From 9f611df3f188082ac5b9b3d87d2cc866c56a00ff Mon Sep 17 00:00:00 2001 From: trav90 Date: Mon, 15 Oct 2018 21:45:30 -0500 Subject: Import aom library This is the reference implementation for the Alliance for Open Media's av1 video code. The commit used was 4d668d7feb1f8abd809d1bca0418570a7f142a36. --- third_party/aom/.clang-format | 94 + third_party/aom/.mailmap | 32 + third_party/aom/AUTHORS | 144 + third_party/aom/CHANGELOG | 630 + third_party/aom/CMakeLists.txt | 359 + third_party/aom/LICENSE | 27 + third_party/aom/PATENTS | 108 + third_party/aom/README | 168 + third_party/aom/aom/aom.h | 160 + third_party/aom/aom/aom_codec.h | 487 + third_party/aom/aom/aom_codec.mk | 42 + third_party/aom/aom/aom_decoder.h | 366 + third_party/aom/aom/aom_encoder.h | 832 ++ third_party/aom/aom/aom_frame_buffer.h | 84 + third_party/aom/aom/aom_image.h | 225 + third_party/aom/aom/aom_integer.h | 64 + third_party/aom/aom/aomcx.h | 750 ++ third_party/aom/aom/aomdx.h | 219 + third_party/aom/aom/exports_com | 16 + third_party/aom/aom/exports_dec | 8 + third_party/aom/aom/exports_enc | 9 + third_party/aom/aom/internal/aom_codec_internal.h | 467 + third_party/aom/aom/src/aom_codec.c | 138 + third_party/aom/aom/src/aom_decoder.c | 189 + third_party/aom/aom/src/aom_encoder.c | 405 + third_party/aom/aom/src/aom_image.c | 240 + third_party/aom/aom_dsp/add_noise.c | 73 + third_party/aom/aom_dsp/ans.h | 44 + third_party/aom/aom_dsp/ansreader.h | 214 + third_party/aom/aom_dsp/answriter.h | 148 + third_party/aom/aom_dsp/aom_convolve.c | 854 ++ third_party/aom/aom_dsp/aom_convolve.h | 57 + third_party/aom/aom_dsp/aom_dsp.cmake | 509 + third_party/aom/aom_dsp/aom_dsp.mk | 428 + third_party/aom/aom_dsp/aom_dsp_common.h | 107 + third_party/aom/aom_dsp/aom_dsp_rtcd.c | 16 + third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl | 1495 +++ third_party/aom/aom_dsp/aom_filter.h | 43 + third_party/aom/aom_dsp/aom_simd.h | 37 + third_party/aom/aom_dsp/aom_simd_inline.h | 21 + .../aom/aom_dsp/arm/aom_convolve8_avg_neon.c | 364 + .../aom/aom_dsp/arm/aom_convolve8_avg_neon_asm.asm | 295 + third_party/aom/aom_dsp/arm/aom_convolve8_neon.c | 331 + .../aom/aom_dsp/arm/aom_convolve8_neon_asm.asm | 273 + .../aom/aom_dsp/arm/aom_convolve_avg_neon.c | 145 + .../aom/aom_dsp/arm/aom_convolve_avg_neon_asm.asm | 119 + .../aom/aom_dsp/arm/aom_convolve_copy_neon.c | 93 + .../aom/aom_dsp/arm/aom_convolve_copy_neon_asm.asm | 87 + third_party/aom/aom_dsp/arm/aom_convolve_neon.c | 66 + third_party/aom/aom_dsp/arm/avg_neon.c | 254 + .../aom/aom_dsp/arm/bilinear_filter_media.asm | 240 + third_party/aom/aom_dsp/arm/fwd_txfm_neon.c | 221 + third_party/aom/aom_dsp/arm/hadamard_neon.c | 200 + .../aom/aom_dsp/arm/idct16x16_1_add_neon.asm | 201 + third_party/aom/aom_dsp/arm/idct16x16_1_add_neon.c | 59 + third_party/aom/aom_dsp/arm/idct16x16_add_neon.asm | 1182 ++ third_party/aom/aom_dsp/arm/idct16x16_add_neon.c | 1295 ++ third_party/aom/aom_dsp/arm/idct16x16_neon.c | 152 + .../aom/aom_dsp/arm/idct32x32_1_add_neon.asm | 147 + third_party/aom/aom_dsp/arm/idct32x32_1_add_neon.c | 141 + third_party/aom/aom_dsp/arm/idct32x32_add_neon.asm | 1302 ++ third_party/aom/aom_dsp/arm/idct32x32_add_neon.c | 686 + third_party/aom/aom_dsp/arm/idct4x4_1_add_neon.asm | 71 + third_party/aom/aom_dsp/arm/idct4x4_1_add_neon.c | 47 + third_party/aom/aom_dsp/arm/idct4x4_add_neon.asm | 193 + third_party/aom/aom_dsp/arm/idct4x4_add_neon.c | 146 + third_party/aom/aom_dsp/arm/idct8x8_1_add_neon.asm | 91 + third_party/aom/aom_dsp/arm/idct8x8_1_add_neon.c | 62 + third_party/aom/aom_dsp/arm/idct8x8_add_neon.asm | 522 + third_party/aom/aom_dsp/arm/idct8x8_add_neon.c | 509 + third_party/aom/aom_dsp/arm/intrapred_neon.c | 757 ++ third_party/aom/aom_dsp/arm/intrapred_neon_asm.asm | 633 + third_party/aom/aom_dsp/arm/loopfilter_16_neon.asm | 202 + third_party/aom/aom_dsp/arm/loopfilter_16_neon.c | 174 + third_party/aom/aom_dsp/arm/loopfilter_4_neon.asm | 252 + third_party/aom/aom_dsp/arm/loopfilter_4_neon.c | 250 + third_party/aom/aom_dsp/arm/loopfilter_8_neon.asm | 428 + third_party/aom/aom_dsp/arm/loopfilter_8_neon.c | 430 + third_party/aom/aom_dsp/arm/loopfilter_mb_neon.asm | 638 + third_party/aom/aom_dsp/arm/loopfilter_neon.c | 49 + third_party/aom/aom_dsp/arm/sad4d_neon.c | 225 + third_party/aom/aom_dsp/arm/sad_media.asm | 98 + third_party/aom/aom_dsp/arm/sad_neon.c | 224 + third_party/aom/aom_dsp/arm/save_reg_neon.asm | 39 + .../aom/aom_dsp/arm/subpel_variance_media.c | 81 + third_party/aom/aom_dsp/arm/subpel_variance_neon.c | 134 + third_party/aom/aom_dsp/arm/subtract_neon.c | 80 + .../arm/variance_halfpixvar16x16_h_media.asm | 185 + .../arm/variance_halfpixvar16x16_hv_media.asm | 225 + .../arm/variance_halfpixvar16x16_v_media.asm | 187 + third_party/aom/aom_dsp/arm/variance_media.asm | 361 + third_party/aom/aom_dsp/arm/variance_neon.c | 400 + third_party/aom/aom_dsp/avg.c | 232 + third_party/aom/aom_dsp/binary_codes_reader.c | 117 + third_party/aom/aom_dsp/binary_codes_reader.h | 38 + third_party/aom/aom_dsp/binary_codes_writer.c | 211 + third_party/aom/aom_dsp/binary_codes_writer.h | 70 + third_party/aom/aom_dsp/bitreader.h | 276 + third_party/aom/aom_dsp/bitreader_buffer.c | 47 + third_party/aom/aom_dsp/bitreader_buffer.h | 48 + third_party/aom/aom_dsp/bitwriter.h | 255 + third_party/aom/aom_dsp/bitwriter_buffer.c | 61 + third_party/aom/aom_dsp/bitwriter_buffer.h | 44 + third_party/aom/aom_dsp/blend.h | 42 + third_party/aom/aom_dsp/blend_a64_hmask.c | 71 + third_party/aom/aom_dsp/blend_a64_mask.c | 145 + third_party/aom/aom_dsp/blend_a64_vmask.c | 73 + third_party/aom/aom_dsp/buf_ans.c | 71 + third_party/aom/aom_dsp/buf_ans.h | 133 + third_party/aom/aom_dsp/daalaboolreader.c | 37 + third_party/aom/aom_dsp/daalaboolreader.h | 164 + third_party/aom/aom_dsp/daalaboolwriter.c | 32 + third_party/aom/aom_dsp/daalaboolwriter.h | 87 + third_party/aom/aom_dsp/dkboolreader.c | 110 + third_party/aom/aom_dsp/dkboolreader.h | 181 + third_party/aom/aom_dsp/dkboolwriter.c | 44 + third_party/aom/aom_dsp/dkboolwriter.h | 104 + third_party/aom/aom_dsp/entcode.c | 53 + third_party/aom/aom_dsp/entcode.h | 46 + third_party/aom/aom_dsp/entdec.c | 300 + third_party/aom/aom_dsp/entdec.h | 91 + third_party/aom/aom_dsp/entenc.c | 507 + third_party/aom/aom_dsp/entenc.h | 91 + third_party/aom/aom_dsp/fastssim.c | 493 + third_party/aom/aom_dsp/fwd_txfm.c | 809 ++ third_party/aom/aom_dsp/fwd_txfm.h | 29 + third_party/aom/aom_dsp/intrapred.c | 971 ++ third_party/aom/aom_dsp/inv_txfm.c | 1445 +++ third_party/aom/aom_dsp/inv_txfm.h | 91 + third_party/aom/aom_dsp/loopfilter.c | 900 ++ third_party/aom/aom_dsp/mips/add_noise_msa.c | 60 + .../aom/aom_dsp/mips/aom_convolve8_avg_horiz_msa.c | 704 + .../aom/aom_dsp/mips/aom_convolve8_avg_msa.c | 605 + .../aom/aom_dsp/mips/aom_convolve8_avg_vert_msa.c | 677 + .../aom/aom_dsp/mips/aom_convolve8_horiz_msa.c | 692 + third_party/aom/aom_dsp/mips/aom_convolve8_msa.c | 630 + .../aom/aom_dsp/mips/aom_convolve8_vert_msa.c | 699 + .../aom/aom_dsp/mips/aom_convolve_avg_msa.c | 233 + .../aom/aom_dsp/mips/aom_convolve_copy_msa.c | 248 + third_party/aom/aom_dsp/mips/aom_convolve_msa.h | 124 + third_party/aom/aom_dsp/mips/avg_msa.c | 57 + third_party/aom/aom_dsp/mips/common_dspr2.c | 31 + third_party/aom/aom_dsp/mips/common_dspr2.h | 49 + third_party/aom/aom_dsp/mips/convolve2_avg_dspr2.c | 256 + .../aom/aom_dsp/mips/convolve2_avg_horiz_dspr2.c | 802 ++ third_party/aom/aom_dsp/mips/convolve2_dspr2.c | 1030 ++ .../aom/aom_dsp/mips/convolve2_horiz_dspr2.c | 681 + .../aom/aom_dsp/mips/convolve2_vert_dspr2.c | 237 + third_party/aom/aom_dsp/mips/convolve8_avg_dspr2.c | 641 + .../aom/aom_dsp/mips/convolve8_avg_horiz_dspr2.c | 998 ++ third_party/aom/aom_dsp/mips/convolve8_dspr2.c | 1590 +++ .../aom/aom_dsp/mips/convolve8_horiz_dspr2.c | 878 ++ .../aom/aom_dsp/mips/convolve8_vert_dspr2.c | 360 + .../aom/aom_dsp/mips/convolve_common_dspr2.h | 59 + third_party/aom/aom_dsp/mips/fwd_dct32x32_msa.c | 948 ++ third_party/aom/aom_dsp/mips/fwd_txfm_msa.c | 246 + third_party/aom/aom_dsp/mips/fwd_txfm_msa.h | 381 + third_party/aom/aom_dsp/mips/idct16x16_msa.c | 486 + third_party/aom/aom_dsp/mips/idct32x32_msa.c | 730 ++ third_party/aom/aom_dsp/mips/idct4x4_msa.c | 99 + third_party/aom/aom_dsp/mips/idct8x8_msa.c | 117 + third_party/aom/aom_dsp/mips/intrapred16_dspr2.c | 325 + third_party/aom/aom_dsp/mips/intrapred4_dspr2.c | 225 + third_party/aom/aom_dsp/mips/intrapred8_dspr2.c | 603 + third_party/aom/aom_dsp/mips/intrapred_msa.c | 739 ++ third_party/aom/aom_dsp/mips/inv_txfm_dspr2.h | 80 + third_party/aom/aom_dsp/mips/inv_txfm_msa.h | 412 + third_party/aom/aom_dsp/mips/itrans16_dspr2.c | 1190 ++ third_party/aom/aom_dsp/mips/itrans32_cols_dspr2.c | 1042 ++ third_party/aom/aom_dsp/mips/itrans32_dspr2.c | 1030 ++ third_party/aom/aom_dsp/mips/itrans4_dspr2.c | 342 + third_party/aom/aom_dsp/mips/itrans8_dspr2.c | 645 + third_party/aom/aom_dsp/mips/loopfilter_16_msa.c | 1487 +++ third_party/aom/aom_dsp/mips/loopfilter_4_msa.c | 147 + third_party/aom/aom_dsp/mips/loopfilter_8_msa.c | 333 + .../aom/aom_dsp/mips/loopfilter_filters_dspr2.c | 327 + .../aom/aom_dsp/mips/loopfilter_filters_dspr2.h | 735 ++ .../aom/aom_dsp/mips/loopfilter_macros_dspr2.h | 436 + .../aom/aom_dsp/mips/loopfilter_masks_dspr2.h | 356 + third_party/aom/aom_dsp/mips/loopfilter_mb_dspr2.c | 589 + .../aom/aom_dsp/mips/loopfilter_mb_horiz_dspr2.c | 734 ++ .../aom/aom_dsp/mips/loopfilter_mb_vert_dspr2.c | 757 ++ third_party/aom/aom_dsp/mips/loopfilter_msa.h | 251 + third_party/aom/aom_dsp/mips/macros_msa.h | 2057 +++ third_party/aom/aom_dsp/mips/sad_msa.c | 1529 +++ .../aom/aom_dsp/mips/sub_pixel_variance_msa.c | 1795 +++ third_party/aom/aom_dsp/mips/subtract_msa.c | 265 + third_party/aom/aom_dsp/mips/txfm_macros_msa.h | 97 + third_party/aom/aom_dsp/mips/variance_msa.c | 632 + third_party/aom/aom_dsp/postproc.h | 26 + third_party/aom/aom_dsp/prob.c | 236 + third_party/aom/aom_dsp/prob.h | 198 + third_party/aom/aom_dsp/psnr.c | 373 + third_party/aom/aom_dsp/psnr.h | 79 + third_party/aom/aom_dsp/psnrhvs.c | 276 + third_party/aom/aom_dsp/quantize.c | 832 ++ third_party/aom/aom_dsp/quantize.h | 120 + third_party/aom/aom_dsp/sad.c | 512 + third_party/aom/aom_dsp/simd/v128_intrinsics.h | 268 + third_party/aom/aom_dsp/simd/v128_intrinsics_arm.h | 671 + third_party/aom/aom_dsp/simd/v128_intrinsics_c.h | 707 ++ third_party/aom/aom_dsp/simd/v128_intrinsics_x86.h | 511 + third_party/aom/aom_dsp/simd/v256_intrinsics.h | 283 + third_party/aom/aom_dsp/simd/v256_intrinsics_arm.h | 17 + third_party/aom/aom_dsp/simd/v256_intrinsics_c.h | 724 ++ .../aom/aom_dsp/simd/v256_intrinsics_v128.h | 545 + third_party/aom/aom_dsp/simd/v256_intrinsics_x86.h | 548 + third_party/aom/aom_dsp/simd/v64_intrinsics.h | 223 + third_party/aom/aom_dsp/simd/v64_intrinsics_arm.h | 583 + third_party/aom/aom_dsp/simd/v64_intrinsics_c.h | 919 ++ third_party/aom/aom_dsp/simd/v64_intrinsics_x86.h | 470 + third_party/aom/aom_dsp/ssim.c | 462 + third_party/aom/aom_dsp/ssim.h | 88 + third_party/aom/aom_dsp/subtract.c | 55 + third_party/aom/aom_dsp/sum_squares.c | 40 + third_party/aom/aom_dsp/txfm_common.h | 70 + third_party/aom/aom_dsp/variance.c | 1249 ++ third_party/aom/aom_dsp/variance.h | 132 + third_party/aom/aom_dsp/x86/aom_asm_stubs.c | 182 + .../aom/aom_dsp/x86/aom_convolve_copy_sse2.asm | 345 + .../aom/aom_dsp/x86/aom_high_subpixel_8t_sse2.asm | 965 ++ .../x86/aom_high_subpixel_bilinear_sse2.asm | 497 + .../aom/aom_dsp/x86/aom_subpixel_8t_intrin_avx2.c | 575 + .../aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c | 920 ++ .../aom/aom_dsp/x86/aom_subpixel_8t_sse2.asm | 990 ++ .../aom/aom_dsp/x86/aom_subpixel_8t_ssse3.asm | 883 ++ .../aom/aom_dsp/x86/aom_subpixel_bilinear_sse2.asm | 451 + .../aom_dsp/x86/aom_subpixel_bilinear_ssse3.asm | 421 + third_party/aom/aom_dsp/x86/avg_intrin_sse2.c | 426 + third_party/aom/aom_dsp/x86/avg_ssse3_x86_64.asm | 124 + third_party/aom/aom_dsp/x86/blend_a64_hmask_sse4.c | 36 + third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c | 924 ++ third_party/aom/aom_dsp/x86/blend_a64_vmask_sse4.c | 285 + third_party/aom/aom_dsp/x86/blend_sse4.h | 146 + third_party/aom/aom_dsp/x86/convolve.h | 288 + third_party/aom/aom_dsp/x86/fwd_dct32_8cols_sse2.c | 862 ++ .../aom/aom_dsp/x86/fwd_dct32x32_impl_avx2.h | 3022 +++++ .../aom/aom_dsp/x86/fwd_dct32x32_impl_sse2.h | 3201 +++++ third_party/aom/aom_dsp/x86/fwd_txfm_avx2.c | 24 + third_party/aom/aom_dsp/x86/fwd_txfm_avx2.h | 35 + third_party/aom/aom_dsp/x86/fwd_txfm_impl_sse2.h | 1014 ++ third_party/aom/aom_dsp/x86/fwd_txfm_sse2.c | 273 + third_party/aom/aom_dsp/x86/fwd_txfm_sse2.h | 362 + .../aom/aom_dsp/x86/fwd_txfm_ssse3_x86_64.asm | 204 + .../aom/aom_dsp/x86/halfpix_variance_impl_sse2.asm | 349 + .../aom/aom_dsp/x86/halfpix_variance_sse2.c | 77 + third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c | 1151 ++ .../aom/aom_dsp/x86/highbd_intrapred_sse2.asm | 456 + .../aom/aom_dsp/x86/highbd_loopfilter_sse2.c | 1140 ++ .../aom/aom_dsp/x86/highbd_quantize_intrin_sse2.c | 155 + third_party/aom/aom_dsp/x86/highbd_sad4d_sse2.asm | 290 + third_party/aom/aom_dsp/x86/highbd_sad_sse2.asm | 366 + .../x86/highbd_subpel_variance_impl_sse2.asm | 1040 ++ third_party/aom/aom_dsp/x86/highbd_subtract_sse2.c | 364 + .../aom/aom_dsp/x86/highbd_variance_impl_sse2.asm | 316 + third_party/aom/aom_dsp/x86/highbd_variance_sse2.c | 695 + third_party/aom/aom_dsp/x86/highbd_variance_sse4.c | 216 + third_party/aom/aom_dsp/x86/intrapred_sse2.asm | 771 ++ third_party/aom/aom_dsp/x86/intrapred_ssse3.asm | 410 + third_party/aom/aom_dsp/x86/inv_txfm_sse2.c | 3631 ++++++ third_party/aom/aom_dsp/x86/inv_txfm_sse2.h | 265 + third_party/aom/aom_dsp/x86/inv_txfm_ssse3.c | 1333 ++ third_party/aom/aom_dsp/x86/inv_wht_sse2.asm | 112 + third_party/aom/aom_dsp/x86/loopfilter_avx2.c | 915 ++ third_party/aom/aom_dsp/x86/loopfilter_sse2.c | 1892 +++ .../aom/aom_dsp/x86/masked_sad_intrin_ssse3.c | 334 + .../aom/aom_dsp/x86/masked_variance_intrin_ssse3.c | 1948 +++ third_party/aom/aom_dsp/x86/obmc_sad_sse4.c | 262 + third_party/aom/aom_dsp/x86/obmc_variance_sse4.c | 355 + .../aom/aom_dsp/x86/quantize_avx_x86_64.asm | 547 + third_party/aom/aom_dsp/x86/quantize_sse2.c | 249 + .../aom/aom_dsp/x86/quantize_ssse3_x86_64.asm | 349 + third_party/aom/aom_dsp/x86/sad4d_avx2.c | 216 + third_party/aom/aom_dsp/x86/sad4d_sse2.asm | 253 + third_party/aom/aom_dsp/x86/sad_avx2.c | 187 + third_party/aom/aom_dsp/x86/sad_highbd_avx2.c | 1043 ++ third_party/aom/aom_dsp/x86/sad_impl_avx2.c | 233 + third_party/aom/aom_dsp/x86/sad_sse2.asm | 345 + third_party/aom/aom_dsp/x86/sad_sse3.asm | 377 + third_party/aom/aom_dsp/x86/sad_sse4.asm | 362 + third_party/aom/aom_dsp/x86/sad_ssse3.asm | 373 + third_party/aom/aom_dsp/x86/ssim_opt_x86_64.asm | 219 + .../aom/aom_dsp/x86/subpel_variance_sse2.asm | 1489 +++ third_party/aom/aom_dsp/x86/subtract_sse2.asm | 150 + third_party/aom/aom_dsp/x86/sum_squares_sse2.c | 210 + third_party/aom/aom_dsp/x86/synonyms.h | 120 + third_party/aom/aom_dsp/x86/txfm_common_avx2.h | 204 + third_party/aom/aom_dsp/x86/txfm_common_intrin.h | 31 + third_party/aom/aom_dsp/x86/txfm_common_sse2.h | 326 + third_party/aom/aom_dsp/x86/variance_avx2.c | 192 + third_party/aom/aom_dsp/x86/variance_impl_avx2.c | 713 ++ third_party/aom/aom_dsp/x86/variance_sse2.c | 690 + third_party/aom/aom_mem/aom_mem.c | 74 + third_party/aom/aom_mem/aom_mem.cmake | 22 + third_party/aom/aom_mem/aom_mem.h | 46 + third_party/aom/aom_mem/aom_mem.mk | 4 + third_party/aom/aom_mem/include/aom_mem_intrnl.h | 32 + third_party/aom/aom_ports/aom_once.h | 141 + third_party/aom/aom_ports/aom_ports.cmake | 66 + third_party/aom/aom_ports/aom_ports.mk | 29 + third_party/aom/aom_ports/aom_timer.h | 111 + third_party/aom/aom_ports/arm.h | 40 + third_party/aom/aom_ports/arm_cpudetect.c | 176 + third_party/aom/aom_ports/bitops.h | 76 + third_party/aom/aom_ports/config.h | 17 + third_party/aom/aom_ports/emmintrin_compat.h | 56 + third_party/aom/aom_ports/emms.asm | 41 + third_party/aom/aom_ports/mem.h | 65 + third_party/aom/aom_ports/mem_ops.h | 229 + third_party/aom/aom_ports/mem_ops_aligned.h | 172 + third_party/aom/aom_ports/msvc.h | 47 + third_party/aom/aom_ports/system_state.h | 23 + third_party/aom/aom_ports/x86.h | 322 + third_party/aom/aom_ports/x86_abi_support.asm | 395 + third_party/aom/aom_scale/aom_scale.cmake | 34 + third_party/aom/aom_scale/aom_scale.h | 23 + third_party/aom/aom_scale/aom_scale.mk | 16 + third_party/aom/aom_scale/aom_scale_rtcd.c | 16 + third_party/aom/aom_scale/aom_scale_rtcd.pl | 38 + third_party/aom/aom_scale/generic/aom_scale.c | 529 + third_party/aom/aom_scale/generic/gen_scalers.c | 200 + third_party/aom/aom_scale/generic/yv12config.c | 199 + third_party/aom/aom_scale/generic/yv12extend.c | 382 + .../aom/aom_scale/mips/dspr2/yv12extend_dspr2.c | 139 + third_party/aom/aom_scale/yv12config.h | 110 + third_party/aom/aom_util/aom_thread.c | 183 + third_party/aom/aom_util/aom_thread.h | 412 + third_party/aom/aom_util/aom_util.cmake | 29 + third_party/aom/aom_util/aom_util.mk | 18 + third_party/aom/aom_util/debug_util.c | 98 + third_party/aom/aom_util/debug_util.h | 54 + third_party/aom/aom_util/endian_inl.h | 120 + third_party/aom/aomdec.c | 1070 ++ third_party/aom/aomenc.c | 2144 ++++ third_party/aom/aomenc.h | 65 + third_party/aom/aomstats.c | 106 + third_party/aom/aomstats.h | 44 + third_party/aom/args.c | 212 + third_party/aom/args.h | 64 + third_party/aom/av1/av1.cmake | 518 + third_party/aom/av1/av1_common.mk | 180 + third_party/aom/av1/av1_cx.mk | 165 + third_party/aom/av1/av1_cx_iface.c | 1605 +++ third_party/aom/av1/av1_dx.mk | 71 + third_party/aom/av1/av1_dx_iface.c | 1223 ++ third_party/aom/av1/av1_iface_common.h | 146 + third_party/aom/av1/common/alloccommon.c | 209 + third_party/aom/av1/common/alloccommon.h | 47 + .../aom/av1/common/arm/neon/iht4x4_add_neon.c | 227 + .../aom/av1/common/arm/neon/iht8x8_add_neon.c | 593 + third_party/aom/av1/common/av1_fwd_txfm1d.c | 2312 ++++ third_party/aom/av1/common/av1_fwd_txfm1d.h | 45 + third_party/aom/av1/common/av1_fwd_txfm2d.c | 230 + third_party/aom/av1/common/av1_fwd_txfm2d_cfg.h | 444 + third_party/aom/av1/common/av1_inv_txfm1d.c | 2334 ++++ third_party/aom/av1/common/av1_inv_txfm1d.h | 45 + third_party/aom/av1/common/av1_inv_txfm2d.c | 256 + third_party/aom/av1/common/av1_inv_txfm2d_cfg.h | 447 + third_party/aom/av1/common/av1_loopfilter.c | 2336 ++++ third_party/aom/av1/common/av1_loopfilter.h | 162 + third_party/aom/av1/common/av1_rtcd.c | 20 + third_party/aom/av1/common/av1_rtcd_defs.pl | 644 + third_party/aom/av1/common/av1_txfm.h | 211 + third_party/aom/av1/common/blockd.c | 296 + third_party/aom/av1/common/blockd.h | 1371 ++ third_party/aom/av1/common/cdef.c | 445 + third_party/aom/av1/common/cdef.h | 53 + third_party/aom/av1/common/cdef_simd.h | 27 + third_party/aom/av1/common/cfl.c | 240 + third_party/aom/av1/common/cfl.h | 52 + third_party/aom/av1/common/clpf.c | 116 + third_party/aom/av1/common/clpf.h | 18 + third_party/aom/av1/common/clpf_neon.c | 14 + third_party/aom/av1/common/clpf_simd.h | 446 + third_party/aom/av1/common/clpf_sse2.c | 14 + third_party/aom/av1/common/clpf_sse4.c | 14 + third_party/aom/av1/common/clpf_ssse3.c | 14 + third_party/aom/av1/common/common.h | 64 + third_party/aom/av1/common/common_data.h | 1405 ++ third_party/aom/av1/common/convolve.c | 775 ++ third_party/aom/av1/common/convolve.h | 119 + third_party/aom/av1/common/debugmodes.c | 90 + third_party/aom/av1/common/entropy.c | 6438 ++++++++++ third_party/aom/av1/common/entropy.h | 428 + third_party/aom/av1/common/entropymode.c | 3792 ++++++ third_party/aom/av1/common/entropymode.h | 575 + third_party/aom/av1/common/entropymv.c | 315 + third_party/aom/av1/common/entropymv.h | 149 + third_party/aom/av1/common/enums.h | 543 + third_party/aom/av1/common/filter.c | 360 + third_party/aom/av1/common/filter.h | 111 + third_party/aom/av1/common/frame_buffers.c | 79 + third_party/aom/av1/common/frame_buffers.h | 54 + third_party/aom/av1/common/generic_code.c | 114 + third_party/aom/av1/common/generic_code.h | 81 + third_party/aom/av1/common/idct.c | 3067 +++++ third_party/aom/av1/common/idct.h | 99 + third_party/aom/av1/common/laplace_tables.c | 657 + .../aom/av1/common/mips/dspr2/av1_itrans16_dspr2.c | 97 + .../aom/av1/common/mips/dspr2/av1_itrans4_dspr2.c | 91 + .../aom/av1/common/mips/dspr2/av1_itrans8_dspr2.c | 85 + .../aom/av1/common/mips/msa/av1_idct16x16_msa.c | 80 + .../aom/av1/common/mips/msa/av1_idct4x4_msa.c | 61 + .../aom/av1/common/mips/msa/av1_idct8x8_msa.c | 79 + third_party/aom/av1/common/mv.h | 302 + third_party/aom/av1/common/mvref_common.c | 1164 ++ third_party/aom/av1/common/mvref_common.h | 580 + third_party/aom/av1/common/od_dering.c | 416 + third_party/aom/av1/common/od_dering.h | 54 + third_party/aom/av1/common/od_dering_neon.c | 14 + third_party/aom/av1/common/od_dering_simd.h | 390 + third_party/aom/av1/common/od_dering_sse2.c | 14 + third_party/aom/av1/common/od_dering_sse4.c | 14 + third_party/aom/av1/common/od_dering_ssse3.c | 14 + third_party/aom/av1/common/odintrin.c | 551 + third_party/aom/av1/common/odintrin.h | 267 + third_party/aom/av1/common/onyxc_int.h | 1027 ++ third_party/aom/av1/common/partition.c | 256 + third_party/aom/av1/common/partition.h | 40 + third_party/aom/av1/common/pred_common.c | 1408 ++ third_party/aom/av1/common/pred_common.h | 266 + third_party/aom/av1/common/pvq.c | 1007 ++ third_party/aom/av1/common/pvq.h | 183 + third_party/aom/av1/common/pvq_state.c | 50 + third_party/aom/av1/common/pvq_state.h | 52 + third_party/aom/av1/common/quant_common.c | 11369 +++++++++++++++++ third_party/aom/av1/common/quant_common.h | 111 + third_party/aom/av1/common/reconinter.c | 3083 +++++ third_party/aom/av1/common/reconinter.h | 828 ++ third_party/aom/av1/common/reconintra.c | 2467 ++++ third_party/aom/av1/common/reconintra.h | 67 + third_party/aom/av1/common/resize.c | 821 ++ third_party/aom/av1/common/resize.h | 69 + third_party/aom/av1/common/restoration.c | 1401 ++ third_party/aom/av1/common/restoration.h | 257 + third_party/aom/av1/common/scale.c | 164 + third_party/aom/av1/common/scale.h | 72 + third_party/aom/av1/common/scan.c | 6853 ++++++++++ third_party/aom/av1/common/scan.h | 95 + third_party/aom/av1/common/seg_common.c | 62 + third_party/aom/av1/common/seg_common.h | 86 + third_party/aom/av1/common/thread_common.c | 529 + third_party/aom/av1/common/thread_common.h | 64 + third_party/aom/av1/common/tile_common.c | 125 + third_party/aom/av1/common/tile_common.h | 59 + third_party/aom/av1/common/txb_common.c | 149 + third_party/aom/av1/common/txb_common.h | 304 + third_party/aom/av1/common/warped_motion.c | 1773 +++ third_party/aom/av1/common/warped_motion.h | 97 + .../aom/av1/common/x86/av1_convolve_ssse3.c | 1029 ++ .../aom/av1/common/x86/av1_fwd_txfm1d_sse4.c | 839 ++ .../aom/av1/common/x86/av1_fwd_txfm2d_sse4.c | 81 + .../aom/av1/common/x86/av1_highbd_convolve_sse4.c | 533 + third_party/aom/av1/common/x86/av1_txfm1d_sse4.h | 144 + third_party/aom/av1/common/x86/filterintra_sse4.c | 898 ++ .../aom/av1/common/x86/highbd_inv_txfm_avx2.c | 557 + .../aom/av1/common/x86/highbd_inv_txfm_sse4.c | 1398 ++ .../aom/av1/common/x86/highbd_txfm_utility_sse4.h | 92 + .../aom/av1/common/x86/highbd_warp_plane_ssse3.c | 286 + .../aom/av1/common/x86/hybrid_inv_txfm_avx2.c | 507 + third_party/aom/av1/common/x86/idct_intrin_sse2.c | 1402 ++ third_party/aom/av1/common/x86/pvq_sse4.c | 252 + third_party/aom/av1/common/x86/pvq_sse4.h | 13 + third_party/aom/av1/common/x86/selfguided_sse4.c | 1805 +++ third_party/aom/av1/common/x86/warp_plane_sse2.c | 297 + third_party/aom/av1/common/zigzag.h | 33 + third_party/aom/av1/common/zigzag16.c | 157 + third_party/aom/av1/common/zigzag32.c | 199 + third_party/aom/av1/common/zigzag4.c | 22 + third_party/aom/av1/common/zigzag8.c | 50 + third_party/aom/av1/decoder/accounting.c | 138 + third_party/aom/av1/decoder/accounting.h | 83 + third_party/aom/av1/decoder/decint.h | 35 + third_party/aom/av1/decoder/decodeframe.c | 5159 ++++++++ third_party/aom/av1/decoder/decodeframe.h | 39 + third_party/aom/av1/decoder/decodemv.c | 2405 ++++ third_party/aom/av1/decoder/decodemv.h | 44 + third_party/aom/av1/decoder/decoder.c | 583 + third_party/aom/av1/decoder/decoder.h | 224 + third_party/aom/av1/decoder/decodetxb.c | 286 + third_party/aom/av1/decoder/decodetxb.h | 31 + third_party/aom/av1/decoder/detokenize.c | 467 + third_party/aom/av1/decoder/detokenize.h | 38 + third_party/aom/av1/decoder/dsubexp.c | 82 + third_party/aom/av1/decoder/dsubexp.h | 32 + third_party/aom/av1/decoder/dthread.c | 194 + third_party/aom/av1/decoder/dthread.h | 75 + third_party/aom/av1/decoder/generic_decoder.c | 110 + third_party/aom/av1/decoder/inspection.c | 103 + third_party/aom/av1/decoder/inspection.h | 82 + third_party/aom/av1/decoder/laplace_decoder.c | 121 + third_party/aom/av1/decoder/pvq_decoder.c | 378 + third_party/aom/av1/decoder/pvq_decoder.h | 40 + third_party/aom/av1/encoder/aq_complexity.c | 163 + third_party/aom/av1/encoder/aq_complexity.h | 37 + third_party/aom/av1/encoder/aq_cyclicrefresh.c | 566 + third_party/aom/av1/encoder/aq_cyclicrefresh.h | 98 + third_party/aom/av1/encoder/aq_variance.c | 207 + third_party/aom/av1/encoder/aq_variance.h | 31 + third_party/aom/av1/encoder/arm/neon/dct_neon.c | 36 + third_party/aom/av1/encoder/arm/neon/error_neon.c | 42 + .../aom/av1/encoder/arm/neon/quantize_neon.c | 118 + third_party/aom/av1/encoder/av1_quantize.c | 1790 +++ third_party/aom/av1/encoder/av1_quantize.h | 184 + third_party/aom/av1/encoder/bitstream.c | 5399 ++++++++ third_party/aom/av1/encoder/bitstream.h | 53 + third_party/aom/av1/encoder/block.h | 241 + third_party/aom/av1/encoder/blockiness.c | 142 + third_party/aom/av1/encoder/context_tree.c | 331 + third_party/aom/av1/encoder/context_tree.h | 111 + third_party/aom/av1/encoder/corner_detect.c | 37 + third_party/aom/av1/encoder/corner_detect.h | 22 + third_party/aom/av1/encoder/corner_match.c | 193 + third_party/aom/av1/encoder/corner_match.h | 29 + third_party/aom/av1/encoder/cost.c | 67 + third_party/aom/av1/encoder/cost.h | 63 + third_party/aom/av1/encoder/daala_compat_enc.c | 30 + third_party/aom/av1/encoder/dct.c | 2228 ++++ third_party/aom/av1/encoder/encint.h | 51 + third_party/aom/av1/encoder/encodeframe.c | 7160 +++++++++++ third_party/aom/av1/encoder/encodeframe.h | 58 + third_party/aom/av1/encoder/encodemb.c | 1671 +++ third_party/aom/av1/encoder/encodemb.h | 92 + third_party/aom/av1/encoder/encodemv.c | 497 + third_party/aom/av1/encoder/encodemv.h | 43 + third_party/aom/av1/encoder/encoder.c | 5980 +++++++++ third_party/aom/av1/encoder/encoder.h | 883 ++ third_party/aom/av1/encoder/encodetxb.c | 784 ++ third_party/aom/av1/encoder/encodetxb.h | 53 + third_party/aom/av1/encoder/ethread.c | 176 + third_party/aom/av1/encoder/ethread.h | 34 + third_party/aom/av1/encoder/extend.c | 192 + third_party/aom/av1/encoder/extend.h | 32 + third_party/aom/av1/encoder/firstpass.c | 3026 +++++ third_party/aom/av1/encoder/firstpass.h | 202 + third_party/aom/av1/encoder/generic_encoder.c | 157 + third_party/aom/av1/encoder/global_motion.c | 319 + third_party/aom/av1/encoder/global_motion.h | 62 + third_party/aom/av1/encoder/hybrid_fwd_txfm.c | 499 + third_party/aom/av1/encoder/hybrid_fwd_txfm.h | 44 + third_party/aom/av1/encoder/laplace_encoder.c | 107 + third_party/aom/av1/encoder/lookahead.c | 225 + third_party/aom/av1/encoder/lookahead.h | 114 + third_party/aom/av1/encoder/mbgraph.c | 398 + third_party/aom/av1/encoder/mbgraph.h | 39 + third_party/aom/av1/encoder/mcomp.c | 3493 +++++ third_party/aom/av1/encoder/mcomp.h | 163 + third_party/aom/av1/encoder/mips/msa/error_msa.c | 108 + .../aom/av1/encoder/mips/msa/fdct16x16_msa.c | 436 + third_party/aom/av1/encoder/mips/msa/fdct4x4_msa.c | 98 + third_party/aom/av1/encoder/mips/msa/fdct8x8_msa.c | 65 + third_party/aom/av1/encoder/mips/msa/fdct_msa.h | 117 + .../aom/av1/encoder/mips/msa/temporal_filter_msa.c | 284 + third_party/aom/av1/encoder/palette.c | 277 + third_party/aom/av1/encoder/palette.h | 73 + third_party/aom/av1/encoder/pickcdef.c | 490 + third_party/aom/av1/encoder/picklpf.c | 211 + third_party/aom/av1/encoder/picklpf.h | 32 + third_party/aom/av1/encoder/pickrst.c | 1269 ++ third_party/aom/av1/encoder/pickrst.h | 30 + third_party/aom/av1/encoder/pvq_encoder.c | 988 ++ third_party/aom/av1/encoder/pvq_encoder.h | 53 + third_party/aom/av1/encoder/ransac.c | 1210 ++ third_party/aom/av1/encoder/ransac.h | 44 + third_party/aom/av1/encoder/ratectrl.c | 1759 +++ third_party/aom/av1/encoder/ratectrl.h | 284 + third_party/aom/av1/encoder/ratectrl_xiph.c | 1244 ++ third_party/aom/av1/encoder/ratectrl_xiph.h | 200 + third_party/aom/av1/encoder/rd.c | 1204 ++ third_party/aom/av1/encoder/rd.h | 505 + third_party/aom/av1/encoder/rdopt.c | 12713 +++++++++++++++++++ third_party/aom/av1/encoder/rdopt.h | 142 + third_party/aom/av1/encoder/segmentation.c | 394 + third_party/aom/av1/encoder/segmentation.h | 51 + third_party/aom/av1/encoder/speed_features.c | 506 + third_party/aom/av1/encoder/speed_features.h | 484 + third_party/aom/av1/encoder/subexp.c | 282 + third_party/aom/av1/encoder/subexp.h | 49 + third_party/aom/av1/encoder/temporal_filter.c | 719 ++ third_party/aom/av1/encoder/temporal_filter.h | 25 + third_party/aom/av1/encoder/tokenize.c | 887 ++ third_party/aom/av1/encoder/tokenize.h | 151 + third_party/aom/av1/encoder/treewriter.c | 59 + third_party/aom/av1/encoder/treewriter.h | 42 + third_party/aom/av1/encoder/variance_tree.c | 61 + third_party/aom/av1/encoder/variance_tree.h | 96 + third_party/aom/av1/encoder/wedge_utils.c | 125 + .../aom/av1/encoder/x86/av1_highbd_quantize_sse4.c | 193 + .../aom/av1/encoder/x86/av1_quantize_sse2.c | 211 + .../av1/encoder/x86/av1_quantize_ssse3_x86_64.asm | 204 + .../aom/av1/encoder/x86/av1_ssim_opt_x86_64.asm | 219 + third_party/aom/av1/encoder/x86/dct_intrin_sse2.c | 3884 ++++++ third_party/aom/av1/encoder/x86/dct_sse2.asm | 87 + third_party/aom/av1/encoder/x86/dct_ssse3.c | 469 + .../aom/av1/encoder/x86/error_intrin_avx2.c | 73 + third_party/aom/av1/encoder/x86/error_sse2.asm | 125 + .../encoder/x86/highbd_block_error_intrin_sse2.c | 72 + .../aom/av1/encoder/x86/highbd_fwd_txfm_sse4.c | 1895 +++ .../aom/av1/encoder/x86/hybrid_fwd_txfm_avx2.c | 1678 +++ .../av1/encoder/x86/temporal_filter_apply_sse2.asm | 215 + third_party/aom/av1/encoder/x86/wedge_utils_sse2.c | 254 + third_party/aom/av1/exports_dec | 2 + third_party/aom/av1/exports_enc | 2 + third_party/aom/build/cmake/aom_config.c.cmake | 15 + .../aom/build/cmake/aom_config_defaults.cmake | 167 + third_party/aom/build/cmake/aom_configure.cmake | 306 + third_party/aom/build/cmake/aom_optimization.cmake | 204 + third_party/aom/build/cmake/aom_version.pl | 93 + third_party/aom/build/cmake/compiler_flags.cmake | 218 + third_party/aom/build/cmake/compiler_tests.cmake | 133 + third_party/aom/build/cmake/cpu.cmake | 72 + .../cmake/generate_aom_config_templates.cmake | 84 + third_party/aom/build/cmake/msvc_runtime.cmake | 26 + third_party/aom/build/cmake/rtcd_config.cmake | 137 + .../build/cmake/toolchains/arm-ios-common.cmake | 31 + .../aom/build/cmake/toolchains/arm64-ios.cmake | 24 + .../build/cmake/toolchains/arm64-linux-gcc.cmake | 35 + .../aom/build/cmake/toolchains/armv7-ios.cmake | 34 + .../build/cmake/toolchains/armv7-linux-gcc.cmake | 48 + .../aom/build/cmake/toolchains/armv7s-ios.cmake | 34 + .../cmake/toolchains/ios-simulator-common.cmake | 23 + .../build/cmake/toolchains/mips32-linux-gcc.cmake | 71 + .../build/cmake/toolchains/mips64-linux-gcc.cmake | 48 + .../build/cmake/toolchains/x86-ios-simulator.cmake | 27 + .../aom/build/cmake/toolchains/x86-linux.cmake | 14 + .../aom/build/cmake/toolchains/x86-macos.cmake | 18 + .../cmake/toolchains/x86_64-ios-simulator.cmake | 24 + third_party/aom/build/make/Android.mk | 201 + third_party/aom/build/make/Makefile | 466 + third_party/aom/build/make/ads2armasm_ms.pl | 39 + third_party/aom/build/make/ads2gas.pl | 236 + third_party/aom/build/make/ads2gas_apple.pl | 235 + third_party/aom/build/make/armlink_adapter.sh | 53 + third_party/aom/build/make/configure.sh | 1556 +++ third_party/aom/build/make/gen_asm_deps.sh | 65 + third_party/aom/build/make/gen_msvs_def.sh | 82 + third_party/aom/build/make/gen_msvs_sln.sh | 254 + third_party/aom/build/make/gen_msvs_vcxproj.sh | 474 + third_party/aom/build/make/ios-Info.plist | 37 + third_party/aom/build/make/iosbuild.sh | 383 + third_party/aom/build/make/msvs_common.sh | 114 + third_party/aom/build/make/rtcd.pl | 424 + third_party/aom/build/make/thumb.pm | 71 + third_party/aom/build/make/version.sh | 77 + third_party/aom/codereview.settings | 5 + third_party/aom/configure | 865 ++ third_party/aom/docs.mk | 50 + third_party/aom/examples.mk | 383 + third_party/aom/examples/analyzer.cc | 695 + third_party/aom/examples/aom_cx_set_ref.c | 323 + third_party/aom/examples/decode_to_md5.c | 133 + third_party/aom/examples/decode_with_drops.c | 149 + third_party/aom/examples/encoder_util.c | 139 + third_party/aom/examples/encoder_util.h | 36 + third_party/aom/examples/inspect.c | 678 + third_party/aom/examples/lossless_encoder.c | 139 + third_party/aom/examples/resize_util.c | 124 + third_party/aom/examples/set_maps.c | 210 + third_party/aom/examples/simple_decoder.c | 150 + third_party/aom/examples/simple_encoder.c | 250 + third_party/aom/examples/twopass_encoder.c | 255 + third_party/aom/ivfdec.c | 102 + third_party/aom/ivfdec.h | 29 + third_party/aom/ivfenc.c | 52 + third_party/aom/ivfenc.h | 34 + third_party/aom/keywords.dox | 51 + third_party/aom/libs.doxy_template | 1295 ++ third_party/aom/libs.mk | 591 + third_party/aom/mainpage.dox | 52 + third_party/aom/md5_utils.c | 249 + third_party/aom/md5_utils.h | 49 + third_party/aom/rate_hist.c | 271 + third_party/aom/rate_hist.h | 41 + third_party/aom/solution.mk | 33 + third_party/aom/test/accounting_test.cc | 78 + third_party/aom/test/acm_random.h | 71 + third_party/aom/test/active_map_refresh_test.cc | 129 + third_party/aom/test/active_map_test.cc | 103 + third_party/aom/test/altref_test.cc | 97 + third_party/aom/test/android/Android.mk | 58 + third_party/aom/test/android/README | 32 + third_party/aom/test/android/get_files.py | 120 + third_party/aom/test/android/scrape_gtest_log.py | 60 + third_party/aom/test/ans_codec_test.cc | 97 + third_party/aom/test/ans_test.cc | 211 + third_party/aom/test/aomcx_set_ref.sh | 58 + third_party/aom/test/aomdec.sh | 124 + third_party/aom/test/aomenc.sh | 241 + third_party/aom/test/aq_segment_test.cc | 129 + third_party/aom/test/arf_freq_test.cc | 232 + third_party/aom/test/av1_convolve_optimz_test.cc | 405 + third_party/aom/test/av1_convolve_test.cc | 522 + third_party/aom/test/av1_dct_test.cc | 108 + third_party/aom/test/av1_ext_tile_test.cc | 200 + third_party/aom/test/av1_fht16x16_test.cc | 275 + third_party/aom/test/av1_fht16x32_test.cc | 144 + third_party/aom/test/av1_fht16x8_test.cc | 129 + third_party/aom/test/av1_fht32x16_test.cc | 144 + third_party/aom/test/av1_fht4x4_test.cc | 222 + third_party/aom/test/av1_fht4x8_test.cc | 129 + third_party/aom/test/av1_fht8x16_test.cc | 128 + third_party/aom/test/av1_fht8x4_test.cc | 128 + third_party/aom/test/av1_fht8x8_test.cc | 220 + third_party/aom/test/av1_fwd_txfm1d_test.cc | 133 + third_party/aom/test/av1_fwd_txfm2d_test.cc | 179 + third_party/aom/test/av1_highbd_iht_test.cc | 236 + third_party/aom/test/av1_inv_txfm1d_test.cc | 87 + third_party/aom/test/av1_inv_txfm2d_test.cc | 158 + third_party/aom/test/av1_inv_txfm_test.cc | 282 + third_party/aom/test/av1_quantize_test.cc | 211 + third_party/aom/test/av1_txfm_test.cc | 164 + third_party/aom/test/av1_txfm_test.h | 100 + third_party/aom/test/av1_wedge_utils_test.cc | 383 + third_party/aom/test/avg_test.cc | 396 + third_party/aom/test/binary_codes_test.cc | 130 + third_party/aom/test/blend_a64_mask_1d_test.cc | 334 + third_party/aom/test/blend_a64_mask_test.cc | 270 + third_party/aom/test/boolcoder_test.cc | 143 + third_party/aom/test/borders_test.cc | 85 + third_party/aom/test/clear_system_state.h | 30 + third_party/aom/test/clpf_test.cc | 437 + third_party/aom/test/codec_factory.h | 166 + third_party/aom/test/convolve_test.cc | 1345 ++ third_party/aom/test/cpu_speed_test.cc | 180 + third_party/aom/test/datarate_test.cc | 253 + third_party/aom/test/dct16x16_test.cc | 876 ++ third_party/aom/test/dct32x32_test.cc | 438 + third_party/aom/test/decode_api_test.cc | 58 + third_party/aom/test/decode_perf_test.cc | 243 + third_party/aom/test/decode_test_driver.cc | 124 + third_party/aom/test/decode_test_driver.h | 164 + third_party/aom/test/decode_to_md5.sh | 67 + third_party/aom/test/decode_with_drops.sh | 67 + third_party/aom/test/dering_test.cc | 388 + third_party/aom/test/divu_small_test.cc | 41 + third_party/aom/test/encode_api_test.cc | 63 + third_party/aom/test/encode_perf_test.cc | 189 + third_party/aom/test/encode_test_driver.cc | 323 + third_party/aom/test/encode_test_driver.h | 247 + .../aom/test/encoder_parms_get_to_decoder.cc | 147 + third_party/aom/test/end_to_end_test.cc | 226 + third_party/aom/test/error_block_test.cc | 173 + third_party/aom/test/error_resilience_test.cc | 235 + third_party/aom/test/ethread_test.cc | 188 + third_party/aom/test/examples.sh | 29 + third_party/aom/test/fdct4x4_test.cc | 344 + third_party/aom/test/fdct8x8_test.cc | 699 + third_party/aom/test/fht32x32_test.cc | 216 + .../aom/test/filterintra_predictors_test.cc | 331 + third_party/aom/test/frame_size_tests.cc | 96 + third_party/aom/test/function_equivalence_test.h | 69 + third_party/aom/test/hadamard_test.cc | 221 + third_party/aom/test/hbd_metrics_test.cc | 237 + third_party/aom/test/i420_video_source.h | 34 + third_party/aom/test/idct8x8_test.cc | 86 + third_party/aom/test/idct_test.cc | 122 + third_party/aom/test/intrabc_test.cc | 157 + third_party/aom/test/intrapred_test.cc | 231 + third_party/aom/test/ivf_video_source.h | 107 + third_party/aom/test/level_test.cc | 117 + third_party/aom/test/lossless_test.cc | 125 + third_party/aom/test/lpf_8_test.cc | 624 + third_party/aom/test/masked_sad_test.cc | 206 + third_party/aom/test/masked_variance_test.cc | 790 ++ third_party/aom/test/md5_helper.h | 76 + third_party/aom/test/minmax_test.cc | 131 + third_party/aom/test/motion_vector_test.cc | 115 + third_party/aom/test/obmc_sad_test.cc | 199 + third_party/aom/test/obmc_variance_test.cc | 297 + third_party/aom/test/partial_idct_test.cc | 485 + third_party/aom/test/quantize_test.cc | 205 + third_party/aom/test/realtime_test.cc | 63 + third_party/aom/test/register_state_check.h | 196 + third_party/aom/test/resize_test.cc | 717 ++ third_party/aom/test/sad_test.cc | 1172 ++ third_party/aom/test/scan_test.cc | 97 + third_party/aom/test/selfguided_filter_test.cc | 274 + third_party/aom/test/set_maps.sh | 52 + third_party/aom/test/simd_cmp_impl.h | 1212 ++ third_party/aom/test/simd_cmp_neon.cc | 17 + third_party/aom/test/simd_cmp_sse2.cc | 18 + third_party/aom/test/simd_cmp_sse4.cc | 18 + third_party/aom/test/simd_cmp_ssse3.cc | 18 + third_party/aom/test/simd_impl.h | 594 + third_party/aom/test/simd_neon_test.cc | 17 + third_party/aom/test/simd_sse2_test.cc | 18 + third_party/aom/test/simd_sse4_test.cc | 18 + third_party/aom/test/simd_ssse3_test.cc | 18 + third_party/aom/test/simple_decoder.sh | 58 + third_party/aom/test/simple_encoder.sh | 53 + third_party/aom/test/subtract_test.cc | 252 + third_party/aom/test/sum_squares_test.cc | 192 + third_party/aom/test/superframe_test.cc | 141 + third_party/aom/test/test-data.mk | 45 + third_party/aom/test/test-data.sha1 | 28 + third_party/aom/test/test.cmake | 315 + third_party/aom/test/test.mk | 241 + third_party/aom/test/test_data_util.cmake | 76 + third_party/aom/test/test_intra_pred_speed.cc | 515 + third_party/aom/test/test_libaom.cc | 67 + third_party/aom/test/test_worker.cmake | 49 + third_party/aom/test/tile_independence_test.cc | 147 + third_party/aom/test/tools_common.sh | 454 + third_party/aom/test/transform_test_base.h | 367 + third_party/aom/test/twopass_encoder.sh | 54 + third_party/aom/test/user_priv_test.cc | 101 + third_party/aom/test/util.h | 46 + third_party/aom/test/variance_test.cc | 1385 ++ third_party/aom/test/video_source.h | 256 + third_party/aom/test/warp_filter_test.cc | 39 + third_party/aom/test/warp_filter_test_util.cc | 268 + third_party/aom/test/warp_filter_test_util.h | 94 + third_party/aom/test/webm_video_source.h | 94 + third_party/aom/test/y4m_test.cc | 192 + third_party/aom/test/y4m_video_source.h | 123 + third_party/aom/test/yuv_video_source.h | 125 + third_party/aom/third_party/fastfeat/LICENSE | 30 + third_party/aom/third_party/fastfeat/README.libvpx | 38 + third_party/aom/third_party/fastfeat/fast.c | 22 + third_party/aom/third_party/fastfeat/fast.h | 20 + third_party/aom/third_party/fastfeat/fast_9.c | 5911 +++++++++ third_party/aom/third_party/fastfeat/nonmax.c | 121 + .../aom/third_party/googletest/README.libaom | 24 + third_party/aom/third_party/googletest/gtest.mk | 1 + .../third_party/googletest/src/googletest/CHANGES | 157 + .../googletest/src/googletest/CMakeLists.txt | 286 + .../googletest/src/googletest/CONTRIBUTORS | 37 + .../third_party/googletest/src/googletest/LICENSE | 28 + .../googletest/src/googletest/README.md | 280 + .../src/googletest/cmake/internal_utils.cmake | 254 + .../googletest/include/gtest/gtest-death-test.h | 294 + .../src/googletest/include/gtest/gtest-message.h | 250 + .../googletest/include/gtest/gtest-param-test.h | 1444 +++ .../include/gtest/gtest-param-test.h.pump | 510 + .../src/googletest/include/gtest/gtest-printers.h | 993 ++ .../src/googletest/include/gtest/gtest-spi.h | 232 + .../src/googletest/include/gtest/gtest-test-part.h | 179 + .../googletest/include/gtest/gtest-typed-test.h | 263 + .../src/googletest/include/gtest/gtest.h | 2236 ++++ .../src/googletest/include/gtest/gtest_pred_impl.h | 358 + .../src/googletest/include/gtest/gtest_prod.h | 58 + .../include/gtest/internal/custom/gtest-port.h | 69 + .../include/gtest/internal/custom/gtest-printers.h | 42 + .../include/gtest/internal/custom/gtest.h | 41 + .../gtest/internal/gtest-death-test-internal.h | 319 + .../include/gtest/internal/gtest-filepath.h | 206 + .../include/gtest/internal/gtest-internal.h | 1238 ++ .../include/gtest/internal/gtest-linked_ptr.h | 243 + .../gtest/internal/gtest-param-util-generated.h | 5146 ++++++++ .../internal/gtest-param-util-generated.h.pump | 286 + .../include/gtest/internal/gtest-param-util.h | 731 ++ .../include/gtest/internal/gtest-port-arch.h | 93 + .../googletest/include/gtest/internal/gtest-port.h | 2554 ++++ .../include/gtest/internal/gtest-string.h | 167 + .../include/gtest/internal/gtest-tuple.h | 1020 ++ .../include/gtest/internal/gtest-tuple.h.pump | 347 + .../include/gtest/internal/gtest-type-util.h | 3331 +++++ .../include/gtest/internal/gtest-type-util.h.pump | 297 + .../googletest/src/googletest/src/gtest-all.cc | 48 + .../src/googletest/src/gtest-death-test.cc | 1342 ++ .../src/googletest/src/gtest-filepath.cc | 387 + .../src/googletest/src/gtest-internal-inl.h | 1183 ++ .../googletest/src/googletest/src/gtest-port.cc | 1259 ++ .../src/googletest/src/gtest-printers.cc | 373 + .../src/googletest/src/gtest-test-part.cc | 110 + .../src/googletest/src/gtest-typed-test.cc | 118 + .../googletest/src/googletest/src/gtest.cc | 5388 ++++++++ .../googletest/src/googletest/src/gtest_main.cc | 38 + third_party/aom/third_party/libwebm/AUTHORS.TXT | 4 + third_party/aom/third_party/libwebm/Android.mk | 17 + third_party/aom/third_party/libwebm/LICENSE.TXT | 30 + third_party/aom/third_party/libwebm/PATENTS.TXT | 23 + third_party/aom/third_party/libwebm/README.libaom | 10 + .../aom/third_party/libwebm/common/file_util.cc | 67 + .../aom/third_party/libwebm/common/file_util.h | 41 + .../aom/third_party/libwebm/common/hdr_util.cc | 182 + .../aom/third_party/libwebm/common/hdr_util.h | 51 + .../aom/third_party/libwebm/common/webmids.h | 184 + .../aom/third_party/libwebm/mkvmuxer/mkvmuxer.cc | 3769 ++++++ .../aom/third_party/libwebm/mkvmuxer/mkvmuxer.h | 1695 +++ .../third_party/libwebm/mkvmuxer/mkvmuxertypes.h | 28 + .../third_party/libwebm/mkvmuxer/mkvmuxerutil.cc | 650 + .../third_party/libwebm/mkvmuxer/mkvmuxerutil.h | 95 + .../aom/third_party/libwebm/mkvmuxer/mkvwriter.cc | 88 + .../aom/third_party/libwebm/mkvmuxer/mkvwriter.h | 51 + .../aom/third_party/libwebm/mkvparser/mkvparser.cc | 7940 ++++++++++++ .../aom/third_party/libwebm/mkvparser/mkvparser.h | 1112 ++ .../aom/third_party/libwebm/mkvparser/mkvreader.cc | 131 + .../aom/third_party/libwebm/mkvparser/mkvreader.h | 45 + third_party/aom/third_party/libyuv/README.libaom | 15 + .../libyuv/include/libyuv/basic_types.h | 119 + .../third_party/libyuv/include/libyuv/compare.h | 79 + .../third_party/libyuv/include/libyuv/convert.h | 246 + .../libyuv/include/libyuv/convert_argb.h | 232 + .../libyuv/include/libyuv/convert_from.h | 182 + .../libyuv/include/libyuv/convert_from_argb.h | 191 + .../aom/third_party/libyuv/include/libyuv/cpu_id.h | 82 + .../libyuv/include/libyuv/mjpeg_decoder.h | 193 + .../libyuv/include/libyuv/planar_functions.h | 454 + .../aom/third_party/libyuv/include/libyuv/rotate.h | 118 + .../libyuv/include/libyuv/rotate_argb.h | 34 + .../third_party/libyuv/include/libyuv/rotate_row.h | 139 + .../aom/third_party/libyuv/include/libyuv/row.h | 1857 +++ .../aom/third_party/libyuv/include/libyuv/scale.h | 104 + .../third_party/libyuv/include/libyuv/scale_argb.h | 58 + .../third_party/libyuv/include/libyuv/scale_row.h | 479 + .../third_party/libyuv/include/libyuv/version.h | 17 + .../libyuv/include/libyuv/video_common.h | 183 + .../aom/third_party/libyuv/source/compare.cc | 373 + .../third_party/libyuv/source/compare_common.cc | 42 + .../aom/third_party/libyuv/source/compare_gcc.cc | 152 + .../aom/third_party/libyuv/source/compare_neon.cc | 65 + .../third_party/libyuv/source/compare_neon64.cc | 63 + .../aom/third_party/libyuv/source/compare_win.cc | 229 + .../aom/third_party/libyuv/source/convert.cc | 1389 ++ .../aom/third_party/libyuv/source/convert_argb.cc | 1155 ++ .../aom/third_party/libyuv/source/convert_from.cc | 1348 ++ .../third_party/libyuv/source/convert_from_argb.cc | 1301 ++ .../aom/third_party/libyuv/source/convert_jpeg.cc | 392 + .../third_party/libyuv/source/convert_to_argb.cc | 306 + .../third_party/libyuv/source/convert_to_i420.cc | 339 + .../aom/third_party/libyuv/source/cpu_id.cc | 307 + .../aom/third_party/libyuv/source/mjpeg_decoder.cc | 572 + .../third_party/libyuv/source/mjpeg_validate.cc | 101 + .../third_party/libyuv/source/planar_functions.cc | 2555 ++++ .../aom/third_party/libyuv/source/rotate.cc | 496 + .../aom/third_party/libyuv/source/rotate_any.cc | 55 + .../aom/third_party/libyuv/source/rotate_argb.cc | 205 + .../aom/third_party/libyuv/source/rotate_common.cc | 92 + .../aom/third_party/libyuv/source/rotate_gcc.cc | 493 + .../aom/third_party/libyuv/source/rotate_mips.cc | 484 + .../aom/third_party/libyuv/source/rotate_neon.cc | 535 + .../aom/third_party/libyuv/source/rotate_neon64.cc | 543 + .../aom/third_party/libyuv/source/rotate_win.cc | 248 + .../aom/third_party/libyuv/source/row_any.cc | 680 + .../aom/third_party/libyuv/source/row_common.cc | 2576 ++++ .../aom/third_party/libyuv/source/row_gcc.cc | 5475 ++++++++ .../aom/third_party/libyuv/source/row_mips.cc | 911 ++ .../aom/third_party/libyuv/source/row_neon.cc | 3084 +++++ .../aom/third_party/libyuv/source/row_neon64.cc | 3087 +++++ .../aom/third_party/libyuv/source/row_win.cc | 6331 +++++++++ .../aom/third_party/libyuv/source/row_x86.asm | 146 + third_party/aom/third_party/libyuv/source/scale.cc | 1689 +++ .../aom/third_party/libyuv/source/scale_any.cc | 200 + .../aom/third_party/libyuv/source/scale_argb.cc | 853 ++ .../aom/third_party/libyuv/source/scale_common.cc | 1137 ++ .../aom/third_party/libyuv/source/scale_gcc.cc | 1089 ++ .../aom/third_party/libyuv/source/scale_mips.cc | 654 + .../aom/third_party/libyuv/source/scale_neon.cc | 1037 ++ .../aom/third_party/libyuv/source/scale_neon64.cc | 1042 ++ .../aom/third_party/libyuv/source/scale_win.cc | 1354 ++ .../aom/third_party/libyuv/source/video_common.cc | 64 + .../aom/third_party/libyuv/source/x86inc.asm | 1136 ++ third_party/aom/third_party/x86inc/LICENSE | 18 + third_party/aom/third_party/x86inc/README.libaom | 20 + third_party/aom/third_party/x86inc/x86inc.asm | 1649 +++ third_party/aom/tools/all_builds.py | 72 + third_party/aom/tools/author_first_release.sh | 15 + third_party/aom/tools/build_inspector.sh | 16 + third_party/aom/tools/cpplint.py | 4756 +++++++ third_party/aom/tools/diff.py | 132 + third_party/aom/tools/ftfy.sh | 158 + third_party/aom/tools/gen_authors.sh | 10 + third_party/aom/tools/gen_constrained_tokenset.py | 120 + third_party/aom/tools/inspect-cli.js | 39 + third_party/aom/tools/inspect-post.js | 1 + third_party/aom/tools/intersect-diffs.py | 78 + third_party/aom/tools/lint-hunks.py | 146 + third_party/aom/tools/wrap-commit-msg.py | 72 + third_party/aom/tools_common.c | 454 + third_party/aom/tools_common.h | 168 + third_party/aom/usage.dox | 135 + third_party/aom/usage_cx.dox | 13 + third_party/aom/usage_dx.dox | 62 + third_party/aom/video_common.h | 24 + third_party/aom/video_reader.c | 78 + third_party/aom/video_reader.h | 52 + third_party/aom/video_writer.c | 77 + third_party/aom/video_writer.h | 45 + third_party/aom/warnings.c | 98 + third_party/aom/warnings.h | 34 + third_party/aom/webmdec.cc | 223 + third_party/aom/webmdec.h | 70 + third_party/aom/webmenc.cc | 96 + third_party/aom/webmenc.h | 56 + third_party/aom/y4menc.c | 65 + third_party/aom/y4menc.h | 34 + third_party/aom/y4minput.c | 1127 ++ third_party/aom/y4minput.h | 69 + 989 files changed, 470949 insertions(+) create mode 100644 third_party/aom/.clang-format create mode 100644 third_party/aom/.mailmap create mode 100644 third_party/aom/AUTHORS create mode 100644 third_party/aom/CHANGELOG create mode 100644 third_party/aom/CMakeLists.txt create mode 100644 third_party/aom/LICENSE create mode 100644 third_party/aom/PATENTS create mode 100644 third_party/aom/README create mode 100644 third_party/aom/aom/aom.h create mode 100644 third_party/aom/aom/aom_codec.h create mode 100644 third_party/aom/aom/aom_codec.mk create mode 100644 third_party/aom/aom/aom_decoder.h create mode 100644 third_party/aom/aom/aom_encoder.h create mode 100644 third_party/aom/aom/aom_frame_buffer.h create mode 100644 third_party/aom/aom/aom_image.h create mode 100644 third_party/aom/aom/aom_integer.h create mode 100644 third_party/aom/aom/aomcx.h create mode 100644 third_party/aom/aom/aomdx.h create mode 100644 third_party/aom/aom/exports_com create mode 100644 third_party/aom/aom/exports_dec create mode 100644 third_party/aom/aom/exports_enc create mode 100644 third_party/aom/aom/internal/aom_codec_internal.h create mode 100644 third_party/aom/aom/src/aom_codec.c create mode 100644 third_party/aom/aom/src/aom_decoder.c create mode 100644 third_party/aom/aom/src/aom_encoder.c create mode 100644 third_party/aom/aom/src/aom_image.c create mode 100644 third_party/aom/aom_dsp/add_noise.c create mode 100644 third_party/aom/aom_dsp/ans.h create mode 100644 third_party/aom/aom_dsp/ansreader.h create mode 100644 third_party/aom/aom_dsp/answriter.h create mode 100644 third_party/aom/aom_dsp/aom_convolve.c create mode 100644 third_party/aom/aom_dsp/aom_convolve.h create mode 100644 third_party/aom/aom_dsp/aom_dsp.cmake create mode 100644 third_party/aom/aom_dsp/aom_dsp.mk create mode 100644 third_party/aom/aom_dsp/aom_dsp_common.h create mode 100644 third_party/aom/aom_dsp/aom_dsp_rtcd.c create mode 100755 third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl create mode 100644 third_party/aom/aom_dsp/aom_filter.h create mode 100644 third_party/aom/aom_dsp/aom_simd.h create mode 100644 third_party/aom/aom_dsp/aom_simd_inline.h create mode 100644 third_party/aom/aom_dsp/arm/aom_convolve8_avg_neon.c create mode 100644 third_party/aom/aom_dsp/arm/aom_convolve8_avg_neon_asm.asm create mode 100644 third_party/aom/aom_dsp/arm/aom_convolve8_neon.c create mode 100644 third_party/aom/aom_dsp/arm/aom_convolve8_neon_asm.asm create mode 100644 third_party/aom/aom_dsp/arm/aom_convolve_avg_neon.c create mode 100644 third_party/aom/aom_dsp/arm/aom_convolve_avg_neon_asm.asm create mode 100644 third_party/aom/aom_dsp/arm/aom_convolve_copy_neon.c create mode 100644 third_party/aom/aom_dsp/arm/aom_convolve_copy_neon_asm.asm create mode 100644 third_party/aom/aom_dsp/arm/aom_convolve_neon.c create mode 100644 third_party/aom/aom_dsp/arm/avg_neon.c create mode 100644 third_party/aom/aom_dsp/arm/bilinear_filter_media.asm create mode 100644 third_party/aom/aom_dsp/arm/fwd_txfm_neon.c create mode 100644 third_party/aom/aom_dsp/arm/hadamard_neon.c create mode 100644 third_party/aom/aom_dsp/arm/idct16x16_1_add_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/idct16x16_1_add_neon.c create mode 100644 third_party/aom/aom_dsp/arm/idct16x16_add_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/idct16x16_add_neon.c create mode 100644 third_party/aom/aom_dsp/arm/idct16x16_neon.c create mode 100644 third_party/aom/aom_dsp/arm/idct32x32_1_add_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/idct32x32_1_add_neon.c create mode 100644 third_party/aom/aom_dsp/arm/idct32x32_add_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/idct32x32_add_neon.c create mode 100644 third_party/aom/aom_dsp/arm/idct4x4_1_add_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/idct4x4_1_add_neon.c create mode 100644 third_party/aom/aom_dsp/arm/idct4x4_add_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/idct4x4_add_neon.c create mode 100644 third_party/aom/aom_dsp/arm/idct8x8_1_add_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/idct8x8_1_add_neon.c create mode 100644 third_party/aom/aom_dsp/arm/idct8x8_add_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/idct8x8_add_neon.c create mode 100644 third_party/aom/aom_dsp/arm/intrapred_neon.c create mode 100644 third_party/aom/aom_dsp/arm/intrapred_neon_asm.asm create mode 100644 third_party/aom/aom_dsp/arm/loopfilter_16_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/loopfilter_16_neon.c create mode 100644 third_party/aom/aom_dsp/arm/loopfilter_4_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/loopfilter_4_neon.c create mode 100644 third_party/aom/aom_dsp/arm/loopfilter_8_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/loopfilter_8_neon.c create mode 100644 third_party/aom/aom_dsp/arm/loopfilter_mb_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/loopfilter_neon.c create mode 100644 third_party/aom/aom_dsp/arm/sad4d_neon.c create mode 100644 third_party/aom/aom_dsp/arm/sad_media.asm create mode 100644 third_party/aom/aom_dsp/arm/sad_neon.c create mode 100644 third_party/aom/aom_dsp/arm/save_reg_neon.asm create mode 100644 third_party/aom/aom_dsp/arm/subpel_variance_media.c create mode 100644 third_party/aom/aom_dsp/arm/subpel_variance_neon.c create mode 100644 third_party/aom/aom_dsp/arm/subtract_neon.c create mode 100644 third_party/aom/aom_dsp/arm/variance_halfpixvar16x16_h_media.asm create mode 100644 third_party/aom/aom_dsp/arm/variance_halfpixvar16x16_hv_media.asm create mode 100644 third_party/aom/aom_dsp/arm/variance_halfpixvar16x16_v_media.asm create mode 100644 third_party/aom/aom_dsp/arm/variance_media.asm create mode 100644 third_party/aom/aom_dsp/arm/variance_neon.c create mode 100644 third_party/aom/aom_dsp/avg.c create mode 100644 third_party/aom/aom_dsp/binary_codes_reader.c create mode 100644 third_party/aom/aom_dsp/binary_codes_reader.h create mode 100644 third_party/aom/aom_dsp/binary_codes_writer.c create mode 100644 third_party/aom/aom_dsp/binary_codes_writer.h create mode 100644 third_party/aom/aom_dsp/bitreader.h create mode 100644 third_party/aom/aom_dsp/bitreader_buffer.c create mode 100644 third_party/aom/aom_dsp/bitreader_buffer.h create mode 100644 third_party/aom/aom_dsp/bitwriter.h create mode 100644 third_party/aom/aom_dsp/bitwriter_buffer.c create mode 100644 third_party/aom/aom_dsp/bitwriter_buffer.h create mode 100644 third_party/aom/aom_dsp/blend.h create mode 100644 third_party/aom/aom_dsp/blend_a64_hmask.c create mode 100644 third_party/aom/aom_dsp/blend_a64_mask.c create mode 100644 third_party/aom/aom_dsp/blend_a64_vmask.c create mode 100644 third_party/aom/aom_dsp/buf_ans.c create mode 100644 third_party/aom/aom_dsp/buf_ans.h create mode 100644 third_party/aom/aom_dsp/daalaboolreader.c create mode 100644 third_party/aom/aom_dsp/daalaboolreader.h create mode 100644 third_party/aom/aom_dsp/daalaboolwriter.c create mode 100644 third_party/aom/aom_dsp/daalaboolwriter.h create mode 100644 third_party/aom/aom_dsp/dkboolreader.c create mode 100644 third_party/aom/aom_dsp/dkboolreader.h create mode 100644 third_party/aom/aom_dsp/dkboolwriter.c create mode 100644 third_party/aom/aom_dsp/dkboolwriter.h create mode 100644 third_party/aom/aom_dsp/entcode.c create mode 100644 third_party/aom/aom_dsp/entcode.h create mode 100644 third_party/aom/aom_dsp/entdec.c create mode 100644 third_party/aom/aom_dsp/entdec.h create mode 100644 third_party/aom/aom_dsp/entenc.c create mode 100644 third_party/aom/aom_dsp/entenc.h create mode 100644 third_party/aom/aom_dsp/fastssim.c create mode 100644 third_party/aom/aom_dsp/fwd_txfm.c create mode 100644 third_party/aom/aom_dsp/fwd_txfm.h create mode 100644 third_party/aom/aom_dsp/intrapred.c create mode 100644 third_party/aom/aom_dsp/inv_txfm.c create mode 100644 third_party/aom/aom_dsp/inv_txfm.h create mode 100644 third_party/aom/aom_dsp/loopfilter.c create mode 100644 third_party/aom/aom_dsp/mips/add_noise_msa.c create mode 100644 third_party/aom/aom_dsp/mips/aom_convolve8_avg_horiz_msa.c create mode 100644 third_party/aom/aom_dsp/mips/aom_convolve8_avg_msa.c create mode 100644 third_party/aom/aom_dsp/mips/aom_convolve8_avg_vert_msa.c create mode 100644 third_party/aom/aom_dsp/mips/aom_convolve8_horiz_msa.c create mode 100644 third_party/aom/aom_dsp/mips/aom_convolve8_msa.c create mode 100644 third_party/aom/aom_dsp/mips/aom_convolve8_vert_msa.c create mode 100644 third_party/aom/aom_dsp/mips/aom_convolve_avg_msa.c create mode 100644 third_party/aom/aom_dsp/mips/aom_convolve_copy_msa.c create mode 100644 third_party/aom/aom_dsp/mips/aom_convolve_msa.h create mode 100644 third_party/aom/aom_dsp/mips/avg_msa.c create mode 100644 third_party/aom/aom_dsp/mips/common_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/common_dspr2.h create mode 100644 third_party/aom/aom_dsp/mips/convolve2_avg_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/convolve2_avg_horiz_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/convolve2_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/convolve2_horiz_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/convolve2_vert_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/convolve8_avg_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/convolve8_avg_horiz_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/convolve8_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/convolve8_horiz_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/convolve8_vert_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/convolve_common_dspr2.h create mode 100644 third_party/aom/aom_dsp/mips/fwd_dct32x32_msa.c create mode 100644 third_party/aom/aom_dsp/mips/fwd_txfm_msa.c create mode 100644 third_party/aom/aom_dsp/mips/fwd_txfm_msa.h create mode 100644 third_party/aom/aom_dsp/mips/idct16x16_msa.c create mode 100644 third_party/aom/aom_dsp/mips/idct32x32_msa.c create mode 100644 third_party/aom/aom_dsp/mips/idct4x4_msa.c create mode 100644 third_party/aom/aom_dsp/mips/idct8x8_msa.c create mode 100644 third_party/aom/aom_dsp/mips/intrapred16_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/intrapred4_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/intrapred8_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/intrapred_msa.c create mode 100644 third_party/aom/aom_dsp/mips/inv_txfm_dspr2.h create mode 100644 third_party/aom/aom_dsp/mips/inv_txfm_msa.h create mode 100644 third_party/aom/aom_dsp/mips/itrans16_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/itrans32_cols_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/itrans32_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/itrans4_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/itrans8_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/loopfilter_16_msa.c create mode 100644 third_party/aom/aom_dsp/mips/loopfilter_4_msa.c create mode 100644 third_party/aom/aom_dsp/mips/loopfilter_8_msa.c create mode 100644 third_party/aom/aom_dsp/mips/loopfilter_filters_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/loopfilter_filters_dspr2.h create mode 100644 third_party/aom/aom_dsp/mips/loopfilter_macros_dspr2.h create mode 100644 third_party/aom/aom_dsp/mips/loopfilter_masks_dspr2.h create mode 100644 third_party/aom/aom_dsp/mips/loopfilter_mb_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/loopfilter_mb_horiz_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/loopfilter_mb_vert_dspr2.c create mode 100644 third_party/aom/aom_dsp/mips/loopfilter_msa.h create mode 100644 third_party/aom/aom_dsp/mips/macros_msa.h create mode 100644 third_party/aom/aom_dsp/mips/sad_msa.c create mode 100644 third_party/aom/aom_dsp/mips/sub_pixel_variance_msa.c create mode 100644 third_party/aom/aom_dsp/mips/subtract_msa.c create mode 100644 third_party/aom/aom_dsp/mips/txfm_macros_msa.h create mode 100644 third_party/aom/aom_dsp/mips/variance_msa.c create mode 100644 third_party/aom/aom_dsp/postproc.h create mode 100644 third_party/aom/aom_dsp/prob.c create mode 100644 third_party/aom/aom_dsp/prob.h create mode 100644 third_party/aom/aom_dsp/psnr.c create mode 100644 third_party/aom/aom_dsp/psnr.h create mode 100644 third_party/aom/aom_dsp/psnrhvs.c create mode 100644 third_party/aom/aom_dsp/quantize.c create mode 100644 third_party/aom/aom_dsp/quantize.h create mode 100644 third_party/aom/aom_dsp/sad.c create mode 100644 third_party/aom/aom_dsp/simd/v128_intrinsics.h create mode 100644 third_party/aom/aom_dsp/simd/v128_intrinsics_arm.h create mode 100644 third_party/aom/aom_dsp/simd/v128_intrinsics_c.h create mode 100644 third_party/aom/aom_dsp/simd/v128_intrinsics_x86.h create mode 100644 third_party/aom/aom_dsp/simd/v256_intrinsics.h create mode 100644 third_party/aom/aom_dsp/simd/v256_intrinsics_arm.h create mode 100644 third_party/aom/aom_dsp/simd/v256_intrinsics_c.h create mode 100644 third_party/aom/aom_dsp/simd/v256_intrinsics_v128.h create mode 100644 third_party/aom/aom_dsp/simd/v256_intrinsics_x86.h create mode 100644 third_party/aom/aom_dsp/simd/v64_intrinsics.h create mode 100644 third_party/aom/aom_dsp/simd/v64_intrinsics_arm.h create mode 100644 third_party/aom/aom_dsp/simd/v64_intrinsics_c.h create mode 100644 third_party/aom/aom_dsp/simd/v64_intrinsics_x86.h create mode 100644 third_party/aom/aom_dsp/ssim.c create mode 100644 third_party/aom/aom_dsp/ssim.h create mode 100644 third_party/aom/aom_dsp/subtract.c create mode 100644 third_party/aom/aom_dsp/sum_squares.c create mode 100644 third_party/aom/aom_dsp/txfm_common.h create mode 100644 third_party/aom/aom_dsp/variance.c create mode 100644 third_party/aom/aom_dsp/variance.h create mode 100644 third_party/aom/aom_dsp/x86/aom_asm_stubs.c create mode 100644 third_party/aom/aom_dsp/x86/aom_convolve_copy_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/aom_high_subpixel_8t_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_avx2.c create mode 100644 third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c create mode 100644 third_party/aom/aom_dsp/x86/aom_subpixel_8t_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/aom_subpixel_8t_ssse3.asm create mode 100644 third_party/aom/aom_dsp/x86/aom_subpixel_bilinear_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/aom_subpixel_bilinear_ssse3.asm create mode 100644 third_party/aom/aom_dsp/x86/avg_intrin_sse2.c create mode 100644 third_party/aom/aom_dsp/x86/avg_ssse3_x86_64.asm create mode 100644 third_party/aom/aom_dsp/x86/blend_a64_hmask_sse4.c create mode 100644 third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c create mode 100644 third_party/aom/aom_dsp/x86/blend_a64_vmask_sse4.c create mode 100644 third_party/aom/aom_dsp/x86/blend_sse4.h create mode 100644 third_party/aom/aom_dsp/x86/convolve.h create mode 100644 third_party/aom/aom_dsp/x86/fwd_dct32_8cols_sse2.c create mode 100644 third_party/aom/aom_dsp/x86/fwd_dct32x32_impl_avx2.h create mode 100644 third_party/aom/aom_dsp/x86/fwd_dct32x32_impl_sse2.h create mode 100644 third_party/aom/aom_dsp/x86/fwd_txfm_avx2.c create mode 100644 third_party/aom/aom_dsp/x86/fwd_txfm_avx2.h create mode 100644 third_party/aom/aom_dsp/x86/fwd_txfm_impl_sse2.h create mode 100644 third_party/aom/aom_dsp/x86/fwd_txfm_sse2.c create mode 100644 third_party/aom/aom_dsp/x86/fwd_txfm_sse2.h create mode 100644 third_party/aom/aom_dsp/x86/fwd_txfm_ssse3_x86_64.asm create mode 100644 third_party/aom/aom_dsp/x86/halfpix_variance_impl_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/halfpix_variance_sse2.c create mode 100644 third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c create mode 100644 third_party/aom/aom_dsp/x86/highbd_intrapred_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c create mode 100644 third_party/aom/aom_dsp/x86/highbd_quantize_intrin_sse2.c create mode 100644 third_party/aom/aom_dsp/x86/highbd_sad4d_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/highbd_sad_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/highbd_subpel_variance_impl_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/highbd_subtract_sse2.c create mode 100644 third_party/aom/aom_dsp/x86/highbd_variance_impl_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/highbd_variance_sse2.c create mode 100644 third_party/aom/aom_dsp/x86/highbd_variance_sse4.c create mode 100644 third_party/aom/aom_dsp/x86/intrapred_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/intrapred_ssse3.asm create mode 100644 third_party/aom/aom_dsp/x86/inv_txfm_sse2.c create mode 100644 third_party/aom/aom_dsp/x86/inv_txfm_sse2.h create mode 100644 third_party/aom/aom_dsp/x86/inv_txfm_ssse3.c create mode 100644 third_party/aom/aom_dsp/x86/inv_wht_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/loopfilter_avx2.c create mode 100644 third_party/aom/aom_dsp/x86/loopfilter_sse2.c create mode 100644 third_party/aom/aom_dsp/x86/masked_sad_intrin_ssse3.c create mode 100644 third_party/aom/aom_dsp/x86/masked_variance_intrin_ssse3.c create mode 100644 third_party/aom/aom_dsp/x86/obmc_sad_sse4.c create mode 100644 third_party/aom/aom_dsp/x86/obmc_variance_sse4.c create mode 100644 third_party/aom/aom_dsp/x86/quantize_avx_x86_64.asm create mode 100644 third_party/aom/aom_dsp/x86/quantize_sse2.c create mode 100644 third_party/aom/aom_dsp/x86/quantize_ssse3_x86_64.asm create mode 100644 third_party/aom/aom_dsp/x86/sad4d_avx2.c create mode 100644 third_party/aom/aom_dsp/x86/sad4d_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/sad_avx2.c create mode 100644 third_party/aom/aom_dsp/x86/sad_highbd_avx2.c create mode 100644 third_party/aom/aom_dsp/x86/sad_impl_avx2.c create mode 100644 third_party/aom/aom_dsp/x86/sad_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/sad_sse3.asm create mode 100644 third_party/aom/aom_dsp/x86/sad_sse4.asm create mode 100644 third_party/aom/aom_dsp/x86/sad_ssse3.asm create mode 100644 third_party/aom/aom_dsp/x86/ssim_opt_x86_64.asm create mode 100644 third_party/aom/aom_dsp/x86/subpel_variance_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/subtract_sse2.asm create mode 100644 third_party/aom/aom_dsp/x86/sum_squares_sse2.c create mode 100644 third_party/aom/aom_dsp/x86/synonyms.h create mode 100644 third_party/aom/aom_dsp/x86/txfm_common_avx2.h create mode 100644 third_party/aom/aom_dsp/x86/txfm_common_intrin.h create mode 100644 third_party/aom/aom_dsp/x86/txfm_common_sse2.h create mode 100644 third_party/aom/aom_dsp/x86/variance_avx2.c create mode 100644 third_party/aom/aom_dsp/x86/variance_impl_avx2.c create mode 100644 third_party/aom/aom_dsp/x86/variance_sse2.c create mode 100644 third_party/aom/aom_mem/aom_mem.c create mode 100644 third_party/aom/aom_mem/aom_mem.cmake create mode 100644 third_party/aom/aom_mem/aom_mem.h create mode 100644 third_party/aom/aom_mem/aom_mem.mk create mode 100644 third_party/aom/aom_mem/include/aom_mem_intrnl.h create mode 100644 third_party/aom/aom_ports/aom_once.h create mode 100644 third_party/aom/aom_ports/aom_ports.cmake create mode 100644 third_party/aom/aom_ports/aom_ports.mk create mode 100644 third_party/aom/aom_ports/aom_timer.h create mode 100644 third_party/aom/aom_ports/arm.h create mode 100644 third_party/aom/aom_ports/arm_cpudetect.c create mode 100644 third_party/aom/aom_ports/bitops.h create mode 100644 third_party/aom/aom_ports/config.h create mode 100644 third_party/aom/aom_ports/emmintrin_compat.h create mode 100644 third_party/aom/aom_ports/emms.asm create mode 100644 third_party/aom/aom_ports/mem.h create mode 100644 third_party/aom/aom_ports/mem_ops.h create mode 100644 third_party/aom/aom_ports/mem_ops_aligned.h create mode 100644 third_party/aom/aom_ports/msvc.h create mode 100644 third_party/aom/aom_ports/system_state.h create mode 100644 third_party/aom/aom_ports/x86.h create mode 100644 third_party/aom/aom_ports/x86_abi_support.asm create mode 100644 third_party/aom/aom_scale/aom_scale.cmake create mode 100644 third_party/aom/aom_scale/aom_scale.h create mode 100644 third_party/aom/aom_scale/aom_scale.mk create mode 100644 third_party/aom/aom_scale/aom_scale_rtcd.c create mode 100644 third_party/aom/aom_scale/aom_scale_rtcd.pl create mode 100644 third_party/aom/aom_scale/generic/aom_scale.c create mode 100644 third_party/aom/aom_scale/generic/gen_scalers.c create mode 100644 third_party/aom/aom_scale/generic/yv12config.c create mode 100644 third_party/aom/aom_scale/generic/yv12extend.c create mode 100644 third_party/aom/aom_scale/mips/dspr2/yv12extend_dspr2.c create mode 100644 third_party/aom/aom_scale/yv12config.h create mode 100644 third_party/aom/aom_util/aom_thread.c create mode 100644 third_party/aom/aom_util/aom_thread.h create mode 100644 third_party/aom/aom_util/aom_util.cmake create mode 100644 third_party/aom/aom_util/aom_util.mk create mode 100644 third_party/aom/aom_util/debug_util.c create mode 100644 third_party/aom/aom_util/debug_util.h create mode 100644 third_party/aom/aom_util/endian_inl.h create mode 100644 third_party/aom/aomdec.c create mode 100644 third_party/aom/aomenc.c create mode 100644 third_party/aom/aomenc.h create mode 100644 third_party/aom/aomstats.c create mode 100644 third_party/aom/aomstats.h create mode 100644 third_party/aom/args.c create mode 100644 third_party/aom/args.h create mode 100644 third_party/aom/av1/av1.cmake create mode 100644 third_party/aom/av1/av1_common.mk create mode 100644 third_party/aom/av1/av1_cx.mk create mode 100644 third_party/aom/av1/av1_cx_iface.c create mode 100644 third_party/aom/av1/av1_dx.mk create mode 100644 third_party/aom/av1/av1_dx_iface.c create mode 100644 third_party/aom/av1/av1_iface_common.h create mode 100644 third_party/aom/av1/common/alloccommon.c create mode 100644 third_party/aom/av1/common/alloccommon.h create mode 100644 third_party/aom/av1/common/arm/neon/iht4x4_add_neon.c create mode 100644 third_party/aom/av1/common/arm/neon/iht8x8_add_neon.c create mode 100644 third_party/aom/av1/common/av1_fwd_txfm1d.c create mode 100644 third_party/aom/av1/common/av1_fwd_txfm1d.h create mode 100644 third_party/aom/av1/common/av1_fwd_txfm2d.c create mode 100644 third_party/aom/av1/common/av1_fwd_txfm2d_cfg.h create mode 100644 third_party/aom/av1/common/av1_inv_txfm1d.c create mode 100644 third_party/aom/av1/common/av1_inv_txfm1d.h create mode 100644 third_party/aom/av1/common/av1_inv_txfm2d.c create mode 100644 third_party/aom/av1/common/av1_inv_txfm2d_cfg.h create mode 100644 third_party/aom/av1/common/av1_loopfilter.c create mode 100644 third_party/aom/av1/common/av1_loopfilter.h create mode 100644 third_party/aom/av1/common/av1_rtcd.c create mode 100755 third_party/aom/av1/common/av1_rtcd_defs.pl create mode 100644 third_party/aom/av1/common/av1_txfm.h create mode 100644 third_party/aom/av1/common/blockd.c create mode 100644 third_party/aom/av1/common/blockd.h create mode 100644 third_party/aom/av1/common/cdef.c create mode 100644 third_party/aom/av1/common/cdef.h create mode 100644 third_party/aom/av1/common/cdef_simd.h create mode 100644 third_party/aom/av1/common/cfl.c create mode 100644 third_party/aom/av1/common/cfl.h create mode 100644 third_party/aom/av1/common/clpf.c create mode 100644 third_party/aom/av1/common/clpf.h create mode 100644 third_party/aom/av1/common/clpf_neon.c create mode 100644 third_party/aom/av1/common/clpf_simd.h create mode 100644 third_party/aom/av1/common/clpf_sse2.c create mode 100644 third_party/aom/av1/common/clpf_sse4.c create mode 100644 third_party/aom/av1/common/clpf_ssse3.c create mode 100644 third_party/aom/av1/common/common.h create mode 100644 third_party/aom/av1/common/common_data.h create mode 100644 third_party/aom/av1/common/convolve.c create mode 100644 third_party/aom/av1/common/convolve.h create mode 100644 third_party/aom/av1/common/debugmodes.c create mode 100644 third_party/aom/av1/common/entropy.c create mode 100644 third_party/aom/av1/common/entropy.h create mode 100644 third_party/aom/av1/common/entropymode.c create mode 100644 third_party/aom/av1/common/entropymode.h create mode 100644 third_party/aom/av1/common/entropymv.c create mode 100644 third_party/aom/av1/common/entropymv.h create mode 100644 third_party/aom/av1/common/enums.h create mode 100644 third_party/aom/av1/common/filter.c create mode 100644 third_party/aom/av1/common/filter.h create mode 100644 third_party/aom/av1/common/frame_buffers.c create mode 100644 third_party/aom/av1/common/frame_buffers.h create mode 100644 third_party/aom/av1/common/generic_code.c create mode 100644 third_party/aom/av1/common/generic_code.h create mode 100644 third_party/aom/av1/common/idct.c create mode 100644 third_party/aom/av1/common/idct.h create mode 100644 third_party/aom/av1/common/laplace_tables.c create mode 100644 third_party/aom/av1/common/mips/dspr2/av1_itrans16_dspr2.c create mode 100644 third_party/aom/av1/common/mips/dspr2/av1_itrans4_dspr2.c create mode 100644 third_party/aom/av1/common/mips/dspr2/av1_itrans8_dspr2.c create mode 100644 third_party/aom/av1/common/mips/msa/av1_idct16x16_msa.c create mode 100644 third_party/aom/av1/common/mips/msa/av1_idct4x4_msa.c create mode 100644 third_party/aom/av1/common/mips/msa/av1_idct8x8_msa.c create mode 100644 third_party/aom/av1/common/mv.h create mode 100644 third_party/aom/av1/common/mvref_common.c create mode 100644 third_party/aom/av1/common/mvref_common.h create mode 100644 third_party/aom/av1/common/od_dering.c create mode 100644 third_party/aom/av1/common/od_dering.h create mode 100644 third_party/aom/av1/common/od_dering_neon.c create mode 100644 third_party/aom/av1/common/od_dering_simd.h create mode 100644 third_party/aom/av1/common/od_dering_sse2.c create mode 100644 third_party/aom/av1/common/od_dering_sse4.c create mode 100644 third_party/aom/av1/common/od_dering_ssse3.c create mode 100644 third_party/aom/av1/common/odintrin.c create mode 100644 third_party/aom/av1/common/odintrin.h create mode 100644 third_party/aom/av1/common/onyxc_int.h create mode 100644 third_party/aom/av1/common/partition.c create mode 100644 third_party/aom/av1/common/partition.h create mode 100644 third_party/aom/av1/common/pred_common.c create mode 100644 third_party/aom/av1/common/pred_common.h create mode 100644 third_party/aom/av1/common/pvq.c create mode 100644 third_party/aom/av1/common/pvq.h create mode 100644 third_party/aom/av1/common/pvq_state.c create mode 100644 third_party/aom/av1/common/pvq_state.h create mode 100644 third_party/aom/av1/common/quant_common.c create mode 100644 third_party/aom/av1/common/quant_common.h create mode 100644 third_party/aom/av1/common/reconinter.c create mode 100644 third_party/aom/av1/common/reconinter.h create mode 100644 third_party/aom/av1/common/reconintra.c create mode 100644 third_party/aom/av1/common/reconintra.h create mode 100644 third_party/aom/av1/common/resize.c create mode 100644 third_party/aom/av1/common/resize.h create mode 100644 third_party/aom/av1/common/restoration.c create mode 100644 third_party/aom/av1/common/restoration.h create mode 100644 third_party/aom/av1/common/scale.c create mode 100644 third_party/aom/av1/common/scale.h create mode 100644 third_party/aom/av1/common/scan.c create mode 100644 third_party/aom/av1/common/scan.h create mode 100644 third_party/aom/av1/common/seg_common.c create mode 100644 third_party/aom/av1/common/seg_common.h create mode 100644 third_party/aom/av1/common/thread_common.c create mode 100644 third_party/aom/av1/common/thread_common.h create mode 100644 third_party/aom/av1/common/tile_common.c create mode 100644 third_party/aom/av1/common/tile_common.h create mode 100644 third_party/aom/av1/common/txb_common.c create mode 100644 third_party/aom/av1/common/txb_common.h create mode 100644 third_party/aom/av1/common/warped_motion.c create mode 100644 third_party/aom/av1/common/warped_motion.h create mode 100644 third_party/aom/av1/common/x86/av1_convolve_ssse3.c create mode 100644 third_party/aom/av1/common/x86/av1_fwd_txfm1d_sse4.c create mode 100644 third_party/aom/av1/common/x86/av1_fwd_txfm2d_sse4.c create mode 100644 third_party/aom/av1/common/x86/av1_highbd_convolve_sse4.c create mode 100644 third_party/aom/av1/common/x86/av1_txfm1d_sse4.h create mode 100644 third_party/aom/av1/common/x86/filterintra_sse4.c create mode 100644 third_party/aom/av1/common/x86/highbd_inv_txfm_avx2.c create mode 100644 third_party/aom/av1/common/x86/highbd_inv_txfm_sse4.c create mode 100644 third_party/aom/av1/common/x86/highbd_txfm_utility_sse4.h create mode 100644 third_party/aom/av1/common/x86/highbd_warp_plane_ssse3.c create mode 100644 third_party/aom/av1/common/x86/hybrid_inv_txfm_avx2.c create mode 100644 third_party/aom/av1/common/x86/idct_intrin_sse2.c create mode 100644 third_party/aom/av1/common/x86/pvq_sse4.c create mode 100644 third_party/aom/av1/common/x86/pvq_sse4.h create mode 100644 third_party/aom/av1/common/x86/selfguided_sse4.c create mode 100644 third_party/aom/av1/common/x86/warp_plane_sse2.c create mode 100644 third_party/aom/av1/common/zigzag.h create mode 100644 third_party/aom/av1/common/zigzag16.c create mode 100644 third_party/aom/av1/common/zigzag32.c create mode 100644 third_party/aom/av1/common/zigzag4.c create mode 100644 third_party/aom/av1/common/zigzag8.c create mode 100644 third_party/aom/av1/decoder/accounting.c create mode 100644 third_party/aom/av1/decoder/accounting.h create mode 100644 third_party/aom/av1/decoder/decint.h create mode 100644 third_party/aom/av1/decoder/decodeframe.c create mode 100644 third_party/aom/av1/decoder/decodeframe.h create mode 100644 third_party/aom/av1/decoder/decodemv.c create mode 100644 third_party/aom/av1/decoder/decodemv.h create mode 100644 third_party/aom/av1/decoder/decoder.c create mode 100644 third_party/aom/av1/decoder/decoder.h create mode 100644 third_party/aom/av1/decoder/decodetxb.c create mode 100644 third_party/aom/av1/decoder/decodetxb.h create mode 100644 third_party/aom/av1/decoder/detokenize.c create mode 100644 third_party/aom/av1/decoder/detokenize.h create mode 100644 third_party/aom/av1/decoder/dsubexp.c create mode 100644 third_party/aom/av1/decoder/dsubexp.h create mode 100644 third_party/aom/av1/decoder/dthread.c create mode 100644 third_party/aom/av1/decoder/dthread.h create mode 100644 third_party/aom/av1/decoder/generic_decoder.c create mode 100644 third_party/aom/av1/decoder/inspection.c create mode 100644 third_party/aom/av1/decoder/inspection.h create mode 100644 third_party/aom/av1/decoder/laplace_decoder.c create mode 100644 third_party/aom/av1/decoder/pvq_decoder.c create mode 100644 third_party/aom/av1/decoder/pvq_decoder.h create mode 100644 third_party/aom/av1/encoder/aq_complexity.c create mode 100644 third_party/aom/av1/encoder/aq_complexity.h create mode 100644 third_party/aom/av1/encoder/aq_cyclicrefresh.c create mode 100644 third_party/aom/av1/encoder/aq_cyclicrefresh.h create mode 100644 third_party/aom/av1/encoder/aq_variance.c create mode 100644 third_party/aom/av1/encoder/aq_variance.h create mode 100644 third_party/aom/av1/encoder/arm/neon/dct_neon.c create mode 100644 third_party/aom/av1/encoder/arm/neon/error_neon.c create mode 100644 third_party/aom/av1/encoder/arm/neon/quantize_neon.c create mode 100644 third_party/aom/av1/encoder/av1_quantize.c create mode 100644 third_party/aom/av1/encoder/av1_quantize.h create mode 100644 third_party/aom/av1/encoder/bitstream.c create mode 100644 third_party/aom/av1/encoder/bitstream.h create mode 100644 third_party/aom/av1/encoder/block.h create mode 100644 third_party/aom/av1/encoder/blockiness.c create mode 100644 third_party/aom/av1/encoder/context_tree.c create mode 100644 third_party/aom/av1/encoder/context_tree.h create mode 100644 third_party/aom/av1/encoder/corner_detect.c create mode 100644 third_party/aom/av1/encoder/corner_detect.h create mode 100644 third_party/aom/av1/encoder/corner_match.c create mode 100644 third_party/aom/av1/encoder/corner_match.h create mode 100644 third_party/aom/av1/encoder/cost.c create mode 100644 third_party/aom/av1/encoder/cost.h create mode 100644 third_party/aom/av1/encoder/daala_compat_enc.c create mode 100644 third_party/aom/av1/encoder/dct.c create mode 100644 third_party/aom/av1/encoder/encint.h create mode 100644 third_party/aom/av1/encoder/encodeframe.c create mode 100644 third_party/aom/av1/encoder/encodeframe.h create mode 100644 third_party/aom/av1/encoder/encodemb.c create mode 100644 third_party/aom/av1/encoder/encodemb.h create mode 100644 third_party/aom/av1/encoder/encodemv.c create mode 100644 third_party/aom/av1/encoder/encodemv.h create mode 100644 third_party/aom/av1/encoder/encoder.c create mode 100644 third_party/aom/av1/encoder/encoder.h create mode 100644 third_party/aom/av1/encoder/encodetxb.c create mode 100644 third_party/aom/av1/encoder/encodetxb.h create mode 100644 third_party/aom/av1/encoder/ethread.c create mode 100644 third_party/aom/av1/encoder/ethread.h create mode 100644 third_party/aom/av1/encoder/extend.c create mode 100644 third_party/aom/av1/encoder/extend.h create mode 100644 third_party/aom/av1/encoder/firstpass.c create mode 100644 third_party/aom/av1/encoder/firstpass.h create mode 100644 third_party/aom/av1/encoder/generic_encoder.c create mode 100644 third_party/aom/av1/encoder/global_motion.c create mode 100644 third_party/aom/av1/encoder/global_motion.h create mode 100644 third_party/aom/av1/encoder/hybrid_fwd_txfm.c create mode 100644 third_party/aom/av1/encoder/hybrid_fwd_txfm.h create mode 100644 third_party/aom/av1/encoder/laplace_encoder.c create mode 100644 third_party/aom/av1/encoder/lookahead.c create mode 100644 third_party/aom/av1/encoder/lookahead.h create mode 100644 third_party/aom/av1/encoder/mbgraph.c create mode 100644 third_party/aom/av1/encoder/mbgraph.h create mode 100644 third_party/aom/av1/encoder/mcomp.c create mode 100644 third_party/aom/av1/encoder/mcomp.h create mode 100644 third_party/aom/av1/encoder/mips/msa/error_msa.c create mode 100644 third_party/aom/av1/encoder/mips/msa/fdct16x16_msa.c create mode 100644 third_party/aom/av1/encoder/mips/msa/fdct4x4_msa.c create mode 100644 third_party/aom/av1/encoder/mips/msa/fdct8x8_msa.c create mode 100644 third_party/aom/av1/encoder/mips/msa/fdct_msa.h create mode 100644 third_party/aom/av1/encoder/mips/msa/temporal_filter_msa.c create mode 100644 third_party/aom/av1/encoder/palette.c create mode 100644 third_party/aom/av1/encoder/palette.h create mode 100644 third_party/aom/av1/encoder/pickcdef.c create mode 100644 third_party/aom/av1/encoder/picklpf.c create mode 100644 third_party/aom/av1/encoder/picklpf.h create mode 100644 third_party/aom/av1/encoder/pickrst.c create mode 100644 third_party/aom/av1/encoder/pickrst.h create mode 100644 third_party/aom/av1/encoder/pvq_encoder.c create mode 100644 third_party/aom/av1/encoder/pvq_encoder.h create mode 100644 third_party/aom/av1/encoder/ransac.c create mode 100644 third_party/aom/av1/encoder/ransac.h create mode 100644 third_party/aom/av1/encoder/ratectrl.c create mode 100644 third_party/aom/av1/encoder/ratectrl.h create mode 100644 third_party/aom/av1/encoder/ratectrl_xiph.c create mode 100644 third_party/aom/av1/encoder/ratectrl_xiph.h create mode 100644 third_party/aom/av1/encoder/rd.c create mode 100644 third_party/aom/av1/encoder/rd.h create mode 100644 third_party/aom/av1/encoder/rdopt.c create mode 100644 third_party/aom/av1/encoder/rdopt.h create mode 100644 third_party/aom/av1/encoder/segmentation.c create mode 100644 third_party/aom/av1/encoder/segmentation.h create mode 100644 third_party/aom/av1/encoder/speed_features.c create mode 100644 third_party/aom/av1/encoder/speed_features.h create mode 100644 third_party/aom/av1/encoder/subexp.c create mode 100644 third_party/aom/av1/encoder/subexp.h create mode 100644 third_party/aom/av1/encoder/temporal_filter.c create mode 100644 third_party/aom/av1/encoder/temporal_filter.h create mode 100644 third_party/aom/av1/encoder/tokenize.c create mode 100644 third_party/aom/av1/encoder/tokenize.h create mode 100644 third_party/aom/av1/encoder/treewriter.c create mode 100644 third_party/aom/av1/encoder/treewriter.h create mode 100644 third_party/aom/av1/encoder/variance_tree.c create mode 100644 third_party/aom/av1/encoder/variance_tree.h create mode 100644 third_party/aom/av1/encoder/wedge_utils.c create mode 100644 third_party/aom/av1/encoder/x86/av1_highbd_quantize_sse4.c create mode 100644 third_party/aom/av1/encoder/x86/av1_quantize_sse2.c create mode 100644 third_party/aom/av1/encoder/x86/av1_quantize_ssse3_x86_64.asm create mode 100644 third_party/aom/av1/encoder/x86/av1_ssim_opt_x86_64.asm create mode 100644 third_party/aom/av1/encoder/x86/dct_intrin_sse2.c create mode 100644 third_party/aom/av1/encoder/x86/dct_sse2.asm create mode 100644 third_party/aom/av1/encoder/x86/dct_ssse3.c create mode 100644 third_party/aom/av1/encoder/x86/error_intrin_avx2.c create mode 100644 third_party/aom/av1/encoder/x86/error_sse2.asm create mode 100644 third_party/aom/av1/encoder/x86/highbd_block_error_intrin_sse2.c create mode 100644 third_party/aom/av1/encoder/x86/highbd_fwd_txfm_sse4.c create mode 100644 third_party/aom/av1/encoder/x86/hybrid_fwd_txfm_avx2.c create mode 100644 third_party/aom/av1/encoder/x86/temporal_filter_apply_sse2.asm create mode 100644 third_party/aom/av1/encoder/x86/wedge_utils_sse2.c create mode 100644 third_party/aom/av1/exports_dec create mode 100644 third_party/aom/av1/exports_enc create mode 100644 third_party/aom/build/cmake/aom_config.c.cmake create mode 100644 third_party/aom/build/cmake/aom_config_defaults.cmake create mode 100644 third_party/aom/build/cmake/aom_configure.cmake create mode 100644 third_party/aom/build/cmake/aom_optimization.cmake create mode 100755 third_party/aom/build/cmake/aom_version.pl create mode 100644 third_party/aom/build/cmake/compiler_flags.cmake create mode 100644 third_party/aom/build/cmake/compiler_tests.cmake create mode 100644 third_party/aom/build/cmake/cpu.cmake create mode 100644 third_party/aom/build/cmake/generate_aom_config_templates.cmake create mode 100644 third_party/aom/build/cmake/msvc_runtime.cmake create mode 100644 third_party/aom/build/cmake/rtcd_config.cmake create mode 100644 third_party/aom/build/cmake/toolchains/arm-ios-common.cmake create mode 100644 third_party/aom/build/cmake/toolchains/arm64-ios.cmake create mode 100644 third_party/aom/build/cmake/toolchains/arm64-linux-gcc.cmake create mode 100644 third_party/aom/build/cmake/toolchains/armv7-ios.cmake create mode 100644 third_party/aom/build/cmake/toolchains/armv7-linux-gcc.cmake create mode 100644 third_party/aom/build/cmake/toolchains/armv7s-ios.cmake create mode 100644 third_party/aom/build/cmake/toolchains/ios-simulator-common.cmake create mode 100644 third_party/aom/build/cmake/toolchains/mips32-linux-gcc.cmake create mode 100644 third_party/aom/build/cmake/toolchains/mips64-linux-gcc.cmake create mode 100644 third_party/aom/build/cmake/toolchains/x86-ios-simulator.cmake create mode 100644 third_party/aom/build/cmake/toolchains/x86-linux.cmake create mode 100644 third_party/aom/build/cmake/toolchains/x86-macos.cmake create mode 100644 third_party/aom/build/cmake/toolchains/x86_64-ios-simulator.cmake create mode 100644 third_party/aom/build/make/Android.mk create mode 100644 third_party/aom/build/make/Makefile create mode 100755 third_party/aom/build/make/ads2armasm_ms.pl create mode 100755 third_party/aom/build/make/ads2gas.pl create mode 100755 third_party/aom/build/make/ads2gas_apple.pl create mode 100755 third_party/aom/build/make/armlink_adapter.sh create mode 100644 third_party/aom/build/make/configure.sh create mode 100755 third_party/aom/build/make/gen_asm_deps.sh create mode 100755 third_party/aom/build/make/gen_msvs_def.sh create mode 100755 third_party/aom/build/make/gen_msvs_sln.sh create mode 100755 third_party/aom/build/make/gen_msvs_vcxproj.sh create mode 100644 third_party/aom/build/make/ios-Info.plist create mode 100755 third_party/aom/build/make/iosbuild.sh create mode 100644 third_party/aom/build/make/msvs_common.sh create mode 100755 third_party/aom/build/make/rtcd.pl create mode 100644 third_party/aom/build/make/thumb.pm create mode 100755 third_party/aom/build/make/version.sh create mode 100644 third_party/aom/codereview.settings create mode 100755 third_party/aom/configure create mode 100644 third_party/aom/docs.mk create mode 100644 third_party/aom/examples.mk create mode 100644 third_party/aom/examples/analyzer.cc create mode 100644 third_party/aom/examples/aom_cx_set_ref.c create mode 100644 third_party/aom/examples/decode_to_md5.c create mode 100644 third_party/aom/examples/decode_with_drops.c create mode 100644 third_party/aom/examples/encoder_util.c create mode 100644 third_party/aom/examples/encoder_util.h create mode 100644 third_party/aom/examples/inspect.c create mode 100644 third_party/aom/examples/lossless_encoder.c create mode 100644 third_party/aom/examples/resize_util.c create mode 100644 third_party/aom/examples/set_maps.c create mode 100644 third_party/aom/examples/simple_decoder.c create mode 100644 third_party/aom/examples/simple_encoder.c create mode 100644 third_party/aom/examples/twopass_encoder.c create mode 100644 third_party/aom/ivfdec.c create mode 100644 third_party/aom/ivfdec.h create mode 100644 third_party/aom/ivfenc.c create mode 100644 third_party/aom/ivfenc.h create mode 100644 third_party/aom/keywords.dox create mode 100644 third_party/aom/libs.doxy_template create mode 100644 third_party/aom/libs.mk create mode 100644 third_party/aom/mainpage.dox create mode 100644 third_party/aom/md5_utils.c create mode 100644 third_party/aom/md5_utils.h create mode 100644 third_party/aom/rate_hist.c create mode 100644 third_party/aom/rate_hist.h create mode 100644 third_party/aom/solution.mk create mode 100644 third_party/aom/test/accounting_test.cc create mode 100644 third_party/aom/test/acm_random.h create mode 100644 third_party/aom/test/active_map_refresh_test.cc create mode 100644 third_party/aom/test/active_map_test.cc create mode 100644 third_party/aom/test/altref_test.cc create mode 100644 third_party/aom/test/android/Android.mk create mode 100644 third_party/aom/test/android/README create mode 100644 third_party/aom/test/android/get_files.py create mode 100644 third_party/aom/test/android/scrape_gtest_log.py create mode 100644 third_party/aom/test/ans_codec_test.cc create mode 100644 third_party/aom/test/ans_test.cc create mode 100755 third_party/aom/test/aomcx_set_ref.sh create mode 100755 third_party/aom/test/aomdec.sh create mode 100755 third_party/aom/test/aomenc.sh create mode 100644 third_party/aom/test/aq_segment_test.cc create mode 100644 third_party/aom/test/arf_freq_test.cc create mode 100644 third_party/aom/test/av1_convolve_optimz_test.cc create mode 100644 third_party/aom/test/av1_convolve_test.cc create mode 100644 third_party/aom/test/av1_dct_test.cc create mode 100644 third_party/aom/test/av1_ext_tile_test.cc create mode 100644 third_party/aom/test/av1_fht16x16_test.cc create mode 100644 third_party/aom/test/av1_fht16x32_test.cc create mode 100644 third_party/aom/test/av1_fht16x8_test.cc create mode 100644 third_party/aom/test/av1_fht32x16_test.cc create mode 100644 third_party/aom/test/av1_fht4x4_test.cc create mode 100644 third_party/aom/test/av1_fht4x8_test.cc create mode 100644 third_party/aom/test/av1_fht8x16_test.cc create mode 100644 third_party/aom/test/av1_fht8x4_test.cc create mode 100644 third_party/aom/test/av1_fht8x8_test.cc create mode 100644 third_party/aom/test/av1_fwd_txfm1d_test.cc create mode 100644 third_party/aom/test/av1_fwd_txfm2d_test.cc create mode 100644 third_party/aom/test/av1_highbd_iht_test.cc create mode 100644 third_party/aom/test/av1_inv_txfm1d_test.cc create mode 100644 third_party/aom/test/av1_inv_txfm2d_test.cc create mode 100644 third_party/aom/test/av1_inv_txfm_test.cc create mode 100644 third_party/aom/test/av1_quantize_test.cc create mode 100644 third_party/aom/test/av1_txfm_test.cc create mode 100644 third_party/aom/test/av1_txfm_test.h create mode 100644 third_party/aom/test/av1_wedge_utils_test.cc create mode 100644 third_party/aom/test/avg_test.cc create mode 100644 third_party/aom/test/binary_codes_test.cc create mode 100644 third_party/aom/test/blend_a64_mask_1d_test.cc create mode 100644 third_party/aom/test/blend_a64_mask_test.cc create mode 100644 third_party/aom/test/boolcoder_test.cc create mode 100644 third_party/aom/test/borders_test.cc create mode 100644 third_party/aom/test/clear_system_state.h create mode 100644 third_party/aom/test/clpf_test.cc create mode 100644 third_party/aom/test/codec_factory.h create mode 100644 third_party/aom/test/convolve_test.cc create mode 100644 third_party/aom/test/cpu_speed_test.cc create mode 100644 third_party/aom/test/datarate_test.cc create mode 100644 third_party/aom/test/dct16x16_test.cc create mode 100644 third_party/aom/test/dct32x32_test.cc create mode 100644 third_party/aom/test/decode_api_test.cc create mode 100644 third_party/aom/test/decode_perf_test.cc create mode 100644 third_party/aom/test/decode_test_driver.cc create mode 100644 third_party/aom/test/decode_test_driver.h create mode 100755 third_party/aom/test/decode_to_md5.sh create mode 100755 third_party/aom/test/decode_with_drops.sh create mode 100644 third_party/aom/test/dering_test.cc create mode 100644 third_party/aom/test/divu_small_test.cc create mode 100644 third_party/aom/test/encode_api_test.cc create mode 100644 third_party/aom/test/encode_perf_test.cc create mode 100644 third_party/aom/test/encode_test_driver.cc create mode 100644 third_party/aom/test/encode_test_driver.h create mode 100644 third_party/aom/test/encoder_parms_get_to_decoder.cc create mode 100644 third_party/aom/test/end_to_end_test.cc create mode 100644 third_party/aom/test/error_block_test.cc create mode 100644 third_party/aom/test/error_resilience_test.cc create mode 100644 third_party/aom/test/ethread_test.cc create mode 100755 third_party/aom/test/examples.sh create mode 100644 third_party/aom/test/fdct4x4_test.cc create mode 100644 third_party/aom/test/fdct8x8_test.cc create mode 100644 third_party/aom/test/fht32x32_test.cc create mode 100644 third_party/aom/test/filterintra_predictors_test.cc create mode 100644 third_party/aom/test/frame_size_tests.cc create mode 100644 third_party/aom/test/function_equivalence_test.h create mode 100644 third_party/aom/test/hadamard_test.cc create mode 100644 third_party/aom/test/hbd_metrics_test.cc create mode 100644 third_party/aom/test/i420_video_source.h create mode 100644 third_party/aom/test/idct8x8_test.cc create mode 100644 third_party/aom/test/idct_test.cc create mode 100644 third_party/aom/test/intrabc_test.cc create mode 100644 third_party/aom/test/intrapred_test.cc create mode 100644 third_party/aom/test/ivf_video_source.h create mode 100644 third_party/aom/test/level_test.cc create mode 100644 third_party/aom/test/lossless_test.cc create mode 100644 third_party/aom/test/lpf_8_test.cc create mode 100644 third_party/aom/test/masked_sad_test.cc create mode 100644 third_party/aom/test/masked_variance_test.cc create mode 100644 third_party/aom/test/md5_helper.h create mode 100644 third_party/aom/test/minmax_test.cc create mode 100644 third_party/aom/test/motion_vector_test.cc create mode 100644 third_party/aom/test/obmc_sad_test.cc create mode 100644 third_party/aom/test/obmc_variance_test.cc create mode 100644 third_party/aom/test/partial_idct_test.cc create mode 100644 third_party/aom/test/quantize_test.cc create mode 100644 third_party/aom/test/realtime_test.cc create mode 100644 third_party/aom/test/register_state_check.h create mode 100644 third_party/aom/test/resize_test.cc create mode 100644 third_party/aom/test/sad_test.cc create mode 100644 third_party/aom/test/scan_test.cc create mode 100644 third_party/aom/test/selfguided_filter_test.cc create mode 100755 third_party/aom/test/set_maps.sh create mode 100644 third_party/aom/test/simd_cmp_impl.h create mode 100644 third_party/aom/test/simd_cmp_neon.cc create mode 100644 third_party/aom/test/simd_cmp_sse2.cc create mode 100644 third_party/aom/test/simd_cmp_sse4.cc create mode 100644 third_party/aom/test/simd_cmp_ssse3.cc create mode 100644 third_party/aom/test/simd_impl.h create mode 100644 third_party/aom/test/simd_neon_test.cc create mode 100644 third_party/aom/test/simd_sse2_test.cc create mode 100644 third_party/aom/test/simd_sse4_test.cc create mode 100644 third_party/aom/test/simd_ssse3_test.cc create mode 100755 third_party/aom/test/simple_decoder.sh create mode 100755 third_party/aom/test/simple_encoder.sh create mode 100644 third_party/aom/test/subtract_test.cc create mode 100644 third_party/aom/test/sum_squares_test.cc create mode 100644 third_party/aom/test/superframe_test.cc create mode 100644 third_party/aom/test/test-data.mk create mode 100644 third_party/aom/test/test-data.sha1 create mode 100644 third_party/aom/test/test.cmake create mode 100644 third_party/aom/test/test.mk create mode 100644 third_party/aom/test/test_data_util.cmake create mode 100644 third_party/aom/test/test_intra_pred_speed.cc create mode 100644 third_party/aom/test/test_libaom.cc create mode 100644 third_party/aom/test/test_worker.cmake create mode 100644 third_party/aom/test/tile_independence_test.cc create mode 100755 third_party/aom/test/tools_common.sh create mode 100644 third_party/aom/test/transform_test_base.h create mode 100755 third_party/aom/test/twopass_encoder.sh create mode 100644 third_party/aom/test/user_priv_test.cc create mode 100644 third_party/aom/test/util.h create mode 100644 third_party/aom/test/variance_test.cc create mode 100644 third_party/aom/test/video_source.h create mode 100644 third_party/aom/test/warp_filter_test.cc create mode 100644 third_party/aom/test/warp_filter_test_util.cc create mode 100644 third_party/aom/test/warp_filter_test_util.h create mode 100644 third_party/aom/test/webm_video_source.h create mode 100644 third_party/aom/test/y4m_test.cc create mode 100644 third_party/aom/test/y4m_video_source.h create mode 100644 third_party/aom/test/yuv_video_source.h create mode 100644 third_party/aom/third_party/fastfeat/LICENSE create mode 100644 third_party/aom/third_party/fastfeat/README.libvpx create mode 100644 third_party/aom/third_party/fastfeat/fast.c create mode 100644 third_party/aom/third_party/fastfeat/fast.h create mode 100644 third_party/aom/third_party/fastfeat/fast_9.c create mode 100644 third_party/aom/third_party/fastfeat/nonmax.c create mode 100644 third_party/aom/third_party/googletest/README.libaom create mode 100644 third_party/aom/third_party/googletest/gtest.mk create mode 100644 third_party/aom/third_party/googletest/src/googletest/CHANGES create mode 100644 third_party/aom/third_party/googletest/src/googletest/CMakeLists.txt create mode 100644 third_party/aom/third_party/googletest/src/googletest/CONTRIBUTORS create mode 100644 third_party/aom/third_party/googletest/src/googletest/LICENSE create mode 100644 third_party/aom/third_party/googletest/src/googletest/README.md create mode 100644 third_party/aom/third_party/googletest/src/googletest/cmake/internal_utils.cmake create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-death-test.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-message.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-param-test.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-param-test.h.pump create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-printers.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-spi.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-test-part.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-typed-test.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest_pred_impl.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest_prod.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/custom/gtest-port.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/custom/gtest-printers.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/custom/gtest.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-death-test-internal.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-filepath.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-internal.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-linked_ptr.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-param-util-generated.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-param-util-generated.h.pump create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-param-util.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-port-arch.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-port.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-string.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-tuple.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-tuple.h.pump create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-type-util.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-type-util.h.pump create mode 100644 third_party/aom/third_party/googletest/src/googletest/src/gtest-all.cc create mode 100644 third_party/aom/third_party/googletest/src/googletest/src/gtest-death-test.cc create mode 100644 third_party/aom/third_party/googletest/src/googletest/src/gtest-filepath.cc create mode 100644 third_party/aom/third_party/googletest/src/googletest/src/gtest-internal-inl.h create mode 100644 third_party/aom/third_party/googletest/src/googletest/src/gtest-port.cc create mode 100644 third_party/aom/third_party/googletest/src/googletest/src/gtest-printers.cc create mode 100644 third_party/aom/third_party/googletest/src/googletest/src/gtest-test-part.cc create mode 100644 third_party/aom/third_party/googletest/src/googletest/src/gtest-typed-test.cc create mode 100644 third_party/aom/third_party/googletest/src/googletest/src/gtest.cc create mode 100644 third_party/aom/third_party/googletest/src/googletest/src/gtest_main.cc create mode 100644 third_party/aom/third_party/libwebm/AUTHORS.TXT create mode 100644 third_party/aom/third_party/libwebm/Android.mk create mode 100644 third_party/aom/third_party/libwebm/LICENSE.TXT create mode 100644 third_party/aom/third_party/libwebm/PATENTS.TXT create mode 100644 third_party/aom/third_party/libwebm/README.libaom create mode 100644 third_party/aom/third_party/libwebm/common/file_util.cc create mode 100644 third_party/aom/third_party/libwebm/common/file_util.h create mode 100644 third_party/aom/third_party/libwebm/common/hdr_util.cc create mode 100644 third_party/aom/third_party/libwebm/common/hdr_util.h create mode 100644 third_party/aom/third_party/libwebm/common/webmids.h create mode 100644 third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.cc create mode 100644 third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.h create mode 100644 third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxertypes.h create mode 100644 third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc create mode 100644 third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.h create mode 100644 third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc create mode 100644 third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.h create mode 100644 third_party/aom/third_party/libwebm/mkvparser/mkvparser.cc create mode 100644 third_party/aom/third_party/libwebm/mkvparser/mkvparser.h create mode 100644 third_party/aom/third_party/libwebm/mkvparser/mkvreader.cc create mode 100644 third_party/aom/third_party/libwebm/mkvparser/mkvreader.h create mode 100644 third_party/aom/third_party/libyuv/README.libaom create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/basic_types.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/compare.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/convert.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/convert_argb.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/convert_from.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/convert_from_argb.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/cpu_id.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/mjpeg_decoder.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/planar_functions.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/rotate.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/rotate_argb.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/rotate_row.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/row.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/scale.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/scale_argb.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/scale_row.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/version.h create mode 100644 third_party/aom/third_party/libyuv/include/libyuv/video_common.h create mode 100644 third_party/aom/third_party/libyuv/source/compare.cc create mode 100644 third_party/aom/third_party/libyuv/source/compare_common.cc create mode 100644 third_party/aom/third_party/libyuv/source/compare_gcc.cc create mode 100644 third_party/aom/third_party/libyuv/source/compare_neon.cc create mode 100644 third_party/aom/third_party/libyuv/source/compare_neon64.cc create mode 100644 third_party/aom/third_party/libyuv/source/compare_win.cc create mode 100644 third_party/aom/third_party/libyuv/source/convert.cc create mode 100644 third_party/aom/third_party/libyuv/source/convert_argb.cc create mode 100644 third_party/aom/third_party/libyuv/source/convert_from.cc create mode 100644 third_party/aom/third_party/libyuv/source/convert_from_argb.cc create mode 100644 third_party/aom/third_party/libyuv/source/convert_jpeg.cc create mode 100644 third_party/aom/third_party/libyuv/source/convert_to_argb.cc create mode 100644 third_party/aom/third_party/libyuv/source/convert_to_i420.cc create mode 100644 third_party/aom/third_party/libyuv/source/cpu_id.cc create mode 100644 third_party/aom/third_party/libyuv/source/mjpeg_decoder.cc create mode 100644 third_party/aom/third_party/libyuv/source/mjpeg_validate.cc create mode 100644 third_party/aom/third_party/libyuv/source/planar_functions.cc create mode 100644 third_party/aom/third_party/libyuv/source/rotate.cc create mode 100644 third_party/aom/third_party/libyuv/source/rotate_any.cc create mode 100644 third_party/aom/third_party/libyuv/source/rotate_argb.cc create mode 100644 third_party/aom/third_party/libyuv/source/rotate_common.cc create mode 100644 third_party/aom/third_party/libyuv/source/rotate_gcc.cc create mode 100644 third_party/aom/third_party/libyuv/source/rotate_mips.cc create mode 100644 third_party/aom/third_party/libyuv/source/rotate_neon.cc create mode 100644 third_party/aom/third_party/libyuv/source/rotate_neon64.cc create mode 100644 third_party/aom/third_party/libyuv/source/rotate_win.cc create mode 100644 third_party/aom/third_party/libyuv/source/row_any.cc create mode 100644 third_party/aom/third_party/libyuv/source/row_common.cc create mode 100644 third_party/aom/third_party/libyuv/source/row_gcc.cc create mode 100644 third_party/aom/third_party/libyuv/source/row_mips.cc create mode 100644 third_party/aom/third_party/libyuv/source/row_neon.cc create mode 100644 third_party/aom/third_party/libyuv/source/row_neon64.cc create mode 100644 third_party/aom/third_party/libyuv/source/row_win.cc create mode 100644 third_party/aom/third_party/libyuv/source/row_x86.asm create mode 100644 third_party/aom/third_party/libyuv/source/scale.cc create mode 100644 third_party/aom/third_party/libyuv/source/scale_any.cc create mode 100644 third_party/aom/third_party/libyuv/source/scale_argb.cc create mode 100644 third_party/aom/third_party/libyuv/source/scale_common.cc create mode 100644 third_party/aom/third_party/libyuv/source/scale_gcc.cc create mode 100644 third_party/aom/third_party/libyuv/source/scale_mips.cc create mode 100644 third_party/aom/third_party/libyuv/source/scale_neon.cc create mode 100644 third_party/aom/third_party/libyuv/source/scale_neon64.cc create mode 100644 third_party/aom/third_party/libyuv/source/scale_win.cc create mode 100644 third_party/aom/third_party/libyuv/source/video_common.cc create mode 100644 third_party/aom/third_party/libyuv/source/x86inc.asm create mode 100644 third_party/aom/third_party/x86inc/LICENSE create mode 100644 third_party/aom/third_party/x86inc/README.libaom create mode 100644 third_party/aom/third_party/x86inc/x86inc.asm create mode 100755 third_party/aom/tools/all_builds.py create mode 100755 third_party/aom/tools/author_first_release.sh create mode 100755 third_party/aom/tools/build_inspector.sh create mode 100755 third_party/aom/tools/cpplint.py create mode 100644 third_party/aom/tools/diff.py create mode 100755 third_party/aom/tools/ftfy.sh create mode 100755 third_party/aom/tools/gen_authors.sh create mode 100755 third_party/aom/tools/gen_constrained_tokenset.py create mode 100644 third_party/aom/tools/inspect-cli.js create mode 100644 third_party/aom/tools/inspect-post.js create mode 100755 third_party/aom/tools/intersect-diffs.py create mode 100755 third_party/aom/tools/lint-hunks.py create mode 100755 third_party/aom/tools/wrap-commit-msg.py create mode 100644 third_party/aom/tools_common.c create mode 100644 third_party/aom/tools_common.h create mode 100644 third_party/aom/usage.dox create mode 100644 third_party/aom/usage_cx.dox create mode 100644 third_party/aom/usage_dx.dox create mode 100644 third_party/aom/video_common.h create mode 100644 third_party/aom/video_reader.c create mode 100644 third_party/aom/video_reader.h create mode 100644 third_party/aom/video_writer.c create mode 100644 third_party/aom/video_writer.h create mode 100644 third_party/aom/warnings.c create mode 100644 third_party/aom/warnings.h create mode 100644 third_party/aom/webmdec.cc create mode 100644 third_party/aom/webmdec.h create mode 100644 third_party/aom/webmenc.cc create mode 100644 third_party/aom/webmenc.h create mode 100644 third_party/aom/y4menc.c create mode 100644 third_party/aom/y4menc.h create mode 100644 third_party/aom/y4minput.c create mode 100644 third_party/aom/y4minput.h (limited to 'third_party/aom') diff --git a/third_party/aom/.clang-format b/third_party/aom/.clang-format new file mode 100644 index 0000000000..7837b7704d --- /dev/null +++ b/third_party/aom/.clang-format @@ -0,0 +1,94 @@ +--- +Language: Cpp +# BasedOnStyle: Google +# Generated with clang-format 3.9.1 +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlinesLeft: true +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: true +AllowShortLoopsOnASingleLine: true +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] +IncludeCategories: + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IndentCaseLabels: true +IndentWidth: 2 +IndentWrappedFunctionNames: false +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: false +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Right +ReflowComments: true +SortIncludes: false +SpaceAfterCStyleCast: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Auto +TabWidth: 8 +UseTab: Never +... + diff --git a/third_party/aom/.mailmap b/third_party/aom/.mailmap new file mode 100644 index 0000000000..4672e5ccb8 --- /dev/null +++ b/third_party/aom/.mailmap @@ -0,0 +1,32 @@ +Adrian Grange +Aℓex Converse +Aℓex Converse +Alexis Ballier +Alpha Lam +Deb Mukherjee +Erik Niemeyer +Guillaume Martres +Hangyu Kuang +Hui Su +Jacky Chen +Jim Bankoski +Johann Koenig +Johann Koenig +Johann Koenig +John Koleszar +Joshua Litt +Marco Paniconi +Marco Paniconi +Pascal Massimino +Paul Wilkins +Ralph Giles +Ralph Giles +Ronald S. Bultje +Sami Pietilä +Tamar Levy +Tamar Levy +Tero Rintaluoma +Timothy B. Terriberry Tim Terriberry +Tom Finegan +Tom Finegan +Yaowu Xu diff --git a/third_party/aom/AUTHORS b/third_party/aom/AUTHORS new file mode 100644 index 0000000000..95c3c8bf2b --- /dev/null +++ b/third_party/aom/AUTHORS @@ -0,0 +1,144 @@ +# This file is automatically generated from the git commit history +# by tools/gen_authors.sh. + +Aaron Watry +Abo Talib Mahfoodh +Adam Xu +Adrian Grange +Aℓex Converse +Ahmad Sharif +Alexander Voronov +Alexis Ballier +Alok Ahuja +Alpha Lam +A.Mahfoodh +Ami Fischman +Andoni Morales Alastruey +Andres Mejia +Andrew Russell +Angie Chiang +Aron Rosenberg +Attila Nagy +Brion Vibber +changjun.yang +Charles 'Buck' Krasic +chm +Christian Duvivier +Daniel Kang +Deb Mukherjee +Dim Temp +Dmitry Kovalev +Dragan Mrdjan +Ed Baker +Ehsan Akhgari +Erik Niemeyer +Fabio Pedretti +Frank Galligan +Fredrik Söderquist +Fritz Koenig +Gaute Strokkenes +Geza Lore +Ghislain MARY +Giuseppe Scrivano +Gordana Cmiljanovic +Guillaume Martres +Guillermo Ballester Valor +Hangyu Kuang +Hanno Böck +Henrik Lundin +Hui Su +Ivan Maltz +Jacek Caban +Jacky Chen +James Berry +James Yu +James Zern +Jan Gerber +Jan Kratochvil +Janne Salonen +Jean-Marc Valin +Jeff Faust +Jeff Muizelaar +Jeff Petkau +Jia Jia +Jian Zhou +Jim Bankoski +Jingning Han +Joey Parrish +Johann Koenig +Johann Koenig +John Koleszar +Johnny Klonaris +John Stark +Joshua Bleecher Snyder +Joshua Litt +Julia Robson +Justin Clift +Justin Lebar +KO Myung-Hun +Lawrence Velázquez +Lou Quillio +Luca Barbato +Makoto Kato +Mans Rullgard +Marco Paniconi +Mark Mentovai +Martin Ettl +Martin Storsjo +Matthew Heaney +Michael Kohler +Mike Frysinger +Mike Hommey +Mikhal Shemer +Minghai Shang +Morton Jonuschat +Nathan E. Egge +Nico Weber +Parag Salasakar +Pascal Massimino +Patrik Westin +Paul Wilkins +Pavol Rusnak +Paweł Hajdan +Pengchong Jin +Peter de Rivaz +Peter de Rivaz +Philip Jägenstedt +Priit Laes +Rafael Ávila de Espíndola +Rafaël Carré +Ralph Giles +Rob Bradford +Ronald S. Bultje +Rui Ueyama +Sami Pietilä +Sasi Inguva +Scott Graham +Scott LaVarnway +Sean McGovern +Sergey Kolomenkin +Sergey Ulanov +Shimon Doodkin +Shunyao Li +Stefan Holmer +Steinar Midtskogen +Suman Sunkara +Taekhyun Kim +Takanori MATSUURA +Tamar Levy +Tao Bai +Tero Rintaluoma +Thijs Vermeir +Thomas Daede +Thomas Davies +Thomas +Tim Kopp +Timothy B. Terriberry +Tom Finegan +Tristan Matthews +Tristan Matthews +Vignesh Venkatasubramanian +Yaowu Xu +Yongzhe Wang +Yunqing Wang +Zoe Liu diff --git a/third_party/aom/CHANGELOG b/third_party/aom/CHANGELOG new file mode 100644 index 0000000000..dad0ea1d29 --- /dev/null +++ b/third_party/aom/CHANGELOG @@ -0,0 +1,630 @@ +Next Release + - Incompatible changes: + The AV1 encoder's default keyframe interval changed to 128 from 9999. + +2016-04-07 v0.1.0 "AOMedia Codec 1" + This release is the first Alliance for Open Media codec. +2015-11-09 v1.5.0 "Javan Whistling Duck" + This release improves upon the VP9 encoder and speeds up the encoding and + decoding processes. + + - Upgrading: + This release is ABI incompatible with 1.4.0. It drops deprecated VP8 + controls and adds a variety of VP9 controls for testing. + + The vpxenc utility now prefers VP9 by default. + + - Enhancements: + Faster VP9 encoding and decoding + Smaller library size by combining functions used by VP8 and VP9 + + - Bug Fixes: + A variety of fuzzing issues + +2015-04-03 v1.4.0 "Indian Runner Duck" + This release includes significant improvements to the VP9 codec. + + - Upgrading: + This release is ABI incompatible with 1.3.0. It drops the compatibility + layer, requiring VPX_IMG_FMT_* instead of IMG_FMT_*, and adds several codec + controls for VP9. + + - Enhancements: + Faster VP9 encoding and decoding + Multithreaded VP9 decoding (tile and frame-based) + Multithreaded VP9 encoding - on by default + YUV 4:2:2 and 4:4:4 support in VP9 + 10 and 12bit support in VP9 + 64bit ARM support by replacing ARM assembly with intrinsics + + - Bug Fixes: + Fixes a VP9 bitstream issue in Profile 1. This only affected non-YUV 4:2:0 + files. + + - Known Issues: + Frame Parallel decoding fails for segmented and non-420 files. + +2013-11-15 v1.3.0 "Forest" + This release introduces the VP9 codec in a backward-compatible way. + All existing users of VP8 can continue to use the library without + modification. However, some VP8 options do not map to VP9 in the same manner. + + The VP9 encoder in this release is not feature complete. Users interested in + the encoder are advised to use the git master branch and discuss issues on + libvpx mailing lists. + + - Upgrading: + This release is ABI and API compatible with Duclair (v1.0.0). Users + of older releases should refer to the Upgrading notes in this document + for that release. + + - Enhancements: + Get rid of bashisms in the main build scripts + Added usage info on command line options + Add lossless compression mode + Dll build of libvpx + Add additional Mac OS X targets: 10.7, 10.8 and 10.9 (darwin11-13) + Add option to disable documentation + configure: add --enable-external-build support + make: support V=1 as short form of verbose=yes + configure: support mingw-w64 + configure: support hardfloat armv7 CHOSTS + configure: add support for android x86 + Add estimated completion time to vpxenc + Don't exit on decode errors in vpxenc + vpxenc: support scaling prior to encoding + vpxdec: support scaling output + vpxenc: improve progress indicators with --skip + msvs: Don't link to winmm.lib + Add a new script for producing vcxproj files + Produce Visual Studio 10 and 11 project files + Produce Windows Phone project files + msvs-build: use msbuild for vs >= 2005 + configure: default configure log to config.log + Add encoding option --static-thresh + + - Speed: + Miscellaneous speed optimizations for VP8 and VP9. + + - Quality: + In general, quality is consistent with the Eider release. + + - Bug Fixes: + This release represents approximately a year of engineering effort, + and contains multiple bug fixes. Please refer to git history for details. + + +2012-12-21 v1.2.0 + This release acts as a checkpoint for a large amount of internal refactoring + and testing. It also contains a number of small bugfixes, so all users are + encouraged to upgrade. + + - Upgrading: + This release is ABI and API compatible with Duclair (v1.0.0). Users + of older releases should refer to the Upgrading notes in this + document for that release. + + - Enhancements: + VP8 optimizations for MIPS dspr2 + vpxenc: add -quiet option + + - Speed: + Encoder and decoder speed is consistent with the Eider release. + + - Quality: + In general, quality is consistent with the Eider release. + + Minor tweaks to ARNR filtering + Minor improvements to real time encoding with multiple temporal layers + + - Bug Fixes: + Fixes multithreaded encoder race condition in loopfilter + Fixes multi-resolution threaded encoding + Fix potential encoder dead-lock after picture resize + + +2012-05-09 v1.1.0 "Eider" + This introduces a number of enhancements, mostly focused on real-time + encoding. In addition, it fixes a decoder bug (first introduced in + Duclair) so all users of that release are encouraged to upgrade. + + - Upgrading: + This release is ABI and API compatible with Duclair (v1.0.0). Users + of older releases should refer to the Upgrading notes in this + document for that release. + + This release introduces a new temporal denoiser, controlled by the + VP8E_SET_NOISE_SENSITIVITY control. The temporal denoiser does not + currently take a strength parameter, so the control is effectively + a boolean - zero (off) or non-zero (on). For compatibility with + existing applications, the values accepted are the same as those + for the spatial denoiser (0-6). The temporal denoiser is enabled + by default, and the older spatial denoiser may be restored by + configuring with --disable-temporal-denoising. The temporal denoiser + is more computationally intensive than the spatial one. + + This release removes support for a legacy, decode only API that was + supported, but deprecated, at the initial release of libvpx + (v0.9.0). This is not expected to have any impact. If you are + impacted, you can apply a reversion to commit 2bf8fb58 locally. + Please update to the latest libvpx API if you are affected. + + - Enhancements: + Adds a motion compensated temporal denoiser to the encoder, which + gives higher quality than the older spatial denoiser. (See above + for notes on upgrading). + + In addition, support for new compilers and platforms were added, + including: + improved support for XCode + Android x86 NDK build + OS/2 support + SunCC support + + Changing resolution with vpx_codec_enc_config_set() is now + supported. Previously, reinitializing the codec was required to + change the input resolution. + + The vpxenc application has initial support for producing multiple + encodes from the same input in one call. Resizing is not yet + supported, but varying other codec parameters is. Use -- to + delineate output streams. Options persist from one stream to the + next. + + Also, the vpxenc application will now use a keyframe interval of + 5 seconds by default. Use the --kf-max-dist option to override. + + - Speed: + Decoder performance improved 2.5% versus Duclair. Encoder speed is + consistent with Duclair for most material. Two pass encoding of + slideshow-like material will see significant improvements. + + Large realtime encoding speed gains at a small quality expense are + possible by configuring the on-the-fly bitpacking experiment with + --enable-onthefly-bitpacking. Realtime encoder can be up to 13% + faster (ARM) depending on the number of threads and bitrate + settings. This technique sees constant gain over the 5-16 speed + range. For VC style input the loss seen is up to 0.2dB. See commit + 52cf4dca for further details. + + - Quality: + On the whole, quality is consistent with the Duclair release. Some + tweaks: + + Reduced blockiness in easy sections by applying a penalty to + intra modes. + + Improved quality of static sections (like slideshows) with + two pass encoding. + + Improved keyframe sizing with multiple temporal layers + + - Bug Fixes: + Corrected alt-ref contribution to frame rate for visible updates + to the alt-ref buffer. This affected applications making manual + usage of the frame reference flags, or temporal layers. + + Additional constraints were added to disable multi-frame quality + enhancement (MFQE) in sections of the frame where there is motion. + (#392) + + Fixed corruption issues when vpx_codec_enc_config_set() was called + with spatial resampling enabled. + + Fixed a decoder error introduced in Duclair where the segmentation + map was not being reinitialized on keyframes (#378) + + +2012-01-27 v1.0.0 "Duclair" + Our fourth named release, focused on performance and features related to + real-time encoding. It also fixes a decoder crash bug introduced in + v0.9.7, so all users of that release are encouraged to upgrade. + + - Upgrading: + This release is ABI incompatible with prior releases of libvpx, so the + "major" version number has been bumped to 1. You must recompile your + applications against the latest version of the libvpx headers. The + API remains compatible, and this should not require code changes in most + applications. + + - Enhancements: + This release introduces several substantial new features to the encoder, + of particular interest to real time streaming applications. + + Temporal scalability allows the encoder to produce a stream that can + be decimated to different frame rates, with independent rate targetting + for each substream. + + Multiframe quality enhancement postprocessing can make visual quality + more consistent in the presence of frames that are substantially + different quality than the surrounding frames, as in the temporal + scalability case and in some forced keyframe scenarios. + + Multiple-resolution encoding support allows the encoding of the + same content at different resolutions faster than encoding them + separately. + + - Speed: + Optimization targets for this release included the decoder and the real- + time modes of the encoder. Decoder speed on x86 has improved 10.5% with + this release. Encoder improvements followed a curve where speeds 1-3 + improved 4.0%-1.5%, speeds 4-8 improved <1%, and speeds 9-16 improved + 1.5% to 10.5%, respectively. "Best" mode speed is consistent with the + Cayuga release. + + - Quality: + Encoder quality in the single stream case is consistent with the Cayuga + release. + + - Bug Fixes: + This release fixes an OOB read decoder crash bug present in v0.9.7 + related to the clamping of motion vectors in SPLITMV blocks. This + behavior could be triggered by corrupt input or by starting + decoding from a P-frame. + + +2011-08-15 v0.9.7-p1 "Cayuga" patch 1 + This is an incremental bugfix release against Cayuga. All users of that + release are strongly encouraged to upgrade. + + - Fix potential OOB reads (cdae03a) + + An unbounded out of bounds read was discovered when the + decoder was requested to perform error concealment (new in + Cayuga) given a frame with corrupt partition sizes. + + A bounded out of bounds read was discovered affecting all + versions of libvpx. Given an multipartition input frame that + is truncated between the mode/mv partition and the first + residiual paritition (in the block of partition offsets), up + to 3 extra bytes could have been read from the source buffer. + The code will not take any action regardless of the contents + of these undefined bytes, as the truncated buffer is detected + immediately following the read based on the calculated + starting position of the coefficient partition. + + - Fix potential error concealment crash when the very first frame + is missing or corrupt (a609be5) + + - Fix significant artifacts in error concealment (a4c2211, 99d870a) + + - Revert 1-pass CBR rate control changes (e961317) + Further testing showed this change produced undesirable visual + artifacts, rolling back for now. + + +2011-08-02 v0.9.7 "Cayuga" + Our third named release, focused on a faster, higher quality, encoder. + + - Upgrading: + This release is backwards compatible with Aylesbury (v0.9.5) and + Bali (v0.9.6). Users of older releases should refer to the Upgrading + notes in this document for that release. + + - Enhancements: + Stereo 3D format support for vpxenc + Runtime detection of available processor cores. + Allow specifying --end-usage by enum name + vpxdec: test for frame corruption + vpxenc: add quantizer histogram display + vpxenc: add rate histogram display + Set VPX_FRAME_IS_DROPPABLE + update configure for ios sdk 4.3 + Avoid text relocations in ARM vp8 decoder + Generate a vpx.pc file for pkg-config. + New ways of passing encoded data between encoder and decoder. + + - Speed: + This release includes across-the-board speed improvements to the + encoder. On x86, these measure at approximately 11.5% in Best mode, + 21.5% in Good mode (speed 0), and 22.5% in Realtime mode (speed 6). + On ARM Cortex A9 with Neon extensions, real-time encoding of video + telephony content is 35% faster than Bali on single core and 48% + faster on multi-core. On the NVidia Tegra2 platform, real time + encoding is 40% faster than Bali. + + Decoder speed was not a priority for this release, but improved + approximately 8.4% on x86. + + Reduce motion vector search on alt-ref frame. + Encoder loopfilter running in its own thread + Reworked loopfilter to precalculate more parameters + SSE2/SSSE3 optimizations for build_predictors_mbuv{,_s}(). + Make hor UV predict ~2x faster (73 vs 132 cycles) using SSSE3. + Removed redundant checks + Reduced structure sizes + utilize preload in ARMv6 MC/LPF/Copy routines + ARM optimized quantization, dfct, variance, subtract + Increase chrow row alignment to 16 bytes. + disable trellis optimization for first pass + Write SSSE3 sub-pixel filter function + Improve SSE2 half-pixel filter funtions + Add vp8_sub_pixel_variance16x8_ssse3 function + Reduce unnecessary distortion computation + Use diamond search to replace full search + Preload reference area in sub-pixel motion search (real-time mode) + + - Quality: + This release focused primarily on one-pass use cases, including + video conferencing. Low latency data rate control was significantly + improved, improving streamability over bandwidth constrained links. + Added support for error concealment, allowing frames to maintain + visual quality in the presence of substantial packet loss. + + Add rc_max_intra_bitrate_pct control + Limit size of initial keyframe in one-pass. + Improve framerate adaptation + Improved 1-pass CBR rate control + Improved KF insertion after fades to still. + Improved key frame detection. + Improved activity masking (lower PSNR impact for same SSIM boost) + Improved interaction between GF and ARFs + Adding error-concealment to the decoder. + Adding support for independent partitions + Adjusted rate-distortion constants + + + - Bug Fixes: + Removed firstpass motion map + Fix parallel make install + Fix multithreaded encoding for 1 MB wide frame + Fixed iwalsh_neon build problems with RVDS4.1 + Fix semaphore emulation, spin-wait intrinsics on Windows + Fix build with xcode4 and simplify GLOBAL. + Mark ARM asm objects as allowing a non-executable stack. + Fix vpxenc encoding incorrect webm file header on big endian + + +2011-03-07 v0.9.6 "Bali" + Our second named release, focused on a faster, higher quality, encoder. + + - Upgrading: + This release is backwards compatible with Aylesbury (v0.9.5). Users + of older releases should refer to the Upgrading notes in this + document for that release. + + - Enhancements: + vpxenc --psnr shows a summary when encode completes + --tune=ssim option to enable activity masking + improved postproc visualizations for development + updated support for Apple iOS to SDK 4.2 + query decoder to determine which reference frames were updated + implemented error tracking in the decoder + fix pipe support on windows + + - Speed: + Primary focus was on good quality mode, speed 0. Average improvement + on x86 about 40%, up to 100% on user-generated content at that speed. + Best quality mode speed improved 35%, and realtime speed 10-20%. This + release also saw significant improvement in realtime encoding speed + on ARM platforms. + + Improved encoder threading + Dont pick encoder filter level when loopfilter is disabled. + Avoid double copying of key frames into alt and golden buffer + FDCT optimizations. + x86 sse2 temporal filter + SSSE3 version of fast quantizer + vp8_rd_pick_best_mbsegmentation code restructure + Adjusted breakout RD for SPLITMV + Changed segmentation check order + Improved rd_pick_intra4x4block + Adds armv6 optimized variance calculation + ARMv6 optimized sad16x16 + ARMv6 optimized half pixel variance calculations + Full search SAD function optimization in SSE4.1 + Improve MV prediction accuracy to achieve performance gain + Improve MV prediction in vp8_pick_inter_mode() for speed>3 + + - Quality: + Best quality mode improved PSNR 6.3%, and SSIM 6.1%. This release + also includes support for "activity masking," which greatly improves + SSIM at the expense of PSNR. For now, this feature is available with + the --tune=ssim option. Further experimentation in this area + is ongoing. This release also introduces a new rate control mode + called "CQ," which changes the allocation of bits within a clip to + the sections where they will have the most visual impact. + + Tuning for the more exact quantizer. + Relax rate control for last few frames + CQ Mode + Limit key frame quantizer for forced key frames. + KF/GF Pulsing + Add simple version of activity masking. + make rdmult adaptive for intra in quantizer RDO + cap the best quantizer for 2nd order DC + change the threshold of DC check for encode breakout + + - Bug Fixes: + Fix crash on Sparc Solaris. + Fix counter of fixed keyframe distance + ARNR filter pointer update bug fix + Fixed use of motion percentage in KF/GF group calc + Changed condition for using RD in Intra Mode + Fix encoder real-time only configuration. + Fix ARM encoder crash with multiple token partitions + Fixed bug first cluster timecode of webm file is wrong. + Fixed various encoder bugs with odd-sized images + vp8e_get_preview fixed when spatial resampling enabled + quantizer: fix assertion in fast quantizer path + Allocate source buffers to be multiples of 16 + Fix for manual Golden frame frequency + Fix drastic undershoot in long form content + + +2010-10-28 v0.9.5 "Aylesbury" + Our first named release, focused on a faster decoder, and a better encoder. + + - Upgrading: + This release incorporates backwards-incompatible changes to the + ivfenc and ivfdec tools. These tools are now called vpxenc and vpxdec. + + vpxdec + * the -q (quiet) option has been removed, and replaced with + -v (verbose). the output is quiet by default. Use -v to see + the version number of the binary. + + * The default behavior is now to write output to a single file + instead of individual frames. The -y option has been removed. + Y4M output is the default. + + * For raw I420/YV12 output instead of Y4M, the --i420 or --yv12 + options must be specified. + + $ ivfdec -o OUTPUT INPUT + $ vpxdec --i420 -o OUTPUT INPUT + + * If an output file is not specified, the default is to write + Y4M to stdout. This makes piping more natural. + + $ ivfdec -y -o - INPUT | ... + $ vpxdec INPUT | ... + + * The output file has additional flexibility for formatting the + filename. It supports escape characters for constructing a + filename from the width, height, and sequence number. This + replaces the -p option. To get the equivalent: + + $ ivfdec -p frame INPUT + $ vpxdec --i420 -o frame-%wx%h-%4.i420 INPUT + + vpxenc + * The output file must be specified with -o, rather than as the + last argument. + + $ ivfenc INPUT OUTPUT + $ vpxenc -o OUTPUT INPUT + + * The output defaults to webm. To get IVF output, use the --ivf + option. + + $ ivfenc INPUT OUTPUT.ivf + $ vpxenc -o OUTPUT.ivf --ivf INPUT + + + - Enhancements: + ivfenc and ivfdec have been renamed to vpxenc, vpxdec. + vpxdec supports .webm input + vpxdec writes .y4m by default + vpxenc writes .webm output by default + vpxenc --psnr now shows the average/overall PSNR at the end + ARM platforms now support runtime cpu detection + vpxdec visualizations added for motion vectors, block modes, references + vpxdec now silent by default + vpxdec --progress shows frame-by-frame timing information + vpxenc supports the distinction between --fps and --timebase + NASM is now a supported assembler + configure: enable PIC for shared libs by default + configure: add --enable-small + configure: support for ppc32-linux-gcc + configure: support for sparc-solaris-gcc + + - Bugs: + Improve handling of invalid frames + Fix valgrind errors in the NEON loop filters. + Fix loopfilter delta zero transitions + Fix valgrind errors in vp8_sixtap_predict8x4_armv6(). + Build fixes for darwin-icc + + - Speed: + 20-40% (average 28%) improvement in libvpx decoder speed, + including: + Rewrite vp8_short_walsh4x4_sse2() + Optimizations on the loopfilters. + Miscellaneous improvements for Atom + Add 4-tap version of 2nd-pass ARMv6 MC filter. + Improved multithread utilization + Better instruction choices on x86 + reorder data to use wider instructions + Update NEON wide idcts + Make block access to frame buffer sequential + Improved subset block search + Bilinear subpixel optimizations for ssse3. + Decrease memory footprint + + Encoder speed improvements (percentage gain not measured): + Skip unnecessary search of identical frames + Add SSE2 subtract functions + Improve bounds checking in vp8_diamond_search_sadx4() + Added vp8_fast_quantize_b_sse2 + + - Quality: + Over 7% overall PSNR improvement (6.3% SSIM) in "best" quality + encoding mode, and up to 60% improvement on very noisy, still + or slow moving source video + + Motion compensated temporal filter for Alt-Ref Noise Reduction + Improved use of trellis quantization on 2nd order Y blocks + Tune effect of motion on KF/GF boost in two pass + Allow coefficient optimization for good quality speed 0. + Improved control of active min quantizer for two pass. + Enable ARFs for non-lagged compress + +2010-09-02 v0.9.2 + - Enhancements: + Disable frame dropping by default + Improved multithreaded performance + Improved Force Key Frame Behaviour + Increased rate control buffer level precision + Fix bug in 1st pass motion compensation + ivfenc: correct fixed kf interval, --disable-kf + - Speed: + Changed above and left context data layout + Rework idct calling structure. + Removed unnecessary MB_MODE_INFO copies + x86: SSSE3 sixtap prediction + Reworked IDCT to include reconstruction (add) step + Swap alt/gold/new/last frame buffer ptrs instead of copying. + Improve SSE2 loopfilter functions + Change bitreader to use a larger window. + Avoid loopfilter reinitialization when possible + - Quality: + Normalize quantizer's zero bin and rounding factors + Add trellis quantization. + Make the quantizer exact. + Updates to ARNR filtering algorithm + Fix breakout thresh computation for golden & AltRef frames + Redo the forward 4x4 dct + Improve the accuracy of forward walsh-hadamard transform + Further adjustment of RD behaviour with Q and Zbin. + - Build System: + Allow linking of libs built with MinGW to MSVC + Fix target auto-detection on mingw32 + Allow --cpu= to work for x86. + configure: pass original arguments through to make dist + Fix builds without runtime CPU detection + msvs: fix install of codec sources + msvs: Change devenv.com command line for better msys support + msvs: Add vs9 targets. + Add x86_64-linux-icc target + - Bugs: + Potential crashes on older MinGW builds + Fix two-pass framrate for Y4M input. + Fixed simple loop filter, other crashes on ARM v6 + arm: fix missing dependency with --enable-shared + configure: support directories containing .o + Replace pinsrw (SSE) with MMX instructions + apple: include proper mach primatives + Fixed rate control bug with long key frame interval. + Fix DSO link errors on x86-64 when not using a version script + Fixed buffer selection for UV in AltRef filtering + + +2010-06-17 v0.9.1 + - Enhancements: + * ivfenc/ivfdec now support YUV4MPEG2 input and pipe I/O + * Speed optimizations + - Bugfixes: + * Rate control + * Prevent out-of-bounds accesses on invalid data + - Build system updates: + * Detect toolchain to be used automatically for native builds + * Support building shared libraries + * Better autotools emulation (--prefix, --libdir, DESTDIR) + - Updated LICENSE + * http://webmproject.blogspot.com/2010/06/changes-to-webm-open-source-license.html + + +2010-05-18 v0.9.0 + - Initial open source release. Welcome to WebM and VP8! + diff --git a/third_party/aom/CMakeLists.txt b/third_party/aom/CMakeLists.txt new file mode 100644 index 0000000000..4ba2d40aaf --- /dev/null +++ b/third_party/aom/CMakeLists.txt @@ -0,0 +1,359 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +cmake_minimum_required(VERSION 3.5) +project(AOM C CXX) + +set(AOM_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") +set(AOM_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}") + +if ("${AOM_ROOT}" STREQUAL "${AOM_CONFIG_DIR}") + message(FATAL_ERROR + "Building from within the aom source tree is not supported.\n" + "Hint: mkdir -p ../aom_build && cd ../aom_build\n" + "Run cmake from there.") +endif () + +include("${AOM_ROOT}/build/cmake/aom_configure.cmake") +include("${AOM_ROOT}/build/cmake/aom_optimization.cmake") +include("${AOM_ROOT}/aom_dsp/aom_dsp.cmake") +include("${AOM_ROOT}/aom_mem/aom_mem.cmake") +include("${AOM_ROOT}/aom_ports/aom_ports.cmake") +include("${AOM_ROOT}/aom_scale/aom_scale.cmake") +include("${AOM_ROOT}/aom_util/aom_util.cmake") +include("${AOM_ROOT}/av1/av1.cmake") +include("${AOM_ROOT}/test/test.cmake") + +set(AOM_RTCD_SOURCES + "${AOM_CONFIG_DIR}/aom_dsp_rtcd.h" + "${AOM_CONFIG_DIR}/aom_scale_rtcd.h" + "${AOM_CONFIG_DIR}/av1_rtcd.h" + "${AOM_ROOT}/aom_dsp/aom_dsp_rtcd_defs.pl" + "${AOM_ROOT}/aom_dsp/aom_dsp_rtcd.c" + "${AOM_ROOT}/aom_scale/aom_scale_rtcd.pl" + "${AOM_ROOT}/aom_scale/aom_scale_rtcd.c" + "${AOM_ROOT}/av1/common/av1_rtcd_defs.pl" + "${AOM_ROOT}/av1/common/av1_rtcd.c" + "${AOM_ROOT}/build/make/rtcd.pl") + +# TODO(tomfinegan): Use libwebm's cmake support directly. +set(AOM_LIBWEBM_SOURCES + "${AOM_ROOT}/third_party/libwebm/common/hdr_util.cc" + "${AOM_ROOT}/third_party/libwebm/common/hdr_util.h" + "${AOM_ROOT}/third_party/libwebm/common/webmids.h" + "${AOM_ROOT}/third_party/libwebm/mkvmuxer/mkvmuxer.cc" + "${AOM_ROOT}/third_party/libwebm/mkvmuxer/mkvmuxer.h" + "${AOM_ROOT}/third_party/libwebm/mkvmuxer/mkvmuxertypes.h" + "${AOM_ROOT}/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc" + "${AOM_ROOT}/third_party/libwebm/mkvmuxer/mkvmuxerutil.h" + "${AOM_ROOT}/third_party/libwebm/mkvmuxer/mkvwriter.cc" + "${AOM_ROOT}/third_party/libwebm/mkvmuxer/mkvwriter.h" + "${AOM_ROOT}/third_party/libwebm/mkvparser/mkvparser.cc" + "${AOM_ROOT}/third_party/libwebm/mkvparser/mkvparser.h" + "${AOM_ROOT}/third_party/libwebm/mkvparser/mkvreader.cc" + "${AOM_ROOT}/third_party/libwebm/mkvparser/mkvreader.h") + +set(AOM_LIBYUV_SOURCES + "${AOM_ROOT}/third_party/libyuv/include/libyuv/basic_types.h" + "${AOM_ROOT}/third_party/libyuv/include/libyuv/convert.h" + "${AOM_ROOT}/third_party/libyuv/include/libyuv/convert_argb.h" + "${AOM_ROOT}/third_party/libyuv/include/libyuv/convert_from.h" + "${AOM_ROOT}/third_party/libyuv/include/libyuv/cpu_id.h" + "${AOM_ROOT}/third_party/libyuv/include/libyuv/planar_functions.h" + "${AOM_ROOT}/third_party/libyuv/include/libyuv/rotate.h" + "${AOM_ROOT}/third_party/libyuv/include/libyuv/row.h" + "${AOM_ROOT}/third_party/libyuv/include/libyuv/scale.h" + "${AOM_ROOT}/third_party/libyuv/include/libyuv/scale_row.h" + "${AOM_ROOT}/third_party/libyuv/source/cpu_id.cc" + "${AOM_ROOT}/third_party/libyuv/source/planar_functions.cc" + "${AOM_ROOT}/third_party/libyuv/source/row_any.cc" + "${AOM_ROOT}/third_party/libyuv/source/row_common.cc" + "${AOM_ROOT}/third_party/libyuv/source/row_gcc.cc" + "${AOM_ROOT}/third_party/libyuv/source/row_mips.cc" + "${AOM_ROOT}/third_party/libyuv/source/row_neon.cc" + "${AOM_ROOT}/third_party/libyuv/source/row_neon64.cc" + "${AOM_ROOT}/third_party/libyuv/source/row_win.cc" + "${AOM_ROOT}/third_party/libyuv/source/scale.cc" + "${AOM_ROOT}/third_party/libyuv/source/scale_any.cc" + "${AOM_ROOT}/third_party/libyuv/source/scale_common.cc" + "${AOM_ROOT}/third_party/libyuv/source/scale_gcc.cc" + "${AOM_ROOT}/third_party/libyuv/source/scale_mips.cc" + "${AOM_ROOT}/third_party/libyuv/source/scale_neon.cc" + "${AOM_ROOT}/third_party/libyuv/source/scale_neon64.cc" + "${AOM_ROOT}/third_party/libyuv/source/scale_win.cc") + +set(AOM_SOURCES + "${AOM_CONFIG_DIR}/aom_config.c" + "${AOM_CONFIG_DIR}/aom_config.h" + "${AOM_ROOT}/aom/aom.h" + "${AOM_ROOT}/aom/aom_codec.h" + "${AOM_ROOT}/aom/aom_decoder.h" + "${AOM_ROOT}/aom/aom_encoder.h" + "${AOM_ROOT}/aom/aom_frame_buffer.h" + "${AOM_ROOT}/aom/aom_image.h" + "${AOM_ROOT}/aom/aom_integer.h" + "${AOM_ROOT}/aom/aomcx.h" + "${AOM_ROOT}/aom/aomdx.h" + "${AOM_ROOT}/aom/internal/aom_codec_internal.h" + "${AOM_ROOT}/aom/src/aom_codec.c" + "${AOM_ROOT}/aom/src/aom_decoder.c" + "${AOM_ROOT}/aom/src/aom_encoder.c" + "${AOM_ROOT}/aom/src/aom_image.c") + +set(AOM_COMMON_APP_UTIL_SOURCES + "${AOM_ROOT}/args.c" + "${AOM_ROOT}/args.h" + "${AOM_ROOT}/md5_utils.c" + "${AOM_ROOT}/md5_utils.h" + "${AOM_ROOT}/tools_common.c" + "${AOM_ROOT}/tools_common.h" + "${AOM_ROOT}/video_common.h") + +set(AOM_DECODER_APP_UTIL_SOURCES + "${AOM_ROOT}/ivfdec.c" + "${AOM_ROOT}/ivfdec.h" + "${AOM_ROOT}/video_reader.c" + "${AOM_ROOT}/video_reader.h" + "${AOM_ROOT}/y4menc.c" + "${AOM_ROOT}/y4menc.h") + +set(AOM_ENCODER_APP_UTIL_SOURCES + "${AOM_ROOT}/ivfenc.c" + "${AOM_ROOT}/ivfenc.h" + "${AOM_ROOT}/video_writer.c" + "${AOM_ROOT}/video_writer.h" + "${AOM_ROOT}/warnings.c" + "${AOM_ROOT}/warnings.h" + "${AOM_ROOT}/y4minput.c" + "${AOM_ROOT}/y4minput.h" + "${AOM_ROOT}/examples/encoder_util.h" + "${AOM_ROOT}/examples/encoder_util.c") + +set(AOM_ENCODER_STATS_SOURCES + "${AOM_ROOT}/aomstats.c" + "${AOM_ROOT}/aomstats.h" + "${AOM_ROOT}/rate_hist.c" + "${AOM_ROOT}/rate_hist.h") + +set(AOM_WEBM_DECODER_SOURCES + "${AOM_ROOT}/webmdec.cc" + "${AOM_ROOT}/webmdec.h") + +set(AOM_WEBM_ENCODER_SOURCES + "${AOM_ROOT}/webmenc.cc" + "${AOM_ROOT}/webmenc.h") + +include_directories(${AOM_ROOT} ${AOM_CONFIG_DIR}) + +# Targets +# TODO(tomfinegan): Move rtcd target setup where it belongs for each rtcd +# source. +add_rtcd_build_step("${AOM_ROOT}/aom_dsp/aom_dsp_rtcd_defs.pl" + "${AOM_CONFIG_DIR}/aom_dsp_rtcd.h" + "${AOM_ROOT}/aom_dsp/aom_dsp_rtcd.c" + "aom_dsp_rtcd") +add_rtcd_build_step("${AOM_ROOT}/aom_scale/aom_scale_rtcd.pl" + "${AOM_CONFIG_DIR}/aom_scale_rtcd.h" + "${AOM_ROOT}/aom_scale/aom_scale_rtcd.c" + "aom_scale_rtcd") +add_rtcd_build_step("${AOM_ROOT}/av1/common/av1_rtcd_defs.pl" + "${AOM_CONFIG_DIR}/av1_rtcd.h" + "${AOM_ROOT}/av1/common/av1_rtcd.c" + "av1_rtcd") + +add_library(aom_rtcd OBJECT ${AOM_RTCD_SOURCES}) +add_library(aom_encoder_stats OBJECT ${AOM_ENCODER_STATS_SOURCES}) +add_library(aom ${AOM_SOURCES} $) + +# List of object and static library targets. +set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_rtcd aom_encoder_stats aom_mem + aom_scale aom) + +# Setup dependencies. +setup_aom_dsp_targets() +setup_aom_mem_targets() +setup_aom_ports_targets() +setup_aom_util_targets() +setup_aom_scale_targets() +setup_av1_targets() + +# Make all library targets depend on aom_rtcd to make sure it builds first. +foreach (aom_lib ${AOM_LIB_TARGETS}) + if (NOT "${aom_lib}" STREQUAL "aom_rtcd") + add_dependencies(${aom_lib} aom_rtcd) + endif () +endforeach () +# +# Application and application support targets. +# +add_library(aom_common_app_util OBJECT ${AOM_COMMON_APP_UTIL_SOURCES}) +add_library(aom_decoder_app_util OBJECT ${AOM_DECODER_APP_UTIL_SOURCES}) +add_library(aom_encoder_app_util OBJECT ${AOM_ENCODER_APP_UTIL_SOURCES}) + +add_executable(aom_cx_set_ref + "${AOM_ROOT}/examples/aom_cx_set_ref.c" + $ + $) + +add_executable(aomdec + "${AOM_ROOT}/aomdec.c" + $ + $) + +add_executable(aomenc + "${AOM_ROOT}/aomenc.c" + $ + $ + $) + +if (CONFIG_ANALYZER) + add_executable(analyzer + "${AOM_ROOT}/examples/analyzer.cc" + $ + $) + target_link_libraries(analyzer PUBLIC ${wxWidgets_LIBRARIES}) + set(AOM_APP_TARGETS ${AOM_APP_TARGETS} analyzer) +endif () + +add_executable(decode_to_md5 + "${AOM_ROOT}/examples/decode_to_md5.c" + $ + $) + +add_executable(decode_with_drops + "${AOM_ROOT}/examples/decode_with_drops.c" + $ + $) + +add_executable(lossless_encoder + "${AOM_ROOT}/examples/lossless_encoder.c" + $ + $) + +add_executable(set_maps + "${AOM_ROOT}/examples/set_maps.c" + $ + $) + +add_executable(simple_decoder + "${AOM_ROOT}/examples/simple_decoder.c" + $ + $) + +if (CONFIG_INSPECTION) +add_executable(inspect + "${AOM_ROOT}/examples/inspect.c" + $ + $) +set(AOM_APP_TARGETS ${AOM_APP_TARGETS} inspect) +endif () + +add_executable(simple_encoder + "${AOM_ROOT}/examples/simple_encoder.c" + $ + $) + +add_executable(twopass_encoder + "${AOM_ROOT}/examples/twopass_encoder.c" + $ + $) + +# List of app targets. +set(AOM_APP_TARGETS ${AOM_APP_TARGETS} aom_cx_set_ref aomdec aomenc + decode_to_md5 decode_with_drops lossless_encoder set_maps simple_decoder + simple_encoder twopass_encoder) + +foreach (aom_app ${AOM_APP_TARGETS}) + target_link_libraries(${aom_app} PUBLIC aom) +endforeach () + +if (CONFIG_LIBYUV) + add_library(yuv OBJECT ${AOM_LIBYUV_SOURCES}) + if (NOT MSVC) + target_compile_options(yuv PRIVATE -Wno-unused-parameter) + endif () + include_directories("${AOM_ROOT}/third_party/libyuv/include") + + # Add to existing targets. + foreach (aom_app ${AOM_APP_TARGETS}) + target_sources(${aom_app} PUBLIC $) + set_property(TARGET ${aom_app} PROPERTY LINKER_LANGUAGE CXX) + endforeach () +endif () + +if (CONFIG_WEBM_IO) + add_library(webm OBJECT ${AOM_LIBWEBM_SOURCES}) + include_directories("${AOM_ROOT}/third_party/libwebm") + + if (NOT MSVC) + target_compile_options(webm PRIVATE -Wno-shadow) + endif () + + # Add to existing targets. + target_sources(aom_decoder_app_util PUBLIC ${AOM_WEBM_DECODER_SOURCES}) + target_sources(aom_encoder_app_util PUBLIC ${AOM_WEBM_ENCODER_SOURCES}) + + foreach (aom_app ${AOM_APP_TARGETS}) + target_sources(${aom_app} PUBLIC $) + set_property(TARGET ${aom_app} PROPERTY LINKER_LANGUAGE CXX) + endforeach () +endif () + +if (CONFIG_UNIT_TESTS) + # Create test_libaom target and the targets it depends on. + setup_aom_test_targets() +endif () + +if (HAVE_PTHREAD_H AND CONFIG_MULTITHREAD) + find_package(Threads) + foreach (app_target ${AOM_APP_TARGETS}) + target_link_libraries(${app_target} PUBLIC Threads::Threads) + endforeach () +endif () + +if (XCODE) + if (CONFIG_LIBYUV OR CONFIG_WEBM_IO) + # The Xcode generator does not obey LINKER_LANGUAGE. Because of the issue + # what looks like a C++ file needs to be in any target that Xcode will link + # when the target contains a C++ dependency. + # Without this Xcode will try to link with the C linker, which always ends + # badly when a dependency actually includes C++. + # Note: LINKER_LANGUAGE is explicitly set to C++ for all targets touched + # here, it really is the Xcode generator's fault, or just a deficiency in + # Xcode itself. + set(XCODE_DUMMY_CXX_FILE "${AOM_CONFIG_DIR}/dummy.cc") + file(WRITE "${XCODE_DUMMY_CXX_FILE}" + "// Xcode needs a C++ file to link, ignore this file.") + foreach (aom_app ${AOM_APP_TARGETS}) + target_sources(${aom_app} PUBLIC "${XCODE_DUMMY_CXX_FILE}") + endforeach () + endif () +endif () + +# Aomedia install rule. +# TODO(tomfinegan): Add the missing pkg-config related bits. +set(AOM_INSTALL_INCS + "${AOM_ROOT}/aom/aom.h" + "${AOM_ROOT}/aom/aomcx.h" + "${AOM_ROOT}/aom/aomdx.h" + "${AOM_ROOT}/aom/aom_codec.h" + "${AOM_ROOT}/aom/aom_frame_buffer.h" + "${AOM_ROOT}/aom/aom_image.h" + "${AOM_ROOT}/aom/aom_integer.h" + "${AOM_ROOT}/aom/aom_decoder.h" + "${AOM_ROOT}/aom/aom_encoder.h" + "${AOM_ROOT}/aom/aom.h") +set(AOM_INSTALL_BINS aomdec aomenc) +set(AOM_INSTALL_LIBS aom) + +install(FILES ${AOM_INSTALL_INCS} + DESTINATION "${CMAKE_INSTALL_PREFIX}/include/aom") +install(TARGETS ${AOM_INSTALL_LIBS} DESTINATION "${CMAKE_INSTALL_PREFIX}/lib") +install(TARGETS ${AOM_INSTALL_BINS} DESTINATION "${CMAKE_INSTALL_PREFIX}/bin") diff --git a/third_party/aom/LICENSE b/third_party/aom/LICENSE new file mode 100644 index 0000000000..fc340c3764 --- /dev/null +++ b/third_party/aom/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2016, Alliance for Open Media. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/third_party/aom/PATENTS b/third_party/aom/PATENTS new file mode 100644 index 0000000000..97842e02f1 --- /dev/null +++ b/third_party/aom/PATENTS @@ -0,0 +1,108 @@ +Alliance for Open Media Patent License 1.0 + +1. License Terms. + +1.1. Patent License. Subject to the terms and conditions of this License, each + Licensor, on behalf of itself and successors in interest and assigns, + grants Licensee a non-sublicensable, perpetual, worldwide, non-exclusive, + no-charge, royalty-free, irrevocable (except as expressly stated in this + License) patent license to its Necessary Claims to make, use, sell, offer + for sale, import or distribute any Implementation. + +1.2. Conditions. + +1.2.1. Availability. As a condition to the grant of rights to Licensee to make, + sell, offer for sale, import or distribute an Implementation under + Section 1.1, Licensee must make its Necessary Claims available under + this License, and must reproduce this License with any Implementation + as follows: + + a. For distribution in source code, by including this License in the + root directory of the source code with its Implementation. + + b. For distribution in any other form (including binary, object form, + and/or hardware description code (e.g., HDL, RTL, Gate Level Netlist, + GDSII, etc.)), by including this License in the documentation, legal + notices, and/or other written materials provided with the + Implementation. + +1.2.2. Additional Conditions. This license is directly from Licensor to + Licensee. Licensee acknowledges as a condition of benefiting from it + that no rights from Licensor are received from suppliers, distributors, + or otherwise in connection with this License. + +1.3. Defensive Termination. If any Licensee, its Affiliates, or its agents + initiates patent litigation or files, maintains, or voluntarily + participates in a lawsuit against another entity or any person asserting + that any Implementation infringes Necessary Claims, any patent licenses + granted under this License directly to the Licensee are immediately + terminated as of the date of the initiation of action unless 1) that suit + was in response to a corresponding suit regarding an Implementation first + brought against an initiating entity, or 2) that suit was brought to + enforce the terms of this License (including intervention in a third-party + action by a Licensee). + +1.4. Disclaimers. The Reference Implementation and Specification are provided + "AS IS" and without warranty. The entire risk as to implementing or + otherwise using the Reference Implementation or Specification is assumed + by the implementer and user. Licensor expressly disclaims any warranties + (express, implied, or otherwise), including implied warranties of + merchantability, non-infringement, fitness for a particular purpose, or + title, related to the material. IN NO EVENT WILL LICENSOR BE LIABLE TO + ANY OTHER PARTY FOR LOST PROFITS OR ANY FORM OF INDIRECT, SPECIAL, + INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER FROM ANY CAUSES OF + ACTION OF ANY KIND WITH RESPECT TO THIS LICENSE, WHETHER BASED ON BREACH + OF CONTRACT, TORT (INCLUDING NEGLIGENCE), OR OTHERWISE, AND WHETHER OR + NOT THE OTHER PARTRY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +2. Definitions. + +2.1. Affiliate. �Affiliate� means an entity that directly or indirectly + Controls, is Controlled by, or is under common Control of that party. + +2.2. Control. �Control� means direct or indirect control of more than 50% of + the voting power to elect directors of that corporation, or for any other + entity, the power to direct management of such entity. + +2.3. Decoder. "Decoder" means any decoder that conforms fully with all + non-optional portions of the Specification. + +2.4. Encoder. "Encoder" means any encoder that produces a bitstream that can + be decoded by a Decoder only to the extent it produces such a bitstream. + +2.5. Final Deliverable. �Final Deliverable� means the final version of a + deliverable approved by the Alliance for Open Media as a Final + Deliverable. + +2.6. Implementation. "Implementation" means any implementation, including the + Reference Implementation, that is an Encoder and/or a Decoder. An + Implementation also includes components of an Implementation only to the + extent they are used as part of an Implementation. + +2.7. License. �License� means this license. + +2.8. Licensee. �Licensee� means any person or entity who exercises patent + rights granted under this License. + +2.9. Licensor. "Licensor" means (i) any Licensee that makes, sells, offers + for sale, imports or distributes any Implementation, or (ii) a person + or entity that has a licensing obligation to the Implementation as a + result of its membership and/or participation in the Alliance for Open + Media working group that developed the Specification. + +2.10. Necessary Claims. "Necessary Claims" means all claims of patents or + patent applications, (a) that currently or at any time in the future, + are owned or controlled by the Licensor, and (b) (i) would be an + Essential Claim as defined by the W3C Policy as of February 5, 2004 + (https://www.w3.org/Consortium/Patent-Policy-20040205/#def-essential) + as if the Specification was a W3C Recommendation; or (ii) are infringed + by the Reference Implementation. + +2.11. Reference Implementation. �Reference Implementation� means an Encoder + and/or Decoder released by the Alliance for Open Media as a Final + Deliverable. + +2.12. Specification. �Specification� means the specification designated by + the Alliance for Open Media as a Final Deliverable for which this + License was issued. + diff --git a/third_party/aom/README b/third_party/aom/README new file mode 100644 index 0000000000..9aa30daa13 --- /dev/null +++ b/third_party/aom/README @@ -0,0 +1,168 @@ +README - 9 March 2017 + +Welcome to the AV1 Codec SDK! + +COMPILING THE APPLICATIONS/LIBRARIES: + The build system used is similar to autotools. Building generally consists of + "configuring" with your desired build options, then using GNU make to build + the application. + + 1. Prerequisites + + * All x86 targets require the Yasm[1] assembler be installed. + * All Windows builds require that Cygwin[2] be installed. + * Building the documentation requires Doxygen[3]. If you do not + have this package, the install-docs option will be disabled. + * Downloading the data for the unit tests requires curl[4] and sha1sum. + sha1sum is provided via the GNU coreutils, installed by default on + many *nix platforms, as well as MinGW and Cygwin. If coreutils is not + available, a compatible version of sha1sum can be built from + source[5]. These requirements are optional if not running the unit + tests. + + [1]: http://www.tortall.net/projects/yasm + [2]: http://www.cygwin.com + [3]: http://www.doxygen.org + [4]: http://curl.haxx.se + [5]: http://www.microbrew.org/tools/md5sha1sum/ + + 2. Out-of-tree builds + Out of tree builds are a supported method of building the application. For + an out of tree build, the source tree is kept separate from the object + files produced during compilation. For instance: + + $ mkdir build + $ cd build + $ ../libaom/configure + $ make + + 3. Configuration options + The 'configure' script supports a number of options. The --help option can be + used to get a list of supported options: + $ ../libaom/configure --help + + 4. Cross development + For cross development, the most notable option is the --target option. The + most up-to-date list of supported targets can be found at the bottom of the + --help output of the configure script. As of this writing, the list of + available targets is: + + armv6-linux-rvct + armv6-linux-gcc + armv6-none-rvct + arm64-darwin-gcc + armv7-android-gcc + armv7-darwin-gcc + armv7-linux-rvct + armv7-linux-gcc + armv7-none-rvct + armv7-win32-vs12 + armv7-win32-vs14 + armv7s-darwin-gcc + mips32-linux-gcc + mips64-linux-gcc + sparc-solaris-gcc + x86-android-gcc + x86-darwin8-gcc + x86-darwin8-icc + x86-darwin9-gcc + x86-darwin9-icc + x86-darwin10-gcc + x86-darwin11-gcc + x86-darwin12-gcc + x86-darwin13-gcc + x86-darwin14-gcc + x86-darwin15-gcc + x86-darwin16-gcc + x86-iphonesimulator-gcc + x86-linux-gcc + x86-linux-icc + x86-os2-gcc + x86-solaris-gcc + x86-win32-gcc + x86-win32-vs12 + x86-win32-vs14 + x86_64-android-gcc + x86_64-darwin9-gcc + x86_64-darwin10-gcc + x86_64-darwin11-gcc + x86_64-darwin12-gcc + x86_64-darwin13-gcc + x86_64-darwin14-gcc + x86_64-darwin15-gcc + x86_64-darwin16-gcc + x86_64-iphonesimulator-gcc + x86_64-linux-gcc + x86_64-linux-icc + x86_64-solaris-gcc + x86_64-win64-gcc + x86_64-win64-vs12 + x86_64-win64-vs14 + generic-gnu + + The generic-gnu target, in conjunction with the CROSS environment variable, + can be used to cross compile architectures that aren't explicitly listed, if + the toolchain is a cross GNU (gcc/binutils) toolchain. Other POSIX toolchains + will likely work as well. For instance, to build using the mipsel-linux-uclibc + toolchain, the following command could be used (note, POSIX SH syntax, adapt + to your shell as necessary): + + $ CROSS=mipsel-linux-uclibc- ../libaom/configure + + In addition, the executables to be invoked can be overridden by specifying the + environment variables: CC, AR, LD, AS, STRIP, NM. Additional flags can be + passed to these executables with CFLAGS, LDFLAGS, and ASFLAGS. + + 5. Configuration errors + If the configuration step fails, the first step is to look in the error log. + This defaults to config.log. This should give a good indication of what went + wrong. If not, contact us for support. + +AV1 TEST VECTORS: + The test vectors can be downloaded and verified using the build system after + running configure. To specify an alternate directory the + LIBAOM_TEST_DATA_PATH environment variable can be used. + + $ ./configure --enable-unit-tests + $ LIBAOM_TEST_DATA_PATH=../-test-data make testdata + +UNIT TESTS: + The unit tests (consisting mainly of the test_libaom binary) can be run using + make. This will download the test data if necessary. + + $ ../libaom/configure --enable-unit-tests + $ make test + + Test may be run in parallel using make -j which supports up to 10 shards by + default. + $ make -j10 test + + If you have additional cores you can scale the tests to match: + $ shards=$(nproc); \ + make -j$shards test \ + NUM_SHARDS=$shards SHARDS="$(seq -s' ' 0 $(( shards - 1 )))" \ + && echo "success" + + The GTEST_FILTER environment variable (equivalent to --gtest_filter) can be + used to control which tests are run while sharding: + $ GTEST_FILTER='SSE2*' make -j10 test + +CODE STYLE: + The coding style used by this project is enforced with clang-format using the + configuration contained in the .clang-format file in the root of the + repository. + + Before pushing changes for review you can format your code with: + # Apply clang-format to modified .c, .h and .cc files + $ clang-format -i --style=file \ + $(git diff --name-only --diff-filter=ACMR '*.[hc]' '*.cc') + + Check the .clang-format file for the version used to generate it if there is + any difference between your local formatting and the review system. + + See also: http://clang.llvm.org/docs/ClangFormat.html + +SUPPORT + This library is an open source project supported by its community. Please + please email webm-discuss@webmproject.org for help. + diff --git a/third_party/aom/aom/aom.h b/third_party/aom/aom/aom.h new file mode 100644 index 0000000000..98366b8709 --- /dev/null +++ b/third_party/aom/aom/aom.h @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/*!\defgroup aom AOM + * \ingroup codecs + * AOM is aom's newest video compression algorithm that uses motion + * compensated prediction, Discrete Cosine Transform (DCT) coding of the + * prediction error signal and context dependent entropy coding techniques + * based on arithmetic principles. It features: + * - YUV 4:2:0 image format + * - Macro-block based coding (16x16 luma plus two 8x8 chroma) + * - 1/4 (1/8) pixel accuracy motion compensated prediction + * - 4x4 DCT transform + * - 128 level linear quantizer + * - In loop deblocking filter + * - Context-based entropy coding + * + * @{ + */ +/*!\file + * \brief Provides controls common to both the AOM encoder and decoder. + */ +#ifndef AOM_AOM_H_ +#define AOM_AOM_H_ + +#include "./aom_codec.h" +#include "./aom_image.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*!\brief Control functions + * + * The set of macros define the control functions of AOM interface + */ +enum aom_com_control_id { + /*!\brief pass in an external frame into decoder to be used as reference frame + */ + AOM_SET_REFERENCE = 1, + AOM_COPY_REFERENCE = 2, /**< get a copy of reference frame from the decoder */ + AOM_SET_POSTPROC = 3, /**< set the decoder's post processing settings */ + AOM_SET_DBG_COLOR_REF_FRAME = + 4, /**< set the reference frames to color for each macroblock */ + AOM_SET_DBG_COLOR_MB_MODES = 5, /**< set which macro block modes to color */ + AOM_SET_DBG_COLOR_B_MODES = 6, /**< set which blocks modes to color */ + AOM_SET_DBG_DISPLAY_MV = 7, /**< set which motion vector modes to draw */ + + /* TODO(jkoleszar): The encoder incorrectly reuses some of these values (5+) + * for its control ids. These should be migrated to something like the + * AOM_DECODER_CTRL_ID_START range next time we're ready to break the ABI. + */ + AV1_GET_REFERENCE = 128, /**< get a pointer to a reference frame */ + AOM_COMMON_CTRL_ID_MAX, + + AV1_GET_NEW_FRAME_IMAGE = 192, /**< get a pointer to the new frame */ + + AOM_DECODER_CTRL_ID_START = 256 +}; + +/*!\brief post process flags + * + * The set of macros define AOM decoder post processing flags + */ +enum aom_postproc_level { + AOM_NOFILTERING = 0, + AOM_DEBLOCK = 1 << 0, + AOM_DEMACROBLOCK = 1 << 1, + AOM_ADDNOISE = 1 << 2, + AOM_DEBUG_TXT_FRAME_INFO = 1 << 3, /**< print frame information */ + AOM_DEBUG_TXT_MBLK_MODES = + 1 << 4, /**< print macro block modes over each macro block */ + AOM_DEBUG_TXT_DC_DIFF = 1 << 5, /**< print dc diff for each macro block */ + AOM_DEBUG_TXT_RATE_INFO = 1 << 6, /**< print video rate info (encoder only) */ + AOM_MFQE = 1 << 10 +}; + +/*!\brief post process flags + * + * This define a structure that describe the post processing settings. For + * the best objective measure (using the PSNR metric) set post_proc_flag + * to AOM_DEBLOCK and deblocking_level to 1. + */ + +typedef struct aom_postproc_cfg { + /*!\brief the types of post processing to be done, should be combination of + * "aom_postproc_level" */ + int post_proc_flag; + int deblocking_level; /**< the strength of deblocking, valid range [0, 16] */ + int noise_level; /**< the strength of additive noise, valid range [0, 16] */ +} aom_postproc_cfg_t; + +/*!\brief reference frame type + * + * The set of macros define the type of AOM reference frames + */ +typedef enum aom_ref_frame_type { + AOM_LAST_FRAME = 1, + AOM_GOLD_FRAME = 2, + AOM_ALTR_FRAME = 4 +} aom_ref_frame_type_t; + +/*!\brief reference frame data struct + * + * Define the data struct to access aom reference frames. + */ +typedef struct aom_ref_frame { + aom_ref_frame_type_t frame_type; /**< which reference frame */ + aom_image_t img; /**< reference frame data in image format */ +} aom_ref_frame_t; + +/*!\brief AV1 specific reference frame data struct + * + * Define the data struct to access av1 reference frames. + */ +typedef struct av1_ref_frame { + int idx; /**< frame index to get (input) */ + aom_image_t img; /**< img structure to populate (output) */ +} av1_ref_frame_t; + +/*!\cond */ +/*!\brief aom decoder control function parameter type + * + * defines the data type for each of AOM decoder control function requires + */ +AOM_CTRL_USE_TYPE(AOM_SET_REFERENCE, aom_ref_frame_t *) +#define AOM_CTRL_AOM_SET_REFERENCE +AOM_CTRL_USE_TYPE(AOM_COPY_REFERENCE, aom_ref_frame_t *) +#define AOM_CTRL_AOM_COPY_REFERENCE +AOM_CTRL_USE_TYPE(AOM_SET_POSTPROC, aom_postproc_cfg_t *) +#define AOM_CTRL_AOM_SET_POSTPROC +AOM_CTRL_USE_TYPE(AOM_SET_DBG_COLOR_REF_FRAME, int) +#define AOM_CTRL_AOM_SET_DBG_COLOR_REF_FRAME +AOM_CTRL_USE_TYPE(AOM_SET_DBG_COLOR_MB_MODES, int) +#define AOM_CTRL_AOM_SET_DBG_COLOR_MB_MODES +AOM_CTRL_USE_TYPE(AOM_SET_DBG_COLOR_B_MODES, int) +#define AOM_CTRL_AOM_SET_DBG_COLOR_B_MODES +AOM_CTRL_USE_TYPE(AOM_SET_DBG_DISPLAY_MV, int) +#define AOM_CTRL_AOM_SET_DBG_DISPLAY_MV +AOM_CTRL_USE_TYPE(AV1_GET_REFERENCE, av1_ref_frame_t *) +#define AOM_CTRL_AV1_GET_REFERENCE +AOM_CTRL_USE_TYPE(AV1_GET_NEW_FRAME_IMAGE, aom_image_t *) +#define AOM_CTRL_AV1_GET_NEW_FRAME_IMAGE + +/*!\endcond */ +/*! @} - end defgroup aom */ + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_AOM_H_ diff --git a/third_party/aom/aom/aom_codec.h b/third_party/aom/aom/aom_codec.h new file mode 100644 index 0000000000..1d301d16b2 --- /dev/null +++ b/third_party/aom/aom/aom_codec.h @@ -0,0 +1,487 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/*!\defgroup codec Common Algorithm Interface + * This abstraction allows applications to easily support multiple video + * formats with minimal code duplication. This section describes the interface + * common to all codecs (both encoders and decoders). + * @{ + */ + +/*!\file + * \brief Describes the codec algorithm interface to applications. + * + * This file describes the interface between an application and a + * video codec algorithm. + * + * An application instantiates a specific codec instance by using + * aom_codec_init() and a pointer to the algorithm's interface structure: + * 
+ *     my_app.c:
+ *       extern aom_codec_iface_t my_codec;
+ *       {
+ *           aom_codec_ctx_t algo;
+ *           res = aom_codec_init(&algo, &my_codec);
+ *       }
+ *
+ * + * Once initialized, the instance is manged using other functions from + * the aom_codec_* family. + */ +#ifndef AOM_AOM_CODEC_H_ +#define AOM_AOM_CODEC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./aom_integer.h" +#include "./aom_image.h" + +/*!\brief Decorator indicating a function is deprecated */ +#ifndef DEPRECATED +#if defined(__GNUC__) && __GNUC__ +#define DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) +#define DEPRECATED +#else +#define DEPRECATED +#endif +#endif /* DEPRECATED */ + +#ifndef DECLSPEC_DEPRECATED +#if defined(__GNUC__) && __GNUC__ +#define DECLSPEC_DEPRECATED /**< \copydoc #DEPRECATED */ +#elif defined(_MSC_VER) +/*!\brief \copydoc #DEPRECATED */ +#define DECLSPEC_DEPRECATED __declspec(deprecated) +#else +#define DECLSPEC_DEPRECATED /**< \copydoc #DEPRECATED */ +#endif +#endif /* DECLSPEC_DEPRECATED */ + +/*!\brief Decorator indicating a function is potentially unused */ +#ifdef UNUSED +#elif defined(__GNUC__) || defined(__clang__) +#define UNUSED __attribute__((unused)) +#else +#define UNUSED +#endif + +/*!\brief Decorator indicating that given struct/union/enum is packed */ +#ifndef ATTRIBUTE_PACKED +#if defined(__GNUC__) && __GNUC__ +#define ATTRIBUTE_PACKED __attribute__((packed)) +#elif defined(_MSC_VER) +#define ATTRIBUTE_PACKED +#else +#define ATTRIBUTE_PACKED +#endif +#endif /* ATTRIBUTE_PACKED */ + +/*!\brief Current ABI version number + * + * \internal + * If this file is altered in any way that changes the ABI, this value + * must be bumped. Examples include, but are not limited to, changing + * types, removing or reassigning enums, adding/removing/rearranging + * fields to structures + */ +#define AOM_CODEC_ABI_VERSION (3 + AOM_IMAGE_ABI_VERSION) /**<\hideinitializer*/ + +/*!\brief Algorithm return codes */ +typedef enum { + /*!\brief Operation completed without error */ + AOM_CODEC_OK, + + /*!\brief Unspecified error */ + AOM_CODEC_ERROR, + + /*!\brief Memory operation failed */ + AOM_CODEC_MEM_ERROR, + + /*!\brief ABI version mismatch */ + AOM_CODEC_ABI_MISMATCH, + + /*!\brief Algorithm does not have required capability */ + AOM_CODEC_INCAPABLE, + + /*!\brief The given bitstream is not supported. + * + * The bitstream was unable to be parsed at the highest level. The decoder + * is unable to proceed. This error \ref SHOULD be treated as fatal to the + * stream. */ + AOM_CODEC_UNSUP_BITSTREAM, + + /*!\brief Encoded bitstream uses an unsupported feature + * + * The decoder does not implement a feature required by the encoder. This + * return code should only be used for features that prevent future + * pictures from being properly decoded. This error \ref MAY be treated as + * fatal to the stream or \ref MAY be treated as fatal to the current GOP. + */ + AOM_CODEC_UNSUP_FEATURE, + + /*!\brief The coded data for this stream is corrupt or incomplete + * + * There was a problem decoding the current frame. This return code + * should only be used for failures that prevent future pictures from + * being properly decoded. This error \ref MAY be treated as fatal to the + * stream or \ref MAY be treated as fatal to the current GOP. If decoding + * is continued for the current GOP, artifacts may be present. + */ + AOM_CODEC_CORRUPT_FRAME, + + /*!\brief An application-supplied parameter is not valid. + * + */ + AOM_CODEC_INVALID_PARAM, + + /*!\brief An iterator reached the end of list. + * + */ + AOM_CODEC_LIST_END + +} aom_codec_err_t; + +/*! \brief Codec capabilities bitfield + * + * Each codec advertises the capabilities it supports as part of its + * ::aom_codec_iface_t interface structure. Capabilities are extra interfaces + * or functionality, and are not required to be supported. + * + * The available flags are specified by AOM_CODEC_CAP_* defines. + */ +typedef long aom_codec_caps_t; +#define AOM_CODEC_CAP_DECODER 0x1 /**< Is a decoder */ +#define AOM_CODEC_CAP_ENCODER 0x2 /**< Is an encoder */ + +/*! \brief Initialization-time Feature Enabling + * + * Certain codec features must be known at initialization time, to allow for + * proper memory allocation. + * + * The available flags are specified by AOM_CODEC_USE_* defines. + */ +typedef long aom_codec_flags_t; + +/*!\brief Codec interface structure. + * + * Contains function pointers and other data private to the codec + * implementation. This structure is opaque to the application. + */ +typedef const struct aom_codec_iface aom_codec_iface_t; + +/*!\brief Codec private data structure. + * + * Contains data private to the codec implementation. This structure is opaque + * to the application. + */ +typedef struct aom_codec_priv aom_codec_priv_t; + +/*!\brief Iterator + * + * Opaque storage used for iterating over lists. + */ +typedef const void *aom_codec_iter_t; + +/*!\brief Codec context structure + * + * All codecs \ref MUST support this context structure fully. In general, + * this data should be considered private to the codec algorithm, and + * not be manipulated or examined by the calling application. Applications + * may reference the 'name' member to get a printable description of the + * algorithm. + */ +typedef struct aom_codec_ctx { + const char *name; /**< Printable interface name */ + aom_codec_iface_t *iface; /**< Interface pointers */ + aom_codec_err_t err; /**< Last returned error */ + const char *err_detail; /**< Detailed info, if available */ + aom_codec_flags_t init_flags; /**< Flags passed at init time */ + union { + /**< Decoder Configuration Pointer */ + const struct aom_codec_dec_cfg *dec; + /**< Encoder Configuration Pointer */ + const struct aom_codec_enc_cfg *enc; + const void *raw; + } config; /**< Configuration pointer aliasing union */ + aom_codec_priv_t *priv; /**< Algorithm private storage */ +} aom_codec_ctx_t; + +/*!\brief Bit depth for codec + * * + * This enumeration determines the bit depth of the codec. + */ +typedef enum aom_bit_depth { + AOM_BITS_8 = 8, /**< 8 bits */ + AOM_BITS_10 = 10, /**< 10 bits */ + AOM_BITS_12 = 12, /**< 12 bits */ +} aom_bit_depth_t; + +/*!\brief Superblock size selection. + * + * Defines the superblock size used for encoding. The superblock size can + * either be fixed at 64x64 or 128x128 pixels, or it can be dynamically + * selected by the encoder for each frame. + */ +typedef enum aom_superblock_size { + AOM_SUPERBLOCK_SIZE_64X64, /**< Always use 64x64 superblocks. */ + AOM_SUPERBLOCK_SIZE_128X128, /**< Always use 128x128 superblocks. */ + AOM_SUPERBLOCK_SIZE_DYNAMIC /**< Select superblock size dynamically. */ +} aom_superblock_size_t; + +/* + * Library Version Number Interface + * + * For example, see the following sample return values: + * aom_codec_version() (1<<16 | 2<<8 | 3) + * aom_codec_version_str() "v1.2.3-rc1-16-gec6a1ba" + * aom_codec_version_extra_str() "rc1-16-gec6a1ba" + */ + +/*!\brief Return the version information (as an integer) + * + * Returns a packed encoding of the library version number. This will only + * include + * the major.minor.patch component of the version number. Note that this encoded + * value should be accessed through the macros provided, as the encoding may + * change + * in the future. + * + */ +int aom_codec_version(void); +#define AOM_VERSION_MAJOR(v) \ + ((v >> 16) & 0xff) /**< extract major from packed version */ +#define AOM_VERSION_MINOR(v) \ + ((v >> 8) & 0xff) /**< extract minor from packed version */ +#define AOM_VERSION_PATCH(v) \ + ((v >> 0) & 0xff) /**< extract patch from packed version */ + +/*!\brief Return the version major number */ +#define aom_codec_version_major() ((aom_codec_version() >> 16) & 0xff) + +/*!\brief Return the version minor number */ +#define aom_codec_version_minor() ((aom_codec_version() >> 8) & 0xff) + +/*!\brief Return the version patch number */ +#define aom_codec_version_patch() ((aom_codec_version() >> 0) & 0xff) + +/*!\brief Return the version information (as a string) + * + * Returns a printable string containing the full library version number. This + * may + * contain additional text following the three digit version number, as to + * indicate + * release candidates, prerelease versions, etc. + * + */ +const char *aom_codec_version_str(void); + +/*!\brief Return the version information (as a string) + * + * Returns a printable "extra string". This is the component of the string + * returned + * by aom_codec_version_str() following the three digit version number. + * + */ +const char *aom_codec_version_extra_str(void); + +/*!\brief Return the build configuration + * + * Returns a printable string containing an encoded version of the build + * configuration. This may be useful to aom support. + * + */ +const char *aom_codec_build_config(void); + +/*!\brief Return the name for a given interface + * + * Returns a human readable string for name of the given codec interface. + * + * \param[in] iface Interface pointer + * + */ +const char *aom_codec_iface_name(aom_codec_iface_t *iface); + +/*!\brief Convert error number to printable string + * + * Returns a human readable string for the last error returned by the + * algorithm. The returned error will be one line and will not contain + * any newline characters. + * + * + * \param[in] err Error number. + * + */ +const char *aom_codec_err_to_string(aom_codec_err_t err); + +/*!\brief Retrieve error synopsis for codec context + * + * Returns a human readable string for the last error returned by the + * algorithm. The returned error will be one line and will not contain + * any newline characters. + * + * + * \param[in] ctx Pointer to this instance's context. + * + */ +const char *aom_codec_error(aom_codec_ctx_t *ctx); + +/*!\brief Retrieve detailed error information for codec context + * + * Returns a human readable string providing detailed information about + * the last error. + * + * \param[in] ctx Pointer to this instance's context. + * + * \retval NULL + * No detailed information is available. + */ +const char *aom_codec_error_detail(aom_codec_ctx_t *ctx); + +/* REQUIRED FUNCTIONS + * + * The following functions are required to be implemented for all codecs. + * They represent the base case functionality expected of all codecs. + */ + +/*!\brief Destroy a codec instance + * + * Destroys a codec context, freeing any associated memory buffers. + * + * \param[in] ctx Pointer to this instance's context + * + * \retval #AOM_CODEC_OK + * The codec algorithm initialized. + * \retval #AOM_CODEC_MEM_ERROR + * Memory allocation failed. + */ +aom_codec_err_t aom_codec_destroy(aom_codec_ctx_t *ctx); + +/*!\brief Get the capabilities of an algorithm. + * + * Retrieves the capabilities bitfield from the algorithm's interface. + * + * \param[in] iface Pointer to the algorithm interface + * + */ +aom_codec_caps_t aom_codec_get_caps(aom_codec_iface_t *iface); + +/*!\brief Control algorithm + * + * This function is used to exchange algorithm specific data with the codec + * instance. This can be used to implement features specific to a particular + * algorithm. + * + * This wrapper function dispatches the request to the helper function + * associated with the given ctrl_id. It tries to call this function + * transparently, but will return #AOM_CODEC_ERROR if the request could not + * be dispatched. + * + * Note that this function should not be used directly. Call the + * #aom_codec_control wrapper macro instead. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] ctrl_id Algorithm specific control identifier + * + * \retval #AOM_CODEC_OK + * The control request was processed. + * \retval #AOM_CODEC_ERROR + * The control request was not processed. + * \retval #AOM_CODEC_INVALID_PARAM + * The data was not valid. + */ +aom_codec_err_t aom_codec_control_(aom_codec_ctx_t *ctx, int ctrl_id, ...); +#if defined(AOM_DISABLE_CTRL_TYPECHECKS) && AOM_DISABLE_CTRL_TYPECHECKS +#define aom_codec_control(ctx, id, data) aom_codec_control_(ctx, id, data) +#define AOM_CTRL_USE_TYPE(id, typ) +#define AOM_CTRL_USE_TYPE_DEPRECATED(id, typ) +#define AOM_CTRL_VOID(id, typ) + +#else +/*!\brief aom_codec_control wrapper macro + * + * This macro allows for type safe conversions across the variadic parameter + * to aom_codec_control_(). + * + * \internal + * It works by dispatching the call to the control function through a wrapper + * function named with the id parameter. + */ +#define aom_codec_control(ctx, id, data) \ + aom_codec_control_##id(ctx, id, data) /**<\hideinitializer*/ + +/*!\brief aom_codec_control type definition macro + * + * This macro allows for type safe conversions across the variadic parameter + * to aom_codec_control_(). It defines the type of the argument for a given + * control identifier. + * + * \internal + * It defines a static function with + * the correctly typed arguments as a wrapper to the type-unsafe internal + * function. + */ +#define AOM_CTRL_USE_TYPE(id, typ) \ + static aom_codec_err_t aom_codec_control_##id(aom_codec_ctx_t *, int, typ) \ + UNUSED; \ + \ + static aom_codec_err_t aom_codec_control_##id(aom_codec_ctx_t *ctx, \ + int ctrl_id, typ data) { \ + return aom_codec_control_(ctx, ctrl_id, data); \ + } /**<\hideinitializer*/ + +/*!\brief aom_codec_control deprecated type definition macro + * + * Like #AOM_CTRL_USE_TYPE, but indicates that the specified control is + * deprecated and should not be used. Consult the documentation for your + * codec for more information. + * + * \internal + * It defines a static function with the correctly typed arguments as a + * wrapper to the type-unsafe internal function. + */ +#define AOM_CTRL_USE_TYPE_DEPRECATED(id, typ) \ + DECLSPEC_DEPRECATED static aom_codec_err_t aom_codec_control_##id( \ + aom_codec_ctx_t *, int, typ) DEPRECATED UNUSED; \ + \ + DECLSPEC_DEPRECATED static aom_codec_err_t aom_codec_control_##id( \ + aom_codec_ctx_t *ctx, int ctrl_id, typ data) { \ + return aom_codec_control_(ctx, ctrl_id, data); \ + } /**<\hideinitializer*/ + +/*!\brief aom_codec_control void type definition macro + * + * This macro allows for type safe conversions across the variadic parameter + * to aom_codec_control_(). It indicates that a given control identifier takes + * no argument. + * + * \internal + * It defines a static function without a data argument as a wrapper to the + * type-unsafe internal function. + */ +#define AOM_CTRL_VOID(id) \ + static aom_codec_err_t aom_codec_control_##id(aom_codec_ctx_t *, int) \ + UNUSED; \ + \ + static aom_codec_err_t aom_codec_control_##id(aom_codec_ctx_t *ctx, \ + int ctrl_id) { \ + return aom_codec_control_(ctx, ctrl_id); \ + } /**<\hideinitializer*/ + +#endif + +/*!@} - end defgroup codec*/ +#ifdef __cplusplus +} +#endif +#endif // AOM_AOM_CODEC_H_ diff --git a/third_party/aom/aom/aom_codec.mk b/third_party/aom/aom/aom_codec.mk new file mode 100644 index 0000000000..33bd3fe3be --- /dev/null +++ b/third_party/aom/aom/aom_codec.mk @@ -0,0 +1,42 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + +API_EXPORTS += exports + +API_SRCS-$(CONFIG_AV1_ENCODER) += aom.h +API_SRCS-$(CONFIG_AV1_ENCODER) += aomcx.h +API_DOC_SRCS-$(CONFIG_AV1_ENCODER) += aom.h +API_DOC_SRCS-$(CONFIG_AV1_ENCODER) += aomcx.h + +API_SRCS-$(CONFIG_AV1_DECODER) += aom.h +API_SRCS-$(CONFIG_AV1_DECODER) += aomdx.h +API_DOC_SRCS-$(CONFIG_AV1_DECODER) += aom.h +API_DOC_SRCS-$(CONFIG_AV1_DECODER) += aomdx.h + +API_DOC_SRCS-yes += aom_codec.h +API_DOC_SRCS-yes += aom_decoder.h +API_DOC_SRCS-yes += aom_encoder.h +API_DOC_SRCS-yes += aom_frame_buffer.h +API_DOC_SRCS-yes += aom_image.h + +API_SRCS-yes += src/aom_decoder.c +API_SRCS-yes += aom_decoder.h +API_SRCS-yes += src/aom_encoder.c +API_SRCS-yes += aom_encoder.h +API_SRCS-yes += internal/aom_codec_internal.h +API_SRCS-yes += src/aom_codec.c +API_SRCS-yes += src/aom_image.c +API_SRCS-yes += aom_codec.h +API_SRCS-yes += aom_codec.mk +API_SRCS-yes += aom_frame_buffer.h +API_SRCS-yes += aom_image.h +API_SRCS-yes += aom_integer.h diff --git a/third_party/aom/aom/aom_decoder.h b/third_party/aom/aom/aom_decoder.h new file mode 100644 index 0000000000..e6f05048a9 --- /dev/null +++ b/third_party/aom/aom/aom_decoder.h @@ -0,0 +1,366 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_AOM_DECODER_H_ +#define AOM_AOM_DECODER_H_ + +/*!\defgroup decoder Decoder Algorithm Interface + * \ingroup codec + * This abstraction allows applications using this decoder to easily support + * multiple video formats with minimal code duplication. This section describes + * the interface common to all decoders. + * @{ + */ + +/*!\file + * \brief Describes the decoder algorithm interface to applications. + * + * This file describes the interface between an application and a + * video decoder algorithm. + * + */ +#ifdef __cplusplus +extern "C" { +#endif + +#include "./aom_codec.h" +#include "./aom_frame_buffer.h" + +/*!\brief Current ABI version number + * + * \internal + * If this file is altered in any way that changes the ABI, this value + * must be bumped. Examples include, but are not limited to, changing + * types, removing or reassigning enums, adding/removing/rearranging + * fields to structures + */ +#define AOM_DECODER_ABI_VERSION \ + (3 + AOM_CODEC_ABI_VERSION) /**<\hideinitializer*/ + +/*! \brief Decoder capabilities bitfield + * + * Each decoder advertises the capabilities it supports as part of its + * ::aom_codec_iface_t interface structure. Capabilities are extra interfaces + * or functionality, and are not required to be supported by a decoder. + * + * The available flags are specified by AOM_CODEC_CAP_* defines. + */ +#define AOM_CODEC_CAP_PUT_SLICE 0x10000 /**< Will issue put_slice callbacks */ +#define AOM_CODEC_CAP_PUT_FRAME 0x20000 /**< Will issue put_frame callbacks */ +#define AOM_CODEC_CAP_POSTPROC 0x40000 /**< Can postprocess decoded frame */ +/*!\brief Can conceal errors due to packet loss */ +#define AOM_CODEC_CAP_ERROR_CONCEALMENT 0x80000 +/*!\brief Can receive encoded frames one fragment at a time */ +#define AOM_CODEC_CAP_INPUT_FRAGMENTS 0x100000 + +/*! \brief Initialization-time Feature Enabling + * + * Certain codec features must be known at initialization time, to allow for + * proper memory allocation. + * + * The available flags are specified by AOM_CODEC_USE_* defines. + */ +/*!\brief Can support frame-based multi-threading */ +#define AOM_CODEC_CAP_FRAME_THREADING 0x200000 +/*!brief Can support external frame buffers */ +#define AOM_CODEC_CAP_EXTERNAL_FRAME_BUFFER 0x400000 + +#define AOM_CODEC_USE_POSTPROC 0x10000 /**< Postprocess decoded frame */ +/*!\brief Conceal errors in decoded frames */ +#define AOM_CODEC_USE_ERROR_CONCEALMENT 0x20000 +/*!\brief The input frame should be passed to the decoder one fragment at a + * time */ +#define AOM_CODEC_USE_INPUT_FRAGMENTS 0x40000 +/*!\brief Enable frame-based multi-threading */ +#define AOM_CODEC_USE_FRAME_THREADING 0x80000 + +/*!\brief Stream properties + * + * This structure is used to query or set properties of the decoded + * stream. Algorithms may extend this structure with data specific + * to their bitstream by setting the sz member appropriately. + */ +typedef struct aom_codec_stream_info { + unsigned int sz; /**< Size of this structure */ + unsigned int w; /**< Width (or 0 for unknown/default) */ + unsigned int h; /**< Height (or 0 for unknown/default) */ + unsigned int is_kf; /**< Current frame is a keyframe */ +} aom_codec_stream_info_t; + +/* REQUIRED FUNCTIONS + * + * The following functions are required to be implemented for all decoders. + * They represent the base case functionality expected of all decoders. + */ + +/*!\brief Initialization Configurations + * + * This structure is used to pass init time configuration options to the + * decoder. + */ +typedef struct aom_codec_dec_cfg { + unsigned int threads; /**< Maximum number of threads to use, default 1 */ + unsigned int w; /**< Width */ + unsigned int h; /**< Height */ +} aom_codec_dec_cfg_t; /**< alias for struct aom_codec_dec_cfg */ + +/*!\brief Initialize a decoder instance + * + * Initializes a decoder context using the given interface. Applications + * should call the aom_codec_dec_init convenience macro instead of this + * function directly, to ensure that the ABI version number parameter + * is properly initialized. + * + * If the library was configured with --disable-multithread, this call + * is not thread safe and should be guarded with a lock if being used + * in a multithreaded context. + * + * \param[in] ctx Pointer to this instance's context. + * \param[in] iface Pointer to the algorithm interface to use. + * \param[in] cfg Configuration to use, if known. May be NULL. + * \param[in] flags Bitfield of AOM_CODEC_USE_* flags + * \param[in] ver ABI version number. Must be set to + * AOM_DECODER_ABI_VERSION + * \retval #AOM_CODEC_OK + * The decoder algorithm initialized. + * \retval #AOM_CODEC_MEM_ERROR + * Memory allocation failed. + */ +aom_codec_err_t aom_codec_dec_init_ver(aom_codec_ctx_t *ctx, + aom_codec_iface_t *iface, + const aom_codec_dec_cfg_t *cfg, + aom_codec_flags_t flags, int ver); + +/*!\brief Convenience macro for aom_codec_dec_init_ver() + * + * Ensures the ABI version parameter is properly set. + */ +#define aom_codec_dec_init(ctx, iface, cfg, flags) \ + aom_codec_dec_init_ver(ctx, iface, cfg, flags, AOM_DECODER_ABI_VERSION) + +/*!\brief Parse stream info from a buffer + * + * Performs high level parsing of the bitstream. Construction of a decoder + * context is not necessary. Can be used to determine if the bitstream is + * of the proper format, and to extract information from the stream. + * + * \param[in] iface Pointer to the algorithm interface + * \param[in] data Pointer to a block of data to parse + * \param[in] data_sz Size of the data buffer + * \param[in,out] si Pointer to stream info to update. The size member + * \ref MUST be properly initialized, but \ref MAY be + * clobbered by the algorithm. This parameter \ref MAY + * be NULL. + * + * \retval #AOM_CODEC_OK + * Bitstream is parsable and stream information updated + */ +aom_codec_err_t aom_codec_peek_stream_info(aom_codec_iface_t *iface, + const uint8_t *data, + unsigned int data_sz, + aom_codec_stream_info_t *si); + +/*!\brief Return information about the current stream. + * + * Returns information about the stream that has been parsed during decoding. + * + * \param[in] ctx Pointer to this instance's context + * \param[in,out] si Pointer to stream info to update. The size member + * \ref MUST be properly initialized, but \ref MAY be + * clobbered by the algorithm. This parameter \ref MAY + * be NULL. + * + * \retval #AOM_CODEC_OK + * Bitstream is parsable and stream information updated + */ +aom_codec_err_t aom_codec_get_stream_info(aom_codec_ctx_t *ctx, + aom_codec_stream_info_t *si); + +/*!\brief Decode data + * + * Processes a buffer of coded data. If the processing results in a new + * decoded frame becoming available, PUT_SLICE and PUT_FRAME events may be + * generated, as appropriate. Encoded data \ref MUST be passed in DTS (decode + * time stamp) order. Frames produced will always be in PTS (presentation + * time stamp) order. + * If the decoder is configured with AOM_CODEC_USE_INPUT_FRAGMENTS enabled, + * data and data_sz can contain a fragment of the encoded frame. Fragment + * \#n must contain at least partition \#n, but can also contain subsequent + * partitions (\#n+1 - \#n+i), and if so, fragments \#n+1, .., \#n+i must + * be empty. When no more data is available, this function should be called + * with NULL as data and 0 as data_sz. The memory passed to this function + * must be available until the frame has been decoded. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] data Pointer to this block of new coded data. If + * NULL, a AOM_CODEC_CB_PUT_FRAME event is posted + * for the previously decoded frame. + * \param[in] data_sz Size of the coded data, in bytes. + * \param[in] user_priv Application specific data to associate with + * this frame. + * \param[in] deadline Soft deadline the decoder should attempt to meet, + * in us. Set to zero for unlimited. + * + * \return Returns #AOM_CODEC_OK if the coded data was processed completely + * and future pictures can be decoded without error. Otherwise, + * see the descriptions of the other error codes in ::aom_codec_err_t + * for recoverability capabilities. + */ +aom_codec_err_t aom_codec_decode(aom_codec_ctx_t *ctx, const uint8_t *data, + unsigned int data_sz, void *user_priv, + long deadline); + +/*!\brief Decoded frames iterator + * + * Iterates over a list of the frames available for display. The iterator + * storage should be initialized to NULL to start the iteration. Iteration is + * complete when this function returns NULL. + * + * The list of available frames becomes valid upon completion of the + * aom_codec_decode call, and remains valid until the next call to + * aom_codec_decode. + * + * \param[in] ctx Pointer to this instance's context + * \param[in,out] iter Iterator storage, initialized to NULL + * + * \return Returns a pointer to an image, if one is ready for display. Frames + * produced will always be in PTS (presentation time stamp) order. + */ +aom_image_t *aom_codec_get_frame(aom_codec_ctx_t *ctx, aom_codec_iter_t *iter); + +/*!\defgroup cap_put_frame Frame-Based Decoding Functions + * + * The following functions are required to be implemented for all decoders + * that advertise the AOM_CODEC_CAP_PUT_FRAME capability. Calling these + * functions + * for codecs that don't advertise this capability will result in an error + * code being returned, usually AOM_CODEC_ERROR + * @{ + */ + +/*!\brief put frame callback prototype + * + * This callback is invoked by the decoder to notify the application of + * the availability of decoded image data. + */ +typedef void (*aom_codec_put_frame_cb_fn_t)(void *user_priv, + const aom_image_t *img); + +/*!\brief Register for notification of frame completion. + * + * Registers a given function to be called when a decoded frame is + * available. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] cb Pointer to the callback function + * \param[in] user_priv User's private data + * + * \retval #AOM_CODEC_OK + * Callback successfully registered. + * \retval #AOM_CODEC_ERROR + * Decoder context not initialized, or algorithm not capable of + * posting slice completion. + */ +aom_codec_err_t aom_codec_register_put_frame_cb(aom_codec_ctx_t *ctx, + aom_codec_put_frame_cb_fn_t cb, + void *user_priv); + +/*!@} - end defgroup cap_put_frame */ + +/*!\defgroup cap_put_slice Slice-Based Decoding Functions + * + * The following functions are required to be implemented for all decoders + * that advertise the AOM_CODEC_CAP_PUT_SLICE capability. Calling these + * functions + * for codecs that don't advertise this capability will result in an error + * code being returned, usually AOM_CODEC_ERROR + * @{ + */ + +/*!\brief put slice callback prototype + * + * This callback is invoked by the decoder to notify the application of + * the availability of partially decoded image data. The + */ +typedef void (*aom_codec_put_slice_cb_fn_t)(void *user_priv, + const aom_image_t *img, + const aom_image_rect_t *valid, + const aom_image_rect_t *update); + +/*!\brief Register for notification of slice completion. + * + * Registers a given function to be called when a decoded slice is + * available. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] cb Pointer to the callback function + * \param[in] user_priv User's private data + * + * \retval #AOM_CODEC_OK + * Callback successfully registered. + * \retval #AOM_CODEC_ERROR + * Decoder context not initialized, or algorithm not capable of + * posting slice completion. + */ +aom_codec_err_t aom_codec_register_put_slice_cb(aom_codec_ctx_t *ctx, + aom_codec_put_slice_cb_fn_t cb, + void *user_priv); + +/*!@} - end defgroup cap_put_slice*/ + +/*!\defgroup cap_external_frame_buffer External Frame Buffer Functions + * + * The following section is required to be implemented for all decoders + * that advertise the AOM_CODEC_CAP_EXTERNAL_FRAME_BUFFER capability. + * Calling this function for codecs that don't advertise this capability + * will result in an error code being returned, usually AOM_CODEC_ERROR. + * + * \note + * Currently this only works with AV1. + * @{ + */ + +/*!\brief Pass in external frame buffers for the decoder to use. + * + * Registers functions to be called when libaom needs a frame buffer + * to decode the current frame and a function to be called when libaom does + * not internally reference the frame buffer. This set function must + * be called before the first call to decode or libaom will assume the + * default behavior of allocating frame buffers internally. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] cb_get Pointer to the get callback function + * \param[in] cb_release Pointer to the release callback function + * \param[in] cb_priv Callback's private data + * + * \retval #AOM_CODEC_OK + * External frame buffers will be used by libaom. + * \retval #AOM_CODEC_INVALID_PARAM + * One or more of the callbacks were NULL. + * \retval #AOM_CODEC_ERROR + * Decoder context not initialized, or algorithm not capable of + * using external frame buffers. + * + * \note + * When decoding AV1, the application may be required to pass in at least + * #AOM_MAXIMUM_WORK_BUFFERS external frame + * buffers. + */ +aom_codec_err_t aom_codec_set_frame_buffer_functions( + aom_codec_ctx_t *ctx, aom_get_frame_buffer_cb_fn_t cb_get, + aom_release_frame_buffer_cb_fn_t cb_release, void *cb_priv); + +/*!@} - end defgroup cap_external_frame_buffer */ + +/*!@} - end defgroup decoder*/ +#ifdef __cplusplus +} +#endif +#endif // AOM_AOM_DECODER_H_ diff --git a/third_party/aom/aom/aom_encoder.h b/third_party/aom/aom/aom_encoder.h new file mode 100644 index 0000000000..14c9f0c263 --- /dev/null +++ b/third_party/aom/aom/aom_encoder.h @@ -0,0 +1,832 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_AOM_ENCODER_H_ +#define AOM_AOM_ENCODER_H_ + +/*!\defgroup encoder Encoder Algorithm Interface + * \ingroup codec + * This abstraction allows applications using this encoder to easily support + * multiple video formats with minimal code duplication. This section describes + * the interface common to all encoders. + * @{ + */ + +/*!\file + * \brief Describes the encoder algorithm interface to applications. + * + * This file describes the interface between an application and a + * video encoder algorithm. + * + */ +#ifdef __cplusplus +extern "C" { +#endif + +#include "./aom_codec.h" + +/*!\brief Current ABI version number + * + * \internal + * If this file is altered in any way that changes the ABI, this value + * must be bumped. Examples include, but are not limited to, changing + * types, removing or reassigning enums, adding/removing/rearranging + * fields to structures + */ +#define AOM_ENCODER_ABI_VERSION \ + (5 + AOM_CODEC_ABI_VERSION) /**<\hideinitializer*/ + +/*! \brief Encoder capabilities bitfield + * + * Each encoder advertises the capabilities it supports as part of its + * ::aom_codec_iface_t interface structure. Capabilities are extra + * interfaces or functionality, and are not required to be supported + * by an encoder. + * + * The available flags are specified by AOM_CODEC_CAP_* defines. + */ +#define AOM_CODEC_CAP_PSNR 0x10000 /**< Can issue PSNR packets */ + +/*! Can output one partition at a time. Each partition is returned in its + * own AOM_CODEC_CX_FRAME_PKT, with the FRAME_IS_FRAGMENT flag set for + * every partition but the last. In this mode all frames are always + * returned partition by partition. + */ +#define AOM_CODEC_CAP_OUTPUT_PARTITION 0x20000 + +/*! Can support input images at greater than 8 bitdepth. + */ +#define AOM_CODEC_CAP_HIGHBITDEPTH 0x40000 + +/*! \brief Initialization-time Feature Enabling + * + * Certain codec features must be known at initialization time, to allow + * for proper memory allocation. + * + * The available flags are specified by AOM_CODEC_USE_* defines. + */ +#define AOM_CODEC_USE_PSNR 0x10000 /**< Calculate PSNR on each frame */ +/*!\brief Make the encoder output one partition at a time. */ +#define AOM_CODEC_USE_OUTPUT_PARTITION 0x20000 +#define AOM_CODEC_USE_HIGHBITDEPTH 0x40000 /**< Use high bitdepth */ + +/*!\brief Generic fixed size buffer structure + * + * This structure is able to hold a reference to any fixed size buffer. + */ +typedef struct aom_fixed_buf { + void *buf; /**< Pointer to the data */ + size_t sz; /**< Length of the buffer, in chars */ +} aom_fixed_buf_t; /**< alias for struct aom_fixed_buf */ + +/*!\brief Time Stamp Type + * + * An integer, which when multiplied by the stream's time base, provides + * the absolute time of a sample. + */ +typedef int64_t aom_codec_pts_t; + +/*!\brief Compressed Frame Flags + * + * This type represents a bitfield containing information about a compressed + * frame that may be useful to an application. The most significant 16 bits + * can be used by an algorithm to provide additional detail, for example to + * support frame types that are codec specific (MPEG-1 D-frames for example) + */ +typedef uint32_t aom_codec_frame_flags_t; +#define AOM_FRAME_IS_KEY 0x1 /**< frame is the start of a GOP */ +/*!\brief frame can be dropped without affecting the stream (no future frame + * depends on this one) */ +#define AOM_FRAME_IS_DROPPABLE 0x2 +/*!\brief frame should be decoded but will not be shown */ +#define AOM_FRAME_IS_INVISIBLE 0x4 +/*!\brief this is a fragment of the encoded frame */ +#define AOM_FRAME_IS_FRAGMENT 0x8 + +/*!\brief Error Resilient flags + * + * These flags define which error resilient features to enable in the + * encoder. The flags are specified through the + * aom_codec_enc_cfg::g_error_resilient variable. + */ +typedef uint32_t aom_codec_er_flags_t; +/*!\brief Improve resiliency against losses of whole frames */ +#define AOM_ERROR_RESILIENT_DEFAULT 0x1 +/*!\brief The frame partitions are independently decodable by the bool decoder, + * meaning that partitions can be decoded even though earlier partitions have + * been lost. Note that intra prediction is still done over the partition + * boundary. */ +#define AOM_ERROR_RESILIENT_PARTITIONS 0x2 + +/*!\brief Encoder output packet variants + * + * This enumeration lists the different kinds of data packets that can be + * returned by calls to aom_codec_get_cx_data(). Algorithms \ref MAY + * extend this list to provide additional functionality. + */ +enum aom_codec_cx_pkt_kind { + AOM_CODEC_CX_FRAME_PKT, /**< Compressed video frame */ + AOM_CODEC_STATS_PKT, /**< Two-pass statistics for this frame */ + AOM_CODEC_FPMB_STATS_PKT, /**< first pass mb statistics for this frame */ + AOM_CODEC_PSNR_PKT, /**< PSNR statistics for this frame */ + AOM_CODEC_CUSTOM_PKT = 256 /**< Algorithm extensions */ +}; + +/*!\brief Encoder output packet + * + * This structure contains the different kinds of output data the encoder + * may produce while compressing a frame. + */ +typedef struct aom_codec_cx_pkt { + enum aom_codec_cx_pkt_kind kind; /**< packet variant */ + union { + struct { + void *buf; /**< compressed data buffer */ + size_t sz; /**< length of compressed data */ + /*!\brief time stamp to show frame (in timebase units) */ + aom_codec_pts_t pts; + /*!\brief duration to show frame (in timebase units) */ + unsigned long duration; + aom_codec_frame_flags_t flags; /**< flags for this frame */ + /*!\brief the partition id defines the decoding order of the partitions. + * Only applicable when "output partition" mode is enabled. First + * partition has id 0.*/ + int partition_id; + } frame; /**< data for compressed frame packet */ + aom_fixed_buf_t twopass_stats; /**< data for two-pass packet */ + aom_fixed_buf_t firstpass_mb_stats; /**< first pass mb packet */ + struct aom_psnr_pkt { + unsigned int samples[4]; /**< Number of samples, total/y/u/v */ + uint64_t sse[4]; /**< sum squared error, total/y/u/v */ + double psnr[4]; /**< PSNR, total/y/u/v */ + } psnr; /**< data for PSNR packet */ + aom_fixed_buf_t raw; /**< data for arbitrary packets */ + + /* This packet size is fixed to allow codecs to extend this + * interface without having to manage storage for raw packets, + * i.e., if it's smaller than 128 bytes, you can store in the + * packet list directly. + */ + char pad[128 - sizeof(enum aom_codec_cx_pkt_kind)]; /**< fixed sz */ + } data; /**< packet data */ +} aom_codec_cx_pkt_t; /**< alias for struct aom_codec_cx_pkt */ + +/*!\brief Rational Number + * + * This structure holds a fractional value. + */ +typedef struct aom_rational { + int num; /**< fraction numerator */ + int den; /**< fraction denominator */ +} aom_rational_t; /**< alias for struct aom_rational */ + +/*!\brief Multi-pass Encoding Pass */ +enum aom_enc_pass { + AOM_RC_ONE_PASS, /**< Single pass mode */ + AOM_RC_FIRST_PASS, /**< First pass of multi-pass mode */ + AOM_RC_LAST_PASS /**< Final pass of multi-pass mode */ +}; + +/*!\brief Rate control mode */ +enum aom_rc_mode { + AOM_VBR, /**< Variable Bit Rate (VBR) mode */ + AOM_CBR, /**< Constant Bit Rate (CBR) mode */ + AOM_CQ, /**< Constrained Quality (CQ) mode */ + AOM_Q, /**< Constant Quality (Q) mode */ +}; + +/*!\brief Keyframe placement mode. + * + * This enumeration determines whether keyframes are placed automatically by + * the encoder or whether this behavior is disabled. Older releases of this + * SDK were implemented such that AOM_KF_FIXED meant keyframes were disabled. + * This name is confusing for this behavior, so the new symbols to be used + * are AOM_KF_AUTO and AOM_KF_DISABLED. + */ +enum aom_kf_mode { + AOM_KF_FIXED, /**< deprecated, implies AOM_KF_DISABLED */ + AOM_KF_AUTO, /**< Encoder determines optimal placement automatically */ + AOM_KF_DISABLED = 0 /**< Encoder does not place keyframes. */ +}; + +/*!\brief Encoded Frame Flags + * + * This type indicates a bitfield to be passed to aom_codec_encode(), defining + * per-frame boolean values. By convention, bits common to all codecs will be + * named AOM_EFLAG_*, and bits specific to an algorithm will be named + * /algo/_eflag_*. The lower order 16 bits are reserved for common use. + */ +typedef long aom_enc_frame_flags_t; +#define AOM_EFLAG_FORCE_KF (1 << 0) /**< Force this frame to be a keyframe */ + +/*!\brief Encoder configuration structure + * + * This structure contains the encoder settings that have common representations + * across all codecs. This doesn't imply that all codecs support all features, + * however. + */ +typedef struct aom_codec_enc_cfg { + /* + * generic settings (g) + */ + + /*!\brief Algorithm specific "usage" value + * + * Algorithms may define multiple values for usage, which may convey the + * intent of how the application intends to use the stream. If this value + * is non-zero, consult the documentation for the codec to determine its + * meaning. + */ + unsigned int g_usage; + + /*!\brief Maximum number of threads to use + * + * For multi-threaded implementations, use no more than this number of + * threads. The codec may use fewer threads than allowed. The value + * 0 is equivalent to the value 1. + */ + unsigned int g_threads; + + /*!\brief Bitstream profile to use + * + * Some codecs support a notion of multiple bitstream profiles. Typically + * this maps to a set of features that are turned on or off. Often the + * profile to use is determined by the features of the intended decoder. + * Consult the documentation for the codec to determine the valid values + * for this parameter, or set to zero for a sane default. + */ + unsigned int g_profile; /**< profile of bitstream to use */ + + /*!\brief Width of the frame + * + * This value identifies the presentation resolution of the frame, + * in pixels. Note that the frames passed as input to the encoder must + * have this resolution. Frames will be presented by the decoder in this + * resolution, independent of any spatial resampling the encoder may do. + */ + unsigned int g_w; + + /*!\brief Height of the frame + * + * This value identifies the presentation resolution of the frame, + * in pixels. Note that the frames passed as input to the encoder must + * have this resolution. Frames will be presented by the decoder in this + * resolution, independent of any spatial resampling the encoder may do. + */ + unsigned int g_h; + + /*!\brief Bit-depth of the codec + * + * This value identifies the bit_depth of the codec, + * Only certain bit-depths are supported as identified in the + * aom_bit_depth_t enum. + */ + aom_bit_depth_t g_bit_depth; + + /*!\brief Bit-depth of the input frames + * + * This value identifies the bit_depth of the input frames in bits. + * Note that the frames passed as input to the encoder must have + * this bit-depth. + */ + unsigned int g_input_bit_depth; + + /*!\brief Stream timebase units + * + * Indicates the smallest interval of time, in seconds, used by the stream. + * For fixed frame rate material, or variable frame rate material where + * frames are timed at a multiple of a given clock (ex: video capture), + * the \ref RECOMMENDED method is to set the timebase to the reciprocal + * of the frame rate (ex: 1001/30000 for 29.970 Hz NTSC). This allows the + * pts to correspond to the frame number, which can be handy. For + * re-encoding video from containers with absolute time timestamps, the + * \ref RECOMMENDED method is to set the timebase to that of the parent + * container or multimedia framework (ex: 1/1000 for ms, as in FLV). + */ + struct aom_rational g_timebase; + + /*!\brief Enable error resilient modes. + * + * The error resilient bitfield indicates to the encoder which features + * it should enable to take measures for streaming over lossy or noisy + * links. + */ + aom_codec_er_flags_t g_error_resilient; + + /*!\brief Multi-pass Encoding Mode + * + * This value should be set to the current phase for multi-pass encoding. + * For single pass, set to #AOM_RC_ONE_PASS. + */ + enum aom_enc_pass g_pass; + + /*!\brief Allow lagged encoding + * + * If set, this value allows the encoder to consume a number of input + * frames before producing output frames. This allows the encoder to + * base decisions for the current frame on future frames. This does + * increase the latency of the encoding pipeline, so it is not appropriate + * in all situations (ex: realtime encoding). + * + * Note that this is a maximum value -- the encoder may produce frames + * sooner than the given limit. Set this value to 0 to disable this + * feature. + */ + unsigned int g_lag_in_frames; + + /* + * rate control settings (rc) + */ + + /*!\brief Temporal resampling configuration, if supported by the codec. + * + * Temporal resampling allows the codec to "drop" frames as a strategy to + * meet its target data rate. This can cause temporal discontinuities in + * the encoded video, which may appear as stuttering during playback. This + * trade-off is often acceptable, but for many applications is not. It can + * be disabled in these cases. + * + * Note that not all codecs support this feature. All aom AVx codecs do. + * For other codecs, consult the documentation for that algorithm. + * + * This threshold is described as a percentage of the target data buffer. + * When the data buffer falls below this percentage of fullness, a + * dropped frame is indicated. Set the threshold to zero (0) to disable + * this feature. + */ + unsigned int rc_dropframe_thresh; + + /*!\brief Enable/disable spatial resampling, if supported by the codec. + * + * Spatial resampling allows the codec to compress a lower resolution + * version of the frame, which is then upscaled by the encoder to the + * correct presentation resolution. This increases visual quality at + * low data rates, at the expense of CPU time on the encoder/decoder. + */ + unsigned int rc_resize_allowed; + + /*!\brief Internal coded frame width. + * + * If spatial resampling is enabled this specifies the width of the + * encoded frame. + */ + unsigned int rc_scaled_width; + + /*!\brief Internal coded frame height. + * + * If spatial resampling is enabled this specifies the height of the + * encoded frame. + */ + unsigned int rc_scaled_height; + + /*!\brief Spatial resampling up watermark. + * + * This threshold is described as a percentage of the target data buffer. + * When the data buffer rises above this percentage of fullness, the + * encoder will step up to a higher resolution version of the frame. + */ + unsigned int rc_resize_up_thresh; + + /*!\brief Spatial resampling down watermark. + * + * This threshold is described as a percentage of the target data buffer. + * When the data buffer falls below this percentage of fullness, the + * encoder will step down to a lower resolution version of the frame. + */ + unsigned int rc_resize_down_thresh; + + /*!\brief Rate control algorithm to use. + * + * Indicates whether the end usage of this stream is to be streamed over + * a bandwidth constrained link, indicating that Constant Bit Rate (CBR) + * mode should be used, or whether it will be played back on a high + * bandwidth link, as from a local disk, where higher variations in + * bitrate are acceptable. + */ + enum aom_rc_mode rc_end_usage; + + /*!\brief Two-pass stats buffer. + * + * A buffer containing all of the stats packets produced in the first + * pass, concatenated. + */ + aom_fixed_buf_t rc_twopass_stats_in; + + /*!\brief first pass mb stats buffer. + * + * A buffer containing all of the first pass mb stats packets produced + * in the first pass, concatenated. + */ + aom_fixed_buf_t rc_firstpass_mb_stats_in; + + /*!\brief Target data rate + * + * Target bandwidth to use for this stream, in kilobits per second. + */ + unsigned int rc_target_bitrate; + + /* + * quantizer settings + */ + + /*!\brief Minimum (Best Quality) Quantizer + * + * The quantizer is the most direct control over the quality of the + * encoded image. The range of valid values for the quantizer is codec + * specific. Consult the documentation for the codec to determine the + * values to use. To determine the range programmatically, call + * aom_codec_enc_config_default() with a usage value of 0. + */ + unsigned int rc_min_quantizer; + + /*!\brief Maximum (Worst Quality) Quantizer + * + * The quantizer is the most direct control over the quality of the + * encoded image. The range of valid values for the quantizer is codec + * specific. Consult the documentation for the codec to determine the + * values to use. To determine the range programmatically, call + * aom_codec_enc_config_default() with a usage value of 0. + */ + unsigned int rc_max_quantizer; + + /* + * bitrate tolerance + */ + + /*!\brief Rate control adaptation undershoot control + * + * This value, expressed as a percentage of the target bitrate, + * controls the maximum allowed adaptation speed of the codec. + * This factor controls the maximum amount of bits that can + * be subtracted from the target bitrate in order to compensate + * for prior overshoot. + * + * Valid values in the range 0-1000. + */ + unsigned int rc_undershoot_pct; + + /*!\brief Rate control adaptation overshoot control + * + * This value, expressed as a percentage of the target bitrate, + * controls the maximum allowed adaptation speed of the codec. + * This factor controls the maximum amount of bits that can + * be added to the target bitrate in order to compensate for + * prior undershoot. + * + * Valid values in the range 0-1000. + */ + unsigned int rc_overshoot_pct; + + /* + * decoder buffer model parameters + */ + + /*!\brief Decoder Buffer Size + * + * This value indicates the amount of data that may be buffered by the + * decoding application. Note that this value is expressed in units of + * time (milliseconds). For example, a value of 5000 indicates that the + * client will buffer (at least) 5000ms worth of encoded data. Use the + * target bitrate (#rc_target_bitrate) to convert to bits/bytes, if + * necessary. + */ + unsigned int rc_buf_sz; + + /*!\brief Decoder Buffer Initial Size + * + * This value indicates the amount of data that will be buffered by the + * decoding application prior to beginning playback. This value is + * expressed in units of time (milliseconds). Use the target bitrate + * (#rc_target_bitrate) to convert to bits/bytes, if necessary. + */ + unsigned int rc_buf_initial_sz; + + /*!\brief Decoder Buffer Optimal Size + * + * This value indicates the amount of data that the encoder should try + * to maintain in the decoder's buffer. This value is expressed in units + * of time (milliseconds). Use the target bitrate (#rc_target_bitrate) + * to convert to bits/bytes, if necessary. + */ + unsigned int rc_buf_optimal_sz; + + /* + * 2 pass rate control parameters + */ + + /*!\brief Two-pass mode CBR/VBR bias + * + * Bias, expressed on a scale of 0 to 100, for determining target size + * for the current frame. The value 0 indicates the optimal CBR mode + * value should be used. The value 100 indicates the optimal VBR mode + * value should be used. Values in between indicate which way the + * encoder should "lean." + */ + unsigned int rc_2pass_vbr_bias_pct; + + /*!\brief Two-pass mode per-GOP minimum bitrate + * + * This value, expressed as a percentage of the target bitrate, indicates + * the minimum bitrate to be used for a single GOP (aka "section") + */ + unsigned int rc_2pass_vbr_minsection_pct; + + /*!\brief Two-pass mode per-GOP maximum bitrate + * + * This value, expressed as a percentage of the target bitrate, indicates + * the maximum bitrate to be used for a single GOP (aka "section") + */ + unsigned int rc_2pass_vbr_maxsection_pct; + + /* + * keyframing settings (kf) + */ + + /*!\brief Keyframe placement mode + * + * This value indicates whether the encoder should place keyframes at a + * fixed interval, or determine the optimal placement automatically + * (as governed by the #kf_min_dist and #kf_max_dist parameters) + */ + enum aom_kf_mode kf_mode; + + /*!\brief Keyframe minimum interval + * + * This value, expressed as a number of frames, prevents the encoder from + * placing a keyframe nearer than kf_min_dist to the previous keyframe. At + * least kf_min_dist frames non-keyframes will be coded before the next + * keyframe. Set kf_min_dist equal to kf_max_dist for a fixed interval. + */ + unsigned int kf_min_dist; + + /*!\brief Keyframe maximum interval + * + * This value, expressed as a number of frames, forces the encoder to code + * a keyframe if one has not been coded in the last kf_max_dist frames. + * A value of 0 implies all frames will be keyframes. Set kf_min_dist + * equal to kf_max_dist for a fixed interval. + */ + unsigned int kf_max_dist; +} aom_codec_enc_cfg_t; /**< alias for struct aom_codec_enc_cfg */ + +/*!\brief Initialize an encoder instance + * + * Initializes a encoder context using the given interface. Applications + * should call the aom_codec_enc_init convenience macro instead of this + * function directly, to ensure that the ABI version number parameter + * is properly initialized. + * + * If the library was configured with --disable-multithread, this call + * is not thread safe and should be guarded with a lock if being used + * in a multithreaded context. + * + * \param[in] ctx Pointer to this instance's context. + * \param[in] iface Pointer to the algorithm interface to use. + * \param[in] cfg Configuration to use, if known. May be NULL. + * \param[in] flags Bitfield of AOM_CODEC_USE_* flags + * \param[in] ver ABI version number. Must be set to + * AOM_ENCODER_ABI_VERSION + * \retval #AOM_CODEC_OK + * The decoder algorithm initialized. + * \retval #AOM_CODEC_MEM_ERROR + * Memory allocation failed. + */ +aom_codec_err_t aom_codec_enc_init_ver(aom_codec_ctx_t *ctx, + aom_codec_iface_t *iface, + const aom_codec_enc_cfg_t *cfg, + aom_codec_flags_t flags, int ver); + +/*!\brief Convenience macro for aom_codec_enc_init_ver() + * + * Ensures the ABI version parameter is properly set. + */ +#define aom_codec_enc_init(ctx, iface, cfg, flags) \ + aom_codec_enc_init_ver(ctx, iface, cfg, flags, AOM_ENCODER_ABI_VERSION) + +/*!\brief Initialize multi-encoder instance + * + * Initializes multi-encoder context using the given interface. + * Applications should call the aom_codec_enc_init_multi convenience macro + * instead of this function directly, to ensure that the ABI version number + * parameter is properly initialized. + * + * \param[in] ctx Pointer to this instance's context. + * \param[in] iface Pointer to the algorithm interface to use. + * \param[in] cfg Configuration to use, if known. May be NULL. + * \param[in] num_enc Total number of encoders. + * \param[in] flags Bitfield of AOM_CODEC_USE_* flags + * \param[in] dsf Pointer to down-sampling factors. + * \param[in] ver ABI version number. Must be set to + * AOM_ENCODER_ABI_VERSION + * \retval #AOM_CODEC_OK + * The decoder algorithm initialized. + * \retval #AOM_CODEC_MEM_ERROR + * Memory allocation failed. + */ +aom_codec_err_t aom_codec_enc_init_multi_ver( + aom_codec_ctx_t *ctx, aom_codec_iface_t *iface, aom_codec_enc_cfg_t *cfg, + int num_enc, aom_codec_flags_t flags, aom_rational_t *dsf, int ver); + +/*!\brief Convenience macro for aom_codec_enc_init_multi_ver() + * + * Ensures the ABI version parameter is properly set. + */ +#define aom_codec_enc_init_multi(ctx, iface, cfg, num_enc, flags, dsf) \ + aom_codec_enc_init_multi_ver(ctx, iface, cfg, num_enc, flags, dsf, \ + AOM_ENCODER_ABI_VERSION) + +/*!\brief Get a default configuration + * + * Initializes a encoder configuration structure with default values. Supports + * the notion of "usages" so that an algorithm may offer different default + * settings depending on the user's intended goal. This function \ref SHOULD + * be called by all applications to initialize the configuration structure + * before specializing the configuration with application specific values. + * + * \param[in] iface Pointer to the algorithm interface to use. + * \param[out] cfg Configuration buffer to populate. + * \param[in] reserved Must set to 0. + * + * \retval #AOM_CODEC_OK + * The configuration was populated. + * \retval #AOM_CODEC_INCAPABLE + * Interface is not an encoder interface. + * \retval #AOM_CODEC_INVALID_PARAM + * A parameter was NULL, or the usage value was not recognized. + */ +aom_codec_err_t aom_codec_enc_config_default(aom_codec_iface_t *iface, + aom_codec_enc_cfg_t *cfg, + unsigned int reserved); + +/*!\brief Set or change configuration + * + * Reconfigures an encoder instance according to the given configuration. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] cfg Configuration buffer to use + * + * \retval #AOM_CODEC_OK + * The configuration was populated. + * \retval #AOM_CODEC_INCAPABLE + * Interface is not an encoder interface. + * \retval #AOM_CODEC_INVALID_PARAM + * A parameter was NULL, or the usage value was not recognized. + */ +aom_codec_err_t aom_codec_enc_config_set(aom_codec_ctx_t *ctx, + const aom_codec_enc_cfg_t *cfg); + +/*!\brief Get global stream headers + * + * Retrieves a stream level global header packet, if supported by the codec. + * + * \param[in] ctx Pointer to this instance's context + * + * \retval NULL + * Encoder does not support global header + * \retval Non-NULL + * Pointer to buffer containing global header packet + */ +aom_fixed_buf_t *aom_codec_get_global_headers(aom_codec_ctx_t *ctx); + +/*!\brief deadline parameter analogous to AVx GOOD QUALITY mode. */ +#define AOM_DL_GOOD_QUALITY (1000000) +/*!\brief Encode a frame + * + * Encodes a video frame at the given "presentation time." The presentation + * time stamp (PTS) \ref MUST be strictly increasing. + * + * The encoder supports the notion of a soft real-time deadline. Given a + * non-zero value to the deadline parameter, the encoder will make a "best + * effort" guarantee to return before the given time slice expires. It is + * implicit that limiting the available time to encode will degrade the + * output quality. The encoder can be given an unlimited time to produce the + * best possible frame by specifying a deadline of '0'. This deadline + * supercedes the AVx notion of "best quality, good quality, realtime". + * Applications that wish to map these former settings to the new deadline + * based system can use the symbol #AOM_DL_GOOD_QUALITY. + * + * When the last frame has been passed to the encoder, this function should + * continue to be called, with the img parameter set to NULL. This will + * signal the end-of-stream condition to the encoder and allow it to encode + * any held buffers. Encoding is complete when aom_codec_encode() is called + * and aom_codec_get_cx_data() returns no data. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] img Image data to encode, NULL to flush. + * \param[in] pts Presentation time stamp, in timebase units. + * \param[in] duration Duration to show frame, in timebase units. + * \param[in] flags Flags to use for encoding this frame. + * \param[in] deadline Time to spend encoding, in microseconds. (0=infinite) + * + * \retval #AOM_CODEC_OK + * The configuration was populated. + * \retval #AOM_CODEC_INCAPABLE + * Interface is not an encoder interface. + * \retval #AOM_CODEC_INVALID_PARAM + * A parameter was NULL, the image format is unsupported, etc. + */ +aom_codec_err_t aom_codec_encode(aom_codec_ctx_t *ctx, const aom_image_t *img, + aom_codec_pts_t pts, unsigned long duration, + aom_enc_frame_flags_t flags, + unsigned long deadline); + +/*!\brief Set compressed data output buffer + * + * Sets the buffer that the codec should output the compressed data + * into. This call effectively sets the buffer pointer returned in the + * next AOM_CODEC_CX_FRAME_PKT packet. Subsequent packets will be + * appended into this buffer. The buffer is preserved across frames, + * so applications must periodically call this function after flushing + * the accumulated compressed data to disk or to the network to reset + * the pointer to the buffer's head. + * + * `pad_before` bytes will be skipped before writing the compressed + * data, and `pad_after` bytes will be appended to the packet. The size + * of the packet will be the sum of the size of the actual compressed + * data, pad_before, and pad_after. The padding bytes will be preserved + * (not overwritten). + * + * Note that calling this function does not guarantee that the returned + * compressed data will be placed into the specified buffer. In the + * event that the encoded data will not fit into the buffer provided, + * the returned packet \ref MAY point to an internal buffer, as it would + * if this call were never used. In this event, the output packet will + * NOT have any padding, and the application must free space and copy it + * to the proper place. This is of particular note in configurations + * that may output multiple packets for a single encoded frame (e.g., lagged + * encoding) or if the application does not reset the buffer periodically. + * + * Applications may restore the default behavior of the codec providing + * the compressed data buffer by calling this function with a NULL + * buffer. + * + * Applications \ref MUSTNOT call this function during iteration of + * aom_codec_get_cx_data(). + * + * \param[in] ctx Pointer to this instance's context + * \param[in] buf Buffer to store compressed data into + * \param[in] pad_before Bytes to skip before writing compressed data + * \param[in] pad_after Bytes to skip after writing compressed data + * + * \retval #AOM_CODEC_OK + * The buffer was set successfully. + * \retval #AOM_CODEC_INVALID_PARAM + * A parameter was NULL, the image format is unsupported, etc. + */ +aom_codec_err_t aom_codec_set_cx_data_buf(aom_codec_ctx_t *ctx, + const aom_fixed_buf_t *buf, + unsigned int pad_before, + unsigned int pad_after); + +/*!\brief Encoded data iterator + * + * Iterates over a list of data packets to be passed from the encoder to the + * application. The different kinds of packets available are enumerated in + * #aom_codec_cx_pkt_kind. + * + * #AOM_CODEC_CX_FRAME_PKT packets should be passed to the application's + * muxer. Multiple compressed frames may be in the list. + * #AOM_CODEC_STATS_PKT packets should be appended to a global buffer. + * + * The application \ref MUST silently ignore any packet kinds that it does + * not recognize or support. + * + * The data buffers returned from this function are only guaranteed to be + * valid until the application makes another call to any aom_codec_* function. + * + * \param[in] ctx Pointer to this instance's context + * \param[in,out] iter Iterator storage, initialized to NULL + * + * \return Returns a pointer to an output data packet (compressed frame data, + * two-pass statistics, etc.) or NULL to signal end-of-list. + * + */ +const aom_codec_cx_pkt_t *aom_codec_get_cx_data(aom_codec_ctx_t *ctx, + aom_codec_iter_t *iter); + +/*!\brief Get Preview Frame + * + * Returns an image that can be used as a preview. Shows the image as it would + * exist at the decompressor. The application \ref MUST NOT write into this + * image buffer. + * + * \param[in] ctx Pointer to this instance's context + * + * \return Returns a pointer to a preview image, or NULL if no image is + * available. + * + */ +const aom_image_t *aom_codec_get_preview_frame(aom_codec_ctx_t *ctx); + +/*!@} - end defgroup encoder*/ +#ifdef __cplusplus +} +#endif +#endif // AOM_AOM_ENCODER_H_ diff --git a/third_party/aom/aom/aom_frame_buffer.h b/third_party/aom/aom/aom_frame_buffer.h new file mode 100644 index 0000000000..c87cf749b6 --- /dev/null +++ b/third_party/aom/aom/aom_frame_buffer.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AOM_FRAME_BUFFER_H_ +#define AOM_AOM_FRAME_BUFFER_H_ + +/*!\file + * \brief Describes the decoder external frame buffer interface. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./aom_integer.h" + +/*!\brief The maximum number of work buffers used by libaom. + * Support maximum 4 threads to decode video in parallel. + * Each thread will use one work buffer. + * TODO(hkuang): Add support to set number of worker threads dynamically. + */ +#define AOM_MAXIMUM_WORK_BUFFERS 8 + +/*!\brief The maximum number of reference buffers that a AV1 encoder may use. + */ +#define AOM_MAXIMUM_REF_BUFFERS 8 + +/*!\brief External frame buffer + * + * This structure holds allocated frame buffers used by the decoder. + */ +typedef struct aom_codec_frame_buffer { + uint8_t *data; /**< Pointer to the data buffer */ + size_t size; /**< Size of data in bytes */ + void *priv; /**< Frame's private data */ +} aom_codec_frame_buffer_t; + +/*!\brief get frame buffer callback prototype + * + * This callback is invoked by the decoder to retrieve data for the frame + * buffer in order for the decode call to complete. The callback must + * allocate at least min_size in bytes and assign it to fb->data. The callback + * must zero out all the data allocated. Then the callback must set fb->size + * to the allocated size. The application does not need to align the allocated + * data. The callback is triggered when the decoder needs a frame buffer to + * decode a compressed image into. This function may be called more than once + * for every call to aom_codec_decode. The application may set fb->priv to + * some data which will be passed back in the ximage and the release function + * call. |fb| is guaranteed to not be NULL. On success the callback must + * return 0. Any failure the callback must return a value less than 0. + * + * \param[in] priv Callback's private data + * \param[in] new_size Size in bytes needed by the buffer + * \param[in,out] fb Pointer to aom_codec_frame_buffer_t + */ +typedef int (*aom_get_frame_buffer_cb_fn_t)(void *priv, size_t min_size, + aom_codec_frame_buffer_t *fb); + +/*!\brief release frame buffer callback prototype + * + * This callback is invoked by the decoder when the frame buffer is not + * referenced by any other buffers. |fb| is guaranteed to not be NULL. On + * success the callback must return 0. Any failure the callback must return + * a value less than 0. + * + * \param[in] priv Callback's private data + * \param[in] fb Pointer to aom_codec_frame_buffer_t + */ +typedef int (*aom_release_frame_buffer_cb_fn_t)(void *priv, + aom_codec_frame_buffer_t *fb); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_AOM_FRAME_BUFFER_H_ diff --git a/third_party/aom/aom/aom_image.h b/third_party/aom/aom/aom_image.h new file mode 100644 index 0000000000..b2f75e6391 --- /dev/null +++ b/third_party/aom/aom/aom_image.h @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/*!\file + * \brief Describes the aom image descriptor and associated operations + * + */ +#ifndef AOM_AOM_IMAGE_H_ +#define AOM_AOM_IMAGE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/*!\brief Current ABI version number + * + * \internal + * If this file is altered in any way that changes the ABI, this value + * must be bumped. Examples include, but are not limited to, changing + * types, removing or reassigning enums, adding/removing/rearranging + * fields to structures + */ +#define AOM_IMAGE_ABI_VERSION (4) /**<\hideinitializer*/ + +#define AOM_IMG_FMT_PLANAR 0x100 /**< Image is a planar format. */ +#define AOM_IMG_FMT_UV_FLIP 0x200 /**< V plane precedes U in memory. */ +#define AOM_IMG_FMT_HAS_ALPHA 0x400 /**< Image has an alpha channel. */ +#define AOM_IMG_FMT_HIGHBITDEPTH 0x800 /**< Image uses 16bit framebuffer. */ + +/*!\brief List of supported image formats */ +typedef enum aom_img_fmt { + AOM_IMG_FMT_NONE, + AOM_IMG_FMT_RGB24, /**< 24 bit per pixel packed RGB */ + AOM_IMG_FMT_RGB32, /**< 32 bit per pixel packed 0RGB */ + AOM_IMG_FMT_RGB565, /**< 16 bit per pixel, 565 */ + AOM_IMG_FMT_RGB555, /**< 16 bit per pixel, 555 */ + AOM_IMG_FMT_UYVY, /**< UYVY packed YUV */ + AOM_IMG_FMT_YUY2, /**< YUYV packed YUV */ + AOM_IMG_FMT_YVYU, /**< YVYU packed YUV */ + AOM_IMG_FMT_BGR24, /**< 24 bit per pixel packed BGR */ + AOM_IMG_FMT_RGB32_LE, /**< 32 bit packed BGR0 */ + AOM_IMG_FMT_ARGB, /**< 32 bit packed ARGB, alpha=255 */ + AOM_IMG_FMT_ARGB_LE, /**< 32 bit packed BGRA, alpha=255 */ + AOM_IMG_FMT_RGB565_LE, /**< 16 bit per pixel, gggbbbbb rrrrrggg */ + AOM_IMG_FMT_RGB555_LE, /**< 16 bit per pixel, gggbbbbb 0rrrrrgg */ + AOM_IMG_FMT_YV12 = + AOM_IMG_FMT_PLANAR | AOM_IMG_FMT_UV_FLIP | 1, /**< planar YVU */ + AOM_IMG_FMT_I420 = AOM_IMG_FMT_PLANAR | 2, + AOM_IMG_FMT_AOMYV12 = AOM_IMG_FMT_PLANAR | AOM_IMG_FMT_UV_FLIP | + 3, /** < planar 4:2:0 format with aom color space */ + AOM_IMG_FMT_AOMI420 = AOM_IMG_FMT_PLANAR | 4, + AOM_IMG_FMT_I422 = AOM_IMG_FMT_PLANAR | 5, + AOM_IMG_FMT_I444 = AOM_IMG_FMT_PLANAR | 6, + AOM_IMG_FMT_I440 = AOM_IMG_FMT_PLANAR | 7, + AOM_IMG_FMT_444A = AOM_IMG_FMT_PLANAR | AOM_IMG_FMT_HAS_ALPHA | 6, + AOM_IMG_FMT_I42016 = AOM_IMG_FMT_I420 | AOM_IMG_FMT_HIGHBITDEPTH, + AOM_IMG_FMT_I42216 = AOM_IMG_FMT_I422 | AOM_IMG_FMT_HIGHBITDEPTH, + AOM_IMG_FMT_I44416 = AOM_IMG_FMT_I444 | AOM_IMG_FMT_HIGHBITDEPTH, + AOM_IMG_FMT_I44016 = AOM_IMG_FMT_I440 | AOM_IMG_FMT_HIGHBITDEPTH +} aom_img_fmt_t; /**< alias for enum aom_img_fmt */ + +/*!\brief List of supported color spaces */ +typedef enum aom_color_space { + AOM_CS_UNKNOWN = 0, /**< Unknown */ + AOM_CS_BT_601 = 1, /**< BT.601 */ + AOM_CS_BT_709 = 2, /**< BT.709 */ + AOM_CS_SMPTE_170 = 3, /**< SMPTE.170 */ + AOM_CS_SMPTE_240 = 4, /**< SMPTE.240 */ + AOM_CS_BT_2020 = 5, /**< BT.2020 */ + AOM_CS_RESERVED = 6, /**< Reserved */ + AOM_CS_SRGB = 7 /**< sRGB */ +} aom_color_space_t; /**< alias for enum aom_color_space */ + +/*!\brief List of supported color range */ +typedef enum aom_color_range { + AOM_CR_STUDIO_RANGE = 0, /**< Y [16..235], UV [16..240] */ + AOM_CR_FULL_RANGE = 1 /**< YUV/RGB [0..255] */ +} aom_color_range_t; /**< alias for enum aom_color_range */ + +/**\brief Image Descriptor */ +typedef struct aom_image { + aom_img_fmt_t fmt; /**< Image Format */ + aom_color_space_t cs; /**< Color Space */ + aom_color_range_t range; /**< Color Range */ + + /* Image storage dimensions */ + unsigned int w; /**< Stored image width */ + unsigned int h; /**< Stored image height */ + unsigned int bit_depth; /**< Stored image bit-depth */ + + /* Image display dimensions */ + unsigned int d_w; /**< Displayed image width */ + unsigned int d_h; /**< Displayed image height */ + + /* Image intended rendering dimensions */ + unsigned int r_w; /**< Intended rendering image width */ + unsigned int r_h; /**< Intended rendering image height */ + + /* Chroma subsampling info */ + unsigned int x_chroma_shift; /**< subsampling order, X */ + unsigned int y_chroma_shift; /**< subsampling order, Y */ + +/* Image data pointers. */ +#define AOM_PLANE_PACKED 0 /**< To be used for all packed formats */ +#define AOM_PLANE_Y 0 /**< Y (Luminance) plane */ +#define AOM_PLANE_U 1 /**< U (Chroma) plane */ +#define AOM_PLANE_V 2 /**< V (Chroma) plane */ +#define AOM_PLANE_ALPHA 3 /**< A (Transparency) plane */ + unsigned char *planes[4]; /**< pointer to the top left pixel for each plane */ + int stride[4]; /**< stride between rows for each plane */ + + int bps; /**< bits per sample (for packed formats) */ + + /*!\brief The following member may be set by the application to associate + * data with this image. + */ + void *user_priv; + + /* The following members should be treated as private. */ + unsigned char *img_data; /**< private */ + int img_data_owner; /**< private */ + int self_allocd; /**< private */ + + void *fb_priv; /**< Frame buffer data associated with the image. */ +} aom_image_t; /**< alias for struct aom_image */ + +/**\brief Representation of a rectangle on a surface */ +typedef struct aom_image_rect { + unsigned int x; /**< leftmost column */ + unsigned int y; /**< topmost row */ + unsigned int w; /**< width */ + unsigned int h; /**< height */ +} aom_image_rect_t; /**< alias for struct aom_image_rect */ + +/*!\brief Open a descriptor, allocating storage for the underlying image + * + * Returns a descriptor for storing an image of the given format. The + * storage for the descriptor is allocated on the heap. + * + * \param[in] img Pointer to storage for descriptor. If this parameter + * is NULL, the storage for the descriptor will be + * allocated on the heap. + * \param[in] fmt Format for the image + * \param[in] d_w Width of the image + * \param[in] d_h Height of the image + * \param[in] align Alignment, in bytes, of the image buffer and + * each row in the image(stride). + * + * \return Returns a pointer to the initialized image descriptor. If the img + * parameter is non-null, the value of the img parameter will be + * returned. + */ +aom_image_t *aom_img_alloc(aom_image_t *img, aom_img_fmt_t fmt, + unsigned int d_w, unsigned int d_h, + unsigned int align); + +/*!\brief Open a descriptor, using existing storage for the underlying image + * + * Returns a descriptor for storing an image of the given format. The + * storage for descriptor has been allocated elsewhere, and a descriptor is + * desired to "wrap" that storage. + * + * \param[in] img Pointer to storage for descriptor. If this parameter + * is NULL, the storage for the descriptor will be + * allocated on the heap. + * \param[in] fmt Format for the image + * \param[in] d_w Width of the image + * \param[in] d_h Height of the image + * \param[in] align Alignment, in bytes, of each row in the image. + * \param[in] img_data Storage to use for the image + * + * \return Returns a pointer to the initialized image descriptor. If the img + * parameter is non-null, the value of the img parameter will be + * returned. + */ +aom_image_t *aom_img_wrap(aom_image_t *img, aom_img_fmt_t fmt, unsigned int d_w, + unsigned int d_h, unsigned int align, + unsigned char *img_data); + +/*!\brief Set the rectangle identifying the displayed portion of the image + * + * Updates the displayed rectangle (aka viewport) on the image surface to + * match the specified coordinates and size. + * + * \param[in] img Image descriptor + * \param[in] x leftmost column + * \param[in] y topmost row + * \param[in] w width + * \param[in] h height + * + * \return 0 if the requested rectangle is valid, nonzero otherwise. + */ +int aom_img_set_rect(aom_image_t *img, unsigned int x, unsigned int y, + unsigned int w, unsigned int h); + +/*!\brief Flip the image vertically (top for bottom) + * + * Adjusts the image descriptor's pointers and strides to make the image + * be referenced upside-down. + * + * \param[in] img Image descriptor + */ +void aom_img_flip(aom_image_t *img); + +/*!\brief Close an image descriptor + * + * Frees all allocated storage associated with an image descriptor. + * + * \param[in] img Image descriptor + */ +void aom_img_free(aom_image_t *img); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_AOM_IMAGE_H_ diff --git a/third_party/aom/aom/aom_integer.h b/third_party/aom/aom/aom_integer.h new file mode 100644 index 0000000000..2e8f23f459 --- /dev/null +++ b/third_party/aom/aom/aom_integer.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AOM_INTEGER_H_ +#define AOM_AOM_INTEGER_H_ + +/* get ptrdiff_t, size_t, wchar_t, NULL */ +#include + +#if defined(_MSC_VER) +#define AOM_FORCE_INLINE __forceinline +#define AOM_INLINE __inline +#else +#define AOM_FORCE_INLINE __inline__ __attribute__((always_inline)) +// TODO(jbb): Allow a way to force inline off for older compilers. +#define AOM_INLINE inline +#endif + +#if defined(AOM_EMULATE_INTTYPES) +typedef signed char int8_t; +typedef signed short int16_t; +typedef signed int int32_t; + +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; + +#ifndef _UINTPTR_T_DEFINED +typedef size_t uintptr_t; +#endif + +#else + +/* Most platforms have the C99 standard integer types. */ + +#if defined(__cplusplus) +#if !defined(__STDC_FORMAT_MACROS) +#define __STDC_FORMAT_MACROS +#endif +#if !defined(__STDC_LIMIT_MACROS) +#define __STDC_LIMIT_MACROS +#endif +#endif // __cplusplus + +#include + +#endif + +/* VS2010 defines stdint.h, but not inttypes.h */ +#if defined(_MSC_VER) && _MSC_VER < 1800 +#define PRId64 "I64d" +#else +#include +#endif + +#endif // AOM_AOM_INTEGER_H_ diff --git a/third_party/aom/aom/aomcx.h b/third_party/aom/aom/aomcx.h new file mode 100644 index 0000000000..debec21b09 --- /dev/null +++ b/third_party/aom/aom/aomcx.h @@ -0,0 +1,750 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_AOMCX_H_ +#define AOM_AOMCX_H_ + +/*!\defgroup aom_encoder AOMedia AOM/AV1 Encoder + * \ingroup aom + * + * @{ + */ +#include "./aom.h" +#include "./aom_encoder.h" + +/*!\file + * \brief Provides definitions for using AOM or AV1 encoder algorithm within the + * aom Codec Interface. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/*!\name Algorithm interface for AV1 + * + * This interface provides the capability to encode raw AV1 streams. + * @{ + */ +extern aom_codec_iface_t aom_codec_av1_cx_algo; +extern aom_codec_iface_t *aom_codec_av1_cx(void); +/*!@} - end algorithm interface member group*/ + +/* + * Algorithm Flags + */ + +/*!\brief Don't reference the last frame + * + * When this flag is set, the encoder will not use the last frame as a + * predictor. When not set, the encoder will choose whether to use the + * last frame or not automatically. + */ +#define AOM_EFLAG_NO_REF_LAST (1 << 16) + +/*!\brief Don't reference the golden frame + * + * When this flag is set, the encoder will not use the golden frame as a + * predictor. When not set, the encoder will choose whether to use the + * golden frame or not automatically. + */ +#define AOM_EFLAG_NO_REF_GF (1 << 17) + +/*!\brief Don't reference the alternate reference frame + * + * When this flag is set, the encoder will not use the alt ref frame as a + * predictor. When not set, the encoder will choose whether to use the + * alt ref frame or not automatically. + */ +#define AOM_EFLAG_NO_REF_ARF (1 << 21) + +/*!\brief Don't update the last frame + * + * When this flag is set, the encoder will not update the last frame with + * the contents of the current frame. + */ +#define AOM_EFLAG_NO_UPD_LAST (1 << 18) + +/*!\brief Don't update the golden frame + * + * When this flag is set, the encoder will not update the golden frame with + * the contents of the current frame. + */ +#define AOM_EFLAG_NO_UPD_GF (1 << 22) + +/*!\brief Don't update the alternate reference frame + * + * When this flag is set, the encoder will not update the alt ref frame with + * the contents of the current frame. + */ +#define AOM_EFLAG_NO_UPD_ARF (1 << 23) + +/*!\brief Force golden frame update + * + * When this flag is set, the encoder copy the contents of the current frame + * to the golden frame buffer. + */ +#define AOM_EFLAG_FORCE_GF (1 << 19) + +/*!\brief Force alternate reference frame update + * + * When this flag is set, the encoder copy the contents of the current frame + * to the alternate reference frame buffer. + */ +#define AOM_EFLAG_FORCE_ARF (1 << 24) + +/*!\brief Disable entropy update + * + * When this flag is set, the encoder will not update its internal entropy + * model based on the entropy of this frame. + */ +#define AOM_EFLAG_NO_UPD_ENTROPY (1 << 20) + +/*!\brief AVx encoder control functions + * + * This set of macros define the control functions available for AVx + * encoder interface. + * + * \sa #aom_codec_control + */ +enum aome_enc_control_id { + /*!\brief Codec control function to set which reference frame encoder can use. + */ + AOME_USE_REFERENCE = 7, + + /*!\brief Codec control function to pass an ROI map to encoder. + */ + AOME_SET_ROI_MAP = 8, + + /*!\brief Codec control function to pass an Active map to encoder. + */ + AOME_SET_ACTIVEMAP, + + /*!\brief Codec control function to set encoder scaling mode. + */ + AOME_SET_SCALEMODE = 11, + + /*!\brief Codec control function to set encoder internal speed settings. + * + * Changes in this value influences, among others, the encoder's selection + * of motion estimation methods. Values greater than 0 will increase encoder + * speed at the expense of quality. + * + * \note Valid range: 0..8 + */ + AOME_SET_CPUUSED = 13, + + /*!\brief Codec control function to enable automatic set and use alf frames. + */ + AOME_SET_ENABLEAUTOALTREF, + + /*!\brief Codec control function to set sharpness. + */ + AOME_SET_SHARPNESS = AOME_SET_ENABLEAUTOALTREF + 2, + + /*!\brief Codec control function to set the threshold for MBs treated static. + */ + AOME_SET_STATIC_THRESHOLD, + + /*!\brief Codec control function to get last quantizer chosen by the encoder. + * + * Return value uses internal quantizer scale defined by the codec. + */ + AOME_GET_LAST_QUANTIZER = AOME_SET_STATIC_THRESHOLD + 2, + + /*!\brief Codec control function to get last quantizer chosen by the encoder. + * + * Return value uses the 0..63 scale as used by the rc_*_quantizer config + * parameters. + */ + AOME_GET_LAST_QUANTIZER_64, + + /*!\brief Codec control function to set the max no of frames to create arf. + */ + AOME_SET_ARNR_MAXFRAMES, + + /*!\brief Codec control function to set the filter strength for the arf. + */ + AOME_SET_ARNR_STRENGTH, + + /*!\brief Codec control function to set visual tuning. + */ + AOME_SET_TUNING = AOME_SET_ARNR_STRENGTH + 2, + + /*!\brief Codec control function to set constrained quality level. + * + * \attention For this value to be used aom_codec_enc_cfg_t::g_usage must be + * set to #AOM_CQ. + * \note Valid range: 0..63 + */ + AOME_SET_CQ_LEVEL, + + /*!\brief Codec control function to set Max data rate for Intra frames. + * + * This value controls additional clamping on the maximum size of a + * keyframe. It is expressed as a percentage of the average + * per-frame bitrate, with the special (and default) value 0 meaning + * unlimited, or no additional clamping beyond the codec's built-in + * algorithm. + * + * For example, to allocate no more than 4.5 frames worth of bitrate + * to a keyframe, set this to 450. + */ + AOME_SET_MAX_INTRA_BITRATE_PCT, + + /*!\brief Codec control function to set max data rate for Inter frames. + * + * This value controls additional clamping on the maximum size of an + * inter frame. It is expressed as a percentage of the average + * per-frame bitrate, with the special (and default) value 0 meaning + * unlimited, or no additional clamping beyond the codec's built-in + * algorithm. + * + * For example, to allow no more than 4.5 frames worth of bitrate + * to an inter frame, set this to 450. + */ + AV1E_SET_MAX_INTER_BITRATE_PCT = AOME_SET_MAX_INTRA_BITRATE_PCT + 2, + + /*!\brief Boost percentage for Golden Frame in CBR mode. + * + * This value controls the amount of boost given to Golden Frame in + * CBR mode. It is expressed as a percentage of the average + * per-frame bitrate, with the special (and default) value 0 meaning + * the feature is off, i.e., no golden frame boost in CBR mode and + * average bitrate target is used. + * + * For example, to allow 100% more bits, i.e, 2X, in a golden frame + * than average frame, set this to 100. + */ + AV1E_SET_GF_CBR_BOOST_PCT, + + /*!\brief Codec control function to set lossless encoding mode. + * + * AV1 can operate in lossless encoding mode, in which the bitstream + * produced will be able to decode and reconstruct a perfect copy of + * input source. This control function provides a mean to switch encoder + * into lossless coding mode(1) or normal coding mode(0) that may be lossy. + * 0 = lossy coding mode + * 1 = lossless coding mode + * + * By default, encoder operates in normal coding mode (maybe lossy). + */ + AV1E_SET_LOSSLESS = AV1E_SET_GF_CBR_BOOST_PCT + 2, + + /*!\brief Codec control function to set number of tile columns. + * + * In encoding and decoding, AV1 allows an input image frame be partitioned + * into separated vertical tile columns, which can be encoded or decoded + * independently. This enables easy implementation of parallel encoding and + * decoding. This control requests the encoder to use column tiles in + * encoding an input frame, with number of tile columns (in Log2 unit) as + * the parameter: + * 0 = 1 tile column + * 1 = 2 tile columns + * 2 = 4 tile columns + * ..... + * n = 2**n tile columns + * The requested tile columns will be capped by encoder based on image size + * limitation (The minimum width of a tile column is 256 pixel, the maximum + * is 4096). + * + * By default, the value is 0, i.e. one single column tile for entire image. + */ + AV1E_SET_TILE_COLUMNS, + + /*!\brief Codec control function to set number of tile rows. + * + * In encoding and decoding, AV1 allows an input image frame be partitioned + * into separated horizontal tile rows. Tile rows are encoded or decoded + * sequentially. Even though encoding/decoding of later tile rows depends on + * earlier ones, this allows the encoder to output data packets for tile rows + * prior to completely processing all tile rows in a frame, thereby reducing + * the latency in processing between input and output. The parameter + * for this control describes the number of tile rows, which has a valid + * range [0, 2]: + * 0 = 1 tile row + * 1 = 2 tile rows + * 2 = 4 tile rows + * + * By default, the value is 0, i.e. one single row tile for entire image. + */ + AV1E_SET_TILE_ROWS, + + /*!\brief Codec control function to enable frame parallel decoding feature. + * + * AV1 has a bitstream feature to reduce decoding dependency between frames + * by turning off backward update of probability context used in encoding + * and decoding. This allows staged parallel processing of more than one + * video frames in the decoder. This control function provides a mean to + * turn this feature on or off for bitstreams produced by encoder. + * + * By default, this feature is off. + */ + AV1E_SET_FRAME_PARALLEL_DECODING, + + /*!\brief Codec control function to set adaptive quantization mode. + * + * AV1 has a segment based feature that allows encoder to adaptively change + * quantization parameter for each segment within a frame to improve the + * subjective quality. This control makes encoder operate in one of the + * several AQ_modes supported. + * + * By default, encoder operates with AQ_Mode 0(adaptive quantization off). + */ + AV1E_SET_AQ_MODE, + + /*!\brief Codec control function to enable/disable periodic Q boost. + * + * One AV1 encoder speed feature is to enable quality boost by lowering + * frame level Q periodically. This control function provides a mean to + * turn on/off this feature. + * 0 = off + * 1 = on + * + * By default, the encoder is allowed to use this feature for appropriate + * encoding modes. + */ + AV1E_SET_FRAME_PERIODIC_BOOST, + + /*!\brief Codec control function to set noise sensitivity. + * + * 0: off, 1: On(YOnly) + */ + AV1E_SET_NOISE_SENSITIVITY, + + /*!\brief Codec control function to set content type. + * \note Valid parameter range: + * AOM_CONTENT_DEFAULT = Regular video content (Default) + * AOM_CONTENT_SCREEN = Screen capture content + */ + AV1E_SET_TUNE_CONTENT, + + /*!\brief Codec control function to set color space info. + * \note Valid ranges: 0..7, default is "UNKNOWN". + * 0 = UNKNOWN, + * 1 = BT_601 + * 2 = BT_709 + * 3 = SMPTE_170 + * 4 = SMPTE_240 + * 5 = BT_2020 + * 6 = RESERVED + * 7 = SRGB + */ + AV1E_SET_COLOR_SPACE, + + /*!\brief Codec control function to set minimum interval between GF/ARF frames + * + * By default the value is set as 4. + */ + AV1E_SET_MIN_GF_INTERVAL, + + /*!\brief Codec control function to set minimum interval between GF/ARF frames + * + * By default the value is set as 16. + */ + AV1E_SET_MAX_GF_INTERVAL, + + /*!\brief Codec control function to get an Active map back from the encoder. + */ + AV1E_GET_ACTIVEMAP, + + /*!\brief Codec control function to set color range bit. + * \note Valid ranges: 0..1, default is 0 + * 0 = Limited range (16..235 or HBD equivalent) + * 1 = Full range (0..255 or HBD equivalent) + */ + AV1E_SET_COLOR_RANGE, + + /*!\brief Codec control function to set intended rendering image size. + * + * By default, this is identical to the image size in pixels. + */ + AV1E_SET_RENDER_SIZE, + + /*!\brief Codec control function to set target level. + * + * 255: off (default); 0: only keep level stats; 10: target for level 1.0; + * 11: target for level 1.1; ... 62: target for level 6.2 + */ + AV1E_SET_TARGET_LEVEL, + + /*!\brief Codec control function to get bitstream level. + */ + AV1E_GET_LEVEL, + + /*!\brief Codec control function to set intended superblock size. + * + * By default, the superblock size is determined separately for each + * frame by the encoder. + * + * Experiment: EXT_PARTITION + */ + AV1E_SET_SUPERBLOCK_SIZE, + + /*!\brief Codec control function to enable automatic set and use + * bwd-pred frames. + * + * Experiment: EXT_REFS + */ + AOME_SET_ENABLEAUTOBWDREF, + + /*!\brief Codec control function to encode with quantisation matrices. + * + * AOM can operate with default quantisation matrices dependent on + * quantisation level and block type. + * 0 = do not use quantisation matrices + * 1 = use quantisation matrices + * + * By default, the encoder operates without quantisation matrices. + * + * Experiment: AOM_QM + */ + AV1E_SET_ENABLE_QM, + + /*!\brief Codec control function to set the min quant matrix flatness. + * + * AOM can operate with different ranges of quantisation matrices. + * As quantisation levels increase, the matrices get flatter. This + * control sets the minimum level of flatness from which the matrices + * are determined. + * + * By default, the encoder sets this minimum at half the available + * range. + * + * Experiment: AOM_QM + */ + AV1E_SET_QM_MIN, + + /*!\brief Codec control function to set the max quant matrix flatness. + * + * AOM can operate with different ranges of quantisation matrices. + * As quantisation levels increase, the matrices get flatter. This + * control sets the maximum level of flatness possible. + * + * By default, the encoder sets this maximum at the top of the + * available range. + * + * Experiment: AOM_QM + */ + AV1E_SET_QM_MAX, + + /*!\brief Codec control function to set a maximum number of tile groups. + * + * This will set the maximum number of tile groups. This will be + * overridden if an MTU size is set. The default value is 1. + * + * Experiment: TILE_GROUPS + */ + AV1E_SET_NUM_TG, + + /*!\brief Codec control function to set an MTU size for a tile group. + * + * This will set the maximum number of bytes in a tile group. This can be + * exceeded only if a single tile is larger than this amount. + * + * By default, the value is 0, in which case a fixed number of tile groups + * is used. + * + * Experiment: TILE_GROUPS + */ + AV1E_SET_MTU, + + /*!\brief Codec control function to set dependent_horz_tiles. + * + * In encoding and decoding, AV1 allows enabling dependent horizontal tile + * The parameter for this control describes the value of this flag, + * which has a valid range [0, 1]: + * 0 = disable dependent horizontal tile + * 1 = enable dependent horizontal tile, + * + * By default, the value is 0, i.e. disable dependent horizontal tile. + */ + AV1E_SET_TILE_DEPENDENT_ROWS, + + /*!\brief Codec control function to set the number of symbols in an ANS data + * window. + * + * The number of ANS symbols (both boolean and non-booleans alphabets) in an + * ANS data window is set to 1 << value. + * + * \note Valid range: [8, 23] + * + * Experiment: ANS + */ + AV1E_SET_ANS_WINDOW_SIZE_LOG2, + + /*!\brief Codec control function to set temporal mv prediction + * enabling/disabling. + * + * This will enable or disable temporal mv predicton. The default value is 0. + * + * Experiment: TEMPMV_SIGNALING + */ + AV1E_SET_DISABLE_TEMPMV, + + /*!\brief Codec control function to set loop_filter_across_tiles_enabled. + * + * In encoding and decoding, AV1 allows disabling loop filter across tile + * boundary The parameter for this control describes the value of this flag, + * which has a valid range [0, 1]: + * 0 = disable loop filter across tile boundary + * 1 = enable loop filter across tile boundary + * + * By default, the value is 1, i.e. enable loop filter across tile boundary. + * + * Experiment: LOOPFILTERING_ACROSS_TILES + */ + AV1E_SET_TILE_LOOPFILTER, + + /*!\brief Codec control function to set the delta q mode + * + * AV1 has a segment based feature that allows encoder to adaptively change + * quantization parameter for each segment within a frame to improve the + * subjective quality. the delta q mode is added on top of segment based + * feature, and allows control per 64x64 q and lf delta.This control makes + * encoder operate in one of the several DELTA_Q_modes supported. + * + * By default, encoder operates with DELTAQ_Mode 0(deltaq signaling off). + */ + AV1E_SET_DELTAQ_MODE, + + /*!\brief Codec control function to set the tile encoding mode to 0 or 1. + * + * 0 means that the tile encoding mode is TILE_NORMAL, and 1 means that the + * tile encoding mode is TILE_VR. + * + * Experiment: EXT_TILE + */ + AV1E_SET_TILE_ENCODING_MODE, + + /*!\brief Codec control function to enable the extreme motion vector unit test + * in AV1. Please note that this is only used in motion vector unit test. + * + * 0 : off, 1 : MAX_EXTREME_MV, 2 : MIN_EXTREME_MV + */ + AV1E_ENABLE_MOTION_VECTOR_UNIT_TEST, +}; + +/*!\brief aom 1-D scaling mode + * + * This set of constants define 1-D aom scaling modes + */ +typedef enum aom_scaling_mode_1d { + AOME_NORMAL = 0, + AOME_FOURFIVE = 1, + AOME_THREEFIVE = 2, + AOME_ONETWO = 3 +} AOM_SCALING_MODE; + +/*!\brief aom region of interest map + * + * These defines the data structures for the region of interest map + * + */ + +typedef struct aom_roi_map { + /*! An id between 0 and 3 for each 16x16 region within a frame. */ + unsigned char *roi_map; + unsigned int rows; /**< Number of rows. */ + unsigned int cols; /**< Number of columns. */ + // TODO(paulwilkins): broken for AV1 which has 8 segments + // q and loop filter deltas for each segment + // (see MAX_MB_SEGMENTS) + int delta_q[4]; /**< Quantizer deltas. */ + int delta_lf[4]; /**< Loop filter deltas. */ + /*! Static breakout threshold for each segment. */ + unsigned int static_threshold[4]; +} aom_roi_map_t; + +/*!\brief aom active region map + * + * These defines the data structures for active region map + * + */ + +typedef struct aom_active_map { + /*!\brief specify an on (1) or off (0) each 16x16 region within a frame */ + unsigned char *active_map; + unsigned int rows; /**< number of rows */ + unsigned int cols; /**< number of cols */ +} aom_active_map_t; + +/*!\brief aom image scaling mode + * + * This defines the data structure for image scaling mode + * + */ +typedef struct aom_scaling_mode { + AOM_SCALING_MODE h_scaling_mode; /**< horizontal scaling mode */ + AOM_SCALING_MODE v_scaling_mode; /**< vertical scaling mode */ +} aom_scaling_mode_t; + +/*!brief AV1 encoder content type */ +typedef enum { + AOM_CONTENT_DEFAULT, + AOM_CONTENT_SCREEN, + AOM_CONTENT_INVALID +} aom_tune_content; + +/*!\brief Model tuning parameters + * + * Changes the encoder to tune for certain types of input material. + * + */ +typedef enum { AOM_TUNE_PSNR, AOM_TUNE_SSIM } aom_tune_metric; + +/*!\cond */ +/*!\brief Encoder control function parameter type + * + * Defines the data types that AOME/AV1E control functions take. Note that + * additional common controls are defined in aom.h + * + */ + +AOM_CTRL_USE_TYPE_DEPRECATED(AOME_USE_REFERENCE, int) +#define AOM_CTRL_AOME_USE_REFERENCE +AOM_CTRL_USE_TYPE(AOME_SET_ROI_MAP, aom_roi_map_t *) +#define AOM_CTRL_AOME_SET_ROI_MAP +AOM_CTRL_USE_TYPE(AOME_SET_ACTIVEMAP, aom_active_map_t *) +#define AOM_CTRL_AOME_SET_ACTIVEMAP +AOM_CTRL_USE_TYPE(AOME_SET_SCALEMODE, aom_scaling_mode_t *) +#define AOM_CTRL_AOME_SET_SCALEMODE + +AOM_CTRL_USE_TYPE(AOME_SET_CPUUSED, int) +#define AOM_CTRL_AOME_SET_CPUUSED +AOM_CTRL_USE_TYPE(AOME_SET_ENABLEAUTOALTREF, unsigned int) +#define AOM_CTRL_AOME_SET_ENABLEAUTOALTREF + +AOM_CTRL_USE_TYPE(AOME_SET_ENABLEAUTOBWDREF, unsigned int) +#define AOM_CTRL_AOME_SET_ENABLEAUTOBWDREF + +AOM_CTRL_USE_TYPE(AOME_SET_SHARPNESS, unsigned int) +#define AOM_CTRL_AOME_SET_SHARPNESS +AOM_CTRL_USE_TYPE(AOME_SET_STATIC_THRESHOLD, unsigned int) +#define AOM_CTRL_AOME_SET_STATIC_THRESHOLD + +AOM_CTRL_USE_TYPE(AOME_SET_ARNR_MAXFRAMES, unsigned int) +#define AOM_CTRL_AOME_SET_ARNR_MAXFRAMES +AOM_CTRL_USE_TYPE(AOME_SET_ARNR_STRENGTH, unsigned int) +#define AOM_CTRL_AOME_SET_ARNR_STRENGTH +AOM_CTRL_USE_TYPE(AOME_SET_TUNING, int) /* aom_tune_metric */ +#define AOM_CTRL_AOME_SET_TUNING +AOM_CTRL_USE_TYPE(AOME_SET_CQ_LEVEL, unsigned int) +#define AOM_CTRL_AOME_SET_CQ_LEVEL + +AOM_CTRL_USE_TYPE(AV1E_SET_TILE_COLUMNS, int) +#define AOM_CTRL_AV1E_SET_TILE_COLUMNS +AOM_CTRL_USE_TYPE(AV1E_SET_TILE_ROWS, int) +#define AOM_CTRL_AV1E_SET_TILE_ROWS + +AOM_CTRL_USE_TYPE(AV1E_SET_TILE_DEPENDENT_ROWS, int) +#define AOM_CTRL_AV1E_SET_TILE_DEPENDENT_ROWS + +AOM_CTRL_USE_TYPE(AV1E_SET_TILE_LOOPFILTER, int) +#define AOM_CTRL_AV1E_SET_TILE_LOOPFILTER + +AOM_CTRL_USE_TYPE(AOME_GET_LAST_QUANTIZER, int *) +#define AOM_CTRL_AOME_GET_LAST_QUANTIZER +AOM_CTRL_USE_TYPE(AOME_GET_LAST_QUANTIZER_64, int *) +#define AOM_CTRL_AOME_GET_LAST_QUANTIZER_64 + +AOM_CTRL_USE_TYPE(AOME_SET_MAX_INTRA_BITRATE_PCT, unsigned int) +#define AOM_CTRL_AOME_SET_MAX_INTRA_BITRATE_PCT +AOM_CTRL_USE_TYPE(AOME_SET_MAX_INTER_BITRATE_PCT, unsigned int) +#define AOM_CTRL_AOME_SET_MAX_INTER_BITRATE_PCT + +AOM_CTRL_USE_TYPE(AV1E_SET_GF_CBR_BOOST_PCT, unsigned int) +#define AOM_CTRL_AV1E_SET_GF_CBR_BOOST_PCT + +AOM_CTRL_USE_TYPE(AV1E_SET_LOSSLESS, unsigned int) +#define AOM_CTRL_AV1E_SET_LOSSLESS + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_QM, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_QM + +AOM_CTRL_USE_TYPE(AV1E_SET_QM_MIN, unsigned int) +#define AOM_CTRL_AV1E_SET_QM_MIN + +AOM_CTRL_USE_TYPE(AV1E_SET_QM_MAX, unsigned int) +#define AOM_CTRL_AV1E_SET_QM_MAX + +AOM_CTRL_USE_TYPE(AV1E_SET_NUM_TG, unsigned int) +#define AOM_CTRL_AV1E_SET_NUM_TG +AOM_CTRL_USE_TYPE(AV1E_SET_MTU, unsigned int) +#define AOM_CTRL_AV1E_SET_MTU + +AOM_CTRL_USE_TYPE(AV1E_SET_DISABLE_TEMPMV, unsigned int) +#define AOM_CTRL_AV1E_SET_DISABLE_TEMPMV + +AOM_CTRL_USE_TYPE(AV1E_SET_FRAME_PARALLEL_DECODING, unsigned int) +#define AOM_CTRL_AV1E_SET_FRAME_PARALLEL_DECODING + +AOM_CTRL_USE_TYPE(AV1E_SET_AQ_MODE, unsigned int) +#define AOM_CTRL_AV1E_SET_AQ_MODE + +AOM_CTRL_USE_TYPE(AV1E_SET_DELTAQ_MODE, unsigned int) +#define AOM_CTRL_AV1E_SET_DELTAQ_MODE + +AOM_CTRL_USE_TYPE(AV1E_SET_FRAME_PERIODIC_BOOST, unsigned int) +#define AOM_CTRL_AV1E_SET_FRAME_PERIODIC_BOOST + +AOM_CTRL_USE_TYPE(AV1E_SET_NOISE_SENSITIVITY, unsigned int) +#define AOM_CTRL_AV1E_SET_NOISE_SENSITIVITY + +AOM_CTRL_USE_TYPE(AV1E_SET_TUNE_CONTENT, int) /* aom_tune_content */ +#define AOM_CTRL_AV1E_SET_TUNE_CONTENT + +AOM_CTRL_USE_TYPE(AV1E_SET_COLOR_SPACE, int) +#define AOM_CTRL_AV1E_SET_COLOR_SPACE + +AOM_CTRL_USE_TYPE(AV1E_SET_MIN_GF_INTERVAL, unsigned int) +#define AOM_CTRL_AV1E_SET_MIN_GF_INTERVAL + +AOM_CTRL_USE_TYPE(AV1E_SET_MAX_GF_INTERVAL, unsigned int) +#define AOM_CTRL_AV1E_SET_MAX_GF_INTERVAL + +AOM_CTRL_USE_TYPE(AV1E_GET_ACTIVEMAP, aom_active_map_t *) +#define AOM_CTRL_AV1E_GET_ACTIVEMAP + +AOM_CTRL_USE_TYPE(AV1E_SET_COLOR_RANGE, int) +#define AOM_CTRL_AV1E_SET_COLOR_RANGE + +/*!\brief + * + * TODO(rbultje) : add support of the control in ffmpeg + */ +#define AOM_CTRL_AV1E_SET_RENDER_SIZE +AOM_CTRL_USE_TYPE(AV1E_SET_RENDER_SIZE, int *) + +AOM_CTRL_USE_TYPE(AV1E_SET_SUPERBLOCK_SIZE, unsigned int) +#define AOM_CTRL_AV1E_SET_SUPERBLOCK_SIZE + +AOM_CTRL_USE_TYPE(AV1E_SET_TARGET_LEVEL, unsigned int) +#define AOM_CTRL_AV1E_SET_TARGET_LEVEL + +AOM_CTRL_USE_TYPE(AV1E_GET_LEVEL, int *) +#define AOM_CTRL_AV1E_GET_LEVEL + +AOM_CTRL_USE_TYPE(AV1E_SET_ANS_WINDOW_SIZE_LOG2, unsigned int) +#define AOM_CTRL_AV1E_SET_ANS_WINDOW_SIZE_LOG2 + +AOM_CTRL_USE_TYPE(AV1E_SET_TILE_ENCODING_MODE, unsigned int) +#define AOM_CTRL_AV1E_SET_TILE_ENCODING_MODE + +AOM_CTRL_USE_TYPE(AV1E_ENABLE_MOTION_VECTOR_UNIT_TEST, unsigned int) +#define AOM_CTRL_AV1E_ENABLE_MOTION_VECTOR_UNIT_TEST + +/*!\endcond */ +/*! @} - end defgroup aom_encoder */ +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_AOMCX_H_ diff --git a/third_party/aom/aom/aomdx.h b/third_party/aom/aom/aomdx.h new file mode 100644 index 0000000000..99f50c17ea --- /dev/null +++ b/third_party/aom/aom/aomdx.h @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/*!\defgroup aom_decoder AOMedia AOM/AV1 Decoder + * \ingroup aom + * + * @{ + */ +/*!\file + * \brief Provides definitions for using AOM or AV1 within the aom Decoder + * interface. + */ +#ifndef AOM_AOMDX_H_ +#define AOM_AOMDX_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Include controls common to both the encoder and decoder */ +#include "./aom.h" + +/*!\name Algorithm interface for AV1 + * + * This interface provides the capability to decode AV1 streams. + * @{ + */ +extern aom_codec_iface_t aom_codec_av1_dx_algo; +extern aom_codec_iface_t *aom_codec_av1_dx(void); +/*!@} - end algorithm interface member group*/ + +/** Data structure that stores bit accounting for debug + */ +typedef struct Accounting Accounting; + +/** Callback that inspects decoder frame data. + */ +typedef void (*aom_inspect_cb)(void *decoder, void *ctx); + +/*!\brief Structure to hold inspection callback and context. + * + * Defines a structure to hold the inspection callback function and calling + * context. + */ +typedef struct aom_inspect_init { + /*! Inspection callback. */ + aom_inspect_cb inspect_cb; + + /*! Inspection context. */ + void *inspect_ctx; +} aom_inspect_init; + +/*!\enum aom_dec_control_id + * \brief AOM decoder control functions + * + * This set of macros define the control functions available for the AOM + * decoder interface. + * + * \sa #aom_codec_control + */ +enum aom_dec_control_id { + /** control function to get info on which reference frames were updated + * by the last decode + */ + AOMD_GET_LAST_REF_UPDATES = AOM_DECODER_CTRL_ID_START, + + /** check if the indicated frame is corrupted */ + AOMD_GET_FRAME_CORRUPTED, + + /** control function to get info on which reference frames were used + * by the last decode + */ + AOMD_GET_LAST_REF_USED, + + /** decryption function to decrypt encoded buffer data immediately + * before decoding. Takes a aom_decrypt_init, which contains + * a callback function and opaque context pointer. + */ + AOMD_SET_DECRYPTOR, + // AOMD_SET_DECRYPTOR = AOMD_SET_DECRYPTOR, + + /** control function to get the dimensions that the current frame is decoded + * at. This may be different to the intended display size for the frame as + * specified in the wrapper or frame header (see AV1D_GET_DISPLAY_SIZE). */ + AV1D_GET_FRAME_SIZE, + + /** control function to get the current frame's intended display dimensions + * (as specified in the wrapper or frame header). This may be different to + * the decoded dimensions of this frame (see AV1D_GET_FRAME_SIZE). */ + AV1D_GET_DISPLAY_SIZE, + + /** control function to get the bit depth of the stream. */ + AV1D_GET_BIT_DEPTH, + + /** control function to set the byte alignment of the planes in the reference + * buffers. Valid values are power of 2, from 32 to 1024. A value of 0 sets + * legacy alignment. I.e. Y plane is aligned to 32 bytes, U plane directly + * follows Y plane, and V plane directly follows U plane. Default value is 0. + */ + AV1_SET_BYTE_ALIGNMENT, + + /** control function to invert the decoding order to from right to left. The + * function is used in a test to confirm the decoding independence of tile + * columns. The function may be used in application where this order + * of decoding is desired. + * + * TODO(yaowu): Rework the unit test that uses this control, and in a future + * release, this test-only control shall be removed. + */ + AV1_INVERT_TILE_DECODE_ORDER, + + /** control function to set the skip loop filter flag. Valid values are + * integers. The decoder will skip the loop filter when its value is set to + * nonzero. If the loop filter is skipped the decoder may accumulate decode + * artifacts. The default value is 0. + */ + AV1_SET_SKIP_LOOP_FILTER, + + /** control function to retrieve a pointer to the Accounting struct. When + * compiled without --enable-accounting, this returns AOM_CODEC_INCAPABLE. + * If called before a frame has been decoded, this returns AOM_CODEC_ERROR. + * The caller should ensure that AOM_CODEC_OK is returned before attempting + * to dereference the Accounting pointer. + */ + AV1_GET_ACCOUNTING, + + /** control function to get last decoded frame quantizer. Returned value uses + * internal quantizer scale defined by the codec. + */ + AOMD_GET_LAST_QUANTIZER, + + /** control function to set the range of tile decoding. A value that is + * greater and equal to zero indicates only the specific row/column is + * decoded. A value that is -1 indicates the whole row/column is decoded. + * A special case is both values are -1 that means the whole frame is + * decoded. + */ + AV1_SET_DECODE_TILE_ROW, + AV1_SET_DECODE_TILE_COL, + + /** control function to set an aom_inspect_cb callback that is invoked each + * time a frame is decoded. When compiled without --enable-inspection, this + * returns AOM_CODEC_INCAPABLE. + */ + AV1_SET_INSPECTION_CALLBACK, + + AOM_DECODER_CTRL_ID_MAX, +}; + +/** Decrypt n bytes of data from input -> output, using the decrypt_state + * passed in AOMD_SET_DECRYPTOR. + */ +typedef void (*aom_decrypt_cb)(void *decrypt_state, const unsigned char *input, + unsigned char *output, int count); + +/*!\brief Structure to hold decryption state + * + * Defines a structure to hold the decryption state and access function. + */ +typedef struct aom_decrypt_init { + /*! Decrypt callback. */ + aom_decrypt_cb decrypt_cb; + + /*! Decryption state. */ + void *decrypt_state; +} aom_decrypt_init; + +/*!\cond */ +/*!\brief AOM decoder control function parameter type + * + * Defines the data types that AOMD control functions take. Note that + * additional common controls are defined in aom.h + * + */ + +AOM_CTRL_USE_TYPE(AOMD_GET_LAST_REF_UPDATES, int *) +#define AOM_CTRL_AOMD_GET_LAST_REF_UPDATES +AOM_CTRL_USE_TYPE(AOMD_GET_FRAME_CORRUPTED, int *) +#define AOM_CTRL_AOMD_GET_FRAME_CORRUPTED +AOM_CTRL_USE_TYPE(AOMD_GET_LAST_REF_USED, int *) +#define AOM_CTRL_AOMD_GET_LAST_REF_USED +AOM_CTRL_USE_TYPE(AOMD_GET_LAST_QUANTIZER, int *) +#define AOM_CTRL_AOMD_GET_LAST_QUANTIZER +AOM_CTRL_USE_TYPE(AOMD_SET_DECRYPTOR, aom_decrypt_init *) +#define AOM_CTRL_AOMD_SET_DECRYPTOR +// AOM_CTRL_USE_TYPE(AOMD_SET_DECRYPTOR, aom_decrypt_init *) +//#define AOM_CTRL_AOMD_SET_DECRYPTOR +AOM_CTRL_USE_TYPE(AV1D_GET_DISPLAY_SIZE, int *) +#define AOM_CTRL_AV1D_GET_DISPLAY_SIZE +AOM_CTRL_USE_TYPE(AV1D_GET_BIT_DEPTH, unsigned int *) +#define AOM_CTRL_AV1D_GET_BIT_DEPTH +AOM_CTRL_USE_TYPE(AV1D_GET_FRAME_SIZE, int *) +#define AOM_CTRL_AV1D_GET_FRAME_SIZE +AOM_CTRL_USE_TYPE(AV1_INVERT_TILE_DECODE_ORDER, int) +#define AOM_CTRL_AV1_INVERT_TILE_DECODE_ORDER +AOM_CTRL_USE_TYPE(AV1_GET_ACCOUNTING, Accounting **) +#define AOM_CTRL_AV1_GET_ACCOUNTING +AOM_CTRL_USE_TYPE(AV1_SET_DECODE_TILE_ROW, int) +#define AOM_CTRL_AV1_SET_DECODE_TILE_ROW +AOM_CTRL_USE_TYPE(AV1_SET_DECODE_TILE_COL, int) +#define AOM_CTRL_AV1_SET_DECODE_TILE_COL +AOM_CTRL_USE_TYPE(AV1_SET_INSPECTION_CALLBACK, aom_inspect_init *) +#define AOM_CTRL_AV1_SET_INSPECTION_CALLBACK +/*!\endcond */ +/*! @} - end defgroup aom_decoder */ + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_AOMDX_H_ diff --git a/third_party/aom/aom/exports_com b/third_party/aom/aom/exports_com new file mode 100644 index 0000000000..0c79fa124b --- /dev/null +++ b/third_party/aom/aom/exports_com @@ -0,0 +1,16 @@ +text aom_codec_build_config +text aom_codec_control_ +text aom_codec_destroy +text aom_codec_err_to_string +text aom_codec_error +text aom_codec_error_detail +text aom_codec_get_caps +text aom_codec_iface_name +text aom_codec_version +text aom_codec_version_extra_str +text aom_codec_version_str +text aom_img_alloc +text aom_img_flip +text aom_img_free +text aom_img_set_rect +text aom_img_wrap diff --git a/third_party/aom/aom/exports_dec b/third_party/aom/aom/exports_dec new file mode 100644 index 0000000000..de8fe449a9 --- /dev/null +++ b/third_party/aom/aom/exports_dec @@ -0,0 +1,8 @@ +text aom_codec_dec_init_ver +text aom_codec_decode +text aom_codec_get_frame +text aom_codec_get_stream_info +text aom_codec_peek_stream_info +text aom_codec_register_put_frame_cb +text aom_codec_register_put_slice_cb +text aom_codec_set_frame_buffer_functions diff --git a/third_party/aom/aom/exports_enc b/third_party/aom/aom/exports_enc new file mode 100644 index 0000000000..0dcca7da35 --- /dev/null +++ b/third_party/aom/aom/exports_enc @@ -0,0 +1,9 @@ +text aom_codec_enc_config_default +text aom_codec_enc_config_set +text aom_codec_enc_init_multi_ver +text aom_codec_enc_init_ver +text aom_codec_encode +text aom_codec_get_cx_data +text aom_codec_get_global_headers +text aom_codec_get_preview_frame +text aom_codec_set_cx_data_buf diff --git a/third_party/aom/aom/internal/aom_codec_internal.h b/third_party/aom/aom/internal/aom_codec_internal.h new file mode 100644 index 0000000000..7f9df7a722 --- /dev/null +++ b/third_party/aom/aom/internal/aom_codec_internal.h @@ -0,0 +1,467 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/*!\file + * \brief Describes the decoder algorithm interface for algorithm + * implementations. + * + * This file defines the private structures and data types that are only + * relevant to implementing an algorithm, as opposed to using it. + * + * To create a decoder algorithm class, an interface structure is put + * into the global namespace: + * 
+ *     my_codec.c:
+ *       aom_codec_iface_t my_codec = {
+ *           "My Codec v1.0",
+ *           AOM_CODEC_ALG_ABI_VERSION,
+ *           ...
+ *       };
+ *
+ * + * An application instantiates a specific decoder instance by using + * aom_codec_init() and a pointer to the algorithm's interface structure: + * 
+ *     my_app.c:
+ *       extern aom_codec_iface_t my_codec;
+ *       {
+ *           aom_codec_ctx_t algo;
+ *           res = aom_codec_init(&algo, &my_codec);
+ *       }
+ *
+ * + * Once initialized, the instance is manged using other functions from + * the aom_codec_* family. + */ +#ifndef AOM_INTERNAL_AOM_CODEC_INTERNAL_H_ +#define AOM_INTERNAL_AOM_CODEC_INTERNAL_H_ +#include "./aom_config.h" +#include "../aom_decoder.h" +#include "../aom_encoder.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/*!\brief Current ABI version number + * + * \internal + * If this file is altered in any way that changes the ABI, this value + * must be bumped. Examples include, but are not limited to, changing + * types, removing or reassigning enums, adding/removing/rearranging + * fields to structures + */ +#define AOM_CODEC_INTERNAL_ABI_VERSION (5) /**<\hideinitializer*/ + +typedef struct aom_codec_alg_priv aom_codec_alg_priv_t; +typedef struct aom_codec_priv_enc_mr_cfg aom_codec_priv_enc_mr_cfg_t; + +/*!\brief init function pointer prototype + * + * Performs algorithm-specific initialization of the decoder context. This + * function is called by the generic aom_codec_init() wrapper function, so + * plugins implementing this interface may trust the input parameters to be + * properly initialized. + * + * \param[in] ctx Pointer to this instance's context + * \retval #AOM_CODEC_OK + * The input stream was recognized and decoder initialized. + * \retval #AOM_CODEC_MEM_ERROR + * Memory operation failed. + */ +typedef aom_codec_err_t (*aom_codec_init_fn_t)( + aom_codec_ctx_t *ctx, aom_codec_priv_enc_mr_cfg_t *data); + +/*!\brief destroy function pointer prototype + * + * Performs algorithm-specific destruction of the decoder context. This + * function is called by the generic aom_codec_destroy() wrapper function, + * so plugins implementing this interface may trust the input parameters + * to be properly initialized. + * + * \param[in] ctx Pointer to this instance's context + * \retval #AOM_CODEC_OK + * The input stream was recognized and decoder initialized. + * \retval #AOM_CODEC_MEM_ERROR + * Memory operation failed. + */ +typedef aom_codec_err_t (*aom_codec_destroy_fn_t)(aom_codec_alg_priv_t *ctx); + +/*!\brief parse stream info function pointer prototype + * + * Performs high level parsing of the bitstream. This function is called by the + * generic aom_codec_peek_stream_info() wrapper function, so plugins + * implementing this interface may trust the input parameters to be properly + * initialized. + * + * \param[in] data Pointer to a block of data to parse + * \param[in] data_sz Size of the data buffer + * \param[in,out] si Pointer to stream info to update. The size member + * \ref MUST be properly initialized, but \ref MAY be + * clobbered by the algorithm. This parameter \ref MAY + * be NULL. + * + * \retval #AOM_CODEC_OK + * Bitstream is parsable and stream information updated + */ +typedef aom_codec_err_t (*aom_codec_peek_si_fn_t)(const uint8_t *data, + unsigned int data_sz, + aom_codec_stream_info_t *si); + +/*!\brief Return information about the current stream. + * + * Returns information about the stream that has been parsed during decoding. + * + * \param[in] ctx Pointer to this instance's context + * \param[in,out] si Pointer to stream info to update. The size member + * \ref MUST be properly initialized, but \ref MAY be + * clobbered by the algorithm. This parameter \ref MAY + * be NULL. + * + * \retval #AOM_CODEC_OK + * Bitstream is parsable and stream information updated + */ +typedef aom_codec_err_t (*aom_codec_get_si_fn_t)(aom_codec_alg_priv_t *ctx, + aom_codec_stream_info_t *si); + +/*!\brief control function pointer prototype + * + * This function is used to exchange algorithm specific data with the decoder + * instance. This can be used to implement features specific to a particular + * algorithm. + * + * This function is called by the generic aom_codec_control() wrapper + * function, so plugins implementing this interface may trust the input + * parameters to be properly initialized. However, this interface does not + * provide type safety for the exchanged data or assign meanings to the + * control codes. Those details should be specified in the algorithm's + * header file. In particular, the ctrl_id parameter is guaranteed to exist + * in the algorithm's control mapping table, and the data parameter may be NULL. + * + * + * \param[in] ctx Pointer to this instance's context + * \param[in] ctrl_id Algorithm specific control identifier + * \param[in,out] data Data to exchange with algorithm instance. + * + * \retval #AOM_CODEC_OK + * The internal state data was deserialized. + */ +typedef aom_codec_err_t (*aom_codec_control_fn_t)(aom_codec_alg_priv_t *ctx, + va_list ap); + +/*!\brief control function pointer mapping + * + * This structure stores the mapping between control identifiers and + * implementing functions. Each algorithm provides a list of these + * mappings. This list is searched by the aom_codec_control() wrapper + * function to determine which function to invoke. The special + * value {0, NULL} is used to indicate end-of-list, and must be + * present. The special value {0, } can be used as a catch-all + * mapping. This implies that ctrl_id values chosen by the algorithm + * \ref MUST be non-zero. + */ +typedef const struct aom_codec_ctrl_fn_map { + int ctrl_id; + aom_codec_control_fn_t fn; +} aom_codec_ctrl_fn_map_t; + +/*!\brief decode data function pointer prototype + * + * Processes a buffer of coded data. If the processing results in a new + * decoded frame becoming available, #AOM_CODEC_CB_PUT_SLICE and + * #AOM_CODEC_CB_PUT_FRAME events are generated as appropriate. This + * function is called by the generic aom_codec_decode() wrapper function, + * so plugins implementing this interface may trust the input parameters + * to be properly initialized. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] data Pointer to this block of new coded data. If + * NULL, a #AOM_CODEC_CB_PUT_FRAME event is posted + * for the previously decoded frame. + * \param[in] data_sz Size of the coded data, in bytes. + * + * \return Returns #AOM_CODEC_OK if the coded data was processed completely + * and future pictures can be decoded without error. Otherwise, + * see the descriptions of the other error codes in ::aom_codec_err_t + * for recoverability capabilities. + */ +typedef aom_codec_err_t (*aom_codec_decode_fn_t)(aom_codec_alg_priv_t *ctx, + const uint8_t *data, + unsigned int data_sz, + void *user_priv, + long deadline); + +/*!\brief Decoded frames iterator + * + * Iterates over a list of the frames available for display. The iterator + * storage should be initialized to NULL to start the iteration. Iteration is + * complete when this function returns NULL. + * + * The list of available frames becomes valid upon completion of the + * aom_codec_decode call, and remains valid until the next call to + * aom_codec_decode. + * + * \param[in] ctx Pointer to this instance's context + * \param[in out] iter Iterator storage, initialized to NULL + * + * \return Returns a pointer to an image, if one is ready for display. Frames + * produced will always be in PTS (presentation time stamp) order. + */ +typedef aom_image_t *(*aom_codec_get_frame_fn_t)(aom_codec_alg_priv_t *ctx, + aom_codec_iter_t *iter); + +/*!\brief Pass in external frame buffers for the decoder to use. + * + * Registers functions to be called when libaom needs a frame buffer + * to decode the current frame and a function to be called when libaom does + * not internally reference the frame buffer. This set function must + * be called before the first call to decode or libaom will assume the + * default behavior of allocating frame buffers internally. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] cb_get Pointer to the get callback function + * \param[in] cb_release Pointer to the release callback function + * \param[in] cb_priv Callback's private data + * + * \retval #AOM_CODEC_OK + * External frame buffers will be used by libaom. + * \retval #AOM_CODEC_INVALID_PARAM + * One or more of the callbacks were NULL. + * \retval #AOM_CODEC_ERROR + * Decoder context not initialized, or algorithm not capable of + * using external frame buffers. + * + * \note + * When decoding AV1, the application may be required to pass in at least + * #AOM_MAXIMUM_WORK_BUFFERS external frame + * buffers. + */ +typedef aom_codec_err_t (*aom_codec_set_fb_fn_t)( + aom_codec_alg_priv_t *ctx, aom_get_frame_buffer_cb_fn_t cb_get, + aom_release_frame_buffer_cb_fn_t cb_release, void *cb_priv); + +typedef aom_codec_err_t (*aom_codec_encode_fn_t)(aom_codec_alg_priv_t *ctx, + const aom_image_t *img, + aom_codec_pts_t pts, + unsigned long duration, + aom_enc_frame_flags_t flags, + unsigned long deadline); +typedef const aom_codec_cx_pkt_t *(*aom_codec_get_cx_data_fn_t)( + aom_codec_alg_priv_t *ctx, aom_codec_iter_t *iter); + +typedef aom_codec_err_t (*aom_codec_enc_config_set_fn_t)( + aom_codec_alg_priv_t *ctx, const aom_codec_enc_cfg_t *cfg); +typedef aom_fixed_buf_t *(*aom_codec_get_global_headers_fn_t)( + aom_codec_alg_priv_t *ctx); + +typedef aom_image_t *(*aom_codec_get_preview_frame_fn_t)( + aom_codec_alg_priv_t *ctx); + +typedef aom_codec_err_t (*aom_codec_enc_mr_get_mem_loc_fn_t)( + const aom_codec_enc_cfg_t *cfg, void **mem_loc); + +/*!\brief usage configuration mapping + * + * This structure stores the mapping between usage identifiers and + * configuration structures. Each algorithm provides a list of these + * mappings. This list is searched by the aom_codec_enc_config_default() + * wrapper function to determine which config to return. The special value + * {-1, {0}} is used to indicate end-of-list, and must be present. At least + * one mapping must be present, in addition to the end-of-list. + * + */ +typedef const struct aom_codec_enc_cfg_map { + int usage; + aom_codec_enc_cfg_t cfg; +} aom_codec_enc_cfg_map_t; + +/*!\brief Decoder algorithm interface interface + * + * All decoders \ref MUST expose a variable of this type. + */ +struct aom_codec_iface { + const char *name; /**< Identification String */ + int abi_version; /**< Implemented ABI version */ + aom_codec_caps_t caps; /**< Decoder capabilities */ + aom_codec_init_fn_t init; /**< \copydoc ::aom_codec_init_fn_t */ + aom_codec_destroy_fn_t destroy; /**< \copydoc ::aom_codec_destroy_fn_t */ + aom_codec_ctrl_fn_map_t *ctrl_maps; /**< \copydoc ::aom_codec_ctrl_fn_map_t */ + struct aom_codec_dec_iface { + aom_codec_peek_si_fn_t peek_si; /**< \copydoc ::aom_codec_peek_si_fn_t */ + aom_codec_get_si_fn_t get_si; /**< \copydoc ::aom_codec_get_si_fn_t */ + aom_codec_decode_fn_t decode; /**< \copydoc ::aom_codec_decode_fn_t */ + aom_codec_get_frame_fn_t + get_frame; /**< \copydoc ::aom_codec_get_frame_fn_t */ + aom_codec_set_fb_fn_t set_fb_fn; /**< \copydoc ::aom_codec_set_fb_fn_t */ + } dec; + struct aom_codec_enc_iface { + int cfg_map_count; + aom_codec_enc_cfg_map_t + *cfg_maps; /**< \copydoc ::aom_codec_enc_cfg_map_t */ + aom_codec_encode_fn_t encode; /**< \copydoc ::aom_codec_encode_fn_t */ + aom_codec_get_cx_data_fn_t + get_cx_data; /**< \copydoc ::aom_codec_get_cx_data_fn_t */ + aom_codec_enc_config_set_fn_t + cfg_set; /**< \copydoc ::aom_codec_enc_config_set_fn_t */ + aom_codec_get_global_headers_fn_t + get_glob_hdrs; /**< \copydoc ::aom_codec_get_global_headers_fn_t */ + aom_codec_get_preview_frame_fn_t + get_preview; /**< \copydoc ::aom_codec_get_preview_frame_fn_t */ + aom_codec_enc_mr_get_mem_loc_fn_t + mr_get_mem_loc; /**< \copydoc ::aom_codec_enc_mr_get_mem_loc_fn_t */ + } enc; +}; + +/*!\brief Callback function pointer / user data pair storage */ +typedef struct aom_codec_priv_cb_pair { + union { + aom_codec_put_frame_cb_fn_t put_frame; + aom_codec_put_slice_cb_fn_t put_slice; + } u; + void *user_priv; +} aom_codec_priv_cb_pair_t; + +/*!\brief Instance private storage + * + * This structure is allocated by the algorithm's init function. It can be + * extended in one of two ways. First, a second, algorithm specific structure + * can be allocated and the priv member pointed to it. Alternatively, this + * structure can be made the first member of the algorithm specific structure, + * and the pointer cast to the proper type. + */ +struct aom_codec_priv { + const char *err_detail; + aom_codec_flags_t init_flags; + struct { + aom_codec_priv_cb_pair_t put_frame_cb; + aom_codec_priv_cb_pair_t put_slice_cb; + } dec; + struct { + aom_fixed_buf_t cx_data_dst_buf; + unsigned int cx_data_pad_before; + unsigned int cx_data_pad_after; + aom_codec_cx_pkt_t cx_data_pkt; + unsigned int total_encoders; + } enc; +}; + +/* + * Multi-resolution encoding internal configuration + */ +struct aom_codec_priv_enc_mr_cfg { + unsigned int mr_total_resolutions; + unsigned int mr_encoder_id; + struct aom_rational mr_down_sampling_factor; + void *mr_low_res_mode_info; +}; + +#undef AOM_CTRL_USE_TYPE +#define AOM_CTRL_USE_TYPE(id, typ) \ + static AOM_INLINE typ id##__value(va_list args) { return va_arg(args, typ); } + +#undef AOM_CTRL_USE_TYPE_DEPRECATED +#define AOM_CTRL_USE_TYPE_DEPRECATED(id, typ) \ + static AOM_INLINE typ id##__value(va_list args) { return va_arg(args, typ); } + +#define CAST(id, arg) id##__value(arg) + +/* CODEC_INTERFACE convenience macro + * + * By convention, each codec interface is a struct with extern linkage, where + * the symbol is suffixed with _algo. A getter function is also defined to + * return a pointer to the struct, since in some cases it's easier to work + * with text symbols than data symbols (see issue #169). This function has + * the same name as the struct, less the _algo suffix. The CODEC_INTERFACE + * macro is provided to define this getter function automatically. + */ +#define CODEC_INTERFACE(id) \ + aom_codec_iface_t *id(void) { return &id##_algo; } \ + aom_codec_iface_t id##_algo + +/* Internal Utility Functions + * + * The following functions are intended to be used inside algorithms as + * utilities for manipulating aom_codec_* data structures. + */ +struct aom_codec_pkt_list { + unsigned int cnt; + unsigned int max; + struct aom_codec_cx_pkt pkts[1]; +}; + +#define aom_codec_pkt_list_decl(n) \ + union { \ + struct aom_codec_pkt_list head; \ + struct { \ + struct aom_codec_pkt_list head; \ + struct aom_codec_cx_pkt pkts[n]; \ + } alloc; \ + } + +#define aom_codec_pkt_list_init(m) \ + (m)->alloc.head.cnt = 0, \ + (m)->alloc.head.max = sizeof((m)->alloc.pkts) / sizeof((m)->alloc.pkts[0]) + +int aom_codec_pkt_list_add(struct aom_codec_pkt_list *, + const struct aom_codec_cx_pkt *); + +const aom_codec_cx_pkt_t *aom_codec_pkt_list_get( + struct aom_codec_pkt_list *list, aom_codec_iter_t *iter); + +#include +#include + +struct aom_internal_error_info { + aom_codec_err_t error_code; + int has_detail; + char detail[80]; + int setjmp; + jmp_buf jmp; +}; + +#define CLANG_ANALYZER_NORETURN +#if defined(__has_feature) +#if __has_feature(attribute_analyzer_noreturn) +#undef CLANG_ANALYZER_NORETURN +#define CLANG_ANALYZER_NORETURN __attribute__((analyzer_noreturn)) +#endif +#endif + +void aom_internal_error(struct aom_internal_error_info *info, + aom_codec_err_t error, const char *fmt, + ...) CLANG_ANALYZER_NORETURN; + +void aom_merge_corrupted_flag(int *corrupted, int value); + +#if CONFIG_DEBUG +#define AOM_CHECK_MEM_ERROR(error_info, lval, expr) \ + do { \ + lval = (expr); \ + if (!lval) \ + aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, \ + "Failed to allocate " #lval " at %s:%d", __FILE__, \ + __LINE__); \ + } while (0) +#else +#define AOM_CHECK_MEM_ERROR(error_info, lval, expr) \ + do { \ + lval = (expr); \ + if (!lval) \ + aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, \ + "Failed to allocate " #lval); \ + } while (0) +#endif +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_INTERNAL_AOM_CODEC_INTERNAL_H_ diff --git a/third_party/aom/aom/src/aom_codec.c b/third_party/aom/aom/src/aom_codec.c new file mode 100644 index 0000000000..873d758768 --- /dev/null +++ b/third_party/aom/aom/src/aom_codec.c @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/*!\file + * \brief Provides the high level interface to wrap decoder algorithms. + * + */ +#include +#include +#include "aom/aom_integer.h" +#include "aom/internal/aom_codec_internal.h" +#include "aom_version.h" + +#define SAVE_STATUS(ctx, var) (ctx ? (ctx->err = var) : var) + +int aom_codec_version(void) { return VERSION_PACKED; } + +const char *aom_codec_version_str(void) { return VERSION_STRING_NOSP; } + +const char *aom_codec_version_extra_str(void) { return VERSION_EXTRA; } + +const char *aom_codec_iface_name(aom_codec_iface_t *iface) { + return iface ? iface->name : ""; +} + +const char *aom_codec_err_to_string(aom_codec_err_t err) { + switch (err) { + case AOM_CODEC_OK: return "Success"; + case AOM_CODEC_ERROR: return "Unspecified internal error"; + case AOM_CODEC_MEM_ERROR: return "Memory allocation error"; + case AOM_CODEC_ABI_MISMATCH: return "ABI version mismatch"; + case AOM_CODEC_INCAPABLE: + return "Codec does not implement requested capability"; + case AOM_CODEC_UNSUP_BITSTREAM: + return "Bitstream not supported by this decoder"; + case AOM_CODEC_UNSUP_FEATURE: + return "Bitstream required feature not supported by this decoder"; + case AOM_CODEC_CORRUPT_FRAME: return "Corrupt frame detected"; + case AOM_CODEC_INVALID_PARAM: return "Invalid parameter"; + case AOM_CODEC_LIST_END: return "End of iterated list"; + } + + return "Unrecognized error code"; +} + +const char *aom_codec_error(aom_codec_ctx_t *ctx) { + return (ctx) ? aom_codec_err_to_string(ctx->err) + : aom_codec_err_to_string(AOM_CODEC_INVALID_PARAM); +} + +const char *aom_codec_error_detail(aom_codec_ctx_t *ctx) { + if (ctx && ctx->err) + return ctx->priv ? ctx->priv->err_detail : ctx->err_detail; + + return NULL; +} + +aom_codec_err_t aom_codec_destroy(aom_codec_ctx_t *ctx) { + aom_codec_err_t res; + + if (!ctx) + res = AOM_CODEC_INVALID_PARAM; + else if (!ctx->iface || !ctx->priv) + res = AOM_CODEC_ERROR; + else { + ctx->iface->destroy((aom_codec_alg_priv_t *)ctx->priv); + + ctx->iface = NULL; + ctx->name = NULL; + ctx->priv = NULL; + res = AOM_CODEC_OK; + } + + return SAVE_STATUS(ctx, res); +} + +aom_codec_caps_t aom_codec_get_caps(aom_codec_iface_t *iface) { + return (iface) ? iface->caps : 0; +} + +aom_codec_err_t aom_codec_control_(aom_codec_ctx_t *ctx, int ctrl_id, ...) { + aom_codec_err_t res; + + if (!ctx || !ctrl_id) + res = AOM_CODEC_INVALID_PARAM; + else if (!ctx->iface || !ctx->priv || !ctx->iface->ctrl_maps) + res = AOM_CODEC_ERROR; + else { + aom_codec_ctrl_fn_map_t *entry; + + res = AOM_CODEC_ERROR; + + for (entry = ctx->iface->ctrl_maps; entry && entry->fn; entry++) { + if (!entry->ctrl_id || entry->ctrl_id == ctrl_id) { + va_list ap; + + va_start(ap, ctrl_id); + res = entry->fn((aom_codec_alg_priv_t *)ctx->priv, ap); + va_end(ap); + break; + } + } + } + + return SAVE_STATUS(ctx, res); +} + +void aom_internal_error(struct aom_internal_error_info *info, + aom_codec_err_t error, const char *fmt, ...) { + va_list ap; + + info->error_code = error; + info->has_detail = 0; + + if (fmt) { + size_t sz = sizeof(info->detail); + + info->has_detail = 1; + va_start(ap, fmt); + vsnprintf(info->detail, sz - 1, fmt, ap); + va_end(ap); + info->detail[sz - 1] = '\0'; + } + + if (info->setjmp) longjmp(info->jmp, info->error_code); +} + +void aom_merge_corrupted_flag(int *corrupted, int value) { + *corrupted |= value; +} diff --git a/third_party/aom/aom/src/aom_decoder.c b/third_party/aom/aom/src/aom_decoder.c new file mode 100644 index 0000000000..c6cec6d79c --- /dev/null +++ b/third_party/aom/aom/src/aom_decoder.c @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/*!\file + * \brief Provides the high level interface to wrap decoder algorithms. + * + */ +#include +#include "aom/internal/aom_codec_internal.h" + +#define SAVE_STATUS(ctx, var) (ctx ? (ctx->err = var) : var) + +static aom_codec_alg_priv_t *get_alg_priv(aom_codec_ctx_t *ctx) { + return (aom_codec_alg_priv_t *)ctx->priv; +} + +aom_codec_err_t aom_codec_dec_init_ver(aom_codec_ctx_t *ctx, + aom_codec_iface_t *iface, + const aom_codec_dec_cfg_t *cfg, + aom_codec_flags_t flags, int ver) { + aom_codec_err_t res; + + if (ver != AOM_DECODER_ABI_VERSION) + res = AOM_CODEC_ABI_MISMATCH; + else if (!ctx || !iface) + res = AOM_CODEC_INVALID_PARAM; + else if (iface->abi_version != AOM_CODEC_INTERNAL_ABI_VERSION) + res = AOM_CODEC_ABI_MISMATCH; + else if ((flags & AOM_CODEC_USE_POSTPROC) && + !(iface->caps & AOM_CODEC_CAP_POSTPROC)) + res = AOM_CODEC_INCAPABLE; + else if ((flags & AOM_CODEC_USE_ERROR_CONCEALMENT) && + !(iface->caps & AOM_CODEC_CAP_ERROR_CONCEALMENT)) + res = AOM_CODEC_INCAPABLE; + else if ((flags & AOM_CODEC_USE_INPUT_FRAGMENTS) && + !(iface->caps & AOM_CODEC_CAP_INPUT_FRAGMENTS)) + res = AOM_CODEC_INCAPABLE; + else if (!(iface->caps & AOM_CODEC_CAP_DECODER)) + res = AOM_CODEC_INCAPABLE; + else { + memset(ctx, 0, sizeof(*ctx)); + ctx->iface = iface; + ctx->name = iface->name; + ctx->priv = NULL; + ctx->init_flags = flags; + ctx->config.dec = cfg; + + res = ctx->iface->init(ctx, NULL); + if (res) { + ctx->err_detail = ctx->priv ? ctx->priv->err_detail : NULL; + aom_codec_destroy(ctx); + } + } + + return SAVE_STATUS(ctx, res); +} + +aom_codec_err_t aom_codec_peek_stream_info(aom_codec_iface_t *iface, + const uint8_t *data, + unsigned int data_sz, + aom_codec_stream_info_t *si) { + aom_codec_err_t res; + + if (!iface || !data || !data_sz || !si || + si->sz < sizeof(aom_codec_stream_info_t)) + res = AOM_CODEC_INVALID_PARAM; + else { + /* Set default/unknown values */ + si->w = 0; + si->h = 0; + + res = iface->dec.peek_si(data, data_sz, si); + } + + return res; +} + +aom_codec_err_t aom_codec_get_stream_info(aom_codec_ctx_t *ctx, + aom_codec_stream_info_t *si) { + aom_codec_err_t res; + + if (!ctx || !si || si->sz < sizeof(aom_codec_stream_info_t)) + res = AOM_CODEC_INVALID_PARAM; + else if (!ctx->iface || !ctx->priv) + res = AOM_CODEC_ERROR; + else { + /* Set default/unknown values */ + si->w = 0; + si->h = 0; + + res = ctx->iface->dec.get_si(get_alg_priv(ctx), si); + } + + return SAVE_STATUS(ctx, res); +} + +aom_codec_err_t aom_codec_decode(aom_codec_ctx_t *ctx, const uint8_t *data, + unsigned int data_sz, void *user_priv, + long deadline) { + aom_codec_err_t res; + + /* Sanity checks */ + /* NULL data ptr allowed if data_sz is 0 too */ + if (!ctx || (!data && data_sz) || (data && !data_sz)) + res = AOM_CODEC_INVALID_PARAM; + else if (!ctx->iface || !ctx->priv) + res = AOM_CODEC_ERROR; + else { + res = ctx->iface->dec.decode(get_alg_priv(ctx), data, data_sz, user_priv, + deadline); + } + + return SAVE_STATUS(ctx, res); +} + +aom_image_t *aom_codec_get_frame(aom_codec_ctx_t *ctx, aom_codec_iter_t *iter) { + aom_image_t *img; + + if (!ctx || !iter || !ctx->iface || !ctx->priv) + img = NULL; + else + img = ctx->iface->dec.get_frame(get_alg_priv(ctx), iter); + + return img; +} + +aom_codec_err_t aom_codec_register_put_frame_cb(aom_codec_ctx_t *ctx, + aom_codec_put_frame_cb_fn_t cb, + void *user_priv) { + aom_codec_err_t res; + + if (!ctx || !cb) + res = AOM_CODEC_INVALID_PARAM; + else if (!ctx->iface || !ctx->priv || + !(ctx->iface->caps & AOM_CODEC_CAP_PUT_FRAME)) + res = AOM_CODEC_ERROR; + else { + ctx->priv->dec.put_frame_cb.u.put_frame = cb; + ctx->priv->dec.put_frame_cb.user_priv = user_priv; + res = AOM_CODEC_OK; + } + + return SAVE_STATUS(ctx, res); +} + +aom_codec_err_t aom_codec_register_put_slice_cb(aom_codec_ctx_t *ctx, + aom_codec_put_slice_cb_fn_t cb, + void *user_priv) { + aom_codec_err_t res; + + if (!ctx || !cb) + res = AOM_CODEC_INVALID_PARAM; + else if (!ctx->iface || !ctx->priv || + !(ctx->iface->caps & AOM_CODEC_CAP_PUT_SLICE)) + res = AOM_CODEC_ERROR; + else { + ctx->priv->dec.put_slice_cb.u.put_slice = cb; + ctx->priv->dec.put_slice_cb.user_priv = user_priv; + res = AOM_CODEC_OK; + } + + return SAVE_STATUS(ctx, res); +} + +aom_codec_err_t aom_codec_set_frame_buffer_functions( + aom_codec_ctx_t *ctx, aom_get_frame_buffer_cb_fn_t cb_get, + aom_release_frame_buffer_cb_fn_t cb_release, void *cb_priv) { + aom_codec_err_t res; + + if (!ctx || !cb_get || !cb_release) { + res = AOM_CODEC_INVALID_PARAM; + } else if (!ctx->iface || !ctx->priv || + !(ctx->iface->caps & AOM_CODEC_CAP_EXTERNAL_FRAME_BUFFER)) { + res = AOM_CODEC_ERROR; + } else { + res = ctx->iface->dec.set_fb_fn(get_alg_priv(ctx), cb_get, cb_release, + cb_priv); + } + + return SAVE_STATUS(ctx, res); +} diff --git a/third_party/aom/aom/src/aom_encoder.c b/third_party/aom/aom/src/aom_encoder.c new file mode 100644 index 0000000000..ac84c888a4 --- /dev/null +++ b/third_party/aom/aom/src/aom_encoder.c @@ -0,0 +1,405 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/*!\file + * \brief Provides the high level interface to wrap encoder algorithms. + * + */ +#include "./aom_config.h" + +#if HAVE_FEXCEPT +#define _GNU_SOURCE +#include +#endif + +#include +#include +#include "aom/internal/aom_codec_internal.h" + +#define SAVE_STATUS(ctx, var) (ctx ? (ctx->err = var) : var) + +static aom_codec_alg_priv_t *get_alg_priv(aom_codec_ctx_t *ctx) { + return (aom_codec_alg_priv_t *)ctx->priv; +} + +aom_codec_err_t aom_codec_enc_init_ver(aom_codec_ctx_t *ctx, + aom_codec_iface_t *iface, + const aom_codec_enc_cfg_t *cfg, + aom_codec_flags_t flags, int ver) { + aom_codec_err_t res; + + if (ver != AOM_ENCODER_ABI_VERSION) + res = AOM_CODEC_ABI_MISMATCH; + else if (!ctx || !iface || !cfg) + res = AOM_CODEC_INVALID_PARAM; + else if (iface->abi_version != AOM_CODEC_INTERNAL_ABI_VERSION) + res = AOM_CODEC_ABI_MISMATCH; + else if (!(iface->caps & AOM_CODEC_CAP_ENCODER)) + res = AOM_CODEC_INCAPABLE; + else if ((flags & AOM_CODEC_USE_PSNR) && !(iface->caps & AOM_CODEC_CAP_PSNR)) + res = AOM_CODEC_INCAPABLE; + else if ((flags & AOM_CODEC_USE_OUTPUT_PARTITION) && + !(iface->caps & AOM_CODEC_CAP_OUTPUT_PARTITION)) + res = AOM_CODEC_INCAPABLE; + else { + ctx->iface = iface; + ctx->name = iface->name; + ctx->priv = NULL; + ctx->init_flags = flags; + ctx->config.enc = cfg; + res = ctx->iface->init(ctx, NULL); + + if (res) { + ctx->err_detail = ctx->priv ? ctx->priv->err_detail : NULL; + aom_codec_destroy(ctx); + } + } + + return SAVE_STATUS(ctx, res); +} + +aom_codec_err_t aom_codec_enc_init_multi_ver( + aom_codec_ctx_t *ctx, aom_codec_iface_t *iface, aom_codec_enc_cfg_t *cfg, + int num_enc, aom_codec_flags_t flags, aom_rational_t *dsf, int ver) { + aom_codec_err_t res = AOM_CODEC_OK; + + if (ver != AOM_ENCODER_ABI_VERSION) + res = AOM_CODEC_ABI_MISMATCH; + else if (!ctx || !iface || !cfg || (num_enc > 16 || num_enc < 1)) + res = AOM_CODEC_INVALID_PARAM; + else if (iface->abi_version != AOM_CODEC_INTERNAL_ABI_VERSION) + res = AOM_CODEC_ABI_MISMATCH; + else if (!(iface->caps & AOM_CODEC_CAP_ENCODER)) + res = AOM_CODEC_INCAPABLE; + else if ((flags & AOM_CODEC_USE_PSNR) && !(iface->caps & AOM_CODEC_CAP_PSNR)) + res = AOM_CODEC_INCAPABLE; + else if ((flags & AOM_CODEC_USE_OUTPUT_PARTITION) && + !(iface->caps & AOM_CODEC_CAP_OUTPUT_PARTITION)) + res = AOM_CODEC_INCAPABLE; + else { + int i; + void *mem_loc = NULL; + + if (!(res = iface->enc.mr_get_mem_loc(cfg, &mem_loc))) { + for (i = 0; i < num_enc; i++) { + aom_codec_priv_enc_mr_cfg_t mr_cfg; + + /* Validate down-sampling factor. */ + if (dsf->num < 1 || dsf->num > 4096 || dsf->den < 1 || + dsf->den > dsf->num) { + res = AOM_CODEC_INVALID_PARAM; + break; + } + + mr_cfg.mr_low_res_mode_info = mem_loc; + mr_cfg.mr_total_resolutions = num_enc; + mr_cfg.mr_encoder_id = num_enc - 1 - i; + mr_cfg.mr_down_sampling_factor.num = dsf->num; + mr_cfg.mr_down_sampling_factor.den = dsf->den; + + /* Force Key-frame synchronization. Namely, encoder at higher + * resolution always use the same frame_type chosen by the + * lowest-resolution encoder. + */ + if (mr_cfg.mr_encoder_id) cfg->kf_mode = AOM_KF_DISABLED; + + ctx->iface = iface; + ctx->name = iface->name; + ctx->priv = NULL; + ctx->init_flags = flags; + ctx->config.enc = cfg; + res = ctx->iface->init(ctx, &mr_cfg); + + if (res) { + const char *error_detail = ctx->priv ? ctx->priv->err_detail : NULL; + /* Destroy current ctx */ + ctx->err_detail = error_detail; + aom_codec_destroy(ctx); + + /* Destroy already allocated high-level ctx */ + while (i) { + ctx--; + ctx->err_detail = error_detail; + aom_codec_destroy(ctx); + i--; + } + } + + if (res) break; + + ctx++; + cfg++; + dsf++; + } + ctx--; + } + } + + return SAVE_STATUS(ctx, res); +} + +aom_codec_err_t aom_codec_enc_config_default(aom_codec_iface_t *iface, + aom_codec_enc_cfg_t *cfg, + unsigned int usage) { + aom_codec_err_t res; + aom_codec_enc_cfg_map_t *map; + int i; + + if (!iface || !cfg || usage > INT_MAX) + res = AOM_CODEC_INVALID_PARAM; + else if (!(iface->caps & AOM_CODEC_CAP_ENCODER)) + res = AOM_CODEC_INCAPABLE; + else { + res = AOM_CODEC_INVALID_PARAM; + + for (i = 0; i < iface->enc.cfg_map_count; ++i) { + map = iface->enc.cfg_maps + i; + if (map->usage == (int)usage) { + *cfg = map->cfg; + cfg->g_usage = usage; + res = AOM_CODEC_OK; + break; + } + } + } + + return res; +} + +/* clang-format off */ +#define FLOATING_POINT_BEGIN_SCOPE do { +#define FLOATING_POINT_END_SCOPE } while (0); +/* clang-format on */ + +#if ARCH_X86 || ARCH_X86_64 +/* On X86, disable the x87 unit's internal 80 bit precision for better + * consistency with the SSE unit's 64 bit precision. + */ +#include "aom_ports/x86.h" +#define FLOATING_POINT_SET_PRECISION \ + unsigned short x87_orig_mode = x87_set_double_precision(); +#define FLOATING_POINT_RESTORE_PRECISION x87_set_control_word(x87_orig_mode); +#else +#define FLOATING_POINT_SET_PRECISION +#define FLOATING_POINT_RESTORE_PRECISION +#endif // ARCH_X86 || ARCH_X86_64 + +#if HAVE_FEXCEPT && CONFIG_DEBUG +#define FLOATING_POINT_SET_EXCEPTIONS \ + const int float_excepts = feenableexcept(FE_DIVBYZERO); +#define FLOATING_POINT_RESTORE_EXCEPTIONS feenableexcept(float_excepts); +#else +#define FLOATING_POINT_SET_EXCEPTIONS +#define FLOATING_POINT_RESTORE_EXCEPTIONS +#endif // HAVE_FEXCEPT && CONFIG_DEBUG + +#define FLOATING_POINT_INIT \ + FLOATING_POINT_BEGIN_SCOPE \ + FLOATING_POINT_SET_PRECISION \ + FLOATING_POINT_SET_EXCEPTIONS + +#define FLOATING_POINT_RESTORE \ + FLOATING_POINT_RESTORE_EXCEPTIONS \ + FLOATING_POINT_RESTORE_PRECISION \ + FLOATING_POINT_END_SCOPE + +aom_codec_err_t aom_codec_encode(aom_codec_ctx_t *ctx, const aom_image_t *img, + aom_codec_pts_t pts, unsigned long duration, + aom_enc_frame_flags_t flags, + unsigned long deadline) { + aom_codec_err_t res = AOM_CODEC_OK; + + if (!ctx || (img && !duration)) + res = AOM_CODEC_INVALID_PARAM; + else if (!ctx->iface || !ctx->priv) + res = AOM_CODEC_ERROR; + else if (!(ctx->iface->caps & AOM_CODEC_CAP_ENCODER)) + res = AOM_CODEC_INCAPABLE; + else { + unsigned int num_enc = ctx->priv->enc.total_encoders; + + /* Execute in a normalized floating point environment, if the platform + * requires it. + */ + FLOATING_POINT_INIT + + if (num_enc == 1) + res = ctx->iface->enc.encode(get_alg_priv(ctx), img, pts, duration, flags, + deadline); + else { + /* Multi-resolution encoding: + * Encode multi-levels in reverse order. For example, + * if mr_total_resolutions = 3, first encode level 2, + * then encode level 1, and finally encode level 0. + */ + int i; + + ctx += num_enc - 1; + if (img) img += num_enc - 1; + + for (i = num_enc - 1; i >= 0; i--) { + if ((res = ctx->iface->enc.encode(get_alg_priv(ctx), img, pts, duration, + flags, deadline))) + break; + + ctx--; + if (img) img--; + } + ctx++; + } + + FLOATING_POINT_RESTORE + } + + return SAVE_STATUS(ctx, res); +} + +const aom_codec_cx_pkt_t *aom_codec_get_cx_data(aom_codec_ctx_t *ctx, + aom_codec_iter_t *iter) { + const aom_codec_cx_pkt_t *pkt = NULL; + + if (ctx) { + if (!iter) + ctx->err = AOM_CODEC_INVALID_PARAM; + else if (!ctx->iface || !ctx->priv) + ctx->err = AOM_CODEC_ERROR; + else if (!(ctx->iface->caps & AOM_CODEC_CAP_ENCODER)) + ctx->err = AOM_CODEC_INCAPABLE; + else + pkt = ctx->iface->enc.get_cx_data(get_alg_priv(ctx), iter); + } + + if (pkt && pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + // If the application has specified a destination area for the + // compressed data, and the codec has not placed the data there, + // and it fits, copy it. + aom_codec_priv_t *const priv = ctx->priv; + char *const dst_buf = (char *)priv->enc.cx_data_dst_buf.buf; + + if (dst_buf && pkt->data.raw.buf != dst_buf && + pkt->data.raw.sz + priv->enc.cx_data_pad_before + + priv->enc.cx_data_pad_after <= + priv->enc.cx_data_dst_buf.sz) { + aom_codec_cx_pkt_t *modified_pkt = &priv->enc.cx_data_pkt; + + memcpy(dst_buf + priv->enc.cx_data_pad_before, pkt->data.raw.buf, + pkt->data.raw.sz); + *modified_pkt = *pkt; + modified_pkt->data.raw.buf = dst_buf; + modified_pkt->data.raw.sz += + priv->enc.cx_data_pad_before + priv->enc.cx_data_pad_after; + pkt = modified_pkt; + } + + if (dst_buf == pkt->data.raw.buf) { + priv->enc.cx_data_dst_buf.buf = dst_buf + pkt->data.raw.sz; + priv->enc.cx_data_dst_buf.sz -= pkt->data.raw.sz; + } + } + + return pkt; +} + +aom_codec_err_t aom_codec_set_cx_data_buf(aom_codec_ctx_t *ctx, + const aom_fixed_buf_t *buf, + unsigned int pad_before, + unsigned int pad_after) { + if (!ctx || !ctx->priv) return AOM_CODEC_INVALID_PARAM; + + if (buf) { + ctx->priv->enc.cx_data_dst_buf = *buf; + ctx->priv->enc.cx_data_pad_before = pad_before; + ctx->priv->enc.cx_data_pad_after = pad_after; + } else { + ctx->priv->enc.cx_data_dst_buf.buf = NULL; + ctx->priv->enc.cx_data_dst_buf.sz = 0; + ctx->priv->enc.cx_data_pad_before = 0; + ctx->priv->enc.cx_data_pad_after = 0; + } + + return AOM_CODEC_OK; +} + +const aom_image_t *aom_codec_get_preview_frame(aom_codec_ctx_t *ctx) { + aom_image_t *img = NULL; + + if (ctx) { + if (!ctx->iface || !ctx->priv) + ctx->err = AOM_CODEC_ERROR; + else if (!(ctx->iface->caps & AOM_CODEC_CAP_ENCODER)) + ctx->err = AOM_CODEC_INCAPABLE; + else if (!ctx->iface->enc.get_preview) + ctx->err = AOM_CODEC_INCAPABLE; + else + img = ctx->iface->enc.get_preview(get_alg_priv(ctx)); + } + + return img; +} + +aom_fixed_buf_t *aom_codec_get_global_headers(aom_codec_ctx_t *ctx) { + aom_fixed_buf_t *buf = NULL; + + if (ctx) { + if (!ctx->iface || !ctx->priv) + ctx->err = AOM_CODEC_ERROR; + else if (!(ctx->iface->caps & AOM_CODEC_CAP_ENCODER)) + ctx->err = AOM_CODEC_INCAPABLE; + else if (!ctx->iface->enc.get_glob_hdrs) + ctx->err = AOM_CODEC_INCAPABLE; + else + buf = ctx->iface->enc.get_glob_hdrs(get_alg_priv(ctx)); + } + + return buf; +} + +aom_codec_err_t aom_codec_enc_config_set(aom_codec_ctx_t *ctx, + const aom_codec_enc_cfg_t *cfg) { + aom_codec_err_t res; + + if (!ctx || !ctx->iface || !ctx->priv || !cfg) + res = AOM_CODEC_INVALID_PARAM; + else if (!(ctx->iface->caps & AOM_CODEC_CAP_ENCODER)) + res = AOM_CODEC_INCAPABLE; + else + res = ctx->iface->enc.cfg_set(get_alg_priv(ctx), cfg); + + return SAVE_STATUS(ctx, res); +} + +int aom_codec_pkt_list_add(struct aom_codec_pkt_list *list, + const struct aom_codec_cx_pkt *pkt) { + if (list->cnt < list->max) { + list->pkts[list->cnt++] = *pkt; + return 0; + } + + return 1; +} + +const aom_codec_cx_pkt_t *aom_codec_pkt_list_get( + struct aom_codec_pkt_list *list, aom_codec_iter_t *iter) { + const aom_codec_cx_pkt_t *pkt; + + if (!(*iter)) { + *iter = list->pkts; + } + + pkt = (const aom_codec_cx_pkt_t *)*iter; + + if ((size_t)(pkt - list->pkts) < list->cnt) + *iter = pkt + 1; + else + pkt = NULL; + + return pkt; +} diff --git a/third_party/aom/aom/src/aom_image.c b/third_party/aom/aom/src/aom_image.c new file mode 100644 index 0000000000..0d54fd46db --- /dev/null +++ b/third_party/aom/aom/src/aom_image.c @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "aom/aom_image.h" +#include "aom/aom_integer.h" +#include "aom_mem/aom_mem.h" + +static aom_image_t *img_alloc_helper(aom_image_t *img, aom_img_fmt_t fmt, + unsigned int d_w, unsigned int d_h, + unsigned int buf_align, + unsigned int stride_align, + unsigned char *img_data) { + unsigned int h, w, s, xcs, ycs, bps; + unsigned int stride_in_bytes; + int align; + + /* Treat align==0 like align==1 */ + if (!buf_align) buf_align = 1; + + /* Validate alignment (must be power of 2) */ + if (buf_align & (buf_align - 1)) goto fail; + + /* Treat align==0 like align==1 */ + if (!stride_align) stride_align = 1; + + /* Validate alignment (must be power of 2) */ + if (stride_align & (stride_align - 1)) goto fail; + + /* Get sample size for this format */ + switch (fmt) { + case AOM_IMG_FMT_RGB32: + case AOM_IMG_FMT_RGB32_LE: + case AOM_IMG_FMT_ARGB: + case AOM_IMG_FMT_ARGB_LE: bps = 32; break; + case AOM_IMG_FMT_RGB24: + case AOM_IMG_FMT_BGR24: bps = 24; break; + case AOM_IMG_FMT_RGB565: + case AOM_IMG_FMT_RGB565_LE: + case AOM_IMG_FMT_RGB555: + case AOM_IMG_FMT_RGB555_LE: + case AOM_IMG_FMT_UYVY: + case AOM_IMG_FMT_YUY2: + case AOM_IMG_FMT_YVYU: bps = 16; break; + case AOM_IMG_FMT_I420: + case AOM_IMG_FMT_YV12: + case AOM_IMG_FMT_AOMI420: + case AOM_IMG_FMT_AOMYV12: bps = 12; break; + case AOM_IMG_FMT_I422: + case AOM_IMG_FMT_I440: bps = 16; break; + case AOM_IMG_FMT_I444: bps = 24; break; + case AOM_IMG_FMT_I42016: bps = 24; break; + case AOM_IMG_FMT_I42216: + case AOM_IMG_FMT_I44016: bps = 32; break; + case AOM_IMG_FMT_I44416: bps = 48; break; + default: bps = 16; break; + } + + /* Get chroma shift values for this format */ + switch (fmt) { + case AOM_IMG_FMT_I420: + case AOM_IMG_FMT_YV12: + case AOM_IMG_FMT_AOMI420: + case AOM_IMG_FMT_AOMYV12: + case AOM_IMG_FMT_I422: + case AOM_IMG_FMT_I42016: + case AOM_IMG_FMT_I42216: xcs = 1; break; + default: xcs = 0; break; + } + + switch (fmt) { + case AOM_IMG_FMT_I420: + case AOM_IMG_FMT_I440: + case AOM_IMG_FMT_YV12: + case AOM_IMG_FMT_AOMI420: + case AOM_IMG_FMT_AOMYV12: + case AOM_IMG_FMT_I42016: + case AOM_IMG_FMT_I44016: ycs = 1; break; + default: ycs = 0; break; + } + + /* Calculate storage sizes given the chroma subsampling */ + align = (1 << xcs) - 1; + w = (d_w + align) & ~align; + align = (1 << ycs) - 1; + h = (d_h + align) & ~align; + s = (fmt & AOM_IMG_FMT_PLANAR) ? w : bps * w / 8; + s = (s + stride_align - 1) & ~(stride_align - 1); + stride_in_bytes = (fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; + + /* Allocate the new image */ + if (!img) { + img = (aom_image_t *)calloc(1, sizeof(aom_image_t)); + + if (!img) goto fail; + + img->self_allocd = 1; + } else { + memset(img, 0, sizeof(aom_image_t)); + } + + img->img_data = img_data; + + if (!img_data) { + const uint64_t alloc_size = (fmt & AOM_IMG_FMT_PLANAR) + ? (uint64_t)h * s * bps / 8 + : (uint64_t)h * s; + + if (alloc_size != (size_t)alloc_size) goto fail; + + img->img_data = (uint8_t *)aom_memalign(buf_align, (size_t)alloc_size); + img->img_data_owner = 1; + } + + if (!img->img_data) goto fail; + + img->fmt = fmt; + img->bit_depth = (fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 16 : 8; + img->w = w; + img->h = h; + img->x_chroma_shift = xcs; + img->y_chroma_shift = ycs; + img->bps = bps; + + /* Calculate strides */ + img->stride[AOM_PLANE_Y] = img->stride[AOM_PLANE_ALPHA] = stride_in_bytes; + img->stride[AOM_PLANE_U] = img->stride[AOM_PLANE_V] = stride_in_bytes >> xcs; + + /* Default viewport to entire image */ + if (!aom_img_set_rect(img, 0, 0, d_w, d_h)) return img; + +fail: + aom_img_free(img); + return NULL; +} + +aom_image_t *aom_img_alloc(aom_image_t *img, aom_img_fmt_t fmt, + unsigned int d_w, unsigned int d_h, + unsigned int align) { + return img_alloc_helper(img, fmt, d_w, d_h, align, align, NULL); +} + +aom_image_t *aom_img_wrap(aom_image_t *img, aom_img_fmt_t fmt, unsigned int d_w, + unsigned int d_h, unsigned int stride_align, + unsigned char *img_data) { + /* By setting buf_align = 1, we don't change buffer alignment in this + * function. */ + return img_alloc_helper(img, fmt, d_w, d_h, 1, stride_align, img_data); +} + +int aom_img_set_rect(aom_image_t *img, unsigned int x, unsigned int y, + unsigned int w, unsigned int h) { + unsigned char *data; + + if (x + w <= img->w && y + h <= img->h) { + img->d_w = w; + img->d_h = h; + + /* Calculate plane pointers */ + if (!(img->fmt & AOM_IMG_FMT_PLANAR)) { + img->planes[AOM_PLANE_PACKED] = + img->img_data + x * img->bps / 8 + y * img->stride[AOM_PLANE_PACKED]; + } else { + const int bytes_per_sample = + (img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1; + data = img->img_data; + + if (img->fmt & AOM_IMG_FMT_HAS_ALPHA) { + img->planes[AOM_PLANE_ALPHA] = + data + x * bytes_per_sample + y * img->stride[AOM_PLANE_ALPHA]; + data += img->h * img->stride[AOM_PLANE_ALPHA]; + } + + img->planes[AOM_PLANE_Y] = + data + x * bytes_per_sample + y * img->stride[AOM_PLANE_Y]; + data += img->h * img->stride[AOM_PLANE_Y]; + + if (!(img->fmt & AOM_IMG_FMT_UV_FLIP)) { + img->planes[AOM_PLANE_U] = + data + (x >> img->x_chroma_shift) * bytes_per_sample + + (y >> img->y_chroma_shift) * img->stride[AOM_PLANE_U]; + data += (img->h >> img->y_chroma_shift) * img->stride[AOM_PLANE_U]; + img->planes[AOM_PLANE_V] = + data + (x >> img->x_chroma_shift) * bytes_per_sample + + (y >> img->y_chroma_shift) * img->stride[AOM_PLANE_V]; + } else { + img->planes[AOM_PLANE_V] = + data + (x >> img->x_chroma_shift) * bytes_per_sample + + (y >> img->y_chroma_shift) * img->stride[AOM_PLANE_V]; + data += (img->h >> img->y_chroma_shift) * img->stride[AOM_PLANE_V]; + img->planes[AOM_PLANE_U] = + data + (x >> img->x_chroma_shift) * bytes_per_sample + + (y >> img->y_chroma_shift) * img->stride[AOM_PLANE_U]; + } + } + return 0; + } + return -1; +} + +void aom_img_flip(aom_image_t *img) { + /* Note: In the calculation pointer adjustment calculation, we want the + * rhs to be promoted to a signed type. Section 6.3.1.8 of the ISO C99 + * standard indicates that if the adjustment parameter is unsigned, the + * stride parameter will be promoted to unsigned, causing errors when + * the lhs is a larger type than the rhs. + */ + img->planes[AOM_PLANE_Y] += (signed)(img->d_h - 1) * img->stride[AOM_PLANE_Y]; + img->stride[AOM_PLANE_Y] = -img->stride[AOM_PLANE_Y]; + + img->planes[AOM_PLANE_U] += (signed)((img->d_h >> img->y_chroma_shift) - 1) * + img->stride[AOM_PLANE_U]; + img->stride[AOM_PLANE_U] = -img->stride[AOM_PLANE_U]; + + img->planes[AOM_PLANE_V] += (signed)((img->d_h >> img->y_chroma_shift) - 1) * + img->stride[AOM_PLANE_V]; + img->stride[AOM_PLANE_V] = -img->stride[AOM_PLANE_V]; + + img->planes[AOM_PLANE_ALPHA] += + (signed)(img->d_h - 1) * img->stride[AOM_PLANE_ALPHA]; + img->stride[AOM_PLANE_ALPHA] = -img->stride[AOM_PLANE_ALPHA]; +} + +void aom_img_free(aom_image_t *img) { + if (img) { + if (img->img_data && img->img_data_owner) aom_free(img->img_data); + + if (img->self_allocd) free(img); + } +} diff --git a/third_party/aom/aom_dsp/add_noise.c b/third_party/aom/aom_dsp/add_noise.c new file mode 100644 index 0000000000..389cf2049f --- /dev/null +++ b/third_party/aom/aom_dsp/add_noise.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" + +void aom_plane_add_noise_c(uint8_t *start, char *noise, char blackclamp[16], + char whiteclamp[16], char bothclamp[16], + unsigned int width, unsigned int height, int pitch) { + unsigned int i, j; + + for (i = 0; i < height; ++i) { + uint8_t *pos = start + i * pitch; + char *ref = (char *)(noise + (rand() & 0xff)); // NOLINT + + for (j = 0; j < width; ++j) { + int v = pos[j]; + + v = clamp(v - blackclamp[0], 0, 255); + v = clamp(v + bothclamp[0], 0, 255); + v = clamp(v - whiteclamp[0], 0, 255); + + pos[j] = v + ref[j]; + } + } +} + +static double gaussian(double sigma, double mu, double x) { + return 1 / (sigma * sqrt(2.0 * 3.14159265)) * + (exp(-(x - mu) * (x - mu) / (2 * sigma * sigma))); +} + +int aom_setup_noise(double sigma, int size, char *noise) { + char char_dist[256]; + int next = 0, i, j; + + // set up a 256 entry lookup that matches gaussian distribution + for (i = -32; i < 32; ++i) { + const int a_i = (int)(0.5 + 256 * gaussian(sigma, 0, i)); + if (a_i) { + for (j = 0; j < a_i; ++j) { + char_dist[next + j] = (char)i; + } + next = next + j; + } + } + + // Rounding error - might mean we have less than 256. + for (; next < 256; ++next) { + char_dist[next] = 0; + } + + for (i = 0; i < size; ++i) { + noise[i] = char_dist[rand() & 0xff]; // NOLINT + } + + // Returns the highest non 0 value used in distribution. + return -char_dist[0]; +} diff --git a/third_party/aom/aom_dsp/ans.h b/third_party/aom/aom_dsp/ans.h new file mode 100644 index 0000000000..a7a2f0eab3 --- /dev/null +++ b/third_party/aom/aom_dsp/ans.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_ANS_H_ +#define AOM_DSP_ANS_H_ +// Constants, types and utilities for Asymmetric Numeral Systems +// http://arxiv.org/abs/1311.2540v2 + +#include +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/prob.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +// Use windowed ANS, size is passed in at initialization +#define ANS_MAX_SYMBOLS 1 +#define ANS_REVERSE 1 + +typedef uint8_t AnsP8; +#define ANS_P8_PRECISION 256u +#define ANS_P8_SHIFT 8 +#define RANS_PROB_BITS 15 +#define RANS_PRECISION (1u << RANS_PROB_BITS) + +// L_BASE is the ANS base state. L_BASE % PRECISION must be 0. +#define L_BASE (1u << 17) +#define IO_BASE 256 +// Range I = { L_BASE, L_BASE + 1, ..., L_BASE * IO_BASE - 1 } + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus +#endif // AOM_DSP_ANS_H_ diff --git a/third_party/aom/aom_dsp/ansreader.h b/third_party/aom/aom_dsp/ansreader.h new file mode 100644 index 0000000000..e50c63b2d1 --- /dev/null +++ b/third_party/aom/aom_dsp/ansreader.h @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_ANSREADER_H_ +#define AOM_DSP_ANSREADER_H_ +// An implementation of Asymmetric Numeral Systems +// http://arxiv.org/abs/1311.2540v2 +// Implements decoding of: +// * rABS (range Asymmetric Binary Systems), a boolean coder +// * rANS (range Asymmetric Numeral Systems), a multi-symbol coder + +#include +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/prob.h" +#include "aom_dsp/ans.h" +#include "aom_ports/mem_ops.h" +#if CONFIG_ACCOUNTING +#include "av1/decoder/accounting.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +struct AnsDecoder { + const uint8_t *buf; + int buf_offset; + uint32_t state; +#if ANS_MAX_SYMBOLS + int symbols_left; + int window_size; +#endif +#if CONFIG_ACCOUNTING + Accounting *accounting; +#endif +}; + +static INLINE int ans_read_reinit(struct AnsDecoder *const ans); + +static INLINE unsigned refill_state(struct AnsDecoder *const ans, + unsigned state) { +#if ANS_REVERSE + while (state < L_BASE && ans->buf_offset < 0) { + state = state * IO_BASE + ans->buf[ans->buf_offset++]; + } +#else + while (state < L_BASE && ans->buf_offset > 0) { + state = state * IO_BASE + ans->buf[--ans->buf_offset]; + } +#endif + return state; +} + +// Decode one rABS encoded boolean where the probability of the value being zero +// is p0. +static INLINE int rabs_read(struct AnsDecoder *ans, AnsP8 p0) { +#if ANS_MAX_SYMBOLS + if (ans->symbols_left-- == 0) { + ans_read_reinit(ans); + ans->symbols_left--; + } +#endif + unsigned state = refill_state(ans, ans->state); + const unsigned quotient = state / ANS_P8_PRECISION; + const unsigned remainder = state % ANS_P8_PRECISION; + const int value = remainder >= p0; + const unsigned qp0 = quotient * p0; + if (value) + state = state - qp0 - p0; + else + state = qp0 + remainder; + ans->state = state; + return value; +} + +// Decode one rABS encoded boolean where the probability of the value being zero +// is one half. +static INLINE int rabs_read_bit(struct AnsDecoder *ans) { +#if ANS_MAX_SYMBOLS + if (ans->symbols_left-- == 0) { + ans_read_reinit(ans); + ans->symbols_left--; + } +#endif + unsigned state = refill_state(ans, ans->state); + const int value = !!(state & 0x80); + ans->state = ((state >> 1) & ~0x7F) | (state & 0x7F); + return value; +} + +struct rans_dec_sym { + uint8_t val; + aom_cdf_prob prob; + aom_cdf_prob cum_prob; // not-inclusive +}; + +static INLINE void fetch_sym(struct rans_dec_sym *out, const aom_cdf_prob *cdf, + aom_cdf_prob rem) { + int i; + aom_cdf_prob cum_prob = 0, top_prob; + // TODO(skal): if critical, could be a binary search. + // Or, better, an O(1) alias-table. + for (i = 0; rem >= (top_prob = cdf[i]); ++i) { + cum_prob = top_prob; + } + out->val = i; + out->prob = top_prob - cum_prob; + out->cum_prob = cum_prob; +} + +static INLINE int rans_read(struct AnsDecoder *ans, const aom_cdf_prob *tab) { + unsigned rem; + unsigned quo; + struct rans_dec_sym sym; +#if ANS_MAX_SYMBOLS + if (ans->symbols_left-- == 0) { + ans_read_reinit(ans); + ans->symbols_left--; + } +#endif + ans->state = refill_state(ans, ans->state); + quo = ans->state / RANS_PRECISION; + rem = ans->state % RANS_PRECISION; + fetch_sym(&sym, tab, rem); + ans->state = quo * sym.prob + rem - sym.cum_prob; + return sym.val; +} + +static INLINE int ans_read_init(struct AnsDecoder *const ans, + const uint8_t *const buf, int offset) { + unsigned x; + if (offset < 1) return 1; +#if ANS_REVERSE + ans->buf = buf + offset; + ans->buf_offset = -offset; + x = buf[0]; + if ((x & 0x80) == 0) { // Marker is 0xxx xxxx + if (offset < 2) return 1; + ans->buf_offset += 2; + ans->state = mem_get_be16(buf) & 0x7FFF; +#if L_BASE * IO_BASE > (1 << 23) + } else if ((x & 0xC0) == 0x80) { // Marker is 10xx xxxx + if (offset < 3) return 1; + ans->buf_offset += 3; + ans->state = mem_get_be24(buf) & 0x3FFFFF; + } else { // Marker is 11xx xxxx + if (offset < 4) return 1; + ans->buf_offset += 4; + ans->state = mem_get_be32(buf) & 0x3FFFFFFF; +#else + } else { // Marker is 1xxx xxxx + if (offset < 3) return 1; + ans->buf_offset += 3; + ans->state = mem_get_be24(buf) & 0x7FFFFF; +#endif + } +#else + ans->buf = buf; + x = buf[offset - 1]; + if ((x & 0x80) == 0) { // Marker is 0xxx xxxx + if (offset < 2) return 1; + ans->buf_offset = offset - 2; + ans->state = mem_get_le16(buf + offset - 2) & 0x7FFF; + } else if ((x & 0xC0) == 0x80) { // Marker is 10xx xxxx + if (offset < 3) return 1; + ans->buf_offset = offset - 3; + ans->state = mem_get_le24(buf + offset - 3) & 0x3FFFFF; + } else if ((x & 0xE0) == 0xE0) { // Marker is 111x xxxx + if (offset < 4) return 1; + ans->buf_offset = offset - 4; + ans->state = mem_get_le32(buf + offset - 4) & 0x1FFFFFFF; + } else { + // Marker 110x xxxx implies this byte is a superframe marker + return 1; + } +#endif // ANS_REVERSE +#if CONFIG_ACCOUNTING + ans->accounting = NULL; +#endif + ans->state += L_BASE; + if (ans->state >= L_BASE * IO_BASE) return 1; +#if ANS_MAX_SYMBOLS + assert(ans->window_size > 1); + ans->symbols_left = ans->window_size; +#endif + return 0; +} + +#if ANS_REVERSE +static INLINE int ans_read_reinit(struct AnsDecoder *const ans) { + return ans_read_init(ans, ans->buf + ans->buf_offset, -ans->buf_offset); +} +#endif + +static INLINE int ans_read_end(const struct AnsDecoder *const ans) { + return ans->buf_offset == 0 && ans->state < L_BASE; +} + +static INLINE int ans_reader_has_error(const struct AnsDecoder *const ans) { + return ans->state < L_BASE / RANS_PRECISION; +} +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus +#endif // AOM_DSP_ANSREADER_H_ diff --git a/third_party/aom/aom_dsp/answriter.h b/third_party/aom/aom_dsp/answriter.h new file mode 100644 index 0000000000..353acf1a94 --- /dev/null +++ b/third_party/aom/aom_dsp/answriter.h @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_ANSWRITER_H_ +#define AOM_DSP_ANSWRITER_H_ +// An implementation of Asymmetric Numeral Systems +// http://arxiv.org/abs/1311.2540v2 +// Implements encoding of: +// * rABS (range Asymmetric Binary Systems), a boolean coder +// * rANS (range Asymmetric Numeral Systems), a multi-symbol coder + +#include +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/ans.h" +#include "aom_dsp/prob.h" +#include "aom_ports/mem_ops.h" +#include "av1/common/odintrin.h" + +#if RANS_PRECISION <= OD_DIVU_DMAX +#define ANS_DIVREM(quotient, remainder, dividend, divisor) \ + do { \ + quotient = OD_DIVU_SMALL((dividend), (divisor)); \ + remainder = (dividend) - (quotient) * (divisor); \ + } while (0) +#else +#define ANS_DIVREM(quotient, remainder, dividend, divisor) \ + do { \ + quotient = (dividend) / (divisor); \ + remainder = (dividend) % (divisor); \ + } while (0) +#endif + +#define ANS_DIV8(dividend, divisor) OD_DIVU_SMALL((dividend), (divisor)) + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +struct AnsCoder { + uint8_t *buf; + int buf_offset; + uint32_t state; +}; + +static INLINE void ans_write_init(struct AnsCoder *const ans, + uint8_t *const buf) { + ans->buf = buf; + ans->buf_offset = 0; + ans->state = L_BASE; +} + +static INLINE int ans_write_end(struct AnsCoder *const ans) { + uint32_t state; + int ans_size; + assert(ans->state >= L_BASE); + assert(ans->state < L_BASE * IO_BASE); + state = ans->state - L_BASE; + if (state < (1u << 15)) { + mem_put_le16(ans->buf + ans->buf_offset, (0x00u << 15) + state); + ans_size = ans->buf_offset + 2; +#if ANS_REVERSE +#if L_BASE * IO_BASE > (1 << 23) + } else if (state < (1u << 22)) { + mem_put_le24(ans->buf + ans->buf_offset, (0x02u << 22) + state); + ans_size = ans->buf_offset + 3; + } else if (state < (1u << 30)) { + mem_put_le32(ans->buf + ans->buf_offset, (0x03u << 30) + state); + ans_size = ans->buf_offset + 4; +#else + } else if (state < (1u << 23)) { + mem_put_le24(ans->buf + ans->buf_offset, (0x01u << 23) + state); + ans_size = ans->buf_offset + 3; +#endif +#else + } else if (state < (1u << 22)) { + mem_put_le24(ans->buf + ans->buf_offset, (0x02u << 22) + state); + ans_size = ans->buf_offset + 3; + } else if (state < (1u << 29)) { + mem_put_le32(ans->buf + ans->buf_offset, (0x07u << 29) + state); + ans_size = ans->buf_offset + 4; +#endif + } else { + assert(0 && "State is too large to be serialized"); + return ans->buf_offset; + } +#if ANS_REVERSE + { + int i; + uint8_t tmp; + for (i = 0; i < (ans_size >> 1); i++) { + tmp = ans->buf[i]; + ans->buf[i] = ans->buf[ans_size - 1 - i]; + ans->buf[ans_size - 1 - i] = tmp; + } + ans->buf += ans_size; + ans->buf_offset = 0; + ans->state = L_BASE; + } +#endif + return ans_size; +} + +// Write one boolean using rABS where p0 is the probability of the value being +// zero. +static INLINE void rabs_write(struct AnsCoder *ans, int value, AnsP8 p0) { + const AnsP8 p = ANS_P8_PRECISION - p0; + const unsigned l_s = value ? p : p0; + unsigned state = ans->state; + while (state >= L_BASE / ANS_P8_PRECISION * IO_BASE * l_s) { + ans->buf[ans->buf_offset++] = state % IO_BASE; + state /= IO_BASE; + } + const unsigned quotient = ANS_DIV8(state, l_s); + const unsigned remainder = state - quotient * l_s; + ans->state = quotient * ANS_P8_PRECISION + remainder + (value ? p0 : 0); +} + +// Encode one symbol using rANS. +// cum_prob: The cumulative probability before this symbol (the offset of +// the symbol in the symbol cycle) +// prob: The probability of this symbol (l_s from the paper) +// RANS_PRECISION takes the place of m from the paper. +static INLINE void rans_write(struct AnsCoder *ans, aom_cdf_prob cum_prob, + aom_cdf_prob prob) { + unsigned quotient, remainder; + while (ans->state >= L_BASE / RANS_PRECISION * IO_BASE * prob) { + ans->buf[ans->buf_offset++] = ans->state % IO_BASE; + ans->state /= IO_BASE; + } + ANS_DIVREM(quotient, remainder, ans->state, prob); + ans->state = quotient * RANS_PRECISION + remainder + cum_prob; +} + +#undef ANS_DIV8 +#undef ANS_DIVREM +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus +#endif // AOM_DSP_ANSWRITER_H_ diff --git a/third_party/aom/aom_dsp/aom_convolve.c b/third_party/aom/aom_dsp/aom_convolve.c new file mode 100644 index 0000000000..74f4c00fbb --- /dev/null +++ b/third_party/aom/aom_dsp/aom_convolve.c @@ -0,0 +1,854 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_dsp/aom_convolve.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/aom_filter.h" +#include "aom_ports/mem.h" + +static void convolve_horiz(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *x_filters, int x0_q4, + int x_step_q4, int w, int h) { + int x, y; + src -= SUBPEL_TAPS / 2 - 1; + for (y = 0; y < h; ++y) { + int x_q4 = x0_q4; + for (x = 0; x < w; ++x) { + const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS]; + const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK]; + int k, sum = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) sum += src_x[k] * x_filter[k]; + dst[x] = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); + x_q4 += x_step_q4; + } + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_avg_horiz(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *x_filters, int x0_q4, + int x_step_q4, int w, int h) { + int x, y; + src -= SUBPEL_TAPS / 2 - 1; + for (y = 0; y < h; ++y) { + int x_q4 = x0_q4; + for (x = 0; x < w; ++x) { + const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS]; + const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK]; + int k, sum = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) sum += src_x[k] * x_filter[k]; + dst[x] = ROUND_POWER_OF_TWO( + dst[x] + clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)), 1); + x_q4 += x_step_q4; + } + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_vert(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *y_filters, int y0_q4, + int y_step_q4, int w, int h) { + int x, y; + src -= src_stride * (SUBPEL_TAPS / 2 - 1); + + for (x = 0; x < w; ++x) { + int y_q4 = y0_q4; + for (y = 0; y < h; ++y) { + const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; + const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK]; + int k, sum = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) + sum += src_y[k * src_stride] * y_filter[k]; + dst[y * dst_stride] = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); + y_q4 += y_step_q4; + } + ++src; + ++dst; + } +} + +static void convolve_avg_vert(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *y_filters, int y0_q4, + int y_step_q4, int w, int h) { + int x, y; + src -= src_stride * (SUBPEL_TAPS / 2 - 1); + + for (x = 0; x < w; ++x) { + int y_q4 = y0_q4; + for (y = 0; y < h; ++y) { + const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; + const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK]; + int k, sum = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) + sum += src_y[k * src_stride] * y_filter[k]; + dst[y * dst_stride] = ROUND_POWER_OF_TWO( + dst[y * dst_stride] + + clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)), + 1); + y_q4 += y_step_q4; + } + ++src; + ++dst; + } +} + +static void convolve(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const InterpKernel *const x_filters, + int x0_q4, int x_step_q4, + const InterpKernel *const y_filters, int y0_q4, + int y_step_q4, int w, int h) { + // Note: Fixed size intermediate buffer, temp, places limits on parameters. + // 2d filtering proceeds in 2 steps: + // (1) Interpolate horizontally into an intermediate buffer, temp. + // (2) Interpolate temp vertically to derive the sub-pixel result. + // Deriving the maximum number of rows in the temp buffer (135): + // --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative). + // --Largest block size is 64x64 pixels. + // --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the + // original frame (in 1/16th pixel units). + // --Must round-up because block may be located at sub-pixel position. + // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails. + // --((64 - 1) * 32 + 15) >> 4 + 8 = 135. + uint8_t temp[MAX_EXT_SIZE * MAX_SB_SIZE]; + int intermediate_height = + (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS; + + assert(w <= MAX_SB_SIZE); + assert(h <= MAX_SB_SIZE); + + assert(y_step_q4 <= 32); + assert(x_step_q4 <= 32); + + convolve_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1), src_stride, temp, + MAX_SB_SIZE, x_filters, x0_q4, x_step_q4, w, + intermediate_height); + convolve_vert(temp + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1), MAX_SB_SIZE, dst, + dst_stride, y_filters, y0_q4, y_step_q4, w, h); +} + +static const InterpKernel *get_filter_base(const int16_t *filter) { + // NOTE: This assumes that the filter table is 256-byte aligned. + // TODO(agrange) Modify to make independent of table alignment. + return (const InterpKernel *)(((intptr_t)filter) & ~((intptr_t)0xFF)); +} + +static int get_filter_offset(const int16_t *f, const InterpKernel *base) { + return (int)((const InterpKernel *)(intptr_t)f - base); +} + +void aom_convolve8_horiz_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + const InterpKernel *const filters_x = get_filter_base(filter_x); + const int x0_q4 = get_filter_offset(filter_x, filters_x); + + (void)filter_y; + (void)y_step_q4; + + convolve_horiz(src, src_stride, dst, dst_stride, filters_x, x0_q4, x_step_q4, + w, h); +} + +void aom_convolve8_avg_horiz_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + const InterpKernel *const filters_x = get_filter_base(filter_x); + const int x0_q4 = get_filter_offset(filter_x, filters_x); + + (void)filter_y; + (void)y_step_q4; + + convolve_avg_horiz(src, src_stride, dst, dst_stride, filters_x, x0_q4, + x_step_q4, w, h); +} + +void aom_convolve8_vert_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + const InterpKernel *const filters_y = get_filter_base(filter_y); + const int y0_q4 = get_filter_offset(filter_y, filters_y); + + (void)filter_x; + (void)x_step_q4; + + convolve_vert(src, src_stride, dst, dst_stride, filters_y, y0_q4, y_step_q4, + w, h); +} + +void aom_convolve8_avg_vert_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + const InterpKernel *const filters_y = get_filter_base(filter_y); + const int y0_q4 = get_filter_offset(filter_y, filters_y); + + (void)filter_x; + (void)x_step_q4; + + convolve_avg_vert(src, src_stride, dst, dst_stride, filters_y, y0_q4, + y_step_q4, w, h); +} + +void aom_convolve8_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, + int x_step_q4, const int16_t *filter_y, int y_step_q4, + int w, int h) { + const InterpKernel *const filters_x = get_filter_base(filter_x); + const int x0_q4 = get_filter_offset(filter_x, filters_x); + + const InterpKernel *const filters_y = get_filter_base(filter_y); + const int y0_q4 = get_filter_offset(filter_y, filters_y); + + convolve(src, src_stride, dst, dst_stride, filters_x, x0_q4, x_step_q4, + filters_y, y0_q4, y_step_q4, w, h); +} + +void aom_convolve8_avg_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, + int x_step_q4, const int16_t *filter_y, int y_step_q4, + int w, int h) { + /* Fixed size intermediate buffer places limits on parameters. */ + DECLARE_ALIGNED(16, uint8_t, temp[MAX_SB_SIZE * MAX_SB_SIZE]); + assert(w <= MAX_SB_SIZE); + assert(h <= MAX_SB_SIZE); + + aom_convolve8_c(src, src_stride, temp, MAX_SB_SIZE, filter_x, x_step_q4, + filter_y, y_step_q4, w, h); + aom_convolve_avg_c(temp, MAX_SB_SIZE, dst, dst_stride, NULL, 0, NULL, 0, w, + h); +} + +void aom_convolve_copy_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, + int filter_x_stride, const int16_t *filter_y, + int filter_y_stride, int w, int h) { + int r; + + (void)filter_x; + (void)filter_x_stride; + (void)filter_y; + (void)filter_y_stride; + + for (r = h; r > 0; --r) { + memcpy(dst, src, w); + src += src_stride; + dst += dst_stride; + } +} + +void aom_convolve_avg_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, + int filter_x_stride, const int16_t *filter_y, + int filter_y_stride, int w, int h) { + int x, y; + + (void)filter_x; + (void)filter_x_stride; + (void)filter_y; + (void)filter_y_stride; + + for (y = 0; y < h; ++y) { + for (x = 0; x < w; ++x) dst[x] = ROUND_POWER_OF_TWO(dst[x] + src[x], 1); + + src += src_stride; + dst += dst_stride; + } +} + +void aom_scaled_horiz_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, + int x_step_q4, const int16_t *filter_y, int y_step_q4, + int w, int h) { + aom_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter_x, x_step_q4, + filter_y, y_step_q4, w, h); +} + +void aom_scaled_vert_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, + int x_step_q4, const int16_t *filter_y, int y_step_q4, + int w, int h) { + aom_convolve8_vert_c(src, src_stride, dst, dst_stride, filter_x, x_step_q4, + filter_y, y_step_q4, w, h); +} + +void aom_scaled_2d_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, + int x_step_q4, const int16_t *filter_y, int y_step_q4, + int w, int h) { + aom_convolve8_c(src, src_stride, dst, dst_stride, filter_x, x_step_q4, + filter_y, y_step_q4, w, h); +} + +void aom_scaled_avg_horiz_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + aom_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); +} + +void aom_scaled_avg_vert_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + aom_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); +} + +void aom_scaled_avg_2d_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, + int x_step_q4, const int16_t *filter_y, int y_step_q4, + int w, int h) { + aom_convolve8_avg_c(src, src_stride, dst, dst_stride, filter_x, x_step_q4, + filter_y, y_step_q4, w, h); +} + +#if CONFIG_LOOP_RESTORATION +static void convolve_add_src_horiz(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *x_filters, int x0_q4, + int x_step_q4, int w, int h) { + int x, y; + src -= SUBPEL_TAPS / 2 - 1; + for (y = 0; y < h; ++y) { + int x_q4 = x0_q4; + for (x = 0; x < w; ++x) { + const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS]; + const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK]; + int k, sum = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) sum += src_x[k] * x_filter[k]; + dst[x] = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS) + + src_x[SUBPEL_TAPS / 2 - 1]); + x_q4 += x_step_q4; + } + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_add_src_vert(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *y_filters, int y0_q4, + int y_step_q4, int w, int h) { + int x, y; + src -= src_stride * (SUBPEL_TAPS / 2 - 1); + + for (x = 0; x < w; ++x) { + int y_q4 = y0_q4; + for (y = 0; y < h; ++y) { + const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; + const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK]; + int k, sum = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) + sum += src_y[k * src_stride] * y_filter[k]; + dst[y * dst_stride] = + clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS) + + src_y[(SUBPEL_TAPS / 2 - 1) * src_stride]); + y_q4 += y_step_q4; + } + ++src; + ++dst; + } +} + +static void convolve_add_src(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *const x_filters, int x0_q4, + int x_step_q4, const InterpKernel *const y_filters, + int y0_q4, int y_step_q4, int w, int h) { + uint8_t temp[MAX_EXT_SIZE * MAX_SB_SIZE]; + int intermediate_height = + (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS; + + assert(w <= MAX_SB_SIZE); + assert(h <= MAX_SB_SIZE); + + assert(y_step_q4 <= 32); + assert(x_step_q4 <= 32); + + convolve_add_src_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1), src_stride, + temp, MAX_SB_SIZE, x_filters, x0_q4, x_step_q4, w, + intermediate_height); + convolve_add_src_vert(temp + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1), MAX_SB_SIZE, + dst, dst_stride, y_filters, y0_q4, y_step_q4, w, h); +} + +void aom_convolve8_add_src_horiz_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h) { + const InterpKernel *const filters_x = get_filter_base(filter_x); + const int x0_q4 = get_filter_offset(filter_x, filters_x); + + (void)filter_y; + (void)y_step_q4; + + convolve_add_src_horiz(src, src_stride, dst, dst_stride, filters_x, x0_q4, + x_step_q4, w, h); +} + +void aom_convolve8_add_src_vert_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + const InterpKernel *const filters_y = get_filter_base(filter_y); + const int y0_q4 = get_filter_offset(filter_y, filters_y); + + (void)filter_x; + (void)x_step_q4; + + convolve_add_src_vert(src, src_stride, dst, dst_stride, filters_y, y0_q4, + y_step_q4, w, h); +} + +void aom_convolve8_add_src_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + const InterpKernel *const filters_x = get_filter_base(filter_x); + const int x0_q4 = get_filter_offset(filter_x, filters_x); + + const InterpKernel *const filters_y = get_filter_base(filter_y); + const int y0_q4 = get_filter_offset(filter_y, filters_y); + + convolve_add_src(src, src_stride, dst, dst_stride, filters_x, x0_q4, + x_step_q4, filters_y, y0_q4, y_step_q4, w, h); +} +#endif // CONFIG_LOOP_RESTORATION + +#if CONFIG_HIGHBITDEPTH +static void highbd_convolve_horiz(const uint8_t *src8, ptrdiff_t src_stride, + uint8_t *dst8, ptrdiff_t dst_stride, + const InterpKernel *x_filters, int x0_q4, + int x_step_q4, int w, int h, int bd) { + int x, y; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + src -= SUBPEL_TAPS / 2 - 1; + for (y = 0; y < h; ++y) { + int x_q4 = x0_q4; + for (x = 0; x < w; ++x) { + const uint16_t *const src_x = &src[x_q4 >> SUBPEL_BITS]; + const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK]; + int k, sum = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) sum += src_x[k] * x_filter[k]; + dst[x] = clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd); + x_q4 += x_step_q4; + } + src += src_stride; + dst += dst_stride; + } +} + +static void highbd_convolve_avg_horiz(const uint8_t *src8, ptrdiff_t src_stride, + uint8_t *dst8, ptrdiff_t dst_stride, + const InterpKernel *x_filters, int x0_q4, + int x_step_q4, int w, int h, int bd) { + int x, y; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + src -= SUBPEL_TAPS / 2 - 1; + for (y = 0; y < h; ++y) { + int x_q4 = x0_q4; + for (x = 0; x < w; ++x) { + const uint16_t *const src_x = &src[x_q4 >> SUBPEL_BITS]; + const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK]; + int k, sum = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) sum += src_x[k] * x_filter[k]; + dst[x] = ROUND_POWER_OF_TWO( + dst[x] + clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd), + 1); + x_q4 += x_step_q4; + } + src += src_stride; + dst += dst_stride; + } +} + +static void highbd_convolve_vert(const uint8_t *src8, ptrdiff_t src_stride, + uint8_t *dst8, ptrdiff_t dst_stride, + const InterpKernel *y_filters, int y0_q4, + int y_step_q4, int w, int h, int bd) { + int x, y; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + src -= src_stride * (SUBPEL_TAPS / 2 - 1); + for (x = 0; x < w; ++x) { + int y_q4 = y0_q4; + for (y = 0; y < h; ++y) { + const uint16_t *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; + const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK]; + int k, sum = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) + sum += src_y[k * src_stride] * y_filter[k]; + dst[y * dst_stride] = + clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd); + y_q4 += y_step_q4; + } + ++src; + ++dst; + } +} + +static void highbd_convolve_avg_vert(const uint8_t *src8, ptrdiff_t src_stride, + uint8_t *dst8, ptrdiff_t dst_stride, + const InterpKernel *y_filters, int y0_q4, + int y_step_q4, int w, int h, int bd) { + int x, y; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + src -= src_stride * (SUBPEL_TAPS / 2 - 1); + for (x = 0; x < w; ++x) { + int y_q4 = y0_q4; + for (y = 0; y < h; ++y) { + const uint16_t *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; + const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK]; + int k, sum = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) + sum += src_y[k * src_stride] * y_filter[k]; + dst[y * dst_stride] = ROUND_POWER_OF_TWO( + dst[y * dst_stride] + + clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd), + 1); + y_q4 += y_step_q4; + } + ++src; + ++dst; + } +} + +static void highbd_convolve(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *const x_filters, int x0_q4, + int x_step_q4, const InterpKernel *const y_filters, + int y0_q4, int y_step_q4, int w, int h, int bd) { + // Note: Fixed size intermediate buffer, temp, places limits on parameters. + // 2d filtering proceeds in 2 steps: + // (1) Interpolate horizontally into an intermediate buffer, temp. + // (2) Interpolate temp vertically to derive the sub-pixel result. + // Deriving the maximum number of rows in the temp buffer (135): + // --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative). + // --Largest block size is 64x64 pixels. + // --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the + // original frame (in 1/16th pixel units). + // --Must round-up because block may be located at sub-pixel position. + // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails. + // --((64 - 1) * 32 + 15) >> 4 + 8 = 135. + uint16_t temp[MAX_EXT_SIZE * MAX_SB_SIZE]; + int intermediate_height = + (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS; + + assert(w <= MAX_SB_SIZE); + assert(h <= MAX_SB_SIZE); + assert(y_step_q4 <= 32); + assert(x_step_q4 <= 32); + + highbd_convolve_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1), src_stride, + CONVERT_TO_BYTEPTR(temp), MAX_SB_SIZE, x_filters, x0_q4, + x_step_q4, w, intermediate_height, bd); + highbd_convolve_vert( + CONVERT_TO_BYTEPTR(temp) + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1), + MAX_SB_SIZE, dst, dst_stride, y_filters, y0_q4, y_step_q4, w, h, bd); +} + +void aom_highbd_convolve8_horiz_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h, int bd) { + const InterpKernel *const filters_x = get_filter_base(filter_x); + const int x0_q4 = get_filter_offset(filter_x, filters_x); + (void)filter_y; + (void)y_step_q4; + + highbd_convolve_horiz(src, src_stride, dst, dst_stride, filters_x, x0_q4, + x_step_q4, w, h, bd); +} + +void aom_highbd_convolve8_avg_horiz_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h, int bd) { + const InterpKernel *const filters_x = get_filter_base(filter_x); + const int x0_q4 = get_filter_offset(filter_x, filters_x); + (void)filter_y; + (void)y_step_q4; + + highbd_convolve_avg_horiz(src, src_stride, dst, dst_stride, filters_x, x0_q4, + x_step_q4, w, h, bd); +} + +void aom_highbd_convolve8_vert_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h, int bd) { + const InterpKernel *const filters_y = get_filter_base(filter_y); + const int y0_q4 = get_filter_offset(filter_y, filters_y); + (void)filter_x; + (void)x_step_q4; + + highbd_convolve_vert(src, src_stride, dst, dst_stride, filters_y, y0_q4, + y_step_q4, w, h, bd); +} + +void aom_highbd_convolve8_avg_vert_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h, int bd) { + const InterpKernel *const filters_y = get_filter_base(filter_y); + const int y0_q4 = get_filter_offset(filter_y, filters_y); + (void)filter_x; + (void)x_step_q4; + + highbd_convolve_avg_vert(src, src_stride, dst, dst_stride, filters_y, y0_q4, + y_step_q4, w, h, bd); +} + +void aom_highbd_convolve8_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h, int bd) { + const InterpKernel *const filters_x = get_filter_base(filter_x); + const int x0_q4 = get_filter_offset(filter_x, filters_x); + + const InterpKernel *const filters_y = get_filter_base(filter_y); + const int y0_q4 = get_filter_offset(filter_y, filters_y); + + highbd_convolve(src, src_stride, dst, dst_stride, filters_x, x0_q4, x_step_q4, + filters_y, y0_q4, y_step_q4, w, h, bd); +} + +void aom_highbd_convolve8_avg_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h, int bd) { + // Fixed size intermediate buffer places limits on parameters. + DECLARE_ALIGNED(16, uint16_t, temp[MAX_SB_SIZE * MAX_SB_SIZE]); + assert(w <= MAX_SB_SIZE); + assert(h <= MAX_SB_SIZE); + + aom_highbd_convolve8_c(src, src_stride, CONVERT_TO_BYTEPTR(temp), MAX_SB_SIZE, + filter_x, x_step_q4, filter_y, y_step_q4, w, h, bd); + aom_highbd_convolve_avg_c(CONVERT_TO_BYTEPTR(temp), MAX_SB_SIZE, dst, + dst_stride, NULL, 0, NULL, 0, w, h, bd); +} + +void aom_highbd_convolve_copy_c(const uint8_t *src8, ptrdiff_t src_stride, + uint8_t *dst8, ptrdiff_t dst_stride, + const int16_t *filter_x, int filter_x_stride, + const int16_t *filter_y, int filter_y_stride, + int w, int h, int bd) { + int r; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + (void)filter_x; + (void)filter_y; + (void)filter_x_stride; + (void)filter_y_stride; + (void)bd; + + for (r = h; r > 0; --r) { + memcpy(dst, src, w * sizeof(uint16_t)); + src += src_stride; + dst += dst_stride; + } +} + +void aom_highbd_convolve_avg_c(const uint8_t *src8, ptrdiff_t src_stride, + uint8_t *dst8, ptrdiff_t dst_stride, + const int16_t *filter_x, int filter_x_stride, + const int16_t *filter_y, int filter_y_stride, + int w, int h, int bd) { + int x, y; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + (void)filter_x; + (void)filter_y; + (void)filter_x_stride; + (void)filter_y_stride; + (void)bd; + + for (y = 0; y < h; ++y) { + for (x = 0; x < w; ++x) { + dst[x] = ROUND_POWER_OF_TWO(dst[x] + src[x], 1); + } + src += src_stride; + dst += dst_stride; + } +} + +#if CONFIG_LOOP_RESTORATION +static void highbd_convolve_add_src_horiz(const uint8_t *src8, + ptrdiff_t src_stride, uint8_t *dst8, + ptrdiff_t dst_stride, + const InterpKernel *x_filters, + int x0_q4, int x_step_q4, int w, + int h, int bd) { + int x, y; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + src -= SUBPEL_TAPS / 2 - 1; + for (y = 0; y < h; ++y) { + int x_q4 = x0_q4; + for (x = 0; x < w; ++x) { + const uint16_t *const src_x = &src[x_q4 >> SUBPEL_BITS]; + const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK]; + int k, sum = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) sum += src_x[k] * x_filter[k]; + dst[x] = clip_pixel_highbd( + ROUND_POWER_OF_TWO(sum, FILTER_BITS) + src_x[SUBPEL_TAPS / 2 - 1], + bd); + x_q4 += x_step_q4; + } + src += src_stride; + dst += dst_stride; + } +} + +static void highbd_convolve_add_src_vert(const uint8_t *src8, + ptrdiff_t src_stride, uint8_t *dst8, + ptrdiff_t dst_stride, + const InterpKernel *y_filters, + int y0_q4, int y_step_q4, int w, int h, + int bd) { + int x, y; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + src -= src_stride * (SUBPEL_TAPS / 2 - 1); + for (x = 0; x < w; ++x) { + int y_q4 = y0_q4; + for (y = 0; y < h; ++y) { + const uint16_t *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; + const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK]; + int k, sum = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) + sum += src_y[k * src_stride] * y_filter[k]; + dst[y * dst_stride] = + clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS) + + src_y[(SUBPEL_TAPS / 2 - 1) * src_stride], + bd); + y_q4 += y_step_q4; + } + ++src; + ++dst; + } +} + +static void highbd_convolve_add_src(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *const x_filters, + int x0_q4, int x_step_q4, + const InterpKernel *const y_filters, + int y0_q4, int y_step_q4, int w, int h, + int bd) { + // Note: Fixed size intermediate buffer, temp, places limits on parameters. + // 2d filtering proceeds in 2 steps: + // (1) Interpolate horizontally into an intermediate buffer, temp. + // (2) Interpolate temp vertically to derive the sub-pixel result. + // Deriving the maximum number of rows in the temp buffer (135): + // --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative). + // --Largest block size is 64x64 pixels. + // --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the + // original frame (in 1/16th pixel units). + // --Must round-up because block may be located at sub-pixel position. + // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails. + // --((64 - 1) * 32 + 15) >> 4 + 8 = 135. + uint16_t temp[MAX_EXT_SIZE * MAX_SB_SIZE]; + int intermediate_height = + (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS; + + assert(w <= MAX_SB_SIZE); + assert(h <= MAX_SB_SIZE); + assert(y_step_q4 <= 32); + assert(x_step_q4 <= 32); + + highbd_convolve_add_src_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1), + src_stride, CONVERT_TO_BYTEPTR(temp), + MAX_SB_SIZE, x_filters, x0_q4, x_step_q4, w, + intermediate_height, bd); + highbd_convolve_add_src_vert( + CONVERT_TO_BYTEPTR(temp) + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1), + MAX_SB_SIZE, dst, dst_stride, y_filters, y0_q4, y_step_q4, w, h, bd); +} + +void aom_highbd_convolve8_add_src_horiz_c( + const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, int h, int bd) { + const InterpKernel *const filters_x = get_filter_base(filter_x); + const int x0_q4 = get_filter_offset(filter_x, filters_x); + (void)filter_y; + (void)y_step_q4; + + highbd_convolve_add_src_horiz(src, src_stride, dst, dst_stride, filters_x, + x0_q4, x_step_q4, w, h, bd); +} + +void aom_highbd_convolve8_add_src_vert_c(const uint8_t *src, + ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h, int bd) { + const InterpKernel *const filters_y = get_filter_base(filter_y); + const int y0_q4 = get_filter_offset(filter_y, filters_y); + (void)filter_x; + (void)x_step_q4; + + highbd_convolve_add_src_vert(src, src_stride, dst, dst_stride, filters_y, + y0_q4, y_step_q4, w, h, bd); +} + +void aom_highbd_convolve8_add_src_c(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h, int bd) { + const InterpKernel *const filters_x = get_filter_base(filter_x); + const int x0_q4 = get_filter_offset(filter_x, filters_x); + + const InterpKernel *const filters_y = get_filter_base(filter_y); + const int y0_q4 = get_filter_offset(filter_y, filters_y); + + highbd_convolve_add_src(src, src_stride, dst, dst_stride, filters_x, x0_q4, + x_step_q4, filters_y, y0_q4, y_step_q4, w, h, bd); +} +#endif // CONFIG_LOOP_RESTORATION +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/aom_convolve.h b/third_party/aom/aom_dsp/aom_convolve.h new file mode 100644 index 0000000000..d0de6c5d20 --- /dev/null +++ b/third_party/aom/aom_dsp/aom_convolve.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_DSP_AOM_CONVOLVE_H_ +#define AOM_DSP_AOM_CONVOLVE_H_ + +#include "./aom_config.h" +#include "aom/aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Note: Fixed size intermediate buffers, place limits on parameters +// of some functions. 2d filtering proceeds in 2 steps: +// (1) Interpolate horizontally into an intermediate buffer, temp. +// (2) Interpolate temp vertically to derive the sub-pixel result. +// Deriving the maximum number of rows in the temp buffer (135): +// --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative). +// --Largest block size is 64x64 pixels. +// --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the +// original frame (in 1/16th pixel units). +// --Must round-up because block may be located at sub-pixel position. +// --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails. +// --((64 - 1) * 32 + 15) >> 4 + 8 = 135. +#if CONFIG_AV1 && CONFIG_EXT_PARTITION +#define MAX_EXT_SIZE 263 +#else +#define MAX_EXT_SIZE 135 +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + +typedef void (*convolve_fn_t)(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h); + +#if CONFIG_HIGHBITDEPTH +typedef void (*highbd_convolve_fn_t)(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h, int bd); +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_AOM_CONVOLVE_H_ diff --git a/third_party/aom/aom_dsp/aom_dsp.cmake b/third_party/aom/aom_dsp/aom_dsp.cmake new file mode 100644 index 0000000000..f00348cbcf --- /dev/null +++ b/third_party/aom/aom_dsp/aom_dsp.cmake @@ -0,0 +1,509 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +set(AOM_DSP_COMMON_SOURCES + "${AOM_ROOT}/aom_dsp/aom_convolve.c" + "${AOM_ROOT}/aom_dsp/aom_convolve.h" + "${AOM_ROOT}/aom_dsp/aom_dsp_common.h" + "${AOM_ROOT}/aom_dsp/aom_filter.h" + "${AOM_ROOT}/aom_dsp/aom_simd.h" + "${AOM_ROOT}/aom_dsp/aom_simd_inline.h" + "${AOM_ROOT}/aom_dsp/blend.h" + "${AOM_ROOT}/aom_dsp/blend_a64_hmask.c" + "${AOM_ROOT}/aom_dsp/blend_a64_mask.c" + "${AOM_ROOT}/aom_dsp/blend_a64_vmask.c" + "${AOM_ROOT}/aom_dsp/intrapred.c" + "${AOM_ROOT}/aom_dsp/loopfilter.c" + "${AOM_ROOT}/aom_dsp/prob.c" + "${AOM_ROOT}/aom_dsp/prob.h" + "${AOM_ROOT}/aom_dsp/sad.c" + "${AOM_ROOT}/aom_dsp/simd/v128_intrinsics.h" + "${AOM_ROOT}/aom_dsp/simd/v128_intrinsics_c.h" + "${AOM_ROOT}/aom_dsp/simd/v256_intrinsics.h" + "${AOM_ROOT}/aom_dsp/simd/v256_intrinsics_c.h" + "${AOM_ROOT}/aom_dsp/simd/v64_intrinsics.h" + "${AOM_ROOT}/aom_dsp/simd/v64_intrinsics_c.h" + "${AOM_ROOT}/aom_dsp/subtract.c" + "${AOM_ROOT}/aom_dsp/txfm_common.h" + "${AOM_ROOT}/aom_dsp/x86/txfm_common_intrin.h") + +set(AOM_DSP_COMMON_ASM_SSE2 + "${AOM_ROOT}/aom_dsp/x86/aom_convolve_copy_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/aom_subpixel_bilinear_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/intrapred_sse2.asm") + +set(AOM_DSP_COMMON_INTRIN_SSE2 + "${AOM_ROOT}/aom_dsp/x86/aom_asm_stubs.c" + "${AOM_ROOT}/aom_dsp/x86/convolve.h" + "${AOM_ROOT}/aom_dsp/x86/txfm_common_sse2.h" + "${AOM_ROOT}/aom_dsp/x86/loopfilter_sse2.c") + +set(AOM_DSP_COMMON_ASM_SSSE3 + "${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_ssse3.asm" + "${AOM_ROOT}/aom_dsp/x86/aom_subpixel_bilinear_ssse3.asm" + "${AOM_ROOT}/aom_dsp/x86/intrapred_ssse3.asm") + +set(AOM_DSP_COMMON_INTRIN_SSSE3 + "${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c" + "${AOM_ROOT}/aom_dsp/x86/inv_txfm_ssse3.c") + +set(AOM_DSP_COMMON_INTRIN_SSE4_1 + "${AOM_ROOT}/aom_dsp/x86/blend_a64_hmask_sse4.c" + "${AOM_ROOT}/aom_dsp/x86/blend_a64_mask_sse4.c" + "${AOM_ROOT}/aom_dsp/x86/blend_a64_vmask_sse4.c") + +set(AOM_DSP_COMMON_INTRIN_AVX2 + "${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_intrin_avx2.c" + "${AOM_ROOT}/aom_dsp/x86/fwd_txfm_avx2.c" + "${AOM_ROOT}/aom_dsp/x86/loopfilter_avx2.c") + +set(AOM_DSP_COMMON_ASM_NEON + "${AOM_ROOT}/aom_dsp/arm/aom_convolve8_avg_neon_asm.asm" + "${AOM_ROOT}/aom_dsp/arm/aom_convolve8_neon_asm.asm" + "${AOM_ROOT}/aom_dsp/arm/aom_convolve_avg_neon_asm.asm" + "${AOM_ROOT}/aom_dsp/arm/aom_convolve_copy_neon_asm.asm" + "${AOM_ROOT}/aom_dsp/arm/idct16x16_1_add_neon.asm" + "${AOM_ROOT}/aom_dsp/arm/idct16x16_add_neon.asm" + "${AOM_ROOT}/aom_dsp/arm/idct32x32_1_add_neon.asm" + "${AOM_ROOT}/aom_dsp/arm/idct32x32_add_neon.asm" + "${AOM_ROOT}/aom_dsp/arm/idct4x4_1_add_neon.asm" + "${AOM_ROOT}/aom_dsp/arm/idct4x4_add_neon.asm" + "${AOM_ROOT}/aom_dsp/arm/idct8x8_1_add_neon.asm" + "${AOM_ROOT}/aom_dsp/arm/idct8x8_add_neon.asm" + "${AOM_ROOT}/aom_dsp/arm/intrapred_neon_asm.asm" + "${AOM_ROOT}/aom_dsp/arm/loopfilter_16_neon.asm" + "${AOM_ROOT}/aom_dsp/arm/loopfilter_4_neon.asm" + "${AOM_ROOT}/aom_dsp/arm/loopfilter_8_neon.asm" + "${AOM_ROOT}/aom_dsp/arm/loopfilter_mb_neon.asm" + "${AOM_ROOT}/aom_dsp/arm/save_reg_neon.asm") + +set(AOM_DSP_COMMON_INTRIN_NEON + "${AOM_ROOT}/aom_dsp/arm/aom_convolve_neon.c" + "${AOM_ROOT}/aom_dsp/arm/avg_neon.c" + "${AOM_ROOT}/aom_dsp/arm/fwd_txfm_neon.c" + "${AOM_ROOT}/aom_dsp/arm/hadamard_neon.c" + "${AOM_ROOT}/aom_dsp/arm/idct16x16_neon.c" + "${AOM_ROOT}/aom_dsp/arm/intrapred_neon.c" + "${AOM_ROOT}/aom_dsp/arm/loopfilter_neon.c" + "${AOM_ROOT}/aom_dsp/arm/sad4d_neon.c" + "${AOM_ROOT}/aom_dsp/arm/sad_neon.c" + "${AOM_ROOT}/aom_dsp/arm/subpel_variance_neon.c" + "${AOM_ROOT}/aom_dsp/arm/subtract_neon.c" + "${AOM_ROOT}/aom_dsp/arm/variance_neon.c") + +if ("${AOM_TARGET_CPU}" STREQUAL "arm64") + set(AOM_DSP_COMMON_INTRIN_NEON + ${AOM_DSP_COMMON_INTRIN_NEON} + "${AOM_ROOT}/aom_dsp/arm/aom_convolve8_avg_neon.c" + "${AOM_ROOT}/aom_dsp/arm/aom_convolve8_neon.c" + "${AOM_ROOT}/aom_dsp/arm/aom_convolve_avg_neon.c" + "${AOM_ROOT}/aom_dsp/arm/aom_convolve_copy_neon.c" + "${AOM_ROOT}/aom_dsp/arm/idct16x16_1_add_neon.c" + "${AOM_ROOT}/aom_dsp/arm/idct16x16_add_neon.c" + "${AOM_ROOT}/aom_dsp/arm/idct32x32_1_add_neon.c" + "${AOM_ROOT}/aom_dsp/arm/idct32x32_add_neon.c" + "${AOM_ROOT}/aom_dsp/arm/idct4x4_1_add_neon.c" + "${AOM_ROOT}/aom_dsp/arm/idct4x4_add_neon.c" + "${AOM_ROOT}/aom_dsp/arm/idct8x8_1_add_neon.c" + "${AOM_ROOT}/aom_dsp/arm/idct8x8_add_neon.c" + "${AOM_ROOT}/aom_dsp/arm/intrapred_neon.c" + "${AOM_ROOT}/aom_dsp/arm/loopfilter_16_neon.c" + "${AOM_ROOT}/aom_dsp/arm/loopfilter_4_neon.c" + "${AOM_ROOT}/aom_dsp/arm/loopfilter_8_neon.c") +endif () + +set(AOM_DSP_COMMON_INTRIN_DSPR2 + "${AOM_ROOT}/aom_dsp/mips/common_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/common_dspr2.h" + "${AOM_ROOT}/aom_dsp/mips/convolve2_avg_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/convolve2_avg_horiz_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/convolve2_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/convolve2_horiz_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/convolve2_vert_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/convolve8_avg_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/convolve8_avg_horiz_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/convolve8_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/convolve8_horiz_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/convolve8_vert_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/convolve_common_dspr2.h" + "${AOM_ROOT}/aom_dsp/mips/intrapred16_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/intrapred4_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/intrapred8_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/inv_txfm_dspr2.h" + "${AOM_ROOT}/aom_dsp/mips/loopfilter_filters_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/loopfilter_filters_dspr2.h" + "${AOM_ROOT}/aom_dsp/mips/loopfilter_macros_dspr2.h" + "${AOM_ROOT}/aom_dsp/mips/loopfilter_masks_dspr2.h" + "${AOM_ROOT}/aom_dsp/mips/loopfilter_mb_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/loopfilter_mb_horiz_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/loopfilter_mb_vert_dspr2.c") + +set(AOM_DSP_COMMON_INTRIN_MSA + "${AOM_ROOT}/aom_dsp/mips/aom_convolve8_avg_horiz_msa.c" + "${AOM_ROOT}/aom_dsp/mips/aom_convolve8_avg_msa.c" + "${AOM_ROOT}/aom_dsp/mips/aom_convolve8_avg_vert_msa.c" + "${AOM_ROOT}/aom_dsp/mips/aom_convolve8_horiz_msa.c" + "${AOM_ROOT}/aom_dsp/mips/aom_convolve8_msa.c" + "${AOM_ROOT}/aom_dsp/mips/aom_convolve8_vert_msa.c" + "${AOM_ROOT}/aom_dsp/mips/aom_convolve_avg_msa.c" + "${AOM_ROOT}/aom_dsp/mips/aom_convolve_copy_msa.c" + "${AOM_ROOT}/aom_dsp/mips/aom_convolve_msa.h" + "${AOM_ROOT}/aom_dsp/mips/fwd_dct32x32_msa.c" + "${AOM_ROOT}/aom_dsp/mips/fwd_txfm_msa.c" + "${AOM_ROOT}/aom_dsp/mips/fwd_txfm_msa.h" + "${AOM_ROOT}/aom_dsp/mips/idct16x16_msa.c" + "${AOM_ROOT}/aom_dsp/mips/idct32x32_msa.c" + "${AOM_ROOT}/aom_dsp/mips/idct4x4_msa.c" + "${AOM_ROOT}/aom_dsp/mips/idct8x8_msa.c" + "${AOM_ROOT}/aom_dsp/mips/intrapred_msa.c" + "${AOM_ROOT}/aom_dsp/mips/inv_txfm_msa.h" + "${AOM_ROOT}/aom_dsp/mips/loopfilter_16_msa.c" + "${AOM_ROOT}/aom_dsp/mips/loopfilter_4_msa.c" + "${AOM_ROOT}/aom_dsp/mips/loopfilter_8_msa.c" + "${AOM_ROOT}/aom_dsp/mips/loopfilter_msa.h" + "${AOM_ROOT}/aom_dsp/mips/macros_msa.h" + "${AOM_ROOT}/aom_dsp/mips/txfm_macros_msa.h") + +if (CONFIG_HIGHBITDEPTH) + set(AOM_DSP_COMMON_ASM_SSE2 + ${AOM_DSP_COMMON_ASM_SSE2} + "${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_sse2.asm") + + set(AOM_DSP_COMMON_INTRIN_SSE2 + ${AOM_DSP_COMMON_INTRIN_SSE2} + "${AOM_ROOT}/aom_dsp/x86/highbd_loopfilter_sse2.c") + + set(AOM_DSP_COMMON_INTRIN_AVX2 + ${AOM_DSP_COMMON_INTRIN_AVX2} + "${AOM_ROOT}/aom_dsp/x86/highbd_convolve_avx2.c") +else () + set(AOM_DSP_COMMON_INTRIN_DSPR2 + ${AOM_DSP_COMMON_INTRIN_DSPR2} + "${AOM_ROOT}/aom_dsp/mips/itrans16_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/itrans32_cols_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/itrans32_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/itrans4_dspr2.c" + "${AOM_ROOT}/aom_dsp/mips/itrans8_dspr2.c") +endif () + +if (CONFIG_ANS) + set(AOM_DSP_COMMON_SOURCES + ${AOM_DSP_COMMON_SOURCES} + "${AOM_ROOT}/aom_dsp/ans.h") +elseif (CONFIG_DAALA_EC) + set(AOM_DSP_COMMON_SOURCES + ${AOM_DSP_COMMON_SOURCES} + "${AOM_ROOT}/aom_dsp/entcode.c" + "${AOM_ROOT}/aom_dsp/entcode.h") +endif () + +if (CONFIG_AV1) + set(AOM_DSP_COMMON_SOURCES + ${AOM_DSP_COMMON_SOURCES} + "${AOM_ROOT}/aom_dsp/inv_txfm.c" + "${AOM_ROOT}/aom_dsp/inv_txfm.h") + + set(AOM_DSP_COMMON_ASM_SSE2 + ${AOM_DSP_COMMON_ASM_SSE2} + "${AOM_ROOT}/aom_dsp/x86/inv_wht_sse2.asm") + + set(AOM_DSP_COMMON_INTRIN_SSE2 + ${AOM_DSP_COMMON_INTRIN_SSE2} + "${AOM_ROOT}/aom_dsp/x86/inv_txfm_sse2.c" + "${AOM_ROOT}/aom_dsp/x86/inv_txfm_sse2.h") +endif () + +if (CONFIG_DECODERS) + set(AOM_DSP_DECODER_SOURCES + "${AOM_ROOT}/aom_dsp/binary_codes_reader.c" + "${AOM_ROOT}/aom_dsp/binary_codes_reader.h" + "${AOM_ROOT}/aom_dsp/bitreader.h" + "${AOM_ROOT}/aom_dsp/bitreader_buffer.c" + "${AOM_ROOT}/aom_dsp/bitreader_buffer.h") + + if (CONFIG_ANS) + set(AOM_DSP_DECODER_SOURCES + ${AOM_DSP_DECODER_SOURCES} + "${AOM_ROOT}/aom_dsp/ansreader.h") + elseif (CONFIG_DAALA_EC) + set(AOM_DSP_DECODER_SOURCES + ${AOM_DSP_DECODER_SOURCES} + "${AOM_ROOT}/aom_dsp/daalaboolreader.c" + "${AOM_ROOT}/aom_dsp/daalaboolreader.h" + "${AOM_ROOT}/aom_dsp/entdec.c" + "${AOM_ROOT}/aom_dsp/entdec.h") + else () + set(AOM_DSP_DECODER_SOURCES + ${AOM_DSP_DECODER_SOURCES} + "${AOM_ROOT}/aom_dsp/dkboolreader.c" + "${AOM_ROOT}/aom_dsp/dkboolreader.h") + endif () +endif () + +if (CONFIG_ENCODERS) + set(AOM_DSP_ENCODER_SOURCES + "${AOM_ROOT}/aom_dsp/binary_codes_writer.c" + "${AOM_ROOT}/aom_dsp/binary_codes_writer.h" + "${AOM_ROOT}/aom_dsp/bitwriter.h" + "${AOM_ROOT}/aom_dsp/bitwriter_buffer.c" + "${AOM_ROOT}/aom_dsp/bitwriter_buffer.h" + "${AOM_ROOT}/aom_dsp/psnr.c" + "${AOM_ROOT}/aom_dsp/psnr.h" + "${AOM_ROOT}/aom_dsp/variance.c" + "${AOM_ROOT}/aom_dsp/variance.h") + + set(AOM_DSP_ENCODER_ASM_SSE2 + ${AOM_DSP_ENCODER_ASM_SSE2} + "${AOM_ROOT}/aom_dsp/x86/halfpix_variance_impl_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/sad4d_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/sad_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/subtract_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/subpel_variance_sse2.asm") + + set(AOM_DSP_ENCODER_INTRIN_SSE2 + "${AOM_ROOT}/aom_dsp/x86/quantize_sse2.c") + + set(AOM_DSP_ENCODER_ASM_SSSE3 + "${AOM_ROOT}/aom_dsp/x86/sad_ssse3.asm") + + set(AOM_DSP_ENCODER_ASM_SSSE3_X86_64 + "${AOM_ROOT}/aom_dsp/x86/fwd_txfm_ssse3_x86_64.asm" + "${AOM_ROOT}/aom_dsp/x86/ssim_opt_x86_64.asm") + + set(AOM_DSP_ENCODER_INTRIN_SSE3 "${AOM_ROOT}/aom_dsp/x86/sad_sse3.asm") + set(AOM_DSP_ENCODER_ASM_SSE4_1 "${AOM_ROOT}/aom_dsp/x86/sad_sse4.asm") + + set(AOM_DSP_ENCODER_INTRIN_AVX2 + "${AOM_ROOT}/aom_dsp/x86/sad4d_avx2.c" + "${AOM_ROOT}/aom_dsp/x86/sad_avx2.c" + "${AOM_ROOT}/aom_dsp/x86/sad_impl_avx2.c" + "${AOM_ROOT}/aom_dsp/x86/variance_avx2.c" + "${AOM_ROOT}/aom_dsp/x86/variance_impl_avx2.c") + + if (CONFIG_AV1_ENCODER) + set(AOM_DSP_ENCODER_SOURCES + ${AOM_DSP_ENCODER_SOURCES} + "${AOM_ROOT}/aom_dsp/avg.c" + "${AOM_ROOT}/aom_dsp/fwd_txfm.c" + "${AOM_ROOT}/aom_dsp/fwd_txfm.h" + "${AOM_ROOT}/aom_dsp/quantize.c" + "${AOM_ROOT}/aom_dsp/quantize.h" + "${AOM_ROOT}/aom_dsp/sum_squares.c") + + set(AOM_DSP_ENCODER_INTRIN_SSE2 + ${AOM_DSP_ENCODER_INTRIN_SSE2} + "${AOM_ROOT}/aom_dsp/x86/avg_intrin_sse2.c" + "${AOM_ROOT}/aom_dsp/x86/fwd_dct32_8cols_sse2.c" + "${AOM_ROOT}/aom_dsp/x86/fwd_dct32x32_impl_sse2.h" + "${AOM_ROOT}/aom_dsp/x86/fwd_txfm_impl_sse2.h" + "${AOM_ROOT}/aom_dsp/x86/fwd_txfm_sse2.c" + "${AOM_ROOT}/aom_dsp/x86/fwd_txfm_sse2.h" + "${AOM_ROOT}/aom_dsp/x86/halfpix_variance_sse2.c" + "${AOM_ROOT}/aom_dsp/x86/variance_sse2.c" + "${AOM_ROOT}/aom_dsp/x86/sum_squares_sse2.c") + + set(AOM_DSP_ENCODER_INTRIN_SSSE3 + ${AOM_DSP_ENCODER_INTRIN_SSSE3} + "${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_ssse3.c" + "${AOM_ROOT}/aom_dsp/x86/masked_variance_intrin_ssse3.c") + + set(AOM_DSP_ENCODER_ASM_SSSE3_X86_64 + ${AOM_DSP_ENCODER_ASM_SSSE3_X86_64} + "${AOM_ROOT}/aom_dsp/x86/avg_ssse3_x86_64.asm" + "${AOM_ROOT}/aom_dsp/x86/quantize_ssse3_x86_64.asm") + + set(AOM_DSP_ENCODER_AVX_ASM_X86_64 + ${AOM_DSP_ENCODER_AVX_ASM_X86_64} + "${AOM_ROOT}/aom_dsp/x86/quantize_avx_x86_64.asm") + + set(AOM_DSP_ENCODER_INTRIN_MSA + "${AOM_ROOT}/aom_dsp/mips/avg_msa.c" + "${AOM_ROOT}/aom_dsp/mips/sad_msa.c" + "${AOM_ROOT}/aom_dsp/mips/subtract_msa.c" + "${AOM_ROOT}/aom_dsp/mips/variance_msa.c" + "${AOM_ROOT}/aom_dsp/mips/sub_pixel_variance_msa.c") + + if (CONFIG_HIGHBITDEPTH) + set(AOM_DSP_ENCODER_INTRIN_SSE2 + ${AOM_DSP_ENCODER_INTRIN_SSE2} + "${AOM_ROOT}/aom_dsp/x86/highbd_quantize_intrin_sse2.c" + "${AOM_ROOT}/aom_dsp/x86/highbd_subtract_sse2.c") + endif () + endif () + + if (CONFIG_HIGHBITDEPTH) + set(AOM_DSP_ENCODER_ASM_SSE2 + ${AOM_DSP_ENCODER_ASM_SSE2} + "${AOM_ROOT}/aom_dsp/x86/highbd_sad4d_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/highbd_sad_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/highbd_subpel_variance_impl_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/highbd_variance_impl_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/aom_high_subpixel_8t_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm") + + set(AOM_DSP_ENCODER_INTRIN_SSE2 + ${AOM_DSP_ENCODER_INTRIN_SSE2} + "${AOM_ROOT}/aom_dsp/x86/highbd_variance_sse2.c") + + set(AOM_DSP_ENCODER_INTRIN_SSE4_1 + ${AOM_DSP_ENCODER_INTRIN_SSE4_1} + "${AOM_ROOT}/aom_dsp/x86/highbd_variance_sse4.c") + + set(AOM_DSP_ENCODER_INTRIN_AVX2 + ${AOM_DSP_ENCODER_INTRIN_AVX2} + "${AOM_ROOT}/aom_dsp/x86/sad_highbd_avx2.c") + endif () + + if (CONFIG_ANS) + set(AOM_DSP_ENCODER_SOURCES + ${AOM_DSP_ENCODER_SOURCES} + "${AOM_ROOT}/aom_dsp/answriter.h" + "${AOM_ROOT}/aom_dsp/buf_ans.c" + "${AOM_ROOT}/aom_dsp/buf_ans.h") + elseif (CONFIG_DAALA_EC) + set(AOM_DSP_ENCODER_SOURCES + ${AOM_DSP_ENCODER_SOURCES} + "${AOM_ROOT}/aom_dsp/daalaboolwriter.c" + "${AOM_ROOT}/aom_dsp/daalaboolwriter.h" + "${AOM_ROOT}/aom_dsp/entenc.c" + "${AOM_ROOT}/aom_dsp/entenc.h") + else () + set(AOM_DSP_ENCODER_SOURCES + ${AOM_DSP_ENCODER_SOURCES} + "${AOM_ROOT}/aom_dsp/dkboolwriter.c" + "${AOM_ROOT}/aom_dsp/dkboolwriter.h") + endif () + + if (CONFIG_INTERNAL_STATS) + set(AOM_DSP_ENCODER_SOURCES + ${AOM_DSP_ENCODER_SOURCES} + "${AOM_ROOT}/aom_dsp/fastssim.c" + "${AOM_ROOT}/aom_dsp/psnrhvs.c" + "${AOM_ROOT}/aom_dsp/ssim.c" + "${AOM_ROOT}/aom_dsp/ssim.h") + endif () +endif () + +if (CONFIG_MOTION_VAR) + set(AOM_DSP_ENCODER_INTRIN_SSE4_1 + ${AOM_DSP_ENCODER_INTRIN_SSE4_1} + "${AOM_ROOT}/aom_dsp/x86/obmc_sad_sse4.c" + "${AOM_ROOT}/aom_dsp/x86/obmc_variance_sse4.c") +endif () + +# Creates aom_dsp build targets. Must not be called until after libaom target +# has been created. +function (setup_aom_dsp_targets) + add_library(aom_dsp_common OBJECT ${AOM_DSP_COMMON_SOURCES}) + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_dsp_common) + target_sources(aom PUBLIC $) + + if (CONFIG_DECODERS) + add_library(aom_dsp_decoder OBJECT ${AOM_DSP_DECODER_SOURCES}) + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_dsp_decoder) + target_sources(aom PUBLIC $) + endif () + + if (CONFIG_ENCODERS) + add_library(aom_dsp_encoder OBJECT ${AOM_DSP_ENCODER_SOURCES}) + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_dsp_encoder) + target_sources(aom PUBLIC $) + endif () + + if (HAVE_SSE2) + add_asm_library("aom_dsp_common_sse2" "AOM_DSP_COMMON_ASM_SSE2" "aom") + add_intrinsics_object_library("-msse2" "sse2" "aom_dsp_common" + "AOM_DSP_COMMON_INTRIN_SSE2") + if (CONFIG_ENCODERS) + add_asm_library("aom_dsp_encoder_sse2" "AOM_DSP_ENCODER_ASM_SSE2" "aom") + add_intrinsics_object_library("-msse2" "sse2" "aom_dsp_encoder" + "AOM_DSP_ENCODER_INTRIN_SSE2") + endif() + endif () + + if (HAVE_SSE3 AND CONFIG_ENCODERS) + add_asm_library("aom_dsp_encoder_sse3" "AOM_DSP_ENCODER_INTRIN_SSE3" "aom") + endif () + + if (HAVE_SSSE3) + add_asm_library("aom_dsp_common_ssse3" "AOM_DSP_COMMON_ASM_SSSE3" "aom") + add_intrinsics_object_library("-mssse3" "ssse3" "aom_dsp_common" + "AOM_DSP_COMMON_INTRIN_SSSE3") + + if (CONFIG_ENCODERS) + if ("${AOM_TARGET_CPU}" STREQUAL "x86_64") + list(APPEND AOM_DSP_ENCODER_ASM_SSSE3 + ${AOM_DSP_ENCODER_ASM_SSSE3_X86_64}) + endif () + add_asm_library("aom_dsp_encoder_ssse3" "AOM_DSP_ENCODER_ASM_SSSE3" "aom") + add_intrinsics_object_library("-mssse3" "ssse3" "aom_dsp_encoder" + "AOM_DSP_ENCODER_INTRIN_SSSE3") + endif () + endif () + + if (HAVE_SSE4_1) + add_intrinsics_object_library("-msse4.1" "sse4_1" "aom_dsp_common" + "AOM_DSP_COMMON_INTRIN_SSE4_1") + if (CONFIG_ENCODERS) + if (AOM_DSP_ENCODER_INTRIN_SSE4_1) + add_intrinsics_object_library("-msse4.1" "sse4_1" "aom_dsp_encoder" + "AOM_DSP_ENCODER_INTRIN_SSE4_1") + endif () + add_asm_library("aom_dsp_encoder_sse4_1" "AOM_DSP_ENCODER_ASM_SSE4_1" + "aom") + endif () + endif () + + if (HAVE_AVX AND "${AOM_TARGET_CPU}" STREQUAL "x86_64") + add_asm_library("aom_dsp_encoder_avx" "AOM_DSP_ENCODER_AVX_ASM_X86_64" + "aom") + endif () + + if (HAVE_AVX2) + add_intrinsics_object_library("-mavx2" "avx2" "aom_dsp_common" + "AOM_DSP_COMMON_INTRIN_AVX2") + if (CONFIG_ENCODERS) + add_intrinsics_object_library("-mavx2" "avx2" "aom_dsp_encoder" + "AOM_DSP_ENCODER_INTRIN_AVX2") + endif () + endif () + + if (HAVE_NEON_ASM) + if (AOM_ADS2GAS_REQUIRED) + add_gas_asm_library("aom_dsp_common_neon" "AOM_DSP_COMMON_ASM_NEON" "aom") + else () + add_asm_library("aom_dsp_common_neon" "AOM_DSP_COMMON_ASM_NEON" "aom") + endif () + endif () + + if (HAVE_NEON) + add_intrinsics_object_library("${AOM_NEON_INTRIN_FLAG}" "neon" + "aom_dsp_common" "AOM_DSP_COMMON_INTRIN_NEON") + endif () + + if (HAVE_DSPR2) + add_intrinsics_object_library("" "dspr2" "aom_dsp_common" + "AOM_DSP_COMMON_INTRIN_DSPR2") + endif () + + if (HAVE_MSA) + add_intrinsics_object_library("" "msa" "aom_dsp_common" + "AOM_DSP_COMMON_INTRIN_MSA") + if (CONFIG_ENCODERS) + add_intrinsics_object_library("" "msa" "aom_dsp_encoder" + "AOM_DSP_ENCODER_INTRIN_MSA") + endif () + endif () + + # Pass the new lib targets up to the parent scope instance of + # $AOM_LIB_TARGETS. + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} PARENT_SCOPE) +endfunction () diff --git a/third_party/aom/aom_dsp/aom_dsp.mk b/third_party/aom/aom_dsp/aom_dsp.mk new file mode 100644 index 0000000000..8c7241b831 --- /dev/null +++ b/third_party/aom/aom_dsp/aom_dsp.mk @@ -0,0 +1,428 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + +DSP_SRCS-yes += aom_dsp.mk +DSP_SRCS-yes += aom_dsp_common.h + +DSP_SRCS-$(HAVE_MSA) += mips/macros_msa.h + +DSP_SRCS-$(ARCH_X86)$(ARCH_X86_64) += x86/synonyms.h + +# bit reader +DSP_SRCS-yes += prob.h +DSP_SRCS-yes += prob.c +DSP_SRCS-$(CONFIG_ANS) += ans.h + +ifeq ($(CONFIG_ENCODERS),yes) +ifeq ($(CONFIG_ANS),yes) +DSP_SRCS-yes += answriter.h +DSP_SRCS-yes += buf_ans.h +DSP_SRCS-yes += buf_ans.c +else ifeq ($(CONFIG_DAALA_EC),yes) +DSP_SRCS-yes += entenc.c +DSP_SRCS-yes += entenc.h +DSP_SRCS-yes += daalaboolwriter.c +DSP_SRCS-yes += daalaboolwriter.h +else +DSP_SRCS-yes += dkboolwriter.h +DSP_SRCS-yes += dkboolwriter.c +endif +DSP_SRCS-yes += bitwriter.h +DSP_SRCS-yes += bitwriter_buffer.c +DSP_SRCS-yes += bitwriter_buffer.h +DSP_SRCS-yes += binary_codes_writer.c +DSP_SRCS-yes += binary_codes_writer.h +DSP_SRCS-yes += psnr.c +DSP_SRCS-yes += psnr.h +DSP_SRCS-$(CONFIG_INTERNAL_STATS) += ssim.c +DSP_SRCS-$(CONFIG_INTERNAL_STATS) += ssim.h +DSP_SRCS-$(CONFIG_INTERNAL_STATS) += psnrhvs.c +DSP_SRCS-$(CONFIG_INTERNAL_STATS) += fastssim.c +endif + +ifeq ($(CONFIG_DECODERS),yes) +ifeq ($(CONFIG_ANS),yes) +DSP_SRCS-yes += ansreader.h +else ifeq ($(CONFIG_DAALA_EC),yes) +DSP_SRCS-yes += entdec.c +DSP_SRCS-yes += entdec.h +DSP_SRCS-yes += daalaboolreader.c +DSP_SRCS-yes += daalaboolreader.h +else +DSP_SRCS-yes += dkboolreader.h +DSP_SRCS-yes += dkboolreader.c +endif +DSP_SRCS-yes += bitreader.h +DSP_SRCS-yes += bitreader_buffer.c +DSP_SRCS-yes += bitreader_buffer.h +DSP_SRCS-yes += binary_codes_reader.c +DSP_SRCS-yes += binary_codes_reader.h +endif + +# intra predictions +DSP_SRCS-yes += intrapred.c + +ifeq ($(CONFIG_DAALA_EC),yes) +DSP_SRCS-yes += entcode.c +DSP_SRCS-yes += entcode.h +endif + +DSP_SRCS-$(HAVE_SSE) += x86/intrapred_sse2.asm +DSP_SRCS-$(HAVE_SSE2) += x86/intrapred_sse2.asm +DSP_SRCS-$(HAVE_SSSE3) += x86/intrapred_ssse3.asm +DSP_SRCS-$(HAVE_SSSE3) += x86/aom_subpixel_8t_ssse3.asm + +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +DSP_SRCS-$(HAVE_SSE) += x86/highbd_intrapred_sse2.asm +DSP_SRCS-$(HAVE_SSE2) += x86/highbd_intrapred_sse2.asm +endif # CONFIG_HIGHBITDEPTH + +DSP_SRCS-$(HAVE_NEON_ASM) += arm/intrapred_neon_asm$(ASM) +DSP_SRCS-$(HAVE_NEON) += arm/intrapred_neon.c +DSP_SRCS-$(HAVE_MSA) += mips/intrapred_msa.c +DSP_SRCS-$(HAVE_DSPR2) += mips/intrapred4_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/intrapred8_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/intrapred16_dspr2.c + +DSP_SRCS-$(HAVE_DSPR2) += mips/common_dspr2.h +DSP_SRCS-$(HAVE_DSPR2) += mips/common_dspr2.c + +# inter predictions +DSP_SRCS-yes += blend.h +DSP_SRCS-yes += blend_a64_mask.c +DSP_SRCS-yes += blend_a64_hmask.c +DSP_SRCS-yes += blend_a64_vmask.c +DSP_SRCS-$(HAVE_SSE4_1) += x86/blend_sse4.h +DSP_SRCS-$(HAVE_SSE4_1) += x86/blend_a64_mask_sse4.c +DSP_SRCS-$(HAVE_SSE4_1) += x86/blend_a64_hmask_sse4.c +DSP_SRCS-$(HAVE_SSE4_1) += x86/blend_a64_vmask_sse4.c + +# interpolation filters +DSP_SRCS-yes += aom_convolve.c +DSP_SRCS-yes += aom_convolve.h +DSP_SRCS-yes += aom_filter.h + +DSP_SRCS-$(ARCH_X86)$(ARCH_X86_64) += x86/convolve.h +DSP_SRCS-$(ARCH_X86)$(ARCH_X86_64) += x86/aom_asm_stubs.c +DSP_SRCS-$(HAVE_SSE2) += x86/aom_subpixel_8t_sse2.asm +DSP_SRCS-$(HAVE_SSE2) += x86/aom_subpixel_bilinear_sse2.asm +DSP_SRCS-$(HAVE_SSSE3) += x86/aom_subpixel_8t_ssse3.asm +DSP_SRCS-$(HAVE_SSSE3) += x86/aom_subpixel_bilinear_ssse3.asm +DSP_SRCS-$(HAVE_AVX2) += x86/aom_subpixel_8t_intrin_avx2.c +DSP_SRCS-$(HAVE_SSSE3) += x86/aom_subpixel_8t_intrin_ssse3.c +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +DSP_SRCS-$(HAVE_SSE2) += x86/aom_high_subpixel_8t_sse2.asm +DSP_SRCS-$(HAVE_SSE2) += x86/aom_high_subpixel_bilinear_sse2.asm +DSP_SRCS-$(HAVE_AVX2) += x86/highbd_convolve_avx2.c +endif +DSP_SRCS-$(HAVE_SSE2) += x86/aom_convolve_copy_sse2.asm + +ifeq ($(HAVE_NEON_ASM),yes) +DSP_SRCS-yes += arm/aom_convolve_copy_neon_asm$(ASM) +DSP_SRCS-yes += arm/aom_convolve8_avg_neon_asm$(ASM) +DSP_SRCS-yes += arm/aom_convolve8_neon_asm$(ASM) +DSP_SRCS-yes += arm/aom_convolve_avg_neon_asm$(ASM) +DSP_SRCS-yes += arm/aom_convolve_neon.c +else +ifeq ($(HAVE_NEON),yes) +DSP_SRCS-yes += arm/aom_convolve_copy_neon.c +DSP_SRCS-yes += arm/aom_convolve8_avg_neon.c +DSP_SRCS-yes += arm/aom_convolve8_neon.c +DSP_SRCS-yes += arm/aom_convolve_avg_neon.c +DSP_SRCS-yes += arm/aom_convolve_neon.c +endif # HAVE_NEON +endif # HAVE_NEON_ASM + +# common (msa) +DSP_SRCS-$(HAVE_MSA) += mips/aom_convolve8_avg_horiz_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/aom_convolve8_avg_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/aom_convolve8_avg_vert_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/aom_convolve8_horiz_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/aom_convolve8_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/aom_convolve8_vert_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/aom_convolve_avg_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/aom_convolve_copy_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/aom_convolve_msa.h + +# common (dspr2) +DSP_SRCS-$(HAVE_DSPR2) += mips/convolve_common_dspr2.h +DSP_SRCS-$(HAVE_DSPR2) += mips/convolve2_avg_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/convolve2_avg_horiz_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/convolve2_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/convolve2_horiz_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/convolve2_vert_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/convolve8_avg_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/convolve8_avg_horiz_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/convolve8_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/convolve8_horiz_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/convolve8_vert_dspr2.c + +# loop filters +DSP_SRCS-yes += loopfilter.c + +DSP_SRCS-$(ARCH_X86)$(ARCH_X86_64) += x86/loopfilter_sse2.c +DSP_SRCS-$(HAVE_AVX2) += x86/loopfilter_avx2.c + +DSP_SRCS-$(HAVE_NEON) += arm/loopfilter_neon.c +ifeq ($(HAVE_NEON_ASM),yes) +DSP_SRCS-yes += arm/loopfilter_mb_neon$(ASM) +DSP_SRCS-yes += arm/loopfilter_16_neon$(ASM) +DSP_SRCS-yes += arm/loopfilter_8_neon$(ASM) +DSP_SRCS-yes += arm/loopfilter_4_neon$(ASM) +else +ifeq ($(HAVE_NEON),yes) +DSP_SRCS-yes += arm/loopfilter_16_neon.c +DSP_SRCS-yes += arm/loopfilter_8_neon.c +DSP_SRCS-yes += arm/loopfilter_4_neon.c +endif # HAVE_NEON +endif # HAVE_NEON_ASM + +DSP_SRCS-$(HAVE_MSA) += mips/loopfilter_msa.h +DSP_SRCS-$(HAVE_MSA) += mips/loopfilter_16_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/loopfilter_8_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/loopfilter_4_msa.c +DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_filters_dspr2.h +DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_filters_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_macros_dspr2.h +DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_masks_dspr2.h +DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_horiz_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_vert_dspr2.c + +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +DSP_SRCS-$(HAVE_SSE2) += x86/highbd_loopfilter_sse2.c +endif # CONFIG_HIGHBITDEPTH + +DSP_SRCS-yes += txfm_common.h +DSP_SRCS-yes += x86/txfm_common_intrin.h +DSP_SRCS-$(HAVE_SSE2) += x86/txfm_common_sse2.h +DSP_SRCS-$(HAVE_MSA) += mips/txfm_macros_msa.h + +# forward transform +ifneq ($(findstring yes,$(CONFIG_AV1)$(CONFIG_PVQ)),) +DSP_SRCS-$(HAVE_AVX2) += x86/txfm_common_avx2.h +ifeq ($(CONFIG_AV1_ENCODER),yes) +DSP_SRCS-yes += fwd_txfm.c +DSP_SRCS-yes += fwd_txfm.h +DSP_SRCS-$(HAVE_SSE2) += x86/fwd_txfm_sse2.h +DSP_SRCS-$(HAVE_SSE2) += x86/fwd_txfm_sse2.c +DSP_SRCS-$(HAVE_SSE2) += x86/fwd_dct32_8cols_sse2.c +DSP_SRCS-$(HAVE_SSE2) += x86/fwd_txfm_impl_sse2.h +DSP_SRCS-$(HAVE_SSE2) += x86/fwd_dct32x32_impl_sse2.h +ifeq ($(ARCH_X86_64),yes) +DSP_SRCS-$(HAVE_SSSE3) += x86/fwd_txfm_ssse3_x86_64.asm +endif +DSP_SRCS-$(HAVE_AVX2) += x86/fwd_txfm_avx2.h +DSP_SRCS-$(HAVE_AVX2) += x86/fwd_txfm_avx2.c +DSP_SRCS-$(HAVE_AVX2) += x86/fwd_dct32x32_impl_avx2.h +DSP_SRCS-$(HAVE_NEON) += arm/fwd_txfm_neon.c +DSP_SRCS-$(HAVE_MSA) += mips/fwd_txfm_msa.h +DSP_SRCS-$(HAVE_MSA) += mips/fwd_txfm_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/fwd_dct32x32_msa.c +endif # CONFIG_AV1_ENCODER +endif # CONFIG_AV1 + +# inverse transform +ifeq ($(CONFIG_AV1), yes) +DSP_SRCS-yes += inv_txfm.h +DSP_SRCS-yes += inv_txfm.c +DSP_SRCS-$(HAVE_SSE2) += x86/inv_txfm_sse2.h +DSP_SRCS-$(HAVE_SSE2) += x86/inv_txfm_sse2.c +DSP_SRCS-$(HAVE_SSE2) += x86/inv_wht_sse2.asm +DSP_SRCS-$(HAVE_SSSE3) += x86/inv_txfm_ssse3.c + +ifeq ($(HAVE_NEON_ASM),yes) +DSP_SRCS-yes += arm/save_reg_neon$(ASM) +DSP_SRCS-yes += arm/idct4x4_1_add_neon$(ASM) +DSP_SRCS-yes += arm/idct4x4_add_neon$(ASM) +DSP_SRCS-yes += arm/idct8x8_1_add_neon$(ASM) +DSP_SRCS-yes += arm/idct8x8_add_neon$(ASM) +DSP_SRCS-yes += arm/idct16x16_1_add_neon$(ASM) +DSP_SRCS-yes += arm/idct16x16_add_neon$(ASM) +DSP_SRCS-yes += arm/idct32x32_1_add_neon$(ASM) +DSP_SRCS-yes += arm/idct32x32_add_neon$(ASM) +else +ifeq ($(HAVE_NEON),yes) +DSP_SRCS-yes += arm/idct4x4_1_add_neon.c +DSP_SRCS-yes += arm/idct4x4_add_neon.c +DSP_SRCS-yes += arm/idct8x8_1_add_neon.c +DSP_SRCS-yes += arm/idct8x8_add_neon.c +DSP_SRCS-yes += arm/idct16x16_1_add_neon.c +DSP_SRCS-yes += arm/idct16x16_add_neon.c +DSP_SRCS-yes += arm/idct32x32_1_add_neon.c +DSP_SRCS-yes += arm/idct32x32_add_neon.c +endif # HAVE_NEON +endif # HAVE_NEON_ASM +DSP_SRCS-$(HAVE_NEON) += arm/idct16x16_neon.c + +DSP_SRCS-$(HAVE_MSA) += mips/inv_txfm_msa.h +DSP_SRCS-$(HAVE_MSA) += mips/idct4x4_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/idct8x8_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/idct16x16_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/idct32x32_msa.c + +ifneq ($(CONFIG_HIGHBITDEPTH),yes) +DSP_SRCS-$(HAVE_DSPR2) += mips/inv_txfm_dspr2.h +DSP_SRCS-$(HAVE_DSPR2) += mips/itrans4_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/itrans8_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/itrans16_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/itrans32_dspr2.c +DSP_SRCS-$(HAVE_DSPR2) += mips/itrans32_cols_dspr2.c +endif # CONFIG_HIGHBITDEPTH +endif # CONFIG_AV1 + +# quantization +ifneq ($(filter yes,$(CONFIG_AV1_ENCODER)),) +DSP_SRCS-yes += quantize.c +DSP_SRCS-yes += quantize.h + +DSP_SRCS-$(HAVE_SSE2) += x86/quantize_sse2.c +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +DSP_SRCS-$(HAVE_SSE2) += x86/highbd_quantize_intrin_sse2.c +endif +ifeq ($(ARCH_X86_64),yes) +DSP_SRCS-$(HAVE_SSSE3) += x86/quantize_ssse3_x86_64.asm +DSP_SRCS-$(HAVE_AVX) += x86/quantize_avx_x86_64.asm +endif + +# avg +DSP_SRCS-yes += avg.c +DSP_SRCS-$(HAVE_SSE2) += x86/avg_intrin_sse2.c +DSP_SRCS-$(HAVE_NEON) += arm/avg_neon.c +DSP_SRCS-$(HAVE_MSA) += mips/avg_msa.c +DSP_SRCS-$(HAVE_NEON) += arm/hadamard_neon.c +ifeq ($(ARCH_X86_64),yes) +DSP_SRCS-$(HAVE_SSSE3) += x86/avg_ssse3_x86_64.asm +endif + +# high bit depth subtract +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +DSP_SRCS-$(HAVE_SSE2) += x86/highbd_subtract_sse2.c +endif + +endif # CONFIG_AV1_ENCODER + +ifeq ($(CONFIG_AV1_ENCODER),yes) +DSP_SRCS-yes += sum_squares.c + +DSP_SRCS-$(HAVE_SSE2) += x86/sum_squares_sse2.c +endif # CONFIG_AV1_ENCODER + +ifeq ($(CONFIG_ENCODERS),yes) +DSP_SRCS-yes += sad.c +DSP_SRCS-yes += subtract.c + +DSP_SRCS-$(HAVE_MEDIA) += arm/sad_media$(ASM) +DSP_SRCS-$(HAVE_NEON) += arm/sad4d_neon.c +DSP_SRCS-$(HAVE_NEON) += arm/sad_neon.c +DSP_SRCS-$(HAVE_NEON) += arm/subtract_neon.c + +DSP_SRCS-$(HAVE_MSA) += mips/sad_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/subtract_msa.c + +DSP_SRCS-$(HAVE_SSE3) += x86/sad_sse3.asm +DSP_SRCS-$(HAVE_SSSE3) += x86/sad_ssse3.asm +DSP_SRCS-$(HAVE_SSE4_1) += x86/sad_sse4.asm +DSP_SRCS-$(HAVE_AVX2) += x86/sad4d_avx2.c +DSP_SRCS-$(HAVE_AVX2) += x86/sad_avx2.c + +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +DSP_SRCS-$(HAVE_AVX2) += x86/sad_highbd_avx2.c +endif + +ifeq ($(CONFIG_AV1_ENCODER),yes) +ifeq ($(CONFIG_EXT_INTER),yes) +DSP_SRCS-$(HAVE_SSSE3) += x86/masked_sad_intrin_ssse3.c +DSP_SRCS-$(HAVE_SSSE3) += x86/masked_variance_intrin_ssse3.c +endif #CONFIG_EXT_INTER +ifeq ($(CONFIG_MOTION_VAR),yes) +DSP_SRCS-$(HAVE_SSE4_1) += x86/obmc_sad_sse4.c +DSP_SRCS-$(HAVE_SSE4_1) += x86/obmc_variance_sse4.c +endif #CONFIG_MOTION_VAR +ifeq ($(CONFIG_EXT_PARTITION),yes) +DSP_SRCS-$(HAVE_AVX2) += x86/sad_impl_avx2.c +endif +endif #CONFIG_AV1_ENCODER + +DSP_SRCS-$(HAVE_SSE) += x86/sad4d_sse2.asm +DSP_SRCS-$(HAVE_SSE) += x86/sad_sse2.asm +DSP_SRCS-$(HAVE_SSE2) += x86/sad4d_sse2.asm +DSP_SRCS-$(HAVE_SSE2) += x86/sad_sse2.asm +DSP_SRCS-$(HAVE_SSE2) += x86/subtract_sse2.asm + +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +DSP_SRCS-$(HAVE_SSE2) += x86/highbd_sad4d_sse2.asm +DSP_SRCS-$(HAVE_SSE2) += x86/highbd_sad_sse2.asm +endif # CONFIG_HIGHBITDEPTH + +endif # CONFIG_ENCODERS + +ifneq ($(filter yes,$(CONFIG_ENCODERS)),) +DSP_SRCS-yes += variance.c +DSP_SRCS-yes += variance.h + +DSP_SRCS-$(HAVE_MEDIA) += arm/bilinear_filter_media$(ASM) +DSP_SRCS-$(HAVE_MEDIA) += arm/subpel_variance_media.c +DSP_SRCS-$(HAVE_MEDIA) += arm/variance_halfpixvar16x16_h_media$(ASM) +DSP_SRCS-$(HAVE_MEDIA) += arm/variance_halfpixvar16x16_hv_media$(ASM) +DSP_SRCS-$(HAVE_MEDIA) += arm/variance_halfpixvar16x16_v_media$(ASM) +DSP_SRCS-$(HAVE_MEDIA) += arm/variance_media$(ASM) +DSP_SRCS-$(HAVE_NEON) += arm/subpel_variance_neon.c +DSP_SRCS-$(HAVE_NEON) += arm/variance_neon.c + +DSP_SRCS-$(HAVE_MSA) += mips/variance_msa.c +DSP_SRCS-$(HAVE_MSA) += mips/sub_pixel_variance_msa.c + +DSP_SRCS-$(HAVE_SSE) += x86/variance_sse2.c +DSP_SRCS-$(HAVE_SSE2) += x86/variance_sse2.c # Contains SSE2 and SSSE3 +DSP_SRCS-$(HAVE_SSE2) += x86/halfpix_variance_sse2.c +DSP_SRCS-$(HAVE_SSE2) += x86/halfpix_variance_impl_sse2.asm +DSP_SRCS-$(HAVE_AVX2) += x86/variance_avx2.c +DSP_SRCS-$(HAVE_AVX2) += x86/variance_impl_avx2.c + +ifeq ($(ARCH_X86_64),yes) +DSP_SRCS-$(HAVE_SSE2) += x86/ssim_opt_x86_64.asm +endif # ARCH_X86_64 + +DSP_SRCS-$(HAVE_SSE) += x86/subpel_variance_sse2.asm +DSP_SRCS-$(HAVE_SSE2) += x86/subpel_variance_sse2.asm # Contains SSE2 and SSSE3 + +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +DSP_SRCS-$(HAVE_SSE2) += x86/highbd_variance_sse2.c +DSP_SRCS-$(HAVE_SSE4_1) += x86/highbd_variance_sse4.c +DSP_SRCS-$(HAVE_SSE2) += x86/highbd_variance_impl_sse2.asm +DSP_SRCS-$(HAVE_SSE2) += x86/highbd_subpel_variance_impl_sse2.asm +endif # CONFIG_HIGHBITDEPTH +endif # CONFIG_ENCODERS + +DSP_SRCS-no += $(DSP_SRCS_REMOVE-yes) + +DSP_SRCS-yes += aom_dsp_rtcd.c +DSP_SRCS-yes += aom_dsp_rtcd_defs.pl + +DSP_SRCS-yes += aom_simd.h +DSP_SRCS-yes += aom_simd_inline.h +DSP_SRCS-yes += simd/v64_intrinsics.h +DSP_SRCS-yes += simd/v64_intrinsics_c.h +DSP_SRCS-yes += simd/v128_intrinsics.h +DSP_SRCS-yes += simd/v128_intrinsics_c.h +DSP_SRCS-yes += simd/v256_intrinsics.h +DSP_SRCS-yes += simd/v256_intrinsics_c.h +DSP_SRCS-yes += simd/v256_intrinsics_v128.h +DSP_SRCS-$(HAVE_SSE2) += simd/v64_intrinsics_x86.h +DSP_SRCS-$(HAVE_SSE2) += simd/v128_intrinsics_x86.h +DSP_SRCS-$(HAVE_SSE2) += simd/v256_intrinsics_x86.h +DSP_SRCS-$(HAVE_NEON) += simd/v64_intrinsics_arm.h +DSP_SRCS-$(HAVE_NEON) += simd/v128_intrinsics_arm.h +DSP_SRCS-$(HAVE_NEON) += simd/v256_intrinsics_arm.h + +$(eval $(call rtcd_h_template,aom_dsp_rtcd,aom_dsp/aom_dsp_rtcd_defs.pl)) diff --git a/third_party/aom/aom_dsp/aom_dsp_common.h b/third_party/aom/aom_dsp/aom_dsp_common.h new file mode 100644 index 0000000000..47ffbeb6cc --- /dev/null +++ b/third_party/aom/aom_dsp/aom_dsp_common.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_AOM_DSP_COMMON_H_ +#define AOM_DSP_AOM_DSP_COMMON_H_ + +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef MAX_SB_SIZE +#if CONFIG_AV1 && CONFIG_EXT_PARTITION +#define MAX_SB_SIZE 128 +#else +#define MAX_SB_SIZE 64 +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION +#endif // ndef MAX_SB_SIZE + +#define AOMMIN(x, y) (((x) < (y)) ? (x) : (y)) +#define AOMMAX(x, y) (((x) > (y)) ? (x) : (y)) + +#define IMPLIES(a, b) (!(a) || (b)) // Logical 'a implies b' (or 'a -> b') + +#define IS_POWER_OF_TWO(x) (((x) & ((x)-1)) == 0) + +/* Left shifting a negative value became undefined behavior in C99 (downgraded + from merely implementation-defined in C89). This should still compile to the + correct thing on any two's-complement machine, but avoid ubsan warnings.*/ +#define AOM_SIGNED_SHL(x, shift) ((x) * (((x)*0 + 1) << (shift))) + +// These can be used to give a hint about branch outcomes. +// This can have an effect, even if your target processor has a +// good branch predictor, as these hints can affect basic block +// ordering by the compiler. +#ifdef __GNUC__ +#define LIKELY(v) __builtin_expect(v, 1) +#define UNLIKELY(v) __builtin_expect(v, 0) +#else +#define LIKELY(v) (v) +#define UNLIKELY(v) (v) +#endif + +#define AOM_SWAP(type, a, b) \ + do { \ + type c = (b); \ + b = a; \ + a = c; \ + } while (0) + +#if CONFIG_AOM_QM +typedef uint16_t qm_val_t; +#define AOM_QM_BITS 6 +#endif +#if CONFIG_HIGHBITDEPTH +// Note: +// tran_low_t is the datatype used for final transform coefficients. +// tran_high_t is the datatype used for intermediate transform stages. +typedef int64_t tran_high_t; +typedef int32_t tran_low_t; +#else +// Note: +// tran_low_t is the datatype used for final transform coefficients. +// tran_high_t is the datatype used for intermediate transform stages. +typedef int32_t tran_high_t; +typedef int16_t tran_low_t; +#endif // CONFIG_HIGHBITDEPTH + +static INLINE uint8_t clip_pixel(int val) { + return (val > 255) ? 255 : (val < 0) ? 0 : val; +} + +static INLINE int clamp(int value, int low, int high) { + return value < low ? low : (value > high ? high : value); +} + +static INLINE double fclamp(double value, double low, double high) { + return value < low ? low : (value > high ? high : value); +} + +#if CONFIG_HIGHBITDEPTH +static INLINE uint16_t clip_pixel_highbd(int val, int bd) { + switch (bd) { + case 8: + default: return (uint16_t)clamp(val, 0, 255); + case 10: return (uint16_t)clamp(val, 0, 1023); + case 12: return (uint16_t)clamp(val, 0, 4095); + } +} +#endif // CONFIG_HIGHBITDEPTH + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_AOM_DSP_COMMON_H_ diff --git a/third_party/aom/aom_dsp/aom_dsp_rtcd.c b/third_party/aom/aom_dsp/aom_dsp_rtcd.c new file mode 100644 index 0000000000..11a57d3822 --- /dev/null +++ b/third_party/aom/aom_dsp/aom_dsp_rtcd.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include "./aom_config.h" +#define RTCD_C +#include "./aom_dsp_rtcd.h" +#include "aom_ports/aom_once.h" + +void aom_dsp_rtcd() { once(setup_rtcd_internal); } diff --git a/third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl b/third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl new file mode 100755 index 0000000000..b4ef0d92f3 --- /dev/null +++ b/third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl @@ -0,0 +1,1495 @@ +sub aom_dsp_forward_decls() { +print < + +#if defined(_WIN32) +#include +#endif + +#include "./aom_config.h" +#include "./aom_simd_inline.h" + +#define SIMD_CHECK 1 // Sanity checks in C equivalents + +#if HAVE_NEON +#include "simd/v256_intrinsics_arm.h" +// VS compiling for 32 bit targets does not support vector types in +// structs as arguments, which makes the v256 type of the intrinsics +// hard to support, so optimizations for this target are disabled. +#elif HAVE_SSE2 && (defined(_WIN64) || !defined(_MSC_VER) || defined(__clang__)) +#include "simd/v256_intrinsics_x86.h" +#else +#include "simd/v256_intrinsics.h" +#endif + +#endif // AOM_DSP_AOM_AOM_SIMD_H_ diff --git a/third_party/aom/aom_dsp/aom_simd_inline.h b/third_party/aom/aom_dsp/aom_simd_inline.h new file mode 100644 index 0000000000..02a8b3a170 --- /dev/null +++ b/third_party/aom/aom_dsp/aom_simd_inline.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_AOM_SIMD_INLINE_H_ +#define AOM_DSP_AOM_SIMD_INLINE_H_ + +#include "aom/aom_integer.h" + +#ifndef SIMD_INLINE +#define SIMD_INLINE static AOM_FORCE_INLINE +#endif + +#endif // AOM_DSP_AOM_SIMD_INLINE_H_ diff --git a/third_party/aom/aom_dsp/arm/aom_convolve8_avg_neon.c b/third_party/aom/aom_dsp/arm/aom_convolve8_avg_neon.c new file mode 100644 index 0000000000..09429d6d26 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/aom_convolve8_avg_neon.c @@ -0,0 +1,364 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" + +static INLINE int32x4_t MULTIPLY_BY_Q0(int16x4_t dsrc0, int16x4_t dsrc1, + int16x4_t dsrc2, int16x4_t dsrc3, + int16x4_t dsrc4, int16x4_t dsrc5, + int16x4_t dsrc6, int16x4_t dsrc7, + int16x8_t q0s16) { + int32x4_t qdst; + int16x4_t d0s16, d1s16; + + d0s16 = vget_low_s16(q0s16); + d1s16 = vget_high_s16(q0s16); + + qdst = vmull_lane_s16(dsrc0, d0s16, 0); + qdst = vmlal_lane_s16(qdst, dsrc1, d0s16, 1); + qdst = vmlal_lane_s16(qdst, dsrc2, d0s16, 2); + qdst = vmlal_lane_s16(qdst, dsrc3, d0s16, 3); + qdst = vmlal_lane_s16(qdst, dsrc4, d1s16, 0); + qdst = vmlal_lane_s16(qdst, dsrc5, d1s16, 1); + qdst = vmlal_lane_s16(qdst, dsrc6, d1s16, 2); + qdst = vmlal_lane_s16(qdst, dsrc7, d1s16, 3); + return qdst; +} + +void aom_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, // unused + int y_step_q4, // unused + int w, int h) { + int width; + const uint8_t *s; + uint8_t *d; + uint8x8_t d2u8, d3u8, d24u8, d25u8, d26u8, d27u8, d28u8, d29u8; + uint32x2_t d2u32, d3u32, d6u32, d7u32, d28u32, d29u32, d30u32, d31u32; + uint8x16_t q1u8, q3u8, q12u8, q13u8, q14u8, q15u8; + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d22s16, d23s16; + int16x4_t d24s16, d25s16, d26s16, d27s16; + uint16x4_t d2u16, d3u16, d4u16, d5u16, d16u16, d17u16, d18u16, d19u16; + int16x8_t q0s16; + uint16x8_t q1u16, q2u16, q8u16, q9u16, q10u16, q11u16, q12u16, q13u16; + int32x4_t q1s32, q2s32, q14s32, q15s32; + uint16x8x2_t q0x2u16; + uint8x8x2_t d0x2u8, d1x2u8; + uint32x2x2_t d0x2u32; + uint16x4x2_t d0x2u16, d1x2u16; + uint32x4x2_t q0x2u32; + + assert(x_step_q4 == 16); + + (void)x_step_q4; + (void)y_step_q4; + (void)filter_y; + + q0s16 = vld1q_s16(filter_x); + + src -= 3; // adjust for taps + for (; h > 0; h -= 4) { // loop_horiz_v + s = src; + d24u8 = vld1_u8(s); + s += src_stride; + d25u8 = vld1_u8(s); + s += src_stride; + d26u8 = vld1_u8(s); + s += src_stride; + d27u8 = vld1_u8(s); + + q12u8 = vcombine_u8(d24u8, d25u8); + q13u8 = vcombine_u8(d26u8, d27u8); + + q0x2u16 = + vtrnq_u16(vreinterpretq_u16_u8(q12u8), vreinterpretq_u16_u8(q13u8)); + d24u8 = vreinterpret_u8_u16(vget_low_u16(q0x2u16.val[0])); + d25u8 = vreinterpret_u8_u16(vget_high_u16(q0x2u16.val[0])); + d26u8 = vreinterpret_u8_u16(vget_low_u16(q0x2u16.val[1])); + d27u8 = vreinterpret_u8_u16(vget_high_u16(q0x2u16.val[1])); + d0x2u8 = vtrn_u8(d24u8, d25u8); + d1x2u8 = vtrn_u8(d26u8, d27u8); + + __builtin_prefetch(src + src_stride * 4); + __builtin_prefetch(src + src_stride * 5); + + q8u16 = vmovl_u8(d0x2u8.val[0]); + q9u16 = vmovl_u8(d0x2u8.val[1]); + q10u16 = vmovl_u8(d1x2u8.val[0]); + q11u16 = vmovl_u8(d1x2u8.val[1]); + + src += 7; + d16u16 = vget_low_u16(q8u16); + d17u16 = vget_high_u16(q8u16); + d18u16 = vget_low_u16(q9u16); + d19u16 = vget_high_u16(q9u16); + q8u16 = vcombine_u16(d16u16, d18u16); // vswp 17 18 + q9u16 = vcombine_u16(d17u16, d19u16); + + d20s16 = vreinterpret_s16_u16(vget_low_u16(q10u16)); + d23s16 = vreinterpret_s16_u16(vget_high_u16(q10u16)); // vmov 23 21 + for (width = w; width > 0; width -= 4, src += 4, dst += 4) { // loop_horiz + s = src; + d28u32 = vld1_dup_u32((const uint32_t *)s); + s += src_stride; + d29u32 = vld1_dup_u32((const uint32_t *)s); + s += src_stride; + d31u32 = vld1_dup_u32((const uint32_t *)s); + s += src_stride; + d30u32 = vld1_dup_u32((const uint32_t *)s); + + __builtin_prefetch(src + 64); + + d0x2u16 = + vtrn_u16(vreinterpret_u16_u32(d28u32), vreinterpret_u16_u32(d31u32)); + d1x2u16 = + vtrn_u16(vreinterpret_u16_u32(d29u32), vreinterpret_u16_u32(d30u32)); + d0x2u8 = vtrn_u8(vreinterpret_u8_u16(d0x2u16.val[0]), // d28 + vreinterpret_u8_u16(d1x2u16.val[0])); // d29 + d1x2u8 = vtrn_u8(vreinterpret_u8_u16(d0x2u16.val[1]), // d31 + vreinterpret_u8_u16(d1x2u16.val[1])); // d30 + + __builtin_prefetch(src + 64 + src_stride); + + q14u8 = vcombine_u8(d0x2u8.val[0], d0x2u8.val[1]); + q15u8 = vcombine_u8(d1x2u8.val[1], d1x2u8.val[0]); + q0x2u32 = + vtrnq_u32(vreinterpretq_u32_u8(q14u8), vreinterpretq_u32_u8(q15u8)); + + d28u8 = vreinterpret_u8_u32(vget_low_u32(q0x2u32.val[0])); + d29u8 = vreinterpret_u8_u32(vget_high_u32(q0x2u32.val[0])); + q12u16 = vmovl_u8(d28u8); + q13u16 = vmovl_u8(d29u8); + + __builtin_prefetch(src + 64 + src_stride * 2); + + d = dst; + d6u32 = vld1_lane_u32((const uint32_t *)d, d6u32, 0); + d += dst_stride; + d7u32 = vld1_lane_u32((const uint32_t *)d, d7u32, 0); + d += dst_stride; + d6u32 = vld1_lane_u32((const uint32_t *)d, d6u32, 1); + d += dst_stride; + d7u32 = vld1_lane_u32((const uint32_t *)d, d7u32, 1); + + d16s16 = vreinterpret_s16_u16(vget_low_u16(q8u16)); + d17s16 = vreinterpret_s16_u16(vget_high_u16(q8u16)); + d18s16 = vreinterpret_s16_u16(vget_low_u16(q9u16)); + d19s16 = vreinterpret_s16_u16(vget_high_u16(q9u16)); + d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16)); + d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16)); + d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16)); + d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16)); + d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16)); + + q1s32 = MULTIPLY_BY_Q0(d16s16, d17s16, d20s16, d22s16, d18s16, d19s16, + d23s16, d24s16, q0s16); + q2s32 = MULTIPLY_BY_Q0(d17s16, d20s16, d22s16, d18s16, d19s16, d23s16, + d24s16, d26s16, q0s16); + q14s32 = MULTIPLY_BY_Q0(d20s16, d22s16, d18s16, d19s16, d23s16, d24s16, + d26s16, d27s16, q0s16); + q15s32 = MULTIPLY_BY_Q0(d22s16, d18s16, d19s16, d23s16, d24s16, d26s16, + d27s16, d25s16, q0s16); + + __builtin_prefetch(src + 64 + src_stride * 3); + + d2u16 = vqrshrun_n_s32(q1s32, 7); + d3u16 = vqrshrun_n_s32(q2s32, 7); + d4u16 = vqrshrun_n_s32(q14s32, 7); + d5u16 = vqrshrun_n_s32(q15s32, 7); + + q1u16 = vcombine_u16(d2u16, d3u16); + q2u16 = vcombine_u16(d4u16, d5u16); + + d2u8 = vqmovn_u16(q1u16); + d3u8 = vqmovn_u16(q2u16); + + d0x2u16 = vtrn_u16(vreinterpret_u16_u8(d2u8), vreinterpret_u16_u8(d3u8)); + d0x2u32 = vtrn_u32(vreinterpret_u32_u16(d0x2u16.val[0]), + vreinterpret_u32_u16(d0x2u16.val[1])); + d0x2u8 = vtrn_u8(vreinterpret_u8_u32(d0x2u32.val[0]), + vreinterpret_u8_u32(d0x2u32.val[1])); + + q1u8 = vcombine_u8(d0x2u8.val[0], d0x2u8.val[1]); + q3u8 = vreinterpretq_u8_u32(vcombine_u32(d6u32, d7u32)); + + q1u8 = vrhaddq_u8(q1u8, q3u8); + + d2u32 = vreinterpret_u32_u8(vget_low_u8(q1u8)); + d3u32 = vreinterpret_u32_u8(vget_high_u8(q1u8)); + + d = dst; + vst1_lane_u32((uint32_t *)d, d2u32, 0); + d += dst_stride; + vst1_lane_u32((uint32_t *)d, d3u32, 0); + d += dst_stride; + vst1_lane_u32((uint32_t *)d, d2u32, 1); + d += dst_stride; + vst1_lane_u32((uint32_t *)d, d3u32, 1); + + q8u16 = q9u16; + d20s16 = d23s16; + q11u16 = q12u16; + q9u16 = q13u16; + d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16)); + } + src += src_stride * 4 - w - 7; + dst += dst_stride * 4 - w; + } + return; +} + +void aom_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, // unused + int x_step_q4, // unused + const int16_t *filter_y, int y_step_q4, int w, + int h) { + int height; + const uint8_t *s; + uint8_t *d; + uint8x8_t d2u8, d3u8; + uint32x2_t d2u32, d3u32, d6u32, d7u32; + uint32x2_t d16u32, d18u32, d20u32, d22u32, d24u32, d26u32; + uint8x16_t q1u8, q3u8; + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16; + int16x4_t d24s16, d25s16, d26s16, d27s16; + uint16x4_t d2u16, d3u16, d4u16, d5u16; + int16x8_t q0s16; + uint16x8_t q1u16, q2u16, q8u16, q9u16, q10u16, q11u16, q12u16, q13u16; + int32x4_t q1s32, q2s32, q14s32, q15s32; + + assert(y_step_q4 == 16); + + (void)x_step_q4; + (void)y_step_q4; + (void)filter_x; + + src -= src_stride * 3; + q0s16 = vld1q_s16(filter_y); + for (; w > 0; w -= 4, src += 4, dst += 4) { // loop_vert_h + s = src; + d16u32 = vld1_lane_u32((const uint32_t *)s, d16u32, 0); + s += src_stride; + d16u32 = vld1_lane_u32((const uint32_t *)s, d16u32, 1); + s += src_stride; + d18u32 = vld1_lane_u32((const uint32_t *)s, d18u32, 0); + s += src_stride; + d18u32 = vld1_lane_u32((const uint32_t *)s, d18u32, 1); + s += src_stride; + d20u32 = vld1_lane_u32((const uint32_t *)s, d20u32, 0); + s += src_stride; + d20u32 = vld1_lane_u32((const uint32_t *)s, d20u32, 1); + s += src_stride; + d22u32 = vld1_lane_u32((const uint32_t *)s, d22u32, 0); + s += src_stride; + + q8u16 = vmovl_u8(vreinterpret_u8_u32(d16u32)); + q9u16 = vmovl_u8(vreinterpret_u8_u32(d18u32)); + q10u16 = vmovl_u8(vreinterpret_u8_u32(d20u32)); + q11u16 = vmovl_u8(vreinterpret_u8_u32(d22u32)); + + d18s16 = vreinterpret_s16_u16(vget_low_u16(q9u16)); + d19s16 = vreinterpret_s16_u16(vget_high_u16(q9u16)); + d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16)); + d = dst; + for (height = h; height > 0; height -= 4) { // loop_vert + d24u32 = vld1_lane_u32((const uint32_t *)s, d24u32, 0); + s += src_stride; + d26u32 = vld1_lane_u32((const uint32_t *)s, d26u32, 0); + s += src_stride; + d26u32 = vld1_lane_u32((const uint32_t *)s, d26u32, 1); + s += src_stride; + d24u32 = vld1_lane_u32((const uint32_t *)s, d24u32, 1); + s += src_stride; + + q12u16 = vmovl_u8(vreinterpret_u8_u32(d24u32)); + q13u16 = vmovl_u8(vreinterpret_u8_u32(d26u32)); + + d6u32 = vld1_lane_u32((const uint32_t *)d, d6u32, 0); + d += dst_stride; + d6u32 = vld1_lane_u32((const uint32_t *)d, d6u32, 1); + d += dst_stride; + d7u32 = vld1_lane_u32((const uint32_t *)d, d7u32, 0); + d += dst_stride; + d7u32 = vld1_lane_u32((const uint32_t *)d, d7u32, 1); + d -= dst_stride * 3; + + d16s16 = vreinterpret_s16_u16(vget_low_u16(q8u16)); + d17s16 = vreinterpret_s16_u16(vget_high_u16(q8u16)); + d20s16 = vreinterpret_s16_u16(vget_low_u16(q10u16)); + d21s16 = vreinterpret_s16_u16(vget_high_u16(q10u16)); + d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16)); + d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16)); + d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16)); + d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16)); + + __builtin_prefetch(s); + __builtin_prefetch(s + src_stride); + q1s32 = MULTIPLY_BY_Q0(d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, + d22s16, d24s16, q0s16); + __builtin_prefetch(s + src_stride * 2); + __builtin_prefetch(s + src_stride * 3); + q2s32 = MULTIPLY_BY_Q0(d17s16, d18s16, d19s16, d20s16, d21s16, d22s16, + d24s16, d26s16, q0s16); + __builtin_prefetch(d); + __builtin_prefetch(d + dst_stride); + q14s32 = MULTIPLY_BY_Q0(d18s16, d19s16, d20s16, d21s16, d22s16, d24s16, + d26s16, d27s16, q0s16); + __builtin_prefetch(d + dst_stride * 2); + __builtin_prefetch(d + dst_stride * 3); + q15s32 = MULTIPLY_BY_Q0(d19s16, d20s16, d21s16, d22s16, d24s16, d26s16, + d27s16, d25s16, q0s16); + + d2u16 = vqrshrun_n_s32(q1s32, 7); + d3u16 = vqrshrun_n_s32(q2s32, 7); + d4u16 = vqrshrun_n_s32(q14s32, 7); + d5u16 = vqrshrun_n_s32(q15s32, 7); + + q1u16 = vcombine_u16(d2u16, d3u16); + q2u16 = vcombine_u16(d4u16, d5u16); + + d2u8 = vqmovn_u16(q1u16); + d3u8 = vqmovn_u16(q2u16); + + q1u8 = vcombine_u8(d2u8, d3u8); + q3u8 = vreinterpretq_u8_u32(vcombine_u32(d6u32, d7u32)); + + q1u8 = vrhaddq_u8(q1u8, q3u8); + + d2u32 = vreinterpret_u32_u8(vget_low_u8(q1u8)); + d3u32 = vreinterpret_u32_u8(vget_high_u8(q1u8)); + + vst1_lane_u32((uint32_t *)d, d2u32, 0); + d += dst_stride; + vst1_lane_u32((uint32_t *)d, d2u32, 1); + d += dst_stride; + vst1_lane_u32((uint32_t *)d, d3u32, 0); + d += dst_stride; + vst1_lane_u32((uint32_t *)d, d3u32, 1); + d += dst_stride; + + q8u16 = q10u16; + d18s16 = d22s16; + d19s16 = d24s16; + q10u16 = q13u16; + d22s16 = d25s16; + } + } + return; +} diff --git a/third_party/aom/aom_dsp/arm/aom_convolve8_avg_neon_asm.asm b/third_party/aom/aom_dsp/arm/aom_convolve8_avg_neon_asm.asm new file mode 100644 index 0000000000..80aef992dd --- /dev/null +++ b/third_party/aom/aom_dsp/arm/aom_convolve8_avg_neon_asm.asm @@ -0,0 +1,295 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + + ; These functions are only valid when: + ; x_step_q4 == 16 + ; w%4 == 0 + ; h%4 == 0 + ; taps == 8 + ; AV1_FILTER_WEIGHT == 128 + ; AV1_FILTER_SHIFT == 7 + + EXPORT |aom_convolve8_avg_horiz_neon| + EXPORT |aom_convolve8_avg_vert_neon| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + + ; Multiply and accumulate by q0 + MACRO + MULTIPLY_BY_Q0 $dst, $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7 + vmull.s16 $dst, $src0, d0[0] + vmlal.s16 $dst, $src1, d0[1] + vmlal.s16 $dst, $src2, d0[2] + vmlal.s16 $dst, $src3, d0[3] + vmlal.s16 $dst, $src4, d1[0] + vmlal.s16 $dst, $src5, d1[1] + vmlal.s16 $dst, $src6, d1[2] + vmlal.s16 $dst, $src7, d1[3] + MEND + +; r0 const uint8_t *src +; r1 int src_stride +; r2 uint8_t *dst +; r3 int dst_stride +; sp[]const int16_t *filter_x +; sp[]int x_step_q4 +; sp[]const int16_t *filter_y ; unused +; sp[]int y_step_q4 ; unused +; sp[]int w +; sp[]int h + +|aom_convolve8_avg_horiz_neon| PROC + push {r4-r10, lr} + + sub r0, r0, #3 ; adjust for taps + + ldr r5, [sp, #32] ; filter_x + ldr r6, [sp, #48] ; w + ldr r7, [sp, #52] ; h + + vld1.s16 {q0}, [r5] ; filter_x + + sub r8, r1, r1, lsl #2 ; -src_stride * 3 + add r8, r8, #4 ; -src_stride * 3 + 4 + + sub r4, r3, r3, lsl #2 ; -dst_stride * 3 + add r4, r4, #4 ; -dst_stride * 3 + 4 + + rsb r9, r6, r1, lsl #2 ; reset src for outer loop + sub r9, r9, #7 + rsb r12, r6, r3, lsl #2 ; reset dst for outer loop + + mov r10, r6 ; w loop counter + +aom_convolve8_avg_loop_horiz_v + vld1.8 {d24}, [r0], r1 + vld1.8 {d25}, [r0], r1 + vld1.8 {d26}, [r0], r1 + vld1.8 {d27}, [r0], r8 + + vtrn.16 q12, q13 + vtrn.8 d24, d25 + vtrn.8 d26, d27 + + pld [r0, r1, lsl #2] + + vmovl.u8 q8, d24 + vmovl.u8 q9, d25 + vmovl.u8 q10, d26 + vmovl.u8 q11, d27 + + ; save a few instructions in the inner loop + vswp d17, d18 + vmov d23, d21 + + add r0, r0, #3 + +aom_convolve8_avg_loop_horiz + add r5, r0, #64 + + vld1.32 {d28[]}, [r0], r1 + vld1.32 {d29[]}, [r0], r1 + vld1.32 {d31[]}, [r0], r1 + vld1.32 {d30[]}, [r0], r8 + + pld [r5] + + vtrn.16 d28, d31 + vtrn.16 d29, d30 + vtrn.8 d28, d29 + vtrn.8 d31, d30 + + pld [r5, r1] + + ; extract to s16 + vtrn.32 q14, q15 + vmovl.u8 q12, d28 + vmovl.u8 q13, d29 + + pld [r5, r1, lsl #1] + + ; slightly out of order load to match the existing data + vld1.u32 {d6[0]}, [r2], r3 + vld1.u32 {d7[0]}, [r2], r3 + vld1.u32 {d6[1]}, [r2], r3 + vld1.u32 {d7[1]}, [r2], r3 + + sub r2, r2, r3, lsl #2 ; reset for store + + ; src[] * filter_x + MULTIPLY_BY_Q0 q1, d16, d17, d20, d22, d18, d19, d23, d24 + MULTIPLY_BY_Q0 q2, d17, d20, d22, d18, d19, d23, d24, d26 + MULTIPLY_BY_Q0 q14, d20, d22, d18, d19, d23, d24, d26, d27 + MULTIPLY_BY_Q0 q15, d22, d18, d19, d23, d24, d26, d27, d25 + + pld [r5, -r8] + + ; += 64 >> 7 + vqrshrun.s32 d2, q1, #7 + vqrshrun.s32 d3, q2, #7 + vqrshrun.s32 d4, q14, #7 + vqrshrun.s32 d5, q15, #7 + + ; saturate + vqmovn.u16 d2, q1 + vqmovn.u16 d3, q2 + + ; transpose + vtrn.16 d2, d3 + vtrn.32 d2, d3 + vtrn.8 d2, d3 + + ; average the new value and the dst value + vrhadd.u8 q1, q1, q3 + + vst1.u32 {d2[0]}, [r2@32], r3 + vst1.u32 {d3[0]}, [r2@32], r3 + vst1.u32 {d2[1]}, [r2@32], r3 + vst1.u32 {d3[1]}, [r2@32], r4 + + vmov q8, q9 + vmov d20, d23 + vmov q11, q12 + vmov q9, q13 + + subs r6, r6, #4 ; w -= 4 + bgt aom_convolve8_avg_loop_horiz + + ; outer loop + mov r6, r10 ; restore w counter + add r0, r0, r9 ; src += src_stride * 4 - w + add r2, r2, r12 ; dst += dst_stride * 4 - w + subs r7, r7, #4 ; h -= 4 + bgt aom_convolve8_avg_loop_horiz_v + + pop {r4-r10, pc} + + ENDP + +|aom_convolve8_avg_vert_neon| PROC + push {r4-r8, lr} + + ; adjust for taps + sub r0, r0, r1 + sub r0, r0, r1, lsl #1 + + ldr r4, [sp, #32] ; filter_y + ldr r6, [sp, #40] ; w + ldr lr, [sp, #44] ; h + + vld1.s16 {q0}, [r4] ; filter_y + + lsl r1, r1, #1 + lsl r3, r3, #1 + +aom_convolve8_avg_loop_vert_h + mov r4, r0 + add r7, r0, r1, asr #1 + mov r5, r2 + add r8, r2, r3, asr #1 + mov r12, lr ; h loop counter + + vld1.u32 {d16[0]}, [r4], r1 + vld1.u32 {d16[1]}, [r7], r1 + vld1.u32 {d18[0]}, [r4], r1 + vld1.u32 {d18[1]}, [r7], r1 + vld1.u32 {d20[0]}, [r4], r1 + vld1.u32 {d20[1]}, [r7], r1 + vld1.u32 {d22[0]}, [r4], r1 + + vmovl.u8 q8, d16 + vmovl.u8 q9, d18 + vmovl.u8 q10, d20 + vmovl.u8 q11, d22 + +aom_convolve8_avg_loop_vert + ; always process a 4x4 block at a time + vld1.u32 {d24[0]}, [r7], r1 + vld1.u32 {d26[0]}, [r4], r1 + vld1.u32 {d26[1]}, [r7], r1 + vld1.u32 {d24[1]}, [r4], r1 + + ; extract to s16 + vmovl.u8 q12, d24 + vmovl.u8 q13, d26 + + vld1.u32 {d6[0]}, [r5@32], r3 + vld1.u32 {d6[1]}, [r8@32], r3 + vld1.u32 {d7[0]}, [r5@32], r3 + vld1.u32 {d7[1]}, [r8@32], r3 + + pld [r7] + pld [r4] + + ; src[] * filter_y + MULTIPLY_BY_Q0 q1, d16, d17, d18, d19, d20, d21, d22, d24 + + pld [r7, r1] + pld [r4, r1] + + MULTIPLY_BY_Q0 q2, d17, d18, d19, d20, d21, d22, d24, d26 + + pld [r5] + pld [r8] + + MULTIPLY_BY_Q0 q14, d18, d19, d20, d21, d22, d24, d26, d27 + + pld [r5, r3] + pld [r8, r3] + + MULTIPLY_BY_Q0 q15, d19, d20, d21, d22, d24, d26, d27, d25 + + ; += 64 >> 7 + vqrshrun.s32 d2, q1, #7 + vqrshrun.s32 d3, q2, #7 + vqrshrun.s32 d4, q14, #7 + vqrshrun.s32 d5, q15, #7 + + ; saturate + vqmovn.u16 d2, q1 + vqmovn.u16 d3, q2 + + ; average the new value and the dst value + vrhadd.u8 q1, q1, q3 + + sub r5, r5, r3, lsl #1 ; reset for store + sub r8, r8, r3, lsl #1 + + vst1.u32 {d2[0]}, [r5@32], r3 + vst1.u32 {d2[1]}, [r8@32], r3 + vst1.u32 {d3[0]}, [r5@32], r3 + vst1.u32 {d3[1]}, [r8@32], r3 + + vmov q8, q10 + vmov d18, d22 + vmov d19, d24 + vmov q10, q13 + vmov d22, d25 + + subs r12, r12, #4 ; h -= 4 + bgt aom_convolve8_avg_loop_vert + + ; outer loop + add r0, r0, #4 + add r2, r2, #4 + subs r6, r6, #4 ; w -= 4 + bgt aom_convolve8_avg_loop_vert_h + + pop {r4-r8, pc} + + ENDP + END diff --git a/third_party/aom/aom_dsp/arm/aom_convolve8_neon.c b/third_party/aom/aom_dsp/arm/aom_convolve8_neon.c new file mode 100644 index 0000000000..8ebffb5f9a --- /dev/null +++ b/third_party/aom/aom_dsp/arm/aom_convolve8_neon.c @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" + +static INLINE int32x4_t MULTIPLY_BY_Q0(int16x4_t dsrc0, int16x4_t dsrc1, + int16x4_t dsrc2, int16x4_t dsrc3, + int16x4_t dsrc4, int16x4_t dsrc5, + int16x4_t dsrc6, int16x4_t dsrc7, + int16x8_t q0s16) { + int32x4_t qdst; + int16x4_t d0s16, d1s16; + + d0s16 = vget_low_s16(q0s16); + d1s16 = vget_high_s16(q0s16); + + qdst = vmull_lane_s16(dsrc0, d0s16, 0); + qdst = vmlal_lane_s16(qdst, dsrc1, d0s16, 1); + qdst = vmlal_lane_s16(qdst, dsrc2, d0s16, 2); + qdst = vmlal_lane_s16(qdst, dsrc3, d0s16, 3); + qdst = vmlal_lane_s16(qdst, dsrc4, d1s16, 0); + qdst = vmlal_lane_s16(qdst, dsrc5, d1s16, 1); + qdst = vmlal_lane_s16(qdst, dsrc6, d1s16, 2); + qdst = vmlal_lane_s16(qdst, dsrc7, d1s16, 3); + return qdst; +} + +void aom_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, // unused + int y_step_q4, // unused + int w, int h) { + int width; + const uint8_t *s, *psrc; + uint8_t *d, *pdst; + uint8x8_t d2u8, d3u8, d24u8, d25u8, d26u8, d27u8, d28u8, d29u8; + uint32x2_t d2u32, d3u32, d28u32, d29u32, d30u32, d31u32; + uint8x16_t q12u8, q13u8, q14u8, q15u8; + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d22s16, d23s16; + int16x4_t d24s16, d25s16, d26s16, d27s16; + uint16x4_t d2u16, d3u16, d4u16, d5u16, d16u16, d17u16, d18u16, d19u16; + int16x8_t q0s16; + uint16x8_t q1u16, q2u16, q8u16, q9u16, q10u16, q11u16, q12u16, q13u16; + int32x4_t q1s32, q2s32, q14s32, q15s32; + uint16x8x2_t q0x2u16; + uint8x8x2_t d0x2u8, d1x2u8; + uint32x2x2_t d0x2u32; + uint16x4x2_t d0x2u16, d1x2u16; + uint32x4x2_t q0x2u32; + + assert(x_step_q4 == 16); + + (void)x_step_q4; + (void)y_step_q4; + (void)filter_y; + + q0s16 = vld1q_s16(filter_x); + + src -= 3; // adjust for taps + for (; h > 0; h -= 4, src += src_stride * 4, + dst += dst_stride * 4) { // loop_horiz_v + s = src; + d24u8 = vld1_u8(s); + s += src_stride; + d25u8 = vld1_u8(s); + s += src_stride; + d26u8 = vld1_u8(s); + s += src_stride; + d27u8 = vld1_u8(s); + + q12u8 = vcombine_u8(d24u8, d25u8); + q13u8 = vcombine_u8(d26u8, d27u8); + + q0x2u16 = + vtrnq_u16(vreinterpretq_u16_u8(q12u8), vreinterpretq_u16_u8(q13u8)); + d24u8 = vreinterpret_u8_u16(vget_low_u16(q0x2u16.val[0])); + d25u8 = vreinterpret_u8_u16(vget_high_u16(q0x2u16.val[0])); + d26u8 = vreinterpret_u8_u16(vget_low_u16(q0x2u16.val[1])); + d27u8 = vreinterpret_u8_u16(vget_high_u16(q0x2u16.val[1])); + d0x2u8 = vtrn_u8(d24u8, d25u8); + d1x2u8 = vtrn_u8(d26u8, d27u8); + + __builtin_prefetch(src + src_stride * 4); + __builtin_prefetch(src + src_stride * 5); + __builtin_prefetch(src + src_stride * 6); + + q8u16 = vmovl_u8(d0x2u8.val[0]); + q9u16 = vmovl_u8(d0x2u8.val[1]); + q10u16 = vmovl_u8(d1x2u8.val[0]); + q11u16 = vmovl_u8(d1x2u8.val[1]); + + d16u16 = vget_low_u16(q8u16); + d17u16 = vget_high_u16(q8u16); + d18u16 = vget_low_u16(q9u16); + d19u16 = vget_high_u16(q9u16); + q8u16 = vcombine_u16(d16u16, d18u16); // vswp 17 18 + q9u16 = vcombine_u16(d17u16, d19u16); + + d20s16 = vreinterpret_s16_u16(vget_low_u16(q10u16)); + d23s16 = vreinterpret_s16_u16(vget_high_u16(q10u16)); // vmov 23 21 + for (width = w, psrc = src + 7, pdst = dst; width > 0; + width -= 4, psrc += 4, pdst += 4) { // loop_horiz + s = psrc; + d28u32 = vld1_dup_u32((const uint32_t *)s); + s += src_stride; + d29u32 = vld1_dup_u32((const uint32_t *)s); + s += src_stride; + d31u32 = vld1_dup_u32((const uint32_t *)s); + s += src_stride; + d30u32 = vld1_dup_u32((const uint32_t *)s); + + __builtin_prefetch(psrc + 64); + + d0x2u16 = + vtrn_u16(vreinterpret_u16_u32(d28u32), vreinterpret_u16_u32(d31u32)); + d1x2u16 = + vtrn_u16(vreinterpret_u16_u32(d29u32), vreinterpret_u16_u32(d30u32)); + d0x2u8 = vtrn_u8(vreinterpret_u8_u16(d0x2u16.val[0]), // d28 + vreinterpret_u8_u16(d1x2u16.val[0])); // d29 + d1x2u8 = vtrn_u8(vreinterpret_u8_u16(d0x2u16.val[1]), // d31 + vreinterpret_u8_u16(d1x2u16.val[1])); // d30 + + __builtin_prefetch(psrc + 64 + src_stride); + + q14u8 = vcombine_u8(d0x2u8.val[0], d0x2u8.val[1]); + q15u8 = vcombine_u8(d1x2u8.val[1], d1x2u8.val[0]); + q0x2u32 = + vtrnq_u32(vreinterpretq_u32_u8(q14u8), vreinterpretq_u32_u8(q15u8)); + + d28u8 = vreinterpret_u8_u32(vget_low_u32(q0x2u32.val[0])); + d29u8 = vreinterpret_u8_u32(vget_high_u32(q0x2u32.val[0])); + q12u16 = vmovl_u8(d28u8); + q13u16 = vmovl_u8(d29u8); + + __builtin_prefetch(psrc + 64 + src_stride * 2); + + d16s16 = vreinterpret_s16_u16(vget_low_u16(q8u16)); + d17s16 = vreinterpret_s16_u16(vget_high_u16(q8u16)); + d18s16 = vreinterpret_s16_u16(vget_low_u16(q9u16)); + d19s16 = vreinterpret_s16_u16(vget_high_u16(q9u16)); + d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16)); + d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16)); + d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16)); + d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16)); + d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16)); + + q1s32 = MULTIPLY_BY_Q0(d16s16, d17s16, d20s16, d22s16, d18s16, d19s16, + d23s16, d24s16, q0s16); + q2s32 = MULTIPLY_BY_Q0(d17s16, d20s16, d22s16, d18s16, d19s16, d23s16, + d24s16, d26s16, q0s16); + q14s32 = MULTIPLY_BY_Q0(d20s16, d22s16, d18s16, d19s16, d23s16, d24s16, + d26s16, d27s16, q0s16); + q15s32 = MULTIPLY_BY_Q0(d22s16, d18s16, d19s16, d23s16, d24s16, d26s16, + d27s16, d25s16, q0s16); + + __builtin_prefetch(psrc + 60 + src_stride * 3); + + d2u16 = vqrshrun_n_s32(q1s32, 7); + d3u16 = vqrshrun_n_s32(q2s32, 7); + d4u16 = vqrshrun_n_s32(q14s32, 7); + d5u16 = vqrshrun_n_s32(q15s32, 7); + + q1u16 = vcombine_u16(d2u16, d3u16); + q2u16 = vcombine_u16(d4u16, d5u16); + + d2u8 = vqmovn_u16(q1u16); + d3u8 = vqmovn_u16(q2u16); + + d0x2u16 = vtrn_u16(vreinterpret_u16_u8(d2u8), vreinterpret_u16_u8(d3u8)); + d0x2u32 = vtrn_u32(vreinterpret_u32_u16(d0x2u16.val[0]), + vreinterpret_u32_u16(d0x2u16.val[1])); + d0x2u8 = vtrn_u8(vreinterpret_u8_u32(d0x2u32.val[0]), + vreinterpret_u8_u32(d0x2u32.val[1])); + + d2u32 = vreinterpret_u32_u8(d0x2u8.val[0]); + d3u32 = vreinterpret_u32_u8(d0x2u8.val[1]); + + d = pdst; + vst1_lane_u32((uint32_t *)d, d2u32, 0); + d += dst_stride; + vst1_lane_u32((uint32_t *)d, d3u32, 0); + d += dst_stride; + vst1_lane_u32((uint32_t *)d, d2u32, 1); + d += dst_stride; + vst1_lane_u32((uint32_t *)d, d3u32, 1); + + q8u16 = q9u16; + d20s16 = d23s16; + q11u16 = q12u16; + q9u16 = q13u16; + d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16)); + } + } + return; +} + +void aom_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, // unused + int x_step_q4, // unused + const int16_t *filter_y, int y_step_q4, int w, + int h) { + int height; + const uint8_t *s; + uint8_t *d; + uint32x2_t d2u32, d3u32; + uint32x2_t d16u32, d18u32, d20u32, d22u32, d24u32, d26u32; + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16; + int16x4_t d24s16, d25s16, d26s16, d27s16; + uint16x4_t d2u16, d3u16, d4u16, d5u16; + int16x8_t q0s16; + uint16x8_t q1u16, q2u16, q8u16, q9u16, q10u16, q11u16, q12u16, q13u16; + int32x4_t q1s32, q2s32, q14s32, q15s32; + + assert(y_step_q4 == 16); + + (void)x_step_q4; + (void)y_step_q4; + (void)filter_x; + + src -= src_stride * 3; + q0s16 = vld1q_s16(filter_y); + for (; w > 0; w -= 4, src += 4, dst += 4) { // loop_vert_h + s = src; + d16u32 = vld1_lane_u32((const uint32_t *)s, d16u32, 0); + s += src_stride; + d16u32 = vld1_lane_u32((const uint32_t *)s, d16u32, 1); + s += src_stride; + d18u32 = vld1_lane_u32((const uint32_t *)s, d18u32, 0); + s += src_stride; + d18u32 = vld1_lane_u32((const uint32_t *)s, d18u32, 1); + s += src_stride; + d20u32 = vld1_lane_u32((const uint32_t *)s, d20u32, 0); + s += src_stride; + d20u32 = vld1_lane_u32((const uint32_t *)s, d20u32, 1); + s += src_stride; + d22u32 = vld1_lane_u32((const uint32_t *)s, d22u32, 0); + s += src_stride; + + q8u16 = vmovl_u8(vreinterpret_u8_u32(d16u32)); + q9u16 = vmovl_u8(vreinterpret_u8_u32(d18u32)); + q10u16 = vmovl_u8(vreinterpret_u8_u32(d20u32)); + q11u16 = vmovl_u8(vreinterpret_u8_u32(d22u32)); + + d18s16 = vreinterpret_s16_u16(vget_low_u16(q9u16)); + d19s16 = vreinterpret_s16_u16(vget_high_u16(q9u16)); + d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16)); + d = dst; + for (height = h; height > 0; height -= 4) { // loop_vert + d24u32 = vld1_lane_u32((const uint32_t *)s, d24u32, 0); + s += src_stride; + d26u32 = vld1_lane_u32((const uint32_t *)s, d26u32, 0); + s += src_stride; + d26u32 = vld1_lane_u32((const uint32_t *)s, d26u32, 1); + s += src_stride; + d24u32 = vld1_lane_u32((const uint32_t *)s, d24u32, 1); + s += src_stride; + + q12u16 = vmovl_u8(vreinterpret_u8_u32(d24u32)); + q13u16 = vmovl_u8(vreinterpret_u8_u32(d26u32)); + + d16s16 = vreinterpret_s16_u16(vget_low_u16(q8u16)); + d17s16 = vreinterpret_s16_u16(vget_high_u16(q8u16)); + d20s16 = vreinterpret_s16_u16(vget_low_u16(q10u16)); + d21s16 = vreinterpret_s16_u16(vget_high_u16(q10u16)); + d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16)); + d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16)); + d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16)); + d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16)); + + __builtin_prefetch(d); + __builtin_prefetch(d + dst_stride); + q1s32 = MULTIPLY_BY_Q0(d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, + d22s16, d24s16, q0s16); + __builtin_prefetch(d + dst_stride * 2); + __builtin_prefetch(d + dst_stride * 3); + q2s32 = MULTIPLY_BY_Q0(d17s16, d18s16, d19s16, d20s16, d21s16, d22s16, + d24s16, d26s16, q0s16); + __builtin_prefetch(s); + __builtin_prefetch(s + src_stride); + q14s32 = MULTIPLY_BY_Q0(d18s16, d19s16, d20s16, d21s16, d22s16, d24s16, + d26s16, d27s16, q0s16); + __builtin_prefetch(s + src_stride * 2); + __builtin_prefetch(s + src_stride * 3); + q15s32 = MULTIPLY_BY_Q0(d19s16, d20s16, d21s16, d22s16, d24s16, d26s16, + d27s16, d25s16, q0s16); + + d2u16 = vqrshrun_n_s32(q1s32, 7); + d3u16 = vqrshrun_n_s32(q2s32, 7); + d4u16 = vqrshrun_n_s32(q14s32, 7); + d5u16 = vqrshrun_n_s32(q15s32, 7); + + q1u16 = vcombine_u16(d2u16, d3u16); + q2u16 = vcombine_u16(d4u16, d5u16); + + d2u32 = vreinterpret_u32_u8(vqmovn_u16(q1u16)); + d3u32 = vreinterpret_u32_u8(vqmovn_u16(q2u16)); + + vst1_lane_u32((uint32_t *)d, d2u32, 0); + d += dst_stride; + vst1_lane_u32((uint32_t *)d, d2u32, 1); + d += dst_stride; + vst1_lane_u32((uint32_t *)d, d3u32, 0); + d += dst_stride; + vst1_lane_u32((uint32_t *)d, d3u32, 1); + d += dst_stride; + + q8u16 = q10u16; + d18s16 = d22s16; + d19s16 = d24s16; + q10u16 = q13u16; + d22s16 = d25s16; + } + } + return; +} diff --git a/third_party/aom/aom_dsp/arm/aom_convolve8_neon_asm.asm b/third_party/aom/aom_dsp/arm/aom_convolve8_neon_asm.asm new file mode 100644 index 0000000000..38207d8649 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/aom_convolve8_neon_asm.asm @@ -0,0 +1,273 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + + ; These functions are only valid when: + ; x_step_q4 == 16 + ; w%4 == 0 + ; h%4 == 0 + ; taps == 8 + ; AV1_FILTER_WEIGHT == 128 + ; AV1_FILTER_SHIFT == 7 + + EXPORT |aom_convolve8_horiz_neon| + EXPORT |aom_convolve8_vert_neon| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + + ; Multiply and accumulate by q0 + MACRO + MULTIPLY_BY_Q0 $dst, $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7 + vmull.s16 $dst, $src0, d0[0] + vmlal.s16 $dst, $src1, d0[1] + vmlal.s16 $dst, $src2, d0[2] + vmlal.s16 $dst, $src3, d0[3] + vmlal.s16 $dst, $src4, d1[0] + vmlal.s16 $dst, $src5, d1[1] + vmlal.s16 $dst, $src6, d1[2] + vmlal.s16 $dst, $src7, d1[3] + MEND + +; r0 const uint8_t *src +; r1 int src_stride +; r2 uint8_t *dst +; r3 int dst_stride +; sp[]const int16_t *filter_x +; sp[]int x_step_q4 +; sp[]const int16_t *filter_y ; unused +; sp[]int y_step_q4 ; unused +; sp[]int w +; sp[]int h + +|aom_convolve8_horiz_neon| PROC + push {r4-r10, lr} + + sub r0, r0, #3 ; adjust for taps + + ldr r5, [sp, #32] ; filter_x + ldr r6, [sp, #48] ; w + ldr r7, [sp, #52] ; h + + vld1.s16 {q0}, [r5] ; filter_x + + sub r8, r1, r1, lsl #2 ; -src_stride * 3 + add r8, r8, #4 ; -src_stride * 3 + 4 + + sub r4, r3, r3, lsl #2 ; -dst_stride * 3 + add r4, r4, #4 ; -dst_stride * 3 + 4 + + rsb r9, r6, r1, lsl #2 ; reset src for outer loop + sub r9, r9, #7 + rsb r12, r6, r3, lsl #2 ; reset dst for outer loop + + mov r10, r6 ; w loop counter + +aom_convolve8_loop_horiz_v + vld1.8 {d24}, [r0], r1 + vld1.8 {d25}, [r0], r1 + vld1.8 {d26}, [r0], r1 + vld1.8 {d27}, [r0], r8 + + vtrn.16 q12, q13 + vtrn.8 d24, d25 + vtrn.8 d26, d27 + + pld [r0, r1, lsl #2] + + vmovl.u8 q8, d24 + vmovl.u8 q9, d25 + vmovl.u8 q10, d26 + vmovl.u8 q11, d27 + + ; save a few instructions in the inner loop + vswp d17, d18 + vmov d23, d21 + + add r0, r0, #3 + +aom_convolve8_loop_horiz + add r5, r0, #64 + + vld1.32 {d28[]}, [r0], r1 + vld1.32 {d29[]}, [r0], r1 + vld1.32 {d31[]}, [r0], r1 + vld1.32 {d30[]}, [r0], r8 + + pld [r5] + + vtrn.16 d28, d31 + vtrn.16 d29, d30 + vtrn.8 d28, d29 + vtrn.8 d31, d30 + + pld [r5, r1] + + ; extract to s16 + vtrn.32 q14, q15 + vmovl.u8 q12, d28 + vmovl.u8 q13, d29 + + pld [r5, r1, lsl #1] + + ; src[] * filter_x + MULTIPLY_BY_Q0 q1, d16, d17, d20, d22, d18, d19, d23, d24 + MULTIPLY_BY_Q0 q2, d17, d20, d22, d18, d19, d23, d24, d26 + MULTIPLY_BY_Q0 q14, d20, d22, d18, d19, d23, d24, d26, d27 + MULTIPLY_BY_Q0 q15, d22, d18, d19, d23, d24, d26, d27, d25 + + pld [r5, -r8] + + ; += 64 >> 7 + vqrshrun.s32 d2, q1, #7 + vqrshrun.s32 d3, q2, #7 + vqrshrun.s32 d4, q14, #7 + vqrshrun.s32 d5, q15, #7 + + ; saturate + vqmovn.u16 d2, q1 + vqmovn.u16 d3, q2 + + ; transpose + vtrn.16 d2, d3 + vtrn.32 d2, d3 + vtrn.8 d2, d3 + + vst1.u32 {d2[0]}, [r2@32], r3 + vst1.u32 {d3[0]}, [r2@32], r3 + vst1.u32 {d2[1]}, [r2@32], r3 + vst1.u32 {d3[1]}, [r2@32], r4 + + vmov q8, q9 + vmov d20, d23 + vmov q11, q12 + vmov q9, q13 + + subs r6, r6, #4 ; w -= 4 + bgt aom_convolve8_loop_horiz + + ; outer loop + mov r6, r10 ; restore w counter + add r0, r0, r9 ; src += src_stride * 4 - w + add r2, r2, r12 ; dst += dst_stride * 4 - w + subs r7, r7, #4 ; h -= 4 + bgt aom_convolve8_loop_horiz_v + + pop {r4-r10, pc} + + ENDP + +|aom_convolve8_vert_neon| PROC + push {r4-r8, lr} + + ; adjust for taps + sub r0, r0, r1 + sub r0, r0, r1, lsl #1 + + ldr r4, [sp, #32] ; filter_y + ldr r6, [sp, #40] ; w + ldr lr, [sp, #44] ; h + + vld1.s16 {q0}, [r4] ; filter_y + + lsl r1, r1, #1 + lsl r3, r3, #1 + +aom_convolve8_loop_vert_h + mov r4, r0 + add r7, r0, r1, asr #1 + mov r5, r2 + add r8, r2, r3, asr #1 + mov r12, lr ; h loop counter + + vld1.u32 {d16[0]}, [r4], r1 + vld1.u32 {d16[1]}, [r7], r1 + vld1.u32 {d18[0]}, [r4], r1 + vld1.u32 {d18[1]}, [r7], r1 + vld1.u32 {d20[0]}, [r4], r1 + vld1.u32 {d20[1]}, [r7], r1 + vld1.u32 {d22[0]}, [r4], r1 + + vmovl.u8 q8, d16 + vmovl.u8 q9, d18 + vmovl.u8 q10, d20 + vmovl.u8 q11, d22 + +aom_convolve8_loop_vert + ; always process a 4x4 block at a time + vld1.u32 {d24[0]}, [r7], r1 + vld1.u32 {d26[0]}, [r4], r1 + vld1.u32 {d26[1]}, [r7], r1 + vld1.u32 {d24[1]}, [r4], r1 + + ; extract to s16 + vmovl.u8 q12, d24 + vmovl.u8 q13, d26 + + pld [r5] + pld [r8] + + ; src[] * filter_y + MULTIPLY_BY_Q0 q1, d16, d17, d18, d19, d20, d21, d22, d24 + + pld [r5, r3] + pld [r8, r3] + + MULTIPLY_BY_Q0 q2, d17, d18, d19, d20, d21, d22, d24, d26 + + pld [r7] + pld [r4] + + MULTIPLY_BY_Q0 q14, d18, d19, d20, d21, d22, d24, d26, d27 + + pld [r7, r1] + pld [r4, r1] + + MULTIPLY_BY_Q0 q15, d19, d20, d21, d22, d24, d26, d27, d25 + + ; += 64 >> 7 + vqrshrun.s32 d2, q1, #7 + vqrshrun.s32 d3, q2, #7 + vqrshrun.s32 d4, q14, #7 + vqrshrun.s32 d5, q15, #7 + + ; saturate + vqmovn.u16 d2, q1 + vqmovn.u16 d3, q2 + + vst1.u32 {d2[0]}, [r5@32], r3 + vst1.u32 {d2[1]}, [r8@32], r3 + vst1.u32 {d3[0]}, [r5@32], r3 + vst1.u32 {d3[1]}, [r8@32], r3 + + vmov q8, q10 + vmov d18, d22 + vmov d19, d24 + vmov q10, q13 + vmov d22, d25 + + subs r12, r12, #4 ; h -= 4 + bgt aom_convolve8_loop_vert + + ; outer loop + add r0, r0, #4 + add r2, r2, #4 + subs r6, r6, #4 ; w -= 4 + bgt aom_convolve8_loop_vert_h + + pop {r4-r8, pc} + + ENDP + END diff --git a/third_party/aom/aom_dsp/arm/aom_convolve_avg_neon.c b/third_party/aom/aom_dsp/arm/aom_convolve_avg_neon.c new file mode 100644 index 0000000000..f05d3ceae2 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/aom_convolve_avg_neon.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +void aom_convolve_avg_neon(const uint8_t *src, // r0 + ptrdiff_t src_stride, // r1 + uint8_t *dst, // r2 + ptrdiff_t dst_stride, // r3 + const int16_t *filter_x, int filter_x_stride, + const int16_t *filter_y, int filter_y_stride, int w, + int h) { + uint8_t *d; + uint8x8_t d0u8, d1u8, d2u8, d3u8; + uint32x2_t d0u32, d2u32; + uint8x16_t q0u8, q1u8, q2u8, q3u8, q8u8, q9u8, q10u8, q11u8; + (void)filter_x; + (void)filter_x_stride; + (void)filter_y; + (void)filter_y_stride; + + d = dst; + if (w > 32) { // avg64 + for (; h > 0; h -= 1) { + q0u8 = vld1q_u8(src); + q1u8 = vld1q_u8(src + 16); + q2u8 = vld1q_u8(src + 32); + q3u8 = vld1q_u8(src + 48); + src += src_stride; + q8u8 = vld1q_u8(d); + q9u8 = vld1q_u8(d + 16); + q10u8 = vld1q_u8(d + 32); + q11u8 = vld1q_u8(d + 48); + d += dst_stride; + + q0u8 = vrhaddq_u8(q0u8, q8u8); + q1u8 = vrhaddq_u8(q1u8, q9u8); + q2u8 = vrhaddq_u8(q2u8, q10u8); + q3u8 = vrhaddq_u8(q3u8, q11u8); + + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q1u8); + vst1q_u8(dst + 32, q2u8); + vst1q_u8(dst + 48, q3u8); + dst += dst_stride; + } + } else if (w == 32) { // avg32 + for (; h > 0; h -= 2) { + q0u8 = vld1q_u8(src); + q1u8 = vld1q_u8(src + 16); + src += src_stride; + q2u8 = vld1q_u8(src); + q3u8 = vld1q_u8(src + 16); + src += src_stride; + q8u8 = vld1q_u8(d); + q9u8 = vld1q_u8(d + 16); + d += dst_stride; + q10u8 = vld1q_u8(d); + q11u8 = vld1q_u8(d + 16); + d += dst_stride; + + q0u8 = vrhaddq_u8(q0u8, q8u8); + q1u8 = vrhaddq_u8(q1u8, q9u8); + q2u8 = vrhaddq_u8(q2u8, q10u8); + q3u8 = vrhaddq_u8(q3u8, q11u8); + + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q1u8); + dst += dst_stride; + vst1q_u8(dst, q2u8); + vst1q_u8(dst + 16, q3u8); + dst += dst_stride; + } + } else if (w > 8) { // avg16 + for (; h > 0; h -= 2) { + q0u8 = vld1q_u8(src); + src += src_stride; + q1u8 = vld1q_u8(src); + src += src_stride; + q2u8 = vld1q_u8(d); + d += dst_stride; + q3u8 = vld1q_u8(d); + d += dst_stride; + + q0u8 = vrhaddq_u8(q0u8, q2u8); + q1u8 = vrhaddq_u8(q1u8, q3u8); + + vst1q_u8(dst, q0u8); + dst += dst_stride; + vst1q_u8(dst, q1u8); + dst += dst_stride; + } + } else if (w == 8) { // avg8 + for (; h > 0; h -= 2) { + d0u8 = vld1_u8(src); + src += src_stride; + d1u8 = vld1_u8(src); + src += src_stride; + d2u8 = vld1_u8(d); + d += dst_stride; + d3u8 = vld1_u8(d); + d += dst_stride; + + q0u8 = vcombine_u8(d0u8, d1u8); + q1u8 = vcombine_u8(d2u8, d3u8); + q0u8 = vrhaddq_u8(q0u8, q1u8); + + vst1_u8(dst, vget_low_u8(q0u8)); + dst += dst_stride; + vst1_u8(dst, vget_high_u8(q0u8)); + dst += dst_stride; + } + } else { // avg4 + for (; h > 0; h -= 2) { + d0u32 = vld1_lane_u32((const uint32_t *)src, d0u32, 0); + src += src_stride; + d0u32 = vld1_lane_u32((const uint32_t *)src, d0u32, 1); + src += src_stride; + d2u32 = vld1_lane_u32((const uint32_t *)d, d2u32, 0); + d += dst_stride; + d2u32 = vld1_lane_u32((const uint32_t *)d, d2u32, 1); + d += dst_stride; + + d0u8 = vrhadd_u8(vreinterpret_u8_u32(d0u32), vreinterpret_u8_u32(d2u32)); + + d0u32 = vreinterpret_u32_u8(d0u8); + vst1_lane_u32((uint32_t *)dst, d0u32, 0); + dst += dst_stride; + vst1_lane_u32((uint32_t *)dst, d0u32, 1); + dst += dst_stride; + } + } + return; +} diff --git a/third_party/aom/aom_dsp/arm/aom_convolve_avg_neon_asm.asm b/third_party/aom/aom_dsp/arm/aom_convolve_avg_neon_asm.asm new file mode 100644 index 0000000000..43c300954f --- /dev/null +++ b/third_party/aom/aom_dsp/arm/aom_convolve_avg_neon_asm.asm @@ -0,0 +1,119 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + EXPORT |aom_convolve_avg_neon| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +|aom_convolve_avg_neon| PROC + push {r4-r6, lr} + ldrd r4, r5, [sp, #32] + mov r6, r2 + + cmp r4, #32 + bgt avg64 + beq avg32 + cmp r4, #8 + bgt avg16 + beq avg8 + b avg4 + +avg64 + sub lr, r1, #32 + sub r4, r3, #32 +avg64_h + pld [r0, r1, lsl #1] + vld1.8 {q0-q1}, [r0]! + vld1.8 {q2-q3}, [r0], lr + pld [r2, r3] + vld1.8 {q8-q9}, [r6@128]! + vld1.8 {q10-q11}, [r6@128], r4 + vrhadd.u8 q0, q0, q8 + vrhadd.u8 q1, q1, q9 + vrhadd.u8 q2, q2, q10 + vrhadd.u8 q3, q3, q11 + vst1.8 {q0-q1}, [r2@128]! + vst1.8 {q2-q3}, [r2@128], r4 + subs r5, r5, #1 + bgt avg64_h + pop {r4-r6, pc} + +avg32 + vld1.8 {q0-q1}, [r0], r1 + vld1.8 {q2-q3}, [r0], r1 + vld1.8 {q8-q9}, [r6@128], r3 + vld1.8 {q10-q11}, [r6@128], r3 + pld [r0] + vrhadd.u8 q0, q0, q8 + pld [r0, r1] + vrhadd.u8 q1, q1, q9 + pld [r6] + vrhadd.u8 q2, q2, q10 + pld [r6, r3] + vrhadd.u8 q3, q3, q11 + vst1.8 {q0-q1}, [r2@128], r3 + vst1.8 {q2-q3}, [r2@128], r3 + subs r5, r5, #2 + bgt avg32 + pop {r4-r6, pc} + +avg16 + vld1.8 {q0}, [r0], r1 + vld1.8 {q1}, [r0], r1 + vld1.8 {q2}, [r6@128], r3 + vld1.8 {q3}, [r6@128], r3 + pld [r0] + pld [r0, r1] + vrhadd.u8 q0, q0, q2 + pld [r6] + pld [r6, r3] + vrhadd.u8 q1, q1, q3 + vst1.8 {q0}, [r2@128], r3 + vst1.8 {q1}, [r2@128], r3 + subs r5, r5, #2 + bgt avg16 + pop {r4-r6, pc} + +avg8 + vld1.8 {d0}, [r0], r1 + vld1.8 {d1}, [r0], r1 + vld1.8 {d2}, [r6@64], r3 + vld1.8 {d3}, [r6@64], r3 + pld [r0] + pld [r0, r1] + vrhadd.u8 q0, q0, q1 + pld [r6] + pld [r6, r3] + vst1.8 {d0}, [r2@64], r3 + vst1.8 {d1}, [r2@64], r3 + subs r5, r5, #2 + bgt avg8 + pop {r4-r6, pc} + +avg4 + vld1.32 {d0[0]}, [r0], r1 + vld1.32 {d0[1]}, [r0], r1 + vld1.32 {d2[0]}, [r6@32], r3 + vld1.32 {d2[1]}, [r6@32], r3 + vrhadd.u8 d0, d0, d2 + vst1.32 {d0[0]}, [r2@32], r3 + vst1.32 {d0[1]}, [r2@32], r3 + subs r5, r5, #2 + bgt avg4 + pop {r4-r6, pc} + ENDP + + END diff --git a/third_party/aom/aom_dsp/arm/aom_convolve_copy_neon.c b/third_party/aom/aom_dsp/arm/aom_convolve_copy_neon.c new file mode 100644 index 0000000000..9e57c7176f --- /dev/null +++ b/third_party/aom/aom_dsp/arm/aom_convolve_copy_neon.c @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +void aom_convolve_copy_neon(const uint8_t *src, // r0 + ptrdiff_t src_stride, // r1 + uint8_t *dst, // r2 + ptrdiff_t dst_stride, // r3 + const int16_t *filter_x, int filter_x_stride, + const int16_t *filter_y, int filter_y_stride, int w, + int h) { + uint8x8_t d0u8, d2u8; + uint8x16_t q0u8, q1u8, q2u8, q3u8; + (void)filter_x; + (void)filter_x_stride; + (void)filter_y; + (void)filter_y_stride; + + if (w > 32) { // copy64 + for (; h > 0; h--) { + q0u8 = vld1q_u8(src); + q1u8 = vld1q_u8(src + 16); + q2u8 = vld1q_u8(src + 32); + q3u8 = vld1q_u8(src + 48); + src += src_stride; + + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q1u8); + vst1q_u8(dst + 32, q2u8); + vst1q_u8(dst + 48, q3u8); + dst += dst_stride; + } + } else if (w == 32) { // copy32 + for (; h > 0; h -= 2) { + q0u8 = vld1q_u8(src); + q1u8 = vld1q_u8(src + 16); + src += src_stride; + q2u8 = vld1q_u8(src); + q3u8 = vld1q_u8(src + 16); + src += src_stride; + + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q1u8); + dst += dst_stride; + vst1q_u8(dst, q2u8); + vst1q_u8(dst + 16, q3u8); + dst += dst_stride; + } + } else if (w > 8) { // copy16 + for (; h > 0; h -= 2) { + q0u8 = vld1q_u8(src); + src += src_stride; + q1u8 = vld1q_u8(src); + src += src_stride; + + vst1q_u8(dst, q0u8); + dst += dst_stride; + vst1q_u8(dst, q1u8); + dst += dst_stride; + } + } else if (w == 8) { // copy8 + for (; h > 0; h -= 2) { + d0u8 = vld1_u8(src); + src += src_stride; + d2u8 = vld1_u8(src); + src += src_stride; + + vst1_u8(dst, d0u8); + dst += dst_stride; + vst1_u8(dst, d2u8); + dst += dst_stride; + } + } else { // copy4 + for (; h > 0; h--) { + *(uint32_t *)dst = *(const uint32_t *)src; + src += src_stride; + dst += dst_stride; + } + } + return; +} diff --git a/third_party/aom/aom_dsp/arm/aom_convolve_copy_neon_asm.asm b/third_party/aom/aom_dsp/arm/aom_convolve_copy_neon_asm.asm new file mode 100644 index 0000000000..443d7178a6 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/aom_convolve_copy_neon_asm.asm @@ -0,0 +1,87 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + EXPORT |aom_convolve_copy_neon| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +|aom_convolve_copy_neon| PROC + push {r4-r5, lr} + ldrd r4, r5, [sp, #28] + + cmp r4, #32 + bgt copy64 + beq copy32 + cmp r4, #8 + bgt copy16 + beq copy8 + b copy4 + +copy64 + sub lr, r1, #32 + sub r3, r3, #32 +copy64_h + pld [r0, r1, lsl #1] + vld1.8 {q0-q1}, [r0]! + vld1.8 {q2-q3}, [r0], lr + vst1.8 {q0-q1}, [r2@128]! + vst1.8 {q2-q3}, [r2@128], r3 + subs r5, r5, #1 + bgt copy64_h + pop {r4-r5, pc} + +copy32 + pld [r0, r1, lsl #1] + vld1.8 {q0-q1}, [r0], r1 + pld [r0, r1, lsl #1] + vld1.8 {q2-q3}, [r0], r1 + vst1.8 {q0-q1}, [r2@128], r3 + vst1.8 {q2-q3}, [r2@128], r3 + subs r5, r5, #2 + bgt copy32 + pop {r4-r5, pc} + +copy16 + pld [r0, r1, lsl #1] + vld1.8 {q0}, [r0], r1 + pld [r0, r1, lsl #1] + vld1.8 {q1}, [r0], r1 + vst1.8 {q0}, [r2@128], r3 + vst1.8 {q1}, [r2@128], r3 + subs r5, r5, #2 + bgt copy16 + pop {r4-r5, pc} + +copy8 + pld [r0, r1, lsl #1] + vld1.8 {d0}, [r0], r1 + pld [r0, r1, lsl #1] + vld1.8 {d2}, [r0], r1 + vst1.8 {d0}, [r2@64], r3 + vst1.8 {d2}, [r2@64], r3 + subs r5, r5, #2 + bgt copy8 + pop {r4-r5, pc} + +copy4 + ldr r12, [r0], r1 + str r12, [r2], r3 + subs r5, r5, #1 + bgt copy4 + pop {r4-r5, pc} + ENDP + + END diff --git a/third_party/aom/aom_dsp/arm/aom_convolve_neon.c b/third_party/aom/aom_dsp/arm/aom_convolve_neon.c new file mode 100644 index 0000000000..6c2997e040 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/aom_convolve_neon.c @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" + +void aom_convolve8_neon(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, + int x_step_q4, const int16_t *filter_y, int y_step_q4, + int w, int h) { + /* Given our constraints: w <= 64, h <= 64, taps == 8 we can reduce the + * maximum buffer size to 64 * 64 + 7 (+ 1 to make it divisible by 4). + */ + DECLARE_ALIGNED(8, uint8_t, temp[64 * 72]); + + // Account for the vertical phase needing 3 lines prior and 4 lines post + int intermediate_height = h + 7; + + assert(y_step_q4 == 16); + assert(x_step_q4 == 16); + + /* Filter starting 3 lines back. The neon implementation will ignore the + * given height and filter a multiple of 4 lines. Since this goes in to + * the temp buffer which has lots of extra room and is subsequently discarded + * this is safe if somewhat less than ideal. + */ + aom_convolve8_horiz_neon(src - src_stride * 3, src_stride, temp, 64, filter_x, + x_step_q4, filter_y, y_step_q4, w, + intermediate_height); + + /* Step into the temp buffer 3 lines to get the actual frame data */ + aom_convolve8_vert_neon(temp + 64 * 3, 64, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); +} + +void aom_convolve8_avg_neon(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + DECLARE_ALIGNED(8, uint8_t, temp[64 * 72]); + int intermediate_height = h + 7; + + assert(y_step_q4 == 16); + assert(x_step_q4 == 16); + + /* This implementation has the same issues as above. In addition, we only want + * to average the values after both passes. + */ + aom_convolve8_horiz_neon(src - src_stride * 3, src_stride, temp, 64, filter_x, + x_step_q4, filter_y, y_step_q4, w, + intermediate_height); + aom_convolve8_avg_vert_neon(temp + 64 * 3, 64, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); +} diff --git a/third_party/aom/aom_dsp/arm/avg_neon.c b/third_party/aom/aom_dsp/arm/avg_neon.c new file mode 100644 index 0000000000..e730ccbccc --- /dev/null +++ b/third_party/aom/aom_dsp/arm/avg_neon.c @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "./aom_config.h" + +#include "aom/aom_integer.h" + +static INLINE unsigned int horizontal_add_u16x8(const uint16x8_t v_16x8) { + const uint32x4_t a = vpaddlq_u16(v_16x8); + const uint64x2_t b = vpaddlq_u32(a); + const uint32x2_t c = vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)), + vreinterpret_u32_u64(vget_high_u64(b))); + return vget_lane_u32(c, 0); +} + +unsigned int aom_avg_4x4_neon(const uint8_t *s, int p) { + uint16x8_t v_sum; + uint32x2_t v_s0 = vdup_n_u32(0); + uint32x2_t v_s1 = vdup_n_u32(0); + v_s0 = vld1_lane_u32((const uint32_t *)s, v_s0, 0); + v_s0 = vld1_lane_u32((const uint32_t *)(s + p), v_s0, 1); + v_s1 = vld1_lane_u32((const uint32_t *)(s + 2 * p), v_s1, 0); + v_s1 = vld1_lane_u32((const uint32_t *)(s + 3 * p), v_s1, 1); + v_sum = vaddl_u8(vreinterpret_u8_u32(v_s0), vreinterpret_u8_u32(v_s1)); + return (horizontal_add_u16x8(v_sum) + 8) >> 4; +} + +unsigned int aom_avg_8x8_neon(const uint8_t *s, int p) { + uint8x8_t v_s0 = vld1_u8(s); + const uint8x8_t v_s1 = vld1_u8(s + p); + uint16x8_t v_sum = vaddl_u8(v_s0, v_s1); + + v_s0 = vld1_u8(s + 2 * p); + v_sum = vaddw_u8(v_sum, v_s0); + + v_s0 = vld1_u8(s + 3 * p); + v_sum = vaddw_u8(v_sum, v_s0); + + v_s0 = vld1_u8(s + 4 * p); + v_sum = vaddw_u8(v_sum, v_s0); + + v_s0 = vld1_u8(s + 5 * p); + v_sum = vaddw_u8(v_sum, v_s0); + + v_s0 = vld1_u8(s + 6 * p); + v_sum = vaddw_u8(v_sum, v_s0); + + v_s0 = vld1_u8(s + 7 * p); + v_sum = vaddw_u8(v_sum, v_s0); + + return (horizontal_add_u16x8(v_sum) + 32) >> 6; +} + +// coeff: 16 bits, dynamic range [-32640, 32640]. +// length: value range {16, 64, 256, 1024}. +int aom_satd_neon(const int16_t *coeff, int length) { + const int16x4_t zero = vdup_n_s16(0); + int32x4_t accum = vdupq_n_s32(0); + + do { + const int16x8_t src0 = vld1q_s16(coeff); + const int16x8_t src8 = vld1q_s16(coeff + 8); + accum = vabal_s16(accum, vget_low_s16(src0), zero); + accum = vabal_s16(accum, vget_high_s16(src0), zero); + accum = vabal_s16(accum, vget_low_s16(src8), zero); + accum = vabal_s16(accum, vget_high_s16(src8), zero); + length -= 16; + coeff += 16; + } while (length != 0); + + { + // satd: 26 bits, dynamic range [-32640 * 1024, 32640 * 1024] + const int64x2_t s0 = vpaddlq_s32(accum); // cascading summation of 'accum'. + const int32x2_t s1 = vadd_s32(vreinterpret_s32_s64(vget_low_s64(s0)), + vreinterpret_s32_s64(vget_high_s64(s0))); + const int satd = vget_lane_s32(s1, 0); + return satd; + } +} + +void aom_int_pro_row_neon(int16_t hbuf[16], uint8_t const *ref, int ref_stride, + int height) { + int i; + uint16x8_t vec_sum_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_hi = vdupq_n_u16(0); + const int shift_factor = ((height >> 5) + 3) * -1; + const int16x8_t vec_shift = vdupq_n_s16(shift_factor); + + for (i = 0; i < height; i += 8) { + const uint8x16_t vec_row1 = vld1q_u8(ref); + const uint8x16_t vec_row2 = vld1q_u8(ref + ref_stride); + const uint8x16_t vec_row3 = vld1q_u8(ref + ref_stride * 2); + const uint8x16_t vec_row4 = vld1q_u8(ref + ref_stride * 3); + const uint8x16_t vec_row5 = vld1q_u8(ref + ref_stride * 4); + const uint8x16_t vec_row6 = vld1q_u8(ref + ref_stride * 5); + const uint8x16_t vec_row7 = vld1q_u8(ref + ref_stride * 6); + const uint8x16_t vec_row8 = vld1q_u8(ref + ref_stride * 7); + + vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row1)); + vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row1)); + + vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row2)); + vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row2)); + + vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row3)); + vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row3)); + + vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row4)); + vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row4)); + + vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row5)); + vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row5)); + + vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row6)); + vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row6)); + + vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row7)); + vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row7)); + + vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row8)); + vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row8)); + + ref += ref_stride * 8; + } + + vec_sum_lo = vshlq_u16(vec_sum_lo, vec_shift); + vec_sum_hi = vshlq_u16(vec_sum_hi, vec_shift); + + vst1q_s16(hbuf, vreinterpretq_s16_u16(vec_sum_lo)); + hbuf += 8; + vst1q_s16(hbuf, vreinterpretq_s16_u16(vec_sum_hi)); +} + +int16_t aom_int_pro_col_neon(uint8_t const *ref, const int width) { + int i; + uint16x8_t vec_sum = vdupq_n_u16(0); + + for (i = 0; i < width; i += 16) { + const uint8x16_t vec_row = vld1q_u8(ref); + vec_sum = vaddw_u8(vec_sum, vget_low_u8(vec_row)); + vec_sum = vaddw_u8(vec_sum, vget_high_u8(vec_row)); + ref += 16; + } + + return horizontal_add_u16x8(vec_sum); +} + +// ref, src = [0, 510] - max diff = 16-bits +// bwl = {2, 3, 4}, width = {16, 32, 64} +int aom_vector_var_neon(int16_t const *ref, int16_t const *src, int bwl) { + int width = 4 << bwl; + int32x4_t sse = vdupq_n_s32(0); + int16x8_t total = vdupq_n_s16(0); + + assert(width >= 8); + assert((width % 8) == 0); + + do { + const int16x8_t r = vld1q_s16(ref); + const int16x8_t s = vld1q_s16(src); + const int16x8_t diff = vsubq_s16(r, s); // [-510, 510], 10 bits. + const int16x4_t diff_lo = vget_low_s16(diff); + const int16x4_t diff_hi = vget_high_s16(diff); + sse = vmlal_s16(sse, diff_lo, diff_lo); // dynamic range 26 bits. + sse = vmlal_s16(sse, diff_hi, diff_hi); + total = vaddq_s16(total, diff); // dynamic range 16 bits. + + ref += 8; + src += 8; + width -= 8; + } while (width != 0); + + { + // Note: 'total''s pairwise addition could be implemented similarly to + // horizontal_add_u16x8(), but one less vpaddl with 'total' when paired + // with the summation of 'sse' performed better on a Cortex-A15. + const int32x4_t t0 = vpaddlq_s16(total); // cascading summation of 'total' + const int32x2_t t1 = vadd_s32(vget_low_s32(t0), vget_high_s32(t0)); + const int32x2_t t2 = vpadd_s32(t1, t1); + const int t = vget_lane_s32(t2, 0); + const int64x2_t s0 = vpaddlq_s32(sse); // cascading summation of 'sse'. + const int32x2_t s1 = vadd_s32(vreinterpret_s32_s64(vget_low_s64(s0)), + vreinterpret_s32_s64(vget_high_s64(s0))); + const int s = vget_lane_s32(s1, 0); + const int shift_factor = bwl + 2; + return s - ((t * t) >> shift_factor); + } +} + +void aom_minmax_8x8_neon(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int *min, int *max) { + // Load and concatenate. + const uint8x16_t a01 = vcombine_u8(vld1_u8(a), vld1_u8(a + a_stride)); + const uint8x16_t a23 = + vcombine_u8(vld1_u8(a + 2 * a_stride), vld1_u8(a + 3 * a_stride)); + const uint8x16_t a45 = + vcombine_u8(vld1_u8(a + 4 * a_stride), vld1_u8(a + 5 * a_stride)); + const uint8x16_t a67 = + vcombine_u8(vld1_u8(a + 6 * a_stride), vld1_u8(a + 7 * a_stride)); + + const uint8x16_t b01 = vcombine_u8(vld1_u8(b), vld1_u8(b + b_stride)); + const uint8x16_t b23 = + vcombine_u8(vld1_u8(b + 2 * b_stride), vld1_u8(b + 3 * b_stride)); + const uint8x16_t b45 = + vcombine_u8(vld1_u8(b + 4 * b_stride), vld1_u8(b + 5 * b_stride)); + const uint8x16_t b67 = + vcombine_u8(vld1_u8(b + 6 * b_stride), vld1_u8(b + 7 * b_stride)); + + // Absolute difference. + const uint8x16_t ab01_diff = vabdq_u8(a01, b01); + const uint8x16_t ab23_diff = vabdq_u8(a23, b23); + const uint8x16_t ab45_diff = vabdq_u8(a45, b45); + const uint8x16_t ab67_diff = vabdq_u8(a67, b67); + + // Max values between the Q vectors. + const uint8x16_t ab0123_max = vmaxq_u8(ab01_diff, ab23_diff); + const uint8x16_t ab4567_max = vmaxq_u8(ab45_diff, ab67_diff); + const uint8x16_t ab0123_min = vminq_u8(ab01_diff, ab23_diff); + const uint8x16_t ab4567_min = vminq_u8(ab45_diff, ab67_diff); + + const uint8x16_t ab07_max = vmaxq_u8(ab0123_max, ab4567_max); + const uint8x16_t ab07_min = vminq_u8(ab0123_min, ab4567_min); + + // Split to D and start doing pairwise. + uint8x8_t ab_max = vmax_u8(vget_high_u8(ab07_max), vget_low_u8(ab07_max)); + uint8x8_t ab_min = vmin_u8(vget_high_u8(ab07_min), vget_low_u8(ab07_min)); + + // Enough runs of vpmax/min propogate the max/min values to every position. + ab_max = vpmax_u8(ab_max, ab_max); + ab_min = vpmin_u8(ab_min, ab_min); + + ab_max = vpmax_u8(ab_max, ab_max); + ab_min = vpmin_u8(ab_min, ab_min); + + ab_max = vpmax_u8(ab_max, ab_max); + ab_min = vpmin_u8(ab_min, ab_min); + + *min = *max = 0; // Clear high bits + // Store directly to avoid costly neon->gpr transfer. + vst1_lane_u8((uint8_t *)max, ab_max, 0); + vst1_lane_u8((uint8_t *)min, ab_min, 0); +} diff --git a/third_party/aom/aom_dsp/arm/bilinear_filter_media.asm b/third_party/aom/aom_dsp/arm/bilinear_filter_media.asm new file mode 100644 index 0000000000..17b7d25f97 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/bilinear_filter_media.asm @@ -0,0 +1,240 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + + EXPORT |aom_filter_block2d_bil_first_pass_media| + EXPORT |aom_filter_block2d_bil_second_pass_media| + + AREA |.text|, CODE, READONLY ; name this block of code + +;------------------------------------- +; r0 unsigned char *src_ptr, +; r1 unsigned short *dst_ptr, +; r2 unsigned int src_pitch, +; r3 unsigned int height, +; stack unsigned int width, +; stack const short *aom_filter +;------------------------------------- +; The output is transposed stroed in output array to make it easy for second pass filtering. +|aom_filter_block2d_bil_first_pass_media| PROC + stmdb sp!, {r4 - r11, lr} + + ldr r11, [sp, #40] ; aom_filter address + ldr r4, [sp, #36] ; width + + mov r12, r3 ; outer-loop counter + + add r7, r2, r4 ; preload next row + pld [r0, r7] + + sub r2, r2, r4 ; src increment for height loop + + ldr r5, [r11] ; load up filter coefficients + + mov r3, r3, lsl #1 ; height*2 + add r3, r3, #2 ; plus 2 to make output buffer 4-bit aligned since height is actually (height+1) + + mov r11, r1 ; save dst_ptr for each row + + cmp r5, #128 ; if filter coef = 128, then skip the filter + beq bil_null_1st_filter + +|bil_height_loop_1st_v6| + ldrb r6, [r0] ; load source data + ldrb r7, [r0, #1] + ldrb r8, [r0, #2] + mov lr, r4, lsr #2 ; 4-in-parellel loop counter + +|bil_width_loop_1st_v6| + ldrb r9, [r0, #3] + ldrb r10, [r0, #4] + + pkhbt r6, r6, r7, lsl #16 ; src[1] | src[0] + pkhbt r7, r7, r8, lsl #16 ; src[2] | src[1] + + smuad r6, r6, r5 ; apply the filter + pkhbt r8, r8, r9, lsl #16 ; src[3] | src[2] + smuad r7, r7, r5 + pkhbt r9, r9, r10, lsl #16 ; src[4] | src[3] + + smuad r8, r8, r5 + smuad r9, r9, r5 + + add r0, r0, #4 + subs lr, lr, #1 + + add r6, r6, #0x40 ; round_shift_and_clamp + add r7, r7, #0x40 + usat r6, #16, r6, asr #7 + usat r7, #16, r7, asr #7 + + strh r6, [r1], r3 ; result is transposed and stored + + add r8, r8, #0x40 ; round_shift_and_clamp + strh r7, [r1], r3 + add r9, r9, #0x40 + usat r8, #16, r8, asr #7 + usat r9, #16, r9, asr #7 + + strh r8, [r1], r3 ; result is transposed and stored + + ldrneb r6, [r0] ; load source data + strh r9, [r1], r3 + + ldrneb r7, [r0, #1] + ldrneb r8, [r0, #2] + + bne bil_width_loop_1st_v6 + + add r0, r0, r2 ; move to next input row + subs r12, r12, #1 + + add r9, r2, r4, lsl #1 ; adding back block width + pld [r0, r9] ; preload next row + + add r11, r11, #2 ; move over to next column + mov r1, r11 + + bne bil_height_loop_1st_v6 + + ldmia sp!, {r4 - r11, pc} + +|bil_null_1st_filter| +|bil_height_loop_null_1st| + mov lr, r4, lsr #2 ; loop counter + +|bil_width_loop_null_1st| + ldrb r6, [r0] ; load data + ldrb r7, [r0, #1] + ldrb r8, [r0, #2] + ldrb r9, [r0, #3] + + strh r6, [r1], r3 ; store it to immediate buffer + add r0, r0, #4 + strh r7, [r1], r3 + subs lr, lr, #1 + strh r8, [r1], r3 + strh r9, [r1], r3 + + bne bil_width_loop_null_1st + + subs r12, r12, #1 + add r0, r0, r2 ; move to next input line + add r11, r11, #2 ; move over to next column + mov r1, r11 + + bne bil_height_loop_null_1st + + ldmia sp!, {r4 - r11, pc} + + ENDP ; |aom_filter_block2d_bil_first_pass_media| + + +;--------------------------------- +; r0 unsigned short *src_ptr, +; r1 unsigned char *dst_ptr, +; r2 int dst_pitch, +; r3 unsigned int height, +; stack unsigned int width, +; stack const short *aom_filter +;--------------------------------- +|aom_filter_block2d_bil_second_pass_media| PROC + stmdb sp!, {r4 - r11, lr} + + ldr r11, [sp, #40] ; aom_filter address + ldr r4, [sp, #36] ; width + + ldr r5, [r11] ; load up filter coefficients + mov r12, r4 ; outer-loop counter = width, since we work on transposed data matrix + mov r11, r1 + + cmp r5, #128 ; if filter coef = 128, then skip the filter + beq bil_null_2nd_filter + +|bil_height_loop_2nd| + ldr r6, [r0] ; load the data + ldr r8, [r0, #4] + ldrh r10, [r0, #8] + mov lr, r3, lsr #2 ; loop counter + +|bil_width_loop_2nd| + pkhtb r7, r6, r8 ; src[1] | src[2] + pkhtb r9, r8, r10 ; src[3] | src[4] + + smuad r6, r6, r5 ; apply filter + smuad r8, r8, r5 ; apply filter + + subs lr, lr, #1 + + smuadx r7, r7, r5 ; apply filter + smuadx r9, r9, r5 ; apply filter + + add r0, r0, #8 + + add r6, r6, #0x40 ; round_shift_and_clamp + add r7, r7, #0x40 + usat r6, #8, r6, asr #7 + usat r7, #8, r7, asr #7 + strb r6, [r1], r2 ; the result is transposed back and stored + + add r8, r8, #0x40 ; round_shift_and_clamp + strb r7, [r1], r2 + add r9, r9, #0x40 + usat r8, #8, r8, asr #7 + usat r9, #8, r9, asr #7 + strb r8, [r1], r2 ; the result is transposed back and stored + + ldrne r6, [r0] ; load data + strb r9, [r1], r2 + ldrne r8, [r0, #4] + ldrneh r10, [r0, #8] + + bne bil_width_loop_2nd + + subs r12, r12, #1 + add r0, r0, #4 ; update src for next row + add r11, r11, #1 + mov r1, r11 + + bne bil_height_loop_2nd + ldmia sp!, {r4 - r11, pc} + +|bil_null_2nd_filter| +|bil_height_loop_null_2nd| + mov lr, r3, lsr #2 + +|bil_width_loop_null_2nd| + ldr r6, [r0], #4 ; load data + subs lr, lr, #1 + ldr r8, [r0], #4 + + strb r6, [r1], r2 ; store data + mov r7, r6, lsr #16 + strb r7, [r1], r2 + mov r9, r8, lsr #16 + strb r8, [r1], r2 + strb r9, [r1], r2 + + bne bil_width_loop_null_2nd + + subs r12, r12, #1 + add r0, r0, #4 + add r11, r11, #1 + mov r1, r11 + + bne bil_height_loop_null_2nd + + ldmia sp!, {r4 - r11, pc} + ENDP ; |aom_filter_block2d_second_pass_media| + + END diff --git a/third_party/aom/aom_dsp/arm/fwd_txfm_neon.c b/third_party/aom/aom_dsp/arm/fwd_txfm_neon.c new file mode 100644 index 0000000000..1cf8a3a6ed --- /dev/null +++ b/third_party/aom/aom_dsp/arm/fwd_txfm_neon.c @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "aom_dsp/txfm_common.h" + +void aom_fdct8x8_neon(const int16_t *input, int16_t *final_output, int stride) { + int i; + // stage 1 + int16x8_t input_0 = vshlq_n_s16(vld1q_s16(&input[0 * stride]), 2); + int16x8_t input_1 = vshlq_n_s16(vld1q_s16(&input[1 * stride]), 2); + int16x8_t input_2 = vshlq_n_s16(vld1q_s16(&input[2 * stride]), 2); + int16x8_t input_3 = vshlq_n_s16(vld1q_s16(&input[3 * stride]), 2); + int16x8_t input_4 = vshlq_n_s16(vld1q_s16(&input[4 * stride]), 2); + int16x8_t input_5 = vshlq_n_s16(vld1q_s16(&input[5 * stride]), 2); + int16x8_t input_6 = vshlq_n_s16(vld1q_s16(&input[6 * stride]), 2); + int16x8_t input_7 = vshlq_n_s16(vld1q_s16(&input[7 * stride]), 2); + for (i = 0; i < 2; ++i) { + int16x8_t out_0, out_1, out_2, out_3, out_4, out_5, out_6, out_7; + const int16x8_t v_s0 = vaddq_s16(input_0, input_7); + const int16x8_t v_s1 = vaddq_s16(input_1, input_6); + const int16x8_t v_s2 = vaddq_s16(input_2, input_5); + const int16x8_t v_s3 = vaddq_s16(input_3, input_4); + const int16x8_t v_s4 = vsubq_s16(input_3, input_4); + const int16x8_t v_s5 = vsubq_s16(input_2, input_5); + const int16x8_t v_s6 = vsubq_s16(input_1, input_6); + const int16x8_t v_s7 = vsubq_s16(input_0, input_7); + // fdct4(step, step); + int16x8_t v_x0 = vaddq_s16(v_s0, v_s3); + int16x8_t v_x1 = vaddq_s16(v_s1, v_s2); + int16x8_t v_x2 = vsubq_s16(v_s1, v_s2); + int16x8_t v_x3 = vsubq_s16(v_s0, v_s3); + // fdct4(step, step); + int32x4_t v_t0_lo = vaddl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1)); + int32x4_t v_t0_hi = vaddl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1)); + int32x4_t v_t1_lo = vsubl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1)); + int32x4_t v_t1_hi = vsubl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1)); + int32x4_t v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), (int16_t)cospi_24_64); + int32x4_t v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), (int16_t)cospi_24_64); + int32x4_t v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), (int16_t)cospi_24_64); + int32x4_t v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), (int16_t)cospi_24_64); + v_t2_lo = vmlal_n_s16(v_t2_lo, vget_low_s16(v_x3), (int16_t)cospi_8_64); + v_t2_hi = vmlal_n_s16(v_t2_hi, vget_high_s16(v_x3), (int16_t)cospi_8_64); + v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x2), (int16_t)cospi_8_64); + v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x2), (int16_t)cospi_8_64); + v_t0_lo = vmulq_n_s32(v_t0_lo, (int32_t)cospi_16_64); + v_t0_hi = vmulq_n_s32(v_t0_hi, (int32_t)cospi_16_64); + v_t1_lo = vmulq_n_s32(v_t1_lo, (int32_t)cospi_16_64); + v_t1_hi = vmulq_n_s32(v_t1_hi, (int32_t)cospi_16_64); + { + const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); + const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); + const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); + const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); + const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS); + const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS); + const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS); + const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS); + out_0 = vcombine_s16(a, c); // 00 01 02 03 40 41 42 43 + out_2 = vcombine_s16(e, g); // 20 21 22 23 60 61 62 63 + out_4 = vcombine_s16(b, d); // 04 05 06 07 44 45 46 47 + out_6 = vcombine_s16(f, h); // 24 25 26 27 64 65 66 67 + } + // Stage 2 + v_x0 = vsubq_s16(v_s6, v_s5); + v_x1 = vaddq_s16(v_s6, v_s5); + v_t0_lo = vmull_n_s16(vget_low_s16(v_x0), (int16_t)cospi_16_64); + v_t0_hi = vmull_n_s16(vget_high_s16(v_x0), (int16_t)cospi_16_64); + v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), (int16_t)cospi_16_64); + v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), (int16_t)cospi_16_64); + { + const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); + const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); + const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); + const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); + const int16x8_t ab = vcombine_s16(a, b); + const int16x8_t cd = vcombine_s16(c, d); + // Stage 3 + v_x0 = vaddq_s16(v_s4, ab); + v_x1 = vsubq_s16(v_s4, ab); + v_x2 = vsubq_s16(v_s7, cd); + v_x3 = vaddq_s16(v_s7, cd); + } + // Stage 4 + v_t0_lo = vmull_n_s16(vget_low_s16(v_x3), (int16_t)cospi_4_64); + v_t0_hi = vmull_n_s16(vget_high_s16(v_x3), (int16_t)cospi_4_64); + v_t0_lo = vmlal_n_s16(v_t0_lo, vget_low_s16(v_x0), (int16_t)cospi_28_64); + v_t0_hi = vmlal_n_s16(v_t0_hi, vget_high_s16(v_x0), (int16_t)cospi_28_64); + v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), (int16_t)cospi_12_64); + v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), (int16_t)cospi_12_64); + v_t1_lo = vmlal_n_s16(v_t1_lo, vget_low_s16(v_x2), (int16_t)cospi_20_64); + v_t1_hi = vmlal_n_s16(v_t1_hi, vget_high_s16(v_x2), (int16_t)cospi_20_64); + v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), (int16_t)cospi_12_64); + v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), (int16_t)cospi_12_64); + v_t2_lo = vmlsl_n_s16(v_t2_lo, vget_low_s16(v_x1), (int16_t)cospi_20_64); + v_t2_hi = vmlsl_n_s16(v_t2_hi, vget_high_s16(v_x1), (int16_t)cospi_20_64); + v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), (int16_t)cospi_28_64); + v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), (int16_t)cospi_28_64); + v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x0), (int16_t)cospi_4_64); + v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x0), (int16_t)cospi_4_64); + { + const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); + const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); + const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); + const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); + const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS); + const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS); + const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS); + const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS); + out_1 = vcombine_s16(a, c); // 10 11 12 13 50 51 52 53 + out_3 = vcombine_s16(e, g); // 30 31 32 33 70 71 72 73 + out_5 = vcombine_s16(b, d); // 14 15 16 17 54 55 56 57 + out_7 = vcombine_s16(f, h); // 34 35 36 37 74 75 76 77 + } + // transpose 8x8 + { + // 00 01 02 03 40 41 42 43 + // 10 11 12 13 50 51 52 53 + // 20 21 22 23 60 61 62 63 + // 30 31 32 33 70 71 72 73 + // 04 05 06 07 44 45 46 47 + // 14 15 16 17 54 55 56 57 + // 24 25 26 27 64 65 66 67 + // 34 35 36 37 74 75 76 77 + const int32x4x2_t r02_s32 = + vtrnq_s32(vreinterpretq_s32_s16(out_0), vreinterpretq_s32_s16(out_2)); + const int32x4x2_t r13_s32 = + vtrnq_s32(vreinterpretq_s32_s16(out_1), vreinterpretq_s32_s16(out_3)); + const int32x4x2_t r46_s32 = + vtrnq_s32(vreinterpretq_s32_s16(out_4), vreinterpretq_s32_s16(out_6)); + const int32x4x2_t r57_s32 = + vtrnq_s32(vreinterpretq_s32_s16(out_5), vreinterpretq_s32_s16(out_7)); + const int16x8x2_t r01_s16 = + vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[0]), + vreinterpretq_s16_s32(r13_s32.val[0])); + const int16x8x2_t r23_s16 = + vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[1]), + vreinterpretq_s16_s32(r13_s32.val[1])); + const int16x8x2_t r45_s16 = + vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[0]), + vreinterpretq_s16_s32(r57_s32.val[0])); + const int16x8x2_t r67_s16 = + vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[1]), + vreinterpretq_s16_s32(r57_s32.val[1])); + input_0 = r01_s16.val[0]; + input_1 = r01_s16.val[1]; + input_2 = r23_s16.val[0]; + input_3 = r23_s16.val[1]; + input_4 = r45_s16.val[0]; + input_5 = r45_s16.val[1]; + input_6 = r67_s16.val[0]; + input_7 = r67_s16.val[1]; + // 00 10 20 30 40 50 60 70 + // 01 11 21 31 41 51 61 71 + // 02 12 22 32 42 52 62 72 + // 03 13 23 33 43 53 63 73 + // 04 14 24 34 44 54 64 74 + // 05 15 25 35 45 55 65 75 + // 06 16 26 36 46 56 66 76 + // 07 17 27 37 47 57 67 77 + } + } // for + { + // from aom_dct_sse2.c + // Post-condition (division by two) + // division of two 16 bits signed numbers using shifts + // n / 2 = (n - (n >> 15)) >> 1 + const int16x8_t sign_in0 = vshrq_n_s16(input_0, 15); + const int16x8_t sign_in1 = vshrq_n_s16(input_1, 15); + const int16x8_t sign_in2 = vshrq_n_s16(input_2, 15); + const int16x8_t sign_in3 = vshrq_n_s16(input_3, 15); + const int16x8_t sign_in4 = vshrq_n_s16(input_4, 15); + const int16x8_t sign_in5 = vshrq_n_s16(input_5, 15); + const int16x8_t sign_in6 = vshrq_n_s16(input_6, 15); + const int16x8_t sign_in7 = vshrq_n_s16(input_7, 15); + input_0 = vhsubq_s16(input_0, sign_in0); + input_1 = vhsubq_s16(input_1, sign_in1); + input_2 = vhsubq_s16(input_2, sign_in2); + input_3 = vhsubq_s16(input_3, sign_in3); + input_4 = vhsubq_s16(input_4, sign_in4); + input_5 = vhsubq_s16(input_5, sign_in5); + input_6 = vhsubq_s16(input_6, sign_in6); + input_7 = vhsubq_s16(input_7, sign_in7); + // store results + vst1q_s16(&final_output[0 * 8], input_0); + vst1q_s16(&final_output[1 * 8], input_1); + vst1q_s16(&final_output[2 * 8], input_2); + vst1q_s16(&final_output[3 * 8], input_3); + vst1q_s16(&final_output[4 * 8], input_4); + vst1q_s16(&final_output[5 * 8], input_5); + vst1q_s16(&final_output[6 * 8], input_6); + vst1q_s16(&final_output[7 * 8], input_7); + } +} + +void aom_fdct8x8_1_neon(const int16_t *input, int16_t *output, int stride) { + int r; + int16x8_t sum = vld1q_s16(&input[0]); + for (r = 1; r < 8; ++r) { + const int16x8_t input_00 = vld1q_s16(&input[r * stride]); + sum = vaddq_s16(sum, input_00); + } + { + const int32x4_t a = vpaddlq_s16(sum); + const int64x2_t b = vpaddlq_s32(a); + const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)), + vreinterpret_s32_s64(vget_high_s64(b))); + output[0] = vget_lane_s16(vreinterpret_s16_s32(c), 0); + output[1] = 0; + } +} diff --git a/third_party/aom/aom_dsp/arm/hadamard_neon.c b/third_party/aom/aom_dsp/arm/hadamard_neon.c new file mode 100644 index 0000000000..9baefae47f --- /dev/null +++ b/third_party/aom/aom_dsp/arm/hadamard_neon.c @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" + +static void hadamard8x8_one_pass(int16x8_t *a0, int16x8_t *a1, int16x8_t *a2, + int16x8_t *a3, int16x8_t *a4, int16x8_t *a5, + int16x8_t *a6, int16x8_t *a7) { + const int16x8_t b0 = vaddq_s16(*a0, *a1); + const int16x8_t b1 = vsubq_s16(*a0, *a1); + const int16x8_t b2 = vaddq_s16(*a2, *a3); + const int16x8_t b3 = vsubq_s16(*a2, *a3); + const int16x8_t b4 = vaddq_s16(*a4, *a5); + const int16x8_t b5 = vsubq_s16(*a4, *a5); + const int16x8_t b6 = vaddq_s16(*a6, *a7); + const int16x8_t b7 = vsubq_s16(*a6, *a7); + + const int16x8_t c0 = vaddq_s16(b0, b2); + const int16x8_t c1 = vaddq_s16(b1, b3); + const int16x8_t c2 = vsubq_s16(b0, b2); + const int16x8_t c3 = vsubq_s16(b1, b3); + const int16x8_t c4 = vaddq_s16(b4, b6); + const int16x8_t c5 = vaddq_s16(b5, b7); + const int16x8_t c6 = vsubq_s16(b4, b6); + const int16x8_t c7 = vsubq_s16(b5, b7); + + *a0 = vaddq_s16(c0, c4); + *a1 = vsubq_s16(c2, c6); + *a2 = vsubq_s16(c0, c4); + *a3 = vaddq_s16(c2, c6); + *a4 = vaddq_s16(c3, c7); + *a5 = vsubq_s16(c3, c7); + *a6 = vsubq_s16(c1, c5); + *a7 = vaddq_s16(c1, c5); +} + +// TODO(johannkoenig): Make a transpose library and dedup with idct. Consider +// reversing transpose order which may make it easier for the compiler to +// reconcile the vtrn.64 moves. +static void transpose8x8(int16x8_t *a0, int16x8_t *a1, int16x8_t *a2, + int16x8_t *a3, int16x8_t *a4, int16x8_t *a5, + int16x8_t *a6, int16x8_t *a7) { + // Swap 64 bit elements. Goes from: + // a0: 00 01 02 03 04 05 06 07 + // a1: 08 09 10 11 12 13 14 15 + // a2: 16 17 18 19 20 21 22 23 + // a3: 24 25 26 27 28 29 30 31 + // a4: 32 33 34 35 36 37 38 39 + // a5: 40 41 42 43 44 45 46 47 + // a6: 48 49 50 51 52 53 54 55 + // a7: 56 57 58 59 60 61 62 63 + // to: + // a04_lo: 00 01 02 03 32 33 34 35 + // a15_lo: 08 09 10 11 40 41 42 43 + // a26_lo: 16 17 18 19 48 49 50 51 + // a37_lo: 24 25 26 27 56 57 58 59 + // a04_hi: 04 05 06 07 36 37 38 39 + // a15_hi: 12 13 14 15 44 45 46 47 + // a26_hi: 20 21 22 23 52 53 54 55 + // a37_hi: 28 29 30 31 60 61 62 63 + const int16x8_t a04_lo = vcombine_s16(vget_low_s16(*a0), vget_low_s16(*a4)); + const int16x8_t a15_lo = vcombine_s16(vget_low_s16(*a1), vget_low_s16(*a5)); + const int16x8_t a26_lo = vcombine_s16(vget_low_s16(*a2), vget_low_s16(*a6)); + const int16x8_t a37_lo = vcombine_s16(vget_low_s16(*a3), vget_low_s16(*a7)); + const int16x8_t a04_hi = vcombine_s16(vget_high_s16(*a0), vget_high_s16(*a4)); + const int16x8_t a15_hi = vcombine_s16(vget_high_s16(*a1), vget_high_s16(*a5)); + const int16x8_t a26_hi = vcombine_s16(vget_high_s16(*a2), vget_high_s16(*a6)); + const int16x8_t a37_hi = vcombine_s16(vget_high_s16(*a3), vget_high_s16(*a7)); + + // Swap 32 bit elements resulting in: + // a0246_lo: + // 00 01 16 17 32 33 48 49 + // 02 03 18 19 34 35 50 51 + // a1357_lo: + // 08 09 24 25 40 41 56 57 + // 10 11 26 27 42 43 58 59 + // a0246_hi: + // 04 05 20 21 36 37 52 53 + // 06 07 22 23 38 39 54 55 + // a1657_hi: + // 12 13 28 29 44 45 60 61 + // 14 15 30 31 46 47 62 63 + const int32x4x2_t a0246_lo = + vtrnq_s32(vreinterpretq_s32_s16(a04_lo), vreinterpretq_s32_s16(a26_lo)); + const int32x4x2_t a1357_lo = + vtrnq_s32(vreinterpretq_s32_s16(a15_lo), vreinterpretq_s32_s16(a37_lo)); + const int32x4x2_t a0246_hi = + vtrnq_s32(vreinterpretq_s32_s16(a04_hi), vreinterpretq_s32_s16(a26_hi)); + const int32x4x2_t a1357_hi = + vtrnq_s32(vreinterpretq_s32_s16(a15_hi), vreinterpretq_s32_s16(a37_hi)); + + // Swap 16 bit elements resulting in: + // b0: + // 00 08 16 24 32 40 48 56 + // 01 09 17 25 33 41 49 57 + // b1: + // 02 10 18 26 34 42 50 58 + // 03 11 19 27 35 43 51 59 + // b2: + // 04 12 20 28 36 44 52 60 + // 05 13 21 29 37 45 53 61 + // b3: + // 06 14 22 30 38 46 54 62 + // 07 15 23 31 39 47 55 63 + const int16x8x2_t b0 = vtrnq_s16(vreinterpretq_s16_s32(a0246_lo.val[0]), + vreinterpretq_s16_s32(a1357_lo.val[0])); + const int16x8x2_t b1 = vtrnq_s16(vreinterpretq_s16_s32(a0246_lo.val[1]), + vreinterpretq_s16_s32(a1357_lo.val[1])); + const int16x8x2_t b2 = vtrnq_s16(vreinterpretq_s16_s32(a0246_hi.val[0]), + vreinterpretq_s16_s32(a1357_hi.val[0])); + const int16x8x2_t b3 = vtrnq_s16(vreinterpretq_s16_s32(a0246_hi.val[1]), + vreinterpretq_s16_s32(a1357_hi.val[1])); + + *a0 = b0.val[0]; + *a1 = b0.val[1]; + *a2 = b1.val[0]; + *a3 = b1.val[1]; + *a4 = b2.val[0]; + *a5 = b2.val[1]; + *a6 = b3.val[0]; + *a7 = b3.val[1]; +} + +void aom_hadamard_8x8_neon(const int16_t *src_diff, int src_stride, + int16_t *coeff) { + int16x8_t a0 = vld1q_s16(src_diff); + int16x8_t a1 = vld1q_s16(src_diff + src_stride); + int16x8_t a2 = vld1q_s16(src_diff + 2 * src_stride); + int16x8_t a3 = vld1q_s16(src_diff + 3 * src_stride); + int16x8_t a4 = vld1q_s16(src_diff + 4 * src_stride); + int16x8_t a5 = vld1q_s16(src_diff + 5 * src_stride); + int16x8_t a6 = vld1q_s16(src_diff + 6 * src_stride); + int16x8_t a7 = vld1q_s16(src_diff + 7 * src_stride); + + hadamard8x8_one_pass(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7); + + transpose8x8(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7); + + hadamard8x8_one_pass(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7); + + // Skip the second transpose because it is not required. + + vst1q_s16(coeff + 0, a0); + vst1q_s16(coeff + 8, a1); + vst1q_s16(coeff + 16, a2); + vst1q_s16(coeff + 24, a3); + vst1q_s16(coeff + 32, a4); + vst1q_s16(coeff + 40, a5); + vst1q_s16(coeff + 48, a6); + vst1q_s16(coeff + 56, a7); +} + +void aom_hadamard_16x16_neon(const int16_t *src_diff, int src_stride, + int16_t *coeff) { + int i; + + /* Rearrange 16x16 to 8x32 and remove stride. + * Top left first. */ + aom_hadamard_8x8_neon(src_diff + 0 + 0 * src_stride, src_stride, coeff + 0); + /* Top right. */ + aom_hadamard_8x8_neon(src_diff + 8 + 0 * src_stride, src_stride, coeff + 64); + /* Bottom left. */ + aom_hadamard_8x8_neon(src_diff + 0 + 8 * src_stride, src_stride, coeff + 128); + /* Bottom right. */ + aom_hadamard_8x8_neon(src_diff + 8 + 8 * src_stride, src_stride, coeff + 192); + + for (i = 0; i < 64; i += 8) { + const int16x8_t a0 = vld1q_s16(coeff + 0); + const int16x8_t a1 = vld1q_s16(coeff + 64); + const int16x8_t a2 = vld1q_s16(coeff + 128); + const int16x8_t a3 = vld1q_s16(coeff + 192); + + const int16x8_t b0 = vhaddq_s16(a0, a1); + const int16x8_t b1 = vhsubq_s16(a0, a1); + const int16x8_t b2 = vhaddq_s16(a2, a3); + const int16x8_t b3 = vhsubq_s16(a2, a3); + + const int16x8_t c0 = vaddq_s16(b0, b2); + const int16x8_t c1 = vaddq_s16(b1, b3); + const int16x8_t c2 = vsubq_s16(b0, b2); + const int16x8_t c3 = vsubq_s16(b1, b3); + + vst1q_s16(coeff + 0, c0); + vst1q_s16(coeff + 64, c1); + vst1q_s16(coeff + 128, c2); + vst1q_s16(coeff + 192, c3); + + coeff += 8; + } +} diff --git a/third_party/aom/aom_dsp/arm/idct16x16_1_add_neon.asm b/third_party/aom/aom_dsp/arm/idct16x16_1_add_neon.asm new file mode 100644 index 0000000000..d01c4bc03e --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct16x16_1_add_neon.asm @@ -0,0 +1,201 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + + + + EXPORT |aom_idct16x16_1_add_neon| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +;void aom_idct16x16_1_add_neon(int16_t *input, uint8_t *dest, +; int dest_stride) +; +; r0 int16_t input +; r1 uint8_t *dest +; r2 int dest_stride) + +|aom_idct16x16_1_add_neon| PROC + ldrsh r0, [r0] + + ; generate cospi_16_64 = 11585 + mov r12, #0x2d00 + add r12, #0x41 + + ; out = dct_const_round_shift(input[0] * cospi_16_64) + mul r0, r0, r12 ; input[0] * cospi_16_64 + add r0, r0, #0x2000 ; +(1 << ((DCT_CONST_BITS) - 1)) + asr r0, r0, #14 ; >> DCT_CONST_BITS + + ; out = dct_const_round_shift(out * cospi_16_64) + mul r0, r0, r12 ; out * cospi_16_64 + mov r12, r1 ; save dest + add r0, r0, #0x2000 ; +(1 << ((DCT_CONST_BITS) - 1)) + asr r0, r0, #14 ; >> DCT_CONST_BITS + + ; a1 = ROUND_POWER_OF_TWO(out, 6) + add r0, r0, #32 ; + (1 <<((6) - 1)) + asr r0, r0, #6 ; >> 6 + + vdup.s16 q0, r0 ; duplicate a1 + mov r0, #8 + sub r2, #8 + + ; load destination data row0 - row3 + vld1.64 {d2}, [r1], r0 + vld1.64 {d3}, [r1], r2 + vld1.64 {d4}, [r1], r0 + vld1.64 {d5}, [r1], r2 + vld1.64 {d6}, [r1], r0 + vld1.64 {d7}, [r1], r2 + vld1.64 {d16}, [r1], r0 + vld1.64 {d17}, [r1], r2 + + vaddw.u8 q9, q0, d2 ; dest[x] + a1 + vaddw.u8 q10, q0, d3 ; dest[x] + a1 + vaddw.u8 q11, q0, d4 ; dest[x] + a1 + vaddw.u8 q12, q0, d5 ; dest[x] + a1 + vqmovun.s16 d2, q9 ; clip_pixel + vqmovun.s16 d3, q10 ; clip_pixel + vqmovun.s16 d30, q11 ; clip_pixel + vqmovun.s16 d31, q12 ; clip_pixel + vst1.64 {d2}, [r12], r0 + vst1.64 {d3}, [r12], r2 + vst1.64 {d30}, [r12], r0 + vst1.64 {d31}, [r12], r2 + + vaddw.u8 q9, q0, d6 ; dest[x] + a1 + vaddw.u8 q10, q0, d7 ; dest[x] + a1 + vaddw.u8 q11, q0, d16 ; dest[x] + a1 + vaddw.u8 q12, q0, d17 ; dest[x] + a1 + vqmovun.s16 d2, q9 ; clip_pixel + vqmovun.s16 d3, q10 ; clip_pixel + vqmovun.s16 d30, q11 ; clip_pixel + vqmovun.s16 d31, q12 ; clip_pixel + vst1.64 {d2}, [r12], r0 + vst1.64 {d3}, [r12], r2 + vst1.64 {d30}, [r12], r0 + vst1.64 {d31}, [r12], r2 + + ; load destination data row4 - row7 + vld1.64 {d2}, [r1], r0 + vld1.64 {d3}, [r1], r2 + vld1.64 {d4}, [r1], r0 + vld1.64 {d5}, [r1], r2 + vld1.64 {d6}, [r1], r0 + vld1.64 {d7}, [r1], r2 + vld1.64 {d16}, [r1], r0 + vld1.64 {d17}, [r1], r2 + + vaddw.u8 q9, q0, d2 ; dest[x] + a1 + vaddw.u8 q10, q0, d3 ; dest[x] + a1 + vaddw.u8 q11, q0, d4 ; dest[x] + a1 + vaddw.u8 q12, q0, d5 ; dest[x] + a1 + vqmovun.s16 d2, q9 ; clip_pixel + vqmovun.s16 d3, q10 ; clip_pixel + vqmovun.s16 d30, q11 ; clip_pixel + vqmovun.s16 d31, q12 ; clip_pixel + vst1.64 {d2}, [r12], r0 + vst1.64 {d3}, [r12], r2 + vst1.64 {d30}, [r12], r0 + vst1.64 {d31}, [r12], r2 + + vaddw.u8 q9, q0, d6 ; dest[x] + a1 + vaddw.u8 q10, q0, d7 ; dest[x] + a1 + vaddw.u8 q11, q0, d16 ; dest[x] + a1 + vaddw.u8 q12, q0, d17 ; dest[x] + a1 + vqmovun.s16 d2, q9 ; clip_pixel + vqmovun.s16 d3, q10 ; clip_pixel + vqmovun.s16 d30, q11 ; clip_pixel + vqmovun.s16 d31, q12 ; clip_pixel + vst1.64 {d2}, [r12], r0 + vst1.64 {d3}, [r12], r2 + vst1.64 {d30}, [r12], r0 + vst1.64 {d31}, [r12], r2 + + ; load destination data row8 - row11 + vld1.64 {d2}, [r1], r0 + vld1.64 {d3}, [r1], r2 + vld1.64 {d4}, [r1], r0 + vld1.64 {d5}, [r1], r2 + vld1.64 {d6}, [r1], r0 + vld1.64 {d7}, [r1], r2 + vld1.64 {d16}, [r1], r0 + vld1.64 {d17}, [r1], r2 + + vaddw.u8 q9, q0, d2 ; dest[x] + a1 + vaddw.u8 q10, q0, d3 ; dest[x] + a1 + vaddw.u8 q11, q0, d4 ; dest[x] + a1 + vaddw.u8 q12, q0, d5 ; dest[x] + a1 + vqmovun.s16 d2, q9 ; clip_pixel + vqmovun.s16 d3, q10 ; clip_pixel + vqmovun.s16 d30, q11 ; clip_pixel + vqmovun.s16 d31, q12 ; clip_pixel + vst1.64 {d2}, [r12], r0 + vst1.64 {d3}, [r12], r2 + vst1.64 {d30}, [r12], r0 + vst1.64 {d31}, [r12], r2 + + vaddw.u8 q9, q0, d6 ; dest[x] + a1 + vaddw.u8 q10, q0, d7 ; dest[x] + a1 + vaddw.u8 q11, q0, d16 ; dest[x] + a1 + vaddw.u8 q12, q0, d17 ; dest[x] + a1 + vqmovun.s16 d2, q9 ; clip_pixel + vqmovun.s16 d3, q10 ; clip_pixel + vqmovun.s16 d30, q11 ; clip_pixel + vqmovun.s16 d31, q12 ; clip_pixel + vst1.64 {d2}, [r12], r0 + vst1.64 {d3}, [r12], r2 + vst1.64 {d30}, [r12], r0 + vst1.64 {d31}, [r12], r2 + + ; load destination data row12 - row15 + vld1.64 {d2}, [r1], r0 + vld1.64 {d3}, [r1], r2 + vld1.64 {d4}, [r1], r0 + vld1.64 {d5}, [r1], r2 + vld1.64 {d6}, [r1], r0 + vld1.64 {d7}, [r1], r2 + vld1.64 {d16}, [r1], r0 + vld1.64 {d17}, [r1], r2 + + vaddw.u8 q9, q0, d2 ; dest[x] + a1 + vaddw.u8 q10, q0, d3 ; dest[x] + a1 + vaddw.u8 q11, q0, d4 ; dest[x] + a1 + vaddw.u8 q12, q0, d5 ; dest[x] + a1 + vqmovun.s16 d2, q9 ; clip_pixel + vqmovun.s16 d3, q10 ; clip_pixel + vqmovun.s16 d30, q11 ; clip_pixel + vqmovun.s16 d31, q12 ; clip_pixel + vst1.64 {d2}, [r12], r0 + vst1.64 {d3}, [r12], r2 + vst1.64 {d30}, [r12], r0 + vst1.64 {d31}, [r12], r2 + + vaddw.u8 q9, q0, d6 ; dest[x] + a1 + vaddw.u8 q10, q0, d7 ; dest[x] + a1 + vaddw.u8 q11, q0, d16 ; dest[x] + a1 + vaddw.u8 q12, q0, d17 ; dest[x] + a1 + vqmovun.s16 d2, q9 ; clip_pixel + vqmovun.s16 d3, q10 ; clip_pixel + vqmovun.s16 d30, q11 ; clip_pixel + vqmovun.s16 d31, q12 ; clip_pixel + vst1.64 {d2}, [r12], r0 + vst1.64 {d3}, [r12], r2 + vst1.64 {d30}, [r12], r0 + vst1.64 {d31}, [r12], r2 + + bx lr + ENDP ; |aom_idct16x16_1_add_neon| + + END diff --git a/third_party/aom/aom_dsp/arm/idct16x16_1_add_neon.c b/third_party/aom/aom_dsp/arm/idct16x16_1_add_neon.c new file mode 100644 index 0000000000..196b2a890d --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct16x16_1_add_neon.c @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom_dsp/inv_txfm.h" +#include "aom_ports/mem.h" + +void aom_idct16x16_1_add_neon(int16_t *input, uint8_t *dest, int dest_stride) { + uint8x8_t d2u8, d3u8, d30u8, d31u8; + uint64x1_t d2u64, d3u64, d4u64, d5u64; + uint16x8_t q0u16, q9u16, q10u16, q11u16, q12u16; + int16x8_t q0s16; + uint8_t *d1, *d2; + int16_t i, j, a1; + int16_t out = dct_const_round_shift(input[0] * cospi_16_64); + out = dct_const_round_shift(out * cospi_16_64); + a1 = ROUND_POWER_OF_TWO(out, 6); + + q0s16 = vdupq_n_s16(a1); + q0u16 = vreinterpretq_u16_s16(q0s16); + + for (d1 = d2 = dest, i = 0; i < 4; i++) { + for (j = 0; j < 2; j++) { + d2u64 = vld1_u64((const uint64_t *)d1); + d3u64 = vld1_u64((const uint64_t *)(d1 + 8)); + d1 += dest_stride; + d4u64 = vld1_u64((const uint64_t *)d1); + d5u64 = vld1_u64((const uint64_t *)(d1 + 8)); + d1 += dest_stride; + + q9u16 = vaddw_u8(q0u16, vreinterpret_u8_u64(d2u64)); + q10u16 = vaddw_u8(q0u16, vreinterpret_u8_u64(d3u64)); + q11u16 = vaddw_u8(q0u16, vreinterpret_u8_u64(d4u64)); + q12u16 = vaddw_u8(q0u16, vreinterpret_u8_u64(d5u64)); + + d2u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16)); + d3u8 = vqmovun_s16(vreinterpretq_s16_u16(q10u16)); + d30u8 = vqmovun_s16(vreinterpretq_s16_u16(q11u16)); + d31u8 = vqmovun_s16(vreinterpretq_s16_u16(q12u16)); + + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d2u8)); + vst1_u64((uint64_t *)(d2 + 8), vreinterpret_u64_u8(d3u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d30u8)); + vst1_u64((uint64_t *)(d2 + 8), vreinterpret_u64_u8(d31u8)); + d2 += dest_stride; + } + } + return; +} diff --git a/third_party/aom/aom_dsp/arm/idct16x16_add_neon.asm b/third_party/aom/aom_dsp/arm/idct16x16_add_neon.asm new file mode 100644 index 0000000000..4a8f8f183a --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct16x16_add_neon.asm @@ -0,0 +1,1182 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + EXPORT |aom_idct16x16_256_add_neon_pass1| + EXPORT |aom_idct16x16_256_add_neon_pass2| + EXPORT |aom_idct16x16_10_add_neon_pass1| + EXPORT |aom_idct16x16_10_add_neon_pass2| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + + ; Transpose a 8x8 16bit data matrix. Datas are loaded in q8-q15. + MACRO + TRANSPOSE8X8 + vswp d17, d24 + vswp d23, d30 + vswp d21, d28 + vswp d19, d26 + vtrn.32 q8, q10 + vtrn.32 q9, q11 + vtrn.32 q12, q14 + vtrn.32 q13, q15 + vtrn.16 q8, q9 + vtrn.16 q10, q11 + vtrn.16 q12, q13 + vtrn.16 q14, q15 + MEND + + AREA Block, CODE, READONLY ; name this block of code +;void |aom_idct16x16_256_add_neon_pass1|(int16_t *input, +; int16_t *output, int output_stride) +; +; r0 int16_t input +; r1 int16_t *output +; r2 int output_stride) + +; idct16 stage1 - stage6 on all the elements loaded in q8-q15. The output +; will be stored back into q8-q15 registers. This function will touch q0-q7 +; registers and use them as buffer during calculation. +|aom_idct16x16_256_add_neon_pass1| PROC + + ; TODO(hkuang): Find a better way to load the elements. + ; load elements of 0, 2, 4, 6, 8, 10, 12, 14 into q8 - q15 + vld2.s16 {q8,q9}, [r0]! + vld2.s16 {q9,q10}, [r0]! + vld2.s16 {q10,q11}, [r0]! + vld2.s16 {q11,q12}, [r0]! + vld2.s16 {q12,q13}, [r0]! + vld2.s16 {q13,q14}, [r0]! + vld2.s16 {q14,q15}, [r0]! + vld2.s16 {q1,q2}, [r0]! + vmov.s16 q15, q1 + + ; generate cospi_28_64 = 3196 + mov r3, #0xc00 + add r3, #0x7c + + ; generate cospi_4_64 = 16069 + mov r12, #0x3e00 + add r12, #0xc5 + + ; transpose the input data + TRANSPOSE8X8 + + ; stage 3 + vdup.16 d0, r3 ; duplicate cospi_28_64 + vdup.16 d1, r12 ; duplicate cospi_4_64 + + ; preloading to avoid stall + ; generate cospi_12_64 = 13623 + mov r3, #0x3500 + add r3, #0x37 + + ; generate cospi_20_64 = 9102 + mov r12, #0x2300 + add r12, #0x8e + + ; step2[4] * cospi_28_64 + vmull.s16 q2, d18, d0 + vmull.s16 q3, d19, d0 + + ; step2[4] * cospi_4_64 + vmull.s16 q5, d18, d1 + vmull.s16 q6, d19, d1 + + ; temp1 = step2[4] * cospi_28_64 - step2[7] * cospi_4_64 + vmlsl.s16 q2, d30, d1 + vmlsl.s16 q3, d31, d1 + + ; temp2 = step2[4] * cospi_4_64 + step2[7] * cospi_28_64 + vmlal.s16 q5, d30, d0 + vmlal.s16 q6, d31, d0 + + vdup.16 d2, r3 ; duplicate cospi_12_64 + vdup.16 d3, r12 ; duplicate cospi_20_64 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d8, q2, #14 ; >> 14 + vqrshrn.s32 d9, q3, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d14, q5, #14 ; >> 14 + vqrshrn.s32 d15, q6, #14 ; >> 14 + + ; preloading to avoid stall + ; generate cospi_16_64 = 11585 + mov r3, #0x2d00 + add r3, #0x41 + + ; generate cospi_24_64 = 6270 + mov r12, #0x1800 + add r12, #0x7e + + ; step2[5] * cospi_12_64 + vmull.s16 q2, d26, d2 + vmull.s16 q3, d27, d2 + + ; step2[5] * cospi_20_64 + vmull.s16 q9, d26, d3 + vmull.s16 q15, d27, d3 + + ; temp1 = input[5] * cospi_12_64 - input[3] * cospi_20_64 + vmlsl.s16 q2, d22, d3 + vmlsl.s16 q3, d23, d3 + + ; temp2 = step2[5] * cospi_20_64 + step2[6] * cospi_12_64 + vmlal.s16 q9, d22, d2 + vmlal.s16 q15, d23, d2 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d10, q2, #14 ; >> 14 + vqrshrn.s32 d11, q3, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d12, q9, #14 ; >> 14 + vqrshrn.s32 d13, q15, #14 ; >> 14 + + ; stage 4 + vdup.16 d30, r3 ; cospi_16_64 + + ; step1[0] * cospi_16_64 + vmull.s16 q2, d16, d30 + vmull.s16 q11, d17, d30 + + ; step1[1] * cospi_16_64 + vmull.s16 q0, d24, d30 + vmull.s16 q1, d25, d30 + + ; generate cospi_8_64 = 15137 + mov r3, #0x3b00 + add r3, #0x21 + + vdup.16 d30, r12 ; duplicate cospi_24_64 + vdup.16 d31, r3 ; duplicate cospi_8_64 + + ; temp1 = (step1[0] + step1[1]) * cospi_16_64 + vadd.s32 q3, q2, q0 + vadd.s32 q12, q11, q1 + + ; temp2 = (step1[0] - step1[1]) * cospi_16_64 + vsub.s32 q13, q2, q0 + vsub.s32 q1, q11, q1 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d16, q3, #14 ; >> 14 + vqrshrn.s32 d17, q12, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d18, q13, #14 ; >> 14 + vqrshrn.s32 d19, q1, #14 ; >> 14 + + ; step1[2] * cospi_24_64 - step1[3] * cospi_8_64; + ; step1[2] * cospi_8_64 + vmull.s16 q0, d20, d31 + vmull.s16 q1, d21, d31 + + ; step1[2] * cospi_24_64 + vmull.s16 q12, d20, d30 + vmull.s16 q13, d21, d30 + + ; temp2 = input[1] * cospi_8_64 + input[3] * cospi_24_64 + vmlal.s16 q0, d28, d30 + vmlal.s16 q1, d29, d30 + + ; temp1 = input[1] * cospi_24_64 - input[3] * cospi_8_64 + vmlsl.s16 q12, d28, d31 + vmlsl.s16 q13, d29, d31 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d22, q0, #14 ; >> 14 + vqrshrn.s32 d23, q1, #14 ; >> 14 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d20, q12, #14 ; >> 14 + vqrshrn.s32 d21, q13, #14 ; >> 14 + + vsub.s16 q13, q4, q5 ; step2[5] = step1[4] - step1[5]; + vadd.s16 q4, q4, q5 ; step2[4] = step1[4] + step1[5]; + vsub.s16 q14, q7, q6 ; step2[6] = -step1[6] + step1[7]; + vadd.s16 q15, q6, q7 ; step2[7] = step1[6] + step1[7]; + + ; generate cospi_16_64 = 11585 + mov r3, #0x2d00 + add r3, #0x41 + + ; stage 5 + vadd.s16 q0, q8, q11 ; step1[0] = step2[0] + step2[3]; + vadd.s16 q1, q9, q10 ; step1[1] = step2[1] + step2[2]; + vsub.s16 q2, q9, q10 ; step1[2] = step2[1] - step2[2]; + vsub.s16 q3, q8, q11 ; step1[3] = step2[0] - step2[3]; + + vdup.16 d16, r3; ; duplicate cospi_16_64 + + ; step2[5] * cospi_16_64 + vmull.s16 q11, d26, d16 + vmull.s16 q12, d27, d16 + + ; step2[6] * cospi_16_64 + vmull.s16 q9, d28, d16 + vmull.s16 q10, d29, d16 + + ; temp1 = (step2[6] - step2[5]) * cospi_16_64 + vsub.s32 q6, q9, q11 + vsub.s32 q13, q10, q12 + + ; temp2 = (step2[5] + step2[6]) * cospi_16_64 + vadd.s32 q9, q9, q11 + vadd.s32 q10, q10, q12 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d10, q6, #14 ; >> 14 + vqrshrn.s32 d11, q13, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d12, q9, #14 ; >> 14 + vqrshrn.s32 d13, q10, #14 ; >> 14 + + ; stage 6 + vadd.s16 q8, q0, q15 ; step2[0] = step1[0] + step1[7]; + vadd.s16 q9, q1, q6 ; step2[1] = step1[1] + step1[6]; + vadd.s16 q10, q2, q5 ; step2[2] = step1[2] + step1[5]; + vadd.s16 q11, q3, q4 ; step2[3] = step1[3] + step1[4]; + vsub.s16 q12, q3, q4 ; step2[4] = step1[3] - step1[4]; + vsub.s16 q13, q2, q5 ; step2[5] = step1[2] - step1[5]; + vsub.s16 q14, q1, q6 ; step2[6] = step1[1] - step1[6]; + vsub.s16 q15, q0, q15 ; step2[7] = step1[0] - step1[7]; + + ; store the data + vst1.64 {d16}, [r1], r2 + vst1.64 {d17}, [r1], r2 + vst1.64 {d18}, [r1], r2 + vst1.64 {d19}, [r1], r2 + vst1.64 {d20}, [r1], r2 + vst1.64 {d21}, [r1], r2 + vst1.64 {d22}, [r1], r2 + vst1.64 {d23}, [r1], r2 + vst1.64 {d24}, [r1], r2 + vst1.64 {d25}, [r1], r2 + vst1.64 {d26}, [r1], r2 + vst1.64 {d27}, [r1], r2 + vst1.64 {d28}, [r1], r2 + vst1.64 {d29}, [r1], r2 + vst1.64 {d30}, [r1], r2 + vst1.64 {d31}, [r1], r2 + + bx lr + ENDP ; |aom_idct16x16_256_add_neon_pass1| + +;void aom_idct16x16_256_add_neon_pass2(int16_t *src, +; int16_t *output, +; int16_t *pass1Output, +; int16_t skip_adding, +; uint8_t *dest, +; int dest_stride) +; +; r0 int16_t *src +; r1 int16_t *output, +; r2 int16_t *pass1Output, +; r3 int16_t skip_adding, +; r4 uint8_t *dest, +; r5 int dest_stride) + +; idct16 stage1 - stage7 on all the elements loaded in q8-q15. The output +; will be stored back into q8-q15 registers. This function will touch q0-q7 +; registers and use them as buffer during calculation. +|aom_idct16x16_256_add_neon_pass2| PROC + push {r3-r9} + + ; TODO(hkuang): Find a better way to load the elements. + ; load elements of 1, 3, 5, 7, 9, 11, 13, 15 into q8 - q15 + vld2.s16 {q8,q9}, [r0]! + vld2.s16 {q9,q10}, [r0]! + vld2.s16 {q10,q11}, [r0]! + vld2.s16 {q11,q12}, [r0]! + vld2.s16 {q12,q13}, [r0]! + vld2.s16 {q13,q14}, [r0]! + vld2.s16 {q14,q15}, [r0]! + vld2.s16 {q0,q1}, [r0]! + vmov.s16 q15, q0; + + ; generate cospi_30_64 = 1606 + mov r3, #0x0600 + add r3, #0x46 + + ; generate cospi_2_64 = 16305 + mov r12, #0x3f00 + add r12, #0xb1 + + ; transpose the input data + TRANSPOSE8X8 + + ; stage 3 + vdup.16 d12, r3 ; duplicate cospi_30_64 + vdup.16 d13, r12 ; duplicate cospi_2_64 + + ; preloading to avoid stall + ; generate cospi_14_64 = 12665 + mov r3, #0x3100 + add r3, #0x79 + + ; generate cospi_18_64 = 10394 + mov r12, #0x2800 + add r12, #0x9a + + ; step1[8] * cospi_30_64 + vmull.s16 q2, d16, d12 + vmull.s16 q3, d17, d12 + + ; step1[8] * cospi_2_64 + vmull.s16 q1, d16, d13 + vmull.s16 q4, d17, d13 + + ; temp1 = step1[8] * cospi_30_64 - step1[15] * cospi_2_64 + vmlsl.s16 q2, d30, d13 + vmlsl.s16 q3, d31, d13 + + ; temp2 = step1[8] * cospi_2_64 + step1[15] * cospi_30_64 + vmlal.s16 q1, d30, d12 + vmlal.s16 q4, d31, d12 + + vdup.16 d30, r3 ; duplicate cospi_14_64 + vdup.16 d31, r12 ; duplicate cospi_18_64 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d0, q2, #14 ; >> 14 + vqrshrn.s32 d1, q3, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d14, q1, #14 ; >> 14 + vqrshrn.s32 d15, q4, #14 ; >> 14 + + ; preloading to avoid stall + ; generate cospi_22_64 = 7723 + mov r3, #0x1e00 + add r3, #0x2b + + ; generate cospi_10_64 = 14449 + mov r12, #0x3800 + add r12, #0x71 + + ; step1[9] * cospi_14_64 + vmull.s16 q2, d24, d30 + vmull.s16 q3, d25, d30 + + ; step1[9] * cospi_18_64 + vmull.s16 q4, d24, d31 + vmull.s16 q5, d25, d31 + + ; temp1 = step1[9] * cospi_14_64 - step1[14] * cospi_18_64 + vmlsl.s16 q2, d22, d31 + vmlsl.s16 q3, d23, d31 + + ; temp2 = step1[9] * cospi_18_64 + step1[14] * cospi_14_64 + vmlal.s16 q4, d22, d30 + vmlal.s16 q5, d23, d30 + + vdup.16 d30, r3 ; duplicate cospi_22_64 + vdup.16 d31, r12 ; duplicate cospi_10_64 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d2, q2, #14 ; >> 14 + vqrshrn.s32 d3, q3, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d12, q4, #14 ; >> 14 + vqrshrn.s32 d13, q5, #14 ; >> 14 + + ; step1[10] * cospi_22_64 + vmull.s16 q11, d20, d30 + vmull.s16 q12, d21, d30 + + ; step1[10] * cospi_10_64 + vmull.s16 q4, d20, d31 + vmull.s16 q5, d21, d31 + + ; temp1 = step1[10] * cospi_22_64 - step1[13] * cospi_10_64 + vmlsl.s16 q11, d26, d31 + vmlsl.s16 q12, d27, d31 + + ; temp2 = step1[10] * cospi_10_64 + step1[13] * cospi_22_64 + vmlal.s16 q4, d26, d30 + vmlal.s16 q5, d27, d30 + + ; preloading to avoid stall + ; generate cospi_6_64 = 15679 + mov r3, #0x3d00 + add r3, #0x3f + + ; generate cospi_26_64 = 4756 + mov r12, #0x1200 + add r12, #0x94 + + vdup.16 d30, r3 ; duplicate cospi_6_64 + vdup.16 d31, r12 ; duplicate cospi_26_64 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d4, q11, #14 ; >> 14 + vqrshrn.s32 d5, q12, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d11, q5, #14 ; >> 14 + vqrshrn.s32 d10, q4, #14 ; >> 14 + + ; step1[11] * cospi_6_64 + vmull.s16 q10, d28, d30 + vmull.s16 q11, d29, d30 + + ; step1[11] * cospi_26_64 + vmull.s16 q12, d28, d31 + vmull.s16 q13, d29, d31 + + ; temp1 = step1[11] * cospi_6_64 - step1[12] * cospi_26_64 + vmlsl.s16 q10, d18, d31 + vmlsl.s16 q11, d19, d31 + + ; temp2 = step1[11] * cospi_26_64 + step1[12] * cospi_6_64 + vmlal.s16 q12, d18, d30 + vmlal.s16 q13, d19, d30 + + vsub.s16 q9, q0, q1 ; step1[9]=step2[8]-step2[9] + vadd.s16 q0, q0, q1 ; step1[8]=step2[8]+step2[9] + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d6, q10, #14 ; >> 14 + vqrshrn.s32 d7, q11, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d8, q12, #14 ; >> 14 + vqrshrn.s32 d9, q13, #14 ; >> 14 + + ; stage 3 + vsub.s16 q10, q3, q2 ; step1[10]=-step2[10]+step2[11] + vadd.s16 q11, q2, q3 ; step1[11]=step2[10]+step2[11] + vadd.s16 q12, q4, q5 ; step1[12]=step2[12]+step2[13] + vsub.s16 q13, q4, q5 ; step1[13]=step2[12]-step2[13] + vsub.s16 q14, q7, q6 ; step1[14]=-step2[14]+tep2[15] + vadd.s16 q7, q6, q7 ; step1[15]=step2[14]+step2[15] + + ; stage 4 + ; generate cospi_24_64 = 6270 + mov r3, #0x1800 + add r3, #0x7e + + ; generate cospi_8_64 = 15137 + mov r12, #0x3b00 + add r12, #0x21 + + ; -step1[9] * cospi_8_64 + step1[14] * cospi_24_64 + vdup.16 d30, r12 ; duplicate cospi_8_64 + vdup.16 d31, r3 ; duplicate cospi_24_64 + + ; step1[9] * cospi_24_64 + vmull.s16 q2, d18, d31 + vmull.s16 q3, d19, d31 + + ; step1[14] * cospi_24_64 + vmull.s16 q4, d28, d31 + vmull.s16 q5, d29, d31 + + ; temp2 = step1[9] * cospi_24_64 + step1[14] * cospi_8_64 + vmlal.s16 q2, d28, d30 + vmlal.s16 q3, d29, d30 + + ; temp1 = -step1[9] * cospi_8_64 + step1[14] * cospi_24_64 + vmlsl.s16 q4, d18, d30 + vmlsl.s16 q5, d19, d30 + + rsb r12, #0 + vdup.16 d30, r12 ; duplicate -cospi_8_64 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d12, q2, #14 ; >> 14 + vqrshrn.s32 d13, q3, #14 ; >> 14 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d2, q4, #14 ; >> 14 + vqrshrn.s32 d3, q5, #14 ; >> 14 + + vmov.s16 q3, q11 + vmov.s16 q4, q12 + + ; - step1[13] * cospi_8_64 + vmull.s16 q11, d26, d30 + vmull.s16 q12, d27, d30 + + ; -step1[10] * cospi_8_64 + vmull.s16 q8, d20, d30 + vmull.s16 q9, d21, d30 + + ; temp2 = -step1[10] * cospi_8_64 + step1[13] * cospi_24_64 + vmlsl.s16 q11, d20, d31 + vmlsl.s16 q12, d21, d31 + + ; temp1 = -step1[10] * cospi_8_64 + step1[13] * cospi_24_64 + vmlal.s16 q8, d26, d31 + vmlal.s16 q9, d27, d31 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d4, q11, #14 ; >> 14 + vqrshrn.s32 d5, q12, #14 ; >> 14 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d10, q8, #14 ; >> 14 + vqrshrn.s32 d11, q9, #14 ; >> 14 + + ; stage 5 + vadd.s16 q8, q0, q3 ; step1[8] = step2[8]+step2[11]; + vadd.s16 q9, q1, q2 ; step1[9] = step2[9]+step2[10]; + vsub.s16 q10, q1, q2 ; step1[10] = step2[9]-step2[10]; + vsub.s16 q11, q0, q3 ; step1[11] = step2[8]-step2[11]; + vsub.s16 q12, q7, q4 ; step1[12] =-step2[12]+step2[15]; + vsub.s16 q13, q6, q5 ; step1[13] =-step2[13]+step2[14]; + vadd.s16 q14, q6, q5 ; step1[14] =step2[13]+step2[14]; + vadd.s16 q15, q7, q4 ; step1[15] =step2[12]+step2[15]; + + ; stage 6. + ; generate cospi_16_64 = 11585 + mov r12, #0x2d00 + add r12, #0x41 + + vdup.16 d14, r12 ; duplicate cospi_16_64 + + ; step1[13] * cospi_16_64 + vmull.s16 q3, d26, d14 + vmull.s16 q4, d27, d14 + + ; step1[10] * cospi_16_64 + vmull.s16 q0, d20, d14 + vmull.s16 q1, d21, d14 + + ; temp1 = (-step1[10] + step1[13]) * cospi_16_64 + vsub.s32 q5, q3, q0 + vsub.s32 q6, q4, q1 + + ; temp2 = (step1[10] + step1[13]) * cospi_16_64 + vadd.s32 q10, q3, q0 + vadd.s32 q4, q4, q1 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d4, q5, #14 ; >> 14 + vqrshrn.s32 d5, q6, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d10, q10, #14 ; >> 14 + vqrshrn.s32 d11, q4, #14 ; >> 14 + + ; step1[11] * cospi_16_64 + vmull.s16 q0, d22, d14 + vmull.s16 q1, d23, d14 + + ; step1[12] * cospi_16_64 + vmull.s16 q13, d24, d14 + vmull.s16 q6, d25, d14 + + ; temp1 = (-step1[11] + step1[12]) * cospi_16_64 + vsub.s32 q10, q13, q0 + vsub.s32 q4, q6, q1 + + ; temp2 = (step1[11] + step1[12]) * cospi_16_64 + vadd.s32 q13, q13, q0 + vadd.s32 q6, q6, q1 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d6, q10, #14 ; >> 14 + vqrshrn.s32 d7, q4, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d8, q13, #14 ; >> 14 + vqrshrn.s32 d9, q6, #14 ; >> 14 + + mov r4, #16 ; pass1Output stride + ldr r3, [sp] ; load skip_adding + cmp r3, #0 ; check if need adding dest data + beq skip_adding_dest + + ldr r7, [sp, #28] ; dest used to save element 0-7 + mov r9, r7 ; save dest pointer for later use + ldr r8, [sp, #32] ; load dest_stride + + ; stage 7 + ; load the data in pass1 + vld1.s16 {q0}, [r2], r4 ; load data step2[0] + vld1.s16 {q1}, [r2], r4 ; load data step2[1] + vld1.s16 {q10}, [r2], r4 ; load data step2[2] + vld1.s16 {q11}, [r2], r4 ; load data step2[3] + vld1.64 {d12}, [r7], r8 ; load destinatoin data + vld1.64 {d13}, [r7], r8 ; load destinatoin data + vadd.s16 q12, q0, q15 ; step2[0] + step2[15] + vadd.s16 q13, q1, q14 ; step2[1] + step2[14] + vrshr.s16 q12, q12, #6 ; ROUND_POWER_OF_TWO + vrshr.s16 q13, q13, #6 ; ROUND_POWER_OF_TWO + vaddw.u8 q12, q12, d12 ; + dest[j * dest_stride + i] + vaddw.u8 q13, q13, d13 ; + dest[j * dest_stride + i] + vqmovun.s16 d12, q12 ; clip pixel + vqmovun.s16 d13, q13 ; clip pixel + vst1.64 {d12}, [r9], r8 ; store the data + vst1.64 {d13}, [r9], r8 ; store the data + vsub.s16 q14, q1, q14 ; step2[1] - step2[14] + vsub.s16 q15, q0, q15 ; step2[0] - step2[15] + vld1.64 {d12}, [r7], r8 ; load destinatoin data + vld1.64 {d13}, [r7], r8 ; load destinatoin data + vadd.s16 q12, q10, q5 ; step2[2] + step2[13] + vadd.s16 q13, q11, q4 ; step2[3] + step2[12] + vrshr.s16 q12, q12, #6 ; ROUND_POWER_OF_TWO + vrshr.s16 q13, q13, #6 ; ROUND_POWER_OF_TWO + vaddw.u8 q12, q12, d12 ; + dest[j * dest_stride + i] + vaddw.u8 q13, q13, d13 ; + dest[j * dest_stride + i] + vqmovun.s16 d12, q12 ; clip pixel + vqmovun.s16 d13, q13 ; clip pixel + vst1.64 {d12}, [r9], r8 ; store the data + vst1.64 {d13}, [r9], r8 ; store the data + vsub.s16 q4, q11, q4 ; step2[3] - step2[12] + vsub.s16 q5, q10, q5 ; step2[2] - step2[13] + vld1.s16 {q0}, [r2], r4 ; load data step2[4] + vld1.s16 {q1}, [r2], r4 ; load data step2[5] + vld1.s16 {q10}, [r2], r4 ; load data step2[6] + vld1.s16 {q11}, [r2], r4 ; load data step2[7] + vld1.64 {d12}, [r7], r8 ; load destinatoin data + vld1.64 {d13}, [r7], r8 ; load destinatoin data + vadd.s16 q12, q0, q3 ; step2[4] + step2[11] + vadd.s16 q13, q1, q2 ; step2[5] + step2[10] + vrshr.s16 q12, q12, #6 ; ROUND_POWER_OF_TWO + vrshr.s16 q13, q13, #6 ; ROUND_POWER_OF_TWO + vaddw.u8 q12, q12, d12 ; + dest[j * dest_stride + i] + vaddw.u8 q13, q13, d13 ; + dest[j * dest_stride + i] + vqmovun.s16 d12, q12 ; clip pixel + vqmovun.s16 d13, q13 ; clip pixel + vst1.64 {d12}, [r9], r8 ; store the data + vst1.64 {d13}, [r9], r8 ; store the data + vsub.s16 q2, q1, q2 ; step2[5] - step2[10] + vsub.s16 q3, q0, q3 ; step2[4] - step2[11] + vld1.64 {d12}, [r7], r8 ; load destinatoin data + vld1.64 {d13}, [r7], r8 ; load destinatoin data + vadd.s16 q12, q10, q9 ; step2[6] + step2[9] + vadd.s16 q13, q11, q8 ; step2[7] + step2[8] + vrshr.s16 q12, q12, #6 ; ROUND_POWER_OF_TWO + vrshr.s16 q13, q13, #6 ; ROUND_POWER_OF_TWO + vaddw.u8 q12, q12, d12 ; + dest[j * dest_stride + i] + vaddw.u8 q13, q13, d13 ; + dest[j * dest_stride + i] + vqmovun.s16 d12, q12 ; clip pixel + vqmovun.s16 d13, q13 ; clip pixel + vst1.64 {d12}, [r9], r8 ; store the data + vst1.64 {d13}, [r9], r8 ; store the data + vld1.64 {d12}, [r7], r8 ; load destinatoin data + vld1.64 {d13}, [r7], r8 ; load destinatoin data + vsub.s16 q8, q11, q8 ; step2[7] - step2[8] + vsub.s16 q9, q10, q9 ; step2[6] - step2[9] + + ; store the data output 8,9,10,11,12,13,14,15 + vrshr.s16 q8, q8, #6 ; ROUND_POWER_OF_TWO + vaddw.u8 q8, q8, d12 ; + dest[j * dest_stride + i] + vqmovun.s16 d12, q8 ; clip pixel + vst1.64 {d12}, [r9], r8 ; store the data + vld1.64 {d12}, [r7], r8 ; load destinatoin data + vrshr.s16 q9, q9, #6 + vaddw.u8 q9, q9, d13 ; + dest[j * dest_stride + i] + vqmovun.s16 d13, q9 ; clip pixel + vst1.64 {d13}, [r9], r8 ; store the data + vld1.64 {d13}, [r7], r8 ; load destinatoin data + vrshr.s16 q2, q2, #6 + vaddw.u8 q2, q2, d12 ; + dest[j * dest_stride + i] + vqmovun.s16 d12, q2 ; clip pixel + vst1.64 {d12}, [r9], r8 ; store the data + vld1.64 {d12}, [r7], r8 ; load destinatoin data + vrshr.s16 q3, q3, #6 + vaddw.u8 q3, q3, d13 ; + dest[j * dest_stride + i] + vqmovun.s16 d13, q3 ; clip pixel + vst1.64 {d13}, [r9], r8 ; store the data + vld1.64 {d13}, [r7], r8 ; load destinatoin data + vrshr.s16 q4, q4, #6 + vaddw.u8 q4, q4, d12 ; + dest[j * dest_stride + i] + vqmovun.s16 d12, q4 ; clip pixel + vst1.64 {d12}, [r9], r8 ; store the data + vld1.64 {d12}, [r7], r8 ; load destinatoin data + vrshr.s16 q5, q5, #6 + vaddw.u8 q5, q5, d13 ; + dest[j * dest_stride + i] + vqmovun.s16 d13, q5 ; clip pixel + vst1.64 {d13}, [r9], r8 ; store the data + vld1.64 {d13}, [r7], r8 ; load destinatoin data + vrshr.s16 q14, q14, #6 + vaddw.u8 q14, q14, d12 ; + dest[j * dest_stride + i] + vqmovun.s16 d12, q14 ; clip pixel + vst1.64 {d12}, [r9], r8 ; store the data + vld1.64 {d12}, [r7], r8 ; load destinatoin data + vrshr.s16 q15, q15, #6 + vaddw.u8 q15, q15, d13 ; + dest[j * dest_stride + i] + vqmovun.s16 d13, q15 ; clip pixel + vst1.64 {d13}, [r9], r8 ; store the data + b end_idct16x16_pass2 + +skip_adding_dest + ; stage 7 + ; load the data in pass1 + mov r5, #24 + mov r3, #8 + + vld1.s16 {q0}, [r2], r4 ; load data step2[0] + vld1.s16 {q1}, [r2], r4 ; load data step2[1] + vadd.s16 q12, q0, q15 ; step2[0] + step2[15] + vadd.s16 q13, q1, q14 ; step2[1] + step2[14] + vld1.s16 {q10}, [r2], r4 ; load data step2[2] + vld1.s16 {q11}, [r2], r4 ; load data step2[3] + vst1.64 {d24}, [r1], r3 ; store output[0] + vst1.64 {d25}, [r1], r5 + vst1.64 {d26}, [r1], r3 ; store output[1] + vst1.64 {d27}, [r1], r5 + vadd.s16 q12, q10, q5 ; step2[2] + step2[13] + vadd.s16 q13, q11, q4 ; step2[3] + step2[12] + vsub.s16 q14, q1, q14 ; step2[1] - step2[14] + vsub.s16 q15, q0, q15 ; step2[0] - step2[15] + vst1.64 {d24}, [r1], r3 ; store output[2] + vst1.64 {d25}, [r1], r5 + vst1.64 {d26}, [r1], r3 ; store output[3] + vst1.64 {d27}, [r1], r5 + vsub.s16 q4, q11, q4 ; step2[3] - step2[12] + vsub.s16 q5, q10, q5 ; step2[2] - step2[13] + vld1.s16 {q0}, [r2], r4 ; load data step2[4] + vld1.s16 {q1}, [r2], r4 ; load data step2[5] + vadd.s16 q12, q0, q3 ; step2[4] + step2[11] + vadd.s16 q13, q1, q2 ; step2[5] + step2[10] + vld1.s16 {q10}, [r2], r4 ; load data step2[6] + vld1.s16 {q11}, [r2], r4 ; load data step2[7] + vst1.64 {d24}, [r1], r3 ; store output[4] + vst1.64 {d25}, [r1], r5 + vst1.64 {d26}, [r1], r3 ; store output[5] + vst1.64 {d27}, [r1], r5 + vadd.s16 q12, q10, q9 ; step2[6] + step2[9] + vadd.s16 q13, q11, q8 ; step2[7] + step2[8] + vsub.s16 q2, q1, q2 ; step2[5] - step2[10] + vsub.s16 q3, q0, q3 ; step2[4] - step2[11] + vsub.s16 q8, q11, q8 ; step2[7] - step2[8] + vsub.s16 q9, q10, q9 ; step2[6] - step2[9] + vst1.64 {d24}, [r1], r3 ; store output[6] + vst1.64 {d25}, [r1], r5 + vst1.64 {d26}, [r1], r3 ; store output[7] + vst1.64 {d27}, [r1], r5 + + ; store the data output 8,9,10,11,12,13,14,15 + vst1.64 {d16}, [r1], r3 + vst1.64 {d17}, [r1], r5 + vst1.64 {d18}, [r1], r3 + vst1.64 {d19}, [r1], r5 + vst1.64 {d4}, [r1], r3 + vst1.64 {d5}, [r1], r5 + vst1.64 {d6}, [r1], r3 + vst1.64 {d7}, [r1], r5 + vst1.64 {d8}, [r1], r3 + vst1.64 {d9}, [r1], r5 + vst1.64 {d10}, [r1], r3 + vst1.64 {d11}, [r1], r5 + vst1.64 {d28}, [r1], r3 + vst1.64 {d29}, [r1], r5 + vst1.64 {d30}, [r1], r3 + vst1.64 {d31}, [r1], r5 +end_idct16x16_pass2 + pop {r3-r9} + bx lr + ENDP ; |aom_idct16x16_256_add_neon_pass2| + +;void |aom_idct16x16_10_add_neon_pass1|(int16_t *input, +; int16_t *output, int output_stride) +; +; r0 int16_t input +; r1 int16_t *output +; r2 int output_stride) + +; idct16 stage1 - stage6 on all the elements loaded in q8-q15. The output +; will be stored back into q8-q15 registers. This function will touch q0-q7 +; registers and use them as buffer during calculation. +|aom_idct16x16_10_add_neon_pass1| PROC + + ; TODO(hkuang): Find a better way to load the elements. + ; load elements of 0, 2, 4, 6, 8, 10, 12, 14 into q8 - q15 + vld2.s16 {q8,q9}, [r0]! + vld2.s16 {q9,q10}, [r0]! + vld2.s16 {q10,q11}, [r0]! + vld2.s16 {q11,q12}, [r0]! + vld2.s16 {q12,q13}, [r0]! + vld2.s16 {q13,q14}, [r0]! + vld2.s16 {q14,q15}, [r0]! + vld2.s16 {q1,q2}, [r0]! + vmov.s16 q15, q1 + + ; generate cospi_28_64*2 = 6392 + mov r3, #0x1800 + add r3, #0xf8 + + ; generate cospi_4_64*2 = 32138 + mov r12, #0x7d00 + add r12, #0x8a + + ; transpose the input data + TRANSPOSE8X8 + + ; stage 3 + vdup.16 q0, r3 ; duplicate cospi_28_64*2 + vdup.16 q1, r12 ; duplicate cospi_4_64*2 + + ; The following instructions use vqrdmulh to do the + ; dct_const_round_shift(step2[4] * cospi_28_64). vvqrdmulh will multiply, + ; double, and return the high 16 bits, effectively giving >> 15. Doubling + ; the constant will change this to >> 14. + ; dct_const_round_shift(step2[4] * cospi_28_64); + vqrdmulh.s16 q4, q9, q0 + + ; preloading to avoid stall + ; generate cospi_16_64*2 = 23170 + mov r3, #0x5a00 + add r3, #0x82 + + ; dct_const_round_shift(step2[4] * cospi_4_64); + vqrdmulh.s16 q7, q9, q1 + + ; stage 4 + vdup.16 q1, r3 ; cospi_16_64*2 + + ; generate cospi_16_64 = 11585 + mov r3, #0x2d00 + add r3, #0x41 + + vdup.16 d4, r3; ; duplicate cospi_16_64 + + ; dct_const_round_shift(step1[0] * cospi_16_64) + vqrdmulh.s16 q8, q8, q1 + + ; step2[6] * cospi_16_64 + vmull.s16 q9, d14, d4 + vmull.s16 q10, d15, d4 + + ; step2[5] * cospi_16_64 + vmull.s16 q12, d9, d4 + vmull.s16 q11, d8, d4 + + ; temp1 = (step2[6] - step2[5]) * cospi_16_64 + vsub.s32 q15, q10, q12 + vsub.s32 q6, q9, q11 + + ; temp2 = (step2[5] + step2[6]) * cospi_16_64 + vadd.s32 q9, q9, q11 + vadd.s32 q10, q10, q12 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d11, q15, #14 ; >> 14 + vqrshrn.s32 d10, q6, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d12, q9, #14 ; >> 14 + vqrshrn.s32 d13, q10, #14 ; >> 14 + + ; stage 6 + vadd.s16 q2, q8, q7 ; step2[0] = step1[0] + step1[7]; + vadd.s16 q10, q8, q5 ; step2[2] = step1[2] + step1[5]; + vadd.s16 q11, q8, q4 ; step2[3] = step1[3] + step1[4]; + vadd.s16 q9, q8, q6 ; step2[1] = step1[1] + step1[6]; + vsub.s16 q12, q8, q4 ; step2[4] = step1[3] - step1[4]; + vsub.s16 q13, q8, q5 ; step2[5] = step1[2] - step1[5]; + vsub.s16 q14, q8, q6 ; step2[6] = step1[1] - step1[6]; + vsub.s16 q15, q8, q7 ; step2[7] = step1[0] - step1[7]; + + ; store the data + vst1.64 {d4}, [r1], r2 + vst1.64 {d5}, [r1], r2 + vst1.64 {d18}, [r1], r2 + vst1.64 {d19}, [r1], r2 + vst1.64 {d20}, [r1], r2 + vst1.64 {d21}, [r1], r2 + vst1.64 {d22}, [r1], r2 + vst1.64 {d23}, [r1], r2 + vst1.64 {d24}, [r1], r2 + vst1.64 {d25}, [r1], r2 + vst1.64 {d26}, [r1], r2 + vst1.64 {d27}, [r1], r2 + vst1.64 {d28}, [r1], r2 + vst1.64 {d29}, [r1], r2 + vst1.64 {d30}, [r1], r2 + vst1.64 {d31}, [r1], r2 + + bx lr + ENDP ; |aom_idct16x16_10_add_neon_pass1| + +;void aom_idct16x16_10_add_neon_pass2(int16_t *src, +; int16_t *output, +; int16_t *pass1Output, +; int16_t skip_adding, +; uint8_t *dest, +; int dest_stride) +; +; r0 int16_t *src +; r1 int16_t *output, +; r2 int16_t *pass1Output, +; r3 int16_t skip_adding, +; r4 uint8_t *dest, +; r5 int dest_stride) + +; idct16 stage1 - stage7 on all the elements loaded in q8-q15. The output +; will be stored back into q8-q15 registers. This function will touch q0-q7 +; registers and use them as buffer during calculation. +|aom_idct16x16_10_add_neon_pass2| PROC + push {r3-r9} + + ; TODO(hkuang): Find a better way to load the elements. + ; load elements of 1, 3, 5, 7, 9, 11, 13, 15 into q8 - q15 + vld2.s16 {q8,q9}, [r0]! + vld2.s16 {q9,q10}, [r0]! + vld2.s16 {q10,q11}, [r0]! + vld2.s16 {q11,q12}, [r0]! + vld2.s16 {q12,q13}, [r0]! + vld2.s16 {q13,q14}, [r0]! + vld2.s16 {q14,q15}, [r0]! + vld2.s16 {q0,q1}, [r0]! + vmov.s16 q15, q0; + + ; generate 2*cospi_30_64 = 3212 + mov r3, #0xc00 + add r3, #0x8c + + ; generate 2*cospi_2_64 = 32610 + mov r12, #0x7f00 + add r12, #0x62 + + ; transpose the input data + TRANSPOSE8X8 + + ; stage 3 + vdup.16 q6, r3 ; duplicate 2*cospi_30_64 + + ; dct_const_round_shift(step1[8] * cospi_30_64) + vqrdmulh.s16 q0, q8, q6 + + vdup.16 q6, r12 ; duplicate 2*cospi_2_64 + + ; dct_const_round_shift(step1[8] * cospi_2_64) + vqrdmulh.s16 q7, q8, q6 + + ; preloading to avoid stall + ; generate 2*cospi_26_64 = 9512 + mov r12, #0x2500 + add r12, #0x28 + rsb r12, #0 + vdup.16 q15, r12 ; duplicate -2*cospi_26_64 + + ; generate 2*cospi_6_64 = 31358 + mov r3, #0x7a00 + add r3, #0x7e + vdup.16 q14, r3 ; duplicate 2*cospi_6_64 + + ; dct_const_round_shift(- step1[12] * cospi_26_64) + vqrdmulh.s16 q3, q9, q15 + + ; dct_const_round_shift(step1[12] * cospi_6_64) + vqrdmulh.s16 q4, q9, q14 + + ; stage 4 + ; generate cospi_24_64 = 6270 + mov r3, #0x1800 + add r3, #0x7e + vdup.16 d31, r3 ; duplicate cospi_24_64 + + ; generate cospi_8_64 = 15137 + mov r12, #0x3b00 + add r12, #0x21 + vdup.16 d30, r12 ; duplicate cospi_8_64 + + ; step1[14] * cospi_24_64 + vmull.s16 q12, d14, d31 + vmull.s16 q5, d15, d31 + + ; step1[9] * cospi_24_64 + vmull.s16 q2, d0, d31 + vmull.s16 q11, d1, d31 + + ; temp1 = -step1[9] * cospi_8_64 + step1[14] * cospi_24_64 + vmlsl.s16 q12, d0, d30 + vmlsl.s16 q5, d1, d30 + + ; temp2 = step1[9] * cospi_24_64 + step1[14] * cospi_8_64 + vmlal.s16 q2, d14, d30 + vmlal.s16 q11, d15, d30 + + rsb r12, #0 + vdup.16 d30, r12 ; duplicate -cospi_8_64 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d2, q12, #14 ; >> 14 + vqrshrn.s32 d3, q5, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d12, q2, #14 ; >> 14 + vqrshrn.s32 d13, q11, #14 ; >> 14 + + ; - step1[13] * cospi_8_64 + vmull.s16 q10, d8, d30 + vmull.s16 q13, d9, d30 + + ; -step1[10] * cospi_8_64 + vmull.s16 q8, d6, d30 + vmull.s16 q9, d7, d30 + + ; temp1 = -step1[10] * cospi_24_64 - step1[13] * cospi_8_64 + vmlsl.s16 q10, d6, d31 + vmlsl.s16 q13, d7, d31 + + ; temp2 = -step1[10] * cospi_8_64 + step1[13] * cospi_24_64 + vmlal.s16 q8, d8, d31 + vmlal.s16 q9, d9, d31 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d4, q10, #14 ; >> 14 + vqrshrn.s32 d5, q13, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d10, q8, #14 ; >> 14 + vqrshrn.s32 d11, q9, #14 ; >> 14 + + ; stage 5 + vadd.s16 q8, q0, q3 ; step1[8] = step2[8]+step2[11]; + vadd.s16 q9, q1, q2 ; step1[9] = step2[9]+step2[10]; + vsub.s16 q10, q1, q2 ; step1[10] = step2[9]-step2[10]; + vsub.s16 q11, q0, q3 ; step1[11] = step2[8]-step2[11]; + vsub.s16 q12, q7, q4 ; step1[12] =-step2[12]+step2[15]; + vsub.s16 q13, q6, q5 ; step1[13] =-step2[13]+step2[14]; + vadd.s16 q14, q6, q5 ; step1[14] =step2[13]+step2[14]; + vadd.s16 q15, q7, q4 ; step1[15] =step2[12]+step2[15]; + + ; stage 6. + ; generate cospi_16_64 = 11585 + mov r12, #0x2d00 + add r12, #0x41 + + vdup.16 d14, r12 ; duplicate cospi_16_64 + + ; step1[13] * cospi_16_64 + vmull.s16 q3, d26, d14 + vmull.s16 q4, d27, d14 + + ; step1[10] * cospi_16_64 + vmull.s16 q0, d20, d14 + vmull.s16 q1, d21, d14 + + ; temp1 = (-step1[10] + step1[13]) * cospi_16_64 + vsub.s32 q5, q3, q0 + vsub.s32 q6, q4, q1 + + ; temp2 = (step1[10] + step1[13]) * cospi_16_64 + vadd.s32 q0, q3, q0 + vadd.s32 q1, q4, q1 + + ; dct_const_round_shift(temp1) + vqrshrn.s32 d4, q5, #14 ; >> 14 + vqrshrn.s32 d5, q6, #14 ; >> 14 + + ; dct_const_round_shift(temp2) + vqrshrn.s32 d10, q0, #14 ; >> 14 + vqrshrn.s32 d11, q1, #14 ; >> 14 + + ; step1[11] * cospi_16_64 + vmull.s16 q0, d22, d14 + vmull.s16 q1, d23, d14 + + ; step1[12] * cospi_16_64 + vmull.s16 q13, d24, d14 + vmull.s16 q6, d25, d14 + + ; temp1 = (-step1[11] + step1[12]) * cospi_16_64 + vsub.s32 q10, q13, q0 + vsub.s32 q4, q6, q1 + + ; temp2 = (step1[11] + step1[12]) * cospi_16_64 + vadd.s32 q13, q13, q0 + vadd.s32 q6, q6, q1 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d6, q10, #14 ; >> 14 + vqrshrn.s32 d7, q4, #14 ; >> 14 + + ; dct_const_round_shift((step1[11] + step1[12]) * cospi_16_64); + vqrshrn.s32 d8, q13, #14 ; >> 14 + vqrshrn.s32 d9, q6, #14 ; >> 14 + + mov r4, #16 ; pass1Output stride + ldr r3, [sp] ; load skip_adding + + ; stage 7 + ; load the data in pass1 + mov r5, #24 + mov r3, #8 + + vld1.s16 {q0}, [r2], r4 ; load data step2[0] + vld1.s16 {q1}, [r2], r4 ; load data step2[1] + vadd.s16 q12, q0, q15 ; step2[0] + step2[15] + vadd.s16 q13, q1, q14 ; step2[1] + step2[14] + vld1.s16 {q10}, [r2], r4 ; load data step2[2] + vld1.s16 {q11}, [r2], r4 ; load data step2[3] + vst1.64 {d24}, [r1], r3 ; store output[0] + vst1.64 {d25}, [r1], r5 + vst1.64 {d26}, [r1], r3 ; store output[1] + vst1.64 {d27}, [r1], r5 + vadd.s16 q12, q10, q5 ; step2[2] + step2[13] + vadd.s16 q13, q11, q4 ; step2[3] + step2[12] + vsub.s16 q14, q1, q14 ; step2[1] - step2[14] + vsub.s16 q15, q0, q15 ; step2[0] - step2[15] + vst1.64 {d24}, [r1], r3 ; store output[2] + vst1.64 {d25}, [r1], r5 + vst1.64 {d26}, [r1], r3 ; store output[3] + vst1.64 {d27}, [r1], r5 + vsub.s16 q4, q11, q4 ; step2[3] - step2[12] + vsub.s16 q5, q10, q5 ; step2[2] - step2[13] + vld1.s16 {q0}, [r2], r4 ; load data step2[4] + vld1.s16 {q1}, [r2], r4 ; load data step2[5] + vadd.s16 q12, q0, q3 ; step2[4] + step2[11] + vadd.s16 q13, q1, q2 ; step2[5] + step2[10] + vld1.s16 {q10}, [r2], r4 ; load data step2[6] + vld1.s16 {q11}, [r2], r4 ; load data step2[7] + vst1.64 {d24}, [r1], r3 ; store output[4] + vst1.64 {d25}, [r1], r5 + vst1.64 {d26}, [r1], r3 ; store output[5] + vst1.64 {d27}, [r1], r5 + vadd.s16 q12, q10, q9 ; step2[6] + step2[9] + vadd.s16 q13, q11, q8 ; step2[7] + step2[8] + vsub.s16 q2, q1, q2 ; step2[5] - step2[10] + vsub.s16 q3, q0, q3 ; step2[4] - step2[11] + vsub.s16 q8, q11, q8 ; step2[7] - step2[8] + vsub.s16 q9, q10, q9 ; step2[6] - step2[9] + vst1.64 {d24}, [r1], r3 ; store output[6] + vst1.64 {d25}, [r1], r5 + vst1.64 {d26}, [r1], r3 ; store output[7] + vst1.64 {d27}, [r1], r5 + + ; store the data output 8,9,10,11,12,13,14,15 + vst1.64 {d16}, [r1], r3 + vst1.64 {d17}, [r1], r5 + vst1.64 {d18}, [r1], r3 + vst1.64 {d19}, [r1], r5 + vst1.64 {d4}, [r1], r3 + vst1.64 {d5}, [r1], r5 + vst1.64 {d6}, [r1], r3 + vst1.64 {d7}, [r1], r5 + vst1.64 {d8}, [r1], r3 + vst1.64 {d9}, [r1], r5 + vst1.64 {d10}, [r1], r3 + vst1.64 {d11}, [r1], r5 + vst1.64 {d28}, [r1], r3 + vst1.64 {d29}, [r1], r5 + vst1.64 {d30}, [r1], r3 + vst1.64 {d31}, [r1], r5 +end_idct10_16x16_pass2 + pop {r3-r9} + bx lr + ENDP ; |aom_idct16x16_10_add_neon_pass2| + END diff --git a/third_party/aom/aom_dsp/arm/idct16x16_add_neon.c b/third_party/aom/aom_dsp/arm/idct16x16_add_neon.c new file mode 100644 index 0000000000..b4cb7a0cd8 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct16x16_add_neon.c @@ -0,0 +1,1295 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "aom_dsp/txfm_common.h" + +static INLINE void TRANSPOSE8X8(int16x8_t *q8s16, int16x8_t *q9s16, + int16x8_t *q10s16, int16x8_t *q11s16, + int16x8_t *q12s16, int16x8_t *q13s16, + int16x8_t *q14s16, int16x8_t *q15s16) { + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16, d23s16; + int16x4_t d24s16, d25s16, d26s16, d27s16, d28s16, d29s16, d30s16, d31s16; + int32x4x2_t q0x2s32, q1x2s32, q2x2s32, q3x2s32; + int16x8x2_t q0x2s16, q1x2s16, q2x2s16, q3x2s16; + + d16s16 = vget_low_s16(*q8s16); + d17s16 = vget_high_s16(*q8s16); + d18s16 = vget_low_s16(*q9s16); + d19s16 = vget_high_s16(*q9s16); + d20s16 = vget_low_s16(*q10s16); + d21s16 = vget_high_s16(*q10s16); + d22s16 = vget_low_s16(*q11s16); + d23s16 = vget_high_s16(*q11s16); + d24s16 = vget_low_s16(*q12s16); + d25s16 = vget_high_s16(*q12s16); + d26s16 = vget_low_s16(*q13s16); + d27s16 = vget_high_s16(*q13s16); + d28s16 = vget_low_s16(*q14s16); + d29s16 = vget_high_s16(*q14s16); + d30s16 = vget_low_s16(*q15s16); + d31s16 = vget_high_s16(*q15s16); + + *q8s16 = vcombine_s16(d16s16, d24s16); // vswp d17, d24 + *q9s16 = vcombine_s16(d18s16, d26s16); // vswp d19, d26 + *q10s16 = vcombine_s16(d20s16, d28s16); // vswp d21, d28 + *q11s16 = vcombine_s16(d22s16, d30s16); // vswp d23, d30 + *q12s16 = vcombine_s16(d17s16, d25s16); + *q13s16 = vcombine_s16(d19s16, d27s16); + *q14s16 = vcombine_s16(d21s16, d29s16); + *q15s16 = vcombine_s16(d23s16, d31s16); + + q0x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(*q8s16), vreinterpretq_s32_s16(*q10s16)); + q1x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(*q9s16), vreinterpretq_s32_s16(*q11s16)); + q2x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(*q12s16), vreinterpretq_s32_s16(*q14s16)); + q3x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(*q13s16), vreinterpretq_s32_s16(*q15s16)); + + q0x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q0x2s32.val[0]), // q8 + vreinterpretq_s16_s32(q1x2s32.val[0])); // q9 + q1x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q0x2s32.val[1]), // q10 + vreinterpretq_s16_s32(q1x2s32.val[1])); // q11 + q2x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q2x2s32.val[0]), // q12 + vreinterpretq_s16_s32(q3x2s32.val[0])); // q13 + q3x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q2x2s32.val[1]), // q14 + vreinterpretq_s16_s32(q3x2s32.val[1])); // q15 + + *q8s16 = q0x2s16.val[0]; + *q9s16 = q0x2s16.val[1]; + *q10s16 = q1x2s16.val[0]; + *q11s16 = q1x2s16.val[1]; + *q12s16 = q2x2s16.val[0]; + *q13s16 = q2x2s16.val[1]; + *q14s16 = q3x2s16.val[0]; + *q15s16 = q3x2s16.val[1]; + return; +} + +void aom_idct16x16_256_add_neon_pass1(int16_t *in, int16_t *out, + int output_stride) { + int16x4_t d0s16, d1s16, d2s16, d3s16; + int16x4_t d8s16, d9s16, d10s16, d11s16, d12s16, d13s16, d14s16, d15s16; + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16, d23s16; + int16x4_t d24s16, d25s16, d26s16, d27s16, d28s16, d29s16, d30s16, d31s16; + uint64x1_t d16u64, d17u64, d18u64, d19u64, d20u64, d21u64, d22u64, d23u64; + uint64x1_t d24u64, d25u64, d26u64, d27u64, d28u64, d29u64, d30u64, d31u64; + int16x8_t q0s16, q1s16, q2s16, q3s16, q4s16, q5s16, q6s16, q7s16; + int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16, q13s16, q14s16, q15s16; + int32x4_t q0s32, q1s32, q2s32, q3s32, q5s32, q6s32, q9s32; + int32x4_t q10s32, q11s32, q12s32, q13s32, q15s32; + int16x8x2_t q0x2s16; + + q0x2s16 = vld2q_s16(in); + q8s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q9s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q10s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q11s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q12s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q13s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q14s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q15s16 = q0x2s16.val[0]; + + TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + d16s16 = vget_low_s16(q8s16); + d17s16 = vget_high_s16(q8s16); + d18s16 = vget_low_s16(q9s16); + d19s16 = vget_high_s16(q9s16); + d20s16 = vget_low_s16(q10s16); + d21s16 = vget_high_s16(q10s16); + d22s16 = vget_low_s16(q11s16); + d23s16 = vget_high_s16(q11s16); + d24s16 = vget_low_s16(q12s16); + d25s16 = vget_high_s16(q12s16); + d26s16 = vget_low_s16(q13s16); + d27s16 = vget_high_s16(q13s16); + d28s16 = vget_low_s16(q14s16); + d29s16 = vget_high_s16(q14s16); + d30s16 = vget_low_s16(q15s16); + d31s16 = vget_high_s16(q15s16); + + // stage 3 + d0s16 = vdup_n_s16((int16_t)cospi_28_64); + d1s16 = vdup_n_s16((int16_t)cospi_4_64); + + q2s32 = vmull_s16(d18s16, d0s16); + q3s32 = vmull_s16(d19s16, d0s16); + q5s32 = vmull_s16(d18s16, d1s16); + q6s32 = vmull_s16(d19s16, d1s16); + + q2s32 = vmlsl_s16(q2s32, d30s16, d1s16); + q3s32 = vmlsl_s16(q3s32, d31s16, d1s16); + q5s32 = vmlal_s16(q5s32, d30s16, d0s16); + q6s32 = vmlal_s16(q6s32, d31s16, d0s16); + + d2s16 = vdup_n_s16((int16_t)cospi_12_64); + d3s16 = vdup_n_s16((int16_t)cospi_20_64); + + d8s16 = vqrshrn_n_s32(q2s32, 14); + d9s16 = vqrshrn_n_s32(q3s32, 14); + d14s16 = vqrshrn_n_s32(q5s32, 14); + d15s16 = vqrshrn_n_s32(q6s32, 14); + q4s16 = vcombine_s16(d8s16, d9s16); + q7s16 = vcombine_s16(d14s16, d15s16); + + q2s32 = vmull_s16(d26s16, d2s16); + q3s32 = vmull_s16(d27s16, d2s16); + q9s32 = vmull_s16(d26s16, d3s16); + q15s32 = vmull_s16(d27s16, d3s16); + + q2s32 = vmlsl_s16(q2s32, d22s16, d3s16); + q3s32 = vmlsl_s16(q3s32, d23s16, d3s16); + q9s32 = vmlal_s16(q9s32, d22s16, d2s16); + q15s32 = vmlal_s16(q15s32, d23s16, d2s16); + + d10s16 = vqrshrn_n_s32(q2s32, 14); + d11s16 = vqrshrn_n_s32(q3s32, 14); + d12s16 = vqrshrn_n_s32(q9s32, 14); + d13s16 = vqrshrn_n_s32(q15s32, 14); + q5s16 = vcombine_s16(d10s16, d11s16); + q6s16 = vcombine_s16(d12s16, d13s16); + + // stage 4 + d30s16 = vdup_n_s16((int16_t)cospi_16_64); + + q2s32 = vmull_s16(d16s16, d30s16); + q11s32 = vmull_s16(d17s16, d30s16); + q0s32 = vmull_s16(d24s16, d30s16); + q1s32 = vmull_s16(d25s16, d30s16); + + d30s16 = vdup_n_s16((int16_t)cospi_24_64); + d31s16 = vdup_n_s16((int16_t)cospi_8_64); + + q3s32 = vaddq_s32(q2s32, q0s32); + q12s32 = vaddq_s32(q11s32, q1s32); + q13s32 = vsubq_s32(q2s32, q0s32); + q1s32 = vsubq_s32(q11s32, q1s32); + + d16s16 = vqrshrn_n_s32(q3s32, 14); + d17s16 = vqrshrn_n_s32(q12s32, 14); + d18s16 = vqrshrn_n_s32(q13s32, 14); + d19s16 = vqrshrn_n_s32(q1s32, 14); + q8s16 = vcombine_s16(d16s16, d17s16); + q9s16 = vcombine_s16(d18s16, d19s16); + + q0s32 = vmull_s16(d20s16, d31s16); + q1s32 = vmull_s16(d21s16, d31s16); + q12s32 = vmull_s16(d20s16, d30s16); + q13s32 = vmull_s16(d21s16, d30s16); + + q0s32 = vmlal_s16(q0s32, d28s16, d30s16); + q1s32 = vmlal_s16(q1s32, d29s16, d30s16); + q12s32 = vmlsl_s16(q12s32, d28s16, d31s16); + q13s32 = vmlsl_s16(q13s32, d29s16, d31s16); + + d22s16 = vqrshrn_n_s32(q0s32, 14); + d23s16 = vqrshrn_n_s32(q1s32, 14); + d20s16 = vqrshrn_n_s32(q12s32, 14); + d21s16 = vqrshrn_n_s32(q13s32, 14); + q10s16 = vcombine_s16(d20s16, d21s16); + q11s16 = vcombine_s16(d22s16, d23s16); + + q13s16 = vsubq_s16(q4s16, q5s16); + q4s16 = vaddq_s16(q4s16, q5s16); + q14s16 = vsubq_s16(q7s16, q6s16); + q15s16 = vaddq_s16(q6s16, q7s16); + d26s16 = vget_low_s16(q13s16); + d27s16 = vget_high_s16(q13s16); + d28s16 = vget_low_s16(q14s16); + d29s16 = vget_high_s16(q14s16); + + // stage 5 + q0s16 = vaddq_s16(q8s16, q11s16); + q1s16 = vaddq_s16(q9s16, q10s16); + q2s16 = vsubq_s16(q9s16, q10s16); + q3s16 = vsubq_s16(q8s16, q11s16); + + d16s16 = vdup_n_s16((int16_t)cospi_16_64); + + q11s32 = vmull_s16(d26s16, d16s16); + q12s32 = vmull_s16(d27s16, d16s16); + q9s32 = vmull_s16(d28s16, d16s16); + q10s32 = vmull_s16(d29s16, d16s16); + + q6s32 = vsubq_s32(q9s32, q11s32); + q13s32 = vsubq_s32(q10s32, q12s32); + q9s32 = vaddq_s32(q9s32, q11s32); + q10s32 = vaddq_s32(q10s32, q12s32); + + d10s16 = vqrshrn_n_s32(q6s32, 14); + d11s16 = vqrshrn_n_s32(q13s32, 14); + d12s16 = vqrshrn_n_s32(q9s32, 14); + d13s16 = vqrshrn_n_s32(q10s32, 14); + q5s16 = vcombine_s16(d10s16, d11s16); + q6s16 = vcombine_s16(d12s16, d13s16); + + // stage 6 + q8s16 = vaddq_s16(q0s16, q15s16); + q9s16 = vaddq_s16(q1s16, q6s16); + q10s16 = vaddq_s16(q2s16, q5s16); + q11s16 = vaddq_s16(q3s16, q4s16); + q12s16 = vsubq_s16(q3s16, q4s16); + q13s16 = vsubq_s16(q2s16, q5s16); + q14s16 = vsubq_s16(q1s16, q6s16); + q15s16 = vsubq_s16(q0s16, q15s16); + + d16u64 = vreinterpret_u64_s16(vget_low_s16(q8s16)); + d17u64 = vreinterpret_u64_s16(vget_high_s16(q8s16)); + d18u64 = vreinterpret_u64_s16(vget_low_s16(q9s16)); + d19u64 = vreinterpret_u64_s16(vget_high_s16(q9s16)); + d20u64 = vreinterpret_u64_s16(vget_low_s16(q10s16)); + d21u64 = vreinterpret_u64_s16(vget_high_s16(q10s16)); + d22u64 = vreinterpret_u64_s16(vget_low_s16(q11s16)); + d23u64 = vreinterpret_u64_s16(vget_high_s16(q11s16)); + d24u64 = vreinterpret_u64_s16(vget_low_s16(q12s16)); + d25u64 = vreinterpret_u64_s16(vget_high_s16(q12s16)); + d26u64 = vreinterpret_u64_s16(vget_low_s16(q13s16)); + d27u64 = vreinterpret_u64_s16(vget_high_s16(q13s16)); + d28u64 = vreinterpret_u64_s16(vget_low_s16(q14s16)); + d29u64 = vreinterpret_u64_s16(vget_high_s16(q14s16)); + d30u64 = vreinterpret_u64_s16(vget_low_s16(q15s16)); + d31u64 = vreinterpret_u64_s16(vget_high_s16(q15s16)); + + // store the data + output_stride >>= 1; // output_stride / 2, out is int16_t + vst1_u64((uint64_t *)out, d16u64); + out += output_stride; + vst1_u64((uint64_t *)out, d17u64); + out += output_stride; + vst1_u64((uint64_t *)out, d18u64); + out += output_stride; + vst1_u64((uint64_t *)out, d19u64); + out += output_stride; + vst1_u64((uint64_t *)out, d20u64); + out += output_stride; + vst1_u64((uint64_t *)out, d21u64); + out += output_stride; + vst1_u64((uint64_t *)out, d22u64); + out += output_stride; + vst1_u64((uint64_t *)out, d23u64); + out += output_stride; + vst1_u64((uint64_t *)out, d24u64); + out += output_stride; + vst1_u64((uint64_t *)out, d25u64); + out += output_stride; + vst1_u64((uint64_t *)out, d26u64); + out += output_stride; + vst1_u64((uint64_t *)out, d27u64); + out += output_stride; + vst1_u64((uint64_t *)out, d28u64); + out += output_stride; + vst1_u64((uint64_t *)out, d29u64); + out += output_stride; + vst1_u64((uint64_t *)out, d30u64); + out += output_stride; + vst1_u64((uint64_t *)out, d31u64); + return; +} + +void aom_idct16x16_256_add_neon_pass2(int16_t *src, int16_t *out, + int16_t *pass1Output, int16_t skip_adding, + uint8_t *dest, int dest_stride) { + uint8_t *d; + uint8x8_t d12u8, d13u8; + int16x4_t d0s16, d1s16, d2s16, d3s16, d4s16, d5s16, d6s16, d7s16; + int16x4_t d8s16, d9s16, d10s16, d11s16, d12s16, d13s16, d14s16, d15s16; + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16, d23s16; + int16x4_t d24s16, d25s16, d26s16, d27s16, d28s16, d29s16, d30s16, d31s16; + uint64x1_t d24u64, d25u64, d26u64, d27u64; + int64x1_t d12s64, d13s64; + uint16x8_t q2u16, q3u16, q4u16, q5u16, q8u16; + uint16x8_t q9u16, q12u16, q13u16, q14u16, q15u16; + int16x8_t q0s16, q1s16, q2s16, q3s16, q4s16, q5s16, q6s16, q7s16; + int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16, q13s16, q14s16, q15s16; + int32x4_t q0s32, q1s32, q2s32, q3s32, q4s32, q5s32, q6s32, q8s32, q9s32; + int32x4_t q10s32, q11s32, q12s32, q13s32; + int16x8x2_t q0x2s16; + + q0x2s16 = vld2q_s16(src); + q8s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q9s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q10s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q11s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q12s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q13s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q14s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q15s16 = q0x2s16.val[0]; + + TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + d16s16 = vget_low_s16(q8s16); + d17s16 = vget_high_s16(q8s16); + d18s16 = vget_low_s16(q9s16); + d19s16 = vget_high_s16(q9s16); + d20s16 = vget_low_s16(q10s16); + d21s16 = vget_high_s16(q10s16); + d22s16 = vget_low_s16(q11s16); + d23s16 = vget_high_s16(q11s16); + d24s16 = vget_low_s16(q12s16); + d25s16 = vget_high_s16(q12s16); + d26s16 = vget_low_s16(q13s16); + d27s16 = vget_high_s16(q13s16); + d28s16 = vget_low_s16(q14s16); + d29s16 = vget_high_s16(q14s16); + d30s16 = vget_low_s16(q15s16); + d31s16 = vget_high_s16(q15s16); + + // stage 3 + d12s16 = vdup_n_s16((int16_t)cospi_30_64); + d13s16 = vdup_n_s16((int16_t)cospi_2_64); + + q2s32 = vmull_s16(d16s16, d12s16); + q3s32 = vmull_s16(d17s16, d12s16); + q1s32 = vmull_s16(d16s16, d13s16); + q4s32 = vmull_s16(d17s16, d13s16); + + q2s32 = vmlsl_s16(q2s32, d30s16, d13s16); + q3s32 = vmlsl_s16(q3s32, d31s16, d13s16); + q1s32 = vmlal_s16(q1s32, d30s16, d12s16); + q4s32 = vmlal_s16(q4s32, d31s16, d12s16); + + d0s16 = vqrshrn_n_s32(q2s32, 14); + d1s16 = vqrshrn_n_s32(q3s32, 14); + d14s16 = vqrshrn_n_s32(q1s32, 14); + d15s16 = vqrshrn_n_s32(q4s32, 14); + q0s16 = vcombine_s16(d0s16, d1s16); + q7s16 = vcombine_s16(d14s16, d15s16); + + d30s16 = vdup_n_s16((int16_t)cospi_14_64); + d31s16 = vdup_n_s16((int16_t)cospi_18_64); + + q2s32 = vmull_s16(d24s16, d30s16); + q3s32 = vmull_s16(d25s16, d30s16); + q4s32 = vmull_s16(d24s16, d31s16); + q5s32 = vmull_s16(d25s16, d31s16); + + q2s32 = vmlsl_s16(q2s32, d22s16, d31s16); + q3s32 = vmlsl_s16(q3s32, d23s16, d31s16); + q4s32 = vmlal_s16(q4s32, d22s16, d30s16); + q5s32 = vmlal_s16(q5s32, d23s16, d30s16); + + d2s16 = vqrshrn_n_s32(q2s32, 14); + d3s16 = vqrshrn_n_s32(q3s32, 14); + d12s16 = vqrshrn_n_s32(q4s32, 14); + d13s16 = vqrshrn_n_s32(q5s32, 14); + q1s16 = vcombine_s16(d2s16, d3s16); + q6s16 = vcombine_s16(d12s16, d13s16); + + d30s16 = vdup_n_s16((int16_t)cospi_22_64); + d31s16 = vdup_n_s16((int16_t)cospi_10_64); + + q11s32 = vmull_s16(d20s16, d30s16); + q12s32 = vmull_s16(d21s16, d30s16); + q4s32 = vmull_s16(d20s16, d31s16); + q5s32 = vmull_s16(d21s16, d31s16); + + q11s32 = vmlsl_s16(q11s32, d26s16, d31s16); + q12s32 = vmlsl_s16(q12s32, d27s16, d31s16); + q4s32 = vmlal_s16(q4s32, d26s16, d30s16); + q5s32 = vmlal_s16(q5s32, d27s16, d30s16); + + d4s16 = vqrshrn_n_s32(q11s32, 14); + d5s16 = vqrshrn_n_s32(q12s32, 14); + d11s16 = vqrshrn_n_s32(q5s32, 14); + d10s16 = vqrshrn_n_s32(q4s32, 14); + q2s16 = vcombine_s16(d4s16, d5s16); + q5s16 = vcombine_s16(d10s16, d11s16); + + d30s16 = vdup_n_s16((int16_t)cospi_6_64); + d31s16 = vdup_n_s16((int16_t)cospi_26_64); + + q10s32 = vmull_s16(d28s16, d30s16); + q11s32 = vmull_s16(d29s16, d30s16); + q12s32 = vmull_s16(d28s16, d31s16); + q13s32 = vmull_s16(d29s16, d31s16); + + q10s32 = vmlsl_s16(q10s32, d18s16, d31s16); + q11s32 = vmlsl_s16(q11s32, d19s16, d31s16); + q12s32 = vmlal_s16(q12s32, d18s16, d30s16); + q13s32 = vmlal_s16(q13s32, d19s16, d30s16); + + d6s16 = vqrshrn_n_s32(q10s32, 14); + d7s16 = vqrshrn_n_s32(q11s32, 14); + d8s16 = vqrshrn_n_s32(q12s32, 14); + d9s16 = vqrshrn_n_s32(q13s32, 14); + q3s16 = vcombine_s16(d6s16, d7s16); + q4s16 = vcombine_s16(d8s16, d9s16); + + // stage 3 + q9s16 = vsubq_s16(q0s16, q1s16); + q0s16 = vaddq_s16(q0s16, q1s16); + q10s16 = vsubq_s16(q3s16, q2s16); + q11s16 = vaddq_s16(q2s16, q3s16); + q12s16 = vaddq_s16(q4s16, q5s16); + q13s16 = vsubq_s16(q4s16, q5s16); + q14s16 = vsubq_s16(q7s16, q6s16); + q7s16 = vaddq_s16(q6s16, q7s16); + + // stage 4 + d18s16 = vget_low_s16(q9s16); + d19s16 = vget_high_s16(q9s16); + d20s16 = vget_low_s16(q10s16); + d21s16 = vget_high_s16(q10s16); + d26s16 = vget_low_s16(q13s16); + d27s16 = vget_high_s16(q13s16); + d28s16 = vget_low_s16(q14s16); + d29s16 = vget_high_s16(q14s16); + + d30s16 = vdup_n_s16((int16_t)cospi_8_64); + d31s16 = vdup_n_s16((int16_t)cospi_24_64); + + q2s32 = vmull_s16(d18s16, d31s16); + q3s32 = vmull_s16(d19s16, d31s16); + q4s32 = vmull_s16(d28s16, d31s16); + q5s32 = vmull_s16(d29s16, d31s16); + + q2s32 = vmlal_s16(q2s32, d28s16, d30s16); + q3s32 = vmlal_s16(q3s32, d29s16, d30s16); + q4s32 = vmlsl_s16(q4s32, d18s16, d30s16); + q5s32 = vmlsl_s16(q5s32, d19s16, d30s16); + + d12s16 = vqrshrn_n_s32(q2s32, 14); + d13s16 = vqrshrn_n_s32(q3s32, 14); + d2s16 = vqrshrn_n_s32(q4s32, 14); + d3s16 = vqrshrn_n_s32(q5s32, 14); + q1s16 = vcombine_s16(d2s16, d3s16); + q6s16 = vcombine_s16(d12s16, d13s16); + + q3s16 = q11s16; + q4s16 = q12s16; + + d30s16 = vdup_n_s16(-cospi_8_64); + q11s32 = vmull_s16(d26s16, d30s16); + q12s32 = vmull_s16(d27s16, d30s16); + q8s32 = vmull_s16(d20s16, d30s16); + q9s32 = vmull_s16(d21s16, d30s16); + + q11s32 = vmlsl_s16(q11s32, d20s16, d31s16); + q12s32 = vmlsl_s16(q12s32, d21s16, d31s16); + q8s32 = vmlal_s16(q8s32, d26s16, d31s16); + q9s32 = vmlal_s16(q9s32, d27s16, d31s16); + + d4s16 = vqrshrn_n_s32(q11s32, 14); + d5s16 = vqrshrn_n_s32(q12s32, 14); + d10s16 = vqrshrn_n_s32(q8s32, 14); + d11s16 = vqrshrn_n_s32(q9s32, 14); + q2s16 = vcombine_s16(d4s16, d5s16); + q5s16 = vcombine_s16(d10s16, d11s16); + + // stage 5 + q8s16 = vaddq_s16(q0s16, q3s16); + q9s16 = vaddq_s16(q1s16, q2s16); + q10s16 = vsubq_s16(q1s16, q2s16); + q11s16 = vsubq_s16(q0s16, q3s16); + q12s16 = vsubq_s16(q7s16, q4s16); + q13s16 = vsubq_s16(q6s16, q5s16); + q14s16 = vaddq_s16(q6s16, q5s16); + q15s16 = vaddq_s16(q7s16, q4s16); + + // stage 6 + d20s16 = vget_low_s16(q10s16); + d21s16 = vget_high_s16(q10s16); + d22s16 = vget_low_s16(q11s16); + d23s16 = vget_high_s16(q11s16); + d24s16 = vget_low_s16(q12s16); + d25s16 = vget_high_s16(q12s16); + d26s16 = vget_low_s16(q13s16); + d27s16 = vget_high_s16(q13s16); + + d14s16 = vdup_n_s16((int16_t)cospi_16_64); + + q3s32 = vmull_s16(d26s16, d14s16); + q4s32 = vmull_s16(d27s16, d14s16); + q0s32 = vmull_s16(d20s16, d14s16); + q1s32 = vmull_s16(d21s16, d14s16); + + q5s32 = vsubq_s32(q3s32, q0s32); + q6s32 = vsubq_s32(q4s32, q1s32); + q10s32 = vaddq_s32(q3s32, q0s32); + q4s32 = vaddq_s32(q4s32, q1s32); + + d4s16 = vqrshrn_n_s32(q5s32, 14); + d5s16 = vqrshrn_n_s32(q6s32, 14); + d10s16 = vqrshrn_n_s32(q10s32, 14); + d11s16 = vqrshrn_n_s32(q4s32, 14); + q2s16 = vcombine_s16(d4s16, d5s16); + q5s16 = vcombine_s16(d10s16, d11s16); + + q0s32 = vmull_s16(d22s16, d14s16); + q1s32 = vmull_s16(d23s16, d14s16); + q13s32 = vmull_s16(d24s16, d14s16); + q6s32 = vmull_s16(d25s16, d14s16); + + q10s32 = vsubq_s32(q13s32, q0s32); + q4s32 = vsubq_s32(q6s32, q1s32); + q13s32 = vaddq_s32(q13s32, q0s32); + q6s32 = vaddq_s32(q6s32, q1s32); + + d6s16 = vqrshrn_n_s32(q10s32, 14); + d7s16 = vqrshrn_n_s32(q4s32, 14); + d8s16 = vqrshrn_n_s32(q13s32, 14); + d9s16 = vqrshrn_n_s32(q6s32, 14); + q3s16 = vcombine_s16(d6s16, d7s16); + q4s16 = vcombine_s16(d8s16, d9s16); + + // stage 7 + if (skip_adding != 0) { + d = dest; + // load the data in pass1 + q0s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q1s16 = vld1q_s16(pass1Output); + pass1Output += 8; + d12s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + d13s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + + q12s16 = vaddq_s16(q0s16, q15s16); + q13s16 = vaddq_s16(q1s16, q14s16); + q12s16 = vrshrq_n_s16(q12s16, 6); + q13s16 = vrshrq_n_s16(q13s16, 6); + q12u16 = + vaddw_u8(vreinterpretq_u16_s16(q12s16), vreinterpret_u8_s64(d12s64)); + q13u16 = + vaddw_u8(vreinterpretq_u16_s16(q13s16), vreinterpret_u8_s64(d13s64)); + d12u8 = vqmovun_s16(vreinterpretq_s16_u16(q12u16)); + d13u8 = vqmovun_s16(vreinterpretq_s16_u16(q13u16)); + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d12u8)); + d += dest_stride; + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d13u8)); + d += dest_stride; + q14s16 = vsubq_s16(q1s16, q14s16); + q15s16 = vsubq_s16(q0s16, q15s16); + + q10s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q11s16 = vld1q_s16(pass1Output); + pass1Output += 8; + d12s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + d13s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + q12s16 = vaddq_s16(q10s16, q5s16); + q13s16 = vaddq_s16(q11s16, q4s16); + q12s16 = vrshrq_n_s16(q12s16, 6); + q13s16 = vrshrq_n_s16(q13s16, 6); + q12u16 = + vaddw_u8(vreinterpretq_u16_s16(q12s16), vreinterpret_u8_s64(d12s64)); + q13u16 = + vaddw_u8(vreinterpretq_u16_s16(q13s16), vreinterpret_u8_s64(d13s64)); + d12u8 = vqmovun_s16(vreinterpretq_s16_u16(q12u16)); + d13u8 = vqmovun_s16(vreinterpretq_s16_u16(q13u16)); + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d12u8)); + d += dest_stride; + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d13u8)); + d += dest_stride; + q4s16 = vsubq_s16(q11s16, q4s16); + q5s16 = vsubq_s16(q10s16, q5s16); + + q0s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q1s16 = vld1q_s16(pass1Output); + pass1Output += 8; + d12s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + d13s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + q12s16 = vaddq_s16(q0s16, q3s16); + q13s16 = vaddq_s16(q1s16, q2s16); + q12s16 = vrshrq_n_s16(q12s16, 6); + q13s16 = vrshrq_n_s16(q13s16, 6); + q12u16 = + vaddw_u8(vreinterpretq_u16_s16(q12s16), vreinterpret_u8_s64(d12s64)); + q13u16 = + vaddw_u8(vreinterpretq_u16_s16(q13s16), vreinterpret_u8_s64(d13s64)); + d12u8 = vqmovun_s16(vreinterpretq_s16_u16(q12u16)); + d13u8 = vqmovun_s16(vreinterpretq_s16_u16(q13u16)); + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d12u8)); + d += dest_stride; + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d13u8)); + d += dest_stride; + q2s16 = vsubq_s16(q1s16, q2s16); + q3s16 = vsubq_s16(q0s16, q3s16); + + q10s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q11s16 = vld1q_s16(pass1Output); + d12s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + d13s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + q12s16 = vaddq_s16(q10s16, q9s16); + q13s16 = vaddq_s16(q11s16, q8s16); + q12s16 = vrshrq_n_s16(q12s16, 6); + q13s16 = vrshrq_n_s16(q13s16, 6); + q12u16 = + vaddw_u8(vreinterpretq_u16_s16(q12s16), vreinterpret_u8_s64(d12s64)); + q13u16 = + vaddw_u8(vreinterpretq_u16_s16(q13s16), vreinterpret_u8_s64(d13s64)); + d12u8 = vqmovun_s16(vreinterpretq_s16_u16(q12u16)); + d13u8 = vqmovun_s16(vreinterpretq_s16_u16(q13u16)); + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d12u8)); + d += dest_stride; + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d13u8)); + d += dest_stride; + q8s16 = vsubq_s16(q11s16, q8s16); + q9s16 = vsubq_s16(q10s16, q9s16); + + // store the data out 8,9,10,11,12,13,14,15 + d12s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + q8s16 = vrshrq_n_s16(q8s16, 6); + q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16), vreinterpret_u8_s64(d12s64)); + d12u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16)); + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d12u8)); + d += dest_stride; + + d12s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + q9s16 = vrshrq_n_s16(q9s16, 6); + q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16), vreinterpret_u8_s64(d12s64)); + d12u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16)); + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d12u8)); + d += dest_stride; + + d12s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + q2s16 = vrshrq_n_s16(q2s16, 6); + q2u16 = vaddw_u8(vreinterpretq_u16_s16(q2s16), vreinterpret_u8_s64(d12s64)); + d12u8 = vqmovun_s16(vreinterpretq_s16_u16(q2u16)); + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d12u8)); + d += dest_stride; + + d12s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + q3s16 = vrshrq_n_s16(q3s16, 6); + q3u16 = vaddw_u8(vreinterpretq_u16_s16(q3s16), vreinterpret_u8_s64(d12s64)); + d12u8 = vqmovun_s16(vreinterpretq_s16_u16(q3u16)); + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d12u8)); + d += dest_stride; + + d12s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + q4s16 = vrshrq_n_s16(q4s16, 6); + q4u16 = vaddw_u8(vreinterpretq_u16_s16(q4s16), vreinterpret_u8_s64(d12s64)); + d12u8 = vqmovun_s16(vreinterpretq_s16_u16(q4u16)); + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d12u8)); + d += dest_stride; + + d12s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + q5s16 = vrshrq_n_s16(q5s16, 6); + q5u16 = vaddw_u8(vreinterpretq_u16_s16(q5s16), vreinterpret_u8_s64(d12s64)); + d12u8 = vqmovun_s16(vreinterpretq_s16_u16(q5u16)); + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d12u8)); + d += dest_stride; + + d12s64 = vld1_s64((int64_t *)dest); + dest += dest_stride; + q14s16 = vrshrq_n_s16(q14s16, 6); + q14u16 = + vaddw_u8(vreinterpretq_u16_s16(q14s16), vreinterpret_u8_s64(d12s64)); + d12u8 = vqmovun_s16(vreinterpretq_s16_u16(q14u16)); + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d12u8)); + d += dest_stride; + + d12s64 = vld1_s64((int64_t *)dest); + q15s16 = vrshrq_n_s16(q15s16, 6); + q15u16 = + vaddw_u8(vreinterpretq_u16_s16(q15s16), vreinterpret_u8_s64(d12s64)); + d12u8 = vqmovun_s16(vreinterpretq_s16_u16(q15u16)); + vst1_u64((uint64_t *)d, vreinterpret_u64_u8(d12u8)); + } else { // skip_adding_dest + q0s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q1s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q12s16 = vaddq_s16(q0s16, q15s16); + q13s16 = vaddq_s16(q1s16, q14s16); + d24u64 = vreinterpret_u64_s16(vget_low_s16(q12s16)); + d25u64 = vreinterpret_u64_s16(vget_high_s16(q12s16)); + d26u64 = vreinterpret_u64_s16(vget_low_s16(q13s16)); + d27u64 = vreinterpret_u64_s16(vget_high_s16(q13s16)); + vst1_u64((uint64_t *)out, d24u64); + out += 4; + vst1_u64((uint64_t *)out, d25u64); + out += 12; + vst1_u64((uint64_t *)out, d26u64); + out += 4; + vst1_u64((uint64_t *)out, d27u64); + out += 12; + q14s16 = vsubq_s16(q1s16, q14s16); + q15s16 = vsubq_s16(q0s16, q15s16); + + q10s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q11s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q12s16 = vaddq_s16(q10s16, q5s16); + q13s16 = vaddq_s16(q11s16, q4s16); + d24u64 = vreinterpret_u64_s16(vget_low_s16(q12s16)); + d25u64 = vreinterpret_u64_s16(vget_high_s16(q12s16)); + d26u64 = vreinterpret_u64_s16(vget_low_s16(q13s16)); + d27u64 = vreinterpret_u64_s16(vget_high_s16(q13s16)); + vst1_u64((uint64_t *)out, d24u64); + out += 4; + vst1_u64((uint64_t *)out, d25u64); + out += 12; + vst1_u64((uint64_t *)out, d26u64); + out += 4; + vst1_u64((uint64_t *)out, d27u64); + out += 12; + q4s16 = vsubq_s16(q11s16, q4s16); + q5s16 = vsubq_s16(q10s16, q5s16); + + q0s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q1s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q12s16 = vaddq_s16(q0s16, q3s16); + q13s16 = vaddq_s16(q1s16, q2s16); + d24u64 = vreinterpret_u64_s16(vget_low_s16(q12s16)); + d25u64 = vreinterpret_u64_s16(vget_high_s16(q12s16)); + d26u64 = vreinterpret_u64_s16(vget_low_s16(q13s16)); + d27u64 = vreinterpret_u64_s16(vget_high_s16(q13s16)); + vst1_u64((uint64_t *)out, d24u64); + out += 4; + vst1_u64((uint64_t *)out, d25u64); + out += 12; + vst1_u64((uint64_t *)out, d26u64); + out += 4; + vst1_u64((uint64_t *)out, d27u64); + out += 12; + q2s16 = vsubq_s16(q1s16, q2s16); + q3s16 = vsubq_s16(q0s16, q3s16); + + q10s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q11s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q12s16 = vaddq_s16(q10s16, q9s16); + q13s16 = vaddq_s16(q11s16, q8s16); + d24u64 = vreinterpret_u64_s16(vget_low_s16(q12s16)); + d25u64 = vreinterpret_u64_s16(vget_high_s16(q12s16)); + d26u64 = vreinterpret_u64_s16(vget_low_s16(q13s16)); + d27u64 = vreinterpret_u64_s16(vget_high_s16(q13s16)); + vst1_u64((uint64_t *)out, d24u64); + out += 4; + vst1_u64((uint64_t *)out, d25u64); + out += 12; + vst1_u64((uint64_t *)out, d26u64); + out += 4; + vst1_u64((uint64_t *)out, d27u64); + out += 12; + q8s16 = vsubq_s16(q11s16, q8s16); + q9s16 = vsubq_s16(q10s16, q9s16); + + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_low_s16(q8s16))); + out += 4; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_high_s16(q8s16))); + out += 12; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_low_s16(q9s16))); + out += 4; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_high_s16(q9s16))); + out += 12; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_low_s16(q2s16))); + out += 4; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_high_s16(q2s16))); + out += 12; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_low_s16(q3s16))); + out += 4; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_high_s16(q3s16))); + out += 12; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_low_s16(q4s16))); + out += 4; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_high_s16(q4s16))); + out += 12; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_low_s16(q5s16))); + out += 4; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_high_s16(q5s16))); + out += 12; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_low_s16(q14s16))); + out += 4; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_high_s16(q14s16))); + out += 12; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_low_s16(q15s16))); + out += 4; + vst1_u64((uint64_t *)out, vreinterpret_u64_s16(vget_high_s16(q15s16))); + } + return; +} + +void aom_idct16x16_10_add_neon_pass1(int16_t *in, int16_t *out, + int output_stride) { + int16x4_t d4s16; + int16x4_t d8s16, d9s16, d10s16, d11s16, d12s16, d13s16, d14s16, d15s16; + uint64x1_t d4u64, d5u64, d18u64, d19u64, d20u64, d21u64, d22u64, d23u64; + uint64x1_t d24u64, d25u64, d26u64, d27u64, d28u64, d29u64, d30u64, d31u64; + int16x8_t q0s16, q1s16, q2s16, q4s16, q5s16, q6s16, q7s16; + int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16, q13s16, q14s16, q15s16; + int32x4_t q6s32, q9s32; + int32x4_t q10s32, q11s32, q12s32, q15s32; + int16x8x2_t q0x2s16; + + q0x2s16 = vld2q_s16(in); + q8s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q9s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q10s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q11s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q12s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q13s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q14s16 = q0x2s16.val[0]; + in += 16; + q0x2s16 = vld2q_s16(in); + q15s16 = q0x2s16.val[0]; + + TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + // stage 3 + q0s16 = vdupq_n_s16((int16_t)(cospi_28_64 * 2)); + q1s16 = vdupq_n_s16((int16_t)(cospi_4_64 * 2)); + + q4s16 = vqrdmulhq_s16(q9s16, q0s16); + q7s16 = vqrdmulhq_s16(q9s16, q1s16); + + // stage 4 + q1s16 = vdupq_n_s16((int16_t)(cospi_16_64 * 2)); + d4s16 = vdup_n_s16((int16_t)cospi_16_64); + + q8s16 = vqrdmulhq_s16(q8s16, q1s16); + + d8s16 = vget_low_s16(q4s16); + d9s16 = vget_high_s16(q4s16); + d14s16 = vget_low_s16(q7s16); + d15s16 = vget_high_s16(q7s16); + q9s32 = vmull_s16(d14s16, d4s16); + q10s32 = vmull_s16(d15s16, d4s16); + q12s32 = vmull_s16(d9s16, d4s16); + q11s32 = vmull_s16(d8s16, d4s16); + + q15s32 = vsubq_s32(q10s32, q12s32); + q6s32 = vsubq_s32(q9s32, q11s32); + q9s32 = vaddq_s32(q9s32, q11s32); + q10s32 = vaddq_s32(q10s32, q12s32); + + d11s16 = vqrshrn_n_s32(q15s32, 14); + d10s16 = vqrshrn_n_s32(q6s32, 14); + d12s16 = vqrshrn_n_s32(q9s32, 14); + d13s16 = vqrshrn_n_s32(q10s32, 14); + q5s16 = vcombine_s16(d10s16, d11s16); + q6s16 = vcombine_s16(d12s16, d13s16); + + // stage 6 + q2s16 = vaddq_s16(q8s16, q7s16); + q9s16 = vaddq_s16(q8s16, q6s16); + q10s16 = vaddq_s16(q8s16, q5s16); + q11s16 = vaddq_s16(q8s16, q4s16); + q12s16 = vsubq_s16(q8s16, q4s16); + q13s16 = vsubq_s16(q8s16, q5s16); + q14s16 = vsubq_s16(q8s16, q6s16); + q15s16 = vsubq_s16(q8s16, q7s16); + + d4u64 = vreinterpret_u64_s16(vget_low_s16(q2s16)); + d5u64 = vreinterpret_u64_s16(vget_high_s16(q2s16)); + d18u64 = vreinterpret_u64_s16(vget_low_s16(q9s16)); + d19u64 = vreinterpret_u64_s16(vget_high_s16(q9s16)); + d20u64 = vreinterpret_u64_s16(vget_low_s16(q10s16)); + d21u64 = vreinterpret_u64_s16(vget_high_s16(q10s16)); + d22u64 = vreinterpret_u64_s16(vget_low_s16(q11s16)); + d23u64 = vreinterpret_u64_s16(vget_high_s16(q11s16)); + d24u64 = vreinterpret_u64_s16(vget_low_s16(q12s16)); + d25u64 = vreinterpret_u64_s16(vget_high_s16(q12s16)); + d26u64 = vreinterpret_u64_s16(vget_low_s16(q13s16)); + d27u64 = vreinterpret_u64_s16(vget_high_s16(q13s16)); + d28u64 = vreinterpret_u64_s16(vget_low_s16(q14s16)); + d29u64 = vreinterpret_u64_s16(vget_high_s16(q14s16)); + d30u64 = vreinterpret_u64_s16(vget_low_s16(q15s16)); + d31u64 = vreinterpret_u64_s16(vget_high_s16(q15s16)); + + // store the data + output_stride >>= 1; // output_stride / 2, out is int16_t + vst1_u64((uint64_t *)out, d4u64); + out += output_stride; + vst1_u64((uint64_t *)out, d5u64); + out += output_stride; + vst1_u64((uint64_t *)out, d18u64); + out += output_stride; + vst1_u64((uint64_t *)out, d19u64); + out += output_stride; + vst1_u64((uint64_t *)out, d20u64); + out += output_stride; + vst1_u64((uint64_t *)out, d21u64); + out += output_stride; + vst1_u64((uint64_t *)out, d22u64); + out += output_stride; + vst1_u64((uint64_t *)out, d23u64); + out += output_stride; + vst1_u64((uint64_t *)out, d24u64); + out += output_stride; + vst1_u64((uint64_t *)out, d25u64); + out += output_stride; + vst1_u64((uint64_t *)out, d26u64); + out += output_stride; + vst1_u64((uint64_t *)out, d27u64); + out += output_stride; + vst1_u64((uint64_t *)out, d28u64); + out += output_stride; + vst1_u64((uint64_t *)out, d29u64); + out += output_stride; + vst1_u64((uint64_t *)out, d30u64); + out += output_stride; + vst1_u64((uint64_t *)out, d31u64); + return; +} + +void aom_idct16x16_10_add_neon_pass2(int16_t *src, int16_t *out, + int16_t *pass1Output, int16_t skip_adding, + uint8_t *dest, int dest_stride) { + int16x4_t d0s16, d1s16, d2s16, d3s16, d4s16, d5s16, d6s16, d7s16; + int16x4_t d8s16, d9s16, d10s16, d11s16, d12s16, d13s16, d14s16, d15s16; + int16x4_t d20s16, d21s16, d22s16, d23s16; + int16x4_t d24s16, d25s16, d26s16, d27s16, d30s16, d31s16; + uint64x1_t d4u64, d5u64, d6u64, d7u64, d8u64, d9u64, d10u64, d11u64; + uint64x1_t d16u64, d17u64, d18u64, d19u64; + uint64x1_t d24u64, d25u64, d26u64, d27u64, d28u64, d29u64, d30u64, d31u64; + int16x8_t q0s16, q1s16, q2s16, q3s16, q4s16, q5s16, q6s16, q7s16; + int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16, q13s16, q14s16, q15s16; + int32x4_t q0s32, q1s32, q2s32, q3s32, q4s32, q5s32, q6s32, q8s32, q9s32; + int32x4_t q10s32, q11s32, q12s32, q13s32; + int16x8x2_t q0x2s16; + (void)skip_adding; + (void)dest; + (void)dest_stride; + + q0x2s16 = vld2q_s16(src); + q8s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q9s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q10s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q11s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q12s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q13s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q14s16 = q0x2s16.val[0]; + src += 16; + q0x2s16 = vld2q_s16(src); + q15s16 = q0x2s16.val[0]; + + TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + // stage 3 + q6s16 = vdupq_n_s16((int16_t)(cospi_30_64 * 2)); + q0s16 = vqrdmulhq_s16(q8s16, q6s16); + q6s16 = vdupq_n_s16((int16_t)(cospi_2_64 * 2)); + q7s16 = vqrdmulhq_s16(q8s16, q6s16); + + q15s16 = vdupq_n_s16(-cospi_26_64 * 2); + q14s16 = vdupq_n_s16((int16_t)(cospi_6_64 * 2)); + q3s16 = vqrdmulhq_s16(q9s16, q15s16); + q4s16 = vqrdmulhq_s16(q9s16, q14s16); + + // stage 4 + d0s16 = vget_low_s16(q0s16); + d1s16 = vget_high_s16(q0s16); + d6s16 = vget_low_s16(q3s16); + d7s16 = vget_high_s16(q3s16); + d8s16 = vget_low_s16(q4s16); + d9s16 = vget_high_s16(q4s16); + d14s16 = vget_low_s16(q7s16); + d15s16 = vget_high_s16(q7s16); + + d30s16 = vdup_n_s16((int16_t)cospi_8_64); + d31s16 = vdup_n_s16((int16_t)cospi_24_64); + + q12s32 = vmull_s16(d14s16, d31s16); + q5s32 = vmull_s16(d15s16, d31s16); + q2s32 = vmull_s16(d0s16, d31s16); + q11s32 = vmull_s16(d1s16, d31s16); + + q12s32 = vmlsl_s16(q12s32, d0s16, d30s16); + q5s32 = vmlsl_s16(q5s32, d1s16, d30s16); + q2s32 = vmlal_s16(q2s32, d14s16, d30s16); + q11s32 = vmlal_s16(q11s32, d15s16, d30s16); + + d2s16 = vqrshrn_n_s32(q12s32, 14); + d3s16 = vqrshrn_n_s32(q5s32, 14); + d12s16 = vqrshrn_n_s32(q2s32, 14); + d13s16 = vqrshrn_n_s32(q11s32, 14); + q1s16 = vcombine_s16(d2s16, d3s16); + q6s16 = vcombine_s16(d12s16, d13s16); + + d30s16 = vdup_n_s16(-cospi_8_64); + q10s32 = vmull_s16(d8s16, d30s16); + q13s32 = vmull_s16(d9s16, d30s16); + q8s32 = vmull_s16(d6s16, d30s16); + q9s32 = vmull_s16(d7s16, d30s16); + + q10s32 = vmlsl_s16(q10s32, d6s16, d31s16); + q13s32 = vmlsl_s16(q13s32, d7s16, d31s16); + q8s32 = vmlal_s16(q8s32, d8s16, d31s16); + q9s32 = vmlal_s16(q9s32, d9s16, d31s16); + + d4s16 = vqrshrn_n_s32(q10s32, 14); + d5s16 = vqrshrn_n_s32(q13s32, 14); + d10s16 = vqrshrn_n_s32(q8s32, 14); + d11s16 = vqrshrn_n_s32(q9s32, 14); + q2s16 = vcombine_s16(d4s16, d5s16); + q5s16 = vcombine_s16(d10s16, d11s16); + + // stage 5 + q8s16 = vaddq_s16(q0s16, q3s16); + q9s16 = vaddq_s16(q1s16, q2s16); + q10s16 = vsubq_s16(q1s16, q2s16); + q11s16 = vsubq_s16(q0s16, q3s16); + q12s16 = vsubq_s16(q7s16, q4s16); + q13s16 = vsubq_s16(q6s16, q5s16); + q14s16 = vaddq_s16(q6s16, q5s16); + q15s16 = vaddq_s16(q7s16, q4s16); + + // stage 6 + d20s16 = vget_low_s16(q10s16); + d21s16 = vget_high_s16(q10s16); + d22s16 = vget_low_s16(q11s16); + d23s16 = vget_high_s16(q11s16); + d24s16 = vget_low_s16(q12s16); + d25s16 = vget_high_s16(q12s16); + d26s16 = vget_low_s16(q13s16); + d27s16 = vget_high_s16(q13s16); + + d14s16 = vdup_n_s16((int16_t)cospi_16_64); + q3s32 = vmull_s16(d26s16, d14s16); + q4s32 = vmull_s16(d27s16, d14s16); + q0s32 = vmull_s16(d20s16, d14s16); + q1s32 = vmull_s16(d21s16, d14s16); + + q5s32 = vsubq_s32(q3s32, q0s32); + q6s32 = vsubq_s32(q4s32, q1s32); + q0s32 = vaddq_s32(q3s32, q0s32); + q4s32 = vaddq_s32(q4s32, q1s32); + + d4s16 = vqrshrn_n_s32(q5s32, 14); + d5s16 = vqrshrn_n_s32(q6s32, 14); + d10s16 = vqrshrn_n_s32(q0s32, 14); + d11s16 = vqrshrn_n_s32(q4s32, 14); + q2s16 = vcombine_s16(d4s16, d5s16); + q5s16 = vcombine_s16(d10s16, d11s16); + + q0s32 = vmull_s16(d22s16, d14s16); + q1s32 = vmull_s16(d23s16, d14s16); + q13s32 = vmull_s16(d24s16, d14s16); + q6s32 = vmull_s16(d25s16, d14s16); + + q10s32 = vsubq_s32(q13s32, q0s32); + q4s32 = vsubq_s32(q6s32, q1s32); + q13s32 = vaddq_s32(q13s32, q0s32); + q6s32 = vaddq_s32(q6s32, q1s32); + + d6s16 = vqrshrn_n_s32(q10s32, 14); + d7s16 = vqrshrn_n_s32(q4s32, 14); + d8s16 = vqrshrn_n_s32(q13s32, 14); + d9s16 = vqrshrn_n_s32(q6s32, 14); + q3s16 = vcombine_s16(d6s16, d7s16); + q4s16 = vcombine_s16(d8s16, d9s16); + + // stage 7 + q0s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q1s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q12s16 = vaddq_s16(q0s16, q15s16); + q13s16 = vaddq_s16(q1s16, q14s16); + d24u64 = vreinterpret_u64_s16(vget_low_s16(q12s16)); + d25u64 = vreinterpret_u64_s16(vget_high_s16(q12s16)); + d26u64 = vreinterpret_u64_s16(vget_low_s16(q13s16)); + d27u64 = vreinterpret_u64_s16(vget_high_s16(q13s16)); + vst1_u64((uint64_t *)out, d24u64); + out += 4; + vst1_u64((uint64_t *)out, d25u64); + out += 12; + vst1_u64((uint64_t *)out, d26u64); + out += 4; + vst1_u64((uint64_t *)out, d27u64); + out += 12; + q14s16 = vsubq_s16(q1s16, q14s16); + q15s16 = vsubq_s16(q0s16, q15s16); + + q10s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q11s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q12s16 = vaddq_s16(q10s16, q5s16); + q13s16 = vaddq_s16(q11s16, q4s16); + d24u64 = vreinterpret_u64_s16(vget_low_s16(q12s16)); + d25u64 = vreinterpret_u64_s16(vget_high_s16(q12s16)); + d26u64 = vreinterpret_u64_s16(vget_low_s16(q13s16)); + d27u64 = vreinterpret_u64_s16(vget_high_s16(q13s16)); + vst1_u64((uint64_t *)out, d24u64); + out += 4; + vst1_u64((uint64_t *)out, d25u64); + out += 12; + vst1_u64((uint64_t *)out, d26u64); + out += 4; + vst1_u64((uint64_t *)out, d27u64); + out += 12; + q4s16 = vsubq_s16(q11s16, q4s16); + q5s16 = vsubq_s16(q10s16, q5s16); + + q0s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q1s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q12s16 = vaddq_s16(q0s16, q3s16); + q13s16 = vaddq_s16(q1s16, q2s16); + d24u64 = vreinterpret_u64_s16(vget_low_s16(q12s16)); + d25u64 = vreinterpret_u64_s16(vget_high_s16(q12s16)); + d26u64 = vreinterpret_u64_s16(vget_low_s16(q13s16)); + d27u64 = vreinterpret_u64_s16(vget_high_s16(q13s16)); + vst1_u64((uint64_t *)out, d24u64); + out += 4; + vst1_u64((uint64_t *)out, d25u64); + out += 12; + vst1_u64((uint64_t *)out, d26u64); + out += 4; + vst1_u64((uint64_t *)out, d27u64); + out += 12; + q2s16 = vsubq_s16(q1s16, q2s16); + q3s16 = vsubq_s16(q0s16, q3s16); + + q10s16 = vld1q_s16(pass1Output); + pass1Output += 8; + q11s16 = vld1q_s16(pass1Output); + q12s16 = vaddq_s16(q10s16, q9s16); + q13s16 = vaddq_s16(q11s16, q8s16); + d24u64 = vreinterpret_u64_s16(vget_low_s16(q12s16)); + d25u64 = vreinterpret_u64_s16(vget_high_s16(q12s16)); + d26u64 = vreinterpret_u64_s16(vget_low_s16(q13s16)); + d27u64 = vreinterpret_u64_s16(vget_high_s16(q13s16)); + vst1_u64((uint64_t *)out, d24u64); + out += 4; + vst1_u64((uint64_t *)out, d25u64); + out += 12; + vst1_u64((uint64_t *)out, d26u64); + out += 4; + vst1_u64((uint64_t *)out, d27u64); + out += 12; + q8s16 = vsubq_s16(q11s16, q8s16); + q9s16 = vsubq_s16(q10s16, q9s16); + + d4u64 = vreinterpret_u64_s16(vget_low_s16(q2s16)); + d5u64 = vreinterpret_u64_s16(vget_high_s16(q2s16)); + d6u64 = vreinterpret_u64_s16(vget_low_s16(q3s16)); + d7u64 = vreinterpret_u64_s16(vget_high_s16(q3s16)); + d8u64 = vreinterpret_u64_s16(vget_low_s16(q4s16)); + d9u64 = vreinterpret_u64_s16(vget_high_s16(q4s16)); + d10u64 = vreinterpret_u64_s16(vget_low_s16(q5s16)); + d11u64 = vreinterpret_u64_s16(vget_high_s16(q5s16)); + d16u64 = vreinterpret_u64_s16(vget_low_s16(q8s16)); + d17u64 = vreinterpret_u64_s16(vget_high_s16(q8s16)); + d18u64 = vreinterpret_u64_s16(vget_low_s16(q9s16)); + d19u64 = vreinterpret_u64_s16(vget_high_s16(q9s16)); + d28u64 = vreinterpret_u64_s16(vget_low_s16(q14s16)); + d29u64 = vreinterpret_u64_s16(vget_high_s16(q14s16)); + d30u64 = vreinterpret_u64_s16(vget_low_s16(q15s16)); + d31u64 = vreinterpret_u64_s16(vget_high_s16(q15s16)); + + vst1_u64((uint64_t *)out, d16u64); + out += 4; + vst1_u64((uint64_t *)out, d17u64); + out += 12; + vst1_u64((uint64_t *)out, d18u64); + out += 4; + vst1_u64((uint64_t *)out, d19u64); + out += 12; + vst1_u64((uint64_t *)out, d4u64); + out += 4; + vst1_u64((uint64_t *)out, d5u64); + out += 12; + vst1_u64((uint64_t *)out, d6u64); + out += 4; + vst1_u64((uint64_t *)out, d7u64); + out += 12; + vst1_u64((uint64_t *)out, d8u64); + out += 4; + vst1_u64((uint64_t *)out, d9u64); + out += 12; + vst1_u64((uint64_t *)out, d10u64); + out += 4; + vst1_u64((uint64_t *)out, d11u64); + out += 12; + vst1_u64((uint64_t *)out, d28u64); + out += 4; + vst1_u64((uint64_t *)out, d29u64); + out += 12; + vst1_u64((uint64_t *)out, d30u64); + out += 4; + vst1_u64((uint64_t *)out, d31u64); + return; +} diff --git a/third_party/aom/aom_dsp/arm/idct16x16_neon.c b/third_party/aom/aom_dsp/arm/idct16x16_neon.c new file mode 100644 index 0000000000..db0d4905b5 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct16x16_neon.c @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/aom_dsp_common.h" + +void aom_idct16x16_256_add_neon_pass1(const int16_t *input, int16_t *output, + int output_stride); +void aom_idct16x16_256_add_neon_pass2(const int16_t *src, int16_t *output, + int16_t *pass1Output, int16_t skip_adding, + uint8_t *dest, int dest_stride); +void aom_idct16x16_10_add_neon_pass1(const int16_t *input, int16_t *output, + int output_stride); +void aom_idct16x16_10_add_neon_pass2(const int16_t *src, int16_t *output, + int16_t *pass1Output, int16_t skip_adding, + uint8_t *dest, int dest_stride); + +#if HAVE_NEON_ASM +/* For ARM NEON, d8-d15 are callee-saved registers, and need to be saved. */ +extern void aom_push_neon(int64_t *store); +extern void aom_pop_neon(int64_t *store); +#endif // HAVE_NEON_ASM + +void aom_idct16x16_256_add_neon(const int16_t *input, uint8_t *dest, + int dest_stride) { +#if HAVE_NEON_ASM + int64_t store_reg[8]; +#endif + int16_t pass1_output[16 * 16] = { 0 }; + int16_t row_idct_output[16 * 16] = { 0 }; + +#if HAVE_NEON_ASM + // save d8-d15 register values. + aom_push_neon(store_reg); +#endif + + /* Parallel idct on the upper 8 rows */ + // First pass processes even elements 0, 2, 4, 6, 8, 10, 12, 14 and save the + // stage 6 result in pass1_output. + aom_idct16x16_256_add_neon_pass1(input, pass1_output, 8); + + // Second pass processes odd elements 1, 3, 5, 7, 9, 11, 13, 15 and combines + // with result in pass1(pass1_output) to calculate final result in stage 7 + // which will be saved into row_idct_output. + aom_idct16x16_256_add_neon_pass2(input + 1, row_idct_output, pass1_output, 0, + dest, dest_stride); + + /* Parallel idct on the lower 8 rows */ + // First pass processes even elements 0, 2, 4, 6, 8, 10, 12, 14 and save the + // stage 6 result in pass1_output. + aom_idct16x16_256_add_neon_pass1(input + 8 * 16, pass1_output, 8); + + // Second pass processes odd elements 1, 3, 5, 7, 9, 11, 13, 15 and combines + // with result in pass1(pass1_output) to calculate final result in stage 7 + // which will be saved into row_idct_output. + aom_idct16x16_256_add_neon_pass2(input + 8 * 16 + 1, row_idct_output + 8, + pass1_output, 0, dest, dest_stride); + + /* Parallel idct on the left 8 columns */ + // First pass processes even elements 0, 2, 4, 6, 8, 10, 12, 14 and save the + // stage 6 result in pass1_output. + aom_idct16x16_256_add_neon_pass1(row_idct_output, pass1_output, 8); + + // Second pass processes odd elements 1, 3, 5, 7, 9, 11, 13, 15 and combines + // with result in pass1(pass1_output) to calculate final result in stage 7. + // Then add the result to the destination data. + aom_idct16x16_256_add_neon_pass2(row_idct_output + 1, row_idct_output, + pass1_output, 1, dest, dest_stride); + + /* Parallel idct on the right 8 columns */ + // First pass processes even elements 0, 2, 4, 6, 8, 10, 12, 14 and save the + // stage 6 result in pass1_output. + aom_idct16x16_256_add_neon_pass1(row_idct_output + 8 * 16, pass1_output, 8); + + // Second pass processes odd elements 1, 3, 5, 7, 9, 11, 13, 15 and combines + // with result in pass1(pass1_output) to calculate final result in stage 7. + // Then add the result to the destination data. + aom_idct16x16_256_add_neon_pass2(row_idct_output + 8 * 16 + 1, + row_idct_output + 8, pass1_output, 1, + dest + 8, dest_stride); + +#if HAVE_NEON_ASM + // restore d8-d15 register values. + aom_pop_neon(store_reg); +#endif + + return; +} + +void aom_idct16x16_10_add_neon(const int16_t *input, uint8_t *dest, + int dest_stride) { +#if HAVE_NEON_ASM + int64_t store_reg[8]; +#endif + int16_t pass1_output[16 * 16] = { 0 }; + int16_t row_idct_output[16 * 16] = { 0 }; + +#if HAVE_NEON_ASM + // save d8-d15 register values. + aom_push_neon(store_reg); +#endif + + /* Parallel idct on the upper 8 rows */ + // First pass processes even elements 0, 2, 4, 6, 8, 10, 12, 14 and save the + // stage 6 result in pass1_output. + aom_idct16x16_10_add_neon_pass1(input, pass1_output, 8); + + // Second pass processes odd elements 1, 3, 5, 7, 9, 11, 13, 15 and combines + // with result in pass1(pass1_output) to calculate final result in stage 7 + // which will be saved into row_idct_output. + aom_idct16x16_10_add_neon_pass2(input + 1, row_idct_output, pass1_output, 0, + dest, dest_stride); + + /* Skip Parallel idct on the lower 8 rows as they are all 0s */ + + /* Parallel idct on the left 8 columns */ + // First pass processes even elements 0, 2, 4, 6, 8, 10, 12, 14 and save the + // stage 6 result in pass1_output. + aom_idct16x16_256_add_neon_pass1(row_idct_output, pass1_output, 8); + + // Second pass processes odd elements 1, 3, 5, 7, 9, 11, 13, 15 and combines + // with result in pass1(pass1_output) to calculate final result in stage 7. + // Then add the result to the destination data. + aom_idct16x16_256_add_neon_pass2(row_idct_output + 1, row_idct_output, + pass1_output, 1, dest, dest_stride); + + /* Parallel idct on the right 8 columns */ + // First pass processes even elements 0, 2, 4, 6, 8, 10, 12, 14 and save the + // stage 6 result in pass1_output. + aom_idct16x16_256_add_neon_pass1(row_idct_output + 8 * 16, pass1_output, 8); + + // Second pass processes odd elements 1, 3, 5, 7, 9, 11, 13, 15 and combines + // with result in pass1(pass1_output) to calculate final result in stage 7. + // Then add the result to the destination data. + aom_idct16x16_256_add_neon_pass2(row_idct_output + 8 * 16 + 1, + row_idct_output + 8, pass1_output, 1, + dest + 8, dest_stride); + +#if HAVE_NEON_ASM + // restore d8-d15 register values. + aom_pop_neon(store_reg); +#endif + + return; +} diff --git a/third_party/aom/aom_dsp/arm/idct32x32_1_add_neon.asm b/third_party/aom/aom_dsp/arm/idct32x32_1_add_neon.asm new file mode 100644 index 0000000000..b04df2d0b8 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct32x32_1_add_neon.asm @@ -0,0 +1,147 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + + + EXPORT |aom_idct32x32_1_add_neon| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + + ;TODO(hkuang): put the following macros in a seperate + ;file so other idct function could also use them. + MACRO + LD_16x8 $src, $stride + vld1.8 {q8}, [$src], $stride + vld1.8 {q9}, [$src], $stride + vld1.8 {q10}, [$src], $stride + vld1.8 {q11}, [$src], $stride + vld1.8 {q12}, [$src], $stride + vld1.8 {q13}, [$src], $stride + vld1.8 {q14}, [$src], $stride + vld1.8 {q15}, [$src], $stride + MEND + + MACRO + ADD_DIFF_16x8 $diff + vqadd.u8 q8, q8, $diff + vqadd.u8 q9, q9, $diff + vqadd.u8 q10, q10, $diff + vqadd.u8 q11, q11, $diff + vqadd.u8 q12, q12, $diff + vqadd.u8 q13, q13, $diff + vqadd.u8 q14, q14, $diff + vqadd.u8 q15, q15, $diff + MEND + + MACRO + SUB_DIFF_16x8 $diff + vqsub.u8 q8, q8, $diff + vqsub.u8 q9, q9, $diff + vqsub.u8 q10, q10, $diff + vqsub.u8 q11, q11, $diff + vqsub.u8 q12, q12, $diff + vqsub.u8 q13, q13, $diff + vqsub.u8 q14, q14, $diff + vqsub.u8 q15, q15, $diff + MEND + + MACRO + ST_16x8 $dst, $stride + vst1.8 {q8}, [$dst], $stride + vst1.8 {q9}, [$dst], $stride + vst1.8 {q10},[$dst], $stride + vst1.8 {q11},[$dst], $stride + vst1.8 {q12},[$dst], $stride + vst1.8 {q13},[$dst], $stride + vst1.8 {q14},[$dst], $stride + vst1.8 {q15},[$dst], $stride + MEND + +;void aom_idct32x32_1_add_neon(int16_t *input, uint8_t *dest, +; int dest_stride) +; +; r0 int16_t input +; r1 uint8_t *dest +; r2 int dest_stride + +|aom_idct32x32_1_add_neon| PROC + push {lr} + pld [r1] + add r3, r1, #16 ; r3 dest + 16 for second loop + ldrsh r0, [r0] + + ; generate cospi_16_64 = 11585 + mov r12, #0x2d00 + add r12, #0x41 + + ; out = dct_const_round_shift(input[0] * cospi_16_64) + mul r0, r0, r12 ; input[0] * cospi_16_64 + add r0, r0, #0x2000 ; +(1 << ((DCT_CONST_BITS) - 1)) + asr r0, r0, #14 ; >> DCT_CONST_BITS + + ; out = dct_const_round_shift(out * cospi_16_64) + mul r0, r0, r12 ; out * cospi_16_64 + mov r12, r1 ; save dest + add r0, r0, #0x2000 ; +(1 << ((DCT_CONST_BITS) - 1)) + asr r0, r0, #14 ; >> DCT_CONST_BITS + + ; a1 = ROUND_POWER_OF_TWO(out, 6) + add r0, r0, #32 ; + (1 <<((6) - 1)) + asrs r0, r0, #6 ; >> 6 + bge diff_positive_32_32 + +diff_negative_32_32 + neg r0, r0 + usat r0, #8, r0 + vdup.u8 q0, r0 + mov r0, #4 + +diff_negative_32_32_loop + sub r0, #1 + LD_16x8 r1, r2 + SUB_DIFF_16x8 q0 + ST_16x8 r12, r2 + + LD_16x8 r1, r2 + SUB_DIFF_16x8 q0 + ST_16x8 r12, r2 + cmp r0, #2 + moveq r1, r3 + moveq r12, r3 + cmp r0, #0 + bne diff_negative_32_32_loop + pop {pc} + +diff_positive_32_32 + usat r0, #8, r0 + vdup.u8 q0, r0 + mov r0, #4 + +diff_positive_32_32_loop + sub r0, #1 + LD_16x8 r1, r2 + ADD_DIFF_16x8 q0 + ST_16x8 r12, r2 + + LD_16x8 r1, r2 + ADD_DIFF_16x8 q0 + ST_16x8 r12, r2 + cmp r0, #2 + moveq r1, r3 + moveq r12, r3 + cmp r0, #0 + bne diff_positive_32_32_loop + pop {pc} + + ENDP ; |aom_idct32x32_1_add_neon| + END diff --git a/third_party/aom/aom_dsp/arm/idct32x32_1_add_neon.c b/third_party/aom/aom_dsp/arm/idct32x32_1_add_neon.c new file mode 100644 index 0000000000..547567c5b4 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct32x32_1_add_neon.c @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" + +#include "aom_dsp/inv_txfm.h" +#include "aom_ports/mem.h" + +static INLINE void LD_16x8(uint8_t *d, int d_stride, uint8x16_t *q8u8, + uint8x16_t *q9u8, uint8x16_t *q10u8, + uint8x16_t *q11u8, uint8x16_t *q12u8, + uint8x16_t *q13u8, uint8x16_t *q14u8, + uint8x16_t *q15u8) { + *q8u8 = vld1q_u8(d); + d += d_stride; + *q9u8 = vld1q_u8(d); + d += d_stride; + *q10u8 = vld1q_u8(d); + d += d_stride; + *q11u8 = vld1q_u8(d); + d += d_stride; + *q12u8 = vld1q_u8(d); + d += d_stride; + *q13u8 = vld1q_u8(d); + d += d_stride; + *q14u8 = vld1q_u8(d); + d += d_stride; + *q15u8 = vld1q_u8(d); + return; +} + +static INLINE void ADD_DIFF_16x8(uint8x16_t qdiffu8, uint8x16_t *q8u8, + uint8x16_t *q9u8, uint8x16_t *q10u8, + uint8x16_t *q11u8, uint8x16_t *q12u8, + uint8x16_t *q13u8, uint8x16_t *q14u8, + uint8x16_t *q15u8) { + *q8u8 = vqaddq_u8(*q8u8, qdiffu8); + *q9u8 = vqaddq_u8(*q9u8, qdiffu8); + *q10u8 = vqaddq_u8(*q10u8, qdiffu8); + *q11u8 = vqaddq_u8(*q11u8, qdiffu8); + *q12u8 = vqaddq_u8(*q12u8, qdiffu8); + *q13u8 = vqaddq_u8(*q13u8, qdiffu8); + *q14u8 = vqaddq_u8(*q14u8, qdiffu8); + *q15u8 = vqaddq_u8(*q15u8, qdiffu8); + return; +} + +static INLINE void SUB_DIFF_16x8(uint8x16_t qdiffu8, uint8x16_t *q8u8, + uint8x16_t *q9u8, uint8x16_t *q10u8, + uint8x16_t *q11u8, uint8x16_t *q12u8, + uint8x16_t *q13u8, uint8x16_t *q14u8, + uint8x16_t *q15u8) { + *q8u8 = vqsubq_u8(*q8u8, qdiffu8); + *q9u8 = vqsubq_u8(*q9u8, qdiffu8); + *q10u8 = vqsubq_u8(*q10u8, qdiffu8); + *q11u8 = vqsubq_u8(*q11u8, qdiffu8); + *q12u8 = vqsubq_u8(*q12u8, qdiffu8); + *q13u8 = vqsubq_u8(*q13u8, qdiffu8); + *q14u8 = vqsubq_u8(*q14u8, qdiffu8); + *q15u8 = vqsubq_u8(*q15u8, qdiffu8); + return; +} + +static INLINE void ST_16x8(uint8_t *d, int d_stride, uint8x16_t *q8u8, + uint8x16_t *q9u8, uint8x16_t *q10u8, + uint8x16_t *q11u8, uint8x16_t *q12u8, + uint8x16_t *q13u8, uint8x16_t *q14u8, + uint8x16_t *q15u8) { + vst1q_u8(d, *q8u8); + d += d_stride; + vst1q_u8(d, *q9u8); + d += d_stride; + vst1q_u8(d, *q10u8); + d += d_stride; + vst1q_u8(d, *q11u8); + d += d_stride; + vst1q_u8(d, *q12u8); + d += d_stride; + vst1q_u8(d, *q13u8); + d += d_stride; + vst1q_u8(d, *q14u8); + d += d_stride; + vst1q_u8(d, *q15u8); + return; +} + +void aom_idct32x32_1_add_neon(int16_t *input, uint8_t *dest, int dest_stride) { + uint8x16_t q0u8, q8u8, q9u8, q10u8, q11u8, q12u8, q13u8, q14u8, q15u8; + int i, j, dest_stride8; + uint8_t *d; + int16_t a1; + int16_t out = dct_const_round_shift(input[0] * cospi_16_64); + + out = dct_const_round_shift(out * cospi_16_64); + a1 = ROUND_POWER_OF_TWO(out, 6); + + dest_stride8 = dest_stride * 8; + if (a1 >= 0) { // diff_positive_32_32 + a1 = a1 < 0 ? 0 : a1 > 255 ? 255 : a1; + q0u8 = vdupq_n_u8(a1); + for (i = 0; i < 2; i++, dest += 16) { // diff_positive_32_32_loop + d = dest; + for (j = 0; j < 4; j++) { + LD_16x8(d, dest_stride, &q8u8, &q9u8, &q10u8, &q11u8, &q12u8, &q13u8, + &q14u8, &q15u8); + ADD_DIFF_16x8(q0u8, &q8u8, &q9u8, &q10u8, &q11u8, &q12u8, &q13u8, + &q14u8, &q15u8); + ST_16x8(d, dest_stride, &q8u8, &q9u8, &q10u8, &q11u8, &q12u8, &q13u8, + &q14u8, &q15u8); + d += dest_stride8; + } + } + } else { // diff_negative_32_32 + a1 = -a1; + a1 = a1 < 0 ? 0 : a1 > 255 ? 255 : a1; + q0u8 = vdupq_n_u8(a1); + for (i = 0; i < 2; i++, dest += 16) { // diff_negative_32_32_loop + d = dest; + for (j = 0; j < 4; j++) { + LD_16x8(d, dest_stride, &q8u8, &q9u8, &q10u8, &q11u8, &q12u8, &q13u8, + &q14u8, &q15u8); + SUB_DIFF_16x8(q0u8, &q8u8, &q9u8, &q10u8, &q11u8, &q12u8, &q13u8, + &q14u8, &q15u8); + ST_16x8(d, dest_stride, &q8u8, &q9u8, &q10u8, &q11u8, &q12u8, &q13u8, + &q14u8, &q15u8); + d += dest_stride8; + } + } + } + return; +} diff --git a/third_party/aom/aom_dsp/arm/idct32x32_add_neon.asm b/third_party/aom/aom_dsp/arm/idct32x32_add_neon.asm new file mode 100644 index 0000000000..e7793fb16e --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct32x32_add_neon.asm @@ -0,0 +1,1302 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +;TODO(cd): adjust these constant to be able to use vqdmulh for faster +; dct_const_round_shift(a * b) within butterfly calculations. +cospi_1_64 EQU 16364 +cospi_2_64 EQU 16305 +cospi_3_64 EQU 16207 +cospi_4_64 EQU 16069 +cospi_5_64 EQU 15893 +cospi_6_64 EQU 15679 +cospi_7_64 EQU 15426 +cospi_8_64 EQU 15137 +cospi_9_64 EQU 14811 +cospi_10_64 EQU 14449 +cospi_11_64 EQU 14053 +cospi_12_64 EQU 13623 +cospi_13_64 EQU 13160 +cospi_14_64 EQU 12665 +cospi_15_64 EQU 12140 +cospi_16_64 EQU 11585 +cospi_17_64 EQU 11003 +cospi_18_64 EQU 10394 +cospi_19_64 EQU 9760 +cospi_20_64 EQU 9102 +cospi_21_64 EQU 8423 +cospi_22_64 EQU 7723 +cospi_23_64 EQU 7005 +cospi_24_64 EQU 6270 +cospi_25_64 EQU 5520 +cospi_26_64 EQU 4756 +cospi_27_64 EQU 3981 +cospi_28_64 EQU 3196 +cospi_29_64 EQU 2404 +cospi_30_64 EQU 1606 +cospi_31_64 EQU 804 + + + EXPORT |aom_idct32x32_1024_add_neon| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + + AREA Block, CODE, READONLY + + ; -------------------------------------------------------------------------- + ; Load from transposed_buffer + ; q13 = transposed_buffer[first_offset] + ; q14 = transposed_buffer[second_offset] + ; for proper address calculation, the last offset used when manipulating + ; transposed_buffer must be passed in. use 0 for first use. + MACRO + LOAD_FROM_TRANSPOSED $prev_offset, $first_offset, $second_offset + ; address calculation with proper stride and loading + add r0, #($first_offset - $prev_offset )*8*2 + vld1.s16 {q14}, [r0] + add r0, #($second_offset - $first_offset)*8*2 + vld1.s16 {q13}, [r0] + ; (used) two registers (q14, q13) + MEND + ; -------------------------------------------------------------------------- + ; Load from output (used as temporary storage) + ; reg1 = output[first_offset] + ; reg2 = output[second_offset] + ; for proper address calculation, the last offset used when manipulating + ; output, whether reading or storing) must be passed in. use 0 for first + ; use. + MACRO + LOAD_FROM_OUTPUT $prev_offset, $first_offset, $second_offset, $reg1, $reg2 + ; address calculation with proper stride and loading + add r1, #($first_offset - $prev_offset )*32*2 + vld1.s16 {$reg1}, [r1] + add r1, #($second_offset - $first_offset)*32*2 + vld1.s16 {$reg2}, [r1] + ; (used) two registers ($reg1, $reg2) + MEND + ; -------------------------------------------------------------------------- + ; Store into output (sometimes as as temporary storage) + ; output[first_offset] = reg1 + ; output[second_offset] = reg2 + ; for proper address calculation, the last offset used when manipulating + ; output, whether reading or storing) must be passed in. use 0 for first + ; use. + MACRO + STORE_IN_OUTPUT $prev_offset, $first_offset, $second_offset, $reg1, $reg2 + ; address calculation with proper stride and storing + add r1, #($first_offset - $prev_offset )*32*2 + vst1.16 {$reg1}, [r1] + add r1, #($second_offset - $first_offset)*32*2 + vst1.16 {$reg2}, [r1] + MEND + ; -------------------------------------------------------------------------- + ; Combine-add results with current destination content + ; q6-q9 contain the results (out[j * 32 + 0-31]) + MACRO + STORE_COMBINE_CENTER_RESULTS + ; load dest[j * dest_stride + 0-31] + vld1.s16 {d8}, [r10], r2 + vld1.s16 {d11}, [r9], r11 + vld1.s16 {d9}, [r10] + vld1.s16 {d10}, [r9] + ; ROUND_POWER_OF_TWO + vrshr.s16 q7, q7, #6 + vrshr.s16 q8, q8, #6 + vrshr.s16 q9, q9, #6 + vrshr.s16 q6, q6, #6 + ; add to dest[j * dest_stride + 0-31] + vaddw.u8 q7, q7, d9 + vaddw.u8 q8, q8, d10 + vaddw.u8 q9, q9, d11 + vaddw.u8 q6, q6, d8 + ; clip pixel + vqmovun.s16 d9, q7 + vqmovun.s16 d10, q8 + vqmovun.s16 d11, q9 + vqmovun.s16 d8, q6 + ; store back into dest[j * dest_stride + 0-31] + vst1.16 {d9}, [r10], r11 + vst1.16 {d10}, [r9], r2 + vst1.16 {d8}, [r10] + vst1.16 {d11}, [r9] + ; update pointers (by dest_stride * 2) + sub r9, r9, r2, lsl #1 + add r10, r10, r2, lsl #1 + MEND + ; -------------------------------------------------------------------------- + ; Combine-add results with current destination content + ; q6-q9 contain the results (out[j * 32 + 0-31]) + MACRO + STORE_COMBINE_CENTER_RESULTS_LAST + ; load dest[j * dest_stride + 0-31] + vld1.s16 {d8}, [r10], r2 + vld1.s16 {d11}, [r9], r11 + vld1.s16 {d9}, [r10] + vld1.s16 {d10}, [r9] + ; ROUND_POWER_OF_TWO + vrshr.s16 q7, q7, #6 + vrshr.s16 q8, q8, #6 + vrshr.s16 q9, q9, #6 + vrshr.s16 q6, q6, #6 + ; add to dest[j * dest_stride + 0-31] + vaddw.u8 q7, q7, d9 + vaddw.u8 q8, q8, d10 + vaddw.u8 q9, q9, d11 + vaddw.u8 q6, q6, d8 + ; clip pixel + vqmovun.s16 d9, q7 + vqmovun.s16 d10, q8 + vqmovun.s16 d11, q9 + vqmovun.s16 d8, q6 + ; store back into dest[j * dest_stride + 0-31] + vst1.16 {d9}, [r10], r11 + vst1.16 {d10}, [r9], r2 + vst1.16 {d8}, [r10]! + vst1.16 {d11}, [r9]! + ; update pointers (by dest_stride * 2) + sub r9, r9, r2, lsl #1 + add r10, r10, r2, lsl #1 + MEND + ; -------------------------------------------------------------------------- + ; Combine-add results with current destination content + ; q4-q7 contain the results (out[j * 32 + 0-31]) + MACRO + STORE_COMBINE_EXTREME_RESULTS + ; load dest[j * dest_stride + 0-31] + vld1.s16 {d4}, [r7], r2 + vld1.s16 {d7}, [r6], r11 + vld1.s16 {d5}, [r7] + vld1.s16 {d6}, [r6] + ; ROUND_POWER_OF_TWO + vrshr.s16 q5, q5, #6 + vrshr.s16 q6, q6, #6 + vrshr.s16 q7, q7, #6 + vrshr.s16 q4, q4, #6 + ; add to dest[j * dest_stride + 0-31] + vaddw.u8 q5, q5, d5 + vaddw.u8 q6, q6, d6 + vaddw.u8 q7, q7, d7 + vaddw.u8 q4, q4, d4 + ; clip pixel + vqmovun.s16 d5, q5 + vqmovun.s16 d6, q6 + vqmovun.s16 d7, q7 + vqmovun.s16 d4, q4 + ; store back into dest[j * dest_stride + 0-31] + vst1.16 {d5}, [r7], r11 + vst1.16 {d6}, [r6], r2 + vst1.16 {d7}, [r6] + vst1.16 {d4}, [r7] + ; update pointers (by dest_stride * 2) + sub r6, r6, r2, lsl #1 + add r7, r7, r2, lsl #1 + MEND + ; -------------------------------------------------------------------------- + ; Combine-add results with current destination content + ; q4-q7 contain the results (out[j * 32 + 0-31]) + MACRO + STORE_COMBINE_EXTREME_RESULTS_LAST + ; load dest[j * dest_stride + 0-31] + vld1.s16 {d4}, [r7], r2 + vld1.s16 {d7}, [r6], r11 + vld1.s16 {d5}, [r7] + vld1.s16 {d6}, [r6] + ; ROUND_POWER_OF_TWO + vrshr.s16 q5, q5, #6 + vrshr.s16 q6, q6, #6 + vrshr.s16 q7, q7, #6 + vrshr.s16 q4, q4, #6 + ; add to dest[j * dest_stride + 0-31] + vaddw.u8 q5, q5, d5 + vaddw.u8 q6, q6, d6 + vaddw.u8 q7, q7, d7 + vaddw.u8 q4, q4, d4 + ; clip pixel + vqmovun.s16 d5, q5 + vqmovun.s16 d6, q6 + vqmovun.s16 d7, q7 + vqmovun.s16 d4, q4 + ; store back into dest[j * dest_stride + 0-31] + vst1.16 {d5}, [r7], r11 + vst1.16 {d6}, [r6], r2 + vst1.16 {d7}, [r6]! + vst1.16 {d4}, [r7]! + ; update pointers (by dest_stride * 2) + sub r6, r6, r2, lsl #1 + add r7, r7, r2, lsl #1 + MEND + ; -------------------------------------------------------------------------- + ; Touches q8-q12, q15 (q13-q14 are preserved) + ; valid output registers are anything but q8-q11 + MACRO + DO_BUTTERFLY $regC, $regD, $regA, $regB, $first_constant, $second_constant, $reg1, $reg2, $reg3, $reg4 + ; TODO(cd): have special case to re-use constants when they are similar for + ; consecutive butterflies + ; TODO(cd): have special case when both constants are the same, do the + ; additions/subtractions before the multiplies. + ; generate the constants + ; generate scalar constants + mov r8, #$first_constant & 0xFF00 + mov r12, #$second_constant & 0xFF00 + add r8, #$first_constant & 0x00FF + add r12, #$second_constant & 0x00FF + ; generate vector constants + vdup.16 d30, r8 + vdup.16 d31, r12 + ; (used) two for inputs (regA-regD), one for constants (q15) + ; do some multiplications (ordered for maximum latency hiding) + vmull.s16 q8, $regC, d30 + vmull.s16 q10, $regA, d31 + vmull.s16 q9, $regD, d30 + vmull.s16 q11, $regB, d31 + vmull.s16 q12, $regC, d31 + ; (used) five for intermediate (q8-q12), one for constants (q15) + ; do some addition/subtractions (to get back two register) + vsub.s32 q8, q8, q10 + vsub.s32 q9, q9, q11 + ; do more multiplications (ordered for maximum latency hiding) + vmull.s16 q10, $regD, d31 + vmull.s16 q11, $regA, d30 + vmull.s16 q15, $regB, d30 + ; (used) six for intermediate (q8-q12, q15) + ; do more addition/subtractions + vadd.s32 q11, q12, q11 + vadd.s32 q10, q10, q15 + ; (used) four for intermediate (q8-q11) + ; dct_const_round_shift + vqrshrn.s32 $reg1, q8, #14 + vqrshrn.s32 $reg2, q9, #14 + vqrshrn.s32 $reg3, q11, #14 + vqrshrn.s32 $reg4, q10, #14 + ; (used) two for results, well four d registers + MEND + ; -------------------------------------------------------------------------- + ; Touches q8-q12, q15 (q13-q14 are preserved) + ; valid output registers are anything but q8-q11 + MACRO + DO_BUTTERFLY_STD $first_constant, $second_constant, $reg1, $reg2, $reg3, $reg4 + DO_BUTTERFLY d28, d29, d26, d27, $first_constant, $second_constant, $reg1, $reg2, $reg3, $reg4 + MEND + ; -------------------------------------------------------------------------- + +;void aom_idct32x32_1024_add_neon(int16_t *input, uint8_t *dest, int dest_stride); +; +; r0 int16_t *input, +; r1 uint8_t *dest, +; r2 int dest_stride) +; loop counters +; r4 bands loop counter +; r5 pass loop counter +; r8 transpose loop counter +; combine-add pointers +; r6 dest + 31 * dest_stride, descending (30, 29, 28, ...) +; r7 dest + 0 * dest_stride, ascending (1, 2, 3, ...) +; r9 dest + 15 * dest_stride, descending (14, 13, 12, ...) +; r10 dest + 16 * dest_stride, ascending (17, 18, 19, ...) + +|aom_idct32x32_1024_add_neon| PROC + ; This function does one pass of idct32x32 transform. + ; + ; This is done by transposing the input and then doing a 1d transform on + ; columns. In the first pass, the transposed columns are the original + ; rows. In the second pass, after the transposition, the colums are the + ; original columns. + ; The 1d transform is done by looping over bands of eight columns (the + ; idct32_bands loop). For each band, the transform input transposition + ; is done on demand, one band of four 8x8 matrices at a time. The four + ; matrices are transposed by pairs (the idct32_transpose_pair loop). + push {r4-r11} + vpush {d8-d15} + ; stack operation + ; internal buffer used to transpose 8 lines into before transforming them + ; int16_t transpose_buffer[32 * 8]; + ; at sp + [4096, 4607] + ; results of the first pass (transpose and transform rows) + ; int16_t pass1[32 * 32]; + ; at sp + [0, 2047] + ; results of the second pass (transpose and transform columns) + ; int16_t pass2[32 * 32]; + ; at sp + [2048, 4095] + sub sp, sp, #512+2048+2048 + + ; r6 = dest + 31 * dest_stride + ; r7 = dest + 0 * dest_stride + ; r9 = dest + 15 * dest_stride + ; r10 = dest + 16 * dest_stride + rsb r6, r2, r2, lsl #5 + rsb r9, r2, r2, lsl #4 + add r10, r1, r2, lsl #4 + mov r7, r1 + add r6, r6, r1 + add r9, r9, r1 + ; r11 = -dest_stride + neg r11, r2 + ; r3 = input + mov r3, r0 + ; parameters for first pass + ; r0 = transpose_buffer[32 * 8] + add r0, sp, #4096 + ; r1 = pass1[32 * 32] + mov r1, sp + + mov r5, #0 ; initialize pass loop counter +idct32_pass_loop + mov r4, #4 ; initialize bands loop counter +idct32_bands_loop + mov r8, #2 ; initialize transpose loop counter +idct32_transpose_pair_loop + ; Load two horizontally consecutive 8x8 16bit data matrices. The first one + ; into q0-q7 and the second one into q8-q15. There is a stride of 64, + ; adjusted to 32 because of the two post-increments. + vld1.s16 {q8}, [r3]! + vld1.s16 {q0}, [r3]! + add r3, #32 + vld1.s16 {q9}, [r3]! + vld1.s16 {q1}, [r3]! + add r3, #32 + vld1.s16 {q10}, [r3]! + vld1.s16 {q2}, [r3]! + add r3, #32 + vld1.s16 {q11}, [r3]! + vld1.s16 {q3}, [r3]! + add r3, #32 + vld1.s16 {q12}, [r3]! + vld1.s16 {q4}, [r3]! + add r3, #32 + vld1.s16 {q13}, [r3]! + vld1.s16 {q5}, [r3]! + add r3, #32 + vld1.s16 {q14}, [r3]! + vld1.s16 {q6}, [r3]! + add r3, #32 + vld1.s16 {q15}, [r3]! + vld1.s16 {q7}, [r3]! + + ; Transpose the two 8x8 16bit data matrices. + vswp d17, d24 + vswp d23, d30 + vswp d21, d28 + vswp d19, d26 + vswp d1, d8 + vswp d7, d14 + vswp d5, d12 + vswp d3, d10 + vtrn.32 q8, q10 + vtrn.32 q9, q11 + vtrn.32 q12, q14 + vtrn.32 q13, q15 + vtrn.32 q0, q2 + vtrn.32 q1, q3 + vtrn.32 q4, q6 + vtrn.32 q5, q7 + vtrn.16 q8, q9 + vtrn.16 q10, q11 + vtrn.16 q12, q13 + vtrn.16 q14, q15 + vtrn.16 q0, q1 + vtrn.16 q2, q3 + vtrn.16 q4, q5 + vtrn.16 q6, q7 + + ; Store both matrices after each other. There is a stride of 32, which + ; adjusts to nothing because of the post-increments. + vst1.16 {q8}, [r0]! + vst1.16 {q9}, [r0]! + vst1.16 {q10}, [r0]! + vst1.16 {q11}, [r0]! + vst1.16 {q12}, [r0]! + vst1.16 {q13}, [r0]! + vst1.16 {q14}, [r0]! + vst1.16 {q15}, [r0]! + vst1.16 {q0}, [r0]! + vst1.16 {q1}, [r0]! + vst1.16 {q2}, [r0]! + vst1.16 {q3}, [r0]! + vst1.16 {q4}, [r0]! + vst1.16 {q5}, [r0]! + vst1.16 {q6}, [r0]! + vst1.16 {q7}, [r0]! + + ; increment pointers by adjusted stride (not necessary for r0/out) + ; go back by 7*32 for the seven lines moved fully by read and add + ; go back by 32 for the eigth line only read + ; advance by 16*2 to go the next pair + sub r3, r3, #7*32*2 + 32 - 16*2 + ; transpose pair loop processing + subs r8, r8, #1 + bne idct32_transpose_pair_loop + + ; restore r0/input to its original value + sub r0, r0, #32*8*2 + + ; Instead of doing the transforms stage by stage, it is done by loading + ; some input values and doing as many stages as possible to minimize the + ; storing/loading of intermediate results. To fit within registers, the + ; final coefficients are cut into four blocks: + ; BLOCK A: 16-19,28-31 + ; BLOCK B: 20-23,24-27 + ; BLOCK C: 8-10,11-15 + ; BLOCK D: 0-3,4-7 + ; Blocks A and C are straight calculation through the various stages. In + ; block B, further calculations are performed using the results from + ; block A. In block D, further calculations are performed using the results + ; from block C and then the final calculations are done using results from + ; block A and B which have been combined at the end of block B. + + ; -------------------------------------------------------------------------- + ; BLOCK A: 16-19,28-31 + ; -------------------------------------------------------------------------- + ; generate 16,17,30,31 + ; -------------------------------------------------------------------------- + ; part of stage 1 + ;temp1 = input[1 * 32] * cospi_31_64 - input[31 * 32] * cospi_1_64; + ;temp2 = input[1 * 32] * cospi_1_64 + input[31 * 32] * cospi_31_64; + ;step1b[16][i] = dct_const_round_shift(temp1); + ;step1b[31][i] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 0, 1, 31 + DO_BUTTERFLY_STD cospi_31_64, cospi_1_64, d0, d1, d4, d5 + ; -------------------------------------------------------------------------- + ; part of stage 1 + ;temp1 = input[17 * 32] * cospi_15_64 - input[15 * 32] * cospi_17_64; + ;temp2 = input[17 * 32] * cospi_17_64 + input[15 * 32] * cospi_15_64; + ;step1b[17][i] = dct_const_round_shift(temp1); + ;step1b[30][i] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 31, 17, 15 + DO_BUTTERFLY_STD cospi_15_64, cospi_17_64, d2, d3, d6, d7 + ; -------------------------------------------------------------------------- + ; part of stage 2 + ;step2[16] = step1b[16][i] + step1b[17][i]; + ;step2[17] = step1b[16][i] - step1b[17][i]; + ;step2[30] = -step1b[30][i] + step1b[31][i]; + ;step2[31] = step1b[30][i] + step1b[31][i]; + vadd.s16 q4, q0, q1 + vsub.s16 q13, q0, q1 + vadd.s16 q6, q2, q3 + vsub.s16 q14, q2, q3 + ; -------------------------------------------------------------------------- + ; part of stage 3 + ;temp1 = step1b[30][i] * cospi_28_64 - step1b[17][i] * cospi_4_64; + ;temp2 = step1b[30][i] * cospi_4_64 - step1b[17][i] * cospi_28_64; + ;step3[17] = dct_const_round_shift(temp1); + ;step3[30] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD cospi_28_64, cospi_4_64, d10, d11, d14, d15 + ; -------------------------------------------------------------------------- + ; generate 18,19,28,29 + ; -------------------------------------------------------------------------- + ; part of stage 1 + ;temp1 = input[9 * 32] * cospi_23_64 - input[23 * 32] * cospi_9_64; + ;temp2 = input[9 * 32] * cospi_9_64 + input[23 * 32] * cospi_23_64; + ;step1b[18][i] = dct_const_round_shift(temp1); + ;step1b[29][i] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 15, 9, 23 + DO_BUTTERFLY_STD cospi_23_64, cospi_9_64, d0, d1, d4, d5 + ; -------------------------------------------------------------------------- + ; part of stage 1 + ;temp1 = input[25 * 32] * cospi_7_64 - input[7 * 32] * cospi_25_64; + ;temp2 = input[25 * 32] * cospi_25_64 + input[7 * 32] * cospi_7_64; + ;step1b[19][i] = dct_const_round_shift(temp1); + ;step1b[28][i] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 23, 25, 7 + DO_BUTTERFLY_STD cospi_7_64, cospi_25_64, d2, d3, d6, d7 + ; -------------------------------------------------------------------------- + ; part of stage 2 + ;step2[18] = -step1b[18][i] + step1b[19][i]; + ;step2[19] = step1b[18][i] + step1b[19][i]; + ;step2[28] = step1b[28][i] + step1b[29][i]; + ;step2[29] = step1b[28][i] - step1b[29][i]; + vsub.s16 q13, q3, q2 + vadd.s16 q3, q3, q2 + vsub.s16 q14, q1, q0 + vadd.s16 q2, q1, q0 + ; -------------------------------------------------------------------------- + ; part of stage 3 + ;temp1 = step1b[18][i] * (-cospi_4_64) - step1b[29][i] * (-cospi_28_64); + ;temp2 = step1b[18][i] * (-cospi_28_64) + step1b[29][i] * (-cospi_4_64); + ;step3[29] = dct_const_round_shift(temp1); + ;step3[18] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD (-cospi_4_64), (-cospi_28_64), d2, d3, d0, d1 + ; -------------------------------------------------------------------------- + ; combine 16-19,28-31 + ; -------------------------------------------------------------------------- + ; part of stage 4 + ;step1[16] = step1b[16][i] + step1b[19][i]; + ;step1[17] = step1b[17][i] + step1b[18][i]; + ;step1[18] = step1b[17][i] - step1b[18][i]; + ;step1[29] = step1b[30][i] - step1b[29][i]; + ;step1[30] = step1b[30][i] + step1b[29][i]; + ;step1[31] = step1b[31][i] + step1b[28][i]; + vadd.s16 q8, q4, q2 + vadd.s16 q9, q5, q0 + vadd.s16 q10, q7, q1 + vadd.s16 q15, q6, q3 + vsub.s16 q13, q5, q0 + vsub.s16 q14, q7, q1 + STORE_IN_OUTPUT 0, 16, 31, q8, q15 + STORE_IN_OUTPUT 31, 17, 30, q9, q10 + ; -------------------------------------------------------------------------- + ; part of stage 5 + ;temp1 = step1b[29][i] * cospi_24_64 - step1b[18][i] * cospi_8_64; + ;temp2 = step1b[29][i] * cospi_8_64 + step1b[18][i] * cospi_24_64; + ;step2[18] = dct_const_round_shift(temp1); + ;step2[29] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD cospi_24_64, cospi_8_64, d0, d1, d2, d3 + STORE_IN_OUTPUT 30, 29, 18, q1, q0 + ; -------------------------------------------------------------------------- + ; part of stage 4 + ;step1[19] = step1b[16][i] - step1b[19][i]; + ;step1[28] = step1b[31][i] - step1b[28][i]; + vsub.s16 q13, q4, q2 + vsub.s16 q14, q6, q3 + ; -------------------------------------------------------------------------- + ; part of stage 5 + ;temp1 = step1b[28][i] * cospi_24_64 - step1b[19][i] * cospi_8_64; + ;temp2 = step1b[28][i] * cospi_8_64 + step1b[19][i] * cospi_24_64; + ;step2[19] = dct_const_round_shift(temp1); + ;step2[28] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD cospi_24_64, cospi_8_64, d8, d9, d12, d13 + STORE_IN_OUTPUT 18, 19, 28, q4, q6 + ; -------------------------------------------------------------------------- + + + ; -------------------------------------------------------------------------- + ; BLOCK B: 20-23,24-27 + ; -------------------------------------------------------------------------- + ; generate 20,21,26,27 + ; -------------------------------------------------------------------------- + ; part of stage 1 + ;temp1 = input[5 * 32] * cospi_27_64 - input[27 * 32] * cospi_5_64; + ;temp2 = input[5 * 32] * cospi_5_64 + input[27 * 32] * cospi_27_64; + ;step1b[20][i] = dct_const_round_shift(temp1); + ;step1b[27][i] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 7, 5, 27 + DO_BUTTERFLY_STD cospi_27_64, cospi_5_64, d0, d1, d4, d5 + ; -------------------------------------------------------------------------- + ; part of stage 1 + ;temp1 = input[21 * 32] * cospi_11_64 - input[11 * 32] * cospi_21_64; + ;temp2 = input[21 * 32] * cospi_21_64 + input[11 * 32] * cospi_11_64; + ;step1b[21][i] = dct_const_round_shift(temp1); + ;step1b[26][i] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 27, 21, 11 + DO_BUTTERFLY_STD cospi_11_64, cospi_21_64, d2, d3, d6, d7 + ; -------------------------------------------------------------------------- + ; part of stage 2 + ;step2[20] = step1b[20][i] + step1b[21][i]; + ;step2[21] = step1b[20][i] - step1b[21][i]; + ;step2[26] = -step1b[26][i] + step1b[27][i]; + ;step2[27] = step1b[26][i] + step1b[27][i]; + vsub.s16 q13, q0, q1 + vadd.s16 q0, q0, q1 + vsub.s16 q14, q2, q3 + vadd.s16 q2, q2, q3 + ; -------------------------------------------------------------------------- + ; part of stage 3 + ;temp1 = step1b[26][i] * cospi_12_64 - step1b[21][i] * cospi_20_64; + ;temp2 = step1b[26][i] * cospi_20_64 + step1b[21][i] * cospi_12_64; + ;step3[21] = dct_const_round_shift(temp1); + ;step3[26] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD cospi_12_64, cospi_20_64, d2, d3, d6, d7 + ; -------------------------------------------------------------------------- + ; generate 22,23,24,25 + ; -------------------------------------------------------------------------- + ; part of stage 1 + ;temp1 = input[13 * 32] * cospi_19_64 - input[19 * 32] * cospi_13_64; + ;temp2 = input[13 * 32] * cospi_13_64 + input[19 * 32] * cospi_19_64; + ;step1b[22][i] = dct_const_round_shift(temp1); + ;step1b[25][i] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 11, 13, 19 + DO_BUTTERFLY_STD cospi_19_64, cospi_13_64, d10, d11, d14, d15 + ; -------------------------------------------------------------------------- + ; part of stage 1 + ;temp1 = input[29 * 32] * cospi_3_64 - input[3 * 32] * cospi_29_64; + ;temp2 = input[29 * 32] * cospi_29_64 + input[3 * 32] * cospi_3_64; + ;step1b[23][i] = dct_const_round_shift(temp1); + ;step1b[24][i] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 19, 29, 3 + DO_BUTTERFLY_STD cospi_3_64, cospi_29_64, d8, d9, d12, d13 + ; -------------------------------------------------------------------------- + ; part of stage 2 + ;step2[22] = -step1b[22][i] + step1b[23][i]; + ;step2[23] = step1b[22][i] + step1b[23][i]; + ;step2[24] = step1b[24][i] + step1b[25][i]; + ;step2[25] = step1b[24][i] - step1b[25][i]; + vsub.s16 q14, q4, q5 + vadd.s16 q5, q4, q5 + vsub.s16 q13, q6, q7 + vadd.s16 q6, q6, q7 + ; -------------------------------------------------------------------------- + ; part of stage 3 + ;temp1 = step1b[22][i] * (-cospi_20_64) - step1b[25][i] * (-cospi_12_64); + ;temp2 = step1b[22][i] * (-cospi_12_64) + step1b[25][i] * (-cospi_20_64); + ;step3[25] = dct_const_round_shift(temp1); + ;step3[22] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD (-cospi_20_64), (-cospi_12_64), d8, d9, d14, d15 + ; -------------------------------------------------------------------------- + ; combine 20-23,24-27 + ; -------------------------------------------------------------------------- + ; part of stage 4 + ;step1[22] = step1b[22][i] + step1b[21][i]; + ;step1[23] = step1b[23][i] + step1b[20][i]; + vadd.s16 q10, q7, q1 + vadd.s16 q11, q5, q0 + ;step1[24] = step1b[24][i] + step1b[27][i]; + ;step1[25] = step1b[25][i] + step1b[26][i]; + vadd.s16 q12, q6, q2 + vadd.s16 q15, q4, q3 + ; -------------------------------------------------------------------------- + ; part of stage 6 + ;step3[16] = step1b[16][i] + step1b[23][i]; + ;step3[17] = step1b[17][i] + step1b[22][i]; + ;step3[22] = step1b[17][i] - step1b[22][i]; + ;step3[23] = step1b[16][i] - step1b[23][i]; + LOAD_FROM_OUTPUT 28, 16, 17, q14, q13 + vadd.s16 q8, q14, q11 + vadd.s16 q9, q13, q10 + vsub.s16 q13, q13, q10 + vsub.s16 q11, q14, q11 + STORE_IN_OUTPUT 17, 17, 16, q9, q8 + ; -------------------------------------------------------------------------- + ; part of stage 6 + ;step3[24] = step1b[31][i] - step1b[24][i]; + ;step3[25] = step1b[30][i] - step1b[25][i]; + ;step3[30] = step1b[30][i] + step1b[25][i]; + ;step3[31] = step1b[31][i] + step1b[24][i]; + LOAD_FROM_OUTPUT 16, 30, 31, q14, q9 + vsub.s16 q8, q9, q12 + vadd.s16 q10, q14, q15 + vsub.s16 q14, q14, q15 + vadd.s16 q12, q9, q12 + STORE_IN_OUTPUT 31, 30, 31, q10, q12 + ; -------------------------------------------------------------------------- + ; TODO(cd) do some register allocation change to remove these push/pop + vpush {q8} ; [24] + vpush {q11} ; [23] + ; -------------------------------------------------------------------------- + ; part of stage 7 + ;temp1 = (step1b[25][i] - step1b[22][i]) * cospi_16_64; + ;temp2 = (step1b[25][i] + step1b[22][i]) * cospi_16_64; + ;step1[22] = dct_const_round_shift(temp1); + ;step1[25] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD cospi_16_64, cospi_16_64, d26, d27, d28, d29 + STORE_IN_OUTPUT 31, 25, 22, q14, q13 + ; -------------------------------------------------------------------------- + ; part of stage 7 + ;temp1 = (step1b[24][i] - step1b[23][i]) * cospi_16_64; + ;temp2 = (step1b[24][i] + step1b[23][i]) * cospi_16_64; + ;step1[23] = dct_const_round_shift(temp1); + ;step1[24] = dct_const_round_shift(temp2); + ; TODO(cd) do some register allocation change to remove these push/pop + vpop {q13} ; [23] + vpop {q14} ; [24] + DO_BUTTERFLY_STD cospi_16_64, cospi_16_64, d26, d27, d28, d29 + STORE_IN_OUTPUT 22, 24, 23, q14, q13 + ; -------------------------------------------------------------------------- + ; part of stage 4 + ;step1[20] = step1b[23][i] - step1b[20][i]; + ;step1[27] = step1b[24][i] - step1b[27][i]; + vsub.s16 q14, q5, q0 + vsub.s16 q13, q6, q2 + ; -------------------------------------------------------------------------- + ; part of stage 5 + ;temp1 = step1b[20][i] * (-cospi_8_64) - step1b[27][i] * (-cospi_24_64); + ;temp2 = step1b[20][i] * (-cospi_24_64) + step1b[27][i] * (-cospi_8_64); + ;step2[27] = dct_const_round_shift(temp1); + ;step2[20] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD (-cospi_8_64), (-cospi_24_64), d10, d11, d12, d13 + ; -------------------------------------------------------------------------- + ; part of stage 4 + ;step1[21] = step1b[22][i] - step1b[21][i]; + ;step1[26] = step1b[25][i] - step1b[26][i]; + vsub.s16 q14, q7, q1 + vsub.s16 q13, q4, q3 + ; -------------------------------------------------------------------------- + ; part of stage 5 + ;temp1 = step1b[21][i] * (-cospi_8_64) - step1b[26][i] * (-cospi_24_64); + ;temp2 = step1b[21][i] * (-cospi_24_64) + step1b[26][i] * (-cospi_8_64); + ;step2[26] = dct_const_round_shift(temp1); + ;step2[21] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD (-cospi_8_64), (-cospi_24_64), d0, d1, d2, d3 + ; -------------------------------------------------------------------------- + ; part of stage 6 + ;step3[18] = step1b[18][i] + step1b[21][i]; + ;step3[19] = step1b[19][i] + step1b[20][i]; + ;step3[20] = step1b[19][i] - step1b[20][i]; + ;step3[21] = step1b[18][i] - step1b[21][i]; + LOAD_FROM_OUTPUT 23, 18, 19, q14, q13 + vadd.s16 q8, q14, q1 + vadd.s16 q9, q13, q6 + vsub.s16 q13, q13, q6 + vsub.s16 q1, q14, q1 + STORE_IN_OUTPUT 19, 18, 19, q8, q9 + ; -------------------------------------------------------------------------- + ; part of stage 6 + ;step3[27] = step1b[28][i] - step1b[27][i]; + ;step3[28] = step1b[28][i] + step1b[27][i]; + ;step3[29] = step1b[29][i] + step1b[26][i]; + ;step3[26] = step1b[29][i] - step1b[26][i]; + LOAD_FROM_OUTPUT 19, 28, 29, q8, q9 + vsub.s16 q14, q8, q5 + vadd.s16 q10, q8, q5 + vadd.s16 q11, q9, q0 + vsub.s16 q0, q9, q0 + STORE_IN_OUTPUT 29, 28, 29, q10, q11 + ; -------------------------------------------------------------------------- + ; part of stage 7 + ;temp1 = (step1b[27][i] - step1b[20][i]) * cospi_16_64; + ;temp2 = (step1b[27][i] + step1b[20][i]) * cospi_16_64; + ;step1[20] = dct_const_round_shift(temp1); + ;step1[27] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD cospi_16_64, cospi_16_64, d26, d27, d28, d29 + STORE_IN_OUTPUT 29, 20, 27, q13, q14 + ; -------------------------------------------------------------------------- + ; part of stage 7 + ;temp1 = (step1b[26][i] - step1b[21][i]) * cospi_16_64; + ;temp2 = (step1b[26][i] + step1b[21][i]) * cospi_16_64; + ;step1[21] = dct_const_round_shift(temp1); + ;step1[26] = dct_const_round_shift(temp2); + DO_BUTTERFLY d0, d1, d2, d3, cospi_16_64, cospi_16_64, d2, d3, d0, d1 + STORE_IN_OUTPUT 27, 21, 26, q1, q0 + ; -------------------------------------------------------------------------- + + + ; -------------------------------------------------------------------------- + ; BLOCK C: 8-10,11-15 + ; -------------------------------------------------------------------------- + ; generate 8,9,14,15 + ; -------------------------------------------------------------------------- + ; part of stage 2 + ;temp1 = input[2 * 32] * cospi_30_64 - input[30 * 32] * cospi_2_64; + ;temp2 = input[2 * 32] * cospi_2_64 + input[30 * 32] * cospi_30_64; + ;step2[8] = dct_const_round_shift(temp1); + ;step2[15] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 3, 2, 30 + DO_BUTTERFLY_STD cospi_30_64, cospi_2_64, d0, d1, d4, d5 + ; -------------------------------------------------------------------------- + ; part of stage 2 + ;temp1 = input[18 * 32] * cospi_14_64 - input[14 * 32] * cospi_18_64; + ;temp2 = input[18 * 32] * cospi_18_64 + input[14 * 32] * cospi_14_64; + ;step2[9] = dct_const_round_shift(temp1); + ;step2[14] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 30, 18, 14 + DO_BUTTERFLY_STD cospi_14_64, cospi_18_64, d2, d3, d6, d7 + ; -------------------------------------------------------------------------- + ; part of stage 3 + ;step3[8] = step1b[8][i] + step1b[9][i]; + ;step3[9] = step1b[8][i] - step1b[9][i]; + ;step3[14] = step1b[15][i] - step1b[14][i]; + ;step3[15] = step1b[15][i] + step1b[14][i]; + vsub.s16 q13, q0, q1 + vadd.s16 q0, q0, q1 + vsub.s16 q14, q2, q3 + vadd.s16 q2, q2, q3 + ; -------------------------------------------------------------------------- + ; part of stage 4 + ;temp1 = step1b[14][i] * cospi_24_64 - step1b[9][i] * cospi_8_64; + ;temp2 = step1b[14][i] * cospi_8_64 + step1b[9][i] * cospi_24_64; + ;step1[9] = dct_const_round_shift(temp1); + ;step1[14] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD cospi_24_64, cospi_8_64, d2, d3, d6, d7 + ; -------------------------------------------------------------------------- + ; generate 10,11,12,13 + ; -------------------------------------------------------------------------- + ; part of stage 2 + ;temp1 = input[10 * 32] * cospi_22_64 - input[22 * 32] * cospi_10_64; + ;temp2 = input[10 * 32] * cospi_10_64 + input[22 * 32] * cospi_22_64; + ;step2[10] = dct_const_round_shift(temp1); + ;step2[13] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 14, 10, 22 + DO_BUTTERFLY_STD cospi_22_64, cospi_10_64, d10, d11, d14, d15 + ; -------------------------------------------------------------------------- + ; part of stage 2 + ;temp1 = input[26 * 32] * cospi_6_64 - input[6 * 32] * cospi_26_64; + ;temp2 = input[26 * 32] * cospi_26_64 + input[6 * 32] * cospi_6_64; + ;step2[11] = dct_const_round_shift(temp1); + ;step2[12] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 22, 26, 6 + DO_BUTTERFLY_STD cospi_6_64, cospi_26_64, d8, d9, d12, d13 + ; -------------------------------------------------------------------------- + ; part of stage 3 + ;step3[10] = step1b[11][i] - step1b[10][i]; + ;step3[11] = step1b[11][i] + step1b[10][i]; + ;step3[12] = step1b[12][i] + step1b[13][i]; + ;step3[13] = step1b[12][i] - step1b[13][i]; + vsub.s16 q14, q4, q5 + vadd.s16 q5, q4, q5 + vsub.s16 q13, q6, q7 + vadd.s16 q6, q6, q7 + ; -------------------------------------------------------------------------- + ; part of stage 4 + ;temp1 = step1b[10][i] * (-cospi_8_64) - step1b[13][i] * (-cospi_24_64); + ;temp2 = step1b[10][i] * (-cospi_24_64) + step1b[13][i] * (-cospi_8_64); + ;step1[13] = dct_const_round_shift(temp1); + ;step1[10] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD (-cospi_8_64), (-cospi_24_64), d8, d9, d14, d15 + ; -------------------------------------------------------------------------- + ; combine 8-10,11-15 + ; -------------------------------------------------------------------------- + ; part of stage 5 + ;step2[8] = step1b[8][i] + step1b[11][i]; + ;step2[9] = step1b[9][i] + step1b[10][i]; + ;step2[10] = step1b[9][i] - step1b[10][i]; + vadd.s16 q8, q0, q5 + vadd.s16 q9, q1, q7 + vsub.s16 q13, q1, q7 + ;step2[13] = step1b[14][i] - step1b[13][i]; + ;step2[14] = step1b[14][i] + step1b[13][i]; + ;step2[15] = step1b[15][i] + step1b[12][i]; + vsub.s16 q14, q3, q4 + vadd.s16 q10, q3, q4 + vadd.s16 q15, q2, q6 + STORE_IN_OUTPUT 26, 8, 15, q8, q15 + STORE_IN_OUTPUT 15, 9, 14, q9, q10 + ; -------------------------------------------------------------------------- + ; part of stage 6 + ;temp1 = (step1b[13][i] - step1b[10][i]) * cospi_16_64; + ;temp2 = (step1b[13][i] + step1b[10][i]) * cospi_16_64; + ;step3[10] = dct_const_round_shift(temp1); + ;step3[13] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD cospi_16_64, cospi_16_64, d2, d3, d6, d7 + STORE_IN_OUTPUT 14, 13, 10, q3, q1 + ; -------------------------------------------------------------------------- + ; part of stage 5 + ;step2[11] = step1b[8][i] - step1b[11][i]; + ;step2[12] = step1b[15][i] - step1b[12][i]; + vsub.s16 q13, q0, q5 + vsub.s16 q14, q2, q6 + ; -------------------------------------------------------------------------- + ; part of stage 6 + ;temp1 = (step1b[12][i] - step1b[11][i]) * cospi_16_64; + ;temp2 = (step1b[12][i] + step1b[11][i]) * cospi_16_64; + ;step3[11] = dct_const_round_shift(temp1); + ;step3[12] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD cospi_16_64, cospi_16_64, d2, d3, d6, d7 + STORE_IN_OUTPUT 10, 11, 12, q1, q3 + ; -------------------------------------------------------------------------- + + + ; -------------------------------------------------------------------------- + ; BLOCK D: 0-3,4-7 + ; -------------------------------------------------------------------------- + ; generate 4,5,6,7 + ; -------------------------------------------------------------------------- + ; part of stage 3 + ;temp1 = input[4 * 32] * cospi_28_64 - input[28 * 32] * cospi_4_64; + ;temp2 = input[4 * 32] * cospi_4_64 + input[28 * 32] * cospi_28_64; + ;step3[4] = dct_const_round_shift(temp1); + ;step3[7] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 6, 4, 28 + DO_BUTTERFLY_STD cospi_28_64, cospi_4_64, d0, d1, d4, d5 + ; -------------------------------------------------------------------------- + ; part of stage 3 + ;temp1 = input[20 * 32] * cospi_12_64 - input[12 * 32] * cospi_20_64; + ;temp2 = input[20 * 32] * cospi_20_64 + input[12 * 32] * cospi_12_64; + ;step3[5] = dct_const_round_shift(temp1); + ;step3[6] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 28, 20, 12 + DO_BUTTERFLY_STD cospi_12_64, cospi_20_64, d2, d3, d6, d7 + ; -------------------------------------------------------------------------- + ; part of stage 4 + ;step1[4] = step1b[4][i] + step1b[5][i]; + ;step1[5] = step1b[4][i] - step1b[5][i]; + ;step1[6] = step1b[7][i] - step1b[6][i]; + ;step1[7] = step1b[7][i] + step1b[6][i]; + vsub.s16 q13, q0, q1 + vadd.s16 q0, q0, q1 + vsub.s16 q14, q2, q3 + vadd.s16 q2, q2, q3 + ; -------------------------------------------------------------------------- + ; part of stage 5 + ;temp1 = (step1b[6][i] - step1b[5][i]) * cospi_16_64; + ;temp2 = (step1b[5][i] + step1b[6][i]) * cospi_16_64; + ;step2[5] = dct_const_round_shift(temp1); + ;step2[6] = dct_const_round_shift(temp2); + DO_BUTTERFLY_STD cospi_16_64, cospi_16_64, d2, d3, d6, d7 + ; -------------------------------------------------------------------------- + ; generate 0,1,2,3 + ; -------------------------------------------------------------------------- + ; part of stage 4 + ;temp1 = (input[0 * 32] - input[16 * 32]) * cospi_16_64; + ;temp2 = (input[0 * 32] + input[16 * 32]) * cospi_16_64; + ;step1[1] = dct_const_round_shift(temp1); + ;step1[0] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 12, 0, 16 + DO_BUTTERFLY_STD cospi_16_64, cospi_16_64, d10, d11, d14, d15 + ; -------------------------------------------------------------------------- + ; part of stage 4 + ;temp1 = input[8 * 32] * cospi_24_64 - input[24 * 32] * cospi_8_64; + ;temp2 = input[8 * 32] * cospi_8_64 + input[24 * 32] * cospi_24_64; + ;step1[2] = dct_const_round_shift(temp1); + ;step1[3] = dct_const_round_shift(temp2); + LOAD_FROM_TRANSPOSED 16, 8, 24 + DO_BUTTERFLY_STD cospi_24_64, cospi_8_64, d28, d29, d12, d13 + ; -------------------------------------------------------------------------- + ; part of stage 5 + ;step2[0] = step1b[0][i] + step1b[3][i]; + ;step2[1] = step1b[1][i] + step1b[2][i]; + ;step2[2] = step1b[1][i] - step1b[2][i]; + ;step2[3] = step1b[0][i] - step1b[3][i]; + vadd.s16 q4, q7, q6 + vsub.s16 q7, q7, q6 + vsub.s16 q6, q5, q14 + vadd.s16 q5, q5, q14 + ; -------------------------------------------------------------------------- + ; combine 0-3,4-7 + ; -------------------------------------------------------------------------- + ; part of stage 6 + ;step3[0] = step1b[0][i] + step1b[7][i]; + ;step3[1] = step1b[1][i] + step1b[6][i]; + ;step3[2] = step1b[2][i] + step1b[5][i]; + ;step3[3] = step1b[3][i] + step1b[4][i]; + vadd.s16 q8, q4, q2 + vadd.s16 q9, q5, q3 + vadd.s16 q10, q6, q1 + vadd.s16 q11, q7, q0 + ;step3[4] = step1b[3][i] - step1b[4][i]; + ;step3[5] = step1b[2][i] - step1b[5][i]; + ;step3[6] = step1b[1][i] - step1b[6][i]; + ;step3[7] = step1b[0][i] - step1b[7][i]; + vsub.s16 q12, q7, q0 + vsub.s16 q13, q6, q1 + vsub.s16 q14, q5, q3 + vsub.s16 q15, q4, q2 + ; -------------------------------------------------------------------------- + ; part of stage 7 + ;step1[0] = step1b[0][i] + step1b[15][i]; + ;step1[1] = step1b[1][i] + step1b[14][i]; + ;step1[14] = step1b[1][i] - step1b[14][i]; + ;step1[15] = step1b[0][i] - step1b[15][i]; + LOAD_FROM_OUTPUT 12, 14, 15, q0, q1 + vadd.s16 q2, q8, q1 + vadd.s16 q3, q9, q0 + vsub.s16 q4, q9, q0 + vsub.s16 q5, q8, q1 + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[14 * 32] = step1b[14][i] + step1b[17][i]; + ;output[15 * 32] = step1b[15][i] + step1b[16][i]; + ;output[16 * 32] = step1b[15][i] - step1b[16][i]; + ;output[17 * 32] = step1b[14][i] - step1b[17][i]; + LOAD_FROM_OUTPUT 15, 16, 17, q0, q1 + vadd.s16 q8, q4, q1 + vadd.s16 q9, q5, q0 + vsub.s16 q6, q5, q0 + vsub.s16 q7, q4, q1 + + cmp r5, #0 + bgt idct32_bands_end_2nd_pass + +idct32_bands_end_1st_pass + STORE_IN_OUTPUT 17, 16, 17, q6, q7 + STORE_IN_OUTPUT 17, 14, 15, q8, q9 + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[ 0 * 32] = step1b[0][i] + step1b[31][i]; + ;output[ 1 * 32] = step1b[1][i] + step1b[30][i]; + ;output[30 * 32] = step1b[1][i] - step1b[30][i]; + ;output[31 * 32] = step1b[0][i] - step1b[31][i]; + LOAD_FROM_OUTPUT 15, 30, 31, q0, q1 + vadd.s16 q4, q2, q1 + vadd.s16 q5, q3, q0 + vsub.s16 q6, q3, q0 + vsub.s16 q7, q2, q1 + STORE_IN_OUTPUT 31, 30, 31, q6, q7 + STORE_IN_OUTPUT 31, 0, 1, q4, q5 + ; -------------------------------------------------------------------------- + ; part of stage 7 + ;step1[2] = step1b[2][i] + step1b[13][i]; + ;step1[3] = step1b[3][i] + step1b[12][i]; + ;step1[12] = step1b[3][i] - step1b[12][i]; + ;step1[13] = step1b[2][i] - step1b[13][i]; + LOAD_FROM_OUTPUT 1, 12, 13, q0, q1 + vadd.s16 q2, q10, q1 + vadd.s16 q3, q11, q0 + vsub.s16 q4, q11, q0 + vsub.s16 q5, q10, q1 + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[12 * 32] = step1b[12][i] + step1b[19][i]; + ;output[13 * 32] = step1b[13][i] + step1b[18][i]; + ;output[18 * 32] = step1b[13][i] - step1b[18][i]; + ;output[19 * 32] = step1b[12][i] - step1b[19][i]; + LOAD_FROM_OUTPUT 13, 18, 19, q0, q1 + vadd.s16 q8, q4, q1 + vadd.s16 q9, q5, q0 + vsub.s16 q6, q5, q0 + vsub.s16 q7, q4, q1 + STORE_IN_OUTPUT 19, 18, 19, q6, q7 + STORE_IN_OUTPUT 19, 12, 13, q8, q9 + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[ 2 * 32] = step1b[2][i] + step1b[29][i]; + ;output[ 3 * 32] = step1b[3][i] + step1b[28][i]; + ;output[28 * 32] = step1b[3][i] - step1b[28][i]; + ;output[29 * 32] = step1b[2][i] - step1b[29][i]; + LOAD_FROM_OUTPUT 13, 28, 29, q0, q1 + vadd.s16 q4, q2, q1 + vadd.s16 q5, q3, q0 + vsub.s16 q6, q3, q0 + vsub.s16 q7, q2, q1 + STORE_IN_OUTPUT 29, 28, 29, q6, q7 + STORE_IN_OUTPUT 29, 2, 3, q4, q5 + ; -------------------------------------------------------------------------- + ; part of stage 7 + ;step1[4] = step1b[4][i] + step1b[11][i]; + ;step1[5] = step1b[5][i] + step1b[10][i]; + ;step1[10] = step1b[5][i] - step1b[10][i]; + ;step1[11] = step1b[4][i] - step1b[11][i]; + LOAD_FROM_OUTPUT 3, 10, 11, q0, q1 + vadd.s16 q2, q12, q1 + vadd.s16 q3, q13, q0 + vsub.s16 q4, q13, q0 + vsub.s16 q5, q12, q1 + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[10 * 32] = step1b[10][i] + step1b[21][i]; + ;output[11 * 32] = step1b[11][i] + step1b[20][i]; + ;output[20 * 32] = step1b[11][i] - step1b[20][i]; + ;output[21 * 32] = step1b[10][i] - step1b[21][i]; + LOAD_FROM_OUTPUT 11, 20, 21, q0, q1 + vadd.s16 q8, q4, q1 + vadd.s16 q9, q5, q0 + vsub.s16 q6, q5, q0 + vsub.s16 q7, q4, q1 + STORE_IN_OUTPUT 21, 20, 21, q6, q7 + STORE_IN_OUTPUT 21, 10, 11, q8, q9 + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[ 4 * 32] = step1b[4][i] + step1b[27][i]; + ;output[ 5 * 32] = step1b[5][i] + step1b[26][i]; + ;output[26 * 32] = step1b[5][i] - step1b[26][i]; + ;output[27 * 32] = step1b[4][i] - step1b[27][i]; + LOAD_FROM_OUTPUT 11, 26, 27, q0, q1 + vadd.s16 q4, q2, q1 + vadd.s16 q5, q3, q0 + vsub.s16 q6, q3, q0 + vsub.s16 q7, q2, q1 + STORE_IN_OUTPUT 27, 26, 27, q6, q7 + STORE_IN_OUTPUT 27, 4, 5, q4, q5 + ; -------------------------------------------------------------------------- + ; part of stage 7 + ;step1[6] = step1b[6][i] + step1b[9][i]; + ;step1[7] = step1b[7][i] + step1b[8][i]; + ;step1[8] = step1b[7][i] - step1b[8][i]; + ;step1[9] = step1b[6][i] - step1b[9][i]; + LOAD_FROM_OUTPUT 5, 8, 9, q0, q1 + vadd.s16 q2, q14, q1 + vadd.s16 q3, q15, q0 + vsub.s16 q4, q15, q0 + vsub.s16 q5, q14, q1 + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[ 8 * 32] = step1b[8][i] + step1b[23][i]; + ;output[ 9 * 32] = step1b[9][i] + step1b[22][i]; + ;output[22 * 32] = step1b[9][i] - step1b[22][i]; + ;output[23 * 32] = step1b[8][i] - step1b[23][i]; + LOAD_FROM_OUTPUT 9, 22, 23, q0, q1 + vadd.s16 q8, q4, q1 + vadd.s16 q9, q5, q0 + vsub.s16 q6, q5, q0 + vsub.s16 q7, q4, q1 + STORE_IN_OUTPUT 23, 22, 23, q6, q7 + STORE_IN_OUTPUT 23, 8, 9, q8, q9 + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[ 6 * 32] = step1b[6][i] + step1b[25][i]; + ;output[ 7 * 32] = step1b[7][i] + step1b[24][i]; + ;output[24 * 32] = step1b[7][i] - step1b[24][i]; + ;output[25 * 32] = step1b[6][i] - step1b[25][i]; + LOAD_FROM_OUTPUT 9, 24, 25, q0, q1 + vadd.s16 q4, q2, q1 + vadd.s16 q5, q3, q0 + vsub.s16 q6, q3, q0 + vsub.s16 q7, q2, q1 + STORE_IN_OUTPUT 25, 24, 25, q6, q7 + STORE_IN_OUTPUT 25, 6, 7, q4, q5 + + ; restore r0 by removing the last offset from the last + ; operation (LOAD_FROM_TRANSPOSED 16, 8, 24) => 24*8*2 + sub r0, r0, #24*8*2 + ; restore r1 by removing the last offset from the last + ; operation (STORE_IN_OUTPUT 24, 6, 7) => 7*32*2 + ; advance by 8 columns => 8*2 + sub r1, r1, #7*32*2 - 8*2 + ; advance by 8 lines (8*32*2) + ; go back by the two pairs from the loop (32*2) + add r3, r3, #8*32*2 - 32*2 + + ; bands loop processing + subs r4, r4, #1 + bne idct32_bands_loop + + ; parameters for second pass + ; the input of pass2 is the result of pass1. we have to remove the offset + ; of 32 columns induced by the above idct32_bands_loop + sub r3, r1, #32*2 + ; r1 = pass2[32 * 32] + add r1, sp, #2048 + + ; pass loop processing + add r5, r5, #1 + b idct32_pass_loop + +idct32_bands_end_2nd_pass + STORE_COMBINE_CENTER_RESULTS + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[ 0 * 32] = step1b[0][i] + step1b[31][i]; + ;output[ 1 * 32] = step1b[1][i] + step1b[30][i]; + ;output[30 * 32] = step1b[1][i] - step1b[30][i]; + ;output[31 * 32] = step1b[0][i] - step1b[31][i]; + LOAD_FROM_OUTPUT 17, 30, 31, q0, q1 + vadd.s16 q4, q2, q1 + vadd.s16 q5, q3, q0 + vsub.s16 q6, q3, q0 + vsub.s16 q7, q2, q1 + STORE_COMBINE_EXTREME_RESULTS + ; -------------------------------------------------------------------------- + ; part of stage 7 + ;step1[2] = step1b[2][i] + step1b[13][i]; + ;step1[3] = step1b[3][i] + step1b[12][i]; + ;step1[12] = step1b[3][i] - step1b[12][i]; + ;step1[13] = step1b[2][i] - step1b[13][i]; + LOAD_FROM_OUTPUT 31, 12, 13, q0, q1 + vadd.s16 q2, q10, q1 + vadd.s16 q3, q11, q0 + vsub.s16 q4, q11, q0 + vsub.s16 q5, q10, q1 + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[12 * 32] = step1b[12][i] + step1b[19][i]; + ;output[13 * 32] = step1b[13][i] + step1b[18][i]; + ;output[18 * 32] = step1b[13][i] - step1b[18][i]; + ;output[19 * 32] = step1b[12][i] - step1b[19][i]; + LOAD_FROM_OUTPUT 13, 18, 19, q0, q1 + vadd.s16 q8, q4, q1 + vadd.s16 q9, q5, q0 + vsub.s16 q6, q5, q0 + vsub.s16 q7, q4, q1 + STORE_COMBINE_CENTER_RESULTS + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[ 2 * 32] = step1b[2][i] + step1b[29][i]; + ;output[ 3 * 32] = step1b[3][i] + step1b[28][i]; + ;output[28 * 32] = step1b[3][i] - step1b[28][i]; + ;output[29 * 32] = step1b[2][i] - step1b[29][i]; + LOAD_FROM_OUTPUT 19, 28, 29, q0, q1 + vadd.s16 q4, q2, q1 + vadd.s16 q5, q3, q0 + vsub.s16 q6, q3, q0 + vsub.s16 q7, q2, q1 + STORE_COMBINE_EXTREME_RESULTS + ; -------------------------------------------------------------------------- + ; part of stage 7 + ;step1[4] = step1b[4][i] + step1b[11][i]; + ;step1[5] = step1b[5][i] + step1b[10][i]; + ;step1[10] = step1b[5][i] - step1b[10][i]; + ;step1[11] = step1b[4][i] - step1b[11][i]; + LOAD_FROM_OUTPUT 29, 10, 11, q0, q1 + vadd.s16 q2, q12, q1 + vadd.s16 q3, q13, q0 + vsub.s16 q4, q13, q0 + vsub.s16 q5, q12, q1 + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[10 * 32] = step1b[10][i] + step1b[21][i]; + ;output[11 * 32] = step1b[11][i] + step1b[20][i]; + ;output[20 * 32] = step1b[11][i] - step1b[20][i]; + ;output[21 * 32] = step1b[10][i] - step1b[21][i]; + LOAD_FROM_OUTPUT 11, 20, 21, q0, q1 + vadd.s16 q8, q4, q1 + vadd.s16 q9, q5, q0 + vsub.s16 q6, q5, q0 + vsub.s16 q7, q4, q1 + STORE_COMBINE_CENTER_RESULTS + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[ 4 * 32] = step1b[4][i] + step1b[27][i]; + ;output[ 5 * 32] = step1b[5][i] + step1b[26][i]; + ;output[26 * 32] = step1b[5][i] - step1b[26][i]; + ;output[27 * 32] = step1b[4][i] - step1b[27][i]; + LOAD_FROM_OUTPUT 21, 26, 27, q0, q1 + vadd.s16 q4, q2, q1 + vadd.s16 q5, q3, q0 + vsub.s16 q6, q3, q0 + vsub.s16 q7, q2, q1 + STORE_COMBINE_EXTREME_RESULTS + ; -------------------------------------------------------------------------- + ; part of stage 7 + ;step1[6] = step1b[6][i] + step1b[9][i]; + ;step1[7] = step1b[7][i] + step1b[8][i]; + ;step1[8] = step1b[7][i] - step1b[8][i]; + ;step1[9] = step1b[6][i] - step1b[9][i]; + LOAD_FROM_OUTPUT 27, 8, 9, q0, q1 + vadd.s16 q2, q14, q1 + vadd.s16 q3, q15, q0 + vsub.s16 q4, q15, q0 + vsub.s16 q5, q14, q1 + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[ 8 * 32] = step1b[8][i] + step1b[23][i]; + ;output[ 9 * 32] = step1b[9][i] + step1b[22][i]; + ;output[22 * 32] = step1b[9][i] - step1b[22][i]; + ;output[23 * 32] = step1b[8][i] - step1b[23][i]; + LOAD_FROM_OUTPUT 9, 22, 23, q0, q1 + vadd.s16 q8, q4, q1 + vadd.s16 q9, q5, q0 + vsub.s16 q6, q5, q0 + vsub.s16 q7, q4, q1 + STORE_COMBINE_CENTER_RESULTS_LAST + ; -------------------------------------------------------------------------- + ; part of final stage + ;output[ 6 * 32] = step1b[6][i] + step1b[25][i]; + ;output[ 7 * 32] = step1b[7][i] + step1b[24][i]; + ;output[24 * 32] = step1b[7][i] - step1b[24][i]; + ;output[25 * 32] = step1b[6][i] - step1b[25][i]; + LOAD_FROM_OUTPUT 23, 24, 25, q0, q1 + vadd.s16 q4, q2, q1 + vadd.s16 q5, q3, q0 + vsub.s16 q6, q3, q0 + vsub.s16 q7, q2, q1 + STORE_COMBINE_EXTREME_RESULTS_LAST + ; -------------------------------------------------------------------------- + ; restore pointers to their initial indices for next band pass by + ; removing/adding dest_stride * 8. The actual increment by eight + ; is taken care of within the _LAST macros. + add r6, r6, r2, lsl #3 + add r9, r9, r2, lsl #3 + sub r7, r7, r2, lsl #3 + sub r10, r10, r2, lsl #3 + + ; restore r0 by removing the last offset from the last + ; operation (LOAD_FROM_TRANSPOSED 16, 8, 24) => 24*8*2 + sub r0, r0, #24*8*2 + ; restore r1 by removing the last offset from the last + ; operation (LOAD_FROM_OUTPUT 23, 24, 25) => 25*32*2 + ; advance by 8 columns => 8*2 + sub r1, r1, #25*32*2 - 8*2 + ; advance by 8 lines (8*32*2) + ; go back by the two pairs from the loop (32*2) + add r3, r3, #8*32*2 - 32*2 + + ; bands loop processing + subs r4, r4, #1 + bne idct32_bands_loop + + ; stack operation + add sp, sp, #512+2048+2048 + vpop {d8-d15} + pop {r4-r11} + bx lr + ENDP ; |aom_idct32x32_1024_add_neon| + END diff --git a/third_party/aom/aom_dsp/arm/idct32x32_add_neon.c b/third_party/aom/aom_dsp/arm/idct32x32_add_neon.c new file mode 100644 index 0000000000..a7562c7d5d --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct32x32_add_neon.c @@ -0,0 +1,686 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "aom_dsp/txfm_common.h" + +#define LOAD_FROM_TRANSPOSED(prev, first, second) \ + q14s16 = vld1q_s16(trans_buf + first * 8); \ + q13s16 = vld1q_s16(trans_buf + second * 8); + +#define LOAD_FROM_OUTPUT(prev, first, second, qA, qB) \ + qA = vld1q_s16(out + first * 32); \ + qB = vld1q_s16(out + second * 32); + +#define STORE_IN_OUTPUT(prev, first, second, qA, qB) \ + vst1q_s16(out + first * 32, qA); \ + vst1q_s16(out + second * 32, qB); + +#define STORE_COMBINE_CENTER_RESULTS(r10, r9) \ + __STORE_COMBINE_CENTER_RESULTS(r10, r9, stride, q6s16, q7s16, q8s16, q9s16); +static INLINE void __STORE_COMBINE_CENTER_RESULTS(uint8_t *p1, uint8_t *p2, + int stride, int16x8_t q6s16, + int16x8_t q7s16, + int16x8_t q8s16, + int16x8_t q9s16) { + int16x4_t d8s16, d9s16, d10s16, d11s16; + + d8s16 = vld1_s16((int16_t *)p1); + p1 += stride; + d11s16 = vld1_s16((int16_t *)p2); + p2 -= stride; + d9s16 = vld1_s16((int16_t *)p1); + d10s16 = vld1_s16((int16_t *)p2); + + q7s16 = vrshrq_n_s16(q7s16, 6); + q8s16 = vrshrq_n_s16(q8s16, 6); + q9s16 = vrshrq_n_s16(q9s16, 6); + q6s16 = vrshrq_n_s16(q6s16, 6); + + q7s16 = vreinterpretq_s16_u16( + vaddw_u8(vreinterpretq_u16_s16(q7s16), vreinterpret_u8_s16(d9s16))); + q8s16 = vreinterpretq_s16_u16( + vaddw_u8(vreinterpretq_u16_s16(q8s16), vreinterpret_u8_s16(d10s16))); + q9s16 = vreinterpretq_s16_u16( + vaddw_u8(vreinterpretq_u16_s16(q9s16), vreinterpret_u8_s16(d11s16))); + q6s16 = vreinterpretq_s16_u16( + vaddw_u8(vreinterpretq_u16_s16(q6s16), vreinterpret_u8_s16(d8s16))); + + d9s16 = vreinterpret_s16_u8(vqmovun_s16(q7s16)); + d10s16 = vreinterpret_s16_u8(vqmovun_s16(q8s16)); + d11s16 = vreinterpret_s16_u8(vqmovun_s16(q9s16)); + d8s16 = vreinterpret_s16_u8(vqmovun_s16(q6s16)); + + vst1_s16((int16_t *)p1, d9s16); + p1 -= stride; + vst1_s16((int16_t *)p2, d10s16); + p2 += stride; + vst1_s16((int16_t *)p1, d8s16); + vst1_s16((int16_t *)p2, d11s16); + return; +} + +#define STORE_COMBINE_EXTREME_RESULTS(r7, r6) \ + ; \ + __STORE_COMBINE_EXTREME_RESULTS(r7, r6, stride, q4s16, q5s16, q6s16, q7s16); +static INLINE void __STORE_COMBINE_EXTREME_RESULTS(uint8_t *p1, uint8_t *p2, + int stride, int16x8_t q4s16, + int16x8_t q5s16, + int16x8_t q6s16, + int16x8_t q7s16) { + int16x4_t d4s16, d5s16, d6s16, d7s16; + + d4s16 = vld1_s16((int16_t *)p1); + p1 += stride; + d7s16 = vld1_s16((int16_t *)p2); + p2 -= stride; + d5s16 = vld1_s16((int16_t *)p1); + d6s16 = vld1_s16((int16_t *)p2); + + q5s16 = vrshrq_n_s16(q5s16, 6); + q6s16 = vrshrq_n_s16(q6s16, 6); + q7s16 = vrshrq_n_s16(q7s16, 6); + q4s16 = vrshrq_n_s16(q4s16, 6); + + q5s16 = vreinterpretq_s16_u16( + vaddw_u8(vreinterpretq_u16_s16(q5s16), vreinterpret_u8_s16(d5s16))); + q6s16 = vreinterpretq_s16_u16( + vaddw_u8(vreinterpretq_u16_s16(q6s16), vreinterpret_u8_s16(d6s16))); + q7s16 = vreinterpretq_s16_u16( + vaddw_u8(vreinterpretq_u16_s16(q7s16), vreinterpret_u8_s16(d7s16))); + q4s16 = vreinterpretq_s16_u16( + vaddw_u8(vreinterpretq_u16_s16(q4s16), vreinterpret_u8_s16(d4s16))); + + d5s16 = vreinterpret_s16_u8(vqmovun_s16(q5s16)); + d6s16 = vreinterpret_s16_u8(vqmovun_s16(q6s16)); + d7s16 = vreinterpret_s16_u8(vqmovun_s16(q7s16)); + d4s16 = vreinterpret_s16_u8(vqmovun_s16(q4s16)); + + vst1_s16((int16_t *)p1, d5s16); + p1 -= stride; + vst1_s16((int16_t *)p2, d6s16); + p2 += stride; + vst1_s16((int16_t *)p2, d7s16); + vst1_s16((int16_t *)p1, d4s16); + return; +} + +#define DO_BUTTERFLY_STD(const_1, const_2, qA, qB) \ + DO_BUTTERFLY(q14s16, q13s16, const_1, const_2, qA, qB); +static INLINE void DO_BUTTERFLY(int16x8_t q14s16, int16x8_t q13s16, + int16_t first_const, int16_t second_const, + int16x8_t *qAs16, int16x8_t *qBs16) { + int16x4_t d30s16, d31s16; + int32x4_t q8s32, q9s32, q10s32, q11s32, q12s32, q15s32; + int16x4_t dCs16, dDs16, dAs16, dBs16; + + dCs16 = vget_low_s16(q14s16); + dDs16 = vget_high_s16(q14s16); + dAs16 = vget_low_s16(q13s16); + dBs16 = vget_high_s16(q13s16); + + d30s16 = vdup_n_s16(first_const); + d31s16 = vdup_n_s16(second_const); + + q8s32 = vmull_s16(dCs16, d30s16); + q10s32 = vmull_s16(dAs16, d31s16); + q9s32 = vmull_s16(dDs16, d30s16); + q11s32 = vmull_s16(dBs16, d31s16); + q12s32 = vmull_s16(dCs16, d31s16); + + q8s32 = vsubq_s32(q8s32, q10s32); + q9s32 = vsubq_s32(q9s32, q11s32); + + q10s32 = vmull_s16(dDs16, d31s16); + q11s32 = vmull_s16(dAs16, d30s16); + q15s32 = vmull_s16(dBs16, d30s16); + + q11s32 = vaddq_s32(q12s32, q11s32); + q10s32 = vaddq_s32(q10s32, q15s32); + + *qAs16 = vcombine_s16(vqrshrn_n_s32(q8s32, 14), vqrshrn_n_s32(q9s32, 14)); + *qBs16 = vcombine_s16(vqrshrn_n_s32(q11s32, 14), vqrshrn_n_s32(q10s32, 14)); + return; +} + +static INLINE void idct32_transpose_pair(int16_t *input, int16_t *t_buf) { + int16_t *in; + int i; + const int stride = 32; + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16, d23s16; + int16x4_t d24s16, d25s16, d26s16, d27s16, d28s16, d29s16, d30s16, d31s16; + int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16, q13s16, q14s16, q15s16; + int32x4x2_t q0x2s32, q1x2s32, q2x2s32, q3x2s32; + int16x8x2_t q0x2s16, q1x2s16, q2x2s16, q3x2s16; + + for (i = 0; i < 4; i++, input += 8) { + in = input; + q8s16 = vld1q_s16(in); + in += stride; + q9s16 = vld1q_s16(in); + in += stride; + q10s16 = vld1q_s16(in); + in += stride; + q11s16 = vld1q_s16(in); + in += stride; + q12s16 = vld1q_s16(in); + in += stride; + q13s16 = vld1q_s16(in); + in += stride; + q14s16 = vld1q_s16(in); + in += stride; + q15s16 = vld1q_s16(in); + + d16s16 = vget_low_s16(q8s16); + d17s16 = vget_high_s16(q8s16); + d18s16 = vget_low_s16(q9s16); + d19s16 = vget_high_s16(q9s16); + d20s16 = vget_low_s16(q10s16); + d21s16 = vget_high_s16(q10s16); + d22s16 = vget_low_s16(q11s16); + d23s16 = vget_high_s16(q11s16); + d24s16 = vget_low_s16(q12s16); + d25s16 = vget_high_s16(q12s16); + d26s16 = vget_low_s16(q13s16); + d27s16 = vget_high_s16(q13s16); + d28s16 = vget_low_s16(q14s16); + d29s16 = vget_high_s16(q14s16); + d30s16 = vget_low_s16(q15s16); + d31s16 = vget_high_s16(q15s16); + + q8s16 = vcombine_s16(d16s16, d24s16); // vswp d17, d24 + q9s16 = vcombine_s16(d18s16, d26s16); // vswp d19, d26 + q10s16 = vcombine_s16(d20s16, d28s16); // vswp d21, d28 + q11s16 = vcombine_s16(d22s16, d30s16); // vswp d23, d30 + q12s16 = vcombine_s16(d17s16, d25s16); + q13s16 = vcombine_s16(d19s16, d27s16); + q14s16 = vcombine_s16(d21s16, d29s16); + q15s16 = vcombine_s16(d23s16, d31s16); + + q0x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(q8s16), vreinterpretq_s32_s16(q10s16)); + q1x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(q9s16), vreinterpretq_s32_s16(q11s16)); + q2x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(q12s16), vreinterpretq_s32_s16(q14s16)); + q3x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(q13s16), vreinterpretq_s32_s16(q15s16)); + + q0x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q0x2s32.val[0]), // q8 + vreinterpretq_s16_s32(q1x2s32.val[0])); // q9 + q1x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q0x2s32.val[1]), // q10 + vreinterpretq_s16_s32(q1x2s32.val[1])); // q11 + q2x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q2x2s32.val[0]), // q12 + vreinterpretq_s16_s32(q3x2s32.val[0])); // q13 + q3x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q2x2s32.val[1]), // q14 + vreinterpretq_s16_s32(q3x2s32.val[1])); // q15 + + vst1q_s16(t_buf, q0x2s16.val[0]); + t_buf += 8; + vst1q_s16(t_buf, q0x2s16.val[1]); + t_buf += 8; + vst1q_s16(t_buf, q1x2s16.val[0]); + t_buf += 8; + vst1q_s16(t_buf, q1x2s16.val[1]); + t_buf += 8; + vst1q_s16(t_buf, q2x2s16.val[0]); + t_buf += 8; + vst1q_s16(t_buf, q2x2s16.val[1]); + t_buf += 8; + vst1q_s16(t_buf, q3x2s16.val[0]); + t_buf += 8; + vst1q_s16(t_buf, q3x2s16.val[1]); + t_buf += 8; + } + return; +} + +static INLINE void idct32_bands_end_1st_pass(int16_t *out, int16x8_t q2s16, + int16x8_t q3s16, int16x8_t q6s16, + int16x8_t q7s16, int16x8_t q8s16, + int16x8_t q9s16, int16x8_t q10s16, + int16x8_t q11s16, int16x8_t q12s16, + int16x8_t q13s16, int16x8_t q14s16, + int16x8_t q15s16) { + int16x8_t q0s16, q1s16, q4s16, q5s16; + + STORE_IN_OUTPUT(17, 16, 17, q6s16, q7s16); + STORE_IN_OUTPUT(17, 14, 15, q8s16, q9s16); + + LOAD_FROM_OUTPUT(15, 30, 31, q0s16, q1s16); + q4s16 = vaddq_s16(q2s16, q1s16); + q5s16 = vaddq_s16(q3s16, q0s16); + q6s16 = vsubq_s16(q3s16, q0s16); + q7s16 = vsubq_s16(q2s16, q1s16); + STORE_IN_OUTPUT(31, 30, 31, q6s16, q7s16); + STORE_IN_OUTPUT(31, 0, 1, q4s16, q5s16); + + LOAD_FROM_OUTPUT(1, 12, 13, q0s16, q1s16); + q2s16 = vaddq_s16(q10s16, q1s16); + q3s16 = vaddq_s16(q11s16, q0s16); + q4s16 = vsubq_s16(q11s16, q0s16); + q5s16 = vsubq_s16(q10s16, q1s16); + + LOAD_FROM_OUTPUT(13, 18, 19, q0s16, q1s16); + q8s16 = vaddq_s16(q4s16, q1s16); + q9s16 = vaddq_s16(q5s16, q0s16); + q6s16 = vsubq_s16(q5s16, q0s16); + q7s16 = vsubq_s16(q4s16, q1s16); + STORE_IN_OUTPUT(19, 18, 19, q6s16, q7s16); + STORE_IN_OUTPUT(19, 12, 13, q8s16, q9s16); + + LOAD_FROM_OUTPUT(13, 28, 29, q0s16, q1s16); + q4s16 = vaddq_s16(q2s16, q1s16); + q5s16 = vaddq_s16(q3s16, q0s16); + q6s16 = vsubq_s16(q3s16, q0s16); + q7s16 = vsubq_s16(q2s16, q1s16); + STORE_IN_OUTPUT(29, 28, 29, q6s16, q7s16); + STORE_IN_OUTPUT(29, 2, 3, q4s16, q5s16); + + LOAD_FROM_OUTPUT(3, 10, 11, q0s16, q1s16); + q2s16 = vaddq_s16(q12s16, q1s16); + q3s16 = vaddq_s16(q13s16, q0s16); + q4s16 = vsubq_s16(q13s16, q0s16); + q5s16 = vsubq_s16(q12s16, q1s16); + + LOAD_FROM_OUTPUT(11, 20, 21, q0s16, q1s16); + q8s16 = vaddq_s16(q4s16, q1s16); + q9s16 = vaddq_s16(q5s16, q0s16); + q6s16 = vsubq_s16(q5s16, q0s16); + q7s16 = vsubq_s16(q4s16, q1s16); + STORE_IN_OUTPUT(21, 20, 21, q6s16, q7s16); + STORE_IN_OUTPUT(21, 10, 11, q8s16, q9s16); + + LOAD_FROM_OUTPUT(11, 26, 27, q0s16, q1s16); + q4s16 = vaddq_s16(q2s16, q1s16); + q5s16 = vaddq_s16(q3s16, q0s16); + q6s16 = vsubq_s16(q3s16, q0s16); + q7s16 = vsubq_s16(q2s16, q1s16); + STORE_IN_OUTPUT(27, 26, 27, q6s16, q7s16); + STORE_IN_OUTPUT(27, 4, 5, q4s16, q5s16); + + LOAD_FROM_OUTPUT(5, 8, 9, q0s16, q1s16); + q2s16 = vaddq_s16(q14s16, q1s16); + q3s16 = vaddq_s16(q15s16, q0s16); + q4s16 = vsubq_s16(q15s16, q0s16); + q5s16 = vsubq_s16(q14s16, q1s16); + + LOAD_FROM_OUTPUT(9, 22, 23, q0s16, q1s16); + q8s16 = vaddq_s16(q4s16, q1s16); + q9s16 = vaddq_s16(q5s16, q0s16); + q6s16 = vsubq_s16(q5s16, q0s16); + q7s16 = vsubq_s16(q4s16, q1s16); + STORE_IN_OUTPUT(23, 22, 23, q6s16, q7s16); + STORE_IN_OUTPUT(23, 8, 9, q8s16, q9s16); + + LOAD_FROM_OUTPUT(9, 24, 25, q0s16, q1s16); + q4s16 = vaddq_s16(q2s16, q1s16); + q5s16 = vaddq_s16(q3s16, q0s16); + q6s16 = vsubq_s16(q3s16, q0s16); + q7s16 = vsubq_s16(q2s16, q1s16); + STORE_IN_OUTPUT(25, 24, 25, q6s16, q7s16); + STORE_IN_OUTPUT(25, 6, 7, q4s16, q5s16); + return; +} + +static INLINE void idct32_bands_end_2nd_pass( + int16_t *out, uint8_t *dest, int stride, int16x8_t q2s16, int16x8_t q3s16, + int16x8_t q6s16, int16x8_t q7s16, int16x8_t q8s16, int16x8_t q9s16, + int16x8_t q10s16, int16x8_t q11s16, int16x8_t q12s16, int16x8_t q13s16, + int16x8_t q14s16, int16x8_t q15s16) { + uint8_t *r6 = dest + 31 * stride; + uint8_t *r7 = dest /* + 0 * stride*/; + uint8_t *r9 = dest + 15 * stride; + uint8_t *r10 = dest + 16 * stride; + int str2 = stride << 1; + int16x8_t q0s16, q1s16, q4s16, q5s16; + + STORE_COMBINE_CENTER_RESULTS(r10, r9); + r10 += str2; + r9 -= str2; + + LOAD_FROM_OUTPUT(17, 30, 31, q0s16, q1s16) + q4s16 = vaddq_s16(q2s16, q1s16); + q5s16 = vaddq_s16(q3s16, q0s16); + q6s16 = vsubq_s16(q3s16, q0s16); + q7s16 = vsubq_s16(q2s16, q1s16); + STORE_COMBINE_EXTREME_RESULTS(r7, r6); + r7 += str2; + r6 -= str2; + + LOAD_FROM_OUTPUT(31, 12, 13, q0s16, q1s16) + q2s16 = vaddq_s16(q10s16, q1s16); + q3s16 = vaddq_s16(q11s16, q0s16); + q4s16 = vsubq_s16(q11s16, q0s16); + q5s16 = vsubq_s16(q10s16, q1s16); + + LOAD_FROM_OUTPUT(13, 18, 19, q0s16, q1s16) + q8s16 = vaddq_s16(q4s16, q1s16); + q9s16 = vaddq_s16(q5s16, q0s16); + q6s16 = vsubq_s16(q5s16, q0s16); + q7s16 = vsubq_s16(q4s16, q1s16); + STORE_COMBINE_CENTER_RESULTS(r10, r9); + r10 += str2; + r9 -= str2; + + LOAD_FROM_OUTPUT(19, 28, 29, q0s16, q1s16) + q4s16 = vaddq_s16(q2s16, q1s16); + q5s16 = vaddq_s16(q3s16, q0s16); + q6s16 = vsubq_s16(q3s16, q0s16); + q7s16 = vsubq_s16(q2s16, q1s16); + STORE_COMBINE_EXTREME_RESULTS(r7, r6); + r7 += str2; + r6 -= str2; + + LOAD_FROM_OUTPUT(29, 10, 11, q0s16, q1s16) + q2s16 = vaddq_s16(q12s16, q1s16); + q3s16 = vaddq_s16(q13s16, q0s16); + q4s16 = vsubq_s16(q13s16, q0s16); + q5s16 = vsubq_s16(q12s16, q1s16); + + LOAD_FROM_OUTPUT(11, 20, 21, q0s16, q1s16) + q8s16 = vaddq_s16(q4s16, q1s16); + q9s16 = vaddq_s16(q5s16, q0s16); + q6s16 = vsubq_s16(q5s16, q0s16); + q7s16 = vsubq_s16(q4s16, q1s16); + STORE_COMBINE_CENTER_RESULTS(r10, r9); + r10 += str2; + r9 -= str2; + + LOAD_FROM_OUTPUT(21, 26, 27, q0s16, q1s16) + q4s16 = vaddq_s16(q2s16, q1s16); + q5s16 = vaddq_s16(q3s16, q0s16); + q6s16 = vsubq_s16(q3s16, q0s16); + q7s16 = vsubq_s16(q2s16, q1s16); + STORE_COMBINE_EXTREME_RESULTS(r7, r6); + r7 += str2; + r6 -= str2; + + LOAD_FROM_OUTPUT(27, 8, 9, q0s16, q1s16) + q2s16 = vaddq_s16(q14s16, q1s16); + q3s16 = vaddq_s16(q15s16, q0s16); + q4s16 = vsubq_s16(q15s16, q0s16); + q5s16 = vsubq_s16(q14s16, q1s16); + + LOAD_FROM_OUTPUT(9, 22, 23, q0s16, q1s16) + q8s16 = vaddq_s16(q4s16, q1s16); + q9s16 = vaddq_s16(q5s16, q0s16); + q6s16 = vsubq_s16(q5s16, q0s16); + q7s16 = vsubq_s16(q4s16, q1s16); + STORE_COMBINE_CENTER_RESULTS(r10, r9); + + LOAD_FROM_OUTPUT(23, 24, 25, q0s16, q1s16) + q4s16 = vaddq_s16(q2s16, q1s16); + q5s16 = vaddq_s16(q3s16, q0s16); + q6s16 = vsubq_s16(q3s16, q0s16); + q7s16 = vsubq_s16(q2s16, q1s16); + STORE_COMBINE_EXTREME_RESULTS(r7, r6); + return; +} + +void aom_idct32x32_1024_add_neon(int16_t *input, uint8_t *dest, int stride) { + int i, idct32_pass_loop; + int16_t trans_buf[32 * 8]; + int16_t pass1[32 * 32]; + int16_t pass2[32 * 32]; + int16_t *out; + int16x8_t q0s16, q1s16, q2s16, q3s16, q4s16, q5s16, q6s16, q7s16; + int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16, q13s16, q14s16, q15s16; + + for (idct32_pass_loop = 0, out = pass1; idct32_pass_loop < 2; + idct32_pass_loop++, + input = pass1, // the input of pass2 is the result of pass1 + out = pass2) { + for (i = 0; i < 4; i++, input += 32 * 8, out += 8) { // idct32_bands_loop + idct32_transpose_pair(input, trans_buf); + + // ----------------------------------------- + // BLOCK A: 16-19,28-31 + // ----------------------------------------- + // generate 16,17,30,31 + // part of stage 1 + LOAD_FROM_TRANSPOSED(0, 1, 31) + DO_BUTTERFLY_STD(cospi_31_64, cospi_1_64, &q0s16, &q2s16) + LOAD_FROM_TRANSPOSED(31, 17, 15) + DO_BUTTERFLY_STD(cospi_15_64, cospi_17_64, &q1s16, &q3s16) + // part of stage 2 + q4s16 = vaddq_s16(q0s16, q1s16); + q13s16 = vsubq_s16(q0s16, q1s16); + q6s16 = vaddq_s16(q2s16, q3s16); + q14s16 = vsubq_s16(q2s16, q3s16); + // part of stage 3 + DO_BUTTERFLY_STD(cospi_28_64, cospi_4_64, &q5s16, &q7s16) + + // generate 18,19,28,29 + // part of stage 1 + LOAD_FROM_TRANSPOSED(15, 9, 23) + DO_BUTTERFLY_STD(cospi_23_64, cospi_9_64, &q0s16, &q2s16) + LOAD_FROM_TRANSPOSED(23, 25, 7) + DO_BUTTERFLY_STD(cospi_7_64, cospi_25_64, &q1s16, &q3s16) + // part of stage 2 + q13s16 = vsubq_s16(q3s16, q2s16); + q3s16 = vaddq_s16(q3s16, q2s16); + q14s16 = vsubq_s16(q1s16, q0s16); + q2s16 = vaddq_s16(q1s16, q0s16); + // part of stage 3 + DO_BUTTERFLY_STD(-cospi_4_64, -cospi_28_64, &q1s16, &q0s16) + // part of stage 4 + q8s16 = vaddq_s16(q4s16, q2s16); + q9s16 = vaddq_s16(q5s16, q0s16); + q10s16 = vaddq_s16(q7s16, q1s16); + q15s16 = vaddq_s16(q6s16, q3s16); + q13s16 = vsubq_s16(q5s16, q0s16); + q14s16 = vsubq_s16(q7s16, q1s16); + STORE_IN_OUTPUT(0, 16, 31, q8s16, q15s16) + STORE_IN_OUTPUT(31, 17, 30, q9s16, q10s16) + // part of stage 5 + DO_BUTTERFLY_STD(cospi_24_64, cospi_8_64, &q0s16, &q1s16) + STORE_IN_OUTPUT(30, 29, 18, q1s16, q0s16) + // part of stage 4 + q13s16 = vsubq_s16(q4s16, q2s16); + q14s16 = vsubq_s16(q6s16, q3s16); + // part of stage 5 + DO_BUTTERFLY_STD(cospi_24_64, cospi_8_64, &q4s16, &q6s16) + STORE_IN_OUTPUT(18, 19, 28, q4s16, q6s16) + + // ----------------------------------------- + // BLOCK B: 20-23,24-27 + // ----------------------------------------- + // generate 20,21,26,27 + // part of stage 1 + LOAD_FROM_TRANSPOSED(7, 5, 27) + DO_BUTTERFLY_STD(cospi_27_64, cospi_5_64, &q0s16, &q2s16) + LOAD_FROM_TRANSPOSED(27, 21, 11) + DO_BUTTERFLY_STD(cospi_11_64, cospi_21_64, &q1s16, &q3s16) + // part of stage 2 + q13s16 = vsubq_s16(q0s16, q1s16); + q0s16 = vaddq_s16(q0s16, q1s16); + q14s16 = vsubq_s16(q2s16, q3s16); + q2s16 = vaddq_s16(q2s16, q3s16); + // part of stage 3 + DO_BUTTERFLY_STD(cospi_12_64, cospi_20_64, &q1s16, &q3s16) + + // generate 22,23,24,25 + // part of stage 1 + LOAD_FROM_TRANSPOSED(11, 13, 19) + DO_BUTTERFLY_STD(cospi_19_64, cospi_13_64, &q5s16, &q7s16) + LOAD_FROM_TRANSPOSED(19, 29, 3) + DO_BUTTERFLY_STD(cospi_3_64, cospi_29_64, &q4s16, &q6s16) + // part of stage 2 + q14s16 = vsubq_s16(q4s16, q5s16); + q5s16 = vaddq_s16(q4s16, q5s16); + q13s16 = vsubq_s16(q6s16, q7s16); + q6s16 = vaddq_s16(q6s16, q7s16); + // part of stage 3 + DO_BUTTERFLY_STD(-cospi_20_64, -cospi_12_64, &q4s16, &q7s16) + // part of stage 4 + q10s16 = vaddq_s16(q7s16, q1s16); + q11s16 = vaddq_s16(q5s16, q0s16); + q12s16 = vaddq_s16(q6s16, q2s16); + q15s16 = vaddq_s16(q4s16, q3s16); + // part of stage 6 + LOAD_FROM_OUTPUT(28, 16, 17, q14s16, q13s16) + q8s16 = vaddq_s16(q14s16, q11s16); + q9s16 = vaddq_s16(q13s16, q10s16); + q13s16 = vsubq_s16(q13s16, q10s16); + q11s16 = vsubq_s16(q14s16, q11s16); + STORE_IN_OUTPUT(17, 17, 16, q9s16, q8s16) + LOAD_FROM_OUTPUT(16, 30, 31, q14s16, q9s16) + q8s16 = vsubq_s16(q9s16, q12s16); + q10s16 = vaddq_s16(q14s16, q15s16); + q14s16 = vsubq_s16(q14s16, q15s16); + q12s16 = vaddq_s16(q9s16, q12s16); + STORE_IN_OUTPUT(31, 30, 31, q10s16, q12s16) + // part of stage 7 + DO_BUTTERFLY_STD(cospi_16_64, cospi_16_64, &q13s16, &q14s16) + STORE_IN_OUTPUT(31, 25, 22, q14s16, q13s16) + q13s16 = q11s16; + q14s16 = q8s16; + DO_BUTTERFLY_STD(cospi_16_64, cospi_16_64, &q13s16, &q14s16) + STORE_IN_OUTPUT(22, 24, 23, q14s16, q13s16) + // part of stage 4 + q14s16 = vsubq_s16(q5s16, q0s16); + q13s16 = vsubq_s16(q6s16, q2s16); + DO_BUTTERFLY_STD(-cospi_8_64, -cospi_24_64, &q5s16, &q6s16); + q14s16 = vsubq_s16(q7s16, q1s16); + q13s16 = vsubq_s16(q4s16, q3s16); + DO_BUTTERFLY_STD(-cospi_8_64, -cospi_24_64, &q0s16, &q1s16); + // part of stage 6 + LOAD_FROM_OUTPUT(23, 18, 19, q14s16, q13s16) + q8s16 = vaddq_s16(q14s16, q1s16); + q9s16 = vaddq_s16(q13s16, q6s16); + q13s16 = vsubq_s16(q13s16, q6s16); + q1s16 = vsubq_s16(q14s16, q1s16); + STORE_IN_OUTPUT(19, 18, 19, q8s16, q9s16) + LOAD_FROM_OUTPUT(19, 28, 29, q8s16, q9s16) + q14s16 = vsubq_s16(q8s16, q5s16); + q10s16 = vaddq_s16(q8s16, q5s16); + q11s16 = vaddq_s16(q9s16, q0s16); + q0s16 = vsubq_s16(q9s16, q0s16); + STORE_IN_OUTPUT(29, 28, 29, q10s16, q11s16) + // part of stage 7 + DO_BUTTERFLY_STD(cospi_16_64, cospi_16_64, &q13s16, &q14s16) + STORE_IN_OUTPUT(29, 20, 27, q13s16, q14s16) + DO_BUTTERFLY(q0s16, q1s16, cospi_16_64, cospi_16_64, &q1s16, &q0s16); + STORE_IN_OUTPUT(27, 21, 26, q1s16, q0s16) + + // ----------------------------------------- + // BLOCK C: 8-10,11-15 + // ----------------------------------------- + // generate 8,9,14,15 + // part of stage 2 + LOAD_FROM_TRANSPOSED(3, 2, 30) + DO_BUTTERFLY_STD(cospi_30_64, cospi_2_64, &q0s16, &q2s16) + LOAD_FROM_TRANSPOSED(30, 18, 14) + DO_BUTTERFLY_STD(cospi_14_64, cospi_18_64, &q1s16, &q3s16) + // part of stage 3 + q13s16 = vsubq_s16(q0s16, q1s16); + q0s16 = vaddq_s16(q0s16, q1s16); + q14s16 = vsubq_s16(q2s16, q3s16); + q2s16 = vaddq_s16(q2s16, q3s16); + // part of stage 4 + DO_BUTTERFLY_STD(cospi_24_64, cospi_8_64, &q1s16, &q3s16) + + // generate 10,11,12,13 + // part of stage 2 + LOAD_FROM_TRANSPOSED(14, 10, 22) + DO_BUTTERFLY_STD(cospi_22_64, cospi_10_64, &q5s16, &q7s16) + LOAD_FROM_TRANSPOSED(22, 26, 6) + DO_BUTTERFLY_STD(cospi_6_64, cospi_26_64, &q4s16, &q6s16) + // part of stage 3 + q14s16 = vsubq_s16(q4s16, q5s16); + q5s16 = vaddq_s16(q4s16, q5s16); + q13s16 = vsubq_s16(q6s16, q7s16); + q6s16 = vaddq_s16(q6s16, q7s16); + // part of stage 4 + DO_BUTTERFLY_STD(-cospi_8_64, -cospi_24_64, &q4s16, &q7s16) + // part of stage 5 + q8s16 = vaddq_s16(q0s16, q5s16); + q9s16 = vaddq_s16(q1s16, q7s16); + q13s16 = vsubq_s16(q1s16, q7s16); + q14s16 = vsubq_s16(q3s16, q4s16); + q10s16 = vaddq_s16(q3s16, q4s16); + q15s16 = vaddq_s16(q2s16, q6s16); + STORE_IN_OUTPUT(26, 8, 15, q8s16, q15s16) + STORE_IN_OUTPUT(15, 9, 14, q9s16, q10s16) + // part of stage 6 + DO_BUTTERFLY_STD(cospi_16_64, cospi_16_64, &q1s16, &q3s16) + STORE_IN_OUTPUT(14, 13, 10, q3s16, q1s16) + q13s16 = vsubq_s16(q0s16, q5s16); + q14s16 = vsubq_s16(q2s16, q6s16); + DO_BUTTERFLY_STD(cospi_16_64, cospi_16_64, &q1s16, &q3s16) + STORE_IN_OUTPUT(10, 11, 12, q1s16, q3s16) + + // ----------------------------------------- + // BLOCK D: 0-3,4-7 + // ----------------------------------------- + // generate 4,5,6,7 + // part of stage 3 + LOAD_FROM_TRANSPOSED(6, 4, 28) + DO_BUTTERFLY_STD(cospi_28_64, cospi_4_64, &q0s16, &q2s16) + LOAD_FROM_TRANSPOSED(28, 20, 12) + DO_BUTTERFLY_STD(cospi_12_64, cospi_20_64, &q1s16, &q3s16) + // part of stage 4 + q13s16 = vsubq_s16(q0s16, q1s16); + q0s16 = vaddq_s16(q0s16, q1s16); + q14s16 = vsubq_s16(q2s16, q3s16); + q2s16 = vaddq_s16(q2s16, q3s16); + // part of stage 5 + DO_BUTTERFLY_STD(cospi_16_64, cospi_16_64, &q1s16, &q3s16) + + // generate 0,1,2,3 + // part of stage 4 + LOAD_FROM_TRANSPOSED(12, 0, 16) + DO_BUTTERFLY_STD(cospi_16_64, cospi_16_64, &q5s16, &q7s16) + LOAD_FROM_TRANSPOSED(16, 8, 24) + DO_BUTTERFLY_STD(cospi_24_64, cospi_8_64, &q14s16, &q6s16) + // part of stage 5 + q4s16 = vaddq_s16(q7s16, q6s16); + q7s16 = vsubq_s16(q7s16, q6s16); + q6s16 = vsubq_s16(q5s16, q14s16); + q5s16 = vaddq_s16(q5s16, q14s16); + // part of stage 6 + q8s16 = vaddq_s16(q4s16, q2s16); + q9s16 = vaddq_s16(q5s16, q3s16); + q10s16 = vaddq_s16(q6s16, q1s16); + q11s16 = vaddq_s16(q7s16, q0s16); + q12s16 = vsubq_s16(q7s16, q0s16); + q13s16 = vsubq_s16(q6s16, q1s16); + q14s16 = vsubq_s16(q5s16, q3s16); + q15s16 = vsubq_s16(q4s16, q2s16); + // part of stage 7 + LOAD_FROM_OUTPUT(12, 14, 15, q0s16, q1s16) + q2s16 = vaddq_s16(q8s16, q1s16); + q3s16 = vaddq_s16(q9s16, q0s16); + q4s16 = vsubq_s16(q9s16, q0s16); + q5s16 = vsubq_s16(q8s16, q1s16); + LOAD_FROM_OUTPUT(15, 16, 17, q0s16, q1s16) + q8s16 = vaddq_s16(q4s16, q1s16); + q9s16 = vaddq_s16(q5s16, q0s16); + q6s16 = vsubq_s16(q5s16, q0s16); + q7s16 = vsubq_s16(q4s16, q1s16); + + if (idct32_pass_loop == 0) { + idct32_bands_end_1st_pass(out, q2s16, q3s16, q6s16, q7s16, q8s16, q9s16, + q10s16, q11s16, q12s16, q13s16, q14s16, + q15s16); + } else { + idct32_bands_end_2nd_pass(out, dest, stride, q2s16, q3s16, q6s16, q7s16, + q8s16, q9s16, q10s16, q11s16, q12s16, q13s16, + q14s16, q15s16); + dest += 8; + } + } + } + return; +} diff --git a/third_party/aom/aom_dsp/arm/idct4x4_1_add_neon.asm b/third_party/aom/aom_dsp/arm/idct4x4_1_add_neon.asm new file mode 100644 index 0000000000..6bd733d5dd --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct4x4_1_add_neon.asm @@ -0,0 +1,71 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + + + + EXPORT |aom_idct4x4_1_add_neon| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +;void aom_idct4x4_1_add_neon(int16_t *input, uint8_t *dest, +; int dest_stride) +; +; r0 int16_t input +; r1 uint8_t *dest +; r2 int dest_stride) + +|aom_idct4x4_1_add_neon| PROC + ldrsh r0, [r0] + + ; generate cospi_16_64 = 11585 + mov r12, #0x2d00 + add r12, #0x41 + + ; out = dct_const_round_shift(input[0] * cospi_16_64) + mul r0, r0, r12 ; input[0] * cospi_16_64 + add r0, r0, #0x2000 ; +(1 << ((DCT_CONST_BITS) - 1)) + asr r0, r0, #14 ; >> DCT_CONST_BITS + + ; out = dct_const_round_shift(out * cospi_16_64) + mul r0, r0, r12 ; out * cospi_16_64 + mov r12, r1 ; save dest + add r0, r0, #0x2000 ; +(1 << ((DCT_CONST_BITS) - 1)) + asr r0, r0, #14 ; >> DCT_CONST_BITS + + ; a1 = ROUND_POWER_OF_TWO(out, 4) + add r0, r0, #8 ; + (1 <<((4) - 1)) + asr r0, r0, #4 ; >> 4 + + vdup.s16 q0, r0 ; duplicate a1 + + vld1.32 {d2[0]}, [r1], r2 + vld1.32 {d2[1]}, [r1], r2 + vld1.32 {d4[0]}, [r1], r2 + vld1.32 {d4[1]}, [r1] + + vaddw.u8 q8, q0, d2 ; dest[x] + a1 + vaddw.u8 q9, q0, d4 + + vqmovun.s16 d6, q8 ; clip_pixel + vqmovun.s16 d7, q9 + + vst1.32 {d6[0]}, [r12], r2 + vst1.32 {d6[1]}, [r12], r2 + vst1.32 {d7[0]}, [r12], r2 + vst1.32 {d7[1]}, [r12] + + bx lr + ENDP ; |aom_idct4x4_1_add_neon| + + END diff --git a/third_party/aom/aom_dsp/arm/idct4x4_1_add_neon.c b/third_party/aom/aom_dsp/arm/idct4x4_1_add_neon.c new file mode 100644 index 0000000000..3df7a901b0 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct4x4_1_add_neon.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom_dsp/inv_txfm.h" +#include "aom_ports/mem.h" + +void aom_idct4x4_1_add_neon(int16_t *input, uint8_t *dest, int dest_stride) { + uint8x8_t d6u8; + uint32x2_t d2u32 = vdup_n_u32(0); + uint16x8_t q8u16; + int16x8_t q0s16; + uint8_t *d1, *d2; + int16_t i, a1; + int16_t out = dct_const_round_shift(input[0] * cospi_16_64); + out = dct_const_round_shift(out * cospi_16_64); + a1 = ROUND_POWER_OF_TWO(out, 4); + + q0s16 = vdupq_n_s16(a1); + + // dc_only_idct_add + d1 = d2 = dest; + for (i = 0; i < 2; i++) { + d2u32 = vld1_lane_u32((const uint32_t *)d1, d2u32, 0); + d1 += dest_stride; + d2u32 = vld1_lane_u32((const uint32_t *)d1, d2u32, 1); + d1 += dest_stride; + + q8u16 = vaddw_u8(vreinterpretq_u16_s16(q0s16), vreinterpret_u8_u32(d2u32)); + d6u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16)); + + vst1_lane_u32((uint32_t *)d2, vreinterpret_u32_u8(d6u8), 0); + d2 += dest_stride; + vst1_lane_u32((uint32_t *)d2, vreinterpret_u32_u8(d6u8), 1); + d2 += dest_stride; + } + return; +} diff --git a/third_party/aom/aom_dsp/arm/idct4x4_add_neon.asm b/third_party/aom/aom_dsp/arm/idct4x4_add_neon.asm new file mode 100644 index 0000000000..127acf6140 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct4x4_add_neon.asm @@ -0,0 +1,193 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + EXPORT |aom_idct4x4_16_add_neon| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + + AREA Block, CODE, READONLY ; name this block of code +;void aom_idct4x4_16_add_neon(int16_t *input, uint8_t *dest, int dest_stride) +; +; r0 int16_t input +; r1 uint8_t *dest +; r2 int dest_stride) + +|aom_idct4x4_16_add_neon| PROC + + ; The 2D transform is done with two passes which are actually pretty + ; similar. We first transform the rows. This is done by transposing + ; the inputs, doing an SIMD column transform (the columns are the + ; transposed rows) and then transpose the results (so that it goes back + ; in normal/row positions). Then, we transform the columns by doing + ; another SIMD column transform. + ; So, two passes of a transpose followed by a column transform. + + ; load the inputs into q8-q9, d16-d19 + vld1.s16 {q8,q9}, [r0]! + + ; generate scalar constants + ; cospi_8_64 = 15137 = 0x3b21 + mov r0, #0x3b00 + add r0, #0x21 + ; cospi_16_64 = 11585 = 0x2d41 + mov r3, #0x2d00 + add r3, #0x41 + ; cospi_24_64 = 6270 = 0x 187e + mov r12, #0x1800 + add r12, #0x7e + + ; transpose the input data + ; 00 01 02 03 d16 + ; 10 11 12 13 d17 + ; 20 21 22 23 d18 + ; 30 31 32 33 d19 + vtrn.16 d16, d17 + vtrn.16 d18, d19 + + ; generate constant vectors + vdup.16 d20, r0 ; replicate cospi_8_64 + vdup.16 d21, r3 ; replicate cospi_16_64 + + ; 00 10 02 12 d16 + ; 01 11 03 13 d17 + ; 20 30 22 32 d18 + ; 21 31 23 33 d19 + vtrn.32 q8, q9 + ; 00 10 20 30 d16 + ; 01 11 21 31 d17 + ; 02 12 22 32 d18 + ; 03 13 23 33 d19 + + vdup.16 d22, r12 ; replicate cospi_24_64 + + ; do the transform on transposed rows + + ; stage 1 + vadd.s16 d23, d16, d18 ; (input[0] + input[2]) + vsub.s16 d24, d16, d18 ; (input[0] - input[2]) + + vmull.s16 q15, d17, d22 ; input[1] * cospi_24_64 + vmull.s16 q1, d17, d20 ; input[1] * cospi_8_64 + + ; (input[0] + input[2]) * cospi_16_64; + ; (input[0] - input[2]) * cospi_16_64; + vmull.s16 q13, d23, d21 + vmull.s16 q14, d24, d21 + + ; input[1] * cospi_24_64 - input[3] * cospi_8_64; + ; input[1] * cospi_8_64 + input[3] * cospi_24_64; + vmlsl.s16 q15, d19, d20 + vmlal.s16 q1, d19, d22 + + ; dct_const_round_shift + vqrshrn.s32 d26, q13, #14 + vqrshrn.s32 d27, q14, #14 + vqrshrn.s32 d29, q15, #14 + vqrshrn.s32 d28, q1, #14 + + ; stage 2 + ; output[0] = step[0] + step[3]; + ; output[1] = step[1] + step[2]; + ; output[3] = step[0] - step[3]; + ; output[2] = step[1] - step[2]; + vadd.s16 q8, q13, q14 + vsub.s16 q9, q13, q14 + vswp d18, d19 + + ; transpose the results + ; 00 01 02 03 d16 + ; 10 11 12 13 d17 + ; 20 21 22 23 d18 + ; 30 31 32 33 d19 + vtrn.16 d16, d17 + vtrn.16 d18, d19 + ; 00 10 02 12 d16 + ; 01 11 03 13 d17 + ; 20 30 22 32 d18 + ; 21 31 23 33 d19 + vtrn.32 q8, q9 + ; 00 10 20 30 d16 + ; 01 11 21 31 d17 + ; 02 12 22 32 d18 + ; 03 13 23 33 d19 + + ; do the transform on columns + + ; stage 1 + vadd.s16 d23, d16, d18 ; (input[0] + input[2]) + vsub.s16 d24, d16, d18 ; (input[0] - input[2]) + + vmull.s16 q15, d17, d22 ; input[1] * cospi_24_64 + vmull.s16 q1, d17, d20 ; input[1] * cospi_8_64 + + ; (input[0] + input[2]) * cospi_16_64; + ; (input[0] - input[2]) * cospi_16_64; + vmull.s16 q13, d23, d21 + vmull.s16 q14, d24, d21 + + ; input[1] * cospi_24_64 - input[3] * cospi_8_64; + ; input[1] * cospi_8_64 + input[3] * cospi_24_64; + vmlsl.s16 q15, d19, d20 + vmlal.s16 q1, d19, d22 + + ; dct_const_round_shift + vqrshrn.s32 d26, q13, #14 + vqrshrn.s32 d27, q14, #14 + vqrshrn.s32 d29, q15, #14 + vqrshrn.s32 d28, q1, #14 + + ; stage 2 + ; output[0] = step[0] + step[3]; + ; output[1] = step[1] + step[2]; + ; output[3] = step[0] - step[3]; + ; output[2] = step[1] - step[2]; + vadd.s16 q8, q13, q14 + vsub.s16 q9, q13, q14 + + ; The results are in two registers, one of them being swapped. This will + ; be taken care of by loading the 'dest' value in a swapped fashion and + ; also storing them in the same swapped fashion. + ; temp_out[0, 1] = d16, d17 = q8 + ; temp_out[2, 3] = d19, d18 = q9 swapped + + ; ROUND_POWER_OF_TWO(temp_out[j], 4) + vrshr.s16 q8, q8, #4 + vrshr.s16 q9, q9, #4 + + vld1.32 {d26[0]}, [r1], r2 + vld1.32 {d26[1]}, [r1], r2 + vld1.32 {d27[1]}, [r1], r2 + vld1.32 {d27[0]}, [r1] ; no post-increment + + ; ROUND_POWER_OF_TWO(temp_out[j], 4) + dest[j * dest_stride + i] + vaddw.u8 q8, q8, d26 + vaddw.u8 q9, q9, d27 + + ; clip_pixel + vqmovun.s16 d26, q8 + vqmovun.s16 d27, q9 + + ; do the stores in reverse order with negative post-increment, by changing + ; the sign of the stride + rsb r2, r2, #0 + vst1.32 {d27[0]}, [r1], r2 + vst1.32 {d27[1]}, [r1], r2 + vst1.32 {d26[1]}, [r1], r2 + vst1.32 {d26[0]}, [r1] ; no post-increment + bx lr + ENDP ; |aom_idct4x4_16_add_neon| + + END diff --git a/third_party/aom/aom_dsp/arm/idct4x4_add_neon.c b/third_party/aom/aom_dsp/arm/idct4x4_add_neon.c new file mode 100644 index 0000000000..763be1ab0e --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct4x4_add_neon.c @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom_dsp/txfm_common.h" + +void aom_idct4x4_16_add_neon(int16_t *input, uint8_t *dest, int dest_stride) { + uint8x8_t d26u8, d27u8; + uint32x2_t d26u32, d27u32; + uint16x8_t q8u16, q9u16; + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16; + int16x4_t d22s16, d23s16, d24s16, d26s16, d27s16, d28s16, d29s16; + int16x8_t q8s16, q9s16, q13s16, q14s16; + int32x4_t q1s32, q13s32, q14s32, q15s32; + int16x4x2_t d0x2s16, d1x2s16; + int32x4x2_t q0x2s32; + uint8_t *d; + + d26u32 = d27u32 = vdup_n_u32(0); + + q8s16 = vld1q_s16(input); + q9s16 = vld1q_s16(input + 8); + + d16s16 = vget_low_s16(q8s16); + d17s16 = vget_high_s16(q8s16); + d18s16 = vget_low_s16(q9s16); + d19s16 = vget_high_s16(q9s16); + + d0x2s16 = vtrn_s16(d16s16, d17s16); + d1x2s16 = vtrn_s16(d18s16, d19s16); + q8s16 = vcombine_s16(d0x2s16.val[0], d0x2s16.val[1]); + q9s16 = vcombine_s16(d1x2s16.val[0], d1x2s16.val[1]); + + d20s16 = vdup_n_s16((int16_t)cospi_8_64); + d21s16 = vdup_n_s16((int16_t)cospi_16_64); + + q0x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(q8s16), vreinterpretq_s32_s16(q9s16)); + d16s16 = vget_low_s16(vreinterpretq_s16_s32(q0x2s32.val[0])); + d17s16 = vget_high_s16(vreinterpretq_s16_s32(q0x2s32.val[0])); + d18s16 = vget_low_s16(vreinterpretq_s16_s32(q0x2s32.val[1])); + d19s16 = vget_high_s16(vreinterpretq_s16_s32(q0x2s32.val[1])); + + d22s16 = vdup_n_s16((int16_t)cospi_24_64); + + // stage 1 + d23s16 = vadd_s16(d16s16, d18s16); + d24s16 = vsub_s16(d16s16, d18s16); + + q15s32 = vmull_s16(d17s16, d22s16); + q1s32 = vmull_s16(d17s16, d20s16); + q13s32 = vmull_s16(d23s16, d21s16); + q14s32 = vmull_s16(d24s16, d21s16); + + q15s32 = vmlsl_s16(q15s32, d19s16, d20s16); + q1s32 = vmlal_s16(q1s32, d19s16, d22s16); + + d26s16 = vqrshrn_n_s32(q13s32, 14); + d27s16 = vqrshrn_n_s32(q14s32, 14); + d29s16 = vqrshrn_n_s32(q15s32, 14); + d28s16 = vqrshrn_n_s32(q1s32, 14); + q13s16 = vcombine_s16(d26s16, d27s16); + q14s16 = vcombine_s16(d28s16, d29s16); + + // stage 2 + q8s16 = vaddq_s16(q13s16, q14s16); + q9s16 = vsubq_s16(q13s16, q14s16); + + d16s16 = vget_low_s16(q8s16); + d17s16 = vget_high_s16(q8s16); + d18s16 = vget_high_s16(q9s16); // vswp d18 d19 + d19s16 = vget_low_s16(q9s16); + + d0x2s16 = vtrn_s16(d16s16, d17s16); + d1x2s16 = vtrn_s16(d18s16, d19s16); + q8s16 = vcombine_s16(d0x2s16.val[0], d0x2s16.val[1]); + q9s16 = vcombine_s16(d1x2s16.val[0], d1x2s16.val[1]); + + q0x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(q8s16), vreinterpretq_s32_s16(q9s16)); + d16s16 = vget_low_s16(vreinterpretq_s16_s32(q0x2s32.val[0])); + d17s16 = vget_high_s16(vreinterpretq_s16_s32(q0x2s32.val[0])); + d18s16 = vget_low_s16(vreinterpretq_s16_s32(q0x2s32.val[1])); + d19s16 = vget_high_s16(vreinterpretq_s16_s32(q0x2s32.val[1])); + + // do the transform on columns + // stage 1 + d23s16 = vadd_s16(d16s16, d18s16); + d24s16 = vsub_s16(d16s16, d18s16); + + q15s32 = vmull_s16(d17s16, d22s16); + q1s32 = vmull_s16(d17s16, d20s16); + q13s32 = vmull_s16(d23s16, d21s16); + q14s32 = vmull_s16(d24s16, d21s16); + + q15s32 = vmlsl_s16(q15s32, d19s16, d20s16); + q1s32 = vmlal_s16(q1s32, d19s16, d22s16); + + d26s16 = vqrshrn_n_s32(q13s32, 14); + d27s16 = vqrshrn_n_s32(q14s32, 14); + d29s16 = vqrshrn_n_s32(q15s32, 14); + d28s16 = vqrshrn_n_s32(q1s32, 14); + q13s16 = vcombine_s16(d26s16, d27s16); + q14s16 = vcombine_s16(d28s16, d29s16); + + // stage 2 + q8s16 = vaddq_s16(q13s16, q14s16); + q9s16 = vsubq_s16(q13s16, q14s16); + + q8s16 = vrshrq_n_s16(q8s16, 4); + q9s16 = vrshrq_n_s16(q9s16, 4); + + d = dest; + d26u32 = vld1_lane_u32((const uint32_t *)d, d26u32, 0); + d += dest_stride; + d26u32 = vld1_lane_u32((const uint32_t *)d, d26u32, 1); + d += dest_stride; + d27u32 = vld1_lane_u32((const uint32_t *)d, d27u32, 1); + d += dest_stride; + d27u32 = vld1_lane_u32((const uint32_t *)d, d27u32, 0); + + q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16), vreinterpret_u8_u32(d26u32)); + q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16), vreinterpret_u8_u32(d27u32)); + + d26u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16)); + d27u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16)); + + d = dest; + vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(d26u8), 0); + d += dest_stride; + vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(d26u8), 1); + d += dest_stride; + vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(d27u8), 1); + d += dest_stride; + vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(d27u8), 0); + return; +} diff --git a/third_party/aom/aom_dsp/arm/idct8x8_1_add_neon.asm b/third_party/aom/aom_dsp/arm/idct8x8_1_add_neon.asm new file mode 100644 index 0000000000..ec07e2053b --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct8x8_1_add_neon.asm @@ -0,0 +1,91 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + + + + EXPORT |aom_idct8x8_1_add_neon| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +;void aom_idct8x8_1_add_neon(int16_t *input, uint8_t *dest, +; int dest_stride) +; +; r0 int16_t input +; r1 uint8_t *dest +; r2 int dest_stride) + +|aom_idct8x8_1_add_neon| PROC + ldrsh r0, [r0] + + ; generate cospi_16_64 = 11585 + mov r12, #0x2d00 + add r12, #0x41 + + ; out = dct_const_round_shift(input[0] * cospi_16_64) + mul r0, r0, r12 ; input[0] * cospi_16_64 + add r0, r0, #0x2000 ; +(1 << ((DCT_CONST_BITS) - 1)) + asr r0, r0, #14 ; >> DCT_CONST_BITS + + ; out = dct_const_round_shift(out * cospi_16_64) + mul r0, r0, r12 ; out * cospi_16_64 + mov r12, r1 ; save dest + add r0, r0, #0x2000 ; +(1 << ((DCT_CONST_BITS) - 1)) + asr r0, r0, #14 ; >> DCT_CONST_BITS + + ; a1 = ROUND_POWER_OF_TWO(out, 5) + add r0, r0, #16 ; + (1 <<((5) - 1)) + asr r0, r0, #5 ; >> 5 + + vdup.s16 q0, r0 ; duplicate a1 + + ; load destination data + vld1.64 {d2}, [r1], r2 + vld1.64 {d3}, [r1], r2 + vld1.64 {d4}, [r1], r2 + vld1.64 {d5}, [r1], r2 + vld1.64 {d6}, [r1], r2 + vld1.64 {d7}, [r1], r2 + vld1.64 {d16}, [r1], r2 + vld1.64 {d17}, [r1] + + vaddw.u8 q9, q0, d2 ; dest[x] + a1 + vaddw.u8 q10, q0, d3 ; dest[x] + a1 + vaddw.u8 q11, q0, d4 ; dest[x] + a1 + vaddw.u8 q12, q0, d5 ; dest[x] + a1 + vqmovun.s16 d2, q9 ; clip_pixel + vqmovun.s16 d3, q10 ; clip_pixel + vqmovun.s16 d30, q11 ; clip_pixel + vqmovun.s16 d31, q12 ; clip_pixel + vst1.64 {d2}, [r12], r2 + vst1.64 {d3}, [r12], r2 + vst1.64 {d30}, [r12], r2 + vst1.64 {d31}, [r12], r2 + + vaddw.u8 q9, q0, d6 ; dest[x] + a1 + vaddw.u8 q10, q0, d7 ; dest[x] + a1 + vaddw.u8 q11, q0, d16 ; dest[x] + a1 + vaddw.u8 q12, q0, d17 ; dest[x] + a1 + vqmovun.s16 d2, q9 ; clip_pixel + vqmovun.s16 d3, q10 ; clip_pixel + vqmovun.s16 d30, q11 ; clip_pixel + vqmovun.s16 d31, q12 ; clip_pixel + vst1.64 {d2}, [r12], r2 + vst1.64 {d3}, [r12], r2 + vst1.64 {d30}, [r12], r2 + vst1.64 {d31}, [r12], r2 + + bx lr + ENDP ; |aom_idct8x8_1_add_neon| + + END diff --git a/third_party/aom/aom_dsp/arm/idct8x8_1_add_neon.c b/third_party/aom/aom_dsp/arm/idct8x8_1_add_neon.c new file mode 100644 index 0000000000..c7926f9e4f --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct8x8_1_add_neon.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom_dsp/inv_txfm.h" +#include "aom_ports/mem.h" + +void aom_idct8x8_1_add_neon(int16_t *input, uint8_t *dest, int dest_stride) { + uint8x8_t d2u8, d3u8, d30u8, d31u8; + uint64x1_t d2u64, d3u64, d4u64, d5u64; + uint16x8_t q0u16, q9u16, q10u16, q11u16, q12u16; + int16x8_t q0s16; + uint8_t *d1, *d2; + int16_t i, a1; + int16_t out = dct_const_round_shift(input[0] * cospi_16_64); + out = dct_const_round_shift(out * cospi_16_64); + a1 = ROUND_POWER_OF_TWO(out, 5); + + q0s16 = vdupq_n_s16(a1); + q0u16 = vreinterpretq_u16_s16(q0s16); + + d1 = d2 = dest; + for (i = 0; i < 2; i++) { + d2u64 = vld1_u64((const uint64_t *)d1); + d1 += dest_stride; + d3u64 = vld1_u64((const uint64_t *)d1); + d1 += dest_stride; + d4u64 = vld1_u64((const uint64_t *)d1); + d1 += dest_stride; + d5u64 = vld1_u64((const uint64_t *)d1); + d1 += dest_stride; + + q9u16 = vaddw_u8(q0u16, vreinterpret_u8_u64(d2u64)); + q10u16 = vaddw_u8(q0u16, vreinterpret_u8_u64(d3u64)); + q11u16 = vaddw_u8(q0u16, vreinterpret_u8_u64(d4u64)); + q12u16 = vaddw_u8(q0u16, vreinterpret_u8_u64(d5u64)); + + d2u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16)); + d3u8 = vqmovun_s16(vreinterpretq_s16_u16(q10u16)); + d30u8 = vqmovun_s16(vreinterpretq_s16_u16(q11u16)); + d31u8 = vqmovun_s16(vreinterpretq_s16_u16(q12u16)); + + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d2u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d3u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d30u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d31u8)); + d2 += dest_stride; + } + return; +} diff --git a/third_party/aom/aom_dsp/arm/idct8x8_add_neon.asm b/third_party/aom/aom_dsp/arm/idct8x8_add_neon.asm new file mode 100644 index 0000000000..f3d5f246d0 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct8x8_add_neon.asm @@ -0,0 +1,522 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + EXPORT |aom_idct8x8_64_add_neon| + EXPORT |aom_idct8x8_12_add_neon| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + + ; Parallel 1D IDCT on all the columns of a 8x8 16bit data matrix which are + ; loaded in q8-q15. The output will be stored back into q8-q15 registers. + ; This macro will touch q0-q7 registers and use them as buffer during + ; calculation. + MACRO + IDCT8x8_1D + ; stage 1 + vdup.16 d0, r3 ; duplicate cospi_28_64 + vdup.16 d1, r4 ; duplicate cospi_4_64 + vdup.16 d2, r5 ; duplicate cospi_12_64 + vdup.16 d3, r6 ; duplicate cospi_20_64 + + ; input[1] * cospi_28_64 + vmull.s16 q2, d18, d0 + vmull.s16 q3, d19, d0 + + ; input[5] * cospi_12_64 + vmull.s16 q5, d26, d2 + vmull.s16 q6, d27, d2 + + ; input[1]*cospi_28_64-input[7]*cospi_4_64 + vmlsl.s16 q2, d30, d1 + vmlsl.s16 q3, d31, d1 + + ; input[5] * cospi_12_64 - input[3] * cospi_20_64 + vmlsl.s16 q5, d22, d3 + vmlsl.s16 q6, d23, d3 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d8, q2, #14 ; >> 14 + vqrshrn.s32 d9, q3, #14 ; >> 14 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d10, q5, #14 ; >> 14 + vqrshrn.s32 d11, q6, #14 ; >> 14 + + ; input[1] * cospi_4_64 + vmull.s16 q2, d18, d1 + vmull.s16 q3, d19, d1 + + ; input[5] * cospi_20_64 + vmull.s16 q9, d26, d3 + vmull.s16 q13, d27, d3 + + ; input[1]*cospi_4_64+input[7]*cospi_28_64 + vmlal.s16 q2, d30, d0 + vmlal.s16 q3, d31, d0 + + ; input[5] * cospi_20_64 + input[3] * cospi_12_64 + vmlal.s16 q9, d22, d2 + vmlal.s16 q13, d23, d2 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d14, q2, #14 ; >> 14 + vqrshrn.s32 d15, q3, #14 ; >> 14 + + ; stage 2 & stage 3 - even half + vdup.16 d0, r7 ; duplicate cospi_16_64 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d12, q9, #14 ; >> 14 + vqrshrn.s32 d13, q13, #14 ; >> 14 + + ; input[0] * cospi_16_64 + vmull.s16 q2, d16, d0 + vmull.s16 q3, d17, d0 + + ; input[0] * cospi_16_64 + vmull.s16 q13, d16, d0 + vmull.s16 q15, d17, d0 + + ; (input[0] + input[2]) * cospi_16_64 + vmlal.s16 q2, d24, d0 + vmlal.s16 q3, d25, d0 + + ; (input[0] - input[2]) * cospi_16_64 + vmlsl.s16 q13, d24, d0 + vmlsl.s16 q15, d25, d0 + + vdup.16 d0, r8 ; duplicate cospi_24_64 + vdup.16 d1, r9 ; duplicate cospi_8_64 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d18, q2, #14 ; >> 14 + vqrshrn.s32 d19, q3, #14 ; >> 14 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d22, q13, #14 ; >> 14 + vqrshrn.s32 d23, q15, #14 ; >> 14 + + ; input[1] * cospi_24_64 - input[3] * cospi_8_64 + ; input[1] * cospi_24_64 + vmull.s16 q2, d20, d0 + vmull.s16 q3, d21, d0 + + ; input[1] * cospi_8_64 + vmull.s16 q8, d20, d1 + vmull.s16 q12, d21, d1 + + ; input[1] * cospi_24_64 - input[3] * cospi_8_64 + vmlsl.s16 q2, d28, d1 + vmlsl.s16 q3, d29, d1 + + ; input[1] * cospi_8_64 + input[3] * cospi_24_64 + vmlal.s16 q8, d28, d0 + vmlal.s16 q12, d29, d0 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d26, q2, #14 ; >> 14 + vqrshrn.s32 d27, q3, #14 ; >> 14 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d30, q8, #14 ; >> 14 + vqrshrn.s32 d31, q12, #14 ; >> 14 + + vadd.s16 q0, q9, q15 ; output[0] = step[0] + step[3] + vadd.s16 q1, q11, q13 ; output[1] = step[1] + step[2] + vsub.s16 q2, q11, q13 ; output[2] = step[1] - step[2] + vsub.s16 q3, q9, q15 ; output[3] = step[0] - step[3] + + ; stage 3 -odd half + vdup.16 d16, r7 ; duplicate cospi_16_64 + + ; stage 2 - odd half + vsub.s16 q13, q4, q5 ; step2[5] = step1[4] - step1[5] + vadd.s16 q4, q4, q5 ; step2[4] = step1[4] + step1[5] + vsub.s16 q14, q7, q6 ; step2[6] = -step1[6] + step1[7] + vadd.s16 q7, q7, q6 ; step2[7] = step1[6] + step1[7] + + ; step2[6] * cospi_16_64 + vmull.s16 q9, d28, d16 + vmull.s16 q10, d29, d16 + + ; step2[6] * cospi_16_64 + vmull.s16 q11, d28, d16 + vmull.s16 q12, d29, d16 + + ; (step2[6] - step2[5]) * cospi_16_64 + vmlsl.s16 q9, d26, d16 + vmlsl.s16 q10, d27, d16 + + ; (step2[5] + step2[6]) * cospi_16_64 + vmlal.s16 q11, d26, d16 + vmlal.s16 q12, d27, d16 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d10, q9, #14 ; >> 14 + vqrshrn.s32 d11, q10, #14 ; >> 14 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d12, q11, #14 ; >> 14 + vqrshrn.s32 d13, q12, #14 ; >> 14 + + ; stage 4 + vadd.s16 q8, q0, q7 ; output[0] = step1[0] + step1[7]; + vadd.s16 q9, q1, q6 ; output[1] = step1[1] + step1[6]; + vadd.s16 q10, q2, q5 ; output[2] = step1[2] + step1[5]; + vadd.s16 q11, q3, q4 ; output[3] = step1[3] + step1[4]; + vsub.s16 q12, q3, q4 ; output[4] = step1[3] - step1[4]; + vsub.s16 q13, q2, q5 ; output[5] = step1[2] - step1[5]; + vsub.s16 q14, q1, q6 ; output[6] = step1[1] - step1[6]; + vsub.s16 q15, q0, q7 ; output[7] = step1[0] - step1[7]; + MEND + + ; Transpose a 8x8 16bit data matrix. Datas are loaded in q8-q15. + MACRO + TRANSPOSE8X8 + vswp d17, d24 + vswp d23, d30 + vswp d21, d28 + vswp d19, d26 + vtrn.32 q8, q10 + vtrn.32 q9, q11 + vtrn.32 q12, q14 + vtrn.32 q13, q15 + vtrn.16 q8, q9 + vtrn.16 q10, q11 + vtrn.16 q12, q13 + vtrn.16 q14, q15 + MEND + + AREA Block, CODE, READONLY ; name this block of code +;void aom_idct8x8_64_add_neon(int16_t *input, uint8_t *dest, int dest_stride) +; +; r0 int16_t input +; r1 uint8_t *dest +; r2 int dest_stride) + +|aom_idct8x8_64_add_neon| PROC + push {r4-r9} + vpush {d8-d15} + vld1.s16 {q8,q9}, [r0]! + vld1.s16 {q10,q11}, [r0]! + vld1.s16 {q12,q13}, [r0]! + vld1.s16 {q14,q15}, [r0]! + + ; transpose the input data + TRANSPOSE8X8 + + ; generate cospi_28_64 = 3196 + mov r3, #0x0c00 + add r3, #0x7c + + ; generate cospi_4_64 = 16069 + mov r4, #0x3e00 + add r4, #0xc5 + + ; generate cospi_12_64 = 13623 + mov r5, #0x3500 + add r5, #0x37 + + ; generate cospi_20_64 = 9102 + mov r6, #0x2300 + add r6, #0x8e + + ; generate cospi_16_64 = 11585 + mov r7, #0x2d00 + add r7, #0x41 + + ; generate cospi_24_64 = 6270 + mov r8, #0x1800 + add r8, #0x7e + + ; generate cospi_8_64 = 15137 + mov r9, #0x3b00 + add r9, #0x21 + + ; First transform rows + IDCT8x8_1D + + ; Transpose the matrix + TRANSPOSE8X8 + + ; Then transform columns + IDCT8x8_1D + + ; ROUND_POWER_OF_TWO(temp_out[j], 5) + vrshr.s16 q8, q8, #5 + vrshr.s16 q9, q9, #5 + vrshr.s16 q10, q10, #5 + vrshr.s16 q11, q11, #5 + vrshr.s16 q12, q12, #5 + vrshr.s16 q13, q13, #5 + vrshr.s16 q14, q14, #5 + vrshr.s16 q15, q15, #5 + + ; save dest pointer + mov r0, r1 + + ; load destination data + vld1.64 {d0}, [r1], r2 + vld1.64 {d1}, [r1], r2 + vld1.64 {d2}, [r1], r2 + vld1.64 {d3}, [r1], r2 + vld1.64 {d4}, [r1], r2 + vld1.64 {d5}, [r1], r2 + vld1.64 {d6}, [r1], r2 + vld1.64 {d7}, [r1] + + ; ROUND_POWER_OF_TWO(temp_out[j], 5) + dest[j * dest_stride + i] + vaddw.u8 q8, q8, d0 + vaddw.u8 q9, q9, d1 + vaddw.u8 q10, q10, d2 + vaddw.u8 q11, q11, d3 + vaddw.u8 q12, q12, d4 + vaddw.u8 q13, q13, d5 + vaddw.u8 q14, q14, d6 + vaddw.u8 q15, q15, d7 + + ; clip_pixel + vqmovun.s16 d0, q8 + vqmovun.s16 d1, q9 + vqmovun.s16 d2, q10 + vqmovun.s16 d3, q11 + vqmovun.s16 d4, q12 + vqmovun.s16 d5, q13 + vqmovun.s16 d6, q14 + vqmovun.s16 d7, q15 + + ; store the data + vst1.64 {d0}, [r0], r2 + vst1.64 {d1}, [r0], r2 + vst1.64 {d2}, [r0], r2 + vst1.64 {d3}, [r0], r2 + vst1.64 {d4}, [r0], r2 + vst1.64 {d5}, [r0], r2 + vst1.64 {d6}, [r0], r2 + vst1.64 {d7}, [r0], r2 + + vpop {d8-d15} + pop {r4-r9} + bx lr + ENDP ; |aom_idct8x8_64_add_neon| + +;void aom_idct8x8_12_add_neon(int16_t *input, uint8_t *dest, int dest_stride) +; +; r0 int16_t input +; r1 uint8_t *dest +; r2 int dest_stride) + +|aom_idct8x8_12_add_neon| PROC + push {r4-r9} + vpush {d8-d15} + vld1.s16 {q8,q9}, [r0]! + vld1.s16 {q10,q11}, [r0]! + vld1.s16 {q12,q13}, [r0]! + vld1.s16 {q14,q15}, [r0]! + + ; transpose the input data + TRANSPOSE8X8 + + ; generate cospi_28_64 = 3196 + mov r3, #0x0c00 + add r3, #0x7c + + ; generate cospi_4_64 = 16069 + mov r4, #0x3e00 + add r4, #0xc5 + + ; generate cospi_12_64 = 13623 + mov r5, #0x3500 + add r5, #0x37 + + ; generate cospi_20_64 = 9102 + mov r6, #0x2300 + add r6, #0x8e + + ; generate cospi_16_64 = 11585 + mov r7, #0x2d00 + add r7, #0x41 + + ; generate cospi_24_64 = 6270 + mov r8, #0x1800 + add r8, #0x7e + + ; generate cospi_8_64 = 15137 + mov r9, #0x3b00 + add r9, #0x21 + + ; First transform rows + ; stage 1 + ; The following instructions use vqrdmulh to do the + ; dct_const_round_shift(input[1] * cospi_28_64). vqrdmulh will do doubling + ; multiply and shift the result by 16 bits instead of 14 bits. So we need + ; to double the constants before multiplying to compensate this. + mov r12, r3, lsl #1 + vdup.16 q0, r12 ; duplicate cospi_28_64*2 + mov r12, r4, lsl #1 + vdup.16 q1, r12 ; duplicate cospi_4_64*2 + + ; dct_const_round_shift(input[1] * cospi_28_64) + vqrdmulh.s16 q4, q9, q0 + + mov r12, r6, lsl #1 + rsb r12, #0 + vdup.16 q0, r12 ; duplicate -cospi_20_64*2 + + ; dct_const_round_shift(input[1] * cospi_4_64) + vqrdmulh.s16 q7, q9, q1 + + mov r12, r5, lsl #1 + vdup.16 q1, r12 ; duplicate cospi_12_64*2 + + ; dct_const_round_shift(- input[3] * cospi_20_64) + vqrdmulh.s16 q5, q11, q0 + + mov r12, r7, lsl #1 + vdup.16 q0, r12 ; duplicate cospi_16_64*2 + + ; dct_const_round_shift(input[3] * cospi_12_64) + vqrdmulh.s16 q6, q11, q1 + + ; stage 2 & stage 3 - even half + mov r12, r8, lsl #1 + vdup.16 q1, r12 ; duplicate cospi_24_64*2 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrdmulh.s16 q9, q8, q0 + + mov r12, r9, lsl #1 + vdup.16 q0, r12 ; duplicate cospi_8_64*2 + + ; dct_const_round_shift(input[1] * cospi_24_64) + vqrdmulh.s16 q13, q10, q1 + + ; dct_const_round_shift(input[1] * cospi_8_64) + vqrdmulh.s16 q15, q10, q0 + + ; stage 3 -odd half + vdup.16 d16, r7 ; duplicate cospi_16_64 + + vadd.s16 q0, q9, q15 ; output[0] = step[0] + step[3] + vadd.s16 q1, q9, q13 ; output[1] = step[1] + step[2] + vsub.s16 q2, q9, q13 ; output[2] = step[1] - step[2] + vsub.s16 q3, q9, q15 ; output[3] = step[0] - step[3] + + ; stage 2 - odd half + vsub.s16 q13, q4, q5 ; step2[5] = step1[4] - step1[5] + vadd.s16 q4, q4, q5 ; step2[4] = step1[4] + step1[5] + vsub.s16 q14, q7, q6 ; step2[6] = -step1[6] + step1[7] + vadd.s16 q7, q7, q6 ; step2[7] = step1[6] + step1[7] + + ; step2[6] * cospi_16_64 + vmull.s16 q9, d28, d16 + vmull.s16 q10, d29, d16 + + ; step2[6] * cospi_16_64 + vmull.s16 q11, d28, d16 + vmull.s16 q12, d29, d16 + + ; (step2[6] - step2[5]) * cospi_16_64 + vmlsl.s16 q9, d26, d16 + vmlsl.s16 q10, d27, d16 + + ; (step2[5] + step2[6]) * cospi_16_64 + vmlal.s16 q11, d26, d16 + vmlal.s16 q12, d27, d16 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d10, q9, #14 ; >> 14 + vqrshrn.s32 d11, q10, #14 ; >> 14 + + ; dct_const_round_shift(input_dc * cospi_16_64) + vqrshrn.s32 d12, q11, #14 ; >> 14 + vqrshrn.s32 d13, q12, #14 ; >> 14 + + ; stage 4 + vadd.s16 q8, q0, q7 ; output[0] = step1[0] + step1[7]; + vadd.s16 q9, q1, q6 ; output[1] = step1[1] + step1[6]; + vadd.s16 q10, q2, q5 ; output[2] = step1[2] + step1[5]; + vadd.s16 q11, q3, q4 ; output[3] = step1[3] + step1[4]; + vsub.s16 q12, q3, q4 ; output[4] = step1[3] - step1[4]; + vsub.s16 q13, q2, q5 ; output[5] = step1[2] - step1[5]; + vsub.s16 q14, q1, q6 ; output[6] = step1[1] - step1[6]; + vsub.s16 q15, q0, q7 ; output[7] = step1[0] - step1[7]; + + ; Transpose the matrix + TRANSPOSE8X8 + + ; Then transform columns + IDCT8x8_1D + + ; ROUND_POWER_OF_TWO(temp_out[j], 5) + vrshr.s16 q8, q8, #5 + vrshr.s16 q9, q9, #5 + vrshr.s16 q10, q10, #5 + vrshr.s16 q11, q11, #5 + vrshr.s16 q12, q12, #5 + vrshr.s16 q13, q13, #5 + vrshr.s16 q14, q14, #5 + vrshr.s16 q15, q15, #5 + + ; save dest pointer + mov r0, r1 + + ; load destination data + vld1.64 {d0}, [r1], r2 + vld1.64 {d1}, [r1], r2 + vld1.64 {d2}, [r1], r2 + vld1.64 {d3}, [r1], r2 + vld1.64 {d4}, [r1], r2 + vld1.64 {d5}, [r1], r2 + vld1.64 {d6}, [r1], r2 + vld1.64 {d7}, [r1] + + ; ROUND_POWER_OF_TWO(temp_out[j], 5) + dest[j * dest_stride + i] + vaddw.u8 q8, q8, d0 + vaddw.u8 q9, q9, d1 + vaddw.u8 q10, q10, d2 + vaddw.u8 q11, q11, d3 + vaddw.u8 q12, q12, d4 + vaddw.u8 q13, q13, d5 + vaddw.u8 q14, q14, d6 + vaddw.u8 q15, q15, d7 + + ; clip_pixel + vqmovun.s16 d0, q8 + vqmovun.s16 d1, q9 + vqmovun.s16 d2, q10 + vqmovun.s16 d3, q11 + vqmovun.s16 d4, q12 + vqmovun.s16 d5, q13 + vqmovun.s16 d6, q14 + vqmovun.s16 d7, q15 + + ; store the data + vst1.64 {d0}, [r0], r2 + vst1.64 {d1}, [r0], r2 + vst1.64 {d2}, [r0], r2 + vst1.64 {d3}, [r0], r2 + vst1.64 {d4}, [r0], r2 + vst1.64 {d5}, [r0], r2 + vst1.64 {d6}, [r0], r2 + vst1.64 {d7}, [r0], r2 + + vpop {d8-d15} + pop {r4-r9} + bx lr + ENDP ; |aom_idct8x8_12_add_neon| + + END diff --git a/third_party/aom/aom_dsp/arm/idct8x8_add_neon.c b/third_party/aom/aom_dsp/arm/idct8x8_add_neon.c new file mode 100644 index 0000000000..8ad70862d6 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/idct8x8_add_neon.c @@ -0,0 +1,509 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "aom_dsp/txfm_common.h" + +static INLINE void TRANSPOSE8X8(int16x8_t *q8s16, int16x8_t *q9s16, + int16x8_t *q10s16, int16x8_t *q11s16, + int16x8_t *q12s16, int16x8_t *q13s16, + int16x8_t *q14s16, int16x8_t *q15s16) { + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16, d23s16; + int16x4_t d24s16, d25s16, d26s16, d27s16, d28s16, d29s16, d30s16, d31s16; + int32x4x2_t q0x2s32, q1x2s32, q2x2s32, q3x2s32; + int16x8x2_t q0x2s16, q1x2s16, q2x2s16, q3x2s16; + + d16s16 = vget_low_s16(*q8s16); + d17s16 = vget_high_s16(*q8s16); + d18s16 = vget_low_s16(*q9s16); + d19s16 = vget_high_s16(*q9s16); + d20s16 = vget_low_s16(*q10s16); + d21s16 = vget_high_s16(*q10s16); + d22s16 = vget_low_s16(*q11s16); + d23s16 = vget_high_s16(*q11s16); + d24s16 = vget_low_s16(*q12s16); + d25s16 = vget_high_s16(*q12s16); + d26s16 = vget_low_s16(*q13s16); + d27s16 = vget_high_s16(*q13s16); + d28s16 = vget_low_s16(*q14s16); + d29s16 = vget_high_s16(*q14s16); + d30s16 = vget_low_s16(*q15s16); + d31s16 = vget_high_s16(*q15s16); + + *q8s16 = vcombine_s16(d16s16, d24s16); // vswp d17, d24 + *q9s16 = vcombine_s16(d18s16, d26s16); // vswp d19, d26 + *q10s16 = vcombine_s16(d20s16, d28s16); // vswp d21, d28 + *q11s16 = vcombine_s16(d22s16, d30s16); // vswp d23, d30 + *q12s16 = vcombine_s16(d17s16, d25s16); + *q13s16 = vcombine_s16(d19s16, d27s16); + *q14s16 = vcombine_s16(d21s16, d29s16); + *q15s16 = vcombine_s16(d23s16, d31s16); + + q0x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(*q8s16), vreinterpretq_s32_s16(*q10s16)); + q1x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(*q9s16), vreinterpretq_s32_s16(*q11s16)); + q2x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(*q12s16), vreinterpretq_s32_s16(*q14s16)); + q3x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(*q13s16), vreinterpretq_s32_s16(*q15s16)); + + q0x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q0x2s32.val[0]), // q8 + vreinterpretq_s16_s32(q1x2s32.val[0])); // q9 + q1x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q0x2s32.val[1]), // q10 + vreinterpretq_s16_s32(q1x2s32.val[1])); // q11 + q2x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q2x2s32.val[0]), // q12 + vreinterpretq_s16_s32(q3x2s32.val[0])); // q13 + q3x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q2x2s32.val[1]), // q14 + vreinterpretq_s16_s32(q3x2s32.val[1])); // q15 + + *q8s16 = q0x2s16.val[0]; + *q9s16 = q0x2s16.val[1]; + *q10s16 = q1x2s16.val[0]; + *q11s16 = q1x2s16.val[1]; + *q12s16 = q2x2s16.val[0]; + *q13s16 = q2x2s16.val[1]; + *q14s16 = q3x2s16.val[0]; + *q15s16 = q3x2s16.val[1]; + return; +} + +static INLINE void IDCT8x8_1D(int16x8_t *q8s16, int16x8_t *q9s16, + int16x8_t *q10s16, int16x8_t *q11s16, + int16x8_t *q12s16, int16x8_t *q13s16, + int16x8_t *q14s16, int16x8_t *q15s16) { + int16x4_t d0s16, d1s16, d2s16, d3s16; + int16x4_t d8s16, d9s16, d10s16, d11s16, d12s16, d13s16, d14s16, d15s16; + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16, d23s16; + int16x4_t d24s16, d25s16, d26s16, d27s16, d28s16, d29s16, d30s16, d31s16; + int16x8_t q0s16, q1s16, q2s16, q3s16, q4s16, q5s16, q6s16, q7s16; + int32x4_t q2s32, q3s32, q5s32, q6s32, q8s32, q9s32; + int32x4_t q10s32, q11s32, q12s32, q13s32, q15s32; + + d0s16 = vdup_n_s16((int16_t)cospi_28_64); + d1s16 = vdup_n_s16((int16_t)cospi_4_64); + d2s16 = vdup_n_s16((int16_t)cospi_12_64); + d3s16 = vdup_n_s16((int16_t)cospi_20_64); + + d16s16 = vget_low_s16(*q8s16); + d17s16 = vget_high_s16(*q8s16); + d18s16 = vget_low_s16(*q9s16); + d19s16 = vget_high_s16(*q9s16); + d20s16 = vget_low_s16(*q10s16); + d21s16 = vget_high_s16(*q10s16); + d22s16 = vget_low_s16(*q11s16); + d23s16 = vget_high_s16(*q11s16); + d24s16 = vget_low_s16(*q12s16); + d25s16 = vget_high_s16(*q12s16); + d26s16 = vget_low_s16(*q13s16); + d27s16 = vget_high_s16(*q13s16); + d28s16 = vget_low_s16(*q14s16); + d29s16 = vget_high_s16(*q14s16); + d30s16 = vget_low_s16(*q15s16); + d31s16 = vget_high_s16(*q15s16); + + q2s32 = vmull_s16(d18s16, d0s16); + q3s32 = vmull_s16(d19s16, d0s16); + q5s32 = vmull_s16(d26s16, d2s16); + q6s32 = vmull_s16(d27s16, d2s16); + + q2s32 = vmlsl_s16(q2s32, d30s16, d1s16); + q3s32 = vmlsl_s16(q3s32, d31s16, d1s16); + q5s32 = vmlsl_s16(q5s32, d22s16, d3s16); + q6s32 = vmlsl_s16(q6s32, d23s16, d3s16); + + d8s16 = vqrshrn_n_s32(q2s32, 14); + d9s16 = vqrshrn_n_s32(q3s32, 14); + d10s16 = vqrshrn_n_s32(q5s32, 14); + d11s16 = vqrshrn_n_s32(q6s32, 14); + q4s16 = vcombine_s16(d8s16, d9s16); + q5s16 = vcombine_s16(d10s16, d11s16); + + q2s32 = vmull_s16(d18s16, d1s16); + q3s32 = vmull_s16(d19s16, d1s16); + q9s32 = vmull_s16(d26s16, d3s16); + q13s32 = vmull_s16(d27s16, d3s16); + + q2s32 = vmlal_s16(q2s32, d30s16, d0s16); + q3s32 = vmlal_s16(q3s32, d31s16, d0s16); + q9s32 = vmlal_s16(q9s32, d22s16, d2s16); + q13s32 = vmlal_s16(q13s32, d23s16, d2s16); + + d14s16 = vqrshrn_n_s32(q2s32, 14); + d15s16 = vqrshrn_n_s32(q3s32, 14); + d12s16 = vqrshrn_n_s32(q9s32, 14); + d13s16 = vqrshrn_n_s32(q13s32, 14); + q6s16 = vcombine_s16(d12s16, d13s16); + q7s16 = vcombine_s16(d14s16, d15s16); + + d0s16 = vdup_n_s16((int16_t)cospi_16_64); + + q2s32 = vmull_s16(d16s16, d0s16); + q3s32 = vmull_s16(d17s16, d0s16); + q13s32 = vmull_s16(d16s16, d0s16); + q15s32 = vmull_s16(d17s16, d0s16); + + q2s32 = vmlal_s16(q2s32, d24s16, d0s16); + q3s32 = vmlal_s16(q3s32, d25s16, d0s16); + q13s32 = vmlsl_s16(q13s32, d24s16, d0s16); + q15s32 = vmlsl_s16(q15s32, d25s16, d0s16); + + d0s16 = vdup_n_s16((int16_t)cospi_24_64); + d1s16 = vdup_n_s16((int16_t)cospi_8_64); + + d18s16 = vqrshrn_n_s32(q2s32, 14); + d19s16 = vqrshrn_n_s32(q3s32, 14); + d22s16 = vqrshrn_n_s32(q13s32, 14); + d23s16 = vqrshrn_n_s32(q15s32, 14); + *q9s16 = vcombine_s16(d18s16, d19s16); + *q11s16 = vcombine_s16(d22s16, d23s16); + + q2s32 = vmull_s16(d20s16, d0s16); + q3s32 = vmull_s16(d21s16, d0s16); + q8s32 = vmull_s16(d20s16, d1s16); + q12s32 = vmull_s16(d21s16, d1s16); + + q2s32 = vmlsl_s16(q2s32, d28s16, d1s16); + q3s32 = vmlsl_s16(q3s32, d29s16, d1s16); + q8s32 = vmlal_s16(q8s32, d28s16, d0s16); + q12s32 = vmlal_s16(q12s32, d29s16, d0s16); + + d26s16 = vqrshrn_n_s32(q2s32, 14); + d27s16 = vqrshrn_n_s32(q3s32, 14); + d30s16 = vqrshrn_n_s32(q8s32, 14); + d31s16 = vqrshrn_n_s32(q12s32, 14); + *q13s16 = vcombine_s16(d26s16, d27s16); + *q15s16 = vcombine_s16(d30s16, d31s16); + + q0s16 = vaddq_s16(*q9s16, *q15s16); + q1s16 = vaddq_s16(*q11s16, *q13s16); + q2s16 = vsubq_s16(*q11s16, *q13s16); + q3s16 = vsubq_s16(*q9s16, *q15s16); + + *q13s16 = vsubq_s16(q4s16, q5s16); + q4s16 = vaddq_s16(q4s16, q5s16); + *q14s16 = vsubq_s16(q7s16, q6s16); + q7s16 = vaddq_s16(q7s16, q6s16); + d26s16 = vget_low_s16(*q13s16); + d27s16 = vget_high_s16(*q13s16); + d28s16 = vget_low_s16(*q14s16); + d29s16 = vget_high_s16(*q14s16); + + d16s16 = vdup_n_s16((int16_t)cospi_16_64); + + q9s32 = vmull_s16(d28s16, d16s16); + q10s32 = vmull_s16(d29s16, d16s16); + q11s32 = vmull_s16(d28s16, d16s16); + q12s32 = vmull_s16(d29s16, d16s16); + + q9s32 = vmlsl_s16(q9s32, d26s16, d16s16); + q10s32 = vmlsl_s16(q10s32, d27s16, d16s16); + q11s32 = vmlal_s16(q11s32, d26s16, d16s16); + q12s32 = vmlal_s16(q12s32, d27s16, d16s16); + + d10s16 = vqrshrn_n_s32(q9s32, 14); + d11s16 = vqrshrn_n_s32(q10s32, 14); + d12s16 = vqrshrn_n_s32(q11s32, 14); + d13s16 = vqrshrn_n_s32(q12s32, 14); + q5s16 = vcombine_s16(d10s16, d11s16); + q6s16 = vcombine_s16(d12s16, d13s16); + + *q8s16 = vaddq_s16(q0s16, q7s16); + *q9s16 = vaddq_s16(q1s16, q6s16); + *q10s16 = vaddq_s16(q2s16, q5s16); + *q11s16 = vaddq_s16(q3s16, q4s16); + *q12s16 = vsubq_s16(q3s16, q4s16); + *q13s16 = vsubq_s16(q2s16, q5s16); + *q14s16 = vsubq_s16(q1s16, q6s16); + *q15s16 = vsubq_s16(q0s16, q7s16); + return; +} + +void aom_idct8x8_64_add_neon(int16_t *input, uint8_t *dest, int dest_stride) { + uint8_t *d1, *d2; + uint8x8_t d0u8, d1u8, d2u8, d3u8; + uint64x1_t d0u64, d1u64, d2u64, d3u64; + int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16, q13s16, q14s16, q15s16; + uint16x8_t q8u16, q9u16, q10u16, q11u16; + + q8s16 = vld1q_s16(input); + q9s16 = vld1q_s16(input + 8); + q10s16 = vld1q_s16(input + 16); + q11s16 = vld1q_s16(input + 24); + q12s16 = vld1q_s16(input + 32); + q13s16 = vld1q_s16(input + 40); + q14s16 = vld1q_s16(input + 48); + q15s16 = vld1q_s16(input + 56); + + TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + IDCT8x8_1D(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + IDCT8x8_1D(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + q8s16 = vrshrq_n_s16(q8s16, 5); + q9s16 = vrshrq_n_s16(q9s16, 5); + q10s16 = vrshrq_n_s16(q10s16, 5); + q11s16 = vrshrq_n_s16(q11s16, 5); + q12s16 = vrshrq_n_s16(q12s16, 5); + q13s16 = vrshrq_n_s16(q13s16, 5); + q14s16 = vrshrq_n_s16(q14s16, 5); + q15s16 = vrshrq_n_s16(q15s16, 5); + + d1 = d2 = dest; + + d0u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d1u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d2u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d3u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + + q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16), vreinterpret_u8_u64(d0u64)); + q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16), vreinterpret_u8_u64(d1u64)); + q10u16 = vaddw_u8(vreinterpretq_u16_s16(q10s16), vreinterpret_u8_u64(d2u64)); + q11u16 = vaddw_u8(vreinterpretq_u16_s16(q11s16), vreinterpret_u8_u64(d3u64)); + + d0u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16)); + d1u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16)); + d2u8 = vqmovun_s16(vreinterpretq_s16_u16(q10u16)); + d3u8 = vqmovun_s16(vreinterpretq_s16_u16(q11u16)); + + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d0u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d1u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d2u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d3u8)); + d2 += dest_stride; + + q8s16 = q12s16; + q9s16 = q13s16; + q10s16 = q14s16; + q11s16 = q15s16; + + d0u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d1u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d2u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d3u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + + q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16), vreinterpret_u8_u64(d0u64)); + q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16), vreinterpret_u8_u64(d1u64)); + q10u16 = vaddw_u8(vreinterpretq_u16_s16(q10s16), vreinterpret_u8_u64(d2u64)); + q11u16 = vaddw_u8(vreinterpretq_u16_s16(q11s16), vreinterpret_u8_u64(d3u64)); + + d0u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16)); + d1u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16)); + d2u8 = vqmovun_s16(vreinterpretq_s16_u16(q10u16)); + d3u8 = vqmovun_s16(vreinterpretq_s16_u16(q11u16)); + + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d0u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d1u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d2u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d3u8)); + d2 += dest_stride; + return; +} + +void aom_idct8x8_12_add_neon(int16_t *input, uint8_t *dest, int dest_stride) { + uint8_t *d1, *d2; + uint8x8_t d0u8, d1u8, d2u8, d3u8; + int16x4_t d10s16, d11s16, d12s16, d13s16, d16s16; + int16x4_t d26s16, d27s16, d28s16, d29s16; + uint64x1_t d0u64, d1u64, d2u64, d3u64; + int16x8_t q0s16, q1s16, q2s16, q3s16, q4s16, q5s16, q6s16, q7s16; + int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16, q13s16, q14s16, q15s16; + uint16x8_t q8u16, q9u16, q10u16, q11u16; + int32x4_t q9s32, q10s32, q11s32, q12s32; + + q8s16 = vld1q_s16(input); + q9s16 = vld1q_s16(input + 8); + q10s16 = vld1q_s16(input + 16); + q11s16 = vld1q_s16(input + 24); + q12s16 = vld1q_s16(input + 32); + q13s16 = vld1q_s16(input + 40); + q14s16 = vld1q_s16(input + 48); + q15s16 = vld1q_s16(input + 56); + + TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + // First transform rows + // stage 1 + q0s16 = vdupq_n_s16((int16_t)cospi_28_64 * 2); + q1s16 = vdupq_n_s16((int16_t)cospi_4_64 * 2); + + q4s16 = vqrdmulhq_s16(q9s16, q0s16); + + q0s16 = vdupq_n_s16(-(int16_t)cospi_20_64 * 2); + + q7s16 = vqrdmulhq_s16(q9s16, q1s16); + + q1s16 = vdupq_n_s16((int16_t)cospi_12_64 * 2); + + q5s16 = vqrdmulhq_s16(q11s16, q0s16); + + q0s16 = vdupq_n_s16((int16_t)cospi_16_64 * 2); + + q6s16 = vqrdmulhq_s16(q11s16, q1s16); + + // stage 2 & stage 3 - even half + q1s16 = vdupq_n_s16((int16_t)cospi_24_64 * 2); + + q9s16 = vqrdmulhq_s16(q8s16, q0s16); + + q0s16 = vdupq_n_s16((int16_t)cospi_8_64 * 2); + + q13s16 = vqrdmulhq_s16(q10s16, q1s16); + + q15s16 = vqrdmulhq_s16(q10s16, q0s16); + + // stage 3 -odd half + q0s16 = vaddq_s16(q9s16, q15s16); + q1s16 = vaddq_s16(q9s16, q13s16); + q2s16 = vsubq_s16(q9s16, q13s16); + q3s16 = vsubq_s16(q9s16, q15s16); + + // stage 2 - odd half + q13s16 = vsubq_s16(q4s16, q5s16); + q4s16 = vaddq_s16(q4s16, q5s16); + q14s16 = vsubq_s16(q7s16, q6s16); + q7s16 = vaddq_s16(q7s16, q6s16); + d26s16 = vget_low_s16(q13s16); + d27s16 = vget_high_s16(q13s16); + d28s16 = vget_low_s16(q14s16); + d29s16 = vget_high_s16(q14s16); + + d16s16 = vdup_n_s16((int16_t)cospi_16_64); + q9s32 = vmull_s16(d28s16, d16s16); + q10s32 = vmull_s16(d29s16, d16s16); + q11s32 = vmull_s16(d28s16, d16s16); + q12s32 = vmull_s16(d29s16, d16s16); + + q9s32 = vmlsl_s16(q9s32, d26s16, d16s16); + q10s32 = vmlsl_s16(q10s32, d27s16, d16s16); + q11s32 = vmlal_s16(q11s32, d26s16, d16s16); + q12s32 = vmlal_s16(q12s32, d27s16, d16s16); + + d10s16 = vqrshrn_n_s32(q9s32, 14); + d11s16 = vqrshrn_n_s32(q10s32, 14); + d12s16 = vqrshrn_n_s32(q11s32, 14); + d13s16 = vqrshrn_n_s32(q12s32, 14); + q5s16 = vcombine_s16(d10s16, d11s16); + q6s16 = vcombine_s16(d12s16, d13s16); + + // stage 4 + q8s16 = vaddq_s16(q0s16, q7s16); + q9s16 = vaddq_s16(q1s16, q6s16); + q10s16 = vaddq_s16(q2s16, q5s16); + q11s16 = vaddq_s16(q3s16, q4s16); + q12s16 = vsubq_s16(q3s16, q4s16); + q13s16 = vsubq_s16(q2s16, q5s16); + q14s16 = vsubq_s16(q1s16, q6s16); + q15s16 = vsubq_s16(q0s16, q7s16); + + TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + IDCT8x8_1D(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + q8s16 = vrshrq_n_s16(q8s16, 5); + q9s16 = vrshrq_n_s16(q9s16, 5); + q10s16 = vrshrq_n_s16(q10s16, 5); + q11s16 = vrshrq_n_s16(q11s16, 5); + q12s16 = vrshrq_n_s16(q12s16, 5); + q13s16 = vrshrq_n_s16(q13s16, 5); + q14s16 = vrshrq_n_s16(q14s16, 5); + q15s16 = vrshrq_n_s16(q15s16, 5); + + d1 = d2 = dest; + + d0u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d1u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d2u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d3u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + + q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16), vreinterpret_u8_u64(d0u64)); + q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16), vreinterpret_u8_u64(d1u64)); + q10u16 = vaddw_u8(vreinterpretq_u16_s16(q10s16), vreinterpret_u8_u64(d2u64)); + q11u16 = vaddw_u8(vreinterpretq_u16_s16(q11s16), vreinterpret_u8_u64(d3u64)); + + d0u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16)); + d1u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16)); + d2u8 = vqmovun_s16(vreinterpretq_s16_u16(q10u16)); + d3u8 = vqmovun_s16(vreinterpretq_s16_u16(q11u16)); + + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d0u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d1u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d2u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d3u8)); + d2 += dest_stride; + + q8s16 = q12s16; + q9s16 = q13s16; + q10s16 = q14s16; + q11s16 = q15s16; + + d0u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d1u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d2u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d3u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + + q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16), vreinterpret_u8_u64(d0u64)); + q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16), vreinterpret_u8_u64(d1u64)); + q10u16 = vaddw_u8(vreinterpretq_u16_s16(q10s16), vreinterpret_u8_u64(d2u64)); + q11u16 = vaddw_u8(vreinterpretq_u16_s16(q11s16), vreinterpret_u8_u64(d3u64)); + + d0u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16)); + d1u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16)); + d2u8 = vqmovun_s16(vreinterpretq_s16_u16(q10u16)); + d3u8 = vqmovun_s16(vreinterpretq_s16_u16(q11u16)); + + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d0u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d1u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d2u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d3u8)); + d2 += dest_stride; + return; +} diff --git a/third_party/aom/aom_dsp/arm/intrapred_neon.c b/third_party/aom/aom_dsp/arm/intrapred_neon.c new file mode 100644 index 0000000000..2dc5b2e568 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/intrapred_neon.c @@ -0,0 +1,757 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +//------------------------------------------------------------------------------ +// DC 4x4 + +// 'do_above' and 'do_left' facilitate branch removal when inlined. +static INLINE void dc_4x4(uint8_t *dst, ptrdiff_t stride, const uint8_t *above, + const uint8_t *left, int do_above, int do_left) { + uint16x8_t sum_top; + uint16x8_t sum_left; + uint8x8_t dc0; + + if (do_above) { + const uint8x8_t A = vld1_u8(above); // top row + const uint16x4_t p0 = vpaddl_u8(A); // cascading summation of the top + const uint16x4_t p1 = vpadd_u16(p0, p0); + sum_top = vcombine_u16(p1, p1); + } + + if (do_left) { + const uint8x8_t L = vld1_u8(left); // left border + const uint16x4_t p0 = vpaddl_u8(L); // cascading summation of the left + const uint16x4_t p1 = vpadd_u16(p0, p0); + sum_left = vcombine_u16(p1, p1); + } + + if (do_above && do_left) { + const uint16x8_t sum = vaddq_u16(sum_left, sum_top); + dc0 = vrshrn_n_u16(sum, 3); + } else if (do_above) { + dc0 = vrshrn_n_u16(sum_top, 2); + } else if (do_left) { + dc0 = vrshrn_n_u16(sum_left, 2); + } else { + dc0 = vdup_n_u8(0x80); + } + + { + const uint8x8_t dc = vdup_lane_u8(dc0, 0); + int i; + for (i = 0; i < 4; ++i) { + vst1_lane_u32((uint32_t *)(dst + i * stride), vreinterpret_u32_u8(dc), 0); + } + } +} + +void aom_dc_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + dc_4x4(dst, stride, above, left, 1, 1); +} + +void aom_dc_left_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + dc_4x4(dst, stride, NULL, left, 0, 1); +} + +void aom_dc_top_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + (void)left; + dc_4x4(dst, stride, above, NULL, 1, 0); +} + +void aom_dc_128_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + (void)left; + dc_4x4(dst, stride, NULL, NULL, 0, 0); +} + +//------------------------------------------------------------------------------ +// DC 8x8 + +// 'do_above' and 'do_left' facilitate branch removal when inlined. +static INLINE void dc_8x8(uint8_t *dst, ptrdiff_t stride, const uint8_t *above, + const uint8_t *left, int do_above, int do_left) { + uint16x8_t sum_top; + uint16x8_t sum_left; + uint8x8_t dc0; + + if (do_above) { + const uint8x8_t A = vld1_u8(above); // top row + const uint16x4_t p0 = vpaddl_u8(A); // cascading summation of the top + const uint16x4_t p1 = vpadd_u16(p0, p0); + const uint16x4_t p2 = vpadd_u16(p1, p1); + sum_top = vcombine_u16(p2, p2); + } + + if (do_left) { + const uint8x8_t L = vld1_u8(left); // left border + const uint16x4_t p0 = vpaddl_u8(L); // cascading summation of the left + const uint16x4_t p1 = vpadd_u16(p0, p0); + const uint16x4_t p2 = vpadd_u16(p1, p1); + sum_left = vcombine_u16(p2, p2); + } + + if (do_above && do_left) { + const uint16x8_t sum = vaddq_u16(sum_left, sum_top); + dc0 = vrshrn_n_u16(sum, 4); + } else if (do_above) { + dc0 = vrshrn_n_u16(sum_top, 3); + } else if (do_left) { + dc0 = vrshrn_n_u16(sum_left, 3); + } else { + dc0 = vdup_n_u8(0x80); + } + + { + const uint8x8_t dc = vdup_lane_u8(dc0, 0); + int i; + for (i = 0; i < 8; ++i) { + vst1_u32((uint32_t *)(dst + i * stride), vreinterpret_u32_u8(dc)); + } + } +} + +void aom_dc_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + dc_8x8(dst, stride, above, left, 1, 1); +} + +void aom_dc_left_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + dc_8x8(dst, stride, NULL, left, 0, 1); +} + +void aom_dc_top_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + (void)left; + dc_8x8(dst, stride, above, NULL, 1, 0); +} + +void aom_dc_128_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + (void)left; + dc_8x8(dst, stride, NULL, NULL, 0, 0); +} + +//------------------------------------------------------------------------------ +// DC 16x16 + +// 'do_above' and 'do_left' facilitate branch removal when inlined. +static INLINE void dc_16x16(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left, + int do_above, int do_left) { + uint16x8_t sum_top; + uint16x8_t sum_left; + uint8x8_t dc0; + + if (do_above) { + const uint8x16_t A = vld1q_u8(above); // top row + const uint16x8_t p0 = vpaddlq_u8(A); // cascading summation of the top + const uint16x4_t p1 = vadd_u16(vget_low_u16(p0), vget_high_u16(p0)); + const uint16x4_t p2 = vpadd_u16(p1, p1); + const uint16x4_t p3 = vpadd_u16(p2, p2); + sum_top = vcombine_u16(p3, p3); + } + + if (do_left) { + const uint8x16_t L = vld1q_u8(left); // left row + const uint16x8_t p0 = vpaddlq_u8(L); // cascading summation of the left + const uint16x4_t p1 = vadd_u16(vget_low_u16(p0), vget_high_u16(p0)); + const uint16x4_t p2 = vpadd_u16(p1, p1); + const uint16x4_t p3 = vpadd_u16(p2, p2); + sum_left = vcombine_u16(p3, p3); + } + + if (do_above && do_left) { + const uint16x8_t sum = vaddq_u16(sum_left, sum_top); + dc0 = vrshrn_n_u16(sum, 5); + } else if (do_above) { + dc0 = vrshrn_n_u16(sum_top, 4); + } else if (do_left) { + dc0 = vrshrn_n_u16(sum_left, 4); + } else { + dc0 = vdup_n_u8(0x80); + } + + { + const uint8x16_t dc = vdupq_lane_u8(dc0, 0); + int i; + for (i = 0; i < 16; ++i) { + vst1q_u8(dst + i * stride, dc); + } + } +} + +void aom_dc_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + dc_16x16(dst, stride, above, left, 1, 1); +} + +void aom_dc_left_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + (void)above; + dc_16x16(dst, stride, NULL, left, 0, 1); +} + +void aom_dc_top_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + (void)left; + dc_16x16(dst, stride, above, NULL, 1, 0); +} + +void aom_dc_128_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + (void)above; + (void)left; + dc_16x16(dst, stride, NULL, NULL, 0, 0); +} + +//------------------------------------------------------------------------------ +// DC 32x32 + +// 'do_above' and 'do_left' facilitate branch removal when inlined. +static INLINE void dc_32x32(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left, + int do_above, int do_left) { + uint16x8_t sum_top; + uint16x8_t sum_left; + uint8x8_t dc0; + + if (do_above) { + const uint8x16_t A0 = vld1q_u8(above); // top row + const uint8x16_t A1 = vld1q_u8(above + 16); + const uint16x8_t p0 = vpaddlq_u8(A0); // cascading summation of the top + const uint16x8_t p1 = vpaddlq_u8(A1); + const uint16x8_t p2 = vaddq_u16(p0, p1); + const uint16x4_t p3 = vadd_u16(vget_low_u16(p2), vget_high_u16(p2)); + const uint16x4_t p4 = vpadd_u16(p3, p3); + const uint16x4_t p5 = vpadd_u16(p4, p4); + sum_top = vcombine_u16(p5, p5); + } + + if (do_left) { + const uint8x16_t L0 = vld1q_u8(left); // left row + const uint8x16_t L1 = vld1q_u8(left + 16); + const uint16x8_t p0 = vpaddlq_u8(L0); // cascading summation of the left + const uint16x8_t p1 = vpaddlq_u8(L1); + const uint16x8_t p2 = vaddq_u16(p0, p1); + const uint16x4_t p3 = vadd_u16(vget_low_u16(p2), vget_high_u16(p2)); + const uint16x4_t p4 = vpadd_u16(p3, p3); + const uint16x4_t p5 = vpadd_u16(p4, p4); + sum_left = vcombine_u16(p5, p5); + } + + if (do_above && do_left) { + const uint16x8_t sum = vaddq_u16(sum_left, sum_top); + dc0 = vrshrn_n_u16(sum, 6); + } else if (do_above) { + dc0 = vrshrn_n_u16(sum_top, 5); + } else if (do_left) { + dc0 = vrshrn_n_u16(sum_left, 5); + } else { + dc0 = vdup_n_u8(0x80); + } + + { + const uint8x16_t dc = vdupq_lane_u8(dc0, 0); + int i; + for (i = 0; i < 32; ++i) { + vst1q_u8(dst + i * stride, dc); + vst1q_u8(dst + i * stride + 16, dc); + } + } +} + +void aom_dc_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + dc_32x32(dst, stride, above, left, 1, 1); +} + +void aom_dc_left_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + (void)above; + dc_32x32(dst, stride, NULL, left, 0, 1); +} + +void aom_dc_top_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + (void)left; + dc_32x32(dst, stride, above, NULL, 1, 0); +} + +void aom_dc_128_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + (void)above; + (void)left; + dc_32x32(dst, stride, NULL, NULL, 0, 0); +} + +// ----------------------------------------------------------------------------- + +void aom_d135_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + const uint8x8_t XABCD_u8 = vld1_u8(above - 1); + const uint64x1_t XABCD = vreinterpret_u64_u8(XABCD_u8); + const uint64x1_t ____XABC = vshl_n_u64(XABCD, 32); + const uint32x2_t zero = vdup_n_u32(0); + const uint32x2_t IJKL = vld1_lane_u32((const uint32_t *)left, zero, 0); + const uint8x8_t IJKL_u8 = vreinterpret_u8_u32(IJKL); + const uint64x1_t LKJI____ = vreinterpret_u64_u8(vrev32_u8(IJKL_u8)); + const uint64x1_t LKJIXABC = vorr_u64(LKJI____, ____XABC); + const uint8x8_t KJIXABC_ = vreinterpret_u8_u64(vshr_n_u64(LKJIXABC, 8)); + const uint8x8_t JIXABC__ = vreinterpret_u8_u64(vshr_n_u64(LKJIXABC, 16)); + const uint8_t D = vget_lane_u8(XABCD_u8, 4); + const uint8x8_t JIXABCD_ = vset_lane_u8(D, JIXABC__, 6); + const uint8x8_t LKJIXABC_u8 = vreinterpret_u8_u64(LKJIXABC); + const uint8x8_t avg1 = vhadd_u8(JIXABCD_, LKJIXABC_u8); + const uint8x8_t avg2 = vrhadd_u8(avg1, KJIXABC_); + const uint64x1_t avg2_u64 = vreinterpret_u64_u8(avg2); + const uint32x2_t r3 = vreinterpret_u32_u8(avg2); + const uint32x2_t r2 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 8)); + const uint32x2_t r1 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 16)); + const uint32x2_t r0 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 24)); + vst1_lane_u32((uint32_t *)(dst + 0 * stride), r0, 0); + vst1_lane_u32((uint32_t *)(dst + 1 * stride), r1, 0); + vst1_lane_u32((uint32_t *)(dst + 2 * stride), r2, 0); + vst1_lane_u32((uint32_t *)(dst + 3 * stride), r3, 0); +} + +#if !HAVE_NEON_ASM + +void aom_v_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int i; + uint32x2_t d0u32 = vdup_n_u32(0); + (void)left; + + d0u32 = vld1_lane_u32((const uint32_t *)above, d0u32, 0); + for (i = 0; i < 4; i++, dst += stride) + vst1_lane_u32((uint32_t *)dst, d0u32, 0); +} + +void aom_v_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int i; + uint8x8_t d0u8 = vdup_n_u8(0); + (void)left; + + d0u8 = vld1_u8(above); + for (i = 0; i < 8; i++, dst += stride) vst1_u8(dst, d0u8); +} + +void aom_v_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int i; + uint8x16_t q0u8 = vdupq_n_u8(0); + (void)left; + + q0u8 = vld1q_u8(above); + for (i = 0; i < 16; i++, dst += stride) vst1q_u8(dst, q0u8); +} + +void aom_v_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int i; + uint8x16_t q0u8 = vdupq_n_u8(0); + uint8x16_t q1u8 = vdupq_n_u8(0); + (void)left; + + q0u8 = vld1q_u8(above); + q1u8 = vld1q_u8(above + 16); + for (i = 0; i < 32; i++, dst += stride) { + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q1u8); + } +} + +void aom_h_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + uint8x8_t d0u8 = vdup_n_u8(0); + uint32x2_t d1u32 = vdup_n_u32(0); + (void)above; + + d1u32 = vld1_lane_u32((const uint32_t *)left, d1u32, 0); + + d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 0); + vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0); + dst += stride; + d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 1); + vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0); + dst += stride; + d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 2); + vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0); + dst += stride; + d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 3); + vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0); +} + +void aom_h_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + uint8x8_t d0u8 = vdup_n_u8(0); + uint64x1_t d1u64 = vdup_n_u64(0); + (void)above; + + d1u64 = vld1_u64((const uint64_t *)left); + + d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 0); + vst1_u8(dst, d0u8); + dst += stride; + d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 1); + vst1_u8(dst, d0u8); + dst += stride; + d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 2); + vst1_u8(dst, d0u8); + dst += stride; + d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 3); + vst1_u8(dst, d0u8); + dst += stride; + d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 4); + vst1_u8(dst, d0u8); + dst += stride; + d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 5); + vst1_u8(dst, d0u8); + dst += stride; + d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 6); + vst1_u8(dst, d0u8); + dst += stride; + d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 7); + vst1_u8(dst, d0u8); +} + +void aom_h_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int j; + uint8x8_t d2u8 = vdup_n_u8(0); + uint8x16_t q0u8 = vdupq_n_u8(0); + uint8x16_t q1u8 = vdupq_n_u8(0); + (void)above; + + q1u8 = vld1q_u8(left); + d2u8 = vget_low_u8(q1u8); + for (j = 0; j < 2; j++, d2u8 = vget_high_u8(q1u8)) { + q0u8 = vdupq_lane_u8(d2u8, 0); + vst1q_u8(dst, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 1); + vst1q_u8(dst, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 2); + vst1q_u8(dst, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 3); + vst1q_u8(dst, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 4); + vst1q_u8(dst, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 5); + vst1q_u8(dst, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 6); + vst1q_u8(dst, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 7); + vst1q_u8(dst, q0u8); + dst += stride; + } +} + +void aom_h_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int j, k; + uint8x8_t d2u8 = vdup_n_u8(0); + uint8x16_t q0u8 = vdupq_n_u8(0); + uint8x16_t q1u8 = vdupq_n_u8(0); + (void)above; + + for (k = 0; k < 2; k++, left += 16) { + q1u8 = vld1q_u8(left); + d2u8 = vget_low_u8(q1u8); + for (j = 0; j < 2; j++, d2u8 = vget_high_u8(q1u8)) { + q0u8 = vdupq_lane_u8(d2u8, 0); + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 1); + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 2); + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 3); + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 4); + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 5); + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 6); + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q0u8); + dst += stride; + q0u8 = vdupq_lane_u8(d2u8, 7); + vst1q_u8(dst, q0u8); + vst1q_u8(dst + 16, q0u8); + dst += stride; + } + } +} + +void aom_tm_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int i; + uint16x8_t q1u16, q3u16; + int16x8_t q1s16; + uint8x8_t d0u8 = vdup_n_u8(0); + uint32x2_t d2u32 = vdup_n_u32(0); + + d0u8 = vld1_dup_u8(above - 1); + d2u32 = vld1_lane_u32((const uint32_t *)above, d2u32, 0); + q3u16 = vsubl_u8(vreinterpret_u8_u32(d2u32), d0u8); + for (i = 0; i < 4; i++, dst += stride) { + q1u16 = vdupq_n_u16((uint16_t)left[i]); + q1s16 = + vaddq_s16(vreinterpretq_s16_u16(q1u16), vreinterpretq_s16_u16(q3u16)); + d0u8 = vqmovun_s16(q1s16); + vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0); + } +} + +void aom_tm_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int j; + uint16x8_t q0u16, q3u16, q10u16; + int16x8_t q0s16; + uint16x4_t d20u16; + uint8x8_t d0u8, d2u8, d30u8; + + d0u8 = vld1_dup_u8(above - 1); + d30u8 = vld1_u8(left); + d2u8 = vld1_u8(above); + q10u16 = vmovl_u8(d30u8); + q3u16 = vsubl_u8(d2u8, d0u8); + d20u16 = vget_low_u16(q10u16); + for (j = 0; j < 2; j++, d20u16 = vget_high_u16(q10u16)) { + q0u16 = vdupq_lane_u16(d20u16, 0); + q0s16 = + vaddq_s16(vreinterpretq_s16_u16(q3u16), vreinterpretq_s16_u16(q0u16)); + d0u8 = vqmovun_s16(q0s16); + vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d0u8)); + dst += stride; + q0u16 = vdupq_lane_u16(d20u16, 1); + q0s16 = + vaddq_s16(vreinterpretq_s16_u16(q3u16), vreinterpretq_s16_u16(q0u16)); + d0u8 = vqmovun_s16(q0s16); + vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d0u8)); + dst += stride; + q0u16 = vdupq_lane_u16(d20u16, 2); + q0s16 = + vaddq_s16(vreinterpretq_s16_u16(q3u16), vreinterpretq_s16_u16(q0u16)); + d0u8 = vqmovun_s16(q0s16); + vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d0u8)); + dst += stride; + q0u16 = vdupq_lane_u16(d20u16, 3); + q0s16 = + vaddq_s16(vreinterpretq_s16_u16(q3u16), vreinterpretq_s16_u16(q0u16)); + d0u8 = vqmovun_s16(q0s16); + vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d0u8)); + dst += stride; + } +} + +void aom_tm_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int j, k; + uint16x8_t q0u16, q2u16, q3u16, q8u16, q10u16; + uint8x16_t q0u8, q1u8; + int16x8_t q0s16, q1s16, q8s16, q11s16; + uint16x4_t d20u16; + uint8x8_t d2u8, d3u8, d18u8, d22u8, d23u8; + + q0u8 = vld1q_dup_u8(above - 1); + q1u8 = vld1q_u8(above); + q2u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q0u8)); + q3u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q0u8)); + for (k = 0; k < 2; k++, left += 8) { + d18u8 = vld1_u8(left); + q10u16 = vmovl_u8(d18u8); + d20u16 = vget_low_u16(q10u16); + for (j = 0; j < 2; j++, d20u16 = vget_high_u16(q10u16)) { + q0u16 = vdupq_lane_u16(d20u16, 0); + q8u16 = vdupq_lane_u16(d20u16, 1); + q1s16 = + vaddq_s16(vreinterpretq_s16_u16(q0u16), vreinterpretq_s16_u16(q2u16)); + q0s16 = + vaddq_s16(vreinterpretq_s16_u16(q0u16), vreinterpretq_s16_u16(q3u16)); + q11s16 = + vaddq_s16(vreinterpretq_s16_u16(q8u16), vreinterpretq_s16_u16(q2u16)); + q8s16 = + vaddq_s16(vreinterpretq_s16_u16(q8u16), vreinterpretq_s16_u16(q3u16)); + d2u8 = vqmovun_s16(q1s16); + d3u8 = vqmovun_s16(q0s16); + d22u8 = vqmovun_s16(q11s16); + d23u8 = vqmovun_s16(q8s16); + vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d2u8)); + vst1_u64((uint64_t *)(dst + 8), vreinterpret_u64_u8(d3u8)); + dst += stride; + vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d22u8)); + vst1_u64((uint64_t *)(dst + 8), vreinterpret_u64_u8(d23u8)); + dst += stride; + + q0u16 = vdupq_lane_u16(d20u16, 2); + q8u16 = vdupq_lane_u16(d20u16, 3); + q1s16 = + vaddq_s16(vreinterpretq_s16_u16(q0u16), vreinterpretq_s16_u16(q2u16)); + q0s16 = + vaddq_s16(vreinterpretq_s16_u16(q0u16), vreinterpretq_s16_u16(q3u16)); + q11s16 = + vaddq_s16(vreinterpretq_s16_u16(q8u16), vreinterpretq_s16_u16(q2u16)); + q8s16 = + vaddq_s16(vreinterpretq_s16_u16(q8u16), vreinterpretq_s16_u16(q3u16)); + d2u8 = vqmovun_s16(q1s16); + d3u8 = vqmovun_s16(q0s16); + d22u8 = vqmovun_s16(q11s16); + d23u8 = vqmovun_s16(q8s16); + vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d2u8)); + vst1_u64((uint64_t *)(dst + 8), vreinterpret_u64_u8(d3u8)); + dst += stride; + vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d22u8)); + vst1_u64((uint64_t *)(dst + 8), vreinterpret_u64_u8(d23u8)); + dst += stride; + } + } +} + +void aom_tm_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int j, k; + uint16x8_t q0u16, q3u16, q8u16, q9u16, q10u16, q11u16; + uint8x16_t q0u8, q1u8, q2u8; + int16x8_t q12s16, q13s16, q14s16, q15s16; + uint16x4_t d6u16; + uint8x8_t d0u8, d1u8, d2u8, d3u8, d26u8; + + q0u8 = vld1q_dup_u8(above - 1); + q1u8 = vld1q_u8(above); + q2u8 = vld1q_u8(above + 16); + q8u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q0u8)); + q9u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q0u8)); + q10u16 = vsubl_u8(vget_low_u8(q2u8), vget_low_u8(q0u8)); + q11u16 = vsubl_u8(vget_high_u8(q2u8), vget_high_u8(q0u8)); + for (k = 0; k < 4; k++, left += 8) { + d26u8 = vld1_u8(left); + q3u16 = vmovl_u8(d26u8); + d6u16 = vget_low_u16(q3u16); + for (j = 0; j < 2; j++, d6u16 = vget_high_u16(q3u16)) { + q0u16 = vdupq_lane_u16(d6u16, 0); + q12s16 = + vaddq_s16(vreinterpretq_s16_u16(q0u16), vreinterpretq_s16_u16(q8u16)); + q13s16 = + vaddq_s16(vreinterpretq_s16_u16(q0u16), vreinterpretq_s16_u16(q9u16)); + q14s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16), + vreinterpretq_s16_u16(q10u16)); + q15s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16), + vreinterpretq_s16_u16(q11u16)); + d0u8 = vqmovun_s16(q12s16); + d1u8 = vqmovun_s16(q13s16); + d2u8 = vqmovun_s16(q14s16); + d3u8 = vqmovun_s16(q15s16); + q0u8 = vcombine_u8(d0u8, d1u8); + q1u8 = vcombine_u8(d2u8, d3u8); + vst1q_u64((uint64_t *)dst, vreinterpretq_u64_u8(q0u8)); + vst1q_u64((uint64_t *)(dst + 16), vreinterpretq_u64_u8(q1u8)); + dst += stride; + + q0u16 = vdupq_lane_u16(d6u16, 1); + q12s16 = + vaddq_s16(vreinterpretq_s16_u16(q0u16), vreinterpretq_s16_u16(q8u16)); + q13s16 = + vaddq_s16(vreinterpretq_s16_u16(q0u16), vreinterpretq_s16_u16(q9u16)); + q14s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16), + vreinterpretq_s16_u16(q10u16)); + q15s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16), + vreinterpretq_s16_u16(q11u16)); + d0u8 = vqmovun_s16(q12s16); + d1u8 = vqmovun_s16(q13s16); + d2u8 = vqmovun_s16(q14s16); + d3u8 = vqmovun_s16(q15s16); + q0u8 = vcombine_u8(d0u8, d1u8); + q1u8 = vcombine_u8(d2u8, d3u8); + vst1q_u64((uint64_t *)dst, vreinterpretq_u64_u8(q0u8)); + vst1q_u64((uint64_t *)(dst + 16), vreinterpretq_u64_u8(q1u8)); + dst += stride; + + q0u16 = vdupq_lane_u16(d6u16, 2); + q12s16 = + vaddq_s16(vreinterpretq_s16_u16(q0u16), vreinterpretq_s16_u16(q8u16)); + q13s16 = + vaddq_s16(vreinterpretq_s16_u16(q0u16), vreinterpretq_s16_u16(q9u16)); + q14s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16), + vreinterpretq_s16_u16(q10u16)); + q15s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16), + vreinterpretq_s16_u16(q11u16)); + d0u8 = vqmovun_s16(q12s16); + d1u8 = vqmovun_s16(q13s16); + d2u8 = vqmovun_s16(q14s16); + d3u8 = vqmovun_s16(q15s16); + q0u8 = vcombine_u8(d0u8, d1u8); + q1u8 = vcombine_u8(d2u8, d3u8); + vst1q_u64((uint64_t *)dst, vreinterpretq_u64_u8(q0u8)); + vst1q_u64((uint64_t *)(dst + 16), vreinterpretq_u64_u8(q1u8)); + dst += stride; + + q0u16 = vdupq_lane_u16(d6u16, 3); + q12s16 = + vaddq_s16(vreinterpretq_s16_u16(q0u16), vreinterpretq_s16_u16(q8u16)); + q13s16 = + vaddq_s16(vreinterpretq_s16_u16(q0u16), vreinterpretq_s16_u16(q9u16)); + q14s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16), + vreinterpretq_s16_u16(q10u16)); + q15s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16), + vreinterpretq_s16_u16(q11u16)); + d0u8 = vqmovun_s16(q12s16); + d1u8 = vqmovun_s16(q13s16); + d2u8 = vqmovun_s16(q14s16); + d3u8 = vqmovun_s16(q15s16); + q0u8 = vcombine_u8(d0u8, d1u8); + q1u8 = vcombine_u8(d2u8, d3u8); + vst1q_u64((uint64_t *)dst, vreinterpretq_u64_u8(q0u8)); + vst1q_u64((uint64_t *)(dst + 16), vreinterpretq_u64_u8(q1u8)); + dst += stride; + } + } +} +#endif // !HAVE_NEON_ASM diff --git a/third_party/aom/aom_dsp/arm/intrapred_neon_asm.asm b/third_party/aom/aom_dsp/arm/intrapred_neon_asm.asm new file mode 100644 index 0000000000..7d04d35539 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/intrapred_neon_asm.asm @@ -0,0 +1,633 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + EXPORT |aom_v_predictor_4x4_neon| + EXPORT |aom_v_predictor_8x8_neon| + EXPORT |aom_v_predictor_16x16_neon| + EXPORT |aom_v_predictor_32x32_neon| + EXPORT |aom_h_predictor_4x4_neon| + EXPORT |aom_h_predictor_8x8_neon| + EXPORT |aom_h_predictor_16x16_neon| + EXPORT |aom_h_predictor_32x32_neon| + EXPORT |aom_tm_predictor_4x4_neon| + EXPORT |aom_tm_predictor_8x8_neon| + EXPORT |aom_tm_predictor_16x16_neon| + EXPORT |aom_tm_predictor_32x32_neon| + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +;void aom_v_predictor_4x4_neon(uint8_t *dst, ptrdiff_t y_stride, +; const uint8_t *above, +; const uint8_t *left) +; r0 uint8_t *dst +; r1 ptrdiff_t y_stride +; r2 const uint8_t *above +; r3 const uint8_t *left + +|aom_v_predictor_4x4_neon| PROC + vld1.32 {d0[0]}, [r2] + vst1.32 {d0[0]}, [r0], r1 + vst1.32 {d0[0]}, [r0], r1 + vst1.32 {d0[0]}, [r0], r1 + vst1.32 {d0[0]}, [r0], r1 + bx lr + ENDP ; |aom_v_predictor_4x4_neon| + +;void aom_v_predictor_8x8_neon(uint8_t *dst, ptrdiff_t y_stride, +; const uint8_t *above, +; const uint8_t *left) +; r0 uint8_t *dst +; r1 ptrdiff_t y_stride +; r2 const uint8_t *above +; r3 const uint8_t *left + +|aom_v_predictor_8x8_neon| PROC + vld1.8 {d0}, [r2] + vst1.8 {d0}, [r0], r1 + vst1.8 {d0}, [r0], r1 + vst1.8 {d0}, [r0], r1 + vst1.8 {d0}, [r0], r1 + vst1.8 {d0}, [r0], r1 + vst1.8 {d0}, [r0], r1 + vst1.8 {d0}, [r0], r1 + vst1.8 {d0}, [r0], r1 + bx lr + ENDP ; |aom_v_predictor_8x8_neon| + +;void aom_v_predictor_16x16_neon(uint8_t *dst, ptrdiff_t y_stride, +; const uint8_t *above, +; const uint8_t *left) +; r0 uint8_t *dst +; r1 ptrdiff_t y_stride +; r2 const uint8_t *above +; r3 const uint8_t *left + +|aom_v_predictor_16x16_neon| PROC + vld1.8 {q0}, [r2] + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + vst1.8 {q0}, [r0], r1 + bx lr + ENDP ; |aom_v_predictor_16x16_neon| + +;void aom_v_predictor_32x32_neon(uint8_t *dst, ptrdiff_t y_stride, +; const uint8_t *above, +; const uint8_t *left) +; r0 uint8_t *dst +; r1 ptrdiff_t y_stride +; r2 const uint8_t *above +; r3 const uint8_t *left + +|aom_v_predictor_32x32_neon| PROC + vld1.8 {q0, q1}, [r2] + mov r2, #2 +loop_v + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + vst1.8 {q0, q1}, [r0], r1 + subs r2, r2, #1 + bgt loop_v + bx lr + ENDP ; |aom_v_predictor_32x32_neon| + +;void aom_h_predictor_4x4_neon(uint8_t *dst, ptrdiff_t y_stride, +; const uint8_t *above, +; const uint8_t *left) +; r0 uint8_t *dst +; r1 ptrdiff_t y_stride +; r2 const uint8_t *above +; r3 const uint8_t *left + +|aom_h_predictor_4x4_neon| PROC + vld1.32 {d1[0]}, [r3] + vdup.8 d0, d1[0] + vst1.32 {d0[0]}, [r0], r1 + vdup.8 d0, d1[1] + vst1.32 {d0[0]}, [r0], r1 + vdup.8 d0, d1[2] + vst1.32 {d0[0]}, [r0], r1 + vdup.8 d0, d1[3] + vst1.32 {d0[0]}, [r0], r1 + bx lr + ENDP ; |aom_h_predictor_4x4_neon| + +;void aom_h_predictor_8x8_neon(uint8_t *dst, ptrdiff_t y_stride, +; const uint8_t *above, +; const uint8_t *left) +; r0 uint8_t *dst +; r1 ptrdiff_t y_stride +; r2 const uint8_t *above +; r3 const uint8_t *left + +|aom_h_predictor_8x8_neon| PROC + vld1.64 {d1}, [r3] + vdup.8 d0, d1[0] + vst1.64 {d0}, [r0], r1 + vdup.8 d0, d1[1] + vst1.64 {d0}, [r0], r1 + vdup.8 d0, d1[2] + vst1.64 {d0}, [r0], r1 + vdup.8 d0, d1[3] + vst1.64 {d0}, [r0], r1 + vdup.8 d0, d1[4] + vst1.64 {d0}, [r0], r1 + vdup.8 d0, d1[5] + vst1.64 {d0}, [r0], r1 + vdup.8 d0, d1[6] + vst1.64 {d0}, [r0], r1 + vdup.8 d0, d1[7] + vst1.64 {d0}, [r0], r1 + bx lr + ENDP ; |aom_h_predictor_8x8_neon| + +;void aom_h_predictor_16x16_neon(uint8_t *dst, ptrdiff_t y_stride, +; const uint8_t *above, +; const uint8_t *left) +; r0 uint8_t *dst +; r1 ptrdiff_t y_stride +; r2 const uint8_t *above +; r3 const uint8_t *left + +|aom_h_predictor_16x16_neon| PROC + vld1.8 {q1}, [r3] + vdup.8 q0, d2[0] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[1] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[2] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[3] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[4] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[5] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[6] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[7] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[0] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[1] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[2] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[3] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[4] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[5] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[6] + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[7] + vst1.8 {q0}, [r0], r1 + bx lr + ENDP ; |aom_h_predictor_16x16_neon| + +;void aom_h_predictor_32x32_neon(uint8_t *dst, ptrdiff_t y_stride, +; const uint8_t *above, +; const uint8_t *left) +; r0 uint8_t *dst +; r1 ptrdiff_t y_stride +; r2 const uint8_t *above +; r3 const uint8_t *left + +|aom_h_predictor_32x32_neon| PROC + sub r1, r1, #16 + mov r2, #2 +loop_h + vld1.8 {q1}, [r3]! + vdup.8 q0, d2[0] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[1] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[2] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[3] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[4] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[5] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[6] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d2[7] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[0] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[1] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[2] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[3] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[4] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[5] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[6] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + vdup.8 q0, d3[7] + vst1.8 {q0}, [r0]! + vst1.8 {q0}, [r0], r1 + subs r2, r2, #1 + bgt loop_h + bx lr + ENDP ; |aom_h_predictor_32x32_neon| + +;void aom_tm_predictor_4x4_neon (uint8_t *dst, ptrdiff_t y_stride, +; const uint8_t *above, +; const uint8_t *left) +; r0 uint8_t *dst +; r1 ptrdiff_t y_stride +; r2 const uint8_t *above +; r3 const uint8_t *left + +|aom_tm_predictor_4x4_neon| PROC + ; Load ytop_left = above[-1]; + sub r12, r2, #1 + vld1.u8 {d0[]}, [r12] + + ; Load above 4 pixels + vld1.32 {d2[0]}, [r2] + + ; Compute above - ytop_left + vsubl.u8 q3, d2, d0 + + ; Load left row by row and compute left + (above - ytop_left) + ; 1st row and 2nd row + vld1.u8 {d2[]}, [r3]! + vld1.u8 {d4[]}, [r3]! + vmovl.u8 q1, d2 + vmovl.u8 q2, d4 + vadd.s16 q1, q1, q3 + vadd.s16 q2, q2, q3 + vqmovun.s16 d0, q1 + vqmovun.s16 d1, q2 + vst1.32 {d0[0]}, [r0], r1 + vst1.32 {d1[0]}, [r0], r1 + + ; 3rd row and 4th row + vld1.u8 {d2[]}, [r3]! + vld1.u8 {d4[]}, [r3] + vmovl.u8 q1, d2 + vmovl.u8 q2, d4 + vadd.s16 q1, q1, q3 + vadd.s16 q2, q2, q3 + vqmovun.s16 d0, q1 + vqmovun.s16 d1, q2 + vst1.32 {d0[0]}, [r0], r1 + vst1.32 {d1[0]}, [r0], r1 + bx lr + ENDP ; |aom_tm_predictor_4x4_neon| + +;void aom_tm_predictor_8x8_neon (uint8_t *dst, ptrdiff_t y_stride, +; const uint8_t *above, +; const uint8_t *left) +; r0 uint8_t *dst +; r1 ptrdiff_t y_stride +; r2 const uint8_t *above +; r3 const uint8_t *left + +|aom_tm_predictor_8x8_neon| PROC + ; Load ytop_left = above[-1]; + sub r12, r2, #1 + vld1.8 {d0[]}, [r12] + + ; preload 8 left + vld1.8 {d30}, [r3] + + ; Load above 8 pixels + vld1.64 {d2}, [r2] + + vmovl.u8 q10, d30 + + ; Compute above - ytop_left + vsubl.u8 q3, d2, d0 + + ; Load left row by row and compute left + (above - ytop_left) + ; 1st row and 2nd row + vdup.16 q0, d20[0] + vdup.16 q1, d20[1] + vadd.s16 q0, q3, q0 + vadd.s16 q1, q3, q1 + + ; 3rd row and 4th row + vdup.16 q8, d20[2] + vdup.16 q9, d20[3] + vadd.s16 q8, q3, q8 + vadd.s16 q9, q3, q9 + + vqmovun.s16 d0, q0 + vqmovun.s16 d1, q1 + vqmovun.s16 d2, q8 + vqmovun.s16 d3, q9 + + vst1.64 {d0}, [r0], r1 + vst1.64 {d1}, [r0], r1 + vst1.64 {d2}, [r0], r1 + vst1.64 {d3}, [r0], r1 + + ; 5th row and 6th row + vdup.16 q0, d21[0] + vdup.16 q1, d21[1] + vadd.s16 q0, q3, q0 + vadd.s16 q1, q3, q1 + + ; 7th row and 8th row + vdup.16 q8, d21[2] + vdup.16 q9, d21[3] + vadd.s16 q8, q3, q8 + vadd.s16 q9, q3, q9 + + vqmovun.s16 d0, q0 + vqmovun.s16 d1, q1 + vqmovun.s16 d2, q8 + vqmovun.s16 d3, q9 + + vst1.64 {d0}, [r0], r1 + vst1.64 {d1}, [r0], r1 + vst1.64 {d2}, [r0], r1 + vst1.64 {d3}, [r0], r1 + + bx lr + ENDP ; |aom_tm_predictor_8x8_neon| + +;void aom_tm_predictor_16x16_neon (uint8_t *dst, ptrdiff_t y_stride, +; const uint8_t *above, +; const uint8_t *left) +; r0 uint8_t *dst +; r1 ptrdiff_t y_stride +; r2 const uint8_t *above +; r3 const uint8_t *left + +|aom_tm_predictor_16x16_neon| PROC + ; Load ytop_left = above[-1]; + sub r12, r2, #1 + vld1.8 {d0[]}, [r12] + + ; Load above 8 pixels + vld1.8 {q1}, [r2] + + ; preload 8 left into r12 + vld1.8 {d18}, [r3]! + + ; Compute above - ytop_left + vsubl.u8 q2, d2, d0 + vsubl.u8 q3, d3, d0 + + vmovl.u8 q10, d18 + + ; Load left row by row and compute left + (above - ytop_left) + ; Process 8 rows in each single loop and loop 2 times to process 16 rows. + mov r2, #2 + +loop_16x16_neon + ; Process two rows. + vdup.16 q0, d20[0] + vdup.16 q8, d20[1] + vadd.s16 q1, q0, q2 + vadd.s16 q0, q0, q3 + vadd.s16 q11, q8, q2 + vadd.s16 q8, q8, q3 + vqmovun.s16 d2, q1 + vqmovun.s16 d3, q0 + vqmovun.s16 d22, q11 + vqmovun.s16 d23, q8 + vdup.16 q0, d20[2] ; proload next 2 rows data + vdup.16 q8, d20[3] + vst1.64 {d2,d3}, [r0], r1 + vst1.64 {d22,d23}, [r0], r1 + + ; Process two rows. + vadd.s16 q1, q0, q2 + vadd.s16 q0, q0, q3 + vadd.s16 q11, q8, q2 + vadd.s16 q8, q8, q3 + vqmovun.s16 d2, q1 + vqmovun.s16 d3, q0 + vqmovun.s16 d22, q11 + vqmovun.s16 d23, q8 + vdup.16 q0, d21[0] ; proload next 2 rows data + vdup.16 q8, d21[1] + vst1.64 {d2,d3}, [r0], r1 + vst1.64 {d22,d23}, [r0], r1 + + vadd.s16 q1, q0, q2 + vadd.s16 q0, q0, q3 + vadd.s16 q11, q8, q2 + vadd.s16 q8, q8, q3 + vqmovun.s16 d2, q1 + vqmovun.s16 d3, q0 + vqmovun.s16 d22, q11 + vqmovun.s16 d23, q8 + vdup.16 q0, d21[2] ; proload next 2 rows data + vdup.16 q8, d21[3] + vst1.64 {d2,d3}, [r0], r1 + vst1.64 {d22,d23}, [r0], r1 + + + vadd.s16 q1, q0, q2 + vadd.s16 q0, q0, q3 + vadd.s16 q11, q8, q2 + vadd.s16 q8, q8, q3 + vqmovun.s16 d2, q1 + vqmovun.s16 d3, q0 + vqmovun.s16 d22, q11 + vqmovun.s16 d23, q8 + vld1.8 {d18}, [r3]! ; preload 8 left into r12 + vmovl.u8 q10, d18 + vst1.64 {d2,d3}, [r0], r1 + vst1.64 {d22,d23}, [r0], r1 + + subs r2, r2, #1 + bgt loop_16x16_neon + + bx lr + ENDP ; |aom_tm_predictor_16x16_neon| + +;void aom_tm_predictor_32x32_neon (uint8_t *dst, ptrdiff_t y_stride, +; const uint8_t *above, +; const uint8_t *left) +; r0 uint8_t *dst +; r1 ptrdiff_t y_stride +; r2 const uint8_t *above +; r3 const uint8_t *left + +|aom_tm_predictor_32x32_neon| PROC + ; Load ytop_left = above[-1]; + sub r12, r2, #1 + vld1.8 {d0[]}, [r12] + + ; Load above 32 pixels + vld1.8 {q1}, [r2]! + vld1.8 {q2}, [r2] + + ; preload 8 left pixels + vld1.8 {d26}, [r3]! + + ; Compute above - ytop_left + vsubl.u8 q8, d2, d0 + vsubl.u8 q9, d3, d0 + vsubl.u8 q10, d4, d0 + vsubl.u8 q11, d5, d0 + + vmovl.u8 q3, d26 + + ; Load left row by row and compute left + (above - ytop_left) + ; Process 8 rows in each single loop and loop 4 times to process 32 rows. + mov r2, #4 + +loop_32x32_neon + ; Process two rows. + vdup.16 q0, d6[0] + vdup.16 q2, d6[1] + vadd.s16 q12, q0, q8 + vadd.s16 q13, q0, q9 + vadd.s16 q14, q0, q10 + vadd.s16 q15, q0, q11 + vqmovun.s16 d0, q12 + vqmovun.s16 d1, q13 + vadd.s16 q12, q2, q8 + vadd.s16 q13, q2, q9 + vqmovun.s16 d2, q14 + vqmovun.s16 d3, q15 + vadd.s16 q14, q2, q10 + vadd.s16 q15, q2, q11 + vst1.64 {d0-d3}, [r0], r1 + vqmovun.s16 d24, q12 + vqmovun.s16 d25, q13 + vqmovun.s16 d26, q14 + vqmovun.s16 d27, q15 + vdup.16 q1, d6[2] + vdup.16 q2, d6[3] + vst1.64 {d24-d27}, [r0], r1 + + ; Process two rows. + vadd.s16 q12, q1, q8 + vadd.s16 q13, q1, q9 + vadd.s16 q14, q1, q10 + vadd.s16 q15, q1, q11 + vqmovun.s16 d0, q12 + vqmovun.s16 d1, q13 + vadd.s16 q12, q2, q8 + vadd.s16 q13, q2, q9 + vqmovun.s16 d2, q14 + vqmovun.s16 d3, q15 + vadd.s16 q14, q2, q10 + vadd.s16 q15, q2, q11 + vst1.64 {d0-d3}, [r0], r1 + vqmovun.s16 d24, q12 + vqmovun.s16 d25, q13 + vqmovun.s16 d26, q14 + vqmovun.s16 d27, q15 + vdup.16 q0, d7[0] + vdup.16 q2, d7[1] + vst1.64 {d24-d27}, [r0], r1 + + ; Process two rows. + vadd.s16 q12, q0, q8 + vadd.s16 q13, q0, q9 + vadd.s16 q14, q0, q10 + vadd.s16 q15, q0, q11 + vqmovun.s16 d0, q12 + vqmovun.s16 d1, q13 + vadd.s16 q12, q2, q8 + vadd.s16 q13, q2, q9 + vqmovun.s16 d2, q14 + vqmovun.s16 d3, q15 + vadd.s16 q14, q2, q10 + vadd.s16 q15, q2, q11 + vst1.64 {d0-d3}, [r0], r1 + vqmovun.s16 d24, q12 + vqmovun.s16 d25, q13 + vqmovun.s16 d26, q14 + vqmovun.s16 d27, q15 + vdup.16 q0, d7[2] + vdup.16 q2, d7[3] + vst1.64 {d24-d27}, [r0], r1 + + ; Process two rows. + vadd.s16 q12, q0, q8 + vadd.s16 q13, q0, q9 + vadd.s16 q14, q0, q10 + vadd.s16 q15, q0, q11 + vqmovun.s16 d0, q12 + vqmovun.s16 d1, q13 + vadd.s16 q12, q2, q8 + vadd.s16 q13, q2, q9 + vqmovun.s16 d2, q14 + vqmovun.s16 d3, q15 + vadd.s16 q14, q2, q10 + vadd.s16 q15, q2, q11 + vst1.64 {d0-d3}, [r0], r1 + vqmovun.s16 d24, q12 + vqmovun.s16 d25, q13 + vld1.8 {d0}, [r3]! ; preload 8 left pixels + vqmovun.s16 d26, q14 + vqmovun.s16 d27, q15 + vmovl.u8 q3, d0 + vst1.64 {d24-d27}, [r0], r1 + + subs r2, r2, #1 + bgt loop_32x32_neon + + bx lr + ENDP ; |aom_tm_predictor_32x32_neon| + + END diff --git a/third_party/aom/aom_dsp/arm/loopfilter_16_neon.asm b/third_party/aom/aom_dsp/arm/loopfilter_16_neon.asm new file mode 100644 index 0000000000..b6e2c9edb4 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/loopfilter_16_neon.asm @@ -0,0 +1,202 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + EXPORT |aom_lpf_horizontal_4_dual_neon| + ARM + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +;void aom_lpf_horizontal_4_dual_neon(uint8_t *s, int p, +; const uint8_t *blimit0, +; const uint8_t *limit0, +; const uint8_t *thresh0, +; const uint8_t *blimit1, +; const uint8_t *limit1, +; const uint8_t *thresh1) +; r0 uint8_t *s, +; r1 int p, +; r2 const uint8_t *blimit0, +; r3 const uint8_t *limit0, +; sp const uint8_t *thresh0, +; sp+4 const uint8_t *blimit1, +; sp+8 const uint8_t *limit1, +; sp+12 const uint8_t *thresh1, + +|aom_lpf_horizontal_4_dual_neon| PROC + push {lr} + + ldr r12, [sp, #4] ; load thresh0 + vld1.8 {d0}, [r2] ; load blimit0 to first half q + vld1.8 {d2}, [r3] ; load limit0 to first half q + + add r1, r1, r1 ; double pitch + ldr r2, [sp, #8] ; load blimit1 + + vld1.8 {d4}, [r12] ; load thresh0 to first half q + + ldr r3, [sp, #12] ; load limit1 + ldr r12, [sp, #16] ; load thresh1 + vld1.8 {d1}, [r2] ; load blimit1 to 2nd half q + + sub r2, r0, r1, lsl #1 ; s[-4 * p] + + vld1.8 {d3}, [r3] ; load limit1 to 2nd half q + vld1.8 {d5}, [r12] ; load thresh1 to 2nd half q + + vpush {d8-d15} ; save neon registers + + add r3, r2, r1, lsr #1 ; s[-3 * p] + + vld1.u8 {q3}, [r2@64], r1 ; p3 + vld1.u8 {q4}, [r3@64], r1 ; p2 + vld1.u8 {q5}, [r2@64], r1 ; p1 + vld1.u8 {q6}, [r3@64], r1 ; p0 + vld1.u8 {q7}, [r2@64], r1 ; q0 + vld1.u8 {q8}, [r3@64], r1 ; q1 + vld1.u8 {q9}, [r2@64] ; q2 + vld1.u8 {q10}, [r3@64] ; q3 + + sub r2, r2, r1, lsl #1 + sub r3, r3, r1, lsl #1 + + bl aom_loop_filter_neon_16 + + vst1.u8 {q5}, [r2@64], r1 ; store op1 + vst1.u8 {q6}, [r3@64], r1 ; store op0 + vst1.u8 {q7}, [r2@64], r1 ; store oq0 + vst1.u8 {q8}, [r3@64], r1 ; store oq1 + + vpop {d8-d15} ; restore neon registers + + pop {pc} + ENDP ; |aom_lpf_horizontal_4_dual_neon| + +; void aom_loop_filter_neon_16(); +; This is a helper function for the loopfilters. The invidual functions do the +; necessary load, transpose (if necessary) and store. This function uses +; registers d8-d15, so the calling function must save those registers. +; +; r0-r3, r12 PRESERVE +; q0 blimit +; q1 limit +; q2 thresh +; q3 p3 +; q4 p2 +; q5 p1 +; q6 p0 +; q7 q0 +; q8 q1 +; q9 q2 +; q10 q3 +; +; Outputs: +; q5 op1 +; q6 op0 +; q7 oq0 +; q8 oq1 +|aom_loop_filter_neon_16| PROC + + ; filter_mask + vabd.u8 q11, q3, q4 ; m1 = abs(p3 - p2) + vabd.u8 q12, q4, q5 ; m2 = abs(p2 - p1) + vabd.u8 q13, q5, q6 ; m3 = abs(p1 - p0) + vabd.u8 q14, q8, q7 ; m4 = abs(q1 - q0) + vabd.u8 q3, q9, q8 ; m5 = abs(q2 - q1) + vabd.u8 q4, q10, q9 ; m6 = abs(q3 - q2) + + ; only compare the largest value to limit + vmax.u8 q11, q11, q12 ; m7 = max(m1, m2) + vmax.u8 q12, q13, q14 ; m8 = max(m3, m4) + + vabd.u8 q9, q6, q7 ; abs(p0 - q0) + + vmax.u8 q3, q3, q4 ; m9 = max(m5, m6) + + vmov.u8 q10, #0x80 + + vmax.u8 q15, q11, q12 ; m10 = max(m7, m8) + + vcgt.u8 q13, q13, q2 ; (abs(p1 - p0) > thresh)*-1 + vcgt.u8 q14, q14, q2 ; (abs(q1 - q0) > thresh)*-1 + vmax.u8 q15, q15, q3 ; m11 = max(m10, m9) + + vabd.u8 q2, q5, q8 ; a = abs(p1 - q1) + vqadd.u8 q9, q9, q9 ; b = abs(p0 - q0) * 2 + + veor q7, q7, q10 ; qs0 + + vcge.u8 q15, q1, q15 ; abs(m11) > limit + + vshr.u8 q2, q2, #1 ; a = a / 2 + veor q6, q6, q10 ; ps0 + + veor q5, q5, q10 ; ps1 + vqadd.u8 q9, q9, q2 ; a = b + a + + veor q8, q8, q10 ; qs1 + + vmov.u16 q4, #3 + + vsubl.s8 q2, d14, d12 ; ( qs0 - ps0) + vsubl.s8 q11, d15, d13 + + vcge.u8 q9, q0, q9 ; a > blimit + + vqsub.s8 q1, q5, q8 ; filter = clamp(ps1-qs1) + vorr q14, q13, q14 ; hev + + vmul.i16 q2, q2, q4 ; 3 * ( qs0 - ps0) + vmul.i16 q11, q11, q4 + + vand q1, q1, q14 ; filter &= hev + vand q15, q15, q9 ; mask + + vmov.u8 q4, #3 + + vaddw.s8 q2, q2, d2 ; filter + 3 * (qs0 - ps0) + vaddw.s8 q11, q11, d3 + + vmov.u8 q9, #4 + + ; filter = clamp(filter + 3 * ( qs0 - ps0)) + vqmovn.s16 d2, q2 + vqmovn.s16 d3, q11 + vand q1, q1, q15 ; filter &= mask + + vqadd.s8 q2, q1, q4 ; filter2 = clamp(filter+3) + vqadd.s8 q1, q1, q9 ; filter1 = clamp(filter+4) + vshr.s8 q2, q2, #3 ; filter2 >>= 3 + vshr.s8 q1, q1, #3 ; filter1 >>= 3 + + + vqadd.s8 q11, q6, q2 ; u = clamp(ps0 + filter2) + vqsub.s8 q0, q7, q1 ; u = clamp(qs0 - filter1) + + ; outer tap adjustments + vrshr.s8 q1, q1, #1 ; filter = ++filter1 >> 1 + + veor q7, q0, q10 ; *oq0 = u^0x80 + + vbic q1, q1, q14 ; filter &= ~hev + + vqadd.s8 q13, q5, q1 ; u = clamp(ps1 + filter) + vqsub.s8 q12, q8, q1 ; u = clamp(qs1 - filter) + + veor q6, q11, q10 ; *op0 = u^0x80 + veor q5, q13, q10 ; *op1 = u^0x80 + veor q8, q12, q10 ; *oq1 = u^0x80 + + bx lr + ENDP ; |aom_loop_filter_neon_16| + + END diff --git a/third_party/aom/aom_dsp/arm/loopfilter_16_neon.c b/third_party/aom/aom_dsp/arm/loopfilter_16_neon.c new file mode 100644 index 0000000000..c0562a6ea5 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/loopfilter_16_neon.c @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "./aom_config.h" +#include "aom/aom_integer.h" + +static INLINE void loop_filter_neon_16(uint8x16_t qblimit, // blimit + uint8x16_t qlimit, // limit + uint8x16_t qthresh, // thresh + uint8x16_t q3, // p3 + uint8x16_t q4, // p2 + uint8x16_t q5, // p1 + uint8x16_t q6, // p0 + uint8x16_t q7, // q0 + uint8x16_t q8, // q1 + uint8x16_t q9, // q2 + uint8x16_t q10, // q3 + uint8x16_t *q5r, // p1 + uint8x16_t *q6r, // p0 + uint8x16_t *q7r, // q0 + uint8x16_t *q8r) { // q1 + uint8x16_t q1u8, q2u8, q11u8, q12u8, q13u8, q14u8, q15u8; + int16x8_t q2s16, q11s16; + uint16x8_t q4u16; + int8x16_t q0s8, q1s8, q2s8, q11s8, q12s8, q13s8; + int8x8_t d2s8, d3s8; + + q11u8 = vabdq_u8(q3, q4); + q12u8 = vabdq_u8(q4, q5); + q13u8 = vabdq_u8(q5, q6); + q14u8 = vabdq_u8(q8, q7); + q3 = vabdq_u8(q9, q8); + q4 = vabdq_u8(q10, q9); + + q11u8 = vmaxq_u8(q11u8, q12u8); + q12u8 = vmaxq_u8(q13u8, q14u8); + q3 = vmaxq_u8(q3, q4); + q15u8 = vmaxq_u8(q11u8, q12u8); + + q9 = vabdq_u8(q6, q7); + + // aom_hevmask + q13u8 = vcgtq_u8(q13u8, qthresh); + q14u8 = vcgtq_u8(q14u8, qthresh); + q15u8 = vmaxq_u8(q15u8, q3); + + q2u8 = vabdq_u8(q5, q8); + q9 = vqaddq_u8(q9, q9); + + q15u8 = vcgeq_u8(qlimit, q15u8); + + // aom_filter() function + // convert to signed + q10 = vdupq_n_u8(0x80); + q8 = veorq_u8(q8, q10); + q7 = veorq_u8(q7, q10); + q6 = veorq_u8(q6, q10); + q5 = veorq_u8(q5, q10); + + q2u8 = vshrq_n_u8(q2u8, 1); + q9 = vqaddq_u8(q9, q2u8); + + q2s16 = vsubl_s8(vget_low_s8(vreinterpretq_s8_u8(q7)), + vget_low_s8(vreinterpretq_s8_u8(q6))); + q11s16 = vsubl_s8(vget_high_s8(vreinterpretq_s8_u8(q7)), + vget_high_s8(vreinterpretq_s8_u8(q6))); + + q9 = vcgeq_u8(qblimit, q9); + + q1s8 = vqsubq_s8(vreinterpretq_s8_u8(q5), vreinterpretq_s8_u8(q8)); + + q14u8 = vorrq_u8(q13u8, q14u8); + + q4u16 = vdupq_n_u16(3); + q2s16 = vmulq_s16(q2s16, vreinterpretq_s16_u16(q4u16)); + q11s16 = vmulq_s16(q11s16, vreinterpretq_s16_u16(q4u16)); + + q1u8 = vandq_u8(vreinterpretq_u8_s8(q1s8), q14u8); + q15u8 = vandq_u8(q15u8, q9); + + q1s8 = vreinterpretq_s8_u8(q1u8); + q2s16 = vaddw_s8(q2s16, vget_low_s8(q1s8)); + q11s16 = vaddw_s8(q11s16, vget_high_s8(q1s8)); + + q4 = vdupq_n_u8(3); + q9 = vdupq_n_u8(4); + // aom_filter = clamp(aom_filter + 3 * ( qs0 - ps0)) + d2s8 = vqmovn_s16(q2s16); + d3s8 = vqmovn_s16(q11s16); + q1s8 = vcombine_s8(d2s8, d3s8); + q1u8 = vandq_u8(vreinterpretq_u8_s8(q1s8), q15u8); + q1s8 = vreinterpretq_s8_u8(q1u8); + + q2s8 = vqaddq_s8(q1s8, vreinterpretq_s8_u8(q4)); + q1s8 = vqaddq_s8(q1s8, vreinterpretq_s8_u8(q9)); + q2s8 = vshrq_n_s8(q2s8, 3); + q1s8 = vshrq_n_s8(q1s8, 3); + + q11s8 = vqaddq_s8(vreinterpretq_s8_u8(q6), q2s8); + q0s8 = vqsubq_s8(vreinterpretq_s8_u8(q7), q1s8); + + q1s8 = vrshrq_n_s8(q1s8, 1); + q1s8 = vbicq_s8(q1s8, vreinterpretq_s8_u8(q14u8)); + + q13s8 = vqaddq_s8(vreinterpretq_s8_u8(q5), q1s8); + q12s8 = vqsubq_s8(vreinterpretq_s8_u8(q8), q1s8); + + *q8r = veorq_u8(vreinterpretq_u8_s8(q12s8), q10); + *q7r = veorq_u8(vreinterpretq_u8_s8(q0s8), q10); + *q6r = veorq_u8(vreinterpretq_u8_s8(q11s8), q10); + *q5r = veorq_u8(vreinterpretq_u8_s8(q13s8), q10); + return; +} + +void aom_lpf_horizontal_4_dual_neon( + uint8_t *s, int p /* pitch */, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, + const uint8_t *limit1, const uint8_t *thresh1) { + uint8x8_t dblimit0, dlimit0, dthresh0, dblimit1, dlimit1, dthresh1; + uint8x16_t qblimit, qlimit, qthresh; + uint8x16_t q3u8, q4u8, q5u8, q6u8, q7u8, q8u8, q9u8, q10u8; + + dblimit0 = vld1_u8(blimit0); + dlimit0 = vld1_u8(limit0); + dthresh0 = vld1_u8(thresh0); + dblimit1 = vld1_u8(blimit1); + dlimit1 = vld1_u8(limit1); + dthresh1 = vld1_u8(thresh1); + qblimit = vcombine_u8(dblimit0, dblimit1); + qlimit = vcombine_u8(dlimit0, dlimit1); + qthresh = vcombine_u8(dthresh0, dthresh1); + + s -= (p << 2); + + q3u8 = vld1q_u8(s); + s += p; + q4u8 = vld1q_u8(s); + s += p; + q5u8 = vld1q_u8(s); + s += p; + q6u8 = vld1q_u8(s); + s += p; + q7u8 = vld1q_u8(s); + s += p; + q8u8 = vld1q_u8(s); + s += p; + q9u8 = vld1q_u8(s); + s += p; + q10u8 = vld1q_u8(s); + + loop_filter_neon_16(qblimit, qlimit, qthresh, q3u8, q4u8, q5u8, q6u8, q7u8, + q8u8, q9u8, q10u8, &q5u8, &q6u8, &q7u8, &q8u8); + + s -= (p * 5); + vst1q_u8(s, q5u8); + s += p; + vst1q_u8(s, q6u8); + s += p; + vst1q_u8(s, q7u8); + s += p; + vst1q_u8(s, q8u8); + return; +} diff --git a/third_party/aom/aom_dsp/arm/loopfilter_4_neon.asm b/third_party/aom/aom_dsp/arm/loopfilter_4_neon.asm new file mode 100644 index 0000000000..8b54984d56 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/loopfilter_4_neon.asm @@ -0,0 +1,252 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + EXPORT |aom_lpf_horizontal_4_neon| + EXPORT |aom_lpf_vertical_4_neon| + ARM + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +; Currently aom only works on iterations 8 at a time. The aom loop filter +; works on 16 iterations at a time. +; +; void aom_lpf_horizontal_4_neon(uint8_t *s, +; int p /* pitch */, +; const uint8_t *blimit, +; const uint8_t *limit, +; const uint8_t *thresh) +; +; r0 uint8_t *s, +; r1 int p, /* pitch */ +; r2 const uint8_t *blimit, +; r3 const uint8_t *limit, +; sp const uint8_t *thresh, +|aom_lpf_horizontal_4_neon| PROC + push {lr} + + vld1.8 {d0[]}, [r2] ; duplicate *blimit + ldr r2, [sp, #4] ; load thresh + add r1, r1, r1 ; double pitch + + vld1.8 {d1[]}, [r3] ; duplicate *limit + vld1.8 {d2[]}, [r2] ; duplicate *thresh + + sub r2, r0, r1, lsl #1 ; move src pointer down by 4 lines + add r3, r2, r1, lsr #1 ; set to 3 lines down + + vld1.u8 {d3}, [r2@64], r1 ; p3 + vld1.u8 {d4}, [r3@64], r1 ; p2 + vld1.u8 {d5}, [r2@64], r1 ; p1 + vld1.u8 {d6}, [r3@64], r1 ; p0 + vld1.u8 {d7}, [r2@64], r1 ; q0 + vld1.u8 {d16}, [r3@64], r1 ; q1 + vld1.u8 {d17}, [r2@64] ; q2 + vld1.u8 {d18}, [r3@64] ; q3 + + sub r2, r2, r1, lsl #1 + sub r3, r3, r1, lsl #1 + + bl aom_loop_filter_neon + + vst1.u8 {d4}, [r2@64], r1 ; store op1 + vst1.u8 {d5}, [r3@64], r1 ; store op0 + vst1.u8 {d6}, [r2@64], r1 ; store oq0 + vst1.u8 {d7}, [r3@64], r1 ; store oq1 + + pop {pc} + ENDP ; |aom_lpf_horizontal_4_neon| + +; Currently aom only works on iterations 8 at a time. The aom loop filter +; works on 16 iterations at a time. +; +; void aom_lpf_vertical_4_neon(uint8_t *s, +; int p /* pitch */, +; const uint8_t *blimit, +; const uint8_t *limit, +; const uint8_t *thresh) +; +; r0 uint8_t *s, +; r1 int p, /* pitch */ +; r2 const uint8_t *blimit, +; r3 const uint8_t *limit, +; sp const uint8_t *thresh, +|aom_lpf_vertical_4_neon| PROC + push {lr} + + vld1.8 {d0[]}, [r2] ; duplicate *blimit + vld1.8 {d1[]}, [r3] ; duplicate *limit + + ldr r3, [sp, #4] ; load thresh + sub r2, r0, #4 ; move s pointer down by 4 columns + + vld1.8 {d2[]}, [r3] ; duplicate *thresh + + vld1.u8 {d3}, [r2], r1 ; load s data + vld1.u8 {d4}, [r2], r1 + vld1.u8 {d5}, [r2], r1 + vld1.u8 {d6}, [r2], r1 + vld1.u8 {d7}, [r2], r1 + vld1.u8 {d16}, [r2], r1 + vld1.u8 {d17}, [r2], r1 + vld1.u8 {d18}, [r2] + + ;transpose to 8x16 matrix + vtrn.32 d3, d7 + vtrn.32 d4, d16 + vtrn.32 d5, d17 + vtrn.32 d6, d18 + + vtrn.16 d3, d5 + vtrn.16 d4, d6 + vtrn.16 d7, d17 + vtrn.16 d16, d18 + + vtrn.8 d3, d4 + vtrn.8 d5, d6 + vtrn.8 d7, d16 + vtrn.8 d17, d18 + + bl aom_loop_filter_neon + + sub r0, r0, #2 + + ;store op1, op0, oq0, oq1 + vst4.8 {d4[0], d5[0], d6[0], d7[0]}, [r0], r1 + vst4.8 {d4[1], d5[1], d6[1], d7[1]}, [r0], r1 + vst4.8 {d4[2], d5[2], d6[2], d7[2]}, [r0], r1 + vst4.8 {d4[3], d5[3], d6[3], d7[3]}, [r0], r1 + vst4.8 {d4[4], d5[4], d6[4], d7[4]}, [r0], r1 + vst4.8 {d4[5], d5[5], d6[5], d7[5]}, [r0], r1 + vst4.8 {d4[6], d5[6], d6[6], d7[6]}, [r0], r1 + vst4.8 {d4[7], d5[7], d6[7], d7[7]}, [r0] + + pop {pc} + ENDP ; |aom_lpf_vertical_4_neon| + +; void aom_loop_filter_neon(); +; This is a helper function for the loopfilters. The invidual functions do the +; necessary load, transpose (if necessary) and store. The function does not use +; registers d8-d15. +; +; Inputs: +; r0-r3, r12 PRESERVE +; d0 blimit +; d1 limit +; d2 thresh +; d3 p3 +; d4 p2 +; d5 p1 +; d6 p0 +; d7 q0 +; d16 q1 +; d17 q2 +; d18 q3 +; +; Outputs: +; d4 op1 +; d5 op0 +; d6 oq0 +; d7 oq1 +|aom_loop_filter_neon| PROC + ; filter_mask + vabd.u8 d19, d3, d4 ; m1 = abs(p3 - p2) + vabd.u8 d20, d4, d5 ; m2 = abs(p2 - p1) + vabd.u8 d21, d5, d6 ; m3 = abs(p1 - p0) + vabd.u8 d22, d16, d7 ; m4 = abs(q1 - q0) + vabd.u8 d3, d17, d16 ; m5 = abs(q2 - q1) + vabd.u8 d4, d18, d17 ; m6 = abs(q3 - q2) + + ; only compare the largest value to limit + vmax.u8 d19, d19, d20 ; m1 = max(m1, m2) + vmax.u8 d20, d21, d22 ; m2 = max(m3, m4) + + vabd.u8 d17, d6, d7 ; abs(p0 - q0) + + vmax.u8 d3, d3, d4 ; m3 = max(m5, m6) + + vmov.u8 d18, #0x80 + + vmax.u8 d23, d19, d20 ; m1 = max(m1, m2) + + ; hevmask + vcgt.u8 d21, d21, d2 ; (abs(p1 - p0) > thresh)*-1 + vcgt.u8 d22, d22, d2 ; (abs(q1 - q0) > thresh)*-1 + vmax.u8 d23, d23, d3 ; m1 = max(m1, m3) + + vabd.u8 d28, d5, d16 ; a = abs(p1 - q1) + vqadd.u8 d17, d17, d17 ; b = abs(p0 - q0) * 2 + + veor d7, d7, d18 ; qs0 + + vcge.u8 d23, d1, d23 ; abs(m1) > limit + + ; filter() function + ; convert to signed + + vshr.u8 d28, d28, #1 ; a = a / 2 + veor d6, d6, d18 ; ps0 + + veor d5, d5, d18 ; ps1 + vqadd.u8 d17, d17, d28 ; a = b + a + + veor d16, d16, d18 ; qs1 + + vmov.u8 d19, #3 + + vsub.s8 d28, d7, d6 ; ( qs0 - ps0) + + vcge.u8 d17, d0, d17 ; a > blimit + + vqsub.s8 d27, d5, d16 ; filter = clamp(ps1-qs1) + vorr d22, d21, d22 ; hevmask + + vmull.s8 q12, d28, d19 ; 3 * ( qs0 - ps0) + + vand d27, d27, d22 ; filter &= hev + vand d23, d23, d17 ; filter_mask + + vaddw.s8 q12, q12, d27 ; filter + 3 * (qs0 - ps0) + + vmov.u8 d17, #4 + + ; filter = clamp(filter + 3 * ( qs0 - ps0)) + vqmovn.s16 d27, q12 + + vand d27, d27, d23 ; filter &= mask + + vqadd.s8 d28, d27, d19 ; filter2 = clamp(filter+3) + vqadd.s8 d27, d27, d17 ; filter1 = clamp(filter+4) + vshr.s8 d28, d28, #3 ; filter2 >>= 3 + vshr.s8 d27, d27, #3 ; filter1 >>= 3 + + vqadd.s8 d19, d6, d28 ; u = clamp(ps0 + filter2) + vqsub.s8 d26, d7, d27 ; u = clamp(qs0 - filter1) + + ; outer tap adjustments + vrshr.s8 d27, d27, #1 ; filter = ++filter1 >> 1 + + veor d6, d26, d18 ; *oq0 = u^0x80 + + vbic d27, d27, d22 ; filter &= ~hev + + vqadd.s8 d21, d5, d27 ; u = clamp(ps1 + filter) + vqsub.s8 d20, d16, d27 ; u = clamp(qs1 - filter) + + veor d5, d19, d18 ; *op0 = u^0x80 + veor d4, d21, d18 ; *op1 = u^0x80 + veor d7, d20, d18 ; *oq1 = u^0x80 + + bx lr + ENDP ; |aom_loop_filter_neon| + + END diff --git a/third_party/aom/aom_dsp/arm/loopfilter_4_neon.c b/third_party/aom/aom_dsp/arm/loopfilter_4_neon.c new file mode 100644 index 0000000000..2b1f80b81e --- /dev/null +++ b/third_party/aom/aom_dsp/arm/loopfilter_4_neon.c @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" + +static INLINE void loop_filter_neon(uint8x8_t dblimit, // flimit + uint8x8_t dlimit, // limit + uint8x8_t dthresh, // thresh + uint8x8_t d3u8, // p3 + uint8x8_t d4u8, // p2 + uint8x8_t d5u8, // p1 + uint8x8_t d6u8, // p0 + uint8x8_t d7u8, // q0 + uint8x8_t d16u8, // q1 + uint8x8_t d17u8, // q2 + uint8x8_t d18u8, // q3 + uint8x8_t *d4ru8, // p1 + uint8x8_t *d5ru8, // p0 + uint8x8_t *d6ru8, // q0 + uint8x8_t *d7ru8) { // q1 + uint8x8_t d19u8, d20u8, d21u8, d22u8, d23u8, d27u8, d28u8; + int16x8_t q12s16; + int8x8_t d19s8, d20s8, d21s8, d26s8, d27s8, d28s8; + + d19u8 = vabd_u8(d3u8, d4u8); + d20u8 = vabd_u8(d4u8, d5u8); + d21u8 = vabd_u8(d5u8, d6u8); + d22u8 = vabd_u8(d16u8, d7u8); + d3u8 = vabd_u8(d17u8, d16u8); + d4u8 = vabd_u8(d18u8, d17u8); + + d19u8 = vmax_u8(d19u8, d20u8); + d20u8 = vmax_u8(d21u8, d22u8); + d3u8 = vmax_u8(d3u8, d4u8); + d23u8 = vmax_u8(d19u8, d20u8); + + d17u8 = vabd_u8(d6u8, d7u8); + + d21u8 = vcgt_u8(d21u8, dthresh); + d22u8 = vcgt_u8(d22u8, dthresh); + d23u8 = vmax_u8(d23u8, d3u8); + + d28u8 = vabd_u8(d5u8, d16u8); + d17u8 = vqadd_u8(d17u8, d17u8); + + d23u8 = vcge_u8(dlimit, d23u8); + + d18u8 = vdup_n_u8(0x80); + d5u8 = veor_u8(d5u8, d18u8); + d6u8 = veor_u8(d6u8, d18u8); + d7u8 = veor_u8(d7u8, d18u8); + d16u8 = veor_u8(d16u8, d18u8); + + d28u8 = vshr_n_u8(d28u8, 1); + d17u8 = vqadd_u8(d17u8, d28u8); + + d19u8 = vdup_n_u8(3); + + d28s8 = vsub_s8(vreinterpret_s8_u8(d7u8), vreinterpret_s8_u8(d6u8)); + + d17u8 = vcge_u8(dblimit, d17u8); + + d27s8 = vqsub_s8(vreinterpret_s8_u8(d5u8), vreinterpret_s8_u8(d16u8)); + + d22u8 = vorr_u8(d21u8, d22u8); + + q12s16 = vmull_s8(d28s8, vreinterpret_s8_u8(d19u8)); + + d27u8 = vand_u8(vreinterpret_u8_s8(d27s8), d22u8); + d23u8 = vand_u8(d23u8, d17u8); + + q12s16 = vaddw_s8(q12s16, vreinterpret_s8_u8(d27u8)); + + d17u8 = vdup_n_u8(4); + + d27s8 = vqmovn_s16(q12s16); + d27u8 = vand_u8(vreinterpret_u8_s8(d27s8), d23u8); + d27s8 = vreinterpret_s8_u8(d27u8); + + d28s8 = vqadd_s8(d27s8, vreinterpret_s8_u8(d19u8)); + d27s8 = vqadd_s8(d27s8, vreinterpret_s8_u8(d17u8)); + d28s8 = vshr_n_s8(d28s8, 3); + d27s8 = vshr_n_s8(d27s8, 3); + + d19s8 = vqadd_s8(vreinterpret_s8_u8(d6u8), d28s8); + d26s8 = vqsub_s8(vreinterpret_s8_u8(d7u8), d27s8); + + d27s8 = vrshr_n_s8(d27s8, 1); + d27s8 = vbic_s8(d27s8, vreinterpret_s8_u8(d22u8)); + + d21s8 = vqadd_s8(vreinterpret_s8_u8(d5u8), d27s8); + d20s8 = vqsub_s8(vreinterpret_s8_u8(d16u8), d27s8); + + *d4ru8 = veor_u8(vreinterpret_u8_s8(d21s8), d18u8); + *d5ru8 = veor_u8(vreinterpret_u8_s8(d19s8), d18u8); + *d6ru8 = veor_u8(vreinterpret_u8_s8(d26s8), d18u8); + *d7ru8 = veor_u8(vreinterpret_u8_s8(d20s8), d18u8); + return; +} + +void aom_lpf_horizontal_4_neon(uint8_t *src, int pitch, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh) { + int i; + uint8_t *s, *psrc; + uint8x8_t dblimit, dlimit, dthresh; + uint8x8_t d3u8, d4u8, d5u8, d6u8, d7u8, d16u8, d17u8, d18u8; + + dblimit = vld1_u8(blimit); + dlimit = vld1_u8(limit); + dthresh = vld1_u8(thresh); + + psrc = src - (pitch << 2); + for (i = 0; i < 1; i++) { + s = psrc + i * 8; + + d3u8 = vld1_u8(s); + s += pitch; + d4u8 = vld1_u8(s); + s += pitch; + d5u8 = vld1_u8(s); + s += pitch; + d6u8 = vld1_u8(s); + s += pitch; + d7u8 = vld1_u8(s); + s += pitch; + d16u8 = vld1_u8(s); + s += pitch; + d17u8 = vld1_u8(s); + s += pitch; + d18u8 = vld1_u8(s); + + loop_filter_neon(dblimit, dlimit, dthresh, d3u8, d4u8, d5u8, d6u8, d7u8, + d16u8, d17u8, d18u8, &d4u8, &d5u8, &d6u8, &d7u8); + + s -= (pitch * 5); + vst1_u8(s, d4u8); + s += pitch; + vst1_u8(s, d5u8); + s += pitch; + vst1_u8(s, d6u8); + s += pitch; + vst1_u8(s, d7u8); + } + return; +} + +void aom_lpf_vertical_4_neon(uint8_t *src, int pitch, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh) { + int i, pitch8; + uint8_t *s; + uint8x8_t dblimit, dlimit, dthresh; + uint8x8_t d3u8, d4u8, d5u8, d6u8, d7u8, d16u8, d17u8, d18u8; + uint32x2x2_t d2tmp0, d2tmp1, d2tmp2, d2tmp3; + uint16x4x2_t d2tmp4, d2tmp5, d2tmp6, d2tmp7; + uint8x8x2_t d2tmp8, d2tmp9, d2tmp10, d2tmp11; + uint8x8x4_t d4Result; + + dblimit = vld1_u8(blimit); + dlimit = vld1_u8(limit); + dthresh = vld1_u8(thresh); + + pitch8 = pitch * 8; + for (i = 0; i < 1; i++, src += pitch8) { + s = src - (i + 1) * 4; + + d3u8 = vld1_u8(s); + s += pitch; + d4u8 = vld1_u8(s); + s += pitch; + d5u8 = vld1_u8(s); + s += pitch; + d6u8 = vld1_u8(s); + s += pitch; + d7u8 = vld1_u8(s); + s += pitch; + d16u8 = vld1_u8(s); + s += pitch; + d17u8 = vld1_u8(s); + s += pitch; + d18u8 = vld1_u8(s); + + d2tmp0 = vtrn_u32(vreinterpret_u32_u8(d3u8), vreinterpret_u32_u8(d7u8)); + d2tmp1 = vtrn_u32(vreinterpret_u32_u8(d4u8), vreinterpret_u32_u8(d16u8)); + d2tmp2 = vtrn_u32(vreinterpret_u32_u8(d5u8), vreinterpret_u32_u8(d17u8)); + d2tmp3 = vtrn_u32(vreinterpret_u32_u8(d6u8), vreinterpret_u32_u8(d18u8)); + + d2tmp4 = vtrn_u16(vreinterpret_u16_u32(d2tmp0.val[0]), + vreinterpret_u16_u32(d2tmp2.val[0])); + d2tmp5 = vtrn_u16(vreinterpret_u16_u32(d2tmp1.val[0]), + vreinterpret_u16_u32(d2tmp3.val[0])); + d2tmp6 = vtrn_u16(vreinterpret_u16_u32(d2tmp0.val[1]), + vreinterpret_u16_u32(d2tmp2.val[1])); + d2tmp7 = vtrn_u16(vreinterpret_u16_u32(d2tmp1.val[1]), + vreinterpret_u16_u32(d2tmp3.val[1])); + + d2tmp8 = vtrn_u8(vreinterpret_u8_u16(d2tmp4.val[0]), + vreinterpret_u8_u16(d2tmp5.val[0])); + d2tmp9 = vtrn_u8(vreinterpret_u8_u16(d2tmp4.val[1]), + vreinterpret_u8_u16(d2tmp5.val[1])); + d2tmp10 = vtrn_u8(vreinterpret_u8_u16(d2tmp6.val[0]), + vreinterpret_u8_u16(d2tmp7.val[0])); + d2tmp11 = vtrn_u8(vreinterpret_u8_u16(d2tmp6.val[1]), + vreinterpret_u8_u16(d2tmp7.val[1])); + + d3u8 = d2tmp8.val[0]; + d4u8 = d2tmp8.val[1]; + d5u8 = d2tmp9.val[0]; + d6u8 = d2tmp9.val[1]; + d7u8 = d2tmp10.val[0]; + d16u8 = d2tmp10.val[1]; + d17u8 = d2tmp11.val[0]; + d18u8 = d2tmp11.val[1]; + + loop_filter_neon(dblimit, dlimit, dthresh, d3u8, d4u8, d5u8, d6u8, d7u8, + d16u8, d17u8, d18u8, &d4u8, &d5u8, &d6u8, &d7u8); + + d4Result.val[0] = d4u8; + d4Result.val[1] = d5u8; + d4Result.val[2] = d6u8; + d4Result.val[3] = d7u8; + + src -= 2; + vst4_lane_u8(src, d4Result, 0); + src += pitch; + vst4_lane_u8(src, d4Result, 1); + src += pitch; + vst4_lane_u8(src, d4Result, 2); + src += pitch; + vst4_lane_u8(src, d4Result, 3); + src += pitch; + vst4_lane_u8(src, d4Result, 4); + src += pitch; + vst4_lane_u8(src, d4Result, 5); + src += pitch; + vst4_lane_u8(src, d4Result, 6); + src += pitch; + vst4_lane_u8(src, d4Result, 7); + } + return; +} diff --git a/third_party/aom/aom_dsp/arm/loopfilter_8_neon.asm b/third_party/aom/aom_dsp/arm/loopfilter_8_neon.asm new file mode 100644 index 0000000000..9f3db66ee0 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/loopfilter_8_neon.asm @@ -0,0 +1,428 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + EXPORT |aom_lpf_horizontal_8_neon| + EXPORT |aom_lpf_vertical_8_neon| + ARM + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +; Currently aom only works on iterations 8 at a time. The aom loop filter +; works on 16 iterations at a time. +; +; void aom_lpf_horizontal_8_neon(uint8_t *s, int p, +; const uint8_t *blimit, +; const uint8_t *limit, +; const uint8_t *thresh) +; r0 uint8_t *s, +; r1 int p, /* pitch */ +; r2 const uint8_t *blimit, +; r3 const uint8_t *limit, +; sp const uint8_t *thresh, +|aom_lpf_horizontal_8_neon| PROC + push {r4-r5, lr} + + vld1.8 {d0[]}, [r2] ; duplicate *blimit + ldr r2, [sp, #12] ; load thresh + add r1, r1, r1 ; double pitch + + vld1.8 {d1[]}, [r3] ; duplicate *limit + vld1.8 {d2[]}, [r2] ; duplicate *thresh + + sub r3, r0, r1, lsl #1 ; move src pointer down by 4 lines + add r2, r3, r1, lsr #1 ; set to 3 lines down + + vld1.u8 {d3}, [r3@64], r1 ; p3 + vld1.u8 {d4}, [r2@64], r1 ; p2 + vld1.u8 {d5}, [r3@64], r1 ; p1 + vld1.u8 {d6}, [r2@64], r1 ; p0 + vld1.u8 {d7}, [r3@64], r1 ; q0 + vld1.u8 {d16}, [r2@64], r1 ; q1 + vld1.u8 {d17}, [r3@64] ; q2 + vld1.u8 {d18}, [r2@64], r1 ; q3 + + sub r3, r3, r1, lsl #1 + sub r2, r2, r1, lsl #2 + + bl aom_mbloop_filter_neon + + vst1.u8 {d0}, [r2@64], r1 ; store op2 + vst1.u8 {d1}, [r3@64], r1 ; store op1 + vst1.u8 {d2}, [r2@64], r1 ; store op0 + vst1.u8 {d3}, [r3@64], r1 ; store oq0 + vst1.u8 {d4}, [r2@64], r1 ; store oq1 + vst1.u8 {d5}, [r3@64], r1 ; store oq2 + + pop {r4-r5, pc} + + ENDP ; |aom_lpf_horizontal_8_neon| + +; void aom_lpf_vertical_8_neon(uint8_t *s, +; int pitch, +; const uint8_t *blimit, +; const uint8_t *limit, +; const uint8_t *thresh) +; +; r0 uint8_t *s, +; r1 int pitch, +; r2 const uint8_t *blimit, +; r3 const uint8_t *limit, +; sp const uint8_t *thresh, +|aom_lpf_vertical_8_neon| PROC + push {r4-r5, lr} + + vld1.8 {d0[]}, [r2] ; duplicate *blimit + vld1.8 {d1[]}, [r3] ; duplicate *limit + + ldr r3, [sp, #12] ; load thresh + sub r2, r0, #4 ; move s pointer down by 4 columns + + vld1.8 {d2[]}, [r3] ; duplicate *thresh + + vld1.u8 {d3}, [r2], r1 ; load s data + vld1.u8 {d4}, [r2], r1 + vld1.u8 {d5}, [r2], r1 + vld1.u8 {d6}, [r2], r1 + vld1.u8 {d7}, [r2], r1 + vld1.u8 {d16}, [r2], r1 + vld1.u8 {d17}, [r2], r1 + vld1.u8 {d18}, [r2] + + ;transpose to 8x16 matrix + vtrn.32 d3, d7 + vtrn.32 d4, d16 + vtrn.32 d5, d17 + vtrn.32 d6, d18 + + vtrn.16 d3, d5 + vtrn.16 d4, d6 + vtrn.16 d7, d17 + vtrn.16 d16, d18 + + vtrn.8 d3, d4 + vtrn.8 d5, d6 + vtrn.8 d7, d16 + vtrn.8 d17, d18 + + sub r2, r0, #3 + add r3, r0, #1 + + bl aom_mbloop_filter_neon + + ;store op2, op1, op0, oq0 + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r2], r1 + vst4.8 {d0[1], d1[1], d2[1], d3[1]}, [r2], r1 + vst4.8 {d0[2], d1[2], d2[2], d3[2]}, [r2], r1 + vst4.8 {d0[3], d1[3], d2[3], d3[3]}, [r2], r1 + vst4.8 {d0[4], d1[4], d2[4], d3[4]}, [r2], r1 + vst4.8 {d0[5], d1[5], d2[5], d3[5]}, [r2], r1 + vst4.8 {d0[6], d1[6], d2[6], d3[6]}, [r2], r1 + vst4.8 {d0[7], d1[7], d2[7], d3[7]}, [r2] + + ;store oq1, oq2 + vst2.8 {d4[0], d5[0]}, [r3], r1 + vst2.8 {d4[1], d5[1]}, [r3], r1 + vst2.8 {d4[2], d5[2]}, [r3], r1 + vst2.8 {d4[3], d5[3]}, [r3], r1 + vst2.8 {d4[4], d5[4]}, [r3], r1 + vst2.8 {d4[5], d5[5]}, [r3], r1 + vst2.8 {d4[6], d5[6]}, [r3], r1 + vst2.8 {d4[7], d5[7]}, [r3] + + pop {r4-r5, pc} + ENDP ; |aom_lpf_vertical_8_neon| + +; void aom_mbloop_filter_neon(); +; This is a helper function for the loopfilters. The invidual functions do the +; necessary load, transpose (if necessary) and store. The function does not use +; registers d8-d15. +; +; Inputs: +; r0-r3, r12 PRESERVE +; d0 blimit +; d1 limit +; d2 thresh +; d3 p3 +; d4 p2 +; d5 p1 +; d6 p0 +; d7 q0 +; d16 q1 +; d17 q2 +; d18 q3 +; +; Outputs: +; d0 op2 +; d1 op1 +; d2 op0 +; d3 oq0 +; d4 oq1 +; d5 oq2 +|aom_mbloop_filter_neon| PROC + ; filter_mask + vabd.u8 d19, d3, d4 ; m1 = abs(p3 - p2) + vabd.u8 d20, d4, d5 ; m2 = abs(p2 - p1) + vabd.u8 d21, d5, d6 ; m3 = abs(p1 - p0) + vabd.u8 d22, d16, d7 ; m4 = abs(q1 - q0) + vabd.u8 d23, d17, d16 ; m5 = abs(q2 - q1) + vabd.u8 d24, d18, d17 ; m6 = abs(q3 - q2) + + ; only compare the largest value to limit + vmax.u8 d19, d19, d20 ; m1 = max(m1, m2) + vmax.u8 d20, d21, d22 ; m2 = max(m3, m4) + + vabd.u8 d25, d6, d4 ; m7 = abs(p0 - p2) + + vmax.u8 d23, d23, d24 ; m3 = max(m5, m6) + + vabd.u8 d26, d7, d17 ; m8 = abs(q0 - q2) + + vmax.u8 d19, d19, d20 + + vabd.u8 d24, d6, d7 ; m9 = abs(p0 - q0) + vabd.u8 d27, d3, d6 ; m10 = abs(p3 - p0) + vabd.u8 d28, d18, d7 ; m11 = abs(q3 - q0) + + vmax.u8 d19, d19, d23 + + vabd.u8 d23, d5, d16 ; a = abs(p1 - q1) + vqadd.u8 d24, d24, d24 ; b = abs(p0 - q0) * 2 + + ; abs () > limit + vcge.u8 d19, d1, d19 + + ; only compare the largest value to thresh + vmax.u8 d25, d25, d26 ; m4 = max(m7, m8) + vmax.u8 d26, d27, d28 ; m5 = max(m10, m11) + + vshr.u8 d23, d23, #1 ; a = a / 2 + + vmax.u8 d25, d25, d26 ; m4 = max(m4, m5) + + vqadd.u8 d24, d24, d23 ; a = b + a + + vmax.u8 d20, d20, d25 ; m2 = max(m2, m4) + + vmov.u8 d23, #1 + vcge.u8 d24, d0, d24 ; a > blimit + + vcgt.u8 d21, d21, d2 ; (abs(p1 - p0) > thresh)*-1 + + vcge.u8 d20, d23, d20 ; flat + + vand d19, d19, d24 ; mask + + vcgt.u8 d23, d22, d2 ; (abs(q1 - q0) > thresh)*-1 + + vand d20, d20, d19 ; flat & mask + + vmov.u8 d22, #0x80 + + vorr d23, d21, d23 ; hev + + ; This instruction will truncate the "flat & mask" masks down to 4 bits + ; each to fit into one 32 bit arm register. The values are stored in + ; q10.64[0]. + vshrn.u16 d30, q10, #4 + vmov.u32 r4, d30[0] ; flat & mask 4bits + + adds r5, r4, #1 ; Check for all 1's + + ; If mask and flat are 1's for all vectors, then we only need to execute + ; the power branch for all vectors. + beq power_branch_only + + cmp r4, #0 ; Check for 0, set flag for later + + ; mbfilter() function + ; filter() function + ; convert to signed + veor d21, d7, d22 ; qs0 + veor d24, d6, d22 ; ps0 + veor d25, d5, d22 ; ps1 + veor d26, d16, d22 ; qs1 + + vmov.u8 d27, #3 + + vsub.s8 d28, d21, d24 ; ( qs0 - ps0) + + vqsub.s8 d29, d25, d26 ; filter = clamp(ps1-qs1) + + vmull.s8 q15, d28, d27 ; 3 * ( qs0 - ps0) + + vand d29, d29, d23 ; filter &= hev + + vaddw.s8 q15, q15, d29 ; filter + 3 * (qs0 - ps0) + + vmov.u8 d29, #4 + + ; filter = clamp(filter + 3 * ( qs0 - ps0)) + vqmovn.s16 d28, q15 + + vand d28, d28, d19 ; filter &= mask + + vqadd.s8 d30, d28, d27 ; filter2 = clamp(filter+3) + vqadd.s8 d29, d28, d29 ; filter1 = clamp(filter+4) + vshr.s8 d30, d30, #3 ; filter2 >>= 3 + vshr.s8 d29, d29, #3 ; filter1 >>= 3 + + vqadd.s8 d24, d24, d30 ; op0 = clamp(ps0 + filter2) + vqsub.s8 d21, d21, d29 ; oq0 = clamp(qs0 - filter1) + + ; outer tap adjustments: ++filter1 >> 1 + vrshr.s8 d29, d29, #1 + vbic d29, d29, d23 ; filter &= ~hev + + vqadd.s8 d25, d25, d29 ; op1 = clamp(ps1 + filter) + vqsub.s8 d26, d26, d29 ; oq1 = clamp(qs1 - filter) + + ; If mask and flat are 0's for all vectors, then we only need to execute + ; the filter branch for all vectors. + beq filter_branch_only + + ; If mask and flat are mixed then we must perform both branches and + ; combine the data. + veor d24, d24, d22 ; *f_op0 = u^0x80 + veor d21, d21, d22 ; *f_oq0 = u^0x80 + veor d25, d25, d22 ; *f_op1 = u^0x80 + veor d26, d26, d22 ; *f_oq1 = u^0x80 + + ; At this point we have already executed the filter branch. The filter + ; branch does not set op2 or oq2, so use p2 and q2. Execute the power + ; branch and combine the data. + vmov.u8 d23, #2 + vaddl.u8 q14, d6, d7 ; r_op2 = p0 + q0 + vmlal.u8 q14, d3, d27 ; r_op2 += p3 * 3 + vmlal.u8 q14, d4, d23 ; r_op2 += p2 * 2 + + vbif d0, d4, d20 ; op2 |= p2 & ~(flat & mask) + + vaddw.u8 q14, d5 ; r_op2 += p1 + + vbif d1, d25, d20 ; op1 |= f_op1 & ~(flat & mask) + + vqrshrn.u16 d30, q14, #3 ; r_op2 + + vsubw.u8 q14, d3 ; r_op1 = r_op2 - p3 + vsubw.u8 q14, d4 ; r_op1 -= p2 + vaddw.u8 q14, d5 ; r_op1 += p1 + vaddw.u8 q14, d16 ; r_op1 += q1 + + vbif d2, d24, d20 ; op0 |= f_op0 & ~(flat & mask) + + vqrshrn.u16 d31, q14, #3 ; r_op1 + + vsubw.u8 q14, d3 ; r_op0 = r_op1 - p3 + vsubw.u8 q14, d5 ; r_op0 -= p1 + vaddw.u8 q14, d6 ; r_op0 += p0 + vaddw.u8 q14, d17 ; r_op0 += q2 + + vbit d0, d30, d20 ; op2 |= r_op2 & (flat & mask) + + vqrshrn.u16 d23, q14, #3 ; r_op0 + + vsubw.u8 q14, d3 ; r_oq0 = r_op0 - p3 + vsubw.u8 q14, d6 ; r_oq0 -= p0 + vaddw.u8 q14, d7 ; r_oq0 += q0 + + vbit d1, d31, d20 ; op1 |= r_op1 & (flat & mask) + + vaddw.u8 q14, d18 ; oq0 += q3 + + vbit d2, d23, d20 ; op0 |= r_op0 & (flat & mask) + + vqrshrn.u16 d22, q14, #3 ; r_oq0 + + vsubw.u8 q14, d4 ; r_oq1 = r_oq0 - p2 + vsubw.u8 q14, d7 ; r_oq1 -= q0 + vaddw.u8 q14, d16 ; r_oq1 += q1 + + vbif d3, d21, d20 ; oq0 |= f_oq0 & ~(flat & mask) + + vaddw.u8 q14, d18 ; r_oq1 += q3 + + vbif d4, d26, d20 ; oq1 |= f_oq1 & ~(flat & mask) + + vqrshrn.u16 d6, q14, #3 ; r_oq1 + + vsubw.u8 q14, d5 ; r_oq2 = r_oq1 - p1 + vsubw.u8 q14, d16 ; r_oq2 -= q1 + vaddw.u8 q14, d17 ; r_oq2 += q2 + vaddw.u8 q14, d18 ; r_oq2 += q3 + + vbif d5, d17, d20 ; oq2 |= q2 & ~(flat & mask) + + vqrshrn.u16 d7, q14, #3 ; r_oq2 + + vbit d3, d22, d20 ; oq0 |= r_oq0 & (flat & mask) + vbit d4, d6, d20 ; oq1 |= r_oq1 & (flat & mask) + vbit d5, d7, d20 ; oq2 |= r_oq2 & (flat & mask) + + bx lr + +power_branch_only + vmov.u8 d27, #3 + vmov.u8 d21, #2 + vaddl.u8 q14, d6, d7 ; op2 = p0 + q0 + vmlal.u8 q14, d3, d27 ; op2 += p3 * 3 + vmlal.u8 q14, d4, d21 ; op2 += p2 * 2 + vaddw.u8 q14, d5 ; op2 += p1 + vqrshrn.u16 d0, q14, #3 ; op2 + + vsubw.u8 q14, d3 ; op1 = op2 - p3 + vsubw.u8 q14, d4 ; op1 -= p2 + vaddw.u8 q14, d5 ; op1 += p1 + vaddw.u8 q14, d16 ; op1 += q1 + vqrshrn.u16 d1, q14, #3 ; op1 + + vsubw.u8 q14, d3 ; op0 = op1 - p3 + vsubw.u8 q14, d5 ; op0 -= p1 + vaddw.u8 q14, d6 ; op0 += p0 + vaddw.u8 q14, d17 ; op0 += q2 + vqrshrn.u16 d2, q14, #3 ; op0 + + vsubw.u8 q14, d3 ; oq0 = op0 - p3 + vsubw.u8 q14, d6 ; oq0 -= p0 + vaddw.u8 q14, d7 ; oq0 += q0 + vaddw.u8 q14, d18 ; oq0 += q3 + vqrshrn.u16 d3, q14, #3 ; oq0 + + vsubw.u8 q14, d4 ; oq1 = oq0 - p2 + vsubw.u8 q14, d7 ; oq1 -= q0 + vaddw.u8 q14, d16 ; oq1 += q1 + vaddw.u8 q14, d18 ; oq1 += q3 + vqrshrn.u16 d4, q14, #3 ; oq1 + + vsubw.u8 q14, d5 ; oq2 = oq1 - p1 + vsubw.u8 q14, d16 ; oq2 -= q1 + vaddw.u8 q14, d17 ; oq2 += q2 + vaddw.u8 q14, d18 ; oq2 += q3 + vqrshrn.u16 d5, q14, #3 ; oq2 + + bx lr + +filter_branch_only + ; TODO(fgalligan): See if we can rearange registers so we do not need to + ; do the 2 vswp. + vswp d0, d4 ; op2 + vswp d5, d17 ; oq2 + veor d2, d24, d22 ; *op0 = u^0x80 + veor d3, d21, d22 ; *oq0 = u^0x80 + veor d1, d25, d22 ; *op1 = u^0x80 + veor d4, d26, d22 ; *oq1 = u^0x80 + + bx lr + + ENDP ; |aom_mbloop_filter_neon| + + END diff --git a/third_party/aom/aom_dsp/arm/loopfilter_8_neon.c b/third_party/aom/aom_dsp/arm/loopfilter_8_neon.c new file mode 100644 index 0000000000..c4502fdb5f --- /dev/null +++ b/third_party/aom/aom_dsp/arm/loopfilter_8_neon.c @@ -0,0 +1,430 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" + +static INLINE void mbloop_filter_neon(uint8x8_t dblimit, // mblimit + uint8x8_t dlimit, // limit + uint8x8_t dthresh, // thresh + uint8x8_t d3u8, // p2 + uint8x8_t d4u8, // p2 + uint8x8_t d5u8, // p1 + uint8x8_t d6u8, // p0 + uint8x8_t d7u8, // q0 + uint8x8_t d16u8, // q1 + uint8x8_t d17u8, // q2 + uint8x8_t d18u8, // q3 + uint8x8_t *d0ru8, // p1 + uint8x8_t *d1ru8, // p1 + uint8x8_t *d2ru8, // p0 + uint8x8_t *d3ru8, // q0 + uint8x8_t *d4ru8, // q1 + uint8x8_t *d5ru8) { // q1 + uint32_t flat; + uint8x8_t d0u8, d1u8, d2u8, d19u8, d20u8, d21u8, d22u8, d23u8, d24u8; + uint8x8_t d25u8, d26u8, d27u8, d28u8, d29u8, d30u8, d31u8; + int16x8_t q15s16; + uint16x8_t q10u16, q14u16; + int8x8_t d21s8, d24s8, d25s8, d26s8, d28s8, d29s8, d30s8; + + d19u8 = vabd_u8(d3u8, d4u8); + d20u8 = vabd_u8(d4u8, d5u8); + d21u8 = vabd_u8(d5u8, d6u8); + d22u8 = vabd_u8(d16u8, d7u8); + d23u8 = vabd_u8(d17u8, d16u8); + d24u8 = vabd_u8(d18u8, d17u8); + + d19u8 = vmax_u8(d19u8, d20u8); + d20u8 = vmax_u8(d21u8, d22u8); + + d25u8 = vabd_u8(d6u8, d4u8); + + d23u8 = vmax_u8(d23u8, d24u8); + + d26u8 = vabd_u8(d7u8, d17u8); + + d19u8 = vmax_u8(d19u8, d20u8); + + d24u8 = vabd_u8(d6u8, d7u8); + d27u8 = vabd_u8(d3u8, d6u8); + d28u8 = vabd_u8(d18u8, d7u8); + + d19u8 = vmax_u8(d19u8, d23u8); + + d23u8 = vabd_u8(d5u8, d16u8); + d24u8 = vqadd_u8(d24u8, d24u8); + + d19u8 = vcge_u8(dlimit, d19u8); + + d25u8 = vmax_u8(d25u8, d26u8); + d26u8 = vmax_u8(d27u8, d28u8); + + d23u8 = vshr_n_u8(d23u8, 1); + + d25u8 = vmax_u8(d25u8, d26u8); + + d24u8 = vqadd_u8(d24u8, d23u8); + + d20u8 = vmax_u8(d20u8, d25u8); + + d23u8 = vdup_n_u8(1); + d24u8 = vcge_u8(dblimit, d24u8); + + d21u8 = vcgt_u8(d21u8, dthresh); + + d20u8 = vcge_u8(d23u8, d20u8); + + d19u8 = vand_u8(d19u8, d24u8); + + d23u8 = vcgt_u8(d22u8, dthresh); + + d20u8 = vand_u8(d20u8, d19u8); + + d22u8 = vdup_n_u8(0x80); + + d23u8 = vorr_u8(d21u8, d23u8); + + q10u16 = vcombine_u16(vreinterpret_u16_u8(d20u8), vreinterpret_u16_u8(d21u8)); + + d30u8 = vshrn_n_u16(q10u16, 4); + flat = vget_lane_u32(vreinterpret_u32_u8(d30u8), 0); + + if (flat == 0xffffffff) { // Check for all 1's, power_branch_only + d27u8 = vdup_n_u8(3); + d21u8 = vdup_n_u8(2); + q14u16 = vaddl_u8(d6u8, d7u8); + q14u16 = vmlal_u8(q14u16, d3u8, d27u8); + q14u16 = vmlal_u8(q14u16, d4u8, d21u8); + q14u16 = vaddw_u8(q14u16, d5u8); + *d0ru8 = vqrshrn_n_u16(q14u16, 3); + + q14u16 = vsubw_u8(q14u16, d3u8); + q14u16 = vsubw_u8(q14u16, d4u8); + q14u16 = vaddw_u8(q14u16, d5u8); + q14u16 = vaddw_u8(q14u16, d16u8); + *d1ru8 = vqrshrn_n_u16(q14u16, 3); + + q14u16 = vsubw_u8(q14u16, d3u8); + q14u16 = vsubw_u8(q14u16, d5u8); + q14u16 = vaddw_u8(q14u16, d6u8); + q14u16 = vaddw_u8(q14u16, d17u8); + *d2ru8 = vqrshrn_n_u16(q14u16, 3); + + q14u16 = vsubw_u8(q14u16, d3u8); + q14u16 = vsubw_u8(q14u16, d6u8); + q14u16 = vaddw_u8(q14u16, d7u8); + q14u16 = vaddw_u8(q14u16, d18u8); + *d3ru8 = vqrshrn_n_u16(q14u16, 3); + + q14u16 = vsubw_u8(q14u16, d4u8); + q14u16 = vsubw_u8(q14u16, d7u8); + q14u16 = vaddw_u8(q14u16, d16u8); + q14u16 = vaddw_u8(q14u16, d18u8); + *d4ru8 = vqrshrn_n_u16(q14u16, 3); + + q14u16 = vsubw_u8(q14u16, d5u8); + q14u16 = vsubw_u8(q14u16, d16u8); + q14u16 = vaddw_u8(q14u16, d17u8); + q14u16 = vaddw_u8(q14u16, d18u8); + *d5ru8 = vqrshrn_n_u16(q14u16, 3); + } else { + d21u8 = veor_u8(d7u8, d22u8); + d24u8 = veor_u8(d6u8, d22u8); + d25u8 = veor_u8(d5u8, d22u8); + d26u8 = veor_u8(d16u8, d22u8); + + d27u8 = vdup_n_u8(3); + + d28s8 = vsub_s8(vreinterpret_s8_u8(d21u8), vreinterpret_s8_u8(d24u8)); + d29s8 = vqsub_s8(vreinterpret_s8_u8(d25u8), vreinterpret_s8_u8(d26u8)); + + q15s16 = vmull_s8(d28s8, vreinterpret_s8_u8(d27u8)); + + d29s8 = vand_s8(d29s8, vreinterpret_s8_u8(d23u8)); + + q15s16 = vaddw_s8(q15s16, d29s8); + + d29u8 = vdup_n_u8(4); + + d28s8 = vqmovn_s16(q15s16); + + d28s8 = vand_s8(d28s8, vreinterpret_s8_u8(d19u8)); + + d30s8 = vqadd_s8(d28s8, vreinterpret_s8_u8(d27u8)); + d29s8 = vqadd_s8(d28s8, vreinterpret_s8_u8(d29u8)); + d30s8 = vshr_n_s8(d30s8, 3); + d29s8 = vshr_n_s8(d29s8, 3); + + d24s8 = vqadd_s8(vreinterpret_s8_u8(d24u8), d30s8); + d21s8 = vqsub_s8(vreinterpret_s8_u8(d21u8), d29s8); + + d29s8 = vrshr_n_s8(d29s8, 1); + d29s8 = vbic_s8(d29s8, vreinterpret_s8_u8(d23u8)); + + d25s8 = vqadd_s8(vreinterpret_s8_u8(d25u8), d29s8); + d26s8 = vqsub_s8(vreinterpret_s8_u8(d26u8), d29s8); + + if (flat == 0) { // filter_branch_only + *d0ru8 = d4u8; + *d1ru8 = veor_u8(vreinterpret_u8_s8(d25s8), d22u8); + *d2ru8 = veor_u8(vreinterpret_u8_s8(d24s8), d22u8); + *d3ru8 = veor_u8(vreinterpret_u8_s8(d21s8), d22u8); + *d4ru8 = veor_u8(vreinterpret_u8_s8(d26s8), d22u8); + *d5ru8 = d17u8; + return; + } + + d21u8 = veor_u8(vreinterpret_u8_s8(d21s8), d22u8); + d24u8 = veor_u8(vreinterpret_u8_s8(d24s8), d22u8); + d25u8 = veor_u8(vreinterpret_u8_s8(d25s8), d22u8); + d26u8 = veor_u8(vreinterpret_u8_s8(d26s8), d22u8); + + d23u8 = vdup_n_u8(2); + q14u16 = vaddl_u8(d6u8, d7u8); + q14u16 = vmlal_u8(q14u16, d3u8, d27u8); + q14u16 = vmlal_u8(q14u16, d4u8, d23u8); + + d0u8 = vbsl_u8(d20u8, dblimit, d4u8); + + q14u16 = vaddw_u8(q14u16, d5u8); + + d1u8 = vbsl_u8(d20u8, dlimit, d25u8); + + d30u8 = vqrshrn_n_u16(q14u16, 3); + + q14u16 = vsubw_u8(q14u16, d3u8); + q14u16 = vsubw_u8(q14u16, d4u8); + q14u16 = vaddw_u8(q14u16, d5u8); + q14u16 = vaddw_u8(q14u16, d16u8); + + d2u8 = vbsl_u8(d20u8, dthresh, d24u8); + + d31u8 = vqrshrn_n_u16(q14u16, 3); + + q14u16 = vsubw_u8(q14u16, d3u8); + q14u16 = vsubw_u8(q14u16, d5u8); + q14u16 = vaddw_u8(q14u16, d6u8); + q14u16 = vaddw_u8(q14u16, d17u8); + + *d0ru8 = vbsl_u8(d20u8, d30u8, d0u8); + + d23u8 = vqrshrn_n_u16(q14u16, 3); + + q14u16 = vsubw_u8(q14u16, d3u8); + q14u16 = vsubw_u8(q14u16, d6u8); + q14u16 = vaddw_u8(q14u16, d7u8); + + *d1ru8 = vbsl_u8(d20u8, d31u8, d1u8); + + q14u16 = vaddw_u8(q14u16, d18u8); + + *d2ru8 = vbsl_u8(d20u8, d23u8, d2u8); + + d22u8 = vqrshrn_n_u16(q14u16, 3); + + q14u16 = vsubw_u8(q14u16, d4u8); + q14u16 = vsubw_u8(q14u16, d7u8); + q14u16 = vaddw_u8(q14u16, d16u8); + + d3u8 = vbsl_u8(d20u8, d3u8, d21u8); + + q14u16 = vaddw_u8(q14u16, d18u8); + + d4u8 = vbsl_u8(d20u8, d4u8, d26u8); + + d6u8 = vqrshrn_n_u16(q14u16, 3); + + q14u16 = vsubw_u8(q14u16, d5u8); + q14u16 = vsubw_u8(q14u16, d16u8); + q14u16 = vaddw_u8(q14u16, d17u8); + q14u16 = vaddw_u8(q14u16, d18u8); + + d5u8 = vbsl_u8(d20u8, d5u8, d17u8); + + d7u8 = vqrshrn_n_u16(q14u16, 3); + + *d3ru8 = vbsl_u8(d20u8, d22u8, d3u8); + *d4ru8 = vbsl_u8(d20u8, d6u8, d4u8); + *d5ru8 = vbsl_u8(d20u8, d7u8, d5u8); + } + return; +} + +void aom_lpf_horizontal_8_neon(uint8_t *src, int pitch, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh) { + int i; + uint8_t *s, *psrc; + uint8x8_t dblimit, dlimit, dthresh; + uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8; + uint8x8_t d16u8, d17u8, d18u8; + + dblimit = vld1_u8(blimit); + dlimit = vld1_u8(limit); + dthresh = vld1_u8(thresh); + + psrc = src - (pitch << 2); + for (i = 0; i < 1; i++) { + s = psrc + i * 8; + + d3u8 = vld1_u8(s); + s += pitch; + d4u8 = vld1_u8(s); + s += pitch; + d5u8 = vld1_u8(s); + s += pitch; + d6u8 = vld1_u8(s); + s += pitch; + d7u8 = vld1_u8(s); + s += pitch; + d16u8 = vld1_u8(s); + s += pitch; + d17u8 = vld1_u8(s); + s += pitch; + d18u8 = vld1_u8(s); + + mbloop_filter_neon(dblimit, dlimit, dthresh, d3u8, d4u8, d5u8, d6u8, d7u8, + d16u8, d17u8, d18u8, &d0u8, &d1u8, &d2u8, &d3u8, &d4u8, + &d5u8); + + s -= (pitch * 6); + vst1_u8(s, d0u8); + s += pitch; + vst1_u8(s, d1u8); + s += pitch; + vst1_u8(s, d2u8); + s += pitch; + vst1_u8(s, d3u8); + s += pitch; + vst1_u8(s, d4u8); + s += pitch; + vst1_u8(s, d5u8); + } + return; +} + +void aom_lpf_vertical_8_neon(uint8_t *src, int pitch, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh) { + int i; + uint8_t *s; + uint8x8_t dblimit, dlimit, dthresh; + uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8; + uint8x8_t d16u8, d17u8, d18u8; + uint32x2x2_t d2tmp0, d2tmp1, d2tmp2, d2tmp3; + uint16x4x2_t d2tmp4, d2tmp5, d2tmp6, d2tmp7; + uint8x8x2_t d2tmp8, d2tmp9, d2tmp10, d2tmp11; + uint8x8x4_t d4Result; + uint8x8x2_t d2Result; + + dblimit = vld1_u8(blimit); + dlimit = vld1_u8(limit); + dthresh = vld1_u8(thresh); + + for (i = 0; i < 1; i++) { + s = src + (i * (pitch << 3)) - 4; + + d3u8 = vld1_u8(s); + s += pitch; + d4u8 = vld1_u8(s); + s += pitch; + d5u8 = vld1_u8(s); + s += pitch; + d6u8 = vld1_u8(s); + s += pitch; + d7u8 = vld1_u8(s); + s += pitch; + d16u8 = vld1_u8(s); + s += pitch; + d17u8 = vld1_u8(s); + s += pitch; + d18u8 = vld1_u8(s); + + d2tmp0 = vtrn_u32(vreinterpret_u32_u8(d3u8), vreinterpret_u32_u8(d7u8)); + d2tmp1 = vtrn_u32(vreinterpret_u32_u8(d4u8), vreinterpret_u32_u8(d16u8)); + d2tmp2 = vtrn_u32(vreinterpret_u32_u8(d5u8), vreinterpret_u32_u8(d17u8)); + d2tmp3 = vtrn_u32(vreinterpret_u32_u8(d6u8), vreinterpret_u32_u8(d18u8)); + + d2tmp4 = vtrn_u16(vreinterpret_u16_u32(d2tmp0.val[0]), + vreinterpret_u16_u32(d2tmp2.val[0])); + d2tmp5 = vtrn_u16(vreinterpret_u16_u32(d2tmp1.val[0]), + vreinterpret_u16_u32(d2tmp3.val[0])); + d2tmp6 = vtrn_u16(vreinterpret_u16_u32(d2tmp0.val[1]), + vreinterpret_u16_u32(d2tmp2.val[1])); + d2tmp7 = vtrn_u16(vreinterpret_u16_u32(d2tmp1.val[1]), + vreinterpret_u16_u32(d2tmp3.val[1])); + + d2tmp8 = vtrn_u8(vreinterpret_u8_u16(d2tmp4.val[0]), + vreinterpret_u8_u16(d2tmp5.val[0])); + d2tmp9 = vtrn_u8(vreinterpret_u8_u16(d2tmp4.val[1]), + vreinterpret_u8_u16(d2tmp5.val[1])); + d2tmp10 = vtrn_u8(vreinterpret_u8_u16(d2tmp6.val[0]), + vreinterpret_u8_u16(d2tmp7.val[0])); + d2tmp11 = vtrn_u8(vreinterpret_u8_u16(d2tmp6.val[1]), + vreinterpret_u8_u16(d2tmp7.val[1])); + + d3u8 = d2tmp8.val[0]; + d4u8 = d2tmp8.val[1]; + d5u8 = d2tmp9.val[0]; + d6u8 = d2tmp9.val[1]; + d7u8 = d2tmp10.val[0]; + d16u8 = d2tmp10.val[1]; + d17u8 = d2tmp11.val[0]; + d18u8 = d2tmp11.val[1]; + + mbloop_filter_neon(dblimit, dlimit, dthresh, d3u8, d4u8, d5u8, d6u8, d7u8, + d16u8, d17u8, d18u8, &d0u8, &d1u8, &d2u8, &d3u8, &d4u8, + &d5u8); + + d4Result.val[0] = d0u8; + d4Result.val[1] = d1u8; + d4Result.val[2] = d2u8; + d4Result.val[3] = d3u8; + + d2Result.val[0] = d4u8; + d2Result.val[1] = d5u8; + + s = src - 3; + vst4_lane_u8(s, d4Result, 0); + s += pitch; + vst4_lane_u8(s, d4Result, 1); + s += pitch; + vst4_lane_u8(s, d4Result, 2); + s += pitch; + vst4_lane_u8(s, d4Result, 3); + s += pitch; + vst4_lane_u8(s, d4Result, 4); + s += pitch; + vst4_lane_u8(s, d4Result, 5); + s += pitch; + vst4_lane_u8(s, d4Result, 6); + s += pitch; + vst4_lane_u8(s, d4Result, 7); + + s = src + 1; + vst2_lane_u8(s, d2Result, 0); + s += pitch; + vst2_lane_u8(s, d2Result, 1); + s += pitch; + vst2_lane_u8(s, d2Result, 2); + s += pitch; + vst2_lane_u8(s, d2Result, 3); + s += pitch; + vst2_lane_u8(s, d2Result, 4); + s += pitch; + vst2_lane_u8(s, d2Result, 5); + s += pitch; + vst2_lane_u8(s, d2Result, 6); + s += pitch; + vst2_lane_u8(s, d2Result, 7); + } + return; +} diff --git a/third_party/aom/aom_dsp/arm/loopfilter_mb_neon.asm b/third_party/aom/aom_dsp/arm/loopfilter_mb_neon.asm new file mode 100644 index 0000000000..675928860f --- /dev/null +++ b/third_party/aom/aom_dsp/arm/loopfilter_mb_neon.asm @@ -0,0 +1,638 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + EXPORT |aom_lpf_horizontal_edge_8_neon| + EXPORT |aom_lpf_horizontal_edge_16_neon| + EXPORT |aom_lpf_vertical_16_neon| + ARM + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +; void mb_lpf_horizontal_edge(uint8_t *s, int p, +; const uint8_t *blimit, +; const uint8_t *limit, +; const uint8_t *thresh, +; int count) +; r0 uint8_t *s, +; r1 int p, /* pitch */ +; r2 const uint8_t *blimit, +; r3 const uint8_t *limit, +; sp const uint8_t *thresh, +; r12 int count +|mb_lpf_horizontal_edge| PROC + push {r4-r8, lr} + vpush {d8-d15} + ldr r4, [sp, #88] ; load thresh + +h_count + vld1.8 {d16[]}, [r2] ; load *blimit + vld1.8 {d17[]}, [r3] ; load *limit + vld1.8 {d18[]}, [r4] ; load *thresh + + sub r8, r0, r1, lsl #3 ; move src pointer down by 8 lines + + vld1.u8 {d0}, [r8@64], r1 ; p7 + vld1.u8 {d1}, [r8@64], r1 ; p6 + vld1.u8 {d2}, [r8@64], r1 ; p5 + vld1.u8 {d3}, [r8@64], r1 ; p4 + vld1.u8 {d4}, [r8@64], r1 ; p3 + vld1.u8 {d5}, [r8@64], r1 ; p2 + vld1.u8 {d6}, [r8@64], r1 ; p1 + vld1.u8 {d7}, [r8@64], r1 ; p0 + vld1.u8 {d8}, [r8@64], r1 ; q0 + vld1.u8 {d9}, [r8@64], r1 ; q1 + vld1.u8 {d10}, [r8@64], r1 ; q2 + vld1.u8 {d11}, [r8@64], r1 ; q3 + vld1.u8 {d12}, [r8@64], r1 ; q4 + vld1.u8 {d13}, [r8@64], r1 ; q5 + vld1.u8 {d14}, [r8@64], r1 ; q6 + vld1.u8 {d15}, [r8@64], r1 ; q7 + + bl aom_wide_mbfilter_neon + + tst r7, #1 + beq h_mbfilter + + ; flat && mask were not set for any of the channels. Just store the values + ; from filter. + sub r8, r0, r1, lsl #1 + + vst1.u8 {d25}, [r8@64], r1 ; store op1 + vst1.u8 {d24}, [r8@64], r1 ; store op0 + vst1.u8 {d23}, [r8@64], r1 ; store oq0 + vst1.u8 {d26}, [r8@64], r1 ; store oq1 + + b h_next + +h_mbfilter + tst r7, #2 + beq h_wide_mbfilter + + ; flat2 was not set for any of the channels. Just store the values from + ; mbfilter. + sub r8, r0, r1, lsl #1 + sub r8, r8, r1 + + vst1.u8 {d18}, [r8@64], r1 ; store op2 + vst1.u8 {d19}, [r8@64], r1 ; store op1 + vst1.u8 {d20}, [r8@64], r1 ; store op0 + vst1.u8 {d21}, [r8@64], r1 ; store oq0 + vst1.u8 {d22}, [r8@64], r1 ; store oq1 + vst1.u8 {d23}, [r8@64], r1 ; store oq2 + + b h_next + +h_wide_mbfilter + sub r8, r0, r1, lsl #3 + add r8, r8, r1 + + vst1.u8 {d16}, [r8@64], r1 ; store op6 + vst1.u8 {d24}, [r8@64], r1 ; store op5 + vst1.u8 {d25}, [r8@64], r1 ; store op4 + vst1.u8 {d26}, [r8@64], r1 ; store op3 + vst1.u8 {d27}, [r8@64], r1 ; store op2 + vst1.u8 {d18}, [r8@64], r1 ; store op1 + vst1.u8 {d19}, [r8@64], r1 ; store op0 + vst1.u8 {d20}, [r8@64], r1 ; store oq0 + vst1.u8 {d21}, [r8@64], r1 ; store oq1 + vst1.u8 {d22}, [r8@64], r1 ; store oq2 + vst1.u8 {d23}, [r8@64], r1 ; store oq3 + vst1.u8 {d1}, [r8@64], r1 ; store oq4 + vst1.u8 {d2}, [r8@64], r1 ; store oq5 + vst1.u8 {d3}, [r8@64], r1 ; store oq6 + +h_next + add r0, r0, #8 + subs r12, r12, #1 + bne h_count + + vpop {d8-d15} + pop {r4-r8, pc} + + ENDP ; |mb_lpf_horizontal_edge| + +; void aom_lpf_horizontal_edge_8_neon(uint8_t *s, int pitch, +; const uint8_t *blimit, +; const uint8_t *limit, +; const uint8_t *thresh) +; r0 uint8_t *s, +; r1 int pitch, +; r2 const uint8_t *blimit, +; r3 const uint8_t *limit, +; sp const uint8_t *thresh +|aom_lpf_horizontal_edge_8_neon| PROC + mov r12, #1 + b mb_lpf_horizontal_edge + ENDP ; |aom_lpf_horizontal_edge_8_neon| + +; void aom_lpf_horizontal_edge_16_neon(uint8_t *s, int pitch, +; const uint8_t *blimit, +; const uint8_t *limit, +; const uint8_t *thresh) +; r0 uint8_t *s, +; r1 int pitch, +; r2 const uint8_t *blimit, +; r3 const uint8_t *limit, +; sp const uint8_t *thresh +|aom_lpf_horizontal_edge_16_neon| PROC + mov r12, #2 + b mb_lpf_horizontal_edge + ENDP ; |aom_lpf_horizontal_edge_16_neon| + +; void aom_lpf_vertical_16_neon(uint8_t *s, int p, +; const uint8_t *blimit, +; const uint8_t *limit, +; const uint8_t *thresh) +; r0 uint8_t *s, +; r1 int p, /* pitch */ +; r2 const uint8_t *blimit, +; r3 const uint8_t *limit, +; sp const uint8_t *thresh, +|aom_lpf_vertical_16_neon| PROC + push {r4-r8, lr} + vpush {d8-d15} + ldr r4, [sp, #88] ; load thresh + + vld1.8 {d16[]}, [r2] ; load *blimit + vld1.8 {d17[]}, [r3] ; load *limit + vld1.8 {d18[]}, [r4] ; load *thresh + + sub r8, r0, #8 + + vld1.8 {d0}, [r8@64], r1 + vld1.8 {d8}, [r0@64], r1 + vld1.8 {d1}, [r8@64], r1 + vld1.8 {d9}, [r0@64], r1 + vld1.8 {d2}, [r8@64], r1 + vld1.8 {d10}, [r0@64], r1 + vld1.8 {d3}, [r8@64], r1 + vld1.8 {d11}, [r0@64], r1 + vld1.8 {d4}, [r8@64], r1 + vld1.8 {d12}, [r0@64], r1 + vld1.8 {d5}, [r8@64], r1 + vld1.8 {d13}, [r0@64], r1 + vld1.8 {d6}, [r8@64], r1 + vld1.8 {d14}, [r0@64], r1 + vld1.8 {d7}, [r8@64], r1 + vld1.8 {d15}, [r0@64], r1 + + sub r0, r0, r1, lsl #3 + + vtrn.32 q0, q2 + vtrn.32 q1, q3 + vtrn.32 q4, q6 + vtrn.32 q5, q7 + + vtrn.16 q0, q1 + vtrn.16 q2, q3 + vtrn.16 q4, q5 + vtrn.16 q6, q7 + + vtrn.8 d0, d1 + vtrn.8 d2, d3 + vtrn.8 d4, d5 + vtrn.8 d6, d7 + + vtrn.8 d8, d9 + vtrn.8 d10, d11 + vtrn.8 d12, d13 + vtrn.8 d14, d15 + + bl aom_wide_mbfilter_neon + + tst r7, #1 + beq v_mbfilter + + ; flat && mask were not set for any of the channels. Just store the values + ; from filter. + sub r8, r0, #2 + + vswp d23, d25 + + vst4.8 {d23[0], d24[0], d25[0], d26[0]}, [r8], r1 + vst4.8 {d23[1], d24[1], d25[1], d26[1]}, [r8], r1 + vst4.8 {d23[2], d24[2], d25[2], d26[2]}, [r8], r1 + vst4.8 {d23[3], d24[3], d25[3], d26[3]}, [r8], r1 + vst4.8 {d23[4], d24[4], d25[4], d26[4]}, [r8], r1 + vst4.8 {d23[5], d24[5], d25[5], d26[5]}, [r8], r1 + vst4.8 {d23[6], d24[6], d25[6], d26[6]}, [r8], r1 + vst4.8 {d23[7], d24[7], d25[7], d26[7]}, [r8], r1 + + b v_end + +v_mbfilter + tst r7, #2 + beq v_wide_mbfilter + + ; flat2 was not set for any of the channels. Just store the values from + ; mbfilter. + sub r8, r0, #3 + + vst3.8 {d18[0], d19[0], d20[0]}, [r8], r1 + vst3.8 {d21[0], d22[0], d23[0]}, [r0], r1 + vst3.8 {d18[1], d19[1], d20[1]}, [r8], r1 + vst3.8 {d21[1], d22[1], d23[1]}, [r0], r1 + vst3.8 {d18[2], d19[2], d20[2]}, [r8], r1 + vst3.8 {d21[2], d22[2], d23[2]}, [r0], r1 + vst3.8 {d18[3], d19[3], d20[3]}, [r8], r1 + vst3.8 {d21[3], d22[3], d23[3]}, [r0], r1 + vst3.8 {d18[4], d19[4], d20[4]}, [r8], r1 + vst3.8 {d21[4], d22[4], d23[4]}, [r0], r1 + vst3.8 {d18[5], d19[5], d20[5]}, [r8], r1 + vst3.8 {d21[5], d22[5], d23[5]}, [r0], r1 + vst3.8 {d18[6], d19[6], d20[6]}, [r8], r1 + vst3.8 {d21[6], d22[6], d23[6]}, [r0], r1 + vst3.8 {d18[7], d19[7], d20[7]}, [r8], r1 + vst3.8 {d21[7], d22[7], d23[7]}, [r0], r1 + + b v_end + +v_wide_mbfilter + sub r8, r0, #8 + + vtrn.32 d0, d26 + vtrn.32 d16, d27 + vtrn.32 d24, d18 + vtrn.32 d25, d19 + + vtrn.16 d0, d24 + vtrn.16 d16, d25 + vtrn.16 d26, d18 + vtrn.16 d27, d19 + + vtrn.8 d0, d16 + vtrn.8 d24, d25 + vtrn.8 d26, d27 + vtrn.8 d18, d19 + + vtrn.32 d20, d1 + vtrn.32 d21, d2 + vtrn.32 d22, d3 + vtrn.32 d23, d15 + + vtrn.16 d20, d22 + vtrn.16 d21, d23 + vtrn.16 d1, d3 + vtrn.16 d2, d15 + + vtrn.8 d20, d21 + vtrn.8 d22, d23 + vtrn.8 d1, d2 + vtrn.8 d3, d15 + + vst1.8 {d0}, [r8@64], r1 + vst1.8 {d20}, [r0@64], r1 + vst1.8 {d16}, [r8@64], r1 + vst1.8 {d21}, [r0@64], r1 + vst1.8 {d24}, [r8@64], r1 + vst1.8 {d22}, [r0@64], r1 + vst1.8 {d25}, [r8@64], r1 + vst1.8 {d23}, [r0@64], r1 + vst1.8 {d26}, [r8@64], r1 + vst1.8 {d1}, [r0@64], r1 + vst1.8 {d27}, [r8@64], r1 + vst1.8 {d2}, [r0@64], r1 + vst1.8 {d18}, [r8@64], r1 + vst1.8 {d3}, [r0@64], r1 + vst1.8 {d19}, [r8@64], r1 + vst1.8 {d15}, [r0@64], r1 + +v_end + vpop {d8-d15} + pop {r4-r8, pc} + + ENDP ; |aom_lpf_vertical_16_neon| + +; void aom_wide_mbfilter_neon(); +; This is a helper function for the loopfilters. The invidual functions do the +; necessary load, transpose (if necessary) and store. +; +; r0-r3 PRESERVE +; d16 blimit +; d17 limit +; d18 thresh +; d0 p7 +; d1 p6 +; d2 p5 +; d3 p4 +; d4 p3 +; d5 p2 +; d6 p1 +; d7 p0 +; d8 q0 +; d9 q1 +; d10 q2 +; d11 q3 +; d12 q4 +; d13 q5 +; d14 q6 +; d15 q7 +|aom_wide_mbfilter_neon| PROC + mov r7, #0 + + ; filter_mask + vabd.u8 d19, d4, d5 ; abs(p3 - p2) + vabd.u8 d20, d5, d6 ; abs(p2 - p1) + vabd.u8 d21, d6, d7 ; abs(p1 - p0) + vabd.u8 d22, d9, d8 ; abs(q1 - q0) + vabd.u8 d23, d10, d9 ; abs(q2 - q1) + vabd.u8 d24, d11, d10 ; abs(q3 - q2) + + ; only compare the largest value to limit + vmax.u8 d19, d19, d20 ; max(abs(p3 - p2), abs(p2 - p1)) + vmax.u8 d20, d21, d22 ; max(abs(p1 - p0), abs(q1 - q0)) + vmax.u8 d23, d23, d24 ; max(abs(q2 - q1), abs(q3 - q2)) + vmax.u8 d19, d19, d20 + + vabd.u8 d24, d7, d8 ; abs(p0 - q0) + + vmax.u8 d19, d19, d23 + + vabd.u8 d23, d6, d9 ; a = abs(p1 - q1) + vqadd.u8 d24, d24, d24 ; b = abs(p0 - q0) * 2 + + ; abs () > limit + vcge.u8 d19, d17, d19 + + ; flatmask4 + vabd.u8 d25, d7, d5 ; abs(p0 - p2) + vabd.u8 d26, d8, d10 ; abs(q0 - q2) + vabd.u8 d27, d4, d7 ; abs(p3 - p0) + vabd.u8 d28, d11, d8 ; abs(q3 - q0) + + ; only compare the largest value to thresh + vmax.u8 d25, d25, d26 ; max(abs(p0 - p2), abs(q0 - q2)) + vmax.u8 d26, d27, d28 ; max(abs(p3 - p0), abs(q3 - q0)) + vmax.u8 d25, d25, d26 + vmax.u8 d20, d20, d25 + + vshr.u8 d23, d23, #1 ; a = a / 2 + vqadd.u8 d24, d24, d23 ; a = b + a + + vmov.u8 d30, #1 + vcge.u8 d24, d16, d24 ; (a > blimit * 2 + limit) * -1 + + vcge.u8 d20, d30, d20 ; flat + + vand d19, d19, d24 ; mask + + ; hevmask + vcgt.u8 d21, d21, d18 ; (abs(p1 - p0) > thresh)*-1 + vcgt.u8 d22, d22, d18 ; (abs(q1 - q0) > thresh)*-1 + vorr d21, d21, d22 ; hev + + vand d16, d20, d19 ; flat && mask + vmov r5, r6, d16 + + ; flatmask5(1, p7, p6, p5, p4, p0, q0, q4, q5, q6, q7) + vabd.u8 d22, d3, d7 ; abs(p4 - p0) + vabd.u8 d23, d12, d8 ; abs(q4 - q0) + vabd.u8 d24, d7, d2 ; abs(p0 - p5) + vabd.u8 d25, d8, d13 ; abs(q0 - q5) + vabd.u8 d26, d1, d7 ; abs(p6 - p0) + vabd.u8 d27, d14, d8 ; abs(q6 - q0) + vabd.u8 d28, d0, d7 ; abs(p7 - p0) + vabd.u8 d29, d15, d8 ; abs(q7 - q0) + + ; only compare the largest value to thresh + vmax.u8 d22, d22, d23 ; max(abs(p4 - p0), abs(q4 - q0)) + vmax.u8 d23, d24, d25 ; max(abs(p0 - p5), abs(q0 - q5)) + vmax.u8 d24, d26, d27 ; max(abs(p6 - p0), abs(q6 - q0)) + vmax.u8 d25, d28, d29 ; max(abs(p7 - p0), abs(q7 - q0)) + + vmax.u8 d26, d22, d23 + vmax.u8 d27, d24, d25 + vmax.u8 d23, d26, d27 + + vcge.u8 d18, d30, d23 ; flat2 + + vmov.u8 d22, #0x80 + + orrs r5, r5, r6 ; Check for 0 + orreq r7, r7, #1 ; Only do filter branch + + vand d17, d18, d16 ; flat2 && flat && mask + vmov r5, r6, d17 + + ; mbfilter() function + + ; filter() function + ; convert to signed + veor d23, d8, d22 ; qs0 + veor d24, d7, d22 ; ps0 + veor d25, d6, d22 ; ps1 + veor d26, d9, d22 ; qs1 + + vmov.u8 d27, #3 + + vsub.s8 d28, d23, d24 ; ( qs0 - ps0) + vqsub.s8 d29, d25, d26 ; filter = clamp(ps1-qs1) + vmull.s8 q15, d28, d27 ; 3 * ( qs0 - ps0) + vand d29, d29, d21 ; filter &= hev + vaddw.s8 q15, q15, d29 ; filter + 3 * (qs0 - ps0) + vmov.u8 d29, #4 + + ; filter = clamp(filter + 3 * ( qs0 - ps0)) + vqmovn.s16 d28, q15 + + vand d28, d28, d19 ; filter &= mask + + vqadd.s8 d30, d28, d27 ; filter2 = clamp(filter+3) + vqadd.s8 d29, d28, d29 ; filter1 = clamp(filter+4) + vshr.s8 d30, d30, #3 ; filter2 >>= 3 + vshr.s8 d29, d29, #3 ; filter1 >>= 3 + + + vqadd.s8 d24, d24, d30 ; op0 = clamp(ps0 + filter2) + vqsub.s8 d23, d23, d29 ; oq0 = clamp(qs0 - filter1) + + ; outer tap adjustments: ++filter1 >> 1 + vrshr.s8 d29, d29, #1 + vbic d29, d29, d21 ; filter &= ~hev + + vqadd.s8 d25, d25, d29 ; op1 = clamp(ps1 + filter) + vqsub.s8 d26, d26, d29 ; oq1 = clamp(qs1 - filter) + + veor d24, d24, d22 ; *f_op0 = u^0x80 + veor d23, d23, d22 ; *f_oq0 = u^0x80 + veor d25, d25, d22 ; *f_op1 = u^0x80 + veor d26, d26, d22 ; *f_oq1 = u^0x80 + + tst r7, #1 + bxne lr + + orrs r5, r5, r6 ; Check for 0 + orreq r7, r7, #2 ; Only do mbfilter branch + + ; mbfilter flat && mask branch + ; TODO(fgalligan): Can I decrease the cycles shifting to consective d's + ; and using vibt on the q's? + vmov.u8 d29, #2 + vaddl.u8 q15, d7, d8 ; op2 = p0 + q0 + vmlal.u8 q15, d4, d27 ; op2 = p0 + q0 + p3 * 3 + vmlal.u8 q15, d5, d29 ; op2 = p0 + q0 + p3 * 3 + p2 * 2 + vaddl.u8 q10, d4, d5 + vaddw.u8 q15, d6 ; op2=p1 + p0 + q0 + p3 * 3 + p2 *2 + vaddl.u8 q14, d6, d9 + vqrshrn.u16 d18, q15, #3 ; r_op2 + + vsub.i16 q15, q10 + vaddl.u8 q10, d4, d6 + vadd.i16 q15, q14 + vaddl.u8 q14, d7, d10 + vqrshrn.u16 d19, q15, #3 ; r_op1 + + vsub.i16 q15, q10 + vadd.i16 q15, q14 + vaddl.u8 q14, d8, d11 + vqrshrn.u16 d20, q15, #3 ; r_op0 + + vsubw.u8 q15, d4 ; oq0 = op0 - p3 + vsubw.u8 q15, d7 ; oq0 -= p0 + vadd.i16 q15, q14 + vaddl.u8 q14, d9, d11 + vqrshrn.u16 d21, q15, #3 ; r_oq0 + + vsubw.u8 q15, d5 ; oq1 = oq0 - p2 + vsubw.u8 q15, d8 ; oq1 -= q0 + vadd.i16 q15, q14 + vaddl.u8 q14, d10, d11 + vqrshrn.u16 d22, q15, #3 ; r_oq1 + + vsubw.u8 q15, d6 ; oq2 = oq0 - p1 + vsubw.u8 q15, d9 ; oq2 -= q1 + vadd.i16 q15, q14 + vqrshrn.u16 d27, q15, #3 ; r_oq2 + + ; Filter does not set op2 or oq2, so use p2 and q2. + vbif d18, d5, d16 ; t_op2 |= p2 & ~(flat & mask) + vbif d19, d25, d16 ; t_op1 |= f_op1 & ~(flat & mask) + vbif d20, d24, d16 ; t_op0 |= f_op0 & ~(flat & mask) + vbif d21, d23, d16 ; t_oq0 |= f_oq0 & ~(flat & mask) + vbif d22, d26, d16 ; t_oq1 |= f_oq1 & ~(flat & mask) + + vbit d23, d27, d16 ; t_oq2 |= r_oq2 & (flat & mask) + vbif d23, d10, d16 ; t_oq2 |= q2 & ~(flat & mask) + + tst r7, #2 + bxne lr + + ; wide_mbfilter flat2 && flat && mask branch + vmov.u8 d16, #7 + vaddl.u8 q15, d7, d8 ; op6 = p0 + q0 + vaddl.u8 q12, d2, d3 + vaddl.u8 q13, d4, d5 + vaddl.u8 q14, d1, d6 + vmlal.u8 q15, d0, d16 ; op6 += p7 * 3 + vadd.i16 q12, q13 + vadd.i16 q15, q14 + vaddl.u8 q14, d2, d9 + vadd.i16 q15, q12 + vaddl.u8 q12, d0, d1 + vaddw.u8 q15, d1 + vaddl.u8 q13, d0, d2 + vadd.i16 q14, q15, q14 + vqrshrn.u16 d16, q15, #4 ; w_op6 + + vsub.i16 q15, q14, q12 + vaddl.u8 q14, d3, d10 + vqrshrn.u16 d24, q15, #4 ; w_op5 + + vsub.i16 q15, q13 + vaddl.u8 q13, d0, d3 + vadd.i16 q15, q14 + vaddl.u8 q14, d4, d11 + vqrshrn.u16 d25, q15, #4 ; w_op4 + + vadd.i16 q15, q14 + vaddl.u8 q14, d0, d4 + vsub.i16 q15, q13 + vsub.i16 q14, q15, q14 + vqrshrn.u16 d26, q15, #4 ; w_op3 + + vaddw.u8 q15, q14, d5 ; op2 += p2 + vaddl.u8 q14, d0, d5 + vaddw.u8 q15, d12 ; op2 += q4 + vbif d26, d4, d17 ; op3 |= p3 & ~(f2 & f & m) + vqrshrn.u16 d27, q15, #4 ; w_op2 + + vsub.i16 q15, q14 + vaddl.u8 q14, d0, d6 + vaddw.u8 q15, d6 ; op1 += p1 + vaddw.u8 q15, d13 ; op1 += q5 + vbif d27, d18, d17 ; op2 |= t_op2 & ~(f2 & f & m) + vqrshrn.u16 d18, q15, #4 ; w_op1 + + vsub.i16 q15, q14 + vaddl.u8 q14, d0, d7 + vaddw.u8 q15, d7 ; op0 += p0 + vaddw.u8 q15, d14 ; op0 += q6 + vbif d18, d19, d17 ; op1 |= t_op1 & ~(f2 & f & m) + vqrshrn.u16 d19, q15, #4 ; w_op0 + + vsub.i16 q15, q14 + vaddl.u8 q14, d1, d8 + vaddw.u8 q15, d8 ; oq0 += q0 + vaddw.u8 q15, d15 ; oq0 += q7 + vbif d19, d20, d17 ; op0 |= t_op0 & ~(f2 & f & m) + vqrshrn.u16 d20, q15, #4 ; w_oq0 + + vsub.i16 q15, q14 + vaddl.u8 q14, d2, d9 + vaddw.u8 q15, d9 ; oq1 += q1 + vaddl.u8 q4, d10, d15 + vaddw.u8 q15, d15 ; oq1 += q7 + vbif d20, d21, d17 ; oq0 |= t_oq0 & ~(f2 & f & m) + vqrshrn.u16 d21, q15, #4 ; w_oq1 + + vsub.i16 q15, q14 + vaddl.u8 q14, d3, d10 + vadd.i16 q15, q4 + vaddl.u8 q4, d11, d15 + vbif d21, d22, d17 ; oq1 |= t_oq1 & ~(f2 & f & m) + vqrshrn.u16 d22, q15, #4 ; w_oq2 + + vsub.i16 q15, q14 + vaddl.u8 q14, d4, d11 + vadd.i16 q15, q4 + vaddl.u8 q4, d12, d15 + vbif d22, d23, d17 ; oq2 |= t_oq2 & ~(f2 & f & m) + vqrshrn.u16 d23, q15, #4 ; w_oq3 + + vsub.i16 q15, q14 + vaddl.u8 q14, d5, d12 + vadd.i16 q15, q4 + vaddl.u8 q4, d13, d15 + vbif d16, d1, d17 ; op6 |= p6 & ~(f2 & f & m) + vqrshrn.u16 d1, q15, #4 ; w_oq4 + + vsub.i16 q15, q14 + vaddl.u8 q14, d6, d13 + vadd.i16 q15, q4 + vaddl.u8 q4, d14, d15 + vbif d24, d2, d17 ; op5 |= p5 & ~(f2 & f & m) + vqrshrn.u16 d2, q15, #4 ; w_oq5 + + vsub.i16 q15, q14 + vbif d25, d3, d17 ; op4 |= p4 & ~(f2 & f & m) + vadd.i16 q15, q4 + vbif d23, d11, d17 ; oq3 |= q3 & ~(f2 & f & m) + vqrshrn.u16 d3, q15, #4 ; w_oq6 + vbif d1, d12, d17 ; oq4 |= q4 & ~(f2 & f & m) + vbif d2, d13, d17 ; oq5 |= q5 & ~(f2 & f & m) + vbif d3, d14, d17 ; oq6 |= q6 & ~(f2 & f & m) + + bx lr + ENDP ; |aom_wide_mbfilter_neon| + + END diff --git a/third_party/aom/aom_dsp/arm/loopfilter_neon.c b/third_party/aom/aom_dsp/arm/loopfilter_neon.c new file mode 100644 index 0000000000..c90d6bfde1 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/loopfilter_neon.c @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "./aom_config.h" +#include "aom/aom_integer.h" + +void aom_lpf_vertical_4_dual_neon(uint8_t *s, int p, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, + const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1) { + aom_lpf_vertical_4_neon(s, p, blimit0, limit0, thresh0); + aom_lpf_vertical_4_neon(s + 8 * p, p, blimit1, limit1, thresh1); +} + +#if HAVE_NEON_ASM +void aom_lpf_horizontal_8_dual_neon( + uint8_t *s, int p /* pitch */, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, + const uint8_t *limit1, const uint8_t *thresh1) { + aom_lpf_horizontal_8_neon(s, p, blimit0, limit0, thresh0); + aom_lpf_horizontal_8_neon(s + 8, p, blimit1, limit1, thresh1); +} + +void aom_lpf_vertical_8_dual_neon(uint8_t *s, int p, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, + const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1) { + aom_lpf_vertical_8_neon(s, p, blimit0, limit0, thresh0); + aom_lpf_vertical_8_neon(s + 8 * p, p, blimit1, limit1, thresh1); +} + +void aom_lpf_vertical_16_dual_neon(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh) { + aom_lpf_vertical_16_neon(s, p, blimit, limit, thresh); + aom_lpf_vertical_16_neon(s + 8 * p, p, blimit, limit, thresh); +} +#endif // HAVE_NEON_ASM diff --git a/third_party/aom/aom_dsp/arm/sad4d_neon.c b/third_party/aom/aom_dsp/arm/sad4d_neon.c new file mode 100644 index 0000000000..a1eeaf4b77 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/sad4d_neon.c @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +static INLINE unsigned int horizontal_long_add_16x8(const uint16x8_t vec_lo, + const uint16x8_t vec_hi) { + const uint32x4_t vec_l_lo = + vaddl_u16(vget_low_u16(vec_lo), vget_high_u16(vec_lo)); + const uint32x4_t vec_l_hi = + vaddl_u16(vget_low_u16(vec_hi), vget_high_u16(vec_hi)); + const uint32x4_t a = vaddq_u32(vec_l_lo, vec_l_hi); + const uint64x2_t b = vpaddlq_u32(a); + const uint32x2_t c = vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)), + vreinterpret_u32_u64(vget_high_u64(b))); + return vget_lane_u32(c, 0); +} + +// Calculate the absolute difference of 64 bytes from vec_src_00, vec_src_16, +// vec_src_32, vec_src_48 and ref. Accumulate partial sums in vec_sum_ref_lo +// and vec_sum_ref_hi. +static void sad_neon_64(const uint8x16_t vec_src_00, + const uint8x16_t vec_src_16, + const uint8x16_t vec_src_32, + const uint8x16_t vec_src_48, const uint8_t *ref, + uint16x8_t *vec_sum_ref_lo, + uint16x8_t *vec_sum_ref_hi) { + const uint8x16_t vec_ref_00 = vld1q_u8(ref); + const uint8x16_t vec_ref_16 = vld1q_u8(ref + 16); + const uint8x16_t vec_ref_32 = vld1q_u8(ref + 32); + const uint8x16_t vec_ref_48 = vld1q_u8(ref + 48); + + *vec_sum_ref_lo = vabal_u8(*vec_sum_ref_lo, vget_low_u8(vec_src_00), + vget_low_u8(vec_ref_00)); + *vec_sum_ref_hi = vabal_u8(*vec_sum_ref_hi, vget_high_u8(vec_src_00), + vget_high_u8(vec_ref_00)); + *vec_sum_ref_lo = vabal_u8(*vec_sum_ref_lo, vget_low_u8(vec_src_16), + vget_low_u8(vec_ref_16)); + *vec_sum_ref_hi = vabal_u8(*vec_sum_ref_hi, vget_high_u8(vec_src_16), + vget_high_u8(vec_ref_16)); + *vec_sum_ref_lo = vabal_u8(*vec_sum_ref_lo, vget_low_u8(vec_src_32), + vget_low_u8(vec_ref_32)); + *vec_sum_ref_hi = vabal_u8(*vec_sum_ref_hi, vget_high_u8(vec_src_32), + vget_high_u8(vec_ref_32)); + *vec_sum_ref_lo = vabal_u8(*vec_sum_ref_lo, vget_low_u8(vec_src_48), + vget_low_u8(vec_ref_48)); + *vec_sum_ref_hi = vabal_u8(*vec_sum_ref_hi, vget_high_u8(vec_src_48), + vget_high_u8(vec_ref_48)); +} + +// Calculate the absolute difference of 32 bytes from vec_src_00, vec_src_16, +// and ref. Accumulate partial sums in vec_sum_ref_lo and vec_sum_ref_hi. +static void sad_neon_32(const uint8x16_t vec_src_00, + const uint8x16_t vec_src_16, const uint8_t *ref, + uint16x8_t *vec_sum_ref_lo, + uint16x8_t *vec_sum_ref_hi) { + const uint8x16_t vec_ref_00 = vld1q_u8(ref); + const uint8x16_t vec_ref_16 = vld1q_u8(ref + 16); + + *vec_sum_ref_lo = vabal_u8(*vec_sum_ref_lo, vget_low_u8(vec_src_00), + vget_low_u8(vec_ref_00)); + *vec_sum_ref_hi = vabal_u8(*vec_sum_ref_hi, vget_high_u8(vec_src_00), + vget_high_u8(vec_ref_00)); + *vec_sum_ref_lo = vabal_u8(*vec_sum_ref_lo, vget_low_u8(vec_src_16), + vget_low_u8(vec_ref_16)); + *vec_sum_ref_hi = vabal_u8(*vec_sum_ref_hi, vget_high_u8(vec_src_16), + vget_high_u8(vec_ref_16)); +} + +void aom_sad64x64x4d_neon(const uint8_t *src, int src_stride, + const uint8_t *const ref[4], int ref_stride, + uint32_t *res) { + int i; + uint16x8_t vec_sum_ref0_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_ref0_hi = vdupq_n_u16(0); + uint16x8_t vec_sum_ref1_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_ref1_hi = vdupq_n_u16(0); + uint16x8_t vec_sum_ref2_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_ref2_hi = vdupq_n_u16(0); + uint16x8_t vec_sum_ref3_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_ref3_hi = vdupq_n_u16(0); + const uint8_t *ref0, *ref1, *ref2, *ref3; + ref0 = ref[0]; + ref1 = ref[1]; + ref2 = ref[2]; + ref3 = ref[3]; + + for (i = 0; i < 64; ++i) { + const uint8x16_t vec_src_00 = vld1q_u8(src); + const uint8x16_t vec_src_16 = vld1q_u8(src + 16); + const uint8x16_t vec_src_32 = vld1q_u8(src + 32); + const uint8x16_t vec_src_48 = vld1q_u8(src + 48); + + sad_neon_64(vec_src_00, vec_src_16, vec_src_32, vec_src_48, ref0, + &vec_sum_ref0_lo, &vec_sum_ref0_hi); + sad_neon_64(vec_src_00, vec_src_16, vec_src_32, vec_src_48, ref1, + &vec_sum_ref1_lo, &vec_sum_ref1_hi); + sad_neon_64(vec_src_00, vec_src_16, vec_src_32, vec_src_48, ref2, + &vec_sum_ref2_lo, &vec_sum_ref2_hi); + sad_neon_64(vec_src_00, vec_src_16, vec_src_32, vec_src_48, ref3, + &vec_sum_ref3_lo, &vec_sum_ref3_hi); + + src += src_stride; + ref0 += ref_stride; + ref1 += ref_stride; + ref2 += ref_stride; + ref3 += ref_stride; + } + + res[0] = horizontal_long_add_16x8(vec_sum_ref0_lo, vec_sum_ref0_hi); + res[1] = horizontal_long_add_16x8(vec_sum_ref1_lo, vec_sum_ref1_hi); + res[2] = horizontal_long_add_16x8(vec_sum_ref2_lo, vec_sum_ref2_hi); + res[3] = horizontal_long_add_16x8(vec_sum_ref3_lo, vec_sum_ref3_hi); +} + +void aom_sad32x32x4d_neon(const uint8_t *src, int src_stride, + const uint8_t *const ref[4], int ref_stride, + uint32_t *res) { + int i; + uint16x8_t vec_sum_ref0_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_ref0_hi = vdupq_n_u16(0); + uint16x8_t vec_sum_ref1_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_ref1_hi = vdupq_n_u16(0); + uint16x8_t vec_sum_ref2_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_ref2_hi = vdupq_n_u16(0); + uint16x8_t vec_sum_ref3_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_ref3_hi = vdupq_n_u16(0); + const uint8_t *ref0, *ref1, *ref2, *ref3; + ref0 = ref[0]; + ref1 = ref[1]; + ref2 = ref[2]; + ref3 = ref[3]; + + for (i = 0; i < 32; ++i) { + const uint8x16_t vec_src_00 = vld1q_u8(src); + const uint8x16_t vec_src_16 = vld1q_u8(src + 16); + + sad_neon_32(vec_src_00, vec_src_16, ref0, &vec_sum_ref0_lo, + &vec_sum_ref0_hi); + sad_neon_32(vec_src_00, vec_src_16, ref1, &vec_sum_ref1_lo, + &vec_sum_ref1_hi); + sad_neon_32(vec_src_00, vec_src_16, ref2, &vec_sum_ref2_lo, + &vec_sum_ref2_hi); + sad_neon_32(vec_src_00, vec_src_16, ref3, &vec_sum_ref3_lo, + &vec_sum_ref3_hi); + + src += src_stride; + ref0 += ref_stride; + ref1 += ref_stride; + ref2 += ref_stride; + ref3 += ref_stride; + } + + res[0] = horizontal_long_add_16x8(vec_sum_ref0_lo, vec_sum_ref0_hi); + res[1] = horizontal_long_add_16x8(vec_sum_ref1_lo, vec_sum_ref1_hi); + res[2] = horizontal_long_add_16x8(vec_sum_ref2_lo, vec_sum_ref2_hi); + res[3] = horizontal_long_add_16x8(vec_sum_ref3_lo, vec_sum_ref3_hi); +} + +void aom_sad16x16x4d_neon(const uint8_t *src, int src_stride, + const uint8_t *const ref[4], int ref_stride, + uint32_t *res) { + int i; + uint16x8_t vec_sum_ref0_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_ref0_hi = vdupq_n_u16(0); + uint16x8_t vec_sum_ref1_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_ref1_hi = vdupq_n_u16(0); + uint16x8_t vec_sum_ref2_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_ref2_hi = vdupq_n_u16(0); + uint16x8_t vec_sum_ref3_lo = vdupq_n_u16(0); + uint16x8_t vec_sum_ref3_hi = vdupq_n_u16(0); + const uint8_t *ref0, *ref1, *ref2, *ref3; + ref0 = ref[0]; + ref1 = ref[1]; + ref2 = ref[2]; + ref3 = ref[3]; + + for (i = 0; i < 16; ++i) { + const uint8x16_t vec_src = vld1q_u8(src); + const uint8x16_t vec_ref0 = vld1q_u8(ref0); + const uint8x16_t vec_ref1 = vld1q_u8(ref1); + const uint8x16_t vec_ref2 = vld1q_u8(ref2); + const uint8x16_t vec_ref3 = vld1q_u8(ref3); + + vec_sum_ref0_lo = + vabal_u8(vec_sum_ref0_lo, vget_low_u8(vec_src), vget_low_u8(vec_ref0)); + vec_sum_ref0_hi = vabal_u8(vec_sum_ref0_hi, vget_high_u8(vec_src), + vget_high_u8(vec_ref0)); + vec_sum_ref1_lo = + vabal_u8(vec_sum_ref1_lo, vget_low_u8(vec_src), vget_low_u8(vec_ref1)); + vec_sum_ref1_hi = vabal_u8(vec_sum_ref1_hi, vget_high_u8(vec_src), + vget_high_u8(vec_ref1)); + vec_sum_ref2_lo = + vabal_u8(vec_sum_ref2_lo, vget_low_u8(vec_src), vget_low_u8(vec_ref2)); + vec_sum_ref2_hi = vabal_u8(vec_sum_ref2_hi, vget_high_u8(vec_src), + vget_high_u8(vec_ref2)); + vec_sum_ref3_lo = + vabal_u8(vec_sum_ref3_lo, vget_low_u8(vec_src), vget_low_u8(vec_ref3)); + vec_sum_ref3_hi = vabal_u8(vec_sum_ref3_hi, vget_high_u8(vec_src), + vget_high_u8(vec_ref3)); + + src += src_stride; + ref0 += ref_stride; + ref1 += ref_stride; + ref2 += ref_stride; + ref3 += ref_stride; + } + + res[0] = horizontal_long_add_16x8(vec_sum_ref0_lo, vec_sum_ref0_hi); + res[1] = horizontal_long_add_16x8(vec_sum_ref1_lo, vec_sum_ref1_hi); + res[2] = horizontal_long_add_16x8(vec_sum_ref2_lo, vec_sum_ref2_hi); + res[3] = horizontal_long_add_16x8(vec_sum_ref3_lo, vec_sum_ref3_hi); +} diff --git a/third_party/aom/aom_dsp/arm/sad_media.asm b/third_party/aom/aom_dsp/arm/sad_media.asm new file mode 100644 index 0000000000..49ddb67642 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/sad_media.asm @@ -0,0 +1,98 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + + EXPORT |aom_sad16x16_media| + + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +; r0 const unsigned char *src_ptr +; r1 int src_stride +; r2 const unsigned char *ref_ptr +; r3 int ref_stride +|aom_sad16x16_media| PROC + stmfd sp!, {r4-r12, lr} + + pld [r0, r1, lsl #0] + pld [r2, r3, lsl #0] + pld [r0, r1, lsl #1] + pld [r2, r3, lsl #1] + + mov r4, #0 ; sad = 0; + mov r5, #8 ; loop count + +loop + ; 1st row + ldr r6, [r0, #0x0] ; load 4 src pixels (1A) + ldr r8, [r2, #0x0] ; load 4 ref pixels (1A) + ldr r7, [r0, #0x4] ; load 4 src pixels (1A) + ldr r9, [r2, #0x4] ; load 4 ref pixels (1A) + ldr r10, [r0, #0x8] ; load 4 src pixels (1B) + ldr r11, [r0, #0xC] ; load 4 src pixels (1B) + + usada8 r4, r8, r6, r4 ; calculate sad for 4 pixels + usad8 r8, r7, r9 ; calculate sad for 4 pixels + + ldr r12, [r2, #0x8] ; load 4 ref pixels (1B) + ldr lr, [r2, #0xC] ; load 4 ref pixels (1B) + + add r0, r0, r1 ; set src pointer to next row + add r2, r2, r3 ; set dst pointer to next row + + pld [r0, r1, lsl #1] + pld [r2, r3, lsl #1] + + usada8 r4, r10, r12, r4 ; calculate sad for 4 pixels + usada8 r8, r11, lr, r8 ; calculate sad for 4 pixels + + ldr r6, [r0, #0x0] ; load 4 src pixels (2A) + ldr r7, [r0, #0x4] ; load 4 src pixels (2A) + add r4, r4, r8 ; add partial sad values + + ; 2nd row + ldr r8, [r2, #0x0] ; load 4 ref pixels (2A) + ldr r9, [r2, #0x4] ; load 4 ref pixels (2A) + ldr r10, [r0, #0x8] ; load 4 src pixels (2B) + ldr r11, [r0, #0xC] ; load 4 src pixels (2B) + + usada8 r4, r6, r8, r4 ; calculate sad for 4 pixels + usad8 r8, r7, r9 ; calculate sad for 4 pixels + + ldr r12, [r2, #0x8] ; load 4 ref pixels (2B) + ldr lr, [r2, #0xC] ; load 4 ref pixels (2B) + + add r0, r0, r1 ; set src pointer to next row + add r2, r2, r3 ; set dst pointer to next row + + usada8 r4, r10, r12, r4 ; calculate sad for 4 pixels + usada8 r8, r11, lr, r8 ; calculate sad for 4 pixels + + pld [r0, r1, lsl #1] + pld [r2, r3, lsl #1] + + subs r5, r5, #1 ; decrement loop counter + add r4, r4, r8 ; add partial sad values + + bne loop + + mov r0, r4 ; return sad + ldmfd sp!, {r4-r12, pc} + + ENDP + + END + diff --git a/third_party/aom/aom_dsp/arm/sad_neon.c b/third_party/aom/aom_dsp/arm/sad_neon.c new file mode 100644 index 0000000000..2f452f55b5 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/sad_neon.c @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" + +#include "aom/aom_integer.h" + +unsigned int aom_sad8x16_neon(unsigned char *src_ptr, int src_stride, + unsigned char *ref_ptr, int ref_stride) { + uint8x8_t d0, d8; + uint16x8_t q12; + uint32x4_t q1; + uint64x2_t q3; + uint32x2_t d5; + int i; + + d0 = vld1_u8(src_ptr); + src_ptr += src_stride; + d8 = vld1_u8(ref_ptr); + ref_ptr += ref_stride; + q12 = vabdl_u8(d0, d8); + + for (i = 0; i < 15; i++) { + d0 = vld1_u8(src_ptr); + src_ptr += src_stride; + d8 = vld1_u8(ref_ptr); + ref_ptr += ref_stride; + q12 = vabal_u8(q12, d0, d8); + } + + q1 = vpaddlq_u16(q12); + q3 = vpaddlq_u32(q1); + d5 = vadd_u32(vreinterpret_u32_u64(vget_low_u64(q3)), + vreinterpret_u32_u64(vget_high_u64(q3))); + + return vget_lane_u32(d5, 0); +} + +unsigned int aom_sad4x4_neon(unsigned char *src_ptr, int src_stride, + unsigned char *ref_ptr, int ref_stride) { + uint8x8_t d0, d8; + uint16x8_t q12; + uint32x2_t d1; + uint64x1_t d3; + int i; + + d0 = vld1_u8(src_ptr); + src_ptr += src_stride; + d8 = vld1_u8(ref_ptr); + ref_ptr += ref_stride; + q12 = vabdl_u8(d0, d8); + + for (i = 0; i < 3; i++) { + d0 = vld1_u8(src_ptr); + src_ptr += src_stride; + d8 = vld1_u8(ref_ptr); + ref_ptr += ref_stride; + q12 = vabal_u8(q12, d0, d8); + } + + d1 = vpaddl_u16(vget_low_u16(q12)); + d3 = vpaddl_u32(d1); + + return vget_lane_u32(vreinterpret_u32_u64(d3), 0); +} + +unsigned int aom_sad16x8_neon(unsigned char *src_ptr, int src_stride, + unsigned char *ref_ptr, int ref_stride) { + uint8x16_t q0, q4; + uint16x8_t q12, q13; + uint32x4_t q1; + uint64x2_t q3; + uint32x2_t d5; + int i; + + q0 = vld1q_u8(src_ptr); + src_ptr += src_stride; + q4 = vld1q_u8(ref_ptr); + ref_ptr += ref_stride; + q12 = vabdl_u8(vget_low_u8(q0), vget_low_u8(q4)); + q13 = vabdl_u8(vget_high_u8(q0), vget_high_u8(q4)); + + for (i = 0; i < 7; i++) { + q0 = vld1q_u8(src_ptr); + src_ptr += src_stride; + q4 = vld1q_u8(ref_ptr); + ref_ptr += ref_stride; + q12 = vabal_u8(q12, vget_low_u8(q0), vget_low_u8(q4)); + q13 = vabal_u8(q13, vget_high_u8(q0), vget_high_u8(q4)); + } + + q12 = vaddq_u16(q12, q13); + q1 = vpaddlq_u16(q12); + q3 = vpaddlq_u32(q1); + d5 = vadd_u32(vreinterpret_u32_u64(vget_low_u64(q3)), + vreinterpret_u32_u64(vget_high_u64(q3))); + + return vget_lane_u32(d5, 0); +} + +static INLINE unsigned int horizontal_long_add_16x8(const uint16x8_t vec_lo, + const uint16x8_t vec_hi) { + const uint32x4_t vec_l_lo = + vaddl_u16(vget_low_u16(vec_lo), vget_high_u16(vec_lo)); + const uint32x4_t vec_l_hi = + vaddl_u16(vget_low_u16(vec_hi), vget_high_u16(vec_hi)); + const uint32x4_t a = vaddq_u32(vec_l_lo, vec_l_hi); + const uint64x2_t b = vpaddlq_u32(a); + const uint32x2_t c = vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)), + vreinterpret_u32_u64(vget_high_u64(b))); + return vget_lane_u32(c, 0); +} +static INLINE unsigned int horizontal_add_16x8(const uint16x8_t vec_16x8) { + const uint32x4_t a = vpaddlq_u16(vec_16x8); + const uint64x2_t b = vpaddlq_u32(a); + const uint32x2_t c = vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)), + vreinterpret_u32_u64(vget_high_u64(b))); + return vget_lane_u32(c, 0); +} + +unsigned int aom_sad64x64_neon(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + int i; + uint16x8_t vec_accum_lo = vdupq_n_u16(0); + uint16x8_t vec_accum_hi = vdupq_n_u16(0); + for (i = 0; i < 64; ++i) { + const uint8x16_t vec_src_00 = vld1q_u8(src); + const uint8x16_t vec_src_16 = vld1q_u8(src + 16); + const uint8x16_t vec_src_32 = vld1q_u8(src + 32); + const uint8x16_t vec_src_48 = vld1q_u8(src + 48); + const uint8x16_t vec_ref_00 = vld1q_u8(ref); + const uint8x16_t vec_ref_16 = vld1q_u8(ref + 16); + const uint8x16_t vec_ref_32 = vld1q_u8(ref + 32); + const uint8x16_t vec_ref_48 = vld1q_u8(ref + 48); + src += src_stride; + ref += ref_stride; + vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src_00), + vget_low_u8(vec_ref_00)); + vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src_00), + vget_high_u8(vec_ref_00)); + vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src_16), + vget_low_u8(vec_ref_16)); + vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src_16), + vget_high_u8(vec_ref_16)); + vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src_32), + vget_low_u8(vec_ref_32)); + vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src_32), + vget_high_u8(vec_ref_32)); + vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src_48), + vget_low_u8(vec_ref_48)); + vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src_48), + vget_high_u8(vec_ref_48)); + } + return horizontal_long_add_16x8(vec_accum_lo, vec_accum_hi); +} + +unsigned int aom_sad32x32_neon(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + int i; + uint16x8_t vec_accum_lo = vdupq_n_u16(0); + uint16x8_t vec_accum_hi = vdupq_n_u16(0); + + for (i = 0; i < 32; ++i) { + const uint8x16_t vec_src_00 = vld1q_u8(src); + const uint8x16_t vec_src_16 = vld1q_u8(src + 16); + const uint8x16_t vec_ref_00 = vld1q_u8(ref); + const uint8x16_t vec_ref_16 = vld1q_u8(ref + 16); + src += src_stride; + ref += ref_stride; + vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src_00), + vget_low_u8(vec_ref_00)); + vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src_00), + vget_high_u8(vec_ref_00)); + vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src_16), + vget_low_u8(vec_ref_16)); + vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src_16), + vget_high_u8(vec_ref_16)); + } + return horizontal_add_16x8(vaddq_u16(vec_accum_lo, vec_accum_hi)); +} + +unsigned int aom_sad16x16_neon(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + int i; + uint16x8_t vec_accum_lo = vdupq_n_u16(0); + uint16x8_t vec_accum_hi = vdupq_n_u16(0); + + for (i = 0; i < 16; ++i) { + const uint8x16_t vec_src = vld1q_u8(src); + const uint8x16_t vec_ref = vld1q_u8(ref); + src += src_stride; + ref += ref_stride; + vec_accum_lo = + vabal_u8(vec_accum_lo, vget_low_u8(vec_src), vget_low_u8(vec_ref)); + vec_accum_hi = + vabal_u8(vec_accum_hi, vget_high_u8(vec_src), vget_high_u8(vec_ref)); + } + return horizontal_add_16x8(vaddq_u16(vec_accum_lo, vec_accum_hi)); +} + +unsigned int aom_sad8x8_neon(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + int i; + uint16x8_t vec_accum = vdupq_n_u16(0); + + for (i = 0; i < 8; ++i) { + const uint8x8_t vec_src = vld1_u8(src); + const uint8x8_t vec_ref = vld1_u8(ref); + src += src_stride; + ref += ref_stride; + vec_accum = vabal_u8(vec_accum, vec_src, vec_ref); + } + return horizontal_add_16x8(vec_accum); +} diff --git a/third_party/aom/aom_dsp/arm/save_reg_neon.asm b/third_party/aom/aom_dsp/arm/save_reg_neon.asm new file mode 100644 index 0000000000..e049698234 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/save_reg_neon.asm @@ -0,0 +1,39 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + + EXPORT |aom_push_neon| + EXPORT |aom_pop_neon| + + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +|aom_push_neon| PROC + vst1.i64 {d8, d9, d10, d11}, [r0]! + vst1.i64 {d12, d13, d14, d15}, [r0]! + bx lr + + ENDP + +|aom_pop_neon| PROC + vld1.i64 {d8, d9, d10, d11}, [r0]! + vld1.i64 {d12, d13, d14, d15}, [r0]! + bx lr + + ENDP + + END + diff --git a/third_party/aom/aom_dsp/arm/subpel_variance_media.c b/third_party/aom/aom_dsp/arm/subpel_variance_media.c new file mode 100644 index 0000000000..46ec028d37 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/subpel_variance_media.c @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" + +#if HAVE_MEDIA +static const int16_t bilinear_filters_media[8][2] = { { 128, 0 }, { 112, 16 }, + { 96, 32 }, { 80, 48 }, + { 64, 64 }, { 48, 80 }, + { 32, 96 }, { 16, 112 } }; + +extern void aom_filter_block2d_bil_first_pass_media( + const uint8_t *src_ptr, uint16_t *dst_ptr, uint32_t src_pitch, + uint32_t height, uint32_t width, const int16_t *filter); + +extern void aom_filter_block2d_bil_second_pass_media( + const uint16_t *src_ptr, uint8_t *dst_ptr, int32_t src_pitch, + uint32_t height, uint32_t width, const int16_t *filter); + +unsigned int aom_sub_pixel_variance8x8_media( + const uint8_t *src_ptr, int src_pixels_per_line, int xoffset, int yoffset, + const uint8_t *dst_ptr, int dst_pixels_per_line, unsigned int *sse) { + uint16_t first_pass[10 * 8]; + uint8_t second_pass[8 * 8]; + const int16_t *HFilter, *VFilter; + + HFilter = bilinear_filters_media[xoffset]; + VFilter = bilinear_filters_media[yoffset]; + + aom_filter_block2d_bil_first_pass_media(src_ptr, first_pass, + src_pixels_per_line, 9, 8, HFilter); + aom_filter_block2d_bil_second_pass_media(first_pass, second_pass, 8, 8, 8, + VFilter); + + return aom_variance8x8_media(second_pass, 8, dst_ptr, dst_pixels_per_line, + sse); +} + +unsigned int aom_sub_pixel_variance16x16_media( + const uint8_t *src_ptr, int src_pixels_per_line, int xoffset, int yoffset, + const uint8_t *dst_ptr, int dst_pixels_per_line, unsigned int *sse) { + uint16_t first_pass[36 * 16]; + uint8_t second_pass[20 * 16]; + const int16_t *HFilter, *VFilter; + unsigned int var; + + if (xoffset == 4 && yoffset == 0) { + var = aom_variance_halfpixvar16x16_h_media( + src_ptr, src_pixels_per_line, dst_ptr, dst_pixels_per_line, sse); + } else if (xoffset == 0 && yoffset == 4) { + var = aom_variance_halfpixvar16x16_v_media( + src_ptr, src_pixels_per_line, dst_ptr, dst_pixels_per_line, sse); + } else if (xoffset == 4 && yoffset == 4) { + var = aom_variance_halfpixvar16x16_hv_media( + src_ptr, src_pixels_per_line, dst_ptr, dst_pixels_per_line, sse); + } else { + HFilter = bilinear_filters_media[xoffset]; + VFilter = bilinear_filters_media[yoffset]; + + aom_filter_block2d_bil_first_pass_media( + src_ptr, first_pass, src_pixels_per_line, 17, 16, HFilter); + aom_filter_block2d_bil_second_pass_media(first_pass, second_pass, 16, 16, + 16, VFilter); + + var = aom_variance16x16_media(second_pass, 16, dst_ptr, dst_pixels_per_line, + sse); + } + return var; +} +#endif // HAVE_MEDIA diff --git a/third_party/aom/aom_dsp/arm/subpel_variance_neon.c b/third_party/aom/aom_dsp/arm/subpel_variance_neon.c new file mode 100644 index 0000000000..064b72d6fc --- /dev/null +++ b/third_party/aom/aom_dsp/arm/subpel_variance_neon.c @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./aom_dsp_rtcd.h" +#include "./aom_config.h" + +#include "aom_ports/mem.h" +#include "aom/aom_integer.h" + +#include "aom_dsp/variance.h" + +static const uint8_t bilinear_filters[8][2] = { + { 128, 0 }, { 112, 16 }, { 96, 32 }, { 80, 48 }, + { 64, 64 }, { 48, 80 }, { 32, 96 }, { 16, 112 }, +}; + +static void var_filter_block2d_bil_w8(const uint8_t *src_ptr, + uint8_t *output_ptr, + unsigned int src_pixels_per_line, + int pixel_step, + unsigned int output_height, + unsigned int output_width, + const uint8_t *filter) { + const uint8x8_t f0 = vmov_n_u8(filter[0]); + const uint8x8_t f1 = vmov_n_u8(filter[1]); + unsigned int i; + for (i = 0; i < output_height; ++i) { + const uint8x8_t src_0 = vld1_u8(&src_ptr[0]); + const uint8x8_t src_1 = vld1_u8(&src_ptr[pixel_step]); + const uint16x8_t a = vmull_u8(src_0, f0); + const uint16x8_t b = vmlal_u8(a, src_1, f1); + const uint8x8_t out = vrshrn_n_u16(b, FILTER_BITS); + vst1_u8(&output_ptr[0], out); + // Next row... + src_ptr += src_pixels_per_line; + output_ptr += output_width; + } +} + +static void var_filter_block2d_bil_w16(const uint8_t *src_ptr, + uint8_t *output_ptr, + unsigned int src_pixels_per_line, + int pixel_step, + unsigned int output_height, + unsigned int output_width, + const uint8_t *filter) { + const uint8x8_t f0 = vmov_n_u8(filter[0]); + const uint8x8_t f1 = vmov_n_u8(filter[1]); + unsigned int i, j; + for (i = 0; i < output_height; ++i) { + for (j = 0; j < output_width; j += 16) { + const uint8x16_t src_0 = vld1q_u8(&src_ptr[j]); + const uint8x16_t src_1 = vld1q_u8(&src_ptr[j + pixel_step]); + const uint16x8_t a = vmull_u8(vget_low_u8(src_0), f0); + const uint16x8_t b = vmlal_u8(a, vget_low_u8(src_1), f1); + const uint8x8_t out_lo = vrshrn_n_u16(b, FILTER_BITS); + const uint16x8_t c = vmull_u8(vget_high_u8(src_0), f0); + const uint16x8_t d = vmlal_u8(c, vget_high_u8(src_1), f1); + const uint8x8_t out_hi = vrshrn_n_u16(d, FILTER_BITS); + vst1q_u8(&output_ptr[j], vcombine_u8(out_lo, out_hi)); + } + // Next row... + src_ptr += src_pixels_per_line; + output_ptr += output_width; + } +} + +unsigned int aom_sub_pixel_variance8x8_neon(const uint8_t *src, int src_stride, + int xoffset, int yoffset, + const uint8_t *dst, int dst_stride, + unsigned int *sse) { + DECLARE_ALIGNED(16, uint8_t, temp2[8 * 8]); + DECLARE_ALIGNED(16, uint8_t, fdata3[9 * 8]); + + var_filter_block2d_bil_w8(src, fdata3, src_stride, 1, 9, 8, + bilinear_filters[xoffset]); + var_filter_block2d_bil_w8(fdata3, temp2, 8, 8, 8, 8, + bilinear_filters[yoffset]); + return aom_variance8x8_neon(temp2, 8, dst, dst_stride, sse); +} + +unsigned int aom_sub_pixel_variance16x16_neon(const uint8_t *src, + int src_stride, int xoffset, + int yoffset, const uint8_t *dst, + int dst_stride, + unsigned int *sse) { + DECLARE_ALIGNED(16, uint8_t, temp2[16 * 16]); + DECLARE_ALIGNED(16, uint8_t, fdata3[17 * 16]); + + var_filter_block2d_bil_w16(src, fdata3, src_stride, 1, 17, 16, + bilinear_filters[xoffset]); + var_filter_block2d_bil_w16(fdata3, temp2, 16, 16, 16, 16, + bilinear_filters[yoffset]); + return aom_variance16x16_neon(temp2, 16, dst, dst_stride, sse); +} + +unsigned int aom_sub_pixel_variance32x32_neon(const uint8_t *src, + int src_stride, int xoffset, + int yoffset, const uint8_t *dst, + int dst_stride, + unsigned int *sse) { + DECLARE_ALIGNED(16, uint8_t, temp2[32 * 32]); + DECLARE_ALIGNED(16, uint8_t, fdata3[33 * 32]); + + var_filter_block2d_bil_w16(src, fdata3, src_stride, 1, 33, 32, + bilinear_filters[xoffset]); + var_filter_block2d_bil_w16(fdata3, temp2, 32, 32, 32, 32, + bilinear_filters[yoffset]); + return aom_variance32x32_neon(temp2, 32, dst, dst_stride, sse); +} + +unsigned int aom_sub_pixel_variance64x64_neon(const uint8_t *src, + int src_stride, int xoffset, + int yoffset, const uint8_t *dst, + int dst_stride, + unsigned int *sse) { + DECLARE_ALIGNED(16, uint8_t, temp2[64 * 64]); + DECLARE_ALIGNED(16, uint8_t, fdata3[65 * 64]); + + var_filter_block2d_bil_w16(src, fdata3, src_stride, 1, 65, 64, + bilinear_filters[xoffset]); + var_filter_block2d_bil_w16(fdata3, temp2, 64, 64, 64, 64, + bilinear_filters[yoffset]); + return aom_variance64x64_neon(temp2, 64, dst, dst_stride, sse); +} diff --git a/third_party/aom/aom_dsp/arm/subtract_neon.c b/third_party/aom/aom_dsp/arm/subtract_neon.c new file mode 100644 index 0000000000..cb8a2daf8a --- /dev/null +++ b/third_party/aom/aom_dsp/arm/subtract_neon.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "aom/aom_integer.h" + +void aom_subtract_block_neon(int rows, int cols, int16_t *diff, + ptrdiff_t diff_stride, const uint8_t *src, + ptrdiff_t src_stride, const uint8_t *pred, + ptrdiff_t pred_stride) { + int r, c; + + if (cols > 16) { + for (r = 0; r < rows; ++r) { + for (c = 0; c < cols; c += 32) { + const uint8x16_t v_src_00 = vld1q_u8(&src[c + 0]); + const uint8x16_t v_src_16 = vld1q_u8(&src[c + 16]); + const uint8x16_t v_pred_00 = vld1q_u8(&pred[c + 0]); + const uint8x16_t v_pred_16 = vld1q_u8(&pred[c + 16]); + const uint16x8_t v_diff_lo_00 = + vsubl_u8(vget_low_u8(v_src_00), vget_low_u8(v_pred_00)); + const uint16x8_t v_diff_hi_00 = + vsubl_u8(vget_high_u8(v_src_00), vget_high_u8(v_pred_00)); + const uint16x8_t v_diff_lo_16 = + vsubl_u8(vget_low_u8(v_src_16), vget_low_u8(v_pred_16)); + const uint16x8_t v_diff_hi_16 = + vsubl_u8(vget_high_u8(v_src_16), vget_high_u8(v_pred_16)); + vst1q_s16(&diff[c + 0], vreinterpretq_s16_u16(v_diff_lo_00)); + vst1q_s16(&diff[c + 8], vreinterpretq_s16_u16(v_diff_hi_00)); + vst1q_s16(&diff[c + 16], vreinterpretq_s16_u16(v_diff_lo_16)); + vst1q_s16(&diff[c + 24], vreinterpretq_s16_u16(v_diff_hi_16)); + } + diff += diff_stride; + pred += pred_stride; + src += src_stride; + } + } else if (cols > 8) { + for (r = 0; r < rows; ++r) { + const uint8x16_t v_src = vld1q_u8(&src[0]); + const uint8x16_t v_pred = vld1q_u8(&pred[0]); + const uint16x8_t v_diff_lo = + vsubl_u8(vget_low_u8(v_src), vget_low_u8(v_pred)); + const uint16x8_t v_diff_hi = + vsubl_u8(vget_high_u8(v_src), vget_high_u8(v_pred)); + vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff_lo)); + vst1q_s16(&diff[8], vreinterpretq_s16_u16(v_diff_hi)); + diff += diff_stride; + pred += pred_stride; + src += src_stride; + } + } else if (cols > 4) { + for (r = 0; r < rows; ++r) { + const uint8x8_t v_src = vld1_u8(&src[0]); + const uint8x8_t v_pred = vld1_u8(&pred[0]); + const uint16x8_t v_diff = vsubl_u8(v_src, v_pred); + vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff)); + diff += diff_stride; + pred += pred_stride; + src += src_stride; + } + } else { + for (r = 0; r < rows; ++r) { + for (c = 0; c < cols; ++c) diff[c] = src[c] - pred[c]; + + diff += diff_stride; + pred += pred_stride; + src += src_stride; + } + } +} diff --git a/third_party/aom/aom_dsp/arm/variance_halfpixvar16x16_h_media.asm b/third_party/aom/aom_dsp/arm/variance_halfpixvar16x16_h_media.asm new file mode 100644 index 0000000000..1e5c9178e6 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/variance_halfpixvar16x16_h_media.asm @@ -0,0 +1,185 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + + EXPORT |aom_variance_halfpixvar16x16_h_media| + + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +; r0 unsigned char *src_ptr +; r1 int source_stride +; r2 unsigned char *ref_ptr +; r3 int recon_stride +; stack unsigned int *sse +|aom_variance_halfpixvar16x16_h_media| PROC + + stmfd sp!, {r4-r12, lr} + + pld [r0, r1, lsl #0] + pld [r2, r3, lsl #0] + + mov r8, #0 ; initialize sum = 0 + ldr r10, c80808080 + mov r11, #0 ; initialize sse = 0 + mov r12, #16 ; set loop counter to 16 (=block height) + mov lr, #0 ; constant zero +loop + ; 1st 4 pixels + ldr r4, [r0, #0] ; load 4 src pixels + ldr r6, [r0, #1] ; load 4 src pixels with 1 byte offset + ldr r5, [r2, #0] ; load 4 ref pixels + + ; bilinear interpolation + mvn r6, r6 + uhsub8 r4, r4, r6 + eor r4, r4, r10 + + usub8 r6, r4, r5 ; calculate difference + pld [r0, r1, lsl #1] + sel r7, r6, lr ; select bytes with positive difference + usub8 r6, r5, r4 ; calculate difference with reversed operands + pld [r2, r3, lsl #1] + sel r6, r6, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + ; calculate total sum + adds r8, r8, r4 ; add positive differences to sum + subs r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r7, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + + ; 2nd 4 pixels + ldr r4, [r0, #4] ; load 4 src pixels + ldr r6, [r0, #5] ; load 4 src pixels with 1 byte offset + ldr r5, [r2, #4] ; load 4 ref pixels + + ; bilinear interpolation + mvn r6, r6 + uhsub8 r4, r4, r6 + eor r4, r4, r10 + + smlad r11, r7, r7, r11 ; dual signed multiply, add and accumulate (2) + + usub8 r6, r4, r5 ; calculate difference + sel r7, r6, lr ; select bytes with positive difference + usub8 r6, r5, r4 ; calculate difference with reversed operands + sel r6, r6, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + + ; calculate total sum + add r8, r8, r4 ; add positive differences to sum + sub r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r7, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + + ; 3rd 4 pixels + ldr r4, [r0, #8] ; load 4 src pixels + ldr r6, [r0, #9] ; load 4 src pixels with 1 byte offset + ldr r5, [r2, #8] ; load 4 ref pixels + + ; bilinear interpolation + mvn r6, r6 + uhsub8 r4, r4, r6 + eor r4, r4, r10 + + smlad r11, r7, r7, r11 ; dual signed multiply, add and accumulate (2) + + usub8 r6, r4, r5 ; calculate difference + sel r7, r6, lr ; select bytes with positive difference + usub8 r6, r5, r4 ; calculate difference with reversed operands + sel r6, r6, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + + ; calculate total sum + add r8, r8, r4 ; add positive differences to sum + sub r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r7, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + + ; 4th 4 pixels + ldr r4, [r0, #12] ; load 4 src pixels + ldr r6, [r0, #13] ; load 4 src pixels with 1 byte offset + ldr r5, [r2, #12] ; load 4 ref pixels + + ; bilinear interpolation + mvn r6, r6 + uhsub8 r4, r4, r6 + eor r4, r4, r10 + + smlad r11, r7, r7, r11 ; dual signed multiply, add and accumulate (2) + + usub8 r6, r4, r5 ; calculate difference + add r0, r0, r1 ; set src_ptr to next row + sel r7, r6, lr ; select bytes with positive difference + usub8 r6, r5, r4 ; calculate difference with reversed operands + add r2, r2, r3 ; set dst_ptr to next row + sel r6, r6, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + + ; calculate total sum + add r8, r8, r4 ; add positive differences to sum + sub r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r7, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + smlad r11, r7, r7, r11 ; dual signed multiply, add and accumulate (2) + + subs r12, r12, #1 + + bne loop + + ; return stuff + ldr r6, [sp, #40] ; get address of sse + mul r0, r8, r8 ; sum * sum + str r11, [r6] ; store sse + sub r0, r11, r0, lsr #8 ; return (sse - ((sum * sum) >> 8)) + + ldmfd sp!, {r4-r12, pc} + + ENDP + +c80808080 + DCD 0x80808080 + + END + diff --git a/third_party/aom/aom_dsp/arm/variance_halfpixvar16x16_hv_media.asm b/third_party/aom/aom_dsp/arm/variance_halfpixvar16x16_hv_media.asm new file mode 100644 index 0000000000..9e0af830ee --- /dev/null +++ b/third_party/aom/aom_dsp/arm/variance_halfpixvar16x16_hv_media.asm @@ -0,0 +1,225 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + + EXPORT |aom_variance_halfpixvar16x16_hv_media| + + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +; r0 unsigned char *src_ptr +; r1 int source_stride +; r2 unsigned char *ref_ptr +; r3 int recon_stride +; stack unsigned int *sse +|aom_variance_halfpixvar16x16_hv_media| PROC + + stmfd sp!, {r4-r12, lr} + + pld [r0, r1, lsl #0] + pld [r2, r3, lsl #0] + + mov r8, #0 ; initialize sum = 0 + ldr r10, c80808080 + mov r11, #0 ; initialize sse = 0 + mov r12, #16 ; set loop counter to 16 (=block height) + mov lr, #0 ; constant zero +loop + add r9, r0, r1 ; pointer to pixels on the next row + ; 1st 4 pixels + ldr r4, [r0, #0] ; load source pixels a, row N + ldr r6, [r0, #1] ; load source pixels b, row N + ldr r5, [r9, #0] ; load source pixels c, row N+1 + ldr r7, [r9, #1] ; load source pixels d, row N+1 + + ; x = (a + b + 1) >> 1, interpolate pixels horizontally on row N + mvn r6, r6 + uhsub8 r4, r4, r6 + eor r4, r4, r10 + ; y = (c + d + 1) >> 1, interpolate pixels horizontally on row N+1 + mvn r7, r7 + uhsub8 r5, r5, r7 + eor r5, r5, r10 + ; z = (x + y + 1) >> 1, interpolate half pixel values vertically + mvn r5, r5 + uhsub8 r4, r4, r5 + ldr r5, [r2, #0] ; load 4 ref pixels + eor r4, r4, r10 + + usub8 r6, r4, r5 ; calculate difference + pld [r0, r1, lsl #1] + sel r7, r6, lr ; select bytes with positive difference + usub8 r6, r5, r4 ; calculate difference with reversed operands + pld [r2, r3, lsl #1] + sel r6, r6, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + ; calculate total sum + adds r8, r8, r4 ; add positive differences to sum + subs r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r7, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + + ; 2nd 4 pixels + ldr r4, [r0, #4] ; load source pixels a, row N + ldr r6, [r0, #5] ; load source pixels b, row N + ldr r5, [r9, #4] ; load source pixels c, row N+1 + + smlad r11, r7, r7, r11 ; dual signed multiply, add and accumulate (2) + + ldr r7, [r9, #5] ; load source pixels d, row N+1 + + ; x = (a + b + 1) >> 1, interpolate pixels horizontally on row N + mvn r6, r6 + uhsub8 r4, r4, r6 + eor r4, r4, r10 + ; y = (c + d + 1) >> 1, interpolate pixels horizontally on row N+1 + mvn r7, r7 + uhsub8 r5, r5, r7 + eor r5, r5, r10 + ; z = (x + y + 1) >> 1, interpolate half pixel values vertically + mvn r5, r5 + uhsub8 r4, r4, r5 + ldr r5, [r2, #4] ; load 4 ref pixels + eor r4, r4, r10 + + usub8 r6, r4, r5 ; calculate difference + sel r7, r6, lr ; select bytes with positive difference + usub8 r6, r5, r4 ; calculate difference with reversed operands + sel r6, r6, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + + ; calculate total sum + add r8, r8, r4 ; add positive differences to sum + sub r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r7, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + + ; 3rd 4 pixels + ldr r4, [r0, #8] ; load source pixels a, row N + ldr r6, [r0, #9] ; load source pixels b, row N + ldr r5, [r9, #8] ; load source pixels c, row N+1 + + smlad r11, r7, r7, r11 ; dual signed multiply, add and accumulate (2) + + ldr r7, [r9, #9] ; load source pixels d, row N+1 + + ; x = (a + b + 1) >> 1, interpolate pixels horizontally on row N + mvn r6, r6 + uhsub8 r4, r4, r6 + eor r4, r4, r10 + ; y = (c + d + 1) >> 1, interpolate pixels horizontally on row N+1 + mvn r7, r7 + uhsub8 r5, r5, r7 + eor r5, r5, r10 + ; z = (x + y + 1) >> 1, interpolate half pixel values vertically + mvn r5, r5 + uhsub8 r4, r4, r5 + ldr r5, [r2, #8] ; load 4 ref pixels + eor r4, r4, r10 + + usub8 r6, r4, r5 ; calculate difference + sel r7, r6, lr ; select bytes with positive difference + usub8 r6, r5, r4 ; calculate difference with reversed operands + sel r6, r6, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + + ; calculate total sum + add r8, r8, r4 ; add positive differences to sum + sub r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r7, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + + ; 4th 4 pixels + ldr r4, [r0, #12] ; load source pixels a, row N + ldr r6, [r0, #13] ; load source pixels b, row N + ldr r5, [r9, #12] ; load source pixels c, row N+1 + smlad r11, r7, r7, r11 ; dual signed multiply, add and accumulate (2) + ldr r7, [r9, #13] ; load source pixels d, row N+1 + + ; x = (a + b + 1) >> 1, interpolate pixels horizontally on row N + mvn r6, r6 + uhsub8 r4, r4, r6 + eor r4, r4, r10 + ; y = (c + d + 1) >> 1, interpolate pixels horizontally on row N+1 + mvn r7, r7 + uhsub8 r5, r5, r7 + eor r5, r5, r10 + ; z = (x + y + 1) >> 1, interpolate half pixel values vertically + mvn r5, r5 + uhsub8 r4, r4, r5 + ldr r5, [r2, #12] ; load 4 ref pixels + eor r4, r4, r10 + + usub8 r6, r4, r5 ; calculate difference + add r0, r0, r1 ; set src_ptr to next row + sel r7, r6, lr ; select bytes with positive difference + usub8 r6, r5, r4 ; calculate difference with reversed operands + add r2, r2, r3 ; set dst_ptr to next row + sel r6, r6, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + + ; calculate total sum + add r8, r8, r4 ; add positive differences to sum + sub r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r7, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + subs r12, r12, #1 + smlad r11, r7, r7, r11 ; dual signed multiply, add and accumulate (2) + + bne loop + + ; return stuff + ldr r6, [sp, #40] ; get address of sse + mul r0, r8, r8 ; sum * sum + str r11, [r6] ; store sse + sub r0, r11, r0, lsr #8 ; return (sse - ((sum * sum) >> 8)) + + ldmfd sp!, {r4-r12, pc} + + ENDP + +c80808080 + DCD 0x80808080 + + END diff --git a/third_party/aom/aom_dsp/arm/variance_halfpixvar16x16_v_media.asm b/third_party/aom/aom_dsp/arm/variance_halfpixvar16x16_v_media.asm new file mode 100644 index 0000000000..545b681794 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/variance_halfpixvar16x16_v_media.asm @@ -0,0 +1,187 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + + EXPORT |aom_variance_halfpixvar16x16_v_media| + + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +; r0 unsigned char *src_ptr +; r1 int source_stride +; r2 unsigned char *ref_ptr +; r3 int recon_stride +; stack unsigned int *sse +|aom_variance_halfpixvar16x16_v_media| PROC + + stmfd sp!, {r4-r12, lr} + + pld [r0, r1, lsl #0] + pld [r2, r3, lsl #0] + + mov r8, #0 ; initialize sum = 0 + ldr r10, c80808080 + mov r11, #0 ; initialize sse = 0 + mov r12, #16 ; set loop counter to 16 (=block height) + mov lr, #0 ; constant zero +loop + add r9, r0, r1 ; set src pointer to next row + ; 1st 4 pixels + ldr r4, [r0, #0] ; load 4 src pixels + ldr r6, [r9, #0] ; load 4 src pixels from next row + ldr r5, [r2, #0] ; load 4 ref pixels + + ; bilinear interpolation + mvn r6, r6 + uhsub8 r4, r4, r6 + eor r4, r4, r10 + + usub8 r6, r4, r5 ; calculate difference + pld [r0, r1, lsl #1] + sel r7, r6, lr ; select bytes with positive difference + usub8 r6, r5, r4 ; calculate difference with reversed operands + pld [r2, r3, lsl #1] + sel r6, r6, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + ; calculate total sum + adds r8, r8, r4 ; add positive differences to sum + subs r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r7, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + + ; 2nd 4 pixels + ldr r4, [r0, #4] ; load 4 src pixels + ldr r6, [r9, #4] ; load 4 src pixels from next row + ldr r5, [r2, #4] ; load 4 ref pixels + + ; bilinear interpolation + mvn r6, r6 + uhsub8 r4, r4, r6 + eor r4, r4, r10 + + smlad r11, r7, r7, r11 ; dual signed multiply, add and accumulate (2) + + usub8 r6, r4, r5 ; calculate difference + sel r7, r6, lr ; select bytes with positive difference + usub8 r6, r5, r4 ; calculate difference with reversed operands + sel r6, r6, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + + ; calculate total sum + add r8, r8, r4 ; add positive differences to sum + sub r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r7, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + + ; 3rd 4 pixels + ldr r4, [r0, #8] ; load 4 src pixels + ldr r6, [r9, #8] ; load 4 src pixels from next row + ldr r5, [r2, #8] ; load 4 ref pixels + + ; bilinear interpolation + mvn r6, r6 + uhsub8 r4, r4, r6 + eor r4, r4, r10 + + smlad r11, r7, r7, r11 ; dual signed multiply, add and accumulate (2) + + usub8 r6, r4, r5 ; calculate difference + sel r7, r6, lr ; select bytes with positive difference + usub8 r6, r5, r4 ; calculate difference with reversed operands + sel r6, r6, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + + ; calculate total sum + add r8, r8, r4 ; add positive differences to sum + sub r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r7, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + + ; 4th 4 pixels + ldr r4, [r0, #12] ; load 4 src pixels + ldr r6, [r9, #12] ; load 4 src pixels from next row + ldr r5, [r2, #12] ; load 4 ref pixels + + ; bilinear interpolation + mvn r6, r6 + uhsub8 r4, r4, r6 + eor r4, r4, r10 + + smlad r11, r7, r7, r11 ; dual signed multiply, add and accumulate (2) + + usub8 r6, r4, r5 ; calculate difference + add r0, r0, r1 ; set src_ptr to next row + sel r7, r6, lr ; select bytes with positive difference + usub8 r6, r5, r4 ; calculate difference with reversed operands + add r2, r2, r3 ; set dst_ptr to next row + sel r6, r6, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + + ; calculate total sum + add r8, r8, r4 ; add positive differences to sum + sub r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r7, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + smlad r11, r7, r7, r11 ; dual signed multiply, add and accumulate (2) + + + subs r12, r12, #1 + + bne loop + + ; return stuff + ldr r6, [sp, #40] ; get address of sse + mul r0, r8, r8 ; sum * sum + str r11, [r6] ; store sse + sub r0, r11, r0, lsr #8 ; return (sse - ((sum * sum) >> 8)) + + ldmfd sp!, {r4-r12, pc} + + ENDP + +c80808080 + DCD 0x80808080 + + END + diff --git a/third_party/aom/aom_dsp/arm/variance_media.asm b/third_party/aom/aom_dsp/arm/variance_media.asm new file mode 100644 index 0000000000..fdc311a81c --- /dev/null +++ b/third_party/aom/aom_dsp/arm/variance_media.asm @@ -0,0 +1,361 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + + EXPORT |aom_variance16x16_media| + EXPORT |aom_variance8x8_media| + EXPORT |aom_mse16x16_media| + + ARM + REQUIRE8 + PRESERVE8 + + AREA ||.text||, CODE, READONLY, ALIGN=2 + +; r0 unsigned char *src_ptr +; r1 int source_stride +; r2 unsigned char *ref_ptr +; r3 int recon_stride +; stack unsigned int *sse +|aom_variance16x16_media| PROC + + stmfd sp!, {r4-r12, lr} + + pld [r0, r1, lsl #0] + pld [r2, r3, lsl #0] + + mov r8, #0 ; initialize sum = 0 + mov r11, #0 ; initialize sse = 0 + mov r12, #16 ; set loop counter to 16 (=block height) + +loop16x16 + ; 1st 4 pixels + ldr r4, [r0, #0] ; load 4 src pixels + ldr r5, [r2, #0] ; load 4 ref pixels + + mov lr, #0 ; constant zero + + usub8 r6, r4, r5 ; calculate difference + pld [r0, r1, lsl #1] + sel r7, r6, lr ; select bytes with positive difference + usub8 r9, r5, r4 ; calculate difference with reversed operands + pld [r2, r3, lsl #1] + sel r6, r9, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + ; calculate total sum + adds r8, r8, r4 ; add positive differences to sum + subs r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r10, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + + ; 2nd 4 pixels + ldr r4, [r0, #4] ; load 4 src pixels + ldr r5, [r2, #4] ; load 4 ref pixels + smlad r11, r10, r10, r11 ; dual signed multiply, add and accumulate (2) + + usub8 r6, r4, r5 ; calculate difference + sel r7, r6, lr ; select bytes with positive difference + usub8 r9, r5, r4 ; calculate difference with reversed operands + sel r6, r9, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + + ; calculate total sum + add r8, r8, r4 ; add positive differences to sum + sub r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r10, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + + ; 3rd 4 pixels + ldr r4, [r0, #8] ; load 4 src pixels + ldr r5, [r2, #8] ; load 4 ref pixels + smlad r11, r10, r10, r11 ; dual signed multiply, add and accumulate (2) + + usub8 r6, r4, r5 ; calculate difference + sel r7, r6, lr ; select bytes with positive difference + usub8 r9, r5, r4 ; calculate difference with reversed operands + sel r6, r9, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + + ; calculate total sum + add r8, r8, r4 ; add positive differences to sum + sub r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r10, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + + ; 4th 4 pixels + ldr r4, [r0, #12] ; load 4 src pixels + ldr r5, [r2, #12] ; load 4 ref pixels + smlad r11, r10, r10, r11 ; dual signed multiply, add and accumulate (2) + + usub8 r6, r4, r5 ; calculate difference + add r0, r0, r1 ; set src_ptr to next row + sel r7, r6, lr ; select bytes with positive difference + usub8 r9, r5, r4 ; calculate difference with reversed operands + add r2, r2, r3 ; set dst_ptr to next row + sel r6, r9, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r4, r7, lr ; calculate sum of positive differences + usad8 r5, r6, lr ; calculate sum of negative differences + orr r6, r6, r7 ; differences of all 4 pixels + + ; calculate total sum + add r8, r8, r4 ; add positive differences to sum + sub r8, r8, r5 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r5, r6 ; byte (two pixels) to halfwords + uxtb16 r10, r6, ror #8 ; another two pixels to halfwords + smlad r11, r5, r5, r11 ; dual signed multiply, add and accumulate (1) + smlad r11, r10, r10, r11 ; dual signed multiply, add and accumulate (2) + + + subs r12, r12, #1 + + bne loop16x16 + + ; return stuff + ldr r6, [sp, #40] ; get address of sse + mul r0, r8, r8 ; sum * sum + str r11, [r6] ; store sse + sub r0, r11, r0, lsr #8 ; return (sse - ((sum * sum) >> 8)) + + ldmfd sp!, {r4-r12, pc} + + ENDP + +; r0 unsigned char *src_ptr +; r1 int source_stride +; r2 unsigned char *ref_ptr +; r3 int recon_stride +; stack unsigned int *sse +|aom_variance8x8_media| PROC + + push {r4-r10, lr} + + pld [r0, r1, lsl #0] + pld [r2, r3, lsl #0] + + mov r12, #8 ; set loop counter to 8 (=block height) + mov r4, #0 ; initialize sum = 0 + mov r5, #0 ; initialize sse = 0 + +loop8x8 + ; 1st 4 pixels + ldr r6, [r0, #0x0] ; load 4 src pixels + ldr r7, [r2, #0x0] ; load 4 ref pixels + + mov lr, #0 ; constant zero + + usub8 r8, r6, r7 ; calculate difference + pld [r0, r1, lsl #1] + sel r10, r8, lr ; select bytes with positive difference + usub8 r9, r7, r6 ; calculate difference with reversed operands + pld [r2, r3, lsl #1] + sel r8, r9, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r6, r10, lr ; calculate sum of positive differences + usad8 r7, r8, lr ; calculate sum of negative differences + orr r8, r8, r10 ; differences of all 4 pixels + ; calculate total sum + add r4, r4, r6 ; add positive differences to sum + sub r4, r4, r7 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r7, r8 ; byte (two pixels) to halfwords + uxtb16 r10, r8, ror #8 ; another two pixels to halfwords + smlad r5, r7, r7, r5 ; dual signed multiply, add and accumulate (1) + + ; 2nd 4 pixels + ldr r6, [r0, #0x4] ; load 4 src pixels + ldr r7, [r2, #0x4] ; load 4 ref pixels + smlad r5, r10, r10, r5 ; dual signed multiply, add and accumulate (2) + + usub8 r8, r6, r7 ; calculate difference + add r0, r0, r1 ; set src_ptr to next row + sel r10, r8, lr ; select bytes with positive difference + usub8 r9, r7, r6 ; calculate difference with reversed operands + add r2, r2, r3 ; set dst_ptr to next row + sel r8, r9, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r6, r10, lr ; calculate sum of positive differences + usad8 r7, r8, lr ; calculate sum of negative differences + orr r8, r8, r10 ; differences of all 4 pixels + + ; calculate total sum + add r4, r4, r6 ; add positive differences to sum + sub r4, r4, r7 ; subtract negative differences from sum + + ; calculate sse + uxtb16 r7, r8 ; byte (two pixels) to halfwords + uxtb16 r10, r8, ror #8 ; another two pixels to halfwords + smlad r5, r7, r7, r5 ; dual signed multiply, add and accumulate (1) + subs r12, r12, #1 ; next row + smlad r5, r10, r10, r5 ; dual signed multiply, add and accumulate (2) + + bne loop8x8 + + ; return stuff + ldr r8, [sp, #32] ; get address of sse + mul r1, r4, r4 ; sum * sum + str r5, [r8] ; store sse + sub r0, r5, r1, ASR #6 ; return (sse - ((sum * sum) >> 6)) + + pop {r4-r10, pc} + + ENDP + +; r0 unsigned char *src_ptr +; r1 int source_stride +; r2 unsigned char *ref_ptr +; r3 int recon_stride +; stack unsigned int *sse +; +;note: Based on aom_variance16x16_media. In this function, sum is never used. +; So, we can remove this part of calculation. + +|aom_mse16x16_media| PROC + + push {r4-r9, lr} + + pld [r0, r1, lsl #0] + pld [r2, r3, lsl #0] + + mov r12, #16 ; set loop counter to 16 (=block height) + mov r4, #0 ; initialize sse = 0 + +loopmse + ; 1st 4 pixels + ldr r5, [r0, #0x0] ; load 4 src pixels + ldr r6, [r2, #0x0] ; load 4 ref pixels + + mov lr, #0 ; constant zero + + usub8 r8, r5, r6 ; calculate difference + pld [r0, r1, lsl #1] + sel r7, r8, lr ; select bytes with positive difference + usub8 r9, r6, r5 ; calculate difference with reversed operands + pld [r2, r3, lsl #1] + sel r8, r9, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r5, r7, lr ; calculate sum of positive differences + usad8 r6, r8, lr ; calculate sum of negative differences + orr r8, r8, r7 ; differences of all 4 pixels + + ldr r5, [r0, #0x4] ; load 4 src pixels + + ; calculate sse + uxtb16 r6, r8 ; byte (two pixels) to halfwords + uxtb16 r7, r8, ror #8 ; another two pixels to halfwords + smlad r4, r6, r6, r4 ; dual signed multiply, add and accumulate (1) + + ; 2nd 4 pixels + ldr r6, [r2, #0x4] ; load 4 ref pixels + smlad r4, r7, r7, r4 ; dual signed multiply, add and accumulate (2) + + usub8 r8, r5, r6 ; calculate difference + sel r7, r8, lr ; select bytes with positive difference + usub8 r9, r6, r5 ; calculate difference with reversed operands + sel r8, r9, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r5, r7, lr ; calculate sum of positive differences + usad8 r6, r8, lr ; calculate sum of negative differences + orr r8, r8, r7 ; differences of all 4 pixels + ldr r5, [r0, #0x8] ; load 4 src pixels + ; calculate sse + uxtb16 r6, r8 ; byte (two pixels) to halfwords + uxtb16 r7, r8, ror #8 ; another two pixels to halfwords + smlad r4, r6, r6, r4 ; dual signed multiply, add and accumulate (1) + + ; 3rd 4 pixels + ldr r6, [r2, #0x8] ; load 4 ref pixels + smlad r4, r7, r7, r4 ; dual signed multiply, add and accumulate (2) + + usub8 r8, r5, r6 ; calculate difference + sel r7, r8, lr ; select bytes with positive difference + usub8 r9, r6, r5 ; calculate difference with reversed operands + sel r8, r9, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r5, r7, lr ; calculate sum of positive differences + usad8 r6, r8, lr ; calculate sum of negative differences + orr r8, r8, r7 ; differences of all 4 pixels + + ldr r5, [r0, #0xc] ; load 4 src pixels + + ; calculate sse + uxtb16 r6, r8 ; byte (two pixels) to halfwords + uxtb16 r7, r8, ror #8 ; another two pixels to halfwords + smlad r4, r6, r6, r4 ; dual signed multiply, add and accumulate (1) + + ; 4th 4 pixels + ldr r6, [r2, #0xc] ; load 4 ref pixels + smlad r4, r7, r7, r4 ; dual signed multiply, add and accumulate (2) + + usub8 r8, r5, r6 ; calculate difference + add r0, r0, r1 ; set src_ptr to next row + sel r7, r8, lr ; select bytes with positive difference + usub8 r9, r6, r5 ; calculate difference with reversed operands + add r2, r2, r3 ; set dst_ptr to next row + sel r8, r9, lr ; select bytes with negative difference + + ; calculate partial sums + usad8 r5, r7, lr ; calculate sum of positive differences + usad8 r6, r8, lr ; calculate sum of negative differences + orr r8, r8, r7 ; differences of all 4 pixels + + subs r12, r12, #1 ; next row + + ; calculate sse + uxtb16 r6, r8 ; byte (two pixels) to halfwords + uxtb16 r7, r8, ror #8 ; another two pixels to halfwords + smlad r4, r6, r6, r4 ; dual signed multiply, add and accumulate (1) + smlad r4, r7, r7, r4 ; dual signed multiply, add and accumulate (2) + + bne loopmse + + ; return stuff + ldr r1, [sp, #28] ; get address of sse + mov r0, r4 ; return sse + str r4, [r1] ; store sse + + pop {r4-r9, pc} + + ENDP + + END diff --git a/third_party/aom/aom_dsp/arm/variance_neon.c b/third_party/aom/aom_dsp/arm/variance_neon.c new file mode 100644 index 0000000000..dbab287e35 --- /dev/null +++ b/third_party/aom/aom_dsp/arm/variance_neon.c @@ -0,0 +1,400 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "./aom_config.h" + +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" + +static INLINE int horizontal_add_s16x8(const int16x8_t v_16x8) { + const int32x4_t a = vpaddlq_s16(v_16x8); + const int64x2_t b = vpaddlq_s32(a); + const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)), + vreinterpret_s32_s64(vget_high_s64(b))); + return vget_lane_s32(c, 0); +} + +static INLINE int horizontal_add_s32x4(const int32x4_t v_32x4) { + const int64x2_t b = vpaddlq_s32(v_32x4); + const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)), + vreinterpret_s32_s64(vget_high_s64(b))); + return vget_lane_s32(c, 0); +} + +// w * h must be less than 2048 or local variable v_sum may overflow. +static void variance_neon_w8(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int w, int h, uint32_t *sse, + int *sum) { + int i, j; + int16x8_t v_sum = vdupq_n_s16(0); + int32x4_t v_sse_lo = vdupq_n_s32(0); + int32x4_t v_sse_hi = vdupq_n_s32(0); + + for (i = 0; i < h; ++i) { + for (j = 0; j < w; j += 8) { + const uint8x8_t v_a = vld1_u8(&a[j]); + const uint8x8_t v_b = vld1_u8(&b[j]); + const uint16x8_t v_diff = vsubl_u8(v_a, v_b); + const int16x8_t sv_diff = vreinterpretq_s16_u16(v_diff); + v_sum = vaddq_s16(v_sum, sv_diff); + v_sse_lo = + vmlal_s16(v_sse_lo, vget_low_s16(sv_diff), vget_low_s16(sv_diff)); + v_sse_hi = + vmlal_s16(v_sse_hi, vget_high_s16(sv_diff), vget_high_s16(sv_diff)); + } + a += a_stride; + b += b_stride; + } + + *sum = horizontal_add_s16x8(v_sum); + *sse = (unsigned int)horizontal_add_s32x4(vaddq_s32(v_sse_lo, v_sse_hi)); +} + +void aom_get8x8var_neon(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, unsigned int *sse, int *sum) { + variance_neon_w8(a, a_stride, b, b_stride, 8, 8, sse, sum); +} + +void aom_get16x16var_neon(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, unsigned int *sse, int *sum) { + variance_neon_w8(a, a_stride, b, b_stride, 16, 16, sse, sum); +} + +unsigned int aom_variance8x8_neon(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + unsigned int *sse) { + int sum; + variance_neon_w8(a, a_stride, b, b_stride, 8, 8, sse, &sum); + return *sse - ((sum * sum) >> 6); +} + +unsigned int aom_variance16x16_neon(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + unsigned int *sse) { + int sum; + variance_neon_w8(a, a_stride, b, b_stride, 16, 16, sse, &sum); + return *sse - (((unsigned int)((int64_t)sum * sum)) >> 8); +} + +unsigned int aom_variance32x32_neon(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + unsigned int *sse) { + int sum; + variance_neon_w8(a, a_stride, b, b_stride, 32, 32, sse, &sum); + return *sse - (unsigned int)(((int64_t)sum * sum) >> 10); +} + +unsigned int aom_variance32x64_neon(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + unsigned int *sse) { + int sum1, sum2; + uint32_t sse1, sse2; + variance_neon_w8(a, a_stride, b, b_stride, 32, 32, &sse1, &sum1); + variance_neon_w8(a + (32 * a_stride), a_stride, b + (32 * b_stride), b_stride, + 32, 32, &sse2, &sum2); + *sse = sse1 + sse2; + sum1 += sum2; + return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 11); +} + +unsigned int aom_variance64x32_neon(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + unsigned int *sse) { + int sum1, sum2; + uint32_t sse1, sse2; + variance_neon_w8(a, a_stride, b, b_stride, 64, 16, &sse1, &sum1); + variance_neon_w8(a + (16 * a_stride), a_stride, b + (16 * b_stride), b_stride, + 64, 16, &sse2, &sum2); + *sse = sse1 + sse2; + sum1 += sum2; + return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 11); +} + +unsigned int aom_variance64x64_neon(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + unsigned int *sse) { + int sum1, sum2; + uint32_t sse1, sse2; + + variance_neon_w8(a, a_stride, b, b_stride, 64, 16, &sse1, &sum1); + variance_neon_w8(a + (16 * a_stride), a_stride, b + (16 * b_stride), b_stride, + 64, 16, &sse2, &sum2); + sse1 += sse2; + sum1 += sum2; + + variance_neon_w8(a + (16 * 2 * a_stride), a_stride, b + (16 * 2 * b_stride), + b_stride, 64, 16, &sse2, &sum2); + sse1 += sse2; + sum1 += sum2; + + variance_neon_w8(a + (16 * 3 * a_stride), a_stride, b + (16 * 3 * b_stride), + b_stride, 64, 16, &sse2, &sum2); + *sse = sse1 + sse2; + sum1 += sum2; + return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 12); +} + +unsigned int aom_variance16x8_neon(const unsigned char *src_ptr, + int source_stride, + const unsigned char *ref_ptr, + int recon_stride, unsigned int *sse) { + int i; + int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16; + uint32x2_t d0u32, d10u32; + int64x1_t d0s64, d1s64; + uint8x16_t q0u8, q1u8, q2u8, q3u8; + uint16x8_t q11u16, q12u16, q13u16, q14u16; + int32x4_t q8s32, q9s32, q10s32; + int64x2_t q0s64, q1s64, q5s64; + + q8s32 = vdupq_n_s32(0); + q9s32 = vdupq_n_s32(0); + q10s32 = vdupq_n_s32(0); + + for (i = 0; i < 4; i++) { + q0u8 = vld1q_u8(src_ptr); + src_ptr += source_stride; + q1u8 = vld1q_u8(src_ptr); + src_ptr += source_stride; + __builtin_prefetch(src_ptr); + + q2u8 = vld1q_u8(ref_ptr); + ref_ptr += recon_stride; + q3u8 = vld1q_u8(ref_ptr); + ref_ptr += recon_stride; + __builtin_prefetch(ref_ptr); + + q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8)); + q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8)); + q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8)); + q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8)); + + d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16)); + d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16)); + q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q11u16)); + q9s32 = vmlal_s16(q9s32, d22s16, d22s16); + q10s32 = vmlal_s16(q10s32, d23s16, d23s16); + + d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16)); + d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16)); + q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q12u16)); + q9s32 = vmlal_s16(q9s32, d24s16, d24s16); + q10s32 = vmlal_s16(q10s32, d25s16, d25s16); + + d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16)); + d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16)); + q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q13u16)); + q9s32 = vmlal_s16(q9s32, d26s16, d26s16); + q10s32 = vmlal_s16(q10s32, d27s16, d27s16); + + d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16)); + d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16)); + q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q14u16)); + q9s32 = vmlal_s16(q9s32, d28s16, d28s16); + q10s32 = vmlal_s16(q10s32, d29s16, d29s16); + } + + q10s32 = vaddq_s32(q10s32, q9s32); + q0s64 = vpaddlq_s32(q8s32); + q1s64 = vpaddlq_s32(q10s32); + + d0s64 = vadd_s64(vget_low_s64(q0s64), vget_high_s64(q0s64)); + d1s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64)); + + q5s64 = vmull_s32(vreinterpret_s32_s64(d0s64), vreinterpret_s32_s64(d0s64)); + vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d1s64), 0); + + d10u32 = vshr_n_u32(vreinterpret_u32_s64(vget_low_s64(q5s64)), 7); + d0u32 = vsub_u32(vreinterpret_u32_s64(d1s64), d10u32); + + return vget_lane_u32(d0u32, 0); +} + +unsigned int aom_variance8x16_neon(const unsigned char *src_ptr, + int source_stride, + const unsigned char *ref_ptr, + int recon_stride, unsigned int *sse) { + int i; + uint8x8_t d0u8, d2u8, d4u8, d6u8; + int16x4_t d22s16, d23s16, d24s16, d25s16; + uint32x2_t d0u32, d10u32; + int64x1_t d0s64, d1s64; + uint16x8_t q11u16, q12u16; + int32x4_t q8s32, q9s32, q10s32; + int64x2_t q0s64, q1s64, q5s64; + + q8s32 = vdupq_n_s32(0); + q9s32 = vdupq_n_s32(0); + q10s32 = vdupq_n_s32(0); + + for (i = 0; i < 8; i++) { + d0u8 = vld1_u8(src_ptr); + src_ptr += source_stride; + d2u8 = vld1_u8(src_ptr); + src_ptr += source_stride; + __builtin_prefetch(src_ptr); + + d4u8 = vld1_u8(ref_ptr); + ref_ptr += recon_stride; + d6u8 = vld1_u8(ref_ptr); + ref_ptr += recon_stride; + __builtin_prefetch(ref_ptr); + + q11u16 = vsubl_u8(d0u8, d4u8); + q12u16 = vsubl_u8(d2u8, d6u8); + + d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16)); + d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16)); + q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q11u16)); + q9s32 = vmlal_s16(q9s32, d22s16, d22s16); + q10s32 = vmlal_s16(q10s32, d23s16, d23s16); + + d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16)); + d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16)); + q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q12u16)); + q9s32 = vmlal_s16(q9s32, d24s16, d24s16); + q10s32 = vmlal_s16(q10s32, d25s16, d25s16); + } + + q10s32 = vaddq_s32(q10s32, q9s32); + q0s64 = vpaddlq_s32(q8s32); + q1s64 = vpaddlq_s32(q10s32); + + d0s64 = vadd_s64(vget_low_s64(q0s64), vget_high_s64(q0s64)); + d1s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64)); + + q5s64 = vmull_s32(vreinterpret_s32_s64(d0s64), vreinterpret_s32_s64(d0s64)); + vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d1s64), 0); + + d10u32 = vshr_n_u32(vreinterpret_u32_s64(vget_low_s64(q5s64)), 7); + d0u32 = vsub_u32(vreinterpret_u32_s64(d1s64), d10u32); + + return vget_lane_u32(d0u32, 0); +} + +unsigned int aom_mse16x16_neon(const unsigned char *src_ptr, int source_stride, + const unsigned char *ref_ptr, int recon_stride, + unsigned int *sse) { + int i; + int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16; + int64x1_t d0s64; + uint8x16_t q0u8, q1u8, q2u8, q3u8; + int32x4_t q7s32, q8s32, q9s32, q10s32; + uint16x8_t q11u16, q12u16, q13u16, q14u16; + int64x2_t q1s64; + + q7s32 = vdupq_n_s32(0); + q8s32 = vdupq_n_s32(0); + q9s32 = vdupq_n_s32(0); + q10s32 = vdupq_n_s32(0); + + for (i = 0; i < 8; i++) { // mse16x16_neon_loop + q0u8 = vld1q_u8(src_ptr); + src_ptr += source_stride; + q1u8 = vld1q_u8(src_ptr); + src_ptr += source_stride; + q2u8 = vld1q_u8(ref_ptr); + ref_ptr += recon_stride; + q3u8 = vld1q_u8(ref_ptr); + ref_ptr += recon_stride; + + q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8)); + q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8)); + q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8)); + q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8)); + + d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16)); + d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16)); + q7s32 = vmlal_s16(q7s32, d22s16, d22s16); + q8s32 = vmlal_s16(q8s32, d23s16, d23s16); + + d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16)); + d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16)); + q9s32 = vmlal_s16(q9s32, d24s16, d24s16); + q10s32 = vmlal_s16(q10s32, d25s16, d25s16); + + d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16)); + d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16)); + q7s32 = vmlal_s16(q7s32, d26s16, d26s16); + q8s32 = vmlal_s16(q8s32, d27s16, d27s16); + + d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16)); + d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16)); + q9s32 = vmlal_s16(q9s32, d28s16, d28s16); + q10s32 = vmlal_s16(q10s32, d29s16, d29s16); + } + + q7s32 = vaddq_s32(q7s32, q8s32); + q9s32 = vaddq_s32(q9s32, q10s32); + q10s32 = vaddq_s32(q7s32, q9s32); + + q1s64 = vpaddlq_s32(q10s32); + d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64)); + + vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d0s64), 0); + return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0); +} + +unsigned int aom_get4x4sse_cs_neon(const unsigned char *src_ptr, + int source_stride, + const unsigned char *ref_ptr, + int recon_stride) { + int16x4_t d22s16, d24s16, d26s16, d28s16; + int64x1_t d0s64; + uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8; + int32x4_t q7s32, q8s32, q9s32, q10s32; + uint16x8_t q11u16, q12u16, q13u16, q14u16; + int64x2_t q1s64; + + d0u8 = vld1_u8(src_ptr); + src_ptr += source_stride; + d4u8 = vld1_u8(ref_ptr); + ref_ptr += recon_stride; + d1u8 = vld1_u8(src_ptr); + src_ptr += source_stride; + d5u8 = vld1_u8(ref_ptr); + ref_ptr += recon_stride; + d2u8 = vld1_u8(src_ptr); + src_ptr += source_stride; + d6u8 = vld1_u8(ref_ptr); + ref_ptr += recon_stride; + d3u8 = vld1_u8(src_ptr); + src_ptr += source_stride; + d7u8 = vld1_u8(ref_ptr); + ref_ptr += recon_stride; + + q11u16 = vsubl_u8(d0u8, d4u8); + q12u16 = vsubl_u8(d1u8, d5u8); + q13u16 = vsubl_u8(d2u8, d6u8); + q14u16 = vsubl_u8(d3u8, d7u8); + + d22s16 = vget_low_s16(vreinterpretq_s16_u16(q11u16)); + d24s16 = vget_low_s16(vreinterpretq_s16_u16(q12u16)); + d26s16 = vget_low_s16(vreinterpretq_s16_u16(q13u16)); + d28s16 = vget_low_s16(vreinterpretq_s16_u16(q14u16)); + + q7s32 = vmull_s16(d22s16, d22s16); + q8s32 = vmull_s16(d24s16, d24s16); + q9s32 = vmull_s16(d26s16, d26s16); + q10s32 = vmull_s16(d28s16, d28s16); + + q7s32 = vaddq_s32(q7s32, q8s32); + q9s32 = vaddq_s32(q9s32, q10s32); + q9s32 = vaddq_s32(q7s32, q9s32); + + q1s64 = vpaddlq_s32(q9s32); + d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64)); + + return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0); +} diff --git a/third_party/aom/aom_dsp/avg.c b/third_party/aom/aom_dsp/avg.c new file mode 100644 index 0000000000..eb60597052 --- /dev/null +++ b/third_party/aom/aom_dsp/avg.c @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_ports/mem.h" + +unsigned int aom_avg_8x8_c(const uint8_t *src, int stride) { + int i, j; + int sum = 0; + for (i = 0; i < 8; ++i, src += stride) + for (j = 0; j < 8; sum += src[j], ++j) { + } + + return ROUND_POWER_OF_TWO(sum, 6); +} + +unsigned int aom_avg_4x4_c(const uint8_t *src, int stride) { + int i, j; + int sum = 0; + for (i = 0; i < 4; ++i, src += stride) + for (j = 0; j < 4; sum += src[j], ++j) { + } + + return ROUND_POWER_OF_TWO(sum, 4); +} + +// src_diff: first pass, 9 bit, dynamic range [-255, 255] +// second pass, 12 bit, dynamic range [-2040, 2040] +static void hadamard_col8(const int16_t *src_diff, int src_stride, + int16_t *coeff) { + int16_t b0 = src_diff[0 * src_stride] + src_diff[1 * src_stride]; + int16_t b1 = src_diff[0 * src_stride] - src_diff[1 * src_stride]; + int16_t b2 = src_diff[2 * src_stride] + src_diff[3 * src_stride]; + int16_t b3 = src_diff[2 * src_stride] - src_diff[3 * src_stride]; + int16_t b4 = src_diff[4 * src_stride] + src_diff[5 * src_stride]; + int16_t b5 = src_diff[4 * src_stride] - src_diff[5 * src_stride]; + int16_t b6 = src_diff[6 * src_stride] + src_diff[7 * src_stride]; + int16_t b7 = src_diff[6 * src_stride] - src_diff[7 * src_stride]; + + int16_t c0 = b0 + b2; + int16_t c1 = b1 + b3; + int16_t c2 = b0 - b2; + int16_t c3 = b1 - b3; + int16_t c4 = b4 + b6; + int16_t c5 = b5 + b7; + int16_t c6 = b4 - b6; + int16_t c7 = b5 - b7; + + coeff[0] = c0 + c4; + coeff[7] = c1 + c5; + coeff[3] = c2 + c6; + coeff[4] = c3 + c7; + coeff[2] = c0 - c4; + coeff[6] = c1 - c5; + coeff[1] = c2 - c6; + coeff[5] = c3 - c7; +} + +// The order of the output coeff of the hadamard is not important. For +// optimization purposes the final transpose may be skipped. +void aom_hadamard_8x8_c(const int16_t *src_diff, int src_stride, + int16_t *coeff) { + int idx; + int16_t buffer[64]; + int16_t *tmp_buf = &buffer[0]; + for (idx = 0; idx < 8; ++idx) { + hadamard_col8(src_diff, src_stride, tmp_buf); // src_diff: 9 bit + // dynamic range [-255, 255] + tmp_buf += 8; + ++src_diff; + } + + tmp_buf = &buffer[0]; + for (idx = 0; idx < 8; ++idx) { + hadamard_col8(tmp_buf, 8, coeff); // tmp_buf: 12 bit + // dynamic range [-2040, 2040] + coeff += 8; // coeff: 15 bit + // dynamic range [-16320, 16320] + ++tmp_buf; + } +} + +// In place 16x16 2D Hadamard transform +void aom_hadamard_16x16_c(const int16_t *src_diff, int src_stride, + int16_t *coeff) { + int idx; + for (idx = 0; idx < 4; ++idx) { + // src_diff: 9 bit, dynamic range [-255, 255] + const int16_t *src_ptr = + src_diff + (idx >> 1) * 8 * src_stride + (idx & 0x01) * 8; + aom_hadamard_8x8_c(src_ptr, src_stride, coeff + idx * 64); + } + + // coeff: 15 bit, dynamic range [-16320, 16320] + for (idx = 0; idx < 64; ++idx) { + int16_t a0 = coeff[0]; + int16_t a1 = coeff[64]; + int16_t a2 = coeff[128]; + int16_t a3 = coeff[192]; + + int16_t b0 = (a0 + a1) >> 1; // (a0 + a1): 16 bit, [-32640, 32640] + int16_t b1 = (a0 - a1) >> 1; // b0-b3: 15 bit, dynamic range + int16_t b2 = (a2 + a3) >> 1; // [-16320, 16320] + int16_t b3 = (a2 - a3) >> 1; + + coeff[0] = b0 + b2; // 16 bit, [-32640, 32640] + coeff[64] = b1 + b3; + coeff[128] = b0 - b2; + coeff[192] = b1 - b3; + + ++coeff; + } +} + +// coeff: 16 bits, dynamic range [-32640, 32640]. +// length: value range {16, 64, 256, 1024}. +int aom_satd_c(const int16_t *coeff, int length) { + int i; + int satd = 0; + for (i = 0; i < length; ++i) satd += abs(coeff[i]); + + // satd: 26 bits, dynamic range [-32640 * 1024, 32640 * 1024] + return satd; +} + +// Integer projection onto row vectors. +// height: value range {16, 32, 64}. +void aom_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, int ref_stride, + int height) { + int idx; + const int norm_factor = height >> 1; + for (idx = 0; idx < 16; ++idx) { + int i; + hbuf[idx] = 0; + // hbuf[idx]: 14 bit, dynamic range [0, 16320]. + for (i = 0; i < height; ++i) hbuf[idx] += ref[i * ref_stride]; + // hbuf[idx]: 9 bit, dynamic range [0, 510]. + hbuf[idx] /= norm_factor; + ++ref; + } +} + +// width: value range {16, 32, 64}. +int16_t aom_int_pro_col_c(const uint8_t *ref, int width) { + int idx; + int16_t sum = 0; + // sum: 14 bit, dynamic range [0, 16320] + for (idx = 0; idx < width; ++idx) sum += ref[idx]; + return sum; +} + +// ref: [0 - 510] +// src: [0 - 510] +// bwl: {2, 3, 4} +int aom_vector_var_c(const int16_t *ref, const int16_t *src, int bwl) { + int i; + int width = 4 << bwl; + int sse = 0, mean = 0, var; + + for (i = 0; i < width; ++i) { + int diff = ref[i] - src[i]; // diff: dynamic range [-510, 510], 10 bits. + mean += diff; // mean: dynamic range 16 bits. + sse += diff * diff; // sse: dynamic range 26 bits. + } + + // (mean * mean): dynamic range 31 bits. + var = sse - ((mean * mean) >> (bwl + 2)); + return var; +} + +void aom_minmax_8x8_c(const uint8_t *src, int src_stride, const uint8_t *ref, + int ref_stride, int *min, int *max) { + int i, j; + *min = 255; + *max = 0; + for (i = 0; i < 8; ++i, src += src_stride, ref += ref_stride) { + for (j = 0; j < 8; ++j) { + int diff = abs(src[j] - ref[j]); + *min = diff < *min ? diff : *min; + *max = diff > *max ? diff : *max; + } + } +} + +#if CONFIG_HIGHBITDEPTH +unsigned int aom_highbd_avg_8x8_c(const uint8_t *src, int stride) { + int i, j; + int sum = 0; + const uint16_t *s = CONVERT_TO_SHORTPTR(src); + for (i = 0; i < 8; ++i, s += stride) + for (j = 0; j < 8; sum += s[j], ++j) { + } + + return ROUND_POWER_OF_TWO(sum, 6); +} + +unsigned int aom_highbd_avg_4x4_c(const uint8_t *src, int stride) { + int i, j; + int sum = 0; + const uint16_t *s = CONVERT_TO_SHORTPTR(src); + for (i = 0; i < 4; ++i, s += stride) + for (j = 0; j < 4; sum += s[j], ++j) { + } + + return ROUND_POWER_OF_TWO(sum, 4); +} + +void aom_highbd_minmax_8x8_c(const uint8_t *s8, int p, const uint8_t *d8, + int dp, int *min, int *max) { + int i, j; + const uint16_t *s = CONVERT_TO_SHORTPTR(s8); + const uint16_t *d = CONVERT_TO_SHORTPTR(d8); + *min = 255; + *max = 0; + for (i = 0; i < 8; ++i, s += p, d += dp) { + for (j = 0; j < 8; ++j) { + int diff = abs(s[j] - d[j]); + *min = diff < *min ? diff : *min; + *max = diff > *max ? diff : *max; + } + } +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/binary_codes_reader.c b/third_party/aom/aom_dsp/binary_codes_reader.c new file mode 100644 index 0000000000..96c4cb436d --- /dev/null +++ b/third_party/aom/aom_dsp/binary_codes_reader.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/bitreader.h" + +#include "av1/common/common.h" + +// Inverse recenters a non-negative literal v around a reference r +static uint16_t inv_recenter_nonneg(uint16_t r, uint16_t v) { + if (v > (r << 1)) + return v; + else if ((v & 1) == 0) + return (v >> 1) + r; + else + return r - ((v + 1) >> 1); +} + +// Inverse recenters a non-negative literal v in [0, n-1] around a +// reference r also in [0, n-1] +static uint16_t inv_recenter_finite_nonneg(uint16_t n, uint16_t r, uint16_t v) { + if ((r << 1) <= n) { + return inv_recenter_nonneg(r, v); + } else { + return n - 1 - inv_recenter_nonneg(n - 1 - r, v); + } +} + +int16_t aom_read_primitive_symmetric(aom_reader *r, unsigned int mag_bits) { + if (aom_read_bit(r, NULL)) { + int s = aom_read_bit(r, NULL); + int16_t x = aom_read_literal(r, mag_bits, NULL) + 1; + return (s > 0 ? -x : x); + } else { + return 0; + } +} + +uint16_t aom_read_primitive_quniform(aom_reader *r, uint16_t n) { + if (n <= 1) return 0; + const int l = get_msb(n - 1) + 1; + const int m = (1 << l) - n; + const int v = aom_read_literal(r, l - 1, NULL); + return v < m ? v : (v << 1) - m + aom_read_bit(r, NULL); +} + +uint16_t aom_read_primitive_refbilevel(aom_reader *r, uint16_t n, uint16_t p, + uint16_t ref) { + if (n <= 1) return 0; + assert(p > 0 && p <= n); + assert(ref < n); + int lolimit = ref - p / 2; + const int hilimit = lolimit + p - 1; + if (lolimit < 0) { + lolimit = 0; + } else if (hilimit >= n) { + lolimit = n - p; + } + int v; + if (aom_read_bit(r, NULL)) { + v = aom_read_primitive_quniform(r, p) + lolimit; + } else { + v = aom_read_primitive_quniform(r, n - p); + if (v >= lolimit) v += p; + } + return v; +} + +// Decode finite subexponential code that for a symbol v in [0, n-1] with +// parameter k +uint16_t aom_read_primitive_subexpfin(aom_reader *r, uint16_t n, uint16_t k) { + int i = 0; + int mk = 0; + uint16_t v; + while (1) { + int b = (i ? k + i - 1 : k); + int a = (1 << b); + if (n <= mk + 3 * a) { + v = aom_read_primitive_quniform(r, n - mk) + mk; + break; + } else { + if (aom_read_bit(r, NULL)) { + i = i + 1; + mk += a; + } else { + v = aom_read_literal(r, b, NULL) + mk; + break; + } + } + } + return v; +} + +// Decode finite subexponential code that for a symbol v in [0, n-1] with +// parameter k +// based on a reference ref also in [0, n-1]. +uint16_t aom_read_primitive_refsubexpfin(aom_reader *r, uint16_t n, uint16_t k, + uint16_t ref) { + return inv_recenter_finite_nonneg(n, ref, + aom_read_primitive_subexpfin(r, n, k)); +} + +// Decode finite subexponential code that for a symbol v in [-(n-1), n-1] with +// parameter k based on a reference ref also in [-(n-1), n-1]. +int16_t aom_read_signed_primitive_refsubexpfin(aom_reader *r, uint16_t n, + uint16_t k, int16_t ref) { + ref += n - 1; + const uint16_t scaled_n = (n << 1) - 1; + return aom_read_primitive_refsubexpfin(r, scaled_n, k, ref) - n + 1; +} diff --git a/third_party/aom/aom_dsp/binary_codes_reader.h b/third_party/aom/aom_dsp/binary_codes_reader.h new file mode 100644 index 0000000000..738d91da83 --- /dev/null +++ b/third_party/aom/aom_dsp/binary_codes_reader.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_BINARY_CODES_READER_H_ +#define AOM_DSP_BINARY_CODES_READER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/bitreader.h" + +int16_t aom_read_primitive_symmetric(aom_reader *r, unsigned int mag_bits); + +uint16_t aom_read_primitive_quniform(aom_reader *r, uint16_t n); +uint16_t aom_read_primitive_refbilevel(aom_reader *r, uint16_t n, uint16_t p, + uint16_t ref); +uint16_t aom_read_primitive_subexpfin(aom_reader *r, uint16_t n, uint16_t k); +uint16_t aom_read_primitive_refsubexpfin(aom_reader *r, uint16_t n, uint16_t k, + uint16_t ref); +int16_t aom_read_signed_primitive_refsubexpfin(aom_reader *r, uint16_t n, + uint16_t k, int16_t ref); +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_BINARY_CODES_READER_H_ diff --git a/third_party/aom/aom_dsp/binary_codes_writer.c b/third_party/aom/aom_dsp/binary_codes_writer.c new file mode 100644 index 0000000000..91e807b299 --- /dev/null +++ b/third_party/aom/aom_dsp/binary_codes_writer.c @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/bitwriter.h" + +#include "av1/common/common.h" + +// Recenters a non-negative literal v around a reference r +static uint16_t recenter_nonneg(uint16_t r, uint16_t v) { + if (v > (r << 1)) + return v; + else if (v >= r) + return ((v - r) << 1); + else + return ((r - v) << 1) - 1; +} + +// Recenters a non-negative literal v in [0, n-1] around a +// reference r also in [0, n-1] +static uint16_t recenter_finite_nonneg(uint16_t n, uint16_t r, uint16_t v) { + if ((r << 1) <= n) { + return recenter_nonneg(r, v); + } else { + return recenter_nonneg(n - 1 - r, n - 1 - v); + } +} + +// Codes a symbol v in [-2^mag_bits, 2^mag_bits]. +// mag_bits is number of bits for magnitude. The alphabet is of size +// 2 * 2^mag_bits + 1, symmetric around 0, where one bit is used to +// indicate 0 or non-zero, mag_bits bits are used to indicate magnitide +// and 1 more bit for the sign if non-zero. +void aom_write_primitive_symmetric(aom_writer *w, int16_t v, + unsigned int abs_bits) { + if (v == 0) { + aom_write_bit(w, 0); + } else { + const int x = abs(v); + const int s = v < 0; + aom_write_bit(w, 1); + aom_write_bit(w, s); + aom_write_literal(w, x - 1, abs_bits); + } +} + +int aom_count_primitive_symmetric(int16_t v, unsigned int abs_bits) { + return (v == 0 ? 1 : abs_bits + 2); +} + +// Encodes a value v in [0, n-1] quasi-uniformly +void aom_write_primitive_quniform(aom_writer *w, uint16_t n, uint16_t v) { + if (n <= 1) return; + const int l = get_msb(n - 1) + 1; + const int m = (1 << l) - n; + if (v < m) { + aom_write_literal(w, v, l - 1); + } else { + aom_write_literal(w, m + ((v - m) >> 1), l - 1); + aom_write_bit(w, (v - m) & 1); + } +} + +int aom_count_primitive_quniform(uint16_t n, uint16_t v) { + if (n <= 1) return 0; + const int l = get_msb(n - 1) + 1; + const int m = (1 << l) - n; + return v < m ? l - 1 : l; +} + +// Encodes a value v in [0, n-1] based on a reference ref also in [0, n-1] +// The closest p values of v from ref are coded using a p-ary quasi-unoform +// short code while the remaining n-p values are coded with a longer code. +void aom_write_primitive_refbilevel(aom_writer *w, uint16_t n, uint16_t p, + uint16_t ref, uint16_t v) { + if (n <= 1) return; + assert(p > 0 && p <= n); + assert(ref < n); + int lolimit = ref - p / 2; + int hilimit = lolimit + p - 1; + if (lolimit < 0) { + lolimit = 0; + hilimit = p - 1; + } else if (hilimit >= n) { + hilimit = n - 1; + lolimit = n - p; + } + if (v >= lolimit && v <= hilimit) { + aom_write_bit(w, 1); + v = v - lolimit; + aom_write_primitive_quniform(w, p, v); + } else { + aom_write_bit(w, 0); + if (v > hilimit) v -= p; + aom_write_primitive_quniform(w, n - p, v); + } +} + +int aom_count_primitive_refbilevel(uint16_t n, uint16_t p, uint16_t ref, + uint16_t v) { + if (n <= 1) return 0; + assert(p > 0 && p <= n); + assert(ref < n); + int lolimit = ref - p / 2; + int hilimit = lolimit + p - 1; + if (lolimit < 0) { + lolimit = 0; + hilimit = p - 1; + } else if (hilimit >= n) { + hilimit = n - 1; + lolimit = n - p; + } + int count = 0; + if (v >= lolimit && v <= hilimit) { + count++; + v = v - lolimit; + count += aom_count_primitive_quniform(p, v); + } else { + count++; + if (v > hilimit) v -= p; + count += aom_count_primitive_quniform(n - p, v); + } + return count; +} + +// Finite subexponential code that codes a symbol v in [0, n-1] with parameter k +void aom_write_primitive_subexpfin(aom_writer *w, uint16_t n, uint16_t k, + uint16_t v) { + int i = 0; + int mk = 0; + while (1) { + int b = (i ? k + i - 1 : k); + int a = (1 << b); + if (n <= mk + 3 * a) { + aom_write_primitive_quniform(w, n - mk, v - mk); + break; + } else { + int t = (v >= mk + a); + aom_write_bit(w, t); + if (t) { + i = i + 1; + mk += a; + } else { + aom_write_literal(w, v - mk, b); + break; + } + } + } +} + +int aom_count_primitive_subexpfin(uint16_t n, uint16_t k, uint16_t v) { + int count = 0; + int i = 0; + int mk = 0; + while (1) { + int b = (i ? k + i - 1 : k); + int a = (1 << b); + if (n <= mk + 3 * a) { + count += aom_count_primitive_quniform(n - mk, v - mk); + break; + } else { + int t = (v >= mk + a); + count++; + if (t) { + i = i + 1; + mk += a; + } else { + count += b; + break; + } + } + } + return count; +} + +// Finite subexponential code that codes a symbol v in [0, n-1] with parameter k +// based on a reference ref also in [0, n-1]. +// Recenters symbol around r first and then uses a finite subexponential code. +void aom_write_primitive_refsubexpfin(aom_writer *w, uint16_t n, uint16_t k, + int16_t ref, int16_t v) { + aom_write_primitive_subexpfin(w, n, k, recenter_finite_nonneg(n, ref, v)); +} + +void aom_write_signed_primitive_refsubexpfin(aom_writer *w, uint16_t n, + uint16_t k, uint16_t ref, + uint16_t v) { + ref += n - 1; + v += n - 1; + const uint16_t scaled_n = (n << 1) - 1; + aom_write_primitive_refsubexpfin(w, scaled_n, k, ref, v); +} + +int aom_count_primitive_refsubexpfin(uint16_t n, uint16_t k, uint16_t ref, + uint16_t v) { + return aom_count_primitive_subexpfin(n, k, recenter_finite_nonneg(n, ref, v)); +} + +int aom_count_signed_primitive_refsubexpfin(uint16_t n, uint16_t k, int16_t ref, + int16_t v) { + ref += n - 1; + v += n - 1; + const uint16_t scaled_n = (n << 1) - 1; + return aom_count_primitive_refsubexpfin(scaled_n, k, ref, v); +} diff --git a/third_party/aom/aom_dsp/binary_codes_writer.h b/third_party/aom/aom_dsp/binary_codes_writer.h new file mode 100644 index 0000000000..ab5ccbf15c --- /dev/null +++ b/third_party/aom/aom_dsp/binary_codes_writer.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_BINARY_CODES_WRITER_H_ +#define AOM_DSP_BINARY_CODES_WRITER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/bitwriter.h" + +// Codes a symbol v in [-2^mag_bits, 2^mag_bits] +// mag_bits is number of bits for magnitude. The alphabet is of size +// 2 * 2^mag_bits + 1, symmetric around 0, where one bit is used to +// indicate 0 or non-zero, mag_bits bits are used to indicate magnitide +// and 1 more bit for the sign if non-zero. +void aom_write_primitive_symmetric(aom_writer *w, int16_t v, + unsigned int mag_bits); + +// Encodes a value v in [0, n-1] quasi-uniformly +void aom_write_primitive_quniform(aom_writer *w, uint16_t n, uint16_t v); + +// Encodes a value v in [0, n-1] based on a reference ref also in [0, n-1] +// The closest p values of v from ref are coded using a p-ary quasi-unoform +// short code while the remaining n-p values are coded with a longer code. +void aom_write_primitive_refbilevel(aom_writer *w, uint16_t n, uint16_t p, + uint16_t ref, uint16_t v); + +// Finite subexponential code that codes a symbol v in [0, n-1] with parameter k +void aom_write_primitive_subexpfin(aom_writer *w, uint16_t n, uint16_t k, + uint16_t v); + +// Finite subexponential code that codes a symbol v in [0, n-1] with parameter k +// based on a reference ref also in [0, n-1]. +void aom_write_primitive_refsubexpfin(aom_writer *w, uint16_t n, uint16_t k, + uint16_t ref, uint16_t v); + +// Finite subexponential code that codes a symbol v in [-(n-1), n-1] with +// parameter k based on a reference ref also in [-(n-1), n-1]. +void aom_write_signed_primitive_refsubexpfin(aom_writer *w, uint16_t n, + uint16_t k, int16_t ref, + int16_t v); + +// Functions that counts bits for the above primitives +int aom_count_primitive_symmetric(int16_t v, unsigned int mag_bits); +int aom_count_primitive_quniform(uint16_t n, uint16_t v); +int aom_count_primitive_refbilevel(uint16_t n, uint16_t p, uint16_t ref, + uint16_t v); +int aom_count_primitive_subexpfin(uint16_t n, uint16_t k, uint16_t v); +int aom_count_primitive_refsubexpfin(uint16_t n, uint16_t k, uint16_t ref, + uint16_t v); +int aom_count_signed_primitive_refsubexpfin(uint16_t n, uint16_t k, int16_t ref, + int16_t v); +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_BINARY_CODES_WRITER_H_ diff --git a/third_party/aom/aom_dsp/bitreader.h b/third_party/aom/aom_dsp/bitreader.h new file mode 100644 index 0000000000..9cd34dd483 --- /dev/null +++ b/third_party/aom/aom_dsp/bitreader.h @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_BITREADER_H_ +#define AOM_DSP_BITREADER_H_ + +#include +#include + +#include "./aom_config.h" +#if CONFIG_EC_ADAPT && !CONFIG_EC_MULTISYMBOL +#error "CONFIG_EC_ADAPT is enabled without enabling CONFIG_EC_MULTISYMBOL." +#endif + +#include "aom/aomdx.h" +#include "aom/aom_integer.h" +#if CONFIG_ANS +#include "aom_dsp/ansreader.h" +#elif CONFIG_DAALA_EC +#include "aom_dsp/daalaboolreader.h" +#else +#include "aom_dsp/dkboolreader.h" +#endif +#include "aom_dsp/prob.h" +#include "av1/common/odintrin.h" + +#if CONFIG_ACCOUNTING +#include "av1/decoder/accounting.h" +#define ACCT_STR_NAME acct_str +#define ACCT_STR_PARAM , const char *ACCT_STR_NAME +#define ACCT_STR_ARG(s) , s +#else +#define ACCT_STR_PARAM +#define ACCT_STR_ARG(s) +#endif + +#define aom_read(r, prob, ACCT_STR_NAME) \ + aom_read_(r, prob ACCT_STR_ARG(ACCT_STR_NAME)) +#define aom_read_bit(r, ACCT_STR_NAME) \ + aom_read_bit_(r ACCT_STR_ARG(ACCT_STR_NAME)) +#define aom_read_tree(r, tree, probs, ACCT_STR_NAME) \ + aom_read_tree_(r, tree, probs ACCT_STR_ARG(ACCT_STR_NAME)) +#define aom_read_literal(r, bits, ACCT_STR_NAME) \ + aom_read_literal_(r, bits ACCT_STR_ARG(ACCT_STR_NAME)) +#define aom_read_cdf(r, cdf, nsymbs, ACCT_STR_NAME) \ + aom_read_cdf_(r, cdf, nsymbs ACCT_STR_ARG(ACCT_STR_NAME)) +#define aom_read_symbol(r, cdf, nsymbs, ACCT_STR_NAME) \ + aom_read_symbol_(r, cdf, nsymbs ACCT_STR_ARG(ACCT_STR_NAME)) + +#ifdef __cplusplus +extern "C" { +#endif + +#if CONFIG_ANS +typedef struct AnsDecoder aom_reader; +#elif CONFIG_DAALA_EC +typedef struct daala_reader aom_reader; +#else +typedef struct aom_dk_reader aom_reader; +#endif + +static INLINE int aom_reader_init(aom_reader *r, const uint8_t *buffer, + size_t size, aom_decrypt_cb decrypt_cb, + void *decrypt_state) { +#if CONFIG_ANS + (void)decrypt_cb; + (void)decrypt_state; + if (size > INT_MAX) return 1; + return ans_read_init(r, buffer, (int)size); +#elif CONFIG_DAALA_EC + (void)decrypt_cb; + (void)decrypt_state; + return aom_daala_reader_init(r, buffer, (int)size); +#else + return aom_dk_reader_init(r, buffer, size, decrypt_cb, decrypt_state); +#endif +} + +static INLINE const uint8_t *aom_reader_find_end(aom_reader *r) { +#if CONFIG_ANS + (void)r; + assert(0 && "Use the raw buffer size with ANS"); + return NULL; +#elif CONFIG_DAALA_EC + return aom_daala_reader_find_end(r); +#else + return aom_dk_reader_find_end(r); +#endif +} + +static INLINE int aom_reader_has_error(aom_reader *r) { +#if CONFIG_ANS + return ans_reader_has_error(r); +#elif CONFIG_DAALA_EC + return aom_daala_reader_has_error(r); +#else + return aom_dk_reader_has_error(r); +#endif +} + +// Returns the position in the bit reader in bits. +static INLINE uint32_t aom_reader_tell(const aom_reader *r) { +#if CONFIG_ANS + (void)r; + assert(0 && "aom_reader_tell() is unimplemented for ANS"); + return 0; +#elif CONFIG_DAALA_EC + return aom_daala_reader_tell(r); +#else + return aom_dk_reader_tell(r); +#endif +} + +// Returns the position in the bit reader in 1/8th bits. +static INLINE uint32_t aom_reader_tell_frac(const aom_reader *r) { +#if CONFIG_ANS + (void)r; + assert(0 && "aom_reader_tell_frac() is unimplemented for ANS"); + return 0; +#elif CONFIG_DAALA_EC + return aom_daala_reader_tell_frac(r); +#else + return aom_dk_reader_tell_frac(r); +#endif +} + +#if CONFIG_ACCOUNTING +static INLINE void aom_process_accounting(const aom_reader *r ACCT_STR_PARAM) { + if (r->accounting != NULL) { + uint32_t tell_frac; + tell_frac = aom_reader_tell_frac(r); + aom_accounting_record(r->accounting, ACCT_STR_NAME, + tell_frac - r->accounting->last_tell_frac); + r->accounting->last_tell_frac = tell_frac; + } +} + +static INLINE void aom_update_symb_counts(const aom_reader *r, int is_binary) { + if (r->accounting != NULL) { + r->accounting->syms.num_multi_syms += !is_binary; + r->accounting->syms.num_binary_syms += !!is_binary; + } +} +#endif + +static INLINE int aom_read_(aom_reader *r, int prob ACCT_STR_PARAM) { + int ret; +#if CONFIG_ANS + ret = rabs_read(r, prob); +#elif CONFIG_DAALA_EC + ret = aom_daala_read(r, prob); +#else + ret = aom_dk_read(r, prob); +#endif +#if CONFIG_ACCOUNTING + if (ACCT_STR_NAME) aom_process_accounting(r, ACCT_STR_NAME); + aom_update_symb_counts(r, 1); +#endif + return ret; +} + +static INLINE int aom_read_bit_(aom_reader *r ACCT_STR_PARAM) { + int ret; +#if CONFIG_ANS + ret = rabs_read_bit(r); // Non trivial optimization at half probability +#elif CONFIG_DAALA_EC && CONFIG_RAWBITS + // Note this uses raw bits and is not the same as aom_daala_read(r, 128); + // Calls to this function are omitted from raw symbol accounting. + ret = aom_daala_read_bit(r); +#else + ret = aom_read(r, 128, NULL); // aom_prob_half +#endif +#if CONFIG_ACCOUNTING + if (ACCT_STR_NAME) aom_process_accounting(r, ACCT_STR_NAME); +#endif + return ret; +} + +static INLINE int aom_read_literal_(aom_reader *r, int bits ACCT_STR_PARAM) { + int literal = 0, bit; + + for (bit = bits - 1; bit >= 0; bit--) literal |= aom_read_bit(r, NULL) << bit; +#if CONFIG_ACCOUNTING + if (ACCT_STR_NAME) aom_process_accounting(r, ACCT_STR_NAME); +#endif + return literal; +} + +static INLINE int aom_read_tree_as_bits(aom_reader *r, + const aom_tree_index *tree, + const aom_prob *probs) { + aom_tree_index i = 0; + + while ((i = tree[i + aom_read(r, probs[i >> 1], NULL)]) > 0) continue; + return -i; +} + +#if CONFIG_EC_MULTISYMBOL +static INLINE int aom_read_cdf_(aom_reader *r, const aom_cdf_prob *cdf, + int nsymbs ACCT_STR_PARAM) { + int ret; +#if CONFIG_ANS + (void)nsymbs; + ret = rans_read(r, cdf); +#elif CONFIG_DAALA_EC + ret = daala_read_symbol(r, cdf, nsymbs); +#else +#error \ + "CONFIG_EC_MULTISYMBOL is selected without a valid backing entropy " \ + "coder. Enable daala_ec or ans for a valid configuration." +#endif + +#if CONFIG_ACCOUNTING + if (ACCT_STR_NAME) aom_process_accounting(r, ACCT_STR_NAME); + aom_update_symb_counts(r, (nsymbs == 2)); +#endif + return ret; +} + +static INLINE int aom_read_symbol_(aom_reader *r, aom_cdf_prob *cdf, + int nsymbs ACCT_STR_PARAM) { + int ret; + ret = aom_read_cdf(r, cdf, nsymbs, ACCT_STR_NAME); +#if CONFIG_EC_ADAPT + update_cdf(cdf, ret, nsymbs); +#endif + return ret; +} + +static INLINE int aom_read_tree_as_cdf(aom_reader *r, + const aom_tree_index *tree, + const aom_prob *probs) { + aom_tree_index i = 0; + do { + aom_cdf_prob cdf[16]; + aom_tree_index index[16]; + int path[16]; + int dist[16]; + int nsymbs; + int symb; + nsymbs = tree_to_cdf(tree, probs, i, cdf, index, path, dist); + symb = aom_read_cdf(r, cdf, nsymbs, NULL); + OD_ASSERT(symb >= 0 && symb < nsymbs); + i = index[symb]; + } while (i > 0); + return -i; +} +#endif // CONFIG_EC_MULTISYMBOL + +static INLINE int aom_read_tree_(aom_reader *r, const aom_tree_index *tree, + const aom_prob *probs ACCT_STR_PARAM) { + int ret; +#if CONFIG_EC_MULTISYMBOL + ret = aom_read_tree_as_cdf(r, tree, probs); +#else + ret = aom_read_tree_as_bits(r, tree, probs); +#endif +#if CONFIG_ACCOUNTING + if (ACCT_STR_NAME) aom_process_accounting(r, ACCT_STR_NAME); +#endif + return ret; +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_BITREADER_H_ diff --git a/third_party/aom/aom_dsp/bitreader_buffer.c b/third_party/aom/aom_dsp/bitreader_buffer.c new file mode 100644 index 0000000000..009682b4c8 --- /dev/null +++ b/third_party/aom/aom_dsp/bitreader_buffer.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include "./aom_config.h" +#include "./bitreader_buffer.h" + +size_t aom_rb_bytes_read(struct aom_read_bit_buffer *rb) { + return (rb->bit_offset + 7) >> 3; +} + +int aom_rb_read_bit(struct aom_read_bit_buffer *rb) { + const uint32_t off = rb->bit_offset; + const uint32_t p = off >> 3; + const int q = 7 - (int)(off & 0x7); + if (rb->bit_buffer + p < rb->bit_buffer_end) { + const int bit = (rb->bit_buffer[p] >> q) & 1; + rb->bit_offset = off + 1; + return bit; + } else { + rb->error_handler(rb->error_handler_data); + return 0; + } +} + +int aom_rb_read_literal(struct aom_read_bit_buffer *rb, int bits) { + int value = 0, bit; + for (bit = bits - 1; bit >= 0; bit--) value |= aom_rb_read_bit(rb) << bit; + return value; +} + +int aom_rb_read_signed_literal(struct aom_read_bit_buffer *rb, int bits) { + const int value = aom_rb_read_literal(rb, bits); + return aom_rb_read_bit(rb) ? -value : value; +} + +int aom_rb_read_inv_signed_literal(struct aom_read_bit_buffer *rb, int bits) { + const int nbits = sizeof(unsigned) * 8 - bits - 1; + const unsigned value = (unsigned)aom_rb_read_literal(rb, bits + 1) << nbits; + return ((int)value) >> nbits; +} diff --git a/third_party/aom/aom_dsp/bitreader_buffer.h b/third_party/aom/aom_dsp/bitreader_buffer.h new file mode 100644 index 0000000000..22187357e0 --- /dev/null +++ b/third_party/aom/aom_dsp/bitreader_buffer.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_BITREADER_BUFFER_H_ +#define AOM_DSP_BITREADER_BUFFER_H_ + +#include + +#include "aom/aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void (*aom_rb_error_handler)(void *data); + +struct aom_read_bit_buffer { + const uint8_t *bit_buffer; + const uint8_t *bit_buffer_end; + uint32_t bit_offset; + + void *error_handler_data; + aom_rb_error_handler error_handler; +}; + +size_t aom_rb_bytes_read(struct aom_read_bit_buffer *rb); + +int aom_rb_read_bit(struct aom_read_bit_buffer *rb); + +int aom_rb_read_literal(struct aom_read_bit_buffer *rb, int bits); + +int aom_rb_read_signed_literal(struct aom_read_bit_buffer *rb, int bits); + +int aom_rb_read_inv_signed_literal(struct aom_read_bit_buffer *rb, int bits); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_BITREADER_BUFFER_H_ diff --git a/third_party/aom/aom_dsp/bitwriter.h b/third_party/aom/aom_dsp/bitwriter.h new file mode 100644 index 0000000000..6e3fac2607 --- /dev/null +++ b/third_party/aom/aom_dsp/bitwriter.h @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_BITWRITER_H_ +#define AOM_DSP_BITWRITER_H_ + +#include +#include "./aom_config.h" +#if CONFIG_EC_ADAPT && !CONFIG_EC_MULTISYMBOL +#error "CONFIG_EC_ADAPT is enabled without enabling CONFIG_EC_MULTISYMBOL" +#endif + +#if CONFIG_ANS +#include "aom_dsp/buf_ans.h" +#elif CONFIG_DAALA_EC +#include "aom_dsp/daalaboolwriter.h" +#else +#include "aom_dsp/dkboolwriter.h" +#endif +#include "aom_dsp/prob.h" + +#if CONFIG_RD_DEBUG +#include "av1/common/blockd.h" +#include "av1/encoder/cost.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#if CONFIG_ANS +typedef struct BufAnsCoder aom_writer; +#elif CONFIG_DAALA_EC +typedef struct daala_writer aom_writer; +#else +typedef struct aom_dk_writer aom_writer; +#endif + +typedef struct TOKEN_STATS { + int cost; +#if CONFIG_VAR_TX +#if CONFIG_RD_DEBUG + int txb_coeff_cost_map[TXB_COEFF_COST_MAP_SIZE][TXB_COEFF_COST_MAP_SIZE]; +#endif +#endif +} TOKEN_STATS; + +static INLINE void init_token_stats(TOKEN_STATS *token_stats) { +#if CONFIG_VAR_TX +#if CONFIG_RD_DEBUG + int r, c; + for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r) { + for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c) { + token_stats->txb_coeff_cost_map[r][c] = 0; + } + } +#endif +#endif + token_stats->cost = 0; +} + +static INLINE void aom_start_encode(aom_writer *bc, uint8_t *buffer) { +#if CONFIG_ANS + (void)bc; + (void)buffer; + assert(0 && "buf_ans requires a more complicated startup procedure"); +#elif CONFIG_DAALA_EC + aom_daala_start_encode(bc, buffer); +#else + aom_dk_start_encode(bc, buffer); +#endif +} + +static INLINE void aom_stop_encode(aom_writer *bc) { +#if CONFIG_ANS + (void)bc; + assert(0 && "buf_ans requires a more complicated shutdown procedure"); +#elif CONFIG_DAALA_EC + aom_daala_stop_encode(bc); +#else + aom_dk_stop_encode(bc); +#endif +} + +static INLINE void aom_write(aom_writer *br, int bit, int probability) { +#if CONFIG_ANS + buf_rabs_write(br, bit, probability); +#elif CONFIG_DAALA_EC + aom_daala_write(br, bit, probability); +#else + aom_dk_write(br, bit, probability); +#endif +} + +static INLINE void aom_write_record(aom_writer *br, int bit, int probability, + TOKEN_STATS *token_stats) { + aom_write(br, bit, probability); +#if CONFIG_RD_DEBUG + token_stats->cost += av1_cost_bit(probability, bit); +#else + (void)token_stats; +#endif +} + +static INLINE void aom_write_bit(aom_writer *w, int bit) { +#if CONFIG_ANS + buf_rabs_write_bit(w, bit); +#elif CONFIG_DAALA_EC && CONFIG_RAWBITS + // Note this uses raw bits and is not the same as aom_daala_write(r, 128); + aom_daala_write_bit(w, bit); +#else + aom_write(w, bit, 128); // aom_prob_half +#endif +} + +static INLINE void aom_write_bit_record(aom_writer *w, int bit, + TOKEN_STATS *token_stats) { + aom_write_bit(w, bit); +#if CONFIG_RD_DEBUG + token_stats->cost += av1_cost_bit(128, bit); // aom_prob_half +#else + (void)token_stats; +#endif +} + +static INLINE void aom_write_literal(aom_writer *w, int data, int bits) { + int bit; + + for (bit = bits - 1; bit >= 0; bit--) aom_write_bit(w, 1 & (data >> bit)); +} + +static INLINE void aom_write_tree_as_bits(aom_writer *w, + const aom_tree_index *tr, + const aom_prob *probs, int bits, + int len, aom_tree_index i) { + do { + const int bit = (bits >> --len) & 1; + aom_write(w, bit, probs[i >> 1]); + i = tr[i + bit]; + } while (len); +} + +static INLINE void aom_write_tree_as_bits_record( + aom_writer *w, const aom_tree_index *tr, const aom_prob *probs, int bits, + int len, aom_tree_index i, TOKEN_STATS *token_stats) { + do { + const int bit = (bits >> --len) & 1; + aom_write_record(w, bit, probs[i >> 1], token_stats); + i = tr[i + bit]; + } while (len); +} + +#if CONFIG_EC_MULTISYMBOL +static INLINE void aom_write_cdf(aom_writer *w, int symb, + const aom_cdf_prob *cdf, int nsymbs) { +#if CONFIG_ANS + (void)nsymbs; + assert(cdf); + const aom_cdf_prob cum_prob = symb > 0 ? cdf[symb - 1] : 0; + const aom_cdf_prob prob = cdf[symb] - cum_prob; + buf_rans_write(w, cum_prob, prob); +#elif CONFIG_DAALA_EC + daala_write_symbol(w, symb, cdf, nsymbs); +#else +#error \ + "CONFIG_EC_MULTISYMBOL is selected without a valid backing entropy " \ + "coder. Enable daala_ec or ans for a valid configuration." +#endif +} + +static INLINE void aom_write_symbol(aom_writer *w, int symb, aom_cdf_prob *cdf, + int nsymbs) { + aom_write_cdf(w, symb, cdf, nsymbs); +#if CONFIG_EC_ADAPT + update_cdf(cdf, symb, nsymbs); +#endif +} + +static INLINE void aom_write_tree_as_cdf(aom_writer *w, + const aom_tree_index *tree, + const aom_prob *probs, int bits, + int len, aom_tree_index i) { + aom_tree_index root; + root = i; + do { + aom_cdf_prob cdf[16]; + aom_tree_index index[16]; + int path[16]; + int dist[16]; + int nsymbs; + int symb; + int j; + /* Compute the CDF of the binary tree using the given probabilities. */ + nsymbs = tree_to_cdf(tree, probs, root, cdf, index, path, dist); + /* Find the symbol to code. */ + symb = -1; + for (j = 0; j < nsymbs; j++) { + /* If this symbol codes a leaf node, */ + if (index[j] <= 0) { + if (len == dist[j] && path[j] == bits) { + symb = j; + break; + } + } else { + if (len > dist[j] && path[j] == bits >> (len - dist[j])) { + symb = j; + break; + } + } + } + OD_ASSERT(symb != -1); + aom_write_cdf(w, symb, cdf, nsymbs); + bits &= (1 << (len - dist[symb])) - 1; + len -= dist[symb]; + } while (len); +} + +#endif // CONFIG_EC_MULTISYMBOL + +static INLINE void aom_write_tree(aom_writer *w, const aom_tree_index *tree, + const aom_prob *probs, int bits, int len, + aom_tree_index i) { +#if CONFIG_EC_MULTISYMBOL + aom_write_tree_as_cdf(w, tree, probs, bits, len, i); +#else + aom_write_tree_as_bits(w, tree, probs, bits, len, i); +#endif +} + +static INLINE void aom_write_tree_record(aom_writer *w, + const aom_tree_index *tree, + const aom_prob *probs, int bits, + int len, aom_tree_index i, + TOKEN_STATS *token_stats) { +#if CONFIG_EC_MULTISYMBOL + (void)token_stats; + aom_write_tree_as_cdf(w, tree, probs, bits, len, i); +#else + aom_write_tree_as_bits_record(w, tree, probs, bits, len, i, token_stats); +#endif +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_BITWRITER_H_ diff --git a/third_party/aom/aom_dsp/bitwriter_buffer.c b/third_party/aom/aom_dsp/bitwriter_buffer.c new file mode 100644 index 0000000000..1b3dd2913e --- /dev/null +++ b/third_party/aom/aom_dsp/bitwriter_buffer.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./bitwriter_buffer.h" + +uint32_t aom_wb_bytes_written(const struct aom_write_bit_buffer *wb) { + return wb->bit_offset / CHAR_BIT + (wb->bit_offset % CHAR_BIT > 0); +} + +void aom_wb_write_bit(struct aom_write_bit_buffer *wb, int bit) { + const int off = (int)wb->bit_offset; + const int p = off / CHAR_BIT; + const int q = CHAR_BIT - 1 - off % CHAR_BIT; + if (q == CHAR_BIT - 1) { + // Zero next char and write bit + wb->bit_buffer[p] = bit << q; + } else { + wb->bit_buffer[p] &= ~(1 << q); + wb->bit_buffer[p] |= bit << q; + } + wb->bit_offset = off + 1; +} + +void aom_wb_overwrite_bit(struct aom_write_bit_buffer *wb, int bit) { + // Do not zero bytes but overwrite exisiting values + const int off = (int)wb->bit_offset; + const int p = off / CHAR_BIT; + const int q = CHAR_BIT - 1 - off % CHAR_BIT; + wb->bit_buffer[p] &= ~(1 << q); + wb->bit_buffer[p] |= bit << q; + wb->bit_offset = off + 1; +} + +void aom_wb_write_literal(struct aom_write_bit_buffer *wb, int data, int bits) { + int bit; + for (bit = bits - 1; bit >= 0; bit--) aom_wb_write_bit(wb, (data >> bit) & 1); +} + +void aom_wb_overwrite_literal(struct aom_write_bit_buffer *wb, int data, + int bits) { + int bit; + for (bit = bits - 1; bit >= 0; bit--) + aom_wb_overwrite_bit(wb, (data >> bit) & 1); +} + +void aom_wb_write_inv_signed_literal(struct aom_write_bit_buffer *wb, int data, + int bits) { + aom_wb_write_literal(wb, data, bits + 1); +} diff --git a/third_party/aom/aom_dsp/bitwriter_buffer.h b/third_party/aom/aom_dsp/bitwriter_buffer.h new file mode 100644 index 0000000000..1f23dc857b --- /dev/null +++ b/third_party/aom/aom_dsp/bitwriter_buffer.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_BITWRITER_BUFFER_H_ +#define AOM_DSP_BITWRITER_BUFFER_H_ + +#include "aom/aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct aom_write_bit_buffer { + uint8_t *bit_buffer; + uint32_t bit_offset; +}; + +uint32_t aom_wb_bytes_written(const struct aom_write_bit_buffer *wb); + +void aom_wb_write_bit(struct aom_write_bit_buffer *wb, int bit); + +void aom_wb_overwrite_bit(struct aom_write_bit_buffer *wb, int bit); + +void aom_wb_write_literal(struct aom_write_bit_buffer *wb, int data, int bits); + +void aom_wb_overwrite_literal(struct aom_write_bit_buffer *wb, int data, + int bits); + +void aom_wb_write_inv_signed_literal(struct aom_write_bit_buffer *wb, int data, + int bits); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_BITWRITER_BUFFER_H_ diff --git a/third_party/aom/aom_dsp/blend.h b/third_party/aom/aom_dsp/blend.h new file mode 100644 index 0000000000..e5297ff83b --- /dev/null +++ b/third_party/aom/aom_dsp/blend.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_BLEND_H_ +#define AOM_DSP_BLEND_H_ + +#include "aom_ports/mem.h" + +// Various blending functions and macros. +// See also the aom_blend_* functions in aom_dsp_rtcd.h + +// Alpha blending with alpha values from the range [0, 64], where 64 +// means use the first input and 0 means use the second input. + +#define AOM_BLEND_A64_ROUND_BITS 6 +#define AOM_BLEND_A64_MAX_ALPHA (1 << AOM_BLEND_A64_ROUND_BITS) // 64 + +#define AOM_BLEND_A64(a, v0, v1) \ + ROUND_POWER_OF_TWO((a) * (v0) + (AOM_BLEND_A64_MAX_ALPHA - (a)) * (v1), \ + AOM_BLEND_A64_ROUND_BITS) + +// Alpha blending with alpha values from the range [0, 256], where 256 +// means use the first input and 0 means use the second input. +#define AOM_BLEND_A256_ROUND_BITS 8 +#define AOM_BLEND_A256_MAX_ALPHA (1 << AOM_BLEND_A256_ROUND_BITS) // 256 + +#define AOM_BLEND_A256(a, v0, v1) \ + ROUND_POWER_OF_TWO((a) * (v0) + (AOM_BLEND_A256_MAX_ALPHA - (a)) * (v1), \ + AOM_BLEND_A256_ROUND_BITS) + +// Blending by averaging. +#define AOM_BLEND_AVG(v0, v1) ROUND_POWER_OF_TWO((v0) + (v1), 1) + +#endif // AOM_DSP_BLEND_H_ diff --git a/third_party/aom/aom_dsp/blend_a64_hmask.c b/third_party/aom/aom_dsp/blend_a64_hmask.c new file mode 100644 index 0000000000..99b4b8a599 --- /dev/null +++ b/third_party/aom/aom_dsp/blend_a64_hmask.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/blend.h" + +#include "./aom_dsp_rtcd.h" + +void aom_blend_a64_hmask_c(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + int i, j; + + assert(IMPLIES(src0 == dst, src0_stride == dst_stride)); + assert(IMPLIES(src1 == dst, src1_stride == dst_stride)); + + assert(h >= 1); + assert(w >= 1); + assert(IS_POWER_OF_TWO(h)); + assert(IS_POWER_OF_TWO(w)); + + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + dst[i * dst_stride + j] = AOM_BLEND_A64( + mask[j], src0[i * src0_stride + j], src1[i * src1_stride + j]); + } + } +} + +#if CONFIG_HIGHBITDEPTH +void aom_highbd_blend_a64_hmask_c(uint8_t *dst_8, uint32_t dst_stride, + const uint8_t *src0_8, uint32_t src0_stride, + const uint8_t *src1_8, uint32_t src1_stride, + const uint8_t *mask, int h, int w, int bd) { + int i, j; + uint16_t *dst = CONVERT_TO_SHORTPTR(dst_8); + const uint16_t *src0 = CONVERT_TO_SHORTPTR(src0_8); + const uint16_t *src1 = CONVERT_TO_SHORTPTR(src1_8); + (void)bd; + + assert(IMPLIES(src0 == dst, src0_stride == dst_stride)); + assert(IMPLIES(src1 == dst, src1_stride == dst_stride)); + + assert(h >= 1); + assert(w >= 1); + assert(IS_POWER_OF_TWO(h)); + assert(IS_POWER_OF_TWO(w)); + + assert(bd == 8 || bd == 10 || bd == 12); + + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + dst[i * dst_stride + j] = AOM_BLEND_A64( + mask[j], src0[i * src0_stride + j], src1[i * src1_stride + j]); + } + } +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/blend_a64_mask.c b/third_party/aom/aom_dsp/blend_a64_mask.c new file mode 100644 index 0000000000..3e15542c93 --- /dev/null +++ b/third_party/aom/aom_dsp/blend_a64_mask.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" +#include "aom_dsp/blend.h" +#include "aom_dsp/aom_dsp_common.h" + +#include "./aom_dsp_rtcd.h" + +// Blending with alpha mask. Mask values come from the range [0, 64], +// as described for AOM_BLEND_A64 in aom_dsp/blend.h. src0 or src1 can +// be the same as dst, or dst can be different from both sources. + +void aom_blend_a64_mask_c(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, + int w, int subh, int subw) { + int i, j; + + assert(IMPLIES(src0 == dst, src0_stride == dst_stride)); + assert(IMPLIES(src1 == dst, src1_stride == dst_stride)); + + assert(h >= 1); + assert(w >= 1); + assert(IS_POWER_OF_TWO(h)); + assert(IS_POWER_OF_TWO(w)); + + if (subw == 0 && subh == 0) { + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + const int m = mask[i * mask_stride + j]; + dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j], + src1[i * src1_stride + j]); + } + } + } else if (subw == 1 && subh == 1) { + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + const int m = ROUND_POWER_OF_TWO( + mask[(2 * i) * mask_stride + (2 * j)] + + mask[(2 * i + 1) * mask_stride + (2 * j)] + + mask[(2 * i) * mask_stride + (2 * j + 1)] + + mask[(2 * i + 1) * mask_stride + (2 * j + 1)], + 2); + dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j], + src1[i * src1_stride + j]); + } + } + } else if (subw == 1 && subh == 0) { + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + const int m = AOM_BLEND_AVG(mask[i * mask_stride + (2 * j)], + mask[i * mask_stride + (2 * j + 1)]); + dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j], + src1[i * src1_stride + j]); + } + } + } else { + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + const int m = AOM_BLEND_AVG(mask[(2 * i) * mask_stride + j], + mask[(2 * i + 1) * mask_stride + j]); + dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j], + src1[i * src1_stride + j]); + } + } + } +} + +#if CONFIG_HIGHBITDEPTH +void aom_highbd_blend_a64_mask_c(uint8_t *dst_8, uint32_t dst_stride, + const uint8_t *src0_8, uint32_t src0_stride, + const uint8_t *src1_8, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, + int h, int w, int subh, int subw, int bd) { + int i, j; + uint16_t *dst = CONVERT_TO_SHORTPTR(dst_8); + const uint16_t *src0 = CONVERT_TO_SHORTPTR(src0_8); + const uint16_t *src1 = CONVERT_TO_SHORTPTR(src1_8); + (void)bd; + + assert(IMPLIES(src0 == dst, src0_stride == dst_stride)); + assert(IMPLIES(src1 == dst, src1_stride == dst_stride)); + + assert(h >= 1); + assert(w >= 1); + assert(IS_POWER_OF_TWO(h)); + assert(IS_POWER_OF_TWO(w)); + + assert(bd == 8 || bd == 10 || bd == 12); + + if (subw == 0 && subh == 0) { + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + const int m = mask[i * mask_stride + j]; + dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j], + src1[i * src1_stride + j]); + } + } + } else if (subw == 1 && subh == 1) { + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + const int m = ROUND_POWER_OF_TWO( + mask[(2 * i) * mask_stride + (2 * j)] + + mask[(2 * i + 1) * mask_stride + (2 * j)] + + mask[(2 * i) * mask_stride + (2 * j + 1)] + + mask[(2 * i + 1) * mask_stride + (2 * j + 1)], + 2); + dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j], + src1[i * src1_stride + j]); + } + } + } else if (subw == 1 && subh == 0) { + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + const int m = AOM_BLEND_AVG(mask[i * mask_stride + (2 * j)], + mask[i * mask_stride + (2 * j + 1)]); + dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j], + src1[i * src1_stride + j]); + } + } + } else { + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + const int m = AOM_BLEND_AVG(mask[(2 * i) * mask_stride + j], + mask[(2 * i + 1) * mask_stride + j]); + dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j], + src1[i * src1_stride + j]); + } + } + } +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/blend_a64_vmask.c b/third_party/aom/aom_dsp/blend_a64_vmask.c new file mode 100644 index 0000000000..1a5e30e315 --- /dev/null +++ b/third_party/aom/aom_dsp/blend_a64_vmask.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/blend.h" + +#include "./aom_dsp_rtcd.h" + +void aom_blend_a64_vmask_c(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + int i, j; + + assert(IMPLIES(src0 == dst, src0_stride == dst_stride)); + assert(IMPLIES(src1 == dst, src1_stride == dst_stride)); + + assert(h >= 1); + assert(w >= 1); + assert(IS_POWER_OF_TWO(h)); + assert(IS_POWER_OF_TWO(w)); + + for (i = 0; i < h; ++i) { + const int m = mask[i]; + for (j = 0; j < w; ++j) { + dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j], + src1[i * src1_stride + j]); + } + } +} + +#if CONFIG_HIGHBITDEPTH +void aom_highbd_blend_a64_vmask_c(uint8_t *dst_8, uint32_t dst_stride, + const uint8_t *src0_8, uint32_t src0_stride, + const uint8_t *src1_8, uint32_t src1_stride, + const uint8_t *mask, int h, int w, int bd) { + int i, j; + uint16_t *dst = CONVERT_TO_SHORTPTR(dst_8); + const uint16_t *src0 = CONVERT_TO_SHORTPTR(src0_8); + const uint16_t *src1 = CONVERT_TO_SHORTPTR(src1_8); + (void)bd; + + assert(IMPLIES(src0 == dst, src0_stride == dst_stride)); + assert(IMPLIES(src1 == dst, src1_stride == dst_stride)); + + assert(h >= 1); + assert(w >= 1); + assert(IS_POWER_OF_TWO(h)); + assert(IS_POWER_OF_TWO(w)); + + assert(bd == 8 || bd == 10 || bd == 12); + + for (i = 0; i < h; ++i) { + const int m = mask[i]; + for (j = 0; j < w; ++j) { + dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j], + src1[i * src1_stride + j]); + } + } +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/buf_ans.c b/third_party/aom/aom_dsp/buf_ans.c new file mode 100644 index 0000000000..8fe1ff7631 --- /dev/null +++ b/third_party/aom/aom_dsp/buf_ans.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom_dsp/buf_ans.h" +#include "aom_mem/aom_mem.h" +#include "aom/internal/aom_codec_internal.h" + +void aom_buf_ans_alloc(struct BufAnsCoder *c, + struct aom_internal_error_info *error, int size) { + c->error = error; + c->size = size; + assert(c->size > 1); + AOM_CHECK_MEM_ERROR(error, c->buf, aom_malloc(c->size * sizeof(*c->buf))); + // Initialize to overfull to trigger the assert in write. + c->offset = c->size + 1; +} + +void aom_buf_ans_free(struct BufAnsCoder *c) { + aom_free(c->buf); + c->buf = NULL; + c->size = 0; +} + +#if !ANS_MAX_SYMBOLS +void aom_buf_ans_grow(struct BufAnsCoder *c) { + struct buffered_ans_symbol *new_buf = NULL; + int new_size = c->size * 2; + AOM_CHECK_MEM_ERROR(c->error, new_buf, + aom_malloc(new_size * sizeof(*new_buf))); + memcpy(new_buf, c->buf, c->size * sizeof(*c->buf)); + aom_free(c->buf); + c->buf = new_buf; + c->size = new_size; +} +#endif + +void aom_buf_ans_flush(struct BufAnsCoder *const c) { + int offset; +#if ANS_MAX_SYMBOLS + if (c->offset == 0) return; +#endif + assert(c->offset > 0); + offset = c->offset - 1; + // Code the first symbol such that it brings the state to the smallest normal + // state from an initial state that would have been a subnormal/refill state. + if (c->buf[offset].method == ANS_METHOD_RANS) { + c->ans.state += c->buf[offset].val_start; + } else { + c->ans.state += c->buf[offset].val_start ? c->buf[offset].prob : 0; + } + for (offset = offset - 1; offset >= 0; --offset) { + if (c->buf[offset].method == ANS_METHOD_RANS) { + rans_write(&c->ans, c->buf[offset].val_start, c->buf[offset].prob); + } else { + rabs_write(&c->ans, (uint8_t)c->buf[offset].val_start, + (AnsP8)c->buf[offset].prob); + } + } + c->offset = 0; + c->output_bytes += ans_write_end(&c->ans); +} diff --git a/third_party/aom/aom_dsp/buf_ans.h b/third_party/aom/aom_dsp/buf_ans.h new file mode 100644 index 0000000000..0768506b35 --- /dev/null +++ b/third_party/aom/aom_dsp/buf_ans.h @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_BUF_ANS_H_ +#define AOM_DSP_BUF_ANS_H_ +// Buffered forward ANS writer. +// Symbols are written to the writer in forward (decode) order and serialized +// backwards due to ANS's stack like behavior. + +#include +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/ans.h" +#include "aom_dsp/answriter.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +#define ANS_METHOD_RABS 0 +#define ANS_METHOD_RANS 1 + +struct buffered_ans_symbol { + unsigned int method : 1; // one of ANS_METHOD_RABS or ANS_METHOD_RANS + // TODO(aconverse): Should be possible to write this in terms of start for ABS + unsigned int val_start : RANS_PROB_BITS; // Boolean value for ABS + // start in symbol cycle for Rans + unsigned int prob : RANS_PROB_BITS; // Probability of this symbol +}; + +struct BufAnsCoder { + struct aom_internal_error_info *error; + struct buffered_ans_symbol *buf; + struct AnsCoder ans; + int size; + int offset; + int output_bytes; +#if ANS_MAX_SYMBOLS + int window_size; +#endif +}; + +// Allocate a buffered ANS coder to store size symbols. +// When ANS_MAX_SYMBOLS is turned on, the size is the fixed size of each ANS +// partition. +// When ANS_MAX_SYMBOLS is turned off, size is merely an initial hint and the +// buffer will grow on demand +void aom_buf_ans_alloc(struct BufAnsCoder *c, + struct aom_internal_error_info *error, int hint); + +void aom_buf_ans_free(struct BufAnsCoder *c); + +#if !ANS_MAX_SYMBOLS +void aom_buf_ans_grow(struct BufAnsCoder *c); +#endif + +void aom_buf_ans_flush(struct BufAnsCoder *const c); + +static INLINE void buf_ans_write_init(struct BufAnsCoder *const c, + uint8_t *const output_buffer) { + c->offset = 0; + c->output_bytes = 0; + ans_write_init(&c->ans, output_buffer); +} + +static INLINE void buf_rabs_write(struct BufAnsCoder *const c, uint8_t val, + AnsP8 prob) { + assert(c->offset <= c->size); +#if !ANS_MAX_SYMBOLS + if (c->offset == c->size) { + aom_buf_ans_grow(c); + } +#endif + c->buf[c->offset].method = ANS_METHOD_RABS; + c->buf[c->offset].val_start = val; + c->buf[c->offset].prob = prob; + ++c->offset; +#if ANS_MAX_SYMBOLS + if (c->offset == c->size) aom_buf_ans_flush(c); +#endif +} + +// Buffer one symbol for encoding using rANS. +// cum_prob: The cumulative probability before this symbol (the offset of +// the symbol in the symbol cycle) +// prob: The probability of this symbol (l_s from the paper) +// RANS_PRECISION takes the place of m from the paper. +static INLINE void buf_rans_write(struct BufAnsCoder *const c, + aom_cdf_prob cum_prob, aom_cdf_prob prob) { + assert(c->offset <= c->size); +#if !ANS_MAX_SYMBOLS + if (c->offset == c->size) { + aom_buf_ans_grow(c); + } +#endif + c->buf[c->offset].method = ANS_METHOD_RANS; + c->buf[c->offset].val_start = cum_prob; + c->buf[c->offset].prob = prob; + ++c->offset; +#if ANS_MAX_SYMBOLS + if (c->offset == c->size) aom_buf_ans_flush(c); +#endif +} + +static INLINE void buf_rabs_write_bit(struct BufAnsCoder *c, int bit) { + buf_rabs_write(c, bit, 128); +} + +static INLINE void buf_rabs_write_literal(struct BufAnsCoder *c, int literal, + int bits) { + int bit; + + assert(bits < 31); + for (bit = bits - 1; bit >= 0; bit--) + buf_rabs_write_bit(c, 1 & (literal >> bit)); +} + +static INLINE int buf_ans_write_end(struct BufAnsCoder *const c) { + assert(c->offset == 0); + return c->output_bytes; +} +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus +#endif // AOM_DSP_BUF_ANS_H_ diff --git a/third_party/aom/aom_dsp/daalaboolreader.c b/third_party/aom/aom_dsp/daalaboolreader.c new file mode 100644 index 0000000000..0fc7b14a5a --- /dev/null +++ b/third_party/aom/aom_dsp/daalaboolreader.c @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/daalaboolreader.h" + +int aom_daala_reader_init(daala_reader *r, const uint8_t *buffer, int size) { + if (size && !buffer) { + return 1; + } + r->buffer_end = buffer + size; + r->buffer = buffer; + od_ec_dec_init(&r->ec, buffer, size - 1); +#if CONFIG_ACCOUNTING + r->accounting = NULL; +#endif + return 0; +} + +const uint8_t *aom_daala_reader_find_end(daala_reader *r) { + return r->buffer_end; +} + +uint32_t aom_daala_reader_tell(const daala_reader *r) { + return od_ec_dec_tell(&r->ec); +} + +uint32_t aom_daala_reader_tell_frac(const daala_reader *r) { + return od_ec_dec_tell_frac(&r->ec); +} diff --git a/third_party/aom/aom_dsp/daalaboolreader.h b/third_party/aom/aom_dsp/daalaboolreader.h new file mode 100644 index 0000000000..428d74db09 --- /dev/null +++ b/third_party/aom/aom_dsp/daalaboolreader.h @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_DAALABOOLREADER_H_ +#define AOM_DSP_DAALABOOLREADER_H_ + +#include "aom/aom_integer.h" +#include "aom_dsp/entdec.h" +#include "aom_dsp/prob.h" +#if CONFIG_ACCOUNTING +#include "av1/decoder/accounting.h" +#endif +#if CONFIG_BITSTREAM_DEBUG +#include +#include "aom_util/debug_util.h" +#endif // CONFIG_BITSTREAM_DEBUG + +#ifdef __cplusplus +extern "C" { +#endif + +struct daala_reader { + const uint8_t *buffer; + const uint8_t *buffer_end; + od_ec_dec ec; +#if CONFIG_ACCOUNTING + Accounting *accounting; +#endif +}; + +typedef struct daala_reader daala_reader; + +int aom_daala_reader_init(daala_reader *r, const uint8_t *buffer, int size); +const uint8_t *aom_daala_reader_find_end(daala_reader *r); +uint32_t aom_daala_reader_tell(const daala_reader *r); +uint32_t aom_daala_reader_tell_frac(const daala_reader *r); + +static INLINE int aom_daala_read(daala_reader *r, int prob) { + int bit; +#if CONFIG_EC_SMALLMUL + int p = (0x7FFFFF - (prob << 15) + prob) >> 8; +#else + int p = ((prob << 15) + 256 - prob) >> 8; +#endif +#if CONFIG_BITSTREAM_DEBUG +/*{ + const int queue_r = bitstream_queue_get_read(); + const int frame_idx = bitstream_queue_get_frame_read(); + if (frame_idx == 0 && queue_r == 0) { + fprintf(stderr, "\n *** bitstream queue at frame_idx_r %d queue_r %d\n", + frame_idx, queue_r); + } +}*/ +#endif + + bit = od_ec_decode_bool_q15(&r->ec, p); + +#if CONFIG_BITSTREAM_DEBUG + { + int i; + int ref_bit, ref_nsymbs; + aom_cdf_prob ref_cdf[16]; + const int queue_r = bitstream_queue_get_read(); + const int frame_idx = bitstream_queue_get_frame_read(); + bitstream_queue_pop(&ref_bit, ref_cdf, &ref_nsymbs); + if (ref_nsymbs != 2) { + fprintf(stderr, + "\n *** [bit] nsymbs error, frame_idx_r %d nsymbs %d ref_nsymbs " + "%d queue_r %d\n", + frame_idx, 2, ref_nsymbs, queue_r); + assert(0); + } + if ((ref_nsymbs != 2) || (ref_cdf[0] != (aom_cdf_prob)p) || + (ref_cdf[1] != 32767)) { + fprintf(stderr, + "\n *** [bit] cdf error, frame_idx_r %d cdf {%d, %d} ref_cdf {%d", + frame_idx, p, 32767, ref_cdf[0]); + for (i = 1; i < ref_nsymbs; ++i) fprintf(stderr, ", %d", ref_cdf[i]); + fprintf(stderr, "} queue_r %d\n", queue_r); + assert(0); + } + if (bit != ref_bit) { + fprintf(stderr, + "\n *** [bit] symb error, frame_idx_r %d symb %d ref_symb %d " + "queue_r %d\n", + frame_idx, bit, ref_bit, queue_r); + assert(0); + } + } +#endif + + return bit; +} + +#if CONFIG_RAWBITS +static INLINE int aom_daala_read_bit(daala_reader *r) { + return od_ec_dec_bits(&r->ec, 1, "aom_bits"); +} +#endif + +static INLINE int aom_daala_reader_has_error(daala_reader *r) { + return r->ec.error; +} + +static INLINE int daala_read_symbol(daala_reader *r, const aom_cdf_prob *cdf, + int nsymbs) { + int symb; + symb = od_ec_decode_cdf_q15(&r->ec, cdf, nsymbs); + +#if CONFIG_BITSTREAM_DEBUG + { + int i; + int cdf_error = 0; + int ref_symb, ref_nsymbs; + aom_cdf_prob ref_cdf[16]; + const int queue_r = bitstream_queue_get_read(); + const int frame_idx = bitstream_queue_get_frame_read(); + bitstream_queue_pop(&ref_symb, ref_cdf, &ref_nsymbs); + if (nsymbs != ref_nsymbs) { + fprintf(stderr, + "\n *** nsymbs error, frame_idx_r %d nsymbs %d ref_nsymbs %d " + "queue_r %d\n", + frame_idx, nsymbs, ref_nsymbs, queue_r); + cdf_error = 0; + assert(0); + } else { + for (i = 0; i < nsymbs; ++i) + if (cdf[i] != ref_cdf[i]) cdf_error = 1; + } + if (cdf_error) { + fprintf(stderr, "\n *** cdf error, frame_idx_r %d cdf {%d", frame_idx, + cdf[0]); + for (i = 1; i < nsymbs; ++i) fprintf(stderr, ", %d", cdf[i]); + fprintf(stderr, "} ref_cdf {%d", ref_cdf[0]); + for (i = 1; i < ref_nsymbs; ++i) fprintf(stderr, ", %d", ref_cdf[i]); + fprintf(stderr, "} queue_r %d\n", queue_r); + assert(0); + } + if (symb != ref_symb) { + fprintf( + stderr, + "\n *** symb error, frame_idx_r %d symb %d ref_symb %d queue_r %d\n", + frame_idx, symb, ref_symb, queue_r); + assert(0); + } + } +#endif + + return symb; +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/third_party/aom/aom_dsp/daalaboolwriter.c b/third_party/aom/aom_dsp/daalaboolwriter.c new file mode 100644 index 0000000000..0ba8f6ab8b --- /dev/null +++ b/third_party/aom/aom_dsp/daalaboolwriter.c @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "aom_dsp/daalaboolwriter.h" + +void aom_daala_start_encode(daala_writer *br, uint8_t *source) { + br->buffer = source; + br->pos = 0; + od_ec_enc_init(&br->ec, 62025); +} + +void aom_daala_stop_encode(daala_writer *br) { + uint32_t daala_bytes; + unsigned char *daala_data; + daala_data = od_ec_enc_done(&br->ec, &daala_bytes); + memcpy(br->buffer, daala_data, daala_bytes); + br->pos = daala_bytes; + /* Prevent ec bitstream from being detected as a superframe marker. + Must always be added, so that rawbits knows the exact length of the + bitstream. */ + br->buffer[br->pos++] = 0; + od_ec_enc_clear(&br->ec); +} diff --git a/third_party/aom/aom_dsp/daalaboolwriter.h b/third_party/aom/aom_dsp/daalaboolwriter.h new file mode 100644 index 0000000000..bbaf53c69f --- /dev/null +++ b/third_party/aom/aom_dsp/daalaboolwriter.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_DAALABOOLWRITER_H_ +#define AOM_DSP_DAALABOOLWRITER_H_ + +#include + +#include "aom_dsp/entenc.h" +#include "aom_dsp/prob.h" +#if CONFIG_BITSTREAM_DEBUG +#include "aom_util/debug_util.h" +#endif // CONFIG_BITSTREAM_DEBUG + +#ifdef __cplusplus +extern "C" { +#endif + +struct daala_writer { + unsigned int pos; + uint8_t *buffer; + od_ec_enc ec; +}; + +typedef struct daala_writer daala_writer; + +void aom_daala_start_encode(daala_writer *w, uint8_t *buffer); +void aom_daala_stop_encode(daala_writer *w); + +static INLINE void aom_daala_write(daala_writer *w, int bit, int prob) { +#if CONFIG_EC_SMALLMUL + int p = (0x7FFFFF - (prob << 15) + prob) >> 8; +#else + int p = ((prob << 15) + 256 - prob) >> 8; +#endif +#if CONFIG_BITSTREAM_DEBUG + aom_cdf_prob cdf[2] = { (aom_cdf_prob)p, 32767 }; + /*int queue_r = 0; + int frame_idx_r = 0; + int queue_w = bitstream_queue_get_write(); + int frame_idx_w = bitstream_queue_get_frame_write(); + if (frame_idx_w == frame_idx_r && queue_w == queue_r) { + fprintf(stderr, "\n *** bitstream queue at frame_idx_w %d queue_w %d\n", + frame_idx_w, queue_w); + }*/ + bitstream_queue_push(bit, cdf, 2); +#endif + + od_ec_encode_bool_q15(&w->ec, bit, p); +} + +#if CONFIG_RAWBITS +static INLINE void aom_daala_write_bit(daala_writer *w, int bit) { + od_ec_enc_bits(&w->ec, bit, 1); +} +#endif + +static INLINE void daala_write_symbol(daala_writer *w, int symb, + const aom_cdf_prob *cdf, int nsymbs) { +#if CONFIG_BITSTREAM_DEBUG + /*int queue_r = 0; + int frame_idx_r = 0; + int queue_w = bitstream_queue_get_write(); + int frame_idx_w = bitstream_queue_get_frame_write(); + if (frame_idx_w == frame_idx_r && queue_w == queue_r) { + fprintf(stderr, "\n *** bitstream queue at frame_idx_w %d queue_w %d\n", + frame_idx_w, queue_w); + }*/ + bitstream_queue_push(symb, cdf, nsymbs); +#endif + + od_ec_encode_cdf_q15(&w->ec, symb, cdf, nsymbs); +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/third_party/aom/aom_dsp/dkboolreader.c b/third_party/aom/aom_dsp/dkboolreader.c new file mode 100644 index 0000000000..288d5f1ce4 --- /dev/null +++ b/third_party/aom/aom_dsp/dkboolreader.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" + +#include "aom_dsp/dkboolreader.h" +#include "aom_dsp/prob.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" +#include "aom_mem/aom_mem.h" +#include "aom_util/endian_inl.h" + +static INLINE int aom_dk_read_bit(struct aom_dk_reader *r) { + return aom_dk_read(r, 128); // aom_prob_half +} + +int aom_dk_reader_init(struct aom_dk_reader *r, const uint8_t *buffer, + size_t size, aom_decrypt_cb decrypt_cb, + void *decrypt_state) { + if (size && !buffer) { + return 1; + } else { + r->buffer_end = buffer + size; + r->buffer_start = r->buffer = buffer; + r->value = 0; + r->count = -8; + r->range = 255; + r->decrypt_cb = decrypt_cb; + r->decrypt_state = decrypt_state; + aom_dk_reader_fill(r); +#if CONFIG_ACCOUNTING + r->accounting = NULL; +#endif + return aom_dk_read_bit(r) != 0; // marker bit + } +} + +void aom_dk_reader_fill(struct aom_dk_reader *r) { + const uint8_t *const buffer_end = r->buffer_end; + const uint8_t *buffer = r->buffer; + const uint8_t *buffer_start = buffer; + BD_VALUE value = r->value; + int count = r->count; + const size_t bytes_left = buffer_end - buffer; + const size_t bits_left = bytes_left * CHAR_BIT; + int shift = BD_VALUE_SIZE - CHAR_BIT - (count + CHAR_BIT); + + if (r->decrypt_cb) { + size_t n = AOMMIN(sizeof(r->clear_buffer), bytes_left); + r->decrypt_cb(r->decrypt_state, buffer, r->clear_buffer, (int)n); + buffer = r->clear_buffer; + buffer_start = r->clear_buffer; + } + if (bits_left > BD_VALUE_SIZE) { + const int bits = (shift & 0xfffffff8) + CHAR_BIT; + BD_VALUE nv; + BD_VALUE big_endian_values; + memcpy(&big_endian_values, buffer, sizeof(BD_VALUE)); +#if SIZE_MAX == 0xffffffffffffffffULL + big_endian_values = HToBE64(big_endian_values); +#else + big_endian_values = HToBE32(big_endian_values); +#endif + nv = big_endian_values >> (BD_VALUE_SIZE - bits); + count += bits; + buffer += (bits >> 3); + value = r->value | (nv << (shift & 0x7)); + } else { + const int bits_over = (int)(shift + CHAR_BIT - (int)bits_left); + int loop_end = 0; + if (bits_over >= 0) { + count += LOTS_OF_BITS; + loop_end = bits_over; + } + + if (bits_over < 0 || bits_left) { + while (shift >= loop_end) { + count += CHAR_BIT; + value |= (BD_VALUE)*buffer++ << shift; + shift -= CHAR_BIT; + } + } + } + + // NOTE: Variable 'buffer' may not relate to 'r->buffer' after decryption, + // so we increase 'r->buffer' by the amount that 'buffer' moved, rather than + // assign 'buffer' to 'r->buffer'. + r->buffer += buffer - buffer_start; + r->value = value; + r->count = count; +} + +const uint8_t *aom_dk_reader_find_end(struct aom_dk_reader *r) { + // Find the end of the coded buffer + while (r->count > CHAR_BIT && r->count < BD_VALUE_SIZE) { + r->count -= CHAR_BIT; + r->buffer--; + } + return r->buffer; +} diff --git a/third_party/aom/aom_dsp/dkboolreader.h b/third_party/aom/aom_dsp/dkboolreader.h new file mode 100644 index 0000000000..f0bc843813 --- /dev/null +++ b/third_party/aom/aom_dsp/dkboolreader.h @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_DKBOOLREADER_H_ +#define AOM_DSP_DKBOOLREADER_H_ + +#include +#include +#include + +#include "./aom_config.h" +#if CONFIG_BITSTREAM_DEBUG +#include +#include +#include "aom_util/debug_util.h" +#endif // CONFIG_BITSTREAM_DEBUG + +#include "aom_ports/mem.h" +#include "aom/aomdx.h" +#include "aom/aom_integer.h" +#include "aom_dsp/prob.h" +#if CONFIG_ACCOUNTING +#include "av1/decoder/accounting.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef size_t BD_VALUE; + +#define BD_VALUE_SIZE ((int)sizeof(BD_VALUE) * CHAR_BIT) + +// This is meant to be a large, positive constant that can still be efficiently +// loaded as an immediate (on platforms like ARM, for example). +// Even relatively modest values like 100 would work fine. +#define LOTS_OF_BITS 0x40000000 + +struct aom_dk_reader { + // Be careful when reordering this struct, it may impact the cache negatively. + BD_VALUE value; + unsigned int range; + int count; + const uint8_t *buffer_start; + const uint8_t *buffer_end; + const uint8_t *buffer; + aom_decrypt_cb decrypt_cb; + void *decrypt_state; + uint8_t clear_buffer[sizeof(BD_VALUE) + 1]; +#if CONFIG_ACCOUNTING + Accounting *accounting; +#endif +}; + +int aom_dk_reader_init(struct aom_dk_reader *r, const uint8_t *buffer, + size_t size, aom_decrypt_cb decrypt_cb, + void *decrypt_state); + +void aom_dk_reader_fill(struct aom_dk_reader *r); + +const uint8_t *aom_dk_reader_find_end(struct aom_dk_reader *r); + +static INLINE uint32_t aom_dk_reader_tell(const struct aom_dk_reader *r) { + const uint32_t bits_read = + (uint32_t)((r->buffer - r->buffer_start) * CHAR_BIT); + const int count = + (r->count < LOTS_OF_BITS) ? r->count : r->count - LOTS_OF_BITS; + assert(r->buffer >= r->buffer_start); + return bits_read - (count + CHAR_BIT); +} + +/*The resolution of fractional-precision bit usage measurements, i.e., + 3 => 1/8th bits.*/ +#define DK_BITRES (3) + +static INLINE uint32_t aom_dk_reader_tell_frac(const struct aom_dk_reader *r) { + uint32_t num_bits; + uint32_t range; + int l; + int i; + num_bits = aom_dk_reader_tell(r) << DK_BITRES; + range = r->range; + l = 0; + for (i = DK_BITRES; i-- > 0;) { + int b; + range = range * range >> 7; + b = (int)(range >> 8); + l = l << 1 | b; + range >>= b; + } + return num_bits - l; +} + +static INLINE int aom_dk_reader_has_error(struct aom_dk_reader *r) { + // Check if we have reached the end of the buffer. + // + // Variable 'count' stores the number of bits in the 'value' buffer, minus + // 8. The top byte is part of the algorithm, and the remainder is buffered + // to be shifted into it. So if count == 8, the top 16 bits of 'value' are + // occupied, 8 for the algorithm and 8 in the buffer. + // + // When reading a byte from the user's buffer, count is filled with 8 and + // one byte is filled into the value buffer. When we reach the end of the + // data, count is additionally filled with LOTS_OF_BITS. So when + // count == LOTS_OF_BITS - 1, the user's data has been exhausted. + // + // 1 if we have tried to decode bits after the end of stream was encountered. + // 0 No error. + return r->count > BD_VALUE_SIZE && r->count < LOTS_OF_BITS; +} + +static INLINE int aom_dk_read(struct aom_dk_reader *r, int prob) { + unsigned int bit = 0; + BD_VALUE value; + BD_VALUE bigsplit; + int count; + unsigned int range; + unsigned int split = (r->range * prob + (256 - prob)) >> CHAR_BIT; + + if (r->count < 0) aom_dk_reader_fill(r); + + value = r->value; + count = r->count; + + bigsplit = (BD_VALUE)split << (BD_VALUE_SIZE - CHAR_BIT); + + range = split; + + if (value >= bigsplit) { + range = r->range - split; + value = value - bigsplit; + bit = 1; + } + + { + register int shift = aom_norm[range]; + range <<= shift; + value <<= shift; + count -= shift; + } + r->value = value; + r->count = count; + r->range = range; + +#if CONFIG_BITSTREAM_DEBUG + { + int ref_bit, ref_prob; + const int queue_r = bitstream_queue_get_read(); + const int frame_idx = bitstream_queue_get_frame_read(); + bitstream_queue_pop(&ref_bit, &ref_prob); + if (prob != ref_prob) { + fprintf( + stderr, + "\n *** prob error, frame_idx_r %d prob %d ref_prob %d queue_r %d\n", + frame_idx, prob, ref_prob, queue_r); + assert(0); + } + if ((int)bit != ref_bit) { + fprintf(stderr, "\n *** bit error, frame_idx_r %d bit %d ref_bit %d\n", + frame_idx, bit, ref_bit); + assert(0); + } + } +#endif // CONFIG_BITSTREAM_DEBUG + + return bit; +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_DKBOOLREADER_H_ diff --git a/third_party/aom/aom_dsp/dkboolwriter.c b/third_party/aom/aom_dsp/dkboolwriter.c new file mode 100644 index 0000000000..fc98e7c9be --- /dev/null +++ b/third_party/aom/aom_dsp/dkboolwriter.c @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./dkboolwriter.h" + +static INLINE void aom_dk_write_bit(aom_dk_writer *w, int bit) { + aom_dk_write(w, bit, 128); // aom_prob_half +} + +void aom_dk_start_encode(aom_dk_writer *br, uint8_t *source) { + br->lowvalue = 0; + br->range = 255; + br->count = -24; + br->buffer = source; + br->pos = 0; + aom_dk_write_bit(br, 0); +} + +void aom_dk_stop_encode(aom_dk_writer *br) { + int i; + +#if CONFIG_BITSTREAM_DEBUG + bitstream_queue_set_skip_write(1); +#endif // CONFIG_BITSTREAM_DEBUG + + for (i = 0; i < 32; i++) aom_dk_write_bit(br, 0); + +#if CONFIG_BITSTREAM_DEBUG + bitstream_queue_set_skip_write(0); +#endif // CONFIG_BITSTREAM_DEBUG + + // Ensure there's no ambigous collision with any index marker bytes + if ((br->buffer[br->pos - 1] & 0xe0) == 0xc0) br->buffer[br->pos++] = 0; +} diff --git a/third_party/aom/aom_dsp/dkboolwriter.h b/third_party/aom/aom_dsp/dkboolwriter.h new file mode 100644 index 0000000000..835436885b --- /dev/null +++ b/third_party/aom/aom_dsp/dkboolwriter.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_DKBOOLWRITER_H_ +#define AOM_DSP_DKBOOLWRITER_H_ + +#include "./aom_config.h" + +#if CONFIG_BITSTREAM_DEBUG +#include +#include "aom_util/debug_util.h" +#endif // CONFIG_BITSTREAM_DEBUG + +#include "aom_dsp/prob.h" +#include "aom_ports/mem.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct aom_dk_writer { + unsigned int lowvalue; + unsigned int range; + int count; + unsigned int pos; + uint8_t *buffer; +} aom_dk_writer; + +void aom_dk_start_encode(aom_dk_writer *bc, uint8_t *buffer); +void aom_dk_stop_encode(aom_dk_writer *bc); + +static INLINE void aom_dk_write(aom_dk_writer *br, int bit, int probability) { + unsigned int split; + int count = br->count; + unsigned int range = br->range; + unsigned int lowvalue = br->lowvalue; + register int shift; + +#if CONFIG_BITSTREAM_DEBUG + // int queue_r = 0; + // int frame_idx_r = 0; + // int queue_w = bitstream_queue_get_write(); + // int frame_idx_w = bitstream_queue_get_frame_write(); + // if (frame_idx_w == frame_idx_r && queue_w == queue_r) { + // fprintf(stderr, "\n *** bitstream queue at frame_idx_w %d queue_w %d\n", + // frame_idx_w, queue_w); + // } + bitstream_queue_push(bit, probability); +#endif // CONFIG_BITSTREAM_DEBUG + + split = 1 + (((range - 1) * probability) >> 8); + + range = split; + + if (bit) { + lowvalue += split; + range = br->range - split; + } + + shift = aom_norm[range]; + + range <<= shift; + count += shift; + + if (count >= 0) { + int offset = shift - count; + + if ((lowvalue << (offset - 1)) & 0x80000000) { + int x = br->pos - 1; + + while (x >= 0 && br->buffer[x] == 0xff) { + br->buffer[x] = 0; + x--; + } + + br->buffer[x] += 1; + } + + br->buffer[br->pos++] = (lowvalue >> (24 - offset)); + lowvalue <<= offset; + shift = count; + lowvalue &= 0xffffff; + count -= 8; + } + + lowvalue <<= shift; + br->count = count; + br->lowvalue = lowvalue; + br->range = range; +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_DKBOOLWRITER_H_ diff --git a/third_party/aom/aom_dsp/entcode.c b/third_party/aom/aom_dsp/entcode.c new file mode 100644 index 0000000000..ad76b7e3ea --- /dev/null +++ b/third_party/aom/aom_dsp/entcode.c @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifdef HAVE_CONFIG_H +#include "./config.h" +#endif + +#include "aom_dsp/entcode.h" + +/*Given the current total integer number of bits used and the current value of + rng, computes the fraction number of bits used to OD_BITRES precision. + This is used by od_ec_enc_tell_frac() and od_ec_dec_tell_frac(). + nbits_total: The number of whole bits currently used, i.e., the value + returned by od_ec_enc_tell() or od_ec_dec_tell(). + rng: The current value of rng from either the encoder or decoder state. + Return: The number of bits scaled by 2**OD_BITRES. + This will always be slightly larger than the exact value (e.g., all + rounding error is in the positive direction).*/ +uint32_t od_ec_tell_frac(uint32_t nbits_total, uint32_t rng) { + uint32_t nbits; + int l; + int i; + /*To handle the non-integral number of bits still left in the encoder/decoder + state, we compute the worst-case number of bits of val that must be + encoded to ensure that the value is inside the range for any possible + subsequent bits. + The computation here is independent of val itself (the decoder does not + even track that value), even though the real number of bits used after + od_ec_enc_done() may be 1 smaller if rng is a power of two and the + corresponding trailing bits of val are all zeros. + If we did try to track that special case, then coding a value with a + probability of 1/(1 << n) might sometimes appear to use more than n bits. + This may help explain the surprising result that a newly initialized + encoder or decoder claims to have used 1 bit.*/ + nbits = nbits_total << OD_BITRES; + l = 0; + for (i = OD_BITRES; i-- > 0;) { + int b; + rng = rng * rng >> 15; + b = (int)(rng >> 16); + l = l << 1 | b; + rng >>= b; + } + return nbits - l; +} diff --git a/third_party/aom/aom_dsp/entcode.h b/third_party/aom/aom_dsp/entcode.h new file mode 100644 index 0000000000..534959e661 --- /dev/null +++ b/third_party/aom/aom_dsp/entcode.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if !defined(_entcode_H) +#define _entcode_H (1) +#include +#include +#include "av1/common/odintrin.h" + +/*OPT: od_ec_window must be at least 32 bits, but if you have fast arithmetic + on a larger type, you can speed up the decoder by using it here.*/ +typedef uint32_t od_ec_window; + +#define OD_EC_WINDOW_SIZE ((int)sizeof(od_ec_window) * CHAR_BIT) + +/*The number of bits to use for the range-coded part of unsigned integers.*/ +#define OD_EC_UINT_BITS (4) + +/*The resolution of fractional-precision bit usage measurements, i.e., + 3 => 1/8th bits.*/ +#define OD_BITRES (3) + +/*With CONFIG_EC_SMALLMUL, the value stored in a CDF is 32768 minus the actual + Q15 cumulative probability (an "inverse" CDF). + This function converts from one representation to the other (and is its own + inverse).*/ +#if CONFIG_EC_SMALLMUL +#define OD_ICDF(x) (32768U - (x)) +#else +#define OD_ICDF(x) (x) +#endif + +/*See entcode.c for further documentation.*/ + +OD_WARN_UNUSED_RESULT uint32_t od_ec_tell_frac(uint32_t nbits_total, + uint32_t rng); + +#endif diff --git a/third_party/aom/aom_dsp/entdec.c b/third_party/aom/aom_dsp/entdec.c new file mode 100644 index 0000000000..49b176cd80 --- /dev/null +++ b/third_party/aom/aom_dsp/entdec.c @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifdef HAVE_CONFIG_H +#include "./config.h" +#endif + +#include "aom_dsp/entdec.h" + +/*A range decoder. + This is an entropy decoder based upon \cite{Mar79}, which is itself a + rediscovery of the FIFO arithmetic code introduced by \cite{Pas76}. + It is very similar to arithmetic encoding, except that encoding is done with + digits in any base, instead of with bits, and so it is faster when using + larger bases (i.e.: a byte). + The author claims an average waste of $\frac{1}{2}\log_b(2b)$ bits, where $b$ + is the base, longer than the theoretical optimum, but to my knowledge there + is no published justification for this claim. + This only seems true when using near-infinite precision arithmetic so that + the process is carried out with no rounding errors. + + An excellent description of implementation details is available at + http://www.arturocampos.com/ac_range.html + A recent work \cite{MNW98} which proposes several changes to arithmetic + encoding for efficiency actually re-discovers many of the principles + behind range encoding, and presents a good theoretical analysis of them. + + End of stream is handled by writing out the smallest number of bits that + ensures that the stream will be correctly decoded regardless of the value of + any subsequent bits. + od_ec_dec_tell() can be used to determine how many bits were needed to decode + all the symbols thus far; other data can be packed in the remaining bits of + the input buffer. + @PHDTHESIS{Pas76, + author="Richard Clark Pasco", + title="Source coding algorithms for fast data compression", + school="Dept. of Electrical Engineering, Stanford University", + address="Stanford, CA", + month=May, + year=1976, + URL="http://www.richpasco.org/scaffdc.pdf" + } + @INPROCEEDINGS{Mar79, + author="Martin, G.N.N.", + title="Range encoding: an algorithm for removing redundancy from a digitised + message", + booktitle="Video & Data Recording Conference", + year=1979, + address="Southampton", + month=Jul, + URL="http://www.compressconsult.com/rangecoder/rngcod.pdf.gz" + } + @ARTICLE{MNW98, + author="Alistair Moffat and Radford Neal and Ian H. Witten", + title="Arithmetic Coding Revisited", + journal="{ACM} Transactions on Information Systems", + year=1998, + volume=16, + number=3, + pages="256--294", + month=Jul, + URL="http://researchcommons.waikato.ac.nz/bitstream/handle/10289/78/content.pdf" + }*/ + +/*This is meant to be a large, positive constant that can still be efficiently + loaded as an immediate (on platforms like ARM, for example). + Even relatively modest values like 100 would work fine.*/ +#define OD_EC_LOTS_OF_BITS (0x4000) + +static void od_ec_dec_refill(od_ec_dec *dec) { + int s; + od_ec_window dif; + int16_t cnt; + const unsigned char *bptr; + const unsigned char *end; + dif = dec->dif; + cnt = dec->cnt; + bptr = dec->bptr; + end = dec->end; + s = OD_EC_WINDOW_SIZE - 9 - (cnt + 15); + for (; s >= 0 && bptr < end; s -= 8, bptr++) { + OD_ASSERT(s <= OD_EC_WINDOW_SIZE - 8); + dif ^= (od_ec_window)bptr[0] << s; + cnt += 8; + } + if (bptr >= end) { + dec->tell_offs += OD_EC_LOTS_OF_BITS - cnt; + cnt = OD_EC_LOTS_OF_BITS; + } + dec->dif = dif; + dec->cnt = cnt; + dec->bptr = bptr; +} + +/*Takes updated dif and range values, renormalizes them so that + 32768 <= rng < 65536 (reading more bytes from the stream into dif if + necessary), and stores them back in the decoder context. + dif: The new value of dif. + rng: The new value of the range. + ret: The value to return. + Return: ret. + This allows the compiler to jump to this function via a tail-call.*/ +static int od_ec_dec_normalize(od_ec_dec *dec, od_ec_window dif, unsigned rng, + int ret) { + int d; + OD_ASSERT(rng <= 65535U); + d = 16 - OD_ILOG_NZ(rng); + dec->cnt -= d; +#if CONFIG_EC_SMALLMUL + /*This is equivalent to shifting in 1's instead of 0's.*/ + dec->dif = ((dif + 1) << d) - 1; +#else + dec->dif = dif << d; +#endif + dec->rng = rng << d; + if (dec->cnt < 0) od_ec_dec_refill(dec); + return ret; +} + +/*Initializes the decoder. + buf: The input buffer to use. + Return: 0 on success, or a negative value on error.*/ +void od_ec_dec_init(od_ec_dec *dec, const unsigned char *buf, + uint32_t storage) { + dec->buf = buf; + dec->eptr = buf + storage; + dec->end_window = 0; + dec->nend_bits = 0; + dec->tell_offs = 10 - (OD_EC_WINDOW_SIZE - 8); + dec->end = buf + storage; + dec->bptr = buf; +#if CONFIG_EC_SMALLMUL + dec->dif = ((od_ec_window)1 << (OD_EC_WINDOW_SIZE - 1)) - 1; +#else + dec->dif = 0; +#endif + dec->rng = 0x8000; + dec->cnt = -15; + dec->error = 0; + od_ec_dec_refill(dec); +} + +/*Decode a single binary value. + {EC_SMALLMUL} f: The probability that the bit is one, scaled by 32768. + {else} f: The probability that the bit is zero, scaled by 32768. + Return: The value decoded (0 or 1).*/ +int od_ec_decode_bool_q15(od_ec_dec *dec, unsigned f) { + od_ec_window dif; + od_ec_window vw; + unsigned r; + unsigned r_new; + unsigned v; + int ret; + OD_ASSERT(0 < f); + OD_ASSERT(f < 32768U); + dif = dec->dif; + r = dec->rng; + OD_ASSERT(dif >> (OD_EC_WINDOW_SIZE - 16) < r); + OD_ASSERT(32768U <= r); +#if CONFIG_EC_SMALLMUL + v = (r >> 8) * (uint32_t)f >> 7; + vw = (od_ec_window)v << (OD_EC_WINDOW_SIZE - 16); + ret = 1; + r_new = v; + if (dif >= vw) { + r_new = r - v; + dif -= vw; + ret = 0; + } +#else + v = f * (uint32_t)r >> 15; + vw = (od_ec_window)v << (OD_EC_WINDOW_SIZE - 16); + ret = 0; + r_new = v; + if (dif >= vw) { + r_new = r - v; + dif -= vw; + ret = 1; + } +#endif + return od_ec_dec_normalize(dec, dif, r_new, ret); +} + +/*Decodes a symbol given a cumulative distribution function (CDF) table in Q15. + cdf: The CDF, such that symbol s falls in the range + [s > 0 ? cdf[s - 1] : 0, cdf[s]). + The values must be monotonically non-increasing, and cdf[nsyms - 1] + must be 32768. + {EC_SMALLMUL}: The CDF contains 32768 minus those values. + nsyms: The number of symbols in the alphabet. + This should be at most 16. + Return: The decoded symbol s.*/ +int od_ec_decode_cdf_q15(od_ec_dec *dec, const uint16_t *cdf, int nsyms) { + od_ec_window dif; + unsigned r; + unsigned c; + unsigned u; + unsigned v; + int ret; + (void)nsyms; + dif = dec->dif; + r = dec->rng; + OD_ASSERT(dif >> (OD_EC_WINDOW_SIZE - 16) < r); + OD_ASSERT(cdf[nsyms - 1] == OD_ICDF(32768U)); + OD_ASSERT(32768U <= r); +#if CONFIG_EC_SMALLMUL + c = (unsigned)(dif >> (OD_EC_WINDOW_SIZE - 16)); + v = r; + ret = -1; + do { + u = v; + v = (r >> 8) * (uint32_t)cdf[++ret] >> 7; + } while (c < v); + OD_ASSERT(v < u); + OD_ASSERT(u <= r); + r = u - v; + dif -= (od_ec_window)v << (OD_EC_WINDOW_SIZE - 16); +#else + c = (unsigned)(dif >> (OD_EC_WINDOW_SIZE - 16)); + v = 0; + ret = -1; + do { + u = v; + v = cdf[++ret] * (uint32_t)r >> 15; + } while (v <= c); + OD_ASSERT(u < v); + OD_ASSERT(v <= r); + r = v - u; + dif -= (od_ec_window)u << (OD_EC_WINDOW_SIZE - 16); +#endif + return od_ec_dec_normalize(dec, dif, r, ret); +} + +#if CONFIG_RAWBITS +/*Extracts a sequence of raw bits from the stream. + The bits must have been encoded with od_ec_enc_bits(). + ftb: The number of bits to extract. + This must be between 0 and 25, inclusive. + Return: The decoded bits.*/ +uint32_t od_ec_dec_bits_(od_ec_dec *dec, unsigned ftb) { + od_ec_window window; + int available; + uint32_t ret; + OD_ASSERT(ftb <= 25); + window = dec->end_window; + available = dec->nend_bits; + if ((unsigned)available < ftb) { + const unsigned char *buf; + const unsigned char *eptr; + buf = dec->buf; + eptr = dec->eptr; + OD_ASSERT(available <= OD_EC_WINDOW_SIZE - 8); + do { + if (eptr <= buf) { + dec->tell_offs += OD_EC_LOTS_OF_BITS - available; + available = OD_EC_LOTS_OF_BITS; + break; + } + window |= (od_ec_window) * --eptr << available; + available += 8; + } while (available <= OD_EC_WINDOW_SIZE - 8); + dec->eptr = eptr; + } + ret = (uint32_t)window & (((uint32_t)1 << ftb) - 1); + window >>= ftb; + available -= ftb; + dec->end_window = window; + dec->nend_bits = available; + return ret; +} +#endif + +/*Returns the number of bits "used" by the decoded symbols so far. + This same number can be computed in either the encoder or the decoder, and is + suitable for making coding decisions. + Return: The number of bits. + This will always be slightly larger than the exact value (e.g., all + rounding error is in the positive direction).*/ +int od_ec_dec_tell(const od_ec_dec *dec) { + return (int)(((dec->end - dec->eptr) + (dec->bptr - dec->buf)) * 8 - + dec->cnt - dec->nend_bits + dec->tell_offs); +} + +/*Returns the number of bits "used" by the decoded symbols so far. + This same number can be computed in either the encoder or the decoder, and is + suitable for making coding decisions. + Return: The number of bits scaled by 2**OD_BITRES. + This will always be slightly larger than the exact value (e.g., all + rounding error is in the positive direction).*/ +uint32_t od_ec_dec_tell_frac(const od_ec_dec *dec) { + return od_ec_tell_frac(od_ec_dec_tell(dec), dec->rng); +} diff --git a/third_party/aom/aom_dsp/entdec.h b/third_party/aom/aom_dsp/entdec.h new file mode 100644 index 0000000000..e1145e81d4 --- /dev/null +++ b/third_party/aom/aom_dsp/entdec.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if !defined(_entdec_H) +#define _entdec_H (1) +#include +#include "aom_dsp/entcode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct od_ec_dec od_ec_dec; + +#if defined(OD_ACCOUNTING) && OD_ACCOUNTING +#define OD_ACC_STR , char *acc_str +#define od_ec_dec_bits(dec, ftb, str) od_ec_dec_bits_(dec, ftb, str) +#else +#define OD_ACC_STR +#define od_ec_dec_bits(dec, ftb, str) od_ec_dec_bits_(dec, ftb) +#endif + +/*The entropy decoder context.*/ +struct od_ec_dec { + /*The start of the current input buffer.*/ + const unsigned char *buf; + /*The read pointer for the raw bits.*/ + const unsigned char *eptr; + /*Bits that will be read from/written at the end.*/ + od_ec_window end_window; + /*Number of valid bits in end_window.*/ + int nend_bits; + /*An offset used to keep track of tell after reaching the end of the stream. + This is constant throughout most of the decoding process, but becomes + important once we hit the end of the buffer and stop incrementing pointers + (and instead pretend cnt/nend_bits have lots of bits).*/ + int32_t tell_offs; + /*The end of the current input buffer.*/ + const unsigned char *end; + /*The read pointer for the entropy-coded bits.*/ + const unsigned char *bptr; + /*The difference between the coded value and the low end of the current + range. + {EC_SMALLMUL} The difference between the high end of the current range, + (low + rng), and the coded value, minus 1. + This stores up to OD_EC_WINDOW_SIZE bits of that difference, but the + decoder only uses the top 16 bits of the window to decode the next symbol. + As we shift up during renormalization, if we don't have enough bits left in + the window to fill the top 16, we'll read in more bits of the coded + value.*/ + od_ec_window dif; + /*The number of values in the current range.*/ + uint16_t rng; + /*The number of bits of data in the current value.*/ + int16_t cnt; + /*Nonzero if an error occurred.*/ + int error; +}; + +/*See entdec.c for further documentation.*/ + +void od_ec_dec_init(od_ec_dec *dec, const unsigned char *buf, uint32_t storage) + OD_ARG_NONNULL(1) OD_ARG_NONNULL(2); + +OD_WARN_UNUSED_RESULT int od_ec_decode_bool_q15(od_ec_dec *dec, unsigned f) + OD_ARG_NONNULL(1); +OD_WARN_UNUSED_RESULT int od_ec_decode_cdf_q15(od_ec_dec *dec, + const uint16_t *cdf, int nsyms) + OD_ARG_NONNULL(1) OD_ARG_NONNULL(2); + +OD_WARN_UNUSED_RESULT uint32_t od_ec_dec_bits_(od_ec_dec *dec, unsigned ftb) + OD_ARG_NONNULL(1); + +OD_WARN_UNUSED_RESULT int od_ec_dec_tell(const od_ec_dec *dec) + OD_ARG_NONNULL(1); +OD_WARN_UNUSED_RESULT uint32_t od_ec_dec_tell_frac(const od_ec_dec *dec) + OD_ARG_NONNULL(1); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/third_party/aom/aom_dsp/entenc.c b/third_party/aom/aom_dsp/entenc.c new file mode 100644 index 0000000000..a350f27f4a --- /dev/null +++ b/third_party/aom/aom_dsp/entenc.c @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifdef HAVE_CONFIG_H +#include "./config.h" +#endif + +#include +#include +#include "aom_dsp/entenc.h" + +/*A range encoder. + See entdec.c and the references for implementation details \cite{Mar79,MNW98}. + + @INPROCEEDINGS{Mar79, + author="Martin, G.N.N.", + title="Range encoding: an algorithm for removing redundancy from a digitised + message", + booktitle="Video \& Data Recording Conference", + year=1979, + address="Southampton", + month=Jul, + URL="http://www.compressconsult.com/rangecoder/rngcod.pdf.gz" + } + @ARTICLE{MNW98, + author="Alistair Moffat and Radford Neal and Ian H. Witten", + title="Arithmetic Coding Revisited", + journal="{ACM} Transactions on Information Systems", + year=1998, + volume=16, + number=3, + pages="256--294", + month=Jul, + URL="http://researchcommons.waikato.ac.nz/bitstream/handle/10289/78/content.pdf" + }*/ + +/*Takes updated low and range values, renormalizes them so that + 32768 <= rng < 65536 (flushing bytes from low to the pre-carry buffer if + necessary), and stores them back in the encoder context. + low: The new value of low. + rng: The new value of the range.*/ +static void od_ec_enc_normalize(od_ec_enc *enc, od_ec_window low, + unsigned rng) { + int d; + int c; + int s; + c = enc->cnt; + OD_ASSERT(rng <= 65535U); + d = 16 - OD_ILOG_NZ(rng); + s = c + d; + /*TODO: Right now we flush every time we have at least one byte available. + Instead we should use an od_ec_window and flush right before we're about to + shift bits off the end of the window. + For a 32-bit window this is about the same amount of work, but for a 64-bit + window it should be a fair win.*/ + if (s >= 0) { + uint16_t *buf; + uint32_t storage; + uint32_t offs; + unsigned m; + buf = enc->precarry_buf; + storage = enc->precarry_storage; + offs = enc->offs; + if (offs + 2 > storage) { + storage = 2 * storage + 2; + buf = (uint16_t *)realloc(buf, sizeof(*buf) * storage); + if (buf == NULL) { + enc->error = -1; + enc->offs = 0; + return; + } + enc->precarry_buf = buf; + enc->precarry_storage = storage; + } + c += 16; + m = (1 << c) - 1; + if (s >= 8) { + OD_ASSERT(offs < storage); + buf[offs++] = (uint16_t)(low >> c); + low &= m; + c -= 8; + m >>= 8; + } + OD_ASSERT(offs < storage); + buf[offs++] = (uint16_t)(low >> c); + s = c + d - 24; + low &= m; + enc->offs = offs; + } + enc->low = low << d; + enc->rng = rng << d; + enc->cnt = s; +} + +/*Initializes the encoder. + size: The initial size of the buffer, in bytes.*/ +void od_ec_enc_init(od_ec_enc *enc, uint32_t size) { + od_ec_enc_reset(enc); + enc->buf = (unsigned char *)malloc(sizeof(*enc->buf) * size); + enc->storage = size; + if (size > 0 && enc->buf == NULL) { + enc->storage = 0; + enc->error = -1; + } + enc->precarry_buf = (uint16_t *)malloc(sizeof(*enc->precarry_buf) * size); + enc->precarry_storage = size; + if (size > 0 && enc->precarry_buf == NULL) { + enc->precarry_storage = 0; + enc->error = -1; + } +} + +/*Reinitializes the encoder.*/ +void od_ec_enc_reset(od_ec_enc *enc) { + enc->end_offs = 0; + enc->end_window = 0; + enc->nend_bits = 0; + enc->offs = 0; + enc->low = 0; + enc->rng = 0x8000; + /*This is initialized to -9 so that it crosses zero after we've accumulated + one byte + one carry bit.*/ + enc->cnt = -9; + enc->error = 0; +#if OD_MEASURE_EC_OVERHEAD + enc->entropy = 0; + enc->nb_symbols = 0; +#endif +} + +/*Frees the buffers used by the encoder.*/ +void od_ec_enc_clear(od_ec_enc *enc) { + free(enc->precarry_buf); + free(enc->buf); +} + +/*Encodes a symbol given its frequency in Q15. + fl: The cumulative frequency of all symbols that come before the one to be + encoded. + fh: The cumulative frequency of all symbols up to and including the one to + be encoded. + {EC_SMALLMUL} Both values are 32768 minus that.*/ +static void od_ec_encode_q15(od_ec_enc *enc, unsigned fl, unsigned fh) { + od_ec_window l; + unsigned r; + unsigned u; + unsigned v; + l = enc->low; + r = enc->rng; + OD_ASSERT(32768U <= r); +#if CONFIG_EC_SMALLMUL + OD_ASSERT(fh < fl); + OD_ASSERT(fl <= 32768U); + if (fl < 32768U) { + u = (r >> 8) * (uint32_t)fl >> 7; + v = (r >> 8) * (uint32_t)fh >> 7; + l += r - u; + r = u - v; + } else { + r -= (r >> 8) * (uint32_t)fh >> 7; + } +#else + OD_ASSERT(fl < fh); + OD_ASSERT(fh <= 32768U); + u = fl * (uint32_t)r >> 15; + v = fh * (uint32_t)r >> 15; + r = v - u; + l += u; +#endif + od_ec_enc_normalize(enc, l, r); +#if OD_MEASURE_EC_OVERHEAD + enc->entropy -= OD_LOG2((double)(OD_ICDF(fh) - OD_ICDF(fl)) / 32768.); + enc->nb_symbols++; +#endif +} + +/*Encode a single binary value. + val: The value to encode (0 or 1). + {EC_SMALLMUL} f: The probability that the val is one, scaled by 32768. + {else} f: The probability that val is zero, scaled by 32768.*/ +void od_ec_encode_bool_q15(od_ec_enc *enc, int val, unsigned f) { + od_ec_window l; + unsigned r; + unsigned v; + OD_ASSERT(0 < f); + OD_ASSERT(f < 32768U); + l = enc->low; + r = enc->rng; + OD_ASSERT(32768U <= r); +#if CONFIG_EC_SMALLMUL + v = (r >> 8) * (uint32_t)f >> 7; + if (val) l += r - v; + r = val ? v : r - v; +#else + v = f * (uint32_t)r >> 15; + if (val) l += v; + r = val ? r - v : v; +#endif + od_ec_enc_normalize(enc, l, r); +#if OD_MEASURE_EC_OVERHEAD + enc->entropy -= + OD_LOG2((double)(val ? 32768 - OD_ICDF(f) : OD_ICDF(f)) / 32768.); + enc->nb_symbols++; +#endif +} + +/*Encodes a symbol given a cumulative distribution function (CDF) table in Q15. + s: The index of the symbol to encode. + cdf: The CDF, such that symbol s falls in the range + [s > 0 ? cdf[s - 1] : 0, cdf[s]). + The values must be monotonically non-decreasing, and the last value + must be exactly 32768. + nsyms: The number of symbols in the alphabet. + This should be at most 16.*/ +void od_ec_encode_cdf_q15(od_ec_enc *enc, int s, const uint16_t *cdf, + int nsyms) { + (void)nsyms; + OD_ASSERT(s >= 0); + OD_ASSERT(s < nsyms); + OD_ASSERT(cdf[nsyms - 1] == OD_ICDF(32768U)); + od_ec_encode_q15(enc, s > 0 ? cdf[s - 1] : OD_ICDF(0), cdf[s]); +} + +#if CONFIG_RAWBITS +/*Encodes a sequence of raw bits in the stream. + fl: The bits to encode. + ftb: The number of bits to encode. + This must be between 0 and 25, inclusive.*/ +void od_ec_enc_bits(od_ec_enc *enc, uint32_t fl, unsigned ftb) { + od_ec_window end_window; + int nend_bits; + OD_ASSERT(ftb <= 25); + OD_ASSERT(fl < (uint32_t)1 << ftb); +#if OD_MEASURE_EC_OVERHEAD + enc->entropy += ftb; +#endif + end_window = enc->end_window; + nend_bits = enc->nend_bits; + if (nend_bits + ftb > OD_EC_WINDOW_SIZE) { + unsigned char *buf; + uint32_t storage; + uint32_t end_offs; + buf = enc->buf; + storage = enc->storage; + end_offs = enc->end_offs; + if (end_offs + (OD_EC_WINDOW_SIZE >> 3) >= storage) { + unsigned char *new_buf; + uint32_t new_storage; + new_storage = 2 * storage + (OD_EC_WINDOW_SIZE >> 3); + new_buf = (unsigned char *)malloc(sizeof(*new_buf) * new_storage); + if (new_buf == NULL) { + enc->error = -1; + enc->end_offs = 0; + return; + } + OD_COPY(new_buf + new_storage - end_offs, buf + storage - end_offs, + end_offs); + storage = new_storage; + free(buf); + enc->buf = buf = new_buf; + enc->storage = storage; + } + do { + OD_ASSERT(end_offs < storage); + buf[storage - ++end_offs] = (unsigned char)end_window; + end_window >>= 8; + nend_bits -= 8; + } while (nend_bits >= 8); + enc->end_offs = end_offs; + } + OD_ASSERT(nend_bits + ftb <= OD_EC_WINDOW_SIZE); + end_window |= (od_ec_window)fl << nend_bits; + nend_bits += ftb; + enc->end_window = end_window; + enc->nend_bits = nend_bits; +} +#endif + +/*Overwrites a few bits at the very start of an existing stream, after they + have already been encoded. + This makes it possible to have a few flags up front, where it is easy for + decoders to access them without parsing the whole stream, even if their + values are not determined until late in the encoding process, without having + to buffer all the intermediate symbols in the encoder. + In order for this to work, at least nbits bits must have already been encoded + using probabilities that are an exact power of two. + The encoder can verify the number of encoded bits is sufficient, but cannot + check this latter condition. + val: The bits to encode (in the least nbits significant bits). + They will be decoded in order from most-significant to least. + nbits: The number of bits to overwrite. + This must be no more than 8.*/ +void od_ec_enc_patch_initial_bits(od_ec_enc *enc, unsigned val, int nbits) { + int shift; + unsigned mask; + OD_ASSERT(nbits >= 0); + OD_ASSERT(nbits <= 8); + OD_ASSERT(val < 1U << nbits); + shift = 8 - nbits; + mask = ((1U << nbits) - 1) << shift; + if (enc->offs > 0) { + /*The first byte has been finalized.*/ + enc->precarry_buf[0] = + (uint16_t)((enc->precarry_buf[0] & ~mask) | val << shift); + } else if (9 + enc->cnt + (enc->rng == 0x8000) > nbits) { + /*The first byte has yet to be output.*/ + enc->low = (enc->low & ~((od_ec_window)mask << (16 + enc->cnt))) | + (od_ec_window)val << (16 + enc->cnt + shift); + } else { + /*The encoder hasn't even encoded _nbits of data yet.*/ + enc->error = -1; + } +} + +#if OD_MEASURE_EC_OVERHEAD +#include +#endif + +/*Indicates that there are no more symbols to encode. + All remaining output bytes are flushed to the output buffer. + od_ec_enc_reset() should be called before using the encoder again. + bytes: Returns the size of the encoded data in the returned buffer. + Return: A pointer to the start of the final buffer, or NULL if there was an + encoding error.*/ +unsigned char *od_ec_enc_done(od_ec_enc *enc, uint32_t *nbytes) { + unsigned char *out; + uint32_t storage; + uint16_t *buf; + uint32_t offs; + uint32_t end_offs; + int nend_bits; + od_ec_window m; + od_ec_window e; + od_ec_window l; + unsigned r; + int c; + int s; + if (enc->error) return NULL; +#if OD_MEASURE_EC_OVERHEAD + { + uint32_t tell; + /* Don't count the 1 bit we lose to raw bits as overhead. */ + tell = od_ec_enc_tell(enc) - 1; + fprintf(stderr, "overhead: %f%%\n", + 100 * (tell - enc->entropy) / enc->entropy); + fprintf(stderr, "efficiency: %f bits/symbol\n", + (double)tell / enc->nb_symbols); + } +#endif + /*We output the minimum number of bits that ensures that the symbols encoded + thus far will be decoded correctly regardless of the bits that follow.*/ + l = enc->low; + r = enc->rng; + c = enc->cnt; + s = 9; + m = 0x7FFF; + e = (l + m) & ~m; + while ((e | m) >= l + r) { + s++; + m >>= 1; + e = (l + m) & ~m; + } + s += c; + offs = enc->offs; + buf = enc->precarry_buf; + if (s > 0) { + unsigned n; + storage = enc->precarry_storage; + if (offs + ((s + 7) >> 3) > storage) { + storage = storage * 2 + ((s + 7) >> 3); + buf = (uint16_t *)realloc(buf, sizeof(*buf) * storage); + if (buf == NULL) { + enc->error = -1; + return NULL; + } + enc->precarry_buf = buf; + enc->precarry_storage = storage; + } + n = (1 << (c + 16)) - 1; + do { + OD_ASSERT(offs < storage); + buf[offs++] = (uint16_t)(e >> (c + 16)); + e &= n; + s -= 8; + c -= 8; + n >>= 8; + } while (s > 0); + } + /*Make sure there's enough room for the entropy-coded bits and the raw + bits.*/ + out = enc->buf; + storage = enc->storage; + end_offs = enc->end_offs; + e = enc->end_window; + nend_bits = enc->nend_bits; + s = -s; + c = OD_MAXI((nend_bits - s + 7) >> 3, 0); + if (offs + end_offs + c > storage) { + storage = offs + end_offs + c; + out = (unsigned char *)realloc(out, sizeof(*out) * storage); + if (out == NULL) { + enc->error = -1; + return NULL; + } + OD_MOVE(out + storage - end_offs, out + enc->storage - end_offs, end_offs); + enc->buf = out; + enc->storage = storage; + } + /*If we have buffered raw bits, flush them as well.*/ + while (nend_bits > s) { + OD_ASSERT(end_offs < storage); + out[storage - ++end_offs] = (unsigned char)e; + e >>= 8; + nend_bits -= 8; + } + *nbytes = offs + end_offs; + /*Perform carry propagation.*/ + OD_ASSERT(offs + end_offs <= storage); + out = out + storage - (offs + end_offs); + c = 0; + end_offs = offs; + while (offs > 0) { + offs--; + c = buf[offs] + c; + out[offs] = (unsigned char)c; + c >>= 8; + } + /*Add any remaining raw bits to the last byte. + There is guaranteed to be enough room, because nend_bits <= s.*/ + OD_ASSERT(nend_bits <= 0 || end_offs > 0); + if (nend_bits > 0) out[end_offs - 1] |= (unsigned char)e; + /*Note: Unless there's an allocation error, if you keep encoding into the + current buffer and call this function again later, everything will work + just fine (you won't get a new packet out, but you will get a single + buffer with the new data appended to the old). + However, this function is O(N) where N is the amount of data coded so far, + so calling it more than once for a given packet is a bad idea.*/ + return out; +} + +/*Returns the number of bits "used" by the encoded symbols so far. + This same number can be computed in either the encoder or the decoder, and is + suitable for making coding decisions. + Warning: The value returned by this function can decrease compared to an + earlier call, even after encoding more data, if there is an encoding error + (i.e., a failure to allocate enough space for the output buffer). + Return: The number of bits. + This will always be slightly larger than the exact value (e.g., all + rounding error is in the positive direction).*/ +int od_ec_enc_tell(const od_ec_enc *enc) { + /*The 10 here counteracts the offset of -9 baked into cnt, and adds 1 extra + bit, which we reserve for terminating the stream.*/ + return (enc->offs + enc->end_offs) * 8 + enc->cnt + enc->nend_bits + 10; +} + +/*Returns the number of bits "used" by the encoded symbols so far. + This same number can be computed in either the encoder or the decoder, and is + suitable for making coding decisions. + Warning: The value returned by this function can decrease compared to an + earlier call, even after encoding more data, if there is an encoding error + (i.e., a failure to allocate enough space for the output buffer). + Return: The number of bits scaled by 2**OD_BITRES. + This will always be slightly larger than the exact value (e.g., all + rounding error is in the positive direction).*/ +uint32_t od_ec_enc_tell_frac(const od_ec_enc *enc) { + return od_ec_tell_frac(od_ec_enc_tell(enc), enc->rng); +} + +/*Saves a entropy coder checkpoint to dst. + This allows an encoder to reverse a series of entropy coder + decisions if it decides that the information would have been + better coded some other way.*/ +void od_ec_enc_checkpoint(od_ec_enc *dst, const od_ec_enc *src) { + OD_COPY(dst, src, 1); +} + +/*Restores an entropy coder checkpoint saved by od_ec_enc_checkpoint. + This can only be used to restore from checkpoints earlier in the target + state's history: you can not switch backwards and forwards or otherwise + switch to a state which isn't a casual ancestor of the current state. + Restore is also incompatible with patching the initial bits, as the + changes will remain in the restored version.*/ +void od_ec_enc_rollback(od_ec_enc *dst, const od_ec_enc *src) { + unsigned char *buf; + uint32_t storage; + uint16_t *precarry_buf; + uint32_t precarry_storage; + OD_ASSERT(dst->storage >= src->storage); + OD_ASSERT(dst->precarry_storage >= src->precarry_storage); + buf = dst->buf; + storage = dst->storage; + precarry_buf = dst->precarry_buf; + precarry_storage = dst->precarry_storage; + OD_COPY(dst, src, 1); + dst->buf = buf; + dst->storage = storage; + dst->precarry_buf = precarry_buf; + dst->precarry_storage = precarry_storage; +} diff --git a/third_party/aom/aom_dsp/entenc.h b/third_party/aom/aom_dsp/entenc.h new file mode 100644 index 0000000000..314b363186 --- /dev/null +++ b/third_party/aom/aom_dsp/entenc.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if !defined(_entenc_H) +#define _entenc_H (1) +#include +#include "aom_dsp/entcode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct od_ec_enc od_ec_enc; + +#define OD_MEASURE_EC_OVERHEAD (0) + +/*The entropy encoder context.*/ +struct od_ec_enc { + /*Buffered output. + This contains only the raw bits until the final call to od_ec_enc_done(), + where all the arithmetic-coded data gets prepended to it.*/ + unsigned char *buf; + /*The size of the buffer.*/ + uint32_t storage; + /*The offset at which the last byte containing raw bits was written.*/ + uint32_t end_offs; + /*Bits that will be read from/written at the end.*/ + od_ec_window end_window; + /*Number of valid bits in end_window.*/ + int nend_bits; + /*A buffer for output bytes with their associated carry flags.*/ + uint16_t *precarry_buf; + /*The size of the pre-carry buffer.*/ + uint32_t precarry_storage; + /*The offset at which the next entropy-coded byte will be written.*/ + uint32_t offs; + /*The low end of the current range.*/ + od_ec_window low; + /*The number of values in the current range.*/ + uint16_t rng; + /*The number of bits of data in the current value.*/ + int16_t cnt; + /*Nonzero if an error occurred.*/ + int error; +#if OD_MEASURE_EC_OVERHEAD + double entropy; + int nb_symbols; +#endif +}; + +/*See entenc.c for further documentation.*/ + +void od_ec_enc_init(od_ec_enc *enc, uint32_t size) OD_ARG_NONNULL(1); +void od_ec_enc_reset(od_ec_enc *enc) OD_ARG_NONNULL(1); +void od_ec_enc_clear(od_ec_enc *enc) OD_ARG_NONNULL(1); + +void od_ec_encode_bool_q15(od_ec_enc *enc, int val, unsigned f_q15) + OD_ARG_NONNULL(1); +void od_ec_encode_cdf_q15(od_ec_enc *enc, int s, const uint16_t *cdf, int nsyms) + OD_ARG_NONNULL(1) OD_ARG_NONNULL(3); + +void od_ec_enc_bits(od_ec_enc *enc, uint32_t fl, unsigned ftb) + OD_ARG_NONNULL(1); + +void od_ec_enc_patch_initial_bits(od_ec_enc *enc, unsigned val, int nbits) + OD_ARG_NONNULL(1); +OD_WARN_UNUSED_RESULT unsigned char *od_ec_enc_done(od_ec_enc *enc, + uint32_t *nbytes) + OD_ARG_NONNULL(1) OD_ARG_NONNULL(2); + +OD_WARN_UNUSED_RESULT int od_ec_enc_tell(const od_ec_enc *enc) + OD_ARG_NONNULL(1); +OD_WARN_UNUSED_RESULT uint32_t od_ec_enc_tell_frac(const od_ec_enc *enc) + OD_ARG_NONNULL(1); + +void od_ec_enc_checkpoint(od_ec_enc *dst, const od_ec_enc *src); +void od_ec_enc_rollback(od_ec_enc *dst, const od_ec_enc *src); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/third_party/aom/aom_dsp/fastssim.c b/third_party/aom/aom_dsp/fastssim.c new file mode 100644 index 0000000000..09d945afc8 --- /dev/null +++ b/third_party/aom/aom_dsp/fastssim.c @@ -0,0 +1,493 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + * + * This code was originally written by: Nathan E. Egge, at the Daala + * project. + */ +#include +#include +#include +#include +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/ssim.h" +#include "aom_ports/system_state.h" + +typedef struct fs_level fs_level; +typedef struct fs_ctx fs_ctx; + +#define SSIM_C1 (255 * 255 * 0.01 * 0.01) +#define SSIM_C2 (255 * 255 * 0.03 * 0.03) +#if CONFIG_HIGHBITDEPTH +#define SSIM_C1_10 (1023 * 1023 * 0.01 * 0.01) +#define SSIM_C1_12 (4095 * 4095 * 0.01 * 0.01) +#define SSIM_C2_10 (1023 * 1023 * 0.03 * 0.03) +#define SSIM_C2_12 (4095 * 4095 * 0.03 * 0.03) +#endif +#define FS_MINI(_a, _b) ((_a) < (_b) ? (_a) : (_b)) +#define FS_MAXI(_a, _b) ((_a) > (_b) ? (_a) : (_b)) + +struct fs_level { + uint32_t *im1; + uint32_t *im2; + double *ssim; + int w; + int h; +}; + +struct fs_ctx { + fs_level *level; + int nlevels; + unsigned *col_buf; +}; + +static void fs_ctx_init(fs_ctx *_ctx, int _w, int _h, int _nlevels) { + unsigned char *data; + size_t data_size; + int lw; + int lh; + int l; + lw = (_w + 1) >> 1; + lh = (_h + 1) >> 1; + data_size = + _nlevels * sizeof(fs_level) + 2 * (lw + 8) * 8 * sizeof(*_ctx->col_buf); + for (l = 0; l < _nlevels; l++) { + size_t im_size; + size_t level_size; + im_size = lw * (size_t)lh; + level_size = 2 * im_size * sizeof(*_ctx->level[l].im1); + level_size += sizeof(*_ctx->level[l].ssim) - 1; + level_size /= sizeof(*_ctx->level[l].ssim); + level_size += im_size; + level_size *= sizeof(*_ctx->level[l].ssim); + data_size += level_size; + lw = (lw + 1) >> 1; + lh = (lh + 1) >> 1; + } + data = (unsigned char *)malloc(data_size); + _ctx->level = (fs_level *)data; + _ctx->nlevels = _nlevels; + data += _nlevels * sizeof(*_ctx->level); + lw = (_w + 1) >> 1; + lh = (_h + 1) >> 1; + for (l = 0; l < _nlevels; l++) { + size_t im_size; + size_t level_size; + _ctx->level[l].w = lw; + _ctx->level[l].h = lh; + im_size = lw * (size_t)lh; + level_size = 2 * im_size * sizeof(*_ctx->level[l].im1); + level_size += sizeof(*_ctx->level[l].ssim) - 1; + level_size /= sizeof(*_ctx->level[l].ssim); + level_size *= sizeof(*_ctx->level[l].ssim); + _ctx->level[l].im1 = (uint32_t *)data; + _ctx->level[l].im2 = _ctx->level[l].im1 + im_size; + data += level_size; + _ctx->level[l].ssim = (double *)data; + data += im_size * sizeof(*_ctx->level[l].ssim); + lw = (lw + 1) >> 1; + lh = (lh + 1) >> 1; + } + _ctx->col_buf = (unsigned *)data; +} + +static void fs_ctx_clear(fs_ctx *_ctx) { free(_ctx->level); } + +static void fs_downsample_level(fs_ctx *_ctx, int _l) { + const uint32_t *src1; + const uint32_t *src2; + uint32_t *dst1; + uint32_t *dst2; + int w2; + int h2; + int w; + int h; + int i; + int j; + w = _ctx->level[_l].w; + h = _ctx->level[_l].h; + dst1 = _ctx->level[_l].im1; + dst2 = _ctx->level[_l].im2; + w2 = _ctx->level[_l - 1].w; + h2 = _ctx->level[_l - 1].h; + src1 = _ctx->level[_l - 1].im1; + src2 = _ctx->level[_l - 1].im2; + for (j = 0; j < h; j++) { + int j0offs; + int j1offs; + j0offs = 2 * j * w2; + j1offs = FS_MINI(2 * j + 1, h2) * w2; + for (i = 0; i < w; i++) { + int i0; + int i1; + i0 = 2 * i; + i1 = FS_MINI(i0 + 1, w2); + dst1[j * w + i] = src1[j0offs + i0] + src1[j0offs + i1] + + src1[j1offs + i0] + src1[j1offs + i1]; + dst2[j * w + i] = src2[j0offs + i0] + src2[j0offs + i1] + + src2[j1offs + i0] + src2[j1offs + i1]; + } + } +} + +static void fs_downsample_level0(fs_ctx *_ctx, const uint8_t *_src1, + int _s1ystride, const uint8_t *_src2, + int _s2ystride, int _w, int _h, uint32_t bd, + uint32_t shift) { + uint32_t *dst1; + uint32_t *dst2; + int w; + int h; + int i; + int j; + w = _ctx->level[0].w; + h = _ctx->level[0].h; + dst1 = _ctx->level[0].im1; + dst2 = _ctx->level[0].im2; + for (j = 0; j < h; j++) { + int j0; + int j1; + j0 = 2 * j; + j1 = FS_MINI(j0 + 1, _h); + for (i = 0; i < w; i++) { + int i0; + int i1; + i0 = 2 * i; + i1 = FS_MINI(i0 + 1, _w); + if (bd == 8 && shift == 0) { + dst1[j * w + i] = + _src1[j0 * _s1ystride + i0] + _src1[j0 * _s1ystride + i1] + + _src1[j1 * _s1ystride + i0] + _src1[j1 * _s1ystride + i1]; + dst2[j * w + i] = + _src2[j0 * _s2ystride + i0] + _src2[j0 * _s2ystride + i1] + + _src2[j1 * _s2ystride + i0] + _src2[j1 * _s2ystride + i1]; + } else { + uint16_t *src1s = CONVERT_TO_SHORTPTR(_src1); + uint16_t *src2s = CONVERT_TO_SHORTPTR(_src2); + dst1[j * w + i] = (src1s[j0 * _s1ystride + i0] >> shift) + + (src1s[j0 * _s1ystride + i1] >> shift) + + (src1s[j1 * _s1ystride + i0] >> shift) + + (src1s[j1 * _s1ystride + i1] >> shift); + dst2[j * w + i] = (src2s[j0 * _s2ystride + i0] >> shift) + + (src2s[j0 * _s2ystride + i1] >> shift) + + (src2s[j1 * _s2ystride + i0] >> shift) + + (src2s[j1 * _s2ystride + i1] >> shift); + } + } + } +} + +static void fs_apply_luminance(fs_ctx *_ctx, int _l, int bit_depth) { + unsigned *col_sums_x; + unsigned *col_sums_y; + uint32_t *im1; + uint32_t *im2; + double *ssim; + double c1; + int w; + int h; + int j0offs; + int j1offs; + int i; + int j; + double ssim_c1 = SSIM_C1; +#if CONFIG_HIGHBITDEPTH + if (bit_depth == 10) ssim_c1 = SSIM_C1_10; + if (bit_depth == 12) ssim_c1 = SSIM_C1_12; +#else + assert(bit_depth == 8); + (void)bit_depth; +#endif + w = _ctx->level[_l].w; + h = _ctx->level[_l].h; + col_sums_x = _ctx->col_buf; + col_sums_y = col_sums_x + w; + im1 = _ctx->level[_l].im1; + im2 = _ctx->level[_l].im2; + for (i = 0; i < w; i++) col_sums_x[i] = 5 * im1[i]; + for (i = 0; i < w; i++) col_sums_y[i] = 5 * im2[i]; + for (j = 1; j < 4; j++) { + j1offs = FS_MINI(j, h - 1) * w; + for (i = 0; i < w; i++) col_sums_x[i] += im1[j1offs + i]; + for (i = 0; i < w; i++) col_sums_y[i] += im2[j1offs + i]; + } + ssim = _ctx->level[_l].ssim; + c1 = (double)(ssim_c1 * 4096 * (1 << 4 * _l)); + for (j = 0; j < h; j++) { + unsigned mux; + unsigned muy; + int i0; + int i1; + mux = 5 * col_sums_x[0]; + muy = 5 * col_sums_y[0]; + for (i = 1; i < 4; i++) { + i1 = FS_MINI(i, w - 1); + mux += col_sums_x[i1]; + muy += col_sums_y[i1]; + } + for (i = 0; i < w; i++) { + ssim[j * w + i] *= (2 * mux * (double)muy + c1) / + (mux * (double)mux + muy * (double)muy + c1); + if (i + 1 < w) { + i0 = FS_MAXI(0, i - 4); + i1 = FS_MINI(i + 4, w - 1); + mux += col_sums_x[i1] - col_sums_x[i0]; + muy += col_sums_x[i1] - col_sums_x[i0]; + } + } + if (j + 1 < h) { + j0offs = FS_MAXI(0, j - 4) * w; + for (i = 0; i < w; i++) col_sums_x[i] -= im1[j0offs + i]; + for (i = 0; i < w; i++) col_sums_y[i] -= im2[j0offs + i]; + j1offs = FS_MINI(j + 4, h - 1) * w; + for (i = 0; i < w; i++) col_sums_x[i] += im1[j1offs + i]; + for (i = 0; i < w; i++) col_sums_y[i] += im2[j1offs + i]; + } + } +} + +#define FS_COL_SET(_col, _joffs, _ioffs) \ + do { \ + unsigned gx; \ + unsigned gy; \ + gx = gx_buf[((j + (_joffs)) & 7) * stride + i + (_ioffs)]; \ + gy = gy_buf[((j + (_joffs)) & 7) * stride + i + (_ioffs)]; \ + col_sums_gx2[(_col)] = gx * (double)gx; \ + col_sums_gy2[(_col)] = gy * (double)gy; \ + col_sums_gxgy[(_col)] = gx * (double)gy; \ + } while (0) + +#define FS_COL_ADD(_col, _joffs, _ioffs) \ + do { \ + unsigned gx; \ + unsigned gy; \ + gx = gx_buf[((j + (_joffs)) & 7) * stride + i + (_ioffs)]; \ + gy = gy_buf[((j + (_joffs)) & 7) * stride + i + (_ioffs)]; \ + col_sums_gx2[(_col)] += gx * (double)gx; \ + col_sums_gy2[(_col)] += gy * (double)gy; \ + col_sums_gxgy[(_col)] += gx * (double)gy; \ + } while (0) + +#define FS_COL_SUB(_col, _joffs, _ioffs) \ + do { \ + unsigned gx; \ + unsigned gy; \ + gx = gx_buf[((j + (_joffs)) & 7) * stride + i + (_ioffs)]; \ + gy = gy_buf[((j + (_joffs)) & 7) * stride + i + (_ioffs)]; \ + col_sums_gx2[(_col)] -= gx * (double)gx; \ + col_sums_gy2[(_col)] -= gy * (double)gy; \ + col_sums_gxgy[(_col)] -= gx * (double)gy; \ + } while (0) + +#define FS_COL_COPY(_col1, _col2) \ + do { \ + col_sums_gx2[(_col1)] = col_sums_gx2[(_col2)]; \ + col_sums_gy2[(_col1)] = col_sums_gy2[(_col2)]; \ + col_sums_gxgy[(_col1)] = col_sums_gxgy[(_col2)]; \ + } while (0) + +#define FS_COL_HALVE(_col1, _col2) \ + do { \ + col_sums_gx2[(_col1)] = col_sums_gx2[(_col2)] * 0.5; \ + col_sums_gy2[(_col1)] = col_sums_gy2[(_col2)] * 0.5; \ + col_sums_gxgy[(_col1)] = col_sums_gxgy[(_col2)] * 0.5; \ + } while (0) + +#define FS_COL_DOUBLE(_col1, _col2) \ + do { \ + col_sums_gx2[(_col1)] = col_sums_gx2[(_col2)] * 2; \ + col_sums_gy2[(_col1)] = col_sums_gy2[(_col2)] * 2; \ + col_sums_gxgy[(_col1)] = col_sums_gxgy[(_col2)] * 2; \ + } while (0) + +static void fs_calc_structure(fs_ctx *_ctx, int _l, int bit_depth) { + uint32_t *im1; + uint32_t *im2; + unsigned *gx_buf; + unsigned *gy_buf; + double *ssim; + double col_sums_gx2[8]; + double col_sums_gy2[8]; + double col_sums_gxgy[8]; + double c2; + int stride; + int w; + int h; + int i; + int j; + double ssim_c2 = SSIM_C2; +#if CONFIG_HIGHBITDEPTH + if (bit_depth == 10) ssim_c2 = SSIM_C2_10; + if (bit_depth == 12) ssim_c2 = SSIM_C2_12; +#else + assert(bit_depth == 8); + (void)bit_depth; +#endif + + w = _ctx->level[_l].w; + h = _ctx->level[_l].h; + im1 = _ctx->level[_l].im1; + im2 = _ctx->level[_l].im2; + ssim = _ctx->level[_l].ssim; + gx_buf = _ctx->col_buf; + stride = w + 8; + gy_buf = gx_buf + 8 * stride; + memset(gx_buf, 0, 2 * 8 * stride * sizeof(*gx_buf)); + c2 = ssim_c2 * (1 << 4 * _l) * 16 * 104; + for (j = 0; j < h + 4; j++) { + if (j < h - 1) { + for (i = 0; i < w - 1; i++) { + unsigned g1; + unsigned g2; + unsigned gx; + unsigned gy; + g1 = abs((int)im1[(j + 1) * w + i + 1] - (int)im1[j * w + i]); + g2 = abs((int)im1[(j + 1) * w + i] - (int)im1[j * w + i + 1]); + gx = 4 * FS_MAXI(g1, g2) + FS_MINI(g1, g2); + g1 = abs((int)im2[(j + 1) * w + i + 1] - (int)im2[j * w + i]); + g2 = abs((int)im2[(j + 1) * w + i] - (int)im2[j * w + i + 1]); + gy = 4 * FS_MAXI(g1, g2) + FS_MINI(g1, g2); + gx_buf[(j & 7) * stride + i + 4] = gx; + gy_buf[(j & 7) * stride + i + 4] = gy; + } + } else { + memset(gx_buf + (j & 7) * stride, 0, stride * sizeof(*gx_buf)); + memset(gy_buf + (j & 7) * stride, 0, stride * sizeof(*gy_buf)); + } + if (j >= 4) { + int k; + col_sums_gx2[3] = col_sums_gx2[2] = col_sums_gx2[1] = col_sums_gx2[0] = 0; + col_sums_gy2[3] = col_sums_gy2[2] = col_sums_gy2[1] = col_sums_gy2[0] = 0; + col_sums_gxgy[3] = col_sums_gxgy[2] = col_sums_gxgy[1] = + col_sums_gxgy[0] = 0; + for (i = 4; i < 8; i++) { + FS_COL_SET(i, -1, 0); + FS_COL_ADD(i, 0, 0); + for (k = 1; k < 8 - i; k++) { + FS_COL_DOUBLE(i, i); + FS_COL_ADD(i, -k - 1, 0); + FS_COL_ADD(i, k, 0); + } + } + for (i = 0; i < w; i++) { + double mugx2; + double mugy2; + double mugxgy; + mugx2 = col_sums_gx2[0]; + for (k = 1; k < 8; k++) mugx2 += col_sums_gx2[k]; + mugy2 = col_sums_gy2[0]; + for (k = 1; k < 8; k++) mugy2 += col_sums_gy2[k]; + mugxgy = col_sums_gxgy[0]; + for (k = 1; k < 8; k++) mugxgy += col_sums_gxgy[k]; + ssim[(j - 4) * w + i] = (2 * mugxgy + c2) / (mugx2 + mugy2 + c2); + if (i + 1 < w) { + FS_COL_SET(0, -1, 1); + FS_COL_ADD(0, 0, 1); + FS_COL_SUB(2, -3, 2); + FS_COL_SUB(2, 2, 2); + FS_COL_HALVE(1, 2); + FS_COL_SUB(3, -4, 3); + FS_COL_SUB(3, 3, 3); + FS_COL_HALVE(2, 3); + FS_COL_COPY(3, 4); + FS_COL_DOUBLE(4, 5); + FS_COL_ADD(4, -4, 5); + FS_COL_ADD(4, 3, 5); + FS_COL_DOUBLE(5, 6); + FS_COL_ADD(5, -3, 6); + FS_COL_ADD(5, 2, 6); + FS_COL_DOUBLE(6, 7); + FS_COL_ADD(6, -2, 7); + FS_COL_ADD(6, 1, 7); + FS_COL_SET(7, -1, 8); + FS_COL_ADD(7, 0, 8); + } + } + } + } +} + +#define FS_NLEVELS (4) + +/*These weights were derived from the default weights found in Wang's original + Matlab implementation: {0.0448, 0.2856, 0.2363, 0.1333}. + We drop the finest scale and renormalize the rest to sum to 1.*/ + +static const double FS_WEIGHTS[FS_NLEVELS] = { + 0.2989654541015625, 0.3141326904296875, 0.2473602294921875, 0.1395416259765625 +}; + +static double fs_average(fs_ctx *_ctx, int _l) { + double *ssim; + double ret; + int w; + int h; + int i; + int j; + w = _ctx->level[_l].w; + h = _ctx->level[_l].h; + ssim = _ctx->level[_l].ssim; + ret = 0; + for (j = 0; j < h; j++) + for (i = 0; i < w; i++) ret += ssim[j * w + i]; + return pow(ret / (w * h), FS_WEIGHTS[_l]); +} + +static double convert_ssim_db(double _ssim, double _weight) { + assert(_weight >= _ssim); + if ((_weight - _ssim) < 1e-10) return MAX_SSIM_DB; + return 10 * (log10(_weight) - log10(_weight - _ssim)); +} + +static double calc_ssim(const uint8_t *_src, int _systride, const uint8_t *_dst, + int _dystride, int _w, int _h, uint32_t _bd, + uint32_t _shift) { + fs_ctx ctx; + double ret; + int l; + ret = 1; + fs_ctx_init(&ctx, _w, _h, FS_NLEVELS); + fs_downsample_level0(&ctx, _src, _systride, _dst, _dystride, _w, _h, _bd, + _shift); + for (l = 0; l < FS_NLEVELS - 1; l++) { + fs_calc_structure(&ctx, l, _bd); + ret *= fs_average(&ctx, l); + fs_downsample_level(&ctx, l + 1); + } + fs_calc_structure(&ctx, l, _bd); + fs_apply_luminance(&ctx, l, _bd); + ret *= fs_average(&ctx, l); + fs_ctx_clear(&ctx); + return ret; +} + +double aom_calc_fastssim(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, double *ssim_y, + double *ssim_u, double *ssim_v, uint32_t bd, + uint32_t in_bd) { + double ssimv; + uint32_t bd_shift = 0; + aom_clear_system_state(); + assert(bd >= in_bd); + + bd_shift = bd - in_bd; + + *ssim_y = calc_ssim(source->y_buffer, source->y_stride, dest->y_buffer, + dest->y_stride, source->y_crop_width, + source->y_crop_height, in_bd, bd_shift); + *ssim_u = calc_ssim(source->u_buffer, source->uv_stride, dest->u_buffer, + dest->uv_stride, source->uv_crop_width, + source->uv_crop_height, in_bd, bd_shift); + *ssim_v = calc_ssim(source->v_buffer, source->uv_stride, dest->v_buffer, + dest->uv_stride, source->uv_crop_width, + source->uv_crop_height, in_bd, bd_shift); + ssimv = (*ssim_y) * .8 + .1 * ((*ssim_u) + (*ssim_v)); + return convert_ssim_db(ssimv, 1.0); +} diff --git a/third_party/aom/aom_dsp/fwd_txfm.c b/third_party/aom/aom_dsp/fwd_txfm.c new file mode 100644 index 0000000000..12ee02ba1f --- /dev/null +++ b/third_party/aom/aom_dsp/fwd_txfm.c @@ -0,0 +1,809 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/fwd_txfm.h" +#include +#include "./aom_dsp_rtcd.h" + +void aom_fdct4x4_c(const int16_t *input, tran_low_t *output, int stride) { + // The 2D transform is done with two passes which are actually pretty + // similar. In the first one, we transform the columns and transpose + // the results. In the second one, we transform the rows. To achieve that, + // as the first pass results are transposed, we transpose the columns (that + // is the transposed rows) and transpose the results (so that it goes back + // in normal/row positions). + int pass; + // We need an intermediate buffer between passes. + tran_low_t intermediate[4 * 4]; + const tran_low_t *in_low = NULL; + tran_low_t *out = intermediate; + // Do the two transform/transpose passes + for (pass = 0; pass < 2; ++pass) { + tran_high_t in_high[4]; // canbe16 + tran_high_t step[4]; // canbe16 + tran_high_t temp1, temp2; // needs32 + int i; + for (i = 0; i < 4; ++i) { + // Load inputs. + if (pass == 0) { + in_high[0] = input[0 * stride] * 16; + in_high[1] = input[1 * stride] * 16; + in_high[2] = input[2 * stride] * 16; + in_high[3] = input[3 * stride] * 16; + if (i == 0 && in_high[0]) { + ++in_high[0]; + } + } else { + assert(in_low != NULL); + in_high[0] = in_low[0 * 4]; + in_high[1] = in_low[1 * 4]; + in_high[2] = in_low[2 * 4]; + in_high[3] = in_low[3 * 4]; + ++in_low; + } + // Transform. + step[0] = in_high[0] + in_high[3]; + step[1] = in_high[1] + in_high[2]; + step[2] = in_high[1] - in_high[2]; + step[3] = in_high[0] - in_high[3]; + temp1 = (step[0] + step[1]) * cospi_16_64; + temp2 = (step[0] - step[1]) * cospi_16_64; + out[0] = (tran_low_t)fdct_round_shift(temp1); + out[2] = (tran_low_t)fdct_round_shift(temp2); + temp1 = step[2] * cospi_24_64 + step[3] * cospi_8_64; + temp2 = -step[2] * cospi_8_64 + step[3] * cospi_24_64; + out[1] = (tran_low_t)fdct_round_shift(temp1); + out[3] = (tran_low_t)fdct_round_shift(temp2); + // Do next column (which is a transposed row in second/horizontal pass) + ++input; + out += 4; + } + // Setup in/out for next pass. + in_low = intermediate; + out = output; + } + + { + int i, j; + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) output[j + i * 4] = (output[j + i * 4] + 1) >> 2; + } + } +} + +void aom_fdct4x4_1_c(const int16_t *input, tran_low_t *output, int stride) { + int r, c; + tran_low_t sum = 0; + for (r = 0; r < 4; ++r) + for (c = 0; c < 4; ++c) sum += input[r * stride + c]; + + output[0] = sum << 1; +} + +void aom_fdct8x8_c(const int16_t *input, tran_low_t *final_output, int stride) { + int i, j; + tran_low_t intermediate[64]; + int pass; + tran_low_t *output = intermediate; + const tran_low_t *in = NULL; + + // Transform columns + for (pass = 0; pass < 2; ++pass) { + tran_high_t s0, s1, s2, s3, s4, s5, s6, s7; // canbe16 + tran_high_t t0, t1, t2, t3; // needs32 + tran_high_t x0, x1, x2, x3; // canbe16 + + for (i = 0; i < 8; i++) { + // stage 1 + if (pass == 0) { + s0 = (input[0 * stride] + input[7 * stride]) * 4; + s1 = (input[1 * stride] + input[6 * stride]) * 4; + s2 = (input[2 * stride] + input[5 * stride]) * 4; + s3 = (input[3 * stride] + input[4 * stride]) * 4; + s4 = (input[3 * stride] - input[4 * stride]) * 4; + s5 = (input[2 * stride] - input[5 * stride]) * 4; + s6 = (input[1 * stride] - input[6 * stride]) * 4; + s7 = (input[0 * stride] - input[7 * stride]) * 4; + ++input; + } else { + s0 = in[0 * 8] + in[7 * 8]; + s1 = in[1 * 8] + in[6 * 8]; + s2 = in[2 * 8] + in[5 * 8]; + s3 = in[3 * 8] + in[4 * 8]; + s4 = in[3 * 8] - in[4 * 8]; + s5 = in[2 * 8] - in[5 * 8]; + s6 = in[1 * 8] - in[6 * 8]; + s7 = in[0 * 8] - in[7 * 8]; + ++in; + } + + // fdct4(step, step); + x0 = s0 + s3; + x1 = s1 + s2; + x2 = s1 - s2; + x3 = s0 - s3; + t0 = (x0 + x1) * cospi_16_64; + t1 = (x0 - x1) * cospi_16_64; + t2 = x2 * cospi_24_64 + x3 * cospi_8_64; + t3 = -x2 * cospi_8_64 + x3 * cospi_24_64; + output[0] = (tran_low_t)fdct_round_shift(t0); + output[2] = (tran_low_t)fdct_round_shift(t2); + output[4] = (tran_low_t)fdct_round_shift(t1); + output[6] = (tran_low_t)fdct_round_shift(t3); + + // Stage 2 + t0 = (s6 - s5) * cospi_16_64; + t1 = (s6 + s5) * cospi_16_64; + t2 = fdct_round_shift(t0); + t3 = fdct_round_shift(t1); + + // Stage 3 + x0 = s4 + t2; + x1 = s4 - t2; + x2 = s7 - t3; + x3 = s7 + t3; + + // Stage 4 + t0 = x0 * cospi_28_64 + x3 * cospi_4_64; + t1 = x1 * cospi_12_64 + x2 * cospi_20_64; + t2 = x2 * cospi_12_64 + x1 * -cospi_20_64; + t3 = x3 * cospi_28_64 + x0 * -cospi_4_64; + output[1] = (tran_low_t)fdct_round_shift(t0); + output[3] = (tran_low_t)fdct_round_shift(t2); + output[5] = (tran_low_t)fdct_round_shift(t1); + output[7] = (tran_low_t)fdct_round_shift(t3); + output += 8; + } + in = intermediate; + output = final_output; + } + + // Rows + for (i = 0; i < 8; ++i) { + for (j = 0; j < 8; ++j) final_output[j + i * 8] /= 2; + } +} + +void aom_fdct8x8_1_c(const int16_t *input, tran_low_t *output, int stride) { + int r, c; + tran_low_t sum = 0; + for (r = 0; r < 8; ++r) + for (c = 0; c < 8; ++c) sum += input[r * stride + c]; + + output[0] = sum; +} + +void aom_fdct16x16_c(const int16_t *input, tran_low_t *output, int stride) { + // The 2D transform is done with two passes which are actually pretty + // similar. In the first one, we transform the columns and transpose + // the results. In the second one, we transform the rows. To achieve that, + // as the first pass results are transposed, we transpose the columns (that + // is the transposed rows) and transpose the results (so that it goes back + // in normal/row positions). + int pass; + // We need an intermediate buffer between passes. + tran_low_t intermediate[256]; + const tran_low_t *in_low = NULL; + tran_low_t *out = intermediate; + // Do the two transform/transpose passes + for (pass = 0; pass < 2; ++pass) { + tran_high_t step1[8]; // canbe16 + tran_high_t step2[8]; // canbe16 + tran_high_t step3[8]; // canbe16 + tran_high_t in_high[8]; // canbe16 + tran_high_t temp1, temp2; // needs32 + int i; + for (i = 0; i < 16; i++) { + if (0 == pass) { + // Calculate input for the first 8 results. + in_high[0] = (input[0 * stride] + input[15 * stride]) * 4; + in_high[1] = (input[1 * stride] + input[14 * stride]) * 4; + in_high[2] = (input[2 * stride] + input[13 * stride]) * 4; + in_high[3] = (input[3 * stride] + input[12 * stride]) * 4; + in_high[4] = (input[4 * stride] + input[11 * stride]) * 4; + in_high[5] = (input[5 * stride] + input[10 * stride]) * 4; + in_high[6] = (input[6 * stride] + input[9 * stride]) * 4; + in_high[7] = (input[7 * stride] + input[8 * stride]) * 4; + // Calculate input for the next 8 results. + step1[0] = (input[7 * stride] - input[8 * stride]) * 4; + step1[1] = (input[6 * stride] - input[9 * stride]) * 4; + step1[2] = (input[5 * stride] - input[10 * stride]) * 4; + step1[3] = (input[4 * stride] - input[11 * stride]) * 4; + step1[4] = (input[3 * stride] - input[12 * stride]) * 4; + step1[5] = (input[2 * stride] - input[13 * stride]) * 4; + step1[6] = (input[1 * stride] - input[14 * stride]) * 4; + step1[7] = (input[0 * stride] - input[15 * stride]) * 4; + } else { + // Calculate input for the first 8 results. + assert(in_low != NULL); + in_high[0] = ((in_low[0 * 16] + 1) >> 2) + ((in_low[15 * 16] + 1) >> 2); + in_high[1] = ((in_low[1 * 16] + 1) >> 2) + ((in_low[14 * 16] + 1) >> 2); + in_high[2] = ((in_low[2 * 16] + 1) >> 2) + ((in_low[13 * 16] + 1) >> 2); + in_high[3] = ((in_low[3 * 16] + 1) >> 2) + ((in_low[12 * 16] + 1) >> 2); + in_high[4] = ((in_low[4 * 16] + 1) >> 2) + ((in_low[11 * 16] + 1) >> 2); + in_high[5] = ((in_low[5 * 16] + 1) >> 2) + ((in_low[10 * 16] + 1) >> 2); + in_high[6] = ((in_low[6 * 16] + 1) >> 2) + ((in_low[9 * 16] + 1) >> 2); + in_high[7] = ((in_low[7 * 16] + 1) >> 2) + ((in_low[8 * 16] + 1) >> 2); + // Calculate input for the next 8 results. + step1[0] = ((in_low[7 * 16] + 1) >> 2) - ((in_low[8 * 16] + 1) >> 2); + step1[1] = ((in_low[6 * 16] + 1) >> 2) - ((in_low[9 * 16] + 1) >> 2); + step1[2] = ((in_low[5 * 16] + 1) >> 2) - ((in_low[10 * 16] + 1) >> 2); + step1[3] = ((in_low[4 * 16] + 1) >> 2) - ((in_low[11 * 16] + 1) >> 2); + step1[4] = ((in_low[3 * 16] + 1) >> 2) - ((in_low[12 * 16] + 1) >> 2); + step1[5] = ((in_low[2 * 16] + 1) >> 2) - ((in_low[13 * 16] + 1) >> 2); + step1[6] = ((in_low[1 * 16] + 1) >> 2) - ((in_low[14 * 16] + 1) >> 2); + step1[7] = ((in_low[0 * 16] + 1) >> 2) - ((in_low[15 * 16] + 1) >> 2); + in_low++; + } + // Work on the first eight values; fdct8(input, even_results); + { + tran_high_t s0, s1, s2, s3, s4, s5, s6, s7; // canbe16 + tran_high_t t0, t1, t2, t3; // needs32 + tran_high_t x0, x1, x2, x3; // canbe16 + + // stage 1 + s0 = in_high[0] + in_high[7]; + s1 = in_high[1] + in_high[6]; + s2 = in_high[2] + in_high[5]; + s3 = in_high[3] + in_high[4]; + s4 = in_high[3] - in_high[4]; + s5 = in_high[2] - in_high[5]; + s6 = in_high[1] - in_high[6]; + s7 = in_high[0] - in_high[7]; + + // fdct4(step, step); + x0 = s0 + s3; + x1 = s1 + s2; + x2 = s1 - s2; + x3 = s0 - s3; + t0 = (x0 + x1) * cospi_16_64; + t1 = (x0 - x1) * cospi_16_64; + t2 = x3 * cospi_8_64 + x2 * cospi_24_64; + t3 = x3 * cospi_24_64 - x2 * cospi_8_64; + out[0] = (tran_low_t)fdct_round_shift(t0); + out[4] = (tran_low_t)fdct_round_shift(t2); + out[8] = (tran_low_t)fdct_round_shift(t1); + out[12] = (tran_low_t)fdct_round_shift(t3); + + // Stage 2 + t0 = (s6 - s5) * cospi_16_64; + t1 = (s6 + s5) * cospi_16_64; + t2 = fdct_round_shift(t0); + t3 = fdct_round_shift(t1); + + // Stage 3 + x0 = s4 + t2; + x1 = s4 - t2; + x2 = s7 - t3; + x3 = s7 + t3; + + // Stage 4 + t0 = x0 * cospi_28_64 + x3 * cospi_4_64; + t1 = x1 * cospi_12_64 + x2 * cospi_20_64; + t2 = x2 * cospi_12_64 + x1 * -cospi_20_64; + t3 = x3 * cospi_28_64 + x0 * -cospi_4_64; + out[2] = (tran_low_t)fdct_round_shift(t0); + out[6] = (tran_low_t)fdct_round_shift(t2); + out[10] = (tran_low_t)fdct_round_shift(t1); + out[14] = (tran_low_t)fdct_round_shift(t3); + } + // Work on the next eight values; step1 -> odd_results + { + // step 2 + temp1 = (step1[5] - step1[2]) * cospi_16_64; + temp2 = (step1[4] - step1[3]) * cospi_16_64; + step2[2] = fdct_round_shift(temp1); + step2[3] = fdct_round_shift(temp2); + temp1 = (step1[4] + step1[3]) * cospi_16_64; + temp2 = (step1[5] + step1[2]) * cospi_16_64; + step2[4] = fdct_round_shift(temp1); + step2[5] = fdct_round_shift(temp2); + // step 3 + step3[0] = step1[0] + step2[3]; + step3[1] = step1[1] + step2[2]; + step3[2] = step1[1] - step2[2]; + step3[3] = step1[0] - step2[3]; + step3[4] = step1[7] - step2[4]; + step3[5] = step1[6] - step2[5]; + step3[6] = step1[6] + step2[5]; + step3[7] = step1[7] + step2[4]; + // step 4 + temp1 = step3[1] * -cospi_8_64 + step3[6] * cospi_24_64; + temp2 = step3[2] * cospi_24_64 + step3[5] * cospi_8_64; + step2[1] = fdct_round_shift(temp1); + step2[2] = fdct_round_shift(temp2); + temp1 = step3[2] * cospi_8_64 - step3[5] * cospi_24_64; + temp2 = step3[1] * cospi_24_64 + step3[6] * cospi_8_64; + step2[5] = fdct_round_shift(temp1); + step2[6] = fdct_round_shift(temp2); + // step 5 + step1[0] = step3[0] + step2[1]; + step1[1] = step3[0] - step2[1]; + step1[2] = step3[3] + step2[2]; + step1[3] = step3[3] - step2[2]; + step1[4] = step3[4] - step2[5]; + step1[5] = step3[4] + step2[5]; + step1[6] = step3[7] - step2[6]; + step1[7] = step3[7] + step2[6]; + // step 6 + temp1 = step1[0] * cospi_30_64 + step1[7] * cospi_2_64; + temp2 = step1[1] * cospi_14_64 + step1[6] * cospi_18_64; + out[1] = (tran_low_t)fdct_round_shift(temp1); + out[9] = (tran_low_t)fdct_round_shift(temp2); + temp1 = step1[2] * cospi_22_64 + step1[5] * cospi_10_64; + temp2 = step1[3] * cospi_6_64 + step1[4] * cospi_26_64; + out[5] = (tran_low_t)fdct_round_shift(temp1); + out[13] = (tran_low_t)fdct_round_shift(temp2); + temp1 = step1[3] * -cospi_26_64 + step1[4] * cospi_6_64; + temp2 = step1[2] * -cospi_10_64 + step1[5] * cospi_22_64; + out[3] = (tran_low_t)fdct_round_shift(temp1); + out[11] = (tran_low_t)fdct_round_shift(temp2); + temp1 = step1[1] * -cospi_18_64 + step1[6] * cospi_14_64; + temp2 = step1[0] * -cospi_2_64 + step1[7] * cospi_30_64; + out[7] = (tran_low_t)fdct_round_shift(temp1); + out[15] = (tran_low_t)fdct_round_shift(temp2); + } + // Do next column (which is a transposed row in second/horizontal pass) + input++; + out += 16; + } + // Setup in/out for next pass. + in_low = intermediate; + out = output; + } +} + +void aom_fdct16x16_1_c(const int16_t *input, tran_low_t *output, int stride) { + int r, c; + int sum = 0; + for (r = 0; r < 16; ++r) + for (c = 0; c < 16; ++c) sum += input[r * stride + c]; + + output[0] = (tran_low_t)(sum >> 1); +} + +static INLINE tran_high_t dct_32_round(tran_high_t input) { + tran_high_t rv = ROUND_POWER_OF_TWO(input, DCT_CONST_BITS); + // TODO(debargha, peter.derivaz): Find new bounds for this assert, + // and make the bounds consts. + // assert(-131072 <= rv && rv <= 131071); + return rv; +} + +static INLINE tran_high_t half_round_shift(tran_high_t input) { + tran_high_t rv = (input + 1 + (input < 0)) >> 2; + return rv; +} + +void aom_fdct32(const tran_high_t *input, tran_high_t *output, int round) { + tran_high_t step[32]; + // Stage 1 + step[0] = input[0] + input[(32 - 1)]; + step[1] = input[1] + input[(32 - 2)]; + step[2] = input[2] + input[(32 - 3)]; + step[3] = input[3] + input[(32 - 4)]; + step[4] = input[4] + input[(32 - 5)]; + step[5] = input[5] + input[(32 - 6)]; + step[6] = input[6] + input[(32 - 7)]; + step[7] = input[7] + input[(32 - 8)]; + step[8] = input[8] + input[(32 - 9)]; + step[9] = input[9] + input[(32 - 10)]; + step[10] = input[10] + input[(32 - 11)]; + step[11] = input[11] + input[(32 - 12)]; + step[12] = input[12] + input[(32 - 13)]; + step[13] = input[13] + input[(32 - 14)]; + step[14] = input[14] + input[(32 - 15)]; + step[15] = input[15] + input[(32 - 16)]; + step[16] = -input[16] + input[(32 - 17)]; + step[17] = -input[17] + input[(32 - 18)]; + step[18] = -input[18] + input[(32 - 19)]; + step[19] = -input[19] + input[(32 - 20)]; + step[20] = -input[20] + input[(32 - 21)]; + step[21] = -input[21] + input[(32 - 22)]; + step[22] = -input[22] + input[(32 - 23)]; + step[23] = -input[23] + input[(32 - 24)]; + step[24] = -input[24] + input[(32 - 25)]; + step[25] = -input[25] + input[(32 - 26)]; + step[26] = -input[26] + input[(32 - 27)]; + step[27] = -input[27] + input[(32 - 28)]; + step[28] = -input[28] + input[(32 - 29)]; + step[29] = -input[29] + input[(32 - 30)]; + step[30] = -input[30] + input[(32 - 31)]; + step[31] = -input[31] + input[(32 - 32)]; + + // Stage 2 + output[0] = step[0] + step[16 - 1]; + output[1] = step[1] + step[16 - 2]; + output[2] = step[2] + step[16 - 3]; + output[3] = step[3] + step[16 - 4]; + output[4] = step[4] + step[16 - 5]; + output[5] = step[5] + step[16 - 6]; + output[6] = step[6] + step[16 - 7]; + output[7] = step[7] + step[16 - 8]; + output[8] = -step[8] + step[16 - 9]; + output[9] = -step[9] + step[16 - 10]; + output[10] = -step[10] + step[16 - 11]; + output[11] = -step[11] + step[16 - 12]; + output[12] = -step[12] + step[16 - 13]; + output[13] = -step[13] + step[16 - 14]; + output[14] = -step[14] + step[16 - 15]; + output[15] = -step[15] + step[16 - 16]; + + output[16] = step[16]; + output[17] = step[17]; + output[18] = step[18]; + output[19] = step[19]; + + output[20] = dct_32_round((-step[20] + step[27]) * cospi_16_64); + output[21] = dct_32_round((-step[21] + step[26]) * cospi_16_64); + output[22] = dct_32_round((-step[22] + step[25]) * cospi_16_64); + output[23] = dct_32_round((-step[23] + step[24]) * cospi_16_64); + + output[24] = dct_32_round((step[24] + step[23]) * cospi_16_64); + output[25] = dct_32_round((step[25] + step[22]) * cospi_16_64); + output[26] = dct_32_round((step[26] + step[21]) * cospi_16_64); + output[27] = dct_32_round((step[27] + step[20]) * cospi_16_64); + + output[28] = step[28]; + output[29] = step[29]; + output[30] = step[30]; + output[31] = step[31]; + + // dump the magnitude by 4, hence the intermediate values are within + // the range of 16 bits. + if (round) { + output[0] = half_round_shift(output[0]); + output[1] = half_round_shift(output[1]); + output[2] = half_round_shift(output[2]); + output[3] = half_round_shift(output[3]); + output[4] = half_round_shift(output[4]); + output[5] = half_round_shift(output[5]); + output[6] = half_round_shift(output[6]); + output[7] = half_round_shift(output[7]); + output[8] = half_round_shift(output[8]); + output[9] = half_round_shift(output[9]); + output[10] = half_round_shift(output[10]); + output[11] = half_round_shift(output[11]); + output[12] = half_round_shift(output[12]); + output[13] = half_round_shift(output[13]); + output[14] = half_round_shift(output[14]); + output[15] = half_round_shift(output[15]); + + output[16] = half_round_shift(output[16]); + output[17] = half_round_shift(output[17]); + output[18] = half_round_shift(output[18]); + output[19] = half_round_shift(output[19]); + output[20] = half_round_shift(output[20]); + output[21] = half_round_shift(output[21]); + output[22] = half_round_shift(output[22]); + output[23] = half_round_shift(output[23]); + output[24] = half_round_shift(output[24]); + output[25] = half_round_shift(output[25]); + output[26] = half_round_shift(output[26]); + output[27] = half_round_shift(output[27]); + output[28] = half_round_shift(output[28]); + output[29] = half_round_shift(output[29]); + output[30] = half_round_shift(output[30]); + output[31] = half_round_shift(output[31]); + } + + // Stage 3 + step[0] = output[0] + output[(8 - 1)]; + step[1] = output[1] + output[(8 - 2)]; + step[2] = output[2] + output[(8 - 3)]; + step[3] = output[3] + output[(8 - 4)]; + step[4] = -output[4] + output[(8 - 5)]; + step[5] = -output[5] + output[(8 - 6)]; + step[6] = -output[6] + output[(8 - 7)]; + step[7] = -output[7] + output[(8 - 8)]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = dct_32_round((-output[10] + output[13]) * cospi_16_64); + step[11] = dct_32_round((-output[11] + output[12]) * cospi_16_64); + step[12] = dct_32_round((output[12] + output[11]) * cospi_16_64); + step[13] = dct_32_round((output[13] + output[10]) * cospi_16_64); + step[14] = output[14]; + step[15] = output[15]; + + step[16] = output[16] + output[23]; + step[17] = output[17] + output[22]; + step[18] = output[18] + output[21]; + step[19] = output[19] + output[20]; + step[20] = -output[20] + output[19]; + step[21] = -output[21] + output[18]; + step[22] = -output[22] + output[17]; + step[23] = -output[23] + output[16]; + step[24] = -output[24] + output[31]; + step[25] = -output[25] + output[30]; + step[26] = -output[26] + output[29]; + step[27] = -output[27] + output[28]; + step[28] = output[28] + output[27]; + step[29] = output[29] + output[26]; + step[30] = output[30] + output[25]; + step[31] = output[31] + output[24]; + + // Stage 4 + output[0] = step[0] + step[3]; + output[1] = step[1] + step[2]; + output[2] = -step[2] + step[1]; + output[3] = -step[3] + step[0]; + output[4] = step[4]; + output[5] = dct_32_round((-step[5] + step[6]) * cospi_16_64); + output[6] = dct_32_round((step[6] + step[5]) * cospi_16_64); + output[7] = step[7]; + output[8] = step[8] + step[11]; + output[9] = step[9] + step[10]; + output[10] = -step[10] + step[9]; + output[11] = -step[11] + step[8]; + output[12] = -step[12] + step[15]; + output[13] = -step[13] + step[14]; + output[14] = step[14] + step[13]; + output[15] = step[15] + step[12]; + + output[16] = step[16]; + output[17] = step[17]; + output[18] = dct_32_round(step[18] * -cospi_8_64 + step[29] * cospi_24_64); + output[19] = dct_32_round(step[19] * -cospi_8_64 + step[28] * cospi_24_64); + output[20] = dct_32_round(step[20] * -cospi_24_64 + step[27] * -cospi_8_64); + output[21] = dct_32_round(step[21] * -cospi_24_64 + step[26] * -cospi_8_64); + output[22] = step[22]; + output[23] = step[23]; + output[24] = step[24]; + output[25] = step[25]; + output[26] = dct_32_round(step[26] * cospi_24_64 + step[21] * -cospi_8_64); + output[27] = dct_32_round(step[27] * cospi_24_64 + step[20] * -cospi_8_64); + output[28] = dct_32_round(step[28] * cospi_8_64 + step[19] * cospi_24_64); + output[29] = dct_32_round(step[29] * cospi_8_64 + step[18] * cospi_24_64); + output[30] = step[30]; + output[31] = step[31]; + + // Stage 5 + step[0] = dct_32_round((output[0] + output[1]) * cospi_16_64); + step[1] = dct_32_round((-output[1] + output[0]) * cospi_16_64); + step[2] = dct_32_round(output[2] * cospi_24_64 + output[3] * cospi_8_64); + step[3] = dct_32_round(output[3] * cospi_24_64 - output[2] * cospi_8_64); + step[4] = output[4] + output[5]; + step[5] = -output[5] + output[4]; + step[6] = -output[6] + output[7]; + step[7] = output[7] + output[6]; + step[8] = output[8]; + step[9] = dct_32_round(output[9] * -cospi_8_64 + output[14] * cospi_24_64); + step[10] = dct_32_round(output[10] * -cospi_24_64 + output[13] * -cospi_8_64); + step[11] = output[11]; + step[12] = output[12]; + step[13] = dct_32_round(output[13] * cospi_24_64 + output[10] * -cospi_8_64); + step[14] = dct_32_round(output[14] * cospi_8_64 + output[9] * cospi_24_64); + step[15] = output[15]; + + step[16] = output[16] + output[19]; + step[17] = output[17] + output[18]; + step[18] = -output[18] + output[17]; + step[19] = -output[19] + output[16]; + step[20] = -output[20] + output[23]; + step[21] = -output[21] + output[22]; + step[22] = output[22] + output[21]; + step[23] = output[23] + output[20]; + step[24] = output[24] + output[27]; + step[25] = output[25] + output[26]; + step[26] = -output[26] + output[25]; + step[27] = -output[27] + output[24]; + step[28] = -output[28] + output[31]; + step[29] = -output[29] + output[30]; + step[30] = output[30] + output[29]; + step[31] = output[31] + output[28]; + + // Stage 6 + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = dct_32_round(step[4] * cospi_28_64 + step[7] * cospi_4_64); + output[5] = dct_32_round(step[5] * cospi_12_64 + step[6] * cospi_20_64); + output[6] = dct_32_round(step[6] * cospi_12_64 + step[5] * -cospi_20_64); + output[7] = dct_32_round(step[7] * cospi_28_64 + step[4] * -cospi_4_64); + output[8] = step[8] + step[9]; + output[9] = -step[9] + step[8]; + output[10] = -step[10] + step[11]; + output[11] = step[11] + step[10]; + output[12] = step[12] + step[13]; + output[13] = -step[13] + step[12]; + output[14] = -step[14] + step[15]; + output[15] = step[15] + step[14]; + + output[16] = step[16]; + output[17] = dct_32_round(step[17] * -cospi_4_64 + step[30] * cospi_28_64); + output[18] = dct_32_round(step[18] * -cospi_28_64 + step[29] * -cospi_4_64); + output[19] = step[19]; + output[20] = step[20]; + output[21] = dct_32_round(step[21] * -cospi_20_64 + step[26] * cospi_12_64); + output[22] = dct_32_round(step[22] * -cospi_12_64 + step[25] * -cospi_20_64); + output[23] = step[23]; + output[24] = step[24]; + output[25] = dct_32_round(step[25] * cospi_12_64 + step[22] * -cospi_20_64); + output[26] = dct_32_round(step[26] * cospi_20_64 + step[21] * cospi_12_64); + output[27] = step[27]; + output[28] = step[28]; + output[29] = dct_32_round(step[29] * cospi_28_64 + step[18] * -cospi_4_64); + output[30] = dct_32_round(step[30] * cospi_4_64 + step[17] * cospi_28_64); + output[31] = step[31]; + + // Stage 7 + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = dct_32_round(output[8] * cospi_30_64 + output[15] * cospi_2_64); + step[9] = dct_32_round(output[9] * cospi_14_64 + output[14] * cospi_18_64); + step[10] = dct_32_round(output[10] * cospi_22_64 + output[13] * cospi_10_64); + step[11] = dct_32_round(output[11] * cospi_6_64 + output[12] * cospi_26_64); + step[12] = dct_32_round(output[12] * cospi_6_64 + output[11] * -cospi_26_64); + step[13] = dct_32_round(output[13] * cospi_22_64 + output[10] * -cospi_10_64); + step[14] = dct_32_round(output[14] * cospi_14_64 + output[9] * -cospi_18_64); + step[15] = dct_32_round(output[15] * cospi_30_64 + output[8] * -cospi_2_64); + + step[16] = output[16] + output[17]; + step[17] = -output[17] + output[16]; + step[18] = -output[18] + output[19]; + step[19] = output[19] + output[18]; + step[20] = output[20] + output[21]; + step[21] = -output[21] + output[20]; + step[22] = -output[22] + output[23]; + step[23] = output[23] + output[22]; + step[24] = output[24] + output[25]; + step[25] = -output[25] + output[24]; + step[26] = -output[26] + output[27]; + step[27] = output[27] + output[26]; + step[28] = output[28] + output[29]; + step[29] = -output[29] + output[28]; + step[30] = -output[30] + output[31]; + step[31] = output[31] + output[30]; + + // Final stage --- outputs indices are bit-reversed. + output[0] = step[0]; + output[16] = step[1]; + output[8] = step[2]; + output[24] = step[3]; + output[4] = step[4]; + output[20] = step[5]; + output[12] = step[6]; + output[28] = step[7]; + output[2] = step[8]; + output[18] = step[9]; + output[10] = step[10]; + output[26] = step[11]; + output[6] = step[12]; + output[22] = step[13]; + output[14] = step[14]; + output[30] = step[15]; + + output[1] = dct_32_round(step[16] * cospi_31_64 + step[31] * cospi_1_64); + output[17] = dct_32_round(step[17] * cospi_15_64 + step[30] * cospi_17_64); + output[9] = dct_32_round(step[18] * cospi_23_64 + step[29] * cospi_9_64); + output[25] = dct_32_round(step[19] * cospi_7_64 + step[28] * cospi_25_64); + output[5] = dct_32_round(step[20] * cospi_27_64 + step[27] * cospi_5_64); + output[21] = dct_32_round(step[21] * cospi_11_64 + step[26] * cospi_21_64); + output[13] = dct_32_round(step[22] * cospi_19_64 + step[25] * cospi_13_64); + output[29] = dct_32_round(step[23] * cospi_3_64 + step[24] * cospi_29_64); + output[3] = dct_32_round(step[24] * cospi_3_64 + step[23] * -cospi_29_64); + output[19] = dct_32_round(step[25] * cospi_19_64 + step[22] * -cospi_13_64); + output[11] = dct_32_round(step[26] * cospi_11_64 + step[21] * -cospi_21_64); + output[27] = dct_32_round(step[27] * cospi_27_64 + step[20] * -cospi_5_64); + output[7] = dct_32_round(step[28] * cospi_7_64 + step[19] * -cospi_25_64); + output[23] = dct_32_round(step[29] * cospi_23_64 + step[18] * -cospi_9_64); + output[15] = dct_32_round(step[30] * cospi_15_64 + step[17] * -cospi_17_64); + output[31] = dct_32_round(step[31] * cospi_31_64 + step[16] * -cospi_1_64); +} + +void aom_fdct32x32_c(const int16_t *input, tran_low_t *out, int stride) { + int i, j; + tran_high_t output[32 * 32]; + + // Columns + for (i = 0; i < 32; ++i) { + tran_high_t temp_in[32], temp_out[32]; + for (j = 0; j < 32; ++j) temp_in[j] = input[j * stride + i] * 4; + aom_fdct32(temp_in, temp_out, 0); + for (j = 0; j < 32; ++j) + output[j * 32 + i] = (temp_out[j] + 1 + (temp_out[j] > 0)) >> 2; + } + + // Rows + for (i = 0; i < 32; ++i) { + tran_high_t temp_in[32], temp_out[32]; + for (j = 0; j < 32; ++j) temp_in[j] = output[j + i * 32]; + aom_fdct32(temp_in, temp_out, 0); + for (j = 0; j < 32; ++j) + out[j + i * 32] = + (tran_low_t)((temp_out[j] + 1 + (temp_out[j] < 0)) >> 2); + } +} + +// Note that although we use dct_32_round in dct32 computation flow, +// this 2d fdct32x32 for rate-distortion optimization loop is operating +// within 16 bits precision. +void aom_fdct32x32_rd_c(const int16_t *input, tran_low_t *out, int stride) { + int i, j; + tran_high_t output[32 * 32]; + + // Columns + for (i = 0; i < 32; ++i) { + tran_high_t temp_in[32], temp_out[32]; + for (j = 0; j < 32; ++j) temp_in[j] = input[j * stride + i] * 4; + aom_fdct32(temp_in, temp_out, 0); + for (j = 0; j < 32; ++j) + // TODO(cd): see quality impact of only doing + // output[j * 32 + i] = (temp_out[j] + 1) >> 2; + // PS: also change code in aom_dsp/x86/aom_dct_sse2.c + output[j * 32 + i] = (temp_out[j] + 1 + (temp_out[j] > 0)) >> 2; + } + + // Rows + for (i = 0; i < 32; ++i) { + tran_high_t temp_in[32], temp_out[32]; + for (j = 0; j < 32; ++j) temp_in[j] = output[j + i * 32]; + aom_fdct32(temp_in, temp_out, 1); + for (j = 0; j < 32; ++j) out[j + i * 32] = (tran_low_t)temp_out[j]; + } +} + +void aom_fdct32x32_1_c(const int16_t *input, tran_low_t *output, int stride) { + int r, c; + int sum = 0; + for (r = 0; r < 32; ++r) + for (c = 0; c < 32; ++c) sum += input[r * stride + c]; + + output[0] = (tran_low_t)(sum >> 3); +} + +#if CONFIG_HIGHBITDEPTH +void aom_highbd_fdct4x4_c(const int16_t *input, tran_low_t *output, + int stride) { + aom_fdct4x4_c(input, output, stride); +} + +void aom_highbd_fdct8x8_c(const int16_t *input, tran_low_t *final_output, + int stride) { + aom_fdct8x8_c(input, final_output, stride); +} + +void aom_highbd_fdct8x8_1_c(const int16_t *input, tran_low_t *final_output, + int stride) { + aom_fdct8x8_1_c(input, final_output, stride); +} + +void aom_highbd_fdct16x16_c(const int16_t *input, tran_low_t *output, + int stride) { + aom_fdct16x16_c(input, output, stride); +} + +void aom_highbd_fdct16x16_1_c(const int16_t *input, tran_low_t *output, + int stride) { + aom_fdct16x16_1_c(input, output, stride); +} + +void aom_highbd_fdct32x32_c(const int16_t *input, tran_low_t *out, int stride) { + aom_fdct32x32_c(input, out, stride); +} + +void aom_highbd_fdct32x32_rd_c(const int16_t *input, tran_low_t *out, + int stride) { + aom_fdct32x32_rd_c(input, out, stride); +} + +void aom_highbd_fdct32x32_1_c(const int16_t *input, tran_low_t *out, + int stride) { + aom_fdct32x32_1_c(input, out, stride); +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/fwd_txfm.h b/third_party/aom/aom_dsp/fwd_txfm.h new file mode 100644 index 0000000000..579dbd06e0 --- /dev/null +++ b/third_party/aom/aom_dsp/fwd_txfm.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_FWD_TXFM_H_ +#define AOM_DSP_FWD_TXFM_H_ + +#include "aom_dsp/txfm_common.h" + +static INLINE tran_high_t saturate_int16(tran_high_t value) { + tran_high_t result; + result = value > INT16_MAX ? INT16_MAX : value; + return result < INT16_MIN ? INT16_MIN : result; +} + +static INLINE tran_high_t fdct_round_shift(tran_high_t input) { + tran_high_t rv = ROUND_POWER_OF_TWO(input, DCT_CONST_BITS); + return rv; +} + +void aom_fdct32(const tran_high_t *input, tran_high_t *output, int round); +#endif // AOM_DSP_FWD_TXFM_H_ diff --git a/third_party/aom/aom_dsp/intrapred.c b/third_party/aom/aom_dsp/intrapred.c new file mode 100644 index 0000000000..1f0870b647 --- /dev/null +++ b/third_party/aom/aom_dsp/intrapred.c @@ -0,0 +1,971 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/bitops.h" + +#define DST(x, y) dst[(x) + (y)*stride] +#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2) +#define AVG2(a, b) (((a) + (b) + 1) >> 1) + +static INLINE void d207e_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int r, c; + (void)above; + + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) { + dst[c] = c & 1 ? AVG3(left[(c >> 1) + r], left[(c >> 1) + r + 1], + left[(c >> 1) + r + 2]) + : AVG2(left[(c >> 1) + r], left[(c >> 1) + r + 1]); + } + dst += stride; + } +} + +static INLINE void d63e_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int r, c; + (void)left; + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) { + dst[c] = r & 1 ? AVG3(above[(r >> 1) + c], above[(r >> 1) + c + 1], + above[(r >> 1) + c + 2]) + : AVG2(above[(r >> 1) + c], above[(r >> 1) + c + 1]); + } + dst += stride; + } +} + +static INLINE void d45e_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int r, c; + (void)left; + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) { + dst[c] = AVG3(above[r + c], above[r + c + 1], + above[r + c + 1 + (r + c + 2 < bs * 2)]); + } + dst += stride; + } +} + +static INLINE void d117_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int r, c; + + // first row + for (c = 0; c < bs; c++) dst[c] = AVG2(above[c - 1], above[c]); + dst += stride; + + // second row + dst[0] = AVG3(left[0], above[-1], above[0]); + for (c = 1; c < bs; c++) dst[c] = AVG3(above[c - 2], above[c - 1], above[c]); + dst += stride; + + // the rest of first col + dst[0] = AVG3(above[-1], left[0], left[1]); + for (r = 3; r < bs; ++r) + dst[(r - 2) * stride] = AVG3(left[r - 3], left[r - 2], left[r - 1]); + + // the rest of the block + for (r = 2; r < bs; ++r) { + for (c = 1; c < bs; c++) dst[c] = dst[-2 * stride + c - 1]; + dst += stride; + } +} + +static INLINE void d135_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int i; +#if CONFIG_TX64X64 +#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ > 7 + // silence a spurious -Warray-bounds warning, possibly related to: + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56273 + uint8_t border[133]; +#else + uint8_t border[64 + 64 - 1]; // outer border from bottom-left to top-right +#endif +#else +#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ > 7 + // silence a spurious -Warray-bounds warning, possibly related to: + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56273 + uint8_t border[69]; +#else + uint8_t border[32 + 32 - 1]; // outer border from bottom-left to top-right +#endif +#endif // CONFIG_TX64X64 + + // dst(bs, bs - 2)[0], i.e., border starting at bottom-left + for (i = 0; i < bs - 2; ++i) { + border[i] = AVG3(left[bs - 3 - i], left[bs - 2 - i], left[bs - 1 - i]); + } + border[bs - 2] = AVG3(above[-1], left[0], left[1]); + border[bs - 1] = AVG3(left[0], above[-1], above[0]); + border[bs - 0] = AVG3(above[-1], above[0], above[1]); + // dst[0][2, size), i.e., remaining top border ascending + for (i = 0; i < bs - 2; ++i) { + border[bs + 1 + i] = AVG3(above[i], above[i + 1], above[i + 2]); + } + + for (i = 0; i < bs; ++i) { + memcpy(dst + i * stride, border + bs - 1 - i, bs); + } +} + +static INLINE void d153_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int r, c; + dst[0] = AVG2(above[-1], left[0]); + for (r = 1; r < bs; r++) dst[r * stride] = AVG2(left[r - 1], left[r]); + dst++; + + dst[0] = AVG3(left[0], above[-1], above[0]); + dst[stride] = AVG3(above[-1], left[0], left[1]); + for (r = 2; r < bs; r++) + dst[r * stride] = AVG3(left[r - 2], left[r - 1], left[r]); + dst++; + + for (c = 0; c < bs - 2; c++) + dst[c] = AVG3(above[c - 1], above[c], above[c + 1]); + dst += stride; + + for (r = 1; r < bs; ++r) { + for (c = 0; c < bs - 2; c++) dst[c] = dst[-stride + c - 2]; + dst += stride; + } +} + +static INLINE void v_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int r; + (void)left; + + for (r = 0; r < bs; r++) { + memcpy(dst, above, bs); + dst += stride; + } +} + +static INLINE void h_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int r; + (void)above; + + for (r = 0; r < bs; r++) { + memset(dst, left[r], bs); + dst += stride; + } +} + +#if CONFIG_ALT_INTRA +static INLINE int abs_diff(int a, int b) { return (a > b) ? a - b : b - a; } + +static INLINE uint16_t paeth_predictor_single(uint16_t left, uint16_t top, + uint16_t top_left) { + const int base = top + left - top_left; + const int p_left = abs_diff(base, left); + const int p_top = abs_diff(base, top); + const int p_top_left = abs_diff(base, top_left); + + // Return nearest to base of left, top and top_left. + return (p_left <= p_top && p_left <= p_top_left) + ? left + : (p_top <= p_top_left) ? top : top_left; +} + +static INLINE void paeth_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int r, c; + const uint8_t ytop_left = above[-1]; + + for (r = 0; r < bs; r++) { + for (c = 0; c < bs; c++) + dst[c] = (uint8_t)paeth_predictor_single(left[r], above[c], ytop_left); + dst += stride; + } +} + +// Weights are quadratic from '1' to '1 / block_size', scaled by +// 2^sm_weight_log2_scale. +static const int sm_weight_log2_scale = 8; + +#if CONFIG_TX64X64 +// max(block_size_wide[BLOCK_LARGEST], block_size_high[BLOCK_LARGEST]) +#define MAX_BLOCK_DIM 64 +#define NUM_BLOCK_DIMS 6 // log2(MAX_BLOCK_DIM) +#else +#define MAX_BLOCK_DIM 32 +#define NUM_BLOCK_DIMS 5 +#endif // CONFIG_TX64X64 + +static const uint8_t sm_weight_arrays[NUM_BLOCK_DIMS][MAX_BLOCK_DIM] = { + // bs = 2 + { 255, 128 }, + // bs = 4 + { 255, 149, 85, 64 }, + // bs = 8 + { 255, 197, 146, 105, 73, 50, 37, 32 }, + // bs = 16 + { 255, 225, 196, 170, 145, 123, 102, 84, 68, 54, 43, 33, 26, 20, 17, 16 }, + // bs = 32 + { + 255, 240, 225, 210, 196, 182, 169, 157, 145, 133, 122, + 111, 101, 92, 83, 74, 66, 59, 52, 45, 39, 34, + 29, 25, 21, 17, 14, 12, 10, 9, 8, 8 }, +#if CONFIG_TX64X64 + // bs = 64 + { 255, 248, 240, 233, 225, 218, 210, 203, 196, 189, 182, 176, 169, + 163, 156, 150, 144, 138, 133, 127, 121, 116, 111, 106, 101, 96, + 91, 86, 82, 77, 73, 69, 65, 61, 57, 54, 50, 47, 44, + 41, 38, 35, 32, 29, 27, 25, 22, 20, 18, 16, 15, 13, + 12, 10, 9, 8, 7, 6, 6, 5, 5, 4, 4, 4 }, +#endif // CONFIG_TX64X64 +}; + +// Some basic checks on weights for smooth predictor. +#define sm_weights_sanity_checks(weights, weights_scale, pred_scale) \ + assert(weights[0] < weights_scale); \ + assert(weights_scale - weights[bs - 1] < weights_scale); \ + assert(pred_scale < 31) // ensures no overflow when calculating predictor. + +#define divide_round(value, bits) (((value) + (1 << ((bits)-1))) >> (bits)) + +static INLINE void smooth_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + const uint8_t below_pred = left[bs - 1]; // estimated by bottom-left pixel + const uint8_t right_pred = above[bs - 1]; // estimated by top-right pixel + const int arr_index = get_msb(bs) - 1; + assert(arr_index >= 0); + assert(arr_index < NUM_BLOCK_DIMS); + const uint8_t *const sm_weights = sm_weight_arrays[arr_index]; + // scale = 2 * 2^sm_weight_log2_scale + const int log2_scale = 1 + sm_weight_log2_scale; + const uint16_t scale = (1 << sm_weight_log2_scale); + sm_weights_sanity_checks(sm_weights, scale, log2_scale + sizeof(*dst)); + int r; + for (r = 0; r < bs; ++r) { + int c; + for (c = 0; c < bs; ++c) { + const uint8_t pixels[] = { above[c], below_pred, left[r], right_pred }; + const uint8_t weights[] = { sm_weights[r], scale - sm_weights[r], + sm_weights[c], scale - sm_weights[c] }; + uint32_t this_pred = 0; + int i; + assert(scale >= sm_weights[r] && scale >= sm_weights[c]); + for (i = 0; i < 4; ++i) { + this_pred += weights[i] * pixels[i]; + } + dst[c] = clip_pixel(divide_round(this_pred, log2_scale)); + } + dst += stride; + } +} + +#else + +static INLINE void tm_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int r, c; + int ytop_left = above[-1]; + + for (r = 0; r < bs; r++) { + for (c = 0; c < bs; c++) + dst[c] = clip_pixel(left[r] + above[c] - ytop_left); + dst += stride; + } +} +#endif // CONFIG_ALT_INTRA + +static INLINE void dc_128_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int r; + (void)above; + (void)left; + + for (r = 0; r < bs; r++) { + memset(dst, 128, bs); + dst += stride; + } +} + +static INLINE void dc_left_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, + const uint8_t *left) { + int i, r, expected_dc, sum = 0; + (void)above; + + for (i = 0; i < bs; i++) sum += left[i]; + expected_dc = (sum + (bs >> 1)) / bs; + + for (r = 0; r < bs; r++) { + memset(dst, expected_dc, bs); + dst += stride; + } +} + +static INLINE void dc_top_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int i, r, expected_dc, sum = 0; + (void)left; + + for (i = 0; i < bs; i++) sum += above[i]; + expected_dc = (sum + (bs >> 1)) / bs; + + for (r = 0; r < bs; r++) { + memset(dst, expected_dc, bs); + dst += stride; + } +} + +static INLINE void dc_predictor(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + int i, r, expected_dc, sum = 0; + const int count = 2 * bs; + + for (i = 0; i < bs; i++) { + sum += above[i]; + sum += left[i]; + } + + expected_dc = (sum + (count >> 1)) / count; + + for (r = 0; r < bs; r++) { + memset(dst, expected_dc, bs); + dst += stride; + } +} + +void aom_d45e_predictor_2x2_c(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + const int A = above[0]; + const int B = above[1]; + const int C = above[2]; + const int D = above[3]; + (void)stride; + (void)left; + + DST(0, 0) = AVG3(A, B, C); + DST(1, 0) = DST(0, 1) = AVG3(B, C, D); + DST(1, 1) = AVG3(C, D, D); +} + +void aom_d117_predictor_2x2_c(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + const int I = left[0]; + const int X = above[-1]; + const int A = above[0]; + const int B = above[1]; + DST(0, 0) = AVG2(X, A); + DST(1, 0) = AVG2(A, B); + DST(0, 1) = AVG3(I, X, A); + DST(1, 1) = AVG3(X, A, B); +} + +void aom_d135_predictor_2x2_c(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + const int I = left[0]; + const int J = left[1]; + const int X = above[-1]; + const int A = above[0]; + const int B = above[1]; + (void)stride; + DST(0, 1) = AVG3(X, I, J); + DST(1, 1) = DST(0, 0) = AVG3(A, X, I); + DST(1, 0) = AVG3(B, A, X); +} + +void aom_d153_predictor_2x2_c(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + const int I = left[0]; + const int J = left[1]; + const int X = above[-1]; + const int A = above[0]; + + DST(0, 0) = AVG2(I, X); + DST(0, 1) = AVG2(J, I); + DST(1, 0) = AVG3(I, X, A); + DST(1, 1) = AVG3(J, I, X); +} + +void aom_d45e_predictor_4x4_c(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + const int A = above[0]; + const int B = above[1]; + const int C = above[2]; + const int D = above[3]; + const int E = above[4]; + const int F = above[5]; + const int G = above[6]; + const int H = above[7]; + (void)stride; + (void)left; + DST(0, 0) = AVG3(A, B, C); + DST(1, 0) = DST(0, 1) = AVG3(B, C, D); + DST(2, 0) = DST(1, 1) = DST(0, 2) = AVG3(C, D, E); + DST(3, 0) = DST(2, 1) = DST(1, 2) = DST(0, 3) = AVG3(D, E, F); + DST(3, 1) = DST(2, 2) = DST(1, 3) = AVG3(E, F, G); + DST(3, 2) = DST(2, 3) = AVG3(F, G, H); + DST(3, 3) = AVG3(G, H, H); +} + +void aom_d117_predictor_4x4_c(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + const int I = left[0]; + const int J = left[1]; + const int K = left[2]; + const int X = above[-1]; + const int A = above[0]; + const int B = above[1]; + const int C = above[2]; + const int D = above[3]; + DST(0, 0) = DST(1, 2) = AVG2(X, A); + DST(1, 0) = DST(2, 2) = AVG2(A, B); + DST(2, 0) = DST(3, 2) = AVG2(B, C); + DST(3, 0) = AVG2(C, D); + + DST(0, 3) = AVG3(K, J, I); + DST(0, 2) = AVG3(J, I, X); + DST(0, 1) = DST(1, 3) = AVG3(I, X, A); + DST(1, 1) = DST(2, 3) = AVG3(X, A, B); + DST(2, 1) = DST(3, 3) = AVG3(A, B, C); + DST(3, 1) = AVG3(B, C, D); +} + +void aom_d135_predictor_4x4_c(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + const int I = left[0]; + const int J = left[1]; + const int K = left[2]; + const int L = left[3]; + const int X = above[-1]; + const int A = above[0]; + const int B = above[1]; + const int C = above[2]; + const int D = above[3]; + (void)stride; + DST(0, 3) = AVG3(J, K, L); + DST(1, 3) = DST(0, 2) = AVG3(I, J, K); + DST(2, 3) = DST(1, 2) = DST(0, 1) = AVG3(X, I, J); + DST(3, 3) = DST(2, 2) = DST(1, 1) = DST(0, 0) = AVG3(A, X, I); + DST(3, 2) = DST(2, 1) = DST(1, 0) = AVG3(B, A, X); + DST(3, 1) = DST(2, 0) = AVG3(C, B, A); + DST(3, 0) = AVG3(D, C, B); +} + +void aom_d153_predictor_4x4_c(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + const int I = left[0]; + const int J = left[1]; + const int K = left[2]; + const int L = left[3]; + const int X = above[-1]; + const int A = above[0]; + const int B = above[1]; + const int C = above[2]; + + DST(0, 0) = DST(2, 1) = AVG2(I, X); + DST(0, 1) = DST(2, 2) = AVG2(J, I); + DST(0, 2) = DST(2, 3) = AVG2(K, J); + DST(0, 3) = AVG2(L, K); + + DST(3, 0) = AVG3(A, B, C); + DST(2, 0) = AVG3(X, A, B); + DST(1, 0) = DST(3, 1) = AVG3(I, X, A); + DST(1, 1) = DST(3, 2) = AVG3(J, I, X); + DST(1, 2) = DST(3, 3) = AVG3(K, J, I); + DST(1, 3) = AVG3(L, K, J); +} + +#if CONFIG_HIGHBITDEPTH +static INLINE void highbd_d207e_predictor(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + int r, c; + (void)above; + (void)bd; + + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) { + dst[c] = c & 1 ? AVG3(left[(c >> 1) + r], left[(c >> 1) + r + 1], + left[(c >> 1) + r + 2]) + : AVG2(left[(c >> 1) + r], left[(c >> 1) + r + 1]); + } + dst += stride; + } +} + +static INLINE void highbd_d63e_predictor(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + int r, c; + (void)left; + (void)bd; + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) { + dst[c] = r & 1 ? AVG3(above[(r >> 1) + c], above[(r >> 1) + c + 1], + above[(r >> 1) + c + 2]) + : AVG2(above[(r >> 1) + c], above[(r >> 1) + c + 1]); + } + dst += stride; + } +} + +static INLINE void highbd_d45e_predictor(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + int r, c; + (void)left; + (void)bd; + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) { + dst[c] = AVG3(above[r + c], above[r + c + 1], + above[r + c + 1 + (r + c + 2 < bs * 2)]); + } + dst += stride; + } +} + +static INLINE void highbd_d117_predictor(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + int r, c; + (void)bd; + + // first row + for (c = 0; c < bs; c++) dst[c] = AVG2(above[c - 1], above[c]); + dst += stride; + + // second row + dst[0] = AVG3(left[0], above[-1], above[0]); + for (c = 1; c < bs; c++) dst[c] = AVG3(above[c - 2], above[c - 1], above[c]); + dst += stride; + + // the rest of first col + dst[0] = AVG3(above[-1], left[0], left[1]); + for (r = 3; r < bs; ++r) + dst[(r - 2) * stride] = AVG3(left[r - 3], left[r - 2], left[r - 1]); + + // the rest of the block + for (r = 2; r < bs; ++r) { + for (c = 1; c < bs; c++) dst[c] = dst[-2 * stride + c - 1]; + dst += stride; + } +} + +static INLINE void highbd_d135_predictor(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + int r, c; + (void)bd; + dst[0] = AVG3(left[0], above[-1], above[0]); + for (c = 1; c < bs; c++) dst[c] = AVG3(above[c - 2], above[c - 1], above[c]); + + dst[stride] = AVG3(above[-1], left[0], left[1]); + for (r = 2; r < bs; ++r) + dst[r * stride] = AVG3(left[r - 2], left[r - 1], left[r]); + + dst += stride; + for (r = 1; r < bs; ++r) { + for (c = 1; c < bs; c++) dst[c] = dst[-stride + c - 1]; + dst += stride; + } +} + +static INLINE void highbd_d153_predictor(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + int r, c; + (void)bd; + dst[0] = AVG2(above[-1], left[0]); + for (r = 1; r < bs; r++) dst[r * stride] = AVG2(left[r - 1], left[r]); + dst++; + + dst[0] = AVG3(left[0], above[-1], above[0]); + dst[stride] = AVG3(above[-1], left[0], left[1]); + for (r = 2; r < bs; r++) + dst[r * stride] = AVG3(left[r - 2], left[r - 1], left[r]); + dst++; + + for (c = 0; c < bs - 2; c++) + dst[c] = AVG3(above[c - 1], above[c], above[c + 1]); + dst += stride; + + for (r = 1; r < bs; ++r) { + for (c = 0; c < bs - 2; c++) dst[c] = dst[-stride + c - 2]; + dst += stride; + } +} + +static INLINE void highbd_v_predictor(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { + int r; + (void)left; + (void)bd; + for (r = 0; r < bs; r++) { + memcpy(dst, above, bs * sizeof(uint16_t)); + dst += stride; + } +} + +static INLINE void highbd_h_predictor(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { + int r; + (void)above; + (void)bd; + for (r = 0; r < bs; r++) { + aom_memset16(dst, left[r], bs); + dst += stride; + } +} + +void aom_highbd_d207_predictor_2x2_c(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const int I = left[0]; + const int J = left[1]; + const int K = left[2]; + const int L = left[3]; + (void)above; + (void)bd; + DST(0, 0) = AVG2(I, J); + DST(0, 1) = AVG2(J, K); + DST(1, 0) = AVG3(I, J, K); + DST(1, 1) = AVG3(J, K, L); +} + +void aom_highbd_d63_predictor_2x2_c(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, const uint16_t *left, + int bd) { + const int A = above[0]; + const int B = above[1]; + const int C = above[2]; + const int D = above[3]; + (void)left; + (void)bd; + DST(0, 0) = AVG2(A, B); + DST(1, 0) = AVG2(B, C); + DST(0, 1) = AVG3(A, B, C); + DST(1, 1) = AVG3(B, C, D); +} + +void aom_highbd_d45e_predictor_2x2_c(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const int A = above[0]; + const int B = above[1]; + const int C = above[2]; + const int D = above[3]; + (void)stride; + (void)left; + (void)bd; + DST(0, 0) = AVG3(A, B, C); + DST(1, 0) = DST(0, 1) = AVG3(B, C, D); + DST(1, 1) = AVG3(C, D, D); +} + +void aom_highbd_d117_predictor_2x2_c(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const int I = left[0]; + const int X = above[-1]; + const int A = above[0]; + const int B = above[1]; + (void)bd; + DST(0, 0) = AVG2(X, A); + DST(1, 0) = AVG2(A, B); + DST(0, 1) = AVG3(I, X, A); + DST(1, 1) = AVG3(X, A, B); +} + +void aom_highbd_d135_predictor_2x2_c(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const int I = left[0]; + const int J = left[1]; + const int X = above[-1]; + const int A = above[0]; + const int B = above[1]; + (void)bd; + DST(0, 1) = AVG3(X, I, J); + DST(1, 1) = DST(0, 0) = AVG3(A, X, I); + DST(1, 0) = AVG3(B, A, X); +} + +void aom_highbd_d153_predictor_2x2_c(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const int I = left[0]; + const int J = left[1]; + const int X = above[-1]; + const int A = above[0]; + (void)bd; + DST(0, 0) = AVG2(I, X); + DST(0, 1) = AVG2(J, I); + DST(1, 0) = AVG3(I, X, A); + DST(1, 1) = AVG3(J, I, X); +} + +#if CONFIG_ALT_INTRA +static INLINE void highbd_paeth_predictor(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + int r, c; + const uint16_t ytop_left = above[-1]; + (void)bd; + + for (r = 0; r < bs; r++) { + for (c = 0; c < bs; c++) + dst[c] = paeth_predictor_single(left[r], above[c], ytop_left); + dst += stride; + } +} + +static INLINE void highbd_smooth_predictor(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + const uint16_t below_pred = left[bs - 1]; // estimated by bottom-left pixel + const uint16_t right_pred = above[bs - 1]; // estimated by top-right pixel + const int arr_index = get_msb(bs) - 1; + assert(arr_index >= 0); + assert(arr_index < NUM_BLOCK_DIMS); + const uint8_t *const sm_weights = sm_weight_arrays[arr_index]; + // scale = 2 * 2^sm_weight_log2_scale + const int log2_scale = 1 + sm_weight_log2_scale; + const uint16_t scale = (1 << sm_weight_log2_scale); + sm_weights_sanity_checks(sm_weights, scale, log2_scale + sizeof(*dst)); + int r; + for (r = 0; r < bs; ++r) { + int c; + for (c = 0; c < bs; ++c) { + const uint16_t pixels[] = { above[c], below_pred, left[r], right_pred }; + const uint8_t weights[] = { sm_weights[r], scale - sm_weights[r], + sm_weights[c], scale - sm_weights[c] }; + uint32_t this_pred = 0; + int i; + assert(scale >= sm_weights[r] && scale >= sm_weights[c]); + for (i = 0; i < 4; ++i) { + this_pred += weights[i] * pixels[i]; + } + dst[c] = clip_pixel_highbd(divide_round(this_pred, log2_scale), bd); + } + dst += stride; + } +} + +#else +static INLINE void highbd_tm_predictor(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { + int r, c; + int ytop_left = above[-1]; + (void)bd; + + for (r = 0; r < bs; r++) { + for (c = 0; c < bs; c++) + dst[c] = clip_pixel_highbd(left[r] + above[c] - ytop_left, bd); + dst += stride; + } +} +#endif // CONFIG_ALT_INTRA + +static INLINE void highbd_dc_128_predictor(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + int r; + (void)above; + (void)left; + + for (r = 0; r < bs; r++) { + aom_memset16(dst, 128 << (bd - 8), bs); + dst += stride; + } +} + +static INLINE void highbd_dc_left_predictor(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + int i, r, expected_dc, sum = 0; + (void)above; + (void)bd; + + for (i = 0; i < bs; i++) sum += left[i]; + expected_dc = (sum + (bs >> 1)) / bs; + + for (r = 0; r < bs; r++) { + aom_memset16(dst, expected_dc, bs); + dst += stride; + } +} + +static INLINE void highbd_dc_top_predictor(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + int i, r, expected_dc, sum = 0; + (void)left; + (void)bd; + + for (i = 0; i < bs; i++) sum += above[i]; + expected_dc = (sum + (bs >> 1)) / bs; + + for (r = 0; r < bs; r++) { + aom_memset16(dst, expected_dc, bs); + dst += stride; + } +} + +static INLINE void highbd_dc_predictor(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { + int i, r, expected_dc, sum = 0; + const int count = 2 * bs; + (void)bd; + + for (i = 0; i < bs; i++) { + sum += above[i]; + sum += left[i]; + } + + expected_dc = (sum + (count >> 1)) / count; + + for (r = 0; r < bs; r++) { + aom_memset16(dst, expected_dc, bs); + dst += stride; + } +} +#endif // CONFIG_HIGHBITDEPTH + +// This serves as a wrapper function, so that all the prediction functions +// can be unified and accessed as a pointer array. Note that the boundary +// above and left are not necessarily used all the time. +#define intra_pred_sized(type, size) \ + void aom_##type##_predictor_##size##x##size##_c( \ + uint8_t *dst, ptrdiff_t stride, const uint8_t *above, \ + const uint8_t *left) { \ + type##_predictor(dst, stride, size, above, left); \ + } + +#if CONFIG_HIGHBITDEPTH +#define intra_pred_highbd_sized(type, size) \ + void aom_highbd_##type##_predictor_##size##x##size##_c( \ + uint16_t *dst, ptrdiff_t stride, const uint16_t *above, \ + const uint16_t *left, int bd) { \ + highbd_##type##_predictor(dst, stride, size, above, left, bd); \ + } + +/* clang-format off */ +#if CONFIG_TX64X64 +#define intra_pred_allsizes(type) \ + intra_pred_sized(type, 2) \ + intra_pred_sized(type, 4) \ + intra_pred_sized(type, 8) \ + intra_pred_sized(type, 16) \ + intra_pred_sized(type, 32) \ + intra_pred_sized(type, 64) \ + intra_pred_highbd_sized(type, 4) \ + intra_pred_highbd_sized(type, 8) \ + intra_pred_highbd_sized(type, 16) \ + intra_pred_highbd_sized(type, 32) \ + intra_pred_highbd_sized(type, 64) + +#define intra_pred_above_4x4(type) \ + intra_pred_sized(type, 8) \ + intra_pred_sized(type, 16) \ + intra_pred_sized(type, 32) \ + intra_pred_sized(type, 64) \ + intra_pred_highbd_sized(type, 4) \ + intra_pred_highbd_sized(type, 8) \ + intra_pred_highbd_sized(type, 16) \ + intra_pred_highbd_sized(type, 32) \ + intra_pred_highbd_sized(type, 64) +#else // CONFIG_TX64X64 +#define intra_pred_allsizes(type) \ + intra_pred_sized(type, 2) \ + intra_pred_sized(type, 4) \ + intra_pred_sized(type, 8) \ + intra_pred_sized(type, 16) \ + intra_pred_sized(type, 32) \ + intra_pred_highbd_sized(type, 2) \ + intra_pred_highbd_sized(type, 4) \ + intra_pred_highbd_sized(type, 8) \ + intra_pred_highbd_sized(type, 16) \ + intra_pred_highbd_sized(type, 32) + +#define intra_pred_above_4x4(type) \ + intra_pred_sized(type, 8) \ + intra_pred_sized(type, 16) \ + intra_pred_sized(type, 32) \ + intra_pred_highbd_sized(type, 4) \ + intra_pred_highbd_sized(type, 8) \ + intra_pred_highbd_sized(type, 16) \ + intra_pred_highbd_sized(type, 32) +#endif // CONFIG_TX64X64 + +#else + +#if CONFIG_TX64X64 +#define intra_pred_allsizes(type) \ + intra_pred_sized(type, 2) \ + intra_pred_sized(type, 4) \ + intra_pred_sized(type, 8) \ + intra_pred_sized(type, 16) \ + intra_pred_sized(type, 32) \ + intra_pred_sized(type, 64) + +#define intra_pred_above_4x4(type) \ + intra_pred_sized(type, 8) \ + intra_pred_sized(type, 16) \ + intra_pred_sized(type, 32) \ + intra_pred_sized(type, 64) +#else // CONFIG_TX64X64 +#define intra_pred_allsizes(type) \ + intra_pred_sized(type, 2) \ + intra_pred_sized(type, 4) \ + intra_pred_sized(type, 8) \ + intra_pred_sized(type, 16) \ + intra_pred_sized(type, 32) + +#define intra_pred_above_4x4(type) \ + intra_pred_sized(type, 8) \ + intra_pred_sized(type, 16) \ + intra_pred_sized(type, 32) +#endif // CONFIG_TX64X64 +#endif // CONFIG_HIGHBITDEPTH + +intra_pred_allsizes(d207e) +intra_pred_allsizes(d63e) +intra_pred_above_4x4(d45e) +intra_pred_above_4x4(d117) +intra_pred_above_4x4(d135) +intra_pred_above_4x4(d153) +intra_pred_allsizes(v) +intra_pred_allsizes(h) +#if CONFIG_ALT_INTRA +intra_pred_allsizes(paeth) +intra_pred_allsizes(smooth) +#else +intra_pred_allsizes(tm) +#endif // CONFIG_ALT_INTRA +intra_pred_allsizes(dc_128) +intra_pred_allsizes(dc_left) +intra_pred_allsizes(dc_top) +intra_pred_allsizes(dc) +/* clang-format on */ +#undef intra_pred_allsizes diff --git a/third_party/aom/aom_dsp/inv_txfm.c b/third_party/aom/aom_dsp/inv_txfm.c new file mode 100644 index 0000000000..bb995856ae --- /dev/null +++ b/third_party/aom/aom_dsp/inv_txfm.c @@ -0,0 +1,1445 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/inv_txfm.h" + +void aom_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride) { + /* 4-point reversible, orthonormal inverse Walsh-Hadamard in 3.5 adds, + 0.5 shifts per pixel. */ + int i; + tran_low_t output[16]; + tran_high_t a1, b1, c1, d1, e1; + const tran_low_t *ip = input; + tran_low_t *op = output; + + for (i = 0; i < 4; i++) { + a1 = ip[0] >> UNIT_QUANT_SHIFT; + c1 = ip[1] >> UNIT_QUANT_SHIFT; + d1 = ip[2] >> UNIT_QUANT_SHIFT; + b1 = ip[3] >> UNIT_QUANT_SHIFT; + a1 += c1; + d1 -= b1; + e1 = (a1 - d1) >> 1; + b1 = e1 - b1; + c1 = e1 - c1; + a1 -= b1; + d1 += c1; + op[0] = WRAPLOW(a1); + op[1] = WRAPLOW(b1); + op[2] = WRAPLOW(c1); + op[3] = WRAPLOW(d1); + ip += 4; + op += 4; + } + + ip = output; + for (i = 0; i < 4; i++) { + a1 = ip[4 * 0]; + c1 = ip[4 * 1]; + d1 = ip[4 * 2]; + b1 = ip[4 * 3]; + a1 += c1; + d1 -= b1; + e1 = (a1 - d1) >> 1; + b1 = e1 - b1; + c1 = e1 - c1; + a1 -= b1; + d1 += c1; + dest[stride * 0] = clip_pixel_add(dest[stride * 0], WRAPLOW(a1)); + dest[stride * 1] = clip_pixel_add(dest[stride * 1], WRAPLOW(b1)); + dest[stride * 2] = clip_pixel_add(dest[stride * 2], WRAPLOW(c1)); + dest[stride * 3] = clip_pixel_add(dest[stride * 3], WRAPLOW(d1)); + + ip++; + dest++; + } +} + +void aom_iwht4x4_1_add_c(const tran_low_t *in, uint8_t *dest, int dest_stride) { + int i; + tran_high_t a1, e1; + tran_low_t tmp[4]; + const tran_low_t *ip = in; + tran_low_t *op = tmp; + + a1 = ip[0] >> UNIT_QUANT_SHIFT; + e1 = a1 >> 1; + a1 -= e1; + op[0] = WRAPLOW(a1); + op[1] = op[2] = op[3] = WRAPLOW(e1); + + ip = tmp; + for (i = 0; i < 4; i++) { + e1 = ip[0] >> 1; + a1 = ip[0] - e1; + dest[dest_stride * 0] = clip_pixel_add(dest[dest_stride * 0], a1); + dest[dest_stride * 1] = clip_pixel_add(dest[dest_stride * 1], e1); + dest[dest_stride * 2] = clip_pixel_add(dest[dest_stride * 2], e1); + dest[dest_stride * 3] = clip_pixel_add(dest[dest_stride * 3], e1); + ip++; + dest++; + } +} + +void aom_idct4_c(const tran_low_t *input, tran_low_t *output) { + tran_low_t step[4]; + tran_high_t temp1, temp2; + // stage 1 + temp1 = (input[0] + input[2]) * cospi_16_64; + temp2 = (input[0] - input[2]) * cospi_16_64; + step[0] = WRAPLOW(dct_const_round_shift(temp1)); + step[1] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = input[1] * cospi_24_64 - input[3] * cospi_8_64; + temp2 = input[1] * cospi_8_64 + input[3] * cospi_24_64; + step[2] = WRAPLOW(dct_const_round_shift(temp1)); + step[3] = WRAPLOW(dct_const_round_shift(temp2)); + + // stage 2 + output[0] = WRAPLOW(step[0] + step[3]); + output[1] = WRAPLOW(step[1] + step[2]); + output[2] = WRAPLOW(step[1] - step[2]); + output[3] = WRAPLOW(step[0] - step[3]); +} + +void aom_idct4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride) { + tran_low_t out[4 * 4]; + tran_low_t *outptr = out; + int i, j; + tran_low_t temp_in[4], temp_out[4]; + + // Rows + for (i = 0; i < 4; ++i) { + aom_idct4_c(input, outptr); + input += 4; + outptr += 4; + } + + // Columns + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) temp_in[j] = out[j * 4 + i]; + aom_idct4_c(temp_in, temp_out); + for (j = 0; j < 4; ++j) { + dest[j * stride + i] = clip_pixel_add(dest[j * stride + i], + ROUND_POWER_OF_TWO(temp_out[j], 4)); + } + } +} + +void aom_idct4x4_1_add_c(const tran_low_t *input, uint8_t *dest, + int dest_stride) { + int i; + tran_high_t a1; + tran_low_t out = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + out = WRAPLOW(dct_const_round_shift(out * cospi_16_64)); + a1 = ROUND_POWER_OF_TWO(out, 4); + + if (a1 == 0) return; + + for (i = 0; i < 4; i++) { + dest[0] = clip_pixel_add(dest[0], a1); + dest[1] = clip_pixel_add(dest[1], a1); + dest[2] = clip_pixel_add(dest[2], a1); + dest[3] = clip_pixel_add(dest[3], a1); + dest += dest_stride; + } +} + +void aom_idct8_c(const tran_low_t *input, tran_low_t *output) { + tran_low_t step1[8], step2[8]; + tran_high_t temp1, temp2; + // stage 1 + step1[0] = input[0]; + step1[2] = input[4]; + step1[1] = input[2]; + step1[3] = input[6]; + temp1 = input[1] * cospi_28_64 - input[7] * cospi_4_64; + temp2 = input[1] * cospi_4_64 + input[7] * cospi_28_64; + step1[4] = WRAPLOW(dct_const_round_shift(temp1)); + step1[7] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = input[5] * cospi_12_64 - input[3] * cospi_20_64; + temp2 = input[5] * cospi_20_64 + input[3] * cospi_12_64; + step1[5] = WRAPLOW(dct_const_round_shift(temp1)); + step1[6] = WRAPLOW(dct_const_round_shift(temp2)); + + // stage 2 + temp1 = (step1[0] + step1[2]) * cospi_16_64; + temp2 = (step1[0] - step1[2]) * cospi_16_64; + step2[0] = WRAPLOW(dct_const_round_shift(temp1)); + step2[1] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = step1[1] * cospi_24_64 - step1[3] * cospi_8_64; + temp2 = step1[1] * cospi_8_64 + step1[3] * cospi_24_64; + step2[2] = WRAPLOW(dct_const_round_shift(temp1)); + step2[3] = WRAPLOW(dct_const_round_shift(temp2)); + step2[4] = WRAPLOW(step1[4] + step1[5]); + step2[5] = WRAPLOW(step1[4] - step1[5]); + step2[6] = WRAPLOW(-step1[6] + step1[7]); + step2[7] = WRAPLOW(step1[6] + step1[7]); + + // stage 3 + step1[0] = WRAPLOW(step2[0] + step2[3]); + step1[1] = WRAPLOW(step2[1] + step2[2]); + step1[2] = WRAPLOW(step2[1] - step2[2]); + step1[3] = WRAPLOW(step2[0] - step2[3]); + step1[4] = step2[4]; + temp1 = (step2[6] - step2[5]) * cospi_16_64; + temp2 = (step2[5] + step2[6]) * cospi_16_64; + step1[5] = WRAPLOW(dct_const_round_shift(temp1)); + step1[6] = WRAPLOW(dct_const_round_shift(temp2)); + step1[7] = step2[7]; + + // stage 4 + output[0] = WRAPLOW(step1[0] + step1[7]); + output[1] = WRAPLOW(step1[1] + step1[6]); + output[2] = WRAPLOW(step1[2] + step1[5]); + output[3] = WRAPLOW(step1[3] + step1[4]); + output[4] = WRAPLOW(step1[3] - step1[4]); + output[5] = WRAPLOW(step1[2] - step1[5]); + output[6] = WRAPLOW(step1[1] - step1[6]); + output[7] = WRAPLOW(step1[0] - step1[7]); +} + +void aom_idct8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride) { + tran_low_t out[8 * 8]; + tran_low_t *outptr = out; + int i, j; + tran_low_t temp_in[8], temp_out[8]; + + // First transform rows + for (i = 0; i < 8; ++i) { + aom_idct8_c(input, outptr); + input += 8; + outptr += 8; + } + + // Then transform columns + for (i = 0; i < 8; ++i) { + for (j = 0; j < 8; ++j) temp_in[j] = out[j * 8 + i]; + aom_idct8_c(temp_in, temp_out); + for (j = 0; j < 8; ++j) { + dest[j * stride + i] = clip_pixel_add(dest[j * stride + i], + ROUND_POWER_OF_TWO(temp_out[j], 5)); + } + } +} + +void aom_idct8x8_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) { + int i, j; + tran_high_t a1; + tran_low_t out = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + out = WRAPLOW(dct_const_round_shift(out * cospi_16_64)); + a1 = ROUND_POWER_OF_TWO(out, 5); + if (a1 == 0) return; + for (j = 0; j < 8; ++j) { + for (i = 0; i < 8; ++i) dest[i] = clip_pixel_add(dest[i], a1); + dest += stride; + } +} + +void aom_iadst4_c(const tran_low_t *input, tran_low_t *output) { + tran_high_t s0, s1, s2, s3, s4, s5, s6, s7; + + tran_low_t x0 = input[0]; + tran_low_t x1 = input[1]; + tran_low_t x2 = input[2]; + tran_low_t x3 = input[3]; + + if (!(x0 | x1 | x2 | x3)) { + output[0] = output[1] = output[2] = output[3] = 0; + return; + } + + s0 = sinpi_1_9 * x0; + s1 = sinpi_2_9 * x0; + s2 = sinpi_3_9 * x1; + s3 = sinpi_4_9 * x2; + s4 = sinpi_1_9 * x2; + s5 = sinpi_2_9 * x3; + s6 = sinpi_4_9 * x3; + s7 = WRAPLOW(x0 - x2 + x3); + + s0 = s0 + s3 + s5; + s1 = s1 - s4 - s6; + s3 = s2; + s2 = sinpi_3_9 * s7; + + // 1-D transform scaling factor is sqrt(2). + // The overall dynamic range is 14b (input) + 14b (multiplication scaling) + // + 1b (addition) = 29b. + // Hence the output bit depth is 15b. + output[0] = WRAPLOW(dct_const_round_shift(s0 + s3)); + output[1] = WRAPLOW(dct_const_round_shift(s1 + s3)); + output[2] = WRAPLOW(dct_const_round_shift(s2)); + output[3] = WRAPLOW(dct_const_round_shift(s0 + s1 - s3)); +} + +void aom_iadst8_c(const tran_low_t *input, tran_low_t *output) { + int s0, s1, s2, s3, s4, s5, s6, s7; + + tran_high_t x0 = input[7]; + tran_high_t x1 = input[0]; + tran_high_t x2 = input[5]; + tran_high_t x3 = input[2]; + tran_high_t x4 = input[3]; + tran_high_t x5 = input[4]; + tran_high_t x6 = input[1]; + tran_high_t x7 = input[6]; + + if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7)) { + output[0] = output[1] = output[2] = output[3] = output[4] = output[5] = + output[6] = output[7] = 0; + return; + } + + // stage 1 + s0 = (int)(cospi_2_64 * x0 + cospi_30_64 * x1); + s1 = (int)(cospi_30_64 * x0 - cospi_2_64 * x1); + s2 = (int)(cospi_10_64 * x2 + cospi_22_64 * x3); + s3 = (int)(cospi_22_64 * x2 - cospi_10_64 * x3); + s4 = (int)(cospi_18_64 * x4 + cospi_14_64 * x5); + s5 = (int)(cospi_14_64 * x4 - cospi_18_64 * x5); + s6 = (int)(cospi_26_64 * x6 + cospi_6_64 * x7); + s7 = (int)(cospi_6_64 * x6 - cospi_26_64 * x7); + + x0 = WRAPLOW(dct_const_round_shift(s0 + s4)); + x1 = WRAPLOW(dct_const_round_shift(s1 + s5)); + x2 = WRAPLOW(dct_const_round_shift(s2 + s6)); + x3 = WRAPLOW(dct_const_round_shift(s3 + s7)); + x4 = WRAPLOW(dct_const_round_shift(s0 - s4)); + x5 = WRAPLOW(dct_const_round_shift(s1 - s5)); + x6 = WRAPLOW(dct_const_round_shift(s2 - s6)); + x7 = WRAPLOW(dct_const_round_shift(s3 - s7)); + + // stage 2 + s0 = (int)x0; + s1 = (int)x1; + s2 = (int)x2; + s3 = (int)x3; + s4 = (int)(cospi_8_64 * x4 + cospi_24_64 * x5); + s5 = (int)(cospi_24_64 * x4 - cospi_8_64 * x5); + s6 = (int)(-cospi_24_64 * x6 + cospi_8_64 * x7); + s7 = (int)(cospi_8_64 * x6 + cospi_24_64 * x7); + + x0 = WRAPLOW(s0 + s2); + x1 = WRAPLOW(s1 + s3); + x2 = WRAPLOW(s0 - s2); + x3 = WRAPLOW(s1 - s3); + x4 = WRAPLOW(dct_const_round_shift(s4 + s6)); + x5 = WRAPLOW(dct_const_round_shift(s5 + s7)); + x6 = WRAPLOW(dct_const_round_shift(s4 - s6)); + x7 = WRAPLOW(dct_const_round_shift(s5 - s7)); + + // stage 3 + s2 = (int)(cospi_16_64 * (x2 + x3)); + s3 = (int)(cospi_16_64 * (x2 - x3)); + s6 = (int)(cospi_16_64 * (x6 + x7)); + s7 = (int)(cospi_16_64 * (x6 - x7)); + + x2 = WRAPLOW(dct_const_round_shift(s2)); + x3 = WRAPLOW(dct_const_round_shift(s3)); + x6 = WRAPLOW(dct_const_round_shift(s6)); + x7 = WRAPLOW(dct_const_round_shift(s7)); + + output[0] = WRAPLOW(x0); + output[1] = WRAPLOW(-x4); + output[2] = WRAPLOW(x6); + output[3] = WRAPLOW(-x2); + output[4] = WRAPLOW(x3); + output[5] = WRAPLOW(-x7); + output[6] = WRAPLOW(x5); + output[7] = WRAPLOW(-x1); +} + +void aom_idct8x8_12_add_c(const tran_low_t *input, uint8_t *dest, int stride) { + tran_low_t out[8 * 8] = { 0 }; + tran_low_t *outptr = out; + int i, j; + tran_low_t temp_in[8], temp_out[8]; + + // First transform rows + // only first 4 row has non-zero coefs + for (i = 0; i < 4; ++i) { + aom_idct8_c(input, outptr); + input += 8; + outptr += 8; + } + + // Then transform columns + for (i = 0; i < 8; ++i) { + for (j = 0; j < 8; ++j) temp_in[j] = out[j * 8 + i]; + aom_idct8_c(temp_in, temp_out); + for (j = 0; j < 8; ++j) { + dest[j * stride + i] = clip_pixel_add(dest[j * stride + i], + ROUND_POWER_OF_TWO(temp_out[j], 5)); + } + } +} + +void aom_idct16_c(const tran_low_t *input, tran_low_t *output) { + tran_low_t step1[16], step2[16]; + tran_high_t temp1, temp2; + + // stage 1 + step1[0] = input[0 / 2]; + step1[1] = input[16 / 2]; + step1[2] = input[8 / 2]; + step1[3] = input[24 / 2]; + step1[4] = input[4 / 2]; + step1[5] = input[20 / 2]; + step1[6] = input[12 / 2]; + step1[7] = input[28 / 2]; + step1[8] = input[2 / 2]; + step1[9] = input[18 / 2]; + step1[10] = input[10 / 2]; + step1[11] = input[26 / 2]; + step1[12] = input[6 / 2]; + step1[13] = input[22 / 2]; + step1[14] = input[14 / 2]; + step1[15] = input[30 / 2]; + + // stage 2 + step2[0] = step1[0]; + step2[1] = step1[1]; + step2[2] = step1[2]; + step2[3] = step1[3]; + step2[4] = step1[4]; + step2[5] = step1[5]; + step2[6] = step1[6]; + step2[7] = step1[7]; + + temp1 = step1[8] * cospi_30_64 - step1[15] * cospi_2_64; + temp2 = step1[8] * cospi_2_64 + step1[15] * cospi_30_64; + step2[8] = WRAPLOW(dct_const_round_shift(temp1)); + step2[15] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = step1[9] * cospi_14_64 - step1[14] * cospi_18_64; + temp2 = step1[9] * cospi_18_64 + step1[14] * cospi_14_64; + step2[9] = WRAPLOW(dct_const_round_shift(temp1)); + step2[14] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = step1[10] * cospi_22_64 - step1[13] * cospi_10_64; + temp2 = step1[10] * cospi_10_64 + step1[13] * cospi_22_64; + step2[10] = WRAPLOW(dct_const_round_shift(temp1)); + step2[13] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = step1[11] * cospi_6_64 - step1[12] * cospi_26_64; + temp2 = step1[11] * cospi_26_64 + step1[12] * cospi_6_64; + step2[11] = WRAPLOW(dct_const_round_shift(temp1)); + step2[12] = WRAPLOW(dct_const_round_shift(temp2)); + + // stage 3 + step1[0] = step2[0]; + step1[1] = step2[1]; + step1[2] = step2[2]; + step1[3] = step2[3]; + + temp1 = step2[4] * cospi_28_64 - step2[7] * cospi_4_64; + temp2 = step2[4] * cospi_4_64 + step2[7] * cospi_28_64; + step1[4] = WRAPLOW(dct_const_round_shift(temp1)); + step1[7] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = step2[5] * cospi_12_64 - step2[6] * cospi_20_64; + temp2 = step2[5] * cospi_20_64 + step2[6] * cospi_12_64; + step1[5] = WRAPLOW(dct_const_round_shift(temp1)); + step1[6] = WRAPLOW(dct_const_round_shift(temp2)); + + step1[8] = WRAPLOW(step2[8] + step2[9]); + step1[9] = WRAPLOW(step2[8] - step2[9]); + step1[10] = WRAPLOW(-step2[10] + step2[11]); + step1[11] = WRAPLOW(step2[10] + step2[11]); + step1[12] = WRAPLOW(step2[12] + step2[13]); + step1[13] = WRAPLOW(step2[12] - step2[13]); + step1[14] = WRAPLOW(-step2[14] + step2[15]); + step1[15] = WRAPLOW(step2[14] + step2[15]); + + // stage 4 + temp1 = (step1[0] + step1[1]) * cospi_16_64; + temp2 = (step1[0] - step1[1]) * cospi_16_64; + step2[0] = WRAPLOW(dct_const_round_shift(temp1)); + step2[1] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = step1[2] * cospi_24_64 - step1[3] * cospi_8_64; + temp2 = step1[2] * cospi_8_64 + step1[3] * cospi_24_64; + step2[2] = WRAPLOW(dct_const_round_shift(temp1)); + step2[3] = WRAPLOW(dct_const_round_shift(temp2)); + step2[4] = WRAPLOW(step1[4] + step1[5]); + step2[5] = WRAPLOW(step1[4] - step1[5]); + step2[6] = WRAPLOW(-step1[6] + step1[7]); + step2[7] = WRAPLOW(step1[6] + step1[7]); + + step2[8] = step1[8]; + step2[15] = step1[15]; + temp1 = -step1[9] * cospi_8_64 + step1[14] * cospi_24_64; + temp2 = step1[9] * cospi_24_64 + step1[14] * cospi_8_64; + step2[9] = WRAPLOW(dct_const_round_shift(temp1)); + step2[14] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = -step1[10] * cospi_24_64 - step1[13] * cospi_8_64; + temp2 = -step1[10] * cospi_8_64 + step1[13] * cospi_24_64; + step2[10] = WRAPLOW(dct_const_round_shift(temp1)); + step2[13] = WRAPLOW(dct_const_round_shift(temp2)); + step2[11] = step1[11]; + step2[12] = step1[12]; + + // stage 5 + step1[0] = WRAPLOW(step2[0] + step2[3]); + step1[1] = WRAPLOW(step2[1] + step2[2]); + step1[2] = WRAPLOW(step2[1] - step2[2]); + step1[3] = WRAPLOW(step2[0] - step2[3]); + step1[4] = step2[4]; + temp1 = (step2[6] - step2[5]) * cospi_16_64; + temp2 = (step2[5] + step2[6]) * cospi_16_64; + step1[5] = WRAPLOW(dct_const_round_shift(temp1)); + step1[6] = WRAPLOW(dct_const_round_shift(temp2)); + step1[7] = step2[7]; + + step1[8] = WRAPLOW(step2[8] + step2[11]); + step1[9] = WRAPLOW(step2[9] + step2[10]); + step1[10] = WRAPLOW(step2[9] - step2[10]); + step1[11] = WRAPLOW(step2[8] - step2[11]); + step1[12] = WRAPLOW(-step2[12] + step2[15]); + step1[13] = WRAPLOW(-step2[13] + step2[14]); + step1[14] = WRAPLOW(step2[13] + step2[14]); + step1[15] = WRAPLOW(step2[12] + step2[15]); + + // stage 6 + step2[0] = WRAPLOW(step1[0] + step1[7]); + step2[1] = WRAPLOW(step1[1] + step1[6]); + step2[2] = WRAPLOW(step1[2] + step1[5]); + step2[3] = WRAPLOW(step1[3] + step1[4]); + step2[4] = WRAPLOW(step1[3] - step1[4]); + step2[5] = WRAPLOW(step1[2] - step1[5]); + step2[6] = WRAPLOW(step1[1] - step1[6]); + step2[7] = WRAPLOW(step1[0] - step1[7]); + step2[8] = step1[8]; + step2[9] = step1[9]; + temp1 = (-step1[10] + step1[13]) * cospi_16_64; + temp2 = (step1[10] + step1[13]) * cospi_16_64; + step2[10] = WRAPLOW(dct_const_round_shift(temp1)); + step2[13] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = (-step1[11] + step1[12]) * cospi_16_64; + temp2 = (step1[11] + step1[12]) * cospi_16_64; + step2[11] = WRAPLOW(dct_const_round_shift(temp1)); + step2[12] = WRAPLOW(dct_const_round_shift(temp2)); + step2[14] = step1[14]; + step2[15] = step1[15]; + + // stage 7 + output[0] = WRAPLOW(step2[0] + step2[15]); + output[1] = WRAPLOW(step2[1] + step2[14]); + output[2] = WRAPLOW(step2[2] + step2[13]); + output[3] = WRAPLOW(step2[3] + step2[12]); + output[4] = WRAPLOW(step2[4] + step2[11]); + output[5] = WRAPLOW(step2[5] + step2[10]); + output[6] = WRAPLOW(step2[6] + step2[9]); + output[7] = WRAPLOW(step2[7] + step2[8]); + output[8] = WRAPLOW(step2[7] - step2[8]); + output[9] = WRAPLOW(step2[6] - step2[9]); + output[10] = WRAPLOW(step2[5] - step2[10]); + output[11] = WRAPLOW(step2[4] - step2[11]); + output[12] = WRAPLOW(step2[3] - step2[12]); + output[13] = WRAPLOW(step2[2] - step2[13]); + output[14] = WRAPLOW(step2[1] - step2[14]); + output[15] = WRAPLOW(step2[0] - step2[15]); +} + +void aom_idct16x16_256_add_c(const tran_low_t *input, uint8_t *dest, + int stride) { + tran_low_t out[16 * 16]; + tran_low_t *outptr = out; + int i, j; + tran_low_t temp_in[16], temp_out[16]; + + // First transform rows + for (i = 0; i < 16; ++i) { + aom_idct16_c(input, outptr); + input += 16; + outptr += 16; + } + + // Then transform columns + for (i = 0; i < 16; ++i) { + for (j = 0; j < 16; ++j) temp_in[j] = out[j * 16 + i]; + aom_idct16_c(temp_in, temp_out); + for (j = 0; j < 16; ++j) { + dest[j * stride + i] = clip_pixel_add(dest[j * stride + i], + ROUND_POWER_OF_TWO(temp_out[j], 6)); + } + } +} + +void aom_iadst16_c(const tran_low_t *input, tran_low_t *output) { + tran_high_t s0, s1, s2, s3, s4, s5, s6, s7, s8; + tran_high_t s9, s10, s11, s12, s13, s14, s15; + + tran_high_t x0 = input[15]; + tran_high_t x1 = input[0]; + tran_high_t x2 = input[13]; + tran_high_t x3 = input[2]; + tran_high_t x4 = input[11]; + tran_high_t x5 = input[4]; + tran_high_t x6 = input[9]; + tran_high_t x7 = input[6]; + tran_high_t x8 = input[7]; + tran_high_t x9 = input[8]; + tran_high_t x10 = input[5]; + tran_high_t x11 = input[10]; + tran_high_t x12 = input[3]; + tran_high_t x13 = input[12]; + tran_high_t x14 = input[1]; + tran_high_t x15 = input[14]; + + if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7 | x8 | x9 | x10 | x11 | x12 | + x13 | x14 | x15)) { + output[0] = output[1] = output[2] = output[3] = output[4] = output[5] = + output[6] = output[7] = output[8] = output[9] = output[10] = + output[11] = output[12] = output[13] = output[14] = output[15] = 0; + return; + } + + // stage 1 + s0 = x0 * cospi_1_64 + x1 * cospi_31_64; + s1 = x0 * cospi_31_64 - x1 * cospi_1_64; + s2 = x2 * cospi_5_64 + x3 * cospi_27_64; + s3 = x2 * cospi_27_64 - x3 * cospi_5_64; + s4 = x4 * cospi_9_64 + x5 * cospi_23_64; + s5 = x4 * cospi_23_64 - x5 * cospi_9_64; + s6 = x6 * cospi_13_64 + x7 * cospi_19_64; + s7 = x6 * cospi_19_64 - x7 * cospi_13_64; + s8 = x8 * cospi_17_64 + x9 * cospi_15_64; + s9 = x8 * cospi_15_64 - x9 * cospi_17_64; + s10 = x10 * cospi_21_64 + x11 * cospi_11_64; + s11 = x10 * cospi_11_64 - x11 * cospi_21_64; + s12 = x12 * cospi_25_64 + x13 * cospi_7_64; + s13 = x12 * cospi_7_64 - x13 * cospi_25_64; + s14 = x14 * cospi_29_64 + x15 * cospi_3_64; + s15 = x14 * cospi_3_64 - x15 * cospi_29_64; + + x0 = WRAPLOW(dct_const_round_shift(s0 + s8)); + x1 = WRAPLOW(dct_const_round_shift(s1 + s9)); + x2 = WRAPLOW(dct_const_round_shift(s2 + s10)); + x3 = WRAPLOW(dct_const_round_shift(s3 + s11)); + x4 = WRAPLOW(dct_const_round_shift(s4 + s12)); + x5 = WRAPLOW(dct_const_round_shift(s5 + s13)); + x6 = WRAPLOW(dct_const_round_shift(s6 + s14)); + x7 = WRAPLOW(dct_const_round_shift(s7 + s15)); + x8 = WRAPLOW(dct_const_round_shift(s0 - s8)); + x9 = WRAPLOW(dct_const_round_shift(s1 - s9)); + x10 = WRAPLOW(dct_const_round_shift(s2 - s10)); + x11 = WRAPLOW(dct_const_round_shift(s3 - s11)); + x12 = WRAPLOW(dct_const_round_shift(s4 - s12)); + x13 = WRAPLOW(dct_const_round_shift(s5 - s13)); + x14 = WRAPLOW(dct_const_round_shift(s6 - s14)); + x15 = WRAPLOW(dct_const_round_shift(s7 - s15)); + + // stage 2 + s0 = x0; + s1 = x1; + s2 = x2; + s3 = x3; + s4 = x4; + s5 = x5; + s6 = x6; + s7 = x7; + s8 = x8 * cospi_4_64 + x9 * cospi_28_64; + s9 = x8 * cospi_28_64 - x9 * cospi_4_64; + s10 = x10 * cospi_20_64 + x11 * cospi_12_64; + s11 = x10 * cospi_12_64 - x11 * cospi_20_64; + s12 = -x12 * cospi_28_64 + x13 * cospi_4_64; + s13 = x12 * cospi_4_64 + x13 * cospi_28_64; + s14 = -x14 * cospi_12_64 + x15 * cospi_20_64; + s15 = x14 * cospi_20_64 + x15 * cospi_12_64; + + x0 = WRAPLOW(s0 + s4); + x1 = WRAPLOW(s1 + s5); + x2 = WRAPLOW(s2 + s6); + x3 = WRAPLOW(s3 + s7); + x4 = WRAPLOW(s0 - s4); + x5 = WRAPLOW(s1 - s5); + x6 = WRAPLOW(s2 - s6); + x7 = WRAPLOW(s3 - s7); + x8 = WRAPLOW(dct_const_round_shift(s8 + s12)); + x9 = WRAPLOW(dct_const_round_shift(s9 + s13)); + x10 = WRAPLOW(dct_const_round_shift(s10 + s14)); + x11 = WRAPLOW(dct_const_round_shift(s11 + s15)); + x12 = WRAPLOW(dct_const_round_shift(s8 - s12)); + x13 = WRAPLOW(dct_const_round_shift(s9 - s13)); + x14 = WRAPLOW(dct_const_round_shift(s10 - s14)); + x15 = WRAPLOW(dct_const_round_shift(s11 - s15)); + + // stage 3 + s0 = x0; + s1 = x1; + s2 = x2; + s3 = x3; + s4 = x4 * cospi_8_64 + x5 * cospi_24_64; + s5 = x4 * cospi_24_64 - x5 * cospi_8_64; + s6 = -x6 * cospi_24_64 + x7 * cospi_8_64; + s7 = x6 * cospi_8_64 + x7 * cospi_24_64; + s8 = x8; + s9 = x9; + s10 = x10; + s11 = x11; + s12 = x12 * cospi_8_64 + x13 * cospi_24_64; + s13 = x12 * cospi_24_64 - x13 * cospi_8_64; + s14 = -x14 * cospi_24_64 + x15 * cospi_8_64; + s15 = x14 * cospi_8_64 + x15 * cospi_24_64; + + x0 = WRAPLOW(s0 + s2); + x1 = WRAPLOW(s1 + s3); + x2 = WRAPLOW(s0 - s2); + x3 = WRAPLOW(s1 - s3); + x4 = WRAPLOW(dct_const_round_shift(s4 + s6)); + x5 = WRAPLOW(dct_const_round_shift(s5 + s7)); + x6 = WRAPLOW(dct_const_round_shift(s4 - s6)); + x7 = WRAPLOW(dct_const_round_shift(s5 - s7)); + x8 = WRAPLOW(s8 + s10); + x9 = WRAPLOW(s9 + s11); + x10 = WRAPLOW(s8 - s10); + x11 = WRAPLOW(s9 - s11); + x12 = WRAPLOW(dct_const_round_shift(s12 + s14)); + x13 = WRAPLOW(dct_const_round_shift(s13 + s15)); + x14 = WRAPLOW(dct_const_round_shift(s12 - s14)); + x15 = WRAPLOW(dct_const_round_shift(s13 - s15)); + + // stage 4 + s2 = (-cospi_16_64) * (x2 + x3); + s3 = cospi_16_64 * (x2 - x3); + s6 = cospi_16_64 * (x6 + x7); + s7 = cospi_16_64 * (-x6 + x7); + s10 = cospi_16_64 * (x10 + x11); + s11 = cospi_16_64 * (-x10 + x11); + s14 = (-cospi_16_64) * (x14 + x15); + s15 = cospi_16_64 * (x14 - x15); + + x2 = WRAPLOW(dct_const_round_shift(s2)); + x3 = WRAPLOW(dct_const_round_shift(s3)); + x6 = WRAPLOW(dct_const_round_shift(s6)); + x7 = WRAPLOW(dct_const_round_shift(s7)); + x10 = WRAPLOW(dct_const_round_shift(s10)); + x11 = WRAPLOW(dct_const_round_shift(s11)); + x14 = WRAPLOW(dct_const_round_shift(s14)); + x15 = WRAPLOW(dct_const_round_shift(s15)); + + output[0] = WRAPLOW(x0); + output[1] = WRAPLOW(-x8); + output[2] = WRAPLOW(x12); + output[3] = WRAPLOW(-x4); + output[4] = WRAPLOW(x6); + output[5] = WRAPLOW(x14); + output[6] = WRAPLOW(x10); + output[7] = WRAPLOW(x2); + output[8] = WRAPLOW(x3); + output[9] = WRAPLOW(x11); + output[10] = WRAPLOW(x15); + output[11] = WRAPLOW(x7); + output[12] = WRAPLOW(x5); + output[13] = WRAPLOW(-x13); + output[14] = WRAPLOW(x9); + output[15] = WRAPLOW(-x1); +} + +void aom_idct16x16_38_add_c(const tran_low_t *input, uint8_t *dest, + int stride) { + int i, j; + tran_low_t out[16 * 16] = { 0 }; + tran_low_t *outptr = out; + tran_low_t temp_in[16], temp_out[16]; + + // First transform rows. Since all non-zero dct coefficients are in + // upper-left 8x8 area, we only need to calculate first 8 rows here. + for (i = 0; i < 8; ++i) { + aom_idct16_c(input, outptr); + input += 16; + outptr += 16; + } + + // Then transform columns + for (i = 0; i < 16; ++i) { + for (j = 0; j < 16; ++j) temp_in[j] = out[j * 16 + i]; + aom_idct16_c(temp_in, temp_out); + for (j = 0; j < 16; ++j) { + dest[j * stride + i] = clip_pixel_add(dest[j * stride + i], + ROUND_POWER_OF_TWO(temp_out[j], 6)); + } + } +} + +void aom_idct16x16_10_add_c(const tran_low_t *input, uint8_t *dest, + int stride) { + tran_low_t out[16 * 16] = { 0 }; + tran_low_t *outptr = out; + int i, j; + tran_low_t temp_in[16], temp_out[16]; + + // First transform rows. Since all non-zero dct coefficients are in + // upper-left 4x4 area, we only need to calculate first 4 rows here. + for (i = 0; i < 4; ++i) { + aom_idct16_c(input, outptr); + input += 16; + outptr += 16; + } + + // Then transform columns + for (i = 0; i < 16; ++i) { + for (j = 0; j < 16; ++j) temp_in[j] = out[j * 16 + i]; + aom_idct16_c(temp_in, temp_out); + for (j = 0; j < 16; ++j) { + dest[j * stride + i] = clip_pixel_add(dest[j * stride + i], + ROUND_POWER_OF_TWO(temp_out[j], 6)); + } + } +} + +void aom_idct16x16_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) { + int i, j; + tran_high_t a1; + tran_low_t out = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + out = WRAPLOW(dct_const_round_shift(out * cospi_16_64)); + a1 = ROUND_POWER_OF_TWO(out, 6); + if (a1 == 0) return; + for (j = 0; j < 16; ++j) { + for (i = 0; i < 16; ++i) dest[i] = clip_pixel_add(dest[i], a1); + dest += stride; + } +} + +void aom_idct32_c(const tran_low_t *input, tran_low_t *output) { + tran_low_t step1[32], step2[32]; + tran_high_t temp1, temp2; + + // stage 1 + step1[0] = input[0]; + step1[1] = input[16]; + step1[2] = input[8]; + step1[3] = input[24]; + step1[4] = input[4]; + step1[5] = input[20]; + step1[6] = input[12]; + step1[7] = input[28]; + step1[8] = input[2]; + step1[9] = input[18]; + step1[10] = input[10]; + step1[11] = input[26]; + step1[12] = input[6]; + step1[13] = input[22]; + step1[14] = input[14]; + step1[15] = input[30]; + + temp1 = input[1] * cospi_31_64 - input[31] * cospi_1_64; + temp2 = input[1] * cospi_1_64 + input[31] * cospi_31_64; + step1[16] = WRAPLOW(dct_const_round_shift(temp1)); + step1[31] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = input[17] * cospi_15_64 - input[15] * cospi_17_64; + temp2 = input[17] * cospi_17_64 + input[15] * cospi_15_64; + step1[17] = WRAPLOW(dct_const_round_shift(temp1)); + step1[30] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = input[9] * cospi_23_64 - input[23] * cospi_9_64; + temp2 = input[9] * cospi_9_64 + input[23] * cospi_23_64; + step1[18] = WRAPLOW(dct_const_round_shift(temp1)); + step1[29] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = input[25] * cospi_7_64 - input[7] * cospi_25_64; + temp2 = input[25] * cospi_25_64 + input[7] * cospi_7_64; + step1[19] = WRAPLOW(dct_const_round_shift(temp1)); + step1[28] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = input[5] * cospi_27_64 - input[27] * cospi_5_64; + temp2 = input[5] * cospi_5_64 + input[27] * cospi_27_64; + step1[20] = WRAPLOW(dct_const_round_shift(temp1)); + step1[27] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = input[21] * cospi_11_64 - input[11] * cospi_21_64; + temp2 = input[21] * cospi_21_64 + input[11] * cospi_11_64; + step1[21] = WRAPLOW(dct_const_round_shift(temp1)); + step1[26] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = input[13] * cospi_19_64 - input[19] * cospi_13_64; + temp2 = input[13] * cospi_13_64 + input[19] * cospi_19_64; + step1[22] = WRAPLOW(dct_const_round_shift(temp1)); + step1[25] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = input[29] * cospi_3_64 - input[3] * cospi_29_64; + temp2 = input[29] * cospi_29_64 + input[3] * cospi_3_64; + step1[23] = WRAPLOW(dct_const_round_shift(temp1)); + step1[24] = WRAPLOW(dct_const_round_shift(temp2)); + + // stage 2 + step2[0] = step1[0]; + step2[1] = step1[1]; + step2[2] = step1[2]; + step2[3] = step1[3]; + step2[4] = step1[4]; + step2[5] = step1[5]; + step2[6] = step1[6]; + step2[7] = step1[7]; + + temp1 = step1[8] * cospi_30_64 - step1[15] * cospi_2_64; + temp2 = step1[8] * cospi_2_64 + step1[15] * cospi_30_64; + step2[8] = WRAPLOW(dct_const_round_shift(temp1)); + step2[15] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = step1[9] * cospi_14_64 - step1[14] * cospi_18_64; + temp2 = step1[9] * cospi_18_64 + step1[14] * cospi_14_64; + step2[9] = WRAPLOW(dct_const_round_shift(temp1)); + step2[14] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = step1[10] * cospi_22_64 - step1[13] * cospi_10_64; + temp2 = step1[10] * cospi_10_64 + step1[13] * cospi_22_64; + step2[10] = WRAPLOW(dct_const_round_shift(temp1)); + step2[13] = WRAPLOW(dct_const_round_shift(temp2)); + + temp1 = step1[11] * cospi_6_64 - step1[12] * cospi_26_64; + temp2 = step1[11] * cospi_26_64 + step1[12] * cospi_6_64; + step2[11] = WRAPLOW(dct_const_round_shift(temp1)); + step2[12] = WRAPLOW(dct_const_round_shift(temp2)); + + step2[16] = WRAPLOW(step1[16] + step1[17]); + step2[17] = WRAPLOW(step1[16] - step1[17]); + step2[18] = WRAPLOW(-step1[18] + step1[19]); + step2[19] = WRAPLOW(step1[18] + step1[19]); + step2[20] = WRAPLOW(step1[20] + step1[21]); + step2[21] = WRAPLOW(step1[20] - step1[21]); + step2[22] = WRAPLOW(-step1[22] + step1[23]); + step2[23] = WRAPLOW(step1[22] + step1[23]); + step2[24] = WRAPLOW(step1[24] + step1[25]); + step2[25] = WRAPLOW(step1[24] - step1[25]); + step2[26] = WRAPLOW(-step1[26] + step1[27]); + step2[27] = WRAPLOW(step1[26] + step1[27]); + step2[28] = WRAPLOW(step1[28] + step1[29]); + step2[29] = WRAPLOW(step1[28] - step1[29]); + step2[30] = WRAPLOW(-step1[30] + step1[31]); + step2[31] = WRAPLOW(step1[30] + step1[31]); + + // stage 3 + step1[0] = step2[0]; + step1[1] = step2[1]; + step1[2] = step2[2]; + step1[3] = step2[3]; + + temp1 = step2[4] * cospi_28_64 - step2[7] * cospi_4_64; + temp2 = step2[4] * cospi_4_64 + step2[7] * cospi_28_64; + step1[4] = WRAPLOW(dct_const_round_shift(temp1)); + step1[7] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = step2[5] * cospi_12_64 - step2[6] * cospi_20_64; + temp2 = step2[5] * cospi_20_64 + step2[6] * cospi_12_64; + step1[5] = WRAPLOW(dct_const_round_shift(temp1)); + step1[6] = WRAPLOW(dct_const_round_shift(temp2)); + + step1[8] = WRAPLOW(step2[8] + step2[9]); + step1[9] = WRAPLOW(step2[8] - step2[9]); + step1[10] = WRAPLOW(-step2[10] + step2[11]); + step1[11] = WRAPLOW(step2[10] + step2[11]); + step1[12] = WRAPLOW(step2[12] + step2[13]); + step1[13] = WRAPLOW(step2[12] - step2[13]); + step1[14] = WRAPLOW(-step2[14] + step2[15]); + step1[15] = WRAPLOW(step2[14] + step2[15]); + + step1[16] = step2[16]; + step1[31] = step2[31]; + temp1 = -step2[17] * cospi_4_64 + step2[30] * cospi_28_64; + temp2 = step2[17] * cospi_28_64 + step2[30] * cospi_4_64; + step1[17] = WRAPLOW(dct_const_round_shift(temp1)); + step1[30] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = -step2[18] * cospi_28_64 - step2[29] * cospi_4_64; + temp2 = -step2[18] * cospi_4_64 + step2[29] * cospi_28_64; + step1[18] = WRAPLOW(dct_const_round_shift(temp1)); + step1[29] = WRAPLOW(dct_const_round_shift(temp2)); + step1[19] = step2[19]; + step1[20] = step2[20]; + temp1 = -step2[21] * cospi_20_64 + step2[26] * cospi_12_64; + temp2 = step2[21] * cospi_12_64 + step2[26] * cospi_20_64; + step1[21] = WRAPLOW(dct_const_round_shift(temp1)); + step1[26] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = -step2[22] * cospi_12_64 - step2[25] * cospi_20_64; + temp2 = -step2[22] * cospi_20_64 + step2[25] * cospi_12_64; + step1[22] = WRAPLOW(dct_const_round_shift(temp1)); + step1[25] = WRAPLOW(dct_const_round_shift(temp2)); + step1[23] = step2[23]; + step1[24] = step2[24]; + step1[27] = step2[27]; + step1[28] = step2[28]; + + // stage 4 + temp1 = (step1[0] + step1[1]) * cospi_16_64; + temp2 = (step1[0] - step1[1]) * cospi_16_64; + step2[0] = WRAPLOW(dct_const_round_shift(temp1)); + step2[1] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = step1[2] * cospi_24_64 - step1[3] * cospi_8_64; + temp2 = step1[2] * cospi_8_64 + step1[3] * cospi_24_64; + step2[2] = WRAPLOW(dct_const_round_shift(temp1)); + step2[3] = WRAPLOW(dct_const_round_shift(temp2)); + step2[4] = WRAPLOW(step1[4] + step1[5]); + step2[5] = WRAPLOW(step1[4] - step1[5]); + step2[6] = WRAPLOW(-step1[6] + step1[7]); + step2[7] = WRAPLOW(step1[6] + step1[7]); + + step2[8] = step1[8]; + step2[15] = step1[15]; + temp1 = -step1[9] * cospi_8_64 + step1[14] * cospi_24_64; + temp2 = step1[9] * cospi_24_64 + step1[14] * cospi_8_64; + step2[9] = WRAPLOW(dct_const_round_shift(temp1)); + step2[14] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = -step1[10] * cospi_24_64 - step1[13] * cospi_8_64; + temp2 = -step1[10] * cospi_8_64 + step1[13] * cospi_24_64; + step2[10] = WRAPLOW(dct_const_round_shift(temp1)); + step2[13] = WRAPLOW(dct_const_round_shift(temp2)); + step2[11] = step1[11]; + step2[12] = step1[12]; + + step2[16] = WRAPLOW(step1[16] + step1[19]); + step2[17] = WRAPLOW(step1[17] + step1[18]); + step2[18] = WRAPLOW(step1[17] - step1[18]); + step2[19] = WRAPLOW(step1[16] - step1[19]); + step2[20] = WRAPLOW(-step1[20] + step1[23]); + step2[21] = WRAPLOW(-step1[21] + step1[22]); + step2[22] = WRAPLOW(step1[21] + step1[22]); + step2[23] = WRAPLOW(step1[20] + step1[23]); + + step2[24] = WRAPLOW(step1[24] + step1[27]); + step2[25] = WRAPLOW(step1[25] + step1[26]); + step2[26] = WRAPLOW(step1[25] - step1[26]); + step2[27] = WRAPLOW(step1[24] - step1[27]); + step2[28] = WRAPLOW(-step1[28] + step1[31]); + step2[29] = WRAPLOW(-step1[29] + step1[30]); + step2[30] = WRAPLOW(step1[29] + step1[30]); + step2[31] = WRAPLOW(step1[28] + step1[31]); + + // stage 5 + step1[0] = WRAPLOW(step2[0] + step2[3]); + step1[1] = WRAPLOW(step2[1] + step2[2]); + step1[2] = WRAPLOW(step2[1] - step2[2]); + step1[3] = WRAPLOW(step2[0] - step2[3]); + step1[4] = step2[4]; + temp1 = (step2[6] - step2[5]) * cospi_16_64; + temp2 = (step2[5] + step2[6]) * cospi_16_64; + step1[5] = WRAPLOW(dct_const_round_shift(temp1)); + step1[6] = WRAPLOW(dct_const_round_shift(temp2)); + step1[7] = step2[7]; + + step1[8] = WRAPLOW(step2[8] + step2[11]); + step1[9] = WRAPLOW(step2[9] + step2[10]); + step1[10] = WRAPLOW(step2[9] - step2[10]); + step1[11] = WRAPLOW(step2[8] - step2[11]); + step1[12] = WRAPLOW(-step2[12] + step2[15]); + step1[13] = WRAPLOW(-step2[13] + step2[14]); + step1[14] = WRAPLOW(step2[13] + step2[14]); + step1[15] = WRAPLOW(step2[12] + step2[15]); + + step1[16] = step2[16]; + step1[17] = step2[17]; + temp1 = -step2[18] * cospi_8_64 + step2[29] * cospi_24_64; + temp2 = step2[18] * cospi_24_64 + step2[29] * cospi_8_64; + step1[18] = WRAPLOW(dct_const_round_shift(temp1)); + step1[29] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = -step2[19] * cospi_8_64 + step2[28] * cospi_24_64; + temp2 = step2[19] * cospi_24_64 + step2[28] * cospi_8_64; + step1[19] = WRAPLOW(dct_const_round_shift(temp1)); + step1[28] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = -step2[20] * cospi_24_64 - step2[27] * cospi_8_64; + temp2 = -step2[20] * cospi_8_64 + step2[27] * cospi_24_64; + step1[20] = WRAPLOW(dct_const_round_shift(temp1)); + step1[27] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = -step2[21] * cospi_24_64 - step2[26] * cospi_8_64; + temp2 = -step2[21] * cospi_8_64 + step2[26] * cospi_24_64; + step1[21] = WRAPLOW(dct_const_round_shift(temp1)); + step1[26] = WRAPLOW(dct_const_round_shift(temp2)); + step1[22] = step2[22]; + step1[23] = step2[23]; + step1[24] = step2[24]; + step1[25] = step2[25]; + step1[30] = step2[30]; + step1[31] = step2[31]; + + // stage 6 + step2[0] = WRAPLOW(step1[0] + step1[7]); + step2[1] = WRAPLOW(step1[1] + step1[6]); + step2[2] = WRAPLOW(step1[2] + step1[5]); + step2[3] = WRAPLOW(step1[3] + step1[4]); + step2[4] = WRAPLOW(step1[3] - step1[4]); + step2[5] = WRAPLOW(step1[2] - step1[5]); + step2[6] = WRAPLOW(step1[1] - step1[6]); + step2[7] = WRAPLOW(step1[0] - step1[7]); + step2[8] = step1[8]; + step2[9] = step1[9]; + temp1 = (-step1[10] + step1[13]) * cospi_16_64; + temp2 = (step1[10] + step1[13]) * cospi_16_64; + step2[10] = WRAPLOW(dct_const_round_shift(temp1)); + step2[13] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = (-step1[11] + step1[12]) * cospi_16_64; + temp2 = (step1[11] + step1[12]) * cospi_16_64; + step2[11] = WRAPLOW(dct_const_round_shift(temp1)); + step2[12] = WRAPLOW(dct_const_round_shift(temp2)); + step2[14] = step1[14]; + step2[15] = step1[15]; + + step2[16] = WRAPLOW(step1[16] + step1[23]); + step2[17] = WRAPLOW(step1[17] + step1[22]); + step2[18] = WRAPLOW(step1[18] + step1[21]); + step2[19] = WRAPLOW(step1[19] + step1[20]); + step2[20] = WRAPLOW(step1[19] - step1[20]); + step2[21] = WRAPLOW(step1[18] - step1[21]); + step2[22] = WRAPLOW(step1[17] - step1[22]); + step2[23] = WRAPLOW(step1[16] - step1[23]); + + step2[24] = WRAPLOW(-step1[24] + step1[31]); + step2[25] = WRAPLOW(-step1[25] + step1[30]); + step2[26] = WRAPLOW(-step1[26] + step1[29]); + step2[27] = WRAPLOW(-step1[27] + step1[28]); + step2[28] = WRAPLOW(step1[27] + step1[28]); + step2[29] = WRAPLOW(step1[26] + step1[29]); + step2[30] = WRAPLOW(step1[25] + step1[30]); + step2[31] = WRAPLOW(step1[24] + step1[31]); + + // stage 7 + step1[0] = WRAPLOW(step2[0] + step2[15]); + step1[1] = WRAPLOW(step2[1] + step2[14]); + step1[2] = WRAPLOW(step2[2] + step2[13]); + step1[3] = WRAPLOW(step2[3] + step2[12]); + step1[4] = WRAPLOW(step2[4] + step2[11]); + step1[5] = WRAPLOW(step2[5] + step2[10]); + step1[6] = WRAPLOW(step2[6] + step2[9]); + step1[7] = WRAPLOW(step2[7] + step2[8]); + step1[8] = WRAPLOW(step2[7] - step2[8]); + step1[9] = WRAPLOW(step2[6] - step2[9]); + step1[10] = WRAPLOW(step2[5] - step2[10]); + step1[11] = WRAPLOW(step2[4] - step2[11]); + step1[12] = WRAPLOW(step2[3] - step2[12]); + step1[13] = WRAPLOW(step2[2] - step2[13]); + step1[14] = WRAPLOW(step2[1] - step2[14]); + step1[15] = WRAPLOW(step2[0] - step2[15]); + + step1[16] = step2[16]; + step1[17] = step2[17]; + step1[18] = step2[18]; + step1[19] = step2[19]; + temp1 = (-step2[20] + step2[27]) * cospi_16_64; + temp2 = (step2[20] + step2[27]) * cospi_16_64; + step1[20] = WRAPLOW(dct_const_round_shift(temp1)); + step1[27] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = (-step2[21] + step2[26]) * cospi_16_64; + temp2 = (step2[21] + step2[26]) * cospi_16_64; + step1[21] = WRAPLOW(dct_const_round_shift(temp1)); + step1[26] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = (-step2[22] + step2[25]) * cospi_16_64; + temp2 = (step2[22] + step2[25]) * cospi_16_64; + step1[22] = WRAPLOW(dct_const_round_shift(temp1)); + step1[25] = WRAPLOW(dct_const_round_shift(temp2)); + temp1 = (-step2[23] + step2[24]) * cospi_16_64; + temp2 = (step2[23] + step2[24]) * cospi_16_64; + step1[23] = WRAPLOW(dct_const_round_shift(temp1)); + step1[24] = WRAPLOW(dct_const_round_shift(temp2)); + step1[28] = step2[28]; + step1[29] = step2[29]; + step1[30] = step2[30]; + step1[31] = step2[31]; + + // final stage + output[0] = WRAPLOW(step1[0] + step1[31]); + output[1] = WRAPLOW(step1[1] + step1[30]); + output[2] = WRAPLOW(step1[2] + step1[29]); + output[3] = WRAPLOW(step1[3] + step1[28]); + output[4] = WRAPLOW(step1[4] + step1[27]); + output[5] = WRAPLOW(step1[5] + step1[26]); + output[6] = WRAPLOW(step1[6] + step1[25]); + output[7] = WRAPLOW(step1[7] + step1[24]); + output[8] = WRAPLOW(step1[8] + step1[23]); + output[9] = WRAPLOW(step1[9] + step1[22]); + output[10] = WRAPLOW(step1[10] + step1[21]); + output[11] = WRAPLOW(step1[11] + step1[20]); + output[12] = WRAPLOW(step1[12] + step1[19]); + output[13] = WRAPLOW(step1[13] + step1[18]); + output[14] = WRAPLOW(step1[14] + step1[17]); + output[15] = WRAPLOW(step1[15] + step1[16]); + output[16] = WRAPLOW(step1[15] - step1[16]); + output[17] = WRAPLOW(step1[14] - step1[17]); + output[18] = WRAPLOW(step1[13] - step1[18]); + output[19] = WRAPLOW(step1[12] - step1[19]); + output[20] = WRAPLOW(step1[11] - step1[20]); + output[21] = WRAPLOW(step1[10] - step1[21]); + output[22] = WRAPLOW(step1[9] - step1[22]); + output[23] = WRAPLOW(step1[8] - step1[23]); + output[24] = WRAPLOW(step1[7] - step1[24]); + output[25] = WRAPLOW(step1[6] - step1[25]); + output[26] = WRAPLOW(step1[5] - step1[26]); + output[27] = WRAPLOW(step1[4] - step1[27]); + output[28] = WRAPLOW(step1[3] - step1[28]); + output[29] = WRAPLOW(step1[2] - step1[29]); + output[30] = WRAPLOW(step1[1] - step1[30]); + output[31] = WRAPLOW(step1[0] - step1[31]); +} + +void aom_idct32x32_1024_add_c(const tran_low_t *input, uint8_t *dest, + int stride) { + tran_low_t out[32 * 32]; + tran_low_t *outptr = out; + int i, j; + tran_low_t temp_in[32], temp_out[32]; + + // Rows + for (i = 0; i < 32; ++i) { + int16_t zero_coeff[16]; + for (j = 0; j < 16; ++j) zero_coeff[j] = input[2 * j] | input[2 * j + 1]; + for (j = 0; j < 8; ++j) + zero_coeff[j] = zero_coeff[2 * j] | zero_coeff[2 * j + 1]; + for (j = 0; j < 4; ++j) + zero_coeff[j] = zero_coeff[2 * j] | zero_coeff[2 * j + 1]; + for (j = 0; j < 2; ++j) + zero_coeff[j] = zero_coeff[2 * j] | zero_coeff[2 * j + 1]; + + if (zero_coeff[0] | zero_coeff[1]) + aom_idct32_c(input, outptr); + else + memset(outptr, 0, sizeof(tran_low_t) * 32); + input += 32; + outptr += 32; + } + + // Columns + for (i = 0; i < 32; ++i) { + for (j = 0; j < 32; ++j) temp_in[j] = out[j * 32 + i]; + aom_idct32_c(temp_in, temp_out); + for (j = 0; j < 32; ++j) { + dest[j * stride + i] = clip_pixel_add(dest[j * stride + i], + ROUND_POWER_OF_TWO(temp_out[j], 6)); + } + } +} + +void aom_idct32x32_135_add_c(const tran_low_t *input, uint8_t *dest, + int stride) { + tran_low_t out[32 * 32] = { 0 }; + tran_low_t *outptr = out; + int i, j; + tran_low_t temp_in[32], temp_out[32]; + + // Rows + // only upper-left 16x16 has non-zero coeff + for (i = 0; i < 16; ++i) { + aom_idct32_c(input, outptr); + input += 32; + outptr += 32; + } + + // Columns + for (i = 0; i < 32; ++i) { + for (j = 0; j < 32; ++j) temp_in[j] = out[j * 32 + i]; + aom_idct32_c(temp_in, temp_out); + for (j = 0; j < 32; ++j) { + dest[j * stride + i] = clip_pixel_add(dest[j * stride + i], + ROUND_POWER_OF_TWO(temp_out[j], 6)); + } + } +} + +void aom_idct32x32_34_add_c(const tran_low_t *input, uint8_t *dest, + int stride) { + tran_low_t out[32 * 32] = { 0 }; + tran_low_t *outptr = out; + int i, j; + tran_low_t temp_in[32], temp_out[32]; + + // Rows + // only upper-left 8x8 has non-zero coeff + for (i = 0; i < 8; ++i) { + aom_idct32_c(input, outptr); + input += 32; + outptr += 32; + } + + // Columns + for (i = 0; i < 32; ++i) { + for (j = 0; j < 32; ++j) temp_in[j] = out[j * 32 + i]; + aom_idct32_c(temp_in, temp_out); + for (j = 0; j < 32; ++j) { + dest[j * stride + i] = clip_pixel_add(dest[j * stride + i], + ROUND_POWER_OF_TWO(temp_out[j], 6)); + } + } +} + +void aom_idct32x32_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) { + int i, j; + tran_high_t a1; + + tran_low_t out = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + out = WRAPLOW(dct_const_round_shift(out * cospi_16_64)); + a1 = ROUND_POWER_OF_TWO(out, 6); + if (a1 == 0) return; + + for (j = 0; j < 32; ++j) { + for (i = 0; i < 32; ++i) dest[i] = clip_pixel_add(dest[i], a1); + dest += stride; + } +} + +#if CONFIG_HIGHBITDEPTH +void aom_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int bd) { + /* 4-point reversible, orthonormal inverse Walsh-Hadamard in 3.5 adds, + 0.5 shifts per pixel. */ + int i; + tran_low_t output[16]; + tran_high_t a1, b1, c1, d1, e1; + const tran_low_t *ip = input; + tran_low_t *op = output; + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + for (i = 0; i < 4; i++) { + a1 = ip[0] >> UNIT_QUANT_SHIFT; + c1 = ip[1] >> UNIT_QUANT_SHIFT; + d1 = ip[2] >> UNIT_QUANT_SHIFT; + b1 = ip[3] >> UNIT_QUANT_SHIFT; + a1 += c1; + d1 -= b1; + e1 = (a1 - d1) >> 1; + b1 = e1 - b1; + c1 = e1 - c1; + a1 -= b1; + d1 += c1; + op[0] = HIGHBD_WRAPLOW(a1, bd); + op[1] = HIGHBD_WRAPLOW(b1, bd); + op[2] = HIGHBD_WRAPLOW(c1, bd); + op[3] = HIGHBD_WRAPLOW(d1, bd); + ip += 4; + op += 4; + } + + ip = output; + for (i = 0; i < 4; i++) { + a1 = ip[4 * 0]; + c1 = ip[4 * 1]; + d1 = ip[4 * 2]; + b1 = ip[4 * 3]; + a1 += c1; + d1 -= b1; + e1 = (a1 - d1) >> 1; + b1 = e1 - b1; + c1 = e1 - c1; + a1 -= b1; + d1 += c1; + dest[stride * 0] = + highbd_clip_pixel_add(dest[stride * 0], HIGHBD_WRAPLOW(a1, bd), bd); + dest[stride * 1] = + highbd_clip_pixel_add(dest[stride * 1], HIGHBD_WRAPLOW(b1, bd), bd); + dest[stride * 2] = + highbd_clip_pixel_add(dest[stride * 2], HIGHBD_WRAPLOW(c1, bd), bd); + dest[stride * 3] = + highbd_clip_pixel_add(dest[stride * 3], HIGHBD_WRAPLOW(d1, bd), bd); + + ip++; + dest++; + } +} + +void aom_highbd_iwht4x4_1_add_c(const tran_low_t *in, uint8_t *dest8, + int dest_stride, int bd) { + int i; + tran_high_t a1, e1; + tran_low_t tmp[4]; + const tran_low_t *ip = in; + tran_low_t *op = tmp; + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + (void)bd; + + a1 = ip[0] >> UNIT_QUANT_SHIFT; + e1 = a1 >> 1; + a1 -= e1; + op[0] = HIGHBD_WRAPLOW(a1, bd); + op[1] = op[2] = op[3] = HIGHBD_WRAPLOW(e1, bd); + + ip = tmp; + for (i = 0; i < 4; i++) { + e1 = ip[0] >> 1; + a1 = ip[0] - e1; + dest[dest_stride * 0] = + highbd_clip_pixel_add(dest[dest_stride * 0], a1, bd); + dest[dest_stride * 1] = + highbd_clip_pixel_add(dest[dest_stride * 1], e1, bd); + dest[dest_stride * 2] = + highbd_clip_pixel_add(dest[dest_stride * 2], e1, bd); + dest[dest_stride * 3] = + highbd_clip_pixel_add(dest[dest_stride * 3], e1, bd); + ip++; + dest++; + } +} + +void aom_highbd_idct4_c(const tran_low_t *input, tran_low_t *output, int bd) { + tran_low_t step[4]; + tran_high_t temp1, temp2; + (void)bd; + // stage 1 + temp1 = (input[0] + input[2]) * cospi_16_64; + temp2 = (input[0] - input[2]) * cospi_16_64; + step[0] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd); + step[1] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd); + temp1 = input[1] * cospi_24_64 - input[3] * cospi_8_64; + temp2 = input[1] * cospi_8_64 + input[3] * cospi_24_64; + step[2] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd); + step[3] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd); + + // stage 2 + output[0] = HIGHBD_WRAPLOW(step[0] + step[3], bd); + output[1] = HIGHBD_WRAPLOW(step[1] + step[2], bd); + output[2] = HIGHBD_WRAPLOW(step[1] - step[2], bd); + output[3] = HIGHBD_WRAPLOW(step[0] - step[3], bd); +} + +void aom_highbd_idct4x4_16_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int bd) { + tran_low_t out[4 * 4]; + tran_low_t *outptr = out; + int i, j; + tran_low_t temp_in[4], temp_out[4]; + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + // Rows + for (i = 0; i < 4; ++i) { + aom_highbd_idct4_c(input, outptr, bd); + input += 4; + outptr += 4; + } + + // Columns + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) temp_in[j] = out[j * 4 + i]; + aom_highbd_idct4_c(temp_in, temp_out, bd); + for (j = 0; j < 4; ++j) { + dest[j * stride + i] = highbd_clip_pixel_add( + dest[j * stride + i], ROUND_POWER_OF_TWO(temp_out[j], 4), bd); + } + } +} + +void aom_highbd_idct4x4_1_add_c(const tran_low_t *input, uint8_t *dest8, + int dest_stride, int bd) { + int i; + tran_high_t a1; + tran_low_t out = + HIGHBD_WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64), bd); + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + out = HIGHBD_WRAPLOW(dct_const_round_shift(out * cospi_16_64), bd); + a1 = ROUND_POWER_OF_TWO(out, 4); + + for (i = 0; i < 4; i++) { + dest[0] = highbd_clip_pixel_add(dest[0], a1, bd); + dest[1] = highbd_clip_pixel_add(dest[1], a1, bd); + dest[2] = highbd_clip_pixel_add(dest[2], a1, bd); + dest[3] = highbd_clip_pixel_add(dest[3], a1, bd); + dest += dest_stride; + } +} + +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/inv_txfm.h b/third_party/aom/aom_dsp/inv_txfm.h new file mode 100644 index 0000000000..e64d463eae --- /dev/null +++ b/third_party/aom/aom_dsp/inv_txfm.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_INV_TXFM_H_ +#define AOM_DSP_INV_TXFM_H_ + +#include + +#include "./aom_config.h" +#include "aom_dsp/txfm_common.h" +#include "aom_ports/mem.h" + +#ifdef __cplusplus +extern "C" { +#endif + +static INLINE tran_high_t dct_const_round_shift(tran_high_t input) { + tran_high_t rv = ROUND_POWER_OF_TWO(input, DCT_CONST_BITS); + return rv; +} + +static INLINE tran_high_t check_range(tran_high_t input, int bd) { +#if CONFIG_COEFFICIENT_RANGE_CHECKING + // For valid AV1 input streams, intermediate stage coefficients should always + // stay within the range of a signed 16 bit integer. Coefficients can go out + // of this range for invalid/corrupt AV1 streams. However, strictly checking + // this range for every intermediate coefficient can burdensome for a decoder, + // therefore the following assertion is only enabled when configured with + // --enable-coefficient-range-checking. + // For valid highbitdepth AV1 streams, intermediate stage coefficients will + // stay within the ranges: + // - 8 bit: signed 16 bit integer + // - 10 bit: signed 18 bit integer + // - 12 bit: signed 20 bit integer + const int32_t int_max = (1 << (7 + bd)) - 1; + const int32_t int_min = -int_max - 1; + assert(int_min <= input); + assert(input <= int_max); + (void)int_min; +#endif // CONFIG_COEFFICIENT_RANGE_CHECKING + (void)bd; + return input; +} + +#define WRAPLOW(x) ((int32_t)check_range(x, 8)) +#if CONFIG_HIGHBITDEPTH +#define HIGHBD_WRAPLOW(x, bd) ((int32_t)check_range((x), bd)) +#endif // CONFIG_HIGHBITDEPTH + +void aom_idct4_c(const tran_low_t *input, tran_low_t *output); +void aom_idct8_c(const tran_low_t *input, tran_low_t *output); +void aom_idct16_c(const tran_low_t *input, tran_low_t *output); +void aom_idct32_c(const tran_low_t *input, tran_low_t *output); +void aom_iadst4_c(const tran_low_t *input, tran_low_t *output); +void aom_iadst8_c(const tran_low_t *input, tran_low_t *output); +void aom_iadst16_c(const tran_low_t *input, tran_low_t *output); + +#if CONFIG_HIGHBITDEPTH +void aom_highbd_idct4_c(const tran_low_t *input, tran_low_t *output, int bd); +void aom_highbd_idct8_c(const tran_low_t *input, tran_low_t *output, int bd); +void aom_highbd_idct16_c(const tran_low_t *input, tran_low_t *output, int bd); +void aom_highbd_idct32_c(const tran_low_t *input, tran_low_t *output, int bd); + +void aom_highbd_iadst4_c(const tran_low_t *input, tran_low_t *output, int bd); +void aom_highbd_iadst8_c(const tran_low_t *input, tran_low_t *output, int bd); +void aom_highbd_iadst16_c(const tran_low_t *input, tran_low_t *output, int bd); + +static INLINE uint16_t highbd_clip_pixel_add(uint16_t dest, tran_high_t trans, + int bd) { + trans = HIGHBD_WRAPLOW(trans, bd); + return clip_pixel_highbd(dest + (int)trans, bd); +} +#endif + +static INLINE uint8_t clip_pixel_add(uint8_t dest, tran_high_t trans) { + trans = WRAPLOW(trans); + return clip_pixel(dest + (int)trans); +} +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_INV_TXFM_H_ diff --git a/third_party/aom/aom_dsp/loopfilter.c b/third_party/aom/aom_dsp/loopfilter.c new file mode 100644 index 0000000000..e2e8392194 --- /dev/null +++ b/third_party/aom/aom_dsp/loopfilter.c @@ -0,0 +1,900 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" + +static INLINE int8_t signed_char_clamp(int t) { + return (int8_t)clamp(t, -128, 127); +} + +#define PARALLEL_DEBLOCKING_11_TAP 0 +#define PARALLEL_DEBLOCKING_9_TAP 0 + +#if CONFIG_HIGHBITDEPTH +static INLINE int16_t signed_char_clamp_high(int t, int bd) { + switch (bd) { + case 10: return (int16_t)clamp(t, -128 * 4, 128 * 4 - 1); + case 12: return (int16_t)clamp(t, -128 * 16, 128 * 16 - 1); + case 8: + default: return (int16_t)clamp(t, -128, 128 - 1); + } +} +#endif +#if CONFIG_PARALLEL_DEBLOCKING +// should we apply any filter at all: 11111111 yes, 00000000 no +static INLINE int8_t filter_mask2(uint8_t limit, uint8_t blimit, uint8_t p1, + uint8_t p0, uint8_t q0, uint8_t q1) { + int8_t mask = 0; + mask |= (abs(p1 - p0) > limit) * -1; + mask |= (abs(q1 - q0) > limit) * -1; + mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + return ~mask; +} +#endif // CONFIG_PARALLEL_DEBLOCKING +static INLINE int8_t filter_mask(uint8_t limit, uint8_t blimit, uint8_t p3, + uint8_t p2, uint8_t p1, uint8_t p0, uint8_t q0, + uint8_t q1, uint8_t q2, uint8_t q3) { + int8_t mask = 0; + mask |= (abs(p3 - p2) > limit) * -1; + mask |= (abs(p2 - p1) > limit) * -1; + mask |= (abs(p1 - p0) > limit) * -1; + mask |= (abs(q1 - q0) > limit) * -1; + mask |= (abs(q2 - q1) > limit) * -1; + mask |= (abs(q3 - q2) > limit) * -1; + mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + return ~mask; +} + +static INLINE int8_t flat_mask4(uint8_t thresh, uint8_t p3, uint8_t p2, + uint8_t p1, uint8_t p0, uint8_t q0, uint8_t q1, + uint8_t q2, uint8_t q3) { + int8_t mask = 0; + mask |= (abs(p1 - p0) > thresh) * -1; + mask |= (abs(q1 - q0) > thresh) * -1; + mask |= (abs(p2 - p0) > thresh) * -1; + mask |= (abs(q2 - q0) > thresh) * -1; + mask |= (abs(p3 - p0) > thresh) * -1; + mask |= (abs(q3 - q0) > thresh) * -1; + return ~mask; +} + +#if PARALLEL_DEBLOCKING_9_TAP +static INLINE int8_t flat_mask2(uint8_t thresh, uint8_t p4, uint8_t p0, + uint8_t q0, uint8_t q4) { + int8_t mask = 0; + mask |= (abs(p4 - p0) > thresh) * -1; + mask |= (abs(q4 - q0) > thresh) * -1; + return ~mask; +} +#endif + +#if PARALLEL_DEBLOCKING_11_TAP +static INLINE int8_t flat_mask3(uint8_t thresh, uint8_t p5, uint8_t p4, + uint8_t p0, uint8_t q0, uint8_t q4, + uint8_t q5) { + int8_t mask = 0; + mask |= (abs(p4 - p0) > thresh) * -1; + mask |= (abs(q4 - q0) > thresh) * -1; + mask |= (abs(p5 - p0) > thresh) * -1; + mask |= (abs(q5 - q0) > thresh) * -1; + return ~mask; +} +#endif + +static INLINE int8_t flat_mask5(uint8_t thresh, uint8_t p4, uint8_t p3, + uint8_t p2, uint8_t p1, uint8_t p0, uint8_t q0, + uint8_t q1, uint8_t q2, uint8_t q3, + uint8_t q4) { + int8_t mask = ~flat_mask4(thresh, p3, p2, p1, p0, q0, q1, q2, q3); + mask |= (abs(p4 - p0) > thresh) * -1; + mask |= (abs(q4 - q0) > thresh) * -1; + return ~mask; +} + +// is there high edge variance internal edge: 11111111 yes, 00000000 no +static INLINE int8_t hev_mask(uint8_t thresh, uint8_t p1, uint8_t p0, + uint8_t q0, uint8_t q1) { + int8_t hev = 0; + hev |= (abs(p1 - p0) > thresh) * -1; + hev |= (abs(q1 - q0) > thresh) * -1; + return hev; +} + +static INLINE void filter4(int8_t mask, uint8_t thresh, uint8_t *op1, + uint8_t *op0, uint8_t *oq0, uint8_t *oq1) { + int8_t filter1, filter2; + + const int8_t ps1 = (int8_t)*op1 ^ 0x80; + const int8_t ps0 = (int8_t)*op0 ^ 0x80; + const int8_t qs0 = (int8_t)*oq0 ^ 0x80; + const int8_t qs1 = (int8_t)*oq1 ^ 0x80; + const uint8_t hev = hev_mask(thresh, *op1, *op0, *oq0, *oq1); + + // add outer taps if we have high edge variance + int8_t filter = signed_char_clamp(ps1 - qs1) & hev; + + // inner taps + filter = signed_char_clamp(filter + 3 * (qs0 - ps0)) & mask; + + // save bottom 3 bits so that we round one side +4 and the other +3 + // if it equals 4 we'll set to adjust by -1 to account for the fact + // we'd round 3 the other way + filter1 = signed_char_clamp(filter + 4) >> 3; + filter2 = signed_char_clamp(filter + 3) >> 3; + + *oq0 = signed_char_clamp(qs0 - filter1) ^ 0x80; + *op0 = signed_char_clamp(ps0 + filter2) ^ 0x80; + + // outer tap adjustments + filter = ROUND_POWER_OF_TWO(filter1, 1) & ~hev; + + *oq1 = signed_char_clamp(qs1 - filter) ^ 0x80; + *op1 = signed_char_clamp(ps1 + filter) ^ 0x80; +} + +void aom_lpf_horizontal_4_c(uint8_t *s, int p /* pitch */, + const uint8_t *blimit, const uint8_t *limit, + const uint8_t *thresh) { + int i; + + // loop filter designed to work using chars so that we can make maximum use + // of 8 bit simd instructions. + for (i = 0; i < 8; ++i) { +#if !CONFIG_PARALLEL_DEBLOCKING + const uint8_t p3 = s[-4 * p], p2 = s[-3 * p], p1 = s[-2 * p], p0 = s[-p]; + const uint8_t q0 = s[0 * p], q1 = s[1 * p], q2 = s[2 * p], q3 = s[3 * p]; + const int8_t mask = + filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3); +#else // CONFIG_PARALLEL_DEBLOCKING + const uint8_t p1 = s[-2 * p], p0 = s[-p]; + const uint8_t q0 = s[0 * p], q1 = s[1 * p]; + const int8_t mask = filter_mask2(*limit, *blimit, p1, p0, q0, q1); +#endif // !CONFIG_PARALLEL_DEBLOCKING + filter4(mask, *thresh, s - 2 * p, s - 1 * p, s, s + 1 * p); + ++s; + } +} + +void aom_lpf_horizontal_4_dual_c(uint8_t *s, int p, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, + const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1) { + aom_lpf_horizontal_4_c(s, p, blimit0, limit0, thresh0); + aom_lpf_horizontal_4_c(s + 8, p, blimit1, limit1, thresh1); +} + +void aom_lpf_vertical_4_c(uint8_t *s, int pitch, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh) { + int i; + + // loop filter designed to work using chars so that we can make maximum use + // of 8 bit simd instructions. + for (i = 0; i < 8; ++i) { +#if !CONFIG_PARALLEL_DEBLOCKING + const uint8_t p3 = s[-4], p2 = s[-3], p1 = s[-2], p0 = s[-1]; + const uint8_t q0 = s[0], q1 = s[1], q2 = s[2], q3 = s[3]; + const int8_t mask = + filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3); +#else // CONFIG_PARALLEL_DEBLOCKING + const uint8_t p1 = s[-2], p0 = s[-1]; + const uint8_t q0 = s[0], q1 = s[1]; + const int8_t mask = filter_mask2(*limit, *blimit, p1, p0, q0, q1); +#endif // !CONFIG_PARALLEL_DEBLOCKING + filter4(mask, *thresh, s - 2, s - 1, s, s + 1); + s += pitch; + } +} + +void aom_lpf_vertical_4_dual_c(uint8_t *s, int pitch, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, + const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1) { + aom_lpf_vertical_4_c(s, pitch, blimit0, limit0, thresh0); + aom_lpf_vertical_4_c(s + 8 * pitch, pitch, blimit1, limit1, thresh1); +} + +static INLINE void filter8(int8_t mask, uint8_t thresh, uint8_t flat, + uint8_t *op3, uint8_t *op2, uint8_t *op1, + uint8_t *op0, uint8_t *oq0, uint8_t *oq1, + uint8_t *oq2, uint8_t *oq3) { + if (flat && mask) { + const uint8_t p3 = *op3, p2 = *op2, p1 = *op1, p0 = *op0; + const uint8_t q0 = *oq0, q1 = *oq1, q2 = *oq2, q3 = *oq3; + + // 7-tap filter [1, 1, 1, 2, 1, 1, 1] + *op2 = ROUND_POWER_OF_TWO(p3 + p3 + p3 + 2 * p2 + p1 + p0 + q0, 3); + *op1 = ROUND_POWER_OF_TWO(p3 + p3 + p2 + 2 * p1 + p0 + q0 + q1, 3); + *op0 = ROUND_POWER_OF_TWO(p3 + p2 + p1 + 2 * p0 + q0 + q1 + q2, 3); + *oq0 = ROUND_POWER_OF_TWO(p2 + p1 + p0 + 2 * q0 + q1 + q2 + q3, 3); + *oq1 = ROUND_POWER_OF_TWO(p1 + p0 + q0 + 2 * q1 + q2 + q3 + q3, 3); + *oq2 = ROUND_POWER_OF_TWO(p0 + q0 + q1 + 2 * q2 + q3 + q3 + q3, 3); + } else { + filter4(mask, thresh, op1, op0, oq0, oq1); + } +} + +void aom_lpf_horizontal_8_c(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh) { + int i; + + // loop filter designed to work using chars so that we can make maximum use + // of 8 bit simd instructions. + for (i = 0; i < 8; ++i) { + const uint8_t p3 = s[-4 * p], p2 = s[-3 * p], p1 = s[-2 * p], p0 = s[-p]; + const uint8_t q0 = s[0 * p], q1 = s[1 * p], q2 = s[2 * p], q3 = s[3 * p]; + + const int8_t mask = + filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3); + const int8_t flat = flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3); + filter8(mask, *thresh, flat, s - 4 * p, s - 3 * p, s - 2 * p, s - 1 * p, s, + s + 1 * p, s + 2 * p, s + 3 * p); + ++s; + } +} + +void aom_lpf_horizontal_8_dual_c(uint8_t *s, int p, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, + const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1) { + aom_lpf_horizontal_8_c(s, p, blimit0, limit0, thresh0); + aom_lpf_horizontal_8_c(s + 8, p, blimit1, limit1, thresh1); +} + +void aom_lpf_vertical_8_c(uint8_t *s, int pitch, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh) { + int i; + + for (i = 0; i < 8; ++i) { + const uint8_t p3 = s[-4], p2 = s[-3], p1 = s[-2], p0 = s[-1]; + const uint8_t q0 = s[0], q1 = s[1], q2 = s[2], q3 = s[3]; + const int8_t mask = + filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3); + const int8_t flat = flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3); + filter8(mask, *thresh, flat, s - 4, s - 3, s - 2, s - 1, s, s + 1, s + 2, + s + 3); + s += pitch; + } +} + +void aom_lpf_vertical_8_dual_c(uint8_t *s, int pitch, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, + const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1) { + aom_lpf_vertical_8_c(s, pitch, blimit0, limit0, thresh0); + aom_lpf_vertical_8_c(s + 8 * pitch, pitch, blimit1, limit1, thresh1); +} + +#if PARALLEL_DEBLOCKING_11_TAP +static INLINE void filter12(int8_t mask, uint8_t thresh, uint8_t flat, + uint8_t flat2, uint8_t *op5, uint8_t *op4, + uint8_t *op3, uint8_t *op2, uint8_t *op1, + uint8_t *op0, uint8_t *oq0, uint8_t *oq1, + uint8_t *oq2, uint8_t *oq3, uint8_t *oq4, + uint8_t *oq5) { + if (flat2 && flat && mask) { + const uint8_t p5 = *op5, p4 = *op4, p3 = *op3, p2 = *op2, p1 = *op1, + p0 = *op0; + const uint8_t q0 = *oq0, q1 = *oq1, q2 = *oq2, q3 = *oq3, q4 = *oq4, + q5 = *oq5; + + // 11-tap filter [1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1] + *op4 = (p5 * 5 + p4 * 2 + p3 + p2 + p1 + p0 + q0 + 6) / 12; + *op3 = (p5 * 4 + p4 + p3 * 2 + p2 + p1 + p0 + q0 + q1 + 6) / 12; + *op2 = (p5 * 3 + p4 + p3 + p2 * 2 + p1 + p0 + q0 + q1 + q2 + 6) / 12; + *op1 = (p5 * 2 + p4 + p3 + p2 + p1 * 2 + p0 + q0 + q1 + q2 + q3 + 6) / 12; + *op0 = (p5 + p4 + p3 + p2 + p1 + p0 * 2 + q0 + q1 + q2 + q3 + q4 + 6) / 12; + *oq0 = (p4 + p3 + p2 + p1 + p0 + q0 * 2 + q1 + q2 + q3 + q4 + q5 + 6) / 12; + *oq1 = (p3 + p2 + p1 + p0 + q0 + q1 * 2 + q2 + q3 + q4 + q5 * 2 + 6) / 12; + *oq2 = (p2 + p1 + p0 + q0 + q1 + q2 * 2 + q3 + q4 + q5 * 3 + 6) / 12; + *oq3 = (p1 + p0 + q0 + q1 + q2 + q3 * 2 + q4 + q5 * 4 + 6) / 12; + *oq4 = (p0 + q0 + q1 + q2 + q3 + q4 * 2 + q5 * 5 + 6) / 12; + } else { + filter8(mask, thresh, flat, op3, op2, op1, op0, oq0, oq1, oq2, oq3); + } +} +#endif + +#if PARALLEL_DEBLOCKING_9_TAP +static INLINE void filter10(int8_t mask, uint8_t thresh, uint8_t flat, + uint8_t flat2, uint8_t *op4, uint8_t *op3, + uint8_t *op2, uint8_t *op1, uint8_t *op0, + uint8_t *oq0, uint8_t *oq1, uint8_t *oq2, + uint8_t *oq3, uint8_t *oq4) { + if (flat2 && flat && mask) { + const uint8_t p4 = *op4, p3 = *op3, p2 = *op2, p1 = *op1, p0 = *op0; + const uint8_t q0 = *oq0, q1 = *oq1, q2 = *oq2, q3 = *oq3, q4 = *oq4; + + // 9-tap filter [1, 1, 1, 1, 2, 1, 1, 1, 1] + *op3 = (p4 * 4 + p3 * 2 + p2 + p1 + p0 + q0 + 5) / 10; + *op2 = (p4 * 3 + p3 + p2 * 2 + p1 + p0 + q0 + q1 + 5) / 10; + *op1 = (p4 * 2 + p3 + p2 + p1 * 2 + p0 + q0 + q1 + q2 + 5) / 10; + *op0 = (p4 + p3 + p2 + p1 + p0 * 2 + q0 + q1 + q2 + q3 + 5) / 10; + *oq0 = (p3 + p2 + p1 + p0 + q0 * 2 + q1 + q2 + q3 + q4 + 5) / 10; + *oq1 = (p2 + p1 + p0 + q0 + q1 * 2 + q2 + q3 + q4 * 2 + 5) / 10; + *oq2 = (p1 + p0 + q0 + q1 + q2 * 2 + q3 + q4 * 3 + 5) / 10; + *oq3 = (p0 + q0 + q1 + q2 + q3 * 2 + q4 * 4 + 5) / 10; + } else { + filter8(mask, thresh, flat, op3, op2, op1, op0, oq0, oq1, oq2, oq3); + } +} +#endif + +static INLINE void filter16(int8_t mask, uint8_t thresh, uint8_t flat, + uint8_t flat2, uint8_t *op7, uint8_t *op6, + uint8_t *op5, uint8_t *op4, uint8_t *op3, + uint8_t *op2, uint8_t *op1, uint8_t *op0, + uint8_t *oq0, uint8_t *oq1, uint8_t *oq2, + uint8_t *oq3, uint8_t *oq4, uint8_t *oq5, + uint8_t *oq6, uint8_t *oq7) { + if (flat2 && flat && mask) { + const uint8_t p7 = *op7, p6 = *op6, p5 = *op5, p4 = *op4, p3 = *op3, + p2 = *op2, p1 = *op1, p0 = *op0; + + const uint8_t q0 = *oq0, q1 = *oq1, q2 = *oq2, q3 = *oq3, q4 = *oq4, + q5 = *oq5, q6 = *oq6, q7 = *oq7; + + // 15-tap filter [1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1] + *op6 = ROUND_POWER_OF_TWO( + p7 * 7 + p6 * 2 + p5 + p4 + p3 + p2 + p1 + p0 + q0, 4); + *op5 = ROUND_POWER_OF_TWO( + p7 * 6 + p6 + p5 * 2 + p4 + p3 + p2 + p1 + p0 + q0 + q1, 4); + *op4 = ROUND_POWER_OF_TWO( + p7 * 5 + p6 + p5 + p4 * 2 + p3 + p2 + p1 + p0 + q0 + q1 + q2, 4); + *op3 = ROUND_POWER_OF_TWO( + p7 * 4 + p6 + p5 + p4 + p3 * 2 + p2 + p1 + p0 + q0 + q1 + q2 + q3, 4); + *op2 = ROUND_POWER_OF_TWO( + p7 * 3 + p6 + p5 + p4 + p3 + p2 * 2 + p1 + p0 + q0 + q1 + q2 + q3 + q4, + 4); + *op1 = ROUND_POWER_OF_TWO(p7 * 2 + p6 + p5 + p4 + p3 + p2 + p1 * 2 + p0 + + q0 + q1 + q2 + q3 + q4 + q5, + 4); + *op0 = ROUND_POWER_OF_TWO(p7 + p6 + p5 + p4 + p3 + p2 + p1 + p0 * 2 + q0 + + q1 + q2 + q3 + q4 + q5 + q6, + 4); + *oq0 = ROUND_POWER_OF_TWO(p6 + p5 + p4 + p3 + p2 + p1 + p0 + q0 * 2 + q1 + + q2 + q3 + q4 + q5 + q6 + q7, + 4); + *oq1 = ROUND_POWER_OF_TWO(p5 + p4 + p3 + p2 + p1 + p0 + q0 + q1 * 2 + q2 + + q3 + q4 + q5 + q6 + q7 * 2, + 4); + *oq2 = ROUND_POWER_OF_TWO( + p4 + p3 + p2 + p1 + p0 + q0 + q1 + q2 * 2 + q3 + q4 + q5 + q6 + q7 * 3, + 4); + *oq3 = ROUND_POWER_OF_TWO( + p3 + p2 + p1 + p0 + q0 + q1 + q2 + q3 * 2 + q4 + q5 + q6 + q7 * 4, 4); + *oq4 = ROUND_POWER_OF_TWO( + p2 + p1 + p0 + q0 + q1 + q2 + q3 + q4 * 2 + q5 + q6 + q7 * 5, 4); + *oq5 = ROUND_POWER_OF_TWO( + p1 + p0 + q0 + q1 + q2 + q3 + q4 + q5 * 2 + q6 + q7 * 6, 4); + *oq6 = ROUND_POWER_OF_TWO( + p0 + q0 + q1 + q2 + q3 + q4 + q5 + q6 * 2 + q7 * 7, 4); + } else { + filter8(mask, thresh, flat, op3, op2, op1, op0, oq0, oq1, oq2, oq3); + } +} + +static void mb_lpf_horizontal_edge_w(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, int count) { + int i; + + // loop filter designed to work using chars so that we can make maximum use + // of 8 bit simd instructions. + for (i = 0; i < 8 * count; ++i) { + const uint8_t p7 = s[-8 * p], p6 = s[-7 * p], p5 = s[-6 * p], + p4 = s[-5 * p], p3 = s[-4 * p], p2 = s[-3 * p], + p1 = s[-2 * p], p0 = s[-p]; + const uint8_t q0 = s[0 * p], q1 = s[1 * p], q2 = s[2 * p], q3 = s[3 * p], + q4 = s[4 * p], q5 = s[5 * p], q6 = s[6 * p], q7 = s[7 * p]; + const int8_t mask = + filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3); + const int8_t flat = flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3); + +#if PARALLEL_DEBLOCKING_11_TAP + const int8_t flat2 = flat_mask3(1, p5, p4, p0, q0, q4, q5); + + filter12(mask, *thresh, flat, flat2, s - 6 * p, s - 5 * p, s - 4 * p, + s - 3 * p, s - 2 * p, s - 1 * p, s, s + 1 * p, s + 2 * p, + s + 3 * p, s + 4 * p, s + 5 * p); + +#elif PARALLEL_DEBLOCKING_9_TAP + const int8_t flat2 = flat_mask2(1, p4, p0, q0, q4); + + filter10(mask, *thresh, flat, flat2, s - 5 * p, s - 4 * p, s - 3 * p, + s - 2 * p, s - 1 * p, s, s + 1 * p, s + 2 * p, s + 3 * p, + s + 4 * p); +#else + const int8_t flat2 = flat_mask5(1, p7, p6, p5, p4, p0, q0, q4, q5, q6, q7); + + filter16(mask, *thresh, flat, flat2, s - 8 * p, s - 7 * p, s - 6 * p, + s - 5 * p, s - 4 * p, s - 3 * p, s - 2 * p, s - 1 * p, s, + s + 1 * p, s + 2 * p, s + 3 * p, s + 4 * p, s + 5 * p, s + 6 * p, + s + 7 * p); +#endif + + ++s; + } +} + +void aom_lpf_horizontal_edge_8_c(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh) { + mb_lpf_horizontal_edge_w(s, p, blimit, limit, thresh, 1); +} + +void aom_lpf_horizontal_edge_16_c(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh) { + mb_lpf_horizontal_edge_w(s, p, blimit, limit, thresh, 2); +} + +static void mb_lpf_vertical_edge_w(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count) { + int i; + + for (i = 0; i < count; ++i) { + const uint8_t p7 = s[-8], p6 = s[-7], p5 = s[-6], p4 = s[-5], p3 = s[-4], + p2 = s[-3], p1 = s[-2], p0 = s[-1]; + const uint8_t q0 = s[0], q1 = s[1], q2 = s[2], q3 = s[3], q4 = s[4], + q5 = s[5], q6 = s[6], q7 = s[7]; + const int8_t mask = + filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3); + const int8_t flat = flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3); + +#if PARALLEL_DEBLOCKING_11_TAP + const int8_t flat2 = flat_mask3(1, p5, p4, p0, q0, q4, q5); + + filter12(mask, *thresh, flat, flat2, s - 6, s - 5, s - 4, s - 3, s - 2, + s - 1, s, s + 1, s + 2, s + 3, s + 4, s + 5); +#elif PARALLEL_DEBLOCKING_9_TAP + const int8_t flat2 = flat_mask2(1, p4, p0, q0, q4); + + filter10(mask, *thresh, flat, flat2, s - 5, s - 4, s - 3, s - 2, s - 1, s, + s + 1, s + 2, s + 3, s + 4); + +#else + const int8_t flat2 = flat_mask5(1, p7, p6, p5, p4, p0, q0, q4, q5, q6, q7); + + filter16(mask, *thresh, flat, flat2, s - 8, s - 7, s - 6, s - 5, s - 4, + s - 3, s - 2, s - 1, s, s + 1, s + 2, s + 3, s + 4, s + 5, s + 6, + s + 7); +#endif + + s += p; + } +} + +void aom_lpf_vertical_16_c(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh) { + mb_lpf_vertical_edge_w(s, p, blimit, limit, thresh, 8); +} + +void aom_lpf_vertical_16_dual_c(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh) { + mb_lpf_vertical_edge_w(s, p, blimit, limit, thresh, 16); +} + +#if CONFIG_HIGHBITDEPTH +#if CONFIG_PARALLEL_DEBLOCKING +// Should we apply any filter at all: 11111111 yes, 00000000 no ? +static INLINE int8_t highbd_filter_mask2(uint8_t limit, uint8_t blimit, + uint16_t p1, uint16_t p0, uint16_t q0, + uint16_t q1, int bd) { + int8_t mask = 0; + int16_t limit16 = (uint16_t)limit << (bd - 8); + int16_t blimit16 = (uint16_t)blimit << (bd - 8); + mask |= (abs(p1 - p0) > limit16) * -1; + mask |= (abs(q1 - q0) > limit16) * -1; + mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit16) * -1; + return ~mask; +} +#endif // CONFIG_PARALLEL_DEBLOCKING + +// Should we apply any filter at all: 11111111 yes, 00000000 no ? +static INLINE int8_t highbd_filter_mask(uint8_t limit, uint8_t blimit, + uint16_t p3, uint16_t p2, uint16_t p1, + uint16_t p0, uint16_t q0, uint16_t q1, + uint16_t q2, uint16_t q3, int bd) { + int8_t mask = 0; + int16_t limit16 = (uint16_t)limit << (bd - 8); + int16_t blimit16 = (uint16_t)blimit << (bd - 8); + mask |= (abs(p3 - p2) > limit16) * -1; + mask |= (abs(p2 - p1) > limit16) * -1; + mask |= (abs(p1 - p0) > limit16) * -1; + mask |= (abs(q1 - q0) > limit16) * -1; + mask |= (abs(q2 - q1) > limit16) * -1; + mask |= (abs(q3 - q2) > limit16) * -1; + mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit16) * -1; + return ~mask; +} + +static INLINE int8_t highbd_flat_mask4(uint8_t thresh, uint16_t p3, uint16_t p2, + uint16_t p1, uint16_t p0, uint16_t q0, + uint16_t q1, uint16_t q2, uint16_t q3, + int bd) { + int8_t mask = 0; + int16_t thresh16 = (uint16_t)thresh << (bd - 8); + mask |= (abs(p1 - p0) > thresh16) * -1; + mask |= (abs(q1 - q0) > thresh16) * -1; + mask |= (abs(p2 - p0) > thresh16) * -1; + mask |= (abs(q2 - q0) > thresh16) * -1; + mask |= (abs(p3 - p0) > thresh16) * -1; + mask |= (abs(q3 - q0) > thresh16) * -1; + return ~mask; +} + +static INLINE int8_t highbd_flat_mask5(uint8_t thresh, uint16_t p4, uint16_t p3, + uint16_t p2, uint16_t p1, uint16_t p0, + uint16_t q0, uint16_t q1, uint16_t q2, + uint16_t q3, uint16_t q4, int bd) { + int8_t mask = ~highbd_flat_mask4(thresh, p3, p2, p1, p0, q0, q1, q2, q3, bd); + int16_t thresh16 = (uint16_t)thresh << (bd - 8); + mask |= (abs(p4 - p0) > thresh16) * -1; + mask |= (abs(q4 - q0) > thresh16) * -1; + return ~mask; +} + +// Is there high edge variance internal edge: +// 11111111_11111111 yes, 00000000_00000000 no ? +static INLINE int16_t highbd_hev_mask(uint8_t thresh, uint16_t p1, uint16_t p0, + uint16_t q0, uint16_t q1, int bd) { + int16_t hev = 0; + int16_t thresh16 = (uint16_t)thresh << (bd - 8); + hev |= (abs(p1 - p0) > thresh16) * -1; + hev |= (abs(q1 - q0) > thresh16) * -1; + return hev; +} + +static INLINE void highbd_filter4(int8_t mask, uint8_t thresh, uint16_t *op1, + uint16_t *op0, uint16_t *oq0, uint16_t *oq1, + int bd) { + int16_t filter1, filter2; + // ^0x80 equivalent to subtracting 0x80 from the values to turn them + // into -128 to +127 instead of 0 to 255. + int shift = bd - 8; + const int16_t ps1 = (int16_t)*op1 - (0x80 << shift); + const int16_t ps0 = (int16_t)*op0 - (0x80 << shift); + const int16_t qs0 = (int16_t)*oq0 - (0x80 << shift); + const int16_t qs1 = (int16_t)*oq1 - (0x80 << shift); + const uint16_t hev = highbd_hev_mask(thresh, *op1, *op0, *oq0, *oq1, bd); + + // Add outer taps if we have high edge variance. + int16_t filter = signed_char_clamp_high(ps1 - qs1, bd) & hev; + + // Inner taps. + filter = signed_char_clamp_high(filter + 3 * (qs0 - ps0), bd) & mask; + + // Save bottom 3 bits so that we round one side +4 and the other +3 + // if it equals 4 we'll set to adjust by -1 to account for the fact + // we'd round 3 the other way. + filter1 = signed_char_clamp_high(filter + 4, bd) >> 3; + filter2 = signed_char_clamp_high(filter + 3, bd) >> 3; + + *oq0 = signed_char_clamp_high(qs0 - filter1, bd) + (0x80 << shift); + *op0 = signed_char_clamp_high(ps0 + filter2, bd) + (0x80 << shift); + + // Outer tap adjustments. + filter = ROUND_POWER_OF_TWO(filter1, 1) & ~hev; + + *oq1 = signed_char_clamp_high(qs1 - filter, bd) + (0x80 << shift); + *op1 = signed_char_clamp_high(ps1 + filter, bd) + (0x80 << shift); +} + +void aom_highbd_lpf_horizontal_4_c(uint16_t *s, int p /* pitch */, + const uint8_t *blimit, const uint8_t *limit, + const uint8_t *thresh, int bd) { + int i; + + // loop filter designed to work using chars so that we can make maximum use + // of 8 bit simd instructions. + for (i = 0; i < 8; ++i) { +#if !CONFIG_PARALLEL_DEBLOCKING + const uint16_t p3 = s[-4 * p]; + const uint16_t p2 = s[-3 * p]; + const uint16_t p1 = s[-2 * p]; + const uint16_t p0 = s[-p]; + const uint16_t q0 = s[0 * p]; + const uint16_t q1 = s[1 * p]; + const uint16_t q2 = s[2 * p]; + const uint16_t q3 = s[3 * p]; + const int8_t mask = + highbd_filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3, bd); +#else // CONFIG_PARALLEL_DEBLOCKING + const uint16_t p1 = s[-2 * p]; + const uint16_t p0 = s[-p]; + const uint16_t q0 = s[0 * p]; + const uint16_t q1 = s[1 * p]; + const int8_t mask = + highbd_filter_mask2(*limit, *blimit, p1, p0, q0, q1, bd); +#endif // !CONFIG_PARALLEL_DEBLOCKING + highbd_filter4(mask, *thresh, s - 2 * p, s - 1 * p, s, s + 1 * p, bd); + ++s; + } +} + +void aom_highbd_lpf_horizontal_4_dual_c( + uint16_t *s, int p, const uint8_t *blimit0, const uint8_t *limit0, + const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1, int bd) { + aom_highbd_lpf_horizontal_4_c(s, p, blimit0, limit0, thresh0, bd); + aom_highbd_lpf_horizontal_4_c(s + 8, p, blimit1, limit1, thresh1, bd); +} + +void aom_highbd_lpf_vertical_4_c(uint16_t *s, int pitch, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int bd) { + int i; + + // loop filter designed to work using chars so that we can make maximum use + // of 8 bit simd instructions. + for (i = 0; i < 8; ++i) { +#if !CONFIG_PARALLEL_DEBLOCKING + const uint16_t p3 = s[-4], p2 = s[-3], p1 = s[-2], p0 = s[-1]; + const uint16_t q0 = s[0], q1 = s[1], q2 = s[2], q3 = s[3]; + const int8_t mask = + highbd_filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3, bd); +#else // CONFIG_PARALLEL_DEBLOCKING + const uint16_t p1 = s[-2], p0 = s[-1]; + const uint16_t q0 = s[0], q1 = s[1]; + const int8_t mask = + highbd_filter_mask2(*limit, *blimit, p1, p0, q0, q1, bd); +#endif // !CONFIG_PARALLEL_DEBLOCKING + highbd_filter4(mask, *thresh, s - 2, s - 1, s, s + 1, bd); + s += pitch; + } +} + +void aom_highbd_lpf_vertical_4_dual_c( + uint16_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, + const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1, int bd) { + aom_highbd_lpf_vertical_4_c(s, pitch, blimit0, limit0, thresh0, bd); + aom_highbd_lpf_vertical_4_c(s + 8 * pitch, pitch, blimit1, limit1, thresh1, + bd); +} + +static INLINE void highbd_filter8(int8_t mask, uint8_t thresh, uint8_t flat, + uint16_t *op3, uint16_t *op2, uint16_t *op1, + uint16_t *op0, uint16_t *oq0, uint16_t *oq1, + uint16_t *oq2, uint16_t *oq3, int bd) { + if (flat && mask) { + const uint16_t p3 = *op3, p2 = *op2, p1 = *op1, p0 = *op0; + const uint16_t q0 = *oq0, q1 = *oq1, q2 = *oq2, q3 = *oq3; + + // 7-tap filter [1, 1, 1, 2, 1, 1, 1] + *op2 = ROUND_POWER_OF_TWO(p3 + p3 + p3 + 2 * p2 + p1 + p0 + q0, 3); + *op1 = ROUND_POWER_OF_TWO(p3 + p3 + p2 + 2 * p1 + p0 + q0 + q1, 3); + *op0 = ROUND_POWER_OF_TWO(p3 + p2 + p1 + 2 * p0 + q0 + q1 + q2, 3); + *oq0 = ROUND_POWER_OF_TWO(p2 + p1 + p0 + 2 * q0 + q1 + q2 + q3, 3); + *oq1 = ROUND_POWER_OF_TWO(p1 + p0 + q0 + 2 * q1 + q2 + q3 + q3, 3); + *oq2 = ROUND_POWER_OF_TWO(p0 + q0 + q1 + 2 * q2 + q3 + q3 + q3, 3); + } else { + highbd_filter4(mask, thresh, op1, op0, oq0, oq1, bd); + } +} + +void aom_highbd_lpf_horizontal_8_c(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int bd) { + int i; + + // loop filter designed to work using chars so that we can make maximum use + // of 8 bit simd instructions. + for (i = 0; i < 8; ++i) { + const uint16_t p3 = s[-4 * p], p2 = s[-3 * p], p1 = s[-2 * p], p0 = s[-p]; + const uint16_t q0 = s[0 * p], q1 = s[1 * p], q2 = s[2 * p], q3 = s[3 * p]; + + const int8_t mask = + highbd_filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3, bd); + const int8_t flat = + highbd_flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3, bd); + highbd_filter8(mask, *thresh, flat, s - 4 * p, s - 3 * p, s - 2 * p, + s - 1 * p, s, s + 1 * p, s + 2 * p, s + 3 * p, bd); + ++s; + } +} + +void aom_highbd_lpf_horizontal_8_dual_c( + uint16_t *s, int p, const uint8_t *blimit0, const uint8_t *limit0, + const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1, int bd) { + aom_highbd_lpf_horizontal_8_c(s, p, blimit0, limit0, thresh0, bd); + aom_highbd_lpf_horizontal_8_c(s + 8, p, blimit1, limit1, thresh1, bd); +} + +void aom_highbd_lpf_vertical_8_c(uint16_t *s, int pitch, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int bd) { + int i; + + for (i = 0; i < 8; ++i) { + const uint16_t p3 = s[-4], p2 = s[-3], p1 = s[-2], p0 = s[-1]; + const uint16_t q0 = s[0], q1 = s[1], q2 = s[2], q3 = s[3]; + const int8_t mask = + highbd_filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3, bd); + const int8_t flat = + highbd_flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3, bd); + highbd_filter8(mask, *thresh, flat, s - 4, s - 3, s - 2, s - 1, s, s + 1, + s + 2, s + 3, bd); + s += pitch; + } +} + +void aom_highbd_lpf_vertical_8_dual_c( + uint16_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, + const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1, int bd) { + aom_highbd_lpf_vertical_8_c(s, pitch, blimit0, limit0, thresh0, bd); + aom_highbd_lpf_vertical_8_c(s + 8 * pitch, pitch, blimit1, limit1, thresh1, + bd); +} + +static INLINE void highbd_filter16(int8_t mask, uint8_t thresh, uint8_t flat, + uint8_t flat2, uint16_t *op7, uint16_t *op6, + uint16_t *op5, uint16_t *op4, uint16_t *op3, + uint16_t *op2, uint16_t *op1, uint16_t *op0, + uint16_t *oq0, uint16_t *oq1, uint16_t *oq2, + uint16_t *oq3, uint16_t *oq4, uint16_t *oq5, + uint16_t *oq6, uint16_t *oq7, int bd) { + if (flat2 && flat && mask) { + const uint16_t p7 = *op7; + const uint16_t p6 = *op6; + const uint16_t p5 = *op5; + const uint16_t p4 = *op4; + const uint16_t p3 = *op3; + const uint16_t p2 = *op2; + const uint16_t p1 = *op1; + const uint16_t p0 = *op0; + const uint16_t q0 = *oq0; + const uint16_t q1 = *oq1; + const uint16_t q2 = *oq2; + const uint16_t q3 = *oq3; + const uint16_t q4 = *oq4; + const uint16_t q5 = *oq5; + const uint16_t q6 = *oq6; + const uint16_t q7 = *oq7; + + // 15-tap filter [1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1] + *op6 = ROUND_POWER_OF_TWO( + p7 * 7 + p6 * 2 + p5 + p4 + p3 + p2 + p1 + p0 + q0, 4); + *op5 = ROUND_POWER_OF_TWO( + p7 * 6 + p6 + p5 * 2 + p4 + p3 + p2 + p1 + p0 + q0 + q1, 4); + *op4 = ROUND_POWER_OF_TWO( + p7 * 5 + p6 + p5 + p4 * 2 + p3 + p2 + p1 + p0 + q0 + q1 + q2, 4); + *op3 = ROUND_POWER_OF_TWO( + p7 * 4 + p6 + p5 + p4 + p3 * 2 + p2 + p1 + p0 + q0 + q1 + q2 + q3, 4); + *op2 = ROUND_POWER_OF_TWO( + p7 * 3 + p6 + p5 + p4 + p3 + p2 * 2 + p1 + p0 + q0 + q1 + q2 + q3 + q4, + 4); + *op1 = ROUND_POWER_OF_TWO(p7 * 2 + p6 + p5 + p4 + p3 + p2 + p1 * 2 + p0 + + q0 + q1 + q2 + q3 + q4 + q5, + 4); + *op0 = ROUND_POWER_OF_TWO(p7 + p6 + p5 + p4 + p3 + p2 + p1 + p0 * 2 + q0 + + q1 + q2 + q3 + q4 + q5 + q6, + 4); + *oq0 = ROUND_POWER_OF_TWO(p6 + p5 + p4 + p3 + p2 + p1 + p0 + q0 * 2 + q1 + + q2 + q3 + q4 + q5 + q6 + q7, + 4); + *oq1 = ROUND_POWER_OF_TWO(p5 + p4 + p3 + p2 + p1 + p0 + q0 + q1 * 2 + q2 + + q3 + q4 + q5 + q6 + q7 * 2, + 4); + *oq2 = ROUND_POWER_OF_TWO( + p4 + p3 + p2 + p1 + p0 + q0 + q1 + q2 * 2 + q3 + q4 + q5 + q6 + q7 * 3, + 4); + *oq3 = ROUND_POWER_OF_TWO( + p3 + p2 + p1 + p0 + q0 + q1 + q2 + q3 * 2 + q4 + q5 + q6 + q7 * 4, 4); + *oq4 = ROUND_POWER_OF_TWO( + p2 + p1 + p0 + q0 + q1 + q2 + q3 + q4 * 2 + q5 + q6 + q7 * 5, 4); + *oq5 = ROUND_POWER_OF_TWO( + p1 + p0 + q0 + q1 + q2 + q3 + q4 + q5 * 2 + q6 + q7 * 6, 4); + *oq6 = ROUND_POWER_OF_TWO( + p0 + q0 + q1 + q2 + q3 + q4 + q5 + q6 * 2 + q7 * 7, 4); + } else { + highbd_filter8(mask, thresh, flat, op3, op2, op1, op0, oq0, oq1, oq2, oq3, + bd); + } +} + +static void highbd_mb_lpf_horizontal_edge_w(uint16_t *s, int p, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, int count, + int bd) { + int i; + + // loop filter designed to work using chars so that we can make maximum use + // of 8 bit simd instructions. + for (i = 0; i < 8 * count; ++i) { + const uint16_t p3 = s[-4 * p]; + const uint16_t p2 = s[-3 * p]; + const uint16_t p1 = s[-2 * p]; + const uint16_t p0 = s[-p]; + const uint16_t q0 = s[0 * p]; + const uint16_t q1 = s[1 * p]; + const uint16_t q2 = s[2 * p]; + const uint16_t q3 = s[3 * p]; + const int8_t mask = + highbd_filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3, bd); + const int8_t flat = + highbd_flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3, bd); + const int8_t flat2 = + highbd_flat_mask5(1, s[-8 * p], s[-7 * p], s[-6 * p], s[-5 * p], p0, q0, + s[4 * p], s[5 * p], s[6 * p], s[7 * p], bd); + + highbd_filter16(mask, *thresh, flat, flat2, s - 8 * p, s - 7 * p, s - 6 * p, + s - 5 * p, s - 4 * p, s - 3 * p, s - 2 * p, s - 1 * p, s, + s + 1 * p, s + 2 * p, s + 3 * p, s + 4 * p, s + 5 * p, + s + 6 * p, s + 7 * p, bd); + ++s; + } +} + +void aom_highbd_lpf_horizontal_edge_8_c(uint16_t *s, int p, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, int bd) { + highbd_mb_lpf_horizontal_edge_w(s, p, blimit, limit, thresh, 1, bd); +} + +void aom_highbd_lpf_horizontal_edge_16_c(uint16_t *s, int p, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, int bd) { + highbd_mb_lpf_horizontal_edge_w(s, p, blimit, limit, thresh, 2, bd); +} + +static void highbd_mb_lpf_vertical_edge_w(uint16_t *s, int p, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, int count, + int bd) { + int i; + + for (i = 0; i < count; ++i) { + const uint16_t p3 = s[-4]; + const uint16_t p2 = s[-3]; + const uint16_t p1 = s[-2]; + const uint16_t p0 = s[-1]; + const uint16_t q0 = s[0]; + const uint16_t q1 = s[1]; + const uint16_t q2 = s[2]; + const uint16_t q3 = s[3]; + const int8_t mask = + highbd_filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3, bd); + const int8_t flat = + highbd_flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3, bd); + const int8_t flat2 = highbd_flat_mask5(1, s[-8], s[-7], s[-6], s[-5], p0, + q0, s[4], s[5], s[6], s[7], bd); + + highbd_filter16(mask, *thresh, flat, flat2, s - 8, s - 7, s - 6, s - 5, + s - 4, s - 3, s - 2, s - 1, s, s + 1, s + 2, s + 3, s + 4, + s + 5, s + 6, s + 7, bd); + s += p; + } +} + +void aom_highbd_lpf_vertical_16_c(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int bd) { + highbd_mb_lpf_vertical_edge_w(s, p, blimit, limit, thresh, 8, bd); +} + +void aom_highbd_lpf_vertical_16_dual_c(uint16_t *s, int p, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, int bd) { + highbd_mb_lpf_vertical_edge_w(s, p, blimit, limit, thresh, 16, bd); +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/mips/add_noise_msa.c b/third_party/aom/aom_dsp/mips/add_noise_msa.c new file mode 100644 index 0000000000..4c6e201e11 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/add_noise_msa.c @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./macros_msa.h" + +void aom_plane_add_noise_msa(uint8_t *start_ptr, char *noise, + char blackclamp[16], char whiteclamp[16], + char bothclamp[16], uint32_t width, + uint32_t height, int32_t pitch) { + uint32_t i, j; + + for (i = 0; i < height / 2; ++i) { + uint8_t *pos0_ptr = start_ptr + (2 * i) * pitch; + int8_t *ref0_ptr = (int8_t *)(noise + (rand() & 0xff)); + uint8_t *pos1_ptr = start_ptr + (2 * i + 1) * pitch; + int8_t *ref1_ptr = (int8_t *)(noise + (rand() & 0xff)); + for (j = width / 16; j--;) { + v16i8 temp00_s, temp01_s; + v16u8 temp00, temp01, black_clamp, white_clamp; + v16u8 pos0, ref0, pos1, ref1; + v16i8 const127 = __msa_ldi_b(127); + + pos0 = LD_UB(pos0_ptr); + ref0 = LD_UB(ref0_ptr); + pos1 = LD_UB(pos1_ptr); + ref1 = LD_UB(ref1_ptr); + black_clamp = (v16u8)__msa_fill_b(blackclamp[0]); + white_clamp = (v16u8)__msa_fill_b(whiteclamp[0]); + temp00 = (pos0 < black_clamp); + pos0 = __msa_bmnz_v(pos0, black_clamp, temp00); + temp01 = (pos1 < black_clamp); + pos1 = __msa_bmnz_v(pos1, black_clamp, temp01); + XORI_B2_128_UB(pos0, pos1); + temp00_s = __msa_adds_s_b((v16i8)white_clamp, const127); + temp00 = (v16u8)(temp00_s < pos0); + pos0 = (v16u8)__msa_bmnz_v((v16u8)pos0, (v16u8)temp00_s, temp00); + temp01_s = __msa_adds_s_b((v16i8)white_clamp, const127); + temp01 = (temp01_s < pos1); + pos1 = (v16u8)__msa_bmnz_v((v16u8)pos1, (v16u8)temp01_s, temp01); + XORI_B2_128_UB(pos0, pos1); + pos0 += ref0; + ST_UB(pos0, pos0_ptr); + pos1 += ref1; + ST_UB(pos1, pos1_ptr); + pos0_ptr += 16; + pos1_ptr += 16; + ref0_ptr += 16; + ref1_ptr += 16; + } + } +} diff --git a/third_party/aom/aom_dsp/mips/aom_convolve8_avg_horiz_msa.c b/third_party/aom/aom_dsp/mips/aom_convolve8_avg_horiz_msa.c new file mode 100644 index 0000000000..847394a3d2 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/aom_convolve8_avg_horiz_msa.c @@ -0,0 +1,704 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/aom_convolve_msa.h" + +static void common_hz_8t_and_aver_dst_4x4_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3; + v16u8 dst0, dst1, dst2, dst3, res2, res3; + v16u8 mask0, mask1, mask2, mask3; + v8i16 filt, res0, res1; + + mask0 = LD_UB(&mc_filt_mask_arr[16]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + LD_SB4(src, src_stride, src0, src1, src2, src3); + XORI_B4_128_SB(src0, src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filt0, filt1, filt2, filt3, res0, res1); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + SRARI_H2_SH(res0, res1, FILTER_BITS); + SAT_SH2_SH(res0, res1, 7); + PCKEV_B2_UB(res0, res0, res1, res1, res2, res3); + ILVR_W2_UB(dst1, dst0, dst3, dst2, dst0, dst2); + XORI_B2_128_UB(res2, res3); + AVER_UB2_UB(res2, dst0, res3, dst2, res2, res3); + ST4x4_UB(res2, res3, 0, 1, 0, 1, dst, dst_stride); +} + +static void common_hz_8t_and_aver_dst_4x8_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3; + v16u8 mask0, mask1, mask2, mask3, res0, res1, res2, res3; + v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + v8i16 filt, vec0, vec1, vec2, vec3; + + mask0 = LD_UB(&mc_filt_mask_arr[16]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + LD_SB4(src, src_stride, src0, src1, src2, src3); + XORI_B4_128_SB(src0, src1, src2, src3); + src += (4 * src_stride); + LD_UB8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filt0, filt1, filt2, filt3, vec0, vec1); + LD_SB4(src, src_stride, src0, src1, src2, src3); + XORI_B4_128_SB(src0, src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filt0, filt1, filt2, filt3, vec2, vec3); + SRARI_H4_SH(vec0, vec1, vec2, vec3, FILTER_BITS); + SAT_SH4_SH(vec0, vec1, vec2, vec3, 7); + PCKEV_B4_UB(vec0, vec0, vec1, vec1, vec2, vec2, vec3, vec3, res0, res1, res2, + res3); + ILVR_D2_UB(res1, res0, res3, res2, res0, res2); + XORI_B2_128_UB(res0, res2); + ILVR_W4_UB(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, dst0, dst2, dst4, + dst6); + ILVR_D2_UB(dst2, dst0, dst6, dst4, dst0, dst4); + AVER_UB2_UB(res0, dst0, res2, dst4, res0, res2); + ST4x8_UB(res0, res2, dst, dst_stride); +} + +static void common_hz_8t_and_aver_dst_4w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (4 == height) { + common_hz_8t_and_aver_dst_4x4_msa(src, src_stride, dst, dst_stride, filter); + } else if (8 == height) { + common_hz_8t_and_aver_dst_4x8_msa(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_8t_and_aver_dst_8w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + int32_t loop_cnt; + v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3; + v16u8 mask0, mask1, mask2, mask3, dst0, dst1, dst2, dst3; + v8i16 filt, out0, out1, out2, out3; + + mask0 = LD_UB(&mc_filt_mask_arr[0]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src0, src1, src2, src3); + XORI_B4_128_SB(src0, src1, src2, src3); + src += (4 * src_stride); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filt0, filt1, filt2, filt3, out0, out1, + out2, out3); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + CONVERT_UB_AVG_ST8x4_UB(out0, out1, out2, out3, dst0, dst1, dst2, dst3, dst, + dst_stride); + dst += (4 * dst_stride); + } +} + +static void common_hz_8t_and_aver_dst_16w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + int32_t loop_cnt; + v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3; + v16u8 mask0, mask1, mask2, mask3, dst0, dst1; + v8i16 filt, out0, out1, out2, out3; + v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8i16 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15; + + mask0 = LD_UB(&mc_filt_mask_arr[0]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + for (loop_cnt = height >> 1; loop_cnt--;) { + LD_SB2(src, src_stride, src0, src2); + LD_SB2(src + 8, src_stride, src1, src3); + src += (2 * src_stride); + + XORI_B4_128_SB(src0, src1, src2, src3); + VSHF_B4_SH(src0, src0, mask0, mask1, mask2, mask3, vec0, vec4, vec8, vec12); + VSHF_B4_SH(src1, src1, mask0, mask1, mask2, mask3, vec1, vec5, vec9, vec13); + VSHF_B4_SH(src2, src2, mask0, mask1, mask2, mask3, vec2, vec6, vec10, + vec14); + VSHF_B4_SH(src3, src3, mask0, mask1, mask2, mask3, vec3, vec7, vec11, + vec15); + DOTP_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + DOTP_SB4_SH(vec8, vec9, vec10, vec11, filt2, filt2, filt2, filt2, vec8, + vec9, vec10, vec11); + DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt1, filt1, filt1, filt1, vec0, vec1, + vec2, vec3); + DPADD_SB4_SH(vec12, vec13, vec14, vec15, filt3, filt3, filt3, filt3, vec8, + vec9, vec10, vec11); + ADDS_SH4_SH(vec0, vec8, vec1, vec9, vec2, vec10, vec3, vec11, out0, out1, + out2, out3); + LD_UB2(dst, dst_stride, dst0, dst1); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + PCKEV_XORI128_AVG_ST_UB(out1, out0, dst0, dst); + dst += dst_stride; + PCKEV_XORI128_AVG_ST_UB(out3, out2, dst1, dst); + dst += dst_stride; + } +} + +static void common_hz_8t_and_aver_dst_32w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3; + v16u8 dst1, dst2, mask0, mask1, mask2, mask3; + v8i16 filt, out0, out1, out2, out3; + v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8i16 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15; + + mask0 = LD_UB(&mc_filt_mask_arr[0]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + for (loop_cnt = height; loop_cnt--;) { + src0 = LD_SB(src); + src2 = LD_SB(src + 16); + src3 = LD_SB(src + 24); + src1 = __msa_sldi_b(src2, src0, 8); + src += src_stride; + + XORI_B4_128_SB(src0, src1, src2, src3); + VSHF_B4_SH(src0, src0, mask0, mask1, mask2, mask3, vec0, vec4, vec8, vec12); + VSHF_B4_SH(src1, src1, mask0, mask1, mask2, mask3, vec1, vec5, vec9, vec13); + VSHF_B4_SH(src2, src2, mask0, mask1, mask2, mask3, vec2, vec6, vec10, + vec14); + VSHF_B4_SH(src3, src3, mask0, mask1, mask2, mask3, vec3, vec7, vec11, + vec15); + DOTP_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + DOTP_SB4_SH(vec8, vec9, vec10, vec11, filt2, filt2, filt2, filt2, vec8, + vec9, vec10, vec11); + DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt1, filt1, filt1, filt1, vec0, vec1, + vec2, vec3); + DPADD_SB4_SH(vec12, vec13, vec14, vec15, filt3, filt3, filt3, filt3, vec8, + vec9, vec10, vec11); + ADDS_SH4_SH(vec0, vec8, vec1, vec9, vec2, vec10, vec3, vec11, out0, out1, + out2, out3); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + LD_UB2(dst, 16, dst1, dst2); + PCKEV_XORI128_AVG_ST_UB(out1, out0, dst1, dst); + PCKEV_XORI128_AVG_ST_UB(out3, out2, dst2, dst + 16); + dst += dst_stride; + } +} + +static void common_hz_8t_and_aver_dst_64w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt, cnt; + v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3; + v16u8 dst1, dst2, mask0, mask1, mask2, mask3; + v8i16 filt, out0, out1, out2, out3; + v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8i16 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15; + + mask0 = LD_UB(&mc_filt_mask_arr[0]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + for (loop_cnt = height; loop_cnt--;) { + for (cnt = 0; cnt < 2; ++cnt) { + src0 = LD_SB(&src[cnt << 5]); + src2 = LD_SB(&src[16 + (cnt << 5)]); + src3 = LD_SB(&src[24 + (cnt << 5)]); + src1 = __msa_sldi_b(src2, src0, 8); + + XORI_B4_128_SB(src0, src1, src2, src3); + VSHF_B4_SH(src0, src0, mask0, mask1, mask2, mask3, vec0, vec4, vec8, + vec12); + VSHF_B4_SH(src1, src1, mask0, mask1, mask2, mask3, vec1, vec5, vec9, + vec13); + VSHF_B4_SH(src2, src2, mask0, mask1, mask2, mask3, vec2, vec6, vec10, + vec14); + VSHF_B4_SH(src3, src3, mask0, mask1, mask2, mask3, vec3, vec7, vec11, + vec15); + DOTP_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, + vec1, vec2, vec3); + DOTP_SB4_SH(vec8, vec9, vec10, vec11, filt2, filt2, filt2, filt2, vec8, + vec9, vec10, vec11); + DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt1, filt1, filt1, filt1, vec0, + vec1, vec2, vec3); + DPADD_SB4_SH(vec12, vec13, vec14, vec15, filt3, filt3, filt3, filt3, vec8, + vec9, vec10, vec11); + ADDS_SH4_SH(vec0, vec8, vec1, vec9, vec2, vec10, vec3, vec11, out0, out1, + out2, out3); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + LD_UB2(&dst[cnt << 5], 16, dst1, dst2); + PCKEV_XORI128_AVG_ST_UB(out1, out0, dst1, &dst[cnt << 5]); + PCKEV_XORI128_AVG_ST_UB(out3, out2, dst2, &dst[16 + (cnt << 5)]); + } + + src += src_stride; + dst += dst_stride; + } +} + +static void common_hz_2t_and_aver_dst_4x4_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + v16i8 src0, src1, src2, src3, mask; + v16u8 filt0, dst0, dst1, dst2, dst3, vec0, vec1, res0, res1; + v8u16 vec2, vec3, filt; + + mask = LD_SB(&mc_filt_mask_arr[16]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB4(src, src_stride, src0, src1, src2, src3); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + VSHF_B2_UB(src0, src1, src2, src3, mask, mask, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt0, filt0, vec2, vec3); + SRARI_H2_UH(vec2, vec3, FILTER_BITS); + PCKEV_B2_UB(vec2, vec2, vec3, vec3, res0, res1); + ILVR_W2_UB(dst1, dst0, dst3, dst2, dst0, dst2); + AVER_UB2_UB(res0, dst0, res1, dst2, res0, res1); + ST4x4_UB(res0, res1, 0, 1, 0, 1, dst, dst_stride); +} + +static void common_hz_2t_and_aver_dst_4x8_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask; + v16u8 filt0, vec0, vec1, vec2, vec3, res0, res1, res2, res3; + v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + v8u16 vec4, vec5, vec6, vec7, filt; + + mask = LD_SB(&mc_filt_mask_arr[16]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + LD_UB8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7); + VSHF_B2_UB(src0, src1, src2, src3, mask, mask, vec0, vec1); + VSHF_B2_UB(src4, src5, src6, src7, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec4, vec5, + vec6, vec7); + SRARI_H4_UH(vec4, vec5, vec6, vec7, FILTER_BITS); + PCKEV_B4_UB(vec4, vec4, vec5, vec5, vec6, vec6, vec7, vec7, res0, res1, res2, + res3); + ILVR_W4_UB(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, dst0, dst2, dst4, + dst6); + AVER_UB4_UB(res0, dst0, res1, dst2, res2, dst4, res3, dst6, res0, res1, res2, + res3); + ST4x4_UB(res0, res1, 0, 1, 0, 1, dst, dst_stride); + dst += (4 * dst_stride); + ST4x4_UB(res2, res3, 0, 1, 0, 1, dst, dst_stride); +} + +static void common_hz_2t_and_aver_dst_4w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (4 == height) { + common_hz_2t_and_aver_dst_4x4_msa(src, src_stride, dst, dst_stride, filter); + } else if (8 == height) { + common_hz_2t_and_aver_dst_4x8_msa(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_2t_and_aver_dst_8x4_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + v16i8 src0, src1, src2, src3, mask; + v16u8 filt0, dst0, dst1, dst2, dst3; + v8u16 vec0, vec1, vec2, vec3, filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB4(src, src_stride, src0, src1, src2, src3); + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + PCKEV_AVG_ST8x4_UB(vec0, dst0, vec1, dst1, vec2, dst2, vec3, dst3, dst, + dst_stride); +} + +static void common_hz_2t_and_aver_dst_8x8mult_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + v16i8 src0, src1, src2, src3, mask; + v16u8 filt0, dst0, dst1, dst2, dst3; + v8u16 vec0, vec1, vec2, vec3, filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + LD_SB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + PCKEV_AVG_ST8x4_UB(vec0, dst0, vec1, dst1, vec2, dst2, vec3, dst3, dst, + dst_stride); + dst += (4 * dst_stride); + + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + PCKEV_AVG_ST8x4_UB(vec0, dst0, vec1, dst1, vec2, dst2, vec3, dst3, dst, + dst_stride); + dst += (4 * dst_stride); + + if (16 == height) { + LD_SB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + LD_SB4(src, src_stride, src0, src1, src2, src3); + PCKEV_AVG_ST8x4_UB(vec0, dst0, vec1, dst1, vec2, dst2, vec3, dst3, dst, + dst_stride); + dst += (4 * dst_stride); + + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + PCKEV_AVG_ST8x4_UB(vec0, dst0, vec1, dst1, vec2, dst2, vec3, dst3, dst, + dst_stride); + } +} + +static void common_hz_2t_and_aver_dst_8w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (4 == height) { + common_hz_2t_and_aver_dst_8x4_msa(src, src_stride, dst, dst_stride, filter); + } else { + common_hz_2t_and_aver_dst_8x8mult_msa(src, src_stride, dst, dst_stride, + filter, height); + } +} + +static void common_hz_2t_and_aver_dst_16w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask; + v16u8 filt0, dst0, dst1, dst2, dst3; + v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8u16 res0, res1, res2, res3, res4, res5, res6, res7, filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB4(src, src_stride, src0, src2, src4, src6); + LD_SB4(src + 8, src_stride, src1, src3, src5, src7); + src += (4 * src_stride); + + VSHF_B2_UB(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UB(src2, src2, src3, src3, mask, mask, vec2, vec3); + VSHF_B2_UB(src4, src4, src5, src5, mask, mask, vec4, vec5); + VSHF_B2_UB(src6, src6, src7, src7, mask, mask, vec6, vec7); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, res0, res1, + res2, res3); + DOTP_UB4_UH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0, res4, res5, + res6, res7); + SRARI_H4_UH(res0, res1, res2, res3, FILTER_BITS); + SRARI_H4_UH(res4, res5, res6, res7, FILTER_BITS); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + PCKEV_AVG_ST_UB(res1, res0, dst0, dst); + dst += dst_stride; + PCKEV_AVG_ST_UB(res3, res2, dst1, dst); + dst += dst_stride; + PCKEV_AVG_ST_UB(res5, res4, dst2, dst); + dst += dst_stride; + PCKEV_AVG_ST_UB(res7, res6, dst3, dst); + dst += dst_stride; + + for (loop_cnt = (height >> 2) - 1; loop_cnt--;) { + LD_SB4(src, src_stride, src0, src2, src4, src6); + LD_SB4(src + 8, src_stride, src1, src3, src5, src7); + src += (4 * src_stride); + + VSHF_B2_UB(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UB(src2, src2, src3, src3, mask, mask, vec2, vec3); + VSHF_B2_UB(src4, src4, src5, src5, mask, mask, vec4, vec5); + VSHF_B2_UB(src6, src6, src7, src7, mask, mask, vec6, vec7); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, res0, res1, + res2, res3); + DOTP_UB4_UH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0, res4, res5, + res6, res7); + SRARI_H4_UH(res0, res1, res2, res3, FILTER_BITS); + SRARI_H4_UH(res4, res5, res6, res7, FILTER_BITS); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + PCKEV_AVG_ST_UB(res1, res0, dst0, dst); + dst += dst_stride; + PCKEV_AVG_ST_UB(res3, res2, dst1, dst); + dst += dst_stride; + PCKEV_AVG_ST_UB(res5, res4, dst2, dst); + dst += dst_stride; + PCKEV_AVG_ST_UB(res7, res6, dst3, dst); + dst += dst_stride; + } +} + +static void common_hz_2t_and_aver_dst_32w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask; + v16u8 filt0, dst0, dst1, dst2, dst3; + v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8u16 res0, res1, res2, res3, res4, res5, res6, res7, filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + for (loop_cnt = (height >> 1); loop_cnt--;) { + src0 = LD_SB(src); + src2 = LD_SB(src + 16); + src3 = LD_SB(src + 24); + src1 = __msa_sldi_b(src2, src0, 8); + src += src_stride; + src4 = LD_SB(src); + src6 = LD_SB(src + 16); + src7 = LD_SB(src + 24); + src5 = __msa_sldi_b(src6, src4, 8); + src += src_stride; + + VSHF_B2_UB(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UB(src2, src2, src3, src3, mask, mask, vec2, vec3); + VSHF_B2_UB(src4, src4, src5, src5, mask, mask, vec4, vec5); + VSHF_B2_UB(src6, src6, src7, src7, mask, mask, vec6, vec7); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, res0, res1, + res2, res3); + DOTP_UB4_UH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0, res4, res5, + res6, res7); + SRARI_H4_UH(res0, res1, res2, res3, FILTER_BITS); + SRARI_H4_UH(res4, res5, res6, res7, FILTER_BITS); + LD_UB2(dst, 16, dst0, dst1); + PCKEV_AVG_ST_UB(res1, res0, dst0, dst); + PCKEV_AVG_ST_UB(res3, res2, dst1, (dst + 16)); + dst += dst_stride; + LD_UB2(dst, 16, dst2, dst3); + PCKEV_AVG_ST_UB(res5, res4, dst2, dst); + PCKEV_AVG_ST_UB(res7, res6, dst3, (dst + 16)); + dst += dst_stride; + } +} + +static void common_hz_2t_and_aver_dst_64w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask; + v16u8 filt0, dst0, dst1, dst2, dst3; + v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8u16 out0, out1, out2, out3, out4, out5, out6, out7, filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + for (loop_cnt = height; loop_cnt--;) { + LD_SB4(src, 16, src0, src2, src4, src6); + src7 = LD_SB(src + 56); + SLDI_B3_SB(src2, src4, src6, src0, src2, src4, src1, src3, src5, 8); + src += src_stride; + + VSHF_B2_UB(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UB(src2, src2, src3, src3, mask, mask, vec2, vec3); + VSHF_B2_UB(src4, src4, src5, src5, mask, mask, vec4, vec5); + VSHF_B2_UB(src6, src6, src7, src7, mask, mask, vec6, vec7); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, out0, out1, + out2, out3); + DOTP_UB4_UH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0, out4, out5, + out6, out7); + SRARI_H4_UH(out0, out1, out2, out3, FILTER_BITS); + SRARI_H4_UH(out4, out5, out6, out7, FILTER_BITS); + LD_UB4(dst, 16, dst0, dst1, dst2, dst3); + PCKEV_AVG_ST_UB(out1, out0, dst0, dst); + PCKEV_AVG_ST_UB(out3, out2, dst1, dst + 16); + PCKEV_AVG_ST_UB(out5, out4, dst2, dst + 32); + PCKEV_AVG_ST_UB(out7, out6, dst3, dst + 48); + dst += dst_stride; + } +} + +void aom_convolve8_avg_horiz_msa(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + int8_t cnt, filt_hor[8]; + + assert(x_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_hor[cnt] = filter_x[cnt]; + } + + if (((const int32_t *)filter_x)[0] == 0) { + switch (w) { + case 4: + common_hz_2t_and_aver_dst_4w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + case 8: + common_hz_2t_and_aver_dst_8w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + case 16: + common_hz_2t_and_aver_dst_16w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + case 32: + common_hz_2t_and_aver_dst_32w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + case 64: + common_hz_2t_and_aver_dst_64w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + default: + aom_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } else { + switch (w) { + case 4: + common_hz_8t_and_aver_dst_4w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + case 8: + common_hz_8t_and_aver_dst_8w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + case 16: + common_hz_8t_and_aver_dst_16w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + case 32: + common_hz_8t_and_aver_dst_32w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + case 64: + common_hz_8t_and_aver_dst_64w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + default: + aom_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } +} diff --git a/third_party/aom/aom_dsp/mips/aom_convolve8_avg_msa.c b/third_party/aom/aom_dsp/mips/aom_convolve8_avg_msa.c new file mode 100644 index 0000000000..bed600d5b9 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/aom_convolve8_avg_msa.c @@ -0,0 +1,605 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/aom_convolve_msa.h" + +static void common_hv_8ht_8vt_and_aver_dst_4w_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + v16u8 dst0, dst1, dst2, dst3, mask0, mask1, mask2, mask3, tmp0, tmp1; + v16i8 filt_hz0, filt_hz1, filt_hz2, filt_hz3; + v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + v8i16 hz_out7, hz_out8, hz_out9, res0, res1, vec0, vec1, vec2, vec3, vec4; + v8i16 filt, filt_vt0, filt_vt1, filt_vt2, filt_vt3; + + mask0 = LD_UB(&mc_filt_mask_arr[16]); + src -= (3 + 3 * src_stride); + + /* rearranging filter */ + filt = LD_SH(filter_horiz); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6); + XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6); + src += (7 * src_stride); + + hz_out0 = HORIZ_8TAP_FILT(src0, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out2 = HORIZ_8TAP_FILT(src2, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out4 = HORIZ_8TAP_FILT(src4, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out5 = HORIZ_8TAP_FILT(src5, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + SLDI_B2_SH(hz_out2, hz_out4, hz_out0, hz_out2, hz_out1, hz_out3, 8); + + filt = LD_SH(filter_vert); + SPLATI_H4_SH(filt, 0, 1, 2, 3, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + + ILVEV_B2_SH(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + vec2 = (v8i16)__msa_ilvev_b((v16i8)hz_out5, (v16i8)hz_out4); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src7, src8, src9, src10); + XORI_B4_128_SB(src7, src8, src9, src10); + src += (4 * src_stride); + + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + hz_out7 = HORIZ_8TAP_FILT(src7, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out6 = (v8i16)__msa_sldi_b((v16i8)hz_out7, (v16i8)hz_out5, 8); + vec3 = (v8i16)__msa_ilvev_b((v16i8)hz_out7, (v16i8)hz_out6); + res0 = FILT_8TAP_DPADD_S_H(vec0, vec1, vec2, vec3, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + + hz_out9 = HORIZ_8TAP_FILT(src9, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out8 = (v8i16)__msa_sldi_b((v16i8)hz_out9, (v16i8)hz_out7, 8); + vec4 = (v8i16)__msa_ilvev_b((v16i8)hz_out9, (v16i8)hz_out8); + res1 = FILT_8TAP_DPADD_S_H(vec1, vec2, vec3, vec4, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + ILVR_W2_UB(dst1, dst0, dst3, dst2, dst0, dst2); + + SRARI_H2_SH(res0, res1, FILTER_BITS); + SAT_SH2_SH(res0, res1, 7); + PCKEV_B2_UB(res0, res0, res1, res1, tmp0, tmp1); + XORI_B2_128_UB(tmp0, tmp1); + AVER_UB2_UB(tmp0, dst0, tmp1, dst2, tmp0, tmp1); + ST4x4_UB(tmp0, tmp1, 0, 1, 0, 1, dst, dst_stride); + dst += (4 * dst_stride); + + hz_out5 = hz_out9; + vec0 = vec2; + vec1 = vec3; + vec2 = vec4; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_8w_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + v16i8 filt_hz0, filt_hz1, filt_hz2, filt_hz3; + v8i16 filt, filt_vt0, filt_vt1, filt_vt2, filt_vt3; + v16u8 dst0, dst1, dst2, dst3, mask0, mask1, mask2, mask3; + v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + v8i16 hz_out7, hz_out8, hz_out9, hz_out10, tmp0, tmp1, tmp2, tmp3; + v8i16 out0, out1, out2, out3, out4, out5, out6, out7, out8, out9; + + mask0 = LD_UB(&mc_filt_mask_arr[0]); + src -= (3 + 3 * src_stride); + + /* rearranging filter */ + filt = LD_SH(filter_horiz); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6); + src += (7 * src_stride); + + XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6); + hz_out0 = HORIZ_8TAP_FILT(src0, src0, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out1 = HORIZ_8TAP_FILT(src1, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out2 = HORIZ_8TAP_FILT(src2, src2, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out3 = HORIZ_8TAP_FILT(src3, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out4 = HORIZ_8TAP_FILT(src4, src4, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out5 = HORIZ_8TAP_FILT(src5, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out6 = HORIZ_8TAP_FILT(src6, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + + filt = LD_SH(filter_vert); + SPLATI_H4_SH(filt, 0, 1, 2, 3, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + + ILVEV_B2_SH(hz_out0, hz_out1, hz_out2, hz_out3, out0, out1); + ILVEV_B2_SH(hz_out4, hz_out5, hz_out1, hz_out2, out2, out4); + ILVEV_B2_SH(hz_out3, hz_out4, hz_out5, hz_out6, out5, out6); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src7, src8, src9, src10); + XORI_B4_128_SB(src7, src8, src9, src10); + src += (4 * src_stride); + + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + + hz_out7 = HORIZ_8TAP_FILT(src7, src7, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + out3 = (v8i16)__msa_ilvev_b((v16i8)hz_out7, (v16i8)hz_out6); + tmp0 = FILT_8TAP_DPADD_S_H(out0, out1, out2, out3, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + + hz_out8 = HORIZ_8TAP_FILT(src8, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + out7 = (v8i16)__msa_ilvev_b((v16i8)hz_out8, (v16i8)hz_out7); + tmp1 = FILT_8TAP_DPADD_S_H(out4, out5, out6, out7, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + + hz_out9 = HORIZ_8TAP_FILT(src9, src9, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + out8 = (v8i16)__msa_ilvev_b((v16i8)hz_out9, (v16i8)hz_out8); + tmp2 = FILT_8TAP_DPADD_S_H(out1, out2, out3, out8, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + + hz_out10 = HORIZ_8TAP_FILT(src10, src10, mask0, mask1, mask2, mask3, + filt_hz0, filt_hz1, filt_hz2, filt_hz3); + out9 = (v8i16)__msa_ilvev_b((v16i8)hz_out10, (v16i8)hz_out9); + tmp3 = FILT_8TAP_DPADD_S_H(out5, out6, out7, out9, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + + SRARI_H4_SH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3, 7); + CONVERT_UB_AVG_ST8x4_UB(tmp0, tmp1, tmp2, tmp3, dst0, dst1, dst2, dst3, dst, + dst_stride); + dst += (4 * dst_stride); + + hz_out6 = hz_out10; + out0 = out2; + out1 = out3; + out2 = out8; + out4 = out6; + out5 = out7; + out6 = out9; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_16w_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 2; multiple8_cnt--;) { + common_hv_8ht_8vt_and_aver_dst_8w_msa(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_32w_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_8ht_8vt_and_aver_dst_8w_msa(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_64w_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 8; multiple8_cnt--;) { + common_hv_8ht_8vt_and_aver_dst_8w_msa(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_2ht_2vt_and_aver_dst_4x4_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert) { + v16i8 src0, src1, src2, src3, src4, mask; + v16u8 filt_hz, filt_vt, vec0, vec1; + v16u8 dst0, dst1, dst2, dst3, res0, res1; + v8u16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, tmp0, tmp1, filt; + + mask = LD_SB(&mc_filt_mask_arr[16]); + + /* rearranging filter */ + filt = LD_UH(filter_horiz); + filt_hz = (v16u8)__msa_splati_h((v8i16)filt, 0); + + filt = LD_UH(filter_vert); + filt_vt = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB5(src, src_stride, src0, src1, src2, src3, src4); + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out1 = (v8u16)__msa_sldi_b((v16i8)hz_out2, (v16i8)hz_out0, 8); + hz_out3 = (v8u16)__msa_pckod_d((v2i64)hz_out4, (v2i64)hz_out2); + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + ILVR_W2_UB(dst1, dst0, dst3, dst2, dst0, dst2); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_B2_UB(tmp0, tmp0, tmp1, tmp1, res0, res1); + AVER_UB2_UB(res0, dst0, res1, dst2, res0, res1); + ST4x4_UB(res0, res1, 0, 1, 0, 1, dst, dst_stride); +} + +static void common_hv_2ht_2vt_and_aver_dst_4x8_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert) { + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, mask; + v16u8 filt_hz, filt_vt, vec0, vec1, vec2, vec3, res0, res1, res2, res3; + v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + v8u16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + v8u16 hz_out7, hz_out8, tmp0, tmp1, tmp2, tmp3; + v8i16 filt; + + mask = LD_SB(&mc_filt_mask_arr[16]); + + /* rearranging filter */ + filt = LD_SH(filter_horiz); + filt_hz = (v16u8)__msa_splati_h(filt, 0); + + filt = LD_SH(filter_vert); + filt_vt = (v16u8)__msa_splati_h(filt, 0); + + LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + src += (8 * src_stride); + src8 = LD_SB(src); + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src5, mask, filt_hz, FILTER_BITS); + hz_out6 = HORIZ_2TAP_FILT_UH(src6, src7, mask, filt_hz, FILTER_BITS); + hz_out8 = HORIZ_2TAP_FILT_UH(src8, src8, mask, filt_hz, FILTER_BITS); + SLDI_B3_UH(hz_out2, hz_out4, hz_out6, hz_out0, hz_out2, hz_out4, hz_out1, + hz_out3, hz_out5, 8); + hz_out7 = (v8u16)__msa_pckod_d((v2i64)hz_out8, (v2i64)hz_out6); + + LD_UB8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7); + ILVR_W4_UB(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, dst0, dst2, dst4, + dst6); + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + ILVEV_B2_UB(hz_out4, hz_out5, hz_out6, hz_out7, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt_vt, filt_vt, filt_vt, filt_vt, tmp0, + tmp1, tmp2, tmp3); + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_B4_UB(tmp0, tmp0, tmp1, tmp1, tmp2, tmp2, tmp3, tmp3, res0, res1, res2, + res3); + AVER_UB4_UB(res0, dst0, res1, dst2, res2, dst4, res3, dst6, res0, res1, res2, + res3); + ST4x4_UB(res0, res1, 0, 1, 0, 1, dst, dst_stride); + dst += (4 * dst_stride); + ST4x4_UB(res2, res3, 0, 1, 0, 1, dst, dst_stride); +} + +static void common_hv_2ht_2vt_and_aver_dst_4w_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + if (4 == height) { + common_hv_2ht_2vt_and_aver_dst_4x4_msa(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert); + } else if (8 == height) { + common_hv_2ht_2vt_and_aver_dst_4x8_msa(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert); + } +} + +static void common_hv_2ht_2vt_and_aver_dst_8x4_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert) { + v16i8 src0, src1, src2, src3, src4, mask; + v16u8 filt_hz, filt_vt, dst0, dst1, dst2, dst3, vec0, vec1, vec2, vec3; + v8u16 hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3; + v8i16 filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_SH(filter_horiz); + filt_hz = (v16u8)__msa_splati_h(filt, 0); + + filt = LD_SH(filter_vert); + filt_vt = (v16u8)__msa_splati_h(filt, 0); + + LD_SB5(src, src_stride, src0, src1, src2, src3, src4); + src += (5 * src_stride); + + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp0 = __msa_dotp_u_h(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec1 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp1 = __msa_dotp_u_h(vec1, filt_vt); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec2 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp2 = __msa_dotp_u_h(vec2, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec3 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp3 = __msa_dotp_u_h(vec3, filt_vt); + + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_AVG_ST8x4_UB(tmp0, dst0, tmp1, dst1, tmp2, dst2, tmp3, dst3, dst, + dst_stride); +} + +static void common_hv_2ht_2vt_and_aver_dst_8x8mult_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, mask; + v16u8 filt_hz, filt_vt, vec0, dst0, dst1, dst2, dst3; + v8u16 hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3; + v8i16 filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_SH(filter_horiz); + filt_hz = (v16u8)__msa_splati_h(filt, 0); + + filt = LD_SH(filter_vert); + filt_vt = (v16u8)__msa_splati_h(filt, 0); + + src0 = LD_SB(src); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp0 = __msa_dotp_u_h(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp1 = __msa_dotp_u_h(vec0, filt_vt); + + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp2 = __msa_dotp_u_h(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp3 = __msa_dotp_u_h(vec0, filt_vt); + + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + PCKEV_AVG_ST8x4_UB(tmp0, dst0, tmp1, dst1, tmp2, dst2, tmp3, dst3, dst, + dst_stride); + dst += (4 * dst_stride); + } +} + +static void common_hv_2ht_2vt_and_aver_dst_8w_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + if (4 == height) { + common_hv_2ht_2vt_and_aver_dst_8x4_msa(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert); + } else { + common_hv_2ht_2vt_and_aver_dst_8x8mult_msa( + src, src_stride, dst, dst_stride, filter_horiz, filter_vert, height); + } +} + +static void common_hv_2ht_2vt_and_aver_dst_16w_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask; + v16u8 filt_hz, filt_vt, vec0, vec1, dst0, dst1, dst2, dst3; + v8u16 hz_out0, hz_out1, hz_out2, hz_out3, tmp0, tmp1; + v8i16 filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_SH(filter_horiz); + filt_hz = (v16u8)__msa_splati_h(filt, 0); + + filt = LD_SH(filter_vert); + filt_vt = (v16u8)__msa_splati_h(filt, 0); + + LD_SB2(src, 8, src0, src1); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src0, src2, src4, src6); + LD_SB4(src + 8, src_stride, src1, src3, src5, src7); + src += (4 * src_stride); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + + hz_out1 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp1, tmp0, dst0, dst); + dst += dst_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp1, tmp0, dst1, dst); + dst += dst_stride; + + hz_out1 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp1, tmp0, dst2, dst); + dst += dst_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp1, tmp0, dst3, dst); + dst += dst_stride; + } +} + +static void common_hv_2ht_2vt_and_aver_dst_32w_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 2; multiple8_cnt--;) { + common_hv_2ht_2vt_and_aver_dst_16w_msa(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 16; + dst += 16; + } +} + +static void common_hv_2ht_2vt_and_aver_dst_64w_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_2ht_2vt_and_aver_dst_16w_msa(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 16; + dst += 16; + } +} + +void aom_convolve8_avg_msa(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + int8_t cnt, filt_hor[8], filt_ver[8]; + + assert(x_step_q4 == 16); + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_hor[cnt] = filter_x[cnt]; + filt_ver[cnt] = filter_y[cnt]; + } + + if (((const int32_t *)filter_x)[0] == 0 && + ((const int32_t *)filter_y)[0] == 0) { + switch (w) { + case 4: + common_hv_2ht_2vt_and_aver_dst_4w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], h); + break; + case 8: + common_hv_2ht_2vt_and_aver_dst_8w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], h); + break; + case 16: + common_hv_2ht_2vt_and_aver_dst_16w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, + &filt_hor[3], &filt_ver[3], h); + break; + case 32: + common_hv_2ht_2vt_and_aver_dst_32w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, + &filt_hor[3], &filt_ver[3], h); + break; + case 64: + common_hv_2ht_2vt_and_aver_dst_64w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, + &filt_hor[3], &filt_ver[3], h); + break; + default: + aom_convolve8_avg_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } else if (((const int32_t *)filter_x)[0] == 0 || + ((const int32_t *)filter_y)[0] == 0) { + aom_convolve8_avg_c(src, src_stride, dst, dst_stride, filter_x, x_step_q4, + filter_y, y_step_q4, w, h); + } else { + switch (w) { + case 4: + common_hv_8ht_8vt_and_aver_dst_4w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + case 8: + common_hv_8ht_8vt_and_aver_dst_8w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + case 16: + common_hv_8ht_8vt_and_aver_dst_16w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + case 32: + common_hv_8ht_8vt_and_aver_dst_32w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + case 64: + common_hv_8ht_8vt_and_aver_dst_64w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + default: + aom_convolve8_avg_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } +} diff --git a/third_party/aom/aom_dsp/mips/aom_convolve8_avg_vert_msa.c b/third_party/aom/aom_dsp/mips/aom_convolve8_avg_vert_msa.c new file mode 100644 index 0000000000..dae771104f --- /dev/null +++ b/third_party/aom/aom_dsp/mips/aom_convolve8_avg_vert_msa.c @@ -0,0 +1,677 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/aom_convolve_msa.h" + +static void common_vt_8t_and_aver_dst_4w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + v16u8 dst0, dst1, dst2, dst3, out; + v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r; + v16i8 src65_r, src87_r, src109_r, src2110, src4332, src6554, src8776; + v16i8 src10998, filt0, filt1, filt2, filt3; + v8i16 filt, out10, out32; + + src -= (3 * src_stride); + + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6); + src += (7 * src_stride); + + ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_r, src32_r, + src54_r, src21_r); + ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r); + ILVR_D3_SB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r, src2110, + src4332, src6554); + XORI_B3_128_SB(src2110, src4332, src6554); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src7, src8, src9, src10); + src += (4 * src_stride); + + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r, + src87_r, src98_r, src109_r); + ILVR_D2_SB(src87_r, src76_r, src109_r, src98_r, src8776, src10998); + XORI_B2_128_SB(src8776, src10998); + out10 = FILT_8TAP_DPADD_S_H(src2110, src4332, src6554, src8776, filt0, + filt1, filt2, filt3); + out32 = FILT_8TAP_DPADD_S_H(src4332, src6554, src8776, src10998, filt0, + filt1, filt2, filt3); + SRARI_H2_SH(out10, out32, FILTER_BITS); + SAT_SH2_SH(out10, out32, 7); + out = PCKEV_XORI128_UB(out10, out32); + ILVR_W2_UB(dst1, dst0, dst3, dst2, dst0, dst2); + + dst0 = (v16u8)__msa_ilvr_d((v2i64)dst2, (v2i64)dst0); + out = __msa_aver_u_b(out, dst0); + + ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride); + dst += (4 * dst_stride); + + src2110 = src6554; + src4332 = src8776; + src6554 = src10998; + src6 = src10; + } +} + +static void common_vt_8t_and_aver_dst_8w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + v16u8 dst0, dst1, dst2, dst3; + v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r; + v16i8 src65_r, src87_r, src109_r, filt0, filt1, filt2, filt3; + v8i16 filt, out0, out1, out2, out3; + + src -= (3 * src_stride); + + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6); + src += (7 * src_stride); + + XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6); + ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_r, src32_r, + src54_r, src21_r); + ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src7, src8, src9, src10); + src += (4 * src_stride); + + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + XORI_B4_128_SB(src7, src8, src9, src10); + ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r, + src87_r, src98_r, src109_r); + out0 = FILT_8TAP_DPADD_S_H(src10_r, src32_r, src54_r, src76_r, filt0, filt1, + filt2, filt3); + out1 = FILT_8TAP_DPADD_S_H(src21_r, src43_r, src65_r, src87_r, filt0, filt1, + filt2, filt3); + out2 = FILT_8TAP_DPADD_S_H(src32_r, src54_r, src76_r, src98_r, filt0, filt1, + filt2, filt3); + out3 = FILT_8TAP_DPADD_S_H(src43_r, src65_r, src87_r, src109_r, filt0, + filt1, filt2, filt3); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + CONVERT_UB_AVG_ST8x4_UB(out0, out1, out2, out3, dst0, dst1, dst2, dst3, dst, + dst_stride); + dst += (4 * dst_stride); + + src10_r = src54_r; + src32_r = src76_r; + src54_r = src98_r; + src21_r = src65_r; + src43_r = src87_r; + src65_r = src109_r; + src6 = src10; + } +} + +static void common_vt_8t_and_aver_dst_16w_mult_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height, int32_t width) { + const uint8_t *src_tmp; + uint8_t *dst_tmp; + uint32_t loop_cnt, cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r; + v16i8 src65_r, src87_r, src109_r, src10_l, src32_l, src54_l, src76_l; + v16i8 src98_l, src21_l, src43_l, src65_l, src87_l, src109_l; + v16i8 filt0, filt1, filt2, filt3; + v16u8 dst0, dst1, dst2, dst3, tmp0, tmp1, tmp2, tmp3; + v8i16 out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l, filt; + + src -= (3 * src_stride); + + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + for (cnt = (width >> 4); cnt--;) { + src_tmp = src; + dst_tmp = dst; + + LD_SB7(src_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6); + XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6); + src_tmp += (7 * src_stride); + + ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_r, src32_r, + src54_r, src21_r); + ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r); + ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_l, src32_l, + src54_l, src21_l); + ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src_tmp, src_stride, src7, src8, src9, src10); + src_tmp += (4 * src_stride); + + LD_UB4(dst_tmp, dst_stride, dst0, dst1, dst2, dst3); + XORI_B4_128_SB(src7, src8, src9, src10); + ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r, + src87_r, src98_r, src109_r); + ILVL_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_l, + src87_l, src98_l, src109_l); + out0_r = FILT_8TAP_DPADD_S_H(src10_r, src32_r, src54_r, src76_r, filt0, + filt1, filt2, filt3); + out1_r = FILT_8TAP_DPADD_S_H(src21_r, src43_r, src65_r, src87_r, filt0, + filt1, filt2, filt3); + out2_r = FILT_8TAP_DPADD_S_H(src32_r, src54_r, src76_r, src98_r, filt0, + filt1, filt2, filt3); + out3_r = FILT_8TAP_DPADD_S_H(src43_r, src65_r, src87_r, src109_r, filt0, + filt1, filt2, filt3); + out0_l = FILT_8TAP_DPADD_S_H(src10_l, src32_l, src54_l, src76_l, filt0, + filt1, filt2, filt3); + out1_l = FILT_8TAP_DPADD_S_H(src21_l, src43_l, src65_l, src87_l, filt0, + filt1, filt2, filt3); + out2_l = FILT_8TAP_DPADD_S_H(src32_l, src54_l, src76_l, src98_l, filt0, + filt1, filt2, filt3); + out3_l = FILT_8TAP_DPADD_S_H(src43_l, src65_l, src87_l, src109_l, filt0, + filt1, filt2, filt3); + SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, FILTER_BITS); + SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, FILTER_BITS); + SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7); + SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7); + PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l, + out3_r, tmp0, tmp1, tmp2, tmp3); + XORI_B4_128_UB(tmp0, tmp1, tmp2, tmp3); + AVER_UB4_UB(tmp0, dst0, tmp1, dst1, tmp2, dst2, tmp3, dst3, dst0, dst1, + dst2, dst3); + ST_UB4(dst0, dst1, dst2, dst3, dst_tmp, dst_stride); + dst_tmp += (4 * dst_stride); + + src10_r = src54_r; + src32_r = src76_r; + src54_r = src98_r; + src21_r = src65_r; + src43_r = src87_r; + src65_r = src109_r; + src10_l = src54_l; + src32_l = src76_l; + src54_l = src98_l; + src21_l = src65_l; + src43_l = src87_l; + src65_l = src109_l; + src6 = src10; + } + + src += 16; + dst += 16; + } +} + +static void common_vt_8t_and_aver_dst_16w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + common_vt_8t_and_aver_dst_16w_mult_msa(src, src_stride, dst, dst_stride, + filter, height, 16); +} + +static void common_vt_8t_and_aver_dst_32w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + common_vt_8t_and_aver_dst_16w_mult_msa(src, src_stride, dst, dst_stride, + filter, height, 32); +} + +static void common_vt_8t_and_aver_dst_64w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + common_vt_8t_and_aver_dst_16w_mult_msa(src, src_stride, dst, dst_stride, + filter, height, 64); +} + +static void common_vt_2t_and_aver_dst_4x4_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + v16i8 src0, src1, src2, src3, src4; + v16u8 dst0, dst1, dst2, dst3, out, filt0, src2110, src4332; + v16i8 src10_r, src32_r, src21_r, src43_r; + v8i16 filt; + v8u16 tmp0, tmp1; + + filt = LD_SH(filter); + filt0 = (v16u8)__msa_splati_h(filt, 0); + + LD_SB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + + src4 = LD_SB(src); + src += src_stride; + + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + ILVR_W2_UB(dst1, dst0, dst3, dst2, dst0, dst1); + dst0 = (v16u8)__msa_ilvr_d((v2i64)dst1, (v2i64)dst0); + ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r, + src32_r, src43_r); + ILVR_D2_UB(src21_r, src10_r, src43_r, src32_r, src2110, src4332); + DOTP_UB2_UH(src2110, src4332, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + + out = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + out = __msa_aver_u_b(out, dst0); + + ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride); +} + +static void common_vt_2t_and_aver_dst_4x8_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src87_r; + v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r; + v16u8 src2110, src4332, src6554, src8776, filt0; + v8u16 tmp0, tmp1, tmp2, tmp3; + v8i16 filt; + + filt = LD_SH(filter); + filt0 = (v16u8)__msa_splati_h(filt, 0); + + LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + src += (8 * src_stride); + src8 = LD_SB(src); + + LD_UB8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7); + ILVR_W4_UB(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, dst0, dst1, dst2, + dst3); + ILVR_D2_UB(dst1, dst0, dst3, dst2, dst0, dst1); + ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r, + src32_r, src43_r); + ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r, + src76_r, src87_r); + ILVR_D4_UB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r, src87_r, + src76_r, src2110, src4332, src6554, src8776); + DOTP_UB4_UH(src2110, src4332, src6554, src8776, filt0, filt0, filt0, filt0, + tmp0, tmp1, tmp2, tmp3); + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, src2110, src4332); + AVER_UB2_UB(src2110, dst0, src4332, dst1, src2110, src4332); + ST4x4_UB(src2110, src2110, 0, 1, 2, 3, dst, dst_stride); + dst += (4 * dst_stride); + ST4x4_UB(src4332, src4332, 0, 1, 2, 3, dst, dst_stride); +} + +static void common_vt_2t_and_aver_dst_4w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (4 == height) { + common_vt_2t_and_aver_dst_4x4_msa(src, src_stride, dst, dst_stride, filter); + } else if (8 == height) { + common_vt_2t_and_aver_dst_4x8_msa(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_vt_2t_and_aver_dst_8x4_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + v16u8 src0, src1, src2, src3, src4; + v16u8 dst0, dst1, dst2, dst3, vec0, vec1, vec2, vec3, filt0; + v8u16 tmp0, tmp1, tmp2, tmp3; + v8i16 filt; + + /* rearranging filter_y */ + filt = LD_SH(filter); + filt0 = (v16u8)__msa_splati_h(filt, 0); + + LD_UB5(src, src_stride, src0, src1, src2, src3, src4); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + ILVR_B2_UB(src1, src0, src2, src1, vec0, vec1); + ILVR_B2_UB(src3, src2, src4, src3, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, tmp0, tmp1, + tmp2, tmp3); + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_AVG_ST8x4_UB(tmp0, dst0, tmp1, dst1, tmp2, dst2, tmp3, dst3, dst, + dst_stride); +} + +static void common_vt_2t_and_aver_dst_8x8mult_msa( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8; + v16u8 dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8; + v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + v8u16 tmp0, tmp1, tmp2, tmp3; + v8i16 filt; + + /* rearranging filter_y */ + filt = LD_SH(filter); + filt0 = (v16u8)__msa_splati_h(filt, 0); + + src0 = LD_UB(src); + src += src_stride; + + for (loop_cnt = (height >> 3); loop_cnt--;) { + LD_UB8(src, src_stride, src1, src2, src3, src4, src5, src6, src7, src8); + src += (8 * src_stride); + LD_UB8(dst, dst_stride, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8); + + ILVR_B4_UB(src1, src0, src2, src1, src3, src2, src4, src3, vec0, vec1, vec2, + vec3); + ILVR_B4_UB(src5, src4, src6, src5, src7, src6, src8, src7, vec4, vec5, vec6, + vec7); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, tmp0, tmp1, + tmp2, tmp3); + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_AVG_ST8x4_UB(tmp0, dst1, tmp1, dst2, tmp2, dst3, tmp3, dst4, dst, + dst_stride); + dst += (4 * dst_stride); + + DOTP_UB4_UH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0, tmp0, tmp1, + tmp2, tmp3); + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_AVG_ST8x4_UB(tmp0, dst5, tmp1, dst6, tmp2, dst7, tmp3, dst8, dst, + dst_stride); + dst += (4 * dst_stride); + + src0 = src8; + } +} + +static void common_vt_2t_and_aver_dst_8w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (4 == height) { + common_vt_2t_and_aver_dst_8x4_msa(src, src_stride, dst, dst_stride, filter); + } else { + common_vt_2t_and_aver_dst_8x8mult_msa(src, src_stride, dst, dst_stride, + filter, height); + } +} + +static void common_vt_2t_and_aver_dst_16w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16u8 src0, src1, src2, src3, src4, dst0, dst1, dst2, dst3, filt0; + v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8u16 tmp0, tmp1, tmp2, tmp3, filt; + + /* rearranging filter_y */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + src0 = LD_UB(src); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + ILVR_B2_UB(src1, src0, src2, src1, vec0, vec2); + ILVL_B2_UB(src1, src0, src2, src1, vec1, vec3); + DOTP_UB2_UH(vec0, vec1, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp1, tmp0, dst0, dst); + dst += dst_stride; + + ILVR_B2_UB(src3, src2, src4, src3, vec4, vec6); + ILVL_B2_UB(src3, src2, src4, src3, vec5, vec7); + DOTP_UB2_UH(vec2, vec3, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp3, tmp2, dst1, dst); + dst += dst_stride; + + DOTP_UB2_UH(vec4, vec5, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp1, tmp0, dst2, dst); + dst += dst_stride; + + DOTP_UB2_UH(vec6, vec7, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp3, tmp2, dst3, dst); + dst += dst_stride; + + src0 = src4; + } +} + +static void common_vt_2t_and_aver_dst_32w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9; + v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + v8u16 tmp0, tmp1, tmp2, tmp3, filt; + + /* rearranging filter_y */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_UB2(src, 16, src0, src5); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + ILVR_B2_UB(src1, src0, src2, src1, vec0, vec2); + ILVL_B2_UB(src1, src0, src2, src1, vec1, vec3); + + LD_UB4(src + 16, src_stride, src6, src7, src8, src9); + LD_UB4(dst + 16, dst_stride, dst4, dst5, dst6, dst7); + src += (4 * src_stride); + + DOTP_UB2_UH(vec0, vec1, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp1, tmp0, dst0, dst); + + DOTP_UB2_UH(vec2, vec3, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp3, tmp2, dst1, dst + dst_stride); + + ILVR_B2_UB(src3, src2, src4, src3, vec4, vec6); + ILVL_B2_UB(src3, src2, src4, src3, vec5, vec7); + DOTP_UB2_UH(vec4, vec5, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp1, tmp0, dst2, dst + 2 * dst_stride); + + DOTP_UB2_UH(vec6, vec7, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp3, tmp2, dst3, dst + 3 * dst_stride); + + ILVR_B2_UB(src6, src5, src7, src6, vec0, vec2); + ILVL_B2_UB(src6, src5, src7, src6, vec1, vec3); + DOTP_UB2_UH(vec0, vec1, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp1, tmp0, dst4, dst + 16); + + DOTP_UB2_UH(vec2, vec3, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp3, tmp2, dst5, dst + 16 + dst_stride); + + ILVR_B2_UB(src8, src7, src9, src8, vec4, vec6); + ILVL_B2_UB(src8, src7, src9, src8, vec5, vec7); + DOTP_UB2_UH(vec4, vec5, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp1, tmp0, dst6, dst + 16 + 2 * dst_stride); + + DOTP_UB2_UH(vec6, vec7, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp3, tmp2, dst7, dst + 16 + 3 * dst_stride); + dst += (4 * dst_stride); + + src0 = src4; + src5 = src9; + } +} + +static void common_vt_2t_and_aver_dst_64w_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16u8 src0, src1, src2, src3, src4, src5; + v16u8 src6, src7, src8, src9, src10, src11, filt0; + v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8u16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + v8u16 filt; + + /* rearranging filter_y */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_UB4(src, 16, src0, src3, src6, src9); + src += src_stride; + + for (loop_cnt = (height >> 1); loop_cnt--;) { + LD_UB2(src, src_stride, src1, src2); + LD_UB2(dst, dst_stride, dst0, dst1); + LD_UB2(src + 16, src_stride, src4, src5); + LD_UB2(dst + 16, dst_stride, dst2, dst3); + LD_UB2(src + 32, src_stride, src7, src8); + LD_UB2(dst + 32, dst_stride, dst4, dst5); + LD_UB2(src + 48, src_stride, src10, src11); + LD_UB2(dst + 48, dst_stride, dst6, dst7); + src += (2 * src_stride); + + ILVR_B2_UB(src1, src0, src2, src1, vec0, vec2); + ILVL_B2_UB(src1, src0, src2, src1, vec1, vec3); + DOTP_UB2_UH(vec0, vec1, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp1, tmp0, dst0, dst); + + DOTP_UB2_UH(vec2, vec3, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp3, tmp2, dst1, dst + dst_stride); + + ILVR_B2_UB(src4, src3, src5, src4, vec4, vec6); + ILVL_B2_UB(src4, src3, src5, src4, vec5, vec7); + DOTP_UB2_UH(vec4, vec5, filt0, filt0, tmp4, tmp5); + SRARI_H2_UH(tmp4, tmp5, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp5, tmp4, dst2, dst + 16); + + DOTP_UB2_UH(vec6, vec7, filt0, filt0, tmp6, tmp7); + SRARI_H2_UH(tmp6, tmp7, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp7, tmp6, dst3, dst + 16 + dst_stride); + + ILVR_B2_UB(src7, src6, src8, src7, vec0, vec2); + ILVL_B2_UB(src7, src6, src8, src7, vec1, vec3); + DOTP_UB2_UH(vec0, vec1, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp1, tmp0, dst4, dst + 32); + + DOTP_UB2_UH(vec2, vec3, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp3, tmp2, dst5, dst + 32 + dst_stride); + + ILVR_B2_UB(src10, src9, src11, src10, vec4, vec6); + ILVL_B2_UB(src10, src9, src11, src10, vec5, vec7); + DOTP_UB2_UH(vec4, vec5, filt0, filt0, tmp4, tmp5); + SRARI_H2_UH(tmp4, tmp5, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp5, tmp4, dst6, (dst + 48)); + + DOTP_UB2_UH(vec6, vec7, filt0, filt0, tmp6, tmp7); + SRARI_H2_UH(tmp6, tmp7, FILTER_BITS); + PCKEV_AVG_ST_UB(tmp7, tmp6, dst7, dst + 48 + dst_stride); + dst += (2 * dst_stride); + + src0 = src2; + src3 = src5; + src6 = src8; + src9 = src11; + } +} + +void aom_convolve8_avg_vert_msa(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + int8_t cnt, filt_ver[8]; + + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_ver[cnt] = filter_y[cnt]; + } + + if (((const int32_t *)filter_y)[0] == 0) { + switch (w) { + case 4: + common_vt_2t_and_aver_dst_4w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + case 8: + common_vt_2t_and_aver_dst_8w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + case 16: + common_vt_2t_and_aver_dst_16w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + case 32: + common_vt_2t_and_aver_dst_32w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + case 64: + common_vt_2t_and_aver_dst_64w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + default: + aom_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } else { + switch (w) { + case 4: + common_vt_8t_and_aver_dst_4w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + break; + case 8: + common_vt_8t_and_aver_dst_8w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + break; + case 16: + common_vt_8t_and_aver_dst_16w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + + break; + case 32: + common_vt_8t_and_aver_dst_32w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + break; + case 64: + common_vt_8t_and_aver_dst_64w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + break; + default: + aom_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } +} diff --git a/third_party/aom/aom_dsp/mips/aom_convolve8_horiz_msa.c b/third_party/aom/aom_dsp/mips/aom_convolve8_horiz_msa.c new file mode 100644 index 0000000000..fc3a823c57 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/aom_convolve8_horiz_msa.c @@ -0,0 +1,692 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/aom_convolve_msa.h" + +static void common_hz_8t_4x4_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + v16u8 mask0, mask1, mask2, mask3, out; + v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3; + v8i16 filt, out0, out1; + + mask0 = LD_UB(&mc_filt_mask_arr[16]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + LD_SB4(src, src_stride, src0, src1, src2, src3); + XORI_B4_128_SB(src0, src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filt0, filt1, filt2, filt3, out0, out1); + SRARI_H2_SH(out0, out1, FILTER_BITS); + SAT_SH2_SH(out0, out1, 7); + out = PCKEV_XORI128_UB(out0, out1); + ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride); +} + +static void common_hz_8t_4x8_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + v16i8 filt0, filt1, filt2, filt3; + v16i8 src0, src1, src2, src3; + v16u8 mask0, mask1, mask2, mask3, out; + v8i16 filt, out0, out1, out2, out3; + + mask0 = LD_UB(&mc_filt_mask_arr[16]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + LD_SB4(src, src_stride, src0, src1, src2, src3); + XORI_B4_128_SB(src0, src1, src2, src3); + src += (4 * src_stride); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filt0, filt1, filt2, filt3, out0, out1); + LD_SB4(src, src_stride, src0, src1, src2, src3); + XORI_B4_128_SB(src0, src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filt0, filt1, filt2, filt3, out2, out3); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + out = PCKEV_XORI128_UB(out0, out1); + ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride); + dst += (4 * dst_stride); + out = PCKEV_XORI128_UB(out2, out3); + ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride); +} + +static void common_hz_8t_4w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (4 == height) { + common_hz_8t_4x4_msa(src, src_stride, dst, dst_stride, filter); + } else if (8 == height) { + common_hz_8t_4x8_msa(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_8t_8x4_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3; + v16u8 mask0, mask1, mask2, mask3, tmp0, tmp1; + v8i16 filt, out0, out1, out2, out3; + + mask0 = LD_UB(&mc_filt_mask_arr[0]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + LD_SB4(src, src_stride, src0, src1, src2, src3); + XORI_B4_128_SB(src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filt0, filt1, filt2, filt3, out0, out1, out2, + out3); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + tmp0 = PCKEV_XORI128_UB(out0, out1); + tmp1 = PCKEV_XORI128_UB(out2, out3); + ST8x4_UB(tmp0, tmp1, dst, dst_stride); +} + +static void common_hz_8t_8x8mult_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3; + v16u8 mask0, mask1, mask2, mask3, tmp0, tmp1; + v8i16 filt, out0, out1, out2, out3; + + mask0 = LD_UB(&mc_filt_mask_arr[0]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src0, src1, src2, src3); + XORI_B4_128_SB(src0, src1, src2, src3); + src += (4 * src_stride); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filt0, filt1, filt2, filt3, out0, out1, + out2, out3); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + tmp0 = PCKEV_XORI128_UB(out0, out1); + tmp1 = PCKEV_XORI128_UB(out2, out3); + ST8x4_UB(tmp0, tmp1, dst, dst_stride); + dst += (4 * dst_stride); + } +} + +static void common_hz_8t_8w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (4 == height) { + common_hz_8t_8x4_msa(src, src_stride, dst, dst_stride, filter); + } else { + common_hz_8t_8x8mult_msa(src, src_stride, dst, dst_stride, filter, height); + } +} + +static void common_hz_8t_16w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3; + v16u8 mask0, mask1, mask2, mask3, out; + v8i16 filt, out0, out1, out2, out3; + + mask0 = LD_UB(&mc_filt_mask_arr[0]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + for (loop_cnt = (height >> 1); loop_cnt--;) { + LD_SB2(src, src_stride, src0, src2); + LD_SB2(src + 8, src_stride, src1, src3); + XORI_B4_128_SB(src0, src1, src2, src3); + src += (2 * src_stride); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filt0, filt1, filt2, filt3, out0, out1, + out2, out3); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + out = PCKEV_XORI128_UB(out0, out1); + ST_UB(out, dst); + dst += dst_stride; + out = PCKEV_XORI128_UB(out2, out3); + ST_UB(out, dst); + dst += dst_stride; + } +} + +static void common_hz_8t_32w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3; + v16u8 mask0, mask1, mask2, mask3, out; + v8i16 filt, out0, out1, out2, out3; + + mask0 = LD_UB(&mc_filt_mask_arr[0]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + for (loop_cnt = (height >> 1); loop_cnt--;) { + src0 = LD_SB(src); + src2 = LD_SB(src + 16); + src3 = LD_SB(src + 24); + src1 = __msa_sldi_b(src2, src0, 8); + src += src_stride; + XORI_B4_128_SB(src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filt0, filt1, filt2, filt3, out0, out1, + out2, out3); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + + src0 = LD_SB(src); + src2 = LD_SB(src + 16); + src3 = LD_SB(src + 24); + src1 = __msa_sldi_b(src2, src0, 8); + src += src_stride; + + out = PCKEV_XORI128_UB(out0, out1); + ST_UB(out, dst); + out = PCKEV_XORI128_UB(out2, out3); + ST_UB(out, dst + 16); + dst += dst_stride; + + XORI_B4_128_SB(src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filt0, filt1, filt2, filt3, out0, out1, + out2, out3); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + out = PCKEV_XORI128_UB(out0, out1); + ST_UB(out, dst); + out = PCKEV_XORI128_UB(out2, out3); + ST_UB(out, dst + 16); + dst += dst_stride; + } +} + +static void common_hz_8t_64w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + int32_t loop_cnt; + v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3; + v16u8 mask0, mask1, mask2, mask3, out; + v8i16 filt, out0, out1, out2, out3; + + mask0 = LD_UB(&mc_filt_mask_arr[0]); + src -= 3; + + /* rearranging filter */ + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + for (loop_cnt = height; loop_cnt--;) { + src0 = LD_SB(src); + src2 = LD_SB(src + 16); + src3 = LD_SB(src + 24); + src1 = __msa_sldi_b(src2, src0, 8); + + XORI_B4_128_SB(src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filt0, filt1, filt2, filt3, out0, out1, + out2, out3); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + out = PCKEV_XORI128_UB(out0, out1); + ST_UB(out, dst); + out = PCKEV_XORI128_UB(out2, out3); + ST_UB(out, dst + 16); + + src0 = LD_SB(src + 32); + src2 = LD_SB(src + 48); + src3 = LD_SB(src + 56); + src1 = __msa_sldi_b(src2, src0, 8); + src += src_stride; + + XORI_B4_128_SB(src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filt0, filt1, filt2, filt3, out0, out1, + out2, out3); + SRARI_H4_SH(out0, out1, out2, out3, FILTER_BITS); + SAT_SH4_SH(out0, out1, out2, out3, 7); + out = PCKEV_XORI128_UB(out0, out1); + ST_UB(out, dst + 32); + out = PCKEV_XORI128_UB(out2, out3); + ST_UB(out, dst + 48); + dst += dst_stride; + } +} + +static void common_hz_2t_4x4_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + v16i8 src0, src1, src2, src3, mask; + v16u8 filt0, vec0, vec1, res0, res1; + v8u16 vec2, vec3, filt; + + mask = LD_SB(&mc_filt_mask_arr[16]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB4(src, src_stride, src0, src1, src2, src3); + VSHF_B2_UB(src0, src1, src2, src3, mask, mask, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt0, filt0, vec2, vec3); + SRARI_H2_UH(vec2, vec3, FILTER_BITS); + PCKEV_B2_UB(vec2, vec2, vec3, vec3, res0, res1); + ST4x4_UB(res0, res1, 0, 1, 0, 1, dst, dst_stride); +} + +static void common_hz_2t_4x8_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + v16u8 vec0, vec1, vec2, vec3, filt0; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask; + v16i8 res0, res1, res2, res3; + v8u16 vec4, vec5, vec6, vec7, filt; + + mask = LD_SB(&mc_filt_mask_arr[16]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + VSHF_B2_UB(src0, src1, src2, src3, mask, mask, vec0, vec1); + VSHF_B2_UB(src4, src5, src6, src7, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec4, vec5, + vec6, vec7); + SRARI_H4_UH(vec4, vec5, vec6, vec7, FILTER_BITS); + PCKEV_B4_SB(vec4, vec4, vec5, vec5, vec6, vec6, vec7, vec7, res0, res1, res2, + res3); + ST4x4_UB(res0, res1, 0, 1, 0, 1, dst, dst_stride); + dst += (4 * dst_stride); + ST4x4_UB(res2, res3, 0, 1, 0, 1, dst, dst_stride); +} + +static void common_hz_2t_4w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (4 == height) { + common_hz_2t_4x4_msa(src, src_stride, dst, dst_stride, filter); + } else if (8 == height) { + common_hz_2t_4x8_msa(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_2t_8x4_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + v16u8 filt0; + v16i8 src0, src1, src2, src3, mask; + v8u16 vec0, vec1, vec2, vec3, filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB4(src, src_stride, src0, src1, src2, src3); + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + PCKEV_B2_SB(vec1, vec0, vec3, vec2, src0, src1); + ST8x4_UB(src0, src1, dst, dst_stride); +} + +static void common_hz_2t_8x8mult_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + v16u8 filt0; + v16i8 src0, src1, src2, src3, mask, out0, out1; + v8u16 vec0, vec1, vec2, vec3, filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + + LD_SB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + + PCKEV_B2_SB(vec1, vec0, vec3, vec2, out0, out1); + ST8x4_UB(out0, out1, dst, dst_stride); + dst += (4 * dst_stride); + + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + PCKEV_B2_SB(vec1, vec0, vec3, vec2, out0, out1); + ST8x4_UB(out0, out1, dst, dst_stride); + dst += (4 * dst_stride); + + if (16 == height) { + LD_SB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + LD_SB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + + PCKEV_B2_SB(vec1, vec0, vec3, vec2, out0, out1); + ST8x4_UB(out0, out1, dst, dst_stride); + + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + PCKEV_B2_SB(vec1, vec0, vec3, vec2, out0, out1); + ST8x4_UB(out0, out1, dst + 4 * dst_stride, dst_stride); + } +} + +static void common_hz_2t_8w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (4 == height) { + common_hz_2t_8x4_msa(src, src_stride, dst, dst_stride, filter); + } else { + common_hz_2t_8x8mult_msa(src, src_stride, dst, dst_stride, filter, height); + } +} + +static void common_hz_2t_16w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask; + v16u8 filt0, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8u16 out0, out1, out2, out3, out4, out5, out6, out7, filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + loop_cnt = (height >> 2) - 1; + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB4(src, src_stride, src0, src2, src4, src6); + LD_SB4(src + 8, src_stride, src1, src3, src5, src7); + src += (4 * src_stride); + + VSHF_B2_UB(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UB(src2, src2, src3, src3, mask, mask, vec2, vec3); + VSHF_B2_UB(src4, src4, src5, src5, mask, mask, vec4, vec5); + VSHF_B2_UB(src6, src6, src7, src7, mask, mask, vec6, vec7); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, out0, out1, + out2, out3); + DOTP_UB4_UH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0, out4, out5, + out6, out7); + SRARI_H4_UH(out0, out1, out2, out3, FILTER_BITS); + SRARI_H4_UH(out4, out5, out6, out7, FILTER_BITS); + PCKEV_ST_SB(out0, out1, dst); + dst += dst_stride; + PCKEV_ST_SB(out2, out3, dst); + dst += dst_stride; + PCKEV_ST_SB(out4, out5, dst); + dst += dst_stride; + PCKEV_ST_SB(out6, out7, dst); + dst += dst_stride; + + for (; loop_cnt--;) { + LD_SB4(src, src_stride, src0, src2, src4, src6); + LD_SB4(src + 8, src_stride, src1, src3, src5, src7); + src += (4 * src_stride); + + VSHF_B2_UB(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UB(src2, src2, src3, src3, mask, mask, vec2, vec3); + VSHF_B2_UB(src4, src4, src5, src5, mask, mask, vec4, vec5); + VSHF_B2_UB(src6, src6, src7, src7, mask, mask, vec6, vec7); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, out0, out1, + out2, out3); + DOTP_UB4_UH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0, out4, out5, + out6, out7); + SRARI_H4_UH(out0, out1, out2, out3, FILTER_BITS); + SRARI_H4_UH(out4, out5, out6, out7, FILTER_BITS); + PCKEV_ST_SB(out0, out1, dst); + dst += dst_stride; + PCKEV_ST_SB(out2, out3, dst); + dst += dst_stride; + PCKEV_ST_SB(out4, out5, dst); + dst += dst_stride; + PCKEV_ST_SB(out6, out7, dst); + dst += dst_stride; + } +} + +static void common_hz_2t_32w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask; + v16u8 filt0, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8u16 out0, out1, out2, out3, out4, out5, out6, out7, filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + for (loop_cnt = height >> 1; loop_cnt--;) { + src0 = LD_SB(src); + src2 = LD_SB(src + 16); + src3 = LD_SB(src + 24); + src1 = __msa_sldi_b(src2, src0, 8); + src += src_stride; + src4 = LD_SB(src); + src6 = LD_SB(src + 16); + src7 = LD_SB(src + 24); + src5 = __msa_sldi_b(src6, src4, 8); + src += src_stride; + + VSHF_B2_UB(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UB(src2, src2, src3, src3, mask, mask, vec2, vec3); + VSHF_B2_UB(src4, src4, src5, src5, mask, mask, vec4, vec5); + VSHF_B2_UB(src6, src6, src7, src7, mask, mask, vec6, vec7); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, out0, out1, + out2, out3); + DOTP_UB4_UH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0, out4, out5, + out6, out7); + SRARI_H4_UH(out0, out1, out2, out3, FILTER_BITS); + SRARI_H4_UH(out4, out5, out6, out7, FILTER_BITS); + PCKEV_ST_SB(out0, out1, dst); + PCKEV_ST_SB(out2, out3, dst + 16); + dst += dst_stride; + PCKEV_ST_SB(out4, out5, dst); + PCKEV_ST_SB(out6, out7, dst + 16); + dst += dst_stride; + } +} + +static void common_hz_2t_64w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask; + v16u8 filt0, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8u16 out0, out1, out2, out3, out4, out5, out6, out7, filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_UH(filter); + filt0 = (v16u8)__msa_splati_h((v8i16)filt, 0); + + for (loop_cnt = height; loop_cnt--;) { + src0 = LD_SB(src); + src2 = LD_SB(src + 16); + src4 = LD_SB(src + 32); + src6 = LD_SB(src + 48); + src7 = LD_SB(src + 56); + SLDI_B3_SB(src2, src4, src6, src0, src2, src4, src1, src3, src5, 8); + src += src_stride; + + VSHF_B2_UB(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UB(src2, src2, src3, src3, mask, mask, vec2, vec3); + VSHF_B2_UB(src4, src4, src5, src5, mask, mask, vec4, vec5); + VSHF_B2_UB(src6, src6, src7, src7, mask, mask, vec6, vec7); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, out0, out1, + out2, out3); + DOTP_UB4_UH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0, out4, out5, + out6, out7); + SRARI_H4_UH(out0, out1, out2, out3, FILTER_BITS); + SRARI_H4_UH(out4, out5, out6, out7, FILTER_BITS); + PCKEV_ST_SB(out0, out1, dst); + PCKEV_ST_SB(out2, out3, dst + 16); + PCKEV_ST_SB(out4, out5, dst + 32); + PCKEV_ST_SB(out6, out7, dst + 48); + dst += dst_stride; + } +} + +void aom_convolve8_horiz_msa(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + int8_t cnt, filt_hor[8]; + + assert(x_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_hor[cnt] = filter_x[cnt]; + } + + if (((const int32_t *)filter_x)[0] == 0) { + switch (w) { + case 4: + common_hz_2t_4w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + case 8: + common_hz_2t_8w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + case 16: + common_hz_2t_16w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + case 32: + common_hz_2t_32w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + case 64: + common_hz_2t_64w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + default: + aom_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } else { + switch (w) { + case 4: + common_hz_8t_4w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + case 8: + common_hz_8t_8w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + case 16: + common_hz_8t_16w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + case 32: + common_hz_8t_32w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + case 64: + common_hz_8t_64w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + default: + aom_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } +} diff --git a/third_party/aom/aom_dsp/mips/aom_convolve8_msa.c b/third_party/aom/aom_dsp/mips/aom_convolve8_msa.c new file mode 100644 index 0000000000..a4d594931a --- /dev/null +++ b/third_party/aom/aom_dsp/mips/aom_convolve8_msa.c @@ -0,0 +1,630 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/aom_convolve_msa.h" + +const uint8_t mc_filt_mask_arr[16 * 3] = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + +static void common_hv_8ht_8vt_4w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + v16i8 filt_hz0, filt_hz1, filt_hz2, filt_hz3; + v16u8 mask0, mask1, mask2, mask3, out; + v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + v8i16 hz_out7, hz_out8, hz_out9, tmp0, tmp1, out0, out1, out2, out3, out4; + v8i16 filt, filt_vt0, filt_vt1, filt_vt2, filt_vt3; + + mask0 = LD_UB(&mc_filt_mask_arr[16]); + src -= (3 + 3 * src_stride); + + /* rearranging filter */ + filt = LD_SH(filter_horiz); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6); + XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6); + src += (7 * src_stride); + + hz_out0 = HORIZ_8TAP_FILT(src0, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out2 = HORIZ_8TAP_FILT(src2, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out4 = HORIZ_8TAP_FILT(src4, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out5 = HORIZ_8TAP_FILT(src5, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + SLDI_B2_SH(hz_out2, hz_out4, hz_out0, hz_out2, hz_out1, hz_out3, 8); + + filt = LD_SH(filter_vert); + SPLATI_H4_SH(filt, 0, 1, 2, 3, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + + ILVEV_B2_SH(hz_out0, hz_out1, hz_out2, hz_out3, out0, out1); + out2 = (v8i16)__msa_ilvev_b((v16i8)hz_out5, (v16i8)hz_out4); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src7, src8, src9, src10); + XORI_B4_128_SB(src7, src8, src9, src10); + src += (4 * src_stride); + + hz_out7 = HORIZ_8TAP_FILT(src7, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out6 = (v8i16)__msa_sldi_b((v16i8)hz_out7, (v16i8)hz_out5, 8); + out3 = (v8i16)__msa_ilvev_b((v16i8)hz_out7, (v16i8)hz_out6); + tmp0 = FILT_8TAP_DPADD_S_H(out0, out1, out2, out3, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + + hz_out9 = HORIZ_8TAP_FILT(src9, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out8 = (v8i16)__msa_sldi_b((v16i8)hz_out9, (v16i8)hz_out7, 8); + out4 = (v8i16)__msa_ilvev_b((v16i8)hz_out9, (v16i8)hz_out8); + tmp1 = FILT_8TAP_DPADD_S_H(out1, out2, out3, out4, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + SRARI_H2_SH(tmp0, tmp1, FILTER_BITS); + SAT_SH2_SH(tmp0, tmp1, 7); + out = PCKEV_XORI128_UB(tmp0, tmp1); + ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride); + dst += (4 * dst_stride); + + hz_out5 = hz_out9; + out0 = out2; + out1 = out3; + out2 = out4; + } +} + +static void common_hv_8ht_8vt_8w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + v16i8 filt_hz0, filt_hz1, filt_hz2, filt_hz3; + v16u8 mask0, mask1, mask2, mask3, vec0, vec1; + v8i16 filt, filt_vt0, filt_vt1, filt_vt2, filt_vt3; + v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + v8i16 hz_out7, hz_out8, hz_out9, hz_out10, tmp0, tmp1, tmp2, tmp3; + v8i16 out0, out1, out2, out3, out4, out5, out6, out7, out8, out9; + + mask0 = LD_UB(&mc_filt_mask_arr[0]); + src -= (3 + 3 * src_stride); + + /* rearranging filter */ + filt = LD_SH(filter_horiz); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + + mask1 = mask0 + 2; + mask2 = mask0 + 4; + mask3 = mask0 + 6; + + LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6); + src += (7 * src_stride); + + XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6); + hz_out0 = HORIZ_8TAP_FILT(src0, src0, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out1 = HORIZ_8TAP_FILT(src1, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out2 = HORIZ_8TAP_FILT(src2, src2, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out3 = HORIZ_8TAP_FILT(src3, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out4 = HORIZ_8TAP_FILT(src4, src4, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out5 = HORIZ_8TAP_FILT(src5, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + hz_out6 = HORIZ_8TAP_FILT(src6, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + + filt = LD_SH(filter_vert); + SPLATI_H4_SH(filt, 0, 1, 2, 3, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + + ILVEV_B2_SH(hz_out0, hz_out1, hz_out2, hz_out3, out0, out1); + ILVEV_B2_SH(hz_out4, hz_out5, hz_out1, hz_out2, out2, out4); + ILVEV_B2_SH(hz_out3, hz_out4, hz_out5, hz_out6, out5, out6); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src7, src8, src9, src10); + src += (4 * src_stride); + + XORI_B4_128_SB(src7, src8, src9, src10); + + hz_out7 = HORIZ_8TAP_FILT(src7, src7, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + out3 = (v8i16)__msa_ilvev_b((v16i8)hz_out7, (v16i8)hz_out6); + tmp0 = FILT_8TAP_DPADD_S_H(out0, out1, out2, out3, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + + hz_out8 = HORIZ_8TAP_FILT(src8, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + out7 = (v8i16)__msa_ilvev_b((v16i8)hz_out8, (v16i8)hz_out7); + tmp1 = FILT_8TAP_DPADD_S_H(out4, out5, out6, out7, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + + hz_out9 = HORIZ_8TAP_FILT(src9, src9, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + out8 = (v8i16)__msa_ilvev_b((v16i8)hz_out9, (v16i8)hz_out8); + tmp2 = FILT_8TAP_DPADD_S_H(out1, out2, out3, out8, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + + hz_out10 = HORIZ_8TAP_FILT(src10, src10, mask0, mask1, mask2, mask3, + filt_hz0, filt_hz1, filt_hz2, filt_hz3); + out9 = (v8i16)__msa_ilvev_b((v16i8)hz_out10, (v16i8)hz_out9); + tmp3 = FILT_8TAP_DPADD_S_H(out5, out6, out7, out9, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + SRARI_H4_SH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3, 7); + vec0 = PCKEV_XORI128_UB(tmp0, tmp1); + vec1 = PCKEV_XORI128_UB(tmp2, tmp3); + ST8x4_UB(vec0, vec1, dst, dst_stride); + dst += (4 * dst_stride); + + hz_out6 = hz_out10; + out0 = out2; + out1 = out3; + out2 = out8; + out4 = out6; + out5 = out7; + out6 = out9; + } +} + +static void common_hv_8ht_8vt_16w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 2; multiple8_cnt--;) { + common_hv_8ht_8vt_8w_msa(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_8ht_8vt_32w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_8ht_8vt_8w_msa(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_8ht_8vt_64w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 8; multiple8_cnt--;) { + common_hv_8ht_8vt_8w_msa(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_2ht_2vt_4x4_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, + int8_t *filter_vert) { + v16i8 src0, src1, src2, src3, src4, mask; + v16u8 filt_vt, filt_hz, vec0, vec1, res0, res1; + v8u16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, filt, tmp0, tmp1; + + mask = LD_SB(&mc_filt_mask_arr[16]); + + /* rearranging filter */ + filt = LD_UH(filter_horiz); + filt_hz = (v16u8)__msa_splati_h((v8i16)filt, 0); + + filt = LD_UH(filter_vert); + filt_vt = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB5(src, src_stride, src0, src1, src2, src3, src4); + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out1 = (v8u16)__msa_sldi_b((v16i8)hz_out2, (v16i8)hz_out0, 8); + hz_out3 = (v8u16)__msa_pckod_d((v2i64)hz_out4, (v2i64)hz_out2); + + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_B2_UB(tmp0, tmp0, tmp1, tmp1, res0, res1); + ST4x4_UB(res0, res1, 0, 1, 0, 1, dst, dst_stride); +} + +static void common_hv_2ht_2vt_4x8_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, + int8_t *filter_vert) { + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, mask; + v16i8 res0, res1, res2, res3; + v16u8 filt_hz, filt_vt, vec0, vec1, vec2, vec3; + v8u16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + v8u16 hz_out7, hz_out8, vec4, vec5, vec6, vec7, filt; + + mask = LD_SB(&mc_filt_mask_arr[16]); + + /* rearranging filter */ + filt = LD_UH(filter_horiz); + filt_hz = (v16u8)__msa_splati_h((v8i16)filt, 0); + + filt = LD_UH(filter_vert); + filt_vt = (v16u8)__msa_splati_h((v8i16)filt, 0); + + LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + src += (8 * src_stride); + src8 = LD_SB(src); + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src5, mask, filt_hz, FILTER_BITS); + hz_out6 = HORIZ_2TAP_FILT_UH(src6, src7, mask, filt_hz, FILTER_BITS); + hz_out8 = HORIZ_2TAP_FILT_UH(src8, src8, mask, filt_hz, FILTER_BITS); + SLDI_B3_UH(hz_out2, hz_out4, hz_out6, hz_out0, hz_out2, hz_out4, hz_out1, + hz_out3, hz_out5, 8); + hz_out7 = (v8u16)__msa_pckod_d((v2i64)hz_out8, (v2i64)hz_out6); + + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + ILVEV_B2_UB(hz_out4, hz_out5, hz_out6, hz_out7, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt_vt, filt_vt, filt_vt, filt_vt, vec4, + vec5, vec6, vec7); + SRARI_H4_UH(vec4, vec5, vec6, vec7, FILTER_BITS); + PCKEV_B4_SB(vec4, vec4, vec5, vec5, vec6, vec6, vec7, vec7, res0, res1, res2, + res3); + ST4x4_UB(res0, res1, 0, 1, 0, 1, dst, dst_stride); + dst += (4 * dst_stride); + ST4x4_UB(res2, res3, 0, 1, 0, 1, dst, dst_stride); +} + +static void common_hv_2ht_2vt_4w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + if (4 == height) { + common_hv_2ht_2vt_4x4_msa(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert); + } else if (8 == height) { + common_hv_2ht_2vt_4x8_msa(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert); + } +} + +static void common_hv_2ht_2vt_8x4_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, + int8_t *filter_vert) { + v16i8 src0, src1, src2, src3, src4, mask, out0, out1; + v16u8 filt_hz, filt_vt, vec0, vec1, vec2, vec3; + v8u16 hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3; + v8i16 filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_SH(filter_horiz); + filt_hz = (v16u8)__msa_splati_h(filt, 0); + + filt = LD_SH(filter_vert); + filt_vt = (v16u8)__msa_splati_h(filt, 0); + + LD_SB5(src, src_stride, src0, src1, src2, src3, src4); + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp0 = __msa_dotp_u_h(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec1 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp1 = __msa_dotp_u_h(vec1, filt_vt); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec2 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp2 = __msa_dotp_u_h(vec2, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec3 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp3 = __msa_dotp_u_h(vec3, filt_vt); + + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_B2_SB(tmp1, tmp0, tmp3, tmp2, out0, out1); + ST8x4_UB(out0, out1, dst, dst_stride); +} + +static void common_hv_2ht_2vt_8x8mult_msa(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter_horiz, + int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, mask, out0, out1; + v16u8 filt_hz, filt_vt, vec0; + v8u16 hz_out0, hz_out1, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; + v8i16 filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_SH(filter_horiz); + filt_hz = (v16u8)__msa_splati_h(filt, 0); + + filt = LD_SH(filter_vert); + filt_vt = (v16u8)__msa_splati_h(filt, 0); + + src0 = LD_SB(src); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + + for (loop_cnt = (height >> 3); loop_cnt--;) { + LD_SB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp1 = __msa_dotp_u_h(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp2 = __msa_dotp_u_h(vec0, filt_vt); + + SRARI_H2_UH(tmp1, tmp2, FILTER_BITS); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp3 = __msa_dotp_u_h(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + LD_SB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp4 = __msa_dotp_u_h(vec0, filt_vt); + + SRARI_H2_UH(tmp3, tmp4, FILTER_BITS); + PCKEV_B2_SB(tmp2, tmp1, tmp4, tmp3, out0, out1); + ST8x4_UB(out0, out1, dst, dst_stride); + dst += (4 * dst_stride); + + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp5 = __msa_dotp_u_h(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp6 = __msa_dotp_u_h(vec0, filt_vt); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp7 = __msa_dotp_u_h(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp8 = __msa_dotp_u_h(vec0, filt_vt); + + SRARI_H4_UH(tmp5, tmp6, tmp7, tmp8, FILTER_BITS); + PCKEV_B2_SB(tmp6, tmp5, tmp8, tmp7, out0, out1); + ST8x4_UB(out0, out1, dst, dst_stride); + dst += (4 * dst_stride); + } +} + +static void common_hv_2ht_2vt_8w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + if (4 == height) { + common_hv_2ht_2vt_8x4_msa(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert); + } else { + common_hv_2ht_2vt_8x8mult_msa(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + } +} + +static void common_hv_2ht_2vt_16w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask; + v16u8 filt_hz, filt_vt, vec0, vec1; + v8u16 tmp1, tmp2, hz_out0, hz_out1, hz_out2, hz_out3; + v8i16 filt; + + mask = LD_SB(&mc_filt_mask_arr[0]); + + /* rearranging filter */ + filt = LD_SH(filter_horiz); + filt_hz = (v16u8)__msa_splati_h(filt, 0); + + filt = LD_SH(filter_vert); + filt_vt = (v16u8)__msa_splati_h(filt, 0); + + LD_SB2(src, 8, src0, src1); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src0, src2, src4, src6); + LD_SB4(src + 8, src_stride, src1, src3, src5, src7); + src += (4 * src_stride); + + hz_out1 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp1, tmp2); + SRARI_H2_UH(tmp1, tmp2, FILTER_BITS); + PCKEV_ST_SB(tmp1, tmp2, dst); + dst += dst_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp1, tmp2); + SRARI_H2_UH(tmp1, tmp2, FILTER_BITS); + PCKEV_ST_SB(tmp1, tmp2, dst); + dst += dst_stride; + + hz_out1 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp1, tmp2); + SRARI_H2_UH(tmp1, tmp2, FILTER_BITS); + PCKEV_ST_SB(tmp1, tmp2, dst); + dst += dst_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp1, tmp2); + SRARI_H2_UH(tmp1, tmp2, FILTER_BITS); + PCKEV_ST_SB(tmp1, tmp2, dst); + dst += dst_stride; + } +} + +static void common_hv_2ht_2vt_32w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 2; multiple8_cnt--;) { + common_hv_2ht_2vt_16w_msa(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 16; + dst += 16; + } +} + +static void common_hv_2ht_2vt_64w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_2ht_2vt_16w_msa(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 16; + dst += 16; + } +} + +void aom_convolve8_msa(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, + int32_t x_step_q4, const int16_t *filter_y, + int32_t y_step_q4, int32_t w, int32_t h) { + int8_t cnt, filt_hor[8], filt_ver[8]; + + assert(x_step_q4 == 16); + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_hor[cnt] = filter_x[cnt]; + filt_ver[cnt] = filter_y[cnt]; + } + + if (((const int32_t *)filter_x)[0] == 0 && + ((const int32_t *)filter_y)[0] == 0) { + switch (w) { + case 4: + common_hv_2ht_2vt_4w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + case 8: + common_hv_2ht_2vt_8w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + case 16: + common_hv_2ht_2vt_16w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + case 32: + common_hv_2ht_2vt_32w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + case 64: + common_hv_2ht_2vt_64w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + default: + aom_convolve8_c(src, src_stride, dst, dst_stride, filter_x, x_step_q4, + filter_y, y_step_q4, w, h); + break; + } + } else if (((const int32_t *)filter_x)[0] == 0 || + ((const int32_t *)filter_y)[0] == 0) { + aom_convolve8_c(src, src_stride, dst, dst_stride, filter_x, x_step_q4, + filter_y, y_step_q4, w, h); + } else { + switch (w) { + case 4: + common_hv_8ht_8vt_4w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + case 8: + common_hv_8ht_8vt_8w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + case 16: + common_hv_8ht_8vt_16w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + case 32: + common_hv_8ht_8vt_32w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + case 64: + common_hv_8ht_8vt_64w_msa(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + default: + aom_convolve8_c(src, src_stride, dst, dst_stride, filter_x, x_step_q4, + filter_y, y_step_q4, w, h); + break; + } + } +} diff --git a/third_party/aom/aom_dsp/mips/aom_convolve8_vert_msa.c b/third_party/aom/aom_dsp/mips/aom_convolve8_vert_msa.c new file mode 100644 index 0000000000..f7bdfc2bd7 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/aom_convolve8_vert_msa.c @@ -0,0 +1,699 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/aom_convolve_msa.h" + +static void common_vt_8t_4w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r; + v16i8 src65_r, src87_r, src109_r, src2110, src4332, src6554, src8776; + v16i8 src10998, filt0, filt1, filt2, filt3; + v16u8 out; + v8i16 filt, out10, out32; + + src -= (3 * src_stride); + + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6); + src += (7 * src_stride); + + ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_r, src32_r, + src54_r, src21_r); + ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r); + ILVR_D3_SB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r, src2110, + src4332, src6554); + XORI_B3_128_SB(src2110, src4332, src6554); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src7, src8, src9, src10); + src += (4 * src_stride); + + ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r, + src87_r, src98_r, src109_r); + ILVR_D2_SB(src87_r, src76_r, src109_r, src98_r, src8776, src10998); + XORI_B2_128_SB(src8776, src10998); + out10 = FILT_8TAP_DPADD_S_H(src2110, src4332, src6554, src8776, filt0, + filt1, filt2, filt3); + out32 = FILT_8TAP_DPADD_S_H(src4332, src6554, src8776, src10998, filt0, + filt1, filt2, filt3); + SRARI_H2_SH(out10, out32, FILTER_BITS); + SAT_SH2_SH(out10, out32, 7); + out = PCKEV_XORI128_UB(out10, out32); + ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride); + dst += (4 * dst_stride); + + src2110 = src6554; + src4332 = src8776; + src6554 = src10998; + src6 = src10; + } +} + +static void common_vt_8t_8w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r; + v16i8 src65_r, src87_r, src109_r, filt0, filt1, filt2, filt3; + v16u8 tmp0, tmp1; + v8i16 filt, out0_r, out1_r, out2_r, out3_r; + + src -= (3 * src_stride); + + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6); + XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6); + src += (7 * src_stride); + ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_r, src32_r, + src54_r, src21_r); + ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src7, src8, src9, src10); + XORI_B4_128_SB(src7, src8, src9, src10); + src += (4 * src_stride); + + ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r, + src87_r, src98_r, src109_r); + out0_r = FILT_8TAP_DPADD_S_H(src10_r, src32_r, src54_r, src76_r, filt0, + filt1, filt2, filt3); + out1_r = FILT_8TAP_DPADD_S_H(src21_r, src43_r, src65_r, src87_r, filt0, + filt1, filt2, filt3); + out2_r = FILT_8TAP_DPADD_S_H(src32_r, src54_r, src76_r, src98_r, filt0, + filt1, filt2, filt3); + out3_r = FILT_8TAP_DPADD_S_H(src43_r, src65_r, src87_r, src109_r, filt0, + filt1, filt2, filt3); + SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, FILTER_BITS); + SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7); + tmp0 = PCKEV_XORI128_UB(out0_r, out1_r); + tmp1 = PCKEV_XORI128_UB(out2_r, out3_r); + ST8x4_UB(tmp0, tmp1, dst, dst_stride); + dst += (4 * dst_stride); + + src10_r = src54_r; + src32_r = src76_r; + src54_r = src98_r; + src21_r = src65_r; + src43_r = src87_r; + src65_r = src109_r; + src6 = src10; + } +} + +static void common_vt_8t_16w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + v16i8 filt0, filt1, filt2, filt3; + v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r; + v16i8 src65_r, src87_r, src109_r, src10_l, src32_l, src54_l, src76_l; + v16i8 src98_l, src21_l, src43_l, src65_l, src87_l, src109_l; + v16u8 tmp0, tmp1, tmp2, tmp3; + v8i16 filt, out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l; + + src -= (3 * src_stride); + + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6); + XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6); + src += (7 * src_stride); + ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_r, src32_r, + src54_r, src21_r); + ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r); + ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_l, src32_l, + src54_l, src21_l); + ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src7, src8, src9, src10); + XORI_B4_128_SB(src7, src8, src9, src10); + src += (4 * src_stride); + + ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r, + src87_r, src98_r, src109_r); + ILVL_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_l, + src87_l, src98_l, src109_l); + out0_r = FILT_8TAP_DPADD_S_H(src10_r, src32_r, src54_r, src76_r, filt0, + filt1, filt2, filt3); + out1_r = FILT_8TAP_DPADD_S_H(src21_r, src43_r, src65_r, src87_r, filt0, + filt1, filt2, filt3); + out2_r = FILT_8TAP_DPADD_S_H(src32_r, src54_r, src76_r, src98_r, filt0, + filt1, filt2, filt3); + out3_r = FILT_8TAP_DPADD_S_H(src43_r, src65_r, src87_r, src109_r, filt0, + filt1, filt2, filt3); + out0_l = FILT_8TAP_DPADD_S_H(src10_l, src32_l, src54_l, src76_l, filt0, + filt1, filt2, filt3); + out1_l = FILT_8TAP_DPADD_S_H(src21_l, src43_l, src65_l, src87_l, filt0, + filt1, filt2, filt3); + out2_l = FILT_8TAP_DPADD_S_H(src32_l, src54_l, src76_l, src98_l, filt0, + filt1, filt2, filt3); + out3_l = FILT_8TAP_DPADD_S_H(src43_l, src65_l, src87_l, src109_l, filt0, + filt1, filt2, filt3); + SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, FILTER_BITS); + SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, FILTER_BITS); + SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7); + SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7); + PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l, out3_r, + tmp0, tmp1, tmp2, tmp3); + XORI_B4_128_UB(tmp0, tmp1, tmp2, tmp3); + ST_UB4(tmp0, tmp1, tmp2, tmp3, dst, dst_stride); + dst += (4 * dst_stride); + + src10_r = src54_r; + src32_r = src76_r; + src54_r = src98_r; + src21_r = src65_r; + src43_r = src87_r; + src65_r = src109_r; + src10_l = src54_l; + src32_l = src76_l; + src54_l = src98_l; + src21_l = src65_l; + src43_l = src87_l; + src65_l = src109_l; + src6 = src10; + } +} + +static void common_vt_8t_16w_mult_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height, + int32_t width) { + const uint8_t *src_tmp; + uint8_t *dst_tmp; + uint32_t loop_cnt, cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + v16i8 filt0, filt1, filt2, filt3; + v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r; + v16i8 src65_r, src87_r, src109_r, src10_l, src32_l, src54_l, src76_l; + v16i8 src98_l, src21_l, src43_l, src65_l, src87_l, src109_l; + v16u8 tmp0, tmp1, tmp2, tmp3; + v8i16 filt, out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l; + + src -= (3 * src_stride); + + filt = LD_SH(filter); + SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3); + + for (cnt = (width >> 4); cnt--;) { + src_tmp = src; + dst_tmp = dst; + + LD_SB7(src_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6); + XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6); + src_tmp += (7 * src_stride); + ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_r, src32_r, + src54_r, src21_r); + ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r); + ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_l, src32_l, + src54_l, src21_l); + ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src_tmp, src_stride, src7, src8, src9, src10); + XORI_B4_128_SB(src7, src8, src9, src10); + src_tmp += (4 * src_stride); + ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r, + src87_r, src98_r, src109_r); + ILVL_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_l, + src87_l, src98_l, src109_l); + out0_r = FILT_8TAP_DPADD_S_H(src10_r, src32_r, src54_r, src76_r, filt0, + filt1, filt2, filt3); + out1_r = FILT_8TAP_DPADD_S_H(src21_r, src43_r, src65_r, src87_r, filt0, + filt1, filt2, filt3); + out2_r = FILT_8TAP_DPADD_S_H(src32_r, src54_r, src76_r, src98_r, filt0, + filt1, filt2, filt3); + out3_r = FILT_8TAP_DPADD_S_H(src43_r, src65_r, src87_r, src109_r, filt0, + filt1, filt2, filt3); + out0_l = FILT_8TAP_DPADD_S_H(src10_l, src32_l, src54_l, src76_l, filt0, + filt1, filt2, filt3); + out1_l = FILT_8TAP_DPADD_S_H(src21_l, src43_l, src65_l, src87_l, filt0, + filt1, filt2, filt3); + out2_l = FILT_8TAP_DPADD_S_H(src32_l, src54_l, src76_l, src98_l, filt0, + filt1, filt2, filt3); + out3_l = FILT_8TAP_DPADD_S_H(src43_l, src65_l, src87_l, src109_l, filt0, + filt1, filt2, filt3); + SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, FILTER_BITS); + SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, FILTER_BITS); + SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7); + SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7); + PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l, + out3_r, tmp0, tmp1, tmp2, tmp3); + XORI_B4_128_UB(tmp0, tmp1, tmp2, tmp3); + ST_UB4(tmp0, tmp1, tmp2, tmp3, dst_tmp, dst_stride); + dst_tmp += (4 * dst_stride); + + src10_r = src54_r; + src32_r = src76_r; + src54_r = src98_r; + src21_r = src65_r; + src43_r = src87_r; + src65_r = src109_r; + src10_l = src54_l; + src32_l = src76_l; + src54_l = src98_l; + src21_l = src65_l; + src43_l = src87_l; + src65_l = src109_l; + src6 = src10; + } + + src += 16; + dst += 16; + } +} + +static void common_vt_8t_32w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + common_vt_8t_16w_mult_msa(src, src_stride, dst, dst_stride, filter, height, + 32); +} + +static void common_vt_8t_64w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + common_vt_8t_16w_mult_msa(src, src_stride, dst, dst_stride, filter, height, + 64); +} + +static void common_vt_2t_4x4_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + v16i8 src0, src1, src2, src3, src4; + v16i8 src10_r, src32_r, src21_r, src43_r, src2110, src4332; + v16u8 filt0; + v8i16 filt; + v8u16 tmp0, tmp1; + + filt = LD_SH(filter); + filt0 = (v16u8)__msa_splati_h(filt, 0); + + LD_SB5(src, src_stride, src0, src1, src2, src3, src4); + src += (5 * src_stride); + + ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r, + src32_r, src43_r); + ILVR_D2_SB(src21_r, src10_r, src43_r, src32_r, src2110, src4332); + DOTP_UB2_UH(src2110, src4332, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + src2110 = __msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + ST4x4_UB(src2110, src2110, 0, 1, 2, 3, dst, dst_stride); +} + +static void common_vt_2t_4x8_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8; + v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r; + v16i8 src65_r, src87_r, src2110, src4332, src6554, src8776; + v8u16 tmp0, tmp1, tmp2, tmp3; + v16u8 filt0; + v8i16 filt; + + filt = LD_SH(filter); + filt0 = (v16u8)__msa_splati_h(filt, 0); + + LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + src += (8 * src_stride); + + src8 = LD_SB(src); + src += src_stride; + + ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r, + src32_r, src43_r); + ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r, + src76_r, src87_r); + ILVR_D4_SB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r, src87_r, + src76_r, src2110, src4332, src6554, src8776); + DOTP_UB4_UH(src2110, src4332, src6554, src8776, filt0, filt0, filt0, filt0, + tmp0, tmp1, tmp2, tmp3); + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_B2_SB(tmp1, tmp0, tmp3, tmp2, src2110, src4332); + ST4x4_UB(src2110, src2110, 0, 1, 2, 3, dst, dst_stride); + ST4x4_UB(src4332, src4332, 0, 1, 2, 3, dst + 4 * dst_stride, dst_stride); +} + +static void common_vt_2t_4w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (4 == height) { + common_vt_2t_4x4_msa(src, src_stride, dst, dst_stride, filter); + } else if (8 == height) { + common_vt_2t_4x8_msa(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_vt_2t_8x4_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + v16u8 src0, src1, src2, src3, src4, vec0, vec1, vec2, vec3, filt0; + v16i8 out0, out1; + v8u16 tmp0, tmp1, tmp2, tmp3; + v8i16 filt; + + /* rearranging filter_y */ + filt = LD_SH(filter); + filt0 = (v16u8)__msa_splati_h(filt, 0); + + LD_UB5(src, src_stride, src0, src1, src2, src3, src4); + ILVR_B2_UB(src1, src0, src2, src1, vec0, vec1); + ILVR_B2_UB(src3, src2, src4, src3, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, tmp0, tmp1, + tmp2, tmp3); + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_B2_SB(tmp1, tmp0, tmp3, tmp2, out0, out1); + ST8x4_UB(out0, out1, dst, dst_stride); +} + +static void common_vt_2t_8x8mult_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8; + v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + v16i8 out0, out1; + v8u16 tmp0, tmp1, tmp2, tmp3; + v8i16 filt; + + /* rearranging filter_y */ + filt = LD_SH(filter); + filt0 = (v16u8)__msa_splati_h(filt, 0); + + src0 = LD_UB(src); + src += src_stride; + + for (loop_cnt = (height >> 3); loop_cnt--;) { + LD_UB8(src, src_stride, src1, src2, src3, src4, src5, src6, src7, src8); + src += (8 * src_stride); + + ILVR_B4_UB(src1, src0, src2, src1, src3, src2, src4, src3, vec0, vec1, vec2, + vec3); + ILVR_B4_UB(src5, src4, src6, src5, src7, src6, src8, src7, vec4, vec5, vec6, + vec7); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, tmp0, tmp1, + tmp2, tmp3); + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_B2_SB(tmp1, tmp0, tmp3, tmp2, out0, out1); + ST8x4_UB(out0, out1, dst, dst_stride); + dst += (4 * dst_stride); + + DOTP_UB4_UH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0, tmp0, tmp1, + tmp2, tmp3); + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_B2_SB(tmp1, tmp0, tmp3, tmp2, out0, out1); + ST8x4_UB(out0, out1, dst, dst_stride); + dst += (4 * dst_stride); + + src0 = src8; + } +} + +static void common_vt_2t_8w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (4 == height) { + common_vt_2t_8x4_msa(src, src_stride, dst, dst_stride, filter); + } else { + common_vt_2t_8x8mult_msa(src, src_stride, dst, dst_stride, filter, height); + } +} + +static void common_vt_2t_16w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16u8 src0, src1, src2, src3, src4; + v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + v8u16 tmp0, tmp1, tmp2, tmp3; + v8i16 filt; + + /* rearranging filter_y */ + filt = LD_SH(filter); + filt0 = (v16u8)__msa_splati_h(filt, 0); + + src0 = LD_UB(src); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + + ILVR_B2_UB(src1, src0, src2, src1, vec0, vec2); + ILVL_B2_UB(src1, src0, src2, src1, vec1, vec3); + DOTP_UB2_UH(vec0, vec1, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_ST_SB(tmp0, tmp1, dst); + dst += dst_stride; + + ILVR_B2_UB(src3, src2, src4, src3, vec4, vec6); + ILVL_B2_UB(src3, src2, src4, src3, vec5, vec7); + DOTP_UB2_UH(vec2, vec3, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_ST_SB(tmp2, tmp3, dst); + dst += dst_stride; + + DOTP_UB2_UH(vec4, vec5, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_ST_SB(tmp0, tmp1, dst); + dst += dst_stride; + + DOTP_UB2_UH(vec6, vec7, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_ST_SB(tmp2, tmp3, dst); + dst += dst_stride; + + src0 = src4; + } +} + +static void common_vt_2t_32w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9; + v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + v8u16 tmp0, tmp1, tmp2, tmp3; + v8i16 filt; + + /* rearranging filter_y */ + filt = LD_SH(filter); + filt0 = (v16u8)__msa_splati_h(filt, 0); + + src0 = LD_UB(src); + src5 = LD_UB(src + 16); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + ILVR_B2_UB(src1, src0, src2, src1, vec0, vec2); + ILVL_B2_UB(src1, src0, src2, src1, vec1, vec3); + + LD_UB4(src + 16, src_stride, src6, src7, src8, src9); + src += (4 * src_stride); + + DOTP_UB2_UH(vec0, vec1, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_ST_SB(tmp0, tmp1, dst); + DOTP_UB2_UH(vec2, vec3, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_ST_SB(tmp2, tmp3, dst + dst_stride); + + ILVR_B2_UB(src3, src2, src4, src3, vec4, vec6); + ILVL_B2_UB(src3, src2, src4, src3, vec5, vec7); + DOTP_UB2_UH(vec4, vec5, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_ST_SB(tmp0, tmp1, dst + 2 * dst_stride); + + DOTP_UB2_UH(vec6, vec7, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_ST_SB(tmp2, tmp3, dst + 3 * dst_stride); + + ILVR_B2_UB(src6, src5, src7, src6, vec0, vec2); + ILVL_B2_UB(src6, src5, src7, src6, vec1, vec3); + DOTP_UB2_UH(vec0, vec1, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_ST_SB(tmp0, tmp1, dst + 16); + + DOTP_UB2_UH(vec2, vec3, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_ST_SB(tmp2, tmp3, dst + 16 + dst_stride); + + ILVR_B2_UB(src8, src7, src9, src8, vec4, vec6); + ILVL_B2_UB(src8, src7, src9, src8, vec5, vec7); + DOTP_UB2_UH(vec4, vec5, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_ST_SB(tmp0, tmp1, dst + 16 + 2 * dst_stride); + + DOTP_UB2_UH(vec6, vec7, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_ST_SB(tmp2, tmp3, dst + 16 + 3 * dst_stride); + dst += (4 * dst_stride); + + src0 = src4; + src5 = src9; + } +} + +static void common_vt_2t_64w_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + v16u8 src11, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + v8u16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + v8i16 filt; + + /* rearranging filter_y */ + filt = LD_SH(filter); + filt0 = (v16u8)__msa_splati_h(filt, 0); + + LD_UB4(src, 16, src0, src3, src6, src9); + src += src_stride; + + for (loop_cnt = (height >> 1); loop_cnt--;) { + LD_UB2(src, src_stride, src1, src2); + LD_UB2(src + 16, src_stride, src4, src5); + LD_UB2(src + 32, src_stride, src7, src8); + LD_UB2(src + 48, src_stride, src10, src11); + src += (2 * src_stride); + + ILVR_B2_UB(src1, src0, src2, src1, vec0, vec2); + ILVL_B2_UB(src1, src0, src2, src1, vec1, vec3); + DOTP_UB2_UH(vec0, vec1, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_ST_SB(tmp0, tmp1, dst); + + DOTP_UB2_UH(vec2, vec3, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_ST_SB(tmp2, tmp3, dst + dst_stride); + + ILVR_B2_UB(src4, src3, src5, src4, vec4, vec6); + ILVL_B2_UB(src4, src3, src5, src4, vec5, vec7); + DOTP_UB2_UH(vec4, vec5, filt0, filt0, tmp4, tmp5); + SRARI_H2_UH(tmp4, tmp5, FILTER_BITS); + PCKEV_ST_SB(tmp4, tmp5, dst + 16); + + DOTP_UB2_UH(vec6, vec7, filt0, filt0, tmp6, tmp7); + SRARI_H2_UH(tmp6, tmp7, FILTER_BITS); + PCKEV_ST_SB(tmp6, tmp7, dst + 16 + dst_stride); + + ILVR_B2_UB(src7, src6, src8, src7, vec0, vec2); + ILVL_B2_UB(src7, src6, src8, src7, vec1, vec3); + DOTP_UB2_UH(vec0, vec1, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + PCKEV_ST_SB(tmp0, tmp1, dst + 32); + + DOTP_UB2_UH(vec2, vec3, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_ST_SB(tmp2, tmp3, dst + 32 + dst_stride); + + ILVR_B2_UB(src10, src9, src11, src10, vec4, vec6); + ILVL_B2_UB(src10, src9, src11, src10, vec5, vec7); + DOTP_UB2_UH(vec4, vec5, filt0, filt0, tmp4, tmp5); + SRARI_H2_UH(tmp4, tmp5, FILTER_BITS); + PCKEV_ST_SB(tmp4, tmp5, dst + 48); + + DOTP_UB2_UH(vec6, vec7, filt0, filt0, tmp6, tmp7); + SRARI_H2_UH(tmp6, tmp7, FILTER_BITS); + PCKEV_ST_SB(tmp6, tmp7, dst + 48 + dst_stride); + dst += (2 * dst_stride); + + src0 = src2; + src3 = src5; + src6 = src8; + src9 = src11; + } +} + +void aom_convolve8_vert_msa(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + int8_t cnt, filt_ver[8]; + + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 8; cnt--;) { + filt_ver[cnt] = filter_y[cnt]; + } + + if (((const int32_t *)filter_y)[0] == 0) { + switch (w) { + case 4: + common_vt_2t_4w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + case 8: + common_vt_2t_8w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + case 16: + common_vt_2t_16w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + case 32: + common_vt_2t_32w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + case 64: + common_vt_2t_64w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + default: + aom_convolve8_vert_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } else { + switch (w) { + case 4: + common_vt_8t_4w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + case 8: + common_vt_8t_8w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + case 16: + common_vt_8t_16w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + case 32: + common_vt_8t_32w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + case 64: + common_vt_8t_64w_msa(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + default: + aom_convolve8_vert_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } +} diff --git a/third_party/aom/aom_dsp/mips/aom_convolve_avg_msa.c b/third_party/aom/aom_dsp/mips/aom_convolve_avg_msa.c new file mode 100644 index 0000000000..75f8c7ea85 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/aom_convolve_avg_msa.c @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/macros_msa.h" + +static void avg_width4_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int32_t height) { + int32_t cnt; + uint32_t out0, out1, out2, out3; + v16u8 src0, src1, src2, src3; + v16u8 dst0, dst1, dst2, dst3; + + if (0 == (height % 4)) { + for (cnt = (height / 4); cnt--;) { + LD_UB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + + AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3, dst0, dst1, + dst2, dst3); + + out0 = __msa_copy_u_w((v4i32)dst0, 0); + out1 = __msa_copy_u_w((v4i32)dst1, 0); + out2 = __msa_copy_u_w((v4i32)dst2, 0); + out3 = __msa_copy_u_w((v4i32)dst3, 0); + SW4(out0, out1, out2, out3, dst, dst_stride); + dst += (4 * dst_stride); + } + } else if (0 == (height % 2)) { + for (cnt = (height / 2); cnt--;) { + LD_UB2(src, src_stride, src0, src1); + src += (2 * src_stride); + + LD_UB2(dst, dst_stride, dst0, dst1); + + AVER_UB2_UB(src0, dst0, src1, dst1, dst0, dst1); + + out0 = __msa_copy_u_w((v4i32)dst0, 0); + out1 = __msa_copy_u_w((v4i32)dst1, 0); + SW(out0, dst); + dst += dst_stride; + SW(out1, dst); + dst += dst_stride; + } + } +} + +static void avg_width8_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int32_t height) { + int32_t cnt; + uint64_t out0, out1, out2, out3; + v16u8 src0, src1, src2, src3; + v16u8 dst0, dst1, dst2, dst3; + + for (cnt = (height / 4); cnt--;) { + LD_UB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + + AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3, dst0, dst1, + dst2, dst3); + + out0 = __msa_copy_u_d((v2i64)dst0, 0); + out1 = __msa_copy_u_d((v2i64)dst1, 0); + out2 = __msa_copy_u_d((v2i64)dst2, 0); + out3 = __msa_copy_u_d((v2i64)dst3, 0); + SD4(out0, out1, out2, out3, dst, dst_stride); + dst += (4 * dst_stride); + } +} + +static void avg_width16_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + + for (cnt = (height / 8); cnt--;) { + LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + src += (8 * src_stride); + LD_UB8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7); + + AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3, dst0, dst1, + dst2, dst3); + AVER_UB4_UB(src4, dst4, src5, dst5, src6, dst6, src7, dst7, dst4, dst5, + dst6, dst7); + ST_UB8(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst, dst_stride); + dst += (8 * dst_stride); + } +} + +static void avg_width32_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt; + uint8_t *dst_dup = dst; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + v16u8 src8, src9, src10, src11, src12, src13, src14, src15; + v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + v16u8 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15; + + for (cnt = (height / 8); cnt--;) { + LD_UB4(src, src_stride, src0, src2, src4, src6); + LD_UB4(src + 16, src_stride, src1, src3, src5, src7); + src += (4 * src_stride); + LD_UB4(dst_dup, dst_stride, dst0, dst2, dst4, dst6); + LD_UB4(dst_dup + 16, dst_stride, dst1, dst3, dst5, dst7); + dst_dup += (4 * dst_stride); + LD_UB4(src, src_stride, src8, src10, src12, src14); + LD_UB4(src + 16, src_stride, src9, src11, src13, src15); + src += (4 * src_stride); + LD_UB4(dst_dup, dst_stride, dst8, dst10, dst12, dst14); + LD_UB4(dst_dup + 16, dst_stride, dst9, dst11, dst13, dst15); + dst_dup += (4 * dst_stride); + + AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3, dst0, dst1, + dst2, dst3); + AVER_UB4_UB(src4, dst4, src5, dst5, src6, dst6, src7, dst7, dst4, dst5, + dst6, dst7); + AVER_UB4_UB(src8, dst8, src9, dst9, src10, dst10, src11, dst11, dst8, dst9, + dst10, dst11); + AVER_UB4_UB(src12, dst12, src13, dst13, src14, dst14, src15, dst15, dst12, + dst13, dst14, dst15); + + ST_UB4(dst0, dst2, dst4, dst6, dst, dst_stride); + ST_UB4(dst1, dst3, dst5, dst7, dst + 16, dst_stride); + dst += (4 * dst_stride); + ST_UB4(dst8, dst10, dst12, dst14, dst, dst_stride); + ST_UB4(dst9, dst11, dst13, dst15, dst + 16, dst_stride); + dst += (4 * dst_stride); + } +} + +static void avg_width64_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt; + uint8_t *dst_dup = dst; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + v16u8 src8, src9, src10, src11, src12, src13, src14, src15; + v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + v16u8 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15; + + for (cnt = (height / 4); cnt--;) { + LD_UB4(src, 16, src0, src1, src2, src3); + src += src_stride; + LD_UB4(src, 16, src4, src5, src6, src7); + src += src_stride; + LD_UB4(src, 16, src8, src9, src10, src11); + src += src_stride; + LD_UB4(src, 16, src12, src13, src14, src15); + src += src_stride; + + LD_UB4(dst_dup, 16, dst0, dst1, dst2, dst3); + dst_dup += dst_stride; + LD_UB4(dst_dup, 16, dst4, dst5, dst6, dst7); + dst_dup += dst_stride; + LD_UB4(dst_dup, 16, dst8, dst9, dst10, dst11); + dst_dup += dst_stride; + LD_UB4(dst_dup, 16, dst12, dst13, dst14, dst15); + dst_dup += dst_stride; + + AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3, dst0, dst1, + dst2, dst3); + AVER_UB4_UB(src4, dst4, src5, dst5, src6, dst6, src7, dst7, dst4, dst5, + dst6, dst7); + AVER_UB4_UB(src8, dst8, src9, dst9, src10, dst10, src11, dst11, dst8, dst9, + dst10, dst11); + AVER_UB4_UB(src12, dst12, src13, dst13, src14, dst14, src15, dst15, dst12, + dst13, dst14, dst15); + + ST_UB4(dst0, dst1, dst2, dst3, dst, 16); + dst += dst_stride; + ST_UB4(dst4, dst5, dst6, dst7, dst, 16); + dst += dst_stride; + ST_UB4(dst8, dst9, dst10, dst11, dst, 16); + dst += dst_stride; + ST_UB4(dst12, dst13, dst14, dst15, dst, 16); + dst += dst_stride; + } +} + +void aom_convolve_avg_msa(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int32_t filter_x_stride, + const int16_t *filter_y, int32_t filter_y_stride, + int32_t w, int32_t h) { + (void)filter_x; + (void)filter_y; + (void)filter_x_stride; + (void)filter_y_stride; + + switch (w) { + case 4: { + avg_width4_msa(src, src_stride, dst, dst_stride, h); + break; + } + case 8: { + avg_width8_msa(src, src_stride, dst, dst_stride, h); + break; + } + case 16: { + avg_width16_msa(src, src_stride, dst, dst_stride, h); + break; + } + case 32: { + avg_width32_msa(src, src_stride, dst, dst_stride, h); + break; + } + case 64: { + avg_width64_msa(src, src_stride, dst, dst_stride, h); + break; + } + default: { + int32_t lp, cnt; + for (cnt = h; cnt--;) { + for (lp = 0; lp < w; ++lp) { + dst[lp] = (((dst[lp] + src[lp]) + 1) >> 1); + } + src += src_stride; + dst += dst_stride; + } + break; + } + } +} diff --git a/third_party/aom/aom_dsp/mips/aom_convolve_copy_msa.c b/third_party/aom/aom_dsp/mips/aom_convolve_copy_msa.c new file mode 100644 index 0000000000..f7f116f4da --- /dev/null +++ b/third_party/aom/aom_dsp/mips/aom_convolve_copy_msa.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "aom_dsp/mips/macros_msa.h" + +static void copy_width8_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt; + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + + if (0 == height % 12) { + for (cnt = (height / 12); cnt--;) { + LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + src += (8 * src_stride); + + out0 = __msa_copy_u_d((v2i64)src0, 0); + out1 = __msa_copy_u_d((v2i64)src1, 0); + out2 = __msa_copy_u_d((v2i64)src2, 0); + out3 = __msa_copy_u_d((v2i64)src3, 0); + out4 = __msa_copy_u_d((v2i64)src4, 0); + out5 = __msa_copy_u_d((v2i64)src5, 0); + out6 = __msa_copy_u_d((v2i64)src6, 0); + out7 = __msa_copy_u_d((v2i64)src7, 0); + + SD4(out0, out1, out2, out3, dst, dst_stride); + dst += (4 * dst_stride); + SD4(out4, out5, out6, out7, dst, dst_stride); + dst += (4 * dst_stride); + + LD_UB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + + out0 = __msa_copy_u_d((v2i64)src0, 0); + out1 = __msa_copy_u_d((v2i64)src1, 0); + out2 = __msa_copy_u_d((v2i64)src2, 0); + out3 = __msa_copy_u_d((v2i64)src3, 0); + SD4(out0, out1, out2, out3, dst, dst_stride); + dst += (4 * dst_stride); + } + } else if (0 == height % 8) { + for (cnt = height >> 3; cnt--;) { + LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + src += (8 * src_stride); + + out0 = __msa_copy_u_d((v2i64)src0, 0); + out1 = __msa_copy_u_d((v2i64)src1, 0); + out2 = __msa_copy_u_d((v2i64)src2, 0); + out3 = __msa_copy_u_d((v2i64)src3, 0); + out4 = __msa_copy_u_d((v2i64)src4, 0); + out5 = __msa_copy_u_d((v2i64)src5, 0); + out6 = __msa_copy_u_d((v2i64)src6, 0); + out7 = __msa_copy_u_d((v2i64)src7, 0); + + SD4(out0, out1, out2, out3, dst, dst_stride); + dst += (4 * dst_stride); + SD4(out4, out5, out6, out7, dst, dst_stride); + dst += (4 * dst_stride); + } + } else if (0 == height % 4) { + for (cnt = (height / 4); cnt--;) { + LD_UB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + out0 = __msa_copy_u_d((v2i64)src0, 0); + out1 = __msa_copy_u_d((v2i64)src1, 0); + out2 = __msa_copy_u_d((v2i64)src2, 0); + out3 = __msa_copy_u_d((v2i64)src3, 0); + + SD4(out0, out1, out2, out3, dst, dst_stride); + dst += (4 * dst_stride); + } + } else if (0 == height % 2) { + for (cnt = (height / 2); cnt--;) { + LD_UB2(src, src_stride, src0, src1); + src += (2 * src_stride); + out0 = __msa_copy_u_d((v2i64)src0, 0); + out1 = __msa_copy_u_d((v2i64)src1, 0); + + SD(out0, dst); + dst += dst_stride; + SD(out1, dst); + dst += dst_stride; + } + } +} + +static void copy_16multx8mult_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int32_t height, int32_t width) { + int32_t cnt, loop_cnt; + const uint8_t *src_tmp; + uint8_t *dst_tmp; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + + for (cnt = (width >> 4); cnt--;) { + src_tmp = src; + dst_tmp = dst; + + for (loop_cnt = (height >> 3); loop_cnt--;) { + LD_UB8(src_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6, + src7); + src_tmp += (8 * src_stride); + + ST_UB8(src0, src1, src2, src3, src4, src5, src6, src7, dst_tmp, + dst_stride); + dst_tmp += (8 * dst_stride); + } + + src += 16; + dst += 16; + } +} + +static void copy_width16_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + + if (0 == height % 12) { + for (cnt = (height / 12); cnt--;) { + LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + src += (8 * src_stride); + ST_UB8(src0, src1, src2, src3, src4, src5, src6, src7, dst, dst_stride); + dst += (8 * dst_stride); + + LD_UB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + ST_UB4(src0, src1, src2, src3, dst, dst_stride); + dst += (4 * dst_stride); + } + } else if (0 == height % 8) { + copy_16multx8mult_msa(src, src_stride, dst, dst_stride, height, 16); + } else if (0 == height % 4) { + for (cnt = (height >> 2); cnt--;) { + LD_UB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + + ST_UB4(src0, src1, src2, src3, dst, dst_stride); + dst += (4 * dst_stride); + } + } +} + +static void copy_width32_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + + if (0 == height % 12) { + for (cnt = (height / 12); cnt--;) { + LD_UB4(src, src_stride, src0, src1, src2, src3); + LD_UB4(src + 16, src_stride, src4, src5, src6, src7); + src += (4 * src_stride); + ST_UB4(src0, src1, src2, src3, dst, dst_stride); + ST_UB4(src4, src5, src6, src7, dst + 16, dst_stride); + dst += (4 * dst_stride); + + LD_UB4(src, src_stride, src0, src1, src2, src3); + LD_UB4(src + 16, src_stride, src4, src5, src6, src7); + src += (4 * src_stride); + ST_UB4(src0, src1, src2, src3, dst, dst_stride); + ST_UB4(src4, src5, src6, src7, dst + 16, dst_stride); + dst += (4 * dst_stride); + + LD_UB4(src, src_stride, src0, src1, src2, src3); + LD_UB4(src + 16, src_stride, src4, src5, src6, src7); + src += (4 * src_stride); + ST_UB4(src0, src1, src2, src3, dst, dst_stride); + ST_UB4(src4, src5, src6, src7, dst + 16, dst_stride); + dst += (4 * dst_stride); + } + } else if (0 == height % 8) { + copy_16multx8mult_msa(src, src_stride, dst, dst_stride, height, 32); + } else if (0 == height % 4) { + for (cnt = (height >> 2); cnt--;) { + LD_UB4(src, src_stride, src0, src1, src2, src3); + LD_UB4(src + 16, src_stride, src4, src5, src6, src7); + src += (4 * src_stride); + ST_UB4(src0, src1, src2, src3, dst, dst_stride); + ST_UB4(src4, src5, src6, src7, dst + 16, dst_stride); + dst += (4 * dst_stride); + } + } +} + +static void copy_width64_msa(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + copy_16multx8mult_msa(src, src_stride, dst, dst_stride, height, 64); +} + +void aom_convolve_copy_msa(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int32_t filter_x_stride, + const int16_t *filter_y, int32_t filter_y_stride, + int32_t w, int32_t h) { + (void)filter_x; + (void)filter_y; + (void)filter_x_stride; + (void)filter_y_stride; + + switch (w) { + case 4: { + uint32_t cnt, tmp; + /* 1 word storage */ + for (cnt = h; cnt--;) { + tmp = LW(src); + SW(tmp, dst); + src += src_stride; + dst += dst_stride; + } + break; + } + case 8: { + copy_width8_msa(src, src_stride, dst, dst_stride, h); + break; + } + case 16: { + copy_width16_msa(src, src_stride, dst, dst_stride, h); + break; + } + case 32: { + copy_width32_msa(src, src_stride, dst, dst_stride, h); + break; + } + case 64: { + copy_width64_msa(src, src_stride, dst, dst_stride, h); + break; + } + default: { + uint32_t cnt; + for (cnt = h; cnt--;) { + memcpy(dst, src, w); + src += src_stride; + dst += dst_stride; + } + break; + } + } +} diff --git a/third_party/aom/aom_dsp/mips/aom_convolve_msa.h b/third_party/aom/aom_dsp/mips/aom_convolve_msa.h new file mode 100644 index 0000000000..1a0ae4d8d4 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/aom_convolve_msa.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_MIPS_AOM_CONVOLVE_MSA_H_ +#define AOM_DSP_MIPS_AOM_CONVOLVE_MSA_H_ + +#include "aom_dsp/mips/macros_msa.h" +#include "aom_dsp/aom_filter.h" + +extern const uint8_t mc_filt_mask_arr[16 * 3]; + +#define FILT_8TAP_DPADD_S_H(vec0, vec1, vec2, vec3, filt0, filt1, filt2, \ + filt3) \ + ({ \ + v8i16 tmp_dpadd_0, tmp_dpadd_1; \ + \ + tmp_dpadd_0 = __msa_dotp_s_h((v16i8)vec0, (v16i8)filt0); \ + tmp_dpadd_0 = __msa_dpadd_s_h(tmp_dpadd_0, (v16i8)vec1, (v16i8)filt1); \ + tmp_dpadd_1 = __msa_dotp_s_h((v16i8)vec2, (v16i8)filt2); \ + tmp_dpadd_1 = __msa_dpadd_s_h(tmp_dpadd_1, (v16i8)vec3, (v16i8)filt3); \ + tmp_dpadd_0 = __msa_adds_s_h(tmp_dpadd_0, tmp_dpadd_1); \ + \ + tmp_dpadd_0; \ + }) + +#define HORIZ_8TAP_FILT(src0, src1, mask0, mask1, mask2, mask3, filt_h0, \ + filt_h1, filt_h2, filt_h3) \ + ({ \ + v16i8 vec0_m, vec1_m, vec2_m, vec3_m; \ + v8i16 hz_out_m; \ + \ + VSHF_B4_SB(src0, src1, mask0, mask1, mask2, mask3, vec0_m, vec1_m, vec2_m, \ + vec3_m); \ + hz_out_m = FILT_8TAP_DPADD_S_H(vec0_m, vec1_m, vec2_m, vec3_m, filt_h0, \ + filt_h1, filt_h2, filt_h3); \ + \ + hz_out_m = __msa_srari_h(hz_out_m, FILTER_BITS); \ + hz_out_m = __msa_sat_s_h(hz_out_m, 7); \ + \ + hz_out_m; \ + }) + +#define HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, \ + mask2, mask3, filt0, filt1, filt2, filt3, \ + out0, out1) \ + { \ + v16i8 vec0_m, vec1_m, vec2_m, vec3_m, vec4_m, vec5_m, vec6_m, vec7_m; \ + v8i16 res0_m, res1_m, res2_m, res3_m; \ + \ + VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0_m, vec1_m); \ + DOTP_SB2_SH(vec0_m, vec1_m, filt0, filt0, res0_m, res1_m); \ + VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2_m, vec3_m); \ + DPADD_SB2_SH(vec2_m, vec3_m, filt1, filt1, res0_m, res1_m); \ + VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4_m, vec5_m); \ + DOTP_SB2_SH(vec4_m, vec5_m, filt2, filt2, res2_m, res3_m); \ + VSHF_B2_SB(src0, src1, src2, src3, mask3, mask3, vec6_m, vec7_m); \ + DPADD_SB2_SH(vec6_m, vec7_m, filt3, filt3, res2_m, res3_m); \ + ADDS_SH2_SH(res0_m, res2_m, res1_m, res3_m, out0, out1); \ + } + +#define HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, \ + mask2, mask3, filt0, filt1, filt2, filt3, \ + out0, out1, out2, out3) \ + { \ + v16i8 vec0_m, vec1_m, vec2_m, vec3_m, vec4_m, vec5_m, vec6_m, vec7_m; \ + v8i16 res0_m, res1_m, res2_m, res3_m, res4_m, res5_m, res6_m, res7_m; \ + \ + VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0_m, vec1_m); \ + VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2_m, vec3_m); \ + DOTP_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt0, filt0, filt0, filt0, \ + res0_m, res1_m, res2_m, res3_m); \ + VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec0_m, vec1_m); \ + VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec2_m, vec3_m); \ + DOTP_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt2, filt2, filt2, filt2, \ + res4_m, res5_m, res6_m, res7_m); \ + VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4_m, vec5_m); \ + VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6_m, vec7_m); \ + DPADD_SB4_SH(vec4_m, vec5_m, vec6_m, vec7_m, filt1, filt1, filt1, filt1, \ + res0_m, res1_m, res2_m, res3_m); \ + VSHF_B2_SB(src0, src0, src1, src1, mask3, mask3, vec4_m, vec5_m); \ + VSHF_B2_SB(src2, src2, src3, src3, mask3, mask3, vec6_m, vec7_m); \ + DPADD_SB4_SH(vec4_m, vec5_m, vec6_m, vec7_m, filt3, filt3, filt3, filt3, \ + res4_m, res5_m, res6_m, res7_m); \ + ADDS_SH4_SH(res0_m, res4_m, res1_m, res5_m, res2_m, res6_m, res3_m, \ + res7_m, out0, out1, out2, out3); \ + } + +#define PCKEV_XORI128_AVG_ST_UB(in0, in1, dst, pdst) \ + { \ + v16u8 tmp_m; \ + \ + tmp_m = PCKEV_XORI128_UB(in1, in0); \ + tmp_m = __msa_aver_u_b(tmp_m, (v16u8)dst); \ + ST_UB(tmp_m, (pdst)); \ + } + +#define PCKEV_AVG_ST_UB(in0, in1, dst, pdst) \ + { \ + v16u8 tmp_m; \ + \ + tmp_m = (v16u8)__msa_pckev_b((v16i8)in0, (v16i8)in1); \ + tmp_m = __msa_aver_u_b(tmp_m, (v16u8)dst); \ + ST_UB(tmp_m, (pdst)); \ + } + +#define PCKEV_AVG_ST8x4_UB(in1, dst0, in2, dst1, in3, dst2, in4, dst3, pdst, \ + stride) \ + { \ + v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + \ + PCKEV_B2_UB(in2, in1, in4, in3, tmp0_m, tmp1_m); \ + PCKEV_D2_UB(dst1, dst0, dst3, dst2, tmp2_m, tmp3_m); \ + AVER_UB2_UB(tmp0_m, tmp2_m, tmp1_m, tmp3_m, tmp0_m, tmp1_m); \ + ST8x4_UB(tmp0_m, tmp1_m, pdst, stride); \ + } +#endif /* AOM_DSP_MIPS_AOM_CONVOLVE_MSA_H_ */ diff --git a/third_party/aom/aom_dsp/mips/avg_msa.c b/third_party/aom/aom_dsp/mips/avg_msa.c new file mode 100644 index 0000000000..0e17281553 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/avg_msa.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/macros_msa.h" + +uint32_t aom_avg_8x8_msa(const uint8_t *src, int32_t src_stride) { + uint32_t sum_out; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + v8u16 sum0, sum1, sum2, sum3, sum4, sum5, sum6, sum7; + v4u32 sum = { 0 }; + + LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + HADD_UB4_UH(src0, src1, src2, src3, sum0, sum1, sum2, sum3); + HADD_UB4_UH(src4, src5, src6, src7, sum4, sum5, sum6, sum7); + ADD4(sum0, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum0, sum2, sum4, sum6); + ADD2(sum0, sum2, sum4, sum6, sum0, sum4); + sum0 += sum4; + + sum = __msa_hadd_u_w(sum0, sum0); + sum0 = (v8u16)__msa_pckev_h((v8i16)sum, (v8i16)sum); + sum = __msa_hadd_u_w(sum0, sum0); + sum = (v4u32)__msa_srari_w((v4i32)sum, 6); + sum_out = __msa_copy_u_w((v4i32)sum, 0); + + return sum_out; +} + +uint32_t aom_avg_4x4_msa(const uint8_t *src, int32_t src_stride) { + uint32_t sum_out; + uint32_t src0, src1, src2, src3; + v16u8 vec = { 0 }; + v8u16 sum0; + v4u32 sum1; + v2u64 sum2; + + LW4(src, src_stride, src0, src1, src2, src3); + INSERT_W4_UB(src0, src1, src2, src3, vec); + + sum0 = __msa_hadd_u_h(vec, vec); + sum1 = __msa_hadd_u_w(sum0, sum0); + sum0 = (v8u16)__msa_pckev_h((v8i16)sum1, (v8i16)sum1); + sum1 = __msa_hadd_u_w(sum0, sum0); + sum2 = __msa_hadd_u_d(sum1, sum1); + sum1 = (v4u32)__msa_srari_w((v4i32)sum2, 4); + sum_out = __msa_copy_u_w((v4i32)sum1, 0); + + return sum_out; +} diff --git a/third_party/aom/aom_dsp/mips/common_dspr2.c b/third_party/aom/aom_dsp/mips/common_dspr2.c new file mode 100644 index 0000000000..00ab75dc31 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/common_dspr2.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/common_dspr2.h" + +#if HAVE_DSPR2 +uint8_t aom_ff_cropTbl_a[256 + 2 * CROP_WIDTH]; +uint8_t *aom_ff_cropTbl; + +void aom_dsputil_static_init(void) { + int i; + + for (i = 0; i < 256; i++) aom_ff_cropTbl_a[i + CROP_WIDTH] = i; + + for (i = 0; i < CROP_WIDTH; i++) { + aom_ff_cropTbl_a[i] = 0; + aom_ff_cropTbl_a[i + CROP_WIDTH + 256] = 255; + } + + aom_ff_cropTbl = &aom_ff_cropTbl_a[CROP_WIDTH]; +} + +#endif diff --git a/third_party/aom/aom_dsp/mips/common_dspr2.h b/third_party/aom/aom_dsp/mips/common_dspr2.h new file mode 100644 index 0000000000..31159fdcdc --- /dev/null +++ b/third_party/aom/aom_dsp/mips/common_dspr2.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_COMMON_MIPS_DSPR2_H_ +#define AOM_COMMON_MIPS_DSPR2_H_ + +#include +#include "./aom_config.h" +#include "aom/aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif +#if HAVE_DSPR2 +#define CROP_WIDTH 512 + +extern uint8_t *aom_ff_cropTbl; // From "aom_dsp/mips/intrapred4_dspr2.c" + +static INLINE void prefetch_load(const unsigned char *src) { + __asm__ __volatile__("pref 0, 0(%[src]) \n\t" : : [src] "r"(src)); +} + +/* prefetch data for store */ +static INLINE void prefetch_store(unsigned char *dst) { + __asm__ __volatile__("pref 1, 0(%[dst]) \n\t" : : [dst] "r"(dst)); +} + +static INLINE void prefetch_load_streamed(const unsigned char *src) { + __asm__ __volatile__("pref 4, 0(%[src]) \n\t" : : [src] "r"(src)); +} + +/* prefetch data for store */ +static INLINE void prefetch_store_streamed(unsigned char *dst) { + __asm__ __volatile__("pref 5, 0(%[dst]) \n\t" : : [dst] "r"(dst)); +} +#endif // #if HAVE_DSPR2 +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_COMMON_MIPS_DSPR2_H_ diff --git a/third_party/aom/aom_dsp/mips/convolve2_avg_dspr2.c b/third_party/aom/aom_dsp/mips/convolve2_avg_dspr2.c new file mode 100644 index 0000000000..d557115b92 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/convolve2_avg_dspr2.c @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/convolve_common_dspr2.h" +#include "aom_dsp/aom_convolve.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +static void convolve_bi_avg_vert_4_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_y, int32_t w, + int32_t h) { + int32_t x, y; + const uint8_t *src_ptr; + uint8_t *dst_ptr; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector4a = 64; + uint32_t load1, load2; + uint32_t p1, p2; + uint32_t scratch1, scratch2; + uint32_t store1, store2; + int32_t Temp1, Temp2; + const int16_t *filter = &filter_y[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_store(dst + dst_stride); + + for (x = 0; x < w; x += 4) { + src_ptr = src + x; + dst_ptr = dst + x; + + __asm__ __volatile__( + "ulw %[load1], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load2], 0(%[src_ptr]) \n\t" + + "mtlo %[vector4a], $ac0 \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac0 \n\t" + "mthi $zero, $ac1 \n\t" + "mthi $zero, $ac2 \n\t" + "mthi $zero, $ac3 \n\t" + + "preceu.ph.qbr %[scratch1], %[load1] \n\t" + "preceu.ph.qbr %[p1], %[load2] \n\t" + "precrq.ph.w %[p2], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac0, %[p1], %[filter45] \n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" + + "preceu.ph.qbl %[scratch1], %[load1] \n\t" + "preceu.ph.qbl %[p1], %[load2] \n\t" + "precrq.ph.w %[p2], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac2, %[p1], %[filter45] \n\t" + "dpa.w.ph $ac3, %[p2], %[filter45] \n\t" + + "extp %[Temp1], $ac0, 31 \n\t" + "extp %[Temp2], $ac1, 31 \n\t" + + "lbu %[scratch1], 0(%[dst_ptr]) \n\t" + "lbu %[scratch2], 1(%[dst_ptr]) \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "addqh_r.w %[store1], %[store1], %[scratch1] \n\t" /* pixel 1 */ + "extp %[Temp1], $ac2, 31 \n\t" + + "lbux %[store2], %[Temp2](%[cm]) \n\t" + "addqh_r.w %[store2], %[store2], %[scratch2] \n\t" /* pixel 2 */ + "extp %[Temp2], $ac3, 31 \n\t" + "lbu %[scratch1], 2(%[dst_ptr]) \n\t" + + "sb %[store1], 0(%[dst_ptr]) \n\t" + "sb %[store2], 1(%[dst_ptr]) \n\t" + "lbu %[scratch2], 3(%[dst_ptr]) \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "lbux %[store2], %[Temp2](%[cm]) \n\t" + "addqh_r.w %[store1], %[store1], %[scratch1] \n\t" /* pixel 3 */ + "addqh_r.w %[store2], %[store2], %[scratch2] \n\t" /* pixel 4 */ + + "sb %[store1], 2(%[dst_ptr]) \n\t" + "sb %[store2], 3(%[dst_ptr]) \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [p1] "=&r"(p1), + [p2] "=&r"(p2), [scratch1] "=&r"(scratch1), + [scratch2] "=&r"(scratch2), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [store1] "=&r"(store1), + [store2] "=&r"(store2), [src_ptr] "+r"(src_ptr) + : [filter45] "r"(filter45), [vector4a] "r"(vector4a), + [src_stride] "r"(src_stride), [cm] "r"(cm), [dst_ptr] "r"(dst_ptr)); + } + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_bi_avg_vert_64_dspr2(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int16_t *filter_y, int32_t h) { + int32_t x, y; + const uint8_t *src_ptr; + uint8_t *dst_ptr; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector4a = 64; + uint32_t load1, load2; + uint32_t p1, p2; + uint32_t scratch1, scratch2; + uint32_t store1, store2; + int32_t Temp1, Temp2; + const int16_t *filter = &filter_y[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_store(dst + dst_stride); + prefetch_store(dst + dst_stride + 32); + + for (x = 0; x < 64; x += 4) { + src_ptr = src + x; + dst_ptr = dst + x; + + __asm__ __volatile__( + "ulw %[load1], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load2], 0(%[src_ptr]) \n\t" + + "mtlo %[vector4a], $ac0 \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac0 \n\t" + "mthi $zero, $ac1 \n\t" + "mthi $zero, $ac2 \n\t" + "mthi $zero, $ac3 \n\t" + + "preceu.ph.qbr %[scratch1], %[load1] \n\t" + "preceu.ph.qbr %[p1], %[load2] \n\t" + "precrq.ph.w %[p2], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac0, %[p1], %[filter45] \n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" + + "preceu.ph.qbl %[scratch1], %[load1] \n\t" + "preceu.ph.qbl %[p1], %[load2] \n\t" + "precrq.ph.w %[p2], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac2, %[p1], %[filter45] \n\t" + "dpa.w.ph $ac3, %[p2], %[filter45] \n\t" + + "extp %[Temp1], $ac0, 31 \n\t" + "extp %[Temp2], $ac1, 31 \n\t" + + "lbu %[scratch1], 0(%[dst_ptr]) \n\t" + "lbu %[scratch2], 1(%[dst_ptr]) \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "addqh_r.w %[store1], %[store1], %[scratch1] \n\t" /* pixel 1 */ + "extp %[Temp1], $ac2, 31 \n\t" + + "lbux %[store2], %[Temp2](%[cm]) \n\t" + "addqh_r.w %[store2], %[store2], %[scratch2] \n\t" /* pixel 2 */ + "extp %[Temp2], $ac3, 31 \n\t" + "lbu %[scratch1], 2(%[dst_ptr]) \n\t" + + "sb %[store1], 0(%[dst_ptr]) \n\t" + "sb %[store2], 1(%[dst_ptr]) \n\t" + "lbu %[scratch2], 3(%[dst_ptr]) \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "lbux %[store2], %[Temp2](%[cm]) \n\t" + "addqh_r.w %[store1], %[store1], %[scratch1] \n\t" /* pixel 3 */ + "addqh_r.w %[store2], %[store2], %[scratch2] \n\t" /* pixel 4 */ + + "sb %[store1], 2(%[dst_ptr]) \n\t" + "sb %[store2], 3(%[dst_ptr]) \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [p1] "=&r"(p1), + [p2] "=&r"(p2), [scratch1] "=&r"(scratch1), + [scratch2] "=&r"(scratch2), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [store1] "=&r"(store1), + [store2] "=&r"(store2), [src_ptr] "+r"(src_ptr) + : [filter45] "r"(filter45), [vector4a] "r"(vector4a), + [src_stride] "r"(src_stride), [cm] "r"(cm), [dst_ptr] "r"(dst_ptr)); + } + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +void aom_convolve2_avg_vert_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + uint32_t pos = 38; + + assert(y_step_q4 == 16); + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + prefetch_store(dst); + + switch (w) { + case 4: + case 8: + case 16: + case 32: + convolve_bi_avg_vert_4_dspr2(src, src_stride, dst, dst_stride, filter_y, + w, h); + break; + case 64: + prefetch_store(dst + 32); + convolve_bi_avg_vert_64_dspr2(src, src_stride, dst, dst_stride, filter_y, + h); + break; + default: + aom_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } +} +#endif diff --git a/third_party/aom/aom_dsp/mips/convolve2_avg_horiz_dspr2.c b/third_party/aom/aom_dsp/mips/convolve2_avg_horiz_dspr2.c new file mode 100644 index 0000000000..efbdcf60fb --- /dev/null +++ b/third_party/aom/aom_dsp/mips/convolve2_avg_horiz_dspr2.c @@ -0,0 +1,802 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/convolve_common_dspr2.h" +#include "aom_dsp/aom_convolve.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +static void convolve_bi_avg_horiz_4_dspr2(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y; + uint8_t *cm = aom_ff_cropTbl; + int32_t Temp1, Temp2, Temp3, Temp4; + uint32_t vector4a = 64; + uint32_t tp1, tp2; + uint32_t p1, p2, p3; + uint32_t tn1, tn2; + const int16_t *filter = &filter_x0[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[tp1] \n\t" + "preceu.ph.qbl %[p2], %[tp1] \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + + /* even 2. pixel */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "balign %[tp2], %[tp1], 3 \n\t" + "dpa.w.ph $ac2, %[p2], %[filter45] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + "lbu %[p2], 3(%[dst]) \n\t" /* load odd 2 */ + + /* odd 1. pixel */ + "lbux %[tp1], %[Temp1](%[cm]) \n\t" /* even 1 */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "lbu %[Temp1], 1(%[dst]) \n\t" /* load odd 1 */ + "preceu.ph.qbr %[p1], %[tp2] \n\t" + "preceu.ph.qbl %[p3], %[tp2] \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + "lbu %[tn2], 0(%[dst]) \n\t" /* load even 1 */ + + /* odd 2. pixel */ + "lbux %[tp2], %[Temp3](%[cm]) \n\t" /* even 2 */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "lbux %[tn1], %[Temp2](%[cm]) \n\t" /* odd 1 */ + "addqh_r.w %[tn2], %[tn2], %[tp1] \n\t" /* average even 1 */ + "dpa.w.ph $ac2, %[p3], %[filter45] \n\t" + "extp %[Temp4], $ac2, 31 \n\t" + + "lbu %[tp1], 2(%[dst]) \n\t" /* load even 2 */ + "sb %[tn2], 0(%[dst]) \n\t" /* store even 1 */ + + /* clamp */ + "addqh_r.w %[Temp1], %[Temp1], %[tn1] \n\t" /* average odd 1 */ + "lbux %[p3], %[Temp4](%[cm]) \n\t" /* odd 2 */ + "sb %[Temp1], 1(%[dst]) \n\t" /* store odd 1 */ + + "addqh_r.w %[tp1], %[tp1], %[tp2] \n\t" /* average even 2 */ + "sb %[tp1], 2(%[dst]) \n\t" /* store even 2 */ + + "addqh_r.w %[p2], %[p2], %[p3] \n\t" /* average odd 2 */ + "sb %[p2], 3(%[dst]) \n\t" /* store odd 2 */ + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tn1] "=&r"(tn1), + [tn2] "=&r"(tn2), [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), + [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3), + [Temp4] "=&r"(Temp4) + : [filter45] "r"(filter45), [vector4a] "r"(vector4a), [cm] "r"(cm), + [dst] "r"(dst), [src] "r"(src)); + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_bi_avg_horiz_8_dspr2(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector4a = 64; + int32_t Temp1, Temp2, Temp3; + uint32_t tp1, tp2, tp3, tp4; + uint32_t p1, p2, p3, p4, n1; + uint32_t st0, st1; + const int16_t *filter = &filter_x0[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tp1] \n\t" + "preceu.ph.qbl %[p2], %[tp1] \n\t" + "preceu.ph.qbr %[p3], %[tp2] \n\t" + "preceu.ph.qbl %[p4], %[tp2] \n\t" + "ulw %[tp3], 8(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + "lbu %[Temp2], 0(%[dst]) \n\t" + "lbu %[tp4], 2(%[dst]) \n\t" + + /* even 2. pixel */ + "dpa.w.ph $ac2, %[p2], %[filter45] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + /* even 3. pixel */ + "lbux %[st0], %[Temp1](%[cm]) \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "lbux %[st1], %[Temp3](%[cm]) \n\t" + "dpa.w.ph $ac1, %[p3], %[filter45] \n\t" + "extp %[Temp1], $ac1, 31 \n\t" + + "addqh_r.w %[Temp2], %[Temp2], %[st0] \n\t" + "addqh_r.w %[tp4], %[tp4], %[st1] \n\t" + "sb %[Temp2], 0(%[dst]) \n\t" + "sb %[tp4], 2(%[dst]) \n\t" + + /* even 4. pixel */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "balign %[tp3], %[tp2], 3 \n\t" + "balign %[tp2], %[tp1], 3 \n\t" + + "lbux %[st0], %[Temp1](%[cm]) \n\t" + "lbu %[Temp2], 4(%[dst]) \n\t" + "addqh_r.w %[Temp2], %[Temp2], %[st0] \n\t" + + "dpa.w.ph $ac2, %[p4], %[filter45] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + /* odd 1. pixel */ + "mtlo %[vector4a], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "sb %[Temp2], 4(%[dst]) \n\t" + "preceu.ph.qbr %[p1], %[tp2] \n\t" + "preceu.ph.qbl %[p2], %[tp2] \n\t" + "preceu.ph.qbr %[p3], %[tp3] \n\t" + "preceu.ph.qbl %[p4], %[tp3] \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + "lbu %[tp1], 6(%[dst]) \n\t" + + /* odd 2. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "lbux %[st0], %[Temp3](%[cm]) \n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" + "extp %[Temp3], $ac1, 31 \n\t" + + "lbu %[tp2], 1(%[dst]) \n\t" + "lbu %[tp3], 3(%[dst]) \n\t" + "addqh_r.w %[tp1], %[tp1], %[st0] \n\t" + + /* odd 3. pixel */ + "lbux %[st1], %[Temp2](%[cm]) \n\t" + "dpa.w.ph $ac3, %[p3], %[filter45] \n\t" + "addqh_r.w %[tp2], %[tp2], %[st1] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + "lbu %[tp4], 5(%[dst]) \n\t" + + /* odd 4. pixel */ + "sb %[tp2], 1(%[dst]) \n\t" + "sb %[tp1], 6(%[dst]) \n\t" + "dpa.w.ph $ac2, %[p4], %[filter45] \n\t" + "extp %[Temp1], $ac2, 31 \n\t" + + "lbu %[tp1], 7(%[dst]) \n\t" + + /* clamp */ + "lbux %[p4], %[Temp3](%[cm]) \n\t" + "addqh_r.w %[tp3], %[tp3], %[p4] \n\t" + + "lbux %[p2], %[Temp2](%[cm]) \n\t" + "addqh_r.w %[tp4], %[tp4], %[p2] \n\t" + + "lbux %[p1], %[Temp1](%[cm]) \n\t" + "addqh_r.w %[tp1], %[tp1], %[p1] \n\t" + + /* store bytes */ + "sb %[tp3], 3(%[dst]) \n\t" + "sb %[tp4], 5(%[dst]) \n\t" + "sb %[tp1], 7(%[dst]) \n\t" + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tp3] "=&r"(tp3), + [tp4] "=&r"(tp4), [st0] "=&r"(st0), [st1] "=&r"(st1), [p1] "=&r"(p1), + [p2] "=&r"(p2), [p3] "=&r"(p3), [p4] "=&r"(p4), [n1] "=&r"(n1), + [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3) + : [filter45] "r"(filter45), [vector4a] "r"(vector4a), [cm] "r"(cm), + [dst] "r"(dst), [src] "r"(src)); + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_bi_avg_horiz_16_dspr2(const uint8_t *src_ptr, + int32_t src_stride, uint8_t *dst_ptr, + int32_t dst_stride, + const int16_t *filter_x0, int32_t h, + int32_t count) { + int32_t y, c; + const uint8_t *src; + uint8_t *dst; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector_64 = 64; + int32_t Temp1, Temp2, Temp3; + uint32_t qload1, qload2, qload3; + uint32_t p1, p2, p3, p4, p5; + uint32_t st1, st2, st3; + const int16_t *filter = &filter_x0[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + src = src_ptr; + dst = dst_ptr; + + /* prefetch data to cache memory */ + prefetch_load(src_ptr + src_stride); + prefetch_load(src_ptr + src_stride + 32); + prefetch_store(dst_ptr + dst_stride); + + for (c = 0; c < count; c++) { + __asm__ __volatile__( + "ulw %[qload1], 0(%[src]) \n\t" + "ulw %[qload2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 1 */ + "mthi $zero, $ac1 \n\t" + "mtlo %[vector_64], $ac2 \n\t" /* even 2 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "ulw %[qload3], 8(%[src]) \n\t" + "dpa.w.ph $ac1, %[p1], %[filter45] \n\t" /* even 1 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 1 */ + "lbu %[st2], 0(%[dst]) \n\t" /* load even 1 from dst */ + + /* even 2. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 3 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "ulw %[qload1], 12(%[src]) \n\t" + "dpa.w.ph $ac2, %[p2], %[filter45] \n\t" /* even 1 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 1 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 1 */ + + "lbu %[qload3], 2(%[dst]) \n\t" /* load even 2 from dst */ + + /* even 3. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 4 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[st2], %[st2], %[st1] \n\t" /* average even 1 */ + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "sb %[st2], 0(%[dst]) \n\t" /* store even 1 to dst */ + "dpa.w.ph $ac3, %[p3], %[filter45] \n\t" /* even 3 */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 3 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 1 */ + + /* even 4. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 5 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st2] \n\t" /* average even 2 */ + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[qload3], 2(%[dst]) \n\t" /* store even 2 to dst */ + "lbu %[qload3], 4(%[dst]) \n\t" /* load even 3 from dst */ + "lbu %[qload1], 6(%[dst]) \n\t" /* load even 4 from dst */ + "dpa.w.ph $ac1, %[p4], %[filter45] \n\t" /* even 4 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 4 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 3 */ + + /* even 5. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 6 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st3] \n\t" /* average even 3 */ + "sb %[qload3], 4(%[dst]) \n\t" /* store even 3 to dst */ + "dpa.w.ph $ac2, %[p1], %[filter45] \n\t" /* even 5 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 5 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 4 */ + + /* even 6. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 7 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[qload1], %[qload1], %[st1] \n\t" /* average even 4 */ + "sb %[qload1], 6(%[dst]) \n\t" /* store even 4 to dst */ + "dpa.w.ph $ac3, %[p5], %[filter45] \n\t" /* even 6 */ + "lbu %[qload2], 8(%[dst]) \n\t" /* load even 5 from dst */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 6 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 5 */ + + /* even 7. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 8 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload2], %[qload2], %[st2] \n\t" /* average even 5 */ + "sb %[qload2], 8(%[dst]) \n\t" /* store even 5 to dst */ + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" /* even 7 */ + "lbu %[qload3], 10(%[dst]) \n\t" /* load even 6 from dst */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 7 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 6 */ + + "lbu %[st2], 12(%[dst]) \n\t" /* load even 7 from dst */ + + /* even 8. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 1 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st3] \n\t" /* average even 6 */ + "dpa.w.ph $ac2, %[p3], %[filter45] \n\t" /* even 8 */ + "sb %[qload3], 10(%[dst]) \n\t" /* store even 6 to dst */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 8 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 7 */ + + /* ODD pixels */ + "ulw %[qload1], 1(%[src]) \n\t" + "ulw %[qload2], 5(%[src]) \n\t" + + "addqh_r.w %[st2], %[st2], %[st1] \n\t" /* average even 7 */ + + /* odd 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 2 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "sb %[st2], 12(%[dst]) \n\t" /* store even 7 to dst */ + "ulw %[qload3], 9(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" /* odd 1 */ + "lbu %[qload2], 14(%[dst]) \n\t" /* load even 8 from dst */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 1 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 8 */ + + "lbu %[st1], 1(%[dst]) \n\t" /* load odd 1 from dst */ + + /* odd 2. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 3 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload2], %[qload2], %[st2] \n\t" /* average even 8 */ + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "sb %[qload2], 14(%[dst]) \n\t" /* store even 8 to dst */ + "ulw %[qload1], 13(%[src]) \n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" /* odd 2 */ + "lbu %[qload3], 3(%[dst]) \n\t" /* load odd 2 from dst */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 2 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 1 */ + + /* odd 3. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 4 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[st3], %[st3], %[st1] \n\t" /* average odd 1 */ + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "dpa.w.ph $ac2, %[p3], %[filter45] \n\t" /* odd 3 */ + "sb %[st3], 1(%[dst]) \n\t" /* store odd 1 to dst */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 3 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 2 */ + + /* odd 4. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 5 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st1] \n\t" /* average odd 2 */ + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[qload3], 3(%[dst]) \n\t" /* store odd 2 to dst */ + "lbu %[qload1], 5(%[dst]) \n\t" /* load odd 3 from dst */ + "dpa.w.ph $ac3, %[p4], %[filter45] \n\t" /* odd 4 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 4 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 3 */ + + "lbu %[st1], 7(%[dst]) \n\t" /* load odd 4 from dst */ + + /* odd 5. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 6 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload1], %[qload1], %[st2] \n\t" /* average odd 3 */ + "sb %[qload1], 5(%[dst]) \n\t" /* store odd 3 to dst */ + "dpa.w.ph $ac1, %[p1], %[filter45] \n\t" /* odd 5 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 5 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 4 */ + + "lbu %[qload1], 9(%[dst]) \n\t" /* load odd 5 from dst */ + + /* odd 6. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 7 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[st1], %[st1], %[st3] \n\t" /* average odd 4 */ + "sb %[st1], 7(%[dst]) \n\t" /* store odd 4 to dst */ + "dpa.w.ph $ac2, %[p5], %[filter45] \n\t" /* odd 6 */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 6 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 5 */ + + /* odd 7. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 8 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[qload1], %[qload1], %[st1] \n\t" /* average odd 5 */ + "sb %[qload1], 9(%[dst]) \n\t" /* store odd 5 to dst */ + "lbu %[qload2], 11(%[dst]) \n\t" /* load odd 6 from dst */ + "dpa.w.ph $ac3, %[p2], %[filter45] \n\t" /* odd 7 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 7 */ + + "lbu %[qload3], 13(%[dst]) \n\t" /* load odd 7 from dst */ + + /* odd 8. pixel */ + "dpa.w.ph $ac1, %[p3], %[filter45] \n\t" /* odd 8 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 8 */ + + "lbu %[qload1], 15(%[dst]) \n\t" /* load odd 8 from dst */ + + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 6 */ + "addqh_r.w %[qload2], %[qload2], %[st2] \n\t" /* average odd 6 */ + + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 7 */ + "addqh_r.w %[qload3], %[qload3], %[st3] \n\t" /* average odd 7 */ + + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 8 */ + "addqh_r.w %[qload1], %[qload1], %[st1] \n\t" /* average odd 8 */ + + "sb %[qload2], 11(%[dst]) \n\t" /* store odd 6 to dst */ + "sb %[qload3], 13(%[dst]) \n\t" /* store odd 7 to dst */ + "sb %[qload1], 15(%[dst]) \n\t" /* store odd 8 to dst */ + + : [qload1] "=&r"(qload1), [qload2] "=&r"(qload2), [st1] "=&r"(st1), + [st2] "=&r"(st2), [st3] "=&r"(st3), [p1] "=&r"(p1), [p2] "=&r"(p2), + [p3] "=&r"(p3), [p4] "=&r"(p4), [qload3] "=&r"(qload3), + [p5] "=&r"(p5), [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), + [Temp3] "=&r"(Temp3) + : [filter45] "r"(filter45), [vector_64] "r"(vector_64), [cm] "r"(cm), + [dst] "r"(dst), [src] "r"(src)); + + src += 16; + dst += 16; + } + + /* Next row... */ + src_ptr += src_stride; + dst_ptr += dst_stride; + } +} + +static void convolve_bi_avg_horiz_64_dspr2(const uint8_t *src_ptr, + int32_t src_stride, uint8_t *dst_ptr, + int32_t dst_stride, + const int16_t *filter_x0, + int32_t h) { + int32_t y, c; + const uint8_t *src; + uint8_t *dst; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector_64 = 64; + int32_t Temp1, Temp2, Temp3; + uint32_t qload1, qload2, qload3; + uint32_t p1, p2, p3, p4, p5; + uint32_t st1, st2, st3; + const int16_t *filter = &filter_x0[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + src = src_ptr; + dst = dst_ptr; + + /* prefetch data to cache memory */ + prefetch_load(src_ptr + src_stride); + prefetch_load(src_ptr + src_stride + 32); + prefetch_load(src_ptr + src_stride + 64); + prefetch_store(dst_ptr + dst_stride); + prefetch_store(dst_ptr + dst_stride + 32); + + for (c = 0; c < 4; c++) { + __asm__ __volatile__( + "ulw %[qload1], 0(%[src]) \n\t" + "ulw %[qload2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 1 */ + "mthi $zero, $ac1 \n\t" + "mtlo %[vector_64], $ac2 \n\t" /* even 2 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "ulw %[qload3], 8(%[src]) \n\t" + "dpa.w.ph $ac1, %[p1], %[filter45] \n\t" /* even 1 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 1 */ + "lbu %[st2], 0(%[dst]) \n\t" /* load even 1 from dst */ + + /* even 2. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 3 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "ulw %[qload1], 12(%[src]) \n\t" + "dpa.w.ph $ac2, %[p2], %[filter45] \n\t" /* even 1 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 1 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 1 */ + + "lbu %[qload3], 2(%[dst]) \n\t" /* load even 2 from dst */ + + /* even 3. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 4 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[st2], %[st2], %[st1] \n\t" /* average even 1 */ + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "sb %[st2], 0(%[dst]) \n\t" /* store even 1 to dst */ + "dpa.w.ph $ac3, %[p3], %[filter45] \n\t" /* even 3 */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 3 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 1 */ + + /* even 4. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 5 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st2] \n\t" /* average even 2 */ + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[qload3], 2(%[dst]) \n\t" /* store even 2 to dst */ + "lbu %[qload3], 4(%[dst]) \n\t" /* load even 3 from dst */ + "lbu %[qload1], 6(%[dst]) \n\t" /* load even 4 from dst */ + "dpa.w.ph $ac1, %[p4], %[filter45] \n\t" /* even 4 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 4 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 3 */ + + /* even 5. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 6 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st3] \n\t" /* average even 3 */ + "sb %[qload3], 4(%[dst]) \n\t" /* store even 3 to dst */ + "dpa.w.ph $ac2, %[p1], %[filter45] \n\t" /* even 5 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 5 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 4 */ + + /* even 6. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 7 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[qload1], %[qload1], %[st1] \n\t" /* average even 4 */ + "sb %[qload1], 6(%[dst]) \n\t" /* store even 4 to dst */ + "dpa.w.ph $ac3, %[p5], %[filter45] \n\t" /* even 6 */ + "lbu %[qload2], 8(%[dst]) \n\t" /* load even 5 from dst */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 6 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 5 */ + + /* even 7. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 8 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload2], %[qload2], %[st2] \n\t" /* average even 5 */ + "sb %[qload2], 8(%[dst]) \n\t" /* store even 5 to dst */ + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" /* even 7 */ + "lbu %[qload3], 10(%[dst]) \n\t" /* load even 6 from dst */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 7 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 6 */ + + "lbu %[st2], 12(%[dst]) \n\t" /* load even 7 from dst */ + + /* even 8. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 1 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st3] \n\t" /* average even 6 */ + "dpa.w.ph $ac2, %[p3], %[filter45] \n\t" /* even 8 */ + "sb %[qload3], 10(%[dst]) \n\t" /* store even 6 to dst */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 8 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 7 */ + + /* ODD pixels */ + "ulw %[qload1], 1(%[src]) \n\t" + "ulw %[qload2], 5(%[src]) \n\t" + + "addqh_r.w %[st2], %[st2], %[st1] \n\t" /* average even 7 */ + + /* odd 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 2 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "sb %[st2], 12(%[dst]) \n\t" /* store even 7 to dst */ + "ulw %[qload3], 9(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" /* odd 1 */ + "lbu %[qload2], 14(%[dst]) \n\t" /* load even 8 from dst */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 1 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 8 */ + + "lbu %[st1], 1(%[dst]) \n\t" /* load odd 1 from dst */ + + /* odd 2. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 3 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload2], %[qload2], %[st2] \n\t" /* average even 8 */ + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "sb %[qload2], 14(%[dst]) \n\t" /* store even 8 to dst */ + "ulw %[qload1], 13(%[src]) \n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" /* odd 2 */ + "lbu %[qload3], 3(%[dst]) \n\t" /* load odd 2 from dst */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 2 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 1 */ + + /* odd 3. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 4 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[st3], %[st3], %[st1] \n\t" /* average odd 1 */ + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "dpa.w.ph $ac2, %[p3], %[filter45] \n\t" /* odd 3 */ + "sb %[st3], 1(%[dst]) \n\t" /* store odd 1 to dst */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 3 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 2 */ + + /* odd 4. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 5 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st1] \n\t" /* average odd 2 */ + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[qload3], 3(%[dst]) \n\t" /* store odd 2 to dst */ + "lbu %[qload1], 5(%[dst]) \n\t" /* load odd 3 from dst */ + "dpa.w.ph $ac3, %[p4], %[filter45] \n\t" /* odd 4 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 4 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 3 */ + + "lbu %[st1], 7(%[dst]) \n\t" /* load odd 4 from dst */ + + /* odd 5. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 6 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload1], %[qload1], %[st2] \n\t" /* average odd 3 */ + "sb %[qload1], 5(%[dst]) \n\t" /* store odd 3 to dst */ + "dpa.w.ph $ac1, %[p1], %[filter45] \n\t" /* odd 5 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 5 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 4 */ + + "lbu %[qload1], 9(%[dst]) \n\t" /* load odd 5 from dst */ + + /* odd 6. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 7 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[st1], %[st1], %[st3] \n\t" /* average odd 4 */ + "sb %[st1], 7(%[dst]) \n\t" /* store odd 4 to dst */ + "dpa.w.ph $ac2, %[p5], %[filter45] \n\t" /* odd 6 */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 6 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 5 */ + + /* odd 7. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 8 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[qload1], %[qload1], %[st1] \n\t" /* average odd 5 */ + "sb %[qload1], 9(%[dst]) \n\t" /* store odd 5 to dst */ + "lbu %[qload2], 11(%[dst]) \n\t" /* load odd 6 from dst */ + "dpa.w.ph $ac3, %[p2], %[filter45] \n\t" /* odd 7 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 7 */ + + "lbu %[qload3], 13(%[dst]) \n\t" /* load odd 7 from dst */ + + /* odd 8. pixel */ + "dpa.w.ph $ac1, %[p3], %[filter45] \n\t" /* odd 8 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 8 */ + + "lbu %[qload1], 15(%[dst]) \n\t" /* load odd 8 from dst */ + + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 6 */ + "addqh_r.w %[qload2], %[qload2], %[st2] \n\t" /* average odd 6 */ + + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 7 */ + "addqh_r.w %[qload3], %[qload3], %[st3] \n\t" /* average odd 7 */ + + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 8 */ + "addqh_r.w %[qload1], %[qload1], %[st1] \n\t" /* average odd 8 */ + + "sb %[qload2], 11(%[dst]) \n\t" /* store odd 6 to dst */ + "sb %[qload3], 13(%[dst]) \n\t" /* store odd 7 to dst */ + "sb %[qload1], 15(%[dst]) \n\t" /* store odd 8 to dst */ + + : [qload1] "=&r"(qload1), [qload2] "=&r"(qload2), [st1] "=&r"(st1), + [st2] "=&r"(st2), [st3] "=&r"(st3), [p1] "=&r"(p1), [p2] "=&r"(p2), + [p3] "=&r"(p3), [p4] "=&r"(p4), [qload3] "=&r"(qload3), + [p5] "=&r"(p5), [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), + [Temp3] "=&r"(Temp3) + : [filter45] "r"(filter45), [vector_64] "r"(vector_64), [cm] "r"(cm), + [dst] "r"(dst), [src] "r"(src)); + + src += 16; + dst += 16; + } + + /* Next row... */ + src_ptr += src_stride; + dst_ptr += dst_stride; + } +} + +void aom_convolve2_avg_horiz_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h) { + uint32_t pos = 38; + + assert(x_step_q4 == 16); + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + /* prefetch data to cache memory */ + prefetch_load(src); + prefetch_load(src + 32); + prefetch_store(dst); + + switch (w) { + case 4: + convolve_bi_avg_horiz_4_dspr2(src, src_stride, dst, dst_stride, filter_x, + h); + break; + case 8: + convolve_bi_avg_horiz_8_dspr2(src, src_stride, dst, dst_stride, filter_x, + h); + break; + case 16: + convolve_bi_avg_horiz_16_dspr2(src, src_stride, dst, dst_stride, filter_x, + h, 1); + break; + case 32: + convolve_bi_avg_horiz_16_dspr2(src, src_stride, dst, dst_stride, filter_x, + h, 2); + break; + case 64: + prefetch_load(src + 64); + prefetch_store(dst + 32); + + convolve_bi_avg_horiz_64_dspr2(src, src_stride, dst, dst_stride, filter_x, + h); + break; + default: + aom_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } +} +#endif diff --git a/third_party/aom/aom_dsp/mips/convolve2_dspr2.c b/third_party/aom/aom_dsp/mips/convolve2_dspr2.c new file mode 100644 index 0000000000..066308315d --- /dev/null +++ b/third_party/aom/aom_dsp/mips/convolve2_dspr2.c @@ -0,0 +1,1030 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/convolve_common_dspr2.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/aom_filter.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +static void convolve_bi_horiz_4_transposed_dspr2( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y; + uint8_t *cm = aom_ff_cropTbl; + uint8_t *dst_ptr; + int32_t Temp1, Temp2; + uint32_t vector4a = 64; + uint32_t tp1, tp2; + uint32_t p1, p2; + const int16_t *filter = &filter_x0[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + dst_ptr = dst; + /* prefetch data to cache memory */ + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[tp1] \n\t" + "preceu.ph.qbl %[p2], %[tp1] \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + + /* even 2. pixel */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "balign %[tp2], %[tp1], 3 \n\t" + "dpa.w.ph $ac2, %[p2], %[filter45] \n\t" + "extp %[Temp2], $ac2, 31 \n\t" + + /* odd 1. pixel */ + "lbux %[tp1], %[Temp1](%[cm]) \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[tp2] \n\t" + "preceu.ph.qbl %[p2], %[tp2] \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + + /* odd 2. pixel */ + "lbux %[tp2], %[Temp2](%[cm]) \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "dpa.w.ph $ac2, %[p2], %[filter45] \n\t" + "extp %[Temp2], $ac2, 31 \n\t" + + /* clamp */ + "lbux %[p1], %[Temp1](%[cm]) \n\t" + "lbux %[p2], %[Temp2](%[cm]) \n\t" + + /* store bytes */ + "sb %[tp1], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_stride] \n\t" + + "sb %[p1], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_stride] \n\t" + + "sb %[tp2], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_stride] \n\t" + + "sb %[p2], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_stride] \n\t" + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [p1] "=&r"(p1), [p2] "=&r"(p2), + [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), [dst_ptr] "+r"(dst_ptr) + : [filter45] "r"(filter45), [vector4a] "r"(vector4a), [cm] "r"(cm), + [src] "r"(src), [dst_stride] "r"(dst_stride)); + + /* Next row... */ + src += src_stride; + dst += 1; + } +} + +static void convolve_bi_horiz_8_transposed_dspr2( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y; + uint8_t *cm = aom_ff_cropTbl; + uint8_t *dst_ptr; + uint32_t vector4a = 64; + int32_t Temp1, Temp2, Temp3; + uint32_t tp1, tp2, tp3; + uint32_t p1, p2, p3, p4; + uint8_t *odd_dst; + uint32_t dst_pitch_2 = (dst_stride << 1); + const int16_t *filter = &filter_x0[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + + dst_ptr = dst; + odd_dst = (dst_ptr + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tp1] \n\t" + "preceu.ph.qbl %[p2], %[tp1] \n\t" + "preceu.ph.qbr %[p3], %[tp2] \n\t" + "preceu.ph.qbl %[p4], %[tp2] \n\t" + "ulw %[tp3], 8(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + + /* even 2. pixel */ + "dpa.w.ph $ac2, %[p2], %[filter45] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + /* even 3. pixel */ + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "balign %[tp3], %[tp2], 3 \n\t" + "balign %[tp2], %[tp1], 3 \n\t" + "dpa.w.ph $ac1, %[p3], %[filter45] \n\t" + "lbux %[tp1], %[Temp3](%[cm]) \n\t" + "extp %[p3], $ac1, 31 \n\t" + + /* even 4. pixel */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "sb %[Temp2], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_pitch_2] \n\t" + "sb %[tp1], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_pitch_2] \n\t" + + "dpa.w.ph $ac2, %[p4], %[filter45] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + "lbux %[Temp1], %[p3](%[cm]) " + "\n\t" + + /* odd 1. pixel */ + "mtlo %[vector4a], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p1], %[tp2] \n\t" + "preceu.ph.qbl %[p2], %[tp2] \n\t" + "preceu.ph.qbr %[p3], %[tp3] \n\t" + "preceu.ph.qbl %[p4], %[tp3] \n\t" + "sb %[Temp1], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_pitch_2] \n\t" + + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + /* odd 2. pixel */ + "lbux %[tp1], %[Temp3](%[cm]) \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" + "sb %[tp1], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_pitch_2] \n\t" + "extp %[Temp3], $ac1, 31 \n\t" + + /* odd 3. pixel */ + "lbux %[tp3], %[Temp2](%[cm]) \n\t" + "dpa.w.ph $ac3, %[p3], %[filter45] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + /* odd 4. pixel */ + "sb %[tp3], 0(%[odd_dst]) \n\t" + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] \n\t" + "dpa.w.ph $ac2, %[p4], %[filter45] \n\t" + "extp %[Temp1], $ac2, 31 \n\t" + + /* clamp */ + "lbux %[p4], %[Temp3](%[cm]) \n\t" + "lbux %[p2], %[Temp2](%[cm]) \n\t" + "lbux %[p1], %[Temp1](%[cm]) \n\t" + + /* store bytes */ + "sb %[p4], 0(%[odd_dst]) \n\t" + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] \n\t" + + "sb %[p2], 0(%[odd_dst]) \n\t" + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] \n\t" + + "sb %[p1], 0(%[odd_dst]) \n\t" + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tp3] "=&r"(tp3), [p1] "=&r"(p1), + [p2] "=&r"(p2), [p3] "=&r"(p3), [p4] "=&r"(p4), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3), [dst_ptr] "+r"(dst_ptr), + [odd_dst] "+r"(odd_dst) + : [filter45] "r"(filter45), [vector4a] "r"(vector4a), [cm] "r"(cm), + [src] "r"(src), [dst_pitch_2] "r"(dst_pitch_2)); + + /* Next row... */ + src += src_stride; + dst += 1; + } +} + +static void convolve_bi_horiz_16_transposed_dspr2( + const uint8_t *src_ptr, int32_t src_stride, uint8_t *dst_ptr, + int32_t dst_stride, const int16_t *filter_x0, int32_t h, int32_t count) { + int32_t c, y; + const uint8_t *src; + uint8_t *dst; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector_64 = 64; + int32_t Temp1, Temp2, Temp3; + uint32_t qload1, qload2; + uint32_t p1, p2, p3, p4, p5; + uint32_t st1, st2, st3; + uint32_t dst_pitch_2 = (dst_stride << 1); + uint8_t *odd_dst; + const int16_t *filter = &filter_x0[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src_ptr + src_stride); + prefetch_load(src_ptr + src_stride + 32); + + src = src_ptr; + dst = dst_ptr; + + odd_dst = (dst + dst_stride); + + for (c = 0; c < count; c++) { + __asm__ __volatile__( + "ulw %[qload1], 0(%[src]) " + "\n\t" + "ulw %[qload2], 4(%[src]) " + "\n\t" + + /* even 1. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* even 1 */ + "mthi $zero, $ac1 " + "\n\t" + "mtlo %[vector_64], $ac2 " + "\n\t" /* even 2 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload1] " + "\n\t" + "preceu.ph.qbl %[p2], %[qload1] " + "\n\t" + "preceu.ph.qbr %[p3], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p4], %[qload2] " + "\n\t" + "ulw %[qload1], 8(%[src]) " + "\n\t" + "dpa.w.ph $ac1, %[p1], %[filter45] " + "\n\t" /* even 1 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* even 1 */ + + /* even 2. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* even 3 */ + "mthi $zero, $ac3 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload1] " + "\n\t" + "preceu.ph.qbl %[p5], %[qload1] " + "\n\t" + "ulw %[qload2], 12(%[src]) " + "\n\t" + "dpa.w.ph $ac2, %[p2], %[filter45] " + "\n\t" /* even 1 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* even 1 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* even 1 */ + + /* even 3. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* even 4 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbr %[p2], %[qload2] " + "\n\t" + "sb %[st1], 0(%[dst]) " + "\n\t" /* even 1 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + " \n\t" + "dpa.w.ph $ac3, %[p3], %[filter45] " + "\n\t" /* even 3 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* even 3 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* even 1 */ + + /* even 4. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* even 5 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbl %[p3], %[qload2] " + "\n\t" + "sb %[st2], 0(%[dst]) " + "\n\t" /* even 2 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac1, %[p4], %[filter45] " + "\n\t" /* even 4 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* even 4 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* even 3 */ + + /* even 5. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* even 6 */ + "mthi $zero, $ac3 " + "\n\t" + "sb %[st3], 0(%[dst]) " + "\n\t" /* even 3 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac2, %[p1], %[filter45] " + "\n\t" /* even 5 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* even 5 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* even 4 */ + + /* even 6. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* even 7 */ + "mthi $zero, $ac1 " + "\n\t" + "sb %[st1], 0(%[dst]) " + "\n\t" /* even 4 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload1], 20(%[src]) " + "\n\t" + "dpa.w.ph $ac3, %[p5], %[filter45] " + "\n\t" /* even 6 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* even 6 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* even 5 */ + + /* even 7. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* even 8 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p5], %[qload1] " + "\n\t" + "sb %[st2], 0(%[dst]) " + "\n\t" /* even 5 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] " + "\n\t" /* even 7 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* even 7 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* even 6 */ + + /* even 8. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* odd 1 */ + "mthi $zero, $ac3 " + "\n\t" + "dpa.w.ph $ac2, %[p3], %[filter45] " + "\n\t" /* even 8 */ + "sb %[st3], 0(%[dst]) " + "\n\t" /* even 6 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "extp %[Temp2], $ac2, 31 " + "\n\t" /* even 8 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* even 7 */ + + /* ODD pixels */ + "ulw %[qload1], 1(%[src]) " + "\n\t" + "ulw %[qload2], 5(%[src]) " + "\n\t" + + /* odd 1. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* odd 2 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload1] " + "\n\t" + "preceu.ph.qbl %[p2], %[qload1] " + "\n\t" + "preceu.ph.qbr %[p3], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p4], %[qload2] " + "\n\t" + "sb %[st1], 0(%[dst]) " + "\n\t" /* even 7 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload2], 9(%[src]) " + "\n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] " + "\n\t" /* odd 1 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* odd 1 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* even 8 */ + + /* odd 2. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* odd 3 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p5], %[qload2] " + "\n\t" + "sb %[st2], 0(%[dst]) " + "\n\t" /* even 8 */ + "ulw %[qload1], 13(%[src]) " + "\n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] " + "\n\t" /* odd 2 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* odd 2 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* odd 1 */ + + /* odd 3. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* odd 4 */ + "mthi $zero, $ac3 " + "\n\t" + "preceu.ph.qbr %[p2], %[qload1] " + "\n\t" + "sb %[st3], 0(%[odd_dst]) " + "\n\t" /* odd 1 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac2, %[p3], %[filter45] " + "\n\t" /* odd 3 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* odd 3 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* odd 2 */ + + /* odd 4. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* odd 5 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbl %[p3], %[qload1] " + "\n\t" + "sb %[st1], 0(%[odd_dst]) " + "\n\t" /* odd 2 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac3, %[p4], %[filter45] " + "\n\t" /* odd 4 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* odd 4 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* odd 3 */ + + /* odd 5. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* odd 6 */ + "mthi $zero, $ac2 " + "\n\t" + "sb %[st2], 0(%[odd_dst]) " + "\n\t" /* odd 3 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac1, %[p1], %[filter45] " + "\n\t" /* odd 5 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* odd 5 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* odd 4 */ + + /* odd 6. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* odd 7 */ + "mthi $zero, $ac3 " + "\n\t" + "sb %[st3], 0(%[odd_dst]) " + "\n\t" /* odd 4 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload1], 21(%[src]) " + "\n\t" + "dpa.w.ph $ac2, %[p5], %[filter45] " + "\n\t" /* odd 6 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* odd 6 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* odd 5 */ + + /* odd 7. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* odd 8 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbr %[p5], %[qload1] " + "\n\t" + "sb %[st1], 0(%[odd_dst]) " + "\n\t" /* odd 5 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac3, %[p2], %[filter45] " + "\n\t" /* odd 7 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* odd 7 */ + + /* odd 8. pixel */ + "dpa.w.ph $ac1, %[p3], %[filter45] " + "\n\t" /* odd 8 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* odd 8 */ + + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* odd 6 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* odd 7 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* odd 8 */ + + "sb %[st2], 0(%[odd_dst]) " + "\n\t" /* odd 6 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + + "sb %[st3], 0(%[odd_dst]) " + "\n\t" /* odd 7 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + + "sb %[st1], 0(%[odd_dst]) " + "\n\t" /* odd 8 */ + + : [qload1] "=&r"(qload1), [qload2] "=&r"(qload2), [p5] "=&r"(p5), + [st1] "=&r"(st1), [st2] "=&r"(st2), [st3] "=&r"(st3), + [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), [p4] "=&r"(p4), + [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3), + [dst] "+r"(dst), [odd_dst] "+r"(odd_dst) + : [filter45] "r"(filter45), [vector_64] "r"(vector_64), [cm] "r"(cm), + [src] "r"(src), [dst_pitch_2] "r"(dst_pitch_2)); + + src += 16; + dst = (dst_ptr + ((c + 1) * 16 * dst_stride)); + odd_dst = (dst + dst_stride); + } + + /* Next row... */ + src_ptr += src_stride; + dst_ptr += 1; + } +} + +static void convolve_bi_horiz_64_transposed_dspr2( + const uint8_t *src_ptr, int32_t src_stride, uint8_t *dst_ptr, + int32_t dst_stride, const int16_t *filter_x0, int32_t h) { + int32_t c, y; + const uint8_t *src; + uint8_t *dst; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector_64 = 64; + int32_t Temp1, Temp2, Temp3; + uint32_t qload1, qload2; + uint32_t p1, p2, p3, p4, p5; + uint32_t st1, st2, st3; + uint32_t dst_pitch_2 = (dst_stride << 1); + uint8_t *odd_dst; + const int16_t *filter = &filter_x0[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src_ptr + src_stride); + prefetch_load(src_ptr + src_stride + 32); + prefetch_load(src_ptr + src_stride + 64); + + src = src_ptr; + dst = dst_ptr; + + odd_dst = (dst + dst_stride); + + for (c = 0; c < 4; c++) { + __asm__ __volatile__( + "ulw %[qload1], 0(%[src]) " + "\n\t" + "ulw %[qload2], 4(%[src]) " + "\n\t" + + /* even 1. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* even 1 */ + "mthi $zero, $ac1 " + "\n\t" + "mtlo %[vector_64], $ac2 " + "\n\t" /* even 2 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload1] " + "\n\t" + "preceu.ph.qbl %[p2], %[qload1] " + "\n\t" + "preceu.ph.qbr %[p3], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p4], %[qload2] " + "\n\t" + "ulw %[qload1], 8(%[src]) " + "\n\t" + "dpa.w.ph $ac1, %[p1], %[filter45] " + "\n\t" /* even 1 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* even 1 */ + + /* even 2. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* even 3 */ + "mthi $zero, $ac3 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload1] " + "\n\t" + "preceu.ph.qbl %[p5], %[qload1] " + "\n\t" + "ulw %[qload2], 12(%[src]) " + "\n\t" + "dpa.w.ph $ac2, %[p2], %[filter45] " + "\n\t" /* even 1 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* even 1 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* even 1 */ + + /* even 3. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* even 4 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbr %[p2], %[qload2] " + "\n\t" + "sb %[st1], 0(%[dst]) " + "\n\t" /* even 1 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + " \n\t" + "dpa.w.ph $ac3, %[p3], %[filter45] " + "\n\t" /* even 3 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* even 3 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* even 1 */ + + /* even 4. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* even 5 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbl %[p3], %[qload2] " + "\n\t" + "sb %[st2], 0(%[dst]) " + "\n\t" /* even 2 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac1, %[p4], %[filter45] " + "\n\t" /* even 4 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* even 4 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* even 3 */ + + /* even 5. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* even 6 */ + "mthi $zero, $ac3 " + "\n\t" + "sb %[st3], 0(%[dst]) " + "\n\t" /* even 3 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac2, %[p1], %[filter45] " + "\n\t" /* even 5 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* even 5 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* even 4 */ + + /* even 6. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* even 7 */ + "mthi $zero, $ac1 " + "\n\t" + "sb %[st1], 0(%[dst]) " + "\n\t" /* even 4 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload1], 20(%[src]) " + "\n\t" + "dpa.w.ph $ac3, %[p5], %[filter45] " + "\n\t" /* even 6 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* even 6 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* even 5 */ + + /* even 7. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* even 8 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p5], %[qload1] " + "\n\t" + "sb %[st2], 0(%[dst]) " + "\n\t" /* even 5 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] " + "\n\t" /* even 7 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* even 7 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* even 6 */ + + /* even 8. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* odd 1 */ + "mthi $zero, $ac3 " + "\n\t" + "dpa.w.ph $ac2, %[p3], %[filter45] " + "\n\t" /* even 8 */ + "sb %[st3], 0(%[dst]) " + "\n\t" /* even 6 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "extp %[Temp2], $ac2, 31 " + "\n\t" /* even 8 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* even 7 */ + + /* ODD pixels */ + "ulw %[qload1], 1(%[src]) " + "\n\t" + "ulw %[qload2], 5(%[src]) " + "\n\t" + + /* odd 1. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* odd 2 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload1] " + "\n\t" + "preceu.ph.qbl %[p2], %[qload1] " + "\n\t" + "preceu.ph.qbr %[p3], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p4], %[qload2] " + "\n\t" + "sb %[st1], 0(%[dst]) " + "\n\t" /* even 7 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload2], 9(%[src]) " + "\n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] " + "\n\t" /* odd 1 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* odd 1 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* even 8 */ + + /* odd 2. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* odd 3 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p5], %[qload2] " + "\n\t" + "sb %[st2], 0(%[dst]) " + "\n\t" /* even 8 */ + "ulw %[qload1], 13(%[src]) " + "\n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] " + "\n\t" /* odd 2 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* odd 2 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* odd 1 */ + + /* odd 3. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* odd 4 */ + "mthi $zero, $ac3 " + "\n\t" + "preceu.ph.qbr %[p2], %[qload1] " + "\n\t" + "sb %[st3], 0(%[odd_dst]) " + "\n\t" /* odd 1 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac2, %[p3], %[filter45] " + "\n\t" /* odd 3 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* odd 3 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* odd 2 */ + + /* odd 4. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* odd 5 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbl %[p3], %[qload1] " + "\n\t" + "sb %[st1], 0(%[odd_dst]) " + "\n\t" /* odd 2 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac3, %[p4], %[filter45] " + "\n\t" /* odd 4 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* odd 4 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* odd 3 */ + + /* odd 5. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* odd 6 */ + "mthi $zero, $ac2 " + "\n\t" + "sb %[st2], 0(%[odd_dst]) " + "\n\t" /* odd 3 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac1, %[p1], %[filter45] " + "\n\t" /* odd 5 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* odd 5 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* odd 4 */ + + /* odd 6. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* odd 7 */ + "mthi $zero, $ac3 " + "\n\t" + "sb %[st3], 0(%[odd_dst]) " + "\n\t" /* odd 4 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload1], 21(%[src]) " + "\n\t" + "dpa.w.ph $ac2, %[p5], %[filter45] " + "\n\t" /* odd 6 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* odd 6 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* odd 5 */ + + /* odd 7. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* odd 8 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbr %[p5], %[qload1] " + "\n\t" + "sb %[st1], 0(%[odd_dst]) " + "\n\t" /* odd 5 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac3, %[p2], %[filter45] " + "\n\t" /* odd 7 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* odd 7 */ + + /* odd 8. pixel */ + "dpa.w.ph $ac1, %[p3], %[filter45] " + "\n\t" /* odd 8 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* odd 8 */ + + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* odd 6 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* odd 7 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* odd 8 */ + + "sb %[st2], 0(%[odd_dst]) " + "\n\t" /* odd 6 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + + "sb %[st3], 0(%[odd_dst]) " + "\n\t" /* odd 7 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + + "sb %[st1], 0(%[odd_dst]) " + "\n\t" /* odd 8 */ + + : [qload1] "=&r"(qload1), [qload2] "=&r"(qload2), [p5] "=&r"(p5), + [st1] "=&r"(st1), [st2] "=&r"(st2), [st3] "=&r"(st3), + [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), [p4] "=&r"(p4), + [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3), + [dst] "+r"(dst), [odd_dst] "+r"(odd_dst) + : [filter45] "r"(filter45), [vector_64] "r"(vector_64), [cm] "r"(cm), + [src] "r"(src), [dst_pitch_2] "r"(dst_pitch_2)); + + src += 16; + dst = (dst_ptr + ((c + 1) * 16 * dst_stride)); + odd_dst = (dst + dst_stride); + } + + /* Next row... */ + src_ptr += src_stride; + dst_ptr += 1; + } +} + +void convolve_bi_horiz_transposed(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter, int w, int h) { + int x, y; + + for (y = 0; y < h; ++y) { + for (x = 0; x < w; ++x) { + int sum = 0; + + sum += src[x] * filter[3]; + sum += src[x + 1] * filter[4]; + + dst[x * dst_stride] = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); + } + + src += src_stride; + dst += 1; + } +} + +void aom_convolve2_dspr2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter, int w, + int h) { + uint32_t pos = 38; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + /* prefetch data to cache memory */ + prefetch_load(src); + prefetch_load(src + 32); + + switch (w) { + case 4: + convolve_bi_horiz_4_transposed_dspr2(src, src_stride, dst, dst_stride, + filter, h); + break; + case 8: + convolve_bi_horiz_8_transposed_dspr2(src, src_stride, dst, dst_stride, + filter, h); + break; + case 16: + case 32: + convolve_bi_horiz_16_transposed_dspr2(src, src_stride, dst, dst_stride, + filter, h, (w / 16)); + break; + case 64: + prefetch_load(src + 32); + convolve_bi_horiz_64_transposed_dspr2(src, src_stride, dst, dst_stride, + filter, h); + break; + default: + convolve_bi_horiz_transposed(src, src_stride, dst, dst_stride, filter, w, + h); + break; + } +} +#endif diff --git a/third_party/aom/aom_dsp/mips/convolve2_horiz_dspr2.c b/third_party/aom/aom_dsp/mips/convolve2_horiz_dspr2.c new file mode 100644 index 0000000000..dc51ab1cb7 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/convolve2_horiz_dspr2.c @@ -0,0 +1,681 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/convolve_common_dspr2.h" +#include "aom_dsp/aom_convolve.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +static void convolve_bi_horiz_4_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y; + uint8_t *cm = aom_ff_cropTbl; + int32_t Temp1, Temp2, Temp3, Temp4; + uint32_t vector4a = 64; + uint32_t tp1, tp2; + uint32_t p1, p2; + const int16_t *filter = &filter_x0[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[tp1] \n\t" + "preceu.ph.qbl %[p2], %[tp1] \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + + /* even 2. pixel */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "balign %[tp2], %[tp1], 3 \n\t" + "dpa.w.ph $ac2, %[p2], %[filter45] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + /* odd 1. pixel */ + "lbux %[tp1], %[Temp1](%[cm]) \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[tp2] \n\t" + "preceu.ph.qbl %[p2], %[tp2] \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + /* odd 2. pixel */ + "lbux %[tp2], %[Temp3](%[cm]) \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "dpa.w.ph $ac2, %[p2], %[filter45] \n\t" + "extp %[Temp4], $ac2, 31 \n\t" + + /* clamp */ + "lbux %[p1], %[Temp2](%[cm]) \n\t" + "lbux %[p2], %[Temp4](%[cm]) \n\t" + + /* store bytes */ + "sb %[tp1], 0(%[dst]) \n\t" + "sb %[p1], 1(%[dst]) \n\t" + "sb %[tp2], 2(%[dst]) \n\t" + "sb %[p2], 3(%[dst]) \n\t" + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [p1] "=&r"(p1), [p2] "=&r"(p2), + [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3), + [Temp4] "=&r"(Temp4) + : [filter45] "r"(filter45), [vector4a] "r"(vector4a), [cm] "r"(cm), + [dst] "r"(dst), [src] "r"(src)); + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_bi_horiz_8_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector4a = 64; + int32_t Temp1, Temp2, Temp3; + uint32_t tp1, tp2, tp3; + uint32_t p1, p2, p3, p4; + uint32_t st0, st1; + const int16_t *filter = &filter_x0[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tp1] \n\t" + "preceu.ph.qbl %[p2], %[tp1] \n\t" + "preceu.ph.qbr %[p3], %[tp2] \n\t" + "preceu.ph.qbl %[p4], %[tp2] \n\t" + "ulw %[tp3], 8(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + + /* even 2. pixel */ + "dpa.w.ph $ac2, %[p2], %[filter45] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + /* even 3. pixel */ + "lbux %[st0], %[Temp1](%[cm]) \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "dpa.w.ph $ac1, %[p3], %[filter45] \n\t" + "extp %[Temp1], $ac1, 31 \n\t" + + /* even 4. pixel */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "sb %[st0], 0(%[dst]) \n\t" + "lbux %[st1], %[Temp3](%[cm]) \n\t" + + "balign %[tp3], %[tp2], 3 \n\t" + "balign %[tp2], %[tp1], 3 \n\t" + + "dpa.w.ph $ac2, %[p4], %[filter45] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + "lbux %[st0], %[Temp1](%[cm]) \n\t" + + /* odd 1. pixel */ + "mtlo %[vector4a], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "sb %[st1], 2(%[dst]) \n\t" + "preceu.ph.qbr %[p1], %[tp2] \n\t" + "preceu.ph.qbl %[p2], %[tp2] \n\t" + "preceu.ph.qbr %[p3], %[tp3] \n\t" + "preceu.ph.qbl %[p4], %[tp3] \n\t" + "sb %[st0], 4(%[dst]) \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + /* odd 2. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "lbux %[st0], %[Temp3](%[cm]) \n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" + "extp %[Temp3], $ac1, 31 \n\t" + + /* odd 3. pixel */ + "lbux %[st1], %[Temp2](%[cm]) \n\t" + "dpa.w.ph $ac3, %[p3], %[filter45] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + /* odd 4. pixel */ + "sb %[st1], 1(%[dst]) \n\t" + "sb %[st0], 6(%[dst]) \n\t" + "dpa.w.ph $ac2, %[p4], %[filter45] \n\t" + "extp %[Temp1], $ac2, 31 \n\t" + + /* clamp */ + "lbux %[p4], %[Temp3](%[cm]) \n\t" + "lbux %[p2], %[Temp2](%[cm]) \n\t" + "lbux %[p1], %[Temp1](%[cm]) \n\t" + + /* store bytes */ + "sb %[p4], 3(%[dst]) \n\t" + "sb %[p2], 5(%[dst]) \n\t" + "sb %[p1], 7(%[dst]) \n\t" + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tp3] "=&r"(tp3), + [st0] "=&r"(st0), [st1] "=&r"(st1), [p1] "=&r"(p1), [p2] "=&r"(p2), + [p3] "=&r"(p3), [p4] "=&r"(p4), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3) + : [filter45] "r"(filter45), [vector4a] "r"(vector4a), [cm] "r"(cm), + [dst] "r"(dst), [src] "r"(src)); + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_bi_horiz_16_dspr2(const uint8_t *src_ptr, + int32_t src_stride, uint8_t *dst_ptr, + int32_t dst_stride, + const int16_t *filter_x0, int32_t h, + int32_t count) { + int32_t y, c; + const uint8_t *src; + uint8_t *dst; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector_64 = 64; + int32_t Temp1, Temp2, Temp3; + uint32_t qload1, qload2, qload3; + uint32_t p1, p2, p3, p4, p5; + uint32_t st1, st2, st3; + const int16_t *filter = &filter_x0[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + src = src_ptr; + dst = dst_ptr; + + /* prefetch data to cache memory */ + prefetch_load(src_ptr + src_stride); + prefetch_load(src_ptr + src_stride + 32); + prefetch_store(dst_ptr + dst_stride); + + for (c = 0; c < count; c++) { + __asm__ __volatile__( + "ulw %[qload1], 0(%[src]) \n\t" + "ulw %[qload2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 1 */ + "mthi $zero, $ac1 \n\t" + "mtlo %[vector_64], $ac2 \n\t" /* even 2 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "ulw %[qload3], 8(%[src]) \n\t" + "dpa.w.ph $ac1, %[p1], %[filter45] \n\t" /* even 1 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 1 */ + + /* even 2. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 3 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "ulw %[qload1], 12(%[src]) \n\t" + "dpa.w.ph $ac2, %[p2], %[filter45] \n\t" /* even 1 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 1 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 1 */ + + /* even 3. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 4 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "sb %[st1], 0(%[dst]) \n\t" /* even 1 */ + "dpa.w.ph $ac3, %[p3], %[filter45] \n\t" /* even 3 */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 3 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 1 */ + + /* even 4. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 5 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[st2], 2(%[dst]) \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p4], %[filter45] \n\t" /* even 4 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 4 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 3 */ + + /* even 5. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 6 */ + "mthi $zero, $ac3 \n\t" + "sb %[st3], 4(%[dst]) \n\t" /* even 3 */ + "dpa.w.ph $ac2, %[p1], %[filter45] \n\t" /* even 5 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 5 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 4 */ + + /* even 6. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 7 */ + "mthi $zero, $ac1 \n\t" + "sb %[st1], 6(%[dst]) \n\t" /* even 4 */ + "dpa.w.ph $ac3, %[p5], %[filter45] \n\t" /* even 6 */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 6 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 5 */ + + /* even 7. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 8 */ + "mthi $zero, $ac2 \n\t" + "sb %[st2], 8(%[dst]) \n\t" /* even 5 */ + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" /* even 7 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 7 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 6 */ + + /* even 8. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 1 */ + "mthi $zero, $ac3 \n\t" + "dpa.w.ph $ac2, %[p3], %[filter45] \n\t" /* even 8 */ + "sb %[st3], 10(%[dst]) \n\t" /* even 6 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 8 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 7 */ + + /* ODD pixels */ + "ulw %[qload1], 1(%[src]) \n\t" + "ulw %[qload2], 5(%[src]) \n\t" + + /* odd 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 2 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "sb %[st1], 12(%[dst]) \n\t" /* even 7 */ + "ulw %[qload3], 9(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" /* odd 1 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 1 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 8 */ + + /* odd 2. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 3 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "sb %[st2], 14(%[dst]) \n\t" /* even 8 */ + "ulw %[qload1], 13(%[src]) \n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" /* odd 2 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 2 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 1 */ + + /* odd 3. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 4 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "sb %[st3], 1(%[dst]) \n\t" /* odd 1 */ + "dpa.w.ph $ac2, %[p3], %[filter45] \n\t" /* odd 3 */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 3 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 2 */ + + /* odd 4. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 5 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[st1], 3(%[dst]) \n\t" /* odd 2 */ + "dpa.w.ph $ac3, %[p4], %[filter45] \n\t" /* odd 4 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 4 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 3 */ + + /* odd 5. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 6 */ + "mthi $zero, $ac2 \n\t" + "sb %[st2], 5(%[dst]) \n\t" /* odd 3 */ + "dpa.w.ph $ac1, %[p1], %[filter45] \n\t" /* odd 5 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 5 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 4 */ + + /* odd 6. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 7 */ + "mthi $zero, $ac3 \n\t" + "sb %[st3], 7(%[dst]) \n\t" /* odd 4 */ + "dpa.w.ph $ac2, %[p5], %[filter45] \n\t" /* odd 6 */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 6 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 5 */ + + /* odd 7. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 8 */ + "mthi $zero, $ac1 \n\t" + "sb %[st1], 9(%[dst]) \n\t" /* odd 5 */ + "dpa.w.ph $ac3, %[p2], %[filter45] \n\t" /* odd 7 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 7 */ + + /* odd 8. pixel */ + "dpa.w.ph $ac1, %[p3], %[filter45] \n\t" /* odd 8 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 8 */ + + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 6 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 7 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 8 */ + + "sb %[st2], 11(%[dst]) \n\t" /* odd 6 */ + "sb %[st3], 13(%[dst]) \n\t" /* odd 7 */ + "sb %[st1], 15(%[dst]) \n\t" /* odd 8 */ + + : [qload1] "=&r"(qload1), [qload2] "=&r"(qload2), + [qload3] "=&r"(qload3), [st1] "=&r"(st1), [st2] "=&r"(st2), + [st3] "=&r"(st3), [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), + [p4] "=&r"(p4), [p5] "=&r"(p5), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3) + : [filter45] "r"(filter45), [vector_64] "r"(vector_64), [cm] "r"(cm), + [dst] "r"(dst), [src] "r"(src)); + + src += 16; + dst += 16; + } + + /* Next row... */ + src_ptr += src_stride; + dst_ptr += dst_stride; + } +} + +static void convolve_bi_horiz_64_dspr2(const uint8_t *src_ptr, + int32_t src_stride, uint8_t *dst_ptr, + int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y, c; + const uint8_t *src; + uint8_t *dst; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector_64 = 64; + int32_t Temp1, Temp2, Temp3; + uint32_t qload1, qload2, qload3; + uint32_t p1, p2, p3, p4, p5; + uint32_t st1, st2, st3; + const int16_t *filter = &filter_x0[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + src = src_ptr; + dst = dst_ptr; + + /* prefetch data to cache memory */ + prefetch_load(src_ptr + src_stride); + prefetch_load(src_ptr + src_stride + 32); + prefetch_load(src_ptr + src_stride + 64); + prefetch_store(dst_ptr + dst_stride); + prefetch_store(dst_ptr + dst_stride + 32); + + for (c = 0; c < 4; c++) { + __asm__ __volatile__( + "ulw %[qload1], 0(%[src]) \n\t" + "ulw %[qload2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 1 */ + "mthi $zero, $ac1 \n\t" + "mtlo %[vector_64], $ac2 \n\t" /* even 2 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "ulw %[qload3], 8(%[src]) \n\t" + "dpa.w.ph $ac1, %[p1], %[filter45] \n\t" /* even 1 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 1 */ + + /* even 2. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 3 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "ulw %[qload1], 12(%[src]) \n\t" + "dpa.w.ph $ac2, %[p2], %[filter45] \n\t" /* even 1 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 1 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 1 */ + + /* even 3. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 4 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "sb %[st1], 0(%[dst]) \n\t" /* even 1 */ + "dpa.w.ph $ac3, %[p3], %[filter45] \n\t" /* even 3 */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 3 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 1 */ + + /* even 4. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 5 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[st2], 2(%[dst]) \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p4], %[filter45] \n\t" /* even 4 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 4 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 3 */ + + /* even 5. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 6 */ + "mthi $zero, $ac3 \n\t" + "sb %[st3], 4(%[dst]) \n\t" /* even 3 */ + "dpa.w.ph $ac2, %[p1], %[filter45] \n\t" /* even 5 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 5 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 4 */ + + /* even 6. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 7 */ + "mthi $zero, $ac1 \n\t" + "sb %[st1], 6(%[dst]) \n\t" /* even 4 */ + "dpa.w.ph $ac3, %[p5], %[filter45] \n\t" /* even 6 */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 6 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 5 */ + + /* even 7. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 8 */ + "mthi $zero, $ac2 \n\t" + "sb %[st2], 8(%[dst]) \n\t" /* even 5 */ + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" /* even 7 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 7 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 6 */ + + /* even 8. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 1 */ + "mthi $zero, $ac3 \n\t" + "dpa.w.ph $ac2, %[p3], %[filter45] \n\t" /* even 8 */ + "sb %[st3], 10(%[dst]) \n\t" /* even 6 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 8 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 7 */ + + /* ODD pixels */ + "ulw %[qload1], 1(%[src]) \n\t" + "ulw %[qload2], 5(%[src]) \n\t" + + /* odd 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 2 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "sb %[st1], 12(%[dst]) \n\t" /* even 7 */ + "ulw %[qload3], 9(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[filter45] \n\t" /* odd 1 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 1 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 8 */ + + /* odd 2. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 3 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "sb %[st2], 14(%[dst]) \n\t" /* even 8 */ + "ulw %[qload1], 13(%[src]) \n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" /* odd 2 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 2 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 1 */ + + /* odd 3. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 4 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "sb %[st3], 1(%[dst]) \n\t" /* odd 1 */ + "dpa.w.ph $ac2, %[p3], %[filter45] \n\t" /* odd 3 */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 3 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 2 */ + + /* odd 4. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 5 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[st1], 3(%[dst]) \n\t" /* odd 2 */ + "dpa.w.ph $ac3, %[p4], %[filter45] \n\t" /* odd 4 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 4 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 3 */ + + /* odd 5. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 6 */ + "mthi $zero, $ac2 \n\t" + "sb %[st2], 5(%[dst]) \n\t" /* odd 3 */ + "dpa.w.ph $ac1, %[p1], %[filter45] \n\t" /* odd 5 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 5 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 4 */ + + /* odd 6. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 7 */ + "mthi $zero, $ac3 \n\t" + "sb %[st3], 7(%[dst]) \n\t" /* odd 4 */ + "dpa.w.ph $ac2, %[p5], %[filter45] \n\t" /* odd 6 */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 6 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 5 */ + + /* odd 7. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 8 */ + "mthi $zero, $ac1 \n\t" + "sb %[st1], 9(%[dst]) \n\t" /* odd 5 */ + "dpa.w.ph $ac3, %[p2], %[filter45] \n\t" /* odd 7 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 7 */ + + /* odd 8. pixel */ + "dpa.w.ph $ac1, %[p3], %[filter45] \n\t" /* odd 8 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 8 */ + + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 6 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 7 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 8 */ + + "sb %[st2], 11(%[dst]) \n\t" /* odd 6 */ + "sb %[st3], 13(%[dst]) \n\t" /* odd 7 */ + "sb %[st1], 15(%[dst]) \n\t" /* odd 8 */ + + : [qload1] "=&r"(qload1), [qload2] "=&r"(qload2), + [qload3] "=&r"(qload3), [st1] "=&r"(st1), [st2] "=&r"(st2), + [st3] "=&r"(st3), [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), + [p4] "=&r"(p4), [p5] "=&r"(p5), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3) + : [filter45] "r"(filter45), [vector_64] "r"(vector_64), [cm] "r"(cm), + [dst] "r"(dst), [src] "r"(src)); + + src += 16; + dst += 16; + } + + /* Next row... */ + src_ptr += src_stride; + dst_ptr += dst_stride; + } +} + +void aom_convolve2_horiz_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + uint32_t pos = 38; + + assert(x_step_q4 == 16); + + prefetch_load((const uint8_t *)filter_x); + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + /* prefetch data to cache memory */ + prefetch_load(src); + prefetch_load(src + 32); + prefetch_store(dst); + + switch (w) { + case 4: + convolve_bi_horiz_4_dspr2(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filter_x, (int32_t)h); + break; + case 8: + convolve_bi_horiz_8_dspr2(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filter_x, (int32_t)h); + break; + case 16: + convolve_bi_horiz_16_dspr2(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filter_x, (int32_t)h, 1); + break; + case 32: + convolve_bi_horiz_16_dspr2(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filter_x, (int32_t)h, 2); + break; + case 64: + prefetch_load(src + 64); + prefetch_store(dst + 32); + + convolve_bi_horiz_64_dspr2(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filter_x, (int32_t)h); + break; + default: + aom_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } +} +#endif diff --git a/third_party/aom/aom_dsp/mips/convolve2_vert_dspr2.c b/third_party/aom/aom_dsp/mips/convolve2_vert_dspr2.c new file mode 100644 index 0000000000..3367be01a2 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/convolve2_vert_dspr2.c @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/convolve_common_dspr2.h" +#include "aom_dsp/aom_convolve.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +static void convolve_bi_vert_4_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_y, int32_t w, + int32_t h) { + int32_t x, y; + const uint8_t *src_ptr; + uint8_t *dst_ptr; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector4a = 64; + uint32_t load1, load2; + uint32_t p1, p2; + uint32_t scratch1; + uint32_t store1, store2; + int32_t Temp1, Temp2; + const int16_t *filter = &filter_y[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_store(dst + dst_stride); + + for (x = 0; x < w; x += 4) { + src_ptr = src + x; + dst_ptr = dst + x; + + __asm__ __volatile__( + "ulw %[load1], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load2], 0(%[src_ptr]) \n\t" + + "mtlo %[vector4a], $ac0 \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac0 \n\t" + "mthi $zero, $ac1 \n\t" + "mthi $zero, $ac2 \n\t" + "mthi $zero, $ac3 \n\t" + + "preceu.ph.qbr %[scratch1], %[load1] \n\t" + "preceu.ph.qbr %[p1], %[load2] \n\t" + + "precrq.ph.w %[p2], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac0, %[p1], %[filter45] \n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" + + "preceu.ph.qbl %[scratch1], %[load1] \n\t" + "preceu.ph.qbl %[p1], %[load2] \n\t" + + "precrq.ph.w %[p2], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac2, %[p1], %[filter45] \n\t" + "dpa.w.ph $ac3, %[p2], %[filter45] \n\t" + + "extp %[Temp1], $ac0, 31 \n\t" + "extp %[Temp2], $ac1, 31 \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "extp %[Temp1], $ac2, 31 \n\t" + + "lbux %[store2], %[Temp2](%[cm]) \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + "sb %[store1], 0(%[dst_ptr]) \n\t" + "sb %[store2], 1(%[dst_ptr]) \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "lbux %[store2], %[Temp2](%[cm]) \n\t" + + "sb %[store1], 2(%[dst_ptr]) \n\t" + "sb %[store2], 3(%[dst_ptr]) \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [p1] "=&r"(p1), + [p2] "=&r"(p2), [scratch1] "=&r"(scratch1), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [store1] "=&r"(store1), + [store2] "=&r"(store2), [src_ptr] "+r"(src_ptr) + : [filter45] "r"(filter45), [vector4a] "r"(vector4a), + [src_stride] "r"(src_stride), [cm] "r"(cm), [dst_ptr] "r"(dst_ptr)); + } + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_bi_vert_64_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_y, int32_t h) { + int32_t x, y; + const uint8_t *src_ptr; + uint8_t *dst_ptr; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector4a = 64; + uint32_t load1, load2; + uint32_t p1, p2; + uint32_t scratch1; + uint32_t store1, store2; + int32_t Temp1, Temp2; + const int16_t *filter = &filter_y[3]; + uint32_t filter45; + + filter45 = ((const int32_t *)filter)[0]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_store(dst + dst_stride); + + for (x = 0; x < 64; x += 4) { + src_ptr = src + x; + dst_ptr = dst + x; + + __asm__ __volatile__( + "ulw %[load1], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load2], 0(%[src_ptr]) \n\t" + + "mtlo %[vector4a], $ac0 \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac0 \n\t" + "mthi $zero, $ac1 \n\t" + "mthi $zero, $ac2 \n\t" + "mthi $zero, $ac3 \n\t" + + "preceu.ph.qbr %[scratch1], %[load1] \n\t" + "preceu.ph.qbr %[p1], %[load2] \n\t" + + "precrq.ph.w %[p2], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac0, %[p1], %[filter45] \n\t" + "dpa.w.ph $ac1, %[p2], %[filter45] \n\t" + + "preceu.ph.qbl %[scratch1], %[load1] \n\t" + "preceu.ph.qbl %[p1], %[load2] \n\t" + + "precrq.ph.w %[p2], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac2, %[p1], %[filter45] \n\t" + "dpa.w.ph $ac3, %[p2], %[filter45] \n\t" + + "extp %[Temp1], $ac0, 31 \n\t" + "extp %[Temp2], $ac1, 31 \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "extp %[Temp1], $ac2, 31 \n\t" + + "lbux %[store2], %[Temp2](%[cm]) \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + "sb %[store1], 0(%[dst_ptr]) \n\t" + "sb %[store2], 1(%[dst_ptr]) \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "lbux %[store2], %[Temp2](%[cm]) \n\t" + + "sb %[store1], 2(%[dst_ptr]) \n\t" + "sb %[store2], 3(%[dst_ptr]) \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [p1] "=&r"(p1), + [p2] "=&r"(p2), [scratch1] "=&r"(scratch1), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [store1] "=&r"(store1), + [store2] "=&r"(store2), [src_ptr] "+r"(src_ptr) + : [filter45] "r"(filter45), [vector4a] "r"(vector4a), + [src_stride] "r"(src_stride), [cm] "r"(cm), [dst_ptr] "r"(dst_ptr)); + } + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +void aom_convolve2_vert_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + uint32_t pos = 38; + + assert(y_step_q4 == 16); + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + prefetch_store(dst); + + switch (w) { + case 4: + case 8: + case 16: + case 32: + convolve_bi_vert_4_dspr2(src, src_stride, dst, dst_stride, filter_y, w, + h); + break; + case 64: + prefetch_store(dst + 32); + convolve_bi_vert_64_dspr2(src, src_stride, dst, dst_stride, filter_y, h); + break; + default: + aom_convolve8_vert_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } +} +#endif diff --git a/third_party/aom/aom_dsp/mips/convolve8_avg_dspr2.c b/third_party/aom/aom_dsp/mips/convolve8_avg_dspr2.c new file mode 100644 index 0000000000..298065adb0 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/convolve8_avg_dspr2.c @@ -0,0 +1,641 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/convolve_common_dspr2.h" +#include "aom_dsp/aom_convolve.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +static void convolve_avg_vert_4_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_y, int32_t w, + int32_t h) { + int32_t x, y; + const uint8_t *src_ptr; + uint8_t *dst_ptr; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector4a = 64; + uint32_t load1, load2, load3, load4; + uint32_t p1, p2; + uint32_t n1, n2; + uint32_t scratch1, scratch2; + uint32_t store1, store2; + int32_t vector1b, vector2b, vector3b, vector4b; + int32_t Temp1, Temp2; + + vector1b = ((const int32_t *)filter_y)[0]; + vector2b = ((const int32_t *)filter_y)[1]; + vector3b = ((const int32_t *)filter_y)[2]; + vector4b = ((const int32_t *)filter_y)[3]; + + src -= 3 * src_stride; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_store(dst + dst_stride); + + for (x = 0; x < w; x += 4) { + src_ptr = src + x; + dst_ptr = dst + x; + + __asm__ __volatile__( + "ulw %[load1], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load2], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load3], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load4], 0(%[src_ptr]) \n\t" + + "mtlo %[vector4a], $ac0 \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac0 \n\t" + "mthi $zero, $ac1 \n\t" + "mthi $zero, $ac2 \n\t" + "mthi $zero, $ac3 \n\t" + + "preceu.ph.qbr %[scratch1], %[load1] \n\t" + "preceu.ph.qbr %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbr %[scratch2], %[load3] \n\t" + "preceu.ph.qbr %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac0, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac0, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac1, %[n1], %[vector1b] \n\t" + "dpa.w.ph $ac1, %[n2], %[vector2b] \n\t" + + "preceu.ph.qbl %[scratch1], %[load1] \n\t" + "preceu.ph.qbl %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbl %[scratch2], %[load3] \n\t" + "preceu.ph.qbl %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac2, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[n1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[n2], %[vector2b] \n\t" + + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load1], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load2], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load3], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load4], 0(%[src_ptr]) \n\t" + + "preceu.ph.qbr %[scratch1], %[load1] \n\t" + "preceu.ph.qbr %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbr %[scratch2], %[load3] \n\t" + "preceu.ph.qbr %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac0, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac0, %[p2], %[vector4b] \n\t" + "extp %[Temp1], $ac0, 31 \n\t" + "dpa.w.ph $ac1, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac1, %[n2], %[vector4b] \n\t" + "extp %[Temp2], $ac1, 31 \n\t" + + "preceu.ph.qbl %[scratch1], %[load1] \n\t" + "preceu.ph.qbl %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "lbu %[scratch1], 0(%[dst_ptr]) \n\t" + "preceu.ph.qbl %[scratch2], %[load3] \n\t" + "preceu.ph.qbl %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + "lbu %[scratch2], 1(%[dst_ptr]) \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "dpa.w.ph $ac2, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector4b] \n\t" + "addqh_r.w %[store1], %[store1], %[scratch1] \n\t" /* pixel 1 */ + "extp %[Temp1], $ac2, 31 \n\t" + + "lbux %[store2], %[Temp2](%[cm]) \n\t" + "dpa.w.ph $ac3, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[n2], %[vector4b] \n\t" + "addqh_r.w %[store2], %[store2], %[scratch2] \n\t" /* pixel 2 */ + "extp %[Temp2], $ac3, 31 \n\t" + "lbu %[scratch1], 2(%[dst_ptr]) \n\t" + + "sb %[store1], 0(%[dst_ptr]) \n\t" + "sb %[store2], 1(%[dst_ptr]) \n\t" + "lbu %[scratch2], 3(%[dst_ptr]) \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "lbux %[store2], %[Temp2](%[cm]) \n\t" + "addqh_r.w %[store1], %[store1], %[scratch1] \n\t" /* pixel 3 */ + "addqh_r.w %[store2], %[store2], %[scratch2] \n\t" /* pixel 4 */ + + "sb %[store1], 2(%[dst_ptr]) \n\t" + "sb %[store2], 3(%[dst_ptr]) \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [p1] "=&r"(p1), [p2] "=&r"(p2), + [n1] "=&r"(n1), [n2] "=&r"(n2), [scratch1] "=&r"(scratch1), + [scratch2] "=&r"(scratch2), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [store1] "=&r"(store1), + [store2] "=&r"(store2), [src_ptr] "+r"(src_ptr) + : [vector1b] "r"(vector1b), [vector2b] "r"(vector2b), + [vector3b] "r"(vector3b), [vector4b] "r"(vector4b), + [vector4a] "r"(vector4a), [src_stride] "r"(src_stride), + [cm] "r"(cm), [dst_ptr] "r"(dst_ptr)); + } + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_avg_vert_64_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_y, int32_t h) { + int32_t x, y; + const uint8_t *src_ptr; + uint8_t *dst_ptr; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector4a = 64; + uint32_t load1, load2, load3, load4; + uint32_t p1, p2; + uint32_t n1, n2; + uint32_t scratch1, scratch2; + uint32_t store1, store2; + int32_t vector1b, vector2b, vector3b, vector4b; + int32_t Temp1, Temp2; + + vector1b = ((const int32_t *)filter_y)[0]; + vector2b = ((const int32_t *)filter_y)[1]; + vector3b = ((const int32_t *)filter_y)[2]; + vector4b = ((const int32_t *)filter_y)[3]; + + src -= 3 * src_stride; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_store(dst + dst_stride); + prefetch_store(dst + dst_stride + 32); + + for (x = 0; x < 64; x += 4) { + src_ptr = src + x; + dst_ptr = dst + x; + + __asm__ __volatile__( + "ulw %[load1], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load2], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load3], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load4], 0(%[src_ptr]) \n\t" + + "mtlo %[vector4a], $ac0 \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac0 \n\t" + "mthi $zero, $ac1 \n\t" + "mthi $zero, $ac2 \n\t" + "mthi $zero, $ac3 \n\t" + + "preceu.ph.qbr %[scratch1], %[load1] \n\t" + "preceu.ph.qbr %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbr %[scratch2], %[load3] \n\t" + "preceu.ph.qbr %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac0, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac0, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac1, %[n1], %[vector1b] \n\t" + "dpa.w.ph $ac1, %[n2], %[vector2b] \n\t" + + "preceu.ph.qbl %[scratch1], %[load1] \n\t" + "preceu.ph.qbl %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbl %[scratch2], %[load3] \n\t" + "preceu.ph.qbl %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac2, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[n1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[n2], %[vector2b] \n\t" + + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load1], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load2], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load3], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load4], 0(%[src_ptr]) \n\t" + + "preceu.ph.qbr %[scratch1], %[load1] \n\t" + "preceu.ph.qbr %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbr %[scratch2], %[load3] \n\t" + "preceu.ph.qbr %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac0, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac0, %[p2], %[vector4b] \n\t" + "extp %[Temp1], $ac0, 31 \n\t" + "dpa.w.ph $ac1, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac1, %[n2], %[vector4b] \n\t" + "extp %[Temp2], $ac1, 31 \n\t" + + "preceu.ph.qbl %[scratch1], %[load1] \n\t" + "preceu.ph.qbl %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "lbu %[scratch1], 0(%[dst_ptr]) \n\t" + "preceu.ph.qbl %[scratch2], %[load3] \n\t" + "preceu.ph.qbl %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + "lbu %[scratch2], 1(%[dst_ptr]) \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "dpa.w.ph $ac2, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector4b] \n\t" + "addqh_r.w %[store1], %[store1], %[scratch1] \n\t" /* pixel 1 */ + "extp %[Temp1], $ac2, 31 \n\t" + + "lbux %[store2], %[Temp2](%[cm]) \n\t" + "dpa.w.ph $ac3, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[n2], %[vector4b] \n\t" + "addqh_r.w %[store2], %[store2], %[scratch2] \n\t" /* pixel 2 */ + "extp %[Temp2], $ac3, 31 \n\t" + "lbu %[scratch1], 2(%[dst_ptr]) \n\t" + + "sb %[store1], 0(%[dst_ptr]) \n\t" + "sb %[store2], 1(%[dst_ptr]) \n\t" + "lbu %[scratch2], 3(%[dst_ptr]) \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "lbux %[store2], %[Temp2](%[cm]) \n\t" + "addqh_r.w %[store1], %[store1], %[scratch1] \n\t" /* pixel 3 */ + "addqh_r.w %[store2], %[store2], %[scratch2] \n\t" /* pixel 4 */ + + "sb %[store1], 2(%[dst_ptr]) \n\t" + "sb %[store2], 3(%[dst_ptr]) \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [p1] "=&r"(p1), [p2] "=&r"(p2), + [n1] "=&r"(n1), [n2] "=&r"(n2), [scratch1] "=&r"(scratch1), + [scratch2] "=&r"(scratch2), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [store1] "=&r"(store1), + [store2] "=&r"(store2), [src_ptr] "+r"(src_ptr) + : [vector1b] "r"(vector1b), [vector2b] "r"(vector2b), + [vector3b] "r"(vector3b), [vector4b] "r"(vector4b), + [vector4a] "r"(vector4a), [src_stride] "r"(src_stride), + [cm] "r"(cm), [dst_ptr] "r"(dst_ptr)); + } + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +void aom_convolve8_avg_vert_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + if (((const int32_t *)filter_y)[0] == 0) { + aom_convolve2_avg_vert_dspr2(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + } else { + uint32_t pos = 38; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + prefetch_store(dst); + + switch (w) { + case 4: + case 8: + case 16: + case 32: + convolve_avg_vert_4_dspr2(src, src_stride, dst, dst_stride, filter_y, w, + h); + break; + case 64: + prefetch_store(dst + 32); + convolve_avg_vert_64_dspr2(src, src_stride, dst, dst_stride, filter_y, + h); + break; + default: + aom_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } +} + +void aom_convolve8_avg_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + /* Fixed size intermediate buffer places limits on parameters. */ + DECLARE_ALIGNED(32, uint8_t, temp[64 * 135]); + int32_t intermediate_height = ((h * y_step_q4) >> 4) + 7; + + assert(w <= 64); + assert(h <= 64); + assert(x_step_q4 == 16); + assert(y_step_q4 == 16); + + if (intermediate_height < h) intermediate_height = h; + + aom_convolve8_horiz(src - (src_stride * 3), src_stride, temp, 64, filter_x, + x_step_q4, filter_y, y_step_q4, w, intermediate_height); + + aom_convolve8_avg_vert(temp + 64 * 3, 64, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); +} + +void aom_convolve_avg_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int filter_x_stride, + const int16_t *filter_y, int filter_y_stride, int w, + int h) { + int x, y; + uint32_t tp1, tp2, tn1; + uint32_t tp3, tp4, tn2; + + /* prefetch data to cache memory */ + prefetch_load(src); + prefetch_load(src + 32); + prefetch_store(dst); + + switch (w) { + case 4: + /* 1 word storage */ + for (y = h; y--;) { + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 0(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "sw %[tn1], 0(%[dst]) \n\t" /* store */ + + : [tn1] "=&r"(tn1), [tp1] "=&r"(tp1), [tp2] "=&r"(tp2) + : [src] "r"(src), [dst] "r"(dst)); + + src += src_stride; + dst += dst_stride; + } + break; + case 8: + /* 2 word storage */ + for (y = h; y--;) { + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 0(%[dst]) \n\t" + "ulw %[tp3], 4(%[src]) \n\t" + "ulw %[tp4], 4(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "sw %[tn1], 0(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 4(%[dst]) \n\t" /* store */ + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tp3] "=&r"(tp3), + [tp4] "=&r"(tp4), [tn1] "=&r"(tn1), [tn2] "=&r"(tn2) + : [src] "r"(src), [dst] "r"(dst)); + + src += src_stride; + dst += dst_stride; + } + break; + case 16: + /* 4 word storage */ + for (y = h; y--;) { + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 0(%[dst]) \n\t" + "ulw %[tp3], 4(%[src]) \n\t" + "ulw %[tp4], 4(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "ulw %[tp1], 8(%[src]) \n\t" + "ulw %[tp2], 8(%[dst]) \n\t" + "sw %[tn1], 0(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 4(%[dst]) \n\t" /* store */ + "ulw %[tp3], 12(%[src]) \n\t" + "ulw %[tp4], 12(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "sw %[tn1], 8(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 12(%[dst]) \n\t" /* store */ + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tp3] "=&r"(tp3), + [tp4] "=&r"(tp4), [tn1] "=&r"(tn1), [tn2] "=&r"(tn2) + : [src] "r"(src), [dst] "r"(dst)); + + src += src_stride; + dst += dst_stride; + } + break; + case 32: + /* 8 word storage */ + for (y = h; y--;) { + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 0(%[dst]) \n\t" + "ulw %[tp3], 4(%[src]) \n\t" + "ulw %[tp4], 4(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "ulw %[tp1], 8(%[src]) \n\t" + "ulw %[tp2], 8(%[dst]) \n\t" + "sw %[tn1], 0(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 4(%[dst]) \n\t" /* store */ + "ulw %[tp3], 12(%[src]) \n\t" + "ulw %[tp4], 12(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "ulw %[tp1], 16(%[src]) \n\t" + "ulw %[tp2], 16(%[dst]) \n\t" + "sw %[tn1], 8(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 12(%[dst]) \n\t" /* store */ + "ulw %[tp3], 20(%[src]) \n\t" + "ulw %[tp4], 20(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "ulw %[tp1], 24(%[src]) \n\t" + "ulw %[tp2], 24(%[dst]) \n\t" + "sw %[tn1], 16(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 20(%[dst]) \n\t" /* store */ + "ulw %[tp3], 28(%[src]) \n\t" + "ulw %[tp4], 28(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "sw %[tn1], 24(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 28(%[dst]) \n\t" /* store */ + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tp3] "=&r"(tp3), + [tp4] "=&r"(tp4), [tn1] "=&r"(tn1), [tn2] "=&r"(tn2) + : [src] "r"(src), [dst] "r"(dst)); + + src += src_stride; + dst += dst_stride; + } + break; + case 64: + prefetch_load(src + 64); + prefetch_store(dst + 32); + + /* 16 word storage */ + for (y = h; y--;) { + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_load(src + src_stride + 64); + prefetch_store(dst + dst_stride); + prefetch_store(dst + dst_stride + 32); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 0(%[dst]) \n\t" + "ulw %[tp3], 4(%[src]) \n\t" + "ulw %[tp4], 4(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "ulw %[tp1], 8(%[src]) \n\t" + "ulw %[tp2], 8(%[dst]) \n\t" + "sw %[tn1], 0(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 4(%[dst]) \n\t" /* store */ + "ulw %[tp3], 12(%[src]) \n\t" + "ulw %[tp4], 12(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "ulw %[tp1], 16(%[src]) \n\t" + "ulw %[tp2], 16(%[dst]) \n\t" + "sw %[tn1], 8(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 12(%[dst]) \n\t" /* store */ + "ulw %[tp3], 20(%[src]) \n\t" + "ulw %[tp4], 20(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "ulw %[tp1], 24(%[src]) \n\t" + "ulw %[tp2], 24(%[dst]) \n\t" + "sw %[tn1], 16(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 20(%[dst]) \n\t" /* store */ + "ulw %[tp3], 28(%[src]) \n\t" + "ulw %[tp4], 28(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "ulw %[tp1], 32(%[src]) \n\t" + "ulw %[tp2], 32(%[dst]) \n\t" + "sw %[tn1], 24(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 28(%[dst]) \n\t" /* store */ + "ulw %[tp3], 36(%[src]) \n\t" + "ulw %[tp4], 36(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "ulw %[tp1], 40(%[src]) \n\t" + "ulw %[tp2], 40(%[dst]) \n\t" + "sw %[tn1], 32(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 36(%[dst]) \n\t" /* store */ + "ulw %[tp3], 44(%[src]) \n\t" + "ulw %[tp4], 44(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "ulw %[tp1], 48(%[src]) \n\t" + "ulw %[tp2], 48(%[dst]) \n\t" + "sw %[tn1], 40(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 44(%[dst]) \n\t" /* store */ + "ulw %[tp3], 52(%[src]) \n\t" + "ulw %[tp4], 52(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "ulw %[tp1], 56(%[src]) \n\t" + "ulw %[tp2], 56(%[dst]) \n\t" + "sw %[tn1], 48(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 52(%[dst]) \n\t" /* store */ + "ulw %[tp3], 60(%[src]) \n\t" + "ulw %[tp4], 60(%[dst]) \n\t" + "adduh_r.qb %[tn1], %[tp2], %[tp1] \n\t" /* average */ + "sw %[tn1], 56(%[dst]) \n\t" /* store */ + "adduh_r.qb %[tn2], %[tp3], %[tp4] \n\t" /* average */ + "sw %[tn2], 60(%[dst]) \n\t" /* store */ + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tp3] "=&r"(tp3), + [tp4] "=&r"(tp4), [tn1] "=&r"(tn1), [tn2] "=&r"(tn2) + : [src] "r"(src), [dst] "r"(dst)); + + src += src_stride; + dst += dst_stride; + } + break; + default: + for (y = h; y > 0; --y) { + for (x = 0; x < w; ++x) { + dst[x] = (dst[x] + src[x] + 1) >> 1; + } + + src += src_stride; + dst += dst_stride; + } + break; + } +} +#endif diff --git a/third_party/aom/aom_dsp/mips/convolve8_avg_horiz_dspr2.c b/third_party/aom/aom_dsp/mips/convolve8_avg_horiz_dspr2.c new file mode 100644 index 0000000000..f6534b4205 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/convolve8_avg_horiz_dspr2.c @@ -0,0 +1,998 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/convolve_common_dspr2.h" +#include "aom_dsp/aom_convolve.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +static void convolve_avg_horiz_4_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y; + uint8_t *cm = aom_ff_cropTbl; + int32_t vector1b, vector2b, vector3b, vector4b; + int32_t Temp1, Temp2, Temp3, Temp4; + uint32_t vector4a = 64; + uint32_t tp1, tp2; + uint32_t p1, p2, p3, p4; + uint32_t n1, n2, n3, n4; + uint32_t tn1, tn2; + + vector1b = ((const int32_t *)filter_x0)[0]; + vector2b = ((const int32_t *)filter_x0)[1]; + vector3b = ((const int32_t *)filter_x0)[2]; + vector4b = ((const int32_t *)filter_x0)[3]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[tp1] \n\t" + "preceu.ph.qbl %[p2], %[tp1] \n\t" + "preceu.ph.qbr %[p3], %[tp2] \n\t" + "preceu.ph.qbl %[p4], %[tp2] \n\t" + "dpa.w.ph $ac3, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector3b] \n\t" + "ulw %[tn2], 8(%[src]) \n\t" + "dpa.w.ph $ac3, %[p4], %[vector4b] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + + /* even 2. pixel */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tn2] \n\t" + "balign %[tn1], %[tn2], 3 \n\t" + "balign %[tn2], %[tp2], 3 \n\t" + "balign %[tp2], %[tp1], 3 \n\t" + "dpa.w.ph $ac2, %[p2], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p3], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[p4], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector4b] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + "lbu %[p2], 3(%[dst]) \n\t" /* load odd 2 */ + + /* odd 1. pixel */ + "lbux %[tp1], %[Temp1](%[cm]) \n\t" /* even 1 */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "lbu %[Temp1], 1(%[dst]) \n\t" /* load odd 1 */ + "preceu.ph.qbr %[n1], %[tp2] \n\t" + "preceu.ph.qbl %[n2], %[tp2] \n\t" + "preceu.ph.qbr %[n3], %[tn2] \n\t" + "preceu.ph.qbl %[n4], %[tn2] \n\t" + "dpa.w.ph $ac3, %[n1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[n2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[n3], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[n4], %[vector4b] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + "lbu %[tn2], 0(%[dst]) \n\t" /* load even 1 */ + + /* odd 2. pixel */ + "lbux %[tp2], %[Temp3](%[cm]) \n\t" /* even 2 */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[n1], %[tn1] \n\t" + "lbux %[tn1], %[Temp2](%[cm]) \n\t" /* odd 1 */ + "addqh_r.w %[tn2], %[tn2], %[tp1] \n\t" /* average even 1 */ + "dpa.w.ph $ac2, %[n2], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[n3], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[n4], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[n1], %[vector4b] \n\t" + "extp %[Temp4], $ac2, 31 \n\t" + + "lbu %[tp1], 2(%[dst]) \n\t" /* load even 2 */ + "sb %[tn2], 0(%[dst]) \n\t" /* store even 1 */ + + /* clamp */ + "addqh_r.w %[Temp1], %[Temp1], %[tn1] \n\t" /* average odd 1 */ + "lbux %[n2], %[Temp4](%[cm]) \n\t" /* odd 2 */ + "sb %[Temp1], 1(%[dst]) \n\t" /* store odd 1 */ + + "addqh_r.w %[tp1], %[tp1], %[tp2] \n\t" /* average even 2 */ + "sb %[tp1], 2(%[dst]) \n\t" /* store even 2 */ + + "addqh_r.w %[p2], %[p2], %[n2] \n\t" /* average odd 2 */ + "sb %[p2], 3(%[dst]) \n\t" /* store odd 2 */ + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tn1] "=&r"(tn1), + [tn2] "=&r"(tn2), [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), + [p4] "=&r"(p4), [n1] "=&r"(n1), [n2] "=&r"(n2), [n3] "=&r"(n3), + [n4] "=&r"(n4), [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), + [Temp3] "=&r"(Temp3), [Temp4] "=&r"(Temp4) + : [vector1b] "r"(vector1b), [vector2b] "r"(vector2b), + [vector3b] "r"(vector3b), [vector4b] "r"(vector4b), + [vector4a] "r"(vector4a), [cm] "r"(cm), [dst] "r"(dst), + [src] "r"(src)); + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_avg_horiz_8_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector4a = 64; + int32_t vector1b, vector2b, vector3b, vector4b; + int32_t Temp1, Temp2, Temp3; + uint32_t tp1, tp2; + uint32_t p1, p2, p3, p4, n1; + uint32_t tn1, tn2, tn3; + uint32_t st0, st1; + + vector1b = ((const int32_t *)filter_x0)[0]; + vector2b = ((const int32_t *)filter_x0)[1]; + vector3b = ((const int32_t *)filter_x0)[2]; + vector4b = ((const int32_t *)filter_x0)[3]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tp1] \n\t" + "preceu.ph.qbl %[p2], %[tp1] \n\t" + "preceu.ph.qbr %[p3], %[tp2] \n\t" + "preceu.ph.qbl %[p4], %[tp2] \n\t" + "ulw %[tn2], 8(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[p4], %[vector4b] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + "lbu %[Temp2], 0(%[dst]) \n\t" + "lbu %[tn3], 2(%[dst]) \n\t" + + /* even 2. pixel */ + "preceu.ph.qbr %[p1], %[tn2] \n\t" + "preceu.ph.qbl %[n1], %[tn2] \n\t" + "ulw %[tn1], 12(%[src]) \n\t" + "dpa.w.ph $ac2, %[p2], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p3], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[p4], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector4b] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + /* even 3. pixel */ + "lbux %[st0], %[Temp1](%[cm]) \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p2], %[tn1] \n\t" + "lbux %[st1], %[Temp3](%[cm]) \n\t" + "dpa.w.ph $ac1, %[p3], %[vector1b] \n\t" + "dpa.w.ph $ac1, %[p4], %[vector2b] \n\t" + "dpa.w.ph $ac1, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac1, %[n1], %[vector4b] \n\t" + "extp %[Temp1], $ac1, 31 \n\t" + + "addqh_r.w %[Temp2], %[Temp2], %[st0] \n\t" + "addqh_r.w %[tn3], %[tn3], %[st1] \n\t" + "sb %[Temp2], 0(%[dst]) \n\t" + "sb %[tn3], 2(%[dst]) \n\t" + + /* even 4. pixel */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "balign %[tn3], %[tn1], 3 \n\t" + "balign %[tn1], %[tn2], 3 \n\t" + "balign %[tn2], %[tp2], 3 \n\t" + "balign %[tp2], %[tp1], 3 \n\t" + + "lbux %[st0], %[Temp1](%[cm]) \n\t" + "lbu %[Temp2], 4(%[dst]) \n\t" + "addqh_r.w %[Temp2], %[Temp2], %[st0] \n\t" + + "dpa.w.ph $ac2, %[p4], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector4b] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + /* odd 1. pixel */ + "mtlo %[vector4a], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "sb %[Temp2], 4(%[dst]) \n\t" + "preceu.ph.qbr %[p1], %[tp2] \n\t" + "preceu.ph.qbl %[p2], %[tp2] \n\t" + "preceu.ph.qbr %[p3], %[tn2] \n\t" + "preceu.ph.qbl %[p4], %[tn2] \n\t" + "dpa.w.ph $ac3, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[p4], %[vector4b] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + "lbu %[tp1], 6(%[dst]) \n\t" + + /* odd 2. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tn1] \n\t" + "preceu.ph.qbl %[n1], %[tn1] \n\t" + "lbux %[st0], %[Temp3](%[cm]) \n\t" + "dpa.w.ph $ac1, %[p2], %[vector1b] \n\t" + "dpa.w.ph $ac1, %[p3], %[vector2b] \n\t" + "dpa.w.ph $ac1, %[p4], %[vector3b] \n\t" + "dpa.w.ph $ac1, %[p1], %[vector4b] \n\t" + "extp %[Temp3], $ac1, 31 \n\t" + + "lbu %[tp2], 1(%[dst]) \n\t" + "lbu %[tn2], 3(%[dst]) \n\t" + "addqh_r.w %[tp1], %[tp1], %[st0] \n\t" + + /* odd 3. pixel */ + "lbux %[st1], %[Temp2](%[cm]) \n\t" + "preceu.ph.qbr %[p2], %[tn3] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p4], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[n1], %[vector4b] \n\t" + "addqh_r.w %[tp2], %[tp2], %[st1] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + "lbu %[tn3], 5(%[dst]) \n\t" + + /* odd 4. pixel */ + "sb %[tp2], 1(%[dst]) \n\t" + "sb %[tp1], 6(%[dst]) \n\t" + "dpa.w.ph $ac2, %[p4], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector4b] \n\t" + "extp %[Temp1], $ac2, 31 \n\t" + + "lbu %[tn1], 7(%[dst]) \n\t" + + /* clamp */ + "lbux %[p4], %[Temp3](%[cm]) \n\t" + "addqh_r.w %[tn2], %[tn2], %[p4] \n\t" + + "lbux %[p2], %[Temp2](%[cm]) \n\t" + "addqh_r.w %[tn3], %[tn3], %[p2] \n\t" + + "lbux %[n1], %[Temp1](%[cm]) \n\t" + "addqh_r.w %[tn1], %[tn1], %[n1] \n\t" + + /* store bytes */ + "sb %[tn2], 3(%[dst]) \n\t" + "sb %[tn3], 5(%[dst]) \n\t" + "sb %[tn1], 7(%[dst]) \n\t" + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tn1] "=&r"(tn1), + [tn2] "=&r"(tn2), [tn3] "=&r"(tn3), [st0] "=&r"(st0), + [st1] "=&r"(st1), [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), + [p4] "=&r"(p4), [n1] "=&r"(n1), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3) + : [vector1b] "r"(vector1b), [vector2b] "r"(vector2b), + [vector3b] "r"(vector3b), [vector4b] "r"(vector4b), + [vector4a] "r"(vector4a), [cm] "r"(cm), [dst] "r"(dst), + [src] "r"(src)); + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_avg_horiz_16_dspr2(const uint8_t *src_ptr, + int32_t src_stride, uint8_t *dst_ptr, + int32_t dst_stride, + const int16_t *filter_x0, int32_t h, + int32_t count) { + int32_t y, c; + const uint8_t *src; + uint8_t *dst; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector_64 = 64; + int32_t filter12, filter34, filter56, filter78; + int32_t Temp1, Temp2, Temp3; + uint32_t qload1, qload2, qload3; + uint32_t p1, p2, p3, p4, p5; + uint32_t st1, st2, st3; + + filter12 = ((const int32_t *)filter_x0)[0]; + filter34 = ((const int32_t *)filter_x0)[1]; + filter56 = ((const int32_t *)filter_x0)[2]; + filter78 = ((const int32_t *)filter_x0)[3]; + + for (y = h; y--;) { + src = src_ptr; + dst = dst_ptr; + + /* prefetch data to cache memory */ + prefetch_load(src_ptr + src_stride); + prefetch_load(src_ptr + src_stride + 32); + prefetch_store(dst_ptr + dst_stride); + + for (c = 0; c < count; c++) { + __asm__ __volatile__( + "ulw %[qload1], 0(%[src]) \n\t" + "ulw %[qload2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 1 */ + "mthi $zero, $ac1 \n\t" + "mtlo %[vector_64], $ac2 \n\t" /* even 2 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "ulw %[qload3], 8(%[src]) \n\t" + "dpa.w.ph $ac1, %[p1], %[filter12] \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p2], %[filter34] \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p3], %[filter56] \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p4], %[filter78] \n\t" /* even 1 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 1 */ + "lbu %[st2], 0(%[dst]) \n\t" /* load even 1 from dst */ + + /* even 2. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 3 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "ulw %[qload1], 12(%[src]) \n\t" + "dpa.w.ph $ac2, %[p2], %[filter12] \n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p3], %[filter34] \n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p4], %[filter56] \n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p1], %[filter78] \n\t" /* even 1 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 1 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 1 */ + + "lbu %[qload3], 2(%[dst]) \n\t" /* load even 2 from dst */ + + /* even 3. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 4 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[st2], %[st2], %[st1] \n\t" /* average even 1 */ + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "sb %[st2], 0(%[dst]) \n\t" /* store even 1 to dst */ + "dpa.w.ph $ac3, %[p3], %[filter12] \n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p4], %[filter34] \n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p1], %[filter56] \n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p5], %[filter78] \n\t" /* even 3 */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 3 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 1 */ + + /* even 4. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 5 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st2] \n\t" /* average even 2 */ + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[qload3], 2(%[dst]) \n\t" /* store even 2 to dst */ + "ulw %[qload2], 16(%[src]) \n\t" + "lbu %[qload3], 4(%[dst]) \n\t" /* load even 3 from dst */ + "lbu %[qload1], 6(%[dst]) \n\t" /* load even 4 from dst */ + "dpa.w.ph $ac1, %[p4], %[filter12] \n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p1], %[filter34] \n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p5], %[filter56] \n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p2], %[filter78] \n\t" /* even 4 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 4 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 3 */ + + /* even 5. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 6 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st3] \n\t" /* average even 3 */ + "preceu.ph.qbr %[p4], %[qload2] \n\t" + "sb %[qload3], 4(%[dst]) \n\t" /* store even 3 to dst */ + "dpa.w.ph $ac2, %[p1], %[filter12] \n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p5], %[filter34] \n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p2], %[filter56] \n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p3], %[filter78] \n\t" /* even 5 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 5 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 4 */ + + /* even 6. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 7 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[qload1], %[qload1], %[st1] \n\t" /* average even 4 */ + "preceu.ph.qbl %[p1], %[qload2] \n\t" + "sb %[qload1], 6(%[dst]) \n\t" /* store even 4 to dst */ + "ulw %[qload3], 20(%[src]) \n\t" + "dpa.w.ph $ac3, %[p5], %[filter12] \n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p2], %[filter34] \n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p3], %[filter56] \n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p4], %[filter78] \n\t" /* even 6 */ + "lbu %[qload2], 8(%[dst]) \n\t" /* load even 5 from dst */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 6 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 5 */ + + /* even 7. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 8 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload2], %[qload2], %[st2] \n\t" /* average even 5 */ + "preceu.ph.qbr %[p5], %[qload3] \n\t" + "sb %[qload2], 8(%[dst]) \n\t" /* store even 5 to dst */ + "dpa.w.ph $ac1, %[p2], %[filter12] \n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p3], %[filter34] \n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p4], %[filter56] \n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p1], %[filter78] \n\t" /* even 7 */ + "lbu %[qload3], 10(%[dst]) \n\t" /* load even 6 from dst */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 7 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 6 */ + + "lbu %[st2], 12(%[dst]) \n\t" /* load even 7 from dst */ + + /* even 8. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 1 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st3] \n\t" /* average even 6 */ + "dpa.w.ph $ac2, %[p3], %[filter12] \n\t" /* even 8 */ + "dpa.w.ph $ac2, %[p4], %[filter34] \n\t" /* even 8 */ + "sb %[qload3], 10(%[dst]) \n\t" /* store even 6 to dst */ + "dpa.w.ph $ac2, %[p1], %[filter56] \n\t" /* even 8 */ + "dpa.w.ph $ac2, %[p5], %[filter78] \n\t" /* even 8 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 8 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 7 */ + + /* ODD pixels */ + "ulw %[qload1], 1(%[src]) \n\t" + "ulw %[qload2], 5(%[src]) \n\t" + + "addqh_r.w %[st2], %[st2], %[st1] \n\t" /* average even 7 */ + + /* odd 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 2 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "sb %[st2], 12(%[dst]) \n\t" /* store even 7 to dst */ + "ulw %[qload3], 9(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[filter12] \n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p2], %[filter34] \n\t" /* odd 1 */ + "lbu %[qload2], 14(%[dst]) \n\t" /* load even 8 from dst */ + "dpa.w.ph $ac3, %[p3], %[filter56] \n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p4], %[filter78] \n\t" /* odd 1 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 1 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 8 */ + + "lbu %[st1], 1(%[dst]) \n\t" /* load odd 1 from dst */ + + /* odd 2. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 3 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload2], %[qload2], %[st2] \n\t" /* average even 8 */ + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "sb %[qload2], 14(%[dst]) \n\t" /* store even 8 to dst */ + "ulw %[qload1], 13(%[src]) \n\t" + "dpa.w.ph $ac1, %[p2], %[filter12] \n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p3], %[filter34] \n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p4], %[filter56] \n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p1], %[filter78] \n\t" /* odd 2 */ + "lbu %[qload3], 3(%[dst]) \n\t" /* load odd 2 from dst */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 2 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 1 */ + + /* odd 3. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 4 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[st3], %[st3], %[st1] \n\t" /* average odd 1 */ + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "dpa.w.ph $ac2, %[p3], %[filter12] \n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p4], %[filter34] \n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p1], %[filter56] \n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p5], %[filter78] \n\t" /* odd 3 */ + "sb %[st3], 1(%[dst]) \n\t" /* store odd 1 to dst */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 3 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 2 */ + + /* odd 4. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 5 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st1] \n\t" /* average odd 2 */ + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[qload3], 3(%[dst]) \n\t" /* store odd 2 to dst */ + "lbu %[qload1], 5(%[dst]) \n\t" /* load odd 3 from dst */ + "ulw %[qload2], 17(%[src]) \n\t" + "dpa.w.ph $ac3, %[p4], %[filter12] \n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p1], %[filter34] \n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p5], %[filter56] \n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p2], %[filter78] \n\t" /* odd 4 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 4 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 3 */ + + "lbu %[st1], 7(%[dst]) \n\t" /* load odd 4 from dst */ + + /* odd 5. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 6 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload1], %[qload1], %[st2] \n\t" /* average odd 3 */ + "preceu.ph.qbr %[p4], %[qload2] \n\t" + "sb %[qload1], 5(%[dst]) \n\t" /* store odd 3 to dst */ + "dpa.w.ph $ac1, %[p1], %[filter12] \n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p5], %[filter34] \n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p2], %[filter56] \n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p3], %[filter78] \n\t" /* odd 5 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 5 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 4 */ + + "lbu %[qload1], 9(%[dst]) \n\t" /* load odd 5 from dst */ + + /* odd 6. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 7 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[st1], %[st1], %[st3] \n\t" /* average odd 4 */ + "preceu.ph.qbl %[p1], %[qload2] \n\t" + "sb %[st1], 7(%[dst]) \n\t" /* store odd 4 to dst */ + "ulw %[qload3], 21(%[src]) \n\t" + "dpa.w.ph $ac2, %[p5], %[filter12] \n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p2], %[filter34] \n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p3], %[filter56] \n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p4], %[filter78] \n\t" /* odd 6 */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 6 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 5 */ + + /* odd 7. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 8 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[qload1], %[qload1], %[st1] \n\t" /* average odd 5 */ + "preceu.ph.qbr %[p5], %[qload3] \n\t" + "sb %[qload1], 9(%[dst]) \n\t" /* store odd 5 to dst */ + "lbu %[qload2], 11(%[dst]) \n\t" /* load odd 6 from dst */ + "dpa.w.ph $ac3, %[p2], %[filter12] \n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p3], %[filter34] \n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p4], %[filter56] \n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p1], %[filter78] \n\t" /* odd 7 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 7 */ + + "lbu %[qload3], 13(%[dst]) \n\t" /* load odd 7 from dst */ + + /* odd 8. pixel */ + "dpa.w.ph $ac1, %[p3], %[filter12] \n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p4], %[filter34] \n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p1], %[filter56] \n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p5], %[filter78] \n\t" /* odd 8 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 8 */ + + "lbu %[qload1], 15(%[dst]) \n\t" /* load odd 8 from dst */ + + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 6 */ + "addqh_r.w %[qload2], %[qload2], %[st2] \n\t" /* average odd 6 */ + + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 7 */ + "addqh_r.w %[qload3], %[qload3], %[st3] \n\t" /* average odd 7 */ + + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 8 */ + "addqh_r.w %[qload1], %[qload1], %[st1] \n\t" /* average odd 8 */ + + "sb %[qload2], 11(%[dst]) \n\t" /* store odd 6 to dst */ + "sb %[qload3], 13(%[dst]) \n\t" /* store odd 7 to dst */ + "sb %[qload1], 15(%[dst]) \n\t" /* store odd 8 to dst */ + + : [qload1] "=&r"(qload1), [qload2] "=&r"(qload2), [st1] "=&r"(st1), + [st2] "=&r"(st2), [st3] "=&r"(st3), [p1] "=&r"(p1), [p2] "=&r"(p2), + [p3] "=&r"(p3), [p4] "=&r"(p4), [qload3] "=&r"(qload3), + [p5] "=&r"(p5), [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), + [Temp3] "=&r"(Temp3) + : [filter12] "r"(filter12), [filter34] "r"(filter34), + [filter56] "r"(filter56), [filter78] "r"(filter78), + [vector_64] "r"(vector_64), [cm] "r"(cm), [dst] "r"(dst), + [src] "r"(src)); + + src += 16; + dst += 16; + } + + /* Next row... */ + src_ptr += src_stride; + dst_ptr += dst_stride; + } +} + +static void convolve_avg_horiz_64_dspr2(const uint8_t *src_ptr, + int32_t src_stride, uint8_t *dst_ptr, + int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y, c; + const uint8_t *src; + uint8_t *dst; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector_64 = 64; + int32_t filter12, filter34, filter56, filter78; + int32_t Temp1, Temp2, Temp3; + uint32_t qload1, qload2, qload3; + uint32_t p1, p2, p3, p4, p5; + uint32_t st1, st2, st3; + + filter12 = ((const int32_t *)filter_x0)[0]; + filter34 = ((const int32_t *)filter_x0)[1]; + filter56 = ((const int32_t *)filter_x0)[2]; + filter78 = ((const int32_t *)filter_x0)[3]; + + for (y = h; y--;) { + src = src_ptr; + dst = dst_ptr; + + /* prefetch data to cache memory */ + prefetch_load(src_ptr + src_stride); + prefetch_load(src_ptr + src_stride + 32); + prefetch_load(src_ptr + src_stride + 64); + prefetch_store(dst_ptr + dst_stride); + prefetch_store(dst_ptr + dst_stride + 32); + + for (c = 0; c < 4; c++) { + __asm__ __volatile__( + "ulw %[qload1], 0(%[src]) \n\t" + "ulw %[qload2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 1 */ + "mthi $zero, $ac1 \n\t" + "mtlo %[vector_64], $ac2 \n\t" /* even 2 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "ulw %[qload3], 8(%[src]) \n\t" + "dpa.w.ph $ac1, %[p1], %[filter12] \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p2], %[filter34] \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p3], %[filter56] \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p4], %[filter78] \n\t" /* even 1 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 1 */ + "lbu %[st2], 0(%[dst]) \n\t" /* load even 1 from dst */ + + /* even 2. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 3 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "ulw %[qload1], 12(%[src]) \n\t" + "dpa.w.ph $ac2, %[p2], %[filter12] \n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p3], %[filter34] \n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p4], %[filter56] \n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p1], %[filter78] \n\t" /* even 1 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 1 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 1 */ + + "lbu %[qload3], 2(%[dst]) \n\t" /* load even 2 from dst */ + + /* even 3. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 4 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[st2], %[st2], %[st1] \n\t" /* average even 1 */ + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "sb %[st2], 0(%[dst]) \n\t" /* store even 1 to dst */ + "dpa.w.ph $ac3, %[p3], %[filter12] \n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p4], %[filter34] \n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p1], %[filter56] \n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p5], %[filter78] \n\t" /* even 3 */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 3 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 1 */ + + /* even 4. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 5 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st2] \n\t" /* average even 2 */ + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[qload3], 2(%[dst]) \n\t" /* store even 2 to dst */ + "ulw %[qload2], 16(%[src]) \n\t" + "lbu %[qload3], 4(%[dst]) \n\t" /* load even 3 from dst */ + "lbu %[qload1], 6(%[dst]) \n\t" /* load even 4 from dst */ + "dpa.w.ph $ac1, %[p4], %[filter12] \n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p1], %[filter34] \n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p5], %[filter56] \n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p2], %[filter78] \n\t" /* even 4 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 4 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 3 */ + + /* even 5. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 6 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st3] \n\t" /* average even 3 */ + "preceu.ph.qbr %[p4], %[qload2] \n\t" + "sb %[qload3], 4(%[dst]) \n\t" /* store even 3 to dst */ + "dpa.w.ph $ac2, %[p1], %[filter12] \n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p5], %[filter34] \n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p2], %[filter56] \n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p3], %[filter78] \n\t" /* even 5 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 5 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 4 */ + + /* even 6. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 7 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[qload1], %[qload1], %[st1] \n\t" /* average even 4 */ + "preceu.ph.qbl %[p1], %[qload2] \n\t" + "sb %[qload1], 6(%[dst]) \n\t" /* store even 4 to dst */ + "ulw %[qload3], 20(%[src]) \n\t" + "dpa.w.ph $ac3, %[p5], %[filter12] \n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p2], %[filter34] \n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p3], %[filter56] \n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p4], %[filter78] \n\t" /* even 6 */ + "lbu %[qload2], 8(%[dst]) \n\t" /* load even 5 from dst */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 6 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 5 */ + + /* even 7. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 8 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload2], %[qload2], %[st2] \n\t" /* average even 5 */ + "preceu.ph.qbr %[p5], %[qload3] \n\t" + "sb %[qload2], 8(%[dst]) \n\t" /* store even 5 to dst */ + "dpa.w.ph $ac1, %[p2], %[filter12] \n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p3], %[filter34] \n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p4], %[filter56] \n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p1], %[filter78] \n\t" /* even 7 */ + "lbu %[qload3], 10(%[dst]) \n\t" /* load even 6 from dst */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 7 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 6 */ + + "lbu %[st2], 12(%[dst]) \n\t" /* load even 7 from dst */ + + /* even 8. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 1 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st3] \n\t" /* average even 6 */ + "dpa.w.ph $ac2, %[p3], %[filter12] \n\t" /* even 8 */ + "dpa.w.ph $ac2, %[p4], %[filter34] \n\t" /* even 8 */ + "sb %[qload3], 10(%[dst]) \n\t" /* store even 6 to dst */ + "dpa.w.ph $ac2, %[p1], %[filter56] \n\t" /* even 8 */ + "dpa.w.ph $ac2, %[p5], %[filter78] \n\t" /* even 8 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 8 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 7 */ + + /* ODD pixels */ + "ulw %[qload1], 1(%[src]) \n\t" + "ulw %[qload2], 5(%[src]) \n\t" + + "addqh_r.w %[st2], %[st2], %[st1] \n\t" /* average even 7 */ + + /* odd 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 2 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "sb %[st2], 12(%[dst]) \n\t" /* store even 7 to dst */ + "ulw %[qload3], 9(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[filter12] \n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p2], %[filter34] \n\t" /* odd 1 */ + "lbu %[qload2], 14(%[dst]) \n\t" /* load even 8 from dst */ + "dpa.w.ph $ac3, %[p3], %[filter56] \n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p4], %[filter78] \n\t" /* odd 1 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 1 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 8 */ + + "lbu %[st1], 1(%[dst]) \n\t" /* load odd 1 from dst */ + + /* odd 2. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 3 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload2], %[qload2], %[st2] \n\t" /* average even 8 */ + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "sb %[qload2], 14(%[dst]) \n\t" /* store even 8 to dst */ + "ulw %[qload1], 13(%[src]) \n\t" + "dpa.w.ph $ac1, %[p2], %[filter12] \n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p3], %[filter34] \n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p4], %[filter56] \n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p1], %[filter78] \n\t" /* odd 2 */ + "lbu %[qload3], 3(%[dst]) \n\t" /* load odd 2 from dst */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 2 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 1 */ + + /* odd 3. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 4 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[st3], %[st3], %[st1] \n\t" /* average odd 1 */ + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "dpa.w.ph $ac2, %[p3], %[filter12] \n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p4], %[filter34] \n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p1], %[filter56] \n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p5], %[filter78] \n\t" /* odd 3 */ + "sb %[st3], 1(%[dst]) \n\t" /* store odd 1 to dst */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 3 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 2 */ + + /* odd 4. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 5 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[qload3], %[qload3], %[st1] \n\t" /* average odd 2 */ + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[qload3], 3(%[dst]) \n\t" /* store odd 2 to dst */ + "lbu %[qload1], 5(%[dst]) \n\t" /* load odd 3 from dst */ + "ulw %[qload2], 17(%[src]) \n\t" + "dpa.w.ph $ac3, %[p4], %[filter12] \n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p1], %[filter34] \n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p5], %[filter56] \n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p2], %[filter78] \n\t" /* odd 4 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 4 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 3 */ + + "lbu %[st1], 7(%[dst]) \n\t" /* load odd 4 from dst */ + + /* odd 5. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 6 */ + "mthi $zero, $ac2 \n\t" + "addqh_r.w %[qload1], %[qload1], %[st2] \n\t" /* average odd 3 */ + "preceu.ph.qbr %[p4], %[qload2] \n\t" + "sb %[qload1], 5(%[dst]) \n\t" /* store odd 3 to dst */ + "dpa.w.ph $ac1, %[p1], %[filter12] \n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p5], %[filter34] \n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p2], %[filter56] \n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p3], %[filter78] \n\t" /* odd 5 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 5 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 4 */ + + "lbu %[qload1], 9(%[dst]) \n\t" /* load odd 5 from dst */ + + /* odd 6. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 7 */ + "mthi $zero, $ac3 \n\t" + "addqh_r.w %[st1], %[st1], %[st3] \n\t" /* average odd 4 */ + "preceu.ph.qbl %[p1], %[qload2] \n\t" + "sb %[st1], 7(%[dst]) \n\t" /* store odd 4 to dst */ + "ulw %[qload3], 21(%[src]) \n\t" + "dpa.w.ph $ac2, %[p5], %[filter12] \n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p2], %[filter34] \n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p3], %[filter56] \n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p4], %[filter78] \n\t" /* odd 6 */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 6 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 5 */ + + /* odd 7. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 8 */ + "mthi $zero, $ac1 \n\t" + "addqh_r.w %[qload1], %[qload1], %[st1] \n\t" /* average odd 5 */ + "preceu.ph.qbr %[p5], %[qload3] \n\t" + "sb %[qload1], 9(%[dst]) \n\t" /* store odd 5 to dst */ + "lbu %[qload2], 11(%[dst]) \n\t" /* load odd 6 from dst */ + "dpa.w.ph $ac3, %[p2], %[filter12] \n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p3], %[filter34] \n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p4], %[filter56] \n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p1], %[filter78] \n\t" /* odd 7 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 7 */ + + "lbu %[qload3], 13(%[dst]) \n\t" /* load odd 7 from dst */ + + /* odd 8. pixel */ + "dpa.w.ph $ac1, %[p3], %[filter12] \n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p4], %[filter34] \n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p1], %[filter56] \n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p5], %[filter78] \n\t" /* odd 8 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 8 */ + + "lbu %[qload1], 15(%[dst]) \n\t" /* load odd 8 from dst */ + + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 6 */ + "addqh_r.w %[qload2], %[qload2], %[st2] \n\t" /* average odd 6 */ + + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 7 */ + "addqh_r.w %[qload3], %[qload3], %[st3] \n\t" /* average odd 7 */ + + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 8 */ + "addqh_r.w %[qload1], %[qload1], %[st1] \n\t" /* average odd 8 */ + + "sb %[qload2], 11(%[dst]) \n\t" /* store odd 6 to dst */ + "sb %[qload3], 13(%[dst]) \n\t" /* store odd 7 to dst */ + "sb %[qload1], 15(%[dst]) \n\t" /* store odd 8 to dst */ + + : [qload1] "=&r"(qload1), [qload2] "=&r"(qload2), [st1] "=&r"(st1), + [st2] "=&r"(st2), [st3] "=&r"(st3), [p1] "=&r"(p1), [p2] "=&r"(p2), + [p3] "=&r"(p3), [p4] "=&r"(p4), [qload3] "=&r"(qload3), + [p5] "=&r"(p5), [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), + [Temp3] "=&r"(Temp3) + : [filter12] "r"(filter12), [filter34] "r"(filter34), + [filter56] "r"(filter56), [filter78] "r"(filter78), + [vector_64] "r"(vector_64), [cm] "r"(cm), [dst] "r"(dst), + [src] "r"(src)); + + src += 16; + dst += 16; + } + + /* Next row... */ + src_ptr += src_stride; + dst_ptr += dst_stride; + } +} + +void aom_convolve8_avg_horiz_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h) { + assert(x_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + + if (((const int32_t *)filter_x)[0] == 0) { + aom_convolve2_avg_horiz_dspr2(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + } else { + uint32_t pos = 38; + + src -= 3; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + /* prefetch data to cache memory */ + prefetch_load(src); + prefetch_load(src + 32); + prefetch_store(dst); + + switch (w) { + case 4: + convolve_avg_horiz_4_dspr2(src, src_stride, dst, dst_stride, filter_x, + h); + break; + case 8: + convolve_avg_horiz_8_dspr2(src, src_stride, dst, dst_stride, filter_x, + h); + break; + case 16: + convolve_avg_horiz_16_dspr2(src, src_stride, dst, dst_stride, filter_x, + h, 1); + break; + case 32: + convolve_avg_horiz_16_dspr2(src, src_stride, dst, dst_stride, filter_x, + h, 2); + break; + case 64: + prefetch_load(src + 64); + prefetch_store(dst + 32); + + convolve_avg_horiz_64_dspr2(src, src_stride, dst, dst_stride, filter_x, + h); + break; + default: + aom_convolve8_avg_horiz_c(src + 3, src_stride, dst, dst_stride, + filter_x, x_step_q4, filter_y, y_step_q4, w, + h); + break; + } + } +} +#endif diff --git a/third_party/aom/aom_dsp/mips/convolve8_dspr2.c b/third_party/aom/aom_dsp/mips/convolve8_dspr2.c new file mode 100644 index 0000000000..c871702f4b --- /dev/null +++ b/third_party/aom/aom_dsp/mips/convolve8_dspr2.c @@ -0,0 +1,1590 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/convolve_common_dspr2.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/aom_filter.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +static void convolve_horiz_4_transposed_dspr2(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int16_t *filter_x0, + int32_t h) { + int32_t y; + uint8_t *cm = aom_ff_cropTbl; + uint8_t *dst_ptr; + int32_t vector1b, vector2b, vector3b, vector4b; + int32_t Temp1, Temp2, Temp3, Temp4; + uint32_t vector4a = 64; + uint32_t tp1, tp2; + uint32_t p1, p2, p3, p4; + uint32_t tn1, tn2; + + vector1b = ((const int32_t *)filter_x0)[0]; + vector2b = ((const int32_t *)filter_x0)[1]; + vector3b = ((const int32_t *)filter_x0)[2]; + vector4b = ((const int32_t *)filter_x0)[3]; + + for (y = h; y--;) { + dst_ptr = dst; + /* prefetch data to cache memory */ + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[tp1] \n\t" + "preceu.ph.qbl %[p2], %[tp1] \n\t" + "preceu.ph.qbr %[p3], %[tp2] \n\t" + "preceu.ph.qbl %[p4], %[tp2] \n\t" + "dpa.w.ph $ac3, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector3b] \n\t" + "ulw %[tn2], 8(%[src]) \n\t" + "dpa.w.ph $ac3, %[p4], %[vector4b] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + + /* even 2. pixel */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tn2] \n\t" + "balign %[tn1], %[tn2], 3 \n\t" + "balign %[tn2], %[tp2], 3 \n\t" + "balign %[tp2], %[tp1], 3 \n\t" + "dpa.w.ph $ac2, %[p2], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p3], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[p4], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector4b] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + /* odd 1. pixel */ + "lbux %[tp1], %[Temp1](%[cm]) \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[tp2] \n\t" + "preceu.ph.qbl %[p2], %[tp2] \n\t" + "preceu.ph.qbr %[p3], %[tn2] \n\t" + "preceu.ph.qbl %[p4], %[tn2] \n\t" + "dpa.w.ph $ac3, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[p4], %[vector4b] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + /* odd 2. pixel */ + "lbux %[tp2], %[Temp3](%[cm]) \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tn1] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p3], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[p4], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector4b] \n\t" + "extp %[Temp4], $ac2, 31 \n\t" + + /* clamp */ + "lbux %[tn1], %[Temp2](%[cm]) \n\t" + "lbux %[p2], %[Temp4](%[cm]) \n\t" + + /* store bytes */ + "sb %[tp1], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_stride] \n\t" + + "sb %[tn1], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_stride] \n\t" + + "sb %[tp2], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_stride] \n\t" + + "sb %[p2], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_stride] \n\t" + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tn1] "=&r"(tn1), + [tn2] "=&r"(tn2), [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), + [p4] "=&r"(p4), [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), + [Temp3] "=&r"(Temp3), [Temp4] "=&r"(Temp4), [dst_ptr] "+r"(dst_ptr) + : [vector1b] "r"(vector1b), [vector2b] "r"(vector2b), + [vector3b] "r"(vector3b), [vector4b] "r"(vector4b), + [vector4a] "r"(vector4a), [cm] "r"(cm), [src] "r"(src), + [dst_stride] "r"(dst_stride)); + + /* Next row... */ + src += src_stride; + dst += 1; + } +} + +static void convolve_horiz_8_transposed_dspr2(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int16_t *filter_x0, + int32_t h) { + int32_t y; + uint8_t *cm = aom_ff_cropTbl; + uint8_t *dst_ptr; + uint32_t vector4a = 64; + int32_t vector1b, vector2b, vector3b, vector4b; + int32_t Temp1, Temp2, Temp3; + uint32_t tp1, tp2, tp3; + uint32_t p1, p2, p3, p4, n1; + uint8_t *odd_dst; + uint32_t dst_pitch_2 = (dst_stride << 1); + + vector1b = ((const int32_t *)filter_x0)[0]; + vector2b = ((const int32_t *)filter_x0)[1]; + vector3b = ((const int32_t *)filter_x0)[2]; + vector4b = ((const int32_t *)filter_x0)[3]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + + dst_ptr = dst; + odd_dst = (dst_ptr + dst_stride); + + __asm__ __volatile__( + "ulw %[tp2], 0(%[src]) \n\t" + "ulw %[tp1], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tp2] \n\t" + "preceu.ph.qbl %[p2], %[tp2] \n\t" + "preceu.ph.qbr %[p3], %[tp1] \n\t" + "preceu.ph.qbl %[p4], %[tp1] \n\t" + "ulw %[tp3], 8(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[p4], %[vector4b] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + + /* even 2. pixel */ + "preceu.ph.qbr %[p1], %[tp3] \n\t" + "preceu.ph.qbl %[n1], %[tp3] \n\t" + "ulw %[tp2], 12(%[src]) \n\t" + "dpa.w.ph $ac2, %[p2], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p3], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[p4], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector4b] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + /* even 3. pixel */ + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p2], %[tp2] \n\t" + "dpa.w.ph $ac1, %[p3], %[vector1b] \n\t" + "dpa.w.ph $ac1, %[p4], %[vector2b] \n\t" + "dpa.w.ph $ac1, %[p1], %[vector3b] \n\t" + "lbux %[tp3], %[Temp3](%[cm]) \n\t" + "dpa.w.ph $ac1, %[n1], %[vector4b] \n\t" + "extp %[p3], $ac1, 31 \n\t" + + /* even 4. pixel */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "sb %[Temp2], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_pitch_2] \n\t" + "sb %[tp3], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_pitch_2] \n\t" + + "ulw %[tp1], 1(%[src]) \n\t" + "ulw %[tp3], 5(%[src]) \n\t" + + "dpa.w.ph $ac2, %[p4], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector4b] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + "lbux %[tp2], %[p3](%[cm]) \n\t" + + /* odd 1. pixel */ + "mtlo %[vector4a], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p1], %[tp1] \n\t" + "preceu.ph.qbl %[p2], %[tp1] \n\t" + "preceu.ph.qbr %[p3], %[tp3] \n\t" + "preceu.ph.qbl %[p4], %[tp3] \n\t" + "sb %[tp2], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_pitch_2] \n\t" + "ulw %[tp2], 9(%[src]) \n\t" + + "dpa.w.ph $ac3, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[p4], %[vector4b] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + /* odd 2. pixel */ + "lbux %[tp1], %[Temp3](%[cm]) \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tp2] \n\t" + "preceu.ph.qbl %[n1], %[tp2] \n\t" + "ulw %[Temp1], 13(%[src]) \n\t" + "dpa.w.ph $ac1, %[p2], %[vector1b] \n\t" + "sb %[tp1], 0(%[dst_ptr]) \n\t" + "addu %[dst_ptr], %[dst_ptr], %[dst_pitch_2] \n\t" + "dpa.w.ph $ac1, %[p3], %[vector2b] \n\t" + "dpa.w.ph $ac1, %[p4], %[vector3b] \n\t" + "dpa.w.ph $ac1, %[p1], %[vector4b] \n\t" + "extp %[Temp3], $ac1, 31 \n\t" + + /* odd 3. pixel */ + "lbux %[tp3], %[Temp2](%[cm]) \n\t" + "preceu.ph.qbr %[p2], %[Temp1] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p4], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[n1], %[vector4b] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + /* odd 4. pixel */ + "sb %[tp3], 0(%[odd_dst]) \n\t" + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] \n\t" + "dpa.w.ph $ac2, %[p4], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector4b] \n\t" + "extp %[Temp1], $ac2, 31 \n\t" + + /* clamp */ + "lbux %[p4], %[Temp3](%[cm]) \n\t" + "lbux %[p2], %[Temp2](%[cm]) \n\t" + "lbux %[n1], %[Temp1](%[cm]) \n\t" + + /* store bytes */ + "sb %[p4], 0(%[odd_dst]) \n\t" + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] \n\t" + + "sb %[p2], 0(%[odd_dst]) \n\t" + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] \n\t" + + "sb %[n1], 0(%[odd_dst]) \n\t" + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tp3] "=&r"(tp3), [p1] "=&r"(p1), + [p2] "=&r"(p2), [p3] "=&r"(p3), [p4] "=&r"(p4), [n1] "=&r"(n1), + [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3), + [dst_ptr] "+r"(dst_ptr), [odd_dst] "+r"(odd_dst) + : [vector1b] "r"(vector1b), [vector2b] "r"(vector2b), + [vector3b] "r"(vector3b), [vector4b] "r"(vector4b), + [vector4a] "r"(vector4a), [cm] "r"(cm), [src] "r"(src), + [dst_pitch_2] "r"(dst_pitch_2)); + + /* Next row... */ + src += src_stride; + dst += 1; + } +} + +static void convolve_horiz_16_transposed_dspr2( + const uint8_t *src_ptr, int32_t src_stride, uint8_t *dst_ptr, + int32_t dst_stride, const int16_t *filter_x0, int32_t h, int32_t count) { + int32_t c, y; + const uint8_t *src; + uint8_t *dst; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector_64 = 64; + int32_t filter12, filter34, filter56, filter78; + int32_t Temp1, Temp2, Temp3; + uint32_t qload1, qload2; + uint32_t p1, p2, p3, p4, p5; + uint32_t st1, st2, st3; + uint32_t dst_pitch_2 = (dst_stride << 1); + uint8_t *odd_dst; + + filter12 = ((const int32_t *)filter_x0)[0]; + filter34 = ((const int32_t *)filter_x0)[1]; + filter56 = ((const int32_t *)filter_x0)[2]; + filter78 = ((const int32_t *)filter_x0)[3]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src_ptr + src_stride); + prefetch_load(src_ptr + src_stride + 32); + + src = src_ptr; + dst = dst_ptr; + + odd_dst = (dst + dst_stride); + + for (c = 0; c < count; c++) { + __asm__ __volatile__( + "ulw %[qload1], 0(%[src]) " + "\n\t" + "ulw %[qload2], 4(%[src]) " + "\n\t" + + /* even 1. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* even 1 */ + "mthi $zero, $ac1 " + "\n\t" + "mtlo %[vector_64], $ac2 " + "\n\t" /* even 2 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p3], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p4], %[qload2] " + "\n\t" + "preceu.ph.qbr %[p1], %[qload1] " + "\n\t" + "preceu.ph.qbl %[p2], %[qload1] " + "\n\t" + "ulw %[qload2], 8(%[src]) " + "\n\t" + "dpa.w.ph $ac1, %[p1], %[filter12] " + "\n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p2], %[filter34] " + "\n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p3], %[filter56] " + "\n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p4], %[filter78] " + "\n\t" /* even 1 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* even 1 */ + + /* even 2. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* even 3 */ + "mthi $zero, $ac3 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p5], %[qload2] " + "\n\t" + "ulw %[qload1], 12(%[src]) " + "\n\t" + "dpa.w.ph $ac2, %[p2], %[filter12] " + "\n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p3], %[filter34] " + "\n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p4], %[filter56] " + "\n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p1], %[filter78] " + "\n\t" /* even 1 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* even 1 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* even 1 */ + + /* even 3. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* even 4 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbr %[p2], %[qload1] " + "\n\t" + "sb %[st1], 0(%[dst]) " + "\n\t" /* even 1 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + " \n\t" + "dpa.w.ph $ac3, %[p3], %[filter12] " + "\n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p4], %[filter34] " + "\n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p1], %[filter56] " + "\n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p5], %[filter78] " + "\n\t" /* even 3 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* even 3 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* even 1 */ + + /* even 4. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* even 5 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbl %[p3], %[qload1] " + "\n\t" + "sb %[st2], 0(%[dst]) " + "\n\t" /* even 2 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload2], 16(%[src]) " + "\n\t" + "dpa.w.ph $ac1, %[p4], %[filter12] " + "\n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p1], %[filter34] " + "\n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p5], %[filter56] " + "\n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p2], %[filter78] " + "\n\t" /* even 4 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* even 4 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* even 3 */ + + /* even 5. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* even 6 */ + "mthi $zero, $ac3 " + "\n\t" + "preceu.ph.qbr %[p4], %[qload2] " + "\n\t" + "sb %[st3], 0(%[dst]) " + "\n\t" /* even 3 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac2, %[p1], %[filter12] " + "\n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p5], %[filter34] " + "\n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p2], %[filter56] " + "\n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p3], %[filter78] " + "\n\t" /* even 5 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* even 5 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* even 4 */ + + /* even 6. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* even 7 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbl %[p1], %[qload2] " + "\n\t" + "sb %[st1], 0(%[dst]) " + "\n\t" /* even 4 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload1], 20(%[src]) " + "\n\t" + "dpa.w.ph $ac3, %[p5], %[filter12] " + "\n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p2], %[filter34] " + "\n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p3], %[filter56] " + "\n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p4], %[filter78] " + "\n\t" /* even 6 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* even 6 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* even 5 */ + + /* even 7. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* even 8 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p5], %[qload1] " + "\n\t" + "sb %[st2], 0(%[dst]) " + "\n\t" /* even 5 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac1, %[p2], %[filter12] " + "\n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p3], %[filter34] " + "\n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p4], %[filter56] " + "\n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p1], %[filter78] " + "\n\t" /* even 7 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* even 7 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* even 6 */ + + /* even 8. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* odd 1 */ + "mthi $zero, $ac3 " + "\n\t" + "dpa.w.ph $ac2, %[p3], %[filter12] " + "\n\t" /* even 8 */ + "dpa.w.ph $ac2, %[p4], %[filter34] " + "\n\t" /* even 8 */ + "sb %[st3], 0(%[dst]) " + "\n\t" /* even 6 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac2, %[p1], %[filter56] " + "\n\t" /* even 8 */ + "dpa.w.ph $ac2, %[p5], %[filter78] " + "\n\t" /* even 8 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* even 8 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* even 7 */ + + /* ODD pixels */ + "ulw %[qload1], 1(%[src]) " + "\n\t" + "ulw %[qload2], 5(%[src]) " + "\n\t" + + /* odd 1. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* odd 2 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload1] " + "\n\t" + "preceu.ph.qbl %[p2], %[qload1] " + "\n\t" + "preceu.ph.qbr %[p3], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p4], %[qload2] " + "\n\t" + "sb %[st1], 0(%[dst]) " + "\n\t" /* even 7 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload2], 9(%[src]) " + "\n\t" + "dpa.w.ph $ac3, %[p1], %[filter12] " + "\n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p2], %[filter34] " + "\n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p3], %[filter56] " + "\n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p4], %[filter78] " + "\n\t" /* odd 1 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* odd 1 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* even 8 */ + + /* odd 2. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* odd 3 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p5], %[qload2] " + "\n\t" + "sb %[st2], 0(%[dst]) " + "\n\t" /* even 8 */ + "ulw %[qload1], 13(%[src]) " + "\n\t" + "dpa.w.ph $ac1, %[p2], %[filter12] " + "\n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p3], %[filter34] " + "\n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p4], %[filter56] " + "\n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p1], %[filter78] " + "\n\t" /* odd 2 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* odd 2 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* odd 1 */ + + /* odd 3. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* odd 4 */ + "mthi $zero, $ac3 " + "\n\t" + "preceu.ph.qbr %[p2], %[qload1] " + "\n\t" + "sb %[st3], 0(%[odd_dst]) " + "\n\t" /* odd 1 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac2, %[p3], %[filter12] " + "\n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p4], %[filter34] " + "\n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p1], %[filter56] " + "\n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p5], %[filter78] " + "\n\t" /* odd 3 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* odd 3 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* odd 2 */ + + /* odd 4. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* odd 5 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbl %[p3], %[qload1] " + "\n\t" + "sb %[st1], 0(%[odd_dst]) " + "\n\t" /* odd 2 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload2], 17(%[src]) " + "\n\t" + "dpa.w.ph $ac3, %[p4], %[filter12] " + "\n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p1], %[filter34] " + "\n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p5], %[filter56] " + "\n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p2], %[filter78] " + "\n\t" /* odd 4 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* odd 4 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* odd 3 */ + + /* odd 5. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* odd 6 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p4], %[qload2] " + "\n\t" + "sb %[st2], 0(%[odd_dst]) " + "\n\t" /* odd 3 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac1, %[p1], %[filter12] " + "\n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p5], %[filter34] " + "\n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p2], %[filter56] " + "\n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p3], %[filter78] " + "\n\t" /* odd 5 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* odd 5 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* odd 4 */ + + /* odd 6. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* odd 7 */ + "mthi $zero, $ac3 " + "\n\t" + "preceu.ph.qbl %[p1], %[qload2] " + "\n\t" + "sb %[st3], 0(%[odd_dst]) " + "\n\t" /* odd 4 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload1], 21(%[src]) " + "\n\t" + "dpa.w.ph $ac2, %[p5], %[filter12] " + "\n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p2], %[filter34] " + "\n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p3], %[filter56] " + "\n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p4], %[filter78] " + "\n\t" /* odd 6 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* odd 6 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* odd 5 */ + + /* odd 7. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* odd 8 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbr %[p5], %[qload1] " + "\n\t" + "sb %[st1], 0(%[odd_dst]) " + "\n\t" /* odd 5 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac3, %[p2], %[filter12] " + "\n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p3], %[filter34] " + "\n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p4], %[filter56] " + "\n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p1], %[filter78] " + "\n\t" /* odd 7 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* odd 7 */ + + /* odd 8. pixel */ + "dpa.w.ph $ac1, %[p3], %[filter12] " + "\n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p4], %[filter34] " + "\n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p1], %[filter56] " + "\n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p5], %[filter78] " + "\n\t" /* odd 8 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* odd 8 */ + + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* odd 6 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* odd 7 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* odd 8 */ + + "sb %[st2], 0(%[odd_dst]) " + "\n\t" /* odd 6 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + + "sb %[st3], 0(%[odd_dst]) " + "\n\t" /* odd 7 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + + "sb %[st1], 0(%[odd_dst]) " + "\n\t" /* odd 8 */ + + : [qload1] "=&r"(qload1), [qload2] "=&r"(qload2), [p5] "=&r"(p5), + [st1] "=&r"(st1), [st2] "=&r"(st2), [st3] "=&r"(st3), + [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), [p4] "=&r"(p4), + [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3), + [dst] "+r"(dst), [odd_dst] "+r"(odd_dst) + : [filter12] "r"(filter12), [filter34] "r"(filter34), + [filter56] "r"(filter56), [filter78] "r"(filter78), + [vector_64] "r"(vector_64), [cm] "r"(cm), [src] "r"(src), + [dst_pitch_2] "r"(dst_pitch_2)); + + src += 16; + dst = (dst_ptr + ((c + 1) * 16 * dst_stride)); + odd_dst = (dst + dst_stride); + } + + /* Next row... */ + src_ptr += src_stride; + + dst_ptr += 1; + } +} + +static void convolve_horiz_64_transposed_dspr2( + const uint8_t *src_ptr, int32_t src_stride, uint8_t *dst_ptr, + int32_t dst_stride, const int16_t *filter_x0, int32_t h) { + int32_t c, y; + const uint8_t *src; + uint8_t *dst; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector_64 = 64; + int32_t filter12, filter34, filter56, filter78; + int32_t Temp1, Temp2, Temp3; + uint32_t qload1, qload2; + uint32_t p1, p2, p3, p4, p5; + uint32_t st1, st2, st3; + uint32_t dst_pitch_2 = (dst_stride << 1); + uint8_t *odd_dst; + + filter12 = ((const int32_t *)filter_x0)[0]; + filter34 = ((const int32_t *)filter_x0)[1]; + filter56 = ((const int32_t *)filter_x0)[2]; + filter78 = ((const int32_t *)filter_x0)[3]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src_ptr + src_stride); + prefetch_load(src_ptr + src_stride + 32); + prefetch_load(src_ptr + src_stride + 64); + + src = src_ptr; + dst = dst_ptr; + + odd_dst = (dst + dst_stride); + + for (c = 0; c < 4; c++) { + __asm__ __volatile__( + "ulw %[qload1], 0(%[src]) " + "\n\t" + "ulw %[qload2], 4(%[src]) " + "\n\t" + + /* even 1. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* even 1 */ + "mthi $zero, $ac1 " + "\n\t" + "mtlo %[vector_64], $ac2 " + "\n\t" /* even 2 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p3], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p4], %[qload2] " + "\n\t" + "preceu.ph.qbr %[p1], %[qload1] " + "\n\t" + "preceu.ph.qbl %[p2], %[qload1] " + "\n\t" + "ulw %[qload2], 8(%[src]) " + "\n\t" + "dpa.w.ph $ac1, %[p1], %[filter12] " + "\n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p2], %[filter34] " + "\n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p3], %[filter56] " + "\n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p4], %[filter78] " + "\n\t" /* even 1 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* even 1 */ + + /* even 2. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* even 3 */ + "mthi $zero, $ac3 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p5], %[qload2] " + "\n\t" + "ulw %[qload1], 12(%[src]) " + "\n\t" + "dpa.w.ph $ac2, %[p2], %[filter12] " + "\n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p3], %[filter34] " + "\n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p4], %[filter56] " + "\n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p1], %[filter78] " + "\n\t" /* even 1 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* even 1 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* even 1 */ + + /* even 3. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* even 4 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbr %[p2], %[qload1] " + "\n\t" + "sb %[st1], 0(%[dst]) " + "\n\t" /* even 1 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + " \n\t" + "dpa.w.ph $ac3, %[p3], %[filter12] " + "\n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p4], %[filter34] " + "\n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p1], %[filter56] " + "\n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p5], %[filter78] " + "\n\t" /* even 3 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* even 3 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* even 1 */ + + /* even 4. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* even 5 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbl %[p3], %[qload1] " + "\n\t" + "sb %[st2], 0(%[dst]) " + "\n\t" /* even 2 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload2], 16(%[src]) " + "\n\t" + "dpa.w.ph $ac1, %[p4], %[filter12] " + "\n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p1], %[filter34] " + "\n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p5], %[filter56] " + "\n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p2], %[filter78] " + "\n\t" /* even 4 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* even 4 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* even 3 */ + + /* even 5. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* even 6 */ + "mthi $zero, $ac3 " + "\n\t" + "preceu.ph.qbr %[p4], %[qload2] " + "\n\t" + "sb %[st3], 0(%[dst]) " + "\n\t" /* even 3 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac2, %[p1], %[filter12] " + "\n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p5], %[filter34] " + "\n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p2], %[filter56] " + "\n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p3], %[filter78] " + "\n\t" /* even 5 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* even 5 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* even 4 */ + + /* even 6. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* even 7 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbl %[p1], %[qload2] " + "\n\t" + "sb %[st1], 0(%[dst]) " + "\n\t" /* even 4 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload1], 20(%[src]) " + "\n\t" + "dpa.w.ph $ac3, %[p5], %[filter12] " + "\n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p2], %[filter34] " + "\n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p3], %[filter56] " + "\n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p4], %[filter78] " + "\n\t" /* even 6 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* even 6 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* even 5 */ + + /* even 7. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* even 8 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p5], %[qload1] " + "\n\t" + "sb %[st2], 0(%[dst]) " + "\n\t" /* even 5 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac1, %[p2], %[filter12] " + "\n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p3], %[filter34] " + "\n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p4], %[filter56] " + "\n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p1], %[filter78] " + "\n\t" /* even 7 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* even 7 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* even 6 */ + + /* even 8. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* odd 1 */ + "mthi $zero, $ac3 " + "\n\t" + "dpa.w.ph $ac2, %[p3], %[filter12] " + "\n\t" /* even 8 */ + "dpa.w.ph $ac2, %[p4], %[filter34] " + "\n\t" /* even 8 */ + "sb %[st3], 0(%[dst]) " + "\n\t" /* even 6 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac2, %[p1], %[filter56] " + "\n\t" /* even 8 */ + "dpa.w.ph $ac2, %[p5], %[filter78] " + "\n\t" /* even 8 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* even 8 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* even 7 */ + + /* ODD pixels */ + "ulw %[qload1], 1(%[src]) " + "\n\t" + "ulw %[qload2], 5(%[src]) " + "\n\t" + + /* odd 1. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* odd 2 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload1] " + "\n\t" + "preceu.ph.qbl %[p2], %[qload1] " + "\n\t" + "preceu.ph.qbr %[p3], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p4], %[qload2] " + "\n\t" + "sb %[st1], 0(%[dst]) " + "\n\t" /* even 7 */ + "addu %[dst], %[dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload2], 9(%[src]) " + "\n\t" + "dpa.w.ph $ac3, %[p1], %[filter12] " + "\n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p2], %[filter34] " + "\n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p3], %[filter56] " + "\n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p4], %[filter78] " + "\n\t" /* odd 1 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* odd 1 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* even 8 */ + + /* odd 2. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* odd 3 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p1], %[qload2] " + "\n\t" + "preceu.ph.qbl %[p5], %[qload2] " + "\n\t" + "sb %[st2], 0(%[dst]) " + "\n\t" /* even 8 */ + "ulw %[qload1], 13(%[src]) " + "\n\t" + "dpa.w.ph $ac1, %[p2], %[filter12] " + "\n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p3], %[filter34] " + "\n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p4], %[filter56] " + "\n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p1], %[filter78] " + "\n\t" /* odd 2 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* odd 2 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* odd 1 */ + + /* odd 3. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* odd 4 */ + "mthi $zero, $ac3 " + "\n\t" + "preceu.ph.qbr %[p2], %[qload1] " + "\n\t" + "sb %[st3], 0(%[odd_dst]) " + "\n\t" /* odd 1 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac2, %[p3], %[filter12] " + "\n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p4], %[filter34] " + "\n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p1], %[filter56] " + "\n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p5], %[filter78] " + "\n\t" /* odd 3 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* odd 3 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* odd 2 */ + + /* odd 4. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* odd 5 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbl %[p3], %[qload1] " + "\n\t" + "sb %[st1], 0(%[odd_dst]) " + "\n\t" /* odd 2 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload2], 17(%[src]) " + "\n\t" + "dpa.w.ph $ac3, %[p4], %[filter12] " + "\n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p1], %[filter34] " + "\n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p5], %[filter56] " + "\n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p2], %[filter78] " + "\n\t" /* odd 4 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* odd 4 */ + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* odd 3 */ + + /* odd 5. pixel */ + "mtlo %[vector_64], $ac2 " + "\n\t" /* odd 6 */ + "mthi $zero, $ac2 " + "\n\t" + "preceu.ph.qbr %[p4], %[qload2] " + "\n\t" + "sb %[st2], 0(%[odd_dst]) " + "\n\t" /* odd 3 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac1, %[p1], %[filter12] " + "\n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p5], %[filter34] " + "\n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p2], %[filter56] " + "\n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p3], %[filter78] " + "\n\t" /* odd 5 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* odd 5 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* odd 4 */ + + /* odd 6. pixel */ + "mtlo %[vector_64], $ac3 " + "\n\t" /* odd 7 */ + "mthi $zero, $ac3 " + "\n\t" + "preceu.ph.qbl %[p1], %[qload2] " + "\n\t" + "sb %[st3], 0(%[odd_dst]) " + "\n\t" /* odd 4 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "ulw %[qload1], 21(%[src]) " + "\n\t" + "dpa.w.ph $ac2, %[p5], %[filter12] " + "\n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p2], %[filter34] " + "\n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p3], %[filter56] " + "\n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p4], %[filter78] " + "\n\t" /* odd 6 */ + "extp %[Temp2], $ac2, 31 " + "\n\t" /* odd 6 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* odd 5 */ + + /* odd 7. pixel */ + "mtlo %[vector_64], $ac1 " + "\n\t" /* odd 8 */ + "mthi $zero, $ac1 " + "\n\t" + "preceu.ph.qbr %[p5], %[qload1] " + "\n\t" + "sb %[st1], 0(%[odd_dst]) " + "\n\t" /* odd 5 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + "dpa.w.ph $ac3, %[p2], %[filter12] " + "\n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p3], %[filter34] " + "\n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p4], %[filter56] " + "\n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p1], %[filter78] " + "\n\t" /* odd 7 */ + "extp %[Temp3], $ac3, 31 " + "\n\t" /* odd 7 */ + + /* odd 8. pixel */ + "dpa.w.ph $ac1, %[p3], %[filter12] " + "\n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p4], %[filter34] " + "\n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p1], %[filter56] " + "\n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p5], %[filter78] " + "\n\t" /* odd 8 */ + "extp %[Temp1], $ac1, 31 " + "\n\t" /* odd 8 */ + + "lbux %[st2], %[Temp2](%[cm]) " + "\n\t" /* odd 6 */ + "lbux %[st3], %[Temp3](%[cm]) " + "\n\t" /* odd 7 */ + "lbux %[st1], %[Temp1](%[cm]) " + "\n\t" /* odd 8 */ + + "sb %[st2], 0(%[odd_dst]) " + "\n\t" /* odd 6 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + + "sb %[st3], 0(%[odd_dst]) " + "\n\t" /* odd 7 */ + "addu %[odd_dst], %[odd_dst], %[dst_pitch_2] " + "\n\t" + + "sb %[st1], 0(%[odd_dst]) " + "\n\t" /* odd 8 */ + + : [qload1] "=&r"(qload1), [qload2] "=&r"(qload2), [p5] "=&r"(p5), + [st1] "=&r"(st1), [st2] "=&r"(st2), [st3] "=&r"(st3), + [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), [p4] "=&r"(p4), + [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3), + [dst] "+r"(dst), [odd_dst] "+r"(odd_dst) + : [filter12] "r"(filter12), [filter34] "r"(filter34), + [filter56] "r"(filter56), [filter78] "r"(filter78), + [vector_64] "r"(vector_64), [cm] "r"(cm), [src] "r"(src), + [dst_pitch_2] "r"(dst_pitch_2)); + + src += 16; + dst = (dst_ptr + ((c + 1) * 16 * dst_stride)); + odd_dst = (dst + dst_stride); + } + + /* Next row... */ + src_ptr += src_stride; + + dst_ptr += 1; + } +} + +void convolve_horiz_transposed(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter, int w, int h) { + int x, y, k; + + for (y = 0; y < h; ++y) { + for (x = 0; x < w; ++x) { + int sum = 0; + + for (k = 0; k < 8; ++k) sum += src[x + k] * filter[k]; + + dst[x * dst_stride] = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); + } + + src += src_stride; + dst += 1; + } +} + +void copy_horiz_transposed(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, int w, int h) { + int x, y; + + for (y = 0; y < h; ++y) { + for (x = 0; x < w; ++x) { + dst[x * dst_stride] = src[x]; + } + + src += src_stride; + dst += 1; + } +} + +void aom_convolve8_dspr2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, + int x_step_q4, const int16_t *filter_y, int y_step_q4, + int w, int h) { + DECLARE_ALIGNED(32, uint8_t, temp[64 * 135]); + int32_t intermediate_height = ((h * y_step_q4) >> 4) + 7; + uint32_t pos = 38; + + assert(x_step_q4 == 16); + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + if (intermediate_height < h) intermediate_height = h; + + /* copy the src to dst */ + if (filter_x[3] == 0x80) { + copy_horiz_transposed(src - src_stride * 3, src_stride, temp, + intermediate_height, w, intermediate_height); + } else if (((const int32_t *)filter_x)[0] == 0) { + aom_convolve2_dspr2(src - src_stride * 3, src_stride, temp, + intermediate_height, filter_x, w, intermediate_height); + } else { + src -= (src_stride * 3 + 3); + + /* prefetch data to cache memory */ + prefetch_load(src); + prefetch_load(src + 32); + + switch (w) { + case 4: + convolve_horiz_4_transposed_dspr2(src, src_stride, temp, + intermediate_height, filter_x, + intermediate_height); + break; + case 8: + convolve_horiz_8_transposed_dspr2(src, src_stride, temp, + intermediate_height, filter_x, + intermediate_height); + break; + case 16: + case 32: + convolve_horiz_16_transposed_dspr2(src, src_stride, temp, + intermediate_height, filter_x, + intermediate_height, (w / 16)); + break; + case 64: + prefetch_load(src + 32); + convolve_horiz_64_transposed_dspr2(src, src_stride, temp, + intermediate_height, filter_x, + intermediate_height); + break; + default: + convolve_horiz_transposed(src, src_stride, temp, intermediate_height, + filter_x, w, intermediate_height); + break; + } + } + + /* copy the src to dst */ + if (filter_y[3] == 0x80) { + copy_horiz_transposed(temp + 3, intermediate_height, dst, dst_stride, h, w); + } else if (((const int32_t *)filter_y)[0] == 0) { + aom_convolve2_dspr2(temp + 3, intermediate_height, dst, dst_stride, + filter_y, h, w); + } else { + switch (h) { + case 4: + convolve_horiz_4_transposed_dspr2(temp, intermediate_height, dst, + dst_stride, filter_y, w); + break; + case 8: + convolve_horiz_8_transposed_dspr2(temp, intermediate_height, dst, + dst_stride, filter_y, w); + break; + case 16: + case 32: + convolve_horiz_16_transposed_dspr2(temp, intermediate_height, dst, + dst_stride, filter_y, w, (h / 16)); + break; + case 64: + convolve_horiz_64_transposed_dspr2(temp, intermediate_height, dst, + dst_stride, filter_y, w); + break; + default: + convolve_horiz_transposed(temp, intermediate_height, dst, dst_stride, + filter_y, h, w); + break; + } + } +} + +void aom_convolve_copy_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int filter_x_stride, + const int16_t *filter_y, int filter_y_stride, + int w, int h) { + int x, y; + + /* prefetch data to cache memory */ + prefetch_load(src); + prefetch_load(src + 32); + prefetch_store(dst); + + switch (w) { + case 4: { + uint32_t tp1; + + /* 1 word storage */ + for (y = h; y--;) { + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], (%[src]) \n\t" + "sw %[tp1], (%[dst]) \n\t" /* store */ + + : [tp1] "=&r"(tp1) + : [src] "r"(src), [dst] "r"(dst)); + + src += src_stride; + dst += dst_stride; + } + } break; + case 8: { + uint32_t tp1, tp2; + + /* 2 word storage */ + for (y = h; y--;) { + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + "sw %[tp1], 0(%[dst]) \n\t" /* store */ + "sw %[tp2], 4(%[dst]) \n\t" /* store */ + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2) + : [src] "r"(src), [dst] "r"(dst)); + + src += src_stride; + dst += dst_stride; + } + } break; + case 16: { + uint32_t tp1, tp2, tp3, tp4; + + /* 4 word storage */ + for (y = h; y--;) { + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + "ulw %[tp3], 8(%[src]) \n\t" + "ulw %[tp4], 12(%[src]) \n\t" + + "sw %[tp1], 0(%[dst]) \n\t" /* store */ + "sw %[tp2], 4(%[dst]) \n\t" /* store */ + "sw %[tp3], 8(%[dst]) \n\t" /* store */ + "sw %[tp4], 12(%[dst]) \n\t" /* store */ + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tp3] "=&r"(tp3), + [tp4] "=&r"(tp4) + : [src] "r"(src), [dst] "r"(dst)); + + src += src_stride; + dst += dst_stride; + } + } break; + case 32: { + uint32_t tp1, tp2, tp3, tp4; + uint32_t tp5, tp6, tp7, tp8; + + /* 8 word storage */ + for (y = h; y--;) { + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + "ulw %[tp3], 8(%[src]) \n\t" + "ulw %[tp4], 12(%[src]) \n\t" + "ulw %[tp5], 16(%[src]) \n\t" + "ulw %[tp6], 20(%[src]) \n\t" + "ulw %[tp7], 24(%[src]) \n\t" + "ulw %[tp8], 28(%[src]) \n\t" + + "sw %[tp1], 0(%[dst]) \n\t" /* store */ + "sw %[tp2], 4(%[dst]) \n\t" /* store */ + "sw %[tp3], 8(%[dst]) \n\t" /* store */ + "sw %[tp4], 12(%[dst]) \n\t" /* store */ + "sw %[tp5], 16(%[dst]) \n\t" /* store */ + "sw %[tp6], 20(%[dst]) \n\t" /* store */ + "sw %[tp7], 24(%[dst]) \n\t" /* store */ + "sw %[tp8], 28(%[dst]) \n\t" /* store */ + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tp3] "=&r"(tp3), + [tp4] "=&r"(tp4), [tp5] "=&r"(tp5), [tp6] "=&r"(tp6), + [tp7] "=&r"(tp7), [tp8] "=&r"(tp8) + : [src] "r"(src), [dst] "r"(dst)); + + src += src_stride; + dst += dst_stride; + } + } break; + case 64: { + uint32_t tp1, tp2, tp3, tp4; + uint32_t tp5, tp6, tp7, tp8; + + prefetch_load(src + 64); + prefetch_store(dst + 32); + + /* 16 word storage */ + for (y = h; y--;) { + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_load(src + src_stride + 64); + prefetch_store(dst + dst_stride); + prefetch_store(dst + dst_stride + 32); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + "ulw %[tp3], 8(%[src]) \n\t" + "ulw %[tp4], 12(%[src]) \n\t" + "ulw %[tp5], 16(%[src]) \n\t" + "ulw %[tp6], 20(%[src]) \n\t" + "ulw %[tp7], 24(%[src]) \n\t" + "ulw %[tp8], 28(%[src]) \n\t" + + "sw %[tp1], 0(%[dst]) \n\t" /* store */ + "sw %[tp2], 4(%[dst]) \n\t" /* store */ + "sw %[tp3], 8(%[dst]) \n\t" /* store */ + "sw %[tp4], 12(%[dst]) \n\t" /* store */ + "sw %[tp5], 16(%[dst]) \n\t" /* store */ + "sw %[tp6], 20(%[dst]) \n\t" /* store */ + "sw %[tp7], 24(%[dst]) \n\t" /* store */ + "sw %[tp8], 28(%[dst]) \n\t" /* store */ + + "ulw %[tp1], 32(%[src]) \n\t" + "ulw %[tp2], 36(%[src]) \n\t" + "ulw %[tp3], 40(%[src]) \n\t" + "ulw %[tp4], 44(%[src]) \n\t" + "ulw %[tp5], 48(%[src]) \n\t" + "ulw %[tp6], 52(%[src]) \n\t" + "ulw %[tp7], 56(%[src]) \n\t" + "ulw %[tp8], 60(%[src]) \n\t" + + "sw %[tp1], 32(%[dst]) \n\t" /* store */ + "sw %[tp2], 36(%[dst]) \n\t" /* store */ + "sw %[tp3], 40(%[dst]) \n\t" /* store */ + "sw %[tp4], 44(%[dst]) \n\t" /* store */ + "sw %[tp5], 48(%[dst]) \n\t" /* store */ + "sw %[tp6], 52(%[dst]) \n\t" /* store */ + "sw %[tp7], 56(%[dst]) \n\t" /* store */ + "sw %[tp8], 60(%[dst]) \n\t" /* store */ + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tp3] "=&r"(tp3), + [tp4] "=&r"(tp4), [tp5] "=&r"(tp5), [tp6] "=&r"(tp6), + [tp7] "=&r"(tp7), [tp8] "=&r"(tp8) + : [src] "r"(src), [dst] "r"(dst)); + + src += src_stride; + dst += dst_stride; + } + } break; + default: + for (y = h; y--;) { + for (x = 0; x < w; ++x) { + dst[x] = src[x]; + } + + src += src_stride; + dst += dst_stride; + } + break; + } +} +#endif diff --git a/third_party/aom/aom_dsp/mips/convolve8_horiz_dspr2.c b/third_party/aom/aom_dsp/mips/convolve8_horiz_dspr2.c new file mode 100644 index 0000000000..c605576179 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/convolve8_horiz_dspr2.c @@ -0,0 +1,878 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/convolve_common_dspr2.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/aom_filter.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +static void convolve_horiz_4_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y; + uint8_t *cm = aom_ff_cropTbl; + int32_t vector1b, vector2b, vector3b, vector4b; + int32_t Temp1, Temp2, Temp3, Temp4; + uint32_t vector4a = 64; + uint32_t tp1, tp2; + uint32_t p1, p2, p3, p4; + uint32_t n1, n2, n3, n4; + uint32_t tn1, tn2; + + vector1b = ((const int32_t *)filter_x0)[0]; + vector2b = ((const int32_t *)filter_x0)[1]; + vector3b = ((const int32_t *)filter_x0)[2]; + vector4b = ((const int32_t *)filter_x0)[3]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[tp1] \n\t" + "preceu.ph.qbl %[p2], %[tp1] \n\t" + "preceu.ph.qbr %[p3], %[tp2] \n\t" + "preceu.ph.qbl %[p4], %[tp2] \n\t" + "dpa.w.ph $ac3, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector3b] \n\t" + "ulw %[tn2], 8(%[src]) \n\t" + "dpa.w.ph $ac3, %[p4], %[vector4b] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + + /* even 2. pixel */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tn2] \n\t" + "balign %[tn1], %[tn2], 3 \n\t" + "balign %[tn2], %[tp2], 3 \n\t" + "balign %[tp2], %[tp1], 3 \n\t" + "dpa.w.ph $ac2, %[p2], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p3], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[p4], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector4b] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + /* odd 1. pixel */ + "lbux %[tp1], %[Temp1](%[cm]) \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[n1], %[tp2] \n\t" + "preceu.ph.qbl %[n2], %[tp2] \n\t" + "preceu.ph.qbr %[n3], %[tn2] \n\t" + "preceu.ph.qbl %[n4], %[tn2] \n\t" + "dpa.w.ph $ac3, %[n1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[n2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[n3], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[n4], %[vector4b] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + /* odd 2. pixel */ + "lbux %[tp2], %[Temp3](%[cm]) \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[n1], %[tn1] \n\t" + "dpa.w.ph $ac2, %[n2], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[n3], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[n4], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[n1], %[vector4b] \n\t" + "extp %[Temp4], $ac2, 31 \n\t" + + /* clamp */ + "lbux %[tn1], %[Temp2](%[cm]) \n\t" + "lbux %[n2], %[Temp4](%[cm]) \n\t" + + /* store bytes */ + "sb %[tp1], 0(%[dst]) \n\t" + "sb %[tn1], 1(%[dst]) \n\t" + "sb %[tp2], 2(%[dst]) \n\t" + "sb %[n2], 3(%[dst]) \n\t" + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tn1] "=&r"(tn1), + [tn2] "=&r"(tn2), [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), + [p4] "=&r"(p4), [n1] "=&r"(n1), [n2] "=&r"(n2), [n3] "=&r"(n3), + [n4] "=&r"(n4), [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), + [Temp3] "=&r"(Temp3), [Temp4] "=&r"(Temp4) + : [vector1b] "r"(vector1b), [vector2b] "r"(vector2b), + [vector3b] "r"(vector3b), [vector4b] "r"(vector4b), + [vector4a] "r"(vector4a), [cm] "r"(cm), [dst] "r"(dst), + [src] "r"(src)); + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_horiz_8_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector4a = 64; + int32_t vector1b, vector2b, vector3b, vector4b; + int32_t Temp1, Temp2, Temp3; + uint32_t tp1, tp2; + uint32_t p1, p2, p3, p4, n1; + uint32_t tn1, tn2, tn3; + uint32_t st0, st1; + + vector1b = ((const int32_t *)filter_x0)[0]; + vector2b = ((const int32_t *)filter_x0)[1]; + vector3b = ((const int32_t *)filter_x0)[2]; + vector4b = ((const int32_t *)filter_x0)[3]; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_load(src + src_stride); + prefetch_load(src + src_stride + 32); + prefetch_store(dst + dst_stride); + + __asm__ __volatile__( + "ulw %[tp1], 0(%[src]) \n\t" + "ulw %[tp2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tp1] \n\t" + "preceu.ph.qbl %[p2], %[tp1] \n\t" + "preceu.ph.qbr %[p3], %[tp2] \n\t" + "preceu.ph.qbl %[p4], %[tp2] \n\t" + "ulw %[tn2], 8(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[p4], %[vector4b] \n\t" + "extp %[Temp1], $ac3, 31 \n\t" + + /* even 2. pixel */ + "preceu.ph.qbr %[p1], %[tn2] \n\t" + "preceu.ph.qbl %[n1], %[tn2] \n\t" + "ulw %[tn1], 12(%[src]) \n\t" + "dpa.w.ph $ac2, %[p2], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p3], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[p4], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector4b] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + /* even 3. pixel */ + "lbux %[st0], %[Temp1](%[cm]) \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p2], %[tn1] \n\t" + "dpa.w.ph $ac1, %[p3], %[vector1b] \n\t" + "dpa.w.ph $ac1, %[p4], %[vector2b] \n\t" + "dpa.w.ph $ac1, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac1, %[n1], %[vector4b] \n\t" + "extp %[Temp1], $ac1, 31 \n\t" + + /* even 4. pixel */ + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "sb %[st0], 0(%[dst]) \n\t" + "lbux %[st1], %[Temp3](%[cm]) \n\t" + + "balign %[tn3], %[tn1], 3 \n\t" + "balign %[tn1], %[tn2], 3 \n\t" + "balign %[tn2], %[tp2], 3 \n\t" + "balign %[tp2], %[tp1], 3 \n\t" + + "dpa.w.ph $ac2, %[p4], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector4b] \n\t" + "extp %[Temp3], $ac2, 31 \n\t" + + "lbux %[st0], %[Temp1](%[cm]) \n\t" + + /* odd 1. pixel */ + "mtlo %[vector4a], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "sb %[st1], 2(%[dst]) \n\t" + "preceu.ph.qbr %[p1], %[tp2] \n\t" + "preceu.ph.qbl %[p2], %[tp2] \n\t" + "preceu.ph.qbr %[p3], %[tn2] \n\t" + "preceu.ph.qbl %[p4], %[tn2] \n\t" + "sb %[st0], 4(%[dst]) \n\t" + "dpa.w.ph $ac3, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[p4], %[vector4b] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + /* odd 2. pixel */ + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[tn1] \n\t" + "preceu.ph.qbl %[n1], %[tn1] \n\t" + "lbux %[st0], %[Temp3](%[cm]) \n\t" + "dpa.w.ph $ac1, %[p2], %[vector1b] \n\t" + "dpa.w.ph $ac1, %[p3], %[vector2b] \n\t" + "dpa.w.ph $ac1, %[p4], %[vector3b] \n\t" + "dpa.w.ph $ac1, %[p1], %[vector4b] \n\t" + "extp %[Temp3], $ac1, 31 \n\t" + + /* odd 3. pixel */ + "lbux %[st1], %[Temp2](%[cm]) \n\t" + "preceu.ph.qbr %[p2], %[tn3] \n\t" + "dpa.w.ph $ac3, %[p3], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[p4], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[n1], %[vector4b] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + /* odd 4. pixel */ + "sb %[st1], 1(%[dst]) \n\t" + "sb %[st0], 6(%[dst]) \n\t" + "dpa.w.ph $ac2, %[p4], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p1], %[vector2b] \n\t" + "dpa.w.ph $ac2, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector4b] \n\t" + "extp %[Temp1], $ac2, 31 \n\t" + + /* clamp */ + "lbux %[p4], %[Temp3](%[cm]) \n\t" + "lbux %[p2], %[Temp2](%[cm]) \n\t" + "lbux %[n1], %[Temp1](%[cm]) \n\t" + + /* store bytes */ + "sb %[p4], 3(%[dst]) \n\t" + "sb %[p2], 5(%[dst]) \n\t" + "sb %[n1], 7(%[dst]) \n\t" + + : [tp1] "=&r"(tp1), [tp2] "=&r"(tp2), [tn1] "=&r"(tn1), + [tn2] "=&r"(tn2), [tn3] "=&r"(tn3), [st0] "=&r"(st0), + [st1] "=&r"(st1), [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), + [p4] "=&r"(p4), [n1] "=&r"(n1), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3) + : [vector1b] "r"(vector1b), [vector2b] "r"(vector2b), + [vector3b] "r"(vector3b), [vector4b] "r"(vector4b), + [vector4a] "r"(vector4a), [cm] "r"(cm), [dst] "r"(dst), + [src] "r"(src)); + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_horiz_16_dspr2(const uint8_t *src_ptr, int32_t src_stride, + uint8_t *dst_ptr, int32_t dst_stride, + const int16_t *filter_x0, int32_t h, + int32_t count) { + int32_t y, c; + const uint8_t *src; + uint8_t *dst; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector_64 = 64; + int32_t filter12, filter34, filter56, filter78; + int32_t Temp1, Temp2, Temp3; + uint32_t qload1, qload2, qload3; + uint32_t p1, p2, p3, p4, p5; + uint32_t st1, st2, st3; + + filter12 = ((const int32_t *)filter_x0)[0]; + filter34 = ((const int32_t *)filter_x0)[1]; + filter56 = ((const int32_t *)filter_x0)[2]; + filter78 = ((const int32_t *)filter_x0)[3]; + + for (y = h; y--;) { + src = src_ptr; + dst = dst_ptr; + + /* prefetch data to cache memory */ + prefetch_load(src_ptr + src_stride); + prefetch_load(src_ptr + src_stride + 32); + prefetch_store(dst_ptr + dst_stride); + + for (c = 0; c < count; c++) { + __asm__ __volatile__( + "ulw %[qload1], 0(%[src]) \n\t" + "ulw %[qload2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 1 */ + "mthi $zero, $ac1 \n\t" + "mtlo %[vector_64], $ac2 \n\t" /* even 2 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "ulw %[qload3], 8(%[src]) \n\t" + "dpa.w.ph $ac1, %[p1], %[filter12] \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p2], %[filter34] \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p3], %[filter56] \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p4], %[filter78] \n\t" /* even 1 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 1 */ + + /* even 2. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 3 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "ulw %[qload1], 12(%[src]) \n\t" + "dpa.w.ph $ac2, %[p2], %[filter12] \n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p3], %[filter34] \n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p4], %[filter56] \n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p1], %[filter78] \n\t" /* even 1 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 1 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 1 */ + + /* even 3. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 4 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "sb %[st1], 0(%[dst]) \n\t" /* even 1 */ + "dpa.w.ph $ac3, %[p3], %[filter12] \n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p4], %[filter34] \n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p1], %[filter56] \n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p5], %[filter78] \n\t" /* even 3 */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 3 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 1 */ + + /* even 4. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 5 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[st2], 2(%[dst]) \n\t" /* even 1 */ + "ulw %[qload2], 16(%[src]) \n\t" + "dpa.w.ph $ac1, %[p4], %[filter12] \n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p1], %[filter34] \n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p5], %[filter56] \n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p2], %[filter78] \n\t" /* even 4 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 4 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 3 */ + + /* even 5. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 6 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p4], %[qload2] \n\t" + "sb %[st3], 4(%[dst]) \n\t" /* even 3 */ + "dpa.w.ph $ac2, %[p1], %[filter12] \n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p5], %[filter34] \n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p2], %[filter56] \n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p3], %[filter78] \n\t" /* even 5 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 5 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 4 */ + + /* even 6. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 7 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbl %[p1], %[qload2] \n\t" + "sb %[st1], 6(%[dst]) \n\t" /* even 4 */ + "ulw %[qload3], 20(%[src]) \n\t" + "dpa.w.ph $ac3, %[p5], %[filter12] \n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p2], %[filter34] \n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p3], %[filter56] \n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p4], %[filter78] \n\t" /* even 6 */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 6 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 5 */ + + /* even 7. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 8 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p5], %[qload3] \n\t" + "sb %[st2], 8(%[dst]) \n\t" /* even 5 */ + "dpa.w.ph $ac1, %[p2], %[filter12] \n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p3], %[filter34] \n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p4], %[filter56] \n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p1], %[filter78] \n\t" /* even 7 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 7 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 6 */ + + /* even 8. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 1 */ + "mthi $zero, $ac3 \n\t" + "dpa.w.ph $ac2, %[p3], %[filter12] \n\t" /* even 8 */ + "dpa.w.ph $ac2, %[p4], %[filter34] \n\t" /* even 8 */ + "sb %[st3], 10(%[dst]) \n\t" /* even 6 */ + "dpa.w.ph $ac2, %[p1], %[filter56] \n\t" /* even 8 */ + "dpa.w.ph $ac2, %[p5], %[filter78] \n\t" /* even 8 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 8 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 7 */ + + /* ODD pixels */ + "ulw %[qload1], 1(%[src]) \n\t" + "ulw %[qload2], 5(%[src]) \n\t" + + /* odd 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 2 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "sb %[st1], 12(%[dst]) \n\t" /* even 7 */ + "ulw %[qload3], 9(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[filter12] \n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p2], %[filter34] \n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p3], %[filter56] \n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p4], %[filter78] \n\t" /* odd 1 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 1 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 8 */ + + /* odd 2. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 3 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "sb %[st2], 14(%[dst]) \n\t" /* even 8 */ + "ulw %[qload1], 13(%[src]) \n\t" + "dpa.w.ph $ac1, %[p2], %[filter12] \n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p3], %[filter34] \n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p4], %[filter56] \n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p1], %[filter78] \n\t" /* odd 2 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 2 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 1 */ + + /* odd 3. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 4 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "sb %[st3], 1(%[dst]) \n\t" /* odd 1 */ + "dpa.w.ph $ac2, %[p3], %[filter12] \n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p4], %[filter34] \n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p1], %[filter56] \n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p5], %[filter78] \n\t" /* odd 3 */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 3 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 2 */ + + /* odd 4. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 5 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[st1], 3(%[dst]) \n\t" /* odd 2 */ + "ulw %[qload2], 17(%[src]) \n\t" + "dpa.w.ph $ac3, %[p4], %[filter12] \n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p1], %[filter34] \n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p5], %[filter56] \n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p2], %[filter78] \n\t" /* odd 4 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 4 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 3 */ + + /* odd 5. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 6 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p4], %[qload2] \n\t" + "sb %[st2], 5(%[dst]) \n\t" /* odd 3 */ + "dpa.w.ph $ac1, %[p1], %[filter12] \n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p5], %[filter34] \n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p2], %[filter56] \n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p3], %[filter78] \n\t" /* odd 5 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 5 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 4 */ + + /* odd 6. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 7 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbl %[p1], %[qload2] \n\t" + "sb %[st3], 7(%[dst]) \n\t" /* odd 4 */ + "ulw %[qload3], 21(%[src]) \n\t" + "dpa.w.ph $ac2, %[p5], %[filter12] \n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p2], %[filter34] \n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p3], %[filter56] \n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p4], %[filter78] \n\t" /* odd 6 */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 6 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 5 */ + + /* odd 7. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 8 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p5], %[qload3] \n\t" + "sb %[st1], 9(%[dst]) \n\t" /* odd 5 */ + "dpa.w.ph $ac3, %[p2], %[filter12] \n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p3], %[filter34] \n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p4], %[filter56] \n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p1], %[filter78] \n\t" /* odd 7 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 7 */ + + /* odd 8. pixel */ + "dpa.w.ph $ac1, %[p3], %[filter12] \n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p4], %[filter34] \n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p1], %[filter56] \n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p5], %[filter78] \n\t" /* odd 8 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 8 */ + + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 6 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 7 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 8 */ + + "sb %[st2], 11(%[dst]) \n\t" /* odd 6 */ + "sb %[st3], 13(%[dst]) \n\t" /* odd 7 */ + "sb %[st1], 15(%[dst]) \n\t" /* odd 8 */ + + : [qload1] "=&r"(qload1), [qload2] "=&r"(qload2), + [qload3] "=&r"(qload3), [st1] "=&r"(st1), [st2] "=&r"(st2), + [st3] "=&r"(st3), [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), + [p4] "=&r"(p4), [p5] "=&r"(p5), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3) + : [filter12] "r"(filter12), [filter34] "r"(filter34), + [filter56] "r"(filter56), [filter78] "r"(filter78), + [vector_64] "r"(vector_64), [cm] "r"(cm), [dst] "r"(dst), + [src] "r"(src)); + + src += 16; + dst += 16; + } + + /* Next row... */ + src_ptr += src_stride; + dst_ptr += dst_stride; + } +} + +static void convolve_horiz_64_dspr2(const uint8_t *src_ptr, int32_t src_stride, + uint8_t *dst_ptr, int32_t dst_stride, + const int16_t *filter_x0, int32_t h) { + int32_t y, c; + const uint8_t *src; + uint8_t *dst; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector_64 = 64; + int32_t filter12, filter34, filter56, filter78; + int32_t Temp1, Temp2, Temp3; + uint32_t qload1, qload2, qload3; + uint32_t p1, p2, p3, p4, p5; + uint32_t st1, st2, st3; + + filter12 = ((const int32_t *)filter_x0)[0]; + filter34 = ((const int32_t *)filter_x0)[1]; + filter56 = ((const int32_t *)filter_x0)[2]; + filter78 = ((const int32_t *)filter_x0)[3]; + + for (y = h; y--;) { + src = src_ptr; + dst = dst_ptr; + + /* prefetch data to cache memory */ + prefetch_load(src_ptr + src_stride); + prefetch_load(src_ptr + src_stride + 32); + prefetch_load(src_ptr + src_stride + 64); + prefetch_store(dst_ptr + dst_stride); + prefetch_store(dst_ptr + dst_stride + 32); + + for (c = 0; c < 4; c++) { + __asm__ __volatile__( + "ulw %[qload1], 0(%[src]) \n\t" + "ulw %[qload2], 4(%[src]) \n\t" + + /* even 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 1 */ + "mthi $zero, $ac1 \n\t" + "mtlo %[vector_64], $ac2 \n\t" /* even 2 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "ulw %[qload3], 8(%[src]) \n\t" + "dpa.w.ph $ac1, %[p1], %[filter12] \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p2], %[filter34] \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p3], %[filter56] \n\t" /* even 1 */ + "dpa.w.ph $ac1, %[p4], %[filter78] \n\t" /* even 1 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 1 */ + + /* even 2. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 3 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "ulw %[qload1], 12(%[src]) \n\t" + "dpa.w.ph $ac2, %[p2], %[filter12] \n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p3], %[filter34] \n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p4], %[filter56] \n\t" /* even 1 */ + "dpa.w.ph $ac2, %[p1], %[filter78] \n\t" /* even 1 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 1 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 1 */ + + /* even 3. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 4 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "sb %[st1], 0(%[dst]) \n\t" /* even 1 */ + "dpa.w.ph $ac3, %[p3], %[filter12] \n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p4], %[filter34] \n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p1], %[filter56] \n\t" /* even 3 */ + "dpa.w.ph $ac3, %[p5], %[filter78] \n\t" /* even 3 */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 3 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 1 */ + + /* even 4. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 5 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[st2], 2(%[dst]) \n\t" /* even 1 */ + "ulw %[qload2], 16(%[src]) \n\t" + "dpa.w.ph $ac1, %[p4], %[filter12] \n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p1], %[filter34] \n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p5], %[filter56] \n\t" /* even 4 */ + "dpa.w.ph $ac1, %[p2], %[filter78] \n\t" /* even 4 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 4 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 3 */ + + /* even 5. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* even 6 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p4], %[qload2] \n\t" + "sb %[st3], 4(%[dst]) \n\t" /* even 3 */ + "dpa.w.ph $ac2, %[p1], %[filter12] \n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p5], %[filter34] \n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p2], %[filter56] \n\t" /* even 5 */ + "dpa.w.ph $ac2, %[p3], %[filter78] \n\t" /* even 5 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 5 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 4 */ + + /* even 6. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* even 7 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbl %[p1], %[qload2] \n\t" + "sb %[st1], 6(%[dst]) \n\t" /* even 4 */ + "ulw %[qload3], 20(%[src]) \n\t" + "dpa.w.ph $ac3, %[p5], %[filter12] \n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p2], %[filter34] \n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p3], %[filter56] \n\t" /* even 6 */ + "dpa.w.ph $ac3, %[p4], %[filter78] \n\t" /* even 6 */ + "extp %[Temp3], $ac3, 31 \n\t" /* even 6 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 5 */ + + /* even 7. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* even 8 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p5], %[qload3] \n\t" + "sb %[st2], 8(%[dst]) \n\t" /* even 5 */ + "dpa.w.ph $ac1, %[p2], %[filter12] \n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p3], %[filter34] \n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p4], %[filter56] \n\t" /* even 7 */ + "dpa.w.ph $ac1, %[p1], %[filter78] \n\t" /* even 7 */ + "extp %[Temp1], $ac1, 31 \n\t" /* even 7 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* even 6 */ + + /* even 8. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 1 */ + "mthi $zero, $ac3 \n\t" + "dpa.w.ph $ac2, %[p3], %[filter12] \n\t" /* even 8 */ + "dpa.w.ph $ac2, %[p4], %[filter34] \n\t" /* even 8 */ + "sb %[st3], 10(%[dst]) \n\t" /* even 6 */ + "dpa.w.ph $ac2, %[p1], %[filter56] \n\t" /* even 8 */ + "dpa.w.ph $ac2, %[p5], %[filter78] \n\t" /* even 8 */ + "extp %[Temp2], $ac2, 31 \n\t" /* even 8 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* even 7 */ + + /* ODD pixels */ + "ulw %[qload1], 1(%[src]) \n\t" + "ulw %[qload2], 5(%[src]) \n\t" + + /* odd 1. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 2 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p1], %[qload1] \n\t" + "preceu.ph.qbl %[p2], %[qload1] \n\t" + "preceu.ph.qbr %[p3], %[qload2] \n\t" + "preceu.ph.qbl %[p4], %[qload2] \n\t" + "sb %[st1], 12(%[dst]) \n\t" /* even 7 */ + "ulw %[qload3], 9(%[src]) \n\t" + "dpa.w.ph $ac3, %[p1], %[filter12] \n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p2], %[filter34] \n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p3], %[filter56] \n\t" /* odd 1 */ + "dpa.w.ph $ac3, %[p4], %[filter78] \n\t" /* odd 1 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 1 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* even 8 */ + + /* odd 2. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 3 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p1], %[qload3] \n\t" + "preceu.ph.qbl %[p5], %[qload3] \n\t" + "sb %[st2], 14(%[dst]) \n\t" /* even 8 */ + "ulw %[qload1], 13(%[src]) \n\t" + "dpa.w.ph $ac1, %[p2], %[filter12] \n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p3], %[filter34] \n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p4], %[filter56] \n\t" /* odd 2 */ + "dpa.w.ph $ac1, %[p1], %[filter78] \n\t" /* odd 2 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 2 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 1 */ + + /* odd 3. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 4 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbr %[p2], %[qload1] \n\t" + "sb %[st3], 1(%[dst]) \n\t" /* odd 1 */ + "dpa.w.ph $ac2, %[p3], %[filter12] \n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p4], %[filter34] \n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p1], %[filter56] \n\t" /* odd 3 */ + "dpa.w.ph $ac2, %[p5], %[filter78] \n\t" /* odd 3 */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 3 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 2 */ + + /* odd 4. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 5 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbl %[p3], %[qload1] \n\t" + "sb %[st1], 3(%[dst]) \n\t" /* odd 2 */ + "ulw %[qload2], 17(%[src]) \n\t" + "dpa.w.ph $ac3, %[p4], %[filter12] \n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p1], %[filter34] \n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p5], %[filter56] \n\t" /* odd 4 */ + "dpa.w.ph $ac3, %[p2], %[filter78] \n\t" /* odd 4 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 4 */ + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 3 */ + + /* odd 5. pixel */ + "mtlo %[vector_64], $ac2 \n\t" /* odd 6 */ + "mthi $zero, $ac2 \n\t" + "preceu.ph.qbr %[p4], %[qload2] \n\t" + "sb %[st2], 5(%[dst]) \n\t" /* odd 3 */ + "dpa.w.ph $ac1, %[p1], %[filter12] \n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p5], %[filter34] \n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p2], %[filter56] \n\t" /* odd 5 */ + "dpa.w.ph $ac1, %[p3], %[filter78] \n\t" /* odd 5 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 5 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 4 */ + + /* odd 6. pixel */ + "mtlo %[vector_64], $ac3 \n\t" /* odd 7 */ + "mthi $zero, $ac3 \n\t" + "preceu.ph.qbl %[p1], %[qload2] \n\t" + "sb %[st3], 7(%[dst]) \n\t" /* odd 4 */ + "ulw %[qload3], 21(%[src]) \n\t" + "dpa.w.ph $ac2, %[p5], %[filter12] \n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p2], %[filter34] \n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p3], %[filter56] \n\t" /* odd 6 */ + "dpa.w.ph $ac2, %[p4], %[filter78] \n\t" /* odd 6 */ + "extp %[Temp2], $ac2, 31 \n\t" /* odd 6 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 5 */ + + /* odd 7. pixel */ + "mtlo %[vector_64], $ac1 \n\t" /* odd 8 */ + "mthi $zero, $ac1 \n\t" + "preceu.ph.qbr %[p5], %[qload3] \n\t" + "sb %[st1], 9(%[dst]) \n\t" /* odd 5 */ + "dpa.w.ph $ac3, %[p2], %[filter12] \n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p3], %[filter34] \n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p4], %[filter56] \n\t" /* odd 7 */ + "dpa.w.ph $ac3, %[p1], %[filter78] \n\t" /* odd 7 */ + "extp %[Temp3], $ac3, 31 \n\t" /* odd 7 */ + + /* odd 8. pixel */ + "dpa.w.ph $ac1, %[p3], %[filter12] \n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p4], %[filter34] \n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p1], %[filter56] \n\t" /* odd 8 */ + "dpa.w.ph $ac1, %[p5], %[filter78] \n\t" /* odd 8 */ + "extp %[Temp1], $ac1, 31 \n\t" /* odd 8 */ + + "lbux %[st2], %[Temp2](%[cm]) \n\t" /* odd 6 */ + "lbux %[st3], %[Temp3](%[cm]) \n\t" /* odd 7 */ + "lbux %[st1], %[Temp1](%[cm]) \n\t" /* odd 8 */ + + "sb %[st2], 11(%[dst]) \n\t" /* odd 6 */ + "sb %[st3], 13(%[dst]) \n\t" /* odd 7 */ + "sb %[st1], 15(%[dst]) \n\t" /* odd 8 */ + + : [qload1] "=&r"(qload1), [qload2] "=&r"(qload2), + [qload3] "=&r"(qload3), [st1] "=&r"(st1), [st2] "=&r"(st2), + [st3] "=&r"(st3), [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), + [p4] "=&r"(p4), [p5] "=&r"(p5), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [Temp3] "=&r"(Temp3) + : [filter12] "r"(filter12), [filter34] "r"(filter34), + [filter56] "r"(filter56), [filter78] "r"(filter78), + [vector_64] "r"(vector_64), [cm] "r"(cm), [dst] "r"(dst), + [src] "r"(src)); + + src += 16; + dst += 16; + } + + /* Next row... */ + src_ptr += src_stride; + dst_ptr += dst_stride; + } +} + +void aom_convolve8_horiz_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + assert(x_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + + if (((const int32_t *)filter_x)[0] == 0) { + aom_convolve2_horiz_dspr2(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + } else { + uint32_t pos = 38; + + prefetch_load((const uint8_t *)filter_x); + src -= 3; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + /* prefetch data to cache memory */ + prefetch_load(src); + prefetch_load(src + 32); + prefetch_store(dst); + + switch (w) { + case 4: + convolve_horiz_4_dspr2(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filter_x, (int32_t)h); + break; + case 8: + convolve_horiz_8_dspr2(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filter_x, (int32_t)h); + break; + case 16: + convolve_horiz_16_dspr2(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filter_x, (int32_t)h, 1); + break; + case 32: + convolve_horiz_16_dspr2(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filter_x, (int32_t)h, 2); + break; + case 64: + prefetch_load(src + 64); + prefetch_store(dst + 32); + + convolve_horiz_64_dspr2(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filter_x, (int32_t)h); + break; + default: + aom_convolve8_horiz_c(src + 3, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } +} +#endif diff --git a/third_party/aom/aom_dsp/mips/convolve8_vert_dspr2.c b/third_party/aom/aom_dsp/mips/convolve8_vert_dspr2.c new file mode 100644 index 0000000000..d8a90b6abd --- /dev/null +++ b/third_party/aom/aom_dsp/mips/convolve8_vert_dspr2.c @@ -0,0 +1,360 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/convolve_common_dspr2.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/aom_filter.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +static void convolve_vert_4_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_y, int32_t w, + int32_t h) { + int32_t x, y; + const uint8_t *src_ptr; + uint8_t *dst_ptr; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector4a = 64; + uint32_t load1, load2, load3, load4; + uint32_t p1, p2; + uint32_t n1, n2; + uint32_t scratch1, scratch2; + uint32_t store1, store2; + int32_t vector1b, vector2b, vector3b, vector4b; + int32_t Temp1, Temp2; + + vector1b = ((const int32_t *)filter_y)[0]; + vector2b = ((const int32_t *)filter_y)[1]; + vector3b = ((const int32_t *)filter_y)[2]; + vector4b = ((const int32_t *)filter_y)[3]; + + src -= 3 * src_stride; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_store(dst + dst_stride); + + for (x = 0; x < w; x += 4) { + src_ptr = src + x; + dst_ptr = dst + x; + + __asm__ __volatile__( + "ulw %[load1], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load2], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load3], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load4], 0(%[src_ptr]) \n\t" + + "mtlo %[vector4a], $ac0 \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac0 \n\t" + "mthi $zero, $ac1 \n\t" + "mthi $zero, $ac2 \n\t" + "mthi $zero, $ac3 \n\t" + + "preceu.ph.qbr %[scratch1], %[load1] \n\t" + "preceu.ph.qbr %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbr %[scratch2], %[load3] \n\t" + "preceu.ph.qbr %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac0, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac0, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac1, %[n1], %[vector1b] \n\t" + "dpa.w.ph $ac1, %[n2], %[vector2b] \n\t" + + "preceu.ph.qbl %[scratch1], %[load1] \n\t" + "preceu.ph.qbl %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbl %[scratch2], %[load3] \n\t" + "preceu.ph.qbl %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac2, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[n1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[n2], %[vector2b] \n\t" + + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load1], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load2], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load3], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load4], 0(%[src_ptr]) \n\t" + + "preceu.ph.qbr %[scratch1], %[load1] \n\t" + "preceu.ph.qbr %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbr %[scratch2], %[load3] \n\t" + "preceu.ph.qbr %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac0, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac0, %[p2], %[vector4b] \n\t" + "extp %[Temp1], $ac0, 31 \n\t" + "dpa.w.ph $ac1, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac1, %[n2], %[vector4b] \n\t" + "extp %[Temp2], $ac1, 31 \n\t" + + "preceu.ph.qbl %[scratch1], %[load1] \n\t" + "preceu.ph.qbl %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbl %[scratch2], %[load3] \n\t" + "preceu.ph.qbl %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "dpa.w.ph $ac2, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector4b] \n\t" + "extp %[Temp1], $ac2, 31 \n\t" + + "lbux %[store2], %[Temp2](%[cm]) \n\t" + "dpa.w.ph $ac3, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[n2], %[vector4b] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + "sb %[store1], 0(%[dst_ptr]) \n\t" + "sb %[store2], 1(%[dst_ptr]) \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "lbux %[store2], %[Temp2](%[cm]) \n\t" + + "sb %[store1], 2(%[dst_ptr]) \n\t" + "sb %[store2], 3(%[dst_ptr]) \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [p1] "=&r"(p1), [p2] "=&r"(p2), + [n1] "=&r"(n1), [n2] "=&r"(n2), [scratch1] "=&r"(scratch1), + [scratch2] "=&r"(scratch2), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [store1] "=&r"(store1), + [store2] "=&r"(store2), [src_ptr] "+r"(src_ptr) + : [vector1b] "r"(vector1b), [vector2b] "r"(vector2b), + [vector3b] "r"(vector3b), [vector4b] "r"(vector4b), + [vector4a] "r"(vector4a), [src_stride] "r"(src_stride), + [cm] "r"(cm), [dst_ptr] "r"(dst_ptr)); + } + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +static void convolve_vert_64_dspr2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int16_t *filter_y, int32_t h) { + int32_t x, y; + const uint8_t *src_ptr; + uint8_t *dst_ptr; + uint8_t *cm = aom_ff_cropTbl; + uint32_t vector4a = 64; + uint32_t load1, load2, load3, load4; + uint32_t p1, p2; + uint32_t n1, n2; + uint32_t scratch1, scratch2; + uint32_t store1, store2; + int32_t vector1b, vector2b, vector3b, vector4b; + int32_t Temp1, Temp2; + + vector1b = ((const int32_t *)filter_y)[0]; + vector2b = ((const int32_t *)filter_y)[1]; + vector3b = ((const int32_t *)filter_y)[2]; + vector4b = ((const int32_t *)filter_y)[3]; + + src -= 3 * src_stride; + + for (y = h; y--;) { + /* prefetch data to cache memory */ + prefetch_store(dst + dst_stride); + prefetch_store(dst + dst_stride + 32); + + for (x = 0; x < 64; x += 4) { + src_ptr = src + x; + dst_ptr = dst + x; + + __asm__ __volatile__( + "ulw %[load1], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load2], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load3], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load4], 0(%[src_ptr]) \n\t" + + "mtlo %[vector4a], $ac0 \n\t" + "mtlo %[vector4a], $ac1 \n\t" + "mtlo %[vector4a], $ac2 \n\t" + "mtlo %[vector4a], $ac3 \n\t" + "mthi $zero, $ac0 \n\t" + "mthi $zero, $ac1 \n\t" + "mthi $zero, $ac2 \n\t" + "mthi $zero, $ac3 \n\t" + + "preceu.ph.qbr %[scratch1], %[load1] \n\t" + "preceu.ph.qbr %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbr %[scratch2], %[load3] \n\t" + "preceu.ph.qbr %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac0, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac0, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac1, %[n1], %[vector1b] \n\t" + "dpa.w.ph $ac1, %[n2], %[vector2b] \n\t" + + "preceu.ph.qbl %[scratch1], %[load1] \n\t" + "preceu.ph.qbl %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbl %[scratch2], %[load3] \n\t" + "preceu.ph.qbl %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac2, %[p1], %[vector1b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector2b] \n\t" + "dpa.w.ph $ac3, %[n1], %[vector1b] \n\t" + "dpa.w.ph $ac3, %[n2], %[vector2b] \n\t" + + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load1], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load2], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load3], 0(%[src_ptr]) \n\t" + "add %[src_ptr], %[src_ptr], %[src_stride] \n\t" + "ulw %[load4], 0(%[src_ptr]) \n\t" + + "preceu.ph.qbr %[scratch1], %[load1] \n\t" + "preceu.ph.qbr %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbr %[scratch2], %[load3] \n\t" + "preceu.ph.qbr %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "dpa.w.ph $ac0, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac0, %[p2], %[vector4b] \n\t" + "extp %[Temp1], $ac0, 31 \n\t" + "dpa.w.ph $ac1, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac1, %[n2], %[vector4b] \n\t" + "extp %[Temp2], $ac1, 31 \n\t" + + "preceu.ph.qbl %[scratch1], %[load1] \n\t" + "preceu.ph.qbl %[p1], %[load2] \n\t" + "precrq.ph.w %[n1], %[p1], %[scratch1] \n\t" /* pixel 2 */ + "append %[p1], %[scratch1], 16 \n\t" /* pixel 1 */ + "preceu.ph.qbl %[scratch2], %[load3] \n\t" + "preceu.ph.qbl %[p2], %[load4] \n\t" + "precrq.ph.w %[n2], %[p2], %[scratch2] \n\t" /* pixel 2 */ + "append %[p2], %[scratch2], 16 \n\t" /* pixel 1 */ + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "dpa.w.ph $ac2, %[p1], %[vector3b] \n\t" + "dpa.w.ph $ac2, %[p2], %[vector4b] \n\t" + "extp %[Temp1], $ac2, 31 \n\t" + + "lbux %[store2], %[Temp2](%[cm]) \n\t" + "dpa.w.ph $ac3, %[n1], %[vector3b] \n\t" + "dpa.w.ph $ac3, %[n2], %[vector4b] \n\t" + "extp %[Temp2], $ac3, 31 \n\t" + + "sb %[store1], 0(%[dst_ptr]) \n\t" + "sb %[store2], 1(%[dst_ptr]) \n\t" + + "lbux %[store1], %[Temp1](%[cm]) \n\t" + "lbux %[store2], %[Temp2](%[cm]) \n\t" + + "sb %[store1], 2(%[dst_ptr]) \n\t" + "sb %[store2], 3(%[dst_ptr]) \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [p1] "=&r"(p1), [p2] "=&r"(p2), + [n1] "=&r"(n1), [n2] "=&r"(n2), [scratch1] "=&r"(scratch1), + [scratch2] "=&r"(scratch2), [Temp1] "=&r"(Temp1), + [Temp2] "=&r"(Temp2), [store1] "=&r"(store1), + [store2] "=&r"(store2), [src_ptr] "+r"(src_ptr) + : [vector1b] "r"(vector1b), [vector2b] "r"(vector2b), + [vector3b] "r"(vector3b), [vector4b] "r"(vector4b), + [vector4a] "r"(vector4a), [src_stride] "r"(src_stride), + [cm] "r"(cm), [dst_ptr] "r"(dst_ptr)); + } + + /* Next row... */ + src += src_stride; + dst += dst_stride; + } +} + +void aom_convolve8_vert_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + if (((const int32_t *)filter_y)[0] == 0) { + aom_convolve2_vert_dspr2(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + } else { + uint32_t pos = 38; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + prefetch_store(dst); + + switch (w) { + case 4: + case 8: + case 16: + case 32: + convolve_vert_4_dspr2(src, src_stride, dst, dst_stride, filter_y, w, h); + break; + case 64: + prefetch_store(dst + 32); + convolve_vert_64_dspr2(src, src_stride, dst, dst_stride, filter_y, h); + break; + default: + aom_convolve8_vert_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h); + break; + } + } +} + +#endif diff --git a/third_party/aom/aom_dsp/mips/convolve_common_dspr2.h b/third_party/aom/aom_dsp/mips/convolve_common_dspr2.h new file mode 100644 index 0000000000..f8fd9e2b6c --- /dev/null +++ b/third_party/aom/aom_dsp/mips/convolve_common_dspr2.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_MIPS_AOM_COMMON_DSPR2_H_ +#define AOM_DSP_MIPS_AOM_COMMON_DSPR2_H_ + +#include + +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/mips/common_dspr2.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if HAVE_DSPR2 +void aom_convolve2_horiz_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h); + +void aom_convolve2_avg_horiz_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h); + +void aom_convolve2_avg_vert_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h); + +void aom_convolve2_dspr2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter, int w, + int h); + +void aom_convolve2_vert_dspr2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h); + +#endif // #if HAVE_DSPR2 +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_MIPS_AOM_COMMON_DSPR2_H_ diff --git a/third_party/aom/aom_dsp/mips/fwd_dct32x32_msa.c b/third_party/aom/aom_dsp/mips/fwd_dct32x32_msa.c new file mode 100644 index 0000000000..dc9c63226a --- /dev/null +++ b/third_party/aom/aom_dsp/mips/fwd_dct32x32_msa.c @@ -0,0 +1,948 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/fwd_txfm_msa.h" + +static void fdct8x32_1d_column_load_butterfly(const int16_t *input, + int32_t src_stride, + int16_t *temp_buff) { + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + v8i16 step0, step1, step2, step3; + v8i16 in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1; + v8i16 step0_1, step1_1, step2_1, step3_1; + + /* 1st and 2nd set */ + LD_SH4(input, src_stride, in0, in1, in2, in3); + LD_SH4(input + (28 * src_stride), src_stride, in4, in5, in6, in7); + LD_SH4(input + (4 * src_stride), src_stride, in0_1, in1_1, in2_1, in3_1); + LD_SH4(input + (24 * src_stride), src_stride, in4_1, in5_1, in6_1, in7_1); + SLLI_4V(in0, in1, in2, in3, 2); + SLLI_4V(in4, in5, in6, in7, 2); + SLLI_4V(in0_1, in1_1, in2_1, in3_1, 2); + SLLI_4V(in4_1, in5_1, in6_1, in7_1, 2); + BUTTERFLY_8(in0, in1, in2, in3, in4, in5, in6, in7, step0, step1, step2, + step3, in4, in5, in6, in7); + BUTTERFLY_8(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, step0_1, + step1_1, step2_1, step3_1, in4_1, in5_1, in6_1, in7_1); + ST_SH4(step0, step1, step2, step3, temp_buff, 8); + ST_SH4(in4, in5, in6, in7, temp_buff + (28 * 8), 8); + ST_SH4(step0_1, step1_1, step2_1, step3_1, temp_buff + (4 * 8), 8); + ST_SH4(in4_1, in5_1, in6_1, in7_1, temp_buff + (24 * 8), 8); + + /* 3rd and 4th set */ + LD_SH4(input + (8 * src_stride), src_stride, in0, in1, in2, in3); + LD_SH4(input + (20 * src_stride), src_stride, in4, in5, in6, in7); + LD_SH4(input + (12 * src_stride), src_stride, in0_1, in1_1, in2_1, in3_1); + LD_SH4(input + (16 * src_stride), src_stride, in4_1, in5_1, in6_1, in7_1); + SLLI_4V(in0, in1, in2, in3, 2); + SLLI_4V(in4, in5, in6, in7, 2); + SLLI_4V(in0_1, in1_1, in2_1, in3_1, 2); + SLLI_4V(in4_1, in5_1, in6_1, in7_1, 2); + BUTTERFLY_8(in0, in1, in2, in3, in4, in5, in6, in7, step0, step1, step2, + step3, in4, in5, in6, in7); + BUTTERFLY_8(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, step0_1, + step1_1, step2_1, step3_1, in4_1, in5_1, in6_1, in7_1); + ST_SH4(step0, step1, step2, step3, temp_buff + (8 * 8), 8); + ST_SH4(in4, in5, in6, in7, temp_buff + (20 * 8), 8); + ST_SH4(step0_1, step1_1, step2_1, step3_1, temp_buff + (12 * 8), 8); + ST_SH4(in4_1, in5_1, in6_1, in7_1, temp_buff + (15 * 8) + 8, 8); +} + +static void fdct8x32_1d_column_even_store(int16_t *input, int16_t *temp) { + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + v8i16 in8, in9, in10, in11, in12, in13, in14, in15; + v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8i16 temp0, temp1; + + /* fdct even */ + LD_SH4(input, 8, in0, in1, in2, in3); + LD_SH4(input + 96, 8, in12, in13, in14, in15); + BUTTERFLY_8(in0, in1, in2, in3, in12, in13, in14, in15, vec0, vec1, vec2, + vec3, in12, in13, in14, in15); + LD_SH4(input + 32, 8, in4, in5, in6, in7); + LD_SH4(input + 64, 8, in8, in9, in10, in11); + BUTTERFLY_8(in4, in5, in6, in7, in8, in9, in10, in11, vec4, vec5, vec6, vec7, + in8, in9, in10, in11); + + /* Stage 3 */ + ADD4(vec0, vec7, vec1, vec6, vec2, vec5, vec3, vec4, in0, in1, in2, in3); + BUTTERFLY_4(in0, in1, in2, in3, temp0, in4, in1, in0); + DOTP_CONST_PAIR(temp0, in4, cospi_16_64, cospi_16_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + ST_SH(temp0, temp); + ST_SH(temp1, temp + 512); + + DOTP_CONST_PAIR(in0, in1, cospi_24_64, cospi_8_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + ST_SH(temp0, temp + 256); + ST_SH(temp1, temp + 768); + + SUB4(vec0, vec7, vec1, vec6, vec2, vec5, vec3, vec4, vec7, vec6, vec5, vec4); + DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6); + ADD2(vec4, vec5, vec7, vec6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + ST_SH(temp0, temp + 128); + ST_SH(temp1, temp + 896); + + SUB2(vec4, vec5, vec7, vec6, vec4, vec7); + DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + ST_SH(temp0, temp + 640); + ST_SH(temp1, temp + 384); + + DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5); + DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4); + ADD4(in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, vec1, vec6, in2); + DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3); + ADD2(in0, in1, in2, in3, vec0, vec7); + DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + ST_SH(temp0, temp + 64); + ST_SH(temp1, temp + 960); + + SUB2(in0, in1, in2, in3, in0, in2); + DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + ST_SH(temp0, temp + 576); + ST_SH(temp1, temp + 448); + + SUB2(in9, vec2, in14, vec5, vec2, vec5); + DOTP_CONST_PAIR((-vec2), vec5, cospi_24_64, cospi_8_64, in2, in1); + SUB4(in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, vec2, vec5); + DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + ST_SH(temp0, temp + 320); + ST_SH(temp1, temp + 704); + + ADD2(in3, in2, in0, in1, vec3, vec4); + DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, temp0, temp1); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + ST_SH(temp0, temp + 192); + ST_SH(temp1, temp + 832); +} + +static void fdct8x32_1d_column_odd_store(int16_t *input, int16_t *temp_ptr) { + v8i16 in16, in17, in18, in19, in20, in21, in22, in23; + v8i16 in24, in25, in26, in27, in28, in29, in30, in31, vec4, vec5; + + in20 = LD_SH(input + 32); + in21 = LD_SH(input + 40); + in26 = LD_SH(input + 80); + in27 = LD_SH(input + 88); + + DOTP_CONST_PAIR(in27, in20, cospi_16_64, cospi_16_64, in20, in27); + DOTP_CONST_PAIR(in26, in21, cospi_16_64, cospi_16_64, in21, in26); + + in18 = LD_SH(input + 16); + in19 = LD_SH(input + 24); + in28 = LD_SH(input + 96); + in29 = LD_SH(input + 104); + + vec4 = in19 - in20; + ST_SH(vec4, input + 32); + vec4 = in18 - in21; + ST_SH(vec4, input + 40); + vec4 = in29 - in26; + ST_SH(vec4, input + 80); + vec4 = in28 - in27; + ST_SH(vec4, input + 88); + + in21 = in18 + in21; + in20 = in19 + in20; + in27 = in28 + in27; + in26 = in29 + in26; + + LD_SH4(input + 48, 8, in22, in23, in24, in25); + DOTP_CONST_PAIR(in25, in22, cospi_16_64, cospi_16_64, in22, in25); + DOTP_CONST_PAIR(in24, in23, cospi_16_64, cospi_16_64, in23, in24); + + in16 = LD_SH(input); + in17 = LD_SH(input + 8); + in30 = LD_SH(input + 112); + in31 = LD_SH(input + 120); + + vec4 = in17 - in22; + ST_SH(vec4, input + 16); + vec4 = in16 - in23; + ST_SH(vec4, input + 24); + vec4 = in31 - in24; + ST_SH(vec4, input + 96); + vec4 = in30 - in25; + ST_SH(vec4, input + 104); + + ADD4(in16, in23, in17, in22, in30, in25, in31, in24, in16, in17, in30, in31); + DOTP_CONST_PAIR(in26, in21, cospi_24_64, cospi_8_64, in18, in29); + DOTP_CONST_PAIR(in27, in20, cospi_24_64, cospi_8_64, in19, in28); + ADD4(in16, in19, in17, in18, in30, in29, in31, in28, in27, in22, in21, in25); + DOTP_CONST_PAIR(in21, in22, cospi_28_64, cospi_4_64, in26, in24); + ADD2(in27, in26, in25, in24, in23, in20); + DOTP_CONST_PAIR(in20, in23, cospi_31_64, cospi_1_64, vec4, vec5); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + ST_SH(vec5, temp_ptr); + ST_SH(vec4, temp_ptr + 960); + + SUB2(in27, in26, in25, in24, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_15_64, cospi_17_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + ST_SH(vec5, temp_ptr + 448); + ST_SH(vec4, temp_ptr + 512); + + SUB4(in17, in18, in16, in19, in31, in28, in30, in29, in23, in26, in24, in20); + DOTP_CONST_PAIR((-in23), in20, cospi_28_64, cospi_4_64, in27, in25); + SUB2(in26, in27, in24, in25, in23, in20); + DOTP_CONST_PAIR(in20, in23, cospi_23_64, cospi_9_64, vec4, vec5); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + ST_SH(vec4, temp_ptr + 704); + ST_SH(vec5, temp_ptr + 256); + + ADD2(in26, in27, in24, in25, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_7_64, cospi_25_64, vec4, vec5); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + ST_SH(vec4, temp_ptr + 192); + ST_SH(vec5, temp_ptr + 768); + + LD_SH4(input + 16, 8, in22, in23, in20, in21); + LD_SH4(input + 80, 8, in26, in27, in24, in25); + in16 = in20; + in17 = in21; + DOTP_CONST_PAIR(-in16, in27, cospi_24_64, cospi_8_64, in20, in27); + DOTP_CONST_PAIR(-in17, in26, cospi_24_64, cospi_8_64, in21, in26); + SUB4(in23, in20, in22, in21, in25, in26, in24, in27, in28, in17, in18, in31); + DOTP_CONST_PAIR(in18, in17, cospi_12_64, cospi_20_64, in29, in30); + ADD2(in28, in29, in31, in30, in16, in19); + DOTP_CONST_PAIR(in19, in16, cospi_27_64, cospi_5_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + ST_SH(vec5, temp_ptr + 832); + ST_SH(vec4, temp_ptr + 128); + + SUB2(in28, in29, in31, in30, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_11_64, cospi_21_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + ST_SH(vec5, temp_ptr + 320); + ST_SH(vec4, temp_ptr + 640); + ADD4(in22, in21, in23, in20, in24, in27, in25, in26, in16, in29, in30, in19); + DOTP_CONST_PAIR(-in16, in19, cospi_12_64, cospi_20_64, in28, in31); + SUB2(in29, in28, in30, in31, in16, in19); + DOTP_CONST_PAIR(in19, in16, cospi_19_64, cospi_13_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + ST_SH(vec5, temp_ptr + 576); + ST_SH(vec4, temp_ptr + 384); + + ADD2(in29, in28, in30, in31, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_3_64, cospi_29_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + ST_SH(vec5, temp_ptr + 64); + ST_SH(vec4, temp_ptr + 896); +} + +static void fdct8x32_1d_column(const int16_t *input, int32_t src_stride, + int16_t *tmp_buf, int16_t *tmp_buf_big) { + fdct8x32_1d_column_load_butterfly(input, src_stride, tmp_buf); + fdct8x32_1d_column_even_store(tmp_buf, tmp_buf_big); + fdct8x32_1d_column_odd_store(tmp_buf + 128, (tmp_buf_big + 32)); +} + +static void fdct8x32_1d_row_load_butterfly(int16_t *temp_buff, + int16_t *output) { + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + v8i16 in8, in9, in10, in11, in12, in13, in14, in15; + v8i16 step0, step1, step2, step3, step4, step5, step6, step7; + + LD_SH8(temp_buff, 32, in0, in1, in2, in3, in4, in5, in6, in7); + LD_SH8(temp_buff + 24, 32, in8, in9, in10, in11, in12, in13, in14, in15); + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + TRANSPOSE8x8_SH_SH(in8, in9, in10, in11, in12, in13, in14, in15, in8, in9, + in10, in11, in12, in13, in14, in15); + BUTTERFLY_16(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11, + in12, in13, in14, in15, step0, step1, step2, step3, step4, step5, + step6, step7, in8, in9, in10, in11, in12, in13, in14, in15); + ST_SH8(step0, step1, step2, step3, step4, step5, step6, step7, output, 8); + ST_SH8(in8, in9, in10, in11, in12, in13, in14, in15, (output + 24 * 8), 8); + + /* 2nd set */ + LD_SH8(temp_buff + 8, 32, in0, in1, in2, in3, in4, in5, in6, in7); + LD_SH8(temp_buff + 16, 32, in8, in9, in10, in11, in12, in13, in14, in15); + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + TRANSPOSE8x8_SH_SH(in8, in9, in10, in11, in12, in13, in14, in15, in8, in9, + in10, in11, in12, in13, in14, in15); + BUTTERFLY_16(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11, + in12, in13, in14, in15, step0, step1, step2, step3, step4, step5, + step6, step7, in8, in9, in10, in11, in12, in13, in14, in15); + ST_SH8(step0, step1, step2, step3, step4, step5, step6, step7, + (output + 8 * 8), 8); + ST_SH8(in8, in9, in10, in11, in12, in13, in14, in15, (output + 16 * 8), 8); +} + +static void fdct8x32_1d_row_even_4x(int16_t *input, int16_t *interm_ptr, + int16_t *out) { + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + v8i16 in8, in9, in10, in11, in12, in13, in14, in15; + v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v4i32 vec0_l, vec1_l, vec2_l, vec3_l, vec4_l, vec5_l, vec6_l, vec7_l; + v4i32 vec0_r, vec1_r, vec2_r, vec3_r, vec4_r, vec5_r, vec6_r, vec7_r; + v4i32 tmp0_w, tmp1_w, tmp2_w, tmp3_w; + + /* fdct32 even */ + /* stage 2 */ + LD_SH8(input, 8, in0, in1, in2, in3, in4, in5, in6, in7); + LD_SH8(input + 64, 8, in8, in9, in10, in11, in12, in13, in14, in15); + + BUTTERFLY_16(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11, + in12, in13, in14, in15, vec0, vec1, vec2, vec3, vec4, vec5, vec6, + vec7, in8, in9, in10, in11, in12, in13, in14, in15); + ST_SH8(vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, interm_ptr, 8); + ST_SH8(in8, in9, in10, in11, in12, in13, in14, in15, interm_ptr + 64, 8); + + /* Stage 3 */ + UNPCK_SH_SW(vec0, vec0_l, vec0_r); + UNPCK_SH_SW(vec1, vec1_l, vec1_r); + UNPCK_SH_SW(vec2, vec2_l, vec2_r); + UNPCK_SH_SW(vec3, vec3_l, vec3_r); + UNPCK_SH_SW(vec4, vec4_l, vec4_r); + UNPCK_SH_SW(vec5, vec5_l, vec5_r); + UNPCK_SH_SW(vec6, vec6_l, vec6_r); + UNPCK_SH_SW(vec7, vec7_l, vec7_r); + ADD4(vec0_r, vec7_r, vec1_r, vec6_r, vec2_r, vec5_r, vec3_r, vec4_r, tmp0_w, + tmp1_w, tmp2_w, tmp3_w); + BUTTERFLY_4(tmp0_w, tmp1_w, tmp2_w, tmp3_w, vec4_r, vec6_r, vec7_r, vec5_r); + ADD4(vec0_l, vec7_l, vec1_l, vec6_l, vec2_l, vec5_l, vec3_l, vec4_l, vec0_r, + vec1_r, vec2_r, vec3_r); + + tmp3_w = vec0_r + vec3_r; + vec0_r = vec0_r - vec3_r; + vec3_r = vec1_r + vec2_r; + vec1_r = vec1_r - vec2_r; + + DOTP_CONST_PAIR_W(vec4_r, vec6_r, tmp3_w, vec3_r, cospi_16_64, cospi_16_64, + vec4_r, tmp3_w, vec6_r, vec3_r); + FDCT32_POSTPROC_NEG_W(vec4_r); + FDCT32_POSTPROC_NEG_W(tmp3_w); + FDCT32_POSTPROC_NEG_W(vec6_r); + FDCT32_POSTPROC_NEG_W(vec3_r); + PCKEV_H2_SH(vec4_r, tmp3_w, vec6_r, vec3_r, vec4, vec5); + ST_SH2(vec5, vec4, out, 8); + + DOTP_CONST_PAIR_W(vec5_r, vec7_r, vec0_r, vec1_r, cospi_24_64, cospi_8_64, + vec4_r, tmp3_w, vec6_r, vec3_r); + FDCT32_POSTPROC_NEG_W(vec4_r); + FDCT32_POSTPROC_NEG_W(tmp3_w); + FDCT32_POSTPROC_NEG_W(vec6_r); + FDCT32_POSTPROC_NEG_W(vec3_r); + PCKEV_H2_SH(vec4_r, tmp3_w, vec6_r, vec3_r, vec4, vec5); + ST_SH2(vec5, vec4, out + 16, 8); + + LD_SH8(interm_ptr, 8, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7); + SUB4(vec3, vec4, vec2, vec5, vec1, vec6, vec0, vec7, vec4, vec5, vec6, vec7); + DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6); + ADD2(vec4, vec5, vec7, vec6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + ST_SH(in4, out + 32); + ST_SH(in5, out + 56); + + SUB2(vec4, vec5, vec7, vec6, vec4, vec7); + DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + ST_SH(in4, out + 40); + ST_SH(in5, out + 48); + + LD_SH8(interm_ptr + 64, 8, in8, in9, in10, in11, in12, in13, in14, in15); + DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5); + DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4); + ADD4(in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, vec1, vec6, in2); + DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3); + ADD2(in0, in1, in2, in3, vec0, vec7); + DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + ST_SH(in4, out + 64); + ST_SH(in5, out + 120); + + SUB2(in0, in1, in2, in3, in0, in2); + DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + ST_SH(in4, out + 72); + ST_SH(in5, out + 112); + + SUB2(in9, vec2, in14, vec5, vec2, vec5); + DOTP_CONST_PAIR((-vec2), vec5, cospi_24_64, cospi_8_64, in2, in1); + SUB4(in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, vec2, vec5); + DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + ST_SH(in4, out + 80); + ST_SH(in5, out + 104); + + ADD2(in3, in2, in0, in1, vec3, vec4); + DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, in4, in5); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + ST_SH(in4, out + 96); + ST_SH(in5, out + 88); +} + +static void fdct8x32_1d_row_even(int16_t *temp, int16_t *out) { + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + v8i16 in8, in9, in10, in11, in12, in13, in14, in15; + v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, temp0, temp1; + + /* fdct32 even */ + /* stage 2 */ + LD_SH8(temp, 8, in0, in1, in2, in3, in4, in5, in6, in7); + LD_SH8(temp + 64, 8, in8, in9, in10, in11, in12, in13, in14, in15); + + BUTTERFLY_16(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11, + in12, in13, in14, in15, vec0, vec1, vec2, vec3, vec4, vec5, vec6, + vec7, in8, in9, in10, in11, in12, in13, in14, in15); + + /* Stage 3 */ + ADD4(vec0, vec7, vec1, vec6, vec2, vec5, vec3, vec4, in0, in1, in2, in3); + BUTTERFLY_4(in0, in1, in2, in3, temp0, in4, in1, in0); + DOTP_CONST_PAIR(temp0, in4, cospi_16_64, cospi_16_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + ST_SH(temp0, out); + ST_SH(temp1, out + 8); + + DOTP_CONST_PAIR(in0, in1, cospi_24_64, cospi_8_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + ST_SH(temp0, out + 16); + ST_SH(temp1, out + 24); + + SUB4(vec3, vec4, vec2, vec5, vec1, vec6, vec0, vec7, vec4, vec5, vec6, vec7); + DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6); + ADD2(vec4, vec5, vec7, vec6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + ST_SH(temp0, out + 32); + ST_SH(temp1, out + 56); + + SUB2(vec4, vec5, vec7, vec6, vec4, vec7); + DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + ST_SH(temp0, out + 40); + ST_SH(temp1, out + 48); + + DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5); + DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4); + ADD4(in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, vec1, vec6, in2); + DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3); + ADD2(in0, in1, in2, in3, vec0, vec7); + DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + ST_SH(temp0, out + 64); + ST_SH(temp1, out + 120); + + SUB2(in0, in1, in2, in3, in0, in2); + DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + ST_SH(temp0, out + 72); + ST_SH(temp1, out + 112); + + SUB2(in9, vec2, in14, vec5, vec2, vec5); + DOTP_CONST_PAIR((-vec2), vec5, cospi_24_64, cospi_8_64, in2, in1); + SUB4(in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, vec2, vec5) + DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + ST_SH(temp0, out + 80); + ST_SH(temp1, out + 104); + + ADD2(in3, in2, in0, in1, vec3, vec4); + DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, temp0, temp1); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + ST_SH(temp0, out + 96); + ST_SH(temp1, out + 88); +} + +static void fdct8x32_1d_row_odd(int16_t *temp, int16_t *interm_ptr, + int16_t *out) { + v8i16 in16, in17, in18, in19, in20, in21, in22, in23; + v8i16 in24, in25, in26, in27, in28, in29, in30, in31, vec4, vec5; + + in20 = LD_SH(temp + 32); + in21 = LD_SH(temp + 40); + in26 = LD_SH(temp + 80); + in27 = LD_SH(temp + 88); + + DOTP_CONST_PAIR(in27, in20, cospi_16_64, cospi_16_64, in20, in27); + DOTP_CONST_PAIR(in26, in21, cospi_16_64, cospi_16_64, in21, in26); + + in18 = LD_SH(temp + 16); + in19 = LD_SH(temp + 24); + in28 = LD_SH(temp + 96); + in29 = LD_SH(temp + 104); + + vec4 = in19 - in20; + ST_SH(vec4, interm_ptr + 32); + vec4 = in18 - in21; + ST_SH(vec4, interm_ptr + 88); + vec4 = in28 - in27; + ST_SH(vec4, interm_ptr + 56); + vec4 = in29 - in26; + ST_SH(vec4, interm_ptr + 64); + + ADD4(in18, in21, in19, in20, in28, in27, in29, in26, in21, in20, in27, in26); + + in22 = LD_SH(temp + 48); + in23 = LD_SH(temp + 56); + in24 = LD_SH(temp + 64); + in25 = LD_SH(temp + 72); + + DOTP_CONST_PAIR(in25, in22, cospi_16_64, cospi_16_64, in22, in25); + DOTP_CONST_PAIR(in24, in23, cospi_16_64, cospi_16_64, in23, in24); + + in16 = LD_SH(temp); + in17 = LD_SH(temp + 8); + in30 = LD_SH(temp + 112); + in31 = LD_SH(temp + 120); + + vec4 = in17 - in22; + ST_SH(vec4, interm_ptr + 40); + vec4 = in30 - in25; + ST_SH(vec4, interm_ptr + 48); + vec4 = in31 - in24; + ST_SH(vec4, interm_ptr + 72); + vec4 = in16 - in23; + ST_SH(vec4, interm_ptr + 80); + + ADD4(in16, in23, in17, in22, in30, in25, in31, in24, in16, in17, in30, in31); + DOTP_CONST_PAIR(in26, in21, cospi_24_64, cospi_8_64, in18, in29); + DOTP_CONST_PAIR(in27, in20, cospi_24_64, cospi_8_64, in19, in28); + + ADD4(in16, in19, in17, in18, in30, in29, in31, in28, in27, in22, in21, in25); + DOTP_CONST_PAIR(in21, in22, cospi_28_64, cospi_4_64, in26, in24); + ADD2(in27, in26, in25, in24, in23, in20); + + DOTP_CONST_PAIR(in20, in23, cospi_31_64, cospi_1_64, vec4, vec5); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + ST_SH(vec5, out); + ST_SH(vec4, out + 120); + + SUB2(in27, in26, in25, in24, in22, in21); + + DOTP_CONST_PAIR(in21, in22, cospi_15_64, cospi_17_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + ST_SH(vec5, out + 112); + ST_SH(vec4, out + 8); + + SUB4(in17, in18, in16, in19, in31, in28, in30, in29, in23, in26, in24, in20); + DOTP_CONST_PAIR((-in23), in20, cospi_28_64, cospi_4_64, in27, in25); + SUB2(in26, in27, in24, in25, in23, in20); + + DOTP_CONST_PAIR(in20, in23, cospi_23_64, cospi_9_64, vec4, vec5); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + ST_SH(vec4, out + 16); + ST_SH(vec5, out + 104); + + ADD2(in26, in27, in24, in25, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_7_64, cospi_25_64, vec4, vec5); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + ST_SH(vec4, out + 24); + ST_SH(vec5, out + 96); + + in20 = LD_SH(interm_ptr + 32); + in21 = LD_SH(interm_ptr + 88); + in27 = LD_SH(interm_ptr + 56); + in26 = LD_SH(interm_ptr + 64); + + in16 = in20; + in17 = in21; + DOTP_CONST_PAIR(-in16, in27, cospi_24_64, cospi_8_64, in20, in27); + DOTP_CONST_PAIR(-in17, in26, cospi_24_64, cospi_8_64, in21, in26); + + in22 = LD_SH(interm_ptr + 40); + in25 = LD_SH(interm_ptr + 48); + in24 = LD_SH(interm_ptr + 72); + in23 = LD_SH(interm_ptr + 80); + + SUB4(in23, in20, in22, in21, in25, in26, in24, in27, in28, in17, in18, in31); + DOTP_CONST_PAIR(in18, in17, cospi_12_64, cospi_20_64, in29, in30); + ADD2(in28, in29, in31, in30, in16, in19); + DOTP_CONST_PAIR(in19, in16, cospi_27_64, cospi_5_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + ST_SH(vec5, out + 32); + ST_SH(vec4, out + 88); + + SUB2(in28, in29, in31, in30, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_11_64, cospi_21_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + ST_SH(vec5, out + 40); + ST_SH(vec4, out + 80); + + ADD4(in22, in21, in23, in20, in24, in27, in25, in26, in16, in29, in30, in19); + DOTP_CONST_PAIR(-in16, in19, cospi_12_64, cospi_20_64, in28, in31); + SUB2(in29, in28, in30, in31, in16, in19); + + DOTP_CONST_PAIR(in19, in16, cospi_19_64, cospi_13_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + ST_SH(vec5, out + 72); + ST_SH(vec4, out + 48); + + ADD2(in29, in28, in30, in31, in17, in18); + + DOTP_CONST_PAIR(in18, in17, cospi_3_64, cospi_29_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + ST_SH(vec4, out + 56); + ST_SH(vec5, out + 64); +} + +static void fdct8x32_1d_row_transpose_store(int16_t *temp, int16_t *output) { + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + v8i16 in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1; + + /* 1st set */ + in0 = LD_SH(temp); + in4 = LD_SH(temp + 32); + in2 = LD_SH(temp + 64); + in6 = LD_SH(temp + 96); + in1 = LD_SH(temp + 128); + in7 = LD_SH(temp + 152); + in3 = LD_SH(temp + 192); + in5 = LD_SH(temp + 216); + + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + + /* 2nd set */ + in0_1 = LD_SH(temp + 16); + in1_1 = LD_SH(temp + 232); + in2_1 = LD_SH(temp + 80); + in3_1 = LD_SH(temp + 168); + in4_1 = LD_SH(temp + 48); + in5_1 = LD_SH(temp + 176); + in6_1 = LD_SH(temp + 112); + in7_1 = LD_SH(temp + 240); + + ST_SH8(in0, in1, in2, in3, in4, in5, in6, in7, output, 32); + TRANSPOSE8x8_SH_SH(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, + in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1); + + /* 3rd set */ + in0 = LD_SH(temp + 8); + in1 = LD_SH(temp + 136); + in2 = LD_SH(temp + 72); + in3 = LD_SH(temp + 200); + in4 = LD_SH(temp + 40); + in5 = LD_SH(temp + 208); + in6 = LD_SH(temp + 104); + in7 = LD_SH(temp + 144); + + ST_SH8(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, output + 8, + 32); + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + ST_SH8(in0, in1, in2, in3, in4, in5, in6, in7, output + 16, 32); + + /* 4th set */ + in0_1 = LD_SH(temp + 24); + in1_1 = LD_SH(temp + 224); + in2_1 = LD_SH(temp + 88); + in3_1 = LD_SH(temp + 160); + in4_1 = LD_SH(temp + 56); + in5_1 = LD_SH(temp + 184); + in6_1 = LD_SH(temp + 120); + in7_1 = LD_SH(temp + 248); + + TRANSPOSE8x8_SH_SH(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, + in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1); + ST_SH8(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, output + 24, + 32); +} + +static void fdct32x8_1d_row(int16_t *temp, int16_t *temp_buf, int16_t *output) { + fdct8x32_1d_row_load_butterfly(temp, temp_buf); + fdct8x32_1d_row_even(temp_buf, temp_buf); + fdct8x32_1d_row_odd(temp_buf + 128, temp, temp_buf + 128); + fdct8x32_1d_row_transpose_store(temp_buf, output); +} + +static void fdct32x8_1d_row_4x(int16_t *tmp_buf_big, int16_t *tmp_buf, + int16_t *output) { + fdct8x32_1d_row_load_butterfly(tmp_buf_big, tmp_buf); + fdct8x32_1d_row_even_4x(tmp_buf, tmp_buf_big, tmp_buf); + fdct8x32_1d_row_odd(tmp_buf + 128, tmp_buf_big, tmp_buf + 128); + fdct8x32_1d_row_transpose_store(tmp_buf, output); +} + +void aom_fdct32x32_msa(const int16_t *input, int16_t *output, + int32_t src_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, tmp_buf_big[1024]); + DECLARE_ALIGNED(32, int16_t, tmp_buf[256]); + + /* column transform */ + for (i = 0; i < 4; ++i) { + fdct8x32_1d_column(input + (8 * i), src_stride, tmp_buf, + tmp_buf_big + (8 * i)); + } + + /* row transform */ + fdct32x8_1d_row_4x(tmp_buf_big, tmp_buf, output); + + /* row transform */ + for (i = 1; i < 4; ++i) { + fdct32x8_1d_row(tmp_buf_big + (i * 256), tmp_buf, output + (i * 256)); + } +} + +static void fdct8x32_1d_row_even_rd(int16_t *temp, int16_t *out) { + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + v8i16 in8, in9, in10, in11, in12, in13, in14, in15; + v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, temp0, temp1; + + /* fdct32 even */ + /* stage 2 */ + LD_SH8(temp, 8, in0, in1, in2, in3, in4, in5, in6, in7); + LD_SH8(temp + 64, 8, in8, in9, in10, in11, in12, in13, in14, in15); + + BUTTERFLY_16(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11, + in12, in13, in14, in15, vec0, vec1, vec2, vec3, vec4, vec5, vec6, + vec7, in8, in9, in10, in11, in12, in13, in14, in15); + FDCT_POSTPROC_2V_NEG_H(vec0, vec1); + FDCT_POSTPROC_2V_NEG_H(vec2, vec3); + FDCT_POSTPROC_2V_NEG_H(vec4, vec5); + FDCT_POSTPROC_2V_NEG_H(vec6, vec7); + FDCT_POSTPROC_2V_NEG_H(in8, in9); + FDCT_POSTPROC_2V_NEG_H(in10, in11); + FDCT_POSTPROC_2V_NEG_H(in12, in13); + FDCT_POSTPROC_2V_NEG_H(in14, in15); + + /* Stage 3 */ + ADD4(vec0, vec7, vec1, vec6, vec2, vec5, vec3, vec4, in0, in1, in2, in3); + + temp0 = in0 + in3; + in0 = in0 - in3; + in3 = in1 + in2; + in1 = in1 - in2; + + DOTP_CONST_PAIR(temp0, in3, cospi_16_64, cospi_16_64, temp1, temp0); + ST_SH(temp0, out); + ST_SH(temp1, out + 8); + + DOTP_CONST_PAIR(in0, in1, cospi_24_64, cospi_8_64, temp1, temp0); + ST_SH(temp0, out + 16); + ST_SH(temp1, out + 24); + + SUB4(vec3, vec4, vec2, vec5, vec1, vec6, vec0, vec7, vec4, vec5, vec6, vec7); + DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6); + ADD2(vec4, vec5, vec7, vec6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, temp1, temp0); + ST_SH(temp0, out + 32); + ST_SH(temp1, out + 56); + + SUB2(vec4, vec5, vec7, vec6, vec4, vec7); + DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, temp1, temp0); + ST_SH(temp0, out + 40); + ST_SH(temp1, out + 48); + + DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5); + DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4); + ADD4(in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, vec1, vec6, in2); + DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3); + ADD2(in0, in1, in2, in3, vec0, vec7); + DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, temp1, temp0); + ST_SH(temp0, out + 64); + ST_SH(temp1, out + 120); + + SUB2(in0, in1, in2, in3, in0, in2); + DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, temp1, temp0); + ST_SH(temp0, out + 72); + ST_SH(temp1, out + 112); + + SUB2(in9, vec2, in14, vec5, vec2, vec5); + DOTP_CONST_PAIR((-vec2), vec5, cospi_24_64, cospi_8_64, in2, in1); + SUB4(in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, vec2, vec5); + DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, temp1, temp0); + ST_SH(temp0, out + 80); + ST_SH(temp1, out + 104); + + ADD2(in3, in2, in0, in1, vec3, vec4); + DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, temp0, temp1); + ST_SH(temp0, out + 96); + ST_SH(temp1, out + 88); +} + +static void fdct8x32_1d_row_odd_rd(int16_t *temp, int16_t *interm_ptr, + int16_t *out) { + v8i16 in16, in17, in18, in19, in20, in21, in22, in23; + v8i16 in24, in25, in26, in27, in28, in29, in30, in31; + v8i16 vec4, vec5; + + in20 = LD_SH(temp + 32); + in21 = LD_SH(temp + 40); + in26 = LD_SH(temp + 80); + in27 = LD_SH(temp + 88); + + DOTP_CONST_PAIR(in27, in20, cospi_16_64, cospi_16_64, in20, in27); + DOTP_CONST_PAIR(in26, in21, cospi_16_64, cospi_16_64, in21, in26); + + FDCT_POSTPROC_2V_NEG_H(in20, in21); + FDCT_POSTPROC_2V_NEG_H(in26, in27); + + in18 = LD_SH(temp + 16); + in19 = LD_SH(temp + 24); + in28 = LD_SH(temp + 96); + in29 = LD_SH(temp + 104); + + FDCT_POSTPROC_2V_NEG_H(in18, in19); + FDCT_POSTPROC_2V_NEG_H(in28, in29); + + vec4 = in19 - in20; + ST_SH(vec4, interm_ptr + 32); + vec4 = in18 - in21; + ST_SH(vec4, interm_ptr + 88); + vec4 = in29 - in26; + ST_SH(vec4, interm_ptr + 64); + vec4 = in28 - in27; + ST_SH(vec4, interm_ptr + 56); + + ADD4(in18, in21, in19, in20, in28, in27, in29, in26, in21, in20, in27, in26); + + in22 = LD_SH(temp + 48); + in23 = LD_SH(temp + 56); + in24 = LD_SH(temp + 64); + in25 = LD_SH(temp + 72); + + DOTP_CONST_PAIR(in25, in22, cospi_16_64, cospi_16_64, in22, in25); + DOTP_CONST_PAIR(in24, in23, cospi_16_64, cospi_16_64, in23, in24); + FDCT_POSTPROC_2V_NEG_H(in22, in23); + FDCT_POSTPROC_2V_NEG_H(in24, in25); + + in16 = LD_SH(temp); + in17 = LD_SH(temp + 8); + in30 = LD_SH(temp + 112); + in31 = LD_SH(temp + 120); + + FDCT_POSTPROC_2V_NEG_H(in16, in17); + FDCT_POSTPROC_2V_NEG_H(in30, in31); + + vec4 = in17 - in22; + ST_SH(vec4, interm_ptr + 40); + vec4 = in30 - in25; + ST_SH(vec4, interm_ptr + 48); + vec4 = in31 - in24; + ST_SH(vec4, interm_ptr + 72); + vec4 = in16 - in23; + ST_SH(vec4, interm_ptr + 80); + + ADD4(in16, in23, in17, in22, in30, in25, in31, in24, in16, in17, in30, in31); + DOTP_CONST_PAIR(in26, in21, cospi_24_64, cospi_8_64, in18, in29); + DOTP_CONST_PAIR(in27, in20, cospi_24_64, cospi_8_64, in19, in28); + ADD4(in16, in19, in17, in18, in30, in29, in31, in28, in27, in22, in21, in25); + DOTP_CONST_PAIR(in21, in22, cospi_28_64, cospi_4_64, in26, in24); + ADD2(in27, in26, in25, in24, in23, in20); + DOTP_CONST_PAIR(in20, in23, cospi_31_64, cospi_1_64, vec4, vec5); + ST_SH(vec5, out); + ST_SH(vec4, out + 120); + + SUB2(in27, in26, in25, in24, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_15_64, cospi_17_64, vec5, vec4); + ST_SH(vec5, out + 112); + ST_SH(vec4, out + 8); + + SUB4(in17, in18, in16, in19, in31, in28, in30, in29, in23, in26, in24, in20); + DOTP_CONST_PAIR((-in23), in20, cospi_28_64, cospi_4_64, in27, in25); + SUB2(in26, in27, in24, in25, in23, in20); + DOTP_CONST_PAIR(in20, in23, cospi_23_64, cospi_9_64, vec4, vec5); + ST_SH(vec4, out + 16); + ST_SH(vec5, out + 104); + + ADD2(in26, in27, in24, in25, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_7_64, cospi_25_64, vec4, vec5); + ST_SH(vec4, out + 24); + ST_SH(vec5, out + 96); + + in20 = LD_SH(interm_ptr + 32); + in21 = LD_SH(interm_ptr + 88); + in27 = LD_SH(interm_ptr + 56); + in26 = LD_SH(interm_ptr + 64); + + in16 = in20; + in17 = in21; + DOTP_CONST_PAIR(-in16, in27, cospi_24_64, cospi_8_64, in20, in27); + DOTP_CONST_PAIR(-in17, in26, cospi_24_64, cospi_8_64, in21, in26); + + in22 = LD_SH(interm_ptr + 40); + in25 = LD_SH(interm_ptr + 48); + in24 = LD_SH(interm_ptr + 72); + in23 = LD_SH(interm_ptr + 80); + + SUB4(in23, in20, in22, in21, in25, in26, in24, in27, in28, in17, in18, in31); + DOTP_CONST_PAIR(in18, in17, cospi_12_64, cospi_20_64, in29, in30); + in16 = in28 + in29; + in19 = in31 + in30; + DOTP_CONST_PAIR(in19, in16, cospi_27_64, cospi_5_64, vec5, vec4); + ST_SH(vec5, out + 32); + ST_SH(vec4, out + 88); + + SUB2(in28, in29, in31, in30, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_11_64, cospi_21_64, vec5, vec4); + ST_SH(vec5, out + 40); + ST_SH(vec4, out + 80); + + ADD4(in22, in21, in23, in20, in24, in27, in25, in26, in16, in29, in30, in19); + DOTP_CONST_PAIR(-in16, in19, cospi_12_64, cospi_20_64, in28, in31); + SUB2(in29, in28, in30, in31, in16, in19); + DOTP_CONST_PAIR(in19, in16, cospi_19_64, cospi_13_64, vec5, vec4); + ST_SH(vec5, out + 72); + ST_SH(vec4, out + 48); + + ADD2(in29, in28, in30, in31, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_3_64, cospi_29_64, vec5, vec4); + ST_SH(vec4, out + 56); + ST_SH(vec5, out + 64); +} + +static void fdct32x8_1d_row_rd(int16_t *tmp_buf_big, int16_t *tmp_buf, + int16_t *output) { + fdct8x32_1d_row_load_butterfly(tmp_buf_big, tmp_buf); + fdct8x32_1d_row_even_rd(tmp_buf, tmp_buf); + fdct8x32_1d_row_odd_rd((tmp_buf + 128), tmp_buf_big, (tmp_buf + 128)); + fdct8x32_1d_row_transpose_store(tmp_buf, output); +} + +void aom_fdct32x32_rd_msa(const int16_t *input, int16_t *out, + int32_t src_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, tmp_buf_big[1024]); + DECLARE_ALIGNED(32, int16_t, tmp_buf[256]); + + /* column transform */ + for (i = 0; i < 4; ++i) { + fdct8x32_1d_column(input + (8 * i), src_stride, &tmp_buf[0], + &tmp_buf_big[0] + (8 * i)); + } + + /* row transform */ + for (i = 0; i < 4; ++i) { + fdct32x8_1d_row_rd(&tmp_buf_big[0] + (8 * i * 32), &tmp_buf[0], + out + (8 * i * 32)); + } +} + +void aom_fdct32x32_1_msa(const int16_t *input, int16_t *out, int32_t stride) { + int sum = LD_HADD(input, stride); + sum += LD_HADD(input + 8, stride); + sum += LD_HADD(input + 16, stride); + sum += LD_HADD(input + 24, stride); + sum += LD_HADD(input + 32 * 8, stride); + sum += LD_HADD(input + 32 * 8 + 8, stride); + sum += LD_HADD(input + 32 * 8 + 16, stride); + sum += LD_HADD(input + 32 * 8 + 24, stride); + sum += LD_HADD(input + 32 * 16, stride); + sum += LD_HADD(input + 32 * 16 + 8, stride); + sum += LD_HADD(input + 32 * 16 + 16, stride); + sum += LD_HADD(input + 32 * 16 + 24, stride); + sum += LD_HADD(input + 32 * 24, stride); + sum += LD_HADD(input + 32 * 24 + 8, stride); + sum += LD_HADD(input + 32 * 24 + 16, stride); + sum += LD_HADD(input + 32 * 24 + 24, stride); + out[0] = (int16_t)(sum >> 3); +} diff --git a/third_party/aom/aom_dsp/mips/fwd_txfm_msa.c b/third_party/aom/aom_dsp/mips/fwd_txfm_msa.c new file mode 100644 index 0000000000..f16d290c82 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/fwd_txfm_msa.c @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/fwd_txfm_msa.h" + +void fdct8x16_1d_column(const int16_t *input, int16_t *tmp_ptr, + int32_t src_stride) { + v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + v8i16 in8, in9, in10, in11, in12, in13, in14, in15; + v8i16 stp21, stp22, stp23, stp24, stp25, stp26, stp30; + v8i16 stp31, stp32, stp33, stp34, stp35, stp36, stp37; + v8i16 vec0, vec1, vec2, vec3, vec4, vec5, cnst0, cnst1, cnst4, cnst5; + v8i16 coeff = { cospi_16_64, -cospi_16_64, cospi_8_64, cospi_24_64, + -cospi_8_64, -cospi_24_64, cospi_12_64, cospi_20_64 }; + v8i16 coeff1 = { cospi_2_64, cospi_30_64, cospi_14_64, cospi_18_64, + cospi_10_64, cospi_22_64, cospi_6_64, cospi_26_64 }; + v8i16 coeff2 = { + -cospi_2_64, -cospi_10_64, -cospi_18_64, -cospi_26_64, 0, 0, 0, 0 + }; + + LD_SH16(input, src_stride, in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, + in10, in11, in12, in13, in14, in15); + SLLI_4V(in0, in1, in2, in3, 2); + SLLI_4V(in4, in5, in6, in7, 2); + SLLI_4V(in8, in9, in10, in11, 2); + SLLI_4V(in12, in13, in14, in15, 2); + ADD4(in0, in15, in1, in14, in2, in13, in3, in12, tmp0, tmp1, tmp2, tmp3); + ADD4(in4, in11, in5, in10, in6, in9, in7, in8, tmp4, tmp5, tmp6, tmp7); + FDCT8x16_EVEN(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp0, tmp1, + tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + ST_SH8(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp_ptr, 32); + SUB4(in0, in15, in1, in14, in2, in13, in3, in12, in15, in14, in13, in12); + SUB4(in4, in11, in5, in10, in6, in9, in7, in8, in11, in10, in9, in8); + + tmp_ptr += 16; + + /* stp 1 */ + ILVL_H2_SH(in10, in13, in11, in12, vec2, vec4); + ILVR_H2_SH(in10, in13, in11, in12, vec3, vec5); + + cnst4 = __msa_splati_h(coeff, 0); + stp25 = DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst4); + + cnst5 = __msa_splati_h(coeff, 1); + cnst5 = __msa_ilvev_h(cnst5, cnst4); + stp22 = DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst5); + stp24 = DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst4); + stp23 = DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst5); + + /* stp2 */ + BUTTERFLY_4(in8, in9, stp22, stp23, stp30, stp31, stp32, stp33); + BUTTERFLY_4(in15, in14, stp25, stp24, stp37, stp36, stp35, stp34); + ILVL_H2_SH(stp36, stp31, stp35, stp32, vec2, vec4); + ILVR_H2_SH(stp36, stp31, stp35, stp32, vec3, vec5); + SPLATI_H2_SH(coeff, 2, 3, cnst0, cnst1); + cnst0 = __msa_ilvev_h(cnst0, cnst1); + stp26 = DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst0); + + cnst0 = __msa_splati_h(coeff, 4); + cnst1 = __msa_ilvev_h(cnst1, cnst0); + stp21 = DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst1); + + BUTTERFLY_4(stp30, stp37, stp26, stp21, in8, in15, in14, in9); + ILVRL_H2_SH(in15, in8, vec1, vec0); + SPLATI_H2_SH(coeff1, 0, 1, cnst0, cnst1); + cnst0 = __msa_ilvev_h(cnst0, cnst1); + + in8 = DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0); + ST_SH(in8, tmp_ptr); + + cnst0 = __msa_splati_h(coeff2, 0); + cnst0 = __msa_ilvev_h(cnst1, cnst0); + in8 = DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0); + ST_SH(in8, tmp_ptr + 224); + + ILVRL_H2_SH(in14, in9, vec1, vec0); + SPLATI_H2_SH(coeff1, 2, 3, cnst0, cnst1); + cnst1 = __msa_ilvev_h(cnst1, cnst0); + + in8 = DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst1); + ST_SH(in8, tmp_ptr + 128); + + cnst1 = __msa_splati_h(coeff2, 2); + cnst0 = __msa_ilvev_h(cnst0, cnst1); + in8 = DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0); + ST_SH(in8, tmp_ptr + 96); + + SPLATI_H2_SH(coeff, 2, 5, cnst0, cnst1); + cnst1 = __msa_ilvev_h(cnst1, cnst0); + + stp25 = DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst1); + + cnst1 = __msa_splati_h(coeff, 3); + cnst1 = __msa_ilvev_h(cnst0, cnst1); + stp22 = DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst1); + + /* stp4 */ + ADD2(stp34, stp25, stp33, stp22, in13, in10); + + ILVRL_H2_SH(in13, in10, vec1, vec0); + SPLATI_H2_SH(coeff1, 4, 5, cnst0, cnst1); + cnst0 = __msa_ilvev_h(cnst0, cnst1); + in8 = DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0); + ST_SH(in8, tmp_ptr + 64); + + cnst0 = __msa_splati_h(coeff2, 1); + cnst0 = __msa_ilvev_h(cnst1, cnst0); + in8 = DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0); + ST_SH(in8, tmp_ptr + 160); + + SUB2(stp34, stp25, stp33, stp22, in12, in11); + ILVRL_H2_SH(in12, in11, vec1, vec0); + SPLATI_H2_SH(coeff1, 6, 7, cnst0, cnst1); + cnst1 = __msa_ilvev_h(cnst1, cnst0); + + in8 = DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst1); + ST_SH(in8, tmp_ptr + 192); + + cnst1 = __msa_splati_h(coeff2, 3); + cnst0 = __msa_ilvev_h(cnst0, cnst1); + in8 = DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0); + ST_SH(in8, tmp_ptr + 32); +} + +void fdct16x8_1d_row(int16_t *input, int16_t *output) { + v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + v8i16 in8, in9, in10, in11, in12, in13, in14, in15; + + LD_SH8(input, 16, in0, in1, in2, in3, in4, in5, in6, in7); + LD_SH8((input + 8), 16, in8, in9, in10, in11, in12, in13, in14, in15); + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + TRANSPOSE8x8_SH_SH(in8, in9, in10, in11, in12, in13, in14, in15, in8, in9, + in10, in11, in12, in13, in14, in15); + ADD4(in0, 1, in1, 1, in2, 1, in3, 1, in0, in1, in2, in3); + ADD4(in4, 1, in5, 1, in6, 1, in7, 1, in4, in5, in6, in7); + ADD4(in8, 1, in9, 1, in10, 1, in11, 1, in8, in9, in10, in11); + ADD4(in12, 1, in13, 1, in14, 1, in15, 1, in12, in13, in14, in15); + SRA_4V(in0, in1, in2, in3, 2); + SRA_4V(in4, in5, in6, in7, 2); + SRA_4V(in8, in9, in10, in11, 2); + SRA_4V(in12, in13, in14, in15, 2); + BUTTERFLY_16(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11, + in12, in13, in14, in15, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, + tmp7, in8, in9, in10, in11, in12, in13, in14, in15); + ST_SH8(in8, in9, in10, in11, in12, in13, in14, in15, input, 16); + FDCT8x16_EVEN(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp0, tmp1, + tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + LD_SH8(input, 16, in8, in9, in10, in11, in12, in13, in14, in15); + FDCT8x16_ODD(in8, in9, in10, in11, in12, in13, in14, in15, in0, in1, in2, in3, + in4, in5, in6, in7); + TRANSPOSE8x8_SH_SH(tmp0, in0, tmp1, in1, tmp2, in2, tmp3, in3, tmp0, in0, + tmp1, in1, tmp2, in2, tmp3, in3); + ST_SH8(tmp0, in0, tmp1, in1, tmp2, in2, tmp3, in3, output, 16); + TRANSPOSE8x8_SH_SH(tmp4, in4, tmp5, in5, tmp6, in6, tmp7, in7, tmp4, in4, + tmp5, in5, tmp6, in6, tmp7, in7); + ST_SH8(tmp4, in4, tmp5, in5, tmp6, in6, tmp7, in7, output + 8, 16); +} + +void aom_fdct4x4_msa(const int16_t *input, int16_t *output, + int32_t src_stride) { + v8i16 in0, in1, in2, in3; + + LD_SH4(input, src_stride, in0, in1, in2, in3); + + /* fdct4 pre-process */ + { + v8i16 vec, mask; + v16i8 zero = { 0 }; + v16i8 one = __msa_ldi_b(1); + + mask = (v8i16)__msa_sldi_b(zero, one, 15); + SLLI_4V(in0, in1, in2, in3, 4); + vec = __msa_ceqi_h(in0, 0); + vec = vec ^ 255; + vec = mask & vec; + in0 += vec; + } + + AOM_FDCT4(in0, in1, in2, in3, in0, in1, in2, in3); + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + AOM_FDCT4(in0, in1, in2, in3, in0, in1, in2, in3); + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + ADD4(in0, 1, in1, 1, in2, 1, in3, 1, in0, in1, in2, in3); + SRA_4V(in0, in1, in2, in3, 2); + PCKEV_D2_SH(in1, in0, in3, in2, in0, in2); + ST_SH2(in0, in2, output, 8); +} + +void aom_fdct8x8_msa(const int16_t *input, int16_t *output, + int32_t src_stride) { + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + + LD_SH8(input, src_stride, in0, in1, in2, in3, in4, in5, in6, in7); + SLLI_4V(in0, in1, in2, in3, 2); + SLLI_4V(in4, in5, in6, in7, 2); + AOM_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + AOM_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + SRLI_AVE_S_4V_H(in0, in1, in2, in3, in4, in5, in6, in7); + ST_SH8(in0, in1, in2, in3, in4, in5, in6, in7, output, 8); +} + +void aom_fdct8x8_1_msa(const int16_t *input, int16_t *out, int32_t stride) { + out[0] = LD_HADD(input, stride); + out[1] = 0; +} + +void aom_fdct16x16_msa(const int16_t *input, int16_t *output, + int32_t src_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, tmp_buf[16 * 16]); + + /* column transform */ + for (i = 0; i < 2; ++i) { + fdct8x16_1d_column((input + 8 * i), (&tmp_buf[0] + 8 * i), src_stride); + } + + /* row transform */ + for (i = 0; i < 2; ++i) { + fdct16x8_1d_row((&tmp_buf[0] + (128 * i)), (output + (128 * i))); + } +} + +void aom_fdct16x16_1_msa(const int16_t *input, int16_t *out, int32_t stride) { + int sum = LD_HADD(input, stride); + sum += LD_HADD(input + 8, stride); + sum += LD_HADD(input + 16 * 8, stride); + sum += LD_HADD(input + 16 * 8 + 8, stride); + out[0] = (int16_t)(sum >> 1); +} diff --git a/third_party/aom/aom_dsp/mips/fwd_txfm_msa.h b/third_party/aom/aom_dsp/mips/fwd_txfm_msa.h new file mode 100644 index 0000000000..ada25dffd9 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/fwd_txfm_msa.h @@ -0,0 +1,381 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_MIPS_FWD_TXFM_MSA_H_ +#define AOM_DSP_MIPS_FWD_TXFM_MSA_H_ + +#include "aom_dsp/mips/txfm_macros_msa.h" +#include "aom_dsp/txfm_common.h" + +#define LD_HADD(psrc, stride) \ + ({ \ + v8i16 in0_m, in1_m, in2_m, in3_m, in4_m, in5_m, in6_m, in7_m; \ + v4i32 vec_w_m; \ + \ + LD_SH4((psrc), stride, in0_m, in1_m, in2_m, in3_m); \ + ADD2(in0_m, in1_m, in2_m, in3_m, in0_m, in2_m); \ + LD_SH4(((psrc) + 4 * stride), stride, in4_m, in5_m, in6_m, in7_m); \ + ADD4(in4_m, in5_m, in6_m, in7_m, in0_m, in2_m, in4_m, in6_m, in4_m, in6_m, \ + in0_m, in4_m); \ + in0_m += in4_m; \ + \ + vec_w_m = __msa_hadd_s_w(in0_m, in0_m); \ + HADD_SW_S32(vec_w_m); \ + }) + +#define AOM_FDCT4(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + v8i16 cnst0_m, cnst1_m, cnst2_m, cnst3_m; \ + v8i16 vec0_m, vec1_m, vec2_m, vec3_m; \ + v4i32 vec4_m, vec5_m, vec6_m, vec7_m; \ + v8i16 coeff_m = { \ + cospi_16_64, -cospi_16_64, cospi_8_64, cospi_24_64, -cospi_8_64, 0, 0, 0 \ + }; \ + \ + BUTTERFLY_4(in0, in1, in2, in3, vec0_m, vec1_m, vec2_m, vec3_m); \ + ILVR_H2_SH(vec1_m, vec0_m, vec3_m, vec2_m, vec0_m, vec2_m); \ + SPLATI_H2_SH(coeff_m, 0, 1, cnst0_m, cnst1_m); \ + cnst1_m = __msa_ilvev_h(cnst1_m, cnst0_m); \ + vec5_m = __msa_dotp_s_w(vec0_m, cnst1_m); \ + \ + SPLATI_H2_SH(coeff_m, 4, 3, cnst2_m, cnst3_m); \ + cnst2_m = __msa_ilvev_h(cnst3_m, cnst2_m); \ + vec7_m = __msa_dotp_s_w(vec2_m, cnst2_m); \ + \ + vec4_m = __msa_dotp_s_w(vec0_m, cnst0_m); \ + cnst2_m = __msa_splati_h(coeff_m, 2); \ + cnst2_m = __msa_ilvev_h(cnst2_m, cnst3_m); \ + vec6_m = __msa_dotp_s_w(vec2_m, cnst2_m); \ + \ + SRARI_W4_SW(vec4_m, vec5_m, vec6_m, vec7_m, DCT_CONST_BITS); \ + PCKEV_H4_SH(vec4_m, vec4_m, vec5_m, vec5_m, vec6_m, vec6_m, vec7_m, \ + vec7_m, out0, out2, out1, out3); \ + } + +#define SRLI_AVE_S_4V_H(in0, in1, in2, in3, in4, in5, in6, in7) \ + { \ + v8i16 vec0_m, vec1_m, vec2_m, vec3_m, vec4_m, vec5_m, vec6_m, vec7_m; \ + \ + SRLI_H4_SH(in0, in1, in2, in3, vec0_m, vec1_m, vec2_m, vec3_m, 15); \ + SRLI_H4_SH(in4, in5, in6, in7, vec4_m, vec5_m, vec6_m, vec7_m, 15); \ + AVE_SH4_SH(vec0_m, in0, vec1_m, in1, vec2_m, in2, vec3_m, in3, in0, in1, \ + in2, in3); \ + AVE_SH4_SH(vec4_m, in4, vec5_m, in5, vec6_m, in6, vec7_m, in7, in4, in5, \ + in6, in7); \ + } + +#define AOM_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, \ + out3, out4, out5, out6, out7) \ + { \ + v8i16 s0_m, s1_m, s2_m, s3_m, s4_m, s5_m, s6_m; \ + v8i16 s7_m, x0_m, x1_m, x2_m, x3_m; \ + v8i16 coeff_m = { cospi_16_64, -cospi_16_64, cospi_8_64, cospi_24_64, \ + cospi_4_64, cospi_28_64, cospi_12_64, cospi_20_64 }; \ + \ + /* FDCT stage1 */ \ + BUTTERFLY_8(in0, in1, in2, in3, in4, in5, in6, in7, s0_m, s1_m, s2_m, \ + s3_m, s4_m, s5_m, s6_m, s7_m); \ + BUTTERFLY_4(s0_m, s1_m, s2_m, s3_m, x0_m, x1_m, x2_m, x3_m); \ + ILVL_H2_SH(x1_m, x0_m, x3_m, x2_m, s0_m, s2_m); \ + ILVR_H2_SH(x1_m, x0_m, x3_m, x2_m, s1_m, s3_m); \ + SPLATI_H2_SH(coeff_m, 0, 1, x0_m, x1_m); \ + x1_m = __msa_ilvev_h(x1_m, x0_m); \ + out4 = DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m); \ + \ + SPLATI_H2_SH(coeff_m, 2, 3, x2_m, x3_m); \ + x2_m = -x2_m; \ + x2_m = __msa_ilvev_h(x3_m, x2_m); \ + out6 = DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m); \ + \ + out0 = DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m); \ + x2_m = __msa_splati_h(coeff_m, 2); \ + x2_m = __msa_ilvev_h(x2_m, x3_m); \ + out2 = DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m); \ + \ + /* stage2 */ \ + ILVRL_H2_SH(s5_m, s6_m, s1_m, s0_m); \ + \ + s6_m = DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m); \ + s5_m = DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m); \ + \ + /* stage3 */ \ + BUTTERFLY_4(s4_m, s7_m, s6_m, s5_m, x0_m, x3_m, x2_m, x1_m); \ + \ + /* stage4 */ \ + ILVL_H2_SH(x3_m, x0_m, x2_m, x1_m, s4_m, s6_m); \ + ILVR_H2_SH(x3_m, x0_m, x2_m, x1_m, s5_m, s7_m); \ + \ + SPLATI_H2_SH(coeff_m, 4, 5, x0_m, x1_m); \ + x1_m = __msa_ilvev_h(x0_m, x1_m); \ + out1 = DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x1_m); \ + \ + SPLATI_H2_SH(coeff_m, 6, 7, x2_m, x3_m); \ + x2_m = __msa_ilvev_h(x3_m, x2_m); \ + out5 = DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m); \ + \ + x1_m = __msa_splati_h(coeff_m, 5); \ + x0_m = -x0_m; \ + x0_m = __msa_ilvev_h(x1_m, x0_m); \ + out7 = DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x0_m); \ + \ + x2_m = __msa_splati_h(coeff_m, 6); \ + x3_m = -x3_m; \ + x2_m = __msa_ilvev_h(x2_m, x3_m); \ + out3 = DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m); \ + } + +#define FDCT8x16_EVEN(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3, out4, out5, out6, out7) \ + { \ + v8i16 s0_m, s1_m, s2_m, s3_m, s4_m, s5_m, s6_m, s7_m; \ + v8i16 x0_m, x1_m, x2_m, x3_m; \ + v8i16 coeff_m = { cospi_16_64, -cospi_16_64, cospi_8_64, cospi_24_64, \ + cospi_4_64, cospi_28_64, cospi_12_64, cospi_20_64 }; \ + \ + /* FDCT stage1 */ \ + BUTTERFLY_8(in0, in1, in2, in3, in4, in5, in6, in7, s0_m, s1_m, s2_m, \ + s3_m, s4_m, s5_m, s6_m, s7_m); \ + BUTTERFLY_4(s0_m, s1_m, s2_m, s3_m, x0_m, x1_m, x2_m, x3_m); \ + ILVL_H2_SH(x1_m, x0_m, x3_m, x2_m, s0_m, s2_m); \ + ILVR_H2_SH(x1_m, x0_m, x3_m, x2_m, s1_m, s3_m); \ + SPLATI_H2_SH(coeff_m, 0, 1, x0_m, x1_m); \ + x1_m = __msa_ilvev_h(x1_m, x0_m); \ + out4 = DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m); \ + \ + SPLATI_H2_SH(coeff_m, 2, 3, x2_m, x3_m); \ + x2_m = -x2_m; \ + x2_m = __msa_ilvev_h(x3_m, x2_m); \ + out6 = DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m); \ + \ + out0 = DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m); \ + x2_m = __msa_splati_h(coeff_m, 2); \ + x2_m = __msa_ilvev_h(x2_m, x3_m); \ + out2 = DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m); \ + \ + /* stage2 */ \ + ILVRL_H2_SH(s5_m, s6_m, s1_m, s0_m); \ + \ + s6_m = DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m); \ + s5_m = DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m); \ + \ + /* stage3 */ \ + BUTTERFLY_4(s4_m, s7_m, s6_m, s5_m, x0_m, x3_m, x2_m, x1_m); \ + \ + /* stage4 */ \ + ILVL_H2_SH(x3_m, x0_m, x2_m, x1_m, s4_m, s6_m); \ + ILVR_H2_SH(x3_m, x0_m, x2_m, x1_m, s5_m, s7_m); \ + \ + SPLATI_H2_SH(coeff_m, 4, 5, x0_m, x1_m); \ + x1_m = __msa_ilvev_h(x0_m, x1_m); \ + out1 = DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x1_m); \ + \ + SPLATI_H2_SH(coeff_m, 6, 7, x2_m, x3_m); \ + x2_m = __msa_ilvev_h(x3_m, x2_m); \ + out5 = DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m); \ + \ + x1_m = __msa_splati_h(coeff_m, 5); \ + x0_m = -x0_m; \ + x0_m = __msa_ilvev_h(x1_m, x0_m); \ + out7 = DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x0_m); \ + \ + x2_m = __msa_splati_h(coeff_m, 6); \ + x3_m = -x3_m; \ + x2_m = __msa_ilvev_h(x2_m, x3_m); \ + out3 = DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m); \ + } + +#define FDCT8x16_ODD(input0, input1, input2, input3, input4, input5, input6, \ + input7, out1, out3, out5, out7, out9, out11, out13, \ + out15) \ + { \ + v8i16 stp21_m, stp22_m, stp23_m, stp24_m, stp25_m, stp26_m; \ + v8i16 stp30_m, stp31_m, stp32_m, stp33_m, stp34_m, stp35_m; \ + v8i16 stp36_m, stp37_m, vec0_m, vec1_m; \ + v8i16 vec2_m, vec3_m, vec4_m, vec5_m, vec6_m; \ + v8i16 cnst0_m, cnst1_m, cnst4_m, cnst5_m; \ + v8i16 coeff_m = { cospi_16_64, -cospi_16_64, cospi_8_64, cospi_24_64, \ + -cospi_8_64, -cospi_24_64, cospi_12_64, cospi_20_64 }; \ + v8i16 coeff1_m = { cospi_2_64, cospi_30_64, cospi_14_64, cospi_18_64, \ + cospi_10_64, cospi_22_64, cospi_6_64, cospi_26_64 }; \ + v8i16 coeff2_m = { \ + -cospi_2_64, -cospi_10_64, -cospi_18_64, -cospi_26_64, 0, 0, 0, 0 \ + }; \ + \ + /* stp 1 */ \ + ILVL_H2_SH(input2, input5, input3, input4, vec2_m, vec4_m); \ + ILVR_H2_SH(input2, input5, input3, input4, vec3_m, vec5_m); \ + \ + cnst4_m = __msa_splati_h(coeff_m, 0); \ + stp25_m = DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst4_m); \ + \ + cnst5_m = __msa_splati_h(coeff_m, 1); \ + cnst5_m = __msa_ilvev_h(cnst5_m, cnst4_m); \ + stp22_m = DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst5_m); \ + stp24_m = DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst4_m); \ + stp23_m = DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst5_m); \ + \ + /* stp2 */ \ + BUTTERFLY_4(input0, input1, stp22_m, stp23_m, stp30_m, stp31_m, stp32_m, \ + stp33_m); \ + BUTTERFLY_4(input7, input6, stp25_m, stp24_m, stp37_m, stp36_m, stp35_m, \ + stp34_m); \ + \ + ILVL_H2_SH(stp36_m, stp31_m, stp35_m, stp32_m, vec2_m, vec4_m); \ + ILVR_H2_SH(stp36_m, stp31_m, stp35_m, stp32_m, vec3_m, vec5_m); \ + \ + SPLATI_H2_SH(coeff_m, 2, 3, cnst0_m, cnst1_m); \ + cnst0_m = __msa_ilvev_h(cnst0_m, cnst1_m); \ + stp26_m = DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst0_m); \ + \ + cnst0_m = __msa_splati_h(coeff_m, 4); \ + cnst1_m = __msa_ilvev_h(cnst1_m, cnst0_m); \ + stp21_m = DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst1_m); \ + \ + SPLATI_H2_SH(coeff_m, 5, 2, cnst0_m, cnst1_m); \ + cnst1_m = __msa_ilvev_h(cnst0_m, cnst1_m); \ + stp25_m = DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst1_m); \ + \ + cnst0_m = __msa_splati_h(coeff_m, 3); \ + cnst1_m = __msa_ilvev_h(cnst1_m, cnst0_m); \ + stp22_m = DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst1_m); \ + \ + /* stp4 */ \ + BUTTERFLY_4(stp30_m, stp37_m, stp26_m, stp21_m, vec6_m, vec2_m, vec4_m, \ + vec5_m); \ + BUTTERFLY_4(stp33_m, stp34_m, stp25_m, stp22_m, stp21_m, stp23_m, stp24_m, \ + stp31_m); \ + \ + ILVRL_H2_SH(vec2_m, vec6_m, vec1_m, vec0_m); \ + SPLATI_H2_SH(coeff1_m, 0, 1, cnst0_m, cnst1_m); \ + cnst0_m = __msa_ilvev_h(cnst0_m, cnst1_m); \ + \ + out1 = DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m); \ + \ + cnst0_m = __msa_splati_h(coeff2_m, 0); \ + cnst0_m = __msa_ilvev_h(cnst1_m, cnst0_m); \ + out15 = DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m); \ + \ + ILVRL_H2_SH(vec4_m, vec5_m, vec1_m, vec0_m); \ + SPLATI_H2_SH(coeff1_m, 2, 3, cnst0_m, cnst1_m); \ + cnst1_m = __msa_ilvev_h(cnst1_m, cnst0_m); \ + \ + out9 = DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst1_m); \ + \ + cnst1_m = __msa_splati_h(coeff2_m, 2); \ + cnst0_m = __msa_ilvev_h(cnst0_m, cnst1_m); \ + out7 = DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m); \ + \ + ILVRL_H2_SH(stp23_m, stp21_m, vec1_m, vec0_m); \ + SPLATI_H2_SH(coeff1_m, 4, 5, cnst0_m, cnst1_m); \ + cnst0_m = __msa_ilvev_h(cnst0_m, cnst1_m); \ + out5 = DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m); \ + \ + cnst0_m = __msa_splati_h(coeff2_m, 1); \ + cnst0_m = __msa_ilvev_h(cnst1_m, cnst0_m); \ + out11 = DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m); \ + \ + ILVRL_H2_SH(stp24_m, stp31_m, vec1_m, vec0_m); \ + SPLATI_H2_SH(coeff1_m, 6, 7, cnst0_m, cnst1_m); \ + cnst1_m = __msa_ilvev_h(cnst1_m, cnst0_m); \ + \ + out13 = DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst1_m); \ + \ + cnst1_m = __msa_splati_h(coeff2_m, 3); \ + cnst0_m = __msa_ilvev_h(cnst0_m, cnst1_m); \ + out3 = DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m); \ + } + +#define FDCT_POSTPROC_2V_NEG_H(vec0, vec1) \ + { \ + v8i16 tp0_m, tp1_m; \ + v8i16 one_m = __msa_ldi_h(1); \ + \ + tp0_m = __msa_clti_s_h(vec0, 0); \ + tp1_m = __msa_clti_s_h(vec1, 0); \ + vec0 += 1; \ + vec1 += 1; \ + tp0_m = one_m & tp0_m; \ + tp1_m = one_m & tp1_m; \ + vec0 += tp0_m; \ + vec1 += tp1_m; \ + vec0 >>= 2; \ + vec1 >>= 2; \ + } + +#define FDCT32_POSTPROC_NEG_W(vec) \ + { \ + v4i32 temp_m; \ + v4i32 one_m = __msa_ldi_w(1); \ + \ + temp_m = __msa_clti_s_w(vec, 0); \ + vec += 1; \ + temp_m = one_m & temp_m; \ + vec += temp_m; \ + vec >>= 2; \ + } + +#define FDCT32_POSTPROC_2V_POS_H(vec0, vec1) \ + { \ + v8i16 tp0_m, tp1_m; \ + v8i16 one = __msa_ldi_h(1); \ + \ + tp0_m = __msa_clei_s_h(vec0, 0); \ + tp1_m = __msa_clei_s_h(vec1, 0); \ + tp0_m = (v8i16)__msa_xori_b((v16u8)tp0_m, 255); \ + tp1_m = (v8i16)__msa_xori_b((v16u8)tp1_m, 255); \ + vec0 += 1; \ + vec1 += 1; \ + tp0_m = one & tp0_m; \ + tp1_m = one & tp1_m; \ + vec0 += tp0_m; \ + vec1 += tp1_m; \ + vec0 >>= 2; \ + vec1 >>= 2; \ + } + +#define DOTP_CONST_PAIR_W(reg0_left, reg1_left, reg0_right, reg1_right, \ + const0, const1, out0, out1, out2, out3) \ + { \ + v4i32 s0_m, s1_m, s2_m, s3_m, s4_m, s5_m, s6_m, s7_m; \ + v2i64 tp0_m, tp1_m, tp2_m, tp3_m; \ + v4i32 k0_m = __msa_fill_w((int32_t)const0); \ + \ + s0_m = __msa_fill_w((int32_t)const1); \ + k0_m = __msa_ilvev_w(s0_m, k0_m); \ + \ + ILVRL_W2_SW(-reg1_left, reg0_left, s1_m, s0_m); \ + ILVRL_W2_SW(reg0_left, reg1_left, s3_m, s2_m); \ + ILVRL_W2_SW(-reg1_right, reg0_right, s5_m, s4_m); \ + ILVRL_W2_SW(reg0_right, reg1_right, s7_m, s6_m); \ + \ + DOTP_SW2_SD(s0_m, s1_m, k0_m, k0_m, tp0_m, tp1_m); \ + DOTP_SW2_SD(s4_m, s5_m, k0_m, k0_m, tp2_m, tp3_m); \ + tp0_m = __msa_srari_d(tp0_m, DCT_CONST_BITS); \ + tp1_m = __msa_srari_d(tp1_m, DCT_CONST_BITS); \ + tp2_m = __msa_srari_d(tp2_m, DCT_CONST_BITS); \ + tp3_m = __msa_srari_d(tp3_m, DCT_CONST_BITS); \ + out0 = __msa_pckev_w((v4i32)tp0_m, (v4i32)tp1_m); \ + out1 = __msa_pckev_w((v4i32)tp2_m, (v4i32)tp3_m); \ + \ + DOTP_SW2_SD(s2_m, s3_m, k0_m, k0_m, tp0_m, tp1_m); \ + DOTP_SW2_SD(s6_m, s7_m, k0_m, k0_m, tp2_m, tp3_m); \ + tp0_m = __msa_srari_d(tp0_m, DCT_CONST_BITS); \ + tp1_m = __msa_srari_d(tp1_m, DCT_CONST_BITS); \ + tp2_m = __msa_srari_d(tp2_m, DCT_CONST_BITS); \ + tp3_m = __msa_srari_d(tp3_m, DCT_CONST_BITS); \ + out2 = __msa_pckev_w((v4i32)tp0_m, (v4i32)tp1_m); \ + out3 = __msa_pckev_w((v4i32)tp2_m, (v4i32)tp3_m); \ + } + +void fdct8x16_1d_column(const int16_t *input, int16_t *tmp_ptr, + int32_t src_stride); +void fdct16x8_1d_row(int16_t *input, int16_t *output); +#endif // AOM_DSP_MIPS_FWD_TXFM_MSA_H_ diff --git a/third_party/aom/aom_dsp/mips/idct16x16_msa.c b/third_party/aom/aom_dsp/mips/idct16x16_msa.c new file mode 100644 index 0000000000..0ea127f524 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/idct16x16_msa.c @@ -0,0 +1,486 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/inv_txfm_msa.h" + +void aom_idct16_1d_rows_msa(const int16_t *input, int16_t *output) { + v8i16 loc0, loc1, loc2, loc3; + v8i16 reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14; + v8i16 reg3, reg13, reg11, reg5, reg7, reg9, reg1, reg15; + v8i16 tmp5, tmp6, tmp7; + + LD_SH8(input, 16, reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7); + input += 8; + LD_SH8(input, 16, reg8, reg9, reg10, reg11, reg12, reg13, reg14, reg15); + + TRANSPOSE8x8_SH_SH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg0, reg1, + reg2, reg3, reg4, reg5, reg6, reg7); + TRANSPOSE8x8_SH_SH(reg8, reg9, reg10, reg11, reg12, reg13, reg14, reg15, reg8, + reg9, reg10, reg11, reg12, reg13, reg14, reg15); + DOTP_CONST_PAIR(reg2, reg14, cospi_28_64, cospi_4_64, reg2, reg14); + DOTP_CONST_PAIR(reg10, reg6, cospi_12_64, cospi_20_64, reg10, reg6); + BUTTERFLY_4(reg2, reg14, reg6, reg10, loc0, loc1, reg14, reg2); + DOTP_CONST_PAIR(reg14, reg2, cospi_16_64, cospi_16_64, loc2, loc3); + DOTP_CONST_PAIR(reg0, reg8, cospi_16_64, cospi_16_64, reg0, reg8); + DOTP_CONST_PAIR(reg4, reg12, cospi_24_64, cospi_8_64, reg4, reg12); + BUTTERFLY_4(reg8, reg0, reg4, reg12, reg2, reg6, reg10, reg14); + SUB4(reg2, loc1, reg14, loc0, reg6, loc3, reg10, loc2, reg0, reg12, reg4, + reg8); + ADD4(reg2, loc1, reg14, loc0, reg6, loc3, reg10, loc2, reg2, reg14, reg6, + reg10); + + /* stage 2 */ + DOTP_CONST_PAIR(reg1, reg15, cospi_30_64, cospi_2_64, reg1, reg15); + DOTP_CONST_PAIR(reg9, reg7, cospi_14_64, cospi_18_64, loc2, loc3); + + reg9 = reg1 - loc2; + reg1 = reg1 + loc2; + reg7 = reg15 - loc3; + reg15 = reg15 + loc3; + + DOTP_CONST_PAIR(reg5, reg11, cospi_22_64, cospi_10_64, reg5, reg11); + DOTP_CONST_PAIR(reg13, reg3, cospi_6_64, cospi_26_64, loc0, loc1); + BUTTERFLY_4(loc0, loc1, reg11, reg5, reg13, reg3, reg11, reg5); + + loc1 = reg15 + reg3; + reg3 = reg15 - reg3; + loc2 = reg2 + loc1; + reg15 = reg2 - loc1; + + loc1 = reg1 + reg13; + reg13 = reg1 - reg13; + loc0 = reg0 + loc1; + loc1 = reg0 - loc1; + tmp6 = loc0; + tmp7 = loc1; + reg0 = loc2; + + DOTP_CONST_PAIR(reg7, reg9, cospi_24_64, cospi_8_64, reg7, reg9); + DOTP_CONST_PAIR((-reg5), (-reg11), cospi_8_64, cospi_24_64, reg5, reg11); + + loc0 = reg9 + reg5; + reg5 = reg9 - reg5; + reg2 = reg6 + loc0; + reg1 = reg6 - loc0; + + loc0 = reg7 + reg11; + reg11 = reg7 - reg11; + loc1 = reg4 + loc0; + loc2 = reg4 - loc0; + tmp5 = loc1; + + DOTP_CONST_PAIR(reg5, reg11, cospi_16_64, cospi_16_64, reg5, reg11); + BUTTERFLY_4(reg8, reg10, reg11, reg5, loc0, reg4, reg9, loc1); + + reg10 = loc0; + reg11 = loc1; + + DOTP_CONST_PAIR(reg3, reg13, cospi_16_64, cospi_16_64, reg3, reg13); + BUTTERFLY_4(reg12, reg14, reg13, reg3, reg8, reg6, reg7, reg5); + + reg13 = loc2; + + /* Transpose and store the output */ + reg12 = tmp5; + reg14 = tmp6; + reg3 = tmp7; + + /* transpose block */ + TRANSPOSE8x8_SH_SH(reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14, reg0, + reg2, reg4, reg6, reg8, reg10, reg12, reg14); + ST_SH8(reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14, output, 16); + + /* transpose block */ + TRANSPOSE8x8_SH_SH(reg3, reg13, reg11, reg5, reg7, reg9, reg1, reg15, reg3, + reg13, reg11, reg5, reg7, reg9, reg1, reg15); + ST_SH8(reg3, reg13, reg11, reg5, reg7, reg9, reg1, reg15, (output + 8), 16); +} + +void aom_idct16_1d_columns_addblk_msa(int16_t *input, uint8_t *dst, + int32_t dst_stride) { + v8i16 loc0, loc1, loc2, loc3; + v8i16 reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14; + v8i16 reg3, reg13, reg11, reg5, reg7, reg9, reg1, reg15; + v8i16 tmp5, tmp6, tmp7; + + /* load up 8x8 */ + LD_SH8(input, 16, reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7); + input += 8 * 16; + /* load bottom 8x8 */ + LD_SH8(input, 16, reg8, reg9, reg10, reg11, reg12, reg13, reg14, reg15); + + DOTP_CONST_PAIR(reg2, reg14, cospi_28_64, cospi_4_64, reg2, reg14); + DOTP_CONST_PAIR(reg10, reg6, cospi_12_64, cospi_20_64, reg10, reg6); + BUTTERFLY_4(reg2, reg14, reg6, reg10, loc0, loc1, reg14, reg2); + DOTP_CONST_PAIR(reg14, reg2, cospi_16_64, cospi_16_64, loc2, loc3); + DOTP_CONST_PAIR(reg0, reg8, cospi_16_64, cospi_16_64, reg0, reg8); + DOTP_CONST_PAIR(reg4, reg12, cospi_24_64, cospi_8_64, reg4, reg12); + BUTTERFLY_4(reg8, reg0, reg4, reg12, reg2, reg6, reg10, reg14); + + reg0 = reg2 - loc1; + reg2 = reg2 + loc1; + reg12 = reg14 - loc0; + reg14 = reg14 + loc0; + reg4 = reg6 - loc3; + reg6 = reg6 + loc3; + reg8 = reg10 - loc2; + reg10 = reg10 + loc2; + + /* stage 2 */ + DOTP_CONST_PAIR(reg1, reg15, cospi_30_64, cospi_2_64, reg1, reg15); + DOTP_CONST_PAIR(reg9, reg7, cospi_14_64, cospi_18_64, loc2, loc3); + + reg9 = reg1 - loc2; + reg1 = reg1 + loc2; + reg7 = reg15 - loc3; + reg15 = reg15 + loc3; + + DOTP_CONST_PAIR(reg5, reg11, cospi_22_64, cospi_10_64, reg5, reg11); + DOTP_CONST_PAIR(reg13, reg3, cospi_6_64, cospi_26_64, loc0, loc1); + BUTTERFLY_4(loc0, loc1, reg11, reg5, reg13, reg3, reg11, reg5); + + loc1 = reg15 + reg3; + reg3 = reg15 - reg3; + loc2 = reg2 + loc1; + reg15 = reg2 - loc1; + + loc1 = reg1 + reg13; + reg13 = reg1 - reg13; + loc0 = reg0 + loc1; + loc1 = reg0 - loc1; + tmp6 = loc0; + tmp7 = loc1; + reg0 = loc2; + + DOTP_CONST_PAIR(reg7, reg9, cospi_24_64, cospi_8_64, reg7, reg9); + DOTP_CONST_PAIR((-reg5), (-reg11), cospi_8_64, cospi_24_64, reg5, reg11); + + loc0 = reg9 + reg5; + reg5 = reg9 - reg5; + reg2 = reg6 + loc0; + reg1 = reg6 - loc0; + + loc0 = reg7 + reg11; + reg11 = reg7 - reg11; + loc1 = reg4 + loc0; + loc2 = reg4 - loc0; + tmp5 = loc1; + + DOTP_CONST_PAIR(reg5, reg11, cospi_16_64, cospi_16_64, reg5, reg11); + BUTTERFLY_4(reg8, reg10, reg11, reg5, loc0, reg4, reg9, loc1); + + reg10 = loc0; + reg11 = loc1; + + DOTP_CONST_PAIR(reg3, reg13, cospi_16_64, cospi_16_64, reg3, reg13); + BUTTERFLY_4(reg12, reg14, reg13, reg3, reg8, reg6, reg7, reg5); + reg13 = loc2; + + /* Transpose and store the output */ + reg12 = tmp5; + reg14 = tmp6; + reg3 = tmp7; + + SRARI_H4_SH(reg0, reg2, reg4, reg6, 6); + AOM_ADDBLK_ST8x4_UB(dst, dst_stride, reg0, reg2, reg4, reg6); + dst += (4 * dst_stride); + SRARI_H4_SH(reg8, reg10, reg12, reg14, 6); + AOM_ADDBLK_ST8x4_UB(dst, dst_stride, reg8, reg10, reg12, reg14); + dst += (4 * dst_stride); + SRARI_H4_SH(reg3, reg13, reg11, reg5, 6); + AOM_ADDBLK_ST8x4_UB(dst, dst_stride, reg3, reg13, reg11, reg5); + dst += (4 * dst_stride); + SRARI_H4_SH(reg7, reg9, reg1, reg15, 6); + AOM_ADDBLK_ST8x4_UB(dst, dst_stride, reg7, reg9, reg1, reg15); +} + +void aom_idct16x16_256_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, out_arr[16 * 16]); + int16_t *out = out_arr; + + /* transform rows */ + for (i = 0; i < 2; ++i) { + /* process 16 * 8 block */ + aom_idct16_1d_rows_msa((input + (i << 7)), (out + (i << 7))); + } + + /* transform columns */ + for (i = 0; i < 2; ++i) { + /* process 8 * 16 block */ + aom_idct16_1d_columns_addblk_msa((out + (i << 3)), (dst + (i << 3)), + dst_stride); + } +} + +void aom_idct16x16_10_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + uint8_t i; + DECLARE_ALIGNED(32, int16_t, out_arr[16 * 16]); + int16_t *out = out_arr; + + /* process 16 * 8 block */ + aom_idct16_1d_rows_msa(input, out); + + /* short case just considers top 4 rows as valid output */ + out += 4 * 16; + for (i = 12; i--;) { + __asm__ __volatile__( + "sw $zero, 0(%[out]) \n\t" + "sw $zero, 4(%[out]) \n\t" + "sw $zero, 8(%[out]) \n\t" + "sw $zero, 12(%[out]) \n\t" + "sw $zero, 16(%[out]) \n\t" + "sw $zero, 20(%[out]) \n\t" + "sw $zero, 24(%[out]) \n\t" + "sw $zero, 28(%[out]) \n\t" + + : + : [out] "r"(out)); + + out += 16; + } + + out = out_arr; + + /* transform columns */ + for (i = 0; i < 2; ++i) { + /* process 8 * 16 block */ + aom_idct16_1d_columns_addblk_msa((out + (i << 3)), (dst + (i << 3)), + dst_stride); + } +} + +void aom_idct16x16_1_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + uint8_t i; + int16_t out; + v8i16 vec, res0, res1, res2, res3, res4, res5, res6, res7; + v16u8 dst0, dst1, dst2, dst3, tmp0, tmp1, tmp2, tmp3; + + out = ROUND_POWER_OF_TWO((input[0] * cospi_16_64), DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO((out * cospi_16_64), DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO(out, 6); + + vec = __msa_fill_h(out); + + for (i = 4; i--;) { + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + UNPCK_UB_SH(dst0, res0, res4); + UNPCK_UB_SH(dst1, res1, res5); + UNPCK_UB_SH(dst2, res2, res6); + UNPCK_UB_SH(dst3, res3, res7); + ADD4(res0, vec, res1, vec, res2, vec, res3, vec, res0, res1, res2, res3); + ADD4(res4, vec, res5, vec, res6, vec, res7, vec, res4, res5, res6, res7); + CLIP_SH4_0_255(res0, res1, res2, res3); + CLIP_SH4_0_255(res4, res5, res6, res7); + PCKEV_B4_UB(res4, res0, res5, res1, res6, res2, res7, res3, tmp0, tmp1, + tmp2, tmp3); + ST_UB4(tmp0, tmp1, tmp2, tmp3, dst, dst_stride); + dst += (4 * dst_stride); + } +} + +void aom_iadst16_1d_rows_msa(const int16_t *input, int16_t *output) { + v8i16 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15; + v8i16 l0, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15; + + /* load input data */ + LD_SH16(input, 8, l0, l8, l1, l9, l2, l10, l3, l11, l4, l12, l5, l13, l6, l14, + l7, l15); + TRANSPOSE8x8_SH_SH(l0, l1, l2, l3, l4, l5, l6, l7, l0, l1, l2, l3, l4, l5, l6, + l7); + TRANSPOSE8x8_SH_SH(l8, l9, l10, l11, l12, l13, l14, l15, l8, l9, l10, l11, + l12, l13, l14, l15); + + /* ADST in horizontal */ + AOM_IADST8x16_1D(l0, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, + l14, l15, r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, + r12, r13, r14, r15); + + l1 = -r8; + l3 = -r4; + l13 = -r13; + l15 = -r1; + + TRANSPOSE8x8_SH_SH(r0, l1, r12, l3, r6, r14, r10, r2, l0, l1, l2, l3, l4, l5, + l6, l7); + ST_SH8(l0, l1, l2, l3, l4, l5, l6, l7, output, 16); + TRANSPOSE8x8_SH_SH(r3, r11, r15, r7, r5, l13, r9, l15, l8, l9, l10, l11, l12, + l13, l14, l15); + ST_SH8(l8, l9, l10, l11, l12, l13, l14, l15, (output + 8), 16); +} + +void aom_iadst16_1d_columns_addblk_msa(int16_t *input, uint8_t *dst, + int32_t dst_stride) { + v8i16 v0, v2, v4, v6, k0, k1, k2, k3; + v8i16 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15; + v8i16 out0, out1, out2, out3, out4, out5, out6, out7; + v8i16 out8, out9, out10, out11, out12, out13, out14, out15; + v8i16 g0, g1, g2, g3, g4, g5, g6, g7, g8, g9, g10, g11, g12, g13, g14, g15; + v8i16 h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11; + v8i16 res0, res1, res2, res3, res4, res5, res6, res7; + v8i16 res8, res9, res10, res11, res12, res13, res14, res15; + v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + v16u8 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15; + v16i8 zero = { 0 }; + + r0 = LD_SH(input + 0 * 16); + r3 = LD_SH(input + 3 * 16); + r4 = LD_SH(input + 4 * 16); + r7 = LD_SH(input + 7 * 16); + r8 = LD_SH(input + 8 * 16); + r11 = LD_SH(input + 11 * 16); + r12 = LD_SH(input + 12 * 16); + r15 = LD_SH(input + 15 * 16); + + /* stage 1 */ + k0 = AOM_SET_COSPI_PAIR(cospi_1_64, cospi_31_64); + k1 = AOM_SET_COSPI_PAIR(cospi_31_64, -cospi_1_64); + k2 = AOM_SET_COSPI_PAIR(cospi_17_64, cospi_15_64); + k3 = AOM_SET_COSPI_PAIR(cospi_15_64, -cospi_17_64); + MADD_BF(r15, r0, r7, r8, k0, k1, k2, k3, g0, g1, g2, g3); + k0 = AOM_SET_COSPI_PAIR(cospi_9_64, cospi_23_64); + k1 = AOM_SET_COSPI_PAIR(cospi_23_64, -cospi_9_64); + k2 = AOM_SET_COSPI_PAIR(cospi_25_64, cospi_7_64); + k3 = AOM_SET_COSPI_PAIR(cospi_7_64, -cospi_25_64); + MADD_BF(r11, r4, r3, r12, k0, k1, k2, k3, g8, g9, g10, g11); + BUTTERFLY_4(g0, g2, g10, g8, h8, h9, v2, v0); + k0 = AOM_SET_COSPI_PAIR(cospi_4_64, cospi_28_64); + k1 = AOM_SET_COSPI_PAIR(cospi_28_64, -cospi_4_64); + k2 = AOM_SET_COSPI_PAIR(-cospi_28_64, cospi_4_64); + MADD_BF(g1, g3, g9, g11, k0, k1, k2, k0, h0, h1, h2, h3); + + r1 = LD_SH(input + 1 * 16); + r2 = LD_SH(input + 2 * 16); + r5 = LD_SH(input + 5 * 16); + r6 = LD_SH(input + 6 * 16); + r9 = LD_SH(input + 9 * 16); + r10 = LD_SH(input + 10 * 16); + r13 = LD_SH(input + 13 * 16); + r14 = LD_SH(input + 14 * 16); + + k0 = AOM_SET_COSPI_PAIR(cospi_5_64, cospi_27_64); + k1 = AOM_SET_COSPI_PAIR(cospi_27_64, -cospi_5_64); + k2 = AOM_SET_COSPI_PAIR(cospi_21_64, cospi_11_64); + k3 = AOM_SET_COSPI_PAIR(cospi_11_64, -cospi_21_64); + MADD_BF(r13, r2, r5, r10, k0, k1, k2, k3, g4, g5, g6, g7); + k0 = AOM_SET_COSPI_PAIR(cospi_13_64, cospi_19_64); + k1 = AOM_SET_COSPI_PAIR(cospi_19_64, -cospi_13_64); + k2 = AOM_SET_COSPI_PAIR(cospi_29_64, cospi_3_64); + k3 = AOM_SET_COSPI_PAIR(cospi_3_64, -cospi_29_64); + MADD_BF(r9, r6, r1, r14, k0, k1, k2, k3, g12, g13, g14, g15); + BUTTERFLY_4(g4, g6, g14, g12, h10, h11, v6, v4); + BUTTERFLY_4(h8, h9, h11, h10, out0, out1, h11, h10); + out1 = -out1; + SRARI_H2_SH(out0, out1, 6); + dst0 = LD_UB(dst + 0 * dst_stride); + dst1 = LD_UB(dst + 15 * dst_stride); + ILVR_B2_SH(zero, dst0, zero, dst1, res0, res1); + ADD2(res0, out0, res1, out1, res0, res1); + CLIP_SH2_0_255(res0, res1); + PCKEV_B2_SH(res0, res0, res1, res1, res0, res1); + ST8x1_UB(res0, dst); + ST8x1_UB(res1, dst + 15 * dst_stride); + + k0 = AOM_SET_COSPI_PAIR(cospi_12_64, cospi_20_64); + k1 = AOM_SET_COSPI_PAIR(-cospi_20_64, cospi_12_64); + k2 = AOM_SET_COSPI_PAIR(cospi_20_64, -cospi_12_64); + MADD_BF(g7, g5, g15, g13, k0, k1, k2, k0, h4, h5, h6, h7); + BUTTERFLY_4(h0, h2, h6, h4, out8, out9, out11, out10); + out8 = -out8; + + SRARI_H2_SH(out8, out9, 6); + dst8 = LD_UB(dst + 1 * dst_stride); + dst9 = LD_UB(dst + 14 * dst_stride); + ILVR_B2_SH(zero, dst8, zero, dst9, res8, res9); + ADD2(res8, out8, res9, out9, res8, res9); + CLIP_SH2_0_255(res8, res9); + PCKEV_B2_SH(res8, res8, res9, res9, res8, res9); + ST8x1_UB(res8, dst + dst_stride); + ST8x1_UB(res9, dst + 14 * dst_stride); + + k0 = AOM_SET_COSPI_PAIR(cospi_8_64, cospi_24_64); + k1 = AOM_SET_COSPI_PAIR(cospi_24_64, -cospi_8_64); + k2 = AOM_SET_COSPI_PAIR(-cospi_24_64, cospi_8_64); + MADD_BF(v0, v2, v4, v6, k0, k1, k2, k0, out4, out6, out5, out7); + out4 = -out4; + SRARI_H2_SH(out4, out5, 6); + dst4 = LD_UB(dst + 3 * dst_stride); + dst5 = LD_UB(dst + 12 * dst_stride); + ILVR_B2_SH(zero, dst4, zero, dst5, res4, res5); + ADD2(res4, out4, res5, out5, res4, res5); + CLIP_SH2_0_255(res4, res5); + PCKEV_B2_SH(res4, res4, res5, res5, res4, res5); + ST8x1_UB(res4, dst + 3 * dst_stride); + ST8x1_UB(res5, dst + 12 * dst_stride); + + MADD_BF(h1, h3, h5, h7, k0, k1, k2, k0, out12, out14, out13, out15); + out13 = -out13; + SRARI_H2_SH(out12, out13, 6); + dst12 = LD_UB(dst + 2 * dst_stride); + dst13 = LD_UB(dst + 13 * dst_stride); + ILVR_B2_SH(zero, dst12, zero, dst13, res12, res13); + ADD2(res12, out12, res13, out13, res12, res13); + CLIP_SH2_0_255(res12, res13); + PCKEV_B2_SH(res12, res12, res13, res13, res12, res13); + ST8x1_UB(res12, dst + 2 * dst_stride); + ST8x1_UB(res13, dst + 13 * dst_stride); + + k0 = AOM_SET_COSPI_PAIR(cospi_16_64, cospi_16_64); + k3 = AOM_SET_COSPI_PAIR(-cospi_16_64, cospi_16_64); + MADD_SHORT(out6, out7, k0, k3, out6, out7); + SRARI_H2_SH(out6, out7, 6); + dst6 = LD_UB(dst + 4 * dst_stride); + dst7 = LD_UB(dst + 11 * dst_stride); + ILVR_B2_SH(zero, dst6, zero, dst7, res6, res7); + ADD2(res6, out6, res7, out7, res6, res7); + CLIP_SH2_0_255(res6, res7); + PCKEV_B2_SH(res6, res6, res7, res7, res6, res7); + ST8x1_UB(res6, dst + 4 * dst_stride); + ST8x1_UB(res7, dst + 11 * dst_stride); + + MADD_SHORT(out10, out11, k0, k3, out10, out11); + SRARI_H2_SH(out10, out11, 6); + dst10 = LD_UB(dst + 6 * dst_stride); + dst11 = LD_UB(dst + 9 * dst_stride); + ILVR_B2_SH(zero, dst10, zero, dst11, res10, res11); + ADD2(res10, out10, res11, out11, res10, res11); + CLIP_SH2_0_255(res10, res11); + PCKEV_B2_SH(res10, res10, res11, res11, res10, res11); + ST8x1_UB(res10, dst + 6 * dst_stride); + ST8x1_UB(res11, dst + 9 * dst_stride); + + k1 = AOM_SET_COSPI_PAIR(-cospi_16_64, -cospi_16_64); + k2 = AOM_SET_COSPI_PAIR(cospi_16_64, -cospi_16_64); + MADD_SHORT(h10, h11, k1, k2, out2, out3); + SRARI_H2_SH(out2, out3, 6); + dst2 = LD_UB(dst + 7 * dst_stride); + dst3 = LD_UB(dst + 8 * dst_stride); + ILVR_B2_SH(zero, dst2, zero, dst3, res2, res3); + ADD2(res2, out2, res3, out3, res2, res3); + CLIP_SH2_0_255(res2, res3); + PCKEV_B2_SH(res2, res2, res3, res3, res2, res3); + ST8x1_UB(res2, dst + 7 * dst_stride); + ST8x1_UB(res3, dst + 8 * dst_stride); + + MADD_SHORT(out14, out15, k1, k2, out14, out15); + SRARI_H2_SH(out14, out15, 6); + dst14 = LD_UB(dst + 5 * dst_stride); + dst15 = LD_UB(dst + 10 * dst_stride); + ILVR_B2_SH(zero, dst14, zero, dst15, res14, res15); + ADD2(res14, out14, res15, out15, res14, res15); + CLIP_SH2_0_255(res14, res15); + PCKEV_B2_SH(res14, res14, res15, res15, res14, res15); + ST8x1_UB(res14, dst + 5 * dst_stride); + ST8x1_UB(res15, dst + 10 * dst_stride); +} diff --git a/third_party/aom/aom_dsp/mips/idct32x32_msa.c b/third_party/aom/aom_dsp/mips/idct32x32_msa.c new file mode 100644 index 0000000000..f1ca757a0d --- /dev/null +++ b/third_party/aom/aom_dsp/mips/idct32x32_msa.c @@ -0,0 +1,730 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/inv_txfm_msa.h" + +static void idct32x8_row_transpose_store(const int16_t *input, + int16_t *tmp_buf) { + v8i16 m0, m1, m2, m3, m4, m5, m6, m7, n0, n1, n2, n3, n4, n5, n6, n7; + + /* 1st & 2nd 8x8 */ + LD_SH8(input, 32, m0, n0, m1, n1, m2, n2, m3, n3); + LD_SH8((input + 8), 32, m4, n4, m5, n5, m6, n6, m7, n7); + TRANSPOSE8x8_SH_SH(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3, + n3); + TRANSPOSE8x8_SH_SH(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7, + n7); + ST_SH8(m0, n0, m1, n1, m2, n2, m3, n3, (tmp_buf), 8); + ST_SH4(m4, n4, m5, n5, (tmp_buf + 8 * 8), 8); + ST_SH4(m6, n6, m7, n7, (tmp_buf + 12 * 8), 8); + + /* 3rd & 4th 8x8 */ + LD_SH8((input + 16), 32, m0, n0, m1, n1, m2, n2, m3, n3); + LD_SH8((input + 24), 32, m4, n4, m5, n5, m6, n6, m7, n7); + TRANSPOSE8x8_SH_SH(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3, + n3); + TRANSPOSE8x8_SH_SH(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7, + n7); + ST_SH4(m0, n0, m1, n1, (tmp_buf + 16 * 8), 8); + ST_SH4(m2, n2, m3, n3, (tmp_buf + 20 * 8), 8); + ST_SH4(m4, n4, m5, n5, (tmp_buf + 24 * 8), 8); + ST_SH4(m6, n6, m7, n7, (tmp_buf + 28 * 8), 8); +} + +static void idct32x8_row_even_process_store(int16_t *tmp_buf, + int16_t *tmp_eve_buf) { + v8i16 vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + v8i16 reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + v8i16 stp0, stp1, stp2, stp3, stp4, stp5, stp6, stp7; + + /* Even stage 1 */ + LD_SH8(tmp_buf, 32, reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7); + + DOTP_CONST_PAIR(reg1, reg7, cospi_28_64, cospi_4_64, reg1, reg7); + DOTP_CONST_PAIR(reg5, reg3, cospi_12_64, cospi_20_64, reg5, reg3); + BUTTERFLY_4(reg1, reg7, reg3, reg5, vec1, vec3, vec2, vec0); + DOTP_CONST_PAIR(vec2, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + + loc1 = vec3; + loc0 = vec1; + + DOTP_CONST_PAIR(reg0, reg4, cospi_16_64, cospi_16_64, reg0, reg4); + DOTP_CONST_PAIR(reg2, reg6, cospi_24_64, cospi_8_64, reg2, reg6); + BUTTERFLY_4(reg4, reg0, reg2, reg6, vec1, vec3, vec2, vec0); + BUTTERFLY_4(vec0, vec1, loc1, loc0, stp3, stp0, stp7, stp4); + BUTTERFLY_4(vec2, vec3, loc3, loc2, stp2, stp1, stp6, stp5); + + /* Even stage 2 */ + LD_SH8((tmp_buf + 16), 32, reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7); + DOTP_CONST_PAIR(reg0, reg7, cospi_30_64, cospi_2_64, reg0, reg7); + DOTP_CONST_PAIR(reg4, reg3, cospi_14_64, cospi_18_64, reg4, reg3); + DOTP_CONST_PAIR(reg2, reg5, cospi_22_64, cospi_10_64, reg2, reg5); + DOTP_CONST_PAIR(reg6, reg1, cospi_6_64, cospi_26_64, reg6, reg1); + + vec0 = reg0 + reg4; + reg0 = reg0 - reg4; + reg4 = reg6 + reg2; + reg6 = reg6 - reg2; + reg2 = reg1 + reg5; + reg1 = reg1 - reg5; + reg5 = reg7 + reg3; + reg7 = reg7 - reg3; + reg3 = vec0; + + vec1 = reg2; + reg2 = reg3 + reg4; + reg3 = reg3 - reg4; + reg4 = reg5 - vec1; + reg5 = reg5 + vec1; + + DOTP_CONST_PAIR(reg7, reg0, cospi_24_64, cospi_8_64, reg0, reg7); + DOTP_CONST_PAIR((-reg6), reg1, cospi_24_64, cospi_8_64, reg6, reg1); + + vec0 = reg0 - reg6; + reg0 = reg0 + reg6; + vec1 = reg7 - reg1; + reg7 = reg7 + reg1; + + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, reg6, reg1); + DOTP_CONST_PAIR(reg4, reg3, cospi_16_64, cospi_16_64, reg3, reg4); + + /* Even stage 3 : Dependency on Even stage 1 & Even stage 2 */ + BUTTERFLY_4(stp0, stp1, reg7, reg5, loc1, loc3, loc2, loc0); + ST_SH(loc0, (tmp_eve_buf + 15 * 8)); + ST_SH(loc1, (tmp_eve_buf)); + ST_SH(loc2, (tmp_eve_buf + 14 * 8)); + ST_SH(loc3, (tmp_eve_buf + 8)); + + BUTTERFLY_4(stp2, stp3, reg4, reg1, loc1, loc3, loc2, loc0); + ST_SH(loc0, (tmp_eve_buf + 13 * 8)); + ST_SH(loc1, (tmp_eve_buf + 2 * 8)); + ST_SH(loc2, (tmp_eve_buf + 12 * 8)); + ST_SH(loc3, (tmp_eve_buf + 3 * 8)); + + /* Store 8 */ + BUTTERFLY_4(stp4, stp5, reg6, reg3, loc1, loc3, loc2, loc0); + ST_SH(loc0, (tmp_eve_buf + 11 * 8)); + ST_SH(loc1, (tmp_eve_buf + 4 * 8)); + ST_SH(loc2, (tmp_eve_buf + 10 * 8)); + ST_SH(loc3, (tmp_eve_buf + 5 * 8)); + + BUTTERFLY_4(stp6, stp7, reg2, reg0, loc1, loc3, loc2, loc0); + ST_SH(loc0, (tmp_eve_buf + 9 * 8)); + ST_SH(loc1, (tmp_eve_buf + 6 * 8)); + ST_SH(loc2, (tmp_eve_buf + 8 * 8)); + ST_SH(loc3, (tmp_eve_buf + 7 * 8)); +} + +static void idct32x8_row_odd_process_store(int16_t *tmp_buf, + int16_t *tmp_odd_buf) { + v8i16 vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + v8i16 reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + + /* Odd stage 1 */ + reg0 = LD_SH(tmp_buf + 8); + reg1 = LD_SH(tmp_buf + 7 * 8); + reg2 = LD_SH(tmp_buf + 9 * 8); + reg3 = LD_SH(tmp_buf + 15 * 8); + reg4 = LD_SH(tmp_buf + 17 * 8); + reg5 = LD_SH(tmp_buf + 23 * 8); + reg6 = LD_SH(tmp_buf + 25 * 8); + reg7 = LD_SH(tmp_buf + 31 * 8); + + DOTP_CONST_PAIR(reg0, reg7, cospi_31_64, cospi_1_64, reg0, reg7); + DOTP_CONST_PAIR(reg4, reg3, cospi_15_64, cospi_17_64, reg3, reg4); + DOTP_CONST_PAIR(reg2, reg5, cospi_23_64, cospi_9_64, reg2, reg5); + DOTP_CONST_PAIR(reg6, reg1, cospi_7_64, cospi_25_64, reg1, reg6); + + vec0 = reg0 + reg3; + reg0 = reg0 - reg3; + reg3 = reg7 + reg4; + reg7 = reg7 - reg4; + reg4 = reg1 + reg2; + reg1 = reg1 - reg2; + reg2 = reg6 + reg5; + reg6 = reg6 - reg5; + reg5 = vec0; + + /* 4 Stores */ + ADD2(reg5, reg4, reg3, reg2, vec0, vec1); + ST_SH2(vec0, vec1, (tmp_odd_buf + 4 * 8), 8); + + SUB2(reg5, reg4, reg3, reg2, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_24_64, cospi_8_64, vec0, vec1); + ST_SH2(vec0, vec1, (tmp_odd_buf), 8); + + /* 4 Stores */ + DOTP_CONST_PAIR(reg7, reg0, cospi_28_64, cospi_4_64, reg0, reg7); + DOTP_CONST_PAIR(reg6, reg1, -cospi_4_64, cospi_28_64, reg1, reg6); + BUTTERFLY_4(reg0, reg7, reg6, reg1, vec0, vec1, vec2, vec3); + ST_SH2(vec0, vec1, (tmp_odd_buf + 6 * 8), 8); + + DOTP_CONST_PAIR(vec2, vec3, cospi_24_64, cospi_8_64, vec2, vec3); + ST_SH2(vec2, vec3, (tmp_odd_buf + 2 * 8), 8); + + /* Odd stage 2 */ + /* 8 loads */ + reg0 = LD_SH(tmp_buf + 3 * 8); + reg1 = LD_SH(tmp_buf + 5 * 8); + reg2 = LD_SH(tmp_buf + 11 * 8); + reg3 = LD_SH(tmp_buf + 13 * 8); + reg4 = LD_SH(tmp_buf + 19 * 8); + reg5 = LD_SH(tmp_buf + 21 * 8); + reg6 = LD_SH(tmp_buf + 27 * 8); + reg7 = LD_SH(tmp_buf + 29 * 8); + + DOTP_CONST_PAIR(reg1, reg6, cospi_27_64, cospi_5_64, reg1, reg6); + DOTP_CONST_PAIR(reg5, reg2, cospi_11_64, cospi_21_64, reg2, reg5); + DOTP_CONST_PAIR(reg3, reg4, cospi_19_64, cospi_13_64, reg3, reg4); + DOTP_CONST_PAIR(reg7, reg0, cospi_3_64, cospi_29_64, reg0, reg7); + + /* 4 Stores */ + SUB4(reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, vec0, vec1, vec2, vec3); + DOTP_CONST_PAIR(vec1, vec0, cospi_12_64, cospi_20_64, loc0, loc1); + DOTP_CONST_PAIR(vec3, vec2, -cospi_20_64, cospi_12_64, loc2, loc3); + + BUTTERFLY_4(loc3, loc2, loc0, loc1, vec1, vec0, vec2, vec3); + ST_SH2(vec0, vec1, (tmp_odd_buf + 12 * 8), 3 * 8); + + DOTP_CONST_PAIR(vec3, vec2, -cospi_8_64, cospi_24_64, vec0, vec1); + ST_SH2(vec0, vec1, (tmp_odd_buf + 10 * 8), 8); + + /* 4 Stores */ + ADD4(reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, vec1, vec2, vec0, vec3); + BUTTERFLY_4(vec0, vec3, vec2, vec1, reg0, reg1, reg3, reg2); + ST_SH(reg0, (tmp_odd_buf + 13 * 8)); + ST_SH(reg1, (tmp_odd_buf + 14 * 8)); + + DOTP_CONST_PAIR(reg3, reg2, -cospi_8_64, cospi_24_64, reg0, reg1); + ST_SH2(reg0, reg1, (tmp_odd_buf + 8 * 8), 8); + + /* Odd stage 3 : Dependency on Odd stage 1 & Odd stage 2 */ + + /* Load 8 & Store 8 */ + LD_SH4(tmp_odd_buf, 8, reg0, reg1, reg2, reg3); + LD_SH4((tmp_odd_buf + 8 * 8), 8, reg4, reg5, reg6, reg7); + + ADD4(reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0, loc1, loc2, loc3); + ST_SH4(loc0, loc1, loc2, loc3, tmp_odd_buf, 8); + + SUB2(reg0, reg4, reg1, reg5, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + + SUB2(reg2, reg6, reg3, reg7, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + ST_SH4(loc0, loc1, loc2, loc3, (tmp_odd_buf + 8 * 8), 8); + + /* Load 8 & Store 8 */ + LD_SH4((tmp_odd_buf + 4 * 8), 8, reg1, reg2, reg0, reg3); + LD_SH4((tmp_odd_buf + 12 * 8), 8, reg4, reg5, reg6, reg7); + + ADD4(reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0, loc1, loc2, loc3); + ST_SH4(loc0, loc1, loc2, loc3, (tmp_odd_buf + 4 * 8), 8); + + SUB2(reg0, reg4, reg3, reg7, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + + SUB2(reg1, reg5, reg2, reg6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + ST_SH4(loc0, loc1, loc2, loc3, (tmp_odd_buf + 12 * 8), 8); +} + +static void idct_butterfly_transpose_store(int16_t *tmp_buf, + int16_t *tmp_eve_buf, + int16_t *tmp_odd_buf, int16_t *dst) { + v8i16 vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + v8i16 m0, m1, m2, m3, m4, m5, m6, m7, n0, n1, n2, n3, n4, n5, n6, n7; + + /* FINAL BUTTERFLY : Dependency on Even & Odd */ + vec0 = LD_SH(tmp_odd_buf); + vec1 = LD_SH(tmp_odd_buf + 9 * 8); + vec2 = LD_SH(tmp_odd_buf + 14 * 8); + vec3 = LD_SH(tmp_odd_buf + 6 * 8); + loc0 = LD_SH(tmp_eve_buf); + loc1 = LD_SH(tmp_eve_buf + 8 * 8); + loc2 = LD_SH(tmp_eve_buf + 4 * 8); + loc3 = LD_SH(tmp_eve_buf + 12 * 8); + + ADD4(loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m0, m4, m2, m6); + + ST_SH((loc0 - vec3), (tmp_buf + 31 * 8)); + ST_SH((loc1 - vec2), (tmp_buf + 23 * 8)); + ST_SH((loc2 - vec1), (tmp_buf + 27 * 8)); + ST_SH((loc3 - vec0), (tmp_buf + 19 * 8)); + + /* Load 8 & Store 8 */ + vec0 = LD_SH(tmp_odd_buf + 4 * 8); + vec1 = LD_SH(tmp_odd_buf + 13 * 8); + vec2 = LD_SH(tmp_odd_buf + 10 * 8); + vec3 = LD_SH(tmp_odd_buf + 3 * 8); + loc0 = LD_SH(tmp_eve_buf + 2 * 8); + loc1 = LD_SH(tmp_eve_buf + 10 * 8); + loc2 = LD_SH(tmp_eve_buf + 6 * 8); + loc3 = LD_SH(tmp_eve_buf + 14 * 8); + + ADD4(loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m1, m5, m3, m7); + + ST_SH((loc0 - vec3), (tmp_buf + 29 * 8)); + ST_SH((loc1 - vec2), (tmp_buf + 21 * 8)); + ST_SH((loc2 - vec1), (tmp_buf + 25 * 8)); + ST_SH((loc3 - vec0), (tmp_buf + 17 * 8)); + + /* Load 8 & Store 8 */ + vec0 = LD_SH(tmp_odd_buf + 2 * 8); + vec1 = LD_SH(tmp_odd_buf + 11 * 8); + vec2 = LD_SH(tmp_odd_buf + 12 * 8); + vec3 = LD_SH(tmp_odd_buf + 7 * 8); + loc0 = LD_SH(tmp_eve_buf + 1 * 8); + loc1 = LD_SH(tmp_eve_buf + 9 * 8); + loc2 = LD_SH(tmp_eve_buf + 5 * 8); + loc3 = LD_SH(tmp_eve_buf + 13 * 8); + + ADD4(loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n0, n4, n2, n6); + + ST_SH((loc0 - vec3), (tmp_buf + 30 * 8)); + ST_SH((loc1 - vec2), (tmp_buf + 22 * 8)); + ST_SH((loc2 - vec1), (tmp_buf + 26 * 8)); + ST_SH((loc3 - vec0), (tmp_buf + 18 * 8)); + + /* Load 8 & Store 8 */ + vec0 = LD_SH(tmp_odd_buf + 5 * 8); + vec1 = LD_SH(tmp_odd_buf + 15 * 8); + vec2 = LD_SH(tmp_odd_buf + 8 * 8); + vec3 = LD_SH(tmp_odd_buf + 1 * 8); + loc0 = LD_SH(tmp_eve_buf + 3 * 8); + loc1 = LD_SH(tmp_eve_buf + 11 * 8); + loc2 = LD_SH(tmp_eve_buf + 7 * 8); + loc3 = LD_SH(tmp_eve_buf + 15 * 8); + + ADD4(loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n1, n5, n3, n7); + + ST_SH((loc0 - vec3), (tmp_buf + 28 * 8)); + ST_SH((loc1 - vec2), (tmp_buf + 20 * 8)); + ST_SH((loc2 - vec1), (tmp_buf + 24 * 8)); + ST_SH((loc3 - vec0), (tmp_buf + 16 * 8)); + + /* Transpose : 16 vectors */ + /* 1st & 2nd 8x8 */ + TRANSPOSE8x8_SH_SH(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3, + n3); + ST_SH4(m0, n0, m1, n1, (dst + 0), 32); + ST_SH4(m2, n2, m3, n3, (dst + 4 * 32), 32); + + TRANSPOSE8x8_SH_SH(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7, + n7); + ST_SH4(m4, n4, m5, n5, (dst + 8), 32); + ST_SH4(m6, n6, m7, n7, (dst + 8 + 4 * 32), 32); + + /* 3rd & 4th 8x8 */ + LD_SH8((tmp_buf + 8 * 16), 8, m0, n0, m1, n1, m2, n2, m3, n3); + LD_SH8((tmp_buf + 12 * 16), 8, m4, n4, m5, n5, m6, n6, m7, n7); + TRANSPOSE8x8_SH_SH(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3, + n3); + ST_SH4(m0, n0, m1, n1, (dst + 16), 32); + ST_SH4(m2, n2, m3, n3, (dst + 16 + 4 * 32), 32); + + TRANSPOSE8x8_SH_SH(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7, + n7); + ST_SH4(m4, n4, m5, n5, (dst + 24), 32); + ST_SH4(m6, n6, m7, n7, (dst + 24 + 4 * 32), 32); +} + +static void idct32x8_1d_rows_msa(const int16_t *input, int16_t *output) { + DECLARE_ALIGNED(32, int16_t, tmp_buf[8 * 32]); + DECLARE_ALIGNED(32, int16_t, tmp_odd_buf[16 * 8]); + DECLARE_ALIGNED(32, int16_t, tmp_eve_buf[16 * 8]); + + idct32x8_row_transpose_store(input, &tmp_buf[0]); + idct32x8_row_even_process_store(&tmp_buf[0], &tmp_eve_buf[0]); + idct32x8_row_odd_process_store(&tmp_buf[0], &tmp_odd_buf[0]); + idct_butterfly_transpose_store(&tmp_buf[0], &tmp_eve_buf[0], &tmp_odd_buf[0], + output); +} + +static void idct8x32_column_even_process_store(int16_t *tmp_buf, + int16_t *tmp_eve_buf) { + v8i16 vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + v8i16 reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + v8i16 stp0, stp1, stp2, stp3, stp4, stp5, stp6, stp7; + + /* Even stage 1 */ + LD_SH8(tmp_buf, (4 * 32), reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7); + tmp_buf += (2 * 32); + + DOTP_CONST_PAIR(reg1, reg7, cospi_28_64, cospi_4_64, reg1, reg7); + DOTP_CONST_PAIR(reg5, reg3, cospi_12_64, cospi_20_64, reg5, reg3); + BUTTERFLY_4(reg1, reg7, reg3, reg5, vec1, vec3, vec2, vec0); + DOTP_CONST_PAIR(vec2, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + + loc1 = vec3; + loc0 = vec1; + + DOTP_CONST_PAIR(reg0, reg4, cospi_16_64, cospi_16_64, reg0, reg4); + DOTP_CONST_PAIR(reg2, reg6, cospi_24_64, cospi_8_64, reg2, reg6); + BUTTERFLY_4(reg4, reg0, reg2, reg6, vec1, vec3, vec2, vec0); + BUTTERFLY_4(vec0, vec1, loc1, loc0, stp3, stp0, stp7, stp4); + BUTTERFLY_4(vec2, vec3, loc3, loc2, stp2, stp1, stp6, stp5); + + /* Even stage 2 */ + /* Load 8 */ + LD_SH8(tmp_buf, (4 * 32), reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7); + + DOTP_CONST_PAIR(reg0, reg7, cospi_30_64, cospi_2_64, reg0, reg7); + DOTP_CONST_PAIR(reg4, reg3, cospi_14_64, cospi_18_64, reg4, reg3); + DOTP_CONST_PAIR(reg2, reg5, cospi_22_64, cospi_10_64, reg2, reg5); + DOTP_CONST_PAIR(reg6, reg1, cospi_6_64, cospi_26_64, reg6, reg1); + + vec0 = reg0 + reg4; + reg0 = reg0 - reg4; + reg4 = reg6 + reg2; + reg6 = reg6 - reg2; + reg2 = reg1 + reg5; + reg1 = reg1 - reg5; + reg5 = reg7 + reg3; + reg7 = reg7 - reg3; + reg3 = vec0; + + vec1 = reg2; + reg2 = reg3 + reg4; + reg3 = reg3 - reg4; + reg4 = reg5 - vec1; + reg5 = reg5 + vec1; + + DOTP_CONST_PAIR(reg7, reg0, cospi_24_64, cospi_8_64, reg0, reg7); + DOTP_CONST_PAIR((-reg6), reg1, cospi_24_64, cospi_8_64, reg6, reg1); + + vec0 = reg0 - reg6; + reg0 = reg0 + reg6; + vec1 = reg7 - reg1; + reg7 = reg7 + reg1; + + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, reg6, reg1); + DOTP_CONST_PAIR(reg4, reg3, cospi_16_64, cospi_16_64, reg3, reg4); + + /* Even stage 3 : Dependency on Even stage 1 & Even stage 2 */ + /* Store 8 */ + BUTTERFLY_4(stp0, stp1, reg7, reg5, loc1, loc3, loc2, loc0); + ST_SH2(loc1, loc3, tmp_eve_buf, 8); + ST_SH2(loc2, loc0, (tmp_eve_buf + 14 * 8), 8); + + BUTTERFLY_4(stp2, stp3, reg4, reg1, loc1, loc3, loc2, loc0); + ST_SH2(loc1, loc3, (tmp_eve_buf + 2 * 8), 8); + ST_SH2(loc2, loc0, (tmp_eve_buf + 12 * 8), 8); + + /* Store 8 */ + BUTTERFLY_4(stp4, stp5, reg6, reg3, loc1, loc3, loc2, loc0); + ST_SH2(loc1, loc3, (tmp_eve_buf + 4 * 8), 8); + ST_SH2(loc2, loc0, (tmp_eve_buf + 10 * 8), 8); + + BUTTERFLY_4(stp6, stp7, reg2, reg0, loc1, loc3, loc2, loc0); + ST_SH2(loc1, loc3, (tmp_eve_buf + 6 * 8), 8); + ST_SH2(loc2, loc0, (tmp_eve_buf + 8 * 8), 8); +} + +static void idct8x32_column_odd_process_store(int16_t *tmp_buf, + int16_t *tmp_odd_buf) { + v8i16 vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + v8i16 reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + + /* Odd stage 1 */ + reg0 = LD_SH(tmp_buf + 32); + reg1 = LD_SH(tmp_buf + 7 * 32); + reg2 = LD_SH(tmp_buf + 9 * 32); + reg3 = LD_SH(tmp_buf + 15 * 32); + reg4 = LD_SH(tmp_buf + 17 * 32); + reg5 = LD_SH(tmp_buf + 23 * 32); + reg6 = LD_SH(tmp_buf + 25 * 32); + reg7 = LD_SH(tmp_buf + 31 * 32); + + DOTP_CONST_PAIR(reg0, reg7, cospi_31_64, cospi_1_64, reg0, reg7); + DOTP_CONST_PAIR(reg4, reg3, cospi_15_64, cospi_17_64, reg3, reg4); + DOTP_CONST_PAIR(reg2, reg5, cospi_23_64, cospi_9_64, reg2, reg5); + DOTP_CONST_PAIR(reg6, reg1, cospi_7_64, cospi_25_64, reg1, reg6); + + vec0 = reg0 + reg3; + reg0 = reg0 - reg3; + reg3 = reg7 + reg4; + reg7 = reg7 - reg4; + reg4 = reg1 + reg2; + reg1 = reg1 - reg2; + reg2 = reg6 + reg5; + reg6 = reg6 - reg5; + reg5 = vec0; + + /* 4 Stores */ + ADD2(reg5, reg4, reg3, reg2, vec0, vec1); + ST_SH2(vec0, vec1, (tmp_odd_buf + 4 * 8), 8); + SUB2(reg5, reg4, reg3, reg2, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_24_64, cospi_8_64, vec0, vec1); + ST_SH2(vec0, vec1, tmp_odd_buf, 8); + + /* 4 Stores */ + DOTP_CONST_PAIR(reg7, reg0, cospi_28_64, cospi_4_64, reg0, reg7); + DOTP_CONST_PAIR(reg6, reg1, -cospi_4_64, cospi_28_64, reg1, reg6); + BUTTERFLY_4(reg0, reg7, reg6, reg1, vec0, vec1, vec2, vec3); + ST_SH2(vec0, vec1, (tmp_odd_buf + 6 * 8), 8); + DOTP_CONST_PAIR(vec2, vec3, cospi_24_64, cospi_8_64, vec2, vec3); + ST_SH2(vec2, vec3, (tmp_odd_buf + 2 * 8), 8); + + /* Odd stage 2 */ + /* 8 loads */ + reg0 = LD_SH(tmp_buf + 3 * 32); + reg1 = LD_SH(tmp_buf + 5 * 32); + reg2 = LD_SH(tmp_buf + 11 * 32); + reg3 = LD_SH(tmp_buf + 13 * 32); + reg4 = LD_SH(tmp_buf + 19 * 32); + reg5 = LD_SH(tmp_buf + 21 * 32); + reg6 = LD_SH(tmp_buf + 27 * 32); + reg7 = LD_SH(tmp_buf + 29 * 32); + + DOTP_CONST_PAIR(reg1, reg6, cospi_27_64, cospi_5_64, reg1, reg6); + DOTP_CONST_PAIR(reg5, reg2, cospi_11_64, cospi_21_64, reg2, reg5); + DOTP_CONST_PAIR(reg3, reg4, cospi_19_64, cospi_13_64, reg3, reg4); + DOTP_CONST_PAIR(reg7, reg0, cospi_3_64, cospi_29_64, reg0, reg7); + + /* 4 Stores */ + SUB4(reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, vec0, vec1, vec2, vec3); + DOTP_CONST_PAIR(vec1, vec0, cospi_12_64, cospi_20_64, loc0, loc1); + DOTP_CONST_PAIR(vec3, vec2, -cospi_20_64, cospi_12_64, loc2, loc3); + BUTTERFLY_4(loc2, loc3, loc1, loc0, vec0, vec1, vec3, vec2); + ST_SH2(vec0, vec1, (tmp_odd_buf + 12 * 8), 3 * 8); + DOTP_CONST_PAIR(vec3, vec2, -cospi_8_64, cospi_24_64, vec0, vec1); + ST_SH2(vec0, vec1, (tmp_odd_buf + 10 * 8), 8); + + /* 4 Stores */ + ADD4(reg0, reg3, reg1, reg2, reg5, reg6, reg4, reg7, vec0, vec1, vec2, vec3); + BUTTERFLY_4(vec0, vec3, vec2, vec1, reg0, reg1, reg3, reg2); + ST_SH2(reg0, reg1, (tmp_odd_buf + 13 * 8), 8); + DOTP_CONST_PAIR(reg3, reg2, -cospi_8_64, cospi_24_64, reg0, reg1); + ST_SH2(reg0, reg1, (tmp_odd_buf + 8 * 8), 8); + + /* Odd stage 3 : Dependency on Odd stage 1 & Odd stage 2 */ + /* Load 8 & Store 8 */ + LD_SH4(tmp_odd_buf, 8, reg0, reg1, reg2, reg3); + LD_SH4((tmp_odd_buf + 8 * 8), 8, reg4, reg5, reg6, reg7); + + ADD4(reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0, loc1, loc2, loc3); + ST_SH4(loc0, loc1, loc2, loc3, tmp_odd_buf, 8); + + SUB2(reg0, reg4, reg1, reg5, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + + SUB2(reg2, reg6, reg3, reg7, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + ST_SH4(loc0, loc1, loc2, loc3, (tmp_odd_buf + 8 * 8), 8); + + /* Load 8 & Store 8 */ + LD_SH4((tmp_odd_buf + 4 * 8), 8, reg1, reg2, reg0, reg3); + LD_SH4((tmp_odd_buf + 12 * 8), 8, reg4, reg5, reg6, reg7); + + ADD4(reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0, loc1, loc2, loc3); + ST_SH4(loc0, loc1, loc2, loc3, (tmp_odd_buf + 4 * 8), 8); + + SUB2(reg0, reg4, reg3, reg7, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + + SUB2(reg1, reg5, reg2, reg6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + ST_SH4(loc0, loc1, loc2, loc3, (tmp_odd_buf + 12 * 8), 8); +} + +static void idct8x32_column_butterfly_addblk(int16_t *tmp_eve_buf, + int16_t *tmp_odd_buf, uint8_t *dst, + int32_t dst_stride) { + v8i16 vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + v8i16 m0, m1, m2, m3, m4, m5, m6, m7, n0, n1, n2, n3, n4, n5, n6, n7; + + /* FINAL BUTTERFLY : Dependency on Even & Odd */ + vec0 = LD_SH(tmp_odd_buf); + vec1 = LD_SH(tmp_odd_buf + 9 * 8); + vec2 = LD_SH(tmp_odd_buf + 14 * 8); + vec3 = LD_SH(tmp_odd_buf + 6 * 8); + loc0 = LD_SH(tmp_eve_buf); + loc1 = LD_SH(tmp_eve_buf + 8 * 8); + loc2 = LD_SH(tmp_eve_buf + 4 * 8); + loc3 = LD_SH(tmp_eve_buf + 12 * 8); + + ADD4(loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m0, m4, m2, m6); + SRARI_H4_SH(m0, m2, m4, m6, 6); + AOM_ADDBLK_ST8x4_UB(dst, (4 * dst_stride), m0, m2, m4, m6); + + SUB4(loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m6, m2, m4, m0); + SRARI_H4_SH(m0, m2, m4, m6, 6); + AOM_ADDBLK_ST8x4_UB((dst + 19 * dst_stride), (4 * dst_stride), m0, m2, m4, + m6); + + /* Load 8 & Store 8 */ + vec0 = LD_SH(tmp_odd_buf + 4 * 8); + vec1 = LD_SH(tmp_odd_buf + 13 * 8); + vec2 = LD_SH(tmp_odd_buf + 10 * 8); + vec3 = LD_SH(tmp_odd_buf + 3 * 8); + loc0 = LD_SH(tmp_eve_buf + 2 * 8); + loc1 = LD_SH(tmp_eve_buf + 10 * 8); + loc2 = LD_SH(tmp_eve_buf + 6 * 8); + loc3 = LD_SH(tmp_eve_buf + 14 * 8); + + ADD4(loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m1, m5, m3, m7); + SRARI_H4_SH(m1, m3, m5, m7, 6); + AOM_ADDBLK_ST8x4_UB((dst + 2 * dst_stride), (4 * dst_stride), m1, m3, m5, m7); + + SUB4(loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m7, m3, m5, m1); + SRARI_H4_SH(m1, m3, m5, m7, 6); + AOM_ADDBLK_ST8x4_UB((dst + 17 * dst_stride), (4 * dst_stride), m1, m3, m5, + m7); + + /* Load 8 & Store 8 */ + vec0 = LD_SH(tmp_odd_buf + 2 * 8); + vec1 = LD_SH(tmp_odd_buf + 11 * 8); + vec2 = LD_SH(tmp_odd_buf + 12 * 8); + vec3 = LD_SH(tmp_odd_buf + 7 * 8); + loc0 = LD_SH(tmp_eve_buf + 1 * 8); + loc1 = LD_SH(tmp_eve_buf + 9 * 8); + loc2 = LD_SH(tmp_eve_buf + 5 * 8); + loc3 = LD_SH(tmp_eve_buf + 13 * 8); + + ADD4(loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n0, n4, n2, n6); + SRARI_H4_SH(n0, n2, n4, n6, 6); + AOM_ADDBLK_ST8x4_UB((dst + 1 * dst_stride), (4 * dst_stride), n0, n2, n4, n6); + + SUB4(loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n6, n2, n4, n0); + SRARI_H4_SH(n0, n2, n4, n6, 6); + AOM_ADDBLK_ST8x4_UB((dst + 18 * dst_stride), (4 * dst_stride), n0, n2, n4, + n6); + + /* Load 8 & Store 8 */ + vec0 = LD_SH(tmp_odd_buf + 5 * 8); + vec1 = LD_SH(tmp_odd_buf + 15 * 8); + vec2 = LD_SH(tmp_odd_buf + 8 * 8); + vec3 = LD_SH(tmp_odd_buf + 1 * 8); + loc0 = LD_SH(tmp_eve_buf + 3 * 8); + loc1 = LD_SH(tmp_eve_buf + 11 * 8); + loc2 = LD_SH(tmp_eve_buf + 7 * 8); + loc3 = LD_SH(tmp_eve_buf + 15 * 8); + + ADD4(loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n1, n5, n3, n7); + SRARI_H4_SH(n1, n3, n5, n7, 6); + AOM_ADDBLK_ST8x4_UB((dst + 3 * dst_stride), (4 * dst_stride), n1, n3, n5, n7); + + SUB4(loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n7, n3, n5, n1); + SRARI_H4_SH(n1, n3, n5, n7, 6); + AOM_ADDBLK_ST8x4_UB((dst + 16 * dst_stride), (4 * dst_stride), n1, n3, n5, + n7); +} + +static void idct8x32_1d_columns_addblk_msa(int16_t *input, uint8_t *dst, + int32_t dst_stride) { + DECLARE_ALIGNED(32, int16_t, tmp_odd_buf[16 * 8]); + DECLARE_ALIGNED(32, int16_t, tmp_eve_buf[16 * 8]); + + idct8x32_column_even_process_store(input, &tmp_eve_buf[0]); + idct8x32_column_odd_process_store(input, &tmp_odd_buf[0]); + idct8x32_column_butterfly_addblk(&tmp_eve_buf[0], &tmp_odd_buf[0], dst, + dst_stride); +} + +void aom_idct32x32_1024_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, out_arr[32 * 32]); + int16_t *out_ptr = out_arr; + + /* transform rows */ + for (i = 0; i < 4; ++i) { + /* process 32 * 8 block */ + idct32x8_1d_rows_msa((input + (i << 8)), (out_ptr + (i << 8))); + } + + /* transform columns */ + for (i = 0; i < 4; ++i) { + /* process 8 * 32 block */ + idct8x32_1d_columns_addblk_msa((out_ptr + (i << 3)), (dst + (i << 3)), + dst_stride); + } +} + +void aom_idct32x32_34_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, out_arr[32 * 32]); + int16_t *out_ptr = out_arr; + + for (i = 32; i--;) { + __asm__ __volatile__( + "sw $zero, 0(%[out_ptr]) \n\t" + "sw $zero, 4(%[out_ptr]) \n\t" + "sw $zero, 8(%[out_ptr]) \n\t" + "sw $zero, 12(%[out_ptr]) \n\t" + "sw $zero, 16(%[out_ptr]) \n\t" + "sw $zero, 20(%[out_ptr]) \n\t" + "sw $zero, 24(%[out_ptr]) \n\t" + "sw $zero, 28(%[out_ptr]) \n\t" + "sw $zero, 32(%[out_ptr]) \n\t" + "sw $zero, 36(%[out_ptr]) \n\t" + "sw $zero, 40(%[out_ptr]) \n\t" + "sw $zero, 44(%[out_ptr]) \n\t" + "sw $zero, 48(%[out_ptr]) \n\t" + "sw $zero, 52(%[out_ptr]) \n\t" + "sw $zero, 56(%[out_ptr]) \n\t" + "sw $zero, 60(%[out_ptr]) \n\t" + + : + : [out_ptr] "r"(out_ptr)); + + out_ptr += 32; + } + + out_ptr = out_arr; + + /* rows: only upper-left 8x8 has non-zero coeff */ + idct32x8_1d_rows_msa(input, out_ptr); + + /* transform columns */ + for (i = 0; i < 4; ++i) { + /* process 8 * 32 block */ + idct8x32_1d_columns_addblk_msa((out_ptr + (i << 3)), (dst + (i << 3)), + dst_stride); + } +} + +void aom_idct32x32_1_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int32_t i; + int16_t out; + v16u8 dst0, dst1, dst2, dst3, tmp0, tmp1, tmp2, tmp3; + v8i16 res0, res1, res2, res3, res4, res5, res6, res7, vec; + + out = ROUND_POWER_OF_TWO((input[0] * cospi_16_64), DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO((out * cospi_16_64), DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO(out, 6); + + vec = __msa_fill_h(out); + + for (i = 16; i--;) { + LD_UB2(dst, 16, dst0, dst1); + LD_UB2(dst + dst_stride, 16, dst2, dst3); + + UNPCK_UB_SH(dst0, res0, res4); + UNPCK_UB_SH(dst1, res1, res5); + UNPCK_UB_SH(dst2, res2, res6); + UNPCK_UB_SH(dst3, res3, res7); + ADD4(res0, vec, res1, vec, res2, vec, res3, vec, res0, res1, res2, res3); + ADD4(res4, vec, res5, vec, res6, vec, res7, vec, res4, res5, res6, res7); + CLIP_SH4_0_255(res0, res1, res2, res3); + CLIP_SH4_0_255(res4, res5, res6, res7); + PCKEV_B4_UB(res4, res0, res5, res1, res6, res2, res7, res3, tmp0, tmp1, + tmp2, tmp3); + + ST_UB2(tmp0, tmp1, dst, 16); + dst += dst_stride; + ST_UB2(tmp2, tmp3, dst, 16); + dst += dst_stride; + } +} diff --git a/third_party/aom/aom_dsp/mips/idct4x4_msa.c b/third_party/aom/aom_dsp/mips/idct4x4_msa.c new file mode 100644 index 0000000000..274818baae --- /dev/null +++ b/third_party/aom/aom_dsp/mips/idct4x4_msa.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/inv_txfm_msa.h" + +void aom_iwht4x4_16_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + v8i16 in0, in1, in2, in3; + v4i32 in0_r, in1_r, in2_r, in3_r, in4_r; + + /* load vector elements of 4x4 block */ + LD4x4_SH(input, in0, in2, in3, in1); + TRANSPOSE4x4_SH_SH(in0, in2, in3, in1, in0, in2, in3, in1); + UNPCK_R_SH_SW(in0, in0_r); + UNPCK_R_SH_SW(in2, in2_r); + UNPCK_R_SH_SW(in3, in3_r); + UNPCK_R_SH_SW(in1, in1_r); + SRA_4V(in0_r, in1_r, in2_r, in3_r, UNIT_QUANT_SHIFT); + + in0_r += in2_r; + in3_r -= in1_r; + in4_r = (in0_r - in3_r) >> 1; + in1_r = in4_r - in1_r; + in2_r = in4_r - in2_r; + in0_r -= in1_r; + in3_r += in2_r; + + TRANSPOSE4x4_SW_SW(in0_r, in1_r, in2_r, in3_r, in0_r, in1_r, in2_r, in3_r); + + in0_r += in1_r; + in2_r -= in3_r; + in4_r = (in0_r - in2_r) >> 1; + in3_r = in4_r - in3_r; + in1_r = in4_r - in1_r; + in0_r -= in3_r; + in2_r += in1_r; + + PCKEV_H4_SH(in0_r, in0_r, in1_r, in1_r, in2_r, in2_r, in3_r, in3_r, in0, in1, + in2, in3); + ADDBLK_ST4x4_UB(in0, in3, in1, in2, dst, dst_stride); +} + +void aom_iwht4x4_1_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int16_t a1, e1; + v8i16 in1, in0 = { 0 }; + + a1 = input[0] >> UNIT_QUANT_SHIFT; + e1 = a1 >> 1; + a1 -= e1; + + in0 = __msa_insert_h(in0, 0, a1); + in0 = __msa_insert_h(in0, 1, e1); + in0 = __msa_insert_h(in0, 2, e1); + in0 = __msa_insert_h(in0, 3, e1); + + in1 = in0 >> 1; + in0 -= in1; + + ADDBLK_ST4x4_UB(in0, in1, in1, in1, dst, dst_stride); +} + +void aom_idct4x4_16_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + v8i16 in0, in1, in2, in3; + + /* load vector elements of 4x4 block */ + LD4x4_SH(input, in0, in1, in2, in3); + /* rows */ + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + AOM_IDCT4x4(in0, in1, in2, in3, in0, in1, in2, in3); + /* columns */ + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + AOM_IDCT4x4(in0, in1, in2, in3, in0, in1, in2, in3); + /* rounding (add 2^3, divide by 2^4) */ + SRARI_H4_SH(in0, in1, in2, in3, 4); + ADDBLK_ST4x4_UB(in0, in1, in2, in3, dst, dst_stride); +} + +void aom_idct4x4_1_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int16_t out; + v8i16 vec; + + out = ROUND_POWER_OF_TWO((input[0] * cospi_16_64), DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO((out * cospi_16_64), DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO(out, 4); + vec = __msa_fill_h(out); + + ADDBLK_ST4x4_UB(vec, vec, vec, vec, dst, dst_stride); +} diff --git a/third_party/aom/aom_dsp/mips/idct8x8_msa.c b/third_party/aom/aom_dsp/mips/idct8x8_msa.c new file mode 100644 index 0000000000..981c103cdc --- /dev/null +++ b/third_party/aom/aom_dsp/mips/idct8x8_msa.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/inv_txfm_msa.h" + +void aom_idct8x8_64_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + + /* load vector elements of 8x8 block */ + LD_SH8(input, 8, in0, in1, in2, in3, in4, in5, in6, in7); + + /* rows transform */ + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + /* 1D idct8x8 */ + AOM_IDCT8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + /* columns transform */ + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + /* 1D idct8x8 */ + AOM_IDCT8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + /* final rounding (add 2^4, divide by 2^5) and shift */ + SRARI_H4_SH(in0, in1, in2, in3, 5); + SRARI_H4_SH(in4, in5, in6, in7, 5); + /* add block and store 8x8 */ + AOM_ADDBLK_ST8x4_UB(dst, dst_stride, in0, in1, in2, in3); + dst += (4 * dst_stride); + AOM_ADDBLK_ST8x4_UB(dst, dst_stride, in4, in5, in6, in7); +} + +void aom_idct8x8_12_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + v8i16 s0, s1, s2, s3, s4, s5, s6, s7, k0, k1, k2, k3, m0, m1, m2, m3; + v4i32 tmp0, tmp1, tmp2, tmp3; + v8i16 zero = { 0 }; + + /* load vector elements of 8x8 block */ + LD_SH8(input, 8, in0, in1, in2, in3, in4, in5, in6, in7); + TRANSPOSE8X4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + + /* stage1 */ + ILVL_H2_SH(in3, in0, in2, in1, s0, s1); + k0 = AOM_SET_COSPI_PAIR(cospi_28_64, -cospi_4_64); + k1 = AOM_SET_COSPI_PAIR(cospi_4_64, cospi_28_64); + k2 = AOM_SET_COSPI_PAIR(-cospi_20_64, cospi_12_64); + k3 = AOM_SET_COSPI_PAIR(cospi_12_64, cospi_20_64); + DOTP_SH4_SW(s0, s0, s1, s1, k0, k1, k2, k3, tmp0, tmp1, tmp2, tmp3); + SRARI_W4_SW(tmp0, tmp1, tmp2, tmp3, DCT_CONST_BITS); + PCKEV_H2_SH(zero, tmp0, zero, tmp1, s0, s1); + PCKEV_H2_SH(zero, tmp2, zero, tmp3, s2, s3); + BUTTERFLY_4(s0, s1, s3, s2, s4, s7, s6, s5); + + /* stage2 */ + ILVR_H2_SH(in3, in1, in2, in0, s1, s0); + k0 = AOM_SET_COSPI_PAIR(cospi_16_64, cospi_16_64); + k1 = AOM_SET_COSPI_PAIR(cospi_16_64, -cospi_16_64); + k2 = AOM_SET_COSPI_PAIR(cospi_24_64, -cospi_8_64); + k3 = AOM_SET_COSPI_PAIR(cospi_8_64, cospi_24_64); + DOTP_SH4_SW(s0, s0, s1, s1, k0, k1, k2, k3, tmp0, tmp1, tmp2, tmp3); + SRARI_W4_SW(tmp0, tmp1, tmp2, tmp3, DCT_CONST_BITS); + PCKEV_H2_SH(zero, tmp0, zero, tmp1, s0, s1); + PCKEV_H2_SH(zero, tmp2, zero, tmp3, s2, s3); + BUTTERFLY_4(s0, s1, s2, s3, m0, m1, m2, m3); + + /* stage3 */ + s0 = __msa_ilvr_h(s6, s5); + + k1 = AOM_SET_COSPI_PAIR(-cospi_16_64, cospi_16_64); + DOTP_SH2_SW(s0, s0, k1, k0, tmp0, tmp1); + SRARI_W2_SW(tmp0, tmp1, DCT_CONST_BITS); + PCKEV_H2_SH(zero, tmp0, zero, tmp1, s2, s3); + + /* stage4 */ + BUTTERFLY_8(m0, m1, m2, m3, s4, s2, s3, s7, in0, in1, in2, in3, in4, in5, in6, + in7); + TRANSPOSE4X8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + AOM_IDCT8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + + /* final rounding (add 2^4, divide by 2^5) and shift */ + SRARI_H4_SH(in0, in1, in2, in3, 5); + SRARI_H4_SH(in4, in5, in6, in7, 5); + + /* add block and store 8x8 */ + AOM_ADDBLK_ST8x4_UB(dst, dst_stride, in0, in1, in2, in3); + dst += (4 * dst_stride); + AOM_ADDBLK_ST8x4_UB(dst, dst_stride, in4, in5, in6, in7); +} + +void aom_idct8x8_1_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int16_t out; + int32_t val; + v8i16 vec; + + out = ROUND_POWER_OF_TWO((input[0] * cospi_16_64), DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO((out * cospi_16_64), DCT_CONST_BITS); + val = ROUND_POWER_OF_TWO(out, 5); + vec = __msa_fill_h(val); + + AOM_ADDBLK_ST8x4_UB(dst, dst_stride, vec, vec, vec, vec); + dst += (4 * dst_stride); + AOM_ADDBLK_ST8x4_UB(dst, dst_stride, vec, vec, vec, vec); +} diff --git a/third_party/aom/aom_dsp/mips/intrapred16_dspr2.c b/third_party/aom/aom_dsp/mips/intrapred16_dspr2.c new file mode 100644 index 0000000000..dc8f20208a --- /dev/null +++ b/third_party/aom/aom_dsp/mips/intrapred16_dspr2.c @@ -0,0 +1,325 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/common_dspr2.h" + +#if HAVE_DSPR2 +void aom_h_predictor_16x16_dspr2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int32_t tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; + int32_t tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15, tmp16; + + __asm__ __volatile__( + "lb %[tmp1], (%[left]) \n\t" + "lb %[tmp2], 1(%[left]) \n\t" + "lb %[tmp3], 2(%[left]) \n\t" + "lb %[tmp4], 3(%[left]) \n\t" + "lb %[tmp5], 4(%[left]) \n\t" + "lb %[tmp6], 5(%[left]) \n\t" + "lb %[tmp7], 6(%[left]) \n\t" + "lb %[tmp8], 7(%[left]) \n\t" + "lb %[tmp9], 8(%[left]) \n\t" + "lb %[tmp10], 9(%[left]) \n\t" + "lb %[tmp11], 10(%[left]) \n\t" + "lb %[tmp12], 11(%[left]) \n\t" + "lb %[tmp13], 12(%[left]) \n\t" + "lb %[tmp14], 13(%[left]) \n\t" + "lb %[tmp15], 14(%[left]) \n\t" + "lb %[tmp16], 15(%[left]) \n\t" + + "replv.qb %[tmp1], %[tmp1] \n\t" + "replv.qb %[tmp2], %[tmp2] \n\t" + "replv.qb %[tmp3], %[tmp3] \n\t" + "replv.qb %[tmp4], %[tmp4] \n\t" + "replv.qb %[tmp5], %[tmp5] \n\t" + "replv.qb %[tmp6], %[tmp6] \n\t" + "replv.qb %[tmp7], %[tmp7] \n\t" + "replv.qb %[tmp8], %[tmp8] \n\t" + "replv.qb %[tmp9], %[tmp9] \n\t" + "replv.qb %[tmp10], %[tmp10] \n\t" + "replv.qb %[tmp11], %[tmp11] \n\t" + "replv.qb %[tmp12], %[tmp12] \n\t" + "replv.qb %[tmp13], %[tmp13] \n\t" + "replv.qb %[tmp14], %[tmp14] \n\t" + "replv.qb %[tmp15], %[tmp15] \n\t" + "replv.qb %[tmp16], %[tmp16] \n\t" + + "sw %[tmp1], (%[dst]) \n\t" + "sw %[tmp1], 4(%[dst]) \n\t" + "sw %[tmp1], 8(%[dst]) \n\t" + "sw %[tmp1], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp2], (%[dst]) \n\t" + "sw %[tmp2], 4(%[dst]) \n\t" + "sw %[tmp2], 8(%[dst]) \n\t" + "sw %[tmp2], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp3], (%[dst]) \n\t" + "sw %[tmp3], 4(%[dst]) \n\t" + "sw %[tmp3], 8(%[dst]) \n\t" + "sw %[tmp3], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp4], (%[dst]) \n\t" + "sw %[tmp4], 4(%[dst]) \n\t" + "sw %[tmp4], 8(%[dst]) \n\t" + "sw %[tmp4], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp5], (%[dst]) \n\t" + "sw %[tmp5], 4(%[dst]) \n\t" + "sw %[tmp5], 8(%[dst]) \n\t" + "sw %[tmp5], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp6], (%[dst]) \n\t" + "sw %[tmp6], 4(%[dst]) \n\t" + "sw %[tmp6], 8(%[dst]) \n\t" + "sw %[tmp6], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp7], (%[dst]) \n\t" + "sw %[tmp7], 4(%[dst]) \n\t" + "sw %[tmp7], 8(%[dst]) \n\t" + "sw %[tmp7], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp8], (%[dst]) \n\t" + "sw %[tmp8], 4(%[dst]) \n\t" + "sw %[tmp8], 8(%[dst]) \n\t" + "sw %[tmp8], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp9], (%[dst]) \n\t" + "sw %[tmp9], 4(%[dst]) \n\t" + "sw %[tmp9], 8(%[dst]) \n\t" + "sw %[tmp9], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp10], (%[dst]) \n\t" + "sw %[tmp10], 4(%[dst]) \n\t" + "sw %[tmp10], 8(%[dst]) \n\t" + "sw %[tmp10], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp11], (%[dst]) \n\t" + "sw %[tmp11], 4(%[dst]) \n\t" + "sw %[tmp11], 8(%[dst]) \n\t" + "sw %[tmp11], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp12], (%[dst]) \n\t" + "sw %[tmp12], 4(%[dst]) \n\t" + "sw %[tmp12], 8(%[dst]) \n\t" + "sw %[tmp12], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp13], (%[dst]) \n\t" + "sw %[tmp13], 4(%[dst]) \n\t" + "sw %[tmp13], 8(%[dst]) \n\t" + "sw %[tmp13], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp14], (%[dst]) \n\t" + "sw %[tmp14], 4(%[dst]) \n\t" + "sw %[tmp14], 8(%[dst]) \n\t" + "sw %[tmp14], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp15], (%[dst]) \n\t" + "sw %[tmp15], 4(%[dst]) \n\t" + "sw %[tmp15], 8(%[dst]) \n\t" + "sw %[tmp15], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp16], (%[dst]) \n\t" + "sw %[tmp16], 4(%[dst]) \n\t" + "sw %[tmp16], 8(%[dst]) \n\t" + "sw %[tmp16], 12(%[dst]) \n\t" + + : [tmp1] "=&r"(tmp1), [tmp2] "=&r"(tmp2), [tmp3] "=&r"(tmp3), + [tmp4] "=&r"(tmp4), [tmp5] "=&r"(tmp5), [tmp7] "=&r"(tmp7), + [tmp6] "=&r"(tmp6), [tmp8] "=&r"(tmp8), [tmp9] "=&r"(tmp9), + [tmp10] "=&r"(tmp10), [tmp11] "=&r"(tmp11), [tmp12] "=&r"(tmp12), + [tmp13] "=&r"(tmp13), [tmp14] "=&r"(tmp14), [tmp15] "=&r"(tmp15), + [tmp16] "=&r"(tmp16) + : [left] "r"(left), [dst] "r"(dst), [stride] "r"(stride)); +} + +void aom_dc_predictor_16x16_dspr2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int32_t expected_dc; + int32_t average; + int32_t tmp, above1, above_l1, above_r1, left1, left_r1, left_l1; + int32_t above2, left2; + + __asm__ __volatile__( + "lw %[above1], (%[above]) \n\t" + "lw %[above2], 4(%[above]) \n\t" + "lw %[left1], (%[left]) \n\t" + "lw %[left2], 4(%[left]) \n\t" + + "preceu.ph.qbl %[above_l1], %[above1] \n\t" + "preceu.ph.qbr %[above_r1], %[above1] \n\t" + "preceu.ph.qbl %[left_l1], %[left1] \n\t" + "preceu.ph.qbr %[left_r1], %[left1] \n\t" + + "addu.ph %[average], %[above_r1], %[above_l1] \n\t" + "addu.ph %[average], %[average], %[left_l1] \n\t" + "addu.ph %[average], %[average], %[left_r1] \n\t" + + "preceu.ph.qbl %[above_l1], %[above2] \n\t" + "preceu.ph.qbr %[above_r1], %[above2] \n\t" + "preceu.ph.qbl %[left_l1], %[left2] \n\t" + "preceu.ph.qbr %[left_r1], %[left2] \n\t" + + "addu.ph %[average], %[average], %[above_l1] \n\t" + "addu.ph %[average], %[average], %[above_r1] \n\t" + "addu.ph %[average], %[average], %[left_l1] \n\t" + "addu.ph %[average], %[average], %[left_r1] \n\t" + + "lw %[above1], 8(%[above]) \n\t" + "lw %[above2], 12(%[above]) \n\t" + "lw %[left1], 8(%[left]) \n\t" + "lw %[left2], 12(%[left]) \n\t" + + "preceu.ph.qbl %[above_l1], %[above1] \n\t" + "preceu.ph.qbr %[above_r1], %[above1] \n\t" + "preceu.ph.qbl %[left_l1], %[left1] \n\t" + "preceu.ph.qbr %[left_r1], %[left1] \n\t" + + "addu.ph %[average], %[average], %[above_l1] \n\t" + "addu.ph %[average], %[average], %[above_r1] \n\t" + "addu.ph %[average], %[average], %[left_l1] \n\t" + "addu.ph %[average], %[average], %[left_r1] \n\t" + + "preceu.ph.qbl %[above_l1], %[above2] \n\t" + "preceu.ph.qbr %[above_r1], %[above2] \n\t" + "preceu.ph.qbl %[left_l1], %[left2] \n\t" + "preceu.ph.qbr %[left_r1], %[left2] \n\t" + + "addu.ph %[average], %[average], %[above_l1] \n\t" + "addu.ph %[average], %[average], %[above_r1] \n\t" + "addu.ph %[average], %[average], %[left_l1] \n\t" + "addu.ph %[average], %[average], %[left_r1] \n\t" + + "addiu %[average], %[average], 16 \n\t" + "srl %[tmp], %[average], 16 \n\t" + "addu.ph %[average], %[tmp], %[average] \n\t" + "srl %[expected_dc], %[average], 5 \n\t" + "replv.qb %[expected_dc], %[expected_dc] \n\t" + + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + "sw %[expected_dc], 8(%[dst]) \n\t" + "sw %[expected_dc], 12(%[dst]) \n\t" + + : [left1] "=&r"(left1), [above1] "=&r"(above1), [left_l1] "=&r"(left_l1), + [above_l1] "=&r"(above_l1), [left_r1] "=&r"(left_r1), + [above_r1] "=&r"(above_r1), [above2] "=&r"(above2), + [left2] "=&r"(left2), [average] "=&r"(average), [tmp] "=&r"(tmp), + [expected_dc] "=&r"(expected_dc) + : [above] "r"(above), [left] "r"(left), [dst] "r"(dst), + [stride] "r"(stride)); +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/aom_dsp/mips/intrapred4_dspr2.c b/third_party/aom/aom_dsp/mips/intrapred4_dspr2.c new file mode 100644 index 0000000000..ea7c02810a --- /dev/null +++ b/third_party/aom/aom_dsp/mips/intrapred4_dspr2.c @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/common_dspr2.h" + +#if HAVE_DSPR2 +void aom_h_predictor_4x4_dspr2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int32_t tmp1, tmp2, tmp3, tmp4; + + __asm__ __volatile__( + "lb %[tmp1], (%[left]) \n\t" + "lb %[tmp2], 1(%[left]) \n\t" + "lb %[tmp3], 2(%[left]) \n\t" + "lb %[tmp4], 3(%[left]) \n\t" + "replv.qb %[tmp1], %[tmp1] \n\t" + "replv.qb %[tmp2], %[tmp2] \n\t" + "replv.qb %[tmp3], %[tmp3] \n\t" + "replv.qb %[tmp4], %[tmp4] \n\t" + "sw %[tmp1], (%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp2], (%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp3], (%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp4], (%[dst]) \n\t" + + : [tmp1] "=&r"(tmp1), [tmp2] "=&r"(tmp2), [tmp3] "=&r"(tmp3), + [tmp4] "=&r"(tmp4) + : [left] "r"(left), [dst] "r"(dst), [stride] "r"(stride)); +} + +void aom_dc_predictor_4x4_dspr2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int32_t expected_dc; + int32_t average; + int32_t tmp, above_c, above_l, above_r, left_c, left_r, left_l; + + __asm__ __volatile__( + "lw %[above_c], (%[above]) \n\t" + "lw %[left_c], (%[left]) \n\t" + + "preceu.ph.qbl %[above_l], %[above_c] \n\t" + "preceu.ph.qbr %[above_r], %[above_c] \n\t" + "preceu.ph.qbl %[left_l], %[left_c] \n\t" + "preceu.ph.qbr %[left_r], %[left_c] \n\t" + + "addu.ph %[average], %[above_r], %[above_l] \n\t" + "addu.ph %[average], %[average], %[left_l] \n\t" + "addu.ph %[average], %[average], %[left_r] \n\t" + "addiu %[average], %[average], 4 \n\t" + "srl %[tmp], %[average], 16 \n\t" + "addu.ph %[average], %[tmp], %[average] \n\t" + "srl %[expected_dc], %[average], 3 \n\t" + "replv.qb %[expected_dc], %[expected_dc] \n\t" + + "sw %[expected_dc], (%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + + : [above_c] "=&r"(above_c), [above_l] "=&r"(above_l), + [above_r] "=&r"(above_r), [left_c] "=&r"(left_c), + [left_l] "=&r"(left_l), [left_r] "=&r"(left_r), + [average] "=&r"(average), [tmp] "=&r"(tmp), + [expected_dc] "=&r"(expected_dc) + : [above] "r"(above), [left] "r"(left), [dst] "r"(dst), + [stride] "r"(stride)); +} + +void aom_tm_predictor_4x4_dspr2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int32_t abovel, abover; + int32_t left0, left1, left2, left3; + int32_t res0, res1; + int32_t resl; + int32_t resr; + int32_t top_left; + uint8_t *cm = aom_ff_cropTbl; + + __asm__ __volatile__( + "ulw %[resl], (%[above]) \n\t" + + "lbu %[left0], (%[left]) \n\t" + "lbu %[left1], 1(%[left]) \n\t" + "lbu %[left2], 2(%[left]) \n\t" + "lbu %[left3], 3(%[left]) \n\t" + + "lbu %[top_left], -1(%[above]) \n\t" + + "preceu.ph.qbl %[abovel], %[resl] \n\t" + "preceu.ph.qbr %[abover], %[resl] \n\t" + + "replv.ph %[left0], %[left0] \n\t" + "replv.ph %[left1], %[left1] \n\t" + "replv.ph %[left2], %[left2] \n\t" + "replv.ph %[left3], %[left3] \n\t" + + "replv.ph %[top_left], %[top_left] \n\t" + + "addu.ph %[resl], %[abovel], %[left0] \n\t" + "subu.ph %[resl], %[resl], %[top_left] \n\t" + + "addu.ph %[resr], %[abover], %[left0] \n\t" + "subu.ph %[resr], %[resr], %[top_left] \n\t" + + "sll %[res0], %[resr], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "lbux %[res0], %[res0](%[cm]) \n\t" + + "sra %[res1], %[resr], 16 \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "sb %[res0], (%[dst]) \n\t" + + "sll %[res0], %[resl], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "lbux %[res0], %[res0](%[cm]) \n\t" + "sb %[res1], 1(%[dst]) \n\t" + + "sra %[res1], %[resl], 16 \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + + "addu.ph %[resl], %[abovel], %[left1] \n\t" + "subu.ph %[resl], %[resl], %[top_left] \n\t" + + "addu.ph %[resr], %[abover], %[left1] \n\t" + "subu.ph %[resr], %[resr], %[top_left] \n\t" + + "sb %[res0], 2(%[dst]) \n\t" + "sb %[res1], 3(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + + "sll %[res0], %[resr], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "lbux %[res0], %[res0](%[cm]) \n\t" + + "sra %[res1], %[resr], 16 \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "sb %[res0], (%[dst]) \n\t" + + "sll %[res0], %[resl], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "lbux %[res0], %[res0](%[cm]) \n\t" + + "sb %[res1], 1(%[dst]) \n\t" + "sra %[res1], %[resl], 16 \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + + "addu.ph %[resl], %[abovel], %[left2] \n\t" + "subu.ph %[resl], %[resl], %[top_left] \n\t" + + "addu.ph %[resr], %[abover], %[left2] \n\t" + "subu.ph %[resr], %[resr], %[top_left] \n\t" + + "sb %[res0], 2(%[dst]) \n\t" + "sb %[res1], 3(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + + "sll %[res0], %[resr], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "lbux %[res0], %[res0](%[cm]) \n\t" + + "sra %[res1], %[resr], 16 \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "sb %[res0], (%[dst]) \n\t" + + "sll %[res0], %[resl], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "lbux %[res0], %[res0](%[cm]) \n\t" + + "sb %[res1], 1(%[dst]) \n\t" + "sra %[res1], %[resl], 16 \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + + "addu.ph %[resl], %[abovel], %[left3] \n\t" + "subu.ph %[resl], %[resl], %[top_left] \n\t" + + "addu.ph %[resr], %[abover], %[left3] \n\t" + "subu.ph %[resr], %[resr], %[top_left] \n\t" + + "sb %[res0], 2(%[dst]) \n\t" + "sb %[res1], 3(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + + "sll %[res0], %[resr], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "lbux %[res0], %[res0](%[cm]) \n\t" + + "sra %[res1], %[resr], 16 \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "sb %[res0], (%[dst]) \n\t" + + "sll %[res0], %[resl], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "lbux %[res0], %[res0](%[cm]) \n\t" + "sb %[res1], 1(%[dst]) \n\t" + + "sra %[res1], %[resl], 16 \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + + "sb %[res0], 2(%[dst]) \n\t" + "sb %[res1], 3(%[dst]) \n\t" + + : [abovel] "=&r"(abovel), [abover] "=&r"(abover), [left0] "=&r"(left0), + [left1] "=&r"(left1), [left2] "=&r"(left2), [res0] "=&r"(res0), + [res1] "=&r"(res1), [left3] "=&r"(left3), [resl] "=&r"(resl), + [resr] "=&r"(resr), [top_left] "=&r"(top_left) + : [above] "r"(above), [left] "r"(left), [dst] "r"(dst), + [stride] "r"(stride), [cm] "r"(cm)); +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/aom_dsp/mips/intrapred8_dspr2.c b/third_party/aom/aom_dsp/mips/intrapred8_dspr2.c new file mode 100644 index 0000000000..1114fbc006 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/intrapred8_dspr2.c @@ -0,0 +1,603 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/common_dspr2.h" + +#if HAVE_DSPR2 +void aom_h_predictor_8x8_dspr2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int32_t tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; + + __asm__ __volatile__( + "lb %[tmp1], (%[left]) \n\t" + "lb %[tmp2], 1(%[left]) \n\t" + "lb %[tmp3], 2(%[left]) \n\t" + "lb %[tmp4], 3(%[left]) \n\t" + "lb %[tmp5], 4(%[left]) \n\t" + "lb %[tmp6], 5(%[left]) \n\t" + "lb %[tmp7], 6(%[left]) \n\t" + "lb %[tmp8], 7(%[left]) \n\t" + + "replv.qb %[tmp1], %[tmp1] \n\t" + "replv.qb %[tmp2], %[tmp2] \n\t" + "replv.qb %[tmp3], %[tmp3] \n\t" + "replv.qb %[tmp4], %[tmp4] \n\t" + "replv.qb %[tmp5], %[tmp5] \n\t" + "replv.qb %[tmp6], %[tmp6] \n\t" + "replv.qb %[tmp7], %[tmp7] \n\t" + "replv.qb %[tmp8], %[tmp8] \n\t" + + "sw %[tmp1], (%[dst]) \n\t" + "sw %[tmp1], 4(%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp2], (%[dst]) \n\t" + "sw %[tmp2], 4(%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp3], (%[dst]) \n\t" + "sw %[tmp3], 4(%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp4], (%[dst]) \n\t" + "sw %[tmp4], 4(%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp5], (%[dst]) \n\t" + "sw %[tmp5], 4(%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp6], (%[dst]) \n\t" + "sw %[tmp6], 4(%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp7], (%[dst]) \n\t" + "sw %[tmp7], 4(%[dst]) \n\t" + "add %[dst], %[dst], %[stride] \n\t" + "sw %[tmp8], (%[dst]) \n\t" + "sw %[tmp8], 4(%[dst]) \n\t" + + : [tmp1] "=&r"(tmp1), [tmp2] "=&r"(tmp2), [tmp3] "=&r"(tmp3), + [tmp4] "=&r"(tmp4), [tmp5] "=&r"(tmp5), [tmp7] "=&r"(tmp7), + [tmp6] "=&r"(tmp6), [tmp8] "=&r"(tmp8) + : [left] "r"(left), [dst] "r"(dst), [stride] "r"(stride)); +} + +void aom_dc_predictor_8x8_dspr2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int32_t expected_dc; + int32_t average; + int32_t tmp, above1, above_l1, above_r1, left1, left_r1, left_l1; + int32_t above2, above_l2, above_r2, left2, left_r2, left_l2; + + __asm__ __volatile__( + "lw %[above1], (%[above]) \n\t" + "lw %[above2], 4(%[above]) \n\t" + "lw %[left1], (%[left]) \n\t" + "lw %[left2], 4(%[left]) \n\t" + + "preceu.ph.qbl %[above_l1], %[above1] \n\t" + "preceu.ph.qbr %[above_r1], %[above1] \n\t" + "preceu.ph.qbl %[left_l1], %[left1] \n\t" + "preceu.ph.qbr %[left_r1], %[left1] \n\t" + + "preceu.ph.qbl %[above_l2], %[above2] \n\t" + "preceu.ph.qbr %[above_r2], %[above2] \n\t" + "preceu.ph.qbl %[left_l2], %[left2] \n\t" + "preceu.ph.qbr %[left_r2], %[left2] \n\t" + + "addu.ph %[average], %[above_r1], %[above_l1] \n\t" + "addu.ph %[average], %[average], %[left_l1] \n\t" + "addu.ph %[average], %[average], %[left_r1] \n\t" + + "addu.ph %[average], %[average], %[above_l2] \n\t" + "addu.ph %[average], %[average], %[above_r2] \n\t" + "addu.ph %[average], %[average], %[left_l2] \n\t" + "addu.ph %[average], %[average], %[left_r2] \n\t" + + "addiu %[average], %[average], 8 \n\t" + + "srl %[tmp], %[average], 16 \n\t" + "addu.ph %[average], %[tmp], %[average] \n\t" + "srl %[expected_dc], %[average], 4 \n\t" + "replv.qb %[expected_dc], %[expected_dc] \n\t" + + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + + "add %[dst], %[dst], %[stride] \n\t" + "sw %[expected_dc], (%[dst]) \n\t" + "sw %[expected_dc], 4(%[dst]) \n\t" + + : [above1] "=&r"(above1), [above_l1] "=&r"(above_l1), + [above_r1] "=&r"(above_r1), [left1] "=&r"(left1), + [left_l1] "=&r"(left_l1), [left_r1] "=&r"(left_r1), + [above2] "=&r"(above2), [above_l2] "=&r"(above_l2), + [above_r2] "=&r"(above_r2), [left2] "=&r"(left2), + [left_l2] "=&r"(left_l2), [left_r2] "=&r"(left_r2), + [average] "=&r"(average), [tmp] "=&r"(tmp), + [expected_dc] "=&r"(expected_dc) + : [above] "r"(above), [left] "r"(left), [dst] "r"(dst), + [stride] "r"(stride)); +} + +void aom_tm_predictor_8x8_dspr2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left) { + int32_t abovel, abover; + int32_t abovel_1, abover_1; + int32_t left0; + int32_t res0, res1, res2, res3; + int32_t reshw; + int32_t top_left; + uint8_t *cm = aom_ff_cropTbl; + + __asm__ __volatile__( + "ulw %[reshw], (%[above]) \n\t" + "ulw %[top_left], 4(%[above]) \n\t" + + "lbu %[left0], (%[left]) \n\t" + + "preceu.ph.qbl %[abovel], %[reshw] \n\t" + "preceu.ph.qbr %[abover], %[reshw] \n\t" + "preceu.ph.qbl %[abovel_1], %[top_left] \n\t" + "preceu.ph.qbr %[abover_1], %[top_left] \n\t" + + "lbu %[top_left], -1(%[above]) \n\t" + "replv.ph %[left0], %[left0] \n\t" + + "replv.ph %[top_left], %[top_left] \n\t" + + "addu.ph %[reshw], %[abovel], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], (%[dst]) \n\t" + "sb %[res1], 1(%[dst]) \n\t" + "sb %[res2], 2(%[dst]) \n\t" + "sb %[res3], 3(%[dst]) \n\t" + + "addu.ph %[reshw], %[abovel_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbu %[left0], 1(%[left]) \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], 4(%[dst]) \n\t" + "sb %[res1], 5(%[dst]) \n\t" + "sb %[res2], 6(%[dst]) \n\t" + "sb %[res3], 7(%[dst]) \n\t" + + "replv.ph %[left0], %[left0] \n\t" + "add %[dst], %[dst], %[stride] \n\t" + + "addu.ph %[reshw], %[abovel], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], (%[dst]) \n\t" + "sb %[res1], 1(%[dst]) \n\t" + "sb %[res2], 2(%[dst]) \n\t" + "sb %[res3], 3(%[dst]) \n\t" + + "addu.ph %[reshw], %[abovel_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbu %[left0], 2(%[left]) \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], 4(%[dst]) \n\t" + "sb %[res1], 5(%[dst]) \n\t" + "sb %[res2], 6(%[dst]) \n\t" + "sb %[res3], 7(%[dst]) \n\t" + + "replv.ph %[left0], %[left0] \n\t" + "add %[dst], %[dst], %[stride] \n\t" + + "addu.ph %[reshw], %[abovel], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], (%[dst]) \n\t" + "sb %[res1], 1(%[dst]) \n\t" + "sb %[res2], 2(%[dst]) \n\t" + "sb %[res3], 3(%[dst]) \n\t" + + "addu.ph %[reshw], %[abovel_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbu %[left0], 3(%[left]) \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], 4(%[dst]) \n\t" + "sb %[res1], 5(%[dst]) \n\t" + "sb %[res2], 6(%[dst]) \n\t" + "sb %[res3], 7(%[dst]) \n\t" + + "replv.ph %[left0], %[left0] \n\t" + "add %[dst], %[dst], %[stride] \n\t" + + "addu.ph %[reshw], %[abovel], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], (%[dst]) \n\t" + "sb %[res1], 1(%[dst]) \n\t" + "sb %[res2], 2(%[dst]) \n\t" + "sb %[res3], 3(%[dst]) \n\t" + + "addu.ph %[reshw], %[abovel_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbu %[left0], 4(%[left]) \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], 4(%[dst]) \n\t" + "sb %[res1], 5(%[dst]) \n\t" + "sb %[res2], 6(%[dst]) \n\t" + "sb %[res3], 7(%[dst]) \n\t" + + "replv.ph %[left0], %[left0] \n\t" + "add %[dst], %[dst], %[stride] \n\t" + + "addu.ph %[reshw], %[abovel], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], (%[dst]) \n\t" + "sb %[res1], 1(%[dst]) \n\t" + "sb %[res2], 2(%[dst]) \n\t" + "sb %[res3], 3(%[dst]) \n\t" + + "addu.ph %[reshw], %[abovel_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbu %[left0], 5(%[left]) \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], 4(%[dst]) \n\t" + "sb %[res1], 5(%[dst]) \n\t" + "sb %[res2], 6(%[dst]) \n\t" + "sb %[res3], 7(%[dst]) \n\t" + + "replv.ph %[left0], %[left0] \n\t" + "add %[dst], %[dst], %[stride] \n\t" + + "addu.ph %[reshw], %[abovel], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], (%[dst]) \n\t" + "sb %[res1], 1(%[dst]) \n\t" + "sb %[res2], 2(%[dst]) \n\t" + "sb %[res3], 3(%[dst]) \n\t" + + "addu.ph %[reshw], %[abovel_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbu %[left0], 6(%[left]) \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], 4(%[dst]) \n\t" + "sb %[res1], 5(%[dst]) \n\t" + "sb %[res2], 6(%[dst]) \n\t" + "sb %[res3], 7(%[dst]) \n\t" + + "replv.ph %[left0], %[left0] \n\t" + "add %[dst], %[dst], %[stride] \n\t" + + "addu.ph %[reshw], %[abovel], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], (%[dst]) \n\t" + "sb %[res1], 1(%[dst]) \n\t" + "sb %[res2], 2(%[dst]) \n\t" + "sb %[res3], 3(%[dst]) \n\t" + + "addu.ph %[reshw], %[abovel_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbu %[left0], 7(%[left]) \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], 4(%[dst]) \n\t" + "sb %[res1], 5(%[dst]) \n\t" + "sb %[res2], 6(%[dst]) \n\t" + "sb %[res3], 7(%[dst]) \n\t" + + "replv.ph %[left0], %[left0] \n\t" + "add %[dst], %[dst], %[stride] \n\t" + + "addu.ph %[reshw], %[abovel], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], (%[dst]) \n\t" + "sb %[res1], 1(%[dst]) \n\t" + "sb %[res2], 2(%[dst]) \n\t" + "sb %[res3], 3(%[dst]) \n\t" + + "addu.ph %[reshw], %[abovel_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res2], %[reshw], 16 \n\t" + "sra %[res2], %[res2], 16 \n\t" + "sra %[res3], %[reshw], 16 \n\t" + + "addu.ph %[reshw], %[abover_1], %[left0] \n\t" + "subu.ph %[reshw], %[reshw], %[top_left] \n\t" + + "sll %[res0], %[reshw], 16 \n\t" + "sra %[res0], %[res0], 16 \n\t" + "sra %[res1], %[reshw], 16 \n\t" + + "lbux %[res0], %[res0](%[cm]) \n\t" + "lbux %[res1], %[res1](%[cm]) \n\t" + "lbux %[res2], %[res2](%[cm]) \n\t" + "lbux %[res3], %[res3](%[cm]) \n\t" + + "sb %[res0], 4(%[dst]) \n\t" + "sb %[res1], 5(%[dst]) \n\t" + "sb %[res2], 6(%[dst]) \n\t" + "sb %[res3], 7(%[dst]) \n\t" + + : [abovel] "=&r"(abovel), [abover] "=&r"(abover), + [abovel_1] "=&r"(abovel_1), [abover_1] "=&r"(abover_1), + [left0] "=&r"(left0), [res2] "=&r"(res2), [res3] "=&r"(res3), + [res0] "=&r"(res0), [res1] "=&r"(res1), [reshw] "=&r"(reshw), + [top_left] "=&r"(top_left) + : [above] "r"(above), [left] "r"(left), [dst] "r"(dst), + [stride] "r"(stride), [cm] "r"(cm)); +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/aom_dsp/mips/intrapred_msa.c b/third_party/aom/aom_dsp/mips/intrapred_msa.c new file mode 100644 index 0000000000..e8eaec7a94 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/intrapred_msa.c @@ -0,0 +1,739 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/macros_msa.h" + +#define IPRED_SUBS_UH2_UH(in0, in1, out0, out1) \ + { \ + out0 = __msa_subs_u_h(out0, in0); \ + out1 = __msa_subs_u_h(out1, in1); \ + } + +static void intra_predict_vert_4x4_msa(const uint8_t *src, uint8_t *dst, + int32_t dst_stride) { + uint32_t src_data; + + src_data = LW(src); + + SW4(src_data, src_data, src_data, src_data, dst, dst_stride); +} + +static void intra_predict_vert_8x8_msa(const uint8_t *src, uint8_t *dst, + int32_t dst_stride) { + uint32_t row; + uint32_t src_data1, src_data2; + + src_data1 = LW(src); + src_data2 = LW(src + 4); + + for (row = 8; row--;) { + SW(src_data1, dst); + SW(src_data2, (dst + 4)); + dst += dst_stride; + } +} + +static void intra_predict_vert_16x16_msa(const uint8_t *src, uint8_t *dst, + int32_t dst_stride) { + uint32_t row; + v16u8 src0; + + src0 = LD_UB(src); + + for (row = 16; row--;) { + ST_UB(src0, dst); + dst += dst_stride; + } +} + +static void intra_predict_vert_32x32_msa(const uint8_t *src, uint8_t *dst, + int32_t dst_stride) { + uint32_t row; + v16u8 src1, src2; + + src1 = LD_UB(src); + src2 = LD_UB(src + 16); + + for (row = 32; row--;) { + ST_UB2(src1, src2, dst, 16); + dst += dst_stride; + } +} + +static void intra_predict_horiz_4x4_msa(const uint8_t *src, uint8_t *dst, + int32_t dst_stride) { + uint32_t out0, out1, out2, out3; + + out0 = src[0] * 0x01010101; + out1 = src[1] * 0x01010101; + out2 = src[2] * 0x01010101; + out3 = src[3] * 0x01010101; + + SW4(out0, out1, out2, out3, dst, dst_stride); +} + +static void intra_predict_horiz_8x8_msa(const uint8_t *src, uint8_t *dst, + int32_t dst_stride) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + out0 = src[0] * 0x0101010101010101ull; + out1 = src[1] * 0x0101010101010101ull; + out2 = src[2] * 0x0101010101010101ull; + out3 = src[3] * 0x0101010101010101ull; + out4 = src[4] * 0x0101010101010101ull; + out5 = src[5] * 0x0101010101010101ull; + out6 = src[6] * 0x0101010101010101ull; + out7 = src[7] * 0x0101010101010101ull; + + SD4(out0, out1, out2, out3, dst, dst_stride); + dst += (4 * dst_stride); + SD4(out4, out5, out6, out7, dst, dst_stride); +} + +static void intra_predict_horiz_16x16_msa(const uint8_t *src, uint8_t *dst, + int32_t dst_stride) { + uint32_t row; + uint8_t inp0, inp1, inp2, inp3; + v16u8 src0, src1, src2, src3; + + for (row = 4; row--;) { + inp0 = src[0]; + inp1 = src[1]; + inp2 = src[2]; + inp3 = src[3]; + src += 4; + + src0 = (v16u8)__msa_fill_b(inp0); + src1 = (v16u8)__msa_fill_b(inp1); + src2 = (v16u8)__msa_fill_b(inp2); + src3 = (v16u8)__msa_fill_b(inp3); + + ST_UB4(src0, src1, src2, src3, dst, dst_stride); + dst += (4 * dst_stride); + } +} + +static void intra_predict_horiz_32x32_msa(const uint8_t *src, uint8_t *dst, + int32_t dst_stride) { + uint32_t row; + uint8_t inp0, inp1, inp2, inp3; + v16u8 src0, src1, src2, src3; + + for (row = 8; row--;) { + inp0 = src[0]; + inp1 = src[1]; + inp2 = src[2]; + inp3 = src[3]; + src += 4; + + src0 = (v16u8)__msa_fill_b(inp0); + src1 = (v16u8)__msa_fill_b(inp1); + src2 = (v16u8)__msa_fill_b(inp2); + src3 = (v16u8)__msa_fill_b(inp3); + + ST_UB2(src0, src0, dst, 16); + dst += dst_stride; + ST_UB2(src1, src1, dst, 16); + dst += dst_stride; + ST_UB2(src2, src2, dst, 16); + dst += dst_stride; + ST_UB2(src3, src3, dst, 16); + dst += dst_stride; + } +} + +static void intra_predict_dc_4x4_msa(const uint8_t *src_top, + const uint8_t *src_left, uint8_t *dst, + int32_t dst_stride) { + uint32_t val0, val1; + v16i8 store, src = { 0 }; + v8u16 sum_h; + v4u32 sum_w; + v2u64 sum_d; + + val0 = LW(src_top); + val1 = LW(src_left); + INSERT_W2_SB(val0, val1, src); + sum_h = __msa_hadd_u_h((v16u8)src, (v16u8)src); + sum_w = __msa_hadd_u_w(sum_h, sum_h); + sum_d = __msa_hadd_u_d(sum_w, sum_w); + sum_w = (v4u32)__msa_srari_w((v4i32)sum_d, 3); + store = __msa_splati_b((v16i8)sum_w, 0); + val0 = __msa_copy_u_w((v4i32)store, 0); + + SW4(val0, val0, val0, val0, dst, dst_stride); +} + +static void intra_predict_dc_tl_4x4_msa(const uint8_t *src, uint8_t *dst, + int32_t dst_stride) { + uint32_t val0; + v16i8 store, data = { 0 }; + v8u16 sum_h; + v4u32 sum_w; + + val0 = LW(src); + data = (v16i8)__msa_insert_w((v4i32)data, 0, val0); + sum_h = __msa_hadd_u_h((v16u8)data, (v16u8)data); + sum_w = __msa_hadd_u_w(sum_h, sum_h); + sum_w = (v4u32)__msa_srari_w((v4i32)sum_w, 2); + store = __msa_splati_b((v16i8)sum_w, 0); + val0 = __msa_copy_u_w((v4i32)store, 0); + + SW4(val0, val0, val0, val0, dst, dst_stride); +} + +static void intra_predict_128dc_4x4_msa(uint8_t *dst, int32_t dst_stride) { + uint32_t out; + const v16i8 store = __msa_ldi_b(128); + + out = __msa_copy_u_w((v4i32)store, 0); + + SW4(out, out, out, out, dst, dst_stride); +} + +static void intra_predict_dc_8x8_msa(const uint8_t *src_top, + const uint8_t *src_left, uint8_t *dst, + int32_t dst_stride) { + uint64_t val0, val1; + v16i8 store; + v16u8 src = { 0 }; + v8u16 sum_h; + v4u32 sum_w; + v2u64 sum_d; + + val0 = LD(src_top); + val1 = LD(src_left); + INSERT_D2_UB(val0, val1, src); + sum_h = __msa_hadd_u_h(src, src); + sum_w = __msa_hadd_u_w(sum_h, sum_h); + sum_d = __msa_hadd_u_d(sum_w, sum_w); + sum_w = (v4u32)__msa_pckev_w((v4i32)sum_d, (v4i32)sum_d); + sum_d = __msa_hadd_u_d(sum_w, sum_w); + sum_w = (v4u32)__msa_srari_w((v4i32)sum_d, 4); + store = __msa_splati_b((v16i8)sum_w, 0); + val0 = __msa_copy_u_d((v2i64)store, 0); + + SD4(val0, val0, val0, val0, dst, dst_stride); + dst += (4 * dst_stride); + SD4(val0, val0, val0, val0, dst, dst_stride); +} + +static void intra_predict_dc_tl_8x8_msa(const uint8_t *src, uint8_t *dst, + int32_t dst_stride) { + uint64_t val0; + v16i8 store; + v16u8 data = { 0 }; + v8u16 sum_h; + v4u32 sum_w; + v2u64 sum_d; + + val0 = LD(src); + data = (v16u8)__msa_insert_d((v2i64)data, 0, val0); + sum_h = __msa_hadd_u_h(data, data); + sum_w = __msa_hadd_u_w(sum_h, sum_h); + sum_d = __msa_hadd_u_d(sum_w, sum_w); + sum_w = (v4u32)__msa_srari_w((v4i32)sum_d, 3); + store = __msa_splati_b((v16i8)sum_w, 0); + val0 = __msa_copy_u_d((v2i64)store, 0); + + SD4(val0, val0, val0, val0, dst, dst_stride); + dst += (4 * dst_stride); + SD4(val0, val0, val0, val0, dst, dst_stride); +} + +static void intra_predict_128dc_8x8_msa(uint8_t *dst, int32_t dst_stride) { + uint64_t out; + const v16i8 store = __msa_ldi_b(128); + + out = __msa_copy_u_d((v2i64)store, 0); + + SD4(out, out, out, out, dst, dst_stride); + dst += (4 * dst_stride); + SD4(out, out, out, out, dst, dst_stride); +} + +static void intra_predict_dc_16x16_msa(const uint8_t *src_top, + const uint8_t *src_left, uint8_t *dst, + int32_t dst_stride) { + v16u8 top, left, out; + v8u16 sum_h, sum_top, sum_left; + v4u32 sum_w; + v2u64 sum_d; + + top = LD_UB(src_top); + left = LD_UB(src_left); + HADD_UB2_UH(top, left, sum_top, sum_left); + sum_h = sum_top + sum_left; + sum_w = __msa_hadd_u_w(sum_h, sum_h); + sum_d = __msa_hadd_u_d(sum_w, sum_w); + sum_w = (v4u32)__msa_pckev_w((v4i32)sum_d, (v4i32)sum_d); + sum_d = __msa_hadd_u_d(sum_w, sum_w); + sum_w = (v4u32)__msa_srari_w((v4i32)sum_d, 5); + out = (v16u8)__msa_splati_b((v16i8)sum_w, 0); + + ST_UB8(out, out, out, out, out, out, out, out, dst, dst_stride); + dst += (8 * dst_stride); + ST_UB8(out, out, out, out, out, out, out, out, dst, dst_stride); +} + +static void intra_predict_dc_tl_16x16_msa(const uint8_t *src, uint8_t *dst, + int32_t dst_stride) { + v16u8 data, out; + v8u16 sum_h; + v4u32 sum_w; + v2u64 sum_d; + + data = LD_UB(src); + sum_h = __msa_hadd_u_h(data, data); + sum_w = __msa_hadd_u_w(sum_h, sum_h); + sum_d = __msa_hadd_u_d(sum_w, sum_w); + sum_w = (v4u32)__msa_pckev_w((v4i32)sum_d, (v4i32)sum_d); + sum_d = __msa_hadd_u_d(sum_w, sum_w); + sum_w = (v4u32)__msa_srari_w((v4i32)sum_d, 4); + out = (v16u8)__msa_splati_b((v16i8)sum_w, 0); + + ST_UB8(out, out, out, out, out, out, out, out, dst, dst_stride); + dst += (8 * dst_stride); + ST_UB8(out, out, out, out, out, out, out, out, dst, dst_stride); +} + +static void intra_predict_128dc_16x16_msa(uint8_t *dst, int32_t dst_stride) { + const v16u8 out = (v16u8)__msa_ldi_b(128); + + ST_UB8(out, out, out, out, out, out, out, out, dst, dst_stride); + dst += (8 * dst_stride); + ST_UB8(out, out, out, out, out, out, out, out, dst, dst_stride); +} + +static void intra_predict_dc_32x32_msa(const uint8_t *src_top, + const uint8_t *src_left, uint8_t *dst, + int32_t dst_stride) { + uint32_t row; + v16u8 top0, top1, left0, left1, out; + v8u16 sum_h, sum_top0, sum_top1, sum_left0, sum_left1; + v4u32 sum_w; + v2u64 sum_d; + + LD_UB2(src_top, 16, top0, top1); + LD_UB2(src_left, 16, left0, left1); + HADD_UB2_UH(top0, top1, sum_top0, sum_top1); + HADD_UB2_UH(left0, left1, sum_left0, sum_left1); + sum_h = sum_top0 + sum_top1; + sum_h += sum_left0 + sum_left1; + sum_w = __msa_hadd_u_w(sum_h, sum_h); + sum_d = __msa_hadd_u_d(sum_w, sum_w); + sum_w = (v4u32)__msa_pckev_w((v4i32)sum_d, (v4i32)sum_d); + sum_d = __msa_hadd_u_d(sum_w, sum_w); + sum_w = (v4u32)__msa_srari_w((v4i32)sum_d, 6); + out = (v16u8)__msa_splati_b((v16i8)sum_w, 0); + + for (row = 16; row--;) { + ST_UB2(out, out, dst, 16); + dst += dst_stride; + ST_UB2(out, out, dst, 16); + dst += dst_stride; + } +} + +static void intra_predict_dc_tl_32x32_msa(const uint8_t *src, uint8_t *dst, + int32_t dst_stride) { + uint32_t row; + v16u8 data0, data1, out; + v8u16 sum_h, sum_data0, sum_data1; + v4u32 sum_w; + v2u64 sum_d; + + LD_UB2(src, 16, data0, data1); + HADD_UB2_UH(data0, data1, sum_data0, sum_data1); + sum_h = sum_data0 + sum_data1; + sum_w = __msa_hadd_u_w(sum_h, sum_h); + sum_d = __msa_hadd_u_d(sum_w, sum_w); + sum_w = (v4u32)__msa_pckev_w((v4i32)sum_d, (v4i32)sum_d); + sum_d = __msa_hadd_u_d(sum_w, sum_w); + sum_w = (v4u32)__msa_srari_w((v4i32)sum_d, 5); + out = (v16u8)__msa_splati_b((v16i8)sum_w, 0); + + for (row = 16; row--;) { + ST_UB2(out, out, dst, 16); + dst += dst_stride; + ST_UB2(out, out, dst, 16); + dst += dst_stride; + } +} + +static void intra_predict_128dc_32x32_msa(uint8_t *dst, int32_t dst_stride) { + uint32_t row; + const v16u8 out = (v16u8)__msa_ldi_b(128); + + for (row = 16; row--;) { + ST_UB2(out, out, dst, 16); + dst += dst_stride; + ST_UB2(out, out, dst, 16); + dst += dst_stride; + } +} + +static void intra_predict_tm_4x4_msa(const uint8_t *src_top_ptr, + const uint8_t *src_left, uint8_t *dst, + int32_t dst_stride) { + uint32_t val; + uint8_t top_left = src_top_ptr[-1]; + v16i8 src_left0, src_left1, src_left2, src_left3, tmp0, tmp1, src_top = { 0 }; + v16u8 src0, src1, src2, src3; + v8u16 src_top_left, vec0, vec1, vec2, vec3; + + src_top_left = (v8u16)__msa_fill_h(top_left); + val = LW(src_top_ptr); + src_top = (v16i8)__msa_insert_w((v4i32)src_top, 0, val); + + src_left0 = __msa_fill_b(src_left[0]); + src_left1 = __msa_fill_b(src_left[1]); + src_left2 = __msa_fill_b(src_left[2]); + src_left3 = __msa_fill_b(src_left[3]); + + ILVR_B4_UB(src_left0, src_top, src_left1, src_top, src_left2, src_top, + src_left3, src_top, src0, src1, src2, src3); + HADD_UB4_UH(src0, src1, src2, src3, vec0, vec1, vec2, vec3); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, vec0, vec1); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, vec2, vec3); + SAT_UH4_UH(vec0, vec1, vec2, vec3, 7); + PCKEV_B2_SB(vec1, vec0, vec3, vec2, tmp0, tmp1); + ST4x4_UB(tmp0, tmp1, 0, 2, 0, 2, dst, dst_stride); +} + +static void intra_predict_tm_8x8_msa(const uint8_t *src_top_ptr, + const uint8_t *src_left, uint8_t *dst, + int32_t dst_stride) { + uint64_t val; + uint8_t top_left = src_top_ptr[-1]; + uint32_t loop_cnt; + v16i8 src_left0, src_left1, src_left2, src_left3, tmp0, tmp1, src_top = { 0 }; + v8u16 src_top_left, vec0, vec1, vec2, vec3; + v16u8 src0, src1, src2, src3; + + val = LD(src_top_ptr); + src_top = (v16i8)__msa_insert_d((v2i64)src_top, 0, val); + src_top_left = (v8u16)__msa_fill_h(top_left); + + for (loop_cnt = 2; loop_cnt--;) { + src_left0 = __msa_fill_b(src_left[0]); + src_left1 = __msa_fill_b(src_left[1]); + src_left2 = __msa_fill_b(src_left[2]); + src_left3 = __msa_fill_b(src_left[3]); + src_left += 4; + + ILVR_B4_UB(src_left0, src_top, src_left1, src_top, src_left2, src_top, + src_left3, src_top, src0, src1, src2, src3); + HADD_UB4_UH(src0, src1, src2, src3, vec0, vec1, vec2, vec3); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, vec0, vec1); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, vec2, vec3); + SAT_UH4_UH(vec0, vec1, vec2, vec3, 7); + PCKEV_B2_SB(vec1, vec0, vec3, vec2, tmp0, tmp1); + ST8x4_UB(tmp0, tmp1, dst, dst_stride); + dst += (4 * dst_stride); + } +} + +static void intra_predict_tm_16x16_msa(const uint8_t *src_top_ptr, + const uint8_t *src_left, uint8_t *dst, + int32_t dst_stride) { + uint8_t top_left = src_top_ptr[-1]; + uint32_t loop_cnt; + v16i8 src_top, src_left0, src_left1, src_left2, src_left3; + v8u16 src_top_left, res_r, res_l; + + src_top = LD_SB(src_top_ptr); + src_top_left = (v8u16)__msa_fill_h(top_left); + + for (loop_cnt = 4; loop_cnt--;) { + src_left0 = __msa_fill_b(src_left[0]); + src_left1 = __msa_fill_b(src_left[1]); + src_left2 = __msa_fill_b(src_left[2]); + src_left3 = __msa_fill_b(src_left[3]); + src_left += 4; + + ILVRL_B2_UH(src_left0, src_top, res_r, res_l); + HADD_UB2_UH(res_r, res_l, res_r, res_l); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, res_r, res_l); + + SAT_UH2_UH(res_r, res_l, 7); + PCKEV_ST_SB(res_r, res_l, dst); + dst += dst_stride; + + ILVRL_B2_UH(src_left1, src_top, res_r, res_l); + HADD_UB2_UH(res_r, res_l, res_r, res_l); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, res_r, res_l); + SAT_UH2_UH(res_r, res_l, 7); + PCKEV_ST_SB(res_r, res_l, dst); + dst += dst_stride; + + ILVRL_B2_UH(src_left2, src_top, res_r, res_l); + HADD_UB2_UH(res_r, res_l, res_r, res_l); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, res_r, res_l); + SAT_UH2_UH(res_r, res_l, 7); + PCKEV_ST_SB(res_r, res_l, dst); + dst += dst_stride; + + ILVRL_B2_UH(src_left3, src_top, res_r, res_l); + HADD_UB2_UH(res_r, res_l, res_r, res_l); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, res_r, res_l); + SAT_UH2_UH(res_r, res_l, 7); + PCKEV_ST_SB(res_r, res_l, dst); + dst += dst_stride; + } +} + +static void intra_predict_tm_32x32_msa(const uint8_t *src_top, + const uint8_t *src_left, uint8_t *dst, + int32_t dst_stride) { + uint8_t top_left = src_top[-1]; + uint32_t loop_cnt; + v16i8 src_top0, src_top1, src_left0, src_left1, src_left2, src_left3; + v8u16 src_top_left, res_r0, res_r1, res_l0, res_l1; + + LD_SB2(src_top, 16, src_top0, src_top1); + src_top_left = (v8u16)__msa_fill_h(top_left); + + for (loop_cnt = 8; loop_cnt--;) { + src_left0 = __msa_fill_b(src_left[0]); + src_left1 = __msa_fill_b(src_left[1]); + src_left2 = __msa_fill_b(src_left[2]); + src_left3 = __msa_fill_b(src_left[3]); + src_left += 4; + + ILVR_B2_UH(src_left0, src_top0, src_left0, src_top1, res_r0, res_r1); + ILVL_B2_UH(src_left0, src_top0, src_left0, src_top1, res_l0, res_l1); + HADD_UB4_UH(res_r0, res_l0, res_r1, res_l1, res_r0, res_l0, res_r1, res_l1); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, res_r0, res_l0); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, res_r1, res_l1); + SAT_UH4_UH(res_r0, res_l0, res_r1, res_l1, 7); + PCKEV_ST_SB(res_r0, res_l0, dst); + PCKEV_ST_SB(res_r1, res_l1, dst + 16); + dst += dst_stride; + + ILVR_B2_UH(src_left1, src_top0, src_left1, src_top1, res_r0, res_r1); + ILVL_B2_UH(src_left1, src_top0, src_left1, src_top1, res_l0, res_l1); + HADD_UB4_UH(res_r0, res_l0, res_r1, res_l1, res_r0, res_l0, res_r1, res_l1); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, res_r0, res_l0); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, res_r1, res_l1); + SAT_UH4_UH(res_r0, res_l0, res_r1, res_l1, 7); + PCKEV_ST_SB(res_r0, res_l0, dst); + PCKEV_ST_SB(res_r1, res_l1, dst + 16); + dst += dst_stride; + + ILVR_B2_UH(src_left2, src_top0, src_left2, src_top1, res_r0, res_r1); + ILVL_B2_UH(src_left2, src_top0, src_left2, src_top1, res_l0, res_l1); + HADD_UB4_UH(res_r0, res_l0, res_r1, res_l1, res_r0, res_l0, res_r1, res_l1); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, res_r0, res_l0); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, res_r1, res_l1); + SAT_UH4_UH(res_r0, res_l0, res_r1, res_l1, 7); + PCKEV_ST_SB(res_r0, res_l0, dst); + PCKEV_ST_SB(res_r1, res_l1, dst + 16); + dst += dst_stride; + + ILVR_B2_UH(src_left3, src_top0, src_left3, src_top1, res_r0, res_r1); + ILVL_B2_UH(src_left3, src_top0, src_left3, src_top1, res_l0, res_l1); + HADD_UB4_UH(res_r0, res_l0, res_r1, res_l1, res_r0, res_l0, res_r1, res_l1); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, res_r0, res_l0); + IPRED_SUBS_UH2_UH(src_top_left, src_top_left, res_r1, res_l1); + SAT_UH4_UH(res_r0, res_l0, res_r1, res_l1, 7); + PCKEV_ST_SB(res_r0, res_l0, dst); + PCKEV_ST_SB(res_r1, res_l1, dst + 16); + dst += dst_stride; + } +} + +void aom_v_predictor_4x4_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)left; + + intra_predict_vert_4x4_msa(above, dst, y_stride); +} + +void aom_v_predictor_8x8_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)left; + + intra_predict_vert_8x8_msa(above, dst, y_stride); +} + +void aom_v_predictor_16x16_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)left; + + intra_predict_vert_16x16_msa(above, dst, y_stride); +} + +void aom_v_predictor_32x32_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)left; + + intra_predict_vert_32x32_msa(above, dst, y_stride); +} + +void aom_h_predictor_4x4_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + + intra_predict_horiz_4x4_msa(left, dst, y_stride); +} + +void aom_h_predictor_8x8_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + + intra_predict_horiz_8x8_msa(left, dst, y_stride); +} + +void aom_h_predictor_16x16_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + + intra_predict_horiz_16x16_msa(left, dst, y_stride); +} + +void aom_h_predictor_32x32_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + + intra_predict_horiz_32x32_msa(left, dst, y_stride); +} + +void aom_dc_predictor_4x4_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + intra_predict_dc_4x4_msa(above, left, dst, y_stride); +} + +void aom_dc_predictor_8x8_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + intra_predict_dc_8x8_msa(above, left, dst, y_stride); +} + +void aom_dc_predictor_16x16_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + intra_predict_dc_16x16_msa(above, left, dst, y_stride); +} + +void aom_dc_predictor_32x32_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + intra_predict_dc_32x32_msa(above, left, dst, y_stride); +} + +void aom_dc_top_predictor_4x4_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)left; + + intra_predict_dc_tl_4x4_msa(above, dst, y_stride); +} + +void aom_dc_top_predictor_8x8_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)left; + + intra_predict_dc_tl_8x8_msa(above, dst, y_stride); +} + +void aom_dc_top_predictor_16x16_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)left; + + intra_predict_dc_tl_16x16_msa(above, dst, y_stride); +} + +void aom_dc_top_predictor_32x32_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)left; + + intra_predict_dc_tl_32x32_msa(above, dst, y_stride); +} + +void aom_dc_left_predictor_4x4_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + + intra_predict_dc_tl_4x4_msa(left, dst, y_stride); +} + +void aom_dc_left_predictor_8x8_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + + intra_predict_dc_tl_8x8_msa(left, dst, y_stride); +} + +void aom_dc_left_predictor_16x16_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, + const uint8_t *left) { + (void)above; + + intra_predict_dc_tl_16x16_msa(left, dst, y_stride); +} + +void aom_dc_left_predictor_32x32_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, + const uint8_t *left) { + (void)above; + + intra_predict_dc_tl_32x32_msa(left, dst, y_stride); +} + +void aom_dc_128_predictor_4x4_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + (void)left; + + intra_predict_128dc_4x4_msa(dst, y_stride); +} + +void aom_dc_128_predictor_8x8_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + (void)left; + + intra_predict_128dc_8x8_msa(dst, y_stride); +} + +void aom_dc_128_predictor_16x16_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + (void)left; + + intra_predict_128dc_16x16_msa(dst, y_stride); +} + +void aom_dc_128_predictor_32x32_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + (void)above; + (void)left; + + intra_predict_128dc_32x32_msa(dst, y_stride); +} + +void aom_tm_predictor_4x4_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + intra_predict_tm_4x4_msa(above, left, dst, y_stride); +} + +void aom_tm_predictor_8x8_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + intra_predict_tm_8x8_msa(above, left, dst, y_stride); +} + +void aom_tm_predictor_16x16_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + intra_predict_tm_16x16_msa(above, left, dst, y_stride); +} + +void aom_tm_predictor_32x32_msa(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + intra_predict_tm_32x32_msa(above, left, dst, y_stride); +} diff --git a/third_party/aom/aom_dsp/mips/inv_txfm_dspr2.h b/third_party/aom/aom_dsp/mips/inv_txfm_dspr2.h new file mode 100644 index 0000000000..8a85e26f3e --- /dev/null +++ b/third_party/aom/aom_dsp/mips/inv_txfm_dspr2.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_MIPS_INV_TXFM_DSPR2_H_ +#define AOM_DSP_MIPS_INV_TXFM_DSPR2_H_ + +#include + +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/inv_txfm.h" +#include "aom_dsp/mips/common_dspr2.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if HAVE_DSPR2 +#define DCT_CONST_ROUND_SHIFT_TWICE_COSPI_16_64(input) \ + ({ \ + \ + int32_t tmp, out; \ + int dct_cost_rounding = DCT_CONST_ROUNDING; \ + int in = input; \ + \ + __asm__ __volatile__(/* out = dct_const_round_shift(dc * cospi_16_64); */ \ + "mtlo %[dct_cost_rounding], $ac1 " \ + " \n\t" \ + "mthi $zero, $ac1 " \ + " \n\t" \ + "madd $ac1, %[in], " \ + "%[cospi_16_64] \n\t" \ + "extp %[tmp], $ac1, " \ + "31 \n\t" \ + \ + /* out = dct_const_round_shift(out * cospi_16_64); */ \ + "mtlo %[dct_cost_rounding], $ac2 " \ + " \n\t" \ + "mthi $zero, $ac2 " \ + " \n\t" \ + "madd $ac2, %[tmp], " \ + "%[cospi_16_64] \n\t" \ + "extp %[out], $ac2, " \ + "31 \n\t" \ + \ + : [tmp] "=&r"(tmp), [out] "=r"(out) \ + : [in] "r"(in), \ + [dct_cost_rounding] "r"(dct_cost_rounding), \ + [cospi_16_64] "r"(cospi_16_64)); \ + out; \ + }) + +void aom_idct32_cols_add_blk_dspr2(int16_t *input, uint8_t *dest, + int dest_stride); +void aom_idct4_rows_dspr2(const int16_t *input, int16_t *output); +void aom_idct4_columns_add_blk_dspr2(int16_t *input, uint8_t *dest, + int dest_stride); +void iadst4_dspr2(const int16_t *input, int16_t *output); +void idct8_rows_dspr2(const int16_t *input, int16_t *output, uint32_t no_rows); +void idct8_columns_add_blk_dspr2(int16_t *input, uint8_t *dest, + int dest_stride); +void iadst8_dspr2(const int16_t *input, int16_t *output); +void idct16_rows_dspr2(const int16_t *input, int16_t *output, uint32_t no_rows); +void idct16_cols_add_blk_dspr2(int16_t *input, uint8_t *dest, int dest_stride); +void iadst16_dspr2(const int16_t *input, int16_t *output); + +#endif // #if HAVE_DSPR2 +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_MIPS_INV_TXFM_DSPR2_H_ diff --git a/third_party/aom/aom_dsp/mips/inv_txfm_msa.h b/third_party/aom/aom_dsp/mips/inv_txfm_msa.h new file mode 100644 index 0000000000..122667aa85 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/inv_txfm_msa.h @@ -0,0 +1,412 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_MIPS_INV_TXFM_MSA_H_ +#define AOM_DSP_MIPS_INV_TXFM_MSA_H_ + +#include "aom_dsp/mips/macros_msa.h" +#include "aom_dsp/mips/txfm_macros_msa.h" +#include "aom_dsp/txfm_common.h" + +#define AOM_ADST8(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, \ + out3, out4, out5, out6, out7) \ + { \ + v8i16 cnst0_m, cnst1_m, cnst2_m, cnst3_m, cnst4_m; \ + v8i16 vec0_m, vec1_m, vec2_m, vec3_m, s0_m, s1_m; \ + v8i16 coeff0_m = { cospi_2_64, cospi_6_64, cospi_10_64, cospi_14_64, \ + cospi_18_64, cospi_22_64, cospi_26_64, cospi_30_64 }; \ + v8i16 coeff1_m = { cospi_8_64, -cospi_8_64, cospi_16_64, -cospi_16_64, \ + cospi_24_64, -cospi_24_64, 0, 0 }; \ + \ + SPLATI_H2_SH(coeff0_m, 0, 7, cnst0_m, cnst1_m); \ + cnst2_m = -cnst0_m; \ + ILVEV_H2_SH(cnst0_m, cnst1_m, cnst1_m, cnst2_m, cnst0_m, cnst1_m); \ + SPLATI_H2_SH(coeff0_m, 4, 3, cnst2_m, cnst3_m); \ + cnst4_m = -cnst2_m; \ + ILVEV_H2_SH(cnst2_m, cnst3_m, cnst3_m, cnst4_m, cnst2_m, cnst3_m); \ + \ + ILVRL_H2_SH(in0, in7, vec1_m, vec0_m); \ + ILVRL_H2_SH(in4, in3, vec3_m, vec2_m); \ + DOT_ADD_SUB_SRARI_PCK(vec0_m, vec1_m, vec2_m, vec3_m, cnst0_m, cnst1_m, \ + cnst2_m, cnst3_m, in7, in0, in4, in3); \ + \ + SPLATI_H2_SH(coeff0_m, 2, 5, cnst0_m, cnst1_m); \ + cnst2_m = -cnst0_m; \ + ILVEV_H2_SH(cnst0_m, cnst1_m, cnst1_m, cnst2_m, cnst0_m, cnst1_m); \ + SPLATI_H2_SH(coeff0_m, 6, 1, cnst2_m, cnst3_m); \ + cnst4_m = -cnst2_m; \ + ILVEV_H2_SH(cnst2_m, cnst3_m, cnst3_m, cnst4_m, cnst2_m, cnst3_m); \ + \ + ILVRL_H2_SH(in2, in5, vec1_m, vec0_m); \ + ILVRL_H2_SH(in6, in1, vec3_m, vec2_m); \ + \ + DOT_ADD_SUB_SRARI_PCK(vec0_m, vec1_m, vec2_m, vec3_m, cnst0_m, cnst1_m, \ + cnst2_m, cnst3_m, in5, in2, in6, in1); \ + BUTTERFLY_4(in7, in0, in2, in5, s1_m, s0_m, in2, in5); \ + out7 = -s0_m; \ + out0 = s1_m; \ + \ + SPLATI_H4_SH(coeff1_m, 0, 4, 1, 5, cnst0_m, cnst1_m, cnst2_m, cnst3_m); \ + \ + ILVEV_H2_SH(cnst3_m, cnst0_m, cnst1_m, cnst2_m, cnst3_m, cnst2_m); \ + cnst0_m = __msa_ilvev_h(cnst1_m, cnst0_m); \ + cnst1_m = cnst0_m; \ + \ + ILVRL_H2_SH(in4, in3, vec1_m, vec0_m); \ + ILVRL_H2_SH(in6, in1, vec3_m, vec2_m); \ + DOT_ADD_SUB_SRARI_PCK(vec0_m, vec1_m, vec2_m, vec3_m, cnst0_m, cnst2_m, \ + cnst3_m, cnst1_m, out1, out6, s0_m, s1_m); \ + \ + SPLATI_H2_SH(coeff1_m, 2, 3, cnst0_m, cnst1_m); \ + cnst1_m = __msa_ilvev_h(cnst1_m, cnst0_m); \ + \ + ILVRL_H2_SH(in2, in5, vec1_m, vec0_m); \ + ILVRL_H2_SH(s0_m, s1_m, vec3_m, vec2_m); \ + out3 = DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m); \ + out4 = DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst1_m); \ + out2 = DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst0_m); \ + out5 = DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst1_m); \ + \ + out1 = -out1; \ + out3 = -out3; \ + out5 = -out5; \ + } + +#define AOM_SET_COSPI_PAIR(c0_h, c1_h) \ + ({ \ + v8i16 out0_m, r0_m, r1_m; \ + \ + r0_m = __msa_fill_h(c0_h); \ + r1_m = __msa_fill_h(c1_h); \ + out0_m = __msa_ilvev_h(r1_m, r0_m); \ + \ + out0_m; \ + }) + +#define AOM_ADDBLK_ST8x4_UB(dst, dst_stride, in0, in1, in2, in3) \ + { \ + uint8_t *dst_m = (uint8_t *)(dst); \ + v16u8 dst0_m, dst1_m, dst2_m, dst3_m; \ + v16i8 tmp0_m, tmp1_m; \ + v16i8 zero_m = { 0 }; \ + v8i16 res0_m, res1_m, res2_m, res3_m; \ + \ + LD_UB4(dst_m, dst_stride, dst0_m, dst1_m, dst2_m, dst3_m); \ + ILVR_B4_SH(zero_m, dst0_m, zero_m, dst1_m, zero_m, dst2_m, zero_m, dst3_m, \ + res0_m, res1_m, res2_m, res3_m); \ + ADD4(res0_m, in0, res1_m, in1, res2_m, in2, res3_m, in3, res0_m, res1_m, \ + res2_m, res3_m); \ + CLIP_SH4_0_255(res0_m, res1_m, res2_m, res3_m); \ + PCKEV_B2_SB(res1_m, res0_m, res3_m, res2_m, tmp0_m, tmp1_m); \ + ST8x4_UB(tmp0_m, tmp1_m, dst_m, dst_stride); \ + } + +#define AOM_IDCT4x4(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + v8i16 c0_m, c1_m, c2_m, c3_m; \ + v8i16 step0_m, step1_m; \ + v4i32 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + \ + c0_m = AOM_SET_COSPI_PAIR(cospi_16_64, cospi_16_64); \ + c1_m = AOM_SET_COSPI_PAIR(cospi_16_64, -cospi_16_64); \ + step0_m = __msa_ilvr_h(in2, in0); \ + DOTP_SH2_SW(step0_m, step0_m, c0_m, c1_m, tmp0_m, tmp1_m); \ + \ + c2_m = AOM_SET_COSPI_PAIR(cospi_24_64, -cospi_8_64); \ + c3_m = AOM_SET_COSPI_PAIR(cospi_8_64, cospi_24_64); \ + step1_m = __msa_ilvr_h(in3, in1); \ + DOTP_SH2_SW(step1_m, step1_m, c2_m, c3_m, tmp2_m, tmp3_m); \ + SRARI_W4_SW(tmp0_m, tmp1_m, tmp2_m, tmp3_m, DCT_CONST_BITS); \ + \ + PCKEV_H2_SW(tmp1_m, tmp0_m, tmp3_m, tmp2_m, tmp0_m, tmp2_m); \ + SLDI_B2_0_SW(tmp0_m, tmp2_m, tmp1_m, tmp3_m, 8); \ + BUTTERFLY_4((v8i16)tmp0_m, (v8i16)tmp1_m, (v8i16)tmp2_m, (v8i16)tmp3_m, \ + out0, out1, out2, out3); \ + } + +#define AOM_IADST4x4(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + v8i16 res0_m, res1_m, c0_m, c1_m; \ + v8i16 k1_m, k2_m, k3_m, k4_m; \ + v8i16 zero_m = { 0 }; \ + v4i32 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + v4i32 int0_m, int1_m, int2_m, int3_m; \ + v8i16 mask_m = { sinpi_1_9, sinpi_2_9, sinpi_3_9, sinpi_4_9, \ + -sinpi_1_9, -sinpi_2_9, -sinpi_3_9, -sinpi_4_9 }; \ + \ + SPLATI_H4_SH(mask_m, 3, 0, 1, 2, c0_m, c1_m, k1_m, k2_m); \ + ILVEV_H2_SH(c0_m, c1_m, k1_m, k2_m, c0_m, c1_m); \ + ILVR_H2_SH(in0, in2, in1, in3, res0_m, res1_m); \ + DOTP_SH2_SW(res0_m, res1_m, c0_m, c1_m, tmp2_m, tmp1_m); \ + int0_m = tmp2_m + tmp1_m; \ + \ + SPLATI_H2_SH(mask_m, 4, 7, k4_m, k3_m); \ + ILVEV_H2_SH(k4_m, k1_m, k3_m, k2_m, c0_m, c1_m); \ + DOTP_SH2_SW(res0_m, res1_m, c0_m, c1_m, tmp0_m, tmp1_m); \ + int1_m = tmp0_m + tmp1_m; \ + \ + c0_m = __msa_splati_h(mask_m, 6); \ + ILVL_H2_SH(k2_m, c0_m, zero_m, k2_m, c0_m, c1_m); \ + ILVR_H2_SH(in0, in2, in1, in3, res0_m, res1_m); \ + DOTP_SH2_SW(res0_m, res1_m, c0_m, c1_m, tmp0_m, tmp1_m); \ + int2_m = tmp0_m + tmp1_m; \ + \ + c0_m = __msa_splati_h(mask_m, 6); \ + c0_m = __msa_ilvev_h(c0_m, k1_m); \ + \ + res0_m = __msa_ilvr_h((in1), (in3)); \ + tmp0_m = __msa_dotp_s_w(res0_m, c0_m); \ + int3_m = tmp2_m + tmp0_m; \ + \ + res0_m = __msa_ilvr_h((in2), (in3)); \ + c1_m = __msa_ilvev_h(k4_m, k3_m); \ + \ + tmp2_m = __msa_dotp_s_w(res0_m, c1_m); \ + res1_m = __msa_ilvr_h((in0), (in2)); \ + c1_m = __msa_ilvev_h(k1_m, zero_m); \ + \ + tmp3_m = __msa_dotp_s_w(res1_m, c1_m); \ + int3_m += tmp2_m; \ + int3_m += tmp3_m; \ + \ + SRARI_W4_SW(int0_m, int1_m, int2_m, int3_m, DCT_CONST_BITS); \ + PCKEV_H2_SH(int0_m, int0_m, int1_m, int1_m, out0, out1); \ + PCKEV_H2_SH(int2_m, int2_m, int3_m, int3_m, out2, out3); \ + } + +#define AV1_SET_CONST_PAIR(mask_h, idx1_h, idx2_h) \ + ({ \ + v8i16 c0_m, c1_m; \ + \ + SPLATI_H2_SH(mask_h, idx1_h, idx2_h, c0_m, c1_m); \ + c0_m = __msa_ilvev_h(c1_m, c0_m); \ + \ + c0_m; \ + }) + +/* multiply and add macro */ +#define AV1_MADD(inp0, inp1, inp2, inp3, cst0, cst1, cst2, cst3, out0, out1, \ + out2, out3) \ + { \ + v8i16 madd_s0_m, madd_s1_m, madd_s2_m, madd_s3_m; \ + v4i32 tmp0_madd, tmp1_madd, tmp2_madd, tmp3_madd; \ + \ + ILVRL_H2_SH(inp1, inp0, madd_s1_m, madd_s0_m); \ + ILVRL_H2_SH(inp3, inp2, madd_s3_m, madd_s2_m); \ + DOTP_SH4_SW(madd_s1_m, madd_s0_m, madd_s1_m, madd_s0_m, cst0, cst0, cst1, \ + cst1, tmp0_madd, tmp1_madd, tmp2_madd, tmp3_madd); \ + SRARI_W4_SW(tmp0_madd, tmp1_madd, tmp2_madd, tmp3_madd, DCT_CONST_BITS); \ + PCKEV_H2_SH(tmp1_madd, tmp0_madd, tmp3_madd, tmp2_madd, out0, out1); \ + DOTP_SH4_SW(madd_s3_m, madd_s2_m, madd_s3_m, madd_s2_m, cst2, cst2, cst3, \ + cst3, tmp0_madd, tmp1_madd, tmp2_madd, tmp3_madd); \ + SRARI_W4_SW(tmp0_madd, tmp1_madd, tmp2_madd, tmp3_madd, DCT_CONST_BITS); \ + PCKEV_H2_SH(tmp1_madd, tmp0_madd, tmp3_madd, tmp2_madd, out2, out3); \ + } + +/* idct 8x8 macro */ +#define AOM_IDCT8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3, out4, out5, out6, out7) \ + { \ + v8i16 tp0_m, tp1_m, tp2_m, tp3_m, tp4_m, tp5_m, tp6_m, tp7_m; \ + v8i16 k0_m, k1_m, k2_m, k3_m, res0_m, res1_m, res2_m, res3_m; \ + v4i32 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + v8i16 mask_m = { cospi_28_64, cospi_4_64, cospi_20_64, cospi_12_64, \ + cospi_16_64, -cospi_4_64, -cospi_20_64, -cospi_16_64 }; \ + \ + k0_m = AV1_SET_CONST_PAIR(mask_m, 0, 5); \ + k1_m = AV1_SET_CONST_PAIR(mask_m, 1, 0); \ + k2_m = AV1_SET_CONST_PAIR(mask_m, 6, 3); \ + k3_m = AV1_SET_CONST_PAIR(mask_m, 3, 2); \ + AV1_MADD(in1, in7, in3, in5, k0_m, k1_m, k2_m, k3_m, in1, in7, in3, in5); \ + SUB2(in1, in3, in7, in5, res0_m, res1_m); \ + k0_m = AV1_SET_CONST_PAIR(mask_m, 4, 7); \ + k1_m = __msa_splati_h(mask_m, 4); \ + \ + ILVRL_H2_SH(res0_m, res1_m, res2_m, res3_m); \ + DOTP_SH4_SW(res2_m, res3_m, res2_m, res3_m, k0_m, k0_m, k1_m, k1_m, \ + tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ + SRARI_W4_SW(tmp0_m, tmp1_m, tmp2_m, tmp3_m, DCT_CONST_BITS); \ + tp4_m = in1 + in3; \ + PCKEV_H2_SH(tmp1_m, tmp0_m, tmp3_m, tmp2_m, tp5_m, tp6_m); \ + tp7_m = in7 + in5; \ + k2_m = AOM_SET_COSPI_PAIR(cospi_24_64, -cospi_8_64); \ + k3_m = AOM_SET_COSPI_PAIR(cospi_8_64, cospi_24_64); \ + AV1_MADD(in0, in4, in2, in6, k1_m, k0_m, k2_m, k3_m, in0, in4, in2, in6); \ + BUTTERFLY_4(in0, in4, in2, in6, tp0_m, tp1_m, tp2_m, tp3_m); \ + BUTTERFLY_8(tp0_m, tp1_m, tp2_m, tp3_m, tp4_m, tp5_m, tp6_m, tp7_m, out0, \ + out1, out2, out3, out4, out5, out6, out7); \ + } + +#define AV1_IADST8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3, out4, out5, out6, out7) \ + { \ + v4i32 r0_m, r1_m, r2_m, r3_m, r4_m, r5_m, r6_m, r7_m; \ + v4i32 m0_m, m1_m, m2_m, m3_m, t0_m, t1_m; \ + v8i16 res0_m, res1_m, res2_m, res3_m, k0_m, k1_m, in_s0, in_s1; \ + v8i16 mask1_m = { cospi_2_64, cospi_30_64, -cospi_2_64, cospi_10_64, \ + cospi_22_64, -cospi_10_64, cospi_18_64, cospi_14_64 }; \ + v8i16 mask2_m = { cospi_14_64, -cospi_18_64, cospi_26_64, cospi_6_64, \ + -cospi_26_64, cospi_8_64, cospi_24_64, -cospi_8_64 }; \ + v8i16 mask3_m = { \ + -cospi_24_64, cospi_8_64, cospi_16_64, -cospi_16_64, 0, 0, 0, 0 \ + }; \ + \ + k0_m = AV1_SET_CONST_PAIR(mask1_m, 0, 1); \ + k1_m = AV1_SET_CONST_PAIR(mask1_m, 1, 2); \ + ILVRL_H2_SH(in1, in0, in_s1, in_s0); \ + DOTP_SH4_SW(in_s1, in_s0, in_s1, in_s0, k0_m, k0_m, k1_m, k1_m, r0_m, \ + r1_m, r2_m, r3_m); \ + k0_m = AV1_SET_CONST_PAIR(mask1_m, 6, 7); \ + k1_m = AV1_SET_CONST_PAIR(mask2_m, 0, 1); \ + ILVRL_H2_SH(in5, in4, in_s1, in_s0); \ + DOTP_SH4_SW(in_s1, in_s0, in_s1, in_s0, k0_m, k0_m, k1_m, k1_m, r4_m, \ + r5_m, r6_m, r7_m); \ + ADD4(r0_m, r4_m, r1_m, r5_m, r2_m, r6_m, r3_m, r7_m, m0_m, m1_m, m2_m, \ + m3_m); \ + SRARI_W4_SW(m0_m, m1_m, m2_m, m3_m, DCT_CONST_BITS); \ + PCKEV_H2_SH(m1_m, m0_m, m3_m, m2_m, res0_m, res1_m); \ + SUB4(r0_m, r4_m, r1_m, r5_m, r2_m, r6_m, r3_m, r7_m, m0_m, m1_m, m2_m, \ + m3_m); \ + SRARI_W4_SW(m0_m, m1_m, m2_m, m3_m, DCT_CONST_BITS); \ + PCKEV_H2_SW(m1_m, m0_m, m3_m, m2_m, t0_m, t1_m); \ + k0_m = AV1_SET_CONST_PAIR(mask1_m, 3, 4); \ + k1_m = AV1_SET_CONST_PAIR(mask1_m, 4, 5); \ + ILVRL_H2_SH(in3, in2, in_s1, in_s0); \ + DOTP_SH4_SW(in_s1, in_s0, in_s1, in_s0, k0_m, k0_m, k1_m, k1_m, r0_m, \ + r1_m, r2_m, r3_m); \ + k0_m = AV1_SET_CONST_PAIR(mask2_m, 2, 3); \ + k1_m = AV1_SET_CONST_PAIR(mask2_m, 3, 4); \ + ILVRL_H2_SH(in7, in6, in_s1, in_s0); \ + DOTP_SH4_SW(in_s1, in_s0, in_s1, in_s0, k0_m, k0_m, k1_m, k1_m, r4_m, \ + r5_m, r6_m, r7_m); \ + ADD4(r0_m, r4_m, r1_m, r5_m, r2_m, r6_m, r3_m, r7_m, m0_m, m1_m, m2_m, \ + m3_m); \ + SRARI_W4_SW(m0_m, m1_m, m2_m, m3_m, DCT_CONST_BITS); \ + PCKEV_H2_SH(m1_m, m0_m, m3_m, m2_m, res2_m, res3_m); \ + SUB4(r0_m, r4_m, r1_m, r5_m, r2_m, r6_m, r3_m, r7_m, m0_m, m1_m, m2_m, \ + m3_m); \ + SRARI_W4_SW(m0_m, m1_m, m2_m, m3_m, DCT_CONST_BITS); \ + PCKEV_H2_SW(m1_m, m0_m, m3_m, m2_m, r2_m, r3_m); \ + ILVRL_H2_SW(r3_m, r2_m, m2_m, m3_m); \ + BUTTERFLY_4(res0_m, res1_m, res3_m, res2_m, out0, in7, in4, in3); \ + k0_m = AV1_SET_CONST_PAIR(mask2_m, 5, 6); \ + k1_m = AV1_SET_CONST_PAIR(mask2_m, 6, 7); \ + ILVRL_H2_SH(t1_m, t0_m, in_s1, in_s0); \ + DOTP_SH4_SW(in_s1, in_s0, in_s1, in_s0, k0_m, k0_m, k1_m, k1_m, r0_m, \ + r1_m, r2_m, r3_m); \ + k1_m = AV1_SET_CONST_PAIR(mask3_m, 0, 1); \ + DOTP_SH4_SW(m2_m, m3_m, m2_m, m3_m, k0_m, k0_m, k1_m, k1_m, r4_m, r5_m, \ + r6_m, r7_m); \ + ADD4(r0_m, r6_m, r1_m, r7_m, r2_m, r4_m, r3_m, r5_m, m0_m, m1_m, m2_m, \ + m3_m); \ + SRARI_W4_SW(m0_m, m1_m, m2_m, m3_m, DCT_CONST_BITS); \ + PCKEV_H2_SH(m1_m, m0_m, m3_m, m2_m, in1, out6); \ + SUB4(r0_m, r6_m, r1_m, r7_m, r2_m, r4_m, r3_m, r5_m, m0_m, m1_m, m2_m, \ + m3_m); \ + SRARI_W4_SW(m0_m, m1_m, m2_m, m3_m, DCT_CONST_BITS); \ + PCKEV_H2_SH(m1_m, m0_m, m3_m, m2_m, in2, in5); \ + k0_m = AV1_SET_CONST_PAIR(mask3_m, 2, 2); \ + k1_m = AV1_SET_CONST_PAIR(mask3_m, 2, 3); \ + ILVRL_H2_SH(in4, in3, in_s1, in_s0); \ + DOTP_SH4_SW(in_s1, in_s0, in_s1, in_s0, k0_m, k0_m, k1_m, k1_m, m0_m, \ + m1_m, m2_m, m3_m); \ + SRARI_W4_SW(m0_m, m1_m, m2_m, m3_m, DCT_CONST_BITS); \ + PCKEV_H2_SH(m1_m, m0_m, m3_m, m2_m, in3, out4); \ + ILVRL_H2_SW(in5, in2, m2_m, m3_m); \ + DOTP_SH4_SW(m2_m, m3_m, m2_m, m3_m, k0_m, k0_m, k1_m, k1_m, m0_m, m1_m, \ + m2_m, m3_m); \ + SRARI_W4_SW(m0_m, m1_m, m2_m, m3_m, DCT_CONST_BITS); \ + PCKEV_H2_SH(m1_m, m0_m, m3_m, m2_m, out2, in5); \ + \ + out1 = -in1; \ + out3 = -in3; \ + out5 = -in5; \ + out7 = -in7; \ + } + +#define AOM_IADST8x16_1D(r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, \ + r12, r13, r14, r15, out0, out1, out2, out3, out4, \ + out5, out6, out7, out8, out9, out10, out11, out12, \ + out13, out14, out15) \ + { \ + v8i16 g0_m, g1_m, g2_m, g3_m, g4_m, g5_m, g6_m, g7_m; \ + v8i16 g8_m, g9_m, g10_m, g11_m, g12_m, g13_m, g14_m, g15_m; \ + v8i16 h0_m, h1_m, h2_m, h3_m, h4_m, h5_m, h6_m, h7_m; \ + v8i16 h8_m, h9_m, h10_m, h11_m; \ + v8i16 k0_m, k1_m, k2_m, k3_m; \ + \ + /* stage 1 */ \ + k0_m = AOM_SET_COSPI_PAIR(cospi_1_64, cospi_31_64); \ + k1_m = AOM_SET_COSPI_PAIR(cospi_31_64, -cospi_1_64); \ + k2_m = AOM_SET_COSPI_PAIR(cospi_17_64, cospi_15_64); \ + k3_m = AOM_SET_COSPI_PAIR(cospi_15_64, -cospi_17_64); \ + MADD_BF(r15, r0, r7, r8, k0_m, k1_m, k2_m, k3_m, g0_m, g1_m, g2_m, g3_m); \ + k0_m = AOM_SET_COSPI_PAIR(cospi_5_64, cospi_27_64); \ + k1_m = AOM_SET_COSPI_PAIR(cospi_27_64, -cospi_5_64); \ + k2_m = AOM_SET_COSPI_PAIR(cospi_21_64, cospi_11_64); \ + k3_m = AOM_SET_COSPI_PAIR(cospi_11_64, -cospi_21_64); \ + MADD_BF(r13, r2, r5, r10, k0_m, k1_m, k2_m, k3_m, g4_m, g5_m, g6_m, g7_m); \ + k0_m = AOM_SET_COSPI_PAIR(cospi_9_64, cospi_23_64); \ + k1_m = AOM_SET_COSPI_PAIR(cospi_23_64, -cospi_9_64); \ + k2_m = AOM_SET_COSPI_PAIR(cospi_25_64, cospi_7_64); \ + k3_m = AOM_SET_COSPI_PAIR(cospi_7_64, -cospi_25_64); \ + MADD_BF(r11, r4, r3, r12, k0_m, k1_m, k2_m, k3_m, g8_m, g9_m, g10_m, \ + g11_m); \ + k0_m = AOM_SET_COSPI_PAIR(cospi_13_64, cospi_19_64); \ + k1_m = AOM_SET_COSPI_PAIR(cospi_19_64, -cospi_13_64); \ + k2_m = AOM_SET_COSPI_PAIR(cospi_29_64, cospi_3_64); \ + k3_m = AOM_SET_COSPI_PAIR(cospi_3_64, -cospi_29_64); \ + MADD_BF(r9, r6, r1, r14, k0_m, k1_m, k2_m, k3_m, g12_m, g13_m, g14_m, \ + g15_m); \ + \ + /* stage 2 */ \ + k0_m = AOM_SET_COSPI_PAIR(cospi_4_64, cospi_28_64); \ + k1_m = AOM_SET_COSPI_PAIR(cospi_28_64, -cospi_4_64); \ + k2_m = AOM_SET_COSPI_PAIR(-cospi_28_64, cospi_4_64); \ + MADD_BF(g1_m, g3_m, g9_m, g11_m, k0_m, k1_m, k2_m, k0_m, h0_m, h1_m, h2_m, \ + h3_m); \ + k0_m = AOM_SET_COSPI_PAIR(cospi_12_64, cospi_20_64); \ + k1_m = AOM_SET_COSPI_PAIR(-cospi_20_64, cospi_12_64); \ + k2_m = AOM_SET_COSPI_PAIR(cospi_20_64, -cospi_12_64); \ + MADD_BF(g7_m, g5_m, g15_m, g13_m, k0_m, k1_m, k2_m, k0_m, h4_m, h5_m, \ + h6_m, h7_m); \ + BUTTERFLY_4(h0_m, h2_m, h6_m, h4_m, out8, out9, out11, out10); \ + BUTTERFLY_8(g0_m, g2_m, g4_m, g6_m, g14_m, g12_m, g10_m, g8_m, h8_m, h9_m, \ + h10_m, h11_m, h6_m, h4_m, h2_m, h0_m); \ + \ + /* stage 3 */ \ + BUTTERFLY_4(h8_m, h9_m, h11_m, h10_m, out0, out1, h11_m, h10_m); \ + k0_m = AOM_SET_COSPI_PAIR(cospi_8_64, cospi_24_64); \ + k1_m = AOM_SET_COSPI_PAIR(cospi_24_64, -cospi_8_64); \ + k2_m = AOM_SET_COSPI_PAIR(-cospi_24_64, cospi_8_64); \ + MADD_BF(h0_m, h2_m, h4_m, h6_m, k0_m, k1_m, k2_m, k0_m, out4, out6, out5, \ + out7); \ + MADD_BF(h1_m, h3_m, h5_m, h7_m, k0_m, k1_m, k2_m, k0_m, out12, out14, \ + out13, out15); \ + \ + /* stage 4 */ \ + k0_m = AOM_SET_COSPI_PAIR(cospi_16_64, cospi_16_64); \ + k1_m = AOM_SET_COSPI_PAIR(-cospi_16_64, -cospi_16_64); \ + k2_m = AOM_SET_COSPI_PAIR(cospi_16_64, -cospi_16_64); \ + k3_m = AOM_SET_COSPI_PAIR(-cospi_16_64, cospi_16_64); \ + MADD_SHORT(h10_m, h11_m, k1_m, k2_m, out2, out3); \ + MADD_SHORT(out6, out7, k0_m, k3_m, out6, out7); \ + MADD_SHORT(out10, out11, k0_m, k3_m, out10, out11); \ + MADD_SHORT(out14, out15, k1_m, k2_m, out14, out15); \ + } + +void aom_idct16_1d_columns_addblk_msa(int16_t *input, uint8_t *dst, + int32_t dst_stride); +void aom_idct16_1d_rows_msa(const int16_t *input, int16_t *output); +void aom_iadst16_1d_columns_addblk_msa(int16_t *input, uint8_t *dst, + int32_t dst_stride); +void aom_iadst16_1d_rows_msa(const int16_t *input, int16_t *output); +#endif // AOM_DSP_MIPS_INV_TXFM_MSA_H_ diff --git a/third_party/aom/aom_dsp/mips/itrans16_dspr2.c b/third_party/aom/aom_dsp/mips/itrans16_dspr2.c new file mode 100644 index 0000000000..c63b1e8570 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/itrans16_dspr2.c @@ -0,0 +1,1190 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/inv_txfm_dspr2.h" +#include "aom_dsp/txfm_common.h" + +#if HAVE_DSPR2 +void idct16_rows_dspr2(const int16_t *input, int16_t *output, + uint32_t no_rows) { + int i; + int step1_0, step1_1, step1_2, step1_3, step1_4, step1_5, step1_6, step1_7; + int step1_10, step1_11, step1_12, step1_13; + int step2_0, step2_1, step2_2, step2_3; + int step2_8, step2_9, step2_10, step2_11; + int step2_12, step2_13, step2_14, step2_15; + int load1, load2, load3, load4, load5, load6, load7, load8; + int result1, result2, result3, result4; + const int const_2_power_13 = 8192; + + for (i = no_rows; i--;) { + /* prefetch row */ + prefetch_load((const uint8_t *)(input + 16)); + + __asm__ __volatile__( + "lh %[load1], 0(%[input]) \n\t" + "lh %[load2], 16(%[input]) \n\t" + "lh %[load3], 8(%[input]) \n\t" + "lh %[load4], 24(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "add %[result1], %[load1], %[load2] \n\t" + "sub %[result2], %[load1], %[load2] \n\t" + "madd $ac1, %[result1], %[cospi_16_64] \n\t" + "madd $ac2, %[result2], %[cospi_16_64] \n\t" + "extp %[step2_0], $ac1, 31 \n\t" + "extp %[step2_1], $ac2, 31 \n\t" + + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "madd $ac3, %[load3], %[cospi_24_64] \n\t" + "msub $ac3, %[load4], %[cospi_8_64] \n\t" + "extp %[step2_2], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "madd $ac1, %[load3], %[cospi_8_64] \n\t" + "madd $ac1, %[load4], %[cospi_24_64] \n\t" + "extp %[step2_3], $ac1, 31 \n\t" + + "add %[step1_0], %[step2_0], %[step2_3] \n\t" + "add %[step1_1], %[step2_1], %[step2_2] \n\t" + "sub %[step1_2], %[step2_1], %[step2_2] \n\t" + "sub %[step1_3], %[step2_0], %[step2_3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [result1] "=&r"(result1), + [result2] "=&r"(result2), [step2_0] "=&r"(step2_0), + [step2_1] "=&r"(step2_1), [step2_2] "=&r"(step2_2), + [step2_3] "=&r"(step2_3), [step1_0] "=r"(step1_0), + [step1_1] "=r"(step1_1), [step1_2] "=r"(step1_2), + [step1_3] "=r"(step1_3) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64), + [cospi_16_64] "r"(cospi_16_64)); + + __asm__ __volatile__( + "lh %[load5], 2(%[input]) \n\t" + "lh %[load6], 30(%[input]) \n\t" + "lh %[load7], 18(%[input]) \n\t" + "lh %[load8], 14(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load5], %[cospi_30_64] \n\t" + "msub $ac1, %[load6], %[cospi_2_64] \n\t" + "extp %[result1], $ac1, 31 \n\t" + + "madd $ac3, %[load7], %[cospi_14_64] \n\t" + "msub $ac3, %[load8], %[cospi_18_64] \n\t" + "extp %[result2], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac1, %[load7], %[cospi_18_64] \n\t" + "madd $ac1, %[load8], %[cospi_14_64] \n\t" + "extp %[result3], $ac1, 31 \n\t" + + "madd $ac2, %[load5], %[cospi_2_64] \n\t" + "madd $ac2, %[load6], %[cospi_30_64] \n\t" + "extp %[result4], $ac2, 31 \n\t" + + "sub %[load5], %[result1], %[result2] \n\t" + "sub %[load6], %[result4], %[result3] \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load6], %[cospi_24_64] \n\t" + "msub $ac1, %[load5], %[cospi_8_64] \n\t" + "madd $ac3, %[load5], %[cospi_24_64] \n\t" + "madd $ac3, %[load6], %[cospi_8_64] \n\t" + + "extp %[step2_9], $ac1, 31 \n\t" + "extp %[step2_14], $ac3, 31 \n\t" + "add %[step2_8], %[result1], %[result2] \n\t" + "add %[step2_15], %[result4], %[result3] \n\t" + + : [load5] "=&r"(load5), [load6] "=&r"(load6), [load7] "=&r"(load7), + [load8] "=&r"(load8), [result1] "=&r"(result1), + [result2] "=&r"(result2), [result3] "=&r"(result3), + [result4] "=&r"(result4), [step2_8] "=r"(step2_8), + [step2_15] "=r"(step2_15), [step2_9] "=r"(step2_9), + [step2_14] "=r"(step2_14) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_30_64] "r"(cospi_30_64), [cospi_2_64] "r"(cospi_2_64), + [cospi_14_64] "r"(cospi_14_64), [cospi_18_64] "r"(cospi_18_64), + [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64)); + + __asm__ __volatile__( + "lh %[load1], 10(%[input]) \n\t" + "lh %[load2], 22(%[input]) \n\t" + "lh %[load3], 26(%[input]) \n\t" + "lh %[load4], 6(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_22_64] \n\t" + "msub $ac1, %[load2], %[cospi_10_64] \n\t" + "extp %[result1], $ac1, 31 \n\t" + + "madd $ac3, %[load3], %[cospi_6_64] \n\t" + "msub $ac3, %[load4], %[cospi_26_64] \n\t" + "extp %[result2], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac1, %[load1], %[cospi_10_64] \n\t" + "madd $ac1, %[load2], %[cospi_22_64] \n\t" + "extp %[result3], $ac1, 31 \n\t" + + "madd $ac2, %[load3], %[cospi_26_64] \n\t" + "madd $ac2, %[load4], %[cospi_6_64] \n\t" + "extp %[result4], $ac2, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[result2], %[result1] \n\t" + "sub %[load2], %[result4], %[result3] \n\t" + + "msub $ac1, %[load1], %[cospi_24_64] \n\t" + "msub $ac1, %[load2], %[cospi_8_64] \n\t" + "madd $ac3, %[load2], %[cospi_24_64] \n\t" + "msub $ac3, %[load1], %[cospi_8_64] \n\t" + + "extp %[step2_10], $ac1, 31 \n\t" + "extp %[step2_13], $ac3, 31 \n\t" + "add %[step2_11], %[result1], %[result2] \n\t" + "add %[step2_12], %[result4], %[result3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [result1] "=&r"(result1), + [result2] "=&r"(result2), [result3] "=&r"(result3), + [result4] "=&r"(result4), [step2_10] "=r"(step2_10), + [step2_11] "=r"(step2_11), [step2_12] "=r"(step2_12), + [step2_13] "=r"(step2_13) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_22_64] "r"(cospi_22_64), [cospi_10_64] "r"(cospi_10_64), + [cospi_6_64] "r"(cospi_6_64), [cospi_26_64] "r"(cospi_26_64), + [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64)); + + __asm__ __volatile__( + "lh %[load5], 4(%[input]) \n\t" + "lh %[load6], 28(%[input]) \n\t" + "lh %[load7], 20(%[input]) \n\t" + "lh %[load8], 12(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load5], %[cospi_28_64] \n\t" + "msub $ac1, %[load6], %[cospi_4_64] \n\t" + "extp %[result1], $ac1, 31 \n\t" + + "madd $ac3, %[load7], %[cospi_12_64] \n\t" + "msub $ac3, %[load8], %[cospi_20_64] \n\t" + "extp %[result2], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac1, %[load7], %[cospi_20_64] \n\t" + "madd $ac1, %[load8], %[cospi_12_64] \n\t" + "extp %[result3], $ac1, 31 \n\t" + + "madd $ac2, %[load5], %[cospi_4_64] \n\t" + "madd $ac2, %[load6], %[cospi_28_64] \n\t" + "extp %[result4], $ac2, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load5], %[result4], %[result3] \n\t" + "sub %[load5], %[load5], %[result1] \n\t" + "add %[load5], %[load5], %[result2] \n\t" + + "sub %[load6], %[result1], %[result2] \n\t" + "sub %[load6], %[load6], %[result3] \n\t" + "add %[load6], %[load6], %[result4] \n\t" + + "madd $ac1, %[load5], %[cospi_16_64] \n\t" + "madd $ac3, %[load6], %[cospi_16_64] \n\t" + + "extp %[step1_5], $ac1, 31 \n\t" + "extp %[step1_6], $ac3, 31 \n\t" + "add %[step1_4], %[result1], %[result2] \n\t" + "add %[step1_7], %[result4], %[result3] \n\t" + + : [load5] "=&r"(load5), [load6] "=&r"(load6), [load7] "=&r"(load7), + [load8] "=&r"(load8), [result1] "=&r"(result1), + [result2] "=&r"(result2), [result3] "=&r"(result3), + [result4] "=&r"(result4), [step1_4] "=r"(step1_4), + [step1_5] "=r"(step1_5), [step1_6] "=r"(step1_6), + [step1_7] "=r"(step1_7) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_20_64] "r"(cospi_20_64), [cospi_12_64] "r"(cospi_12_64), + [cospi_4_64] "r"(cospi_4_64), [cospi_28_64] "r"(cospi_28_64), + [cospi_16_64] "r"(cospi_16_64)); + + __asm__ __volatile__( + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + + "sub %[load5], %[step2_14], %[step2_13] \n\t" + "sub %[load5], %[load5], %[step2_9] \n\t" + "add %[load5], %[load5], %[step2_10] \n\t" + + "madd $ac0, %[load5], %[cospi_16_64] \n\t" + + "sub %[load6], %[step2_14], %[step2_13] \n\t" + "sub %[load6], %[load6], %[step2_10] \n\t" + "add %[load6], %[load6], %[step2_9] \n\t" + + "madd $ac1, %[load6], %[cospi_16_64] \n\t" + + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load5], %[step2_15], %[step2_12] \n\t" + "sub %[load5], %[load5], %[step2_8] \n\t" + "add %[load5], %[load5], %[step2_11] \n\t" + + "madd $ac2, %[load5], %[cospi_16_64] \n\t" + + "sub %[load6], %[step2_15], %[step2_12] \n\t" + "sub %[load6], %[load6], %[step2_11] \n\t" + "add %[load6], %[load6], %[step2_8] \n\t" + + "madd $ac3, %[load6], %[cospi_16_64] \n\t" + + "extp %[step1_10], $ac0, 31 \n\t" + "extp %[step1_13], $ac1, 31 \n\t" + "extp %[step1_11], $ac2, 31 \n\t" + "extp %[step1_12], $ac3, 31 \n\t" + + : [load5] "=&r"(load5), [load6] "=&r"(load6), [step1_10] "=r"(step1_10), + [step1_11] "=r"(step1_11), [step1_12] "=r"(step1_12), + [step1_13] "=r"(step1_13) + : [const_2_power_13] "r"(const_2_power_13), [step2_14] "r"(step2_14), + [step2_13] "r"(step2_13), [step2_9] "r"(step2_9), + [step2_10] "r"(step2_10), [step2_15] "r"(step2_15), + [step2_12] "r"(step2_12), [step2_8] "r"(step2_8), + [step2_11] "r"(step2_11), [cospi_16_64] "r"(cospi_16_64)); + + __asm__ __volatile__( + "add %[load5], %[step1_0], %[step1_7] \n\t" + "add %[load5], %[load5], %[step2_12] \n\t" + "add %[load5], %[load5], %[step2_15] \n\t" + "add %[load6], %[step1_1], %[step1_6] \n\t" + "add %[load6], %[load6], %[step2_13] \n\t" + "add %[load6], %[load6], %[step2_14] \n\t" + "sh %[load5], 0(%[output]) \n\t" + "sh %[load6], 32(%[output]) \n\t" + "sub %[load5], %[step1_1], %[step1_6] \n\t" + "add %[load5], %[load5], %[step2_9] \n\t" + "add %[load5], %[load5], %[step2_10] \n\t" + "sub %[load6], %[step1_0], %[step1_7] \n\t" + "add %[load6], %[load6], %[step2_8] \n\t" + "add %[load6], %[load6], %[step2_11] \n\t" + "sh %[load5], 192(%[output]) \n\t" + "sh %[load6], 224(%[output]) \n\t" + "sub %[load5], %[step1_0], %[step1_7] \n\t" + "sub %[load5], %[load5], %[step2_8] \n\t" + "sub %[load5], %[load5], %[step2_11] \n\t" + "sub %[load6], %[step1_1], %[step1_6] \n\t" + "sub %[load6], %[load6], %[step2_9] \n\t" + "sub %[load6], %[load6], %[step2_10] \n\t" + "sh %[load5], 256(%[output]) \n\t" + "sh %[load6], 288(%[output]) \n\t" + "add %[load5], %[step1_1], %[step1_6] \n\t" + "sub %[load5], %[load5], %[step2_13] \n\t" + "sub %[load5], %[load5], %[step2_14] \n\t" + "add %[load6], %[step1_0], %[step1_7] \n\t" + "sub %[load6], %[load6], %[step2_12] \n\t" + "sub %[load6], %[load6], %[step2_15] \n\t" + "sh %[load5], 448(%[output]) \n\t" + "sh %[load6], 480(%[output]) \n\t" + + : [load5] "=&r"(load5), [load6] "=&r"(load6) + : [output] "r"(output), [step1_0] "r"(step1_0), [step1_1] "r"(step1_1), + [step1_6] "r"(step1_6), [step1_7] "r"(step1_7), + [step2_8] "r"(step2_8), [step2_9] "r"(step2_9), + [step2_10] "r"(step2_10), [step2_11] "r"(step2_11), + [step2_12] "r"(step2_12), [step2_13] "r"(step2_13), + [step2_14] "r"(step2_14), [step2_15] "r"(step2_15)); + + __asm__ __volatile__( + "add %[load5], %[step1_2], %[step1_5] \n\t" + "add %[load5], %[load5], %[step1_13] \n\t" + "add %[load6], %[step1_3], %[step1_4] \n\t" + "add %[load6], %[load6], %[step1_12] \n\t" + "sh %[load5], 64(%[output]) \n\t" + "sh %[load6], 96(%[output]) \n\t" + "sub %[load5], %[step1_3], %[step1_4] \n\t" + "add %[load5], %[load5], %[step1_11] \n\t" + "sub %[load6], %[step1_2], %[step1_5] \n\t" + "add %[load6], %[load6], %[step1_10] \n\t" + "sh %[load5], 128(%[output]) \n\t" + "sh %[load6], 160(%[output]) \n\t" + "sub %[load5], %[step1_2], %[step1_5] \n\t" + "sub %[load5], %[load5], %[step1_10] \n\t" + "sub %[load6], %[step1_3], %[step1_4] \n\t" + "sub %[load6], %[load6], %[step1_11] \n\t" + "sh %[load5], 320(%[output]) \n\t" + "sh %[load6], 352(%[output]) \n\t" + "add %[load5], %[step1_3], %[step1_4] \n\t" + "sub %[load5], %[load5], %[step1_12] \n\t" + "add %[load6], %[step1_2], %[step1_5] \n\t" + "sub %[load6], %[load6], %[step1_13] \n\t" + "sh %[load5], 384(%[output]) \n\t" + "sh %[load6], 416(%[output]) \n\t" + + : [load5] "=&r"(load5), [load6] "=&r"(load6) + : [output] "r"(output), [step1_2] "r"(step1_2), [step1_3] "r"(step1_3), + [step1_4] "r"(step1_4), [step1_5] "r"(step1_5), + [step1_10] "r"(step1_10), [step1_11] "r"(step1_11), + [step1_12] "r"(step1_12), [step1_13] "r"(step1_13)); + + input += 16; + output += 1; + } +} + +void idct16_cols_add_blk_dspr2(int16_t *input, uint8_t *dest, int dest_stride) { + int i; + int step1_0, step1_1, step1_2, step1_3, step1_4, step1_5, step1_6, step1_7; + int step1_8, step1_9, step1_10, step1_11; + int step1_12, step1_13, step1_14, step1_15; + int step2_0, step2_1, step2_2, step2_3; + int step2_8, step2_9, step2_10, step2_11; + int step2_12, step2_13, step2_14, step2_15; + int load1, load2, load3, load4, load5, load6, load7, load8; + int result1, result2, result3, result4; + const int const_2_power_13 = 8192; + uint8_t *dest_pix; + uint8_t *cm = aom_ff_cropTbl; + + /* prefetch aom_ff_cropTbl */ + prefetch_load(aom_ff_cropTbl); + prefetch_load(aom_ff_cropTbl + 32); + prefetch_load(aom_ff_cropTbl + 64); + prefetch_load(aom_ff_cropTbl + 96); + prefetch_load(aom_ff_cropTbl + 128); + prefetch_load(aom_ff_cropTbl + 160); + prefetch_load(aom_ff_cropTbl + 192); + prefetch_load(aom_ff_cropTbl + 224); + + for (i = 0; i < 16; ++i) { + dest_pix = (dest + i); + __asm__ __volatile__( + "lh %[load1], 0(%[input]) \n\t" + "lh %[load2], 16(%[input]) \n\t" + "lh %[load3], 8(%[input]) \n\t" + "lh %[load4], 24(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "add %[result1], %[load1], %[load2] \n\t" + "sub %[result2], %[load1], %[load2] \n\t" + "madd $ac1, %[result1], %[cospi_16_64] \n\t" + "madd $ac2, %[result2], %[cospi_16_64] \n\t" + "extp %[step2_0], $ac1, 31 \n\t" + "extp %[step2_1], $ac2, 31 \n\t" + + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "madd $ac3, %[load3], %[cospi_24_64] \n\t" + "msub $ac3, %[load4], %[cospi_8_64] \n\t" + "extp %[step2_2], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "madd $ac1, %[load3], %[cospi_8_64] \n\t" + "madd $ac1, %[load4], %[cospi_24_64] \n\t" + "extp %[step2_3], $ac1, 31 \n\t" + + "add %[step1_0], %[step2_0], %[step2_3] \n\t" + "add %[step1_1], %[step2_1], %[step2_2] \n\t" + "sub %[step1_2], %[step2_1], %[step2_2] \n\t" + "sub %[step1_3], %[step2_0], %[step2_3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [result1] "=&r"(result1), + [result2] "=&r"(result2), [step2_0] "=&r"(step2_0), + [step2_1] "=&r"(step2_1), [step2_2] "=&r"(step2_2), + [step2_3] "=&r"(step2_3), [step1_0] "=r"(step1_0), + [step1_1] "=r"(step1_1), [step1_2] "=r"(step1_2), + [step1_3] "=r"(step1_3) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64), + [cospi_16_64] "r"(cospi_16_64)); + + __asm__ __volatile__( + "lh %[load5], 2(%[input]) \n\t" + "lh %[load6], 30(%[input]) \n\t" + "lh %[load7], 18(%[input]) \n\t" + "lh %[load8], 14(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load5], %[cospi_30_64] \n\t" + "msub $ac1, %[load6], %[cospi_2_64] \n\t" + "extp %[result1], $ac1, 31 \n\t" + + "madd $ac3, %[load7], %[cospi_14_64] \n\t" + "msub $ac3, %[load8], %[cospi_18_64] \n\t" + "extp %[result2], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac1, %[load7], %[cospi_18_64] \n\t" + "madd $ac1, %[load8], %[cospi_14_64] \n\t" + "extp %[result3], $ac1, 31 \n\t" + + "madd $ac2, %[load5], %[cospi_2_64] \n\t" + "madd $ac2, %[load6], %[cospi_30_64] \n\t" + "extp %[result4], $ac2, 31 \n\t" + + "sub %[load5], %[result1], %[result2] \n\t" + "sub %[load6], %[result4], %[result3] \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load6], %[cospi_24_64] \n\t" + "msub $ac1, %[load5], %[cospi_8_64] \n\t" + "madd $ac3, %[load5], %[cospi_24_64] \n\t" + "madd $ac3, %[load6], %[cospi_8_64] \n\t" + + "extp %[step2_9], $ac1, 31 \n\t" + "extp %[step2_14], $ac3, 31 \n\t" + "add %[step2_8], %[result1], %[result2] \n\t" + "add %[step2_15], %[result4], %[result3] \n\t" + + : [load5] "=&r"(load5), [load6] "=&r"(load6), [load7] "=&r"(load7), + [load8] "=&r"(load8), [result1] "=&r"(result1), + [result2] "=&r"(result2), [result3] "=&r"(result3), + [result4] "=&r"(result4), [step2_8] "=r"(step2_8), + [step2_15] "=r"(step2_15), [step2_9] "=r"(step2_9), + [step2_14] "=r"(step2_14) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_30_64] "r"(cospi_30_64), [cospi_2_64] "r"(cospi_2_64), + [cospi_14_64] "r"(cospi_14_64), [cospi_18_64] "r"(cospi_18_64), + [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64)); + + __asm__ __volatile__( + "lh %[load1], 10(%[input]) \n\t" + "lh %[load2], 22(%[input]) \n\t" + "lh %[load3], 26(%[input]) \n\t" + "lh %[load4], 6(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_22_64] \n\t" + "msub $ac1, %[load2], %[cospi_10_64] \n\t" + "extp %[result1], $ac1, 31 \n\t" + + "madd $ac3, %[load3], %[cospi_6_64] \n\t" + "msub $ac3, %[load4], %[cospi_26_64] \n\t" + "extp %[result2], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac1, %[load1], %[cospi_10_64] \n\t" + "madd $ac1, %[load2], %[cospi_22_64] \n\t" + "extp %[result3], $ac1, 31 \n\t" + + "madd $ac2, %[load3], %[cospi_26_64] \n\t" + "madd $ac2, %[load4], %[cospi_6_64] \n\t" + "extp %[result4], $ac2, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[result2], %[result1] \n\t" + "sub %[load2], %[result4], %[result3] \n\t" + + "msub $ac1, %[load1], %[cospi_24_64] \n\t" + "msub $ac1, %[load2], %[cospi_8_64] \n\t" + "madd $ac3, %[load2], %[cospi_24_64] \n\t" + "msub $ac3, %[load1], %[cospi_8_64] \n\t" + + "extp %[step2_10], $ac1, 31 \n\t" + "extp %[step2_13], $ac3, 31 \n\t" + "add %[step2_11], %[result1], %[result2] \n\t" + "add %[step2_12], %[result4], %[result3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [result1] "=&r"(result1), + [result2] "=&r"(result2), [result3] "=&r"(result3), + [result4] "=&r"(result4), [step2_10] "=r"(step2_10), + [step2_11] "=r"(step2_11), [step2_12] "=r"(step2_12), + [step2_13] "=r"(step2_13) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_22_64] "r"(cospi_22_64), [cospi_10_64] "r"(cospi_10_64), + [cospi_6_64] "r"(cospi_6_64), [cospi_26_64] "r"(cospi_26_64), + [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64)); + + __asm__ __volatile__( + "lh %[load5], 4(%[input]) \n\t" + "lh %[load6], 28(%[input]) \n\t" + "lh %[load7], 20(%[input]) \n\t" + "lh %[load8], 12(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load5], %[cospi_28_64] \n\t" + "msub $ac1, %[load6], %[cospi_4_64] \n\t" + "extp %[result1], $ac1, 31 \n\t" + + "madd $ac3, %[load7], %[cospi_12_64] \n\t" + "msub $ac3, %[load8], %[cospi_20_64] \n\t" + "extp %[result2], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac1, %[load7], %[cospi_20_64] \n\t" + "madd $ac1, %[load8], %[cospi_12_64] \n\t" + "extp %[result3], $ac1, 31 \n\t" + + "madd $ac2, %[load5], %[cospi_4_64] \n\t" + "madd $ac2, %[load6], %[cospi_28_64] \n\t" + "extp %[result4], $ac2, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load5], %[result4], %[result3] \n\t" + "sub %[load5], %[load5], %[result1] \n\t" + "add %[load5], %[load5], %[result2] \n\t" + + "sub %[load6], %[result1], %[result2] \n\t" + "sub %[load6], %[load6], %[result3] \n\t" + "add %[load6], %[load6], %[result4] \n\t" + + "madd $ac1, %[load5], %[cospi_16_64] \n\t" + "madd $ac3, %[load6], %[cospi_16_64] \n\t" + + "extp %[step1_5], $ac1, 31 \n\t" + "extp %[step1_6], $ac3, 31 \n\t" + + "add %[step1_4], %[result1], %[result2] \n\t" + "add %[step1_7], %[result4], %[result3] \n\t" + + : [load5] "=&r"(load5), [load6] "=&r"(load6), [load7] "=&r"(load7), + [load8] "=&r"(load8), [result1] "=&r"(result1), + [result2] "=&r"(result2), [result3] "=&r"(result3), + [result4] "=&r"(result4), [step1_4] "=r"(step1_4), + [step1_5] "=r"(step1_5), [step1_6] "=r"(step1_6), + [step1_7] "=r"(step1_7) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_20_64] "r"(cospi_20_64), [cospi_12_64] "r"(cospi_12_64), + [cospi_4_64] "r"(cospi_4_64), [cospi_28_64] "r"(cospi_28_64), + [cospi_16_64] "r"(cospi_16_64)); + + __asm__ __volatile__( + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + + "sub %[load5], %[step2_14], %[step2_13] \n\t" + "sub %[load5], %[load5], %[step2_9] \n\t" + "add %[load5], %[load5], %[step2_10] \n\t" + + "madd $ac0, %[load5], %[cospi_16_64] \n\t" + + "sub %[load6], %[step2_14], %[step2_13] \n\t" + "sub %[load6], %[load6], %[step2_10] \n\t" + "add %[load6], %[load6], %[step2_9] \n\t" + + "madd $ac1, %[load6], %[cospi_16_64] \n\t" + + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load5], %[step2_15], %[step2_12] \n\t" + "sub %[load5], %[load5], %[step2_8] \n\t" + "add %[load5], %[load5], %[step2_11] \n\t" + + "madd $ac2, %[load5], %[cospi_16_64] \n\t" + + "sub %[load6], %[step2_15], %[step2_12] \n\t" + "sub %[load6], %[load6], %[step2_11] \n\t" + "add %[load6], %[load6], %[step2_8] \n\t" + + "madd $ac3, %[load6], %[cospi_16_64] \n\t" + + "extp %[step1_10], $ac0, 31 \n\t" + "extp %[step1_13], $ac1, 31 \n\t" + "extp %[step1_11], $ac2, 31 \n\t" + "extp %[step1_12], $ac3, 31 \n\t" + + : [load5] "=&r"(load5), [load6] "=&r"(load6), [step1_10] "=r"(step1_10), + [step1_11] "=r"(step1_11), [step1_12] "=r"(step1_12), + [step1_13] "=r"(step1_13) + : [const_2_power_13] "r"(const_2_power_13), [step2_14] "r"(step2_14), + [step2_13] "r"(step2_13), [step2_9] "r"(step2_9), + [step2_10] "r"(step2_10), [step2_15] "r"(step2_15), + [step2_12] "r"(step2_12), [step2_8] "r"(step2_8), + [step2_11] "r"(step2_11), [cospi_16_64] "r"(cospi_16_64)); + + step1_8 = step2_8 + step2_11; + step1_9 = step2_9 + step2_10; + step1_14 = step2_13 + step2_14; + step1_15 = step2_12 + step2_15; + + __asm__ __volatile__( + "lbu %[load7], 0(%[dest_pix]) \n\t" + "add %[load5], %[step1_0], %[step1_7] \n\t" + "add %[load5], %[load5], %[step1_15] \n\t" + "addi %[load5], %[load5], 32 \n\t" + "sra %[load5], %[load5], 6 \n\t" + "add %[load7], %[load7], %[load5] \n\t" + "lbux %[load5], %[load7](%[cm]) \n\t" + "add %[load6], %[step1_1], %[step1_6] \n\t" + "add %[load6], %[load6], %[step1_14] \n\t" + "sb %[load5], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[load8], 0(%[dest_pix]) \n\t" + "addi %[load6], %[load6], 32 \n\t" + "sra %[load6], %[load6], 6 \n\t" + "add %[load8], %[load8], %[load6] \n\t" + "lbux %[load6], %[load8](%[cm]) \n\t" + "sb %[load6], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[load7], 0(%[dest_pix]) \n\t" + "add %[load5], %[step1_2], %[step1_5] \n\t" + "add %[load5], %[load5], %[step1_13] \n\t" + "addi %[load5], %[load5], 32 \n\t" + "sra %[load5], %[load5], 6 \n\t" + "add %[load7], %[load7], %[load5] \n\t" + "lbux %[load5], %[load7](%[cm]) \n\t" + "add %[load6], %[step1_3], %[step1_4] \n\t" + "add %[load6], %[load6], %[step1_12] \n\t" + "sb %[load5], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[load8], 0(%[dest_pix]) \n\t" + "addi %[load6], %[load6], 32 \n\t" + "sra %[load6], %[load6], 6 \n\t" + "add %[load8], %[load8], %[load6] \n\t" + "lbux %[load6], %[load8](%[cm]) \n\t" + "sb %[load6], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[load7], 0(%[dest_pix]) \n\t" + "sub %[load5], %[step1_3], %[step1_4] \n\t" + "add %[load5], %[load5], %[step1_11] \n\t" + "addi %[load5], %[load5], 32 \n\t" + "sra %[load5], %[load5], 6 \n\t" + "add %[load7], %[load7], %[load5] \n\t" + "lbux %[load5], %[load7](%[cm]) \n\t" + "sub %[load6], %[step1_2], %[step1_5] \n\t" + "add %[load6], %[load6], %[step1_10] \n\t" + "sb %[load5], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[load8], 0(%[dest_pix]) \n\t" + "addi %[load6], %[load6], 32 \n\t" + "sra %[load6], %[load6], 6 \n\t" + "add %[load8], %[load8], %[load6] \n\t" + "lbux %[load6], %[load8](%[cm]) \n\t" + "sb %[load6], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "sub %[load5], %[step1_1], %[step1_6] \n\t" + "lbu %[load7], 0(%[dest_pix]) \n\t" + "add %[load5], %[load5], %[step1_9] \n\t" + "addi %[load5], %[load5], 32 \n\t" + "sra %[load5], %[load5], 6 \n\t" + "add %[load7], %[load7], %[load5] \n\t" + "lbux %[load5], %[load7](%[cm]) \n\t" + "sub %[load6], %[step1_0], %[step1_7] \n\t" + "add %[load6], %[load6], %[step1_8] \n\t" + "sb %[load5], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[load8], 0(%[dest_pix]) \n\t" + "addi %[load6], %[load6], 32 \n\t" + "sra %[load6], %[load6], 6 \n\t" + "add %[load8], %[load8], %[load6] \n\t" + "lbux %[load6], %[load8](%[cm]) \n\t" + "sb %[load6], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[load7], 0(%[dest_pix]) \n\t" + "sub %[load5], %[step1_0], %[step1_7] \n\t" + "sub %[load5], %[load5], %[step1_8] \n\t" + "addi %[load5], %[load5], 32 \n\t" + "sra %[load5], %[load5], 6 \n\t" + "add %[load7], %[load7], %[load5] \n\t" + "lbux %[load5], %[load7](%[cm]) \n\t" + "sub %[load6], %[step1_1], %[step1_6] \n\t" + "sub %[load6], %[load6], %[step1_9] \n\t" + "sb %[load5], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[load8], 0(%[dest_pix]) \n\t" + "addi %[load6], %[load6], 32 \n\t" + "sra %[load6], %[load6], 6 \n\t" + "add %[load8], %[load8], %[load6] \n\t" + "lbux %[load6], %[load8](%[cm]) \n\t" + "sb %[load6], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[load7], 0(%[dest_pix]) \n\t" + "sub %[load5], %[step1_2], %[step1_5] \n\t" + "sub %[load5], %[load5], %[step1_10] \n\t" + "addi %[load5], %[load5], 32 \n\t" + "sra %[load5], %[load5], 6 \n\t" + "add %[load7], %[load7], %[load5] \n\t" + "lbux %[load5], %[load7](%[cm]) \n\t" + "sub %[load6], %[step1_3], %[step1_4] \n\t" + "sub %[load6], %[load6], %[step1_11] \n\t" + "sb %[load5], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[load8], 0(%[dest_pix]) \n\t" + "addi %[load6], %[load6], 32 \n\t" + "sra %[load6], %[load6], 6 \n\t" + "add %[load8], %[load8], %[load6] \n\t" + "lbux %[load6], %[load8](%[cm]) \n\t" + "sb %[load6], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[load7], 0(%[dest_pix]) \n\t" + "add %[load5], %[step1_3], %[step1_4] \n\t" + "sub %[load5], %[load5], %[step1_12] \n\t" + "addi %[load5], %[load5], 32 \n\t" + "sra %[load5], %[load5], 6 \n\t" + "add %[load7], %[load7], %[load5] \n\t" + "lbux %[load5], %[load7](%[cm]) \n\t" + "add %[load6], %[step1_2], %[step1_5] \n\t" + "sub %[load6], %[load6], %[step1_13] \n\t" + "sb %[load5], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[load8], 0(%[dest_pix]) \n\t" + "addi %[load6], %[load6], 32 \n\t" + "sra %[load6], %[load6], 6 \n\t" + "add %[load8], %[load8], %[load6] \n\t" + "lbux %[load6], %[load8](%[cm]) \n\t" + "sb %[load6], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[load7], 0(%[dest_pix]) \n\t" + "add %[load5], %[step1_1], %[step1_6] \n\t" + "sub %[load5], %[load5], %[step1_14] \n\t" + "addi %[load5], %[load5], 32 \n\t" + "sra %[load5], %[load5], 6 \n\t" + "add %[load7], %[load7], %[load5] \n\t" + "lbux %[load5], %[load7](%[cm]) \n\t" + "add %[load6], %[step1_0], %[step1_7] \n\t" + "sub %[load6], %[load6], %[step1_15] \n\t" + "sb %[load5], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[load8], 0(%[dest_pix]) \n\t" + "addi %[load6], %[load6], 32 \n\t" + "sra %[load6], %[load6], 6 \n\t" + "add %[load8], %[load8], %[load6] \n\t" + "lbux %[load6], %[load8](%[cm]) \n\t" + "sb %[load6], 0(%[dest_pix]) \n\t" + + : [load5] "=&r"(load5), [load6] "=&r"(load6), [load7] "=&r"(load7), + [load8] "=&r"(load8), [dest_pix] "+r"(dest_pix) + : + [cm] "r"(cm), [dest_stride] "r"(dest_stride), [step1_0] "r"(step1_0), + [step1_1] "r"(step1_1), [step1_2] "r"(step1_2), [step1_3] "r"(step1_3), + [step1_4] "r"(step1_4), [step1_5] "r"(step1_5), [step1_6] "r"(step1_6), + [step1_7] "r"(step1_7), [step1_8] "r"(step1_8), [step1_9] "r"(step1_9), + [step1_10] "r"(step1_10), [step1_11] "r"(step1_11), + [step1_12] "r"(step1_12), [step1_13] "r"(step1_13), + [step1_14] "r"(step1_14), [step1_15] "r"(step1_15)); + + input += 16; + } +} + +void aom_idct16x16_256_add_dspr2(const int16_t *input, uint8_t *dest, + int dest_stride) { + DECLARE_ALIGNED(32, int16_t, out[16 * 16]); + uint32_t pos = 45; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" : : [pos] "r"(pos)); + + // First transform rows + idct16_rows_dspr2(input, out, 16); + + // Then transform columns and add to dest + idct16_cols_add_blk_dspr2(out, dest, dest_stride); +} + +void aom_idct16x16_10_add_dspr2(const int16_t *input, uint8_t *dest, + int dest_stride) { + DECLARE_ALIGNED(32, int16_t, out[16 * 16]); + int16_t *outptr = out; + uint32_t i; + uint32_t pos = 45; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" : : [pos] "r"(pos)); + + // First transform rows. Since all non-zero dct coefficients are in + // upper-left 4x4 area, we only need to calculate first 4 rows here. + idct16_rows_dspr2(input, outptr, 4); + + outptr += 4; + for (i = 0; i < 6; ++i) { + __asm__ __volatile__( + "sw $zero, 0(%[outptr]) \n\t" + "sw $zero, 32(%[outptr]) \n\t" + "sw $zero, 64(%[outptr]) \n\t" + "sw $zero, 96(%[outptr]) \n\t" + "sw $zero, 128(%[outptr]) \n\t" + "sw $zero, 160(%[outptr]) \n\t" + "sw $zero, 192(%[outptr]) \n\t" + "sw $zero, 224(%[outptr]) \n\t" + "sw $zero, 256(%[outptr]) \n\t" + "sw $zero, 288(%[outptr]) \n\t" + "sw $zero, 320(%[outptr]) \n\t" + "sw $zero, 352(%[outptr]) \n\t" + "sw $zero, 384(%[outptr]) \n\t" + "sw $zero, 416(%[outptr]) \n\t" + "sw $zero, 448(%[outptr]) \n\t" + "sw $zero, 480(%[outptr]) \n\t" + + : + : [outptr] "r"(outptr)); + + outptr += 2; + } + + // Then transform columns + idct16_cols_add_blk_dspr2(out, dest, dest_stride); +} + +void aom_idct16x16_1_add_dspr2(const int16_t *input, uint8_t *dest, + int dest_stride) { + uint32_t pos = 45; + int32_t out; + int32_t r; + int32_t a1, absa1; + int32_t vector_a1; + int32_t t1, t2, t3, t4; + int32_t vector_1, vector_2, vector_3, vector_4; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + + : + : [pos] "r"(pos)); + + out = DCT_CONST_ROUND_SHIFT_TWICE_COSPI_16_64(input[0]); + __asm__ __volatile__( + "addi %[out], %[out], 32 \n\t" + "sra %[a1], %[out], 6 \n\t" + + : [out] "+r"(out), [a1] "=r"(a1) + :); + + if (a1 < 0) { + /* use quad-byte + * input and output memory are four byte aligned */ + __asm__ __volatile__( + "abs %[absa1], %[a1] \n\t" + "replv.qb %[vector_a1], %[absa1] \n\t" + + : [absa1] "=r"(absa1), [vector_a1] "=r"(vector_a1) + : [a1] "r"(a1)); + + for (r = 16; r--;) { + __asm__ __volatile__( + "lw %[t1], 0(%[dest]) \n\t" + "lw %[t2], 4(%[dest]) \n\t" + "lw %[t3], 8(%[dest]) \n\t" + "lw %[t4], 12(%[dest]) \n\t" + "subu_s.qb %[vector_1], %[t1], %[vector_a1] \n\t" + "subu_s.qb %[vector_2], %[t2], %[vector_a1] \n\t" + "subu_s.qb %[vector_3], %[t3], %[vector_a1] \n\t" + "subu_s.qb %[vector_4], %[t4], %[vector_a1] \n\t" + "sw %[vector_1], 0(%[dest]) \n\t" + "sw %[vector_2], 4(%[dest]) \n\t" + "sw %[vector_3], 8(%[dest]) \n\t" + "sw %[vector_4], 12(%[dest]) \n\t" + "add %[dest], %[dest], %[dest_stride] \n\t" + + : [t1] "=&r"(t1), [t2] "=&r"(t2), [t3] "=&r"(t3), [t4] "=&r"(t4), + [vector_1] "=&r"(vector_1), [vector_2] "=&r"(vector_2), + [vector_3] "=&r"(vector_3), [vector_4] "=&r"(vector_4), + [dest] "+&r"(dest) + : [dest_stride] "r"(dest_stride), [vector_a1] "r"(vector_a1)); + } + } else { + /* use quad-byte + * input and output memory are four byte aligned */ + __asm__ __volatile__("replv.qb %[vector_a1], %[a1] \n\t" + + : [vector_a1] "=r"(vector_a1) + : [a1] "r"(a1)); + + for (r = 16; r--;) { + __asm__ __volatile__( + "lw %[t1], 0(%[dest]) \n\t" + "lw %[t2], 4(%[dest]) \n\t" + "lw %[t3], 8(%[dest]) \n\t" + "lw %[t4], 12(%[dest]) \n\t" + "addu_s.qb %[vector_1], %[t1], %[vector_a1] \n\t" + "addu_s.qb %[vector_2], %[t2], %[vector_a1] \n\t" + "addu_s.qb %[vector_3], %[t3], %[vector_a1] \n\t" + "addu_s.qb %[vector_4], %[t4], %[vector_a1] \n\t" + "sw %[vector_1], 0(%[dest]) \n\t" + "sw %[vector_2], 4(%[dest]) \n\t" + "sw %[vector_3], 8(%[dest]) \n\t" + "sw %[vector_4], 12(%[dest]) \n\t" + "add %[dest], %[dest], %[dest_stride] \n\t" + + : [t1] "=&r"(t1), [t2] "=&r"(t2), [t3] "=&r"(t3), [t4] "=&r"(t4), + [vector_1] "=&r"(vector_1), [vector_2] "=&r"(vector_2), + [vector_3] "=&r"(vector_3), [vector_4] "=&r"(vector_4), + [dest] "+&r"(dest) + : [dest_stride] "r"(dest_stride), [vector_a1] "r"(vector_a1)); + } + } +} + +void iadst16_dspr2(const int16_t *input, int16_t *output) { + int s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, s15; + + int x0 = input[15]; + int x1 = input[0]; + int x2 = input[13]; + int x3 = input[2]; + int x4 = input[11]; + int x5 = input[4]; + int x6 = input[9]; + int x7 = input[6]; + int x8 = input[7]; + int x9 = input[8]; + int x10 = input[5]; + int x11 = input[10]; + int x12 = input[3]; + int x13 = input[12]; + int x14 = input[1]; + int x15 = input[14]; + + if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7 | x8 | x9 | x10 | x11 | x12 | + x13 | x14 | x15)) { + output[0] = output[1] = output[2] = output[3] = output[4] = output[5] = + output[6] = output[7] = output[8] = output[9] = output[10] = + output[11] = output[12] = output[13] = output[14] = output[15] = 0; + return; + } + + // stage 1 + s0 = x0 * cospi_1_64 + x1 * cospi_31_64; + s1 = x0 * cospi_31_64 - x1 * cospi_1_64; + s2 = x2 * cospi_5_64 + x3 * cospi_27_64; + s3 = x2 * cospi_27_64 - x3 * cospi_5_64; + s4 = x4 * cospi_9_64 + x5 * cospi_23_64; + s5 = x4 * cospi_23_64 - x5 * cospi_9_64; + s6 = x6 * cospi_13_64 + x7 * cospi_19_64; + s7 = x6 * cospi_19_64 - x7 * cospi_13_64; + s8 = x8 * cospi_17_64 + x9 * cospi_15_64; + s9 = x8 * cospi_15_64 - x9 * cospi_17_64; + s10 = x10 * cospi_21_64 + x11 * cospi_11_64; + s11 = x10 * cospi_11_64 - x11 * cospi_21_64; + s12 = x12 * cospi_25_64 + x13 * cospi_7_64; + s13 = x12 * cospi_7_64 - x13 * cospi_25_64; + s14 = x14 * cospi_29_64 + x15 * cospi_3_64; + s15 = x14 * cospi_3_64 - x15 * cospi_29_64; + + x0 = dct_const_round_shift(s0 + s8); + x1 = dct_const_round_shift(s1 + s9); + x2 = dct_const_round_shift(s2 + s10); + x3 = dct_const_round_shift(s3 + s11); + x4 = dct_const_round_shift(s4 + s12); + x5 = dct_const_round_shift(s5 + s13); + x6 = dct_const_round_shift(s6 + s14); + x7 = dct_const_round_shift(s7 + s15); + x8 = dct_const_round_shift(s0 - s8); + x9 = dct_const_round_shift(s1 - s9); + x10 = dct_const_round_shift(s2 - s10); + x11 = dct_const_round_shift(s3 - s11); + x12 = dct_const_round_shift(s4 - s12); + x13 = dct_const_round_shift(s5 - s13); + x14 = dct_const_round_shift(s6 - s14); + x15 = dct_const_round_shift(s7 - s15); + + // stage 2 + s0 = x0; + s1 = x1; + s2 = x2; + s3 = x3; + s4 = x4; + s5 = x5; + s6 = x6; + s7 = x7; + s8 = x8 * cospi_4_64 + x9 * cospi_28_64; + s9 = x8 * cospi_28_64 - x9 * cospi_4_64; + s10 = x10 * cospi_20_64 + x11 * cospi_12_64; + s11 = x10 * cospi_12_64 - x11 * cospi_20_64; + s12 = -x12 * cospi_28_64 + x13 * cospi_4_64; + s13 = x12 * cospi_4_64 + x13 * cospi_28_64; + s14 = -x14 * cospi_12_64 + x15 * cospi_20_64; + s15 = x14 * cospi_20_64 + x15 * cospi_12_64; + + x0 = s0 + s4; + x1 = s1 + s5; + x2 = s2 + s6; + x3 = s3 + s7; + x4 = s0 - s4; + x5 = s1 - s5; + x6 = s2 - s6; + x7 = s3 - s7; + x8 = dct_const_round_shift(s8 + s12); + x9 = dct_const_round_shift(s9 + s13); + x10 = dct_const_round_shift(s10 + s14); + x11 = dct_const_round_shift(s11 + s15); + x12 = dct_const_round_shift(s8 - s12); + x13 = dct_const_round_shift(s9 - s13); + x14 = dct_const_round_shift(s10 - s14); + x15 = dct_const_round_shift(s11 - s15); + + // stage 3 + s0 = x0; + s1 = x1; + s2 = x2; + s3 = x3; + s4 = x4 * cospi_8_64 + x5 * cospi_24_64; + s5 = x4 * cospi_24_64 - x5 * cospi_8_64; + s6 = -x6 * cospi_24_64 + x7 * cospi_8_64; + s7 = x6 * cospi_8_64 + x7 * cospi_24_64; + s8 = x8; + s9 = x9; + s10 = x10; + s11 = x11; + s12 = x12 * cospi_8_64 + x13 * cospi_24_64; + s13 = x12 * cospi_24_64 - x13 * cospi_8_64; + s14 = -x14 * cospi_24_64 + x15 * cospi_8_64; + s15 = x14 * cospi_8_64 + x15 * cospi_24_64; + + x0 = s0 + s2; + x1 = s1 + s3; + x2 = s0 - s2; + x3 = s1 - s3; + x4 = dct_const_round_shift(s4 + s6); + x5 = dct_const_round_shift(s5 + s7); + x6 = dct_const_round_shift(s4 - s6); + x7 = dct_const_round_shift(s5 - s7); + x8 = s8 + s10; + x9 = s9 + s11; + x10 = s8 - s10; + x11 = s9 - s11; + x12 = dct_const_round_shift(s12 + s14); + x13 = dct_const_round_shift(s13 + s15); + x14 = dct_const_round_shift(s12 - s14); + x15 = dct_const_round_shift(s13 - s15); + + // stage 4 + s2 = (-cospi_16_64) * (x2 + x3); + s3 = cospi_16_64 * (x2 - x3); + s6 = cospi_16_64 * (x6 + x7); + s7 = cospi_16_64 * (-x6 + x7); + s10 = cospi_16_64 * (x10 + x11); + s11 = cospi_16_64 * (-x10 + x11); + s14 = (-cospi_16_64) * (x14 + x15); + s15 = cospi_16_64 * (x14 - x15); + + x2 = dct_const_round_shift(s2); + x3 = dct_const_round_shift(s3); + x6 = dct_const_round_shift(s6); + x7 = dct_const_round_shift(s7); + x10 = dct_const_round_shift(s10); + x11 = dct_const_round_shift(s11); + x14 = dct_const_round_shift(s14); + x15 = dct_const_round_shift(s15); + + output[0] = x0; + output[1] = -x8; + output[2] = x12; + output[3] = -x4; + output[4] = x6; + output[5] = x14; + output[6] = x10; + output[7] = x2; + output[8] = x3; + output[9] = x11; + output[10] = x15; + output[11] = x7; + output[12] = x5; + output[13] = -x13; + output[14] = x9; + output[15] = -x1; +} + +#endif // HAVE_DSPR2 diff --git a/third_party/aom/aom_dsp/mips/itrans32_cols_dspr2.c b/third_party/aom/aom_dsp/mips/itrans32_cols_dspr2.c new file mode 100644 index 0000000000..d469d1ad0b --- /dev/null +++ b/third_party/aom/aom_dsp/mips/itrans32_cols_dspr2.c @@ -0,0 +1,1042 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" +#include "aom_dsp/mips/inv_txfm_dspr2.h" +#include "aom_dsp/txfm_common.h" + +#if HAVE_DSPR2 +void aom_idct32_cols_add_blk_dspr2(int16_t *input, uint8_t *dest, + int dest_stride) { + int16_t step1_0, step1_1, step1_2, step1_3, step1_4, step1_5, step1_6; + int16_t step1_7, step1_8, step1_9, step1_10, step1_11, step1_12, step1_13; + int16_t step1_14, step1_15, step1_16, step1_17, step1_18, step1_19; + int16_t step1_20, step1_21, step1_22, step1_23, step1_24, step1_25, step1_26; + int16_t step1_27, step1_28, step1_29, step1_30, step1_31; + int16_t step2_0, step2_1, step2_2, step2_3, step2_4, step2_5, step2_6; + int16_t step2_7, step2_8, step2_9, step2_10, step2_11, step2_12, step2_13; + int16_t step2_14, step2_15, step2_16, step2_17, step2_18, step2_19, step2_20; + int16_t step2_21, step2_22, step2_23, step2_24, step2_25, step2_26, step2_27; + int16_t step2_28, step2_29, step2_30, step2_31; + int16_t step3_8, step3_9, step3_10, step3_11, step3_12, step3_13, step3_14; + int16_t step3_15, step3_16, step3_17, step3_18, step3_19, step3_20, step3_21; + int16_t step3_22, step3_23, step3_24, step3_25, step3_26, step3_27; + int16_t step3_28, step3_29, step3_30, step3_31; + int temp0, temp1, temp2, temp3; + int load1, load2, load3, load4; + int result1, result2; + int i, temp21; + uint8_t *dest_pix, *dest_pix1; + const int const_2_power_13 = 8192; + uint8_t *cm = aom_ff_cropTbl; + + /* prefetch aom_ff_cropTbl */ + prefetch_load(aom_ff_cropTbl); + prefetch_load(aom_ff_cropTbl + 32); + prefetch_load(aom_ff_cropTbl + 64); + prefetch_load(aom_ff_cropTbl + 96); + prefetch_load(aom_ff_cropTbl + 128); + prefetch_load(aom_ff_cropTbl + 160); + prefetch_load(aom_ff_cropTbl + 192); + prefetch_load(aom_ff_cropTbl + 224); + + for (i = 0; i < 32; ++i) { + dest_pix = dest + i; + dest_pix1 = dest + i + 31 * dest_stride; + + __asm__ __volatile__( + "lh %[load1], 2(%[input]) \n\t" + "lh %[load2], 62(%[input]) \n\t" + "lh %[load3], 34(%[input]) \n\t" + "lh %[load4], 30(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_31_64] \n\t" + "msub $ac1, %[load2], %[cospi_1_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + + "madd $ac3, %[load1], %[cospi_1_64] \n\t" + "madd $ac3, %[load2], %[cospi_31_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_15_64] \n\t" + "msub $ac2, %[load4], %[cospi_17_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + + "madd $ac1, %[load3], %[cospi_17_64] \n\t" + "madd $ac1, %[load4], %[cospi_15_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp3], %[temp2] \n\t" + "sub %[load2], %[temp0], %[temp1] \n\t" + + "madd $ac1, %[load1], %[cospi_28_64] \n\t" + "msub $ac1, %[load2], %[cospi_4_64] \n\t" + "madd $ac3, %[load1], %[cospi_4_64] \n\t" + "madd $ac3, %[load2], %[cospi_28_64] \n\t" + + "extp %[step1_17], $ac1, 31 \n\t" + "extp %[step1_30], $ac3, 31 \n\t" + "add %[step1_16], %[temp0], %[temp1] \n\t" + "add %[step1_31], %[temp2], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step1_16] "=r"(step1_16), + [step1_17] "=r"(step1_17), [step1_30] "=r"(step1_30), + [step1_31] "=r"(step1_31) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_31_64] "r"(cospi_31_64), [cospi_1_64] "r"(cospi_1_64), + [cospi_4_64] "r"(cospi_4_64), [cospi_17_64] "r"(cospi_17_64), + [cospi_15_64] "r"(cospi_15_64), [cospi_28_64] "r"(cospi_28_64)); + + __asm__ __volatile__( + "lh %[load1], 18(%[input]) \n\t" + "lh %[load2], 46(%[input]) \n\t" + "lh %[load3], 50(%[input]) \n\t" + "lh %[load4], 14(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_23_64] \n\t" + "msub $ac1, %[load2], %[cospi_9_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + + "madd $ac3, %[load1], %[cospi_9_64] \n\t" + "madd $ac3, %[load2], %[cospi_23_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_7_64] \n\t" + "msub $ac2, %[load4], %[cospi_25_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + + "madd $ac1, %[load3], %[cospi_25_64] \n\t" + "madd $ac1, %[load4], %[cospi_7_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp1], %[temp0] \n\t" + "sub %[load2], %[temp2], %[temp3] \n\t" + + "msub $ac1, %[load1], %[cospi_28_64] \n\t" + "msub $ac1, %[load2], %[cospi_4_64] \n\t" + "msub $ac3, %[load1], %[cospi_4_64] \n\t" + "madd $ac3, %[load2], %[cospi_28_64] \n\t" + + "extp %[step1_18], $ac1, 31 \n\t" + "extp %[step1_29], $ac3, 31 \n\t" + "add %[step1_19], %[temp0], %[temp1] \n\t" + "add %[step1_28], %[temp2], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step1_18] "=r"(step1_18), + [step1_19] "=r"(step1_19), [step1_28] "=r"(step1_28), + [step1_29] "=r"(step1_29) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_23_64] "r"(cospi_23_64), [cospi_9_64] "r"(cospi_9_64), + [cospi_4_64] "r"(cospi_4_64), [cospi_7_64] "r"(cospi_7_64), + [cospi_25_64] "r"(cospi_25_64), [cospi_28_64] "r"(cospi_28_64)); + + __asm__ __volatile__( + "lh %[load1], 10(%[input]) \n\t" + "lh %[load2], 54(%[input]) \n\t" + "lh %[load3], 42(%[input]) \n\t" + "lh %[load4], 22(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_27_64] \n\t" + "msub $ac1, %[load2], %[cospi_5_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + + "madd $ac3, %[load1], %[cospi_5_64] \n\t" + "madd $ac3, %[load2], %[cospi_27_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_11_64] \n\t" + "msub $ac2, %[load4], %[cospi_21_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + + "madd $ac1, %[load3], %[cospi_21_64] \n\t" + "madd $ac1, %[load4], %[cospi_11_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp0], %[temp1] \n\t" + "sub %[load2], %[temp3], %[temp2] \n\t" + + "madd $ac1, %[load2], %[cospi_12_64] \n\t" + "msub $ac1, %[load1], %[cospi_20_64] \n\t" + "madd $ac3, %[load1], %[cospi_12_64] \n\t" + "madd $ac3, %[load2], %[cospi_20_64] \n\t" + + "extp %[step1_21], $ac1, 31 \n\t" + "extp %[step1_26], $ac3, 31 \n\t" + "add %[step1_20], %[temp0], %[temp1] \n\t" + "add %[step1_27], %[temp2], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step1_20] "=r"(step1_20), + [step1_21] "=r"(step1_21), [step1_26] "=r"(step1_26), + [step1_27] "=r"(step1_27) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_27_64] "r"(cospi_27_64), [cospi_5_64] "r"(cospi_5_64), + [cospi_11_64] "r"(cospi_11_64), [cospi_21_64] "r"(cospi_21_64), + [cospi_12_64] "r"(cospi_12_64), [cospi_20_64] "r"(cospi_20_64)); + + __asm__ __volatile__( + "lh %[load1], 26(%[input]) \n\t" + "lh %[load2], 38(%[input]) \n\t" + "lh %[load3], 58(%[input]) \n\t" + "lh %[load4], 6(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_19_64] \n\t" + "msub $ac1, %[load2], %[cospi_13_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + "madd $ac3, %[load1], %[cospi_13_64] \n\t" + "madd $ac3, %[load2], %[cospi_19_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_3_64] \n\t" + "msub $ac2, %[load4], %[cospi_29_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + "madd $ac1, %[load3], %[cospi_29_64] \n\t" + "madd $ac1, %[load4], %[cospi_3_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp1], %[temp0] \n\t" + "sub %[load2], %[temp2], %[temp3] \n\t" + "msub $ac1, %[load1], %[cospi_12_64] \n\t" + "msub $ac1, %[load2], %[cospi_20_64] \n\t" + "msub $ac3, %[load1], %[cospi_20_64] \n\t" + "madd $ac3, %[load2], %[cospi_12_64] \n\t" + "extp %[step1_22], $ac1, 31 \n\t" + "extp %[step1_25], $ac3, 31 \n\t" + "add %[step1_23], %[temp0], %[temp1] \n\t" + "add %[step1_24], %[temp2], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step1_22] "=r"(step1_22), + [step1_23] "=r"(step1_23), [step1_24] "=r"(step1_24), + [step1_25] "=r"(step1_25) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_19_64] "r"(cospi_19_64), [cospi_13_64] "r"(cospi_13_64), + [cospi_3_64] "r"(cospi_3_64), [cospi_29_64] "r"(cospi_29_64), + [cospi_12_64] "r"(cospi_12_64), [cospi_20_64] "r"(cospi_20_64)); + + __asm__ __volatile__( + "lh %[load1], 4(%[input]) \n\t" + "lh %[load2], 60(%[input]) \n\t" + "lh %[load3], 36(%[input]) \n\t" + "lh %[load4], 28(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_30_64] \n\t" + "msub $ac1, %[load2], %[cospi_2_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + "madd $ac3, %[load1], %[cospi_2_64] \n\t" + "madd $ac3, %[load2], %[cospi_30_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_14_64] \n\t" + "msub $ac2, %[load4], %[cospi_18_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + "madd $ac1, %[load3], %[cospi_18_64] \n\t" + "madd $ac1, %[load4], %[cospi_14_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp0], %[temp1] \n\t" + "sub %[load2], %[temp3], %[temp2] \n\t" + "msub $ac1, %[load1], %[cospi_8_64] \n\t" + "madd $ac1, %[load2], %[cospi_24_64] \n\t" + "madd $ac3, %[load1], %[cospi_24_64] \n\t" + "madd $ac3, %[load2], %[cospi_8_64] \n\t" + "extp %[step2_9], $ac1, 31 \n\t" + "extp %[step2_14], $ac3, 31 \n\t" + "add %[step2_8], %[temp0], %[temp1] \n\t" + "add %[step2_15], %[temp2], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step2_8] "=r"(step2_8), + [step2_9] "=r"(step2_9), [step2_14] "=r"(step2_14), + [step2_15] "=r"(step2_15) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_30_64] "r"(cospi_30_64), [cospi_2_64] "r"(cospi_2_64), + [cospi_14_64] "r"(cospi_14_64), [cospi_18_64] "r"(cospi_18_64), + [cospi_8_64] "r"(cospi_8_64), [cospi_24_64] "r"(cospi_24_64)); + + __asm__ __volatile__( + "lh %[load1], 20(%[input]) \n\t" + "lh %[load2], 44(%[input]) \n\t" + "lh %[load3], 52(%[input]) \n\t" + "lh %[load4], 12(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_22_64] \n\t" + "msub $ac1, %[load2], %[cospi_10_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + "madd $ac3, %[load1], %[cospi_10_64] \n\t" + "madd $ac3, %[load2], %[cospi_22_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_6_64] \n\t" + "msub $ac2, %[load4], %[cospi_26_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + "madd $ac1, %[load3], %[cospi_26_64] \n\t" + "madd $ac1, %[load4], %[cospi_6_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp1], %[temp0] \n\t" + "sub %[load2], %[temp2], %[temp3] \n\t" + "msub $ac1, %[load1], %[cospi_24_64] \n\t" + "msub $ac1, %[load2], %[cospi_8_64] \n\t" + "madd $ac3, %[load2], %[cospi_24_64] \n\t" + "msub $ac3, %[load1], %[cospi_8_64] \n\t" + "extp %[step2_10], $ac1, 31 \n\t" + "extp %[step2_13], $ac3, 31 \n\t" + "add %[step2_11], %[temp0], %[temp1] \n\t" + "add %[step2_12], %[temp2], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step2_10] "=r"(step2_10), + [step2_11] "=r"(step2_11), [step2_12] "=r"(step2_12), + [step2_13] "=r"(step2_13) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_22_64] "r"(cospi_22_64), [cospi_10_64] "r"(cospi_10_64), + [cospi_6_64] "r"(cospi_6_64), [cospi_26_64] "r"(cospi_26_64), + [cospi_8_64] "r"(cospi_8_64), [cospi_24_64] "r"(cospi_24_64)); + + __asm__ __volatile__( + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "sub %[temp0], %[step2_14], %[step2_13] \n\t" + "sub %[temp0], %[temp0], %[step2_9] \n\t" + "add %[temp0], %[temp0], %[step2_10] \n\t" + "madd $ac0, %[temp0], %[cospi_16_64] \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "sub %[temp1], %[step2_14], %[step2_13] \n\t" + "add %[temp1], %[temp1], %[step2_9] \n\t" + "sub %[temp1], %[temp1], %[step2_10] \n\t" + "madd $ac1, %[temp1], %[cospi_16_64] \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "sub %[temp0], %[step2_15], %[step2_12] \n\t" + "sub %[temp0], %[temp0], %[step2_8] \n\t" + "add %[temp0], %[temp0], %[step2_11] \n\t" + "madd $ac2, %[temp0], %[cospi_16_64] \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "sub %[temp1], %[step2_15], %[step2_12] \n\t" + "add %[temp1], %[temp1], %[step2_8] \n\t" + "sub %[temp1], %[temp1], %[step2_11] \n\t" + "madd $ac3, %[temp1], %[cospi_16_64] \n\t" + + "add %[step3_8], %[step2_8], %[step2_11] \n\t" + "add %[step3_9], %[step2_9], %[step2_10] \n\t" + "add %[step3_14], %[step2_13], %[step2_14] \n\t" + "add %[step3_15], %[step2_12], %[step2_15] \n\t" + "extp %[step3_10], $ac0, 31 \n\t" + "extp %[step3_13], $ac1, 31 \n\t" + "extp %[step3_11], $ac2, 31 \n\t" + "extp %[step3_12], $ac3, 31 \n\t" + + : [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), [step3_8] "=r"(step3_8), + [step3_9] "=r"(step3_9), [step3_10] "=r"(step3_10), + [step3_11] "=r"(step3_11), [step3_12] "=r"(step3_12), + [step3_13] "=r"(step3_13), [step3_14] "=r"(step3_14), + [step3_15] "=r"(step3_15) + : [const_2_power_13] "r"(const_2_power_13), [step2_8] "r"(step2_8), + [step2_9] "r"(step2_9), [step2_10] "r"(step2_10), + [step2_11] "r"(step2_11), [step2_12] "r"(step2_12), + [step2_13] "r"(step2_13), [step2_14] "r"(step2_14), + [step2_15] "r"(step2_15), [cospi_16_64] "r"(cospi_16_64)); + + step2_18 = step1_17 - step1_18; + step2_29 = step1_30 - step1_29; + + __asm__ __volatile__( + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "msub $ac0, %[step2_18], %[cospi_8_64] \n\t" + "madd $ac0, %[step2_29], %[cospi_24_64] \n\t" + "extp %[step3_18], $ac0, 31 \n\t" + + : [step3_18] "=r"(step3_18) + : [const_2_power_13] "r"(const_2_power_13), [step2_18] "r"(step2_18), + [step2_29] "r"(step2_29), [cospi_24_64] "r"(cospi_24_64), + [cospi_8_64] "r"(cospi_8_64)); + + temp21 = step2_18 * cospi_24_64 + step2_29 * cospi_8_64; + step3_29 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + step2_19 = step1_16 - step1_19; + step2_28 = step1_31 - step1_28; + + __asm__ __volatile__( + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "msub $ac0, %[step2_19], %[cospi_8_64] \n\t" + "madd $ac0, %[step2_28], %[cospi_24_64] \n\t" + "extp %[step3_19], $ac0, 31 \n\t" + + : [step3_19] "=r"(step3_19) + : [const_2_power_13] "r"(const_2_power_13), [step2_19] "r"(step2_19), + [step2_28] "r"(step2_28), [cospi_24_64] "r"(cospi_24_64), + [cospi_8_64] "r"(cospi_8_64)); + + temp21 = step2_19 * cospi_24_64 + step2_28 * cospi_8_64; + step3_28 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + step3_16 = step1_16 + step1_19; + step3_17 = step1_17 + step1_18; + step3_30 = step1_29 + step1_30; + step3_31 = step1_28 + step1_31; + + step2_20 = step1_23 - step1_20; + step2_27 = step1_24 - step1_27; + + __asm__ __volatile__( + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "msub $ac0, %[step2_20], %[cospi_24_64] \n\t" + "msub $ac0, %[step2_27], %[cospi_8_64] \n\t" + "extp %[step3_20], $ac0, 31 \n\t" + + : [step3_20] "=r"(step3_20) + : [const_2_power_13] "r"(const_2_power_13), [step2_20] "r"(step2_20), + [step2_27] "r"(step2_27), [cospi_24_64] "r"(cospi_24_64), + [cospi_8_64] "r"(cospi_8_64)); + + temp21 = -step2_20 * cospi_8_64 + step2_27 * cospi_24_64; + step3_27 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + step2_21 = step1_22 - step1_21; + step2_26 = step1_25 - step1_26; + + __asm__ __volatile__( + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "msub $ac1, %[step2_21], %[cospi_24_64] \n\t" + "msub $ac1, %[step2_26], %[cospi_8_64] \n\t" + "extp %[step3_21], $ac1, 31 \n\t" + + : [step3_21] "=r"(step3_21) + : [const_2_power_13] "r"(const_2_power_13), [step2_21] "r"(step2_21), + [step2_26] "r"(step2_26), [cospi_24_64] "r"(cospi_24_64), + [cospi_8_64] "r"(cospi_8_64)); + + temp21 = -step2_21 * cospi_8_64 + step2_26 * cospi_24_64; + step3_26 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + step3_22 = step1_21 + step1_22; + step3_23 = step1_20 + step1_23; + step3_24 = step1_24 + step1_27; + step3_25 = step1_25 + step1_26; + + step2_16 = step3_16 + step3_23; + step2_17 = step3_17 + step3_22; + step2_18 = step3_18 + step3_21; + step2_19 = step3_19 + step3_20; + step2_20 = step3_19 - step3_20; + step2_21 = step3_18 - step3_21; + step2_22 = step3_17 - step3_22; + step2_23 = step3_16 - step3_23; + + step2_24 = step3_31 - step3_24; + step2_25 = step3_30 - step3_25; + step2_26 = step3_29 - step3_26; + step2_27 = step3_28 - step3_27; + step2_28 = step3_28 + step3_27; + step2_29 = step3_29 + step3_26; + step2_30 = step3_30 + step3_25; + step2_31 = step3_31 + step3_24; + + __asm__ __volatile__( + "lh %[load1], 0(%[input]) \n\t" + "lh %[load2], 32(%[input]) \n\t" + "lh %[load3], 16(%[input]) \n\t" + "lh %[load4], 48(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "add %[result1], %[load1], %[load2] \n\t" + "sub %[result2], %[load1], %[load2] \n\t" + "madd $ac1, %[result1], %[cospi_16_64] \n\t" + "madd $ac2, %[result2], %[cospi_16_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + "extp %[temp1], $ac2, 31 \n\t" + + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "madd $ac3, %[load3], %[cospi_24_64] \n\t" + "msub $ac3, %[load4], %[cospi_8_64] \n\t" + "extp %[temp2], $ac3, 31 \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "madd $ac1, %[load3], %[cospi_8_64] \n\t" + "madd $ac1, %[load4], %[cospi_24_64] \n\t" + "extp %[temp3], $ac1, 31 \n\t" + "add %[step1_0], %[temp0], %[temp3] \n\t" + "add %[step1_1], %[temp1], %[temp2] \n\t" + "sub %[step1_2], %[temp1], %[temp2] \n\t" + "sub %[step1_3], %[temp0], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [result1] "=&r"(result1), + [result2] "=&r"(result2), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step1_0] "=r"(step1_0), + [step1_1] "=r"(step1_1), [step1_2] "=r"(step1_2), + [step1_3] "=r"(step1_3) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64), + [cospi_16_64] "r"(cospi_16_64)); + + __asm__ __volatile__( + "lh %[load1], 8(%[input]) \n\t" + "lh %[load2], 56(%[input]) \n\t" + "lh %[load3], 40(%[input]) \n\t" + "lh %[load4], 24(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_28_64] \n\t" + "msub $ac1, %[load2], %[cospi_4_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + "madd $ac3, %[load1], %[cospi_4_64] \n\t" + "madd $ac3, %[load2], %[cospi_28_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_12_64] \n\t" + "msub $ac2, %[load4], %[cospi_20_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + "madd $ac1, %[load3], %[cospi_20_64] \n\t" + "madd $ac1, %[load4], %[cospi_12_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp3], %[temp2] \n\t" + "sub %[load1], %[load1], %[temp0] \n\t" + "add %[load1], %[load1], %[temp1] \n\t" + "sub %[load2], %[temp0], %[temp1] \n\t" + "sub %[load2], %[load2], %[temp2] \n\t" + "add %[load2], %[load2], %[temp3] \n\t" + "madd $ac1, %[load1], %[cospi_16_64] \n\t" + "madd $ac3, %[load2], %[cospi_16_64] \n\t" + + "extp %[step1_5], $ac1, 31 \n\t" + "extp %[step1_6], $ac3, 31 \n\t" + "add %[step1_4], %[temp0], %[temp1] \n\t" + "add %[step1_7], %[temp3], %[temp2] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step1_4] "=r"(step1_4), + [step1_5] "=r"(step1_5), [step1_6] "=r"(step1_6), + [step1_7] "=r"(step1_7) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_20_64] "r"(cospi_20_64), [cospi_12_64] "r"(cospi_12_64), + [cospi_4_64] "r"(cospi_4_64), [cospi_28_64] "r"(cospi_28_64), + [cospi_16_64] "r"(cospi_16_64)); + + step2_0 = step1_0 + step1_7; + step2_1 = step1_1 + step1_6; + step2_2 = step1_2 + step1_5; + step2_3 = step1_3 + step1_4; + step2_4 = step1_3 - step1_4; + step2_5 = step1_2 - step1_5; + step2_6 = step1_1 - step1_6; + step2_7 = step1_0 - step1_7; + + // stage 7 + step1_0 = step2_0 + step3_15; + step1_1 = step2_1 + step3_14; + step1_2 = step2_2 + step3_13; + step1_3 = step2_3 + step3_12; + step1_4 = step2_4 + step3_11; + step1_5 = step2_5 + step3_10; + step1_6 = step2_6 + step3_9; + step1_7 = step2_7 + step3_8; + step1_8 = step2_7 - step3_8; + step1_9 = step2_6 - step3_9; + step1_10 = step2_5 - step3_10; + step1_11 = step2_4 - step3_11; + step1_12 = step2_3 - step3_12; + step1_13 = step2_2 - step3_13; + step1_14 = step2_1 - step3_14; + step1_15 = step2_0 - step3_15; + + __asm__ __volatile__( + "sub %[temp0], %[step2_27], %[step2_20] \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "madd $ac0, %[temp0], %[cospi_16_64] \n\t" + "extp %[step1_20], $ac0, 31 \n\t" + + : [temp0] "=&r"(temp0), [step1_20] "=r"(step1_20) + : [const_2_power_13] "r"(const_2_power_13), [step2_20] "r"(step2_20), + [step2_27] "r"(step2_27), [cospi_16_64] "r"(cospi_16_64)); + + temp21 = (step2_20 + step2_27) * cospi_16_64; + step1_27 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + __asm__ __volatile__( + "sub %[temp0], %[step2_26], %[step2_21] \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "madd $ac0, %[temp0], %[cospi_16_64] \n\t" + "extp %[step1_21], $ac0, 31 \n\t" + + : [temp0] "=&r"(temp0), [step1_21] "=r"(step1_21) + : [const_2_power_13] "r"(const_2_power_13), [step2_26] "r"(step2_26), + [step2_21] "r"(step2_21), [cospi_16_64] "r"(cospi_16_64)); + + temp21 = (step2_21 + step2_26) * cospi_16_64; + step1_26 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + __asm__ __volatile__( + "sub %[temp0], %[step2_25], %[step2_22] \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "madd $ac0, %[temp0], %[cospi_16_64] \n\t" + "extp %[step1_22], $ac0, 31 \n\t" + + : [temp0] "=&r"(temp0), [step1_22] "=r"(step1_22) + : [const_2_power_13] "r"(const_2_power_13), [step2_25] "r"(step2_25), + [step2_22] "r"(step2_22), [cospi_16_64] "r"(cospi_16_64)); + + temp21 = (step2_22 + step2_25) * cospi_16_64; + step1_25 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + __asm__ __volatile__( + "sub %[temp0], %[step2_24], %[step2_23] \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "madd $ac0, %[temp0], %[cospi_16_64] \n\t" + "extp %[step1_23], $ac0, 31 \n\t" + + : [temp0] "=&r"(temp0), [step1_23] "=r"(step1_23) + : [const_2_power_13] "r"(const_2_power_13), [step2_24] "r"(step2_24), + [step2_23] "r"(step2_23), [cospi_16_64] "r"(cospi_16_64)); + + temp21 = (step2_23 + step2_24) * cospi_16_64; + step1_24 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + __asm__ __volatile__( + "lbu %[temp2], 0(%[dest_pix]) \n\t" + "add %[temp0], %[step1_0], %[step2_31] \n\t" + "addi %[temp0], %[temp0], 32 \n\t" + "sra %[temp0], %[temp0], 6 \n\t" + "add %[temp2], %[temp2], %[temp0] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "add %[temp1], %[step1_1], %[step2_30] \n\t" + "sb %[temp0], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix]) \n\t" + "addi %[temp1], %[temp1], 32 \n\t" + "sra %[temp1], %[temp1], 6 \n\t" + "add %[temp3], %[temp3], %[temp1] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[temp2], 0(%[dest_pix]) \n\t" + "add %[temp0], %[step1_2], %[step2_29] \n\t" + "addi %[temp0], %[temp0], 32 \n\t" + "sra %[temp0], %[temp0], 6 \n\t" + "add %[temp2], %[temp2], %[temp0] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "add %[temp1], %[step1_3], %[step2_28] \n\t" + "sb %[temp0], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix]) \n\t" + "addi %[temp1], %[temp1], 32 \n\t" + "sra %[temp1], %[temp1], 6 \n\t" + "add %[temp3], %[temp3], %[temp1] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + : [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2), + [temp3] "=&r"(temp3), [dest_pix] "+r"(dest_pix) + : [cm] "r"(cm), [dest_stride] "r"(dest_stride), [step1_0] "r"(step1_0), + [step1_1] "r"(step1_1), [step1_2] "r"(step1_2), + [step1_3] "r"(step1_3), [step2_28] "r"(step2_28), + [step2_29] "r"(step2_29), [step2_30] "r"(step2_30), + [step2_31] "r"(step2_31)); + + step3_12 = ROUND_POWER_OF_TWO((step1_3 - step2_28), 6); + step3_13 = ROUND_POWER_OF_TWO((step1_2 - step2_29), 6); + step3_14 = ROUND_POWER_OF_TWO((step1_1 - step2_30), 6); + step3_15 = ROUND_POWER_OF_TWO((step1_0 - step2_31), 6); + + __asm__ __volatile__( + "lbu %[temp2], 0(%[dest_pix1]) \n\t" + "add %[temp2], %[temp2], %[step3_15] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "sb %[temp0], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix1]) \n\t" + "add %[temp3], %[temp3], %[step3_14] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + + "lbu %[temp2], 0(%[dest_pix1]) \n\t" + "add %[temp2], %[temp2], %[step3_13] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "sb %[temp0], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix1]) \n\t" + "add %[temp3], %[temp3], %[step3_12] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + + : [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2), + [temp3] "=&r"(temp3), [dest_pix1] "+r"(dest_pix1) + : [cm] "r"(cm), [dest_stride] "r"(dest_stride), + [step3_12] "r"(step3_12), [step3_13] "r"(step3_13), + [step3_14] "r"(step3_14), [step3_15] "r"(step3_15)); + + __asm__ __volatile__( + "lbu %[temp2], 0(%[dest_pix]) \n\t" + "add %[temp0], %[step1_4], %[step1_27] \n\t" + "addi %[temp0], %[temp0], 32 \n\t" + "sra %[temp0], %[temp0], 6 \n\t" + "add %[temp2], %[temp2], %[temp0] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "add %[temp1], %[step1_5], %[step1_26] \n\t" + "sb %[temp0], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix]) \n\t" + "addi %[temp1], %[temp1], 32 \n\t" + "sra %[temp1], %[temp1], 6 \n\t" + "add %[temp3], %[temp3], %[temp1] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[temp2], 0(%[dest_pix]) \n\t" + "add %[temp0], %[step1_6], %[step1_25] \n\t" + "addi %[temp0], %[temp0], 32 \n\t" + "sra %[temp0], %[temp0], 6 \n\t" + "add %[temp2], %[temp2], %[temp0] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "add %[temp1], %[step1_7], %[step1_24] \n\t" + "sb %[temp0], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix]) \n\t" + "addi %[temp1], %[temp1], 32 \n\t" + "sra %[temp1], %[temp1], 6 \n\t" + "add %[temp3], %[temp3], %[temp1] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + : [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2), + [temp3] "=&r"(temp3), [dest_pix] "+r"(dest_pix) + : [cm] "r"(cm), [dest_stride] "r"(dest_stride), [step1_4] "r"(step1_4), + [step1_5] "r"(step1_5), [step1_6] "r"(step1_6), + [step1_7] "r"(step1_7), [step1_24] "r"(step1_24), + [step1_25] "r"(step1_25), [step1_26] "r"(step1_26), + [step1_27] "r"(step1_27)); + + step3_12 = ROUND_POWER_OF_TWO((step1_7 - step1_24), 6); + step3_13 = ROUND_POWER_OF_TWO((step1_6 - step1_25), 6); + step3_14 = ROUND_POWER_OF_TWO((step1_5 - step1_26), 6); + step3_15 = ROUND_POWER_OF_TWO((step1_4 - step1_27), 6); + + __asm__ __volatile__( + "lbu %[temp2], 0(%[dest_pix1]) \n\t" + "add %[temp2], %[temp2], %[step3_15] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "sb %[temp0], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix1]) \n\t" + "add %[temp3], %[temp3], %[step3_14] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + + "lbu %[temp2], 0(%[dest_pix1]) \n\t" + "add %[temp2], %[temp2], %[step3_13] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "sb %[temp0], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix1]) \n\t" + "add %[temp3], %[temp3], %[step3_12] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + + : [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2), + [temp3] "=&r"(temp3), [dest_pix1] "+r"(dest_pix1) + : [cm] "r"(cm), [dest_stride] "r"(dest_stride), + [step3_12] "r"(step3_12), [step3_13] "r"(step3_13), + [step3_14] "r"(step3_14), [step3_15] "r"(step3_15)); + + __asm__ __volatile__( + "lbu %[temp2], 0(%[dest_pix]) \n\t" + "add %[temp0], %[step1_8], %[step1_23] \n\t" + "addi %[temp0], %[temp0], 32 \n\t" + "sra %[temp0], %[temp0], 6 \n\t" + "add %[temp2], %[temp2], %[temp0] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "add %[temp1], %[step1_9], %[step1_22] \n\t" + "sb %[temp0], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix]) \n\t" + "addi %[temp1], %[temp1], 32 \n\t" + "sra %[temp1], %[temp1], 6 \n\t" + "add %[temp3], %[temp3], %[temp1] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[temp2], 0(%[dest_pix]) \n\t" + "add %[temp0], %[step1_10], %[step1_21] \n\t" + "addi %[temp0], %[temp0], 32 \n\t" + "sra %[temp0], %[temp0], 6 \n\t" + "add %[temp2], %[temp2], %[temp0] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "add %[temp1], %[step1_11], %[step1_20] \n\t" + "sb %[temp0], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix]) \n\t" + "addi %[temp1], %[temp1], 32 \n\t" + "sra %[temp1], %[temp1], 6 \n\t" + "add %[temp3], %[temp3], %[temp1] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + : [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2), + [temp3] "=&r"(temp3), [dest_pix] "+r"(dest_pix) + : [cm] "r"(cm), [dest_stride] "r"(dest_stride), [step1_8] "r"(step1_8), + [step1_9] "r"(step1_9), [step1_10] "r"(step1_10), + [step1_11] "r"(step1_11), [step1_20] "r"(step1_20), + [step1_21] "r"(step1_21), [step1_22] "r"(step1_22), + [step1_23] "r"(step1_23)); + + step3_12 = ROUND_POWER_OF_TWO((step1_11 - step1_20), 6); + step3_13 = ROUND_POWER_OF_TWO((step1_10 - step1_21), 6); + step3_14 = ROUND_POWER_OF_TWO((step1_9 - step1_22), 6); + step3_15 = ROUND_POWER_OF_TWO((step1_8 - step1_23), 6); + + __asm__ __volatile__( + "lbu %[temp2], 0(%[dest_pix1]) \n\t" + "add %[temp2], %[temp2], %[step3_15] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "sb %[temp0], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix1]) \n\t" + "add %[temp3], %[temp3], %[step3_14] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + + "lbu %[temp2], 0(%[dest_pix1]) \n\t" + "add %[temp2], %[temp2], %[step3_13] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "sb %[temp0], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix1]) \n\t" + "add %[temp3], %[temp3], %[step3_12] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + + : [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2), + [temp3] "=&r"(temp3), [dest_pix1] "+r"(dest_pix1) + : [cm] "r"(cm), [dest_stride] "r"(dest_stride), + [step3_12] "r"(step3_12), [step3_13] "r"(step3_13), + [step3_14] "r"(step3_14), [step3_15] "r"(step3_15)); + + __asm__ __volatile__( + "lbu %[temp2], 0(%[dest_pix]) \n\t" + "add %[temp0], %[step1_12], %[step2_19] \n\t" + "addi %[temp0], %[temp0], 32 \n\t" + "sra %[temp0], %[temp0], 6 \n\t" + "add %[temp2], %[temp2], %[temp0] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "add %[temp1], %[step1_13], %[step2_18] \n\t" + "sb %[temp0], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix]) \n\t" + "addi %[temp1], %[temp1], 32 \n\t" + "sra %[temp1], %[temp1], 6 \n\t" + "add %[temp3], %[temp3], %[temp1] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[temp2], 0(%[dest_pix]) \n\t" + "add %[temp0], %[step1_14], %[step2_17] \n\t" + "addi %[temp0], %[temp0], 32 \n\t" + "sra %[temp0], %[temp0], 6 \n\t" + "add %[temp2], %[temp2], %[temp0] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "add %[temp1], %[step1_15], %[step2_16] \n\t" + "sb %[temp0], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix]) \n\t" + "addi %[temp1], %[temp1], 32 \n\t" + "sra %[temp1], %[temp1], 6 \n\t" + "add %[temp3], %[temp3], %[temp1] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix]) \n\t" + + : [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2), + [temp3] "=&r"(temp3), [dest_pix] "+r"(dest_pix) + : [cm] "r"(cm), [dest_stride] "r"(dest_stride), + [step1_12] "r"(step1_12), [step1_13] "r"(step1_13), + [step1_14] "r"(step1_14), [step1_15] "r"(step1_15), + [step2_16] "r"(step2_16), [step2_17] "r"(step2_17), + [step2_18] "r"(step2_18), [step2_19] "r"(step2_19)); + + step3_12 = ROUND_POWER_OF_TWO((step1_15 - step2_16), 6); + step3_13 = ROUND_POWER_OF_TWO((step1_14 - step2_17), 6); + step3_14 = ROUND_POWER_OF_TWO((step1_13 - step2_18), 6); + step3_15 = ROUND_POWER_OF_TWO((step1_12 - step2_19), 6); + + __asm__ __volatile__( + "lbu %[temp2], 0(%[dest_pix1]) \n\t" + "add %[temp2], %[temp2], %[step3_15] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "sb %[temp0], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix1]) \n\t" + "add %[temp3], %[temp3], %[step3_14] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + + "lbu %[temp2], 0(%[dest_pix1]) \n\t" + "add %[temp2], %[temp2], %[step3_13] \n\t" + "lbux %[temp0], %[temp2](%[cm]) \n\t" + "sb %[temp0], 0(%[dest_pix1]) \n\t" + "subu %[dest_pix1], %[dest_pix1], %[dest_stride] \n\t" + "lbu %[temp3], 0(%[dest_pix1]) \n\t" + "add %[temp3], %[temp3], %[step3_12] \n\t" + "lbux %[temp1], %[temp3](%[cm]) \n\t" + "sb %[temp1], 0(%[dest_pix1]) \n\t" + + : [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2), + [temp3] "=&r"(temp3), [dest_pix1] "+r"(dest_pix1) + : [cm] "r"(cm), [dest_stride] "r"(dest_stride), + [step3_12] "r"(step3_12), [step3_13] "r"(step3_13), + [step3_14] "r"(step3_14), [step3_15] "r"(step3_15)); + + input += 32; + } +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/aom_dsp/mips/itrans32_dspr2.c b/third_party/aom/aom_dsp/mips/itrans32_dspr2.c new file mode 100644 index 0000000000..fa77032176 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/itrans32_dspr2.c @@ -0,0 +1,1030 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "aom_dsp/mips/inv_txfm_dspr2.h" +#include "aom_dsp/txfm_common.h" + +#if HAVE_DSPR2 +static void idct32_rows_dspr2(const int16_t *input, int16_t *output, + uint32_t no_rows) { + int16_t step1_0, step1_1, step1_2, step1_3, step1_4, step1_5, step1_6; + int16_t step1_7, step1_8, step1_9, step1_10, step1_11, step1_12, step1_13; + int16_t step1_14, step1_15, step1_16, step1_17, step1_18, step1_19, step1_20; + int16_t step1_21, step1_22, step1_23, step1_24, step1_25, step1_26, step1_27; + int16_t step1_28, step1_29, step1_30, step1_31; + int16_t step2_0, step2_1, step2_2, step2_3, step2_4, step2_5, step2_6; + int16_t step2_7, step2_8, step2_9, step2_10, step2_11, step2_12, step2_13; + int16_t step2_14, step2_15, step2_16, step2_17, step2_18, step2_19, step2_20; + int16_t step2_21, step2_22, step2_23, step2_24, step2_25, step2_26, step2_27; + int16_t step2_28, step2_29, step2_30, step2_31; + int16_t step3_8, step3_9, step3_10, step3_11, step3_12, step3_13, step3_14; + int16_t step3_15, step3_16, step3_17, step3_18, step3_19, step3_20, step3_21; + int16_t step3_22, step3_23, step3_24, step3_25, step3_26, step3_27, step3_28; + int16_t step3_29, step3_30, step3_31; + int temp0, temp1, temp2, temp3; + int load1, load2, load3, load4; + int result1, result2; + int temp21; + int i; + const int const_2_power_13 = 8192; + const int32_t *input_int; + + for (i = no_rows; i--;) { + input_int = (const int32_t *)input; + + if (!(input_int[0] | input_int[1] | input_int[2] | input_int[3] | + input_int[4] | input_int[5] | input_int[6] | input_int[7] | + input_int[8] | input_int[9] | input_int[10] | input_int[11] | + input_int[12] | input_int[13] | input_int[14] | input_int[15])) { + input += 32; + + __asm__ __volatile__( + "sh $zero, 0(%[output]) \n\t" + "sh $zero, 64(%[output]) \n\t" + "sh $zero, 128(%[output]) \n\t" + "sh $zero, 192(%[output]) \n\t" + "sh $zero, 256(%[output]) \n\t" + "sh $zero, 320(%[output]) \n\t" + "sh $zero, 384(%[output]) \n\t" + "sh $zero, 448(%[output]) \n\t" + "sh $zero, 512(%[output]) \n\t" + "sh $zero, 576(%[output]) \n\t" + "sh $zero, 640(%[output]) \n\t" + "sh $zero, 704(%[output]) \n\t" + "sh $zero, 768(%[output]) \n\t" + "sh $zero, 832(%[output]) \n\t" + "sh $zero, 896(%[output]) \n\t" + "sh $zero, 960(%[output]) \n\t" + "sh $zero, 1024(%[output]) \n\t" + "sh $zero, 1088(%[output]) \n\t" + "sh $zero, 1152(%[output]) \n\t" + "sh $zero, 1216(%[output]) \n\t" + "sh $zero, 1280(%[output]) \n\t" + "sh $zero, 1344(%[output]) \n\t" + "sh $zero, 1408(%[output]) \n\t" + "sh $zero, 1472(%[output]) \n\t" + "sh $zero, 1536(%[output]) \n\t" + "sh $zero, 1600(%[output]) \n\t" + "sh $zero, 1664(%[output]) \n\t" + "sh $zero, 1728(%[output]) \n\t" + "sh $zero, 1792(%[output]) \n\t" + "sh $zero, 1856(%[output]) \n\t" + "sh $zero, 1920(%[output]) \n\t" + "sh $zero, 1984(%[output]) \n\t" + + : + : [output] "r"(output)); + + output += 1; + + continue; + } + + /* prefetch row */ + prefetch_load((const uint8_t *)(input + 32)); + prefetch_load((const uint8_t *)(input + 48)); + + __asm__ __volatile__( + "lh %[load1], 2(%[input]) \n\t" + "lh %[load2], 62(%[input]) \n\t" + "lh %[load3], 34(%[input]) \n\t" + "lh %[load4], 30(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_31_64] \n\t" + "msub $ac1, %[load2], %[cospi_1_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + + "madd $ac3, %[load1], %[cospi_1_64] \n\t" + "madd $ac3, %[load2], %[cospi_31_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_15_64] \n\t" + "msub $ac2, %[load4], %[cospi_17_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + + "madd $ac1, %[load3], %[cospi_17_64] \n\t" + "madd $ac1, %[load4], %[cospi_15_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp3], %[temp2] \n\t" + "sub %[load2], %[temp0], %[temp1] \n\t" + + "madd $ac1, %[load1], %[cospi_28_64] \n\t" + "msub $ac1, %[load2], %[cospi_4_64] \n\t" + "madd $ac3, %[load1], %[cospi_4_64] \n\t" + "madd $ac3, %[load2], %[cospi_28_64] \n\t" + + "extp %[step1_17], $ac1, 31 \n\t" + "extp %[step1_30], $ac3, 31 \n\t" + "add %[step1_16], %[temp0], %[temp1] \n\t" + "add %[step1_31], %[temp2], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step1_16] "=r"(step1_16), + [step1_17] "=r"(step1_17), [step1_30] "=r"(step1_30), + [step1_31] "=r"(step1_31) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_31_64] "r"(cospi_31_64), [cospi_1_64] "r"(cospi_1_64), + [cospi_4_64] "r"(cospi_4_64), [cospi_17_64] "r"(cospi_17_64), + [cospi_15_64] "r"(cospi_15_64), [cospi_28_64] "r"(cospi_28_64)); + + __asm__ __volatile__( + "lh %[load1], 18(%[input]) \n\t" + "lh %[load2], 46(%[input]) \n\t" + "lh %[load3], 50(%[input]) \n\t" + "lh %[load4], 14(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_23_64] \n\t" + "msub $ac1, %[load2], %[cospi_9_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + + "madd $ac3, %[load1], %[cospi_9_64] \n\t" + "madd $ac3, %[load2], %[cospi_23_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_7_64] \n\t" + "msub $ac2, %[load4], %[cospi_25_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + + "madd $ac1, %[load3], %[cospi_25_64] \n\t" + "madd $ac1, %[load4], %[cospi_7_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp1], %[temp0] \n\t" + "sub %[load2], %[temp2], %[temp3] \n\t" + + "msub $ac1, %[load1], %[cospi_28_64] \n\t" + "msub $ac1, %[load2], %[cospi_4_64] \n\t" + "msub $ac3, %[load1], %[cospi_4_64] \n\t" + "madd $ac3, %[load2], %[cospi_28_64] \n\t" + + "extp %[step1_18], $ac1, 31 \n\t" + "extp %[step1_29], $ac3, 31 \n\t" + "add %[step1_19], %[temp0], %[temp1] \n\t" + "add %[step1_28], %[temp2], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step1_18] "=r"(step1_18), + [step1_19] "=r"(step1_19), [step1_28] "=r"(step1_28), + [step1_29] "=r"(step1_29) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_23_64] "r"(cospi_23_64), [cospi_9_64] "r"(cospi_9_64), + [cospi_4_64] "r"(cospi_4_64), [cospi_7_64] "r"(cospi_7_64), + [cospi_25_64] "r"(cospi_25_64), [cospi_28_64] "r"(cospi_28_64)); + + __asm__ __volatile__( + "lh %[load1], 10(%[input]) \n\t" + "lh %[load2], 54(%[input]) \n\t" + "lh %[load3], 42(%[input]) \n\t" + "lh %[load4], 22(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_27_64] \n\t" + "msub $ac1, %[load2], %[cospi_5_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + + "madd $ac3, %[load1], %[cospi_5_64] \n\t" + "madd $ac3, %[load2], %[cospi_27_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_11_64] \n\t" + "msub $ac2, %[load4], %[cospi_21_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + + "madd $ac1, %[load3], %[cospi_21_64] \n\t" + "madd $ac1, %[load4], %[cospi_11_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp0], %[temp1] \n\t" + "sub %[load2], %[temp3], %[temp2] \n\t" + + "madd $ac1, %[load2], %[cospi_12_64] \n\t" + "msub $ac1, %[load1], %[cospi_20_64] \n\t" + "madd $ac3, %[load1], %[cospi_12_64] \n\t" + "madd $ac3, %[load2], %[cospi_20_64] \n\t" + + "extp %[step1_21], $ac1, 31 \n\t" + "extp %[step1_26], $ac3, 31 \n\t" + "add %[step1_20], %[temp0], %[temp1] \n\t" + "add %[step1_27], %[temp2], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step1_20] "=r"(step1_20), + [step1_21] "=r"(step1_21), [step1_26] "=r"(step1_26), + [step1_27] "=r"(step1_27) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_27_64] "r"(cospi_27_64), [cospi_5_64] "r"(cospi_5_64), + [cospi_11_64] "r"(cospi_11_64), [cospi_21_64] "r"(cospi_21_64), + [cospi_12_64] "r"(cospi_12_64), [cospi_20_64] "r"(cospi_20_64)); + + __asm__ __volatile__( + "lh %[load1], 26(%[input]) \n\t" + "lh %[load2], 38(%[input]) \n\t" + "lh %[load3], 58(%[input]) \n\t" + "lh %[load4], 6(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_19_64] \n\t" + "msub $ac1, %[load2], %[cospi_13_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + + "madd $ac3, %[load1], %[cospi_13_64] \n\t" + "madd $ac3, %[load2], %[cospi_19_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_3_64] \n\t" + "msub $ac2, %[load4], %[cospi_29_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + + "madd $ac1, %[load3], %[cospi_29_64] \n\t" + "madd $ac1, %[load4], %[cospi_3_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp1], %[temp0] \n\t" + "sub %[load2], %[temp2], %[temp3] \n\t" + + "msub $ac1, %[load1], %[cospi_12_64] \n\t" + "msub $ac1, %[load2], %[cospi_20_64] \n\t" + "msub $ac3, %[load1], %[cospi_20_64] \n\t" + "madd $ac3, %[load2], %[cospi_12_64] \n\t" + + "extp %[step1_22], $ac1, 31 \n\t" + "extp %[step1_25], $ac3, 31 \n\t" + "add %[step1_23], %[temp0], %[temp1] \n\t" + "add %[step1_24], %[temp2], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step1_22] "=r"(step1_22), + [step1_23] "=r"(step1_23), [step1_24] "=r"(step1_24), + [step1_25] "=r"(step1_25) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_19_64] "r"(cospi_19_64), [cospi_13_64] "r"(cospi_13_64), + [cospi_3_64] "r"(cospi_3_64), [cospi_29_64] "r"(cospi_29_64), + [cospi_12_64] "r"(cospi_12_64), [cospi_20_64] "r"(cospi_20_64)); + + __asm__ __volatile__( + "lh %[load1], 4(%[input]) \n\t" + "lh %[load2], 60(%[input]) \n\t" + "lh %[load3], 36(%[input]) \n\t" + "lh %[load4], 28(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_30_64] \n\t" + "msub $ac1, %[load2], %[cospi_2_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + + "madd $ac3, %[load1], %[cospi_2_64] \n\t" + "madd $ac3, %[load2], %[cospi_30_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_14_64] \n\t" + "msub $ac2, %[load4], %[cospi_18_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + + "madd $ac1, %[load3], %[cospi_18_64] \n\t" + "madd $ac1, %[load4], %[cospi_14_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp0], %[temp1] \n\t" + "sub %[load2], %[temp3], %[temp2] \n\t" + + "msub $ac1, %[load1], %[cospi_8_64] \n\t" + "madd $ac1, %[load2], %[cospi_24_64] \n\t" + "madd $ac3, %[load1], %[cospi_24_64] \n\t" + "madd $ac3, %[load2], %[cospi_8_64] \n\t" + + "extp %[step2_9], $ac1, 31 \n\t" + "extp %[step2_14], $ac3, 31 \n\t" + "add %[step2_8], %[temp0], %[temp1] \n\t" + "add %[step2_15], %[temp2], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step2_8] "=r"(step2_8), + [step2_9] "=r"(step2_9), [step2_14] "=r"(step2_14), + [step2_15] "=r"(step2_15) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_30_64] "r"(cospi_30_64), [cospi_2_64] "r"(cospi_2_64), + [cospi_14_64] "r"(cospi_14_64), [cospi_18_64] "r"(cospi_18_64), + [cospi_8_64] "r"(cospi_8_64), [cospi_24_64] "r"(cospi_24_64)); + + __asm__ __volatile__( + "lh %[load1], 20(%[input]) \n\t" + "lh %[load2], 44(%[input]) \n\t" + "lh %[load3], 52(%[input]) \n\t" + "lh %[load4], 12(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_22_64] \n\t" + "msub $ac1, %[load2], %[cospi_10_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + + "madd $ac3, %[load1], %[cospi_10_64] \n\t" + "madd $ac3, %[load2], %[cospi_22_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_6_64] \n\t" + "msub $ac2, %[load4], %[cospi_26_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + + "madd $ac1, %[load3], %[cospi_26_64] \n\t" + "madd $ac1, %[load4], %[cospi_6_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp1], %[temp0] \n\t" + "sub %[load2], %[temp2], %[temp3] \n\t" + + "msub $ac1, %[load1], %[cospi_24_64] \n\t" + "msub $ac1, %[load2], %[cospi_8_64] \n\t" + "madd $ac3, %[load2], %[cospi_24_64] \n\t" + "msub $ac3, %[load1], %[cospi_8_64] \n\t" + + "extp %[step2_10], $ac1, 31 \n\t" + "extp %[step2_13], $ac3, 31 \n\t" + "add %[step2_11], %[temp0], %[temp1] \n\t" + "add %[step2_12], %[temp2], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step2_10] "=r"(step2_10), + [step2_11] "=r"(step2_11), [step2_12] "=r"(step2_12), + [step2_13] "=r"(step2_13) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_22_64] "r"(cospi_22_64), [cospi_10_64] "r"(cospi_10_64), + [cospi_6_64] "r"(cospi_6_64), [cospi_26_64] "r"(cospi_26_64), + [cospi_8_64] "r"(cospi_8_64), [cospi_24_64] "r"(cospi_24_64)); + + __asm__ __volatile__( + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "sub %[temp0], %[step2_14], %[step2_13] \n\t" + "sub %[temp0], %[temp0], %[step2_9] \n\t" + "add %[temp0], %[temp0], %[step2_10] \n\t" + "madd $ac0, %[temp0], %[cospi_16_64] \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "sub %[temp1], %[step2_14], %[step2_13] \n\t" + "add %[temp1], %[temp1], %[step2_9] \n\t" + "sub %[temp1], %[temp1], %[step2_10] \n\t" + "madd $ac1, %[temp1], %[cospi_16_64] \n\t" + + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "sub %[temp0], %[step2_15], %[step2_12] \n\t" + "sub %[temp0], %[temp0], %[step2_8] \n\t" + "add %[temp0], %[temp0], %[step2_11] \n\t" + "madd $ac2, %[temp0], %[cospi_16_64] \n\t" + + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "sub %[temp1], %[step2_15], %[step2_12] \n\t" + "add %[temp1], %[temp1], %[step2_8] \n\t" + "sub %[temp1], %[temp1], %[step2_11] \n\t" + "madd $ac3, %[temp1], %[cospi_16_64] \n\t" + + "add %[step3_8], %[step2_8], %[step2_11] \n\t" + "add %[step3_9], %[step2_9], %[step2_10] \n\t" + "add %[step3_14], %[step2_13], %[step2_14] \n\t" + "add %[step3_15], %[step2_12], %[step2_15] \n\t" + + "extp %[step3_10], $ac0, 31 \n\t" + "extp %[step3_13], $ac1, 31 \n\t" + "extp %[step3_11], $ac2, 31 \n\t" + "extp %[step3_12], $ac3, 31 \n\t" + + : [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), [step3_8] "=r"(step3_8), + [step3_9] "=r"(step3_9), [step3_10] "=r"(step3_10), + [step3_11] "=r"(step3_11), [step3_12] "=r"(step3_12), + [step3_13] "=r"(step3_13), [step3_14] "=r"(step3_14), + [step3_15] "=r"(step3_15) + : [const_2_power_13] "r"(const_2_power_13), [step2_8] "r"(step2_8), + [step2_9] "r"(step2_9), [step2_10] "r"(step2_10), + [step2_11] "r"(step2_11), [step2_12] "r"(step2_12), + [step2_13] "r"(step2_13), [step2_14] "r"(step2_14), + [step2_15] "r"(step2_15), [cospi_16_64] "r"(cospi_16_64)); + + step2_18 = step1_17 - step1_18; + step2_29 = step1_30 - step1_29; + + __asm__ __volatile__( + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "msub $ac0, %[step2_18], %[cospi_8_64] \n\t" + "madd $ac0, %[step2_29], %[cospi_24_64] \n\t" + "extp %[step3_18], $ac0, 31 \n\t" + + : [step3_18] "=r"(step3_18) + : [const_2_power_13] "r"(const_2_power_13), [step2_18] "r"(step2_18), + [step2_29] "r"(step2_29), [cospi_24_64] "r"(cospi_24_64), + [cospi_8_64] "r"(cospi_8_64)); + + temp21 = step2_18 * cospi_24_64 + step2_29 * cospi_8_64; + step3_29 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + step2_19 = step1_16 - step1_19; + step2_28 = step1_31 - step1_28; + + __asm__ __volatile__( + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "msub $ac0, %[step2_19], %[cospi_8_64] \n\t" + "madd $ac0, %[step2_28], %[cospi_24_64] \n\t" + "extp %[step3_19], $ac0, 31 \n\t" + + : [step3_19] "=r"(step3_19) + : [const_2_power_13] "r"(const_2_power_13), [step2_19] "r"(step2_19), + [step2_28] "r"(step2_28), [cospi_24_64] "r"(cospi_24_64), + [cospi_8_64] "r"(cospi_8_64)); + + temp21 = step2_19 * cospi_24_64 + step2_28 * cospi_8_64; + step3_28 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + step3_16 = step1_16 + step1_19; + step3_17 = step1_17 + step1_18; + step3_30 = step1_29 + step1_30; + step3_31 = step1_28 + step1_31; + + step2_20 = step1_23 - step1_20; + step2_27 = step1_24 - step1_27; + + __asm__ __volatile__( + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "msub $ac0, %[step2_20], %[cospi_24_64] \n\t" + "msub $ac0, %[step2_27], %[cospi_8_64] \n\t" + "extp %[step3_20], $ac0, 31 \n\t" + + : [step3_20] "=r"(step3_20) + : [const_2_power_13] "r"(const_2_power_13), [step2_20] "r"(step2_20), + [step2_27] "r"(step2_27), [cospi_24_64] "r"(cospi_24_64), + [cospi_8_64] "r"(cospi_8_64)); + + temp21 = -step2_20 * cospi_8_64 + step2_27 * cospi_24_64; + step3_27 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + step2_21 = step1_22 - step1_21; + step2_26 = step1_25 - step1_26; + + __asm__ __volatile__( + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "msub $ac1, %[step2_21], %[cospi_24_64] \n\t" + "msub $ac1, %[step2_26], %[cospi_8_64] \n\t" + "extp %[step3_21], $ac1, 31 \n\t" + + : [step3_21] "=r"(step3_21) + : [const_2_power_13] "r"(const_2_power_13), [step2_21] "r"(step2_21), + [step2_26] "r"(step2_26), [cospi_24_64] "r"(cospi_24_64), + [cospi_8_64] "r"(cospi_8_64)); + + temp21 = -step2_21 * cospi_8_64 + step2_26 * cospi_24_64; + step3_26 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + step3_22 = step1_21 + step1_22; + step3_23 = step1_20 + step1_23; + step3_24 = step1_24 + step1_27; + step3_25 = step1_25 + step1_26; + + step2_16 = step3_16 + step3_23; + step2_17 = step3_17 + step3_22; + step2_18 = step3_18 + step3_21; + step2_19 = step3_19 + step3_20; + step2_20 = step3_19 - step3_20; + step2_21 = step3_18 - step3_21; + step2_22 = step3_17 - step3_22; + step2_23 = step3_16 - step3_23; + + step2_24 = step3_31 - step3_24; + step2_25 = step3_30 - step3_25; + step2_26 = step3_29 - step3_26; + step2_27 = step3_28 - step3_27; + step2_28 = step3_28 + step3_27; + step2_29 = step3_29 + step3_26; + step2_30 = step3_30 + step3_25; + step2_31 = step3_31 + step3_24; + + __asm__ __volatile__( + "lh %[load1], 0(%[input]) \n\t" + "lh %[load2], 32(%[input]) \n\t" + "lh %[load3], 16(%[input]) \n\t" + "lh %[load4], 48(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + "add %[result1], %[load1], %[load2] \n\t" + "sub %[result2], %[load1], %[load2] \n\t" + "madd $ac1, %[result1], %[cospi_16_64] \n\t" + "madd $ac2, %[result2], %[cospi_16_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + "extp %[temp1], $ac2, 31 \n\t" + + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + "madd $ac3, %[load3], %[cospi_24_64] \n\t" + "msub $ac3, %[load4], %[cospi_8_64] \n\t" + "extp %[temp2], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "madd $ac1, %[load3], %[cospi_8_64] \n\t" + "madd $ac1, %[load4], %[cospi_24_64] \n\t" + "extp %[temp3], $ac1, 31 \n\t" + + "add %[step1_0], %[temp0], %[temp3] \n\t" + "add %[step1_1], %[temp1], %[temp2] \n\t" + "sub %[step1_2], %[temp1], %[temp2] \n\t" + "sub %[step1_3], %[temp0], %[temp3] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [result1] "=&r"(result1), + [result2] "=&r"(result2), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step1_0] "=r"(step1_0), + [step1_1] "=r"(step1_1), [step1_2] "=r"(step1_2), + [step1_3] "=r"(step1_3) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_16_64] "r"(cospi_16_64), [cospi_24_64] "r"(cospi_24_64), + [cospi_8_64] "r"(cospi_8_64) + + ); + + __asm__ __volatile__( + "lh %[load1], 8(%[input]) \n\t" + "lh %[load2], 56(%[input]) \n\t" + "lh %[load3], 40(%[input]) \n\t" + "lh %[load4], 24(%[input]) \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "madd $ac1, %[load1], %[cospi_28_64] \n\t" + "msub $ac1, %[load2], %[cospi_4_64] \n\t" + "extp %[temp0], $ac1, 31 \n\t" + + "madd $ac3, %[load1], %[cospi_4_64] \n\t" + "madd $ac3, %[load2], %[cospi_28_64] \n\t" + "extp %[temp3], $ac3, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac2 \n\t" + "mthi $zero, $ac2 \n\t" + + "madd $ac2, %[load3], %[cospi_12_64] \n\t" + "msub $ac2, %[load4], %[cospi_20_64] \n\t" + "extp %[temp1], $ac2, 31 \n\t" + + "madd $ac1, %[load3], %[cospi_20_64] \n\t" + "madd $ac1, %[load4], %[cospi_12_64] \n\t" + "extp %[temp2], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "mtlo %[const_2_power_13], $ac3 \n\t" + "mthi $zero, $ac3 \n\t" + + "sub %[load1], %[temp3], %[temp2] \n\t" + "sub %[load1], %[load1], %[temp0] \n\t" + "add %[load1], %[load1], %[temp1] \n\t" + + "sub %[load2], %[temp0], %[temp1] \n\t" + "sub %[load2], %[load2], %[temp2] \n\t" + "add %[load2], %[load2], %[temp3] \n\t" + + "madd $ac1, %[load1], %[cospi_16_64] \n\t" + "madd $ac3, %[load2], %[cospi_16_64] \n\t" + + "extp %[step1_5], $ac1, 31 \n\t" + "extp %[step1_6], $ac3, 31 \n\t" + "add %[step1_4], %[temp0], %[temp1] \n\t" + "add %[step1_7], %[temp3], %[temp2] \n\t" + + : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3), + [load4] "=&r"(load4), [temp0] "=&r"(temp0), [temp1] "=&r"(temp1), + [temp2] "=&r"(temp2), [temp3] "=&r"(temp3), [step1_4] "=r"(step1_4), + [step1_5] "=r"(step1_5), [step1_6] "=r"(step1_6), + [step1_7] "=r"(step1_7) + : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input), + [cospi_20_64] "r"(cospi_20_64), [cospi_12_64] "r"(cospi_12_64), + [cospi_4_64] "r"(cospi_4_64), [cospi_28_64] "r"(cospi_28_64), + [cospi_16_64] "r"(cospi_16_64)); + + step2_0 = step1_0 + step1_7; + step2_1 = step1_1 + step1_6; + step2_2 = step1_2 + step1_5; + step2_3 = step1_3 + step1_4; + step2_4 = step1_3 - step1_4; + step2_5 = step1_2 - step1_5; + step2_6 = step1_1 - step1_6; + step2_7 = step1_0 - step1_7; + + step1_0 = step2_0 + step3_15; + step1_1 = step2_1 + step3_14; + step1_2 = step2_2 + step3_13; + step1_3 = step2_3 + step3_12; + step1_4 = step2_4 + step3_11; + step1_5 = step2_5 + step3_10; + step1_6 = step2_6 + step3_9; + step1_7 = step2_7 + step3_8; + step1_8 = step2_7 - step3_8; + step1_9 = step2_6 - step3_9; + step1_10 = step2_5 - step3_10; + step1_11 = step2_4 - step3_11; + step1_12 = step2_3 - step3_12; + step1_13 = step2_2 - step3_13; + step1_14 = step2_1 - step3_14; + step1_15 = step2_0 - step3_15; + + __asm__ __volatile__( + "sub %[temp0], %[step2_27], %[step2_20] \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "madd $ac0, %[temp0], %[cospi_16_64] \n\t" + "extp %[step1_20], $ac0, 31 \n\t" + + : [temp0] "=&r"(temp0), [step1_20] "=r"(step1_20) + : [const_2_power_13] "r"(const_2_power_13), [step2_20] "r"(step2_20), + [step2_27] "r"(step2_27), [cospi_16_64] "r"(cospi_16_64)); + + temp21 = (step2_20 + step2_27) * cospi_16_64; + step1_27 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + __asm__ __volatile__( + "sub %[temp0], %[step2_26], %[step2_21] \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "madd $ac0, %[temp0], %[cospi_16_64] \n\t" + "extp %[step1_21], $ac0, 31 \n\t" + + : [temp0] "=&r"(temp0), [step1_21] "=r"(step1_21) + : [const_2_power_13] "r"(const_2_power_13), [step2_26] "r"(step2_26), + [step2_21] "r"(step2_21), [cospi_16_64] "r"(cospi_16_64)); + + temp21 = (step2_21 + step2_26) * cospi_16_64; + step1_26 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + __asm__ __volatile__( + "sub %[temp0], %[step2_25], %[step2_22] \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "madd $ac0, %[temp0], %[cospi_16_64] \n\t" + "extp %[step1_22], $ac0, 31 \n\t" + + : [temp0] "=&r"(temp0), [step1_22] "=r"(step1_22) + : [const_2_power_13] "r"(const_2_power_13), [step2_25] "r"(step2_25), + [step2_22] "r"(step2_22), [cospi_16_64] "r"(cospi_16_64)); + + temp21 = (step2_22 + step2_25) * cospi_16_64; + step1_25 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + __asm__ __volatile__( + "sub %[temp0], %[step2_24], %[step2_23] \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "madd $ac0, %[temp0], %[cospi_16_64] \n\t" + "extp %[step1_23], $ac0, 31 \n\t" + + : [temp0] "=&r"(temp0), [step1_23] "=r"(step1_23) + : [const_2_power_13] "r"(const_2_power_13), [step2_24] "r"(step2_24), + [step2_23] "r"(step2_23), [cospi_16_64] "r"(cospi_16_64)); + + temp21 = (step2_23 + step2_24) * cospi_16_64; + step1_24 = (temp21 + DCT_CONST_ROUNDING) >> DCT_CONST_BITS; + + // final stage + output[0 * 32] = step1_0 + step2_31; + output[1 * 32] = step1_1 + step2_30; + output[2 * 32] = step1_2 + step2_29; + output[3 * 32] = step1_3 + step2_28; + output[4 * 32] = step1_4 + step1_27; + output[5 * 32] = step1_5 + step1_26; + output[6 * 32] = step1_6 + step1_25; + output[7 * 32] = step1_7 + step1_24; + output[8 * 32] = step1_8 + step1_23; + output[9 * 32] = step1_9 + step1_22; + output[10 * 32] = step1_10 + step1_21; + output[11 * 32] = step1_11 + step1_20; + output[12 * 32] = step1_12 + step2_19; + output[13 * 32] = step1_13 + step2_18; + output[14 * 32] = step1_14 + step2_17; + output[15 * 32] = step1_15 + step2_16; + output[16 * 32] = step1_15 - step2_16; + output[17 * 32] = step1_14 - step2_17; + output[18 * 32] = step1_13 - step2_18; + output[19 * 32] = step1_12 - step2_19; + output[20 * 32] = step1_11 - step1_20; + output[21 * 32] = step1_10 - step1_21; + output[22 * 32] = step1_9 - step1_22; + output[23 * 32] = step1_8 - step1_23; + output[24 * 32] = step1_7 - step1_24; + output[25 * 32] = step1_6 - step1_25; + output[26 * 32] = step1_5 - step1_26; + output[27 * 32] = step1_4 - step1_27; + output[28 * 32] = step1_3 - step2_28; + output[29 * 32] = step1_2 - step2_29; + output[30 * 32] = step1_1 - step2_30; + output[31 * 32] = step1_0 - step2_31; + + input += 32; + output += 1; + } +} + +void aom_idct32x32_1024_add_dspr2(const int16_t *input, uint8_t *dest, + int dest_stride) { + DECLARE_ALIGNED(32, int16_t, out[32 * 32]); + int16_t *outptr = out; + uint32_t pos = 45; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + // Rows + idct32_rows_dspr2(input, outptr, 32); + + // Columns + aom_idct32_cols_add_blk_dspr2(out, dest, dest_stride); +} + +void aom_idct32x32_34_add_dspr2(const int16_t *input, uint8_t *dest, + int stride) { + DECLARE_ALIGNED(32, int16_t, out[32 * 32]); + int16_t *outptr = out; + uint32_t i; + uint32_t pos = 45; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + // Rows + idct32_rows_dspr2(input, outptr, 8); + + outptr += 8; + __asm__ __volatile__( + "sw $zero, 0(%[outptr]) \n\t" + "sw $zero, 4(%[outptr]) \n\t" + "sw $zero, 8(%[outptr]) \n\t" + "sw $zero, 12(%[outptr]) \n\t" + "sw $zero, 16(%[outptr]) \n\t" + "sw $zero, 20(%[outptr]) \n\t" + "sw $zero, 24(%[outptr]) \n\t" + "sw $zero, 28(%[outptr]) \n\t" + "sw $zero, 32(%[outptr]) \n\t" + "sw $zero, 36(%[outptr]) \n\t" + "sw $zero, 40(%[outptr]) \n\t" + "sw $zero, 44(%[outptr]) \n\t" + + : + : [outptr] "r"(outptr)); + + for (i = 0; i < 31; ++i) { + outptr += 32; + + __asm__ __volatile__( + "sw $zero, 0(%[outptr]) \n\t" + "sw $zero, 4(%[outptr]) \n\t" + "sw $zero, 8(%[outptr]) \n\t" + "sw $zero, 12(%[outptr]) \n\t" + "sw $zero, 16(%[outptr]) \n\t" + "sw $zero, 20(%[outptr]) \n\t" + "sw $zero, 24(%[outptr]) \n\t" + "sw $zero, 28(%[outptr]) \n\t" + "sw $zero, 32(%[outptr]) \n\t" + "sw $zero, 36(%[outptr]) \n\t" + "sw $zero, 40(%[outptr]) \n\t" + "sw $zero, 44(%[outptr]) \n\t" + + : + : [outptr] "r"(outptr)); + } + + // Columns + aom_idct32_cols_add_blk_dspr2(out, dest, stride); +} + +void aom_idct32x32_1_add_dspr2(const int16_t *input, uint8_t *dest, + int stride) { + int r, out; + int32_t a1, absa1; + int32_t vector_a1; + int32_t t1, t2, t3, t4; + int32_t vector_1, vector_2, vector_3, vector_4; + uint32_t pos = 45; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + + : + : [pos] "r"(pos)); + + out = DCT_CONST_ROUND_SHIFT_TWICE_COSPI_16_64(input[0]); + __asm__ __volatile__( + "addi %[out], %[out], 32 \n\t" + "sra %[a1], %[out], 6 \n\t" + + : [out] "+r"(out), [a1] "=r"(a1) + :); + + if (a1 < 0) { + /* use quad-byte + * input and output memory are four byte aligned */ + __asm__ __volatile__( + "abs %[absa1], %[a1] \n\t" + "replv.qb %[vector_a1], %[absa1] \n\t" + + : [absa1] "=r"(absa1), [vector_a1] "=r"(vector_a1) + : [a1] "r"(a1)); + + for (r = 32; r--;) { + __asm__ __volatile__( + "lw %[t1], 0(%[dest]) \n\t" + "lw %[t2], 4(%[dest]) \n\t" + "lw %[t3], 8(%[dest]) \n\t" + "lw %[t4], 12(%[dest]) \n\t" + "subu_s.qb %[vector_1], %[t1], %[vector_a1] \n\t" + "subu_s.qb %[vector_2], %[t2], %[vector_a1] \n\t" + "subu_s.qb %[vector_3], %[t3], %[vector_a1] \n\t" + "subu_s.qb %[vector_4], %[t4], %[vector_a1] \n\t" + "sw %[vector_1], 0(%[dest]) \n\t" + "sw %[vector_2], 4(%[dest]) \n\t" + "sw %[vector_3], 8(%[dest]) \n\t" + "sw %[vector_4], 12(%[dest]) \n\t" + + "lw %[t1], 16(%[dest]) \n\t" + "lw %[t2], 20(%[dest]) \n\t" + "lw %[t3], 24(%[dest]) \n\t" + "lw %[t4], 28(%[dest]) \n\t" + "subu_s.qb %[vector_1], %[t1], %[vector_a1] \n\t" + "subu_s.qb %[vector_2], %[t2], %[vector_a1] \n\t" + "subu_s.qb %[vector_3], %[t3], %[vector_a1] \n\t" + "subu_s.qb %[vector_4], %[t4], %[vector_a1] \n\t" + "sw %[vector_1], 16(%[dest]) \n\t" + "sw %[vector_2], 20(%[dest]) \n\t" + "sw %[vector_3], 24(%[dest]) \n\t" + "sw %[vector_4], 28(%[dest]) \n\t" + + "add %[dest], %[dest], %[stride] \n\t" + + : [t1] "=&r"(t1), [t2] "=&r"(t2), [t3] "=&r"(t3), [t4] "=&r"(t4), + [vector_1] "=&r"(vector_1), [vector_2] "=&r"(vector_2), + [vector_3] "=&r"(vector_3), [vector_4] "=&r"(vector_4), + [dest] "+&r"(dest) + : [stride] "r"(stride), [vector_a1] "r"(vector_a1)); + } + } else { + /* use quad-byte + * input and output memory are four byte aligned */ + __asm__ __volatile__("replv.qb %[vector_a1], %[a1] \n\t" + + : [vector_a1] "=r"(vector_a1) + : [a1] "r"(a1)); + + for (r = 32; r--;) { + __asm__ __volatile__( + "lw %[t1], 0(%[dest]) \n\t" + "lw %[t2], 4(%[dest]) \n\t" + "lw %[t3], 8(%[dest]) \n\t" + "lw %[t4], 12(%[dest]) \n\t" + "addu_s.qb %[vector_1], %[t1], %[vector_a1] \n\t" + "addu_s.qb %[vector_2], %[t2], %[vector_a1] \n\t" + "addu_s.qb %[vector_3], %[t3], %[vector_a1] \n\t" + "addu_s.qb %[vector_4], %[t4], %[vector_a1] \n\t" + "sw %[vector_1], 0(%[dest]) \n\t" + "sw %[vector_2], 4(%[dest]) \n\t" + "sw %[vector_3], 8(%[dest]) \n\t" + "sw %[vector_4], 12(%[dest]) \n\t" + + "lw %[t1], 16(%[dest]) \n\t" + "lw %[t2], 20(%[dest]) \n\t" + "lw %[t3], 24(%[dest]) \n\t" + "lw %[t4], 28(%[dest]) \n\t" + "addu_s.qb %[vector_1], %[t1], %[vector_a1] \n\t" + "addu_s.qb %[vector_2], %[t2], %[vector_a1] \n\t" + "addu_s.qb %[vector_3], %[t3], %[vector_a1] \n\t" + "addu_s.qb %[vector_4], %[t4], %[vector_a1] \n\t" + "sw %[vector_1], 16(%[dest]) \n\t" + "sw %[vector_2], 20(%[dest]) \n\t" + "sw %[vector_3], 24(%[dest]) \n\t" + "sw %[vector_4], 28(%[dest]) \n\t" + + "add %[dest], %[dest], %[stride] \n\t" + + : [t1] "=&r"(t1), [t2] "=&r"(t2), [t3] "=&r"(t3), [t4] "=&r"(t4), + [vector_1] "=&r"(vector_1), [vector_2] "=&r"(vector_2), + [vector_3] "=&r"(vector_3), [vector_4] "=&r"(vector_4), + [dest] "+&r"(dest) + : [stride] "r"(stride), [vector_a1] "r"(vector_a1)); + } + } +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/aom_dsp/mips/itrans4_dspr2.c b/third_party/aom/aom_dsp/mips/itrans4_dspr2.c new file mode 100644 index 0000000000..e6d0367cd2 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/itrans4_dspr2.c @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/inv_txfm_dspr2.h" +#include "aom_dsp/txfm_common.h" + +#if HAVE_DSPR2 +void aom_idct4_rows_dspr2(const int16_t *input, int16_t *output) { + int16_t step_0, step_1, step_2, step_3; + int Temp0, Temp1, Temp2, Temp3; + const int const_2_power_13 = 8192; + int i; + + for (i = 4; i--;) { + __asm__ __volatile__( + /* + temp_1 = (input[0] + input[2]) * cospi_16_64; + step_0 = dct_const_round_shift(temp_1); + + temp_2 = (input[0] - input[2]) * cospi_16_64; + step_1 = dct_const_round_shift(temp_2); + */ + "lh %[Temp0], 0(%[input]) \n\t" + "lh %[Temp1], 4(%[input]) \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "add %[Temp2], %[Temp0], %[Temp1] \n\t" + "sub %[Temp3], %[Temp0], %[Temp1] \n\t" + "madd $ac0, %[Temp2], %[cospi_16_64] \n\t" + "lh %[Temp0], 2(%[input]) \n\t" + "lh %[Temp1], 6(%[input]) \n\t" + "extp %[step_0], $ac0, 31 \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + + "madd $ac1, %[Temp3], %[cospi_16_64] \n\t" + "extp %[step_1], $ac1, 31 \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + + /* + temp1 = input[1] * cospi_24_64 - input[3] * cospi_8_64; + step_2 = dct_const_round_shift(temp1); + */ + "madd $ac0, %[Temp0], %[cospi_24_64] \n\t" + "msub $ac0, %[Temp1], %[cospi_8_64] \n\t" + "extp %[step_2], $ac0, 31 \n\t" + + /* + temp2 = input[1] * cospi_8_64 + input[3] * cospi_24_64; + step_3 = dct_const_round_shift(temp2); + */ + "madd $ac1, %[Temp0], %[cospi_8_64] \n\t" + "madd $ac1, %[Temp1], %[cospi_24_64] \n\t" + "extp %[step_3], $ac1, 31 \n\t" + + /* + output[0] = step_0 + step_3; + output[4] = step_1 + step_2; + output[8] = step_1 - step_2; + output[12] = step_0 - step_3; + */ + "add %[Temp0], %[step_0], %[step_3] \n\t" + "sh %[Temp0], 0(%[output]) \n\t" + + "add %[Temp1], %[step_1], %[step_2] \n\t" + "sh %[Temp1], 8(%[output]) \n\t" + + "sub %[Temp2], %[step_1], %[step_2] \n\t" + "sh %[Temp2], 16(%[output]) \n\t" + + "sub %[Temp3], %[step_0], %[step_3] \n\t" + "sh %[Temp3], 24(%[output]) \n\t" + + : [Temp0] "=&r"(Temp0), [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), + [Temp3] "=&r"(Temp3), [step_0] "=&r"(step_0), [step_1] "=&r"(step_1), + [step_2] "=&r"(step_2), [step_3] "=&r"(step_3), [output] "+r"(output) + : [const_2_power_13] "r"(const_2_power_13), + [cospi_8_64] "r"(cospi_8_64), [cospi_16_64] "r"(cospi_16_64), + [cospi_24_64] "r"(cospi_24_64), [input] "r"(input)); + + input += 4; + output += 1; + } +} + +void aom_idct4_columns_add_blk_dspr2(int16_t *input, uint8_t *dest, + int dest_stride) { + int16_t step_0, step_1, step_2, step_3; + int Temp0, Temp1, Temp2, Temp3; + const int const_2_power_13 = 8192; + int i; + uint8_t *dest_pix; + uint8_t *cm = aom_ff_cropTbl; + + /* prefetch aom_ff_cropTbl */ + prefetch_load(aom_ff_cropTbl); + prefetch_load(aom_ff_cropTbl + 32); + prefetch_load(aom_ff_cropTbl + 64); + prefetch_load(aom_ff_cropTbl + 96); + prefetch_load(aom_ff_cropTbl + 128); + prefetch_load(aom_ff_cropTbl + 160); + prefetch_load(aom_ff_cropTbl + 192); + prefetch_load(aom_ff_cropTbl + 224); + + for (i = 0; i < 4; ++i) { + dest_pix = (dest + i); + + __asm__ __volatile__( + /* + temp_1 = (input[0] + input[2]) * cospi_16_64; + step_0 = dct_const_round_shift(temp_1); + + temp_2 = (input[0] - input[2]) * cospi_16_64; + step_1 = dct_const_round_shift(temp_2); + */ + "lh %[Temp0], 0(%[input]) \n\t" + "lh %[Temp1], 4(%[input]) \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "add %[Temp2], %[Temp0], %[Temp1] \n\t" + "sub %[Temp3], %[Temp0], %[Temp1] \n\t" + "madd $ac0, %[Temp2], %[cospi_16_64] \n\t" + "lh %[Temp0], 2(%[input]) \n\t" + "lh %[Temp1], 6(%[input]) \n\t" + "extp %[step_0], $ac0, 31 \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + + "madd $ac1, %[Temp3], %[cospi_16_64] \n\t" + "extp %[step_1], $ac1, 31 \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + + /* + temp1 = input[1] * cospi_24_64 - input[3] * cospi_8_64; + step_2 = dct_const_round_shift(temp1); + */ + "madd $ac0, %[Temp0], %[cospi_24_64] \n\t" + "msub $ac0, %[Temp1], %[cospi_8_64] \n\t" + "extp %[step_2], $ac0, 31 \n\t" + + /* + temp2 = input[1] * cospi_8_64 + input[3] * cospi_24_64; + step_3 = dct_const_round_shift(temp2); + */ + "madd $ac1, %[Temp0], %[cospi_8_64] \n\t" + "madd $ac1, %[Temp1], %[cospi_24_64] \n\t" + "extp %[step_3], $ac1, 31 \n\t" + + /* + output[0] = step_0 + step_3; + output[4] = step_1 + step_2; + output[8] = step_1 - step_2; + output[12] = step_0 - step_3; + */ + "add %[Temp0], %[step_0], %[step_3] \n\t" + "addi %[Temp0], %[Temp0], 8 \n\t" + "sra %[Temp0], %[Temp0], 4 \n\t" + "lbu %[Temp1], 0(%[dest_pix]) \n\t" + "add %[Temp1], %[Temp1], %[Temp0] \n\t" + "add %[Temp0], %[step_1], %[step_2] \n\t" + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "sb %[Temp2], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "addi %[Temp0], %[Temp0], 8 \n\t" + "sra %[Temp0], %[Temp0], 4 \n\t" + "lbu %[Temp1], 0(%[dest_pix]) \n\t" + "add %[Temp1], %[Temp1], %[Temp0] \n\t" + "sub %[Temp0], %[step_1], %[step_2] \n\t" + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "sb %[Temp2], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "addi %[Temp0], %[Temp0], 8 \n\t" + "sra %[Temp0], %[Temp0], 4 \n\t" + "lbu %[Temp1], 0(%[dest_pix]) \n\t" + "add %[Temp1], %[Temp1], %[Temp0] \n\t" + "sub %[Temp0], %[step_0], %[step_3] \n\t" + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "sb %[Temp2], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "addi %[Temp0], %[Temp0], 8 \n\t" + "sra %[Temp0], %[Temp0], 4 \n\t" + "lbu %[Temp1], 0(%[dest_pix]) \n\t" + "add %[Temp1], %[Temp1], %[Temp0] \n\t" + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "sb %[Temp2], 0(%[dest_pix]) \n\t" + + : [Temp0] "=&r"(Temp0), [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), + [Temp3] "=&r"(Temp3), [step_0] "=&r"(step_0), [step_1] "=&r"(step_1), + [step_2] "=&r"(step_2), [step_3] "=&r"(step_3), + [dest_pix] "+r"(dest_pix) + : [const_2_power_13] "r"(const_2_power_13), + [cospi_8_64] "r"(cospi_8_64), [cospi_16_64] "r"(cospi_16_64), + [cospi_24_64] "r"(cospi_24_64), [input] "r"(input), [cm] "r"(cm), + [dest_stride] "r"(dest_stride)); + + input += 4; + } +} + +void aom_idct4x4_16_add_dspr2(const int16_t *input, uint8_t *dest, + int dest_stride) { + DECLARE_ALIGNED(32, int16_t, out[4 * 4]); + int16_t *outptr = out; + uint32_t pos = 45; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + // Rows + aom_idct4_rows_dspr2(input, outptr); + + // Columns + aom_idct4_columns_add_blk_dspr2(&out[0], dest, dest_stride); +} + +void aom_idct4x4_1_add_dspr2(const int16_t *input, uint8_t *dest, + int dest_stride) { + int a1, absa1; + int r; + int32_t out; + int t2, vector_a1, vector_a; + uint32_t pos = 45; + int16_t input_dc = input[0]; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + + : + : [pos] "r"(pos)); + + out = DCT_CONST_ROUND_SHIFT_TWICE_COSPI_16_64(input_dc); + __asm__ __volatile__( + "addi %[out], %[out], 8 \n\t" + "sra %[a1], %[out], 4 \n\t" + + : [out] "+r"(out), [a1] "=r"(a1) + :); + + if (a1 < 0) { + /* use quad-byte + * input and output memory are four byte aligned */ + __asm__ __volatile__( + "abs %[absa1], %[a1] \n\t" + "replv.qb %[vector_a1], %[absa1] \n\t" + + : [absa1] "=r"(absa1), [vector_a1] "=r"(vector_a1) + : [a1] "r"(a1)); + + for (r = 4; r--;) { + __asm__ __volatile__( + "lw %[t2], 0(%[dest]) \n\t" + "subu_s.qb %[vector_a], %[t2], %[vector_a1] \n\t" + "sw %[vector_a], 0(%[dest]) \n\t" + "add %[dest], %[dest], %[dest_stride] \n\t" + + : [t2] "=&r"(t2), [vector_a] "=&r"(vector_a), [dest] "+&r"(dest) + : [dest_stride] "r"(dest_stride), [vector_a1] "r"(vector_a1)); + } + } else { + /* use quad-byte + * input and output memory are four byte aligned */ + __asm__ __volatile__("replv.qb %[vector_a1], %[a1] \n\t" + : [vector_a1] "=r"(vector_a1) + : [a1] "r"(a1)); + + for (r = 4; r--;) { + __asm__ __volatile__( + "lw %[t2], 0(%[dest]) \n\t" + "addu_s.qb %[vector_a], %[t2], %[vector_a1] \n\t" + "sw %[vector_a], 0(%[dest]) \n\t" + "add %[dest], %[dest], %[dest_stride] \n\t" + + : [t2] "=&r"(t2), [vector_a] "=&r"(vector_a), [dest] "+&r"(dest) + : [dest_stride] "r"(dest_stride), [vector_a1] "r"(vector_a1)); + } + } +} + +void iadst4_dspr2(const int16_t *input, int16_t *output) { + int s0, s1, s2, s3, s4, s5, s6, s7; + int x0, x1, x2, x3; + + x0 = input[0]; + x1 = input[1]; + x2 = input[2]; + x3 = input[3]; + + if (!(x0 | x1 | x2 | x3)) { + output[0] = output[1] = output[2] = output[3] = 0; + return; + } + + s0 = sinpi_1_9 * x0; + s1 = sinpi_2_9 * x0; + s2 = sinpi_3_9 * x1; + s3 = sinpi_4_9 * x2; + s4 = sinpi_1_9 * x2; + s5 = sinpi_2_9 * x3; + s6 = sinpi_4_9 * x3; + s7 = x0 - x2 + x3; + + x0 = s0 + s3 + s5; + x1 = s1 - s4 - s6; + x2 = sinpi_3_9 * s7; + x3 = s2; + + s0 = x0 + x3; + s1 = x1 + x3; + s2 = x2; + s3 = x0 + x1 - x3; + + // 1-D transform scaling factor is sqrt(2). + // The overall dynamic range is 14b (input) + 14b (multiplication scaling) + // + 1b (addition) = 29b. + // Hence the output bit depth is 15b. + output[0] = dct_const_round_shift(s0); + output[1] = dct_const_round_shift(s1); + output[2] = dct_const_round_shift(s2); + output[3] = dct_const_round_shift(s3); +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/aom_dsp/mips/itrans8_dspr2.c b/third_party/aom/aom_dsp/mips/itrans8_dspr2.c new file mode 100644 index 0000000000..0a20f76f2d --- /dev/null +++ b/third_party/aom/aom_dsp/mips/itrans8_dspr2.c @@ -0,0 +1,645 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/inv_txfm_dspr2.h" +#include "aom_dsp/txfm_common.h" + +#if HAVE_DSPR2 +void idct8_rows_dspr2(const int16_t *input, int16_t *output, uint32_t no_rows) { + int step1_0, step1_1, step1_2, step1_3, step1_4, step1_5, step1_6, step1_7; + const int const_2_power_13 = 8192; + int Temp0, Temp1, Temp2, Temp3, Temp4; + int i; + + for (i = no_rows; i--;) { + __asm__ __volatile__( + /* + temp_1 = (input[0] + input[4]) * cospi_16_64; + step2_0 = dct_const_round_shift(temp_1); + + temp_2 = (input[0] - input[4]) * cospi_16_64; + step2_1 = dct_const_round_shift(temp_2); + */ + "lh %[Temp0], 0(%[input]) \n\t" + "lh %[Temp1], 8(%[input]) \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "add %[Temp2], %[Temp0], %[Temp1] \n\t" + "madd $ac0, %[Temp2], %[cospi_16_64] \n\t" + "extp %[Temp4], $ac0, 31 \n\t" + + "sub %[Temp3], %[Temp0], %[Temp1] \n\t" + "madd $ac1, %[Temp3], %[cospi_16_64] \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "extp %[Temp2], $ac1, 31 \n\t" + + /* + temp_1 = input[2] * cospi_24_64 - input[6] * cospi_8_64; + step2_2 = dct_const_round_shift(temp_1); + */ + "lh %[Temp0], 4(%[input]) \n\t" + "lh %[Temp1], 12(%[input]) \n\t" + "madd $ac0, %[Temp0], %[cospi_24_64] \n\t" + "msub $ac0, %[Temp1], %[cospi_8_64] \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "extp %[Temp3], $ac0, 31 \n\t" + + /* + step1_1 = step2_1 + step2_2; + step1_2 = step2_1 - step2_2; + */ + "add %[step1_1], %[Temp2], %[Temp3] \n\t" + "sub %[step1_2], %[Temp2], %[Temp3] \n\t" + + /* + temp_2 = input[2] * cospi_8_64 + input[6] * cospi_24_64; + step2_3 = dct_const_round_shift(temp_2); + */ + "madd $ac1, %[Temp0], %[cospi_8_64] \n\t" + "madd $ac1, %[Temp1], %[cospi_24_64] \n\t" + "extp %[Temp1], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + + /* + step1_0 = step2_0 + step2_3; + step1_3 = step2_0 - step2_3; + */ + "add %[step1_0], %[Temp4], %[Temp1] \n\t" + "sub %[step1_3], %[Temp4], %[Temp1] \n\t" + + /* + temp_1 = input[1] * cospi_28_64 - input[7] * cospi_4_64; + step1_4 = dct_const_round_shift(temp_1); + */ + "lh %[Temp0], 2(%[input]) \n\t" + "madd $ac0, %[Temp0], %[cospi_28_64] \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "lh %[Temp1], 14(%[input]) \n\t" + "lh %[Temp0], 2(%[input]) \n\t" + "msub $ac0, %[Temp1], %[cospi_4_64] \n\t" + "extp %[step1_4], $ac0, 31 \n\t" + + /* + temp_2 = input[1] * cospi_4_64 + input[7] * cospi_28_64; + step1_7 = dct_const_round_shift(temp_2); + */ + "madd $ac1, %[Temp0], %[cospi_4_64] \n\t" + "madd $ac1, %[Temp1], %[cospi_28_64] \n\t" + "extp %[step1_7], $ac1, 31 \n\t" + + /* + temp_1 = input[5] * cospi_12_64 - input[3] * cospi_20_64; + step1_5 = dct_const_round_shift(temp_1); + */ + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "lh %[Temp0], 10(%[input]) \n\t" + "madd $ac0, %[Temp0], %[cospi_12_64] \n\t" + "lh %[Temp1], 6(%[input]) \n\t" + "msub $ac0, %[Temp1], %[cospi_20_64] \n\t" + "extp %[step1_5], $ac0, 31 \n\t" + + /* + temp_2 = input[5] * cospi_20_64 + input[3] * cospi_12_64; + step1_6 = dct_const_round_shift(temp_2); + */ + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "lh %[Temp0], 10(%[input]) \n\t" + "madd $ac1, %[Temp0], %[cospi_20_64] \n\t" + "lh %[Temp1], 6(%[input]) \n\t" + "madd $ac1, %[Temp1], %[cospi_12_64] \n\t" + "extp %[step1_6], $ac1, 31 \n\t" + + /* + temp_1 = (step1_7 - step1_6 - step1_4 + step1_5) * cospi_16_64; + temp_2 = (step1_4 - step1_5 - step1_6 + step1_7) * cospi_16_64; + */ + "sub %[Temp0], %[step1_7], %[step1_6] \n\t" + "sub %[Temp0], %[Temp0], %[step1_4] \n\t" + "add %[Temp0], %[Temp0], %[step1_5] \n\t" + "sub %[Temp1], %[step1_4], %[step1_5] \n\t" + "sub %[Temp1], %[Temp1], %[step1_6] \n\t" + "add %[Temp1], %[Temp1], %[step1_7] \n\t" + + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + + "madd $ac0, %[Temp0], %[cospi_16_64] \n\t" + "madd $ac1, %[Temp1], %[cospi_16_64] \n\t" + + /* + step1_4 = step1_4 + step1_5; + step1_7 = step1_6 + step1_7; + */ + "add %[step1_4], %[step1_4], %[step1_5] \n\t" + "add %[step1_7], %[step1_7], %[step1_6] \n\t" + + "extp %[step1_5], $ac0, 31 \n\t" + "extp %[step1_6], $ac1, 31 \n\t" + + "add %[Temp0], %[step1_0], %[step1_7] \n\t" + "sh %[Temp0], 0(%[output]) \n\t" + "add %[Temp1], %[step1_1], %[step1_6] \n\t" + "sh %[Temp1], 16(%[output]) \n\t" + "add %[Temp0], %[step1_2], %[step1_5] \n\t" + "sh %[Temp0], 32(%[output]) \n\t" + "add %[Temp1], %[step1_3], %[step1_4] \n\t" + "sh %[Temp1], 48(%[output]) \n\t" + + "sub %[Temp0], %[step1_3], %[step1_4] \n\t" + "sh %[Temp0], 64(%[output]) \n\t" + "sub %[Temp1], %[step1_2], %[step1_5] \n\t" + "sh %[Temp1], 80(%[output]) \n\t" + "sub %[Temp0], %[step1_1], %[step1_6] \n\t" + "sh %[Temp0], 96(%[output]) \n\t" + "sub %[Temp1], %[step1_0], %[step1_7] \n\t" + "sh %[Temp1], 112(%[output]) \n\t" + + : [step1_0] "=&r"(step1_0), [step1_1] "=&r"(step1_1), + [step1_2] "=&r"(step1_2), [step1_3] "=&r"(step1_3), + [step1_4] "=&r"(step1_4), [step1_5] "=&r"(step1_5), + [step1_6] "=&r"(step1_6), [step1_7] "=&r"(step1_7), + [Temp0] "=&r"(Temp0), [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), + [Temp3] "=&r"(Temp3), [Temp4] "=&r"(Temp4) + : [const_2_power_13] "r"(const_2_power_13), + [cospi_16_64] "r"(cospi_16_64), [cospi_28_64] "r"(cospi_28_64), + [cospi_4_64] "r"(cospi_4_64), [cospi_12_64] "r"(cospi_12_64), + [cospi_20_64] "r"(cospi_20_64), [cospi_8_64] "r"(cospi_8_64), + [cospi_24_64] "r"(cospi_24_64), [output] "r"(output), + [input] "r"(input)); + + input += 8; + output += 1; + } +} + +void idct8_columns_add_blk_dspr2(int16_t *input, uint8_t *dest, + int dest_stride) { + int step1_0, step1_1, step1_2, step1_3, step1_4, step1_5, step1_6, step1_7; + int Temp0, Temp1, Temp2, Temp3; + int i; + const int const_2_power_13 = 8192; + uint8_t *dest_pix; + uint8_t *cm = aom_ff_cropTbl; + + /* prefetch aom_ff_cropTbl */ + prefetch_load(aom_ff_cropTbl); + prefetch_load(aom_ff_cropTbl + 32); + prefetch_load(aom_ff_cropTbl + 64); + prefetch_load(aom_ff_cropTbl + 96); + prefetch_load(aom_ff_cropTbl + 128); + prefetch_load(aom_ff_cropTbl + 160); + prefetch_load(aom_ff_cropTbl + 192); + prefetch_load(aom_ff_cropTbl + 224); + + for (i = 0; i < 8; ++i) { + dest_pix = (dest + i); + + __asm__ __volatile__( + /* + temp_1 = (input[0] + input[4]) * cospi_16_64; + step2_0 = dct_const_round_shift(temp_1); + + temp_2 = (input[0] - input[4]) * cospi_16_64; + step2_1 = dct_const_round_shift(temp_2); + */ + "lh %[Temp0], 0(%[input]) \n\t" + "lh %[Temp1], 8(%[input]) \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "add %[Temp2], %[Temp0], %[Temp1] \n\t" + "madd $ac0, %[Temp2], %[cospi_16_64] \n\t" + "extp %[step1_6], $ac0, 31 \n\t" + + "sub %[Temp3], %[Temp0], %[Temp1] \n\t" + "madd $ac1, %[Temp3], %[cospi_16_64] \n\t" + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "extp %[Temp2], $ac1, 31 \n\t" + + /* + temp_1 = input[2] * cospi_24_64 - input[6] * cospi_8_64; + step2_2 = dct_const_round_shift(temp_1); + */ + "lh %[Temp0], 4(%[input]) \n\t" + "lh %[Temp1], 12(%[input]) \n\t" + "madd $ac0, %[Temp0], %[cospi_24_64] \n\t" + "msub $ac0, %[Temp1], %[cospi_8_64] \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "extp %[Temp3], $ac0, 31 \n\t" + + /* + step1_1 = step2_1 + step2_2; + step1_2 = step2_1 - step2_2; + */ + "add %[step1_1], %[Temp2], %[Temp3] \n\t" + "sub %[step1_2], %[Temp2], %[Temp3] \n\t" + + /* + temp_2 = input[2] * cospi_8_64 + input[6] * cospi_24_64; + step2_3 = dct_const_round_shift(temp_2); + */ + "madd $ac1, %[Temp0], %[cospi_8_64] \n\t" + "madd $ac1, %[Temp1], %[cospi_24_64] \n\t" + "extp %[Temp1], $ac1, 31 \n\t" + + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + + /* + step1_0 = step2_0 + step2_3; + step1_3 = step2_0 - step2_3; + */ + "add %[step1_0], %[step1_6], %[Temp1] \n\t" + "sub %[step1_3], %[step1_6], %[Temp1] \n\t" + + /* + temp_1 = input[1] * cospi_28_64 - input[7] * cospi_4_64; + step1_4 = dct_const_round_shift(temp_1); + */ + "lh %[Temp0], 2(%[input]) \n\t" + "madd $ac0, %[Temp0], %[cospi_28_64] \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "lh %[Temp1], 14(%[input]) \n\t" + "lh %[Temp0], 2(%[input]) \n\t" + "msub $ac0, %[Temp1], %[cospi_4_64] \n\t" + "extp %[step1_4], $ac0, 31 \n\t" + + /* + temp_2 = input[1] * cospi_4_64 + input[7] * cospi_28_64; + step1_7 = dct_const_round_shift(temp_2); + */ + "madd $ac1, %[Temp0], %[cospi_4_64] \n\t" + "madd $ac1, %[Temp1], %[cospi_28_64] \n\t" + "extp %[step1_7], $ac1, 31 \n\t" + + /* + temp_1 = input[5] * cospi_12_64 - input[3] * cospi_20_64; + step1_5 = dct_const_round_shift(temp_1); + */ + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "lh %[Temp0], 10(%[input]) \n\t" + "madd $ac0, %[Temp0], %[cospi_12_64] \n\t" + "lh %[Temp1], 6(%[input]) \n\t" + "msub $ac0, %[Temp1], %[cospi_20_64] \n\t" + "extp %[step1_5], $ac0, 31 \n\t" + + /* + temp_2 = input[5] * cospi_20_64 + input[3] * cospi_12_64; + step1_6 = dct_const_round_shift(temp_2); + */ + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + "lh %[Temp0], 10(%[input]) \n\t" + "madd $ac1, %[Temp0], %[cospi_20_64] \n\t" + "lh %[Temp1], 6(%[input]) \n\t" + "madd $ac1, %[Temp1], %[cospi_12_64] \n\t" + "extp %[step1_6], $ac1, 31 \n\t" + + /* + temp_1 = (step1_7 - step1_6 - step1_4 + step1_5) * cospi_16_64; + temp_2 = (step1_4 - step1_5 - step1_6 + step1_7) * cospi_16_64; + */ + "sub %[Temp0], %[step1_7], %[step1_6] \n\t" + "sub %[Temp0], %[Temp0], %[step1_4] \n\t" + "add %[Temp0], %[Temp0], %[step1_5] \n\t" + "sub %[Temp1], %[step1_4], %[step1_5] \n\t" + "sub %[Temp1], %[Temp1], %[step1_6] \n\t" + "add %[Temp1], %[Temp1], %[step1_7] \n\t" + + "mtlo %[const_2_power_13], $ac0 \n\t" + "mthi $zero, $ac0 \n\t" + "mtlo %[const_2_power_13], $ac1 \n\t" + "mthi $zero, $ac1 \n\t" + + "madd $ac0, %[Temp0], %[cospi_16_64] \n\t" + "madd $ac1, %[Temp1], %[cospi_16_64] \n\t" + + /* + step1_4 = step1_4 + step1_5; + step1_7 = step1_6 + step1_7; + */ + "add %[step1_4], %[step1_4], %[step1_5] \n\t" + "add %[step1_7], %[step1_7], %[step1_6] \n\t" + + "extp %[step1_5], $ac0, 31 \n\t" + "extp %[step1_6], $ac1, 31 \n\t" + + /* add block */ + "lbu %[Temp1], 0(%[dest_pix]) \n\t" + "add %[Temp0], %[step1_0], %[step1_7] \n\t" + "addi %[Temp0], %[Temp0], 16 \n\t" + "sra %[Temp0], %[Temp0], 5 \n\t" + "add %[Temp1], %[Temp1], %[Temp0] \n\t" + "add %[Temp0], %[step1_1], %[step1_6] \n\t" + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "sb %[Temp2], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[Temp1], 0(%[dest_pix]) \n\t" + "addi %[Temp0], %[Temp0], 16 \n\t" + "sra %[Temp0], %[Temp0], 5 \n\t" + "add %[Temp1], %[Temp1], %[Temp0] \n\t" + "add %[Temp0], %[step1_2], %[step1_5] \n\t" + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "sb %[Temp2], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[Temp1], 0(%[dest_pix]) \n\t" + "addi %[Temp0], %[Temp0], 16 \n\t" + "sra %[Temp0], %[Temp0], 5 \n\t" + "add %[Temp1], %[Temp1], %[Temp0] \n\t" + "add %[Temp0], %[step1_3], %[step1_4] \n\t" + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "sb %[Temp2], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[Temp1], 0(%[dest_pix]) \n\t" + "addi %[Temp0], %[Temp0], 16 \n\t" + "sra %[Temp0], %[Temp0], 5 \n\t" + "add %[Temp1], %[Temp1], %[Temp0] \n\t" + "sub %[Temp0], %[step1_3], %[step1_4] \n\t" + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "sb %[Temp2], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[Temp1], 0(%[dest_pix]) \n\t" + "addi %[Temp0], %[Temp0], 16 \n\t" + "sra %[Temp0], %[Temp0], 5 \n\t" + "add %[Temp1], %[Temp1], %[Temp0] \n\t" + "sub %[Temp0], %[step1_2], %[step1_5] \n\t" + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "sb %[Temp2], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[Temp1], 0(%[dest_pix]) \n\t" + "addi %[Temp0], %[Temp0], 16 \n\t" + "sra %[Temp0], %[Temp0], 5 \n\t" + "add %[Temp1], %[Temp1], %[Temp0] \n\t" + "sub %[Temp0], %[step1_1], %[step1_6] \n\t" + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "sb %[Temp2], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[Temp1], 0(%[dest_pix]) \n\t" + "addi %[Temp0], %[Temp0], 16 \n\t" + "sra %[Temp0], %[Temp0], 5 \n\t" + "add %[Temp1], %[Temp1], %[Temp0] \n\t" + "sub %[Temp0], %[step1_0], %[step1_7] \n\t" + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "sb %[Temp2], 0(%[dest_pix]) \n\t" + "addu %[dest_pix], %[dest_pix], %[dest_stride] \n\t" + + "lbu %[Temp1], 0(%[dest_pix]) \n\t" + "addi %[Temp0], %[Temp0], 16 \n\t" + "sra %[Temp0], %[Temp0], 5 \n\t" + "add %[Temp1], %[Temp1], %[Temp0] \n\t" + "lbux %[Temp2], %[Temp1](%[cm]) \n\t" + "sb %[Temp2], 0(%[dest_pix]) \n\t" + + : [step1_0] "=&r"(step1_0), [step1_1] "=&r"(step1_1), + [step1_2] "=&r"(step1_2), [step1_3] "=&r"(step1_3), + [step1_4] "=&r"(step1_4), [step1_5] "=&r"(step1_5), + [step1_6] "=&r"(step1_6), [step1_7] "=&r"(step1_7), + [Temp0] "=&r"(Temp0), [Temp1] "=&r"(Temp1), [Temp2] "=&r"(Temp2), + [Temp3] "=&r"(Temp3), [dest_pix] "+r"(dest_pix) + : [const_2_power_13] "r"(const_2_power_13), + [cospi_16_64] "r"(cospi_16_64), [cospi_28_64] "r"(cospi_28_64), + [cospi_4_64] "r"(cospi_4_64), [cospi_12_64] "r"(cospi_12_64), + [cospi_20_64] "r"(cospi_20_64), [cospi_8_64] "r"(cospi_8_64), + [cospi_24_64] "r"(cospi_24_64), [input] "r"(input), [cm] "r"(cm), + [dest_stride] "r"(dest_stride)); + + input += 8; + } +} + +void aom_idct8x8_64_add_dspr2(const int16_t *input, uint8_t *dest, + int dest_stride) { + DECLARE_ALIGNED(32, int16_t, out[8 * 8]); + int16_t *outptr = out; + uint32_t pos = 45; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" : : [pos] "r"(pos)); + + // First transform rows + idct8_rows_dspr2(input, outptr, 8); + + // Then transform columns and add to dest + idct8_columns_add_blk_dspr2(&out[0], dest, dest_stride); +} + +void aom_idct8x8_12_add_dspr2(const int16_t *input, uint8_t *dest, + int dest_stride) { + DECLARE_ALIGNED(32, int16_t, out[8 * 8]); + int16_t *outptr = out; + uint32_t pos = 45; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" : : [pos] "r"(pos)); + + // First transform rows + idct8_rows_dspr2(input, outptr, 4); + + outptr += 4; + + __asm__ __volatile__( + "sw $zero, 0(%[outptr]) \n\t" + "sw $zero, 4(%[outptr]) \n\t" + "sw $zero, 16(%[outptr]) \n\t" + "sw $zero, 20(%[outptr]) \n\t" + "sw $zero, 32(%[outptr]) \n\t" + "sw $zero, 36(%[outptr]) \n\t" + "sw $zero, 48(%[outptr]) \n\t" + "sw $zero, 52(%[outptr]) \n\t" + "sw $zero, 64(%[outptr]) \n\t" + "sw $zero, 68(%[outptr]) \n\t" + "sw $zero, 80(%[outptr]) \n\t" + "sw $zero, 84(%[outptr]) \n\t" + "sw $zero, 96(%[outptr]) \n\t" + "sw $zero, 100(%[outptr]) \n\t" + "sw $zero, 112(%[outptr]) \n\t" + "sw $zero, 116(%[outptr]) \n\t" + + : + : [outptr] "r"(outptr)); + + // Then transform columns and add to dest + idct8_columns_add_blk_dspr2(&out[0], dest, dest_stride); +} + +void aom_idct8x8_1_add_dspr2(const int16_t *input, uint8_t *dest, + int dest_stride) { + uint32_t pos = 45; + int32_t out; + int32_t r; + int32_t a1, absa1; + int32_t t1, t2, vector_a1, vector_1, vector_2; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + + : + : [pos] "r"(pos)); + + out = DCT_CONST_ROUND_SHIFT_TWICE_COSPI_16_64(input[0]); + __asm__ __volatile__( + "addi %[out], %[out], 16 \n\t" + "sra %[a1], %[out], 5 \n\t" + + : [out] "+r"(out), [a1] "=r"(a1) + :); + + if (a1 < 0) { + /* use quad-byte + * input and output memory are four byte aligned */ + __asm__ __volatile__( + "abs %[absa1], %[a1] \n\t" + "replv.qb %[vector_a1], %[absa1] \n\t" + + : [absa1] "=r"(absa1), [vector_a1] "=r"(vector_a1) + : [a1] "r"(a1)); + + for (r = 8; r--;) { + __asm__ __volatile__( + "lw %[t1], 0(%[dest]) \n\t" + "lw %[t2], 4(%[dest]) \n\t" + "subu_s.qb %[vector_1], %[t1], %[vector_a1] \n\t" + "subu_s.qb %[vector_2], %[t2], %[vector_a1] \n\t" + "sw %[vector_1], 0(%[dest]) \n\t" + "sw %[vector_2], 4(%[dest]) \n\t" + "add %[dest], %[dest], %[dest_stride] \n\t" + + : [t1] "=&r"(t1), [t2] "=&r"(t2), [vector_1] "=&r"(vector_1), + [vector_2] "=&r"(vector_2), [dest] "+&r"(dest) + : [dest_stride] "r"(dest_stride), [vector_a1] "r"(vector_a1)); + } + } else { + /* use quad-byte + * input and output memory are four byte aligned */ + __asm__ __volatile__("replv.qb %[vector_a1], %[a1] \n\t" + + : [vector_a1] "=r"(vector_a1) + : [a1] "r"(a1)); + + for (r = 8; r--;) { + __asm__ __volatile__( + "lw %[t1], 0(%[dest]) \n\t" + "lw %[t2], 4(%[dest]) \n\t" + "addu_s.qb %[vector_1], %[t1], %[vector_a1] \n\t" + "addu_s.qb %[vector_2], %[t2], %[vector_a1] \n\t" + "sw %[vector_1], 0(%[dest]) \n\t" + "sw %[vector_2], 4(%[dest]) \n\t" + "add %[dest], %[dest], %[dest_stride] \n\t" + + : [t1] "=&r"(t1), [t2] "=&r"(t2), [vector_1] "=&r"(vector_1), + [vector_2] "=&r"(vector_2), [dest] "+r"(dest) + : [dest_stride] "r"(dest_stride), [vector_a1] "r"(vector_a1)); + } + } +} + +void iadst8_dspr2(const int16_t *input, int16_t *output) { + int s0, s1, s2, s3, s4, s5, s6, s7; + int x0, x1, x2, x3, x4, x5, x6, x7; + + x0 = input[7]; + x1 = input[0]; + x2 = input[5]; + x3 = input[2]; + x4 = input[3]; + x5 = input[4]; + x6 = input[1]; + x7 = input[6]; + + if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7)) { + output[0] = output[1] = output[2] = output[3] = output[4] = output[5] = + output[6] = output[7] = 0; + return; + } + + // stage 1 + s0 = cospi_2_64 * x0 + cospi_30_64 * x1; + s1 = cospi_30_64 * x0 - cospi_2_64 * x1; + s2 = cospi_10_64 * x2 + cospi_22_64 * x3; + s3 = cospi_22_64 * x2 - cospi_10_64 * x3; + s4 = cospi_18_64 * x4 + cospi_14_64 * x5; + s5 = cospi_14_64 * x4 - cospi_18_64 * x5; + s6 = cospi_26_64 * x6 + cospi_6_64 * x7; + s7 = cospi_6_64 * x6 - cospi_26_64 * x7; + + x0 = ROUND_POWER_OF_TWO((s0 + s4), DCT_CONST_BITS); + x1 = ROUND_POWER_OF_TWO((s1 + s5), DCT_CONST_BITS); + x2 = ROUND_POWER_OF_TWO((s2 + s6), DCT_CONST_BITS); + x3 = ROUND_POWER_OF_TWO((s3 + s7), DCT_CONST_BITS); + x4 = ROUND_POWER_OF_TWO((s0 - s4), DCT_CONST_BITS); + x5 = ROUND_POWER_OF_TWO((s1 - s5), DCT_CONST_BITS); + x6 = ROUND_POWER_OF_TWO((s2 - s6), DCT_CONST_BITS); + x7 = ROUND_POWER_OF_TWO((s3 - s7), DCT_CONST_BITS); + + // stage 2 + s0 = x0; + s1 = x1; + s2 = x2; + s3 = x3; + s4 = cospi_8_64 * x4 + cospi_24_64 * x5; + s5 = cospi_24_64 * x4 - cospi_8_64 * x5; + s6 = -cospi_24_64 * x6 + cospi_8_64 * x7; + s7 = cospi_8_64 * x6 + cospi_24_64 * x7; + + x0 = s0 + s2; + x1 = s1 + s3; + x2 = s0 - s2; + x3 = s1 - s3; + x4 = ROUND_POWER_OF_TWO((s4 + s6), DCT_CONST_BITS); + x5 = ROUND_POWER_OF_TWO((s5 + s7), DCT_CONST_BITS); + x6 = ROUND_POWER_OF_TWO((s4 - s6), DCT_CONST_BITS); + x7 = ROUND_POWER_OF_TWO((s5 - s7), DCT_CONST_BITS); + + // stage 3 + s2 = cospi_16_64 * (x2 + x3); + s3 = cospi_16_64 * (x2 - x3); + s6 = cospi_16_64 * (x6 + x7); + s7 = cospi_16_64 * (x6 - x7); + + x2 = ROUND_POWER_OF_TWO((s2), DCT_CONST_BITS); + x3 = ROUND_POWER_OF_TWO((s3), DCT_CONST_BITS); + x6 = ROUND_POWER_OF_TWO((s6), DCT_CONST_BITS); + x7 = ROUND_POWER_OF_TWO((s7), DCT_CONST_BITS); + + output[0] = x0; + output[1] = -x4; + output[2] = x6; + output[3] = -x2; + output[4] = x3; + output[5] = -x7; + output[6] = x5; + output[7] = -x1; +} +#endif // HAVE_DSPR2 diff --git a/third_party/aom/aom_dsp/mips/loopfilter_16_msa.c b/third_party/aom/aom_dsp/mips/loopfilter_16_msa.c new file mode 100644 index 0000000000..fc0c32ce39 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/loopfilter_16_msa.c @@ -0,0 +1,1487 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_ports/mem.h" +#include "aom_dsp/mips/loopfilter_msa.h" + +int32_t aom_hz_lpf_t4_and_t8_16w(uint8_t *src, int32_t pitch, uint8_t *filter48, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + v16u8 p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + v16u8 flat, mask, hev, thresh, b_limit, limit; + v8u16 p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r; + v8u16 p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + v8i16 p2_filt8_r, p1_filt8_r, p0_filt8_r, q0_filt8_r, q1_filt8_r, q2_filt8_r; + v8i16 p2_filt8_l, p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l; + v16u8 zero = { 0 }; + + /* load vector elements */ + LD_UB8(src - (4 * pitch), pitch, p3, p2, p1, p0, q0, q1, q2, q3); + + thresh = (v16u8)__msa_fill_b(*thresh_ptr); + b_limit = (v16u8)__msa_fill_b(*b_limit_ptr); + limit = (v16u8)__msa_fill_b(*limit_ptr); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + AOM_FLAT4(p3, p2, p0, q0, q2, q3, flat); + AOM_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + if (__msa_test_bz_v(flat)) { + ST_UB4(p1_out, p0_out, q0_out, q1_out, (src - 2 * pitch), pitch); + + return 1; + } else { + ILVR_B8_UH(zero, p3, zero, p2, zero, p1, zero, p0, zero, q0, zero, q1, zero, + q2, zero, q3, p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r); + AOM_FILTER8(p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r, p2_filt8_r, + p1_filt8_r, p0_filt8_r, q0_filt8_r, q1_filt8_r, q2_filt8_r); + + ILVL_B4_UH(zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, p1_l, p0_l); + ILVL_B4_UH(zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, q2_l, q3_l); + AOM_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + /* convert 16 bit output data into 8 bit */ + PCKEV_B4_SH(p2_filt8_l, p2_filt8_r, p1_filt8_l, p1_filt8_r, p0_filt8_l, + p0_filt8_r, q0_filt8_l, q0_filt8_r, p2_filt8_r, p1_filt8_r, + p0_filt8_r, q0_filt8_r); + PCKEV_B2_SH(q1_filt8_l, q1_filt8_r, q2_filt8_l, q2_filt8_r, q1_filt8_r, + q2_filt8_r); + + /* store pixel values */ + p2_out = __msa_bmnz_v(p2, (v16u8)p2_filt8_r, flat); + p1_out = __msa_bmnz_v(p1_out, (v16u8)p1_filt8_r, flat); + p0_out = __msa_bmnz_v(p0_out, (v16u8)p0_filt8_r, flat); + q0_out = __msa_bmnz_v(q0_out, (v16u8)q0_filt8_r, flat); + q1_out = __msa_bmnz_v(q1_out, (v16u8)q1_filt8_r, flat); + q2_out = __msa_bmnz_v(q2, (v16u8)q2_filt8_r, flat); + + ST_UB4(p2_out, p1_out, p0_out, q0_out, filter48, 16); + filter48 += (4 * 16); + ST_UB2(q1_out, q2_out, filter48, 16); + filter48 += (2 * 16); + ST_UB(flat, filter48); + + return 0; + } +} + +void aom_hz_lpf_t16_16w(uint8_t *src, int32_t pitch, uint8_t *filter48) { + v16u8 flat, flat2, filter8; + v16i8 zero = { 0 }; + v16u8 p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + v8u16 p7_r_in, p6_r_in, p5_r_in, p4_r_in, p3_r_in, p2_r_in, p1_r_in, p0_r_in; + v8u16 q7_r_in, q6_r_in, q5_r_in, q4_r_in, q3_r_in, q2_r_in, q1_r_in, q0_r_in; + v8u16 p7_l_in, p6_l_in, p5_l_in, p4_l_in, p3_l_in, p2_l_in, p1_l_in, p0_l_in; + v8u16 q7_l_in, q6_l_in, q5_l_in, q4_l_in, q3_l_in, q2_l_in, q1_l_in, q0_l_in; + v8u16 tmp0_r, tmp1_r, tmp0_l, tmp1_l; + v8i16 l_out, r_out; + + flat = LD_UB(filter48 + 96); + + LD_UB8((src - 8 * pitch), pitch, p7, p6, p5, p4, p3, p2, p1, p0); + LD_UB8(src, pitch, q0, q1, q2, q3, q4, q5, q6, q7); + AOM_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + + if (__msa_test_bz_v(flat2)) { + LD_UB4(filter48, 16, p2, p1, p0, q0); + LD_UB2(filter48 + 4 * 16, 16, q1, q2); + + src -= 3 * pitch; + ST_UB4(p2, p1, p0, q0, src, pitch); + src += (4 * pitch); + ST_UB2(q1, q2, src, pitch); + } else { + src -= 7 * pitch; + + ILVR_B8_UH(zero, p7, zero, p6, zero, p5, zero, p4, zero, p3, zero, p2, zero, + p1, zero, p0, p7_r_in, p6_r_in, p5_r_in, p4_r_in, p3_r_in, + p2_r_in, p1_r_in, p0_r_in); + + q0_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q0); + + tmp0_r = p7_r_in << 3; + tmp0_r -= p7_r_in; + tmp0_r += p6_r_in; + tmp0_r += q0_r_in; + tmp1_r = p6_r_in + p5_r_in; + tmp1_r += p4_r_in; + tmp1_r += p3_r_in; + tmp1_r += p2_r_in; + tmp1_r += p1_r_in; + tmp1_r += p0_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + ILVL_B4_UH(zero, p7, zero, p6, zero, p5, zero, p4, p7_l_in, p6_l_in, + p5_l_in, p4_l_in); + ILVL_B4_UH(zero, p3, zero, p2, zero, p1, zero, p0, p3_l_in, p2_l_in, + p1_l_in, p0_l_in); + q0_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q0); + + tmp0_l = p7_l_in << 3; + tmp0_l -= p7_l_in; + tmp0_l += p6_l_in; + tmp0_l += q0_l_in; + tmp1_l = p6_l_in + p5_l_in; + tmp1_l += p4_l_in; + tmp1_l += p3_l_in; + tmp1_l += p2_l_in; + tmp1_l += p1_l_in; + tmp1_l += p0_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + p6 = __msa_bmnz_v(p6, (v16u8)r_out, flat2); + ST_UB(p6, src); + src += pitch; + + /* p5 */ + q1_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q1); + tmp0_r = p5_r_in - p6_r_in; + tmp0_r += q1_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + q1_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q1); + tmp0_l = p5_l_in - p6_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + p5 = __msa_bmnz_v(p5, (v16u8)r_out, flat2); + ST_UB(p5, src); + src += pitch; + + /* p4 */ + q2_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q2); + tmp0_r = p4_r_in - p5_r_in; + tmp0_r += q2_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = (v8i16)__msa_srari_h((v8i16)tmp1_r, 4); + + q2_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q2); + tmp0_l = p4_l_in - p5_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + p4 = __msa_bmnz_v(p4, (v16u8)r_out, flat2); + ST_UB(p4, src); + src += pitch; + + /* p3 */ + q3_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q3); + tmp0_r = p3_r_in - p4_r_in; + tmp0_r += q3_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + q3_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q3); + tmp0_l = p3_l_in - p4_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + p3 = __msa_bmnz_v(p3, (v16u8)r_out, flat2); + ST_UB(p3, src); + src += pitch; + + /* p2 */ + q4_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q4); + filter8 = LD_UB(filter48); + tmp0_r = p2_r_in - p3_r_in; + tmp0_r += q4_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + q4_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q4); + tmp0_l = p2_l_in - p3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST_UB(filter8, src); + src += pitch; + + /* p1 */ + q5_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q5); + filter8 = LD_UB(filter48 + 16); + tmp0_r = p1_r_in - p2_r_in; + tmp0_r += q5_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + q5_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q5); + tmp0_l = p1_l_in - p2_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST_UB(filter8, src); + src += pitch; + + /* p0 */ + q6_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q6); + filter8 = LD_UB(filter48 + 32); + tmp0_r = p0_r_in - p1_r_in; + tmp0_r += q6_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + q6_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q6); + tmp0_l = p0_l_in - p1_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST_UB(filter8, src); + src += pitch; + + /* q0 */ + q7_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q7); + filter8 = LD_UB(filter48 + 48); + tmp0_r = q7_r_in - p0_r_in; + tmp0_r += q0_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + q7_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q7); + tmp0_l = q7_l_in - p0_l_in; + tmp0_l += q0_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST_UB(filter8, src); + src += pitch; + + /* q1 */ + filter8 = LD_UB(filter48 + 64); + tmp0_r = q7_r_in - q0_r_in; + tmp0_r += q1_r_in; + tmp0_r -= p6_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + tmp0_l = q7_l_in - q0_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p6_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST_UB(filter8, src); + src += pitch; + + /* q2 */ + filter8 = LD_UB(filter48 + 80); + tmp0_r = q7_r_in - q1_r_in; + tmp0_r += q2_r_in; + tmp0_r -= p5_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + tmp0_l = q7_l_in - q1_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p5_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST_UB(filter8, src); + src += pitch; + + /* q3 */ + tmp0_r = q7_r_in - q2_r_in; + tmp0_r += q3_r_in; + tmp0_r -= p4_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + tmp0_l = q7_l_in - q2_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p4_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + q3 = __msa_bmnz_v(q3, (v16u8)r_out, flat2); + ST_UB(q3, src); + src += pitch; + + /* q4 */ + tmp0_r = q7_r_in - q3_r_in; + tmp0_r += q4_r_in; + tmp0_r -= p3_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + tmp0_l = q7_l_in - q3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p3_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + q4 = __msa_bmnz_v(q4, (v16u8)r_out, flat2); + ST_UB(q4, src); + src += pitch; + + /* q5 */ + tmp0_r = q7_r_in - q4_r_in; + tmp0_r += q5_r_in; + tmp0_r -= p2_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + tmp0_l = q7_l_in - q4_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p2_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + q5 = __msa_bmnz_v(q5, (v16u8)r_out, flat2); + ST_UB(q5, src); + src += pitch; + + /* q6 */ + tmp0_r = q7_r_in - q5_r_in; + tmp0_r += q6_r_in; + tmp0_r -= p1_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + tmp0_l = q7_l_in - q5_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p1_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + q6 = __msa_bmnz_v(q6, (v16u8)r_out, flat2); + ST_UB(q6, src); + } +} + +void aom_lpf_horizontal_16_dual_msa(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr, int32_t count) { + DECLARE_ALIGNED(32, uint8_t, filter48[16 * 8]); + uint8_t early_exit = 0; + + (void)count; + + early_exit = aom_hz_lpf_t4_and_t8_16w(src, pitch, &filter48[0], b_limit_ptr, + limit_ptr, thresh_ptr); + + if (0 == early_exit) { + aom_hz_lpf_t16_16w(src, pitch, filter48); + } +} + +static void mb_lpf_horizontal_edge(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr, int32_t count) { + if (1 == count) { + uint64_t p2_d, p1_d, p0_d, q0_d, q1_d, q2_d; + uint64_t dword0, dword1; + v16u8 flat2, mask, hev, flat, thresh, b_limit, limit; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0, p7, p6, p5, p4, q4, q5, q6, q7; + v16u8 p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + v16u8 p0_filter16, p1_filter16; + v8i16 p2_filter8, p1_filter8, p0_filter8; + v8i16 q0_filter8, q1_filter8, q2_filter8; + v8u16 p7_r, p6_r, p5_r, p4_r, q7_r, q6_r, q5_r, q4_r; + v8u16 p3_r, p2_r, p1_r, p0_r, q3_r, q2_r, q1_r, q0_r; + v16i8 zero = { 0 }; + v8u16 tmp0, tmp1, tmp2; + + /* load vector elements */ + LD_UB8((src - 4 * pitch), pitch, p3, p2, p1, p0, q0, q1, q2, q3); + + thresh = (v16u8)__msa_fill_b(*thresh_ptr); + b_limit = (v16u8)__msa_fill_b(*b_limit_ptr); + limit = (v16u8)__msa_fill_b(*limit_ptr); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + AOM_FLAT4(p3, p2, p0, q0, q2, q3, flat); + AOM_LPF_FILTER4_8W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + flat = (v16u8)__msa_ilvr_d((v2i64)zero, (v2i64)flat); + + if (__msa_test_bz_v(flat)) { + p1_d = __msa_copy_u_d((v2i64)p1_out, 0); + p0_d = __msa_copy_u_d((v2i64)p0_out, 0); + q0_d = __msa_copy_u_d((v2i64)q0_out, 0); + q1_d = __msa_copy_u_d((v2i64)q1_out, 0); + SD4(p1_d, p0_d, q0_d, q1_d, src - 2 * pitch, pitch); + } else { + /* convert 8 bit input data into 16 bit */ + ILVR_B8_UH(zero, p3, zero, p2, zero, p1, zero, p0, zero, q0, zero, q1, + zero, q2, zero, q3, p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, + q3_r); + AOM_FILTER8(p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r, p2_filter8, + p1_filter8, p0_filter8, q0_filter8, q1_filter8, q2_filter8); + + /* convert 16 bit output data into 8 bit */ + PCKEV_B4_SH(zero, p2_filter8, zero, p1_filter8, zero, p0_filter8, zero, + q0_filter8, p2_filter8, p1_filter8, p0_filter8, q0_filter8); + PCKEV_B2_SH(zero, q1_filter8, zero, q2_filter8, q1_filter8, q2_filter8); + + /* store pixel values */ + p2_out = __msa_bmnz_v(p2, (v16u8)p2_filter8, flat); + p1_out = __msa_bmnz_v(p1_out, (v16u8)p1_filter8, flat); + p0_out = __msa_bmnz_v(p0_out, (v16u8)p0_filter8, flat); + q0_out = __msa_bmnz_v(q0_out, (v16u8)q0_filter8, flat); + q1_out = __msa_bmnz_v(q1_out, (v16u8)q1_filter8, flat); + q2_out = __msa_bmnz_v(q2, (v16u8)q2_filter8, flat); + + /* load 16 vector elements */ + LD_UB4((src - 8 * pitch), pitch, p7, p6, p5, p4); + LD_UB4(src + (4 * pitch), pitch, q4, q5, q6, q7); + + AOM_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + + if (__msa_test_bz_v(flat2)) { + p2_d = __msa_copy_u_d((v2i64)p2_out, 0); + p1_d = __msa_copy_u_d((v2i64)p1_out, 0); + p0_d = __msa_copy_u_d((v2i64)p0_out, 0); + q0_d = __msa_copy_u_d((v2i64)q0_out, 0); + q1_d = __msa_copy_u_d((v2i64)q1_out, 0); + q2_d = __msa_copy_u_d((v2i64)q2_out, 0); + + SD4(p2_d, p1_d, p0_d, q0_d, src - 3 * pitch, pitch); + SD(q1_d, src + pitch); + SD(q2_d, src + 2 * pitch); + } else { + /* LSB(right) 8 pixel operation */ + ILVR_B8_UH(zero, p7, zero, p6, zero, p5, zero, p4, zero, q4, zero, q5, + zero, q6, zero, q7, p7_r, p6_r, p5_r, p4_r, q4_r, q5_r, q6_r, + q7_r); + + tmp0 = p7_r << 3; + tmp0 -= p7_r; + tmp0 += p6_r; + tmp0 += q0_r; + + src -= 7 * pitch; + + /* calculation of p6 and p5 */ + tmp1 = p6_r + p5_r + p4_r + p3_r; + tmp1 += (p2_r + p1_r + p0_r); + tmp1 += tmp0; + p0_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + tmp0 = p5_r - p6_r + q1_r - p7_r; + tmp1 += tmp0; + p1_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + PCKEV_B2_UB(zero, p0_filter16, zero, p1_filter16, p0_filter16, + p1_filter16); + p0_filter16 = __msa_bmnz_v(p6, p0_filter16, flat2); + p1_filter16 = __msa_bmnz_v(p5, p1_filter16, flat2); + dword0 = __msa_copy_u_d((v2i64)p0_filter16, 0); + dword1 = __msa_copy_u_d((v2i64)p1_filter16, 0); + SD(dword0, src); + src += pitch; + SD(dword1, src); + src += pitch; + + /* calculation of p4 and p3 */ + tmp0 = p4_r - p5_r + q2_r - p7_r; + tmp2 = p3_r - p4_r + q3_r - p7_r; + tmp1 += tmp0; + p0_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + tmp1 += tmp2; + p1_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + PCKEV_B2_UB(zero, p0_filter16, zero, p1_filter16, p0_filter16, + p1_filter16); + p0_filter16 = __msa_bmnz_v(p4, p0_filter16, flat2); + p1_filter16 = __msa_bmnz_v(p3, p1_filter16, flat2); + dword0 = __msa_copy_u_d((v2i64)p0_filter16, 0); + dword1 = __msa_copy_u_d((v2i64)p1_filter16, 0); + SD(dword0, src); + src += pitch; + SD(dword1, src); + src += pitch; + + /* calculation of p2 and p1 */ + tmp0 = p2_r - p3_r + q4_r - p7_r; + tmp2 = p1_r - p2_r + q5_r - p7_r; + tmp1 += tmp0; + p0_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + tmp1 += tmp2; + p1_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + PCKEV_B2_UB(zero, p0_filter16, zero, p1_filter16, p0_filter16, + p1_filter16); + p0_filter16 = __msa_bmnz_v(p2_out, p0_filter16, flat2); + p1_filter16 = __msa_bmnz_v(p1_out, p1_filter16, flat2); + dword0 = __msa_copy_u_d((v2i64)p0_filter16, 0); + dword1 = __msa_copy_u_d((v2i64)p1_filter16, 0); + SD(dword0, src); + src += pitch; + SD(dword1, src); + src += pitch; + + /* calculation of p0 and q0 */ + tmp0 = (p0_r - p1_r) + (q6_r - p7_r); + tmp2 = (q7_r - p0_r) + (q0_r - p7_r); + tmp1 += tmp0; + p0_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + tmp1 += tmp2; + p1_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + PCKEV_B2_UB(zero, p0_filter16, zero, p1_filter16, p0_filter16, + p1_filter16); + p0_filter16 = __msa_bmnz_v(p0_out, p0_filter16, flat2); + p1_filter16 = __msa_bmnz_v(q0_out, p1_filter16, flat2); + dword0 = __msa_copy_u_d((v2i64)p0_filter16, 0); + dword1 = __msa_copy_u_d((v2i64)p1_filter16, 0); + SD(dword0, src); + src += pitch; + SD(dword1, src); + src += pitch; + + /* calculation of q1 and q2 */ + tmp0 = q7_r - q0_r + q1_r - p6_r; + tmp2 = q7_r - q1_r + q2_r - p5_r; + tmp1 += tmp0; + p0_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + tmp1 += tmp2; + p1_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + PCKEV_B2_UB(zero, p0_filter16, zero, p1_filter16, p0_filter16, + p1_filter16); + p0_filter16 = __msa_bmnz_v(q1_out, p0_filter16, flat2); + p1_filter16 = __msa_bmnz_v(q2_out, p1_filter16, flat2); + dword0 = __msa_copy_u_d((v2i64)p0_filter16, 0); + dword1 = __msa_copy_u_d((v2i64)p1_filter16, 0); + SD(dword0, src); + src += pitch; + SD(dword1, src); + src += pitch; + + /* calculation of q3 and q4 */ + tmp0 = (q7_r - q2_r) + (q3_r - p4_r); + tmp2 = (q7_r - q3_r) + (q4_r - p3_r); + tmp1 += tmp0; + p0_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + tmp1 += tmp2; + p1_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + PCKEV_B2_UB(zero, p0_filter16, zero, p1_filter16, p0_filter16, + p1_filter16); + p0_filter16 = __msa_bmnz_v(q3, p0_filter16, flat2); + p1_filter16 = __msa_bmnz_v(q4, p1_filter16, flat2); + dword0 = __msa_copy_u_d((v2i64)p0_filter16, 0); + dword1 = __msa_copy_u_d((v2i64)p1_filter16, 0); + SD(dword0, src); + src += pitch; + SD(dword1, src); + src += pitch; + + /* calculation of q5 and q6 */ + tmp0 = (q7_r - q4_r) + (q5_r - p2_r); + tmp2 = (q7_r - q5_r) + (q6_r - p1_r); + tmp1 += tmp0; + p0_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + tmp1 += tmp2; + p1_filter16 = (v16u8)__msa_srari_h((v8i16)tmp1, 4); + PCKEV_B2_UB(zero, p0_filter16, zero, p1_filter16, p0_filter16, + p1_filter16); + p0_filter16 = __msa_bmnz_v(q5, p0_filter16, flat2); + p1_filter16 = __msa_bmnz_v(q6, p1_filter16, flat2); + dword0 = __msa_copy_u_d((v2i64)p0_filter16, 0); + dword1 = __msa_copy_u_d((v2i64)p1_filter16, 0); + SD(dword0, src); + src += pitch; + SD(dword1, src); + } + } + } else { + aom_lpf_horizontal_16_dual_msa(src, pitch, b_limit_ptr, limit_ptr, + thresh_ptr, count); + } +} + +void aom_lpf_horizontal_edge_8_msa(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + mb_lpf_horizontal_edge(src, pitch, b_limit_ptr, limit_ptr, thresh_ptr, 1); +} + +void aom_lpf_horizontal_edge_16_msa(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + mb_lpf_horizontal_edge(src, pitch, b_limit_ptr, limit_ptr, thresh_ptr, 2); +} + +static void transpose_16x8_to_8x16(uint8_t *input, int32_t in_pitch, + uint8_t *output, int32_t out_pitch) { + v16u8 p7_org, p6_org, p5_org, p4_org, p3_org, p2_org, p1_org, p0_org; + v16i8 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + v16u8 p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + + LD_UB8(input, in_pitch, p7_org, p6_org, p5_org, p4_org, p3_org, p2_org, + p1_org, p0_org); + /* 8x8 transpose */ + TRANSPOSE8x8_UB_UB(p7_org, p6_org, p5_org, p4_org, p3_org, p2_org, p1_org, + p0_org, p7, p6, p5, p4, p3, p2, p1, p0); + /* 8x8 transpose */ + ILVL_B4_SB(p5_org, p7_org, p4_org, p6_org, p1_org, p3_org, p0_org, p2_org, + tmp0, tmp1, tmp2, tmp3); + ILVR_B2_SB(tmp1, tmp0, tmp3, tmp2, tmp4, tmp6); + ILVL_B2_SB(tmp1, tmp0, tmp3, tmp2, tmp5, tmp7); + ILVR_W2_UB(tmp6, tmp4, tmp7, tmp5, q0, q4); + ILVL_W2_UB(tmp6, tmp4, tmp7, tmp5, q2, q6); + SLDI_B4_0_UB(q0, q2, q4, q6, q1, q3, q5, q7, 8); + + ST_UB8(p7, p6, p5, p4, p3, p2, p1, p0, output, out_pitch); + output += (8 * out_pitch); + ST_UB8(q0, q1, q2, q3, q4, q5, q6, q7, output, out_pitch); +} + +static void transpose_8x16_to_16x8(uint8_t *input, int32_t in_pitch, + uint8_t *output, int32_t out_pitch) { + v16u8 p7_o, p6_o, p5_o, p4_o, p3_o, p2_o, p1_o, p0_o; + v16u8 p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + + LD_UB8(input, in_pitch, p7, p6, p5, p4, p3, p2, p1, p0); + LD_UB8(input + (8 * in_pitch), in_pitch, q0, q1, q2, q3, q4, q5, q6, q7); + TRANSPOSE16x8_UB_UB(p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, + q6, q7, p7_o, p6_o, p5_o, p4_o, p3_o, p2_o, p1_o, p0_o); + ST_UB8(p7_o, p6_o, p5_o, p4_o, p3_o, p2_o, p1_o, p0_o, output, out_pitch); +} + +static void transpose_16x16(uint8_t *input, int32_t in_pitch, uint8_t *output, + int32_t out_pitch) { + v16u8 row0, row1, row2, row3, row4, row5, row6, row7; + v16u8 row8, row9, row10, row11, row12, row13, row14, row15; + v16u8 p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + v8i16 tmp0, tmp1, tmp4, tmp5, tmp6, tmp7; + v4i32 tmp2, tmp3; + + LD_UB8(input, in_pitch, row0, row1, row2, row3, row4, row5, row6, row7); + input += (8 * in_pitch); + LD_UB8(input, in_pitch, row8, row9, row10, row11, row12, row13, row14, row15); + + TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p7, p6, + p5, p4, p3, p2, p1, p0); + + /* transpose 16x8 matrix into 8x16 */ + /* total 8 intermediate register and 32 instructions */ + q7 = (v16u8)__msa_ilvod_d((v2i64)row8, (v2i64)row0); + q6 = (v16u8)__msa_ilvod_d((v2i64)row9, (v2i64)row1); + q5 = (v16u8)__msa_ilvod_d((v2i64)row10, (v2i64)row2); + q4 = (v16u8)__msa_ilvod_d((v2i64)row11, (v2i64)row3); + q3 = (v16u8)__msa_ilvod_d((v2i64)row12, (v2i64)row4); + q2 = (v16u8)__msa_ilvod_d((v2i64)row13, (v2i64)row5); + q1 = (v16u8)__msa_ilvod_d((v2i64)row14, (v2i64)row6); + q0 = (v16u8)__msa_ilvod_d((v2i64)row15, (v2i64)row7); + + ILVEV_B2_SH(q7, q6, q5, q4, tmp0, tmp1); + tmp4 = (v8i16)__msa_ilvod_b((v16i8)q6, (v16i8)q7); + tmp5 = (v8i16)__msa_ilvod_b((v16i8)q4, (v16i8)q5); + + ILVEV_B2_UB(q3, q2, q1, q0, q5, q7); + tmp6 = (v8i16)__msa_ilvod_b((v16i8)q2, (v16i8)q3); + tmp7 = (v8i16)__msa_ilvod_b((v16i8)q0, (v16i8)q1); + + ILVEV_H2_SW(tmp0, tmp1, q5, q7, tmp2, tmp3); + q0 = (v16u8)__msa_ilvev_w(tmp3, tmp2); + q4 = (v16u8)__msa_ilvod_w(tmp3, tmp2); + + tmp2 = (v4i32)__msa_ilvod_h(tmp1, tmp0); + tmp3 = (v4i32)__msa_ilvod_h((v8i16)q7, (v8i16)q5); + q2 = (v16u8)__msa_ilvev_w(tmp3, tmp2); + q6 = (v16u8)__msa_ilvod_w(tmp3, tmp2); + + ILVEV_H2_SW(tmp4, tmp5, tmp6, tmp7, tmp2, tmp3); + q1 = (v16u8)__msa_ilvev_w(tmp3, tmp2); + q5 = (v16u8)__msa_ilvod_w(tmp3, tmp2); + + tmp2 = (v4i32)__msa_ilvod_h(tmp5, tmp4); + tmp3 = (v4i32)__msa_ilvod_h(tmp7, tmp6); + q3 = (v16u8)__msa_ilvev_w(tmp3, tmp2); + q7 = (v16u8)__msa_ilvod_w(tmp3, tmp2); + + ST_UB8(p7, p6, p5, p4, p3, p2, p1, p0, output, out_pitch); + output += (8 * out_pitch); + ST_UB8(q0, q1, q2, q3, q4, q5, q6, q7, output, out_pitch); +} + +int32_t aom_vt_lpf_t4_and_t8_8w(uint8_t *src, uint8_t *filter48, + uint8_t *src_org, int32_t pitch_org, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + v16u8 p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + v16u8 flat, mask, hev, thresh, b_limit, limit; + v8u16 p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r; + v8i16 p2_filt8_r, p1_filt8_r, p0_filt8_r, q0_filt8_r, q1_filt8_r, q2_filt8_r; + v16i8 zero = { 0 }; + v8i16 vec0, vec1, vec2, vec3; + + /* load vector elements */ + LD_UB8(src - (4 * 16), 16, p3, p2, p1, p0, q0, q1, q2, q3); + + thresh = (v16u8)__msa_fill_b(*thresh_ptr); + b_limit = (v16u8)__msa_fill_b(*b_limit_ptr); + limit = (v16u8)__msa_fill_b(*limit_ptr); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + /* flat4 */ + AOM_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + AOM_LPF_FILTER4_8W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + flat = (v16u8)__msa_ilvr_d((v2i64)zero, (v2i64)flat); + + if (__msa_test_bz_v(flat)) { + ILVR_B2_SH(p0_out, p1_out, q1_out, q0_out, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec2, vec3); + ST4x8_UB(vec2, vec3, (src_org - 2), pitch_org); + return 1; + } else { + ILVR_B8_UH(zero, p3, zero, p2, zero, p1, zero, p0, zero, q0, zero, q1, zero, + q2, zero, q3, p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r); + AOM_FILTER8(p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r, p2_filt8_r, + p1_filt8_r, p0_filt8_r, q0_filt8_r, q1_filt8_r, q2_filt8_r); + + /* convert 16 bit output data into 8 bit */ + p2_r = (v8u16)__msa_pckev_b((v16i8)p2_filt8_r, (v16i8)p2_filt8_r); + p1_r = (v8u16)__msa_pckev_b((v16i8)p1_filt8_r, (v16i8)p1_filt8_r); + p0_r = (v8u16)__msa_pckev_b((v16i8)p0_filt8_r, (v16i8)p0_filt8_r); + q0_r = (v8u16)__msa_pckev_b((v16i8)q0_filt8_r, (v16i8)q0_filt8_r); + q1_r = (v8u16)__msa_pckev_b((v16i8)q1_filt8_r, (v16i8)q1_filt8_r); + q2_r = (v8u16)__msa_pckev_b((v16i8)q2_filt8_r, (v16i8)q2_filt8_r); + + /* store pixel values */ + p2_out = __msa_bmnz_v(p2, (v16u8)p2_r, flat); + p1_out = __msa_bmnz_v(p1_out, (v16u8)p1_r, flat); + p0_out = __msa_bmnz_v(p0_out, (v16u8)p0_r, flat); + q0_out = __msa_bmnz_v(q0_out, (v16u8)q0_r, flat); + q1_out = __msa_bmnz_v(q1_out, (v16u8)q1_r, flat); + q2_out = __msa_bmnz_v(q2, (v16u8)q2_r, flat); + + ST_UB4(p2_out, p1_out, p0_out, q0_out, filter48, 16); + filter48 += (4 * 16); + ST_UB2(q1_out, q2_out, filter48, 16); + filter48 += (2 * 16); + ST_UB(flat, filter48); + + return 0; + } +} + +int32_t aom_vt_lpf_t16_8w(uint8_t *src, uint8_t *src_org, int32_t pitch, + uint8_t *filter48) { + v16i8 zero = { 0 }; + v16u8 filter8, flat, flat2; + v16u8 p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + v8u16 p7_r_in, p6_r_in, p5_r_in, p4_r_in, p3_r_in, p2_r_in, p1_r_in, p0_r_in; + v8u16 q7_r_in, q6_r_in, q5_r_in, q4_r_in, q3_r_in, q2_r_in, q1_r_in, q0_r_in; + v8u16 tmp0_r, tmp1_r; + v8i16 r_out; + + flat = LD_UB(filter48 + 6 * 16); + + LD_UB8((src - 8 * 16), 16, p7, p6, p5, p4, p3, p2, p1, p0); + LD_UB8(src, 16, q0, q1, q2, q3, q4, q5, q6, q7); + + AOM_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + + if (__msa_test_bz_v(flat2)) { + v8i16 vec0, vec1, vec2, vec3, vec4; + + LD_UB4(filter48, 16, p2, p1, p0, q0); + LD_UB2(filter48 + 4 * 16, 16, q1, q2); + + ILVR_B2_SH(p1, p2, q0, p0, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec3, vec4); + vec2 = (v8i16)__msa_ilvr_b((v16i8)q2, (v16i8)q1); + + src_org -= 3; + ST4x4_UB(vec3, vec3, 0, 1, 2, 3, src_org, pitch); + ST2x4_UB(vec2, 0, (src_org + 4), pitch); + src_org += (4 * pitch); + ST4x4_UB(vec4, vec4, 0, 1, 2, 3, src_org, pitch); + ST2x4_UB(vec2, 4, (src_org + 4), pitch); + + return 1; + } else { + src -= 7 * 16; + + ILVR_B8_UH(zero, p7, zero, p6, zero, p5, zero, p4, zero, p3, zero, p2, zero, + p1, zero, p0, p7_r_in, p6_r_in, p5_r_in, p4_r_in, p3_r_in, + p2_r_in, p1_r_in, p0_r_in); + q0_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q0); + + tmp0_r = p7_r_in << 3; + tmp0_r -= p7_r_in; + tmp0_r += p6_r_in; + tmp0_r += q0_r_in; + tmp1_r = p6_r_in + p5_r_in; + tmp1_r += p4_r_in; + tmp1_r += p3_r_in; + tmp1_r += p2_r_in; + tmp1_r += p1_r_in; + tmp1_r += p0_r_in; + tmp1_r += tmp0_r; + + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + p6 = __msa_bmnz_v(p6, (v16u8)r_out, flat2); + ST8x1_UB(p6, src); + src += 16; + + /* p5 */ + q1_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q1); + tmp0_r = p5_r_in - p6_r_in; + tmp0_r += q1_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + p5 = __msa_bmnz_v(p5, (v16u8)r_out, flat2); + ST8x1_UB(p5, src); + src += 16; + + /* p4 */ + q2_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q2); + tmp0_r = p4_r_in - p5_r_in; + tmp0_r += q2_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + p4 = __msa_bmnz_v(p4, (v16u8)r_out, flat2); + ST8x1_UB(p4, src); + src += 16; + + /* p3 */ + q3_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q3); + tmp0_r = p3_r_in - p4_r_in; + tmp0_r += q3_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + p3 = __msa_bmnz_v(p3, (v16u8)r_out, flat2); + ST8x1_UB(p3, src); + src += 16; + + /* p2 */ + q4_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q4); + filter8 = LD_UB(filter48); + tmp0_r = p2_r_in - p3_r_in; + tmp0_r += q4_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST8x1_UB(filter8, src); + src += 16; + + /* p1 */ + q5_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q5); + filter8 = LD_UB(filter48 + 16); + tmp0_r = p1_r_in - p2_r_in; + tmp0_r += q5_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST8x1_UB(filter8, src); + src += 16; + + /* p0 */ + q6_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q6); + filter8 = LD_UB(filter48 + 32); + tmp0_r = p0_r_in - p1_r_in; + tmp0_r += q6_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST8x1_UB(filter8, src); + src += 16; + + /* q0 */ + q7_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q7); + filter8 = LD_UB(filter48 + 48); + tmp0_r = q7_r_in - p0_r_in; + tmp0_r += q0_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST8x1_UB(filter8, src); + src += 16; + + /* q1 */ + filter8 = LD_UB(filter48 + 64); + tmp0_r = q7_r_in - q0_r_in; + tmp0_r += q1_r_in; + tmp0_r -= p6_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST8x1_UB(filter8, src); + src += 16; + + /* q2 */ + filter8 = LD_UB(filter48 + 80); + tmp0_r = q7_r_in - q1_r_in; + tmp0_r += q2_r_in; + tmp0_r -= p5_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST8x1_UB(filter8, src); + src += 16; + + /* q3 */ + tmp0_r = q7_r_in - q2_r_in; + tmp0_r += q3_r_in; + tmp0_r -= p4_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + q3 = __msa_bmnz_v(q3, (v16u8)r_out, flat2); + ST8x1_UB(q3, src); + src += 16; + + /* q4 */ + tmp0_r = q7_r_in - q3_r_in; + tmp0_r += q4_r_in; + tmp0_r -= p3_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + q4 = __msa_bmnz_v(q4, (v16u8)r_out, flat2); + ST8x1_UB(q4, src); + src += 16; + + /* q5 */ + tmp0_r = q7_r_in - q4_r_in; + tmp0_r += q5_r_in; + tmp0_r -= p2_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + q5 = __msa_bmnz_v(q5, (v16u8)r_out, flat2); + ST8x1_UB(q5, src); + src += 16; + + /* q6 */ + tmp0_r = q7_r_in - q5_r_in; + tmp0_r += q6_r_in; + tmp0_r -= p1_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)r_out, (v16i8)r_out); + q6 = __msa_bmnz_v(q6, (v16u8)r_out, flat2); + ST8x1_UB(q6, src); + + return 0; + } +} + +void aom_lpf_vertical_16_msa(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + uint8_t early_exit = 0; + DECLARE_ALIGNED(32, uint8_t, transposed_input[16 * 24]); + uint8_t *filter48 = &transposed_input[16 * 16]; + + transpose_16x8_to_8x16(src - 8, pitch, transposed_input, 16); + + early_exit = + aom_vt_lpf_t4_and_t8_8w((transposed_input + 16 * 8), &filter48[0], src, + pitch, b_limit_ptr, limit_ptr, thresh_ptr); + + if (0 == early_exit) { + early_exit = aom_vt_lpf_t16_8w((transposed_input + 16 * 8), src, pitch, + &filter48[0]); + + if (0 == early_exit) { + transpose_8x16_to_16x8(transposed_input, 16, src - 8, pitch); + } + } +} + +int32_t aom_vt_lpf_t4_and_t8_16w(uint8_t *src, uint8_t *filter48, + uint8_t *src_org, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + v16u8 p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + v16u8 flat, mask, hev, thresh, b_limit, limit; + v8u16 p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r; + v8u16 p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + v8i16 p2_filt8_r, p1_filt8_r, p0_filt8_r, q0_filt8_r, q1_filt8_r, q2_filt8_r; + v8i16 p2_filt8_l, p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l; + v16i8 zero = { 0 }; + v8i16 vec0, vec1, vec2, vec3, vec4, vec5; + + /* load vector elements */ + LD_UB8(src - (4 * 16), 16, p3, p2, p1, p0, q0, q1, q2, q3); + + thresh = (v16u8)__msa_fill_b(*thresh_ptr); + b_limit = (v16u8)__msa_fill_b(*b_limit_ptr); + limit = (v16u8)__msa_fill_b(*limit_ptr); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + /* flat4 */ + AOM_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + AOM_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + if (__msa_test_bz_v(flat)) { + ILVR_B2_SH(p0_out, p1_out, q1_out, q0_out, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec2, vec3); + ILVL_B2_SH(p0_out, p1_out, q1_out, q0_out, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec4, vec5); + + src_org -= 2; + ST4x8_UB(vec2, vec3, src_org, pitch); + src_org += 8 * pitch; + ST4x8_UB(vec4, vec5, src_org, pitch); + + return 1; + } else { + ILVR_B8_UH(zero, p3, zero, p2, zero, p1, zero, p0, zero, q0, zero, q1, zero, + q2, zero, q3, p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r); + AOM_FILTER8(p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r, p2_filt8_r, + p1_filt8_r, p0_filt8_r, q0_filt8_r, q1_filt8_r, q2_filt8_r); + ILVL_B4_UH(zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, p1_l, p0_l); + ILVL_B4_UH(zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, q2_l, q3_l); + AOM_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + /* convert 16 bit output data into 8 bit */ + PCKEV_B4_SH(p2_filt8_l, p2_filt8_r, p1_filt8_l, p1_filt8_r, p0_filt8_l, + p0_filt8_r, q0_filt8_l, q0_filt8_r, p2_filt8_r, p1_filt8_r, + p0_filt8_r, q0_filt8_r); + PCKEV_B2_SH(q1_filt8_l, q1_filt8_r, q2_filt8_l, q2_filt8_r, q1_filt8_r, + q2_filt8_r); + + /* store pixel values */ + p2_out = __msa_bmnz_v(p2, (v16u8)p2_filt8_r, flat); + p1_out = __msa_bmnz_v(p1_out, (v16u8)p1_filt8_r, flat); + p0_out = __msa_bmnz_v(p0_out, (v16u8)p0_filt8_r, flat); + q0_out = __msa_bmnz_v(q0_out, (v16u8)q0_filt8_r, flat); + q1_out = __msa_bmnz_v(q1_out, (v16u8)q1_filt8_r, flat); + q2_out = __msa_bmnz_v(q2, (v16u8)q2_filt8_r, flat); + + ST_UB4(p2_out, p1_out, p0_out, q0_out, filter48, 16); + filter48 += (4 * 16); + ST_UB2(q1_out, q2_out, filter48, 16); + filter48 += (2 * 16); + ST_UB(flat, filter48); + + return 0; + } +} + +int32_t aom_vt_lpf_t16_16w(uint8_t *src, uint8_t *src_org, int32_t pitch, + uint8_t *filter48) { + v16u8 flat, flat2, filter8; + v16i8 zero = { 0 }; + v16u8 p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + v8u16 p7_r_in, p6_r_in, p5_r_in, p4_r_in, p3_r_in, p2_r_in, p1_r_in, p0_r_in; + v8u16 q7_r_in, q6_r_in, q5_r_in, q4_r_in, q3_r_in, q2_r_in, q1_r_in, q0_r_in; + v8u16 p7_l_in, p6_l_in, p5_l_in, p4_l_in, p3_l_in, p2_l_in, p1_l_in, p0_l_in; + v8u16 q7_l_in, q6_l_in, q5_l_in, q4_l_in, q3_l_in, q2_l_in, q1_l_in, q0_l_in; + v8u16 tmp0_r, tmp1_r, tmp0_l, tmp1_l; + v8i16 l_out, r_out; + + flat = LD_UB(filter48 + 6 * 16); + + LD_UB8((src - 8 * 16), 16, p7, p6, p5, p4, p3, p2, p1, p0); + LD_UB8(src, 16, q0, q1, q2, q3, q4, q5, q6, q7); + + AOM_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + + if (__msa_test_bz_v(flat2)) { + v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + + LD_UB4(filter48, 16, p2, p1, p0, q0); + LD_UB2(filter48 + 4 * 16, 16, q1, q2); + + ILVR_B2_SH(p1, p2, q0, p0, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec3, vec4); + ILVL_B2_SH(p1, p2, q0, p0, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec6, vec7); + ILVRL_B2_SH(q2, q1, vec2, vec5); + + src_org -= 3; + ST4x4_UB(vec3, vec3, 0, 1, 2, 3, src_org, pitch); + ST2x4_UB(vec2, 0, (src_org + 4), pitch); + src_org += (4 * pitch); + ST4x4_UB(vec4, vec4, 0, 1, 2, 3, src_org, pitch); + ST2x4_UB(vec2, 4, (src_org + 4), pitch); + src_org += (4 * pitch); + ST4x4_UB(vec6, vec6, 0, 1, 2, 3, src_org, pitch); + ST2x4_UB(vec5, 0, (src_org + 4), pitch); + src_org += (4 * pitch); + ST4x4_UB(vec7, vec7, 0, 1, 2, 3, src_org, pitch); + ST2x4_UB(vec5, 4, (src_org + 4), pitch); + + return 1; + } else { + src -= 7 * 16; + + ILVR_B8_UH(zero, p7, zero, p6, zero, p5, zero, p4, zero, p3, zero, p2, zero, + p1, zero, p0, p7_r_in, p6_r_in, p5_r_in, p4_r_in, p3_r_in, + p2_r_in, p1_r_in, p0_r_in); + q0_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q0); + + tmp0_r = p7_r_in << 3; + tmp0_r -= p7_r_in; + tmp0_r += p6_r_in; + tmp0_r += q0_r_in; + tmp1_r = p6_r_in + p5_r_in; + tmp1_r += p4_r_in; + tmp1_r += p3_r_in; + tmp1_r += p2_r_in; + tmp1_r += p1_r_in; + tmp1_r += p0_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + + ILVL_B4_UH(zero, p7, zero, p6, zero, p5, zero, p4, p7_l_in, p6_l_in, + p5_l_in, p4_l_in); + ILVL_B4_UH(zero, p3, zero, p2, zero, p1, zero, p0, p3_l_in, p2_l_in, + p1_l_in, p0_l_in); + q0_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q0); + + tmp0_l = p7_l_in << 3; + tmp0_l -= p7_l_in; + tmp0_l += p6_l_in; + tmp0_l += q0_l_in; + tmp1_l = p6_l_in + p5_l_in; + tmp1_l += p4_l_in; + tmp1_l += p3_l_in; + tmp1_l += p2_l_in; + tmp1_l += p1_l_in; + tmp1_l += p0_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + p6 = __msa_bmnz_v(p6, (v16u8)r_out, flat2); + ST_UB(p6, src); + src += 16; + + /* p5 */ + q1_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q1); + tmp0_r = p5_r_in - p6_r_in; + tmp0_r += q1_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + q1_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q1); + tmp0_l = p5_l_in - p6_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + p5 = __msa_bmnz_v(p5, (v16u8)r_out, flat2); + ST_UB(p5, src); + src += 16; + + /* p4 */ + q2_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q2); + tmp0_r = p4_r_in - p5_r_in; + tmp0_r += q2_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + q2_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q2); + tmp0_l = p4_l_in - p5_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + p4 = __msa_bmnz_v(p4, (v16u8)r_out, flat2); + ST_UB(p4, src); + src += 16; + + /* p3 */ + q3_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q3); + tmp0_r = p3_r_in - p4_r_in; + tmp0_r += q3_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + q3_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q3); + tmp0_l = p3_l_in - p4_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + p3 = __msa_bmnz_v(p3, (v16u8)r_out, flat2); + ST_UB(p3, src); + src += 16; + + /* p2 */ + q4_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q4); + filter8 = LD_UB(filter48); + tmp0_r = p2_r_in - p3_r_in; + tmp0_r += q4_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + q4_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q4); + tmp0_l = p2_l_in - p3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST_UB(filter8, src); + src += 16; + + /* p1 */ + q5_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q5); + filter8 = LD_UB(filter48 + 16); + tmp0_r = p1_r_in - p2_r_in; + tmp0_r += q5_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + q5_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q5); + tmp0_l = p1_l_in - p2_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)(tmp1_l), 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST_UB(filter8, src); + src += 16; + + /* p0 */ + q6_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q6); + filter8 = LD_UB(filter48 + 32); + tmp0_r = p0_r_in - p1_r_in; + tmp0_r += q6_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + q6_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q6); + tmp0_l = p0_l_in - p1_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST_UB(filter8, src); + src += 16; + + /* q0 */ + q7_r_in = (v8u16)__msa_ilvr_b(zero, (v16i8)q7); + filter8 = LD_UB(filter48 + 48); + tmp0_r = q7_r_in - p0_r_in; + tmp0_r += q0_r_in; + tmp0_r -= p7_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + q7_l_in = (v8u16)__msa_ilvl_b(zero, (v16i8)q7); + tmp0_l = q7_l_in - p0_l_in; + tmp0_l += q0_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST_UB(filter8, src); + src += 16; + + /* q1 */ + filter8 = LD_UB(filter48 + 64); + tmp0_r = q7_r_in - q0_r_in; + tmp0_r += q1_r_in; + tmp0_r -= p6_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + tmp0_l = q7_l_in - q0_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p6_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST_UB(filter8, src); + src += 16; + + /* q2 */ + filter8 = LD_UB(filter48 + 80); + tmp0_r = q7_r_in - q1_r_in; + tmp0_r += q2_r_in; + tmp0_r -= p5_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + tmp0_l = q7_l_in - q1_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p5_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + filter8 = __msa_bmnz_v(filter8, (v16u8)r_out, flat2); + ST_UB(filter8, src); + src += 16; + + /* q3 */ + tmp0_r = q7_r_in - q2_r_in; + tmp0_r += q3_r_in; + tmp0_r -= p4_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + tmp0_l = q7_l_in - q2_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p4_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + q3 = __msa_bmnz_v(q3, (v16u8)r_out, flat2); + ST_UB(q3, src); + src += 16; + + /* q4 */ + tmp0_r = q7_r_in - q3_r_in; + tmp0_r += q4_r_in; + tmp0_r -= p3_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + tmp0_l = q7_l_in - q3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p3_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + q4 = __msa_bmnz_v(q4, (v16u8)r_out, flat2); + ST_UB(q4, src); + src += 16; + + /* q5 */ + tmp0_r = q7_r_in - q4_r_in; + tmp0_r += q5_r_in; + tmp0_r -= p2_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + tmp0_l = q7_l_in - q4_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p2_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + q5 = __msa_bmnz_v(q5, (v16u8)r_out, flat2); + ST_UB(q5, src); + src += 16; + + /* q6 */ + tmp0_r = q7_r_in - q5_r_in; + tmp0_r += q6_r_in; + tmp0_r -= p1_r_in; + tmp1_r += tmp0_r; + r_out = __msa_srari_h((v8i16)tmp1_r, 4); + tmp0_l = q7_l_in - q5_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p1_l_in; + tmp1_l += tmp0_l; + l_out = __msa_srari_h((v8i16)tmp1_l, 4); + r_out = (v8i16)__msa_pckev_b((v16i8)l_out, (v16i8)r_out); + q6 = __msa_bmnz_v(q6, (v16u8)r_out, flat2); + ST_UB(q6, src); + + return 0; + } +} + +void aom_lpf_vertical_16_dual_msa(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + uint8_t early_exit = 0; + DECLARE_ALIGNED(32, uint8_t, transposed_input[16 * 24]); + uint8_t *filter48 = &transposed_input[16 * 16]; + + transpose_16x16((src - 8), pitch, &transposed_input[0], 16); + + early_exit = + aom_vt_lpf_t4_and_t8_16w((transposed_input + 16 * 8), &filter48[0], src, + pitch, b_limit_ptr, limit_ptr, thresh_ptr); + + if (0 == early_exit) { + early_exit = aom_vt_lpf_t16_16w((transposed_input + 16 * 8), src, pitch, + &filter48[0]); + + if (0 == early_exit) { + transpose_16x16(transposed_input, 16, (src - 8), pitch); + } + } +} diff --git a/third_party/aom/aom_dsp/mips/loopfilter_4_msa.c b/third_party/aom/aom_dsp/mips/loopfilter_4_msa.c new file mode 100644 index 0000000000..dc0a977645 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/loopfilter_4_msa.c @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/loopfilter_msa.h" + +void aom_lpf_horizontal_4_msa(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + uint64_t p1_d, p0_d, q0_d, q1_d; + v16u8 mask, hev, flat, thresh, b_limit, limit; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0, p1_out, p0_out, q0_out, q1_out; + + /* load vector elements */ + LD_UB8((src - 4 * pitch), pitch, p3, p2, p1, p0, q0, q1, q2, q3); + + thresh = (v16u8)__msa_fill_b(*thresh_ptr); + b_limit = (v16u8)__msa_fill_b(*b_limit_ptr); + limit = (v16u8)__msa_fill_b(*limit_ptr); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + AOM_LPF_FILTER4_8W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + p1_d = __msa_copy_u_d((v2i64)p1_out, 0); + p0_d = __msa_copy_u_d((v2i64)p0_out, 0); + q0_d = __msa_copy_u_d((v2i64)q0_out, 0); + q1_d = __msa_copy_u_d((v2i64)q1_out, 0); + SD4(p1_d, p0_d, q0_d, q1_d, (src - 2 * pitch), pitch); +} + +void aom_lpf_horizontal_4_dual_msa(uint8_t *src, int32_t pitch, + const uint8_t *b_limit0_ptr, + const uint8_t *limit0_ptr, + const uint8_t *thresh0_ptr, + const uint8_t *b_limit1_ptr, + const uint8_t *limit1_ptr, + const uint8_t *thresh1_ptr) { + v16u8 mask, hev, flat, thresh0, b_limit0, limit0, thresh1, b_limit1, limit1; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + + /* load vector elements */ + LD_UB8((src - 4 * pitch), pitch, p3, p2, p1, p0, q0, q1, q2, q3); + + thresh0 = (v16u8)__msa_fill_b(*thresh0_ptr); + thresh1 = (v16u8)__msa_fill_b(*thresh1_ptr); + thresh0 = (v16u8)__msa_ilvr_d((v2i64)thresh1, (v2i64)thresh0); + + b_limit0 = (v16u8)__msa_fill_b(*b_limit0_ptr); + b_limit1 = (v16u8)__msa_fill_b(*b_limit1_ptr); + b_limit0 = (v16u8)__msa_ilvr_d((v2i64)b_limit1, (v2i64)b_limit0); + + limit0 = (v16u8)__msa_fill_b(*limit0_ptr); + limit1 = (v16u8)__msa_fill_b(*limit1_ptr); + limit0 = (v16u8)__msa_ilvr_d((v2i64)limit1, (v2i64)limit0); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, hev, + mask, flat); + AOM_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1); + + ST_UB4(p1, p0, q0, q1, (src - 2 * pitch), pitch); +} + +void aom_lpf_vertical_4_msa(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + v16u8 mask, hev, flat, limit, thresh, b_limit; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + v8i16 vec0, vec1, vec2, vec3; + + LD_UB8((src - 4), pitch, p3, p2, p1, p0, q0, q1, q2, q3); + + thresh = (v16u8)__msa_fill_b(*thresh_ptr); + b_limit = (v16u8)__msa_fill_b(*b_limit_ptr); + limit = (v16u8)__msa_fill_b(*limit_ptr); + + TRANSPOSE8x8_UB_UB(p3, p2, p1, p0, q0, q1, q2, q3, p3, p2, p1, p0, q0, q1, q2, + q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + AOM_LPF_FILTER4_8W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1); + ILVR_B2_SH(p0, p1, q1, q0, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec2, vec3); + + src -= 2; + ST4x4_UB(vec2, vec2, 0, 1, 2, 3, src, pitch); + src += 4 * pitch; + ST4x4_UB(vec3, vec3, 0, 1, 2, 3, src, pitch); +} + +void aom_lpf_vertical_4_dual_msa(uint8_t *src, int32_t pitch, + const uint8_t *b_limit0_ptr, + const uint8_t *limit0_ptr, + const uint8_t *thresh0_ptr, + const uint8_t *b_limit1_ptr, + const uint8_t *limit1_ptr, + const uint8_t *thresh1_ptr) { + v16u8 mask, hev, flat; + v16u8 thresh0, b_limit0, limit0, thresh1, b_limit1, limit1; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + v16u8 row0, row1, row2, row3, row4, row5, row6, row7; + v16u8 row8, row9, row10, row11, row12, row13, row14, row15; + v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + + LD_UB8(src - 4, pitch, row0, row1, row2, row3, row4, row5, row6, row7); + LD_UB8(src - 4 + (8 * pitch), pitch, row8, row9, row10, row11, row12, row13, + row14, row15); + + TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p3, p2, + p1, p0, q0, q1, q2, q3); + + thresh0 = (v16u8)__msa_fill_b(*thresh0_ptr); + thresh1 = (v16u8)__msa_fill_b(*thresh1_ptr); + thresh0 = (v16u8)__msa_ilvr_d((v2i64)thresh1, (v2i64)thresh0); + + b_limit0 = (v16u8)__msa_fill_b(*b_limit0_ptr); + b_limit1 = (v16u8)__msa_fill_b(*b_limit1_ptr); + b_limit0 = (v16u8)__msa_ilvr_d((v2i64)b_limit1, (v2i64)b_limit0); + + limit0 = (v16u8)__msa_fill_b(*limit0_ptr); + limit1 = (v16u8)__msa_fill_b(*limit1_ptr); + limit0 = (v16u8)__msa_ilvr_d((v2i64)limit1, (v2i64)limit0); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, hev, + mask, flat); + AOM_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1); + ILVR_B2_SH(p0, p1, q1, q0, tmp0, tmp1); + ILVRL_H2_SH(tmp1, tmp0, tmp2, tmp3); + ILVL_B2_SH(p0, p1, q1, q0, tmp0, tmp1); + ILVRL_H2_SH(tmp1, tmp0, tmp4, tmp5); + + src -= 2; + + ST4x8_UB(tmp2, tmp3, src, pitch); + src += (8 * pitch); + ST4x8_UB(tmp4, tmp5, src, pitch); +} diff --git a/third_party/aom/aom_dsp/mips/loopfilter_8_msa.c b/third_party/aom/aom_dsp/mips/loopfilter_8_msa.c new file mode 100644 index 0000000000..dc203e79cf --- /dev/null +++ b/third_party/aom/aom_dsp/mips/loopfilter_8_msa.c @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/mips/loopfilter_msa.h" + +void aom_lpf_horizontal_8_msa(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + uint64_t p2_d, p1_d, p0_d, q0_d, q1_d, q2_d; + v16u8 mask, hev, flat, thresh, b_limit, limit; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + v16u8 p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + v8i16 p2_filter8, p1_filter8, p0_filter8, q0_filter8, q1_filter8, q2_filter8; + v8u16 p3_r, p2_r, p1_r, p0_r, q3_r, q2_r, q1_r, q0_r; + v16i8 zero = { 0 }; + + /* load vector elements */ + LD_UB8((src - 4 * pitch), pitch, p3, p2, p1, p0, q0, q1, q2, q3); + + thresh = (v16u8)__msa_fill_b(*thresh_ptr); + b_limit = (v16u8)__msa_fill_b(*b_limit_ptr); + limit = (v16u8)__msa_fill_b(*limit_ptr); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + AOM_FLAT4(p3, p2, p0, q0, q2, q3, flat); + AOM_LPF_FILTER4_8W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + flat = (v16u8)__msa_ilvr_d((v2i64)zero, (v2i64)flat); + + if (__msa_test_bz_v(flat)) { + p1_d = __msa_copy_u_d((v2i64)p1_out, 0); + p0_d = __msa_copy_u_d((v2i64)p0_out, 0); + q0_d = __msa_copy_u_d((v2i64)q0_out, 0); + q1_d = __msa_copy_u_d((v2i64)q1_out, 0); + SD4(p1_d, p0_d, q0_d, q1_d, (src - 2 * pitch), pitch); + } else { + ILVR_B8_UH(zero, p3, zero, p2, zero, p1, zero, p0, zero, q0, zero, q1, zero, + q2, zero, q3, p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r); + AOM_FILTER8(p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r, p2_filter8, + p1_filter8, p0_filter8, q0_filter8, q1_filter8, q2_filter8); + + /* convert 16 bit output data into 8 bit */ + PCKEV_B4_SH(zero, p2_filter8, zero, p1_filter8, zero, p0_filter8, zero, + q0_filter8, p2_filter8, p1_filter8, p0_filter8, q0_filter8); + PCKEV_B2_SH(zero, q1_filter8, zero, q2_filter8, q1_filter8, q2_filter8); + + /* store pixel values */ + p2_out = __msa_bmnz_v(p2, (v16u8)p2_filter8, flat); + p1_out = __msa_bmnz_v(p1_out, (v16u8)p1_filter8, flat); + p0_out = __msa_bmnz_v(p0_out, (v16u8)p0_filter8, flat); + q0_out = __msa_bmnz_v(q0_out, (v16u8)q0_filter8, flat); + q1_out = __msa_bmnz_v(q1_out, (v16u8)q1_filter8, flat); + q2_out = __msa_bmnz_v(q2, (v16u8)q2_filter8, flat); + + p2_d = __msa_copy_u_d((v2i64)p2_out, 0); + p1_d = __msa_copy_u_d((v2i64)p1_out, 0); + p0_d = __msa_copy_u_d((v2i64)p0_out, 0); + q0_d = __msa_copy_u_d((v2i64)q0_out, 0); + q1_d = __msa_copy_u_d((v2i64)q1_out, 0); + q2_d = __msa_copy_u_d((v2i64)q2_out, 0); + + src -= 3 * pitch; + + SD4(p2_d, p1_d, p0_d, q0_d, src, pitch); + src += (4 * pitch); + SD(q1_d, src); + src += pitch; + SD(q2_d, src); + } +} + +void aom_lpf_horizontal_8_dual_msa( + uint8_t *src, int32_t pitch, const uint8_t *b_limit0, const uint8_t *limit0, + const uint8_t *thresh0, const uint8_t *b_limit1, const uint8_t *limit1, + const uint8_t *thresh1) { + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + v16u8 p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + v16u8 flat, mask, hev, tmp, thresh, b_limit, limit; + v8u16 p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r; + v8u16 p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + v8i16 p2_filt8_r, p1_filt8_r, p0_filt8_r, q0_filt8_r, q1_filt8_r, q2_filt8_r; + v8i16 p2_filt8_l, p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l; + v16u8 zero = { 0 }; + + /* load vector elements */ + LD_UB8(src - (4 * pitch), pitch, p3, p2, p1, p0, q0, q1, q2, q3); + + thresh = (v16u8)__msa_fill_b(*thresh0); + tmp = (v16u8)__msa_fill_b(*thresh1); + thresh = (v16u8)__msa_ilvr_d((v2i64)tmp, (v2i64)thresh); + + b_limit = (v16u8)__msa_fill_b(*b_limit0); + tmp = (v16u8)__msa_fill_b(*b_limit1); + b_limit = (v16u8)__msa_ilvr_d((v2i64)tmp, (v2i64)b_limit); + + limit = (v16u8)__msa_fill_b(*limit0); + tmp = (v16u8)__msa_fill_b(*limit1); + limit = (v16u8)__msa_ilvr_d((v2i64)tmp, (v2i64)limit); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + AOM_FLAT4(p3, p2, p0, q0, q2, q3, flat); + AOM_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + if (__msa_test_bz_v(flat)) { + ST_UB4(p1_out, p0_out, q0_out, q1_out, (src - 2 * pitch), pitch); + } else { + ILVR_B8_UH(zero, p3, zero, p2, zero, p1, zero, p0, zero, q0, zero, q1, zero, + q2, zero, q3, p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r); + AOM_FILTER8(p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r, p2_filt8_r, + p1_filt8_r, p0_filt8_r, q0_filt8_r, q1_filt8_r, q2_filt8_r); + + ILVL_B4_UH(zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, p1_l, p0_l); + ILVL_B4_UH(zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, q2_l, q3_l); + AOM_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + /* convert 16 bit output data into 8 bit */ + PCKEV_B4_SH(p2_filt8_l, p2_filt8_r, p1_filt8_l, p1_filt8_r, p0_filt8_l, + p0_filt8_r, q0_filt8_l, q0_filt8_r, p2_filt8_r, p1_filt8_r, + p0_filt8_r, q0_filt8_r); + PCKEV_B2_SH(q1_filt8_l, q1_filt8_r, q2_filt8_l, q2_filt8_r, q1_filt8_r, + q2_filt8_r); + + /* store pixel values */ + p2_out = __msa_bmnz_v(p2, (v16u8)p2_filt8_r, flat); + p1_out = __msa_bmnz_v(p1_out, (v16u8)p1_filt8_r, flat); + p0_out = __msa_bmnz_v(p0_out, (v16u8)p0_filt8_r, flat); + q0_out = __msa_bmnz_v(q0_out, (v16u8)q0_filt8_r, flat); + q1_out = __msa_bmnz_v(q1_out, (v16u8)q1_filt8_r, flat); + q2_out = __msa_bmnz_v(q2, (v16u8)q2_filt8_r, flat); + + src -= 3 * pitch; + + ST_UB4(p2_out, p1_out, p0_out, q0_out, src, pitch); + src += (4 * pitch); + ST_UB2(q1_out, q2_out, src, pitch); + src += (2 * pitch); + } +} + +void aom_lpf_vertical_8_msa(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + v16u8 p1_out, p0_out, q0_out, q1_out; + v16u8 flat, mask, hev, thresh, b_limit, limit; + v8u16 p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r; + v8i16 p2_filt8_r, p1_filt8_r, p0_filt8_r, q0_filt8_r, q1_filt8_r, q2_filt8_r; + v16u8 zero = { 0 }; + v8i16 vec0, vec1, vec2, vec3, vec4; + + /* load vector elements */ + LD_UB8(src - 4, pitch, p3, p2, p1, p0, q0, q1, q2, q3); + + TRANSPOSE8x8_UB_UB(p3, p2, p1, p0, q0, q1, q2, q3, p3, p2, p1, p0, q0, q1, q2, + q3); + + thresh = (v16u8)__msa_fill_b(*thresh_ptr); + b_limit = (v16u8)__msa_fill_b(*b_limit_ptr); + limit = (v16u8)__msa_fill_b(*limit_ptr); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + /* flat4 */ + AOM_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + AOM_LPF_FILTER4_8W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + flat = (v16u8)__msa_ilvr_d((v2i64)zero, (v2i64)flat); + + if (__msa_test_bz_v(flat)) { + /* Store 4 pixels p1-_q1 */ + ILVR_B2_SH(p0_out, p1_out, q1_out, q0_out, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec2, vec3); + + src -= 2; + ST4x4_UB(vec2, vec2, 0, 1, 2, 3, src, pitch); + src += 4 * pitch; + ST4x4_UB(vec3, vec3, 0, 1, 2, 3, src, pitch); + } else { + ILVR_B8_UH(zero, p3, zero, p2, zero, p1, zero, p0, zero, q0, zero, q1, zero, + q2, zero, q3, p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r); + AOM_FILTER8(p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r, p2_filt8_r, + p1_filt8_r, p0_filt8_r, q0_filt8_r, q1_filt8_r, q2_filt8_r); + /* convert 16 bit output data into 8 bit */ + PCKEV_B4_SH(p2_filt8_r, p2_filt8_r, p1_filt8_r, p1_filt8_r, p0_filt8_r, + p0_filt8_r, q0_filt8_r, q0_filt8_r, p2_filt8_r, p1_filt8_r, + p0_filt8_r, q0_filt8_r); + PCKEV_B2_SH(q1_filt8_r, q1_filt8_r, q2_filt8_r, q2_filt8_r, q1_filt8_r, + q2_filt8_r); + + /* store pixel values */ + p2 = __msa_bmnz_v(p2, (v16u8)p2_filt8_r, flat); + p1 = __msa_bmnz_v(p1_out, (v16u8)p1_filt8_r, flat); + p0 = __msa_bmnz_v(p0_out, (v16u8)p0_filt8_r, flat); + q0 = __msa_bmnz_v(q0_out, (v16u8)q0_filt8_r, flat); + q1 = __msa_bmnz_v(q1_out, (v16u8)q1_filt8_r, flat); + q2 = __msa_bmnz_v(q2, (v16u8)q2_filt8_r, flat); + + /* Store 6 pixels p2-_q2 */ + ILVR_B2_SH(p1, p2, q0, p0, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec2, vec3); + vec4 = (v8i16)__msa_ilvr_b((v16i8)q2, (v16i8)q1); + + src -= 3; + ST4x4_UB(vec2, vec2, 0, 1, 2, 3, src, pitch); + ST2x4_UB(vec4, 0, src + 4, pitch); + src += (4 * pitch); + ST4x4_UB(vec3, vec3, 0, 1, 2, 3, src, pitch); + ST2x4_UB(vec4, 4, src + 4, pitch); + } +} + +void aom_lpf_vertical_8_dual_msa(uint8_t *src, int32_t pitch, + const uint8_t *b_limit0, const uint8_t *limit0, + const uint8_t *thresh0, + const uint8_t *b_limit1, const uint8_t *limit1, + const uint8_t *thresh1) { + uint8_t *temp_src; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + v16u8 p1_out, p0_out, q0_out, q1_out; + v16u8 flat, mask, hev, thresh, b_limit, limit; + v16u8 row4, row5, row6, row7, row12, row13, row14, row15; + v8u16 p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r; + v8u16 p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + v8i16 p2_filt8_r, p1_filt8_r, p0_filt8_r, q0_filt8_r, q1_filt8_r, q2_filt8_r; + v8i16 p2_filt8_l, p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l; + v16u8 zero = { 0 }; + v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + + temp_src = src - 4; + + LD_UB8(temp_src, pitch, p0, p1, p2, p3, row4, row5, row6, row7); + temp_src += (8 * pitch); + LD_UB8(temp_src, pitch, q3, q2, q1, q0, row12, row13, row14, row15); + + /* transpose 16x8 matrix into 8x16 */ + TRANSPOSE16x8_UB_UB(p0, p1, p2, p3, row4, row5, row6, row7, q3, q2, q1, q0, + row12, row13, row14, row15, p3, p2, p1, p0, q0, q1, q2, + q3); + + thresh = (v16u8)__msa_fill_b(*thresh0); + vec0 = (v8i16)__msa_fill_b(*thresh1); + thresh = (v16u8)__msa_ilvr_d((v2i64)vec0, (v2i64)thresh); + + b_limit = (v16u8)__msa_fill_b(*b_limit0); + vec0 = (v8i16)__msa_fill_b(*b_limit1); + b_limit = (v16u8)__msa_ilvr_d((v2i64)vec0, (v2i64)b_limit); + + limit = (v16u8)__msa_fill_b(*limit0); + vec0 = (v8i16)__msa_fill_b(*limit1); + limit = (v16u8)__msa_ilvr_d((v2i64)vec0, (v2i64)limit); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + /* flat4 */ + AOM_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + AOM_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + if (__msa_test_bz_v(flat)) { + ILVR_B2_SH(p0_out, p1_out, q1_out, q0_out, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec2, vec3); + ILVL_B2_SH(p0_out, p1_out, q1_out, q0_out, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec4, vec5); + + src -= 2; + ST4x8_UB(vec2, vec3, src, pitch); + src += 8 * pitch; + ST4x8_UB(vec4, vec5, src, pitch); + } else { + ILVR_B8_UH(zero, p3, zero, p2, zero, p1, zero, p0, zero, q0, zero, q1, zero, + q2, zero, q3, p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r); + AOM_FILTER8(p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r, p2_filt8_r, + p1_filt8_r, p0_filt8_r, q0_filt8_r, q1_filt8_r, q2_filt8_r); + + ILVL_B4_UH(zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, p1_l, p0_l); + ILVL_B4_UH(zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, q2_l, q3_l); + + /* filter8 */ + AOM_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + /* convert 16 bit output data into 8 bit */ + PCKEV_B4_SH(p2_filt8_l, p2_filt8_r, p1_filt8_l, p1_filt8_r, p0_filt8_l, + p0_filt8_r, q0_filt8_l, q0_filt8_r, p2_filt8_r, p1_filt8_r, + p0_filt8_r, q0_filt8_r); + PCKEV_B2_SH(q1_filt8_l, q1_filt8_r, q2_filt8_l, q2_filt8_r, q1_filt8_r, + q2_filt8_r); + + /* store pixel values */ + p2 = __msa_bmnz_v(p2, (v16u8)p2_filt8_r, flat); + p1 = __msa_bmnz_v(p1_out, (v16u8)p1_filt8_r, flat); + p0 = __msa_bmnz_v(p0_out, (v16u8)p0_filt8_r, flat); + q0 = __msa_bmnz_v(q0_out, (v16u8)q0_filt8_r, flat); + q1 = __msa_bmnz_v(q1_out, (v16u8)q1_filt8_r, flat); + q2 = __msa_bmnz_v(q2, (v16u8)q2_filt8_r, flat); + + ILVR_B2_SH(p1, p2, q0, p0, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec3, vec4); + ILVL_B2_SH(p1, p2, q0, p0, vec0, vec1); + ILVRL_H2_SH(vec1, vec0, vec6, vec7); + ILVRL_B2_SH(q2, q1, vec2, vec5); + + src -= 3; + ST4x4_UB(vec3, vec3, 0, 1, 2, 3, src, pitch); + ST2x4_UB(vec2, 0, src + 4, pitch); + src += (4 * pitch); + ST4x4_UB(vec4, vec4, 0, 1, 2, 3, src, pitch); + ST2x4_UB(vec2, 4, src + 4, pitch); + src += (4 * pitch); + ST4x4_UB(vec6, vec6, 0, 1, 2, 3, src, pitch); + ST2x4_UB(vec5, 0, src + 4, pitch); + src += (4 * pitch); + ST4x4_UB(vec7, vec7, 0, 1, 2, 3, src, pitch); + ST2x4_UB(vec5, 4, src + 4, pitch); + } +} diff --git a/third_party/aom/aom_dsp/mips/loopfilter_filters_dspr2.c b/third_party/aom/aom_dsp/mips/loopfilter_filters_dspr2.c new file mode 100644 index 0000000000..883d0523d3 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/loopfilter_filters_dspr2.c @@ -0,0 +1,327 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_dsp/mips/common_dspr2.h" +#include "aom_dsp/mips/loopfilter_filters_dspr2.h" +#include "aom_dsp/mips/loopfilter_macros_dspr2.h" +#include "aom_dsp/mips/loopfilter_masks_dspr2.h" +#include "aom_mem/aom_mem.h" + +#if HAVE_DSPR2 +void aom_lpf_horizontal_4_dspr2(unsigned char *s, int pitch, + const uint8_t *blimit, const uint8_t *limit, + const uint8_t *thresh) { + uint8_t i; + uint32_t mask; + uint32_t hev; + uint32_t pm1, p0, p1, p2, p3, p4, p5, p6; + uint8_t *sm1, *s0, *s1, *s2, *s3, *s4, *s5, *s6; + uint32_t thresh_vec, flimit_vec, limit_vec; + uint32_t uflimit, ulimit, uthresh; + + uflimit = *blimit; + ulimit = *limit; + uthresh = *thresh; + + /* create quad-byte */ + __asm__ __volatile__( + "replv.qb %[thresh_vec], %[uthresh] \n\t" + "replv.qb %[flimit_vec], %[uflimit] \n\t" + "replv.qb %[limit_vec], %[ulimit] \n\t" + + : [thresh_vec] "=&r"(thresh_vec), [flimit_vec] "=&r"(flimit_vec), + [limit_vec] "=r"(limit_vec) + : [uthresh] "r"(uthresh), [uflimit] "r"(uflimit), [ulimit] "r"(ulimit)); + + /* prefetch data for store */ + prefetch_store(s); + + /* loop filter designed to work using chars so that we can make maximum use + of 8 bit simd instructions. */ + for (i = 0; i < 2; i++) { + sm1 = s - (pitch << 2); + s0 = sm1 + pitch; + s1 = s0 + pitch; + s2 = s - pitch; + s3 = s; + s4 = s + pitch; + s5 = s4 + pitch; + s6 = s5 + pitch; + + __asm__ __volatile__( + "lw %[p1], (%[s1]) \n\t" + "lw %[p2], (%[s2]) \n\t" + "lw %[p3], (%[s3]) \n\t" + "lw %[p4], (%[s4]) \n\t" + + : [p1] "=&r"(p1), [p2] "=&r"(p2), [p3] "=&r"(p3), [p4] "=&r"(p4) + : [s1] "r"(s1), [s2] "r"(s2), [s3] "r"(s3), [s4] "r"(s4)); + + /* if (p1 - p4 == 0) and (p2 - p3 == 0) + mask will be zero and filtering is not needed */ + if (!(((p1 - p4) == 0) && ((p2 - p3) == 0))) { + __asm__ __volatile__( + "lw %[pm1], (%[sm1]) \n\t" + "lw %[p0], (%[s0]) \n\t" + "lw %[p5], (%[s5]) \n\t" + "lw %[p6], (%[s6]) \n\t" + + : [pm1] "=&r"(pm1), [p0] "=&r"(p0), [p5] "=&r"(p5), [p6] "=&r"(p6) + : [sm1] "r"(sm1), [s0] "r"(s0), [s5] "r"(s5), [s6] "r"(s6)); + + filter_hev_mask_dspr2(limit_vec, flimit_vec, p1, p2, pm1, p0, p3, p4, p5, + p6, thresh_vec, &hev, &mask); + + /* if mask == 0 do filtering is not needed */ + if (mask) { + /* filtering */ + filter_dspr2(mask, hev, &p1, &p2, &p3, &p4); + + __asm__ __volatile__( + "sw %[p1], (%[s1]) \n\t" + "sw %[p2], (%[s2]) \n\t" + "sw %[p3], (%[s3]) \n\t" + "sw %[p4], (%[s4]) \n\t" + + : + : [p1] "r"(p1), [p2] "r"(p2), [p3] "r"(p3), [p4] "r"(p4), + [s1] "r"(s1), [s2] "r"(s2), [s3] "r"(s3), [s4] "r"(s4)); + } + } + + s = s + 4; + } +} + +void aom_lpf_vertical_4_dspr2(unsigned char *s, int pitch, + const uint8_t *blimit, const uint8_t *limit, + const uint8_t *thresh) { + uint8_t i; + uint32_t mask, hev; + uint32_t pm1, p0, p1, p2, p3, p4, p5, p6; + uint8_t *s1, *s2, *s3, *s4; + uint32_t prim1, prim2, sec3, sec4, prim3, prim4; + uint32_t thresh_vec, flimit_vec, limit_vec; + uint32_t uflimit, ulimit, uthresh; + + uflimit = *blimit; + ulimit = *limit; + uthresh = *thresh; + + /* create quad-byte */ + __asm__ __volatile__( + "replv.qb %[thresh_vec], %[uthresh] \n\t" + "replv.qb %[flimit_vec], %[uflimit] \n\t" + "replv.qb %[limit_vec], %[ulimit] \n\t" + + : [thresh_vec] "=&r"(thresh_vec), [flimit_vec] "=&r"(flimit_vec), + [limit_vec] "=r"(limit_vec) + : [uthresh] "r"(uthresh), [uflimit] "r"(uflimit), [ulimit] "r"(ulimit)); + + /* prefetch data for store */ + prefetch_store(s + pitch); + + for (i = 0; i < 2; i++) { + s1 = s; + s2 = s + pitch; + s3 = s2 + pitch; + s4 = s3 + pitch; + s = s4 + pitch; + + /* load quad-byte vectors + * memory is 4 byte aligned + */ + p2 = *((uint32_t *)(s1 - 4)); + p6 = *((uint32_t *)(s1)); + p1 = *((uint32_t *)(s2 - 4)); + p5 = *((uint32_t *)(s2)); + p0 = *((uint32_t *)(s3 - 4)); + p4 = *((uint32_t *)(s3)); + pm1 = *((uint32_t *)(s4 - 4)); + p3 = *((uint32_t *)(s4)); + + /* transpose pm1, p0, p1, p2 */ + __asm__ __volatile__( + "precrq.qb.ph %[prim1], %[p2], %[p1] \n\t" + "precr.qb.ph %[prim2], %[p2], %[p1] \n\t" + "precrq.qb.ph %[prim3], %[p0], %[pm1] \n\t" + "precr.qb.ph %[prim4], %[p0], %[pm1] \n\t" + + "precrq.qb.ph %[p1], %[prim1], %[prim2] \n\t" + "precr.qb.ph %[pm1], %[prim1], %[prim2] \n\t" + "precrq.qb.ph %[sec3], %[prim3], %[prim4] \n\t" + "precr.qb.ph %[sec4], %[prim3], %[prim4] \n\t" + + "precrq.ph.w %[p2], %[p1], %[sec3] \n\t" + "precrq.ph.w %[p0], %[pm1], %[sec4] \n\t" + "append %[p1], %[sec3], 16 \n\t" + "append %[pm1], %[sec4], 16 \n\t" + + : [prim1] "=&r"(prim1), [prim2] "=&r"(prim2), [prim3] "=&r"(prim3), + [prim4] "=&r"(prim4), [p2] "+r"(p2), [p1] "+r"(p1), [p0] "+r"(p0), + [pm1] "+r"(pm1), [sec3] "=&r"(sec3), [sec4] "=&r"(sec4) + :); + + /* transpose p3, p4, p5, p6 */ + __asm__ __volatile__( + "precrq.qb.ph %[prim1], %[p6], %[p5] \n\t" + "precr.qb.ph %[prim2], %[p6], %[p5] \n\t" + "precrq.qb.ph %[prim3], %[p4], %[p3] \n\t" + "precr.qb.ph %[prim4], %[p4], %[p3] \n\t" + + "precrq.qb.ph %[p5], %[prim1], %[prim2] \n\t" + "precr.qb.ph %[p3], %[prim1], %[prim2] \n\t" + "precrq.qb.ph %[sec3], %[prim3], %[prim4] \n\t" + "precr.qb.ph %[sec4], %[prim3], %[prim4] \n\t" + + "precrq.ph.w %[p6], %[p5], %[sec3] \n\t" + "precrq.ph.w %[p4], %[p3], %[sec4] \n\t" + "append %[p5], %[sec3], 16 \n\t" + "append %[p3], %[sec4], 16 \n\t" + + : [prim1] "=&r"(prim1), [prim2] "=&r"(prim2), [prim3] "=&r"(prim3), + [prim4] "=&r"(prim4), [p6] "+r"(p6), [p5] "+r"(p5), [p4] "+r"(p4), + [p3] "+r"(p3), [sec3] "=&r"(sec3), [sec4] "=&r"(sec4) + :); + + /* if (p1 - p4 == 0) and (p2 - p3 == 0) + * mask will be zero and filtering is not needed + */ + if (!(((p1 - p4) == 0) && ((p2 - p3) == 0))) { + filter_hev_mask_dspr2(limit_vec, flimit_vec, p1, p2, pm1, p0, p3, p4, p5, + p6, thresh_vec, &hev, &mask); + + /* if mask == 0 do filtering is not needed */ + if (mask) { + /* filtering */ + filter_dspr2(mask, hev, &p1, &p2, &p3, &p4); + + /* unpack processed 4x4 neighborhood + * don't use transpose on output data + * because memory isn't aligned + */ + __asm__ __volatile__( + "sb %[p4], 1(%[s4]) \n\t" + "sb %[p3], 0(%[s4]) \n\t" + "sb %[p2], -1(%[s4]) \n\t" + "sb %[p1], -2(%[s4]) \n\t" + + : + : [p4] "r"(p4), [p3] "r"(p3), [p2] "r"(p2), [p1] "r"(p1), + [s4] "r"(s4)); + + __asm__ __volatile__( + "srl %[p4], %[p4], 8 \n\t" + "srl %[p3], %[p3], 8 \n\t" + "srl %[p2], %[p2], 8 \n\t" + "srl %[p1], %[p1], 8 \n\t" + + : [p4] "+r"(p4), [p3] "+r"(p3), [p2] "+r"(p2), [p1] "+r"(p1) + :); + + __asm__ __volatile__( + "sb %[p4], 1(%[s3]) \n\t" + "sb %[p3], 0(%[s3]) \n\t" + "sb %[p2], -1(%[s3]) \n\t" + "sb %[p1], -2(%[s3]) \n\t" + + : [p1] "+r"(p1) + : [p4] "r"(p4), [p3] "r"(p3), [p2] "r"(p2), [s3] "r"(s3)); + + __asm__ __volatile__( + "srl %[p4], %[p4], 8 \n\t" + "srl %[p3], %[p3], 8 \n\t" + "srl %[p2], %[p2], 8 \n\t" + "srl %[p1], %[p1], 8 \n\t" + + : [p4] "+r"(p4), [p3] "+r"(p3), [p2] "+r"(p2), [p1] "+r"(p1) + :); + + __asm__ __volatile__( + "sb %[p4], 1(%[s2]) \n\t" + "sb %[p3], 0(%[s2]) \n\t" + "sb %[p2], -1(%[s2]) \n\t" + "sb %[p1], -2(%[s2]) \n\t" + + : + : [p4] "r"(p4), [p3] "r"(p3), [p2] "r"(p2), [p1] "r"(p1), + [s2] "r"(s2)); + + __asm__ __volatile__( + "srl %[p4], %[p4], 8 \n\t" + "srl %[p3], %[p3], 8 \n\t" + "srl %[p2], %[p2], 8 \n\t" + "srl %[p1], %[p1], 8 \n\t" + + : [p4] "+r"(p4), [p3] "+r"(p3), [p2] "+r"(p2), [p1] "+r"(p1) + :); + + __asm__ __volatile__( + "sb %[p4], 1(%[s1]) \n\t" + "sb %[p3], 0(%[s1]) \n\t" + "sb %[p2], -1(%[s1]) \n\t" + "sb %[p1], -2(%[s1]) \n\t" + + : + : [p4] "r"(p4), [p3] "r"(p3), [p2] "r"(p2), [p1] "r"(p1), + [s1] "r"(s1)); + } + } + } +} + +void aom_lpf_horizontal_4_dual_dspr2( + uint8_t *s, int p /* pitch */, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, + const uint8_t *limit1, const uint8_t *thresh1) { + aom_lpf_horizontal_4_dspr2(s, p, blimit0, limit0, thresh0); + aom_lpf_horizontal_4_dspr2(s + 8, p, blimit1, limit1, thresh1); +} + +void aom_lpf_horizontal_8_dual_dspr2( + uint8_t *s, int p /* pitch */, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, + const uint8_t *limit1, const uint8_t *thresh1) { + aom_lpf_horizontal_8_dspr2(s, p, blimit0, limit0, thresh0); + aom_lpf_horizontal_8_dspr2(s + 8, p, blimit1, limit1, thresh1); +} + +void aom_lpf_vertical_4_dual_dspr2(uint8_t *s, int p, const uint8_t *blimit0, + const uint8_t *limit0, + const uint8_t *thresh0, + const uint8_t *blimit1, + const uint8_t *limit1, + const uint8_t *thresh1) { + aom_lpf_vertical_4_dspr2(s, p, blimit0, limit0, thresh0); + aom_lpf_vertical_4_dspr2(s + 8 * p, p, blimit1, limit1, thresh1); +} + +void aom_lpf_vertical_8_dual_dspr2(uint8_t *s, int p, const uint8_t *blimit0, + const uint8_t *limit0, + const uint8_t *thresh0, + const uint8_t *blimit1, + const uint8_t *limit1, + const uint8_t *thresh1) { + aom_lpf_vertical_8_dspr2(s, p, blimit0, limit0, thresh0); + aom_lpf_vertical_8_dspr2(s + 8 * p, p, blimit1, limit1, thresh1); +} + +void aom_lpf_vertical_16_dual_dspr2(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh) { + aom_lpf_vertical_16_dspr2(s, p, blimit, limit, thresh); + aom_lpf_vertical_16_dspr2(s + 8 * p, p, blimit, limit, thresh); +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/aom_dsp/mips/loopfilter_filters_dspr2.h b/third_party/aom/aom_dsp/mips/loopfilter_filters_dspr2.h new file mode 100644 index 0000000000..72df098236 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/loopfilter_filters_dspr2.h @@ -0,0 +1,735 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_MIPS_LOOPFILTER_FILTERS_DSPR2_H_ +#define AOM_DSP_MIPS_LOOPFILTER_FILTERS_DSPR2_H_ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if HAVE_DSPR2 +/* inputs & outputs are quad-byte vectors */ +static INLINE void filter_dspr2(uint32_t mask, uint32_t hev, uint32_t *ps1, + uint32_t *ps0, uint32_t *qs0, uint32_t *qs1) { + int32_t aom_filter_l, aom_filter_r; + int32_t Filter1_l, Filter1_r, Filter2_l, Filter2_r; + int32_t subr_r, subr_l; + uint32_t t1, t2, HWM, t3; + uint32_t hev_l, hev_r, mask_l, mask_r, invhev_l, invhev_r; + int32_t vps1, vps0, vqs0, vqs1; + int32_t vps1_l, vps1_r, vps0_l, vps0_r, vqs0_l, vqs0_r, vqs1_l, vqs1_r; + uint32_t N128; + + N128 = 0x80808080; + t1 = 0x03000300; + t2 = 0x04000400; + t3 = 0x01000100; + HWM = 0xFF00FF00; + + vps0 = (*ps0) ^ N128; + vps1 = (*ps1) ^ N128; + vqs0 = (*qs0) ^ N128; + vqs1 = (*qs1) ^ N128; + + /* use halfword pairs instead quad-bytes because of accuracy */ + vps0_l = vps0 & HWM; + vps0_r = vps0 << 8; + vps0_r = vps0_r & HWM; + + vps1_l = vps1 & HWM; + vps1_r = vps1 << 8; + vps1_r = vps1_r & HWM; + + vqs0_l = vqs0 & HWM; + vqs0_r = vqs0 << 8; + vqs0_r = vqs0_r & HWM; + + vqs1_l = vqs1 & HWM; + vqs1_r = vqs1 << 8; + vqs1_r = vqs1_r & HWM; + + mask_l = mask & HWM; + mask_r = mask << 8; + mask_r = mask_r & HWM; + + hev_l = hev & HWM; + hev_r = hev << 8; + hev_r = hev_r & HWM; + + __asm__ __volatile__( + /* aom_filter = aom_signed_char_clamp(ps1 - qs1); */ + "subq_s.ph %[aom_filter_l], %[vps1_l], %[vqs1_l] \n\t" + "subq_s.ph %[aom_filter_r], %[vps1_r], %[vqs1_r] \n\t" + + /* qs0 - ps0 */ + "subq_s.ph %[subr_l], %[vqs0_l], %[vps0_l] \n\t" + "subq_s.ph %[subr_r], %[vqs0_r], %[vps0_r] \n\t" + + /* aom_filter &= hev; */ + "and %[aom_filter_l], %[aom_filter_l], %[hev_l] \n\t" + "and %[aom_filter_r], %[aom_filter_r], %[hev_r] \n\t" + + /* aom_filter = aom_signed_char_clamp(aom_filter + 3 * (qs0 - ps0)); */ + "addq_s.ph %[aom_filter_l], %[aom_filter_l], %[subr_l] \n\t" + "addq_s.ph %[aom_filter_r], %[aom_filter_r], %[subr_r] \n\t" + "xor %[invhev_l], %[hev_l], %[HWM] \n\t" + "addq_s.ph %[aom_filter_l], %[aom_filter_l], %[subr_l] \n\t" + "addq_s.ph %[aom_filter_r], %[aom_filter_r], %[subr_r] \n\t" + "xor %[invhev_r], %[hev_r], %[HWM] \n\t" + "addq_s.ph %[aom_filter_l], %[aom_filter_l], %[subr_l] \n\t" + "addq_s.ph %[aom_filter_r], %[aom_filter_r], %[subr_r] \n\t" + + /* aom_filter &= mask; */ + "and %[aom_filter_l], %[aom_filter_l], %[mask_l] \n\t" + "and %[aom_filter_r], %[aom_filter_r], %[mask_r] \n\t" + + : [aom_filter_l] "=&r"(aom_filter_l), [aom_filter_r] "=&r"(aom_filter_r), + [subr_l] "=&r"(subr_l), [subr_r] "=&r"(subr_r), + [invhev_l] "=&r"(invhev_l), [invhev_r] "=&r"(invhev_r) + : [vps0_l] "r"(vps0_l), [vps0_r] "r"(vps0_r), [vps1_l] "r"(vps1_l), + [vps1_r] "r"(vps1_r), [vqs0_l] "r"(vqs0_l), [vqs0_r] "r"(vqs0_r), + [vqs1_l] "r"(vqs1_l), [vqs1_r] "r"(vqs1_r), [mask_l] "r"(mask_l), + [mask_r] "r"(mask_r), [hev_l] "r"(hev_l), [hev_r] "r"(hev_r), + [HWM] "r"(HWM)); + + /* save bottom 3 bits so that we round one side +4 and the other +3 */ + __asm__ __volatile__( + /* Filter2 = aom_signed_char_clamp(aom_filter + 3) >>= 3; */ + "addq_s.ph %[Filter1_l], %[aom_filter_l], %[t2] \n\t" + "addq_s.ph %[Filter1_r], %[aom_filter_r], %[t2] \n\t" + + /* Filter1 = aom_signed_char_clamp(aom_filter + 4) >>= 3; */ + "addq_s.ph %[Filter2_l], %[aom_filter_l], %[t1] \n\t" + "addq_s.ph %[Filter2_r], %[aom_filter_r], %[t1] \n\t" + "shra.ph %[Filter1_r], %[Filter1_r], 3 \n\t" + "shra.ph %[Filter1_l], %[Filter1_l], 3 \n\t" + + "shra.ph %[Filter2_l], %[Filter2_l], 3 \n\t" + "shra.ph %[Filter2_r], %[Filter2_r], 3 \n\t" + + "and %[Filter1_l], %[Filter1_l], %[HWM] \n\t" + "and %[Filter1_r], %[Filter1_r], %[HWM] \n\t" + + /* vps0 = aom_signed_char_clamp(ps0 + Filter2); */ + "addq_s.ph %[vps0_l], %[vps0_l], %[Filter2_l] \n\t" + "addq_s.ph %[vps0_r], %[vps0_r], %[Filter2_r] \n\t" + + /* vqs0 = aom_signed_char_clamp(qs0 - Filter1); */ + "subq_s.ph %[vqs0_l], %[vqs0_l], %[Filter1_l] \n\t" + "subq_s.ph %[vqs0_r], %[vqs0_r], %[Filter1_r] \n\t" + + : [Filter1_l] "=&r"(Filter1_l), [Filter1_r] "=&r"(Filter1_r), + [Filter2_l] "=&r"(Filter2_l), [Filter2_r] "=&r"(Filter2_r), + [vps0_l] "+r"(vps0_l), [vps0_r] "+r"(vps0_r), [vqs0_l] "+r"(vqs0_l), + [vqs0_r] "+r"(vqs0_r) + : [t1] "r"(t1), [t2] "r"(t2), [HWM] "r"(HWM), + [aom_filter_l] "r"(aom_filter_l), [aom_filter_r] "r"(aom_filter_r)); + + __asm__ __volatile__( + /* (aom_filter += 1) >>= 1 */ + "addqh.ph %[Filter1_l], %[Filter1_l], %[t3] \n\t" + "addqh.ph %[Filter1_r], %[Filter1_r], %[t3] \n\t" + + /* aom_filter &= ~hev; */ + "and %[Filter1_l], %[Filter1_l], %[invhev_l] \n\t" + "and %[Filter1_r], %[Filter1_r], %[invhev_r] \n\t" + + /* vps1 = aom_signed_char_clamp(ps1 + aom_filter); */ + "addq_s.ph %[vps1_l], %[vps1_l], %[Filter1_l] \n\t" + "addq_s.ph %[vps1_r], %[vps1_r], %[Filter1_r] \n\t" + + /* vqs1 = aom_signed_char_clamp(qs1 - aom_filter); */ + "subq_s.ph %[vqs1_l], %[vqs1_l], %[Filter1_l] \n\t" + "subq_s.ph %[vqs1_r], %[vqs1_r], %[Filter1_r] \n\t" + + : [Filter1_l] "+r"(Filter1_l), [Filter1_r] "+r"(Filter1_r), + [vps1_l] "+r"(vps1_l), [vps1_r] "+r"(vps1_r), [vqs1_l] "+r"(vqs1_l), + [vqs1_r] "+r"(vqs1_r) + : [t3] "r"(t3), [invhev_l] "r"(invhev_l), [invhev_r] "r"(invhev_r)); + + /* Create quad-bytes from halfword pairs */ + vqs0_l = vqs0_l & HWM; + vqs1_l = vqs1_l & HWM; + vps0_l = vps0_l & HWM; + vps1_l = vps1_l & HWM; + + __asm__ __volatile__( + "shrl.ph %[vqs0_r], %[vqs0_r], 8 \n\t" + "shrl.ph %[vps0_r], %[vps0_r], 8 \n\t" + "shrl.ph %[vqs1_r], %[vqs1_r], 8 \n\t" + "shrl.ph %[vps1_r], %[vps1_r], 8 \n\t" + + : [vps1_r] "+r"(vps1_r), [vqs1_r] "+r"(vqs1_r), [vps0_r] "+r"(vps0_r), + [vqs0_r] "+r"(vqs0_r) + :); + + vqs0 = vqs0_l | vqs0_r; + vqs1 = vqs1_l | vqs1_r; + vps0 = vps0_l | vps0_r; + vps1 = vps1_l | vps1_r; + + *ps0 = vps0 ^ N128; + *ps1 = vps1 ^ N128; + *qs0 = vqs0 ^ N128; + *qs1 = vqs1 ^ N128; +} + +static INLINE void filter1_dspr2(uint32_t mask, uint32_t hev, uint32_t ps1, + uint32_t ps0, uint32_t qs0, uint32_t qs1, + uint32_t *p1_f0, uint32_t *p0_f0, + uint32_t *q0_f0, uint32_t *q1_f0) { + int32_t aom_filter_l, aom_filter_r; + int32_t Filter1_l, Filter1_r, Filter2_l, Filter2_r; + int32_t subr_r, subr_l; + uint32_t t1, t2, HWM, t3; + uint32_t hev_l, hev_r, mask_l, mask_r, invhev_l, invhev_r; + int32_t vps1, vps0, vqs0, vqs1; + int32_t vps1_l, vps1_r, vps0_l, vps0_r, vqs0_l, vqs0_r, vqs1_l, vqs1_r; + uint32_t N128; + + N128 = 0x80808080; + t1 = 0x03000300; + t2 = 0x04000400; + t3 = 0x01000100; + HWM = 0xFF00FF00; + + vps0 = (ps0) ^ N128; + vps1 = (ps1) ^ N128; + vqs0 = (qs0) ^ N128; + vqs1 = (qs1) ^ N128; + + /* use halfword pairs instead quad-bytes because of accuracy */ + vps0_l = vps0 & HWM; + vps0_r = vps0 << 8; + vps0_r = vps0_r & HWM; + + vps1_l = vps1 & HWM; + vps1_r = vps1 << 8; + vps1_r = vps1_r & HWM; + + vqs0_l = vqs0 & HWM; + vqs0_r = vqs0 << 8; + vqs0_r = vqs0_r & HWM; + + vqs1_l = vqs1 & HWM; + vqs1_r = vqs1 << 8; + vqs1_r = vqs1_r & HWM; + + mask_l = mask & HWM; + mask_r = mask << 8; + mask_r = mask_r & HWM; + + hev_l = hev & HWM; + hev_r = hev << 8; + hev_r = hev_r & HWM; + + __asm__ __volatile__( + /* aom_filter = aom_signed_char_clamp(ps1 - qs1); */ + "subq_s.ph %[aom_filter_l], %[vps1_l], %[vqs1_l] \n\t" + "subq_s.ph %[aom_filter_r], %[vps1_r], %[vqs1_r] \n\t" + + /* qs0 - ps0 */ + "subq_s.ph %[subr_l], %[vqs0_l], %[vps0_l] \n\t" + "subq_s.ph %[subr_r], %[vqs0_r], %[vps0_r] \n\t" + + /* aom_filter &= hev; */ + "and %[aom_filter_l], %[aom_filter_l], %[hev_l] \n\t" + "and %[aom_filter_r], %[aom_filter_r], %[hev_r] \n\t" + + /* aom_filter = aom_signed_char_clamp(aom_filter + 3 * (qs0 - ps0)); */ + "addq_s.ph %[aom_filter_l], %[aom_filter_l], %[subr_l] \n\t" + "addq_s.ph %[aom_filter_r], %[aom_filter_r], %[subr_r] \n\t" + "xor %[invhev_l], %[hev_l], %[HWM] \n\t" + "addq_s.ph %[aom_filter_l], %[aom_filter_l], %[subr_l] \n\t" + "addq_s.ph %[aom_filter_r], %[aom_filter_r], %[subr_r] \n\t" + "xor %[invhev_r], %[hev_r], %[HWM] \n\t" + "addq_s.ph %[aom_filter_l], %[aom_filter_l], %[subr_l] \n\t" + "addq_s.ph %[aom_filter_r], %[aom_filter_r], %[subr_r] \n\t" + + /* aom_filter &= mask; */ + "and %[aom_filter_l], %[aom_filter_l], %[mask_l] \n\t" + "and %[aom_filter_r], %[aom_filter_r], %[mask_r] \n\t" + + : [aom_filter_l] "=&r"(aom_filter_l), [aom_filter_r] "=&r"(aom_filter_r), + [subr_l] "=&r"(subr_l), [subr_r] "=&r"(subr_r), + [invhev_l] "=&r"(invhev_l), [invhev_r] "=&r"(invhev_r) + : [vps0_l] "r"(vps0_l), [vps0_r] "r"(vps0_r), [vps1_l] "r"(vps1_l), + [vps1_r] "r"(vps1_r), [vqs0_l] "r"(vqs0_l), [vqs0_r] "r"(vqs0_r), + [vqs1_l] "r"(vqs1_l), [vqs1_r] "r"(vqs1_r), [mask_l] "r"(mask_l), + [mask_r] "r"(mask_r), [hev_l] "r"(hev_l), [hev_r] "r"(hev_r), + [HWM] "r"(HWM)); + + /* save bottom 3 bits so that we round one side +4 and the other +3 */ + __asm__ __volatile__( + /* Filter2 = aom_signed_char_clamp(aom_filter + 3) >>= 3; */ + "addq_s.ph %[Filter1_l], %[aom_filter_l], %[t2] \n\t" + "addq_s.ph %[Filter1_r], %[aom_filter_r], %[t2] \n\t" + + /* Filter1 = aom_signed_char_clamp(aom_filter + 4) >>= 3; */ + "addq_s.ph %[Filter2_l], %[aom_filter_l], %[t1] \n\t" + "addq_s.ph %[Filter2_r], %[aom_filter_r], %[t1] \n\t" + "shra.ph %[Filter1_r], %[Filter1_r], 3 \n\t" + "shra.ph %[Filter1_l], %[Filter1_l], 3 \n\t" + + "shra.ph %[Filter2_l], %[Filter2_l], 3 \n\t" + "shra.ph %[Filter2_r], %[Filter2_r], 3 \n\t" + + "and %[Filter1_l], %[Filter1_l], %[HWM] \n\t" + "and %[Filter1_r], %[Filter1_r], %[HWM] \n\t" + + /* vps0 = aom_signed_char_clamp(ps0 + Filter2); */ + "addq_s.ph %[vps0_l], %[vps0_l], %[Filter2_l] \n\t" + "addq_s.ph %[vps0_r], %[vps0_r], %[Filter2_r] \n\t" + + /* vqs0 = aom_signed_char_clamp(qs0 - Filter1); */ + "subq_s.ph %[vqs0_l], %[vqs0_l], %[Filter1_l] \n\t" + "subq_s.ph %[vqs0_r], %[vqs0_r], %[Filter1_r] \n\t" + + : [Filter1_l] "=&r"(Filter1_l), [Filter1_r] "=&r"(Filter1_r), + [Filter2_l] "=&r"(Filter2_l), [Filter2_r] "=&r"(Filter2_r), + [vps0_l] "+r"(vps0_l), [vps0_r] "+r"(vps0_r), [vqs0_l] "+r"(vqs0_l), + [vqs0_r] "+r"(vqs0_r) + : [t1] "r"(t1), [t2] "r"(t2), [HWM] "r"(HWM), + [aom_filter_l] "r"(aom_filter_l), [aom_filter_r] "r"(aom_filter_r)); + + __asm__ __volatile__( + /* (aom_filter += 1) >>= 1 */ + "addqh.ph %[Filter1_l], %[Filter1_l], %[t3] \n\t" + "addqh.ph %[Filter1_r], %[Filter1_r], %[t3] \n\t" + + /* aom_filter &= ~hev; */ + "and %[Filter1_l], %[Filter1_l], %[invhev_l] \n\t" + "and %[Filter1_r], %[Filter1_r], %[invhev_r] \n\t" + + /* vps1 = aom_signed_char_clamp(ps1 + aom_filter); */ + "addq_s.ph %[vps1_l], %[vps1_l], %[Filter1_l] \n\t" + "addq_s.ph %[vps1_r], %[vps1_r], %[Filter1_r] \n\t" + + /* vqs1 = aom_signed_char_clamp(qs1 - aom_filter); */ + "subq_s.ph %[vqs1_l], %[vqs1_l], %[Filter1_l] \n\t" + "subq_s.ph %[vqs1_r], %[vqs1_r], %[Filter1_r] \n\t" + + : [Filter1_l] "+r"(Filter1_l), [Filter1_r] "+r"(Filter1_r), + [vps1_l] "+r"(vps1_l), [vps1_r] "+r"(vps1_r), [vqs1_l] "+r"(vqs1_l), + [vqs1_r] "+r"(vqs1_r) + : [t3] "r"(t3), [invhev_l] "r"(invhev_l), [invhev_r] "r"(invhev_r)); + + /* Create quad-bytes from halfword pairs */ + vqs0_l = vqs0_l & HWM; + vqs1_l = vqs1_l & HWM; + vps0_l = vps0_l & HWM; + vps1_l = vps1_l & HWM; + + __asm__ __volatile__( + "shrl.ph %[vqs0_r], %[vqs0_r], 8 \n\t" + "shrl.ph %[vps0_r], %[vps0_r], 8 \n\t" + "shrl.ph %[vqs1_r], %[vqs1_r], 8 \n\t" + "shrl.ph %[vps1_r], %[vps1_r], 8 \n\t" + + : [vps1_r] "+r"(vps1_r), [vqs1_r] "+r"(vqs1_r), [vps0_r] "+r"(vps0_r), + [vqs0_r] "+r"(vqs0_r) + :); + + vqs0 = vqs0_l | vqs0_r; + vqs1 = vqs1_l | vqs1_r; + vps0 = vps0_l | vps0_r; + vps1 = vps1_l | vps1_r; + + *p0_f0 = vps0 ^ N128; + *p1_f0 = vps1 ^ N128; + *q0_f0 = vqs0 ^ N128; + *q1_f0 = vqs1 ^ N128; +} + +static INLINE void mbfilter_dspr2(uint32_t *op3, uint32_t *op2, uint32_t *op1, + uint32_t *op0, uint32_t *oq0, uint32_t *oq1, + uint32_t *oq2, uint32_t *oq3) { + /* use a 7 tap filter [1, 1, 1, 2, 1, 1, 1] for flat line */ + const uint32_t p3 = *op3, p2 = *op2, p1 = *op1, p0 = *op0; + const uint32_t q0 = *oq0, q1 = *oq1, q2 = *oq2, q3 = *oq3; + uint32_t res_op2, res_op1, res_op0; + uint32_t res_oq0, res_oq1, res_oq2; + uint32_t tmp; + uint32_t add_p210_q012; + uint32_t u32Four = 0x00040004; + + /* *op2 = ROUND_POWER_OF_TWO(p3 + p3 + p3 + p2 + p2 + p1 + p0 + q0, 3) 1 */ + /* *op1 = ROUND_POWER_OF_TWO(p3 + p3 + p2 + p1 + p1 + p0 + q0 + q1, 3) 2 */ + /* *op0 = ROUND_POWER_OF_TWO(p3 + p2 + p1 + p0 + p0 + q0 + q1 + q2, 3) 3 */ + /* *oq0 = ROUND_POWER_OF_TWO(p2 + p1 + p0 + q0 + q0 + q1 + q2 + q3, 3) 4 */ + /* *oq1 = ROUND_POWER_OF_TWO(p1 + p0 + q0 + q1 + q1 + q2 + q3 + q3, 3) 5 */ + /* *oq2 = ROUND_POWER_OF_TWO(p0 + q0 + q1 + q2 + q2 + q3 + q3 + q3, 3) 6 */ + + __asm__ __volatile__( + "addu.ph %[add_p210_q012], %[p2], %[p1] \n\t" + "addu.ph %[add_p210_q012], %[add_p210_q012], %[p0] \n\t" + "addu.ph %[add_p210_q012], %[add_p210_q012], %[q0] \n\t" + "addu.ph %[add_p210_q012], %[add_p210_q012], %[q1] \n\t" + "addu.ph %[add_p210_q012], %[add_p210_q012], %[q2] \n\t" + "addu.ph %[add_p210_q012], %[add_p210_q012], %[u32Four] \n\t" + + "shll.ph %[tmp], %[p3], 1 \n\t" + "addu.ph %[res_op2], %[tmp], %[p3] \n\t" + "addu.ph %[res_op1], %[p3], %[p3] \n\t" + "addu.ph %[res_op2], %[res_op2], %[p2] \n\t" + "addu.ph %[res_op1], %[res_op1], %[p1] \n\t" + "addu.ph %[res_op2], %[res_op2], %[add_p210_q012] \n\t" + "addu.ph %[res_op1], %[res_op1], %[add_p210_q012] \n\t" + "subu.ph %[res_op2], %[res_op2], %[q1] \n\t" + "subu.ph %[res_op1], %[res_op1], %[q2] \n\t" + "subu.ph %[res_op2], %[res_op2], %[q2] \n\t" + "shrl.ph %[res_op1], %[res_op1], 3 \n\t" + "shrl.ph %[res_op2], %[res_op2], 3 \n\t" + "addu.ph %[res_op0], %[p3], %[p0] \n\t" + "addu.ph %[res_oq0], %[q0], %[q3] \n\t" + "addu.ph %[res_op0], %[res_op0], %[add_p210_q012] \n\t" + "addu.ph %[res_oq0], %[res_oq0], %[add_p210_q012] \n\t" + "addu.ph %[res_oq1], %[q3], %[q3] \n\t" + "shll.ph %[tmp], %[q3], 1 \n\t" + "addu.ph %[res_oq1], %[res_oq1], %[q1] \n\t" + "addu.ph %[res_oq2], %[tmp], %[q3] \n\t" + "addu.ph %[res_oq1], %[res_oq1], %[add_p210_q012] \n\t" + "addu.ph %[res_oq2], %[res_oq2], %[add_p210_q012] \n\t" + "subu.ph %[res_oq1], %[res_oq1], %[p2] \n\t" + "addu.ph %[res_oq2], %[res_oq2], %[q2] \n\t" + "shrl.ph %[res_oq1], %[res_oq1], 3 \n\t" + "subu.ph %[res_oq2], %[res_oq2], %[p2] \n\t" + "shrl.ph %[res_oq0], %[res_oq0], 3 \n\t" + "subu.ph %[res_oq2], %[res_oq2], %[p1] \n\t" + "shrl.ph %[res_op0], %[res_op0], 3 \n\t" + "shrl.ph %[res_oq2], %[res_oq2], 3 \n\t" + + : [add_p210_q012] "=&r"(add_p210_q012), [tmp] "=&r"(tmp), + [res_op2] "=&r"(res_op2), [res_op1] "=&r"(res_op1), + [res_op0] "=&r"(res_op0), [res_oq0] "=&r"(res_oq0), + [res_oq1] "=&r"(res_oq1), [res_oq2] "=&r"(res_oq2) + : [p0] "r"(p0), [q0] "r"(q0), [p1] "r"(p1), [q1] "r"(q1), [p2] "r"(p2), + [q2] "r"(q2), [p3] "r"(p3), [q3] "r"(q3), [u32Four] "r"(u32Four)); + + *op2 = res_op2; + *op1 = res_op1; + *op0 = res_op0; + *oq0 = res_oq0; + *oq1 = res_oq1; + *oq2 = res_oq2; +} + +static INLINE void mbfilter1_dspr2(uint32_t p3, uint32_t p2, uint32_t p1, + uint32_t p0, uint32_t q0, uint32_t q1, + uint32_t q2, uint32_t q3, uint32_t *op2_f1, + uint32_t *op1_f1, uint32_t *op0_f1, + uint32_t *oq0_f1, uint32_t *oq1_f1, + uint32_t *oq2_f1) { + /* use a 7 tap filter [1, 1, 1, 2, 1, 1, 1] for flat line */ + uint32_t res_op2, res_op1, res_op0; + uint32_t res_oq0, res_oq1, res_oq2; + uint32_t tmp; + uint32_t add_p210_q012; + uint32_t u32Four = 0x00040004; + + /* *op2 = ROUND_POWER_OF_TWO(p3 + p3 + p3 + p2 + p2 + p1 + p0 + q0, 3) 1 */ + /* *op1 = ROUND_POWER_OF_TWO(p3 + p3 + p2 + p1 + p1 + p0 + q0 + q1, 3) 2 */ + /* *op0 = ROUND_POWER_OF_TWO(p3 + p2 + p1 + p0 + p0 + q0 + q1 + q2, 3) 3 */ + /* *oq0 = ROUND_POWER_OF_TWO(p2 + p1 + p0 + q0 + q0 + q1 + q2 + q3, 3) 4 */ + /* *oq1 = ROUND_POWER_OF_TWO(p1 + p0 + q0 + q1 + q1 + q2 + q3 + q3, 3) 5 */ + /* *oq2 = ROUND_POWER_OF_TWO(p0 + q0 + q1 + q2 + q2 + q3 + q3 + q3, 3) 6 */ + + __asm__ __volatile__( + "addu.ph %[add_p210_q012], %[p2], %[p1] \n\t" + "addu.ph %[add_p210_q012], %[add_p210_q012], %[p0] \n\t" + "addu.ph %[add_p210_q012], %[add_p210_q012], %[q0] \n\t" + "addu.ph %[add_p210_q012], %[add_p210_q012], %[q1] \n\t" + "addu.ph %[add_p210_q012], %[add_p210_q012], %[q2] \n\t" + "addu.ph %[add_p210_q012], %[add_p210_q012], %[u32Four] \n\t" + + "shll.ph %[tmp], %[p3], 1 \n\t" + "addu.ph %[res_op2], %[tmp], %[p3] \n\t" + "addu.ph %[res_op1], %[p3], %[p3] \n\t" + "addu.ph %[res_op2], %[res_op2], %[p2] \n\t" + "addu.ph %[res_op1], %[res_op1], %[p1] \n\t" + "addu.ph %[res_op2], %[res_op2], %[add_p210_q012] \n\t" + "addu.ph %[res_op1], %[res_op1], %[add_p210_q012] \n\t" + "subu.ph %[res_op2], %[res_op2], %[q1] \n\t" + "subu.ph %[res_op1], %[res_op1], %[q2] \n\t" + "subu.ph %[res_op2], %[res_op2], %[q2] \n\t" + "shrl.ph %[res_op1], %[res_op1], 3 \n\t" + "shrl.ph %[res_op2], %[res_op2], 3 \n\t" + "addu.ph %[res_op0], %[p3], %[p0] \n\t" + "addu.ph %[res_oq0], %[q0], %[q3] \n\t" + "addu.ph %[res_op0], %[res_op0], %[add_p210_q012] \n\t" + "addu.ph %[res_oq0], %[res_oq0], %[add_p210_q012] \n\t" + "addu.ph %[res_oq1], %[q3], %[q3] \n\t" + "shll.ph %[tmp], %[q3], 1 \n\t" + "addu.ph %[res_oq1], %[res_oq1], %[q1] \n\t" + "addu.ph %[res_oq2], %[tmp], %[q3] \n\t" + "addu.ph %[res_oq1], %[res_oq1], %[add_p210_q012] \n\t" + "addu.ph %[res_oq2], %[res_oq2], %[add_p210_q012] \n\t" + "subu.ph %[res_oq1], %[res_oq1], %[p2] \n\t" + "addu.ph %[res_oq2], %[res_oq2], %[q2] \n\t" + "shrl.ph %[res_oq1], %[res_oq1], 3 \n\t" + "subu.ph %[res_oq2], %[res_oq2], %[p2] \n\t" + "shrl.ph %[res_oq0], %[res_oq0], 3 \n\t" + "subu.ph %[res_oq2], %[res_oq2], %[p1] \n\t" + "shrl.ph %[res_op0], %[res_op0], 3 \n\t" + "shrl.ph %[res_oq2], %[res_oq2], 3 \n\t" + + : [add_p210_q012] "=&r"(add_p210_q012), [tmp] "=&r"(tmp), + [res_op2] "=&r"(res_op2), [res_op1] "=&r"(res_op1), + [res_op0] "=&r"(res_op0), [res_oq0] "=&r"(res_oq0), + [res_oq1] "=&r"(res_oq1), [res_oq2] "=&r"(res_oq2) + : [p0] "r"(p0), [q0] "r"(q0), [p1] "r"(p1), [q1] "r"(q1), [p2] "r"(p2), + [q2] "r"(q2), [p3] "r"(p3), [q3] "r"(q3), [u32Four] "r"(u32Four)); + + *op2_f1 = res_op2; + *op1_f1 = res_op1; + *op0_f1 = res_op0; + *oq0_f1 = res_oq0; + *oq1_f1 = res_oq1; + *oq2_f1 = res_oq2; +} + +static INLINE void wide_mbfilter_dspr2( + uint32_t *op7, uint32_t *op6, uint32_t *op5, uint32_t *op4, uint32_t *op3, + uint32_t *op2, uint32_t *op1, uint32_t *op0, uint32_t *oq0, uint32_t *oq1, + uint32_t *oq2, uint32_t *oq3, uint32_t *oq4, uint32_t *oq5, uint32_t *oq6, + uint32_t *oq7) { + const uint32_t p7 = *op7, p6 = *op6, p5 = *op5, p4 = *op4; + const uint32_t p3 = *op3, p2 = *op2, p1 = *op1, p0 = *op0; + const uint32_t q0 = *oq0, q1 = *oq1, q2 = *oq2, q3 = *oq3; + const uint32_t q4 = *oq4, q5 = *oq5, q6 = *oq6, q7 = *oq7; + uint32_t res_op6, res_op5, res_op4, res_op3, res_op2, res_op1, res_op0; + uint32_t res_oq0, res_oq1, res_oq2, res_oq3, res_oq4, res_oq5, res_oq6; + uint32_t tmp; + uint32_t add_p6toq6; + uint32_t u32Eight = 0x00080008; + + __asm__ __volatile__( + /* addition of p6,p5,p4,p3,p2,p1,p0,q0,q1,q2,q3,q4,q5,q6 + which is used most of the time */ + "addu.ph %[add_p6toq6], %[p6], %[p5] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[p4] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[p3] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[p2] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[p1] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[p0] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[q0] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[q1] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[q2] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[q3] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[q4] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[q5] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[q6] \n\t" + "addu.ph %[add_p6toq6], %[add_p6toq6], %[u32Eight] \n\t" + + : [add_p6toq6] "=&r"(add_p6toq6) + : [p6] "r"(p6), [p5] "r"(p5), [p4] "r"(p4), [p3] "r"(p3), [p2] "r"(p2), + [p1] "r"(p1), [p0] "r"(p0), [q0] "r"(q0), [q1] "r"(q1), [q2] "r"(q2), + [q3] "r"(q3), [q4] "r"(q4), [q5] "r"(q5), [q6] "r"(q6), + [u32Eight] "r"(u32Eight)); + + __asm__ __volatile__( + /* *op6 = ROUND_POWER_OF_TWO(p7 * 7 + p6 * 2 + p5 + p4 + + p3 + p2 + p1 + p0 + q0, 4) */ + "shll.ph %[tmp], %[p7], 3 \n\t" + "subu.ph %[res_op6], %[tmp], %[p7] \n\t" + "addu.ph %[res_op6], %[res_op6], %[p6] \n\t" + "addu.ph %[res_op6], %[res_op6], %[add_p6toq6] \n\t" + "subu.ph %[res_op6], %[res_op6], %[q1] \n\t" + "subu.ph %[res_op6], %[res_op6], %[q2] \n\t" + "subu.ph %[res_op6], %[res_op6], %[q3] \n\t" + "subu.ph %[res_op6], %[res_op6], %[q4] \n\t" + "subu.ph %[res_op6], %[res_op6], %[q5] \n\t" + "subu.ph %[res_op6], %[res_op6], %[q6] \n\t" + "shrl.ph %[res_op6], %[res_op6], 4 \n\t" + + /* *op5 = ROUND_POWER_OF_TWO(p7 * 6 + p6 + p5 * 2 + p4 + p3 + + p2 + p1 + p0 + q0 + q1, 4) */ + "shll.ph %[tmp], %[p7], 2 \n\t" + "addu.ph %[res_op5], %[tmp], %[p7] \n\t" + "addu.ph %[res_op5], %[res_op5], %[p7] \n\t" + "addu.ph %[res_op5], %[res_op5], %[p5] \n\t" + "addu.ph %[res_op5], %[res_op5], %[add_p6toq6] \n\t" + "subu.ph %[res_op5], %[res_op5], %[q2] \n\t" + "subu.ph %[res_op5], %[res_op5], %[q3] \n\t" + "subu.ph %[res_op5], %[res_op5], %[q4] \n\t" + "subu.ph %[res_op5], %[res_op5], %[q5] \n\t" + "subu.ph %[res_op5], %[res_op5], %[q6] \n\t" + "shrl.ph %[res_op5], %[res_op5], 4 \n\t" + + /* *op4 = ROUND_POWER_OF_TWO(p7 * 5 + p6 + p5 + p4 * 2 + p3 + p2 + + p1 + p0 + q0 + q1 + q2, 4) */ + "shll.ph %[tmp], %[p7], 2 \n\t" + "addu.ph %[res_op4], %[tmp], %[p7] \n\t" + "addu.ph %[res_op4], %[res_op4], %[p4] \n\t" + "addu.ph %[res_op4], %[res_op4], %[add_p6toq6] \n\t" + "subu.ph %[res_op4], %[res_op4], %[q3] \n\t" + "subu.ph %[res_op4], %[res_op4], %[q4] \n\t" + "subu.ph %[res_op4], %[res_op4], %[q5] \n\t" + "subu.ph %[res_op4], %[res_op4], %[q6] \n\t" + "shrl.ph %[res_op4], %[res_op4], 4 \n\t" + + /* *op3 = ROUND_POWER_OF_TWO(p7 * 4 + p6 + p5 + p4 + p3 * 2 + p2 + + p1 + p0 + q0 + q1 + q2 + q3, 4) */ + "shll.ph %[tmp], %[p7], 2 \n\t" + "addu.ph %[res_op3], %[tmp], %[p3] \n\t" + "addu.ph %[res_op3], %[res_op3], %[add_p6toq6] \n\t" + "subu.ph %[res_op3], %[res_op3], %[q4] \n\t" + "subu.ph %[res_op3], %[res_op3], %[q5] \n\t" + "subu.ph %[res_op3], %[res_op3], %[q6] \n\t" + "shrl.ph %[res_op3], %[res_op3], 4 \n\t" + + /* *op2 = ROUND_POWER_OF_TWO(p7 * 3 + p6 + p5 + p4 + p3 + p2 * 2 + p1 + + p0 + q0 + q1 + q2 + q3 + q4, 4) */ + "shll.ph %[tmp], %[p7], 1 \n\t" + "addu.ph %[res_op2], %[tmp], %[p7] \n\t" + "addu.ph %[res_op2], %[res_op2], %[p2] \n\t" + "addu.ph %[res_op2], %[res_op2], %[add_p6toq6] \n\t" + "subu.ph %[res_op2], %[res_op2], %[q5] \n\t" + "subu.ph %[res_op2], %[res_op2], %[q6] \n\t" + "shrl.ph %[res_op2], %[res_op2], 4 \n\t" + + /* *op1 = ROUND_POWER_OF_TWO(p7 * 2 + p6 + p5 + p4 + p3 + p2 + p1 * 2 + + p0 + q0 + q1 + q2 + q3 + q4 + q5, 4); */ + "shll.ph %[tmp], %[p7], 1 \n\t" + "addu.ph %[res_op1], %[tmp], %[p1] \n\t" + "addu.ph %[res_op1], %[res_op1], %[add_p6toq6] \n\t" + "subu.ph %[res_op1], %[res_op1], %[q6] \n\t" + "shrl.ph %[res_op1], %[res_op1], 4 \n\t" + + /* *op0 = ROUND_POWER_OF_TWO(p7 + p6 + p5 + p4 + p3 + p2 + p1 + p0 * 2 + + q0 + q1 + q2 + q3 + q4 + q5 + q6, 4) */ + "addu.ph %[res_op0], %[p7], %[p0] \n\t" + "addu.ph %[res_op0], %[res_op0], %[add_p6toq6] \n\t" + "shrl.ph %[res_op0], %[res_op0], 4 \n\t" + + : [res_op6] "=&r"(res_op6), [res_op5] "=&r"(res_op5), + [res_op4] "=&r"(res_op4), [res_op3] "=&r"(res_op3), + [res_op2] "=&r"(res_op2), [res_op1] "=&r"(res_op1), + [res_op0] "=&r"(res_op0), [tmp] "=&r"(tmp) + : [p7] "r"(p7), [p6] "r"(p6), [p5] "r"(p5), [p4] "r"(p4), [p3] "r"(p3), + [p2] "r"(p2), [p1] "r"(p1), [p0] "r"(p0), [q2] "r"(q2), [q1] "r"(q1), + [q3] "r"(q3), [q4] "r"(q4), [q5] "r"(q5), [q6] "r"(q6), + [add_p6toq6] "r"(add_p6toq6)); + + *op6 = res_op6; + *op5 = res_op5; + *op4 = res_op4; + *op3 = res_op3; + *op2 = res_op2; + *op1 = res_op1; + *op0 = res_op0; + + __asm__ __volatile__( + /* *oq0 = ROUND_POWER_OF_TWO(p6 + p5 + p4 + p3 + p2 + p1 + p0 + q0 * 2 + + q1 + q2 + q3 + q4 + q5 + q6 + q7, 4); */ + "addu.ph %[res_oq0], %[q7], %[q0] \n\t" + "addu.ph %[res_oq0], %[res_oq0], %[add_p6toq6] \n\t" + "shrl.ph %[res_oq0], %[res_oq0], 4 \n\t" + + /* *oq1 = ROUND_POWER_OF_TWO(p5 + p4 + p3 + p2 + p1 + p0 + q0 + q1 * 2 + + q2 + q3 + q4 + q5 + q6 + q7 * 2, 4) */ + "shll.ph %[tmp], %[q7], 1 \n\t" + "addu.ph %[res_oq1], %[tmp], %[q1] \n\t" + "addu.ph %[res_oq1], %[res_oq1], %[add_p6toq6] \n\t" + "subu.ph %[res_oq1], %[res_oq1], %[p6] \n\t" + "shrl.ph %[res_oq1], %[res_oq1], 4 \n\t" + + /* *oq2 = ROUND_POWER_OF_TWO(p4 + p3 + p2 + p1 + p0 + q0 + q1 + q2 * 2 + + q3 + q4 + q5 + q6 + q7 * 3, 4) */ + "shll.ph %[tmp], %[q7], 1 \n\t" + "addu.ph %[res_oq2], %[tmp], %[q7] \n\t" + "addu.ph %[res_oq2], %[res_oq2], %[q2] \n\t" + "addu.ph %[res_oq2], %[res_oq2], %[add_p6toq6] \n\t" + "subu.ph %[res_oq2], %[res_oq2], %[p5] \n\t" + "subu.ph %[res_oq2], %[res_oq2], %[p6] \n\t" + "shrl.ph %[res_oq2], %[res_oq2], 4 \n\t" + + /* *oq3 = ROUND_POWER_OF_TWO(p3 + p2 + p1 + p0 + q0 + q1 + q2 + + q3 * 2 + q4 + q5 + q6 + q7 * 4, 4) */ + "shll.ph %[tmp], %[q7], 2 \n\t" + "addu.ph %[res_oq3], %[tmp], %[q3] \n\t" + "addu.ph %[res_oq3], %[res_oq3], %[add_p6toq6] \n\t" + "subu.ph %[res_oq3], %[res_oq3], %[p4] \n\t" + "subu.ph %[res_oq3], %[res_oq3], %[p5] \n\t" + "subu.ph %[res_oq3], %[res_oq3], %[p6] \n\t" + "shrl.ph %[res_oq3], %[res_oq3], 4 \n\t" + + /* *oq4 = ROUND_POWER_OF_TWO(p2 + p1 + p0 + q0 + q1 + q2 + q3 + + q4 * 2 + q5 + q6 + q7 * 5, 4) */ + "shll.ph %[tmp], %[q7], 2 \n\t" + "addu.ph %[res_oq4], %[tmp], %[q7] \n\t" + "addu.ph %[res_oq4], %[res_oq4], %[q4] \n\t" + "addu.ph %[res_oq4], %[res_oq4], %[add_p6toq6] \n\t" + "subu.ph %[res_oq4], %[res_oq4], %[p3] \n\t" + "subu.ph %[res_oq4], %[res_oq4], %[p4] \n\t" + "subu.ph %[res_oq4], %[res_oq4], %[p5] \n\t" + "subu.ph %[res_oq4], %[res_oq4], %[p6] \n\t" + "shrl.ph %[res_oq4], %[res_oq4], 4 \n\t" + + /* *oq5 = ROUND_POWER_OF_TWO(p1 + p0 + q0 + q1 + q2 + q3 + q4 + + q5 * 2 + q6 + q7 * 6, 4) */ + "shll.ph %[tmp], %[q7], 2 \n\t" + "addu.ph %[res_oq5], %[tmp], %[q7] \n\t" + "addu.ph %[res_oq5], %[res_oq5], %[q7] \n\t" + "addu.ph %[res_oq5], %[res_oq5], %[q5] \n\t" + "addu.ph %[res_oq5], %[res_oq5], %[add_p6toq6] \n\t" + "subu.ph %[res_oq5], %[res_oq5], %[p2] \n\t" + "subu.ph %[res_oq5], %[res_oq5], %[p3] \n\t" + "subu.ph %[res_oq5], %[res_oq5], %[p4] \n\t" + "subu.ph %[res_oq5], %[res_oq5], %[p5] \n\t" + "subu.ph %[res_oq5], %[res_oq5], %[p6] \n\t" + "shrl.ph %[res_oq5], %[res_oq5], 4 \n\t" + + /* *oq6 = ROUND_POWER_OF_TWO(p0 + q0 + q1 + q2 + q3 + + q4 + q5 + q6 * 2 + q7 * 7, 4) */ + "shll.ph %[tmp], %[q7], 3 \n\t" + "subu.ph %[res_oq6], %[tmp], %[q7] \n\t" + "addu.ph %[res_oq6], %[res_oq6], %[q6] \n\t" + "addu.ph %[res_oq6], %[res_oq6], %[add_p6toq6] \n\t" + "subu.ph %[res_oq6], %[res_oq6], %[p1] \n\t" + "subu.ph %[res_oq6], %[res_oq6], %[p2] \n\t" + "subu.ph %[res_oq6], %[res_oq6], %[p3] \n\t" + "subu.ph %[res_oq6], %[res_oq6], %[p4] \n\t" + "subu.ph %[res_oq6], %[res_oq6], %[p5] \n\t" + "subu.ph %[res_oq6], %[res_oq6], %[p6] \n\t" + "shrl.ph %[res_oq6], %[res_oq6], 4 \n\t" + + : [res_oq6] "=&r"(res_oq6), [res_oq5] "=&r"(res_oq5), + [res_oq4] "=&r"(res_oq4), [res_oq3] "=&r"(res_oq3), + [res_oq2] "=&r"(res_oq2), [res_oq1] "=&r"(res_oq1), + [res_oq0] "=&r"(res_oq0), [tmp] "=&r"(tmp) + : [q7] "r"(q7), [q6] "r"(q6), [q5] "r"(q5), [q4] "r"(q4), [q3] "r"(q3), + [q2] "r"(q2), [q1] "r"(q1), [q0] "r"(q0), [p1] "r"(p1), [p2] "r"(p2), + [p3] "r"(p3), [p4] "r"(p4), [p5] "r"(p5), [p6] "r"(p6), + [add_p6toq6] "r"(add_p6toq6)); + + *oq0 = res_oq0; + *oq1 = res_oq1; + *oq2 = res_oq2; + *oq3 = res_oq3; + *oq4 = res_oq4; + *oq5 = res_oq5; + *oq6 = res_oq6; +} +#endif // #if HAVE_DSPR2 +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_MIPS_LOOPFILTER_FILTERS_DSPR2_H_ diff --git a/third_party/aom/aom_dsp/mips/loopfilter_macros_dspr2.h b/third_party/aom/aom_dsp/mips/loopfilter_macros_dspr2.h new file mode 100644 index 0000000000..3e69947144 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/loopfilter_macros_dspr2.h @@ -0,0 +1,436 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_MIPS_LOOPFILTER_MACROS_DSPR2_H_ +#define AOM_DSP_MIPS_LOOPFILTER_MACROS_DSPR2_H_ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_mem/aom_mem.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if HAVE_DSPR2 +#define STORE_F0() \ + { \ + __asm__ __volatile__( \ + "sb %[q1_f0], 1(%[s4]) \n\t" \ + "sb %[q0_f0], 0(%[s4]) \n\t" \ + "sb %[p0_f0], -1(%[s4]) \n\t" \ + "sb %[p1_f0], -2(%[s4]) \n\t" \ + \ + : \ + : [q1_f0] "r"(q1_f0), [q0_f0] "r"(q0_f0), [p0_f0] "r"(p0_f0), \ + [p1_f0] "r"(p1_f0), [s4] "r"(s4)); \ + \ + __asm__ __volatile__( \ + "srl %[q1_f0], %[q1_f0], 8 \n\t" \ + "srl %[q0_f0], %[q0_f0], 8 \n\t" \ + "srl %[p0_f0], %[p0_f0], 8 \n\t" \ + "srl %[p1_f0], %[p1_f0], 8 \n\t" \ + \ + : [q1_f0] "+r"(q1_f0), [q0_f0] "+r"(q0_f0), [p0_f0] "+r"(p0_f0), \ + [p1_f0] "+r"(p1_f0) \ + :); \ + \ + __asm__ __volatile__( \ + "sb %[q1_f0], 1(%[s3]) \n\t" \ + "sb %[q0_f0], 0(%[s3]) \n\t" \ + "sb %[p0_f0], -1(%[s3]) \n\t" \ + "sb %[p1_f0], -2(%[s3]) \n\t" \ + \ + : [p1_f0] "+r"(p1_f0) \ + : [q1_f0] "r"(q1_f0), [q0_f0] "r"(q0_f0), [s3] "r"(s3), \ + [p0_f0] "r"(p0_f0)); \ + \ + __asm__ __volatile__( \ + "srl %[q1_f0], %[q1_f0], 8 \n\t" \ + "srl %[q0_f0], %[q0_f0], 8 \n\t" \ + "srl %[p0_f0], %[p0_f0], 8 \n\t" \ + "srl %[p1_f0], %[p1_f0], 8 \n\t" \ + \ + : [q1_f0] "+r"(q1_f0), [q0_f0] "+r"(q0_f0), [p0_f0] "+r"(p0_f0), \ + [p1_f0] "+r"(p1_f0) \ + :); \ + \ + __asm__ __volatile__( \ + "sb %[q1_f0], 1(%[s2]) \n\t" \ + "sb %[q0_f0], 0(%[s2]) \n\t" \ + "sb %[p0_f0], -1(%[s2]) \n\t" \ + "sb %[p1_f0], -2(%[s2]) \n\t" \ + \ + : \ + : [q1_f0] "r"(q1_f0), [q0_f0] "r"(q0_f0), [p0_f0] "r"(p0_f0), \ + [p1_f0] "r"(p1_f0), [s2] "r"(s2)); \ + \ + __asm__ __volatile__( \ + "srl %[q1_f0], %[q1_f0], 8 \n\t" \ + "srl %[q0_f0], %[q0_f0], 8 \n\t" \ + "srl %[p0_f0], %[p0_f0], 8 \n\t" \ + "srl %[p1_f0], %[p1_f0], 8 \n\t" \ + \ + : [q1_f0] "+r"(q1_f0), [q0_f0] "+r"(q0_f0), [p0_f0] "+r"(p0_f0), \ + [p1_f0] "+r"(p1_f0) \ + :); \ + \ + __asm__ __volatile__( \ + "sb %[q1_f0], 1(%[s1]) \n\t" \ + "sb %[q0_f0], 0(%[s1]) \n\t" \ + "sb %[p0_f0], -1(%[s1]) \n\t" \ + "sb %[p1_f0], -2(%[s1]) \n\t" \ + \ + : \ + : [q1_f0] "r"(q1_f0), [q0_f0] "r"(q0_f0), [p0_f0] "r"(p0_f0), \ + [p1_f0] "r"(p1_f0), [s1] "r"(s1)); \ + } + +#define STORE_F1() \ + { \ + __asm__ __volatile__( \ + "sb %[q2_r], 2(%[s4]) \n\t" \ + "sb %[q1_r], 1(%[s4]) \n\t" \ + "sb %[q0_r], 0(%[s4]) \n\t" \ + "sb %[p0_r], -1(%[s4]) \n\t" \ + "sb %[p1_r], -2(%[s4]) \n\t" \ + "sb %[p2_r], -3(%[s4]) \n\t" \ + \ + : \ + : [q2_r] "r"(q2_r), [q1_r] "r"(q1_r), [q0_r] "r"(q0_r), \ + [p0_r] "r"(p0_r), [p1_r] "r"(p1_r), [p2_r] "r"(p2_r), [s4] "r"(s4)); \ + \ + __asm__ __volatile__( \ + "srl %[q2_r], %[q2_r], 16 \n\t" \ + "srl %[q1_r], %[q1_r], 16 \n\t" \ + "srl %[q0_r], %[q0_r], 16 \n\t" \ + "srl %[p0_r], %[p0_r], 16 \n\t" \ + "srl %[p1_r], %[p1_r], 16 \n\t" \ + "srl %[p2_r], %[p2_r], 16 \n\t" \ + \ + : [q2_r] "+r"(q2_r), [q1_r] "+r"(q1_r), [q0_r] "+r"(q0_r), \ + [p0_r] "+r"(p0_r), [p1_r] "+r"(p1_r), [p2_r] "+r"(p2_r) \ + :); \ + \ + __asm__ __volatile__( \ + "sb %[q2_r], 2(%[s3]) \n\t" \ + "sb %[q1_r], 1(%[s3]) \n\t" \ + "sb %[q0_r], 0(%[s3]) \n\t" \ + "sb %[p0_r], -1(%[s3]) \n\t" \ + "sb %[p1_r], -2(%[s3]) \n\t" \ + "sb %[p2_r], -3(%[s3]) \n\t" \ + \ + : \ + : [q2_r] "r"(q2_r), [q1_r] "r"(q1_r), [q0_r] "r"(q0_r), \ + [p0_r] "r"(p0_r), [p1_r] "r"(p1_r), [p2_r] "r"(p2_r), [s3] "r"(s3)); \ + \ + __asm__ __volatile__( \ + "sb %[q2_l], 2(%[s2]) \n\t" \ + "sb %[q1_l], 1(%[s2]) \n\t" \ + "sb %[q0_l], 0(%[s2]) \n\t" \ + "sb %[p0_l], -1(%[s2]) \n\t" \ + "sb %[p1_l], -2(%[s2]) \n\t" \ + "sb %[p2_l], -3(%[s2]) \n\t" \ + \ + : \ + : [q2_l] "r"(q2_l), [q1_l] "r"(q1_l), [q0_l] "r"(q0_l), \ + [p0_l] "r"(p0_l), [p1_l] "r"(p1_l), [p2_l] "r"(p2_l), [s2] "r"(s2)); \ + \ + __asm__ __volatile__( \ + "srl %[q2_l], %[q2_l], 16 \n\t" \ + "srl %[q1_l], %[q1_l], 16 \n\t" \ + "srl %[q0_l], %[q0_l], 16 \n\t" \ + "srl %[p0_l], %[p0_l], 16 \n\t" \ + "srl %[p1_l], %[p1_l], 16 \n\t" \ + "srl %[p2_l], %[p2_l], 16 \n\t" \ + \ + : [q2_l] "+r"(q2_l), [q1_l] "+r"(q1_l), [q0_l] "+r"(q0_l), \ + [p0_l] "+r"(p0_l), [p1_l] "+r"(p1_l), [p2_l] "+r"(p2_l) \ + :); \ + \ + __asm__ __volatile__( \ + "sb %[q2_l], 2(%[s1]) \n\t" \ + "sb %[q1_l], 1(%[s1]) \n\t" \ + "sb %[q0_l], 0(%[s1]) \n\t" \ + "sb %[p0_l], -1(%[s1]) \n\t" \ + "sb %[p1_l], -2(%[s1]) \n\t" \ + "sb %[p2_l], -3(%[s1]) \n\t" \ + \ + : \ + : [q2_l] "r"(q2_l), [q1_l] "r"(q1_l), [q0_l] "r"(q0_l), \ + [p0_l] "r"(p0_l), [p1_l] "r"(p1_l), [p2_l] "r"(p2_l), [s1] "r"(s1)); \ + } + +#define STORE_F2() \ + { \ + __asm__ __volatile__( \ + "sb %[q6_r], 6(%[s4]) \n\t" \ + "sb %[q5_r], 5(%[s4]) \n\t" \ + "sb %[q4_r], 4(%[s4]) \n\t" \ + "sb %[q3_r], 3(%[s4]) \n\t" \ + "sb %[q2_r], 2(%[s4]) \n\t" \ + "sb %[q1_r], 1(%[s4]) \n\t" \ + "sb %[q0_r], 0(%[s4]) \n\t" \ + "sb %[p0_r], -1(%[s4]) \n\t" \ + "sb %[p1_r], -2(%[s4]) \n\t" \ + "sb %[p2_r], -3(%[s4]) \n\t" \ + "sb %[p3_r], -4(%[s4]) \n\t" \ + "sb %[p4_r], -5(%[s4]) \n\t" \ + "sb %[p5_r], -6(%[s4]) \n\t" \ + "sb %[p6_r], -7(%[s4]) \n\t" \ + \ + : \ + : [q6_r] "r"(q6_r), [q5_r] "r"(q5_r), [q4_r] "r"(q4_r), \ + [q3_r] "r"(q3_r), [q2_r] "r"(q2_r), [q1_r] "r"(q1_r), \ + [q0_r] "r"(q0_r), [p0_r] "r"(p0_r), [p1_r] "r"(p1_r), \ + [p2_r] "r"(p2_r), [p3_r] "r"(p3_r), [p4_r] "r"(p4_r), \ + [p5_r] "r"(p5_r), [p6_r] "r"(p6_r), [s4] "r"(s4)); \ + \ + __asm__ __volatile__( \ + "srl %[q6_r], %[q6_r], 16 \n\t" \ + "srl %[q5_r], %[q5_r], 16 \n\t" \ + "srl %[q4_r], %[q4_r], 16 \n\t" \ + "srl %[q3_r], %[q3_r], 16 \n\t" \ + "srl %[q2_r], %[q2_r], 16 \n\t" \ + "srl %[q1_r], %[q1_r], 16 \n\t" \ + "srl %[q0_r], %[q0_r], 16 \n\t" \ + "srl %[p0_r], %[p0_r], 16 \n\t" \ + "srl %[p1_r], %[p1_r], 16 \n\t" \ + "srl %[p2_r], %[p2_r], 16 \n\t" \ + "srl %[p3_r], %[p3_r], 16 \n\t" \ + "srl %[p4_r], %[p4_r], 16 \n\t" \ + "srl %[p5_r], %[p5_r], 16 \n\t" \ + "srl %[p6_r], %[p6_r], 16 \n\t" \ + \ + : [q6_r] "+r"(q6_r), [q5_r] "+r"(q5_r), [q4_r] "+r"(q4_r), \ + [q3_r] "+r"(q3_r), [q2_r] "+r"(q2_r), [q1_r] "+r"(q1_r), \ + [q0_r] "+r"(q0_r), [p0_r] "+r"(p0_r), [p1_r] "+r"(p1_r), \ + [p2_r] "+r"(p2_r), [p3_r] "+r"(p3_r), [p4_r] "+r"(p4_r), \ + [p5_r] "+r"(p5_r), [p6_r] "+r"(p6_r) \ + :); \ + \ + __asm__ __volatile__( \ + "sb %[q6_r], 6(%[s3]) \n\t" \ + "sb %[q5_r], 5(%[s3]) \n\t" \ + "sb %[q4_r], 4(%[s3]) \n\t" \ + "sb %[q3_r], 3(%[s3]) \n\t" \ + "sb %[q2_r], 2(%[s3]) \n\t" \ + "sb %[q1_r], 1(%[s3]) \n\t" \ + "sb %[q0_r], 0(%[s3]) \n\t" \ + "sb %[p0_r], -1(%[s3]) \n\t" \ + "sb %[p1_r], -2(%[s3]) \n\t" \ + "sb %[p2_r], -3(%[s3]) \n\t" \ + "sb %[p3_r], -4(%[s3]) \n\t" \ + "sb %[p4_r], -5(%[s3]) \n\t" \ + "sb %[p5_r], -6(%[s3]) \n\t" \ + "sb %[p6_r], -7(%[s3]) \n\t" \ + \ + : \ + : [q6_r] "r"(q6_r), [q5_r] "r"(q5_r), [q4_r] "r"(q4_r), \ + [q3_r] "r"(q3_r), [q2_r] "r"(q2_r), [q1_r] "r"(q1_r), \ + [q0_r] "r"(q0_r), [p0_r] "r"(p0_r), [p1_r] "r"(p1_r), \ + [p2_r] "r"(p2_r), [p3_r] "r"(p3_r), [p4_r] "r"(p4_r), \ + [p5_r] "r"(p5_r), [p6_r] "r"(p6_r), [s3] "r"(s3)); \ + \ + __asm__ __volatile__( \ + "sb %[q6_l], 6(%[s2]) \n\t" \ + "sb %[q5_l], 5(%[s2]) \n\t" \ + "sb %[q4_l], 4(%[s2]) \n\t" \ + "sb %[q3_l], 3(%[s2]) \n\t" \ + "sb %[q2_l], 2(%[s2]) \n\t" \ + "sb %[q1_l], 1(%[s2]) \n\t" \ + "sb %[q0_l], 0(%[s2]) \n\t" \ + "sb %[p0_l], -1(%[s2]) \n\t" \ + "sb %[p1_l], -2(%[s2]) \n\t" \ + "sb %[p2_l], -3(%[s2]) \n\t" \ + "sb %[p3_l], -4(%[s2]) \n\t" \ + "sb %[p4_l], -5(%[s2]) \n\t" \ + "sb %[p5_l], -6(%[s2]) \n\t" \ + "sb %[p6_l], -7(%[s2]) \n\t" \ + \ + : \ + : [q6_l] "r"(q6_l), [q5_l] "r"(q5_l), [q4_l] "r"(q4_l), \ + [q3_l] "r"(q3_l), [q2_l] "r"(q2_l), [q1_l] "r"(q1_l), \ + [q0_l] "r"(q0_l), [p0_l] "r"(p0_l), [p1_l] "r"(p1_l), \ + [p2_l] "r"(p2_l), [p3_l] "r"(p3_l), [p4_l] "r"(p4_l), \ + [p5_l] "r"(p5_l), [p6_l] "r"(p6_l), [s2] "r"(s2)); \ + \ + __asm__ __volatile__( \ + "srl %[q6_l], %[q6_l], 16 \n\t" \ + "srl %[q5_l], %[q5_l], 16 \n\t" \ + "srl %[q4_l], %[q4_l], 16 \n\t" \ + "srl %[q3_l], %[q3_l], 16 \n\t" \ + "srl %[q2_l], %[q2_l], 16 \n\t" \ + "srl %[q1_l], %[q1_l], 16 \n\t" \ + "srl %[q0_l], %[q0_l], 16 \n\t" \ + "srl %[p0_l], %[p0_l], 16 \n\t" \ + "srl %[p1_l], %[p1_l], 16 \n\t" \ + "srl %[p2_l], %[p2_l], 16 \n\t" \ + "srl %[p3_l], %[p3_l], 16 \n\t" \ + "srl %[p4_l], %[p4_l], 16 \n\t" \ + "srl %[p5_l], %[p5_l], 16 \n\t" \ + "srl %[p6_l], %[p6_l], 16 \n\t" \ + \ + : [q6_l] "+r"(q6_l), [q5_l] "+r"(q5_l), [q4_l] "+r"(q4_l), \ + [q3_l] "+r"(q3_l), [q2_l] "+r"(q2_l), [q1_l] "+r"(q1_l), \ + [q0_l] "+r"(q0_l), [p0_l] "+r"(p0_l), [p1_l] "+r"(p1_l), \ + [p2_l] "+r"(p2_l), [p3_l] "+r"(p3_l), [p4_l] "+r"(p4_l), \ + [p5_l] "+r"(p5_l), [p6_l] "+r"(p6_l) \ + :); \ + \ + __asm__ __volatile__( \ + "sb %[q6_l], 6(%[s1]) \n\t" \ + "sb %[q5_l], 5(%[s1]) \n\t" \ + "sb %[q4_l], 4(%[s1]) \n\t" \ + "sb %[q3_l], 3(%[s1]) \n\t" \ + "sb %[q2_l], 2(%[s1]) \n\t" \ + "sb %[q1_l], 1(%[s1]) \n\t" \ + "sb %[q0_l], 0(%[s1]) \n\t" \ + "sb %[p0_l], -1(%[s1]) \n\t" \ + "sb %[p1_l], -2(%[s1]) \n\t" \ + "sb %[p2_l], -3(%[s1]) \n\t" \ + "sb %[p3_l], -4(%[s1]) \n\t" \ + "sb %[p4_l], -5(%[s1]) \n\t" \ + "sb %[p5_l], -6(%[s1]) \n\t" \ + "sb %[p6_l], -7(%[s1]) \n\t" \ + \ + : \ + : [q6_l] "r"(q6_l), [q5_l] "r"(q5_l), [q4_l] "r"(q4_l), \ + [q3_l] "r"(q3_l), [q2_l] "r"(q2_l), [q1_l] "r"(q1_l), \ + [q0_l] "r"(q0_l), [p0_l] "r"(p0_l), [p1_l] "r"(p1_l), \ + [p2_l] "r"(p2_l), [p3_l] "r"(p3_l), [p4_l] "r"(p4_l), \ + [p5_l] "r"(p5_l), [p6_l] "r"(p6_l), [s1] "r"(s1)); \ + } + +#define PACK_LEFT_0TO3() \ + { \ + __asm__ __volatile__( \ + "preceu.ph.qbl %[p3_l], %[p3] \n\t" \ + "preceu.ph.qbl %[p2_l], %[p2] \n\t" \ + "preceu.ph.qbl %[p1_l], %[p1] \n\t" \ + "preceu.ph.qbl %[p0_l], %[p0] \n\t" \ + "preceu.ph.qbl %[q0_l], %[q0] \n\t" \ + "preceu.ph.qbl %[q1_l], %[q1] \n\t" \ + "preceu.ph.qbl %[q2_l], %[q2] \n\t" \ + "preceu.ph.qbl %[q3_l], %[q3] \n\t" \ + \ + : [p3_l] "=&r"(p3_l), [p2_l] "=&r"(p2_l), [p1_l] "=&r"(p1_l), \ + [p0_l] "=&r"(p0_l), [q0_l] "=&r"(q0_l), [q1_l] "=&r"(q1_l), \ + [q2_l] "=&r"(q2_l), [q3_l] "=&r"(q3_l) \ + : [p3] "r"(p3), [p2] "r"(p2), [p1] "r"(p1), [p0] "r"(p0), \ + [q0] "r"(q0), [q1] "r"(q1), [q2] "r"(q2), [q3] "r"(q3)); \ + } + +#define PACK_LEFT_4TO7() \ + { \ + __asm__ __volatile__( \ + "preceu.ph.qbl %[p7_l], %[p7] \n\t" \ + "preceu.ph.qbl %[p6_l], %[p6] \n\t" \ + "preceu.ph.qbl %[p5_l], %[p5] \n\t" \ + "preceu.ph.qbl %[p4_l], %[p4] \n\t" \ + "preceu.ph.qbl %[q4_l], %[q4] \n\t" \ + "preceu.ph.qbl %[q5_l], %[q5] \n\t" \ + "preceu.ph.qbl %[q6_l], %[q6] \n\t" \ + "preceu.ph.qbl %[q7_l], %[q7] \n\t" \ + \ + : [p7_l] "=&r"(p7_l), [p6_l] "=&r"(p6_l), [p5_l] "=&r"(p5_l), \ + [p4_l] "=&r"(p4_l), [q4_l] "=&r"(q4_l), [q5_l] "=&r"(q5_l), \ + [q6_l] "=&r"(q6_l), [q7_l] "=&r"(q7_l) \ + : [p7] "r"(p7), [p6] "r"(p6), [p5] "r"(p5), [p4] "r"(p4), \ + [q4] "r"(q4), [q5] "r"(q5), [q6] "r"(q6), [q7] "r"(q7)); \ + } + +#define PACK_RIGHT_0TO3() \ + { \ + __asm__ __volatile__( \ + "preceu.ph.qbr %[p3_r], %[p3] \n\t" \ + "preceu.ph.qbr %[p2_r], %[p2] \n\t" \ + "preceu.ph.qbr %[p1_r], %[p1] \n\t" \ + "preceu.ph.qbr %[p0_r], %[p0] \n\t" \ + "preceu.ph.qbr %[q0_r], %[q0] \n\t" \ + "preceu.ph.qbr %[q1_r], %[q1] \n\t" \ + "preceu.ph.qbr %[q2_r], %[q2] \n\t" \ + "preceu.ph.qbr %[q3_r], %[q3] \n\t" \ + \ + : [p3_r] "=&r"(p3_r), [p2_r] "=&r"(p2_r), [p1_r] "=&r"(p1_r), \ + [p0_r] "=&r"(p0_r), [q0_r] "=&r"(q0_r), [q1_r] "=&r"(q1_r), \ + [q2_r] "=&r"(q2_r), [q3_r] "=&r"(q3_r) \ + : [p3] "r"(p3), [p2] "r"(p2), [p1] "r"(p1), [p0] "r"(p0), \ + [q0] "r"(q0), [q1] "r"(q1), [q2] "r"(q2), [q3] "r"(q3)); \ + } + +#define PACK_RIGHT_4TO7() \ + { \ + __asm__ __volatile__( \ + "preceu.ph.qbr %[p7_r], %[p7] \n\t" \ + "preceu.ph.qbr %[p6_r], %[p6] \n\t" \ + "preceu.ph.qbr %[p5_r], %[p5] \n\t" \ + "preceu.ph.qbr %[p4_r], %[p4] \n\t" \ + "preceu.ph.qbr %[q4_r], %[q4] \n\t" \ + "preceu.ph.qbr %[q5_r], %[q5] \n\t" \ + "preceu.ph.qbr %[q6_r], %[q6] \n\t" \ + "preceu.ph.qbr %[q7_r], %[q7] \n\t" \ + \ + : [p7_r] "=&r"(p7_r), [p6_r] "=&r"(p6_r), [p5_r] "=&r"(p5_r), \ + [p4_r] "=&r"(p4_r), [q4_r] "=&r"(q4_r), [q5_r] "=&r"(q5_r), \ + [q6_r] "=&r"(q6_r), [q7_r] "=&r"(q7_r) \ + : [p7] "r"(p7), [p6] "r"(p6), [p5] "r"(p5), [p4] "r"(p4), \ + [q4] "r"(q4), [q5] "r"(q5), [q6] "r"(q6), [q7] "r"(q7)); \ + } + +#define COMBINE_LEFT_RIGHT_0TO2() \ + { \ + __asm__ __volatile__( \ + "precr.qb.ph %[p2], %[p2_l], %[p2_r] \n\t" \ + "precr.qb.ph %[p1], %[p1_l], %[p1_r] \n\t" \ + "precr.qb.ph %[p0], %[p0_l], %[p0_r] \n\t" \ + "precr.qb.ph %[q0], %[q0_l], %[q0_r] \n\t" \ + "precr.qb.ph %[q1], %[q1_l], %[q1_r] \n\t" \ + "precr.qb.ph %[q2], %[q2_l], %[q2_r] \n\t" \ + \ + : [p2] "=&r"(p2), [p1] "=&r"(p1), [p0] "=&r"(p0), [q0] "=&r"(q0), \ + [q1] "=&r"(q1), [q2] "=&r"(q2) \ + : [p2_l] "r"(p2_l), [p2_r] "r"(p2_r), [p1_l] "r"(p1_l), \ + [p1_r] "r"(p1_r), [p0_l] "r"(p0_l), [p0_r] "r"(p0_r), \ + [q0_l] "r"(q0_l), [q0_r] "r"(q0_r), [q1_l] "r"(q1_l), \ + [q1_r] "r"(q1_r), [q2_l] "r"(q2_l), [q2_r] "r"(q2_r)); \ + } + +#define COMBINE_LEFT_RIGHT_3TO6() \ + { \ + __asm__ __volatile__( \ + "precr.qb.ph %[p6], %[p6_l], %[p6_r] \n\t" \ + "precr.qb.ph %[p5], %[p5_l], %[p5_r] \n\t" \ + "precr.qb.ph %[p4], %[p4_l], %[p4_r] \n\t" \ + "precr.qb.ph %[p3], %[p3_l], %[p3_r] \n\t" \ + "precr.qb.ph %[q3], %[q3_l], %[q3_r] \n\t" \ + "precr.qb.ph %[q4], %[q4_l], %[q4_r] \n\t" \ + "precr.qb.ph %[q5], %[q5_l], %[q5_r] \n\t" \ + "precr.qb.ph %[q6], %[q6_l], %[q6_r] \n\t" \ + \ + : [p6] "=&r"(p6), [p5] "=&r"(p5), [p4] "=&r"(p4), [p3] "=&r"(p3), \ + [q3] "=&r"(q3), [q4] "=&r"(q4), [q5] "=&r"(q5), [q6] "=&r"(q6) \ + : [p6_l] "r"(p6_l), [p5_l] "r"(p5_l), [p4_l] "r"(p4_l), \ + [p3_l] "r"(p3_l), [p6_r] "r"(p6_r), [p5_r] "r"(p5_r), \ + [p4_r] "r"(p4_r), [p3_r] "r"(p3_r), [q3_l] "r"(q3_l), \ + [q4_l] "r"(q4_l), [q5_l] "r"(q5_l), [q6_l] "r"(q6_l), \ + [q3_r] "r"(q3_r), [q4_r] "r"(q4_r), [q5_r] "r"(q5_r), \ + [q6_r] "r"(q6_r)); \ + } + +#endif // #if HAVE_DSPR2 +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_MIPS_LOOPFILTER_MACROS_DSPR2_H_ diff --git a/third_party/aom/aom_dsp/mips/loopfilter_masks_dspr2.h b/third_party/aom/aom_dsp/mips/loopfilter_masks_dspr2.h new file mode 100644 index 0000000000..8db3e521f6 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/loopfilter_masks_dspr2.h @@ -0,0 +1,356 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_MIPS_LOOPFILTER_MASKS_DSPR2_H_ +#define AOM_DSP_MIPS_LOOPFILTER_MASKS_DSPR2_H_ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_mem/aom_mem.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if HAVE_DSPR2 +/* processing 4 pixels at the same time + * compute hev and mask in the same function */ +static INLINE void filter_hev_mask_dspr2(uint32_t limit, uint32_t flimit, + uint32_t p1, uint32_t p0, uint32_t p3, + uint32_t p2, uint32_t q0, uint32_t q1, + uint32_t q2, uint32_t q3, + uint32_t thresh, uint32_t *hev, + uint32_t *mask) { + uint32_t c, r, r3, r_k; + uint32_t s1, s2, s3; + uint32_t ones = 0xFFFFFFFF; + uint32_t hev1; + + __asm__ __volatile__( + /* mask |= (abs(p3 - p2) > limit) */ + "subu_s.qb %[c], %[p3], %[p2] \n\t" + "subu_s.qb %[r_k], %[p2], %[p3] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[limit], %[r_k] \n\t" + "or %[r], $0, %[c] \n\t" + + /* mask |= (abs(p2 - p1) > limit) */ + "subu_s.qb %[c], %[p2], %[p1] \n\t" + "subu_s.qb %[r_k], %[p1], %[p2] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[limit], %[r_k] \n\t" + "or %[r], %[r], %[c] \n\t" + + /* mask |= (abs(p1 - p0) > limit) + * hev |= (abs(p1 - p0) > thresh) + */ + "subu_s.qb %[c], %[p1], %[p0] \n\t" + "subu_s.qb %[r_k], %[p0], %[p1] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[thresh], %[r_k] \n\t" + "or %[r3], $0, %[c] \n\t" + "cmpgu.lt.qb %[c], %[limit], %[r_k] \n\t" + "or %[r], %[r], %[c] \n\t" + + /* mask |= (abs(q1 - q0) > limit) + * hev |= (abs(q1 - q0) > thresh) + */ + "subu_s.qb %[c], %[q1], %[q0] \n\t" + "subu_s.qb %[r_k], %[q0], %[q1] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[thresh], %[r_k] \n\t" + "or %[r3], %[r3], %[c] \n\t" + "cmpgu.lt.qb %[c], %[limit], %[r_k] \n\t" + "or %[r], %[r], %[c] \n\t" + + /* mask |= (abs(q2 - q1) > limit) */ + "subu_s.qb %[c], %[q2], %[q1] \n\t" + "subu_s.qb %[r_k], %[q1], %[q2] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[limit], %[r_k] \n\t" + "or %[r], %[r], %[c] \n\t" + "sll %[r3], %[r3], 24 \n\t" + + /* mask |= (abs(q3 - q2) > limit) */ + "subu_s.qb %[c], %[q3], %[q2] \n\t" + "subu_s.qb %[r_k], %[q2], %[q3] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[limit], %[r_k] \n\t" + "or %[r], %[r], %[c] \n\t" + + : [c] "=&r"(c), [r_k] "=&r"(r_k), [r] "=&r"(r), [r3] "=&r"(r3) + : [limit] "r"(limit), [p3] "r"(p3), [p2] "r"(p2), [p1] "r"(p1), + [p0] "r"(p0), [q1] "r"(q1), [q0] "r"(q0), [q2] "r"(q2), [q3] "r"(q3), + [thresh] "r"(thresh)); + + __asm__ __volatile__( + /* abs(p0 - q0) */ + "subu_s.qb %[c], %[p0], %[q0] \n\t" + "subu_s.qb %[r_k], %[q0], %[p0] \n\t" + "wrdsp %[r3] \n\t" + "or %[s1], %[r_k], %[c] \n\t" + + /* abs(p1 - q1) */ + "subu_s.qb %[c], %[p1], %[q1] \n\t" + "addu_s.qb %[s3], %[s1], %[s1] \n\t" + "pick.qb %[hev1], %[ones], $0 \n\t" + "subu_s.qb %[r_k], %[q1], %[p1] \n\t" + "or %[s2], %[r_k], %[c] \n\t" + + /* abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > flimit * 2 + limit */ + "shrl.qb %[s2], %[s2], 1 \n\t" + "addu_s.qb %[s1], %[s2], %[s3] \n\t" + "cmpgu.lt.qb %[c], %[flimit], %[s1] \n\t" + "or %[r], %[r], %[c] \n\t" + "sll %[r], %[r], 24 \n\t" + + "wrdsp %[r] \n\t" + "pick.qb %[s2], $0, %[ones] \n\t" + + : [c] "=&r"(c), [r_k] "=&r"(r_k), [s1] "=&r"(s1), [hev1] "=&r"(hev1), + [s2] "=&r"(s2), [r] "+r"(r), [s3] "=&r"(s3) + : [p0] "r"(p0), [q0] "r"(q0), [p1] "r"(p1), [r3] "r"(r3), [q1] "r"(q1), + [ones] "r"(ones), [flimit] "r"(flimit)); + + *hev = hev1; + *mask = s2; +} + +static INLINE void filter_hev_mask_flatmask4_dspr2( + uint32_t limit, uint32_t flimit, uint32_t thresh, uint32_t p1, uint32_t p0, + uint32_t p3, uint32_t p2, uint32_t q0, uint32_t q1, uint32_t q2, + uint32_t q3, uint32_t *hev, uint32_t *mask, uint32_t *flat) { + uint32_t c, r, r3, r_k, r_flat; + uint32_t s1, s2, s3; + uint32_t ones = 0xFFFFFFFF; + uint32_t flat_thresh = 0x01010101; + uint32_t hev1; + uint32_t flat1; + + __asm__ __volatile__( + /* mask |= (abs(p3 - p2) > limit) */ + "subu_s.qb %[c], %[p3], %[p2] \n\t" + "subu_s.qb %[r_k], %[p2], %[p3] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[limit], %[r_k] \n\t" + "or %[r], $0, %[c] \n\t" + + /* mask |= (abs(p2 - p1) > limit) */ + "subu_s.qb %[c], %[p2], %[p1] \n\t" + "subu_s.qb %[r_k], %[p1], %[p2] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[limit], %[r_k] \n\t" + "or %[r], %[r], %[c] \n\t" + + /* mask |= (abs(p1 - p0) > limit) + * hev |= (abs(p1 - p0) > thresh) + * flat |= (abs(p1 - p0) > thresh) + */ + "subu_s.qb %[c], %[p1], %[p0] \n\t" + "subu_s.qb %[r_k], %[p0], %[p1] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[thresh], %[r_k] \n\t" + "or %[r3], $0, %[c] \n\t" + "cmpgu.lt.qb %[c], %[limit], %[r_k] \n\t" + "or %[r], %[r], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r_flat], $0, %[c] \n\t" + + /* mask |= (abs(q1 - q0) > limit) + * hev |= (abs(q1 - q0) > thresh) + * flat |= (abs(q1 - q0) > thresh) + */ + "subu_s.qb %[c], %[q1], %[q0] \n\t" + "subu_s.qb %[r_k], %[q0], %[q1] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[thresh], %[r_k] \n\t" + "or %[r3], %[r3], %[c] \n\t" + "cmpgu.lt.qb %[c], %[limit], %[r_k] \n\t" + "or %[r], %[r], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r_flat], %[r_flat], %[c] \n\t" + + /* flat |= (abs(p0 - p2) > thresh) */ + "subu_s.qb %[c], %[p0], %[p2] \n\t" + "subu_s.qb %[r_k], %[p2], %[p0] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r_flat], %[r_flat], %[c] \n\t" + + /* flat |= (abs(q0 - q2) > thresh) */ + "subu_s.qb %[c], %[q0], %[q2] \n\t" + "subu_s.qb %[r_k], %[q2], %[q0] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r_flat], %[r_flat], %[c] \n\t" + + /* flat |= (abs(p3 - p0) > thresh) */ + "subu_s.qb %[c], %[p3], %[p0] \n\t" + "subu_s.qb %[r_k], %[p0], %[p3] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r_flat], %[r_flat], %[c] \n\t" + + /* flat |= (abs(q3 - q0) > thresh) */ + "subu_s.qb %[c], %[q3], %[q0] \n\t" + "subu_s.qb %[r_k], %[q0], %[q3] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r_flat], %[r_flat], %[c] \n\t" + "sll %[r_flat], %[r_flat], 24 \n\t" + /* look at stall here */ + "wrdsp %[r_flat] \n\t" + "pick.qb %[flat1], $0, %[ones] \n\t" + + /* mask |= (abs(q2 - q1) > limit) */ + "subu_s.qb %[c], %[q2], %[q1] \n\t" + "subu_s.qb %[r_k], %[q1], %[q2] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[limit], %[r_k] \n\t" + "or %[r], %[r], %[c] \n\t" + "sll %[r3], %[r3], 24 \n\t" + + /* mask |= (abs(q3 - q2) > limit) */ + "subu_s.qb %[c], %[q3], %[q2] \n\t" + "subu_s.qb %[r_k], %[q2], %[q3] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[limit], %[r_k] \n\t" + "or %[r], %[r], %[c] \n\t" + + : [c] "=&r"(c), [r_k] "=&r"(r_k), [r] "=&r"(r), [r3] "=&r"(r3), + [r_flat] "=&r"(r_flat), [flat1] "=&r"(flat1) + : [limit] "r"(limit), [p3] "r"(p3), [p2] "r"(p2), [p1] "r"(p1), + [p0] "r"(p0), [q1] "r"(q1), [q0] "r"(q0), [q2] "r"(q2), [q3] "r"(q3), + [thresh] "r"(thresh), [flat_thresh] "r"(flat_thresh), [ones] "r"(ones)); + + __asm__ __volatile__( + /* abs(p0 - q0) */ + "subu_s.qb %[c], %[p0], %[q0] \n\t" + "subu_s.qb %[r_k], %[q0], %[p0] \n\t" + "wrdsp %[r3] \n\t" + "or %[s1], %[r_k], %[c] \n\t" + + /* abs(p1 - q1) */ + "subu_s.qb %[c], %[p1], %[q1] \n\t" + "addu_s.qb %[s3], %[s1], %[s1] \n\t" + "pick.qb %[hev1], %[ones], $0 \n\t" + "subu_s.qb %[r_k], %[q1], %[p1] \n\t" + "or %[s2], %[r_k], %[c] \n\t" + + /* abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > flimit * 2 + limit */ + "shrl.qb %[s2], %[s2], 1 \n\t" + "addu_s.qb %[s1], %[s2], %[s3] \n\t" + "cmpgu.lt.qb %[c], %[flimit], %[s1] \n\t" + "or %[r], %[r], %[c] \n\t" + "sll %[r], %[r], 24 \n\t" + + "wrdsp %[r] \n\t" + "pick.qb %[s2], $0, %[ones] \n\t" + + : [c] "=&r"(c), [r_k] "=&r"(r_k), [s1] "=&r"(s1), [hev1] "=&r"(hev1), + [s2] "=&r"(s2), [r] "+r"(r), [s3] "=&r"(s3) + : [p0] "r"(p0), [q0] "r"(q0), [p1] "r"(p1), [r3] "r"(r3), [q1] "r"(q1), + [ones] "r"(ones), [flimit] "r"(flimit)); + + *hev = hev1; + *mask = s2; + *flat = flat1; +} + +static INLINE void flatmask5(uint32_t p4, uint32_t p3, uint32_t p2, uint32_t p1, + uint32_t p0, uint32_t q0, uint32_t q1, uint32_t q2, + uint32_t q3, uint32_t q4, uint32_t *flat2) { + uint32_t c, r, r_k, r_flat; + uint32_t ones = 0xFFFFFFFF; + uint32_t flat_thresh = 0x01010101; + uint32_t flat1, flat3; + + __asm__ __volatile__( + /* flat |= (abs(p4 - p0) > thresh) */ + "subu_s.qb %[c], %[p4], %[p0] \n\t" + "subu_s.qb %[r_k], %[p0], %[p4] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r], $0, %[c] \n\t" + + /* flat |= (abs(q4 - q0) > thresh) */ + "subu_s.qb %[c], %[q4], %[q0] \n\t" + "subu_s.qb %[r_k], %[q0], %[q4] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r], %[r], %[c] \n\t" + "sll %[r], %[r], 24 \n\t" + "wrdsp %[r] \n\t" + "pick.qb %[flat3], $0, %[ones] \n\t" + + /* flat |= (abs(p1 - p0) > thresh) */ + "subu_s.qb %[c], %[p1], %[p0] \n\t" + "subu_s.qb %[r_k], %[p0], %[p1] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r_flat], $0, %[c] \n\t" + + /* flat |= (abs(q1 - q0) > thresh) */ + "subu_s.qb %[c], %[q1], %[q0] \n\t" + "subu_s.qb %[r_k], %[q0], %[q1] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r_flat], %[r_flat], %[c] \n\t" + + /* flat |= (abs(p0 - p2) > thresh) */ + "subu_s.qb %[c], %[p0], %[p2] \n\t" + "subu_s.qb %[r_k], %[p2], %[p0] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r_flat], %[r_flat], %[c] \n\t" + + /* flat |= (abs(q0 - q2) > thresh) */ + "subu_s.qb %[c], %[q0], %[q2] \n\t" + "subu_s.qb %[r_k], %[q2], %[q0] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r_flat], %[r_flat], %[c] \n\t" + + /* flat |= (abs(p3 - p0) > thresh) */ + "subu_s.qb %[c], %[p3], %[p0] \n\t" + "subu_s.qb %[r_k], %[p0], %[p3] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r_flat], %[r_flat], %[c] \n\t" + + /* flat |= (abs(q3 - q0) > thresh) */ + "subu_s.qb %[c], %[q3], %[q0] \n\t" + "subu_s.qb %[r_k], %[q0], %[q3] \n\t" + "or %[r_k], %[r_k], %[c] \n\t" + "cmpgu.lt.qb %[c], %[flat_thresh], %[r_k] \n\t" + "or %[r_flat], %[r_flat], %[c] \n\t" + "sll %[r_flat], %[r_flat], 24 \n\t" + "wrdsp %[r_flat] \n\t" + "pick.qb %[flat1], $0, %[ones] \n\t" + /* flat & flatmask4(thresh, p3, p2, p1, p0, q0, q1, q2, q3) */ + "and %[flat1], %[flat3], %[flat1] \n\t" + + : [c] "=&r"(c), [r_k] "=&r"(r_k), [r] "=&r"(r), [r_flat] "=&r"(r_flat), + [flat1] "=&r"(flat1), [flat3] "=&r"(flat3) + : [p4] "r"(p4), [p3] "r"(p3), [p2] "r"(p2), [p1] "r"(p1), [p0] "r"(p0), + [q0] "r"(q0), [q1] "r"(q1), [q2] "r"(q2), [q3] "r"(q3), [q4] "r"(q4), + [flat_thresh] "r"(flat_thresh), [ones] "r"(ones)); + + *flat2 = flat1; +} +#endif // #if HAVE_DSPR2 +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_MIPS_LOOPFILTER_MASKS_DSPR2_H_ diff --git a/third_party/aom/aom_dsp/mips/loopfilter_mb_dspr2.c b/third_party/aom/aom_dsp/mips/loopfilter_mb_dspr2.c new file mode 100644 index 0000000000..a3b5a9eb1c --- /dev/null +++ b/third_party/aom/aom_dsp/mips/loopfilter_mb_dspr2.c @@ -0,0 +1,589 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_dsp/mips/common_dspr2.h" +#include "aom_dsp/mips/loopfilter_filters_dspr2.h" +#include "aom_dsp/mips/loopfilter_macros_dspr2.h" +#include "aom_dsp/mips/loopfilter_masks_dspr2.h" +#include "aom_mem/aom_mem.h" + +#if HAVE_DSPR2 +void aom_lpf_horizontal_8_dspr2(unsigned char *s, int pitch, + const uint8_t *blimit, const uint8_t *limit, + const uint8_t *thresh) { + uint32_t mask; + uint32_t hev, flat; + uint8_t i; + uint8_t *sp3, *sp2, *sp1, *sp0, *sq0, *sq1, *sq2, *sq3; + uint32_t thresh_vec, flimit_vec, limit_vec; + uint32_t uflimit, ulimit, uthresh; + uint32_t p1_f0, p0_f0, q0_f0, q1_f0; + uint32_t p3, p2, p1, p0, q0, q1, q2, q3; + uint32_t p0_l, p1_l, p2_l, p3_l, q0_l, q1_l, q2_l, q3_l; + uint32_t p0_r, p1_r, p2_r, p3_r, q0_r, q1_r, q2_r, q3_r; + + uflimit = *blimit; + ulimit = *limit; + uthresh = *thresh; + + /* create quad-byte */ + __asm__ __volatile__( + "replv.qb %[thresh_vec], %[uthresh] \n\t" + "replv.qb %[flimit_vec], %[uflimit] \n\t" + "replv.qb %[limit_vec], %[ulimit] \n\t" + + : [thresh_vec] "=&r"(thresh_vec), [flimit_vec] "=&r"(flimit_vec), + [limit_vec] "=r"(limit_vec) + : [uthresh] "r"(uthresh), [uflimit] "r"(uflimit), [ulimit] "r"(ulimit)); + + /* prefetch data for store */ + prefetch_store(s); + + for (i = 0; i < 2; i++) { + sp3 = s - (pitch << 2); + sp2 = sp3 + pitch; + sp1 = sp2 + pitch; + sp0 = sp1 + pitch; + sq0 = s; + sq1 = s + pitch; + sq2 = sq1 + pitch; + sq3 = sq2 + pitch; + + __asm__ __volatile__( + "lw %[p3], (%[sp3]) \n\t" + "lw %[p2], (%[sp2]) \n\t" + "lw %[p1], (%[sp1]) \n\t" + "lw %[p0], (%[sp0]) \n\t" + "lw %[q0], (%[sq0]) \n\t" + "lw %[q1], (%[sq1]) \n\t" + "lw %[q2], (%[sq2]) \n\t" + "lw %[q3], (%[sq3]) \n\t" + + : [p3] "=&r"(p3), [p2] "=&r"(p2), [p1] "=&r"(p1), [p0] "=&r"(p0), + [q3] "=&r"(q3), [q2] "=&r"(q2), [q1] "=&r"(q1), [q0] "=&r"(q0) + : [sp3] "r"(sp3), [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq3] "r"(sq3), [sq2] "r"(sq2), [sq1] "r"(sq1), [sq0] "r"(sq0)); + + filter_hev_mask_flatmask4_dspr2(limit_vec, flimit_vec, thresh_vec, p1, p0, + p3, p2, q0, q1, q2, q3, &hev, &mask, &flat); + + if ((flat == 0) && (mask != 0)) { + filter1_dspr2(mask, hev, p1, p0, q0, q1, &p1_f0, &p0_f0, &q0_f0, &q1_f0); + + __asm__ __volatile__( + "sw %[p1_f0], (%[sp1]) \n\t" + "sw %[p0_f0], (%[sp0]) \n\t" + "sw %[q0_f0], (%[sq0]) \n\t" + "sw %[q1_f0], (%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), + [sq1] "r"(sq1)); + } else if ((mask & flat) == 0xFFFFFFFF) { + /* left 2 element operation */ + PACK_LEFT_0TO3() + mbfilter_dspr2(&p3_l, &p2_l, &p1_l, &p0_l, &q0_l, &q1_l, &q2_l, &q3_l); + + /* right 2 element operation */ + PACK_RIGHT_0TO3() + mbfilter_dspr2(&p3_r, &p2_r, &p1_r, &p0_r, &q0_r, &q1_r, &q2_r, &q3_r); + + COMBINE_LEFT_RIGHT_0TO2() + + __asm__ __volatile__( + "sw %[p2], (%[sp2]) \n\t" + "sw %[p1], (%[sp1]) \n\t" + "sw %[p0], (%[sp0]) \n\t" + "sw %[q0], (%[sq0]) \n\t" + "sw %[q1], (%[sq1]) \n\t" + "sw %[q2], (%[sq2]) \n\t" + + : + : [p2] "r"(p2), [p1] "r"(p1), [p0] "r"(p0), [q0] "r"(q0), + [q1] "r"(q1), [q2] "r"(q2), [sp2] "r"(sp2), [sp1] "r"(sp1), + [sp0] "r"(sp0), [sq0] "r"(sq0), [sq1] "r"(sq1), [sq2] "r"(sq2)); + } else if ((flat != 0) && (mask != 0)) { + /* filtering */ + filter1_dspr2(mask, hev, p1, p0, q0, q1, &p1_f0, &p0_f0, &q0_f0, &q1_f0); + + /* left 2 element operation */ + PACK_LEFT_0TO3() + mbfilter_dspr2(&p3_l, &p2_l, &p1_l, &p0_l, &q0_l, &q1_l, &q2_l, &q3_l); + + /* right 2 element operation */ + PACK_RIGHT_0TO3() + mbfilter_dspr2(&p3_r, &p2_r, &p1_r, &p0_r, &q0_r, &q1_r, &q2_r, &q3_r); + + if (mask & flat & 0x000000FF) { + __asm__ __volatile__( + "sb %[p2_r], (%[sp2]) \n\t" + "sb %[p1_r], (%[sp1]) \n\t" + "sb %[p0_r], (%[sp0]) \n\t" + "sb %[q0_r], (%[sq0]) \n\t" + "sb %[q1_r], (%[sq1]) \n\t" + "sb %[q2_r], (%[sq2]) \n\t" + + : + : [p2_r] "r"(p2_r), [p1_r] "r"(p1_r), [p0_r] "r"(p0_r), + [q0_r] "r"(q0_r), [q1_r] "r"(q1_r), [q2_r] "r"(q2_r), + [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), + [sq1] "r"(sq1), [sq2] "r"(sq2)); + } else if (mask & 0x000000FF) { + __asm__ __volatile__( + "sb %[p1_f0], (%[sp1]) \n\t" + "sb %[p0_f0], (%[sp0]) \n\t" + "sb %[q0_f0], (%[sq0]) \n\t" + "sb %[q1_f0], (%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq0] "r"(sq0), [sq1] "r"(sq1)); + } + + __asm__ __volatile__( + "srl %[p2_r], %[p2_r], 16 \n\t" + "srl %[p1_r], %[p1_r], 16 \n\t" + "srl %[p0_r], %[p0_r], 16 \n\t" + "srl %[q0_r], %[q0_r], 16 \n\t" + "srl %[q1_r], %[q1_r], 16 \n\t" + "srl %[q2_r], %[q2_r], 16 \n\t" + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2_r] "+r"(p2_r), [p1_r] "+r"(p1_r), [p0_r] "+r"(p0_r), + [q0_r] "+r"(q0_r), [q1_r] "+r"(q1_r), [q2_r] "+r"(q2_r), + [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p2_r], +1(%[sp2]) \n\t" + "sb %[p1_r], +1(%[sp1]) \n\t" + "sb %[p0_r], +1(%[sp0]) \n\t" + "sb %[q0_r], +1(%[sq0]) \n\t" + "sb %[q1_r], +1(%[sq1]) \n\t" + "sb %[q2_r], +1(%[sq2]) \n\t" + + : + : [p2_r] "r"(p2_r), [p1_r] "r"(p1_r), [p0_r] "r"(p0_r), + [q0_r] "r"(q0_r), [q1_r] "r"(q1_r), [q2_r] "r"(q2_r), + [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), + [sq1] "r"(sq1), [sq2] "r"(sq2)); + } else if (mask & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p1_f0], +1(%[sp1]) \n\t" + "sb %[p0_f0], +1(%[sp0]) \n\t" + "sb %[q0_f0], +1(%[sq0]) \n\t" + "sb %[q1_f0], +1(%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq0] "r"(sq0), [sq1] "r"(sq1)); + } + + __asm__ __volatile__( + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2] "+r"(p2), [p1] "+r"(p1), [p0] "+r"(p0), [q0] "+r"(q0), + [q1] "+r"(q1), [q2] "+r"(q2), [p1_f0] "+r"(p1_f0), + [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p2_l], +2(%[sp2]) \n\t" + "sb %[p1_l], +2(%[sp1]) \n\t" + "sb %[p0_l], +2(%[sp0]) \n\t" + "sb %[q0_l], +2(%[sq0]) \n\t" + "sb %[q1_l], +2(%[sq1]) \n\t" + "sb %[q2_l], +2(%[sq2]) \n\t" + + : + : [p2_l] "r"(p2_l), [p1_l] "r"(p1_l), [p0_l] "r"(p0_l), + [q0_l] "r"(q0_l), [q1_l] "r"(q1_l), [q2_l] "r"(q2_l), + [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), + [sq1] "r"(sq1), [sq2] "r"(sq2)); + } else if (mask & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p1_f0], +2(%[sp1]) \n\t" + "sb %[p0_f0], +2(%[sp0]) \n\t" + "sb %[q0_f0], +2(%[sq0]) \n\t" + "sb %[q1_f0], +2(%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq0] "r"(sq0), [sq1] "r"(sq1)); + } + + __asm__ __volatile__( + "srl %[p2_l], %[p2_l], 16 \n\t" + "srl %[p1_l], %[p1_l], 16 \n\t" + "srl %[p0_l], %[p0_l], 16 \n\t" + "srl %[q0_l], %[q0_l], 16 \n\t" + "srl %[q1_l], %[q1_l], 16 \n\t" + "srl %[q2_l], %[q2_l], 16 \n\t" + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2_l] "+r"(p2_l), [p1_l] "+r"(p1_l), [p0_l] "+r"(p0_l), + [q0_l] "+r"(q0_l), [q1_l] "+r"(q1_l), [q2_l] "+r"(q2_l), + [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & 0xFF000000) { + __asm__ __volatile__( + "sb %[p2_l], +3(%[sp2]) \n\t" + "sb %[p1_l], +3(%[sp1]) \n\t" + "sb %[p0_l], +3(%[sp0]) \n\t" + "sb %[q0_l], +3(%[sq0]) \n\t" + "sb %[q1_l], +3(%[sq1]) \n\t" + "sb %[q2_l], +3(%[sq2]) \n\t" + + : + : [p2_l] "r"(p2_l), [p1_l] "r"(p1_l), [p0_l] "r"(p0_l), + [q0_l] "r"(q0_l), [q1_l] "r"(q1_l), [q2_l] "r"(q2_l), + [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), + [sq1] "r"(sq1), [sq2] "r"(sq2)); + } else if (mask & 0xFF000000) { + __asm__ __volatile__( + "sb %[p1_f0], +3(%[sp1]) \n\t" + "sb %[p0_f0], +3(%[sp0]) \n\t" + "sb %[q0_f0], +3(%[sq0]) \n\t" + "sb %[q1_f0], +3(%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq0] "r"(sq0), [sq1] "r"(sq1)); + } + } + + s = s + 4; + } +} + +void aom_lpf_vertical_8_dspr2(unsigned char *s, int pitch, + const uint8_t *blimit, const uint8_t *limit, + const uint8_t *thresh) { + uint8_t i; + uint32_t mask, hev, flat; + uint8_t *s1, *s2, *s3, *s4; + uint32_t prim1, prim2, sec3, sec4, prim3, prim4; + uint32_t thresh_vec, flimit_vec, limit_vec; + uint32_t uflimit, ulimit, uthresh; + uint32_t p3, p2, p1, p0, q3, q2, q1, q0; + uint32_t p1_f0, p0_f0, q0_f0, q1_f0; + uint32_t p0_l, p1_l, p2_l, p3_l, q0_l, q1_l, q2_l, q3_l; + uint32_t p0_r, p1_r, p2_r, p3_r, q0_r, q1_r, q2_r, q3_r; + + uflimit = *blimit; + ulimit = *limit; + uthresh = *thresh; + + /* create quad-byte */ + __asm__ __volatile__( + "replv.qb %[thresh_vec], %[uthresh] \n\t" + "replv.qb %[flimit_vec], %[uflimit] \n\t" + "replv.qb %[limit_vec], %[ulimit] \n\t" + + : [thresh_vec] "=&r"(thresh_vec), [flimit_vec] "=&r"(flimit_vec), + [limit_vec] "=r"(limit_vec) + : [uthresh] "r"(uthresh), [uflimit] "r"(uflimit), [ulimit] "r"(ulimit)); + + prefetch_store(s + pitch); + + for (i = 0; i < 2; i++) { + s1 = s; + s2 = s + pitch; + s3 = s2 + pitch; + s4 = s3 + pitch; + s = s4 + pitch; + + __asm__ __volatile__( + "lw %[p0], -4(%[s1]) \n\t" + "lw %[p1], -4(%[s2]) \n\t" + "lw %[p2], -4(%[s3]) \n\t" + "lw %[p3], -4(%[s4]) \n\t" + "lw %[q3], (%[s1]) \n\t" + "lw %[q2], (%[s2]) \n\t" + "lw %[q1], (%[s3]) \n\t" + "lw %[q0], (%[s4]) \n\t" + + : [p3] "=&r"(p3), [p2] "=&r"(p2), [p1] "=&r"(p1), [p0] "=&r"(p0), + [q0] "=&r"(q0), [q1] "=&r"(q1), [q2] "=&r"(q2), [q3] "=&r"(q3) + : [s1] "r"(s1), [s2] "r"(s2), [s3] "r"(s3), [s4] "r"(s4)); + + /* transpose p3, p2, p1, p0 + original (when loaded from memory) + register -4 -3 -2 -1 + p0 p0_0 p0_1 p0_2 p0_3 + p1 p1_0 p1_1 p1_2 p1_3 + p2 p2_0 p2_1 p2_2 p2_3 + p3 p3_0 p3_1 p3_2 p3_3 + + after transpose + register + p0 p3_3 p2_3 p1_3 p0_3 + p1 p3_2 p2_2 p1_2 p0_2 + p2 p3_1 p2_1 p1_1 p0_1 + p3 p3_0 p2_0 p1_0 p0_0 + */ + __asm__ __volatile__( + "precrq.qb.ph %[prim1], %[p0], %[p1] \n\t" + "precr.qb.ph %[prim2], %[p0], %[p1] \n\t" + "precrq.qb.ph %[prim3], %[p2], %[p3] \n\t" + "precr.qb.ph %[prim4], %[p2], %[p3] \n\t" + + "precrq.qb.ph %[p1], %[prim1], %[prim2] \n\t" + "precr.qb.ph %[p3], %[prim1], %[prim2] \n\t" + "precrq.qb.ph %[sec3], %[prim3], %[prim4] \n\t" + "precr.qb.ph %[sec4], %[prim3], %[prim4] \n\t" + + "precrq.ph.w %[p0], %[p1], %[sec3] \n\t" + "precrq.ph.w %[p2], %[p3], %[sec4] \n\t" + "append %[p1], %[sec3], 16 \n\t" + "append %[p3], %[sec4], 16 \n\t" + + : [prim1] "=&r"(prim1), [prim2] "=&r"(prim2), [prim3] "=&r"(prim3), + [prim4] "=&r"(prim4), [p0] "+r"(p0), [p1] "+r"(p1), [p2] "+r"(p2), + [p3] "+r"(p3), [sec3] "=&r"(sec3), [sec4] "=&r"(sec4) + :); + + /* transpose q0, q1, q2, q3 + original (when loaded from memory) + register +1 +2 +3 +4 + q3 q3_0 q3_1 q3_2 q3_3 + q2 q2_0 q2_1 q2_2 q2_3 + q1 q1_0 q1_1 q1_2 q1_3 + q0 q0_0 q0_1 q0_2 q0_3 + + after transpose + register + q3 q0_3 q1_3 q2_3 q3_3 + q2 q0_2 q1_2 q2_2 q3_2 + q1 q0_1 q1_1 q2_1 q3_1 + q0 q0_0 q1_0 q2_0 q3_0 + */ + __asm__ __volatile__( + "precrq.qb.ph %[prim1], %[q3], %[q2] \n\t" + "precr.qb.ph %[prim2], %[q3], %[q2] \n\t" + "precrq.qb.ph %[prim3], %[q1], %[q0] \n\t" + "precr.qb.ph %[prim4], %[q1], %[q0] \n\t" + + "precrq.qb.ph %[q2], %[prim1], %[prim2] \n\t" + "precr.qb.ph %[q0], %[prim1], %[prim2] \n\t" + "precrq.qb.ph %[sec3], %[prim3], %[prim4] \n\t" + "precr.qb.ph %[sec4], %[prim3], %[prim4] \n\t" + + "precrq.ph.w %[q3], %[q2], %[sec3] \n\t" + "precrq.ph.w %[q1], %[q0], %[sec4] \n\t" + "append %[q2], %[sec3], 16 \n\t" + "append %[q0], %[sec4], 16 \n\t" + + : [prim1] "=&r"(prim1), [prim2] "=&r"(prim2), [prim3] "=&r"(prim3), + [prim4] "=&r"(prim4), [q3] "+r"(q3), [q2] "+r"(q2), [q1] "+r"(q1), + [q0] "+r"(q0), [sec3] "=&r"(sec3), [sec4] "=&r"(sec4) + :); + + filter_hev_mask_flatmask4_dspr2(limit_vec, flimit_vec, thresh_vec, p1, p0, + p3, p2, q0, q1, q2, q3, &hev, &mask, &flat); + + if ((flat == 0) && (mask != 0)) { + filter1_dspr2(mask, hev, p1, p0, q0, q1, &p1_f0, &p0_f0, &q0_f0, &q1_f0); + STORE_F0() + } else if ((mask & flat) == 0xFFFFFFFF) { + /* left 2 element operation */ + PACK_LEFT_0TO3() + mbfilter_dspr2(&p3_l, &p2_l, &p1_l, &p0_l, &q0_l, &q1_l, &q2_l, &q3_l); + + /* right 2 element operation */ + PACK_RIGHT_0TO3() + mbfilter_dspr2(&p3_r, &p2_r, &p1_r, &p0_r, &q0_r, &q1_r, &q2_r, &q3_r); + + STORE_F1() + } else if ((flat != 0) && (mask != 0)) { + filter1_dspr2(mask, hev, p1, p0, q0, q1, &p1_f0, &p0_f0, &q0_f0, &q1_f0); + + /* left 2 element operation */ + PACK_LEFT_0TO3() + mbfilter_dspr2(&p3_l, &p2_l, &p1_l, &p0_l, &q0_l, &q1_l, &q2_l, &q3_l); + + /* right 2 element operation */ + PACK_RIGHT_0TO3() + mbfilter_dspr2(&p3_r, &p2_r, &p1_r, &p0_r, &q0_r, &q1_r, &q2_r, &q3_r); + + if (mask & flat & 0x000000FF) { + __asm__ __volatile__( + "sb %[p2_r], -3(%[s4]) \n\t" + "sb %[p1_r], -2(%[s4]) \n\t" + "sb %[p0_r], -1(%[s4]) \n\t" + "sb %[q0_r], (%[s4]) \n\t" + "sb %[q1_r], +1(%[s4]) \n\t" + "sb %[q2_r], +2(%[s4]) \n\t" + + : + : [p2_r] "r"(p2_r), [p1_r] "r"(p1_r), [p0_r] "r"(p0_r), + [q0_r] "r"(q0_r), [q1_r] "r"(q1_r), [q2_r] "r"(q2_r), + [s4] "r"(s4)); + } else if (mask & 0x000000FF) { + __asm__ __volatile__( + "sb %[p1_f0], -2(%[s4]) \n\t" + "sb %[p0_f0], -1(%[s4]) \n\t" + "sb %[q0_f0], (%[s4]) \n\t" + "sb %[q1_f0], +1(%[s4]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [s4] "r"(s4)); + } + + __asm__ __volatile__( + "srl %[p2_r], %[p2_r], 16 \n\t" + "srl %[p1_r], %[p1_r], 16 \n\t" + "srl %[p0_r], %[p0_r], 16 \n\t" + "srl %[q0_r], %[q0_r], 16 \n\t" + "srl %[q1_r], %[q1_r], 16 \n\t" + "srl %[q2_r], %[q2_r], 16 \n\t" + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2_r] "+r"(p2_r), [p1_r] "+r"(p1_r), [p0_r] "+r"(p0_r), + [q0_r] "+r"(q0_r), [q1_r] "+r"(q1_r), [q2_r] "+r"(q2_r), + [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p2_r], -3(%[s3]) \n\t" + "sb %[p1_r], -2(%[s3]) \n\t" + "sb %[p0_r], -1(%[s3]) \n\t" + "sb %[q0_r], (%[s3]) \n\t" + "sb %[q1_r], +1(%[s3]) \n\t" + "sb %[q2_r], +2(%[s3]) \n\t" + + : + : [p2_r] "r"(p2_r), [p1_r] "r"(p1_r), [p0_r] "r"(p0_r), + [q0_r] "r"(q0_r), [q1_r] "r"(q1_r), [q2_r] "r"(q2_r), + [s3] "r"(s3)); + } else if (mask & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p1_f0], -2(%[s3]) \n\t" + "sb %[p0_f0], -1(%[s3]) \n\t" + "sb %[q0_f0], (%[s3]) \n\t" + "sb %[q1_f0], +1(%[s3]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [s3] "r"(s3)); + } + + __asm__ __volatile__( + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2] "+r"(p2), [p1] "+r"(p1), [p0] "+r"(p0), [q0] "+r"(q0), + [q1] "+r"(q1), [q2] "+r"(q2), [p1_f0] "+r"(p1_f0), + [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p2_l], -3(%[s2]) \n\t" + "sb %[p1_l], -2(%[s2]) \n\t" + "sb %[p0_l], -1(%[s2]) \n\t" + "sb %[q0_l], (%[s2]) \n\t" + "sb %[q1_l], +1(%[s2]) \n\t" + "sb %[q2_l], +2(%[s2]) \n\t" + + : + : [p2_l] "r"(p2_l), [p1_l] "r"(p1_l), [p0_l] "r"(p0_l), + [q0_l] "r"(q0_l), [q1_l] "r"(q1_l), [q2_l] "r"(q2_l), + [s2] "r"(s2)); + } else if (mask & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p1_f0], -2(%[s2]) \n\t" + "sb %[p0_f0], -1(%[s2]) \n\t" + "sb %[q0_f0], (%[s2]) \n\t" + "sb %[q1_f0], +1(%[s2]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [s2] "r"(s2)); + } + + __asm__ __volatile__( + "srl %[p2_l], %[p2_l], 16 \n\t" + "srl %[p1_l], %[p1_l], 16 \n\t" + "srl %[p0_l], %[p0_l], 16 \n\t" + "srl %[q0_l], %[q0_l], 16 \n\t" + "srl %[q1_l], %[q1_l], 16 \n\t" + "srl %[q2_l], %[q2_l], 16 \n\t" + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2_l] "+r"(p2_l), [p1_l] "+r"(p1_l), [p0_l] "+r"(p0_l), + [q0_l] "+r"(q0_l), [q1_l] "+r"(q1_l), [q2_l] "+r"(q2_l), + [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & 0xFF000000) { + __asm__ __volatile__( + "sb %[p2_l], -3(%[s1]) \n\t" + "sb %[p1_l], -2(%[s1]) \n\t" + "sb %[p0_l], -1(%[s1]) \n\t" + "sb %[q0_l], (%[s1]) \n\t" + "sb %[q1_l], +1(%[s1]) \n\t" + "sb %[q2_l], +2(%[s1]) \n\t" + + : + : [p2_l] "r"(p2_l), [p1_l] "r"(p1_l), [p0_l] "r"(p0_l), + [q0_l] "r"(q0_l), [q1_l] "r"(q1_l), [q2_l] "r"(q2_l), + [s1] "r"(s1)); + } else if (mask & 0xFF000000) { + __asm__ __volatile__( + "sb %[p1_f0], -2(%[s1]) \n\t" + "sb %[p0_f0], -1(%[s1]) \n\t" + "sb %[q0_f0], (%[s1]) \n\t" + "sb %[q1_f0], +1(%[s1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [s1] "r"(s1)); + } + } + } +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/aom_dsp/mips/loopfilter_mb_horiz_dspr2.c b/third_party/aom/aom_dsp/mips/loopfilter_mb_horiz_dspr2.c new file mode 100644 index 0000000000..8d2fd69f7f --- /dev/null +++ b/third_party/aom/aom_dsp/mips/loopfilter_mb_horiz_dspr2.c @@ -0,0 +1,734 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_dsp/mips/common_dspr2.h" +#include "aom_dsp/mips/loopfilter_filters_dspr2.h" +#include "aom_dsp/mips/loopfilter_macros_dspr2.h" +#include "aom_dsp/mips/loopfilter_masks_dspr2.h" +#include "aom_mem/aom_mem.h" + +#if HAVE_DSPR2 +static void mb_lpf_horizontal_edge(unsigned char *s, int pitch, + const uint8_t *blimit, const uint8_t *limit, + const uint8_t *thresh, int count) { + uint32_t mask; + uint32_t hev, flat, flat2; + uint8_t i; + uint8_t *sp7, *sp6, *sp5, *sp4, *sp3, *sp2, *sp1, *sp0; + uint8_t *sq0, *sq1, *sq2, *sq3, *sq4, *sq5, *sq6, *sq7; + uint32_t thresh_vec, flimit_vec, limit_vec; + uint32_t uflimit, ulimit, uthresh; + uint32_t p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + uint32_t p1_f0, p0_f0, q0_f0, q1_f0; + uint32_t p7_l, p6_l, p5_l, p4_l, p3_l, p2_l, p1_l, p0_l; + uint32_t q0_l, q1_l, q2_l, q3_l, q4_l, q5_l, q6_l, q7_l; + uint32_t p7_r, p6_r, p5_r, p4_r, p3_r, p2_r, p1_r, p0_r; + uint32_t q0_r, q1_r, q2_r, q3_r, q4_r, q5_r, q6_r, q7_r; + uint32_t p2_l_f1, p1_l_f1, p0_l_f1, p2_r_f1, p1_r_f1, p0_r_f1; + uint32_t q0_l_f1, q1_l_f1, q2_l_f1, q0_r_f1, q1_r_f1, q2_r_f1; + + uflimit = *blimit; + ulimit = *limit; + uthresh = *thresh; + + /* create quad-byte */ + __asm__ __volatile__( + "replv.qb %[thresh_vec], %[uthresh] \n\t" + "replv.qb %[flimit_vec], %[uflimit] \n\t" + "replv.qb %[limit_vec], %[ulimit] \n\t" + + : [thresh_vec] "=&r"(thresh_vec), [flimit_vec] "=&r"(flimit_vec), + [limit_vec] "=r"(limit_vec) + : [uthresh] "r"(uthresh), [uflimit] "r"(uflimit), [ulimit] "r"(ulimit)); + + /* prefetch data for store */ + prefetch_store(s); + + for (i = 0; i < (2 * count); i++) { + sp7 = s - (pitch << 3); + sp6 = sp7 + pitch; + sp5 = sp6 + pitch; + sp4 = sp5 + pitch; + sp3 = sp4 + pitch; + sp2 = sp3 + pitch; + sp1 = sp2 + pitch; + sp0 = sp1 + pitch; + sq0 = s; + sq1 = s + pitch; + sq2 = sq1 + pitch; + sq3 = sq2 + pitch; + sq4 = sq3 + pitch; + sq5 = sq4 + pitch; + sq6 = sq5 + pitch; + sq7 = sq6 + pitch; + + __asm__ __volatile__( + "lw %[p7], (%[sp7]) \n\t" + "lw %[p6], (%[sp6]) \n\t" + "lw %[p5], (%[sp5]) \n\t" + "lw %[p4], (%[sp4]) \n\t" + "lw %[p3], (%[sp3]) \n\t" + "lw %[p2], (%[sp2]) \n\t" + "lw %[p1], (%[sp1]) \n\t" + "lw %[p0], (%[sp0]) \n\t" + + : [p3] "=&r"(p3), [p2] "=&r"(p2), [p1] "=&r"(p1), [p0] "=&r"(p0), + [p7] "=&r"(p7), [p6] "=&r"(p6), [p5] "=&r"(p5), [p4] "=&r"(p4) + : [sp3] "r"(sp3), [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sp4] "r"(sp4), [sp5] "r"(sp5), [sp6] "r"(sp6), [sp7] "r"(sp7)); + + __asm__ __volatile__( + "lw %[q0], (%[sq0]) \n\t" + "lw %[q1], (%[sq1]) \n\t" + "lw %[q2], (%[sq2]) \n\t" + "lw %[q3], (%[sq3]) \n\t" + "lw %[q4], (%[sq4]) \n\t" + "lw %[q5], (%[sq5]) \n\t" + "lw %[q6], (%[sq6]) \n\t" + "lw %[q7], (%[sq7]) \n\t" + + : [q3] "=&r"(q3), [q2] "=&r"(q2), [q1] "=&r"(q1), [q0] "=&r"(q0), + [q7] "=&r"(q7), [q6] "=&r"(q6), [q5] "=&r"(q5), [q4] "=&r"(q4) + : [sq3] "r"(sq3), [sq2] "r"(sq2), [sq1] "r"(sq1), [sq0] "r"(sq0), + [sq4] "r"(sq4), [sq5] "r"(sq5), [sq6] "r"(sq6), [sq7] "r"(sq7)); + + filter_hev_mask_flatmask4_dspr2(limit_vec, flimit_vec, thresh_vec, p1, p0, + p3, p2, q0, q1, q2, q3, &hev, &mask, &flat); + + flatmask5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, &flat2); + + /* f0 */ + if (((flat2 == 0) && (flat == 0) && (mask != 0)) || + ((flat2 != 0) && (flat == 0) && (mask != 0))) { + filter1_dspr2(mask, hev, p1, p0, q0, q1, &p1_f0, &p0_f0, &q0_f0, &q1_f0); + + __asm__ __volatile__( + "sw %[p1_f0], (%[sp1]) \n\t" + "sw %[p0_f0], (%[sp0]) \n\t" + "sw %[q0_f0], (%[sq0]) \n\t" + "sw %[q1_f0], (%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), + [sq1] "r"(sq1)); + } else if ((flat2 == 0XFFFFFFFF) && (flat == 0xFFFFFFFF) && + (mask == 0xFFFFFFFF)) { + /* f2 */ + PACK_LEFT_0TO3() + PACK_LEFT_4TO7() + wide_mbfilter_dspr2(&p7_l, &p6_l, &p5_l, &p4_l, &p3_l, &p2_l, &p1_l, + &p0_l, &q0_l, &q1_l, &q2_l, &q3_l, &q4_l, &q5_l, + &q6_l, &q7_l); + + PACK_RIGHT_0TO3() + PACK_RIGHT_4TO7() + wide_mbfilter_dspr2(&p7_r, &p6_r, &p5_r, &p4_r, &p3_r, &p2_r, &p1_r, + &p0_r, &q0_r, &q1_r, &q2_r, &q3_r, &q4_r, &q5_r, + &q6_r, &q7_r); + + COMBINE_LEFT_RIGHT_0TO2() + COMBINE_LEFT_RIGHT_3TO6() + + __asm__ __volatile__( + "sw %[p6], (%[sp6]) \n\t" + "sw %[p5], (%[sp5]) \n\t" + "sw %[p4], (%[sp4]) \n\t" + "sw %[p3], (%[sp3]) \n\t" + "sw %[p2], (%[sp2]) \n\t" + "sw %[p1], (%[sp1]) \n\t" + "sw %[p0], (%[sp0]) \n\t" + + : + : [p6] "r"(p6), [p5] "r"(p5), [p4] "r"(p4), [p3] "r"(p3), + [p2] "r"(p2), [p1] "r"(p1), [p0] "r"(p0), [sp6] "r"(sp6), + [sp5] "r"(sp5), [sp4] "r"(sp4), [sp3] "r"(sp3), [sp2] "r"(sp2), + [sp1] "r"(sp1), [sp0] "r"(sp0)); + + __asm__ __volatile__( + "sw %[q6], (%[sq6]) \n\t" + "sw %[q5], (%[sq5]) \n\t" + "sw %[q4], (%[sq4]) \n\t" + "sw %[q3], (%[sq3]) \n\t" + "sw %[q2], (%[sq2]) \n\t" + "sw %[q1], (%[sq1]) \n\t" + "sw %[q0], (%[sq0]) \n\t" + + : + : [q6] "r"(q6), [q5] "r"(q5), [q4] "r"(q4), [q3] "r"(q3), + [q2] "r"(q2), [q1] "r"(q1), [q0] "r"(q0), [sq6] "r"(sq6), + [sq5] "r"(sq5), [sq4] "r"(sq4), [sq3] "r"(sq3), [sq2] "r"(sq2), + [sq1] "r"(sq1), [sq0] "r"(sq0)); + } else if ((flat2 == 0) && (flat == 0xFFFFFFFF) && (mask == 0xFFFFFFFF)) { + /* f1 */ + /* left 2 element operation */ + PACK_LEFT_0TO3() + mbfilter_dspr2(&p3_l, &p2_l, &p1_l, &p0_l, &q0_l, &q1_l, &q2_l, &q3_l); + + /* right 2 element operation */ + PACK_RIGHT_0TO3() + mbfilter_dspr2(&p3_r, &p2_r, &p1_r, &p0_r, &q0_r, &q1_r, &q2_r, &q3_r); + + COMBINE_LEFT_RIGHT_0TO2() + + __asm__ __volatile__( + "sw %[p2], (%[sp2]) \n\t" + "sw %[p1], (%[sp1]) \n\t" + "sw %[p0], (%[sp0]) \n\t" + "sw %[q0], (%[sq0]) \n\t" + "sw %[q1], (%[sq1]) \n\t" + "sw %[q2], (%[sq2]) \n\t" + + : + : [p2] "r"(p2), [p1] "r"(p1), [p0] "r"(p0), [q0] "r"(q0), + [q1] "r"(q1), [q2] "r"(q2), [sp2] "r"(sp2), [sp1] "r"(sp1), + [sp0] "r"(sp0), [sq0] "r"(sq0), [sq1] "r"(sq1), [sq2] "r"(sq2)); + } else if ((flat2 == 0) && (flat != 0) && (mask != 0)) { + /* f0+f1 */ + filter1_dspr2(mask, hev, p1, p0, q0, q1, &p1_f0, &p0_f0, &q0_f0, &q1_f0); + + /* left 2 element operation */ + PACK_LEFT_0TO3() + mbfilter_dspr2(&p3_l, &p2_l, &p1_l, &p0_l, &q0_l, &q1_l, &q2_l, &q3_l); + + /* right 2 element operation */ + PACK_RIGHT_0TO3() + mbfilter_dspr2(&p3_r, &p2_r, &p1_r, &p0_r, &q0_r, &q1_r, &q2_r, &q3_r); + + if (mask & flat & 0x000000FF) { + __asm__ __volatile__( + "sb %[p2_r], (%[sp2]) \n\t" + "sb %[p1_r], (%[sp1]) \n\t" + "sb %[p0_r], (%[sp0]) \n\t" + "sb %[q0_r], (%[sq0]) \n\t" + "sb %[q1_r], (%[sq1]) \n\t" + "sb %[q2_r], (%[sq2]) \n\t" + + : + : [p2_r] "r"(p2_r), [p1_r] "r"(p1_r), [p0_r] "r"(p0_r), + [q0_r] "r"(q0_r), [q1_r] "r"(q1_r), [q2_r] "r"(q2_r), + [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), + [sq1] "r"(sq1), [sq2] "r"(sq2)); + } else if (mask & 0x000000FF) { + __asm__ __volatile__( + "sb %[p1_f0], (%[sp1]) \n\t" + "sb %[p0_f0], (%[sp0]) \n\t" + "sb %[q0_f0], (%[sq0]) \n\t" + "sb %[q1_f0], (%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq0] "r"(sq0), [sq1] "r"(sq1)); + } + + __asm__ __volatile__( + "srl %[p2_r], %[p2_r], 16 \n\t" + "srl %[p1_r], %[p1_r], 16 \n\t" + "srl %[p0_r], %[p0_r], 16 \n\t" + "srl %[q0_r], %[q0_r], 16 \n\t" + "srl %[q1_r], %[q1_r], 16 \n\t" + "srl %[q2_r], %[q2_r], 16 \n\t" + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2_r] "+r"(p2_r), [p1_r] "+r"(p1_r), [p0_r] "+r"(p0_r), + [q0_r] "+r"(q0_r), [q1_r] "+r"(q1_r), [q2_r] "+r"(q2_r), + [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p2_r], +1(%[sp2]) \n\t" + "sb %[p1_r], +1(%[sp1]) \n\t" + "sb %[p0_r], +1(%[sp0]) \n\t" + "sb %[q0_r], +1(%[sq0]) \n\t" + "sb %[q1_r], +1(%[sq1]) \n\t" + "sb %[q2_r], +1(%[sq2]) \n\t" + + : + : [p2_r] "r"(p2_r), [p1_r] "r"(p1_r), [p0_r] "r"(p0_r), + [q0_r] "r"(q0_r), [q1_r] "r"(q1_r), [q2_r] "r"(q2_r), + [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), + [sq1] "r"(sq1), [sq2] "r"(sq2)); + } else if (mask & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p1_f0], +1(%[sp1]) \n\t" + "sb %[p0_f0], +1(%[sp0]) \n\t" + "sb %[q0_f0], +1(%[sq0]) \n\t" + "sb %[q1_f0], +1(%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq0] "r"(sq0), [sq1] "r"(sq1)); + } + + __asm__ __volatile__( + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p2_l], +2(%[sp2]) \n\t" + "sb %[p1_l], +2(%[sp1]) \n\t" + "sb %[p0_l], +2(%[sp0]) \n\t" + "sb %[q0_l], +2(%[sq0]) \n\t" + "sb %[q1_l], +2(%[sq1]) \n\t" + "sb %[q2_l], +2(%[sq2]) \n\t" + + : + : [p2_l] "r"(p2_l), [p1_l] "r"(p1_l), [p0_l] "r"(p0_l), + [q0_l] "r"(q0_l), [q1_l] "r"(q1_l), [q2_l] "r"(q2_l), + [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), + [sq1] "r"(sq1), [sq2] "r"(sq2)); + } else if (mask & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p1_f0], +2(%[sp1]) \n\t" + "sb %[p0_f0], +2(%[sp0]) \n\t" + "sb %[q0_f0], +2(%[sq0]) \n\t" + "sb %[q1_f0], +2(%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq0] "r"(sq0), [sq1] "r"(sq1)); + } + + __asm__ __volatile__( + "srl %[p2_l], %[p2_l], 16 \n\t" + "srl %[p1_l], %[p1_l], 16 \n\t" + "srl %[p0_l], %[p0_l], 16 \n\t" + "srl %[q0_l], %[q0_l], 16 \n\t" + "srl %[q1_l], %[q1_l], 16 \n\t" + "srl %[q2_l], %[q2_l], 16 \n\t" + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2_l] "+r"(p2_l), [p1_l] "+r"(p1_l), [p0_l] "+r"(p0_l), + [q0_l] "+r"(q0_l), [q1_l] "+r"(q1_l), [q2_l] "+r"(q2_l), + [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & 0xFF000000) { + __asm__ __volatile__( + "sb %[p2_l], +3(%[sp2]) \n\t" + "sb %[p1_l], +3(%[sp1]) \n\t" + "sb %[p0_l], +3(%[sp0]) \n\t" + "sb %[q0_l], +3(%[sq0]) \n\t" + "sb %[q1_l], +3(%[sq1]) \n\t" + "sb %[q2_l], +3(%[sq2]) \n\t" + + : + : [p2_l] "r"(p2_l), [p1_l] "r"(p1_l), [p0_l] "r"(p0_l), + [q0_l] "r"(q0_l), [q1_l] "r"(q1_l), [q2_l] "r"(q2_l), + [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), + [sq1] "r"(sq1), [sq2] "r"(sq2)); + } else if (mask & 0xFF000000) { + __asm__ __volatile__( + "sb %[p1_f0], +3(%[sp1]) \n\t" + "sb %[p0_f0], +3(%[sp0]) \n\t" + "sb %[q0_f0], +3(%[sq0]) \n\t" + "sb %[q1_f0], +3(%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq0] "r"(sq0), [sq1] "r"(sq1)); + } + } else if ((flat2 != 0) && (flat != 0) && (mask != 0)) { + /* f0 + f1 + f2 */ + /* f0 function */ + filter1_dspr2(mask, hev, p1, p0, q0, q1, &p1_f0, &p0_f0, &q0_f0, &q1_f0); + + /* f1 function */ + /* left 2 element operation */ + PACK_LEFT_0TO3() + mbfilter1_dspr2(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, &p2_l_f1, + &p1_l_f1, &p0_l_f1, &q0_l_f1, &q1_l_f1, &q2_l_f1); + + /* right 2 element operation */ + PACK_RIGHT_0TO3() + mbfilter1_dspr2(p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r, &p2_r_f1, + &p1_r_f1, &p0_r_f1, &q0_r_f1, &q1_r_f1, &q2_r_f1); + + /* f2 function */ + PACK_LEFT_4TO7() + wide_mbfilter_dspr2(&p7_l, &p6_l, &p5_l, &p4_l, &p3_l, &p2_l, &p1_l, + &p0_l, &q0_l, &q1_l, &q2_l, &q3_l, &q4_l, &q5_l, + &q6_l, &q7_l); + + PACK_RIGHT_4TO7() + wide_mbfilter_dspr2(&p7_r, &p6_r, &p5_r, &p4_r, &p3_r, &p2_r, &p1_r, + &p0_r, &q0_r, &q1_r, &q2_r, &q3_r, &q4_r, &q5_r, + &q6_r, &q7_r); + + if (mask & flat & flat2 & 0x000000FF) { + __asm__ __volatile__( + "sb %[p6_r], (%[sp6]) \n\t" + "sb %[p5_r], (%[sp5]) \n\t" + "sb %[p4_r], (%[sp4]) \n\t" + "sb %[p3_r], (%[sp3]) \n\t" + "sb %[p2_r], (%[sp2]) \n\t" + "sb %[p1_r], (%[sp1]) \n\t" + "sb %[p0_r], (%[sp0]) \n\t" + + : + : [p6_r] "r"(p6_r), [p5_r] "r"(p5_r), [p4_r] "r"(p4_r), + [p3_r] "r"(p3_r), [p2_r] "r"(p2_r), [p1_r] "r"(p1_r), + [sp6] "r"(sp6), [sp5] "r"(sp5), [sp4] "r"(sp4), [sp3] "r"(sp3), + [sp2] "r"(sp2), [sp1] "r"(sp1), [p0_r] "r"(p0_r), [sp0] "r"(sp0)); + + __asm__ __volatile__( + "sb %[q0_r], (%[sq0]) \n\t" + "sb %[q1_r], (%[sq1]) \n\t" + "sb %[q2_r], (%[sq2]) \n\t" + "sb %[q3_r], (%[sq3]) \n\t" + "sb %[q4_r], (%[sq4]) \n\t" + "sb %[q5_r], (%[sq5]) \n\t" + "sb %[q6_r], (%[sq6]) \n\t" + + : + : [q0_r] "r"(q0_r), [q1_r] "r"(q1_r), [q2_r] "r"(q2_r), + [q3_r] "r"(q3_r), [q4_r] "r"(q4_r), [q5_r] "r"(q5_r), + [q6_r] "r"(q6_r), [sq0] "r"(sq0), [sq1] "r"(sq1), [sq2] "r"(sq2), + [sq3] "r"(sq3), [sq4] "r"(sq4), [sq5] "r"(sq5), [sq6] "r"(sq6)); + } else if (mask & flat & 0x000000FF) { + __asm__ __volatile__( + "sb %[p2_r_f1], (%[sp2]) \n\t" + "sb %[p1_r_f1], (%[sp1]) \n\t" + "sb %[p0_r_f1], (%[sp0]) \n\t" + "sb %[q0_r_f1], (%[sq0]) \n\t" + "sb %[q1_r_f1], (%[sq1]) \n\t" + "sb %[q2_r_f1], (%[sq2]) \n\t" + + : + : [p2_r_f1] "r"(p2_r_f1), [p1_r_f1] "r"(p1_r_f1), + [p0_r_f1] "r"(p0_r_f1), [q0_r_f1] "r"(q0_r_f1), + [q1_r_f1] "r"(q1_r_f1), [q2_r_f1] "r"(q2_r_f1), [sp2] "r"(sp2), + [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), [sq1] "r"(sq1), + [sq2] "r"(sq2)); + } else if (mask & 0x000000FF) { + __asm__ __volatile__( + "sb %[p1_f0], (%[sp1]) \n\t" + "sb %[p0_f0], (%[sp0]) \n\t" + "sb %[q0_f0], (%[sq0]) \n\t" + "sb %[q1_f0], (%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq0] "r"(sq0), [sq1] "r"(sq1)); + } + + __asm__ __volatile__( + "srl %[p6_r], %[p6_r], 16 \n\t" + "srl %[p5_r], %[p5_r], 16 \n\t" + "srl %[p4_r], %[p4_r], 16 \n\t" + "srl %[p3_r], %[p3_r], 16 \n\t" + "srl %[p2_r], %[p2_r], 16 \n\t" + "srl %[p1_r], %[p1_r], 16 \n\t" + "srl %[p0_r], %[p0_r], 16 \n\t" + "srl %[q0_r], %[q0_r], 16 \n\t" + "srl %[q1_r], %[q1_r], 16 \n\t" + "srl %[q2_r], %[q2_r], 16 \n\t" + "srl %[q3_r], %[q3_r], 16 \n\t" + "srl %[q4_r], %[q4_r], 16 \n\t" + "srl %[q5_r], %[q5_r], 16 \n\t" + "srl %[q6_r], %[q6_r], 16 \n\t" + + : [q0_r] "+r"(q0_r), [q1_r] "+r"(q1_r), [q2_r] "+r"(q2_r), + [q3_r] "+r"(q3_r), [q4_r] "+r"(q4_r), [q5_r] "+r"(q5_r), + [p6_r] "+r"(p6_r), [p5_r] "+r"(p5_r), [p4_r] "+r"(p4_r), + [p3_r] "+r"(p3_r), [p2_r] "+r"(p2_r), [p1_r] "+r"(p1_r), + [q6_r] "+r"(q6_r), [p0_r] "+r"(p0_r) + :); + + __asm__ __volatile__( + "srl %[p2_r_f1], %[p2_r_f1], 16 \n\t" + "srl %[p1_r_f1], %[p1_r_f1], 16 \n\t" + "srl %[p0_r_f1], %[p0_r_f1], 16 \n\t" + "srl %[q0_r_f1], %[q0_r_f1], 16 \n\t" + "srl %[q1_r_f1], %[q1_r_f1], 16 \n\t" + "srl %[q2_r_f1], %[q2_r_f1], 16 \n\t" + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2_r_f1] "+r"(p2_r_f1), [p1_r_f1] "+r"(p1_r_f1), + [p0_r_f1] "+r"(p0_r_f1), [q0_r_f1] "+r"(q0_r_f1), + [q1_r_f1] "+r"(q1_r_f1), [q2_r_f1] "+r"(q2_r_f1), + [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & flat2 & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p6_r], +1(%[sp6]) \n\t" + "sb %[p5_r], +1(%[sp5]) \n\t" + "sb %[p4_r], +1(%[sp4]) \n\t" + "sb %[p3_r], +1(%[sp3]) \n\t" + "sb %[p2_r], +1(%[sp2]) \n\t" + "sb %[p1_r], +1(%[sp1]) \n\t" + "sb %[p0_r], +1(%[sp0]) \n\t" + + : + : [p6_r] "r"(p6_r), [p5_r] "r"(p5_r), [p4_r] "r"(p4_r), + [p3_r] "r"(p3_r), [p2_r] "r"(p2_r), [p1_r] "r"(p1_r), + [p0_r] "r"(p0_r), [sp6] "r"(sp6), [sp5] "r"(sp5), [sp4] "r"(sp4), + [sp3] "r"(sp3), [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0)); + + __asm__ __volatile__( + "sb %[q0_r], +1(%[sq0]) \n\t" + "sb %[q1_r], +1(%[sq1]) \n\t" + "sb %[q2_r], +1(%[sq2]) \n\t" + "sb %[q3_r], +1(%[sq3]) \n\t" + "sb %[q4_r], +1(%[sq4]) \n\t" + "sb %[q5_r], +1(%[sq5]) \n\t" + "sb %[q6_r], +1(%[sq6]) \n\t" + + : + : [q0_r] "r"(q0_r), [q1_r] "r"(q1_r), [q2_r] "r"(q2_r), + [q3_r] "r"(q3_r), [q4_r] "r"(q4_r), [q5_r] "r"(q5_r), + [q6_r] "r"(q6_r), [sq0] "r"(sq0), [sq1] "r"(sq1), [sq2] "r"(sq2), + [sq3] "r"(sq3), [sq4] "r"(sq4), [sq5] "r"(sq5), [sq6] "r"(sq6)); + } else if (mask & flat & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p2_r_f1], +1(%[sp2]) \n\t" + "sb %[p1_r_f1], +1(%[sp1]) \n\t" + "sb %[p0_r_f1], +1(%[sp0]) \n\t" + "sb %[q0_r_f1], +1(%[sq0]) \n\t" + "sb %[q1_r_f1], +1(%[sq1]) \n\t" + "sb %[q2_r_f1], +1(%[sq2]) \n\t" + + : + : [p2_r_f1] "r"(p2_r_f1), [p1_r_f1] "r"(p1_r_f1), + [p0_r_f1] "r"(p0_r_f1), [q0_r_f1] "r"(q0_r_f1), + [q1_r_f1] "r"(q1_r_f1), [q2_r_f1] "r"(q2_r_f1), [sp2] "r"(sp2), + [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), [sq1] "r"(sq1), + [sq2] "r"(sq2)); + } else if (mask & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p1_f0], +1(%[sp1]) \n\t" + "sb %[p0_f0], +1(%[sp0]) \n\t" + "sb %[q0_f0], +1(%[sq0]) \n\t" + "sb %[q1_f0], +1(%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq0] "r"(sq0), [sq1] "r"(sq1)); + } + + __asm__ __volatile__( + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & flat2 & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p6_l], +2(%[sp6]) \n\t" + "sb %[p5_l], +2(%[sp5]) \n\t" + "sb %[p4_l], +2(%[sp4]) \n\t" + "sb %[p3_l], +2(%[sp3]) \n\t" + "sb %[p2_l], +2(%[sp2]) \n\t" + "sb %[p1_l], +2(%[sp1]) \n\t" + "sb %[p0_l], +2(%[sp0]) \n\t" + + : + : [p6_l] "r"(p6_l), [p5_l] "r"(p5_l), [p4_l] "r"(p4_l), + [p3_l] "r"(p3_l), [p2_l] "r"(p2_l), [p1_l] "r"(p1_l), + [p0_l] "r"(p0_l), [sp6] "r"(sp6), [sp5] "r"(sp5), [sp4] "r"(sp4), + [sp3] "r"(sp3), [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0)); + + __asm__ __volatile__( + "sb %[q0_l], +2(%[sq0]) \n\t" + "sb %[q1_l], +2(%[sq1]) \n\t" + "sb %[q2_l], +2(%[sq2]) \n\t" + "sb %[q3_l], +2(%[sq3]) \n\t" + "sb %[q4_l], +2(%[sq4]) \n\t" + "sb %[q5_l], +2(%[sq5]) \n\t" + "sb %[q6_l], +2(%[sq6]) \n\t" + + : + : [q0_l] "r"(q0_l), [q1_l] "r"(q1_l), [q2_l] "r"(q2_l), + [q3_l] "r"(q3_l), [q4_l] "r"(q4_l), [q5_l] "r"(q5_l), + [q6_l] "r"(q6_l), [sq0] "r"(sq0), [sq1] "r"(sq1), [sq2] "r"(sq2), + [sq3] "r"(sq3), [sq4] "r"(sq4), [sq5] "r"(sq5), [sq6] "r"(sq6)); + } else if (mask & flat & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p2_l_f1], +2(%[sp2]) \n\t" + "sb %[p1_l_f1], +2(%[sp1]) \n\t" + "sb %[p0_l_f1], +2(%[sp0]) \n\t" + "sb %[q0_l_f1], +2(%[sq0]) \n\t" + "sb %[q1_l_f1], +2(%[sq1]) \n\t" + "sb %[q2_l_f1], +2(%[sq2]) \n\t" + + : + : [p2_l_f1] "r"(p2_l_f1), [p1_l_f1] "r"(p1_l_f1), + [p0_l_f1] "r"(p0_l_f1), [q0_l_f1] "r"(q0_l_f1), + [q1_l_f1] "r"(q1_l_f1), [q2_l_f1] "r"(q2_l_f1), [sp2] "r"(sp2), + [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), [sq1] "r"(sq1), + [sq2] "r"(sq2)); + } else if (mask & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p1_f0], +2(%[sp1]) \n\t" + "sb %[p0_f0], +2(%[sp0]) \n\t" + "sb %[q0_f0], +2(%[sq0]) \n\t" + "sb %[q1_f0], +2(%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq0] "r"(sq0), [sq1] "r"(sq1)); + } + + __asm__ __volatile__( + "srl %[p6_l], %[p6_l], 16 \n\t" + "srl %[p5_l], %[p5_l], 16 \n\t" + "srl %[p4_l], %[p4_l], 16 \n\t" + "srl %[p3_l], %[p3_l], 16 \n\t" + "srl %[p2_l], %[p2_l], 16 \n\t" + "srl %[p1_l], %[p1_l], 16 \n\t" + "srl %[p0_l], %[p0_l], 16 \n\t" + "srl %[q0_l], %[q0_l], 16 \n\t" + "srl %[q1_l], %[q1_l], 16 \n\t" + "srl %[q2_l], %[q2_l], 16 \n\t" + "srl %[q3_l], %[q3_l], 16 \n\t" + "srl %[q4_l], %[q4_l], 16 \n\t" + "srl %[q5_l], %[q5_l], 16 \n\t" + "srl %[q6_l], %[q6_l], 16 \n\t" + + : [q0_l] "+r"(q0_l), [q1_l] "+r"(q1_l), [q2_l] "+r"(q2_l), + [q3_l] "+r"(q3_l), [q4_l] "+r"(q4_l), [q5_l] "+r"(q5_l), + [q6_l] "+r"(q6_l), [p6_l] "+r"(p6_l), [p5_l] "+r"(p5_l), + [p4_l] "+r"(p4_l), [p3_l] "+r"(p3_l), [p2_l] "+r"(p2_l), + [p1_l] "+r"(p1_l), [p0_l] "+r"(p0_l) + :); + + __asm__ __volatile__( + "srl %[p2_l_f1], %[p2_l_f1], 16 \n\t" + "srl %[p1_l_f1], %[p1_l_f1], 16 \n\t" + "srl %[p0_l_f1], %[p0_l_f1], 16 \n\t" + "srl %[q0_l_f1], %[q0_l_f1], 16 \n\t" + "srl %[q1_l_f1], %[q1_l_f1], 16 \n\t" + "srl %[q2_l_f1], %[q2_l_f1], 16 \n\t" + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2_l_f1] "+r"(p2_l_f1), [p1_l_f1] "+r"(p1_l_f1), + [p0_l_f1] "+r"(p0_l_f1), [q0_l_f1] "+r"(q0_l_f1), + [q1_l_f1] "+r"(q1_l_f1), [q2_l_f1] "+r"(q2_l_f1), + [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & flat2 & 0xFF000000) { + __asm__ __volatile__( + "sb %[p6_l], +3(%[sp6]) \n\t" + "sb %[p5_l], +3(%[sp5]) \n\t" + "sb %[p4_l], +3(%[sp4]) \n\t" + "sb %[p3_l], +3(%[sp3]) \n\t" + "sb %[p2_l], +3(%[sp2]) \n\t" + "sb %[p1_l], +3(%[sp1]) \n\t" + "sb %[p0_l], +3(%[sp0]) \n\t" + + : + : [p6_l] "r"(p6_l), [p5_l] "r"(p5_l), [p4_l] "r"(p4_l), + [p3_l] "r"(p3_l), [p2_l] "r"(p2_l), [p1_l] "r"(p1_l), + [p0_l] "r"(p0_l), [sp6] "r"(sp6), [sp5] "r"(sp5), [sp4] "r"(sp4), + [sp3] "r"(sp3), [sp2] "r"(sp2), [sp1] "r"(sp1), [sp0] "r"(sp0)); + + __asm__ __volatile__( + "sb %[q0_l], +3(%[sq0]) \n\t" + "sb %[q1_l], +3(%[sq1]) \n\t" + "sb %[q2_l], +3(%[sq2]) \n\t" + "sb %[q3_l], +3(%[sq3]) \n\t" + "sb %[q4_l], +3(%[sq4]) \n\t" + "sb %[q5_l], +3(%[sq5]) \n\t" + "sb %[q6_l], +3(%[sq6]) \n\t" + + : + : [q0_l] "r"(q0_l), [q1_l] "r"(q1_l), [q2_l] "r"(q2_l), + [q3_l] "r"(q3_l), [q4_l] "r"(q4_l), [q5_l] "r"(q5_l), + [sq0] "r"(sq0), [sq1] "r"(sq1), [sq2] "r"(sq2), [sq3] "r"(sq3), + [sq4] "r"(sq4), [sq5] "r"(sq5), [q6_l] "r"(q6_l), [sq6] "r"(sq6)); + } else if (mask & flat & 0xFF000000) { + __asm__ __volatile__( + "sb %[p2_l_f1], +3(%[sp2]) \n\t" + "sb %[p1_l_f1], +3(%[sp1]) \n\t" + "sb %[p0_l_f1], +3(%[sp0]) \n\t" + "sb %[q0_l_f1], +3(%[sq0]) \n\t" + "sb %[q1_l_f1], +3(%[sq1]) \n\t" + "sb %[q2_l_f1], +3(%[sq2]) \n\t" + + : + : [p2_l_f1] "r"(p2_l_f1), [p1_l_f1] "r"(p1_l_f1), + [p0_l_f1] "r"(p0_l_f1), [q0_l_f1] "r"(q0_l_f1), + [q1_l_f1] "r"(q1_l_f1), [q2_l_f1] "r"(q2_l_f1), [sp2] "r"(sp2), + [sp1] "r"(sp1), [sp0] "r"(sp0), [sq0] "r"(sq0), [sq1] "r"(sq1), + [sq2] "r"(sq2)); + } else if (mask & 0xFF000000) { + __asm__ __volatile__( + "sb %[p1_f0], +3(%[sp1]) \n\t" + "sb %[p0_f0], +3(%[sp0]) \n\t" + "sb %[q0_f0], +3(%[sq0]) \n\t" + "sb %[q1_f0], +3(%[sq1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [sp1] "r"(sp1), [sp0] "r"(sp0), + [sq0] "r"(sq0), [sq1] "r"(sq1)); + } + } + + s = s + 4; + } +} + +void aom_lpf_horizontal_edge_8_dspr2(unsigned char *s, int pitch, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh) { + mb_lpf_horizontal_edge(s, pitch, blimit, limit, thresh, 1); +} + +void aom_lpf_horizontal_edge_16_dspr2(unsigned char *s, int pitch, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh) { + mb_lpf_horizontal_edge(s, pitch, blimit, limit, thresh, 2); +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/aom_dsp/mips/loopfilter_mb_vert_dspr2.c b/third_party/aom/aom_dsp/mips/loopfilter_mb_vert_dspr2.c new file mode 100644 index 0000000000..28528869b4 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/loopfilter_mb_vert_dspr2.c @@ -0,0 +1,757 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_dsp/mips/common_dspr2.h" +#include "aom_dsp/mips/loopfilter_filters_dspr2.h" +#include "aom_dsp/mips/loopfilter_macros_dspr2.h" +#include "aom_dsp/mips/loopfilter_masks_dspr2.h" +#include "aom_mem/aom_mem.h" + +#if HAVE_DSPR2 +void aom_lpf_vertical_16_dspr2(uint8_t *s, int pitch, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh) { + uint8_t i; + uint32_t mask, hev, flat, flat2; + uint8_t *s1, *s2, *s3, *s4; + uint32_t prim1, prim2, sec3, sec4, prim3, prim4; + uint32_t thresh_vec, flimit_vec, limit_vec; + uint32_t uflimit, ulimit, uthresh; + uint32_t p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + uint32_t p1_f0, p0_f0, q0_f0, q1_f0; + uint32_t p7_l, p6_l, p5_l, p4_l, p3_l, p2_l, p1_l, p0_l; + uint32_t q0_l, q1_l, q2_l, q3_l, q4_l, q5_l, q6_l, q7_l; + uint32_t p7_r, p6_r, p5_r, p4_r, p3_r, p2_r, p1_r, p0_r; + uint32_t q0_r, q1_r, q2_r, q3_r, q4_r, q5_r, q6_r, q7_r; + uint32_t p2_l_f1, p1_l_f1, p0_l_f1, p2_r_f1, p1_r_f1, p0_r_f1; + uint32_t q0_l_f1, q1_l_f1, q2_l_f1, q0_r_f1, q1_r_f1, q2_r_f1; + + uflimit = *blimit; + ulimit = *limit; + uthresh = *thresh; + + /* create quad-byte */ + __asm__ __volatile__( + "replv.qb %[thresh_vec], %[uthresh] \n\t" + "replv.qb %[flimit_vec], %[uflimit] \n\t" + "replv.qb %[limit_vec], %[ulimit] \n\t" + + : [thresh_vec] "=&r"(thresh_vec), [flimit_vec] "=&r"(flimit_vec), + [limit_vec] "=r"(limit_vec) + : [uthresh] "r"(uthresh), [uflimit] "r"(uflimit), [ulimit] "r"(ulimit)); + + prefetch_store(s + pitch); + + for (i = 0; i < 2; i++) { + s1 = s; + s2 = s + pitch; + s3 = s2 + pitch; + s4 = s3 + pitch; + s = s4 + pitch; + + __asm__ __volatile__( + "lw %[p0], -4(%[s1]) \n\t" + "lw %[p1], -4(%[s2]) \n\t" + "lw %[p2], -4(%[s3]) \n\t" + "lw %[p3], -4(%[s4]) \n\t" + "lw %[p4], -8(%[s1]) \n\t" + "lw %[p5], -8(%[s2]) \n\t" + "lw %[p6], -8(%[s3]) \n\t" + "lw %[p7], -8(%[s4]) \n\t" + + : [p3] "=&r"(p3), [p2] "=&r"(p2), [p1] "=&r"(p1), [p0] "=&r"(p0), + [p7] "=&r"(p7), [p6] "=&r"(p6), [p5] "=&r"(p5), [p4] "=&r"(p4) + : [s1] "r"(s1), [s2] "r"(s2), [s3] "r"(s3), [s4] "r"(s4)); + + __asm__ __volatile__( + "lw %[q3], (%[s1]) \n\t" + "lw %[q2], (%[s2]) \n\t" + "lw %[q1], (%[s3]) \n\t" + "lw %[q0], (%[s4]) \n\t" + "lw %[q7], +4(%[s1]) \n\t" + "lw %[q6], +4(%[s2]) \n\t" + "lw %[q5], +4(%[s3]) \n\t" + "lw %[q4], +4(%[s4]) \n\t" + + : [q3] "=&r"(q3), [q2] "=&r"(q2), [q1] "=&r"(q1), [q0] "=&r"(q0), + [q7] "=&r"(q7), [q6] "=&r"(q6), [q5] "=&r"(q5), [q4] "=&r"(q4) + : [s1] "r"(s1), [s2] "r"(s2), [s3] "r"(s3), [s4] "r"(s4)); + + /* transpose p3, p2, p1, p0 + original (when loaded from memory) + register -4 -3 -2 -1 + p0 p0_0 p0_1 p0_2 p0_3 + p1 p1_0 p1_1 p1_2 p1_3 + p2 p2_0 p2_1 p2_2 p2_3 + p3 p3_0 p3_1 p3_2 p3_3 + + after transpose + register + p0 p3_3 p2_3 p1_3 p0_3 + p1 p3_2 p2_2 p1_2 p0_2 + p2 p3_1 p2_1 p1_1 p0_1 + p3 p3_0 p2_0 p1_0 p0_0 + */ + __asm__ __volatile__( + "precrq.qb.ph %[prim1], %[p0], %[p1] \n\t" + "precr.qb.ph %[prim2], %[p0], %[p1] \n\t" + "precrq.qb.ph %[prim3], %[p2], %[p3] \n\t" + "precr.qb.ph %[prim4], %[p2], %[p3] \n\t" + + "precrq.qb.ph %[p1], %[prim1], %[prim2] \n\t" + "precr.qb.ph %[p3], %[prim1], %[prim2] \n\t" + "precrq.qb.ph %[sec3], %[prim3], %[prim4] \n\t" + "precr.qb.ph %[sec4], %[prim3], %[prim4] \n\t" + + "precrq.ph.w %[p0], %[p1], %[sec3] \n\t" + "precrq.ph.w %[p2], %[p3], %[sec4] \n\t" + "append %[p1], %[sec3], 16 \n\t" + "append %[p3], %[sec4], 16 \n\t" + + : [prim1] "=&r"(prim1), [prim2] "=&r"(prim2), [prim3] "=&r"(prim3), + [prim4] "=&r"(prim4), [p0] "+r"(p0), [p1] "+r"(p1), [p2] "+r"(p2), + [p3] "+r"(p3), [sec3] "=&r"(sec3), [sec4] "=&r"(sec4) + :); + + /* transpose q0, q1, q2, q3 + original (when loaded from memory) + register +1 +2 +3 +4 + q3 q3_0 q3_1 q3_2 q3_3 + q2 q2_0 q2_1 q2_2 q2_3 + q1 q1_0 q1_1 q1_2 q1_3 + q0 q0_0 q0_1 q0_2 q0_3 + + after transpose + register + q3 q0_3 q1_3 q2_3 q3_3 + q2 q0_2 q1_2 q2_2 q3_2 + q1 q0_1 q1_1 q2_1 q3_1 + q0 q0_0 q1_0 q2_0 q3_0 + */ + __asm__ __volatile__( + "precrq.qb.ph %[prim1], %[q3], %[q2] \n\t" + "precr.qb.ph %[prim2], %[q3], %[q2] \n\t" + "precrq.qb.ph %[prim3], %[q1], %[q0] \n\t" + "precr.qb.ph %[prim4], %[q1], %[q0] \n\t" + + "precrq.qb.ph %[q2], %[prim1], %[prim2] \n\t" + "precr.qb.ph %[q0], %[prim1], %[prim2] \n\t" + "precrq.qb.ph %[sec3], %[prim3], %[prim4] \n\t" + "precr.qb.ph %[sec4], %[prim3], %[prim4] \n\t" + + "precrq.ph.w %[q3], %[q2], %[sec3] \n\t" + "precrq.ph.w %[q1], %[q0], %[sec4] \n\t" + "append %[q2], %[sec3], 16 \n\t" + "append %[q0], %[sec4], 16 \n\t" + + : [prim1] "=&r"(prim1), [prim2] "=&r"(prim2), [prim3] "=&r"(prim3), + [prim4] "=&r"(prim4), [q3] "+r"(q3), [q2] "+r"(q2), [q1] "+r"(q1), + [q0] "+r"(q0), [sec3] "=&r"(sec3), [sec4] "=&r"(sec4) + :); + + /* transpose p7, p6, p5, p4 + original (when loaded from memory) + register -8 -7 -6 -5 + p4 p4_0 p4_1 p4_2 p4_3 + p5 p5_0 p5_1 p5_2 p5_3 + p6 p6_0 p6_1 p6_2 p6_3 + p7 p7_0 p7_1 p7_2 p7_3 + + after transpose + register + p4 p7_3 p6_3 p5_3 p4_3 + p5 p7_2 p6_2 p5_2 p4_2 + p6 p7_1 p6_1 p5_1 p4_1 + p7 p7_0 p6_0 p5_0 p4_0 + */ + __asm__ __volatile__( + "precrq.qb.ph %[prim1], %[p4], %[p5] \n\t" + "precr.qb.ph %[prim2], %[p4], %[p5] \n\t" + "precrq.qb.ph %[prim3], %[p6], %[p7] \n\t" + "precr.qb.ph %[prim4], %[p6], %[p7] \n\t" + + "precrq.qb.ph %[p5], %[prim1], %[prim2] \n\t" + "precr.qb.ph %[p7], %[prim1], %[prim2] \n\t" + "precrq.qb.ph %[sec3], %[prim3], %[prim4] \n\t" + "precr.qb.ph %[sec4], %[prim3], %[prim4] \n\t" + + "precrq.ph.w %[p4], %[p5], %[sec3] \n\t" + "precrq.ph.w %[p6], %[p7], %[sec4] \n\t" + "append %[p5], %[sec3], 16 \n\t" + "append %[p7], %[sec4], 16 \n\t" + + : [prim1] "=&r"(prim1), [prim2] "=&r"(prim2), [prim3] "=&r"(prim3), + [prim4] "=&r"(prim4), [p4] "+r"(p4), [p5] "+r"(p5), [p6] "+r"(p6), + [p7] "+r"(p7), [sec3] "=&r"(sec3), [sec4] "=&r"(sec4) + :); + + /* transpose q4, q5, q6, q7 + original (when loaded from memory) + register +5 +6 +7 +8 + q7 q7_0 q7_1 q7_2 q7_3 + q6 q6_0 q6_1 q6_2 q6_3 + q5 q5_0 q5_1 q5_2 q5_3 + q4 q4_0 q4_1 q4_2 q4_3 + + after transpose + register + q7 q4_3 q5_3 q26_3 q7_3 + q6 q4_2 q5_2 q26_2 q7_2 + q5 q4_1 q5_1 q26_1 q7_1 + q4 q4_0 q5_0 q26_0 q7_0 + */ + __asm__ __volatile__( + "precrq.qb.ph %[prim1], %[q7], %[q6] \n\t" + "precr.qb.ph %[prim2], %[q7], %[q6] \n\t" + "precrq.qb.ph %[prim3], %[q5], %[q4] \n\t" + "precr.qb.ph %[prim4], %[q5], %[q4] \n\t" + + "precrq.qb.ph %[q6], %[prim1], %[prim2] \n\t" + "precr.qb.ph %[q4], %[prim1], %[prim2] \n\t" + "precrq.qb.ph %[sec3], %[prim3], %[prim4] \n\t" + "precr.qb.ph %[sec4], %[prim3], %[prim4] \n\t" + + "precrq.ph.w %[q7], %[q6], %[sec3] \n\t" + "precrq.ph.w %[q5], %[q4], %[sec4] \n\t" + "append %[q6], %[sec3], 16 \n\t" + "append %[q4], %[sec4], 16 \n\t" + + : [prim1] "=&r"(prim1), [prim2] "=&r"(prim2), [prim3] "=&r"(prim3), + [prim4] "=&r"(prim4), [q7] "+r"(q7), [q6] "+r"(q6), [q5] "+r"(q5), + [q4] "+r"(q4), [sec3] "=&r"(sec3), [sec4] "=&r"(sec4) + :); + + filter_hev_mask_flatmask4_dspr2(limit_vec, flimit_vec, thresh_vec, p1, p0, + p3, p2, q0, q1, q2, q3, &hev, &mask, &flat); + + flatmask5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, &flat2); + + /* f0 */ + if (((flat2 == 0) && (flat == 0) && (mask != 0)) || + ((flat2 != 0) && (flat == 0) && (mask != 0))) { + filter1_dspr2(mask, hev, p1, p0, q0, q1, &p1_f0, &p0_f0, &q0_f0, &q1_f0); + STORE_F0() + } else if ((flat2 == 0XFFFFFFFF) && (flat == 0xFFFFFFFF) && + (mask == 0xFFFFFFFF)) { + /* f2 */ + PACK_LEFT_0TO3() + PACK_LEFT_4TO7() + wide_mbfilter_dspr2(&p7_l, &p6_l, &p5_l, &p4_l, &p3_l, &p2_l, &p1_l, + &p0_l, &q0_l, &q1_l, &q2_l, &q3_l, &q4_l, &q5_l, + &q6_l, &q7_l); + + PACK_RIGHT_0TO3() + PACK_RIGHT_4TO7() + wide_mbfilter_dspr2(&p7_r, &p6_r, &p5_r, &p4_r, &p3_r, &p2_r, &p1_r, + &p0_r, &q0_r, &q1_r, &q2_r, &q3_r, &q4_r, &q5_r, + &q6_r, &q7_r); + + STORE_F2() + } else if ((flat2 == 0) && (flat == 0xFFFFFFFF) && (mask == 0xFFFFFFFF)) { + /* f1 */ + PACK_LEFT_0TO3() + mbfilter_dspr2(&p3_l, &p2_l, &p1_l, &p0_l, &q0_l, &q1_l, &q2_l, &q3_l); + + PACK_RIGHT_0TO3() + mbfilter_dspr2(&p3_r, &p2_r, &p1_r, &p0_r, &q0_r, &q1_r, &q2_r, &q3_r); + + STORE_F1() + } else if ((flat2 == 0) && (flat != 0) && (mask != 0)) { + /* f0 + f1 */ + filter1_dspr2(mask, hev, p1, p0, q0, q1, &p1_f0, &p0_f0, &q0_f0, &q1_f0); + + /* left 2 element operation */ + PACK_LEFT_0TO3() + mbfilter_dspr2(&p3_l, &p2_l, &p1_l, &p0_l, &q0_l, &q1_l, &q2_l, &q3_l); + + /* right 2 element operation */ + PACK_RIGHT_0TO3() + mbfilter_dspr2(&p3_r, &p2_r, &p1_r, &p0_r, &q0_r, &q1_r, &q2_r, &q3_r); + + if (mask & flat & 0x000000FF) { + __asm__ __volatile__( + "sb %[p2_r], -3(%[s4]) \n\t" + "sb %[p1_r], -2(%[s4]) \n\t" + "sb %[p0_r], -1(%[s4]) \n\t" + "sb %[q0_r], (%[s4]) \n\t" + "sb %[q1_r], +1(%[s4]) \n\t" + "sb %[q2_r], +2(%[s4]) \n\t" + + : + : [p2_r] "r"(p2_r), [p1_r] "r"(p1_r), [p0_r] "r"(p0_r), + [q0_r] "r"(q0_r), [q1_r] "r"(q1_r), [q2_r] "r"(q2_r), + [s4] "r"(s4)); + } else if (mask & 0x000000FF) { + __asm__ __volatile__( + "sb %[p1_f0], -2(%[s4]) \n\t" + "sb %[p0_f0], -1(%[s4]) \n\t" + "sb %[q0_f0], (%[s4]) \n\t" + "sb %[q1_f0], +1(%[s4]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [s4] "r"(s4)); + } + + __asm__ __volatile__( + "srl %[p2_r], %[p2_r], 16 \n\t" + "srl %[p1_r], %[p1_r], 16 \n\t" + "srl %[p0_r], %[p0_r], 16 \n\t" + "srl %[q0_r], %[q0_r], 16 \n\t" + "srl %[q1_r], %[q1_r], 16 \n\t" + "srl %[q2_r], %[q2_r], 16 \n\t" + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2_r] "+r"(p2_r), [p1_r] "+r"(p1_r), [p0_r] "+r"(p0_r), + [q0_r] "+r"(q0_r), [q1_r] "+r"(q1_r), [q2_r] "+r"(q2_r), + [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p2_r], -3(%[s3]) \n\t" + "sb %[p1_r], -2(%[s3]) \n\t" + "sb %[p0_r], -1(%[s3]) \n\t" + "sb %[q0_r], (%[s3]) \n\t" + "sb %[q1_r], +1(%[s3]) \n\t" + "sb %[q2_r], +2(%[s3]) \n\t" + + : + : [p2_r] "r"(p2_r), [p1_r] "r"(p1_r), [p0_r] "r"(p0_r), + [q0_r] "r"(q0_r), [q1_r] "r"(q1_r), [q2_r] "r"(q2_r), + [s3] "r"(s3)); + } else if (mask & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p1_f0], -2(%[s3]) \n\t" + "sb %[p0_f0], -1(%[s3]) \n\t" + "sb %[q0_f0], (%[s3]) \n\t" + "sb %[q1_f0], +1(%[s3]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [s3] "r"(s3)); + } + + __asm__ __volatile__( + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p2_l], -3(%[s2]) \n\t" + "sb %[p1_l], -2(%[s2]) \n\t" + "sb %[p0_l], -1(%[s2]) \n\t" + "sb %[q0_l], (%[s2]) \n\t" + "sb %[q1_l], +1(%[s2]) \n\t" + "sb %[q2_l], +2(%[s2]) \n\t" + + : + : [p2_l] "r"(p2_l), [p1_l] "r"(p1_l), [p0_l] "r"(p0_l), + [q0_l] "r"(q0_l), [q1_l] "r"(q1_l), [q2_l] "r"(q2_l), + [s2] "r"(s2)); + } else if (mask & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p1_f0], -2(%[s2]) \n\t" + "sb %[p0_f0], -1(%[s2]) \n\t" + "sb %[q0_f0], (%[s2]) \n\t" + "sb %[q1_f0], +1(%[s2]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [s2] "r"(s2)); + } + + __asm__ __volatile__( + "srl %[p2_l], %[p2_l], 16 \n\t" + "srl %[p1_l], %[p1_l], 16 \n\t" + "srl %[p0_l], %[p0_l], 16 \n\t" + "srl %[q0_l], %[q0_l], 16 \n\t" + "srl %[q1_l], %[q1_l], 16 \n\t" + "srl %[q2_l], %[q2_l], 16 \n\t" + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2_l] "+r"(p2_l), [p1_l] "+r"(p1_l), [p0_l] "+r"(p0_l), + [q0_l] "+r"(q0_l), [q1_l] "+r"(q1_l), [q2_l] "+r"(q2_l), + [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & 0xFF000000) { + __asm__ __volatile__( + "sb %[p2_l], -3(%[s1]) \n\t" + "sb %[p1_l], -2(%[s1]) \n\t" + "sb %[p0_l], -1(%[s1]) \n\t" + "sb %[q0_l], (%[s1]) \n\t" + "sb %[q1_l], +1(%[s1]) \n\t" + "sb %[q2_l], +2(%[s1]) \n\t" + + : + : [p2_l] "r"(p2_l), [p1_l] "r"(p1_l), [p0_l] "r"(p0_l), + [q0_l] "r"(q0_l), [q1_l] "r"(q1_l), [q2_l] "r"(q2_l), + [s1] "r"(s1)); + } else if (mask & 0xFF000000) { + __asm__ __volatile__( + "sb %[p1_f0], -2(%[s1]) \n\t" + "sb %[p0_f0], -1(%[s1]) \n\t" + "sb %[q0_f0], (%[s1]) \n\t" + "sb %[q1_f0], +1(%[s1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [s1] "r"(s1)); + } + } else if ((flat2 != 0) && (flat != 0) && (mask != 0)) { + /* f0+f1+f2 */ + filter1_dspr2(mask, hev, p1, p0, q0, q1, &p1_f0, &p0_f0, &q0_f0, &q1_f0); + + PACK_LEFT_0TO3() + mbfilter1_dspr2(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, &p2_l_f1, + &p1_l_f1, &p0_l_f1, &q0_l_f1, &q1_l_f1, &q2_l_f1); + + PACK_RIGHT_0TO3() + mbfilter1_dspr2(p3_r, p2_r, p1_r, p0_r, q0_r, q1_r, q2_r, q3_r, &p2_r_f1, + &p1_r_f1, &p0_r_f1, &q0_r_f1, &q1_r_f1, &q2_r_f1); + + PACK_LEFT_4TO7() + wide_mbfilter_dspr2(&p7_l, &p6_l, &p5_l, &p4_l, &p3_l, &p2_l, &p1_l, + &p0_l, &q0_l, &q1_l, &q2_l, &q3_l, &q4_l, &q5_l, + &q6_l, &q7_l); + + PACK_RIGHT_4TO7() + wide_mbfilter_dspr2(&p7_r, &p6_r, &p5_r, &p4_r, &p3_r, &p2_r, &p1_r, + &p0_r, &q0_r, &q1_r, &q2_r, &q3_r, &q4_r, &q5_r, + &q6_r, &q7_r); + + if (mask & flat & flat2 & 0x000000FF) { + __asm__ __volatile__( + "sb %[p6_r], -7(%[s4]) \n\t" + "sb %[p5_r], -6(%[s4]) \n\t" + "sb %[p4_r], -5(%[s4]) \n\t" + "sb %[p3_r], -4(%[s4]) \n\t" + "sb %[p2_r], -3(%[s4]) \n\t" + "sb %[p1_r], -2(%[s4]) \n\t" + "sb %[p0_r], -1(%[s4]) \n\t" + + : + : [p6_r] "r"(p6_r), [p5_r] "r"(p5_r), [p4_r] "r"(p4_r), + [p3_r] "r"(p3_r), [p2_r] "r"(p2_r), [p1_r] "r"(p1_r), + [p0_r] "r"(p0_r), [s4] "r"(s4)); + + __asm__ __volatile__( + "sb %[q0_r], (%[s4]) \n\t" + "sb %[q1_r], +1(%[s4]) \n\t" + "sb %[q2_r], +2(%[s4]) \n\t" + "sb %[q3_r], +3(%[s4]) \n\t" + "sb %[q4_r], +4(%[s4]) \n\t" + "sb %[q5_r], +5(%[s4]) \n\t" + "sb %[q6_r], +6(%[s4]) \n\t" + + : + : [q0_r] "r"(q0_r), [q1_r] "r"(q1_r), [q2_r] "r"(q2_r), + [q3_r] "r"(q3_r), [q4_r] "r"(q4_r), [q5_r] "r"(q5_r), + [q6_r] "r"(q6_r), [s4] "r"(s4)); + } else if (mask & flat & 0x000000FF) { + __asm__ __volatile__( + "sb %[p2_r_f1], -3(%[s4]) \n\t" + "sb %[p1_r_f1], -2(%[s4]) \n\t" + "sb %[p0_r_f1], -1(%[s4]) \n\t" + "sb %[q0_r_f1], (%[s4]) \n\t" + "sb %[q1_r_f1], +1(%[s4]) \n\t" + "sb %[q2_r_f1], +2(%[s4]) \n\t" + + : + : [p2_r_f1] "r"(p2_r_f1), [p1_r_f1] "r"(p1_r_f1), + [p0_r_f1] "r"(p0_r_f1), [q0_r_f1] "r"(q0_r_f1), + [q1_r_f1] "r"(q1_r_f1), [q2_r_f1] "r"(q2_r_f1), [s4] "r"(s4)); + } else if (mask & 0x000000FF) { + __asm__ __volatile__( + "sb %[p1_f0], -2(%[s4]) \n\t" + "sb %[p0_f0], -1(%[s4]) \n\t" + "sb %[q0_f0], (%[s4]) \n\t" + "sb %[q1_f0], +1(%[s4]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [s4] "r"(s4)); + } + + __asm__ __volatile__( + "srl %[p6_r], %[p6_r], 16 \n\t" + "srl %[p5_r], %[p5_r], 16 \n\t" + "srl %[p4_r], %[p4_r], 16 \n\t" + "srl %[p3_r], %[p3_r], 16 \n\t" + "srl %[p2_r], %[p2_r], 16 \n\t" + "srl %[p1_r], %[p1_r], 16 \n\t" + "srl %[p0_r], %[p0_r], 16 \n\t" + "srl %[q0_r], %[q0_r], 16 \n\t" + "srl %[q1_r], %[q1_r], 16 \n\t" + "srl %[q2_r], %[q2_r], 16 \n\t" + "srl %[q3_r], %[q3_r], 16 \n\t" + "srl %[q4_r], %[q4_r], 16 \n\t" + "srl %[q5_r], %[q5_r], 16 \n\t" + "srl %[q6_r], %[q6_r], 16 \n\t" + + : [q0_r] "+r"(q0_r), [q1_r] "+r"(q1_r), [q2_r] "+r"(q2_r), + [q3_r] "+r"(q3_r), [q4_r] "+r"(q4_r), [q5_r] "+r"(q5_r), + [q6_r] "+r"(q6_r), [p6_r] "+r"(p6_r), [p5_r] "+r"(p5_r), + [p4_r] "+r"(p4_r), [p3_r] "+r"(p3_r), [p2_r] "+r"(p2_r), + [p1_r] "+r"(p1_r), [p0_r] "+r"(p0_r) + :); + + __asm__ __volatile__( + "srl %[p2_r_f1], %[p2_r_f1], 16 \n\t" + "srl %[p1_r_f1], %[p1_r_f1], 16 \n\t" + "srl %[p0_r_f1], %[p0_r_f1], 16 \n\t" + "srl %[q0_r_f1], %[q0_r_f1], 16 \n\t" + "srl %[q1_r_f1], %[q1_r_f1], 16 \n\t" + "srl %[q2_r_f1], %[q2_r_f1], 16 \n\t" + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2_r_f1] "+r"(p2_r_f1), [p1_r_f1] "+r"(p1_r_f1), + [p0_r_f1] "+r"(p0_r_f1), [q0_r_f1] "+r"(q0_r_f1), + [q1_r_f1] "+r"(q1_r_f1), [q2_r_f1] "+r"(q2_r_f1), + [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & flat2 & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p6_r], -7(%[s3]) \n\t" + "sb %[p5_r], -6(%[s3]) \n\t" + "sb %[p4_r], -5(%[s3]) \n\t" + "sb %[p3_r], -4(%[s3]) \n\t" + "sb %[p2_r], -3(%[s3]) \n\t" + "sb %[p1_r], -2(%[s3]) \n\t" + "sb %[p0_r], -1(%[s3]) \n\t" + + : + : [p6_r] "r"(p6_r), [p5_r] "r"(p5_r), [p4_r] "r"(p4_r), + [p3_r] "r"(p3_r), [p2_r] "r"(p2_r), [p1_r] "r"(p1_r), + [p0_r] "r"(p0_r), [s3] "r"(s3)); + + __asm__ __volatile__( + "sb %[q0_r], (%[s3]) \n\t" + "sb %[q1_r], +1(%[s3]) \n\t" + "sb %[q2_r], +2(%[s3]) \n\t" + "sb %[q3_r], +3(%[s3]) \n\t" + "sb %[q4_r], +4(%[s3]) \n\t" + "sb %[q5_r], +5(%[s3]) \n\t" + "sb %[q6_r], +6(%[s3]) \n\t" + + : + : [q0_r] "r"(q0_r), [q1_r] "r"(q1_r), [q2_r] "r"(q2_r), + [q3_r] "r"(q3_r), [q4_r] "r"(q4_r), [q5_r] "r"(q5_r), + [q6_r] "r"(q6_r), [s3] "r"(s3)); + } else if (mask & flat & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p2_r_f1], -3(%[s3]) \n\t" + "sb %[p1_r_f1], -2(%[s3]) \n\t" + "sb %[p0_r_f1], -1(%[s3]) \n\t" + "sb %[q0_r_f1], (%[s3]) \n\t" + "sb %[q1_r_f1], +1(%[s3]) \n\t" + "sb %[q2_r_f1], +2(%[s3]) \n\t" + + : + : [p2_r_f1] "r"(p2_r_f1), [p1_r_f1] "r"(p1_r_f1), + [p0_r_f1] "r"(p0_r_f1), [q0_r_f1] "r"(q0_r_f1), + [q1_r_f1] "r"(q1_r_f1), [q2_r_f1] "r"(q2_r_f1), [s3] "r"(s3)); + } else if (mask & 0x0000FF00) { + __asm__ __volatile__( + "sb %[p1_f0], -2(%[s3]) \n\t" + "sb %[p0_f0], -1(%[s3]) \n\t" + "sb %[q0_f0], (%[s3]) \n\t" + "sb %[q1_f0], +1(%[s3]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [s3] "r"(s3)); + } + + __asm__ __volatile__( + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & flat2 & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p6_l], -7(%[s2]) \n\t" + "sb %[p5_l], -6(%[s2]) \n\t" + "sb %[p4_l], -5(%[s2]) \n\t" + "sb %[p3_l], -4(%[s2]) \n\t" + "sb %[p2_l], -3(%[s2]) \n\t" + "sb %[p1_l], -2(%[s2]) \n\t" + "sb %[p0_l], -1(%[s2]) \n\t" + + : + : [p6_l] "r"(p6_l), [p5_l] "r"(p5_l), [p4_l] "r"(p4_l), + [p3_l] "r"(p3_l), [p2_l] "r"(p2_l), [p1_l] "r"(p1_l), + [p0_l] "r"(p0_l), [s2] "r"(s2)); + + __asm__ __volatile__( + "sb %[q0_l], (%[s2]) \n\t" + "sb %[q1_l], +1(%[s2]) \n\t" + "sb %[q2_l], +2(%[s2]) \n\t" + "sb %[q3_l], +3(%[s2]) \n\t" + "sb %[q4_l], +4(%[s2]) \n\t" + "sb %[q5_l], +5(%[s2]) \n\t" + "sb %[q6_l], +6(%[s2]) \n\t" + + : + : [q0_l] "r"(q0_l), [q1_l] "r"(q1_l), [q2_l] "r"(q2_l), + [q3_l] "r"(q3_l), [q4_l] "r"(q4_l), [q5_l] "r"(q5_l), + [q6_l] "r"(q6_l), [s2] "r"(s2)); + } else if (mask & flat & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p2_l_f1], -3(%[s2]) \n\t" + "sb %[p1_l_f1], -2(%[s2]) \n\t" + "sb %[p0_l_f1], -1(%[s2]) \n\t" + "sb %[q0_l_f1], (%[s2]) \n\t" + "sb %[q1_l_f1], +1(%[s2]) \n\t" + "sb %[q2_l_f1], +2(%[s2]) \n\t" + + : + : [p2_l_f1] "r"(p2_l_f1), [p1_l_f1] "r"(p1_l_f1), + [p0_l_f1] "r"(p0_l_f1), [q0_l_f1] "r"(q0_l_f1), + [q1_l_f1] "r"(q1_l_f1), [q2_l_f1] "r"(q2_l_f1), [s2] "r"(s2)); + } else if (mask & 0x00FF0000) { + __asm__ __volatile__( + "sb %[p1_f0], -2(%[s2]) \n\t" + "sb %[p0_f0], -1(%[s2]) \n\t" + "sb %[q0_f0], (%[s2]) \n\t" + "sb %[q1_f0], +1(%[s2]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [s2] "r"(s2)); + } + + __asm__ __volatile__( + "srl %[p6_l], %[p6_l], 16 \n\t" + "srl %[p5_l], %[p5_l], 16 \n\t" + "srl %[p4_l], %[p4_l], 16 \n\t" + "srl %[p3_l], %[p3_l], 16 \n\t" + "srl %[p2_l], %[p2_l], 16 \n\t" + "srl %[p1_l], %[p1_l], 16 \n\t" + "srl %[p0_l], %[p0_l], 16 \n\t" + "srl %[q0_l], %[q0_l], 16 \n\t" + "srl %[q1_l], %[q1_l], 16 \n\t" + "srl %[q2_l], %[q2_l], 16 \n\t" + "srl %[q3_l], %[q3_l], 16 \n\t" + "srl %[q4_l], %[q4_l], 16 \n\t" + "srl %[q5_l], %[q5_l], 16 \n\t" + "srl %[q6_l], %[q6_l], 16 \n\t" + + : [q0_l] "+r"(q0_l), [q1_l] "+r"(q1_l), [q2_l] "+r"(q2_l), + [q3_l] "+r"(q3_l), [q4_l] "+r"(q4_l), [q5_l] "+r"(q5_l), + [q6_l] "+r"(q6_l), [p6_l] "+r"(p6_l), [p5_l] "+r"(p5_l), + [p4_l] "+r"(p4_l), [p3_l] "+r"(p3_l), [p2_l] "+r"(p2_l), + [p1_l] "+r"(p1_l), [p0_l] "+r"(p0_l) + :); + + __asm__ __volatile__( + "srl %[p2_l_f1], %[p2_l_f1], 16 \n\t" + "srl %[p1_l_f1], %[p1_l_f1], 16 \n\t" + "srl %[p0_l_f1], %[p0_l_f1], 16 \n\t" + "srl %[q0_l_f1], %[q0_l_f1], 16 \n\t" + "srl %[q1_l_f1], %[q1_l_f1], 16 \n\t" + "srl %[q2_l_f1], %[q2_l_f1], 16 \n\t" + "srl %[p1_f0], %[p1_f0], 8 \n\t" + "srl %[p0_f0], %[p0_f0], 8 \n\t" + "srl %[q0_f0], %[q0_f0], 8 \n\t" + "srl %[q1_f0], %[q1_f0], 8 \n\t" + + : [p2_l_f1] "+r"(p2_l_f1), [p1_l_f1] "+r"(p1_l_f1), + [p0_l_f1] "+r"(p0_l_f1), [q0_l_f1] "+r"(q0_l_f1), + [q1_l_f1] "+r"(q1_l_f1), [q2_l_f1] "+r"(q2_l_f1), + [p1_f0] "+r"(p1_f0), [p0_f0] "+r"(p0_f0), [q0_f0] "+r"(q0_f0), + [q1_f0] "+r"(q1_f0) + :); + + if (mask & flat & flat2 & 0xFF000000) { + __asm__ __volatile__( + "sb %[p6_l], -7(%[s1]) \n\t" + "sb %[p5_l], -6(%[s1]) \n\t" + "sb %[p4_l], -5(%[s1]) \n\t" + "sb %[p3_l], -4(%[s1]) \n\t" + "sb %[p2_l], -3(%[s1]) \n\t" + "sb %[p1_l], -2(%[s1]) \n\t" + "sb %[p0_l], -1(%[s1]) \n\t" + + : + : [p6_l] "r"(p6_l), [p5_l] "r"(p5_l), [p4_l] "r"(p4_l), + [p3_l] "r"(p3_l), [p2_l] "r"(p2_l), [p1_l] "r"(p1_l), + [p0_l] "r"(p0_l), [s1] "r"(s1)); + + __asm__ __volatile__( + "sb %[q0_l], (%[s1]) \n\t" + "sb %[q1_l], 1(%[s1]) \n\t" + "sb %[q2_l], 2(%[s1]) \n\t" + "sb %[q3_l], 3(%[s1]) \n\t" + "sb %[q4_l], 4(%[s1]) \n\t" + "sb %[q5_l], 5(%[s1]) \n\t" + "sb %[q6_l], 6(%[s1]) \n\t" + + : + : [q0_l] "r"(q0_l), [q1_l] "r"(q1_l), [q2_l] "r"(q2_l), + [q3_l] "r"(q3_l), [q4_l] "r"(q4_l), [q5_l] "r"(q5_l), + [q6_l] "r"(q6_l), [s1] "r"(s1)); + } else if (mask & flat & 0xFF000000) { + __asm__ __volatile__( + "sb %[p2_l_f1], -3(%[s1]) \n\t" + "sb %[p1_l_f1], -2(%[s1]) \n\t" + "sb %[p0_l_f1], -1(%[s1]) \n\t" + "sb %[q0_l_f1], (%[s1]) \n\t" + "sb %[q1_l_f1], +1(%[s1]) \n\t" + "sb %[q2_l_f1], +2(%[s1]) \n\t" + + : + : [p2_l_f1] "r"(p2_l_f1), [p1_l_f1] "r"(p1_l_f1), + [p0_l_f1] "r"(p0_l_f1), [q0_l_f1] "r"(q0_l_f1), + [q1_l_f1] "r"(q1_l_f1), [q2_l_f1] "r"(q2_l_f1), [s1] "r"(s1)); + } else if (mask & 0xFF000000) { + __asm__ __volatile__( + "sb %[p1_f0], -2(%[s1]) \n\t" + "sb %[p0_f0], -1(%[s1]) \n\t" + "sb %[q0_f0], (%[s1]) \n\t" + "sb %[q1_f0], +1(%[s1]) \n\t" + + : + : [p1_f0] "r"(p1_f0), [p0_f0] "r"(p0_f0), [q0_f0] "r"(q0_f0), + [q1_f0] "r"(q1_f0), [s1] "r"(s1)); + } + } + } +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/aom_dsp/mips/loopfilter_msa.h b/third_party/aom/aom_dsp/mips/loopfilter_msa.h new file mode 100644 index 0000000000..450594262c --- /dev/null +++ b/third_party/aom/aom_dsp/mips/loopfilter_msa.h @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_LOOPFILTER_MSA_H_ +#define AOM_DSP_LOOPFILTER_MSA_H_ + +#include "aom_dsp/mips/macros_msa.h" + +#define AOM_LPF_FILTER4_8W(p1_in, p0_in, q0_in, q1_in, mask_in, hev_in, \ + p1_out, p0_out, q0_out, q1_out) \ + { \ + v16i8 p1_m, p0_m, q0_m, q1_m, q0_sub_p0, filt_sign; \ + v16i8 filt, filt1, filt2, cnst4b, cnst3b; \ + v8i16 q0_sub_p0_r, filt_r, cnst3h; \ + \ + p1_m = (v16i8)__msa_xori_b(p1_in, 0x80); \ + p0_m = (v16i8)__msa_xori_b(p0_in, 0x80); \ + q0_m = (v16i8)__msa_xori_b(q0_in, 0x80); \ + q1_m = (v16i8)__msa_xori_b(q1_in, 0x80); \ + \ + filt = __msa_subs_s_b(p1_m, q1_m); \ + filt = filt & (v16i8)hev_in; \ + q0_sub_p0 = q0_m - p0_m; \ + filt_sign = __msa_clti_s_b(filt, 0); \ + \ + cnst3h = __msa_ldi_h(3); \ + q0_sub_p0_r = (v8i16)__msa_ilvr_b(q0_sub_p0, q0_sub_p0); \ + q0_sub_p0_r = __msa_dotp_s_h((v16i8)q0_sub_p0_r, (v16i8)cnst3h); \ + filt_r = (v8i16)__msa_ilvr_b(filt_sign, filt); \ + filt_r += q0_sub_p0_r; \ + filt_r = __msa_sat_s_h(filt_r, 7); \ + \ + /* combine left and right part */ \ + filt = __msa_pckev_b((v16i8)filt_r, (v16i8)filt_r); \ + \ + filt = filt & (v16i8)mask_in; \ + cnst4b = __msa_ldi_b(4); \ + filt1 = __msa_adds_s_b(filt, cnst4b); \ + filt1 >>= 3; \ + \ + cnst3b = __msa_ldi_b(3); \ + filt2 = __msa_adds_s_b(filt, cnst3b); \ + filt2 >>= 3; \ + \ + q0_m = __msa_subs_s_b(q0_m, filt1); \ + q0_out = __msa_xori_b((v16u8)q0_m, 0x80); \ + p0_m = __msa_adds_s_b(p0_m, filt2); \ + p0_out = __msa_xori_b((v16u8)p0_m, 0x80); \ + \ + filt = __msa_srari_b(filt1, 1); \ + hev_in = __msa_xori_b((v16u8)hev_in, 0xff); \ + filt = filt & (v16i8)hev_in; \ + \ + q1_m = __msa_subs_s_b(q1_m, filt); \ + q1_out = __msa_xori_b((v16u8)q1_m, 0x80); \ + p1_m = __msa_adds_s_b(p1_m, filt); \ + p1_out = __msa_xori_b((v16u8)p1_m, 0x80); \ + } + +#define AOM_LPF_FILTER4_4W(p1_in, p0_in, q0_in, q1_in, mask_in, hev_in, \ + p1_out, p0_out, q0_out, q1_out) \ + { \ + v16i8 p1_m, p0_m, q0_m, q1_m, q0_sub_p0, filt_sign; \ + v16i8 filt, filt1, filt2, cnst4b, cnst3b; \ + v8i16 q0_sub_p0_r, q0_sub_p0_l, filt_l, filt_r, cnst3h; \ + \ + p1_m = (v16i8)__msa_xori_b(p1_in, 0x80); \ + p0_m = (v16i8)__msa_xori_b(p0_in, 0x80); \ + q0_m = (v16i8)__msa_xori_b(q0_in, 0x80); \ + q1_m = (v16i8)__msa_xori_b(q1_in, 0x80); \ + \ + filt = __msa_subs_s_b(p1_m, q1_m); \ + \ + filt = filt & (v16i8)hev_in; \ + \ + q0_sub_p0 = q0_m - p0_m; \ + filt_sign = __msa_clti_s_b(filt, 0); \ + \ + cnst3h = __msa_ldi_h(3); \ + q0_sub_p0_r = (v8i16)__msa_ilvr_b(q0_sub_p0, q0_sub_p0); \ + q0_sub_p0_r = __msa_dotp_s_h((v16i8)q0_sub_p0_r, (v16i8)cnst3h); \ + filt_r = (v8i16)__msa_ilvr_b(filt_sign, filt); \ + filt_r += q0_sub_p0_r; \ + filt_r = __msa_sat_s_h(filt_r, 7); \ + \ + q0_sub_p0_l = (v8i16)__msa_ilvl_b(q0_sub_p0, q0_sub_p0); \ + q0_sub_p0_l = __msa_dotp_s_h((v16i8)q0_sub_p0_l, (v16i8)cnst3h); \ + filt_l = (v8i16)__msa_ilvl_b(filt_sign, filt); \ + filt_l += q0_sub_p0_l; \ + filt_l = __msa_sat_s_h(filt_l, 7); \ + \ + filt = __msa_pckev_b((v16i8)filt_l, (v16i8)filt_r); \ + filt = filt & (v16i8)mask_in; \ + \ + cnst4b = __msa_ldi_b(4); \ + filt1 = __msa_adds_s_b(filt, cnst4b); \ + filt1 >>= 3; \ + \ + cnst3b = __msa_ldi_b(3); \ + filt2 = __msa_adds_s_b(filt, cnst3b); \ + filt2 >>= 3; \ + \ + q0_m = __msa_subs_s_b(q0_m, filt1); \ + q0_out = __msa_xori_b((v16u8)q0_m, 0x80); \ + p0_m = __msa_adds_s_b(p0_m, filt2); \ + p0_out = __msa_xori_b((v16u8)p0_m, 0x80); \ + \ + filt = __msa_srari_b(filt1, 1); \ + hev_in = __msa_xori_b((v16u8)hev_in, 0xff); \ + filt = filt & (v16i8)hev_in; \ + \ + q1_m = __msa_subs_s_b(q1_m, filt); \ + q1_out = __msa_xori_b((v16u8)q1_m, 0x80); \ + p1_m = __msa_adds_s_b(p1_m, filt); \ + p1_out = __msa_xori_b((v16u8)p1_m, 0x80); \ + } + +#define AOM_FLAT4(p3_in, p2_in, p0_in, q0_in, q2_in, q3_in, flat_out) \ + { \ + v16u8 tmp_flat4, p2_a_sub_p0, q2_a_sub_q0, p3_a_sub_p0, q3_a_sub_q0; \ + v16u8 zero_in = { 0 }; \ + \ + tmp_flat4 = __msa_ori_b(zero_in, 1); \ + p2_a_sub_p0 = __msa_asub_u_b(p2_in, p0_in); \ + q2_a_sub_q0 = __msa_asub_u_b(q2_in, q0_in); \ + p3_a_sub_p0 = __msa_asub_u_b(p3_in, p0_in); \ + q3_a_sub_q0 = __msa_asub_u_b(q3_in, q0_in); \ + \ + p2_a_sub_p0 = __msa_max_u_b(p2_a_sub_p0, q2_a_sub_q0); \ + flat_out = __msa_max_u_b(p2_a_sub_p0, flat_out); \ + p3_a_sub_p0 = __msa_max_u_b(p3_a_sub_p0, q3_a_sub_q0); \ + flat_out = __msa_max_u_b(p3_a_sub_p0, flat_out); \ + \ + flat_out = (tmp_flat4 < (v16u8)flat_out); \ + flat_out = __msa_xori_b(flat_out, 0xff); \ + flat_out = flat_out & (mask); \ + } + +#define AOM_FLAT5(p7_in, p6_in, p5_in, p4_in, p0_in, q0_in, q4_in, q5_in, \ + q6_in, q7_in, flat_in, flat2_out) \ + { \ + v16u8 tmp_flat5, zero_in = { 0 }; \ + v16u8 p4_a_sub_p0, q4_a_sub_q0, p5_a_sub_p0, q5_a_sub_q0; \ + v16u8 p6_a_sub_p0, q6_a_sub_q0, p7_a_sub_p0, q7_a_sub_q0; \ + \ + tmp_flat5 = __msa_ori_b(zero_in, 1); \ + p4_a_sub_p0 = __msa_asub_u_b(p4_in, p0_in); \ + q4_a_sub_q0 = __msa_asub_u_b(q4_in, q0_in); \ + p5_a_sub_p0 = __msa_asub_u_b(p5_in, p0_in); \ + q5_a_sub_q0 = __msa_asub_u_b(q5_in, q0_in); \ + p6_a_sub_p0 = __msa_asub_u_b(p6_in, p0_in); \ + q6_a_sub_q0 = __msa_asub_u_b(q6_in, q0_in); \ + p7_a_sub_p0 = __msa_asub_u_b(p7_in, p0_in); \ + q7_a_sub_q0 = __msa_asub_u_b(q7_in, q0_in); \ + \ + p4_a_sub_p0 = __msa_max_u_b(p4_a_sub_p0, q4_a_sub_q0); \ + flat2_out = __msa_max_u_b(p5_a_sub_p0, q5_a_sub_q0); \ + flat2_out = __msa_max_u_b(p4_a_sub_p0, flat2_out); \ + p6_a_sub_p0 = __msa_max_u_b(p6_a_sub_p0, q6_a_sub_q0); \ + flat2_out = __msa_max_u_b(p6_a_sub_p0, flat2_out); \ + p7_a_sub_p0 = __msa_max_u_b(p7_a_sub_p0, q7_a_sub_q0); \ + flat2_out = __msa_max_u_b(p7_a_sub_p0, flat2_out); \ + \ + flat2_out = (tmp_flat5 < (v16u8)flat2_out); \ + flat2_out = __msa_xori_b(flat2_out, 0xff); \ + flat2_out = flat2_out & flat_in; \ + } + +#define AOM_FILTER8(p3_in, p2_in, p1_in, p0_in, q0_in, q1_in, q2_in, q3_in, \ + p2_filt8_out, p1_filt8_out, p0_filt8_out, q0_filt8_out, \ + q1_filt8_out, q2_filt8_out) \ + { \ + v8u16 tmp_filt8_0, tmp_filt8_1, tmp_filt8_2; \ + \ + tmp_filt8_2 = p2_in + p1_in + p0_in; \ + tmp_filt8_0 = p3_in << 1; \ + \ + tmp_filt8_0 = tmp_filt8_0 + tmp_filt8_2 + q0_in; \ + tmp_filt8_1 = tmp_filt8_0 + p3_in + p2_in; \ + p2_filt8_out = (v8i16)__msa_srari_h((v8i16)tmp_filt8_1, 3); \ + \ + tmp_filt8_1 = tmp_filt8_0 + p1_in + q1_in; \ + p1_filt8_out = (v8i16)__msa_srari_h((v8i16)tmp_filt8_1, 3); \ + \ + tmp_filt8_1 = q2_in + q1_in + q0_in; \ + tmp_filt8_2 = tmp_filt8_2 + tmp_filt8_1; \ + tmp_filt8_0 = tmp_filt8_2 + (p0_in); \ + tmp_filt8_0 = tmp_filt8_0 + (p3_in); \ + p0_filt8_out = (v8i16)__msa_srari_h((v8i16)tmp_filt8_0, 3); \ + \ + tmp_filt8_0 = q2_in + q3_in; \ + tmp_filt8_0 = p0_in + tmp_filt8_1 + tmp_filt8_0; \ + tmp_filt8_1 = q3_in + q3_in; \ + tmp_filt8_1 = tmp_filt8_1 + tmp_filt8_0; \ + q2_filt8_out = (v8i16)__msa_srari_h((v8i16)tmp_filt8_1, 3); \ + \ + tmp_filt8_0 = tmp_filt8_2 + q3_in; \ + tmp_filt8_1 = tmp_filt8_0 + q0_in; \ + q0_filt8_out = (v8i16)__msa_srari_h((v8i16)tmp_filt8_1, 3); \ + \ + tmp_filt8_1 = tmp_filt8_0 - p2_in; \ + tmp_filt8_0 = q1_in + q3_in; \ + tmp_filt8_1 = tmp_filt8_0 + tmp_filt8_1; \ + q1_filt8_out = (v8i16)__msa_srari_h((v8i16)tmp_filt8_1, 3); \ + } + +#define LPF_MASK_HEV(p3_in, p2_in, p1_in, p0_in, q0_in, q1_in, q2_in, q3_in, \ + limit_in, b_limit_in, thresh_in, hev_out, mask_out, \ + flat_out) \ + { \ + v16u8 p3_asub_p2_m, p2_asub_p1_m, p1_asub_p0_m, q1_asub_q0_m; \ + v16u8 p1_asub_q1_m, p0_asub_q0_m, q3_asub_q2_m, q2_asub_q1_m; \ + \ + /* absolute subtraction of pixel values */ \ + p3_asub_p2_m = __msa_asub_u_b(p3_in, p2_in); \ + p2_asub_p1_m = __msa_asub_u_b(p2_in, p1_in); \ + p1_asub_p0_m = __msa_asub_u_b(p1_in, p0_in); \ + q1_asub_q0_m = __msa_asub_u_b(q1_in, q0_in); \ + q2_asub_q1_m = __msa_asub_u_b(q2_in, q1_in); \ + q3_asub_q2_m = __msa_asub_u_b(q3_in, q2_in); \ + p0_asub_q0_m = __msa_asub_u_b(p0_in, q0_in); \ + p1_asub_q1_m = __msa_asub_u_b(p1_in, q1_in); \ + \ + /* calculation of hev */ \ + flat_out = __msa_max_u_b(p1_asub_p0_m, q1_asub_q0_m); \ + hev_out = thresh_in < (v16u8)flat_out; \ + \ + /* calculation of mask */ \ + p0_asub_q0_m = __msa_adds_u_b(p0_asub_q0_m, p0_asub_q0_m); \ + p1_asub_q1_m >>= 1; \ + p0_asub_q0_m = __msa_adds_u_b(p0_asub_q0_m, p1_asub_q1_m); \ + \ + mask_out = b_limit_in < p0_asub_q0_m; \ + mask_out = __msa_max_u_b(flat_out, mask_out); \ + p3_asub_p2_m = __msa_max_u_b(p3_asub_p2_m, p2_asub_p1_m); \ + mask_out = __msa_max_u_b(p3_asub_p2_m, mask_out); \ + q2_asub_q1_m = __msa_max_u_b(q2_asub_q1_m, q3_asub_q2_m); \ + mask_out = __msa_max_u_b(q2_asub_q1_m, mask_out); \ + \ + mask_out = limit_in < (v16u8)mask_out; \ + mask_out = __msa_xori_b(mask_out, 0xff); \ + } +#endif /* AOM_DSP_LOOPFILTER_MSA_H_ */ diff --git a/third_party/aom/aom_dsp/mips/macros_msa.h b/third_party/aom/aom_dsp/mips/macros_msa.h new file mode 100644 index 0000000000..48fbcfd47f --- /dev/null +++ b/third_party/aom/aom_dsp/mips/macros_msa.h @@ -0,0 +1,2057 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_MIPS_MACROS_MSA_H_ +#define AOM_DSP_MIPS_MACROS_MSA_H_ + +#include + +#include "./aom_config.h" +#include "aom/aom_integer.h" + +#define LD_B(RTYPE, psrc) *((const RTYPE *)(psrc)) +#define LD_UB(...) LD_B(v16u8, __VA_ARGS__) +#define LD_SB(...) LD_B(v16i8, __VA_ARGS__) + +#define LD_H(RTYPE, psrc) *((const RTYPE *)(psrc)) +#define LD_UH(...) LD_H(v8u16, __VA_ARGS__) +#define LD_SH(...) LD_H(v8i16, __VA_ARGS__) + +#define LD_W(RTYPE, psrc) *((const RTYPE *)(psrc)) +#define LD_SW(...) LD_W(v4i32, __VA_ARGS__) + +#define ST_B(RTYPE, in, pdst) *((RTYPE *)(pdst)) = (in) +#define ST_UB(...) ST_B(v16u8, __VA_ARGS__) +#define ST_SB(...) ST_B(v16i8, __VA_ARGS__) + +#define ST_H(RTYPE, in, pdst) *((RTYPE *)(pdst)) = (in) +#define ST_SH(...) ST_H(v8i16, __VA_ARGS__) + +#define ST_W(RTYPE, in, pdst) *((RTYPE *)(pdst)) = (in) +#define ST_SW(...) ST_W(v4i32, __VA_ARGS__) + +#if (__mips_isa_rev >= 6) +#define LH(psrc) \ + ({ \ + const uint8_t *psrc_m = (const uint8_t *)(psrc); \ + uint16_t val_m; \ + \ + __asm__ __volatile__("lh %[val_m], %[psrc_m] \n\t" \ + \ + : [val_m] "=r"(val_m) \ + : [psrc_m] "m"(*psrc_m)); \ + \ + val_m; \ + }) + +#define LW(psrc) \ + ({ \ + const uint8_t *psrc_m = (const uint8_t *)(psrc); \ + uint32_t val_m; \ + \ + __asm__ __volatile__("lw %[val_m], %[psrc_m] \n\t" \ + \ + : [val_m] "=r"(val_m) \ + : [psrc_m] "m"(*psrc_m)); \ + \ + val_m; \ + }) + +#if (__mips == 64) +#define LD(psrc) \ + ({ \ + const uint8_t *psrc_m = (const uint8_t *)(psrc); \ + uint64_t val_m = 0; \ + \ + __asm__ __volatile__("ld %[val_m], %[psrc_m] \n\t" \ + \ + : [val_m] "=r"(val_m) \ + : [psrc_m] "m"(*psrc_m)); \ + \ + val_m; \ + }) +#else // !(__mips == 64) +#define LD(psrc) \ + ({ \ + const uint8_t *psrc_m = (const uint8_t *)(psrc); \ + uint32_t val0_m, val1_m; \ + uint64_t val_m = 0; \ + \ + val0_m = LW(psrc_m); \ + val1_m = LW(psrc_m + 4); \ + \ + val_m = (uint64_t)(val1_m); \ + val_m = (uint64_t)((val_m << 32) & 0xFFFFFFFF00000000); \ + val_m = (uint64_t)(val_m | (uint64_t)val0_m); \ + \ + val_m; \ + }) +#endif // (__mips == 64) + +#define SH(val, pdst) \ + { \ + uint8_t *pdst_m = (uint8_t *)(pdst); \ + const uint16_t val_m = (val); \ + \ + __asm__ __volatile__("sh %[val_m], %[pdst_m] \n\t" \ + \ + : [pdst_m] "=m"(*pdst_m) \ + : [val_m] "r"(val_m)); \ + } + +#define SW(val, pdst) \ + { \ + uint8_t *pdst_m = (uint8_t *)(pdst); \ + const uint32_t val_m = (val); \ + \ + __asm__ __volatile__("sw %[val_m], %[pdst_m] \n\t" \ + \ + : [pdst_m] "=m"(*pdst_m) \ + : [val_m] "r"(val_m)); \ + } + +#define SD(val, pdst) \ + { \ + uint8_t *pdst_m = (uint8_t *)(pdst); \ + const uint64_t val_m = (val); \ + \ + __asm__ __volatile__("sd %[val_m], %[pdst_m] \n\t" \ + \ + : [pdst_m] "=m"(*pdst_m) \ + : [val_m] "r"(val_m)); \ + } +#else // !(__mips_isa_rev >= 6) +#define LH(psrc) \ + ({ \ + const uint8_t *psrc_m = (const uint8_t *)(psrc); \ + uint16_t val_m; \ + \ + __asm__ __volatile__("ulh %[val_m], %[psrc_m] \n\t" \ + \ + : [val_m] "=r"(val_m) \ + : [psrc_m] "m"(*psrc_m)); \ + \ + val_m; \ + }) + +#define LW(psrc) \ + ({ \ + const uint8_t *psrc_m = (const uint8_t *)(psrc); \ + uint32_t val_m; \ + \ + __asm__ __volatile__("ulw %[val_m], %[psrc_m] \n\t" \ + \ + : [val_m] "=r"(val_m) \ + : [psrc_m] "m"(*psrc_m)); \ + \ + val_m; \ + }) + +#if (__mips == 64) +#define LD(psrc) \ + ({ \ + const uint8_t *psrc_m = (const uint8_t *)(psrc); \ + uint64_t val_m = 0; \ + \ + __asm__ __volatile__("uld %[val_m], %[psrc_m] \n\t" \ + \ + : [val_m] "=r"(val_m) \ + : [psrc_m] "m"(*psrc_m)); \ + \ + val_m; \ + }) +#else // !(__mips == 64) +#define LD(psrc) \ + ({ \ + const uint8_t *psrc_m1 = (const uint8_t *)(psrc); \ + uint32_t val0_m, val1_m; \ + uint64_t val_m_combined = 0; \ + \ + val0_m = LW(psrc_m1); \ + val1_m = LW(psrc_m1 + 4); \ + \ + val_m_combined = (uint64_t)(val1_m); \ + val_m_combined = (uint64_t)((val_m_combined << 32) & 0xFFFFFFFF00000000); \ + val_m_combined = (uint64_t)(val_m_combined | (uint64_t)val0_m); \ + \ + val_m_combined; \ + }) +#endif // (__mips == 64) + +#define SH(val, pdst) \ + { \ + uint8_t *pdst_m = (uint8_t *)(pdst); \ + const uint16_t val_m = (val); \ + \ + __asm__ __volatile__("ush %[val_m], %[pdst_m] \n\t" \ + \ + : [pdst_m] "=m"(*pdst_m) \ + : [val_m] "r"(val_m)); \ + } + +#define SW(val, pdst) \ + { \ + uint8_t *pdst_m = (uint8_t *)(pdst); \ + const uint32_t val_m = (val); \ + \ + __asm__ __volatile__("usw %[val_m], %[pdst_m] \n\t" \ + \ + : [pdst_m] "=m"(*pdst_m) \ + : [val_m] "r"(val_m)); \ + } + +#define SD(val, pdst) \ + { \ + uint8_t *pdst_m1 = (uint8_t *)(pdst); \ + uint32_t val0_m, val1_m; \ + \ + val0_m = (uint32_t)((val)&0x00000000FFFFFFFF); \ + val1_m = (uint32_t)(((val) >> 32) & 0x00000000FFFFFFFF); \ + \ + SW(val0_m, pdst_m1); \ + SW(val1_m, pdst_m1 + 4); \ + } +#endif // (__mips_isa_rev >= 6) + +/* Description : Load 4 words with stride + Arguments : Inputs - psrc, stride + Outputs - out0, out1, out2, out3 + Details : Load word in 'out0' from (psrc) + Load word in 'out1' from (psrc + stride) + Load word in 'out2' from (psrc + 2 * stride) + Load word in 'out3' from (psrc + 3 * stride) +*/ +#define LW4(psrc, stride, out0, out1, out2, out3) \ + { \ + out0 = LW((psrc)); \ + out1 = LW((psrc) + stride); \ + out2 = LW((psrc) + 2 * stride); \ + out3 = LW((psrc) + 3 * stride); \ + } + +/* Description : Load double words with stride + Arguments : Inputs - psrc, stride + Outputs - out0, out1 + Details : Load double word in 'out0' from (psrc) + Load double word in 'out1' from (psrc + stride) +*/ +#define LD2(psrc, stride, out0, out1) \ + { \ + out0 = LD((psrc)); \ + out1 = LD((psrc) + stride); \ + } +#define LD4(psrc, stride, out0, out1, out2, out3) \ + { \ + LD2((psrc), stride, out0, out1); \ + LD2((psrc) + 2 * stride, stride, out2, out3); \ + } + +/* Description : Store 4 words with stride + Arguments : Inputs - in0, in1, in2, in3, pdst, stride + Details : Store word from 'in0' to (pdst) + Store word from 'in1' to (pdst + stride) + Store word from 'in2' to (pdst + 2 * stride) + Store word from 'in3' to (pdst + 3 * stride) +*/ +#define SW4(in0, in1, in2, in3, pdst, stride) \ + { \ + SW(in0, (pdst)) \ + SW(in1, (pdst) + stride); \ + SW(in2, (pdst) + 2 * stride); \ + SW(in3, (pdst) + 3 * stride); \ + } + +/* Description : Store 4 double words with stride + Arguments : Inputs - in0, in1, in2, in3, pdst, stride + Details : Store double word from 'in0' to (pdst) + Store double word from 'in1' to (pdst + stride) + Store double word from 'in2' to (pdst + 2 * stride) + Store double word from 'in3' to (pdst + 3 * stride) +*/ +#define SD4(in0, in1, in2, in3, pdst, stride) \ + { \ + SD(in0, (pdst)) \ + SD(in1, (pdst) + stride); \ + SD(in2, (pdst) + 2 * stride); \ + SD(in3, (pdst) + 3 * stride); \ + } + +/* Description : Load vectors with 16 byte elements with stride + Arguments : Inputs - psrc, stride + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Load 16 byte elements in 'out0' from (psrc) + Load 16 byte elements in 'out1' from (psrc + stride) +*/ +#define LD_B2(RTYPE, psrc, stride, out0, out1) \ + { \ + out0 = LD_B(RTYPE, (psrc)); \ + out1 = LD_B(RTYPE, (psrc) + stride); \ + } +#define LD_UB2(...) LD_B2(v16u8, __VA_ARGS__) +#define LD_SB2(...) LD_B2(v16i8, __VA_ARGS__) + +#define LD_B3(RTYPE, psrc, stride, out0, out1, out2) \ + { \ + LD_B2(RTYPE, (psrc), stride, out0, out1); \ + out2 = LD_B(RTYPE, (psrc) + 2 * stride); \ + } +#define LD_UB3(...) LD_B3(v16u8, __VA_ARGS__) + +#define LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3) \ + { \ + LD_B2(RTYPE, (psrc), stride, out0, out1); \ + LD_B2(RTYPE, (psrc) + 2 * stride, stride, out2, out3); \ + } +#define LD_UB4(...) LD_B4(v16u8, __VA_ARGS__) +#define LD_SB4(...) LD_B4(v16i8, __VA_ARGS__) + +#define LD_B5(RTYPE, psrc, stride, out0, out1, out2, out3, out4) \ + { \ + LD_B4(RTYPE, (psrc), stride, out0, out1, out2, out3); \ + out4 = LD_B(RTYPE, (psrc) + 4 * stride); \ + } +#define LD_UB5(...) LD_B5(v16u8, __VA_ARGS__) +#define LD_SB5(...) LD_B5(v16i8, __VA_ARGS__) + +#define LD_B7(RTYPE, psrc, stride, out0, out1, out2, out3, out4, out5, out6) \ + { \ + LD_B5(RTYPE, (psrc), stride, out0, out1, out2, out3, out4); \ + LD_B2(RTYPE, (psrc) + 5 * stride, stride, out5, out6); \ + } +#define LD_SB7(...) LD_B7(v16i8, __VA_ARGS__) + +#define LD_B8(RTYPE, psrc, stride, out0, out1, out2, out3, out4, out5, out6, \ + out7) \ + { \ + LD_B4(RTYPE, (psrc), stride, out0, out1, out2, out3); \ + LD_B4(RTYPE, (psrc) + 4 * stride, stride, out4, out5, out6, out7); \ + } +#define LD_UB8(...) LD_B8(v16u8, __VA_ARGS__) +#define LD_SB8(...) LD_B8(v16i8, __VA_ARGS__) + +/* Description : Load vectors with 8 halfword elements with stride + Arguments : Inputs - psrc, stride + Outputs - out0, out1 + Details : Load 8 halfword elements in 'out0' from (psrc) + Load 8 halfword elements in 'out1' from (psrc + stride) +*/ +#define LD_H2(RTYPE, psrc, stride, out0, out1) \ + { \ + out0 = LD_H(RTYPE, (psrc)); \ + out1 = LD_H(RTYPE, (psrc) + (stride)); \ + } +#define LD_SH2(...) LD_H2(v8i16, __VA_ARGS__) + +#define LD_H4(RTYPE, psrc, stride, out0, out1, out2, out3) \ + { \ + LD_H2(RTYPE, (psrc), stride, out0, out1); \ + LD_H2(RTYPE, (psrc) + 2 * stride, stride, out2, out3); \ + } +#define LD_SH4(...) LD_H4(v8i16, __VA_ARGS__) + +#define LD_H8(RTYPE, psrc, stride, out0, out1, out2, out3, out4, out5, out6, \ + out7) \ + { \ + LD_H4(RTYPE, (psrc), stride, out0, out1, out2, out3); \ + LD_H4(RTYPE, (psrc) + 4 * stride, stride, out4, out5, out6, out7); \ + } +#define LD_SH8(...) LD_H8(v8i16, __VA_ARGS__) + +#define LD_H16(RTYPE, psrc, stride, out0, out1, out2, out3, out4, out5, out6, \ + out7, out8, out9, out10, out11, out12, out13, out14, out15) \ + { \ + LD_H8(RTYPE, (psrc), stride, out0, out1, out2, out3, out4, out5, out6, \ + out7); \ + LD_H8(RTYPE, (psrc) + 8 * stride, stride, out8, out9, out10, out11, out12, \ + out13, out14, out15); \ + } +#define LD_SH16(...) LD_H16(v8i16, __VA_ARGS__) + +/* Description : Load 4x4 block of signed halfword elements from 1D source + data into 4 vectors (Each vector with 4 signed halfwords) + Arguments : Input - psrc + Outputs - out0, out1, out2, out3 +*/ +#define LD4x4_SH(psrc, out0, out1, out2, out3) \ + { \ + out0 = LD_SH(psrc); \ + out2 = LD_SH(psrc + 8); \ + out1 = (v8i16)__msa_ilvl_d((v2i64)out0, (v2i64)out0); \ + out3 = (v8i16)__msa_ilvl_d((v2i64)out2, (v2i64)out2); \ + } + +/* Description : Load 2 vectors of signed word elements with stride + Arguments : Inputs - psrc, stride + Outputs - out0, out1 + Return Type - signed word +*/ +#define LD_SW2(psrc, stride, out0, out1) \ + { \ + out0 = LD_SW((psrc)); \ + out1 = LD_SW((psrc) + stride); \ + } + +/* Description : Store vectors of 16 byte elements with stride + Arguments : Inputs - in0, in1, pdst, stride + Details : Store 16 byte elements from 'in0' to (pdst) + Store 16 byte elements from 'in1' to (pdst + stride) +*/ +#define ST_B2(RTYPE, in0, in1, pdst, stride) \ + { \ + ST_B(RTYPE, in0, (pdst)); \ + ST_B(RTYPE, in1, (pdst) + stride); \ + } +#define ST_UB2(...) ST_B2(v16u8, __VA_ARGS__) + +#define ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride) \ + { \ + ST_B2(RTYPE, in0, in1, (pdst), stride); \ + ST_B2(RTYPE, in2, in3, (pdst) + 2 * stride, stride); \ + } +#define ST_UB4(...) ST_B4(v16u8, __VA_ARGS__) + +#define ST_B8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ + { \ + ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride); \ + ST_B4(RTYPE, in4, in5, in6, in7, (pdst) + 4 * stride, stride); \ + } +#define ST_UB8(...) ST_B8(v16u8, __VA_ARGS__) + +/* Description : Store vectors of 8 halfword elements with stride + Arguments : Inputs - in0, in1, pdst, stride + Details : Store 8 halfword elements from 'in0' to (pdst) + Store 8 halfword elements from 'in1' to (pdst + stride) +*/ +#define ST_H2(RTYPE, in0, in1, pdst, stride) \ + { \ + ST_H(RTYPE, in0, (pdst)); \ + ST_H(RTYPE, in1, (pdst) + stride); \ + } +#define ST_SH2(...) ST_H2(v8i16, __VA_ARGS__) + +#define ST_H4(RTYPE, in0, in1, in2, in3, pdst, stride) \ + { \ + ST_H2(RTYPE, in0, in1, (pdst), stride); \ + ST_H2(RTYPE, in2, in3, (pdst) + 2 * stride, stride); \ + } +#define ST_SH4(...) ST_H4(v8i16, __VA_ARGS__) + +#define ST_H8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ + { \ + ST_H4(RTYPE, in0, in1, in2, in3, (pdst), stride); \ + ST_H4(RTYPE, in4, in5, in6, in7, (pdst) + 4 * stride, stride); \ + } +#define ST_SH8(...) ST_H8(v8i16, __VA_ARGS__) + +/* Description : Store vectors of word elements with stride + Arguments : Inputs - in0, in1, pdst, stride + Details : Store 4 word elements from 'in0' to (pdst) + Store 4 word elements from 'in1' to (pdst + stride) +*/ +#define ST_SW2(in0, in1, pdst, stride) \ + { \ + ST_SW(in0, (pdst)); \ + ST_SW(in1, (pdst) + stride); \ + } + +/* Description : Store 2x4 byte block to destination memory from input vector + Arguments : Inputs - in, stidx, pdst, stride + Details : Index 'stidx' halfword element from 'in' vector is copied to + the GP register and stored to (pdst) + Index 'stidx+1' halfword element from 'in' vector is copied to + the GP register and stored to (pdst + stride) + Index 'stidx+2' halfword element from 'in' vector is copied to + the GP register and stored to (pdst + 2 * stride) + Index 'stidx+3' halfword element from 'in' vector is copied to + the GP register and stored to (pdst + 3 * stride) +*/ +#define ST2x4_UB(in, stidx, pdst, stride) \ + { \ + uint16_t out0_m, out1_m, out2_m, out3_m; \ + uint8_t *pblk_2x4_m = (uint8_t *)(pdst); \ + \ + out0_m = __msa_copy_u_h((v8i16)in, (stidx)); \ + out1_m = __msa_copy_u_h((v8i16)in, (stidx + 1)); \ + out2_m = __msa_copy_u_h((v8i16)in, (stidx + 2)); \ + out3_m = __msa_copy_u_h((v8i16)in, (stidx + 3)); \ + \ + SH(out0_m, pblk_2x4_m); \ + SH(out1_m, pblk_2x4_m + stride); \ + SH(out2_m, pblk_2x4_m + 2 * stride); \ + SH(out3_m, pblk_2x4_m + 3 * stride); \ + } + +/* Description : Store 4x2 byte block to destination memory from input vector + Arguments : Inputs - in, pdst, stride + Details : Index 0 word element from 'in' vector is copied to the GP + register and stored to (pdst) + Index 1 word element from 'in' vector is copied to the GP + register and stored to (pdst + stride) +*/ +#define ST4x2_UB(in, pdst, stride) \ + { \ + uint32_t out0_m, out1_m; \ + uint8_t *pblk_4x2_m = (uint8_t *)(pdst); \ + \ + out0_m = __msa_copy_u_w((v4i32)in, 0); \ + out1_m = __msa_copy_u_w((v4i32)in, 1); \ + \ + SW(out0_m, pblk_4x2_m); \ + SW(out1_m, pblk_4x2_m + stride); \ + } + +/* Description : Store 4x4 byte block to destination memory from input vector + Arguments : Inputs - in0, in1, pdst, stride + Details : 'Idx0' word element from input vector 'in0' is copied to the + GP register and stored to (pdst) + 'Idx1' word element from input vector 'in0' is copied to the + GP register and stored to (pdst + stride) + 'Idx2' word element from input vector 'in0' is copied to the + GP register and stored to (pdst + 2 * stride) + 'Idx3' word element from input vector 'in0' is copied to the + GP register and stored to (pdst + 3 * stride) +*/ +#define ST4x4_UB(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) \ + { \ + uint32_t out0_m, out1_m, out2_m, out3_m; \ + uint8_t *pblk_4x4_m = (uint8_t *)(pdst); \ + \ + out0_m = __msa_copy_u_w((v4i32)in0, idx0); \ + out1_m = __msa_copy_u_w((v4i32)in0, idx1); \ + out2_m = __msa_copy_u_w((v4i32)in1, idx2); \ + out3_m = __msa_copy_u_w((v4i32)in1, idx3); \ + \ + SW4(out0_m, out1_m, out2_m, out3_m, pblk_4x4_m, stride); \ + } +#define ST4x8_UB(in0, in1, pdst, stride) \ + { \ + uint8_t *pblk_4x8 = (uint8_t *)(pdst); \ + \ + ST4x4_UB(in0, in0, 0, 1, 2, 3, pblk_4x8, stride); \ + ST4x4_UB(in1, in1, 0, 1, 2, 3, pblk_4x8 + 4 * stride, stride); \ + } + +/* Description : Store 8x1 byte block to destination memory from input vector + Arguments : Inputs - in, pdst + Details : Index 0 double word element from 'in' vector is copied to the + GP register and stored to (pdst) +*/ +#define ST8x1_UB(in, pdst) \ + { \ + uint64_t out0_m; \ + \ + out0_m = __msa_copy_u_d((v2i64)in, 0); \ + SD(out0_m, pdst); \ + } + +/* Description : Store 8x2 byte block to destination memory from input vector + Arguments : Inputs - in, pdst, stride + Details : Index 0 double word element from 'in' vector is copied to the + GP register and stored to (pdst) + Index 1 double word element from 'in' vector is copied to the + GP register and stored to (pdst + stride) +*/ +#define ST8x2_UB(in, pdst, stride) \ + { \ + uint64_t out0_m, out1_m; \ + uint8_t *pblk_8x2_m = (uint8_t *)(pdst); \ + \ + out0_m = __msa_copy_u_d((v2i64)in, 0); \ + out1_m = __msa_copy_u_d((v2i64)in, 1); \ + \ + SD(out0_m, pblk_8x2_m); \ + SD(out1_m, pblk_8x2_m + stride); \ + } + +/* Description : Store 8x4 byte block to destination memory from input + vectors + Arguments : Inputs - in0, in1, pdst, stride + Details : Index 0 double word element from 'in0' vector is copied to the + GP register and stored to (pdst) + Index 1 double word element from 'in0' vector is copied to the + GP register and stored to (pdst + stride) + Index 0 double word element from 'in1' vector is copied to the + GP register and stored to (pdst + 2 * stride) + Index 1 double word element from 'in1' vector is copied to the + GP register and stored to (pdst + 3 * stride) +*/ +#define ST8x4_UB(in0, in1, pdst, stride) \ + { \ + uint64_t out0_m, out1_m, out2_m, out3_m; \ + uint8_t *pblk_8x4_m = (uint8_t *)(pdst); \ + \ + out0_m = __msa_copy_u_d((v2i64)in0, 0); \ + out1_m = __msa_copy_u_d((v2i64)in0, 1); \ + out2_m = __msa_copy_u_d((v2i64)in1, 0); \ + out3_m = __msa_copy_u_d((v2i64)in1, 1); \ + \ + SD4(out0_m, out1_m, out2_m, out3_m, pblk_8x4_m, stride); \ + } + +/* Description : average with rounding (in0 + in1 + 1) / 2. + Arguments : Inputs - in0, in1, in2, in3, + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Each unsigned byte element from 'in0' vector is added with + each unsigned byte element from 'in1' vector. Then the average + with rounding is calculated and written to 'out0' +*/ +#define AVER_UB2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_aver_u_b((v16u8)in0, (v16u8)in1); \ + out1 = (RTYPE)__msa_aver_u_b((v16u8)in2, (v16u8)in3); \ + } +#define AVER_UB2_UB(...) AVER_UB2(v16u8, __VA_ARGS__) + +#define AVER_UB4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3) \ + { \ + AVER_UB2(RTYPE, in0, in1, in2, in3, out0, out1) \ + AVER_UB2(RTYPE, in4, in5, in6, in7, out2, out3) \ + } +#define AVER_UB4_UB(...) AVER_UB4(v16u8, __VA_ARGS__) + +/* Description : Immediate number of elements to slide with zero + Arguments : Inputs - in0, in1, slide_val + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Byte elements from 'zero_m' vector are slid into 'in0' by + value specified in the 'slide_val' +*/ +#define SLDI_B2_0(RTYPE, in0, in1, out0, out1, slide_val) \ + { \ + v16i8 zero_m = { 0 }; \ + out0 = (RTYPE)__msa_sldi_b((v16i8)zero_m, (v16i8)in0, slide_val); \ + out1 = (RTYPE)__msa_sldi_b((v16i8)zero_m, (v16i8)in1, slide_val); \ + } +#define SLDI_B2_0_SW(...) SLDI_B2_0(v4i32, __VA_ARGS__) + +#define SLDI_B4_0(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3, \ + slide_val) \ + { \ + SLDI_B2_0(RTYPE, in0, in1, out0, out1, slide_val); \ + SLDI_B2_0(RTYPE, in2, in3, out2, out3, slide_val); \ + } +#define SLDI_B4_0_UB(...) SLDI_B4_0(v16u8, __VA_ARGS__) + +/* Description : Immediate number of elements to slide + Arguments : Inputs - in0_0, in0_1, in1_0, in1_1, slide_val + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Byte elements from 'in0_0' vector are slid into 'in1_0' by + value specified in the 'slide_val' +*/ +#define SLDI_B2(RTYPE, in0_0, in0_1, in1_0, in1_1, out0, out1, slide_val) \ + { \ + out0 = (RTYPE)__msa_sldi_b((v16i8)in0_0, (v16i8)in1_0, slide_val); \ + out1 = (RTYPE)__msa_sldi_b((v16i8)in0_1, (v16i8)in1_1, slide_val); \ + } +#define SLDI_B2_UB(...) SLDI_B2(v16u8, __VA_ARGS__) +#define SLDI_B2_SH(...) SLDI_B2(v8i16, __VA_ARGS__) + +#define SLDI_B3(RTYPE, in0_0, in0_1, in0_2, in1_0, in1_1, in1_2, out0, out1, \ + out2, slide_val) \ + { \ + SLDI_B2(RTYPE, in0_0, in0_1, in1_0, in1_1, out0, out1, slide_val) \ + out2 = (RTYPE)__msa_sldi_b((v16i8)in0_2, (v16i8)in1_2, slide_val); \ + } +#define SLDI_B3_SB(...) SLDI_B3(v16i8, __VA_ARGS__) +#define SLDI_B3_UH(...) SLDI_B3(v8u16, __VA_ARGS__) + +/* Description : Shuffle byte vector elements as per mask vector + Arguments : Inputs - in0, in1, in2, in3, mask0, mask1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Byte elements from 'in0' & 'in1' are copied selectively to + 'out0' as per control vector 'mask0' +*/ +#define VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_vshf_b((v16i8)mask0, (v16i8)in1, (v16i8)in0); \ + out1 = (RTYPE)__msa_vshf_b((v16i8)mask1, (v16i8)in3, (v16i8)in2); \ + } +#define VSHF_B2_UB(...) VSHF_B2(v16u8, __VA_ARGS__) +#define VSHF_B2_SB(...) VSHF_B2(v16i8, __VA_ARGS__) +#define VSHF_B2_UH(...) VSHF_B2(v8u16, __VA_ARGS__) + +#define VSHF_B4(RTYPE, in0, in1, mask0, mask1, mask2, mask3, out0, out1, out2, \ + out3) \ + { \ + VSHF_B2(RTYPE, in0, in1, in0, in1, mask0, mask1, out0, out1); \ + VSHF_B2(RTYPE, in0, in1, in0, in1, mask2, mask3, out2, out3); \ + } +#define VSHF_B4_SB(...) VSHF_B4(v16i8, __VA_ARGS__) +#define VSHF_B4_SH(...) VSHF_B4(v8i16, __VA_ARGS__) + +/* Description : Dot product of byte vector elements + Arguments : Inputs - mult0, mult1, cnst0, cnst1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Unsigned byte elements from 'mult0' are multiplied with + unsigned byte elements from 'cnst0' producing a result + twice the size of input i.e. unsigned halfword. + The multiplication result of adjacent odd-even elements + are added together and written to the 'out0' vector +*/ +#define DOTP_UB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_dotp_u_h((v16u8)mult0, (v16u8)cnst0); \ + out1 = (RTYPE)__msa_dotp_u_h((v16u8)mult1, (v16u8)cnst1); \ + } +#define DOTP_UB2_UH(...) DOTP_UB2(v8u16, __VA_ARGS__) + +#define DOTP_UB4(RTYPE, mult0, mult1, mult2, mult3, cnst0, cnst1, cnst2, \ + cnst3, out0, out1, out2, out3) \ + { \ + DOTP_UB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ + DOTP_UB2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ + } +#define DOTP_UB4_UH(...) DOTP_UB4(v8u16, __VA_ARGS__) + +/* Description : Dot product of byte vector elements + Arguments : Inputs - mult0, mult1, cnst0, cnst1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Signed byte elements from 'mult0' are multiplied with + signed byte elements from 'cnst0' producing a result + twice the size of input i.e. signed halfword. + The multiplication result of adjacent odd-even elements + are added together and written to the 'out0' vector +*/ +#define DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_dotp_s_h((v16i8)mult0, (v16i8)cnst0); \ + out1 = (RTYPE)__msa_dotp_s_h((v16i8)mult1, (v16i8)cnst1); \ + } +#define DOTP_SB2_SH(...) DOTP_SB2(v8i16, __VA_ARGS__) + +#define DOTP_SB4(RTYPE, mult0, mult1, mult2, mult3, cnst0, cnst1, cnst2, \ + cnst3, out0, out1, out2, out3) \ + { \ + DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ + DOTP_SB2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ + } +#define DOTP_SB4_SH(...) DOTP_SB4(v8i16, __VA_ARGS__) + +/* Description : Dot product of halfword vector elements + Arguments : Inputs - mult0, mult1, cnst0, cnst1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Signed halfword elements from 'mult0' are multiplied with + signed halfword elements from 'cnst0' producing a result + twice the size of input i.e. signed word. + The multiplication result of adjacent odd-even elements + are added together and written to the 'out0' vector +*/ +#define DOTP_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_dotp_s_w((v8i16)mult0, (v8i16)cnst0); \ + out1 = (RTYPE)__msa_dotp_s_w((v8i16)mult1, (v8i16)cnst1); \ + } +#define DOTP_SH2_SW(...) DOTP_SH2(v4i32, __VA_ARGS__) + +#define DOTP_SH4(RTYPE, mult0, mult1, mult2, mult3, cnst0, cnst1, cnst2, \ + cnst3, out0, out1, out2, out3) \ + { \ + DOTP_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ + DOTP_SH2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ + } +#define DOTP_SH4_SW(...) DOTP_SH4(v4i32, __VA_ARGS__) + +/* Description : Dot product of word vector elements + Arguments : Inputs - mult0, mult1, cnst0, cnst1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Signed word elements from 'mult0' are multiplied with + signed word elements from 'cnst0' producing a result + twice the size of input i.e. signed double word. + The multiplication result of adjacent odd-even elements + are added together and written to the 'out0' vector +*/ +#define DOTP_SW2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_dotp_s_d((v4i32)mult0, (v4i32)cnst0); \ + out1 = (RTYPE)__msa_dotp_s_d((v4i32)mult1, (v4i32)cnst1); \ + } +#define DOTP_SW2_SD(...) DOTP_SW2(v2i64, __VA_ARGS__) + +/* Description : Dot product & addition of byte vector elements + Arguments : Inputs - mult0, mult1, cnst0, cnst1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Signed byte elements from 'mult0' are multiplied with + signed byte elements from 'cnst0' producing a result + twice the size of input i.e. signed halfword. + The multiplication result of adjacent odd-even elements + are added to the 'out0' vector +*/ +#define DPADD_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_dpadd_s_h((v8i16)out0, (v16i8)mult0, (v16i8)cnst0); \ + out1 = (RTYPE)__msa_dpadd_s_h((v8i16)out1, (v16i8)mult1, (v16i8)cnst1); \ + } +#define DPADD_SB2_SH(...) DPADD_SB2(v8i16, __VA_ARGS__) + +#define DPADD_SB4(RTYPE, mult0, mult1, mult2, mult3, cnst0, cnst1, cnst2, \ + cnst3, out0, out1, out2, out3) \ + { \ + DPADD_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ + DPADD_SB2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ + } +#define DPADD_SB4_SH(...) DPADD_SB4(v8i16, __VA_ARGS__) + +/* Description : Dot product & addition of halfword vector elements + Arguments : Inputs - mult0, mult1, cnst0, cnst1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Signed halfword elements from 'mult0' are multiplied with + signed halfword elements from 'cnst0' producing a result + twice the size of input i.e. signed word. + The multiplication result of adjacent odd-even elements + are added to the 'out0' vector +*/ +#define DPADD_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_dpadd_s_w((v4i32)out0, (v8i16)mult0, (v8i16)cnst0); \ + out1 = (RTYPE)__msa_dpadd_s_w((v4i32)out1, (v8i16)mult1, (v8i16)cnst1); \ + } +#define DPADD_SH2_SW(...) DPADD_SH2(v4i32, __VA_ARGS__) + +/* Description : Dot product & addition of double word vector elements + Arguments : Inputs - mult0, mult1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Each signed word element from 'mult0' is multiplied with itself + producing an intermediate result twice the size of input + i.e. signed double word + The multiplication result of adjacent odd-even elements + are added to the 'out0' vector +*/ +#define DPADD_SD2(RTYPE, mult0, mult1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_dpadd_s_d((v2i64)out0, (v4i32)mult0, (v4i32)mult0); \ + out1 = (RTYPE)__msa_dpadd_s_d((v2i64)out1, (v4i32)mult1, (v4i32)mult1); \ + } +#define DPADD_SD2_SD(...) DPADD_SD2(v2i64, __VA_ARGS__) + +/* Description : Minimum values between unsigned elements of + either vector are copied to the output vector + Arguments : Inputs - in0, in1, min_vec + Outputs - in place operation + Return Type - as per RTYPE + Details : Minimum of unsigned halfword element values from 'in0' and + 'min_vec' are written to output vector 'in0' +*/ +#define MIN_UH2(RTYPE, in0, in1, min_vec) \ + { \ + in0 = (RTYPE)__msa_min_u_h((v8u16)in0, min_vec); \ + in1 = (RTYPE)__msa_min_u_h((v8u16)in1, min_vec); \ + } +#define MIN_UH2_UH(...) MIN_UH2(v8u16, __VA_ARGS__) + +#define MIN_UH4(RTYPE, in0, in1, in2, in3, min_vec) \ + { \ + MIN_UH2(RTYPE, in0, in1, min_vec); \ + MIN_UH2(RTYPE, in2, in3, min_vec); \ + } +#define MIN_UH4_UH(...) MIN_UH4(v8u16, __VA_ARGS__) + +/* Description : Clips all signed halfword elements of input vector + between 0 & 255 + Arguments : Input - in + Output - out_m + Return Type - signed halfword +*/ +#define CLIP_SH_0_255(in) \ + ({ \ + v8i16 max_m = __msa_ldi_h(255); \ + v8i16 out_m; \ + \ + out_m = __msa_maxi_s_h((v8i16)in, 0); \ + out_m = __msa_min_s_h((v8i16)max_m, (v8i16)out_m); \ + out_m; \ + }) +#define CLIP_SH2_0_255(in0, in1) \ + { \ + in0 = CLIP_SH_0_255(in0); \ + in1 = CLIP_SH_0_255(in1); \ + } +#define CLIP_SH4_0_255(in0, in1, in2, in3) \ + { \ + CLIP_SH2_0_255(in0, in1); \ + CLIP_SH2_0_255(in2, in3); \ + } + +/* Description : Horizontal addition of 4 signed word elements of input vector + Arguments : Input - in (signed word vector) + Output - sum_m (i32 sum) + Return Type - signed word (GP) + Details : 4 signed word elements of 'in' vector are added together and + the resulting integer sum is returned +*/ +#define HADD_SW_S32(in) \ + ({ \ + v2i64 res0_m, res1_m; \ + int32_t sum_m; \ + \ + res0_m = __msa_hadd_s_d((v4i32)in, (v4i32)in); \ + res1_m = __msa_splati_d(res0_m, 1); \ + res0_m = res0_m + res1_m; \ + sum_m = __msa_copy_s_w((v4i32)res0_m, 0); \ + sum_m; \ + }) + +/* Description : Horizontal addition of 8 unsigned halfword elements + Arguments : Inputs - in (unsigned halfword vector) + Outputs - sum_m (u32 sum) + Return Type - unsigned word + Details : 8 unsigned halfword elements of input vector are added + together and the resulting integer sum is returned +*/ +#define HADD_UH_U32(in) \ + ({ \ + v4u32 res_m; \ + v2u64 res0_m, res1_m; \ + uint32_t sum_m; \ + \ + res_m = __msa_hadd_u_w((v8u16)in, (v8u16)in); \ + res0_m = __msa_hadd_u_d(res_m, res_m); \ + res1_m = (v2u64)__msa_splati_d((v2i64)res0_m, 1); \ + res0_m = res0_m + res1_m; \ + sum_m = __msa_copy_u_w((v4i32)res0_m, 0); \ + sum_m; \ + }) + +/* Description : Horizontal addition of unsigned byte vector elements + Arguments : Inputs - in0, in1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Each unsigned odd byte element from 'in0' is added to + even unsigned byte element from 'in0' (pairwise) and the + halfword result is written to 'out0' +*/ +#define HADD_UB2(RTYPE, in0, in1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_hadd_u_h((v16u8)in0, (v16u8)in0); \ + out1 = (RTYPE)__msa_hadd_u_h((v16u8)in1, (v16u8)in1); \ + } +#define HADD_UB2_UH(...) HADD_UB2(v8u16, __VA_ARGS__) + +#define HADD_UB4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + HADD_UB2(RTYPE, in0, in1, out0, out1); \ + HADD_UB2(RTYPE, in2, in3, out2, out3); \ + } +#define HADD_UB4_UH(...) HADD_UB4(v8u16, __VA_ARGS__) + +/* Description : Horizontal subtraction of unsigned byte vector elements + Arguments : Inputs - in0, in1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Each unsigned odd byte element from 'in0' is subtracted from + even unsigned byte element from 'in0' (pairwise) and the + halfword result is written to 'out0' +*/ +#define HSUB_UB2(RTYPE, in0, in1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_hsub_u_h((v16u8)in0, (v16u8)in0); \ + out1 = (RTYPE)__msa_hsub_u_h((v16u8)in1, (v16u8)in1); \ + } +#define HSUB_UB2_SH(...) HSUB_UB2(v8i16, __VA_ARGS__) + +/* Description : SAD (Sum of Absolute Difference) + Arguments : Inputs - in0, in1, ref0, ref1 + Outputs - sad_m (halfword vector) + Return Type - unsigned halfword + Details : Absolute difference of all the byte elements from 'in0' with + 'ref0' is calculated and preserved in 'diff0'. Then even-odd + pairs are added together to generate 8 halfword results. +*/ +#define SAD_UB2_UH(in0, in1, ref0, ref1) \ + ({ \ + v16u8 diff0_m, diff1_m; \ + v8u16 sad_m = { 0 }; \ + \ + diff0_m = __msa_asub_u_b((v16u8)in0, (v16u8)ref0); \ + diff1_m = __msa_asub_u_b((v16u8)in1, (v16u8)ref1); \ + \ + sad_m += __msa_hadd_u_h((v16u8)diff0_m, (v16u8)diff0_m); \ + sad_m += __msa_hadd_u_h((v16u8)diff1_m, (v16u8)diff1_m); \ + \ + sad_m; \ + }) + +/* Description : Horizontal subtraction of signed halfword vector elements + Arguments : Inputs - in0, in1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Each signed odd halfword element from 'in0' is subtracted from + even signed halfword element from 'in0' (pairwise) and the + word result is written to 'out0' +*/ +#define HSUB_UH2(RTYPE, in0, in1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_hsub_s_w((v8i16)in0, (v8i16)in0); \ + out1 = (RTYPE)__msa_hsub_s_w((v8i16)in1, (v8i16)in1); \ + } +#define HSUB_UH2_SW(...) HSUB_UH2(v4i32, __VA_ARGS__) + +/* Description : Set element n input vector to GPR value + Arguments : Inputs - in0, in1, in2, in3 + Output - out + Return Type - as per RTYPE + Details : Set element 0 in vector 'out' to value specified in 'in0' +*/ +#define INSERT_W2(RTYPE, in0, in1, out) \ + { \ + out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \ + out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \ + } +#define INSERT_W2_SB(...) INSERT_W2(v16i8, __VA_ARGS__) + +#define INSERT_W4(RTYPE, in0, in1, in2, in3, out) \ + { \ + out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \ + out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \ + out = (RTYPE)__msa_insert_w((v4i32)out, 2, in2); \ + out = (RTYPE)__msa_insert_w((v4i32)out, 3, in3); \ + } +#define INSERT_W4_UB(...) INSERT_W4(v16u8, __VA_ARGS__) +#define INSERT_W4_SB(...) INSERT_W4(v16i8, __VA_ARGS__) + +#define INSERT_D2(RTYPE, in0, in1, out) \ + { \ + out = (RTYPE)__msa_insert_d((v2i64)out, 0, in0); \ + out = (RTYPE)__msa_insert_d((v2i64)out, 1, in1); \ + } +#define INSERT_D2_UB(...) INSERT_D2(v16u8, __VA_ARGS__) +#define INSERT_D2_SB(...) INSERT_D2(v16i8, __VA_ARGS__) + +/* Description : Interleave even byte elements from vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Even byte elements of 'in0' and 'in1' are interleaved + and written to 'out0' +*/ +#define ILVEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvev_b((v16i8)in1, (v16i8)in0); \ + out1 = (RTYPE)__msa_ilvev_b((v16i8)in3, (v16i8)in2); \ + } +#define ILVEV_B2_UB(...) ILVEV_B2(v16u8, __VA_ARGS__) +#define ILVEV_B2_SH(...) ILVEV_B2(v8i16, __VA_ARGS__) + +/* Description : Interleave even halfword elements from vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Even halfword elements of 'in0' and 'in1' are interleaved + and written to 'out0' +*/ +#define ILVEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0); \ + out1 = (RTYPE)__msa_ilvev_h((v8i16)in3, (v8i16)in2); \ + } +#define ILVEV_H2_UB(...) ILVEV_H2(v16u8, __VA_ARGS__) +#define ILVEV_H2_SH(...) ILVEV_H2(v8i16, __VA_ARGS__) +#define ILVEV_H2_SW(...) ILVEV_H2(v4i32, __VA_ARGS__) + +/* Description : Interleave even word elements from vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Even word elements of 'in0' and 'in1' are interleaved + and written to 'out0' +*/ +#define ILVEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0); \ + out1 = (RTYPE)__msa_ilvev_w((v4i32)in3, (v4i32)in2); \ + } +#define ILVEV_W2_SB(...) ILVEV_W2(v16i8, __VA_ARGS__) + +/* Description : Interleave even double word elements from vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Even double word elements of 'in0' and 'in1' are interleaved + and written to 'out0' +*/ +#define ILVEV_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvev_d((v2i64)in1, (v2i64)in0); \ + out1 = (RTYPE)__msa_ilvev_d((v2i64)in3, (v2i64)in2); \ + } +#define ILVEV_D2_UB(...) ILVEV_D2(v16u8, __VA_ARGS__) + +/* Description : Interleave left half of byte elements from vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Left half of byte elements of 'in0' and 'in1' are interleaved + and written to 'out0'. +*/ +#define ILVL_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1); \ + out1 = (RTYPE)__msa_ilvl_b((v16i8)in2, (v16i8)in3); \ + } +#define ILVL_B2_UB(...) ILVL_B2(v16u8, __VA_ARGS__) +#define ILVL_B2_SB(...) ILVL_B2(v16i8, __VA_ARGS__) +#define ILVL_B2_UH(...) ILVL_B2(v8u16, __VA_ARGS__) +#define ILVL_B2_SH(...) ILVL_B2(v8i16, __VA_ARGS__) + +#define ILVL_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3) \ + { \ + ILVL_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ + ILVL_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ + } +#define ILVL_B4_SB(...) ILVL_B4(v16i8, __VA_ARGS__) +#define ILVL_B4_SH(...) ILVL_B4(v8i16, __VA_ARGS__) +#define ILVL_B4_UH(...) ILVL_B4(v8u16, __VA_ARGS__) + +/* Description : Interleave left half of halfword elements from vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Left half of halfword elements of 'in0' and 'in1' are + interleaved and written to 'out0'. +*/ +#define ILVL_H2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvl_h((v8i16)in0, (v8i16)in1); \ + out1 = (RTYPE)__msa_ilvl_h((v8i16)in2, (v8i16)in3); \ + } +#define ILVL_H2_SH(...) ILVL_H2(v8i16, __VA_ARGS__) +#define ILVL_H2_SW(...) ILVL_H2(v4i32, __VA_ARGS__) + +/* Description : Interleave left half of word elements from vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Left half of word elements of 'in0' and 'in1' are interleaved + and written to 'out0'. +*/ +#define ILVL_W2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvl_w((v4i32)in0, (v4i32)in1); \ + out1 = (RTYPE)__msa_ilvl_w((v4i32)in2, (v4i32)in3); \ + } +#define ILVL_W2_UB(...) ILVL_W2(v16u8, __VA_ARGS__) +#define ILVL_W2_SH(...) ILVL_W2(v8i16, __VA_ARGS__) + +/* Description : Interleave right half of byte elements from vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Right half of byte elements of 'in0' and 'in1' are interleaved + and written to out0. +*/ +#define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \ + out1 = (RTYPE)__msa_ilvr_b((v16i8)in2, (v16i8)in3); \ + } +#define ILVR_B2_UB(...) ILVR_B2(v16u8, __VA_ARGS__) +#define ILVR_B2_SB(...) ILVR_B2(v16i8, __VA_ARGS__) +#define ILVR_B2_UH(...) ILVR_B2(v8u16, __VA_ARGS__) +#define ILVR_B2_SH(...) ILVR_B2(v8i16, __VA_ARGS__) + +#define ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3) \ + { \ + ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ + ILVR_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ + } +#define ILVR_B4_UB(...) ILVR_B4(v16u8, __VA_ARGS__) +#define ILVR_B4_SB(...) ILVR_B4(v16i8, __VA_ARGS__) +#define ILVR_B4_UH(...) ILVR_B4(v8u16, __VA_ARGS__) +#define ILVR_B4_SH(...) ILVR_B4(v8i16, __VA_ARGS__) + +#define ILVR_B8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, \ + in11, in12, in13, in14, in15, out0, out1, out2, out3, out4, \ + out5, out6, out7) \ + { \ + ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, \ + out3); \ + ILVR_B4(RTYPE, in8, in9, in10, in11, in12, in13, in14, in15, out4, out5, \ + out6, out7); \ + } +#define ILVR_B8_UH(...) ILVR_B8(v8u16, __VA_ARGS__) + +/* Description : Interleave right half of halfword elements from vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Right half of halfword elements of 'in0' and 'in1' are + interleaved and written to 'out0'. +*/ +#define ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \ + out1 = (RTYPE)__msa_ilvr_h((v8i16)in2, (v8i16)in3); \ + } +#define ILVR_H2_SH(...) ILVR_H2(v8i16, __VA_ARGS__) +#define ILVR_H2_SW(...) ILVR_H2(v4i32, __VA_ARGS__) + +#define ILVR_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3) \ + { \ + ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ + ILVR_H2(RTYPE, in4, in5, in6, in7, out2, out3); \ + } +#define ILVR_H4_SH(...) ILVR_H4(v8i16, __VA_ARGS__) + +#define ILVR_W2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvr_w((v4i32)in0, (v4i32)in1); \ + out1 = (RTYPE)__msa_ilvr_w((v4i32)in2, (v4i32)in3); \ + } +#define ILVR_W2_UB(...) ILVR_W2(v16u8, __VA_ARGS__) +#define ILVR_W2_SH(...) ILVR_W2(v8i16, __VA_ARGS__) + +#define ILVR_W4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3) \ + { \ + ILVR_W2(RTYPE, in0, in1, in2, in3, out0, out1); \ + ILVR_W2(RTYPE, in4, in5, in6, in7, out2, out3); \ + } +#define ILVR_W4_UB(...) ILVR_W4(v16u8, __VA_ARGS__) + +/* Description : Interleave right half of double word elements from vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Right half of double word elements of 'in0' and 'in1' are + interleaved and written to 'out0'. +*/ +#define ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvr_d((v2i64)(in0), (v2i64)(in1)); \ + out1 = (RTYPE)__msa_ilvr_d((v2i64)(in2), (v2i64)(in3)); \ + } +#define ILVR_D2_UB(...) ILVR_D2(v16u8, __VA_ARGS__) +#define ILVR_D2_SB(...) ILVR_D2(v16i8, __VA_ARGS__) +#define ILVR_D2_SH(...) ILVR_D2(v8i16, __VA_ARGS__) + +#define ILVR_D3(RTYPE, in0, in1, in2, in3, in4, in5, out0, out1, out2) \ + { \ + ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1); \ + out2 = (RTYPE)__msa_ilvr_d((v2i64)(in4), (v2i64)(in5)); \ + } +#define ILVR_D3_SB(...) ILVR_D3(v16i8, __VA_ARGS__) + +#define ILVR_D4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3) \ + { \ + ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1); \ + ILVR_D2(RTYPE, in4, in5, in6, in7, out2, out3); \ + } +#define ILVR_D4_SB(...) ILVR_D4(v16i8, __VA_ARGS__) +#define ILVR_D4_UB(...) ILVR_D4(v16u8, __VA_ARGS__) + +/* Description : Interleave both left and right half of input vectors + Arguments : Inputs - in0, in1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Right half of byte elements from 'in0' and 'in1' are + interleaved and written to 'out0' +*/ +#define ILVRL_B2(RTYPE, in0, in1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \ + out1 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1); \ + } +#define ILVRL_B2_UB(...) ILVRL_B2(v16u8, __VA_ARGS__) +#define ILVRL_B2_SB(...) ILVRL_B2(v16i8, __VA_ARGS__) +#define ILVRL_B2_UH(...) ILVRL_B2(v8u16, __VA_ARGS__) +#define ILVRL_B2_SH(...) ILVRL_B2(v8i16, __VA_ARGS__) + +#define ILVRL_H2(RTYPE, in0, in1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \ + out1 = (RTYPE)__msa_ilvl_h((v8i16)in0, (v8i16)in1); \ + } +#define ILVRL_H2_SH(...) ILVRL_H2(v8i16, __VA_ARGS__) +#define ILVRL_H2_SW(...) ILVRL_H2(v4i32, __VA_ARGS__) + +#define ILVRL_W2(RTYPE, in0, in1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_ilvr_w((v4i32)in0, (v4i32)in1); \ + out1 = (RTYPE)__msa_ilvl_w((v4i32)in0, (v4i32)in1); \ + } +#define ILVRL_W2_UB(...) ILVRL_W2(v16u8, __VA_ARGS__) +#define ILVRL_W2_SH(...) ILVRL_W2(v8i16, __VA_ARGS__) +#define ILVRL_W2_SW(...) ILVRL_W2(v4i32, __VA_ARGS__) + +/* Description : Saturate the halfword element values to the max + unsigned value of (sat_val + 1) bits + The element data width remains unchanged + Arguments : Inputs - in0, in1, sat_val + Outputs - in place operation + Return Type - as per RTYPE + Details : Each unsigned halfword element from 'in0' is saturated to the + value generated with (sat_val + 1) bit range. + The results are written in place +*/ +#define SAT_UH2(RTYPE, in0, in1, sat_val) \ + { \ + in0 = (RTYPE)__msa_sat_u_h((v8u16)in0, sat_val); \ + in1 = (RTYPE)__msa_sat_u_h((v8u16)in1, sat_val); \ + } +#define SAT_UH2_UH(...) SAT_UH2(v8u16, __VA_ARGS__) + +#define SAT_UH4(RTYPE, in0, in1, in2, in3, sat_val) \ + { \ + SAT_UH2(RTYPE, in0, in1, sat_val); \ + SAT_UH2(RTYPE, in2, in3, sat_val) \ + } +#define SAT_UH4_UH(...) SAT_UH4(v8u16, __VA_ARGS__) + +/* Description : Saturate the halfword element values to the max + unsigned value of (sat_val + 1) bits + The element data width remains unchanged + Arguments : Inputs - in0, in1, sat_val + Outputs - in place operation + Return Type - as per RTYPE + Details : Each unsigned halfword element from 'in0' is saturated to the + value generated with (sat_val + 1) bit range + The results are written in place +*/ +#define SAT_SH2(RTYPE, in0, in1, sat_val) \ + { \ + in0 = (RTYPE)__msa_sat_s_h((v8i16)in0, sat_val); \ + in1 = (RTYPE)__msa_sat_s_h((v8i16)in1, sat_val); \ + } +#define SAT_SH2_SH(...) SAT_SH2(v8i16, __VA_ARGS__) + +#define SAT_SH4(RTYPE, in0, in1, in2, in3, sat_val) \ + { \ + SAT_SH2(RTYPE, in0, in1, sat_val); \ + SAT_SH2(RTYPE, in2, in3, sat_val); \ + } +#define SAT_SH4_SH(...) SAT_SH4(v8i16, __VA_ARGS__) + +/* Description : Indexed halfword element values are replicated to all + elements in output vector + Arguments : Inputs - in, idx0, idx1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : 'idx0' element value from 'in' vector is replicated to all + elements in 'out0' vector + Valid index range for halfword operation is 0-7 +*/ +#define SPLATI_H2(RTYPE, in, idx0, idx1, out0, out1) \ + { \ + out0 = (RTYPE)__msa_splati_h((v8i16)in, idx0); \ + out1 = (RTYPE)__msa_splati_h((v8i16)in, idx1); \ + } +#define SPLATI_H2_SH(...) SPLATI_H2(v8i16, __VA_ARGS__) + +#define SPLATI_H4(RTYPE, in, idx0, idx1, idx2, idx3, out0, out1, out2, out3) \ + { \ + SPLATI_H2(RTYPE, in, idx0, idx1, out0, out1); \ + SPLATI_H2(RTYPE, in, idx2, idx3, out2, out3); \ + } +#define SPLATI_H4_SB(...) SPLATI_H4(v16i8, __VA_ARGS__) +#define SPLATI_H4_SH(...) SPLATI_H4(v8i16, __VA_ARGS__) + +/* Description : Pack even byte elements of vector pairs + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Even byte elements of 'in0' are copied to the left half of + 'out0' & even byte elements of 'in1' are copied to the right + half of 'out0'. +*/ +#define PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_pckev_b((v16i8)in0, (v16i8)in1); \ + out1 = (RTYPE)__msa_pckev_b((v16i8)in2, (v16i8)in3); \ + } +#define PCKEV_B2_SB(...) PCKEV_B2(v16i8, __VA_ARGS__) +#define PCKEV_B2_UB(...) PCKEV_B2(v16u8, __VA_ARGS__) +#define PCKEV_B2_SH(...) PCKEV_B2(v8i16, __VA_ARGS__) + +#define PCKEV_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3) \ + { \ + PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ + PCKEV_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ + } +#define PCKEV_B4_SB(...) PCKEV_B4(v16i8, __VA_ARGS__) +#define PCKEV_B4_UB(...) PCKEV_B4(v16u8, __VA_ARGS__) +#define PCKEV_B4_SH(...) PCKEV_B4(v8i16, __VA_ARGS__) + +/* Description : Pack even halfword elements of vector pairs + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Even halfword elements of 'in0' are copied to the left half of + 'out0' & even halfword elements of 'in1' are copied to the + right half of 'out0'. +*/ +#define PCKEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_pckev_h((v8i16)in0, (v8i16)in1); \ + out1 = (RTYPE)__msa_pckev_h((v8i16)in2, (v8i16)in3); \ + } +#define PCKEV_H2_SH(...) PCKEV_H2(v8i16, __VA_ARGS__) +#define PCKEV_H2_SW(...) PCKEV_H2(v4i32, __VA_ARGS__) + +#define PCKEV_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3) \ + { \ + PCKEV_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ + PCKEV_H2(RTYPE, in4, in5, in6, in7, out2, out3); \ + } +#define PCKEV_H4_SH(...) PCKEV_H4(v8i16, __VA_ARGS__) + +/* Description : Pack even double word elements of vector pairs + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Even double elements of 'in0' are copied to the left half of + 'out0' & even double elements of 'in1' are copied to the right + half of 'out0'. +*/ +#define PCKEV_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_pckev_d((v2i64)in0, (v2i64)in1); \ + out1 = (RTYPE)__msa_pckev_d((v2i64)in2, (v2i64)in3); \ + } +#define PCKEV_D2_UB(...) PCKEV_D2(v16u8, __VA_ARGS__) +#define PCKEV_D2_SH(...) PCKEV_D2(v8i16, __VA_ARGS__) + +#define PCKEV_D4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3) \ + { \ + PCKEV_D2(RTYPE, in0, in1, in2, in3, out0, out1); \ + PCKEV_D2(RTYPE, in4, in5, in6, in7, out2, out3); \ + } +#define PCKEV_D4_UB(...) PCKEV_D4(v16u8, __VA_ARGS__) + +/* Description : Each byte element is logically xor'ed with immediate 128 + Arguments : Inputs - in0, in1 + Outputs - in place operation + Return Type - as per RTYPE + Details : Each unsigned byte element from input vector 'in0' is + logically xor'ed with 128 and the result is stored in-place. +*/ +#define XORI_B2_128(RTYPE, in0, in1) \ + { \ + in0 = (RTYPE)__msa_xori_b((v16u8)in0, 128); \ + in1 = (RTYPE)__msa_xori_b((v16u8)in1, 128); \ + } +#define XORI_B2_128_UB(...) XORI_B2_128(v16u8, __VA_ARGS__) +#define XORI_B2_128_SB(...) XORI_B2_128(v16i8, __VA_ARGS__) + +#define XORI_B3_128(RTYPE, in0, in1, in2) \ + { \ + XORI_B2_128(RTYPE, in0, in1); \ + in2 = (RTYPE)__msa_xori_b((v16u8)in2, 128); \ + } +#define XORI_B3_128_SB(...) XORI_B3_128(v16i8, __VA_ARGS__) + +#define XORI_B4_128(RTYPE, in0, in1, in2, in3) \ + { \ + XORI_B2_128(RTYPE, in0, in1); \ + XORI_B2_128(RTYPE, in2, in3); \ + } +#define XORI_B4_128_UB(...) XORI_B4_128(v16u8, __VA_ARGS__) +#define XORI_B4_128_SB(...) XORI_B4_128(v16i8, __VA_ARGS__) + +#define XORI_B7_128(RTYPE, in0, in1, in2, in3, in4, in5, in6) \ + { \ + XORI_B4_128(RTYPE, in0, in1, in2, in3); \ + XORI_B3_128(RTYPE, in4, in5, in6); \ + } +#define XORI_B7_128_SB(...) XORI_B7_128(v16i8, __VA_ARGS__) + +/* Description : Average of signed halfword elements -> (a + b) / 2 + Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7 + Outputs - out0, out1, out2, out3 + Return Type - as per RTYPE + Details : Each signed halfword element from 'in0' is added to each + signed halfword element of 'in1' with full precision resulting + in one extra bit in the result. The result is then divided by + 2 and written to 'out0' +*/ +#define AVE_SH4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3) \ + { \ + out0 = (RTYPE)__msa_ave_s_h((v8i16)in0, (v8i16)in1); \ + out1 = (RTYPE)__msa_ave_s_h((v8i16)in2, (v8i16)in3); \ + out2 = (RTYPE)__msa_ave_s_h((v8i16)in4, (v8i16)in5); \ + out3 = (RTYPE)__msa_ave_s_h((v8i16)in6, (v8i16)in7); \ + } +#define AVE_SH4_SH(...) AVE_SH4(v8i16, __VA_ARGS__) + +/* Description : Addition of signed halfword elements and signed saturation + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Signed halfword elements from 'in0' are added to signed + halfword elements of 'in1'. The result is then signed saturated + between halfword data type range +*/ +#define ADDS_SH2(RTYPE, in0, in1, in2, in3, out0, out1) \ + { \ + out0 = (RTYPE)__msa_adds_s_h((v8i16)in0, (v8i16)in1); \ + out1 = (RTYPE)__msa_adds_s_h((v8i16)in2, (v8i16)in3); \ + } +#define ADDS_SH2_SH(...) ADDS_SH2(v8i16, __VA_ARGS__) + +#define ADDS_SH4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3) \ + { \ + ADDS_SH2(RTYPE, in0, in1, in2, in3, out0, out1); \ + ADDS_SH2(RTYPE, in4, in5, in6, in7, out2, out3); \ + } +#define ADDS_SH4_SH(...) ADDS_SH4(v8i16, __VA_ARGS__) + +/* Description : Shift left all elements of vector (generic for all data types) + Arguments : Inputs - in0, in1, in2, in3, shift + Outputs - in place operation + Return Type - as per input vector RTYPE + Details : Each element of vector 'in0' is left shifted by 'shift' and + the result is written in-place. +*/ +#define SLLI_4V(in0, in1, in2, in3, shift) \ + { \ + in0 = in0 << shift; \ + in1 = in1 << shift; \ + in2 = in2 << shift; \ + in3 = in3 << shift; \ + } + +/* Description : Arithmetic shift right all elements of vector + (generic for all data types) + Arguments : Inputs - in0, in1, in2, in3, shift + Outputs - in place operation + Return Type - as per input vector RTYPE + Details : Each element of vector 'in0' is right shifted by 'shift' and + the result is written in-place. 'shift' is a GP variable. +*/ +#define SRA_4V(in0, in1, in2, in3, shift) \ + { \ + in0 = in0 >> shift; \ + in1 = in1 >> shift; \ + in2 = in2 >> shift; \ + in3 = in3 >> shift; \ + } + +/* Description : Shift right arithmetic rounded words + Arguments : Inputs - in0, in1, shift + Outputs - in place operation + Return Type - as per RTYPE + Details : Each element of vector 'in0' is shifted right arithmetically by + the number of bits in the corresponding element in the vector + 'shift'. The last discarded bit is added to shifted value for + rounding and the result is written in-place. + 'shift' is a vector. +*/ +#define SRAR_W2(RTYPE, in0, in1, shift) \ + { \ + in0 = (RTYPE)__msa_srar_w((v4i32)in0, (v4i32)shift); \ + in1 = (RTYPE)__msa_srar_w((v4i32)in1, (v4i32)shift); \ + } + +#define SRAR_W4(RTYPE, in0, in1, in2, in3, shift) \ + { \ + SRAR_W2(RTYPE, in0, in1, shift) \ + SRAR_W2(RTYPE, in2, in3, shift) \ + } +#define SRAR_W4_SW(...) SRAR_W4(v4i32, __VA_ARGS__) + +/* Description : Shift right arithmetic rounded (immediate) + Arguments : Inputs - in0, in1, shift + Outputs - in place operation + Return Type - as per RTYPE + Details : Each element of vector 'in0' is shifted right arithmetically by + the value in 'shift'. The last discarded bit is added to the + shifted value for rounding and the result is written in-place. + 'shift' is an immediate value. +*/ +#define SRARI_H2(RTYPE, in0, in1, shift) \ + { \ + in0 = (RTYPE)__msa_srari_h((v8i16)in0, shift); \ + in1 = (RTYPE)__msa_srari_h((v8i16)in1, shift); \ + } +#define SRARI_H2_UH(...) SRARI_H2(v8u16, __VA_ARGS__) +#define SRARI_H2_SH(...) SRARI_H2(v8i16, __VA_ARGS__) + +#define SRARI_H4(RTYPE, in0, in1, in2, in3, shift) \ + { \ + SRARI_H2(RTYPE, in0, in1, shift); \ + SRARI_H2(RTYPE, in2, in3, shift); \ + } +#define SRARI_H4_UH(...) SRARI_H4(v8u16, __VA_ARGS__) +#define SRARI_H4_SH(...) SRARI_H4(v8i16, __VA_ARGS__) + +#define SRARI_W2(RTYPE, in0, in1, shift) \ + { \ + in0 = (RTYPE)__msa_srari_w((v4i32)in0, shift); \ + in1 = (RTYPE)__msa_srari_w((v4i32)in1, shift); \ + } +#define SRARI_W2_SW(...) SRARI_W2(v4i32, __VA_ARGS__) + +#define SRARI_W4(RTYPE, in0, in1, in2, in3, shift) \ + { \ + SRARI_W2(RTYPE, in0, in1, shift); \ + SRARI_W2(RTYPE, in2, in3, shift); \ + } +#define SRARI_W4_SW(...) SRARI_W4(v4i32, __VA_ARGS__) + +/* Description : Logical shift right all elements of vector (immediate) + Arguments : Inputs - in0, in1, in2, in3, shift + Outputs - out0, out1, out2, out3 + Return Type - as per RTYPE + Details : Each element of vector 'in0' is right shifted by 'shift' and + the result is written in-place. 'shift' is an immediate value. +*/ +#define SRLI_H4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3, shift) \ + { \ + out0 = (RTYPE)__msa_srli_h((v8i16)in0, shift); \ + out1 = (RTYPE)__msa_srli_h((v8i16)in1, shift); \ + out2 = (RTYPE)__msa_srli_h((v8i16)in2, shift); \ + out3 = (RTYPE)__msa_srli_h((v8i16)in3, shift); \ + } +#define SRLI_H4_SH(...) SRLI_H4(v8i16, __VA_ARGS__) + +/* Description : Multiplication of pairs of vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Details : Each element from 'in0' is multiplied with elements from 'in1' + and the result is written to 'out0' +*/ +#define MUL2(in0, in1, in2, in3, out0, out1) \ + { \ + out0 = in0 * in1; \ + out1 = in2 * in3; \ + } +#define MUL4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3) \ + { \ + MUL2(in0, in1, in2, in3, out0, out1); \ + MUL2(in4, in5, in6, in7, out2, out3); \ + } + +/* Description : Addition of 2 pairs of vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Details : Each element in 'in0' is added to 'in1' and result is written + to 'out0'. +*/ +#define ADD2(in0, in1, in2, in3, out0, out1) \ + { \ + out0 = in0 + in1; \ + out1 = in2 + in3; \ + } +#define ADD4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3) \ + { \ + ADD2(in0, in1, in2, in3, out0, out1); \ + ADD2(in4, in5, in6, in7, out2, out3); \ + } + +/* Description : Subtraction of 2 pairs of vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1 + Details : Each element in 'in1' is subtracted from 'in0' and result is + written to 'out0'. +*/ +#define SUB2(in0, in1, in2, in3, out0, out1) \ + { \ + out0 = in0 - in1; \ + out1 = in2 - in3; \ + } +#define SUB4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3) \ + { \ + out0 = in0 - in1; \ + out1 = in2 - in3; \ + out2 = in4 - in5; \ + out3 = in6 - in7; \ + } + +/* Description : Sign extend halfword elements from right half of the vector + Arguments : Input - in (halfword vector) + Output - out (sign extended word vector) + Return Type - signed word + Details : Sign bit of halfword elements from input vector 'in' is + extracted and interleaved with same vector 'in0' to generate + 4 word elements keeping sign intact +*/ +#define UNPCK_R_SH_SW(in, out) \ + { \ + v8i16 sign_m; \ + \ + sign_m = __msa_clti_s_h((v8i16)in, 0); \ + out = (v4i32)__msa_ilvr_h(sign_m, (v8i16)in); \ + } + +/* Description : Zero extend unsigned byte elements to halfword elements + Arguments : Input - in (unsigned byte vector) + Outputs - out0, out1 (unsigned halfword vectors) + Return Type - signed halfword + Details : Zero extended right half of vector is returned in 'out0' + Zero extended left half of vector is returned in 'out1' +*/ +#define UNPCK_UB_SH(in, out0, out1) \ + { \ + v16i8 zero_m = { 0 }; \ + \ + ILVRL_B2_SH(zero_m, in, out0, out1); \ + } + +/* Description : Sign extend halfword elements from input vector and return + the result in pair of vectors + Arguments : Input - in (halfword vector) + Outputs - out0, out1 (sign extended word vectors) + Return Type - signed word + Details : Sign bit of halfword elements from input vector 'in' is + extracted and interleaved right with same vector 'in0' to + generate 4 signed word elements in 'out0' + Then interleaved left with same vector 'in0' to + generate 4 signed word elements in 'out1' +*/ +#define UNPCK_SH_SW(in, out0, out1) \ + { \ + v8i16 tmp_m; \ + \ + tmp_m = __msa_clti_s_h((v8i16)in, 0); \ + ILVRL_H2_SW(tmp_m, in, out0, out1); \ + } + +/* Description : Butterfly of 4 input vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1, out2, out3 + Details : Butterfly operation +*/ +#define BUTTERFLY_4(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + out0 = in0 + in3; \ + out1 = in1 + in2; \ + \ + out2 = in1 - in2; \ + out3 = in0 - in3; \ + } + +/* Description : Butterfly of 8 input vectors + Arguments : Inputs - in0 ... in7 + Outputs - out0 .. out7 + Details : Butterfly operation +*/ +#define BUTTERFLY_8(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, \ + out3, out4, out5, out6, out7) \ + { \ + out0 = in0 + in7; \ + out1 = in1 + in6; \ + out2 = in2 + in5; \ + out3 = in3 + in4; \ + \ + out4 = in3 - in4; \ + out5 = in2 - in5; \ + out6 = in1 - in6; \ + out7 = in0 - in7; \ + } + +/* Description : Butterfly of 16 input vectors + Arguments : Inputs - in0 ... in15 + Outputs - out0 .. out15 + Details : Butterfly operation +*/ +#define BUTTERFLY_16(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, \ + in11, in12, in13, in14, in15, out0, out1, out2, out3, \ + out4, out5, out6, out7, out8, out9, out10, out11, out12, \ + out13, out14, out15) \ + { \ + out0 = in0 + in15; \ + out1 = in1 + in14; \ + out2 = in2 + in13; \ + out3 = in3 + in12; \ + out4 = in4 + in11; \ + out5 = in5 + in10; \ + out6 = in6 + in9; \ + out7 = in7 + in8; \ + \ + out8 = in7 - in8; \ + out9 = in6 - in9; \ + out10 = in5 - in10; \ + out11 = in4 - in11; \ + out12 = in3 - in12; \ + out13 = in2 - in13; \ + out14 = in1 - in14; \ + out15 = in0 - in15; \ + } + +/* Description : Transpose input 8x8 byte block + Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7 + Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + Return Type - as per RTYPE +*/ +#define TRANSPOSE8x8_UB(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, \ + out1, out2, out3, out4, out5, out6, out7) \ + { \ + v16i8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + v16i8 tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ + \ + ILVR_B4_SB(in2, in0, in3, in1, in6, in4, in7, in5, tmp0_m, tmp1_m, tmp2_m, \ + tmp3_m); \ + ILVRL_B2_SB(tmp1_m, tmp0_m, tmp4_m, tmp5_m); \ + ILVRL_B2_SB(tmp3_m, tmp2_m, tmp6_m, tmp7_m); \ + ILVRL_W2(RTYPE, tmp6_m, tmp4_m, out0, out2); \ + ILVRL_W2(RTYPE, tmp7_m, tmp5_m, out4, out6); \ + SLDI_B2_0(RTYPE, out0, out2, out1, out3, 8); \ + SLDI_B2_0(RTYPE, out4, out6, out5, out7, 8); \ + } +#define TRANSPOSE8x8_UB_UB(...) TRANSPOSE8x8_UB(v16u8, __VA_ARGS__) + +/* Description : Transpose 16x8 block into 8x16 with byte elements in vectors + Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, + in8, in9, in10, in11, in12, in13, in14, in15 + Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + Return Type - unsigned byte +*/ +#define TRANSPOSE16x8_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, \ + in10, in11, in12, in13, in14, in15, out0, out1, \ + out2, out3, out4, out5, out6, out7) \ + { \ + v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + v16u8 tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ + \ + ILVEV_D2_UB(in0, in8, in1, in9, out7, out6); \ + ILVEV_D2_UB(in2, in10, in3, in11, out5, out4); \ + ILVEV_D2_UB(in4, in12, in5, in13, out3, out2); \ + ILVEV_D2_UB(in6, in14, in7, in15, out1, out0); \ + \ + tmp0_m = (v16u8)__msa_ilvev_b((v16i8)out6, (v16i8)out7); \ + tmp4_m = (v16u8)__msa_ilvod_b((v16i8)out6, (v16i8)out7); \ + tmp1_m = (v16u8)__msa_ilvev_b((v16i8)out4, (v16i8)out5); \ + tmp5_m = (v16u8)__msa_ilvod_b((v16i8)out4, (v16i8)out5); \ + out5 = (v16u8)__msa_ilvev_b((v16i8)out2, (v16i8)out3); \ + tmp6_m = (v16u8)__msa_ilvod_b((v16i8)out2, (v16i8)out3); \ + out7 = (v16u8)__msa_ilvev_b((v16i8)out0, (v16i8)out1); \ + tmp7_m = (v16u8)__msa_ilvod_b((v16i8)out0, (v16i8)out1); \ + \ + ILVEV_H2_UB(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m); \ + out0 = (v16u8)__msa_ilvev_w((v4i32)tmp3_m, (v4i32)tmp2_m); \ + out4 = (v16u8)__msa_ilvod_w((v4i32)tmp3_m, (v4i32)tmp2_m); \ + \ + tmp2_m = (v16u8)__msa_ilvod_h((v8i16)tmp1_m, (v8i16)tmp0_m); \ + tmp3_m = (v16u8)__msa_ilvod_h((v8i16)out7, (v8i16)out5); \ + out2 = (v16u8)__msa_ilvev_w((v4i32)tmp3_m, (v4i32)tmp2_m); \ + out6 = (v16u8)__msa_ilvod_w((v4i32)tmp3_m, (v4i32)tmp2_m); \ + \ + ILVEV_H2_UB(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m); \ + out1 = (v16u8)__msa_ilvev_w((v4i32)tmp3_m, (v4i32)tmp2_m); \ + out5 = (v16u8)__msa_ilvod_w((v4i32)tmp3_m, (v4i32)tmp2_m); \ + \ + tmp2_m = (v16u8)__msa_ilvod_h((v8i16)tmp5_m, (v8i16)tmp4_m); \ + tmp2_m = (v16u8)__msa_ilvod_h((v8i16)tmp5_m, (v8i16)tmp4_m); \ + tmp3_m = (v16u8)__msa_ilvod_h((v8i16)tmp7_m, (v8i16)tmp6_m); \ + tmp3_m = (v16u8)__msa_ilvod_h((v8i16)tmp7_m, (v8i16)tmp6_m); \ + out3 = (v16u8)__msa_ilvev_w((v4i32)tmp3_m, (v4i32)tmp2_m); \ + out7 = (v16u8)__msa_ilvod_w((v4i32)tmp3_m, (v4i32)tmp2_m); \ + } + +/* Description : Transpose 4x4 block with half word elements in vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1, out2, out3 + Return Type - signed halfword +*/ +#define TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + v8i16 s0_m, s1_m; \ + \ + ILVR_H2_SH(in1, in0, in3, in2, s0_m, s1_m); \ + ILVRL_W2_SH(s1_m, s0_m, out0, out2); \ + out1 = (v8i16)__msa_ilvl_d((v2i64)out0, (v2i64)out0); \ + out3 = (v8i16)__msa_ilvl_d((v2i64)out0, (v2i64)out2); \ + } + +/* Description : Transpose 4x8 block with half word elements in vectors + Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7 + Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + Return Type - signed halfword +*/ +#define TRANSPOSE4X8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3, out4, out5, out6, out7) \ + { \ + v8i16 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + v8i16 tmp0_n, tmp1_n, tmp2_n, tmp3_n; \ + v8i16 zero_m = { 0 }; \ + \ + ILVR_H4_SH(in1, in0, in3, in2, in5, in4, in7, in6, tmp0_n, tmp1_n, tmp2_n, \ + tmp3_n); \ + ILVRL_W2_SH(tmp1_n, tmp0_n, tmp0_m, tmp2_m); \ + ILVRL_W2_SH(tmp3_n, tmp2_n, tmp1_m, tmp3_m); \ + \ + out0 = (v8i16)__msa_ilvr_d((v2i64)tmp1_m, (v2i64)tmp0_m); \ + out1 = (v8i16)__msa_ilvl_d((v2i64)tmp1_m, (v2i64)tmp0_m); \ + out2 = (v8i16)__msa_ilvr_d((v2i64)tmp3_m, (v2i64)tmp2_m); \ + out3 = (v8i16)__msa_ilvl_d((v2i64)tmp3_m, (v2i64)tmp2_m); \ + \ + out4 = zero_m; \ + out5 = zero_m; \ + out6 = zero_m; \ + out7 = zero_m; \ + } + +/* Description : Transpose 8x4 block with half word elements in vectors + Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7 + Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + Return Type - signed halfword +*/ +#define TRANSPOSE8X4_SH_SH(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + v8i16 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + \ + ILVR_H2_SH(in1, in0, in3, in2, tmp0_m, tmp1_m); \ + ILVL_H2_SH(in1, in0, in3, in2, tmp2_m, tmp3_m); \ + ILVR_W2_SH(tmp1_m, tmp0_m, tmp3_m, tmp2_m, out0, out2); \ + ILVL_W2_SH(tmp1_m, tmp0_m, tmp3_m, tmp2_m, out1, out3); \ + } + +/* Description : Transpose 8x8 block with half word elements in vectors + Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7 + Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + Return Type - as per RTYPE +*/ +#define TRANSPOSE8x8_H(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, out0, \ + out1, out2, out3, out4, out5, out6, out7) \ + { \ + v8i16 s0_m, s1_m; \ + v8i16 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + v8i16 tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ + \ + ILVR_H2_SH(in6, in4, in7, in5, s0_m, s1_m); \ + ILVRL_H2_SH(s1_m, s0_m, tmp0_m, tmp1_m); \ + ILVL_H2_SH(in6, in4, in7, in5, s0_m, s1_m); \ + ILVRL_H2_SH(s1_m, s0_m, tmp2_m, tmp3_m); \ + ILVR_H2_SH(in2, in0, in3, in1, s0_m, s1_m); \ + ILVRL_H2_SH(s1_m, s0_m, tmp4_m, tmp5_m); \ + ILVL_H2_SH(in2, in0, in3, in1, s0_m, s1_m); \ + ILVRL_H2_SH(s1_m, s0_m, tmp6_m, tmp7_m); \ + PCKEV_D4(RTYPE, tmp0_m, tmp4_m, tmp1_m, tmp5_m, tmp2_m, tmp6_m, tmp3_m, \ + tmp7_m, out0, out2, out4, out6); \ + out1 = (RTYPE)__msa_pckod_d((v2i64)tmp0_m, (v2i64)tmp4_m); \ + out3 = (RTYPE)__msa_pckod_d((v2i64)tmp1_m, (v2i64)tmp5_m); \ + out5 = (RTYPE)__msa_pckod_d((v2i64)tmp2_m, (v2i64)tmp6_m); \ + out7 = (RTYPE)__msa_pckod_d((v2i64)tmp3_m, (v2i64)tmp7_m); \ + } +#define TRANSPOSE8x8_SH_SH(...) TRANSPOSE8x8_H(v8i16, __VA_ARGS__) + +/* Description : Transpose 4x4 block with word elements in vectors + Arguments : Inputs - in0, in1, in2, in3 + Outputs - out0, out1, out2, out3 + Return Type - signed word +*/ +#define TRANSPOSE4x4_SW_SW(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + v4i32 s0_m, s1_m, s2_m, s3_m; \ + \ + ILVRL_W2_SW(in1, in0, s0_m, s1_m); \ + ILVRL_W2_SW(in3, in2, s2_m, s3_m); \ + \ + out0 = (v4i32)__msa_ilvr_d((v2i64)s2_m, (v2i64)s0_m); \ + out1 = (v4i32)__msa_ilvl_d((v2i64)s2_m, (v2i64)s0_m); \ + out2 = (v4i32)__msa_ilvr_d((v2i64)s3_m, (v2i64)s1_m); \ + out3 = (v4i32)__msa_ilvl_d((v2i64)s3_m, (v2i64)s1_m); \ + } + +/* Description : Add block 4x4 + Arguments : Inputs - in0, in1, in2, in3, pdst, stride + Details : Least significant 4 bytes from each input vector are added to + the destination bytes, clipped between 0-255 and stored. +*/ +#define ADDBLK_ST4x4_UB(in0, in1, in2, in3, pdst, stride) \ + { \ + uint32_t src0_m, src1_m, src2_m, src3_m; \ + v8i16 inp0_m, inp1_m, res0_m, res1_m; \ + v16i8 dst0_m = { 0 }; \ + v16i8 dst1_m = { 0 }; \ + v16i8 zero_m = { 0 }; \ + \ + ILVR_D2_SH(in1, in0, in3, in2, inp0_m, inp1_m) \ + LW4(pdst, stride, src0_m, src1_m, src2_m, src3_m); \ + INSERT_W2_SB(src0_m, src1_m, dst0_m); \ + INSERT_W2_SB(src2_m, src3_m, dst1_m); \ + ILVR_B2_SH(zero_m, dst0_m, zero_m, dst1_m, res0_m, res1_m); \ + ADD2(res0_m, inp0_m, res1_m, inp1_m, res0_m, res1_m); \ + CLIP_SH2_0_255(res0_m, res1_m); \ + PCKEV_B2_SB(res0_m, res0_m, res1_m, res1_m, dst0_m, dst1_m); \ + ST4x4_UB(dst0_m, dst1_m, 0, 1, 0, 1, pdst, stride); \ + } + +/* Description : Pack even elements of input vectors & xor with 128 + Arguments : Inputs - in0, in1 + Output - out_m + Return Type - unsigned byte + Details : Signed byte even elements from 'in0' and 'in1' are packed + together in one vector and the resulting vector is xor'ed with + 128 to shift the range from signed to unsigned byte +*/ +#define PCKEV_XORI128_UB(in0, in1) \ + ({ \ + v16u8 out_m; \ + \ + out_m = (v16u8)__msa_pckev_b((v16i8)in1, (v16i8)in0); \ + out_m = (v16u8)__msa_xori_b((v16u8)out_m, 128); \ + out_m; \ + }) + +/* Description : Converts inputs to unsigned bytes, interleave, average & store + as 8x4 unsigned byte block + Arguments : Inputs - in0, in1, in2, in3, dst0, dst1, dst2, dst3, + pdst, stride +*/ +#define CONVERT_UB_AVG_ST8x4_UB(in0, in1, in2, in3, dst0, dst1, dst2, dst3, \ + pdst, stride) \ + { \ + v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + \ + tmp0_m = PCKEV_XORI128_UB(in0, in1); \ + tmp1_m = PCKEV_XORI128_UB(in2, in3); \ + ILVR_D2_UB(dst1, dst0, dst3, dst2, tmp2_m, tmp3_m); \ + AVER_UB2_UB(tmp0_m, tmp2_m, tmp1_m, tmp3_m, tmp0_m, tmp1_m); \ + ST8x4_UB(tmp0_m, tmp1_m, pdst, stride); \ + } + +/* Description : Pack even byte elements and store byte vector in destination + memory + Arguments : Inputs - in0, in1, pdst +*/ +#define PCKEV_ST_SB(in0, in1, pdst) \ + { \ + v16i8 tmp_m; \ + \ + tmp_m = __msa_pckev_b((v16i8)in1, (v16i8)in0); \ + ST_SB(tmp_m, (pdst)); \ + } + +/* Description : Horizontal 2 tap filter kernel code + Arguments : Inputs - in0, in1, mask, coeff, shift +*/ +#define HORIZ_2TAP_FILT_UH(in0, in1, mask, coeff, shift) \ + ({ \ + v16i8 tmp0_m; \ + v8u16 tmp1_m; \ + \ + tmp0_m = __msa_vshf_b((v16i8)mask, (v16i8)in1, (v16i8)in0); \ + tmp1_m = __msa_dotp_u_h((v16u8)tmp0_m, (v16u8)coeff); \ + tmp1_m = (v8u16)__msa_srari_h((v8i16)tmp1_m, shift); \ + \ + tmp1_m; \ + }) +#endif /* AOM_DSP_MIPS_MACROS_MSA_H_ */ diff --git a/third_party/aom/aom_dsp/mips/sad_msa.c b/third_party/aom/aom_dsp/mips/sad_msa.c new file mode 100644 index 0000000000..258eb5c079 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/sad_msa.c @@ -0,0 +1,1529 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/macros_msa.h" + +#define SAD_INSVE_W4(RTYPE, in0, in1, in2, in3, out) \ + { \ + out = (RTYPE)__msa_insve_w((v4i32)out, 0, (v4i32)in0); \ + out = (RTYPE)__msa_insve_w((v4i32)out, 1, (v4i32)in1); \ + out = (RTYPE)__msa_insve_w((v4i32)out, 2, (v4i32)in2); \ + out = (RTYPE)__msa_insve_w((v4i32)out, 3, (v4i32)in3); \ + } +#define SAD_INSVE_W4_UB(...) SAD_INSVE_W4(v16u8, __VA_ARGS__) + +static uint32_t sad_4width_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt; + uint32_t src0, src1, src2, src3, ref0, ref1, ref2, ref3; + v16u8 src = { 0 }; + v16u8 ref = { 0 }; + v16u8 diff; + v8u16 sad = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LW4(src_ptr, src_stride, src0, src1, src2, src3); + src_ptr += (4 * src_stride); + LW4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); + ref_ptr += (4 * ref_stride); + + INSERT_W4_UB(src0, src1, src2, src3, src); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + + diff = __msa_asub_u_b(src, ref); + sad += __msa_hadd_u_h(diff, diff); + } + + return HADD_UH_U32(sad); +} + +static uint32_t sad_8width_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3, ref0, ref1, ref2, ref3; + v8u16 sad = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + LD_UB4(ref, ref_stride, ref0, ref1, ref2, ref3); + ref += (4 * ref_stride); + + PCKEV_D4_UB(src1, src0, src3, src2, ref1, ref0, ref3, ref2, src0, src1, + ref0, ref1); + sad += SAD_UB2_UH(src0, src1, ref0, ref1); + } + + return HADD_UH_U32(sad); +} + +static uint32_t sad_16width_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt; + v16u8 src0, src1, ref0, ref1; + v8u16 sad = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB2(src, src_stride, src0, src1); + src += (2 * src_stride); + LD_UB2(ref, ref_stride, ref0, ref1); + ref += (2 * ref_stride); + sad += SAD_UB2_UH(src0, src1, ref0, ref1); + + LD_UB2(src, src_stride, src0, src1); + src += (2 * src_stride); + LD_UB2(ref, ref_stride, ref0, ref1); + ref += (2 * ref_stride); + sad += SAD_UB2_UH(src0, src1, ref0, ref1); + } + + return HADD_UH_U32(sad); +} + +static uint32_t sad_32width_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt; + v16u8 src0, src1, ref0, ref1; + v8u16 sad = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB2(src, 16, src0, src1); + src += src_stride; + LD_UB2(ref, 16, ref0, ref1); + ref += ref_stride; + sad += SAD_UB2_UH(src0, src1, ref0, ref1); + + LD_UB2(src, 16, src0, src1); + src += src_stride; + LD_UB2(ref, 16, ref0, ref1); + ref += ref_stride; + sad += SAD_UB2_UH(src0, src1, ref0, ref1); + + LD_UB2(src, 16, src0, src1); + src += src_stride; + LD_UB2(ref, 16, ref0, ref1); + ref += ref_stride; + sad += SAD_UB2_UH(src0, src1, ref0, ref1); + + LD_UB2(src, 16, src0, src1); + src += src_stride; + LD_UB2(ref, 16, ref0, ref1); + ref += ref_stride; + sad += SAD_UB2_UH(src0, src1, ref0, ref1); + } + + return HADD_UH_U32(sad); +} + +static uint32_t sad_64width_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt; + uint32_t sad = 0; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + + for (ht_cnt = (height >> 1); ht_cnt--;) { + LD_UB4(src, 16, src0, src1, src2, src3); + src += src_stride; + LD_UB4(ref, 16, ref0, ref1, ref2, ref3); + ref += ref_stride; + sad0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad1 += SAD_UB2_UH(src2, src3, ref2, ref3); + + LD_UB4(src, 16, src0, src1, src2, src3); + src += src_stride; + LD_UB4(ref, 16, ref0, ref1, ref2, ref3); + ref += ref_stride; + sad0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad1 += SAD_UB2_UH(src2, src3, ref2, ref3); + } + + sad = HADD_UH_U32(sad0); + sad += HADD_UH_U32(sad1); + + return sad; +} + +static void sad_4width_x3_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, uint32_t *sad_array) { + int32_t ht_cnt; + uint32_t src0, src1, src2, src3; + v16u8 src = { 0 }; + v16u8 ref = { 0 }; + v16u8 ref0, ref1, ref2, ref3, diff; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v8u16 sad2 = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LW4(src_ptr, src_stride, src0, src1, src2, src3); + src_ptr += (4 * src_stride); + INSERT_W4_UB(src0, src1, src2, src3, src); + + LD_UB4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); + ref_ptr += (4 * ref_stride); + SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); + diff = __msa_asub_u_b(src, ref); + sad0 += __msa_hadd_u_h(diff, diff); + + SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); + SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); + SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); + diff = __msa_asub_u_b(src, ref); + sad1 += __msa_hadd_u_h(diff, diff); + + SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); + SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); + SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); + diff = __msa_asub_u_b(src, ref); + sad2 += __msa_hadd_u_h(diff, diff); + } + + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); +} + +static void sad_8width_x3_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, uint32_t *sad_array) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref00, ref11, ref22, ref33; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v8u16 sad2 = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + LD_UB4(ref, ref_stride, ref00, ref11, ref22, ref33); + ref += (4 * ref_stride); + PCKEV_D4_UB(src1, src0, src3, src2, ref11, ref00, ref33, ref22, src0, src1, + ref0, ref1); + sad0 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); + SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); + PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); + sad1 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); + SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); + PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); + sad2 += SAD_UB2_UH(src0, src1, ref0, ref1); + } + + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); +} + +static void sad_16width_x3_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, uint32_t *sad_array) { + int32_t ht_cnt; + v16u8 src, ref, ref0, ref1, diff; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v8u16 sad2 = { 0 }; + + for (ht_cnt = (height >> 1); ht_cnt--;) { + src = LD_UB(src_ptr); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + + diff = __msa_asub_u_b(src, ref0); + sad0 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1); + diff = __msa_asub_u_b(src, ref); + sad1 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2); + diff = __msa_asub_u_b(src, ref); + sad2 += __msa_hadd_u_h(diff, diff); + + src = LD_UB(src_ptr); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + + diff = __msa_asub_u_b(src, ref0); + sad0 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1); + diff = __msa_asub_u_b(src, ref); + sad1 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2); + diff = __msa_asub_u_b(src, ref); + sad2 += __msa_hadd_u_h(diff, diff); + } + + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); +} + +static void sad_32width_x3_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, uint32_t *sad_array) { + int32_t ht_cnt; + v16u8 src0, src1, ref0_0, ref0_1, ref0_2, ref0, ref1; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v8u16 sad2 = { 0 }; + + for (ht_cnt = height >> 1; ht_cnt--;) { + LD_UB2(src, 16, src0, src1); + src += src_stride; + LD_UB3(ref, 16, ref0_0, ref0_1, ref0_2); + ref += ref_stride; + + sad0 += SAD_UB2_UH(src0, src1, ref0_0, ref0_1); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 1); + sad1 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 2); + sad2 += SAD_UB2_UH(src0, src1, ref0, ref1); + + LD_UB2(src, 16, src0, src1); + src += src_stride; + LD_UB3(ref, 16, ref0_0, ref0_1, ref0_2); + ref += ref_stride; + + sad0 += SAD_UB2_UH(src0, src1, ref0_0, ref0_1); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 1); + sad1 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 2); + sad2 += SAD_UB2_UH(src0, src1, ref0, ref1); + } + + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); +} + +static void sad_64width_x3_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, uint32_t *sad_array) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0_0, ref0_1, ref0_2, ref0_3, ref0_4, ref0, ref1, ref2, ref3; + v8u16 sad0_0 = { 0 }; + v8u16 sad0_1 = { 0 }; + v8u16 sad1_0 = { 0 }; + v8u16 sad1_1 = { 0 }; + v8u16 sad2_0 = { 0 }; + v8u16 sad2_1 = { 0 }; + v4u32 sad; + + for (ht_cnt = height; ht_cnt--;) { + LD_UB4(src, 16, src0, src1, src2, src3); + src += src_stride; + LD_UB4(ref, 16, ref0_0, ref0_1, ref0_2, ref0_3); + ref0_4 = LD_UB(ref + 64); + ref += ref_stride; + + sad0_0 += SAD_UB2_UH(src0, src1, ref0_0, ref0_1); + sad0_1 += SAD_UB2_UH(src2, src3, ref0_2, ref0_3); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 1); + SLDI_B2_UB(ref0_3, ref0_4, ref0_2, ref0_3, ref2, ref3, 1); + sad1_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad1_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 2); + SLDI_B2_UB(ref0_3, ref0_4, ref0_2, ref0_3, ref2, ref3, 2); + sad2_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad2_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + } + + sad = __msa_hadd_u_w(sad0_0, sad0_0); + sad += __msa_hadd_u_w(sad0_1, sad0_1); + sad_array[0] = HADD_SW_S32((v4i32)sad); + + sad = __msa_hadd_u_w(sad1_0, sad1_0); + sad += __msa_hadd_u_w(sad1_1, sad1_1); + sad_array[1] = HADD_SW_S32((v4i32)sad); + + sad = __msa_hadd_u_w(sad2_0, sad2_0); + sad += __msa_hadd_u_w(sad2_1, sad2_1); + sad_array[2] = HADD_SW_S32((v4i32)sad); +} + +static void sad_4width_x8_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, uint32_t *sad_array) { + int32_t ht_cnt; + uint32_t src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3, diff; + v16u8 src = { 0 }; + v16u8 ref = { 0 }; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v8u16 sad2 = { 0 }; + v8u16 sad3 = { 0 }; + v8u16 sad4 = { 0 }; + v8u16 sad5 = { 0 }; + v8u16 sad6 = { 0 }; + v8u16 sad7 = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LW4(src_ptr, src_stride, src0, src1, src2, src3); + INSERT_W4_UB(src0, src1, src2, src3, src); + src_ptr += (4 * src_stride); + LD_UB4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); + ref_ptr += (4 * ref_stride); + + SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); + diff = __msa_asub_u_b(src, ref); + sad0 += __msa_hadd_u_h(diff, diff); + + SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); + SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); + SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); + diff = __msa_asub_u_b(src, ref); + sad1 += __msa_hadd_u_h(diff, diff); + + SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); + SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); + SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); + diff = __msa_asub_u_b(src, ref); + sad2 += __msa_hadd_u_h(diff, diff); + + SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); + SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); + SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); + diff = __msa_asub_u_b(src, ref); + sad3 += __msa_hadd_u_h(diff, diff); + + SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); + SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); + SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); + diff = __msa_asub_u_b(src, ref); + sad4 += __msa_hadd_u_h(diff, diff); + + SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); + SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); + SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); + diff = __msa_asub_u_b(src, ref); + sad5 += __msa_hadd_u_h(diff, diff); + + SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); + SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); + SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); + diff = __msa_asub_u_b(src, ref); + sad6 += __msa_hadd_u_h(diff, diff); + + SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); + SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); + SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); + diff = __msa_asub_u_b(src, ref); + sad7 += __msa_hadd_u_h(diff, diff); + } + + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); + sad_array[4] = HADD_UH_U32(sad4); + sad_array[5] = HADD_UH_U32(sad5); + sad_array[6] = HADD_UH_U32(sad6); + sad_array[7] = HADD_UH_U32(sad7); +} + +static void sad_8width_x8_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, uint32_t *sad_array) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref00, ref11, ref22, ref33; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v8u16 sad2 = { 0 }; + v8u16 sad3 = { 0 }; + v8u16 sad4 = { 0 }; + v8u16 sad5 = { 0 }; + v8u16 sad6 = { 0 }; + v8u16 sad7 = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + LD_UB4(ref, ref_stride, ref00, ref11, ref22, ref33); + ref += (4 * ref_stride); + PCKEV_D4_UB(src1, src0, src3, src2, ref11, ref00, ref33, ref22, src0, src1, + ref0, ref1); + sad0 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); + SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); + PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); + sad1 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); + SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); + PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); + sad2 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); + SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); + PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); + sad3 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); + SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); + PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); + sad4 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); + SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); + PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); + sad5 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); + SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); + PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); + sad6 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); + SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); + PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); + sad7 += SAD_UB2_UH(src0, src1, ref0, ref1); + } + + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); + sad_array[4] = HADD_UH_U32(sad4); + sad_array[5] = HADD_UH_U32(sad5); + sad_array[6] = HADD_UH_U32(sad6); + sad_array[7] = HADD_UH_U32(sad7); +} + +static void sad_16width_x8_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, uint32_t *sad_array) { + int32_t ht_cnt; + v16u8 src, ref0, ref1, ref; + v16u8 diff; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v8u16 sad2 = { 0 }; + v8u16 sad3 = { 0 }; + v8u16 sad4 = { 0 }; + v8u16 sad5 = { 0 }; + v8u16 sad6 = { 0 }; + v8u16 sad7 = { 0 }; + + for (ht_cnt = (height >> 1); ht_cnt--;) { + src = LD_UB(src_ptr); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + + diff = __msa_asub_u_b(src, ref0); + sad0 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1); + diff = __msa_asub_u_b(src, ref); + sad1 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2); + diff = __msa_asub_u_b(src, ref); + sad2 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 3); + diff = __msa_asub_u_b(src, ref); + sad3 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 4); + diff = __msa_asub_u_b(src, ref); + sad4 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 5); + diff = __msa_asub_u_b(src, ref); + sad5 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 6); + diff = __msa_asub_u_b(src, ref); + sad6 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 7); + diff = __msa_asub_u_b(src, ref); + sad7 += __msa_hadd_u_h(diff, diff); + + src = LD_UB(src_ptr); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + + diff = __msa_asub_u_b(src, ref0); + sad0 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1); + diff = __msa_asub_u_b(src, ref); + sad1 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2); + diff = __msa_asub_u_b(src, ref); + sad2 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 3); + diff = __msa_asub_u_b(src, ref); + sad3 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 4); + diff = __msa_asub_u_b(src, ref); + sad4 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 5); + diff = __msa_asub_u_b(src, ref); + sad5 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 6); + diff = __msa_asub_u_b(src, ref); + sad6 += __msa_hadd_u_h(diff, diff); + + ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 7); + diff = __msa_asub_u_b(src, ref); + sad7 += __msa_hadd_u_h(diff, diff); + } + + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); + sad_array[4] = HADD_UH_U32(sad4); + sad_array[5] = HADD_UH_U32(sad5); + sad_array[6] = HADD_UH_U32(sad6); + sad_array[7] = HADD_UH_U32(sad7); +} + +static void sad_32width_x8_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, uint32_t *sad_array) { + int32_t ht_cnt; + v16u8 src0, src1; + v16u8 ref0, ref1, ref0_0, ref0_1, ref0_2; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v8u16 sad2 = { 0 }; + v8u16 sad3 = { 0 }; + v8u16 sad4 = { 0 }; + v8u16 sad5 = { 0 }; + v8u16 sad6 = { 0 }; + v8u16 sad7 = { 0 }; + + for (ht_cnt = height; ht_cnt--;) { + LD_UB2(src, 16, src0, src1); + src += src_stride; + LD_UB3(ref, 16, ref0_0, ref0_1, ref0_2); + ref += ref_stride; + + sad0 += SAD_UB2_UH(src0, src1, ref0_0, ref0_1); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 1); + sad1 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 2); + sad2 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 3); + sad3 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 4); + sad4 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 5); + sad5 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 6); + sad6 += SAD_UB2_UH(src0, src1, ref0, ref1); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 7); + sad7 += SAD_UB2_UH(src0, src1, ref0, ref1); + } + + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); + sad_array[4] = HADD_UH_U32(sad4); + sad_array[5] = HADD_UH_U32(sad5); + sad_array[6] = HADD_UH_U32(sad6); + sad_array[7] = HADD_UH_U32(sad7); +} + +static void sad_64width_x8_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, uint32_t *sad_array) { + const uint8_t *src_dup, *ref_dup; + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0_0, ref0_1, ref0_2, ref0_3, ref0_4; + v16u8 ref0, ref1, ref2, ref3; + v8u16 sad0_0 = { 0 }; + v8u16 sad0_1 = { 0 }; + v8u16 sad1_0 = { 0 }; + v8u16 sad1_1 = { 0 }; + v8u16 sad2_0 = { 0 }; + v8u16 sad2_1 = { 0 }; + v8u16 sad3_0 = { 0 }; + v8u16 sad3_1 = { 0 }; + v4u32 sad; + + src_dup = src; + ref_dup = ref; + + for (ht_cnt = height; ht_cnt--;) { + LD_UB4(src, 16, src0, src1, src2, src3); + src += src_stride; + LD_UB5(ref, 16, ref0_0, ref0_1, ref0_2, ref0_3, ref0_4); + ref += ref_stride; + + sad0_0 += SAD_UB2_UH(src0, src1, ref0_0, ref0_1); + sad0_1 += SAD_UB2_UH(src2, src3, ref0_2, ref0_3); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 1); + SLDI_B2_UB(ref0_3, ref0_4, ref0_2, ref0_3, ref2, ref3, 1); + sad1_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad1_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 2); + SLDI_B2_UB(ref0_3, ref0_4, ref0_2, ref0_3, ref2, ref3, 2); + sad2_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad2_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 3); + SLDI_B2_UB(ref0_3, ref0_4, ref0_2, ref0_3, ref2, ref3, 3); + sad3_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad3_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + } + + sad = __msa_hadd_u_w(sad0_0, sad0_0); + sad += __msa_hadd_u_w(sad0_1, sad0_1); + sad_array[0] = HADD_SW_S32(sad); + + sad = __msa_hadd_u_w(sad1_0, sad1_0); + sad += __msa_hadd_u_w(sad1_1, sad1_1); + sad_array[1] = HADD_SW_S32(sad); + + sad = __msa_hadd_u_w(sad2_0, sad2_0); + sad += __msa_hadd_u_w(sad2_1, sad2_1); + sad_array[2] = HADD_SW_S32(sad); + + sad = __msa_hadd_u_w(sad3_0, sad3_0); + sad += __msa_hadd_u_w(sad3_1, sad3_1); + sad_array[3] = HADD_SW_S32(sad); + + sad0_0 = (v8u16)__msa_ldi_h(0); + sad0_1 = (v8u16)__msa_ldi_h(0); + sad1_0 = (v8u16)__msa_ldi_h(0); + sad1_1 = (v8u16)__msa_ldi_h(0); + sad2_0 = (v8u16)__msa_ldi_h(0); + sad2_1 = (v8u16)__msa_ldi_h(0); + sad3_0 = (v8u16)__msa_ldi_h(0); + sad3_1 = (v8u16)__msa_ldi_h(0); + + for (ht_cnt = 64; ht_cnt--;) { + LD_UB4(src_dup, 16, src0, src1, src2, src3); + src_dup += src_stride; + LD_UB5(ref_dup, 16, ref0_0, ref0_1, ref0_2, ref0_3, ref0_4); + ref_dup += ref_stride; + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 4); + SLDI_B2_UB(ref0_3, ref0_4, ref0_2, ref0_3, ref2, ref3, 4); + sad0_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad0_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 5); + SLDI_B2_UB(ref0_3, ref0_4, ref0_2, ref0_3, ref2, ref3, 5); + sad1_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad1_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 6); + SLDI_B2_UB(ref0_3, ref0_4, ref0_2, ref0_3, ref2, ref3, 6); + sad2_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad2_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + + SLDI_B2_UB(ref0_1, ref0_2, ref0_0, ref0_1, ref0, ref1, 7); + SLDI_B2_UB(ref0_3, ref0_4, ref0_2, ref0_3, ref2, ref3, 7); + sad3_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad3_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + } + + sad = __msa_hadd_u_w(sad0_0, sad0_0); + sad += __msa_hadd_u_w(sad0_1, sad0_1); + sad_array[4] = HADD_SW_S32(sad); + + sad = __msa_hadd_u_w(sad1_0, sad1_0); + sad += __msa_hadd_u_w(sad1_1, sad1_1); + sad_array[5] = HADD_SW_S32(sad); + + sad = __msa_hadd_u_w(sad2_0, sad2_0); + sad += __msa_hadd_u_w(sad2_1, sad2_1); + sad_array[6] = HADD_SW_S32(sad); + + sad = __msa_hadd_u_w(sad3_0, sad3_0); + sad += __msa_hadd_u_w(sad3_1, sad3_1); + sad_array[7] = HADD_SW_S32(sad); +} + +static void sad_4width_x4d_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + int32_t ht_cnt; + uint32_t src0, src1, src2, src3; + uint32_t ref0, ref1, ref2, ref3; + v16u8 src = { 0 }; + v16u8 ref = { 0 }; + v16u8 diff; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v8u16 sad2 = { 0 }; + v8u16 sad3 = { 0 }; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LW4(src_ptr, src_stride, src0, src1, src2, src3); + INSERT_W4_UB(src0, src1, src2, src3, src); + src_ptr += (4 * src_stride); + + LW4(ref0_ptr, ref_stride, ref0, ref1, ref2, ref3); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + ref0_ptr += (4 * ref_stride); + + diff = __msa_asub_u_b(src, ref); + sad0 += __msa_hadd_u_h(diff, diff); + + LW4(ref1_ptr, ref_stride, ref0, ref1, ref2, ref3); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + ref1_ptr += (4 * ref_stride); + + diff = __msa_asub_u_b(src, ref); + sad1 += __msa_hadd_u_h(diff, diff); + + LW4(ref2_ptr, ref_stride, ref0, ref1, ref2, ref3); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + ref2_ptr += (4 * ref_stride); + + diff = __msa_asub_u_b(src, ref); + sad2 += __msa_hadd_u_h(diff, diff); + + LW4(ref3_ptr, ref_stride, ref0, ref1, ref2, ref3); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + ref3_ptr += (4 * ref_stride); + + diff = __msa_asub_u_b(src, ref); + sad3 += __msa_hadd_u_h(diff, diff); + } + + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); +} + +static void sad_8width_x4d_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + int32_t ht_cnt; + const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7; + v16u8 ref8, ref9, ref10, ref11, ref12, ref13, ref14, ref15; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v8u16 sad2 = { 0 }; + v8u16 sad3 = { 0 }; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB4(src_ptr, src_stride, src0, src1, src2, src3); + src_ptr += (4 * src_stride); + LD_UB4(ref0_ptr, ref_stride, ref0, ref1, ref2, ref3); + ref0_ptr += (4 * ref_stride); + LD_UB4(ref1_ptr, ref_stride, ref4, ref5, ref6, ref7); + ref1_ptr += (4 * ref_stride); + LD_UB4(ref2_ptr, ref_stride, ref8, ref9, ref10, ref11); + ref2_ptr += (4 * ref_stride); + LD_UB4(ref3_ptr, ref_stride, ref12, ref13, ref14, ref15); + ref3_ptr += (4 * ref_stride); + + PCKEV_D2_UB(src1, src0, src3, src2, src0, src1); + PCKEV_D2_UB(ref1, ref0, ref3, ref2, ref0, ref1); + sad0 += SAD_UB2_UH(src0, src1, ref0, ref1); + + PCKEV_D2_UB(ref5, ref4, ref7, ref6, ref0, ref1); + sad1 += SAD_UB2_UH(src0, src1, ref0, ref1); + + PCKEV_D2_UB(ref9, ref8, ref11, ref10, ref0, ref1); + sad2 += SAD_UB2_UH(src0, src1, ref0, ref1); + + PCKEV_D2_UB(ref13, ref12, ref15, ref14, ref0, ref1); + sad3 += SAD_UB2_UH(src0, src1, ref0, ref1); + } + + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); +} + +static void sad_16width_x4d_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + int32_t ht_cnt; + const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + v16u8 src, ref0, ref1, ref2, ref3, diff; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v8u16 sad2 = { 0 }; + v8u16 sad3 = { 0 }; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (ht_cnt = (height >> 1); ht_cnt--;) { + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref0 = LD_UB(ref0_ptr); + ref0_ptr += ref_stride; + ref1 = LD_UB(ref1_ptr); + ref1_ptr += ref_stride; + ref2 = LD_UB(ref2_ptr); + ref2_ptr += ref_stride; + ref3 = LD_UB(ref3_ptr); + ref3_ptr += ref_stride; + + diff = __msa_asub_u_b(src, ref0); + sad0 += __msa_hadd_u_h(diff, diff); + diff = __msa_asub_u_b(src, ref1); + sad1 += __msa_hadd_u_h(diff, diff); + diff = __msa_asub_u_b(src, ref2); + sad2 += __msa_hadd_u_h(diff, diff); + diff = __msa_asub_u_b(src, ref3); + sad3 += __msa_hadd_u_h(diff, diff); + + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref0 = LD_UB(ref0_ptr); + ref0_ptr += ref_stride; + ref1 = LD_UB(ref1_ptr); + ref1_ptr += ref_stride; + ref2 = LD_UB(ref2_ptr); + ref2_ptr += ref_stride; + ref3 = LD_UB(ref3_ptr); + ref3_ptr += ref_stride; + + diff = __msa_asub_u_b(src, ref0); + sad0 += __msa_hadd_u_h(diff, diff); + diff = __msa_asub_u_b(src, ref1); + sad1 += __msa_hadd_u_h(diff, diff); + diff = __msa_asub_u_b(src, ref2); + sad2 += __msa_hadd_u_h(diff, diff); + diff = __msa_asub_u_b(src, ref3); + sad3 += __msa_hadd_u_h(diff, diff); + } + + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); +} + +static void sad_32width_x4d_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + int32_t ht_cnt; + v16u8 src0, src1, ref0, ref1; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v8u16 sad2 = { 0 }; + v8u16 sad3 = { 0 }; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (ht_cnt = height; ht_cnt--;) { + LD_UB2(src, 16, src0, src1); + src += src_stride; + + LD_UB2(ref0_ptr, 16, ref0, ref1); + ref0_ptr += ref_stride; + sad0 += SAD_UB2_UH(src0, src1, ref0, ref1); + + LD_UB2(ref1_ptr, 16, ref0, ref1); + ref1_ptr += ref_stride; + sad1 += SAD_UB2_UH(src0, src1, ref0, ref1); + + LD_UB2(ref2_ptr, 16, ref0, ref1); + ref2_ptr += ref_stride; + sad2 += SAD_UB2_UH(src0, src1, ref0, ref1); + + LD_UB2(ref3_ptr, 16, ref0, ref1); + ref3_ptr += ref_stride; + sad3 += SAD_UB2_UH(src0, src1, ref0, ref1); + } + + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); +} + +static void sad_64width_x4d_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3; + v8u16 sad0_0 = { 0 }; + v8u16 sad0_1 = { 0 }; + v8u16 sad1_0 = { 0 }; + v8u16 sad1_1 = { 0 }; + v8u16 sad2_0 = { 0 }; + v8u16 sad2_1 = { 0 }; + v8u16 sad3_0 = { 0 }; + v8u16 sad3_1 = { 0 }; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (ht_cnt = height; ht_cnt--;) { + LD_UB4(src, 16, src0, src1, src2, src3); + src += src_stride; + + LD_UB4(ref0_ptr, 16, ref0, ref1, ref2, ref3); + ref0_ptr += ref_stride; + sad0_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad0_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + + LD_UB4(ref1_ptr, 16, ref0, ref1, ref2, ref3); + ref1_ptr += ref_stride; + sad1_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad1_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + + LD_UB4(ref2_ptr, 16, ref0, ref1, ref2, ref3); + ref2_ptr += ref_stride; + sad2_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad2_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + + LD_UB4(ref3_ptr, 16, ref0, ref1, ref2, ref3); + ref3_ptr += ref_stride; + sad3_0 += SAD_UB2_UH(src0, src1, ref0, ref1); + sad3_1 += SAD_UB2_UH(src2, src3, ref2, ref3); + } + + sad_array[0] = HADD_UH_U32(sad0_0); + sad_array[0] += HADD_UH_U32(sad0_1); + sad_array[1] = HADD_UH_U32(sad1_0); + sad_array[1] += HADD_UH_U32(sad1_1); + sad_array[2] = HADD_UH_U32(sad2_0); + sad_array[2] += HADD_UH_U32(sad2_1); + sad_array[3] = HADD_UH_U32(sad3_0); + sad_array[3] += HADD_UH_U32(sad3_1); +} + +static uint32_t avgsad_4width_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, const uint8_t *sec_pred) { + int32_t ht_cnt; + uint32_t src0, src1, src2, src3, ref0, ref1, ref2, ref3; + v16u8 src = { 0 }; + v16u8 ref = { 0 }; + v16u8 diff, pred, comp; + v8u16 sad = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LW4(src_ptr, src_stride, src0, src1, src2, src3); + src_ptr += (4 * src_stride); + LW4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); + ref_ptr += (4 * ref_stride); + pred = LD_UB(sec_pred); + sec_pred += 16; + + INSERT_W4_UB(src0, src1, src2, src3, src); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + + comp = __msa_aver_u_b(pred, ref); + diff = __msa_asub_u_b(src, comp); + sad += __msa_hadd_u_h(diff, diff); + } + + return HADD_UH_U32(sad); +} + +static uint32_t avgsad_8width_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, const uint8_t *sec_pred) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3, ref0, ref1, ref2, ref3; + v16u8 diff0, diff1, pred0, pred1; + v8u16 sad = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + LD_UB4(ref, ref_stride, ref0, ref1, ref2, ref3); + ref += (4 * ref_stride); + LD_UB2(sec_pred, 16, pred0, pred1); + sec_pred += 32; + PCKEV_D4_UB(src1, src0, src3, src2, ref1, ref0, ref3, ref2, src0, src1, + ref0, ref1); + AVER_UB2_UB(pred0, ref0, pred1, ref1, diff0, diff1); + sad += SAD_UB2_UH(src0, src1, diff0, diff1); + } + + return HADD_UH_U32(sad); +} + +static uint32_t avgsad_16width_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, const uint8_t *sec_pred) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3, ref0, ref1, ref2, ref3; + v16u8 pred0, pred1, pred2, pred3, comp0, comp1; + v8u16 sad = { 0 }; + + for (ht_cnt = (height >> 3); ht_cnt--;) { + LD_UB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + LD_UB4(ref, ref_stride, ref0, ref1, ref2, ref3); + ref += (4 * ref_stride); + LD_UB4(sec_pred, 16, pred0, pred1, pred2, pred3); + sec_pred += (4 * 16); + AVER_UB2_UB(pred0, ref0, pred1, ref1, comp0, comp1); + sad += SAD_UB2_UH(src0, src1, comp0, comp1); + AVER_UB2_UB(pred2, ref2, pred3, ref3, comp0, comp1); + sad += SAD_UB2_UH(src2, src3, comp0, comp1); + + LD_UB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + LD_UB4(ref, ref_stride, ref0, ref1, ref2, ref3); + ref += (4 * ref_stride); + LD_UB4(sec_pred, 16, pred0, pred1, pred2, pred3); + sec_pred += (4 * 16); + AVER_UB2_UB(pred0, ref0, pred1, ref1, comp0, comp1); + sad += SAD_UB2_UH(src0, src1, comp0, comp1); + AVER_UB2_UB(pred2, ref2, pred3, ref3, comp0, comp1); + sad += SAD_UB2_UH(src2, src3, comp0, comp1); + } + + return HADD_UH_U32(sad); +} + +static uint32_t avgsad_32width_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, const uint8_t *sec_pred) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + v16u8 ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7; + v16u8 pred0, pred1, pred2, pred3, pred4, pred5, pred6, pred7; + v16u8 comp0, comp1; + v8u16 sad = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB4(src, src_stride, src0, src2, src4, src6); + LD_UB4(src + 16, src_stride, src1, src3, src5, src7); + src += (4 * src_stride); + + LD_UB4(ref, ref_stride, ref0, ref2, ref4, ref6); + LD_UB4(ref + 16, ref_stride, ref1, ref3, ref5, ref7); + ref += (4 * ref_stride); + + LD_UB4(sec_pred, 32, pred0, pred2, pred4, pred6); + LD_UB4(sec_pred + 16, 32, pred1, pred3, pred5, pred7); + sec_pred += (4 * 32); + + AVER_UB2_UB(pred0, ref0, pred1, ref1, comp0, comp1); + sad += SAD_UB2_UH(src0, src1, comp0, comp1); + AVER_UB2_UB(pred2, ref2, pred3, ref3, comp0, comp1); + sad += SAD_UB2_UH(src2, src3, comp0, comp1); + AVER_UB2_UB(pred4, ref4, pred5, ref5, comp0, comp1); + sad += SAD_UB2_UH(src4, src5, comp0, comp1); + AVER_UB2_UB(pred6, ref6, pred7, ref7, comp0, comp1); + sad += SAD_UB2_UH(src6, src7, comp0, comp1); + } + + return HADD_UH_U32(sad); +} + +static uint32_t avgsad_64width_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, const uint8_t *sec_pred) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3; + v16u8 comp0, comp1, comp2, comp3; + v16u8 pred0, pred1, pred2, pred3; + v8u16 sad0 = { 0 }; + v8u16 sad1 = { 0 }; + v4u32 sad; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB4(src, 16, src0, src1, src2, src3); + src += src_stride; + LD_UB4(ref, 16, ref0, ref1, ref2, ref3); + ref += ref_stride; + LD_UB4(sec_pred, 16, pred0, pred1, pred2, pred3); + sec_pred += 64; + AVER_UB4_UB(pred0, ref0, pred1, ref1, pred2, ref2, pred3, ref3, comp0, + comp1, comp2, comp3); + sad0 += SAD_UB2_UH(src0, src1, comp0, comp1); + sad1 += SAD_UB2_UH(src2, src3, comp2, comp3); + + LD_UB4(src, 16, src0, src1, src2, src3); + src += src_stride; + LD_UB4(ref, 16, ref0, ref1, ref2, ref3); + ref += ref_stride; + LD_UB4(sec_pred, 16, pred0, pred1, pred2, pred3); + sec_pred += 64; + AVER_UB4_UB(pred0, ref0, pred1, ref1, pred2, ref2, pred3, ref3, comp0, + comp1, comp2, comp3); + sad0 += SAD_UB2_UH(src0, src1, comp0, comp1); + sad1 += SAD_UB2_UH(src2, src3, comp2, comp3); + + LD_UB4(src, 16, src0, src1, src2, src3); + src += src_stride; + LD_UB4(ref, 16, ref0, ref1, ref2, ref3); + ref += ref_stride; + LD_UB4(sec_pred, 16, pred0, pred1, pred2, pred3); + sec_pred += 64; + AVER_UB4_UB(pred0, ref0, pred1, ref1, pred2, ref2, pred3, ref3, comp0, + comp1, comp2, comp3); + sad0 += SAD_UB2_UH(src0, src1, comp0, comp1); + sad1 += SAD_UB2_UH(src2, src3, comp2, comp3); + + LD_UB4(src, 16, src0, src1, src2, src3); + src += src_stride; + LD_UB4(ref, 16, ref0, ref1, ref2, ref3); + ref += ref_stride; + LD_UB4(sec_pred, 16, pred0, pred1, pred2, pred3); + sec_pred += 64; + AVER_UB4_UB(pred0, ref0, pred1, ref1, pred2, ref2, pred3, ref3, comp0, + comp1, comp2, comp3); + sad0 += SAD_UB2_UH(src0, src1, comp0, comp1); + sad1 += SAD_UB2_UH(src2, src3, comp2, comp3); + } + + sad = __msa_hadd_u_w(sad0, sad0); + sad += __msa_hadd_u_w(sad1, sad1); + + return HADD_SW_S32(sad); +} + +#define AOM_SAD_4xHEIGHT_MSA(height) \ + uint32_t aom_sad4x##height##_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride) { \ + return sad_4width_msa(src, src_stride, ref, ref_stride, height); \ + } + +#define AOM_SAD_8xHEIGHT_MSA(height) \ + uint32_t aom_sad8x##height##_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride) { \ + return sad_8width_msa(src, src_stride, ref, ref_stride, height); \ + } + +#define AOM_SAD_16xHEIGHT_MSA(height) \ + uint32_t aom_sad16x##height##_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride) { \ + return sad_16width_msa(src, src_stride, ref, ref_stride, height); \ + } + +#define AOM_SAD_32xHEIGHT_MSA(height) \ + uint32_t aom_sad32x##height##_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride) { \ + return sad_32width_msa(src, src_stride, ref, ref_stride, height); \ + } + +#define AOM_SAD_64xHEIGHT_MSA(height) \ + uint32_t aom_sad64x##height##_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride) { \ + return sad_64width_msa(src, src_stride, ref, ref_stride, height); \ + } + +#define AOM_SAD_4xHEIGHTx3_MSA(height) \ + void aom_sad4x##height##x3_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sads) { \ + sad_4width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \ + } + +#define AOM_SAD_8xHEIGHTx3_MSA(height) \ + void aom_sad8x##height##x3_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sads) { \ + sad_8width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \ + } + +#define AOM_SAD_16xHEIGHTx3_MSA(height) \ + void aom_sad16x##height##x3_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sads) { \ + sad_16width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \ + } + +#define AOM_SAD_32xHEIGHTx3_MSA(height) \ + void aom_sad32x##height##x3_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sads) { \ + sad_32width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \ + } + +#define AOM_SAD_64xHEIGHTx3_MSA(height) \ + void aom_sad64x##height##x3_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sads) { \ + sad_64width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \ + } + +#define AOM_SAD_4xHEIGHTx8_MSA(height) \ + void aom_sad4x##height##x8_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sads) { \ + sad_4width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \ + } + +#define AOM_SAD_8xHEIGHTx8_MSA(height) \ + void aom_sad8x##height##x8_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sads) { \ + sad_8width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \ + } + +#define AOM_SAD_16xHEIGHTx8_MSA(height) \ + void aom_sad16x##height##x8_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sads) { \ + sad_16width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \ + } + +#define AOM_SAD_32xHEIGHTx8_MSA(height) \ + void aom_sad32x##height##x8_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sads) { \ + sad_32width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \ + } + +#define AOM_SAD_64xHEIGHTx8_MSA(height) \ + void aom_sad64x##height##x8_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sads) { \ + sad_64width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \ + } + +#define AOM_SAD_4xHEIGHTx4D_MSA(height) \ + void aom_sad4x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[], \ + int32_t ref_stride, uint32_t *sads) { \ + sad_4width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define AOM_SAD_8xHEIGHTx4D_MSA(height) \ + void aom_sad8x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[], \ + int32_t ref_stride, uint32_t *sads) { \ + sad_8width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define AOM_SAD_16xHEIGHTx4D_MSA(height) \ + void aom_sad16x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[], \ + int32_t ref_stride, uint32_t *sads) { \ + sad_16width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define AOM_SAD_32xHEIGHTx4D_MSA(height) \ + void aom_sad32x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[], \ + int32_t ref_stride, uint32_t *sads) { \ + sad_32width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define AOM_SAD_64xHEIGHTx4D_MSA(height) \ + void aom_sad64x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[], \ + int32_t ref_stride, uint32_t *sads) { \ + sad_64width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define AOM_AVGSAD_4xHEIGHT_MSA(height) \ + uint32_t aom_sad4x##height##_avg_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride, \ + const uint8_t *second_pred) { \ + return avgsad_4width_msa(src, src_stride, ref, ref_stride, height, \ + second_pred); \ + } + +#define AOM_AVGSAD_8xHEIGHT_MSA(height) \ + uint32_t aom_sad8x##height##_avg_msa(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride, \ + const uint8_t *second_pred) { \ + return avgsad_8width_msa(src, src_stride, ref, ref_stride, height, \ + second_pred); \ + } + +#define AOM_AVGSAD_16xHEIGHT_MSA(height) \ + uint32_t aom_sad16x##height##_avg_msa( \ + const uint8_t *src, int32_t src_stride, const uint8_t *ref, \ + int32_t ref_stride, const uint8_t *second_pred) { \ + return avgsad_16width_msa(src, src_stride, ref, ref_stride, height, \ + second_pred); \ + } + +#define AOM_AVGSAD_32xHEIGHT_MSA(height) \ + uint32_t aom_sad32x##height##_avg_msa( \ + const uint8_t *src, int32_t src_stride, const uint8_t *ref, \ + int32_t ref_stride, const uint8_t *second_pred) { \ + return avgsad_32width_msa(src, src_stride, ref, ref_stride, height, \ + second_pred); \ + } + +#define AOM_AVGSAD_64xHEIGHT_MSA(height) \ + uint32_t aom_sad64x##height##_avg_msa( \ + const uint8_t *src, int32_t src_stride, const uint8_t *ref, \ + int32_t ref_stride, const uint8_t *second_pred) { \ + return avgsad_64width_msa(src, src_stride, ref, ref_stride, height, \ + second_pred); \ + } + +/* clang-format off */ +// 64x64 +AOM_SAD_64xHEIGHT_MSA(64) +AOM_SAD_64xHEIGHTx3_MSA(64) +AOM_SAD_64xHEIGHTx8_MSA(64) +AOM_SAD_64xHEIGHTx4D_MSA(64) +AOM_AVGSAD_64xHEIGHT_MSA(64) + +// 64x32 +AOM_SAD_64xHEIGHT_MSA(32) +AOM_SAD_64xHEIGHTx3_MSA(32) +AOM_SAD_64xHEIGHTx8_MSA(32) +AOM_SAD_64xHEIGHTx4D_MSA(32) +AOM_AVGSAD_64xHEIGHT_MSA(32) + +// 32x64 +AOM_SAD_32xHEIGHT_MSA(64) +AOM_SAD_32xHEIGHTx3_MSA(64) +AOM_SAD_32xHEIGHTx8_MSA(64) +AOM_SAD_32xHEIGHTx4D_MSA(64) +AOM_AVGSAD_32xHEIGHT_MSA(64) + +// 32x32 +AOM_SAD_32xHEIGHT_MSA(32) +AOM_SAD_32xHEIGHTx3_MSA(32) +AOM_SAD_32xHEIGHTx8_MSA(32) +AOM_SAD_32xHEIGHTx4D_MSA(32) +AOM_AVGSAD_32xHEIGHT_MSA(32) + +// 32x16 +AOM_SAD_32xHEIGHT_MSA(16) +AOM_SAD_32xHEIGHTx3_MSA(16) +AOM_SAD_32xHEIGHTx8_MSA(16) +AOM_SAD_32xHEIGHTx4D_MSA(16) +AOM_AVGSAD_32xHEIGHT_MSA(16) + +// 16x32 +AOM_SAD_16xHEIGHT_MSA(32) +AOM_SAD_16xHEIGHTx3_MSA(32) +AOM_SAD_16xHEIGHTx8_MSA(32) +AOM_SAD_16xHEIGHTx4D_MSA(32) +AOM_AVGSAD_16xHEIGHT_MSA(32) + +// 16x16 +AOM_SAD_16xHEIGHT_MSA(16) +AOM_SAD_16xHEIGHTx3_MSA(16) +AOM_SAD_16xHEIGHTx8_MSA(16) +AOM_SAD_16xHEIGHTx4D_MSA(16) +AOM_AVGSAD_16xHEIGHT_MSA(16) + +// 16x8 +AOM_SAD_16xHEIGHT_MSA(8) +AOM_SAD_16xHEIGHTx3_MSA(8) +AOM_SAD_16xHEIGHTx8_MSA(8) +AOM_SAD_16xHEIGHTx4D_MSA(8) +AOM_AVGSAD_16xHEIGHT_MSA(8) + +// 8x16 +AOM_SAD_8xHEIGHT_MSA(16) +AOM_SAD_8xHEIGHTx3_MSA(16) +AOM_SAD_8xHEIGHTx8_MSA(16) +AOM_SAD_8xHEIGHTx4D_MSA(16) +AOM_AVGSAD_8xHEIGHT_MSA(16) + +// 8x8 +AOM_SAD_8xHEIGHT_MSA(8) +AOM_SAD_8xHEIGHTx3_MSA(8) +AOM_SAD_8xHEIGHTx8_MSA(8) +AOM_SAD_8xHEIGHTx4D_MSA(8) +AOM_AVGSAD_8xHEIGHT_MSA(8) + +// 8x4 +AOM_SAD_8xHEIGHT_MSA(4) +AOM_SAD_8xHEIGHTx3_MSA(4) +AOM_SAD_8xHEIGHTx8_MSA(4) +AOM_SAD_8xHEIGHTx4D_MSA(4) +AOM_AVGSAD_8xHEIGHT_MSA(4) + +// 4x8 +AOM_SAD_4xHEIGHT_MSA(8) +AOM_SAD_4xHEIGHTx3_MSA(8) +AOM_SAD_4xHEIGHTx8_MSA(8) +AOM_SAD_4xHEIGHTx4D_MSA(8) +AOM_AVGSAD_4xHEIGHT_MSA(8) + +// 4x4 +AOM_SAD_4xHEIGHT_MSA(4) +AOM_SAD_4xHEIGHTx3_MSA(4) +AOM_SAD_4xHEIGHTx8_MSA(4) +AOM_SAD_4xHEIGHTx4D_MSA(4) +AOM_AVGSAD_4xHEIGHT_MSA(4) + /* clang-format on */ diff --git a/third_party/aom/aom_dsp/mips/sub_pixel_variance_msa.c b/third_party/aom/aom_dsp/mips/sub_pixel_variance_msa.c new file mode 100644 index 0000000000..3eb85107d1 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/sub_pixel_variance_msa.c @@ -0,0 +1,1795 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_dsp_rtcd.h" +#include "aom_ports/mem.h" +#include "aom_dsp/mips/macros_msa.h" +#include "aom_dsp/variance.h" + +static const uint8_t bilinear_filters_msa[8][2] = { + { 128, 0 }, { 112, 16 }, { 96, 32 }, { 80, 48 }, + { 64, 64 }, { 48, 80 }, { 32, 96 }, { 16, 112 }, +}; + +#define CALC_MSE_AVG_B(src, ref, var, sub) \ + { \ + v16u8 src_l0_m, src_l1_m; \ + v8i16 res_l0_m, res_l1_m; \ + \ + ILVRL_B2_UB(src, ref, src_l0_m, src_l1_m); \ + HSUB_UB2_SH(src_l0_m, src_l1_m, res_l0_m, res_l1_m); \ + DPADD_SH2_SW(res_l0_m, res_l1_m, res_l0_m, res_l1_m, var, var); \ + \ + sub += res_l0_m + res_l1_m; \ + } + +#define VARIANCE_WxH(sse, diff, shift) sse - (((uint32_t)diff * diff) >> shift) + +#define VARIANCE_LARGE_WxH(sse, diff, shift) \ + sse - (((int64_t)diff * diff) >> shift) + +static uint32_t avg_sse_diff_4width_msa(const uint8_t *src_ptr, + int32_t src_stride, + const uint8_t *ref_ptr, + int32_t ref_stride, + const uint8_t *sec_pred, int32_t height, + int32_t *diff) { + int32_t ht_cnt; + uint32_t src0, src1, src2, src3; + uint32_t ref0, ref1, ref2, ref3; + v16u8 pred, src = { 0 }; + v16u8 ref = { 0 }; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + pred = LD_UB(sec_pred); + sec_pred += 16; + LW4(src_ptr, src_stride, src0, src1, src2, src3); + src_ptr += (4 * src_stride); + LW4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); + ref_ptr += (4 * ref_stride); + + INSERT_W4_UB(src0, src1, src2, src3, src); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + + src = __msa_aver_u_b(src, pred); + CALC_MSE_AVG_B(src, ref, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t avg_sse_diff_8width_msa(const uint8_t *src_ptr, + int32_t src_stride, + const uint8_t *ref_ptr, + int32_t ref_stride, + const uint8_t *sec_pred, int32_t height, + int32_t *diff) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3; + v16u8 pred0, pred1; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB2(sec_pred, 16, pred0, pred1); + sec_pred += 32; + LD_UB4(src_ptr, src_stride, src0, src1, src2, src3); + src_ptr += (4 * src_stride); + LD_UB4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); + ref_ptr += (4 * ref_stride); + + PCKEV_D4_UB(src1, src0, src3, src2, ref1, ref0, ref3, ref2, src0, src1, + ref0, ref1); + AVER_UB2_UB(src0, pred0, src1, pred1, src0, src1); + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t avg_sse_diff_16width_msa(const uint8_t *src_ptr, + int32_t src_stride, + const uint8_t *ref_ptr, + int32_t ref_stride, + const uint8_t *sec_pred, + int32_t height, int32_t *diff) { + int32_t ht_cnt; + v16u8 src, ref, pred; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + pred = LD_UB(sec_pred); + sec_pred += 16; + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref = LD_UB(ref_ptr); + ref_ptr += ref_stride; + src = __msa_aver_u_b(src, pred); + CALC_MSE_AVG_B(src, ref, var, avg); + + pred = LD_UB(sec_pred); + sec_pred += 16; + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref = LD_UB(ref_ptr); + ref_ptr += ref_stride; + src = __msa_aver_u_b(src, pred); + CALC_MSE_AVG_B(src, ref, var, avg); + + pred = LD_UB(sec_pred); + sec_pred += 16; + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref = LD_UB(ref_ptr); + ref_ptr += ref_stride; + src = __msa_aver_u_b(src, pred); + CALC_MSE_AVG_B(src, ref, var, avg); + + pred = LD_UB(sec_pred); + sec_pred += 16; + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref = LD_UB(ref_ptr); + ref_ptr += ref_stride; + src = __msa_aver_u_b(src, pred); + CALC_MSE_AVG_B(src, ref, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t avg_sse_diff_32width_msa(const uint8_t *src_ptr, + int32_t src_stride, + const uint8_t *ref_ptr, + int32_t ref_stride, + const uint8_t *sec_pred, + int32_t height, int32_t *diff) { + int32_t ht_cnt; + v16u8 src0, src1, ref0, ref1, pred0, pred1; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB2(sec_pred, 16, pred0, pred1); + sec_pred += 32; + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + AVER_UB2_UB(src0, pred0, src1, pred1, src0, src1); + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + LD_UB2(sec_pred, 16, pred0, pred1); + sec_pred += 32; + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + AVER_UB2_UB(src0, pred0, src1, pred1, src0, src1); + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + LD_UB2(sec_pred, 16, pred0, pred1); + sec_pred += 32; + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + AVER_UB2_UB(src0, pred0, src1, pred1, src0, src1); + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + LD_UB2(sec_pred, 16, pred0, pred1); + sec_pred += 32; + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + AVER_UB2_UB(src0, pred0, src1, pred1, src0, src1); + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t avg_sse_diff_32x64_msa(const uint8_t *src_ptr, + int32_t src_stride, + const uint8_t *ref_ptr, + int32_t ref_stride, + const uint8_t *sec_pred, int32_t *diff) { + int32_t ht_cnt; + v16u8 src0, src1, ref0, ref1, pred0, pred1; + v8i16 avg0 = { 0 }; + v8i16 avg1 = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = 16; ht_cnt--;) { + LD_UB2(sec_pred, 16, pred0, pred1); + sec_pred += 32; + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + AVER_UB2_UB(src0, pred0, src1, pred1, src0, src1); + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + + LD_UB2(sec_pred, 16, pred0, pred1); + sec_pred += 32; + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + AVER_UB2_UB(src0, pred0, src1, pred1, src0, src1); + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + + LD_UB2(sec_pred, 16, pred0, pred1); + sec_pred += 32; + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + AVER_UB2_UB(src0, pred0, src1, pred1, src0, src1); + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + + LD_UB2(sec_pred, 16, pred0, pred1); + sec_pred += 32; + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + AVER_UB2_UB(src0, pred0, src1, pred1, src0, src1); + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + } + + vec = __msa_hadd_s_w(avg0, avg0); + vec += __msa_hadd_s_w(avg1, avg1); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t avg_sse_diff_64x32_msa(const uint8_t *src_ptr, + int32_t src_stride, + const uint8_t *ref_ptr, + int32_t ref_stride, + const uint8_t *sec_pred, int32_t *diff) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3; + v16u8 pred0, pred1, pred2, pred3; + v8i16 avg0 = { 0 }; + v8i16 avg1 = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = 16; ht_cnt--;) { + LD_UB4(sec_pred, 16, pred0, pred1, pred2, pred3); + sec_pred += 64; + LD_UB4(src_ptr, 16, src0, src1, src2, src3); + src_ptr += src_stride; + LD_UB4(ref_ptr, 16, ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + AVER_UB4_UB(src0, pred0, src1, pred1, src2, pred2, src3, pred3, src0, src1, + src2, src3); + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src2, ref2, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src3, ref3, var, avg1); + + LD_UB4(sec_pred, 16, pred0, pred1, pred2, pred3); + sec_pred += 64; + LD_UB4(src_ptr, 16, src0, src1, src2, src3); + src_ptr += src_stride; + LD_UB4(ref_ptr, 16, ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + AVER_UB4_UB(src0, pred0, src1, pred1, src2, pred2, src3, pred3, src0, src1, + src2, src3); + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src2, ref2, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src3, ref3, var, avg1); + } + + vec = __msa_hadd_s_w(avg0, avg0); + vec += __msa_hadd_s_w(avg1, avg1); + + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t avg_sse_diff_64x64_msa(const uint8_t *src_ptr, + int32_t src_stride, + const uint8_t *ref_ptr, + int32_t ref_stride, + const uint8_t *sec_pred, int32_t *diff) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3; + v16u8 pred0, pred1, pred2, pred3; + v8i16 avg0 = { 0 }; + v8i16 avg1 = { 0 }; + v8i16 avg2 = { 0 }; + v8i16 avg3 = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = 32; ht_cnt--;) { + LD_UB4(sec_pred, 16, pred0, pred1, pred2, pred3); + sec_pred += 64; + LD_UB4(src_ptr, 16, src0, src1, src2, src3); + src_ptr += src_stride; + LD_UB4(ref_ptr, 16, ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + AVER_UB4_UB(src0, pred0, src1, pred1, src2, pred2, src3, pred3, src0, src1, + src2, src3); + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src2, ref2, var, avg2); + CALC_MSE_AVG_B(src3, ref3, var, avg3); + + LD_UB4(sec_pred, 16, pred0, pred1, pred2, pred3); + sec_pred += 64; + LD_UB4(src_ptr, 16, src0, src1, src2, src3); + src_ptr += src_stride; + LD_UB4(ref_ptr, 16, ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + AVER_UB4_UB(src0, pred0, src1, pred1, src2, pred2, src3, pred3, src0, src1, + src2, src3); + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src2, ref2, var, avg2); + CALC_MSE_AVG_B(src3, ref3, var, avg3); + } + + vec = __msa_hadd_s_w(avg0, avg0); + vec += __msa_hadd_s_w(avg1, avg1); + vec += __msa_hadd_s_w(avg2, avg2); + vec += __msa_hadd_s_w(avg3, avg3); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_sse_diff_4width_h_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + uint32_t ref0, ref1, ref2, ref3; + v16u8 filt0, ref = { 0 }; + v16i8 src0, src1, src2, src3; + v16i8 mask = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; + v8u16 vec0, vec1, vec2, vec3; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter); + filt0 = (v16u8)__msa_fill_h(filtval); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + LW4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + PCKEV_B4_SB(vec0, vec0, vec1, vec1, vec2, vec2, vec3, vec3, src0, src1, + src2, src3); + ILVEV_W2_SB(src0, src1, src2, src3, src0, src2); + src0 = (v16i8)__msa_ilvev_d((v2i64)src2, (v2i64)src0); + CALC_MSE_AVG_B(src0, ref, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_sse_diff_8width_h_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + v16u8 filt0, out, ref0, ref1, ref2, ref3; + v16i8 src0, src1, src2, src3; + v16i8 mask = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; + v8u16 vec0, vec1, vec2, vec3; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter); + filt0 = (v16u8)__msa_fill_h(filtval); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + LD_UB4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + + PCKEV_D2_UB(ref1, ref0, ref3, ref2, ref0, ref1); + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + PCKEV_B4_SB(vec0, vec0, vec1, vec1, vec2, vec2, vec3, vec3, src0, src1, + src2, src3); + out = (v16u8)__msa_ilvev_d((v2i64)src1, (v2i64)src0); + CALC_MSE_AVG_B(out, ref0, var, avg); + out = (v16u8)__msa_ilvev_d((v2i64)src3, (v2i64)src2); + CALC_MSE_AVG_B(out, ref1, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_sse_diff_16width_h_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7; + v16i8 mask = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; + v16u8 dst0, dst1, dst2, dst3, filt0; + v8u16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8u16 out0, out1, out2, out3, out4, out5, out6, out7; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter); + filt0 = (v16u8)__msa_fill_h(filtval); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src0, src2, src4, src6); + LD_SB4(src + 8, src_stride, src1, src3, src5, src7); + src += (4 * src_stride); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + dst += (4 * dst_stride); + + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + VSHF_B2_UH(src4, src4, src5, src5, mask, mask, vec4, vec5); + VSHF_B2_UH(src6, src6, src7, src7, mask, mask, vec6, vec7); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, out0, out1, + out2, out3); + DOTP_UB4_UH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0, out4, out5, + out6, out7); + SRARI_H4_UH(out0, out1, out2, out3, FILTER_BITS); + SRARI_H4_UH(out4, out5, out6, out7, FILTER_BITS); + PCKEV_B4_SB(out1, out0, out3, out2, out5, out4, out7, out6, src0, src1, + src2, src3); + CALC_MSE_AVG_B(src0, dst0, var, avg); + CALC_MSE_AVG_B(src1, dst1, var, avg); + CALC_MSE_AVG_B(src2, dst2, var, avg); + CALC_MSE_AVG_B(src3, dst3, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_sse_diff_32width_h_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[2]; + + for (loop_cnt = 0; loop_cnt < 2; ++loop_cnt) { + sse += sub_pixel_sse_diff_16width_h_msa(src, src_stride, dst, dst_stride, + filter, height, &diff0[loop_cnt]); + src += 16; + dst += 16; + } + + *diff = diff0[0] + diff0[1]; + + return sse; +} + +static uint32_t sub_pixel_sse_diff_64width_h_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[4]; + + for (loop_cnt = 0; loop_cnt < 4; ++loop_cnt) { + sse += sub_pixel_sse_diff_16width_h_msa(src, src_stride, dst, dst_stride, + filter, height, &diff0[loop_cnt]); + src += 16; + dst += 16; + } + + *diff = diff0[0] + diff0[1] + diff0[2] + diff0[3]; + + return sse; +} + +static uint32_t sub_pixel_sse_diff_4width_v_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + uint32_t ref0, ref1, ref2, ref3; + v16u8 src0, src1, src2, src3, src4, out; + v16u8 src10_r, src32_r, src21_r, src43_r; + v16u8 ref = { 0 }; + v16u8 src2110, src4332; + v16u8 filt0; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + v8u16 tmp0, tmp1; + + filtval = LH(filter); + filt0 = (v16u8)__msa_fill_h(filtval); + + src0 = LD_UB(src); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + LW4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + ILVR_B4_UB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r, + src32_r, src43_r); + ILVR_D2_UB(src21_r, src10_r, src43_r, src32_r, src2110, src4332); + DOTP_UB2_UH(src2110, src4332, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + out = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + CALC_MSE_AVG_B(out, ref, var, avg); + src0 = src4; + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_sse_diff_8width_v_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + v16u8 src0, src1, src2, src3, src4; + v16u8 ref0, ref1, ref2, ref3; + v8u16 vec0, vec1, vec2, vec3; + v8u16 tmp0, tmp1, tmp2, tmp3; + v16u8 filt0; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter); + filt0 = (v16u8)__msa_fill_h(filtval); + + src0 = LD_UB(src); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + LD_UB4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + + PCKEV_D2_UB(ref1, ref0, ref3, ref2, ref0, ref1); + ILVR_B4_UH(src1, src0, src2, src1, src3, src2, src4, src3, vec0, vec1, vec2, + vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, tmp0, tmp1, + tmp2, tmp3); + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, src0, src1); + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + src0 = src4; + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_sse_diff_16width_v_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + v16u8 ref0, ref1, ref2, ref3; + v16u8 src0, src1, src2, src3, src4; + v16u8 out0, out1, out2, out3; + v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8u16 tmp0, tmp1, tmp2, tmp3; + v16u8 filt0; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter); + filt0 = (v16u8)__msa_fill_h(filtval); + + src0 = LD_UB(src); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + LD_UB4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + + ILVR_B2_UB(src1, src0, src2, src1, vec0, vec2); + ILVL_B2_UB(src1, src0, src2, src1, vec1, vec3); + DOTP_UB2_UH(vec0, vec1, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + out0 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + + ILVR_B2_UB(src3, src2, src4, src3, vec4, vec6); + ILVL_B2_UB(src3, src2, src4, src3, vec5, vec7); + DOTP_UB2_UH(vec2, vec3, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + out1 = (v16u8)__msa_pckev_b((v16i8)tmp3, (v16i8)tmp2); + + DOTP_UB2_UH(vec4, vec5, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + out2 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + DOTP_UB2_UH(vec6, vec7, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + out3 = (v16u8)__msa_pckev_b((v16i8)tmp3, (v16i8)tmp2); + + src0 = src4; + + CALC_MSE_AVG_B(out0, ref0, var, avg); + CALC_MSE_AVG_B(out1, ref1, var, avg); + CALC_MSE_AVG_B(out2, ref2, var, avg); + CALC_MSE_AVG_B(out3, ref3, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_sse_diff_32width_v_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[2]; + + for (loop_cnt = 0; loop_cnt < 2; ++loop_cnt) { + sse += sub_pixel_sse_diff_16width_v_msa(src, src_stride, dst, dst_stride, + filter, height, &diff0[loop_cnt]); + src += 16; + dst += 16; + } + + *diff = diff0[0] + diff0[1]; + + return sse; +} + +static uint32_t sub_pixel_sse_diff_64width_v_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[4]; + + for (loop_cnt = 0; loop_cnt < 4; ++loop_cnt) { + sse += sub_pixel_sse_diff_16width_v_msa(src, src_stride, dst, dst_stride, + filter, height, &diff0[loop_cnt]); + src += 16; + dst += 16; + } + + *diff = diff0[0] + diff0[1] + diff0[2] + diff0[3]; + + return sse; +} + +static uint32_t sub_pixel_sse_diff_4width_hv_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter_horiz, const uint8_t *filter_vert, + int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + uint32_t ref0, ref1, ref2, ref3; + v16u8 src0, src1, src2, src3, src4; + v16u8 out, ref = { 0 }; + v16u8 filt_vt, filt_hz, vec0, vec1; + v16u8 mask = { 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20 }; + v8u16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4; + v8u16 tmp0, tmp1; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter_horiz); + filt_hz = (v16u8)__msa_fill_h(filtval); + filtval = LH(filter_vert); + filt_vt = (v16u8)__msa_fill_h(filtval); + + src0 = LD_UB(src); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + LW4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out1 = (v8u16)__msa_sldi_b((v16i8)hz_out2, (v16i8)hz_out0, 8); + hz_out3 = (v8u16)__msa_pckod_d((v2i64)hz_out4, (v2i64)hz_out2); + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + out = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + CALC_MSE_AVG_B(out, ref, var, avg); + src0 = src4; + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_sse_diff_8width_hv_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter_horiz, const uint8_t *filter_vert, + int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + v16u8 ref0, ref1, ref2, ref3; + v16u8 src0, src1, src2, src3, src4; + v16u8 out0, out1; + v16u8 mask = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; + v8u16 hz_out0, hz_out1; + v8u16 tmp0, tmp1, tmp2, tmp3; + v16u8 filt_vt, filt_hz, vec0; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter_horiz); + filt_hz = (v16u8)__msa_fill_h(filtval); + filtval = LH(filter_vert); + filt_vt = (v16u8)__msa_fill_h(filtval); + + src0 = LD_UB(src); + src += src_stride; + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + LD_UB4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + + PCKEV_D2_UB(ref1, ref0, ref3, ref2, ref0, ref1); + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp0 = __msa_dotp_u_h(vec0, filt_vt); + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp1 = __msa_dotp_u_h(vec0, filt_vt); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp2 = __msa_dotp_u_h(vec0, filt_vt); + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp3 = __msa_dotp_u_h(vec0, filt_vt); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1); + CALC_MSE_AVG_B(out0, ref0, var, avg); + CALC_MSE_AVG_B(out1, ref1, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_sse_diff_16width_hv_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter_horiz, const uint8_t *filter_vert, + int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + v16u8 ref0, ref1, ref2, ref3; + v16u8 filt_hz, filt_vt, vec0, vec1; + v16u8 mask = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; + v8u16 hz_out0, hz_out1, hz_out2, hz_out3; + v8u16 tmp0, tmp1; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter_horiz); + filt_hz = (v16u8)__msa_fill_h(filtval); + filtval = LH(filter_vert); + filt_vt = (v16u8)__msa_fill_h(filtval); + + LD_UB2(src, 8, src0, src1); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src0, src2, src4, src6); + LD_UB4(src + 8, src_stride, src1, src3, src5, src7); + src += (4 * src_stride); + LD_UB4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + + hz_out1 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + src0 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + src1 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + + hz_out1 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + src2 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + + hz_out0 = HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + src3 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + CALC_MSE_AVG_B(src2, ref2, var, avg); + CALC_MSE_AVG_B(src3, ref3, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_sse_diff_32width_hv_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter_horiz, const uint8_t *filter_vert, + int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[2]; + + for (loop_cnt = 0; loop_cnt < 2; ++loop_cnt) { + sse += sub_pixel_sse_diff_16width_hv_msa(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height, + &diff0[loop_cnt]); + src += 16; + dst += 16; + } + + *diff = diff0[0] + diff0[1]; + + return sse; +} + +static uint32_t sub_pixel_sse_diff_64width_hv_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter_horiz, const uint8_t *filter_vert, + int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[4]; + + for (loop_cnt = 0; loop_cnt < 4; ++loop_cnt) { + sse += sub_pixel_sse_diff_16width_hv_msa(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height, + &diff0[loop_cnt]); + src += 16; + dst += 16; + } + + *diff = diff0[0] + diff0[1] + diff0[2] + diff0[3]; + + return sse; +} + +static uint32_t sub_pixel_avg_sse_diff_4width_h_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + uint32_t ref0, ref1, ref2, ref3; + v16u8 out, pred, filt0, ref = { 0 }; + v16i8 src0, src1, src2, src3; + v16i8 mask = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; + v8u16 vec0, vec1, vec2, vec3; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter); + filt0 = (v16u8)__msa_fill_h(filtval); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + pred = LD_UB(sec_pred); + sec_pred += 16; + LW4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + PCKEV_B4_SB(vec0, vec0, vec1, vec1, vec2, vec2, vec3, vec3, src0, src1, + src2, src3); + ILVEV_W2_SB(src0, src1, src2, src3, src0, src2); + out = (v16u8)__msa_ilvev_d((v2i64)src2, (v2i64)src0); + out = __msa_aver_u_b(out, pred); + CALC_MSE_AVG_B(out, ref, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_avg_sse_diff_8width_h_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + v16u8 out, pred, filt0; + v16u8 ref0, ref1, ref2, ref3; + v16i8 src0, src1, src2, src3; + v16i8 mask = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; + v8u16 vec0, vec1, vec2, vec3; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter); + filt0 = (v16u8)__msa_fill_h(filtval); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src0, src1, src2, src3); + src += (4 * src_stride); + LD_UB4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + + PCKEV_D2_UB(ref1, ref0, ref3, ref2, ref0, ref1); + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, vec0, vec1, + vec2, vec3); + SRARI_H4_UH(vec0, vec1, vec2, vec3, FILTER_BITS); + PCKEV_B4_SB(vec0, vec0, vec1, vec1, vec2, vec2, vec3, vec3, src0, src1, + src2, src3); + out = (v16u8)__msa_ilvev_d((v2i64)src1, (v2i64)src0); + + pred = LD_UB(sec_pred); + sec_pred += 16; + out = __msa_aver_u_b(out, pred); + CALC_MSE_AVG_B(out, ref0, var, avg); + out = (v16u8)__msa_ilvev_d((v2i64)src3, (v2i64)src2); + pred = LD_UB(sec_pred); + sec_pred += 16; + out = __msa_aver_u_b(out, pred); + CALC_MSE_AVG_B(out, ref1, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t subpel_avg_ssediff_16w_h_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff, int32_t width) { + int16_t filtval; + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7; + v16i8 mask = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; + v16u8 dst0, dst1, dst2, dst3; + v16u8 tmp0, tmp1, tmp2, tmp3; + v16u8 pred0, pred1, pred2, pred3, filt0; + v8u16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8u16 out0, out1, out2, out3, out4, out5, out6, out7; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter); + filt0 = (v16u8)__msa_fill_h(filtval); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_SB4(src, src_stride, src0, src2, src4, src6); + LD_SB4(src + 8, src_stride, src1, src3, src5, src7); + src += (4 * src_stride); + LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3); + dst += (4 * dst_stride); + LD_UB4(sec_pred, width, pred0, pred1, pred2, pred3); + sec_pred += (4 * width); + + VSHF_B2_UH(src0, src0, src1, src1, mask, mask, vec0, vec1); + VSHF_B2_UH(src2, src2, src3, src3, mask, mask, vec2, vec3); + VSHF_B2_UH(src4, src4, src5, src5, mask, mask, vec4, vec5); + VSHF_B2_UH(src6, src6, src7, src7, mask, mask, vec6, vec7); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, out0, out1, + out2, out3); + DOTP_UB4_UH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0, out4, out5, + out6, out7); + SRARI_H4_UH(out0, out1, out2, out3, FILTER_BITS); + SRARI_H4_UH(out4, out5, out6, out7, FILTER_BITS); + PCKEV_B4_UB(out1, out0, out3, out2, out5, out4, out7, out6, tmp0, tmp1, + tmp2, tmp3); + AVER_UB4_UB(tmp0, pred0, tmp1, pred1, tmp2, pred2, tmp3, pred3, tmp0, tmp1, + tmp2, tmp3); + + CALC_MSE_AVG_B(tmp0, dst0, var, avg); + CALC_MSE_AVG_B(tmp1, dst1, var, avg); + CALC_MSE_AVG_B(tmp2, dst2, var, avg); + CALC_MSE_AVG_B(tmp3, dst3, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_avg_sse_diff_16width_h_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff) { + return subpel_avg_ssediff_16w_h_msa(src, src_stride, dst, dst_stride, + sec_pred, filter, height, diff, 16); +} + +static uint32_t sub_pixel_avg_sse_diff_32width_h_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[2]; + + for (loop_cnt = 0; loop_cnt < 2; ++loop_cnt) { + sse += + subpel_avg_ssediff_16w_h_msa(src, src_stride, dst, dst_stride, sec_pred, + filter, height, &diff0[loop_cnt], 32); + src += 16; + dst += 16; + sec_pred += 16; + } + + *diff = diff0[0] + diff0[1]; + + return sse; +} + +static uint32_t sub_pixel_avg_sse_diff_64width_h_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[4]; + + for (loop_cnt = 0; loop_cnt < 4; ++loop_cnt) { + sse += + subpel_avg_ssediff_16w_h_msa(src, src_stride, dst, dst_stride, sec_pred, + filter, height, &diff0[loop_cnt], 64); + src += 16; + dst += 16; + sec_pred += 16; + } + + *diff = diff0[0] + diff0[1] + diff0[2] + diff0[3]; + + return sse; +} + +static uint32_t sub_pixel_avg_sse_diff_4width_v_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + uint32_t ref0, ref1, ref2, ref3; + v16u8 src0, src1, src2, src3, src4; + v16u8 src10_r, src32_r, src21_r, src43_r; + v16u8 out, pred, ref = { 0 }; + v16u8 src2110, src4332, filt0; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + v8u16 tmp0, tmp1; + + filtval = LH(filter); + filt0 = (v16u8)__msa_fill_h(filtval); + + src0 = LD_UB(src); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + pred = LD_UB(sec_pred); + sec_pred += 16; + LW4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + ILVR_B4_UB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r, + src32_r, src43_r); + ILVR_D2_UB(src21_r, src10_r, src43_r, src32_r, src2110, src4332); + DOTP_UB2_UH(src2110, src4332, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + + out = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + out = __msa_aver_u_b(out, pred); + CALC_MSE_AVG_B(out, ref, var, avg); + src0 = src4; + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_avg_sse_diff_8width_v_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + v16u8 src0, src1, src2, src3, src4; + v16u8 ref0, ref1, ref2, ref3; + v16u8 pred0, pred1, filt0; + v8u16 vec0, vec1, vec2, vec3; + v8u16 tmp0, tmp1, tmp2, tmp3; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter); + filt0 = (v16u8)__msa_fill_h(filtval); + + src0 = LD_UB(src); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + LD_UB2(sec_pred, 16, pred0, pred1); + sec_pred += 32; + LD_UB4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + PCKEV_D2_UB(ref1, ref0, ref3, ref2, ref0, ref1); + ILVR_B4_UH(src1, src0, src2, src1, src3, src2, src4, src3, vec0, vec1, vec2, + vec3); + DOTP_UB4_UH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, tmp0, tmp1, + tmp2, tmp3); + SRARI_H4_UH(tmp0, tmp1, tmp2, tmp3, FILTER_BITS); + PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, src0, src1); + AVER_UB2_UB(src0, pred0, src1, pred1, src0, src1); + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + src0 = src4; + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t subpel_avg_ssediff_16w_v_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff, int32_t width) { + int16_t filtval; + uint32_t loop_cnt; + v16u8 ref0, ref1, ref2, ref3; + v16u8 pred0, pred1, pred2, pred3; + v16u8 src0, src1, src2, src3, src4; + v16u8 out0, out1, out2, out3, filt0; + v8u16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + v8u16 tmp0, tmp1, tmp2, tmp3; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter); + filt0 = (v16u8)__msa_fill_h(filtval); + + src0 = LD_UB(src); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + LD_UB4(sec_pred, width, pred0, pred1, pred2, pred3); + sec_pred += (4 * width); + + ILVR_B2_UH(src1, src0, src2, src1, vec0, vec2); + ILVL_B2_UH(src1, src0, src2, src1, vec1, vec3); + DOTP_UB2_UH(vec0, vec1, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + out0 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + + ILVR_B2_UH(src3, src2, src4, src3, vec4, vec6); + ILVL_B2_UH(src3, src2, src4, src3, vec5, vec7); + DOTP_UB2_UH(vec2, vec3, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + out1 = (v16u8)__msa_pckev_b((v16i8)tmp3, (v16i8)tmp2); + + DOTP_UB2_UH(vec4, vec5, filt0, filt0, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + out2 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + + DOTP_UB2_UH(vec6, vec7, filt0, filt0, tmp2, tmp3); + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + out3 = (v16u8)__msa_pckev_b((v16i8)tmp3, (v16i8)tmp2); + + src0 = src4; + LD_UB4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + + AVER_UB4_UB(out0, pred0, out1, pred1, out2, pred2, out3, pred3, out0, out1, + out2, out3); + + CALC_MSE_AVG_B(out0, ref0, var, avg); + CALC_MSE_AVG_B(out1, ref1, var, avg); + CALC_MSE_AVG_B(out2, ref2, var, avg); + CALC_MSE_AVG_B(out3, ref3, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_avg_sse_diff_16width_v_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff) { + return subpel_avg_ssediff_16w_v_msa(src, src_stride, dst, dst_stride, + sec_pred, filter, height, diff, 16); +} + +static uint32_t sub_pixel_avg_sse_diff_32width_v_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[2]; + + for (loop_cnt = 0; loop_cnt < 2; ++loop_cnt) { + sse += + subpel_avg_ssediff_16w_v_msa(src, src_stride, dst, dst_stride, sec_pred, + filter, height, &diff0[loop_cnt], 32); + src += 16; + dst += 16; + sec_pred += 16; + } + + *diff = diff0[0] + diff0[1]; + + return sse; +} + +static uint32_t sub_pixel_avg_sse_diff_64width_v_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[4]; + + for (loop_cnt = 0; loop_cnt < 4; ++loop_cnt) { + sse += + subpel_avg_ssediff_16w_v_msa(src, src_stride, dst, dst_stride, sec_pred, + filter, height, &diff0[loop_cnt], 64); + src += 16; + dst += 16; + sec_pred += 16; + } + + *diff = diff0[0] + diff0[1] + diff0[2] + diff0[3]; + + return sse; +} + +static uint32_t sub_pixel_avg_sse_diff_4width_hv_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter_horiz, + const uint8_t *filter_vert, int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + uint32_t ref0, ref1, ref2, ref3; + v16u8 src0, src1, src2, src3, src4; + v16u8 mask = { 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20 }; + v16u8 filt_hz, filt_vt, vec0, vec1; + v16u8 out, pred, ref = { 0 }; + v8u16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, tmp0, tmp1; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter_horiz); + filt_hz = (v16u8)__msa_fill_h(filtval); + filtval = LH(filter_vert); + filt_vt = (v16u8)__msa_fill_h(filtval); + + src0 = LD_UB(src); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + pred = LD_UB(sec_pred); + sec_pred += 16; + LW4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out1 = (v8u16)__msa_sldi_b((v16i8)hz_out2, (v16i8)hz_out0, 8); + hz_out3 = (v8u16)__msa_pckod_d((v2i64)hz_out4, (v2i64)hz_out2); + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + out = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + out = __msa_aver_u_b(out, pred); + CALC_MSE_AVG_B(out, ref, var, avg); + src0 = src4; + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_avg_sse_diff_8width_hv_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter_horiz, + const uint8_t *filter_vert, int32_t height, int32_t *diff) { + int16_t filtval; + uint32_t loop_cnt; + v16u8 ref0, ref1, ref2, ref3; + v16u8 src0, src1, src2, src3, src4; + v16u8 pred0, pred1, out0, out1; + v16u8 filt_hz, filt_vt, vec0; + v16u8 mask = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; + v8u16 hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter_horiz); + filt_hz = (v16u8)__msa_fill_h(filtval); + filtval = LH(filter_vert); + filt_vt = (v16u8)__msa_fill_h(filtval); + + src0 = LD_UB(src); + src += src_stride; + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src1, src2, src3, src4); + src += (4 * src_stride); + LD_UB2(sec_pred, 16, pred0, pred1); + sec_pred += 32; + LD_UB4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + + PCKEV_D2_UB(ref1, ref0, ref3, ref2, ref0, ref1); + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp0 = __msa_dotp_u_h(vec0, filt_vt); + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp1 = __msa_dotp_u_h(vec0, filt_vt); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out1, (v16i8)hz_out0); + tmp2 = __msa_dotp_u_h(vec0, filt_vt); + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + + vec0 = (v16u8)__msa_ilvev_b((v16i8)hz_out0, (v16i8)hz_out1); + tmp3 = __msa_dotp_u_h(vec0, filt_vt); + + SRARI_H2_UH(tmp2, tmp3, FILTER_BITS); + PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1); + AVER_UB2_UB(out0, pred0, out1, pred1, out0, out1); + + CALC_MSE_AVG_B(out0, ref0, var, avg); + CALC_MSE_AVG_B(out1, ref1, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t subpel_avg_ssediff_16w_hv_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter_horiz, + const uint8_t *filter_vert, int32_t height, int32_t *diff, int32_t width) { + int16_t filtval; + uint32_t loop_cnt; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + v16u8 ref0, ref1, ref2, ref3; + v16u8 pred0, pred1, pred2, pred3; + v16u8 out0, out1, out2, out3; + v16u8 filt_hz, filt_vt, vec0, vec1; + v16u8 mask = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; + v8u16 hz_out0, hz_out1, hz_out2, hz_out3, tmp0, tmp1; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + filtval = LH(filter_horiz); + filt_hz = (v16u8)__msa_fill_h(filtval); + filtval = LH(filter_vert); + filt_vt = (v16u8)__msa_fill_h(filtval); + + LD_UB2(src, 8, src0, src1); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LD_UB4(src, src_stride, src0, src2, src4, src6); + LD_UB4(src + 8, src_stride, src1, src3, src5, src7); + src += (4 * src_stride); + LD_UB4(sec_pred, width, pred0, pred1, pred2, pred3); + sec_pred += (4 * width); + + hz_out1 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + out0 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + out1 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + + hz_out1 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + out2 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + + hz_out0 = HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS); + ILVEV_B2_UB(hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DOTP_UB2_UH(vec0, vec1, filt_vt, filt_vt, tmp0, tmp1); + SRARI_H2_UH(tmp0, tmp1, FILTER_BITS); + out3 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0); + + LD_UB4(dst, dst_stride, ref0, ref1, ref2, ref3); + dst += (4 * dst_stride); + + AVER_UB4_UB(out0, pred0, out1, pred1, out2, pred2, out3, pred3, out0, out1, + out2, out3); + + CALC_MSE_AVG_B(out0, ref0, var, avg); + CALC_MSE_AVG_B(out1, ref1, var, avg); + CALC_MSE_AVG_B(out2, ref2, var, avg); + CALC_MSE_AVG_B(out3, ref3, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sub_pixel_avg_sse_diff_16width_hv_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter_horiz, + const uint8_t *filter_vert, int32_t height, int32_t *diff) { + return subpel_avg_ssediff_16w_hv_msa(src, src_stride, dst, dst_stride, + sec_pred, filter_horiz, filter_vert, + height, diff, 16); +} + +static uint32_t sub_pixel_avg_sse_diff_32width_hv_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter_horiz, + const uint8_t *filter_vert, int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[2]; + + for (loop_cnt = 0; loop_cnt < 2; ++loop_cnt) { + sse += subpel_avg_ssediff_16w_hv_msa(src, src_stride, dst, dst_stride, + sec_pred, filter_horiz, filter_vert, + height, &diff0[loop_cnt], 32); + src += 16; + dst += 16; + sec_pred += 16; + } + + *diff = diff0[0] + diff0[1]; + + return sse; +} + +static uint32_t sub_pixel_avg_sse_diff_64width_hv_msa( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter_horiz, + const uint8_t *filter_vert, int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[4]; + + for (loop_cnt = 0; loop_cnt < 4; ++loop_cnt) { + sse += subpel_avg_ssediff_16w_hv_msa(src, src_stride, dst, dst_stride, + sec_pred, filter_horiz, filter_vert, + height, &diff0[loop_cnt], 64); + src += 16; + dst += 16; + sec_pred += 16; + } + + *diff = diff0[0] + diff0[1] + diff0[2] + diff0[3]; + + return sse; +} + +#define VARIANCE_4Wx4H(sse, diff) VARIANCE_WxH(sse, diff, 4); +#define VARIANCE_4Wx8H(sse, diff) VARIANCE_WxH(sse, diff, 5); +#define VARIANCE_8Wx4H(sse, diff) VARIANCE_WxH(sse, diff, 5); +#define VARIANCE_8Wx8H(sse, diff) VARIANCE_WxH(sse, diff, 6); +#define VARIANCE_8Wx16H(sse, diff) VARIANCE_WxH(sse, diff, 7); +#define VARIANCE_16Wx8H(sse, diff) VARIANCE_WxH(sse, diff, 7); +#define VARIANCE_16Wx16H(sse, diff) VARIANCE_WxH(sse, diff, 8); + +#define VARIANCE_16Wx32H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 9); +#define VARIANCE_32Wx16H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 9); +#define VARIANCE_32Wx32H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 10); +#define VARIANCE_32Wx64H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 11); +#define VARIANCE_64Wx32H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 11); +#define VARIANCE_64Wx64H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 12); + +#define AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(wd, ht) \ + uint32_t aom_sub_pixel_variance##wd##x##ht##_msa( \ + const uint8_t *src, int32_t src_stride, int32_t xoffset, \ + int32_t yoffset, const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sse) { \ + int32_t diff; \ + uint32_t var; \ + const uint8_t *h_filter = bilinear_filters_msa[xoffset]; \ + const uint8_t *v_filter = bilinear_filters_msa[yoffset]; \ + \ + if (yoffset) { \ + if (xoffset) { \ + *sse = sub_pixel_sse_diff_##wd##width_hv_msa( \ + src, src_stride, ref, ref_stride, h_filter, v_filter, ht, &diff); \ + } else { \ + *sse = sub_pixel_sse_diff_##wd##width_v_msa( \ + src, src_stride, ref, ref_stride, v_filter, ht, &diff); \ + } \ + \ + var = VARIANCE_##wd##Wx##ht##H(*sse, diff); \ + } else { \ + if (xoffset) { \ + *sse = sub_pixel_sse_diff_##wd##width_h_msa( \ + src, src_stride, ref, ref_stride, h_filter, ht, &diff); \ + \ + var = VARIANCE_##wd##Wx##ht##H(*sse, diff); \ + } else { \ + var = aom_variance##wd##x##ht##_msa(src, src_stride, ref, ref_stride, \ + sse); \ + } \ + } \ + \ + return var; \ + } + +/* clang-format off */ +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(4, 4) +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(4, 8) + +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(8, 4) +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(8, 8) +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(8, 16) + +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(16, 8) +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(16, 16) +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(16, 32) + +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(32, 16) +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(32, 32) +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(32, 64) + +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(64, 32) +AOM_SUB_PIXEL_VARIANCE_WDXHT_MSA(64, 64) +/* clang-format on */ + +#define AOM_SUB_PIXEL_AVG_VARIANCE_WDXHT_MSA(wd, ht) \ + uint32_t aom_sub_pixel_avg_variance##wd##x##ht##_msa( \ + const uint8_t *src_ptr, int32_t src_stride, int32_t xoffset, \ + int32_t yoffset, const uint8_t *ref_ptr, int32_t ref_stride, \ + uint32_t *sse, const uint8_t *sec_pred) { \ + int32_t diff; \ + const uint8_t *h_filter = bilinear_filters_msa[xoffset]; \ + const uint8_t *v_filter = bilinear_filters_msa[yoffset]; \ + \ + if (yoffset) { \ + if (xoffset) { \ + *sse = sub_pixel_avg_sse_diff_##wd##width_hv_msa( \ + src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, h_filter, \ + v_filter, ht, &diff); \ + } else { \ + *sse = sub_pixel_avg_sse_diff_##wd##width_v_msa( \ + src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, v_filter, ht, \ + &diff); \ + } \ + } else { \ + if (xoffset) { \ + *sse = sub_pixel_avg_sse_diff_##wd##width_h_msa( \ + src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, h_filter, ht, \ + &diff); \ + } else { \ + *sse = avg_sse_diff_##wd##width_msa(src_ptr, src_stride, ref_ptr, \ + ref_stride, sec_pred, ht, &diff); \ + } \ + } \ + \ + return VARIANCE_##wd##Wx##ht##H(*sse, diff); \ + } + +/* clang-format off */ +AOM_SUB_PIXEL_AVG_VARIANCE_WDXHT_MSA(4, 4) +AOM_SUB_PIXEL_AVG_VARIANCE_WDXHT_MSA(4, 8) + +AOM_SUB_PIXEL_AVG_VARIANCE_WDXHT_MSA(8, 4) +AOM_SUB_PIXEL_AVG_VARIANCE_WDXHT_MSA(8, 8) +AOM_SUB_PIXEL_AVG_VARIANCE_WDXHT_MSA(8, 16) + +AOM_SUB_PIXEL_AVG_VARIANCE_WDXHT_MSA(16, 8) +AOM_SUB_PIXEL_AVG_VARIANCE_WDXHT_MSA(16, 16) +AOM_SUB_PIXEL_AVG_VARIANCE_WDXHT_MSA(16, 32) + +AOM_SUB_PIXEL_AVG_VARIANCE_WDXHT_MSA(32, 16) +AOM_SUB_PIXEL_AVG_VARIANCE_WDXHT_MSA(32, 32) +/* clang-format on */ + +uint32_t aom_sub_pixel_avg_variance32x64_msa(const uint8_t *src_ptr, + int32_t src_stride, + int32_t xoffset, int32_t yoffset, + const uint8_t *ref_ptr, + int32_t ref_stride, uint32_t *sse, + const uint8_t *sec_pred) { + int32_t diff; + const uint8_t *h_filter = bilinear_filters_msa[xoffset]; + const uint8_t *v_filter = bilinear_filters_msa[yoffset]; + + if (yoffset) { + if (xoffset) { + *sse = sub_pixel_avg_sse_diff_32width_hv_msa( + src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, h_filter, + v_filter, 64, &diff); + } else { + *sse = sub_pixel_avg_sse_diff_32width_v_msa(src_ptr, src_stride, ref_ptr, + ref_stride, sec_pred, + v_filter, 64, &diff); + } + } else { + if (xoffset) { + *sse = sub_pixel_avg_sse_diff_32width_h_msa(src_ptr, src_stride, ref_ptr, + ref_stride, sec_pred, + h_filter, 64, &diff); + } else { + *sse = avg_sse_diff_32x64_msa(src_ptr, src_stride, ref_ptr, ref_stride, + sec_pred, &diff); + } + } + + return VARIANCE_32Wx64H(*sse, diff); +} + +#define AOM_SUB_PIXEL_AVG_VARIANCE64XHEIGHT_MSA(ht) \ + uint32_t aom_sub_pixel_avg_variance64x##ht##_msa( \ + const uint8_t *src_ptr, int32_t src_stride, int32_t xoffset, \ + int32_t yoffset, const uint8_t *ref_ptr, int32_t ref_stride, \ + uint32_t *sse, const uint8_t *sec_pred) { \ + int32_t diff; \ + const uint8_t *h_filter = bilinear_filters_msa[xoffset]; \ + const uint8_t *v_filter = bilinear_filters_msa[yoffset]; \ + \ + if (yoffset) { \ + if (xoffset) { \ + *sse = sub_pixel_avg_sse_diff_64width_hv_msa( \ + src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, h_filter, \ + v_filter, ht, &diff); \ + } else { \ + *sse = sub_pixel_avg_sse_diff_64width_v_msa( \ + src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, v_filter, ht, \ + &diff); \ + } \ + } else { \ + if (xoffset) { \ + *sse = sub_pixel_avg_sse_diff_64width_h_msa( \ + src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, h_filter, ht, \ + &diff); \ + } else { \ + *sse = avg_sse_diff_64x##ht##_msa(src_ptr, src_stride, ref_ptr, \ + ref_stride, sec_pred, &diff); \ + } \ + } \ + \ + return VARIANCE_64Wx##ht##H(*sse, diff); \ + } + +/* clang-format off */ +AOM_SUB_PIXEL_AVG_VARIANCE64XHEIGHT_MSA(32) +AOM_SUB_PIXEL_AVG_VARIANCE64XHEIGHT_MSA(64) +/* clang-format on */ diff --git a/third_party/aom/aom_dsp/mips/subtract_msa.c b/third_party/aom/aom_dsp/mips/subtract_msa.c new file mode 100644 index 0000000000..37b89765db --- /dev/null +++ b/third_party/aom/aom_dsp/mips/subtract_msa.c @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/macros_msa.h" + +static void sub_blk_4x4_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *pred_ptr, int32_t pred_stride, + int16_t *diff_ptr, int32_t diff_stride) { + uint32_t src0, src1, src2, src3; + uint32_t pred0, pred1, pred2, pred3; + v16i8 src = { 0 }; + v16i8 pred = { 0 }; + v16u8 src_l0, src_l1; + v8i16 diff0, diff1; + + LW4(src_ptr, src_stride, src0, src1, src2, src3); + LW4(pred_ptr, pred_stride, pred0, pred1, pred2, pred3); + INSERT_W4_SB(src0, src1, src2, src3, src); + INSERT_W4_SB(pred0, pred1, pred2, pred3, pred); + ILVRL_B2_UB(src, pred, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST8x4_UB(diff0, diff1, diff_ptr, (2 * diff_stride)); +} + +static void sub_blk_8x8_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *pred_ptr, int32_t pred_stride, + int16_t *diff_ptr, int32_t diff_stride) { + uint32_t loop_cnt; + uint64_t src0, src1, pred0, pred1; + v16i8 src = { 0 }; + v16i8 pred = { 0 }; + v16u8 src_l0, src_l1; + v8i16 diff0, diff1; + + for (loop_cnt = 4; loop_cnt--;) { + LD2(src_ptr, src_stride, src0, src1); + src_ptr += (2 * src_stride); + LD2(pred_ptr, pred_stride, pred0, pred1); + pred_ptr += (2 * pred_stride); + + INSERT_D2_SB(src0, src1, src); + INSERT_D2_SB(pred0, pred1, pred); + ILVRL_B2_UB(src, pred, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff_ptr, diff_stride); + diff_ptr += (2 * diff_stride); + } +} + +static void sub_blk_16x16_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *pred, int32_t pred_stride, + int16_t *diff, int32_t diff_stride) { + int8_t count; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7; + v16i8 pred0, pred1, pred2, pred3, pred4, pred5, pred6, pred7; + v16u8 src_l0, src_l1; + v8i16 diff0, diff1; + + for (count = 2; count--;) { + LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + src += (8 * src_stride); + + LD_SB8(pred, pred_stride, pred0, pred1, pred2, pred3, pred4, pred5, pred6, + pred7); + pred += (8 * pred_stride); + + ILVRL_B2_UB(src0, pred0, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + diff += diff_stride; + + ILVRL_B2_UB(src1, pred1, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + diff += diff_stride; + + ILVRL_B2_UB(src2, pred2, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + diff += diff_stride; + + ILVRL_B2_UB(src3, pred3, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + diff += diff_stride; + + ILVRL_B2_UB(src4, pred4, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + diff += diff_stride; + + ILVRL_B2_UB(src5, pred5, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + diff += diff_stride; + + ILVRL_B2_UB(src6, pred6, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + diff += diff_stride; + + ILVRL_B2_UB(src7, pred7, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + diff += diff_stride; + } +} + +static void sub_blk_32x32_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *pred, int32_t pred_stride, + int16_t *diff, int32_t diff_stride) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7; + v16i8 pred0, pred1, pred2, pred3, pred4, pred5, pred6, pred7; + v16u8 src_l0, src_l1; + v8i16 diff0, diff1; + + for (loop_cnt = 8; loop_cnt--;) { + LD_SB2(src, 16, src0, src1); + src += src_stride; + LD_SB2(src, 16, src2, src3); + src += src_stride; + LD_SB2(src, 16, src4, src5); + src += src_stride; + LD_SB2(src, 16, src6, src7); + src += src_stride; + + LD_SB2(pred, 16, pred0, pred1); + pred += pred_stride; + LD_SB2(pred, 16, pred2, pred3); + pred += pred_stride; + LD_SB2(pred, 16, pred4, pred5); + pred += pred_stride; + LD_SB2(pred, 16, pred6, pred7); + pred += pred_stride; + + ILVRL_B2_UB(src0, pred0, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + ILVRL_B2_UB(src1, pred1, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff + 16, 8); + diff += diff_stride; + + ILVRL_B2_UB(src2, pred2, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + ILVRL_B2_UB(src3, pred3, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff + 16, 8); + diff += diff_stride; + + ILVRL_B2_UB(src4, pred4, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + ILVRL_B2_UB(src5, pred5, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff + 16, 8); + diff += diff_stride; + + ILVRL_B2_UB(src6, pred6, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + ILVRL_B2_UB(src7, pred7, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff + 16, 8); + diff += diff_stride; + } +} + +static void sub_blk_64x64_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *pred, int32_t pred_stride, + int16_t *diff, int32_t diff_stride) { + uint32_t loop_cnt; + v16i8 src0, src1, src2, src3, src4, src5, src6, src7; + v16i8 pred0, pred1, pred2, pred3, pred4, pred5, pred6, pred7; + v16u8 src_l0, src_l1; + v8i16 diff0, diff1; + + for (loop_cnt = 32; loop_cnt--;) { + LD_SB4(src, 16, src0, src1, src2, src3); + src += src_stride; + LD_SB4(src, 16, src4, src5, src6, src7); + src += src_stride; + + LD_SB4(pred, 16, pred0, pred1, pred2, pred3); + pred += pred_stride; + LD_SB4(pred, 16, pred4, pred5, pred6, pred7); + pred += pred_stride; + + ILVRL_B2_UB(src0, pred0, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + ILVRL_B2_UB(src1, pred1, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff + 16, 8); + ILVRL_B2_UB(src2, pred2, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff + 32, 8); + ILVRL_B2_UB(src3, pred3, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff + 48, 8); + diff += diff_stride; + + ILVRL_B2_UB(src4, pred4, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff, 8); + ILVRL_B2_UB(src5, pred5, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff + 16, 8); + ILVRL_B2_UB(src6, pred6, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff + 32, 8); + ILVRL_B2_UB(src7, pred7, src_l0, src_l1); + HSUB_UB2_SH(src_l0, src_l1, diff0, diff1); + ST_SH2(diff0, diff1, diff + 48, 8); + diff += diff_stride; + } +} + +void aom_subtract_block_msa(int32_t rows, int32_t cols, int16_t *diff_ptr, + ptrdiff_t diff_stride, const uint8_t *src_ptr, + ptrdiff_t src_stride, const uint8_t *pred_ptr, + ptrdiff_t pred_stride) { + if (rows == cols) { + switch (rows) { + case 4: + sub_blk_4x4_msa(src_ptr, src_stride, pred_ptr, pred_stride, diff_ptr, + diff_stride); + break; + case 8: + sub_blk_8x8_msa(src_ptr, src_stride, pred_ptr, pred_stride, diff_ptr, + diff_stride); + break; + case 16: + sub_blk_16x16_msa(src_ptr, src_stride, pred_ptr, pred_stride, diff_ptr, + diff_stride); + break; + case 32: + sub_blk_32x32_msa(src_ptr, src_stride, pred_ptr, pred_stride, diff_ptr, + diff_stride); + break; + case 64: + sub_blk_64x64_msa(src_ptr, src_stride, pred_ptr, pred_stride, diff_ptr, + diff_stride); + break; + default: + aom_subtract_block_c(rows, cols, diff_ptr, diff_stride, src_ptr, + src_stride, pred_ptr, pred_stride); + break; + } + } else { + aom_subtract_block_c(rows, cols, diff_ptr, diff_stride, src_ptr, src_stride, + pred_ptr, pred_stride); + } +} diff --git a/third_party/aom/aom_dsp/mips/txfm_macros_msa.h b/third_party/aom/aom_dsp/mips/txfm_macros_msa.h new file mode 100644 index 0000000000..cba5d4445a --- /dev/null +++ b/third_party/aom/aom_dsp/mips/txfm_macros_msa.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_MIPS_TXFM_MACROS_MIPS_MSA_H_ +#define AOM_DSP_MIPS_TXFM_MACROS_MIPS_MSA_H_ + +#include "aom_dsp/mips/macros_msa.h" + +#define DOTP_CONST_PAIR(reg0, reg1, cnst0, cnst1, out0, out1) \ + { \ + v8i16 k0_m = __msa_fill_h(cnst0); \ + v4i32 s0_m, s1_m, s2_m, s3_m; \ + \ + s0_m = (v4i32)__msa_fill_h(cnst1); \ + k0_m = __msa_ilvev_h((v8i16)s0_m, k0_m); \ + \ + ILVRL_H2_SW((-reg1), reg0, s1_m, s0_m); \ + ILVRL_H2_SW(reg0, reg1, s3_m, s2_m); \ + DOTP_SH2_SW(s1_m, s0_m, k0_m, k0_m, s1_m, s0_m); \ + SRARI_W2_SW(s1_m, s0_m, DCT_CONST_BITS); \ + out0 = __msa_pckev_h((v8i16)s0_m, (v8i16)s1_m); \ + \ + DOTP_SH2_SW(s3_m, s2_m, k0_m, k0_m, s1_m, s0_m); \ + SRARI_W2_SW(s1_m, s0_m, DCT_CONST_BITS); \ + out1 = __msa_pckev_h((v8i16)s0_m, (v8i16)s1_m); \ + } + +#define DOT_ADD_SUB_SRARI_PCK(in0, in1, in2, in3, in4, in5, in6, in7, dst0, \ + dst1, dst2, dst3) \ + { \ + v4i32 tp0_m, tp1_m, tp2_m, tp3_m, tp4_m; \ + v4i32 tp5_m, tp6_m, tp7_m, tp8_m, tp9_m; \ + \ + DOTP_SH4_SW(in0, in1, in0, in1, in4, in4, in5, in5, tp0_m, tp2_m, tp3_m, \ + tp4_m); \ + DOTP_SH4_SW(in2, in3, in2, in3, in6, in6, in7, in7, tp5_m, tp6_m, tp7_m, \ + tp8_m); \ + BUTTERFLY_4(tp0_m, tp3_m, tp7_m, tp5_m, tp1_m, tp9_m, tp7_m, tp5_m); \ + BUTTERFLY_4(tp2_m, tp4_m, tp8_m, tp6_m, tp3_m, tp0_m, tp4_m, tp2_m); \ + SRARI_W4_SW(tp1_m, tp9_m, tp7_m, tp5_m, DCT_CONST_BITS); \ + SRARI_W4_SW(tp3_m, tp0_m, tp4_m, tp2_m, DCT_CONST_BITS); \ + PCKEV_H4_SH(tp1_m, tp3_m, tp9_m, tp0_m, tp7_m, tp4_m, tp5_m, tp2_m, dst0, \ + dst1, dst2, dst3); \ + } + +#define DOT_SHIFT_RIGHT_PCK_H(in0, in1, in2) \ + ({ \ + v8i16 dst_m; \ + v4i32 tp0_m, tp1_m; \ + \ + DOTP_SH2_SW(in0, in1, in2, in2, tp1_m, tp0_m); \ + SRARI_W2_SW(tp1_m, tp0_m, DCT_CONST_BITS); \ + dst_m = __msa_pckev_h((v8i16)tp1_m, (v8i16)tp0_m); \ + \ + dst_m; \ + }) + +#define MADD_SHORT(m0, m1, c0, c1, res0, res1) \ + { \ + v4i32 madd0_m, madd1_m, madd2_m, madd3_m; \ + v8i16 madd_s0_m, madd_s1_m; \ + \ + ILVRL_H2_SH(m1, m0, madd_s0_m, madd_s1_m); \ + DOTP_SH4_SW(madd_s0_m, madd_s1_m, madd_s0_m, madd_s1_m, c0, c0, c1, c1, \ + madd0_m, madd1_m, madd2_m, madd3_m); \ + SRARI_W4_SW(madd0_m, madd1_m, madd2_m, madd3_m, DCT_CONST_BITS); \ + PCKEV_H2_SH(madd1_m, madd0_m, madd3_m, madd2_m, res0, res1); \ + } + +#define MADD_BF(inp0, inp1, inp2, inp3, cst0, cst1, cst2, cst3, out0, out1, \ + out2, out3) \ + { \ + v8i16 madd_s0_m, madd_s1_m, madd_s2_m, madd_s3_m; \ + v4i32 tmp0_m, tmp1_m, tmp2_m, tmp3_m, m4_m, m5_m; \ + \ + ILVRL_H2_SH(inp1, inp0, madd_s0_m, madd_s1_m); \ + ILVRL_H2_SH(inp3, inp2, madd_s2_m, madd_s3_m); \ + DOTP_SH4_SW(madd_s0_m, madd_s1_m, madd_s2_m, madd_s3_m, cst0, cst0, cst2, \ + cst2, tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ + BUTTERFLY_4(tmp0_m, tmp1_m, tmp3_m, tmp2_m, m4_m, m5_m, tmp3_m, tmp2_m); \ + SRARI_W4_SW(m4_m, m5_m, tmp2_m, tmp3_m, DCT_CONST_BITS); \ + PCKEV_H2_SH(m5_m, m4_m, tmp3_m, tmp2_m, out0, out1); \ + DOTP_SH4_SW(madd_s0_m, madd_s1_m, madd_s2_m, madd_s3_m, cst1, cst1, cst3, \ + cst3, tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ + BUTTERFLY_4(tmp0_m, tmp1_m, tmp3_m, tmp2_m, m4_m, m5_m, tmp3_m, tmp2_m); \ + SRARI_W4_SW(m4_m, m5_m, tmp2_m, tmp3_m, DCT_CONST_BITS); \ + PCKEV_H2_SH(m5_m, m4_m, tmp3_m, tmp2_m, out2, out3); \ + } +#endif // AOM_DSP_MIPS_TXFM_MACROS_MIPS_MSA_H_ diff --git a/third_party/aom/aom_dsp/mips/variance_msa.c b/third_party/aom/aom_dsp/mips/variance_msa.c new file mode 100644 index 0000000000..745fdfc9c5 --- /dev/null +++ b/third_party/aom/aom_dsp/mips/variance_msa.c @@ -0,0 +1,632 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/mips/macros_msa.h" + +#define CALC_MSE_B(src, ref, var) \ + { \ + v16u8 src_l0_m, src_l1_m; \ + v8i16 res_l0_m, res_l1_m; \ + \ + ILVRL_B2_UB(src, ref, src_l0_m, src_l1_m); \ + HSUB_UB2_SH(src_l0_m, src_l1_m, res_l0_m, res_l1_m); \ + DPADD_SH2_SW(res_l0_m, res_l1_m, res_l0_m, res_l1_m, var, var); \ + } + +#define CALC_MSE_AVG_B(src, ref, var, sub) \ + { \ + v16u8 src_l0_m, src_l1_m; \ + v8i16 res_l0_m, res_l1_m; \ + \ + ILVRL_B2_UB(src, ref, src_l0_m, src_l1_m); \ + HSUB_UB2_SH(src_l0_m, src_l1_m, res_l0_m, res_l1_m); \ + DPADD_SH2_SW(res_l0_m, res_l1_m, res_l0_m, res_l1_m, var, var); \ + \ + sub += res_l0_m + res_l1_m; \ + } + +#define VARIANCE_WxH(sse, diff, shift) sse - (((uint32_t)diff * diff) >> shift) + +#define VARIANCE_LARGE_WxH(sse, diff, shift) \ + sse - (((int64_t)diff * diff) >> shift) + +static uint32_t sse_diff_4width_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, int32_t *diff) { + uint32_t src0, src1, src2, src3; + uint32_t ref0, ref1, ref2, ref3; + int32_t ht_cnt; + v16u8 src = { 0 }; + v16u8 ref = { 0 }; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LW4(src_ptr, src_stride, src0, src1, src2, src3); + src_ptr += (4 * src_stride); + LW4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); + ref_ptr += (4 * ref_stride); + + INSERT_W4_UB(src0, src1, src2, src3, src); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + CALC_MSE_AVG_B(src, ref, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sse_diff_8width_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, int32_t *diff) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB4(src_ptr, src_stride, src0, src1, src2, src3); + src_ptr += (4 * src_stride); + LD_UB4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); + ref_ptr += (4 * ref_stride); + + PCKEV_D4_UB(src1, src0, src3, src2, ref1, ref0, ref3, ref2, src0, src1, + ref0, ref1); + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sse_diff_16width_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, int32_t *diff) { + int32_t ht_cnt; + v16u8 src, ref; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref = LD_UB(ref_ptr); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src, ref, var, avg); + + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref = LD_UB(ref_ptr); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src, ref, var, avg); + + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref = LD_UB(ref_ptr); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src, ref, var, avg); + + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref = LD_UB(ref_ptr); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src, ref, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sse_diff_32width_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, int32_t *diff) { + int32_t ht_cnt; + v16u8 src0, src1, ref0, ref1; + v8i16 avg = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + } + + vec = __msa_hadd_s_w(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sse_diff_32x64_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t *diff) { + int32_t ht_cnt; + v16u8 src0, src1, ref0, ref1; + v8i16 avg0 = { 0 }; + v8i16 avg1 = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = 16; ht_cnt--;) { + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + } + + vec = __msa_hadd_s_w(avg0, avg0); + vec += __msa_hadd_s_w(avg1, avg1); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sse_diff_64x32_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t *diff) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3; + v8i16 avg0 = { 0 }; + v8i16 avg1 = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = 16; ht_cnt--;) { + LD_UB4(src_ptr, 16, src0, src1, src2, src3); + src_ptr += src_stride; + LD_UB4(ref_ptr, 16, ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src2, ref2, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src3, ref3, var, avg1); + + LD_UB4(src_ptr, 16, src0, src1, src2, src3); + src_ptr += src_stride; + LD_UB4(ref_ptr, 16, ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src2, ref2, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src3, ref3, var, avg1); + } + + vec = __msa_hadd_s_w(avg0, avg0); + vec += __msa_hadd_s_w(avg1, avg1); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sse_diff_64x64_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t *diff) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3; + v8i16 avg0 = { 0 }; + v8i16 avg1 = { 0 }; + v8i16 avg2 = { 0 }; + v8i16 avg3 = { 0 }; + v4i32 vec, var = { 0 }; + + for (ht_cnt = 32; ht_cnt--;) { + LD_UB4(src_ptr, 16, src0, src1, src2, src3); + src_ptr += src_stride; + LD_UB4(ref_ptr, 16, ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src2, ref2, var, avg2); + CALC_MSE_AVG_B(src3, ref3, var, avg3); + LD_UB4(src_ptr, 16, src0, src1, src2, src3); + src_ptr += src_stride; + LD_UB4(ref_ptr, 16, ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src2, ref2, var, avg2); + CALC_MSE_AVG_B(src3, ref3, var, avg3); + } + + vec = __msa_hadd_s_w(avg0, avg0); + vec += __msa_hadd_s_w(avg1, avg1); + vec += __msa_hadd_s_w(avg2, avg2); + vec += __msa_hadd_s_w(avg3, avg3); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t get_mb_ss_msa(const int16_t *src) { + uint32_t sum, cnt; + v8i16 src0, src1, src2, src3; + v4i32 src0_l, src1_l, src2_l, src3_l; + v4i32 src0_r, src1_r, src2_r, src3_r; + v2i64 sq_src_l = { 0 }; + v2i64 sq_src_r = { 0 }; + + for (cnt = 8; cnt--;) { + LD_SH4(src, 8, src0, src1, src2, src3); + src += 4 * 8; + + UNPCK_SH_SW(src0, src0_l, src0_r); + UNPCK_SH_SW(src1, src1_l, src1_r); + UNPCK_SH_SW(src2, src2_l, src2_r); + UNPCK_SH_SW(src3, src3_l, src3_r); + + DPADD_SD2_SD(src0_l, src0_r, sq_src_l, sq_src_r); + DPADD_SD2_SD(src1_l, src1_r, sq_src_l, sq_src_r); + DPADD_SD2_SD(src2_l, src2_r, sq_src_l, sq_src_r); + DPADD_SD2_SD(src3_l, src3_r, sq_src_l, sq_src_r); + } + + sq_src_l += __msa_splati_d(sq_src_l, 1); + sq_src_r += __msa_splati_d(sq_src_r, 1); + + sum = __msa_copy_s_d(sq_src_l, 0); + sum += __msa_copy_s_d(sq_src_r, 0); + + return sum; +} + +static uint32_t sse_4width_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt; + uint32_t src0, src1, src2, src3; + uint32_t ref0, ref1, ref2, ref3; + v16u8 src = { 0 }; + v16u8 ref = { 0 }; + v4i32 var = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LW4(src_ptr, src_stride, src0, src1, src2, src3); + src_ptr += (4 * src_stride); + LW4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); + ref_ptr += (4 * ref_stride); + + INSERT_W4_UB(src0, src1, src2, src3, src); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + CALC_MSE_B(src, ref, var); + } + + return HADD_SW_S32(var); +} + +static uint32_t sse_8width_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3; + v4i32 var = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB4(src_ptr, src_stride, src0, src1, src2, src3); + src_ptr += (4 * src_stride); + LD_UB4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); + ref_ptr += (4 * ref_stride); + + PCKEV_D4_UB(src1, src0, src3, src2, ref1, ref0, ref3, ref2, src0, src1, + ref0, ref1); + CALC_MSE_B(src0, ref0, var); + CALC_MSE_B(src1, ref1, var); + } + + return HADD_SW_S32(var); +} + +static uint32_t sse_16width_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt; + v16u8 src, ref; + v4i32 var = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref = LD_UB(ref_ptr); + ref_ptr += ref_stride; + CALC_MSE_B(src, ref, var); + + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref = LD_UB(ref_ptr); + ref_ptr += ref_stride; + CALC_MSE_B(src, ref, var); + + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref = LD_UB(ref_ptr); + ref_ptr += ref_stride; + CALC_MSE_B(src, ref, var); + + src = LD_UB(src_ptr); + src_ptr += src_stride; + ref = LD_UB(ref_ptr); + ref_ptr += ref_stride; + CALC_MSE_B(src, ref, var); + } + + return HADD_SW_S32(var); +} + +static uint32_t sse_32width_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt; + v16u8 src0, src1, ref0, ref1; + v4i32 var = { 0 }; + + for (ht_cnt = (height >> 2); ht_cnt--;) { + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_B(src0, ref0, var); + CALC_MSE_B(src1, ref1, var); + + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_B(src0, ref0, var); + CALC_MSE_B(src1, ref1, var); + + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_B(src0, ref0, var); + CALC_MSE_B(src1, ref1, var); + + LD_UB2(src_ptr, 16, src0, src1); + src_ptr += src_stride; + LD_UB2(ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_B(src0, ref0, var); + CALC_MSE_B(src1, ref1, var); + } + + return HADD_SW_S32(var); +} + +static uint32_t sse_64width_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt; + v16u8 src0, src1, src2, src3; + v16u8 ref0, ref1, ref2, ref3; + v4i32 var = { 0 }; + + for (ht_cnt = height >> 1; ht_cnt--;) { + LD_UB4(src_ptr, 16, src0, src1, src2, src3); + src_ptr += src_stride; + LD_UB4(ref_ptr, 16, ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + CALC_MSE_B(src0, ref0, var); + CALC_MSE_B(src2, ref2, var); + CALC_MSE_B(src1, ref1, var); + CALC_MSE_B(src3, ref3, var); + + LD_UB4(src_ptr, 16, src0, src1, src2, src3); + src_ptr += src_stride; + LD_UB4(ref_ptr, 16, ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + CALC_MSE_B(src0, ref0, var); + CALC_MSE_B(src2, ref2, var); + CALC_MSE_B(src1, ref1, var); + CALC_MSE_B(src3, ref3, var); + } + + return HADD_SW_S32(var); +} + +uint32_t aom_get4x4sse_cs_msa(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride) { + uint32_t err = 0; + uint32_t src0, src1, src2, src3; + uint32_t ref0, ref1, ref2, ref3; + v16i8 src = { 0 }; + v16i8 ref = { 0 }; + v16u8 src_vec0, src_vec1; + v8i16 diff0, diff1; + v4i32 err0 = { 0 }; + v4i32 err1 = { 0 }; + + LW4(src_ptr, src_stride, src0, src1, src2, src3); + LW4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); + INSERT_W4_SB(src0, src1, src2, src3, src); + INSERT_W4_SB(ref0, ref1, ref2, ref3, ref); + ILVRL_B2_UB(src, ref, src_vec0, src_vec1); + HSUB_UB2_SH(src_vec0, src_vec1, diff0, diff1); + DPADD_SH2_SW(diff0, diff1, diff0, diff1, err0, err1); + err = HADD_SW_S32(err0); + err += HADD_SW_S32(err1); + + return err; +} + +#define VARIANCE_4Wx4H(sse, diff) VARIANCE_WxH(sse, diff, 4); +#define VARIANCE_4Wx8H(sse, diff) VARIANCE_WxH(sse, diff, 5); +#define VARIANCE_8Wx4H(sse, diff) VARIANCE_WxH(sse, diff, 5); +#define VARIANCE_8Wx8H(sse, diff) VARIANCE_WxH(sse, diff, 6); +#define VARIANCE_8Wx16H(sse, diff) VARIANCE_WxH(sse, diff, 7); +#define VARIANCE_16Wx8H(sse, diff) VARIANCE_WxH(sse, diff, 7); +#define VARIANCE_16Wx16H(sse, diff) VARIANCE_WxH(sse, diff, 8); + +#define VARIANCE_16Wx32H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 9); +#define VARIANCE_32Wx16H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 9); +#define VARIANCE_32Wx32H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 10); +#define VARIANCE_32Wx64H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 11); +#define VARIANCE_64Wx32H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 11); +#define VARIANCE_64Wx64H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 12); + +#define AOM_VARIANCE_WDXHT_MSA(wd, ht) \ + uint32_t aom_variance##wd##x##ht##_msa( \ + const uint8_t *src, int32_t src_stride, const uint8_t *ref, \ + int32_t ref_stride, uint32_t *sse) { \ + int32_t diff; \ + \ + *sse = \ + sse_diff_##wd##width_msa(src, src_stride, ref, ref_stride, ht, &diff); \ + \ + return VARIANCE_##wd##Wx##ht##H(*sse, diff); \ + } + +/* clang-format off */ +AOM_VARIANCE_WDXHT_MSA(4, 4) +AOM_VARIANCE_WDXHT_MSA(4, 8) + +AOM_VARIANCE_WDXHT_MSA(8, 4) +AOM_VARIANCE_WDXHT_MSA(8, 8) +AOM_VARIANCE_WDXHT_MSA(8, 16) + +AOM_VARIANCE_WDXHT_MSA(16, 8) +AOM_VARIANCE_WDXHT_MSA(16, 16) +AOM_VARIANCE_WDXHT_MSA(16, 32) + +AOM_VARIANCE_WDXHT_MSA(32, 16) +AOM_VARIANCE_WDXHT_MSA(32, 32) +/* clang-format on */ + +uint32_t aom_variance32x64_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + uint32_t *sse) { + int32_t diff; + + *sse = sse_diff_32x64_msa(src, src_stride, ref, ref_stride, &diff); + + return VARIANCE_32Wx64H(*sse, diff); +} + +uint32_t aom_variance64x32_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + uint32_t *sse) { + int32_t diff; + + *sse = sse_diff_64x32_msa(src, src_stride, ref, ref_stride, &diff); + + return VARIANCE_64Wx32H(*sse, diff); +} + +uint32_t aom_variance64x64_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + uint32_t *sse) { + int32_t diff; + + *sse = sse_diff_64x64_msa(src, src_stride, ref, ref_stride, &diff); + + return VARIANCE_64Wx64H(*sse, diff); +} + +uint32_t aom_mse8x8_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, uint32_t *sse) { + *sse = sse_8width_msa(src, src_stride, ref, ref_stride, 8); + + return *sse; +} + +uint32_t aom_mse8x16_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + uint32_t *sse) { + *sse = sse_8width_msa(src, src_stride, ref, ref_stride, 16); + + return *sse; +} + +uint32_t aom_mse16x8_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + uint32_t *sse) { + *sse = sse_16width_msa(src, src_stride, ref, ref_stride, 8); + + return *sse; +} + +uint32_t aom_mse16x16_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + uint32_t *sse) { + *sse = sse_16width_msa(src, src_stride, ref, ref_stride, 16); + + return *sse; +} + +void aom_get8x8var_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, uint32_t *sse, + int32_t *sum) { + *sse = sse_diff_8width_msa(src, src_stride, ref, ref_stride, 8, sum); +} + +void aom_get16x16var_msa(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, uint32_t *sse, + int32_t *sum) { + *sse = sse_diff_16width_msa(src, src_stride, ref, ref_stride, 16, sum); +} + +uint32_t aom_get_mb_ss_msa(const int16_t *src) { return get_mb_ss_msa(src); } diff --git a/third_party/aom/aom_dsp/postproc.h b/third_party/aom/aom_dsp/postproc.h new file mode 100644 index 0000000000..11a8c5ad78 --- /dev/null +++ b/third_party/aom/aom_dsp/postproc.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_POSTPROC_H_ +#define AOM_DSP_POSTPROC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// Fills a noise buffer with gaussian noise strength determined by sigma. +int aom_setup_noise(double sigma, int size, char *noise); + +#ifdef __cplusplus +} +#endif + +#endif // AOM_DSP_POSTPROC_H_ diff --git a/third_party/aom/aom_dsp/prob.c b/third_party/aom/aom_dsp/prob.c new file mode 100644 index 0000000000..c60bfdac5b --- /dev/null +++ b/third_party/aom/aom_dsp/prob.c @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" + +#if CONFIG_EC_MULTISYMBOL +#include +#endif + +#include "aom_dsp/prob.h" + +const uint8_t aom_norm[256] = { + 0, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +static unsigned int tree_merge_probs_impl(unsigned int i, + const aom_tree_index *tree, + const aom_prob *pre_probs, + const unsigned int *counts, + aom_prob *probs) { + const int l = tree[i]; + const unsigned int left_count = + (l <= 0) ? counts[-l] + : tree_merge_probs_impl(l, tree, pre_probs, counts, probs); + const int r = tree[i + 1]; + const unsigned int right_count = + (r <= 0) ? counts[-r] + : tree_merge_probs_impl(r, tree, pre_probs, counts, probs); + const unsigned int ct[2] = { left_count, right_count }; + probs[i >> 1] = mode_mv_merge_probs(pre_probs[i >> 1], ct); + return left_count + right_count; +} + +void aom_tree_merge_probs(const aom_tree_index *tree, const aom_prob *pre_probs, + const unsigned int *counts, aom_prob *probs) { + tree_merge_probs_impl(0, tree, pre_probs, counts, probs); +} + +#if CONFIG_EC_MULTISYMBOL +typedef struct tree_node tree_node; + +struct tree_node { + aom_tree_index index; + uint8_t probs[16]; + uint8_t prob; + int path; + int len; + int l; + int r; + aom_cdf_prob pdf; +}; + +/* Compute the probability of this node in Q23 */ +static uint32_t tree_node_prob(tree_node n, int i) { + uint32_t prob; + /* 1.0 in Q23 */ + prob = 16777216; + for (; i < n.len; i++) { + prob = prob * n.probs[i] >> 8; + } + return prob; +} + +static int tree_node_cmp(tree_node a, tree_node b) { + int i; + uint32_t pa; + uint32_t pb; + for (i = 0; i < AOMMIN(a.len, b.len) && a.probs[i] == b.probs[i]; i++) { + } + pa = tree_node_prob(a, i); + pb = tree_node_prob(b, i); + return pa > pb ? 1 : pa < pb ? -1 : 0; +} + +/* Given a Q15 probability for symbol subtree rooted at tree[n], this function + computes the probability of each symbol (defined as a node that has no + children). */ +static aom_cdf_prob tree_node_compute_probs(tree_node *tree, int n, + aom_cdf_prob pdf) { + if (tree[n].l == 0) { + /* This prevents probability computations in Q15 that underflow from + producing a symbol that has zero probability. */ + if (pdf == 0) pdf = 1; + tree[n].pdf = pdf; + return pdf; + } else { + /* We process the smaller probability first, */ + if (tree[n].prob < 128) { + aom_cdf_prob lp; + aom_cdf_prob rp; + lp = (((uint32_t)pdf) * tree[n].prob + 128) >> 8; + lp = tree_node_compute_probs(tree, tree[n].l, lp); + rp = tree_node_compute_probs(tree, tree[n].r, lp > pdf ? 0 : pdf - lp); + return lp + rp; + } else { + aom_cdf_prob rp; + aom_cdf_prob lp; + rp = (((uint32_t)pdf) * (256 - tree[n].prob) + 128) >> 8; + rp = tree_node_compute_probs(tree, tree[n].r, rp); + lp = tree_node_compute_probs(tree, tree[n].l, rp > pdf ? 0 : pdf - rp); + return lp + rp; + } + } +} + +static int tree_node_extract(tree_node *tree, int n, int symb, + aom_cdf_prob *pdf, aom_tree_index *index, + int *path, int *len) { + if (tree[n].l == 0) { + pdf[symb] = tree[n].pdf; + if (index != NULL) index[symb] = tree[n].index; + if (path != NULL) path[symb] = tree[n].path; + if (len != NULL) len[symb] = tree[n].len; + return symb + 1; + } else { + symb = tree_node_extract(tree, tree[n].l, symb, pdf, index, path, len); + return tree_node_extract(tree, tree[n].r, symb, pdf, index, path, len); + } +} + +int tree_to_cdf(const aom_tree_index *tree, const aom_prob *probs, + aom_tree_index root, aom_cdf_prob *cdf, aom_tree_index *index, + int *path, int *len) { + tree_node symb[2 * 16 - 1]; + int nodes; + int next[16]; + int size; + int nsymbs; + int i; + /* Create the root node with probability 1 in Q15. */ + symb[0].index = root; + symb[0].path = 0; + symb[0].len = 0; + symb[0].l = symb[0].r = 0; + nodes = 1; + next[0] = 0; + size = 1; + nsymbs = 1; + while (size > 0 && nsymbs < 16) { + int m; + tree_node n; + aom_tree_index j; + uint8_t prob; + m = 0; + /* Find the internal node with the largest probability. */ + for (i = 1; i < size; i++) { + if (tree_node_cmp(symb[next[i]], symb[next[m]]) > 0) m = i; + } + i = next[m]; + memmove(&next[m], &next[m + 1], sizeof(*next) * (size - (m + 1))); + size--; + /* Split this symbol into two symbols */ + n = symb[i]; + j = n.index; + prob = probs[j >> 1]; + /* Left */ + n.index = tree[j]; + n.path <<= 1; + n.len++; + n.probs[n.len - 1] = prob; + symb[nodes] = n; + if (n.index > 0) { + next[size++] = nodes; + } + /* Right */ + n.index = tree[j + 1]; + n.path += 1; + n.probs[n.len - 1] = 256 - prob; + symb[nodes + 1] = n; + if (n.index > 0) { + next[size++] = nodes + 1; + } + symb[i].prob = prob; + symb[i].l = nodes; + symb[i].r = nodes + 1; + nodes += 2; + nsymbs++; + } + /* Compute the probabilities of each symbol in Q15 */ + tree_node_compute_probs(symb, 0, CDF_PROB_TOP); + /* Extract the cdf, index, path and length */ + tree_node_extract(symb, 0, 0, cdf, index, path, len); + /* Convert to CDF */ + cdf[0] = AOM_ICDF(cdf[0]); + for (i = 1; i < nsymbs; i++) { + cdf[i] = AOM_ICDF(AOM_ICDF(cdf[i - 1]) + cdf[i]); + } +// Store symbol count at the end of the CDF +#if CONFIG_EC_ADAPT + cdf[nsymbs] = 0; +#endif + return nsymbs; +} + +/* This code assumes that tree contains as unique leaf nodes the integer values + 0 to len - 1 and produces the forward and inverse mapping tables in ind[] + and inv[] respectively. */ +static void tree_to_index(int *stack_index, int *ind, int *inv, + const aom_tree_index *tree, int value, int index) { + value *= 2; + + do { + const aom_tree_index content = tree[index]; + ++index; + if (content <= 0) { + inv[*stack_index] = -content; + ind[-content] = *stack_index; + ++(*stack_index); + } else { + tree_to_index(stack_index, ind, inv, tree, value, content); + } + } while (++value & 1); +} + +void av1_indices_from_tree(int *ind, int *inv, const aom_tree_index *tree) { + int stack_index = 0; + tree_to_index(&stack_index, ind, inv, tree, 0, 0); +} +#endif diff --git a/third_party/aom/aom_dsp/prob.h b/third_party/aom/aom_dsp/prob.h new file mode 100644 index 0000000000..8085929238 --- /dev/null +++ b/third_party/aom/aom_dsp/prob.h @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_PROB_H_ +#define AOM_DSP_PROB_H_ + +#include + +#include "./aom_config.h" +#include "./aom_dsp_common.h" + +#include "aom_ports/bitops.h" +#include "aom_ports/mem.h" + +#if CONFIG_DAALA_EC +#include "aom_dsp/entcode.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef uint8_t aom_prob; + +// TODO(negge): Rename this aom_prob once we remove vpxbool. +typedef uint16_t aom_cdf_prob; + +#if CONFIG_EC_MULTISYMBOL +#define CDF_SIZE(x) ((x) + 1) +#endif + +#define CDF_PROB_BITS 15 +#define CDF_PROB_TOP (1 << CDF_PROB_BITS) + +#if CONFIG_DAALA_EC +#define AOM_ICDF OD_ICDF +#else +#define AOM_ICDF(x) (x) +#endif + +#define MAX_PROB 255 + +#define aom_prob_half ((aom_prob)128) + +typedef int8_t aom_tree_index; + +#define TREE_SIZE(leaf_count) (-2 + 2 * (leaf_count)) + +#define MODE_MV_COUNT_SAT 20 + +/* We build coding trees compactly in arrays. + Each node of the tree is a pair of aom_tree_indices. + Array index often references a corresponding probability table. + Index <= 0 means done encoding/decoding and value = -Index, + Index > 0 means need another bit, specification at index. + Nonnegative indices are always even; processing begins at node 0. */ + +typedef const aom_tree_index aom_tree[]; + +static INLINE aom_prob get_prob(unsigned int num, unsigned int den) { + assert(den != 0); + { + const int p = (int)(((uint64_t)num * 256 + (den >> 1)) / den); + // (p > 255) ? 255 : (p < 1) ? 1 : p; + const int clipped_prob = p | ((255 - p) >> 23) | (p == 0); + return (aom_prob)clipped_prob; + } +} + +static INLINE aom_prob get_binary_prob(unsigned int n0, unsigned int n1) { + const unsigned int den = n0 + n1; + if (den == 0) return 128u; + return get_prob(n0, den); +} + +/* This function assumes prob1 and prob2 are already within [1,255] range. */ +static INLINE aom_prob weighted_prob(int prob1, int prob2, int factor) { + return ROUND_POWER_OF_TWO(prob1 * (256 - factor) + prob2 * factor, 8); +} + +static INLINE aom_prob merge_probs(aom_prob pre_prob, const unsigned int ct[2], + unsigned int count_sat, + unsigned int max_update_factor) { + const aom_prob prob = get_binary_prob(ct[0], ct[1]); + const unsigned int count = AOMMIN(ct[0] + ct[1], count_sat); + const unsigned int factor = max_update_factor * count / count_sat; + return weighted_prob(pre_prob, prob, factor); +} + +// MODE_MV_MAX_UPDATE_FACTOR (128) * count / MODE_MV_COUNT_SAT; +static const int count_to_update_factor[MODE_MV_COUNT_SAT + 1] = { + 0, 6, 12, 19, 25, 32, 38, 44, 51, 57, 64, + 70, 76, 83, 89, 96, 102, 108, 115, 121, 128 +}; + +static INLINE aom_prob mode_mv_merge_probs(aom_prob pre_prob, + const unsigned int ct[2]) { + const unsigned int den = ct[0] + ct[1]; + if (den == 0) { + return pre_prob; + } else { + const unsigned int count = AOMMIN(den, MODE_MV_COUNT_SAT); + const unsigned int factor = count_to_update_factor[count]; + const aom_prob prob = get_prob(ct[0], den); + return weighted_prob(pre_prob, prob, factor); + } +} + +void aom_tree_merge_probs(const aom_tree_index *tree, const aom_prob *pre_probs, + const unsigned int *counts, aom_prob *probs); + +#if CONFIG_EC_MULTISYMBOL +int tree_to_cdf(const aom_tree_index *tree, const aom_prob *probs, + aom_tree_index root, aom_cdf_prob *cdf, aom_tree_index *ind, + int *pth, int *len); + +static INLINE void av1_tree_to_cdf(const aom_tree_index *tree, + const aom_prob *probs, aom_cdf_prob *cdf) { + aom_tree_index index[16]; + int path[16]; + int dist[16]; + tree_to_cdf(tree, probs, 0, cdf, index, path, dist); +} + +#define av1_tree_to_cdf_1D(tree, probs, cdf, u) \ + do { \ + int i; \ + for (i = 0; i < u; i++) { \ + av1_tree_to_cdf(tree, probs[i], cdf[i]); \ + } \ + } while (0) + +#define av1_tree_to_cdf_2D(tree, probs, cdf, v, u) \ + do { \ + int j; \ + int i; \ + for (j = 0; j < v; j++) { \ + for (i = 0; i < u; i++) { \ + av1_tree_to_cdf(tree, probs[j][i], cdf[j][i]); \ + } \ + } \ + } while (0) + +void av1_indices_from_tree(int *ind, int *inv, const aom_tree_index *tree); +#endif + +DECLARE_ALIGNED(16, extern const uint8_t, aom_norm[256]); + +#if CONFIG_EC_ADAPT +static INLINE void update_cdf(aom_cdf_prob *cdf, int val, int nsymbs) { + const int rate = 4 + (cdf[nsymbs] > 31) + get_msb(nsymbs); + const int rate2 = 5; + int i, tmp; + int diff; +#if 1 + const int tmp0 = 1 << rate2; + tmp = AOM_ICDF(tmp0); + diff = ((CDF_PROB_TOP - (nsymbs << rate2)) >> rate) << rate; +// Single loop (faster) +#if CONFIG_DAALA_EC && CONFIG_EC_SMALLMUL + for (i = 0; i < nsymbs - 1; ++i, tmp -= tmp0) { + tmp -= (i == val ? diff : 0); + cdf[i] += ((tmp - cdf[i]) >> rate); + } +#else + for (i = 0; i < nsymbs - 1; ++i, tmp += tmp0) { + tmp += (i == val ? diff : 0); + cdf[i] -= ((cdf[i] - tmp) >> rate); + } +#endif +#else + for (i = 0; i < nsymbs; ++i) { + tmp = (i + 1) << rate2; + cdf[i] -= ((cdf[i] - tmp) >> rate); + } + diff = CDF_PROB_TOP - cdf[nsymbs - 1]; + + for (i = val; i < nsymbs; ++i) { + cdf[i] += diff; + } +#endif + cdf[nsymbs] += (cdf[nsymbs] < 32); +} +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_PROB_H_ diff --git a/third_party/aom/aom_dsp/psnr.c b/third_party/aom/aom_dsp/psnr.c new file mode 100644 index 0000000000..461c13729c --- /dev/null +++ b/third_party/aom/aom_dsp/psnr.c @@ -0,0 +1,373 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/psnr.h" +#include "aom_scale/yv12config.h" + +double aom_sse_to_psnr(double samples, double peak, double sse) { + if (sse > 0.0) { + const double psnr = 10.0 * log10(samples * peak * peak / sse); + return psnr > MAX_PSNR ? MAX_PSNR : psnr; + } else { + return MAX_PSNR; + } +} + +/* TODO(yaowu): The block_variance calls the unoptimized versions of variance() +* and highbd_8_variance(). It should not. +*/ +static void encoder_variance(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int w, int h, unsigned int *sse, + int *sum) { + int i, j; + + *sum = 0; + *sse = 0; + + for (i = 0; i < h; i++) { + for (j = 0; j < w; j++) { + const int diff = a[j] - b[j]; + *sum += diff; + *sse += diff * diff; + } + + a += a_stride; + b += b_stride; + } +} + +#if CONFIG_HIGHBITDEPTH +static void encoder_highbd_variance64(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, int w, + int h, uint64_t *sse, int64_t *sum) { + int i, j; + + uint16_t *a = CONVERT_TO_SHORTPTR(a8); + uint16_t *b = CONVERT_TO_SHORTPTR(b8); + *sum = 0; + *sse = 0; + + for (i = 0; i < h; i++) { + for (j = 0; j < w; j++) { + const int diff = a[j] - b[j]; + *sum += diff; + *sse += diff * diff; + } + a += a_stride; + b += b_stride; + } +} + +static void encoder_highbd_8_variance(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, int w, + int h, unsigned int *sse, int *sum) { + uint64_t sse_long = 0; + int64_t sum_long = 0; + encoder_highbd_variance64(a8, a_stride, b8, b_stride, w, h, &sse_long, + &sum_long); + *sse = (unsigned int)sse_long; + *sum = (int)sum_long; +} +#endif // CONFIG_HIGHBITDEPTH + +static int64_t get_sse(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int width, int height) { + const int dw = width % 16; + const int dh = height % 16; + int64_t total_sse = 0; + unsigned int sse = 0; + int sum = 0; + int x, y; + + if (dw > 0) { + encoder_variance(&a[width - dw], a_stride, &b[width - dw], b_stride, dw, + height, &sse, &sum); + total_sse += sse; + } + + if (dh > 0) { + encoder_variance(&a[(height - dh) * a_stride], a_stride, + &b[(height - dh) * b_stride], b_stride, width - dw, dh, + &sse, &sum); + total_sse += sse; + } + + for (y = 0; y < height / 16; ++y) { + const uint8_t *pa = a; + const uint8_t *pb = b; + for (x = 0; x < width / 16; ++x) { + aom_mse16x16(pa, a_stride, pb, b_stride, &sse); + total_sse += sse; + + pa += 16; + pb += 16; + } + + a += 16 * a_stride; + b += 16 * b_stride; + } + + return total_sse; +} + +#if CONFIG_HIGHBITDEPTH +static int64_t highbd_get_sse_shift(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, int width, + int height, unsigned int input_shift) { + const uint16_t *a = CONVERT_TO_SHORTPTR(a8); + const uint16_t *b = CONVERT_TO_SHORTPTR(b8); + int64_t total_sse = 0; + int x, y; + for (y = 0; y < height; ++y) { + for (x = 0; x < width; ++x) { + int64_t diff; + diff = (a[x] >> input_shift) - (b[x] >> input_shift); + total_sse += diff * diff; + } + a += a_stride; + b += b_stride; + } + return total_sse; +} + +static int64_t highbd_get_sse(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int width, int height) { + int64_t total_sse = 0; + int x, y; + const int dw = width % 16; + const int dh = height % 16; + unsigned int sse = 0; + int sum = 0; + if (dw > 0) { + encoder_highbd_8_variance(&a[width - dw], a_stride, &b[width - dw], + b_stride, dw, height, &sse, &sum); + total_sse += sse; + } + if (dh > 0) { + encoder_highbd_8_variance(&a[(height - dh) * a_stride], a_stride, + &b[(height - dh) * b_stride], b_stride, + width - dw, dh, &sse, &sum); + total_sse += sse; + } + for (y = 0; y < height / 16; ++y) { + const uint8_t *pa = a; + const uint8_t *pb = b; + for (x = 0; x < width / 16; ++x) { + aom_highbd_8_mse16x16(pa, a_stride, pb, b_stride, &sse); + total_sse += sse; + pa += 16; + pb += 16; + } + a += 16 * a_stride; + b += 16 * b_stride; + } + return total_sse; +} +#endif // CONFIG_HIGHBITDEPTH + +int64_t aom_get_y_sse_part(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, int hstart, int width, + int vstart, int height) { + return get_sse(a->y_buffer + vstart * a->y_stride + hstart, a->y_stride, + b->y_buffer + vstart * b->y_stride + hstart, b->y_stride, + width, height); +} + +int64_t aom_get_y_sse(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b) { + assert(a->y_crop_width == b->y_crop_width); + assert(a->y_crop_height == b->y_crop_height); + + return get_sse(a->y_buffer, a->y_stride, b->y_buffer, b->y_stride, + a->y_crop_width, a->y_crop_height); +} + +int64_t aom_get_u_sse_part(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, int hstart, int width, + int vstart, int height) { + return get_sse(a->u_buffer + vstart * a->uv_stride + hstart, a->uv_stride, + b->u_buffer + vstart * b->uv_stride + hstart, b->uv_stride, + width, height); +} + +int64_t aom_get_u_sse(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b) { + assert(a->uv_crop_width == b->uv_crop_width); + assert(a->uv_crop_height == b->uv_crop_height); + + return get_sse(a->u_buffer, a->uv_stride, b->u_buffer, b->uv_stride, + a->uv_crop_width, a->uv_crop_height); +} + +int64_t aom_get_v_sse_part(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, int hstart, int width, + int vstart, int height) { + return get_sse(a->v_buffer + vstart * a->uv_stride + hstart, a->uv_stride, + b->v_buffer + vstart * b->uv_stride + hstart, b->uv_stride, + width, height); +} + +int64_t aom_get_v_sse(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b) { + assert(a->uv_crop_width == b->uv_crop_width); + assert(a->uv_crop_height == b->uv_crop_height); + + return get_sse(a->v_buffer, a->uv_stride, b->v_buffer, b->uv_stride, + a->uv_crop_width, a->uv_crop_height); +} + +#if CONFIG_HIGHBITDEPTH +int64_t aom_highbd_get_y_sse_part(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, int hstart, + int width, int vstart, int height) { + return highbd_get_sse( + a->y_buffer + vstart * a->y_stride + hstart, a->y_stride, + b->y_buffer + vstart * b->y_stride + hstart, b->y_stride, width, height); +} + +int64_t aom_highbd_get_y_sse(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b) { + assert(a->y_crop_width == b->y_crop_width); + assert(a->y_crop_height == b->y_crop_height); + assert((a->flags & YV12_FLAG_HIGHBITDEPTH) != 0); + assert((b->flags & YV12_FLAG_HIGHBITDEPTH) != 0); + + return highbd_get_sse(a->y_buffer, a->y_stride, b->y_buffer, b->y_stride, + a->y_crop_width, a->y_crop_height); +} + +int64_t aom_highbd_get_u_sse_part(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, int hstart, + int width, int vstart, int height) { + return highbd_get_sse(a->u_buffer + vstart * a->uv_stride + hstart, + a->uv_stride, + b->u_buffer + vstart * b->uv_stride + hstart, + b->uv_stride, width, height); +} + +int64_t aom_highbd_get_u_sse(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b) { + assert(a->uv_crop_width == b->uv_crop_width); + assert(a->uv_crop_height == b->uv_crop_height); + assert((a->flags & YV12_FLAG_HIGHBITDEPTH) != 0); + assert((b->flags & YV12_FLAG_HIGHBITDEPTH) != 0); + + return highbd_get_sse(a->u_buffer, a->uv_stride, b->u_buffer, b->uv_stride, + a->uv_crop_width, a->uv_crop_height); +} + +int64_t aom_highbd_get_v_sse_part(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, int hstart, + int width, int vstart, int height) { + return highbd_get_sse(a->v_buffer + vstart * a->uv_stride + hstart, + a->uv_stride, + b->v_buffer + vstart * b->uv_stride + hstart, + b->uv_stride, width, height); +} + +int64_t aom_highbd_get_v_sse(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b) { + assert(a->uv_crop_width == b->uv_crop_width); + assert(a->uv_crop_height == b->uv_crop_height); + assert((a->flags & YV12_FLAG_HIGHBITDEPTH) != 0); + assert((b->flags & YV12_FLAG_HIGHBITDEPTH) != 0); + + return highbd_get_sse(a->v_buffer, a->uv_stride, b->v_buffer, b->uv_stride, + a->uv_crop_width, a->uv_crop_height); +} +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_HIGHBITDEPTH +void aom_calc_highbd_psnr(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, PSNR_STATS *psnr, + uint32_t bit_depth, uint32_t in_bit_depth) { + const int widths[3] = { a->y_crop_width, a->uv_crop_width, a->uv_crop_width }; + const int heights[3] = { a->y_crop_height, a->uv_crop_height, + a->uv_crop_height }; + const uint8_t *a_planes[3] = { a->y_buffer, a->u_buffer, a->v_buffer }; + const int a_strides[3] = { a->y_stride, a->uv_stride, a->uv_stride }; + const uint8_t *b_planes[3] = { b->y_buffer, b->u_buffer, b->v_buffer }; + const int b_strides[3] = { b->y_stride, b->uv_stride, b->uv_stride }; + int i; + uint64_t total_sse = 0; + uint32_t total_samples = 0; + const double peak = (double)((1 << in_bit_depth) - 1); + const unsigned int input_shift = bit_depth - in_bit_depth; + + for (i = 0; i < 3; ++i) { + const int w = widths[i]; + const int h = heights[i]; + const uint32_t samples = w * h; + uint64_t sse; + if (a->flags & YV12_FLAG_HIGHBITDEPTH) { + if (input_shift) { + sse = highbd_get_sse_shift(a_planes[i], a_strides[i], b_planes[i], + b_strides[i], w, h, input_shift); + } else { + sse = highbd_get_sse(a_planes[i], a_strides[i], b_planes[i], + b_strides[i], w, h); + } + } else { + sse = get_sse(a_planes[i], a_strides[i], b_planes[i], b_strides[i], w, h); + } + psnr->sse[1 + i] = sse; + psnr->samples[1 + i] = samples; + psnr->psnr[1 + i] = aom_sse_to_psnr(samples, peak, (double)sse); + + total_sse += sse; + total_samples += samples; + } + + psnr->sse[0] = total_sse; + psnr->samples[0] = total_samples; + psnr->psnr[0] = + aom_sse_to_psnr((double)total_samples, peak, (double)total_sse); +} + +#endif // !CONFIG_HIGHBITDEPTH + +void aom_calc_psnr(const YV12_BUFFER_CONFIG *a, const YV12_BUFFER_CONFIG *b, + PSNR_STATS *psnr) { + static const double peak = 255.0; + const int widths[3] = { a->y_crop_width, a->uv_crop_width, a->uv_crop_width }; + const int heights[3] = { a->y_crop_height, a->uv_crop_height, + a->uv_crop_height }; + const uint8_t *a_planes[3] = { a->y_buffer, a->u_buffer, a->v_buffer }; + const int a_strides[3] = { a->y_stride, a->uv_stride, a->uv_stride }; + const uint8_t *b_planes[3] = { b->y_buffer, b->u_buffer, b->v_buffer }; + const int b_strides[3] = { b->y_stride, b->uv_stride, b->uv_stride }; + int i; + uint64_t total_sse = 0; + uint32_t total_samples = 0; + + for (i = 0; i < 3; ++i) { + const int w = widths[i]; + const int h = heights[i]; + const uint32_t samples = w * h; + const uint64_t sse = + get_sse(a_planes[i], a_strides[i], b_planes[i], b_strides[i], w, h); + psnr->sse[1 + i] = sse; + psnr->samples[1 + i] = samples; + psnr->psnr[1 + i] = aom_sse_to_psnr(samples, peak, (double)sse); + + total_sse += sse; + total_samples += samples; + } + + psnr->sse[0] = total_sse; + psnr->samples[0] = total_samples; + psnr->psnr[0] = + aom_sse_to_psnr((double)total_samples, peak, (double)total_sse); +} diff --git a/third_party/aom/aom_dsp/psnr.h b/third_party/aom/aom_dsp/psnr.h new file mode 100644 index 0000000000..480140e6f0 --- /dev/null +++ b/third_party/aom/aom_dsp/psnr.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_PSNR_H_ +#define AOM_DSP_PSNR_H_ + +#include "aom_scale/yv12config.h" + +#define MAX_PSNR 100.0 + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + double psnr[4]; // total/y/u/v + uint64_t sse[4]; // total/y/u/v + uint32_t samples[4]; // total/y/u/v +} PSNR_STATS; + +/*!\brief Converts SSE to PSNR +* +* Converts sum of squared errros (SSE) to peak signal-to-noise ratio (PNSR). +* +* \param[in] samples Number of samples +* \param[in] peak Max sample value +* \param[in] sse Sum of squared errors +*/ +double aom_sse_to_psnr(double samples, double peak, double sse); +int64_t aom_get_y_sse_part(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, int hstart, int width, + int vstart, int height); +int64_t aom_get_y_sse(const YV12_BUFFER_CONFIG *a, const YV12_BUFFER_CONFIG *b); +int64_t aom_get_u_sse_part(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, int hstart, int width, + int vstart, int height); +int64_t aom_get_u_sse(const YV12_BUFFER_CONFIG *a, const YV12_BUFFER_CONFIG *b); +int64_t aom_get_v_sse_part(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, int hstart, int width, + int vstart, int height); +int64_t aom_get_v_sse(const YV12_BUFFER_CONFIG *a, const YV12_BUFFER_CONFIG *b); +#if CONFIG_HIGHBITDEPTH +int64_t aom_highbd_get_y_sse_part(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, int hstart, + int width, int vstart, int height); +int64_t aom_highbd_get_y_sse(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b); +int64_t aom_highbd_get_u_sse_part(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, int hstart, + int width, int vstart, int height); +int64_t aom_highbd_get_u_sse(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b); +int64_t aom_highbd_get_v_sse_part(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, int hstart, + int width, int vstart, int height); +int64_t aom_highbd_get_v_sse(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b); +void aom_calc_highbd_psnr(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b, PSNR_STATS *psnr, + unsigned int bit_depth, unsigned int in_bit_depth); +#endif +void aom_calc_psnr(const YV12_BUFFER_CONFIG *a, const YV12_BUFFER_CONFIG *b, + PSNR_STATS *psnr); + +double aom_psnrhvs(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, double *phvs_y, + double *phvs_u, double *phvs_v, uint32_t bd, uint32_t in_bd); +#ifdef __cplusplus +} // extern "C" +#endif +#endif // AOM_DSP_PSNR_H_ diff --git a/third_party/aom/aom_dsp/psnrhvs.c b/third_party/aom/aom_dsp/psnrhvs.c new file mode 100644 index 0000000000..aeefd5908e --- /dev/null +++ b/third_party/aom/aom_dsp/psnrhvs.c @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + * + * This code was originally written by: Gregory Maxwell, at the Daala + * project. + */ + +#include +#include +#include +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/psnr.h" +#include "aom_dsp/ssim.h" +#include "aom_ports/system_state.h" + +#if !defined(M_PI) +#define M_PI (3.141592653589793238462643) +#endif +#include + +static void od_bin_fdct8x8(tran_low_t *y, int ystride, const int16_t *x, + int xstride) { + int i, j; + (void)xstride; + aom_fdct8x8(x, y, ystride); + for (i = 0; i < 8; i++) + for (j = 0; j < 8; j++) + *(y + ystride * i + j) = (*(y + ystride * i + j) + 4) >> 3; +} + +#if CONFIG_HIGHBITDEPTH +static void hbd_od_bin_fdct8x8(tran_low_t *y, int ystride, const int16_t *x, + int xstride) { + int i, j; + (void)xstride; + aom_highbd_fdct8x8(x, y, ystride); + for (i = 0; i < 8; i++) + for (j = 0; j < 8; j++) + *(y + ystride * i + j) = (*(y + ystride * i + j) + 4) >> 3; +} +#endif + +/* Normalized inverse quantization matrix for 8x8 DCT at the point of + * transparency. This is not the JPEG based matrix from the paper, + this one gives a slightly higher MOS agreement.*/ +static const double csf_y[8][8] = { + { 1.6193873005, 2.2901594831, 2.08509755623, 1.48366094411, 1.00227514334, + 0.678296995242, 0.466224900598, 0.3265091542 }, + { 2.2901594831, 1.94321815382, 2.04793073064, 1.68731108984, 1.2305666963, + 0.868920337363, 0.61280991668, 0.436405793551 }, + { 2.08509755623, 2.04793073064, 1.34329019223, 1.09205635862, 0.875748795257, + 0.670882927016, 0.501731932449, 0.372504254596 }, + { 1.48366094411, 1.68731108984, 1.09205635862, 0.772819797575, 0.605636379554, + 0.48309405692, 0.380429446972, 0.295774038565 }, + { 1.00227514334, 1.2305666963, 0.875748795257, 0.605636379554, 0.448996256676, + 0.352889268808, 0.283006984131, 0.226951348204 }, + { 0.678296995242, 0.868920337363, 0.670882927016, 0.48309405692, + 0.352889268808, 0.27032073436, 0.215017739696, 0.17408067321 }, + { 0.466224900598, 0.61280991668, 0.501731932449, 0.380429446972, + 0.283006984131, 0.215017739696, 0.168869545842, 0.136153931001 }, + { 0.3265091542, 0.436405793551, 0.372504254596, 0.295774038565, + 0.226951348204, 0.17408067321, 0.136153931001, 0.109083846276 } +}; +static const double csf_cb420[8][8] = { + { 1.91113096927, 2.46074210438, 1.18284184739, 1.14982565193, 1.05017074788, + 0.898018824055, 0.74725392039, 0.615105596242 }, + { 2.46074210438, 1.58529308355, 1.21363250036, 1.38190029285, 1.33100189972, + 1.17428548929, 0.996404342439, 0.830890433625 }, + { 1.18284184739, 1.21363250036, 0.978712413627, 1.02624506078, 1.03145147362, + 0.960060382087, 0.849823426169, 0.731221236837 }, + { 1.14982565193, 1.38190029285, 1.02624506078, 0.861317501629, 0.801821139099, + 0.751437590932, 0.685398513368, 0.608694761374 }, + { 1.05017074788, 1.33100189972, 1.03145147362, 0.801821139099, 0.676555426187, + 0.605503172737, 0.55002013668, 0.495804539034 }, + { 0.898018824055, 1.17428548929, 0.960060382087, 0.751437590932, + 0.605503172737, 0.514674450957, 0.454353482512, 0.407050308965 }, + { 0.74725392039, 0.996404342439, 0.849823426169, 0.685398513368, + 0.55002013668, 0.454353482512, 0.389234902883, 0.342353999733 }, + { 0.615105596242, 0.830890433625, 0.731221236837, 0.608694761374, + 0.495804539034, 0.407050308965, 0.342353999733, 0.295530605237 } +}; +static const double csf_cr420[8][8] = { + { 2.03871978502, 2.62502345193, 1.26180942886, 1.11019789803, 1.01397751469, + 0.867069376285, 0.721500455585, 0.593906509971 }, + { 2.62502345193, 1.69112867013, 1.17180569821, 1.3342742857, 1.28513006198, + 1.13381474809, 0.962064122248, 0.802254508198 }, + { 1.26180942886, 1.17180569821, 0.944981930573, 0.990876405848, + 0.995903384143, 0.926972725286, 0.820534991409, 0.706020324706 }, + { 1.11019789803, 1.3342742857, 0.990876405848, 0.831632933426, 0.77418706195, + 0.725539939514, 0.661776842059, 0.587716619023 }, + { 1.01397751469, 1.28513006198, 0.995903384143, 0.77418706195, 0.653238524286, + 0.584635025748, 0.531064164893, 0.478717061273 }, + { 0.867069376285, 1.13381474809, 0.926972725286, 0.725539939514, + 0.584635025748, 0.496936637883, 0.438694579826, 0.393021669543 }, + { 0.721500455585, 0.962064122248, 0.820534991409, 0.661776842059, + 0.531064164893, 0.438694579826, 0.375820256136, 0.330555063063 }, + { 0.593906509971, 0.802254508198, 0.706020324706, 0.587716619023, + 0.478717061273, 0.393021669543, 0.330555063063, 0.285345396658 } +}; + +static double convert_score_db(double _score, double _weight, int bit_depth) { + int16_t pix_max = 255; + assert(_score * _weight >= 0.0); + if (bit_depth == 10) + pix_max = 1023; + else if (bit_depth == 12) + pix_max = 4095; + + if (_weight * _score < pix_max * pix_max * 1e-10) return MAX_PSNR; + return 10 * (log10(pix_max * pix_max) - log10(_weight * _score)); +} + +static double calc_psnrhvs(const unsigned char *src, int _systride, + const unsigned char *dst, int _dystride, double _par, + int _w, int _h, int _step, const double _csf[8][8], + uint32_t bit_depth, uint32_t _shift) { + double ret; + const uint8_t *_src8 = src; + const uint8_t *_dst8 = dst; + const uint16_t *_src16 = CONVERT_TO_SHORTPTR(src); + const uint16_t *_dst16 = CONVERT_TO_SHORTPTR(dst); + int16_t dct_s[8 * 8], dct_d[8 * 8]; + tran_low_t dct_s_coef[8 * 8], dct_d_coef[8 * 8]; + double mask[8][8]; + int pixels; + int x; + int y; + (void)_par; + ret = pixels = 0; + /*In the PSNR-HVS-M paper[1] the authors describe the construction of + their masking table as "we have used the quantization table for the + color component Y of JPEG [6] that has been also obtained on the + basis of CSF. Note that the values in quantization table JPEG have + been normalized and then squared." Their CSF matrix (from PSNR-HVS) + was also constructed from the JPEG matrices. I can not find any obvious + scheme of normalizing to produce their table, but if I multiply their + CSF by 0.38857 and square the result I get their masking table. + I have no idea where this constant comes from, but deviating from it + too greatly hurts MOS agreement. + + [1] Nikolay Ponomarenko, Flavia Silvestri, Karen Egiazarian, Marco Carli, + Jaakko Astola, Vladimir Lukin, "On between-coefficient contrast masking + of DCT basis functions", CD-ROM Proceedings of the Third + International Workshop on Video Processing and Quality Metrics for Consumer + Electronics VPQM-07, Scottsdale, Arizona, USA, 25-26 January, 2007, 4 p.*/ + for (x = 0; x < 8; x++) + for (y = 0; y < 8; y++) + mask[x][y] = + (_csf[x][y] * 0.3885746225901003) * (_csf[x][y] * 0.3885746225901003); + for (y = 0; y < _h - 7; y += _step) { + for (x = 0; x < _w - 7; x += _step) { + int i; + int j; + double s_means[4]; + double d_means[4]; + double s_vars[4]; + double d_vars[4]; + double s_gmean = 0; + double d_gmean = 0; + double s_gvar = 0; + double d_gvar = 0; + double s_mask = 0; + double d_mask = 0; + for (i = 0; i < 4; i++) + s_means[i] = d_means[i] = s_vars[i] = d_vars[i] = 0; + for (i = 0; i < 8; i++) { + for (j = 0; j < 8; j++) { + int sub = ((i & 12) >> 2) + ((j & 12) >> 1); + if (bit_depth == 8 && _shift == 0) { + dct_s[i * 8 + j] = _src8[(y + i) * _systride + (j + x)]; + dct_d[i * 8 + j] = _dst8[(y + i) * _dystride + (j + x)]; + } else if (bit_depth == 10 || bit_depth == 12) { + dct_s[i * 8 + j] = _src16[(y + i) * _systride + (j + x)] >> _shift; + dct_d[i * 8 + j] = _dst16[(y + i) * _dystride + (j + x)] >> _shift; + } + s_gmean += dct_s[i * 8 + j]; + d_gmean += dct_d[i * 8 + j]; + s_means[sub] += dct_s[i * 8 + j]; + d_means[sub] += dct_d[i * 8 + j]; + } + } + s_gmean /= 64.f; + d_gmean /= 64.f; + for (i = 0; i < 4; i++) s_means[i] /= 16.f; + for (i = 0; i < 4; i++) d_means[i] /= 16.f; + for (i = 0; i < 8; i++) { + for (j = 0; j < 8; j++) { + int sub = ((i & 12) >> 2) + ((j & 12) >> 1); + s_gvar += (dct_s[i * 8 + j] - s_gmean) * (dct_s[i * 8 + j] - s_gmean); + d_gvar += (dct_d[i * 8 + j] - d_gmean) * (dct_d[i * 8 + j] - d_gmean); + s_vars[sub] += (dct_s[i * 8 + j] - s_means[sub]) * + (dct_s[i * 8 + j] - s_means[sub]); + d_vars[sub] += (dct_d[i * 8 + j] - d_means[sub]) * + (dct_d[i * 8 + j] - d_means[sub]); + } + } + s_gvar *= 1 / 63.f * 64; + d_gvar *= 1 / 63.f * 64; + for (i = 0; i < 4; i++) s_vars[i] *= 1 / 15.f * 16; + for (i = 0; i < 4; i++) d_vars[i] *= 1 / 15.f * 16; + if (s_gvar > 0) + s_gvar = (s_vars[0] + s_vars[1] + s_vars[2] + s_vars[3]) / s_gvar; + if (d_gvar > 0) + d_gvar = (d_vars[0] + d_vars[1] + d_vars[2] + d_vars[3]) / d_gvar; +#if CONFIG_HIGHBITDEPTH + if (bit_depth == 10 || bit_depth == 12) { + hbd_od_bin_fdct8x8(dct_s_coef, 8, dct_s, 8); + hbd_od_bin_fdct8x8(dct_d_coef, 8, dct_d, 8); + } +#endif + if (bit_depth == 8) { + od_bin_fdct8x8(dct_s_coef, 8, dct_s, 8); + od_bin_fdct8x8(dct_d_coef, 8, dct_d, 8); + } + for (i = 0; i < 8; i++) + for (j = (i == 0); j < 8; j++) + s_mask += dct_s_coef[i * 8 + j] * dct_s_coef[i * 8 + j] * mask[i][j]; + for (i = 0; i < 8; i++) + for (j = (i == 0); j < 8; j++) + d_mask += dct_d_coef[i * 8 + j] * dct_d_coef[i * 8 + j] * mask[i][j]; + s_mask = sqrt(s_mask * s_gvar) / 32.f; + d_mask = sqrt(d_mask * d_gvar) / 32.f; + if (d_mask > s_mask) s_mask = d_mask; + for (i = 0; i < 8; i++) { + for (j = 0; j < 8; j++) { + double err; + err = fabs((double)(dct_s_coef[i * 8 + j] - dct_d_coef[i * 8 + j])); + if (i != 0 || j != 0) + err = err < s_mask / mask[i][j] ? 0 : err - s_mask / mask[i][j]; + ret += (err * _csf[i][j]) * (err * _csf[i][j]); + pixels++; + } + } + } + } + if (pixels <= 0) return 0; + ret /= pixels; + return ret; +} + +double aom_psnrhvs(const YV12_BUFFER_CONFIG *src, const YV12_BUFFER_CONFIG *dst, + double *y_psnrhvs, double *u_psnrhvs, double *v_psnrhvs, + uint32_t bd, uint32_t in_bd) { + double psnrhvs; + const double par = 1.0; + const int step = 7; + uint32_t bd_shift = 0; + aom_clear_system_state(); + + assert(bd == 8 || bd == 10 || bd == 12); + assert(bd >= in_bd); + + bd_shift = bd - in_bd; + + *y_psnrhvs = calc_psnrhvs(src->y_buffer, src->y_stride, dst->y_buffer, + dst->y_stride, par, src->y_crop_width, + src->y_crop_height, step, csf_y, bd, bd_shift); + *u_psnrhvs = calc_psnrhvs(src->u_buffer, src->uv_stride, dst->u_buffer, + dst->uv_stride, par, src->uv_crop_width, + src->uv_crop_height, step, csf_cb420, bd, bd_shift); + *v_psnrhvs = calc_psnrhvs(src->v_buffer, src->uv_stride, dst->v_buffer, + dst->uv_stride, par, src->uv_crop_width, + src->uv_crop_height, step, csf_cr420, bd, bd_shift); + psnrhvs = (*y_psnrhvs) * .8 + .1 * ((*u_psnrhvs) + (*v_psnrhvs)); + return convert_score_db(psnrhvs, 1.0, in_bd); +} diff --git a/third_party/aom/aom_dsp/quantize.c b/third_party/aom/aom_dsp/quantize.c new file mode 100644 index 0000000000..0759c22e31 --- /dev/null +++ b/third_party/aom/aom_dsp/quantize.c @@ -0,0 +1,832 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/quantize.h" +#include "aom_mem/aom_mem.h" + +static void quantize_b_helper_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, + const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, +#if CONFIG_AOM_QM + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, +#endif + const int log_scale) { + const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale), + ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) }; + const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; + int i, non_zero_count = (int)n_coeffs, eob = -1; + (void)iscan; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + // Pre-scan pass + for (i = (int)n_coeffs - 1; i >= 0; i--) { + const int rc = scan[i]; +#if CONFIG_AOM_QM + const qm_val_t wt = qm_ptr[rc]; + const int coeff = coeff_ptr[rc] * wt; +#else + const int coeff = coeff_ptr[rc]; +#endif // CONFIG_AOM_QM + +#if CONFIG_AOM_QM + if (coeff < (zbins[rc != 0] << AOM_QM_BITS) && + coeff > (nzbins[rc != 0] << AOM_QM_BITS)) + non_zero_count--; +#else + if (coeff < zbins[rc != 0] && coeff > nzbins[rc != 0]) non_zero_count--; +#endif // CONFIG_AOM_QM + else + break; + } + + // Quantization pass: All coefficients with index >= zero_flag are + // skippable. Note: zero_flag can be zero. + for (i = 0; i < non_zero_count; i++) { + const int rc = scan[i]; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int tmp32; + +#if CONFIG_AOM_QM + const qm_val_t wt = qm_ptr[rc]; + if (abs_coeff * wt >= (zbins[rc != 0] << AOM_QM_BITS)) { +#else + if (abs_coeff >= zbins[rc != 0]) { +#endif // CONFIG_AOM_QM + int64_t tmp = + clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale), + INT16_MIN, INT16_MAX); +#if CONFIG_AOM_QM + tmp *= wt; + tmp32 = (int)(((((tmp * quant_ptr[rc != 0]) >> 16) + tmp) * + quant_shift_ptr[rc != 0]) >> + (16 - log_scale + AOM_QM_BITS)); // quantization +#else + tmp32 = (int)(((((tmp * quant_ptr[rc != 0]) >> 16) + tmp) * + quant_shift_ptr[rc != 0]) >> + (16 - log_scale)); // quantization +#endif // CONFIG_AOM_QM + qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign; +#if CONFIG_AOM_QM + const int dequant = + (dequant_ptr[rc != 0] * iqm_ptr[rc] + (1 << (AOM_QM_BITS - 1))) >> + AOM_QM_BITS; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant / (1 << log_scale); +#else + dqcoeff_ptr[rc] = + qcoeff_ptr[rc] * dequant_ptr[rc != 0] / (1 << log_scale); +#endif // CONFIG_AOM_QM + + if (tmp32) eob = i; + } + } + } + *eob_ptr = eob + 1; +} + +void aom_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const int16_t *iscan +#if CONFIG_AOM_QM + , + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr +#endif + ) { + quantize_b_helper_c(coeff_ptr, n_coeffs, skip_block, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, + dequant_ptr, eob_ptr, scan, iscan, +#if CONFIG_AOM_QM + qm_ptr, iqm_ptr, +#endif + 0); +} + +void aom_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan +#if CONFIG_AOM_QM + , + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr +#endif + ) { + quantize_b_helper_c(coeff_ptr, n_coeffs, skip_block, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, + dequant_ptr, eob_ptr, scan, iscan, +#if CONFIG_AOM_QM + qm_ptr, iqm_ptr, +#endif + 1); +} + +#if CONFIG_TX64X64 +void aom_quantize_b_64x64_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan +#if CONFIG_AOM_QM + , + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr +#endif + ) { + quantize_b_helper_c(coeff_ptr, n_coeffs, skip_block, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, + dequant_ptr, eob_ptr, scan, iscan, +#if CONFIG_AOM_QM + qm_ptr, iqm_ptr, +#endif + 2); +} +#endif // CONFIG_TX64X64 + +#if CONFIG_AOM_QM +void aom_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, int skip_block, + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr, + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr) { + const int rc = 0; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int64_t tmp, eob = -1; + int32_t tmp32; + int dequant = + (dequant_ptr * iqm_ptr[rc] + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX); + tmp32 = (int32_t)((tmp * qm_ptr[rc] * quant) >> (16 + AOM_QM_BITS)); + qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant; + if (tmp32) eob = 0; + } + *eob_ptr = eob + 1; +} + +void aom_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr, + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr) { + const int n_coeffs = 1024; + const int rc = 0; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int64_t tmp, eob = -1; + int32_t tmp32; + int dequant; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1), + INT16_MIN, INT16_MAX); + tmp32 = (int32_t)((tmp * qm_ptr[rc] * quant) >> (15 + AOM_QM_BITS)); + qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign; + dequant = + (dequant_ptr * iqm_ptr[rc] + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS; + dqcoeff_ptr[rc] = (qcoeff_ptr[rc] * dequant) / 2; + if (tmp32) eob = 0; + } + *eob_ptr = eob + 1; +} + +#if CONFIG_TX64X64 +void aom_quantize_dc_64x64(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr, + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr) { + const int n_coeffs = 1024; + const int rc = 0; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int64_t tmp, eob = -1; + int32_t tmp32; + int dequant; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 2), + INT16_MIN, INT16_MAX); + tmp32 = (int32_t)((tmp * qm_ptr[rc] * quant) >> (14 + AOM_QM_BITS)); + qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign; + dequant = + (dequant_ptr * iqm_ptr[rc] + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS; + dqcoeff_ptr[rc] = (qcoeff_ptr[rc] * dequant) / 4; + if (tmp32) eob = 0; + } + *eob_ptr = eob + 1; +} +#endif // CONFIG_TX64X64 + +#if CONFIG_HIGHBITDEPTH +void aom_highbd_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, + int skip_block, const int16_t *round_ptr, + const int16_t quant, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t dequant_ptr, + uint16_t *eob_ptr, const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr) { + int eob = -1; + int dequant = + (dequant_ptr * iqm_ptr[0] + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + const int coeff = coeff_ptr[0]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp = abs_coeff + round_ptr[0]; + const uint32_t abs_qcoeff = + (uint32_t)((tmp * qm_ptr[0] * quant) >> (16 + AOM_QM_BITS)); + qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant; + if (abs_qcoeff) eob = 0; + } + *eob_ptr = eob + 1; +} + +void aom_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr, + const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr) { + const int n_coeffs = 1024; + int eob = -1; + int dequant; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + const int coeff = coeff_ptr[0]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[0], 1); + const uint32_t abs_qcoeff = + (uint32_t)((tmp * qm_ptr[0] * quant) >> (15 + AOM_QM_BITS)); + qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dequant = + (dequant_ptr * iqm_ptr[0] + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS; + dqcoeff_ptr[0] = (qcoeff_ptr[0] * dequant) / 2; + if (abs_qcoeff) eob = 0; + } + *eob_ptr = eob + 1; +} + +#if CONFIG_TX64X64 +void aom_highbd_quantize_dc_64x64(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr, + const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr) { + const int n_coeffs = 1024; + int eob = -1; + int dequant; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + const int coeff = coeff_ptr[0]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[0], 2); + const uint32_t abs_qcoeff = + (uint32_t)((tmp * qm_ptr[0] * quant) >> (14 + AOM_QM_BITS)); + qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dequant = + (dequant_ptr * iqm_ptr[0] + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS; + dqcoeff_ptr[0] = (qcoeff_ptr[0] * dequant) / 4; + if (abs_qcoeff) eob = 0; + } + *eob_ptr = eob + 1; +} +#endif // CONFIG_TX64X64 + +void aom_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr) { + int i, non_zero_count = (int)n_coeffs, eob = -1; + const int zbins[2] = { zbin_ptr[0], zbin_ptr[1] }; + const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; + int dequant; + (void)iscan; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + // Pre-scan pass + for (i = (int)n_coeffs - 1; i >= 0; i--) { + const int rc = scan[i]; + const qm_val_t wt = qm_ptr[rc]; + const int coeff = coeff_ptr[rc] * wt; + + if (coeff < (zbins[rc != 0] << AOM_QM_BITS) && + coeff > (nzbins[rc != 0] << AOM_QM_BITS)) + non_zero_count--; + else + break; + } + + // Quantization pass: All coefficients with index >= zero_flag are + // skippable. Note: zero_flag can be zero. + for (i = 0; i < non_zero_count; i++) { + const int rc = scan[i]; + const int coeff = coeff_ptr[rc]; + const qm_val_t wt = qm_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + + if (abs_coeff * wt >= (zbins[rc != 0] << AOM_QM_BITS)) { + const int64_t tmp1 = abs_coeff + round_ptr[rc != 0]; + const int64_t tmpw = tmp1 * wt; + const int64_t tmp2 = ((tmpw * quant_ptr[rc != 0]) >> 16) + tmpw; + const uint32_t abs_qcoeff = + (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> (16 + AOM_QM_BITS)); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dequant = + (dequant_ptr[rc != 0] * iqm_ptr[rc] + (1 << (AOM_QM_BITS - 1))) >> + AOM_QM_BITS; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant; + if (abs_qcoeff) eob = i; + } + } + } + *eob_ptr = eob + 1; +} + +void aom_highbd_quantize_b_32x32_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr) { + const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], 1), + ROUND_POWER_OF_TWO(zbin_ptr[1], 1) }; + const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; + + int idx = 0; + int idx_arr[1024]; + int i, eob = -1; + int dequant; + (void)iscan; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + // Pre-scan pass + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + const qm_val_t wt = qm_ptr[rc]; + const int coeff = coeff_ptr[rc] * wt; + + // If the coefficient is out of the base ZBIN range, keep it for + // quantization. + if (coeff >= (zbins[rc != 0] << AOM_QM_BITS) || + coeff <= (nzbins[rc != 0] << AOM_QM_BITS)) + idx_arr[idx++] = i; + } + + // Quantization pass: only process the coefficients selected in + // pre-scan pass. Note: idx can be zero. + for (i = 0; i < idx; i++) { + const int rc = scan[idx_arr[i]]; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const qm_val_t wt = qm_ptr[rc]; + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp1 = + abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1); + const int64_t tmpw = tmp1 * wt; + const int64_t tmp2 = ((tmpw * quant_ptr[rc != 0]) >> 16) + tmpw; + const uint32_t abs_qcoeff = + (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> (15 + AOM_QM_BITS)); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dequant = + (dequant_ptr[rc != 0] * iqm_ptr[rc] + (1 << (AOM_QM_BITS - 1))) >> + AOM_QM_BITS; + dqcoeff_ptr[rc] = (qcoeff_ptr[rc] * dequant) / 2; + if (abs_qcoeff) eob = idx_arr[i]; + } + } + *eob_ptr = eob + 1; +} + +#if CONFIG_TX64X64 +void aom_highbd_quantize_b_64x64_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr) { + const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], 2), + ROUND_POWER_OF_TWO(zbin_ptr[1], 2) }; + const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; + + int idx = 0; + int idx_arr[4096]; + int i, eob = -1; + int dequant; + (void)iscan; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + // Pre-scan pass + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + const qm_val_t wt = qm_ptr[rc]; + const int coeff = coeff_ptr[rc] * wt; + + // If the coefficient is out of the base ZBIN range, keep it for + // quantization. + if (coeff >= (zbins[rc != 0] << AOM_QM_BITS) || + coeff <= (nzbins[rc != 0] << AOM_QM_BITS)) + idx_arr[idx++] = i; + } + + // Quantization pass: only process the coefficients selected in + // pre-scan pass. Note: idx can be zero. + for (i = 0; i < idx; i++) { + const int rc = scan[idx_arr[i]]; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const qm_val_t wt = qm_ptr[rc]; + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp1 = + abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 2); + const int64_t tmpw = tmp1 * wt; + const int64_t tmp2 = ((tmpw * quant_ptr[rc != 0]) >> 16) + tmpw; + const uint32_t abs_qcoeff = + (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> (14 + AOM_QM_BITS)); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dequant = + (dequant_ptr[rc != 0] * iqm_ptr[rc] + (1 << (AOM_QM_BITS - 1))) >> + AOM_QM_BITS; + dqcoeff_ptr[rc] = (qcoeff_ptr[rc] * dequant) / 4; + if (abs_qcoeff) eob = idx_arr[i]; + } + } + *eob_ptr = eob + 1; +} +#endif // CONFIG_TX64X64 +#endif // CONFIG_HIGHBITDEPTH + +#else // CONFIG_AOM_QM + +void aom_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, int skip_block, + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr) { + const int rc = 0; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int tmp, eob = -1; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX); + tmp = (tmp * quant) >> 16; + qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr; + if (tmp) eob = 0; + } + *eob_ptr = eob + 1; +} + +void aom_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr) { + const int n_coeffs = 1024; + const int rc = 0; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int tmp, eob = -1; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1), + INT16_MIN, INT16_MAX); + tmp = (tmp * quant) >> 15; + qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr / 2; + if (tmp) eob = 0; + } + *eob_ptr = eob + 1; +} + +#if CONFIG_TX64X64 +void aom_quantize_dc_64x64(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr) { + const int n_coeffs = 4096; + const int rc = 0; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int tmp, eob = -1; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 2), + INT16_MIN, INT16_MAX); + tmp = (tmp * quant) >> 14; + qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr / 4; + if (tmp) eob = 0; + } + *eob_ptr = eob + 1; +} +#endif // CONFIG_TX64X64 + +#if CONFIG_HIGHBITDEPTH +void aom_highbd_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, + int skip_block, const int16_t *round_ptr, + const int16_t quant, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t dequant_ptr, + uint16_t *eob_ptr) { + int eob = -1; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + const int coeff = coeff_ptr[0]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp = abs_coeff + round_ptr[0]; + const uint32_t abs_qcoeff = (uint32_t)((tmp * quant) >> 16); + qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant_ptr; + if (abs_qcoeff) eob = 0; + } + *eob_ptr = eob + 1; +} + +void aom_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, + uint16_t *eob_ptr) { + const int n_coeffs = 1024; + int eob = -1; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + const int coeff = coeff_ptr[0]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[0], 1); + const uint32_t abs_qcoeff = (uint32_t)((tmp * quant) >> 15); + qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant_ptr / 2; + if (abs_qcoeff) eob = 0; + } + *eob_ptr = eob + 1; +} + +#if CONFIG_TX64X64 +void aom_highbd_quantize_dc_64x64(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, + uint16_t *eob_ptr) { + const int n_coeffs = 4096; + int eob = -1; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + const int coeff = coeff_ptr[0]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[0], 2); + const uint32_t abs_qcoeff = (uint32_t)((tmp * quant) >> 14); + qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant_ptr / 4; + if (abs_qcoeff) eob = 0; + } + *eob_ptr = eob + 1; +} +#endif // CONFIG_TX64X64 + +void aom_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + int i, non_zero_count = (int)n_coeffs, eob = -1; + const int zbins[2] = { zbin_ptr[0], zbin_ptr[1] }; + const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; + (void)iscan; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + // Pre-scan pass + for (i = (int)n_coeffs - 1; i >= 0; i--) { + const int rc = scan[i]; + const int coeff = coeff_ptr[rc]; + + if (coeff < zbins[rc != 0] && coeff > nzbins[rc != 0]) + non_zero_count--; + else + break; + } + + // Quantization pass: All coefficients with index >= zero_flag are + // skippable. Note: zero_flag can be zero. + for (i = 0; i < non_zero_count; i++) { + const int rc = scan[i]; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + + if (abs_coeff >= zbins[rc != 0]) { + const int64_t tmp1 = abs_coeff + round_ptr[rc != 0]; + const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; + const uint32_t abs_qcoeff = + (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 16); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0]; + if (abs_qcoeff) eob = i; + } + } + } + *eob_ptr = eob + 1; +} + +void aom_highbd_quantize_b_32x32_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], 1), + ROUND_POWER_OF_TWO(zbin_ptr[1], 1) }; + const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; + + int idx = 0; + int idx_arr[1024]; + int i, eob = -1; + (void)iscan; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + // Pre-scan pass + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + const int coeff = coeff_ptr[rc]; + + // If the coefficient is out of the base ZBIN range, keep it for + // quantization. + if (coeff >= zbins[rc != 0] || coeff <= nzbins[rc != 0]) + idx_arr[idx++] = i; + } + + // Quantization pass: only process the coefficients selected in + // pre-scan pass. Note: idx can be zero. + for (i = 0; i < idx; i++) { + const int rc = scan[idx_arr[i]]; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp1 = + abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1); + const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; + const uint32_t abs_qcoeff = + (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2; + if (abs_qcoeff) eob = idx_arr[i]; + } + } + *eob_ptr = eob + 1; +} + +#if CONFIG_TX64X64 +void aom_highbd_quantize_b_64x64_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], 2), + ROUND_POWER_OF_TWO(zbin_ptr[1], 2) }; + const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; + + int idx = 0; + int idx_arr[4096]; + int i, eob = -1; + (void)iscan; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + // Pre-scan pass + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + const int coeff = coeff_ptr[rc]; + + // If the coefficient is out of the base ZBIN range, keep it for + // quantization. + if (coeff >= zbins[rc != 0] || coeff <= nzbins[rc != 0]) + idx_arr[idx++] = i; + } + + // Quantization pass: only process the coefficients selected in + // pre-scan pass. Note: idx can be zero. + for (i = 0; i < idx; i++) { + const int rc = scan[idx_arr[i]]; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp1 = + abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 2); + const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; + const uint32_t abs_qcoeff = + (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 14); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 4; + if (abs_qcoeff) eob = idx_arr[i]; + } + } + *eob_ptr = eob + 1; +} +#endif // CONFIG_TX64X64 +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_AOM_QM diff --git a/third_party/aom/aom_dsp/quantize.h b/third_party/aom/aom_dsp/quantize.h new file mode 100644 index 0000000000..fe49b830f1 --- /dev/null +++ b/third_party/aom/aom_dsp/quantize.h @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_QUANTIZE_H_ +#define AOM_DSP_QUANTIZE_H_ + +#include "./aom_config.h" +#include "aom_dsp/aom_dsp_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if CONFIG_AOM_QM +void aom_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, int skip_block, + const int16_t *round_ptr, const int16_t quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr, + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr); +void aom_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr, + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr); +#if CONFIG_TX64X64 +void aom_quantize_dc_64x64(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr, + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr); +#endif // CONFIG_TX64X64 +void aom_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const int16_t *iscan, const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr); +#if CONFIG_HIGHBITDEPTH +void aom_highbd_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, + int skip_block, const int16_t *round_ptr, + const int16_t quant_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t dequant_ptr, + uint16_t *eob_ptr, const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr); +void aom_highbd_quantize_dc_32x32( + const tran_low_t *coeff_ptr, int skip_block, const int16_t *round_ptr, + const int16_t quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr, const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr); +#if CONFIG_TX64X64 +void aom_highbd_quantize_dc_64x64( + const tran_low_t *coeff_ptr, int skip_block, const int16_t *round_ptr, + const int16_t quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr, const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr); +#endif // CONFIG_TX64X64 +void aom_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr); +#endif // CONFIG_HIGHBITDEPTH + +#else // CONFIG_AOM_QM + +void aom_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, int skip_block, + const int16_t *round_ptr, const int16_t quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr); +void aom_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr); +#if CONFIG_TX64X64 +void aom_quantize_dc_64x64(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr); +#endif // CONFIG_TX64X64 +#if CONFIG_HIGHBITDEPTH +void aom_highbd_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, + int skip_block, const int16_t *round_ptr, + const int16_t quant_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t dequant_ptr, + uint16_t *eob_ptr); +void aom_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, + const int16_t quant_ptr, + tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr); +#if CONFIG_TX64X64 +void aom_highbd_quantize_dc_64x64(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, + const int16_t quant_ptr, + tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, + const int16_t dequant_ptr, uint16_t *eob_ptr); +#endif // CONFIG_TX64X64 +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_AOM_QM + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_QUANTIZE_H_ diff --git a/third_party/aom/aom_dsp/sad.c b/third_party/aom/aom_dsp/sad.c new file mode 100644 index 0000000000..3e10705195 --- /dev/null +++ b/third_party/aom/aom_dsp/sad.c @@ -0,0 +1,512 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" + +/* Sum the difference between every corresponding element of the buffers. */ +static INLINE unsigned int sad(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int width, int height) { + int y, x; + unsigned int sad = 0; + + for (y = 0; y < height; y++) { + for (x = 0; x < width; x++) sad += abs(a[x] - b[x]); + + a += a_stride; + b += b_stride; + } + return sad; +} + +#define sadMxN(m, n) \ + unsigned int aom_sad##m##x##n##_c(const uint8_t *src, int src_stride, \ + const uint8_t *ref, int ref_stride) { \ + return sad(src, src_stride, ref, ref_stride, m, n); \ + } \ + unsigned int aom_sad##m##x##n##_avg_c(const uint8_t *src, int src_stride, \ + const uint8_t *ref, int ref_stride, \ + const uint8_t *second_pred) { \ + uint8_t comp_pred[m * n]; \ + aom_comp_avg_pred_c(comp_pred, second_pred, m, n, ref, ref_stride); \ + return sad(src, src_stride, comp_pred, m, m, n); \ + } + +// depending on call sites, pass **ref_array to avoid & in subsequent call and +// de-dup with 4D below. +#define sadMxNxK(m, n, k) \ + void aom_sad##m##x##n##x##k##_c(const uint8_t *src, int src_stride, \ + const uint8_t *ref_array, int ref_stride, \ + uint32_t *sad_array) { \ + int i; \ + for (i = 0; i < k; ++i) \ + sad_array[i] = \ + aom_sad##m##x##n##_c(src, src_stride, &ref_array[i], ref_stride); \ + } + +// This appears to be equivalent to the above when k == 4 and refs is const +#define sadMxNx4D(m, n) \ + void aom_sad##m##x##n##x4d_c(const uint8_t *src, int src_stride, \ + const uint8_t *const ref_array[], \ + int ref_stride, uint32_t *sad_array) { \ + int i; \ + for (i = 0; i < 4; ++i) \ + sad_array[i] = \ + aom_sad##m##x##n##_c(src, src_stride, ref_array[i], ref_stride); \ + } + +/* clang-format off */ +#if CONFIG_AV1 && CONFIG_EXT_PARTITION +// 128x128 +sadMxN(128, 128) +sadMxNxK(128, 128, 3) +sadMxNxK(128, 128, 8) +sadMxNx4D(128, 128) + +// 128x64 +sadMxN(128, 64) +sadMxNx4D(128, 64) + +// 64x128 +sadMxN(64, 128) +sadMxNx4D(64, 128) +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + +// 64x64 +sadMxN(64, 64) +sadMxNxK(64, 64, 3) +sadMxNxK(64, 64, 8) +sadMxNx4D(64, 64) + +// 64x32 +sadMxN(64, 32) +sadMxNx4D(64, 32) + +// 32x64 +sadMxN(32, 64) +sadMxNx4D(32, 64) + +// 32x32 +sadMxN(32, 32) +sadMxNxK(32, 32, 3) +sadMxNxK(32, 32, 8) +sadMxNx4D(32, 32) + +// 32x16 +sadMxN(32, 16) +sadMxNx4D(32, 16) + +// 16x32 +sadMxN(16, 32) +sadMxNx4D(16, 32) + +// 16x16 +sadMxN(16, 16) +sadMxNxK(16, 16, 3) +sadMxNxK(16, 16, 8) +sadMxNx4D(16, 16) + +// 16x8 +sadMxN(16, 8) +sadMxNxK(16, 8, 3) +sadMxNxK(16, 8, 8) +sadMxNx4D(16, 8) + +// 8x16 +sadMxN(8, 16) +sadMxNxK(8, 16, 3) +sadMxNxK(8, 16, 8) +sadMxNx4D(8, 16) + +// 8x8 +sadMxN(8, 8) +sadMxNxK(8, 8, 3) +sadMxNxK(8, 8, 8) +sadMxNx4D(8, 8) + +// 8x4 +sadMxN(8, 4) +sadMxNxK(8, 4, 8) +sadMxNx4D(8, 4) + +// 4x8 +sadMxN(4, 8) +sadMxNxK(4, 8, 8) +sadMxNx4D(4, 8) + +// 4x4 +sadMxN(4, 4) +sadMxNxK(4, 4, 3) +sadMxNxK(4, 4, 8) +sadMxNx4D(4, 4) +/* clang-format on */ + +#if CONFIG_HIGHBITDEPTH + static INLINE + unsigned int highbd_sad(const uint8_t *a8, int a_stride, const uint8_t *b8, + int b_stride, int width, int height) { + int y, x; + unsigned int sad = 0; + const uint16_t *a = CONVERT_TO_SHORTPTR(a8); + const uint16_t *b = CONVERT_TO_SHORTPTR(b8); + for (y = 0; y < height; y++) { + for (x = 0; x < width; x++) sad += abs(a[x] - b[x]); + + a += a_stride; + b += b_stride; + } + return sad; +} + +static INLINE unsigned int highbd_sadb(const uint8_t *a8, int a_stride, + const uint16_t *b, int b_stride, + int width, int height) { + int y, x; + unsigned int sad = 0; + const uint16_t *a = CONVERT_TO_SHORTPTR(a8); + for (y = 0; y < height; y++) { + for (x = 0; x < width; x++) sad += abs(a[x] - b[x]); + + a += a_stride; + b += b_stride; + } + return sad; +} + +#define highbd_sadMxN(m, n) \ + unsigned int aom_highbd_sad##m##x##n##_c(const uint8_t *src, int src_stride, \ + const uint8_t *ref, \ + int ref_stride) { \ + return highbd_sad(src, src_stride, ref, ref_stride, m, n); \ + } \ + unsigned int aom_highbd_sad##m##x##n##_avg_c( \ + const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ + const uint8_t *second_pred) { \ + uint16_t comp_pred[m * n]; \ + aom_highbd_comp_avg_pred_c(comp_pred, second_pred, m, n, ref, ref_stride); \ + return highbd_sadb(src, src_stride, comp_pred, m, m, n); \ + } + +#define highbd_sadMxNxK(m, n, k) \ + void aom_highbd_sad##m##x##n##x##k##_c( \ + const uint8_t *src, int src_stride, const uint8_t *ref_array, \ + int ref_stride, uint32_t *sad_array) { \ + int i; \ + for (i = 0; i < k; ++i) { \ + sad_array[i] = aom_highbd_sad##m##x##n##_c(src, src_stride, \ + &ref_array[i], ref_stride); \ + } \ + } + +#define highbd_sadMxNx4D(m, n) \ + void aom_highbd_sad##m##x##n##x4d_c(const uint8_t *src, int src_stride, \ + const uint8_t *const ref_array[], \ + int ref_stride, uint32_t *sad_array) { \ + int i; \ + for (i = 0; i < 4; ++i) { \ + sad_array[i] = aom_highbd_sad##m##x##n##_c(src, src_stride, \ + ref_array[i], ref_stride); \ + } \ + } + +/* clang-format off */ +#if CONFIG_AV1 && CONFIG_EXT_PARTITION +// 128x128 +highbd_sadMxN(128, 128) +highbd_sadMxNxK(128, 128, 3) +highbd_sadMxNxK(128, 128, 8) +highbd_sadMxNx4D(128, 128) + +// 128x64 +highbd_sadMxN(128, 64) +highbd_sadMxNx4D(128, 64) + +// 64x128 +highbd_sadMxN(64, 128) +highbd_sadMxNx4D(64, 128) +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + +// 64x64 +highbd_sadMxN(64, 64) +highbd_sadMxNxK(64, 64, 3) +highbd_sadMxNxK(64, 64, 8) +highbd_sadMxNx4D(64, 64) + +// 64x32 +highbd_sadMxN(64, 32) +highbd_sadMxNx4D(64, 32) + +// 32x64 +highbd_sadMxN(32, 64) +highbd_sadMxNx4D(32, 64) + +// 32x32 +highbd_sadMxN(32, 32) +highbd_sadMxNxK(32, 32, 3) +highbd_sadMxNxK(32, 32, 8) +highbd_sadMxNx4D(32, 32) + +// 32x16 +highbd_sadMxN(32, 16) +highbd_sadMxNx4D(32, 16) + +// 16x32 +highbd_sadMxN(16, 32) +highbd_sadMxNx4D(16, 32) + +// 16x16 +highbd_sadMxN(16, 16) +highbd_sadMxNxK(16, 16, 3) +highbd_sadMxNxK(16, 16, 8) +highbd_sadMxNx4D(16, 16) + +// 16x8 +highbd_sadMxN(16, 8) +highbd_sadMxNxK(16, 8, 3) +highbd_sadMxNxK(16, 8, 8) +highbd_sadMxNx4D(16, 8) + +// 8x16 +highbd_sadMxN(8, 16) +highbd_sadMxNxK(8, 16, 3) +highbd_sadMxNxK(8, 16, 8) +highbd_sadMxNx4D(8, 16) + +// 8x8 +highbd_sadMxN(8, 8) +highbd_sadMxNxK(8, 8, 3) +highbd_sadMxNxK(8, 8, 8) +highbd_sadMxNx4D(8, 8) + +// 8x4 +highbd_sadMxN(8, 4) +highbd_sadMxNxK(8, 4, 8) +highbd_sadMxNx4D(8, 4) + +// 4x8 +highbd_sadMxN(4, 8) +highbd_sadMxNxK(4, 8, 8) +highbd_sadMxNx4D(4, 8) + +// 4x4 +highbd_sadMxN(4, 4) +highbd_sadMxNxK(4, 4, 3) +highbd_sadMxNxK(4, 4, 8) +highbd_sadMxNx4D(4, 4) +/* clang-format on */ +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_AV1 && CONFIG_EXT_INTER + static INLINE + unsigned int masked_sad(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, const uint8_t *m, int m_stride, + int width, int height) { + int y, x; + unsigned int sad = 0; + + for (y = 0; y < height; y++) { + for (x = 0; x < width; x++) sad += m[x] * abs(a[x] - b[x]); + + a += a_stride; + b += b_stride; + m += m_stride; + } + sad = (sad + 31) >> 6; + + return sad; +} + +#define MASKSADMxN(m, n) \ + unsigned int aom_masked_sad##m##x##n##_c( \ + const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ + const uint8_t *msk, int msk_stride) { \ + return masked_sad(src, src_stride, ref, ref_stride, msk, msk_stride, m, \ + n); \ + } + +/* clang-format off */ +#if CONFIG_EXT_PARTITION +MASKSADMxN(128, 128) +MASKSADMxN(128, 64) +MASKSADMxN(64, 128) +#endif // CONFIG_EXT_PARTITION +MASKSADMxN(64, 64) +MASKSADMxN(64, 32) +MASKSADMxN(32, 64) +MASKSADMxN(32, 32) +MASKSADMxN(32, 16) +MASKSADMxN(16, 32) +MASKSADMxN(16, 16) +MASKSADMxN(16, 8) +MASKSADMxN(8, 16) +MASKSADMxN(8, 8) +MASKSADMxN(8, 4) +MASKSADMxN(4, 8) +MASKSADMxN(4, 4) +/* clang-format on */ + +#if CONFIG_HIGHBITDEPTH + static INLINE + unsigned int highbd_masked_sad(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, + const uint8_t *m, int m_stride, int width, + int height) { + int y, x; + unsigned int sad = 0; + const uint16_t *a = CONVERT_TO_SHORTPTR(a8); + const uint16_t *b = CONVERT_TO_SHORTPTR(b8); + + for (y = 0; y < height; y++) { + for (x = 0; x < width; x++) sad += m[x] * abs(a[x] - b[x]); + + a += a_stride; + b += b_stride; + m += m_stride; + } + sad = (sad + 31) >> 6; + + return sad; +} + +#define HIGHBD_MASKSADMXN(m, n) \ + unsigned int aom_highbd_masked_sad##m##x##n##_c( \ + const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ + const uint8_t *msk, int msk_stride) { \ + return highbd_masked_sad(src, src_stride, ref, ref_stride, msk, \ + msk_stride, m, n); \ + } + +#if CONFIG_EXT_PARTITION +HIGHBD_MASKSADMXN(128, 128) +HIGHBD_MASKSADMXN(128, 64) +HIGHBD_MASKSADMXN(64, 128) +#endif // CONFIG_EXT_PARTITION +HIGHBD_MASKSADMXN(64, 64) +HIGHBD_MASKSADMXN(64, 32) +HIGHBD_MASKSADMXN(32, 64) +HIGHBD_MASKSADMXN(32, 32) +HIGHBD_MASKSADMXN(32, 16) +HIGHBD_MASKSADMXN(16, 32) +HIGHBD_MASKSADMXN(16, 16) +HIGHBD_MASKSADMXN(16, 8) +HIGHBD_MASKSADMXN(8, 16) +HIGHBD_MASKSADMXN(8, 8) +HIGHBD_MASKSADMXN(8, 4) +HIGHBD_MASKSADMXN(4, 8) +HIGHBD_MASKSADMXN(4, 4) +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_AV1 && CONFIG_EXT_INTER + +#if CONFIG_AV1 && CONFIG_MOTION_VAR +// pre: predictor being evaluated +// wsrc: target weighted prediction (has been *4096 to keep precision) +// mask: 2d weights (scaled by 4096) +static INLINE unsigned int obmc_sad(const uint8_t *pre, int pre_stride, + const int32_t *wsrc, const int32_t *mask, + int width, int height) { + int y, x; + unsigned int sad = 0; + + for (y = 0; y < height; y++) { + for (x = 0; x < width; x++) + sad += ROUND_POWER_OF_TWO(abs(wsrc[x] - pre[x] * mask[x]), 12); + + pre += pre_stride; + wsrc += width; + mask += width; + } + + return sad; +} + +#define OBMCSADMxN(m, n) \ + unsigned int aom_obmc_sad##m##x##n##_c(const uint8_t *ref, int ref_stride, \ + const int32_t *wsrc, \ + const int32_t *mask) { \ + return obmc_sad(ref, ref_stride, wsrc, mask, m, n); \ + } + +/* clang-format off */ +#if CONFIG_EXT_PARTITION +OBMCSADMxN(128, 128) +OBMCSADMxN(128, 64) +OBMCSADMxN(64, 128) +#endif // CONFIG_EXT_PARTITION +OBMCSADMxN(64, 64) +OBMCSADMxN(64, 32) +OBMCSADMxN(32, 64) +OBMCSADMxN(32, 32) +OBMCSADMxN(32, 16) +OBMCSADMxN(16, 32) +OBMCSADMxN(16, 16) +OBMCSADMxN(16, 8) +OBMCSADMxN(8, 16) +OBMCSADMxN(8, 8) +OBMCSADMxN(8, 4) +OBMCSADMxN(4, 8) +OBMCSADMxN(4, 4) +/* clang-format on */ + +#if CONFIG_HIGHBITDEPTH + static INLINE + unsigned int highbd_obmc_sad(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, const int32_t *mask, + int width, int height) { + int y, x; + unsigned int sad = 0; + const uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); + + for (y = 0; y < height; y++) { + for (x = 0; x < width; x++) + sad += ROUND_POWER_OF_TWO(abs(wsrc[x] - pre[x] * mask[x]), 12); + + pre += pre_stride; + wsrc += width; + mask += width; + } + + return sad; +} + +#define HIGHBD_OBMCSADMXN(m, n) \ + unsigned int aom_highbd_obmc_sad##m##x##n##_c( \ + const uint8_t *ref, int ref_stride, const int32_t *wsrc, \ + const int32_t *mask) { \ + return highbd_obmc_sad(ref, ref_stride, wsrc, mask, m, n); \ + } + +/* clang-format off */ +#if CONFIG_EXT_PARTITION +HIGHBD_OBMCSADMXN(128, 128) +HIGHBD_OBMCSADMXN(128, 64) +HIGHBD_OBMCSADMXN(64, 128) +#endif // CONFIG_EXT_PARTITION +HIGHBD_OBMCSADMXN(64, 64) +HIGHBD_OBMCSADMXN(64, 32) +HIGHBD_OBMCSADMXN(32, 64) +HIGHBD_OBMCSADMXN(32, 32) +HIGHBD_OBMCSADMXN(32, 16) +HIGHBD_OBMCSADMXN(16, 32) +HIGHBD_OBMCSADMXN(16, 16) +HIGHBD_OBMCSADMXN(16, 8) +HIGHBD_OBMCSADMXN(8, 16) +HIGHBD_OBMCSADMXN(8, 8) +HIGHBD_OBMCSADMXN(8, 4) +HIGHBD_OBMCSADMXN(4, 8) +HIGHBD_OBMCSADMXN(4, 4) +/* clang-format on */ +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_AV1 && CONFIG_MOTION_VAR diff --git a/third_party/aom/aom_dsp/simd/v128_intrinsics.h b/third_party/aom/aom_dsp/simd/v128_intrinsics.h new file mode 100644 index 0000000000..8f6509383a --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v128_intrinsics.h @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V128_INTRINSICS_H +#define _V128_INTRINSICS_H + +#include +#include +#include +#include "./v128_intrinsics_c.h" +#include "./v64_intrinsics.h" + +/* Fallback to plain, unoptimised C. */ + +typedef c_v128 v128; + +SIMD_INLINE uint32_t v128_low_u32(v128 a) { return c_v128_low_u32(a); } +SIMD_INLINE v64 v128_low_v64(v128 a) { return c_v128_low_v64(a); } +SIMD_INLINE v64 v128_high_v64(v128 a) { return c_v128_high_v64(a); } +SIMD_INLINE v128 v128_from_64(uint64_t hi, uint64_t lo) { + return c_v128_from_64(hi, lo); +} +SIMD_INLINE v128 v128_from_v64(v64 hi, v64 lo) { + return c_v128_from_v64(hi, lo); +} +SIMD_INLINE v128 v128_from_32(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + return c_v128_from_32(a, b, c, d); +} + +SIMD_INLINE v128 v128_load_unaligned(const void *p) { + return c_v128_load_unaligned(p); +} +SIMD_INLINE v128 v128_load_aligned(const void *p) { + return c_v128_load_aligned(p); +} + +SIMD_INLINE void v128_store_unaligned(void *p, v128 a) { + c_v128_store_unaligned(p, a); +} +SIMD_INLINE void v128_store_aligned(void *p, v128 a) { + c_v128_store_aligned(p, a); +} + +SIMD_INLINE v128 v128_align(v128 a, v128 b, unsigned int c) { + return c_v128_align(a, b, c); +} + +SIMD_INLINE v128 v128_zero() { return c_v128_zero(); } +SIMD_INLINE v128 v128_dup_8(uint8_t x) { return c_v128_dup_8(x); } +SIMD_INLINE v128 v128_dup_16(uint16_t x) { return c_v128_dup_16(x); } +SIMD_INLINE v128 v128_dup_32(uint32_t x) { return c_v128_dup_32(x); } + +typedef uint32_t sad128_internal; +SIMD_INLINE sad128_internal v128_sad_u8_init() { return c_v128_sad_u8_init(); } +SIMD_INLINE sad128_internal v128_sad_u8(sad128_internal s, v128 a, v128 b) { + return c_v128_sad_u8(s, a, b); +} +SIMD_INLINE uint32_t v128_sad_u8_sum(sad128_internal s) { + return c_v128_sad_u8_sum(s); +} +typedef uint32_t ssd128_internal; +SIMD_INLINE ssd128_internal v128_ssd_u8_init() { return c_v128_ssd_u8_init(); } +SIMD_INLINE ssd128_internal v128_ssd_u8(ssd128_internal s, v128 a, v128 b) { + return c_v128_ssd_u8(s, a, b); +} +SIMD_INLINE uint32_t v128_ssd_u8_sum(ssd128_internal s) { + return c_v128_ssd_u8_sum(s); +} +SIMD_INLINE int64_t v128_dotp_s16(v128 a, v128 b) { + return c_v128_dotp_s16(a, b); +} +SIMD_INLINE uint64_t v128_hadd_u8(v128 a) { return c_v128_hadd_u8(a); } + +SIMD_INLINE v128 v128_or(v128 a, v128 b) { return c_v128_or(a, b); } +SIMD_INLINE v128 v128_xor(v128 a, v128 b) { return c_v128_xor(a, b); } +SIMD_INLINE v128 v128_and(v128 a, v128 b) { return c_v128_and(a, b); } +SIMD_INLINE v128 v128_andn(v128 a, v128 b) { return c_v128_andn(a, b); } + +SIMD_INLINE v128 v128_add_8(v128 a, v128 b) { return c_v128_add_8(a, b); } +SIMD_INLINE v128 v128_add_16(v128 a, v128 b) { return c_v128_add_16(a, b); } +SIMD_INLINE v128 v128_sadd_s16(v128 a, v128 b) { return c_v128_sadd_s16(a, b); } +SIMD_INLINE v128 v128_add_32(v128 a, v128 b) { return c_v128_add_32(a, b); } +SIMD_INLINE v128 v128_padd_s16(v128 a) { return c_v128_padd_s16(a); } +SIMD_INLINE v128 v128_sub_8(v128 a, v128 b) { return c_v128_sub_8(a, b); } +SIMD_INLINE v128 v128_ssub_u8(v128 a, v128 b) { return c_v128_ssub_u8(a, b); } +SIMD_INLINE v128 v128_ssub_s8(v128 a, v128 b) { return c_v128_ssub_s8(a, b); } +SIMD_INLINE v128 v128_sub_16(v128 a, v128 b) { return c_v128_sub_16(a, b); } +SIMD_INLINE v128 v128_ssub_s16(v128 a, v128 b) { return c_v128_ssub_s16(a, b); } +SIMD_INLINE v128 v128_ssub_u16(v128 a, v128 b) { return c_v128_ssub_u16(a, b); } +SIMD_INLINE v128 v128_sub_32(v128 a, v128 b) { return c_v128_sub_32(a, b); } +SIMD_INLINE v128 v128_abs_s16(v128 a) { return c_v128_abs_s16(a); } +SIMD_INLINE v128 v128_abs_s8(v128 a) { return c_v128_abs_s8(a); } + +SIMD_INLINE v128 v128_mul_s16(v64 a, v64 b) { return c_v128_mul_s16(a, b); } +SIMD_INLINE v128 v128_mullo_s16(v128 a, v128 b) { + return c_v128_mullo_s16(a, b); +} +SIMD_INLINE v128 v128_mulhi_s16(v128 a, v128 b) { + return c_v128_mulhi_s16(a, b); +} +SIMD_INLINE v128 v128_mullo_s32(v128 a, v128 b) { + return c_v128_mullo_s32(a, b); +} +SIMD_INLINE v128 v128_madd_s16(v128 a, v128 b) { return c_v128_madd_s16(a, b); } +SIMD_INLINE v128 v128_madd_us8(v128 a, v128 b) { return c_v128_madd_us8(a, b); } + +SIMD_INLINE v128 v128_avg_u8(v128 a, v128 b) { return c_v128_avg_u8(a, b); } +SIMD_INLINE v128 v128_rdavg_u8(v128 a, v128 b) { return c_v128_rdavg_u8(a, b); } +SIMD_INLINE v128 v128_avg_u16(v128 a, v128 b) { return c_v128_avg_u16(a, b); } +SIMD_INLINE v128 v128_min_u8(v128 a, v128 b) { return c_v128_min_u8(a, b); } +SIMD_INLINE v128 v128_max_u8(v128 a, v128 b) { return c_v128_max_u8(a, b); } +SIMD_INLINE v128 v128_min_s8(v128 a, v128 b) { return c_v128_min_s8(a, b); } +SIMD_INLINE v128 v128_max_s8(v128 a, v128 b) { return c_v128_max_s8(a, b); } +SIMD_INLINE v128 v128_min_s16(v128 a, v128 b) { return c_v128_min_s16(a, b); } +SIMD_INLINE v128 v128_max_s16(v128 a, v128 b) { return c_v128_max_s16(a, b); } + +SIMD_INLINE v128 v128_ziplo_8(v128 a, v128 b) { return c_v128_ziplo_8(a, b); } +SIMD_INLINE v128 v128_ziphi_8(v128 a, v128 b) { return c_v128_ziphi_8(a, b); } +SIMD_INLINE v128 v128_ziplo_16(v128 a, v128 b) { return c_v128_ziplo_16(a, b); } +SIMD_INLINE v128 v128_ziphi_16(v128 a, v128 b) { return c_v128_ziphi_16(a, b); } +SIMD_INLINE v128 v128_ziplo_32(v128 a, v128 b) { return c_v128_ziplo_32(a, b); } +SIMD_INLINE v128 v128_ziphi_32(v128 a, v128 b) { return c_v128_ziphi_32(a, b); } +SIMD_INLINE v128 v128_ziplo_64(v128 a, v128 b) { return c_v128_ziplo_64(a, b); } +SIMD_INLINE v128 v128_ziphi_64(v128 a, v128 b) { return c_v128_ziphi_64(a, b); } +SIMD_INLINE v128 v128_zip_8(v64 a, v64 b) { return c_v128_zip_8(a, b); } +SIMD_INLINE v128 v128_zip_16(v64 a, v64 b) { return c_v128_zip_16(a, b); } +SIMD_INLINE v128 v128_zip_32(v64 a, v64 b) { return c_v128_zip_32(a, b); } +SIMD_INLINE v128 v128_unziplo_8(v128 a, v128 b) { + return c_v128_unziplo_8(a, b); +} +SIMD_INLINE v128 v128_unziphi_8(v128 a, v128 b) { + return c_v128_unziphi_8(a, b); +} +SIMD_INLINE v128 v128_unziplo_16(v128 a, v128 b) { + return c_v128_unziplo_16(a, b); +} +SIMD_INLINE v128 v128_unziphi_16(v128 a, v128 b) { + return c_v128_unziphi_16(a, b); +} +SIMD_INLINE v128 v128_unziplo_32(v128 a, v128 b) { + return c_v128_unziplo_32(a, b); +} +SIMD_INLINE v128 v128_unziphi_32(v128 a, v128 b) { + return c_v128_unziphi_32(a, b); +} +SIMD_INLINE v128 v128_unpack_u8_s16(v64 a) { return c_v128_unpack_u8_s16(a); } +SIMD_INLINE v128 v128_unpacklo_u8_s16(v128 a) { + return c_v128_unpacklo_u8_s16(a); +} +SIMD_INLINE v128 v128_unpackhi_u8_s16(v128 a) { + return c_v128_unpackhi_u8_s16(a); +} +SIMD_INLINE v128 v128_unpack_s8_s16(v64 a) { return c_v128_unpack_s8_s16(a); } +SIMD_INLINE v128 v128_unpacklo_s8_s16(v128 a) { + return c_v128_unpacklo_s8_s16(a); +} +SIMD_INLINE v128 v128_unpackhi_s8_s16(v128 a) { + return c_v128_unpackhi_s8_s16(a); +} +SIMD_INLINE v128 v128_pack_s32_s16(v128 a, v128 b) { + return c_v128_pack_s32_s16(a, b); +} +SIMD_INLINE v128 v128_pack_s16_u8(v128 a, v128 b) { + return c_v128_pack_s16_u8(a, b); +} +SIMD_INLINE v128 v128_pack_s16_s8(v128 a, v128 b) { + return c_v128_pack_s16_s8(a, b); +} +SIMD_INLINE v128 v128_unpack_u16_s32(v64 a) { return c_v128_unpack_u16_s32(a); } +SIMD_INLINE v128 v128_unpack_s16_s32(v64 a) { return c_v128_unpack_s16_s32(a); } +SIMD_INLINE v128 v128_unpacklo_u16_s32(v128 a) { + return c_v128_unpacklo_u16_s32(a); +} +SIMD_INLINE v128 v128_unpacklo_s16_s32(v128 a) { + return c_v128_unpacklo_s16_s32(a); +} +SIMD_INLINE v128 v128_unpackhi_u16_s32(v128 a) { + return c_v128_unpackhi_u16_s32(a); +} +SIMD_INLINE v128 v128_unpackhi_s16_s32(v128 a) { + return c_v128_unpackhi_s16_s32(a); +} +SIMD_INLINE v128 v128_shuffle_8(v128 a, v128 pattern) { + return c_v128_shuffle_8(a, pattern); +} + +SIMD_INLINE v128 v128_cmpgt_s8(v128 a, v128 b) { return c_v128_cmpgt_s8(a, b); } +SIMD_INLINE v128 v128_cmplt_s8(v128 a, v128 b) { return c_v128_cmplt_s8(a, b); } +SIMD_INLINE v128 v128_cmpeq_8(v128 a, v128 b) { return c_v128_cmpeq_8(a, b); } +SIMD_INLINE v128 v128_cmpgt_s16(v128 a, v128 b) { + return c_v128_cmpgt_s16(a, b); +} +SIMD_INLINE v128 v128_cmplt_s16(v128 a, v128 b) { + return c_v128_cmplt_s16(a, b); +} +SIMD_INLINE v128 v128_cmpeq_16(v128 a, v128 b) { return c_v128_cmpeq_16(a, b); } + +SIMD_INLINE v128 v128_shl_8(v128 a, unsigned int c) { + return c_v128_shl_8(a, c); +} +SIMD_INLINE v128 v128_shr_u8(v128 a, unsigned int c) { + return c_v128_shr_u8(a, c); +} +SIMD_INLINE v128 v128_shr_s8(v128 a, unsigned int c) { + return c_v128_shr_s8(a, c); +} +SIMD_INLINE v128 v128_shl_16(v128 a, unsigned int c) { + return c_v128_shl_16(a, c); +} +SIMD_INLINE v128 v128_shr_u16(v128 a, unsigned int c) { + return c_v128_shr_u16(a, c); +} +SIMD_INLINE v128 v128_shr_s16(v128 a, unsigned int c) { + return c_v128_shr_s16(a, c); +} +SIMD_INLINE v128 v128_shl_32(v128 a, unsigned int c) { + return c_v128_shl_32(a, c); +} +SIMD_INLINE v128 v128_shr_u32(v128 a, unsigned int c) { + return c_v128_shr_u32(a, c); +} +SIMD_INLINE v128 v128_shr_s32(v128 a, unsigned int c) { + return c_v128_shr_s32(a, c); +} + +SIMD_INLINE v128 v128_shr_n_byte(v128 a, unsigned int n) { + return c_v128_shr_n_byte(a, n); +} +SIMD_INLINE v128 v128_shl_n_byte(v128 a, unsigned int n) { + return c_v128_shl_n_byte(a, n); +} +SIMD_INLINE v128 v128_shl_n_8(v128 a, unsigned int n) { + return c_v128_shl_n_8(a, n); +} +SIMD_INLINE v128 v128_shl_n_16(v128 a, unsigned int n) { + return c_v128_shl_n_16(a, n); +} +SIMD_INLINE v128 v128_shl_n_32(v128 a, unsigned int n) { + return c_v128_shl_n_32(a, n); +} +SIMD_INLINE v128 v128_shr_n_u8(v128 a, unsigned int n) { + return c_v128_shr_n_u8(a, n); +} +SIMD_INLINE v128 v128_shr_n_u16(v128 a, unsigned int n) { + return c_v128_shr_n_u16(a, n); +} +SIMD_INLINE v128 v128_shr_n_u32(v128 a, unsigned int n) { + return c_v128_shr_n_u32(a, n); +} +SIMD_INLINE v128 v128_shr_n_s8(v128 a, unsigned int n) { + return c_v128_shr_n_s8(a, n); +} +SIMD_INLINE v128 v128_shr_n_s16(v128 a, unsigned int n) { + return c_v128_shr_n_s16(a, n); +} +SIMD_INLINE v128 v128_shr_n_s32(v128 a, unsigned int n) { + return c_v128_shr_n_s32(a, n); +} + +#endif /* _V128_INTRINSICS_H */ diff --git a/third_party/aom/aom_dsp/simd/v128_intrinsics_arm.h b/third_party/aom/aom_dsp/simd/v128_intrinsics_arm.h new file mode 100644 index 0000000000..0377d4ce10 --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v128_intrinsics_arm.h @@ -0,0 +1,671 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V128_INTRINSICS_H +#define _V128_INTRINSICS_H + +#include +#include "./v64_intrinsics_arm.h" + +typedef int64x2_t v128; + +SIMD_INLINE uint32_t v128_low_u32(v128 a) { + return v64_low_u32(vget_low_s64(a)); +} + +SIMD_INLINE v64 v128_low_v64(v128 a) { return vget_low_s64(a); } + +SIMD_INLINE v64 v128_high_v64(v128 a) { return vget_high_s64(a); } + +SIMD_INLINE v128 v128_from_v64(v64 a, v64 b) { return vcombine_s64(b, a); } + +SIMD_INLINE v128 v128_from_64(uint64_t a, uint64_t b) { + return vcombine_s64((uint64x1_t)b, (uint64x1_t)a); +} + +SIMD_INLINE v128 v128_from_32(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + return vcombine_s64(v64_from_32(c, d), v64_from_32(a, b)); +} + +SIMD_INLINE v128 v128_load_aligned(const void *p) { + return vreinterpretq_s64_u8(vld1q_u8((const uint8_t *)p)); +} + +SIMD_INLINE v128 v128_load_unaligned(const void *p) { + return v128_load_aligned(p); +} + +SIMD_INLINE void v128_store_aligned(void *p, v128 r) { + vst1q_u8((uint8_t *)p, vreinterpretq_u8_s64(r)); +} + +SIMD_INLINE void v128_store_unaligned(void *p, v128 r) { + vst1q_u8((uint8_t *)p, vreinterpretq_u8_s64(r)); +} + +SIMD_INLINE v128 v128_align(v128 a, v128 b, unsigned int c) { +// The following functions require an immediate. +// Some compilers will check this during optimisation, others wont. +#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__) + return c ? vreinterpretq_s64_s8( + vextq_s8(vreinterpretq_s8_s64(b), vreinterpretq_s8_s64(a), c)) + : b; +#else + return c < 8 ? v128_from_v64(v64_align(v128_low_v64(a), v128_high_v64(b), c), + v64_align(v128_high_v64(b), v128_low_v64(b), c)) + : v128_from_v64( + v64_align(v128_high_v64(a), v128_low_v64(a), c - 8), + v64_align(v128_low_v64(a), v128_high_v64(b), c - 8)); +#endif +} + +SIMD_INLINE v128 v128_zero() { return vreinterpretq_s64_u8(vdupq_n_u8(0)); } + +SIMD_INLINE v128 v128_ones() { return vreinterpretq_s64_u8(vdupq_n_u8(-1)); } + +SIMD_INLINE v128 v128_dup_8(uint8_t x) { + return vreinterpretq_s64_u8(vdupq_n_u8(x)); +} + +SIMD_INLINE v128 v128_dup_16(uint16_t x) { + return vreinterpretq_s64_u16(vdupq_n_u16(x)); +} + +SIMD_INLINE v128 v128_dup_32(uint32_t x) { + return vreinterpretq_s64_u32(vdupq_n_u32(x)); +} + +SIMD_INLINE int64_t v128_dotp_s16(v128 a, v128 b) { + return v64_dotp_s16(vget_high_s64(a), vget_high_s64(b)) + + v64_dotp_s16(vget_low_s64(a), vget_low_s64(b)); +} + +SIMD_INLINE uint64_t v128_hadd_u8(v128 x) { + uint64x2_t t = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vreinterpretq_u8_s64(x)))); + return vget_lane_s32( + vreinterpret_s32_u64(vadd_u64(vget_high_u64(t), vget_low_u64(t))), 0); +} + +SIMD_INLINE v128 v128_padd_s16(v128 a) { + return vreinterpretq_s64_s32(vpaddlq_s16(vreinterpretq_s16_s64(a))); +} + +typedef struct { sad64_internal hi, lo; } sad128_internal; + +SIMD_INLINE sad128_internal v128_sad_u8_init() { + sad128_internal s; + s.hi = s.lo = vdupq_n_u16(0); + return s; +} + +/* Implementation dependent return value. Result must be finalised with + v128_sad_u8_sum(). + The result for more than 32 v128_sad_u8() calls is undefined. */ +SIMD_INLINE sad128_internal v128_sad_u8(sad128_internal s, v128 a, v128 b) { + sad128_internal r; + r.hi = v64_sad_u8(s.hi, vget_high_s64(a), vget_high_s64(b)); + r.lo = v64_sad_u8(s.lo, vget_low_s64(a), vget_low_s64(b)); + return r; +} + +SIMD_INLINE uint32_t v128_sad_u8_sum(sad128_internal s) { + return (uint32_t)(v64_sad_u8_sum(s.hi) + v64_sad_u8_sum(s.lo)); +} + +typedef struct { ssd64_internal hi, lo; } ssd128_internal; + +SIMD_INLINE ssd128_internal v128_ssd_u8_init() { + ssd128_internal s; + s.hi = s.lo = (ssd64_internal)(uint64_t)0; + return s; +} + +/* Implementation dependent return value. Result must be finalised with + * v128_ssd_u8_sum(). */ +SIMD_INLINE ssd128_internal v128_ssd_u8(ssd128_internal s, v128 a, v128 b) { + ssd128_internal r; + r.hi = v64_ssd_u8(s.hi, vget_high_s64(a), vget_high_s64(b)); + r.lo = v64_ssd_u8(s.lo, vget_low_s64(a), vget_low_s64(b)); + return r; +} + +SIMD_INLINE uint32_t v128_ssd_u8_sum(ssd128_internal s) { + return (uint32_t)(v64_ssd_u8_sum(s.hi) + v64_ssd_u8_sum(s.lo)); +} + +SIMD_INLINE v128 v128_or(v128 x, v128 y) { return vorrq_s64(x, y); } + +SIMD_INLINE v128 v128_xor(v128 x, v128 y) { return veorq_s64(x, y); } + +SIMD_INLINE v128 v128_and(v128 x, v128 y) { return vandq_s64(x, y); } + +SIMD_INLINE v128 v128_andn(v128 x, v128 y) { return vbicq_s64(x, y); } + +SIMD_INLINE v128 v128_add_8(v128 x, v128 y) { + return vreinterpretq_s64_u8( + vaddq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); +} + +SIMD_INLINE v128 v128_add_16(v128 x, v128 y) { + return vreinterpretq_s64_s16( + vaddq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); +} + +SIMD_INLINE v128 v128_sadd_s16(v128 x, v128 y) { + return vreinterpretq_s64_s16( + vqaddq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); +} + +SIMD_INLINE v128 v128_add_32(v128 x, v128 y) { + return vreinterpretq_s64_u32( + vaddq_u32(vreinterpretq_u32_s64(x), vreinterpretq_u32_s64(y))); +} + +SIMD_INLINE v128 v128_sub_8(v128 x, v128 y) { + return vreinterpretq_s64_u8( + vsubq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); +} + +SIMD_INLINE v128 v128_sub_16(v128 x, v128 y) { + return vreinterpretq_s64_s16( + vsubq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); +} + +SIMD_INLINE v128 v128_ssub_s16(v128 x, v128 y) { + return vreinterpretq_s64_s16( + vqsubq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); +} + +SIMD_INLINE v128 v128_ssub_u16(v128 x, v128 y) { + return vreinterpretq_s64_u16( + vqsubq_u16(vreinterpretq_u16_s64(x), vreinterpretq_u16_s64(y))); +} + +SIMD_INLINE v128 v128_ssub_u8(v128 x, v128 y) { + return vreinterpretq_s64_u8( + vqsubq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); +} + +SIMD_INLINE v128 v128_ssub_s8(v128 x, v128 y) { + return vreinterpretq_s64_s8( + vqsubq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y))); +} + +SIMD_INLINE v128 v128_sub_32(v128 x, v128 y) { + return vreinterpretq_s64_s32( + vsubq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y))); +} + +SIMD_INLINE v128 v128_abs_s16(v128 x) { + return vreinterpretq_s64_s16(vabsq_s16(vreinterpretq_s16_s64(x))); +} + +SIMD_INLINE v128 v128_abs_s8(v128 x) { + return vreinterpretq_s64_s8(vabsq_s8(vreinterpretq_s8_s64(x))); +} + +SIMD_INLINE v128 v128_mul_s16(v64 a, v64 b) { + return vreinterpretq_s64_s32( + vmull_s16(vreinterpret_s16_s64(a), vreinterpret_s16_s64(b))); +} + +SIMD_INLINE v128 v128_mullo_s16(v128 a, v128 b) { + return vreinterpretq_s64_s16( + vmulq_s16(vreinterpretq_s16_s64(a), vreinterpretq_s16_s64(b))); +} + +SIMD_INLINE v128 v128_mulhi_s16(v128 a, v128 b) { + return v128_from_v64(v64_mulhi_s16(vget_high_s64(a), vget_high_s64(b)), + v64_mulhi_s16(vget_low_s64(a), vget_low_s64(b))); +} + +SIMD_INLINE v128 v128_mullo_s32(v128 a, v128 b) { + return vreinterpretq_s64_s32( + vmulq_s32(vreinterpretq_s32_s64(a), vreinterpretq_s32_s64(b))); +} + +SIMD_INLINE v128 v128_madd_s16(v128 a, v128 b) { + return v128_from_v64(v64_madd_s16(vget_high_s64(a), vget_high_s64(b)), + v64_madd_s16(vget_low_s64(a), vget_low_s64(b))); +} + +SIMD_INLINE v128 v128_madd_us8(v128 a, v128 b) { + return v128_from_v64(v64_madd_us8(vget_high_s64(a), vget_high_s64(b)), + v64_madd_us8(vget_low_s64(a), vget_low_s64(b))); +} + +SIMD_INLINE v128 v128_avg_u8(v128 x, v128 y) { + return vreinterpretq_s64_u8( + vrhaddq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); +} + +SIMD_INLINE v128 v128_rdavg_u8(v128 x, v128 y) { + return vreinterpretq_s64_u8( + vhaddq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); +} + +SIMD_INLINE v128 v128_avg_u16(v128 x, v128 y) { + return vreinterpretq_s64_u16( + vrhaddq_u16(vreinterpretq_u16_s64(x), vreinterpretq_u16_s64(y))); +} + +SIMD_INLINE v128 v128_min_u8(v128 x, v128 y) { + return vreinterpretq_s64_u8( + vminq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); +} + +SIMD_INLINE v128 v128_max_u8(v128 x, v128 y) { + return vreinterpretq_s64_u8( + vmaxq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); +} + +SIMD_INLINE v128 v128_min_s8(v128 x, v128 y) { + return vreinterpretq_s64_s8( + vminq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y))); +} + +SIMD_INLINE v128 v128_max_s8(v128 x, v128 y) { + return vreinterpretq_s64_s8( + vmaxq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y))); +} + +SIMD_INLINE v128 v128_min_s16(v128 x, v128 y) { + return vreinterpretq_s64_s16( + vminq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); +} + +SIMD_INLINE v128 v128_max_s16(v128 x, v128 y) { + return vreinterpretq_s64_s16( + vmaxq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); +} + +SIMD_INLINE v128 v128_ziplo_8(v128 x, v128 y) { + uint8x16x2_t r = vzipq_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x)); + return vreinterpretq_s64_u8(r.val[0]); +} + +SIMD_INLINE v128 v128_ziphi_8(v128 x, v128 y) { + uint8x16x2_t r = vzipq_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x)); + return vreinterpretq_s64_u8(r.val[1]); +} + +SIMD_INLINE v128 v128_zip_8(v64 x, v64 y) { + uint8x8x2_t r = vzip_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)); + return vreinterpretq_s64_u8(vcombine_u8(r.val[0], r.val[1])); +} + +SIMD_INLINE v128 v128_ziplo_16(v128 x, v128 y) { + int16x8x2_t r = vzipq_s16(vreinterpretq_s16_s64(y), vreinterpretq_s16_s64(x)); + return vreinterpretq_s64_s16(r.val[0]); +} + +SIMD_INLINE v128 v128_ziphi_16(v128 x, v128 y) { + int16x8x2_t r = vzipq_s16(vreinterpretq_s16_s64(y), vreinterpretq_s16_s64(x)); + return vreinterpretq_s64_s16(r.val[1]); +} + +SIMD_INLINE v128 v128_zip_16(v64 x, v64 y) { + uint16x4x2_t r = vzip_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x)); + return vreinterpretq_s64_u16(vcombine_u16(r.val[0], r.val[1])); +} + +SIMD_INLINE v128 v128_ziplo_32(v128 x, v128 y) { + int32x4x2_t r = vzipq_s32(vreinterpretq_s32_s64(y), vreinterpretq_s32_s64(x)); + return vreinterpretq_s64_s32(r.val[0]); +} + +SIMD_INLINE v128 v128_ziphi_32(v128 x, v128 y) { + int32x4x2_t r = vzipq_s32(vreinterpretq_s32_s64(y), vreinterpretq_s32_s64(x)); + return vreinterpretq_s64_s32(r.val[1]); +} + +SIMD_INLINE v128 v128_zip_32(v64 x, v64 y) { + uint32x2x2_t r = vzip_u32(vreinterpret_u32_s64(y), vreinterpret_u32_s64(x)); + return vreinterpretq_s64_u32(vcombine_u32(r.val[0], r.val[1])); +} + +SIMD_INLINE v128 v128_ziplo_64(v128 a, v128 b) { + return v128_from_v64(vget_low_u64((uint64x2_t)a), + vget_low_u64((uint64x2_t)b)); +} + +SIMD_INLINE v128 v128_ziphi_64(v128 a, v128 b) { + return v128_from_v64(vget_high_u64((uint64x2_t)a), + vget_high_u64((uint64x2_t)b)); +} + +SIMD_INLINE v128 v128_unziplo_8(v128 x, v128 y) { + uint8x16x2_t r = vuzpq_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x)); + return vreinterpretq_s64_u8(r.val[0]); +} + +SIMD_INLINE v128 v128_unziphi_8(v128 x, v128 y) { + uint8x16x2_t r = vuzpq_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x)); + return vreinterpretq_s64_u8(r.val[1]); +} + +SIMD_INLINE v128 v128_unziplo_16(v128 x, v128 y) { + uint16x8x2_t r = + vuzpq_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x)); + return vreinterpretq_s64_u16(r.val[0]); +} + +SIMD_INLINE v128 v128_unziphi_16(v128 x, v128 y) { + uint16x8x2_t r = + vuzpq_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x)); + return vreinterpretq_s64_u16(r.val[1]); +} + +SIMD_INLINE v128 v128_unziplo_32(v128 x, v128 y) { + uint32x4x2_t r = + vuzpq_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x)); + return vreinterpretq_s64_u32(r.val[0]); +} + +SIMD_INLINE v128 v128_unziphi_32(v128 x, v128 y) { + uint32x4x2_t r = + vuzpq_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x)); + return vreinterpretq_s64_u32(r.val[1]); +} + +SIMD_INLINE v128 v128_unpack_u8_s16(v64 a) { + return vreinterpretq_s64_u16(vmovl_u8(vreinterpret_u8_s64(a))); +} + +SIMD_INLINE v128 v128_unpacklo_u8_s16(v128 a) { + return vreinterpretq_s64_u16(vmovl_u8(vreinterpret_u8_s64(vget_low_s64(a)))); +} + +SIMD_INLINE v128 v128_unpackhi_u8_s16(v128 a) { + return vreinterpretq_s64_u16(vmovl_u8(vreinterpret_u8_s64(vget_high_s64(a)))); +} + +SIMD_INLINE v128 v128_unpack_s8_s16(v64 a) { + return vreinterpretq_s64_s16(vmovl_s8(vreinterpret_s8_s64(a))); +} + +SIMD_INLINE v128 v128_unpacklo_s8_s16(v128 a) { + return vreinterpretq_s64_s16(vmovl_s8(vreinterpret_s8_s64(vget_low_s64(a)))); +} + +SIMD_INLINE v128 v128_unpackhi_s8_s16(v128 a) { + return vreinterpretq_s64_s16(vmovl_s8(vreinterpret_s8_s64(vget_high_s64(a)))); +} + +SIMD_INLINE v128 v128_pack_s32_s16(v128 a, v128 b) { + return v128_from_v64( + vreinterpret_s64_s16(vqmovn_s32(vreinterpretq_s32_s64(a))), + vreinterpret_s64_s16(vqmovn_s32(vreinterpretq_s32_s64(b)))); +} + +SIMD_INLINE v128 v128_pack_s16_u8(v128 a, v128 b) { + return v128_from_v64( + vreinterpret_s64_u8(vqmovun_s16(vreinterpretq_s16_s64(a))), + vreinterpret_s64_u8(vqmovun_s16(vreinterpretq_s16_s64(b)))); +} + +SIMD_INLINE v128 v128_pack_s16_s8(v128 a, v128 b) { + return v128_from_v64( + vreinterpret_s64_s8(vqmovn_s16(vreinterpretq_s16_s64(a))), + vreinterpret_s64_s8(vqmovn_s16(vreinterpretq_s16_s64(b)))); +} + +SIMD_INLINE v128 v128_unpack_u16_s32(v64 a) { + return vreinterpretq_s64_u32(vmovl_u16(vreinterpret_u16_s64(a))); +} + +SIMD_INLINE v128 v128_unpack_s16_s32(v64 a) { + return vreinterpretq_s64_s32(vmovl_s16(vreinterpret_s16_s64(a))); +} + +SIMD_INLINE v128 v128_unpacklo_u16_s32(v128 a) { + return vreinterpretq_s64_u32( + vmovl_u16(vreinterpret_u16_s64(vget_low_s64(a)))); +} + +SIMD_INLINE v128 v128_unpacklo_s16_s32(v128 a) { + return vreinterpretq_s64_s32( + vmovl_s16(vreinterpret_s16_s64(vget_low_s64(a)))); +} + +SIMD_INLINE v128 v128_unpackhi_u16_s32(v128 a) { + return vreinterpretq_s64_u32( + vmovl_u16(vreinterpret_u16_s64(vget_high_s64(a)))); +} + +SIMD_INLINE v128 v128_unpackhi_s16_s32(v128 a) { + return vreinterpretq_s64_s32( + vmovl_s16(vreinterpret_s16_s64(vget_high_s64(a)))); +} + +SIMD_INLINE v128 v128_shuffle_8(v128 x, v128 pattern) { + return v128_from_64( + (uint64_t)vreinterpret_s64_u8( + vtbl2_u8((uint8x8x2_t){ { vget_low_u8(vreinterpretq_u8_s64(x)), + vget_high_u8(vreinterpretq_u8_s64(x)) } }, + vreinterpret_u8_s64(vget_high_s64(pattern)))), + (uint64_t)vreinterpret_s64_u8( + vtbl2_u8((uint8x8x2_t){ { vget_low_u8(vreinterpretq_u8_s64(x)), + vget_high_u8(vreinterpretq_u8_s64(x)) } }, + vreinterpret_u8_s64(vget_low_s64(pattern))))); +} + +SIMD_INLINE v128 v128_cmpgt_s8(v128 x, v128 y) { + return vreinterpretq_s64_u8( + vcgtq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y))); +} + +SIMD_INLINE v128 v128_cmplt_s8(v128 x, v128 y) { + return vreinterpretq_s64_u8( + vcltq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y))); +} + +SIMD_INLINE v128 v128_cmpeq_8(v128 x, v128 y) { + return vreinterpretq_s64_u8( + vceqq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); +} + +SIMD_INLINE v128 v128_cmpgt_s16(v128 x, v128 y) { + return vreinterpretq_s64_u16( + vcgtq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); +} + +SIMD_INLINE v128 v128_cmplt_s16(v128 x, v128 y) { + return vreinterpretq_s64_u16( + vcltq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); +} + +SIMD_INLINE v128 v128_cmpeq_16(v128 x, v128 y) { + return vreinterpretq_s64_u16( + vceqq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); +} + +SIMD_INLINE v128 v128_shl_8(v128 a, unsigned int c) { + return (c > 7) ? v128_zero() : vreinterpretq_s64_u8(vshlq_u8( + vreinterpretq_u8_s64(a), vdupq_n_s8(c))); +} + +SIMD_INLINE v128 v128_shr_u8(v128 a, unsigned int c) { + return (c > 7) ? v128_zero() : vreinterpretq_s64_u8(vshlq_u8( + vreinterpretq_u8_s64(a), vdupq_n_s8(-c))); +} + +SIMD_INLINE v128 v128_shr_s8(v128 a, unsigned int c) { + return (c > 7) ? v128_ones() : vreinterpretq_s64_s8(vshlq_s8( + vreinterpretq_s8_s64(a), vdupq_n_s8(-c))); +} + +SIMD_INLINE v128 v128_shl_16(v128 a, unsigned int c) { + return (c > 15) ? v128_zero() + : vreinterpretq_s64_u16( + vshlq_u16(vreinterpretq_u16_s64(a), vdupq_n_s16(c))); +} + +SIMD_INLINE v128 v128_shr_u16(v128 a, unsigned int c) { + return (c > 15) ? v128_zero() + : vreinterpretq_s64_u16( + vshlq_u16(vreinterpretq_u16_s64(a), vdupq_n_s16(-c))); +} + +SIMD_INLINE v128 v128_shr_s16(v128 a, unsigned int c) { + return (c > 15) ? v128_ones() + : vreinterpretq_s64_s16( + vshlq_s16(vreinterpretq_s16_s64(a), vdupq_n_s16(-c))); +} + +SIMD_INLINE v128 v128_shl_32(v128 a, unsigned int c) { + return (c > 31) ? v128_zero() + : vreinterpretq_s64_u32( + vshlq_u32(vreinterpretq_u32_s64(a), vdupq_n_s32(c))); +} + +SIMD_INLINE v128 v128_shr_u32(v128 a, unsigned int c) { + return (c > 31) ? v128_zero() + : vreinterpretq_s64_u32( + vshlq_u32(vreinterpretq_u32_s64(a), vdupq_n_s32(-c))); +} + +SIMD_INLINE v128 v128_shr_s32(v128 a, unsigned int c) { + return (c > 31) ? v128_ones() + : vreinterpretq_s64_s32( + vshlq_s32(vreinterpretq_s32_s64(a), vdupq_n_s32(-c))); +} + +#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__) + +SIMD_INLINE v128 v128_shl_n_byte(v128 a, unsigned int n) { + return n < 8 + ? v128_from_64( + (uint64_t)vorr_u64( + vshl_n_u64(vreinterpret_u64_s64(vget_high_s64(a)), + n * 8), + vshr_n_u64(vreinterpret_u64_s64(vget_low_s64(a)), + (8 - n) * 8)), + (uint64_t)vshl_n_u64(vreinterpret_u64_s64(vget_low_s64(a)), + n * 8)) + : (n == 8 ? v128_from_64( + (uint64_t)vreinterpret_u64_s64(vget_low_s64(a)), 0) + : v128_from_64((uint64_t)vshl_n_u64( + vreinterpret_u64_s64(vget_low_s64(a)), + (n - 8) * 8), + 0)); +} + +SIMD_INLINE v128 v128_shr_n_byte(v128 a, unsigned int n) { + return n < 8 + ? v128_from_64( + vshr_n_u64(vreinterpret_u64_s64(vget_high_s64(a)), n * 8), + vorr_u64( + vshr_n_u64(vreinterpret_u64_s64(vget_low_s64(a)), n * 8), + vshl_n_u64(vreinterpret_u64_s64(vget_high_s64(a)), + (8 - n) * 8))) + : (n == 8 + ? v128_from_64(0, vreinterpret_u64_s64(vget_high_s64(a))) + : v128_from_64( + 0, vshr_n_u64(vreinterpret_u64_s64(vget_high_s64(a)), + (n - 8) * 8))); +} + +SIMD_INLINE v128 v128_shl_n_8(v128 a, unsigned int c) { + return vreinterpretq_s64_u8(vshlq_n_u8(vreinterpretq_u8_s64(a), c)); +} + +SIMD_INLINE v128 v128_shr_n_u8(v128 a, unsigned int c) { + return vreinterpretq_s64_u8(vshrq_n_u8(vreinterpretq_u8_s64(a), c)); +} + +SIMD_INLINE v128 v128_shr_n_s8(v128 a, unsigned int c) { + return vreinterpretq_s64_s8(vshrq_n_s8(vreinterpretq_s8_s64(a), c)); +} + +SIMD_INLINE v128 v128_shl_n_16(v128 a, unsigned int c) { + return vreinterpretq_s64_u16(vshlq_n_u16(vreinterpretq_u16_s64(a), c)); +} + +SIMD_INLINE v128 v128_shr_n_u16(v128 a, unsigned int c) { + return vreinterpretq_s64_u16(vshrq_n_u16(vreinterpretq_u16_s64(a), c)); +} + +SIMD_INLINE v128 v128_shr_n_s16(v128 a, unsigned int c) { + return vreinterpretq_s64_s16(vshrq_n_s16(vreinterpretq_s16_s64(a), c)); +} + +SIMD_INLINE v128 v128_shl_n_32(v128 a, unsigned int c) { + return vreinterpretq_s64_u32(vshlq_n_u32(vreinterpretq_u32_s64(a), c)); +} + +SIMD_INLINE v128 v128_shr_n_u32(v128 a, unsigned int c) { + return vreinterpretq_s64_u32(vshrq_n_u32(vreinterpretq_u32_s64(a), c)); +} + +SIMD_INLINE v128 v128_shr_n_s32(v128 a, unsigned int c) { + return vreinterpretq_s64_s32(vshrq_n_s32(vreinterpretq_s32_s64(a), c)); +} + +#else + +SIMD_INLINE v128 v128_shl_n_byte(v128 a, unsigned int n) { + if (n < 8) + return v128_from_v64(v64_or(v64_shl_n_byte(v128_high_v64(a), n), + v64_shr_n_byte(v128_low_v64(a), 8 - n)), + v64_shl_n_byte(v128_low_v64(a), n)); + else + return v128_from_v64(v64_shl_n_byte(v128_low_v64(a), n - 8), v64_zero()); +} + +SIMD_INLINE v128 v128_shr_n_byte(v128 a, unsigned int n) { + if (n < 8) + return v128_from_v64(v64_shr_n_byte(v128_high_v64(a), n), + v64_or(v64_shr_n_byte(v128_low_v64(a), n), + v64_shl_n_byte(v128_high_v64(a), 8 - n))); + else + return v128_from_v64(v64_zero(), v64_shr_n_byte(v128_high_v64(a), n - 8)); +} + +SIMD_INLINE v128 v128_shl_n_8(v128 a, unsigned int c) { + return v128_shl_8(a, c); +} + +SIMD_INLINE v128 v128_shr_n_u8(v128 a, unsigned int c) { + return v128_shr_u8(a, c); +} + +SIMD_INLINE v128 v128_shr_n_s8(v128 a, unsigned int c) { + return v128_shr_s8(a, c); +} + +SIMD_INLINE v128 v128_shl_n_16(v128 a, unsigned int c) { + return v128_shl_16(a, c); +} + +SIMD_INLINE v128 v128_shr_n_u16(v128 a, unsigned int c) { + return v128_shr_u16(a, c); +} + +SIMD_INLINE v128 v128_shr_n_s16(v128 a, unsigned int c) { + return v128_shr_s16(a, c); +} + +SIMD_INLINE v128 v128_shl_n_32(v128 a, unsigned int c) { + return v128_shl_32(a, c); +} + +SIMD_INLINE v128 v128_shr_n_u32(v128 a, unsigned int c) { + return v128_shr_u32(a, c); +} + +SIMD_INLINE v128 v128_shr_n_s32(v128 a, unsigned int c) { + return v128_shr_s32(a, c); +} + +#endif + +#endif /* _V128_INTRINSICS_H */ diff --git a/third_party/aom/aom_dsp/simd/v128_intrinsics_c.h b/third_party/aom/aom_dsp/simd/v128_intrinsics_c.h new file mode 100644 index 0000000000..32e7c32de5 --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v128_intrinsics_c.h @@ -0,0 +1,707 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V128_INTRINSICS_C_H +#define _V128_INTRINSICS_C_H + +#include +#include +#include "./v64_intrinsics_c.h" +#include "./aom_config.h" + +typedef union { + uint8_t u8[16]; + uint16_t u16[8]; + uint32_t u32[4]; + uint64_t u64[2]; + int8_t s8[16]; + int16_t s16[8]; + int32_t s32[4]; + int64_t s64[2]; + c_v64 v64[2]; +} c_v128; + +SIMD_INLINE uint32_t c_v128_low_u32(c_v128 a) { return a.u32[0]; } + +SIMD_INLINE c_v64 c_v128_low_v64(c_v128 a) { return a.v64[0]; } + +SIMD_INLINE c_v64 c_v128_high_v64(c_v128 a) { return a.v64[1]; } + +SIMD_INLINE c_v128 c_v128_from_64(uint64_t hi, uint64_t lo) { + c_v128 t; + t.u64[1] = hi; + t.u64[0] = lo; + return t; +} + +SIMD_INLINE c_v128 c_v128_from_v64(c_v64 hi, c_v64 lo) { + c_v128 t; + t.v64[1] = hi; + t.v64[0] = lo; + return t; +} + +SIMD_INLINE c_v128 c_v128_from_32(uint32_t a, uint32_t b, uint32_t c, + uint32_t d) { + c_v128 t; + t.u32[3] = a; + t.u32[2] = b; + t.u32[1] = c; + t.u32[0] = d; + return t; +} + +SIMD_INLINE c_v128 c_v128_load_unaligned(const void *p) { + c_v128 t; + uint8_t *pp = (uint8_t *)p; + uint8_t *q = (uint8_t *)&t; + int c; + for (c = 0; c < 16; c++) q[c] = pp[c]; + return t; +} + +SIMD_INLINE c_v128 c_v128_load_aligned(const void *p) { + if (SIMD_CHECK && (uintptr_t)p & 15) { + fprintf(stderr, "Error: unaligned v128 load at %p\n", p); + abort(); + } + return c_v128_load_unaligned(p); +} + +SIMD_INLINE void c_v128_store_unaligned(void *p, c_v128 a) { + uint8_t *pp = (uint8_t *)p; + uint8_t *q = (uint8_t *)&a; + int c; + for (c = 0; c < 16; c++) pp[c] = q[c]; +} + +SIMD_INLINE void c_v128_store_aligned(void *p, c_v128 a) { + if (SIMD_CHECK && (uintptr_t)p & 15) { + fprintf(stderr, "Error: unaligned v128 store at %p\n", p); + abort(); + } + c_v128_store_unaligned(p, a); +} + +SIMD_INLINE c_v128 c_v128_zero() { + c_v128 t; + t.u64[1] = t.u64[0] = 0; + return t; +} + +SIMD_INLINE c_v128 c_v128_dup_8(uint8_t x) { + c_v128 t; + t.v64[1] = t.v64[0] = c_v64_dup_8(x); + return t; +} + +SIMD_INLINE c_v128 c_v128_dup_16(uint16_t x) { + c_v128 t; + t.v64[1] = t.v64[0] = c_v64_dup_16(x); + return t; +} + +SIMD_INLINE c_v128 c_v128_dup_32(uint32_t x) { + c_v128 t; + t.v64[1] = t.v64[0] = c_v64_dup_32(x); + return t; +} + +SIMD_INLINE int64_t c_v128_dotp_s16(c_v128 a, c_v128 b) { + return c_v64_dotp_s16(a.v64[1], b.v64[1]) + + c_v64_dotp_s16(a.v64[0], b.v64[0]); +} + +SIMD_INLINE uint64_t c_v128_hadd_u8(c_v128 a) { + return c_v64_hadd_u8(a.v64[1]) + c_v64_hadd_u8(a.v64[0]); +} + +typedef uint32_t c_sad128_internal; + +SIMD_INLINE c_sad128_internal c_v128_sad_u8_init() { return 0; } + +/* Implementation dependent return value. Result must be finalised with + v128_sad_u8_sum(). + The result for more than 32 v128_sad_u8() calls is undefined. */ +SIMD_INLINE c_sad128_internal c_v128_sad_u8(c_sad128_internal s, c_v128 a, + c_v128 b) { + int c; + for (c = 0; c < 16; c++) + s += a.u8[c] > b.u8[c] ? a.u8[c] - b.u8[c] : b.u8[c] - a.u8[c]; + return s; +} + +SIMD_INLINE uint32_t c_v128_sad_u8_sum(c_sad128_internal s) { return s; } + +typedef uint32_t c_ssd128_internal; + +SIMD_INLINE c_ssd128_internal c_v128_ssd_u8_init() { return 0; } + +/* Implementation dependent return value. Result must be finalised with + * v128_ssd_u8_sum(). */ +SIMD_INLINE c_ssd128_internal c_v128_ssd_u8(c_ssd128_internal s, c_v128 a, + c_v128 b) { + int c; + for (c = 0; c < 16; c++) s += (a.u8[c] - b.u8[c]) * (a.u8[c] - b.u8[c]); + return s; +} + +SIMD_INLINE uint32_t c_v128_ssd_u8_sum(c_ssd128_internal s) { return s; } + +SIMD_INLINE c_v128 c_v128_or(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_or(a.v64[1], b.v64[1]), + c_v64_or(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_xor(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_xor(a.v64[1], b.v64[1]), + c_v64_xor(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_and(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_and(a.v64[1], b.v64[1]), + c_v64_and(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_andn(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_andn(a.v64[1], b.v64[1]), + c_v64_andn(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_add_8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_add_8(a.v64[1], b.v64[1]), + c_v64_add_8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_add_16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_add_16(a.v64[1], b.v64[1]), + c_v64_add_16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_sadd_s16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_sadd_s16(a.v64[1], b.v64[1]), + c_v64_sadd_s16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_add_32(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_add_32(a.v64[1], b.v64[1]), + c_v64_add_32(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_padd_s16(c_v128 a) { + c_v128 t; + t.s32[0] = (int32_t)a.s16[0] + (int32_t)a.s16[1]; + t.s32[1] = (int32_t)a.s16[2] + (int32_t)a.s16[3]; + t.s32[2] = (int32_t)a.s16[4] + (int32_t)a.s16[5]; + t.s32[3] = (int32_t)a.s16[6] + (int32_t)a.s16[7]; + return t; +} + +SIMD_INLINE c_v128 c_v128_sub_8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_sub_8(a.v64[1], b.v64[1]), + c_v64_sub_8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_ssub_u8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_ssub_u8(a.v64[1], b.v64[1]), + c_v64_ssub_u8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_ssub_s8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_ssub_s8(a.v64[1], b.v64[1]), + c_v64_ssub_s8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_sub_16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_sub_16(a.v64[1], b.v64[1]), + c_v64_sub_16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_ssub_s16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_ssub_s16(a.v64[1], b.v64[1]), + c_v64_ssub_s16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_ssub_u16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_ssub_u16(a.v64[1], b.v64[1]), + c_v64_ssub_u16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_sub_32(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_sub_32(a.v64[1], b.v64[1]), + c_v64_sub_32(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_abs_s16(c_v128 a) { + return c_v128_from_v64(c_v64_abs_s16(a.v64[1]), c_v64_abs_s16(a.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_abs_s8(c_v128 a) { + return c_v128_from_v64(c_v64_abs_s8(a.v64[1]), c_v64_abs_s8(a.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_mul_s16(c_v64 a, c_v64 b) { + c_v64 lo_bits = c_v64_mullo_s16(a, b); + c_v64 hi_bits = c_v64_mulhi_s16(a, b); + return c_v128_from_v64(c_v64_ziphi_16(hi_bits, lo_bits), + c_v64_ziplo_16(hi_bits, lo_bits)); +} + +SIMD_INLINE c_v128 c_v128_mullo_s16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_mullo_s16(a.v64[1], b.v64[1]), + c_v64_mullo_s16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_mulhi_s16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_mulhi_s16(a.v64[1], b.v64[1]), + c_v64_mulhi_s16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_mullo_s32(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_mullo_s32(a.v64[1], b.v64[1]), + c_v64_mullo_s32(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_madd_s16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_madd_s16(a.v64[1], b.v64[1]), + c_v64_madd_s16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_madd_us8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_madd_us8(a.v64[1], b.v64[1]), + c_v64_madd_us8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_avg_u8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_avg_u8(a.v64[1], b.v64[1]), + c_v64_avg_u8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_rdavg_u8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_rdavg_u8(a.v64[1], b.v64[1]), + c_v64_rdavg_u8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_avg_u16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_avg_u16(a.v64[1], b.v64[1]), + c_v64_avg_u16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_min_u8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_min_u8(a.v64[1], b.v64[1]), + c_v64_min_u8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_max_u8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_max_u8(a.v64[1], b.v64[1]), + c_v64_max_u8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_min_s8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_min_s8(a.v64[1], b.v64[1]), + c_v64_min_s8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_max_s8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_max_s8(a.v64[1], b.v64[1]), + c_v64_max_s8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_min_s16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_min_s16(a.v64[1], b.v64[1]), + c_v64_min_s16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_max_s16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_max_s16(a.v64[1], b.v64[1]), + c_v64_max_s16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_ziplo_8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_ziphi_8(a.v64[0], b.v64[0]), + c_v64_ziplo_8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_ziphi_8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_ziphi_8(a.v64[1], b.v64[1]), + c_v64_ziplo_8(a.v64[1], b.v64[1])); +} + +SIMD_INLINE c_v128 c_v128_ziplo_16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_ziphi_16(a.v64[0], b.v64[0]), + c_v64_ziplo_16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_ziphi_16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_ziphi_16(a.v64[1], b.v64[1]), + c_v64_ziplo_16(a.v64[1], b.v64[1])); +} + +SIMD_INLINE c_v128 c_v128_ziplo_32(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_ziphi_32(a.v64[0], b.v64[0]), + c_v64_ziplo_32(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_ziphi_32(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_ziphi_32(a.v64[1], b.v64[1]), + c_v64_ziplo_32(a.v64[1], b.v64[1])); +} + +SIMD_INLINE c_v128 c_v128_ziplo_64(c_v128 a, c_v128 b) { + return c_v128_from_v64(a.v64[0], b.v64[0]); +} + +SIMD_INLINE c_v128 c_v128_ziphi_64(c_v128 a, c_v128 b) { + return c_v128_from_v64(a.v64[1], b.v64[1]); +} + +SIMD_INLINE c_v128 c_v128_zip_8(c_v64 a, c_v64 b) { + return c_v128_from_v64(c_v64_ziphi_8(a, b), c_v64_ziplo_8(a, b)); +} + +SIMD_INLINE c_v128 c_v128_zip_16(c_v64 a, c_v64 b) { + return c_v128_from_v64(c_v64_ziphi_16(a, b), c_v64_ziplo_16(a, b)); +} + +SIMD_INLINE c_v128 c_v128_zip_32(c_v64 a, c_v64 b) { + return c_v128_from_v64(c_v64_ziphi_32(a, b), c_v64_ziplo_32(a, b)); +} + +SIMD_INLINE c_v128 _c_v128_unzip_8(c_v128 a, c_v128 b, int mode) { + c_v128 t; + if (mode) { + t.u8[15] = b.u8[15]; + t.u8[14] = b.u8[13]; + t.u8[13] = b.u8[11]; + t.u8[12] = b.u8[9]; + t.u8[11] = b.u8[7]; + t.u8[10] = b.u8[5]; + t.u8[9] = b.u8[3]; + t.u8[8] = b.u8[1]; + t.u8[7] = a.u8[15]; + t.u8[6] = a.u8[13]; + t.u8[5] = a.u8[11]; + t.u8[4] = a.u8[9]; + t.u8[3] = a.u8[7]; + t.u8[2] = a.u8[5]; + t.u8[1] = a.u8[3]; + t.u8[0] = a.u8[1]; + } else { + t.u8[15] = a.u8[14]; + t.u8[14] = a.u8[12]; + t.u8[13] = a.u8[10]; + t.u8[12] = a.u8[8]; + t.u8[11] = a.u8[6]; + t.u8[10] = a.u8[4]; + t.u8[9] = a.u8[2]; + t.u8[8] = a.u8[0]; + t.u8[7] = b.u8[14]; + t.u8[6] = b.u8[12]; + t.u8[5] = b.u8[10]; + t.u8[4] = b.u8[8]; + t.u8[3] = b.u8[6]; + t.u8[2] = b.u8[4]; + t.u8[1] = b.u8[2]; + t.u8[0] = b.u8[0]; + } + return t; +} + +SIMD_INLINE c_v128 c_v128_unziplo_8(c_v128 a, c_v128 b) { + return CONFIG_BIG_ENDIAN ? _c_v128_unzip_8(a, b, 1) + : _c_v128_unzip_8(a, b, 0); +} + +SIMD_INLINE c_v128 c_v128_unziphi_8(c_v128 a, c_v128 b) { + return CONFIG_BIG_ENDIAN ? _c_v128_unzip_8(b, a, 0) + : _c_v128_unzip_8(b, a, 1); +} + +SIMD_INLINE c_v128 _c_v128_unzip_16(c_v128 a, c_v128 b, int mode) { + c_v128 t; + if (mode) { + t.u16[7] = b.u16[7]; + t.u16[6] = b.u16[5]; + t.u16[5] = b.u16[3]; + t.u16[4] = b.u16[1]; + t.u16[3] = a.u16[7]; + t.u16[2] = a.u16[5]; + t.u16[1] = a.u16[3]; + t.u16[0] = a.u16[1]; + } else { + t.u16[7] = a.u16[6]; + t.u16[6] = a.u16[4]; + t.u16[5] = a.u16[2]; + t.u16[4] = a.u16[0]; + t.u16[3] = b.u16[6]; + t.u16[2] = b.u16[4]; + t.u16[1] = b.u16[2]; + t.u16[0] = b.u16[0]; + } + return t; +} + +SIMD_INLINE c_v128 c_v128_unziplo_16(c_v128 a, c_v128 b) { + return CONFIG_BIG_ENDIAN ? _c_v128_unzip_16(a, b, 1) + : _c_v128_unzip_16(a, b, 0); +} + +SIMD_INLINE c_v128 c_v128_unziphi_16(c_v128 a, c_v128 b) { + return CONFIG_BIG_ENDIAN ? _c_v128_unzip_16(b, a, 0) + : _c_v128_unzip_16(b, a, 1); +} + +SIMD_INLINE c_v128 _c_v128_unzip_32(c_v128 a, c_v128 b, int mode) { + c_v128 t; + if (mode) { + t.u32[3] = b.u32[3]; + t.u32[2] = b.u32[1]; + t.u32[1] = a.u32[3]; + t.u32[0] = a.u32[1]; + } else { + t.u32[3] = a.u32[2]; + t.u32[2] = a.u32[0]; + t.u32[1] = b.u32[2]; + t.u32[0] = b.u32[0]; + } + return t; +} + +SIMD_INLINE c_v128 c_v128_unziplo_32(c_v128 a, c_v128 b) { + return CONFIG_BIG_ENDIAN ? _c_v128_unzip_32(a, b, 1) + : _c_v128_unzip_32(a, b, 0); +} + +SIMD_INLINE c_v128 c_v128_unziphi_32(c_v128 a, c_v128 b) { + return CONFIG_BIG_ENDIAN ? _c_v128_unzip_32(b, a, 0) + : _c_v128_unzip_32(b, a, 1); +} + +SIMD_INLINE c_v128 c_v128_unpack_u8_s16(c_v64 a) { + return c_v128_from_v64(c_v64_unpackhi_u8_s16(a), c_v64_unpacklo_u8_s16(a)); +} + +SIMD_INLINE c_v128 c_v128_unpacklo_u8_s16(c_v128 a) { + return c_v128_from_v64(c_v64_unpackhi_u8_s16(a.v64[0]), + c_v64_unpacklo_u8_s16(a.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_unpackhi_u8_s16(c_v128 a) { + return c_v128_from_v64(c_v64_unpackhi_u8_s16(a.v64[1]), + c_v64_unpacklo_u8_s16(a.v64[1])); +} + +SIMD_INLINE c_v128 c_v128_unpack_s8_s16(c_v64 a) { + return c_v128_from_v64(c_v64_unpackhi_s8_s16(a), c_v64_unpacklo_s8_s16(a)); +} + +SIMD_INLINE c_v128 c_v128_unpacklo_s8_s16(c_v128 a) { + return c_v128_from_v64(c_v64_unpackhi_s8_s16(a.v64[0]), + c_v64_unpacklo_s8_s16(a.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_unpackhi_s8_s16(c_v128 a) { + return c_v128_from_v64(c_v64_unpackhi_s8_s16(a.v64[1]), + c_v64_unpacklo_s8_s16(a.v64[1])); +} + +SIMD_INLINE c_v128 c_v128_pack_s32_s16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_pack_s32_s16(a.v64[1], a.v64[0]), + c_v64_pack_s32_s16(b.v64[1], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_pack_s16_u8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_pack_s16_u8(a.v64[1], a.v64[0]), + c_v64_pack_s16_u8(b.v64[1], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_pack_s16_s8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_pack_s16_s8(a.v64[1], a.v64[0]), + c_v64_pack_s16_s8(b.v64[1], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_unpack_u16_s32(c_v64 a) { + return c_v128_from_v64(c_v64_unpackhi_u16_s32(a), c_v64_unpacklo_u16_s32(a)); +} + +SIMD_INLINE c_v128 c_v128_unpack_s16_s32(c_v64 a) { + return c_v128_from_v64(c_v64_unpackhi_s16_s32(a), c_v64_unpacklo_s16_s32(a)); +} + +SIMD_INLINE c_v128 c_v128_unpacklo_u16_s32(c_v128 a) { + return c_v128_from_v64(c_v64_unpackhi_u16_s32(a.v64[0]), + c_v64_unpacklo_u16_s32(a.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_unpacklo_s16_s32(c_v128 a) { + return c_v128_from_v64(c_v64_unpackhi_s16_s32(a.v64[0]), + c_v64_unpacklo_s16_s32(a.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_unpackhi_u16_s32(c_v128 a) { + return c_v128_from_v64(c_v64_unpackhi_u16_s32(a.v64[1]), + c_v64_unpacklo_u16_s32(a.v64[1])); +} + +SIMD_INLINE c_v128 c_v128_unpackhi_s16_s32(c_v128 a) { + return c_v128_from_v64(c_v64_unpackhi_s16_s32(a.v64[1]), + c_v64_unpacklo_s16_s32(a.v64[1])); +} + +SIMD_INLINE c_v128 c_v128_shuffle_8(c_v128 a, c_v128 pattern) { + c_v128 t; + int c; + for (c = 0; c < 16; c++) { + if (pattern.u8[c] & ~15) { + fprintf(stderr, "Undefined v128_shuffle_8 index %d/%d\n", pattern.u8[c], + c); + abort(); + } + t.u8[c] = a.u8[CONFIG_BIG_ENDIAN ? 15 - (pattern.u8[c] & 15) + : pattern.u8[c] & 15]; + } + return t; +} + +SIMD_INLINE c_v128 c_v128_cmpgt_s8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_cmpgt_s8(a.v64[1], b.v64[1]), + c_v64_cmpgt_s8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_cmplt_s8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_cmplt_s8(a.v64[1], b.v64[1]), + c_v64_cmplt_s8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_cmpeq_8(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_cmpeq_8(a.v64[1], b.v64[1]), + c_v64_cmpeq_8(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_cmpgt_s16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_cmpgt_s16(a.v64[1], b.v64[1]), + c_v64_cmpgt_s16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_cmplt_s16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_cmplt_s16(a.v64[1], b.v64[1]), + c_v64_cmplt_s16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_cmpeq_16(c_v128 a, c_v128 b) { + return c_v128_from_v64(c_v64_cmpeq_16(a.v64[1], b.v64[1]), + c_v64_cmpeq_16(a.v64[0], b.v64[0])); +} + +SIMD_INLINE c_v128 c_v128_shl_n_byte(c_v128 a, unsigned int n) { + if (n < 8) + return c_v128_from_v64(c_v64_or(c_v64_shl_n_byte(a.v64[1], n), + c_v64_shr_n_byte(a.v64[0], 8 - n)), + c_v64_shl_n_byte(a.v64[0], n)); + else + return c_v128_from_v64(c_v64_shl_n_byte(a.v64[0], n - 8), c_v64_zero()); +} + +SIMD_INLINE c_v128 c_v128_shr_n_byte(c_v128 a, unsigned int n) { + if (n < 8) + return c_v128_from_v64(c_v64_shr_n_byte(a.v64[1], n), + c_v64_or(c_v64_shr_n_byte(a.v64[0], n), + c_v64_shl_n_byte(a.v64[1], 8 - n))); + else + return c_v128_from_v64(c_v64_zero(), c_v64_shr_n_byte(a.v64[1], n - 8)); +} + +SIMD_INLINE c_v128 c_v128_align(c_v128 a, c_v128 b, unsigned int c) { + if (SIMD_CHECK && c > 15) { + fprintf(stderr, "Error: undefined alignment %d\n", c); + abort(); + } + return c ? c_v128_or(c_v128_shr_n_byte(b, c), c_v128_shl_n_byte(a, 16 - c)) + : b; +} + +SIMD_INLINE c_v128 c_v128_shl_8(c_v128 a, unsigned int c) { + return c_v128_from_v64(c_v64_shl_8(a.v64[1], c), c_v64_shl_8(a.v64[0], c)); +} + +SIMD_INLINE c_v128 c_v128_shr_u8(c_v128 a, unsigned int c) { + return c_v128_from_v64(c_v64_shr_u8(a.v64[1], c), c_v64_shr_u8(a.v64[0], c)); +} + +SIMD_INLINE c_v128 c_v128_shr_s8(c_v128 a, unsigned int c) { + return c_v128_from_v64(c_v64_shr_s8(a.v64[1], c), c_v64_shr_s8(a.v64[0], c)); +} + +SIMD_INLINE c_v128 c_v128_shl_16(c_v128 a, unsigned int c) { + return c_v128_from_v64(c_v64_shl_16(a.v64[1], c), c_v64_shl_16(a.v64[0], c)); +} + +SIMD_INLINE c_v128 c_v128_shr_u16(c_v128 a, unsigned int c) { + return c_v128_from_v64(c_v64_shr_u16(a.v64[1], c), + c_v64_shr_u16(a.v64[0], c)); +} + +SIMD_INLINE c_v128 c_v128_shr_s16(c_v128 a, unsigned int c) { + return c_v128_from_v64(c_v64_shr_s16(a.v64[1], c), + c_v64_shr_s16(a.v64[0], c)); +} + +SIMD_INLINE c_v128 c_v128_shl_32(c_v128 a, unsigned int c) { + return c_v128_from_v64(c_v64_shl_32(a.v64[1], c), c_v64_shl_32(a.v64[0], c)); +} + +SIMD_INLINE c_v128 c_v128_shr_u32(c_v128 a, unsigned int c) { + return c_v128_from_v64(c_v64_shr_u32(a.v64[1], c), + c_v64_shr_u32(a.v64[0], c)); +} + +SIMD_INLINE c_v128 c_v128_shr_s32(c_v128 a, unsigned int c) { + return c_v128_from_v64(c_v64_shr_s32(a.v64[1], c), + c_v64_shr_s32(a.v64[0], c)); +} + +SIMD_INLINE c_v128 c_v128_shl_n_8(c_v128 a, unsigned int n) { + return c_v128_shl_8(a, n); +} + +SIMD_INLINE c_v128 c_v128_shl_n_16(c_v128 a, unsigned int n) { + return c_v128_shl_16(a, n); +} + +SIMD_INLINE c_v128 c_v128_shl_n_32(c_v128 a, unsigned int n) { + return c_v128_shl_32(a, n); +} + +SIMD_INLINE c_v128 c_v128_shr_n_u8(c_v128 a, unsigned int n) { + return c_v128_shr_u8(a, n); +} + +SIMD_INLINE c_v128 c_v128_shr_n_u16(c_v128 a, unsigned int n) { + return c_v128_shr_u16(a, n); +} + +SIMD_INLINE c_v128 c_v128_shr_n_u32(c_v128 a, unsigned int n) { + return c_v128_shr_u32(a, n); +} + +SIMD_INLINE c_v128 c_v128_shr_n_s8(c_v128 a, unsigned int n) { + return c_v128_shr_s8(a, n); +} + +SIMD_INLINE c_v128 c_v128_shr_n_s16(c_v128 a, unsigned int n) { + return c_v128_shr_s16(a, n); +} + +SIMD_INLINE c_v128 c_v128_shr_n_s32(c_v128 a, unsigned int n) { + return c_v128_shr_s32(a, n); +} + +#endif /* _V128_INTRINSICS_C_H */ diff --git a/third_party/aom/aom_dsp/simd/v128_intrinsics_x86.h b/third_party/aom/aom_dsp/simd/v128_intrinsics_x86.h new file mode 100644 index 0000000000..cca1788d54 --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v128_intrinsics_x86.h @@ -0,0 +1,511 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V128_INTRINSICS_H +#define _V128_INTRINSICS_H + +#include "./v64_intrinsics_x86.h" + +typedef __m128i v128; + +SIMD_INLINE uint32_t v128_low_u32(v128 a) { + return (uint32_t)_mm_cvtsi128_si32(a); +} + +SIMD_INLINE v64 v128_low_v64(v128 a) { + return _mm_unpacklo_epi64(a, v64_zero()); +} + +SIMD_INLINE v64 v128_high_v64(v128 a) { return _mm_srli_si128(a, 8); } + +SIMD_INLINE v128 v128_from_v64(v64 a, v64 b) { + return _mm_unpacklo_epi64(b, a); +} + +SIMD_INLINE v128 v128_from_64(uint64_t a, uint64_t b) { + return v128_from_v64(v64_from_64(a), v64_from_64(b)); +} + +SIMD_INLINE v128 v128_from_32(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + return _mm_set_epi32(a, b, c, d); +} + +SIMD_INLINE v128 v128_load_aligned(const void *p) { + return _mm_load_si128((__m128i *)p); +} + +SIMD_INLINE v128 v128_load_unaligned(const void *p) { +#if defined(__SSSE3__) + return (__m128i)_mm_lddqu_si128((__m128i *)p); +#else + return _mm_loadu_si128((__m128i *)p); +#endif +} + +SIMD_INLINE void v128_store_aligned(void *p, v128 a) { + _mm_store_si128((__m128i *)p, a); +} + +SIMD_INLINE void v128_store_unaligned(void *p, v128 a) { + _mm_storeu_si128((__m128i *)p, a); +} + +// The following function requires an immediate. +// Some compilers will check this during optimisation, others wont. +#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__) +#if defined(__SSSE3__) +SIMD_INLINE v128 v128_align(v128 a, v128 b, unsigned int c) { + return c ? _mm_alignr_epi8(a, b, c) : b; +} +#else +#define v128_align(a, b, c) \ + ((c) ? _mm_or_si128(_mm_srli_si128(b, c), _mm_slli_si128(a, 16 - (c))) : (b)) +#endif +#else +#if defined(__SSSE3__) +#define v128_align(a, b, c) ((c) ? _mm_alignr_epi8(a, b, c) : (b)) +#else +#define v128_align(a, b, c) \ + ((c) ? _mm_or_si128(_mm_srli_si128(b, c), _mm_slli_si128(a, 16 - (c))) : (b)) +#endif +#endif + +SIMD_INLINE v128 v128_zero() { return _mm_setzero_si128(); } + +SIMD_INLINE v128 v128_dup_8(uint8_t x) { return _mm_set1_epi8(x); } + +SIMD_INLINE v128 v128_dup_16(uint16_t x) { return _mm_set1_epi16(x); } + +SIMD_INLINE v128 v128_dup_32(uint32_t x) { return _mm_set1_epi32(x); } + +SIMD_INLINE v128 v128_add_8(v128 a, v128 b) { return _mm_add_epi8(a, b); } + +SIMD_INLINE v128 v128_add_16(v128 a, v128 b) { return _mm_add_epi16(a, b); } + +SIMD_INLINE v128 v128_sadd_s16(v128 a, v128 b) { return _mm_adds_epi16(a, b); } + +SIMD_INLINE v128 v128_add_32(v128 a, v128 b) { return _mm_add_epi32(a, b); } + +SIMD_INLINE v128 v128_padd_s16(v128 a) { + return _mm_madd_epi16(a, _mm_set1_epi16(1)); +} + +SIMD_INLINE v128 v128_sub_8(v128 a, v128 b) { return _mm_sub_epi8(a, b); } + +SIMD_INLINE v128 v128_ssub_u8(v128 a, v128 b) { return _mm_subs_epu8(a, b); } + +SIMD_INLINE v128 v128_ssub_s8(v128 a, v128 b) { return _mm_subs_epi8(a, b); } + +SIMD_INLINE v128 v128_sub_16(v128 a, v128 b) { return _mm_sub_epi16(a, b); } + +SIMD_INLINE v128 v128_ssub_s16(v128 a, v128 b) { return _mm_subs_epi16(a, b); } + +SIMD_INLINE v128 v128_ssub_u16(v128 a, v128 b) { return _mm_subs_epu16(a, b); } + +SIMD_INLINE v128 v128_sub_32(v128 a, v128 b) { return _mm_sub_epi32(a, b); } + +SIMD_INLINE v128 v128_abs_s16(v128 a) { +#if defined(__SSSE3__) + return _mm_abs_epi16(a); +#else + return _mm_max_epi16(a, _mm_sub_epi16(_mm_setzero_si128(), a)); +#endif +} + +SIMD_INLINE v128 v128_abs_s8(v128 a) { +#if defined(__SSSE3__) + return _mm_abs_epi8(a); +#else + v128 sign = _mm_cmplt_epi8(a, _mm_setzero_si128()); + return _mm_xor_si128(sign, _mm_add_epi8(a, sign)); +#endif +} + +SIMD_INLINE v128 v128_ziplo_8(v128 a, v128 b) { + return _mm_unpacklo_epi8(b, a); +} + +SIMD_INLINE v128 v128_ziphi_8(v128 a, v128 b) { + return _mm_unpackhi_epi8(b, a); +} + +SIMD_INLINE v128 v128_ziplo_16(v128 a, v128 b) { + return _mm_unpacklo_epi16(b, a); +} + +SIMD_INLINE v128 v128_ziphi_16(v128 a, v128 b) { + return _mm_unpackhi_epi16(b, a); +} + +SIMD_INLINE v128 v128_ziplo_32(v128 a, v128 b) { + return _mm_unpacklo_epi32(b, a); +} + +SIMD_INLINE v128 v128_ziphi_32(v128 a, v128 b) { + return _mm_unpackhi_epi32(b, a); +} + +SIMD_INLINE v128 v128_ziplo_64(v128 a, v128 b) { + return _mm_unpacklo_epi64(b, a); +} + +SIMD_INLINE v128 v128_ziphi_64(v128 a, v128 b) { + return _mm_unpackhi_epi64(b, a); +} + +SIMD_INLINE v128 v128_zip_8(v64 a, v64 b) { return _mm_unpacklo_epi8(b, a); } + +SIMD_INLINE v128 v128_zip_16(v64 a, v64 b) { return _mm_unpacklo_epi16(b, a); } + +SIMD_INLINE v128 v128_zip_32(v64 a, v64 b) { return _mm_unpacklo_epi32(b, a); } + +SIMD_INLINE v128 v128_unziphi_8(v128 a, v128 b) { + return _mm_packs_epi16(_mm_srai_epi16(b, 8), _mm_srai_epi16(a, 8)); +} + +SIMD_INLINE v128 v128_unziplo_8(v128 a, v128 b) { +#if defined(__SSSE3__) +#ifdef __x86_64__ + v128 order = _mm_cvtsi64_si128(0x0e0c0a0806040200LL); +#else + v128 order = _mm_set_epi32(0, 0, 0x0e0c0a08, 0x06040200); +#endif + return _mm_unpacklo_epi64(_mm_shuffle_epi8(b, order), + _mm_shuffle_epi8(a, order)); +#else + return v128_unziphi_8(_mm_slli_si128(a, 1), _mm_slli_si128(b, 1)); +#endif +} + +SIMD_INLINE v128 v128_unziphi_16(v128 a, v128 b) { + return _mm_packs_epi32(_mm_srai_epi32(b, 16), _mm_srai_epi32(a, 16)); +} + +SIMD_INLINE v128 v128_unziplo_16(v128 a, v128 b) { +#if defined(__SSSE3__) +#ifdef __x86_64__ + v128 order = _mm_cvtsi64_si128(0x0d0c090805040100LL); +#else + v128 order = _mm_set_epi32(0, 0, 0x0d0c0908, 0x05040100); +#endif + return _mm_unpacklo_epi64(_mm_shuffle_epi8(b, order), + _mm_shuffle_epi8(a, order)); +#else + return v128_unziphi_16(_mm_slli_si128(a, 2), _mm_slli_si128(b, 2)); +#endif +} + +SIMD_INLINE v128 v128_unziphi_32(v128 a, v128 b) { + return _mm_castps_si128(_mm_shuffle_ps( + _mm_castsi128_ps(b), _mm_castsi128_ps(a), _MM_SHUFFLE(3, 1, 3, 1))); +} + +SIMD_INLINE v128 v128_unziplo_32(v128 a, v128 b) { + return _mm_castps_si128(_mm_shuffle_ps( + _mm_castsi128_ps(b), _mm_castsi128_ps(a), _MM_SHUFFLE(2, 0, 2, 0))); +} + +SIMD_INLINE v128 v128_unpack_u8_s16(v64 a) { + return _mm_unpacklo_epi8(a, _mm_setzero_si128()); +} + +SIMD_INLINE v128 v128_unpacklo_u8_s16(v128 a) { + return _mm_unpacklo_epi8(a, _mm_setzero_si128()); +} + +SIMD_INLINE v128 v128_unpackhi_u8_s16(v128 a) { + return _mm_unpackhi_epi8(a, _mm_setzero_si128()); +} + +SIMD_INLINE v128 v128_unpack_s8_s16(v64 a) { + return _mm_srai_epi16(_mm_unpacklo_epi8(a, a), 8); +} + +SIMD_INLINE v128 v128_unpacklo_s8_s16(v128 a) { + return _mm_srai_epi16(_mm_unpacklo_epi8(a, a), 8); +} + +SIMD_INLINE v128 v128_unpackhi_s8_s16(v128 a) { + return _mm_srai_epi16(_mm_unpackhi_epi8(a, a), 8); +} + +SIMD_INLINE v128 v128_pack_s32_s16(v128 a, v128 b) { + return _mm_packs_epi32(b, a); +} + +SIMD_INLINE v128 v128_pack_s16_u8(v128 a, v128 b) { + return _mm_packus_epi16(b, a); +} + +SIMD_INLINE v128 v128_pack_s16_s8(v128 a, v128 b) { + return _mm_packs_epi16(b, a); +} + +SIMD_INLINE v128 v128_unpack_u16_s32(v64 a) { + return _mm_unpacklo_epi16(a, _mm_setzero_si128()); +} + +SIMD_INLINE v128 v128_unpack_s16_s32(v64 a) { + return _mm_srai_epi32(_mm_unpacklo_epi16(a, a), 16); +} + +SIMD_INLINE v128 v128_unpacklo_u16_s32(v128 a) { + return _mm_unpacklo_epi16(a, _mm_setzero_si128()); +} + +SIMD_INLINE v128 v128_unpacklo_s16_s32(v128 a) { + return _mm_srai_epi32(_mm_unpacklo_epi16(a, a), 16); +} + +SIMD_INLINE v128 v128_unpackhi_u16_s32(v128 a) { + return _mm_unpackhi_epi16(a, _mm_setzero_si128()); +} + +SIMD_INLINE v128 v128_unpackhi_s16_s32(v128 a) { + return _mm_srai_epi32(_mm_unpackhi_epi16(a, a), 16); +} + +SIMD_INLINE v128 v128_shuffle_8(v128 x, v128 pattern) { +#if defined(__SSSE3__) + return _mm_shuffle_epi8(x, pattern); +#else + v128 output; + unsigned char *input = (unsigned char *)&x; + unsigned char *index = (unsigned char *)&pattern; + char *selected = (char *)&output; + int counter; + + for (counter = 0; counter < 16; counter++) { + selected[counter] = input[index[counter] & 15]; + } + + return output; +#endif +} + +SIMD_INLINE int64_t v128_dotp_s16(v128 a, v128 b) { + v128 r = _mm_madd_epi16(a, b); +#if defined(__SSE4_1__) && defined(__x86_64__) + v128 c = _mm_add_epi64(_mm_cvtepi32_epi64(r), + _mm_cvtepi32_epi64(_mm_srli_si128(r, 8))); + return _mm_cvtsi128_si64(_mm_add_epi64(c, _mm_srli_si128(c, 8))); +#else + return (int64_t)_mm_cvtsi128_si32(r) + + (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(r, 4)) + + (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(r, 8)) + + (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(r, 12)); +#endif +} + +SIMD_INLINE uint64_t v128_hadd_u8(v128 a) { + v128 t = _mm_sad_epu8(a, _mm_setzero_si128()); + return v64_low_u32(v128_low_v64(t)) + v64_low_u32(v128_high_v64(t)); +} + +typedef v128 sad128_internal; + +SIMD_INLINE sad128_internal v128_sad_u8_init() { return _mm_setzero_si128(); } + +/* Implementation dependent return value. Result must be finalised with + v128_sad_sum(). + The result for more than 32 v128_sad_u8() calls is undefined. */ +SIMD_INLINE sad128_internal v128_sad_u8(sad128_internal s, v128 a, v128 b) { + return _mm_add_epi64(s, _mm_sad_epu8(a, b)); +} + +SIMD_INLINE uint32_t v128_sad_u8_sum(sad128_internal s) { + return v128_low_u32(_mm_add_epi32(s, _mm_unpackhi_epi64(s, s))); +} + +typedef v128 ssd128_internal; + +SIMD_INLINE ssd128_internal v128_ssd_u8_init() { return _mm_setzero_si128(); } + +/* Implementation dependent return value. Result must be finalised with + * v128_ssd_sum(). */ +SIMD_INLINE ssd128_internal v128_ssd_u8(ssd128_internal s, v128 a, v128 b) { + v128 l = _mm_sub_epi16(_mm_unpacklo_epi8(a, _mm_setzero_si128()), + _mm_unpacklo_epi8(b, _mm_setzero_si128())); + v128 h = _mm_sub_epi16(_mm_unpackhi_epi8(a, _mm_setzero_si128()), + _mm_unpackhi_epi8(b, _mm_setzero_si128())); + v128 rl = _mm_madd_epi16(l, l); + v128 rh = _mm_madd_epi16(h, h); + v128 c = _mm_cvtsi32_si128(32); + rl = _mm_add_epi32(rl, _mm_srli_si128(rl, 8)); + rl = _mm_add_epi32(rl, _mm_srli_si128(rl, 4)); + rh = _mm_add_epi32(rh, _mm_srli_si128(rh, 8)); + rh = _mm_add_epi32(rh, _mm_srli_si128(rh, 4)); + return _mm_add_epi64( + s, _mm_srl_epi64(_mm_sll_epi64(_mm_unpacklo_epi64(rl, rh), c), c)); +} + +SIMD_INLINE uint32_t v128_ssd_u8_sum(ssd128_internal s) { + return v128_low_u32(_mm_add_epi32(s, _mm_unpackhi_epi64(s, s))); +} + +SIMD_INLINE v128 v128_or(v128 a, v128 b) { return _mm_or_si128(a, b); } + +SIMD_INLINE v128 v128_xor(v128 a, v128 b) { return _mm_xor_si128(a, b); } + +SIMD_INLINE v128 v128_and(v128 a, v128 b) { return _mm_and_si128(a, b); } + +SIMD_INLINE v128 v128_andn(v128 a, v128 b) { return _mm_andnot_si128(b, a); } + +SIMD_INLINE v128 v128_mul_s16(v64 a, v64 b) { + v64 lo_bits = v64_mullo_s16(a, b); + v64 hi_bits = v64_mulhi_s16(a, b); + return v128_from_v64(v64_ziphi_16(hi_bits, lo_bits), + v64_ziplo_16(hi_bits, lo_bits)); +} + +SIMD_INLINE v128 v128_mullo_s16(v128 a, v128 b) { + return _mm_mullo_epi16(a, b); +} + +SIMD_INLINE v128 v128_mulhi_s16(v128 a, v128 b) { + return _mm_mulhi_epi16(a, b); +} + +SIMD_INLINE v128 v128_mullo_s32(v128 a, v128 b) { +#if defined(__SSE4_1__) + return _mm_mullo_epi32(a, b); +#else + return _mm_unpacklo_epi32( + _mm_shuffle_epi32(_mm_mul_epu32(a, b), 8), + _mm_shuffle_epi32( + _mm_mul_epu32(_mm_srli_si128(a, 4), _mm_srli_si128(b, 4)), 8)); +#endif +} + +SIMD_INLINE v128 v128_madd_s16(v128 a, v128 b) { return _mm_madd_epi16(a, b); } + +SIMD_INLINE v128 v128_madd_us8(v128 a, v128 b) { +#if defined(__SSSE3__) + return _mm_maddubs_epi16(a, b); +#else + return _mm_packs_epi32( + _mm_madd_epi16(_mm_unpacklo_epi8(a, _mm_setzero_si128()), + _mm_srai_epi16(_mm_unpacklo_epi8(b, b), 8)), + _mm_madd_epi16(_mm_unpackhi_epi8(a, _mm_setzero_si128()), + _mm_srai_epi16(_mm_unpackhi_epi8(b, b), 8))); +#endif +} + +SIMD_INLINE v128 v128_avg_u8(v128 a, v128 b) { return _mm_avg_epu8(a, b); } + +SIMD_INLINE v128 v128_rdavg_u8(v128 a, v128 b) { + return _mm_sub_epi8(_mm_avg_epu8(a, b), + _mm_and_si128(_mm_xor_si128(a, b), v128_dup_8(1))); +} + +SIMD_INLINE v128 v128_avg_u16(v128 a, v128 b) { return _mm_avg_epu16(a, b); } + +SIMD_INLINE v128 v128_min_u8(v128 a, v128 b) { return _mm_min_epu8(a, b); } + +SIMD_INLINE v128 v128_max_u8(v128 a, v128 b) { return _mm_max_epu8(a, b); } + +SIMD_INLINE v128 v128_min_s8(v128 a, v128 b) { +#if defined(__SSE4_1__) + return _mm_min_epi8(a, b); +#else + v128 mask = _mm_cmplt_epi8(a, b); + return _mm_or_si128(_mm_andnot_si128(mask, b), _mm_and_si128(mask, a)); +#endif +} + +SIMD_INLINE v128 v128_max_s8(v128 a, v128 b) { +#if defined(__SSE4_1__) + return _mm_max_epi8(a, b); +#else + v128 mask = _mm_cmplt_epi8(b, a); + return _mm_or_si128(_mm_andnot_si128(mask, b), _mm_and_si128(mask, a)); +#endif +} + +SIMD_INLINE v128 v128_min_s16(v128 a, v128 b) { return _mm_min_epi16(a, b); } + +SIMD_INLINE v128 v128_max_s16(v128 a, v128 b) { return _mm_max_epi16(a, b); } + +SIMD_INLINE v128 v128_cmpgt_s8(v128 a, v128 b) { return _mm_cmpgt_epi8(a, b); } + +SIMD_INLINE v128 v128_cmplt_s8(v128 a, v128 b) { return _mm_cmplt_epi8(a, b); } + +SIMD_INLINE v128 v128_cmpeq_8(v128 a, v128 b) { return _mm_cmpeq_epi8(a, b); } + +SIMD_INLINE v128 v128_cmpgt_s16(v128 a, v128 b) { + return _mm_cmpgt_epi16(a, b); +} + +SIMD_INLINE v128 v128_cmplt_s16(v128 a, v128 b) { + return _mm_cmplt_epi16(a, b); +} + +SIMD_INLINE v128 v128_cmpeq_16(v128 a, v128 b) { return _mm_cmpeq_epi16(a, b); } + +SIMD_INLINE v128 v128_shl_8(v128 a, unsigned int c) { + return _mm_and_si128(_mm_set1_epi8((uint8_t)(0xff << c)), + _mm_sll_epi16(a, _mm_cvtsi32_si128(c))); +} + +SIMD_INLINE v128 v128_shr_u8(v128 a, unsigned int c) { + return _mm_and_si128(_mm_set1_epi8(0xff >> c), + _mm_srl_epi16(a, _mm_cvtsi32_si128(c))); +} + +SIMD_INLINE v128 v128_shr_s8(v128 a, unsigned int c) { + __m128i x = _mm_cvtsi32_si128(c + 8); + return _mm_packs_epi16(_mm_sra_epi16(_mm_unpacklo_epi8(a, a), x), + _mm_sra_epi16(_mm_unpackhi_epi8(a, a), x)); +} + +SIMD_INLINE v128 v128_shl_16(v128 a, unsigned int c) { + return _mm_sll_epi16(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v128 v128_shr_u16(v128 a, unsigned int c) { + return _mm_srl_epi16(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v128 v128_shr_s16(v128 a, unsigned int c) { + return _mm_sra_epi16(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v128 v128_shl_32(v128 a, unsigned int c) { + return _mm_sll_epi32(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v128 v128_shr_u32(v128 a, unsigned int c) { + return _mm_srl_epi32(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v128 v128_shr_s32(v128 a, unsigned int c) { + return _mm_sra_epi32(a, _mm_cvtsi32_si128(c)); +} + +/* These intrinsics require immediate values, so we must use #defines + to enforce that. */ +#define v128_shl_n_byte(a, c) _mm_slli_si128(a, c) +#define v128_shr_n_byte(a, c) _mm_srli_si128(a, c) +#define v128_shl_n_8(a, c) \ + _mm_and_si128(_mm_set1_epi8((uint8_t)(0xff << (c))), _mm_slli_epi16(a, c)) +#define v128_shr_n_u8(a, c) \ + _mm_and_si128(_mm_set1_epi8(0xff >> (c)), _mm_srli_epi16(a, c)) +#define v128_shr_n_s8(a, c) \ + _mm_packs_epi16(_mm_srai_epi16(_mm_unpacklo_epi8(a, a), (c) + 8), \ + _mm_srai_epi16(_mm_unpackhi_epi8(a, a), (c) + 8)) +#define v128_shl_n_16(a, c) _mm_slli_epi16(a, c) +#define v128_shr_n_u16(a, c) _mm_srli_epi16(a, c) +#define v128_shr_n_s16(a, c) _mm_srai_epi16(a, c) +#define v128_shl_n_32(a, c) _mm_slli_epi32(a, c) +#define v128_shr_n_u32(a, c) _mm_srli_epi32(a, c) +#define v128_shr_n_s32(a, c) _mm_srai_epi32(a, c) + +#endif /* _V128_INTRINSICS_H */ diff --git a/third_party/aom/aom_dsp/simd/v256_intrinsics.h b/third_party/aom/aom_dsp/simd/v256_intrinsics.h new file mode 100644 index 0000000000..1896374ee3 --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v256_intrinsics.h @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V256_INTRINSICS_H +#define _V256_INTRINSICS_H + +#include +#include +#include +#include "./v256_intrinsics_c.h" +#include "./v128_intrinsics.h" +#include "./v64_intrinsics.h" + +/* Fallback to plain, unoptimised C. */ + +typedef c_v256 v256; + +SIMD_INLINE uint32_t v256_low_u32(v256 a) { return c_v256_low_u32(a); } +SIMD_INLINE v64 v256_low_v64(v256 a) { return c_v256_low_v64(a); } +SIMD_INLINE v128 v256_low_v128(v256 a) { return c_v256_low_v128(a); } +SIMD_INLINE v128 v256_high_v128(v256 a) { return c_v256_high_v128(a); } +SIMD_INLINE v256 v256_from_v128(v128 hi, v128 lo) { + return c_v256_from_v128(hi, lo); +} +SIMD_INLINE v256 v256_from_64(uint64_t a, uint64_t b, uint64_t c, uint64_t d) { + return c_v256_from_64(a, b, c, d); +} +SIMD_INLINE v256 v256_from_v64(v64 a, v64 b, v64 c, v64 d) { + return c_v256_from_v64(a, b, c, d); +} + +SIMD_INLINE v256 v256_load_unaligned(const void *p) { + return c_v256_load_unaligned(p); +} +SIMD_INLINE v256 v256_load_aligned(const void *p) { + return c_v256_load_aligned(p); +} + +SIMD_INLINE void v256_store_unaligned(void *p, v256 a) { + c_v256_store_unaligned(p, a); +} +SIMD_INLINE void v256_store_aligned(void *p, v256 a) { + c_v256_store_aligned(p, a); +} + +SIMD_INLINE v256 v256_align(v256 a, v256 b, unsigned int c) { + return c_v256_align(a, b, c); +} + +SIMD_INLINE v256 v256_zero() { return c_v256_zero(); } +SIMD_INLINE v256 v256_dup_8(uint8_t x) { return c_v256_dup_8(x); } +SIMD_INLINE v256 v256_dup_16(uint16_t x) { return c_v256_dup_16(x); } +SIMD_INLINE v256 v256_dup_32(uint32_t x) { return c_v256_dup_32(x); } + +typedef uint32_t sad256_internal; +SIMD_INLINE sad256_internal v256_sad_u8_init() { return c_v256_sad_u8_init(); } +SIMD_INLINE sad256_internal v256_sad_u8(sad256_internal s, v256 a, v256 b) { + return c_v256_sad_u8(s, a, b); +} +SIMD_INLINE uint32_t v256_sad_u8_sum(sad256_internal s) { + return c_v256_sad_u8_sum(s); +} +typedef uint32_t ssd256_internal; +SIMD_INLINE ssd256_internal v256_ssd_u8_init() { return c_v256_ssd_u8_init(); } +SIMD_INLINE ssd256_internal v256_ssd_u8(ssd256_internal s, v256 a, v256 b) { + return c_v256_ssd_u8(s, a, b); +} +SIMD_INLINE uint32_t v256_ssd_u8_sum(ssd256_internal s) { + return c_v256_ssd_u8_sum(s); +} +SIMD_INLINE int64_t v256_dotp_s16(v256 a, v256 b) { + return c_v256_dotp_s16(a, b); +} +SIMD_INLINE uint64_t v256_hadd_u8(v256 a) { return c_v256_hadd_u8(a); } + +SIMD_INLINE v256 v256_or(v256 a, v256 b) { return c_v256_or(a, b); } +SIMD_INLINE v256 v256_xor(v256 a, v256 b) { return c_v256_xor(a, b); } +SIMD_INLINE v256 v256_and(v256 a, v256 b) { return c_v256_and(a, b); } +SIMD_INLINE v256 v256_andn(v256 a, v256 b) { return c_v256_andn(a, b); } + +SIMD_INLINE v256 v256_add_8(v256 a, v256 b) { return c_v256_add_8(a, b); } +SIMD_INLINE v256 v256_add_16(v256 a, v256 b) { return c_v256_add_16(a, b); } +SIMD_INLINE v256 v256_sadd_s16(v256 a, v256 b) { return c_v256_sadd_s16(a, b); } +SIMD_INLINE v256 v256_add_32(v256 a, v256 b) { return c_v256_add_32(a, b); } +SIMD_INLINE v256 v256_padd_s16(v256 a) { return c_v256_padd_s16(a); } +SIMD_INLINE v256 v256_sub_8(v256 a, v256 b) { return c_v256_sub_8(a, b); } +SIMD_INLINE v256 v256_ssub_u8(v256 a, v256 b) { return c_v256_ssub_u8(a, b); } +SIMD_INLINE v256 v256_ssub_s8(v256 a, v256 b) { return c_v256_ssub_s8(a, b); } +SIMD_INLINE v256 v256_sub_16(v256 a, v256 b) { return c_v256_sub_16(a, b); } +SIMD_INLINE v256 v256_ssub_s16(v256 a, v256 b) { return c_v256_ssub_s16(a, b); } +SIMD_INLINE v256 v256_ssub_u16(v256 a, v256 b) { return c_v256_ssub_u16(a, b); } +SIMD_INLINE v256 v256_sub_32(v256 a, v256 b) { return c_v256_sub_32(a, b); } +SIMD_INLINE v256 v256_abs_s16(v256 a) { return c_v256_abs_s16(a); } +SIMD_INLINE v256 v256_abs_s8(v256 a) { return c_v256_abs_s8(a); } + +SIMD_INLINE v256 v256_mul_s16(v128 a, v128 b) { return c_v256_mul_s16(a, b); } +SIMD_INLINE v256 v256_mullo_s16(v256 a, v256 b) { + return c_v256_mullo_s16(a, b); +} +SIMD_INLINE v256 v256_mulhi_s16(v256 a, v256 b) { + return c_v256_mulhi_s16(a, b); +} +SIMD_INLINE v256 v256_mullo_s32(v256 a, v256 b) { + return c_v256_mullo_s32(a, b); +} +SIMD_INLINE v256 v256_madd_s16(v256 a, v256 b) { return c_v256_madd_s16(a, b); } +SIMD_INLINE v256 v256_madd_us8(v256 a, v256 b) { return c_v256_madd_us8(a, b); } + +SIMD_INLINE v256 v256_avg_u8(v256 a, v256 b) { return c_v256_avg_u8(a, b); } +SIMD_INLINE v256 v256_rdavg_u8(v256 a, v256 b) { return c_v256_rdavg_u8(a, b); } +SIMD_INLINE v256 v256_avg_u16(v256 a, v256 b) { return c_v256_avg_u16(a, b); } +SIMD_INLINE v256 v256_min_u8(v256 a, v256 b) { return c_v256_min_u8(a, b); } +SIMD_INLINE v256 v256_max_u8(v256 a, v256 b) { return c_v256_max_u8(a, b); } +SIMD_INLINE v256 v256_min_s8(v256 a, v256 b) { return c_v256_min_s8(a, b); } +SIMD_INLINE v256 v256_max_s8(v256 a, v256 b) { return c_v256_max_s8(a, b); } +SIMD_INLINE v256 v256_min_s16(v256 a, v256 b) { return c_v256_min_s16(a, b); } +SIMD_INLINE v256 v256_max_s16(v256 a, v256 b) { return c_v256_max_s16(a, b); } + +SIMD_INLINE v256 v256_ziplo_8(v256 a, v256 b) { return c_v256_ziplo_8(a, b); } +SIMD_INLINE v256 v256_ziphi_8(v256 a, v256 b) { return c_v256_ziphi_8(a, b); } +SIMD_INLINE v256 v256_ziplo_16(v256 a, v256 b) { return c_v256_ziplo_16(a, b); } +SIMD_INLINE v256 v256_ziphi_16(v256 a, v256 b) { return c_v256_ziphi_16(a, b); } +SIMD_INLINE v256 v256_ziplo_32(v256 a, v256 b) { return c_v256_ziplo_32(a, b); } +SIMD_INLINE v256 v256_ziphi_32(v256 a, v256 b) { return c_v256_ziphi_32(a, b); } +SIMD_INLINE v256 v256_ziplo_64(v256 a, v256 b) { return c_v256_ziplo_64(a, b); } +SIMD_INLINE v256 v256_ziphi_64(v256 a, v256 b) { return c_v256_ziphi_64(a, b); } +SIMD_INLINE v256 v256_ziplo_128(v256 a, v256 b) { + return c_v256_ziplo_128(a, b); +} +SIMD_INLINE v256 v256_ziphi_128(v256 a, v256 b) { + return c_v256_ziphi_128(a, b); +} +SIMD_INLINE v256 v256_zip_8(v128 a, v128 b) { return c_v256_zip_8(a, b); } +SIMD_INLINE v256 v256_zip_16(v128 a, v128 b) { return c_v256_zip_16(a, b); } +SIMD_INLINE v256 v256_zip_32(v128 a, v128 b) { return c_v256_zip_32(a, b); } +SIMD_INLINE v256 v256_unziplo_8(v256 a, v256 b) { + return c_v256_unziplo_8(a, b); +} +SIMD_INLINE v256 v256_unziphi_8(v256 a, v256 b) { + return c_v256_unziphi_8(a, b); +} +SIMD_INLINE v256 v256_unziplo_16(v256 a, v256 b) { + return c_v256_unziplo_16(a, b); +} +SIMD_INLINE v256 v256_unziphi_16(v256 a, v256 b) { + return c_v256_unziphi_16(a, b); +} +SIMD_INLINE v256 v256_unziplo_32(v256 a, v256 b) { + return c_v256_unziplo_32(a, b); +} +SIMD_INLINE v256 v256_unziphi_32(v256 a, v256 b) { + return c_v256_unziphi_32(a, b); +} +SIMD_INLINE v256 v256_unpack_u8_s16(v128 a) { return c_v256_unpack_u8_s16(a); } +SIMD_INLINE v256 v256_unpacklo_u8_s16(v256 a) { + return c_v256_unpacklo_u8_s16(a); +} +SIMD_INLINE v256 v256_unpackhi_u8_s16(v256 a) { + return c_v256_unpackhi_u8_s16(a); +} +SIMD_INLINE v256 v256_unpack_s8_s16(v128 a) { return c_v256_unpack_s8_s16(a); } +SIMD_INLINE v256 v256_unpacklo_s8_s16(v256 a) { + return c_v256_unpacklo_s8_s16(a); +} +SIMD_INLINE v256 v256_unpackhi_s8_s16(v256 a) { + return c_v256_unpackhi_s8_s16(a); +} +SIMD_INLINE v256 v256_pack_s32_s16(v256 a, v256 b) { + return c_v256_pack_s32_s16(a, b); +} +SIMD_INLINE v256 v256_pack_s16_u8(v256 a, v256 b) { + return c_v256_pack_s16_u8(a, b); +} +SIMD_INLINE v256 v256_pack_s16_s8(v256 a, v256 b) { + return c_v256_pack_s16_s8(a, b); +} +SIMD_INLINE v256 v256_unpack_u16_s32(v128 a) { + return c_v256_unpack_u16_s32(a); +} +SIMD_INLINE v256 v256_unpack_s16_s32(v128 a) { + return c_v256_unpack_s16_s32(a); +} +SIMD_INLINE v256 v256_unpacklo_u16_s32(v256 a) { + return c_v256_unpacklo_u16_s32(a); +} +SIMD_INLINE v256 v256_unpacklo_s16_s32(v256 a) { + return c_v256_unpacklo_s16_s32(a); +} +SIMD_INLINE v256 v256_unpackhi_u16_s32(v256 a) { + return c_v256_unpackhi_u16_s32(a); +} +SIMD_INLINE v256 v256_unpackhi_s16_s32(v256 a) { + return c_v256_unpackhi_s16_s32(a); +} +SIMD_INLINE v256 v256_shuffle_8(v256 a, v256 pattern) { + return c_v256_shuffle_8(a, pattern); +} +SIMD_INLINE v256 v256_pshuffle_8(v256 a, v256 pattern) { + return c_v256_pshuffle_8(a, pattern); +} + +SIMD_INLINE v256 v256_cmpgt_s8(v256 a, v256 b) { return c_v256_cmpgt_s8(a, b); } +SIMD_INLINE v256 v256_cmplt_s8(v256 a, v256 b) { return c_v256_cmplt_s8(a, b); } +SIMD_INLINE v256 v256_cmpeq_8(v256 a, v256 b) { return c_v256_cmpeq_8(a, b); } +SIMD_INLINE v256 v256_cmpgt_s16(v256 a, v256 b) { + return c_v256_cmpgt_s16(a, b); +} +SIMD_INLINE v256 v256_cmplt_s16(v256 a, v256 b) { + return c_v256_cmplt_s16(a, b); +} +SIMD_INLINE v256 v256_cmpeq_16(v256 a, v256 b) { return c_v256_cmpeq_16(a, b); } + +SIMD_INLINE v256 v256_shl_8(v256 a, unsigned int c) { + return c_v256_shl_8(a, c); +} +SIMD_INLINE v256 v256_shr_u8(v256 a, unsigned int c) { + return c_v256_shr_u8(a, c); +} +SIMD_INLINE v256 v256_shr_s8(v256 a, unsigned int c) { + return c_v256_shr_s8(a, c); +} +SIMD_INLINE v256 v256_shl_16(v256 a, unsigned int c) { + return c_v256_shl_16(a, c); +} +SIMD_INLINE v256 v256_shr_u16(v256 a, unsigned int c) { + return c_v256_shr_u16(a, c); +} +SIMD_INLINE v256 v256_shr_s16(v256 a, unsigned int c) { + return c_v256_shr_s16(a, c); +} +SIMD_INLINE v256 v256_shl_32(v256 a, unsigned int c) { + return c_v256_shl_32(a, c); +} +SIMD_INLINE v256 v256_shr_u32(v256 a, unsigned int c) { + return c_v256_shr_u32(a, c); +} +SIMD_INLINE v256 v256_shr_s32(v256 a, unsigned int c) { + return c_v256_shr_s32(a, c); +} + +SIMD_INLINE v256 v256_shr_n_byte(v256 a, unsigned int n) { + return c_v256_shr_n_byte(a, n); +} +SIMD_INLINE v256 v256_shl_n_byte(v256 a, unsigned int n) { + return c_v256_shl_n_byte(a, n); +} +SIMD_INLINE v256 v256_shl_n_8(v256 a, unsigned int n) { + return c_v256_shl_n_8(a, n); +} +SIMD_INLINE v256 v256_shl_n_16(v256 a, unsigned int n) { + return c_v256_shl_n_16(a, n); +} +SIMD_INLINE v256 v256_shl_n_32(v256 a, unsigned int n) { + return c_v256_shl_n_32(a, n); +} +SIMD_INLINE v256 v256_shr_n_u8(v256 a, unsigned int n) { + return c_v256_shr_n_u8(a, n); +} +SIMD_INLINE v256 v256_shr_n_u16(v256 a, unsigned int n) { + return c_v256_shr_n_u16(a, n); +} +SIMD_INLINE v256 v256_shr_n_u32(v256 a, unsigned int n) { + return c_v256_shr_n_u32(a, n); +} +SIMD_INLINE v256 v256_shr_n_s8(v256 a, unsigned int n) { + return c_v256_shr_n_s8(a, n); +} +SIMD_INLINE v256 v256_shr_n_s16(v256 a, unsigned int n) { + return c_v256_shr_n_s16(a, n); +} +SIMD_INLINE v256 v256_shr_n_s32(v256 a, unsigned int n) { + return c_v256_shr_n_s32(a, n); +} + +#endif /* _V256_INTRINSICS_H */ diff --git a/third_party/aom/aom_dsp/simd/v256_intrinsics_arm.h b/third_party/aom/aom_dsp/simd/v256_intrinsics_arm.h new file mode 100644 index 0000000000..ba4ed719df --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v256_intrinsics_arm.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V256_INTRINSICS_H +#define _V256_INTRINSICS_H + +#include "./v256_intrinsics_v128.h" + +#endif /* _V256_INTRINSICS_H */ diff --git a/third_party/aom/aom_dsp/simd/v256_intrinsics_c.h b/third_party/aom/aom_dsp/simd/v256_intrinsics_c.h new file mode 100644 index 0000000000..f96ca7fa6a --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v256_intrinsics_c.h @@ -0,0 +1,724 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V256_INTRINSICS_C_H +#define _V256_INTRINSICS_C_H + +#include +#include +#include "./v128_intrinsics_c.h" +#include "./aom_config.h" + +typedef union { + uint8_t u8[32]; + uint16_t u16[16]; + uint32_t u32[8]; + uint64_t u64[4]; + int8_t s8[32]; + int16_t s16[16]; + int32_t s32[8]; + int64_t s64[4]; + c_v64 v64[4]; + c_v128 v128[2]; +} c_v256; + +SIMD_INLINE uint32_t c_v256_low_u32(c_v256 a) { return a.u32[0]; } + +SIMD_INLINE c_v64 c_v256_low_v64(c_v256 a) { return a.v64[0]; } + +SIMD_INLINE c_v128 c_v256_low_v128(c_v256 a) { return a.v128[0]; } + +SIMD_INLINE c_v128 c_v256_high_v128(c_v256 a) { return a.v128[1]; } + +SIMD_INLINE c_v256 c_v256_from_v128(c_v128 hi, c_v128 lo) { + c_v256 t; + t.v128[1] = hi; + t.v128[0] = lo; + return t; +} + +SIMD_INLINE c_v256 c_v256_from_64(uint64_t a, uint64_t b, uint64_t c, + uint64_t d) { + c_v256 t; + t.u64[3] = a; + t.u64[2] = b; + t.u64[1] = c; + t.u64[0] = d; + return t; +} + +SIMD_INLINE c_v256 c_v256_from_v64(c_v64 a, c_v64 b, c_v64 c, c_v64 d) { + c_v256 t; + t.u64[3] = a.u64; + t.u64[2] = b.u64; + t.u64[1] = c.u64; + t.u64[0] = d.u64; + return t; +} + +SIMD_INLINE c_v256 c_v256_load_unaligned(const void *p) { + c_v256 t; + uint8_t *pp = (uint8_t *)p; + uint8_t *q = (uint8_t *)&t; + int c; + for (c = 0; c < 32; c++) q[c] = pp[c]; + return t; +} + +SIMD_INLINE c_v256 c_v256_load_aligned(const void *p) { + if (SIMD_CHECK && (uintptr_t)p & 31) { + fprintf(stderr, "Error: unaligned v256 load at %p\n", p); + abort(); + } + return c_v256_load_unaligned(p); +} + +SIMD_INLINE void c_v256_store_unaligned(void *p, c_v256 a) { + uint8_t *pp = (uint8_t *)p; + uint8_t *q = (uint8_t *)&a; + int c; + for (c = 0; c < 32; c++) pp[c] = q[c]; +} + +SIMD_INLINE void c_v256_store_aligned(void *p, c_v256 a) { + if (SIMD_CHECK && (uintptr_t)p & 31) { + fprintf(stderr, "Error: unaligned v256 store at %p\n", p); + abort(); + } + c_v256_store_unaligned(p, a); +} + +SIMD_INLINE c_v256 c_v256_zero() { + c_v256 t; + t.u64[3] = t.u64[2] = t.u64[1] = t.u64[0] = 0; + return t; +} + +SIMD_INLINE c_v256 c_v256_dup_8(uint8_t x) { + c_v256 t; + t.v64[3] = t.v64[2] = t.v64[1] = t.v64[0] = c_v64_dup_8(x); + return t; +} + +SIMD_INLINE c_v256 c_v256_dup_16(uint16_t x) { + c_v256 t; + t.v64[3] = t.v64[2] = t.v64[1] = t.v64[0] = c_v64_dup_16(x); + return t; +} + +SIMD_INLINE c_v256 c_v256_dup_32(uint32_t x) { + c_v256 t; + t.v64[3] = t.v64[2] = t.v64[1] = t.v64[0] = c_v64_dup_32(x); + return t; +} + +SIMD_INLINE int64_t c_v256_dotp_s16(c_v256 a, c_v256 b) { + return c_v128_dotp_s16(a.v128[1], b.v128[1]) + + c_v128_dotp_s16(a.v128[0], b.v128[0]); +} + +SIMD_INLINE uint64_t c_v256_hadd_u8(c_v256 a) { + return c_v128_hadd_u8(a.v128[1]) + c_v128_hadd_u8(a.v128[0]); +} + +typedef uint32_t c_sad256_internal; + +SIMD_INLINE c_sad128_internal c_v256_sad_u8_init() { return 0; } + +/* Implementation dependent return value. Result must be finalised with + v256_sad_u8_sum(). + The result for more than 16 v256_sad_u8() calls is undefined. */ +SIMD_INLINE c_sad128_internal c_v256_sad_u8(c_sad256_internal s, c_v256 a, + c_v256 b) { + int c; + for (c = 0; c < 32; c++) + s += a.u8[c] > b.u8[c] ? a.u8[c] - b.u8[c] : b.u8[c] - a.u8[c]; + return s; +} + +SIMD_INLINE uint32_t c_v256_sad_u8_sum(c_sad256_internal s) { return s; } + +typedef uint32_t c_ssd256_internal; + +SIMD_INLINE c_ssd256_internal c_v256_ssd_u8_init() { return 0; } + +/* Implementation dependent return value. Result must be finalised with + * v256_ssd_u8_sum(). */ +SIMD_INLINE c_ssd256_internal c_v256_ssd_u8(c_ssd256_internal s, c_v256 a, + c_v256 b) { + int c; + for (c = 0; c < 32; c++) s += (a.u8[c] - b.u8[c]) * (a.u8[c] - b.u8[c]); + return s; +} + +SIMD_INLINE uint32_t c_v256_ssd_u8_sum(c_ssd256_internal s) { return s; } + +SIMD_INLINE c_v256 c_v256_or(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_or(a.v128[1], b.v128[1]), + c_v128_or(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_xor(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_xor(a.v128[1], b.v128[1]), + c_v128_xor(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_and(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_and(a.v128[1], b.v128[1]), + c_v128_and(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_andn(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_andn(a.v128[1], b.v128[1]), + c_v128_andn(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_add_8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_add_8(a.v128[1], b.v128[1]), + c_v128_add_8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_add_16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_add_16(a.v128[1], b.v128[1]), + c_v128_add_16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_sadd_s16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_sadd_s16(a.v128[1], b.v128[1]), + c_v128_sadd_s16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_add_32(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_add_32(a.v128[1], b.v128[1]), + c_v128_add_32(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_padd_s16(c_v256 a) { + c_v256 t; + t.s32[0] = (int32_t)a.s16[0] + (int32_t)a.s16[1]; + t.s32[1] = (int32_t)a.s16[2] + (int32_t)a.s16[3]; + t.s32[2] = (int32_t)a.s16[4] + (int32_t)a.s16[5]; + t.s32[3] = (int32_t)a.s16[6] + (int32_t)a.s16[7]; + t.s32[4] = (int32_t)a.s16[8] + (int32_t)a.s16[9]; + t.s32[5] = (int32_t)a.s16[10] + (int32_t)a.s16[11]; + t.s32[6] = (int32_t)a.s16[12] + (int32_t)a.s16[13]; + t.s32[7] = (int32_t)a.s16[14] + (int32_t)a.s16[15]; + return t; +} + +SIMD_INLINE c_v256 c_v256_sub_8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_sub_8(a.v128[1], b.v128[1]), + c_v128_sub_8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_ssub_u8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_ssub_u8(a.v128[1], b.v128[1]), + c_v128_ssub_u8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_ssub_s8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_ssub_s8(a.v128[1], b.v128[1]), + c_v128_ssub_s8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_sub_16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_sub_16(a.v128[1], b.v128[1]), + c_v128_sub_16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_ssub_s16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_ssub_s16(a.v128[1], b.v128[1]), + c_v128_ssub_s16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_ssub_u16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_ssub_u16(a.v128[1], b.v128[1]), + c_v128_ssub_u16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_sub_32(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_sub_32(a.v128[1], b.v128[1]), + c_v128_sub_32(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_abs_s16(c_v256 a) { + return c_v256_from_v128(c_v128_abs_s16(a.v128[1]), c_v128_abs_s16(a.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_abs_s8(c_v256 a) { + return c_v256_from_v128(c_v128_abs_s8(a.v128[1]), c_v128_abs_s8(a.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_mul_s16(c_v128 a, c_v128 b) { + c_v128 lo_bits = c_v128_mullo_s16(a, b); + c_v128 hi_bits = c_v128_mulhi_s16(a, b); + return c_v256_from_v128(c_v128_ziphi_16(hi_bits, lo_bits), + c_v128_ziplo_16(hi_bits, lo_bits)); +} + +SIMD_INLINE c_v256 c_v256_mullo_s16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_mullo_s16(a.v128[1], b.v128[1]), + c_v128_mullo_s16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_mulhi_s16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_mulhi_s16(a.v128[1], b.v128[1]), + c_v128_mulhi_s16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_mullo_s32(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_mullo_s32(a.v128[1], b.v128[1]), + c_v128_mullo_s32(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_madd_s16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_madd_s16(a.v128[1], b.v128[1]), + c_v128_madd_s16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_madd_us8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_madd_us8(a.v128[1], b.v128[1]), + c_v128_madd_us8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_avg_u8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_avg_u8(a.v128[1], b.v128[1]), + c_v128_avg_u8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_rdavg_u8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_rdavg_u8(a.v128[1], b.v128[1]), + c_v128_rdavg_u8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_avg_u16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_avg_u16(a.v128[1], b.v128[1]), + c_v128_avg_u16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_min_u8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_min_u8(a.v128[1], b.v128[1]), + c_v128_min_u8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_max_u8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_max_u8(a.v128[1], b.v128[1]), + c_v128_max_u8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_min_s8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_min_s8(a.v128[1], b.v128[1]), + c_v128_min_s8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_max_s8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_max_s8(a.v128[1], b.v128[1]), + c_v128_max_s8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_min_s16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_min_s16(a.v128[1], b.v128[1]), + c_v128_min_s16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_max_s16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_max_s16(a.v128[1], b.v128[1]), + c_v128_max_s16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_ziplo_8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_ziphi_8(a.v128[0], b.v128[0]), + c_v128_ziplo_8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_ziphi_8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_ziphi_8(a.v128[1], b.v128[1]), + c_v128_ziplo_8(a.v128[1], b.v128[1])); +} + +SIMD_INLINE c_v256 c_v256_ziplo_16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_ziphi_16(a.v128[0], b.v128[0]), + c_v128_ziplo_16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_ziphi_16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_ziphi_16(a.v128[1], b.v128[1]), + c_v128_ziplo_16(a.v128[1], b.v128[1])); +} + +SIMD_INLINE c_v256 c_v256_ziplo_32(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_ziphi_32(a.v128[0], b.v128[0]), + c_v128_ziplo_32(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_ziphi_32(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_ziphi_32(a.v128[1], b.v128[1]), + c_v128_ziplo_32(a.v128[1], b.v128[1])); +} + +SIMD_INLINE c_v256 c_v256_ziplo_64(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_ziphi_64(a.v128[0], b.v128[0]), + c_v128_ziplo_64(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_ziphi_64(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_ziphi_64(a.v128[1], b.v128[1]), + c_v128_ziplo_64(a.v128[1], b.v128[1])); +} + +SIMD_INLINE c_v256 c_v256_ziplo_128(c_v256 a, c_v256 b) { + return c_v256_from_v128(a.v128[0], b.v128[0]); +} + +SIMD_INLINE c_v256 c_v256_ziphi_128(c_v256 a, c_v256 b) { + return c_v256_from_v128(a.v128[1], b.v128[1]); +} + +SIMD_INLINE c_v256 c_v256_zip_8(c_v128 a, c_v128 b) { + return c_v256_from_v128(c_v128_ziphi_8(a, b), c_v128_ziplo_8(a, b)); +} + +SIMD_INLINE c_v256 c_v256_zip_16(c_v128 a, c_v128 b) { + return c_v256_from_v128(c_v128_ziphi_16(a, b), c_v128_ziplo_16(a, b)); +} + +SIMD_INLINE c_v256 c_v256_zip_32(c_v128 a, c_v128 b) { + return c_v256_from_v128(c_v128_ziphi_32(a, b), c_v128_ziplo_32(a, b)); +} + +SIMD_INLINE c_v256 _c_v256_unzip_8(c_v256 a, c_v256 b, int mode) { + c_v256 t; + int i; + if (mode) { + for (i = 0; i < 16; i++) { + t.u8[i] = a.u8[i * 2 + 1]; + t.u8[i + 16] = b.u8[i * 2 + 1]; + } + } else { + for (i = 0; i < 16; i++) { + t.u8[i] = b.u8[i * 2]; + t.u8[i + 16] = a.u8[i * 2]; + } + } + return t; +} + +SIMD_INLINE c_v256 c_v256_unziplo_8(c_v256 a, c_v256 b) { + return CONFIG_BIG_ENDIAN ? _c_v256_unzip_8(a, b, 1) + : _c_v256_unzip_8(a, b, 0); +} + +SIMD_INLINE c_v256 c_v256_unziphi_8(c_v256 a, c_v256 b) { + return CONFIG_BIG_ENDIAN ? _c_v256_unzip_8(b, a, 0) + : _c_v256_unzip_8(b, a, 1); +} + +SIMD_INLINE c_v256 _c_v256_unzip_16(c_v256 a, c_v256 b, int mode) { + c_v256 t; + int i; + if (mode) { + for (i = 0; i < 8; i++) { + t.u16[i] = a.u16[i * 2 + 1]; + t.u16[i + 8] = b.u16[i * 2 + 1]; + } + } else { + for (i = 0; i < 8; i++) { + t.u16[i] = b.u16[i * 2]; + t.u16[i + 8] = a.u16[i * 2]; + } + } + return t; +} + +SIMD_INLINE c_v256 c_v256_unziplo_16(c_v256 a, c_v256 b) { + return CONFIG_BIG_ENDIAN ? _c_v256_unzip_16(a, b, 1) + : _c_v256_unzip_16(a, b, 0); +} + +SIMD_INLINE c_v256 c_v256_unziphi_16(c_v256 a, c_v256 b) { + return CONFIG_BIG_ENDIAN ? _c_v256_unzip_16(b, a, 0) + : _c_v256_unzip_16(b, a, 1); +} + +SIMD_INLINE c_v256 _c_v256_unzip_32(c_v256 a, c_v256 b, int mode) { + c_v256 t; + if (mode) { + t.u32[7] = b.u32[7]; + t.u32[6] = b.u32[5]; + t.u32[5] = b.u32[3]; + t.u32[4] = b.u32[1]; + t.u32[3] = a.u32[7]; + t.u32[2] = a.u32[5]; + t.u32[1] = a.u32[3]; + t.u32[0] = a.u32[1]; + } else { + t.u32[7] = a.u32[6]; + t.u32[6] = a.u32[4]; + t.u32[5] = a.u32[2]; + t.u32[4] = a.u32[0]; + t.u32[3] = b.u32[6]; + t.u32[2] = b.u32[4]; + t.u32[1] = b.u32[2]; + t.u32[0] = b.u32[0]; + } + return t; +} + +SIMD_INLINE c_v256 c_v256_unziplo_32(c_v256 a, c_v256 b) { + return CONFIG_BIG_ENDIAN ? _c_v256_unzip_32(a, b, 1) + : _c_v256_unzip_32(a, b, 0); +} + +SIMD_INLINE c_v256 c_v256_unziphi_32(c_v256 a, c_v256 b) { + return CONFIG_BIG_ENDIAN ? _c_v256_unzip_32(b, a, 0) + : _c_v256_unzip_32(b, a, 1); +} + +SIMD_INLINE c_v256 c_v256_unpack_u8_s16(c_v128 a) { + return c_v256_from_v128(c_v128_unpackhi_u8_s16(a), c_v128_unpacklo_u8_s16(a)); +} + +SIMD_INLINE c_v256 c_v256_unpacklo_u8_s16(c_v256 a) { + return c_v256_from_v128(c_v128_unpackhi_u8_s16(a.v128[0]), + c_v128_unpacklo_u8_s16(a.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_unpackhi_u8_s16(c_v256 a) { + return c_v256_from_v128(c_v128_unpackhi_u8_s16(a.v128[1]), + c_v128_unpacklo_u8_s16(a.v128[1])); +} + +SIMD_INLINE c_v256 c_v256_unpack_s8_s16(c_v128 a) { + return c_v256_from_v128(c_v128_unpackhi_s8_s16(a), c_v128_unpacklo_s8_s16(a)); +} + +SIMD_INLINE c_v256 c_v256_unpacklo_s8_s16(c_v256 a) { + return c_v256_from_v128(c_v128_unpackhi_s8_s16(a.v128[0]), + c_v128_unpacklo_s8_s16(a.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_unpackhi_s8_s16(c_v256 a) { + return c_v256_from_v128(c_v128_unpackhi_s8_s16(a.v128[1]), + c_v128_unpacklo_s8_s16(a.v128[1])); +} + +SIMD_INLINE c_v256 c_v256_pack_s32_s16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_pack_s32_s16(a.v128[1], a.v128[0]), + c_v128_pack_s32_s16(b.v128[1], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_pack_s16_u8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_pack_s16_u8(a.v128[1], a.v128[0]), + c_v128_pack_s16_u8(b.v128[1], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_pack_s16_s8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_pack_s16_s8(a.v128[1], a.v128[0]), + c_v128_pack_s16_s8(b.v128[1], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_unpack_u16_s32(c_v128 a) { + return c_v256_from_v128(c_v128_unpackhi_u16_s32(a), + c_v128_unpacklo_u16_s32(a)); +} + +SIMD_INLINE c_v256 c_v256_unpack_s16_s32(c_v128 a) { + return c_v256_from_v128(c_v128_unpackhi_s16_s32(a), + c_v128_unpacklo_s16_s32(a)); +} + +SIMD_INLINE c_v256 c_v256_unpacklo_u16_s32(c_v256 a) { + return c_v256_from_v128(c_v128_unpackhi_u16_s32(a.v128[0]), + c_v128_unpacklo_u16_s32(a.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_unpacklo_s16_s32(c_v256 a) { + return c_v256_from_v128(c_v128_unpackhi_s16_s32(a.v128[0]), + c_v128_unpacklo_s16_s32(a.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_unpackhi_u16_s32(c_v256 a) { + return c_v256_from_v128(c_v128_unpackhi_u16_s32(a.v128[1]), + c_v128_unpacklo_u16_s32(a.v128[1])); +} + +SIMD_INLINE c_v256 c_v256_unpackhi_s16_s32(c_v256 a) { + return c_v256_from_v128(c_v128_unpackhi_s16_s32(a.v128[1]), + c_v128_unpacklo_s16_s32(a.v128[1])); +} + +SIMD_INLINE c_v256 c_v256_shuffle_8(c_v256 a, c_v256 pattern) { + c_v256 t; + int c; + for (c = 0; c < 32; c++) { + if (pattern.u8[c] & ~31) { + fprintf(stderr, "Undefined v256_shuffle_8 index %d/%d\n", pattern.u8[c], + c); + abort(); + } + t.u8[c] = a.u8[CONFIG_BIG_ENDIAN ? 31 - (pattern.u8[c] & 31) + : pattern.u8[c] & 31]; + } + return t; +} + +// Pairwise / dual-lane shuffle: shuffle two 128 bit lates. +SIMD_INLINE c_v256 c_v256_pshuffle_8(c_v256 a, c_v256 pattern) { + return c_v256_from_v128( + c_v128_shuffle_8(c_v256_high_v128(a), c_v256_high_v128(pattern)), + c_v128_shuffle_8(c_v256_low_v128(a), c_v256_low_v128(pattern))); +} + +SIMD_INLINE c_v256 c_v256_cmpgt_s8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_cmpgt_s8(a.v128[1], b.v128[1]), + c_v128_cmpgt_s8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_cmplt_s8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_cmplt_s8(a.v128[1], b.v128[1]), + c_v128_cmplt_s8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_cmpeq_8(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_cmpeq_8(a.v128[1], b.v128[1]), + c_v128_cmpeq_8(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_cmpgt_s16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_cmpgt_s16(a.v128[1], b.v128[1]), + c_v128_cmpgt_s16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_cmplt_s16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_cmplt_s16(a.v128[1], b.v128[1]), + c_v128_cmplt_s16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_cmpeq_16(c_v256 a, c_v256 b) { + return c_v256_from_v128(c_v128_cmpeq_16(a.v128[1], b.v128[1]), + c_v128_cmpeq_16(a.v128[0], b.v128[0])); +} + +SIMD_INLINE c_v256 c_v256_shl_n_byte(c_v256 a, unsigned int n) { + if (n < 16) + return c_v256_from_v128(c_v128_or(c_v128_shl_n_byte(a.v128[1], n), + c_v128_shr_n_byte(a.v128[0], 16 - n)), + c_v128_shl_n_byte(a.v128[0], n)); + else if (n > 16) + return c_v256_from_v128(c_v128_shl_n_byte(a.v128[0], n - 16), + c_v128_zero()); + else + return c_v256_from_v128(c_v256_low_v128(a), c_v128_zero()); +} + +SIMD_INLINE c_v256 c_v256_shr_n_byte(c_v256 a, unsigned int n) { + if (n < 16) + return c_v256_from_v128(c_v128_shr_n_byte(a.v128[1], n), + c_v128_or(c_v128_shr_n_byte(a.v128[0], n), + c_v128_shl_n_byte(a.v128[1], 16 - n))); + else if (n > 16) + return c_v256_from_v128(c_v128_zero(), + c_v128_shr_n_byte(a.v128[1], n - 16)); + else + return c_v256_from_v128(c_v128_zero(), c_v256_high_v128(a)); +} + +SIMD_INLINE c_v256 c_v256_align(c_v256 a, c_v256 b, unsigned int c) { + if (SIMD_CHECK && c > 31) { + fprintf(stderr, "Error: undefined alignment %d\n", c); + abort(); + } + return c ? c_v256_or(c_v256_shr_n_byte(b, c), c_v256_shl_n_byte(a, 32 - c)) + : b; +} + +SIMD_INLINE c_v256 c_v256_shl_8(c_v256 a, unsigned int c) { + return c_v256_from_v128(c_v128_shl_8(a.v128[1], c), + c_v128_shl_8(a.v128[0], c)); +} + +SIMD_INLINE c_v256 c_v256_shr_u8(c_v256 a, unsigned int c) { + return c_v256_from_v128(c_v128_shr_u8(a.v128[1], c), + c_v128_shr_u8(a.v128[0], c)); +} + +SIMD_INLINE c_v256 c_v256_shr_s8(c_v256 a, unsigned int c) { + return c_v256_from_v128(c_v128_shr_s8(a.v128[1], c), + c_v128_shr_s8(a.v128[0], c)); +} + +SIMD_INLINE c_v256 c_v256_shl_16(c_v256 a, unsigned int c) { + return c_v256_from_v128(c_v128_shl_16(a.v128[1], c), + c_v128_shl_16(a.v128[0], c)); +} + +SIMD_INLINE c_v256 c_v256_shr_u16(c_v256 a, unsigned int c) { + return c_v256_from_v128(c_v128_shr_u16(a.v128[1], c), + c_v128_shr_u16(a.v128[0], c)); +} + +SIMD_INLINE c_v256 c_v256_shr_s16(c_v256 a, unsigned int c) { + return c_v256_from_v128(c_v128_shr_s16(a.v128[1], c), + c_v128_shr_s16(a.v128[0], c)); +} + +SIMD_INLINE c_v256 c_v256_shl_32(c_v256 a, unsigned int c) { + return c_v256_from_v128(c_v128_shl_32(a.v128[1], c), + c_v128_shl_32(a.v128[0], c)); +} + +SIMD_INLINE c_v256 c_v256_shr_u32(c_v256 a, unsigned int c) { + return c_v256_from_v128(c_v128_shr_u32(a.v128[1], c), + c_v128_shr_u32(a.v128[0], c)); +} + +SIMD_INLINE c_v256 c_v256_shr_s32(c_v256 a, unsigned int c) { + return c_v256_from_v128(c_v128_shr_s32(a.v128[1], c), + c_v128_shr_s32(a.v128[0], c)); +} + +SIMD_INLINE c_v256 c_v256_shl_n_8(c_v256 a, unsigned int n) { + return c_v256_shl_8(a, n); +} + +SIMD_INLINE c_v256 c_v256_shl_n_16(c_v256 a, unsigned int n) { + return c_v256_shl_16(a, n); +} + +SIMD_INLINE c_v256 c_v256_shl_n_32(c_v256 a, unsigned int n) { + return c_v256_shl_32(a, n); +} + +SIMD_INLINE c_v256 c_v256_shr_n_u8(c_v256 a, unsigned int n) { + return c_v256_shr_u8(a, n); +} + +SIMD_INLINE c_v256 c_v256_shr_n_u16(c_v256 a, unsigned int n) { + return c_v256_shr_u16(a, n); +} + +SIMD_INLINE c_v256 c_v256_shr_n_u32(c_v256 a, unsigned int n) { + return c_v256_shr_u32(a, n); +} + +SIMD_INLINE c_v256 c_v256_shr_n_s8(c_v256 a, unsigned int n) { + return c_v256_shr_s8(a, n); +} + +SIMD_INLINE c_v256 c_v256_shr_n_s16(c_v256 a, unsigned int n) { + return c_v256_shr_s16(a, n); +} + +SIMD_INLINE c_v256 c_v256_shr_n_s32(c_v256 a, unsigned int n) { + return c_v256_shr_s32(a, n); +} + +#endif /* _V256_INTRINSICS_C_H */ diff --git a/third_party/aom/aom_dsp/simd/v256_intrinsics_v128.h b/third_party/aom/aom_dsp/simd/v256_intrinsics_v128.h new file mode 100644 index 0000000000..a4b334ea65 --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v256_intrinsics_v128.h @@ -0,0 +1,545 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V256_INTRINSICS_V128_H +#define _V256_INTRINSICS_V128_H + +#if HAVE_NEON +#include "./v128_intrinsics_arm.h" +#elif HAVE_SSE2 +#include "./v128_intrinsics_x86.h" +#else +#include "./v128_intrinsics.h" +#endif + +typedef struct { v128 lo, hi; } v256; + +SIMD_INLINE uint32_t v256_low_u32(v256 a) { return v128_low_u32(a.lo); } + +SIMD_INLINE v64 v256_low_v64(v256 a) { return v128_low_v64(a.lo); } + +SIMD_INLINE v128 v256_low_v128(v256 a) { return a.lo; } + +SIMD_INLINE v128 v256_high_v128(v256 a) { return a.hi; } + +SIMD_INLINE v256 v256_from_v128(v128 hi, v128 lo) { + v256 t; + t.hi = hi; + t.lo = lo; + return t; +} + +SIMD_INLINE v256 v256_from_64(uint64_t a, uint64_t b, uint64_t c, uint64_t d) { + return v256_from_v128(v128_from_64(a, b), v128_from_64(c, d)); +} + +SIMD_INLINE v256 v256_from_v64(v64 a, v64 b, v64 c, v64 d) { + return v256_from_v128(v128_from_v64(a, b), v128_from_v64(c, d)); +} + +SIMD_INLINE v256 v256_load_unaligned(const void *p) { + return v256_from_v128(v128_load_unaligned((uint8_t *)p + 16), + v128_load_unaligned(p)); +} + +SIMD_INLINE v256 v256_load_aligned(const void *p) { + return v256_from_v128(v128_load_aligned((uint8_t *)p + 16), + v128_load_aligned(p)); +} + +SIMD_INLINE void v256_store_unaligned(void *p, v256 a) { + v128_store_unaligned(p, a.lo); + v128_store_unaligned((uint8_t *)p + 16, a.hi); +} + +SIMD_INLINE void v256_store_aligned(void *p, v256 a) { + v128_store_aligned(p, a.lo); + v128_store_aligned((uint8_t *)p + 16, a.hi); +} + +SIMD_INLINE v256 v256_zero() { + return v256_from_v128(v128_zero(), v128_zero()); +} + +SIMD_INLINE v256 v256_dup_8(uint8_t x) { + v128 t = v128_dup_8(x); + return v256_from_v128(t, t); +} + +SIMD_INLINE v256 v256_dup_16(uint16_t x) { + v128 t = v128_dup_16(x); + return v256_from_v128(t, t); +} + +SIMD_INLINE v256 v256_dup_32(uint32_t x) { + v128 t = v128_dup_32(x); + return v256_from_v128(t, t); +} + +SIMD_INLINE int64_t v256_dotp_s16(v256 a, v256 b) { + return v128_dotp_s16(a.hi, b.hi) + v128_dotp_s16(a.lo, b.lo); +} + +SIMD_INLINE uint64_t v256_hadd_u8(v256 a) { + return v128_hadd_u8(a.hi) + v128_hadd_u8(a.lo); +} + +typedef struct { + sad128_internal hi; + sad128_internal lo; +} sad256_internal; + +SIMD_INLINE sad256_internal v256_sad_u8_init() { + sad256_internal t; + t.hi = v128_sad_u8_init(); + t.lo = v128_sad_u8_init(); + return t; +} + +/* Implementation dependent return value. Result must be finalised with + v256_sad_u8_sum(). + The result for more than 16 v256_sad_u8() calls is undefined. */ +SIMD_INLINE sad256_internal v256_sad_u8(sad256_internal s, v256 a, v256 b) { + sad256_internal t; + t.hi = v128_sad_u8(s.hi, a.hi, b.hi); + t.lo = v128_sad_u8(s.lo, a.lo, b.lo); + return t; +} + +SIMD_INLINE uint32_t v256_sad_u8_sum(sad256_internal s) { + return v128_sad_u8_sum(s.hi) + v128_sad_u8_sum(s.lo); +} + +typedef struct { + ssd128_internal hi; + ssd128_internal lo; +} ssd256_internal; + +SIMD_INLINE ssd256_internal v256_ssd_u8_init() { + ssd256_internal t; + t.hi = v128_ssd_u8_init(); + t.lo = v128_ssd_u8_init(); + return t; +} + +/* Implementation dependent return value. Result must be finalised with + * v256_ssd_u8_sum(). */ +SIMD_INLINE ssd256_internal v256_ssd_u8(ssd256_internal s, v256 a, v256 b) { + ssd256_internal t; + t.hi = v128_ssd_u8(s.hi, a.hi, b.hi); + t.lo = v128_ssd_u8(s.lo, a.lo, b.lo); + return t; +} + +SIMD_INLINE uint32_t v256_ssd_u8_sum(ssd256_internal s) { + return v128_ssd_u8_sum(s.hi) + v128_ssd_u8_sum(s.lo); +} + +SIMD_INLINE v256 v256_or(v256 a, v256 b) { + return v256_from_v128(v128_or(a.hi, b.hi), v128_or(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_xor(v256 a, v256 b) { + return v256_from_v128(v128_xor(a.hi, b.hi), v128_xor(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_and(v256 a, v256 b) { + return v256_from_v128(v128_and(a.hi, b.hi), v128_and(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_andn(v256 a, v256 b) { + return v256_from_v128(v128_andn(a.hi, b.hi), v128_andn(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_add_8(v256 a, v256 b) { + return v256_from_v128(v128_add_8(a.hi, b.hi), v128_add_8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_add_16(v256 a, v256 b) { + return v256_from_v128(v128_add_16(a.hi, b.hi), v128_add_16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_sadd_s16(v256 a, v256 b) { + return v256_from_v128(v128_sadd_s16(a.hi, b.hi), v128_sadd_s16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_add_32(v256 a, v256 b) { + return v256_from_v128(v128_add_32(a.hi, b.hi), v128_add_32(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_padd_s16(v256 a) { + return v256_from_v128(v128_padd_s16(a.hi), v128_padd_s16(a.lo)); +} + +SIMD_INLINE v256 v256_sub_8(v256 a, v256 b) { + return v256_from_v128(v128_sub_8(a.hi, b.hi), v128_sub_8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_ssub_u8(v256 a, v256 b) { + return v256_from_v128(v128_ssub_u8(a.hi, b.hi), v128_ssub_u8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_ssub_s8(v256 a, v256 b) { + return v256_from_v128(v128_ssub_s8(a.hi, b.hi), v128_ssub_s8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_sub_16(v256 a, v256 b) { + return v256_from_v128(v128_sub_16(a.hi, b.hi), v128_sub_16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_ssub_s16(v256 a, v256 b) { + return v256_from_v128(v128_ssub_s16(a.hi, b.hi), v128_ssub_s16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_ssub_u16(v256 a, v256 b) { + return v256_from_v128(v128_ssub_u16(a.hi, b.hi), v128_ssub_u16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_sub_32(v256 a, v256 b) { + return v256_from_v128(v128_sub_32(a.hi, b.hi), v128_sub_32(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_abs_s16(v256 a) { + return v256_from_v128(v128_abs_s16(a.hi), v128_abs_s16(a.lo)); +} + +SIMD_INLINE v256 v256_abs_s8(v256 a) { + return v256_from_v128(v128_abs_s8(a.hi), v128_abs_s8(a.lo)); +} + +SIMD_INLINE v256 v256_mul_s16(v128 a, v128 b) { + v128 lo_bits = v128_mullo_s16(a, b); + v128 hi_bits = v128_mulhi_s16(a, b); + return v256_from_v128(v128_ziphi_16(hi_bits, lo_bits), + v128_ziplo_16(hi_bits, lo_bits)); +} + +SIMD_INLINE v256 v256_mullo_s16(v256 a, v256 b) { + return v256_from_v128(v128_mullo_s16(a.hi, b.hi), v128_mullo_s16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_mulhi_s16(v256 a, v256 b) { + return v256_from_v128(v128_mulhi_s16(a.hi, b.hi), v128_mulhi_s16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_mullo_s32(v256 a, v256 b) { + return v256_from_v128(v128_mullo_s32(a.hi, b.hi), v128_mullo_s32(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_madd_s16(v256 a, v256 b) { + return v256_from_v128(v128_madd_s16(a.hi, b.hi), v128_madd_s16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_madd_us8(v256 a, v256 b) { + return v256_from_v128(v128_madd_us8(a.hi, b.hi), v128_madd_us8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_avg_u8(v256 a, v256 b) { + return v256_from_v128(v128_avg_u8(a.hi, b.hi), v128_avg_u8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_rdavg_u8(v256 a, v256 b) { + return v256_from_v128(v128_rdavg_u8(a.hi, b.hi), v128_rdavg_u8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_avg_u16(v256 a, v256 b) { + return v256_from_v128(v128_avg_u16(a.hi, b.hi), v128_avg_u16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_min_u8(v256 a, v256 b) { + return v256_from_v128(v128_min_u8(a.hi, b.hi), v128_min_u8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_max_u8(v256 a, v256 b) { + return v256_from_v128(v128_max_u8(a.hi, b.hi), v128_max_u8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_min_s8(v256 a, v256 b) { + return v256_from_v128(v128_min_s8(a.hi, b.hi), v128_min_s8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_max_s8(v256 a, v256 b) { + return v256_from_v128(v128_max_s8(a.hi, b.hi), v128_max_s8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_min_s16(v256 a, v256 b) { + return v256_from_v128(v128_min_s16(a.hi, b.hi), v128_min_s16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_max_s16(v256 a, v256 b) { + return v256_from_v128(v128_max_s16(a.hi, b.hi), v128_max_s16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_ziplo_8(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_8(a.lo, b.lo), v128_ziplo_8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_ziphi_8(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_8(a.hi, b.hi), v128_ziplo_8(a.hi, b.hi)); +} + +SIMD_INLINE v256 v256_ziplo_16(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_16(a.lo, b.lo), v128_ziplo_16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_ziphi_16(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_16(a.hi, b.hi), v128_ziplo_16(a.hi, b.hi)); +} + +SIMD_INLINE v256 v256_ziplo_32(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_32(a.lo, b.lo), v128_ziplo_32(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_ziphi_32(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_32(a.hi, b.hi), v128_ziplo_32(a.hi, b.hi)); +} + +SIMD_INLINE v256 v256_ziplo_64(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_64(a.lo, b.lo), v128_ziplo_64(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_ziphi_64(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_64(a.hi, b.hi), v128_ziplo_64(a.hi, b.hi)); +} + +SIMD_INLINE v256 v256_ziplo_128(v256 a, v256 b) { + return v256_from_v128(a.lo, b.lo); +} + +SIMD_INLINE v256 v256_ziphi_128(v256 a, v256 b) { + return v256_from_v128(a.hi, b.hi); +} + +SIMD_INLINE v256 v256_zip_8(v128 a, v128 b) { + return v256_from_v128(v128_ziphi_8(a, b), v128_ziplo_8(a, b)); +} + +SIMD_INLINE v256 v256_zip_16(v128 a, v128 b) { + return v256_from_v128(v128_ziphi_16(a, b), v128_ziplo_16(a, b)); +} + +SIMD_INLINE v256 v256_zip_32(v128 a, v128 b) { + return v256_from_v128(v128_ziphi_32(a, b), v128_ziplo_32(a, b)); +} + +SIMD_INLINE v256 v256_unziplo_8(v256 a, v256 b) { + return v256_from_v128(v128_unziplo_8(a.hi, a.lo), v128_unziplo_8(b.hi, b.lo)); +} + +SIMD_INLINE v256 v256_unziphi_8(v256 a, v256 b) { + return v256_from_v128(v128_unziphi_8(a.hi, a.lo), v128_unziphi_8(b.hi, b.lo)); +} + +SIMD_INLINE v256 v256_unziplo_16(v256 a, v256 b) { + return v256_from_v128(v128_unziplo_16(a.hi, a.lo), + v128_unziplo_16(b.hi, b.lo)); +} + +SIMD_INLINE v256 v256_unziphi_16(v256 a, v256 b) { + return v256_from_v128(v128_unziphi_16(a.hi, a.lo), + v128_unziphi_16(b.hi, b.lo)); +} + +SIMD_INLINE v256 v256_unziplo_32(v256 a, v256 b) { + return v256_from_v128(v128_unziplo_32(a.hi, a.lo), + v128_unziplo_32(b.hi, b.lo)); +} + +SIMD_INLINE v256 v256_unziphi_32(v256 a, v256 b) { + return v256_from_v128(v128_unziphi_32(a.hi, a.lo), + v128_unziphi_32(b.hi, b.lo)); +} + +SIMD_INLINE v256 v256_unpack_u8_s16(v128 a) { + return v256_from_v128(v128_unpackhi_u8_s16(a), v128_unpacklo_u8_s16(a)); +} + +SIMD_INLINE v256 v256_unpacklo_u8_s16(v256 a) { + return v256_from_v128(v128_unpackhi_u8_s16(a.lo), v128_unpacklo_u8_s16(a.lo)); +} + +SIMD_INLINE v256 v256_unpackhi_u8_s16(v256 a) { + return v256_from_v128(v128_unpackhi_u8_s16(a.hi), v128_unpacklo_u8_s16(a.hi)); +} + +SIMD_INLINE v256 v256_unpack_s8_s16(v128 a) { + return v256_from_v128(v128_unpackhi_s8_s16(a), v128_unpacklo_s8_s16(a)); +} + +SIMD_INLINE v256 v256_unpacklo_s8_s16(v256 a) { + return v256_from_v128(v128_unpackhi_s8_s16(a.lo), v128_unpacklo_s8_s16(a.lo)); +} + +SIMD_INLINE v256 v256_unpackhi_s8_s16(v256 a) { + return v256_from_v128(v128_unpackhi_s8_s16(a.hi), v128_unpacklo_s8_s16(a.hi)); +} + +SIMD_INLINE v256 v256_pack_s32_s16(v256 a, v256 b) { + return v256_from_v128(v128_pack_s32_s16(a.hi, a.lo), + v128_pack_s32_s16(b.hi, b.lo)); +} + +SIMD_INLINE v256 v256_pack_s16_u8(v256 a, v256 b) { + return v256_from_v128(v128_pack_s16_u8(a.hi, a.lo), + v128_pack_s16_u8(b.hi, b.lo)); +} + +SIMD_INLINE v256 v256_pack_s16_s8(v256 a, v256 b) { + return v256_from_v128(v128_pack_s16_s8(a.hi, a.lo), + v128_pack_s16_s8(b.hi, b.lo)); +} + +SIMD_INLINE v256 v256_unpack_u16_s32(v128 a) { + return v256_from_v128(v128_unpackhi_u16_s32(a), v128_unpacklo_u16_s32(a)); +} + +SIMD_INLINE v256 v256_unpack_s16_s32(v128 a) { + return v256_from_v128(v128_unpackhi_s16_s32(a), v128_unpacklo_s16_s32(a)); +} + +SIMD_INLINE v256 v256_unpacklo_u16_s32(v256 a) { + return v256_from_v128(v128_unpackhi_u16_s32(a.lo), + v128_unpacklo_u16_s32(a.lo)); +} + +SIMD_INLINE v256 v256_unpacklo_s16_s32(v256 a) { + return v256_from_v128(v128_unpackhi_s16_s32(a.lo), + v128_unpacklo_s16_s32(a.lo)); +} + +SIMD_INLINE v256 v256_unpackhi_u16_s32(v256 a) { + return v256_from_v128(v128_unpackhi_u16_s32(a.hi), + v128_unpacklo_u16_s32(a.hi)); +} + +SIMD_INLINE v256 v256_unpackhi_s16_s32(v256 a) { + return v256_from_v128(v128_unpackhi_s16_s32(a.hi), + v128_unpacklo_s16_s32(a.hi)); +} + +SIMD_INLINE v256 v256_shuffle_8(v256 a, v256 pattern) { + v128 c16 = v128_dup_8(16); + v128 maskhi = v128_cmplt_s8(pattern.hi, c16); + v128 masklo = v128_cmplt_s8(pattern.lo, c16); + return v256_from_v128( + v128_or( + v128_and(v128_shuffle_8(a.lo, pattern.hi), maskhi), + v128_andn(v128_shuffle_8(a.hi, v128_sub_8(pattern.hi, c16)), maskhi)), + v128_or(v128_and(v128_shuffle_8(a.lo, pattern.lo), masklo), + v128_andn(v128_shuffle_8(a.hi, v128_sub_8(pattern.lo, c16)), + masklo))); +} + +SIMD_INLINE v256 v256_pshuffle_8(v256 a, v256 pattern) { + return v256_from_v128( + v128_shuffle_8(v256_high_v128(a), v256_high_v128(pattern)), + v128_shuffle_8(v256_low_v128(a), v256_low_v128(pattern))); +} + +SIMD_INLINE v256 v256_cmpgt_s8(v256 a, v256 b) { + return v256_from_v128(v128_cmpgt_s8(a.hi, b.hi), v128_cmpgt_s8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_cmplt_s8(v256 a, v256 b) { + return v256_from_v128(v128_cmplt_s8(a.hi, b.hi), v128_cmplt_s8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_cmpeq_8(v256 a, v256 b) { + return v256_from_v128(v128_cmpeq_8(a.hi, b.hi), v128_cmpeq_8(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_cmpgt_s16(v256 a, v256 b) { + return v256_from_v128(v128_cmpgt_s16(a.hi, b.hi), v128_cmpgt_s16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_cmplt_s16(v256 a, v256 b) { + return v256_from_v128(v128_cmplt_s16(a.hi, b.hi), v128_cmplt_s16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_cmpeq_16(v256 a, v256 b) { + return v256_from_v128(v128_cmpeq_16(a.hi, b.hi), v128_cmpeq_16(a.lo, b.lo)); +} + +SIMD_INLINE v256 v256_shl_8(v256 a, unsigned int c) { + return v256_from_v128(v128_shl_8(a.hi, c), v128_shl_8(a.lo, c)); +} + +SIMD_INLINE v256 v256_shr_u8(v256 a, unsigned int c) { + return v256_from_v128(v128_shr_u8(a.hi, c), v128_shr_u8(a.lo, c)); +} + +SIMD_INLINE v256 v256_shr_s8(v256 a, unsigned int c) { + return v256_from_v128(v128_shr_s8(a.hi, c), v128_shr_s8(a.lo, c)); +} + +SIMD_INLINE v256 v256_shl_16(v256 a, unsigned int c) { + return v256_from_v128(v128_shl_16(a.hi, c), v128_shl_16(a.lo, c)); +} + +SIMD_INLINE v256 v256_shr_u16(v256 a, unsigned int c) { + return v256_from_v128(v128_shr_u16(a.hi, c), v128_shr_u16(a.lo, c)); +} + +SIMD_INLINE v256 v256_shr_s16(v256 a, unsigned int c) { + return v256_from_v128(v128_shr_s16(a.hi, c), v128_shr_s16(a.lo, c)); +} + +SIMD_INLINE v256 v256_shl_32(v256 a, unsigned int c) { + return v256_from_v128(v128_shl_32(a.hi, c), v128_shl_32(a.lo, c)); +} + +SIMD_INLINE v256 v256_shr_u32(v256 a, unsigned int c) { + return v256_from_v128(v128_shr_u32(a.hi, c), v128_shr_u32(a.lo, c)); +} + +SIMD_INLINE v256 v256_shr_s32(v256 a, unsigned int c) { + return v256_from_v128(v128_shr_s32(a.hi, c), v128_shr_s32(a.lo, c)); +} + +/* These intrinsics require immediate values, so we must use #defines + to enforce that. */ +#define v256_shl_n_byte(a, n) \ + ((n) < 16 ? v256_from_v128(v128_or(v128_shl_n_byte(a.hi, n), \ + v128_shr_n_byte(a.lo, 16 - (n))), \ + v128_shl_n_byte(a.lo, (n))) \ + : v256_from_v128((n) > 16 ? v128_shl_n_byte(a.lo, (n)-16) : a.lo, \ + v128_zero())) + +#define v256_shr_n_byte(a, n) \ + ((n) < 16 ? v256_from_v128(v128_shr_n_byte(a.hi, n), \ + v128_or(v128_shr_n_byte(a.lo, n), \ + v128_shl_n_byte(a.hi, 16 - (n)))) \ + : v256_from_v128(v128_zero(), \ + (n) > 16 ? v128_shr_n_byte(a.hi, (n)-16) : a.hi)) + +#define v256_align(a, b, c) \ + ((c) ? v256_or(v256_shr_n_byte(b, c), v256_shl_n_byte(a, 32 - (c))) : b) + +#define v256_shl_n_8(a, n) \ + v256_from_v128(v128_shl_n_8(a.hi, n), v128_shl_n_8(a.lo, n)) +#define v256_shl_n_16(a, n) \ + v256_from_v128(v128_shl_n_16(a.hi, n), v128_shl_n_16(a.lo, n)) +#define v256_shl_n_32(a, n) \ + v256_from_v128(v128_shl_n_32(a.hi, n), v128_shl_n_32(a.lo, n)) +#define v256_shr_n_u8(a, n) \ + v256_from_v128(v128_shr_n_u8(a.hi, n), v128_shr_n_u8(a.lo, n)) +#define v256_shr_n_u16(a, n) \ + v256_from_v128(v128_shr_n_u16(a.hi, n), v128_shr_n_u16(a.lo, n)) +#define v256_shr_n_u32(a, n) \ + v256_from_v128(v128_shr_n_u32(a.hi, n), v128_shr_n_u32(a.lo, n)) +#define v256_shr_n_s8(a, n) \ + v256_from_v128(v128_shr_n_s8(a.hi, n), v128_shr_n_s8(a.lo, n)) +#define v256_shr_n_s16(a, n) \ + v256_from_v128(v128_shr_n_s16(a.hi, n), v128_shr_n_s16(a.lo, n)) +#define v256_shr_n_s32(a, n) \ + v256_from_v128(v128_shr_n_s32(a.hi, n), v128_shr_n_s32(a.lo, n)) + +#endif /* _V256_INTRINSICS_V128_H */ diff --git a/third_party/aom/aom_dsp/simd/v256_intrinsics_x86.h b/third_party/aom/aom_dsp/simd/v256_intrinsics_x86.h new file mode 100644 index 0000000000..b82daab686 --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v256_intrinsics_x86.h @@ -0,0 +1,548 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V256_INTRINSICS_H +#define _V256_INTRINSICS_H + +#if !defined(__AVX2__) + +#include "./v256_intrinsics_v128.h" + +#else + +// The _m256i type seems to cause problems for g++'s mangling prior to +// version 5, but adding -fabi-version=0 fixes this. +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 && \ + defined(__AVX2__) && defined(__cplusplus) +#pragma GCC optimize "-fabi-version=0" +#endif + +#include +#include "./v128_intrinsics_x86.h" + +typedef __m256i v256; + +SIMD_INLINE uint32_t v256_low_u32(v256 a) { + return (uint32_t)_mm_cvtsi128_si32(_mm256_extracti128_si256(a, 0)); +} + +SIMD_INLINE v64 v256_low_v64(v256 a) { + return _mm_unpacklo_epi64(_mm256_extracti128_si256(a, 0), v64_zero()); +} + +SIMD_INLINE v128 v256_low_v128(v256 a) { + return _mm256_extracti128_si256(a, 0); +} + +SIMD_INLINE v128 v256_high_v128(v256 a) { + return _mm256_extracti128_si256(a, 1); +} + +SIMD_INLINE v256 v256_from_v128(v128 a, v128 b) { + // gcc seems to be missing _mm256_set_m128i() + return _mm256_insertf128_si256( + _mm256_insertf128_si256(_mm256_setzero_si256(), b, 0), a, 1); +} + +SIMD_INLINE v256 v256_from_v64(v64 a, v64 b, v64 c, v64 d) { + return v256_from_v128(v128_from_v64(a, b), v128_from_v64(c, d)); +} + +SIMD_INLINE v256 v256_from_64(uint64_t a, uint64_t b, uint64_t c, uint64_t d) { + return v256_from_v128(v128_from_64(a, b), v128_from_64(c, d)); +} + +SIMD_INLINE v256 v256_load_aligned(const void *p) { + return _mm256_load_si256((const __m256i *)p); +} + +SIMD_INLINE v256 v256_load_unaligned(const void *p) { + return _mm256_loadu_si256((const __m256i *)p); +} + +SIMD_INLINE void v256_store_aligned(void *p, v256 a) { + _mm256_store_si256((__m256i *)p, a); +} + +SIMD_INLINE void v256_store_unaligned(void *p, v256 a) { + _mm256_storeu_si256((__m256i *)p, a); +} + +SIMD_INLINE v256 v256_zero() { return _mm256_setzero_si256(); } + +SIMD_INLINE v256 v256_dup_8(uint8_t x) { return _mm256_set1_epi8(x); } + +SIMD_INLINE v256 v256_dup_16(uint16_t x) { return _mm256_set1_epi16(x); } + +SIMD_INLINE v256 v256_dup_32(uint32_t x) { return _mm256_set1_epi32(x); } + +SIMD_INLINE v256 v256_add_8(v256 a, v256 b) { return _mm256_add_epi8(a, b); } + +SIMD_INLINE v256 v256_add_16(v256 a, v256 b) { return _mm256_add_epi16(a, b); } + +SIMD_INLINE v256 v256_sadd_s16(v256 a, v256 b) { + return _mm256_adds_epi16(a, b); +} + +SIMD_INLINE v256 v256_add_32(v256 a, v256 b) { return _mm256_add_epi32(a, b); } + +SIMD_INLINE v256 v256_padd_s16(v256 a) { + return _mm256_madd_epi16(a, _mm256_set1_epi16(1)); +} + +SIMD_INLINE v256 v256_sub_8(v256 a, v256 b) { return _mm256_sub_epi8(a, b); } + +SIMD_INLINE v256 v256_ssub_u8(v256 a, v256 b) { return _mm256_subs_epu8(a, b); } + +SIMD_INLINE v256 v256_ssub_s8(v256 a, v256 b) { return _mm256_subs_epi8(a, b); } + +SIMD_INLINE v256 v256_sub_16(v256 a, v256 b) { return _mm256_sub_epi16(a, b); } + +SIMD_INLINE v256 v256_ssub_s16(v256 a, v256 b) { + return _mm256_subs_epi16(a, b); +} + +SIMD_INLINE v256 v256_ssub_u16(v256 a, v256 b) { + return _mm256_subs_epu16(a, b); +} + +SIMD_INLINE v256 v256_sub_32(v256 a, v256 b) { return _mm256_sub_epi32(a, b); } + +SIMD_INLINE v256 v256_abs_s16(v256 a) { return _mm256_abs_epi16(a); } + +SIMD_INLINE v256 v256_abs_s8(v256 a) { return _mm256_abs_epi8(a); } + +// AVX doesn't have the direct intrinsics to zip/unzip 8, 16, 32 bit +// lanes of lower or upper halves of a 256bit vector because the +// unpack/pack intrinsics operate on the 256 bit input vector as 2 +// independent 128 bit vectors. +SIMD_INLINE v256 v256_ziplo_8(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_8(v256_low_v128(a), v256_low_v128(b)), + v128_ziplo_8(v256_low_v128(a), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_ziphi_8(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_8(v256_high_v128(a), v256_high_v128(b)), + v128_ziplo_8(v256_high_v128(a), v256_high_v128(b))); +} + +SIMD_INLINE v256 v256_ziplo_16(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_16(v256_low_v128(a), v256_low_v128(b)), + v128_ziplo_16(v256_low_v128(a), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_ziphi_16(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_16(v256_high_v128(a), v256_high_v128(b)), + v128_ziplo_16(v256_high_v128(a), v256_high_v128(b))); +} + +SIMD_INLINE v256 v256_ziplo_32(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_32(v256_low_v128(a), v256_low_v128(b)), + v128_ziplo_32(v256_low_v128(a), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_ziphi_32(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_32(v256_high_v128(a), v256_high_v128(b)), + v128_ziplo_32(v256_high_v128(a), v256_high_v128(b))); +} + +SIMD_INLINE v256 v256_ziplo_64(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_64(v256_low_v128(a), v256_low_v128(b)), + v128_ziplo_64(v256_low_v128(a), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_ziphi_64(v256 a, v256 b) { + return v256_from_v128(v128_ziphi_64(v256_high_v128(a), v256_high_v128(b)), + v128_ziplo_64(v256_high_v128(a), v256_high_v128(b))); +} + +SIMD_INLINE v256 v256_ziplo_128(v256 a, v256 b) { + return v256_from_v128(v256_low_v128(a), v256_low_v128(b)); +} + +SIMD_INLINE v256 v256_ziphi_128(v256 a, v256 b) { + return v256_from_v128(v256_high_v128(a), v256_high_v128(b)); +} + +SIMD_INLINE v256 v256_zip_8(v128 a, v128 b) { + return v256_from_v128(v128_ziphi_8(a, b), v128_ziplo_8(a, b)); +} + +SIMD_INLINE v256 v256_zip_16(v128 a, v128 b) { + return v256_from_v128(v128_ziphi_16(a, b), v128_ziplo_16(a, b)); +} + +SIMD_INLINE v256 v256_zip_32(v128 a, v128 b) { + return v256_from_v128(v128_ziphi_32(a, b), v128_ziplo_32(a, b)); +} + +SIMD_INLINE v256 v256_unziplo_8(v256 a, v256 b) { + return v256_from_v128(v128_unziplo_8(v256_high_v128(a), v256_low_v128(a)), + v128_unziplo_8(v256_high_v128(b), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_unziphi_8(v256 a, v256 b) { + return v256_from_v128(v128_unziphi_8(v256_high_v128(a), v256_low_v128(a)), + v128_unziphi_8(v256_high_v128(b), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_unziplo_16(v256 a, v256 b) { + return v256_from_v128(v128_unziplo_16(v256_high_v128(a), v256_low_v128(a)), + v128_unziplo_16(v256_high_v128(b), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_unziphi_16(v256 a, v256 b) { + return v256_from_v128(v128_unziphi_16(v256_high_v128(a), v256_low_v128(a)), + v128_unziphi_16(v256_high_v128(b), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_unziplo_32(v256 a, v256 b) { + return v256_from_v128(v128_unziplo_32(v256_high_v128(a), v256_low_v128(a)), + v128_unziplo_32(v256_high_v128(b), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_unziphi_32(v256 a, v256 b) { + return v256_from_v128(v128_unziphi_32(v256_high_v128(a), v256_low_v128(a)), + v128_unziphi_32(v256_high_v128(b), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_unpack_u8_s16(v128 a) { + return v256_from_v128(v128_unpackhi_u8_s16(a), v128_unpacklo_u8_s16(a)); +} + +SIMD_INLINE v256 v256_unpacklo_u8_s16(v256 a) { + return v256_from_v128(v128_unpackhi_u8_s16(v256_low_v128(a)), + v128_unpacklo_u8_s16(v256_low_v128(a))); +} + +SIMD_INLINE v256 v256_unpackhi_u8_s16(v256 a) { + return v256_from_v128(v128_unpackhi_u8_s16(v256_high_v128(a)), + v128_unpacklo_u8_s16(v256_high_v128(a))); +} + +SIMD_INLINE v256 v256_unpack_s8_s16(v128 a) { + return v256_from_v128(v128_unpackhi_s8_s16(a), v128_unpacklo_s8_s16(a)); +} + +SIMD_INLINE v256 v256_unpacklo_s8_s16(v256 a) { + return v256_from_v128(v128_unpackhi_s8_s16(v256_low_v128(a)), + v128_unpacklo_s8_s16(v256_low_v128(a))); +} + +SIMD_INLINE v256 v256_unpackhi_s8_s16(v256 a) { + return v256_from_v128(v128_unpackhi_s8_s16(v256_high_v128(a)), + v128_unpacklo_s8_s16(v256_high_v128(a))); +} + +SIMD_INLINE v256 v256_pack_s32_s16(v256 a, v256 b) { + return v256_from_v128(v128_pack_s32_s16(v256_high_v128(a), v256_low_v128(a)), + v128_pack_s32_s16(v256_high_v128(b), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_pack_s16_u8(v256 a, v256 b) { + return v256_from_v128(v128_pack_s16_u8(v256_high_v128(a), v256_low_v128(a)), + v128_pack_s16_u8(v256_high_v128(b), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_pack_s16_s8(v256 a, v256 b) { + return v256_from_v128(v128_pack_s16_s8(v256_high_v128(a), v256_low_v128(a)), + v128_pack_s16_s8(v256_high_v128(b), v256_low_v128(b))); +} + +SIMD_INLINE v256 v256_unpack_u16_s32(v128 a) { + return v256_from_v128(v128_unpackhi_u16_s32(a), v128_unpacklo_u16_s32(a)); +} + +SIMD_INLINE v256 v256_unpack_s16_s32(v128 a) { + return v256_from_v128(v128_unpackhi_s16_s32(a), v128_unpacklo_s16_s32(a)); +} + +SIMD_INLINE v256 v256_unpacklo_u16_s32(v256 a) { + return v256_from_v128(v128_unpackhi_u16_s32(v256_low_v128(a)), + v128_unpacklo_u16_s32(v256_low_v128(a))); +} + +SIMD_INLINE v256 v256_unpacklo_s16_s32(v256 a) { + return v256_from_v128(v128_unpackhi_s16_s32(v256_low_v128(a)), + v128_unpacklo_s16_s32(v256_low_v128(a))); +} + +SIMD_INLINE v256 v256_unpackhi_u16_s32(v256 a) { + return v256_from_v128(v128_unpackhi_u16_s32(v256_high_v128(a)), + v128_unpacklo_u16_s32(v256_high_v128(a))); +} + +SIMD_INLINE v256 v256_unpackhi_s16_s32(v256 a) { + return v256_from_v128(v128_unpackhi_s16_s32(v256_high_v128(a)), + v128_unpacklo_s16_s32(v256_high_v128(a))); +} +SIMD_INLINE v256 v256_shuffle_8(v256 a, v256 pattern) { + v128 c16 = v128_dup_8(16); + v128 hi = v256_high_v128(pattern); + v128 lo = v256_low_v128(pattern); + v128 maskhi = v128_cmplt_s8(hi, c16); + v128 masklo = v128_cmplt_s8(lo, c16); + return v256_from_v128( + v128_or(v128_and(v128_shuffle_8(v256_low_v128(a), hi), maskhi), + v128_andn(v128_shuffle_8(v256_high_v128(a), v128_sub_8(hi, c16)), + maskhi)), + v128_or(v128_and(v128_shuffle_8(v256_low_v128(a), lo), masklo), + v128_andn(v128_shuffle_8(v256_high_v128(a), v128_sub_8(lo, c16)), + masklo))); +} + +SIMD_INLINE v256 v256_pshuffle_8(v256 a, v256 pattern) { + return _mm256_shuffle_epi8(a, pattern); +} + +SIMD_INLINE int64_t v256_dotp_s16(v256 a, v256 b) { + v256 r = _mm256_madd_epi16(a, b); +#if defined(__x86_64__) + v128 t; + r = _mm256_add_epi64(_mm256_cvtepi32_epi64(v256_high_v128(r)), + _mm256_cvtepi32_epi64(v256_low_v128(r))); + t = v256_low_v128(_mm256_add_epi64( + r, _mm256_permute2x128_si256(r, r, _MM_SHUFFLE(2, 0, 0, 1)))); + return _mm_cvtsi128_si64(_mm_add_epi64(t, _mm_srli_si128(t, 8))); +#else + v128 l = v256_low_v128(r); + v128 h = v256_high_v128(r); + return (int64_t)_mm_cvtsi128_si32(l) + + (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(l, 4)) + + (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(l, 8)) + + (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(l, 12)) + + (int64_t)_mm_cvtsi128_si32(h) + + (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(h, 4)) + + (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(h, 8)) + + (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(h, 12)); +#endif +} + +SIMD_INLINE uint64_t v256_hadd_u8(v256 a) { + v256 t = _mm256_sad_epu8(a, _mm256_setzero_si256()); + v128 lo = v256_low_v128(t); + v128 hi = v256_high_v128(t); + lo = v128_add_32(lo, hi); + return v64_low_u32(v128_low_v64(lo)) + v128_low_u32(v128_high_v64(lo)); +} + +typedef v256 sad256_internal; + +SIMD_INLINE sad256_internal v256_sad_u8_init() { + return _mm256_setzero_si256(); +} + +/* Implementation dependent return value. Result must be finalised with + v256_sad_sum(). + The result for more than 32 v256_sad_u8() calls is undefined. */ +SIMD_INLINE sad256_internal v256_sad_u8(sad256_internal s, v256 a, v256 b) { + return _mm256_add_epi64(s, _mm256_sad_epu8(a, b)); +} + +SIMD_INLINE uint32_t v256_sad_u8_sum(sad256_internal s) { + v256 t = _mm256_add_epi32(s, _mm256_unpackhi_epi64(s, s)); + return v128_low_u32(_mm_add_epi32(v256_high_v128(t), v256_low_v128(t))); +} + +typedef v256 ssd256_internal; + +SIMD_INLINE ssd256_internal v256_ssd_u8_init() { + return _mm256_setzero_si256(); +} + +/* Implementation dependent return value. Result must be finalised with + * v256_ssd_sum(). */ +SIMD_INLINE ssd256_internal v256_ssd_u8(ssd256_internal s, v256 a, v256 b) { + v256 l = _mm256_sub_epi16(_mm256_unpacklo_epi8(a, _mm256_setzero_si256()), + _mm256_unpacklo_epi8(b, _mm256_setzero_si256())); + v256 h = _mm256_sub_epi16(_mm256_unpackhi_epi8(a, _mm256_setzero_si256()), + _mm256_unpackhi_epi8(b, _mm256_setzero_si256())); + v256 rl = _mm256_madd_epi16(l, l); + v256 rh = _mm256_madd_epi16(h, h); + v128 c = _mm_cvtsi32_si128(32); + rl = _mm256_add_epi32(rl, _mm256_srli_si256(rl, 8)); + rl = _mm256_add_epi32(rl, _mm256_srli_si256(rl, 4)); + rh = _mm256_add_epi32(rh, _mm256_srli_si256(rh, 8)); + rh = _mm256_add_epi32(rh, _mm256_srli_si256(rh, 4)); + return _mm256_add_epi64( + s, + _mm256_srl_epi64(_mm256_sll_epi64(_mm256_unpacklo_epi64(rl, rh), c), c)); +} + +SIMD_INLINE uint32_t v256_ssd_u8_sum(ssd256_internal s) { + v256 t = _mm256_add_epi32(s, _mm256_unpackhi_epi64(s, s)); + return v128_low_u32(_mm_add_epi32(v256_high_v128(t), v256_low_v128(t))); +} + +SIMD_INLINE v256 v256_or(v256 a, v256 b) { return _mm256_or_si256(a, b); } + +SIMD_INLINE v256 v256_xor(v256 a, v256 b) { return _mm256_xor_si256(a, b); } + +SIMD_INLINE v256 v256_and(v256 a, v256 b) { return _mm256_and_si256(a, b); } + +SIMD_INLINE v256 v256_andn(v256 a, v256 b) { return _mm256_andnot_si256(b, a); } + +SIMD_INLINE v256 v256_mul_s16(v64 a, v64 b) { + v128 lo_bits = v128_mullo_s16(a, b); + v128 hi_bits = v128_mulhi_s16(a, b); + return v256_from_v128(v128_ziphi_16(hi_bits, lo_bits), + v128_ziplo_16(hi_bits, lo_bits)); +} + +SIMD_INLINE v256 v256_mullo_s16(v256 a, v256 b) { + return _mm256_mullo_epi16(a, b); +} + +SIMD_INLINE v256 v256_mulhi_s16(v256 a, v256 b) { + return _mm256_mulhi_epi16(a, b); +} + +SIMD_INLINE v256 v256_mullo_s32(v256 a, v256 b) { + return _mm256_mullo_epi32(a, b); +} + +SIMD_INLINE v256 v256_madd_s16(v256 a, v256 b) { + return _mm256_madd_epi16(a, b); +} + +SIMD_INLINE v256 v256_madd_us8(v256 a, v256 b) { + return _mm256_maddubs_epi16(a, b); +} + +SIMD_INLINE v256 v256_avg_u8(v256 a, v256 b) { return _mm256_avg_epu8(a, b); } + +SIMD_INLINE v256 v256_rdavg_u8(v256 a, v256 b) { + return _mm256_sub_epi8( + _mm256_avg_epu8(a, b), + _mm256_and_si256(_mm256_xor_si256(a, b), v256_dup_8(1))); +} + +SIMD_INLINE v256 v256_avg_u16(v256 a, v256 b) { return _mm256_avg_epu16(a, b); } + +SIMD_INLINE v256 v256_min_u8(v256 a, v256 b) { return _mm256_min_epu8(a, b); } + +SIMD_INLINE v256 v256_max_u8(v256 a, v256 b) { return _mm256_max_epu8(a, b); } + +SIMD_INLINE v256 v256_min_s8(v256 a, v256 b) { return _mm256_min_epi8(a, b); } + +SIMD_INLINE v256 v256_max_s8(v256 a, v256 b) { return _mm256_max_epi8(a, b); } + +SIMD_INLINE v256 v256_min_s16(v256 a, v256 b) { return _mm256_min_epi16(a, b); } + +SIMD_INLINE v256 v256_max_s16(v256 a, v256 b) { return _mm256_max_epi16(a, b); } + +SIMD_INLINE v256 v256_cmpgt_s8(v256 a, v256 b) { + return _mm256_cmpgt_epi8(a, b); +} + +SIMD_INLINE v256 v256_cmplt_s8(v256 a, v256 b) { + return v256_andn(_mm256_cmpgt_epi8(b, a), _mm256_cmpeq_epi8(b, a)); +} + +SIMD_INLINE v256 v256_cmpeq_8(v256 a, v256 b) { + return _mm256_cmpeq_epi8(a, b); +} + +SIMD_INLINE v256 v256_cmpgt_s16(v256 a, v256 b) { + return _mm256_cmpgt_epi16(a, b); +} + +SIMD_INLINE v256 v256_cmplt_s16(v256 a, v256 b) { + return v256_andn(_mm256_cmpgt_epi16(b, a), _mm256_cmpeq_epi16(b, a)); +} + +SIMD_INLINE v256 v256_cmpeq_16(v256 a, v256 b) { + return _mm256_cmpeq_epi16(a, b); +} + +SIMD_INLINE v256 v256_shl_8(v256 a, unsigned int c) { + return _mm256_and_si256(_mm256_set1_epi8((uint8_t)(0xff << c)), + _mm256_sll_epi16(a, _mm_cvtsi32_si128(c))); +} + +SIMD_INLINE v256 v256_shr_u8(v256 a, unsigned int c) { + return _mm256_and_si256(_mm256_set1_epi8(0xff >> c), + _mm256_srl_epi16(a, _mm_cvtsi32_si128(c))); +} + +SIMD_INLINE v256 v256_shr_s8(v256 a, unsigned int c) { + __m128i x = _mm_cvtsi32_si128(c + 8); + return _mm256_packs_epi16(_mm256_sra_epi16(_mm256_unpacklo_epi8(a, a), x), + _mm256_sra_epi16(_mm256_unpackhi_epi8(a, a), x)); +} + +SIMD_INLINE v256 v256_shl_16(v256 a, unsigned int c) { + return _mm256_sll_epi16(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v256 v256_shr_u16(v256 a, unsigned int c) { + return _mm256_srl_epi16(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v256 v256_shr_s16(v256 a, unsigned int c) { + return _mm256_sra_epi16(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v256 v256_shl_32(v256 a, unsigned int c) { + return _mm256_sll_epi32(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v256 v256_shr_u32(v256 a, unsigned int c) { + return _mm256_srl_epi32(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v256 v256_shr_s32(v256 a, unsigned int c) { + return _mm256_sra_epi32(a, _mm_cvtsi32_si128(c)); +} + +/* These intrinsics require immediate values, so we must use #defines + to enforce that. */ +// _mm256_slli_si256 works on 128 bit lanes and can't be used +#define v256_shl_n_byte(a, n) \ + ((n) < 16 \ + ? v256_from_v128(v128_or(v128_shl_n_byte(v256_high_v128(a), n), \ + v128_shr_n_byte(v256_low_v128(a), 16 - (n))), \ + v128_shl_n_byte(v256_low_v128(a), n)) \ + : v256_from_v128(v128_shl_n_byte(v256_low_v128(a), (n)-16), \ + v128_zero())) + +// _mm256_srli_si256 works on 128 bit lanes and can't be used +#define v256_shr_n_byte(a, n) \ + ((n) < 16 \ + ? _mm256_alignr_epi8( \ + _mm256_permute2x128_si256(a, a, _MM_SHUFFLE(2, 0, 0, 1)), a, n) \ + : ((n) > 16 \ + ? _mm256_srli_si256( \ + _mm256_permute2x128_si256(a, a, _MM_SHUFFLE(2, 0, 0, 1)), \ + (n)-16) \ + : _mm256_permute2x128_si256(a, a, _MM_SHUFFLE(2, 0, 0, 1)))) + +// _mm256_alignr_epi8 works on two 128 bit lanes and can't be used +#define v256_align(a, b, c) \ + ((c) ? v256_or(v256_shr_n_byte(b, c), v256_shl_n_byte(a, 32 - c)) : b) + +#define v256_shl_n_8(a, c) \ + _mm256_and_si256(_mm256_set1_epi8((uint8_t)(0xff << (c))), \ + _mm256_slli_epi16(a, c)) +#define v256_shr_n_u8(a, c) \ + _mm256_and_si256(_mm256_set1_epi8(0xff >> (c)), _mm256_srli_epi16(a, c)) +#define v256_shr_n_s8(a, c) \ + _mm256_packs_epi16(_mm256_srai_epi16(_mm256_unpacklo_epi8(a, a), (c) + 8), \ + _mm256_srai_epi16(_mm256_unpackhi_epi8(a, a), (c) + 8)) +#define v256_shl_n_16(a, c) _mm256_slli_epi16(a, c) +#define v256_shr_n_u16(a, c) _mm256_srli_epi16(a, c) +#define v256_shr_n_s16(a, c) _mm256_srai_epi16(a, c) +#define v256_shl_n_32(a, c) _mm256_slli_epi32(a, c) +#define v256_shr_n_u32(a, c) _mm256_srli_epi32(a, c) +#define v256_shr_n_s32(a, c) _mm256_srai_epi32(a, c) +#endif + +#endif /* _V256_INTRINSICS_H */ diff --git a/third_party/aom/aom_dsp/simd/v64_intrinsics.h b/third_party/aom/aom_dsp/simd/v64_intrinsics.h new file mode 100644 index 0000000000..ee2b683a44 --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v64_intrinsics.h @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V64_INTRINSICS_H +#define _V64_INTRINSICS_H + +#include +#include +#include "./v64_intrinsics_c.h" + +/* Fallback to plain, unoptimised C. */ + +typedef c_v64 v64; + +SIMD_INLINE uint32_t v64_low_u32(v64 a) { return c_v64_low_u32(a); } +SIMD_INLINE uint32_t v64_high_u32(v64 a) { return c_v64_high_u32(a); } +SIMD_INLINE int32_t v64_low_s32(v64 a) { return c_v64_low_s32(a); } +SIMD_INLINE int32_t v64_high_s32(v64 a) { return c_v64_high_s32(a); } +SIMD_INLINE v64 v64_from_32(uint32_t x, uint32_t y) { + return c_v64_from_32(x, y); +} +SIMD_INLINE v64 v64_from_64(uint64_t x) { return c_v64_from_64(x); } +SIMD_INLINE uint64_t v64_u64(v64 x) { return c_v64_u64(x); } +SIMD_INLINE v64 v64_from_16(uint16_t a, uint16_t b, uint16_t c, uint16_t d) { + return c_v64_from_16(a, b, c, d); +} + +SIMD_INLINE uint32_t u32_load_unaligned(const void *p) { + return c_u32_load_unaligned(p); +} +SIMD_INLINE uint32_t u32_load_aligned(const void *p) { + return c_u32_load_aligned(p); +} +SIMD_INLINE void u32_store_unaligned(void *p, uint32_t a) { + c_u32_store_unaligned(p, a); +} +SIMD_INLINE void u32_store_aligned(void *p, uint32_t a) { + c_u32_store_aligned(p, a); +} + +SIMD_INLINE v64 v64_load_unaligned(const void *p) { + return c_v64_load_unaligned(p); +} +SIMD_INLINE v64 v64_load_aligned(const void *p) { + return c_v64_load_aligned(p); +} + +SIMD_INLINE void v64_store_unaligned(void *p, v64 a) { + c_v64_store_unaligned(p, a); +} +SIMD_INLINE void v64_store_aligned(void *p, v64 a) { + c_v64_store_aligned(p, a); +} + +SIMD_INLINE v64 v64_align(v64 a, v64 b, c) { return c_v64_align(a, b, c); } + +SIMD_INLINE v64 v64_zero() { return c_v64_zero(); } +SIMD_INLINE v64 v64_dup_8(uint8_t x) { return c_v64_dup_8(x); } +SIMD_INLINE v64 v64_dup_16(uint16_t x) { return c_v64_dup_16(x); } +SIMD_INLINE v64 v64_dup_32(uint32_t x) { return c_v64_dup_32(x); } + +SIMD_INLINE v64 v64_add_8(v64 a, v64 b) { return c_v64_add_8(a, b); } +SIMD_INLINE v64 v64_add_16(v64 a, v64 b) { return c_v64_add_16(a, b); } +SIMD_INLINE v64 v64_sadd_s16(v64 a, v64 b) { return c_v64_sadd_s16(a, b); } +SIMD_INLINE v64 v64_add_32(v64 a, v64 b) { return c_v64_add_32(a, b); } +SIMD_INLINE v64 v64_sub_8(v64 a, v64 b) { return c_v64_sub_8(a, b); } +SIMD_INLINE v64 v64_ssub_u8(v64 a, v64 b) { return c_v64_ssub_u8(a, b); } +SIMD_INLINE v64 v64_ssub_s8(v64 a, v64 b) { return c_v64_ssub_s8(a, b); } +SIMD_INLINE v64 v64_sub_16(v64 a, v64 b) { return c_v64_sub_16(a, b); } +SIMD_INLINE v64 v64_ssub_s16(v64 a, v64 b) { return c_v64_ssub_s16(a, b); } +SIMD_INLINE v64 v64_ssub_u16(v64 a, v64 b) { return c_v64_ssub_u16(a, b); } +SIMD_INLINE v64 v64_sub_32(v64 a, v64 b) { return c_v64_sub_32(a, b); } +SIMD_INLINE v64 v64_abs_s16(v64 a) { return c_v64_abs_s16(a); } +SIMD_INLINE v64 v64_abs_s8(v64 a) { return c_v64_abs_s8(a); } + +SIMD_INLINE v64 v64_ziplo_8(v64 a, v64 b) { return c_v64_ziplo_8(a, b); } +SIMD_INLINE v64 v64_ziphi_8(v64 a, v64 b) { return c_v64_ziphi_8(a, b); } +SIMD_INLINE v64 v64_ziplo_16(v64 a, v64 b) { return c_v64_ziplo_16(a, b); } +SIMD_INLINE v64 v64_ziphi_16(v64 a, v64 b) { return c_v64_ziphi_16(a, b); } +SIMD_INLINE v64 v64_ziplo_32(v64 a, v64 b) { return c_v64_ziplo_32(a, b); } +SIMD_INLINE v64 v64_ziphi_32(v64 a, v64 b) { return c_v64_ziphi_32(a, b); } +SIMD_INLINE v64 v64_unziplo_8(v64 a, v64 b) { return c_v64_unziplo_8(a, b); } +SIMD_INLINE v64 v64_unziphi_8(v64 a, v64 b) { return c_v64_unziphi_8(a, b); } +SIMD_INLINE v64 v64_unziplo_16(v64 a, v64 b) { return c_v64_unziplo_16(a, b); } +SIMD_INLINE v64 v64_unziphi_16(v64 a, v64 b) { return c_v64_unziphi_16(a, b); } +SIMD_INLINE v64 v64_unpacklo_u8_s16(v64 a) { return c_v64_unpacklo_u8_s16(a); } +SIMD_INLINE v64 v64_unpackhi_u8_s16(v64 a) { return c_v64_unpackhi_u8_s16(a); } +SIMD_INLINE v64 v64_unpacklo_s8_s16(v64 a) { return c_v64_unpacklo_s8_s16(a); } +SIMD_INLINE v64 v64_unpackhi_s8_s16(v64 a) { return c_v64_unpackhi_s8_s16(a); } +SIMD_INLINE v64 v64_pack_s32_s16(v64 a, v64 b) { + return c_v64_pack_s32_s16(a, b); +} +SIMD_INLINE v64 v64_pack_s16_u8(v64 a, v64 b) { + return c_v64_pack_s16_u8(a, b); +} +SIMD_INLINE v64 v64_pack_s16_s8(v64 a, v64 b) { + return c_v64_pack_s16_s8(a, b); +} +SIMD_INLINE v64 v64_unpacklo_u16_s32(v64 a) { + return c_v64_unpacklo_u16_s32(a); +} +SIMD_INLINE v64 v64_unpacklo_s16_s32(v64 a) { + return c_v64_unpacklo_s16_s32(a); +} +SIMD_INLINE v64 v64_unpackhi_u16_s32(v64 a) { + return c_v64_unpackhi_u16_s32(a); +} +SIMD_INLINE v64 v64_unpackhi_s16_s32(v64 a) { + return c_v64_unpackhi_s16_s32(a); +} +SIMD_INLINE v64 v64_shuffle_8(v64 a, v64 pattern) { + return c_v64_shuffle_8(a, pattern); +} + +typedef uint32_t sad64_internal; +SIMD_INLINE sad64_internal v64_sad_u8_init() { return c_v64_sad_u8_init(); } +SIMD_INLINE sad64_internal v64_sad_u8(sad64_internal s, v64 a, v64 b) { + return c_v64_sad_u8(s, a, b); +} +SIMD_INLINE uint32_t v64_sad_u8_sum(sad64_internal s) { + return c_v64_sad_u8_sum(s); +} +typedef uint32_t ssd64_internal; +SIMD_INLINE ssd64_internal v64_ssd_u8_init() { return c_v64_ssd_u8_init(); } +SIMD_INLINE ssd64_internal v64_ssd_u8(ssd64_internal s, v64 a, v64 b) { + return c_v64_ssd_u8(s, a, b); +} +SIMD_INLINE uint32_t v64_ssd_u8_sum(ssd64_internal s) { + return c_v64_ssd_u8_sum(s); +} +SIMD_INLINE int64_t v64_dotp_su8(v64 a, v64 b) { return c_v64_dotp_su8(a, b); } +SIMD_INLINE int64_t v64_dotp_s16(v64 a, v64 b) { return c_v64_dotp_s16(a, b); } +SIMD_INLINE uint64_t v64_hadd_u8(v64 a) { return c_v64_hadd_u8(a); } +SIMD_INLINE int64_t v64_hadd_s16(v64 a) { return c_v64_hadd_s16(a); } + +SIMD_INLINE v64 v64_or(v64 a, v64 b) { return c_v64_or(a, b); } +SIMD_INLINE v64 v64_xor(v64 a, v64 b) { return c_v64_xor(a, b); } +SIMD_INLINE v64 v64_and(v64 a, v64 b) { return c_v64_and(a, b); } +SIMD_INLINE v64 v64_andn(v64 a, v64 b) { return c_v64_andn(a, b); } + +SIMD_INLINE v64 v64_mullo_s16(v64 a, v64 b) { return c_v64_mullo_s16(a, b); } +SIMD_INLINE v64 v64_mulhi_s16(v64 a, v64 b) { return c_v64_mulhi_s16(a, b); } +SIMD_INLINE v64 v64_mullo_s32(v64 a, v64 b) { return c_v64_mullo_s32(a, b); } +SIMD_INLINE v64 v64_madd_s16(v64 a, v64 b) { return c_v64_madd_s16(a, b); } +SIMD_INLINE v64 v64_madd_us8(v64 a, v64 b) { return c_v64_madd_us8(a, b); } + +SIMD_INLINE v64 v64_avg_u8(v64 a, v64 b) { return c_v64_avg_u8(a, b); } +SIMD_INLINE v64 v64_rdavg_u8(v64 a, v64 b) { return c_v64_rdavg_u8(a, b); } +SIMD_INLINE v64 v64_avg_u16(v64 a, v64 b) { return c_v64_avg_u16(a, b); } +SIMD_INLINE v64 v64_min_u8(v64 a, v64 b) { return c_v64_min_u8(a, b); } +SIMD_INLINE v64 v64_max_u8(v64 a, v64 b) { return c_v64_max_u8(a, b); } +SIMD_INLINE v64 v64_min_s8(v64 a, v64 b) { return c_v64_min_s8(a, b); } +SIMD_INLINE v64 v64_max_s8(v64 a, v64 b) { return c_v64_max_s8(a, b); } +SIMD_INLINE v64 v64_min_s16(v64 a, v64 b) { return c_v64_min_s16(a, b); } +SIMD_INLINE v64 v64_max_s16(v64 a, v64 b) { return c_v64_max_s16(a, b); } + +SIMD_INLINE v64 v64_cmpgt_s8(v64 a, v64 b) { return c_v64_cmpgt_s8(a, b); } +SIMD_INLINE v64 v64_cmplt_s8(v64 a, v64 b) { return c_v64_cmplt_s8(a, b); } +SIMD_INLINE v64 v64_cmpeq_8(v64 a, v64 b) { return c_v64_cmpeq_8(a, b); } +SIMD_INLINE v64 v64_cmpgt_s16(v64 a, v64 b) { return c_v64_cmpgt_s16(a, b); } +SIMD_INLINE v64 v64_cmplt_s16(v64 a, v64 b) { return c_v64_cmplt_s16(a, b); } +SIMD_INLINE v64 v64_cmpeq_16(v64 a, v64 b) { return c_v64_cmpeq_16(a, b); } + +SIMD_INLINE v64 v64_shl_8(v64 a, unsigned int n) { return c_v64_shl_8(a, n); } +SIMD_INLINE v64 v64_shr_u8(v64 a, unsigned int n) { return c_v64_shr_u8(a, n); } +SIMD_INLINE v64 v64_shr_s8(v64 a, unsigned int n) { return c_v64_shr_s8(a, n); } +SIMD_INLINE v64 v64_shl_16(v64 a, unsigned int n) { return c_v64_shl_16(a, n); } +SIMD_INLINE v64 v64_shr_u16(v64 a, unsigned int n) { + return c_v64_shr_u16(a, n); +} +SIMD_INLINE v64 v64_shr_s16(v64 a, unsigned int n) { + return c_v64_shr_s16(a, n); +} +SIMD_INLINE v64 v64_shl_32(v64 a, unsigned int n) { return c_v64_shl_32(a, n); } +SIMD_INLINE v64 v64_shr_u32(v64 a, unsigned int n) { + return c_v64_shr_u32(a, n); +} +SIMD_INLINE v64 v64_shr_s32(v64 a, unsigned int n) { + return c_v64_shr_s32(a, n); +} +SIMD_INLINE v64 v64_shr_n_byte(v64 a, unsigned int n) { + return c_v64_shr_n_byte(a, n); +} +SIMD_INLINE v64 v64_shl_n_byte(v64 a, unsigned int n) { + return c_v64_shl_n_byte(a, n); +} +SIMD_INLINE v64 v64_shl_n_8(v64 a, unsigned int c) { + return c_v64_shl_n_8(a, c); +} +SIMD_INLINE v64 v64_shr_n_u8(v64 a, unsigned int c) { + return c_v64_shr_n_u8(a, c); +} +SIMD_INLINE v64 v64_shr_n_s8(v64 a, unsigned int c) { + return c_v64_shr_n_s8(a, c); +} +SIMD_INLINE v64 v64_shl_n_16(v64 a, unsigned int c) { + return c_v64_shl_n_16(a, c); +} +SIMD_INLINE v64 v64_shr_n_u16(v64 a, unsigned int c) { + return c_v64_shr_n_u16(a, c); +} +SIMD_INLINE v64 v64_shr_n_s16(v64 a, unsigned int c) { + return c_v64_shr_n_s16(a, c); +} +SIMD_INLINE v64 v64_shl_n_32(v64 a, unsigned int c) { + return c_v64_shl_n_32(a, c); +} +SIMD_INLINE v64 v64_shr_n_u32(v64 a, unsigned int c) { + return c_v64_shr_n_u32(a, c); +} +SIMD_INLINE v64 v64_shr_n_s32(v64 a, unsigned int c) { + return c_v64_shr_n_s32(a, c); +} + +#endif /* _V64_INTRINSICS_H */ diff --git a/third_party/aom/aom_dsp/simd/v64_intrinsics_arm.h b/third_party/aom/aom_dsp/simd/v64_intrinsics_arm.h new file mode 100644 index 0000000000..c7574eef50 --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v64_intrinsics_arm.h @@ -0,0 +1,583 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V64_INTRINSICS_H +#define _V64_INTRINSICS_H + +#include +#include "./v64_intrinsics_arm.h" +#include "aom_ports/arm.h" + +#ifdef AOM_INCOMPATIBLE_GCC +#error Incompatible gcc +#endif + +typedef int64x1_t v64; + +SIMD_INLINE uint32_t v64_low_u32(v64 a) { + return vget_lane_u32(vreinterpret_u32_s64(a), 0); +} + +SIMD_INLINE uint32_t v64_high_u32(v64 a) { + return vget_lane_u32(vreinterpret_u32_s64(a), 1); +} + +SIMD_INLINE int32_t v64_low_s32(v64 a) { + return vget_lane_s32(vreinterpret_s32_s64(a), 0); +} + +SIMD_INLINE int32_t v64_high_s32(v64 a) { + return vget_lane_s32(vreinterpret_s32_s64(a), 1); +} + +SIMD_INLINE v64 v64_from_16(uint16_t a, uint16_t b, uint16_t c, uint16_t d) { + return vcreate_s64((uint64_t)a << 48 | (uint64_t)b << 32 | (uint64_t)c << 16 | + d); +} + +SIMD_INLINE v64 v64_from_32(uint32_t x, uint32_t y) { + return vcreate_s64((uint64_t)x << 32 | y); +} + +SIMD_INLINE v64 v64_from_64(uint64_t x) { return vcreate_s64(x); } + +SIMD_INLINE uint64_t v64_u64(v64 x) { return (uint64_t)x; } + +SIMD_INLINE uint32_t u32_load_aligned(const void *p) { + return *((uint32_t *)p); +} + +SIMD_INLINE uint32_t u32_load_unaligned(const void *p) { + return vget_lane_u32(vreinterpret_u32_u8(vld1_u8((const uint8_t *)p)), 0); +} + +SIMD_INLINE void u32_store_aligned(void *p, uint32_t a) { + *((uint32_t *)p) = a; +} + +SIMD_INLINE void u32_store_unaligned(void *p, uint32_t a) { +#if defined(__clang__) + vst1_lane_u32((uint32_t *)p, vreinterpret_u32_s64((uint64x1_t)(uint64_t)a), + 0); +#elif defined(__CC_ARM) + *(__packed uint32_t *)p) = a; +#elif defined(__GNUC__) + *((__attribute((packed)) uint32_t *)p) = a; +#else + vst1_lane_u32((uint32_t *)p, vreinterpret_u32_s64((uint64x1_t)(uint64_t)a), + 0); +#endif +} + +SIMD_INLINE v64 v64_load_aligned(const void *p) { + return vreinterpret_s64_u8(vld1_u8((const uint8_t *)p)); +} + +SIMD_INLINE v64 v64_load_unaligned(const void *p) { + return v64_load_aligned(p); +} + +SIMD_INLINE void v64_store_aligned(void *p, v64 r) { + vst1_u8((uint8_t *)p, vreinterpret_u8_s64(r)); +} + +SIMD_INLINE void v64_store_unaligned(void *p, v64 r) { + vst1_u8((uint8_t *)p, vreinterpret_u8_s64(r)); +} + +// The following function requires an immediate. +// Some compilers will check this if it's optimising, others wont. +SIMD_INLINE v64 v64_align(v64 a, v64 b, unsigned int c) { +#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__) + return c ? vreinterpret_s64_s8( + vext_s8(vreinterpret_s8_s64(b), vreinterpret_s8_s64(a), c)) + : b; +#else + return c ? v64_from_64(((uint64_t)b >> c * 8) | ((uint64_t)a << (8 - c) * 8)) + : b; +#endif +} + +SIMD_INLINE v64 v64_zero() { return vreinterpret_s64_u8(vdup_n_u8(0)); } + +SIMD_INLINE v64 v64_dup_8(uint8_t x) { + return vreinterpret_s64_u8(vdup_n_u8(x)); +} + +SIMD_INLINE v64 v64_dup_16(uint16_t x) { + return vreinterpret_s64_u16(vdup_n_u16(x)); +} + +SIMD_INLINE v64 v64_dup_32(uint32_t x) { + return vreinterpret_s64_u32(vdup_n_u32(x)); +} + +SIMD_INLINE int64_t v64_dotp_su8(v64 x, v64 y) { + int64x2_t r = vpaddlq_s32(vpaddlq_s16( + vmulq_s16(vmovl_s8(vreinterpret_s8_s64(x)), + vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(y)))))); + return (int64_t)vadd_s64(vget_high_s64(r), vget_low_s64(r)); +} + +SIMD_INLINE int64_t v64_dotp_s16(v64 x, v64 y) { + int64x2_t r = + vpaddlq_s32(vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); + return (int64_t)(vget_high_s64(r) + vget_low_s64(r)); +} + +SIMD_INLINE uint64_t v64_hadd_u8(v64 x) { + return (uint64_t)vpaddl_u32(vpaddl_u16(vpaddl_u8(vreinterpret_u8_s64(x)))); +} + +SIMD_INLINE int64_t v64_hadd_s16(v64 a) { + return (int64_t)vpaddl_s32(vpaddl_s16(vreinterpret_s16_s64(a))); +} + +typedef uint16x8_t sad64_internal; + +SIMD_INLINE sad64_internal v64_sad_u8_init() { return vdupq_n_u16(0); } + +/* Implementation dependent return value. Result must be finalised with + v64_sad_u8_sum(). + The result for more than 32 v64_sad_u8() calls is undefined. */ +SIMD_INLINE sad64_internal v64_sad_u8(sad64_internal s, v64 a, v64 b) { + return vabal_u8(s, vreinterpret_u8_s64(a), vreinterpret_u8_s64(b)); +} + +SIMD_INLINE uint32_t v64_sad_u8_sum(sad64_internal s) { + uint64x2_t r = vpaddlq_u32(vpaddlq_u16(s)); + return (uint32_t)(uint64_t)(vget_high_u64(r) + vget_low_u64(r)); +} + +typedef int64x1_t ssd64_internal; + +SIMD_INLINE ssd64_internal v64_ssd_u8_init() { + return (ssd64_internal)(uint64_t)0; +} + +/* Implementation dependent return value. Result must be finalised with + * v64_ssd_u8_sum(). */ +SIMD_INLINE ssd64_internal v64_ssd_u8(ssd64_internal s, v64 a, v64 b) { + uint8x8_t t = vabd_u8(vreinterpret_u8_s64(a), vreinterpret_u8_s64(b)); + uint64x2_t r = vpaddlq_u32(vpaddlq_u16(vmull_u8(t, t))); + return vadd_u64(s, vadd_u64(vget_high_u64(r), vget_low_u64(r))); +} + +SIMD_INLINE uint32_t v64_ssd_u8_sum(ssd64_internal s) { + return (uint32_t)(uint64_t)s; +} + +SIMD_INLINE v64 v64_or(v64 x, v64 y) { return vorr_s64(x, y); } + +SIMD_INLINE v64 v64_xor(v64 x, v64 y) { return veor_s64(x, y); } + +SIMD_INLINE v64 v64_and(v64 x, v64 y) { return vand_s64(x, y); } + +SIMD_INLINE v64 v64_andn(v64 x, v64 y) { return vbic_s64(x, y); } + +SIMD_INLINE v64 v64_add_8(v64 x, v64 y) { + return vreinterpret_s64_u8( + vadd_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); +} + +SIMD_INLINE v64 v64_add_16(v64 x, v64 y) { + return vreinterpret_s64_s16( + vadd_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); +} + +SIMD_INLINE v64 v64_sadd_s16(v64 x, v64 y) { + return vreinterpret_s64_s16( + vqadd_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); +} + +SIMD_INLINE v64 v64_add_32(v64 x, v64 y) { + return vreinterpret_s64_u32( + vadd_u32(vreinterpret_u32_s64(x), vreinterpret_u32_s64(y))); +} + +SIMD_INLINE v64 v64_sub_8(v64 x, v64 y) { + return vreinterpret_s64_u8( + vsub_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); +} + +SIMD_INLINE v64 v64_sub_16(v64 x, v64 y) { + return vreinterpret_s64_s16( + vsub_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); +} + +SIMD_INLINE v64 v64_ssub_s16(v64 x, v64 y) { + return vreinterpret_s64_s16( + vqsub_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); +} + +SIMD_INLINE v64 v64_ssub_u16(v64 x, v64 y) { + return vreinterpret_s64_u16( + vqsub_u16(vreinterpret_u16_s64(x), vreinterpret_u16_s64(y))); +} + +SIMD_INLINE v64 v64_ssub_u8(v64 x, v64 y) { + return vreinterpret_s64_u8( + vqsub_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); +} + +SIMD_INLINE v64 v64_ssub_s8(v64 x, v64 y) { + return vreinterpret_s64_s8( + vqsub_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y))); +} + +SIMD_INLINE v64 v64_sub_32(v64 x, v64 y) { + return vreinterpret_s64_s32( + vsub_s32(vreinterpret_s32_s64(x), vreinterpret_s32_s64(y))); +} + +SIMD_INLINE v64 v64_abs_s16(v64 x) { + return vreinterpret_s64_s16(vabs_s16(vreinterpret_s16_s64(x))); +} + +SIMD_INLINE v64 v64_abs_s8(v64 x) { + return vreinterpret_s64_s8(vabs_s8(vreinterpret_s8_s64(x))); +} + +SIMD_INLINE v64 v64_mullo_s16(v64 x, v64 y) { + return vreinterpret_s64_s16( + vmul_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); +} + +SIMD_INLINE v64 v64_mulhi_s16(v64 x, v64 y) { + return vreinterpret_s64_s16(vmovn_s32(vshrq_n_s32( + vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)), 16))); +} + +SIMD_INLINE v64 v64_mullo_s32(v64 x, v64 y) { + return vreinterpret_s64_s32( + vmul_s32(vreinterpret_s32_s64(x), vreinterpret_s32_s64(y))); +} + +SIMD_INLINE v64 v64_madd_s16(v64 x, v64 y) { + int32x4_t t = vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)); + return vreinterpret_s64_s32( + vpadd_s32(vreinterpret_s32_s64(vget_low_s64(vreinterpretq_s64_s32(t))), + vreinterpret_s32_s64(vget_high_s64(vreinterpretq_s64_s32(t))))); +} + +SIMD_INLINE v64 v64_madd_us8(v64 x, v64 y) { + return vreinterpret_s64_s16(vqmovn_s32(vpaddlq_s16( + vaddq_s16(vmull_s8(vadd_s8(vreinterpret_s8_s64(x), vdup_n_s8(-128)), + vreinterpret_s8_s64(y)), + vshlq_n_s16(vmovl_s8(vreinterpret_s8_s64(y)), 7))))); +} + +SIMD_INLINE v64 v64_avg_u8(v64 x, v64 y) { + return vreinterpret_s64_u8( + vrhadd_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); +} + +SIMD_INLINE v64 v64_rdavg_u8(v64 x, v64 y) { + return vreinterpret_s64_u8( + vhadd_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); +} + +SIMD_INLINE v64 v64_avg_u16(v64 x, v64 y) { + return vreinterpret_s64_u16( + vrhadd_u16(vreinterpret_u16_s64(x), vreinterpret_u16_s64(y))); +} + +SIMD_INLINE v64 v64_max_u8(v64 x, v64 y) { + return vreinterpret_s64_u8( + vmax_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); +} + +SIMD_INLINE v64 v64_min_u8(v64 x, v64 y) { + return vreinterpret_s64_u8( + vmin_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); +} + +SIMD_INLINE v64 v64_max_s8(v64 x, v64 y) { + return vreinterpret_s64_s8( + vmax_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y))); +} + +SIMD_INLINE v64 v64_min_s8(v64 x, v64 y) { + return vreinterpret_s64_s8( + vmin_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y))); +} + +SIMD_INLINE v64 v64_max_s16(v64 x, v64 y) { + return vreinterpret_s64_s16( + vmax_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); +} + +SIMD_INLINE v64 v64_min_s16(v64 x, v64 y) { + return vreinterpret_s64_s16( + vmin_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); +} + +SIMD_INLINE v64 v64_ziplo_8(v64 x, v64 y) { + uint8x8x2_t r = vzip_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)); + return vreinterpret_s64_u8(r.val[0]); +} + +SIMD_INLINE v64 v64_ziphi_8(v64 x, v64 y) { + uint8x8x2_t r = vzip_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)); + return vreinterpret_s64_u8(r.val[1]); +} + +SIMD_INLINE v64 v64_ziplo_16(v64 x, v64 y) { + int16x4x2_t r = vzip_s16(vreinterpret_s16_s64(y), vreinterpret_s16_s64(x)); + return vreinterpret_s64_s16(r.val[0]); +} + +SIMD_INLINE v64 v64_ziphi_16(v64 x, v64 y) { + int16x4x2_t r = vzip_s16(vreinterpret_s16_s64(y), vreinterpret_s16_s64(x)); + return vreinterpret_s64_s16(r.val[1]); +} + +SIMD_INLINE v64 v64_ziplo_32(v64 x, v64 y) { + int32x2x2_t r = vzip_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x)); + return vreinterpret_s64_s32(r.val[0]); +} + +SIMD_INLINE v64 v64_ziphi_32(v64 x, v64 y) { + int32x2x2_t r = vzip_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x)); + return vreinterpret_s64_s32(r.val[1]); +} + +SIMD_INLINE v64 v64_unpacklo_u8_s16(v64 a) { + return vreinterpret_s64_u16(vget_low_u16(vmovl_u8(vreinterpret_u8_s64(a)))); +} + +SIMD_INLINE v64 v64_unpackhi_u8_s16(v64 a) { + return vreinterpret_s64_u16(vget_high_u16(vmovl_u8(vreinterpret_u8_s64(a)))); +} + +SIMD_INLINE v64 v64_unpacklo_s8_s16(v64 a) { + return vreinterpret_s64_s16(vget_low_s16(vmovl_s8(vreinterpret_s8_s64(a)))); +} + +SIMD_INLINE v64 v64_unpackhi_s8_s16(v64 a) { + return vreinterpret_s64_s16(vget_high_s16(vmovl_s8(vreinterpret_s8_s64(a)))); +} + +SIMD_INLINE v64 v64_pack_s32_s16(v64 x, v64 y) { + return vreinterpret_s64_s16(vqmovn_s32( + vcombine_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x)))); +} + +SIMD_INLINE v64 v64_pack_s16_u8(v64 x, v64 y) { + return vreinterpret_s64_u8(vqmovun_s16(vreinterpretq_s16_s32( + vcombine_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x))))); +} + +SIMD_INLINE v64 v64_pack_s16_s8(v64 x, v64 y) { + return vreinterpret_s64_s8(vqmovn_s16(vreinterpretq_s16_s32( + vcombine_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x))))); +} + +SIMD_INLINE v64 v64_unziplo_8(v64 x, v64 y) { + uint8x8x2_t r = vuzp_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)); + return vreinterpret_s64_u8(r.val[0]); +} + +SIMD_INLINE v64 v64_unziphi_8(v64 x, v64 y) { + uint8x8x2_t r = vuzp_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)); + return vreinterpret_s64_u8(r.val[1]); +} + +SIMD_INLINE v64 v64_unziplo_16(v64 x, v64 y) { + uint16x4x2_t r = vuzp_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x)); + return vreinterpret_s64_u16(r.val[0]); +} + +SIMD_INLINE v64 v64_unziphi_16(v64 x, v64 y) { + uint16x4x2_t r = vuzp_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x)); + return vreinterpret_s64_u16(r.val[1]); +} + +SIMD_INLINE v64 v64_unpacklo_s16_s32(v64 x) { + return vreinterpret_s64_s32(vget_low_s32(vmovl_s16(vreinterpret_s16_s64(x)))); +} + +SIMD_INLINE v64 v64_unpacklo_u16_s32(v64 x) { + return vreinterpret_s64_u32(vget_low_u32(vmovl_u16(vreinterpret_u16_s64(x)))); +} + +SIMD_INLINE v64 v64_unpackhi_s16_s32(v64 x) { + return vreinterpret_s64_s32( + vget_high_s32(vmovl_s16(vreinterpret_s16_s64(x)))); +} + +SIMD_INLINE v64 v64_unpackhi_u16_s32(v64 x) { + return vreinterpret_s64_u32( + vget_high_u32(vmovl_u16(vreinterpret_u16_s64(x)))); +} + +SIMD_INLINE v64 v64_shuffle_8(v64 x, v64 pattern) { + return vreinterpret_s64_u8( + vtbl1_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(pattern))); +} + +SIMD_INLINE v64 v64_cmpgt_s8(v64 x, v64 y) { + return vreinterpret_s64_u8( + vcgt_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y))); +} + +SIMD_INLINE v64 v64_cmplt_s8(v64 x, v64 y) { + return vreinterpret_s64_u8( + vclt_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y))); +} + +SIMD_INLINE v64 v64_cmpeq_8(v64 x, v64 y) { + return vreinterpret_s64_u8( + vceq_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); +} + +SIMD_INLINE v64 v64_cmpgt_s16(v64 x, v64 y) { + return vreinterpret_s64_u16( + vcgt_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); +} + +SIMD_INLINE v64 v64_cmplt_s16(v64 x, v64 y) { + return vreinterpret_s64_u16( + vclt_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); +} + +SIMD_INLINE v64 v64_cmpeq_16(v64 x, v64 y) { + return vreinterpret_s64_u16( + vceq_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); +} + +SIMD_INLINE v64 v64_shl_8(v64 a, unsigned int c) { + return vreinterpret_s64_u8(vshl_u8(vreinterpret_u8_s64(a), vdup_n_s8(c))); +} + +SIMD_INLINE v64 v64_shr_u8(v64 a, unsigned int c) { + return vreinterpret_s64_u8(vshl_u8(vreinterpret_u8_s64(a), vdup_n_s8(-c))); +} + +SIMD_INLINE v64 v64_shr_s8(v64 a, unsigned int c) { + return vreinterpret_s64_s8(vshl_s8(vreinterpret_s8_s64(a), vdup_n_s8(-c))); +} + +SIMD_INLINE v64 v64_shl_16(v64 a, unsigned int c) { + return vreinterpret_s64_u16(vshl_u16(vreinterpret_u16_s64(a), vdup_n_s16(c))); +} + +SIMD_INLINE v64 v64_shr_u16(v64 a, unsigned int c) { + return vreinterpret_s64_u16( + vshl_u16(vreinterpret_u16_s64(a), vdup_n_s16(-(int)c))); +} + +SIMD_INLINE v64 v64_shr_s16(v64 a, unsigned int c) { + return vreinterpret_s64_s16( + vshl_s16(vreinterpret_s16_s64(a), vdup_n_s16(-(int)c))); +} + +SIMD_INLINE v64 v64_shl_32(v64 a, unsigned int c) { + return vreinterpret_s64_u32(vshl_u32(vreinterpret_u32_s64(a), vdup_n_s32(c))); +} + +SIMD_INLINE v64 v64_shr_u32(v64 a, unsigned int c) { + return vreinterpret_s64_u32( + vshl_u32(vreinterpret_u32_s64(a), vdup_n_s32(-(int)c))); +} + +SIMD_INLINE v64 v64_shr_s32(v64 a, unsigned int c) { + return vreinterpret_s64_s32( + vshl_s32(vreinterpret_s32_s64(a), vdup_n_s32(-(int)c))); +} + +// The following functions require an immediate. +// Some compilers will check this during optimisation, others wont. +#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__) + +SIMD_INLINE v64 v64_shl_n_byte(v64 a, unsigned int c) { + return vshl_n_s64(a, c * 8); +} + +SIMD_INLINE v64 v64_shr_n_byte(v64 a, unsigned int c) { + return c ? (v64)vshr_n_u64(vreinterpret_u64_s64(a), c * 8) : a; +} + +SIMD_INLINE v64 v64_shl_n_8(v64 a, unsigned int c) { + return vreinterpret_s64_u8(vshl_n_u8(vreinterpret_u8_s64(a), c)); +} + +SIMD_INLINE v64 v64_shr_n_u8(v64 a, unsigned int c) { + return vreinterpret_s64_u8(vshr_n_u8(vreinterpret_u8_s64(a), c)); +} + +SIMD_INLINE v64 v64_shr_n_s8(v64 a, unsigned int c) { + return vreinterpret_s64_s8(vshr_n_s8(vreinterpret_s8_s64(a), c)); +} + +SIMD_INLINE v64 v64_shl_n_16(v64 a, unsigned int c) { + return vreinterpret_s64_u16(vshl_n_u16(vreinterpret_u16_s64(a), c)); +} + +SIMD_INLINE v64 v64_shr_n_u16(v64 a, unsigned int c) { + return vreinterpret_s64_u16(vshr_n_u16(vreinterpret_u16_s64(a), c)); +} + +SIMD_INLINE v64 v64_shr_n_s16(v64 a, unsigned int c) { + return vreinterpret_s64_s16(vshr_n_s16(vreinterpret_s16_s64(a), c)); +} + +SIMD_INLINE v64 v64_shl_n_32(v64 a, unsigned int c) { + return vreinterpret_s64_u32(vshl_n_u32(vreinterpret_u32_s64(a), c)); +} + +SIMD_INLINE v64 v64_shr_n_u32(v64 a, unsigned int c) { + return vreinterpret_s64_u32(vshr_n_u32(vreinterpret_u32_s64(a), c)); +} + +SIMD_INLINE v64 v64_shr_n_s32(v64 a, unsigned int c) { + return vreinterpret_s64_s32(vshr_n_s32(vreinterpret_s32_s64(a), c)); +} + +#else + +SIMD_INLINE v64 v64_shl_n_byte(v64 a, unsigned int c) { + return v64_from_64(v64_u64(a) << c * 8); +} + +SIMD_INLINE v64 v64_shr_n_byte(v64 a, unsigned int c) { + return v64_from_64(v64_u64(a) >> c * 8); +} + +SIMD_INLINE v64 v64_shl_n_8(v64 a, unsigned int c) { return v64_shl_8(a, c); } + +SIMD_INLINE v64 v64_shr_n_u8(v64 a, unsigned int c) { return v64_shr_u8(a, c); } + +SIMD_INLINE v64 v64_shr_n_s8(v64 a, unsigned int c) { return v64_shr_s8(a, c); } + +SIMD_INLINE v64 v64_shl_n_16(v64 a, unsigned int c) { return v64_shl_16(a, c); } + +SIMD_INLINE v64 v64_shr_n_u16(v64 a, unsigned int c) { + return v64_shr_u16(a, c); +} + +SIMD_INLINE v64 v64_shr_n_s16(v64 a, unsigned int c) { + return v64_shr_s16(a, c); +} + +SIMD_INLINE v64 v64_shl_n_32(v64 a, unsigned int c) { return v64_shl_32(a, c); } + +SIMD_INLINE v64 v64_shr_n_u32(v64 a, unsigned int c) { + return v64_shr_u32(a, c); +} + +SIMD_INLINE v64 v64_shr_n_s32(v64 a, unsigned int c) { + return v64_shr_s32(a, c); +} + +#endif + +#endif /* _V64_INTRINSICS_H */ diff --git a/third_party/aom/aom_dsp/simd/v64_intrinsics_c.h b/third_party/aom/aom_dsp/simd/v64_intrinsics_c.h new file mode 100644 index 0000000000..5032238b66 --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v64_intrinsics_c.h @@ -0,0 +1,919 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V64_INTRINSICS_C_H +#define _V64_INTRINSICS_C_H + +/* Note: This implements the intrinsics in plain, unoptimised C. + Intended for reference, porting or debugging. */ + +#include +#include +#include "./aom_config.h" + +typedef union { + uint8_t u8[8]; + uint16_t u16[4]; + uint32_t u32[2]; + uint64_t u64; + int8_t s8[8]; + int16_t s16[4]; + int32_t s32[2]; + int64_t s64; +} c_v64; + +SIMD_INLINE uint32_t c_v64_low_u32(c_v64 a) { return a.u32[CONFIG_BIG_ENDIAN]; } + +SIMD_INLINE uint32_t c_v64_high_u32(c_v64 a) { + return a.u32[!CONFIG_BIG_ENDIAN]; +} + +SIMD_INLINE int32_t c_v64_low_s32(c_v64 a) { return a.s32[CONFIG_BIG_ENDIAN]; } + +SIMD_INLINE int32_t c_v64_high_s32(c_v64 a) { + return a.s32[!CONFIG_BIG_ENDIAN]; +} + +SIMD_INLINE c_v64 c_v64_from_32(uint32_t x, uint32_t y) { + c_v64 t; + t.u32[!CONFIG_BIG_ENDIAN] = x; + t.u32[CONFIG_BIG_ENDIAN] = y; + return t; +} + +SIMD_INLINE c_v64 c_v64_from_64(uint64_t x) { + c_v64 t; + t.u64 = x; + return t; +} + +SIMD_INLINE uint64_t c_v64_u64(c_v64 x) { return x.u64; } + +SIMD_INLINE c_v64 c_v64_from_16(uint16_t a, uint16_t b, uint16_t c, + uint16_t d) { + c_v64 t; + if (CONFIG_BIG_ENDIAN) { + t.u16[0] = a; + t.u16[1] = b; + t.u16[2] = c; + t.u16[3] = d; + } else { + t.u16[3] = a; + t.u16[2] = b; + t.u16[1] = c; + t.u16[0] = d; + } + return t; +} + +SIMD_INLINE uint32_t c_u32_load_unaligned(const void *p) { + uint32_t t; + uint8_t *pp = (uint8_t *)p; + uint8_t *q = (uint8_t *)&t; + int c; + for (c = 0; c < 4; c++) q[c] = pp[c]; + return t; +} + +SIMD_INLINE void c_u32_store_unaligned(void *p, uint32_t a) { + uint8_t *pp = (uint8_t *)p; + uint8_t *q = (uint8_t *)&a; + int c; + for (c = 0; c < 4; c++) pp[c] = q[c]; +} + +SIMD_INLINE uint32_t c_u32_load_aligned(const void *p) { + if (SIMD_CHECK && (uintptr_t)p & 3) { + fprintf(stderr, "Error: Unaligned u32 load at %p\n", p); + abort(); + } + return c_u32_load_unaligned(p); +} + +SIMD_INLINE void c_u32_store_aligned(void *p, uint32_t a) { + if (SIMD_CHECK && (uintptr_t)p & 3) { + fprintf(stderr, "Error: Unaligned u32 store at %p\n", p); + abort(); + } + c_u32_store_unaligned(p, a); +} + +SIMD_INLINE c_v64 c_v64_load_unaligned(const void *p) { + c_v64 t; + uint8_t *pp = (uint8_t *)p; + uint8_t *q = (uint8_t *)&t; + int c; + for (c = 0; c < 8; c++) q[c] = pp[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_load_aligned(const void *p) { + if (SIMD_CHECK && (uintptr_t)p & 7) { + fprintf(stderr, "Error: Unaligned c_v64 load at %p\n", p); + abort(); + } + return c_v64_load_unaligned(p); +} + +SIMD_INLINE void c_v64_store_unaligned(void *p, c_v64 a) { + uint8_t *q = (uint8_t *)p; + uint8_t *r = (uint8_t *)&a; + int c; + for (c = 0; c < 8; c++) q[c] = r[c]; +} + +SIMD_INLINE void c_v64_store_aligned(void *p, c_v64 a) { + if (SIMD_CHECK && (uintptr_t)p & 7) { + fprintf(stderr, "Error: Unaligned c_v64 store at %p\n", p); + abort(); + } + c_v64_store_unaligned(p, a); +} + +SIMD_INLINE c_v64 c_v64_zero() { + c_v64 t; + t.u64 = 0; + return t; +} + +SIMD_INLINE c_v64 c_v64_dup_8(uint8_t x) { + c_v64 t; + t.u8[0] = t.u8[1] = t.u8[2] = t.u8[3] = t.u8[4] = t.u8[5] = t.u8[6] = + t.u8[7] = x; + return t; +} + +SIMD_INLINE c_v64 c_v64_dup_16(uint16_t x) { + c_v64 t; + t.u16[0] = t.u16[1] = t.u16[2] = t.u16[3] = x; + return t; +} + +SIMD_INLINE c_v64 c_v64_dup_32(uint32_t x) { + c_v64 t; + t.u32[0] = t.u32[1] = x; + return t; +} + +SIMD_INLINE c_v64 c_v64_add_8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) t.u8[c] = a.u8[c] + b.u8[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_add_16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) t.u16[c] = a.u16[c] + b.u16[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_sadd_s16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) + t.s16[c] = (int32_t)a.s16[c] + (int32_t)b.s16[c] > 32767 + ? 32767 + : (int32_t)a.s16[c] + (int32_t)b.s16[c] < -32768 + ? -32768 + : (int32_t)a.s16[c] + (int32_t)b.s16[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_add_32(c_v64 a, c_v64 b) { + c_v64 t; + t.u32[0] = (uint32_t)((uint64_t)a.u32[0] + b.u32[0]); + t.u32[1] = (uint32_t)((uint64_t)a.u32[1] + b.u32[1]); + return t; +} + +SIMD_INLINE c_v64 c_v64_sub_8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) t.u8[c] = a.u8[c] - b.u8[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_ssub_u8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) + t.u8[c] = (int32_t)a.u8[c] - (int32_t)b.u8[c] < 0 ? 0 : a.u8[c] - b.u8[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_ssub_s8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) { + int16_t d = (int16_t)a.s8[c] - (int16_t)b.s8[c]; + t.s8[c] = d > 127 ? 127 : (d < -128 ? -128 : d); + } + return t; +} + +SIMD_INLINE c_v64 c_v64_sub_16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) t.u16[c] = a.u16[c] - b.u16[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_ssub_s16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) + t.s16[c] = (int32_t)a.s16[c] - (int32_t)b.s16[c] < -32768 + ? -32768 + : (int32_t)a.s16[c] - (int32_t)b.s16[c] > 32767 + ? 32767 + : (int32_t)a.s16[c] - (int32_t)b.s16[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_ssub_u16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) + t.u16[c] = + (int32_t)a.u16[c] - (int32_t)b.u16[c] < 0 ? 0 : a.u16[c] - b.u16[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_sub_32(c_v64 a, c_v64 b) { + c_v64 t; + t.u32[0] = (uint32_t)((int64_t)a.u32[0] - b.u32[0]); + t.u32[1] = (uint32_t)((int64_t)a.u32[1] - b.u32[1]); + return t; +} + +SIMD_INLINE c_v64 c_v64_abs_s16(c_v64 a) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) + t.u16[c] = (int16_t)a.u16[c] > 0 ? a.u16[c] : -a.u16[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_abs_s8(c_v64 a) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) t.u8[c] = (int8_t)a.u8[c] > 0 ? a.u8[c] : -a.u8[c]; + return t; +} + +SIMD_INLINE c_v64 _c_v64_zip_8(c_v64 a, c_v64 b, int mode) { + c_v64 t; + if (mode) { + t.u8[7] = a.u8[7]; + t.u8[6] = b.u8[7]; + t.u8[5] = a.u8[6]; + t.u8[4] = b.u8[6]; + t.u8[3] = a.u8[5]; + t.u8[2] = b.u8[5]; + t.u8[1] = a.u8[4]; + t.u8[0] = b.u8[4]; + } else { + t.u8[7] = a.u8[3]; + t.u8[6] = b.u8[3]; + t.u8[5] = a.u8[2]; + t.u8[4] = b.u8[2]; + t.u8[3] = a.u8[1]; + t.u8[2] = b.u8[1]; + t.u8[1] = a.u8[0]; + t.u8[0] = b.u8[0]; + } + return t; +} + +SIMD_INLINE c_v64 c_v64_ziplo_8(c_v64 a, c_v64 b) { + return CONFIG_BIG_ENDIAN ? _c_v64_zip_8(b, a, 1) : _c_v64_zip_8(a, b, 0); +} + +SIMD_INLINE c_v64 c_v64_ziphi_8(c_v64 a, c_v64 b) { + return CONFIG_BIG_ENDIAN ? _c_v64_zip_8(b, a, 0) : _c_v64_zip_8(a, b, 1); +} + +SIMD_INLINE c_v64 _c_v64_zip_16(c_v64 a, c_v64 b, int mode) { + c_v64 t; + if (mode) { + t.u16[3] = a.u16[3]; + t.u16[2] = b.u16[3]; + t.u16[1] = a.u16[2]; + t.u16[0] = b.u16[2]; + } else { + t.u16[3] = a.u16[1]; + t.u16[2] = b.u16[1]; + t.u16[1] = a.u16[0]; + t.u16[0] = b.u16[0]; + } + return t; +} + +SIMD_INLINE c_v64 c_v64_ziplo_16(c_v64 a, c_v64 b) { + return CONFIG_BIG_ENDIAN ? _c_v64_zip_16(b, a, 1) : _c_v64_zip_16(a, b, 0); +} + +SIMD_INLINE c_v64 c_v64_ziphi_16(c_v64 a, c_v64 b) { + return CONFIG_BIG_ENDIAN ? _c_v64_zip_16(b, a, 0) : _c_v64_zip_16(a, b, 1); +} + +SIMD_INLINE c_v64 _c_v64_zip_32(c_v64 a, c_v64 b, int mode) { + c_v64 t; + if (mode) { + t.u32[1] = a.u32[1]; + t.u32[0] = b.u32[1]; + } else { + t.u32[1] = a.u32[0]; + t.u32[0] = b.u32[0]; + } + return t; +} + +SIMD_INLINE c_v64 c_v64_ziplo_32(c_v64 a, c_v64 b) { + return CONFIG_BIG_ENDIAN ? _c_v64_zip_32(b, a, 1) : _c_v64_zip_32(a, b, 0); +} + +SIMD_INLINE c_v64 c_v64_ziphi_32(c_v64 a, c_v64 b) { + return CONFIG_BIG_ENDIAN ? _c_v64_zip_32(b, a, 0) : _c_v64_zip_32(a, b, 1); +} + +SIMD_INLINE c_v64 _c_v64_unzip_8(c_v64 a, c_v64 b, int mode) { + c_v64 t; + if (mode) { + t.u8[7] = b.u8[7]; + t.u8[6] = b.u8[5]; + t.u8[5] = b.u8[3]; + t.u8[4] = b.u8[1]; + t.u8[3] = a.u8[7]; + t.u8[2] = a.u8[5]; + t.u8[1] = a.u8[3]; + t.u8[0] = a.u8[1]; + } else { + t.u8[7] = a.u8[6]; + t.u8[6] = a.u8[4]; + t.u8[5] = a.u8[2]; + t.u8[4] = a.u8[0]; + t.u8[3] = b.u8[6]; + t.u8[2] = b.u8[4]; + t.u8[1] = b.u8[2]; + t.u8[0] = b.u8[0]; + } + return t; +} + +SIMD_INLINE c_v64 c_v64_unziplo_8(c_v64 a, c_v64 b) { + return CONFIG_BIG_ENDIAN ? _c_v64_unzip_8(a, b, 1) : _c_v64_unzip_8(a, b, 0); +} + +SIMD_INLINE c_v64 c_v64_unziphi_8(c_v64 a, c_v64 b) { + return CONFIG_BIG_ENDIAN ? _c_v64_unzip_8(b, a, 0) : _c_v64_unzip_8(b, a, 1); +} + +SIMD_INLINE c_v64 _c_v64_unzip_16(c_v64 a, c_v64 b, int mode) { + c_v64 t; + if (mode) { + t.u16[3] = b.u16[3]; + t.u16[2] = b.u16[1]; + t.u16[1] = a.u16[3]; + t.u16[0] = a.u16[1]; + } else { + t.u16[3] = a.u16[2]; + t.u16[2] = a.u16[0]; + t.u16[1] = b.u16[2]; + t.u16[0] = b.u16[0]; + } + return t; +} + +SIMD_INLINE c_v64 c_v64_unziplo_16(c_v64 a, c_v64 b) { + return CONFIG_BIG_ENDIAN ? _c_v64_unzip_16(a, b, 1) + : _c_v64_unzip_16(a, b, 0); +} + +SIMD_INLINE c_v64 c_v64_unziphi_16(c_v64 a, c_v64 b) { + return CONFIG_BIG_ENDIAN ? _c_v64_unzip_16(b, a, 0) + : _c_v64_unzip_16(b, a, 1); +} + +SIMD_INLINE c_v64 c_v64_unpacklo_u8_s16(c_v64 a) { + c_v64 t; + int endian = !!CONFIG_BIG_ENDIAN * 4; + t.s16[3] = (int16_t)a.u8[3 + endian]; + t.s16[2] = (int16_t)a.u8[2 + endian]; + t.s16[1] = (int16_t)a.u8[1 + endian]; + t.s16[0] = (int16_t)a.u8[0 + endian]; + return t; +} + +SIMD_INLINE c_v64 c_v64_unpackhi_u8_s16(c_v64 a) { + c_v64 t; + int endian = !!CONFIG_BIG_ENDIAN * 4; + t.s16[3] = (int16_t)a.u8[7 - endian]; + t.s16[2] = (int16_t)a.u8[6 - endian]; + t.s16[1] = (int16_t)a.u8[5 - endian]; + t.s16[0] = (int16_t)a.u8[4 - endian]; + return t; +} + +SIMD_INLINE c_v64 c_v64_unpacklo_s8_s16(c_v64 a) { + c_v64 t; + int endian = !!CONFIG_BIG_ENDIAN * 4; + t.s16[3] = (int16_t)a.s8[3 + endian]; + t.s16[2] = (int16_t)a.s8[2 + endian]; + t.s16[1] = (int16_t)a.s8[1 + endian]; + t.s16[0] = (int16_t)a.s8[0 + endian]; + return t; +} + +SIMD_INLINE c_v64 c_v64_unpackhi_s8_s16(c_v64 a) { + c_v64 t; + int endian = !!CONFIG_BIG_ENDIAN * 4; + t.s16[3] = (int16_t)a.s8[7 - endian]; + t.s16[2] = (int16_t)a.s8[6 - endian]; + t.s16[1] = (int16_t)a.s8[5 - endian]; + t.s16[0] = (int16_t)a.s8[4 - endian]; + return t; +} + +SIMD_INLINE c_v64 c_v64_pack_s32_s16(c_v64 a, c_v64 b) { + c_v64 t; + if (CONFIG_BIG_ENDIAN) { + c_v64 u = a; + a = b; + b = u; + } + t.s16[3] = a.s32[1] > 32767 ? 32767 : a.s32[1] < -32768 ? -32768 : a.s32[1]; + t.s16[2] = a.s32[0] > 32767 ? 32767 : a.s32[0] < -32768 ? -32768 : a.s32[0]; + t.s16[1] = b.s32[1] > 32767 ? 32767 : b.s32[1] < -32768 ? -32768 : b.s32[1]; + t.s16[0] = b.s32[0] > 32767 ? 32767 : b.s32[0] < -32768 ? -32768 : b.s32[0]; + return t; +} + +SIMD_INLINE c_v64 c_v64_pack_s16_u8(c_v64 a, c_v64 b) { + c_v64 t; + if (CONFIG_BIG_ENDIAN) { + c_v64 u = a; + a = b; + b = u; + } + t.u8[7] = a.s16[3] > 255 ? 255 : a.s16[3] < 0 ? 0 : a.s16[3]; + t.u8[6] = a.s16[2] > 255 ? 255 : a.s16[2] < 0 ? 0 : a.s16[2]; + t.u8[5] = a.s16[1] > 255 ? 255 : a.s16[1] < 0 ? 0 : a.s16[1]; + t.u8[4] = a.s16[0] > 255 ? 255 : a.s16[0] < 0 ? 0 : a.s16[0]; + t.u8[3] = b.s16[3] > 255 ? 255 : b.s16[3] < 0 ? 0 : b.s16[3]; + t.u8[2] = b.s16[2] > 255 ? 255 : b.s16[2] < 0 ? 0 : b.s16[2]; + t.u8[1] = b.s16[1] > 255 ? 255 : b.s16[1] < 0 ? 0 : b.s16[1]; + t.u8[0] = b.s16[0] > 255 ? 255 : b.s16[0] < 0 ? 0 : b.s16[0]; + return t; +} + +SIMD_INLINE c_v64 c_v64_pack_s16_s8(c_v64 a, c_v64 b) { + c_v64 t; + if (CONFIG_BIG_ENDIAN) { + c_v64 u = a; + a = b; + b = u; + } + t.u8[7] = a.s16[3] > 127 ? 127 : a.s16[3] < -128 ? 128 : a.s16[3]; + t.u8[6] = a.s16[2] > 127 ? 127 : a.s16[2] < -128 ? 128 : a.s16[2]; + t.u8[5] = a.s16[1] > 127 ? 127 : a.s16[1] < -128 ? 128 : a.s16[1]; + t.u8[4] = a.s16[0] > 127 ? 127 : a.s16[0] < -128 ? 128 : a.s16[0]; + t.u8[3] = b.s16[3] > 127 ? 127 : b.s16[3] < -128 ? 128 : b.s16[3]; + t.u8[2] = b.s16[2] > 127 ? 127 : b.s16[2] < -128 ? 128 : b.s16[2]; + t.u8[1] = b.s16[1] > 127 ? 127 : b.s16[1] < -128 ? 128 : b.s16[1]; + t.u8[0] = b.s16[0] > 127 ? 127 : b.s16[0] < -128 ? 128 : b.s16[0]; + return t; +} + +SIMD_INLINE c_v64 c_v64_unpacklo_u16_s32(c_v64 a) { + c_v64 t; + t.s32[1] = a.u16[1 + !!CONFIG_BIG_ENDIAN * 2]; + t.s32[0] = a.u16[0 + !!CONFIG_BIG_ENDIAN * 2]; + return t; +} + +SIMD_INLINE c_v64 c_v64_unpacklo_s16_s32(c_v64 a) { + c_v64 t; + t.s32[1] = a.s16[1 + !!CONFIG_BIG_ENDIAN * 2]; + t.s32[0] = a.s16[0 + !!CONFIG_BIG_ENDIAN * 2]; + return t; +} + +SIMD_INLINE c_v64 c_v64_unpackhi_u16_s32(c_v64 a) { + c_v64 t; + t.s32[1] = a.u16[3 - !!CONFIG_BIG_ENDIAN * 2]; + t.s32[0] = a.u16[2 - !!CONFIG_BIG_ENDIAN * 2]; + return t; +} + +SIMD_INLINE c_v64 c_v64_unpackhi_s16_s32(c_v64 a) { + c_v64 t; + t.s32[1] = a.s16[3 - !!CONFIG_BIG_ENDIAN * 2]; + t.s32[0] = a.s16[2 - !!CONFIG_BIG_ENDIAN * 2]; + return t; +} + +SIMD_INLINE c_v64 c_v64_shuffle_8(c_v64 a, c_v64 pattern) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) { + if (SIMD_CHECK && (pattern.u8[c] & ~7)) { + fprintf(stderr, "Error: Undefined v64_shuffle_8 index %d/%d\n", + pattern.u8[c], c); + abort(); + } + t.u8[c] = + a.u8[CONFIG_BIG_ENDIAN ? 7 - (pattern.u8[c] & 7) : pattern.u8[c] & 7]; + } + return t; +} + +SIMD_INLINE int64_t c_v64_dotp_su8(c_v64 a, c_v64 b) { + return a.s8[7] * b.u8[7] + a.s8[6] * b.u8[6] + a.s8[5] * b.u8[5] + + a.s8[4] * b.u8[4] + a.s8[3] * b.u8[3] + a.s8[2] * b.u8[2] + + a.s8[1] * b.u8[1] + a.s8[0] * b.u8[0]; +} + +SIMD_INLINE int64_t c_v64_dotp_s16(c_v64 a, c_v64 b) { + return (int64_t)(a.s16[3] * b.s16[3] + a.s16[2] * b.s16[2]) + + (int64_t)(a.s16[1] * b.s16[1] + a.s16[0] * b.s16[0]); +} + +SIMD_INLINE uint64_t c_v64_hadd_u8(c_v64 a) { + return a.u8[7] + a.u8[6] + a.u8[5] + a.u8[4] + a.u8[3] + a.u8[2] + a.u8[1] + + a.u8[0]; +} + +SIMD_INLINE int64_t c_v64_hadd_s16(c_v64 a) { + return a.s16[3] + a.s16[2] + a.s16[1] + a.s16[0]; +} + +typedef uint32_t c_sad64_internal; + +/* Implementation dependent return value. Result must be finalised with + v64_sad_u8_sum(). + The result for more than 32 v64_sad_u8() calls is undefined. */ +SIMD_INLINE c_sad64_internal c_v64_sad_u8_init() { return 0; } + +SIMD_INLINE c_sad64_internal c_v64_sad_u8(c_sad64_internal s, c_v64 a, + c_v64 b) { + int c; + for (c = 0; c < 8; c++) + s += a.u8[c] > b.u8[c] ? a.u8[c] - b.u8[c] : b.u8[c] - a.u8[c]; + return s; +} + +SIMD_INLINE uint32_t c_v64_sad_u8_sum(c_sad64_internal s) { return s; } + +typedef uint32_t c_ssd64_internal; + +/* Implementation dependent return value. Result must be finalised with + * v64_ssd_u8_sum(). */ +SIMD_INLINE c_ssd64_internal c_v64_ssd_u8_init() { return 0; } + +SIMD_INLINE c_ssd64_internal c_v64_ssd_u8(c_ssd64_internal s, c_v64 a, + c_v64 b) { + int c; + for (c = 0; c < 8; c++) s += (a.u8[c] - b.u8[c]) * (a.u8[c] - b.u8[c]); + return s; +} + +SIMD_INLINE uint32_t c_v64_ssd_u8_sum(c_ssd64_internal s) { return s; } + +SIMD_INLINE c_v64 c_v64_or(c_v64 a, c_v64 b) { + c_v64 t; + t.u64 = a.u64 | b.u64; + return t; +} + +SIMD_INLINE c_v64 c_v64_xor(c_v64 a, c_v64 b) { + c_v64 t; + t.u64 = a.u64 ^ b.u64; + return t; +} + +SIMD_INLINE c_v64 c_v64_and(c_v64 a, c_v64 b) { + c_v64 t; + t.u64 = a.u64 & b.u64; + return t; +} + +SIMD_INLINE c_v64 c_v64_andn(c_v64 a, c_v64 b) { + c_v64 t; + t.u64 = a.u64 & ~b.u64; + return t; +} + +SIMD_INLINE c_v64 c_v64_mullo_s16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) t.s16[c] = (int16_t)(a.s16[c] * b.s16[c]); + return t; +} + +SIMD_INLINE c_v64 c_v64_mulhi_s16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) t.s16[c] = (a.s16[c] * b.s16[c]) >> 16; + return t; +} + +SIMD_INLINE c_v64 c_v64_mullo_s32(c_v64 a, c_v64 b) { + c_v64 t; + t.s32[0] = (int32_t)((int64_t)a.s32[0] * b.s32[0]); + t.s32[1] = (int32_t)((int64_t)a.s32[1] * b.s32[1]); + return t; +} + +SIMD_INLINE c_v64 c_v64_madd_s16(c_v64 a, c_v64 b) { + c_v64 t; + t.s32[0] = a.s16[0] * b.s16[0] + a.s16[1] * b.s16[1]; + t.s32[1] = a.s16[2] * b.s16[2] + a.s16[3] * b.s16[3]; + return t; +} + +SIMD_INLINE c_v64 c_v64_madd_us8(c_v64 a, c_v64 b) { + c_v64 t; + int32_t u; + u = a.u8[0] * b.s8[0] + a.u8[1] * b.s8[1]; + t.s16[0] = u > 32767 ? 32767 : u < -32768 ? -32768 : u; + u = a.u8[2] * b.s8[2] + a.u8[3] * b.s8[3]; + t.s16[1] = u > 32767 ? 32767 : u < -32768 ? -32768 : u; + u = a.u8[4] * b.s8[4] + a.u8[5] * b.s8[5]; + t.s16[2] = u > 32767 ? 32767 : u < -32768 ? -32768 : u; + u = a.u8[6] * b.s8[6] + a.u8[7] * b.s8[7]; + t.s16[3] = u > 32767 ? 32767 : u < -32768 ? -32768 : u; + return t; +} + +SIMD_INLINE c_v64 c_v64_avg_u8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) t.u8[c] = (a.u8[c] + b.u8[c] + 1) >> 1; + return t; +} + +SIMD_INLINE c_v64 c_v64_rdavg_u8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) t.u8[c] = (a.u8[c] + b.u8[c]) >> 1; + return t; +} + +SIMD_INLINE c_v64 c_v64_avg_u16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) t.u16[c] = (a.u16[c] + b.u16[c] + 1) >> 1; + return t; +} + +SIMD_INLINE c_v64 c_v64_min_u8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) t.u8[c] = a.u8[c] > b.u8[c] ? b.u8[c] : a.u8[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_max_u8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) t.u8[c] = a.u8[c] > b.u8[c] ? a.u8[c] : b.u8[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_min_s8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) t.s8[c] = a.s8[c] > b.s8[c] ? b.s8[c] : a.s8[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_max_s8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) t.s8[c] = a.s8[c] > b.s8[c] ? a.s8[c] : b.s8[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_min_s16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) t.s16[c] = a.s16[c] > b.s16[c] ? b.s16[c] : a.s16[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_max_s16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) t.s16[c] = a.s16[c] > b.s16[c] ? a.s16[c] : b.s16[c]; + return t; +} + +SIMD_INLINE c_v64 c_v64_cmpgt_s8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) t.s8[c] = -(a.s8[c] > b.s8[c]); + return t; +} + +SIMD_INLINE c_v64 c_v64_cmplt_s8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) t.s8[c] = -(a.s8[c] < b.s8[c]); + return t; +} + +SIMD_INLINE c_v64 c_v64_cmpeq_8(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 8; c++) t.s8[c] = -(a.u8[c] == b.u8[c]); + return t; +} + +SIMD_INLINE c_v64 c_v64_cmpgt_s16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) t.s16[c] = -(a.s16[c] > b.s16[c]); + return t; +} + +SIMD_INLINE c_v64 c_v64_cmplt_s16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) t.s16[c] = -(a.s16[c] < b.s16[c]); + return t; +} + +SIMD_INLINE c_v64 c_v64_cmpeq_16(c_v64 a, c_v64 b) { + c_v64 t; + int c; + for (c = 0; c < 4; c++) t.s16[c] = -(a.u16[c] == b.u16[c]); + return t; +} + +SIMD_INLINE c_v64 c_v64_shl_8(c_v64 a, unsigned int n) { + c_v64 t; + int c; + if (SIMD_CHECK && n > 7) { + fprintf(stderr, "Error: Undefined u8 shift left %d\n", n); + abort(); + } + for (c = 0; c < 8; c++) t.s8[c] = a.u8[c] << n; + return t; +} + +SIMD_INLINE c_v64 c_v64_shr_u8(c_v64 a, unsigned int n) { + c_v64 t; + int c; + if (SIMD_CHECK && n > 7) { + fprintf(stderr, "Error: Undefined u8 shift right %d\n", n); + abort(); + } + for (c = 0; c < 8; c++) t.u8[c] = a.u8[c] >> n; + return t; +} + +SIMD_INLINE c_v64 c_v64_shr_s8(c_v64 a, unsigned int n) { + c_v64 t; + int c; + if (SIMD_CHECK && n > 7) { + fprintf(stderr, "Error: Undefined s8 shift right %d\n", n); + abort(); + } + for (c = 0; c < 8; c++) t.s8[c] = a.s8[c] >> n; + return t; +} + +SIMD_INLINE c_v64 c_v64_shl_16(c_v64 a, unsigned int n) { + c_v64 t; + int c; + if (SIMD_CHECK && n > 15) { + fprintf(stderr, "Error: Undefined u16 shift left %d\n", n); + abort(); + } + for (c = 0; c < 4; c++) t.u16[c] = a.u16[c] << n; + return t; +} + +SIMD_INLINE c_v64 c_v64_shr_u16(c_v64 a, unsigned int n) { + c_v64 t; + int c; + if (SIMD_CHECK && n > 15) { + fprintf(stderr, "Error: Undefined u16 shift right %d\n", n); + abort(); + } + for (c = 0; c < 4; c++) t.u16[c] = a.u16[c] >> n; + return t; +} + +SIMD_INLINE c_v64 c_v64_shr_s16(c_v64 a, unsigned int n) { + c_v64 t; + int c; + if (SIMD_CHECK && n > 15) { + fprintf(stderr, "Error: undefined s16 shift right %d\n", n); + abort(); + } + for (c = 0; c < 4; c++) t.s16[c] = a.s16[c] >> n; + return t; +} + +SIMD_INLINE c_v64 c_v64_shl_32(c_v64 a, unsigned int n) { + c_v64 t; + if (SIMD_CHECK && n > 31) { + fprintf(stderr, "Error: undefined u32 shift left %d\n", n); + abort(); + } + t.u32[1] = a.u32[1] << n; + t.u32[0] = a.u32[0] << n; + return t; +} + +SIMD_INLINE c_v64 c_v64_shr_u32(c_v64 a, unsigned int n) { + c_v64 t; + if (SIMD_CHECK && n > 31) { + fprintf(stderr, "Error: undefined u32 shift right %d\n", n); + abort(); + } + t.u32[1] = a.u32[1] >> n; + t.u32[0] = a.u32[0] >> n; + return t; +} + +SIMD_INLINE c_v64 c_v64_shr_s32(c_v64 a, unsigned int n) { + c_v64 t; + if (SIMD_CHECK && n > 31) { + fprintf(stderr, "Error: undefined s32 shift right %d\n", n); + abort(); + } + t.s32[1] = a.s32[1] >> n; + t.s32[0] = a.s32[0] >> n; + return t; +} + +SIMD_INLINE c_v64 c_v64_shr_n_byte(c_v64 x, unsigned int i) { + c_v64 t; + t.u64 = x.u64 >> i * 8; + return t; +} + +SIMD_INLINE c_v64 c_v64_shl_n_byte(c_v64 x, unsigned int i) { + c_v64 t; + t.u64 = x.u64 << i * 8; + return t; +} + +SIMD_INLINE c_v64 c_v64_align(c_v64 a, c_v64 b, unsigned int c) { + if (SIMD_CHECK && c > 7) { + fprintf(stderr, "Error: undefined alignment %d\n", c); + abort(); + } + return c ? c_v64_or(c_v64_shr_n_byte(b, c), c_v64_shl_n_byte(a, 8 - c)) : b; +} + +SIMD_INLINE c_v64 c_v64_shl_n_8(c_v64 a, unsigned int c) { + return c_v64_shl_8(a, c); +} + +SIMD_INLINE c_v64 c_v64_shr_n_u8(c_v64 a, unsigned int c) { + return c_v64_shr_u8(a, c); +} + +SIMD_INLINE c_v64 c_v64_shr_n_s8(c_v64 a, unsigned int c) { + return c_v64_shr_s8(a, c); +} + +SIMD_INLINE c_v64 c_v64_shl_n_16(c_v64 a, unsigned int c) { + return c_v64_shl_16(a, c); +} + +SIMD_INLINE c_v64 c_v64_shr_n_u16(c_v64 a, unsigned int c) { + return c_v64_shr_u16(a, c); +} + +SIMD_INLINE c_v64 c_v64_shr_n_s16(c_v64 a, unsigned int c) { + return c_v64_shr_s16(a, c); +} + +SIMD_INLINE c_v64 c_v64_shl_n_32(c_v64 a, unsigned int c) { + return c_v64_shl_32(a, c); +} + +SIMD_INLINE c_v64 c_v64_shr_n_u32(c_v64 a, unsigned int c) { + return c_v64_shr_u32(a, c); +} + +SIMD_INLINE c_v64 c_v64_shr_n_s32(c_v64 a, unsigned int c) { + return c_v64_shr_s32(a, c); +} + +#endif /* _V64_INTRINSICS_C_H */ diff --git a/third_party/aom/aom_dsp/simd/v64_intrinsics_x86.h b/third_party/aom/aom_dsp/simd/v64_intrinsics_x86.h new file mode 100644 index 0000000000..8dcc9f6fc7 --- /dev/null +++ b/third_party/aom/aom_dsp/simd/v64_intrinsics_x86.h @@ -0,0 +1,470 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _V64_INTRINSICS_H +#define _V64_INTRINSICS_H + +#include +#if defined(__SSSE3__) +#include +#endif +#if defined(__SSE4_1__) +#include +#endif + +typedef __m128i v64; + +SIMD_INLINE uint32_t v64_low_u32(v64 a) { + return (uint32_t)_mm_cvtsi128_si32(a); +} + +SIMD_INLINE uint32_t v64_high_u32(v64 a) { + return (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(a, 4)); +} + +SIMD_INLINE int32_t v64_low_s32(v64 a) { return (int32_t)_mm_cvtsi128_si32(a); } + +SIMD_INLINE int32_t v64_high_s32(v64 a) { + return (int32_t)_mm_cvtsi128_si32(_mm_srli_si128(a, 4)); +} + +SIMD_INLINE v64 v64_from_16(uint16_t a, uint16_t b, uint16_t c, uint16_t d) { + return _mm_packs_epi32( + _mm_set_epi32((int16_t)a, (int16_t)b, (int16_t)c, (int16_t)d), + _mm_setzero_si128()); +} + +SIMD_INLINE v64 v64_from_32(uint32_t x, uint32_t y) { + return _mm_set_epi32(0, 0, x, y); +} + +SIMD_INLINE v64 v64_from_64(uint64_t x) { +#ifdef __x86_64__ + return _mm_cvtsi64_si128(x); +#else + return _mm_set_epi32(0, 0, x >> 32, (uint32_t)x); +#endif +} + +SIMD_INLINE uint64_t v64_u64(v64 x) { + return (uint64_t)v64_low_u32(x) | ((uint64_t)v64_high_u32(x) << 32); +} + +SIMD_INLINE uint32_t u32_load_aligned(const void *p) { + return *((uint32_t *)p); +} + +SIMD_INLINE uint32_t u32_load_unaligned(const void *p) { + return *((uint32_t *)p); +} + +SIMD_INLINE void u32_store_aligned(void *p, uint32_t a) { + *((uint32_t *)p) = a; +} + +SIMD_INLINE void u32_store_unaligned(void *p, uint32_t a) { + *((uint32_t *)p) = a; +} + +SIMD_INLINE v64 v64_load_aligned(const void *p) { + return _mm_loadl_epi64((__m128i *)p); +} + +SIMD_INLINE v64 v64_load_unaligned(const void *p) { + return _mm_loadl_epi64((__m128i *)p); +} + +SIMD_INLINE void v64_store_aligned(void *p, v64 a) { + _mm_storel_epi64((__m128i *)p, a); +} + +SIMD_INLINE void v64_store_unaligned(void *p, v64 a) { + _mm_storel_epi64((__m128i *)p, a); +} + +// The following function requires an immediate. +#if defined(__OPTIMIZE__) && __OPTIMIZE__ +#define v64_align(a, b, c) \ + ((c) ? _mm_srli_si128(_mm_unpacklo_epi64(b, a), (c)) : b) +#else +#define v64_align(a, b, c) \ + ((c) ? v64_from_64((v64_u64(b) >> (c)*8) | (v64_u64(a) << (8 - (c)) * 8)) \ + : (b)) +#endif + +SIMD_INLINE v64 v64_zero() { return _mm_setzero_si128(); } + +SIMD_INLINE v64 v64_dup_8(uint8_t x) { return _mm_set1_epi8(x); } + +SIMD_INLINE v64 v64_dup_16(uint16_t x) { return _mm_set1_epi16(x); } + +SIMD_INLINE v64 v64_dup_32(uint32_t x) { return _mm_set1_epi32(x); } + +SIMD_INLINE v64 v64_add_8(v64 a, v64 b) { return _mm_add_epi8(a, b); } + +SIMD_INLINE v64 v64_add_16(v64 a, v64 b) { return _mm_add_epi16(a, b); } + +SIMD_INLINE v64 v64_sadd_s16(v64 a, v64 b) { return _mm_adds_epi16(a, b); } + +SIMD_INLINE v64 v64_add_32(v64 a, v64 b) { return _mm_add_epi32(a, b); } + +SIMD_INLINE v64 v64_sub_8(v64 a, v64 b) { return _mm_sub_epi8(a, b); } + +SIMD_INLINE v64 v64_ssub_u8(v64 a, v64 b) { return _mm_subs_epu8(a, b); } + +SIMD_INLINE v64 v64_ssub_s8(v64 a, v64 b) { return _mm_subs_epi8(a, b); } + +SIMD_INLINE v64 v64_sub_16(v64 a, v64 b) { return _mm_sub_epi16(a, b); } + +SIMD_INLINE v64 v64_ssub_s16(v64 a, v64 b) { return _mm_subs_epi16(a, b); } + +SIMD_INLINE v64 v64_ssub_u16(v64 a, v64 b) { return _mm_subs_epu16(a, b); } + +SIMD_INLINE v64 v64_sub_32(v64 a, v64 b) { return _mm_sub_epi32(a, b); } + +SIMD_INLINE v64 v64_abs_s16(v64 a) { +#if defined(__SSSE3__) + return _mm_abs_epi16(a); +#else + return _mm_max_epi16(a, _mm_sub_epi16(_mm_setzero_si128(), a)); +#endif +} + +SIMD_INLINE v64 v64_abs_s8(v64 a) { +#if defined(__SSSE3__) + return _mm_abs_epi8(a); +#else + v64 sign = _mm_cmplt_epi8(a, _mm_setzero_si128()); + return _mm_xor_si128(sign, _mm_add_epi8(a, sign)); +#endif +} + +SIMD_INLINE v64 v64_ziplo_8(v64 a, v64 b) { return _mm_unpacklo_epi8(b, a); } + +SIMD_INLINE v64 v64_ziphi_8(v64 a, v64 b) { + return _mm_srli_si128(_mm_unpacklo_epi8(b, a), 8); +} + +SIMD_INLINE v64 v64_ziplo_16(v64 a, v64 b) { return _mm_unpacklo_epi16(b, a); } + +SIMD_INLINE v64 v64_ziphi_16(v64 a, v64 b) { + return _mm_srli_si128(_mm_unpacklo_epi16(b, a), 8); +} + +SIMD_INLINE v64 v64_ziplo_32(v64 a, v64 b) { return _mm_unpacklo_epi32(b, a); } + +SIMD_INLINE v64 v64_ziphi_32(v64 a, v64 b) { + return _mm_srli_si128(_mm_unpacklo_epi32(b, a), 8); +} + +SIMD_INLINE v64 v64_pack_s32_s16(v64 a, v64 b) { + __m128i t = _mm_unpacklo_epi64(b, a); + return _mm_packs_epi32(t, t); +} + +SIMD_INLINE v64 v64_pack_s16_u8(v64 a, v64 b) { + __m128i t = _mm_unpacklo_epi64(b, a); + return _mm_packus_epi16(t, t); +} + +SIMD_INLINE v64 v64_pack_s16_s8(v64 a, v64 b) { + __m128i t = _mm_unpacklo_epi64(b, a); + return _mm_packs_epi16(t, t); +} + +SIMD_INLINE v64 v64_unziphi_8(v64 a, v64 b) { +#if defined(__SSSE3__) + return _mm_shuffle_epi8(_mm_unpacklo_epi64(b, a), + v64_from_64(0x0f0d0b0907050301LL)); +#else + return _mm_packus_epi16( + _mm_unpacklo_epi64(_mm_srli_epi16(b, 8), _mm_srli_epi16(a, 8)), + _mm_setzero_si128()); +#endif +} + +SIMD_INLINE v64 v64_unziplo_8(v64 a, v64 b) { +#if defined(__SSSE3__) + return _mm_shuffle_epi8(_mm_unpacklo_epi64(b, a), + v64_from_64(0x0e0c0a0806040200LL)); +#else + return v64_unziphi_8(_mm_slli_si128(a, 1), _mm_slli_si128(b, 1)); +#endif +} + +SIMD_INLINE v64 v64_unziphi_16(v64 a, v64 b) { +#if defined(__SSSE3__) + return _mm_shuffle_epi8(_mm_unpacklo_epi64(b, a), + v64_from_64(0x0f0e0b0a07060302LL)); +#else + return _mm_packs_epi32( + _mm_unpacklo_epi64(_mm_srai_epi32(b, 16), _mm_srai_epi32(a, 16)), + _mm_setzero_si128()); +#endif +} + +SIMD_INLINE v64 v64_unziplo_16(v64 a, v64 b) { +#if defined(__SSSE3__) + return _mm_shuffle_epi8(_mm_unpacklo_epi64(b, a), + v64_from_64(0x0d0c090805040100LL)); +#else + return v64_unziphi_16(_mm_slli_si128(a, 2), _mm_slli_si128(b, 2)); +#endif +} + +SIMD_INLINE v64 v64_unpacklo_u8_s16(v64 a) { + return _mm_unpacklo_epi8(a, _mm_setzero_si128()); +} + +SIMD_INLINE v64 v64_unpackhi_u8_s16(v64 a) { + return _mm_srli_si128(_mm_unpacklo_epi8(a, _mm_setzero_si128()), 8); +} + +SIMD_INLINE v64 v64_unpacklo_s8_s16(v64 a) { + return _mm_srai_epi16(_mm_unpacklo_epi8(a, a), 8); +} + +SIMD_INLINE v64 v64_unpackhi_s8_s16(v64 a) { + return _mm_srli_si128(_mm_srai_epi16(_mm_unpacklo_epi8(a, a), 8), 8); +} + +SIMD_INLINE v64 v64_unpacklo_u16_s32(v64 a) { + return _mm_unpacklo_epi16(a, _mm_setzero_si128()); +} + +SIMD_INLINE v64 v64_unpacklo_s16_s32(v64 a) { + return _mm_srai_epi32(_mm_unpacklo_epi16(_mm_setzero_si128(), a), 16); +} + +SIMD_INLINE v64 v64_unpackhi_u16_s32(v64 a) { + return _mm_srli_si128(_mm_unpacklo_epi16(a, _mm_setzero_si128()), 8); +} + +SIMD_INLINE v64 v64_unpackhi_s16_s32(v64 a) { + return _mm_srli_si128( + _mm_srai_epi32(_mm_unpacklo_epi16(_mm_setzero_si128(), a), 16), 8); +} + +SIMD_INLINE v64 v64_shuffle_8(v64 x, v64 pattern) { +#if defined(__SSSE3__) + return _mm_shuffle_epi8(x, pattern); +#else + v64 output; + unsigned char *input = (unsigned char *)&x; + unsigned char *index = (unsigned char *)&pattern; + char *selected = (char *)&output; + int counter; + + for (counter = 0; counter < 8; counter++) { + selected[counter] = input[index[counter]]; + } + + return output; +#endif +} + +SIMD_INLINE int64_t v64_dotp_su8(v64 a, v64 b) { + __m128i r, r1, r2, z; + z = _mm_setzero_si128(); + r1 = _mm_madd_epi16(_mm_slli_epi16(_mm_unpacklo_epi8(a, z), 8), + _mm_unpacklo_epi8(b, z)); + r2 = _mm_srli_si128(r1, 8); + r = _mm_add_epi32(r1, r2); + r = _mm_add_epi32(r, _mm_srli_si128(r, 4)); + return ((int32_t)v64_low_u32(r)) >> 8; +} + +SIMD_INLINE int64_t v64_dotp_s16(v64 a, v64 b) { + __m128i r = _mm_madd_epi16(a, b); +#if defined(__SSE4_1__) && defined(__x86_64__) + __m128i x = _mm_cvtepi32_epi64(r); + return _mm_cvtsi128_si64(_mm_add_epi64(x, _mm_srli_si128(x, 8))); +#else + return (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(r, 4)) + + (int64_t)_mm_cvtsi128_si32(r); +#endif +} + +SIMD_INLINE uint64_t v64_hadd_u8(v64 a) { + return v64_low_u32(_mm_sad_epu8(a, _mm_setzero_si128())); +} + +SIMD_INLINE int64_t v64_hadd_s16(v64 a) { + return v64_dotp_s16(a, v64_dup_16(1)); +} + +typedef v64 sad64_internal; + +SIMD_INLINE sad64_internal v64_sad_u8_init() { return _mm_setzero_si128(); } + +/* Implementation dependent return value. Result must be finalised with + v64_sad_u8_sum(). + The result for more than 32 v64_sad_u8() calls is undefined. */ +SIMD_INLINE sad64_internal v64_sad_u8(sad64_internal s, v64 a, v64 b) { + return _mm_add_epi64(s, _mm_sad_epu8(a, b)); +} + +SIMD_INLINE uint32_t v64_sad_u8_sum(sad64_internal s) { return v64_low_u32(s); } + +typedef v64 ssd64_internal; + +SIMD_INLINE ssd64_internal v64_ssd_u8_init() { return _mm_setzero_si128(); } + +/* Implementation dependent return value. Result must be finalised with + * v64_ssd_u8_sum(). */ +SIMD_INLINE ssd64_internal v64_ssd_u8(ssd64_internal s, v64 a, v64 b) { + v64 l = v64_sub_16(v64_ziplo_8(v64_zero(), a), v64_ziplo_8(v64_zero(), b)); + v64 h = v64_sub_16(v64_ziphi_8(v64_zero(), a), v64_ziphi_8(v64_zero(), b)); + v64 r = v64_add_32(_mm_madd_epi16(l, l), _mm_madd_epi16(h, h)); + return _mm_add_epi64( + s, v64_ziplo_32(v64_zero(), _mm_add_epi32(r, _mm_srli_si128(r, 4)))); +} + +SIMD_INLINE uint32_t v64_ssd_u8_sum(sad64_internal s) { return v64_low_u32(s); } + +SIMD_INLINE v64 v64_or(v64 a, v64 b) { return _mm_or_si128(a, b); } + +SIMD_INLINE v64 v64_xor(v64 a, v64 b) { return _mm_xor_si128(a, b); } + +SIMD_INLINE v64 v64_and(v64 a, v64 b) { return _mm_and_si128(a, b); } + +SIMD_INLINE v64 v64_andn(v64 a, v64 b) { return _mm_andnot_si128(b, a); } + +SIMD_INLINE v64 v64_mullo_s16(v64 a, v64 b) { return _mm_mullo_epi16(a, b); } + +SIMD_INLINE v64 v64_mulhi_s16(v64 a, v64 b) { return _mm_mulhi_epi16(a, b); } + +SIMD_INLINE v64 v64_mullo_s32(v64 a, v64 b) { +#if defined(__SSE4_1__) + return _mm_mullo_epi32(a, b); +#else + return _mm_unpacklo_epi32( + _mm_mul_epu32(a, b), + _mm_mul_epu32(_mm_srli_si128(a, 4), _mm_srli_si128(b, 4))); +#endif +} + +SIMD_INLINE v64 v64_madd_s16(v64 a, v64 b) { return _mm_madd_epi16(a, b); } + +SIMD_INLINE v64 v64_madd_us8(v64 a, v64 b) { +#if defined(__SSSE3__) + return _mm_maddubs_epi16(a, b); +#else + __m128i t = _mm_madd_epi16(_mm_unpacklo_epi8(a, _mm_setzero_si128()), + _mm_srai_epi16(_mm_unpacklo_epi8(b, b), 8)); + return _mm_packs_epi32(t, t); +#endif +} + +SIMD_INLINE v64 v64_avg_u8(v64 a, v64 b) { return _mm_avg_epu8(a, b); } + +SIMD_INLINE v64 v64_rdavg_u8(v64 a, v64 b) { + return _mm_sub_epi8(_mm_avg_epu8(a, b), + _mm_and_si128(_mm_xor_si128(a, b), v64_dup_8(1))); +} + +SIMD_INLINE v64 v64_avg_u16(v64 a, v64 b) { return _mm_avg_epu16(a, b); } + +SIMD_INLINE v64 v64_min_u8(v64 a, v64 b) { return _mm_min_epu8(a, b); } + +SIMD_INLINE v64 v64_max_u8(v64 a, v64 b) { return _mm_max_epu8(a, b); } + +SIMD_INLINE v64 v64_min_s8(v64 a, v64 b) { +#if defined(__SSE4_1__) + return _mm_min_epi8(a, b); +#else + v64 mask = _mm_cmplt_epi8(a, b); + return _mm_or_si128(_mm_andnot_si128(mask, b), _mm_and_si128(mask, a)); +#endif +} + +SIMD_INLINE v64 v64_max_s8(v64 a, v64 b) { +#if defined(__SSE4_1__) + return _mm_max_epi8(a, b); +#else + v64 mask = _mm_cmplt_epi8(b, a); + return _mm_or_si128(_mm_andnot_si128(mask, b), _mm_and_si128(mask, a)); +#endif +} + +SIMD_INLINE v64 v64_min_s16(v64 a, v64 b) { return _mm_min_epi16(a, b); } + +SIMD_INLINE v64 v64_max_s16(v64 a, v64 b) { return _mm_max_epi16(a, b); } + +SIMD_INLINE v64 v64_cmpgt_s8(v64 a, v64 b) { return _mm_cmpgt_epi8(a, b); } + +SIMD_INLINE v64 v64_cmplt_s8(v64 a, v64 b) { return _mm_cmplt_epi8(a, b); } + +SIMD_INLINE v64 v64_cmpeq_8(v64 a, v64 b) { return _mm_cmpeq_epi8(a, b); } + +SIMD_INLINE v64 v64_cmpgt_s16(v64 a, v64 b) { return _mm_cmpgt_epi16(a, b); } + +SIMD_INLINE v64 v64_cmplt_s16(v64 a, v64 b) { return _mm_cmplt_epi16(a, b); } + +SIMD_INLINE v64 v64_cmpeq_16(v64 a, v64 b) { return _mm_cmpeq_epi16(a, b); } + +SIMD_INLINE v64 v64_shl_8(v64 a, unsigned int c) { + return _mm_and_si128(_mm_set1_epi8((uint8_t)(0xff << c)), + _mm_sll_epi16(a, _mm_cvtsi32_si128(c))); +} + +SIMD_INLINE v64 v64_shr_u8(v64 a, unsigned int c) { + return _mm_and_si128(_mm_set1_epi8(0xff >> c), + _mm_srl_epi16(a, _mm_cvtsi32_si128(c))); +} + +SIMD_INLINE v64 v64_shr_s8(v64 a, unsigned int c) { + return _mm_packs_epi16( + _mm_sra_epi16(_mm_unpacklo_epi8(a, a), _mm_cvtsi32_si128(c + 8)), a); +} + +SIMD_INLINE v64 v64_shl_16(v64 a, unsigned int c) { + return _mm_sll_epi16(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v64 v64_shr_u16(v64 a, unsigned int c) { + return _mm_srl_epi16(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v64 v64_shr_s16(v64 a, unsigned int c) { + return _mm_sra_epi16(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v64 v64_shl_32(v64 a, unsigned int c) { + return _mm_sll_epi32(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v64 v64_shr_u32(v64 a, unsigned int c) { + return _mm_srl_epi32(a, _mm_cvtsi32_si128(c)); +} + +SIMD_INLINE v64 v64_shr_s32(v64 a, unsigned int c) { + return _mm_sra_epi32(a, _mm_cvtsi32_si128(c)); +} + +/* These intrinsics require immediate values, so we must use #defines + to enforce that. */ +#define v64_shl_n_byte(a, c) _mm_slli_si128(a, c) +#define v64_shr_n_byte(a, c) _mm_srli_si128(_mm_unpacklo_epi64(a, a), c + 8) +#define v64_shl_n_8(a, c) \ + _mm_and_si128(_mm_set1_epi8((uint8_t)(0xff << (c))), _mm_slli_epi16(a, c)) +#define v64_shr_n_u8(a, c) \ + _mm_and_si128(_mm_set1_epi8(0xff >> (c)), _mm_srli_epi16(a, c)) +#define v64_shr_n_s8(a, c) \ + _mm_packs_epi16(_mm_srai_epi16(_mm_unpacklo_epi8(a, a), (c) + 8), a) +#define v64_shl_n_16(a, c) _mm_slli_epi16(a, c) +#define v64_shr_n_u16(a, c) _mm_srli_epi16(a, c) +#define v64_shr_n_s16(a, c) _mm_srai_epi16(a, c) +#define v64_shl_n_32(a, c) _mm_slli_epi32(a, c) +#define v64_shr_n_u32(a, c) _mm_srli_epi32(a, c) +#define v64_shr_n_s32(a, c) _mm_srai_epi32(a, c) + +#endif /* _V64_INTRINSICS_H */ diff --git a/third_party/aom/aom_dsp/ssim.c b/third_party/aom/aom_dsp/ssim.c new file mode 100644 index 0000000000..141bf01c74 --- /dev/null +++ b/third_party/aom/aom_dsp/ssim.c @@ -0,0 +1,462 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/ssim.h" +#include "aom_ports/mem.h" +#include "aom_ports/system_state.h" + +void aom_ssim_parms_16x16_c(const uint8_t *s, int sp, const uint8_t *r, int rp, + uint32_t *sum_s, uint32_t *sum_r, + uint32_t *sum_sq_s, uint32_t *sum_sq_r, + uint32_t *sum_sxr) { + int i, j; + for (i = 0; i < 16; i++, s += sp, r += rp) { + for (j = 0; j < 16; j++) { + *sum_s += s[j]; + *sum_r += r[j]; + *sum_sq_s += s[j] * s[j]; + *sum_sq_r += r[j] * r[j]; + *sum_sxr += s[j] * r[j]; + } + } +} +void aom_ssim_parms_8x8_c(const uint8_t *s, int sp, const uint8_t *r, int rp, + uint32_t *sum_s, uint32_t *sum_r, uint32_t *sum_sq_s, + uint32_t *sum_sq_r, uint32_t *sum_sxr) { + int i, j; + for (i = 0; i < 8; i++, s += sp, r += rp) { + for (j = 0; j < 8; j++) { + *sum_s += s[j]; + *sum_r += r[j]; + *sum_sq_s += s[j] * s[j]; + *sum_sq_r += r[j] * r[j]; + *sum_sxr += s[j] * r[j]; + } + } +} + +#if CONFIG_HIGHBITDEPTH +void aom_highbd_ssim_parms_8x8_c(const uint16_t *s, int sp, const uint16_t *r, + int rp, uint32_t *sum_s, uint32_t *sum_r, + uint32_t *sum_sq_s, uint32_t *sum_sq_r, + uint32_t *sum_sxr) { + int i, j; + for (i = 0; i < 8; i++, s += sp, r += rp) { + for (j = 0; j < 8; j++) { + *sum_s += s[j]; + *sum_r += r[j]; + *sum_sq_s += s[j] * s[j]; + *sum_sq_r += r[j] * r[j]; + *sum_sxr += s[j] * r[j]; + } + } +} +#endif // CONFIG_HIGHBITDEPTH + +static const int64_t cc1 = 26634; // (64^2*(.01*255)^2 +static const int64_t cc2 = 239708; // (64^2*(.03*255)^2 +static const int64_t cc1_10 = 428658; // (64^2*(.01*1023)^2 +static const int64_t cc2_10 = 3857925; // (64^2*(.03*1023)^2 +static const int64_t cc1_12 = 6868593; // (64^2*(.01*4095)^2 +static const int64_t cc2_12 = 61817334; // (64^2*(.03*4095)^2 + +static double similarity(uint32_t sum_s, uint32_t sum_r, uint32_t sum_sq_s, + uint32_t sum_sq_r, uint32_t sum_sxr, int count, + uint32_t bd) { + int64_t ssim_n, ssim_d; + int64_t c1, c2; + if (bd == 8) { + // scale the constants by number of pixels + c1 = (cc1 * count * count) >> 12; + c2 = (cc2 * count * count) >> 12; + } else if (bd == 10) { + c1 = (cc1_10 * count * count) >> 12; + c2 = (cc2_10 * count * count) >> 12; + } else if (bd == 12) { + c1 = (cc1_12 * count * count) >> 12; + c2 = (cc2_12 * count * count) >> 12; + } else { + c1 = c2 = 0; + assert(0); + } + + ssim_n = (2 * sum_s * sum_r + c1) * + ((int64_t)2 * count * sum_sxr - (int64_t)2 * sum_s * sum_r + c2); + + ssim_d = (sum_s * sum_s + sum_r * sum_r + c1) * + ((int64_t)count * sum_sq_s - (int64_t)sum_s * sum_s + + (int64_t)count * sum_sq_r - (int64_t)sum_r * sum_r + c2); + + return ssim_n * 1.0 / ssim_d; +} + +static double ssim_8x8(const uint8_t *s, int sp, const uint8_t *r, int rp) { + uint32_t sum_s = 0, sum_r = 0, sum_sq_s = 0, sum_sq_r = 0, sum_sxr = 0; + aom_ssim_parms_8x8(s, sp, r, rp, &sum_s, &sum_r, &sum_sq_s, &sum_sq_r, + &sum_sxr); + return similarity(sum_s, sum_r, sum_sq_s, sum_sq_r, sum_sxr, 64, 8); +} + +#if CONFIG_HIGHBITDEPTH +static double highbd_ssim_8x8(const uint16_t *s, int sp, const uint16_t *r, + int rp, uint32_t bd, uint32_t shift) { + uint32_t sum_s = 0, sum_r = 0, sum_sq_s = 0, sum_sq_r = 0, sum_sxr = 0; + aom_highbd_ssim_parms_8x8(s, sp, r, rp, &sum_s, &sum_r, &sum_sq_s, &sum_sq_r, + &sum_sxr); + return similarity(sum_s >> shift, sum_r >> shift, sum_sq_s >> (2 * shift), + sum_sq_r >> (2 * shift), sum_sxr >> (2 * shift), 64, bd); +} +#endif // CONFIG_HIGHBITDEPTH + +// We are using a 8x8 moving window with starting location of each 8x8 window +// on the 4x4 pixel grid. Such arrangement allows the windows to overlap +// block boundaries to penalize blocking artifacts. +static double aom_ssim2(const uint8_t *img1, const uint8_t *img2, + int stride_img1, int stride_img2, int width, + int height) { + int i, j; + int samples = 0; + double ssim_total = 0; + + // sample point start with each 4x4 location + for (i = 0; i <= height - 8; + i += 4, img1 += stride_img1 * 4, img2 += stride_img2 * 4) { + for (j = 0; j <= width - 8; j += 4) { + double v = ssim_8x8(img1 + j, stride_img1, img2 + j, stride_img2); + ssim_total += v; + samples++; + } + } + ssim_total /= samples; + return ssim_total; +} + +#if CONFIG_HIGHBITDEPTH +static double aom_highbd_ssim2(const uint8_t *img1, const uint8_t *img2, + int stride_img1, int stride_img2, int width, + int height, uint32_t bd, uint32_t shift) { + int i, j; + int samples = 0; + double ssim_total = 0; + + // sample point start with each 4x4 location + for (i = 0; i <= height - 8; + i += 4, img1 += stride_img1 * 4, img2 += stride_img2 * 4) { + for (j = 0; j <= width - 8; j += 4) { + double v = highbd_ssim_8x8(CONVERT_TO_SHORTPTR(img1 + j), stride_img1, + CONVERT_TO_SHORTPTR(img2 + j), stride_img2, bd, + shift); + ssim_total += v; + samples++; + } + } + ssim_total /= samples; + return ssim_total; +} +#endif // CONFIG_HIGHBITDEPTH + +double aom_calc_ssim(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, double *weight) { + double a, b, c; + double ssimv; + + a = aom_ssim2(source->y_buffer, dest->y_buffer, source->y_stride, + dest->y_stride, source->y_crop_width, source->y_crop_height); + + b = aom_ssim2(source->u_buffer, dest->u_buffer, source->uv_stride, + dest->uv_stride, source->uv_crop_width, source->uv_crop_height); + + c = aom_ssim2(source->v_buffer, dest->v_buffer, source->uv_stride, + dest->uv_stride, source->uv_crop_width, source->uv_crop_height); + + ssimv = a * .8 + .1 * (b + c); + + *weight = 1; + + return ssimv; +} + +// traditional ssim as per: http://en.wikipedia.org/wiki/Structural_similarity +// +// Re working out the math -> +// +// ssim(x,y) = (2*mean(x)*mean(y) + c1)*(2*cov(x,y)+c2) / +// ((mean(x)^2+mean(y)^2+c1)*(var(x)+var(y)+c2)) +// +// mean(x) = sum(x) / n +// +// cov(x,y) = (n*sum(xi*yi)-sum(x)*sum(y))/(n*n) +// +// var(x) = (n*sum(xi*xi)-sum(xi)*sum(xi))/(n*n) +// +// ssim(x,y) = +// (2*sum(x)*sum(y)/(n*n) + c1)*(2*(n*sum(xi*yi)-sum(x)*sum(y))/(n*n)+c2) / +// (((sum(x)*sum(x)+sum(y)*sum(y))/(n*n) +c1) * +// ((n*sum(xi*xi) - sum(xi)*sum(xi))/(n*n)+ +// (n*sum(yi*yi) - sum(yi)*sum(yi))/(n*n)+c2))) +// +// factoring out n*n +// +// ssim(x,y) = +// (2*sum(x)*sum(y) + n*n*c1)*(2*(n*sum(xi*yi)-sum(x)*sum(y))+n*n*c2) / +// (((sum(x)*sum(x)+sum(y)*sum(y)) + n*n*c1) * +// (n*sum(xi*xi)-sum(xi)*sum(xi)+n*sum(yi*yi)-sum(yi)*sum(yi)+n*n*c2)) +// +// Replace c1 with n*n * c1 for the final step that leads to this code: +// The final step scales by 12 bits so we don't lose precision in the constants. + +static double ssimv_similarity(const Ssimv *sv, int64_t n) { + // Scale the constants by number of pixels. + const int64_t c1 = (cc1 * n * n) >> 12; + const int64_t c2 = (cc2 * n * n) >> 12; + + const double l = 1.0 * (2 * sv->sum_s * sv->sum_r + c1) / + (sv->sum_s * sv->sum_s + sv->sum_r * sv->sum_r + c1); + + // Since these variables are unsigned sums, convert to double so + // math is done in double arithmetic. + const double v = (2.0 * n * sv->sum_sxr - 2 * sv->sum_s * sv->sum_r + c2) / + (n * sv->sum_sq_s - sv->sum_s * sv->sum_s + + n * sv->sum_sq_r - sv->sum_r * sv->sum_r + c2); + + return l * v; +} + +// The first term of the ssim metric is a luminance factor. +// +// (2*mean(x)*mean(y) + c1)/ (mean(x)^2+mean(y)^2+c1) +// +// This luminance factor is super sensitive to the dark side of luminance +// values and completely insensitive on the white side. check out 2 sets +// (1,3) and (250,252) the term gives ( 2*1*3/(1+9) = .60 +// 2*250*252/ (250^2+252^2) => .99999997 +// +// As a result in this tweaked version of the calculation in which the +// luminance is taken as percentage off from peak possible. +// +// 255 * 255 - (sum_s - sum_r) / count * (sum_s - sum_r) / count +// +static double ssimv_similarity2(const Ssimv *sv, int64_t n) { + // Scale the constants by number of pixels. + const int64_t c1 = (cc1 * n * n) >> 12; + const int64_t c2 = (cc2 * n * n) >> 12; + + const double mean_diff = (1.0 * sv->sum_s - sv->sum_r) / n; + const double l = (255 * 255 - mean_diff * mean_diff + c1) / (255 * 255 + c1); + + // Since these variables are unsigned, sums convert to double so + // math is done in double arithmetic. + const double v = (2.0 * n * sv->sum_sxr - 2 * sv->sum_s * sv->sum_r + c2) / + (n * sv->sum_sq_s - sv->sum_s * sv->sum_s + + n * sv->sum_sq_r - sv->sum_r * sv->sum_r + c2); + + return l * v; +} +static void ssimv_parms(uint8_t *img1, int img1_pitch, uint8_t *img2, + int img2_pitch, Ssimv *sv) { + aom_ssim_parms_8x8(img1, img1_pitch, img2, img2_pitch, &sv->sum_s, &sv->sum_r, + &sv->sum_sq_s, &sv->sum_sq_r, &sv->sum_sxr); +} + +double aom_get_ssim_metrics(uint8_t *img1, int img1_pitch, uint8_t *img2, + int img2_pitch, int width, int height, Ssimv *sv2, + Metrics *m, int do_inconsistency) { + double dssim_total = 0; + double ssim_total = 0; + double ssim2_total = 0; + double inconsistency_total = 0; + int i, j; + int c = 0; + double norm; + double old_ssim_total = 0; + aom_clear_system_state(); + // We can sample points as frequently as we like start with 1 per 4x4. + for (i = 0; i < height; + i += 4, img1 += img1_pitch * 4, img2 += img2_pitch * 4) { + for (j = 0; j < width; j += 4, ++c) { + Ssimv sv = { 0 }; + double ssim; + double ssim2; + double dssim; + uint32_t var_new; + uint32_t var_old; + uint32_t mean_new; + uint32_t mean_old; + double ssim_new; + double ssim_old; + + // Not sure there's a great way to handle the edge pixels + // in ssim when using a window. Seems biased against edge pixels + // however you handle this. This uses only samples that are + // fully in the frame. + if (j + 8 <= width && i + 8 <= height) { + ssimv_parms(img1 + j, img1_pitch, img2 + j, img2_pitch, &sv); + } + + ssim = ssimv_similarity(&sv, 64); + ssim2 = ssimv_similarity2(&sv, 64); + + sv.ssim = ssim2; + + // dssim is calculated to use as an actual error metric and + // is scaled up to the same range as sum square error. + // Since we are subsampling every 16th point maybe this should be + // *16 ? + dssim = 255 * 255 * (1 - ssim2) / 2; + + // Here I introduce a new error metric: consistency-weighted + // SSIM-inconsistency. This metric isolates frames where the + // SSIM 'suddenly' changes, e.g. if one frame in every 8 is much + // sharper or blurrier than the others. Higher values indicate a + // temporally inconsistent SSIM. There are two ideas at work: + // + // 1) 'SSIM-inconsistency': the total inconsistency value + // reflects how much SSIM values are changing between this + // source / reference frame pair and the previous pair. + // + // 2) 'consistency-weighted': weights de-emphasize areas in the + // frame where the scene content has changed. Changes in scene + // content are detected via changes in local variance and local + // mean. + // + // Thus the overall measure reflects how inconsistent the SSIM + // values are, over consistent regions of the frame. + // + // The metric has three terms: + // + // term 1 -> uses change in scene Variance to weight error score + // 2 * var(Fi)*var(Fi-1) / (var(Fi)^2+var(Fi-1)^2) + // larger changes from one frame to the next mean we care + // less about consistency. + // + // term 2 -> uses change in local scene luminance to weight error + // 2 * avg(Fi)*avg(Fi-1) / (avg(Fi)^2+avg(Fi-1)^2) + // larger changes from one frame to the next mean we care + // less about consistency. + // + // term3 -> measures inconsistency in ssim scores between frames + // 1 - ( 2 * ssim(Fi)*ssim(Fi-1)/(ssim(Fi)^2+sssim(Fi-1)^2). + // + // This term compares the ssim score for the same location in 2 + // subsequent frames. + var_new = sv.sum_sq_s - sv.sum_s * sv.sum_s / 64; + var_old = sv2[c].sum_sq_s - sv2[c].sum_s * sv2[c].sum_s / 64; + mean_new = sv.sum_s; + mean_old = sv2[c].sum_s; + ssim_new = sv.ssim; + ssim_old = sv2[c].ssim; + + if (do_inconsistency) { + // We do the metric once for every 4x4 block in the image. Since + // we are scaling the error to SSE for use in a psnr calculation + // 1.0 = 4x4x255x255 the worst error we can possibly have. + static const double kScaling = 4. * 4 * 255 * 255; + + // The constants have to be non 0 to avoid potential divide by 0 + // issues other than that they affect kind of a weighting between + // the terms. No testing of what the right terms should be has been + // done. + static const double c1 = 1, c2 = 1, c3 = 1; + + // This measures how much consistent variance is in two consecutive + // source frames. 1.0 means they have exactly the same variance. + const double variance_term = + (2.0 * var_old * var_new + c1) / + (1.0 * var_old * var_old + 1.0 * var_new * var_new + c1); + + // This measures how consistent the local mean are between two + // consecutive frames. 1.0 means they have exactly the same mean. + const double mean_term = + (2.0 * mean_old * mean_new + c2) / + (1.0 * mean_old * mean_old + 1.0 * mean_new * mean_new + c2); + + // This measures how consistent the ssims of two + // consecutive frames is. 1.0 means they are exactly the same. + double ssim_term = + pow((2.0 * ssim_old * ssim_new + c3) / + (ssim_old * ssim_old + ssim_new * ssim_new + c3), + 5); + + double this_inconsistency; + + // Floating point math sometimes makes this > 1 by a tiny bit. + // We want the metric to scale between 0 and 1.0 so we can convert + // it to an snr scaled value. + if (ssim_term > 1) ssim_term = 1; + + // This converts the consistency metric to an inconsistency metric + // ( so we can scale it like psnr to something like sum square error. + // The reason for the variance and mean terms is the assumption that + // if there are big changes in the source we shouldn't penalize + // inconsistency in ssim scores a bit less as it will be less visible + // to the user. + this_inconsistency = (1 - ssim_term) * variance_term * mean_term; + + this_inconsistency *= kScaling; + inconsistency_total += this_inconsistency; + } + sv2[c] = sv; + ssim_total += ssim; + ssim2_total += ssim2; + dssim_total += dssim; + + old_ssim_total += ssim_old; + } + old_ssim_total += 0; + } + + norm = 1. / (width / 4) / (height / 4); + ssim_total *= norm; + ssim2_total *= norm; + m->ssim2 = ssim2_total; + m->ssim = ssim_total; + if (old_ssim_total == 0) inconsistency_total = 0; + + m->ssimc = inconsistency_total; + + m->dssim = dssim_total; + return inconsistency_total; +} + +#if CONFIG_HIGHBITDEPTH +double aom_highbd_calc_ssim(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, double *weight, + uint32_t bd, uint32_t in_bd) { + double a, b, c; + double ssimv; + uint32_t shift = 0; + + assert(bd >= in_bd); + shift = bd - in_bd; + + a = aom_highbd_ssim2(source->y_buffer, dest->y_buffer, source->y_stride, + dest->y_stride, source->y_crop_width, + source->y_crop_height, in_bd, shift); + + b = aom_highbd_ssim2(source->u_buffer, dest->u_buffer, source->uv_stride, + dest->uv_stride, source->uv_crop_width, + source->uv_crop_height, in_bd, shift); + + c = aom_highbd_ssim2(source->v_buffer, dest->v_buffer, source->uv_stride, + dest->uv_stride, source->uv_crop_width, + source->uv_crop_height, in_bd, shift); + + ssimv = a * .8 + .1 * (b + c); + + *weight = 1; + + return ssimv; +} + +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/ssim.h b/third_party/aom/aom_dsp/ssim.h new file mode 100644 index 0000000000..902735e50e --- /dev/null +++ b/third_party/aom/aom_dsp/ssim.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_SSIM_H_ +#define AOM_DSP_SSIM_H_ + +#define MAX_SSIM_DB 100.0; + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./aom_config.h" +#include "aom_scale/yv12config.h" + +// metrics used for calculating ssim, ssim2, dssim, and ssimc +typedef struct { + // source sum ( over 8x8 region ) + uint32_t sum_s; + + // reference sum (over 8x8 region ) + uint32_t sum_r; + + // source sum squared ( over 8x8 region ) + uint32_t sum_sq_s; + + // reference sum squared (over 8x8 region ) + uint32_t sum_sq_r; + + // sum of source times reference (over 8x8 region) + uint32_t sum_sxr; + + // calculated ssim score between source and reference + double ssim; +} Ssimv; + +// metrics collected on a frame basis +typedef struct { + // ssim consistency error metric ( see code for explanation ) + double ssimc; + + // standard ssim + double ssim; + + // revised ssim ( see code for explanation) + double ssim2; + + // ssim restated as an error metric like sse + double dssim; + + // dssim converted to decibels + double dssimd; + + // ssimc converted to decibels + double ssimcd; +} Metrics; + +double aom_get_ssim_metrics(uint8_t *img1, int img1_pitch, uint8_t *img2, + int img2_pitch, int width, int height, Ssimv *sv2, + Metrics *m, int do_inconsistency); + +double aom_calc_ssim(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, double *weight); + +double aom_calc_fastssim(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, double *ssim_y, + double *ssim_u, double *ssim_v, uint32_t bd, + uint32_t in_bd); + +#if CONFIG_HIGHBITDEPTH +double aom_highbd_calc_ssim(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, double *weight, + uint32_t bd, uint32_t in_bd); +#endif // CONFIG_HIGHBITDEPTH + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_SSIM_H_ diff --git a/third_party/aom/aom_dsp/subtract.c b/third_party/aom/aom_dsp/subtract.c new file mode 100644 index 0000000000..8dda96efbc --- /dev/null +++ b/third_party/aom/aom_dsp/subtract.c @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" + +void aom_subtract_block_c(int rows, int cols, int16_t *diff, + ptrdiff_t diff_stride, const uint8_t *src, + ptrdiff_t src_stride, const uint8_t *pred, + ptrdiff_t pred_stride) { + int r, c; + + for (r = 0; r < rows; r++) { + for (c = 0; c < cols; c++) diff[c] = src[c] - pred[c]; + + diff += diff_stride; + pred += pred_stride; + src += src_stride; + } +} + +#if CONFIG_HIGHBITDEPTH +void aom_highbd_subtract_block_c(int rows, int cols, int16_t *diff, + ptrdiff_t diff_stride, const uint8_t *src8, + ptrdiff_t src_stride, const uint8_t *pred8, + ptrdiff_t pred_stride, int bd) { + int r, c; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *pred = CONVERT_TO_SHORTPTR(pred8); + (void)bd; + + for (r = 0; r < rows; r++) { + for (c = 0; c < cols; c++) { + diff[c] = src[c] - pred[c]; + } + + diff += diff_stride; + pred += pred_stride; + src += src_stride; + } +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/sum_squares.c b/third_party/aom/aom_dsp/sum_squares.c new file mode 100644 index 0000000000..b9155fdc05 --- /dev/null +++ b/third_party/aom/aom_dsp/sum_squares.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" + +uint64_t aom_sum_squares_2d_i16_c(const int16_t *src, int src_stride, int width, + int height) { + int r, c; + uint64_t ss = 0; + + for (r = 0; r < height; r++) { + for (c = 0; c < width; c++) { + const int16_t v = src[c]; + ss += v * v; + } + src += src_stride; + } + + return ss; +} + +uint64_t aom_sum_squares_i16_c(const int16_t *src, uint32_t n) { + uint64_t ss = 0; + do { + const int16_t v = *src++; + ss += v * v; + } while (--n); + + return ss; +} diff --git a/third_party/aom/aom_dsp/txfm_common.h b/third_party/aom/aom_dsp/txfm_common.h new file mode 100644 index 0000000000..a5e964aadb --- /dev/null +++ b/third_party/aom/aom_dsp/txfm_common.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_TXFM_COMMON_H_ +#define AOM_DSP_TXFM_COMMON_H_ + +#include "aom_dsp/aom_dsp_common.h" + +// Constants and Macros used by all idct/dct functions +#define DCT_CONST_BITS 14 +#define DCT_CONST_ROUNDING (1 << (DCT_CONST_BITS - 1)) + +#define UNIT_QUANT_SHIFT 2 +#define UNIT_QUANT_FACTOR (1 << UNIT_QUANT_SHIFT) + +// Constants: +// for (int i = 1; i< 32; ++i) +// printf("static const int cospi_%d_64 = %.0f;\n", i, +// round(16384 * cos(i*M_PI/64))); +// Note: sin(k*Pi/64) = cos((32-k)*Pi/64) +static const tran_high_t cospi_1_64 = 16364; +static const tran_high_t cospi_2_64 = 16305; +static const tran_high_t cospi_3_64 = 16207; +static const tran_high_t cospi_4_64 = 16069; +static const tran_high_t cospi_5_64 = 15893; +static const tran_high_t cospi_6_64 = 15679; +static const tran_high_t cospi_7_64 = 15426; +static const tran_high_t cospi_8_64 = 15137; +static const tran_high_t cospi_9_64 = 14811; +static const tran_high_t cospi_10_64 = 14449; +static const tran_high_t cospi_11_64 = 14053; +static const tran_high_t cospi_12_64 = 13623; +static const tran_high_t cospi_13_64 = 13160; +static const tran_high_t cospi_14_64 = 12665; +static const tran_high_t cospi_15_64 = 12140; +static const tran_high_t cospi_16_64 = 11585; +static const tran_high_t cospi_17_64 = 11003; +static const tran_high_t cospi_18_64 = 10394; +static const tran_high_t cospi_19_64 = 9760; +static const tran_high_t cospi_20_64 = 9102; +static const tran_high_t cospi_21_64 = 8423; +static const tran_high_t cospi_22_64 = 7723; +static const tran_high_t cospi_23_64 = 7005; +static const tran_high_t cospi_24_64 = 6270; +static const tran_high_t cospi_25_64 = 5520; +static const tran_high_t cospi_26_64 = 4756; +static const tran_high_t cospi_27_64 = 3981; +static const tran_high_t cospi_28_64 = 3196; +static const tran_high_t cospi_29_64 = 2404; +static const tran_high_t cospi_30_64 = 1606; +static const tran_high_t cospi_31_64 = 804; + +// 16384 * sqrt(2) * sin(kPi/9) * 2 / 3 +static const tran_high_t sinpi_1_9 = 5283; +static const tran_high_t sinpi_2_9 = 9929; +static const tran_high_t sinpi_3_9 = 13377; +static const tran_high_t sinpi_4_9 = 15212; + +// 16384 * sqrt(2) +static const tran_high_t Sqrt2 = 23170; + +#endif // AOM_DSP_TXFM_COMMON_H_ diff --git a/third_party/aom/aom_dsp/variance.c b/third_party/aom/aom_dsp/variance.c new file mode 100644 index 0000000000..9fc0db783f --- /dev/null +++ b/third_party/aom/aom_dsp/variance.c @@ -0,0 +1,1249 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "aom_ports/mem.h" +#include "aom/aom_integer.h" + +#include "aom_dsp/variance.h" +#include "aom_dsp/aom_filter.h" + +uint32_t aom_get4x4sse_cs_c(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride) { + int distortion = 0; + int r, c; + + for (r = 0; r < 4; ++r) { + for (c = 0; c < 4; ++c) { + int diff = a[c] - b[c]; + distortion += diff * diff; + } + + a += a_stride; + b += b_stride; + } + + return distortion; +} + +uint32_t aom_get_mb_ss_c(const int16_t *a) { + unsigned int i, sum = 0; + + for (i = 0; i < 256; ++i) { + sum += a[i] * a[i]; + } + + return sum; +} + +uint32_t aom_variance_halfpixvar16x16_h_c(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + uint32_t *sse) { + return aom_sub_pixel_variance16x16_c(a, a_stride, 4, 0, b, b_stride, sse); +} + +uint32_t aom_variance_halfpixvar16x16_v_c(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + uint32_t *sse) { + return aom_sub_pixel_variance16x16_c(a, a_stride, 0, 4, b, b_stride, sse); +} + +uint32_t aom_variance_halfpixvar16x16_hv_c(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + uint32_t *sse) { + return aom_sub_pixel_variance16x16_c(a, a_stride, 4, 4, b, b_stride, sse); +} + +static void variance(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int w, int h, uint32_t *sse, int *sum) { + int i, j; + + *sum = 0; + *sse = 0; + + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + const int diff = a[j] - b[j]; + *sum += diff; + *sse += diff * diff; + } + + a += a_stride; + b += b_stride; + } +} + +uint32_t aom_sse_odd_size(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int w, int h) { + uint32_t sse; + int sum; + variance(a, a_stride, b, b_stride, w, h, &sse, &sum); + return sse; +} + +// Applies a 1-D 2-tap bilinear filter to the source block in either horizontal +// or vertical direction to produce the filtered output block. Used to implement +// the first-pass of 2-D separable filter. +// +// Produces int16_t output to retain precision for the next pass. Two filter +// taps should sum to FILTER_WEIGHT. pixel_step defines whether the filter is +// applied horizontally (pixel_step = 1) or vertically (pixel_step = stride). +// It defines the offset required to move from one input to the next. +static void var_filter_block2d_bil_first_pass(const uint8_t *a, uint16_t *b, + unsigned int src_pixels_per_line, + int pixel_step, + unsigned int output_height, + unsigned int output_width, + const uint8_t *filter) { + unsigned int i, j; + + for (i = 0; i < output_height; ++i) { + for (j = 0; j < output_width; ++j) { + b[j] = ROUND_POWER_OF_TWO( + (int)a[0] * filter[0] + (int)a[pixel_step] * filter[1], FILTER_BITS); + + ++a; + } + + a += src_pixels_per_line - output_width; + b += output_width; + } +} + +// Applies a 1-D 2-tap bilinear filter to the source block in either horizontal +// or vertical direction to produce the filtered output block. Used to implement +// the second-pass of 2-D separable filter. +// +// Requires 16-bit input as produced by filter_block2d_bil_first_pass. Two +// filter taps should sum to FILTER_WEIGHT. pixel_step defines whether the +// filter is applied horizontally (pixel_step = 1) or vertically +// (pixel_step = stride). It defines the offset required to move from one input +// to the next. Output is 8-bit. +static void var_filter_block2d_bil_second_pass(const uint16_t *a, uint8_t *b, + unsigned int src_pixels_per_line, + unsigned int pixel_step, + unsigned int output_height, + unsigned int output_width, + const uint8_t *filter) { + unsigned int i, j; + + for (i = 0; i < output_height; ++i) { + for (j = 0; j < output_width; ++j) { + b[j] = ROUND_POWER_OF_TWO( + (int)a[0] * filter[0] + (int)a[pixel_step] * filter[1], FILTER_BITS); + ++a; + } + + a += src_pixels_per_line - output_width; + b += output_width; + } +} + +#define VAR(W, H) \ + uint32_t aom_variance##W##x##H##_c(const uint8_t *a, int a_stride, \ + const uint8_t *b, int b_stride, \ + uint32_t *sse) { \ + int sum; \ + variance(a, a_stride, b, b_stride, W, H, sse, &sum); \ + return *sse - (uint32_t)(((int64_t)sum * sum) / (W * H)); \ + } + +#define SUBPIX_VAR(W, H) \ + uint32_t aom_sub_pixel_variance##W##x##H##_c( \ + const uint8_t *a, int a_stride, int xoffset, int yoffset, \ + const uint8_t *b, int b_stride, uint32_t *sse) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint8_t temp2[H * W]; \ + \ + var_filter_block2d_bil_first_pass(a, fdata3, a_stride, 1, H + 1, W, \ + bilinear_filters_2t[xoffset]); \ + var_filter_block2d_bil_second_pass(fdata3, temp2, W, W, H, W, \ + bilinear_filters_2t[yoffset]); \ + \ + return aom_variance##W##x##H##_c(temp2, W, b, b_stride, sse); \ + } + +#define SUBPIX_AVG_VAR(W, H) \ + uint32_t aom_sub_pixel_avg_variance##W##x##H##_c( \ + const uint8_t *a, int a_stride, int xoffset, int yoffset, \ + const uint8_t *b, int b_stride, uint32_t *sse, \ + const uint8_t *second_pred) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint8_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \ + \ + var_filter_block2d_bil_first_pass(a, fdata3, a_stride, 1, H + 1, W, \ + bilinear_filters_2t[xoffset]); \ + var_filter_block2d_bil_second_pass(fdata3, temp2, W, W, H, W, \ + bilinear_filters_2t[yoffset]); \ + \ + aom_comp_avg_pred(temp3, second_pred, W, H, temp2, W); \ + \ + return aom_variance##W##x##H##_c(temp3, W, b, b_stride, sse); \ + } + +/* Identical to the variance call except it takes an additional parameter, sum, + * and returns that value using pass-by-reference instead of returning + * sse - sum^2 / w*h + */ +#define GET_VAR(W, H) \ + void aom_get##W##x##H##var_c(const uint8_t *a, int a_stride, \ + const uint8_t *b, int b_stride, uint32_t *sse, \ + int *sum) { \ + variance(a, a_stride, b, b_stride, W, H, sse, sum); \ + } + +/* Identical to the variance call except it does not calculate the + * sse - sum^2 / w*h and returns sse in addtion to modifying the passed in + * variable. + */ +#define MSE(W, H) \ + uint32_t aom_mse##W##x##H##_c(const uint8_t *a, int a_stride, \ + const uint8_t *b, int b_stride, \ + uint32_t *sse) { \ + int sum; \ + variance(a, a_stride, b, b_stride, W, H, sse, &sum); \ + return *sse; \ + } + +/* All three forms of the variance are available in the same sizes. */ +#define VARIANCES(W, H) \ + VAR(W, H) \ + SUBPIX_VAR(W, H) \ + SUBPIX_AVG_VAR(W, H) + +#if CONFIG_AV1 && CONFIG_EXT_PARTITION +VARIANCES(128, 128) +VARIANCES(128, 64) +VARIANCES(64, 128) +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION +VARIANCES(64, 64) +VARIANCES(64, 32) +VARIANCES(32, 64) +VARIANCES(32, 32) +VARIANCES(32, 16) +VARIANCES(16, 32) +VARIANCES(16, 16) +VARIANCES(16, 8) +VARIANCES(8, 16) +VARIANCES(8, 8) +VARIANCES(8, 4) +VARIANCES(4, 8) +VARIANCES(4, 4) +VARIANCES(4, 2) +VARIANCES(2, 4) +VARIANCES(2, 2) + +GET_VAR(16, 16) +GET_VAR(8, 8) + +MSE(16, 16) +MSE(16, 8) +MSE(8, 16) +MSE(8, 8) + +void aom_comp_avg_pred_c(uint8_t *comp_pred, const uint8_t *pred, int width, + int height, const uint8_t *ref, int ref_stride) { + int i, j; + + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const int tmp = pred[j] + ref[j]; + comp_pred[j] = ROUND_POWER_OF_TWO(tmp, 1); + } + comp_pred += width; + pred += width; + ref += ref_stride; + } +} + +// Get pred block from up-sampled reference. +void aom_upsampled_pred_c(uint8_t *comp_pred, int width, int height, + const uint8_t *ref, int ref_stride) { + int i, j, k; + int stride = ref_stride << 3; + + for (i = 0; i < height; i++) { + for (j = 0, k = 0; j < width; j++, k += 8) { + comp_pred[j] = ref[k]; + } + comp_pred += width; + ref += stride; + } +} + +void aom_comp_avg_upsampled_pred_c(uint8_t *comp_pred, const uint8_t *pred, + int width, int height, const uint8_t *ref, + int ref_stride) { + int i, j; + int stride = ref_stride << 3; + + for (i = 0; i < height; i++) { + for (j = 0; j < width; j++) { + const int tmp = ref[(j << 3)] + pred[j]; + comp_pred[j] = ROUND_POWER_OF_TWO(tmp, 1); + } + comp_pred += width; + pred += width; + ref += stride; + } +} + +#if CONFIG_HIGHBITDEPTH +static void highbd_variance64(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, int w, int h, + uint64_t *sse, int64_t *sum) { + int i, j; + + uint16_t *a = CONVERT_TO_SHORTPTR(a8); + uint16_t *b = CONVERT_TO_SHORTPTR(b8); + *sum = 0; + *sse = 0; + + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + const int diff = a[j] - b[j]; + *sum += diff; + *sse += diff * diff; + } + a += a_stride; + b += b_stride; + } +} + +uint64_t aom_highbd_sse_odd_size(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, int w, int h) { + uint64_t sse; + int64_t sum; + highbd_variance64(a, a_stride, b, b_stride, w, h, &sse, &sum); + return sse; +} + +static void highbd_8_variance(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, int w, int h, + uint32_t *sse, int *sum) { + uint64_t sse_long = 0; + int64_t sum_long = 0; + highbd_variance64(a8, a_stride, b8, b_stride, w, h, &sse_long, &sum_long); + *sse = (uint32_t)sse_long; + *sum = (int)sum_long; +} + +static void highbd_10_variance(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, int w, int h, + uint32_t *sse, int *sum) { + uint64_t sse_long = 0; + int64_t sum_long = 0; + highbd_variance64(a8, a_stride, b8, b_stride, w, h, &sse_long, &sum_long); + *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 4); + *sum = (int)ROUND_POWER_OF_TWO(sum_long, 2); +} + +static void highbd_12_variance(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, int w, int h, + uint32_t *sse, int *sum) { + uint64_t sse_long = 0; + int64_t sum_long = 0; + highbd_variance64(a8, a_stride, b8, b_stride, w, h, &sse_long, &sum_long); + *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 8); + *sum = (int)ROUND_POWER_OF_TWO(sum_long, 4); +} + +#define HIGHBD_VAR(W, H) \ + uint32_t aom_highbd_8_variance##W##x##H##_c(const uint8_t *a, int a_stride, \ + const uint8_t *b, int b_stride, \ + uint32_t *sse) { \ + int sum; \ + highbd_8_variance(a, a_stride, b, b_stride, W, H, sse, &sum); \ + return *sse - (uint32_t)(((int64_t)sum * sum) / (W * H)); \ + } \ + \ + uint32_t aom_highbd_10_variance##W##x##H##_c(const uint8_t *a, int a_stride, \ + const uint8_t *b, int b_stride, \ + uint32_t *sse) { \ + int sum; \ + int64_t var; \ + highbd_10_variance(a, a_stride, b, b_stride, W, H, sse, &sum); \ + var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } \ + \ + uint32_t aom_highbd_12_variance##W##x##H##_c(const uint8_t *a, int a_stride, \ + const uint8_t *b, int b_stride, \ + uint32_t *sse) { \ + int sum; \ + int64_t var; \ + highbd_12_variance(a, a_stride, b, b_stride, W, H, sse, &sum); \ + var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } + +#define HIGHBD_GET_VAR(S) \ + void aom_highbd_8_get##S##x##S##var_c(const uint8_t *src, int src_stride, \ + const uint8_t *ref, int ref_stride, \ + uint32_t *sse, int *sum) { \ + highbd_8_variance(src, src_stride, ref, ref_stride, S, S, sse, sum); \ + } \ + \ + void aom_highbd_10_get##S##x##S##var_c(const uint8_t *src, int src_stride, \ + const uint8_t *ref, int ref_stride, \ + uint32_t *sse, int *sum) { \ + highbd_10_variance(src, src_stride, ref, ref_stride, S, S, sse, sum); \ + } \ + \ + void aom_highbd_12_get##S##x##S##var_c(const uint8_t *src, int src_stride, \ + const uint8_t *ref, int ref_stride, \ + uint32_t *sse, int *sum) { \ + highbd_12_variance(src, src_stride, ref, ref_stride, S, S, sse, sum); \ + } + +#define HIGHBD_MSE(W, H) \ + uint32_t aom_highbd_8_mse##W##x##H##_c(const uint8_t *src, int src_stride, \ + const uint8_t *ref, int ref_stride, \ + uint32_t *sse) { \ + int sum; \ + highbd_8_variance(src, src_stride, ref, ref_stride, W, H, sse, &sum); \ + return *sse; \ + } \ + \ + uint32_t aom_highbd_10_mse##W##x##H##_c(const uint8_t *src, int src_stride, \ + const uint8_t *ref, int ref_stride, \ + uint32_t *sse) { \ + int sum; \ + highbd_10_variance(src, src_stride, ref, ref_stride, W, H, sse, &sum); \ + return *sse; \ + } \ + \ + uint32_t aom_highbd_12_mse##W##x##H##_c(const uint8_t *src, int src_stride, \ + const uint8_t *ref, int ref_stride, \ + uint32_t *sse) { \ + int sum; \ + highbd_12_variance(src, src_stride, ref, ref_stride, W, H, sse, &sum); \ + return *sse; \ + } + +void aom_highbd_var_filter_block2d_bil_first_pass( + const uint8_t *src_ptr8, uint16_t *output_ptr, + unsigned int src_pixels_per_line, int pixel_step, + unsigned int output_height, unsigned int output_width, + const uint8_t *filter) { + unsigned int i, j; + uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src_ptr8); + for (i = 0; i < output_height; ++i) { + for (j = 0; j < output_width; ++j) { + output_ptr[j] = ROUND_POWER_OF_TWO( + (int)src_ptr[0] * filter[0] + (int)src_ptr[pixel_step] * filter[1], + FILTER_BITS); + + ++src_ptr; + } + + // Next row... + src_ptr += src_pixels_per_line - output_width; + output_ptr += output_width; + } +} + +void aom_highbd_var_filter_block2d_bil_second_pass( + const uint16_t *src_ptr, uint16_t *output_ptr, + unsigned int src_pixels_per_line, unsigned int pixel_step, + unsigned int output_height, unsigned int output_width, + const uint8_t *filter) { + unsigned int i, j; + + for (i = 0; i < output_height; ++i) { + for (j = 0; j < output_width; ++j) { + output_ptr[j] = ROUND_POWER_OF_TWO( + (int)src_ptr[0] * filter[0] + (int)src_ptr[pixel_step] * filter[1], + FILTER_BITS); + ++src_ptr; + } + + src_ptr += src_pixels_per_line - output_width; + output_ptr += output_width; + } +} + +#define HIGHBD_SUBPIX_VAR(W, H) \ + uint32_t aom_highbd_8_sub_pixel_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, uint32_t *sse) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + return aom_highbd_8_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp2), W, \ + dst, dst_stride, sse); \ + } \ + \ + uint32_t aom_highbd_10_sub_pixel_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, uint32_t *sse) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + return aom_highbd_10_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp2), W, \ + dst, dst_stride, sse); \ + } \ + \ + uint32_t aom_highbd_12_sub_pixel_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, uint32_t *sse) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + return aom_highbd_12_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp2), W, \ + dst, dst_stride, sse); \ + } + +#define HIGHBD_SUBPIX_AVG_VAR(W, H) \ + uint32_t aom_highbd_8_sub_pixel_avg_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, uint32_t *sse, \ + const uint8_t *second_pred) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + aom_highbd_comp_avg_pred_c(temp3, second_pred, W, H, \ + CONVERT_TO_BYTEPTR(temp2), W); \ + \ + return aom_highbd_8_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \ + dst, dst_stride, sse); \ + } \ + \ + uint32_t aom_highbd_10_sub_pixel_avg_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, uint32_t *sse, \ + const uint8_t *second_pred) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + aom_highbd_comp_avg_pred_c(temp3, second_pred, W, H, \ + CONVERT_TO_BYTEPTR(temp2), W); \ + \ + return aom_highbd_10_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \ + dst, dst_stride, sse); \ + } \ + \ + uint32_t aom_highbd_12_sub_pixel_avg_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, uint32_t *sse, \ + const uint8_t *second_pred) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + aom_highbd_comp_avg_pred_c(temp3, second_pred, W, H, \ + CONVERT_TO_BYTEPTR(temp2), W); \ + \ + return aom_highbd_12_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \ + dst, dst_stride, sse); \ + } + +/* All three forms of the variance are available in the same sizes. */ +#define HIGHBD_VARIANCES(W, H) \ + HIGHBD_VAR(W, H) \ + HIGHBD_SUBPIX_VAR(W, H) \ + HIGHBD_SUBPIX_AVG_VAR(W, H) + +#if CONFIG_AV1 && CONFIG_EXT_PARTITION +HIGHBD_VARIANCES(128, 128) +HIGHBD_VARIANCES(128, 64) +HIGHBD_VARIANCES(64, 128) +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION +HIGHBD_VARIANCES(64, 64) +HIGHBD_VARIANCES(64, 32) +HIGHBD_VARIANCES(32, 64) +HIGHBD_VARIANCES(32, 32) +HIGHBD_VARIANCES(32, 16) +HIGHBD_VARIANCES(16, 32) +HIGHBD_VARIANCES(16, 16) +HIGHBD_VARIANCES(16, 8) +HIGHBD_VARIANCES(8, 16) +HIGHBD_VARIANCES(8, 8) +HIGHBD_VARIANCES(8, 4) +HIGHBD_VARIANCES(4, 8) +HIGHBD_VARIANCES(4, 4) +HIGHBD_VARIANCES(4, 2) +HIGHBD_VARIANCES(2, 4) +HIGHBD_VARIANCES(2, 2) + +HIGHBD_GET_VAR(8) +HIGHBD_GET_VAR(16) + +HIGHBD_MSE(16, 16) +HIGHBD_MSE(16, 8) +HIGHBD_MSE(8, 16) +HIGHBD_MSE(8, 8) + +void aom_highbd_comp_avg_pred_c(uint16_t *comp_pred, const uint8_t *pred8, + int width, int height, const uint8_t *ref8, + int ref_stride) { + int i, j; + uint16_t *pred = CONVERT_TO_SHORTPTR(pred8); + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const int tmp = pred[j] + ref[j]; + comp_pred[j] = ROUND_POWER_OF_TWO(tmp, 1); + } + comp_pred += width; + pred += width; + ref += ref_stride; + } +} + +void aom_highbd_upsampled_pred_c(uint16_t *comp_pred, int width, int height, + const uint8_t *ref8, int ref_stride) { + int i, j; + int stride = ref_stride << 3; + + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + comp_pred[j] = ref[(j << 3)]; + } + comp_pred += width; + ref += stride; + } +} + +void aom_highbd_comp_avg_upsampled_pred_c(uint16_t *comp_pred, + const uint8_t *pred8, int width, + int height, const uint8_t *ref8, + int ref_stride) { + int i, j; + int stride = ref_stride << 3; + + uint16_t *pred = CONVERT_TO_SHORTPTR(pred8); + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const int tmp = pred[j] + ref[(j << 3)]; + comp_pred[j] = ROUND_POWER_OF_TWO(tmp, 1); + } + comp_pred += width; + pred += width; + ref += stride; + } +} +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_AV1 && CONFIG_EXT_INTER +void masked_variance(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, const uint8_t *m, int m_stride, int w, int h, + unsigned int *sse, int *sum) { + int i, j; + + int64_t sum64 = 0; + uint64_t sse64 = 0; + + for (i = 0; i < h; i++) { + for (j = 0; j < w; j++) { + const int diff = (a[j] - b[j]) * (m[j]); + sum64 += diff; + sse64 += diff * diff; + } + + a += a_stride; + b += b_stride; + m += m_stride; + } + sum64 = (sum64 >= 0) ? sum64 : -sum64; + *sum = (int)ROUND_POWER_OF_TWO(sum64, 6); + *sse = (uint32_t)ROUND_POWER_OF_TWO(sse64, 12); +} + +#define MASK_VAR(W, H) \ + unsigned int aom_masked_variance##W##x##H##_c( \ + const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, \ + const uint8_t *m, int m_stride, unsigned int *sse) { \ + int sum; \ + masked_variance(a, a_stride, b, b_stride, m, m_stride, W, H, sse, &sum); \ + return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \ + } + +#define MASK_SUBPIX_VAR(W, H) \ + unsigned int aom_masked_sub_pixel_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, const uint8_t *msk, int msk_stride, \ + unsigned int *sse) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint8_t temp2[H * W]; \ + \ + var_filter_block2d_bil_first_pass(src, fdata3, src_stride, 1, H + 1, W, \ + bilinear_filters_2t[xoffset]); \ + var_filter_block2d_bil_second_pass(fdata3, temp2, W, W, H, W, \ + bilinear_filters_2t[yoffset]); \ + \ + return aom_masked_variance##W##x##H##_c(temp2, W, dst, dst_stride, msk, \ + msk_stride, sse); \ + } + +MASK_VAR(4, 4) +MASK_SUBPIX_VAR(4, 4) + +MASK_VAR(4, 8) +MASK_SUBPIX_VAR(4, 8) + +MASK_VAR(8, 4) +MASK_SUBPIX_VAR(8, 4) + +MASK_VAR(8, 8) +MASK_SUBPIX_VAR(8, 8) + +MASK_VAR(8, 16) +MASK_SUBPIX_VAR(8, 16) + +MASK_VAR(16, 8) +MASK_SUBPIX_VAR(16, 8) + +MASK_VAR(16, 16) +MASK_SUBPIX_VAR(16, 16) + +MASK_VAR(16, 32) +MASK_SUBPIX_VAR(16, 32) + +MASK_VAR(32, 16) +MASK_SUBPIX_VAR(32, 16) + +MASK_VAR(32, 32) +MASK_SUBPIX_VAR(32, 32) + +MASK_VAR(32, 64) +MASK_SUBPIX_VAR(32, 64) + +MASK_VAR(64, 32) +MASK_SUBPIX_VAR(64, 32) + +MASK_VAR(64, 64) +MASK_SUBPIX_VAR(64, 64) + +#if CONFIG_EXT_PARTITION +MASK_VAR(64, 128) +MASK_SUBPIX_VAR(64, 128) + +MASK_VAR(128, 64) +MASK_SUBPIX_VAR(128, 64) + +MASK_VAR(128, 128) +MASK_SUBPIX_VAR(128, 128) +#endif // CONFIG_EXT_PARTITION + +#if CONFIG_HIGHBITDEPTH +void highbd_masked_variance64(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, const uint8_t *m, + int m_stride, int w, int h, uint64_t *sse, + int64_t *sum) { + int i, j; + uint16_t *a = CONVERT_TO_SHORTPTR(a8); + uint16_t *b = CONVERT_TO_SHORTPTR(b8); + + *sum = 0; + *sse = 0; + + for (i = 0; i < h; i++) { + for (j = 0; j < w; j++) { + const int diff = (a[j] - b[j]) * (m[j]); + *sum += (int64_t)diff; + *sse += (int64_t)diff * diff; + } + + a += a_stride; + b += b_stride; + m += m_stride; + } + *sum = (*sum >= 0) ? *sum : -*sum; + *sum = ROUND_POWER_OF_TWO(*sum, 6); + *sse = ROUND_POWER_OF_TWO(*sse, 12); +} + +void highbd_masked_variance(const uint8_t *a8, int a_stride, const uint8_t *b8, + int b_stride, const uint8_t *m, int m_stride, int w, + int h, unsigned int *sse, int *sum) { + int64_t sum64; + uint64_t sse64; + highbd_masked_variance64(a8, a_stride, b8, b_stride, m, m_stride, w, h, + &sse64, &sum64); + *sum = (int)sum64; + *sse = (unsigned int)sse64; +} + +void highbd_10_masked_variance(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, + const uint8_t *m, int m_stride, int w, int h, + unsigned int *sse, int *sum) { + int64_t sum64; + uint64_t sse64; + highbd_masked_variance64(a8, a_stride, b8, b_stride, m, m_stride, w, h, + &sse64, &sum64); + *sum = (int)ROUND_POWER_OF_TWO(sum64, 2); + *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 4); +} + +void highbd_12_masked_variance(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, + const uint8_t *m, int m_stride, int w, int h, + unsigned int *sse, int *sum) { + int64_t sum64; + uint64_t sse64; + highbd_masked_variance64(a8, a_stride, b8, b_stride, m, m_stride, w, h, + &sse64, &sum64); + *sum = (int)ROUND_POWER_OF_TWO(sum64, 4); + *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 8); +} + +#define HIGHBD_MASK_VAR(W, H) \ + unsigned int aom_highbd_masked_variance##W##x##H##_c( \ + const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, \ + const uint8_t *m, int m_stride, unsigned int *sse) { \ + int sum; \ + highbd_masked_variance(a, a_stride, b, b_stride, m, m_stride, W, H, sse, \ + &sum); \ + return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \ + } \ + \ + unsigned int aom_highbd_10_masked_variance##W##x##H##_c( \ + const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, \ + const uint8_t *m, int m_stride, unsigned int *sse) { \ + int sum; \ + int64_t var; \ + highbd_10_masked_variance(a, a_stride, b, b_stride, m, m_stride, W, H, \ + sse, &sum); \ + var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } \ + \ + unsigned int aom_highbd_12_masked_variance##W##x##H##_c( \ + const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, \ + const uint8_t *m, int m_stride, unsigned int *sse) { \ + int sum; \ + int64_t var; \ + highbd_12_masked_variance(a, a_stride, b, b_stride, m, m_stride, W, H, \ + sse, &sum); \ + var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } + +#define HIGHBD_MASK_SUBPIX_VAR(W, H) \ + unsigned int aom_highbd_masked_sub_pixel_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, const uint8_t *msk, int msk_stride, \ + unsigned int *sse) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + return aom_highbd_masked_variance##W##x##H##_c( \ + CONVERT_TO_BYTEPTR(temp2), W, dst, dst_stride, msk, msk_stride, sse); \ + } \ + \ + unsigned int aom_highbd_10_masked_sub_pixel_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, const uint8_t *msk, int msk_stride, \ + unsigned int *sse) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + return aom_highbd_10_masked_variance##W##x##H##_c( \ + CONVERT_TO_BYTEPTR(temp2), W, dst, dst_stride, msk, msk_stride, sse); \ + } \ + \ + unsigned int aom_highbd_12_masked_sub_pixel_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, const uint8_t *msk, int msk_stride, \ + unsigned int *sse) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + return aom_highbd_12_masked_variance##W##x##H##_c( \ + CONVERT_TO_BYTEPTR(temp2), W, dst, dst_stride, msk, msk_stride, sse); \ + } + +HIGHBD_MASK_VAR(4, 4) +HIGHBD_MASK_SUBPIX_VAR(4, 4) + +HIGHBD_MASK_VAR(4, 8) +HIGHBD_MASK_SUBPIX_VAR(4, 8) + +HIGHBD_MASK_VAR(8, 4) +HIGHBD_MASK_SUBPIX_VAR(8, 4) + +HIGHBD_MASK_VAR(8, 8) +HIGHBD_MASK_SUBPIX_VAR(8, 8) + +HIGHBD_MASK_VAR(8, 16) +HIGHBD_MASK_SUBPIX_VAR(8, 16) + +HIGHBD_MASK_VAR(16, 8) +HIGHBD_MASK_SUBPIX_VAR(16, 8) + +HIGHBD_MASK_VAR(16, 16) +HIGHBD_MASK_SUBPIX_VAR(16, 16) + +HIGHBD_MASK_VAR(16, 32) +HIGHBD_MASK_SUBPIX_VAR(16, 32) + +HIGHBD_MASK_VAR(32, 16) +HIGHBD_MASK_SUBPIX_VAR(32, 16) + +HIGHBD_MASK_VAR(32, 32) +HIGHBD_MASK_SUBPIX_VAR(32, 32) + +HIGHBD_MASK_VAR(32, 64) +HIGHBD_MASK_SUBPIX_VAR(32, 64) + +HIGHBD_MASK_VAR(64, 32) +HIGHBD_MASK_SUBPIX_VAR(64, 32) + +HIGHBD_MASK_VAR(64, 64) +HIGHBD_MASK_SUBPIX_VAR(64, 64) + +#if CONFIG_EXT_PARTITION +HIGHBD_MASK_VAR(64, 128) +HIGHBD_MASK_SUBPIX_VAR(64, 128) + +HIGHBD_MASK_VAR(128, 64) +HIGHBD_MASK_SUBPIX_VAR(128, 64) + +HIGHBD_MASK_VAR(128, 128) +HIGHBD_MASK_SUBPIX_VAR(128, 128) +#endif // CONFIG_EXT_PARTITION +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_AV1 && CONFIG_EXT_INTER + +#if CONFIG_AV1 && CONFIG_MOTION_VAR +static INLINE void obmc_variance(const uint8_t *pre, int pre_stride, + const int32_t *wsrc, const int32_t *mask, + int w, int h, unsigned int *sse, int *sum) { + int i, j; + + *sse = 0; + *sum = 0; + + for (i = 0; i < h; i++) { + for (j = 0; j < w; j++) { + int diff = ROUND_POWER_OF_TWO_SIGNED(wsrc[j] - pre[j] * mask[j], 12); + *sum += diff; + *sse += diff * diff; + } + + pre += pre_stride; + wsrc += w; + mask += w; + } +} + +#define OBMC_VAR(W, H) \ + unsigned int aom_obmc_variance##W##x##H##_c( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask, unsigned int *sse) { \ + int sum; \ + obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ + return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \ + } + +#define OBMC_SUBPIX_VAR(W, H) \ + unsigned int aom_obmc_sub_pixel_variance##W##x##H##_c( \ + const uint8_t *pre, int pre_stride, int xoffset, int yoffset, \ + const int32_t *wsrc, const int32_t *mask, unsigned int *sse) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint8_t temp2[H * W]; \ + \ + var_filter_block2d_bil_first_pass(pre, fdata3, pre_stride, 1, H + 1, W, \ + bilinear_filters_2t[xoffset]); \ + var_filter_block2d_bil_second_pass(fdata3, temp2, W, W, H, W, \ + bilinear_filters_2t[yoffset]); \ + \ + return aom_obmc_variance##W##x##H##_c(temp2, W, wsrc, mask, sse); \ + } + +OBMC_VAR(4, 4) +OBMC_SUBPIX_VAR(4, 4) + +OBMC_VAR(4, 8) +OBMC_SUBPIX_VAR(4, 8) + +OBMC_VAR(8, 4) +OBMC_SUBPIX_VAR(8, 4) + +OBMC_VAR(8, 8) +OBMC_SUBPIX_VAR(8, 8) + +OBMC_VAR(8, 16) +OBMC_SUBPIX_VAR(8, 16) + +OBMC_VAR(16, 8) +OBMC_SUBPIX_VAR(16, 8) + +OBMC_VAR(16, 16) +OBMC_SUBPIX_VAR(16, 16) + +OBMC_VAR(16, 32) +OBMC_SUBPIX_VAR(16, 32) + +OBMC_VAR(32, 16) +OBMC_SUBPIX_VAR(32, 16) + +OBMC_VAR(32, 32) +OBMC_SUBPIX_VAR(32, 32) + +OBMC_VAR(32, 64) +OBMC_SUBPIX_VAR(32, 64) + +OBMC_VAR(64, 32) +OBMC_SUBPIX_VAR(64, 32) + +OBMC_VAR(64, 64) +OBMC_SUBPIX_VAR(64, 64) + +#if CONFIG_EXT_PARTITION +OBMC_VAR(64, 128) +OBMC_SUBPIX_VAR(64, 128) + +OBMC_VAR(128, 64) +OBMC_SUBPIX_VAR(128, 64) + +OBMC_VAR(128, 128) +OBMC_SUBPIX_VAR(128, 128) +#endif // CONFIG_EXT_PARTITION + +#if CONFIG_HIGHBITDEPTH +static INLINE void highbd_obmc_variance64(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, int w, int h, + uint64_t *sse, int64_t *sum) { + int i, j; + uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); + + *sse = 0; + *sum = 0; + + for (i = 0; i < h; i++) { + for (j = 0; j < w; j++) { + int diff = ROUND_POWER_OF_TWO_SIGNED(wsrc[j] - pre[j] * mask[j], 12); + *sum += diff; + *sse += diff * diff; + } + + pre += pre_stride; + wsrc += w; + mask += w; + } +} + +static INLINE void highbd_obmc_variance(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, int w, int h, + unsigned int *sse, int *sum) { + int64_t sum64; + uint64_t sse64; + highbd_obmc_variance64(pre8, pre_stride, wsrc, mask, w, h, &sse64, &sum64); + *sum = (int)sum64; + *sse = (unsigned int)sse64; +} + +static INLINE void highbd_10_obmc_variance(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, int w, int h, + unsigned int *sse, int *sum) { + int64_t sum64; + uint64_t sse64; + highbd_obmc_variance64(pre8, pre_stride, wsrc, mask, w, h, &sse64, &sum64); + *sum = (int)ROUND_POWER_OF_TWO(sum64, 2); + *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 4); +} + +static INLINE void highbd_12_obmc_variance(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, int w, int h, + unsigned int *sse, int *sum) { + int64_t sum64; + uint64_t sse64; + highbd_obmc_variance64(pre8, pre_stride, wsrc, mask, w, h, &sse64, &sum64); + *sum = (int)ROUND_POWER_OF_TWO(sum64, 4); + *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 8); +} + +#define HIGHBD_OBMC_VAR(W, H) \ + unsigned int aom_highbd_obmc_variance##W##x##H##_c( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask, unsigned int *sse) { \ + int sum; \ + highbd_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ + return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \ + } \ + \ + unsigned int aom_highbd_10_obmc_variance##W##x##H##_c( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask, unsigned int *sse) { \ + int sum; \ + int64_t var; \ + highbd_10_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ + var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } \ + \ + unsigned int aom_highbd_12_obmc_variance##W##x##H##_c( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask, unsigned int *sse) { \ + int sum; \ + int64_t var; \ + highbd_12_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ + var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } + +#define HIGHBD_OBMC_SUBPIX_VAR(W, H) \ + unsigned int aom_highbd_obmc_sub_pixel_variance##W##x##H##_c( \ + const uint8_t *pre, int pre_stride, int xoffset, int yoffset, \ + const int32_t *wsrc, const int32_t *mask, unsigned int *sse) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + pre, fdata3, pre_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + return aom_highbd_obmc_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp2), W, \ + wsrc, mask, sse); \ + } \ + \ + unsigned int aom_highbd_10_obmc_sub_pixel_variance##W##x##H##_c( \ + const uint8_t *pre, int pre_stride, int xoffset, int yoffset, \ + const int32_t *wsrc, const int32_t *mask, unsigned int *sse) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + pre, fdata3, pre_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + return aom_highbd_10_obmc_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp2), \ + W, wsrc, mask, sse); \ + } \ + \ + unsigned int aom_highbd_12_obmc_sub_pixel_variance##W##x##H##_c( \ + const uint8_t *pre, int pre_stride, int xoffset, int yoffset, \ + const int32_t *wsrc, const int32_t *mask, unsigned int *sse) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + pre, fdata3, pre_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + return aom_highbd_12_obmc_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp2), \ + W, wsrc, mask, sse); \ + } + +HIGHBD_OBMC_VAR(4, 4) +HIGHBD_OBMC_SUBPIX_VAR(4, 4) + +HIGHBD_OBMC_VAR(4, 8) +HIGHBD_OBMC_SUBPIX_VAR(4, 8) + +HIGHBD_OBMC_VAR(8, 4) +HIGHBD_OBMC_SUBPIX_VAR(8, 4) + +HIGHBD_OBMC_VAR(8, 8) +HIGHBD_OBMC_SUBPIX_VAR(8, 8) + +HIGHBD_OBMC_VAR(8, 16) +HIGHBD_OBMC_SUBPIX_VAR(8, 16) + +HIGHBD_OBMC_VAR(16, 8) +HIGHBD_OBMC_SUBPIX_VAR(16, 8) + +HIGHBD_OBMC_VAR(16, 16) +HIGHBD_OBMC_SUBPIX_VAR(16, 16) + +HIGHBD_OBMC_VAR(16, 32) +HIGHBD_OBMC_SUBPIX_VAR(16, 32) + +HIGHBD_OBMC_VAR(32, 16) +HIGHBD_OBMC_SUBPIX_VAR(32, 16) + +HIGHBD_OBMC_VAR(32, 32) +HIGHBD_OBMC_SUBPIX_VAR(32, 32) + +HIGHBD_OBMC_VAR(32, 64) +HIGHBD_OBMC_SUBPIX_VAR(32, 64) + +HIGHBD_OBMC_VAR(64, 32) +HIGHBD_OBMC_SUBPIX_VAR(64, 32) + +HIGHBD_OBMC_VAR(64, 64) +HIGHBD_OBMC_SUBPIX_VAR(64, 64) + +#if CONFIG_EXT_PARTITION +HIGHBD_OBMC_VAR(64, 128) +HIGHBD_OBMC_SUBPIX_VAR(64, 128) + +HIGHBD_OBMC_VAR(128, 64) +HIGHBD_OBMC_SUBPIX_VAR(128, 64) + +HIGHBD_OBMC_VAR(128, 128) +HIGHBD_OBMC_SUBPIX_VAR(128, 128) +#endif // CONFIG_EXT_PARTITION +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_AV1 && CONFIG_MOTION_VAR diff --git a/third_party/aom/aom_dsp/variance.h b/third_party/aom/aom_dsp/variance.h new file mode 100644 index 0000000000..7c925cfac0 --- /dev/null +++ b/third_party/aom/aom_dsp/variance.h @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_VARIANCE_H_ +#define AOM_DSP_VARIANCE_H_ + +#include "./aom_config.h" + +#include "aom/aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define FILTER_BITS 7 +#define FILTER_WEIGHT 128 + +typedef unsigned int (*aom_sad_fn_t)(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride); + +typedef unsigned int (*aom_sad_avg_fn_t)(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + const uint8_t *second_pred); + +typedef void (*aom_copy32xn_fn_t)(const uint8_t *a, int a_stride, uint8_t *b, + int b_stride, int n); + +typedef void (*aom_sad_multi_fn_t)(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + unsigned int *sad_array); + +typedef void (*aom_sad_multi_d_fn_t)(const uint8_t *a, int a_stride, + const uint8_t *const b_array[], + int b_stride, unsigned int *sad_array); + +typedef unsigned int (*aom_variance_fn_t)(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + unsigned int *sse); + +typedef unsigned int (*aom_subpixvariance_fn_t)(const uint8_t *a, int a_stride, + int xoffset, int yoffset, + const uint8_t *b, int b_stride, + unsigned int *sse); + +typedef unsigned int (*aom_subp_avg_variance_fn_t)( + const uint8_t *a, int a_stride, int xoffset, int yoffset, const uint8_t *b, + int b_stride, unsigned int *sse, const uint8_t *second_pred); + +#if CONFIG_AV1 && CONFIG_EXT_INTER +typedef unsigned int (*aom_masked_sad_fn_t)(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *msk_ptr, + int msk_stride); +typedef unsigned int (*aom_masked_variance_fn_t)( + const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, + const uint8_t *msk, int msk_stride, unsigned int *sse); +typedef unsigned int (*aom_masked_subpixvariance_fn_t)( + const uint8_t *src, int src_stride, int xoffset, int yoffset, + const uint8_t *ref, int ref_stride, const uint8_t *msk, int msk_stride, + unsigned int *sse); +#endif // CONFIG_AV1 && CONFIG_EXT_INTER + +#if CONFIG_AV1 && CONFIG_MOTION_VAR +typedef unsigned int (*aom_obmc_sad_fn_t)(const uint8_t *pred, int pred_stride, + const int32_t *wsrc, + const int32_t *msk); +typedef unsigned int (*aom_obmc_variance_fn_t)(const uint8_t *pred, + int pred_stride, + const int32_t *wsrc, + const int32_t *msk, + unsigned int *sse); +typedef unsigned int (*aom_obmc_subpixvariance_fn_t)( + const uint8_t *pred, int pred_stride, int xoffset, int yoffset, + const int32_t *wsrc, const int32_t *msk, unsigned int *sse); +#endif // CONFIG_AV1 && CONFIG_MOTION_VAR + +#if CONFIG_AV1 +typedef struct aom_variance_vtable { + aom_sad_fn_t sdf; + aom_sad_avg_fn_t sdaf; + aom_variance_fn_t vf; + aom_subpixvariance_fn_t svf; + aom_subp_avg_variance_fn_t svaf; + aom_sad_multi_fn_t sdx3f; + aom_sad_multi_fn_t sdx8f; + aom_sad_multi_d_fn_t sdx4df; +#if CONFIG_EXT_INTER + aom_masked_sad_fn_t msdf; + aom_masked_variance_fn_t mvf; + aom_masked_subpixvariance_fn_t msvf; +#endif // CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR + aom_obmc_sad_fn_t osdf; + aom_obmc_variance_fn_t ovf; + aom_obmc_subpixvariance_fn_t osvf; +#endif // CONFIG_MOTION_VAR +} aom_variance_fn_ptr_t; +#endif // CONFIG_AV1 + +void aom_highbd_var_filter_block2d_bil_first_pass( + const uint8_t *src_ptr8, uint16_t *output_ptr, + unsigned int src_pixels_per_line, int pixel_step, + unsigned int output_height, unsigned int output_width, + const uint8_t *filter); + +void aom_highbd_var_filter_block2d_bil_second_pass( + const uint16_t *src_ptr, uint16_t *output_ptr, + unsigned int src_pixels_per_line, unsigned int pixel_step, + unsigned int output_height, unsigned int output_width, + const uint8_t *filter); + +uint32_t aom_sse_odd_size(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int w, int h); + +#if CONFIG_HIGHBITDEPTH +uint64_t aom_highbd_sse_odd_size(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, int w, int h); +#endif // CONFIG_HIGHBITDEPTH + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_VARIANCE_H_ diff --git a/third_party/aom/aom_dsp/x86/aom_asm_stubs.c b/third_party/aom/aom_dsp/x86/aom_asm_stubs.c new file mode 100644 index 0000000000..4067b0b531 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/aom_asm_stubs.c @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/x86/convolve.h" + +#if HAVE_SSE2 +filter8_1dfunction aom_filter_block1d16_v8_sse2; +filter8_1dfunction aom_filter_block1d16_h8_sse2; +filter8_1dfunction aom_filter_block1d8_v8_sse2; +filter8_1dfunction aom_filter_block1d8_h8_sse2; +filter8_1dfunction aom_filter_block1d4_v8_sse2; +filter8_1dfunction aom_filter_block1d4_h8_sse2; +filter8_1dfunction aom_filter_block1d16_v8_avg_sse2; +filter8_1dfunction aom_filter_block1d16_h8_avg_sse2; +filter8_1dfunction aom_filter_block1d8_v8_avg_sse2; +filter8_1dfunction aom_filter_block1d8_h8_avg_sse2; +filter8_1dfunction aom_filter_block1d4_v8_avg_sse2; +filter8_1dfunction aom_filter_block1d4_h8_avg_sse2; + +filter8_1dfunction aom_filter_block1d16_v2_sse2; +filter8_1dfunction aom_filter_block1d16_h2_sse2; +filter8_1dfunction aom_filter_block1d8_v2_sse2; +filter8_1dfunction aom_filter_block1d8_h2_sse2; +filter8_1dfunction aom_filter_block1d4_v2_sse2; +filter8_1dfunction aom_filter_block1d4_h2_sse2; +filter8_1dfunction aom_filter_block1d16_v2_avg_sse2; +filter8_1dfunction aom_filter_block1d16_h2_avg_sse2; +filter8_1dfunction aom_filter_block1d8_v2_avg_sse2; +filter8_1dfunction aom_filter_block1d8_h2_avg_sse2; +filter8_1dfunction aom_filter_block1d4_v2_avg_sse2; +filter8_1dfunction aom_filter_block1d4_h2_avg_sse2; + +// void aom_convolve8_horiz_sse2(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +// void aom_convolve8_vert_sse2(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +// void aom_convolve8_avg_horiz_sse2(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +// void aom_convolve8_avg_vert_sse2(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , sse2); +FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , sse2); +FUN_CONV_1D(avg_horiz, x_step_q4, filter_x, h, src, avg_, sse2); +FUN_CONV_1D(avg_vert, y_step_q4, filter_y, v, src - src_stride * 3, avg_, sse2); + +// void aom_convolve8_sse2(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +// void aom_convolve8_avg_sse2(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +FUN_CONV_2D(, sse2); +FUN_CONV_2D(avg_, sse2); + +#if CONFIG_HIGHBITDEPTH && ARCH_X86_64 +highbd_filter8_1dfunction aom_highbd_filter_block1d16_v8_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d16_h8_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d8_v8_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d8_h8_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d4_v8_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d4_h8_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d16_v8_avg_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d16_h8_avg_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d8_v8_avg_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d8_h8_avg_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d4_v8_avg_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d4_h8_avg_sse2; + +highbd_filter8_1dfunction aom_highbd_filter_block1d16_v2_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d16_h2_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d8_v2_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d8_h2_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d4_v2_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d4_h2_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d16_v2_avg_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d16_h2_avg_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d8_v2_avg_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d8_h2_avg_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d4_v2_avg_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d4_h2_avg_sse2; + +// void aom_highbd_convolve8_horiz_sse2(const uint8_t *src, +// ptrdiff_t src_stride, +// uint8_t *dst, +// ptrdiff_t dst_stride, +// const int16_t *filter_x, +// int x_step_q4, +// const int16_t *filter_y, +// int y_step_q4, +// int w, int h, int bd); +// void aom_highbd_convolve8_vert_sse2(const uint8_t *src, +// ptrdiff_t src_stride, +// uint8_t *dst, +// ptrdiff_t dst_stride, +// const int16_t *filter_x, +// int x_step_q4, +// const int16_t *filter_y, +// int y_step_q4, +// int w, int h, int bd); +// void aom_highbd_convolve8_avg_horiz_sse2(const uint8_t *src, +// ptrdiff_t src_stride, +// uint8_t *dst, +// ptrdiff_t dst_stride, +// const int16_t *filter_x, +// int x_step_q4, +// const int16_t *filter_y, +// int y_step_q4, +// int w, int h, int bd); +// void aom_highbd_convolve8_avg_vert_sse2(const uint8_t *src, +// ptrdiff_t src_stride, +// uint8_t *dst, +// ptrdiff_t dst_stride, +// const int16_t *filter_x, +// int x_step_q4, +// const int16_t *filter_y, +// int y_step_q4, +// int w, int h, int bd); +HIGH_FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , sse2); +HIGH_FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , sse2); +HIGH_FUN_CONV_1D(avg_horiz, x_step_q4, filter_x, h, src, avg_, sse2); +HIGH_FUN_CONV_1D(avg_vert, y_step_q4, filter_y, v, src - src_stride * 3, avg_, + sse2); + +// void aom_highbd_convolve8_sse2(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h, int bd); +// void aom_highbd_convolve8_avg_sse2(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h, int bd); +HIGH_FUN_CONV_2D(, sse2); +HIGH_FUN_CONV_2D(avg_, sse2); + +#if CONFIG_LOOP_RESTORATION +// The SSE2 highbd convolve functions can deal with coefficients up to 32767. +// So redirect highbd_convolve8_add_src to regular highbd_convolve8. +void aom_highbd_convolve8_add_src_sse2(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h, int bd) { + assert(x_step_q4 == 16); + assert(y_step_q4 == 16); + ((int16_t *)filter_x)[3] += 128; + ((int16_t *)filter_y)[3] += 128; + aom_highbd_convolve8_sse2(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, filter_y, y_step_q4, w, h, bd); + ((int16_t *)filter_x)[3] -= 128; + ((int16_t *)filter_y)[3] -= 128; +} +#endif // CONFIG_LOOP_RESTORATION +#endif // CONFIG_HIGHBITDEPTH && ARCH_X86_64 +#endif // HAVE_SSE2 diff --git a/third_party/aom/aom_dsp/x86/aom_convolve_copy_sse2.asm b/third_party/aom/aom_dsp/x86/aom_convolve_copy_sse2.asm new file mode 100644 index 0000000000..4d31428671 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/aom_convolve_copy_sse2.asm @@ -0,0 +1,345 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION .text + +%macro convolve_fn 1-2 +%ifidn %1, avg +%define AUX_XMM_REGS 4 +%else +%define AUX_XMM_REGS 0 +%endif +%ifidn %2, highbd +%define pavg pavgw +cglobal %2_convolve_%1, 4, 7, 4+AUX_XMM_REGS, src, src_stride, \ + dst, dst_stride, \ + fx, fxs, fy, fys, w, h, bd +%else +%define pavg pavgb +cglobal convolve_%1, 4, 7, 4+AUX_XMM_REGS, src, src_stride, \ + dst, dst_stride, \ + fx, fxs, fy, fys, w, h +%endif + mov r4d, dword wm +%ifidn %2, highbd + shl r4d, 1 + shl srcq, 1 + shl src_strideq, 1 + shl dstq, 1 + shl dst_strideq, 1 +%else + cmp r4d, 4 + je .w4 +%endif + cmp r4d, 8 + je .w8 + cmp r4d, 16 + je .w16 + cmp r4d, 32 + je .w32 + +%if CONFIG_AV1 && CONFIG_EXT_PARTITION + cmp r4d, 64 + je .w64 +%ifidn %2, highbd + cmp r4d, 128 + je .w128 + +.w256: + mov r4d, dword hm +.loop256: + movu m0, [srcq] + movu m1, [srcq+16] + movu m2, [srcq+32] + movu m3, [srcq+48] +%ifidn %1, avg + pavg m0, [dstq] + pavg m1, [dstq+16] + pavg m2, [dstq+32] + pavg m3, [dstq+48] +%endif + mova [dstq ], m0 + mova [dstq+16], m1 + mova [dstq+32], m2 + mova [dstq+48], m3 + movu m0, [srcq+64] + movu m1, [srcq+80] + movu m2, [srcq+96] + movu m3, [srcq+112] +%ifidn %1, avg + pavg m0, [dstq+64] + pavg m1, [dstq+80] + pavg m2, [dstq+96] + pavg m3, [dstq+112] +%endif + mova [dstq+64], m0 + mova [dstq+80], m1 + mova [dstq+96], m2 + mova [dstq+112], m3 + movu m0, [srcq+128] + movu m1, [srcq+128+16] + movu m2, [srcq+128+32] + movu m3, [srcq+128+48] +%ifidn %1, avg + pavg m0, [dstq+128] + pavg m1, [dstq+128+16] + pavg m2, [dstq+128+32] + pavg m3, [dstq+128+48] +%endif + mova [dstq+128 ], m0 + mova [dstq+128+16], m1 + mova [dstq+128+32], m2 + mova [dstq+128+48], m3 + movu m0, [srcq+128+64] + movu m1, [srcq+128+80] + movu m2, [srcq+128+96] + movu m3, [srcq+128+112] + add srcq, src_strideq +%ifidn %1, avg + pavg m0, [dstq+128+64] + pavg m1, [dstq+128+80] + pavg m2, [dstq+128+96] + pavg m3, [dstq+128+112] +%endif + mova [dstq+128+64], m0 + mova [dstq+128+80], m1 + mova [dstq+128+96], m2 + mova [dstq+128+112], m3 + add dstq, dst_strideq + sub r4d, 1 + jnz .loop256 + RET +%endif + +.w128: + mov r4d, dword hm +.loop128: + movu m0, [srcq] + movu m1, [srcq+16] + movu m2, [srcq+32] + movu m3, [srcq+48] +%ifidn %1, avg + pavg m0, [dstq] + pavg m1, [dstq+16] + pavg m2, [dstq+32] + pavg m3, [dstq+48] +%endif + mova [dstq ], m0 + mova [dstq+16], m1 + mova [dstq+32], m2 + mova [dstq+48], m3 + movu m0, [srcq+64] + movu m1, [srcq+80] + movu m2, [srcq+96] + movu m3, [srcq+112] + add srcq, src_strideq +%ifidn %1, avg + pavg m0, [dstq+64] + pavg m1, [dstq+80] + pavg m2, [dstq+96] + pavg m3, [dstq+112] +%endif + mova [dstq+64], m0 + mova [dstq+80], m1 + mova [dstq+96], m2 + mova [dstq+112], m3 + add dstq, dst_strideq + sub r4d, 1 + jnz .loop128 + RET + +%else ; CONFIG_AV1 && CONFIG_EXT_PARTITION + +%ifidn %2, highbd + cmp r4d, 64 + je .w64 + + mov r4d, dword hm +.loop128: + movu m0, [srcq] + movu m1, [srcq+16] + movu m2, [srcq+32] + movu m3, [srcq+48] +%ifidn %1, avg + pavg m0, [dstq] + pavg m1, [dstq+16] + pavg m2, [dstq+32] + pavg m3, [dstq+48] +%endif + mova [dstq ], m0 + mova [dstq+16], m1 + mova [dstq+32], m2 + mova [dstq+48], m3 + movu m0, [srcq+64] + movu m1, [srcq+80] + movu m2, [srcq+96] + movu m3, [srcq+112] + add srcq, src_strideq +%ifidn %1, avg + pavg m0, [dstq+64] + pavg m1, [dstq+80] + pavg m2, [dstq+96] + pavg m3, [dstq+112] +%endif + mova [dstq+64], m0 + mova [dstq+80], m1 + mova [dstq+96], m2 + mova [dstq+112], m3 + add dstq, dst_strideq + sub r4d, 1 + jnz .loop128 + RET +%endif +%endif ; CONFIG_AV1 && CONFIG_EXT_PARTITION + +.w64: + mov r4d, dword hm +.loop64: + movu m0, [srcq] + movu m1, [srcq+16] + movu m2, [srcq+32] + movu m3, [srcq+48] + add srcq, src_strideq +%ifidn %1, avg + pavg m0, [dstq] + pavg m1, [dstq+16] + pavg m2, [dstq+32] + pavg m3, [dstq+48] +%endif + mova [dstq ], m0 + mova [dstq+16], m1 + mova [dstq+32], m2 + mova [dstq+48], m3 + add dstq, dst_strideq + sub r4d, 1 + jnz .loop64 + RET + +.w32: + mov r4d, dword hm +.loop32: + movu m0, [srcq] + movu m1, [srcq+16] + movu m2, [srcq+src_strideq] + movu m3, [srcq+src_strideq+16] + lea srcq, [srcq+src_strideq*2] +%ifidn %1, avg + pavg m0, [dstq] + pavg m1, [dstq +16] + pavg m2, [dstq+dst_strideq] + pavg m3, [dstq+dst_strideq+16] +%endif + mova [dstq ], m0 + mova [dstq +16], m1 + mova [dstq+dst_strideq ], m2 + mova [dstq+dst_strideq+16], m3 + lea dstq, [dstq+dst_strideq*2] + sub r4d, 2 + jnz .loop32 + RET + +.w16: + mov r4d, dword hm + lea r5q, [src_strideq*3] + lea r6q, [dst_strideq*3] +.loop16: + movu m0, [srcq] + movu m1, [srcq+src_strideq] + movu m2, [srcq+src_strideq*2] + movu m3, [srcq+r5q] + lea srcq, [srcq+src_strideq*4] +%ifidn %1, avg + pavg m0, [dstq] + pavg m1, [dstq+dst_strideq] + pavg m2, [dstq+dst_strideq*2] + pavg m3, [dstq+r6q] +%endif + mova [dstq ], m0 + mova [dstq+dst_strideq ], m1 + mova [dstq+dst_strideq*2], m2 + mova [dstq+r6q ], m3 + lea dstq, [dstq+dst_strideq*4] + sub r4d, 4 + jnz .loop16 + RET + +.w8: + mov r4d, dword hm + lea r5q, [src_strideq*3] + lea r6q, [dst_strideq*3] +.loop8: + movh m0, [srcq] + movh m1, [srcq+src_strideq] + movh m2, [srcq+src_strideq*2] + movh m3, [srcq+r5q] + lea srcq, [srcq+src_strideq*4] +%ifidn %1, avg + movh m4, [dstq] + movh m5, [dstq+dst_strideq] + movh m6, [dstq+dst_strideq*2] + movh m7, [dstq+r6q] + pavg m0, m4 + pavg m1, m5 + pavg m2, m6 + pavg m3, m7 +%endif + movh [dstq ], m0 + movh [dstq+dst_strideq ], m1 + movh [dstq+dst_strideq*2], m2 + movh [dstq+r6q ], m3 + lea dstq, [dstq+dst_strideq*4] + sub r4d, 4 + jnz .loop8 + RET + +%ifnidn %2, highbd +.w4: + mov r4d, dword hm + lea r5q, [src_strideq*3] + lea r6q, [dst_strideq*3] +.loop4: + movd m0, [srcq] + movd m1, [srcq+src_strideq] + movd m2, [srcq+src_strideq*2] + movd m3, [srcq+r5q] + lea srcq, [srcq+src_strideq*4] +%ifidn %1, avg + movd m4, [dstq] + movd m5, [dstq+dst_strideq] + movd m6, [dstq+dst_strideq*2] + movd m7, [dstq+r6q] + pavg m0, m4 + pavg m1, m5 + pavg m2, m6 + pavg m3, m7 +%endif + movd [dstq ], m0 + movd [dstq+dst_strideq ], m1 + movd [dstq+dst_strideq*2], m2 + movd [dstq+r6q ], m3 + lea dstq, [dstq+dst_strideq*4] + sub r4d, 4 + jnz .loop4 + RET +%endif +%endmacro + +INIT_XMM sse2 +convolve_fn copy +convolve_fn avg +%if CONFIG_HIGHBITDEPTH +convolve_fn copy, highbd +convolve_fn avg, highbd +%endif diff --git a/third_party/aom/aom_dsp/x86/aom_high_subpixel_8t_sse2.asm b/third_party/aom/aom_dsp/x86/aom_high_subpixel_8t_sse2.asm new file mode 100644 index 0000000000..e6d357ba3f --- /dev/null +++ b/third_party/aom/aom_dsp/x86/aom_high_subpixel_8t_sse2.asm @@ -0,0 +1,965 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + +%include "aom_ports/x86_abi_support.asm" + +;Note: tap3 and tap4 have to be applied and added after other taps to avoid +;overflow. + +%macro HIGH_GET_FILTERS_4 0 + mov rdx, arg(5) ;filter ptr + mov rcx, 0x00000040 + + movdqa xmm7, [rdx] ;load filters + pshuflw xmm0, xmm7, 0b ;k0 + pshuflw xmm1, xmm7, 01010101b ;k1 + pshuflw xmm2, xmm7, 10101010b ;k2 + pshuflw xmm3, xmm7, 11111111b ;k3 + psrldq xmm7, 8 + pshuflw xmm4, xmm7, 0b ;k4 + pshuflw xmm5, xmm7, 01010101b ;k5 + pshuflw xmm6, xmm7, 10101010b ;k6 + pshuflw xmm7, xmm7, 11111111b ;k7 + + punpcklwd xmm0, xmm6 + punpcklwd xmm2, xmm5 + punpcklwd xmm3, xmm4 + punpcklwd xmm1, xmm7 + + movdqa k0k6, xmm0 + movdqa k2k5, xmm2 + movdqa k3k4, xmm3 + movdqa k1k7, xmm1 + + movq xmm6, rcx + pshufd xmm6, xmm6, 0 + movdqa krd, xmm6 + + ;Compute max and min values of a pixel + mov rdx, 0x00010001 + movsxd rcx, DWORD PTR arg(6) ;bps + movq xmm0, rdx + movq xmm1, rcx + pshufd xmm0, xmm0, 0b + movdqa xmm2, xmm0 + psllw xmm0, xmm1 + psubw xmm0, xmm2 + pxor xmm1, xmm1 + movdqa max, xmm0 ;max value (for clamping) + movdqa min, xmm1 ;min value (for clamping) + +%endm + +%macro HIGH_APPLY_FILTER_4 1 + punpcklwd xmm0, xmm6 ;two row in one register + punpcklwd xmm1, xmm7 + punpcklwd xmm2, xmm5 + punpcklwd xmm3, xmm4 + + pmaddwd xmm0, k0k6 ;multiply the filter factors + pmaddwd xmm1, k1k7 + pmaddwd xmm2, k2k5 + pmaddwd xmm3, k3k4 + + paddd xmm0, xmm1 ;sum + paddd xmm0, xmm2 + paddd xmm0, xmm3 + + paddd xmm0, krd ;rounding + psrad xmm0, 7 ;shift + packssdw xmm0, xmm0 ;pack to word + + ;clamp the values + pminsw xmm0, max + pmaxsw xmm0, min + +%if %1 + movq xmm1, [rdi] + pavgw xmm0, xmm1 +%endif + movq [rdi], xmm0 +%endm + +%macro HIGH_GET_FILTERS 0 + mov rdx, arg(5) ;filter ptr + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + mov rcx, 0x00000040 + + movdqa xmm7, [rdx] ;load filters + pshuflw xmm0, xmm7, 0b ;k0 + pshuflw xmm1, xmm7, 01010101b ;k1 + pshuflw xmm2, xmm7, 10101010b ;k2 + pshuflw xmm3, xmm7, 11111111b ;k3 + pshufhw xmm4, xmm7, 0b ;k4 + pshufhw xmm5, xmm7, 01010101b ;k5 + pshufhw xmm6, xmm7, 10101010b ;k6 + pshufhw xmm7, xmm7, 11111111b ;k7 + punpcklqdq xmm2, xmm2 + punpcklqdq xmm3, xmm3 + punpcklwd xmm0, xmm1 + punpckhwd xmm6, xmm7 + punpckhwd xmm2, xmm5 + punpckhwd xmm3, xmm4 + + movdqa k0k1, xmm0 ;store filter factors on stack + movdqa k6k7, xmm6 + movdqa k2k5, xmm2 + movdqa k3k4, xmm3 + + movq xmm6, rcx + pshufd xmm6, xmm6, 0 + movdqa krd, xmm6 ;rounding + + ;Compute max and min values of a pixel + mov rdx, 0x00010001 + movsxd rcx, DWORD PTR arg(6) ;bps + movq xmm0, rdx + movq xmm1, rcx + pshufd xmm0, xmm0, 0b + movdqa xmm2, xmm0 + psllw xmm0, xmm1 + psubw xmm0, xmm2 + pxor xmm1, xmm1 + movdqa max, xmm0 ;max value (for clamping) + movdqa min, xmm1 ;min value (for clamping) +%endm + +%macro LOAD_VERT_8 1 + movdqu xmm0, [rsi + %1] ;0 + movdqu xmm1, [rsi + rax + %1] ;1 + movdqu xmm6, [rsi + rdx * 2 + %1] ;6 + lea rsi, [rsi + rax] + movdqu xmm7, [rsi + rdx * 2 + %1] ;7 + movdqu xmm2, [rsi + rax + %1] ;2 + movdqu xmm3, [rsi + rax * 2 + %1] ;3 + movdqu xmm4, [rsi + rdx + %1] ;4 + movdqu xmm5, [rsi + rax * 4 + %1] ;5 +%endm + +%macro HIGH_APPLY_FILTER_8 2 + movdqu temp, xmm4 + movdqa xmm4, xmm0 + punpcklwd xmm0, xmm1 + punpckhwd xmm4, xmm1 + movdqa xmm1, xmm6 + punpcklwd xmm6, xmm7 + punpckhwd xmm1, xmm7 + movdqa xmm7, xmm2 + punpcklwd xmm2, xmm5 + punpckhwd xmm7, xmm5 + + movdqu xmm5, temp + movdqu temp, xmm4 + movdqa xmm4, xmm3 + punpcklwd xmm3, xmm5 + punpckhwd xmm4, xmm5 + movdqu xmm5, temp + + pmaddwd xmm0, k0k1 + pmaddwd xmm5, k0k1 + pmaddwd xmm6, k6k7 + pmaddwd xmm1, k6k7 + pmaddwd xmm2, k2k5 + pmaddwd xmm7, k2k5 + pmaddwd xmm3, k3k4 + pmaddwd xmm4, k3k4 + + paddd xmm0, xmm6 + paddd xmm0, xmm2 + paddd xmm0, xmm3 + paddd xmm5, xmm1 + paddd xmm5, xmm7 + paddd xmm5, xmm4 + + paddd xmm0, krd ;rounding + paddd xmm5, krd + psrad xmm0, 7 ;shift + psrad xmm5, 7 + packssdw xmm0, xmm5 ;pack back to word + + ;clamp the values + pminsw xmm0, max + pmaxsw xmm0, min + +%if %1 + movdqu xmm1, [rdi + %2] + pavgw xmm0, xmm1 +%endif + movdqu [rdi + %2], xmm0 +%endm + +;void aom_filter_block1d4_v8_sse2 +;( +; unsigned char *src_ptr, +; unsigned int src_pitch, +; unsigned char *output_ptr, +; unsigned int out_pitch, +; unsigned int output_height, +; short *filter +;) +global sym(aom_highbd_filter_block1d4_v8_sse2) PRIVATE +sym(aom_highbd_filter_block1d4_v8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + push rsi + push rdi + push rbx + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 7 + %define k0k6 [rsp + 16 * 0] + %define k2k5 [rsp + 16 * 1] + %define k3k4 [rsp + 16 * 2] + %define k1k7 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define max [rsp + 16 * 5] + %define min [rsp + 16 * 6] + + HIGH_GET_FILTERS_4 + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rbx, DWORD PTR arg(3) ;out_pitch + lea rax, [rax + rax] ;bytes per line + lea rbx, [rbx + rbx] + lea rdx, [rax + rax * 2] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movq xmm0, [rsi] ;load src: row 0 + movq xmm1, [rsi + rax] ;1 + movq xmm6, [rsi + rdx * 2] ;6 + lea rsi, [rsi + rax] + movq xmm7, [rsi + rdx * 2] ;7 + movq xmm2, [rsi + rax] ;2 + movq xmm3, [rsi + rax * 2] ;3 + movq xmm4, [rsi + rdx] ;4 + movq xmm5, [rsi + rax * 4] ;5 + + HIGH_APPLY_FILTER_4 0 + + lea rdi, [rdi + rbx] + dec rcx + jnz .loop + + add rsp, 16 * 7 + pop rsp + pop rbx + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void aom_filter_block1d8_v8_sse2 +;( +; unsigned char *src_ptr, +; unsigned int src_pitch, +; unsigned char *output_ptr, +; unsigned int out_pitch, +; unsigned int output_height, +; short *filter +;) +global sym(aom_highbd_filter_block1d8_v8_sse2) PRIVATE +sym(aom_highbd_filter_block1d8_v8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + push rsi + push rdi + push rbx + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 8 + %define k0k1 [rsp + 16 * 0] + %define k6k7 [rsp + 16 * 1] + %define k2k5 [rsp + 16 * 2] + %define k3k4 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define temp [rsp + 16 * 5] + %define max [rsp + 16 * 6] + %define min [rsp + 16 * 7] + + HIGH_GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rbx, DWORD PTR arg(3) ;out_pitch + lea rax, [rax + rax] ;bytes per line + lea rbx, [rbx + rbx] + lea rdx, [rax + rax * 2] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + LOAD_VERT_8 0 + HIGH_APPLY_FILTER_8 0, 0 + + lea rdi, [rdi + rbx] + dec rcx + jnz .loop + + add rsp, 16 * 8 + pop rsp + pop rbx + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void aom_filter_block1d16_v8_sse2 +;( +; unsigned char *src_ptr, +; unsigned int src_pitch, +; unsigned char *output_ptr, +; unsigned int out_pitch, +; unsigned int output_height, +; short *filter +;) +global sym(aom_highbd_filter_block1d16_v8_sse2) PRIVATE +sym(aom_highbd_filter_block1d16_v8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + push rsi + push rdi + push rbx + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 8 + %define k0k1 [rsp + 16 * 0] + %define k6k7 [rsp + 16 * 1] + %define k2k5 [rsp + 16 * 2] + %define k3k4 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define temp [rsp + 16 * 5] + %define max [rsp + 16 * 6] + %define min [rsp + 16 * 7] + + HIGH_GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rbx, DWORD PTR arg(3) ;out_pitch + lea rax, [rax + rax] ;bytes per line + lea rbx, [rbx + rbx] + lea rdx, [rax + rax * 2] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + LOAD_VERT_8 0 + HIGH_APPLY_FILTER_8 0, 0 + sub rsi, rax + + LOAD_VERT_8 16 + HIGH_APPLY_FILTER_8 0, 16 + add rdi, rbx + + dec rcx + jnz .loop + + add rsp, 16 * 8 + pop rsp + pop rbx + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_highbd_filter_block1d4_v8_avg_sse2) PRIVATE +sym(aom_highbd_filter_block1d4_v8_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + push rsi + push rdi + push rbx + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 7 + %define k0k6 [rsp + 16 * 0] + %define k2k5 [rsp + 16 * 1] + %define k3k4 [rsp + 16 * 2] + %define k1k7 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define max [rsp + 16 * 5] + %define min [rsp + 16 * 6] + + HIGH_GET_FILTERS_4 + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rbx, DWORD PTR arg(3) ;out_pitch + lea rax, [rax + rax] ;bytes per line + lea rbx, [rbx + rbx] + lea rdx, [rax + rax * 2] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movq xmm0, [rsi] ;load src: row 0 + movq xmm1, [rsi + rax] ;1 + movq xmm6, [rsi + rdx * 2] ;6 + lea rsi, [rsi + rax] + movq xmm7, [rsi + rdx * 2] ;7 + movq xmm2, [rsi + rax] ;2 + movq xmm3, [rsi + rax * 2] ;3 + movq xmm4, [rsi + rdx] ;4 + movq xmm5, [rsi + rax * 4] ;5 + + HIGH_APPLY_FILTER_4 1 + + lea rdi, [rdi + rbx] + dec rcx + jnz .loop + + add rsp, 16 * 7 + pop rsp + pop rbx + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_highbd_filter_block1d8_v8_avg_sse2) PRIVATE +sym(aom_highbd_filter_block1d8_v8_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + push rsi + push rdi + push rbx + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 8 + %define k0k1 [rsp + 16 * 0] + %define k6k7 [rsp + 16 * 1] + %define k2k5 [rsp + 16 * 2] + %define k3k4 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define temp [rsp + 16 * 5] + %define max [rsp + 16 * 6] + %define min [rsp + 16 * 7] + + HIGH_GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rbx, DWORD PTR arg(3) ;out_pitch + lea rax, [rax + rax] ;bytes per line + lea rbx, [rbx + rbx] + lea rdx, [rax + rax * 2] + movsxd rcx, DWORD PTR arg(4) ;output_height +.loop: + LOAD_VERT_8 0 + HIGH_APPLY_FILTER_8 1, 0 + + lea rdi, [rdi + rbx] + dec rcx + jnz .loop + + add rsp, 16 * 8 + pop rsp + pop rbx + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_highbd_filter_block1d16_v8_avg_sse2) PRIVATE +sym(aom_highbd_filter_block1d16_v8_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + push rsi + push rdi + push rbx + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 8 + %define k0k1 [rsp + 16 * 0] + %define k6k7 [rsp + 16 * 1] + %define k2k5 [rsp + 16 * 2] + %define k3k4 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define temp [rsp + 16 * 5] + %define max [rsp + 16 * 6] + %define min [rsp + 16 * 7] + + HIGH_GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rbx, DWORD PTR arg(3) ;out_pitch + lea rax, [rax + rax] ;bytes per line + lea rbx, [rbx + rbx] + lea rdx, [rax + rax * 2] + movsxd rcx, DWORD PTR arg(4) ;output_height +.loop: + LOAD_VERT_8 0 + HIGH_APPLY_FILTER_8 1, 0 + sub rsi, rax + + LOAD_VERT_8 16 + HIGH_APPLY_FILTER_8 1, 16 + add rdi, rbx + + dec rcx + jnz .loop + + add rsp, 16 * 8 + pop rsp + pop rbx + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void aom_filter_block1d4_h8_sse2 +;( +; unsigned char *src_ptr, +; unsigned int src_pixels_per_line, +; unsigned char *output_ptr, +; unsigned int output_pitch, +; unsigned int output_height, +; short *filter +;) +global sym(aom_highbd_filter_block1d4_h8_sse2) PRIVATE +sym(aom_highbd_filter_block1d4_h8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 7 + %define k0k6 [rsp + 16 * 0] + %define k2k5 [rsp + 16 * 1] + %define k3k4 [rsp + 16 * 2] + %define k1k7 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define max [rsp + 16 * 5] + %define min [rsp + 16 * 6] + + HIGH_GET_FILTERS_4 + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + lea rax, [rax + rax] ;bytes per line + lea rdx, [rdx + rdx] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movdqu xmm0, [rsi - 6] ;load src + movdqu xmm4, [rsi + 2] + movdqa xmm1, xmm0 + movdqa xmm6, xmm4 + movdqa xmm7, xmm4 + movdqa xmm2, xmm0 + movdqa xmm3, xmm0 + movdqa xmm5, xmm4 + + psrldq xmm1, 2 + psrldq xmm6, 4 + psrldq xmm7, 6 + psrldq xmm2, 4 + psrldq xmm3, 6 + psrldq xmm5, 2 + + HIGH_APPLY_FILTER_4 0 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx + jnz .loop + + add rsp, 16 * 7 + pop rsp + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void aom_filter_block1d8_h8_sse2 +;( +; unsigned char *src_ptr, +; unsigned int src_pixels_per_line, +; unsigned char *output_ptr, +; unsigned int output_pitch, +; unsigned int output_height, +; short *filter +;) +global sym(aom_highbd_filter_block1d8_h8_sse2) PRIVATE +sym(aom_highbd_filter_block1d8_h8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 8 + %define k0k1 [rsp + 16 * 0] + %define k6k7 [rsp + 16 * 1] + %define k2k5 [rsp + 16 * 2] + %define k3k4 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define temp [rsp + 16 * 5] + %define max [rsp + 16 * 6] + %define min [rsp + 16 * 7] + + HIGH_GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + lea rax, [rax + rax] ;bytes per line + lea rdx, [rdx + rdx] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movdqu xmm0, [rsi - 6] ;load src + movdqu xmm1, [rsi - 4] + movdqu xmm2, [rsi - 2] + movdqu xmm3, [rsi] + movdqu xmm4, [rsi + 2] + movdqu xmm5, [rsi + 4] + movdqu xmm6, [rsi + 6] + movdqu xmm7, [rsi + 8] + + HIGH_APPLY_FILTER_8 0, 0 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx + jnz .loop + + add rsp, 16 * 8 + pop rsp + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void aom_filter_block1d16_h8_sse2 +;( +; unsigned char *src_ptr, +; unsigned int src_pixels_per_line, +; unsigned char *output_ptr, +; unsigned int output_pitch, +; unsigned int output_height, +; short *filter +;) +global sym(aom_highbd_filter_block1d16_h8_sse2) PRIVATE +sym(aom_highbd_filter_block1d16_h8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 8 + %define k0k1 [rsp + 16 * 0] + %define k6k7 [rsp + 16 * 1] + %define k2k5 [rsp + 16 * 2] + %define k3k4 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define temp [rsp + 16 * 5] + %define max [rsp + 16 * 6] + %define min [rsp + 16 * 7] + + HIGH_GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + lea rax, [rax + rax] ;bytes per line + lea rdx, [rdx + rdx] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movdqu xmm0, [rsi - 6] ;load src + movdqu xmm1, [rsi - 4] + movdqu xmm2, [rsi - 2] + movdqu xmm3, [rsi] + movdqu xmm4, [rsi + 2] + movdqu xmm5, [rsi + 4] + movdqu xmm6, [rsi + 6] + movdqu xmm7, [rsi + 8] + + HIGH_APPLY_FILTER_8 0, 0 + + movdqu xmm0, [rsi + 10] ;load src + movdqu xmm1, [rsi + 12] + movdqu xmm2, [rsi + 14] + movdqu xmm3, [rsi + 16] + movdqu xmm4, [rsi + 18] + movdqu xmm5, [rsi + 20] + movdqu xmm6, [rsi + 22] + movdqu xmm7, [rsi + 24] + + HIGH_APPLY_FILTER_8 0, 16 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx + jnz .loop + + add rsp, 16 * 8 + pop rsp + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_highbd_filter_block1d4_h8_avg_sse2) PRIVATE +sym(aom_highbd_filter_block1d4_h8_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 7 + %define k0k6 [rsp + 16 * 0] + %define k2k5 [rsp + 16 * 1] + %define k3k4 [rsp + 16 * 2] + %define k1k7 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define max [rsp + 16 * 5] + %define min [rsp + 16 * 6] + + HIGH_GET_FILTERS_4 + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + lea rax, [rax + rax] ;bytes per line + lea rdx, [rdx + rdx] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movdqu xmm0, [rsi - 6] ;load src + movdqu xmm4, [rsi + 2] + movdqa xmm1, xmm0 + movdqa xmm6, xmm4 + movdqa xmm7, xmm4 + movdqa xmm2, xmm0 + movdqa xmm3, xmm0 + movdqa xmm5, xmm4 + + psrldq xmm1, 2 + psrldq xmm6, 4 + psrldq xmm7, 6 + psrldq xmm2, 4 + psrldq xmm3, 6 + psrldq xmm5, 2 + + HIGH_APPLY_FILTER_4 1 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx + jnz .loop + + add rsp, 16 * 7 + pop rsp + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_highbd_filter_block1d8_h8_avg_sse2) PRIVATE +sym(aom_highbd_filter_block1d8_h8_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 8 + %define k0k1 [rsp + 16 * 0] + %define k6k7 [rsp + 16 * 1] + %define k2k5 [rsp + 16 * 2] + %define k3k4 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define temp [rsp + 16 * 5] + %define max [rsp + 16 * 6] + %define min [rsp + 16 * 7] + + HIGH_GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + lea rax, [rax + rax] ;bytes per line + lea rdx, [rdx + rdx] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movdqu xmm0, [rsi - 6] ;load src + movdqu xmm1, [rsi - 4] + movdqu xmm2, [rsi - 2] + movdqu xmm3, [rsi] + movdqu xmm4, [rsi + 2] + movdqu xmm5, [rsi + 4] + movdqu xmm6, [rsi + 6] + movdqu xmm7, [rsi + 8] + + HIGH_APPLY_FILTER_8 1, 0 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx + jnz .loop + + add rsp, 16 * 8 + pop rsp + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_highbd_filter_block1d16_h8_avg_sse2) PRIVATE +sym(aom_highbd_filter_block1d16_h8_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 8 + %define k0k1 [rsp + 16 * 0] + %define k6k7 [rsp + 16 * 1] + %define k2k5 [rsp + 16 * 2] + %define k3k4 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define temp [rsp + 16 * 5] + %define max [rsp + 16 * 6] + %define min [rsp + 16 * 7] + + HIGH_GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + lea rax, [rax + rax] ;bytes per line + lea rdx, [rdx + rdx] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movdqu xmm0, [rsi - 6] ;load src + movdqu xmm1, [rsi - 4] + movdqu xmm2, [rsi - 2] + movdqu xmm3, [rsi] + movdqu xmm4, [rsi + 2] + movdqu xmm5, [rsi + 4] + movdqu xmm6, [rsi + 6] + movdqu xmm7, [rsi + 8] + + HIGH_APPLY_FILTER_8 1, 0 + + movdqu xmm0, [rsi + 10] ;load src + movdqu xmm1, [rsi + 12] + movdqu xmm2, [rsi + 14] + movdqu xmm3, [rsi + 16] + movdqu xmm4, [rsi + 18] + movdqu xmm5, [rsi + 20] + movdqu xmm6, [rsi + 22] + movdqu xmm7, [rsi + 24] + + HIGH_APPLY_FILTER_8 1, 16 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx + jnz .loop + + add rsp, 16 * 8 + pop rsp + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret diff --git a/third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm b/third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm new file mode 100644 index 0000000000..9e2ec748c7 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm @@ -0,0 +1,497 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "aom_ports/x86_abi_support.asm" + +%macro HIGH_GET_PARAM_4 0 + mov rdx, arg(5) ;filter ptr + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + mov rcx, 0x00000040 + + movdqa xmm3, [rdx] ;load filters + pshuflw xmm4, xmm3, 11111111b ;k3 + psrldq xmm3, 8 + pshuflw xmm3, xmm3, 0b ;k4 + punpcklwd xmm4, xmm3 ;k3k4 + + movq xmm3, rcx ;rounding + pshufd xmm3, xmm3, 0 + + mov rdx, 0x00010001 + movsxd rcx, DWORD PTR arg(6) ;bps + movq xmm5, rdx + movq xmm2, rcx + pshufd xmm5, xmm5, 0b + movdqa xmm1, xmm5 + psllw xmm5, xmm2 + psubw xmm5, xmm1 ;max value (for clamping) + pxor xmm2, xmm2 ;min value (for clamping) + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + movsxd rcx, DWORD PTR arg(4) ;output_height +%endm + +%macro HIGH_APPLY_FILTER_4 1 + + punpcklwd xmm0, xmm1 ;two row in one register + pmaddwd xmm0, xmm4 ;multiply the filter factors + + paddd xmm0, xmm3 ;rounding + psrad xmm0, 7 ;shift + packssdw xmm0, xmm0 ;pack to word + + ;clamp the values + pminsw xmm0, xmm5 + pmaxsw xmm0, xmm2 + +%if %1 + movq xmm1, [rdi] + pavgw xmm0, xmm1 +%endif + + movq [rdi], xmm0 + lea rsi, [rsi + 2*rax] + lea rdi, [rdi + 2*rdx] + dec rcx +%endm + +%if ARCH_X86_64 +%macro HIGH_GET_PARAM 0 + mov rdx, arg(5) ;filter ptr + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + mov rcx, 0x00000040 + + movdqa xmm6, [rdx] ;load filters + + pshuflw xmm7, xmm6, 11111111b ;k3 + pshufhw xmm6, xmm6, 0b ;k4 + psrldq xmm6, 8 + punpcklwd xmm7, xmm6 ;k3k4k3k4k3k4k3k4 + + movq xmm4, rcx ;rounding + pshufd xmm4, xmm4, 0 + + mov rdx, 0x00010001 + movsxd rcx, DWORD PTR arg(6) ;bps + movq xmm8, rdx + movq xmm5, rcx + pshufd xmm8, xmm8, 0b + movdqa xmm1, xmm8 + psllw xmm8, xmm5 + psubw xmm8, xmm1 ;max value (for clamping) + pxor xmm5, xmm5 ;min value (for clamping) + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + movsxd rcx, DWORD PTR arg(4) ;output_height +%endm + +%macro HIGH_APPLY_FILTER_8 1 + movdqa xmm6, xmm0 + punpckhwd xmm6, xmm1 + punpcklwd xmm0, xmm1 + pmaddwd xmm6, xmm7 + pmaddwd xmm0, xmm7 + + paddd xmm6, xmm4 ;rounding + paddd xmm0, xmm4 ;rounding + psrad xmm6, 7 ;shift + psrad xmm0, 7 ;shift + packssdw xmm0, xmm6 ;pack back to word + + ;clamp the values + pminsw xmm0, xmm8 + pmaxsw xmm0, xmm5 + +%if %1 + movdqu xmm1, [rdi] + pavgw xmm0, xmm1 +%endif + movdqu [rdi], xmm0 ;store the result + + lea rsi, [rsi + 2*rax] + lea rdi, [rdi + 2*rdx] + dec rcx +%endm + +%macro HIGH_APPLY_FILTER_16 1 + movdqa xmm9, xmm0 + movdqa xmm6, xmm2 + punpckhwd xmm9, xmm1 + punpckhwd xmm6, xmm3 + punpcklwd xmm0, xmm1 + punpcklwd xmm2, xmm3 + + pmaddwd xmm9, xmm7 + pmaddwd xmm6, xmm7 + pmaddwd xmm0, xmm7 + pmaddwd xmm2, xmm7 + + paddd xmm9, xmm4 ;rounding + paddd xmm6, xmm4 + paddd xmm0, xmm4 + paddd xmm2, xmm4 + + psrad xmm9, 7 ;shift + psrad xmm6, 7 + psrad xmm0, 7 + psrad xmm2, 7 + + packssdw xmm0, xmm9 ;pack back to word + packssdw xmm2, xmm6 ;pack back to word + + ;clamp the values + pminsw xmm0, xmm8 + pmaxsw xmm0, xmm5 + pminsw xmm2, xmm8 + pmaxsw xmm2, xmm5 + +%if %1 + movdqu xmm1, [rdi] + movdqu xmm3, [rdi + 16] + pavgw xmm0, xmm1 + pavgw xmm2, xmm3 +%endif + movdqu [rdi], xmm0 ;store the result + movdqu [rdi + 16], xmm2 ;store the result + + lea rsi, [rsi + 2*rax] + lea rdi, [rdi + 2*rdx] + dec rcx +%endm +%endif + +global sym(aom_highbd_filter_block1d4_v2_sse2) PRIVATE +sym(aom_highbd_filter_block1d4_v2_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + push rsi + push rdi + ; end prolog + + HIGH_GET_PARAM_4 +.loop: + movq xmm0, [rsi] ;load src + movq xmm1, [rsi + 2*rax] + + HIGH_APPLY_FILTER_4 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + +%if ARCH_X86_64 +global sym(aom_highbd_filter_block1d8_v2_sse2) PRIVATE +sym(aom_highbd_filter_block1d8_v2_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 8 + push rsi + push rdi + ; end prolog + + HIGH_GET_PARAM +.loop: + movdqu xmm0, [rsi] ;0 + movdqu xmm1, [rsi + 2*rax] ;1 + + HIGH_APPLY_FILTER_8 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_highbd_filter_block1d16_v2_sse2) PRIVATE +sym(aom_highbd_filter_block1d16_v2_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 9 + push rsi + push rdi + ; end prolog + + HIGH_GET_PARAM +.loop: + movdqu xmm0, [rsi] ;0 + movdqu xmm2, [rsi + 16] + movdqu xmm1, [rsi + 2*rax] ;1 + movdqu xmm3, [rsi + 2*rax + 16] + + HIGH_APPLY_FILTER_16 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret +%endif + +global sym(aom_highbd_filter_block1d4_v2_avg_sse2) PRIVATE +sym(aom_highbd_filter_block1d4_v2_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + push rsi + push rdi + ; end prolog + + HIGH_GET_PARAM_4 +.loop: + movq xmm0, [rsi] ;load src + movq xmm1, [rsi + 2*rax] + + HIGH_APPLY_FILTER_4 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + +%if ARCH_X86_64 +global sym(aom_highbd_filter_block1d8_v2_avg_sse2) PRIVATE +sym(aom_highbd_filter_block1d8_v2_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 8 + push rsi + push rdi + ; end prolog + + HIGH_GET_PARAM +.loop: + movdqu xmm0, [rsi] ;0 + movdqu xmm1, [rsi + 2*rax] ;1 + + HIGH_APPLY_FILTER_8 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_highbd_filter_block1d16_v2_avg_sse2) PRIVATE +sym(aom_highbd_filter_block1d16_v2_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 9 + push rsi + push rdi + ; end prolog + + HIGH_GET_PARAM +.loop: + movdqu xmm0, [rsi] ;0 + movdqu xmm1, [rsi + 2*rax] ;1 + movdqu xmm2, [rsi + 16] + movdqu xmm3, [rsi + 2*rax + 16] + + HIGH_APPLY_FILTER_16 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret +%endif + +global sym(aom_highbd_filter_block1d4_h2_sse2) PRIVATE +sym(aom_highbd_filter_block1d4_h2_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + push rsi + push rdi + ; end prolog + + HIGH_GET_PARAM_4 +.loop: + movdqu xmm0, [rsi] ;load src + movdqa xmm1, xmm0 + psrldq xmm1, 2 + + HIGH_APPLY_FILTER_4 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + +%if ARCH_X86_64 +global sym(aom_highbd_filter_block1d8_h2_sse2) PRIVATE +sym(aom_highbd_filter_block1d8_h2_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 8 + push rsi + push rdi + ; end prolog + + HIGH_GET_PARAM +.loop: + movdqu xmm0, [rsi] ;load src + movdqu xmm1, [rsi + 2] + + HIGH_APPLY_FILTER_8 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_highbd_filter_block1d16_h2_sse2) PRIVATE +sym(aom_highbd_filter_block1d16_h2_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 9 + push rsi + push rdi + ; end prolog + + HIGH_GET_PARAM +.loop: + movdqu xmm0, [rsi] ;load src + movdqu xmm1, [rsi + 2] + movdqu xmm2, [rsi + 16] + movdqu xmm3, [rsi + 18] + + HIGH_APPLY_FILTER_16 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret +%endif + +global sym(aom_highbd_filter_block1d4_h2_avg_sse2) PRIVATE +sym(aom_highbd_filter_block1d4_h2_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + push rsi + push rdi + ; end prolog + + HIGH_GET_PARAM_4 +.loop: + movdqu xmm0, [rsi] ;load src + movdqa xmm1, xmm0 + psrldq xmm1, 2 + + HIGH_APPLY_FILTER_4 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + +%if ARCH_X86_64 +global sym(aom_highbd_filter_block1d8_h2_avg_sse2) PRIVATE +sym(aom_highbd_filter_block1d8_h2_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 8 + push rsi + push rdi + ; end prolog + + HIGH_GET_PARAM +.loop: + movdqu xmm0, [rsi] ;load src + movdqu xmm1, [rsi + 2] + + HIGH_APPLY_FILTER_8 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_highbd_filter_block1d16_h2_avg_sse2) PRIVATE +sym(aom_highbd_filter_block1d16_h2_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 9 + push rsi + push rdi + ; end prolog + + HIGH_GET_PARAM +.loop: + movdqu xmm0, [rsi] ;load src + movdqu xmm1, [rsi + 2] + movdqu xmm2, [rsi + 16] + movdqu xmm3, [rsi + 18] + + HIGH_APPLY_FILTER_16 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret +%endif diff --git a/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_avx2.c b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_avx2.c new file mode 100644 index 0000000000..61476b8be6 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_avx2.c @@ -0,0 +1,575 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/x86/convolve.h" +#include "aom_ports/mem.h" + +// filters for 16_h8 and 16_v8 +DECLARE_ALIGNED(32, static const uint8_t, filt1_global_avx2[32]) = { + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 +}; + +DECLARE_ALIGNED(32, static const uint8_t, filt2_global_avx2[32]) = { + 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, + 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10 +}; + +DECLARE_ALIGNED(32, static const uint8_t, filt3_global_avx2[32]) = { + 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, + 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12 +}; + +DECLARE_ALIGNED(32, static const uint8_t, filt4_global_avx2[32]) = { + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14 +}; + +#if defined(__clang__) +#if (__clang_major__ > 0 && __clang_major__ < 3) || \ + (__clang_major__ == 3 && __clang_minor__ <= 3) || \ + (defined(__APPLE__) && defined(__apple_build_version__) && \ + ((__clang_major__ == 4 && __clang_minor__ <= 2) || \ + (__clang_major__ == 5 && __clang_minor__ == 0))) +#define MM256_BROADCASTSI128_SI256(x) \ + _mm_broadcastsi128_si256((__m128i const *)&(x)) +#else // clang > 3.3, and not 5.0 on macosx. +#define MM256_BROADCASTSI128_SI256(x) _mm256_broadcastsi128_si256(x) +#endif // clang <= 3.3 +#elif defined(__GNUC__) +#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ <= 6) +#define MM256_BROADCASTSI128_SI256(x) \ + _mm_broadcastsi128_si256((__m128i const *)&(x)) +#elif __GNUC__ == 4 && __GNUC_MINOR__ == 7 +#define MM256_BROADCASTSI128_SI256(x) _mm_broadcastsi128_si256(x) +#else // gcc > 4.7 +#define MM256_BROADCASTSI128_SI256(x) _mm256_broadcastsi128_si256(x) +#endif // gcc <= 4.6 +#else // !(gcc || clang) +#define MM256_BROADCASTSI128_SI256(x) _mm256_broadcastsi128_si256(x) +#endif // __clang__ + +static void aom_filter_block1d16_h8_avx2( + const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr, + ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) { + __m128i filtersReg; + __m256i addFilterReg64, filt1Reg, filt2Reg, filt3Reg, filt4Reg; + __m256i firstFilters, secondFilters, thirdFilters, forthFilters; + __m256i srcRegFilt32b1_1, srcRegFilt32b2_1, srcRegFilt32b2, srcRegFilt32b3; + __m256i srcReg32b1, srcReg32b2, filtersReg32; + unsigned int i; + ptrdiff_t src_stride, dst_stride; + + // create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64 + addFilterReg64 = _mm256_set1_epi32((int)0x0400040u); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + // converting the 16 bit (short) to 8 bit (byte) and have the same data + // in both lanes of 128 bit register. + filtersReg = _mm_packs_epi16(filtersReg, filtersReg); + // have the same data in both lanes of a 256 bit register + filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg); + + // duplicate only the first 16 bits (first and second byte) + // across 256 bit register + firstFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x100u)); + // duplicate only the second 16 bits (third and forth byte) + // across 256 bit register + secondFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x302u)); + // duplicate only the third 16 bits (fifth and sixth byte) + // across 256 bit register + thirdFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x504u)); + // duplicate only the forth 16 bits (seventh and eighth byte) + // across 256 bit register + forthFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x706u)); + + filt1Reg = _mm256_load_si256((__m256i const *)filt1_global_avx2); + filt2Reg = _mm256_load_si256((__m256i const *)filt2_global_avx2); + filt3Reg = _mm256_load_si256((__m256i const *)filt3_global_avx2); + filt4Reg = _mm256_load_si256((__m256i const *)filt4_global_avx2); + + // multiple the size of the source and destination stride by two + src_stride = src_pixels_per_line << 1; + dst_stride = output_pitch << 1; + for (i = output_height; i > 1; i -= 2) { + // load the 2 strides of source + srcReg32b1 = + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(src_ptr - 3))); + srcReg32b1 = _mm256_inserti128_si256( + srcReg32b1, + _mm_loadu_si128((const __m128i *)(src_ptr + src_pixels_per_line - 3)), + 1); + + // filter the source buffer + srcRegFilt32b1_1 = _mm256_shuffle_epi8(srcReg32b1, filt1Reg); + srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b1, filt4Reg); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt32b1_1 = _mm256_maddubs_epi16(srcRegFilt32b1_1, firstFilters); + srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, forthFilters); + + // add and saturate the results together + srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, srcRegFilt32b2); + + // filter the source buffer + srcRegFilt32b3 = _mm256_shuffle_epi8(srcReg32b1, filt2Reg); + srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b1, filt3Reg); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt32b3 = _mm256_maddubs_epi16(srcRegFilt32b3, secondFilters); + srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, thirdFilters); + + // add and saturate the results together + srcRegFilt32b1_1 = _mm256_adds_epi16( + srcRegFilt32b1_1, _mm256_min_epi16(srcRegFilt32b3, srcRegFilt32b2)); + + // reading 2 strides of the next 16 bytes + // (part of it was being read by earlier read) + srcReg32b2 = + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(src_ptr + 5))); + srcReg32b2 = _mm256_inserti128_si256( + srcReg32b2, + _mm_loadu_si128((const __m128i *)(src_ptr + src_pixels_per_line + 5)), + 1); + + // add and saturate the results together + srcRegFilt32b1_1 = _mm256_adds_epi16( + srcRegFilt32b1_1, _mm256_max_epi16(srcRegFilt32b3, srcRegFilt32b2)); + + // filter the source buffer + srcRegFilt32b2_1 = _mm256_shuffle_epi8(srcReg32b2, filt1Reg); + srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b2, filt4Reg); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt32b2_1 = _mm256_maddubs_epi16(srcRegFilt32b2_1, firstFilters); + srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, forthFilters); + + // add and saturate the results together + srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1, srcRegFilt32b2); + + // filter the source buffer + srcRegFilt32b3 = _mm256_shuffle_epi8(srcReg32b2, filt2Reg); + srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b2, filt3Reg); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt32b3 = _mm256_maddubs_epi16(srcRegFilt32b3, secondFilters); + srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, thirdFilters); + + // add and saturate the results together + srcRegFilt32b2_1 = _mm256_adds_epi16( + srcRegFilt32b2_1, _mm256_min_epi16(srcRegFilt32b3, srcRegFilt32b2)); + srcRegFilt32b2_1 = _mm256_adds_epi16( + srcRegFilt32b2_1, _mm256_max_epi16(srcRegFilt32b3, srcRegFilt32b2)); + + srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, addFilterReg64); + + srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1, addFilterReg64); + + // shift by 7 bit each 16 bit + srcRegFilt32b1_1 = _mm256_srai_epi16(srcRegFilt32b1_1, 7); + srcRegFilt32b2_1 = _mm256_srai_epi16(srcRegFilt32b2_1, 7); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + srcRegFilt32b1_1 = _mm256_packus_epi16(srcRegFilt32b1_1, srcRegFilt32b2_1); + + src_ptr += src_stride; + + // save 16 bytes + _mm_store_si128((__m128i *)output_ptr, + _mm256_castsi256_si128(srcRegFilt32b1_1)); + + // save the next 16 bits + _mm_store_si128((__m128i *)(output_ptr + output_pitch), + _mm256_extractf128_si256(srcRegFilt32b1_1, 1)); + output_ptr += dst_stride; + } + + // if the number of strides is odd. + // process only 16 bytes + if (i > 0) { + __m128i srcReg1, srcReg2, srcRegFilt1_1, srcRegFilt2_1; + __m128i srcRegFilt2, srcRegFilt3; + + srcReg1 = _mm_loadu_si128((const __m128i *)(src_ptr - 3)); + + // filter the source buffer + srcRegFilt1_1 = _mm_shuffle_epi8(srcReg1, _mm256_castsi256_si128(filt1Reg)); + srcRegFilt2 = _mm_shuffle_epi8(srcReg1, _mm256_castsi256_si128(filt4Reg)); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt1_1 = + _mm_maddubs_epi16(srcRegFilt1_1, _mm256_castsi256_si128(firstFilters)); + srcRegFilt2 = + _mm_maddubs_epi16(srcRegFilt2, _mm256_castsi256_si128(forthFilters)); + + // add and saturate the results together + srcRegFilt1_1 = _mm_adds_epi16(srcRegFilt1_1, srcRegFilt2); + + // filter the source buffer + srcRegFilt3 = _mm_shuffle_epi8(srcReg1, _mm256_castsi256_si128(filt2Reg)); + srcRegFilt2 = _mm_shuffle_epi8(srcReg1, _mm256_castsi256_si128(filt3Reg)); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt3 = + _mm_maddubs_epi16(srcRegFilt3, _mm256_castsi256_si128(secondFilters)); + srcRegFilt2 = + _mm_maddubs_epi16(srcRegFilt2, _mm256_castsi256_si128(thirdFilters)); + + // add and saturate the results together + srcRegFilt1_1 = + _mm_adds_epi16(srcRegFilt1_1, _mm_min_epi16(srcRegFilt3, srcRegFilt2)); + + // reading the next 16 bytes + // (part of it was being read by earlier read) + srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + 5)); + + // add and saturate the results together + srcRegFilt1_1 = + _mm_adds_epi16(srcRegFilt1_1, _mm_max_epi16(srcRegFilt3, srcRegFilt2)); + + // filter the source buffer + srcRegFilt2_1 = _mm_shuffle_epi8(srcReg2, _mm256_castsi256_si128(filt1Reg)); + srcRegFilt2 = _mm_shuffle_epi8(srcReg2, _mm256_castsi256_si128(filt4Reg)); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt2_1 = + _mm_maddubs_epi16(srcRegFilt2_1, _mm256_castsi256_si128(firstFilters)); + srcRegFilt2 = + _mm_maddubs_epi16(srcRegFilt2, _mm256_castsi256_si128(forthFilters)); + + // add and saturate the results together + srcRegFilt2_1 = _mm_adds_epi16(srcRegFilt2_1, srcRegFilt2); + + // filter the source buffer + srcRegFilt3 = _mm_shuffle_epi8(srcReg2, _mm256_castsi256_si128(filt2Reg)); + srcRegFilt2 = _mm_shuffle_epi8(srcReg2, _mm256_castsi256_si128(filt3Reg)); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt3 = + _mm_maddubs_epi16(srcRegFilt3, _mm256_castsi256_si128(secondFilters)); + srcRegFilt2 = + _mm_maddubs_epi16(srcRegFilt2, _mm256_castsi256_si128(thirdFilters)); + + // add and saturate the results together + srcRegFilt2_1 = + _mm_adds_epi16(srcRegFilt2_1, _mm_min_epi16(srcRegFilt3, srcRegFilt2)); + srcRegFilt2_1 = + _mm_adds_epi16(srcRegFilt2_1, _mm_max_epi16(srcRegFilt3, srcRegFilt2)); + + srcRegFilt1_1 = + _mm_adds_epi16(srcRegFilt1_1, _mm256_castsi256_si128(addFilterReg64)); + + srcRegFilt2_1 = + _mm_adds_epi16(srcRegFilt2_1, _mm256_castsi256_si128(addFilterReg64)); + + // shift by 7 bit each 16 bit + srcRegFilt1_1 = _mm_srai_epi16(srcRegFilt1_1, 7); + srcRegFilt2_1 = _mm_srai_epi16(srcRegFilt2_1, 7); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + srcRegFilt1_1 = _mm_packus_epi16(srcRegFilt1_1, srcRegFilt2_1); + + // save 16 bytes + _mm_store_si128((__m128i *)output_ptr, srcRegFilt1_1); + } +} + +static void aom_filter_block1d16_v8_avx2( + const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr, + ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) { + __m128i filtersReg; + __m256i addFilterReg64; + __m256i srcReg32b1, srcReg32b2, srcReg32b3, srcReg32b4, srcReg32b5; + __m256i srcReg32b6, srcReg32b7, srcReg32b8, srcReg32b9, srcReg32b10; + __m256i srcReg32b11, srcReg32b12, filtersReg32; + __m256i firstFilters, secondFilters, thirdFilters, forthFilters; + unsigned int i; + ptrdiff_t src_stride, dst_stride; + + // create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64 + addFilterReg64 = _mm256_set1_epi32((int)0x0400040u); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + // converting the 16 bit (short) to 8 bit (byte) and have the + // same data in both lanes of 128 bit register. + filtersReg = _mm_packs_epi16(filtersReg, filtersReg); + // have the same data in both lanes of a 256 bit register + filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg); + + // duplicate only the first 16 bits (first and second byte) + // across 256 bit register + firstFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x100u)); + // duplicate only the second 16 bits (third and forth byte) + // across 256 bit register + secondFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x302u)); + // duplicate only the third 16 bits (fifth and sixth byte) + // across 256 bit register + thirdFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x504u)); + // duplicate only the forth 16 bits (seventh and eighth byte) + // across 256 bit register + forthFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x706u)); + + // multiple the size of the source and destination stride by two + src_stride = src_pitch << 1; + dst_stride = out_pitch << 1; + + // load 16 bytes 7 times in stride of src_pitch + srcReg32b1 = + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(src_ptr))); + srcReg32b2 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch))); + srcReg32b3 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2))); + srcReg32b4 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3))); + srcReg32b5 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4))); + srcReg32b6 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5))); + srcReg32b7 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6))); + + // have each consecutive loads on the same 256 register + srcReg32b1 = _mm256_inserti128_si256(srcReg32b1, + _mm256_castsi256_si128(srcReg32b2), 1); + srcReg32b2 = _mm256_inserti128_si256(srcReg32b2, + _mm256_castsi256_si128(srcReg32b3), 1); + srcReg32b3 = _mm256_inserti128_si256(srcReg32b3, + _mm256_castsi256_si128(srcReg32b4), 1); + srcReg32b4 = _mm256_inserti128_si256(srcReg32b4, + _mm256_castsi256_si128(srcReg32b5), 1); + srcReg32b5 = _mm256_inserti128_si256(srcReg32b5, + _mm256_castsi256_si128(srcReg32b6), 1); + srcReg32b6 = _mm256_inserti128_si256(srcReg32b6, + _mm256_castsi256_si128(srcReg32b7), 1); + + // merge every two consecutive registers except the last one + srcReg32b10 = _mm256_unpacklo_epi8(srcReg32b1, srcReg32b2); + srcReg32b1 = _mm256_unpackhi_epi8(srcReg32b1, srcReg32b2); + + // save + srcReg32b11 = _mm256_unpacklo_epi8(srcReg32b3, srcReg32b4); + + // save + srcReg32b3 = _mm256_unpackhi_epi8(srcReg32b3, srcReg32b4); + + // save + srcReg32b2 = _mm256_unpacklo_epi8(srcReg32b5, srcReg32b6); + + // save + srcReg32b5 = _mm256_unpackhi_epi8(srcReg32b5, srcReg32b6); + + for (i = output_height; i > 1; i -= 2) { + // load the last 2 loads of 16 bytes and have every two + // consecutive loads in the same 256 bit register + srcReg32b8 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 7))); + srcReg32b7 = _mm256_inserti128_si256(srcReg32b7, + _mm256_castsi256_si128(srcReg32b8), 1); + srcReg32b9 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 8))); + srcReg32b8 = _mm256_inserti128_si256(srcReg32b8, + _mm256_castsi256_si128(srcReg32b9), 1); + + // merge every two consecutive registers + // save + srcReg32b4 = _mm256_unpacklo_epi8(srcReg32b7, srcReg32b8); + srcReg32b7 = _mm256_unpackhi_epi8(srcReg32b7, srcReg32b8); + + // multiply 2 adjacent elements with the filter and add the result + srcReg32b10 = _mm256_maddubs_epi16(srcReg32b10, firstFilters); + srcReg32b6 = _mm256_maddubs_epi16(srcReg32b4, forthFilters); + + // add and saturate the results together + srcReg32b10 = _mm256_adds_epi16(srcReg32b10, srcReg32b6); + + // multiply 2 adjacent elements with the filter and add the result + srcReg32b8 = _mm256_maddubs_epi16(srcReg32b11, secondFilters); + srcReg32b12 = _mm256_maddubs_epi16(srcReg32b2, thirdFilters); + + // add and saturate the results together + srcReg32b10 = _mm256_adds_epi16(srcReg32b10, + _mm256_min_epi16(srcReg32b8, srcReg32b12)); + srcReg32b10 = _mm256_adds_epi16(srcReg32b10, + _mm256_max_epi16(srcReg32b8, srcReg32b12)); + + // multiply 2 adjacent elements with the filter and add the result + srcReg32b1 = _mm256_maddubs_epi16(srcReg32b1, firstFilters); + srcReg32b6 = _mm256_maddubs_epi16(srcReg32b7, forthFilters); + + srcReg32b1 = _mm256_adds_epi16(srcReg32b1, srcReg32b6); + + // multiply 2 adjacent elements with the filter and add the result + srcReg32b8 = _mm256_maddubs_epi16(srcReg32b3, secondFilters); + srcReg32b12 = _mm256_maddubs_epi16(srcReg32b5, thirdFilters); + + // add and saturate the results together + srcReg32b1 = _mm256_adds_epi16(srcReg32b1, + _mm256_min_epi16(srcReg32b8, srcReg32b12)); + srcReg32b1 = _mm256_adds_epi16(srcReg32b1, + _mm256_max_epi16(srcReg32b8, srcReg32b12)); + + srcReg32b10 = _mm256_adds_epi16(srcReg32b10, addFilterReg64); + srcReg32b1 = _mm256_adds_epi16(srcReg32b1, addFilterReg64); + + // shift by 7 bit each 16 bit + srcReg32b10 = _mm256_srai_epi16(srcReg32b10, 7); + srcReg32b1 = _mm256_srai_epi16(srcReg32b1, 7); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + srcReg32b1 = _mm256_packus_epi16(srcReg32b10, srcReg32b1); + + src_ptr += src_stride; + + // save 16 bytes + _mm_store_si128((__m128i *)output_ptr, _mm256_castsi256_si128(srcReg32b1)); + + // save the next 16 bits + _mm_store_si128((__m128i *)(output_ptr + out_pitch), + _mm256_extractf128_si256(srcReg32b1, 1)); + + output_ptr += dst_stride; + + // save part of the registers for next strides + srcReg32b10 = srcReg32b11; + srcReg32b1 = srcReg32b3; + srcReg32b11 = srcReg32b2; + srcReg32b3 = srcReg32b5; + srcReg32b2 = srcReg32b4; + srcReg32b5 = srcReg32b7; + srcReg32b7 = srcReg32b9; + } + if (i > 0) { + __m128i srcRegFilt1, srcRegFilt3, srcRegFilt4, srcRegFilt5; + __m128i srcRegFilt6, srcRegFilt7, srcRegFilt8; + // load the last 16 bytes + srcRegFilt8 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 7)); + + // merge the last 2 results together + srcRegFilt4 = + _mm_unpacklo_epi8(_mm256_castsi256_si128(srcReg32b7), srcRegFilt8); + srcRegFilt7 = + _mm_unpackhi_epi8(_mm256_castsi256_si128(srcReg32b7), srcRegFilt8); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt1 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b10), + _mm256_castsi256_si128(firstFilters)); + srcRegFilt4 = + _mm_maddubs_epi16(srcRegFilt4, _mm256_castsi256_si128(forthFilters)); + srcRegFilt3 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b1), + _mm256_castsi256_si128(firstFilters)); + srcRegFilt7 = + _mm_maddubs_epi16(srcRegFilt7, _mm256_castsi256_si128(forthFilters)); + + // add and saturate the results together + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt4); + srcRegFilt3 = _mm_adds_epi16(srcRegFilt3, srcRegFilt7); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt4 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b11), + _mm256_castsi256_si128(secondFilters)); + srcRegFilt5 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b3), + _mm256_castsi256_si128(secondFilters)); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt6 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b2), + _mm256_castsi256_si128(thirdFilters)); + srcRegFilt7 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b5), + _mm256_castsi256_si128(thirdFilters)); + + // add and saturate the results together + srcRegFilt1 = + _mm_adds_epi16(srcRegFilt1, _mm_min_epi16(srcRegFilt4, srcRegFilt6)); + srcRegFilt3 = + _mm_adds_epi16(srcRegFilt3, _mm_min_epi16(srcRegFilt5, srcRegFilt7)); + + // add and saturate the results together + srcRegFilt1 = + _mm_adds_epi16(srcRegFilt1, _mm_max_epi16(srcRegFilt4, srcRegFilt6)); + srcRegFilt3 = + _mm_adds_epi16(srcRegFilt3, _mm_max_epi16(srcRegFilt5, srcRegFilt7)); + + srcRegFilt1 = + _mm_adds_epi16(srcRegFilt1, _mm256_castsi256_si128(addFilterReg64)); + srcRegFilt3 = + _mm_adds_epi16(srcRegFilt3, _mm256_castsi256_si128(addFilterReg64)); + + // shift by 7 bit each 16 bit + srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 7); + srcRegFilt3 = _mm_srai_epi16(srcRegFilt3, 7); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + srcRegFilt1 = _mm_packus_epi16(srcRegFilt1, srcRegFilt3); + + // save 16 bytes + _mm_store_si128((__m128i *)output_ptr, srcRegFilt1); + } +} + +#if HAVE_AVX2 && HAVE_SSSE3 +filter8_1dfunction aom_filter_block1d4_v8_ssse3; +#if ARCH_X86_64 +filter8_1dfunction aom_filter_block1d8_v8_intrin_ssse3; +filter8_1dfunction aom_filter_block1d8_h8_intrin_ssse3; +filter8_1dfunction aom_filter_block1d4_h8_intrin_ssse3; +#define aom_filter_block1d8_v8_avx2 aom_filter_block1d8_v8_intrin_ssse3 +#define aom_filter_block1d8_h8_avx2 aom_filter_block1d8_h8_intrin_ssse3 +#define aom_filter_block1d4_h8_avx2 aom_filter_block1d4_h8_intrin_ssse3 +#else // ARCH_X86 +filter8_1dfunction aom_filter_block1d8_v8_ssse3; +filter8_1dfunction aom_filter_block1d8_h8_ssse3; +filter8_1dfunction aom_filter_block1d4_h8_ssse3; +#define aom_filter_block1d8_v8_avx2 aom_filter_block1d8_v8_ssse3 +#define aom_filter_block1d8_h8_avx2 aom_filter_block1d8_h8_ssse3 +#define aom_filter_block1d4_h8_avx2 aom_filter_block1d4_h8_ssse3 +#endif // ARCH_X86_64 +filter8_1dfunction aom_filter_block1d16_v2_ssse3; +filter8_1dfunction aom_filter_block1d16_h2_ssse3; +filter8_1dfunction aom_filter_block1d8_v2_ssse3; +filter8_1dfunction aom_filter_block1d8_h2_ssse3; +filter8_1dfunction aom_filter_block1d4_v2_ssse3; +filter8_1dfunction aom_filter_block1d4_h2_ssse3; +#define aom_filter_block1d4_v8_avx2 aom_filter_block1d4_v8_ssse3 +#define aom_filter_block1d16_v2_avx2 aom_filter_block1d16_v2_ssse3 +#define aom_filter_block1d16_h2_avx2 aom_filter_block1d16_h2_ssse3 +#define aom_filter_block1d8_v2_avx2 aom_filter_block1d8_v2_ssse3 +#define aom_filter_block1d8_h2_avx2 aom_filter_block1d8_h2_ssse3 +#define aom_filter_block1d4_v2_avx2 aom_filter_block1d4_v2_ssse3 +#define aom_filter_block1d4_h2_avx2 aom_filter_block1d4_h2_ssse3 +// void aom_convolve8_horiz_avx2(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +// void aom_convolve8_vert_avx2(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , avx2); +FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , avx2); + +// void aom_convolve8_avx2(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +FUN_CONV_2D(, avx2); +#endif // HAVE_AX2 && HAVE_SSSE3 diff --git a/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c new file mode 100644 index 0000000000..be37738df6 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c @@ -0,0 +1,920 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/aom_filter.h" +#include "aom_dsp/x86/convolve.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" +#include "aom_ports/emmintrin_compat.h" + +// filters only for the 4_h8 convolution +DECLARE_ALIGNED(16, static const uint8_t, filt1_4_h8[16]) = { + 0, 1, 1, 2, 2, 3, 3, 4, 2, 3, 3, 4, 4, 5, 5, 6 +}; + +DECLARE_ALIGNED(16, static const uint8_t, filt2_4_h8[16]) = { + 4, 5, 5, 6, 6, 7, 7, 8, 6, 7, 7, 8, 8, 9, 9, 10 +}; + +// filters for 8_h8 and 16_h8 +DECLARE_ALIGNED(16, static const uint8_t, filt1_global[16]) = { + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 +}; + +DECLARE_ALIGNED(16, static const uint8_t, filt2_global[16]) = { + 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10 +}; + +DECLARE_ALIGNED(16, static const uint8_t, filt3_global[16]) = { + 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12 +}; + +DECLARE_ALIGNED(16, static const uint8_t, filt4_global[16]) = { + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14 +}; + +// These are reused by the avx2 intrinsics. +filter8_1dfunction aom_filter_block1d8_v8_intrin_ssse3; +filter8_1dfunction aom_filter_block1d8_h8_intrin_ssse3; +filter8_1dfunction aom_filter_block1d4_h8_intrin_ssse3; + +void aom_filter_block1d4_h8_intrin_ssse3( + const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr, + ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) { + __m128i firstFilters, secondFilters, shuffle1, shuffle2; + __m128i srcRegFilt1, srcRegFilt2, srcRegFilt3, srcRegFilt4; + __m128i addFilterReg64, filtersReg, srcReg, minReg; + unsigned int i; + + // create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64 + addFilterReg64 = _mm_set1_epi32((int)0x0400040u); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + // converting the 16 bit (short) to 8 bit (byte) and have the same data + // in both lanes of 128 bit register. + filtersReg = _mm_packs_epi16(filtersReg, filtersReg); + + // duplicate only the first 16 bits in the filter into the first lane + firstFilters = _mm_shufflelo_epi16(filtersReg, 0); + // duplicate only the third 16 bit in the filter into the first lane + secondFilters = _mm_shufflelo_epi16(filtersReg, 0xAAu); + // duplicate only the seconds 16 bits in the filter into the second lane + // firstFilters: k0 k1 k0 k1 k0 k1 k0 k1 k2 k3 k2 k3 k2 k3 k2 k3 + firstFilters = _mm_shufflehi_epi16(firstFilters, 0x55u); + // duplicate only the forth 16 bits in the filter into the second lane + // secondFilters: k4 k5 k4 k5 k4 k5 k4 k5 k6 k7 k6 k7 k6 k7 k6 k7 + secondFilters = _mm_shufflehi_epi16(secondFilters, 0xFFu); + + // loading the local filters + shuffle1 = _mm_load_si128((__m128i const *)filt1_4_h8); + shuffle2 = _mm_load_si128((__m128i const *)filt2_4_h8); + + for (i = 0; i < output_height; i++) { + srcReg = _mm_loadu_si128((const __m128i *)(src_ptr - 3)); + + // filter the source buffer + srcRegFilt1 = _mm_shuffle_epi8(srcReg, shuffle1); + srcRegFilt2 = _mm_shuffle_epi8(srcReg, shuffle2); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt1 = _mm_maddubs_epi16(srcRegFilt1, firstFilters); + srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2, secondFilters); + + // extract the higher half of the lane + srcRegFilt3 = _mm_srli_si128(srcRegFilt1, 8); + srcRegFilt4 = _mm_srli_si128(srcRegFilt2, 8); + + minReg = _mm_min_epi16(srcRegFilt3, srcRegFilt2); + + // add and saturate all the results together + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt4); + srcRegFilt3 = _mm_max_epi16(srcRegFilt3, srcRegFilt2); + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, minReg); + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt3); + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, addFilterReg64); + + // shift by 7 bit each 16 bits + srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 7); + + // shrink to 8 bit each 16 bits + srcRegFilt1 = _mm_packus_epi16(srcRegFilt1, srcRegFilt1); + src_ptr += src_pixels_per_line; + + // save only 4 bytes + *((int *)&output_ptr[0]) = _mm_cvtsi128_si32(srcRegFilt1); + + output_ptr += output_pitch; + } +} + +void aom_filter_block1d8_h8_intrin_ssse3( + const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr, + ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) { + __m128i firstFilters, secondFilters, thirdFilters, forthFilters, srcReg; + __m128i filt1Reg, filt2Reg, filt3Reg, filt4Reg; + __m128i srcRegFilt1, srcRegFilt2, srcRegFilt3, srcRegFilt4; + __m128i addFilterReg64, filtersReg, minReg; + unsigned int i; + + // create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64 + addFilterReg64 = _mm_set1_epi32((int)0x0400040u); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + // converting the 16 bit (short) to 8 bit (byte) and have the same data + // in both lanes of 128 bit register. + filtersReg = _mm_packs_epi16(filtersReg, filtersReg); + + // duplicate only the first 16 bits (first and second byte) + // across 128 bit register + firstFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x100u)); + // duplicate only the second 16 bits (third and forth byte) + // across 128 bit register + secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u)); + // duplicate only the third 16 bits (fifth and sixth byte) + // across 128 bit register + thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u)); + // duplicate only the forth 16 bits (seventh and eighth byte) + // across 128 bit register + forthFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x706u)); + + filt1Reg = _mm_load_si128((__m128i const *)filt1_global); + filt2Reg = _mm_load_si128((__m128i const *)filt2_global); + filt3Reg = _mm_load_si128((__m128i const *)filt3_global); + filt4Reg = _mm_load_si128((__m128i const *)filt4_global); + + for (i = 0; i < output_height; i++) { + srcReg = _mm_loadu_si128((const __m128i *)(src_ptr - 3)); + + // filter the source buffer + srcRegFilt1 = _mm_shuffle_epi8(srcReg, filt1Reg); + srcRegFilt2 = _mm_shuffle_epi8(srcReg, filt2Reg); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt1 = _mm_maddubs_epi16(srcRegFilt1, firstFilters); + srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2, secondFilters); + + // filter the source buffer + srcRegFilt3 = _mm_shuffle_epi8(srcReg, filt3Reg); + srcRegFilt4 = _mm_shuffle_epi8(srcReg, filt4Reg); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt3 = _mm_maddubs_epi16(srcRegFilt3, thirdFilters); + srcRegFilt4 = _mm_maddubs_epi16(srcRegFilt4, forthFilters); + + // add and saturate all the results together + minReg = _mm_min_epi16(srcRegFilt2, srcRegFilt3); + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt4); + + srcRegFilt2 = _mm_max_epi16(srcRegFilt2, srcRegFilt3); + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, minReg); + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt2); + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, addFilterReg64); + + // shift by 7 bit each 16 bits + srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 7); + + // shrink to 8 bit each 16 bits + srcRegFilt1 = _mm_packus_epi16(srcRegFilt1, srcRegFilt1); + + src_ptr += src_pixels_per_line; + + // save only 8 bytes + _mm_storel_epi64((__m128i *)&output_ptr[0], srcRegFilt1); + + output_ptr += output_pitch; + } +} + +void aom_filter_block1d8_v8_intrin_ssse3( + const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr, + ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) { + __m128i addFilterReg64, filtersReg, minReg; + __m128i firstFilters, secondFilters, thirdFilters, forthFilters; + __m128i srcRegFilt1, srcRegFilt2, srcRegFilt3, srcRegFilt5; + __m128i srcReg1, srcReg2, srcReg3, srcReg4, srcReg5, srcReg6, srcReg7; + __m128i srcReg8; + unsigned int i; + + // create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64 + addFilterReg64 = _mm_set1_epi32((int)0x0400040u); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + // converting the 16 bit (short) to 8 bit (byte) and have the same data + // in both lanes of 128 bit register. + filtersReg = _mm_packs_epi16(filtersReg, filtersReg); + + // duplicate only the first 16 bits in the filter + firstFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x100u)); + // duplicate only the second 16 bits in the filter + secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u)); + // duplicate only the third 16 bits in the filter + thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u)); + // duplicate only the forth 16 bits in the filter + forthFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x706u)); + + // load the first 7 rows of 8 bytes + srcReg1 = _mm_loadl_epi64((const __m128i *)src_ptr); + srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch)); + srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2)); + srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3)); + srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4)); + srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5)); + srcReg7 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6)); + + for (i = 0; i < output_height; i++) { + // load the last 8 bytes + srcReg8 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 7)); + + // merge the result together + srcRegFilt1 = _mm_unpacklo_epi8(srcReg1, srcReg2); + srcRegFilt3 = _mm_unpacklo_epi8(srcReg3, srcReg4); + + // merge the result together + srcRegFilt2 = _mm_unpacklo_epi8(srcReg5, srcReg6); + srcRegFilt5 = _mm_unpacklo_epi8(srcReg7, srcReg8); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt1 = _mm_maddubs_epi16(srcRegFilt1, firstFilters); + srcRegFilt3 = _mm_maddubs_epi16(srcRegFilt3, secondFilters); + srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2, thirdFilters); + srcRegFilt5 = _mm_maddubs_epi16(srcRegFilt5, forthFilters); + + // add and saturate the results together + minReg = _mm_min_epi16(srcRegFilt2, srcRegFilt3); + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt5); + srcRegFilt2 = _mm_max_epi16(srcRegFilt2, srcRegFilt3); + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, minReg); + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt2); + srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, addFilterReg64); + + // shift by 7 bit each 16 bit + srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 7); + + // shrink to 8 bit each 16 bits + srcRegFilt1 = _mm_packus_epi16(srcRegFilt1, srcRegFilt1); + + src_ptr += src_pitch; + + // shift down a row + srcReg1 = srcReg2; + srcReg2 = srcReg3; + srcReg3 = srcReg4; + srcReg4 = srcReg5; + srcReg5 = srcReg6; + srcReg6 = srcReg7; + srcReg7 = srcReg8; + + // save only 8 bytes convolve result + _mm_storel_epi64((__m128i *)&output_ptr[0], srcRegFilt1); + + output_ptr += out_pitch; + } +} + +filter8_1dfunction aom_filter_block1d16_v8_ssse3; +filter8_1dfunction aom_filter_block1d16_h8_ssse3; +filter8_1dfunction aom_filter_block1d8_v8_ssse3; +filter8_1dfunction aom_filter_block1d8_h8_ssse3; +filter8_1dfunction aom_filter_block1d4_v8_ssse3; +filter8_1dfunction aom_filter_block1d4_h8_ssse3; +filter8_1dfunction aom_filter_block1d16_v8_avg_ssse3; +filter8_1dfunction aom_filter_block1d16_h8_avg_ssse3; +filter8_1dfunction aom_filter_block1d8_v8_avg_ssse3; +filter8_1dfunction aom_filter_block1d8_h8_avg_ssse3; +filter8_1dfunction aom_filter_block1d4_v8_avg_ssse3; +filter8_1dfunction aom_filter_block1d4_h8_avg_ssse3; +#if CONFIG_LOOP_RESTORATION +filter8_1dfunction aom_filter_block1d16_v8_add_src_ssse3; +filter8_1dfunction aom_filter_block1d16_h8_add_src_ssse3; +filter8_1dfunction aom_filter_block1d8_v8_add_src_ssse3; +filter8_1dfunction aom_filter_block1d8_h8_add_src_ssse3; +filter8_1dfunction aom_filter_block1d4_v8_add_src_ssse3; +filter8_1dfunction aom_filter_block1d4_h8_add_src_ssse3; +#endif + +filter8_1dfunction aom_filter_block1d16_v2_ssse3; +filter8_1dfunction aom_filter_block1d16_h2_ssse3; +filter8_1dfunction aom_filter_block1d8_v2_ssse3; +filter8_1dfunction aom_filter_block1d8_h2_ssse3; +filter8_1dfunction aom_filter_block1d4_v2_ssse3; +filter8_1dfunction aom_filter_block1d4_h2_ssse3; +filter8_1dfunction aom_filter_block1d16_v2_avg_ssse3; +filter8_1dfunction aom_filter_block1d16_h2_avg_ssse3; +filter8_1dfunction aom_filter_block1d8_v2_avg_ssse3; +filter8_1dfunction aom_filter_block1d8_h2_avg_ssse3; +filter8_1dfunction aom_filter_block1d4_v2_avg_ssse3; +filter8_1dfunction aom_filter_block1d4_h2_avg_ssse3; + +// void aom_convolve8_horiz_ssse3(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +// void aom_convolve8_vert_ssse3(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +// void aom_convolve8_avg_horiz_ssse3(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +// void aom_convolve8_avg_vert_ssse3(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , ssse3); +FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , ssse3); +FUN_CONV_1D(avg_horiz, x_step_q4, filter_x, h, src, avg_, ssse3); +FUN_CONV_1D(avg_vert, y_step_q4, filter_y, v, src - src_stride * 3, avg_, + ssse3); + +#if CONFIG_LOOP_RESTORATION +FUN_CONV_1D_NO_BILINEAR(add_src_horiz, x_step_q4, filter_x, h, src, add_src_, + ssse3); +FUN_CONV_1D_NO_BILINEAR(add_src_vert, y_step_q4, filter_y, v, + src - src_stride * 3, add_src_, ssse3); +#endif + +#define TRANSPOSE_8X8(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3, out4, out5, out6, out7) \ + { \ + const __m128i tr0_0 = _mm_unpacklo_epi8(in0, in1); \ + const __m128i tr0_1 = _mm_unpacklo_epi8(in2, in3); \ + const __m128i tr0_2 = _mm_unpacklo_epi8(in4, in5); \ + const __m128i tr0_3 = _mm_unpacklo_epi8(in6, in7); \ + \ + const __m128i tr1_0 = _mm_unpacklo_epi16(tr0_0, tr0_1); \ + const __m128i tr1_1 = _mm_unpackhi_epi16(tr0_0, tr0_1); \ + const __m128i tr1_2 = _mm_unpacklo_epi16(tr0_2, tr0_3); \ + const __m128i tr1_3 = _mm_unpackhi_epi16(tr0_2, tr0_3); \ + \ + const __m128i tr2_0 = _mm_unpacklo_epi32(tr1_0, tr1_2); \ + const __m128i tr2_1 = _mm_unpackhi_epi32(tr1_0, tr1_2); \ + const __m128i tr2_2 = _mm_unpacklo_epi32(tr1_1, tr1_3); \ + const __m128i tr2_3 = _mm_unpackhi_epi32(tr1_1, tr1_3); \ + \ + out0 = _mm_unpacklo_epi64(tr2_0, tr2_0); \ + out1 = _mm_unpackhi_epi64(tr2_0, tr2_0); \ + out2 = _mm_unpacklo_epi64(tr2_1, tr2_1); \ + out3 = _mm_unpackhi_epi64(tr2_1, tr2_1); \ + out4 = _mm_unpacklo_epi64(tr2_2, tr2_2); \ + out5 = _mm_unpackhi_epi64(tr2_2, tr2_2); \ + out6 = _mm_unpacklo_epi64(tr2_3, tr2_3); \ + out7 = _mm_unpackhi_epi64(tr2_3, tr2_3); \ + } + +static void filter_horiz_w8_ssse3(const uint8_t *src_x, ptrdiff_t src_pitch, + uint8_t *dst, const int16_t *x_filter) { + const __m128i k_256 = _mm_set1_epi16(1 << 8); + const __m128i f_values = _mm_load_si128((const __m128i *)x_filter); + // pack and duplicate the filter values + const __m128i f1f0 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0200u)); + const __m128i f3f2 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0604u)); + const __m128i f5f4 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0a08u)); + const __m128i f7f6 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0e0cu)); + const __m128i A = _mm_loadl_epi64((const __m128i *)src_x); + const __m128i B = _mm_loadl_epi64((const __m128i *)(src_x + src_pitch)); + const __m128i C = _mm_loadl_epi64((const __m128i *)(src_x + src_pitch * 2)); + const __m128i D = _mm_loadl_epi64((const __m128i *)(src_x + src_pitch * 3)); + const __m128i E = _mm_loadl_epi64((const __m128i *)(src_x + src_pitch * 4)); + const __m128i F = _mm_loadl_epi64((const __m128i *)(src_x + src_pitch * 5)); + const __m128i G = _mm_loadl_epi64((const __m128i *)(src_x + src_pitch * 6)); + const __m128i H = _mm_loadl_epi64((const __m128i *)(src_x + src_pitch * 7)); + // 00 01 10 11 02 03 12 13 04 05 14 15 06 07 16 17 + const __m128i tr0_0 = _mm_unpacklo_epi16(A, B); + // 20 21 30 31 22 23 32 33 24 25 34 35 26 27 36 37 + const __m128i tr0_1 = _mm_unpacklo_epi16(C, D); + // 40 41 50 51 42 43 52 53 44 45 54 55 46 47 56 57 + const __m128i tr0_2 = _mm_unpacklo_epi16(E, F); + // 60 61 70 71 62 63 72 73 64 65 74 75 66 67 76 77 + const __m128i tr0_3 = _mm_unpacklo_epi16(G, H); + // 00 01 10 11 20 21 30 31 02 03 12 13 22 23 32 33 + const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); + // 04 05 14 15 24 25 34 35 06 07 16 17 26 27 36 37 + const __m128i tr1_1 = _mm_unpackhi_epi32(tr0_0, tr0_1); + // 40 41 50 51 60 61 70 71 42 43 52 53 62 63 72 73 + const __m128i tr1_2 = _mm_unpacklo_epi32(tr0_2, tr0_3); + // 44 45 54 55 64 65 74 75 46 47 56 57 66 67 76 77 + const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3); + // 00 01 10 11 20 21 30 31 40 41 50 51 60 61 70 71 + const __m128i s1s0 = _mm_unpacklo_epi64(tr1_0, tr1_2); + const __m128i s3s2 = _mm_unpackhi_epi64(tr1_0, tr1_2); + const __m128i s5s4 = _mm_unpacklo_epi64(tr1_1, tr1_3); + const __m128i s7s6 = _mm_unpackhi_epi64(tr1_1, tr1_3); + // multiply 2 adjacent elements with the filter and add the result + const __m128i x0 = _mm_maddubs_epi16(s1s0, f1f0); + const __m128i x1 = _mm_maddubs_epi16(s3s2, f3f2); + const __m128i x2 = _mm_maddubs_epi16(s5s4, f5f4); + const __m128i x3 = _mm_maddubs_epi16(s7s6, f7f6); + // add and saturate the results together + const __m128i min_x2x1 = _mm_min_epi16(x2, x1); + const __m128i max_x2x1 = _mm_max_epi16(x2, x1); + __m128i temp = _mm_adds_epi16(x0, x3); + temp = _mm_adds_epi16(temp, min_x2x1); + temp = _mm_adds_epi16(temp, max_x2x1); + // round and shift by 7 bit each 16 bit + temp = _mm_mulhrs_epi16(temp, k_256); + // shrink to 8 bit each 16 bits + temp = _mm_packus_epi16(temp, temp); + // save only 8 bytes convolve result + _mm_storel_epi64((__m128i *)dst, temp); +} + +static void transpose8x8_to_dst(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride) { + __m128i A, B, C, D, E, F, G, H; + + A = _mm_loadl_epi64((const __m128i *)src); + B = _mm_loadl_epi64((const __m128i *)(src + src_stride)); + C = _mm_loadl_epi64((const __m128i *)(src + src_stride * 2)); + D = _mm_loadl_epi64((const __m128i *)(src + src_stride * 3)); + E = _mm_loadl_epi64((const __m128i *)(src + src_stride * 4)); + F = _mm_loadl_epi64((const __m128i *)(src + src_stride * 5)); + G = _mm_loadl_epi64((const __m128i *)(src + src_stride * 6)); + H = _mm_loadl_epi64((const __m128i *)(src + src_stride * 7)); + + TRANSPOSE_8X8(A, B, C, D, E, F, G, H, A, B, C, D, E, F, G, H); + + _mm_storel_epi64((__m128i *)dst, A); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 1), B); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 2), C); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 3), D); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 4), E); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 5), F); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 6), G); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 7), H); +} + +static void scaledconvolve_horiz_w8(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *x_filters, int x0_q4, + int x_step_q4, int w, int h) { + DECLARE_ALIGNED(16, uint8_t, temp[8 * 8]); + int x, y, z; + src -= SUBPEL_TAPS / 2 - 1; + + // This function processes 8x8 areas. The intermediate height is not always + // a multiple of 8, so force it to be a multiple of 8 here. + y = h + (8 - (h & 0x7)); + + do { + int x_q4 = x0_q4; + for (x = 0; x < w; x += 8) { + // process 8 src_x steps + for (z = 0; z < 8; ++z) { + const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS]; + const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK]; + if (x_q4 & SUBPEL_MASK) { + filter_horiz_w8_ssse3(src_x, src_stride, temp + (z * 8), x_filter); + } else { + int i; + for (i = 0; i < 8; ++i) { + temp[z * 8 + i] = src_x[i * src_stride + 3]; + } + } + x_q4 += x_step_q4; + } + + // transpose the 8x8 filters values back to dst + transpose8x8_to_dst(temp, 8, dst + x, dst_stride); + } + + src += src_stride * 8; + dst += dst_stride * 8; + } while (y -= 8); +} + +static void filter_horiz_w4_ssse3(const uint8_t *src_ptr, ptrdiff_t src_pitch, + uint8_t *dst, const int16_t *filter) { + const __m128i k_256 = _mm_set1_epi16(1 << 8); + const __m128i f_values = _mm_load_si128((const __m128i *)filter); + // pack and duplicate the filter values + const __m128i f1f0 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0200u)); + const __m128i f3f2 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0604u)); + const __m128i f5f4 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0a08u)); + const __m128i f7f6 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0e0cu)); + const __m128i A = _mm_loadl_epi64((const __m128i *)src_ptr); + const __m128i B = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch)); + const __m128i C = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2)); + const __m128i D = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3)); + // TRANSPOSE... + // 00 01 02 03 04 05 06 07 + // 10 11 12 13 14 15 16 17 + // 20 21 22 23 24 25 26 27 + // 30 31 32 33 34 35 36 37 + // + // TO + // + // 00 10 20 30 + // 01 11 21 31 + // 02 12 22 32 + // 03 13 23 33 + // 04 14 24 34 + // 05 15 25 35 + // 06 16 26 36 + // 07 17 27 37 + // + // 00 01 10 11 02 03 12 13 04 05 14 15 06 07 16 17 + const __m128i tr0_0 = _mm_unpacklo_epi16(A, B); + // 20 21 30 31 22 23 32 33 24 25 34 35 26 27 36 37 + const __m128i tr0_1 = _mm_unpacklo_epi16(C, D); + // 00 01 10 11 20 21 30 31 02 03 12 13 22 23 32 33 + const __m128i s1s0 = _mm_unpacklo_epi32(tr0_0, tr0_1); + // 04 05 14 15 24 25 34 35 06 07 16 17 26 27 36 37 + const __m128i s5s4 = _mm_unpackhi_epi32(tr0_0, tr0_1); + // 02 03 12 13 22 23 32 33 + const __m128i s3s2 = _mm_srli_si128(s1s0, 8); + // 06 07 16 17 26 27 36 37 + const __m128i s7s6 = _mm_srli_si128(s5s4, 8); + // multiply 2 adjacent elements with the filter and add the result + const __m128i x0 = _mm_maddubs_epi16(s1s0, f1f0); + const __m128i x1 = _mm_maddubs_epi16(s3s2, f3f2); + const __m128i x2 = _mm_maddubs_epi16(s5s4, f5f4); + const __m128i x3 = _mm_maddubs_epi16(s7s6, f7f6); + // add and saturate the results together + const __m128i min_x2x1 = _mm_min_epi16(x2, x1); + const __m128i max_x2x1 = _mm_max_epi16(x2, x1); + __m128i temp = _mm_adds_epi16(x0, x3); + temp = _mm_adds_epi16(temp, min_x2x1); + temp = _mm_adds_epi16(temp, max_x2x1); + // round and shift by 7 bit each 16 bit + temp = _mm_mulhrs_epi16(temp, k_256); + // shrink to 8 bit each 16 bits + temp = _mm_packus_epi16(temp, temp); + // save only 4 bytes + *(int *)dst = _mm_cvtsi128_si32(temp); +} + +static void transpose4x4_to_dst(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride) { + __m128i A = _mm_cvtsi32_si128(*(const int *)src); + __m128i B = _mm_cvtsi32_si128(*(const int *)(src + src_stride)); + __m128i C = _mm_cvtsi32_si128(*(const int *)(src + src_stride * 2)); + __m128i D = _mm_cvtsi32_si128(*(const int *)(src + src_stride * 3)); + // 00 10 01 11 02 12 03 13 + const __m128i tr0_0 = _mm_unpacklo_epi8(A, B); + // 20 30 21 31 22 32 23 33 + const __m128i tr0_1 = _mm_unpacklo_epi8(C, D); + // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 + A = _mm_unpacklo_epi16(tr0_0, tr0_1); + B = _mm_srli_si128(A, 4); + C = _mm_srli_si128(A, 8); + D = _mm_srli_si128(A, 12); + + *(int *)(dst) = _mm_cvtsi128_si32(A); + *(int *)(dst + dst_stride) = _mm_cvtsi128_si32(B); + *(int *)(dst + dst_stride * 2) = _mm_cvtsi128_si32(C); + *(int *)(dst + dst_stride * 3) = _mm_cvtsi128_si32(D); +} + +static void scaledconvolve_horiz_w4(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *x_filters, int x0_q4, + int x_step_q4, int w, int h) { + DECLARE_ALIGNED(16, uint8_t, temp[4 * 4]); + int x, y, z; + src -= SUBPEL_TAPS / 2 - 1; + + for (y = 0; y < h; y += 4) { + int x_q4 = x0_q4; + for (x = 0; x < w; x += 4) { + // process 4 src_x steps + for (z = 0; z < 4; ++z) { + const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS]; + const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK]; + if (x_q4 & SUBPEL_MASK) { + filter_horiz_w4_ssse3(src_x, src_stride, temp + (z * 4), x_filter); + } else { + int i; + for (i = 0; i < 4; ++i) { + temp[z * 4 + i] = src_x[i * src_stride + 3]; + } + } + x_q4 += x_step_q4; + } + + // transpose the 4x4 filters values back to dst + transpose4x4_to_dst(temp, 4, dst + x, dst_stride); + } + + src += src_stride * 4; + dst += dst_stride * 4; + } +} + +static void filter_vert_w4_ssse3(const uint8_t *src_ptr, ptrdiff_t src_pitch, + uint8_t *dst, const int16_t *filter) { + const __m128i k_256 = _mm_set1_epi16(1 << 8); + const __m128i f_values = _mm_load_si128((const __m128i *)filter); + // pack and duplicate the filter values + const __m128i f1f0 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0200u)); + const __m128i f3f2 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0604u)); + const __m128i f5f4 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0a08u)); + const __m128i f7f6 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0e0cu)); + const __m128i A = _mm_cvtsi32_si128(*(const int *)src_ptr); + const __m128i B = _mm_cvtsi32_si128(*(const int *)(src_ptr + src_pitch)); + const __m128i C = _mm_cvtsi32_si128(*(const int *)(src_ptr + src_pitch * 2)); + const __m128i D = _mm_cvtsi32_si128(*(const int *)(src_ptr + src_pitch * 3)); + const __m128i E = _mm_cvtsi32_si128(*(const int *)(src_ptr + src_pitch * 4)); + const __m128i F = _mm_cvtsi32_si128(*(const int *)(src_ptr + src_pitch * 5)); + const __m128i G = _mm_cvtsi32_si128(*(const int *)(src_ptr + src_pitch * 6)); + const __m128i H = _mm_cvtsi32_si128(*(const int *)(src_ptr + src_pitch * 7)); + const __m128i s1s0 = _mm_unpacklo_epi8(A, B); + const __m128i s3s2 = _mm_unpacklo_epi8(C, D); + const __m128i s5s4 = _mm_unpacklo_epi8(E, F); + const __m128i s7s6 = _mm_unpacklo_epi8(G, H); + // multiply 2 adjacent elements with the filter and add the result + const __m128i x0 = _mm_maddubs_epi16(s1s0, f1f0); + const __m128i x1 = _mm_maddubs_epi16(s3s2, f3f2); + const __m128i x2 = _mm_maddubs_epi16(s5s4, f5f4); + const __m128i x3 = _mm_maddubs_epi16(s7s6, f7f6); + // add and saturate the results together + const __m128i min_x2x1 = _mm_min_epi16(x2, x1); + const __m128i max_x2x1 = _mm_max_epi16(x2, x1); + __m128i temp = _mm_adds_epi16(x0, x3); + temp = _mm_adds_epi16(temp, min_x2x1); + temp = _mm_adds_epi16(temp, max_x2x1); + // round and shift by 7 bit each 16 bit + temp = _mm_mulhrs_epi16(temp, k_256); + // shrink to 8 bit each 16 bits + temp = _mm_packus_epi16(temp, temp); + // save only 4 bytes + *(int *)dst = _mm_cvtsi128_si32(temp); +} + +static void scaledconvolve_vert_w4(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *y_filters, int y0_q4, + int y_step_q4, int w, int h) { + int y; + int y_q4 = y0_q4; + + src -= src_stride * (SUBPEL_TAPS / 2 - 1); + for (y = 0; y < h; ++y) { + const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; + const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK]; + + if (y_q4 & SUBPEL_MASK) { + filter_vert_w4_ssse3(src_y, src_stride, &dst[y * dst_stride], y_filter); + } else { + memcpy(&dst[y * dst_stride], &src_y[3 * src_stride], w); + } + + y_q4 += y_step_q4; + } +} + +static void filter_vert_w8_ssse3(const uint8_t *src_ptr, ptrdiff_t src_pitch, + uint8_t *dst, const int16_t *filter) { + const __m128i k_256 = _mm_set1_epi16(1 << 8); + const __m128i f_values = _mm_load_si128((const __m128i *)filter); + // pack and duplicate the filter values + const __m128i f1f0 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0200u)); + const __m128i f3f2 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0604u)); + const __m128i f5f4 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0a08u)); + const __m128i f7f6 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0e0cu)); + const __m128i A = _mm_loadl_epi64((const __m128i *)src_ptr); + const __m128i B = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch)); + const __m128i C = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2)); + const __m128i D = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3)); + const __m128i E = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4)); + const __m128i F = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5)); + const __m128i G = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6)); + const __m128i H = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 7)); + const __m128i s1s0 = _mm_unpacklo_epi8(A, B); + const __m128i s3s2 = _mm_unpacklo_epi8(C, D); + const __m128i s5s4 = _mm_unpacklo_epi8(E, F); + const __m128i s7s6 = _mm_unpacklo_epi8(G, H); + // multiply 2 adjacent elements with the filter and add the result + const __m128i x0 = _mm_maddubs_epi16(s1s0, f1f0); + const __m128i x1 = _mm_maddubs_epi16(s3s2, f3f2); + const __m128i x2 = _mm_maddubs_epi16(s5s4, f5f4); + const __m128i x3 = _mm_maddubs_epi16(s7s6, f7f6); + // add and saturate the results together + const __m128i min_x2x1 = _mm_min_epi16(x2, x1); + const __m128i max_x2x1 = _mm_max_epi16(x2, x1); + __m128i temp = _mm_adds_epi16(x0, x3); + temp = _mm_adds_epi16(temp, min_x2x1); + temp = _mm_adds_epi16(temp, max_x2x1); + // round and shift by 7 bit each 16 bit + temp = _mm_mulhrs_epi16(temp, k_256); + // shrink to 8 bit each 16 bits + temp = _mm_packus_epi16(temp, temp); + // save only 8 bytes convolve result + _mm_storel_epi64((__m128i *)dst, temp); +} + +static void scaledconvolve_vert_w8(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *y_filters, int y0_q4, + int y_step_q4, int w, int h) { + int y; + int y_q4 = y0_q4; + + src -= src_stride * (SUBPEL_TAPS / 2 - 1); + for (y = 0; y < h; ++y) { + const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; + const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK]; + if (y_q4 & SUBPEL_MASK) { + filter_vert_w8_ssse3(src_y, src_stride, &dst[y * dst_stride], y_filter); + } else { + memcpy(&dst[y * dst_stride], &src_y[3 * src_stride], w); + } + y_q4 += y_step_q4; + } +} + +static void filter_vert_w16_ssse3(const uint8_t *src_ptr, ptrdiff_t src_pitch, + uint8_t *dst, const int16_t *filter, int w) { + const __m128i k_256 = _mm_set1_epi16(1 << 8); + const __m128i f_values = _mm_load_si128((const __m128i *)filter); + // pack and duplicate the filter values + const __m128i f1f0 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0200u)); + const __m128i f3f2 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0604u)); + const __m128i f5f4 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0a08u)); + const __m128i f7f6 = _mm_shuffle_epi8(f_values, _mm_set1_epi16(0x0e0cu)); + int i; + + for (i = 0; i < w; i += 16) { + const __m128i A = _mm_loadu_si128((const __m128i *)src_ptr); + const __m128i B = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch)); + const __m128i C = + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2)); + const __m128i D = + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3)); + const __m128i E = + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4)); + const __m128i F = + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5)); + const __m128i G = + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6)); + const __m128i H = + _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 7)); + // merge the result together + const __m128i s1s0_lo = _mm_unpacklo_epi8(A, B); + const __m128i s7s6_lo = _mm_unpacklo_epi8(G, H); + const __m128i s1s0_hi = _mm_unpackhi_epi8(A, B); + const __m128i s7s6_hi = _mm_unpackhi_epi8(G, H); + // multiply 2 adjacent elements with the filter and add the result + const __m128i x0_lo = _mm_maddubs_epi16(s1s0_lo, f1f0); + const __m128i x3_lo = _mm_maddubs_epi16(s7s6_lo, f7f6); + const __m128i x0_hi = _mm_maddubs_epi16(s1s0_hi, f1f0); + const __m128i x3_hi = _mm_maddubs_epi16(s7s6_hi, f7f6); + // add and saturate the results together + const __m128i x3x0_lo = _mm_adds_epi16(x0_lo, x3_lo); + const __m128i x3x0_hi = _mm_adds_epi16(x0_hi, x3_hi); + // merge the result together + const __m128i s3s2_lo = _mm_unpacklo_epi8(C, D); + const __m128i s3s2_hi = _mm_unpackhi_epi8(C, D); + // multiply 2 adjacent elements with the filter and add the result + const __m128i x1_lo = _mm_maddubs_epi16(s3s2_lo, f3f2); + const __m128i x1_hi = _mm_maddubs_epi16(s3s2_hi, f3f2); + // merge the result together + const __m128i s5s4_lo = _mm_unpacklo_epi8(E, F); + const __m128i s5s4_hi = _mm_unpackhi_epi8(E, F); + // multiply 2 adjacent elements with the filter and add the result + const __m128i x2_lo = _mm_maddubs_epi16(s5s4_lo, f5f4); + const __m128i x2_hi = _mm_maddubs_epi16(s5s4_hi, f5f4); + // add and saturate the results together + __m128i temp_lo = _mm_adds_epi16(x3x0_lo, _mm_min_epi16(x1_lo, x2_lo)); + __m128i temp_hi = _mm_adds_epi16(x3x0_hi, _mm_min_epi16(x1_hi, x2_hi)); + + // add and saturate the results together + temp_lo = _mm_adds_epi16(temp_lo, _mm_max_epi16(x1_lo, x2_lo)); + temp_hi = _mm_adds_epi16(temp_hi, _mm_max_epi16(x1_hi, x2_hi)); + // round and shift by 7 bit each 16 bit + temp_lo = _mm_mulhrs_epi16(temp_lo, k_256); + temp_hi = _mm_mulhrs_epi16(temp_hi, k_256); + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + temp_hi = _mm_packus_epi16(temp_lo, temp_hi); + src_ptr += 16; + // save 16 bytes convolve result + _mm_store_si128((__m128i *)&dst[i], temp_hi); + } +} + +static void scaledconvolve_vert_w16(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *y_filters, int y0_q4, + int y_step_q4, int w, int h) { + int y; + int y_q4 = y0_q4; + + src -= src_stride * (SUBPEL_TAPS / 2 - 1); + for (y = 0; y < h; ++y) { + const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; + const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK]; + if (y_q4 & SUBPEL_MASK) { + filter_vert_w16_ssse3(src_y, src_stride, &dst[y * dst_stride], y_filter, + w); + } else { + memcpy(&dst[y * dst_stride], &src_y[3 * src_stride], w); + } + y_q4 += y_step_q4; + } +} + +static void scaledconvolve2d(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *const x_filters, int x0_q4, + int x_step_q4, const InterpKernel *const y_filters, + int y0_q4, int y_step_q4, int w, int h) { + // Note: Fixed size intermediate buffer, temp, places limits on parameters. + // 2d filtering proceeds in 2 steps: + // (1) Interpolate horizontally into an intermediate buffer, temp. + // (2) Interpolate temp vertically to derive the sub-pixel result. + // Deriving the maximum number of rows in the temp buffer (135): + // --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative). + // --Largest block size is 64x64 pixels. + // --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the + // original frame (in 1/16th pixel units). + // --Must round-up because block may be located at sub-pixel position. + // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails. + // --((64 - 1) * 32 + 15) >> 4 + 8 = 135. + // --Require an additional 8 rows for the horiz_w8 transpose tail. + DECLARE_ALIGNED(16, uint8_t, temp[(MAX_EXT_SIZE + 8) * MAX_SB_SIZE]); + const int intermediate_height = + (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS; + + assert(w <= MAX_SB_SIZE); + assert(h <= MAX_SB_SIZE); + assert(y_step_q4 <= 32); + assert(x_step_q4 <= 32); + + if (w >= 8) { + scaledconvolve_horiz_w8(src - src_stride * (SUBPEL_TAPS / 2 - 1), + src_stride, temp, MAX_SB_SIZE, x_filters, x0_q4, + x_step_q4, w, intermediate_height); + } else { + scaledconvolve_horiz_w4(src - src_stride * (SUBPEL_TAPS / 2 - 1), + src_stride, temp, MAX_SB_SIZE, x_filters, x0_q4, + x_step_q4, w, intermediate_height); + } + + if (w >= 16) { + scaledconvolve_vert_w16(temp + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1), + MAX_SB_SIZE, dst, dst_stride, y_filters, y0_q4, + y_step_q4, w, h); + } else if (w == 8) { + scaledconvolve_vert_w8(temp + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1), + MAX_SB_SIZE, dst, dst_stride, y_filters, y0_q4, + y_step_q4, w, h); + } else { + scaledconvolve_vert_w4(temp + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1), + MAX_SB_SIZE, dst, dst_stride, y_filters, y0_q4, + y_step_q4, w, h); + } +} + +static const InterpKernel *get_filter_base(const int16_t *filter) { + // NOTE: This assumes that the filter table is 256-byte aligned. + // TODO(agrange) Modify to make independent of table alignment. + return (const InterpKernel *)(((intptr_t)filter) & ~((intptr_t)0xFF)); +} + +static int get_filter_offset(const int16_t *f, const InterpKernel *base) { + return (int)((const InterpKernel *)(intptr_t)f - base); +} + +void aom_scaled_2d_ssse3(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, + int x_step_q4, const int16_t *filter_y, int y_step_q4, + int w, int h) { + const InterpKernel *const filters_x = get_filter_base(filter_x); + const int x0_q4 = get_filter_offset(filter_x, filters_x); + + const InterpKernel *const filters_y = get_filter_base(filter_y); + const int y0_q4 = get_filter_offset(filter_y, filters_y); + + scaledconvolve2d(src, src_stride, dst, dst_stride, filters_x, x0_q4, + x_step_q4, filters_y, y0_q4, y_step_q4, w, h); +} + +// void aom_convolve8_ssse3(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +// void aom_convolve8_avg_ssse3(const uint8_t *src, ptrdiff_t src_stride, +// uint8_t *dst, ptrdiff_t dst_stride, +// const int16_t *filter_x, int x_step_q4, +// const int16_t *filter_y, int y_step_q4, +// int w, int h); +FUN_CONV_2D(, ssse3); +FUN_CONV_2D(avg_, ssse3); +#if CONFIG_LOOP_RESTORATION +FUN_CONV_2D_NO_BILINEAR(add_src_, add_src_, ssse3); +#endif diff --git a/third_party/aom/aom_dsp/x86/aom_subpixel_8t_sse2.asm b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_sse2.asm new file mode 100644 index 0000000000..b946010d32 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_sse2.asm @@ -0,0 +1,990 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + +%include "aom_ports/x86_abi_support.asm" + +;Note: tap3 and tap4 have to be applied and added after other taps to avoid +;overflow. + +%macro GET_FILTERS_4 0 + mov rdx, arg(5) ;filter ptr + mov rcx, 0x0400040 + + movdqa xmm7, [rdx] ;load filters + pshuflw xmm0, xmm7, 0b ;k0 + pshuflw xmm1, xmm7, 01010101b ;k1 + pshuflw xmm2, xmm7, 10101010b ;k2 + pshuflw xmm3, xmm7, 11111111b ;k3 + psrldq xmm7, 8 + pshuflw xmm4, xmm7, 0b ;k4 + pshuflw xmm5, xmm7, 01010101b ;k5 + pshuflw xmm6, xmm7, 10101010b ;k6 + pshuflw xmm7, xmm7, 11111111b ;k7 + + punpcklqdq xmm0, xmm1 + punpcklqdq xmm2, xmm3 + punpcklqdq xmm5, xmm4 + punpcklqdq xmm6, xmm7 + + movdqa k0k1, xmm0 + movdqa k2k3, xmm2 + movdqa k5k4, xmm5 + movdqa k6k7, xmm6 + + movq xmm6, rcx + pshufd xmm6, xmm6, 0 + movdqa krd, xmm6 + + pxor xmm7, xmm7 + movdqa zero, xmm7 +%endm + +%macro APPLY_FILTER_4 1 + punpckldq xmm0, xmm1 ;two row in one register + punpckldq xmm6, xmm7 + punpckldq xmm2, xmm3 + punpckldq xmm5, xmm4 + + punpcklbw xmm0, zero ;unpack to word + punpcklbw xmm6, zero + punpcklbw xmm2, zero + punpcklbw xmm5, zero + + pmullw xmm0, k0k1 ;multiply the filter factors + pmullw xmm6, k6k7 + pmullw xmm2, k2k3 + pmullw xmm5, k5k4 + + paddsw xmm0, xmm6 ;sum + movdqa xmm1, xmm0 + psrldq xmm1, 8 + paddsw xmm0, xmm1 + paddsw xmm0, xmm2 + psrldq xmm2, 8 + paddsw xmm0, xmm5 + psrldq xmm5, 8 + paddsw xmm0, xmm2 + paddsw xmm0, xmm5 + + paddsw xmm0, krd ;rounding + psraw xmm0, 7 ;shift + packuswb xmm0, xmm0 ;pack to byte + +%if %1 + movd xmm1, [rdi] + pavgb xmm0, xmm1 +%endif + movd [rdi], xmm0 +%endm + +%macro GET_FILTERS 0 + mov rdx, arg(5) ;filter ptr + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + mov rcx, 0x0400040 + + movdqa xmm7, [rdx] ;load filters + pshuflw xmm0, xmm7, 0b ;k0 + pshuflw xmm1, xmm7, 01010101b ;k1 + pshuflw xmm2, xmm7, 10101010b ;k2 + pshuflw xmm3, xmm7, 11111111b ;k3 + pshufhw xmm4, xmm7, 0b ;k4 + pshufhw xmm5, xmm7, 01010101b ;k5 + pshufhw xmm6, xmm7, 10101010b ;k6 + pshufhw xmm7, xmm7, 11111111b ;k7 + + punpcklwd xmm0, xmm0 + punpcklwd xmm1, xmm1 + punpcklwd xmm2, xmm2 + punpcklwd xmm3, xmm3 + punpckhwd xmm4, xmm4 + punpckhwd xmm5, xmm5 + punpckhwd xmm6, xmm6 + punpckhwd xmm7, xmm7 + + movdqa k0, xmm0 ;store filter factors on stack + movdqa k1, xmm1 + movdqa k2, xmm2 + movdqa k3, xmm3 + movdqa k4, xmm4 + movdqa k5, xmm5 + movdqa k6, xmm6 + movdqa k7, xmm7 + + movq xmm6, rcx + pshufd xmm6, xmm6, 0 + movdqa krd, xmm6 ;rounding + + pxor xmm7, xmm7 + movdqa zero, xmm7 +%endm + +%macro LOAD_VERT_8 1 + movq xmm0, [rsi + %1] ;0 + movq xmm1, [rsi + rax + %1] ;1 + movq xmm6, [rsi + rdx * 2 + %1] ;6 + lea rsi, [rsi + rax] + movq xmm7, [rsi + rdx * 2 + %1] ;7 + movq xmm2, [rsi + rax + %1] ;2 + movq xmm3, [rsi + rax * 2 + %1] ;3 + movq xmm4, [rsi + rdx + %1] ;4 + movq xmm5, [rsi + rax * 4 + %1] ;5 +%endm + +%macro APPLY_FILTER_8 2 + punpcklbw xmm0, zero + punpcklbw xmm1, zero + punpcklbw xmm6, zero + punpcklbw xmm7, zero + punpcklbw xmm2, zero + punpcklbw xmm5, zero + punpcklbw xmm3, zero + punpcklbw xmm4, zero + + pmullw xmm0, k0 + pmullw xmm1, k1 + pmullw xmm6, k6 + pmullw xmm7, k7 + pmullw xmm2, k2 + pmullw xmm5, k5 + pmullw xmm3, k3 + pmullw xmm4, k4 + + paddsw xmm0, xmm1 + paddsw xmm0, xmm6 + paddsw xmm0, xmm7 + paddsw xmm0, xmm2 + paddsw xmm0, xmm5 + paddsw xmm0, xmm3 + paddsw xmm0, xmm4 + + paddsw xmm0, krd ;rounding + psraw xmm0, 7 ;shift + packuswb xmm0, xmm0 ;pack back to byte +%if %1 + movq xmm1, [rdi + %2] + pavgb xmm0, xmm1 +%endif + movq [rdi + %2], xmm0 +%endm + +;void aom_filter_block1d4_v8_sse2 +;( +; unsigned char *src_ptr, +; unsigned int src_pitch, +; unsigned char *output_ptr, +; unsigned int out_pitch, +; unsigned int output_height, +; short *filter +;) +global sym(aom_filter_block1d4_v8_sse2) PRIVATE +sym(aom_filter_block1d4_v8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + push rbx + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 6 + %define k0k1 [rsp + 16 * 0] + %define k2k3 [rsp + 16 * 1] + %define k5k4 [rsp + 16 * 2] + %define k6k7 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define zero [rsp + 16 * 5] + + GET_FILTERS_4 + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rbx, DWORD PTR arg(3) ;out_pitch + lea rdx, [rax + rax * 2] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movd xmm0, [rsi] ;load src: row 0 + movd xmm1, [rsi + rax] ;1 + movd xmm6, [rsi + rdx * 2] ;6 + lea rsi, [rsi + rax] + movd xmm7, [rsi + rdx * 2] ;7 + movd xmm2, [rsi + rax] ;2 + movd xmm3, [rsi + rax * 2] ;3 + movd xmm4, [rsi + rdx] ;4 + movd xmm5, [rsi + rax * 4] ;5 + + APPLY_FILTER_4 0 + + lea rdi, [rdi + rbx] + dec rcx + jnz .loop + + add rsp, 16 * 6 + pop rsp + pop rbx + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void aom_filter_block1d8_v8_sse2 +;( +; unsigned char *src_ptr, +; unsigned int src_pitch, +; unsigned char *output_ptr, +; unsigned int out_pitch, +; unsigned int output_height, +; short *filter +;) +global sym(aom_filter_block1d8_v8_sse2) PRIVATE +sym(aom_filter_block1d8_v8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + push rbx + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 10 + %define k0 [rsp + 16 * 0] + %define k1 [rsp + 16 * 1] + %define k2 [rsp + 16 * 2] + %define k3 [rsp + 16 * 3] + %define k4 [rsp + 16 * 4] + %define k5 [rsp + 16 * 5] + %define k6 [rsp + 16 * 6] + %define k7 [rsp + 16 * 7] + %define krd [rsp + 16 * 8] + %define zero [rsp + 16 * 9] + + GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rbx, DWORD PTR arg(3) ;out_pitch + lea rdx, [rax + rax * 2] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + LOAD_VERT_8 0 + APPLY_FILTER_8 0, 0 + + lea rdi, [rdi + rbx] + dec rcx + jnz .loop + + add rsp, 16 * 10 + pop rsp + pop rbx + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void aom_filter_block1d16_v8_sse2 +;( +; unsigned char *src_ptr, +; unsigned int src_pitch, +; unsigned char *output_ptr, +; unsigned int out_pitch, +; unsigned int output_height, +; short *filter +;) +global sym(aom_filter_block1d16_v8_sse2) PRIVATE +sym(aom_filter_block1d16_v8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + push rbx + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 10 + %define k0 [rsp + 16 * 0] + %define k1 [rsp + 16 * 1] + %define k2 [rsp + 16 * 2] + %define k3 [rsp + 16 * 3] + %define k4 [rsp + 16 * 4] + %define k5 [rsp + 16 * 5] + %define k6 [rsp + 16 * 6] + %define k7 [rsp + 16 * 7] + %define krd [rsp + 16 * 8] + %define zero [rsp + 16 * 9] + + GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rbx, DWORD PTR arg(3) ;out_pitch + lea rdx, [rax + rax * 2] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + LOAD_VERT_8 0 + APPLY_FILTER_8 0, 0 + sub rsi, rax + + LOAD_VERT_8 8 + APPLY_FILTER_8 0, 8 + add rdi, rbx + + dec rcx + jnz .loop + + add rsp, 16 * 10 + pop rsp + pop rbx + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d4_v8_avg_sse2) PRIVATE +sym(aom_filter_block1d4_v8_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + push rbx + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 6 + %define k0k1 [rsp + 16 * 0] + %define k2k3 [rsp + 16 * 1] + %define k5k4 [rsp + 16 * 2] + %define k6k7 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define zero [rsp + 16 * 5] + + GET_FILTERS_4 + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rbx, DWORD PTR arg(3) ;out_pitch + lea rdx, [rax + rax * 2] + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movd xmm0, [rsi] ;load src: row 0 + movd xmm1, [rsi + rax] ;1 + movd xmm6, [rsi + rdx * 2] ;6 + lea rsi, [rsi + rax] + movd xmm7, [rsi + rdx * 2] ;7 + movd xmm2, [rsi + rax] ;2 + movd xmm3, [rsi + rax * 2] ;3 + movd xmm4, [rsi + rdx] ;4 + movd xmm5, [rsi + rax * 4] ;5 + + APPLY_FILTER_4 1 + + lea rdi, [rdi + rbx] + dec rcx + jnz .loop + + add rsp, 16 * 6 + pop rsp + pop rbx + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d8_v8_avg_sse2) PRIVATE +sym(aom_filter_block1d8_v8_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + push rbx + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 10 + %define k0 [rsp + 16 * 0] + %define k1 [rsp + 16 * 1] + %define k2 [rsp + 16 * 2] + %define k3 [rsp + 16 * 3] + %define k4 [rsp + 16 * 4] + %define k5 [rsp + 16 * 5] + %define k6 [rsp + 16 * 6] + %define k7 [rsp + 16 * 7] + %define krd [rsp + 16 * 8] + %define zero [rsp + 16 * 9] + + GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rbx, DWORD PTR arg(3) ;out_pitch + lea rdx, [rax + rax * 2] + movsxd rcx, DWORD PTR arg(4) ;output_height +.loop: + LOAD_VERT_8 0 + APPLY_FILTER_8 1, 0 + + lea rdi, [rdi + rbx] + dec rcx + jnz .loop + + add rsp, 16 * 10 + pop rsp + pop rbx + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d16_v8_avg_sse2) PRIVATE +sym(aom_filter_block1d16_v8_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + push rbx + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 10 + %define k0 [rsp + 16 * 0] + %define k1 [rsp + 16 * 1] + %define k2 [rsp + 16 * 2] + %define k3 [rsp + 16 * 3] + %define k4 [rsp + 16 * 4] + %define k5 [rsp + 16 * 5] + %define k6 [rsp + 16 * 6] + %define k7 [rsp + 16 * 7] + %define krd [rsp + 16 * 8] + %define zero [rsp + 16 * 9] + + GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rbx, DWORD PTR arg(3) ;out_pitch + lea rdx, [rax + rax * 2] + movsxd rcx, DWORD PTR arg(4) ;output_height +.loop: + LOAD_VERT_8 0 + APPLY_FILTER_8 1, 0 + sub rsi, rax + + LOAD_VERT_8 8 + APPLY_FILTER_8 1, 8 + add rdi, rbx + + dec rcx + jnz .loop + + add rsp, 16 * 10 + pop rsp + pop rbx + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void aom_filter_block1d4_h8_sse2 +;( +; unsigned char *src_ptr, +; unsigned int src_pixels_per_line, +; unsigned char *output_ptr, +; unsigned int output_pitch, +; unsigned int output_height, +; short *filter +;) +global sym(aom_filter_block1d4_h8_sse2) PRIVATE +sym(aom_filter_block1d4_h8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 6 + %define k0k1 [rsp + 16 * 0] + %define k2k3 [rsp + 16 * 1] + %define k5k4 [rsp + 16 * 2] + %define k6k7 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define zero [rsp + 16 * 5] + + GET_FILTERS_4 + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movdqu xmm0, [rsi - 3] ;load src + + movdqa xmm1, xmm0 + movdqa xmm6, xmm0 + movdqa xmm7, xmm0 + movdqa xmm2, xmm0 + movdqa xmm3, xmm0 + movdqa xmm5, xmm0 + movdqa xmm4, xmm0 + + psrldq xmm1, 1 + psrldq xmm6, 6 + psrldq xmm7, 7 + psrldq xmm2, 2 + psrldq xmm3, 3 + psrldq xmm5, 5 + psrldq xmm4, 4 + + APPLY_FILTER_4 0 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx + jnz .loop + + add rsp, 16 * 6 + pop rsp + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void aom_filter_block1d8_h8_sse2 +;( +; unsigned char *src_ptr, +; unsigned int src_pixels_per_line, +; unsigned char *output_ptr, +; unsigned int output_pitch, +; unsigned int output_height, +; short *filter +;) +global sym(aom_filter_block1d8_h8_sse2) PRIVATE +sym(aom_filter_block1d8_h8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 10 + %define k0 [rsp + 16 * 0] + %define k1 [rsp + 16 * 1] + %define k2 [rsp + 16 * 2] + %define k3 [rsp + 16 * 3] + %define k4 [rsp + 16 * 4] + %define k5 [rsp + 16 * 5] + %define k6 [rsp + 16 * 6] + %define k7 [rsp + 16 * 7] + %define krd [rsp + 16 * 8] + %define zero [rsp + 16 * 9] + + GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movdqu xmm0, [rsi - 3] ;load src + + movdqa xmm1, xmm0 + movdqa xmm6, xmm0 + movdqa xmm7, xmm0 + movdqa xmm2, xmm0 + movdqa xmm5, xmm0 + movdqa xmm3, xmm0 + movdqa xmm4, xmm0 + + psrldq xmm1, 1 + psrldq xmm6, 6 + psrldq xmm7, 7 + psrldq xmm2, 2 + psrldq xmm5, 5 + psrldq xmm3, 3 + psrldq xmm4, 4 + + APPLY_FILTER_8 0, 0 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx + jnz .loop + + add rsp, 16 * 10 + pop rsp + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void aom_filter_block1d16_h8_sse2 +;( +; unsigned char *src_ptr, +; unsigned int src_pixels_per_line, +; unsigned char *output_ptr, +; unsigned int output_pitch, +; unsigned int output_height, +; short *filter +;) +global sym(aom_filter_block1d16_h8_sse2) PRIVATE +sym(aom_filter_block1d16_h8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 10 + %define k0 [rsp + 16 * 0] + %define k1 [rsp + 16 * 1] + %define k2 [rsp + 16 * 2] + %define k3 [rsp + 16 * 3] + %define k4 [rsp + 16 * 4] + %define k5 [rsp + 16 * 5] + %define k6 [rsp + 16 * 6] + %define k7 [rsp + 16 * 7] + %define krd [rsp + 16 * 8] + %define zero [rsp + 16 * 9] + + GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movdqu xmm0, [rsi - 3] ;load src + + movdqa xmm1, xmm0 + movdqa xmm6, xmm0 + movdqa xmm7, xmm0 + movdqa xmm2, xmm0 + movdqa xmm5, xmm0 + movdqa xmm3, xmm0 + movdqa xmm4, xmm0 + + psrldq xmm1, 1 + psrldq xmm6, 6 + psrldq xmm7, 7 + psrldq xmm2, 2 + psrldq xmm5, 5 + psrldq xmm3, 3 + psrldq xmm4, 4 + + APPLY_FILTER_8 0, 0 + + movdqu xmm0, [rsi + 5] ;load src + + movdqa xmm1, xmm0 + movdqa xmm6, xmm0 + movdqa xmm7, xmm0 + movdqa xmm2, xmm0 + movdqa xmm5, xmm0 + movdqa xmm3, xmm0 + movdqa xmm4, xmm0 + + psrldq xmm1, 1 + psrldq xmm6, 6 + psrldq xmm7, 7 + psrldq xmm2, 2 + psrldq xmm5, 5 + psrldq xmm3, 3 + psrldq xmm4, 4 + + APPLY_FILTER_8 0, 8 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx + jnz .loop + + add rsp, 16 * 10 + pop rsp + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d4_h8_avg_sse2) PRIVATE +sym(aom_filter_block1d4_h8_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 6 + %define k0k1 [rsp + 16 * 0] + %define k2k3 [rsp + 16 * 1] + %define k5k4 [rsp + 16 * 2] + %define k6k7 [rsp + 16 * 3] + %define krd [rsp + 16 * 4] + %define zero [rsp + 16 * 5] + + GET_FILTERS_4 + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movdqu xmm0, [rsi - 3] ;load src + + movdqa xmm1, xmm0 + movdqa xmm6, xmm0 + movdqa xmm7, xmm0 + movdqa xmm2, xmm0 + movdqa xmm3, xmm0 + movdqa xmm5, xmm0 + movdqa xmm4, xmm0 + + psrldq xmm1, 1 + psrldq xmm6, 6 + psrldq xmm7, 7 + psrldq xmm2, 2 + psrldq xmm3, 3 + psrldq xmm5, 5 + psrldq xmm4, 4 + + APPLY_FILTER_4 1 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx + jnz .loop + + add rsp, 16 * 6 + pop rsp + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d8_h8_avg_sse2) PRIVATE +sym(aom_filter_block1d8_h8_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 10 + %define k0 [rsp + 16 * 0] + %define k1 [rsp + 16 * 1] + %define k2 [rsp + 16 * 2] + %define k3 [rsp + 16 * 3] + %define k4 [rsp + 16 * 4] + %define k5 [rsp + 16 * 5] + %define k6 [rsp + 16 * 6] + %define k7 [rsp + 16 * 7] + %define krd [rsp + 16 * 8] + %define zero [rsp + 16 * 9] + + GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movdqu xmm0, [rsi - 3] ;load src + + movdqa xmm1, xmm0 + movdqa xmm6, xmm0 + movdqa xmm7, xmm0 + movdqa xmm2, xmm0 + movdqa xmm5, xmm0 + movdqa xmm3, xmm0 + movdqa xmm4, xmm0 + + psrldq xmm1, 1 + psrldq xmm6, 6 + psrldq xmm7, 7 + psrldq xmm2, 2 + psrldq xmm5, 5 + psrldq xmm3, 3 + psrldq xmm4, 4 + + APPLY_FILTER_8 1, 0 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx + jnz .loop + + add rsp, 16 * 10 + pop rsp + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d16_h8_avg_sse2) PRIVATE +sym(aom_filter_block1d16_h8_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + ALIGN_STACK 16, rax + sub rsp, 16 * 10 + %define k0 [rsp + 16 * 0] + %define k1 [rsp + 16 * 1] + %define k2 [rsp + 16 * 2] + %define k3 [rsp + 16 * 3] + %define k4 [rsp + 16 * 4] + %define k5 [rsp + 16 * 5] + %define k6 [rsp + 16 * 6] + %define k7 [rsp + 16 * 7] + %define krd [rsp + 16 * 8] + %define zero [rsp + 16 * 9] + + GET_FILTERS + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + movsxd rcx, DWORD PTR arg(4) ;output_height + +.loop: + movdqu xmm0, [rsi - 3] ;load src + + movdqa xmm1, xmm0 + movdqa xmm6, xmm0 + movdqa xmm7, xmm0 + movdqa xmm2, xmm0 + movdqa xmm5, xmm0 + movdqa xmm3, xmm0 + movdqa xmm4, xmm0 + + psrldq xmm1, 1 + psrldq xmm6, 6 + psrldq xmm7, 7 + psrldq xmm2, 2 + psrldq xmm5, 5 + psrldq xmm3, 3 + psrldq xmm4, 4 + + APPLY_FILTER_8 1, 0 + + movdqu xmm0, [rsi + 5] ;load src + + movdqa xmm1, xmm0 + movdqa xmm6, xmm0 + movdqa xmm7, xmm0 + movdqa xmm2, xmm0 + movdqa xmm5, xmm0 + movdqa xmm3, xmm0 + movdqa xmm4, xmm0 + + psrldq xmm1, 1 + psrldq xmm6, 6 + psrldq xmm7, 7 + psrldq xmm2, 2 + psrldq xmm5, 5 + psrldq xmm3, 3 + psrldq xmm4, 4 + + APPLY_FILTER_8 1, 8 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx + jnz .loop + + add rsp, 16 * 10 + pop rsp + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret diff --git a/third_party/aom/aom_dsp/x86/aom_subpixel_8t_ssse3.asm b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_ssse3.asm new file mode 100644 index 0000000000..357f374016 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_ssse3.asm @@ -0,0 +1,883 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION_RODATA +pw_64: times 8 dw 64 +even_byte_mask: times 8 dw 0x00ff + +; %define USE_PMULHRSW +; NOTE: pmulhrsw has a latency of 5 cycles. Tests showed a performance loss +; when using this instruction. +; +; The add order below (based on ffav1) must be followed to prevent outranges. +; x = k0k1 + k4k5 +; y = k2k3 + k6k7 +; z = signed SAT(x + y) + +SECTION .text +%define LOCAL_VARS_SIZE 16*6 + +%macro SETUP_LOCAL_VARS 0 + ; TODO(slavarnway): using xmm registers for these on ARCH_X86_64 + + ; pmaddubsw has a higher latency on some platforms, this might be eased by + ; interleaving the instructions. + %define k0k1 [rsp + 16*0] + %define k2k3 [rsp + 16*1] + %define k4k5 [rsp + 16*2] + %define k6k7 [rsp + 16*3] + packsswb m4, m4 + ; TODO(slavarnway): multiple pshufb instructions had a higher latency on + ; some platforms. + pshuflw m0, m4, 0b ;k0_k1 + pshuflw m1, m4, 01010101b ;k2_k3 + pshuflw m2, m4, 10101010b ;k4_k5 + pshuflw m3, m4, 11111111b ;k6_k7 + punpcklqdq m0, m0 + punpcklqdq m1, m1 + punpcklqdq m2, m2 + punpcklqdq m3, m3 + mova k0k1, m0 + mova k2k3, m1 + mova k4k5, m2 + mova k6k7, m3 +%if ARCH_X86_64 + %define krd m12 + %define tmp0 [rsp + 16*4] + %define tmp1 [rsp + 16*5] + mova krd, [GLOBAL(pw_64)] +%else + %define krd [rsp + 16*4] +%if CONFIG_PIC=0 + mova m6, [GLOBAL(pw_64)] +%else + ; build constants without accessing global memory + pcmpeqb m6, m6 ;all ones + psrlw m6, 15 + psllw m6, 6 ;aka pw_64 +%endif + mova krd, m6 +%endif +%endm + +;------------------------------------------------------------------------------- +%if ARCH_X86_64 + %define LOCAL_VARS_SIZE_H4 0 +%else + %define LOCAL_VARS_SIZE_H4 16*4 +%endif + +%macro SUBPIX_HFILTER4 1 +cglobal filter_block1d4_%1, 6, 6, 11, LOCAL_VARS_SIZE_H4, \ + src, sstride, dst, dstride, height, filter + mova m4, [filterq] + packsswb m4, m4 +%if ARCH_X86_64 + %define k0k1k4k5 m8 + %define k2k3k6k7 m9 + %define krd m10 + mova krd, [GLOBAL(pw_64)] + pshuflw k0k1k4k5, m4, 0b ;k0_k1 + pshufhw k0k1k4k5, k0k1k4k5, 10101010b ;k0_k1_k4_k5 + pshuflw k2k3k6k7, m4, 01010101b ;k2_k3 + pshufhw k2k3k6k7, k2k3k6k7, 11111111b ;k2_k3_k6_k7 +%else + %define k0k1k4k5 [rsp + 16*0] + %define k2k3k6k7 [rsp + 16*1] + %define krd [rsp + 16*2] + pshuflw m6, m4, 0b ;k0_k1 + pshufhw m6, m6, 10101010b ;k0_k1_k4_k5 + pshuflw m7, m4, 01010101b ;k2_k3 + pshufhw m7, m7, 11111111b ;k2_k3_k6_k7 +%if CONFIG_PIC=0 + mova m1, [GLOBAL(pw_64)] +%else + ; build constants without accessing global memory + pcmpeqb m1, m1 ;all ones + psrlw m1, 15 + psllw m1, 6 ;aka pw_64 +%endif + mova k0k1k4k5, m6 + mova k2k3k6k7, m7 + mova krd, m1 +%endif + dec heightd + +.loop: + ;Do two rows at once + movu m4, [srcq - 3] + movu m5, [srcq + sstrideq - 3] + punpckhbw m1, m4, m4 + punpcklbw m4, m4 + punpckhbw m3, m5, m5 + punpcklbw m5, m5 + palignr m0, m1, m4, 1 + pmaddubsw m0, k0k1k4k5 + palignr m1, m4, 5 + pmaddubsw m1, k2k3k6k7 + palignr m2, m3, m5, 1 + pmaddubsw m2, k0k1k4k5 + palignr m3, m5, 5 + pmaddubsw m3, k2k3k6k7 + punpckhqdq m4, m0, m2 + punpcklqdq m0, m2 + punpckhqdq m5, m1, m3 + punpcklqdq m1, m3 + paddsw m0, m4 + paddsw m1, m5 +%ifidn %1, h8_avg + movd m4, [dstq] + movd m5, [dstq + dstrideq] +%endif + paddsw m0, m1 + paddsw m0, krd + psraw m0, 7 +%ifidn %1, h8_add_src + pxor m3, m3 + movu m4, [srcq] + movu m5, [srcq + sstrideq] + punpckldq m4, m5 ; Bytes 0,1,2,3 from row 0, then 0,1,2,3 from row 2 + punpcklbw m4, m3 + paddsw m0, m4 +%endif + packuswb m0, m0 + psrldq m1, m0, 4 + +%ifidn %1, h8_avg + pavgb m0, m4 + pavgb m1, m5 +%endif + movd [dstq], m0 + movd [dstq + dstrideq], m1 + + lea srcq, [srcq + sstrideq ] + prefetcht0 [srcq + 4 * sstrideq - 3] + lea srcq, [srcq + sstrideq ] + lea dstq, [dstq + 2 * dstrideq ] + prefetcht0 [srcq + 2 * sstrideq - 3] + + sub heightd, 2 + jg .loop + + ; Do last row if output_height is odd + jne .done + + movu m4, [srcq - 3] + punpckhbw m1, m4, m4 + punpcklbw m4, m4 + palignr m0, m1, m4, 1 + palignr m1, m4, 5 + pmaddubsw m0, k0k1k4k5 + pmaddubsw m1, k2k3k6k7 + psrldq m2, m0, 8 + psrldq m3, m1, 8 + paddsw m0, m2 + paddsw m1, m3 + paddsw m0, m1 + paddsw m0, krd + psraw m0, 7 +%ifidn %1, h8_add_src + pxor m3, m3 + movu m4, [srcq] + punpcklbw m4, m3 + paddsw m0, m4 +%endif + packuswb m0, m0 +%ifidn %1, h8_avg + movd m4, [dstq] + pavgb m0, m4 +%endif + movd [dstq], m0 +.done: + REP_RET +%endm + +;------------------------------------------------------------------------------- +%macro SUBPIX_HFILTER8 1 +cglobal filter_block1d8_%1, 6, 6, 14, LOCAL_VARS_SIZE, \ + src, sstride, dst, dstride, height, filter + mova m4, [filterq] + SETUP_LOCAL_VARS + dec heightd + +.loop: + ;Do two rows at once + movu m0, [srcq - 3] + movu m4, [srcq + sstrideq - 3] + punpckhbw m1, m0, m0 + punpcklbw m0, m0 + palignr m5, m1, m0, 13 + pmaddubsw m5, k6k7 + palignr m2, m1, m0, 5 + palignr m3, m1, m0, 9 + palignr m1, m0, 1 + pmaddubsw m1, k0k1 + punpckhbw m6, m4, m4 + punpcklbw m4, m4 + pmaddubsw m2, k2k3 + pmaddubsw m3, k4k5 + + palignr m7, m6, m4, 13 + palignr m0, m6, m4, 5 + pmaddubsw m7, k6k7 + paddsw m1, m3 + paddsw m2, m5 + paddsw m1, m2 +%ifidn %1, h8_avg + movh m2, [dstq] + movhps m2, [dstq + dstrideq] +%endif + palignr m5, m6, m4, 9 + palignr m6, m4, 1 + pmaddubsw m0, k2k3 + pmaddubsw m6, k0k1 + paddsw m1, krd + pmaddubsw m5, k4k5 + psraw m1, 7 + paddsw m0, m7 + paddsw m6, m5 + paddsw m6, m0 + paddsw m6, krd + psraw m6, 7 +%ifidn %1, h8_add_src + pxor m3, m3 + movu m4, [srcq] + movu m5, [srcq + sstrideq] + punpcklbw m4, m3 + punpcklbw m5, m3 + paddsw m1, m4 + paddsw m6, m5 +%endif + packuswb m1, m6 +%ifidn %1, h8_avg + pavgb m1, m2 +%endif + movh [dstq], m1 + movhps [dstq + dstrideq], m1 + + lea srcq, [srcq + sstrideq ] + prefetcht0 [srcq + 4 * sstrideq - 3] + lea srcq, [srcq + sstrideq ] + lea dstq, [dstq + 2 * dstrideq ] + prefetcht0 [srcq + 2 * sstrideq - 3] + sub heightd, 2 + jg .loop + + ; Do last row if output_height is odd + jne .done + + movu m0, [srcq - 3] + punpckhbw m3, m0, m0 + punpcklbw m0, m0 + palignr m1, m3, m0, 1 + palignr m2, m3, m0, 5 + palignr m4, m3, m0, 13 + palignr m3, m0, 9 + pmaddubsw m1, k0k1 + pmaddubsw m2, k2k3 + pmaddubsw m3, k4k5 + pmaddubsw m4, k6k7 + paddsw m1, m3 + paddsw m4, m2 + paddsw m1, m4 + paddsw m1, krd + psraw m1, 7 +%ifidn %1, h8_add_src + pxor m6, m6 + movu m5, [srcq] + punpcklbw m5, m6 + paddsw m1, m5 +%endif + packuswb m1, m1 +%ifidn %1, h8_avg + movh m0, [dstq] + pavgb m1, m0 +%endif + movh [dstq], m1 +.done: + REP_RET +%endm + +;------------------------------------------------------------------------------- +%macro SUBPIX_HFILTER16 1 +cglobal filter_block1d16_%1, 6, 6, 14, LOCAL_VARS_SIZE, \ + src, sstride, dst, dstride, height, filter + mova m4, [filterq] + SETUP_LOCAL_VARS + +.loop: + prefetcht0 [srcq + 2 * sstrideq -3] + + movu m0, [srcq - 3] + movu m4, [srcq - 2] + pmaddubsw m0, k0k1 + pmaddubsw m4, k0k1 + movu m1, [srcq - 1] + movu m5, [srcq + 0] + pmaddubsw m1, k2k3 + pmaddubsw m5, k2k3 + movu m2, [srcq + 1] + movu m6, [srcq + 2] + pmaddubsw m2, k4k5 + pmaddubsw m6, k4k5 + movu m3, [srcq + 3] + movu m7, [srcq + 4] + pmaddubsw m3, k6k7 + pmaddubsw m7, k6k7 + paddsw m0, m2 + paddsw m1, m3 + paddsw m0, m1 + paddsw m4, m6 + paddsw m5, m7 + paddsw m4, m5 + paddsw m0, krd + paddsw m4, krd + psraw m0, 7 + psraw m4, 7 +%ifidn %1, h8_add_src + movu m5, [srcq] + mova m7, m5 + pand m5, [even_byte_mask] + psrlw m7, 8 + paddsw m0, m5 + paddsw m4, m7 +%endif + packuswb m0, m0 + packuswb m4, m4 + punpcklbw m0, m4 +%ifidn %1, h8_avg + pavgb m0, [dstq] +%endif + lea srcq, [srcq + sstrideq] + mova [dstq], m0 + lea dstq, [dstq + dstrideq] + dec heightd + jnz .loop + REP_RET +%endm + +INIT_XMM ssse3 +SUBPIX_HFILTER16 h8 +SUBPIX_HFILTER16 h8_avg +SUBPIX_HFILTER8 h8 +SUBPIX_HFILTER8 h8_avg +SUBPIX_HFILTER4 h8 +SUBPIX_HFILTER4 h8_avg + +%if CONFIG_LOOP_RESTORATION +SUBPIX_HFILTER16 h8_add_src +SUBPIX_HFILTER8 h8_add_src +SUBPIX_HFILTER4 h8_add_src +%endif + +;------------------------------------------------------------------------------- + +; TODO(Linfeng): Detect cpu type and choose the code with better performance. +%define X86_SUBPIX_VFILTER_PREFER_SLOW_CELERON 1 + +%if ARCH_X86_64 && X86_SUBPIX_VFILTER_PREFER_SLOW_CELERON + %define NUM_GENERAL_REG_USED 9 +%else + %define NUM_GENERAL_REG_USED 6 +%endif + +%macro SUBPIX_VFILTER 2 +cglobal filter_block1d%2_%1, 6, NUM_GENERAL_REG_USED, 15, LOCAL_VARS_SIZE, \ + src, sstride, dst, dstride, height, filter + mova m4, [filterq] + SETUP_LOCAL_VARS + +%ifidn %2, 8 + %define movx movh +%else + %define movx movd +%endif + + dec heightd + +%if ARCH_X86 || X86_SUBPIX_VFILTER_PREFER_SLOW_CELERON + +%if ARCH_X86_64 + %define src1q r7 + %define sstride6q r8 + %define dst_stride dstrideq +%else + %define src1q filterq + %define sstride6q dstrideq + %define dst_stride dstridemp +%endif + mov src1q, srcq + add src1q, sstrideq + lea sstride6q, [sstrideq + sstrideq * 4] + add sstride6q, sstrideq ;pitch * 6 + +.loop: + ;Do two rows at once + movx m0, [srcq ] ;A + movx m1, [src1q ] ;B + punpcklbw m0, m1 ;A B + movx m2, [srcq + sstrideq * 2 ] ;C + pmaddubsw m0, k0k1 + mova m6, m2 + movx m3, [src1q + sstrideq * 2] ;D + punpcklbw m2, m3 ;C D + pmaddubsw m2, k2k3 + movx m4, [srcq + sstrideq * 4 ] ;E + mova m7, m4 + movx m5, [src1q + sstrideq * 4] ;F + punpcklbw m4, m5 ;E F + pmaddubsw m4, k4k5 + punpcklbw m1, m6 ;A B next iter + movx m6, [srcq + sstride6q ] ;G + punpcklbw m5, m6 ;E F next iter + punpcklbw m3, m7 ;C D next iter + pmaddubsw m5, k4k5 + movx m7, [src1q + sstride6q ] ;H + punpcklbw m6, m7 ;G H + pmaddubsw m6, k6k7 + pmaddubsw m3, k2k3 + pmaddubsw m1, k0k1 + paddsw m0, m4 + paddsw m2, m6 + movx m6, [srcq + sstrideq * 8 ] ;H next iter + punpcklbw m7, m6 + pmaddubsw m7, k6k7 + paddsw m0, m2 + paddsw m0, krd + psraw m0, 7 + paddsw m1, m5 +%ifidn %1, v8_add_src + pxor m6, m6 + movu m4, [srcq] + punpcklbw m4, m6 + paddsw m0, m4 +%endif + packuswb m0, m0 + + paddsw m3, m7 + paddsw m1, m3 + paddsw m1, krd + psraw m1, 7 +%ifidn %1, v8_add_src + movu m4, [src1q] + punpcklbw m4, m6 + paddsw m1, m4 +%endif + lea srcq, [srcq + sstrideq * 2 ] + lea src1q, [src1q + sstrideq * 2] + packuswb m1, m1 + +%ifidn %1, v8_avg + movx m2, [dstq] + pavgb m0, m2 +%endif + movx [dstq], m0 + add dstq, dst_stride +%ifidn %1, v8_avg + movx m3, [dstq] + pavgb m1, m3 +%endif + movx [dstq], m1 + add dstq, dst_stride + sub heightd, 2 + jg .loop + + ; Do last row if output_height is odd + jne .done + + movx m0, [srcq ] ;A + movx m1, [srcq + sstrideq ] ;B + movx m6, [srcq + sstride6q ] ;G + punpcklbw m0, m1 ;A B + movx m7, [src1q + sstride6q ] ;H + pmaddubsw m0, k0k1 + movx m2, [srcq + sstrideq * 2 ] ;C + punpcklbw m6, m7 ;G H + movx m3, [src1q + sstrideq * 2] ;D + pmaddubsw m6, k6k7 + movx m4, [srcq + sstrideq * 4 ] ;E + punpcklbw m2, m3 ;C D + movx m5, [src1q + sstrideq * 4] ;F + punpcklbw m4, m5 ;E F + pmaddubsw m2, k2k3 + pmaddubsw m4, k4k5 + paddsw m2, m6 + paddsw m0, m4 + paddsw m0, m2 + paddsw m0, krd + psraw m0, 7 +%ifidn %1, v8_add_src + pxor m6, m6 + movu m4, [srcq] + punpcklbw m4, m6 + paddsw m0, m4 +%endif + packuswb m0, m0 +%ifidn %1, v8_avg + movx m1, [dstq] + pavgb m0, m1 +%endif + movx [dstq], m0 + +%else + ; ARCH_X86_64 + + movx m0, [srcq ] ;A + movx m1, [srcq + sstrideq ] ;B + lea srcq, [srcq + sstrideq * 2 ] + movx m2, [srcq] ;C + movx m3, [srcq + sstrideq] ;D + lea srcq, [srcq + sstrideq * 2 ] + movx m4, [srcq] ;E + movx m5, [srcq + sstrideq] ;F + lea srcq, [srcq + sstrideq * 2 ] + movx m6, [srcq] ;G + punpcklbw m0, m1 ;A B + punpcklbw m1, m2 ;A B next iter + punpcklbw m2, m3 ;C D + punpcklbw m3, m4 ;C D next iter + punpcklbw m4, m5 ;E F + punpcklbw m5, m6 ;E F next iter + +.loop: + ;Do two rows at once + movx m7, [srcq + sstrideq] ;H + lea srcq, [srcq + sstrideq * 2 ] + movx m14, [srcq] ;H next iter + punpcklbw m6, m7 ;G H + punpcklbw m7, m14 ;G H next iter + pmaddubsw m8, m0, k0k1 + pmaddubsw m9, m1, k0k1 + mova m0, m2 + mova m1, m3 + pmaddubsw m10, m2, k2k3 + pmaddubsw m11, m3, k2k3 + mova m2, m4 + mova m3, m5 + pmaddubsw m4, k4k5 + pmaddubsw m5, k4k5 + paddsw m8, m4 + paddsw m9, m5 + mova m4, m6 + mova m5, m7 + pmaddubsw m6, k6k7 + pmaddubsw m7, k6k7 + paddsw m10, m6 + paddsw m11, m7 + paddsw m8, m10 + paddsw m9, m11 + mova m6, m14 + paddsw m8, krd + paddsw m9, krd + psraw m8, 7 + psraw m9, 7 +%ifidn %2, 4 + packuswb m8, m8 + packuswb m9, m9 +%else + packuswb m8, m9 +%endif + +%ifidn %1, v8_avg + movx m7, [dstq] +%ifidn %2, 4 + movx m10, [dstq + dstrideq] + pavgb m9, m10 +%else + movhpd m7, [dstq + dstrideq] +%endif + pavgb m8, m7 +%endif + movx [dstq], m8 +%ifidn %2, 4 + movx [dstq + dstrideq], m9 +%else + movhpd [dstq + dstrideq], m8 +%endif + + lea dstq, [dstq + dstrideq * 2 ] + sub heightd, 2 + jg .loop + + ; Do last row if output_height is odd + jne .done + + movx m7, [srcq + sstrideq] ;H + punpcklbw m6, m7 ;G H + pmaddubsw m0, k0k1 + pmaddubsw m2, k2k3 + pmaddubsw m4, k4k5 + pmaddubsw m6, k6k7 + paddsw m0, m4 + paddsw m2, m6 + paddsw m0, m2 + paddsw m0, krd + psraw m0, 7 + packuswb m0, m0 +%ifidn %1, v8_avg + movx m1, [dstq] + pavgb m0, m1 +%endif + movx [dstq], m0 + +%endif ; ARCH_X86_64 + +.done: + REP_RET + +%endm + +;------------------------------------------------------------------------------- +%macro SUBPIX_VFILTER16 1 +cglobal filter_block1d16_%1, 6, NUM_GENERAL_REG_USED, 16, LOCAL_VARS_SIZE, \ + src, sstride, dst, dstride, height, filter + mova m4, [filterq] + SETUP_LOCAL_VARS + +%if ARCH_X86 || X86_SUBPIX_VFILTER_PREFER_SLOW_CELERON + +%if ARCH_X86_64 + %define src1q r7 + %define sstride6q r8 + %define dst_stride dstrideq +%else + %define src1q filterq + %define sstride6q dstrideq + %define dst_stride dstridemp +%endif + lea src1q, [srcq + sstrideq] + lea sstride6q, [sstrideq + sstrideq * 4] + add sstride6q, sstrideq ;pitch * 6 + +.loop: + movh m0, [srcq ] ;A + movh m1, [src1q ] ;B + movh m2, [srcq + sstrideq * 2 ] ;C + movh m3, [src1q + sstrideq * 2] ;D + movh m4, [srcq + sstrideq * 4 ] ;E + movh m5, [src1q + sstrideq * 4] ;F + + punpcklbw m0, m1 ;A B + movh m6, [srcq + sstride6q] ;G + punpcklbw m2, m3 ;C D + movh m7, [src1q + sstride6q] ;H + punpcklbw m4, m5 ;E F + pmaddubsw m0, k0k1 + movh m3, [srcq + 8] ;A + pmaddubsw m2, k2k3 + punpcklbw m6, m7 ;G H + movh m5, [srcq + sstrideq + 8] ;B + pmaddubsw m4, k4k5 + punpcklbw m3, m5 ;A B + movh m7, [srcq + sstrideq * 2 + 8] ;C + pmaddubsw m6, k6k7 + movh m5, [src1q + sstrideq * 2 + 8] ;D + punpcklbw m7, m5 ;C D + paddsw m2, m6 + pmaddubsw m3, k0k1 + movh m1, [srcq + sstrideq * 4 + 8] ;E + paddsw m0, m4 + pmaddubsw m7, k2k3 + movh m6, [src1q + sstrideq * 4 + 8] ;F + punpcklbw m1, m6 ;E F + paddsw m0, m2 + paddsw m0, krd + movh m2, [srcq + sstride6q + 8] ;G + pmaddubsw m1, k4k5 + movh m5, [src1q + sstride6q + 8] ;H + psraw m0, 7 + punpcklbw m2, m5 ;G H + pmaddubsw m2, k6k7 + paddsw m7, m2 + paddsw m3, m1 + paddsw m3, m7 + paddsw m3, krd + psraw m3, 7 +%ifidn %1, v8_add_src + pxor m6, m6 + movu m4, [src1q + 2 * sstrideq] ; Fetch from 3 rows down + mova m5, m4 + punpcklbw m4, m6 + punpckhbw m5, m6 + paddsw m0, m4 + paddsw m3, m5 +%endif + packuswb m0, m3 + + add srcq, sstrideq + add src1q, sstrideq +%ifidn %1, v8_avg + pavgb m0, [dstq] +%endif + mova [dstq], m0 + add dstq, dst_stride + dec heightd + jnz .loop + REP_RET + +%else + ; ARCH_X86_64 + dec heightd + + movu m1, [srcq ] ;A + movu m3, [srcq + sstrideq ] ;B + lea srcq, [srcq + sstrideq * 2] + punpcklbw m0, m1, m3 ;A B + punpckhbw m1, m3 ;A B + movu m5, [srcq] ;C + punpcklbw m2, m3, m5 ;A B next iter + punpckhbw m3, m5 ;A B next iter + mova tmp0, m2 ;store to stack + mova tmp1, m3 ;store to stack + movu m7, [srcq + sstrideq] ;D + lea srcq, [srcq + sstrideq * 2] + punpcklbw m4, m5, m7 ;C D + punpckhbw m5, m7 ;C D + movu m9, [srcq] ;E + punpcklbw m6, m7, m9 ;C D next iter + punpckhbw m7, m9 ;C D next iter + movu m11, [srcq + sstrideq] ;F + lea srcq, [srcq + sstrideq * 2] + punpcklbw m8, m9, m11 ;E F + punpckhbw m9, m11 ;E F + movu m2, [srcq] ;G + punpcklbw m10, m11, m2 ;E F next iter + punpckhbw m11, m2 ;E F next iter + +.loop: + ;Do two rows at once + pmaddubsw m13, m0, k0k1 + mova m0, m4 + pmaddubsw m14, m8, k4k5 + pmaddubsw m15, m4, k2k3 + mova m4, m8 + paddsw m13, m14 + movu m3, [srcq + sstrideq] ;H + lea srcq, [srcq + sstrideq * 2] + punpcklbw m14, m2, m3 ;G H + mova m8, m14 + pmaddubsw m14, k6k7 + paddsw m15, m14 + paddsw m13, m15 + paddsw m13, krd + psraw m13, 7 + + pmaddubsw m14, m1, k0k1 + pmaddubsw m1, m9, k4k5 + pmaddubsw m15, m5, k2k3 + paddsw m14, m1 + mova m1, m5 + mova m5, m9 + punpckhbw m2, m3 ;G H + mova m9, m2 + pmaddubsw m2, k6k7 + paddsw m15, m2 + paddsw m14, m15 + paddsw m14, krd + psraw m14, 7 + packuswb m13, m14 +%ifidn %1, v8_avg + pavgb m13, [dstq] +%endif + mova [dstq], m13 + + ; next iter + pmaddubsw m15, tmp0, k0k1 + pmaddubsw m14, m10, k4k5 + pmaddubsw m13, m6, k2k3 + paddsw m15, m14 + mova tmp0, m6 + mova m6, m10 + movu m2, [srcq] ;G next iter + punpcklbw m14, m3, m2 ;G H next iter + mova m10, m14 + pmaddubsw m14, k6k7 + paddsw m13, m14 + paddsw m15, m13 + paddsw m15, krd + psraw m15, 7 + + pmaddubsw m14, tmp1, k0k1 + mova tmp1, m7 + pmaddubsw m13, m7, k2k3 + mova m7, m11 + pmaddubsw m11, k4k5 + paddsw m14, m11 + punpckhbw m3, m2 ;G H next iter + mova m11, m3 + pmaddubsw m3, k6k7 + paddsw m13, m3 + paddsw m14, m13 + paddsw m14, krd + psraw m14, 7 + packuswb m15, m14 +%ifidn %1, v8_avg + pavgb m15, [dstq + dstrideq] +%endif + mova [dstq + dstrideq], m15 + lea dstq, [dstq + dstrideq * 2] + sub heightd, 2 + jg .loop + + ; Do last row if output_height is odd + jne .done + + movu m3, [srcq + sstrideq] ;H + punpcklbw m6, m2, m3 ;G H + punpckhbw m2, m3 ;G H + pmaddubsw m0, k0k1 + pmaddubsw m1, k0k1 + pmaddubsw m4, k2k3 + pmaddubsw m5, k2k3 + pmaddubsw m8, k4k5 + pmaddubsw m9, k4k5 + pmaddubsw m6, k6k7 + pmaddubsw m2, k6k7 + paddsw m0, m8 + paddsw m1, m9 + paddsw m4, m6 + paddsw m5, m2 + paddsw m0, m4 + paddsw m1, m5 + paddsw m0, krd + paddsw m1, krd + psraw m0, 7 + psraw m1, 7 + packuswb m0, m1 +%ifidn %1, v8_avg + pavgb m0, [dstq] +%endif + mova [dstq], m0 + +.done: + REP_RET + +%endif ; ARCH_X86_64 + +%endm + +INIT_XMM ssse3 +SUBPIX_VFILTER16 v8 +SUBPIX_VFILTER16 v8_avg +SUBPIX_VFILTER v8, 8 +SUBPIX_VFILTER v8_avg, 8 +SUBPIX_VFILTER v8, 4 +SUBPIX_VFILTER v8_avg, 4 + +%if (ARCH_X86 || X86_SUBPIX_VFILTER_PREFER_SLOW_CELERON) && \ + CONFIG_LOOP_RESTORATION +SUBPIX_VFILTER16 v8_add_src +SUBPIX_VFILTER v8_add_src, 8 +SUBPIX_VFILTER v8_add_src, 4 +%endif diff --git a/third_party/aom/aom_dsp/x86/aom_subpixel_bilinear_sse2.asm b/third_party/aom/aom_dsp/x86/aom_subpixel_bilinear_sse2.asm new file mode 100644 index 0000000000..8f025a8be4 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/aom_subpixel_bilinear_sse2.asm @@ -0,0 +1,451 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "aom_ports/x86_abi_support.asm" + +%macro GET_PARAM_4 0 + mov rdx, arg(5) ;filter ptr + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + mov rcx, 0x0400040 + + movdqa xmm3, [rdx] ;load filters + pshuflw xmm4, xmm3, 11111111b ;k3 + psrldq xmm3, 8 + pshuflw xmm3, xmm3, 0b ;k4 + punpcklqdq xmm4, xmm3 ;k3k4 + + movq xmm3, rcx ;rounding + pshufd xmm3, xmm3, 0 + + pxor xmm2, xmm2 + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + movsxd rcx, DWORD PTR arg(4) ;output_height +%endm + +%macro APPLY_FILTER_4 1 + + punpckldq xmm0, xmm1 ;two row in one register + punpcklbw xmm0, xmm2 ;unpack to word + pmullw xmm0, xmm4 ;multiply the filter factors + + movdqa xmm1, xmm0 + psrldq xmm1, 8 + paddsw xmm0, xmm1 + + paddsw xmm0, xmm3 ;rounding + psraw xmm0, 7 ;shift + packuswb xmm0, xmm0 ;pack to byte + +%if %1 + movd xmm1, [rdi] + pavgb xmm0, xmm1 +%endif + + movd [rdi], xmm0 + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx +%endm + +%macro GET_PARAM 0 + mov rdx, arg(5) ;filter ptr + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + mov rcx, 0x0400040 + + movdqa xmm7, [rdx] ;load filters + + pshuflw xmm6, xmm7, 11111111b ;k3 + pshufhw xmm7, xmm7, 0b ;k4 + punpcklwd xmm6, xmm6 + punpckhwd xmm7, xmm7 + + movq xmm4, rcx ;rounding + pshufd xmm4, xmm4, 0 + + pxor xmm5, xmm5 + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + movsxd rcx, DWORD PTR arg(4) ;output_height +%endm + +%macro APPLY_FILTER_8 1 + punpcklbw xmm0, xmm5 + punpcklbw xmm1, xmm5 + + pmullw xmm0, xmm6 + pmullw xmm1, xmm7 + paddsw xmm0, xmm1 + paddsw xmm0, xmm4 ;rounding + psraw xmm0, 7 ;shift + packuswb xmm0, xmm0 ;pack back to byte +%if %1 + movq xmm1, [rdi] + pavgb xmm0, xmm1 +%endif + movq [rdi], xmm0 ;store the result + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx +%endm + +%macro APPLY_FILTER_16 1 + punpcklbw xmm0, xmm5 + punpcklbw xmm1, xmm5 + punpckhbw xmm2, xmm5 + punpckhbw xmm3, xmm5 + + pmullw xmm0, xmm6 + pmullw xmm1, xmm7 + pmullw xmm2, xmm6 + pmullw xmm3, xmm7 + + paddsw xmm0, xmm1 + paddsw xmm2, xmm3 + + paddsw xmm0, xmm4 ;rounding + paddsw xmm2, xmm4 + psraw xmm0, 7 ;shift + psraw xmm2, 7 + packuswb xmm0, xmm2 ;pack back to byte +%if %1 + movdqu xmm1, [rdi] + pavgb xmm0, xmm1 +%endif + movdqu [rdi], xmm0 ;store the result + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx +%endm + +global sym(aom_filter_block1d4_v2_sse2) PRIVATE +sym(aom_filter_block1d4_v2_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + push rsi + push rdi + ; end prolog + + GET_PARAM_4 +.loop: + movd xmm0, [rsi] ;load src + movd xmm1, [rsi + rax] + + APPLY_FILTER_4 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d8_v2_sse2) PRIVATE +sym(aom_filter_block1d8_v2_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movq xmm0, [rsi] ;0 + movq xmm1, [rsi + rax] ;1 + + APPLY_FILTER_8 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d16_v2_sse2) PRIVATE +sym(aom_filter_block1d16_v2_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movdqu xmm0, [rsi] ;0 + movdqu xmm1, [rsi + rax] ;1 + movdqa xmm2, xmm0 + movdqa xmm3, xmm1 + + APPLY_FILTER_16 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d4_v2_avg_sse2) PRIVATE +sym(aom_filter_block1d4_v2_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + push rsi + push rdi + ; end prolog + + GET_PARAM_4 +.loop: + movd xmm0, [rsi] ;load src + movd xmm1, [rsi + rax] + + APPLY_FILTER_4 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d8_v2_avg_sse2) PRIVATE +sym(aom_filter_block1d8_v2_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movq xmm0, [rsi] ;0 + movq xmm1, [rsi + rax] ;1 + + APPLY_FILTER_8 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d16_v2_avg_sse2) PRIVATE +sym(aom_filter_block1d16_v2_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movdqu xmm0, [rsi] ;0 + movdqu xmm1, [rsi + rax] ;1 + movdqa xmm2, xmm0 + movdqa xmm3, xmm1 + + APPLY_FILTER_16 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d4_h2_sse2) PRIVATE +sym(aom_filter_block1d4_h2_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + push rsi + push rdi + ; end prolog + + GET_PARAM_4 +.loop: + movdqu xmm0, [rsi] ;load src + movdqa xmm1, xmm0 + psrldq xmm1, 1 + + APPLY_FILTER_4 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d8_h2_sse2) PRIVATE +sym(aom_filter_block1d8_h2_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movdqu xmm0, [rsi] ;load src + movdqa xmm1, xmm0 + psrldq xmm1, 1 + + APPLY_FILTER_8 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d16_h2_sse2) PRIVATE +sym(aom_filter_block1d16_h2_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movdqu xmm0, [rsi] ;load src + movdqu xmm1, [rsi + 1] + movdqa xmm2, xmm0 + movdqa xmm3, xmm1 + + APPLY_FILTER_16 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d4_h2_avg_sse2) PRIVATE +sym(aom_filter_block1d4_h2_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + push rsi + push rdi + ; end prolog + + GET_PARAM_4 +.loop: + movdqu xmm0, [rsi] ;load src + movdqa xmm1, xmm0 + psrldq xmm1, 1 + + APPLY_FILTER_4 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d8_h2_avg_sse2) PRIVATE +sym(aom_filter_block1d8_h2_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movdqu xmm0, [rsi] ;load src + movdqa xmm1, xmm0 + psrldq xmm1, 1 + + APPLY_FILTER_8 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d16_h2_avg_sse2) PRIVATE +sym(aom_filter_block1d16_h2_avg_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movdqu xmm0, [rsi] ;load src + movdqu xmm1, [rsi + 1] + movdqa xmm2, xmm0 + movdqa xmm3, xmm1 + + APPLY_FILTER_16 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret diff --git a/third_party/aom/aom_dsp/x86/aom_subpixel_bilinear_ssse3.asm b/third_party/aom/aom_dsp/x86/aom_subpixel_bilinear_ssse3.asm new file mode 100644 index 0000000000..b9b2da0bea --- /dev/null +++ b/third_party/aom/aom_dsp/x86/aom_subpixel_bilinear_ssse3.asm @@ -0,0 +1,421 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "aom_ports/x86_abi_support.asm" + +%macro GET_PARAM_4 0 + mov rdx, arg(5) ;filter ptr + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + mov ecx, 0x01000100 + + movdqa xmm3, [rdx] ;load filters + psrldq xmm3, 6 + packsswb xmm3, xmm3 + pshuflw xmm3, xmm3, 0b ;k3_k4 + + movd xmm2, ecx ;rounding_shift + pshufd xmm2, xmm2, 0 + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + movsxd rcx, DWORD PTR arg(4) ;output_height +%endm + +%macro APPLY_FILTER_4 1 + punpcklbw xmm0, xmm1 + pmaddubsw xmm0, xmm3 + + pmulhrsw xmm0, xmm2 ;rounding(+64)+shift(>>7) + packuswb xmm0, xmm0 ;pack to byte + +%if %1 + movd xmm1, [rdi] + pavgb xmm0, xmm1 +%endif + movd [rdi], xmm0 + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx +%endm + +%macro GET_PARAM 0 + mov rdx, arg(5) ;filter ptr + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;output_ptr + mov ecx, 0x01000100 + + movdqa xmm7, [rdx] ;load filters + psrldq xmm7, 6 + packsswb xmm7, xmm7 + pshuflw xmm7, xmm7, 0b ;k3_k4 + punpcklwd xmm7, xmm7 + + movd xmm6, ecx ;rounding_shift + pshufd xmm6, xmm6, 0 + + movsxd rax, DWORD PTR arg(1) ;pixels_per_line + movsxd rdx, DWORD PTR arg(3) ;out_pitch + movsxd rcx, DWORD PTR arg(4) ;output_height +%endm + +%macro APPLY_FILTER_8 1 + punpcklbw xmm0, xmm1 + pmaddubsw xmm0, xmm7 + + pmulhrsw xmm0, xmm6 ;rounding(+64)+shift(>>7) + packuswb xmm0, xmm0 ;pack back to byte + +%if %1 + movq xmm1, [rdi] + pavgb xmm0, xmm1 +%endif + movq [rdi], xmm0 ;store the result + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx +%endm + +%macro APPLY_FILTER_16 1 + punpcklbw xmm0, xmm1 + punpckhbw xmm2, xmm1 + pmaddubsw xmm0, xmm7 + pmaddubsw xmm2, xmm7 + + pmulhrsw xmm0, xmm6 ;rounding(+64)+shift(>>7) + pmulhrsw xmm2, xmm6 + packuswb xmm0, xmm2 ;pack back to byte + +%if %1 + movdqu xmm1, [rdi] + pavgb xmm0, xmm1 +%endif + movdqu [rdi], xmm0 ;store the result + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + dec rcx +%endm + +global sym(aom_filter_block1d4_v2_ssse3) PRIVATE +sym(aom_filter_block1d4_v2_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + push rsi + push rdi + ; end prolog + + GET_PARAM_4 +.loop: + movd xmm0, [rsi] ;load src + movd xmm1, [rsi + rax] + + APPLY_FILTER_4 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d8_v2_ssse3) PRIVATE +sym(aom_filter_block1d8_v2_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movq xmm0, [rsi] ;0 + movq xmm1, [rsi + rax] ;1 + + APPLY_FILTER_8 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d16_v2_ssse3) PRIVATE +sym(aom_filter_block1d16_v2_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movdqu xmm0, [rsi] ;0 + movdqu xmm1, [rsi + rax] ;1 + movdqa xmm2, xmm0 + + APPLY_FILTER_16 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d4_v2_avg_ssse3) PRIVATE +sym(aom_filter_block1d4_v2_avg_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + push rsi + push rdi + ; end prolog + + GET_PARAM_4 +.loop: + movd xmm0, [rsi] ;load src + movd xmm1, [rsi + rax] + + APPLY_FILTER_4 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d8_v2_avg_ssse3) PRIVATE +sym(aom_filter_block1d8_v2_avg_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movq xmm0, [rsi] ;0 + movq xmm1, [rsi + rax] ;1 + + APPLY_FILTER_8 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d16_v2_avg_ssse3) PRIVATE +sym(aom_filter_block1d16_v2_avg_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movdqu xmm0, [rsi] ;0 + movdqu xmm1, [rsi + rax] ;1 + movdqa xmm2, xmm0 + + APPLY_FILTER_16 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d4_h2_ssse3) PRIVATE +sym(aom_filter_block1d4_h2_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + push rsi + push rdi + ; end prolog + + GET_PARAM_4 +.loop: + movdqu xmm0, [rsi] ;load src + movdqa xmm1, xmm0 + psrldq xmm1, 1 + + APPLY_FILTER_4 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d8_h2_ssse3) PRIVATE +sym(aom_filter_block1d8_h2_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movdqu xmm0, [rsi] ;load src + movdqa xmm1, xmm0 + psrldq xmm1, 1 + + APPLY_FILTER_8 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d16_h2_ssse3) PRIVATE +sym(aom_filter_block1d16_h2_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movdqu xmm0, [rsi] ;load src + movdqu xmm1, [rsi + 1] + movdqa xmm2, xmm0 + + APPLY_FILTER_16 0 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d4_h2_avg_ssse3) PRIVATE +sym(aom_filter_block1d4_h2_avg_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + push rsi + push rdi + ; end prolog + + GET_PARAM_4 +.loop: + movdqu xmm0, [rsi] ;load src + movdqa xmm1, xmm0 + psrldq xmm1, 1 + + APPLY_FILTER_4 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d8_h2_avg_ssse3) PRIVATE +sym(aom_filter_block1d8_h2_avg_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movdqu xmm0, [rsi] ;load src + movdqa xmm1, xmm0 + psrldq xmm1, 1 + + APPLY_FILTER_8 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +global sym(aom_filter_block1d16_h2_avg_ssse3) PRIVATE +sym(aom_filter_block1d16_h2_avg_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rsi + push rdi + ; end prolog + + GET_PARAM +.loop: + movdqu xmm0, [rsi] ;load src + movdqu xmm1, [rsi + 1] + movdqa xmm2, xmm0 + + APPLY_FILTER_16 1 + jnz .loop + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret diff --git a/third_party/aom/aom_dsp/x86/avg_intrin_sse2.c b/third_party/aom/aom_dsp/x86/avg_intrin_sse2.c new file mode 100644 index 0000000000..bcdc20f638 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/avg_intrin_sse2.c @@ -0,0 +1,426 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom_dsp/x86/synonyms.h" + +#include "./aom_dsp_rtcd.h" +#include "aom_ports/mem.h" + +void aom_minmax_8x8_sse2(const uint8_t *s, int p, const uint8_t *d, int dp, + int *min, int *max) { + __m128i u0, s0, d0, diff, maxabsdiff, minabsdiff, negdiff, absdiff0, absdiff; + u0 = _mm_setzero_si128(); + // Row 0 + s0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s)), u0); + d0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(d)), u0); + diff = _mm_subs_epi16(s0, d0); + negdiff = _mm_subs_epi16(u0, diff); + absdiff0 = _mm_max_epi16(diff, negdiff); + // Row 1 + s0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + p)), u0); + d0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(d + dp)), u0); + diff = _mm_subs_epi16(s0, d0); + negdiff = _mm_subs_epi16(u0, diff); + absdiff = _mm_max_epi16(diff, negdiff); + maxabsdiff = _mm_max_epi16(absdiff0, absdiff); + minabsdiff = _mm_min_epi16(absdiff0, absdiff); + // Row 2 + s0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + 2 * p)), u0); + d0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(d + 2 * dp)), u0); + diff = _mm_subs_epi16(s0, d0); + negdiff = _mm_subs_epi16(u0, diff); + absdiff = _mm_max_epi16(diff, negdiff); + maxabsdiff = _mm_max_epi16(maxabsdiff, absdiff); + minabsdiff = _mm_min_epi16(minabsdiff, absdiff); + // Row 3 + s0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + 3 * p)), u0); + d0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(d + 3 * dp)), u0); + diff = _mm_subs_epi16(s0, d0); + negdiff = _mm_subs_epi16(u0, diff); + absdiff = _mm_max_epi16(diff, negdiff); + maxabsdiff = _mm_max_epi16(maxabsdiff, absdiff); + minabsdiff = _mm_min_epi16(minabsdiff, absdiff); + // Row 4 + s0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + 4 * p)), u0); + d0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(d + 4 * dp)), u0); + diff = _mm_subs_epi16(s0, d0); + negdiff = _mm_subs_epi16(u0, diff); + absdiff = _mm_max_epi16(diff, negdiff); + maxabsdiff = _mm_max_epi16(maxabsdiff, absdiff); + minabsdiff = _mm_min_epi16(minabsdiff, absdiff); + // Row 5 + s0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + 5 * p)), u0); + d0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(d + 5 * dp)), u0); + diff = _mm_subs_epi16(s0, d0); + negdiff = _mm_subs_epi16(u0, diff); + absdiff = _mm_max_epi16(diff, negdiff); + maxabsdiff = _mm_max_epi16(maxabsdiff, absdiff); + minabsdiff = _mm_min_epi16(minabsdiff, absdiff); + // Row 6 + s0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + 6 * p)), u0); + d0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(d + 6 * dp)), u0); + diff = _mm_subs_epi16(s0, d0); + negdiff = _mm_subs_epi16(u0, diff); + absdiff = _mm_max_epi16(diff, negdiff); + maxabsdiff = _mm_max_epi16(maxabsdiff, absdiff); + minabsdiff = _mm_min_epi16(minabsdiff, absdiff); + // Row 7 + s0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + 7 * p)), u0); + d0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(d + 7 * dp)), u0); + diff = _mm_subs_epi16(s0, d0); + negdiff = _mm_subs_epi16(u0, diff); + absdiff = _mm_max_epi16(diff, negdiff); + maxabsdiff = _mm_max_epi16(maxabsdiff, absdiff); + minabsdiff = _mm_min_epi16(minabsdiff, absdiff); + + maxabsdiff = _mm_max_epi16(maxabsdiff, _mm_srli_si128(maxabsdiff, 8)); + maxabsdiff = _mm_max_epi16(maxabsdiff, _mm_srli_epi64(maxabsdiff, 32)); + maxabsdiff = _mm_max_epi16(maxabsdiff, _mm_srli_epi64(maxabsdiff, 16)); + *max = _mm_extract_epi16(maxabsdiff, 0); + + minabsdiff = _mm_min_epi16(minabsdiff, _mm_srli_si128(minabsdiff, 8)); + minabsdiff = _mm_min_epi16(minabsdiff, _mm_srli_epi64(minabsdiff, 32)); + minabsdiff = _mm_min_epi16(minabsdiff, _mm_srli_epi64(minabsdiff, 16)); + *min = _mm_extract_epi16(minabsdiff, 0); +} + +unsigned int aom_avg_8x8_sse2(const uint8_t *s, int p) { + __m128i s0, s1, u0; + unsigned int avg = 0; + u0 = _mm_setzero_si128(); + s0 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s)), u0); + s1 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + p)), u0); + s0 = _mm_adds_epu16(s0, s1); + s1 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + 2 * p)), u0); + s0 = _mm_adds_epu16(s0, s1); + s1 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + 3 * p)), u0); + s0 = _mm_adds_epu16(s0, s1); + s1 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + 4 * p)), u0); + s0 = _mm_adds_epu16(s0, s1); + s1 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + 5 * p)), u0); + s0 = _mm_adds_epu16(s0, s1); + s1 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + 6 * p)), u0); + s0 = _mm_adds_epu16(s0, s1); + s1 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(s + 7 * p)), u0); + s0 = _mm_adds_epu16(s0, s1); + + s0 = _mm_adds_epu16(s0, _mm_srli_si128(s0, 8)); + s0 = _mm_adds_epu16(s0, _mm_srli_epi64(s0, 32)); + s0 = _mm_adds_epu16(s0, _mm_srli_epi64(s0, 16)); + avg = _mm_extract_epi16(s0, 0); + return (avg + 32) >> 6; +} + +unsigned int aom_avg_4x4_sse2(const uint8_t *s, int p) { + __m128i s0, s1, u0; + unsigned int avg = 0; + + u0 = _mm_setzero_si128(); + s0 = _mm_unpacklo_epi8(xx_loadl_32(s), u0); + s1 = _mm_unpacklo_epi8(xx_loadl_32(s + p), u0); + s0 = _mm_adds_epu16(s0, s1); + s1 = _mm_unpacklo_epi8(xx_loadl_32(s + 2 * p), u0); + s0 = _mm_adds_epu16(s0, s1); + s1 = _mm_unpacklo_epi8(xx_loadl_32(s + 3 * p), u0); + s0 = _mm_adds_epu16(s0, s1); + + s0 = _mm_adds_epu16(s0, _mm_srli_si128(s0, 4)); + s0 = _mm_adds_epu16(s0, _mm_srli_epi64(s0, 16)); + avg = _mm_extract_epi16(s0, 0); + return (avg + 8) >> 4; +} + +static void hadamard_col8_sse2(__m128i *in, int iter) { + __m128i a0 = in[0]; + __m128i a1 = in[1]; + __m128i a2 = in[2]; + __m128i a3 = in[3]; + __m128i a4 = in[4]; + __m128i a5 = in[5]; + __m128i a6 = in[6]; + __m128i a7 = in[7]; + + __m128i b0 = _mm_add_epi16(a0, a1); + __m128i b1 = _mm_sub_epi16(a0, a1); + __m128i b2 = _mm_add_epi16(a2, a3); + __m128i b3 = _mm_sub_epi16(a2, a3); + __m128i b4 = _mm_add_epi16(a4, a5); + __m128i b5 = _mm_sub_epi16(a4, a5); + __m128i b6 = _mm_add_epi16(a6, a7); + __m128i b7 = _mm_sub_epi16(a6, a7); + + a0 = _mm_add_epi16(b0, b2); + a1 = _mm_add_epi16(b1, b3); + a2 = _mm_sub_epi16(b0, b2); + a3 = _mm_sub_epi16(b1, b3); + a4 = _mm_add_epi16(b4, b6); + a5 = _mm_add_epi16(b5, b7); + a6 = _mm_sub_epi16(b4, b6); + a7 = _mm_sub_epi16(b5, b7); + + if (iter == 0) { + b0 = _mm_add_epi16(a0, a4); + b7 = _mm_add_epi16(a1, a5); + b3 = _mm_add_epi16(a2, a6); + b4 = _mm_add_epi16(a3, a7); + b2 = _mm_sub_epi16(a0, a4); + b6 = _mm_sub_epi16(a1, a5); + b1 = _mm_sub_epi16(a2, a6); + b5 = _mm_sub_epi16(a3, a7); + + a0 = _mm_unpacklo_epi16(b0, b1); + a1 = _mm_unpacklo_epi16(b2, b3); + a2 = _mm_unpackhi_epi16(b0, b1); + a3 = _mm_unpackhi_epi16(b2, b3); + a4 = _mm_unpacklo_epi16(b4, b5); + a5 = _mm_unpacklo_epi16(b6, b7); + a6 = _mm_unpackhi_epi16(b4, b5); + a7 = _mm_unpackhi_epi16(b6, b7); + + b0 = _mm_unpacklo_epi32(a0, a1); + b1 = _mm_unpacklo_epi32(a4, a5); + b2 = _mm_unpackhi_epi32(a0, a1); + b3 = _mm_unpackhi_epi32(a4, a5); + b4 = _mm_unpacklo_epi32(a2, a3); + b5 = _mm_unpacklo_epi32(a6, a7); + b6 = _mm_unpackhi_epi32(a2, a3); + b7 = _mm_unpackhi_epi32(a6, a7); + + in[0] = _mm_unpacklo_epi64(b0, b1); + in[1] = _mm_unpackhi_epi64(b0, b1); + in[2] = _mm_unpacklo_epi64(b2, b3); + in[3] = _mm_unpackhi_epi64(b2, b3); + in[4] = _mm_unpacklo_epi64(b4, b5); + in[5] = _mm_unpackhi_epi64(b4, b5); + in[6] = _mm_unpacklo_epi64(b6, b7); + in[7] = _mm_unpackhi_epi64(b6, b7); + } else { + in[0] = _mm_add_epi16(a0, a4); + in[7] = _mm_add_epi16(a1, a5); + in[3] = _mm_add_epi16(a2, a6); + in[4] = _mm_add_epi16(a3, a7); + in[2] = _mm_sub_epi16(a0, a4); + in[6] = _mm_sub_epi16(a1, a5); + in[1] = _mm_sub_epi16(a2, a6); + in[5] = _mm_sub_epi16(a3, a7); + } +} + +void aom_hadamard_8x8_sse2(int16_t const *src_diff, int src_stride, + int16_t *coeff) { + __m128i src[8]; + src[0] = _mm_load_si128((const __m128i *)src_diff); + src[1] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + src[2] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + src[3] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + src[4] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + src[5] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + src[6] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + src[7] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + + hadamard_col8_sse2(src, 0); + hadamard_col8_sse2(src, 1); + + _mm_store_si128((__m128i *)coeff, src[0]); + coeff += 8; + _mm_store_si128((__m128i *)coeff, src[1]); + coeff += 8; + _mm_store_si128((__m128i *)coeff, src[2]); + coeff += 8; + _mm_store_si128((__m128i *)coeff, src[3]); + coeff += 8; + _mm_store_si128((__m128i *)coeff, src[4]); + coeff += 8; + _mm_store_si128((__m128i *)coeff, src[5]); + coeff += 8; + _mm_store_si128((__m128i *)coeff, src[6]); + coeff += 8; + _mm_store_si128((__m128i *)coeff, src[7]); +} + +void aom_hadamard_16x16_sse2(int16_t const *src_diff, int src_stride, + int16_t *coeff) { + int idx; + for (idx = 0; idx < 4; ++idx) { + int16_t const *src_ptr = + src_diff + (idx >> 1) * 8 * src_stride + (idx & 0x01) * 8; + aom_hadamard_8x8_sse2(src_ptr, src_stride, coeff + idx * 64); + } + + for (idx = 0; idx < 64; idx += 8) { + __m128i coeff0 = _mm_load_si128((const __m128i *)coeff); + __m128i coeff1 = _mm_load_si128((const __m128i *)(coeff + 64)); + __m128i coeff2 = _mm_load_si128((const __m128i *)(coeff + 128)); + __m128i coeff3 = _mm_load_si128((const __m128i *)(coeff + 192)); + + __m128i b0 = _mm_add_epi16(coeff0, coeff1); + __m128i b1 = _mm_sub_epi16(coeff0, coeff1); + __m128i b2 = _mm_add_epi16(coeff2, coeff3); + __m128i b3 = _mm_sub_epi16(coeff2, coeff3); + + b0 = _mm_srai_epi16(b0, 1); + b1 = _mm_srai_epi16(b1, 1); + b2 = _mm_srai_epi16(b2, 1); + b3 = _mm_srai_epi16(b3, 1); + + coeff0 = _mm_add_epi16(b0, b2); + coeff1 = _mm_add_epi16(b1, b3); + _mm_store_si128((__m128i *)coeff, coeff0); + _mm_store_si128((__m128i *)(coeff + 64), coeff1); + + coeff2 = _mm_sub_epi16(b0, b2); + coeff3 = _mm_sub_epi16(b1, b3); + _mm_store_si128((__m128i *)(coeff + 128), coeff2); + _mm_store_si128((__m128i *)(coeff + 192), coeff3); + + coeff += 8; + } +} + +int aom_satd_sse2(const int16_t *coeff, int length) { + int i; + const __m128i zero = _mm_setzero_si128(); + __m128i accum = zero; + + for (i = 0; i < length; i += 8) { + const __m128i src_line = _mm_load_si128((const __m128i *)coeff); + const __m128i inv = _mm_sub_epi16(zero, src_line); + const __m128i abs = _mm_max_epi16(src_line, inv); // abs(src_line) + const __m128i abs_lo = _mm_unpacklo_epi16(abs, zero); + const __m128i abs_hi = _mm_unpackhi_epi16(abs, zero); + const __m128i sum = _mm_add_epi32(abs_lo, abs_hi); + accum = _mm_add_epi32(accum, sum); + coeff += 8; + } + + { // cascading summation of accum + __m128i hi = _mm_srli_si128(accum, 8); + accum = _mm_add_epi32(accum, hi); + hi = _mm_srli_epi64(accum, 32); + accum = _mm_add_epi32(accum, hi); + } + + return _mm_cvtsi128_si32(accum); +} + +void aom_int_pro_row_sse2(int16_t *hbuf, uint8_t const *ref, int ref_stride, + int height) { + int idx; + __m128i zero = _mm_setzero_si128(); + __m128i src_line = _mm_loadu_si128((const __m128i *)ref); + __m128i s0 = _mm_unpacklo_epi8(src_line, zero); + __m128i s1 = _mm_unpackhi_epi8(src_line, zero); + __m128i t0, t1; + int height_1 = height - 1; + ref += ref_stride; + + for (idx = 1; idx < height_1; idx += 2) { + src_line = _mm_loadu_si128((const __m128i *)ref); + t0 = _mm_unpacklo_epi8(src_line, zero); + t1 = _mm_unpackhi_epi8(src_line, zero); + s0 = _mm_adds_epu16(s0, t0); + s1 = _mm_adds_epu16(s1, t1); + ref += ref_stride; + + src_line = _mm_loadu_si128((const __m128i *)ref); + t0 = _mm_unpacklo_epi8(src_line, zero); + t1 = _mm_unpackhi_epi8(src_line, zero); + s0 = _mm_adds_epu16(s0, t0); + s1 = _mm_adds_epu16(s1, t1); + ref += ref_stride; + } + + src_line = _mm_loadu_si128((const __m128i *)ref); + t0 = _mm_unpacklo_epi8(src_line, zero); + t1 = _mm_unpackhi_epi8(src_line, zero); + s0 = _mm_adds_epu16(s0, t0); + s1 = _mm_adds_epu16(s1, t1); + + if (height == 64) { + s0 = _mm_srai_epi16(s0, 5); + s1 = _mm_srai_epi16(s1, 5); + } else if (height == 32) { + s0 = _mm_srai_epi16(s0, 4); + s1 = _mm_srai_epi16(s1, 4); + } else { + s0 = _mm_srai_epi16(s0, 3); + s1 = _mm_srai_epi16(s1, 3); + } + + _mm_storeu_si128((__m128i *)hbuf, s0); + hbuf += 8; + _mm_storeu_si128((__m128i *)hbuf, s1); +} + +int16_t aom_int_pro_col_sse2(uint8_t const *ref, int width) { + __m128i zero = _mm_setzero_si128(); + __m128i src_line = _mm_load_si128((const __m128i *)ref); + __m128i s0 = _mm_sad_epu8(src_line, zero); + __m128i s1; + int i; + + for (i = 16; i < width; i += 16) { + ref += 16; + src_line = _mm_load_si128((const __m128i *)ref); + s1 = _mm_sad_epu8(src_line, zero); + s0 = _mm_adds_epu16(s0, s1); + } + + s1 = _mm_srli_si128(s0, 8); + s0 = _mm_adds_epu16(s0, s1); + + return _mm_extract_epi16(s0, 0); +} + +int aom_vector_var_sse2(int16_t const *ref, int16_t const *src, int bwl) { + int idx; + int width = 4 << bwl; + int16_t mean; + __m128i v0 = _mm_loadu_si128((const __m128i *)ref); + __m128i v1 = _mm_load_si128((const __m128i *)src); + __m128i diff = _mm_subs_epi16(v0, v1); + __m128i sum = diff; + __m128i sse = _mm_madd_epi16(diff, diff); + + ref += 8; + src += 8; + + for (idx = 8; idx < width; idx += 8) { + v0 = _mm_loadu_si128((const __m128i *)ref); + v1 = _mm_load_si128((const __m128i *)src); + diff = _mm_subs_epi16(v0, v1); + + sum = _mm_add_epi16(sum, diff); + v0 = _mm_madd_epi16(diff, diff); + sse = _mm_add_epi32(sse, v0); + + ref += 8; + src += 8; + } + + v0 = _mm_srli_si128(sum, 8); + sum = _mm_add_epi16(sum, v0); + v0 = _mm_srli_epi64(sum, 32); + sum = _mm_add_epi16(sum, v0); + v0 = _mm_srli_epi32(sum, 16); + sum = _mm_add_epi16(sum, v0); + + v1 = _mm_srli_si128(sse, 8); + sse = _mm_add_epi32(sse, v1); + v1 = _mm_srli_epi64(sse, 32); + sse = _mm_add_epi32(sse, v1); + + mean = _mm_extract_epi16(sum, 0); + + return _mm_cvtsi128_si32(sse) - ((mean * mean) >> (bwl + 2)); +} diff --git a/third_party/aom/aom_dsp/x86/avg_ssse3_x86_64.asm b/third_party/aom/aom_dsp/x86/avg_ssse3_x86_64.asm new file mode 100644 index 0000000000..b2d150296b --- /dev/null +++ b/third_party/aom/aom_dsp/x86/avg_ssse3_x86_64.asm @@ -0,0 +1,124 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%define private_prefix aom + +%include "third_party/x86inc/x86inc.asm" + +; This file provides SSSE3 version of the hadamard transformation. Part +; of the macro definitions are originally derived from the ffmpeg project. +; The current version applies to x86 64-bit only. + +SECTION .text + +%if ARCH_X86_64 +; matrix transpose +%macro INTERLEAVE_2X 4 + punpckh%1 m%4, m%2, m%3 + punpckl%1 m%2, m%3 + SWAP %3, %4 +%endmacro + +%macro TRANSPOSE8X8 9 + INTERLEAVE_2X wd, %1, %2, %9 + INTERLEAVE_2X wd, %3, %4, %9 + INTERLEAVE_2X wd, %5, %6, %9 + INTERLEAVE_2X wd, %7, %8, %9 + + INTERLEAVE_2X dq, %1, %3, %9 + INTERLEAVE_2X dq, %2, %4, %9 + INTERLEAVE_2X dq, %5, %7, %9 + INTERLEAVE_2X dq, %6, %8, %9 + + INTERLEAVE_2X qdq, %1, %5, %9 + INTERLEAVE_2X qdq, %3, %7, %9 + INTERLEAVE_2X qdq, %2, %6, %9 + INTERLEAVE_2X qdq, %4, %8, %9 + + SWAP %2, %5 + SWAP %4, %7 +%endmacro + +%macro HMD8_1D 0 + psubw m8, m0, m1 + psubw m9, m2, m3 + paddw m0, m1 + paddw m2, m3 + SWAP 1, 8 + SWAP 3, 9 + psubw m8, m4, m5 + psubw m9, m6, m7 + paddw m4, m5 + paddw m6, m7 + SWAP 5, 8 + SWAP 7, 9 + + psubw m8, m0, m2 + psubw m9, m1, m3 + paddw m0, m2 + paddw m1, m3 + SWAP 2, 8 + SWAP 3, 9 + psubw m8, m4, m6 + psubw m9, m5, m7 + paddw m4, m6 + paddw m5, m7 + SWAP 6, 8 + SWAP 7, 9 + + psubw m8, m0, m4 + psubw m9, m1, m5 + paddw m0, m4 + paddw m1, m5 + SWAP 4, 8 + SWAP 5, 9 + psubw m8, m2, m6 + psubw m9, m3, m7 + paddw m2, m6 + paddw m3, m7 + SWAP 6, 8 + SWAP 7, 9 +%endmacro + +INIT_XMM ssse3 +cglobal hadamard_8x8, 3, 5, 10, input, stride, output + lea r3, [2 * strideq] + lea r4, [4 * strideq] + + mova m0, [inputq] + mova m1, [inputq + r3] + lea inputq, [inputq + r4] + mova m2, [inputq] + mova m3, [inputq + r3] + lea inputq, [inputq + r4] + mova m4, [inputq] + mova m5, [inputq + r3] + lea inputq, [inputq + r4] + mova m6, [inputq] + mova m7, [inputq + r3] + + HMD8_1D + TRANSPOSE8X8 0, 1, 2, 3, 4, 5, 6, 7, 9 + HMD8_1D + + mova [outputq + 0], m0 + mova [outputq + 16], m1 + mova [outputq + 32], m2 + mova [outputq + 48], m3 + mova [outputq + 64], m4 + mova [outputq + 80], m5 + mova [outputq + 96], m6 + mova [outputq + 112], m7 + + RET +%endif diff --git a/third_party/aom/aom_dsp/x86/blend_a64_hmask_sse4.c b/third_party/aom/aom_dsp/x86/blend_a64_hmask_sse4.c new file mode 100644 index 0000000000..e916e4ff9e --- /dev/null +++ b/third_party/aom/aom_dsp/x86/blend_a64_hmask_sse4.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom/aom_integer.h" + +#include "./aom_dsp_rtcd.h" + +// To start out, just dispatch to the function using the 2D mask and +// pass mask stride as 0. This can be improved upon if necessary. + +void aom_blend_a64_hmask_sse4_1(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + aom_blend_a64_mask_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, 0, h, w, 0, 0); +} + +#if CONFIG_HIGHBITDEPTH +void aom_highbd_blend_a64_hmask_sse4_1( + uint8_t *dst_8, uint32_t dst_stride, const uint8_t *src0_8, + uint32_t src0_stride, const uint8_t *src1_8, uint32_t src1_stride, + const uint8_t *mask, int h, int w, int bd) { + aom_highbd_blend_a64_mask_sse4_1(dst_8, dst_stride, src0_8, src0_stride, + src1_8, src1_stride, mask, 0, h, w, 0, 0, + bd); +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c b/third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c new file mode 100644 index 0000000000..68d74e5176 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c @@ -0,0 +1,924 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // SSE4.1 + +#include + +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/blend.h" + +#include "aom_dsp/x86/synonyms.h" +#include "aom_dsp/x86/blend_sse4.h" + +#include "./aom_dsp_rtcd.h" + +////////////////////////////////////////////////////////////////////////////// +// No sub-sampling +////////////////////////////////////////////////////////////////////////////// + +static void blend_a64_mask_w4_sse4_1(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, + int h, int w) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + (void)w; + + do { + const __m128i v_m0_b = xx_loadl_32(mask); + const __m128i v_m0_w = _mm_cvtepu8_epi16(v_m0_b); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend_4(src0, src1, v_m0_w, v_m1_w); + + const __m128i v_res_b = _mm_packus_epi16(v_res_w, v_res_w); + + xx_storel_32(dst, v_res_b); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += mask_stride; + } while (--h); +} + +static void blend_a64_mask_w8_sse4_1(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, + int h, int w) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + (void)w; + + do { + const __m128i v_m0_b = xx_loadl_64(mask); + const __m128i v_m0_w = _mm_cvtepu8_epi16(v_m0_b); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend_8(src0, src1, v_m0_w, v_m1_w); + + const __m128i v_res_b = _mm_packus_epi16(v_res_w, v_res_w); + + xx_storel_64(dst, v_res_b); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += mask_stride; + } while (--h); +} + +static void blend_a64_mask_w16n_sse4_1( + uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + int c; + for (c = 0; c < w; c += 16) { + const __m128i v_m0l_b = xx_loadl_64(mask + c); + const __m128i v_m0h_b = xx_loadl_64(mask + c + 8); + const __m128i v_m0l_w = _mm_cvtepu8_epi16(v_m0l_b); + const __m128i v_m0h_w = _mm_cvtepu8_epi16(v_m0h_b); + const __m128i v_m1l_w = _mm_sub_epi16(v_maxval_w, v_m0l_w); + const __m128i v_m1h_w = _mm_sub_epi16(v_maxval_w, v_m0h_w); + + const __m128i v_resl_w = blend_8(src0 + c, src1 + c, v_m0l_w, v_m1l_w); + const __m128i v_resh_w = + blend_8(src0 + c + 8, src1 + c + 8, v_m0h_w, v_m1h_w); + + const __m128i v_res_b = _mm_packus_epi16(v_resl_w, v_resh_w); + + xx_storeu_128(dst + c, v_res_b); + } + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += mask_stride; + } while (--h); +} + +////////////////////////////////////////////////////////////////////////////// +// Horizontal sub-sampling +////////////////////////////////////////////////////////////////////////////// + +static void blend_a64_mask_sx_w4_sse4_1( + uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + const __m128i v_zmask_b = _mm_set_epi8(0, 0xff, 0, 0xff, 0, 0xff, 0, 0xff, 0, + 0xff, 0, 0xff, 0, 0xff, 0, 0xff); + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + (void)w; + + do { + const __m128i v_r_b = xx_loadl_64(mask); + const __m128i v_a_b = _mm_avg_epu8(v_r_b, _mm_srli_si128(v_r_b, 1)); + + const __m128i v_m0_w = _mm_and_si128(v_a_b, v_zmask_b); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend_4(src0, src1, v_m0_w, v_m1_w); + + const __m128i v_res_b = _mm_packus_epi16(v_res_w, v_res_w); + + xx_storel_32(dst, v_res_b); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += mask_stride; + } while (--h); +} + +static void blend_a64_mask_sx_w8_sse4_1( + uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + const __m128i v_zmask_b = _mm_set_epi8(0, 0xff, 0, 0xff, 0, 0xff, 0, 0xff, 0, + 0xff, 0, 0xff, 0, 0xff, 0, 0xff); + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + (void)w; + + do { + const __m128i v_r_b = xx_loadu_128(mask); + const __m128i v_a_b = _mm_avg_epu8(v_r_b, _mm_srli_si128(v_r_b, 1)); + + const __m128i v_m0_w = _mm_and_si128(v_a_b, v_zmask_b); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend_8(src0, src1, v_m0_w, v_m1_w); + + const __m128i v_res_b = _mm_packus_epi16(v_res_w, v_res_w); + + xx_storel_64(dst, v_res_b); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += mask_stride; + } while (--h); +} + +static void blend_a64_mask_sx_w16n_sse4_1( + uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + const __m128i v_zmask_b = _mm_set_epi8(0, 0xff, 0, 0xff, 0, 0xff, 0, 0xff, 0, + 0xff, 0, 0xff, 0, 0xff, 0, 0xff); + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + int c; + for (c = 0; c < w; c += 16) { + const __m128i v_rl_b = xx_loadu_128(mask + 2 * c); + const __m128i v_rh_b = xx_loadu_128(mask + 2 * c + 16); + const __m128i v_al_b = _mm_avg_epu8(v_rl_b, _mm_srli_si128(v_rl_b, 1)); + const __m128i v_ah_b = _mm_avg_epu8(v_rh_b, _mm_srli_si128(v_rh_b, 1)); + + const __m128i v_m0l_w = _mm_and_si128(v_al_b, v_zmask_b); + const __m128i v_m0h_w = _mm_and_si128(v_ah_b, v_zmask_b); + const __m128i v_m1l_w = _mm_sub_epi16(v_maxval_w, v_m0l_w); + const __m128i v_m1h_w = _mm_sub_epi16(v_maxval_w, v_m0h_w); + + const __m128i v_resl_w = blend_8(src0 + c, src1 + c, v_m0l_w, v_m1l_w); + const __m128i v_resh_w = + blend_8(src0 + c + 8, src1 + c + 8, v_m0h_w, v_m1h_w); + + const __m128i v_res_b = _mm_packus_epi16(v_resl_w, v_resh_w); + + xx_storeu_128(dst + c, v_res_b); + } + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += mask_stride; + } while (--h); +} + +////////////////////////////////////////////////////////////////////////////// +// Vertical sub-sampling +////////////////////////////////////////////////////////////////////////////// + +static void blend_a64_mask_sy_w4_sse4_1( + uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + (void)w; + + do { + const __m128i v_ra_b = xx_loadl_32(mask); + const __m128i v_rb_b = xx_loadl_32(mask + mask_stride); + const __m128i v_a_b = _mm_avg_epu8(v_ra_b, v_rb_b); + + const __m128i v_m0_w = _mm_cvtepu8_epi16(v_a_b); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend_4(src0, src1, v_m0_w, v_m1_w); + + const __m128i v_res_b = _mm_packus_epi16(v_res_w, v_res_w); + + xx_storel_32(dst, v_res_b); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 2 * mask_stride; + } while (--h); +} + +static void blend_a64_mask_sy_w8_sse4_1( + uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + (void)w; + + do { + const __m128i v_ra_b = xx_loadl_64(mask); + const __m128i v_rb_b = xx_loadl_64(mask + mask_stride); + const __m128i v_a_b = _mm_avg_epu8(v_ra_b, v_rb_b); + + const __m128i v_m0_w = _mm_cvtepu8_epi16(v_a_b); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend_8(src0, src1, v_m0_w, v_m1_w); + + const __m128i v_res_b = _mm_packus_epi16(v_res_w, v_res_w); + + xx_storel_64(dst, v_res_b); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 2 * mask_stride; + } while (--h); +} + +static void blend_a64_mask_sy_w16n_sse4_1( + uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + const __m128i v_zero = _mm_setzero_si128(); + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + int c; + for (c = 0; c < w; c += 16) { + const __m128i v_ra_b = xx_loadu_128(mask + c); + const __m128i v_rb_b = xx_loadu_128(mask + c + mask_stride); + const __m128i v_a_b = _mm_avg_epu8(v_ra_b, v_rb_b); + + const __m128i v_m0l_w = _mm_cvtepu8_epi16(v_a_b); + const __m128i v_m0h_w = _mm_unpackhi_epi8(v_a_b, v_zero); + const __m128i v_m1l_w = _mm_sub_epi16(v_maxval_w, v_m0l_w); + const __m128i v_m1h_w = _mm_sub_epi16(v_maxval_w, v_m0h_w); + + const __m128i v_resl_w = blend_8(src0 + c, src1 + c, v_m0l_w, v_m1l_w); + const __m128i v_resh_w = + blend_8(src0 + c + 8, src1 + c + 8, v_m0h_w, v_m1h_w); + + const __m128i v_res_b = _mm_packus_epi16(v_resl_w, v_resh_w); + + xx_storeu_128(dst + c, v_res_b); + } + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 2 * mask_stride; + } while (--h); +} + +////////////////////////////////////////////////////////////////////////////// +// Horizontal and Vertical sub-sampling +////////////////////////////////////////////////////////////////////////////// + +static void blend_a64_mask_sx_sy_w4_sse4_1( + uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + const __m128i v_zmask_b = _mm_set_epi8(0, 0xff, 0, 0xff, 0, 0xff, 0, 0xff, 0, + 0xff, 0, 0xff, 0, 0xff, 0, 0xff); + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + (void)w; + + do { + const __m128i v_ra_b = xx_loadl_64(mask); + const __m128i v_rb_b = xx_loadl_64(mask + mask_stride); + const __m128i v_rvs_b = _mm_add_epi8(v_ra_b, v_rb_b); + const __m128i v_rvsa_w = _mm_and_si128(v_rvs_b, v_zmask_b); + const __m128i v_rvsb_w = + _mm_and_si128(_mm_srli_si128(v_rvs_b, 1), v_zmask_b); + const __m128i v_rs_w = _mm_add_epi16(v_rvsa_w, v_rvsb_w); + + const __m128i v_m0_w = xx_roundn_epu16(v_rs_w, 2); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend_4(src0, src1, v_m0_w, v_m1_w); + + const __m128i v_res_b = _mm_packus_epi16(v_res_w, v_res_w); + + xx_storel_32(dst, v_res_b); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 2 * mask_stride; + } while (--h); +} + +static void blend_a64_mask_sx_sy_w8_sse4_1( + uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + const __m128i v_zmask_b = _mm_set_epi8(0, 0xff, 0, 0xff, 0, 0xff, 0, 0xff, 0, + 0xff, 0, 0xff, 0, 0xff, 0, 0xff); + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + (void)w; + + do { + const __m128i v_ra_b = xx_loadu_128(mask); + const __m128i v_rb_b = xx_loadu_128(mask + mask_stride); + const __m128i v_rvs_b = _mm_add_epi8(v_ra_b, v_rb_b); + const __m128i v_rvsa_w = _mm_and_si128(v_rvs_b, v_zmask_b); + const __m128i v_rvsb_w = + _mm_and_si128(_mm_srli_si128(v_rvs_b, 1), v_zmask_b); + const __m128i v_rs_w = _mm_add_epi16(v_rvsa_w, v_rvsb_w); + + const __m128i v_m0_w = xx_roundn_epu16(v_rs_w, 2); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend_8(src0, src1, v_m0_w, v_m1_w); + + const __m128i v_res_b = _mm_packus_epi16(v_res_w, v_res_w); + + xx_storel_64(dst, v_res_b); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 2 * mask_stride; + } while (--h); +} + +static void blend_a64_mask_sx_sy_w16n_sse4_1( + uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + const __m128i v_zmask_b = _mm_set_epi8(0, 0xff, 0, 0xff, 0, 0xff, 0, 0xff, 0, + 0xff, 0, 0xff, 0, 0xff, 0, 0xff); + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + int c; + for (c = 0; c < w; c += 16) { + const __m128i v_ral_b = xx_loadu_128(mask + 2 * c); + const __m128i v_rah_b = xx_loadu_128(mask + 2 * c + 16); + const __m128i v_rbl_b = xx_loadu_128(mask + mask_stride + 2 * c); + const __m128i v_rbh_b = xx_loadu_128(mask + mask_stride + 2 * c + 16); + const __m128i v_rvsl_b = _mm_add_epi8(v_ral_b, v_rbl_b); + const __m128i v_rvsh_b = _mm_add_epi8(v_rah_b, v_rbh_b); + const __m128i v_rvsal_w = _mm_and_si128(v_rvsl_b, v_zmask_b); + const __m128i v_rvsah_w = _mm_and_si128(v_rvsh_b, v_zmask_b); + const __m128i v_rvsbl_w = + _mm_and_si128(_mm_srli_si128(v_rvsl_b, 1), v_zmask_b); + const __m128i v_rvsbh_w = + _mm_and_si128(_mm_srli_si128(v_rvsh_b, 1), v_zmask_b); + const __m128i v_rsl_w = _mm_add_epi16(v_rvsal_w, v_rvsbl_w); + const __m128i v_rsh_w = _mm_add_epi16(v_rvsah_w, v_rvsbh_w); + + const __m128i v_m0l_w = xx_roundn_epu16(v_rsl_w, 2); + const __m128i v_m0h_w = xx_roundn_epu16(v_rsh_w, 2); + const __m128i v_m1l_w = _mm_sub_epi16(v_maxval_w, v_m0l_w); + const __m128i v_m1h_w = _mm_sub_epi16(v_maxval_w, v_m0h_w); + + const __m128i v_resl_w = blend_8(src0 + c, src1 + c, v_m0l_w, v_m1l_w); + const __m128i v_resh_w = + blend_8(src0 + c + 8, src1 + c + 8, v_m0h_w, v_m1h_w); + + const __m128i v_res_b = _mm_packus_epi16(v_resl_w, v_resh_w); + + xx_storeu_128(dst + c, v_res_b); + } + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 2 * mask_stride; + } while (--h); +} + +////////////////////////////////////////////////////////////////////////////// +// Dispatch +////////////////////////////////////////////////////////////////////////////// + +void aom_blend_a64_mask_sse4_1(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, + int w, int suby, int subx) { + typedef void (*blend_fn)( + uint8_t * dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w); + + // Dimensions are: width_index X subx X suby + static const blend_fn blend[3][2][2] = { + { // w % 16 == 0 + { blend_a64_mask_w16n_sse4_1, blend_a64_mask_sy_w16n_sse4_1 }, + { blend_a64_mask_sx_w16n_sse4_1, blend_a64_mask_sx_sy_w16n_sse4_1 } }, + { // w == 4 + { blend_a64_mask_w4_sse4_1, blend_a64_mask_sy_w4_sse4_1 }, + { blend_a64_mask_sx_w4_sse4_1, blend_a64_mask_sx_sy_w4_sse4_1 } }, + { // w == 8 + { blend_a64_mask_w8_sse4_1, blend_a64_mask_sy_w8_sse4_1 }, + { blend_a64_mask_sx_w8_sse4_1, blend_a64_mask_sx_sy_w8_sse4_1 } } + }; + + assert(IMPLIES(src0 == dst, src0_stride == dst_stride)); + assert(IMPLIES(src1 == dst, src1_stride == dst_stride)); + + assert(h >= 1); + assert(w >= 1); + assert(IS_POWER_OF_TWO(h)); + assert(IS_POWER_OF_TWO(w)); + + if (UNLIKELY((h | w) & 3)) { // if (w <= 2 || h <= 2) + aom_blend_a64_mask_c(dst, dst_stride, src0, src0_stride, src1, src1_stride, + mask, mask_stride, h, w, suby, subx); + } else { + blend[(w >> 2) & 3][subx != 0][suby != 0](dst, dst_stride, src0, + src0_stride, src1, src1_stride, + mask, mask_stride, h, w); + } +} + +#if CONFIG_HIGHBITDEPTH +////////////////////////////////////////////////////////////////////////////// +// No sub-sampling +////////////////////////////////////////////////////////////////////////////// + +static INLINE void blend_a64_mask_bn_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, blend_unit_fn blend) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + const __m128i v_m0_b = xx_loadl_32(mask); + const __m128i v_m0_w = _mm_cvtepu8_epi16(v_m0_b); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend(src0, src1, v_m0_w, v_m1_w); + + xx_storel_64(dst, v_res_w); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += mask_stride; + } while (--h); +} + +static void blend_a64_mask_b10_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + (void)w; + blend_a64_mask_bn_w4_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, blend_4_b10); +} + +static void blend_a64_mask_b12_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + (void)w; + blend_a64_mask_bn_w4_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, blend_4_b12); +} + +static INLINE void blend_a64_mask_bn_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w, + blend_unit_fn blend) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + int c; + for (c = 0; c < w; c += 8) { + const __m128i v_m0_b = xx_loadl_64(mask + c); + const __m128i v_m0_w = _mm_cvtepu8_epi16(v_m0_b); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend(src0 + c, src1 + c, v_m0_w, v_m1_w); + + xx_storeu_128(dst + c, v_res_w); + } + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += mask_stride; + } while (--h); +} + +static void blend_a64_mask_b10_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + blend_a64_mask_bn_w8n_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, w, + blend_8_b10); +} + +static void blend_a64_mask_b12_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + blend_a64_mask_bn_w8n_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, w, + blend_8_b12); +} + +////////////////////////////////////////////////////////////////////////////// +// Horizontal sub-sampling +////////////////////////////////////////////////////////////////////////////// + +static INLINE void blend_a64_mask_bn_sx_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, blend_unit_fn blend) { + const __m128i v_zmask_b = _mm_set_epi8(0, 0xff, 0, 0xff, 0, 0xff, 0, 0xff, 0, + 0xff, 0, 0xff, 0, 0xff, 0, 0xff); + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + const __m128i v_r_b = xx_loadl_64(mask); + const __m128i v_a_b = _mm_avg_epu8(v_r_b, _mm_srli_si128(v_r_b, 1)); + + const __m128i v_m0_w = _mm_and_si128(v_a_b, v_zmask_b); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend(src0, src1, v_m0_w, v_m1_w); + + xx_storel_64(dst, v_res_w); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += mask_stride; + } while (--h); +} + +static void blend_a64_mask_b10_sx_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + (void)w; + blend_a64_mask_bn_sx_w4_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, + blend_4_b10); +} + +static void blend_a64_mask_b12_sx_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + (void)w; + blend_a64_mask_bn_sx_w4_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, + blend_4_b12); +} + +static INLINE void blend_a64_mask_bn_sx_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w, + blend_unit_fn blend) { + const __m128i v_zmask_b = _mm_set_epi8(0, 0xff, 0, 0xff, 0, 0xff, 0, 0xff, 0, + 0xff, 0, 0xff, 0, 0xff, 0, 0xff); + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + int c; + for (c = 0; c < w; c += 8) { + const __m128i v_r_b = xx_loadu_128(mask + 2 * c); + const __m128i v_a_b = _mm_avg_epu8(v_r_b, _mm_srli_si128(v_r_b, 1)); + + const __m128i v_m0_w = _mm_and_si128(v_a_b, v_zmask_b); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend(src0 + c, src1 + c, v_m0_w, v_m1_w); + + xx_storeu_128(dst + c, v_res_w); + } + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += mask_stride; + } while (--h); +} + +static void blend_a64_mask_b10_sx_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + blend_a64_mask_bn_sx_w8n_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, w, + blend_8_b10); +} + +static void blend_a64_mask_b12_sx_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + blend_a64_mask_bn_sx_w8n_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, w, + blend_8_b12); +} + +////////////////////////////////////////////////////////////////////////////// +// Vertical sub-sampling +////////////////////////////////////////////////////////////////////////////// + +static INLINE void blend_a64_mask_bn_sy_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, blend_unit_fn blend) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + const __m128i v_ra_b = xx_loadl_32(mask); + const __m128i v_rb_b = xx_loadl_32(mask + mask_stride); + const __m128i v_a_b = _mm_avg_epu8(v_ra_b, v_rb_b); + + const __m128i v_m0_w = _mm_cvtepu8_epi16(v_a_b); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend(src0, src1, v_m0_w, v_m1_w); + + xx_storel_64(dst, v_res_w); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 2 * mask_stride; + } while (--h); +} + +static void blend_a64_mask_b10_sy_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + (void)w; + blend_a64_mask_bn_sy_w4_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, + blend_4_b10); +} + +static void blend_a64_mask_b12_sy_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + (void)w; + blend_a64_mask_bn_sy_w4_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, + blend_4_b12); +} + +static INLINE void blend_a64_mask_bn_sy_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w, + blend_unit_fn blend) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + int c; + for (c = 0; c < w; c += 8) { + const __m128i v_ra_b = xx_loadl_64(mask + c); + const __m128i v_rb_b = xx_loadl_64(mask + c + mask_stride); + const __m128i v_a_b = _mm_avg_epu8(v_ra_b, v_rb_b); + + const __m128i v_m0_w = _mm_cvtepu8_epi16(v_a_b); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend(src0 + c, src1 + c, v_m0_w, v_m1_w); + + xx_storeu_128(dst + c, v_res_w); + } + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 2 * mask_stride; + } while (--h); +} + +static void blend_a64_mask_b10_sy_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + blend_a64_mask_bn_sy_w8n_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, w, + blend_8_b10); +} + +static void blend_a64_mask_b12_sy_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + blend_a64_mask_bn_sy_w8n_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, w, + blend_8_b12); +} + +////////////////////////////////////////////////////////////////////////////// +// Horizontal and Vertical sub-sampling +////////////////////////////////////////////////////////////////////////////// + +static INLINE void blend_a64_mask_bn_sx_sy_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, blend_unit_fn blend) { + const __m128i v_zmask_b = _mm_set_epi8(0, 0xff, 0, 0xff, 0, 0xff, 0, 0xff, 0, + 0xff, 0, 0xff, 0, 0xff, 0, 0xff); + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + const __m128i v_ra_b = xx_loadl_64(mask); + const __m128i v_rb_b = xx_loadl_64(mask + mask_stride); + const __m128i v_rvs_b = _mm_add_epi8(v_ra_b, v_rb_b); + const __m128i v_rvsa_w = _mm_and_si128(v_rvs_b, v_zmask_b); + const __m128i v_rvsb_w = + _mm_and_si128(_mm_srli_si128(v_rvs_b, 1), v_zmask_b); + const __m128i v_rs_w = _mm_add_epi16(v_rvsa_w, v_rvsb_w); + + const __m128i v_m0_w = xx_roundn_epu16(v_rs_w, 2); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend(src0, src1, v_m0_w, v_m1_w); + + xx_storel_64(dst, v_res_w); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 2 * mask_stride; + } while (--h); +} + +static void blend_a64_mask_b10_sx_sy_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + (void)w; + blend_a64_mask_bn_sx_sy_w4_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, + blend_4_b10); +} + +static void blend_a64_mask_b12_sx_sy_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + (void)w; + blend_a64_mask_bn_sx_sy_w4_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, + blend_4_b12); +} + +static INLINE void blend_a64_mask_bn_sx_sy_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w, + blend_unit_fn blend) { + const __m128i v_zmask_b = _mm_set_epi8(0, 0xff, 0, 0xff, 0, 0xff, 0, 0xff, 0, + 0xff, 0, 0xff, 0, 0xff, 0, 0xff); + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + int c; + for (c = 0; c < w; c += 8) { + const __m128i v_ra_b = xx_loadu_128(mask + 2 * c); + const __m128i v_rb_b = xx_loadu_128(mask + 2 * c + mask_stride); + const __m128i v_rvs_b = _mm_add_epi8(v_ra_b, v_rb_b); + const __m128i v_rvsa_w = _mm_and_si128(v_rvs_b, v_zmask_b); + const __m128i v_rvsb_w = + _mm_and_si128(_mm_srli_si128(v_rvs_b, 1), v_zmask_b); + const __m128i v_rs_w = _mm_add_epi16(v_rvsa_w, v_rvsb_w); + + const __m128i v_m0_w = xx_roundn_epu16(v_rs_w, 2); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend(src0 + c, src1 + c, v_m0_w, v_m1_w); + + xx_storeu_128(dst + c, v_res_w); + } + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 2 * mask_stride; + } while (--h); +} + +static void blend_a64_mask_b10_sx_sy_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + blend_a64_mask_bn_sx_sy_w8n_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, w, + blend_8_b10); +} + +static void blend_a64_mask_b12_sx_sy_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w) { + blend_a64_mask_bn_sx_sy_w8n_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, h, w, + blend_8_b12); +} + +////////////////////////////////////////////////////////////////////////////// +// Dispatch +////////////////////////////////////////////////////////////////////////////// + +void aom_highbd_blend_a64_mask_sse4_1(uint8_t *dst_8, uint32_t dst_stride, + const uint8_t *src0_8, + uint32_t src0_stride, + const uint8_t *src1_8, + uint32_t src1_stride, const uint8_t *mask, + uint32_t mask_stride, int h, int w, + int suby, int subx, int bd) { + typedef void (*blend_fn)( + uint16_t * dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w); + + // Dimensions are: bd_index X width_index X subx X suby + static const blend_fn blend[2][2][2][2] = { + { // bd == 8 or 10 + { // w % 8 == 0 + { blend_a64_mask_b10_w8n_sse4_1, blend_a64_mask_b10_sy_w8n_sse4_1 }, + { blend_a64_mask_b10_sx_w8n_sse4_1, + blend_a64_mask_b10_sx_sy_w8n_sse4_1 } }, + { // w == 4 + { blend_a64_mask_b10_w4_sse4_1, blend_a64_mask_b10_sy_w4_sse4_1 }, + { blend_a64_mask_b10_sx_w4_sse4_1, + blend_a64_mask_b10_sx_sy_w4_sse4_1 } } }, + { // bd == 12 + { // w % 8 == 0 + { blend_a64_mask_b12_w8n_sse4_1, blend_a64_mask_b12_sy_w8n_sse4_1 }, + { blend_a64_mask_b12_sx_w8n_sse4_1, + blend_a64_mask_b12_sx_sy_w8n_sse4_1 } }, + { // w == 4 + { blend_a64_mask_b12_w4_sse4_1, blend_a64_mask_b12_sy_w4_sse4_1 }, + { blend_a64_mask_b12_sx_w4_sse4_1, + blend_a64_mask_b12_sx_sy_w4_sse4_1 } } } + }; + + assert(IMPLIES(src0_8 == dst_8, src0_stride == dst_stride)); + assert(IMPLIES(src1_8 == dst_8, src1_stride == dst_stride)); + + assert(h >= 1); + assert(w >= 1); + assert(IS_POWER_OF_TWO(h)); + assert(IS_POWER_OF_TWO(w)); + + assert(bd == 8 || bd == 10 || bd == 12); + if (UNLIKELY((h | w) & 3)) { // if (w <= 2 || h <= 2) + aom_highbd_blend_a64_mask_c(dst_8, dst_stride, src0_8, src0_stride, src1_8, + src1_stride, mask, mask_stride, h, w, suby, + subx, bd); + } else { + uint16_t *const dst = CONVERT_TO_SHORTPTR(dst_8); + const uint16_t *const src0 = CONVERT_TO_SHORTPTR(src0_8); + const uint16_t *const src1 = CONVERT_TO_SHORTPTR(src1_8); + + blend[bd == 12][(w >> 2) & 1][subx != 0][suby != 0]( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, w); + } +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/x86/blend_a64_vmask_sse4.c b/third_party/aom/aom_dsp/x86/blend_a64_vmask_sse4.c new file mode 100644 index 0000000000..9dabe5b798 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/blend_a64_vmask_sse4.c @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // SSE4.1 + +#include + +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/blend.h" + +#include "aom_dsp/x86/synonyms.h" +#include "aom_dsp/x86/blend_sse4.h" + +#include "./aom_dsp_rtcd.h" + +////////////////////////////////////////////////////////////////////////////// +// Implementation - No sub-sampling +////////////////////////////////////////////////////////////////////////////// + +static void blend_a64_vmask_w4_sse4_1(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + (void)w; + + do { + const __m128i v_m0_w = _mm_set1_epi16(*mask); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend_4(src0, src1, v_m0_w, v_m1_w); + + const __m128i v_res_b = _mm_packus_epi16(v_res_w, v_res_w); + + xx_storel_32(dst, v_res_b); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 1; + } while (--h); +} + +static void blend_a64_vmask_w8_sse4_1(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + (void)w; + + do { + const __m128i v_m0_w = _mm_set1_epi16(*mask); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend_8(src0, src1, v_m0_w, v_m1_w); + + const __m128i v_res_b = _mm_packus_epi16(v_res_w, v_res_w); + + xx_storel_64(dst, v_res_b); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 1; + } while (--h); +} + +static void blend_a64_vmask_w16n_sse4_1(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, + uint32_t src0_stride, + const uint8_t *src1, + uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + int c; + const __m128i v_m0_w = _mm_set1_epi16(*mask); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + for (c = 0; c < w; c += 16) { + const __m128i v_resl_w = blend_8(src0 + c, src1 + c, v_m0_w, v_m1_w); + const __m128i v_resh_w = + blend_8(src0 + c + 8, src1 + c + 8, v_m0_w, v_m1_w); + + const __m128i v_res_b = _mm_packus_epi16(v_resl_w, v_resh_w); + + xx_storeu_128(dst + c, v_res_b); + } + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 1; + } while (--h); +} + +////////////////////////////////////////////////////////////////////////////// +// Dispatch +////////////////////////////////////////////////////////////////////////////// + +void aom_blend_a64_vmask_sse4_1(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + typedef void (*blend_fn)(uint8_t * dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w); + + // Dimension: width_index + static const blend_fn blend[9] = { + blend_a64_vmask_w16n_sse4_1, // w % 16 == 0 + aom_blend_a64_vmask_c, // w == 1 + aom_blend_a64_vmask_c, // w == 2 + NULL, // INVALID + blend_a64_vmask_w4_sse4_1, // w == 4 + NULL, // INVALID + NULL, // INVALID + NULL, // INVALID + blend_a64_vmask_w8_sse4_1, // w == 8 + }; + + assert(IMPLIES(src0 == dst, src0_stride == dst_stride)); + assert(IMPLIES(src1 == dst, src1_stride == dst_stride)); + + assert(h >= 1); + assert(w >= 1); + assert(IS_POWER_OF_TWO(h)); + assert(IS_POWER_OF_TWO(w)); + + blend[w & 0xf](dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, h, + w); +} + +#if CONFIG_HIGHBITDEPTH +////////////////////////////////////////////////////////////////////////////// +// Implementation - No sub-sampling +////////////////////////////////////////////////////////////////////////////// + +static INLINE void blend_a64_vmask_bn_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, blend_unit_fn blend) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + const __m128i v_m0_w = _mm_set1_epi16(*mask); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + + const __m128i v_res_w = blend(src0, src1, v_m0_w, v_m1_w); + + xx_storel_64(dst, v_res_w); + + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 1; + } while (--h); +} + +static void blend_a64_vmask_b10_w4_sse4_1(uint16_t *dst, uint32_t dst_stride, + const uint16_t *src0, + uint32_t src0_stride, + const uint16_t *src1, + uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + (void)w; + blend_a64_vmask_bn_w4_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, h, blend_4_b10); +} + +static void blend_a64_vmask_b12_w4_sse4_1(uint16_t *dst, uint32_t dst_stride, + const uint16_t *src0, + uint32_t src0_stride, + const uint16_t *src1, + uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + (void)w; + blend_a64_vmask_bn_w4_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, h, blend_4_b12); +} + +static INLINE void blend_a64_vmask_bn_w8n_sse4_1( + uint16_t *dst, uint32_t dst_stride, const uint16_t *src0, + uint32_t src0_stride, const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w, blend_unit_fn blend) { + const __m128i v_maxval_w = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + do { + int c; + const __m128i v_m0_w = _mm_set1_epi16(*mask); + const __m128i v_m1_w = _mm_sub_epi16(v_maxval_w, v_m0_w); + for (c = 0; c < w; c += 8) { + const __m128i v_res_w = blend(src0 + c, src1 + c, v_m0_w, v_m1_w); + + xx_storeu_128(dst + c, v_res_w); + } + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += 1; + } while (--h); +} + +static void blend_a64_vmask_b10_w8n_sse4_1(uint16_t *dst, uint32_t dst_stride, + const uint16_t *src0, + uint32_t src0_stride, + const uint16_t *src1, + uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + blend_a64_vmask_bn_w8n_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, h, w, blend_8_b10); +} + +static void blend_a64_vmask_b12_w8n_sse4_1(uint16_t *dst, uint32_t dst_stride, + const uint16_t *src0, + uint32_t src0_stride, + const uint16_t *src1, + uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + blend_a64_vmask_bn_w8n_sse4_1(dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, h, w, blend_8_b12); +} + +////////////////////////////////////////////////////////////////////////////// +// Dispatch +////////////////////////////////////////////////////////////////////////////// + +void aom_highbd_blend_a64_vmask_sse4_1( + uint8_t *dst_8, uint32_t dst_stride, const uint8_t *src0_8, + uint32_t src0_stride, const uint8_t *src1_8, uint32_t src1_stride, + const uint8_t *mask, int h, int w, int bd) { + typedef void (*blend_fn)(uint16_t * dst, uint32_t dst_stride, + const uint16_t *src0, uint32_t src0_stride, + const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w); + + // Dimensions are: bd_index X width_index + static const blend_fn blend[2][2] = { + { + // bd == 8 or 10 + blend_a64_vmask_b10_w8n_sse4_1, // w % 8 == 0 + blend_a64_vmask_b10_w4_sse4_1, // w == 4 + }, + { + // bd == 12 + blend_a64_vmask_b12_w8n_sse4_1, // w % 8 == 0 + blend_a64_vmask_b12_w4_sse4_1, // w == 4 + } + }; + + assert(IMPLIES(src0_8 == dst_8, src0_stride == dst_stride)); + assert(IMPLIES(src1_8 == dst_8, src1_stride == dst_stride)); + + assert(h >= 1); + assert(w >= 1); + assert(IS_POWER_OF_TWO(h)); + assert(IS_POWER_OF_TWO(w)); + + assert(bd == 8 || bd == 10 || bd == 12); + + if (UNLIKELY((h | w) & 3)) { // if (w <= 2 || h <= 2) + aom_highbd_blend_a64_vmask_c(dst_8, dst_stride, src0_8, src0_stride, src1_8, + src1_stride, mask, h, w, bd); + } else { + uint16_t *const dst = CONVERT_TO_SHORTPTR(dst_8); + const uint16_t *const src0 = CONVERT_TO_SHORTPTR(src0_8); + const uint16_t *const src1 = CONVERT_TO_SHORTPTR(src1_8); + + blend[bd == 12][(w >> 2) & 1](dst, dst_stride, src0, src0_stride, src1, + src1_stride, mask, h, w); + } +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/x86/blend_sse4.h b/third_party/aom/aom_dsp/x86/blend_sse4.h new file mode 100644 index 0000000000..daa2b2b3ae --- /dev/null +++ b/third_party/aom/aom_dsp/x86/blend_sse4.h @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_X86_BLEND_SSE4_H_ +#define AOM_DSP_X86_BLEND_SSE4_H_ + +#include "aom_dsp/blend.h" +#include "aom_dsp/x86/synonyms.h" + +////////////////////////////////////////////////////////////////////////////// +// Common kernels +////////////////////////////////////////////////////////////////////////////// + +static INLINE __m128i blend_4(const uint8_t *src0, const uint8_t *src1, + const __m128i v_m0_w, const __m128i v_m1_w) { + const __m128i v_s0_b = xx_loadl_32(src0); + const __m128i v_s1_b = xx_loadl_32(src1); + const __m128i v_s0_w = _mm_cvtepu8_epi16(v_s0_b); + const __m128i v_s1_w = _mm_cvtepu8_epi16(v_s1_b); + + const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, v_m0_w); + const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, v_m1_w); + + const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); + + const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); + + return v_res_w; +} + +static INLINE __m128i blend_8(const uint8_t *src0, const uint8_t *src1, + const __m128i v_m0_w, const __m128i v_m1_w) { + const __m128i v_s0_b = xx_loadl_64(src0); + const __m128i v_s1_b = xx_loadl_64(src1); + const __m128i v_s0_w = _mm_cvtepu8_epi16(v_s0_b); + const __m128i v_s1_w = _mm_cvtepu8_epi16(v_s1_b); + + const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, v_m0_w); + const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, v_m1_w); + + const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); + + const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); + + return v_res_w; +} + +#if CONFIG_HIGHBITDEPTH +typedef __m128i (*blend_unit_fn)(const uint16_t *src0, const uint16_t *src1, + const __m128i v_m0_w, const __m128i v_m1_w); + +static INLINE __m128i blend_4_b10(const uint16_t *src0, const uint16_t *src1, + const __m128i v_m0_w, const __m128i v_m1_w) { + const __m128i v_s0_w = xx_loadl_64(src0); + const __m128i v_s1_w = xx_loadl_64(src1); + + const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, v_m0_w); + const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, v_m1_w); + + const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); + + const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); + + return v_res_w; +} + +static INLINE __m128i blend_8_b10(const uint16_t *src0, const uint16_t *src1, + const __m128i v_m0_w, const __m128i v_m1_w) { + const __m128i v_s0_w = xx_loadu_128(src0); + const __m128i v_s1_w = xx_loadu_128(src1); + + const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, v_m0_w); + const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, v_m1_w); + + const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); + + const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); + + return v_res_w; +} + +static INLINE __m128i blend_4_b12(const uint16_t *src0, const uint16_t *src1, + const __m128i v_m0_w, const __m128i v_m1_w) { + const __m128i v_s0_w = xx_loadl_64(src0); + const __m128i v_s1_w = xx_loadl_64(src1); + + // Interleave + const __m128i v_m01_w = _mm_unpacklo_epi16(v_m0_w, v_m1_w); + const __m128i v_s01_w = _mm_unpacklo_epi16(v_s0_w, v_s1_w); + + // Multiply-Add + const __m128i v_sum_d = _mm_madd_epi16(v_s01_w, v_m01_w); + + // Scale + const __m128i v_ssum_d = + _mm_srli_epi32(v_sum_d, AOM_BLEND_A64_ROUND_BITS - 1); + + // Pack + const __m128i v_pssum_d = _mm_packs_epi32(v_ssum_d, v_ssum_d); + + // Round + const __m128i v_res_w = xx_round_epu16(v_pssum_d); + + return v_res_w; +} + +static INLINE __m128i blend_8_b12(const uint16_t *src0, const uint16_t *src1, + const __m128i v_m0_w, const __m128i v_m1_w) { + const __m128i v_s0_w = xx_loadu_128(src0); + const __m128i v_s1_w = xx_loadu_128(src1); + + // Interleave + const __m128i v_m01l_w = _mm_unpacklo_epi16(v_m0_w, v_m1_w); + const __m128i v_m01h_w = _mm_unpackhi_epi16(v_m0_w, v_m1_w); + const __m128i v_s01l_w = _mm_unpacklo_epi16(v_s0_w, v_s1_w); + const __m128i v_s01h_w = _mm_unpackhi_epi16(v_s0_w, v_s1_w); + + // Multiply-Add + const __m128i v_suml_d = _mm_madd_epi16(v_s01l_w, v_m01l_w); + const __m128i v_sumh_d = _mm_madd_epi16(v_s01h_w, v_m01h_w); + + // Scale + const __m128i v_ssuml_d = + _mm_srli_epi32(v_suml_d, AOM_BLEND_A64_ROUND_BITS - 1); + const __m128i v_ssumh_d = + _mm_srli_epi32(v_sumh_d, AOM_BLEND_A64_ROUND_BITS - 1); + + // Pack + const __m128i v_pssum_d = _mm_packs_epi32(v_ssuml_d, v_ssumh_d); + + // Round + const __m128i v_res_w = xx_round_epu16(v_pssum_d); + + return v_res_w; +} +#endif // CONFIG_HIGHBITDEPTH + +#endif // AOM_DSP_X86_BLEND_SSE4_H_ diff --git a/third_party/aom/aom_dsp/x86/convolve.h b/third_party/aom/aom_dsp/x86/convolve.h new file mode 100644 index 0000000000..8641164db9 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/convolve.h @@ -0,0 +1,288 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_DSP_X86_CONVOLVE_H_ +#define AOM_DSP_X86_CONVOLVE_H_ + +#include + +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" +#include "aom_dsp/aom_convolve.h" + +typedef void filter8_1dfunction(const uint8_t *src_ptr, ptrdiff_t src_pitch, + uint8_t *output_ptr, ptrdiff_t out_pitch, + uint32_t output_height, const int16_t *filter); + +#define FUN_CONV_1D(name, step_q4, filter, dir, src_start, avg, opt) \ + void aom_convolve8_##name##_##opt( \ + const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \ + ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, \ + const int16_t *filter_y, int y_step_q4, int w, int h) { \ + (void)filter_x; \ + (void)x_step_q4; \ + (void)filter_y; \ + (void)y_step_q4; \ + assert((-128 <= filter[3]) && (filter[3] <= 127)); \ + assert(step_q4 == 16); \ + if (filter[0] | filter[1] | filter[2]) { \ + while (w >= 16) { \ + aom_filter_block1d16_##dir##8_##avg##opt(src_start, src_stride, dst, \ + dst_stride, h, filter); \ + src += 16; \ + dst += 16; \ + w -= 16; \ + } \ + while (w >= 8) { \ + aom_filter_block1d8_##dir##8_##avg##opt(src_start, src_stride, dst, \ + dst_stride, h, filter); \ + src += 8; \ + dst += 8; \ + w -= 8; \ + } \ + while (w >= 4) { \ + aom_filter_block1d4_##dir##8_##avg##opt(src_start, src_stride, dst, \ + dst_stride, h, filter); \ + src += 4; \ + dst += 4; \ + w -= 4; \ + } \ + } else { \ + while (w >= 16) { \ + aom_filter_block1d16_##dir##2_##avg##opt(src, src_stride, dst, \ + dst_stride, h, filter); \ + src += 16; \ + dst += 16; \ + w -= 16; \ + } \ + while (w >= 8) { \ + aom_filter_block1d8_##dir##2_##avg##opt(src, src_stride, dst, \ + dst_stride, h, filter); \ + src += 8; \ + dst += 8; \ + w -= 8; \ + } \ + while (w >= 4) { \ + aom_filter_block1d4_##dir##2_##avg##opt(src, src_stride, dst, \ + dst_stride, h, filter); \ + src += 4; \ + dst += 4; \ + w -= 4; \ + } \ + } \ + if (w) { \ + aom_convolve8_##name##_c(src, src_stride, dst, dst_stride, filter_x, \ + x_step_q4, filter_y, y_step_q4, w, h); \ + } \ + } + +#define FUN_CONV_2D(avg, opt) \ + void aom_convolve8_##avg##opt( \ + const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \ + ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, \ + const int16_t *filter_y, int y_step_q4, int w, int h) { \ + assert((-128 <= filter_x[3]) && (filter_x[3] <= 127)); \ + assert((-128 <= filter_y[3]) && (filter_y[3] <= 127)); \ + assert(w <= MAX_SB_SIZE); \ + assert(h <= MAX_SB_SIZE); \ + assert(x_step_q4 == 16); \ + assert(y_step_q4 == 16); \ + if (filter_x[0] || filter_x[1] || filter_x[2] || filter_y[0] || \ + filter_y[1] || filter_y[2]) { \ + DECLARE_ALIGNED(16, uint8_t, fdata2[MAX_SB_SIZE * (MAX_SB_SIZE + 7)]); \ + aom_convolve8_horiz_##opt(src - 3 * src_stride, src_stride, fdata2, \ + MAX_SB_SIZE, filter_x, x_step_q4, filter_y, \ + y_step_q4, w, h + 7); \ + aom_convolve8_##avg##vert_##opt(fdata2 + 3 * MAX_SB_SIZE, MAX_SB_SIZE, \ + dst, dst_stride, filter_x, x_step_q4, \ + filter_y, y_step_q4, w, h); \ + } else { \ + DECLARE_ALIGNED(16, uint8_t, fdata2[MAX_SB_SIZE * (MAX_SB_SIZE + 1)]); \ + aom_convolve8_horiz_##opt(src, src_stride, fdata2, MAX_SB_SIZE, \ + filter_x, x_step_q4, filter_y, y_step_q4, w, \ + h + 1); \ + aom_convolve8_##avg##vert_##opt(fdata2, MAX_SB_SIZE, dst, dst_stride, \ + filter_x, x_step_q4, filter_y, \ + y_step_q4, w, h); \ + } \ + } + +#if CONFIG_LOOP_RESTORATION +// convolve_add_src is only used by the Wiener filter, which will never +// end up calling the bilinear functions (it uses a symmetric filter, so +// the possible numbers of taps are 1,3,5,7) +#define FUN_CONV_1D_NO_BILINEAR(name, step_q4, filter, dir, src_start, avg, \ + opt) \ + void aom_convolve8_##name##_##opt( \ + const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \ + ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, \ + const int16_t *filter_y, int y_step_q4, int w, int h) { \ + (void)filter_x; \ + (void)x_step_q4; \ + (void)filter_y; \ + (void)y_step_q4; \ + assert((-128 <= filter[3]) && (filter[3] <= 127)); \ + assert(step_q4 == 16); \ + while (w >= 16) { \ + aom_filter_block1d16_##dir##8_##avg##opt(src_start, src_stride, dst, \ + dst_stride, h, filter); \ + src += 16; \ + dst += 16; \ + w -= 16; \ + } \ + while (w >= 8) { \ + aom_filter_block1d8_##dir##8_##avg##opt(src_start, src_stride, dst, \ + dst_stride, h, filter); \ + src += 8; \ + dst += 8; \ + w -= 8; \ + } \ + while (w >= 4) { \ + aom_filter_block1d4_##dir##8_##avg##opt(src_start, src_stride, dst, \ + dst_stride, h, filter); \ + src += 4; \ + dst += 4; \ + w -= 4; \ + } \ + if (w) { \ + aom_convolve8_##name##_c(src, src_stride, dst, dst_stride, filter_x, \ + x_step_q4, filter_y, y_step_q4, w, h); \ + } \ + } + +#define FUN_CONV_2D_NO_BILINEAR(type, htype, opt) \ + void aom_convolve8_##type##opt( \ + const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \ + ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, \ + const int16_t *filter_y, int y_step_q4, int w, int h) { \ + DECLARE_ALIGNED(16, uint8_t, fdata2[MAX_SB_SIZE * (MAX_SB_SIZE + 7)]); \ + assert((-128 <= filter_x[3]) && (filter_x[3] <= 127)); \ + assert((-128 <= filter_y[3]) && (filter_y[3] <= 127)); \ + assert(w <= MAX_SB_SIZE); \ + assert(h <= MAX_SB_SIZE); \ + assert(x_step_q4 == 16); \ + assert(y_step_q4 == 16); \ + aom_convolve8_##htype##horiz_##opt( \ + src - 3 * src_stride, src_stride, fdata2, MAX_SB_SIZE, filter_x, \ + x_step_q4, filter_y, y_step_q4, w, h + 7); \ + aom_convolve8_##type##vert_##opt(fdata2 + 3 * MAX_SB_SIZE, MAX_SB_SIZE, \ + dst, dst_stride, filter_x, x_step_q4, \ + filter_y, y_step_q4, w, h); \ + } +#endif + +#if CONFIG_HIGHBITDEPTH +typedef void highbd_filter8_1dfunction(const uint16_t *src_ptr, + const ptrdiff_t src_pitch, + uint16_t *output_ptr, + ptrdiff_t out_pitch, + unsigned int output_height, + const int16_t *filter, int bd); + +#define HIGH_FUN_CONV_1D(name, step_q4, filter, dir, src_start, avg, opt) \ + void aom_highbd_convolve8_##name##_##opt( \ + const uint8_t *src8, ptrdiff_t src_stride, uint8_t *dst8, \ + ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, \ + const int16_t *filter_y, int y_step_q4, int w, int h, int bd) { \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ + if (step_q4 == 16 && filter[3] != 128) { \ + if (filter[0] | filter[1] | filter[2]) { \ + while (w >= 16) { \ + aom_highbd_filter_block1d16_##dir##8_##avg##opt( \ + src_start, src_stride, dst, dst_stride, h, filter, bd); \ + src += 16; \ + dst += 16; \ + w -= 16; \ + } \ + while (w >= 8) { \ + aom_highbd_filter_block1d8_##dir##8_##avg##opt( \ + src_start, src_stride, dst, dst_stride, h, filter, bd); \ + src += 8; \ + dst += 8; \ + w -= 8; \ + } \ + while (w >= 4) { \ + aom_highbd_filter_block1d4_##dir##8_##avg##opt( \ + src_start, src_stride, dst, dst_stride, h, filter, bd); \ + src += 4; \ + dst += 4; \ + w -= 4; \ + } \ + } else { \ + while (w >= 16) { \ + aom_highbd_filter_block1d16_##dir##2_##avg##opt( \ + src, src_stride, dst, dst_stride, h, filter, bd); \ + src += 16; \ + dst += 16; \ + w -= 16; \ + } \ + while (w >= 8) { \ + aom_highbd_filter_block1d8_##dir##2_##avg##opt( \ + src, src_stride, dst, dst_stride, h, filter, bd); \ + src += 8; \ + dst += 8; \ + w -= 8; \ + } \ + while (w >= 4) { \ + aom_highbd_filter_block1d4_##dir##2_##avg##opt( \ + src, src_stride, dst, dst_stride, h, filter, bd); \ + src += 4; \ + dst += 4; \ + w -= 4; \ + } \ + } \ + } \ + if (w) { \ + aom_highbd_convolve8_##name##_c( \ + CONVERT_TO_BYTEPTR(src), src_stride, CONVERT_TO_BYTEPTR(dst), \ + dst_stride, filter_x, x_step_q4, filter_y, y_step_q4, w, h, bd); \ + } \ + } + +#define HIGH_FUN_CONV_2D(avg, opt) \ + void aom_highbd_convolve8_##avg##opt( \ + const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \ + ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, \ + const int16_t *filter_y, int y_step_q4, int w, int h, int bd) { \ + assert(w <= MAX_SB_SIZE); \ + assert(h <= MAX_SB_SIZE); \ + if (x_step_q4 == 16 && y_step_q4 == 16) { \ + if (filter_x[0] || filter_x[1] || filter_x[2] || filter_x[3] == 128 || \ + filter_y[0] || filter_y[1] || filter_y[2] || filter_y[3] == 128) { \ + DECLARE_ALIGNED(16, uint16_t, \ + fdata2[MAX_SB_SIZE * (MAX_SB_SIZE + 7)]); \ + aom_highbd_convolve8_horiz_##opt(src - 3 * src_stride, src_stride, \ + CONVERT_TO_BYTEPTR(fdata2), \ + MAX_SB_SIZE, filter_x, x_step_q4, \ + filter_y, y_step_q4, w, h + 7, bd); \ + aom_highbd_convolve8_##avg##vert_##opt( \ + CONVERT_TO_BYTEPTR(fdata2) + 3 * MAX_SB_SIZE, MAX_SB_SIZE, dst, \ + dst_stride, filter_x, x_step_q4, filter_y, y_step_q4, w, h, bd); \ + } else { \ + DECLARE_ALIGNED(16, uint16_t, \ + fdata2[MAX_SB_SIZE * (MAX_SB_SIZE + 1)]); \ + aom_highbd_convolve8_horiz_##opt( \ + src, src_stride, CONVERT_TO_BYTEPTR(fdata2), MAX_SB_SIZE, \ + filter_x, x_step_q4, filter_y, y_step_q4, w, h + 1, bd); \ + aom_highbd_convolve8_##avg##vert_##opt( \ + CONVERT_TO_BYTEPTR(fdata2), MAX_SB_SIZE, dst, dst_stride, \ + filter_x, x_step_q4, filter_y, y_step_q4, w, h, bd); \ + } \ + } else { \ + aom_highbd_convolve8_##avg##c(src, src_stride, dst, dst_stride, \ + filter_x, x_step_q4, filter_y, y_step_q4, \ + w, h, bd); \ + } \ + } +#endif // CONFIG_HIGHBITDEPTH + +#endif // AOM_DSP_X86_CONVOLVE_H_ diff --git a/third_party/aom/aom_dsp/x86/fwd_dct32_8cols_sse2.c b/third_party/aom/aom_dsp/x86/fwd_dct32_8cols_sse2.c new file mode 100644 index 0000000000..b8ec08de79 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/fwd_dct32_8cols_sse2.c @@ -0,0 +1,862 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // SSE2 + +#include "aom_dsp/fwd_txfm.h" +#include "aom_dsp/txfm_common.h" +#include "aom_dsp/x86/txfm_common_sse2.h" + +// Apply a 32-element IDCT to 8 columns. This does not do any transposition +// of its output - the caller is expected to do that. +// The input buffers are the top and bottom halves of an 8x32 block. +void fdct32_8col(__m128i *in0, __m128i *in1) { + // Constants + // When we use them, in one case, they are all the same. In all others + // it's a pair of them that we need to repeat four times. This is done + // by constructing the 32 bit constant corresponding to that pair. + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(+cospi_16_64, -cospi_16_64); + const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + const __m128i k__cospi_p24_p08 = pair_set_epi16(+cospi_24_64, cospi_8_64); + const __m128i k__cospi_p12_p20 = pair_set_epi16(+cospi_12_64, cospi_20_64); + const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i k__cospi_p28_p04 = pair_set_epi16(+cospi_28_64, cospi_4_64); + const __m128i k__cospi_m28_m04 = pair_set_epi16(-cospi_28_64, -cospi_4_64); + const __m128i k__cospi_m12_m20 = pair_set_epi16(-cospi_12_64, -cospi_20_64); + const __m128i k__cospi_p30_p02 = pair_set_epi16(+cospi_30_64, cospi_2_64); + const __m128i k__cospi_p14_p18 = pair_set_epi16(+cospi_14_64, cospi_18_64); + const __m128i k__cospi_p22_p10 = pair_set_epi16(+cospi_22_64, cospi_10_64); + const __m128i k__cospi_p06_p26 = pair_set_epi16(+cospi_6_64, cospi_26_64); + const __m128i k__cospi_m26_p06 = pair_set_epi16(-cospi_26_64, cospi_6_64); + const __m128i k__cospi_m10_p22 = pair_set_epi16(-cospi_10_64, cospi_22_64); + const __m128i k__cospi_m18_p14 = pair_set_epi16(-cospi_18_64, cospi_14_64); + const __m128i k__cospi_m02_p30 = pair_set_epi16(-cospi_2_64, cospi_30_64); + const __m128i k__cospi_p31_p01 = pair_set_epi16(+cospi_31_64, cospi_1_64); + const __m128i k__cospi_p15_p17 = pair_set_epi16(+cospi_15_64, cospi_17_64); + const __m128i k__cospi_p23_p09 = pair_set_epi16(+cospi_23_64, cospi_9_64); + const __m128i k__cospi_p07_p25 = pair_set_epi16(+cospi_7_64, cospi_25_64); + const __m128i k__cospi_m25_p07 = pair_set_epi16(-cospi_25_64, cospi_7_64); + const __m128i k__cospi_m09_p23 = pair_set_epi16(-cospi_9_64, cospi_23_64); + const __m128i k__cospi_m17_p15 = pair_set_epi16(-cospi_17_64, cospi_15_64); + const __m128i k__cospi_m01_p31 = pair_set_epi16(-cospi_1_64, cospi_31_64); + const __m128i k__cospi_p27_p05 = pair_set_epi16(+cospi_27_64, cospi_5_64); + const __m128i k__cospi_p11_p21 = pair_set_epi16(+cospi_11_64, cospi_21_64); + const __m128i k__cospi_p19_p13 = pair_set_epi16(+cospi_19_64, cospi_13_64); + const __m128i k__cospi_p03_p29 = pair_set_epi16(+cospi_3_64, cospi_29_64); + const __m128i k__cospi_m29_p03 = pair_set_epi16(-cospi_29_64, cospi_3_64); + const __m128i k__cospi_m13_p19 = pair_set_epi16(-cospi_13_64, cospi_19_64); + const __m128i k__cospi_m21_p11 = pair_set_epi16(-cospi_21_64, cospi_11_64); + const __m128i k__cospi_m05_p27 = pair_set_epi16(-cospi_5_64, cospi_27_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + + __m128i step1[32]; + __m128i step2[32]; + __m128i step3[32]; + __m128i out[32]; + // Stage 1 + { + const __m128i *ina = in0; + const __m128i *inb = in1 + 15; + __m128i *step1a = &step1[0]; + __m128i *step1b = &step1[31]; + const __m128i ina0 = _mm_loadu_si128((const __m128i *)(ina)); + const __m128i ina1 = _mm_loadu_si128((const __m128i *)(ina + 1)); + const __m128i ina2 = _mm_loadu_si128((const __m128i *)(ina + 2)); + const __m128i ina3 = _mm_loadu_si128((const __m128i *)(ina + 3)); + const __m128i inb3 = _mm_loadu_si128((const __m128i *)(inb - 3)); + const __m128i inb2 = _mm_loadu_si128((const __m128i *)(inb - 2)); + const __m128i inb1 = _mm_loadu_si128((const __m128i *)(inb - 1)); + const __m128i inb0 = _mm_loadu_si128((const __m128i *)(inb)); + step1a[0] = _mm_add_epi16(ina0, inb0); + step1a[1] = _mm_add_epi16(ina1, inb1); + step1a[2] = _mm_add_epi16(ina2, inb2); + step1a[3] = _mm_add_epi16(ina3, inb3); + step1b[-3] = _mm_sub_epi16(ina3, inb3); + step1b[-2] = _mm_sub_epi16(ina2, inb2); + step1b[-1] = _mm_sub_epi16(ina1, inb1); + step1b[-0] = _mm_sub_epi16(ina0, inb0); + } + { + const __m128i *ina = in0 + 4; + const __m128i *inb = in1 + 11; + __m128i *step1a = &step1[4]; + __m128i *step1b = &step1[27]; + const __m128i ina0 = _mm_loadu_si128((const __m128i *)(ina)); + const __m128i ina1 = _mm_loadu_si128((const __m128i *)(ina + 1)); + const __m128i ina2 = _mm_loadu_si128((const __m128i *)(ina + 2)); + const __m128i ina3 = _mm_loadu_si128((const __m128i *)(ina + 3)); + const __m128i inb3 = _mm_loadu_si128((const __m128i *)(inb - 3)); + const __m128i inb2 = _mm_loadu_si128((const __m128i *)(inb - 2)); + const __m128i inb1 = _mm_loadu_si128((const __m128i *)(inb - 1)); + const __m128i inb0 = _mm_loadu_si128((const __m128i *)(inb)); + step1a[0] = _mm_add_epi16(ina0, inb0); + step1a[1] = _mm_add_epi16(ina1, inb1); + step1a[2] = _mm_add_epi16(ina2, inb2); + step1a[3] = _mm_add_epi16(ina3, inb3); + step1b[-3] = _mm_sub_epi16(ina3, inb3); + step1b[-2] = _mm_sub_epi16(ina2, inb2); + step1b[-1] = _mm_sub_epi16(ina1, inb1); + step1b[-0] = _mm_sub_epi16(ina0, inb0); + } + { + const __m128i *ina = in0 + 8; + const __m128i *inb = in1 + 7; + __m128i *step1a = &step1[8]; + __m128i *step1b = &step1[23]; + const __m128i ina0 = _mm_loadu_si128((const __m128i *)(ina)); + const __m128i ina1 = _mm_loadu_si128((const __m128i *)(ina + 1)); + const __m128i ina2 = _mm_loadu_si128((const __m128i *)(ina + 2)); + const __m128i ina3 = _mm_loadu_si128((const __m128i *)(ina + 3)); + const __m128i inb3 = _mm_loadu_si128((const __m128i *)(inb - 3)); + const __m128i inb2 = _mm_loadu_si128((const __m128i *)(inb - 2)); + const __m128i inb1 = _mm_loadu_si128((const __m128i *)(inb - 1)); + const __m128i inb0 = _mm_loadu_si128((const __m128i *)(inb)); + step1a[0] = _mm_add_epi16(ina0, inb0); + step1a[1] = _mm_add_epi16(ina1, inb1); + step1a[2] = _mm_add_epi16(ina2, inb2); + step1a[3] = _mm_add_epi16(ina3, inb3); + step1b[-3] = _mm_sub_epi16(ina3, inb3); + step1b[-2] = _mm_sub_epi16(ina2, inb2); + step1b[-1] = _mm_sub_epi16(ina1, inb1); + step1b[-0] = _mm_sub_epi16(ina0, inb0); + } + { + const __m128i *ina = in0 + 12; + const __m128i *inb = in1 + 3; + __m128i *step1a = &step1[12]; + __m128i *step1b = &step1[19]; + const __m128i ina0 = _mm_loadu_si128((const __m128i *)(ina)); + const __m128i ina1 = _mm_loadu_si128((const __m128i *)(ina + 1)); + const __m128i ina2 = _mm_loadu_si128((const __m128i *)(ina + 2)); + const __m128i ina3 = _mm_loadu_si128((const __m128i *)(ina + 3)); + const __m128i inb3 = _mm_loadu_si128((const __m128i *)(inb - 3)); + const __m128i inb2 = _mm_loadu_si128((const __m128i *)(inb - 2)); + const __m128i inb1 = _mm_loadu_si128((const __m128i *)(inb - 1)); + const __m128i inb0 = _mm_loadu_si128((const __m128i *)(inb)); + step1a[0] = _mm_add_epi16(ina0, inb0); + step1a[1] = _mm_add_epi16(ina1, inb1); + step1a[2] = _mm_add_epi16(ina2, inb2); + step1a[3] = _mm_add_epi16(ina3, inb3); + step1b[-3] = _mm_sub_epi16(ina3, inb3); + step1b[-2] = _mm_sub_epi16(ina2, inb2); + step1b[-1] = _mm_sub_epi16(ina1, inb1); + step1b[-0] = _mm_sub_epi16(ina0, inb0); + } + // Stage 2 + { + step2[0] = _mm_add_epi16(step1[0], step1[15]); + step2[1] = _mm_add_epi16(step1[1], step1[14]); + step2[2] = _mm_add_epi16(step1[2], step1[13]); + step2[3] = _mm_add_epi16(step1[3], step1[12]); + step2[4] = _mm_add_epi16(step1[4], step1[11]); + step2[5] = _mm_add_epi16(step1[5], step1[10]); + step2[6] = _mm_add_epi16(step1[6], step1[9]); + step2[7] = _mm_add_epi16(step1[7], step1[8]); + step2[8] = _mm_sub_epi16(step1[7], step1[8]); + step2[9] = _mm_sub_epi16(step1[6], step1[9]); + step2[10] = _mm_sub_epi16(step1[5], step1[10]); + step2[11] = _mm_sub_epi16(step1[4], step1[11]); + step2[12] = _mm_sub_epi16(step1[3], step1[12]); + step2[13] = _mm_sub_epi16(step1[2], step1[13]); + step2[14] = _mm_sub_epi16(step1[1], step1[14]); + step2[15] = _mm_sub_epi16(step1[0], step1[15]); + } + { + const __m128i s2_20_0 = _mm_unpacklo_epi16(step1[27], step1[20]); + const __m128i s2_20_1 = _mm_unpackhi_epi16(step1[27], step1[20]); + const __m128i s2_21_0 = _mm_unpacklo_epi16(step1[26], step1[21]); + const __m128i s2_21_1 = _mm_unpackhi_epi16(step1[26], step1[21]); + const __m128i s2_22_0 = _mm_unpacklo_epi16(step1[25], step1[22]); + const __m128i s2_22_1 = _mm_unpackhi_epi16(step1[25], step1[22]); + const __m128i s2_23_0 = _mm_unpacklo_epi16(step1[24], step1[23]); + const __m128i s2_23_1 = _mm_unpackhi_epi16(step1[24], step1[23]); + const __m128i s2_20_2 = _mm_madd_epi16(s2_20_0, k__cospi_p16_m16); + const __m128i s2_20_3 = _mm_madd_epi16(s2_20_1, k__cospi_p16_m16); + const __m128i s2_21_2 = _mm_madd_epi16(s2_21_0, k__cospi_p16_m16); + const __m128i s2_21_3 = _mm_madd_epi16(s2_21_1, k__cospi_p16_m16); + const __m128i s2_22_2 = _mm_madd_epi16(s2_22_0, k__cospi_p16_m16); + const __m128i s2_22_3 = _mm_madd_epi16(s2_22_1, k__cospi_p16_m16); + const __m128i s2_23_2 = _mm_madd_epi16(s2_23_0, k__cospi_p16_m16); + const __m128i s2_23_3 = _mm_madd_epi16(s2_23_1, k__cospi_p16_m16); + const __m128i s2_24_2 = _mm_madd_epi16(s2_23_0, k__cospi_p16_p16); + const __m128i s2_24_3 = _mm_madd_epi16(s2_23_1, k__cospi_p16_p16); + const __m128i s2_25_2 = _mm_madd_epi16(s2_22_0, k__cospi_p16_p16); + const __m128i s2_25_3 = _mm_madd_epi16(s2_22_1, k__cospi_p16_p16); + const __m128i s2_26_2 = _mm_madd_epi16(s2_21_0, k__cospi_p16_p16); + const __m128i s2_26_3 = _mm_madd_epi16(s2_21_1, k__cospi_p16_p16); + const __m128i s2_27_2 = _mm_madd_epi16(s2_20_0, k__cospi_p16_p16); + const __m128i s2_27_3 = _mm_madd_epi16(s2_20_1, k__cospi_p16_p16); + // dct_const_round_shift + const __m128i s2_20_4 = _mm_add_epi32(s2_20_2, k__DCT_CONST_ROUNDING); + const __m128i s2_20_5 = _mm_add_epi32(s2_20_3, k__DCT_CONST_ROUNDING); + const __m128i s2_21_4 = _mm_add_epi32(s2_21_2, k__DCT_CONST_ROUNDING); + const __m128i s2_21_5 = _mm_add_epi32(s2_21_3, k__DCT_CONST_ROUNDING); + const __m128i s2_22_4 = _mm_add_epi32(s2_22_2, k__DCT_CONST_ROUNDING); + const __m128i s2_22_5 = _mm_add_epi32(s2_22_3, k__DCT_CONST_ROUNDING); + const __m128i s2_23_4 = _mm_add_epi32(s2_23_2, k__DCT_CONST_ROUNDING); + const __m128i s2_23_5 = _mm_add_epi32(s2_23_3, k__DCT_CONST_ROUNDING); + const __m128i s2_24_4 = _mm_add_epi32(s2_24_2, k__DCT_CONST_ROUNDING); + const __m128i s2_24_5 = _mm_add_epi32(s2_24_3, k__DCT_CONST_ROUNDING); + const __m128i s2_25_4 = _mm_add_epi32(s2_25_2, k__DCT_CONST_ROUNDING); + const __m128i s2_25_5 = _mm_add_epi32(s2_25_3, k__DCT_CONST_ROUNDING); + const __m128i s2_26_4 = _mm_add_epi32(s2_26_2, k__DCT_CONST_ROUNDING); + const __m128i s2_26_5 = _mm_add_epi32(s2_26_3, k__DCT_CONST_ROUNDING); + const __m128i s2_27_4 = _mm_add_epi32(s2_27_2, k__DCT_CONST_ROUNDING); + const __m128i s2_27_5 = _mm_add_epi32(s2_27_3, k__DCT_CONST_ROUNDING); + const __m128i s2_20_6 = _mm_srai_epi32(s2_20_4, DCT_CONST_BITS); + const __m128i s2_20_7 = _mm_srai_epi32(s2_20_5, DCT_CONST_BITS); + const __m128i s2_21_6 = _mm_srai_epi32(s2_21_4, DCT_CONST_BITS); + const __m128i s2_21_7 = _mm_srai_epi32(s2_21_5, DCT_CONST_BITS); + const __m128i s2_22_6 = _mm_srai_epi32(s2_22_4, DCT_CONST_BITS); + const __m128i s2_22_7 = _mm_srai_epi32(s2_22_5, DCT_CONST_BITS); + const __m128i s2_23_6 = _mm_srai_epi32(s2_23_4, DCT_CONST_BITS); + const __m128i s2_23_7 = _mm_srai_epi32(s2_23_5, DCT_CONST_BITS); + const __m128i s2_24_6 = _mm_srai_epi32(s2_24_4, DCT_CONST_BITS); + const __m128i s2_24_7 = _mm_srai_epi32(s2_24_5, DCT_CONST_BITS); + const __m128i s2_25_6 = _mm_srai_epi32(s2_25_4, DCT_CONST_BITS); + const __m128i s2_25_7 = _mm_srai_epi32(s2_25_5, DCT_CONST_BITS); + const __m128i s2_26_6 = _mm_srai_epi32(s2_26_4, DCT_CONST_BITS); + const __m128i s2_26_7 = _mm_srai_epi32(s2_26_5, DCT_CONST_BITS); + const __m128i s2_27_6 = _mm_srai_epi32(s2_27_4, DCT_CONST_BITS); + const __m128i s2_27_7 = _mm_srai_epi32(s2_27_5, DCT_CONST_BITS); + // Combine + step2[20] = _mm_packs_epi32(s2_20_6, s2_20_7); + step2[21] = _mm_packs_epi32(s2_21_6, s2_21_7); + step2[22] = _mm_packs_epi32(s2_22_6, s2_22_7); + step2[23] = _mm_packs_epi32(s2_23_6, s2_23_7); + step2[24] = _mm_packs_epi32(s2_24_6, s2_24_7); + step2[25] = _mm_packs_epi32(s2_25_6, s2_25_7); + step2[26] = _mm_packs_epi32(s2_26_6, s2_26_7); + step2[27] = _mm_packs_epi32(s2_27_6, s2_27_7); + } + // Stage 3 + { + step3[0] = _mm_add_epi16(step2[(8 - 1)], step2[0]); + step3[1] = _mm_add_epi16(step2[(8 - 2)], step2[1]); + step3[2] = _mm_add_epi16(step2[(8 - 3)], step2[2]); + step3[3] = _mm_add_epi16(step2[(8 - 4)], step2[3]); + step3[4] = _mm_sub_epi16(step2[(8 - 5)], step2[4]); + step3[5] = _mm_sub_epi16(step2[(8 - 6)], step2[5]); + step3[6] = _mm_sub_epi16(step2[(8 - 7)], step2[6]); + step3[7] = _mm_sub_epi16(step2[(8 - 8)], step2[7]); + } + { + const __m128i s3_10_0 = _mm_unpacklo_epi16(step2[13], step2[10]); + const __m128i s3_10_1 = _mm_unpackhi_epi16(step2[13], step2[10]); + const __m128i s3_11_0 = _mm_unpacklo_epi16(step2[12], step2[11]); + const __m128i s3_11_1 = _mm_unpackhi_epi16(step2[12], step2[11]); + const __m128i s3_10_2 = _mm_madd_epi16(s3_10_0, k__cospi_p16_m16); + const __m128i s3_10_3 = _mm_madd_epi16(s3_10_1, k__cospi_p16_m16); + const __m128i s3_11_2 = _mm_madd_epi16(s3_11_0, k__cospi_p16_m16); + const __m128i s3_11_3 = _mm_madd_epi16(s3_11_1, k__cospi_p16_m16); + const __m128i s3_12_2 = _mm_madd_epi16(s3_11_0, k__cospi_p16_p16); + const __m128i s3_12_3 = _mm_madd_epi16(s3_11_1, k__cospi_p16_p16); + const __m128i s3_13_2 = _mm_madd_epi16(s3_10_0, k__cospi_p16_p16); + const __m128i s3_13_3 = _mm_madd_epi16(s3_10_1, k__cospi_p16_p16); + // dct_const_round_shift + const __m128i s3_10_4 = _mm_add_epi32(s3_10_2, k__DCT_CONST_ROUNDING); + const __m128i s3_10_5 = _mm_add_epi32(s3_10_3, k__DCT_CONST_ROUNDING); + const __m128i s3_11_4 = _mm_add_epi32(s3_11_2, k__DCT_CONST_ROUNDING); + const __m128i s3_11_5 = _mm_add_epi32(s3_11_3, k__DCT_CONST_ROUNDING); + const __m128i s3_12_4 = _mm_add_epi32(s3_12_2, k__DCT_CONST_ROUNDING); + const __m128i s3_12_5 = _mm_add_epi32(s3_12_3, k__DCT_CONST_ROUNDING); + const __m128i s3_13_4 = _mm_add_epi32(s3_13_2, k__DCT_CONST_ROUNDING); + const __m128i s3_13_5 = _mm_add_epi32(s3_13_3, k__DCT_CONST_ROUNDING); + const __m128i s3_10_6 = _mm_srai_epi32(s3_10_4, DCT_CONST_BITS); + const __m128i s3_10_7 = _mm_srai_epi32(s3_10_5, DCT_CONST_BITS); + const __m128i s3_11_6 = _mm_srai_epi32(s3_11_4, DCT_CONST_BITS); + const __m128i s3_11_7 = _mm_srai_epi32(s3_11_5, DCT_CONST_BITS); + const __m128i s3_12_6 = _mm_srai_epi32(s3_12_4, DCT_CONST_BITS); + const __m128i s3_12_7 = _mm_srai_epi32(s3_12_5, DCT_CONST_BITS); + const __m128i s3_13_6 = _mm_srai_epi32(s3_13_4, DCT_CONST_BITS); + const __m128i s3_13_7 = _mm_srai_epi32(s3_13_5, DCT_CONST_BITS); + // Combine + step3[10] = _mm_packs_epi32(s3_10_6, s3_10_7); + step3[11] = _mm_packs_epi32(s3_11_6, s3_11_7); + step3[12] = _mm_packs_epi32(s3_12_6, s3_12_7); + step3[13] = _mm_packs_epi32(s3_13_6, s3_13_7); + } + { + step3[16] = _mm_add_epi16(step2[23], step1[16]); + step3[17] = _mm_add_epi16(step2[22], step1[17]); + step3[18] = _mm_add_epi16(step2[21], step1[18]); + step3[19] = _mm_add_epi16(step2[20], step1[19]); + step3[20] = _mm_sub_epi16(step1[19], step2[20]); + step3[21] = _mm_sub_epi16(step1[18], step2[21]); + step3[22] = _mm_sub_epi16(step1[17], step2[22]); + step3[23] = _mm_sub_epi16(step1[16], step2[23]); + step3[24] = _mm_sub_epi16(step1[31], step2[24]); + step3[25] = _mm_sub_epi16(step1[30], step2[25]); + step3[26] = _mm_sub_epi16(step1[29], step2[26]); + step3[27] = _mm_sub_epi16(step1[28], step2[27]); + step3[28] = _mm_add_epi16(step2[27], step1[28]); + step3[29] = _mm_add_epi16(step2[26], step1[29]); + step3[30] = _mm_add_epi16(step2[25], step1[30]); + step3[31] = _mm_add_epi16(step2[24], step1[31]); + } + + // Stage 4 + { + step1[0] = _mm_add_epi16(step3[3], step3[0]); + step1[1] = _mm_add_epi16(step3[2], step3[1]); + step1[2] = _mm_sub_epi16(step3[1], step3[2]); + step1[3] = _mm_sub_epi16(step3[0], step3[3]); + step1[8] = _mm_add_epi16(step3[11], step2[8]); + step1[9] = _mm_add_epi16(step3[10], step2[9]); + step1[10] = _mm_sub_epi16(step2[9], step3[10]); + step1[11] = _mm_sub_epi16(step2[8], step3[11]); + step1[12] = _mm_sub_epi16(step2[15], step3[12]); + step1[13] = _mm_sub_epi16(step2[14], step3[13]); + step1[14] = _mm_add_epi16(step3[13], step2[14]); + step1[15] = _mm_add_epi16(step3[12], step2[15]); + } + { + const __m128i s1_05_0 = _mm_unpacklo_epi16(step3[6], step3[5]); + const __m128i s1_05_1 = _mm_unpackhi_epi16(step3[6], step3[5]); + const __m128i s1_05_2 = _mm_madd_epi16(s1_05_0, k__cospi_p16_m16); + const __m128i s1_05_3 = _mm_madd_epi16(s1_05_1, k__cospi_p16_m16); + const __m128i s1_06_2 = _mm_madd_epi16(s1_05_0, k__cospi_p16_p16); + const __m128i s1_06_3 = _mm_madd_epi16(s1_05_1, k__cospi_p16_p16); + // dct_const_round_shift + const __m128i s1_05_4 = _mm_add_epi32(s1_05_2, k__DCT_CONST_ROUNDING); + const __m128i s1_05_5 = _mm_add_epi32(s1_05_3, k__DCT_CONST_ROUNDING); + const __m128i s1_06_4 = _mm_add_epi32(s1_06_2, k__DCT_CONST_ROUNDING); + const __m128i s1_06_5 = _mm_add_epi32(s1_06_3, k__DCT_CONST_ROUNDING); + const __m128i s1_05_6 = _mm_srai_epi32(s1_05_4, DCT_CONST_BITS); + const __m128i s1_05_7 = _mm_srai_epi32(s1_05_5, DCT_CONST_BITS); + const __m128i s1_06_6 = _mm_srai_epi32(s1_06_4, DCT_CONST_BITS); + const __m128i s1_06_7 = _mm_srai_epi32(s1_06_5, DCT_CONST_BITS); + // Combine + step1[5] = _mm_packs_epi32(s1_05_6, s1_05_7); + step1[6] = _mm_packs_epi32(s1_06_6, s1_06_7); + } + { + const __m128i s1_18_0 = _mm_unpacklo_epi16(step3[18], step3[29]); + const __m128i s1_18_1 = _mm_unpackhi_epi16(step3[18], step3[29]); + const __m128i s1_19_0 = _mm_unpacklo_epi16(step3[19], step3[28]); + const __m128i s1_19_1 = _mm_unpackhi_epi16(step3[19], step3[28]); + const __m128i s1_20_0 = _mm_unpacklo_epi16(step3[20], step3[27]); + const __m128i s1_20_1 = _mm_unpackhi_epi16(step3[20], step3[27]); + const __m128i s1_21_0 = _mm_unpacklo_epi16(step3[21], step3[26]); + const __m128i s1_21_1 = _mm_unpackhi_epi16(step3[21], step3[26]); + const __m128i s1_18_2 = _mm_madd_epi16(s1_18_0, k__cospi_m08_p24); + const __m128i s1_18_3 = _mm_madd_epi16(s1_18_1, k__cospi_m08_p24); + const __m128i s1_19_2 = _mm_madd_epi16(s1_19_0, k__cospi_m08_p24); + const __m128i s1_19_3 = _mm_madd_epi16(s1_19_1, k__cospi_m08_p24); + const __m128i s1_20_2 = _mm_madd_epi16(s1_20_0, k__cospi_m24_m08); + const __m128i s1_20_3 = _mm_madd_epi16(s1_20_1, k__cospi_m24_m08); + const __m128i s1_21_2 = _mm_madd_epi16(s1_21_0, k__cospi_m24_m08); + const __m128i s1_21_3 = _mm_madd_epi16(s1_21_1, k__cospi_m24_m08); + const __m128i s1_26_2 = _mm_madd_epi16(s1_21_0, k__cospi_m08_p24); + const __m128i s1_26_3 = _mm_madd_epi16(s1_21_1, k__cospi_m08_p24); + const __m128i s1_27_2 = _mm_madd_epi16(s1_20_0, k__cospi_m08_p24); + const __m128i s1_27_3 = _mm_madd_epi16(s1_20_1, k__cospi_m08_p24); + const __m128i s1_28_2 = _mm_madd_epi16(s1_19_0, k__cospi_p24_p08); + const __m128i s1_28_3 = _mm_madd_epi16(s1_19_1, k__cospi_p24_p08); + const __m128i s1_29_2 = _mm_madd_epi16(s1_18_0, k__cospi_p24_p08); + const __m128i s1_29_3 = _mm_madd_epi16(s1_18_1, k__cospi_p24_p08); + // dct_const_round_shift + const __m128i s1_18_4 = _mm_add_epi32(s1_18_2, k__DCT_CONST_ROUNDING); + const __m128i s1_18_5 = _mm_add_epi32(s1_18_3, k__DCT_CONST_ROUNDING); + const __m128i s1_19_4 = _mm_add_epi32(s1_19_2, k__DCT_CONST_ROUNDING); + const __m128i s1_19_5 = _mm_add_epi32(s1_19_3, k__DCT_CONST_ROUNDING); + const __m128i s1_20_4 = _mm_add_epi32(s1_20_2, k__DCT_CONST_ROUNDING); + const __m128i s1_20_5 = _mm_add_epi32(s1_20_3, k__DCT_CONST_ROUNDING); + const __m128i s1_21_4 = _mm_add_epi32(s1_21_2, k__DCT_CONST_ROUNDING); + const __m128i s1_21_5 = _mm_add_epi32(s1_21_3, k__DCT_CONST_ROUNDING); + const __m128i s1_26_4 = _mm_add_epi32(s1_26_2, k__DCT_CONST_ROUNDING); + const __m128i s1_26_5 = _mm_add_epi32(s1_26_3, k__DCT_CONST_ROUNDING); + const __m128i s1_27_4 = _mm_add_epi32(s1_27_2, k__DCT_CONST_ROUNDING); + const __m128i s1_27_5 = _mm_add_epi32(s1_27_3, k__DCT_CONST_ROUNDING); + const __m128i s1_28_4 = _mm_add_epi32(s1_28_2, k__DCT_CONST_ROUNDING); + const __m128i s1_28_5 = _mm_add_epi32(s1_28_3, k__DCT_CONST_ROUNDING); + const __m128i s1_29_4 = _mm_add_epi32(s1_29_2, k__DCT_CONST_ROUNDING); + const __m128i s1_29_5 = _mm_add_epi32(s1_29_3, k__DCT_CONST_ROUNDING); + const __m128i s1_18_6 = _mm_srai_epi32(s1_18_4, DCT_CONST_BITS); + const __m128i s1_18_7 = _mm_srai_epi32(s1_18_5, DCT_CONST_BITS); + const __m128i s1_19_6 = _mm_srai_epi32(s1_19_4, DCT_CONST_BITS); + const __m128i s1_19_7 = _mm_srai_epi32(s1_19_5, DCT_CONST_BITS); + const __m128i s1_20_6 = _mm_srai_epi32(s1_20_4, DCT_CONST_BITS); + const __m128i s1_20_7 = _mm_srai_epi32(s1_20_5, DCT_CONST_BITS); + const __m128i s1_21_6 = _mm_srai_epi32(s1_21_4, DCT_CONST_BITS); + const __m128i s1_21_7 = _mm_srai_epi32(s1_21_5, DCT_CONST_BITS); + const __m128i s1_26_6 = _mm_srai_epi32(s1_26_4, DCT_CONST_BITS); + const __m128i s1_26_7 = _mm_srai_epi32(s1_26_5, DCT_CONST_BITS); + const __m128i s1_27_6 = _mm_srai_epi32(s1_27_4, DCT_CONST_BITS); + const __m128i s1_27_7 = _mm_srai_epi32(s1_27_5, DCT_CONST_BITS); + const __m128i s1_28_6 = _mm_srai_epi32(s1_28_4, DCT_CONST_BITS); + const __m128i s1_28_7 = _mm_srai_epi32(s1_28_5, DCT_CONST_BITS); + const __m128i s1_29_6 = _mm_srai_epi32(s1_29_4, DCT_CONST_BITS); + const __m128i s1_29_7 = _mm_srai_epi32(s1_29_5, DCT_CONST_BITS); + // Combine + step1[18] = _mm_packs_epi32(s1_18_6, s1_18_7); + step1[19] = _mm_packs_epi32(s1_19_6, s1_19_7); + step1[20] = _mm_packs_epi32(s1_20_6, s1_20_7); + step1[21] = _mm_packs_epi32(s1_21_6, s1_21_7); + step1[26] = _mm_packs_epi32(s1_26_6, s1_26_7); + step1[27] = _mm_packs_epi32(s1_27_6, s1_27_7); + step1[28] = _mm_packs_epi32(s1_28_6, s1_28_7); + step1[29] = _mm_packs_epi32(s1_29_6, s1_29_7); + } + // Stage 5 + { + step2[4] = _mm_add_epi16(step1[5], step3[4]); + step2[5] = _mm_sub_epi16(step3[4], step1[5]); + step2[6] = _mm_sub_epi16(step3[7], step1[6]); + step2[7] = _mm_add_epi16(step1[6], step3[7]); + } + { + const __m128i out_00_0 = _mm_unpacklo_epi16(step1[0], step1[1]); + const __m128i out_00_1 = _mm_unpackhi_epi16(step1[0], step1[1]); + const __m128i out_08_0 = _mm_unpacklo_epi16(step1[2], step1[3]); + const __m128i out_08_1 = _mm_unpackhi_epi16(step1[2], step1[3]); + const __m128i out_00_2 = _mm_madd_epi16(out_00_0, k__cospi_p16_p16); + const __m128i out_00_3 = _mm_madd_epi16(out_00_1, k__cospi_p16_p16); + const __m128i out_16_2 = _mm_madd_epi16(out_00_0, k__cospi_p16_m16); + const __m128i out_16_3 = _mm_madd_epi16(out_00_1, k__cospi_p16_m16); + const __m128i out_08_2 = _mm_madd_epi16(out_08_0, k__cospi_p24_p08); + const __m128i out_08_3 = _mm_madd_epi16(out_08_1, k__cospi_p24_p08); + const __m128i out_24_2 = _mm_madd_epi16(out_08_0, k__cospi_m08_p24); + const __m128i out_24_3 = _mm_madd_epi16(out_08_1, k__cospi_m08_p24); + // dct_const_round_shift + const __m128i out_00_4 = _mm_add_epi32(out_00_2, k__DCT_CONST_ROUNDING); + const __m128i out_00_5 = _mm_add_epi32(out_00_3, k__DCT_CONST_ROUNDING); + const __m128i out_16_4 = _mm_add_epi32(out_16_2, k__DCT_CONST_ROUNDING); + const __m128i out_16_5 = _mm_add_epi32(out_16_3, k__DCT_CONST_ROUNDING); + const __m128i out_08_4 = _mm_add_epi32(out_08_2, k__DCT_CONST_ROUNDING); + const __m128i out_08_5 = _mm_add_epi32(out_08_3, k__DCT_CONST_ROUNDING); + const __m128i out_24_4 = _mm_add_epi32(out_24_2, k__DCT_CONST_ROUNDING); + const __m128i out_24_5 = _mm_add_epi32(out_24_3, k__DCT_CONST_ROUNDING); + const __m128i out_00_6 = _mm_srai_epi32(out_00_4, DCT_CONST_BITS); + const __m128i out_00_7 = _mm_srai_epi32(out_00_5, DCT_CONST_BITS); + const __m128i out_16_6 = _mm_srai_epi32(out_16_4, DCT_CONST_BITS); + const __m128i out_16_7 = _mm_srai_epi32(out_16_5, DCT_CONST_BITS); + const __m128i out_08_6 = _mm_srai_epi32(out_08_4, DCT_CONST_BITS); + const __m128i out_08_7 = _mm_srai_epi32(out_08_5, DCT_CONST_BITS); + const __m128i out_24_6 = _mm_srai_epi32(out_24_4, DCT_CONST_BITS); + const __m128i out_24_7 = _mm_srai_epi32(out_24_5, DCT_CONST_BITS); + // Combine + out[0] = _mm_packs_epi32(out_00_6, out_00_7); + out[16] = _mm_packs_epi32(out_16_6, out_16_7); + out[8] = _mm_packs_epi32(out_08_6, out_08_7); + out[24] = _mm_packs_epi32(out_24_6, out_24_7); + } + { + const __m128i s2_09_0 = _mm_unpacklo_epi16(step1[9], step1[14]); + const __m128i s2_09_1 = _mm_unpackhi_epi16(step1[9], step1[14]); + const __m128i s2_10_0 = _mm_unpacklo_epi16(step1[10], step1[13]); + const __m128i s2_10_1 = _mm_unpackhi_epi16(step1[10], step1[13]); + const __m128i s2_09_2 = _mm_madd_epi16(s2_09_0, k__cospi_m08_p24); + const __m128i s2_09_3 = _mm_madd_epi16(s2_09_1, k__cospi_m08_p24); + const __m128i s2_10_2 = _mm_madd_epi16(s2_10_0, k__cospi_m24_m08); + const __m128i s2_10_3 = _mm_madd_epi16(s2_10_1, k__cospi_m24_m08); + const __m128i s2_13_2 = _mm_madd_epi16(s2_10_0, k__cospi_m08_p24); + const __m128i s2_13_3 = _mm_madd_epi16(s2_10_1, k__cospi_m08_p24); + const __m128i s2_14_2 = _mm_madd_epi16(s2_09_0, k__cospi_p24_p08); + const __m128i s2_14_3 = _mm_madd_epi16(s2_09_1, k__cospi_p24_p08); + // dct_const_round_shift + const __m128i s2_09_4 = _mm_add_epi32(s2_09_2, k__DCT_CONST_ROUNDING); + const __m128i s2_09_5 = _mm_add_epi32(s2_09_3, k__DCT_CONST_ROUNDING); + const __m128i s2_10_4 = _mm_add_epi32(s2_10_2, k__DCT_CONST_ROUNDING); + const __m128i s2_10_5 = _mm_add_epi32(s2_10_3, k__DCT_CONST_ROUNDING); + const __m128i s2_13_4 = _mm_add_epi32(s2_13_2, k__DCT_CONST_ROUNDING); + const __m128i s2_13_5 = _mm_add_epi32(s2_13_3, k__DCT_CONST_ROUNDING); + const __m128i s2_14_4 = _mm_add_epi32(s2_14_2, k__DCT_CONST_ROUNDING); + const __m128i s2_14_5 = _mm_add_epi32(s2_14_3, k__DCT_CONST_ROUNDING); + const __m128i s2_09_6 = _mm_srai_epi32(s2_09_4, DCT_CONST_BITS); + const __m128i s2_09_7 = _mm_srai_epi32(s2_09_5, DCT_CONST_BITS); + const __m128i s2_10_6 = _mm_srai_epi32(s2_10_4, DCT_CONST_BITS); + const __m128i s2_10_7 = _mm_srai_epi32(s2_10_5, DCT_CONST_BITS); + const __m128i s2_13_6 = _mm_srai_epi32(s2_13_4, DCT_CONST_BITS); + const __m128i s2_13_7 = _mm_srai_epi32(s2_13_5, DCT_CONST_BITS); + const __m128i s2_14_6 = _mm_srai_epi32(s2_14_4, DCT_CONST_BITS); + const __m128i s2_14_7 = _mm_srai_epi32(s2_14_5, DCT_CONST_BITS); + // Combine + step2[9] = _mm_packs_epi32(s2_09_6, s2_09_7); + step2[10] = _mm_packs_epi32(s2_10_6, s2_10_7); + step2[13] = _mm_packs_epi32(s2_13_6, s2_13_7); + step2[14] = _mm_packs_epi32(s2_14_6, s2_14_7); + } + { + step2[16] = _mm_add_epi16(step1[19], step3[16]); + step2[17] = _mm_add_epi16(step1[18], step3[17]); + step2[18] = _mm_sub_epi16(step3[17], step1[18]); + step2[19] = _mm_sub_epi16(step3[16], step1[19]); + step2[20] = _mm_sub_epi16(step3[23], step1[20]); + step2[21] = _mm_sub_epi16(step3[22], step1[21]); + step2[22] = _mm_add_epi16(step1[21], step3[22]); + step2[23] = _mm_add_epi16(step1[20], step3[23]); + step2[24] = _mm_add_epi16(step1[27], step3[24]); + step2[25] = _mm_add_epi16(step1[26], step3[25]); + step2[26] = _mm_sub_epi16(step3[25], step1[26]); + step2[27] = _mm_sub_epi16(step3[24], step1[27]); + step2[28] = _mm_sub_epi16(step3[31], step1[28]); + step2[29] = _mm_sub_epi16(step3[30], step1[29]); + step2[30] = _mm_add_epi16(step1[29], step3[30]); + step2[31] = _mm_add_epi16(step1[28], step3[31]); + } + // Stage 6 + { + const __m128i out_04_0 = _mm_unpacklo_epi16(step2[4], step2[7]); + const __m128i out_04_1 = _mm_unpackhi_epi16(step2[4], step2[7]); + const __m128i out_20_0 = _mm_unpacklo_epi16(step2[5], step2[6]); + const __m128i out_20_1 = _mm_unpackhi_epi16(step2[5], step2[6]); + const __m128i out_12_0 = _mm_unpacklo_epi16(step2[5], step2[6]); + const __m128i out_12_1 = _mm_unpackhi_epi16(step2[5], step2[6]); + const __m128i out_28_0 = _mm_unpacklo_epi16(step2[4], step2[7]); + const __m128i out_28_1 = _mm_unpackhi_epi16(step2[4], step2[7]); + const __m128i out_04_2 = _mm_madd_epi16(out_04_0, k__cospi_p28_p04); + const __m128i out_04_3 = _mm_madd_epi16(out_04_1, k__cospi_p28_p04); + const __m128i out_20_2 = _mm_madd_epi16(out_20_0, k__cospi_p12_p20); + const __m128i out_20_3 = _mm_madd_epi16(out_20_1, k__cospi_p12_p20); + const __m128i out_12_2 = _mm_madd_epi16(out_12_0, k__cospi_m20_p12); + const __m128i out_12_3 = _mm_madd_epi16(out_12_1, k__cospi_m20_p12); + const __m128i out_28_2 = _mm_madd_epi16(out_28_0, k__cospi_m04_p28); + const __m128i out_28_3 = _mm_madd_epi16(out_28_1, k__cospi_m04_p28); + // dct_const_round_shift + const __m128i out_04_4 = _mm_add_epi32(out_04_2, k__DCT_CONST_ROUNDING); + const __m128i out_04_5 = _mm_add_epi32(out_04_3, k__DCT_CONST_ROUNDING); + const __m128i out_20_4 = _mm_add_epi32(out_20_2, k__DCT_CONST_ROUNDING); + const __m128i out_20_5 = _mm_add_epi32(out_20_3, k__DCT_CONST_ROUNDING); + const __m128i out_12_4 = _mm_add_epi32(out_12_2, k__DCT_CONST_ROUNDING); + const __m128i out_12_5 = _mm_add_epi32(out_12_3, k__DCT_CONST_ROUNDING); + const __m128i out_28_4 = _mm_add_epi32(out_28_2, k__DCT_CONST_ROUNDING); + const __m128i out_28_5 = _mm_add_epi32(out_28_3, k__DCT_CONST_ROUNDING); + const __m128i out_04_6 = _mm_srai_epi32(out_04_4, DCT_CONST_BITS); + const __m128i out_04_7 = _mm_srai_epi32(out_04_5, DCT_CONST_BITS); + const __m128i out_20_6 = _mm_srai_epi32(out_20_4, DCT_CONST_BITS); + const __m128i out_20_7 = _mm_srai_epi32(out_20_5, DCT_CONST_BITS); + const __m128i out_12_6 = _mm_srai_epi32(out_12_4, DCT_CONST_BITS); + const __m128i out_12_7 = _mm_srai_epi32(out_12_5, DCT_CONST_BITS); + const __m128i out_28_6 = _mm_srai_epi32(out_28_4, DCT_CONST_BITS); + const __m128i out_28_7 = _mm_srai_epi32(out_28_5, DCT_CONST_BITS); + // Combine + out[4] = _mm_packs_epi32(out_04_6, out_04_7); + out[20] = _mm_packs_epi32(out_20_6, out_20_7); + out[12] = _mm_packs_epi32(out_12_6, out_12_7); + out[28] = _mm_packs_epi32(out_28_6, out_28_7); + } + { + step3[8] = _mm_add_epi16(step2[9], step1[8]); + step3[9] = _mm_sub_epi16(step1[8], step2[9]); + step3[10] = _mm_sub_epi16(step1[11], step2[10]); + step3[11] = _mm_add_epi16(step2[10], step1[11]); + step3[12] = _mm_add_epi16(step2[13], step1[12]); + step3[13] = _mm_sub_epi16(step1[12], step2[13]); + step3[14] = _mm_sub_epi16(step1[15], step2[14]); + step3[15] = _mm_add_epi16(step2[14], step1[15]); + } + { + const __m128i s3_17_0 = _mm_unpacklo_epi16(step2[17], step2[30]); + const __m128i s3_17_1 = _mm_unpackhi_epi16(step2[17], step2[30]); + const __m128i s3_18_0 = _mm_unpacklo_epi16(step2[18], step2[29]); + const __m128i s3_18_1 = _mm_unpackhi_epi16(step2[18], step2[29]); + const __m128i s3_21_0 = _mm_unpacklo_epi16(step2[21], step2[26]); + const __m128i s3_21_1 = _mm_unpackhi_epi16(step2[21], step2[26]); + const __m128i s3_22_0 = _mm_unpacklo_epi16(step2[22], step2[25]); + const __m128i s3_22_1 = _mm_unpackhi_epi16(step2[22], step2[25]); + const __m128i s3_17_2 = _mm_madd_epi16(s3_17_0, k__cospi_m04_p28); + const __m128i s3_17_3 = _mm_madd_epi16(s3_17_1, k__cospi_m04_p28); + const __m128i s3_18_2 = _mm_madd_epi16(s3_18_0, k__cospi_m28_m04); + const __m128i s3_18_3 = _mm_madd_epi16(s3_18_1, k__cospi_m28_m04); + const __m128i s3_21_2 = _mm_madd_epi16(s3_21_0, k__cospi_m20_p12); + const __m128i s3_21_3 = _mm_madd_epi16(s3_21_1, k__cospi_m20_p12); + const __m128i s3_22_2 = _mm_madd_epi16(s3_22_0, k__cospi_m12_m20); + const __m128i s3_22_3 = _mm_madd_epi16(s3_22_1, k__cospi_m12_m20); + const __m128i s3_25_2 = _mm_madd_epi16(s3_22_0, k__cospi_m20_p12); + const __m128i s3_25_3 = _mm_madd_epi16(s3_22_1, k__cospi_m20_p12); + const __m128i s3_26_2 = _mm_madd_epi16(s3_21_0, k__cospi_p12_p20); + const __m128i s3_26_3 = _mm_madd_epi16(s3_21_1, k__cospi_p12_p20); + const __m128i s3_29_2 = _mm_madd_epi16(s3_18_0, k__cospi_m04_p28); + const __m128i s3_29_3 = _mm_madd_epi16(s3_18_1, k__cospi_m04_p28); + const __m128i s3_30_2 = _mm_madd_epi16(s3_17_0, k__cospi_p28_p04); + const __m128i s3_30_3 = _mm_madd_epi16(s3_17_1, k__cospi_p28_p04); + // dct_const_round_shift + const __m128i s3_17_4 = _mm_add_epi32(s3_17_2, k__DCT_CONST_ROUNDING); + const __m128i s3_17_5 = _mm_add_epi32(s3_17_3, k__DCT_CONST_ROUNDING); + const __m128i s3_18_4 = _mm_add_epi32(s3_18_2, k__DCT_CONST_ROUNDING); + const __m128i s3_18_5 = _mm_add_epi32(s3_18_3, k__DCT_CONST_ROUNDING); + const __m128i s3_21_4 = _mm_add_epi32(s3_21_2, k__DCT_CONST_ROUNDING); + const __m128i s3_21_5 = _mm_add_epi32(s3_21_3, k__DCT_CONST_ROUNDING); + const __m128i s3_22_4 = _mm_add_epi32(s3_22_2, k__DCT_CONST_ROUNDING); + const __m128i s3_22_5 = _mm_add_epi32(s3_22_3, k__DCT_CONST_ROUNDING); + const __m128i s3_17_6 = _mm_srai_epi32(s3_17_4, DCT_CONST_BITS); + const __m128i s3_17_7 = _mm_srai_epi32(s3_17_5, DCT_CONST_BITS); + const __m128i s3_18_6 = _mm_srai_epi32(s3_18_4, DCT_CONST_BITS); + const __m128i s3_18_7 = _mm_srai_epi32(s3_18_5, DCT_CONST_BITS); + const __m128i s3_21_6 = _mm_srai_epi32(s3_21_4, DCT_CONST_BITS); + const __m128i s3_21_7 = _mm_srai_epi32(s3_21_5, DCT_CONST_BITS); + const __m128i s3_22_6 = _mm_srai_epi32(s3_22_4, DCT_CONST_BITS); + const __m128i s3_22_7 = _mm_srai_epi32(s3_22_5, DCT_CONST_BITS); + const __m128i s3_25_4 = _mm_add_epi32(s3_25_2, k__DCT_CONST_ROUNDING); + const __m128i s3_25_5 = _mm_add_epi32(s3_25_3, k__DCT_CONST_ROUNDING); + const __m128i s3_26_4 = _mm_add_epi32(s3_26_2, k__DCT_CONST_ROUNDING); + const __m128i s3_26_5 = _mm_add_epi32(s3_26_3, k__DCT_CONST_ROUNDING); + const __m128i s3_29_4 = _mm_add_epi32(s3_29_2, k__DCT_CONST_ROUNDING); + const __m128i s3_29_5 = _mm_add_epi32(s3_29_3, k__DCT_CONST_ROUNDING); + const __m128i s3_30_4 = _mm_add_epi32(s3_30_2, k__DCT_CONST_ROUNDING); + const __m128i s3_30_5 = _mm_add_epi32(s3_30_3, k__DCT_CONST_ROUNDING); + const __m128i s3_25_6 = _mm_srai_epi32(s3_25_4, DCT_CONST_BITS); + const __m128i s3_25_7 = _mm_srai_epi32(s3_25_5, DCT_CONST_BITS); + const __m128i s3_26_6 = _mm_srai_epi32(s3_26_4, DCT_CONST_BITS); + const __m128i s3_26_7 = _mm_srai_epi32(s3_26_5, DCT_CONST_BITS); + const __m128i s3_29_6 = _mm_srai_epi32(s3_29_4, DCT_CONST_BITS); + const __m128i s3_29_7 = _mm_srai_epi32(s3_29_5, DCT_CONST_BITS); + const __m128i s3_30_6 = _mm_srai_epi32(s3_30_4, DCT_CONST_BITS); + const __m128i s3_30_7 = _mm_srai_epi32(s3_30_5, DCT_CONST_BITS); + // Combine + step3[17] = _mm_packs_epi32(s3_17_6, s3_17_7); + step3[18] = _mm_packs_epi32(s3_18_6, s3_18_7); + step3[21] = _mm_packs_epi32(s3_21_6, s3_21_7); + step3[22] = _mm_packs_epi32(s3_22_6, s3_22_7); + // Combine + step3[25] = _mm_packs_epi32(s3_25_6, s3_25_7); + step3[26] = _mm_packs_epi32(s3_26_6, s3_26_7); + step3[29] = _mm_packs_epi32(s3_29_6, s3_29_7); + step3[30] = _mm_packs_epi32(s3_30_6, s3_30_7); + } + // Stage 7 + { + const __m128i out_02_0 = _mm_unpacklo_epi16(step3[8], step3[15]); + const __m128i out_02_1 = _mm_unpackhi_epi16(step3[8], step3[15]); + const __m128i out_18_0 = _mm_unpacklo_epi16(step3[9], step3[14]); + const __m128i out_18_1 = _mm_unpackhi_epi16(step3[9], step3[14]); + const __m128i out_10_0 = _mm_unpacklo_epi16(step3[10], step3[13]); + const __m128i out_10_1 = _mm_unpackhi_epi16(step3[10], step3[13]); + const __m128i out_26_0 = _mm_unpacklo_epi16(step3[11], step3[12]); + const __m128i out_26_1 = _mm_unpackhi_epi16(step3[11], step3[12]); + const __m128i out_02_2 = _mm_madd_epi16(out_02_0, k__cospi_p30_p02); + const __m128i out_02_3 = _mm_madd_epi16(out_02_1, k__cospi_p30_p02); + const __m128i out_18_2 = _mm_madd_epi16(out_18_0, k__cospi_p14_p18); + const __m128i out_18_3 = _mm_madd_epi16(out_18_1, k__cospi_p14_p18); + const __m128i out_10_2 = _mm_madd_epi16(out_10_0, k__cospi_p22_p10); + const __m128i out_10_3 = _mm_madd_epi16(out_10_1, k__cospi_p22_p10); + const __m128i out_26_2 = _mm_madd_epi16(out_26_0, k__cospi_p06_p26); + const __m128i out_26_3 = _mm_madd_epi16(out_26_1, k__cospi_p06_p26); + const __m128i out_06_2 = _mm_madd_epi16(out_26_0, k__cospi_m26_p06); + const __m128i out_06_3 = _mm_madd_epi16(out_26_1, k__cospi_m26_p06); + const __m128i out_22_2 = _mm_madd_epi16(out_10_0, k__cospi_m10_p22); + const __m128i out_22_3 = _mm_madd_epi16(out_10_1, k__cospi_m10_p22); + const __m128i out_14_2 = _mm_madd_epi16(out_18_0, k__cospi_m18_p14); + const __m128i out_14_3 = _mm_madd_epi16(out_18_1, k__cospi_m18_p14); + const __m128i out_30_2 = _mm_madd_epi16(out_02_0, k__cospi_m02_p30); + const __m128i out_30_3 = _mm_madd_epi16(out_02_1, k__cospi_m02_p30); + // dct_const_round_shift + const __m128i out_02_4 = _mm_add_epi32(out_02_2, k__DCT_CONST_ROUNDING); + const __m128i out_02_5 = _mm_add_epi32(out_02_3, k__DCT_CONST_ROUNDING); + const __m128i out_18_4 = _mm_add_epi32(out_18_2, k__DCT_CONST_ROUNDING); + const __m128i out_18_5 = _mm_add_epi32(out_18_3, k__DCT_CONST_ROUNDING); + const __m128i out_10_4 = _mm_add_epi32(out_10_2, k__DCT_CONST_ROUNDING); + const __m128i out_10_5 = _mm_add_epi32(out_10_3, k__DCT_CONST_ROUNDING); + const __m128i out_26_4 = _mm_add_epi32(out_26_2, k__DCT_CONST_ROUNDING); + const __m128i out_26_5 = _mm_add_epi32(out_26_3, k__DCT_CONST_ROUNDING); + const __m128i out_06_4 = _mm_add_epi32(out_06_2, k__DCT_CONST_ROUNDING); + const __m128i out_06_5 = _mm_add_epi32(out_06_3, k__DCT_CONST_ROUNDING); + const __m128i out_22_4 = _mm_add_epi32(out_22_2, k__DCT_CONST_ROUNDING); + const __m128i out_22_5 = _mm_add_epi32(out_22_3, k__DCT_CONST_ROUNDING); + const __m128i out_14_4 = _mm_add_epi32(out_14_2, k__DCT_CONST_ROUNDING); + const __m128i out_14_5 = _mm_add_epi32(out_14_3, k__DCT_CONST_ROUNDING); + const __m128i out_30_4 = _mm_add_epi32(out_30_2, k__DCT_CONST_ROUNDING); + const __m128i out_30_5 = _mm_add_epi32(out_30_3, k__DCT_CONST_ROUNDING); + const __m128i out_02_6 = _mm_srai_epi32(out_02_4, DCT_CONST_BITS); + const __m128i out_02_7 = _mm_srai_epi32(out_02_5, DCT_CONST_BITS); + const __m128i out_18_6 = _mm_srai_epi32(out_18_4, DCT_CONST_BITS); + const __m128i out_18_7 = _mm_srai_epi32(out_18_5, DCT_CONST_BITS); + const __m128i out_10_6 = _mm_srai_epi32(out_10_4, DCT_CONST_BITS); + const __m128i out_10_7 = _mm_srai_epi32(out_10_5, DCT_CONST_BITS); + const __m128i out_26_6 = _mm_srai_epi32(out_26_4, DCT_CONST_BITS); + const __m128i out_26_7 = _mm_srai_epi32(out_26_5, DCT_CONST_BITS); + const __m128i out_06_6 = _mm_srai_epi32(out_06_4, DCT_CONST_BITS); + const __m128i out_06_7 = _mm_srai_epi32(out_06_5, DCT_CONST_BITS); + const __m128i out_22_6 = _mm_srai_epi32(out_22_4, DCT_CONST_BITS); + const __m128i out_22_7 = _mm_srai_epi32(out_22_5, DCT_CONST_BITS); + const __m128i out_14_6 = _mm_srai_epi32(out_14_4, DCT_CONST_BITS); + const __m128i out_14_7 = _mm_srai_epi32(out_14_5, DCT_CONST_BITS); + const __m128i out_30_6 = _mm_srai_epi32(out_30_4, DCT_CONST_BITS); + const __m128i out_30_7 = _mm_srai_epi32(out_30_5, DCT_CONST_BITS); + // Combine + out[2] = _mm_packs_epi32(out_02_6, out_02_7); + out[18] = _mm_packs_epi32(out_18_6, out_18_7); + out[10] = _mm_packs_epi32(out_10_6, out_10_7); + out[26] = _mm_packs_epi32(out_26_6, out_26_7); + out[6] = _mm_packs_epi32(out_06_6, out_06_7); + out[22] = _mm_packs_epi32(out_22_6, out_22_7); + out[14] = _mm_packs_epi32(out_14_6, out_14_7); + out[30] = _mm_packs_epi32(out_30_6, out_30_7); + } + { + step1[16] = _mm_add_epi16(step3[17], step2[16]); + step1[17] = _mm_sub_epi16(step2[16], step3[17]); + step1[18] = _mm_sub_epi16(step2[19], step3[18]); + step1[19] = _mm_add_epi16(step3[18], step2[19]); + step1[20] = _mm_add_epi16(step3[21], step2[20]); + step1[21] = _mm_sub_epi16(step2[20], step3[21]); + step1[22] = _mm_sub_epi16(step2[23], step3[22]); + step1[23] = _mm_add_epi16(step3[22], step2[23]); + step1[24] = _mm_add_epi16(step3[25], step2[24]); + step1[25] = _mm_sub_epi16(step2[24], step3[25]); + step1[26] = _mm_sub_epi16(step2[27], step3[26]); + step1[27] = _mm_add_epi16(step3[26], step2[27]); + step1[28] = _mm_add_epi16(step3[29], step2[28]); + step1[29] = _mm_sub_epi16(step2[28], step3[29]); + step1[30] = _mm_sub_epi16(step2[31], step3[30]); + step1[31] = _mm_add_epi16(step3[30], step2[31]); + } + // Final stage --- outputs indices are bit-reversed. + { + const __m128i out_01_0 = _mm_unpacklo_epi16(step1[16], step1[31]); + const __m128i out_01_1 = _mm_unpackhi_epi16(step1[16], step1[31]); + const __m128i out_17_0 = _mm_unpacklo_epi16(step1[17], step1[30]); + const __m128i out_17_1 = _mm_unpackhi_epi16(step1[17], step1[30]); + const __m128i out_09_0 = _mm_unpacklo_epi16(step1[18], step1[29]); + const __m128i out_09_1 = _mm_unpackhi_epi16(step1[18], step1[29]); + const __m128i out_25_0 = _mm_unpacklo_epi16(step1[19], step1[28]); + const __m128i out_25_1 = _mm_unpackhi_epi16(step1[19], step1[28]); + const __m128i out_01_2 = _mm_madd_epi16(out_01_0, k__cospi_p31_p01); + const __m128i out_01_3 = _mm_madd_epi16(out_01_1, k__cospi_p31_p01); + const __m128i out_17_2 = _mm_madd_epi16(out_17_0, k__cospi_p15_p17); + const __m128i out_17_3 = _mm_madd_epi16(out_17_1, k__cospi_p15_p17); + const __m128i out_09_2 = _mm_madd_epi16(out_09_0, k__cospi_p23_p09); + const __m128i out_09_3 = _mm_madd_epi16(out_09_1, k__cospi_p23_p09); + const __m128i out_25_2 = _mm_madd_epi16(out_25_0, k__cospi_p07_p25); + const __m128i out_25_3 = _mm_madd_epi16(out_25_1, k__cospi_p07_p25); + const __m128i out_07_2 = _mm_madd_epi16(out_25_0, k__cospi_m25_p07); + const __m128i out_07_3 = _mm_madd_epi16(out_25_1, k__cospi_m25_p07); + const __m128i out_23_2 = _mm_madd_epi16(out_09_0, k__cospi_m09_p23); + const __m128i out_23_3 = _mm_madd_epi16(out_09_1, k__cospi_m09_p23); + const __m128i out_15_2 = _mm_madd_epi16(out_17_0, k__cospi_m17_p15); + const __m128i out_15_3 = _mm_madd_epi16(out_17_1, k__cospi_m17_p15); + const __m128i out_31_2 = _mm_madd_epi16(out_01_0, k__cospi_m01_p31); + const __m128i out_31_3 = _mm_madd_epi16(out_01_1, k__cospi_m01_p31); + // dct_const_round_shift + const __m128i out_01_4 = _mm_add_epi32(out_01_2, k__DCT_CONST_ROUNDING); + const __m128i out_01_5 = _mm_add_epi32(out_01_3, k__DCT_CONST_ROUNDING); + const __m128i out_17_4 = _mm_add_epi32(out_17_2, k__DCT_CONST_ROUNDING); + const __m128i out_17_5 = _mm_add_epi32(out_17_3, k__DCT_CONST_ROUNDING); + const __m128i out_09_4 = _mm_add_epi32(out_09_2, k__DCT_CONST_ROUNDING); + const __m128i out_09_5 = _mm_add_epi32(out_09_3, k__DCT_CONST_ROUNDING); + const __m128i out_25_4 = _mm_add_epi32(out_25_2, k__DCT_CONST_ROUNDING); + const __m128i out_25_5 = _mm_add_epi32(out_25_3, k__DCT_CONST_ROUNDING); + const __m128i out_07_4 = _mm_add_epi32(out_07_2, k__DCT_CONST_ROUNDING); + const __m128i out_07_5 = _mm_add_epi32(out_07_3, k__DCT_CONST_ROUNDING); + const __m128i out_23_4 = _mm_add_epi32(out_23_2, k__DCT_CONST_ROUNDING); + const __m128i out_23_5 = _mm_add_epi32(out_23_3, k__DCT_CONST_ROUNDING); + const __m128i out_15_4 = _mm_add_epi32(out_15_2, k__DCT_CONST_ROUNDING); + const __m128i out_15_5 = _mm_add_epi32(out_15_3, k__DCT_CONST_ROUNDING); + const __m128i out_31_4 = _mm_add_epi32(out_31_2, k__DCT_CONST_ROUNDING); + const __m128i out_31_5 = _mm_add_epi32(out_31_3, k__DCT_CONST_ROUNDING); + const __m128i out_01_6 = _mm_srai_epi32(out_01_4, DCT_CONST_BITS); + const __m128i out_01_7 = _mm_srai_epi32(out_01_5, DCT_CONST_BITS); + const __m128i out_17_6 = _mm_srai_epi32(out_17_4, DCT_CONST_BITS); + const __m128i out_17_7 = _mm_srai_epi32(out_17_5, DCT_CONST_BITS); + const __m128i out_09_6 = _mm_srai_epi32(out_09_4, DCT_CONST_BITS); + const __m128i out_09_7 = _mm_srai_epi32(out_09_5, DCT_CONST_BITS); + const __m128i out_25_6 = _mm_srai_epi32(out_25_4, DCT_CONST_BITS); + const __m128i out_25_7 = _mm_srai_epi32(out_25_5, DCT_CONST_BITS); + const __m128i out_07_6 = _mm_srai_epi32(out_07_4, DCT_CONST_BITS); + const __m128i out_07_7 = _mm_srai_epi32(out_07_5, DCT_CONST_BITS); + const __m128i out_23_6 = _mm_srai_epi32(out_23_4, DCT_CONST_BITS); + const __m128i out_23_7 = _mm_srai_epi32(out_23_5, DCT_CONST_BITS); + const __m128i out_15_6 = _mm_srai_epi32(out_15_4, DCT_CONST_BITS); + const __m128i out_15_7 = _mm_srai_epi32(out_15_5, DCT_CONST_BITS); + const __m128i out_31_6 = _mm_srai_epi32(out_31_4, DCT_CONST_BITS); + const __m128i out_31_7 = _mm_srai_epi32(out_31_5, DCT_CONST_BITS); + // Combine + out[1] = _mm_packs_epi32(out_01_6, out_01_7); + out[17] = _mm_packs_epi32(out_17_6, out_17_7); + out[9] = _mm_packs_epi32(out_09_6, out_09_7); + out[25] = _mm_packs_epi32(out_25_6, out_25_7); + out[7] = _mm_packs_epi32(out_07_6, out_07_7); + out[23] = _mm_packs_epi32(out_23_6, out_23_7); + out[15] = _mm_packs_epi32(out_15_6, out_15_7); + out[31] = _mm_packs_epi32(out_31_6, out_31_7); + } + { + const __m128i out_05_0 = _mm_unpacklo_epi16(step1[20], step1[27]); + const __m128i out_05_1 = _mm_unpackhi_epi16(step1[20], step1[27]); + const __m128i out_21_0 = _mm_unpacklo_epi16(step1[21], step1[26]); + const __m128i out_21_1 = _mm_unpackhi_epi16(step1[21], step1[26]); + const __m128i out_13_0 = _mm_unpacklo_epi16(step1[22], step1[25]); + const __m128i out_13_1 = _mm_unpackhi_epi16(step1[22], step1[25]); + const __m128i out_29_0 = _mm_unpacklo_epi16(step1[23], step1[24]); + const __m128i out_29_1 = _mm_unpackhi_epi16(step1[23], step1[24]); + const __m128i out_05_2 = _mm_madd_epi16(out_05_0, k__cospi_p27_p05); + const __m128i out_05_3 = _mm_madd_epi16(out_05_1, k__cospi_p27_p05); + const __m128i out_21_2 = _mm_madd_epi16(out_21_0, k__cospi_p11_p21); + const __m128i out_21_3 = _mm_madd_epi16(out_21_1, k__cospi_p11_p21); + const __m128i out_13_2 = _mm_madd_epi16(out_13_0, k__cospi_p19_p13); + const __m128i out_13_3 = _mm_madd_epi16(out_13_1, k__cospi_p19_p13); + const __m128i out_29_2 = _mm_madd_epi16(out_29_0, k__cospi_p03_p29); + const __m128i out_29_3 = _mm_madd_epi16(out_29_1, k__cospi_p03_p29); + const __m128i out_03_2 = _mm_madd_epi16(out_29_0, k__cospi_m29_p03); + const __m128i out_03_3 = _mm_madd_epi16(out_29_1, k__cospi_m29_p03); + const __m128i out_19_2 = _mm_madd_epi16(out_13_0, k__cospi_m13_p19); + const __m128i out_19_3 = _mm_madd_epi16(out_13_1, k__cospi_m13_p19); + const __m128i out_11_2 = _mm_madd_epi16(out_21_0, k__cospi_m21_p11); + const __m128i out_11_3 = _mm_madd_epi16(out_21_1, k__cospi_m21_p11); + const __m128i out_27_2 = _mm_madd_epi16(out_05_0, k__cospi_m05_p27); + const __m128i out_27_3 = _mm_madd_epi16(out_05_1, k__cospi_m05_p27); + // dct_const_round_shift + const __m128i out_05_4 = _mm_add_epi32(out_05_2, k__DCT_CONST_ROUNDING); + const __m128i out_05_5 = _mm_add_epi32(out_05_3, k__DCT_CONST_ROUNDING); + const __m128i out_21_4 = _mm_add_epi32(out_21_2, k__DCT_CONST_ROUNDING); + const __m128i out_21_5 = _mm_add_epi32(out_21_3, k__DCT_CONST_ROUNDING); + const __m128i out_13_4 = _mm_add_epi32(out_13_2, k__DCT_CONST_ROUNDING); + const __m128i out_13_5 = _mm_add_epi32(out_13_3, k__DCT_CONST_ROUNDING); + const __m128i out_29_4 = _mm_add_epi32(out_29_2, k__DCT_CONST_ROUNDING); + const __m128i out_29_5 = _mm_add_epi32(out_29_3, k__DCT_CONST_ROUNDING); + const __m128i out_03_4 = _mm_add_epi32(out_03_2, k__DCT_CONST_ROUNDING); + const __m128i out_03_5 = _mm_add_epi32(out_03_3, k__DCT_CONST_ROUNDING); + const __m128i out_19_4 = _mm_add_epi32(out_19_2, k__DCT_CONST_ROUNDING); + const __m128i out_19_5 = _mm_add_epi32(out_19_3, k__DCT_CONST_ROUNDING); + const __m128i out_11_4 = _mm_add_epi32(out_11_2, k__DCT_CONST_ROUNDING); + const __m128i out_11_5 = _mm_add_epi32(out_11_3, k__DCT_CONST_ROUNDING); + const __m128i out_27_4 = _mm_add_epi32(out_27_2, k__DCT_CONST_ROUNDING); + const __m128i out_27_5 = _mm_add_epi32(out_27_3, k__DCT_CONST_ROUNDING); + const __m128i out_05_6 = _mm_srai_epi32(out_05_4, DCT_CONST_BITS); + const __m128i out_05_7 = _mm_srai_epi32(out_05_5, DCT_CONST_BITS); + const __m128i out_21_6 = _mm_srai_epi32(out_21_4, DCT_CONST_BITS); + const __m128i out_21_7 = _mm_srai_epi32(out_21_5, DCT_CONST_BITS); + const __m128i out_13_6 = _mm_srai_epi32(out_13_4, DCT_CONST_BITS); + const __m128i out_13_7 = _mm_srai_epi32(out_13_5, DCT_CONST_BITS); + const __m128i out_29_6 = _mm_srai_epi32(out_29_4, DCT_CONST_BITS); + const __m128i out_29_7 = _mm_srai_epi32(out_29_5, DCT_CONST_BITS); + const __m128i out_03_6 = _mm_srai_epi32(out_03_4, DCT_CONST_BITS); + const __m128i out_03_7 = _mm_srai_epi32(out_03_5, DCT_CONST_BITS); + const __m128i out_19_6 = _mm_srai_epi32(out_19_4, DCT_CONST_BITS); + const __m128i out_19_7 = _mm_srai_epi32(out_19_5, DCT_CONST_BITS); + const __m128i out_11_6 = _mm_srai_epi32(out_11_4, DCT_CONST_BITS); + const __m128i out_11_7 = _mm_srai_epi32(out_11_5, DCT_CONST_BITS); + const __m128i out_27_6 = _mm_srai_epi32(out_27_4, DCT_CONST_BITS); + const __m128i out_27_7 = _mm_srai_epi32(out_27_5, DCT_CONST_BITS); + // Combine + out[5] = _mm_packs_epi32(out_05_6, out_05_7); + out[21] = _mm_packs_epi32(out_21_6, out_21_7); + out[13] = _mm_packs_epi32(out_13_6, out_13_7); + out[29] = _mm_packs_epi32(out_29_6, out_29_7); + out[3] = _mm_packs_epi32(out_03_6, out_03_7); + out[19] = _mm_packs_epi32(out_19_6, out_19_7); + out[11] = _mm_packs_epi32(out_11_6, out_11_7); + out[27] = _mm_packs_epi32(out_27_6, out_27_7); + } + + // Output results + { + int j; + for (j = 0; j < 16; ++j) { + _mm_storeu_si128((__m128i *)(in0 + j), out[j]); + _mm_storeu_si128((__m128i *)(in1 + j), out[j + 16]); + } + } +} // NOLINT diff --git a/third_party/aom/aom_dsp/x86/fwd_dct32x32_impl_avx2.h b/third_party/aom/aom_dsp/x86/fwd_dct32x32_impl_avx2.h new file mode 100644 index 0000000000..216739581b --- /dev/null +++ b/third_party/aom/aom_dsp/x86/fwd_dct32x32_impl_avx2.h @@ -0,0 +1,3022 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // AVX2 + +#include "aom_dsp/txfm_common.h" +#include "aom_dsp/x86/txfm_common_intrin.h" +#include "aom_dsp/x86/txfm_common_avx2.h" + +#if FDCT32x32_HIGH_PRECISION +static INLINE __m256i k_madd_epi32_avx2(__m256i a, __m256i b) { + __m256i buf0, buf1; + buf0 = _mm256_mul_epu32(a, b); + a = _mm256_srli_epi64(a, 32); + b = _mm256_srli_epi64(b, 32); + buf1 = _mm256_mul_epu32(a, b); + return _mm256_add_epi64(buf0, buf1); +} + +static INLINE __m256i k_packs_epi64_avx2(__m256i a, __m256i b) { + __m256i buf0 = _mm256_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 2, 0)); + __m256i buf1 = _mm256_shuffle_epi32(b, _MM_SHUFFLE(0, 0, 2, 0)); + return _mm256_unpacklo_epi64(buf0, buf1); +} +#endif + +#ifndef STORE_COEFF_FUNC +#define STORE_COEFF_FUNC +static void store_coeff(const __m256i *coeff, tran_low_t *curr, + tran_low_t *next) { + __m128i u = _mm256_castsi256_si128(*coeff); + storeu_output(&u, curr); + u = _mm256_extractf128_si256(*coeff, 1); + storeu_output(&u, next); +} +#endif + +void FDCT32x32_2D_AVX2(const int16_t *input, tran_low_t *output_org, + int stride) { + // Calculate pre-multiplied strides + const int str1 = stride; + const int str2 = 2 * stride; + const int str3 = 2 * stride + str1; + // We need an intermediate buffer between passes. + DECLARE_ALIGNED(32, int16_t, intermediate[32 * 32]); + // Constants + // When we use them, in one case, they are all the same. In all others + // it's a pair of them that we need to repeat four times. This is done + // by constructing the 32 bit constant corresponding to that pair. + const __m256i k__cospi_p16_p16 = _mm256_set1_epi16((int16_t)cospi_16_64); + const __m256i k__cospi_p16_m16 = + pair256_set_epi16(+cospi_16_64, -cospi_16_64); + const __m256i k__cospi_m08_p24 = pair256_set_epi16(-cospi_8_64, cospi_24_64); + const __m256i k__cospi_m24_m08 = pair256_set_epi16(-cospi_24_64, -cospi_8_64); + const __m256i k__cospi_p24_p08 = pair256_set_epi16(+cospi_24_64, cospi_8_64); + const __m256i k__cospi_p12_p20 = pair256_set_epi16(+cospi_12_64, cospi_20_64); + const __m256i k__cospi_m20_p12 = pair256_set_epi16(-cospi_20_64, cospi_12_64); + const __m256i k__cospi_m04_p28 = pair256_set_epi16(-cospi_4_64, cospi_28_64); + const __m256i k__cospi_p28_p04 = pair256_set_epi16(+cospi_28_64, cospi_4_64); + const __m256i k__cospi_m28_m04 = pair256_set_epi16(-cospi_28_64, -cospi_4_64); + const __m256i k__cospi_m12_m20 = + pair256_set_epi16(-cospi_12_64, -cospi_20_64); + const __m256i k__cospi_p30_p02 = pair256_set_epi16(+cospi_30_64, cospi_2_64); + const __m256i k__cospi_p14_p18 = pair256_set_epi16(+cospi_14_64, cospi_18_64); + const __m256i k__cospi_p22_p10 = pair256_set_epi16(+cospi_22_64, cospi_10_64); + const __m256i k__cospi_p06_p26 = pair256_set_epi16(+cospi_6_64, cospi_26_64); + const __m256i k__cospi_m26_p06 = pair256_set_epi16(-cospi_26_64, cospi_6_64); + const __m256i k__cospi_m10_p22 = pair256_set_epi16(-cospi_10_64, cospi_22_64); + const __m256i k__cospi_m18_p14 = pair256_set_epi16(-cospi_18_64, cospi_14_64); + const __m256i k__cospi_m02_p30 = pair256_set_epi16(-cospi_2_64, cospi_30_64); + const __m256i k__cospi_p31_p01 = pair256_set_epi16(+cospi_31_64, cospi_1_64); + const __m256i k__cospi_p15_p17 = pair256_set_epi16(+cospi_15_64, cospi_17_64); + const __m256i k__cospi_p23_p09 = pair256_set_epi16(+cospi_23_64, cospi_9_64); + const __m256i k__cospi_p07_p25 = pair256_set_epi16(+cospi_7_64, cospi_25_64); + const __m256i k__cospi_m25_p07 = pair256_set_epi16(-cospi_25_64, cospi_7_64); + const __m256i k__cospi_m09_p23 = pair256_set_epi16(-cospi_9_64, cospi_23_64); + const __m256i k__cospi_m17_p15 = pair256_set_epi16(-cospi_17_64, cospi_15_64); + const __m256i k__cospi_m01_p31 = pair256_set_epi16(-cospi_1_64, cospi_31_64); + const __m256i k__cospi_p27_p05 = pair256_set_epi16(+cospi_27_64, cospi_5_64); + const __m256i k__cospi_p11_p21 = pair256_set_epi16(+cospi_11_64, cospi_21_64); + const __m256i k__cospi_p19_p13 = pair256_set_epi16(+cospi_19_64, cospi_13_64); + const __m256i k__cospi_p03_p29 = pair256_set_epi16(+cospi_3_64, cospi_29_64); + const __m256i k__cospi_m29_p03 = pair256_set_epi16(-cospi_29_64, cospi_3_64); + const __m256i k__cospi_m13_p19 = pair256_set_epi16(-cospi_13_64, cospi_19_64); + const __m256i k__cospi_m21_p11 = pair256_set_epi16(-cospi_21_64, cospi_11_64); + const __m256i k__cospi_m05_p27 = pair256_set_epi16(-cospi_5_64, cospi_27_64); + const __m256i k__DCT_CONST_ROUNDING = _mm256_set1_epi32(DCT_CONST_ROUNDING); + const __m256i kZero = _mm256_set1_epi16(0); + const __m256i kOne = _mm256_set1_epi16(1); + // Do the two transform/transpose passes + int pass; + for (pass = 0; pass < 2; ++pass) { + // We process sixteen columns (transposed rows in second pass) at a time. + int column_start; + for (column_start = 0; column_start < 32; column_start += 16) { + __m256i step1[32]; + __m256i step2[32]; + __m256i step3[32]; + __m256i out[32]; + // Stage 1 + // Note: even though all the loads below are aligned, using the aligned + // intrinsic make the code slightly slower. + if (0 == pass) { + const int16_t *in = &input[column_start]; + // step1[i] = (in[ 0 * stride] + in[(32 - 1) * stride]) << 2; + // Note: the next four blocks could be in a loop. That would help the + // instruction cache but is actually slower. + { + const int16_t *ina = in + 0 * str1; + const int16_t *inb = in + 31 * str1; + __m256i *step1a = &step1[0]; + __m256i *step1b = &step1[31]; + const __m256i ina0 = _mm256_loadu_si256((const __m256i *)(ina)); + const __m256i ina1 = + _mm256_loadu_si256((const __m256i *)(ina + str1)); + const __m256i ina2 = + _mm256_loadu_si256((const __m256i *)(ina + str2)); + const __m256i ina3 = + _mm256_loadu_si256((const __m256i *)(ina + str3)); + const __m256i inb3 = + _mm256_loadu_si256((const __m256i *)(inb - str3)); + const __m256i inb2 = + _mm256_loadu_si256((const __m256i *)(inb - str2)); + const __m256i inb1 = + _mm256_loadu_si256((const __m256i *)(inb - str1)); + const __m256i inb0 = _mm256_loadu_si256((const __m256i *)(inb)); + step1a[0] = _mm256_add_epi16(ina0, inb0); + step1a[1] = _mm256_add_epi16(ina1, inb1); + step1a[2] = _mm256_add_epi16(ina2, inb2); + step1a[3] = _mm256_add_epi16(ina3, inb3); + step1b[-3] = _mm256_sub_epi16(ina3, inb3); + step1b[-2] = _mm256_sub_epi16(ina2, inb2); + step1b[-1] = _mm256_sub_epi16(ina1, inb1); + step1b[-0] = _mm256_sub_epi16(ina0, inb0); + step1a[0] = _mm256_slli_epi16(step1a[0], 2); + step1a[1] = _mm256_slli_epi16(step1a[1], 2); + step1a[2] = _mm256_slli_epi16(step1a[2], 2); + step1a[3] = _mm256_slli_epi16(step1a[3], 2); + step1b[-3] = _mm256_slli_epi16(step1b[-3], 2); + step1b[-2] = _mm256_slli_epi16(step1b[-2], 2); + step1b[-1] = _mm256_slli_epi16(step1b[-1], 2); + step1b[-0] = _mm256_slli_epi16(step1b[-0], 2); + } + { + const int16_t *ina = in + 4 * str1; + const int16_t *inb = in + 27 * str1; + __m256i *step1a = &step1[4]; + __m256i *step1b = &step1[27]; + const __m256i ina0 = _mm256_loadu_si256((const __m256i *)(ina)); + const __m256i ina1 = + _mm256_loadu_si256((const __m256i *)(ina + str1)); + const __m256i ina2 = + _mm256_loadu_si256((const __m256i *)(ina + str2)); + const __m256i ina3 = + _mm256_loadu_si256((const __m256i *)(ina + str3)); + const __m256i inb3 = + _mm256_loadu_si256((const __m256i *)(inb - str3)); + const __m256i inb2 = + _mm256_loadu_si256((const __m256i *)(inb - str2)); + const __m256i inb1 = + _mm256_loadu_si256((const __m256i *)(inb - str1)); + const __m256i inb0 = _mm256_loadu_si256((const __m256i *)(inb)); + step1a[0] = _mm256_add_epi16(ina0, inb0); + step1a[1] = _mm256_add_epi16(ina1, inb1); + step1a[2] = _mm256_add_epi16(ina2, inb2); + step1a[3] = _mm256_add_epi16(ina3, inb3); + step1b[-3] = _mm256_sub_epi16(ina3, inb3); + step1b[-2] = _mm256_sub_epi16(ina2, inb2); + step1b[-1] = _mm256_sub_epi16(ina1, inb1); + step1b[-0] = _mm256_sub_epi16(ina0, inb0); + step1a[0] = _mm256_slli_epi16(step1a[0], 2); + step1a[1] = _mm256_slli_epi16(step1a[1], 2); + step1a[2] = _mm256_slli_epi16(step1a[2], 2); + step1a[3] = _mm256_slli_epi16(step1a[3], 2); + step1b[-3] = _mm256_slli_epi16(step1b[-3], 2); + step1b[-2] = _mm256_slli_epi16(step1b[-2], 2); + step1b[-1] = _mm256_slli_epi16(step1b[-1], 2); + step1b[-0] = _mm256_slli_epi16(step1b[-0], 2); + } + { + const int16_t *ina = in + 8 * str1; + const int16_t *inb = in + 23 * str1; + __m256i *step1a = &step1[8]; + __m256i *step1b = &step1[23]; + const __m256i ina0 = _mm256_loadu_si256((const __m256i *)(ina)); + const __m256i ina1 = + _mm256_loadu_si256((const __m256i *)(ina + str1)); + const __m256i ina2 = + _mm256_loadu_si256((const __m256i *)(ina + str2)); + const __m256i ina3 = + _mm256_loadu_si256((const __m256i *)(ina + str3)); + const __m256i inb3 = + _mm256_loadu_si256((const __m256i *)(inb - str3)); + const __m256i inb2 = + _mm256_loadu_si256((const __m256i *)(inb - str2)); + const __m256i inb1 = + _mm256_loadu_si256((const __m256i *)(inb - str1)); + const __m256i inb0 = _mm256_loadu_si256((const __m256i *)(inb)); + step1a[0] = _mm256_add_epi16(ina0, inb0); + step1a[1] = _mm256_add_epi16(ina1, inb1); + step1a[2] = _mm256_add_epi16(ina2, inb2); + step1a[3] = _mm256_add_epi16(ina3, inb3); + step1b[-3] = _mm256_sub_epi16(ina3, inb3); + step1b[-2] = _mm256_sub_epi16(ina2, inb2); + step1b[-1] = _mm256_sub_epi16(ina1, inb1); + step1b[-0] = _mm256_sub_epi16(ina0, inb0); + step1a[0] = _mm256_slli_epi16(step1a[0], 2); + step1a[1] = _mm256_slli_epi16(step1a[1], 2); + step1a[2] = _mm256_slli_epi16(step1a[2], 2); + step1a[3] = _mm256_slli_epi16(step1a[3], 2); + step1b[-3] = _mm256_slli_epi16(step1b[-3], 2); + step1b[-2] = _mm256_slli_epi16(step1b[-2], 2); + step1b[-1] = _mm256_slli_epi16(step1b[-1], 2); + step1b[-0] = _mm256_slli_epi16(step1b[-0], 2); + } + { + const int16_t *ina = in + 12 * str1; + const int16_t *inb = in + 19 * str1; + __m256i *step1a = &step1[12]; + __m256i *step1b = &step1[19]; + const __m256i ina0 = _mm256_loadu_si256((const __m256i *)(ina)); + const __m256i ina1 = + _mm256_loadu_si256((const __m256i *)(ina + str1)); + const __m256i ina2 = + _mm256_loadu_si256((const __m256i *)(ina + str2)); + const __m256i ina3 = + _mm256_loadu_si256((const __m256i *)(ina + str3)); + const __m256i inb3 = + _mm256_loadu_si256((const __m256i *)(inb - str3)); + const __m256i inb2 = + _mm256_loadu_si256((const __m256i *)(inb - str2)); + const __m256i inb1 = + _mm256_loadu_si256((const __m256i *)(inb - str1)); + const __m256i inb0 = _mm256_loadu_si256((const __m256i *)(inb)); + step1a[0] = _mm256_add_epi16(ina0, inb0); + step1a[1] = _mm256_add_epi16(ina1, inb1); + step1a[2] = _mm256_add_epi16(ina2, inb2); + step1a[3] = _mm256_add_epi16(ina3, inb3); + step1b[-3] = _mm256_sub_epi16(ina3, inb3); + step1b[-2] = _mm256_sub_epi16(ina2, inb2); + step1b[-1] = _mm256_sub_epi16(ina1, inb1); + step1b[-0] = _mm256_sub_epi16(ina0, inb0); + step1a[0] = _mm256_slli_epi16(step1a[0], 2); + step1a[1] = _mm256_slli_epi16(step1a[1], 2); + step1a[2] = _mm256_slli_epi16(step1a[2], 2); + step1a[3] = _mm256_slli_epi16(step1a[3], 2); + step1b[-3] = _mm256_slli_epi16(step1b[-3], 2); + step1b[-2] = _mm256_slli_epi16(step1b[-2], 2); + step1b[-1] = _mm256_slli_epi16(step1b[-1], 2); + step1b[-0] = _mm256_slli_epi16(step1b[-0], 2); + } + } else { + int16_t *in = &intermediate[column_start]; + // step1[i] = in[ 0 * 32] + in[(32 - 1) * 32]; + // Note: using the same approach as above to have common offset is + // counter-productive as all offsets can be calculated at compile + // time. + // Note: the next four blocks could be in a loop. That would help the + // instruction cache but is actually slower. + { + __m256i in00 = _mm256_loadu_si256((const __m256i *)(in + 0 * 32)); + __m256i in01 = _mm256_loadu_si256((const __m256i *)(in + 1 * 32)); + __m256i in02 = _mm256_loadu_si256((const __m256i *)(in + 2 * 32)); + __m256i in03 = _mm256_loadu_si256((const __m256i *)(in + 3 * 32)); + __m256i in28 = _mm256_loadu_si256((const __m256i *)(in + 28 * 32)); + __m256i in29 = _mm256_loadu_si256((const __m256i *)(in + 29 * 32)); + __m256i in30 = _mm256_loadu_si256((const __m256i *)(in + 30 * 32)); + __m256i in31 = _mm256_loadu_si256((const __m256i *)(in + 31 * 32)); + step1[0] = _mm256_add_epi16(in00, in31); + step1[1] = _mm256_add_epi16(in01, in30); + step1[2] = _mm256_add_epi16(in02, in29); + step1[3] = _mm256_add_epi16(in03, in28); + step1[28] = _mm256_sub_epi16(in03, in28); + step1[29] = _mm256_sub_epi16(in02, in29); + step1[30] = _mm256_sub_epi16(in01, in30); + step1[31] = _mm256_sub_epi16(in00, in31); + } + { + __m256i in04 = _mm256_loadu_si256((const __m256i *)(in + 4 * 32)); + __m256i in05 = _mm256_loadu_si256((const __m256i *)(in + 5 * 32)); + __m256i in06 = _mm256_loadu_si256((const __m256i *)(in + 6 * 32)); + __m256i in07 = _mm256_loadu_si256((const __m256i *)(in + 7 * 32)); + __m256i in24 = _mm256_loadu_si256((const __m256i *)(in + 24 * 32)); + __m256i in25 = _mm256_loadu_si256((const __m256i *)(in + 25 * 32)); + __m256i in26 = _mm256_loadu_si256((const __m256i *)(in + 26 * 32)); + __m256i in27 = _mm256_loadu_si256((const __m256i *)(in + 27 * 32)); + step1[4] = _mm256_add_epi16(in04, in27); + step1[5] = _mm256_add_epi16(in05, in26); + step1[6] = _mm256_add_epi16(in06, in25); + step1[7] = _mm256_add_epi16(in07, in24); + step1[24] = _mm256_sub_epi16(in07, in24); + step1[25] = _mm256_sub_epi16(in06, in25); + step1[26] = _mm256_sub_epi16(in05, in26); + step1[27] = _mm256_sub_epi16(in04, in27); + } + { + __m256i in08 = _mm256_loadu_si256((const __m256i *)(in + 8 * 32)); + __m256i in09 = _mm256_loadu_si256((const __m256i *)(in + 9 * 32)); + __m256i in10 = _mm256_loadu_si256((const __m256i *)(in + 10 * 32)); + __m256i in11 = _mm256_loadu_si256((const __m256i *)(in + 11 * 32)); + __m256i in20 = _mm256_loadu_si256((const __m256i *)(in + 20 * 32)); + __m256i in21 = _mm256_loadu_si256((const __m256i *)(in + 21 * 32)); + __m256i in22 = _mm256_loadu_si256((const __m256i *)(in + 22 * 32)); + __m256i in23 = _mm256_loadu_si256((const __m256i *)(in + 23 * 32)); + step1[8] = _mm256_add_epi16(in08, in23); + step1[9] = _mm256_add_epi16(in09, in22); + step1[10] = _mm256_add_epi16(in10, in21); + step1[11] = _mm256_add_epi16(in11, in20); + step1[20] = _mm256_sub_epi16(in11, in20); + step1[21] = _mm256_sub_epi16(in10, in21); + step1[22] = _mm256_sub_epi16(in09, in22); + step1[23] = _mm256_sub_epi16(in08, in23); + } + { + __m256i in12 = _mm256_loadu_si256((const __m256i *)(in + 12 * 32)); + __m256i in13 = _mm256_loadu_si256((const __m256i *)(in + 13 * 32)); + __m256i in14 = _mm256_loadu_si256((const __m256i *)(in + 14 * 32)); + __m256i in15 = _mm256_loadu_si256((const __m256i *)(in + 15 * 32)); + __m256i in16 = _mm256_loadu_si256((const __m256i *)(in + 16 * 32)); + __m256i in17 = _mm256_loadu_si256((const __m256i *)(in + 17 * 32)); + __m256i in18 = _mm256_loadu_si256((const __m256i *)(in + 18 * 32)); + __m256i in19 = _mm256_loadu_si256((const __m256i *)(in + 19 * 32)); + step1[12] = _mm256_add_epi16(in12, in19); + step1[13] = _mm256_add_epi16(in13, in18); + step1[14] = _mm256_add_epi16(in14, in17); + step1[15] = _mm256_add_epi16(in15, in16); + step1[16] = _mm256_sub_epi16(in15, in16); + step1[17] = _mm256_sub_epi16(in14, in17); + step1[18] = _mm256_sub_epi16(in13, in18); + step1[19] = _mm256_sub_epi16(in12, in19); + } + } + // Stage 2 + { + step2[0] = _mm256_add_epi16(step1[0], step1[15]); + step2[1] = _mm256_add_epi16(step1[1], step1[14]); + step2[2] = _mm256_add_epi16(step1[2], step1[13]); + step2[3] = _mm256_add_epi16(step1[3], step1[12]); + step2[4] = _mm256_add_epi16(step1[4], step1[11]); + step2[5] = _mm256_add_epi16(step1[5], step1[10]); + step2[6] = _mm256_add_epi16(step1[6], step1[9]); + step2[7] = _mm256_add_epi16(step1[7], step1[8]); + step2[8] = _mm256_sub_epi16(step1[7], step1[8]); + step2[9] = _mm256_sub_epi16(step1[6], step1[9]); + step2[10] = _mm256_sub_epi16(step1[5], step1[10]); + step2[11] = _mm256_sub_epi16(step1[4], step1[11]); + step2[12] = _mm256_sub_epi16(step1[3], step1[12]); + step2[13] = _mm256_sub_epi16(step1[2], step1[13]); + step2[14] = _mm256_sub_epi16(step1[1], step1[14]); + step2[15] = _mm256_sub_epi16(step1[0], step1[15]); + } + { + const __m256i s2_20_0 = _mm256_unpacklo_epi16(step1[27], step1[20]); + const __m256i s2_20_1 = _mm256_unpackhi_epi16(step1[27], step1[20]); + const __m256i s2_21_0 = _mm256_unpacklo_epi16(step1[26], step1[21]); + const __m256i s2_21_1 = _mm256_unpackhi_epi16(step1[26], step1[21]); + const __m256i s2_22_0 = _mm256_unpacklo_epi16(step1[25], step1[22]); + const __m256i s2_22_1 = _mm256_unpackhi_epi16(step1[25], step1[22]); + const __m256i s2_23_0 = _mm256_unpacklo_epi16(step1[24], step1[23]); + const __m256i s2_23_1 = _mm256_unpackhi_epi16(step1[24], step1[23]); + const __m256i s2_20_2 = _mm256_madd_epi16(s2_20_0, k__cospi_p16_m16); + const __m256i s2_20_3 = _mm256_madd_epi16(s2_20_1, k__cospi_p16_m16); + const __m256i s2_21_2 = _mm256_madd_epi16(s2_21_0, k__cospi_p16_m16); + const __m256i s2_21_3 = _mm256_madd_epi16(s2_21_1, k__cospi_p16_m16); + const __m256i s2_22_2 = _mm256_madd_epi16(s2_22_0, k__cospi_p16_m16); + const __m256i s2_22_3 = _mm256_madd_epi16(s2_22_1, k__cospi_p16_m16); + const __m256i s2_23_2 = _mm256_madd_epi16(s2_23_0, k__cospi_p16_m16); + const __m256i s2_23_3 = _mm256_madd_epi16(s2_23_1, k__cospi_p16_m16); + const __m256i s2_24_2 = _mm256_madd_epi16(s2_23_0, k__cospi_p16_p16); + const __m256i s2_24_3 = _mm256_madd_epi16(s2_23_1, k__cospi_p16_p16); + const __m256i s2_25_2 = _mm256_madd_epi16(s2_22_0, k__cospi_p16_p16); + const __m256i s2_25_3 = _mm256_madd_epi16(s2_22_1, k__cospi_p16_p16); + const __m256i s2_26_2 = _mm256_madd_epi16(s2_21_0, k__cospi_p16_p16); + const __m256i s2_26_3 = _mm256_madd_epi16(s2_21_1, k__cospi_p16_p16); + const __m256i s2_27_2 = _mm256_madd_epi16(s2_20_0, k__cospi_p16_p16); + const __m256i s2_27_3 = _mm256_madd_epi16(s2_20_1, k__cospi_p16_p16); + // dct_const_round_shift + const __m256i s2_20_4 = + _mm256_add_epi32(s2_20_2, k__DCT_CONST_ROUNDING); + const __m256i s2_20_5 = + _mm256_add_epi32(s2_20_3, k__DCT_CONST_ROUNDING); + const __m256i s2_21_4 = + _mm256_add_epi32(s2_21_2, k__DCT_CONST_ROUNDING); + const __m256i s2_21_5 = + _mm256_add_epi32(s2_21_3, k__DCT_CONST_ROUNDING); + const __m256i s2_22_4 = + _mm256_add_epi32(s2_22_2, k__DCT_CONST_ROUNDING); + const __m256i s2_22_5 = + _mm256_add_epi32(s2_22_3, k__DCT_CONST_ROUNDING); + const __m256i s2_23_4 = + _mm256_add_epi32(s2_23_2, k__DCT_CONST_ROUNDING); + const __m256i s2_23_5 = + _mm256_add_epi32(s2_23_3, k__DCT_CONST_ROUNDING); + const __m256i s2_24_4 = + _mm256_add_epi32(s2_24_2, k__DCT_CONST_ROUNDING); + const __m256i s2_24_5 = + _mm256_add_epi32(s2_24_3, k__DCT_CONST_ROUNDING); + const __m256i s2_25_4 = + _mm256_add_epi32(s2_25_2, k__DCT_CONST_ROUNDING); + const __m256i s2_25_5 = + _mm256_add_epi32(s2_25_3, k__DCT_CONST_ROUNDING); + const __m256i s2_26_4 = + _mm256_add_epi32(s2_26_2, k__DCT_CONST_ROUNDING); + const __m256i s2_26_5 = + _mm256_add_epi32(s2_26_3, k__DCT_CONST_ROUNDING); + const __m256i s2_27_4 = + _mm256_add_epi32(s2_27_2, k__DCT_CONST_ROUNDING); + const __m256i s2_27_5 = + _mm256_add_epi32(s2_27_3, k__DCT_CONST_ROUNDING); + const __m256i s2_20_6 = _mm256_srai_epi32(s2_20_4, DCT_CONST_BITS); + const __m256i s2_20_7 = _mm256_srai_epi32(s2_20_5, DCT_CONST_BITS); + const __m256i s2_21_6 = _mm256_srai_epi32(s2_21_4, DCT_CONST_BITS); + const __m256i s2_21_7 = _mm256_srai_epi32(s2_21_5, DCT_CONST_BITS); + const __m256i s2_22_6 = _mm256_srai_epi32(s2_22_4, DCT_CONST_BITS); + const __m256i s2_22_7 = _mm256_srai_epi32(s2_22_5, DCT_CONST_BITS); + const __m256i s2_23_6 = _mm256_srai_epi32(s2_23_4, DCT_CONST_BITS); + const __m256i s2_23_7 = _mm256_srai_epi32(s2_23_5, DCT_CONST_BITS); + const __m256i s2_24_6 = _mm256_srai_epi32(s2_24_4, DCT_CONST_BITS); + const __m256i s2_24_7 = _mm256_srai_epi32(s2_24_5, DCT_CONST_BITS); + const __m256i s2_25_6 = _mm256_srai_epi32(s2_25_4, DCT_CONST_BITS); + const __m256i s2_25_7 = _mm256_srai_epi32(s2_25_5, DCT_CONST_BITS); + const __m256i s2_26_6 = _mm256_srai_epi32(s2_26_4, DCT_CONST_BITS); + const __m256i s2_26_7 = _mm256_srai_epi32(s2_26_5, DCT_CONST_BITS); + const __m256i s2_27_6 = _mm256_srai_epi32(s2_27_4, DCT_CONST_BITS); + const __m256i s2_27_7 = _mm256_srai_epi32(s2_27_5, DCT_CONST_BITS); + // Combine + step2[20] = _mm256_packs_epi32(s2_20_6, s2_20_7); + step2[21] = _mm256_packs_epi32(s2_21_6, s2_21_7); + step2[22] = _mm256_packs_epi32(s2_22_6, s2_22_7); + step2[23] = _mm256_packs_epi32(s2_23_6, s2_23_7); + step2[24] = _mm256_packs_epi32(s2_24_6, s2_24_7); + step2[25] = _mm256_packs_epi32(s2_25_6, s2_25_7); + step2[26] = _mm256_packs_epi32(s2_26_6, s2_26_7); + step2[27] = _mm256_packs_epi32(s2_27_6, s2_27_7); + } + +#if !FDCT32x32_HIGH_PRECISION + // dump the magnitude by half, hence the intermediate values are within + // the range of 16 bits. + if (1 == pass) { + __m256i s3_00_0 = _mm256_cmpgt_epi16(kZero, step2[0]); + __m256i s3_01_0 = _mm256_cmpgt_epi16(kZero, step2[1]); + __m256i s3_02_0 = _mm256_cmpgt_epi16(kZero, step2[2]); + __m256i s3_03_0 = _mm256_cmpgt_epi16(kZero, step2[3]); + __m256i s3_04_0 = _mm256_cmpgt_epi16(kZero, step2[4]); + __m256i s3_05_0 = _mm256_cmpgt_epi16(kZero, step2[5]); + __m256i s3_06_0 = _mm256_cmpgt_epi16(kZero, step2[6]); + __m256i s3_07_0 = _mm256_cmpgt_epi16(kZero, step2[7]); + __m256i s2_08_0 = _mm256_cmpgt_epi16(kZero, step2[8]); + __m256i s2_09_0 = _mm256_cmpgt_epi16(kZero, step2[9]); + __m256i s3_10_0 = _mm256_cmpgt_epi16(kZero, step2[10]); + __m256i s3_11_0 = _mm256_cmpgt_epi16(kZero, step2[11]); + __m256i s3_12_0 = _mm256_cmpgt_epi16(kZero, step2[12]); + __m256i s3_13_0 = _mm256_cmpgt_epi16(kZero, step2[13]); + __m256i s2_14_0 = _mm256_cmpgt_epi16(kZero, step2[14]); + __m256i s2_15_0 = _mm256_cmpgt_epi16(kZero, step2[15]); + __m256i s3_16_0 = _mm256_cmpgt_epi16(kZero, step1[16]); + __m256i s3_17_0 = _mm256_cmpgt_epi16(kZero, step1[17]); + __m256i s3_18_0 = _mm256_cmpgt_epi16(kZero, step1[18]); + __m256i s3_19_0 = _mm256_cmpgt_epi16(kZero, step1[19]); + __m256i s3_20_0 = _mm256_cmpgt_epi16(kZero, step2[20]); + __m256i s3_21_0 = _mm256_cmpgt_epi16(kZero, step2[21]); + __m256i s3_22_0 = _mm256_cmpgt_epi16(kZero, step2[22]); + __m256i s3_23_0 = _mm256_cmpgt_epi16(kZero, step2[23]); + __m256i s3_24_0 = _mm256_cmpgt_epi16(kZero, step2[24]); + __m256i s3_25_0 = _mm256_cmpgt_epi16(kZero, step2[25]); + __m256i s3_26_0 = _mm256_cmpgt_epi16(kZero, step2[26]); + __m256i s3_27_0 = _mm256_cmpgt_epi16(kZero, step2[27]); + __m256i s3_28_0 = _mm256_cmpgt_epi16(kZero, step1[28]); + __m256i s3_29_0 = _mm256_cmpgt_epi16(kZero, step1[29]); + __m256i s3_30_0 = _mm256_cmpgt_epi16(kZero, step1[30]); + __m256i s3_31_0 = _mm256_cmpgt_epi16(kZero, step1[31]); + + step2[0] = _mm256_sub_epi16(step2[0], s3_00_0); + step2[1] = _mm256_sub_epi16(step2[1], s3_01_0); + step2[2] = _mm256_sub_epi16(step2[2], s3_02_0); + step2[3] = _mm256_sub_epi16(step2[3], s3_03_0); + step2[4] = _mm256_sub_epi16(step2[4], s3_04_0); + step2[5] = _mm256_sub_epi16(step2[5], s3_05_0); + step2[6] = _mm256_sub_epi16(step2[6], s3_06_0); + step2[7] = _mm256_sub_epi16(step2[7], s3_07_0); + step2[8] = _mm256_sub_epi16(step2[8], s2_08_0); + step2[9] = _mm256_sub_epi16(step2[9], s2_09_0); + step2[10] = _mm256_sub_epi16(step2[10], s3_10_0); + step2[11] = _mm256_sub_epi16(step2[11], s3_11_0); + step2[12] = _mm256_sub_epi16(step2[12], s3_12_0); + step2[13] = _mm256_sub_epi16(step2[13], s3_13_0); + step2[14] = _mm256_sub_epi16(step2[14], s2_14_0); + step2[15] = _mm256_sub_epi16(step2[15], s2_15_0); + step1[16] = _mm256_sub_epi16(step1[16], s3_16_0); + step1[17] = _mm256_sub_epi16(step1[17], s3_17_0); + step1[18] = _mm256_sub_epi16(step1[18], s3_18_0); + step1[19] = _mm256_sub_epi16(step1[19], s3_19_0); + step2[20] = _mm256_sub_epi16(step2[20], s3_20_0); + step2[21] = _mm256_sub_epi16(step2[21], s3_21_0); + step2[22] = _mm256_sub_epi16(step2[22], s3_22_0); + step2[23] = _mm256_sub_epi16(step2[23], s3_23_0); + step2[24] = _mm256_sub_epi16(step2[24], s3_24_0); + step2[25] = _mm256_sub_epi16(step2[25], s3_25_0); + step2[26] = _mm256_sub_epi16(step2[26], s3_26_0); + step2[27] = _mm256_sub_epi16(step2[27], s3_27_0); + step1[28] = _mm256_sub_epi16(step1[28], s3_28_0); + step1[29] = _mm256_sub_epi16(step1[29], s3_29_0); + step1[30] = _mm256_sub_epi16(step1[30], s3_30_0); + step1[31] = _mm256_sub_epi16(step1[31], s3_31_0); + + step2[0] = _mm256_add_epi16(step2[0], kOne); + step2[1] = _mm256_add_epi16(step2[1], kOne); + step2[2] = _mm256_add_epi16(step2[2], kOne); + step2[3] = _mm256_add_epi16(step2[3], kOne); + step2[4] = _mm256_add_epi16(step2[4], kOne); + step2[5] = _mm256_add_epi16(step2[5], kOne); + step2[6] = _mm256_add_epi16(step2[6], kOne); + step2[7] = _mm256_add_epi16(step2[7], kOne); + step2[8] = _mm256_add_epi16(step2[8], kOne); + step2[9] = _mm256_add_epi16(step2[9], kOne); + step2[10] = _mm256_add_epi16(step2[10], kOne); + step2[11] = _mm256_add_epi16(step2[11], kOne); + step2[12] = _mm256_add_epi16(step2[12], kOne); + step2[13] = _mm256_add_epi16(step2[13], kOne); + step2[14] = _mm256_add_epi16(step2[14], kOne); + step2[15] = _mm256_add_epi16(step2[15], kOne); + step1[16] = _mm256_add_epi16(step1[16], kOne); + step1[17] = _mm256_add_epi16(step1[17], kOne); + step1[18] = _mm256_add_epi16(step1[18], kOne); + step1[19] = _mm256_add_epi16(step1[19], kOne); + step2[20] = _mm256_add_epi16(step2[20], kOne); + step2[21] = _mm256_add_epi16(step2[21], kOne); + step2[22] = _mm256_add_epi16(step2[22], kOne); + step2[23] = _mm256_add_epi16(step2[23], kOne); + step2[24] = _mm256_add_epi16(step2[24], kOne); + step2[25] = _mm256_add_epi16(step2[25], kOne); + step2[26] = _mm256_add_epi16(step2[26], kOne); + step2[27] = _mm256_add_epi16(step2[27], kOne); + step1[28] = _mm256_add_epi16(step1[28], kOne); + step1[29] = _mm256_add_epi16(step1[29], kOne); + step1[30] = _mm256_add_epi16(step1[30], kOne); + step1[31] = _mm256_add_epi16(step1[31], kOne); + + step2[0] = _mm256_srai_epi16(step2[0], 2); + step2[1] = _mm256_srai_epi16(step2[1], 2); + step2[2] = _mm256_srai_epi16(step2[2], 2); + step2[3] = _mm256_srai_epi16(step2[3], 2); + step2[4] = _mm256_srai_epi16(step2[4], 2); + step2[5] = _mm256_srai_epi16(step2[5], 2); + step2[6] = _mm256_srai_epi16(step2[6], 2); + step2[7] = _mm256_srai_epi16(step2[7], 2); + step2[8] = _mm256_srai_epi16(step2[8], 2); + step2[9] = _mm256_srai_epi16(step2[9], 2); + step2[10] = _mm256_srai_epi16(step2[10], 2); + step2[11] = _mm256_srai_epi16(step2[11], 2); + step2[12] = _mm256_srai_epi16(step2[12], 2); + step2[13] = _mm256_srai_epi16(step2[13], 2); + step2[14] = _mm256_srai_epi16(step2[14], 2); + step2[15] = _mm256_srai_epi16(step2[15], 2); + step1[16] = _mm256_srai_epi16(step1[16], 2); + step1[17] = _mm256_srai_epi16(step1[17], 2); + step1[18] = _mm256_srai_epi16(step1[18], 2); + step1[19] = _mm256_srai_epi16(step1[19], 2); + step2[20] = _mm256_srai_epi16(step2[20], 2); + step2[21] = _mm256_srai_epi16(step2[21], 2); + step2[22] = _mm256_srai_epi16(step2[22], 2); + step2[23] = _mm256_srai_epi16(step2[23], 2); + step2[24] = _mm256_srai_epi16(step2[24], 2); + step2[25] = _mm256_srai_epi16(step2[25], 2); + step2[26] = _mm256_srai_epi16(step2[26], 2); + step2[27] = _mm256_srai_epi16(step2[27], 2); + step1[28] = _mm256_srai_epi16(step1[28], 2); + step1[29] = _mm256_srai_epi16(step1[29], 2); + step1[30] = _mm256_srai_epi16(step1[30], 2); + step1[31] = _mm256_srai_epi16(step1[31], 2); + } +#endif + +#if FDCT32x32_HIGH_PRECISION + if (pass == 0) { +#endif + // Stage 3 + { + step3[0] = _mm256_add_epi16(step2[(8 - 1)], step2[0]); + step3[1] = _mm256_add_epi16(step2[(8 - 2)], step2[1]); + step3[2] = _mm256_add_epi16(step2[(8 - 3)], step2[2]); + step3[3] = _mm256_add_epi16(step2[(8 - 4)], step2[3]); + step3[4] = _mm256_sub_epi16(step2[(8 - 5)], step2[4]); + step3[5] = _mm256_sub_epi16(step2[(8 - 6)], step2[5]); + step3[6] = _mm256_sub_epi16(step2[(8 - 7)], step2[6]); + step3[7] = _mm256_sub_epi16(step2[(8 - 8)], step2[7]); + } + { + const __m256i s3_10_0 = _mm256_unpacklo_epi16(step2[13], step2[10]); + const __m256i s3_10_1 = _mm256_unpackhi_epi16(step2[13], step2[10]); + const __m256i s3_11_0 = _mm256_unpacklo_epi16(step2[12], step2[11]); + const __m256i s3_11_1 = _mm256_unpackhi_epi16(step2[12], step2[11]); + const __m256i s3_10_2 = _mm256_madd_epi16(s3_10_0, k__cospi_p16_m16); + const __m256i s3_10_3 = _mm256_madd_epi16(s3_10_1, k__cospi_p16_m16); + const __m256i s3_11_2 = _mm256_madd_epi16(s3_11_0, k__cospi_p16_m16); + const __m256i s3_11_3 = _mm256_madd_epi16(s3_11_1, k__cospi_p16_m16); + const __m256i s3_12_2 = _mm256_madd_epi16(s3_11_0, k__cospi_p16_p16); + const __m256i s3_12_3 = _mm256_madd_epi16(s3_11_1, k__cospi_p16_p16); + const __m256i s3_13_2 = _mm256_madd_epi16(s3_10_0, k__cospi_p16_p16); + const __m256i s3_13_3 = _mm256_madd_epi16(s3_10_1, k__cospi_p16_p16); + // dct_const_round_shift + const __m256i s3_10_4 = + _mm256_add_epi32(s3_10_2, k__DCT_CONST_ROUNDING); + const __m256i s3_10_5 = + _mm256_add_epi32(s3_10_3, k__DCT_CONST_ROUNDING); + const __m256i s3_11_4 = + _mm256_add_epi32(s3_11_2, k__DCT_CONST_ROUNDING); + const __m256i s3_11_5 = + _mm256_add_epi32(s3_11_3, k__DCT_CONST_ROUNDING); + const __m256i s3_12_4 = + _mm256_add_epi32(s3_12_2, k__DCT_CONST_ROUNDING); + const __m256i s3_12_5 = + _mm256_add_epi32(s3_12_3, k__DCT_CONST_ROUNDING); + const __m256i s3_13_4 = + _mm256_add_epi32(s3_13_2, k__DCT_CONST_ROUNDING); + const __m256i s3_13_5 = + _mm256_add_epi32(s3_13_3, k__DCT_CONST_ROUNDING); + const __m256i s3_10_6 = _mm256_srai_epi32(s3_10_4, DCT_CONST_BITS); + const __m256i s3_10_7 = _mm256_srai_epi32(s3_10_5, DCT_CONST_BITS); + const __m256i s3_11_6 = _mm256_srai_epi32(s3_11_4, DCT_CONST_BITS); + const __m256i s3_11_7 = _mm256_srai_epi32(s3_11_5, DCT_CONST_BITS); + const __m256i s3_12_6 = _mm256_srai_epi32(s3_12_4, DCT_CONST_BITS); + const __m256i s3_12_7 = _mm256_srai_epi32(s3_12_5, DCT_CONST_BITS); + const __m256i s3_13_6 = _mm256_srai_epi32(s3_13_4, DCT_CONST_BITS); + const __m256i s3_13_7 = _mm256_srai_epi32(s3_13_5, DCT_CONST_BITS); + // Combine + step3[10] = _mm256_packs_epi32(s3_10_6, s3_10_7); + step3[11] = _mm256_packs_epi32(s3_11_6, s3_11_7); + step3[12] = _mm256_packs_epi32(s3_12_6, s3_12_7); + step3[13] = _mm256_packs_epi32(s3_13_6, s3_13_7); + } + { + step3[16] = _mm256_add_epi16(step2[23], step1[16]); + step3[17] = _mm256_add_epi16(step2[22], step1[17]); + step3[18] = _mm256_add_epi16(step2[21], step1[18]); + step3[19] = _mm256_add_epi16(step2[20], step1[19]); + step3[20] = _mm256_sub_epi16(step1[19], step2[20]); + step3[21] = _mm256_sub_epi16(step1[18], step2[21]); + step3[22] = _mm256_sub_epi16(step1[17], step2[22]); + step3[23] = _mm256_sub_epi16(step1[16], step2[23]); + step3[24] = _mm256_sub_epi16(step1[31], step2[24]); + step3[25] = _mm256_sub_epi16(step1[30], step2[25]); + step3[26] = _mm256_sub_epi16(step1[29], step2[26]); + step3[27] = _mm256_sub_epi16(step1[28], step2[27]); + step3[28] = _mm256_add_epi16(step2[27], step1[28]); + step3[29] = _mm256_add_epi16(step2[26], step1[29]); + step3[30] = _mm256_add_epi16(step2[25], step1[30]); + step3[31] = _mm256_add_epi16(step2[24], step1[31]); + } + + // Stage 4 + { + step1[0] = _mm256_add_epi16(step3[3], step3[0]); + step1[1] = _mm256_add_epi16(step3[2], step3[1]); + step1[2] = _mm256_sub_epi16(step3[1], step3[2]); + step1[3] = _mm256_sub_epi16(step3[0], step3[3]); + step1[8] = _mm256_add_epi16(step3[11], step2[8]); + step1[9] = _mm256_add_epi16(step3[10], step2[9]); + step1[10] = _mm256_sub_epi16(step2[9], step3[10]); + step1[11] = _mm256_sub_epi16(step2[8], step3[11]); + step1[12] = _mm256_sub_epi16(step2[15], step3[12]); + step1[13] = _mm256_sub_epi16(step2[14], step3[13]); + step1[14] = _mm256_add_epi16(step3[13], step2[14]); + step1[15] = _mm256_add_epi16(step3[12], step2[15]); + } + { + const __m256i s1_05_0 = _mm256_unpacklo_epi16(step3[6], step3[5]); + const __m256i s1_05_1 = _mm256_unpackhi_epi16(step3[6], step3[5]); + const __m256i s1_05_2 = _mm256_madd_epi16(s1_05_0, k__cospi_p16_m16); + const __m256i s1_05_3 = _mm256_madd_epi16(s1_05_1, k__cospi_p16_m16); + const __m256i s1_06_2 = _mm256_madd_epi16(s1_05_0, k__cospi_p16_p16); + const __m256i s1_06_3 = _mm256_madd_epi16(s1_05_1, k__cospi_p16_p16); + // dct_const_round_shift + const __m256i s1_05_4 = + _mm256_add_epi32(s1_05_2, k__DCT_CONST_ROUNDING); + const __m256i s1_05_5 = + _mm256_add_epi32(s1_05_3, k__DCT_CONST_ROUNDING); + const __m256i s1_06_4 = + _mm256_add_epi32(s1_06_2, k__DCT_CONST_ROUNDING); + const __m256i s1_06_5 = + _mm256_add_epi32(s1_06_3, k__DCT_CONST_ROUNDING); + const __m256i s1_05_6 = _mm256_srai_epi32(s1_05_4, DCT_CONST_BITS); + const __m256i s1_05_7 = _mm256_srai_epi32(s1_05_5, DCT_CONST_BITS); + const __m256i s1_06_6 = _mm256_srai_epi32(s1_06_4, DCT_CONST_BITS); + const __m256i s1_06_7 = _mm256_srai_epi32(s1_06_5, DCT_CONST_BITS); + // Combine + step1[5] = _mm256_packs_epi32(s1_05_6, s1_05_7); + step1[6] = _mm256_packs_epi32(s1_06_6, s1_06_7); + } + { + const __m256i s1_18_0 = _mm256_unpacklo_epi16(step3[18], step3[29]); + const __m256i s1_18_1 = _mm256_unpackhi_epi16(step3[18], step3[29]); + const __m256i s1_19_0 = _mm256_unpacklo_epi16(step3[19], step3[28]); + const __m256i s1_19_1 = _mm256_unpackhi_epi16(step3[19], step3[28]); + const __m256i s1_20_0 = _mm256_unpacklo_epi16(step3[20], step3[27]); + const __m256i s1_20_1 = _mm256_unpackhi_epi16(step3[20], step3[27]); + const __m256i s1_21_0 = _mm256_unpacklo_epi16(step3[21], step3[26]); + const __m256i s1_21_1 = _mm256_unpackhi_epi16(step3[21], step3[26]); + const __m256i s1_18_2 = _mm256_madd_epi16(s1_18_0, k__cospi_m08_p24); + const __m256i s1_18_3 = _mm256_madd_epi16(s1_18_1, k__cospi_m08_p24); + const __m256i s1_19_2 = _mm256_madd_epi16(s1_19_0, k__cospi_m08_p24); + const __m256i s1_19_3 = _mm256_madd_epi16(s1_19_1, k__cospi_m08_p24); + const __m256i s1_20_2 = _mm256_madd_epi16(s1_20_0, k__cospi_m24_m08); + const __m256i s1_20_3 = _mm256_madd_epi16(s1_20_1, k__cospi_m24_m08); + const __m256i s1_21_2 = _mm256_madd_epi16(s1_21_0, k__cospi_m24_m08); + const __m256i s1_21_3 = _mm256_madd_epi16(s1_21_1, k__cospi_m24_m08); + const __m256i s1_26_2 = _mm256_madd_epi16(s1_21_0, k__cospi_m08_p24); + const __m256i s1_26_3 = _mm256_madd_epi16(s1_21_1, k__cospi_m08_p24); + const __m256i s1_27_2 = _mm256_madd_epi16(s1_20_0, k__cospi_m08_p24); + const __m256i s1_27_3 = _mm256_madd_epi16(s1_20_1, k__cospi_m08_p24); + const __m256i s1_28_2 = _mm256_madd_epi16(s1_19_0, k__cospi_p24_p08); + const __m256i s1_28_3 = _mm256_madd_epi16(s1_19_1, k__cospi_p24_p08); + const __m256i s1_29_2 = _mm256_madd_epi16(s1_18_0, k__cospi_p24_p08); + const __m256i s1_29_3 = _mm256_madd_epi16(s1_18_1, k__cospi_p24_p08); + // dct_const_round_shift + const __m256i s1_18_4 = + _mm256_add_epi32(s1_18_2, k__DCT_CONST_ROUNDING); + const __m256i s1_18_5 = + _mm256_add_epi32(s1_18_3, k__DCT_CONST_ROUNDING); + const __m256i s1_19_4 = + _mm256_add_epi32(s1_19_2, k__DCT_CONST_ROUNDING); + const __m256i s1_19_5 = + _mm256_add_epi32(s1_19_3, k__DCT_CONST_ROUNDING); + const __m256i s1_20_4 = + _mm256_add_epi32(s1_20_2, k__DCT_CONST_ROUNDING); + const __m256i s1_20_5 = + _mm256_add_epi32(s1_20_3, k__DCT_CONST_ROUNDING); + const __m256i s1_21_4 = + _mm256_add_epi32(s1_21_2, k__DCT_CONST_ROUNDING); + const __m256i s1_21_5 = + _mm256_add_epi32(s1_21_3, k__DCT_CONST_ROUNDING); + const __m256i s1_26_4 = + _mm256_add_epi32(s1_26_2, k__DCT_CONST_ROUNDING); + const __m256i s1_26_5 = + _mm256_add_epi32(s1_26_3, k__DCT_CONST_ROUNDING); + const __m256i s1_27_4 = + _mm256_add_epi32(s1_27_2, k__DCT_CONST_ROUNDING); + const __m256i s1_27_5 = + _mm256_add_epi32(s1_27_3, k__DCT_CONST_ROUNDING); + const __m256i s1_28_4 = + _mm256_add_epi32(s1_28_2, k__DCT_CONST_ROUNDING); + const __m256i s1_28_5 = + _mm256_add_epi32(s1_28_3, k__DCT_CONST_ROUNDING); + const __m256i s1_29_4 = + _mm256_add_epi32(s1_29_2, k__DCT_CONST_ROUNDING); + const __m256i s1_29_5 = + _mm256_add_epi32(s1_29_3, k__DCT_CONST_ROUNDING); + const __m256i s1_18_6 = _mm256_srai_epi32(s1_18_4, DCT_CONST_BITS); + const __m256i s1_18_7 = _mm256_srai_epi32(s1_18_5, DCT_CONST_BITS); + const __m256i s1_19_6 = _mm256_srai_epi32(s1_19_4, DCT_CONST_BITS); + const __m256i s1_19_7 = _mm256_srai_epi32(s1_19_5, DCT_CONST_BITS); + const __m256i s1_20_6 = _mm256_srai_epi32(s1_20_4, DCT_CONST_BITS); + const __m256i s1_20_7 = _mm256_srai_epi32(s1_20_5, DCT_CONST_BITS); + const __m256i s1_21_6 = _mm256_srai_epi32(s1_21_4, DCT_CONST_BITS); + const __m256i s1_21_7 = _mm256_srai_epi32(s1_21_5, DCT_CONST_BITS); + const __m256i s1_26_6 = _mm256_srai_epi32(s1_26_4, DCT_CONST_BITS); + const __m256i s1_26_7 = _mm256_srai_epi32(s1_26_5, DCT_CONST_BITS); + const __m256i s1_27_6 = _mm256_srai_epi32(s1_27_4, DCT_CONST_BITS); + const __m256i s1_27_7 = _mm256_srai_epi32(s1_27_5, DCT_CONST_BITS); + const __m256i s1_28_6 = _mm256_srai_epi32(s1_28_4, DCT_CONST_BITS); + const __m256i s1_28_7 = _mm256_srai_epi32(s1_28_5, DCT_CONST_BITS); + const __m256i s1_29_6 = _mm256_srai_epi32(s1_29_4, DCT_CONST_BITS); + const __m256i s1_29_7 = _mm256_srai_epi32(s1_29_5, DCT_CONST_BITS); + // Combine + step1[18] = _mm256_packs_epi32(s1_18_6, s1_18_7); + step1[19] = _mm256_packs_epi32(s1_19_6, s1_19_7); + step1[20] = _mm256_packs_epi32(s1_20_6, s1_20_7); + step1[21] = _mm256_packs_epi32(s1_21_6, s1_21_7); + step1[26] = _mm256_packs_epi32(s1_26_6, s1_26_7); + step1[27] = _mm256_packs_epi32(s1_27_6, s1_27_7); + step1[28] = _mm256_packs_epi32(s1_28_6, s1_28_7); + step1[29] = _mm256_packs_epi32(s1_29_6, s1_29_7); + } + // Stage 5 + { + step2[4] = _mm256_add_epi16(step1[5], step3[4]); + step2[5] = _mm256_sub_epi16(step3[4], step1[5]); + step2[6] = _mm256_sub_epi16(step3[7], step1[6]); + step2[7] = _mm256_add_epi16(step1[6], step3[7]); + } + { + const __m256i out_00_0 = _mm256_unpacklo_epi16(step1[0], step1[1]); + const __m256i out_00_1 = _mm256_unpackhi_epi16(step1[0], step1[1]); + const __m256i out_08_0 = _mm256_unpacklo_epi16(step1[2], step1[3]); + const __m256i out_08_1 = _mm256_unpackhi_epi16(step1[2], step1[3]); + const __m256i out_00_2 = + _mm256_madd_epi16(out_00_0, k__cospi_p16_p16); + const __m256i out_00_3 = + _mm256_madd_epi16(out_00_1, k__cospi_p16_p16); + const __m256i out_16_2 = + _mm256_madd_epi16(out_00_0, k__cospi_p16_m16); + const __m256i out_16_3 = + _mm256_madd_epi16(out_00_1, k__cospi_p16_m16); + const __m256i out_08_2 = + _mm256_madd_epi16(out_08_0, k__cospi_p24_p08); + const __m256i out_08_3 = + _mm256_madd_epi16(out_08_1, k__cospi_p24_p08); + const __m256i out_24_2 = + _mm256_madd_epi16(out_08_0, k__cospi_m08_p24); + const __m256i out_24_3 = + _mm256_madd_epi16(out_08_1, k__cospi_m08_p24); + // dct_const_round_shift + const __m256i out_00_4 = + _mm256_add_epi32(out_00_2, k__DCT_CONST_ROUNDING); + const __m256i out_00_5 = + _mm256_add_epi32(out_00_3, k__DCT_CONST_ROUNDING); + const __m256i out_16_4 = + _mm256_add_epi32(out_16_2, k__DCT_CONST_ROUNDING); + const __m256i out_16_5 = + _mm256_add_epi32(out_16_3, k__DCT_CONST_ROUNDING); + const __m256i out_08_4 = + _mm256_add_epi32(out_08_2, k__DCT_CONST_ROUNDING); + const __m256i out_08_5 = + _mm256_add_epi32(out_08_3, k__DCT_CONST_ROUNDING); + const __m256i out_24_4 = + _mm256_add_epi32(out_24_2, k__DCT_CONST_ROUNDING); + const __m256i out_24_5 = + _mm256_add_epi32(out_24_3, k__DCT_CONST_ROUNDING); + const __m256i out_00_6 = _mm256_srai_epi32(out_00_4, DCT_CONST_BITS); + const __m256i out_00_7 = _mm256_srai_epi32(out_00_5, DCT_CONST_BITS); + const __m256i out_16_6 = _mm256_srai_epi32(out_16_4, DCT_CONST_BITS); + const __m256i out_16_7 = _mm256_srai_epi32(out_16_5, DCT_CONST_BITS); + const __m256i out_08_6 = _mm256_srai_epi32(out_08_4, DCT_CONST_BITS); + const __m256i out_08_7 = _mm256_srai_epi32(out_08_5, DCT_CONST_BITS); + const __m256i out_24_6 = _mm256_srai_epi32(out_24_4, DCT_CONST_BITS); + const __m256i out_24_7 = _mm256_srai_epi32(out_24_5, DCT_CONST_BITS); + // Combine + out[0] = _mm256_packs_epi32(out_00_6, out_00_7); + out[16] = _mm256_packs_epi32(out_16_6, out_16_7); + out[8] = _mm256_packs_epi32(out_08_6, out_08_7); + out[24] = _mm256_packs_epi32(out_24_6, out_24_7); + } + { + const __m256i s2_09_0 = _mm256_unpacklo_epi16(step1[9], step1[14]); + const __m256i s2_09_1 = _mm256_unpackhi_epi16(step1[9], step1[14]); + const __m256i s2_10_0 = _mm256_unpacklo_epi16(step1[10], step1[13]); + const __m256i s2_10_1 = _mm256_unpackhi_epi16(step1[10], step1[13]); + const __m256i s2_09_2 = _mm256_madd_epi16(s2_09_0, k__cospi_m08_p24); + const __m256i s2_09_3 = _mm256_madd_epi16(s2_09_1, k__cospi_m08_p24); + const __m256i s2_10_2 = _mm256_madd_epi16(s2_10_0, k__cospi_m24_m08); + const __m256i s2_10_3 = _mm256_madd_epi16(s2_10_1, k__cospi_m24_m08); + const __m256i s2_13_2 = _mm256_madd_epi16(s2_10_0, k__cospi_m08_p24); + const __m256i s2_13_3 = _mm256_madd_epi16(s2_10_1, k__cospi_m08_p24); + const __m256i s2_14_2 = _mm256_madd_epi16(s2_09_0, k__cospi_p24_p08); + const __m256i s2_14_3 = _mm256_madd_epi16(s2_09_1, k__cospi_p24_p08); + // dct_const_round_shift + const __m256i s2_09_4 = + _mm256_add_epi32(s2_09_2, k__DCT_CONST_ROUNDING); + const __m256i s2_09_5 = + _mm256_add_epi32(s2_09_3, k__DCT_CONST_ROUNDING); + const __m256i s2_10_4 = + _mm256_add_epi32(s2_10_2, k__DCT_CONST_ROUNDING); + const __m256i s2_10_5 = + _mm256_add_epi32(s2_10_3, k__DCT_CONST_ROUNDING); + const __m256i s2_13_4 = + _mm256_add_epi32(s2_13_2, k__DCT_CONST_ROUNDING); + const __m256i s2_13_5 = + _mm256_add_epi32(s2_13_3, k__DCT_CONST_ROUNDING); + const __m256i s2_14_4 = + _mm256_add_epi32(s2_14_2, k__DCT_CONST_ROUNDING); + const __m256i s2_14_5 = + _mm256_add_epi32(s2_14_3, k__DCT_CONST_ROUNDING); + const __m256i s2_09_6 = _mm256_srai_epi32(s2_09_4, DCT_CONST_BITS); + const __m256i s2_09_7 = _mm256_srai_epi32(s2_09_5, DCT_CONST_BITS); + const __m256i s2_10_6 = _mm256_srai_epi32(s2_10_4, DCT_CONST_BITS); + const __m256i s2_10_7 = _mm256_srai_epi32(s2_10_5, DCT_CONST_BITS); + const __m256i s2_13_6 = _mm256_srai_epi32(s2_13_4, DCT_CONST_BITS); + const __m256i s2_13_7 = _mm256_srai_epi32(s2_13_5, DCT_CONST_BITS); + const __m256i s2_14_6 = _mm256_srai_epi32(s2_14_4, DCT_CONST_BITS); + const __m256i s2_14_7 = _mm256_srai_epi32(s2_14_5, DCT_CONST_BITS); + // Combine + step2[9] = _mm256_packs_epi32(s2_09_6, s2_09_7); + step2[10] = _mm256_packs_epi32(s2_10_6, s2_10_7); + step2[13] = _mm256_packs_epi32(s2_13_6, s2_13_7); + step2[14] = _mm256_packs_epi32(s2_14_6, s2_14_7); + } + { + step2[16] = _mm256_add_epi16(step1[19], step3[16]); + step2[17] = _mm256_add_epi16(step1[18], step3[17]); + step2[18] = _mm256_sub_epi16(step3[17], step1[18]); + step2[19] = _mm256_sub_epi16(step3[16], step1[19]); + step2[20] = _mm256_sub_epi16(step3[23], step1[20]); + step2[21] = _mm256_sub_epi16(step3[22], step1[21]); + step2[22] = _mm256_add_epi16(step1[21], step3[22]); + step2[23] = _mm256_add_epi16(step1[20], step3[23]); + step2[24] = _mm256_add_epi16(step1[27], step3[24]); + step2[25] = _mm256_add_epi16(step1[26], step3[25]); + step2[26] = _mm256_sub_epi16(step3[25], step1[26]); + step2[27] = _mm256_sub_epi16(step3[24], step1[27]); + step2[28] = _mm256_sub_epi16(step3[31], step1[28]); + step2[29] = _mm256_sub_epi16(step3[30], step1[29]); + step2[30] = _mm256_add_epi16(step1[29], step3[30]); + step2[31] = _mm256_add_epi16(step1[28], step3[31]); + } + // Stage 6 + { + const __m256i out_04_0 = _mm256_unpacklo_epi16(step2[4], step2[7]); + const __m256i out_04_1 = _mm256_unpackhi_epi16(step2[4], step2[7]); + const __m256i out_20_0 = _mm256_unpacklo_epi16(step2[5], step2[6]); + const __m256i out_20_1 = _mm256_unpackhi_epi16(step2[5], step2[6]); + const __m256i out_12_0 = _mm256_unpacklo_epi16(step2[5], step2[6]); + const __m256i out_12_1 = _mm256_unpackhi_epi16(step2[5], step2[6]); + const __m256i out_28_0 = _mm256_unpacklo_epi16(step2[4], step2[7]); + const __m256i out_28_1 = _mm256_unpackhi_epi16(step2[4], step2[7]); + const __m256i out_04_2 = + _mm256_madd_epi16(out_04_0, k__cospi_p28_p04); + const __m256i out_04_3 = + _mm256_madd_epi16(out_04_1, k__cospi_p28_p04); + const __m256i out_20_2 = + _mm256_madd_epi16(out_20_0, k__cospi_p12_p20); + const __m256i out_20_3 = + _mm256_madd_epi16(out_20_1, k__cospi_p12_p20); + const __m256i out_12_2 = + _mm256_madd_epi16(out_12_0, k__cospi_m20_p12); + const __m256i out_12_3 = + _mm256_madd_epi16(out_12_1, k__cospi_m20_p12); + const __m256i out_28_2 = + _mm256_madd_epi16(out_28_0, k__cospi_m04_p28); + const __m256i out_28_3 = + _mm256_madd_epi16(out_28_1, k__cospi_m04_p28); + // dct_const_round_shift + const __m256i out_04_4 = + _mm256_add_epi32(out_04_2, k__DCT_CONST_ROUNDING); + const __m256i out_04_5 = + _mm256_add_epi32(out_04_3, k__DCT_CONST_ROUNDING); + const __m256i out_20_4 = + _mm256_add_epi32(out_20_2, k__DCT_CONST_ROUNDING); + const __m256i out_20_5 = + _mm256_add_epi32(out_20_3, k__DCT_CONST_ROUNDING); + const __m256i out_12_4 = + _mm256_add_epi32(out_12_2, k__DCT_CONST_ROUNDING); + const __m256i out_12_5 = + _mm256_add_epi32(out_12_3, k__DCT_CONST_ROUNDING); + const __m256i out_28_4 = + _mm256_add_epi32(out_28_2, k__DCT_CONST_ROUNDING); + const __m256i out_28_5 = + _mm256_add_epi32(out_28_3, k__DCT_CONST_ROUNDING); + const __m256i out_04_6 = _mm256_srai_epi32(out_04_4, DCT_CONST_BITS); + const __m256i out_04_7 = _mm256_srai_epi32(out_04_5, DCT_CONST_BITS); + const __m256i out_20_6 = _mm256_srai_epi32(out_20_4, DCT_CONST_BITS); + const __m256i out_20_7 = _mm256_srai_epi32(out_20_5, DCT_CONST_BITS); + const __m256i out_12_6 = _mm256_srai_epi32(out_12_4, DCT_CONST_BITS); + const __m256i out_12_7 = _mm256_srai_epi32(out_12_5, DCT_CONST_BITS); + const __m256i out_28_6 = _mm256_srai_epi32(out_28_4, DCT_CONST_BITS); + const __m256i out_28_7 = _mm256_srai_epi32(out_28_5, DCT_CONST_BITS); + // Combine + out[4] = _mm256_packs_epi32(out_04_6, out_04_7); + out[20] = _mm256_packs_epi32(out_20_6, out_20_7); + out[12] = _mm256_packs_epi32(out_12_6, out_12_7); + out[28] = _mm256_packs_epi32(out_28_6, out_28_7); + } + { + step3[8] = _mm256_add_epi16(step2[9], step1[8]); + step3[9] = _mm256_sub_epi16(step1[8], step2[9]); + step3[10] = _mm256_sub_epi16(step1[11], step2[10]); + step3[11] = _mm256_add_epi16(step2[10], step1[11]); + step3[12] = _mm256_add_epi16(step2[13], step1[12]); + step3[13] = _mm256_sub_epi16(step1[12], step2[13]); + step3[14] = _mm256_sub_epi16(step1[15], step2[14]); + step3[15] = _mm256_add_epi16(step2[14], step1[15]); + } + { + const __m256i s3_17_0 = _mm256_unpacklo_epi16(step2[17], step2[30]); + const __m256i s3_17_1 = _mm256_unpackhi_epi16(step2[17], step2[30]); + const __m256i s3_18_0 = _mm256_unpacklo_epi16(step2[18], step2[29]); + const __m256i s3_18_1 = _mm256_unpackhi_epi16(step2[18], step2[29]); + const __m256i s3_21_0 = _mm256_unpacklo_epi16(step2[21], step2[26]); + const __m256i s3_21_1 = _mm256_unpackhi_epi16(step2[21], step2[26]); + const __m256i s3_22_0 = _mm256_unpacklo_epi16(step2[22], step2[25]); + const __m256i s3_22_1 = _mm256_unpackhi_epi16(step2[22], step2[25]); + const __m256i s3_17_2 = _mm256_madd_epi16(s3_17_0, k__cospi_m04_p28); + const __m256i s3_17_3 = _mm256_madd_epi16(s3_17_1, k__cospi_m04_p28); + const __m256i s3_18_2 = _mm256_madd_epi16(s3_18_0, k__cospi_m28_m04); + const __m256i s3_18_3 = _mm256_madd_epi16(s3_18_1, k__cospi_m28_m04); + const __m256i s3_21_2 = _mm256_madd_epi16(s3_21_0, k__cospi_m20_p12); + const __m256i s3_21_3 = _mm256_madd_epi16(s3_21_1, k__cospi_m20_p12); + const __m256i s3_22_2 = _mm256_madd_epi16(s3_22_0, k__cospi_m12_m20); + const __m256i s3_22_3 = _mm256_madd_epi16(s3_22_1, k__cospi_m12_m20); + const __m256i s3_25_2 = _mm256_madd_epi16(s3_22_0, k__cospi_m20_p12); + const __m256i s3_25_3 = _mm256_madd_epi16(s3_22_1, k__cospi_m20_p12); + const __m256i s3_26_2 = _mm256_madd_epi16(s3_21_0, k__cospi_p12_p20); + const __m256i s3_26_3 = _mm256_madd_epi16(s3_21_1, k__cospi_p12_p20); + const __m256i s3_29_2 = _mm256_madd_epi16(s3_18_0, k__cospi_m04_p28); + const __m256i s3_29_3 = _mm256_madd_epi16(s3_18_1, k__cospi_m04_p28); + const __m256i s3_30_2 = _mm256_madd_epi16(s3_17_0, k__cospi_p28_p04); + const __m256i s3_30_3 = _mm256_madd_epi16(s3_17_1, k__cospi_p28_p04); + // dct_const_round_shift + const __m256i s3_17_4 = + _mm256_add_epi32(s3_17_2, k__DCT_CONST_ROUNDING); + const __m256i s3_17_5 = + _mm256_add_epi32(s3_17_3, k__DCT_CONST_ROUNDING); + const __m256i s3_18_4 = + _mm256_add_epi32(s3_18_2, k__DCT_CONST_ROUNDING); + const __m256i s3_18_5 = + _mm256_add_epi32(s3_18_3, k__DCT_CONST_ROUNDING); + const __m256i s3_21_4 = + _mm256_add_epi32(s3_21_2, k__DCT_CONST_ROUNDING); + const __m256i s3_21_5 = + _mm256_add_epi32(s3_21_3, k__DCT_CONST_ROUNDING); + const __m256i s3_22_4 = + _mm256_add_epi32(s3_22_2, k__DCT_CONST_ROUNDING); + const __m256i s3_22_5 = + _mm256_add_epi32(s3_22_3, k__DCT_CONST_ROUNDING); + const __m256i s3_17_6 = _mm256_srai_epi32(s3_17_4, DCT_CONST_BITS); + const __m256i s3_17_7 = _mm256_srai_epi32(s3_17_5, DCT_CONST_BITS); + const __m256i s3_18_6 = _mm256_srai_epi32(s3_18_4, DCT_CONST_BITS); + const __m256i s3_18_7 = _mm256_srai_epi32(s3_18_5, DCT_CONST_BITS); + const __m256i s3_21_6 = _mm256_srai_epi32(s3_21_4, DCT_CONST_BITS); + const __m256i s3_21_7 = _mm256_srai_epi32(s3_21_5, DCT_CONST_BITS); + const __m256i s3_22_6 = _mm256_srai_epi32(s3_22_4, DCT_CONST_BITS); + const __m256i s3_22_7 = _mm256_srai_epi32(s3_22_5, DCT_CONST_BITS); + const __m256i s3_25_4 = + _mm256_add_epi32(s3_25_2, k__DCT_CONST_ROUNDING); + const __m256i s3_25_5 = + _mm256_add_epi32(s3_25_3, k__DCT_CONST_ROUNDING); + const __m256i s3_26_4 = + _mm256_add_epi32(s3_26_2, k__DCT_CONST_ROUNDING); + const __m256i s3_26_5 = + _mm256_add_epi32(s3_26_3, k__DCT_CONST_ROUNDING); + const __m256i s3_29_4 = + _mm256_add_epi32(s3_29_2, k__DCT_CONST_ROUNDING); + const __m256i s3_29_5 = + _mm256_add_epi32(s3_29_3, k__DCT_CONST_ROUNDING); + const __m256i s3_30_4 = + _mm256_add_epi32(s3_30_2, k__DCT_CONST_ROUNDING); + const __m256i s3_30_5 = + _mm256_add_epi32(s3_30_3, k__DCT_CONST_ROUNDING); + const __m256i s3_25_6 = _mm256_srai_epi32(s3_25_4, DCT_CONST_BITS); + const __m256i s3_25_7 = _mm256_srai_epi32(s3_25_5, DCT_CONST_BITS); + const __m256i s3_26_6 = _mm256_srai_epi32(s3_26_4, DCT_CONST_BITS); + const __m256i s3_26_7 = _mm256_srai_epi32(s3_26_5, DCT_CONST_BITS); + const __m256i s3_29_6 = _mm256_srai_epi32(s3_29_4, DCT_CONST_BITS); + const __m256i s3_29_7 = _mm256_srai_epi32(s3_29_5, DCT_CONST_BITS); + const __m256i s3_30_6 = _mm256_srai_epi32(s3_30_4, DCT_CONST_BITS); + const __m256i s3_30_7 = _mm256_srai_epi32(s3_30_5, DCT_CONST_BITS); + // Combine + step3[17] = _mm256_packs_epi32(s3_17_6, s3_17_7); + step3[18] = _mm256_packs_epi32(s3_18_6, s3_18_7); + step3[21] = _mm256_packs_epi32(s3_21_6, s3_21_7); + step3[22] = _mm256_packs_epi32(s3_22_6, s3_22_7); + // Combine + step3[25] = _mm256_packs_epi32(s3_25_6, s3_25_7); + step3[26] = _mm256_packs_epi32(s3_26_6, s3_26_7); + step3[29] = _mm256_packs_epi32(s3_29_6, s3_29_7); + step3[30] = _mm256_packs_epi32(s3_30_6, s3_30_7); + } + // Stage 7 + { + const __m256i out_02_0 = _mm256_unpacklo_epi16(step3[8], step3[15]); + const __m256i out_02_1 = _mm256_unpackhi_epi16(step3[8], step3[15]); + const __m256i out_18_0 = _mm256_unpacklo_epi16(step3[9], step3[14]); + const __m256i out_18_1 = _mm256_unpackhi_epi16(step3[9], step3[14]); + const __m256i out_10_0 = _mm256_unpacklo_epi16(step3[10], step3[13]); + const __m256i out_10_1 = _mm256_unpackhi_epi16(step3[10], step3[13]); + const __m256i out_26_0 = _mm256_unpacklo_epi16(step3[11], step3[12]); + const __m256i out_26_1 = _mm256_unpackhi_epi16(step3[11], step3[12]); + const __m256i out_02_2 = + _mm256_madd_epi16(out_02_0, k__cospi_p30_p02); + const __m256i out_02_3 = + _mm256_madd_epi16(out_02_1, k__cospi_p30_p02); + const __m256i out_18_2 = + _mm256_madd_epi16(out_18_0, k__cospi_p14_p18); + const __m256i out_18_3 = + _mm256_madd_epi16(out_18_1, k__cospi_p14_p18); + const __m256i out_10_2 = + _mm256_madd_epi16(out_10_0, k__cospi_p22_p10); + const __m256i out_10_3 = + _mm256_madd_epi16(out_10_1, k__cospi_p22_p10); + const __m256i out_26_2 = + _mm256_madd_epi16(out_26_0, k__cospi_p06_p26); + const __m256i out_26_3 = + _mm256_madd_epi16(out_26_1, k__cospi_p06_p26); + const __m256i out_06_2 = + _mm256_madd_epi16(out_26_0, k__cospi_m26_p06); + const __m256i out_06_3 = + _mm256_madd_epi16(out_26_1, k__cospi_m26_p06); + const __m256i out_22_2 = + _mm256_madd_epi16(out_10_0, k__cospi_m10_p22); + const __m256i out_22_3 = + _mm256_madd_epi16(out_10_1, k__cospi_m10_p22); + const __m256i out_14_2 = + _mm256_madd_epi16(out_18_0, k__cospi_m18_p14); + const __m256i out_14_3 = + _mm256_madd_epi16(out_18_1, k__cospi_m18_p14); + const __m256i out_30_2 = + _mm256_madd_epi16(out_02_0, k__cospi_m02_p30); + const __m256i out_30_3 = + _mm256_madd_epi16(out_02_1, k__cospi_m02_p30); + // dct_const_round_shift + const __m256i out_02_4 = + _mm256_add_epi32(out_02_2, k__DCT_CONST_ROUNDING); + const __m256i out_02_5 = + _mm256_add_epi32(out_02_3, k__DCT_CONST_ROUNDING); + const __m256i out_18_4 = + _mm256_add_epi32(out_18_2, k__DCT_CONST_ROUNDING); + const __m256i out_18_5 = + _mm256_add_epi32(out_18_3, k__DCT_CONST_ROUNDING); + const __m256i out_10_4 = + _mm256_add_epi32(out_10_2, k__DCT_CONST_ROUNDING); + const __m256i out_10_5 = + _mm256_add_epi32(out_10_3, k__DCT_CONST_ROUNDING); + const __m256i out_26_4 = + _mm256_add_epi32(out_26_2, k__DCT_CONST_ROUNDING); + const __m256i out_26_5 = + _mm256_add_epi32(out_26_3, k__DCT_CONST_ROUNDING); + const __m256i out_06_4 = + _mm256_add_epi32(out_06_2, k__DCT_CONST_ROUNDING); + const __m256i out_06_5 = + _mm256_add_epi32(out_06_3, k__DCT_CONST_ROUNDING); + const __m256i out_22_4 = + _mm256_add_epi32(out_22_2, k__DCT_CONST_ROUNDING); + const __m256i out_22_5 = + _mm256_add_epi32(out_22_3, k__DCT_CONST_ROUNDING); + const __m256i out_14_4 = + _mm256_add_epi32(out_14_2, k__DCT_CONST_ROUNDING); + const __m256i out_14_5 = + _mm256_add_epi32(out_14_3, k__DCT_CONST_ROUNDING); + const __m256i out_30_4 = + _mm256_add_epi32(out_30_2, k__DCT_CONST_ROUNDING); + const __m256i out_30_5 = + _mm256_add_epi32(out_30_3, k__DCT_CONST_ROUNDING); + const __m256i out_02_6 = _mm256_srai_epi32(out_02_4, DCT_CONST_BITS); + const __m256i out_02_7 = _mm256_srai_epi32(out_02_5, DCT_CONST_BITS); + const __m256i out_18_6 = _mm256_srai_epi32(out_18_4, DCT_CONST_BITS); + const __m256i out_18_7 = _mm256_srai_epi32(out_18_5, DCT_CONST_BITS); + const __m256i out_10_6 = _mm256_srai_epi32(out_10_4, DCT_CONST_BITS); + const __m256i out_10_7 = _mm256_srai_epi32(out_10_5, DCT_CONST_BITS); + const __m256i out_26_6 = _mm256_srai_epi32(out_26_4, DCT_CONST_BITS); + const __m256i out_26_7 = _mm256_srai_epi32(out_26_5, DCT_CONST_BITS); + const __m256i out_06_6 = _mm256_srai_epi32(out_06_4, DCT_CONST_BITS); + const __m256i out_06_7 = _mm256_srai_epi32(out_06_5, DCT_CONST_BITS); + const __m256i out_22_6 = _mm256_srai_epi32(out_22_4, DCT_CONST_BITS); + const __m256i out_22_7 = _mm256_srai_epi32(out_22_5, DCT_CONST_BITS); + const __m256i out_14_6 = _mm256_srai_epi32(out_14_4, DCT_CONST_BITS); + const __m256i out_14_7 = _mm256_srai_epi32(out_14_5, DCT_CONST_BITS); + const __m256i out_30_6 = _mm256_srai_epi32(out_30_4, DCT_CONST_BITS); + const __m256i out_30_7 = _mm256_srai_epi32(out_30_5, DCT_CONST_BITS); + // Combine + out[2] = _mm256_packs_epi32(out_02_6, out_02_7); + out[18] = _mm256_packs_epi32(out_18_6, out_18_7); + out[10] = _mm256_packs_epi32(out_10_6, out_10_7); + out[26] = _mm256_packs_epi32(out_26_6, out_26_7); + out[6] = _mm256_packs_epi32(out_06_6, out_06_7); + out[22] = _mm256_packs_epi32(out_22_6, out_22_7); + out[14] = _mm256_packs_epi32(out_14_6, out_14_7); + out[30] = _mm256_packs_epi32(out_30_6, out_30_7); + } + { + step1[16] = _mm256_add_epi16(step3[17], step2[16]); + step1[17] = _mm256_sub_epi16(step2[16], step3[17]); + step1[18] = _mm256_sub_epi16(step2[19], step3[18]); + step1[19] = _mm256_add_epi16(step3[18], step2[19]); + step1[20] = _mm256_add_epi16(step3[21], step2[20]); + step1[21] = _mm256_sub_epi16(step2[20], step3[21]); + step1[22] = _mm256_sub_epi16(step2[23], step3[22]); + step1[23] = _mm256_add_epi16(step3[22], step2[23]); + step1[24] = _mm256_add_epi16(step3[25], step2[24]); + step1[25] = _mm256_sub_epi16(step2[24], step3[25]); + step1[26] = _mm256_sub_epi16(step2[27], step3[26]); + step1[27] = _mm256_add_epi16(step3[26], step2[27]); + step1[28] = _mm256_add_epi16(step3[29], step2[28]); + step1[29] = _mm256_sub_epi16(step2[28], step3[29]); + step1[30] = _mm256_sub_epi16(step2[31], step3[30]); + step1[31] = _mm256_add_epi16(step3[30], step2[31]); + } + // Final stage --- outputs indices are bit-reversed. + { + const __m256i out_01_0 = _mm256_unpacklo_epi16(step1[16], step1[31]); + const __m256i out_01_1 = _mm256_unpackhi_epi16(step1[16], step1[31]); + const __m256i out_17_0 = _mm256_unpacklo_epi16(step1[17], step1[30]); + const __m256i out_17_1 = _mm256_unpackhi_epi16(step1[17], step1[30]); + const __m256i out_09_0 = _mm256_unpacklo_epi16(step1[18], step1[29]); + const __m256i out_09_1 = _mm256_unpackhi_epi16(step1[18], step1[29]); + const __m256i out_25_0 = _mm256_unpacklo_epi16(step1[19], step1[28]); + const __m256i out_25_1 = _mm256_unpackhi_epi16(step1[19], step1[28]); + const __m256i out_01_2 = + _mm256_madd_epi16(out_01_0, k__cospi_p31_p01); + const __m256i out_01_3 = + _mm256_madd_epi16(out_01_1, k__cospi_p31_p01); + const __m256i out_17_2 = + _mm256_madd_epi16(out_17_0, k__cospi_p15_p17); + const __m256i out_17_3 = + _mm256_madd_epi16(out_17_1, k__cospi_p15_p17); + const __m256i out_09_2 = + _mm256_madd_epi16(out_09_0, k__cospi_p23_p09); + const __m256i out_09_3 = + _mm256_madd_epi16(out_09_1, k__cospi_p23_p09); + const __m256i out_25_2 = + _mm256_madd_epi16(out_25_0, k__cospi_p07_p25); + const __m256i out_25_3 = + _mm256_madd_epi16(out_25_1, k__cospi_p07_p25); + const __m256i out_07_2 = + _mm256_madd_epi16(out_25_0, k__cospi_m25_p07); + const __m256i out_07_3 = + _mm256_madd_epi16(out_25_1, k__cospi_m25_p07); + const __m256i out_23_2 = + _mm256_madd_epi16(out_09_0, k__cospi_m09_p23); + const __m256i out_23_3 = + _mm256_madd_epi16(out_09_1, k__cospi_m09_p23); + const __m256i out_15_2 = + _mm256_madd_epi16(out_17_0, k__cospi_m17_p15); + const __m256i out_15_3 = + _mm256_madd_epi16(out_17_1, k__cospi_m17_p15); + const __m256i out_31_2 = + _mm256_madd_epi16(out_01_0, k__cospi_m01_p31); + const __m256i out_31_3 = + _mm256_madd_epi16(out_01_1, k__cospi_m01_p31); + // dct_const_round_shift + const __m256i out_01_4 = + _mm256_add_epi32(out_01_2, k__DCT_CONST_ROUNDING); + const __m256i out_01_5 = + _mm256_add_epi32(out_01_3, k__DCT_CONST_ROUNDING); + const __m256i out_17_4 = + _mm256_add_epi32(out_17_2, k__DCT_CONST_ROUNDING); + const __m256i out_17_5 = + _mm256_add_epi32(out_17_3, k__DCT_CONST_ROUNDING); + const __m256i out_09_4 = + _mm256_add_epi32(out_09_2, k__DCT_CONST_ROUNDING); + const __m256i out_09_5 = + _mm256_add_epi32(out_09_3, k__DCT_CONST_ROUNDING); + const __m256i out_25_4 = + _mm256_add_epi32(out_25_2, k__DCT_CONST_ROUNDING); + const __m256i out_25_5 = + _mm256_add_epi32(out_25_3, k__DCT_CONST_ROUNDING); + const __m256i out_07_4 = + _mm256_add_epi32(out_07_2, k__DCT_CONST_ROUNDING); + const __m256i out_07_5 = + _mm256_add_epi32(out_07_3, k__DCT_CONST_ROUNDING); + const __m256i out_23_4 = + _mm256_add_epi32(out_23_2, k__DCT_CONST_ROUNDING); + const __m256i out_23_5 = + _mm256_add_epi32(out_23_3, k__DCT_CONST_ROUNDING); + const __m256i out_15_4 = + _mm256_add_epi32(out_15_2, k__DCT_CONST_ROUNDING); + const __m256i out_15_5 = + _mm256_add_epi32(out_15_3, k__DCT_CONST_ROUNDING); + const __m256i out_31_4 = + _mm256_add_epi32(out_31_2, k__DCT_CONST_ROUNDING); + const __m256i out_31_5 = + _mm256_add_epi32(out_31_3, k__DCT_CONST_ROUNDING); + const __m256i out_01_6 = _mm256_srai_epi32(out_01_4, DCT_CONST_BITS); + const __m256i out_01_7 = _mm256_srai_epi32(out_01_5, DCT_CONST_BITS); + const __m256i out_17_6 = _mm256_srai_epi32(out_17_4, DCT_CONST_BITS); + const __m256i out_17_7 = _mm256_srai_epi32(out_17_5, DCT_CONST_BITS); + const __m256i out_09_6 = _mm256_srai_epi32(out_09_4, DCT_CONST_BITS); + const __m256i out_09_7 = _mm256_srai_epi32(out_09_5, DCT_CONST_BITS); + const __m256i out_25_6 = _mm256_srai_epi32(out_25_4, DCT_CONST_BITS); + const __m256i out_25_7 = _mm256_srai_epi32(out_25_5, DCT_CONST_BITS); + const __m256i out_07_6 = _mm256_srai_epi32(out_07_4, DCT_CONST_BITS); + const __m256i out_07_7 = _mm256_srai_epi32(out_07_5, DCT_CONST_BITS); + const __m256i out_23_6 = _mm256_srai_epi32(out_23_4, DCT_CONST_BITS); + const __m256i out_23_7 = _mm256_srai_epi32(out_23_5, DCT_CONST_BITS); + const __m256i out_15_6 = _mm256_srai_epi32(out_15_4, DCT_CONST_BITS); + const __m256i out_15_7 = _mm256_srai_epi32(out_15_5, DCT_CONST_BITS); + const __m256i out_31_6 = _mm256_srai_epi32(out_31_4, DCT_CONST_BITS); + const __m256i out_31_7 = _mm256_srai_epi32(out_31_5, DCT_CONST_BITS); + // Combine + out[1] = _mm256_packs_epi32(out_01_6, out_01_7); + out[17] = _mm256_packs_epi32(out_17_6, out_17_7); + out[9] = _mm256_packs_epi32(out_09_6, out_09_7); + out[25] = _mm256_packs_epi32(out_25_6, out_25_7); + out[7] = _mm256_packs_epi32(out_07_6, out_07_7); + out[23] = _mm256_packs_epi32(out_23_6, out_23_7); + out[15] = _mm256_packs_epi32(out_15_6, out_15_7); + out[31] = _mm256_packs_epi32(out_31_6, out_31_7); + } + { + const __m256i out_05_0 = _mm256_unpacklo_epi16(step1[20], step1[27]); + const __m256i out_05_1 = _mm256_unpackhi_epi16(step1[20], step1[27]); + const __m256i out_21_0 = _mm256_unpacklo_epi16(step1[21], step1[26]); + const __m256i out_21_1 = _mm256_unpackhi_epi16(step1[21], step1[26]); + const __m256i out_13_0 = _mm256_unpacklo_epi16(step1[22], step1[25]); + const __m256i out_13_1 = _mm256_unpackhi_epi16(step1[22], step1[25]); + const __m256i out_29_0 = _mm256_unpacklo_epi16(step1[23], step1[24]); + const __m256i out_29_1 = _mm256_unpackhi_epi16(step1[23], step1[24]); + const __m256i out_05_2 = + _mm256_madd_epi16(out_05_0, k__cospi_p27_p05); + const __m256i out_05_3 = + _mm256_madd_epi16(out_05_1, k__cospi_p27_p05); + const __m256i out_21_2 = + _mm256_madd_epi16(out_21_0, k__cospi_p11_p21); + const __m256i out_21_3 = + _mm256_madd_epi16(out_21_1, k__cospi_p11_p21); + const __m256i out_13_2 = + _mm256_madd_epi16(out_13_0, k__cospi_p19_p13); + const __m256i out_13_3 = + _mm256_madd_epi16(out_13_1, k__cospi_p19_p13); + const __m256i out_29_2 = + _mm256_madd_epi16(out_29_0, k__cospi_p03_p29); + const __m256i out_29_3 = + _mm256_madd_epi16(out_29_1, k__cospi_p03_p29); + const __m256i out_03_2 = + _mm256_madd_epi16(out_29_0, k__cospi_m29_p03); + const __m256i out_03_3 = + _mm256_madd_epi16(out_29_1, k__cospi_m29_p03); + const __m256i out_19_2 = + _mm256_madd_epi16(out_13_0, k__cospi_m13_p19); + const __m256i out_19_3 = + _mm256_madd_epi16(out_13_1, k__cospi_m13_p19); + const __m256i out_11_2 = + _mm256_madd_epi16(out_21_0, k__cospi_m21_p11); + const __m256i out_11_3 = + _mm256_madd_epi16(out_21_1, k__cospi_m21_p11); + const __m256i out_27_2 = + _mm256_madd_epi16(out_05_0, k__cospi_m05_p27); + const __m256i out_27_3 = + _mm256_madd_epi16(out_05_1, k__cospi_m05_p27); + // dct_const_round_shift + const __m256i out_05_4 = + _mm256_add_epi32(out_05_2, k__DCT_CONST_ROUNDING); + const __m256i out_05_5 = + _mm256_add_epi32(out_05_3, k__DCT_CONST_ROUNDING); + const __m256i out_21_4 = + _mm256_add_epi32(out_21_2, k__DCT_CONST_ROUNDING); + const __m256i out_21_5 = + _mm256_add_epi32(out_21_3, k__DCT_CONST_ROUNDING); + const __m256i out_13_4 = + _mm256_add_epi32(out_13_2, k__DCT_CONST_ROUNDING); + const __m256i out_13_5 = + _mm256_add_epi32(out_13_3, k__DCT_CONST_ROUNDING); + const __m256i out_29_4 = + _mm256_add_epi32(out_29_2, k__DCT_CONST_ROUNDING); + const __m256i out_29_5 = + _mm256_add_epi32(out_29_3, k__DCT_CONST_ROUNDING); + const __m256i out_03_4 = + _mm256_add_epi32(out_03_2, k__DCT_CONST_ROUNDING); + const __m256i out_03_5 = + _mm256_add_epi32(out_03_3, k__DCT_CONST_ROUNDING); + const __m256i out_19_4 = + _mm256_add_epi32(out_19_2, k__DCT_CONST_ROUNDING); + const __m256i out_19_5 = + _mm256_add_epi32(out_19_3, k__DCT_CONST_ROUNDING); + const __m256i out_11_4 = + _mm256_add_epi32(out_11_2, k__DCT_CONST_ROUNDING); + const __m256i out_11_5 = + _mm256_add_epi32(out_11_3, k__DCT_CONST_ROUNDING); + const __m256i out_27_4 = + _mm256_add_epi32(out_27_2, k__DCT_CONST_ROUNDING); + const __m256i out_27_5 = + _mm256_add_epi32(out_27_3, k__DCT_CONST_ROUNDING); + const __m256i out_05_6 = _mm256_srai_epi32(out_05_4, DCT_CONST_BITS); + const __m256i out_05_7 = _mm256_srai_epi32(out_05_5, DCT_CONST_BITS); + const __m256i out_21_6 = _mm256_srai_epi32(out_21_4, DCT_CONST_BITS); + const __m256i out_21_7 = _mm256_srai_epi32(out_21_5, DCT_CONST_BITS); + const __m256i out_13_6 = _mm256_srai_epi32(out_13_4, DCT_CONST_BITS); + const __m256i out_13_7 = _mm256_srai_epi32(out_13_5, DCT_CONST_BITS); + const __m256i out_29_6 = _mm256_srai_epi32(out_29_4, DCT_CONST_BITS); + const __m256i out_29_7 = _mm256_srai_epi32(out_29_5, DCT_CONST_BITS); + const __m256i out_03_6 = _mm256_srai_epi32(out_03_4, DCT_CONST_BITS); + const __m256i out_03_7 = _mm256_srai_epi32(out_03_5, DCT_CONST_BITS); + const __m256i out_19_6 = _mm256_srai_epi32(out_19_4, DCT_CONST_BITS); + const __m256i out_19_7 = _mm256_srai_epi32(out_19_5, DCT_CONST_BITS); + const __m256i out_11_6 = _mm256_srai_epi32(out_11_4, DCT_CONST_BITS); + const __m256i out_11_7 = _mm256_srai_epi32(out_11_5, DCT_CONST_BITS); + const __m256i out_27_6 = _mm256_srai_epi32(out_27_4, DCT_CONST_BITS); + const __m256i out_27_7 = _mm256_srai_epi32(out_27_5, DCT_CONST_BITS); + // Combine + out[5] = _mm256_packs_epi32(out_05_6, out_05_7); + out[21] = _mm256_packs_epi32(out_21_6, out_21_7); + out[13] = _mm256_packs_epi32(out_13_6, out_13_7); + out[29] = _mm256_packs_epi32(out_29_6, out_29_7); + out[3] = _mm256_packs_epi32(out_03_6, out_03_7); + out[19] = _mm256_packs_epi32(out_19_6, out_19_7); + out[11] = _mm256_packs_epi32(out_11_6, out_11_7); + out[27] = _mm256_packs_epi32(out_27_6, out_27_7); + } +#if FDCT32x32_HIGH_PRECISION + } else { + __m256i lstep1[64], lstep2[64], lstep3[64]; + __m256i u[32], v[32], sign[16]; + const __m256i K32One = _mm256_set_epi32(1, 1, 1, 1, 1, 1, 1, 1); + // start using 32-bit operations + // stage 3 + { + // expanding to 32-bit length priori to addition operations + lstep2[0] = _mm256_unpacklo_epi16(step2[0], kZero); + lstep2[1] = _mm256_unpackhi_epi16(step2[0], kZero); + lstep2[2] = _mm256_unpacklo_epi16(step2[1], kZero); + lstep2[3] = _mm256_unpackhi_epi16(step2[1], kZero); + lstep2[4] = _mm256_unpacklo_epi16(step2[2], kZero); + lstep2[5] = _mm256_unpackhi_epi16(step2[2], kZero); + lstep2[6] = _mm256_unpacklo_epi16(step2[3], kZero); + lstep2[7] = _mm256_unpackhi_epi16(step2[3], kZero); + lstep2[8] = _mm256_unpacklo_epi16(step2[4], kZero); + lstep2[9] = _mm256_unpackhi_epi16(step2[4], kZero); + lstep2[10] = _mm256_unpacklo_epi16(step2[5], kZero); + lstep2[11] = _mm256_unpackhi_epi16(step2[5], kZero); + lstep2[12] = _mm256_unpacklo_epi16(step2[6], kZero); + lstep2[13] = _mm256_unpackhi_epi16(step2[6], kZero); + lstep2[14] = _mm256_unpacklo_epi16(step2[7], kZero); + lstep2[15] = _mm256_unpackhi_epi16(step2[7], kZero); + lstep2[0] = _mm256_madd_epi16(lstep2[0], kOne); + lstep2[1] = _mm256_madd_epi16(lstep2[1], kOne); + lstep2[2] = _mm256_madd_epi16(lstep2[2], kOne); + lstep2[3] = _mm256_madd_epi16(lstep2[3], kOne); + lstep2[4] = _mm256_madd_epi16(lstep2[4], kOne); + lstep2[5] = _mm256_madd_epi16(lstep2[5], kOne); + lstep2[6] = _mm256_madd_epi16(lstep2[6], kOne); + lstep2[7] = _mm256_madd_epi16(lstep2[7], kOne); + lstep2[8] = _mm256_madd_epi16(lstep2[8], kOne); + lstep2[9] = _mm256_madd_epi16(lstep2[9], kOne); + lstep2[10] = _mm256_madd_epi16(lstep2[10], kOne); + lstep2[11] = _mm256_madd_epi16(lstep2[11], kOne); + lstep2[12] = _mm256_madd_epi16(lstep2[12], kOne); + lstep2[13] = _mm256_madd_epi16(lstep2[13], kOne); + lstep2[14] = _mm256_madd_epi16(lstep2[14], kOne); + lstep2[15] = _mm256_madd_epi16(lstep2[15], kOne); + + lstep3[0] = _mm256_add_epi32(lstep2[14], lstep2[0]); + lstep3[1] = _mm256_add_epi32(lstep2[15], lstep2[1]); + lstep3[2] = _mm256_add_epi32(lstep2[12], lstep2[2]); + lstep3[3] = _mm256_add_epi32(lstep2[13], lstep2[3]); + lstep3[4] = _mm256_add_epi32(lstep2[10], lstep2[4]); + lstep3[5] = _mm256_add_epi32(lstep2[11], lstep2[5]); + lstep3[6] = _mm256_add_epi32(lstep2[8], lstep2[6]); + lstep3[7] = _mm256_add_epi32(lstep2[9], lstep2[7]); + lstep3[8] = _mm256_sub_epi32(lstep2[6], lstep2[8]); + lstep3[9] = _mm256_sub_epi32(lstep2[7], lstep2[9]); + lstep3[10] = _mm256_sub_epi32(lstep2[4], lstep2[10]); + lstep3[11] = _mm256_sub_epi32(lstep2[5], lstep2[11]); + lstep3[12] = _mm256_sub_epi32(lstep2[2], lstep2[12]); + lstep3[13] = _mm256_sub_epi32(lstep2[3], lstep2[13]); + lstep3[14] = _mm256_sub_epi32(lstep2[0], lstep2[14]); + lstep3[15] = _mm256_sub_epi32(lstep2[1], lstep2[15]); + } + { + const __m256i s3_10_0 = _mm256_unpacklo_epi16(step2[13], step2[10]); + const __m256i s3_10_1 = _mm256_unpackhi_epi16(step2[13], step2[10]); + const __m256i s3_11_0 = _mm256_unpacklo_epi16(step2[12], step2[11]); + const __m256i s3_11_1 = _mm256_unpackhi_epi16(step2[12], step2[11]); + const __m256i s3_10_2 = _mm256_madd_epi16(s3_10_0, k__cospi_p16_m16); + const __m256i s3_10_3 = _mm256_madd_epi16(s3_10_1, k__cospi_p16_m16); + const __m256i s3_11_2 = _mm256_madd_epi16(s3_11_0, k__cospi_p16_m16); + const __m256i s3_11_3 = _mm256_madd_epi16(s3_11_1, k__cospi_p16_m16); + const __m256i s3_12_2 = _mm256_madd_epi16(s3_11_0, k__cospi_p16_p16); + const __m256i s3_12_3 = _mm256_madd_epi16(s3_11_1, k__cospi_p16_p16); + const __m256i s3_13_2 = _mm256_madd_epi16(s3_10_0, k__cospi_p16_p16); + const __m256i s3_13_3 = _mm256_madd_epi16(s3_10_1, k__cospi_p16_p16); + // dct_const_round_shift + const __m256i s3_10_4 = + _mm256_add_epi32(s3_10_2, k__DCT_CONST_ROUNDING); + const __m256i s3_10_5 = + _mm256_add_epi32(s3_10_3, k__DCT_CONST_ROUNDING); + const __m256i s3_11_4 = + _mm256_add_epi32(s3_11_2, k__DCT_CONST_ROUNDING); + const __m256i s3_11_5 = + _mm256_add_epi32(s3_11_3, k__DCT_CONST_ROUNDING); + const __m256i s3_12_4 = + _mm256_add_epi32(s3_12_2, k__DCT_CONST_ROUNDING); + const __m256i s3_12_5 = + _mm256_add_epi32(s3_12_3, k__DCT_CONST_ROUNDING); + const __m256i s3_13_4 = + _mm256_add_epi32(s3_13_2, k__DCT_CONST_ROUNDING); + const __m256i s3_13_5 = + _mm256_add_epi32(s3_13_3, k__DCT_CONST_ROUNDING); + lstep3[20] = _mm256_srai_epi32(s3_10_4, DCT_CONST_BITS); + lstep3[21] = _mm256_srai_epi32(s3_10_5, DCT_CONST_BITS); + lstep3[22] = _mm256_srai_epi32(s3_11_4, DCT_CONST_BITS); + lstep3[23] = _mm256_srai_epi32(s3_11_5, DCT_CONST_BITS); + lstep3[24] = _mm256_srai_epi32(s3_12_4, DCT_CONST_BITS); + lstep3[25] = _mm256_srai_epi32(s3_12_5, DCT_CONST_BITS); + lstep3[26] = _mm256_srai_epi32(s3_13_4, DCT_CONST_BITS); + lstep3[27] = _mm256_srai_epi32(s3_13_5, DCT_CONST_BITS); + } + { + lstep2[40] = _mm256_unpacklo_epi16(step2[20], kZero); + lstep2[41] = _mm256_unpackhi_epi16(step2[20], kZero); + lstep2[42] = _mm256_unpacklo_epi16(step2[21], kZero); + lstep2[43] = _mm256_unpackhi_epi16(step2[21], kZero); + lstep2[44] = _mm256_unpacklo_epi16(step2[22], kZero); + lstep2[45] = _mm256_unpackhi_epi16(step2[22], kZero); + lstep2[46] = _mm256_unpacklo_epi16(step2[23], kZero); + lstep2[47] = _mm256_unpackhi_epi16(step2[23], kZero); + lstep2[48] = _mm256_unpacklo_epi16(step2[24], kZero); + lstep2[49] = _mm256_unpackhi_epi16(step2[24], kZero); + lstep2[50] = _mm256_unpacklo_epi16(step2[25], kZero); + lstep2[51] = _mm256_unpackhi_epi16(step2[25], kZero); + lstep2[52] = _mm256_unpacklo_epi16(step2[26], kZero); + lstep2[53] = _mm256_unpackhi_epi16(step2[26], kZero); + lstep2[54] = _mm256_unpacklo_epi16(step2[27], kZero); + lstep2[55] = _mm256_unpackhi_epi16(step2[27], kZero); + lstep2[40] = _mm256_madd_epi16(lstep2[40], kOne); + lstep2[41] = _mm256_madd_epi16(lstep2[41], kOne); + lstep2[42] = _mm256_madd_epi16(lstep2[42], kOne); + lstep2[43] = _mm256_madd_epi16(lstep2[43], kOne); + lstep2[44] = _mm256_madd_epi16(lstep2[44], kOne); + lstep2[45] = _mm256_madd_epi16(lstep2[45], kOne); + lstep2[46] = _mm256_madd_epi16(lstep2[46], kOne); + lstep2[47] = _mm256_madd_epi16(lstep2[47], kOne); + lstep2[48] = _mm256_madd_epi16(lstep2[48], kOne); + lstep2[49] = _mm256_madd_epi16(lstep2[49], kOne); + lstep2[50] = _mm256_madd_epi16(lstep2[50], kOne); + lstep2[51] = _mm256_madd_epi16(lstep2[51], kOne); + lstep2[52] = _mm256_madd_epi16(lstep2[52], kOne); + lstep2[53] = _mm256_madd_epi16(lstep2[53], kOne); + lstep2[54] = _mm256_madd_epi16(lstep2[54], kOne); + lstep2[55] = _mm256_madd_epi16(lstep2[55], kOne); + + lstep1[32] = _mm256_unpacklo_epi16(step1[16], kZero); + lstep1[33] = _mm256_unpackhi_epi16(step1[16], kZero); + lstep1[34] = _mm256_unpacklo_epi16(step1[17], kZero); + lstep1[35] = _mm256_unpackhi_epi16(step1[17], kZero); + lstep1[36] = _mm256_unpacklo_epi16(step1[18], kZero); + lstep1[37] = _mm256_unpackhi_epi16(step1[18], kZero); + lstep1[38] = _mm256_unpacklo_epi16(step1[19], kZero); + lstep1[39] = _mm256_unpackhi_epi16(step1[19], kZero); + lstep1[56] = _mm256_unpacklo_epi16(step1[28], kZero); + lstep1[57] = _mm256_unpackhi_epi16(step1[28], kZero); + lstep1[58] = _mm256_unpacklo_epi16(step1[29], kZero); + lstep1[59] = _mm256_unpackhi_epi16(step1[29], kZero); + lstep1[60] = _mm256_unpacklo_epi16(step1[30], kZero); + lstep1[61] = _mm256_unpackhi_epi16(step1[30], kZero); + lstep1[62] = _mm256_unpacklo_epi16(step1[31], kZero); + lstep1[63] = _mm256_unpackhi_epi16(step1[31], kZero); + lstep1[32] = _mm256_madd_epi16(lstep1[32], kOne); + lstep1[33] = _mm256_madd_epi16(lstep1[33], kOne); + lstep1[34] = _mm256_madd_epi16(lstep1[34], kOne); + lstep1[35] = _mm256_madd_epi16(lstep1[35], kOne); + lstep1[36] = _mm256_madd_epi16(lstep1[36], kOne); + lstep1[37] = _mm256_madd_epi16(lstep1[37], kOne); + lstep1[38] = _mm256_madd_epi16(lstep1[38], kOne); + lstep1[39] = _mm256_madd_epi16(lstep1[39], kOne); + lstep1[56] = _mm256_madd_epi16(lstep1[56], kOne); + lstep1[57] = _mm256_madd_epi16(lstep1[57], kOne); + lstep1[58] = _mm256_madd_epi16(lstep1[58], kOne); + lstep1[59] = _mm256_madd_epi16(lstep1[59], kOne); + lstep1[60] = _mm256_madd_epi16(lstep1[60], kOne); + lstep1[61] = _mm256_madd_epi16(lstep1[61], kOne); + lstep1[62] = _mm256_madd_epi16(lstep1[62], kOne); + lstep1[63] = _mm256_madd_epi16(lstep1[63], kOne); + + lstep3[32] = _mm256_add_epi32(lstep2[46], lstep1[32]); + lstep3[33] = _mm256_add_epi32(lstep2[47], lstep1[33]); + + lstep3[34] = _mm256_add_epi32(lstep2[44], lstep1[34]); + lstep3[35] = _mm256_add_epi32(lstep2[45], lstep1[35]); + lstep3[36] = _mm256_add_epi32(lstep2[42], lstep1[36]); + lstep3[37] = _mm256_add_epi32(lstep2[43], lstep1[37]); + lstep3[38] = _mm256_add_epi32(lstep2[40], lstep1[38]); + lstep3[39] = _mm256_add_epi32(lstep2[41], lstep1[39]); + lstep3[40] = _mm256_sub_epi32(lstep1[38], lstep2[40]); + lstep3[41] = _mm256_sub_epi32(lstep1[39], lstep2[41]); + lstep3[42] = _mm256_sub_epi32(lstep1[36], lstep2[42]); + lstep3[43] = _mm256_sub_epi32(lstep1[37], lstep2[43]); + lstep3[44] = _mm256_sub_epi32(lstep1[34], lstep2[44]); + lstep3[45] = _mm256_sub_epi32(lstep1[35], lstep2[45]); + lstep3[46] = _mm256_sub_epi32(lstep1[32], lstep2[46]); + lstep3[47] = _mm256_sub_epi32(lstep1[33], lstep2[47]); + lstep3[48] = _mm256_sub_epi32(lstep1[62], lstep2[48]); + lstep3[49] = _mm256_sub_epi32(lstep1[63], lstep2[49]); + lstep3[50] = _mm256_sub_epi32(lstep1[60], lstep2[50]); + lstep3[51] = _mm256_sub_epi32(lstep1[61], lstep2[51]); + lstep3[52] = _mm256_sub_epi32(lstep1[58], lstep2[52]); + lstep3[53] = _mm256_sub_epi32(lstep1[59], lstep2[53]); + lstep3[54] = _mm256_sub_epi32(lstep1[56], lstep2[54]); + lstep3[55] = _mm256_sub_epi32(lstep1[57], lstep2[55]); + lstep3[56] = _mm256_add_epi32(lstep2[54], lstep1[56]); + lstep3[57] = _mm256_add_epi32(lstep2[55], lstep1[57]); + lstep3[58] = _mm256_add_epi32(lstep2[52], lstep1[58]); + lstep3[59] = _mm256_add_epi32(lstep2[53], lstep1[59]); + lstep3[60] = _mm256_add_epi32(lstep2[50], lstep1[60]); + lstep3[61] = _mm256_add_epi32(lstep2[51], lstep1[61]); + lstep3[62] = _mm256_add_epi32(lstep2[48], lstep1[62]); + lstep3[63] = _mm256_add_epi32(lstep2[49], lstep1[63]); + } + + // stage 4 + { + // expanding to 32-bit length priori to addition operations + lstep2[16] = _mm256_unpacklo_epi16(step2[8], kZero); + lstep2[17] = _mm256_unpackhi_epi16(step2[8], kZero); + lstep2[18] = _mm256_unpacklo_epi16(step2[9], kZero); + lstep2[19] = _mm256_unpackhi_epi16(step2[9], kZero); + lstep2[28] = _mm256_unpacklo_epi16(step2[14], kZero); + lstep2[29] = _mm256_unpackhi_epi16(step2[14], kZero); + lstep2[30] = _mm256_unpacklo_epi16(step2[15], kZero); + lstep2[31] = _mm256_unpackhi_epi16(step2[15], kZero); + lstep2[16] = _mm256_madd_epi16(lstep2[16], kOne); + lstep2[17] = _mm256_madd_epi16(lstep2[17], kOne); + lstep2[18] = _mm256_madd_epi16(lstep2[18], kOne); + lstep2[19] = _mm256_madd_epi16(lstep2[19], kOne); + lstep2[28] = _mm256_madd_epi16(lstep2[28], kOne); + lstep2[29] = _mm256_madd_epi16(lstep2[29], kOne); + lstep2[30] = _mm256_madd_epi16(lstep2[30], kOne); + lstep2[31] = _mm256_madd_epi16(lstep2[31], kOne); + + lstep1[0] = _mm256_add_epi32(lstep3[6], lstep3[0]); + lstep1[1] = _mm256_add_epi32(lstep3[7], lstep3[1]); + lstep1[2] = _mm256_add_epi32(lstep3[4], lstep3[2]); + lstep1[3] = _mm256_add_epi32(lstep3[5], lstep3[3]); + lstep1[4] = _mm256_sub_epi32(lstep3[2], lstep3[4]); + lstep1[5] = _mm256_sub_epi32(lstep3[3], lstep3[5]); + lstep1[6] = _mm256_sub_epi32(lstep3[0], lstep3[6]); + lstep1[7] = _mm256_sub_epi32(lstep3[1], lstep3[7]); + lstep1[16] = _mm256_add_epi32(lstep3[22], lstep2[16]); + lstep1[17] = _mm256_add_epi32(lstep3[23], lstep2[17]); + lstep1[18] = _mm256_add_epi32(lstep3[20], lstep2[18]); + lstep1[19] = _mm256_add_epi32(lstep3[21], lstep2[19]); + lstep1[20] = _mm256_sub_epi32(lstep2[18], lstep3[20]); + lstep1[21] = _mm256_sub_epi32(lstep2[19], lstep3[21]); + lstep1[22] = _mm256_sub_epi32(lstep2[16], lstep3[22]); + lstep1[23] = _mm256_sub_epi32(lstep2[17], lstep3[23]); + lstep1[24] = _mm256_sub_epi32(lstep2[30], lstep3[24]); + lstep1[25] = _mm256_sub_epi32(lstep2[31], lstep3[25]); + lstep1[26] = _mm256_sub_epi32(lstep2[28], lstep3[26]); + lstep1[27] = _mm256_sub_epi32(lstep2[29], lstep3[27]); + lstep1[28] = _mm256_add_epi32(lstep3[26], lstep2[28]); + lstep1[29] = _mm256_add_epi32(lstep3[27], lstep2[29]); + lstep1[30] = _mm256_add_epi32(lstep3[24], lstep2[30]); + lstep1[31] = _mm256_add_epi32(lstep3[25], lstep2[31]); + } + { + // to be continued... + // + const __m256i k32_p16_p16 = + pair256_set_epi32(cospi_16_64, cospi_16_64); + const __m256i k32_p16_m16 = + pair256_set_epi32(cospi_16_64, -cospi_16_64); + + u[0] = _mm256_unpacklo_epi32(lstep3[12], lstep3[10]); + u[1] = _mm256_unpackhi_epi32(lstep3[12], lstep3[10]); + u[2] = _mm256_unpacklo_epi32(lstep3[13], lstep3[11]); + u[3] = _mm256_unpackhi_epi32(lstep3[13], lstep3[11]); + + // TODO(jingning): manually inline k_madd_epi32_avx2_ to further hide + // instruction latency. + v[0] = k_madd_epi32_avx2(u[0], k32_p16_m16); + v[1] = k_madd_epi32_avx2(u[1], k32_p16_m16); + v[2] = k_madd_epi32_avx2(u[2], k32_p16_m16); + v[3] = k_madd_epi32_avx2(u[3], k32_p16_m16); + v[4] = k_madd_epi32_avx2(u[0], k32_p16_p16); + v[5] = k_madd_epi32_avx2(u[1], k32_p16_p16); + v[6] = k_madd_epi32_avx2(u[2], k32_p16_p16); + v[7] = k_madd_epi32_avx2(u[3], k32_p16_p16); + + u[0] = k_packs_epi64_avx2(v[0], v[1]); + u[1] = k_packs_epi64_avx2(v[2], v[3]); + u[2] = k_packs_epi64_avx2(v[4], v[5]); + u[3] = k_packs_epi64_avx2(v[6], v[7]); + + v[0] = _mm256_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm256_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm256_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm256_add_epi32(u[3], k__DCT_CONST_ROUNDING); + + lstep1[10] = _mm256_srai_epi32(v[0], DCT_CONST_BITS); + lstep1[11] = _mm256_srai_epi32(v[1], DCT_CONST_BITS); + lstep1[12] = _mm256_srai_epi32(v[2], DCT_CONST_BITS); + lstep1[13] = _mm256_srai_epi32(v[3], DCT_CONST_BITS); + } + { + const __m256i k32_m08_p24 = + pair256_set_epi32(-cospi_8_64, cospi_24_64); + const __m256i k32_m24_m08 = + pair256_set_epi32(-cospi_24_64, -cospi_8_64); + const __m256i k32_p24_p08 = + pair256_set_epi32(cospi_24_64, cospi_8_64); + + u[0] = _mm256_unpacklo_epi32(lstep3[36], lstep3[58]); + u[1] = _mm256_unpackhi_epi32(lstep3[36], lstep3[58]); + u[2] = _mm256_unpacklo_epi32(lstep3[37], lstep3[59]); + u[3] = _mm256_unpackhi_epi32(lstep3[37], lstep3[59]); + u[4] = _mm256_unpacklo_epi32(lstep3[38], lstep3[56]); + u[5] = _mm256_unpackhi_epi32(lstep3[38], lstep3[56]); + u[6] = _mm256_unpacklo_epi32(lstep3[39], lstep3[57]); + u[7] = _mm256_unpackhi_epi32(lstep3[39], lstep3[57]); + u[8] = _mm256_unpacklo_epi32(lstep3[40], lstep3[54]); + u[9] = _mm256_unpackhi_epi32(lstep3[40], lstep3[54]); + u[10] = _mm256_unpacklo_epi32(lstep3[41], lstep3[55]); + u[11] = _mm256_unpackhi_epi32(lstep3[41], lstep3[55]); + u[12] = _mm256_unpacklo_epi32(lstep3[42], lstep3[52]); + u[13] = _mm256_unpackhi_epi32(lstep3[42], lstep3[52]); + u[14] = _mm256_unpacklo_epi32(lstep3[43], lstep3[53]); + u[15] = _mm256_unpackhi_epi32(lstep3[43], lstep3[53]); + + v[0] = k_madd_epi32_avx2(u[0], k32_m08_p24); + v[1] = k_madd_epi32_avx2(u[1], k32_m08_p24); + v[2] = k_madd_epi32_avx2(u[2], k32_m08_p24); + v[3] = k_madd_epi32_avx2(u[3], k32_m08_p24); + v[4] = k_madd_epi32_avx2(u[4], k32_m08_p24); + v[5] = k_madd_epi32_avx2(u[5], k32_m08_p24); + v[6] = k_madd_epi32_avx2(u[6], k32_m08_p24); + v[7] = k_madd_epi32_avx2(u[7], k32_m08_p24); + v[8] = k_madd_epi32_avx2(u[8], k32_m24_m08); + v[9] = k_madd_epi32_avx2(u[9], k32_m24_m08); + v[10] = k_madd_epi32_avx2(u[10], k32_m24_m08); + v[11] = k_madd_epi32_avx2(u[11], k32_m24_m08); + v[12] = k_madd_epi32_avx2(u[12], k32_m24_m08); + v[13] = k_madd_epi32_avx2(u[13], k32_m24_m08); + v[14] = k_madd_epi32_avx2(u[14], k32_m24_m08); + v[15] = k_madd_epi32_avx2(u[15], k32_m24_m08); + v[16] = k_madd_epi32_avx2(u[12], k32_m08_p24); + v[17] = k_madd_epi32_avx2(u[13], k32_m08_p24); + v[18] = k_madd_epi32_avx2(u[14], k32_m08_p24); + v[19] = k_madd_epi32_avx2(u[15], k32_m08_p24); + v[20] = k_madd_epi32_avx2(u[8], k32_m08_p24); + v[21] = k_madd_epi32_avx2(u[9], k32_m08_p24); + v[22] = k_madd_epi32_avx2(u[10], k32_m08_p24); + v[23] = k_madd_epi32_avx2(u[11], k32_m08_p24); + v[24] = k_madd_epi32_avx2(u[4], k32_p24_p08); + v[25] = k_madd_epi32_avx2(u[5], k32_p24_p08); + v[26] = k_madd_epi32_avx2(u[6], k32_p24_p08); + v[27] = k_madd_epi32_avx2(u[7], k32_p24_p08); + v[28] = k_madd_epi32_avx2(u[0], k32_p24_p08); + v[29] = k_madd_epi32_avx2(u[1], k32_p24_p08); + v[30] = k_madd_epi32_avx2(u[2], k32_p24_p08); + v[31] = k_madd_epi32_avx2(u[3], k32_p24_p08); + + u[0] = k_packs_epi64_avx2(v[0], v[1]); + u[1] = k_packs_epi64_avx2(v[2], v[3]); + u[2] = k_packs_epi64_avx2(v[4], v[5]); + u[3] = k_packs_epi64_avx2(v[6], v[7]); + u[4] = k_packs_epi64_avx2(v[8], v[9]); + u[5] = k_packs_epi64_avx2(v[10], v[11]); + u[6] = k_packs_epi64_avx2(v[12], v[13]); + u[7] = k_packs_epi64_avx2(v[14], v[15]); + u[8] = k_packs_epi64_avx2(v[16], v[17]); + u[9] = k_packs_epi64_avx2(v[18], v[19]); + u[10] = k_packs_epi64_avx2(v[20], v[21]); + u[11] = k_packs_epi64_avx2(v[22], v[23]); + u[12] = k_packs_epi64_avx2(v[24], v[25]); + u[13] = k_packs_epi64_avx2(v[26], v[27]); + u[14] = k_packs_epi64_avx2(v[28], v[29]); + u[15] = k_packs_epi64_avx2(v[30], v[31]); + + v[0] = _mm256_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm256_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm256_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm256_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm256_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm256_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm256_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm256_add_epi32(u[7], k__DCT_CONST_ROUNDING); + v[8] = _mm256_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm256_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm256_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm256_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm256_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm256_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm256_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm256_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + lstep1[36] = _mm256_srai_epi32(v[0], DCT_CONST_BITS); + lstep1[37] = _mm256_srai_epi32(v[1], DCT_CONST_BITS); + lstep1[38] = _mm256_srai_epi32(v[2], DCT_CONST_BITS); + lstep1[39] = _mm256_srai_epi32(v[3], DCT_CONST_BITS); + lstep1[40] = _mm256_srai_epi32(v[4], DCT_CONST_BITS); + lstep1[41] = _mm256_srai_epi32(v[5], DCT_CONST_BITS); + lstep1[42] = _mm256_srai_epi32(v[6], DCT_CONST_BITS); + lstep1[43] = _mm256_srai_epi32(v[7], DCT_CONST_BITS); + lstep1[52] = _mm256_srai_epi32(v[8], DCT_CONST_BITS); + lstep1[53] = _mm256_srai_epi32(v[9], DCT_CONST_BITS); + lstep1[54] = _mm256_srai_epi32(v[10], DCT_CONST_BITS); + lstep1[55] = _mm256_srai_epi32(v[11], DCT_CONST_BITS); + lstep1[56] = _mm256_srai_epi32(v[12], DCT_CONST_BITS); + lstep1[57] = _mm256_srai_epi32(v[13], DCT_CONST_BITS); + lstep1[58] = _mm256_srai_epi32(v[14], DCT_CONST_BITS); + lstep1[59] = _mm256_srai_epi32(v[15], DCT_CONST_BITS); + } + // stage 5 + { + lstep2[8] = _mm256_add_epi32(lstep1[10], lstep3[8]); + lstep2[9] = _mm256_add_epi32(lstep1[11], lstep3[9]); + lstep2[10] = _mm256_sub_epi32(lstep3[8], lstep1[10]); + lstep2[11] = _mm256_sub_epi32(lstep3[9], lstep1[11]); + lstep2[12] = _mm256_sub_epi32(lstep3[14], lstep1[12]); + lstep2[13] = _mm256_sub_epi32(lstep3[15], lstep1[13]); + lstep2[14] = _mm256_add_epi32(lstep1[12], lstep3[14]); + lstep2[15] = _mm256_add_epi32(lstep1[13], lstep3[15]); + } + { + const __m256i k32_p16_p16 = + pair256_set_epi32(cospi_16_64, cospi_16_64); + const __m256i k32_p16_m16 = + pair256_set_epi32(cospi_16_64, -cospi_16_64); + const __m256i k32_p24_p08 = + pair256_set_epi32(cospi_24_64, cospi_8_64); + const __m256i k32_m08_p24 = + pair256_set_epi32(-cospi_8_64, cospi_24_64); + + u[0] = _mm256_unpacklo_epi32(lstep1[0], lstep1[2]); + u[1] = _mm256_unpackhi_epi32(lstep1[0], lstep1[2]); + u[2] = _mm256_unpacklo_epi32(lstep1[1], lstep1[3]); + u[3] = _mm256_unpackhi_epi32(lstep1[1], lstep1[3]); + u[4] = _mm256_unpacklo_epi32(lstep1[4], lstep1[6]); + u[5] = _mm256_unpackhi_epi32(lstep1[4], lstep1[6]); + u[6] = _mm256_unpacklo_epi32(lstep1[5], lstep1[7]); + u[7] = _mm256_unpackhi_epi32(lstep1[5], lstep1[7]); + + // TODO(jingning): manually inline k_madd_epi32_avx2_ to further hide + // instruction latency. + v[0] = k_madd_epi32_avx2(u[0], k32_p16_p16); + v[1] = k_madd_epi32_avx2(u[1], k32_p16_p16); + v[2] = k_madd_epi32_avx2(u[2], k32_p16_p16); + v[3] = k_madd_epi32_avx2(u[3], k32_p16_p16); + v[4] = k_madd_epi32_avx2(u[0], k32_p16_m16); + v[5] = k_madd_epi32_avx2(u[1], k32_p16_m16); + v[6] = k_madd_epi32_avx2(u[2], k32_p16_m16); + v[7] = k_madd_epi32_avx2(u[3], k32_p16_m16); + v[8] = k_madd_epi32_avx2(u[4], k32_p24_p08); + v[9] = k_madd_epi32_avx2(u[5], k32_p24_p08); + v[10] = k_madd_epi32_avx2(u[6], k32_p24_p08); + v[11] = k_madd_epi32_avx2(u[7], k32_p24_p08); + v[12] = k_madd_epi32_avx2(u[4], k32_m08_p24); + v[13] = k_madd_epi32_avx2(u[5], k32_m08_p24); + v[14] = k_madd_epi32_avx2(u[6], k32_m08_p24); + v[15] = k_madd_epi32_avx2(u[7], k32_m08_p24); + + u[0] = k_packs_epi64_avx2(v[0], v[1]); + u[1] = k_packs_epi64_avx2(v[2], v[3]); + u[2] = k_packs_epi64_avx2(v[4], v[5]); + u[3] = k_packs_epi64_avx2(v[6], v[7]); + u[4] = k_packs_epi64_avx2(v[8], v[9]); + u[5] = k_packs_epi64_avx2(v[10], v[11]); + u[6] = k_packs_epi64_avx2(v[12], v[13]); + u[7] = k_packs_epi64_avx2(v[14], v[15]); + + v[0] = _mm256_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm256_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm256_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm256_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm256_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm256_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm256_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm256_add_epi32(u[7], k__DCT_CONST_ROUNDING); + + u[0] = _mm256_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm256_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm256_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm256_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm256_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm256_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm256_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm256_srai_epi32(v[7], DCT_CONST_BITS); + + sign[0] = _mm256_cmpgt_epi32(kZero, u[0]); + sign[1] = _mm256_cmpgt_epi32(kZero, u[1]); + sign[2] = _mm256_cmpgt_epi32(kZero, u[2]); + sign[3] = _mm256_cmpgt_epi32(kZero, u[3]); + sign[4] = _mm256_cmpgt_epi32(kZero, u[4]); + sign[5] = _mm256_cmpgt_epi32(kZero, u[5]); + sign[6] = _mm256_cmpgt_epi32(kZero, u[6]); + sign[7] = _mm256_cmpgt_epi32(kZero, u[7]); + + u[0] = _mm256_sub_epi32(u[0], sign[0]); + u[1] = _mm256_sub_epi32(u[1], sign[1]); + u[2] = _mm256_sub_epi32(u[2], sign[2]); + u[3] = _mm256_sub_epi32(u[3], sign[3]); + u[4] = _mm256_sub_epi32(u[4], sign[4]); + u[5] = _mm256_sub_epi32(u[5], sign[5]); + u[6] = _mm256_sub_epi32(u[6], sign[6]); + u[7] = _mm256_sub_epi32(u[7], sign[7]); + + u[0] = _mm256_add_epi32(u[0], K32One); + u[1] = _mm256_add_epi32(u[1], K32One); + u[2] = _mm256_add_epi32(u[2], K32One); + u[3] = _mm256_add_epi32(u[3], K32One); + u[4] = _mm256_add_epi32(u[4], K32One); + u[5] = _mm256_add_epi32(u[5], K32One); + u[6] = _mm256_add_epi32(u[6], K32One); + u[7] = _mm256_add_epi32(u[7], K32One); + + u[0] = _mm256_srai_epi32(u[0], 2); + u[1] = _mm256_srai_epi32(u[1], 2); + u[2] = _mm256_srai_epi32(u[2], 2); + u[3] = _mm256_srai_epi32(u[3], 2); + u[4] = _mm256_srai_epi32(u[4], 2); + u[5] = _mm256_srai_epi32(u[5], 2); + u[6] = _mm256_srai_epi32(u[6], 2); + u[7] = _mm256_srai_epi32(u[7], 2); + + // Combine + out[0] = _mm256_packs_epi32(u[0], u[1]); + out[16] = _mm256_packs_epi32(u[2], u[3]); + out[8] = _mm256_packs_epi32(u[4], u[5]); + out[24] = _mm256_packs_epi32(u[6], u[7]); + } + { + const __m256i k32_m08_p24 = + pair256_set_epi32(-cospi_8_64, cospi_24_64); + const __m256i k32_m24_m08 = + pair256_set_epi32(-cospi_24_64, -cospi_8_64); + const __m256i k32_p24_p08 = + pair256_set_epi32(cospi_24_64, cospi_8_64); + + u[0] = _mm256_unpacklo_epi32(lstep1[18], lstep1[28]); + u[1] = _mm256_unpackhi_epi32(lstep1[18], lstep1[28]); + u[2] = _mm256_unpacklo_epi32(lstep1[19], lstep1[29]); + u[3] = _mm256_unpackhi_epi32(lstep1[19], lstep1[29]); + u[4] = _mm256_unpacklo_epi32(lstep1[20], lstep1[26]); + u[5] = _mm256_unpackhi_epi32(lstep1[20], lstep1[26]); + u[6] = _mm256_unpacklo_epi32(lstep1[21], lstep1[27]); + u[7] = _mm256_unpackhi_epi32(lstep1[21], lstep1[27]); + + v[0] = k_madd_epi32_avx2(u[0], k32_m08_p24); + v[1] = k_madd_epi32_avx2(u[1], k32_m08_p24); + v[2] = k_madd_epi32_avx2(u[2], k32_m08_p24); + v[3] = k_madd_epi32_avx2(u[3], k32_m08_p24); + v[4] = k_madd_epi32_avx2(u[4], k32_m24_m08); + v[5] = k_madd_epi32_avx2(u[5], k32_m24_m08); + v[6] = k_madd_epi32_avx2(u[6], k32_m24_m08); + v[7] = k_madd_epi32_avx2(u[7], k32_m24_m08); + v[8] = k_madd_epi32_avx2(u[4], k32_m08_p24); + v[9] = k_madd_epi32_avx2(u[5], k32_m08_p24); + v[10] = k_madd_epi32_avx2(u[6], k32_m08_p24); + v[11] = k_madd_epi32_avx2(u[7], k32_m08_p24); + v[12] = k_madd_epi32_avx2(u[0], k32_p24_p08); + v[13] = k_madd_epi32_avx2(u[1], k32_p24_p08); + v[14] = k_madd_epi32_avx2(u[2], k32_p24_p08); + v[15] = k_madd_epi32_avx2(u[3], k32_p24_p08); + + u[0] = k_packs_epi64_avx2(v[0], v[1]); + u[1] = k_packs_epi64_avx2(v[2], v[3]); + u[2] = k_packs_epi64_avx2(v[4], v[5]); + u[3] = k_packs_epi64_avx2(v[6], v[7]); + u[4] = k_packs_epi64_avx2(v[8], v[9]); + u[5] = k_packs_epi64_avx2(v[10], v[11]); + u[6] = k_packs_epi64_avx2(v[12], v[13]); + u[7] = k_packs_epi64_avx2(v[14], v[15]); + + u[0] = _mm256_add_epi32(u[0], k__DCT_CONST_ROUNDING); + u[1] = _mm256_add_epi32(u[1], k__DCT_CONST_ROUNDING); + u[2] = _mm256_add_epi32(u[2], k__DCT_CONST_ROUNDING); + u[3] = _mm256_add_epi32(u[3], k__DCT_CONST_ROUNDING); + u[4] = _mm256_add_epi32(u[4], k__DCT_CONST_ROUNDING); + u[5] = _mm256_add_epi32(u[5], k__DCT_CONST_ROUNDING); + u[6] = _mm256_add_epi32(u[6], k__DCT_CONST_ROUNDING); + u[7] = _mm256_add_epi32(u[7], k__DCT_CONST_ROUNDING); + + lstep2[18] = _mm256_srai_epi32(u[0], DCT_CONST_BITS); + lstep2[19] = _mm256_srai_epi32(u[1], DCT_CONST_BITS); + lstep2[20] = _mm256_srai_epi32(u[2], DCT_CONST_BITS); + lstep2[21] = _mm256_srai_epi32(u[3], DCT_CONST_BITS); + lstep2[26] = _mm256_srai_epi32(u[4], DCT_CONST_BITS); + lstep2[27] = _mm256_srai_epi32(u[5], DCT_CONST_BITS); + lstep2[28] = _mm256_srai_epi32(u[6], DCT_CONST_BITS); + lstep2[29] = _mm256_srai_epi32(u[7], DCT_CONST_BITS); + } + { + lstep2[32] = _mm256_add_epi32(lstep1[38], lstep3[32]); + lstep2[33] = _mm256_add_epi32(lstep1[39], lstep3[33]); + lstep2[34] = _mm256_add_epi32(lstep1[36], lstep3[34]); + lstep2[35] = _mm256_add_epi32(lstep1[37], lstep3[35]); + lstep2[36] = _mm256_sub_epi32(lstep3[34], lstep1[36]); + lstep2[37] = _mm256_sub_epi32(lstep3[35], lstep1[37]); + lstep2[38] = _mm256_sub_epi32(lstep3[32], lstep1[38]); + lstep2[39] = _mm256_sub_epi32(lstep3[33], lstep1[39]); + lstep2[40] = _mm256_sub_epi32(lstep3[46], lstep1[40]); + lstep2[41] = _mm256_sub_epi32(lstep3[47], lstep1[41]); + lstep2[42] = _mm256_sub_epi32(lstep3[44], lstep1[42]); + lstep2[43] = _mm256_sub_epi32(lstep3[45], lstep1[43]); + lstep2[44] = _mm256_add_epi32(lstep1[42], lstep3[44]); + lstep2[45] = _mm256_add_epi32(lstep1[43], lstep3[45]); + lstep2[46] = _mm256_add_epi32(lstep1[40], lstep3[46]); + lstep2[47] = _mm256_add_epi32(lstep1[41], lstep3[47]); + lstep2[48] = _mm256_add_epi32(lstep1[54], lstep3[48]); + lstep2[49] = _mm256_add_epi32(lstep1[55], lstep3[49]); + lstep2[50] = _mm256_add_epi32(lstep1[52], lstep3[50]); + lstep2[51] = _mm256_add_epi32(lstep1[53], lstep3[51]); + lstep2[52] = _mm256_sub_epi32(lstep3[50], lstep1[52]); + lstep2[53] = _mm256_sub_epi32(lstep3[51], lstep1[53]); + lstep2[54] = _mm256_sub_epi32(lstep3[48], lstep1[54]); + lstep2[55] = _mm256_sub_epi32(lstep3[49], lstep1[55]); + lstep2[56] = _mm256_sub_epi32(lstep3[62], lstep1[56]); + lstep2[57] = _mm256_sub_epi32(lstep3[63], lstep1[57]); + lstep2[58] = _mm256_sub_epi32(lstep3[60], lstep1[58]); + lstep2[59] = _mm256_sub_epi32(lstep3[61], lstep1[59]); + lstep2[60] = _mm256_add_epi32(lstep1[58], lstep3[60]); + lstep2[61] = _mm256_add_epi32(lstep1[59], lstep3[61]); + lstep2[62] = _mm256_add_epi32(lstep1[56], lstep3[62]); + lstep2[63] = _mm256_add_epi32(lstep1[57], lstep3[63]); + } + // stage 6 + { + const __m256i k32_p28_p04 = + pair256_set_epi32(cospi_28_64, cospi_4_64); + const __m256i k32_p12_p20 = + pair256_set_epi32(cospi_12_64, cospi_20_64); + const __m256i k32_m20_p12 = + pair256_set_epi32(-cospi_20_64, cospi_12_64); + const __m256i k32_m04_p28 = + pair256_set_epi32(-cospi_4_64, cospi_28_64); + + u[0] = _mm256_unpacklo_epi32(lstep2[8], lstep2[14]); + u[1] = _mm256_unpackhi_epi32(lstep2[8], lstep2[14]); + u[2] = _mm256_unpacklo_epi32(lstep2[9], lstep2[15]); + u[3] = _mm256_unpackhi_epi32(lstep2[9], lstep2[15]); + u[4] = _mm256_unpacklo_epi32(lstep2[10], lstep2[12]); + u[5] = _mm256_unpackhi_epi32(lstep2[10], lstep2[12]); + u[6] = _mm256_unpacklo_epi32(lstep2[11], lstep2[13]); + u[7] = _mm256_unpackhi_epi32(lstep2[11], lstep2[13]); + u[8] = _mm256_unpacklo_epi32(lstep2[10], lstep2[12]); + u[9] = _mm256_unpackhi_epi32(lstep2[10], lstep2[12]); + u[10] = _mm256_unpacklo_epi32(lstep2[11], lstep2[13]); + u[11] = _mm256_unpackhi_epi32(lstep2[11], lstep2[13]); + u[12] = _mm256_unpacklo_epi32(lstep2[8], lstep2[14]); + u[13] = _mm256_unpackhi_epi32(lstep2[8], lstep2[14]); + u[14] = _mm256_unpacklo_epi32(lstep2[9], lstep2[15]); + u[15] = _mm256_unpackhi_epi32(lstep2[9], lstep2[15]); + + v[0] = k_madd_epi32_avx2(u[0], k32_p28_p04); + v[1] = k_madd_epi32_avx2(u[1], k32_p28_p04); + v[2] = k_madd_epi32_avx2(u[2], k32_p28_p04); + v[3] = k_madd_epi32_avx2(u[3], k32_p28_p04); + v[4] = k_madd_epi32_avx2(u[4], k32_p12_p20); + v[5] = k_madd_epi32_avx2(u[5], k32_p12_p20); + v[6] = k_madd_epi32_avx2(u[6], k32_p12_p20); + v[7] = k_madd_epi32_avx2(u[7], k32_p12_p20); + v[8] = k_madd_epi32_avx2(u[8], k32_m20_p12); + v[9] = k_madd_epi32_avx2(u[9], k32_m20_p12); + v[10] = k_madd_epi32_avx2(u[10], k32_m20_p12); + v[11] = k_madd_epi32_avx2(u[11], k32_m20_p12); + v[12] = k_madd_epi32_avx2(u[12], k32_m04_p28); + v[13] = k_madd_epi32_avx2(u[13], k32_m04_p28); + v[14] = k_madd_epi32_avx2(u[14], k32_m04_p28); + v[15] = k_madd_epi32_avx2(u[15], k32_m04_p28); + + u[0] = k_packs_epi64_avx2(v[0], v[1]); + u[1] = k_packs_epi64_avx2(v[2], v[3]); + u[2] = k_packs_epi64_avx2(v[4], v[5]); + u[3] = k_packs_epi64_avx2(v[6], v[7]); + u[4] = k_packs_epi64_avx2(v[8], v[9]); + u[5] = k_packs_epi64_avx2(v[10], v[11]); + u[6] = k_packs_epi64_avx2(v[12], v[13]); + u[7] = k_packs_epi64_avx2(v[14], v[15]); + + v[0] = _mm256_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm256_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm256_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm256_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm256_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm256_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm256_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm256_add_epi32(u[7], k__DCT_CONST_ROUNDING); + + u[0] = _mm256_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm256_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm256_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm256_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm256_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm256_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm256_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm256_srai_epi32(v[7], DCT_CONST_BITS); + + sign[0] = _mm256_cmpgt_epi32(kZero, u[0]); + sign[1] = _mm256_cmpgt_epi32(kZero, u[1]); + sign[2] = _mm256_cmpgt_epi32(kZero, u[2]); + sign[3] = _mm256_cmpgt_epi32(kZero, u[3]); + sign[4] = _mm256_cmpgt_epi32(kZero, u[4]); + sign[5] = _mm256_cmpgt_epi32(kZero, u[5]); + sign[6] = _mm256_cmpgt_epi32(kZero, u[6]); + sign[7] = _mm256_cmpgt_epi32(kZero, u[7]); + + u[0] = _mm256_sub_epi32(u[0], sign[0]); + u[1] = _mm256_sub_epi32(u[1], sign[1]); + u[2] = _mm256_sub_epi32(u[2], sign[2]); + u[3] = _mm256_sub_epi32(u[3], sign[3]); + u[4] = _mm256_sub_epi32(u[4], sign[4]); + u[5] = _mm256_sub_epi32(u[5], sign[5]); + u[6] = _mm256_sub_epi32(u[6], sign[6]); + u[7] = _mm256_sub_epi32(u[7], sign[7]); + + u[0] = _mm256_add_epi32(u[0], K32One); + u[1] = _mm256_add_epi32(u[1], K32One); + u[2] = _mm256_add_epi32(u[2], K32One); + u[3] = _mm256_add_epi32(u[3], K32One); + u[4] = _mm256_add_epi32(u[4], K32One); + u[5] = _mm256_add_epi32(u[5], K32One); + u[6] = _mm256_add_epi32(u[6], K32One); + u[7] = _mm256_add_epi32(u[7], K32One); + + u[0] = _mm256_srai_epi32(u[0], 2); + u[1] = _mm256_srai_epi32(u[1], 2); + u[2] = _mm256_srai_epi32(u[2], 2); + u[3] = _mm256_srai_epi32(u[3], 2); + u[4] = _mm256_srai_epi32(u[4], 2); + u[5] = _mm256_srai_epi32(u[5], 2); + u[6] = _mm256_srai_epi32(u[6], 2); + u[7] = _mm256_srai_epi32(u[7], 2); + + out[4] = _mm256_packs_epi32(u[0], u[1]); + out[20] = _mm256_packs_epi32(u[2], u[3]); + out[12] = _mm256_packs_epi32(u[4], u[5]); + out[28] = _mm256_packs_epi32(u[6], u[7]); + } + { + lstep3[16] = _mm256_add_epi32(lstep2[18], lstep1[16]); + lstep3[17] = _mm256_add_epi32(lstep2[19], lstep1[17]); + lstep3[18] = _mm256_sub_epi32(lstep1[16], lstep2[18]); + lstep3[19] = _mm256_sub_epi32(lstep1[17], lstep2[19]); + lstep3[20] = _mm256_sub_epi32(lstep1[22], lstep2[20]); + lstep3[21] = _mm256_sub_epi32(lstep1[23], lstep2[21]); + lstep3[22] = _mm256_add_epi32(lstep2[20], lstep1[22]); + lstep3[23] = _mm256_add_epi32(lstep2[21], lstep1[23]); + lstep3[24] = _mm256_add_epi32(lstep2[26], lstep1[24]); + lstep3[25] = _mm256_add_epi32(lstep2[27], lstep1[25]); + lstep3[26] = _mm256_sub_epi32(lstep1[24], lstep2[26]); + lstep3[27] = _mm256_sub_epi32(lstep1[25], lstep2[27]); + lstep3[28] = _mm256_sub_epi32(lstep1[30], lstep2[28]); + lstep3[29] = _mm256_sub_epi32(lstep1[31], lstep2[29]); + lstep3[30] = _mm256_add_epi32(lstep2[28], lstep1[30]); + lstep3[31] = _mm256_add_epi32(lstep2[29], lstep1[31]); + } + { + const __m256i k32_m04_p28 = + pair256_set_epi32(-cospi_4_64, cospi_28_64); + const __m256i k32_m28_m04 = + pair256_set_epi32(-cospi_28_64, -cospi_4_64); + const __m256i k32_m20_p12 = + pair256_set_epi32(-cospi_20_64, cospi_12_64); + const __m256i k32_m12_m20 = + pair256_set_epi32(-cospi_12_64, -cospi_20_64); + const __m256i k32_p12_p20 = + pair256_set_epi32(cospi_12_64, cospi_20_64); + const __m256i k32_p28_p04 = + pair256_set_epi32(cospi_28_64, cospi_4_64); + + u[0] = _mm256_unpacklo_epi32(lstep2[34], lstep2[60]); + u[1] = _mm256_unpackhi_epi32(lstep2[34], lstep2[60]); + u[2] = _mm256_unpacklo_epi32(lstep2[35], lstep2[61]); + u[3] = _mm256_unpackhi_epi32(lstep2[35], lstep2[61]); + u[4] = _mm256_unpacklo_epi32(lstep2[36], lstep2[58]); + u[5] = _mm256_unpackhi_epi32(lstep2[36], lstep2[58]); + u[6] = _mm256_unpacklo_epi32(lstep2[37], lstep2[59]); + u[7] = _mm256_unpackhi_epi32(lstep2[37], lstep2[59]); + u[8] = _mm256_unpacklo_epi32(lstep2[42], lstep2[52]); + u[9] = _mm256_unpackhi_epi32(lstep2[42], lstep2[52]); + u[10] = _mm256_unpacklo_epi32(lstep2[43], lstep2[53]); + u[11] = _mm256_unpackhi_epi32(lstep2[43], lstep2[53]); + u[12] = _mm256_unpacklo_epi32(lstep2[44], lstep2[50]); + u[13] = _mm256_unpackhi_epi32(lstep2[44], lstep2[50]); + u[14] = _mm256_unpacklo_epi32(lstep2[45], lstep2[51]); + u[15] = _mm256_unpackhi_epi32(lstep2[45], lstep2[51]); + + v[0] = k_madd_epi32_avx2(u[0], k32_m04_p28); + v[1] = k_madd_epi32_avx2(u[1], k32_m04_p28); + v[2] = k_madd_epi32_avx2(u[2], k32_m04_p28); + v[3] = k_madd_epi32_avx2(u[3], k32_m04_p28); + v[4] = k_madd_epi32_avx2(u[4], k32_m28_m04); + v[5] = k_madd_epi32_avx2(u[5], k32_m28_m04); + v[6] = k_madd_epi32_avx2(u[6], k32_m28_m04); + v[7] = k_madd_epi32_avx2(u[7], k32_m28_m04); + v[8] = k_madd_epi32_avx2(u[8], k32_m20_p12); + v[9] = k_madd_epi32_avx2(u[9], k32_m20_p12); + v[10] = k_madd_epi32_avx2(u[10], k32_m20_p12); + v[11] = k_madd_epi32_avx2(u[11], k32_m20_p12); + v[12] = k_madd_epi32_avx2(u[12], k32_m12_m20); + v[13] = k_madd_epi32_avx2(u[13], k32_m12_m20); + v[14] = k_madd_epi32_avx2(u[14], k32_m12_m20); + v[15] = k_madd_epi32_avx2(u[15], k32_m12_m20); + v[16] = k_madd_epi32_avx2(u[12], k32_m20_p12); + v[17] = k_madd_epi32_avx2(u[13], k32_m20_p12); + v[18] = k_madd_epi32_avx2(u[14], k32_m20_p12); + v[19] = k_madd_epi32_avx2(u[15], k32_m20_p12); + v[20] = k_madd_epi32_avx2(u[8], k32_p12_p20); + v[21] = k_madd_epi32_avx2(u[9], k32_p12_p20); + v[22] = k_madd_epi32_avx2(u[10], k32_p12_p20); + v[23] = k_madd_epi32_avx2(u[11], k32_p12_p20); + v[24] = k_madd_epi32_avx2(u[4], k32_m04_p28); + v[25] = k_madd_epi32_avx2(u[5], k32_m04_p28); + v[26] = k_madd_epi32_avx2(u[6], k32_m04_p28); + v[27] = k_madd_epi32_avx2(u[7], k32_m04_p28); + v[28] = k_madd_epi32_avx2(u[0], k32_p28_p04); + v[29] = k_madd_epi32_avx2(u[1], k32_p28_p04); + v[30] = k_madd_epi32_avx2(u[2], k32_p28_p04); + v[31] = k_madd_epi32_avx2(u[3], k32_p28_p04); + + u[0] = k_packs_epi64_avx2(v[0], v[1]); + u[1] = k_packs_epi64_avx2(v[2], v[3]); + u[2] = k_packs_epi64_avx2(v[4], v[5]); + u[3] = k_packs_epi64_avx2(v[6], v[7]); + u[4] = k_packs_epi64_avx2(v[8], v[9]); + u[5] = k_packs_epi64_avx2(v[10], v[11]); + u[6] = k_packs_epi64_avx2(v[12], v[13]); + u[7] = k_packs_epi64_avx2(v[14], v[15]); + u[8] = k_packs_epi64_avx2(v[16], v[17]); + u[9] = k_packs_epi64_avx2(v[18], v[19]); + u[10] = k_packs_epi64_avx2(v[20], v[21]); + u[11] = k_packs_epi64_avx2(v[22], v[23]); + u[12] = k_packs_epi64_avx2(v[24], v[25]); + u[13] = k_packs_epi64_avx2(v[26], v[27]); + u[14] = k_packs_epi64_avx2(v[28], v[29]); + u[15] = k_packs_epi64_avx2(v[30], v[31]); + + v[0] = _mm256_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm256_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm256_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm256_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm256_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm256_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm256_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm256_add_epi32(u[7], k__DCT_CONST_ROUNDING); + v[8] = _mm256_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm256_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm256_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm256_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm256_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm256_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm256_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm256_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + lstep3[34] = _mm256_srai_epi32(v[0], DCT_CONST_BITS); + lstep3[35] = _mm256_srai_epi32(v[1], DCT_CONST_BITS); + lstep3[36] = _mm256_srai_epi32(v[2], DCT_CONST_BITS); + lstep3[37] = _mm256_srai_epi32(v[3], DCT_CONST_BITS); + lstep3[42] = _mm256_srai_epi32(v[4], DCT_CONST_BITS); + lstep3[43] = _mm256_srai_epi32(v[5], DCT_CONST_BITS); + lstep3[44] = _mm256_srai_epi32(v[6], DCT_CONST_BITS); + lstep3[45] = _mm256_srai_epi32(v[7], DCT_CONST_BITS); + lstep3[50] = _mm256_srai_epi32(v[8], DCT_CONST_BITS); + lstep3[51] = _mm256_srai_epi32(v[9], DCT_CONST_BITS); + lstep3[52] = _mm256_srai_epi32(v[10], DCT_CONST_BITS); + lstep3[53] = _mm256_srai_epi32(v[11], DCT_CONST_BITS); + lstep3[58] = _mm256_srai_epi32(v[12], DCT_CONST_BITS); + lstep3[59] = _mm256_srai_epi32(v[13], DCT_CONST_BITS); + lstep3[60] = _mm256_srai_epi32(v[14], DCT_CONST_BITS); + lstep3[61] = _mm256_srai_epi32(v[15], DCT_CONST_BITS); + } + // stage 7 + { + const __m256i k32_p30_p02 = + pair256_set_epi32(cospi_30_64, cospi_2_64); + const __m256i k32_p14_p18 = + pair256_set_epi32(cospi_14_64, cospi_18_64); + const __m256i k32_p22_p10 = + pair256_set_epi32(cospi_22_64, cospi_10_64); + const __m256i k32_p06_p26 = + pair256_set_epi32(cospi_6_64, cospi_26_64); + const __m256i k32_m26_p06 = + pair256_set_epi32(-cospi_26_64, cospi_6_64); + const __m256i k32_m10_p22 = + pair256_set_epi32(-cospi_10_64, cospi_22_64); + const __m256i k32_m18_p14 = + pair256_set_epi32(-cospi_18_64, cospi_14_64); + const __m256i k32_m02_p30 = + pair256_set_epi32(-cospi_2_64, cospi_30_64); + + u[0] = _mm256_unpacklo_epi32(lstep3[16], lstep3[30]); + u[1] = _mm256_unpackhi_epi32(lstep3[16], lstep3[30]); + u[2] = _mm256_unpacklo_epi32(lstep3[17], lstep3[31]); + u[3] = _mm256_unpackhi_epi32(lstep3[17], lstep3[31]); + u[4] = _mm256_unpacklo_epi32(lstep3[18], lstep3[28]); + u[5] = _mm256_unpackhi_epi32(lstep3[18], lstep3[28]); + u[6] = _mm256_unpacklo_epi32(lstep3[19], lstep3[29]); + u[7] = _mm256_unpackhi_epi32(lstep3[19], lstep3[29]); + u[8] = _mm256_unpacklo_epi32(lstep3[20], lstep3[26]); + u[9] = _mm256_unpackhi_epi32(lstep3[20], lstep3[26]); + u[10] = _mm256_unpacklo_epi32(lstep3[21], lstep3[27]); + u[11] = _mm256_unpackhi_epi32(lstep3[21], lstep3[27]); + u[12] = _mm256_unpacklo_epi32(lstep3[22], lstep3[24]); + u[13] = _mm256_unpackhi_epi32(lstep3[22], lstep3[24]); + u[14] = _mm256_unpacklo_epi32(lstep3[23], lstep3[25]); + u[15] = _mm256_unpackhi_epi32(lstep3[23], lstep3[25]); + + v[0] = k_madd_epi32_avx2(u[0], k32_p30_p02); + v[1] = k_madd_epi32_avx2(u[1], k32_p30_p02); + v[2] = k_madd_epi32_avx2(u[2], k32_p30_p02); + v[3] = k_madd_epi32_avx2(u[3], k32_p30_p02); + v[4] = k_madd_epi32_avx2(u[4], k32_p14_p18); + v[5] = k_madd_epi32_avx2(u[5], k32_p14_p18); + v[6] = k_madd_epi32_avx2(u[6], k32_p14_p18); + v[7] = k_madd_epi32_avx2(u[7], k32_p14_p18); + v[8] = k_madd_epi32_avx2(u[8], k32_p22_p10); + v[9] = k_madd_epi32_avx2(u[9], k32_p22_p10); + v[10] = k_madd_epi32_avx2(u[10], k32_p22_p10); + v[11] = k_madd_epi32_avx2(u[11], k32_p22_p10); + v[12] = k_madd_epi32_avx2(u[12], k32_p06_p26); + v[13] = k_madd_epi32_avx2(u[13], k32_p06_p26); + v[14] = k_madd_epi32_avx2(u[14], k32_p06_p26); + v[15] = k_madd_epi32_avx2(u[15], k32_p06_p26); + v[16] = k_madd_epi32_avx2(u[12], k32_m26_p06); + v[17] = k_madd_epi32_avx2(u[13], k32_m26_p06); + v[18] = k_madd_epi32_avx2(u[14], k32_m26_p06); + v[19] = k_madd_epi32_avx2(u[15], k32_m26_p06); + v[20] = k_madd_epi32_avx2(u[8], k32_m10_p22); + v[21] = k_madd_epi32_avx2(u[9], k32_m10_p22); + v[22] = k_madd_epi32_avx2(u[10], k32_m10_p22); + v[23] = k_madd_epi32_avx2(u[11], k32_m10_p22); + v[24] = k_madd_epi32_avx2(u[4], k32_m18_p14); + v[25] = k_madd_epi32_avx2(u[5], k32_m18_p14); + v[26] = k_madd_epi32_avx2(u[6], k32_m18_p14); + v[27] = k_madd_epi32_avx2(u[7], k32_m18_p14); + v[28] = k_madd_epi32_avx2(u[0], k32_m02_p30); + v[29] = k_madd_epi32_avx2(u[1], k32_m02_p30); + v[30] = k_madd_epi32_avx2(u[2], k32_m02_p30); + v[31] = k_madd_epi32_avx2(u[3], k32_m02_p30); + + u[0] = k_packs_epi64_avx2(v[0], v[1]); + u[1] = k_packs_epi64_avx2(v[2], v[3]); + u[2] = k_packs_epi64_avx2(v[4], v[5]); + u[3] = k_packs_epi64_avx2(v[6], v[7]); + u[4] = k_packs_epi64_avx2(v[8], v[9]); + u[5] = k_packs_epi64_avx2(v[10], v[11]); + u[6] = k_packs_epi64_avx2(v[12], v[13]); + u[7] = k_packs_epi64_avx2(v[14], v[15]); + u[8] = k_packs_epi64_avx2(v[16], v[17]); + u[9] = k_packs_epi64_avx2(v[18], v[19]); + u[10] = k_packs_epi64_avx2(v[20], v[21]); + u[11] = k_packs_epi64_avx2(v[22], v[23]); + u[12] = k_packs_epi64_avx2(v[24], v[25]); + u[13] = k_packs_epi64_avx2(v[26], v[27]); + u[14] = k_packs_epi64_avx2(v[28], v[29]); + u[15] = k_packs_epi64_avx2(v[30], v[31]); + + v[0] = _mm256_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm256_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm256_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm256_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm256_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm256_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm256_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm256_add_epi32(u[7], k__DCT_CONST_ROUNDING); + v[8] = _mm256_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm256_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm256_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm256_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm256_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm256_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm256_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm256_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + u[0] = _mm256_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm256_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm256_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm256_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm256_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm256_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm256_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm256_srai_epi32(v[7], DCT_CONST_BITS); + u[8] = _mm256_srai_epi32(v[8], DCT_CONST_BITS); + u[9] = _mm256_srai_epi32(v[9], DCT_CONST_BITS); + u[10] = _mm256_srai_epi32(v[10], DCT_CONST_BITS); + u[11] = _mm256_srai_epi32(v[11], DCT_CONST_BITS); + u[12] = _mm256_srai_epi32(v[12], DCT_CONST_BITS); + u[13] = _mm256_srai_epi32(v[13], DCT_CONST_BITS); + u[14] = _mm256_srai_epi32(v[14], DCT_CONST_BITS); + u[15] = _mm256_srai_epi32(v[15], DCT_CONST_BITS); + + v[0] = _mm256_cmpgt_epi32(kZero, u[0]); + v[1] = _mm256_cmpgt_epi32(kZero, u[1]); + v[2] = _mm256_cmpgt_epi32(kZero, u[2]); + v[3] = _mm256_cmpgt_epi32(kZero, u[3]); + v[4] = _mm256_cmpgt_epi32(kZero, u[4]); + v[5] = _mm256_cmpgt_epi32(kZero, u[5]); + v[6] = _mm256_cmpgt_epi32(kZero, u[6]); + v[7] = _mm256_cmpgt_epi32(kZero, u[7]); + v[8] = _mm256_cmpgt_epi32(kZero, u[8]); + v[9] = _mm256_cmpgt_epi32(kZero, u[9]); + v[10] = _mm256_cmpgt_epi32(kZero, u[10]); + v[11] = _mm256_cmpgt_epi32(kZero, u[11]); + v[12] = _mm256_cmpgt_epi32(kZero, u[12]); + v[13] = _mm256_cmpgt_epi32(kZero, u[13]); + v[14] = _mm256_cmpgt_epi32(kZero, u[14]); + v[15] = _mm256_cmpgt_epi32(kZero, u[15]); + + u[0] = _mm256_sub_epi32(u[0], v[0]); + u[1] = _mm256_sub_epi32(u[1], v[1]); + u[2] = _mm256_sub_epi32(u[2], v[2]); + u[3] = _mm256_sub_epi32(u[3], v[3]); + u[4] = _mm256_sub_epi32(u[4], v[4]); + u[5] = _mm256_sub_epi32(u[5], v[5]); + u[6] = _mm256_sub_epi32(u[6], v[6]); + u[7] = _mm256_sub_epi32(u[7], v[7]); + u[8] = _mm256_sub_epi32(u[8], v[8]); + u[9] = _mm256_sub_epi32(u[9], v[9]); + u[10] = _mm256_sub_epi32(u[10], v[10]); + u[11] = _mm256_sub_epi32(u[11], v[11]); + u[12] = _mm256_sub_epi32(u[12], v[12]); + u[13] = _mm256_sub_epi32(u[13], v[13]); + u[14] = _mm256_sub_epi32(u[14], v[14]); + u[15] = _mm256_sub_epi32(u[15], v[15]); + + v[0] = _mm256_add_epi32(u[0], K32One); + v[1] = _mm256_add_epi32(u[1], K32One); + v[2] = _mm256_add_epi32(u[2], K32One); + v[3] = _mm256_add_epi32(u[3], K32One); + v[4] = _mm256_add_epi32(u[4], K32One); + v[5] = _mm256_add_epi32(u[5], K32One); + v[6] = _mm256_add_epi32(u[6], K32One); + v[7] = _mm256_add_epi32(u[7], K32One); + v[8] = _mm256_add_epi32(u[8], K32One); + v[9] = _mm256_add_epi32(u[9], K32One); + v[10] = _mm256_add_epi32(u[10], K32One); + v[11] = _mm256_add_epi32(u[11], K32One); + v[12] = _mm256_add_epi32(u[12], K32One); + v[13] = _mm256_add_epi32(u[13], K32One); + v[14] = _mm256_add_epi32(u[14], K32One); + v[15] = _mm256_add_epi32(u[15], K32One); + + u[0] = _mm256_srai_epi32(v[0], 2); + u[1] = _mm256_srai_epi32(v[1], 2); + u[2] = _mm256_srai_epi32(v[2], 2); + u[3] = _mm256_srai_epi32(v[3], 2); + u[4] = _mm256_srai_epi32(v[4], 2); + u[5] = _mm256_srai_epi32(v[5], 2); + u[6] = _mm256_srai_epi32(v[6], 2); + u[7] = _mm256_srai_epi32(v[7], 2); + u[8] = _mm256_srai_epi32(v[8], 2); + u[9] = _mm256_srai_epi32(v[9], 2); + u[10] = _mm256_srai_epi32(v[10], 2); + u[11] = _mm256_srai_epi32(v[11], 2); + u[12] = _mm256_srai_epi32(v[12], 2); + u[13] = _mm256_srai_epi32(v[13], 2); + u[14] = _mm256_srai_epi32(v[14], 2); + u[15] = _mm256_srai_epi32(v[15], 2); + + out[2] = _mm256_packs_epi32(u[0], u[1]); + out[18] = _mm256_packs_epi32(u[2], u[3]); + out[10] = _mm256_packs_epi32(u[4], u[5]); + out[26] = _mm256_packs_epi32(u[6], u[7]); + out[6] = _mm256_packs_epi32(u[8], u[9]); + out[22] = _mm256_packs_epi32(u[10], u[11]); + out[14] = _mm256_packs_epi32(u[12], u[13]); + out[30] = _mm256_packs_epi32(u[14], u[15]); + } + { + lstep1[32] = _mm256_add_epi32(lstep3[34], lstep2[32]); + lstep1[33] = _mm256_add_epi32(lstep3[35], lstep2[33]); + lstep1[34] = _mm256_sub_epi32(lstep2[32], lstep3[34]); + lstep1[35] = _mm256_sub_epi32(lstep2[33], lstep3[35]); + lstep1[36] = _mm256_sub_epi32(lstep2[38], lstep3[36]); + lstep1[37] = _mm256_sub_epi32(lstep2[39], lstep3[37]); + lstep1[38] = _mm256_add_epi32(lstep3[36], lstep2[38]); + lstep1[39] = _mm256_add_epi32(lstep3[37], lstep2[39]); + lstep1[40] = _mm256_add_epi32(lstep3[42], lstep2[40]); + lstep1[41] = _mm256_add_epi32(lstep3[43], lstep2[41]); + lstep1[42] = _mm256_sub_epi32(lstep2[40], lstep3[42]); + lstep1[43] = _mm256_sub_epi32(lstep2[41], lstep3[43]); + lstep1[44] = _mm256_sub_epi32(lstep2[46], lstep3[44]); + lstep1[45] = _mm256_sub_epi32(lstep2[47], lstep3[45]); + lstep1[46] = _mm256_add_epi32(lstep3[44], lstep2[46]); + lstep1[47] = _mm256_add_epi32(lstep3[45], lstep2[47]); + lstep1[48] = _mm256_add_epi32(lstep3[50], lstep2[48]); + lstep1[49] = _mm256_add_epi32(lstep3[51], lstep2[49]); + lstep1[50] = _mm256_sub_epi32(lstep2[48], lstep3[50]); + lstep1[51] = _mm256_sub_epi32(lstep2[49], lstep3[51]); + lstep1[52] = _mm256_sub_epi32(lstep2[54], lstep3[52]); + lstep1[53] = _mm256_sub_epi32(lstep2[55], lstep3[53]); + lstep1[54] = _mm256_add_epi32(lstep3[52], lstep2[54]); + lstep1[55] = _mm256_add_epi32(lstep3[53], lstep2[55]); + lstep1[56] = _mm256_add_epi32(lstep3[58], lstep2[56]); + lstep1[57] = _mm256_add_epi32(lstep3[59], lstep2[57]); + lstep1[58] = _mm256_sub_epi32(lstep2[56], lstep3[58]); + lstep1[59] = _mm256_sub_epi32(lstep2[57], lstep3[59]); + lstep1[60] = _mm256_sub_epi32(lstep2[62], lstep3[60]); + lstep1[61] = _mm256_sub_epi32(lstep2[63], lstep3[61]); + lstep1[62] = _mm256_add_epi32(lstep3[60], lstep2[62]); + lstep1[63] = _mm256_add_epi32(lstep3[61], lstep2[63]); + } + // stage 8 + { + const __m256i k32_p31_p01 = + pair256_set_epi32(cospi_31_64, cospi_1_64); + const __m256i k32_p15_p17 = + pair256_set_epi32(cospi_15_64, cospi_17_64); + const __m256i k32_p23_p09 = + pair256_set_epi32(cospi_23_64, cospi_9_64); + const __m256i k32_p07_p25 = + pair256_set_epi32(cospi_7_64, cospi_25_64); + const __m256i k32_m25_p07 = + pair256_set_epi32(-cospi_25_64, cospi_7_64); + const __m256i k32_m09_p23 = + pair256_set_epi32(-cospi_9_64, cospi_23_64); + const __m256i k32_m17_p15 = + pair256_set_epi32(-cospi_17_64, cospi_15_64); + const __m256i k32_m01_p31 = + pair256_set_epi32(-cospi_1_64, cospi_31_64); + + u[0] = _mm256_unpacklo_epi32(lstep1[32], lstep1[62]); + u[1] = _mm256_unpackhi_epi32(lstep1[32], lstep1[62]); + u[2] = _mm256_unpacklo_epi32(lstep1[33], lstep1[63]); + u[3] = _mm256_unpackhi_epi32(lstep1[33], lstep1[63]); + u[4] = _mm256_unpacklo_epi32(lstep1[34], lstep1[60]); + u[5] = _mm256_unpackhi_epi32(lstep1[34], lstep1[60]); + u[6] = _mm256_unpacklo_epi32(lstep1[35], lstep1[61]); + u[7] = _mm256_unpackhi_epi32(lstep1[35], lstep1[61]); + u[8] = _mm256_unpacklo_epi32(lstep1[36], lstep1[58]); + u[9] = _mm256_unpackhi_epi32(lstep1[36], lstep1[58]); + u[10] = _mm256_unpacklo_epi32(lstep1[37], lstep1[59]); + u[11] = _mm256_unpackhi_epi32(lstep1[37], lstep1[59]); + u[12] = _mm256_unpacklo_epi32(lstep1[38], lstep1[56]); + u[13] = _mm256_unpackhi_epi32(lstep1[38], lstep1[56]); + u[14] = _mm256_unpacklo_epi32(lstep1[39], lstep1[57]); + u[15] = _mm256_unpackhi_epi32(lstep1[39], lstep1[57]); + + v[0] = k_madd_epi32_avx2(u[0], k32_p31_p01); + v[1] = k_madd_epi32_avx2(u[1], k32_p31_p01); + v[2] = k_madd_epi32_avx2(u[2], k32_p31_p01); + v[3] = k_madd_epi32_avx2(u[3], k32_p31_p01); + v[4] = k_madd_epi32_avx2(u[4], k32_p15_p17); + v[5] = k_madd_epi32_avx2(u[5], k32_p15_p17); + v[6] = k_madd_epi32_avx2(u[6], k32_p15_p17); + v[7] = k_madd_epi32_avx2(u[7], k32_p15_p17); + v[8] = k_madd_epi32_avx2(u[8], k32_p23_p09); + v[9] = k_madd_epi32_avx2(u[9], k32_p23_p09); + v[10] = k_madd_epi32_avx2(u[10], k32_p23_p09); + v[11] = k_madd_epi32_avx2(u[11], k32_p23_p09); + v[12] = k_madd_epi32_avx2(u[12], k32_p07_p25); + v[13] = k_madd_epi32_avx2(u[13], k32_p07_p25); + v[14] = k_madd_epi32_avx2(u[14], k32_p07_p25); + v[15] = k_madd_epi32_avx2(u[15], k32_p07_p25); + v[16] = k_madd_epi32_avx2(u[12], k32_m25_p07); + v[17] = k_madd_epi32_avx2(u[13], k32_m25_p07); + v[18] = k_madd_epi32_avx2(u[14], k32_m25_p07); + v[19] = k_madd_epi32_avx2(u[15], k32_m25_p07); + v[20] = k_madd_epi32_avx2(u[8], k32_m09_p23); + v[21] = k_madd_epi32_avx2(u[9], k32_m09_p23); + v[22] = k_madd_epi32_avx2(u[10], k32_m09_p23); + v[23] = k_madd_epi32_avx2(u[11], k32_m09_p23); + v[24] = k_madd_epi32_avx2(u[4], k32_m17_p15); + v[25] = k_madd_epi32_avx2(u[5], k32_m17_p15); + v[26] = k_madd_epi32_avx2(u[6], k32_m17_p15); + v[27] = k_madd_epi32_avx2(u[7], k32_m17_p15); + v[28] = k_madd_epi32_avx2(u[0], k32_m01_p31); + v[29] = k_madd_epi32_avx2(u[1], k32_m01_p31); + v[30] = k_madd_epi32_avx2(u[2], k32_m01_p31); + v[31] = k_madd_epi32_avx2(u[3], k32_m01_p31); + + u[0] = k_packs_epi64_avx2(v[0], v[1]); + u[1] = k_packs_epi64_avx2(v[2], v[3]); + u[2] = k_packs_epi64_avx2(v[4], v[5]); + u[3] = k_packs_epi64_avx2(v[6], v[7]); + u[4] = k_packs_epi64_avx2(v[8], v[9]); + u[5] = k_packs_epi64_avx2(v[10], v[11]); + u[6] = k_packs_epi64_avx2(v[12], v[13]); + u[7] = k_packs_epi64_avx2(v[14], v[15]); + u[8] = k_packs_epi64_avx2(v[16], v[17]); + u[9] = k_packs_epi64_avx2(v[18], v[19]); + u[10] = k_packs_epi64_avx2(v[20], v[21]); + u[11] = k_packs_epi64_avx2(v[22], v[23]); + u[12] = k_packs_epi64_avx2(v[24], v[25]); + u[13] = k_packs_epi64_avx2(v[26], v[27]); + u[14] = k_packs_epi64_avx2(v[28], v[29]); + u[15] = k_packs_epi64_avx2(v[30], v[31]); + + v[0] = _mm256_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm256_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm256_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm256_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm256_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm256_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm256_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm256_add_epi32(u[7], k__DCT_CONST_ROUNDING); + v[8] = _mm256_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm256_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm256_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm256_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm256_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm256_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm256_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm256_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + u[0] = _mm256_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm256_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm256_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm256_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm256_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm256_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm256_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm256_srai_epi32(v[7], DCT_CONST_BITS); + u[8] = _mm256_srai_epi32(v[8], DCT_CONST_BITS); + u[9] = _mm256_srai_epi32(v[9], DCT_CONST_BITS); + u[10] = _mm256_srai_epi32(v[10], DCT_CONST_BITS); + u[11] = _mm256_srai_epi32(v[11], DCT_CONST_BITS); + u[12] = _mm256_srai_epi32(v[12], DCT_CONST_BITS); + u[13] = _mm256_srai_epi32(v[13], DCT_CONST_BITS); + u[14] = _mm256_srai_epi32(v[14], DCT_CONST_BITS); + u[15] = _mm256_srai_epi32(v[15], DCT_CONST_BITS); + + v[0] = _mm256_cmpgt_epi32(kZero, u[0]); + v[1] = _mm256_cmpgt_epi32(kZero, u[1]); + v[2] = _mm256_cmpgt_epi32(kZero, u[2]); + v[3] = _mm256_cmpgt_epi32(kZero, u[3]); + v[4] = _mm256_cmpgt_epi32(kZero, u[4]); + v[5] = _mm256_cmpgt_epi32(kZero, u[5]); + v[6] = _mm256_cmpgt_epi32(kZero, u[6]); + v[7] = _mm256_cmpgt_epi32(kZero, u[7]); + v[8] = _mm256_cmpgt_epi32(kZero, u[8]); + v[9] = _mm256_cmpgt_epi32(kZero, u[9]); + v[10] = _mm256_cmpgt_epi32(kZero, u[10]); + v[11] = _mm256_cmpgt_epi32(kZero, u[11]); + v[12] = _mm256_cmpgt_epi32(kZero, u[12]); + v[13] = _mm256_cmpgt_epi32(kZero, u[13]); + v[14] = _mm256_cmpgt_epi32(kZero, u[14]); + v[15] = _mm256_cmpgt_epi32(kZero, u[15]); + + u[0] = _mm256_sub_epi32(u[0], v[0]); + u[1] = _mm256_sub_epi32(u[1], v[1]); + u[2] = _mm256_sub_epi32(u[2], v[2]); + u[3] = _mm256_sub_epi32(u[3], v[3]); + u[4] = _mm256_sub_epi32(u[4], v[4]); + u[5] = _mm256_sub_epi32(u[5], v[5]); + u[6] = _mm256_sub_epi32(u[6], v[6]); + u[7] = _mm256_sub_epi32(u[7], v[7]); + u[8] = _mm256_sub_epi32(u[8], v[8]); + u[9] = _mm256_sub_epi32(u[9], v[9]); + u[10] = _mm256_sub_epi32(u[10], v[10]); + u[11] = _mm256_sub_epi32(u[11], v[11]); + u[12] = _mm256_sub_epi32(u[12], v[12]); + u[13] = _mm256_sub_epi32(u[13], v[13]); + u[14] = _mm256_sub_epi32(u[14], v[14]); + u[15] = _mm256_sub_epi32(u[15], v[15]); + + v[0] = _mm256_add_epi32(u[0], K32One); + v[1] = _mm256_add_epi32(u[1], K32One); + v[2] = _mm256_add_epi32(u[2], K32One); + v[3] = _mm256_add_epi32(u[3], K32One); + v[4] = _mm256_add_epi32(u[4], K32One); + v[5] = _mm256_add_epi32(u[5], K32One); + v[6] = _mm256_add_epi32(u[6], K32One); + v[7] = _mm256_add_epi32(u[7], K32One); + v[8] = _mm256_add_epi32(u[8], K32One); + v[9] = _mm256_add_epi32(u[9], K32One); + v[10] = _mm256_add_epi32(u[10], K32One); + v[11] = _mm256_add_epi32(u[11], K32One); + v[12] = _mm256_add_epi32(u[12], K32One); + v[13] = _mm256_add_epi32(u[13], K32One); + v[14] = _mm256_add_epi32(u[14], K32One); + v[15] = _mm256_add_epi32(u[15], K32One); + + u[0] = _mm256_srai_epi32(v[0], 2); + u[1] = _mm256_srai_epi32(v[1], 2); + u[2] = _mm256_srai_epi32(v[2], 2); + u[3] = _mm256_srai_epi32(v[3], 2); + u[4] = _mm256_srai_epi32(v[4], 2); + u[5] = _mm256_srai_epi32(v[5], 2); + u[6] = _mm256_srai_epi32(v[6], 2); + u[7] = _mm256_srai_epi32(v[7], 2); + u[8] = _mm256_srai_epi32(v[8], 2); + u[9] = _mm256_srai_epi32(v[9], 2); + u[10] = _mm256_srai_epi32(v[10], 2); + u[11] = _mm256_srai_epi32(v[11], 2); + u[12] = _mm256_srai_epi32(v[12], 2); + u[13] = _mm256_srai_epi32(v[13], 2); + u[14] = _mm256_srai_epi32(v[14], 2); + u[15] = _mm256_srai_epi32(v[15], 2); + + out[1] = _mm256_packs_epi32(u[0], u[1]); + out[17] = _mm256_packs_epi32(u[2], u[3]); + out[9] = _mm256_packs_epi32(u[4], u[5]); + out[25] = _mm256_packs_epi32(u[6], u[7]); + out[7] = _mm256_packs_epi32(u[8], u[9]); + out[23] = _mm256_packs_epi32(u[10], u[11]); + out[15] = _mm256_packs_epi32(u[12], u[13]); + out[31] = _mm256_packs_epi32(u[14], u[15]); + } + { + const __m256i k32_p27_p05 = + pair256_set_epi32(cospi_27_64, cospi_5_64); + const __m256i k32_p11_p21 = + pair256_set_epi32(cospi_11_64, cospi_21_64); + const __m256i k32_p19_p13 = + pair256_set_epi32(cospi_19_64, cospi_13_64); + const __m256i k32_p03_p29 = + pair256_set_epi32(cospi_3_64, cospi_29_64); + const __m256i k32_m29_p03 = + pair256_set_epi32(-cospi_29_64, cospi_3_64); + const __m256i k32_m13_p19 = + pair256_set_epi32(-cospi_13_64, cospi_19_64); + const __m256i k32_m21_p11 = + pair256_set_epi32(-cospi_21_64, cospi_11_64); + const __m256i k32_m05_p27 = + pair256_set_epi32(-cospi_5_64, cospi_27_64); + + u[0] = _mm256_unpacklo_epi32(lstep1[40], lstep1[54]); + u[1] = _mm256_unpackhi_epi32(lstep1[40], lstep1[54]); + u[2] = _mm256_unpacklo_epi32(lstep1[41], lstep1[55]); + u[3] = _mm256_unpackhi_epi32(lstep1[41], lstep1[55]); + u[4] = _mm256_unpacklo_epi32(lstep1[42], lstep1[52]); + u[5] = _mm256_unpackhi_epi32(lstep1[42], lstep1[52]); + u[6] = _mm256_unpacklo_epi32(lstep1[43], lstep1[53]); + u[7] = _mm256_unpackhi_epi32(lstep1[43], lstep1[53]); + u[8] = _mm256_unpacklo_epi32(lstep1[44], lstep1[50]); + u[9] = _mm256_unpackhi_epi32(lstep1[44], lstep1[50]); + u[10] = _mm256_unpacklo_epi32(lstep1[45], lstep1[51]); + u[11] = _mm256_unpackhi_epi32(lstep1[45], lstep1[51]); + u[12] = _mm256_unpacklo_epi32(lstep1[46], lstep1[48]); + u[13] = _mm256_unpackhi_epi32(lstep1[46], lstep1[48]); + u[14] = _mm256_unpacklo_epi32(lstep1[47], lstep1[49]); + u[15] = _mm256_unpackhi_epi32(lstep1[47], lstep1[49]); + + v[0] = k_madd_epi32_avx2(u[0], k32_p27_p05); + v[1] = k_madd_epi32_avx2(u[1], k32_p27_p05); + v[2] = k_madd_epi32_avx2(u[2], k32_p27_p05); + v[3] = k_madd_epi32_avx2(u[3], k32_p27_p05); + v[4] = k_madd_epi32_avx2(u[4], k32_p11_p21); + v[5] = k_madd_epi32_avx2(u[5], k32_p11_p21); + v[6] = k_madd_epi32_avx2(u[6], k32_p11_p21); + v[7] = k_madd_epi32_avx2(u[7], k32_p11_p21); + v[8] = k_madd_epi32_avx2(u[8], k32_p19_p13); + v[9] = k_madd_epi32_avx2(u[9], k32_p19_p13); + v[10] = k_madd_epi32_avx2(u[10], k32_p19_p13); + v[11] = k_madd_epi32_avx2(u[11], k32_p19_p13); + v[12] = k_madd_epi32_avx2(u[12], k32_p03_p29); + v[13] = k_madd_epi32_avx2(u[13], k32_p03_p29); + v[14] = k_madd_epi32_avx2(u[14], k32_p03_p29); + v[15] = k_madd_epi32_avx2(u[15], k32_p03_p29); + v[16] = k_madd_epi32_avx2(u[12], k32_m29_p03); + v[17] = k_madd_epi32_avx2(u[13], k32_m29_p03); + v[18] = k_madd_epi32_avx2(u[14], k32_m29_p03); + v[19] = k_madd_epi32_avx2(u[15], k32_m29_p03); + v[20] = k_madd_epi32_avx2(u[8], k32_m13_p19); + v[21] = k_madd_epi32_avx2(u[9], k32_m13_p19); + v[22] = k_madd_epi32_avx2(u[10], k32_m13_p19); + v[23] = k_madd_epi32_avx2(u[11], k32_m13_p19); + v[24] = k_madd_epi32_avx2(u[4], k32_m21_p11); + v[25] = k_madd_epi32_avx2(u[5], k32_m21_p11); + v[26] = k_madd_epi32_avx2(u[6], k32_m21_p11); + v[27] = k_madd_epi32_avx2(u[7], k32_m21_p11); + v[28] = k_madd_epi32_avx2(u[0], k32_m05_p27); + v[29] = k_madd_epi32_avx2(u[1], k32_m05_p27); + v[30] = k_madd_epi32_avx2(u[2], k32_m05_p27); + v[31] = k_madd_epi32_avx2(u[3], k32_m05_p27); + + u[0] = k_packs_epi64_avx2(v[0], v[1]); + u[1] = k_packs_epi64_avx2(v[2], v[3]); + u[2] = k_packs_epi64_avx2(v[4], v[5]); + u[3] = k_packs_epi64_avx2(v[6], v[7]); + u[4] = k_packs_epi64_avx2(v[8], v[9]); + u[5] = k_packs_epi64_avx2(v[10], v[11]); + u[6] = k_packs_epi64_avx2(v[12], v[13]); + u[7] = k_packs_epi64_avx2(v[14], v[15]); + u[8] = k_packs_epi64_avx2(v[16], v[17]); + u[9] = k_packs_epi64_avx2(v[18], v[19]); + u[10] = k_packs_epi64_avx2(v[20], v[21]); + u[11] = k_packs_epi64_avx2(v[22], v[23]); + u[12] = k_packs_epi64_avx2(v[24], v[25]); + u[13] = k_packs_epi64_avx2(v[26], v[27]); + u[14] = k_packs_epi64_avx2(v[28], v[29]); + u[15] = k_packs_epi64_avx2(v[30], v[31]); + + v[0] = _mm256_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm256_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm256_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm256_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm256_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm256_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm256_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm256_add_epi32(u[7], k__DCT_CONST_ROUNDING); + v[8] = _mm256_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm256_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm256_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm256_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm256_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm256_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm256_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm256_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + u[0] = _mm256_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm256_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm256_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm256_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm256_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm256_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm256_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm256_srai_epi32(v[7], DCT_CONST_BITS); + u[8] = _mm256_srai_epi32(v[8], DCT_CONST_BITS); + u[9] = _mm256_srai_epi32(v[9], DCT_CONST_BITS); + u[10] = _mm256_srai_epi32(v[10], DCT_CONST_BITS); + u[11] = _mm256_srai_epi32(v[11], DCT_CONST_BITS); + u[12] = _mm256_srai_epi32(v[12], DCT_CONST_BITS); + u[13] = _mm256_srai_epi32(v[13], DCT_CONST_BITS); + u[14] = _mm256_srai_epi32(v[14], DCT_CONST_BITS); + u[15] = _mm256_srai_epi32(v[15], DCT_CONST_BITS); + + v[0] = _mm256_cmpgt_epi32(kZero, u[0]); + v[1] = _mm256_cmpgt_epi32(kZero, u[1]); + v[2] = _mm256_cmpgt_epi32(kZero, u[2]); + v[3] = _mm256_cmpgt_epi32(kZero, u[3]); + v[4] = _mm256_cmpgt_epi32(kZero, u[4]); + v[5] = _mm256_cmpgt_epi32(kZero, u[5]); + v[6] = _mm256_cmpgt_epi32(kZero, u[6]); + v[7] = _mm256_cmpgt_epi32(kZero, u[7]); + v[8] = _mm256_cmpgt_epi32(kZero, u[8]); + v[9] = _mm256_cmpgt_epi32(kZero, u[9]); + v[10] = _mm256_cmpgt_epi32(kZero, u[10]); + v[11] = _mm256_cmpgt_epi32(kZero, u[11]); + v[12] = _mm256_cmpgt_epi32(kZero, u[12]); + v[13] = _mm256_cmpgt_epi32(kZero, u[13]); + v[14] = _mm256_cmpgt_epi32(kZero, u[14]); + v[15] = _mm256_cmpgt_epi32(kZero, u[15]); + + u[0] = _mm256_sub_epi32(u[0], v[0]); + u[1] = _mm256_sub_epi32(u[1], v[1]); + u[2] = _mm256_sub_epi32(u[2], v[2]); + u[3] = _mm256_sub_epi32(u[3], v[3]); + u[4] = _mm256_sub_epi32(u[4], v[4]); + u[5] = _mm256_sub_epi32(u[5], v[5]); + u[6] = _mm256_sub_epi32(u[6], v[6]); + u[7] = _mm256_sub_epi32(u[7], v[7]); + u[8] = _mm256_sub_epi32(u[8], v[8]); + u[9] = _mm256_sub_epi32(u[9], v[9]); + u[10] = _mm256_sub_epi32(u[10], v[10]); + u[11] = _mm256_sub_epi32(u[11], v[11]); + u[12] = _mm256_sub_epi32(u[12], v[12]); + u[13] = _mm256_sub_epi32(u[13], v[13]); + u[14] = _mm256_sub_epi32(u[14], v[14]); + u[15] = _mm256_sub_epi32(u[15], v[15]); + + v[0] = _mm256_add_epi32(u[0], K32One); + v[1] = _mm256_add_epi32(u[1], K32One); + v[2] = _mm256_add_epi32(u[2], K32One); + v[3] = _mm256_add_epi32(u[3], K32One); + v[4] = _mm256_add_epi32(u[4], K32One); + v[5] = _mm256_add_epi32(u[5], K32One); + v[6] = _mm256_add_epi32(u[6], K32One); + v[7] = _mm256_add_epi32(u[7], K32One); + v[8] = _mm256_add_epi32(u[8], K32One); + v[9] = _mm256_add_epi32(u[9], K32One); + v[10] = _mm256_add_epi32(u[10], K32One); + v[11] = _mm256_add_epi32(u[11], K32One); + v[12] = _mm256_add_epi32(u[12], K32One); + v[13] = _mm256_add_epi32(u[13], K32One); + v[14] = _mm256_add_epi32(u[14], K32One); + v[15] = _mm256_add_epi32(u[15], K32One); + + u[0] = _mm256_srai_epi32(v[0], 2); + u[1] = _mm256_srai_epi32(v[1], 2); + u[2] = _mm256_srai_epi32(v[2], 2); + u[3] = _mm256_srai_epi32(v[3], 2); + u[4] = _mm256_srai_epi32(v[4], 2); + u[5] = _mm256_srai_epi32(v[5], 2); + u[6] = _mm256_srai_epi32(v[6], 2); + u[7] = _mm256_srai_epi32(v[7], 2); + u[8] = _mm256_srai_epi32(v[8], 2); + u[9] = _mm256_srai_epi32(v[9], 2); + u[10] = _mm256_srai_epi32(v[10], 2); + u[11] = _mm256_srai_epi32(v[11], 2); + u[12] = _mm256_srai_epi32(v[12], 2); + u[13] = _mm256_srai_epi32(v[13], 2); + u[14] = _mm256_srai_epi32(v[14], 2); + u[15] = _mm256_srai_epi32(v[15], 2); + + out[5] = _mm256_packs_epi32(u[0], u[1]); + out[21] = _mm256_packs_epi32(u[2], u[3]); + out[13] = _mm256_packs_epi32(u[4], u[5]); + out[29] = _mm256_packs_epi32(u[6], u[7]); + out[3] = _mm256_packs_epi32(u[8], u[9]); + out[19] = _mm256_packs_epi32(u[10], u[11]); + out[11] = _mm256_packs_epi32(u[12], u[13]); + out[27] = _mm256_packs_epi32(u[14], u[15]); + } + } +#endif + // Transpose the results, do it as four 8x8 transposes. + { + int transpose_block; + int16_t *output_currStep, *output_nextStep; + tran_low_t *curr_out, *next_out; + // Pass 0 + output_currStep = &intermediate[column_start * 32]; + output_nextStep = &intermediate[(column_start + 8) * 32]; + // Pass 1 + curr_out = &output_org[column_start * 32]; + next_out = &output_org[(column_start + 8) * 32]; + + for (transpose_block = 0; transpose_block < 4; ++transpose_block) { + __m256i *this_out = &out[8 * transpose_block]; + // 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 + // 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 + // 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 + // 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 + // 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 + // 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 + // 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 + // 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 + const __m256i tr0_0 = _mm256_unpacklo_epi16(this_out[0], this_out[1]); + const __m256i tr0_1 = _mm256_unpacklo_epi16(this_out[2], this_out[3]); + const __m256i tr0_2 = _mm256_unpackhi_epi16(this_out[0], this_out[1]); + const __m256i tr0_3 = _mm256_unpackhi_epi16(this_out[2], this_out[3]); + const __m256i tr0_4 = _mm256_unpacklo_epi16(this_out[4], this_out[5]); + const __m256i tr0_5 = _mm256_unpacklo_epi16(this_out[6], this_out[7]); + const __m256i tr0_6 = _mm256_unpackhi_epi16(this_out[4], this_out[5]); + const __m256i tr0_7 = _mm256_unpackhi_epi16(this_out[6], this_out[7]); + // 00 20 01 21 02 22 03 23 08 28 09 29 10 30 11 31 + // 40 60 41 61 42 62 43 63 48 68 49 69 50 70 51 71 + // 04 24 05 25 06 26 07 27 12 32 13 33 14 34 15 35 + // 44 64 45 65 46 66 47 67 52 72 53 73 54 74 55 75 + // 80 100 81 101 82 102 83 103 88 108 89 109 90 110 91 101 + // 120 140 121 141 122 142 123 143 128 148 129 149 130 150 131 151 + // 84 104 85 105 86 106 87 107 92 112 93 113 94 114 95 115 + // 124 144 125 145 126 146 127 147 132 152 133 153 134 154 135 155 + + const __m256i tr1_0 = _mm256_unpacklo_epi32(tr0_0, tr0_1); + const __m256i tr1_1 = _mm256_unpacklo_epi32(tr0_2, tr0_3); + const __m256i tr1_2 = _mm256_unpackhi_epi32(tr0_0, tr0_1); + const __m256i tr1_3 = _mm256_unpackhi_epi32(tr0_2, tr0_3); + const __m256i tr1_4 = _mm256_unpacklo_epi32(tr0_4, tr0_5); + const __m256i tr1_5 = _mm256_unpacklo_epi32(tr0_6, tr0_7); + const __m256i tr1_6 = _mm256_unpackhi_epi32(tr0_4, tr0_5); + const __m256i tr1_7 = _mm256_unpackhi_epi32(tr0_6, tr0_7); + // 00 20 40 60 01 21 41 61 08 28 48 68 09 29 49 69 + // 04 24 44 64 05 25 45 65 12 32 52 72 13 33 53 73 + // 02 22 42 62 03 23 43 63 10 30 50 70 11 31 51 71 + // 06 26 46 66 07 27 47 67 14 34 54 74 15 35 55 75 + // 80 100 120 140 81 101 121 141 88 108 128 148 89 109 129 149 + // 84 104 124 144 85 105 125 145 92 112 132 152 93 113 133 153 + // 82 102 122 142 83 103 123 143 90 110 130 150 91 101 131 151 + // 86 106 126 146 87 107 127 147 94 114 134 154 95 115 135 155 + __m256i tr2_0 = _mm256_unpacklo_epi64(tr1_0, tr1_4); + __m256i tr2_1 = _mm256_unpackhi_epi64(tr1_0, tr1_4); + __m256i tr2_2 = _mm256_unpacklo_epi64(tr1_2, tr1_6); + __m256i tr2_3 = _mm256_unpackhi_epi64(tr1_2, tr1_6); + __m256i tr2_4 = _mm256_unpacklo_epi64(tr1_1, tr1_5); + __m256i tr2_5 = _mm256_unpackhi_epi64(tr1_1, tr1_5); + __m256i tr2_6 = _mm256_unpacklo_epi64(tr1_3, tr1_7); + __m256i tr2_7 = _mm256_unpackhi_epi64(tr1_3, tr1_7); + // 00 20 40 60 80 100 120 140 08 28 48 68 88 108 128 148 + // 01 21 41 61 81 101 121 141 09 29 49 69 89 109 129 149 + // 02 22 42 62 82 102 122 142 10 30 50 70 90 110 130 150 + // 03 23 43 63 83 103 123 143 11 31 51 71 91 101 131 151 + // 04 24 44 64 84 104 124 144 12 32 52 72 92 112 132 152 + // 05 25 45 65 85 105 125 145 13 33 53 73 93 113 133 153 + // 06 26 46 66 86 106 126 146 14 34 54 74 94 114 134 154 + // 07 27 47 67 87 107 127 147 15 35 55 75 95 115 135 155 + if (0 == pass) { + // output[j] = (output[j] + 1 + (output[j] > 0)) >> 2; + // TODO(cd): see quality impact of only doing + // output[j] = (output[j] + 1) >> 2; + // which would remove the code between here ... + __m256i tr2_0_0 = _mm256_cmpgt_epi16(tr2_0, kZero); + __m256i tr2_1_0 = _mm256_cmpgt_epi16(tr2_1, kZero); + __m256i tr2_2_0 = _mm256_cmpgt_epi16(tr2_2, kZero); + __m256i tr2_3_0 = _mm256_cmpgt_epi16(tr2_3, kZero); + __m256i tr2_4_0 = _mm256_cmpgt_epi16(tr2_4, kZero); + __m256i tr2_5_0 = _mm256_cmpgt_epi16(tr2_5, kZero); + __m256i tr2_6_0 = _mm256_cmpgt_epi16(tr2_6, kZero); + __m256i tr2_7_0 = _mm256_cmpgt_epi16(tr2_7, kZero); + tr2_0 = _mm256_sub_epi16(tr2_0, tr2_0_0); + tr2_1 = _mm256_sub_epi16(tr2_1, tr2_1_0); + tr2_2 = _mm256_sub_epi16(tr2_2, tr2_2_0); + tr2_3 = _mm256_sub_epi16(tr2_3, tr2_3_0); + tr2_4 = _mm256_sub_epi16(tr2_4, tr2_4_0); + tr2_5 = _mm256_sub_epi16(tr2_5, tr2_5_0); + tr2_6 = _mm256_sub_epi16(tr2_6, tr2_6_0); + tr2_7 = _mm256_sub_epi16(tr2_7, tr2_7_0); + // ... and here. + // PS: also change code in av1/encoder/av1_dct.c + tr2_0 = _mm256_add_epi16(tr2_0, kOne); + tr2_1 = _mm256_add_epi16(tr2_1, kOne); + tr2_2 = _mm256_add_epi16(tr2_2, kOne); + tr2_3 = _mm256_add_epi16(tr2_3, kOne); + tr2_4 = _mm256_add_epi16(tr2_4, kOne); + tr2_5 = _mm256_add_epi16(tr2_5, kOne); + tr2_6 = _mm256_add_epi16(tr2_6, kOne); + tr2_7 = _mm256_add_epi16(tr2_7, kOne); + tr2_0 = _mm256_srai_epi16(tr2_0, 2); + tr2_1 = _mm256_srai_epi16(tr2_1, 2); + tr2_2 = _mm256_srai_epi16(tr2_2, 2); + tr2_3 = _mm256_srai_epi16(tr2_3, 2); + tr2_4 = _mm256_srai_epi16(tr2_4, 2); + tr2_5 = _mm256_srai_epi16(tr2_5, 2); + tr2_6 = _mm256_srai_epi16(tr2_6, 2); + tr2_7 = _mm256_srai_epi16(tr2_7, 2); + } + if (0 == pass) { + // Note: even though all these stores are aligned, using the aligned + // intrinsic make the code slightly slower. + _mm_storeu_si128((__m128i *)(output_currStep + 0 * 32), + _mm256_castsi256_si128(tr2_0)); + _mm_storeu_si128((__m128i *)(output_currStep + 1 * 32), + _mm256_castsi256_si128(tr2_1)); + _mm_storeu_si128((__m128i *)(output_currStep + 2 * 32), + _mm256_castsi256_si128(tr2_2)); + _mm_storeu_si128((__m128i *)(output_currStep + 3 * 32), + _mm256_castsi256_si128(tr2_3)); + _mm_storeu_si128((__m128i *)(output_currStep + 4 * 32), + _mm256_castsi256_si128(tr2_4)); + _mm_storeu_si128((__m128i *)(output_currStep + 5 * 32), + _mm256_castsi256_si128(tr2_5)); + _mm_storeu_si128((__m128i *)(output_currStep + 6 * 32), + _mm256_castsi256_si128(tr2_6)); + _mm_storeu_si128((__m128i *)(output_currStep + 7 * 32), + _mm256_castsi256_si128(tr2_7)); + + _mm_storeu_si128((__m128i *)(output_nextStep + 0 * 32), + _mm256_extractf128_si256(tr2_0, 1)); + _mm_storeu_si128((__m128i *)(output_nextStep + 1 * 32), + _mm256_extractf128_si256(tr2_1, 1)); + _mm_storeu_si128((__m128i *)(output_nextStep + 2 * 32), + _mm256_extractf128_si256(tr2_2, 1)); + _mm_storeu_si128((__m128i *)(output_nextStep + 3 * 32), + _mm256_extractf128_si256(tr2_3, 1)); + _mm_storeu_si128((__m128i *)(output_nextStep + 4 * 32), + _mm256_extractf128_si256(tr2_4, 1)); + _mm_storeu_si128((__m128i *)(output_nextStep + 5 * 32), + _mm256_extractf128_si256(tr2_5, 1)); + _mm_storeu_si128((__m128i *)(output_nextStep + 6 * 32), + _mm256_extractf128_si256(tr2_6, 1)); + _mm_storeu_si128((__m128i *)(output_nextStep + 7 * 32), + _mm256_extractf128_si256(tr2_7, 1)); + // Process next 8x8 + output_currStep += 8; + output_nextStep += 8; + } + if (1 == pass) { + store_coeff(&tr2_0, curr_out + 0 * 32, next_out + 0 * 32); + store_coeff(&tr2_1, curr_out + 1 * 32, next_out + 1 * 32); + store_coeff(&tr2_2, curr_out + 2 * 32, next_out + 2 * 32); + store_coeff(&tr2_3, curr_out + 3 * 32, next_out + 3 * 32); + store_coeff(&tr2_4, curr_out + 4 * 32, next_out + 4 * 32); + store_coeff(&tr2_5, curr_out + 5 * 32, next_out + 5 * 32); + store_coeff(&tr2_6, curr_out + 6 * 32, next_out + 6 * 32); + store_coeff(&tr2_7, curr_out + 7 * 32, next_out + 7 * 32); + curr_out += 8; + next_out += 8; + } + } + } + } + } + _mm256_zeroupper(); +} // NOLINT diff --git a/third_party/aom/aom_dsp/x86/fwd_dct32x32_impl_sse2.h b/third_party/aom/aom_dsp/x86/fwd_dct32x32_impl_sse2.h new file mode 100644 index 0000000000..69dd6af119 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/fwd_dct32x32_impl_sse2.h @@ -0,0 +1,3201 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // SSE2 + +#include "aom_dsp/fwd_txfm.h" +#include "aom_dsp/txfm_common.h" +#include "aom_dsp/x86/txfm_common_sse2.h" + +// TODO(jingning) The high bit-depth version needs re-work for performance. +// The current SSE2 implementation also causes cross reference to the static +// functions in the C implementation file. +#if DCT_HIGH_BIT_DEPTH +#define ADD_EPI16 _mm_adds_epi16 +#define SUB_EPI16 _mm_subs_epi16 +#if FDCT32x32_HIGH_PRECISION +void aom_fdct32x32_rows_c(const int16_t *intermediate, tran_low_t *out) { + int i, j; + for (i = 0; i < 32; ++i) { + tran_high_t temp_in[32], temp_out[32]; + for (j = 0; j < 32; ++j) temp_in[j] = intermediate[j * 32 + i]; + aom_fdct32(temp_in, temp_out, 0); + for (j = 0; j < 32; ++j) + out[j + i * 32] = + (tran_low_t)((temp_out[j] + 1 + (temp_out[j] < 0)) >> 2); + } +} +#define HIGH_FDCT32x32_2D_C aom_highbd_fdct32x32_c +#define HIGH_FDCT32x32_2D_ROWS_C aom_fdct32x32_rows_c +#else +void aom_fdct32x32_rd_rows_c(const int16_t *intermediate, tran_low_t *out) { + int i, j; + for (i = 0; i < 32; ++i) { + tran_high_t temp_in[32], temp_out[32]; + for (j = 0; j < 32; ++j) temp_in[j] = intermediate[j * 32 + i]; + aom_fdct32(temp_in, temp_out, 1); + for (j = 0; j < 32; ++j) out[j + i * 32] = (tran_low_t)temp_out[j]; + } +} +#define HIGH_FDCT32x32_2D_C aom_highbd_fdct32x32_rd_c +#define HIGH_FDCT32x32_2D_ROWS_C aom_fdct32x32_rd_rows_c +#endif // FDCT32x32_HIGH_PRECISION +#else +#define ADD_EPI16 _mm_add_epi16 +#define SUB_EPI16 _mm_sub_epi16 +#endif // DCT_HIGH_BIT_DEPTH + +void FDCT32x32_2D(const int16_t *input, tran_low_t *output_org, int stride) { + // Calculate pre-multiplied strides + const int str1 = stride; + const int str2 = 2 * stride; + const int str3 = 2 * stride + str1; + // We need an intermediate buffer between passes. + DECLARE_ALIGNED(16, int16_t, intermediate[32 * 32]); + // Constants + // When we use them, in one case, they are all the same. In all others + // it's a pair of them that we need to repeat four times. This is done + // by constructing the 32 bit constant corresponding to that pair. + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(+cospi_16_64, -cospi_16_64); + const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + const __m128i k__cospi_p24_p08 = pair_set_epi16(+cospi_24_64, cospi_8_64); + const __m128i k__cospi_p12_p20 = pair_set_epi16(+cospi_12_64, cospi_20_64); + const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i k__cospi_p28_p04 = pair_set_epi16(+cospi_28_64, cospi_4_64); + const __m128i k__cospi_m28_m04 = pair_set_epi16(-cospi_28_64, -cospi_4_64); + const __m128i k__cospi_m12_m20 = pair_set_epi16(-cospi_12_64, -cospi_20_64); + const __m128i k__cospi_p30_p02 = pair_set_epi16(+cospi_30_64, cospi_2_64); + const __m128i k__cospi_p14_p18 = pair_set_epi16(+cospi_14_64, cospi_18_64); + const __m128i k__cospi_p22_p10 = pair_set_epi16(+cospi_22_64, cospi_10_64); + const __m128i k__cospi_p06_p26 = pair_set_epi16(+cospi_6_64, cospi_26_64); + const __m128i k__cospi_m26_p06 = pair_set_epi16(-cospi_26_64, cospi_6_64); + const __m128i k__cospi_m10_p22 = pair_set_epi16(-cospi_10_64, cospi_22_64); + const __m128i k__cospi_m18_p14 = pair_set_epi16(-cospi_18_64, cospi_14_64); + const __m128i k__cospi_m02_p30 = pair_set_epi16(-cospi_2_64, cospi_30_64); + const __m128i k__cospi_p31_p01 = pair_set_epi16(+cospi_31_64, cospi_1_64); + const __m128i k__cospi_p15_p17 = pair_set_epi16(+cospi_15_64, cospi_17_64); + const __m128i k__cospi_p23_p09 = pair_set_epi16(+cospi_23_64, cospi_9_64); + const __m128i k__cospi_p07_p25 = pair_set_epi16(+cospi_7_64, cospi_25_64); + const __m128i k__cospi_m25_p07 = pair_set_epi16(-cospi_25_64, cospi_7_64); + const __m128i k__cospi_m09_p23 = pair_set_epi16(-cospi_9_64, cospi_23_64); + const __m128i k__cospi_m17_p15 = pair_set_epi16(-cospi_17_64, cospi_15_64); + const __m128i k__cospi_m01_p31 = pair_set_epi16(-cospi_1_64, cospi_31_64); + const __m128i k__cospi_p27_p05 = pair_set_epi16(+cospi_27_64, cospi_5_64); + const __m128i k__cospi_p11_p21 = pair_set_epi16(+cospi_11_64, cospi_21_64); + const __m128i k__cospi_p19_p13 = pair_set_epi16(+cospi_19_64, cospi_13_64); + const __m128i k__cospi_p03_p29 = pair_set_epi16(+cospi_3_64, cospi_29_64); + const __m128i k__cospi_m29_p03 = pair_set_epi16(-cospi_29_64, cospi_3_64); + const __m128i k__cospi_m13_p19 = pair_set_epi16(-cospi_13_64, cospi_19_64); + const __m128i k__cospi_m21_p11 = pair_set_epi16(-cospi_21_64, cospi_11_64); + const __m128i k__cospi_m05_p27 = pair_set_epi16(-cospi_5_64, cospi_27_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i kZero = _mm_set1_epi16(0); + const __m128i kOne = _mm_set1_epi16(1); + // Do the two transform/transpose passes + int pass; +#if DCT_HIGH_BIT_DEPTH + int overflow; +#endif + for (pass = 0; pass < 2; ++pass) { + // We process eight columns (transposed rows in second pass) at a time. + int column_start; + for (column_start = 0; column_start < 32; column_start += 8) { + __m128i step1[32]; + __m128i step2[32]; + __m128i step3[32]; + __m128i out[32]; + // Stage 1 + // Note: even though all the loads below are aligned, using the aligned + // intrinsic make the code slightly slower. + if (0 == pass) { + const int16_t *in = &input[column_start]; + // step1[i] = (in[ 0 * stride] + in[(32 - 1) * stride]) << 2; + // Note: the next four blocks could be in a loop. That would help the + // instruction cache but is actually slower. + { + const int16_t *ina = in + 0 * str1; + const int16_t *inb = in + 31 * str1; + __m128i *step1a = &step1[0]; + __m128i *step1b = &step1[31]; + const __m128i ina0 = _mm_loadu_si128((const __m128i *)(ina)); + const __m128i ina1 = _mm_loadu_si128((const __m128i *)(ina + str1)); + const __m128i ina2 = _mm_loadu_si128((const __m128i *)(ina + str2)); + const __m128i ina3 = _mm_loadu_si128((const __m128i *)(ina + str3)); + const __m128i inb3 = _mm_loadu_si128((const __m128i *)(inb - str3)); + const __m128i inb2 = _mm_loadu_si128((const __m128i *)(inb - str2)); + const __m128i inb1 = _mm_loadu_si128((const __m128i *)(inb - str1)); + const __m128i inb0 = _mm_loadu_si128((const __m128i *)(inb)); + step1a[0] = _mm_add_epi16(ina0, inb0); + step1a[1] = _mm_add_epi16(ina1, inb1); + step1a[2] = _mm_add_epi16(ina2, inb2); + step1a[3] = _mm_add_epi16(ina3, inb3); + step1b[-3] = _mm_sub_epi16(ina3, inb3); + step1b[-2] = _mm_sub_epi16(ina2, inb2); + step1b[-1] = _mm_sub_epi16(ina1, inb1); + step1b[-0] = _mm_sub_epi16(ina0, inb0); + step1a[0] = _mm_slli_epi16(step1a[0], 2); + step1a[1] = _mm_slli_epi16(step1a[1], 2); + step1a[2] = _mm_slli_epi16(step1a[2], 2); + step1a[3] = _mm_slli_epi16(step1a[3], 2); + step1b[-3] = _mm_slli_epi16(step1b[-3], 2); + step1b[-2] = _mm_slli_epi16(step1b[-2], 2); + step1b[-1] = _mm_slli_epi16(step1b[-1], 2); + step1b[-0] = _mm_slli_epi16(step1b[-0], 2); + } + { + const int16_t *ina = in + 4 * str1; + const int16_t *inb = in + 27 * str1; + __m128i *step1a = &step1[4]; + __m128i *step1b = &step1[27]; + const __m128i ina0 = _mm_loadu_si128((const __m128i *)(ina)); + const __m128i ina1 = _mm_loadu_si128((const __m128i *)(ina + str1)); + const __m128i ina2 = _mm_loadu_si128((const __m128i *)(ina + str2)); + const __m128i ina3 = _mm_loadu_si128((const __m128i *)(ina + str3)); + const __m128i inb3 = _mm_loadu_si128((const __m128i *)(inb - str3)); + const __m128i inb2 = _mm_loadu_si128((const __m128i *)(inb - str2)); + const __m128i inb1 = _mm_loadu_si128((const __m128i *)(inb - str1)); + const __m128i inb0 = _mm_loadu_si128((const __m128i *)(inb)); + step1a[0] = _mm_add_epi16(ina0, inb0); + step1a[1] = _mm_add_epi16(ina1, inb1); + step1a[2] = _mm_add_epi16(ina2, inb2); + step1a[3] = _mm_add_epi16(ina3, inb3); + step1b[-3] = _mm_sub_epi16(ina3, inb3); + step1b[-2] = _mm_sub_epi16(ina2, inb2); + step1b[-1] = _mm_sub_epi16(ina1, inb1); + step1b[-0] = _mm_sub_epi16(ina0, inb0); + step1a[0] = _mm_slli_epi16(step1a[0], 2); + step1a[1] = _mm_slli_epi16(step1a[1], 2); + step1a[2] = _mm_slli_epi16(step1a[2], 2); + step1a[3] = _mm_slli_epi16(step1a[3], 2); + step1b[-3] = _mm_slli_epi16(step1b[-3], 2); + step1b[-2] = _mm_slli_epi16(step1b[-2], 2); + step1b[-1] = _mm_slli_epi16(step1b[-1], 2); + step1b[-0] = _mm_slli_epi16(step1b[-0], 2); + } + { + const int16_t *ina = in + 8 * str1; + const int16_t *inb = in + 23 * str1; + __m128i *step1a = &step1[8]; + __m128i *step1b = &step1[23]; + const __m128i ina0 = _mm_loadu_si128((const __m128i *)(ina)); + const __m128i ina1 = _mm_loadu_si128((const __m128i *)(ina + str1)); + const __m128i ina2 = _mm_loadu_si128((const __m128i *)(ina + str2)); + const __m128i ina3 = _mm_loadu_si128((const __m128i *)(ina + str3)); + const __m128i inb3 = _mm_loadu_si128((const __m128i *)(inb - str3)); + const __m128i inb2 = _mm_loadu_si128((const __m128i *)(inb - str2)); + const __m128i inb1 = _mm_loadu_si128((const __m128i *)(inb - str1)); + const __m128i inb0 = _mm_loadu_si128((const __m128i *)(inb)); + step1a[0] = _mm_add_epi16(ina0, inb0); + step1a[1] = _mm_add_epi16(ina1, inb1); + step1a[2] = _mm_add_epi16(ina2, inb2); + step1a[3] = _mm_add_epi16(ina3, inb3); + step1b[-3] = _mm_sub_epi16(ina3, inb3); + step1b[-2] = _mm_sub_epi16(ina2, inb2); + step1b[-1] = _mm_sub_epi16(ina1, inb1); + step1b[-0] = _mm_sub_epi16(ina0, inb0); + step1a[0] = _mm_slli_epi16(step1a[0], 2); + step1a[1] = _mm_slli_epi16(step1a[1], 2); + step1a[2] = _mm_slli_epi16(step1a[2], 2); + step1a[3] = _mm_slli_epi16(step1a[3], 2); + step1b[-3] = _mm_slli_epi16(step1b[-3], 2); + step1b[-2] = _mm_slli_epi16(step1b[-2], 2); + step1b[-1] = _mm_slli_epi16(step1b[-1], 2); + step1b[-0] = _mm_slli_epi16(step1b[-0], 2); + } + { + const int16_t *ina = in + 12 * str1; + const int16_t *inb = in + 19 * str1; + __m128i *step1a = &step1[12]; + __m128i *step1b = &step1[19]; + const __m128i ina0 = _mm_loadu_si128((const __m128i *)(ina)); + const __m128i ina1 = _mm_loadu_si128((const __m128i *)(ina + str1)); + const __m128i ina2 = _mm_loadu_si128((const __m128i *)(ina + str2)); + const __m128i ina3 = _mm_loadu_si128((const __m128i *)(ina + str3)); + const __m128i inb3 = _mm_loadu_si128((const __m128i *)(inb - str3)); + const __m128i inb2 = _mm_loadu_si128((const __m128i *)(inb - str2)); + const __m128i inb1 = _mm_loadu_si128((const __m128i *)(inb - str1)); + const __m128i inb0 = _mm_loadu_si128((const __m128i *)(inb)); + step1a[0] = _mm_add_epi16(ina0, inb0); + step1a[1] = _mm_add_epi16(ina1, inb1); + step1a[2] = _mm_add_epi16(ina2, inb2); + step1a[3] = _mm_add_epi16(ina3, inb3); + step1b[-3] = _mm_sub_epi16(ina3, inb3); + step1b[-2] = _mm_sub_epi16(ina2, inb2); + step1b[-1] = _mm_sub_epi16(ina1, inb1); + step1b[-0] = _mm_sub_epi16(ina0, inb0); + step1a[0] = _mm_slli_epi16(step1a[0], 2); + step1a[1] = _mm_slli_epi16(step1a[1], 2); + step1a[2] = _mm_slli_epi16(step1a[2], 2); + step1a[3] = _mm_slli_epi16(step1a[3], 2); + step1b[-3] = _mm_slli_epi16(step1b[-3], 2); + step1b[-2] = _mm_slli_epi16(step1b[-2], 2); + step1b[-1] = _mm_slli_epi16(step1b[-1], 2); + step1b[-0] = _mm_slli_epi16(step1b[-0], 2); + } + } else { + int16_t *in = &intermediate[column_start]; + // step1[i] = in[ 0 * 32] + in[(32 - 1) * 32]; + // Note: using the same approach as above to have common offset is + // counter-productive as all offsets can be calculated at compile + // time. + // Note: the next four blocks could be in a loop. That would help the + // instruction cache but is actually slower. + { + __m128i in00 = _mm_loadu_si128((const __m128i *)(in + 0 * 32)); + __m128i in01 = _mm_loadu_si128((const __m128i *)(in + 1 * 32)); + __m128i in02 = _mm_loadu_si128((const __m128i *)(in + 2 * 32)); + __m128i in03 = _mm_loadu_si128((const __m128i *)(in + 3 * 32)); + __m128i in28 = _mm_loadu_si128((const __m128i *)(in + 28 * 32)); + __m128i in29 = _mm_loadu_si128((const __m128i *)(in + 29 * 32)); + __m128i in30 = _mm_loadu_si128((const __m128i *)(in + 30 * 32)); + __m128i in31 = _mm_loadu_si128((const __m128i *)(in + 31 * 32)); + step1[0] = ADD_EPI16(in00, in31); + step1[1] = ADD_EPI16(in01, in30); + step1[2] = ADD_EPI16(in02, in29); + step1[3] = ADD_EPI16(in03, in28); + step1[28] = SUB_EPI16(in03, in28); + step1[29] = SUB_EPI16(in02, in29); + step1[30] = SUB_EPI16(in01, in30); + step1[31] = SUB_EPI16(in00, in31); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x8(&step1[0], &step1[1], &step1[2], + &step1[3], &step1[28], &step1[29], + &step1[30], &step1[31]); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + __m128i in04 = _mm_loadu_si128((const __m128i *)(in + 4 * 32)); + __m128i in05 = _mm_loadu_si128((const __m128i *)(in + 5 * 32)); + __m128i in06 = _mm_loadu_si128((const __m128i *)(in + 6 * 32)); + __m128i in07 = _mm_loadu_si128((const __m128i *)(in + 7 * 32)); + __m128i in24 = _mm_loadu_si128((const __m128i *)(in + 24 * 32)); + __m128i in25 = _mm_loadu_si128((const __m128i *)(in + 25 * 32)); + __m128i in26 = _mm_loadu_si128((const __m128i *)(in + 26 * 32)); + __m128i in27 = _mm_loadu_si128((const __m128i *)(in + 27 * 32)); + step1[4] = ADD_EPI16(in04, in27); + step1[5] = ADD_EPI16(in05, in26); + step1[6] = ADD_EPI16(in06, in25); + step1[7] = ADD_EPI16(in07, in24); + step1[24] = SUB_EPI16(in07, in24); + step1[25] = SUB_EPI16(in06, in25); + step1[26] = SUB_EPI16(in05, in26); + step1[27] = SUB_EPI16(in04, in27); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x8(&step1[4], &step1[5], &step1[6], + &step1[7], &step1[24], &step1[25], + &step1[26], &step1[27]); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + __m128i in08 = _mm_loadu_si128((const __m128i *)(in + 8 * 32)); + __m128i in09 = _mm_loadu_si128((const __m128i *)(in + 9 * 32)); + __m128i in10 = _mm_loadu_si128((const __m128i *)(in + 10 * 32)); + __m128i in11 = _mm_loadu_si128((const __m128i *)(in + 11 * 32)); + __m128i in20 = _mm_loadu_si128((const __m128i *)(in + 20 * 32)); + __m128i in21 = _mm_loadu_si128((const __m128i *)(in + 21 * 32)); + __m128i in22 = _mm_loadu_si128((const __m128i *)(in + 22 * 32)); + __m128i in23 = _mm_loadu_si128((const __m128i *)(in + 23 * 32)); + step1[8] = ADD_EPI16(in08, in23); + step1[9] = ADD_EPI16(in09, in22); + step1[10] = ADD_EPI16(in10, in21); + step1[11] = ADD_EPI16(in11, in20); + step1[20] = SUB_EPI16(in11, in20); + step1[21] = SUB_EPI16(in10, in21); + step1[22] = SUB_EPI16(in09, in22); + step1[23] = SUB_EPI16(in08, in23); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x8(&step1[8], &step1[9], &step1[10], + &step1[11], &step1[20], &step1[21], + &step1[22], &step1[23]); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + __m128i in12 = _mm_loadu_si128((const __m128i *)(in + 12 * 32)); + __m128i in13 = _mm_loadu_si128((const __m128i *)(in + 13 * 32)); + __m128i in14 = _mm_loadu_si128((const __m128i *)(in + 14 * 32)); + __m128i in15 = _mm_loadu_si128((const __m128i *)(in + 15 * 32)); + __m128i in16 = _mm_loadu_si128((const __m128i *)(in + 16 * 32)); + __m128i in17 = _mm_loadu_si128((const __m128i *)(in + 17 * 32)); + __m128i in18 = _mm_loadu_si128((const __m128i *)(in + 18 * 32)); + __m128i in19 = _mm_loadu_si128((const __m128i *)(in + 19 * 32)); + step1[12] = ADD_EPI16(in12, in19); + step1[13] = ADD_EPI16(in13, in18); + step1[14] = ADD_EPI16(in14, in17); + step1[15] = ADD_EPI16(in15, in16); + step1[16] = SUB_EPI16(in15, in16); + step1[17] = SUB_EPI16(in14, in17); + step1[18] = SUB_EPI16(in13, in18); + step1[19] = SUB_EPI16(in12, in19); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x8(&step1[12], &step1[13], &step1[14], + &step1[15], &step1[16], &step1[17], + &step1[18], &step1[19]); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + } + // Stage 2 + { + step2[0] = ADD_EPI16(step1[0], step1[15]); + step2[1] = ADD_EPI16(step1[1], step1[14]); + step2[2] = ADD_EPI16(step1[2], step1[13]); + step2[3] = ADD_EPI16(step1[3], step1[12]); + step2[4] = ADD_EPI16(step1[4], step1[11]); + step2[5] = ADD_EPI16(step1[5], step1[10]); + step2[6] = ADD_EPI16(step1[6], step1[9]); + step2[7] = ADD_EPI16(step1[7], step1[8]); + step2[8] = SUB_EPI16(step1[7], step1[8]); + step2[9] = SUB_EPI16(step1[6], step1[9]); + step2[10] = SUB_EPI16(step1[5], step1[10]); + step2[11] = SUB_EPI16(step1[4], step1[11]); + step2[12] = SUB_EPI16(step1[3], step1[12]); + step2[13] = SUB_EPI16(step1[2], step1[13]); + step2[14] = SUB_EPI16(step1[1], step1[14]); + step2[15] = SUB_EPI16(step1[0], step1[15]); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x16( + &step2[0], &step2[1], &step2[2], &step2[3], &step2[4], &step2[5], + &step2[6], &step2[7], &step2[8], &step2[9], &step2[10], &step2[11], + &step2[12], &step2[13], &step2[14], &step2[15]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + const __m128i s2_20_0 = _mm_unpacklo_epi16(step1[27], step1[20]); + const __m128i s2_20_1 = _mm_unpackhi_epi16(step1[27], step1[20]); + const __m128i s2_21_0 = _mm_unpacklo_epi16(step1[26], step1[21]); + const __m128i s2_21_1 = _mm_unpackhi_epi16(step1[26], step1[21]); + const __m128i s2_22_0 = _mm_unpacklo_epi16(step1[25], step1[22]); + const __m128i s2_22_1 = _mm_unpackhi_epi16(step1[25], step1[22]); + const __m128i s2_23_0 = _mm_unpacklo_epi16(step1[24], step1[23]); + const __m128i s2_23_1 = _mm_unpackhi_epi16(step1[24], step1[23]); + const __m128i s2_20_2 = _mm_madd_epi16(s2_20_0, k__cospi_p16_m16); + const __m128i s2_20_3 = _mm_madd_epi16(s2_20_1, k__cospi_p16_m16); + const __m128i s2_21_2 = _mm_madd_epi16(s2_21_0, k__cospi_p16_m16); + const __m128i s2_21_3 = _mm_madd_epi16(s2_21_1, k__cospi_p16_m16); + const __m128i s2_22_2 = _mm_madd_epi16(s2_22_0, k__cospi_p16_m16); + const __m128i s2_22_3 = _mm_madd_epi16(s2_22_1, k__cospi_p16_m16); + const __m128i s2_23_2 = _mm_madd_epi16(s2_23_0, k__cospi_p16_m16); + const __m128i s2_23_3 = _mm_madd_epi16(s2_23_1, k__cospi_p16_m16); + const __m128i s2_24_2 = _mm_madd_epi16(s2_23_0, k__cospi_p16_p16); + const __m128i s2_24_3 = _mm_madd_epi16(s2_23_1, k__cospi_p16_p16); + const __m128i s2_25_2 = _mm_madd_epi16(s2_22_0, k__cospi_p16_p16); + const __m128i s2_25_3 = _mm_madd_epi16(s2_22_1, k__cospi_p16_p16); + const __m128i s2_26_2 = _mm_madd_epi16(s2_21_0, k__cospi_p16_p16); + const __m128i s2_26_3 = _mm_madd_epi16(s2_21_1, k__cospi_p16_p16); + const __m128i s2_27_2 = _mm_madd_epi16(s2_20_0, k__cospi_p16_p16); + const __m128i s2_27_3 = _mm_madd_epi16(s2_20_1, k__cospi_p16_p16); + // dct_const_round_shift + const __m128i s2_20_4 = _mm_add_epi32(s2_20_2, k__DCT_CONST_ROUNDING); + const __m128i s2_20_5 = _mm_add_epi32(s2_20_3, k__DCT_CONST_ROUNDING); + const __m128i s2_21_4 = _mm_add_epi32(s2_21_2, k__DCT_CONST_ROUNDING); + const __m128i s2_21_5 = _mm_add_epi32(s2_21_3, k__DCT_CONST_ROUNDING); + const __m128i s2_22_4 = _mm_add_epi32(s2_22_2, k__DCT_CONST_ROUNDING); + const __m128i s2_22_5 = _mm_add_epi32(s2_22_3, k__DCT_CONST_ROUNDING); + const __m128i s2_23_4 = _mm_add_epi32(s2_23_2, k__DCT_CONST_ROUNDING); + const __m128i s2_23_5 = _mm_add_epi32(s2_23_3, k__DCT_CONST_ROUNDING); + const __m128i s2_24_4 = _mm_add_epi32(s2_24_2, k__DCT_CONST_ROUNDING); + const __m128i s2_24_5 = _mm_add_epi32(s2_24_3, k__DCT_CONST_ROUNDING); + const __m128i s2_25_4 = _mm_add_epi32(s2_25_2, k__DCT_CONST_ROUNDING); + const __m128i s2_25_5 = _mm_add_epi32(s2_25_3, k__DCT_CONST_ROUNDING); + const __m128i s2_26_4 = _mm_add_epi32(s2_26_2, k__DCT_CONST_ROUNDING); + const __m128i s2_26_5 = _mm_add_epi32(s2_26_3, k__DCT_CONST_ROUNDING); + const __m128i s2_27_4 = _mm_add_epi32(s2_27_2, k__DCT_CONST_ROUNDING); + const __m128i s2_27_5 = _mm_add_epi32(s2_27_3, k__DCT_CONST_ROUNDING); + const __m128i s2_20_6 = _mm_srai_epi32(s2_20_4, DCT_CONST_BITS); + const __m128i s2_20_7 = _mm_srai_epi32(s2_20_5, DCT_CONST_BITS); + const __m128i s2_21_6 = _mm_srai_epi32(s2_21_4, DCT_CONST_BITS); + const __m128i s2_21_7 = _mm_srai_epi32(s2_21_5, DCT_CONST_BITS); + const __m128i s2_22_6 = _mm_srai_epi32(s2_22_4, DCT_CONST_BITS); + const __m128i s2_22_7 = _mm_srai_epi32(s2_22_5, DCT_CONST_BITS); + const __m128i s2_23_6 = _mm_srai_epi32(s2_23_4, DCT_CONST_BITS); + const __m128i s2_23_7 = _mm_srai_epi32(s2_23_5, DCT_CONST_BITS); + const __m128i s2_24_6 = _mm_srai_epi32(s2_24_4, DCT_CONST_BITS); + const __m128i s2_24_7 = _mm_srai_epi32(s2_24_5, DCT_CONST_BITS); + const __m128i s2_25_6 = _mm_srai_epi32(s2_25_4, DCT_CONST_BITS); + const __m128i s2_25_7 = _mm_srai_epi32(s2_25_5, DCT_CONST_BITS); + const __m128i s2_26_6 = _mm_srai_epi32(s2_26_4, DCT_CONST_BITS); + const __m128i s2_26_7 = _mm_srai_epi32(s2_26_5, DCT_CONST_BITS); + const __m128i s2_27_6 = _mm_srai_epi32(s2_27_4, DCT_CONST_BITS); + const __m128i s2_27_7 = _mm_srai_epi32(s2_27_5, DCT_CONST_BITS); + // Combine + step2[20] = _mm_packs_epi32(s2_20_6, s2_20_7); + step2[21] = _mm_packs_epi32(s2_21_6, s2_21_7); + step2[22] = _mm_packs_epi32(s2_22_6, s2_22_7); + step2[23] = _mm_packs_epi32(s2_23_6, s2_23_7); + step2[24] = _mm_packs_epi32(s2_24_6, s2_24_7); + step2[25] = _mm_packs_epi32(s2_25_6, s2_25_7); + step2[26] = _mm_packs_epi32(s2_26_6, s2_26_7); + step2[27] = _mm_packs_epi32(s2_27_6, s2_27_7); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x8(&step2[20], &step2[21], &step2[22], + &step2[23], &step2[24], &step2[25], + &step2[26], &step2[27]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + +#if !FDCT32x32_HIGH_PRECISION + // dump the magnitude by half, hence the intermediate values are within + // the range of 16 bits. + if (1 == pass) { + __m128i s3_00_0 = _mm_cmplt_epi16(step2[0], kZero); + __m128i s3_01_0 = _mm_cmplt_epi16(step2[1], kZero); + __m128i s3_02_0 = _mm_cmplt_epi16(step2[2], kZero); + __m128i s3_03_0 = _mm_cmplt_epi16(step2[3], kZero); + __m128i s3_04_0 = _mm_cmplt_epi16(step2[4], kZero); + __m128i s3_05_0 = _mm_cmplt_epi16(step2[5], kZero); + __m128i s3_06_0 = _mm_cmplt_epi16(step2[6], kZero); + __m128i s3_07_0 = _mm_cmplt_epi16(step2[7], kZero); + __m128i s2_08_0 = _mm_cmplt_epi16(step2[8], kZero); + __m128i s2_09_0 = _mm_cmplt_epi16(step2[9], kZero); + __m128i s3_10_0 = _mm_cmplt_epi16(step2[10], kZero); + __m128i s3_11_0 = _mm_cmplt_epi16(step2[11], kZero); + __m128i s3_12_0 = _mm_cmplt_epi16(step2[12], kZero); + __m128i s3_13_0 = _mm_cmplt_epi16(step2[13], kZero); + __m128i s2_14_0 = _mm_cmplt_epi16(step2[14], kZero); + __m128i s2_15_0 = _mm_cmplt_epi16(step2[15], kZero); + __m128i s3_16_0 = _mm_cmplt_epi16(step1[16], kZero); + __m128i s3_17_0 = _mm_cmplt_epi16(step1[17], kZero); + __m128i s3_18_0 = _mm_cmplt_epi16(step1[18], kZero); + __m128i s3_19_0 = _mm_cmplt_epi16(step1[19], kZero); + __m128i s3_20_0 = _mm_cmplt_epi16(step2[20], kZero); + __m128i s3_21_0 = _mm_cmplt_epi16(step2[21], kZero); + __m128i s3_22_0 = _mm_cmplt_epi16(step2[22], kZero); + __m128i s3_23_0 = _mm_cmplt_epi16(step2[23], kZero); + __m128i s3_24_0 = _mm_cmplt_epi16(step2[24], kZero); + __m128i s3_25_0 = _mm_cmplt_epi16(step2[25], kZero); + __m128i s3_26_0 = _mm_cmplt_epi16(step2[26], kZero); + __m128i s3_27_0 = _mm_cmplt_epi16(step2[27], kZero); + __m128i s3_28_0 = _mm_cmplt_epi16(step1[28], kZero); + __m128i s3_29_0 = _mm_cmplt_epi16(step1[29], kZero); + __m128i s3_30_0 = _mm_cmplt_epi16(step1[30], kZero); + __m128i s3_31_0 = _mm_cmplt_epi16(step1[31], kZero); + + step2[0] = SUB_EPI16(step2[0], s3_00_0); + step2[1] = SUB_EPI16(step2[1], s3_01_0); + step2[2] = SUB_EPI16(step2[2], s3_02_0); + step2[3] = SUB_EPI16(step2[3], s3_03_0); + step2[4] = SUB_EPI16(step2[4], s3_04_0); + step2[5] = SUB_EPI16(step2[5], s3_05_0); + step2[6] = SUB_EPI16(step2[6], s3_06_0); + step2[7] = SUB_EPI16(step2[7], s3_07_0); + step2[8] = SUB_EPI16(step2[8], s2_08_0); + step2[9] = SUB_EPI16(step2[9], s2_09_0); + step2[10] = SUB_EPI16(step2[10], s3_10_0); + step2[11] = SUB_EPI16(step2[11], s3_11_0); + step2[12] = SUB_EPI16(step2[12], s3_12_0); + step2[13] = SUB_EPI16(step2[13], s3_13_0); + step2[14] = SUB_EPI16(step2[14], s2_14_0); + step2[15] = SUB_EPI16(step2[15], s2_15_0); + step1[16] = SUB_EPI16(step1[16], s3_16_0); + step1[17] = SUB_EPI16(step1[17], s3_17_0); + step1[18] = SUB_EPI16(step1[18], s3_18_0); + step1[19] = SUB_EPI16(step1[19], s3_19_0); + step2[20] = SUB_EPI16(step2[20], s3_20_0); + step2[21] = SUB_EPI16(step2[21], s3_21_0); + step2[22] = SUB_EPI16(step2[22], s3_22_0); + step2[23] = SUB_EPI16(step2[23], s3_23_0); + step2[24] = SUB_EPI16(step2[24], s3_24_0); + step2[25] = SUB_EPI16(step2[25], s3_25_0); + step2[26] = SUB_EPI16(step2[26], s3_26_0); + step2[27] = SUB_EPI16(step2[27], s3_27_0); + step1[28] = SUB_EPI16(step1[28], s3_28_0); + step1[29] = SUB_EPI16(step1[29], s3_29_0); + step1[30] = SUB_EPI16(step1[30], s3_30_0); + step1[31] = SUB_EPI16(step1[31], s3_31_0); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x32( + &step2[0], &step2[1], &step2[2], &step2[3], &step2[4], &step2[5], + &step2[6], &step2[7], &step2[8], &step2[9], &step2[10], &step2[11], + &step2[12], &step2[13], &step2[14], &step2[15], &step1[16], + &step1[17], &step1[18], &step1[19], &step2[20], &step2[21], + &step2[22], &step2[23], &step2[24], &step2[25], &step2[26], + &step2[27], &step1[28], &step1[29], &step1[30], &step1[31]); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + step2[0] = _mm_add_epi16(step2[0], kOne); + step2[1] = _mm_add_epi16(step2[1], kOne); + step2[2] = _mm_add_epi16(step2[2], kOne); + step2[3] = _mm_add_epi16(step2[3], kOne); + step2[4] = _mm_add_epi16(step2[4], kOne); + step2[5] = _mm_add_epi16(step2[5], kOne); + step2[6] = _mm_add_epi16(step2[6], kOne); + step2[7] = _mm_add_epi16(step2[7], kOne); + step2[8] = _mm_add_epi16(step2[8], kOne); + step2[9] = _mm_add_epi16(step2[9], kOne); + step2[10] = _mm_add_epi16(step2[10], kOne); + step2[11] = _mm_add_epi16(step2[11], kOne); + step2[12] = _mm_add_epi16(step2[12], kOne); + step2[13] = _mm_add_epi16(step2[13], kOne); + step2[14] = _mm_add_epi16(step2[14], kOne); + step2[15] = _mm_add_epi16(step2[15], kOne); + step1[16] = _mm_add_epi16(step1[16], kOne); + step1[17] = _mm_add_epi16(step1[17], kOne); + step1[18] = _mm_add_epi16(step1[18], kOne); + step1[19] = _mm_add_epi16(step1[19], kOne); + step2[20] = _mm_add_epi16(step2[20], kOne); + step2[21] = _mm_add_epi16(step2[21], kOne); + step2[22] = _mm_add_epi16(step2[22], kOne); + step2[23] = _mm_add_epi16(step2[23], kOne); + step2[24] = _mm_add_epi16(step2[24], kOne); + step2[25] = _mm_add_epi16(step2[25], kOne); + step2[26] = _mm_add_epi16(step2[26], kOne); + step2[27] = _mm_add_epi16(step2[27], kOne); + step1[28] = _mm_add_epi16(step1[28], kOne); + step1[29] = _mm_add_epi16(step1[29], kOne); + step1[30] = _mm_add_epi16(step1[30], kOne); + step1[31] = _mm_add_epi16(step1[31], kOne); + + step2[0] = _mm_srai_epi16(step2[0], 2); + step2[1] = _mm_srai_epi16(step2[1], 2); + step2[2] = _mm_srai_epi16(step2[2], 2); + step2[3] = _mm_srai_epi16(step2[3], 2); + step2[4] = _mm_srai_epi16(step2[4], 2); + step2[5] = _mm_srai_epi16(step2[5], 2); + step2[6] = _mm_srai_epi16(step2[6], 2); + step2[7] = _mm_srai_epi16(step2[7], 2); + step2[8] = _mm_srai_epi16(step2[8], 2); + step2[9] = _mm_srai_epi16(step2[9], 2); + step2[10] = _mm_srai_epi16(step2[10], 2); + step2[11] = _mm_srai_epi16(step2[11], 2); + step2[12] = _mm_srai_epi16(step2[12], 2); + step2[13] = _mm_srai_epi16(step2[13], 2); + step2[14] = _mm_srai_epi16(step2[14], 2); + step2[15] = _mm_srai_epi16(step2[15], 2); + step1[16] = _mm_srai_epi16(step1[16], 2); + step1[17] = _mm_srai_epi16(step1[17], 2); + step1[18] = _mm_srai_epi16(step1[18], 2); + step1[19] = _mm_srai_epi16(step1[19], 2); + step2[20] = _mm_srai_epi16(step2[20], 2); + step2[21] = _mm_srai_epi16(step2[21], 2); + step2[22] = _mm_srai_epi16(step2[22], 2); + step2[23] = _mm_srai_epi16(step2[23], 2); + step2[24] = _mm_srai_epi16(step2[24], 2); + step2[25] = _mm_srai_epi16(step2[25], 2); + step2[26] = _mm_srai_epi16(step2[26], 2); + step2[27] = _mm_srai_epi16(step2[27], 2); + step1[28] = _mm_srai_epi16(step1[28], 2); + step1[29] = _mm_srai_epi16(step1[29], 2); + step1[30] = _mm_srai_epi16(step1[30], 2); + step1[31] = _mm_srai_epi16(step1[31], 2); + } +#endif // !FDCT32x32_HIGH_PRECISION + +#if FDCT32x32_HIGH_PRECISION + if (pass == 0) { +#endif + // Stage 3 + { + step3[0] = ADD_EPI16(step2[(8 - 1)], step2[0]); + step3[1] = ADD_EPI16(step2[(8 - 2)], step2[1]); + step3[2] = ADD_EPI16(step2[(8 - 3)], step2[2]); + step3[3] = ADD_EPI16(step2[(8 - 4)], step2[3]); + step3[4] = SUB_EPI16(step2[(8 - 5)], step2[4]); + step3[5] = SUB_EPI16(step2[(8 - 6)], step2[5]); + step3[6] = SUB_EPI16(step2[(8 - 7)], step2[6]); + step3[7] = SUB_EPI16(step2[(8 - 8)], step2[7]); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x8(&step3[0], &step3[1], &step3[2], + &step3[3], &step3[4], &step3[5], + &step3[6], &step3[7]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + const __m128i s3_10_0 = _mm_unpacklo_epi16(step2[13], step2[10]); + const __m128i s3_10_1 = _mm_unpackhi_epi16(step2[13], step2[10]); + const __m128i s3_11_0 = _mm_unpacklo_epi16(step2[12], step2[11]); + const __m128i s3_11_1 = _mm_unpackhi_epi16(step2[12], step2[11]); + const __m128i s3_10_2 = _mm_madd_epi16(s3_10_0, k__cospi_p16_m16); + const __m128i s3_10_3 = _mm_madd_epi16(s3_10_1, k__cospi_p16_m16); + const __m128i s3_11_2 = _mm_madd_epi16(s3_11_0, k__cospi_p16_m16); + const __m128i s3_11_3 = _mm_madd_epi16(s3_11_1, k__cospi_p16_m16); + const __m128i s3_12_2 = _mm_madd_epi16(s3_11_0, k__cospi_p16_p16); + const __m128i s3_12_3 = _mm_madd_epi16(s3_11_1, k__cospi_p16_p16); + const __m128i s3_13_2 = _mm_madd_epi16(s3_10_0, k__cospi_p16_p16); + const __m128i s3_13_3 = _mm_madd_epi16(s3_10_1, k__cospi_p16_p16); + // dct_const_round_shift + const __m128i s3_10_4 = _mm_add_epi32(s3_10_2, k__DCT_CONST_ROUNDING); + const __m128i s3_10_5 = _mm_add_epi32(s3_10_3, k__DCT_CONST_ROUNDING); + const __m128i s3_11_4 = _mm_add_epi32(s3_11_2, k__DCT_CONST_ROUNDING); + const __m128i s3_11_5 = _mm_add_epi32(s3_11_3, k__DCT_CONST_ROUNDING); + const __m128i s3_12_4 = _mm_add_epi32(s3_12_2, k__DCT_CONST_ROUNDING); + const __m128i s3_12_5 = _mm_add_epi32(s3_12_3, k__DCT_CONST_ROUNDING); + const __m128i s3_13_4 = _mm_add_epi32(s3_13_2, k__DCT_CONST_ROUNDING); + const __m128i s3_13_5 = _mm_add_epi32(s3_13_3, k__DCT_CONST_ROUNDING); + const __m128i s3_10_6 = _mm_srai_epi32(s3_10_4, DCT_CONST_BITS); + const __m128i s3_10_7 = _mm_srai_epi32(s3_10_5, DCT_CONST_BITS); + const __m128i s3_11_6 = _mm_srai_epi32(s3_11_4, DCT_CONST_BITS); + const __m128i s3_11_7 = _mm_srai_epi32(s3_11_5, DCT_CONST_BITS); + const __m128i s3_12_6 = _mm_srai_epi32(s3_12_4, DCT_CONST_BITS); + const __m128i s3_12_7 = _mm_srai_epi32(s3_12_5, DCT_CONST_BITS); + const __m128i s3_13_6 = _mm_srai_epi32(s3_13_4, DCT_CONST_BITS); + const __m128i s3_13_7 = _mm_srai_epi32(s3_13_5, DCT_CONST_BITS); + // Combine + step3[10] = _mm_packs_epi32(s3_10_6, s3_10_7); + step3[11] = _mm_packs_epi32(s3_11_6, s3_11_7); + step3[12] = _mm_packs_epi32(s3_12_6, s3_12_7); + step3[13] = _mm_packs_epi32(s3_13_6, s3_13_7); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x4(&step3[10], &step3[11], &step3[12], + &step3[13]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + step3[16] = ADD_EPI16(step2[23], step1[16]); + step3[17] = ADD_EPI16(step2[22], step1[17]); + step3[18] = ADD_EPI16(step2[21], step1[18]); + step3[19] = ADD_EPI16(step2[20], step1[19]); + step3[20] = SUB_EPI16(step1[19], step2[20]); + step3[21] = SUB_EPI16(step1[18], step2[21]); + step3[22] = SUB_EPI16(step1[17], step2[22]); + step3[23] = SUB_EPI16(step1[16], step2[23]); + step3[24] = SUB_EPI16(step1[31], step2[24]); + step3[25] = SUB_EPI16(step1[30], step2[25]); + step3[26] = SUB_EPI16(step1[29], step2[26]); + step3[27] = SUB_EPI16(step1[28], step2[27]); + step3[28] = ADD_EPI16(step2[27], step1[28]); + step3[29] = ADD_EPI16(step2[26], step1[29]); + step3[30] = ADD_EPI16(step2[25], step1[30]); + step3[31] = ADD_EPI16(step2[24], step1[31]); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x16( + &step3[16], &step3[17], &step3[18], &step3[19], &step3[20], + &step3[21], &step3[22], &step3[23], &step3[24], &step3[25], + &step3[26], &step3[27], &step3[28], &step3[29], &step3[30], + &step3[31]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + + // Stage 4 + { + step1[0] = ADD_EPI16(step3[3], step3[0]); + step1[1] = ADD_EPI16(step3[2], step3[1]); + step1[2] = SUB_EPI16(step3[1], step3[2]); + step1[3] = SUB_EPI16(step3[0], step3[3]); + step1[8] = ADD_EPI16(step3[11], step2[8]); + step1[9] = ADD_EPI16(step3[10], step2[9]); + step1[10] = SUB_EPI16(step2[9], step3[10]); + step1[11] = SUB_EPI16(step2[8], step3[11]); + step1[12] = SUB_EPI16(step2[15], step3[12]); + step1[13] = SUB_EPI16(step2[14], step3[13]); + step1[14] = ADD_EPI16(step3[13], step2[14]); + step1[15] = ADD_EPI16(step3[12], step2[15]); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x16( + &step1[0], &step1[1], &step1[2], &step1[3], &step1[4], &step1[5], + &step1[6], &step1[7], &step1[8], &step1[9], &step1[10], + &step1[11], &step1[12], &step1[13], &step1[14], &step1[15]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + const __m128i s1_05_0 = _mm_unpacklo_epi16(step3[6], step3[5]); + const __m128i s1_05_1 = _mm_unpackhi_epi16(step3[6], step3[5]); + const __m128i s1_05_2 = _mm_madd_epi16(s1_05_0, k__cospi_p16_m16); + const __m128i s1_05_3 = _mm_madd_epi16(s1_05_1, k__cospi_p16_m16); + const __m128i s1_06_2 = _mm_madd_epi16(s1_05_0, k__cospi_p16_p16); + const __m128i s1_06_3 = _mm_madd_epi16(s1_05_1, k__cospi_p16_p16); + // dct_const_round_shift + const __m128i s1_05_4 = _mm_add_epi32(s1_05_2, k__DCT_CONST_ROUNDING); + const __m128i s1_05_5 = _mm_add_epi32(s1_05_3, k__DCT_CONST_ROUNDING); + const __m128i s1_06_4 = _mm_add_epi32(s1_06_2, k__DCT_CONST_ROUNDING); + const __m128i s1_06_5 = _mm_add_epi32(s1_06_3, k__DCT_CONST_ROUNDING); + const __m128i s1_05_6 = _mm_srai_epi32(s1_05_4, DCT_CONST_BITS); + const __m128i s1_05_7 = _mm_srai_epi32(s1_05_5, DCT_CONST_BITS); + const __m128i s1_06_6 = _mm_srai_epi32(s1_06_4, DCT_CONST_BITS); + const __m128i s1_06_7 = _mm_srai_epi32(s1_06_5, DCT_CONST_BITS); + // Combine + step1[5] = _mm_packs_epi32(s1_05_6, s1_05_7); + step1[6] = _mm_packs_epi32(s1_06_6, s1_06_7); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x2(&step1[5], &step1[6]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + const __m128i s1_18_0 = _mm_unpacklo_epi16(step3[18], step3[29]); + const __m128i s1_18_1 = _mm_unpackhi_epi16(step3[18], step3[29]); + const __m128i s1_19_0 = _mm_unpacklo_epi16(step3[19], step3[28]); + const __m128i s1_19_1 = _mm_unpackhi_epi16(step3[19], step3[28]); + const __m128i s1_20_0 = _mm_unpacklo_epi16(step3[20], step3[27]); + const __m128i s1_20_1 = _mm_unpackhi_epi16(step3[20], step3[27]); + const __m128i s1_21_0 = _mm_unpacklo_epi16(step3[21], step3[26]); + const __m128i s1_21_1 = _mm_unpackhi_epi16(step3[21], step3[26]); + const __m128i s1_18_2 = _mm_madd_epi16(s1_18_0, k__cospi_m08_p24); + const __m128i s1_18_3 = _mm_madd_epi16(s1_18_1, k__cospi_m08_p24); + const __m128i s1_19_2 = _mm_madd_epi16(s1_19_0, k__cospi_m08_p24); + const __m128i s1_19_3 = _mm_madd_epi16(s1_19_1, k__cospi_m08_p24); + const __m128i s1_20_2 = _mm_madd_epi16(s1_20_0, k__cospi_m24_m08); + const __m128i s1_20_3 = _mm_madd_epi16(s1_20_1, k__cospi_m24_m08); + const __m128i s1_21_2 = _mm_madd_epi16(s1_21_0, k__cospi_m24_m08); + const __m128i s1_21_3 = _mm_madd_epi16(s1_21_1, k__cospi_m24_m08); + const __m128i s1_26_2 = _mm_madd_epi16(s1_21_0, k__cospi_m08_p24); + const __m128i s1_26_3 = _mm_madd_epi16(s1_21_1, k__cospi_m08_p24); + const __m128i s1_27_2 = _mm_madd_epi16(s1_20_0, k__cospi_m08_p24); + const __m128i s1_27_3 = _mm_madd_epi16(s1_20_1, k__cospi_m08_p24); + const __m128i s1_28_2 = _mm_madd_epi16(s1_19_0, k__cospi_p24_p08); + const __m128i s1_28_3 = _mm_madd_epi16(s1_19_1, k__cospi_p24_p08); + const __m128i s1_29_2 = _mm_madd_epi16(s1_18_0, k__cospi_p24_p08); + const __m128i s1_29_3 = _mm_madd_epi16(s1_18_1, k__cospi_p24_p08); + // dct_const_round_shift + const __m128i s1_18_4 = _mm_add_epi32(s1_18_2, k__DCT_CONST_ROUNDING); + const __m128i s1_18_5 = _mm_add_epi32(s1_18_3, k__DCT_CONST_ROUNDING); + const __m128i s1_19_4 = _mm_add_epi32(s1_19_2, k__DCT_CONST_ROUNDING); + const __m128i s1_19_5 = _mm_add_epi32(s1_19_3, k__DCT_CONST_ROUNDING); + const __m128i s1_20_4 = _mm_add_epi32(s1_20_2, k__DCT_CONST_ROUNDING); + const __m128i s1_20_5 = _mm_add_epi32(s1_20_3, k__DCT_CONST_ROUNDING); + const __m128i s1_21_4 = _mm_add_epi32(s1_21_2, k__DCT_CONST_ROUNDING); + const __m128i s1_21_5 = _mm_add_epi32(s1_21_3, k__DCT_CONST_ROUNDING); + const __m128i s1_26_4 = _mm_add_epi32(s1_26_2, k__DCT_CONST_ROUNDING); + const __m128i s1_26_5 = _mm_add_epi32(s1_26_3, k__DCT_CONST_ROUNDING); + const __m128i s1_27_4 = _mm_add_epi32(s1_27_2, k__DCT_CONST_ROUNDING); + const __m128i s1_27_5 = _mm_add_epi32(s1_27_3, k__DCT_CONST_ROUNDING); + const __m128i s1_28_4 = _mm_add_epi32(s1_28_2, k__DCT_CONST_ROUNDING); + const __m128i s1_28_5 = _mm_add_epi32(s1_28_3, k__DCT_CONST_ROUNDING); + const __m128i s1_29_4 = _mm_add_epi32(s1_29_2, k__DCT_CONST_ROUNDING); + const __m128i s1_29_5 = _mm_add_epi32(s1_29_3, k__DCT_CONST_ROUNDING); + const __m128i s1_18_6 = _mm_srai_epi32(s1_18_4, DCT_CONST_BITS); + const __m128i s1_18_7 = _mm_srai_epi32(s1_18_5, DCT_CONST_BITS); + const __m128i s1_19_6 = _mm_srai_epi32(s1_19_4, DCT_CONST_BITS); + const __m128i s1_19_7 = _mm_srai_epi32(s1_19_5, DCT_CONST_BITS); + const __m128i s1_20_6 = _mm_srai_epi32(s1_20_4, DCT_CONST_BITS); + const __m128i s1_20_7 = _mm_srai_epi32(s1_20_5, DCT_CONST_BITS); + const __m128i s1_21_6 = _mm_srai_epi32(s1_21_4, DCT_CONST_BITS); + const __m128i s1_21_7 = _mm_srai_epi32(s1_21_5, DCT_CONST_BITS); + const __m128i s1_26_6 = _mm_srai_epi32(s1_26_4, DCT_CONST_BITS); + const __m128i s1_26_7 = _mm_srai_epi32(s1_26_5, DCT_CONST_BITS); + const __m128i s1_27_6 = _mm_srai_epi32(s1_27_4, DCT_CONST_BITS); + const __m128i s1_27_7 = _mm_srai_epi32(s1_27_5, DCT_CONST_BITS); + const __m128i s1_28_6 = _mm_srai_epi32(s1_28_4, DCT_CONST_BITS); + const __m128i s1_28_7 = _mm_srai_epi32(s1_28_5, DCT_CONST_BITS); + const __m128i s1_29_6 = _mm_srai_epi32(s1_29_4, DCT_CONST_BITS); + const __m128i s1_29_7 = _mm_srai_epi32(s1_29_5, DCT_CONST_BITS); + // Combine + step1[18] = _mm_packs_epi32(s1_18_6, s1_18_7); + step1[19] = _mm_packs_epi32(s1_19_6, s1_19_7); + step1[20] = _mm_packs_epi32(s1_20_6, s1_20_7); + step1[21] = _mm_packs_epi32(s1_21_6, s1_21_7); + step1[26] = _mm_packs_epi32(s1_26_6, s1_26_7); + step1[27] = _mm_packs_epi32(s1_27_6, s1_27_7); + step1[28] = _mm_packs_epi32(s1_28_6, s1_28_7); + step1[29] = _mm_packs_epi32(s1_29_6, s1_29_7); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x8(&step1[18], &step1[19], &step1[20], + &step1[21], &step1[26], &step1[27], + &step1[28], &step1[29]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + // Stage 5 + { + step2[4] = ADD_EPI16(step1[5], step3[4]); + step2[5] = SUB_EPI16(step3[4], step1[5]); + step2[6] = SUB_EPI16(step3[7], step1[6]); + step2[7] = ADD_EPI16(step1[6], step3[7]); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x4(&step2[4], &step2[5], &step2[6], + &step2[7]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + const __m128i out_00_0 = _mm_unpacklo_epi16(step1[0], step1[1]); + const __m128i out_00_1 = _mm_unpackhi_epi16(step1[0], step1[1]); + const __m128i out_08_0 = _mm_unpacklo_epi16(step1[2], step1[3]); + const __m128i out_08_1 = _mm_unpackhi_epi16(step1[2], step1[3]); + const __m128i out_00_2 = _mm_madd_epi16(out_00_0, k__cospi_p16_p16); + const __m128i out_00_3 = _mm_madd_epi16(out_00_1, k__cospi_p16_p16); + const __m128i out_16_2 = _mm_madd_epi16(out_00_0, k__cospi_p16_m16); + const __m128i out_16_3 = _mm_madd_epi16(out_00_1, k__cospi_p16_m16); + const __m128i out_08_2 = _mm_madd_epi16(out_08_0, k__cospi_p24_p08); + const __m128i out_08_3 = _mm_madd_epi16(out_08_1, k__cospi_p24_p08); + const __m128i out_24_2 = _mm_madd_epi16(out_08_0, k__cospi_m08_p24); + const __m128i out_24_3 = _mm_madd_epi16(out_08_1, k__cospi_m08_p24); + // dct_const_round_shift + const __m128i out_00_4 = + _mm_add_epi32(out_00_2, k__DCT_CONST_ROUNDING); + const __m128i out_00_5 = + _mm_add_epi32(out_00_3, k__DCT_CONST_ROUNDING); + const __m128i out_16_4 = + _mm_add_epi32(out_16_2, k__DCT_CONST_ROUNDING); + const __m128i out_16_5 = + _mm_add_epi32(out_16_3, k__DCT_CONST_ROUNDING); + const __m128i out_08_4 = + _mm_add_epi32(out_08_2, k__DCT_CONST_ROUNDING); + const __m128i out_08_5 = + _mm_add_epi32(out_08_3, k__DCT_CONST_ROUNDING); + const __m128i out_24_4 = + _mm_add_epi32(out_24_2, k__DCT_CONST_ROUNDING); + const __m128i out_24_5 = + _mm_add_epi32(out_24_3, k__DCT_CONST_ROUNDING); + const __m128i out_00_6 = _mm_srai_epi32(out_00_4, DCT_CONST_BITS); + const __m128i out_00_7 = _mm_srai_epi32(out_00_5, DCT_CONST_BITS); + const __m128i out_16_6 = _mm_srai_epi32(out_16_4, DCT_CONST_BITS); + const __m128i out_16_7 = _mm_srai_epi32(out_16_5, DCT_CONST_BITS); + const __m128i out_08_6 = _mm_srai_epi32(out_08_4, DCT_CONST_BITS); + const __m128i out_08_7 = _mm_srai_epi32(out_08_5, DCT_CONST_BITS); + const __m128i out_24_6 = _mm_srai_epi32(out_24_4, DCT_CONST_BITS); + const __m128i out_24_7 = _mm_srai_epi32(out_24_5, DCT_CONST_BITS); + // Combine + out[0] = _mm_packs_epi32(out_00_6, out_00_7); + out[16] = _mm_packs_epi32(out_16_6, out_16_7); + out[8] = _mm_packs_epi32(out_08_6, out_08_7); + out[24] = _mm_packs_epi32(out_24_6, out_24_7); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x4(&out[0], &out[16], &out[8], &out[24]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + const __m128i s2_09_0 = _mm_unpacklo_epi16(step1[9], step1[14]); + const __m128i s2_09_1 = _mm_unpackhi_epi16(step1[9], step1[14]); + const __m128i s2_10_0 = _mm_unpacklo_epi16(step1[10], step1[13]); + const __m128i s2_10_1 = _mm_unpackhi_epi16(step1[10], step1[13]); + const __m128i s2_09_2 = _mm_madd_epi16(s2_09_0, k__cospi_m08_p24); + const __m128i s2_09_3 = _mm_madd_epi16(s2_09_1, k__cospi_m08_p24); + const __m128i s2_10_2 = _mm_madd_epi16(s2_10_0, k__cospi_m24_m08); + const __m128i s2_10_3 = _mm_madd_epi16(s2_10_1, k__cospi_m24_m08); + const __m128i s2_13_2 = _mm_madd_epi16(s2_10_0, k__cospi_m08_p24); + const __m128i s2_13_3 = _mm_madd_epi16(s2_10_1, k__cospi_m08_p24); + const __m128i s2_14_2 = _mm_madd_epi16(s2_09_0, k__cospi_p24_p08); + const __m128i s2_14_3 = _mm_madd_epi16(s2_09_1, k__cospi_p24_p08); + // dct_const_round_shift + const __m128i s2_09_4 = _mm_add_epi32(s2_09_2, k__DCT_CONST_ROUNDING); + const __m128i s2_09_5 = _mm_add_epi32(s2_09_3, k__DCT_CONST_ROUNDING); + const __m128i s2_10_4 = _mm_add_epi32(s2_10_2, k__DCT_CONST_ROUNDING); + const __m128i s2_10_5 = _mm_add_epi32(s2_10_3, k__DCT_CONST_ROUNDING); + const __m128i s2_13_4 = _mm_add_epi32(s2_13_2, k__DCT_CONST_ROUNDING); + const __m128i s2_13_5 = _mm_add_epi32(s2_13_3, k__DCT_CONST_ROUNDING); + const __m128i s2_14_4 = _mm_add_epi32(s2_14_2, k__DCT_CONST_ROUNDING); + const __m128i s2_14_5 = _mm_add_epi32(s2_14_3, k__DCT_CONST_ROUNDING); + const __m128i s2_09_6 = _mm_srai_epi32(s2_09_4, DCT_CONST_BITS); + const __m128i s2_09_7 = _mm_srai_epi32(s2_09_5, DCT_CONST_BITS); + const __m128i s2_10_6 = _mm_srai_epi32(s2_10_4, DCT_CONST_BITS); + const __m128i s2_10_7 = _mm_srai_epi32(s2_10_5, DCT_CONST_BITS); + const __m128i s2_13_6 = _mm_srai_epi32(s2_13_4, DCT_CONST_BITS); + const __m128i s2_13_7 = _mm_srai_epi32(s2_13_5, DCT_CONST_BITS); + const __m128i s2_14_6 = _mm_srai_epi32(s2_14_4, DCT_CONST_BITS); + const __m128i s2_14_7 = _mm_srai_epi32(s2_14_5, DCT_CONST_BITS); + // Combine + step2[9] = _mm_packs_epi32(s2_09_6, s2_09_7); + step2[10] = _mm_packs_epi32(s2_10_6, s2_10_7); + step2[13] = _mm_packs_epi32(s2_13_6, s2_13_7); + step2[14] = _mm_packs_epi32(s2_14_6, s2_14_7); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x4(&step2[9], &step2[10], &step2[13], + &step2[14]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + step2[16] = ADD_EPI16(step1[19], step3[16]); + step2[17] = ADD_EPI16(step1[18], step3[17]); + step2[18] = SUB_EPI16(step3[17], step1[18]); + step2[19] = SUB_EPI16(step3[16], step1[19]); + step2[20] = SUB_EPI16(step3[23], step1[20]); + step2[21] = SUB_EPI16(step3[22], step1[21]); + step2[22] = ADD_EPI16(step1[21], step3[22]); + step2[23] = ADD_EPI16(step1[20], step3[23]); + step2[24] = ADD_EPI16(step1[27], step3[24]); + step2[25] = ADD_EPI16(step1[26], step3[25]); + step2[26] = SUB_EPI16(step3[25], step1[26]); + step2[27] = SUB_EPI16(step3[24], step1[27]); + step2[28] = SUB_EPI16(step3[31], step1[28]); + step2[29] = SUB_EPI16(step3[30], step1[29]); + step2[30] = ADD_EPI16(step1[29], step3[30]); + step2[31] = ADD_EPI16(step1[28], step3[31]); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x16( + &step2[16], &step2[17], &step2[18], &step2[19], &step2[20], + &step2[21], &step2[22], &step2[23], &step2[24], &step2[25], + &step2[26], &step2[27], &step2[28], &step2[29], &step2[30], + &step2[31]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + // Stage 6 + { + const __m128i out_04_0 = _mm_unpacklo_epi16(step2[4], step2[7]); + const __m128i out_04_1 = _mm_unpackhi_epi16(step2[4], step2[7]); + const __m128i out_20_0 = _mm_unpacklo_epi16(step2[5], step2[6]); + const __m128i out_20_1 = _mm_unpackhi_epi16(step2[5], step2[6]); + const __m128i out_12_0 = _mm_unpacklo_epi16(step2[5], step2[6]); + const __m128i out_12_1 = _mm_unpackhi_epi16(step2[5], step2[6]); + const __m128i out_28_0 = _mm_unpacklo_epi16(step2[4], step2[7]); + const __m128i out_28_1 = _mm_unpackhi_epi16(step2[4], step2[7]); + const __m128i out_04_2 = _mm_madd_epi16(out_04_0, k__cospi_p28_p04); + const __m128i out_04_3 = _mm_madd_epi16(out_04_1, k__cospi_p28_p04); + const __m128i out_20_2 = _mm_madd_epi16(out_20_0, k__cospi_p12_p20); + const __m128i out_20_3 = _mm_madd_epi16(out_20_1, k__cospi_p12_p20); + const __m128i out_12_2 = _mm_madd_epi16(out_12_0, k__cospi_m20_p12); + const __m128i out_12_3 = _mm_madd_epi16(out_12_1, k__cospi_m20_p12); + const __m128i out_28_2 = _mm_madd_epi16(out_28_0, k__cospi_m04_p28); + const __m128i out_28_3 = _mm_madd_epi16(out_28_1, k__cospi_m04_p28); + // dct_const_round_shift + const __m128i out_04_4 = + _mm_add_epi32(out_04_2, k__DCT_CONST_ROUNDING); + const __m128i out_04_5 = + _mm_add_epi32(out_04_3, k__DCT_CONST_ROUNDING); + const __m128i out_20_4 = + _mm_add_epi32(out_20_2, k__DCT_CONST_ROUNDING); + const __m128i out_20_5 = + _mm_add_epi32(out_20_3, k__DCT_CONST_ROUNDING); + const __m128i out_12_4 = + _mm_add_epi32(out_12_2, k__DCT_CONST_ROUNDING); + const __m128i out_12_5 = + _mm_add_epi32(out_12_3, k__DCT_CONST_ROUNDING); + const __m128i out_28_4 = + _mm_add_epi32(out_28_2, k__DCT_CONST_ROUNDING); + const __m128i out_28_5 = + _mm_add_epi32(out_28_3, k__DCT_CONST_ROUNDING); + const __m128i out_04_6 = _mm_srai_epi32(out_04_4, DCT_CONST_BITS); + const __m128i out_04_7 = _mm_srai_epi32(out_04_5, DCT_CONST_BITS); + const __m128i out_20_6 = _mm_srai_epi32(out_20_4, DCT_CONST_BITS); + const __m128i out_20_7 = _mm_srai_epi32(out_20_5, DCT_CONST_BITS); + const __m128i out_12_6 = _mm_srai_epi32(out_12_4, DCT_CONST_BITS); + const __m128i out_12_7 = _mm_srai_epi32(out_12_5, DCT_CONST_BITS); + const __m128i out_28_6 = _mm_srai_epi32(out_28_4, DCT_CONST_BITS); + const __m128i out_28_7 = _mm_srai_epi32(out_28_5, DCT_CONST_BITS); + // Combine + out[4] = _mm_packs_epi32(out_04_6, out_04_7); + out[20] = _mm_packs_epi32(out_20_6, out_20_7); + out[12] = _mm_packs_epi32(out_12_6, out_12_7); + out[28] = _mm_packs_epi32(out_28_6, out_28_7); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x4(&out[4], &out[20], &out[12], &out[28]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + step3[8] = ADD_EPI16(step2[9], step1[8]); + step3[9] = SUB_EPI16(step1[8], step2[9]); + step3[10] = SUB_EPI16(step1[11], step2[10]); + step3[11] = ADD_EPI16(step2[10], step1[11]); + step3[12] = ADD_EPI16(step2[13], step1[12]); + step3[13] = SUB_EPI16(step1[12], step2[13]); + step3[14] = SUB_EPI16(step1[15], step2[14]); + step3[15] = ADD_EPI16(step2[14], step1[15]); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x8(&step3[8], &step3[9], &step3[10], + &step3[11], &step3[12], &step3[13], + &step3[14], &step3[15]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + const __m128i s3_17_0 = _mm_unpacklo_epi16(step2[17], step2[30]); + const __m128i s3_17_1 = _mm_unpackhi_epi16(step2[17], step2[30]); + const __m128i s3_18_0 = _mm_unpacklo_epi16(step2[18], step2[29]); + const __m128i s3_18_1 = _mm_unpackhi_epi16(step2[18], step2[29]); + const __m128i s3_21_0 = _mm_unpacklo_epi16(step2[21], step2[26]); + const __m128i s3_21_1 = _mm_unpackhi_epi16(step2[21], step2[26]); + const __m128i s3_22_0 = _mm_unpacklo_epi16(step2[22], step2[25]); + const __m128i s3_22_1 = _mm_unpackhi_epi16(step2[22], step2[25]); + const __m128i s3_17_2 = _mm_madd_epi16(s3_17_0, k__cospi_m04_p28); + const __m128i s3_17_3 = _mm_madd_epi16(s3_17_1, k__cospi_m04_p28); + const __m128i s3_18_2 = _mm_madd_epi16(s3_18_0, k__cospi_m28_m04); + const __m128i s3_18_3 = _mm_madd_epi16(s3_18_1, k__cospi_m28_m04); + const __m128i s3_21_2 = _mm_madd_epi16(s3_21_0, k__cospi_m20_p12); + const __m128i s3_21_3 = _mm_madd_epi16(s3_21_1, k__cospi_m20_p12); + const __m128i s3_22_2 = _mm_madd_epi16(s3_22_0, k__cospi_m12_m20); + const __m128i s3_22_3 = _mm_madd_epi16(s3_22_1, k__cospi_m12_m20); + const __m128i s3_25_2 = _mm_madd_epi16(s3_22_0, k__cospi_m20_p12); + const __m128i s3_25_3 = _mm_madd_epi16(s3_22_1, k__cospi_m20_p12); + const __m128i s3_26_2 = _mm_madd_epi16(s3_21_0, k__cospi_p12_p20); + const __m128i s3_26_3 = _mm_madd_epi16(s3_21_1, k__cospi_p12_p20); + const __m128i s3_29_2 = _mm_madd_epi16(s3_18_0, k__cospi_m04_p28); + const __m128i s3_29_3 = _mm_madd_epi16(s3_18_1, k__cospi_m04_p28); + const __m128i s3_30_2 = _mm_madd_epi16(s3_17_0, k__cospi_p28_p04); + const __m128i s3_30_3 = _mm_madd_epi16(s3_17_1, k__cospi_p28_p04); + // dct_const_round_shift + const __m128i s3_17_4 = _mm_add_epi32(s3_17_2, k__DCT_CONST_ROUNDING); + const __m128i s3_17_5 = _mm_add_epi32(s3_17_3, k__DCT_CONST_ROUNDING); + const __m128i s3_18_4 = _mm_add_epi32(s3_18_2, k__DCT_CONST_ROUNDING); + const __m128i s3_18_5 = _mm_add_epi32(s3_18_3, k__DCT_CONST_ROUNDING); + const __m128i s3_21_4 = _mm_add_epi32(s3_21_2, k__DCT_CONST_ROUNDING); + const __m128i s3_21_5 = _mm_add_epi32(s3_21_3, k__DCT_CONST_ROUNDING); + const __m128i s3_22_4 = _mm_add_epi32(s3_22_2, k__DCT_CONST_ROUNDING); + const __m128i s3_22_5 = _mm_add_epi32(s3_22_3, k__DCT_CONST_ROUNDING); + const __m128i s3_17_6 = _mm_srai_epi32(s3_17_4, DCT_CONST_BITS); + const __m128i s3_17_7 = _mm_srai_epi32(s3_17_5, DCT_CONST_BITS); + const __m128i s3_18_6 = _mm_srai_epi32(s3_18_4, DCT_CONST_BITS); + const __m128i s3_18_7 = _mm_srai_epi32(s3_18_5, DCT_CONST_BITS); + const __m128i s3_21_6 = _mm_srai_epi32(s3_21_4, DCT_CONST_BITS); + const __m128i s3_21_7 = _mm_srai_epi32(s3_21_5, DCT_CONST_BITS); + const __m128i s3_22_6 = _mm_srai_epi32(s3_22_4, DCT_CONST_BITS); + const __m128i s3_22_7 = _mm_srai_epi32(s3_22_5, DCT_CONST_BITS); + const __m128i s3_25_4 = _mm_add_epi32(s3_25_2, k__DCT_CONST_ROUNDING); + const __m128i s3_25_5 = _mm_add_epi32(s3_25_3, k__DCT_CONST_ROUNDING); + const __m128i s3_26_4 = _mm_add_epi32(s3_26_2, k__DCT_CONST_ROUNDING); + const __m128i s3_26_5 = _mm_add_epi32(s3_26_3, k__DCT_CONST_ROUNDING); + const __m128i s3_29_4 = _mm_add_epi32(s3_29_2, k__DCT_CONST_ROUNDING); + const __m128i s3_29_5 = _mm_add_epi32(s3_29_3, k__DCT_CONST_ROUNDING); + const __m128i s3_30_4 = _mm_add_epi32(s3_30_2, k__DCT_CONST_ROUNDING); + const __m128i s3_30_5 = _mm_add_epi32(s3_30_3, k__DCT_CONST_ROUNDING); + const __m128i s3_25_6 = _mm_srai_epi32(s3_25_4, DCT_CONST_BITS); + const __m128i s3_25_7 = _mm_srai_epi32(s3_25_5, DCT_CONST_BITS); + const __m128i s3_26_6 = _mm_srai_epi32(s3_26_4, DCT_CONST_BITS); + const __m128i s3_26_7 = _mm_srai_epi32(s3_26_5, DCT_CONST_BITS); + const __m128i s3_29_6 = _mm_srai_epi32(s3_29_4, DCT_CONST_BITS); + const __m128i s3_29_7 = _mm_srai_epi32(s3_29_5, DCT_CONST_BITS); + const __m128i s3_30_6 = _mm_srai_epi32(s3_30_4, DCT_CONST_BITS); + const __m128i s3_30_7 = _mm_srai_epi32(s3_30_5, DCT_CONST_BITS); + // Combine + step3[17] = _mm_packs_epi32(s3_17_6, s3_17_7); + step3[18] = _mm_packs_epi32(s3_18_6, s3_18_7); + step3[21] = _mm_packs_epi32(s3_21_6, s3_21_7); + step3[22] = _mm_packs_epi32(s3_22_6, s3_22_7); + // Combine + step3[25] = _mm_packs_epi32(s3_25_6, s3_25_7); + step3[26] = _mm_packs_epi32(s3_26_6, s3_26_7); + step3[29] = _mm_packs_epi32(s3_29_6, s3_29_7); + step3[30] = _mm_packs_epi32(s3_30_6, s3_30_7); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x8(&step3[17], &step3[18], &step3[21], + &step3[22], &step3[25], &step3[26], + &step3[29], &step3[30]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + // Stage 7 + { + const __m128i out_02_0 = _mm_unpacklo_epi16(step3[8], step3[15]); + const __m128i out_02_1 = _mm_unpackhi_epi16(step3[8], step3[15]); + const __m128i out_18_0 = _mm_unpacklo_epi16(step3[9], step3[14]); + const __m128i out_18_1 = _mm_unpackhi_epi16(step3[9], step3[14]); + const __m128i out_10_0 = _mm_unpacklo_epi16(step3[10], step3[13]); + const __m128i out_10_1 = _mm_unpackhi_epi16(step3[10], step3[13]); + const __m128i out_26_0 = _mm_unpacklo_epi16(step3[11], step3[12]); + const __m128i out_26_1 = _mm_unpackhi_epi16(step3[11], step3[12]); + const __m128i out_02_2 = _mm_madd_epi16(out_02_0, k__cospi_p30_p02); + const __m128i out_02_3 = _mm_madd_epi16(out_02_1, k__cospi_p30_p02); + const __m128i out_18_2 = _mm_madd_epi16(out_18_0, k__cospi_p14_p18); + const __m128i out_18_3 = _mm_madd_epi16(out_18_1, k__cospi_p14_p18); + const __m128i out_10_2 = _mm_madd_epi16(out_10_0, k__cospi_p22_p10); + const __m128i out_10_3 = _mm_madd_epi16(out_10_1, k__cospi_p22_p10); + const __m128i out_26_2 = _mm_madd_epi16(out_26_0, k__cospi_p06_p26); + const __m128i out_26_3 = _mm_madd_epi16(out_26_1, k__cospi_p06_p26); + const __m128i out_06_2 = _mm_madd_epi16(out_26_0, k__cospi_m26_p06); + const __m128i out_06_3 = _mm_madd_epi16(out_26_1, k__cospi_m26_p06); + const __m128i out_22_2 = _mm_madd_epi16(out_10_0, k__cospi_m10_p22); + const __m128i out_22_3 = _mm_madd_epi16(out_10_1, k__cospi_m10_p22); + const __m128i out_14_2 = _mm_madd_epi16(out_18_0, k__cospi_m18_p14); + const __m128i out_14_3 = _mm_madd_epi16(out_18_1, k__cospi_m18_p14); + const __m128i out_30_2 = _mm_madd_epi16(out_02_0, k__cospi_m02_p30); + const __m128i out_30_3 = _mm_madd_epi16(out_02_1, k__cospi_m02_p30); + // dct_const_round_shift + const __m128i out_02_4 = + _mm_add_epi32(out_02_2, k__DCT_CONST_ROUNDING); + const __m128i out_02_5 = + _mm_add_epi32(out_02_3, k__DCT_CONST_ROUNDING); + const __m128i out_18_4 = + _mm_add_epi32(out_18_2, k__DCT_CONST_ROUNDING); + const __m128i out_18_5 = + _mm_add_epi32(out_18_3, k__DCT_CONST_ROUNDING); + const __m128i out_10_4 = + _mm_add_epi32(out_10_2, k__DCT_CONST_ROUNDING); + const __m128i out_10_5 = + _mm_add_epi32(out_10_3, k__DCT_CONST_ROUNDING); + const __m128i out_26_4 = + _mm_add_epi32(out_26_2, k__DCT_CONST_ROUNDING); + const __m128i out_26_5 = + _mm_add_epi32(out_26_3, k__DCT_CONST_ROUNDING); + const __m128i out_06_4 = + _mm_add_epi32(out_06_2, k__DCT_CONST_ROUNDING); + const __m128i out_06_5 = + _mm_add_epi32(out_06_3, k__DCT_CONST_ROUNDING); + const __m128i out_22_4 = + _mm_add_epi32(out_22_2, k__DCT_CONST_ROUNDING); + const __m128i out_22_5 = + _mm_add_epi32(out_22_3, k__DCT_CONST_ROUNDING); + const __m128i out_14_4 = + _mm_add_epi32(out_14_2, k__DCT_CONST_ROUNDING); + const __m128i out_14_5 = + _mm_add_epi32(out_14_3, k__DCT_CONST_ROUNDING); + const __m128i out_30_4 = + _mm_add_epi32(out_30_2, k__DCT_CONST_ROUNDING); + const __m128i out_30_5 = + _mm_add_epi32(out_30_3, k__DCT_CONST_ROUNDING); + const __m128i out_02_6 = _mm_srai_epi32(out_02_4, DCT_CONST_BITS); + const __m128i out_02_7 = _mm_srai_epi32(out_02_5, DCT_CONST_BITS); + const __m128i out_18_6 = _mm_srai_epi32(out_18_4, DCT_CONST_BITS); + const __m128i out_18_7 = _mm_srai_epi32(out_18_5, DCT_CONST_BITS); + const __m128i out_10_6 = _mm_srai_epi32(out_10_4, DCT_CONST_BITS); + const __m128i out_10_7 = _mm_srai_epi32(out_10_5, DCT_CONST_BITS); + const __m128i out_26_6 = _mm_srai_epi32(out_26_4, DCT_CONST_BITS); + const __m128i out_26_7 = _mm_srai_epi32(out_26_5, DCT_CONST_BITS); + const __m128i out_06_6 = _mm_srai_epi32(out_06_4, DCT_CONST_BITS); + const __m128i out_06_7 = _mm_srai_epi32(out_06_5, DCT_CONST_BITS); + const __m128i out_22_6 = _mm_srai_epi32(out_22_4, DCT_CONST_BITS); + const __m128i out_22_7 = _mm_srai_epi32(out_22_5, DCT_CONST_BITS); + const __m128i out_14_6 = _mm_srai_epi32(out_14_4, DCT_CONST_BITS); + const __m128i out_14_7 = _mm_srai_epi32(out_14_5, DCT_CONST_BITS); + const __m128i out_30_6 = _mm_srai_epi32(out_30_4, DCT_CONST_BITS); + const __m128i out_30_7 = _mm_srai_epi32(out_30_5, DCT_CONST_BITS); + // Combine + out[2] = _mm_packs_epi32(out_02_6, out_02_7); + out[18] = _mm_packs_epi32(out_18_6, out_18_7); + out[10] = _mm_packs_epi32(out_10_6, out_10_7); + out[26] = _mm_packs_epi32(out_26_6, out_26_7); + out[6] = _mm_packs_epi32(out_06_6, out_06_7); + out[22] = _mm_packs_epi32(out_22_6, out_22_7); + out[14] = _mm_packs_epi32(out_14_6, out_14_7); + out[30] = _mm_packs_epi32(out_30_6, out_30_7); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x8(&out[2], &out[18], &out[10], &out[26], + &out[6], &out[22], &out[14], &out[30]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + step1[16] = ADD_EPI16(step3[17], step2[16]); + step1[17] = SUB_EPI16(step2[16], step3[17]); + step1[18] = SUB_EPI16(step2[19], step3[18]); + step1[19] = ADD_EPI16(step3[18], step2[19]); + step1[20] = ADD_EPI16(step3[21], step2[20]); + step1[21] = SUB_EPI16(step2[20], step3[21]); + step1[22] = SUB_EPI16(step2[23], step3[22]); + step1[23] = ADD_EPI16(step3[22], step2[23]); + step1[24] = ADD_EPI16(step3[25], step2[24]); + step1[25] = SUB_EPI16(step2[24], step3[25]); + step1[26] = SUB_EPI16(step2[27], step3[26]); + step1[27] = ADD_EPI16(step3[26], step2[27]); + step1[28] = ADD_EPI16(step3[29], step2[28]); + step1[29] = SUB_EPI16(step2[28], step3[29]); + step1[30] = SUB_EPI16(step2[31], step3[30]); + step1[31] = ADD_EPI16(step3[30], step2[31]); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x16( + &step1[16], &step1[17], &step1[18], &step1[19], &step1[20], + &step1[21], &step1[22], &step1[23], &step1[24], &step1[25], + &step1[26], &step1[27], &step1[28], &step1[29], &step1[30], + &step1[31]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + // Final stage --- outputs indices are bit-reversed. + { + const __m128i out_01_0 = _mm_unpacklo_epi16(step1[16], step1[31]); + const __m128i out_01_1 = _mm_unpackhi_epi16(step1[16], step1[31]); + const __m128i out_17_0 = _mm_unpacklo_epi16(step1[17], step1[30]); + const __m128i out_17_1 = _mm_unpackhi_epi16(step1[17], step1[30]); + const __m128i out_09_0 = _mm_unpacklo_epi16(step1[18], step1[29]); + const __m128i out_09_1 = _mm_unpackhi_epi16(step1[18], step1[29]); + const __m128i out_25_0 = _mm_unpacklo_epi16(step1[19], step1[28]); + const __m128i out_25_1 = _mm_unpackhi_epi16(step1[19], step1[28]); + const __m128i out_01_2 = _mm_madd_epi16(out_01_0, k__cospi_p31_p01); + const __m128i out_01_3 = _mm_madd_epi16(out_01_1, k__cospi_p31_p01); + const __m128i out_17_2 = _mm_madd_epi16(out_17_0, k__cospi_p15_p17); + const __m128i out_17_3 = _mm_madd_epi16(out_17_1, k__cospi_p15_p17); + const __m128i out_09_2 = _mm_madd_epi16(out_09_0, k__cospi_p23_p09); + const __m128i out_09_3 = _mm_madd_epi16(out_09_1, k__cospi_p23_p09); + const __m128i out_25_2 = _mm_madd_epi16(out_25_0, k__cospi_p07_p25); + const __m128i out_25_3 = _mm_madd_epi16(out_25_1, k__cospi_p07_p25); + const __m128i out_07_2 = _mm_madd_epi16(out_25_0, k__cospi_m25_p07); + const __m128i out_07_3 = _mm_madd_epi16(out_25_1, k__cospi_m25_p07); + const __m128i out_23_2 = _mm_madd_epi16(out_09_0, k__cospi_m09_p23); + const __m128i out_23_3 = _mm_madd_epi16(out_09_1, k__cospi_m09_p23); + const __m128i out_15_2 = _mm_madd_epi16(out_17_0, k__cospi_m17_p15); + const __m128i out_15_3 = _mm_madd_epi16(out_17_1, k__cospi_m17_p15); + const __m128i out_31_2 = _mm_madd_epi16(out_01_0, k__cospi_m01_p31); + const __m128i out_31_3 = _mm_madd_epi16(out_01_1, k__cospi_m01_p31); + // dct_const_round_shift + const __m128i out_01_4 = + _mm_add_epi32(out_01_2, k__DCT_CONST_ROUNDING); + const __m128i out_01_5 = + _mm_add_epi32(out_01_3, k__DCT_CONST_ROUNDING); + const __m128i out_17_4 = + _mm_add_epi32(out_17_2, k__DCT_CONST_ROUNDING); + const __m128i out_17_5 = + _mm_add_epi32(out_17_3, k__DCT_CONST_ROUNDING); + const __m128i out_09_4 = + _mm_add_epi32(out_09_2, k__DCT_CONST_ROUNDING); + const __m128i out_09_5 = + _mm_add_epi32(out_09_3, k__DCT_CONST_ROUNDING); + const __m128i out_25_4 = + _mm_add_epi32(out_25_2, k__DCT_CONST_ROUNDING); + const __m128i out_25_5 = + _mm_add_epi32(out_25_3, k__DCT_CONST_ROUNDING); + const __m128i out_07_4 = + _mm_add_epi32(out_07_2, k__DCT_CONST_ROUNDING); + const __m128i out_07_5 = + _mm_add_epi32(out_07_3, k__DCT_CONST_ROUNDING); + const __m128i out_23_4 = + _mm_add_epi32(out_23_2, k__DCT_CONST_ROUNDING); + const __m128i out_23_5 = + _mm_add_epi32(out_23_3, k__DCT_CONST_ROUNDING); + const __m128i out_15_4 = + _mm_add_epi32(out_15_2, k__DCT_CONST_ROUNDING); + const __m128i out_15_5 = + _mm_add_epi32(out_15_3, k__DCT_CONST_ROUNDING); + const __m128i out_31_4 = + _mm_add_epi32(out_31_2, k__DCT_CONST_ROUNDING); + const __m128i out_31_5 = + _mm_add_epi32(out_31_3, k__DCT_CONST_ROUNDING); + const __m128i out_01_6 = _mm_srai_epi32(out_01_4, DCT_CONST_BITS); + const __m128i out_01_7 = _mm_srai_epi32(out_01_5, DCT_CONST_BITS); + const __m128i out_17_6 = _mm_srai_epi32(out_17_4, DCT_CONST_BITS); + const __m128i out_17_7 = _mm_srai_epi32(out_17_5, DCT_CONST_BITS); + const __m128i out_09_6 = _mm_srai_epi32(out_09_4, DCT_CONST_BITS); + const __m128i out_09_7 = _mm_srai_epi32(out_09_5, DCT_CONST_BITS); + const __m128i out_25_6 = _mm_srai_epi32(out_25_4, DCT_CONST_BITS); + const __m128i out_25_7 = _mm_srai_epi32(out_25_5, DCT_CONST_BITS); + const __m128i out_07_6 = _mm_srai_epi32(out_07_4, DCT_CONST_BITS); + const __m128i out_07_7 = _mm_srai_epi32(out_07_5, DCT_CONST_BITS); + const __m128i out_23_6 = _mm_srai_epi32(out_23_4, DCT_CONST_BITS); + const __m128i out_23_7 = _mm_srai_epi32(out_23_5, DCT_CONST_BITS); + const __m128i out_15_6 = _mm_srai_epi32(out_15_4, DCT_CONST_BITS); + const __m128i out_15_7 = _mm_srai_epi32(out_15_5, DCT_CONST_BITS); + const __m128i out_31_6 = _mm_srai_epi32(out_31_4, DCT_CONST_BITS); + const __m128i out_31_7 = _mm_srai_epi32(out_31_5, DCT_CONST_BITS); + // Combine + out[1] = _mm_packs_epi32(out_01_6, out_01_7); + out[17] = _mm_packs_epi32(out_17_6, out_17_7); + out[9] = _mm_packs_epi32(out_09_6, out_09_7); + out[25] = _mm_packs_epi32(out_25_6, out_25_7); + out[7] = _mm_packs_epi32(out_07_6, out_07_7); + out[23] = _mm_packs_epi32(out_23_6, out_23_7); + out[15] = _mm_packs_epi32(out_15_6, out_15_7); + out[31] = _mm_packs_epi32(out_31_6, out_31_7); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x8(&out[1], &out[17], &out[9], &out[25], + &out[7], &out[23], &out[15], &out[31]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + const __m128i out_05_0 = _mm_unpacklo_epi16(step1[20], step1[27]); + const __m128i out_05_1 = _mm_unpackhi_epi16(step1[20], step1[27]); + const __m128i out_21_0 = _mm_unpacklo_epi16(step1[21], step1[26]); + const __m128i out_21_1 = _mm_unpackhi_epi16(step1[21], step1[26]); + const __m128i out_13_0 = _mm_unpacklo_epi16(step1[22], step1[25]); + const __m128i out_13_1 = _mm_unpackhi_epi16(step1[22], step1[25]); + const __m128i out_29_0 = _mm_unpacklo_epi16(step1[23], step1[24]); + const __m128i out_29_1 = _mm_unpackhi_epi16(step1[23], step1[24]); + const __m128i out_05_2 = _mm_madd_epi16(out_05_0, k__cospi_p27_p05); + const __m128i out_05_3 = _mm_madd_epi16(out_05_1, k__cospi_p27_p05); + const __m128i out_21_2 = _mm_madd_epi16(out_21_0, k__cospi_p11_p21); + const __m128i out_21_3 = _mm_madd_epi16(out_21_1, k__cospi_p11_p21); + const __m128i out_13_2 = _mm_madd_epi16(out_13_0, k__cospi_p19_p13); + const __m128i out_13_3 = _mm_madd_epi16(out_13_1, k__cospi_p19_p13); + const __m128i out_29_2 = _mm_madd_epi16(out_29_0, k__cospi_p03_p29); + const __m128i out_29_3 = _mm_madd_epi16(out_29_1, k__cospi_p03_p29); + const __m128i out_03_2 = _mm_madd_epi16(out_29_0, k__cospi_m29_p03); + const __m128i out_03_3 = _mm_madd_epi16(out_29_1, k__cospi_m29_p03); + const __m128i out_19_2 = _mm_madd_epi16(out_13_0, k__cospi_m13_p19); + const __m128i out_19_3 = _mm_madd_epi16(out_13_1, k__cospi_m13_p19); + const __m128i out_11_2 = _mm_madd_epi16(out_21_0, k__cospi_m21_p11); + const __m128i out_11_3 = _mm_madd_epi16(out_21_1, k__cospi_m21_p11); + const __m128i out_27_2 = _mm_madd_epi16(out_05_0, k__cospi_m05_p27); + const __m128i out_27_3 = _mm_madd_epi16(out_05_1, k__cospi_m05_p27); + // dct_const_round_shift + const __m128i out_05_4 = + _mm_add_epi32(out_05_2, k__DCT_CONST_ROUNDING); + const __m128i out_05_5 = + _mm_add_epi32(out_05_3, k__DCT_CONST_ROUNDING); + const __m128i out_21_4 = + _mm_add_epi32(out_21_2, k__DCT_CONST_ROUNDING); + const __m128i out_21_5 = + _mm_add_epi32(out_21_3, k__DCT_CONST_ROUNDING); + const __m128i out_13_4 = + _mm_add_epi32(out_13_2, k__DCT_CONST_ROUNDING); + const __m128i out_13_5 = + _mm_add_epi32(out_13_3, k__DCT_CONST_ROUNDING); + const __m128i out_29_4 = + _mm_add_epi32(out_29_2, k__DCT_CONST_ROUNDING); + const __m128i out_29_5 = + _mm_add_epi32(out_29_3, k__DCT_CONST_ROUNDING); + const __m128i out_03_4 = + _mm_add_epi32(out_03_2, k__DCT_CONST_ROUNDING); + const __m128i out_03_5 = + _mm_add_epi32(out_03_3, k__DCT_CONST_ROUNDING); + const __m128i out_19_4 = + _mm_add_epi32(out_19_2, k__DCT_CONST_ROUNDING); + const __m128i out_19_5 = + _mm_add_epi32(out_19_3, k__DCT_CONST_ROUNDING); + const __m128i out_11_4 = + _mm_add_epi32(out_11_2, k__DCT_CONST_ROUNDING); + const __m128i out_11_5 = + _mm_add_epi32(out_11_3, k__DCT_CONST_ROUNDING); + const __m128i out_27_4 = + _mm_add_epi32(out_27_2, k__DCT_CONST_ROUNDING); + const __m128i out_27_5 = + _mm_add_epi32(out_27_3, k__DCT_CONST_ROUNDING); + const __m128i out_05_6 = _mm_srai_epi32(out_05_4, DCT_CONST_BITS); + const __m128i out_05_7 = _mm_srai_epi32(out_05_5, DCT_CONST_BITS); + const __m128i out_21_6 = _mm_srai_epi32(out_21_4, DCT_CONST_BITS); + const __m128i out_21_7 = _mm_srai_epi32(out_21_5, DCT_CONST_BITS); + const __m128i out_13_6 = _mm_srai_epi32(out_13_4, DCT_CONST_BITS); + const __m128i out_13_7 = _mm_srai_epi32(out_13_5, DCT_CONST_BITS); + const __m128i out_29_6 = _mm_srai_epi32(out_29_4, DCT_CONST_BITS); + const __m128i out_29_7 = _mm_srai_epi32(out_29_5, DCT_CONST_BITS); + const __m128i out_03_6 = _mm_srai_epi32(out_03_4, DCT_CONST_BITS); + const __m128i out_03_7 = _mm_srai_epi32(out_03_5, DCT_CONST_BITS); + const __m128i out_19_6 = _mm_srai_epi32(out_19_4, DCT_CONST_BITS); + const __m128i out_19_7 = _mm_srai_epi32(out_19_5, DCT_CONST_BITS); + const __m128i out_11_6 = _mm_srai_epi32(out_11_4, DCT_CONST_BITS); + const __m128i out_11_7 = _mm_srai_epi32(out_11_5, DCT_CONST_BITS); + const __m128i out_27_6 = _mm_srai_epi32(out_27_4, DCT_CONST_BITS); + const __m128i out_27_7 = _mm_srai_epi32(out_27_5, DCT_CONST_BITS); + // Combine + out[5] = _mm_packs_epi32(out_05_6, out_05_7); + out[21] = _mm_packs_epi32(out_21_6, out_21_7); + out[13] = _mm_packs_epi32(out_13_6, out_13_7); + out[29] = _mm_packs_epi32(out_29_6, out_29_7); + out[3] = _mm_packs_epi32(out_03_6, out_03_7); + out[19] = _mm_packs_epi32(out_19_6, out_19_7); + out[11] = _mm_packs_epi32(out_11_6, out_11_7); + out[27] = _mm_packs_epi32(out_27_6, out_27_7); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x8(&out[5], &out[21], &out[13], &out[29], + &out[3], &out[19], &out[11], &out[27]); + if (overflow) { + if (pass == 0) + HIGH_FDCT32x32_2D_C(input, output_org, stride); + else + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } +#if FDCT32x32_HIGH_PRECISION + } else { + __m128i lstep1[64], lstep2[64], lstep3[64]; + __m128i u[32], v[32], sign[16]; + const __m128i K32One = _mm_set_epi32(1, 1, 1, 1); + // start using 32-bit operations + // stage 3 + { + // expanding to 32-bit length priori to addition operations + lstep2[0] = _mm_unpacklo_epi16(step2[0], kZero); + lstep2[1] = _mm_unpackhi_epi16(step2[0], kZero); + lstep2[2] = _mm_unpacklo_epi16(step2[1], kZero); + lstep2[3] = _mm_unpackhi_epi16(step2[1], kZero); + lstep2[4] = _mm_unpacklo_epi16(step2[2], kZero); + lstep2[5] = _mm_unpackhi_epi16(step2[2], kZero); + lstep2[6] = _mm_unpacklo_epi16(step2[3], kZero); + lstep2[7] = _mm_unpackhi_epi16(step2[3], kZero); + lstep2[8] = _mm_unpacklo_epi16(step2[4], kZero); + lstep2[9] = _mm_unpackhi_epi16(step2[4], kZero); + lstep2[10] = _mm_unpacklo_epi16(step2[5], kZero); + lstep2[11] = _mm_unpackhi_epi16(step2[5], kZero); + lstep2[12] = _mm_unpacklo_epi16(step2[6], kZero); + lstep2[13] = _mm_unpackhi_epi16(step2[6], kZero); + lstep2[14] = _mm_unpacklo_epi16(step2[7], kZero); + lstep2[15] = _mm_unpackhi_epi16(step2[7], kZero); + lstep2[0] = _mm_madd_epi16(lstep2[0], kOne); + lstep2[1] = _mm_madd_epi16(lstep2[1], kOne); + lstep2[2] = _mm_madd_epi16(lstep2[2], kOne); + lstep2[3] = _mm_madd_epi16(lstep2[3], kOne); + lstep2[4] = _mm_madd_epi16(lstep2[4], kOne); + lstep2[5] = _mm_madd_epi16(lstep2[5], kOne); + lstep2[6] = _mm_madd_epi16(lstep2[6], kOne); + lstep2[7] = _mm_madd_epi16(lstep2[7], kOne); + lstep2[8] = _mm_madd_epi16(lstep2[8], kOne); + lstep2[9] = _mm_madd_epi16(lstep2[9], kOne); + lstep2[10] = _mm_madd_epi16(lstep2[10], kOne); + lstep2[11] = _mm_madd_epi16(lstep2[11], kOne); + lstep2[12] = _mm_madd_epi16(lstep2[12], kOne); + lstep2[13] = _mm_madd_epi16(lstep2[13], kOne); + lstep2[14] = _mm_madd_epi16(lstep2[14], kOne); + lstep2[15] = _mm_madd_epi16(lstep2[15], kOne); + + lstep3[0] = _mm_add_epi32(lstep2[14], lstep2[0]); + lstep3[1] = _mm_add_epi32(lstep2[15], lstep2[1]); + lstep3[2] = _mm_add_epi32(lstep2[12], lstep2[2]); + lstep3[3] = _mm_add_epi32(lstep2[13], lstep2[3]); + lstep3[4] = _mm_add_epi32(lstep2[10], lstep2[4]); + lstep3[5] = _mm_add_epi32(lstep2[11], lstep2[5]); + lstep3[6] = _mm_add_epi32(lstep2[8], lstep2[6]); + lstep3[7] = _mm_add_epi32(lstep2[9], lstep2[7]); + lstep3[8] = _mm_sub_epi32(lstep2[6], lstep2[8]); + lstep3[9] = _mm_sub_epi32(lstep2[7], lstep2[9]); + lstep3[10] = _mm_sub_epi32(lstep2[4], lstep2[10]); + lstep3[11] = _mm_sub_epi32(lstep2[5], lstep2[11]); + lstep3[12] = _mm_sub_epi32(lstep2[2], lstep2[12]); + lstep3[13] = _mm_sub_epi32(lstep2[3], lstep2[13]); + lstep3[14] = _mm_sub_epi32(lstep2[0], lstep2[14]); + lstep3[15] = _mm_sub_epi32(lstep2[1], lstep2[15]); + } + { + const __m128i s3_10_0 = _mm_unpacklo_epi16(step2[13], step2[10]); + const __m128i s3_10_1 = _mm_unpackhi_epi16(step2[13], step2[10]); + const __m128i s3_11_0 = _mm_unpacklo_epi16(step2[12], step2[11]); + const __m128i s3_11_1 = _mm_unpackhi_epi16(step2[12], step2[11]); + const __m128i s3_10_2 = _mm_madd_epi16(s3_10_0, k__cospi_p16_m16); + const __m128i s3_10_3 = _mm_madd_epi16(s3_10_1, k__cospi_p16_m16); + const __m128i s3_11_2 = _mm_madd_epi16(s3_11_0, k__cospi_p16_m16); + const __m128i s3_11_3 = _mm_madd_epi16(s3_11_1, k__cospi_p16_m16); + const __m128i s3_12_2 = _mm_madd_epi16(s3_11_0, k__cospi_p16_p16); + const __m128i s3_12_3 = _mm_madd_epi16(s3_11_1, k__cospi_p16_p16); + const __m128i s3_13_2 = _mm_madd_epi16(s3_10_0, k__cospi_p16_p16); + const __m128i s3_13_3 = _mm_madd_epi16(s3_10_1, k__cospi_p16_p16); + // dct_const_round_shift + const __m128i s3_10_4 = _mm_add_epi32(s3_10_2, k__DCT_CONST_ROUNDING); + const __m128i s3_10_5 = _mm_add_epi32(s3_10_3, k__DCT_CONST_ROUNDING); + const __m128i s3_11_4 = _mm_add_epi32(s3_11_2, k__DCT_CONST_ROUNDING); + const __m128i s3_11_5 = _mm_add_epi32(s3_11_3, k__DCT_CONST_ROUNDING); + const __m128i s3_12_4 = _mm_add_epi32(s3_12_2, k__DCT_CONST_ROUNDING); + const __m128i s3_12_5 = _mm_add_epi32(s3_12_3, k__DCT_CONST_ROUNDING); + const __m128i s3_13_4 = _mm_add_epi32(s3_13_2, k__DCT_CONST_ROUNDING); + const __m128i s3_13_5 = _mm_add_epi32(s3_13_3, k__DCT_CONST_ROUNDING); + lstep3[20] = _mm_srai_epi32(s3_10_4, DCT_CONST_BITS); + lstep3[21] = _mm_srai_epi32(s3_10_5, DCT_CONST_BITS); + lstep3[22] = _mm_srai_epi32(s3_11_4, DCT_CONST_BITS); + lstep3[23] = _mm_srai_epi32(s3_11_5, DCT_CONST_BITS); + lstep3[24] = _mm_srai_epi32(s3_12_4, DCT_CONST_BITS); + lstep3[25] = _mm_srai_epi32(s3_12_5, DCT_CONST_BITS); + lstep3[26] = _mm_srai_epi32(s3_13_4, DCT_CONST_BITS); + lstep3[27] = _mm_srai_epi32(s3_13_5, DCT_CONST_BITS); + } + { + lstep2[40] = _mm_unpacklo_epi16(step2[20], kZero); + lstep2[41] = _mm_unpackhi_epi16(step2[20], kZero); + lstep2[42] = _mm_unpacklo_epi16(step2[21], kZero); + lstep2[43] = _mm_unpackhi_epi16(step2[21], kZero); + lstep2[44] = _mm_unpacklo_epi16(step2[22], kZero); + lstep2[45] = _mm_unpackhi_epi16(step2[22], kZero); + lstep2[46] = _mm_unpacklo_epi16(step2[23], kZero); + lstep2[47] = _mm_unpackhi_epi16(step2[23], kZero); + lstep2[48] = _mm_unpacklo_epi16(step2[24], kZero); + lstep2[49] = _mm_unpackhi_epi16(step2[24], kZero); + lstep2[50] = _mm_unpacklo_epi16(step2[25], kZero); + lstep2[51] = _mm_unpackhi_epi16(step2[25], kZero); + lstep2[52] = _mm_unpacklo_epi16(step2[26], kZero); + lstep2[53] = _mm_unpackhi_epi16(step2[26], kZero); + lstep2[54] = _mm_unpacklo_epi16(step2[27], kZero); + lstep2[55] = _mm_unpackhi_epi16(step2[27], kZero); + lstep2[40] = _mm_madd_epi16(lstep2[40], kOne); + lstep2[41] = _mm_madd_epi16(lstep2[41], kOne); + lstep2[42] = _mm_madd_epi16(lstep2[42], kOne); + lstep2[43] = _mm_madd_epi16(lstep2[43], kOne); + lstep2[44] = _mm_madd_epi16(lstep2[44], kOne); + lstep2[45] = _mm_madd_epi16(lstep2[45], kOne); + lstep2[46] = _mm_madd_epi16(lstep2[46], kOne); + lstep2[47] = _mm_madd_epi16(lstep2[47], kOne); + lstep2[48] = _mm_madd_epi16(lstep2[48], kOne); + lstep2[49] = _mm_madd_epi16(lstep2[49], kOne); + lstep2[50] = _mm_madd_epi16(lstep2[50], kOne); + lstep2[51] = _mm_madd_epi16(lstep2[51], kOne); + lstep2[52] = _mm_madd_epi16(lstep2[52], kOne); + lstep2[53] = _mm_madd_epi16(lstep2[53], kOne); + lstep2[54] = _mm_madd_epi16(lstep2[54], kOne); + lstep2[55] = _mm_madd_epi16(lstep2[55], kOne); + + lstep1[32] = _mm_unpacklo_epi16(step1[16], kZero); + lstep1[33] = _mm_unpackhi_epi16(step1[16], kZero); + lstep1[34] = _mm_unpacklo_epi16(step1[17], kZero); + lstep1[35] = _mm_unpackhi_epi16(step1[17], kZero); + lstep1[36] = _mm_unpacklo_epi16(step1[18], kZero); + lstep1[37] = _mm_unpackhi_epi16(step1[18], kZero); + lstep1[38] = _mm_unpacklo_epi16(step1[19], kZero); + lstep1[39] = _mm_unpackhi_epi16(step1[19], kZero); + lstep1[56] = _mm_unpacklo_epi16(step1[28], kZero); + lstep1[57] = _mm_unpackhi_epi16(step1[28], kZero); + lstep1[58] = _mm_unpacklo_epi16(step1[29], kZero); + lstep1[59] = _mm_unpackhi_epi16(step1[29], kZero); + lstep1[60] = _mm_unpacklo_epi16(step1[30], kZero); + lstep1[61] = _mm_unpackhi_epi16(step1[30], kZero); + lstep1[62] = _mm_unpacklo_epi16(step1[31], kZero); + lstep1[63] = _mm_unpackhi_epi16(step1[31], kZero); + lstep1[32] = _mm_madd_epi16(lstep1[32], kOne); + lstep1[33] = _mm_madd_epi16(lstep1[33], kOne); + lstep1[34] = _mm_madd_epi16(lstep1[34], kOne); + lstep1[35] = _mm_madd_epi16(lstep1[35], kOne); + lstep1[36] = _mm_madd_epi16(lstep1[36], kOne); + lstep1[37] = _mm_madd_epi16(lstep1[37], kOne); + lstep1[38] = _mm_madd_epi16(lstep1[38], kOne); + lstep1[39] = _mm_madd_epi16(lstep1[39], kOne); + lstep1[56] = _mm_madd_epi16(lstep1[56], kOne); + lstep1[57] = _mm_madd_epi16(lstep1[57], kOne); + lstep1[58] = _mm_madd_epi16(lstep1[58], kOne); + lstep1[59] = _mm_madd_epi16(lstep1[59], kOne); + lstep1[60] = _mm_madd_epi16(lstep1[60], kOne); + lstep1[61] = _mm_madd_epi16(lstep1[61], kOne); + lstep1[62] = _mm_madd_epi16(lstep1[62], kOne); + lstep1[63] = _mm_madd_epi16(lstep1[63], kOne); + + lstep3[32] = _mm_add_epi32(lstep2[46], lstep1[32]); + lstep3[33] = _mm_add_epi32(lstep2[47], lstep1[33]); + + lstep3[34] = _mm_add_epi32(lstep2[44], lstep1[34]); + lstep3[35] = _mm_add_epi32(lstep2[45], lstep1[35]); + lstep3[36] = _mm_add_epi32(lstep2[42], lstep1[36]); + lstep3[37] = _mm_add_epi32(lstep2[43], lstep1[37]); + lstep3[38] = _mm_add_epi32(lstep2[40], lstep1[38]); + lstep3[39] = _mm_add_epi32(lstep2[41], lstep1[39]); + lstep3[40] = _mm_sub_epi32(lstep1[38], lstep2[40]); + lstep3[41] = _mm_sub_epi32(lstep1[39], lstep2[41]); + lstep3[42] = _mm_sub_epi32(lstep1[36], lstep2[42]); + lstep3[43] = _mm_sub_epi32(lstep1[37], lstep2[43]); + lstep3[44] = _mm_sub_epi32(lstep1[34], lstep2[44]); + lstep3[45] = _mm_sub_epi32(lstep1[35], lstep2[45]); + lstep3[46] = _mm_sub_epi32(lstep1[32], lstep2[46]); + lstep3[47] = _mm_sub_epi32(lstep1[33], lstep2[47]); + lstep3[48] = _mm_sub_epi32(lstep1[62], lstep2[48]); + lstep3[49] = _mm_sub_epi32(lstep1[63], lstep2[49]); + lstep3[50] = _mm_sub_epi32(lstep1[60], lstep2[50]); + lstep3[51] = _mm_sub_epi32(lstep1[61], lstep2[51]); + lstep3[52] = _mm_sub_epi32(lstep1[58], lstep2[52]); + lstep3[53] = _mm_sub_epi32(lstep1[59], lstep2[53]); + lstep3[54] = _mm_sub_epi32(lstep1[56], lstep2[54]); + lstep3[55] = _mm_sub_epi32(lstep1[57], lstep2[55]); + lstep3[56] = _mm_add_epi32(lstep2[54], lstep1[56]); + lstep3[57] = _mm_add_epi32(lstep2[55], lstep1[57]); + lstep3[58] = _mm_add_epi32(lstep2[52], lstep1[58]); + lstep3[59] = _mm_add_epi32(lstep2[53], lstep1[59]); + lstep3[60] = _mm_add_epi32(lstep2[50], lstep1[60]); + lstep3[61] = _mm_add_epi32(lstep2[51], lstep1[61]); + lstep3[62] = _mm_add_epi32(lstep2[48], lstep1[62]); + lstep3[63] = _mm_add_epi32(lstep2[49], lstep1[63]); + } + + // stage 4 + { + // expanding to 32-bit length priori to addition operations + lstep2[16] = _mm_unpacklo_epi16(step2[8], kZero); + lstep2[17] = _mm_unpackhi_epi16(step2[8], kZero); + lstep2[18] = _mm_unpacklo_epi16(step2[9], kZero); + lstep2[19] = _mm_unpackhi_epi16(step2[9], kZero); + lstep2[28] = _mm_unpacklo_epi16(step2[14], kZero); + lstep2[29] = _mm_unpackhi_epi16(step2[14], kZero); + lstep2[30] = _mm_unpacklo_epi16(step2[15], kZero); + lstep2[31] = _mm_unpackhi_epi16(step2[15], kZero); + lstep2[16] = _mm_madd_epi16(lstep2[16], kOne); + lstep2[17] = _mm_madd_epi16(lstep2[17], kOne); + lstep2[18] = _mm_madd_epi16(lstep2[18], kOne); + lstep2[19] = _mm_madd_epi16(lstep2[19], kOne); + lstep2[28] = _mm_madd_epi16(lstep2[28], kOne); + lstep2[29] = _mm_madd_epi16(lstep2[29], kOne); + lstep2[30] = _mm_madd_epi16(lstep2[30], kOne); + lstep2[31] = _mm_madd_epi16(lstep2[31], kOne); + + lstep1[0] = _mm_add_epi32(lstep3[6], lstep3[0]); + lstep1[1] = _mm_add_epi32(lstep3[7], lstep3[1]); + lstep1[2] = _mm_add_epi32(lstep3[4], lstep3[2]); + lstep1[3] = _mm_add_epi32(lstep3[5], lstep3[3]); + lstep1[4] = _mm_sub_epi32(lstep3[2], lstep3[4]); + lstep1[5] = _mm_sub_epi32(lstep3[3], lstep3[5]); + lstep1[6] = _mm_sub_epi32(lstep3[0], lstep3[6]); + lstep1[7] = _mm_sub_epi32(lstep3[1], lstep3[7]); + lstep1[16] = _mm_add_epi32(lstep3[22], lstep2[16]); + lstep1[17] = _mm_add_epi32(lstep3[23], lstep2[17]); + lstep1[18] = _mm_add_epi32(lstep3[20], lstep2[18]); + lstep1[19] = _mm_add_epi32(lstep3[21], lstep2[19]); + lstep1[20] = _mm_sub_epi32(lstep2[18], lstep3[20]); + lstep1[21] = _mm_sub_epi32(lstep2[19], lstep3[21]); + lstep1[22] = _mm_sub_epi32(lstep2[16], lstep3[22]); + lstep1[23] = _mm_sub_epi32(lstep2[17], lstep3[23]); + lstep1[24] = _mm_sub_epi32(lstep2[30], lstep3[24]); + lstep1[25] = _mm_sub_epi32(lstep2[31], lstep3[25]); + lstep1[26] = _mm_sub_epi32(lstep2[28], lstep3[26]); + lstep1[27] = _mm_sub_epi32(lstep2[29], lstep3[27]); + lstep1[28] = _mm_add_epi32(lstep3[26], lstep2[28]); + lstep1[29] = _mm_add_epi32(lstep3[27], lstep2[29]); + lstep1[30] = _mm_add_epi32(lstep3[24], lstep2[30]); + lstep1[31] = _mm_add_epi32(lstep3[25], lstep2[31]); + } + { + // to be continued... + // + const __m128i k32_p16_p16 = pair_set_epi32(cospi_16_64, cospi_16_64); + const __m128i k32_p16_m16 = pair_set_epi32(cospi_16_64, -cospi_16_64); + + u[0] = _mm_unpacklo_epi32(lstep3[12], lstep3[10]); + u[1] = _mm_unpackhi_epi32(lstep3[12], lstep3[10]); + u[2] = _mm_unpacklo_epi32(lstep3[13], lstep3[11]); + u[3] = _mm_unpackhi_epi32(lstep3[13], lstep3[11]); + + // TODO(jingning): manually inline k_madd_epi32_ to further hide + // instruction latency. + v[0] = k_madd_epi32(u[0], k32_p16_m16); + v[1] = k_madd_epi32(u[1], k32_p16_m16); + v[2] = k_madd_epi32(u[2], k32_p16_m16); + v[3] = k_madd_epi32(u[3], k32_p16_m16); + v[4] = k_madd_epi32(u[0], k32_p16_p16); + v[5] = k_madd_epi32(u[1], k32_p16_p16); + v[6] = k_madd_epi32(u[2], k32_p16_p16); + v[7] = k_madd_epi32(u[3], k32_p16_p16); +#if DCT_HIGH_BIT_DEPTH + overflow = k_check_epi32_overflow_8(&v[0], &v[1], &v[2], &v[3], &v[4], + &v[5], &v[6], &v[7], &kZero); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + u[0] = k_packs_epi64(v[0], v[1]); + u[1] = k_packs_epi64(v[2], v[3]); + u[2] = k_packs_epi64(v[4], v[5]); + u[3] = k_packs_epi64(v[6], v[7]); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + + lstep1[10] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + lstep1[11] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + lstep1[12] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + lstep1[13] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + } + { + const __m128i k32_m08_p24 = pair_set_epi32(-cospi_8_64, cospi_24_64); + const __m128i k32_m24_m08 = pair_set_epi32(-cospi_24_64, -cospi_8_64); + const __m128i k32_p24_p08 = pair_set_epi32(cospi_24_64, cospi_8_64); + + u[0] = _mm_unpacklo_epi32(lstep3[36], lstep3[58]); + u[1] = _mm_unpackhi_epi32(lstep3[36], lstep3[58]); + u[2] = _mm_unpacklo_epi32(lstep3[37], lstep3[59]); + u[3] = _mm_unpackhi_epi32(lstep3[37], lstep3[59]); + u[4] = _mm_unpacklo_epi32(lstep3[38], lstep3[56]); + u[5] = _mm_unpackhi_epi32(lstep3[38], lstep3[56]); + u[6] = _mm_unpacklo_epi32(lstep3[39], lstep3[57]); + u[7] = _mm_unpackhi_epi32(lstep3[39], lstep3[57]); + u[8] = _mm_unpacklo_epi32(lstep3[40], lstep3[54]); + u[9] = _mm_unpackhi_epi32(lstep3[40], lstep3[54]); + u[10] = _mm_unpacklo_epi32(lstep3[41], lstep3[55]); + u[11] = _mm_unpackhi_epi32(lstep3[41], lstep3[55]); + u[12] = _mm_unpacklo_epi32(lstep3[42], lstep3[52]); + u[13] = _mm_unpackhi_epi32(lstep3[42], lstep3[52]); + u[14] = _mm_unpacklo_epi32(lstep3[43], lstep3[53]); + u[15] = _mm_unpackhi_epi32(lstep3[43], lstep3[53]); + + v[0] = k_madd_epi32(u[0], k32_m08_p24); + v[1] = k_madd_epi32(u[1], k32_m08_p24); + v[2] = k_madd_epi32(u[2], k32_m08_p24); + v[3] = k_madd_epi32(u[3], k32_m08_p24); + v[4] = k_madd_epi32(u[4], k32_m08_p24); + v[5] = k_madd_epi32(u[5], k32_m08_p24); + v[6] = k_madd_epi32(u[6], k32_m08_p24); + v[7] = k_madd_epi32(u[7], k32_m08_p24); + v[8] = k_madd_epi32(u[8], k32_m24_m08); + v[9] = k_madd_epi32(u[9], k32_m24_m08); + v[10] = k_madd_epi32(u[10], k32_m24_m08); + v[11] = k_madd_epi32(u[11], k32_m24_m08); + v[12] = k_madd_epi32(u[12], k32_m24_m08); + v[13] = k_madd_epi32(u[13], k32_m24_m08); + v[14] = k_madd_epi32(u[14], k32_m24_m08); + v[15] = k_madd_epi32(u[15], k32_m24_m08); + v[16] = k_madd_epi32(u[12], k32_m08_p24); + v[17] = k_madd_epi32(u[13], k32_m08_p24); + v[18] = k_madd_epi32(u[14], k32_m08_p24); + v[19] = k_madd_epi32(u[15], k32_m08_p24); + v[20] = k_madd_epi32(u[8], k32_m08_p24); + v[21] = k_madd_epi32(u[9], k32_m08_p24); + v[22] = k_madd_epi32(u[10], k32_m08_p24); + v[23] = k_madd_epi32(u[11], k32_m08_p24); + v[24] = k_madd_epi32(u[4], k32_p24_p08); + v[25] = k_madd_epi32(u[5], k32_p24_p08); + v[26] = k_madd_epi32(u[6], k32_p24_p08); + v[27] = k_madd_epi32(u[7], k32_p24_p08); + v[28] = k_madd_epi32(u[0], k32_p24_p08); + v[29] = k_madd_epi32(u[1], k32_p24_p08); + v[30] = k_madd_epi32(u[2], k32_p24_p08); + v[31] = k_madd_epi32(u[3], k32_p24_p08); + +#if DCT_HIGH_BIT_DEPTH + overflow = k_check_epi32_overflow_32( + &v[0], &v[1], &v[2], &v[3], &v[4], &v[5], &v[6], &v[7], &v[8], + &v[9], &v[10], &v[11], &v[12], &v[13], &v[14], &v[15], &v[16], + &v[17], &v[18], &v[19], &v[20], &v[21], &v[22], &v[23], &v[24], + &v[25], &v[26], &v[27], &v[28], &v[29], &v[30], &v[31], &kZero); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + u[0] = k_packs_epi64(v[0], v[1]); + u[1] = k_packs_epi64(v[2], v[3]); + u[2] = k_packs_epi64(v[4], v[5]); + u[3] = k_packs_epi64(v[6], v[7]); + u[4] = k_packs_epi64(v[8], v[9]); + u[5] = k_packs_epi64(v[10], v[11]); + u[6] = k_packs_epi64(v[12], v[13]); + u[7] = k_packs_epi64(v[14], v[15]); + u[8] = k_packs_epi64(v[16], v[17]); + u[9] = k_packs_epi64(v[18], v[19]); + u[10] = k_packs_epi64(v[20], v[21]); + u[11] = k_packs_epi64(v[22], v[23]); + u[12] = k_packs_epi64(v[24], v[25]); + u[13] = k_packs_epi64(v[26], v[27]); + u[14] = k_packs_epi64(v[28], v[29]); + u[15] = k_packs_epi64(v[30], v[31]); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + v[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + lstep1[36] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + lstep1[37] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + lstep1[38] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + lstep1[39] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + lstep1[40] = _mm_srai_epi32(v[4], DCT_CONST_BITS); + lstep1[41] = _mm_srai_epi32(v[5], DCT_CONST_BITS); + lstep1[42] = _mm_srai_epi32(v[6], DCT_CONST_BITS); + lstep1[43] = _mm_srai_epi32(v[7], DCT_CONST_BITS); + lstep1[52] = _mm_srai_epi32(v[8], DCT_CONST_BITS); + lstep1[53] = _mm_srai_epi32(v[9], DCT_CONST_BITS); + lstep1[54] = _mm_srai_epi32(v[10], DCT_CONST_BITS); + lstep1[55] = _mm_srai_epi32(v[11], DCT_CONST_BITS); + lstep1[56] = _mm_srai_epi32(v[12], DCT_CONST_BITS); + lstep1[57] = _mm_srai_epi32(v[13], DCT_CONST_BITS); + lstep1[58] = _mm_srai_epi32(v[14], DCT_CONST_BITS); + lstep1[59] = _mm_srai_epi32(v[15], DCT_CONST_BITS); + } + // stage 5 + { + lstep2[8] = _mm_add_epi32(lstep1[10], lstep3[8]); + lstep2[9] = _mm_add_epi32(lstep1[11], lstep3[9]); + lstep2[10] = _mm_sub_epi32(lstep3[8], lstep1[10]); + lstep2[11] = _mm_sub_epi32(lstep3[9], lstep1[11]); + lstep2[12] = _mm_sub_epi32(lstep3[14], lstep1[12]); + lstep2[13] = _mm_sub_epi32(lstep3[15], lstep1[13]); + lstep2[14] = _mm_add_epi32(lstep1[12], lstep3[14]); + lstep2[15] = _mm_add_epi32(lstep1[13], lstep3[15]); + } + { + const __m128i k32_p16_p16 = pair_set_epi32(cospi_16_64, cospi_16_64); + const __m128i k32_p16_m16 = pair_set_epi32(cospi_16_64, -cospi_16_64); + const __m128i k32_p24_p08 = pair_set_epi32(cospi_24_64, cospi_8_64); + const __m128i k32_m08_p24 = pair_set_epi32(-cospi_8_64, cospi_24_64); + + u[0] = _mm_unpacklo_epi32(lstep1[0], lstep1[2]); + u[1] = _mm_unpackhi_epi32(lstep1[0], lstep1[2]); + u[2] = _mm_unpacklo_epi32(lstep1[1], lstep1[3]); + u[3] = _mm_unpackhi_epi32(lstep1[1], lstep1[3]); + u[4] = _mm_unpacklo_epi32(lstep1[4], lstep1[6]); + u[5] = _mm_unpackhi_epi32(lstep1[4], lstep1[6]); + u[6] = _mm_unpacklo_epi32(lstep1[5], lstep1[7]); + u[7] = _mm_unpackhi_epi32(lstep1[5], lstep1[7]); + + // TODO(jingning): manually inline k_madd_epi32_ to further hide + // instruction latency. + v[0] = k_madd_epi32(u[0], k32_p16_p16); + v[1] = k_madd_epi32(u[1], k32_p16_p16); + v[2] = k_madd_epi32(u[2], k32_p16_p16); + v[3] = k_madd_epi32(u[3], k32_p16_p16); + v[4] = k_madd_epi32(u[0], k32_p16_m16); + v[5] = k_madd_epi32(u[1], k32_p16_m16); + v[6] = k_madd_epi32(u[2], k32_p16_m16); + v[7] = k_madd_epi32(u[3], k32_p16_m16); + v[8] = k_madd_epi32(u[4], k32_p24_p08); + v[9] = k_madd_epi32(u[5], k32_p24_p08); + v[10] = k_madd_epi32(u[6], k32_p24_p08); + v[11] = k_madd_epi32(u[7], k32_p24_p08); + v[12] = k_madd_epi32(u[4], k32_m08_p24); + v[13] = k_madd_epi32(u[5], k32_m08_p24); + v[14] = k_madd_epi32(u[6], k32_m08_p24); + v[15] = k_madd_epi32(u[7], k32_m08_p24); + +#if DCT_HIGH_BIT_DEPTH + overflow = k_check_epi32_overflow_16( + &v[0], &v[1], &v[2], &v[3], &v[4], &v[5], &v[6], &v[7], &v[8], + &v[9], &v[10], &v[11], &v[12], &v[13], &v[14], &v[15], &kZero); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + u[0] = k_packs_epi64(v[0], v[1]); + u[1] = k_packs_epi64(v[2], v[3]); + u[2] = k_packs_epi64(v[4], v[5]); + u[3] = k_packs_epi64(v[6], v[7]); + u[4] = k_packs_epi64(v[8], v[9]); + u[5] = k_packs_epi64(v[10], v[11]); + u[6] = k_packs_epi64(v[12], v[13]); + u[7] = k_packs_epi64(v[14], v[15]); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS); + + sign[0] = _mm_cmplt_epi32(u[0], kZero); + sign[1] = _mm_cmplt_epi32(u[1], kZero); + sign[2] = _mm_cmplt_epi32(u[2], kZero); + sign[3] = _mm_cmplt_epi32(u[3], kZero); + sign[4] = _mm_cmplt_epi32(u[4], kZero); + sign[5] = _mm_cmplt_epi32(u[5], kZero); + sign[6] = _mm_cmplt_epi32(u[6], kZero); + sign[7] = _mm_cmplt_epi32(u[7], kZero); + + u[0] = _mm_sub_epi32(u[0], sign[0]); + u[1] = _mm_sub_epi32(u[1], sign[1]); + u[2] = _mm_sub_epi32(u[2], sign[2]); + u[3] = _mm_sub_epi32(u[3], sign[3]); + u[4] = _mm_sub_epi32(u[4], sign[4]); + u[5] = _mm_sub_epi32(u[5], sign[5]); + u[6] = _mm_sub_epi32(u[6], sign[6]); + u[7] = _mm_sub_epi32(u[7], sign[7]); + + u[0] = _mm_add_epi32(u[0], K32One); + u[1] = _mm_add_epi32(u[1], K32One); + u[2] = _mm_add_epi32(u[2], K32One); + u[3] = _mm_add_epi32(u[3], K32One); + u[4] = _mm_add_epi32(u[4], K32One); + u[5] = _mm_add_epi32(u[5], K32One); + u[6] = _mm_add_epi32(u[6], K32One); + u[7] = _mm_add_epi32(u[7], K32One); + + u[0] = _mm_srai_epi32(u[0], 2); + u[1] = _mm_srai_epi32(u[1], 2); + u[2] = _mm_srai_epi32(u[2], 2); + u[3] = _mm_srai_epi32(u[3], 2); + u[4] = _mm_srai_epi32(u[4], 2); + u[5] = _mm_srai_epi32(u[5], 2); + u[6] = _mm_srai_epi32(u[6], 2); + u[7] = _mm_srai_epi32(u[7], 2); + + // Combine + out[0] = _mm_packs_epi32(u[0], u[1]); + out[16] = _mm_packs_epi32(u[2], u[3]); + out[8] = _mm_packs_epi32(u[4], u[5]); + out[24] = _mm_packs_epi32(u[6], u[7]); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x4(&out[0], &out[16], &out[8], &out[24]); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + const __m128i k32_m08_p24 = pair_set_epi32(-cospi_8_64, cospi_24_64); + const __m128i k32_m24_m08 = pair_set_epi32(-cospi_24_64, -cospi_8_64); + const __m128i k32_p24_p08 = pair_set_epi32(cospi_24_64, cospi_8_64); + + u[0] = _mm_unpacklo_epi32(lstep1[18], lstep1[28]); + u[1] = _mm_unpackhi_epi32(lstep1[18], lstep1[28]); + u[2] = _mm_unpacklo_epi32(lstep1[19], lstep1[29]); + u[3] = _mm_unpackhi_epi32(lstep1[19], lstep1[29]); + u[4] = _mm_unpacklo_epi32(lstep1[20], lstep1[26]); + u[5] = _mm_unpackhi_epi32(lstep1[20], lstep1[26]); + u[6] = _mm_unpacklo_epi32(lstep1[21], lstep1[27]); + u[7] = _mm_unpackhi_epi32(lstep1[21], lstep1[27]); + + v[0] = k_madd_epi32(u[0], k32_m08_p24); + v[1] = k_madd_epi32(u[1], k32_m08_p24); + v[2] = k_madd_epi32(u[2], k32_m08_p24); + v[3] = k_madd_epi32(u[3], k32_m08_p24); + v[4] = k_madd_epi32(u[4], k32_m24_m08); + v[5] = k_madd_epi32(u[5], k32_m24_m08); + v[6] = k_madd_epi32(u[6], k32_m24_m08); + v[7] = k_madd_epi32(u[7], k32_m24_m08); + v[8] = k_madd_epi32(u[4], k32_m08_p24); + v[9] = k_madd_epi32(u[5], k32_m08_p24); + v[10] = k_madd_epi32(u[6], k32_m08_p24); + v[11] = k_madd_epi32(u[7], k32_m08_p24); + v[12] = k_madd_epi32(u[0], k32_p24_p08); + v[13] = k_madd_epi32(u[1], k32_p24_p08); + v[14] = k_madd_epi32(u[2], k32_p24_p08); + v[15] = k_madd_epi32(u[3], k32_p24_p08); + +#if DCT_HIGH_BIT_DEPTH + overflow = k_check_epi32_overflow_16( + &v[0], &v[1], &v[2], &v[3], &v[4], &v[5], &v[6], &v[7], &v[8], + &v[9], &v[10], &v[11], &v[12], &v[13], &v[14], &v[15], &kZero); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + u[0] = k_packs_epi64(v[0], v[1]); + u[1] = k_packs_epi64(v[2], v[3]); + u[2] = k_packs_epi64(v[4], v[5]); + u[3] = k_packs_epi64(v[6], v[7]); + u[4] = k_packs_epi64(v[8], v[9]); + u[5] = k_packs_epi64(v[10], v[11]); + u[6] = k_packs_epi64(v[12], v[13]); + u[7] = k_packs_epi64(v[14], v[15]); + + u[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + + lstep2[18] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + lstep2[19] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + lstep2[20] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + lstep2[21] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + lstep2[26] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + lstep2[27] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + lstep2[28] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + lstep2[29] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + } + { + lstep2[32] = _mm_add_epi32(lstep1[38], lstep3[32]); + lstep2[33] = _mm_add_epi32(lstep1[39], lstep3[33]); + lstep2[34] = _mm_add_epi32(lstep1[36], lstep3[34]); + lstep2[35] = _mm_add_epi32(lstep1[37], lstep3[35]); + lstep2[36] = _mm_sub_epi32(lstep3[34], lstep1[36]); + lstep2[37] = _mm_sub_epi32(lstep3[35], lstep1[37]); + lstep2[38] = _mm_sub_epi32(lstep3[32], lstep1[38]); + lstep2[39] = _mm_sub_epi32(lstep3[33], lstep1[39]); + lstep2[40] = _mm_sub_epi32(lstep3[46], lstep1[40]); + lstep2[41] = _mm_sub_epi32(lstep3[47], lstep1[41]); + lstep2[42] = _mm_sub_epi32(lstep3[44], lstep1[42]); + lstep2[43] = _mm_sub_epi32(lstep3[45], lstep1[43]); + lstep2[44] = _mm_add_epi32(lstep1[42], lstep3[44]); + lstep2[45] = _mm_add_epi32(lstep1[43], lstep3[45]); + lstep2[46] = _mm_add_epi32(lstep1[40], lstep3[46]); + lstep2[47] = _mm_add_epi32(lstep1[41], lstep3[47]); + lstep2[48] = _mm_add_epi32(lstep1[54], lstep3[48]); + lstep2[49] = _mm_add_epi32(lstep1[55], lstep3[49]); + lstep2[50] = _mm_add_epi32(lstep1[52], lstep3[50]); + lstep2[51] = _mm_add_epi32(lstep1[53], lstep3[51]); + lstep2[52] = _mm_sub_epi32(lstep3[50], lstep1[52]); + lstep2[53] = _mm_sub_epi32(lstep3[51], lstep1[53]); + lstep2[54] = _mm_sub_epi32(lstep3[48], lstep1[54]); + lstep2[55] = _mm_sub_epi32(lstep3[49], lstep1[55]); + lstep2[56] = _mm_sub_epi32(lstep3[62], lstep1[56]); + lstep2[57] = _mm_sub_epi32(lstep3[63], lstep1[57]); + lstep2[58] = _mm_sub_epi32(lstep3[60], lstep1[58]); + lstep2[59] = _mm_sub_epi32(lstep3[61], lstep1[59]); + lstep2[60] = _mm_add_epi32(lstep1[58], lstep3[60]); + lstep2[61] = _mm_add_epi32(lstep1[59], lstep3[61]); + lstep2[62] = _mm_add_epi32(lstep1[56], lstep3[62]); + lstep2[63] = _mm_add_epi32(lstep1[57], lstep3[63]); + } + // stage 6 + { + const __m128i k32_p28_p04 = pair_set_epi32(cospi_28_64, cospi_4_64); + const __m128i k32_p12_p20 = pair_set_epi32(cospi_12_64, cospi_20_64); + const __m128i k32_m20_p12 = pair_set_epi32(-cospi_20_64, cospi_12_64); + const __m128i k32_m04_p28 = pair_set_epi32(-cospi_4_64, cospi_28_64); + + u[0] = _mm_unpacklo_epi32(lstep2[8], lstep2[14]); + u[1] = _mm_unpackhi_epi32(lstep2[8], lstep2[14]); + u[2] = _mm_unpacklo_epi32(lstep2[9], lstep2[15]); + u[3] = _mm_unpackhi_epi32(lstep2[9], lstep2[15]); + u[4] = _mm_unpacklo_epi32(lstep2[10], lstep2[12]); + u[5] = _mm_unpackhi_epi32(lstep2[10], lstep2[12]); + u[6] = _mm_unpacklo_epi32(lstep2[11], lstep2[13]); + u[7] = _mm_unpackhi_epi32(lstep2[11], lstep2[13]); + u[8] = _mm_unpacklo_epi32(lstep2[10], lstep2[12]); + u[9] = _mm_unpackhi_epi32(lstep2[10], lstep2[12]); + u[10] = _mm_unpacklo_epi32(lstep2[11], lstep2[13]); + u[11] = _mm_unpackhi_epi32(lstep2[11], lstep2[13]); + u[12] = _mm_unpacklo_epi32(lstep2[8], lstep2[14]); + u[13] = _mm_unpackhi_epi32(lstep2[8], lstep2[14]); + u[14] = _mm_unpacklo_epi32(lstep2[9], lstep2[15]); + u[15] = _mm_unpackhi_epi32(lstep2[9], lstep2[15]); + + v[0] = k_madd_epi32(u[0], k32_p28_p04); + v[1] = k_madd_epi32(u[1], k32_p28_p04); + v[2] = k_madd_epi32(u[2], k32_p28_p04); + v[3] = k_madd_epi32(u[3], k32_p28_p04); + v[4] = k_madd_epi32(u[4], k32_p12_p20); + v[5] = k_madd_epi32(u[5], k32_p12_p20); + v[6] = k_madd_epi32(u[6], k32_p12_p20); + v[7] = k_madd_epi32(u[7], k32_p12_p20); + v[8] = k_madd_epi32(u[8], k32_m20_p12); + v[9] = k_madd_epi32(u[9], k32_m20_p12); + v[10] = k_madd_epi32(u[10], k32_m20_p12); + v[11] = k_madd_epi32(u[11], k32_m20_p12); + v[12] = k_madd_epi32(u[12], k32_m04_p28); + v[13] = k_madd_epi32(u[13], k32_m04_p28); + v[14] = k_madd_epi32(u[14], k32_m04_p28); + v[15] = k_madd_epi32(u[15], k32_m04_p28); + +#if DCT_HIGH_BIT_DEPTH + overflow = k_check_epi32_overflow_16( + &v[0], &v[1], &v[2], &v[3], &v[4], &v[5], &v[6], &v[7], &v[8], + &v[9], &v[10], &v[11], &v[12], &v[13], &v[14], &v[15], &kZero); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + u[0] = k_packs_epi64(v[0], v[1]); + u[1] = k_packs_epi64(v[2], v[3]); + u[2] = k_packs_epi64(v[4], v[5]); + u[3] = k_packs_epi64(v[6], v[7]); + u[4] = k_packs_epi64(v[8], v[9]); + u[5] = k_packs_epi64(v[10], v[11]); + u[6] = k_packs_epi64(v[12], v[13]); + u[7] = k_packs_epi64(v[14], v[15]); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS); + + sign[0] = _mm_cmplt_epi32(u[0], kZero); + sign[1] = _mm_cmplt_epi32(u[1], kZero); + sign[2] = _mm_cmplt_epi32(u[2], kZero); + sign[3] = _mm_cmplt_epi32(u[3], kZero); + sign[4] = _mm_cmplt_epi32(u[4], kZero); + sign[5] = _mm_cmplt_epi32(u[5], kZero); + sign[6] = _mm_cmplt_epi32(u[6], kZero); + sign[7] = _mm_cmplt_epi32(u[7], kZero); + + u[0] = _mm_sub_epi32(u[0], sign[0]); + u[1] = _mm_sub_epi32(u[1], sign[1]); + u[2] = _mm_sub_epi32(u[2], sign[2]); + u[3] = _mm_sub_epi32(u[3], sign[3]); + u[4] = _mm_sub_epi32(u[4], sign[4]); + u[5] = _mm_sub_epi32(u[5], sign[5]); + u[6] = _mm_sub_epi32(u[6], sign[6]); + u[7] = _mm_sub_epi32(u[7], sign[7]); + + u[0] = _mm_add_epi32(u[0], K32One); + u[1] = _mm_add_epi32(u[1], K32One); + u[2] = _mm_add_epi32(u[2], K32One); + u[3] = _mm_add_epi32(u[3], K32One); + u[4] = _mm_add_epi32(u[4], K32One); + u[5] = _mm_add_epi32(u[5], K32One); + u[6] = _mm_add_epi32(u[6], K32One); + u[7] = _mm_add_epi32(u[7], K32One); + + u[0] = _mm_srai_epi32(u[0], 2); + u[1] = _mm_srai_epi32(u[1], 2); + u[2] = _mm_srai_epi32(u[2], 2); + u[3] = _mm_srai_epi32(u[3], 2); + u[4] = _mm_srai_epi32(u[4], 2); + u[5] = _mm_srai_epi32(u[5], 2); + u[6] = _mm_srai_epi32(u[6], 2); + u[7] = _mm_srai_epi32(u[7], 2); + + out[4] = _mm_packs_epi32(u[0], u[1]); + out[20] = _mm_packs_epi32(u[2], u[3]); + out[12] = _mm_packs_epi32(u[4], u[5]); + out[28] = _mm_packs_epi32(u[6], u[7]); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x4(&out[4], &out[20], &out[12], &out[28]); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + lstep3[16] = _mm_add_epi32(lstep2[18], lstep1[16]); + lstep3[17] = _mm_add_epi32(lstep2[19], lstep1[17]); + lstep3[18] = _mm_sub_epi32(lstep1[16], lstep2[18]); + lstep3[19] = _mm_sub_epi32(lstep1[17], lstep2[19]); + lstep3[20] = _mm_sub_epi32(lstep1[22], lstep2[20]); + lstep3[21] = _mm_sub_epi32(lstep1[23], lstep2[21]); + lstep3[22] = _mm_add_epi32(lstep2[20], lstep1[22]); + lstep3[23] = _mm_add_epi32(lstep2[21], lstep1[23]); + lstep3[24] = _mm_add_epi32(lstep2[26], lstep1[24]); + lstep3[25] = _mm_add_epi32(lstep2[27], lstep1[25]); + lstep3[26] = _mm_sub_epi32(lstep1[24], lstep2[26]); + lstep3[27] = _mm_sub_epi32(lstep1[25], lstep2[27]); + lstep3[28] = _mm_sub_epi32(lstep1[30], lstep2[28]); + lstep3[29] = _mm_sub_epi32(lstep1[31], lstep2[29]); + lstep3[30] = _mm_add_epi32(lstep2[28], lstep1[30]); + lstep3[31] = _mm_add_epi32(lstep2[29], lstep1[31]); + } + { + const __m128i k32_m04_p28 = pair_set_epi32(-cospi_4_64, cospi_28_64); + const __m128i k32_m28_m04 = pair_set_epi32(-cospi_28_64, -cospi_4_64); + const __m128i k32_m20_p12 = pair_set_epi32(-cospi_20_64, cospi_12_64); + const __m128i k32_m12_m20 = + pair_set_epi32(-cospi_12_64, -cospi_20_64); + const __m128i k32_p12_p20 = pair_set_epi32(cospi_12_64, cospi_20_64); + const __m128i k32_p28_p04 = pair_set_epi32(cospi_28_64, cospi_4_64); + + u[0] = _mm_unpacklo_epi32(lstep2[34], lstep2[60]); + u[1] = _mm_unpackhi_epi32(lstep2[34], lstep2[60]); + u[2] = _mm_unpacklo_epi32(lstep2[35], lstep2[61]); + u[3] = _mm_unpackhi_epi32(lstep2[35], lstep2[61]); + u[4] = _mm_unpacklo_epi32(lstep2[36], lstep2[58]); + u[5] = _mm_unpackhi_epi32(lstep2[36], lstep2[58]); + u[6] = _mm_unpacklo_epi32(lstep2[37], lstep2[59]); + u[7] = _mm_unpackhi_epi32(lstep2[37], lstep2[59]); + u[8] = _mm_unpacklo_epi32(lstep2[42], lstep2[52]); + u[9] = _mm_unpackhi_epi32(lstep2[42], lstep2[52]); + u[10] = _mm_unpacklo_epi32(lstep2[43], lstep2[53]); + u[11] = _mm_unpackhi_epi32(lstep2[43], lstep2[53]); + u[12] = _mm_unpacklo_epi32(lstep2[44], lstep2[50]); + u[13] = _mm_unpackhi_epi32(lstep2[44], lstep2[50]); + u[14] = _mm_unpacklo_epi32(lstep2[45], lstep2[51]); + u[15] = _mm_unpackhi_epi32(lstep2[45], lstep2[51]); + + v[0] = k_madd_epi32(u[0], k32_m04_p28); + v[1] = k_madd_epi32(u[1], k32_m04_p28); + v[2] = k_madd_epi32(u[2], k32_m04_p28); + v[3] = k_madd_epi32(u[3], k32_m04_p28); + v[4] = k_madd_epi32(u[4], k32_m28_m04); + v[5] = k_madd_epi32(u[5], k32_m28_m04); + v[6] = k_madd_epi32(u[6], k32_m28_m04); + v[7] = k_madd_epi32(u[7], k32_m28_m04); + v[8] = k_madd_epi32(u[8], k32_m20_p12); + v[9] = k_madd_epi32(u[9], k32_m20_p12); + v[10] = k_madd_epi32(u[10], k32_m20_p12); + v[11] = k_madd_epi32(u[11], k32_m20_p12); + v[12] = k_madd_epi32(u[12], k32_m12_m20); + v[13] = k_madd_epi32(u[13], k32_m12_m20); + v[14] = k_madd_epi32(u[14], k32_m12_m20); + v[15] = k_madd_epi32(u[15], k32_m12_m20); + v[16] = k_madd_epi32(u[12], k32_m20_p12); + v[17] = k_madd_epi32(u[13], k32_m20_p12); + v[18] = k_madd_epi32(u[14], k32_m20_p12); + v[19] = k_madd_epi32(u[15], k32_m20_p12); + v[20] = k_madd_epi32(u[8], k32_p12_p20); + v[21] = k_madd_epi32(u[9], k32_p12_p20); + v[22] = k_madd_epi32(u[10], k32_p12_p20); + v[23] = k_madd_epi32(u[11], k32_p12_p20); + v[24] = k_madd_epi32(u[4], k32_m04_p28); + v[25] = k_madd_epi32(u[5], k32_m04_p28); + v[26] = k_madd_epi32(u[6], k32_m04_p28); + v[27] = k_madd_epi32(u[7], k32_m04_p28); + v[28] = k_madd_epi32(u[0], k32_p28_p04); + v[29] = k_madd_epi32(u[1], k32_p28_p04); + v[30] = k_madd_epi32(u[2], k32_p28_p04); + v[31] = k_madd_epi32(u[3], k32_p28_p04); + +#if DCT_HIGH_BIT_DEPTH + overflow = k_check_epi32_overflow_32( + &v[0], &v[1], &v[2], &v[3], &v[4], &v[5], &v[6], &v[7], &v[8], + &v[9], &v[10], &v[11], &v[12], &v[13], &v[14], &v[15], &v[16], + &v[17], &v[18], &v[19], &v[20], &v[21], &v[22], &v[23], &v[24], + &v[25], &v[26], &v[27], &v[28], &v[29], &v[30], &v[31], &kZero); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + u[0] = k_packs_epi64(v[0], v[1]); + u[1] = k_packs_epi64(v[2], v[3]); + u[2] = k_packs_epi64(v[4], v[5]); + u[3] = k_packs_epi64(v[6], v[7]); + u[4] = k_packs_epi64(v[8], v[9]); + u[5] = k_packs_epi64(v[10], v[11]); + u[6] = k_packs_epi64(v[12], v[13]); + u[7] = k_packs_epi64(v[14], v[15]); + u[8] = k_packs_epi64(v[16], v[17]); + u[9] = k_packs_epi64(v[18], v[19]); + u[10] = k_packs_epi64(v[20], v[21]); + u[11] = k_packs_epi64(v[22], v[23]); + u[12] = k_packs_epi64(v[24], v[25]); + u[13] = k_packs_epi64(v[26], v[27]); + u[14] = k_packs_epi64(v[28], v[29]); + u[15] = k_packs_epi64(v[30], v[31]); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + v[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + lstep3[34] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + lstep3[35] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + lstep3[36] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + lstep3[37] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + lstep3[42] = _mm_srai_epi32(v[4], DCT_CONST_BITS); + lstep3[43] = _mm_srai_epi32(v[5], DCT_CONST_BITS); + lstep3[44] = _mm_srai_epi32(v[6], DCT_CONST_BITS); + lstep3[45] = _mm_srai_epi32(v[7], DCT_CONST_BITS); + lstep3[50] = _mm_srai_epi32(v[8], DCT_CONST_BITS); + lstep3[51] = _mm_srai_epi32(v[9], DCT_CONST_BITS); + lstep3[52] = _mm_srai_epi32(v[10], DCT_CONST_BITS); + lstep3[53] = _mm_srai_epi32(v[11], DCT_CONST_BITS); + lstep3[58] = _mm_srai_epi32(v[12], DCT_CONST_BITS); + lstep3[59] = _mm_srai_epi32(v[13], DCT_CONST_BITS); + lstep3[60] = _mm_srai_epi32(v[14], DCT_CONST_BITS); + lstep3[61] = _mm_srai_epi32(v[15], DCT_CONST_BITS); + } + // stage 7 + { + const __m128i k32_p30_p02 = pair_set_epi32(cospi_30_64, cospi_2_64); + const __m128i k32_p14_p18 = pair_set_epi32(cospi_14_64, cospi_18_64); + const __m128i k32_p22_p10 = pair_set_epi32(cospi_22_64, cospi_10_64); + const __m128i k32_p06_p26 = pair_set_epi32(cospi_6_64, cospi_26_64); + const __m128i k32_m26_p06 = pair_set_epi32(-cospi_26_64, cospi_6_64); + const __m128i k32_m10_p22 = pair_set_epi32(-cospi_10_64, cospi_22_64); + const __m128i k32_m18_p14 = pair_set_epi32(-cospi_18_64, cospi_14_64); + const __m128i k32_m02_p30 = pair_set_epi32(-cospi_2_64, cospi_30_64); + + u[0] = _mm_unpacklo_epi32(lstep3[16], lstep3[30]); + u[1] = _mm_unpackhi_epi32(lstep3[16], lstep3[30]); + u[2] = _mm_unpacklo_epi32(lstep3[17], lstep3[31]); + u[3] = _mm_unpackhi_epi32(lstep3[17], lstep3[31]); + u[4] = _mm_unpacklo_epi32(lstep3[18], lstep3[28]); + u[5] = _mm_unpackhi_epi32(lstep3[18], lstep3[28]); + u[6] = _mm_unpacklo_epi32(lstep3[19], lstep3[29]); + u[7] = _mm_unpackhi_epi32(lstep3[19], lstep3[29]); + u[8] = _mm_unpacklo_epi32(lstep3[20], lstep3[26]); + u[9] = _mm_unpackhi_epi32(lstep3[20], lstep3[26]); + u[10] = _mm_unpacklo_epi32(lstep3[21], lstep3[27]); + u[11] = _mm_unpackhi_epi32(lstep3[21], lstep3[27]); + u[12] = _mm_unpacklo_epi32(lstep3[22], lstep3[24]); + u[13] = _mm_unpackhi_epi32(lstep3[22], lstep3[24]); + u[14] = _mm_unpacklo_epi32(lstep3[23], lstep3[25]); + u[15] = _mm_unpackhi_epi32(lstep3[23], lstep3[25]); + + v[0] = k_madd_epi32(u[0], k32_p30_p02); + v[1] = k_madd_epi32(u[1], k32_p30_p02); + v[2] = k_madd_epi32(u[2], k32_p30_p02); + v[3] = k_madd_epi32(u[3], k32_p30_p02); + v[4] = k_madd_epi32(u[4], k32_p14_p18); + v[5] = k_madd_epi32(u[5], k32_p14_p18); + v[6] = k_madd_epi32(u[6], k32_p14_p18); + v[7] = k_madd_epi32(u[7], k32_p14_p18); + v[8] = k_madd_epi32(u[8], k32_p22_p10); + v[9] = k_madd_epi32(u[9], k32_p22_p10); + v[10] = k_madd_epi32(u[10], k32_p22_p10); + v[11] = k_madd_epi32(u[11], k32_p22_p10); + v[12] = k_madd_epi32(u[12], k32_p06_p26); + v[13] = k_madd_epi32(u[13], k32_p06_p26); + v[14] = k_madd_epi32(u[14], k32_p06_p26); + v[15] = k_madd_epi32(u[15], k32_p06_p26); + v[16] = k_madd_epi32(u[12], k32_m26_p06); + v[17] = k_madd_epi32(u[13], k32_m26_p06); + v[18] = k_madd_epi32(u[14], k32_m26_p06); + v[19] = k_madd_epi32(u[15], k32_m26_p06); + v[20] = k_madd_epi32(u[8], k32_m10_p22); + v[21] = k_madd_epi32(u[9], k32_m10_p22); + v[22] = k_madd_epi32(u[10], k32_m10_p22); + v[23] = k_madd_epi32(u[11], k32_m10_p22); + v[24] = k_madd_epi32(u[4], k32_m18_p14); + v[25] = k_madd_epi32(u[5], k32_m18_p14); + v[26] = k_madd_epi32(u[6], k32_m18_p14); + v[27] = k_madd_epi32(u[7], k32_m18_p14); + v[28] = k_madd_epi32(u[0], k32_m02_p30); + v[29] = k_madd_epi32(u[1], k32_m02_p30); + v[30] = k_madd_epi32(u[2], k32_m02_p30); + v[31] = k_madd_epi32(u[3], k32_m02_p30); + +#if DCT_HIGH_BIT_DEPTH + overflow = k_check_epi32_overflow_32( + &v[0], &v[1], &v[2], &v[3], &v[4], &v[5], &v[6], &v[7], &v[8], + &v[9], &v[10], &v[11], &v[12], &v[13], &v[14], &v[15], &v[16], + &v[17], &v[18], &v[19], &v[20], &v[21], &v[22], &v[23], &v[24], + &v[25], &v[26], &v[27], &v[28], &v[29], &v[30], &v[31], &kZero); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + u[0] = k_packs_epi64(v[0], v[1]); + u[1] = k_packs_epi64(v[2], v[3]); + u[2] = k_packs_epi64(v[4], v[5]); + u[3] = k_packs_epi64(v[6], v[7]); + u[4] = k_packs_epi64(v[8], v[9]); + u[5] = k_packs_epi64(v[10], v[11]); + u[6] = k_packs_epi64(v[12], v[13]); + u[7] = k_packs_epi64(v[14], v[15]); + u[8] = k_packs_epi64(v[16], v[17]); + u[9] = k_packs_epi64(v[18], v[19]); + u[10] = k_packs_epi64(v[20], v[21]); + u[11] = k_packs_epi64(v[22], v[23]); + u[12] = k_packs_epi64(v[24], v[25]); + u[13] = k_packs_epi64(v[26], v[27]); + u[14] = k_packs_epi64(v[28], v[29]); + u[15] = k_packs_epi64(v[30], v[31]); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + v[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS); + u[8] = _mm_srai_epi32(v[8], DCT_CONST_BITS); + u[9] = _mm_srai_epi32(v[9], DCT_CONST_BITS); + u[10] = _mm_srai_epi32(v[10], DCT_CONST_BITS); + u[11] = _mm_srai_epi32(v[11], DCT_CONST_BITS); + u[12] = _mm_srai_epi32(v[12], DCT_CONST_BITS); + u[13] = _mm_srai_epi32(v[13], DCT_CONST_BITS); + u[14] = _mm_srai_epi32(v[14], DCT_CONST_BITS); + u[15] = _mm_srai_epi32(v[15], DCT_CONST_BITS); + + v[0] = _mm_cmplt_epi32(u[0], kZero); + v[1] = _mm_cmplt_epi32(u[1], kZero); + v[2] = _mm_cmplt_epi32(u[2], kZero); + v[3] = _mm_cmplt_epi32(u[3], kZero); + v[4] = _mm_cmplt_epi32(u[4], kZero); + v[5] = _mm_cmplt_epi32(u[5], kZero); + v[6] = _mm_cmplt_epi32(u[6], kZero); + v[7] = _mm_cmplt_epi32(u[7], kZero); + v[8] = _mm_cmplt_epi32(u[8], kZero); + v[9] = _mm_cmplt_epi32(u[9], kZero); + v[10] = _mm_cmplt_epi32(u[10], kZero); + v[11] = _mm_cmplt_epi32(u[11], kZero); + v[12] = _mm_cmplt_epi32(u[12], kZero); + v[13] = _mm_cmplt_epi32(u[13], kZero); + v[14] = _mm_cmplt_epi32(u[14], kZero); + v[15] = _mm_cmplt_epi32(u[15], kZero); + + u[0] = _mm_sub_epi32(u[0], v[0]); + u[1] = _mm_sub_epi32(u[1], v[1]); + u[2] = _mm_sub_epi32(u[2], v[2]); + u[3] = _mm_sub_epi32(u[3], v[3]); + u[4] = _mm_sub_epi32(u[4], v[4]); + u[5] = _mm_sub_epi32(u[5], v[5]); + u[6] = _mm_sub_epi32(u[6], v[6]); + u[7] = _mm_sub_epi32(u[7], v[7]); + u[8] = _mm_sub_epi32(u[8], v[8]); + u[9] = _mm_sub_epi32(u[9], v[9]); + u[10] = _mm_sub_epi32(u[10], v[10]); + u[11] = _mm_sub_epi32(u[11], v[11]); + u[12] = _mm_sub_epi32(u[12], v[12]); + u[13] = _mm_sub_epi32(u[13], v[13]); + u[14] = _mm_sub_epi32(u[14], v[14]); + u[15] = _mm_sub_epi32(u[15], v[15]); + + v[0] = _mm_add_epi32(u[0], K32One); + v[1] = _mm_add_epi32(u[1], K32One); + v[2] = _mm_add_epi32(u[2], K32One); + v[3] = _mm_add_epi32(u[3], K32One); + v[4] = _mm_add_epi32(u[4], K32One); + v[5] = _mm_add_epi32(u[5], K32One); + v[6] = _mm_add_epi32(u[6], K32One); + v[7] = _mm_add_epi32(u[7], K32One); + v[8] = _mm_add_epi32(u[8], K32One); + v[9] = _mm_add_epi32(u[9], K32One); + v[10] = _mm_add_epi32(u[10], K32One); + v[11] = _mm_add_epi32(u[11], K32One); + v[12] = _mm_add_epi32(u[12], K32One); + v[13] = _mm_add_epi32(u[13], K32One); + v[14] = _mm_add_epi32(u[14], K32One); + v[15] = _mm_add_epi32(u[15], K32One); + + u[0] = _mm_srai_epi32(v[0], 2); + u[1] = _mm_srai_epi32(v[1], 2); + u[2] = _mm_srai_epi32(v[2], 2); + u[3] = _mm_srai_epi32(v[3], 2); + u[4] = _mm_srai_epi32(v[4], 2); + u[5] = _mm_srai_epi32(v[5], 2); + u[6] = _mm_srai_epi32(v[6], 2); + u[7] = _mm_srai_epi32(v[7], 2); + u[8] = _mm_srai_epi32(v[8], 2); + u[9] = _mm_srai_epi32(v[9], 2); + u[10] = _mm_srai_epi32(v[10], 2); + u[11] = _mm_srai_epi32(v[11], 2); + u[12] = _mm_srai_epi32(v[12], 2); + u[13] = _mm_srai_epi32(v[13], 2); + u[14] = _mm_srai_epi32(v[14], 2); + u[15] = _mm_srai_epi32(v[15], 2); + + out[2] = _mm_packs_epi32(u[0], u[1]); + out[18] = _mm_packs_epi32(u[2], u[3]); + out[10] = _mm_packs_epi32(u[4], u[5]); + out[26] = _mm_packs_epi32(u[6], u[7]); + out[6] = _mm_packs_epi32(u[8], u[9]); + out[22] = _mm_packs_epi32(u[10], u[11]); + out[14] = _mm_packs_epi32(u[12], u[13]); + out[30] = _mm_packs_epi32(u[14], u[15]); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x8(&out[2], &out[18], &out[10], &out[26], + &out[6], &out[22], &out[14], &out[30]); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + lstep1[32] = _mm_add_epi32(lstep3[34], lstep2[32]); + lstep1[33] = _mm_add_epi32(lstep3[35], lstep2[33]); + lstep1[34] = _mm_sub_epi32(lstep2[32], lstep3[34]); + lstep1[35] = _mm_sub_epi32(lstep2[33], lstep3[35]); + lstep1[36] = _mm_sub_epi32(lstep2[38], lstep3[36]); + lstep1[37] = _mm_sub_epi32(lstep2[39], lstep3[37]); + lstep1[38] = _mm_add_epi32(lstep3[36], lstep2[38]); + lstep1[39] = _mm_add_epi32(lstep3[37], lstep2[39]); + lstep1[40] = _mm_add_epi32(lstep3[42], lstep2[40]); + lstep1[41] = _mm_add_epi32(lstep3[43], lstep2[41]); + lstep1[42] = _mm_sub_epi32(lstep2[40], lstep3[42]); + lstep1[43] = _mm_sub_epi32(lstep2[41], lstep3[43]); + lstep1[44] = _mm_sub_epi32(lstep2[46], lstep3[44]); + lstep1[45] = _mm_sub_epi32(lstep2[47], lstep3[45]); + lstep1[46] = _mm_add_epi32(lstep3[44], lstep2[46]); + lstep1[47] = _mm_add_epi32(lstep3[45], lstep2[47]); + lstep1[48] = _mm_add_epi32(lstep3[50], lstep2[48]); + lstep1[49] = _mm_add_epi32(lstep3[51], lstep2[49]); + lstep1[50] = _mm_sub_epi32(lstep2[48], lstep3[50]); + lstep1[51] = _mm_sub_epi32(lstep2[49], lstep3[51]); + lstep1[52] = _mm_sub_epi32(lstep2[54], lstep3[52]); + lstep1[53] = _mm_sub_epi32(lstep2[55], lstep3[53]); + lstep1[54] = _mm_add_epi32(lstep3[52], lstep2[54]); + lstep1[55] = _mm_add_epi32(lstep3[53], lstep2[55]); + lstep1[56] = _mm_add_epi32(lstep3[58], lstep2[56]); + lstep1[57] = _mm_add_epi32(lstep3[59], lstep2[57]); + lstep1[58] = _mm_sub_epi32(lstep2[56], lstep3[58]); + lstep1[59] = _mm_sub_epi32(lstep2[57], lstep3[59]); + lstep1[60] = _mm_sub_epi32(lstep2[62], lstep3[60]); + lstep1[61] = _mm_sub_epi32(lstep2[63], lstep3[61]); + lstep1[62] = _mm_add_epi32(lstep3[60], lstep2[62]); + lstep1[63] = _mm_add_epi32(lstep3[61], lstep2[63]); + } + // stage 8 + { + const __m128i k32_p31_p01 = pair_set_epi32(cospi_31_64, cospi_1_64); + const __m128i k32_p15_p17 = pair_set_epi32(cospi_15_64, cospi_17_64); + const __m128i k32_p23_p09 = pair_set_epi32(cospi_23_64, cospi_9_64); + const __m128i k32_p07_p25 = pair_set_epi32(cospi_7_64, cospi_25_64); + const __m128i k32_m25_p07 = pair_set_epi32(-cospi_25_64, cospi_7_64); + const __m128i k32_m09_p23 = pair_set_epi32(-cospi_9_64, cospi_23_64); + const __m128i k32_m17_p15 = pair_set_epi32(-cospi_17_64, cospi_15_64); + const __m128i k32_m01_p31 = pair_set_epi32(-cospi_1_64, cospi_31_64); + + u[0] = _mm_unpacklo_epi32(lstep1[32], lstep1[62]); + u[1] = _mm_unpackhi_epi32(lstep1[32], lstep1[62]); + u[2] = _mm_unpacklo_epi32(lstep1[33], lstep1[63]); + u[3] = _mm_unpackhi_epi32(lstep1[33], lstep1[63]); + u[4] = _mm_unpacklo_epi32(lstep1[34], lstep1[60]); + u[5] = _mm_unpackhi_epi32(lstep1[34], lstep1[60]); + u[6] = _mm_unpacklo_epi32(lstep1[35], lstep1[61]); + u[7] = _mm_unpackhi_epi32(lstep1[35], lstep1[61]); + u[8] = _mm_unpacklo_epi32(lstep1[36], lstep1[58]); + u[9] = _mm_unpackhi_epi32(lstep1[36], lstep1[58]); + u[10] = _mm_unpacklo_epi32(lstep1[37], lstep1[59]); + u[11] = _mm_unpackhi_epi32(lstep1[37], lstep1[59]); + u[12] = _mm_unpacklo_epi32(lstep1[38], lstep1[56]); + u[13] = _mm_unpackhi_epi32(lstep1[38], lstep1[56]); + u[14] = _mm_unpacklo_epi32(lstep1[39], lstep1[57]); + u[15] = _mm_unpackhi_epi32(lstep1[39], lstep1[57]); + + v[0] = k_madd_epi32(u[0], k32_p31_p01); + v[1] = k_madd_epi32(u[1], k32_p31_p01); + v[2] = k_madd_epi32(u[2], k32_p31_p01); + v[3] = k_madd_epi32(u[3], k32_p31_p01); + v[4] = k_madd_epi32(u[4], k32_p15_p17); + v[5] = k_madd_epi32(u[5], k32_p15_p17); + v[6] = k_madd_epi32(u[6], k32_p15_p17); + v[7] = k_madd_epi32(u[7], k32_p15_p17); + v[8] = k_madd_epi32(u[8], k32_p23_p09); + v[9] = k_madd_epi32(u[9], k32_p23_p09); + v[10] = k_madd_epi32(u[10], k32_p23_p09); + v[11] = k_madd_epi32(u[11], k32_p23_p09); + v[12] = k_madd_epi32(u[12], k32_p07_p25); + v[13] = k_madd_epi32(u[13], k32_p07_p25); + v[14] = k_madd_epi32(u[14], k32_p07_p25); + v[15] = k_madd_epi32(u[15], k32_p07_p25); + v[16] = k_madd_epi32(u[12], k32_m25_p07); + v[17] = k_madd_epi32(u[13], k32_m25_p07); + v[18] = k_madd_epi32(u[14], k32_m25_p07); + v[19] = k_madd_epi32(u[15], k32_m25_p07); + v[20] = k_madd_epi32(u[8], k32_m09_p23); + v[21] = k_madd_epi32(u[9], k32_m09_p23); + v[22] = k_madd_epi32(u[10], k32_m09_p23); + v[23] = k_madd_epi32(u[11], k32_m09_p23); + v[24] = k_madd_epi32(u[4], k32_m17_p15); + v[25] = k_madd_epi32(u[5], k32_m17_p15); + v[26] = k_madd_epi32(u[6], k32_m17_p15); + v[27] = k_madd_epi32(u[7], k32_m17_p15); + v[28] = k_madd_epi32(u[0], k32_m01_p31); + v[29] = k_madd_epi32(u[1], k32_m01_p31); + v[30] = k_madd_epi32(u[2], k32_m01_p31); + v[31] = k_madd_epi32(u[3], k32_m01_p31); + +#if DCT_HIGH_BIT_DEPTH + overflow = k_check_epi32_overflow_32( + &v[0], &v[1], &v[2], &v[3], &v[4], &v[5], &v[6], &v[7], &v[8], + &v[9], &v[10], &v[11], &v[12], &v[13], &v[14], &v[15], &v[16], + &v[17], &v[18], &v[19], &v[20], &v[21], &v[22], &v[23], &v[24], + &v[25], &v[26], &v[27], &v[28], &v[29], &v[30], &v[31], &kZero); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + u[0] = k_packs_epi64(v[0], v[1]); + u[1] = k_packs_epi64(v[2], v[3]); + u[2] = k_packs_epi64(v[4], v[5]); + u[3] = k_packs_epi64(v[6], v[7]); + u[4] = k_packs_epi64(v[8], v[9]); + u[5] = k_packs_epi64(v[10], v[11]); + u[6] = k_packs_epi64(v[12], v[13]); + u[7] = k_packs_epi64(v[14], v[15]); + u[8] = k_packs_epi64(v[16], v[17]); + u[9] = k_packs_epi64(v[18], v[19]); + u[10] = k_packs_epi64(v[20], v[21]); + u[11] = k_packs_epi64(v[22], v[23]); + u[12] = k_packs_epi64(v[24], v[25]); + u[13] = k_packs_epi64(v[26], v[27]); + u[14] = k_packs_epi64(v[28], v[29]); + u[15] = k_packs_epi64(v[30], v[31]); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + v[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS); + u[8] = _mm_srai_epi32(v[8], DCT_CONST_BITS); + u[9] = _mm_srai_epi32(v[9], DCT_CONST_BITS); + u[10] = _mm_srai_epi32(v[10], DCT_CONST_BITS); + u[11] = _mm_srai_epi32(v[11], DCT_CONST_BITS); + u[12] = _mm_srai_epi32(v[12], DCT_CONST_BITS); + u[13] = _mm_srai_epi32(v[13], DCT_CONST_BITS); + u[14] = _mm_srai_epi32(v[14], DCT_CONST_BITS); + u[15] = _mm_srai_epi32(v[15], DCT_CONST_BITS); + + v[0] = _mm_cmplt_epi32(u[0], kZero); + v[1] = _mm_cmplt_epi32(u[1], kZero); + v[2] = _mm_cmplt_epi32(u[2], kZero); + v[3] = _mm_cmplt_epi32(u[3], kZero); + v[4] = _mm_cmplt_epi32(u[4], kZero); + v[5] = _mm_cmplt_epi32(u[5], kZero); + v[6] = _mm_cmplt_epi32(u[6], kZero); + v[7] = _mm_cmplt_epi32(u[7], kZero); + v[8] = _mm_cmplt_epi32(u[8], kZero); + v[9] = _mm_cmplt_epi32(u[9], kZero); + v[10] = _mm_cmplt_epi32(u[10], kZero); + v[11] = _mm_cmplt_epi32(u[11], kZero); + v[12] = _mm_cmplt_epi32(u[12], kZero); + v[13] = _mm_cmplt_epi32(u[13], kZero); + v[14] = _mm_cmplt_epi32(u[14], kZero); + v[15] = _mm_cmplt_epi32(u[15], kZero); + + u[0] = _mm_sub_epi32(u[0], v[0]); + u[1] = _mm_sub_epi32(u[1], v[1]); + u[2] = _mm_sub_epi32(u[2], v[2]); + u[3] = _mm_sub_epi32(u[3], v[3]); + u[4] = _mm_sub_epi32(u[4], v[4]); + u[5] = _mm_sub_epi32(u[5], v[5]); + u[6] = _mm_sub_epi32(u[6], v[6]); + u[7] = _mm_sub_epi32(u[7], v[7]); + u[8] = _mm_sub_epi32(u[8], v[8]); + u[9] = _mm_sub_epi32(u[9], v[9]); + u[10] = _mm_sub_epi32(u[10], v[10]); + u[11] = _mm_sub_epi32(u[11], v[11]); + u[12] = _mm_sub_epi32(u[12], v[12]); + u[13] = _mm_sub_epi32(u[13], v[13]); + u[14] = _mm_sub_epi32(u[14], v[14]); + u[15] = _mm_sub_epi32(u[15], v[15]); + + v[0] = _mm_add_epi32(u[0], K32One); + v[1] = _mm_add_epi32(u[1], K32One); + v[2] = _mm_add_epi32(u[2], K32One); + v[3] = _mm_add_epi32(u[3], K32One); + v[4] = _mm_add_epi32(u[4], K32One); + v[5] = _mm_add_epi32(u[5], K32One); + v[6] = _mm_add_epi32(u[6], K32One); + v[7] = _mm_add_epi32(u[7], K32One); + v[8] = _mm_add_epi32(u[8], K32One); + v[9] = _mm_add_epi32(u[9], K32One); + v[10] = _mm_add_epi32(u[10], K32One); + v[11] = _mm_add_epi32(u[11], K32One); + v[12] = _mm_add_epi32(u[12], K32One); + v[13] = _mm_add_epi32(u[13], K32One); + v[14] = _mm_add_epi32(u[14], K32One); + v[15] = _mm_add_epi32(u[15], K32One); + + u[0] = _mm_srai_epi32(v[0], 2); + u[1] = _mm_srai_epi32(v[1], 2); + u[2] = _mm_srai_epi32(v[2], 2); + u[3] = _mm_srai_epi32(v[3], 2); + u[4] = _mm_srai_epi32(v[4], 2); + u[5] = _mm_srai_epi32(v[5], 2); + u[6] = _mm_srai_epi32(v[6], 2); + u[7] = _mm_srai_epi32(v[7], 2); + u[8] = _mm_srai_epi32(v[8], 2); + u[9] = _mm_srai_epi32(v[9], 2); + u[10] = _mm_srai_epi32(v[10], 2); + u[11] = _mm_srai_epi32(v[11], 2); + u[12] = _mm_srai_epi32(v[12], 2); + u[13] = _mm_srai_epi32(v[13], 2); + u[14] = _mm_srai_epi32(v[14], 2); + u[15] = _mm_srai_epi32(v[15], 2); + + out[1] = _mm_packs_epi32(u[0], u[1]); + out[17] = _mm_packs_epi32(u[2], u[3]); + out[9] = _mm_packs_epi32(u[4], u[5]); + out[25] = _mm_packs_epi32(u[6], u[7]); + out[7] = _mm_packs_epi32(u[8], u[9]); + out[23] = _mm_packs_epi32(u[10], u[11]); + out[15] = _mm_packs_epi32(u[12], u[13]); + out[31] = _mm_packs_epi32(u[14], u[15]); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x8(&out[1], &out[17], &out[9], &out[25], + &out[7], &out[23], &out[15], &out[31]); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + const __m128i k32_p27_p05 = pair_set_epi32(cospi_27_64, cospi_5_64); + const __m128i k32_p11_p21 = pair_set_epi32(cospi_11_64, cospi_21_64); + const __m128i k32_p19_p13 = pair_set_epi32(cospi_19_64, cospi_13_64); + const __m128i k32_p03_p29 = pair_set_epi32(cospi_3_64, cospi_29_64); + const __m128i k32_m29_p03 = pair_set_epi32(-cospi_29_64, cospi_3_64); + const __m128i k32_m13_p19 = pair_set_epi32(-cospi_13_64, cospi_19_64); + const __m128i k32_m21_p11 = pair_set_epi32(-cospi_21_64, cospi_11_64); + const __m128i k32_m05_p27 = pair_set_epi32(-cospi_5_64, cospi_27_64); + + u[0] = _mm_unpacklo_epi32(lstep1[40], lstep1[54]); + u[1] = _mm_unpackhi_epi32(lstep1[40], lstep1[54]); + u[2] = _mm_unpacklo_epi32(lstep1[41], lstep1[55]); + u[3] = _mm_unpackhi_epi32(lstep1[41], lstep1[55]); + u[4] = _mm_unpacklo_epi32(lstep1[42], lstep1[52]); + u[5] = _mm_unpackhi_epi32(lstep1[42], lstep1[52]); + u[6] = _mm_unpacklo_epi32(lstep1[43], lstep1[53]); + u[7] = _mm_unpackhi_epi32(lstep1[43], lstep1[53]); + u[8] = _mm_unpacklo_epi32(lstep1[44], lstep1[50]); + u[9] = _mm_unpackhi_epi32(lstep1[44], lstep1[50]); + u[10] = _mm_unpacklo_epi32(lstep1[45], lstep1[51]); + u[11] = _mm_unpackhi_epi32(lstep1[45], lstep1[51]); + u[12] = _mm_unpacklo_epi32(lstep1[46], lstep1[48]); + u[13] = _mm_unpackhi_epi32(lstep1[46], lstep1[48]); + u[14] = _mm_unpacklo_epi32(lstep1[47], lstep1[49]); + u[15] = _mm_unpackhi_epi32(lstep1[47], lstep1[49]); + + v[0] = k_madd_epi32(u[0], k32_p27_p05); + v[1] = k_madd_epi32(u[1], k32_p27_p05); + v[2] = k_madd_epi32(u[2], k32_p27_p05); + v[3] = k_madd_epi32(u[3], k32_p27_p05); + v[4] = k_madd_epi32(u[4], k32_p11_p21); + v[5] = k_madd_epi32(u[5], k32_p11_p21); + v[6] = k_madd_epi32(u[6], k32_p11_p21); + v[7] = k_madd_epi32(u[7], k32_p11_p21); + v[8] = k_madd_epi32(u[8], k32_p19_p13); + v[9] = k_madd_epi32(u[9], k32_p19_p13); + v[10] = k_madd_epi32(u[10], k32_p19_p13); + v[11] = k_madd_epi32(u[11], k32_p19_p13); + v[12] = k_madd_epi32(u[12], k32_p03_p29); + v[13] = k_madd_epi32(u[13], k32_p03_p29); + v[14] = k_madd_epi32(u[14], k32_p03_p29); + v[15] = k_madd_epi32(u[15], k32_p03_p29); + v[16] = k_madd_epi32(u[12], k32_m29_p03); + v[17] = k_madd_epi32(u[13], k32_m29_p03); + v[18] = k_madd_epi32(u[14], k32_m29_p03); + v[19] = k_madd_epi32(u[15], k32_m29_p03); + v[20] = k_madd_epi32(u[8], k32_m13_p19); + v[21] = k_madd_epi32(u[9], k32_m13_p19); + v[22] = k_madd_epi32(u[10], k32_m13_p19); + v[23] = k_madd_epi32(u[11], k32_m13_p19); + v[24] = k_madd_epi32(u[4], k32_m21_p11); + v[25] = k_madd_epi32(u[5], k32_m21_p11); + v[26] = k_madd_epi32(u[6], k32_m21_p11); + v[27] = k_madd_epi32(u[7], k32_m21_p11); + v[28] = k_madd_epi32(u[0], k32_m05_p27); + v[29] = k_madd_epi32(u[1], k32_m05_p27); + v[30] = k_madd_epi32(u[2], k32_m05_p27); + v[31] = k_madd_epi32(u[3], k32_m05_p27); + +#if DCT_HIGH_BIT_DEPTH + overflow = k_check_epi32_overflow_32( + &v[0], &v[1], &v[2], &v[3], &v[4], &v[5], &v[6], &v[7], &v[8], + &v[9], &v[10], &v[11], &v[12], &v[13], &v[14], &v[15], &v[16], + &v[17], &v[18], &v[19], &v[20], &v[21], &v[22], &v[23], &v[24], + &v[25], &v[26], &v[27], &v[28], &v[29], &v[30], &v[31], &kZero); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + u[0] = k_packs_epi64(v[0], v[1]); + u[1] = k_packs_epi64(v[2], v[3]); + u[2] = k_packs_epi64(v[4], v[5]); + u[3] = k_packs_epi64(v[6], v[7]); + u[4] = k_packs_epi64(v[8], v[9]); + u[5] = k_packs_epi64(v[10], v[11]); + u[6] = k_packs_epi64(v[12], v[13]); + u[7] = k_packs_epi64(v[14], v[15]); + u[8] = k_packs_epi64(v[16], v[17]); + u[9] = k_packs_epi64(v[18], v[19]); + u[10] = k_packs_epi64(v[20], v[21]); + u[11] = k_packs_epi64(v[22], v[23]); + u[12] = k_packs_epi64(v[24], v[25]); + u[13] = k_packs_epi64(v[26], v[27]); + u[14] = k_packs_epi64(v[28], v[29]); + u[15] = k_packs_epi64(v[30], v[31]); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + v[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS); + u[8] = _mm_srai_epi32(v[8], DCT_CONST_BITS); + u[9] = _mm_srai_epi32(v[9], DCT_CONST_BITS); + u[10] = _mm_srai_epi32(v[10], DCT_CONST_BITS); + u[11] = _mm_srai_epi32(v[11], DCT_CONST_BITS); + u[12] = _mm_srai_epi32(v[12], DCT_CONST_BITS); + u[13] = _mm_srai_epi32(v[13], DCT_CONST_BITS); + u[14] = _mm_srai_epi32(v[14], DCT_CONST_BITS); + u[15] = _mm_srai_epi32(v[15], DCT_CONST_BITS); + + v[0] = _mm_cmplt_epi32(u[0], kZero); + v[1] = _mm_cmplt_epi32(u[1], kZero); + v[2] = _mm_cmplt_epi32(u[2], kZero); + v[3] = _mm_cmplt_epi32(u[3], kZero); + v[4] = _mm_cmplt_epi32(u[4], kZero); + v[5] = _mm_cmplt_epi32(u[5], kZero); + v[6] = _mm_cmplt_epi32(u[6], kZero); + v[7] = _mm_cmplt_epi32(u[7], kZero); + v[8] = _mm_cmplt_epi32(u[8], kZero); + v[9] = _mm_cmplt_epi32(u[9], kZero); + v[10] = _mm_cmplt_epi32(u[10], kZero); + v[11] = _mm_cmplt_epi32(u[11], kZero); + v[12] = _mm_cmplt_epi32(u[12], kZero); + v[13] = _mm_cmplt_epi32(u[13], kZero); + v[14] = _mm_cmplt_epi32(u[14], kZero); + v[15] = _mm_cmplt_epi32(u[15], kZero); + + u[0] = _mm_sub_epi32(u[0], v[0]); + u[1] = _mm_sub_epi32(u[1], v[1]); + u[2] = _mm_sub_epi32(u[2], v[2]); + u[3] = _mm_sub_epi32(u[3], v[3]); + u[4] = _mm_sub_epi32(u[4], v[4]); + u[5] = _mm_sub_epi32(u[5], v[5]); + u[6] = _mm_sub_epi32(u[6], v[6]); + u[7] = _mm_sub_epi32(u[7], v[7]); + u[8] = _mm_sub_epi32(u[8], v[8]); + u[9] = _mm_sub_epi32(u[9], v[9]); + u[10] = _mm_sub_epi32(u[10], v[10]); + u[11] = _mm_sub_epi32(u[11], v[11]); + u[12] = _mm_sub_epi32(u[12], v[12]); + u[13] = _mm_sub_epi32(u[13], v[13]); + u[14] = _mm_sub_epi32(u[14], v[14]); + u[15] = _mm_sub_epi32(u[15], v[15]); + + v[0] = _mm_add_epi32(u[0], K32One); + v[1] = _mm_add_epi32(u[1], K32One); + v[2] = _mm_add_epi32(u[2], K32One); + v[3] = _mm_add_epi32(u[3], K32One); + v[4] = _mm_add_epi32(u[4], K32One); + v[5] = _mm_add_epi32(u[5], K32One); + v[6] = _mm_add_epi32(u[6], K32One); + v[7] = _mm_add_epi32(u[7], K32One); + v[8] = _mm_add_epi32(u[8], K32One); + v[9] = _mm_add_epi32(u[9], K32One); + v[10] = _mm_add_epi32(u[10], K32One); + v[11] = _mm_add_epi32(u[11], K32One); + v[12] = _mm_add_epi32(u[12], K32One); + v[13] = _mm_add_epi32(u[13], K32One); + v[14] = _mm_add_epi32(u[14], K32One); + v[15] = _mm_add_epi32(u[15], K32One); + + u[0] = _mm_srai_epi32(v[0], 2); + u[1] = _mm_srai_epi32(v[1], 2); + u[2] = _mm_srai_epi32(v[2], 2); + u[3] = _mm_srai_epi32(v[3], 2); + u[4] = _mm_srai_epi32(v[4], 2); + u[5] = _mm_srai_epi32(v[5], 2); + u[6] = _mm_srai_epi32(v[6], 2); + u[7] = _mm_srai_epi32(v[7], 2); + u[8] = _mm_srai_epi32(v[8], 2); + u[9] = _mm_srai_epi32(v[9], 2); + u[10] = _mm_srai_epi32(v[10], 2); + u[11] = _mm_srai_epi32(v[11], 2); + u[12] = _mm_srai_epi32(v[12], 2); + u[13] = _mm_srai_epi32(v[13], 2); + u[14] = _mm_srai_epi32(v[14], 2); + u[15] = _mm_srai_epi32(v[15], 2); + + out[5] = _mm_packs_epi32(u[0], u[1]); + out[21] = _mm_packs_epi32(u[2], u[3]); + out[13] = _mm_packs_epi32(u[4], u[5]); + out[29] = _mm_packs_epi32(u[6], u[7]); + out[3] = _mm_packs_epi32(u[8], u[9]); + out[19] = _mm_packs_epi32(u[10], u[11]); + out[11] = _mm_packs_epi32(u[12], u[13]); + out[27] = _mm_packs_epi32(u[14], u[15]); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x8(&out[5], &out[21], &out[13], &out[29], + &out[3], &out[19], &out[11], &out[27]); + if (overflow) { + HIGH_FDCT32x32_2D_ROWS_C(intermediate, output_org); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + } +#endif // FDCT32x32_HIGH_PRECISION + // Transpose the results, do it as four 8x8 transposes. + { + int transpose_block; + int16_t *output0 = &intermediate[column_start * 32]; + tran_low_t *output1 = &output_org[column_start * 32]; + for (transpose_block = 0; transpose_block < 4; ++transpose_block) { + __m128i *this_out = &out[8 * transpose_block]; + // 00 01 02 03 04 05 06 07 + // 10 11 12 13 14 15 16 17 + // 20 21 22 23 24 25 26 27 + // 30 31 32 33 34 35 36 37 + // 40 41 42 43 44 45 46 47 + // 50 51 52 53 54 55 56 57 + // 60 61 62 63 64 65 66 67 + // 70 71 72 73 74 75 76 77 + const __m128i tr0_0 = _mm_unpacklo_epi16(this_out[0], this_out[1]); + const __m128i tr0_1 = _mm_unpacklo_epi16(this_out[2], this_out[3]); + const __m128i tr0_2 = _mm_unpackhi_epi16(this_out[0], this_out[1]); + const __m128i tr0_3 = _mm_unpackhi_epi16(this_out[2], this_out[3]); + const __m128i tr0_4 = _mm_unpacklo_epi16(this_out[4], this_out[5]); + const __m128i tr0_5 = _mm_unpacklo_epi16(this_out[6], this_out[7]); + const __m128i tr0_6 = _mm_unpackhi_epi16(this_out[4], this_out[5]); + const __m128i tr0_7 = _mm_unpackhi_epi16(this_out[6], this_out[7]); + // 00 10 01 11 02 12 03 13 + // 20 30 21 31 22 32 23 33 + // 04 14 05 15 06 16 07 17 + // 24 34 25 35 26 36 27 37 + // 40 50 41 51 42 52 43 53 + // 60 70 61 71 62 72 63 73 + // 54 54 55 55 56 56 57 57 + // 64 74 65 75 66 76 67 77 + const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); + const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3); + const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); + const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3); + const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); + const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); + const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); + const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); + // 00 10 20 30 01 11 21 31 + // 40 50 60 70 41 51 61 71 + // 02 12 22 32 03 13 23 33 + // 42 52 62 72 43 53 63 73 + // 04 14 24 34 05 15 21 36 + // 44 54 64 74 45 55 61 76 + // 06 16 26 36 07 17 27 37 + // 46 56 66 76 47 57 67 77 + __m128i tr2_0 = _mm_unpacklo_epi64(tr1_0, tr1_4); + __m128i tr2_1 = _mm_unpackhi_epi64(tr1_0, tr1_4); + __m128i tr2_2 = _mm_unpacklo_epi64(tr1_2, tr1_6); + __m128i tr2_3 = _mm_unpackhi_epi64(tr1_2, tr1_6); + __m128i tr2_4 = _mm_unpacklo_epi64(tr1_1, tr1_5); + __m128i tr2_5 = _mm_unpackhi_epi64(tr1_1, tr1_5); + __m128i tr2_6 = _mm_unpacklo_epi64(tr1_3, tr1_7); + __m128i tr2_7 = _mm_unpackhi_epi64(tr1_3, tr1_7); + // 00 10 20 30 40 50 60 70 + // 01 11 21 31 41 51 61 71 + // 02 12 22 32 42 52 62 72 + // 03 13 23 33 43 53 63 73 + // 04 14 24 34 44 54 64 74 + // 05 15 25 35 45 55 65 75 + // 06 16 26 36 46 56 66 76 + // 07 17 27 37 47 57 67 77 + if (0 == pass) { + // output[j] = (output[j] + 1 + (output[j] > 0)) >> 2; + // TODO(cd): see quality impact of only doing + // output[j] = (output[j] + 1) >> 2; + // which would remove the code between here ... + __m128i tr2_0_0 = _mm_cmpgt_epi16(tr2_0, kZero); + __m128i tr2_1_0 = _mm_cmpgt_epi16(tr2_1, kZero); + __m128i tr2_2_0 = _mm_cmpgt_epi16(tr2_2, kZero); + __m128i tr2_3_0 = _mm_cmpgt_epi16(tr2_3, kZero); + __m128i tr2_4_0 = _mm_cmpgt_epi16(tr2_4, kZero); + __m128i tr2_5_0 = _mm_cmpgt_epi16(tr2_5, kZero); + __m128i tr2_6_0 = _mm_cmpgt_epi16(tr2_6, kZero); + __m128i tr2_7_0 = _mm_cmpgt_epi16(tr2_7, kZero); + tr2_0 = _mm_sub_epi16(tr2_0, tr2_0_0); + tr2_1 = _mm_sub_epi16(tr2_1, tr2_1_0); + tr2_2 = _mm_sub_epi16(tr2_2, tr2_2_0); + tr2_3 = _mm_sub_epi16(tr2_3, tr2_3_0); + tr2_4 = _mm_sub_epi16(tr2_4, tr2_4_0); + tr2_5 = _mm_sub_epi16(tr2_5, tr2_5_0); + tr2_6 = _mm_sub_epi16(tr2_6, tr2_6_0); + tr2_7 = _mm_sub_epi16(tr2_7, tr2_7_0); + // ... and here. + // PS: also change code in av1/encoder/av1_dct.c + tr2_0 = _mm_add_epi16(tr2_0, kOne); + tr2_1 = _mm_add_epi16(tr2_1, kOne); + tr2_2 = _mm_add_epi16(tr2_2, kOne); + tr2_3 = _mm_add_epi16(tr2_3, kOne); + tr2_4 = _mm_add_epi16(tr2_4, kOne); + tr2_5 = _mm_add_epi16(tr2_5, kOne); + tr2_6 = _mm_add_epi16(tr2_6, kOne); + tr2_7 = _mm_add_epi16(tr2_7, kOne); + tr2_0 = _mm_srai_epi16(tr2_0, 2); + tr2_1 = _mm_srai_epi16(tr2_1, 2); + tr2_2 = _mm_srai_epi16(tr2_2, 2); + tr2_3 = _mm_srai_epi16(tr2_3, 2); + tr2_4 = _mm_srai_epi16(tr2_4, 2); + tr2_5 = _mm_srai_epi16(tr2_5, 2); + tr2_6 = _mm_srai_epi16(tr2_6, 2); + tr2_7 = _mm_srai_epi16(tr2_7, 2); + } + // Note: even though all these stores are aligned, using the aligned + // intrinsic make the code slightly slower. + if (pass == 0) { + _mm_storeu_si128((__m128i *)(output0 + 0 * 32), tr2_0); + _mm_storeu_si128((__m128i *)(output0 + 1 * 32), tr2_1); + _mm_storeu_si128((__m128i *)(output0 + 2 * 32), tr2_2); + _mm_storeu_si128((__m128i *)(output0 + 3 * 32), tr2_3); + _mm_storeu_si128((__m128i *)(output0 + 4 * 32), tr2_4); + _mm_storeu_si128((__m128i *)(output0 + 5 * 32), tr2_5); + _mm_storeu_si128((__m128i *)(output0 + 6 * 32), tr2_6); + _mm_storeu_si128((__m128i *)(output0 + 7 * 32), tr2_7); + // Process next 8x8 + output0 += 8; + } else { + storeu_output(&tr2_0, (output1 + 0 * 32)); + storeu_output(&tr2_1, (output1 + 1 * 32)); + storeu_output(&tr2_2, (output1 + 2 * 32)); + storeu_output(&tr2_3, (output1 + 3 * 32)); + storeu_output(&tr2_4, (output1 + 4 * 32)); + storeu_output(&tr2_5, (output1 + 5 * 32)); + storeu_output(&tr2_6, (output1 + 6 * 32)); + storeu_output(&tr2_7, (output1 + 7 * 32)); + // Process next 8x8 + output1 += 8; + } + } + } + } + } +} // NOLINT + +#undef ADD_EPI16 +#undef SUB_EPI16 +#undef HIGH_FDCT32x32_2D_C +#undef HIGH_FDCT32x32_2D_ROWS_C diff --git a/third_party/aom/aom_dsp/x86/fwd_txfm_avx2.c b/third_party/aom/aom_dsp/x86/fwd_txfm_avx2.c new file mode 100644 index 0000000000..670f864d07 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/fwd_txfm_avx2.c @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" + +#define FDCT32x32_2D_AVX2 aom_fdct32x32_rd_avx2 +#define FDCT32x32_HIGH_PRECISION 0 +#include "aom_dsp/x86/fwd_dct32x32_impl_avx2.h" +#undef FDCT32x32_2D_AVX2 +#undef FDCT32x32_HIGH_PRECISION + +#define FDCT32x32_2D_AVX2 aom_fdct32x32_avx2 +#define FDCT32x32_HIGH_PRECISION 1 +#include "aom_dsp/x86/fwd_dct32x32_impl_avx2.h" // NOLINT +#undef FDCT32x32_2D_AVX2 +#undef FDCT32x32_HIGH_PRECISION diff --git a/third_party/aom/aom_dsp/x86/fwd_txfm_avx2.h b/third_party/aom/aom_dsp/x86/fwd_txfm_avx2.h new file mode 100644 index 0000000000..d3aceae009 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/fwd_txfm_avx2.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_X86_FWD_TXFM_AVX2_H +#define AOM_DSP_X86_FWD_TXFM_AVX2_H + +#include "./aom_config.h" + +static INLINE void storeu_output_avx2(const __m256i *coeff, tran_low_t *out) { +#if CONFIG_HIGHBITDEPTH + const __m256i zero = _mm256_setzero_si256(); + const __m256i sign = _mm256_cmpgt_epi16(zero, *coeff); + + __m256i x0 = _mm256_unpacklo_epi16(*coeff, sign); + __m256i x1 = _mm256_unpackhi_epi16(*coeff, sign); + + __m256i y0 = _mm256_permute2x128_si256(x0, x1, 0x20); + __m256i y1 = _mm256_permute2x128_si256(x0, x1, 0x31); + + _mm256_storeu_si256((__m256i *)out, y0); + _mm256_storeu_si256((__m256i *)(out + 8), y1); +#else + _mm256_storeu_si256((__m256i *)out, *coeff); +#endif +} + +#endif // AOM_DSP_X86_FWD_TXFM_AVX2_H diff --git a/third_party/aom/aom_dsp/x86/fwd_txfm_impl_sse2.h b/third_party/aom/aom_dsp/x86/fwd_txfm_impl_sse2.h new file mode 100644 index 0000000000..7bb1db70af --- /dev/null +++ b/third_party/aom/aom_dsp/x86/fwd_txfm_impl_sse2.h @@ -0,0 +1,1014 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // SSE2 + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/txfm_common.h" +#include "aom_dsp/x86/fwd_txfm_sse2.h" +#include "aom_dsp/x86/txfm_common_sse2.h" +#include "aom_ports/mem.h" + +// TODO(jingning) The high bit-depth functions need rework for performance. +// After we properly fix the high bit-depth function implementations, this +// file's dependency should be substantially simplified. +#if DCT_HIGH_BIT_DEPTH +#define ADD_EPI16 _mm_adds_epi16 +#define SUB_EPI16 _mm_subs_epi16 + +#else +#define ADD_EPI16 _mm_add_epi16 +#define SUB_EPI16 _mm_sub_epi16 +#endif + +void FDCT4x4_2D(const int16_t *input, tran_low_t *output, int stride) { + // This 2D transform implements 4 vertical 1D transforms followed + // by 4 horizontal 1D transforms. The multiplies and adds are as given + // by Chen, Smith and Fralick ('77). The commands for moving the data + // around have been minimized by hand. + // For the purposes of the comments, the 16 inputs are referred to at i0 + // through iF (in raster order), intermediate variables are a0, b0, c0 + // through f, and correspond to the in-place computations mapped to input + // locations. The outputs, o0 through oF are labeled according to the + // output locations. + + // Constants + // These are the coefficients used for the multiplies. + // In the comments, pN means cos(N pi /64) and mN is -cos(N pi /64), + // where cospi_N_64 = cos(N pi /64) + const __m128i k__cospi_A = + octa_set_epi16(cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64, + cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64); + const __m128i k__cospi_B = + octa_set_epi16(cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64, + cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64); + const __m128i k__cospi_C = + octa_set_epi16(cospi_8_64, cospi_24_64, cospi_8_64, cospi_24_64, + cospi_24_64, -cospi_8_64, cospi_24_64, -cospi_8_64); + const __m128i k__cospi_D = + octa_set_epi16(cospi_24_64, -cospi_8_64, cospi_24_64, -cospi_8_64, + cospi_8_64, cospi_24_64, cospi_8_64, cospi_24_64); + const __m128i k__cospi_E = + octa_set_epi16(cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64, + cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64); + const __m128i k__cospi_F = + octa_set_epi16(cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64, + cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64); + const __m128i k__cospi_G = + octa_set_epi16(cospi_8_64, cospi_24_64, cospi_8_64, cospi_24_64, + -cospi_8_64, -cospi_24_64, -cospi_8_64, -cospi_24_64); + const __m128i k__cospi_H = + octa_set_epi16(cospi_24_64, -cospi_8_64, cospi_24_64, -cospi_8_64, + -cospi_24_64, cospi_8_64, -cospi_24_64, cospi_8_64); + + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + // This second rounding constant saves doing some extra adds at the end + const __m128i k__DCT_CONST_ROUNDING2 = + _mm_set1_epi32(DCT_CONST_ROUNDING + (DCT_CONST_ROUNDING << 1)); + const int DCT_CONST_BITS2 = DCT_CONST_BITS + 2; + const __m128i k__nonzero_bias_a = _mm_setr_epi16(0, 1, 1, 1, 1, 1, 1, 1); + const __m128i k__nonzero_bias_b = _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0); + __m128i in0, in1; +#if DCT_HIGH_BIT_DEPTH + __m128i cmp0, cmp1; + int test, overflow; +#endif + + // Load inputs. + in0 = _mm_loadl_epi64((const __m128i *)(input + 0 * stride)); + in1 = _mm_loadl_epi64((const __m128i *)(input + 1 * stride)); + // in0 = [i0 i1 i2 i3 iC iD iE iF] + // in1 = [i4 i5 i6 i7 i8 i9 iA iB] + in1 = _mm_unpacklo_epi64( + in1, _mm_loadl_epi64((const __m128i *)(input + 2 * stride))); + in0 = _mm_unpacklo_epi64( + in0, _mm_loadl_epi64((const __m128i *)(input + 3 * stride))); +#if DCT_HIGH_BIT_DEPTH + // Check inputs small enough to use optimised code + cmp0 = _mm_xor_si128(_mm_cmpgt_epi16(in0, _mm_set1_epi16(0x3ff)), + _mm_cmplt_epi16(in0, _mm_set1_epi16(0xfc00))); + cmp1 = _mm_xor_si128(_mm_cmpgt_epi16(in1, _mm_set1_epi16(0x3ff)), + _mm_cmplt_epi16(in1, _mm_set1_epi16(0xfc00))); + test = _mm_movemask_epi8(_mm_or_si128(cmp0, cmp1)); + if (test) { + aom_highbd_fdct4x4_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + + // multiply by 16 to give some extra precision + in0 = _mm_slli_epi16(in0, 4); + in1 = _mm_slli_epi16(in1, 4); + // if (i == 0 && input[0]) input[0] += 1; + // add 1 to the upper left pixel if it is non-zero, which helps reduce + // the round-trip error + { + // The mask will only contain whether the first value is zero, all + // other comparison will fail as something shifted by 4 (above << 4) + // can never be equal to one. To increment in the non-zero case, we + // add the mask and one for the first element: + // - if zero, mask = -1, v = v - 1 + 1 = v + // - if non-zero, mask = 0, v = v + 0 + 1 = v + 1 + __m128i mask = _mm_cmpeq_epi16(in0, k__nonzero_bias_a); + in0 = _mm_add_epi16(in0, mask); + in0 = _mm_add_epi16(in0, k__nonzero_bias_b); + } + // There are 4 total stages, alternating between an add/subtract stage + // followed by an multiply-and-add stage. + { + // Stage 1: Add/subtract + + // in0 = [i0 i1 i2 i3 iC iD iE iF] + // in1 = [i4 i5 i6 i7 i8 i9 iA iB] + const __m128i r0 = _mm_unpacklo_epi16(in0, in1); + const __m128i r1 = _mm_unpackhi_epi16(in0, in1); + // r0 = [i0 i4 i1 i5 i2 i6 i3 i7] + // r1 = [iC i8 iD i9 iE iA iF iB] + const __m128i r2 = _mm_shuffle_epi32(r0, 0xB4); + const __m128i r3 = _mm_shuffle_epi32(r1, 0xB4); + // r2 = [i0 i4 i1 i5 i3 i7 i2 i6] + // r3 = [iC i8 iD i9 iF iB iE iA] + + const __m128i t0 = _mm_add_epi16(r2, r3); + const __m128i t1 = _mm_sub_epi16(r2, r3); + // t0 = [a0 a4 a1 a5 a3 a7 a2 a6] + // t1 = [aC a8 aD a9 aF aB aE aA] + + // Stage 2: multiply by constants (which gets us into 32 bits). + // The constants needed here are: + // k__cospi_A = [p16 p16 p16 p16 p16 m16 p16 m16] + // k__cospi_B = [p16 m16 p16 m16 p16 p16 p16 p16] + // k__cospi_C = [p08 p24 p08 p24 p24 m08 p24 m08] + // k__cospi_D = [p24 m08 p24 m08 p08 p24 p08 p24] + const __m128i u0 = _mm_madd_epi16(t0, k__cospi_A); + const __m128i u2 = _mm_madd_epi16(t0, k__cospi_B); + const __m128i u1 = _mm_madd_epi16(t1, k__cospi_C); + const __m128i u3 = _mm_madd_epi16(t1, k__cospi_D); + // Then add and right-shift to get back to 16-bit range + const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); + const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); + const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); + const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); + const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + // w0 = [b0 b1 b7 b6] + // w1 = [b8 b9 bF bE] + // w2 = [b4 b5 b3 b2] + // w3 = [bC bD bB bA] + const __m128i x0 = _mm_packs_epi32(w0, w1); + const __m128i x1 = _mm_packs_epi32(w2, w3); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x2(&x0, &x1); + if (overflow) { + aom_highbd_fdct4x4_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + // x0 = [b0 b1 b7 b6 b8 b9 bF bE] + // x1 = [b4 b5 b3 b2 bC bD bB bA] + in0 = _mm_shuffle_epi32(x0, 0xD8); + in1 = _mm_shuffle_epi32(x1, 0x8D); + // in0 = [b0 b1 b8 b9 b7 b6 bF bE] + // in1 = [b3 b2 bB bA b4 b5 bC bD] + } + { + // vertical DCTs finished. Now we do the horizontal DCTs. + // Stage 3: Add/subtract + + // t0 = [c0 c1 c8 c9 c4 c5 cC cD] + // t1 = [c3 c2 cB cA -c7 -c6 -cF -cE] + const __m128i t0 = ADD_EPI16(in0, in1); + const __m128i t1 = SUB_EPI16(in0, in1); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x2(&t0, &t1); + if (overflow) { + aom_highbd_fdct4x4_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + + // Stage 4: multiply by constants (which gets us into 32 bits). + { + // The constants needed here are: + // k__cospi_E = [p16 p16 p16 p16 p16 p16 p16 p16] + // k__cospi_F = [p16 m16 p16 m16 p16 m16 p16 m16] + // k__cospi_G = [p08 p24 p08 p24 m08 m24 m08 m24] + // k__cospi_H = [p24 m08 p24 m08 m24 p08 m24 p08] + const __m128i u0 = _mm_madd_epi16(t0, k__cospi_E); + const __m128i u1 = _mm_madd_epi16(t0, k__cospi_F); + const __m128i u2 = _mm_madd_epi16(t1, k__cospi_G); + const __m128i u3 = _mm_madd_epi16(t1, k__cospi_H); + // Then add and right-shift to get back to 16-bit range + // but this combines the final right-shift as well to save operations + // This unusual rounding operations is to maintain bit-accurate + // compatibility with the c version of this function which has two + // rounding steps in a row. + const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING2); + const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING2); + const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING2); + const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING2); + const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS2); + const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS2); + const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS2); + const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS2); + // w0 = [o0 o4 o8 oC] + // w1 = [o2 o6 oA oE] + // w2 = [o1 o5 o9 oD] + // w3 = [o3 o7 oB oF] + // remember the o's are numbered according to the correct output location + const __m128i x0 = _mm_packs_epi32(w0, w1); + const __m128i x1 = _mm_packs_epi32(w2, w3); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x2(&x0, &x1); + if (overflow) { + aom_highbd_fdct4x4_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + { + // x0 = [o0 o4 o8 oC o2 o6 oA oE] + // x1 = [o1 o5 o9 oD o3 o7 oB oF] + const __m128i y0 = _mm_unpacklo_epi16(x0, x1); + const __m128i y1 = _mm_unpackhi_epi16(x0, x1); + // y0 = [o0 o1 o4 o5 o8 o9 oC oD] + // y1 = [o2 o3 o6 o7 oA oB oE oF] + in0 = _mm_unpacklo_epi32(y0, y1); + // in0 = [o0 o1 o2 o3 o4 o5 o6 o7] + in1 = _mm_unpackhi_epi32(y0, y1); + // in1 = [o8 o9 oA oB oC oD oE oF] + } + } + } + // Post-condition (v + 1) >> 2 is now incorporated into previous + // add and right-shift commands. Only 2 store instructions needed + // because we are using the fact that 1/3 are stored just after 0/2. + storeu_output(&in0, output + 0 * 4); + storeu_output(&in1, output + 2 * 4); +} + +void FDCT8x8_2D(const int16_t *input, tran_low_t *output, int stride) { + int pass; + // Constants + // When we use them, in one case, they are all the same. In all others + // it's a pair of them that we need to repeat four times. This is done + // by constructing the 32 bit constant corresponding to that pair. + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); +#if DCT_HIGH_BIT_DEPTH + int overflow; +#endif + // Load input + __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride)); + __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride)); + __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride)); + __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride)); + __m128i in4 = _mm_load_si128((const __m128i *)(input + 4 * stride)); + __m128i in5 = _mm_load_si128((const __m128i *)(input + 5 * stride)); + __m128i in6 = _mm_load_si128((const __m128i *)(input + 6 * stride)); + __m128i in7 = _mm_load_si128((const __m128i *)(input + 7 * stride)); + // Pre-condition input (shift by two) + in0 = _mm_slli_epi16(in0, 2); + in1 = _mm_slli_epi16(in1, 2); + in2 = _mm_slli_epi16(in2, 2); + in3 = _mm_slli_epi16(in3, 2); + in4 = _mm_slli_epi16(in4, 2); + in5 = _mm_slli_epi16(in5, 2); + in6 = _mm_slli_epi16(in6, 2); + in7 = _mm_slli_epi16(in7, 2); + + // We do two passes, first the columns, then the rows. The results of the + // first pass are transposed so that the same column code can be reused. The + // results of the second pass are also transposed so that the rows (processed + // as columns) are put back in row positions. + for (pass = 0; pass < 2; pass++) { + // To store results of each pass before the transpose. + __m128i res0, res1, res2, res3, res4, res5, res6, res7; + // Add/subtract + const __m128i q0 = ADD_EPI16(in0, in7); + const __m128i q1 = ADD_EPI16(in1, in6); + const __m128i q2 = ADD_EPI16(in2, in5); + const __m128i q3 = ADD_EPI16(in3, in4); + const __m128i q4 = SUB_EPI16(in3, in4); + const __m128i q5 = SUB_EPI16(in2, in5); + const __m128i q6 = SUB_EPI16(in1, in6); + const __m128i q7 = SUB_EPI16(in0, in7); +#if DCT_HIGH_BIT_DEPTH + if (pass == 1) { + overflow = + check_epi16_overflow_x8(&q0, &q1, &q2, &q3, &q4, &q5, &q6, &q7); + if (overflow) { + aom_highbd_fdct8x8_c(input, output, stride); + return; + } + } +#endif // DCT_HIGH_BIT_DEPTH + // Work on first four results + { + // Add/subtract + const __m128i r0 = ADD_EPI16(q0, q3); + const __m128i r1 = ADD_EPI16(q1, q2); + const __m128i r2 = SUB_EPI16(q1, q2); + const __m128i r3 = SUB_EPI16(q0, q3); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x4(&r0, &r1, &r2, &r3); + if (overflow) { + aom_highbd_fdct8x8_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + // Interleave to do the multiply by constants which gets us into 32bits + { + const __m128i t0 = _mm_unpacklo_epi16(r0, r1); + const __m128i t1 = _mm_unpackhi_epi16(r0, r1); + const __m128i t2 = _mm_unpacklo_epi16(r2, r3); + const __m128i t3 = _mm_unpackhi_epi16(r2, r3); + const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16); + const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16); + const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16); + const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16); + const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08); + const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08); + const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24); + const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24); + // dct_const_round_shift + const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); + const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); + const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); + const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); + const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); + const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); + const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); + const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); + const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS); + const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS); + const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS); + const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS); + // Combine + res0 = _mm_packs_epi32(w0, w1); + res4 = _mm_packs_epi32(w2, w3); + res2 = _mm_packs_epi32(w4, w5); + res6 = _mm_packs_epi32(w6, w7); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x4(&res0, &res4, &res2, &res6); + if (overflow) { + aom_highbd_fdct8x8_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + } + // Work on next four results + { + // Interleave to do the multiply by constants which gets us into 32bits + const __m128i d0 = _mm_unpacklo_epi16(q6, q5); + const __m128i d1 = _mm_unpackhi_epi16(q6, q5); + const __m128i e0 = _mm_madd_epi16(d0, k__cospi_p16_m16); + const __m128i e1 = _mm_madd_epi16(d1, k__cospi_p16_m16); + const __m128i e2 = _mm_madd_epi16(d0, k__cospi_p16_p16); + const __m128i e3 = _mm_madd_epi16(d1, k__cospi_p16_p16); + // dct_const_round_shift + const __m128i f0 = _mm_add_epi32(e0, k__DCT_CONST_ROUNDING); + const __m128i f1 = _mm_add_epi32(e1, k__DCT_CONST_ROUNDING); + const __m128i f2 = _mm_add_epi32(e2, k__DCT_CONST_ROUNDING); + const __m128i f3 = _mm_add_epi32(e3, k__DCT_CONST_ROUNDING); + const __m128i s0 = _mm_srai_epi32(f0, DCT_CONST_BITS); + const __m128i s1 = _mm_srai_epi32(f1, DCT_CONST_BITS); + const __m128i s2 = _mm_srai_epi32(f2, DCT_CONST_BITS); + const __m128i s3 = _mm_srai_epi32(f3, DCT_CONST_BITS); + // Combine + const __m128i r0 = _mm_packs_epi32(s0, s1); + const __m128i r1 = _mm_packs_epi32(s2, s3); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x2(&r0, &r1); + if (overflow) { + aom_highbd_fdct8x8_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + { + // Add/subtract + const __m128i x0 = ADD_EPI16(q4, r0); + const __m128i x1 = SUB_EPI16(q4, r0); + const __m128i x2 = SUB_EPI16(q7, r1); + const __m128i x3 = ADD_EPI16(q7, r1); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x4(&x0, &x1, &x2, &x3); + if (overflow) { + aom_highbd_fdct8x8_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + // Interleave to do the multiply by constants which gets us into 32bits + { + const __m128i t0 = _mm_unpacklo_epi16(x0, x3); + const __m128i t1 = _mm_unpackhi_epi16(x0, x3); + const __m128i t2 = _mm_unpacklo_epi16(x1, x2); + const __m128i t3 = _mm_unpackhi_epi16(x1, x2); + const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04); + const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04); + const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28); + const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28); + const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20); + const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20); + const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12); + const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12); + // dct_const_round_shift + const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); + const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); + const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); + const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); + const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); + const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); + const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); + const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); + const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS); + const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS); + const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS); + const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS); + // Combine + res1 = _mm_packs_epi32(w0, w1); + res7 = _mm_packs_epi32(w2, w3); + res5 = _mm_packs_epi32(w4, w5); + res3 = _mm_packs_epi32(w6, w7); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x4(&res1, &res7, &res5, &res3); + if (overflow) { + aom_highbd_fdct8x8_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + } + } + // Transpose the 8x8. + { + // 00 01 02 03 04 05 06 07 + // 10 11 12 13 14 15 16 17 + // 20 21 22 23 24 25 26 27 + // 30 31 32 33 34 35 36 37 + // 40 41 42 43 44 45 46 47 + // 50 51 52 53 54 55 56 57 + // 60 61 62 63 64 65 66 67 + // 70 71 72 73 74 75 76 77 + const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1); + const __m128i tr0_1 = _mm_unpacklo_epi16(res2, res3); + const __m128i tr0_2 = _mm_unpackhi_epi16(res0, res1); + const __m128i tr0_3 = _mm_unpackhi_epi16(res2, res3); + const __m128i tr0_4 = _mm_unpacklo_epi16(res4, res5); + const __m128i tr0_5 = _mm_unpacklo_epi16(res6, res7); + const __m128i tr0_6 = _mm_unpackhi_epi16(res4, res5); + const __m128i tr0_7 = _mm_unpackhi_epi16(res6, res7); + // 00 10 01 11 02 12 03 13 + // 20 30 21 31 22 32 23 33 + // 04 14 05 15 06 16 07 17 + // 24 34 25 35 26 36 27 37 + // 40 50 41 51 42 52 43 53 + // 60 70 61 71 62 72 63 73 + // 54 54 55 55 56 56 57 57 + // 64 74 65 75 66 76 67 77 + const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); + const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3); + const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); + const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3); + const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); + const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); + const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); + const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); + // 00 10 20 30 01 11 21 31 + // 40 50 60 70 41 51 61 71 + // 02 12 22 32 03 13 23 33 + // 42 52 62 72 43 53 63 73 + // 04 14 24 34 05 15 21 36 + // 44 54 64 74 45 55 61 76 + // 06 16 26 36 07 17 27 37 + // 46 56 66 76 47 57 67 77 + in0 = _mm_unpacklo_epi64(tr1_0, tr1_4); + in1 = _mm_unpackhi_epi64(tr1_0, tr1_4); + in2 = _mm_unpacklo_epi64(tr1_2, tr1_6); + in3 = _mm_unpackhi_epi64(tr1_2, tr1_6); + in4 = _mm_unpacklo_epi64(tr1_1, tr1_5); + in5 = _mm_unpackhi_epi64(tr1_1, tr1_5); + in6 = _mm_unpacklo_epi64(tr1_3, tr1_7); + in7 = _mm_unpackhi_epi64(tr1_3, tr1_7); + // 00 10 20 30 40 50 60 70 + // 01 11 21 31 41 51 61 71 + // 02 12 22 32 42 52 62 72 + // 03 13 23 33 43 53 63 73 + // 04 14 24 34 44 54 64 74 + // 05 15 25 35 45 55 65 75 + // 06 16 26 36 46 56 66 76 + // 07 17 27 37 47 57 67 77 + } + } + // Post-condition output and store it + { + // Post-condition (division by two) + // division of two 16 bits signed numbers using shifts + // n / 2 = (n - (n >> 15)) >> 1 + const __m128i sign_in0 = _mm_srai_epi16(in0, 15); + const __m128i sign_in1 = _mm_srai_epi16(in1, 15); + const __m128i sign_in2 = _mm_srai_epi16(in2, 15); + const __m128i sign_in3 = _mm_srai_epi16(in3, 15); + const __m128i sign_in4 = _mm_srai_epi16(in4, 15); + const __m128i sign_in5 = _mm_srai_epi16(in5, 15); + const __m128i sign_in6 = _mm_srai_epi16(in6, 15); + const __m128i sign_in7 = _mm_srai_epi16(in7, 15); + in0 = _mm_sub_epi16(in0, sign_in0); + in1 = _mm_sub_epi16(in1, sign_in1); + in2 = _mm_sub_epi16(in2, sign_in2); + in3 = _mm_sub_epi16(in3, sign_in3); + in4 = _mm_sub_epi16(in4, sign_in4); + in5 = _mm_sub_epi16(in5, sign_in5); + in6 = _mm_sub_epi16(in6, sign_in6); + in7 = _mm_sub_epi16(in7, sign_in7); + in0 = _mm_srai_epi16(in0, 1); + in1 = _mm_srai_epi16(in1, 1); + in2 = _mm_srai_epi16(in2, 1); + in3 = _mm_srai_epi16(in3, 1); + in4 = _mm_srai_epi16(in4, 1); + in5 = _mm_srai_epi16(in5, 1); + in6 = _mm_srai_epi16(in6, 1); + in7 = _mm_srai_epi16(in7, 1); + // store results + store_output(&in0, (output + 0 * 8)); + store_output(&in1, (output + 1 * 8)); + store_output(&in2, (output + 2 * 8)); + store_output(&in3, (output + 3 * 8)); + store_output(&in4, (output + 4 * 8)); + store_output(&in5, (output + 5 * 8)); + store_output(&in6, (output + 6 * 8)); + store_output(&in7, (output + 7 * 8)); + } +} + +void FDCT16x16_2D(const int16_t *input, tran_low_t *output, int stride) { + // The 2D transform is done with two passes which are actually pretty + // similar. In the first one, we transform the columns and transpose + // the results. In the second one, we transform the rows. To achieve that, + // as the first pass results are transposed, we transpose the columns (that + // is the transposed rows) and transpose the results (so that it goes back + // in normal/row positions). + int pass; + // We need an intermediate buffer between passes. + DECLARE_ALIGNED(16, int16_t, intermediate[256]); + const int16_t *in = input; + int16_t *out0 = intermediate; + tran_low_t *out1 = output; + // Constants + // When we use them, in one case, they are all the same. In all others + // it's a pair of them that we need to repeat four times. This is done + // by constructing the 32 bit constant corresponding to that pair. + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i k__cospi_p08_m24 = pair_set_epi16(cospi_8_64, -cospi_24_64); + const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i k__cospi_p30_p02 = pair_set_epi16(cospi_30_64, cospi_2_64); + const __m128i k__cospi_p14_p18 = pair_set_epi16(cospi_14_64, cospi_18_64); + const __m128i k__cospi_m02_p30 = pair_set_epi16(-cospi_2_64, cospi_30_64); + const __m128i k__cospi_m18_p14 = pair_set_epi16(-cospi_18_64, cospi_14_64); + const __m128i k__cospi_p22_p10 = pair_set_epi16(cospi_22_64, cospi_10_64); + const __m128i k__cospi_p06_p26 = pair_set_epi16(cospi_6_64, cospi_26_64); + const __m128i k__cospi_m10_p22 = pair_set_epi16(-cospi_10_64, cospi_22_64); + const __m128i k__cospi_m26_p06 = pair_set_epi16(-cospi_26_64, cospi_6_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i kOne = _mm_set1_epi16(1); + // Do the two transform/transpose passes + for (pass = 0; pass < 2; ++pass) { + // We process eight columns (transposed rows in second pass) at a time. + int column_start; +#if DCT_HIGH_BIT_DEPTH + int overflow; +#endif + for (column_start = 0; column_start < 16; column_start += 8) { + __m128i in00, in01, in02, in03, in04, in05, in06, in07; + __m128i in08, in09, in10, in11, in12, in13, in14, in15; + __m128i input0, input1, input2, input3, input4, input5, input6, input7; + __m128i step1_0, step1_1, step1_2, step1_3; + __m128i step1_4, step1_5, step1_6, step1_7; + __m128i step2_1, step2_2, step2_3, step2_4, step2_5, step2_6; + __m128i step3_0, step3_1, step3_2, step3_3; + __m128i step3_4, step3_5, step3_6, step3_7; + __m128i res00, res01, res02, res03, res04, res05, res06, res07; + __m128i res08, res09, res10, res11, res12, res13, res14, res15; + // Load and pre-condition input. + if (0 == pass) { + in00 = _mm_load_si128((const __m128i *)(in + 0 * stride)); + in01 = _mm_load_si128((const __m128i *)(in + 1 * stride)); + in02 = _mm_load_si128((const __m128i *)(in + 2 * stride)); + in03 = _mm_load_si128((const __m128i *)(in + 3 * stride)); + in04 = _mm_load_si128((const __m128i *)(in + 4 * stride)); + in05 = _mm_load_si128((const __m128i *)(in + 5 * stride)); + in06 = _mm_load_si128((const __m128i *)(in + 6 * stride)); + in07 = _mm_load_si128((const __m128i *)(in + 7 * stride)); + in08 = _mm_load_si128((const __m128i *)(in + 8 * stride)); + in09 = _mm_load_si128((const __m128i *)(in + 9 * stride)); + in10 = _mm_load_si128((const __m128i *)(in + 10 * stride)); + in11 = _mm_load_si128((const __m128i *)(in + 11 * stride)); + in12 = _mm_load_si128((const __m128i *)(in + 12 * stride)); + in13 = _mm_load_si128((const __m128i *)(in + 13 * stride)); + in14 = _mm_load_si128((const __m128i *)(in + 14 * stride)); + in15 = _mm_load_si128((const __m128i *)(in + 15 * stride)); + // x = x << 2 + in00 = _mm_slli_epi16(in00, 2); + in01 = _mm_slli_epi16(in01, 2); + in02 = _mm_slli_epi16(in02, 2); + in03 = _mm_slli_epi16(in03, 2); + in04 = _mm_slli_epi16(in04, 2); + in05 = _mm_slli_epi16(in05, 2); + in06 = _mm_slli_epi16(in06, 2); + in07 = _mm_slli_epi16(in07, 2); + in08 = _mm_slli_epi16(in08, 2); + in09 = _mm_slli_epi16(in09, 2); + in10 = _mm_slli_epi16(in10, 2); + in11 = _mm_slli_epi16(in11, 2); + in12 = _mm_slli_epi16(in12, 2); + in13 = _mm_slli_epi16(in13, 2); + in14 = _mm_slli_epi16(in14, 2); + in15 = _mm_slli_epi16(in15, 2); + } else { + in00 = _mm_load_si128((const __m128i *)(in + 0 * 16)); + in01 = _mm_load_si128((const __m128i *)(in + 1 * 16)); + in02 = _mm_load_si128((const __m128i *)(in + 2 * 16)); + in03 = _mm_load_si128((const __m128i *)(in + 3 * 16)); + in04 = _mm_load_si128((const __m128i *)(in + 4 * 16)); + in05 = _mm_load_si128((const __m128i *)(in + 5 * 16)); + in06 = _mm_load_si128((const __m128i *)(in + 6 * 16)); + in07 = _mm_load_si128((const __m128i *)(in + 7 * 16)); + in08 = _mm_load_si128((const __m128i *)(in + 8 * 16)); + in09 = _mm_load_si128((const __m128i *)(in + 9 * 16)); + in10 = _mm_load_si128((const __m128i *)(in + 10 * 16)); + in11 = _mm_load_si128((const __m128i *)(in + 11 * 16)); + in12 = _mm_load_si128((const __m128i *)(in + 12 * 16)); + in13 = _mm_load_si128((const __m128i *)(in + 13 * 16)); + in14 = _mm_load_si128((const __m128i *)(in + 14 * 16)); + in15 = _mm_load_si128((const __m128i *)(in + 15 * 16)); + // x = (x + 1) >> 2 + in00 = _mm_add_epi16(in00, kOne); + in01 = _mm_add_epi16(in01, kOne); + in02 = _mm_add_epi16(in02, kOne); + in03 = _mm_add_epi16(in03, kOne); + in04 = _mm_add_epi16(in04, kOne); + in05 = _mm_add_epi16(in05, kOne); + in06 = _mm_add_epi16(in06, kOne); + in07 = _mm_add_epi16(in07, kOne); + in08 = _mm_add_epi16(in08, kOne); + in09 = _mm_add_epi16(in09, kOne); + in10 = _mm_add_epi16(in10, kOne); + in11 = _mm_add_epi16(in11, kOne); + in12 = _mm_add_epi16(in12, kOne); + in13 = _mm_add_epi16(in13, kOne); + in14 = _mm_add_epi16(in14, kOne); + in15 = _mm_add_epi16(in15, kOne); + in00 = _mm_srai_epi16(in00, 2); + in01 = _mm_srai_epi16(in01, 2); + in02 = _mm_srai_epi16(in02, 2); + in03 = _mm_srai_epi16(in03, 2); + in04 = _mm_srai_epi16(in04, 2); + in05 = _mm_srai_epi16(in05, 2); + in06 = _mm_srai_epi16(in06, 2); + in07 = _mm_srai_epi16(in07, 2); + in08 = _mm_srai_epi16(in08, 2); + in09 = _mm_srai_epi16(in09, 2); + in10 = _mm_srai_epi16(in10, 2); + in11 = _mm_srai_epi16(in11, 2); + in12 = _mm_srai_epi16(in12, 2); + in13 = _mm_srai_epi16(in13, 2); + in14 = _mm_srai_epi16(in14, 2); + in15 = _mm_srai_epi16(in15, 2); + } + in += 8; + // Calculate input for the first 8 results. + { + input0 = ADD_EPI16(in00, in15); + input1 = ADD_EPI16(in01, in14); + input2 = ADD_EPI16(in02, in13); + input3 = ADD_EPI16(in03, in12); + input4 = ADD_EPI16(in04, in11); + input5 = ADD_EPI16(in05, in10); + input6 = ADD_EPI16(in06, in09); + input7 = ADD_EPI16(in07, in08); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x8(&input0, &input1, &input2, &input3, + &input4, &input5, &input6, &input7); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + // Calculate input for the next 8 results. + { + step1_0 = SUB_EPI16(in07, in08); + step1_1 = SUB_EPI16(in06, in09); + step1_2 = SUB_EPI16(in05, in10); + step1_3 = SUB_EPI16(in04, in11); + step1_4 = SUB_EPI16(in03, in12); + step1_5 = SUB_EPI16(in02, in13); + step1_6 = SUB_EPI16(in01, in14); + step1_7 = SUB_EPI16(in00, in15); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x8(&step1_0, &step1_1, &step1_2, &step1_3, + &step1_4, &step1_5, &step1_6, &step1_7); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + // Work on the first eight values; fdct8(input, even_results); + { + // Add/subtract + const __m128i q0 = ADD_EPI16(input0, input7); + const __m128i q1 = ADD_EPI16(input1, input6); + const __m128i q2 = ADD_EPI16(input2, input5); + const __m128i q3 = ADD_EPI16(input3, input4); + const __m128i q4 = SUB_EPI16(input3, input4); + const __m128i q5 = SUB_EPI16(input2, input5); + const __m128i q6 = SUB_EPI16(input1, input6); + const __m128i q7 = SUB_EPI16(input0, input7); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x8(&q0, &q1, &q2, &q3, &q4, &q5, &q6, &q7); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + // Work on first four results + { + // Add/subtract + const __m128i r0 = ADD_EPI16(q0, q3); + const __m128i r1 = ADD_EPI16(q1, q2); + const __m128i r2 = SUB_EPI16(q1, q2); + const __m128i r3 = SUB_EPI16(q0, q3); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x4(&r0, &r1, &r2, &r3); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + // Interleave to do the multiply by constants which gets us + // into 32 bits. + { + const __m128i t0 = _mm_unpacklo_epi16(r0, r1); + const __m128i t1 = _mm_unpackhi_epi16(r0, r1); + const __m128i t2 = _mm_unpacklo_epi16(r2, r3); + const __m128i t3 = _mm_unpackhi_epi16(r2, r3); + res00 = mult_round_shift(&t0, &t1, &k__cospi_p16_p16, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + res08 = mult_round_shift(&t0, &t1, &k__cospi_p16_m16, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + res04 = mult_round_shift(&t2, &t3, &k__cospi_p24_p08, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + res12 = mult_round_shift(&t2, &t3, &k__cospi_m08_p24, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x4(&res00, &res08, &res04, &res12); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + } + // Work on next four results + { + // Interleave to do the multiply by constants which gets us + // into 32 bits. + const __m128i d0 = _mm_unpacklo_epi16(q6, q5); + const __m128i d1 = _mm_unpackhi_epi16(q6, q5); + const __m128i r0 = + mult_round_shift(&d0, &d1, &k__cospi_p16_m16, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + const __m128i r1 = + mult_round_shift(&d0, &d1, &k__cospi_p16_p16, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x2(&r0, &r1); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + { + // Add/subtract + const __m128i x0 = ADD_EPI16(q4, r0); + const __m128i x1 = SUB_EPI16(q4, r0); + const __m128i x2 = SUB_EPI16(q7, r1); + const __m128i x3 = ADD_EPI16(q7, r1); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x4(&x0, &x1, &x2, &x3); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + // Interleave to do the multiply by constants which gets us + // into 32 bits. + { + const __m128i t0 = _mm_unpacklo_epi16(x0, x3); + const __m128i t1 = _mm_unpackhi_epi16(x0, x3); + const __m128i t2 = _mm_unpacklo_epi16(x1, x2); + const __m128i t3 = _mm_unpackhi_epi16(x1, x2); + res02 = mult_round_shift(&t0, &t1, &k__cospi_p28_p04, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + res14 = mult_round_shift(&t0, &t1, &k__cospi_m04_p28, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + res10 = mult_round_shift(&t2, &t3, &k__cospi_p12_p20, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + res06 = mult_round_shift(&t2, &t3, &k__cospi_m20_p12, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x4(&res02, &res14, &res10, &res06); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + } + } + } + // Work on the next eight values; step1 -> odd_results + { + // step 2 + { + const __m128i t0 = _mm_unpacklo_epi16(step1_5, step1_2); + const __m128i t1 = _mm_unpackhi_epi16(step1_5, step1_2); + const __m128i t2 = _mm_unpacklo_epi16(step1_4, step1_3); + const __m128i t3 = _mm_unpackhi_epi16(step1_4, step1_3); + step2_2 = mult_round_shift(&t0, &t1, &k__cospi_p16_m16, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + step2_3 = mult_round_shift(&t2, &t3, &k__cospi_p16_m16, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + step2_5 = mult_round_shift(&t0, &t1, &k__cospi_p16_p16, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + step2_4 = mult_round_shift(&t2, &t3, &k__cospi_p16_p16, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x4(&step2_2, &step2_3, &step2_5, &step2_4); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + // step 3 + { + step3_0 = ADD_EPI16(step1_0, step2_3); + step3_1 = ADD_EPI16(step1_1, step2_2); + step3_2 = SUB_EPI16(step1_1, step2_2); + step3_3 = SUB_EPI16(step1_0, step2_3); + step3_4 = SUB_EPI16(step1_7, step2_4); + step3_5 = SUB_EPI16(step1_6, step2_5); + step3_6 = ADD_EPI16(step1_6, step2_5); + step3_7 = ADD_EPI16(step1_7, step2_4); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x8(&step3_0, &step3_1, &step3_2, &step3_3, + &step3_4, &step3_5, &step3_6, &step3_7); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + // step 4 + { + const __m128i t0 = _mm_unpacklo_epi16(step3_1, step3_6); + const __m128i t1 = _mm_unpackhi_epi16(step3_1, step3_6); + const __m128i t2 = _mm_unpacklo_epi16(step3_2, step3_5); + const __m128i t3 = _mm_unpackhi_epi16(step3_2, step3_5); + step2_1 = mult_round_shift(&t0, &t1, &k__cospi_m08_p24, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + step2_2 = mult_round_shift(&t2, &t3, &k__cospi_p24_p08, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + step2_6 = mult_round_shift(&t0, &t1, &k__cospi_p24_p08, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + step2_5 = mult_round_shift(&t2, &t3, &k__cospi_p08_m24, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x4(&step2_1, &step2_2, &step2_6, &step2_5); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + // step 5 + { + step1_0 = ADD_EPI16(step3_0, step2_1); + step1_1 = SUB_EPI16(step3_0, step2_1); + step1_2 = ADD_EPI16(step3_3, step2_2); + step1_3 = SUB_EPI16(step3_3, step2_2); + step1_4 = SUB_EPI16(step3_4, step2_5); + step1_5 = ADD_EPI16(step3_4, step2_5); + step1_6 = SUB_EPI16(step3_7, step2_6); + step1_7 = ADD_EPI16(step3_7, step2_6); +#if DCT_HIGH_BIT_DEPTH + overflow = + check_epi16_overflow_x8(&step1_0, &step1_1, &step1_2, &step1_3, + &step1_4, &step1_5, &step1_6, &step1_7); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + // step 6 + { + const __m128i t0 = _mm_unpacklo_epi16(step1_0, step1_7); + const __m128i t1 = _mm_unpackhi_epi16(step1_0, step1_7); + const __m128i t2 = _mm_unpacklo_epi16(step1_1, step1_6); + const __m128i t3 = _mm_unpackhi_epi16(step1_1, step1_6); + res01 = mult_round_shift(&t0, &t1, &k__cospi_p30_p02, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + res09 = mult_round_shift(&t2, &t3, &k__cospi_p14_p18, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + res15 = mult_round_shift(&t0, &t1, &k__cospi_m02_p30, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + res07 = mult_round_shift(&t2, &t3, &k__cospi_m18_p14, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x4(&res01, &res09, &res15, &res07); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + { + const __m128i t0 = _mm_unpacklo_epi16(step1_2, step1_5); + const __m128i t1 = _mm_unpackhi_epi16(step1_2, step1_5); + const __m128i t2 = _mm_unpacklo_epi16(step1_3, step1_4); + const __m128i t3 = _mm_unpackhi_epi16(step1_3, step1_4); + res05 = mult_round_shift(&t0, &t1, &k__cospi_p22_p10, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + res13 = mult_round_shift(&t2, &t3, &k__cospi_p06_p26, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + res11 = mult_round_shift(&t0, &t1, &k__cospi_m10_p22, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); + res03 = mult_round_shift(&t2, &t3, &k__cospi_m26_p06, + &k__DCT_CONST_ROUNDING, DCT_CONST_BITS); +#if DCT_HIGH_BIT_DEPTH + overflow = check_epi16_overflow_x4(&res05, &res13, &res11, &res03); + if (overflow) { + aom_highbd_fdct16x16_c(input, output, stride); + return; + } +#endif // DCT_HIGH_BIT_DEPTH + } + } + // Transpose the results, do it as two 8x8 transposes. + transpose_and_output8x8(&res00, &res01, &res02, &res03, &res04, &res05, + &res06, &res07, pass, out0, out1); + transpose_and_output8x8(&res08, &res09, &res10, &res11, &res12, &res13, + &res14, &res15, pass, out0 + 8, out1 + 8); + if (pass == 0) { + out0 += 8 * 16; + } else { + out1 += 8 * 16; + } + } + // Setup in/out for next pass. + in = intermediate; + } +} + +#undef ADD_EPI16 +#undef SUB_EPI16 diff --git a/third_party/aom/aom_dsp/x86/fwd_txfm_sse2.c b/third_party/aom/aom_dsp/x86/fwd_txfm_sse2.c new file mode 100644 index 0000000000..a337e618de --- /dev/null +++ b/third_party/aom/aom_dsp/x86/fwd_txfm_sse2.c @@ -0,0 +1,273 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // SSE2 + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/x86/fwd_txfm_sse2.h" + +void aom_fdct4x4_1_sse2(const int16_t *input, tran_low_t *output, int stride) { + __m128i in0, in1; + __m128i tmp; + const __m128i zero = _mm_setzero_si128(); + in0 = _mm_loadl_epi64((const __m128i *)(input + 0 * stride)); + in1 = _mm_loadl_epi64((const __m128i *)(input + 1 * stride)); + in1 = _mm_unpacklo_epi64( + in1, _mm_loadl_epi64((const __m128i *)(input + 2 * stride))); + in0 = _mm_unpacklo_epi64( + in0, _mm_loadl_epi64((const __m128i *)(input + 3 * stride))); + + tmp = _mm_add_epi16(in0, in1); + in0 = _mm_unpacklo_epi16(zero, tmp); + in1 = _mm_unpackhi_epi16(zero, tmp); + in0 = _mm_srai_epi32(in0, 16); + in1 = _mm_srai_epi32(in1, 16); + + tmp = _mm_add_epi32(in0, in1); + in0 = _mm_unpacklo_epi32(tmp, zero); + in1 = _mm_unpackhi_epi32(tmp, zero); + + tmp = _mm_add_epi32(in0, in1); + in0 = _mm_srli_si128(tmp, 8); + + in1 = _mm_add_epi32(tmp, in0); + in0 = _mm_slli_epi32(in1, 1); + output[0] = (tran_low_t)_mm_cvtsi128_si32(in0); +} + +void aom_fdct8x8_1_sse2(const int16_t *input, tran_low_t *output, int stride) { + __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride)); + __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride)); + __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride)); + __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride)); + __m128i u0, u1, sum; + + u0 = _mm_add_epi16(in0, in1); + u1 = _mm_add_epi16(in2, in3); + + in0 = _mm_load_si128((const __m128i *)(input + 4 * stride)); + in1 = _mm_load_si128((const __m128i *)(input + 5 * stride)); + in2 = _mm_load_si128((const __m128i *)(input + 6 * stride)); + in3 = _mm_load_si128((const __m128i *)(input + 7 * stride)); + + sum = _mm_add_epi16(u0, u1); + + in0 = _mm_add_epi16(in0, in1); + in2 = _mm_add_epi16(in2, in3); + sum = _mm_add_epi16(sum, in0); + + u0 = _mm_setzero_si128(); + sum = _mm_add_epi16(sum, in2); + + in0 = _mm_unpacklo_epi16(u0, sum); + in1 = _mm_unpackhi_epi16(u0, sum); + in0 = _mm_srai_epi32(in0, 16); + in1 = _mm_srai_epi32(in1, 16); + + sum = _mm_add_epi32(in0, in1); + in0 = _mm_unpacklo_epi32(sum, u0); + in1 = _mm_unpackhi_epi32(sum, u0); + + sum = _mm_add_epi32(in0, in1); + in0 = _mm_srli_si128(sum, 8); + + in1 = _mm_add_epi32(sum, in0); + output[0] = (tran_low_t)_mm_cvtsi128_si32(in1); +} + +void aom_fdct16x16_1_sse2(const int16_t *input, tran_low_t *output, + int stride) { + __m128i in0, in1, in2, in3; + __m128i u0, u1; + __m128i sum = _mm_setzero_si128(); + int i; + + for (i = 0; i < 2; ++i) { + in0 = _mm_load_si128((const __m128i *)(input + 0 * stride + 0)); + in1 = _mm_load_si128((const __m128i *)(input + 0 * stride + 8)); + in2 = _mm_load_si128((const __m128i *)(input + 1 * stride + 0)); + in3 = _mm_load_si128((const __m128i *)(input + 1 * stride + 8)); + + u0 = _mm_add_epi16(in0, in1); + u1 = _mm_add_epi16(in2, in3); + sum = _mm_add_epi16(sum, u0); + + in0 = _mm_load_si128((const __m128i *)(input + 2 * stride + 0)); + in1 = _mm_load_si128((const __m128i *)(input + 2 * stride + 8)); + in2 = _mm_load_si128((const __m128i *)(input + 3 * stride + 0)); + in3 = _mm_load_si128((const __m128i *)(input + 3 * stride + 8)); + + sum = _mm_add_epi16(sum, u1); + u0 = _mm_add_epi16(in0, in1); + u1 = _mm_add_epi16(in2, in3); + sum = _mm_add_epi16(sum, u0); + + in0 = _mm_load_si128((const __m128i *)(input + 4 * stride + 0)); + in1 = _mm_load_si128((const __m128i *)(input + 4 * stride + 8)); + in2 = _mm_load_si128((const __m128i *)(input + 5 * stride + 0)); + in3 = _mm_load_si128((const __m128i *)(input + 5 * stride + 8)); + + sum = _mm_add_epi16(sum, u1); + u0 = _mm_add_epi16(in0, in1); + u1 = _mm_add_epi16(in2, in3); + sum = _mm_add_epi16(sum, u0); + + in0 = _mm_load_si128((const __m128i *)(input + 6 * stride + 0)); + in1 = _mm_load_si128((const __m128i *)(input + 6 * stride + 8)); + in2 = _mm_load_si128((const __m128i *)(input + 7 * stride + 0)); + in3 = _mm_load_si128((const __m128i *)(input + 7 * stride + 8)); + + sum = _mm_add_epi16(sum, u1); + u0 = _mm_add_epi16(in0, in1); + u1 = _mm_add_epi16(in2, in3); + sum = _mm_add_epi16(sum, u0); + + sum = _mm_add_epi16(sum, u1); + input += 8 * stride; + } + + u0 = _mm_setzero_si128(); + in0 = _mm_unpacklo_epi16(u0, sum); + in1 = _mm_unpackhi_epi16(u0, sum); + in0 = _mm_srai_epi32(in0, 16); + in1 = _mm_srai_epi32(in1, 16); + + sum = _mm_add_epi32(in0, in1); + in0 = _mm_unpacklo_epi32(sum, u0); + in1 = _mm_unpackhi_epi32(sum, u0); + + sum = _mm_add_epi32(in0, in1); + in0 = _mm_srli_si128(sum, 8); + + in1 = _mm_add_epi32(sum, in0); + in1 = _mm_srai_epi32(in1, 1); + output[0] = (tran_low_t)_mm_cvtsi128_si32(in1); +} + +void aom_fdct32x32_1_sse2(const int16_t *input, tran_low_t *output, + int stride) { + __m128i in0, in1, in2, in3; + __m128i u0, u1; + __m128i sum = _mm_setzero_si128(); + int i; + + for (i = 0; i < 8; ++i) { + in0 = _mm_load_si128((const __m128i *)(input + 0)); + in1 = _mm_load_si128((const __m128i *)(input + 8)); + in2 = _mm_load_si128((const __m128i *)(input + 16)); + in3 = _mm_load_si128((const __m128i *)(input + 24)); + + input += stride; + u0 = _mm_add_epi16(in0, in1); + u1 = _mm_add_epi16(in2, in3); + sum = _mm_add_epi16(sum, u0); + + in0 = _mm_load_si128((const __m128i *)(input + 0)); + in1 = _mm_load_si128((const __m128i *)(input + 8)); + in2 = _mm_load_si128((const __m128i *)(input + 16)); + in3 = _mm_load_si128((const __m128i *)(input + 24)); + + input += stride; + sum = _mm_add_epi16(sum, u1); + u0 = _mm_add_epi16(in0, in1); + u1 = _mm_add_epi16(in2, in3); + sum = _mm_add_epi16(sum, u0); + + in0 = _mm_load_si128((const __m128i *)(input + 0)); + in1 = _mm_load_si128((const __m128i *)(input + 8)); + in2 = _mm_load_si128((const __m128i *)(input + 16)); + in3 = _mm_load_si128((const __m128i *)(input + 24)); + + input += stride; + sum = _mm_add_epi16(sum, u1); + u0 = _mm_add_epi16(in0, in1); + u1 = _mm_add_epi16(in2, in3); + sum = _mm_add_epi16(sum, u0); + + in0 = _mm_load_si128((const __m128i *)(input + 0)); + in1 = _mm_load_si128((const __m128i *)(input + 8)); + in2 = _mm_load_si128((const __m128i *)(input + 16)); + in3 = _mm_load_si128((const __m128i *)(input + 24)); + + input += stride; + sum = _mm_add_epi16(sum, u1); + u0 = _mm_add_epi16(in0, in1); + u1 = _mm_add_epi16(in2, in3); + sum = _mm_add_epi16(sum, u0); + + sum = _mm_add_epi16(sum, u1); + } + + u0 = _mm_setzero_si128(); + in0 = _mm_unpacklo_epi16(u0, sum); + in1 = _mm_unpackhi_epi16(u0, sum); + in0 = _mm_srai_epi32(in0, 16); + in1 = _mm_srai_epi32(in1, 16); + + sum = _mm_add_epi32(in0, in1); + in0 = _mm_unpacklo_epi32(sum, u0); + in1 = _mm_unpackhi_epi32(sum, u0); + + sum = _mm_add_epi32(in0, in1); + in0 = _mm_srli_si128(sum, 8); + + in1 = _mm_add_epi32(sum, in0); + in1 = _mm_srai_epi32(in1, 3); + output[0] = (tran_low_t)_mm_cvtsi128_si32(in1); +} + +#define DCT_HIGH_BIT_DEPTH 0 +#define FDCT4x4_2D aom_fdct4x4_sse2 +#define FDCT8x8_2D aom_fdct8x8_sse2 +#define FDCT16x16_2D aom_fdct16x16_sse2 +#include "aom_dsp/x86/fwd_txfm_impl_sse2.h" +#undef FDCT4x4_2D +#undef FDCT8x8_2D +#undef FDCT16x16_2D + +#define FDCT32x32_2D aom_fdct32x32_rd_sse2 +#define FDCT32x32_HIGH_PRECISION 0 +#include "aom_dsp/x86/fwd_dct32x32_impl_sse2.h" +#undef FDCT32x32_2D +#undef FDCT32x32_HIGH_PRECISION + +#define FDCT32x32_2D aom_fdct32x32_sse2 +#define FDCT32x32_HIGH_PRECISION 1 +#include "aom_dsp/x86/fwd_dct32x32_impl_sse2.h" // NOLINT +#undef FDCT32x32_2D +#undef FDCT32x32_HIGH_PRECISION +#undef DCT_HIGH_BIT_DEPTH + +#if CONFIG_HIGHBITDEPTH +#define DCT_HIGH_BIT_DEPTH 1 +#define FDCT4x4_2D aom_highbd_fdct4x4_sse2 +#define FDCT8x8_2D aom_highbd_fdct8x8_sse2 +#define FDCT16x16_2D aom_highbd_fdct16x16_sse2 +#include "aom_dsp/x86/fwd_txfm_impl_sse2.h" // NOLINT +#undef FDCT4x4_2D +#undef FDCT8x8_2D +#undef FDCT16x16_2D + +#define FDCT32x32_2D aom_highbd_fdct32x32_rd_sse2 +#define FDCT32x32_HIGH_PRECISION 0 +#include "aom_dsp/x86/fwd_dct32x32_impl_sse2.h" // NOLINT +#undef FDCT32x32_2D +#undef FDCT32x32_HIGH_PRECISION + +#define FDCT32x32_2D aom_highbd_fdct32x32_sse2 +#define FDCT32x32_HIGH_PRECISION 1 +#include "aom_dsp/x86/fwd_dct32x32_impl_sse2.h" // NOLINT +#undef FDCT32x32_2D +#undef FDCT32x32_HIGH_PRECISION +#undef DCT_HIGH_BIT_DEPTH +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/x86/fwd_txfm_sse2.h b/third_party/aom/aom_dsp/x86/fwd_txfm_sse2.h new file mode 100644 index 0000000000..26b2db2e02 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/fwd_txfm_sse2.h @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_X86_FWD_TXFM_SSE2_H_ +#define AOM_DSP_X86_FWD_TXFM_SSE2_H_ + +#include "aom_dsp/x86/txfm_common_intrin.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define pair_set_epi32(a, b) \ + _mm_set_epi32((int)(b), (int)(a), (int)(b), (int)(a)) + +static INLINE __m128i k_madd_epi32(__m128i a, __m128i b) { + __m128i buf0, buf1; + buf0 = _mm_mul_epu32(a, b); + a = _mm_srli_epi64(a, 32); + b = _mm_srli_epi64(b, 32); + buf1 = _mm_mul_epu32(a, b); + return _mm_add_epi64(buf0, buf1); +} + +static INLINE __m128i k_packs_epi64(__m128i a, __m128i b) { + __m128i buf0 = _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 2, 0)); + __m128i buf1 = _mm_shuffle_epi32(b, _MM_SHUFFLE(0, 0, 2, 0)); + return _mm_unpacklo_epi64(buf0, buf1); +} + +static INLINE int check_epi16_overflow_x2(const __m128i *preg0, + const __m128i *preg1) { + const __m128i max_overflow = _mm_set1_epi16(0x7fff); + const __m128i min_overflow = _mm_set1_epi16(0x8000); + __m128i cmp0 = _mm_or_si128(_mm_cmpeq_epi16(*preg0, max_overflow), + _mm_cmpeq_epi16(*preg0, min_overflow)); + __m128i cmp1 = _mm_or_si128(_mm_cmpeq_epi16(*preg1, max_overflow), + _mm_cmpeq_epi16(*preg1, min_overflow)); + cmp0 = _mm_or_si128(cmp0, cmp1); + return _mm_movemask_epi8(cmp0); +} + +static INLINE int check_epi16_overflow_x4(const __m128i *preg0, + const __m128i *preg1, + const __m128i *preg2, + const __m128i *preg3) { + const __m128i max_overflow = _mm_set1_epi16(0x7fff); + const __m128i min_overflow = _mm_set1_epi16(0x8000); + __m128i cmp0 = _mm_or_si128(_mm_cmpeq_epi16(*preg0, max_overflow), + _mm_cmpeq_epi16(*preg0, min_overflow)); + __m128i cmp1 = _mm_or_si128(_mm_cmpeq_epi16(*preg1, max_overflow), + _mm_cmpeq_epi16(*preg1, min_overflow)); + __m128i cmp2 = _mm_or_si128(_mm_cmpeq_epi16(*preg2, max_overflow), + _mm_cmpeq_epi16(*preg2, min_overflow)); + __m128i cmp3 = _mm_or_si128(_mm_cmpeq_epi16(*preg3, max_overflow), + _mm_cmpeq_epi16(*preg3, min_overflow)); + cmp0 = _mm_or_si128(_mm_or_si128(cmp0, cmp1), _mm_or_si128(cmp2, cmp3)); + return _mm_movemask_epi8(cmp0); +} + +static INLINE int check_epi16_overflow_x8( + const __m128i *preg0, const __m128i *preg1, const __m128i *preg2, + const __m128i *preg3, const __m128i *preg4, const __m128i *preg5, + const __m128i *preg6, const __m128i *preg7) { + int res0, res1; + res0 = check_epi16_overflow_x4(preg0, preg1, preg2, preg3); + res1 = check_epi16_overflow_x4(preg4, preg5, preg6, preg7); + return res0 + res1; +} + +static INLINE int check_epi16_overflow_x12( + const __m128i *preg0, const __m128i *preg1, const __m128i *preg2, + const __m128i *preg3, const __m128i *preg4, const __m128i *preg5, + const __m128i *preg6, const __m128i *preg7, const __m128i *preg8, + const __m128i *preg9, const __m128i *preg10, const __m128i *preg11) { + int res0, res1; + res0 = check_epi16_overflow_x4(preg0, preg1, preg2, preg3); + res1 = check_epi16_overflow_x4(preg4, preg5, preg6, preg7); + if (!res0) res0 = check_epi16_overflow_x4(preg8, preg9, preg10, preg11); + return res0 + res1; +} + +static INLINE int check_epi16_overflow_x16( + const __m128i *preg0, const __m128i *preg1, const __m128i *preg2, + const __m128i *preg3, const __m128i *preg4, const __m128i *preg5, + const __m128i *preg6, const __m128i *preg7, const __m128i *preg8, + const __m128i *preg9, const __m128i *preg10, const __m128i *preg11, + const __m128i *preg12, const __m128i *preg13, const __m128i *preg14, + const __m128i *preg15) { + int res0, res1; + res0 = check_epi16_overflow_x4(preg0, preg1, preg2, preg3); + res1 = check_epi16_overflow_x4(preg4, preg5, preg6, preg7); + if (!res0) { + res0 = check_epi16_overflow_x4(preg8, preg9, preg10, preg11); + if (!res1) res1 = check_epi16_overflow_x4(preg12, preg13, preg14, preg15); + } + return res0 + res1; +} + +static INLINE int check_epi16_overflow_x32( + const __m128i *preg0, const __m128i *preg1, const __m128i *preg2, + const __m128i *preg3, const __m128i *preg4, const __m128i *preg5, + const __m128i *preg6, const __m128i *preg7, const __m128i *preg8, + const __m128i *preg9, const __m128i *preg10, const __m128i *preg11, + const __m128i *preg12, const __m128i *preg13, const __m128i *preg14, + const __m128i *preg15, const __m128i *preg16, const __m128i *preg17, + const __m128i *preg18, const __m128i *preg19, const __m128i *preg20, + const __m128i *preg21, const __m128i *preg22, const __m128i *preg23, + const __m128i *preg24, const __m128i *preg25, const __m128i *preg26, + const __m128i *preg27, const __m128i *preg28, const __m128i *preg29, + const __m128i *preg30, const __m128i *preg31) { + int res0, res1; + res0 = check_epi16_overflow_x4(preg0, preg1, preg2, preg3); + res1 = check_epi16_overflow_x4(preg4, preg5, preg6, preg7); + if (!res0) { + res0 = check_epi16_overflow_x4(preg8, preg9, preg10, preg11); + if (!res1) { + res1 = check_epi16_overflow_x4(preg12, preg13, preg14, preg15); + if (!res0) { + res0 = check_epi16_overflow_x4(preg16, preg17, preg18, preg19); + if (!res1) { + res1 = check_epi16_overflow_x4(preg20, preg21, preg22, preg23); + if (!res0) { + res0 = check_epi16_overflow_x4(preg24, preg25, preg26, preg27); + if (!res1) + res1 = check_epi16_overflow_x4(preg28, preg29, preg30, preg31); + } + } + } + } + } + return res0 + res1; +} + +static INLINE int k_check_epi32_overflow_4(const __m128i *preg0, + const __m128i *preg1, + const __m128i *preg2, + const __m128i *preg3, + const __m128i *zero) { + __m128i minus_one = _mm_set1_epi32(-1); + // Check for overflows + __m128i reg0_shifted = _mm_slli_epi64(*preg0, 1); + __m128i reg1_shifted = _mm_slli_epi64(*preg1, 1); + __m128i reg2_shifted = _mm_slli_epi64(*preg2, 1); + __m128i reg3_shifted = _mm_slli_epi64(*preg3, 1); + __m128i reg0_top_dwords = + _mm_shuffle_epi32(reg0_shifted, _MM_SHUFFLE(0, 0, 3, 1)); + __m128i reg1_top_dwords = + _mm_shuffle_epi32(reg1_shifted, _MM_SHUFFLE(0, 0, 3, 1)); + __m128i reg2_top_dwords = + _mm_shuffle_epi32(reg2_shifted, _MM_SHUFFLE(0, 0, 3, 1)); + __m128i reg3_top_dwords = + _mm_shuffle_epi32(reg3_shifted, _MM_SHUFFLE(0, 0, 3, 1)); + __m128i top_dwords_01 = _mm_unpacklo_epi64(reg0_top_dwords, reg1_top_dwords); + __m128i top_dwords_23 = _mm_unpacklo_epi64(reg2_top_dwords, reg3_top_dwords); + __m128i valid_positve_01 = _mm_cmpeq_epi32(top_dwords_01, *zero); + __m128i valid_positve_23 = _mm_cmpeq_epi32(top_dwords_23, *zero); + __m128i valid_negative_01 = _mm_cmpeq_epi32(top_dwords_01, minus_one); + __m128i valid_negative_23 = _mm_cmpeq_epi32(top_dwords_23, minus_one); + int overflow_01 = + _mm_movemask_epi8(_mm_cmpeq_epi32(valid_positve_01, valid_negative_01)); + int overflow_23 = + _mm_movemask_epi8(_mm_cmpeq_epi32(valid_positve_23, valid_negative_23)); + return (overflow_01 + overflow_23); +} + +static INLINE int k_check_epi32_overflow_8( + const __m128i *preg0, const __m128i *preg1, const __m128i *preg2, + const __m128i *preg3, const __m128i *preg4, const __m128i *preg5, + const __m128i *preg6, const __m128i *preg7, const __m128i *zero) { + int overflow = k_check_epi32_overflow_4(preg0, preg1, preg2, preg3, zero); + if (!overflow) { + overflow = k_check_epi32_overflow_4(preg4, preg5, preg6, preg7, zero); + } + return overflow; +} + +static INLINE int k_check_epi32_overflow_16( + const __m128i *preg0, const __m128i *preg1, const __m128i *preg2, + const __m128i *preg3, const __m128i *preg4, const __m128i *preg5, + const __m128i *preg6, const __m128i *preg7, const __m128i *preg8, + const __m128i *preg9, const __m128i *preg10, const __m128i *preg11, + const __m128i *preg12, const __m128i *preg13, const __m128i *preg14, + const __m128i *preg15, const __m128i *zero) { + int overflow = k_check_epi32_overflow_4(preg0, preg1, preg2, preg3, zero); + if (!overflow) { + overflow = k_check_epi32_overflow_4(preg4, preg5, preg6, preg7, zero); + if (!overflow) { + overflow = k_check_epi32_overflow_4(preg8, preg9, preg10, preg11, zero); + if (!overflow) { + overflow = + k_check_epi32_overflow_4(preg12, preg13, preg14, preg15, zero); + } + } + } + return overflow; +} + +static INLINE int k_check_epi32_overflow_32( + const __m128i *preg0, const __m128i *preg1, const __m128i *preg2, + const __m128i *preg3, const __m128i *preg4, const __m128i *preg5, + const __m128i *preg6, const __m128i *preg7, const __m128i *preg8, + const __m128i *preg9, const __m128i *preg10, const __m128i *preg11, + const __m128i *preg12, const __m128i *preg13, const __m128i *preg14, + const __m128i *preg15, const __m128i *preg16, const __m128i *preg17, + const __m128i *preg18, const __m128i *preg19, const __m128i *preg20, + const __m128i *preg21, const __m128i *preg22, const __m128i *preg23, + const __m128i *preg24, const __m128i *preg25, const __m128i *preg26, + const __m128i *preg27, const __m128i *preg28, const __m128i *preg29, + const __m128i *preg30, const __m128i *preg31, const __m128i *zero) { + int overflow = k_check_epi32_overflow_4(preg0, preg1, preg2, preg3, zero); + if (!overflow) { + overflow = k_check_epi32_overflow_4(preg4, preg5, preg6, preg7, zero); + if (!overflow) { + overflow = k_check_epi32_overflow_4(preg8, preg9, preg10, preg11, zero); + if (!overflow) { + overflow = + k_check_epi32_overflow_4(preg12, preg13, preg14, preg15, zero); + if (!overflow) { + overflow = + k_check_epi32_overflow_4(preg16, preg17, preg18, preg19, zero); + if (!overflow) { + overflow = + k_check_epi32_overflow_4(preg20, preg21, preg22, preg23, zero); + if (!overflow) { + overflow = k_check_epi32_overflow_4(preg24, preg25, preg26, + preg27, zero); + if (!overflow) { + overflow = k_check_epi32_overflow_4(preg28, preg29, preg30, + preg31, zero); + } + } + } + } + } + } + } + return overflow; +} + +static INLINE void store_output(const __m128i *poutput, tran_low_t *dst_ptr) { +#if CONFIG_HIGHBITDEPTH + const __m128i zero = _mm_setzero_si128(); + const __m128i sign_bits = _mm_cmplt_epi16(*poutput, zero); + __m128i out0 = _mm_unpacklo_epi16(*poutput, sign_bits); + __m128i out1 = _mm_unpackhi_epi16(*poutput, sign_bits); + _mm_store_si128((__m128i *)(dst_ptr), out0); + _mm_store_si128((__m128i *)(dst_ptr + 4), out1); +#else + _mm_store_si128((__m128i *)(dst_ptr), *poutput); +#endif // CONFIG_HIGHBITDEPTH +} + +static INLINE __m128i mult_round_shift(const __m128i *pin0, const __m128i *pin1, + const __m128i *pmultiplier, + const __m128i *prounding, int shift) { + const __m128i u0 = _mm_madd_epi16(*pin0, *pmultiplier); + const __m128i u1 = _mm_madd_epi16(*pin1, *pmultiplier); + const __m128i v0 = _mm_add_epi32(u0, *prounding); + const __m128i v1 = _mm_add_epi32(u1, *prounding); + const __m128i w0 = _mm_srai_epi32(v0, shift); + const __m128i w1 = _mm_srai_epi32(v1, shift); + return _mm_packs_epi32(w0, w1); +} + +static INLINE void transpose_and_output8x8( + const __m128i *pin00, const __m128i *pin01, const __m128i *pin02, + const __m128i *pin03, const __m128i *pin04, const __m128i *pin05, + const __m128i *pin06, const __m128i *pin07, int pass, int16_t *out0_ptr, + tran_low_t *out1_ptr) { + // 00 01 02 03 04 05 06 07 + // 10 11 12 13 14 15 16 17 + // 20 21 22 23 24 25 26 27 + // 30 31 32 33 34 35 36 37 + // 40 41 42 43 44 45 46 47 + // 50 51 52 53 54 55 56 57 + // 60 61 62 63 64 65 66 67 + // 70 71 72 73 74 75 76 77 + const __m128i tr0_0 = _mm_unpacklo_epi16(*pin00, *pin01); + const __m128i tr0_1 = _mm_unpacklo_epi16(*pin02, *pin03); + const __m128i tr0_2 = _mm_unpackhi_epi16(*pin00, *pin01); + const __m128i tr0_3 = _mm_unpackhi_epi16(*pin02, *pin03); + const __m128i tr0_4 = _mm_unpacklo_epi16(*pin04, *pin05); + const __m128i tr0_5 = _mm_unpacklo_epi16(*pin06, *pin07); + const __m128i tr0_6 = _mm_unpackhi_epi16(*pin04, *pin05); + const __m128i tr0_7 = _mm_unpackhi_epi16(*pin06, *pin07); + // 00 10 01 11 02 12 03 13 + // 20 30 21 31 22 32 23 33 + // 04 14 05 15 06 16 07 17 + // 24 34 25 35 26 36 27 37 + // 40 50 41 51 42 52 43 53 + // 60 70 61 71 62 72 63 73 + // 54 54 55 55 56 56 57 57 + // 64 74 65 75 66 76 67 77 + const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); + const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3); + const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); + const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3); + const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); + const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); + const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); + const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); + // 00 10 20 30 01 11 21 31 + // 40 50 60 70 41 51 61 71 + // 02 12 22 32 03 13 23 33 + // 42 52 62 72 43 53 63 73 + // 04 14 24 34 05 15 21 36 + // 44 54 64 74 45 55 61 76 + // 06 16 26 36 07 17 27 37 + // 46 56 66 76 47 57 67 77 + const __m128i tr2_0 = _mm_unpacklo_epi64(tr1_0, tr1_4); + const __m128i tr2_1 = _mm_unpackhi_epi64(tr1_0, tr1_4); + const __m128i tr2_2 = _mm_unpacklo_epi64(tr1_2, tr1_6); + const __m128i tr2_3 = _mm_unpackhi_epi64(tr1_2, tr1_6); + const __m128i tr2_4 = _mm_unpacklo_epi64(tr1_1, tr1_5); + const __m128i tr2_5 = _mm_unpackhi_epi64(tr1_1, tr1_5); + const __m128i tr2_6 = _mm_unpacklo_epi64(tr1_3, tr1_7); + const __m128i tr2_7 = _mm_unpackhi_epi64(tr1_3, tr1_7); + // 00 10 20 30 40 50 60 70 + // 01 11 21 31 41 51 61 71 + // 02 12 22 32 42 52 62 72 + // 03 13 23 33 43 53 63 73 + // 04 14 24 34 44 54 64 74 + // 05 15 25 35 45 55 65 75 + // 06 16 26 36 46 56 66 76 + // 07 17 27 37 47 57 67 77 + if (pass == 0) { + _mm_storeu_si128((__m128i *)(out0_ptr + 0 * 16), tr2_0); + _mm_storeu_si128((__m128i *)(out0_ptr + 1 * 16), tr2_1); + _mm_storeu_si128((__m128i *)(out0_ptr + 2 * 16), tr2_2); + _mm_storeu_si128((__m128i *)(out0_ptr + 3 * 16), tr2_3); + _mm_storeu_si128((__m128i *)(out0_ptr + 4 * 16), tr2_4); + _mm_storeu_si128((__m128i *)(out0_ptr + 5 * 16), tr2_5); + _mm_storeu_si128((__m128i *)(out0_ptr + 6 * 16), tr2_6); + _mm_storeu_si128((__m128i *)(out0_ptr + 7 * 16), tr2_7); + } else { + storeu_output(&tr2_0, (out1_ptr + 0 * 16)); + storeu_output(&tr2_1, (out1_ptr + 1 * 16)); + storeu_output(&tr2_2, (out1_ptr + 2 * 16)); + storeu_output(&tr2_3, (out1_ptr + 3 * 16)); + storeu_output(&tr2_4, (out1_ptr + 4 * 16)); + storeu_output(&tr2_5, (out1_ptr + 5 * 16)); + storeu_output(&tr2_6, (out1_ptr + 6 * 16)); + storeu_output(&tr2_7, (out1_ptr + 7 * 16)); + } +} + +void fdct32_8col(__m128i *in0, __m128i *in1); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_DSP_X86_FWD_TXFM_SSE2_H_ diff --git a/third_party/aom/aom_dsp/x86/fwd_txfm_ssse3_x86_64.asm b/third_party/aom/aom_dsp/x86/fwd_txfm_ssse3_x86_64.asm new file mode 100644 index 0000000000..8fa1c04d0c --- /dev/null +++ b/third_party/aom/aom_dsp/x86/fwd_txfm_ssse3_x86_64.asm @@ -0,0 +1,204 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +; This file provides SSSE3 version of the forward transformation. Part +; of the macro definitions are originally derived from the ffmpeg project. +; The current version applies to x86 64-bit only. + +SECTION_RODATA + +pw_11585x2: times 8 dw 23170 +pd_8192: times 4 dd 8192 + +%macro TRANSFORM_COEFFS 2 +pw_%1_%2: dw %1, %2, %1, %2, %1, %2, %1, %2 +pw_%2_m%1: dw %2, -%1, %2, -%1, %2, -%1, %2, -%1 +%endmacro + +TRANSFORM_COEFFS 11585, 11585 +TRANSFORM_COEFFS 15137, 6270 +TRANSFORM_COEFFS 16069, 3196 +TRANSFORM_COEFFS 9102, 13623 + +SECTION .text + +%if ARCH_X86_64 +%macro SUM_SUB 3 + psubw m%3, m%1, m%2 + paddw m%1, m%2 + SWAP %2, %3 +%endmacro + +; butterfly operation +%macro MUL_ADD_2X 6 ; dst1, dst2, src, round, coefs1, coefs2 + pmaddwd m%1, m%3, %5 + pmaddwd m%2, m%3, %6 + paddd m%1, %4 + paddd m%2, %4 + psrad m%1, 14 + psrad m%2, 14 +%endmacro + +%macro BUTTERFLY_4X 7 ; dst1, dst2, coef1, coef2, round, tmp1, tmp2 + punpckhwd m%6, m%2, m%1 + MUL_ADD_2X %7, %6, %6, %5, [pw_%4_%3], [pw_%3_m%4] + punpcklwd m%2, m%1 + MUL_ADD_2X %1, %2, %2, %5, [pw_%4_%3], [pw_%3_m%4] + packssdw m%1, m%7 + packssdw m%2, m%6 +%endmacro + +; matrix transpose +%macro INTERLEAVE_2X 4 + punpckh%1 m%4, m%2, m%3 + punpckl%1 m%2, m%3 + SWAP %3, %4 +%endmacro + +%macro TRANSPOSE8X8 9 + INTERLEAVE_2X wd, %1, %2, %9 + INTERLEAVE_2X wd, %3, %4, %9 + INTERLEAVE_2X wd, %5, %6, %9 + INTERLEAVE_2X wd, %7, %8, %9 + + INTERLEAVE_2X dq, %1, %3, %9 + INTERLEAVE_2X dq, %2, %4, %9 + INTERLEAVE_2X dq, %5, %7, %9 + INTERLEAVE_2X dq, %6, %8, %9 + + INTERLEAVE_2X qdq, %1, %5, %9 + INTERLEAVE_2X qdq, %3, %7, %9 + INTERLEAVE_2X qdq, %2, %6, %9 + INTERLEAVE_2X qdq, %4, %8, %9 + + SWAP %2, %5 + SWAP %4, %7 +%endmacro + +; 1D forward 8x8 DCT transform +%macro FDCT8_1D 1 + SUM_SUB 0, 7, 9 + SUM_SUB 1, 6, 9 + SUM_SUB 2, 5, 9 + SUM_SUB 3, 4, 9 + + SUM_SUB 0, 3, 9 + SUM_SUB 1, 2, 9 + SUM_SUB 6, 5, 9 +%if %1 == 0 + SUM_SUB 0, 1, 9 +%endif + + BUTTERFLY_4X 2, 3, 6270, 15137, m8, 9, 10 + + pmulhrsw m6, m12 + pmulhrsw m5, m12 +%if %1 == 0 + pmulhrsw m0, m12 + pmulhrsw m1, m12 +%else + BUTTERFLY_4X 1, 0, 11585, 11585, m8, 9, 10 + SWAP 0, 1 +%endif + + SUM_SUB 4, 5, 9 + SUM_SUB 7, 6, 9 + BUTTERFLY_4X 4, 7, 3196, 16069, m8, 9, 10 + BUTTERFLY_4X 5, 6, 13623, 9102, m8, 9, 10 + SWAP 1, 4 + SWAP 3, 6 +%endmacro + +%macro DIVIDE_ROUND_2X 4 ; dst1, dst2, tmp1, tmp2 + psraw m%3, m%1, 15 + psraw m%4, m%2, 15 + psubw m%1, m%3 + psubw m%2, m%4 + psraw m%1, 1 + psraw m%2, 1 +%endmacro + +%macro STORE_OUTPUT 2 ; index, result +%if CONFIG_HIGHBITDEPTH + ; const __m128i sign_bits = _mm_cmplt_epi16(*poutput, zero); + ; __m128i out0 = _mm_unpacklo_epi16(*poutput, sign_bits); + ; __m128i out1 = _mm_unpackhi_epi16(*poutput, sign_bits); + ; _mm_store_si128((__m128i *)(dst_ptr), out0); + ; _mm_store_si128((__m128i *)(dst_ptr + 4), out1); + pxor m11, m11 + pcmpgtw m11, m%2 + movdqa m12, m%2 + punpcklwd m%2, m11 + punpckhwd m12, m11 + mova [outputq + 4*%1 + 0], m%2 + mova [outputq + 4*%1 + 16], m12 +%else + mova [outputq + 2*%1], m%2 +%endif +%endmacro + +INIT_XMM ssse3 +cglobal fdct8x8, 3, 5, 13, input, output, stride + + mova m8, [pd_8192] + mova m12, [pw_11585x2] + + lea r3, [2 * strideq] + lea r4, [4 * strideq] + mova m0, [inputq] + mova m1, [inputq + r3] + lea inputq, [inputq + r4] + mova m2, [inputq] + mova m3, [inputq + r3] + lea inputq, [inputq + r4] + mova m4, [inputq] + mova m5, [inputq + r3] + lea inputq, [inputq + r4] + mova m6, [inputq] + mova m7, [inputq + r3] + + ; left shift by 2 to increase forward transformation precision + psllw m0, 2 + psllw m1, 2 + psllw m2, 2 + psllw m3, 2 + psllw m4, 2 + psllw m5, 2 + psllw m6, 2 + psllw m7, 2 + + ; column transform + FDCT8_1D 0 + TRANSPOSE8X8 0, 1, 2, 3, 4, 5, 6, 7, 9 + + FDCT8_1D 1 + TRANSPOSE8X8 0, 1, 2, 3, 4, 5, 6, 7, 9 + + DIVIDE_ROUND_2X 0, 1, 9, 10 + DIVIDE_ROUND_2X 2, 3, 9, 10 + DIVIDE_ROUND_2X 4, 5, 9, 10 + DIVIDE_ROUND_2X 6, 7, 9, 10 + + STORE_OUTPUT 0, 0 + STORE_OUTPUT 8, 1 + STORE_OUTPUT 16, 2 + STORE_OUTPUT 24, 3 + STORE_OUTPUT 32, 4 + STORE_OUTPUT 40, 5 + STORE_OUTPUT 48, 6 + STORE_OUTPUT 56, 7 + + RET +%endif diff --git a/third_party/aom/aom_dsp/x86/halfpix_variance_impl_sse2.asm b/third_party/aom/aom_dsp/x86/halfpix_variance_impl_sse2.asm new file mode 100644 index 0000000000..60446b0869 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/halfpix_variance_impl_sse2.asm @@ -0,0 +1,349 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "aom_ports/x86_abi_support.asm" + +;void aom_half_horiz_vert_variance16x_h_sse2(unsigned char *ref, +; int ref_stride, +; unsigned char *src, +; int src_stride, +; unsigned int height, +; int *sum, +; unsigned int *sumsquared) +global sym(aom_half_horiz_vert_variance16x_h_sse2) PRIVATE +sym(aom_half_horiz_vert_variance16x_h_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + GET_GOT rbx + push rsi + push rdi + ; end prolog + + pxor xmm6, xmm6 ; error accumulator + pxor xmm7, xmm7 ; sse eaccumulator + mov rsi, arg(0) ;ref + + mov rdi, arg(2) ;src + movsxd rcx, dword ptr arg(4) ;height + movsxd rax, dword ptr arg(1) ;ref_stride + movsxd rdx, dword ptr arg(3) ;src_stride + + pxor xmm0, xmm0 ; + + movdqu xmm5, XMMWORD PTR [rsi] + movdqu xmm3, XMMWORD PTR [rsi+1] + pavgb xmm5, xmm3 ; xmm5 = avg(xmm1,xmm3) horizontal line 1 + + lea rsi, [rsi + rax] + +aom_half_horiz_vert_variance16x_h_1: + movdqu xmm1, XMMWORD PTR [rsi] ; + movdqu xmm2, XMMWORD PTR [rsi+1] ; + pavgb xmm1, xmm2 ; xmm1 = avg(xmm1,xmm3) horizontal line i+1 + + pavgb xmm5, xmm1 ; xmm = vertical average of the above + + movdqa xmm4, xmm5 + punpcklbw xmm5, xmm0 ; xmm5 = words of above + punpckhbw xmm4, xmm0 + + movq xmm3, QWORD PTR [rdi] ; xmm3 = d0,d1,d2..d7 + punpcklbw xmm3, xmm0 ; xmm3 = words of above + psubw xmm5, xmm3 ; xmm5 -= xmm3 + + movq xmm3, QWORD PTR [rdi+8] + punpcklbw xmm3, xmm0 + psubw xmm4, xmm3 + + paddw xmm6, xmm5 ; xmm6 += accumulated column differences + paddw xmm6, xmm4 + pmaddwd xmm5, xmm5 ; xmm5 *= xmm5 + pmaddwd xmm4, xmm4 + paddd xmm7, xmm5 ; xmm7 += accumulated square column differences + paddd xmm7, xmm4 + + movdqa xmm5, xmm1 ; save xmm1 for use on the next row + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + + sub rcx, 1 ; + jnz aom_half_horiz_vert_variance16x_h_1 ; + + pxor xmm1, xmm1 + pxor xmm5, xmm5 + + punpcklwd xmm0, xmm6 + punpckhwd xmm1, xmm6 + psrad xmm0, 16 + psrad xmm1, 16 + paddd xmm0, xmm1 + movdqa xmm1, xmm0 + + movdqa xmm6, xmm7 + punpckldq xmm6, xmm5 + punpckhdq xmm7, xmm5 + paddd xmm6, xmm7 + + punpckldq xmm0, xmm5 + punpckhdq xmm1, xmm5 + paddd xmm0, xmm1 + + movdqa xmm7, xmm6 + movdqa xmm1, xmm0 + + psrldq xmm7, 8 + psrldq xmm1, 8 + + paddd xmm6, xmm7 + paddd xmm0, xmm1 + + mov rsi, arg(5) ;[Sum] + mov rdi, arg(6) ;[SSE] + + movd [rsi], xmm0 + movd [rdi], xmm6 + + ; begin epilog + pop rdi + pop rsi + RESTORE_GOT + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + + +;void aom_half_vert_variance16x_h_sse2(unsigned char *ref, +; int ref_stride, +; unsigned char *src, +; int src_stride, +; unsigned int height, +; int *sum, +; unsigned int *sumsquared) +global sym(aom_half_vert_variance16x_h_sse2) PRIVATE +sym(aom_half_vert_variance16x_h_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + GET_GOT rbx + push rsi + push rdi + ; end prolog + + pxor xmm6, xmm6 ; error accumulator + pxor xmm7, xmm7 ; sse eaccumulator + mov rsi, arg(0) ;ref + + mov rdi, arg(2) ;src + movsxd rcx, dword ptr arg(4) ;height + movsxd rax, dword ptr arg(1) ;ref_stride + movsxd rdx, dword ptr arg(3) ;src_stride + + movdqu xmm5, XMMWORD PTR [rsi] + lea rsi, [rsi + rax ] + pxor xmm0, xmm0 + +aom_half_vert_variance16x_h_1: + movdqu xmm3, XMMWORD PTR [rsi] + + pavgb xmm5, xmm3 ; xmm5 = avg(xmm1,xmm3) + movdqa xmm4, xmm5 + punpcklbw xmm5, xmm0 + punpckhbw xmm4, xmm0 + + movq xmm2, QWORD PTR [rdi] + punpcklbw xmm2, xmm0 + psubw xmm5, xmm2 + movq xmm2, QWORD PTR [rdi+8] + punpcklbw xmm2, xmm0 + psubw xmm4, xmm2 + + paddw xmm6, xmm5 ; xmm6 += accumulated column differences + paddw xmm6, xmm4 + pmaddwd xmm5, xmm5 ; xmm5 *= xmm5 + pmaddwd xmm4, xmm4 + paddd xmm7, xmm5 ; xmm7 += accumulated square column differences + paddd xmm7, xmm4 + + movdqa xmm5, xmm3 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + + sub rcx, 1 + jnz aom_half_vert_variance16x_h_1 + + pxor xmm1, xmm1 + pxor xmm5, xmm5 + + punpcklwd xmm0, xmm6 + punpckhwd xmm1, xmm6 + psrad xmm0, 16 + psrad xmm1, 16 + paddd xmm0, xmm1 + movdqa xmm1, xmm0 + + movdqa xmm6, xmm7 + punpckldq xmm6, xmm5 + punpckhdq xmm7, xmm5 + paddd xmm6, xmm7 + + punpckldq xmm0, xmm5 + punpckhdq xmm1, xmm5 + paddd xmm0, xmm1 + + movdqa xmm7, xmm6 + movdqa xmm1, xmm0 + + psrldq xmm7, 8 + psrldq xmm1, 8 + + paddd xmm6, xmm7 + paddd xmm0, xmm1 + + mov rsi, arg(5) ;[Sum] + mov rdi, arg(6) ;[SSE] + + movd [rsi], xmm0 + movd [rdi], xmm6 + + ; begin epilog + pop rdi + pop rsi + RESTORE_GOT + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + + +;void aom_half_horiz_variance16x_h_sse2(unsigned char *ref, +; int ref_stride +; unsigned char *src, +; int src_stride, +; unsigned int height, +; int *sum, +; unsigned int *sumsquared) +global sym(aom_half_horiz_variance16x_h_sse2) PRIVATE +sym(aom_half_horiz_variance16x_h_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 7 + SAVE_XMM 7 + GET_GOT rbx + push rsi + push rdi + ; end prolog + + pxor xmm6, xmm6 ; error accumulator + pxor xmm7, xmm7 ; sse eaccumulator + mov rsi, arg(0) ;ref + + mov rdi, arg(2) ;src + movsxd rcx, dword ptr arg(4) ;height + movsxd rax, dword ptr arg(1) ;ref_stride + movsxd rdx, dword ptr arg(3) ;src_stride + + pxor xmm0, xmm0 ; + +aom_half_horiz_variance16x_h_1: + movdqu xmm5, XMMWORD PTR [rsi] ; xmm5 = s0,s1,s2..s15 + movdqu xmm3, XMMWORD PTR [rsi+1] ; xmm3 = s1,s2,s3..s16 + + pavgb xmm5, xmm3 ; xmm5 = avg(xmm1,xmm3) + movdqa xmm1, xmm5 + punpcklbw xmm5, xmm0 ; xmm5 = words of above + punpckhbw xmm1, xmm0 + + movq xmm3, QWORD PTR [rdi] ; xmm3 = d0,d1,d2..d7 + punpcklbw xmm3, xmm0 ; xmm3 = words of above + movq xmm2, QWORD PTR [rdi+8] + punpcklbw xmm2, xmm0 + + psubw xmm5, xmm3 ; xmm5 -= xmm3 + psubw xmm1, xmm2 + paddw xmm6, xmm5 ; xmm6 += accumulated column differences + paddw xmm6, xmm1 + pmaddwd xmm5, xmm5 ; xmm5 *= xmm5 + pmaddwd xmm1, xmm1 + paddd xmm7, xmm5 ; xmm7 += accumulated square column differences + paddd xmm7, xmm1 + + lea rsi, [rsi + rax] + lea rdi, [rdi + rdx] + + sub rcx, 1 ; + jnz aom_half_horiz_variance16x_h_1 ; + + pxor xmm1, xmm1 + pxor xmm5, xmm5 + + punpcklwd xmm0, xmm6 + punpckhwd xmm1, xmm6 + psrad xmm0, 16 + psrad xmm1, 16 + paddd xmm0, xmm1 + movdqa xmm1, xmm0 + + movdqa xmm6, xmm7 + punpckldq xmm6, xmm5 + punpckhdq xmm7, xmm5 + paddd xmm6, xmm7 + + punpckldq xmm0, xmm5 + punpckhdq xmm1, xmm5 + paddd xmm0, xmm1 + + movdqa xmm7, xmm6 + movdqa xmm1, xmm0 + + psrldq xmm7, 8 + psrldq xmm1, 8 + + paddd xmm6, xmm7 + paddd xmm0, xmm1 + + mov rsi, arg(5) ;[Sum] + mov rdi, arg(6) ;[SSE] + + movd [rsi], xmm0 + movd [rdi], xmm6 + + ; begin epilog + pop rdi + pop rsi + RESTORE_GOT + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +SECTION_RODATA +; short xmm_bi_rd[8] = { 64, 64, 64, 64,64, 64, 64, 64}; +align 16 +xmm_bi_rd: + times 8 dw 64 +align 16 +aom_bilinear_filters_sse2: + dw 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0 + dw 112, 112, 112, 112, 112, 112, 112, 112, 16, 16, 16, 16, 16, 16, 16, 16 + dw 96, 96, 96, 96, 96, 96, 96, 96, 32, 32, 32, 32, 32, 32, 32, 32 + dw 80, 80, 80, 80, 80, 80, 80, 80, 48, 48, 48, 48, 48, 48, 48, 48 + dw 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 + dw 48, 48, 48, 48, 48, 48, 48, 48, 80, 80, 80, 80, 80, 80, 80, 80 + dw 32, 32, 32, 32, 32, 32, 32, 32, 96, 96, 96, 96, 96, 96, 96, 96 + dw 16, 16, 16, 16, 16, 16, 16, 16, 112, 112, 112, 112, 112, 112, 112, 112 diff --git a/third_party/aom/aom_dsp/x86/halfpix_variance_sse2.c b/third_party/aom/aom_dsp/x86/halfpix_variance_sse2.c new file mode 100644 index 0000000000..a99c0b40e0 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/halfpix_variance_sse2.c @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +void aom_half_horiz_vert_variance16x_h_sse2(const unsigned char *ref, + int ref_stride, + const unsigned char *src, + int src_stride, unsigned int height, + int *sum, unsigned int *sumsquared); +void aom_half_horiz_variance16x_h_sse2(const unsigned char *ref, int ref_stride, + const unsigned char *src, int src_stride, + unsigned int height, int *sum, + unsigned int *sumsquared); +void aom_half_vert_variance16x_h_sse2(const unsigned char *ref, int ref_stride, + const unsigned char *src, int src_stride, + unsigned int height, int *sum, + unsigned int *sumsquared); + +uint32_t aom_variance_halfpixvar16x16_h_sse2(const unsigned char *src, + int src_stride, + const unsigned char *dst, + int dst_stride, uint32_t *sse) { + int xsum0; + unsigned int xxsum0; + + aom_half_horiz_variance16x_h_sse2(src, src_stride, dst, dst_stride, 16, + &xsum0, &xxsum0); + + *sse = xxsum0; + assert(xsum0 <= 255 * 16 * 16); + assert(xsum0 >= -255 * 16 * 16); + return (xxsum0 - ((uint32_t)((int64_t)xsum0 * xsum0) >> 8)); +} + +uint32_t aom_variance_halfpixvar16x16_v_sse2(const unsigned char *src, + int src_stride, + const unsigned char *dst, + int dst_stride, uint32_t *sse) { + int xsum0; + unsigned int xxsum0; + aom_half_vert_variance16x_h_sse2(src, src_stride, dst, dst_stride, 16, &xsum0, + &xxsum0); + + *sse = xxsum0; + assert(xsum0 <= 255 * 16 * 16); + assert(xsum0 >= -255 * 16 * 16); + return (xxsum0 - ((uint32_t)((int64_t)xsum0 * xsum0) >> 8)); +} + +uint32_t aom_variance_halfpixvar16x16_hv_sse2(const unsigned char *src, + int src_stride, + const unsigned char *dst, + int dst_stride, uint32_t *sse) { + int xsum0; + unsigned int xxsum0; + + aom_half_horiz_vert_variance16x_h_sse2(src, src_stride, dst, dst_stride, 16, + &xsum0, &xxsum0); + + *sse = xxsum0; + assert(xsum0 <= 255 * 16 * 16); + assert(xsum0 >= -255 * 16 * 16); + return (xxsum0 - ((uint32_t)((int64_t)xsum0 * xsum0) >> 8)); +} diff --git a/third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c b/third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c new file mode 100644 index 0000000000..7d96e26ae4 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c @@ -0,0 +1,1151 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/x86/convolve.h" + +#define CONV8_ROUNDING_BITS (7) + +static const uint8_t signal_pattern_0[32] = { 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, + 7, 6, 7, 8, 9, 0, 1, 2, 3, 2, 3, + 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 }; + +static const uint8_t signal_pattern_1[32] = { 4, 5, 6, 7, 6, 7, 8, 9, + 8, 9, 10, 11, 10, 11, 12, 13, + 4, 5, 6, 7, 6, 7, 8, 9, + 8, 9, 10, 11, 10, 11, 12, 13 }; + +static const uint8_t signal_pattern_2[32] = { 6, 7, 8, 9, 8, 9, 10, 11, + 10, 11, 12, 13, 12, 13, 14, 15, + 6, 7, 8, 9, 8, 9, 10, 11, + 10, 11, 12, 13, 12, 13, 14, 15 }; + +static const uint32_t signal_index[8] = { 2, 3, 4, 5, 2, 3, 4, 5 }; + +typedef enum { PACK_8x1, PACK_8x2, PACK_16x1 } PixelPackFormat; + +typedef void (*WritePixels)(const __m256i *y0, const __m256i *y1, + const __m256i *mask, uint16_t *dst, + ptrdiff_t pitch); + +// ----------------------------------------------------------------------------- +// Copy and average + +void aom_highbd_convolve_copy_avx2(const uint8_t *src8, ptrdiff_t src_stride, + uint8_t *dst8, ptrdiff_t dst_stride, + const int16_t *filter_x, int filter_x_stride, + const int16_t *filter_y, int filter_y_stride, + int width, int h, int bd) { + const uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + (void)filter_x; + (void)filter_y; + (void)filter_x_stride; + (void)filter_y_stride; + (void)bd; + + assert(width % 4 == 0); + if (width > 32) { // width = 64 + do { + const __m256i p0 = _mm256_loadu_si256((const __m256i *)src); + const __m256i p1 = _mm256_loadu_si256((const __m256i *)(src + 16)); + const __m256i p2 = _mm256_loadu_si256((const __m256i *)(src + 32)); + const __m256i p3 = _mm256_loadu_si256((const __m256i *)(src + 48)); + src += src_stride; + _mm256_storeu_si256((__m256i *)dst, p0); + _mm256_storeu_si256((__m256i *)(dst + 16), p1); + _mm256_storeu_si256((__m256i *)(dst + 32), p2); + _mm256_storeu_si256((__m256i *)(dst + 48), p3); + dst += dst_stride; + h--; + } while (h > 0); + } else if (width > 16) { // width = 32 + do { + const __m256i p0 = _mm256_loadu_si256((const __m256i *)src); + const __m256i p1 = _mm256_loadu_si256((const __m256i *)(src + 16)); + src += src_stride; + _mm256_storeu_si256((__m256i *)dst, p0); + _mm256_storeu_si256((__m256i *)(dst + 16), p1); + dst += dst_stride; + h--; + } while (h > 0); + } else if (width > 8) { // width = 16 + __m256i p0, p1; + do { + p0 = _mm256_loadu_si256((const __m256i *)src); + src += src_stride; + p1 = _mm256_loadu_si256((const __m256i *)src); + src += src_stride; + + _mm256_storeu_si256((__m256i *)dst, p0); + dst += dst_stride; + _mm256_storeu_si256((__m256i *)dst, p1); + dst += dst_stride; + h -= 2; + } while (h > 0); + } else if (width > 4) { // width = 8 + __m128i p0, p1; + do { + p0 = _mm_loadu_si128((const __m128i *)src); + src += src_stride; + p1 = _mm_loadu_si128((const __m128i *)src); + src += src_stride; + + _mm_storeu_si128((__m128i *)dst, p0); + dst += dst_stride; + _mm_storeu_si128((__m128i *)dst, p1); + dst += dst_stride; + h -= 2; + } while (h > 0); + } else { // width = 4 + __m128i p0, p1; + do { + p0 = _mm_loadl_epi64((const __m128i *)src); + src += src_stride; + p1 = _mm_loadl_epi64((const __m128i *)src); + src += src_stride; + + _mm_storel_epi64((__m128i *)dst, p0); + dst += dst_stride; + _mm_storel_epi64((__m128i *)dst, p1); + dst += dst_stride; + h -= 2; + } while (h > 0); + } +} + +void aom_highbd_convolve_avg_avx2(const uint8_t *src8, ptrdiff_t src_stride, + uint8_t *dst8, ptrdiff_t dst_stride, + const int16_t *filter_x, int filter_x_stride, + const int16_t *filter_y, int filter_y_stride, + int width, int h, int bd) { + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + (void)filter_x; + (void)filter_y; + (void)filter_x_stride; + (void)filter_y_stride; + (void)bd; + + assert(width % 4 == 0); + if (width > 32) { // width = 64 + __m256i p0, p1, p2, p3, u0, u1, u2, u3; + do { + p0 = _mm256_loadu_si256((const __m256i *)src); + p1 = _mm256_loadu_si256((const __m256i *)(src + 16)); + p2 = _mm256_loadu_si256((const __m256i *)(src + 32)); + p3 = _mm256_loadu_si256((const __m256i *)(src + 48)); + src += src_stride; + u0 = _mm256_loadu_si256((const __m256i *)dst); + u1 = _mm256_loadu_si256((const __m256i *)(dst + 16)); + u2 = _mm256_loadu_si256((const __m256i *)(dst + 32)); + u3 = _mm256_loadu_si256((const __m256i *)(dst + 48)); + _mm256_storeu_si256((__m256i *)dst, _mm256_avg_epu16(p0, u0)); + _mm256_storeu_si256((__m256i *)(dst + 16), _mm256_avg_epu16(p1, u1)); + _mm256_storeu_si256((__m256i *)(dst + 32), _mm256_avg_epu16(p2, u2)); + _mm256_storeu_si256((__m256i *)(dst + 48), _mm256_avg_epu16(p3, u3)); + dst += dst_stride; + h--; + } while (h > 0); + } else if (width > 16) { // width = 32 + __m256i p0, p1, u0, u1; + do { + p0 = _mm256_loadu_si256((const __m256i *)src); + p1 = _mm256_loadu_si256((const __m256i *)(src + 16)); + src += src_stride; + u0 = _mm256_loadu_si256((const __m256i *)dst); + u1 = _mm256_loadu_si256((const __m256i *)(dst + 16)); + _mm256_storeu_si256((__m256i *)dst, _mm256_avg_epu16(p0, u0)); + _mm256_storeu_si256((__m256i *)(dst + 16), _mm256_avg_epu16(p1, u1)); + dst += dst_stride; + h--; + } while (h > 0); + } else if (width > 8) { // width = 16 + __m256i p0, p1, u0, u1; + do { + p0 = _mm256_loadu_si256((const __m256i *)src); + p1 = _mm256_loadu_si256((const __m256i *)(src + src_stride)); + src += src_stride << 1; + u0 = _mm256_loadu_si256((const __m256i *)dst); + u1 = _mm256_loadu_si256((const __m256i *)(dst + dst_stride)); + + _mm256_storeu_si256((__m256i *)dst, _mm256_avg_epu16(p0, u0)); + _mm256_storeu_si256((__m256i *)(dst + dst_stride), + _mm256_avg_epu16(p1, u1)); + dst += dst_stride << 1; + h -= 2; + } while (h > 0); + } else if (width > 4) { // width = 8 + __m128i p0, p1, u0, u1; + do { + p0 = _mm_loadu_si128((const __m128i *)src); + p1 = _mm_loadu_si128((const __m128i *)(src + src_stride)); + src += src_stride << 1; + u0 = _mm_loadu_si128((const __m128i *)dst); + u1 = _mm_loadu_si128((const __m128i *)(dst + dst_stride)); + + _mm_storeu_si128((__m128i *)dst, _mm_avg_epu16(p0, u0)); + _mm_storeu_si128((__m128i *)(dst + dst_stride), _mm_avg_epu16(p1, u1)); + dst += dst_stride << 1; + h -= 2; + } while (h > 0); + } else { // width = 4 + __m128i p0, p1, u0, u1; + do { + p0 = _mm_loadl_epi64((const __m128i *)src); + p1 = _mm_loadl_epi64((const __m128i *)(src + src_stride)); + src += src_stride << 1; + u0 = _mm_loadl_epi64((const __m128i *)dst); + u1 = _mm_loadl_epi64((const __m128i *)(dst + dst_stride)); + + _mm_storel_epi64((__m128i *)dst, _mm_avg_epu16(u0, p0)); + _mm_storel_epi64((__m128i *)(dst + dst_stride), _mm_avg_epu16(u1, p1)); + dst += dst_stride << 1; + h -= 2; + } while (h > 0); + } +} + +// ----------------------------------------------------------------------------- +// Horizontal Filtering + +static INLINE void pack_pixels(const __m256i *s, __m256i *p /*p[4]*/) { + const __m256i idx = _mm256_loadu_si256((const __m256i *)signal_index); + const __m256i sf0 = _mm256_loadu_si256((const __m256i *)signal_pattern_0); + const __m256i sf1 = _mm256_loadu_si256((const __m256i *)signal_pattern_1); + const __m256i c = _mm256_permutevar8x32_epi32(*s, idx); + + p[0] = _mm256_shuffle_epi8(*s, sf0); // x0x6 + p[1] = _mm256_shuffle_epi8(*s, sf1); // x1x7 + p[2] = _mm256_shuffle_epi8(c, sf0); // x2x4 + p[3] = _mm256_shuffle_epi8(c, sf1); // x3x5 +} + +// Note: +// Shared by 8x2 and 16x1 block +static INLINE void pack_16_pixels(const __m256i *s0, const __m256i *s1, + __m256i *x /*x[8]*/) { + __m256i pp[8]; + pack_pixels(s0, pp); + pack_pixels(s1, &pp[4]); + x[0] = _mm256_permute2x128_si256(pp[0], pp[4], 0x20); + x[1] = _mm256_permute2x128_si256(pp[1], pp[5], 0x20); + x[2] = _mm256_permute2x128_si256(pp[2], pp[6], 0x20); + x[3] = _mm256_permute2x128_si256(pp[3], pp[7], 0x20); + x[4] = x[2]; + x[5] = x[3]; + x[6] = _mm256_permute2x128_si256(pp[0], pp[4], 0x31); + x[7] = _mm256_permute2x128_si256(pp[1], pp[5], 0x31); +} + +static INLINE void pack_pixels_with_format(const uint16_t *src, + PixelPackFormat fmt, + ptrdiff_t stride, __m256i *x) { + switch (fmt) { + case PACK_8x1: { + __m256i pp[8]; + __m256i s0; + s0 = _mm256_loadu_si256((const __m256i *)src); + pack_pixels(&s0, pp); + x[0] = _mm256_permute2x128_si256(pp[0], pp[2], 0x30); + x[1] = _mm256_permute2x128_si256(pp[1], pp[3], 0x30); + x[2] = _mm256_permute2x128_si256(pp[2], pp[0], 0x30); + x[3] = _mm256_permute2x128_si256(pp[3], pp[1], 0x30); + break; + } + case PACK_8x2: { + __m256i s0, s1; + s0 = _mm256_loadu_si256((const __m256i *)src); + s1 = _mm256_loadu_si256((const __m256i *)(src + stride)); + pack_16_pixels(&s0, &s1, x); + break; + } + case PACK_16x1: { + __m256i s0, s1; + s0 = _mm256_loadu_si256((const __m256i *)src); + s1 = _mm256_loadu_si256((const __m256i *)(src + 8)); + pack_16_pixels(&s0, &s1, x); + break; + } + default: { assert(0); } + } +} + +static INLINE void pack_8x1_pixels(const uint16_t *src, const ptrdiff_t pitch, + __m256i *x /*x[4]*/) { + pack_pixels_with_format(src, PACK_8x1, pitch, x); +} + +static INLINE void pack_8x2_pixels(const uint16_t *src, const ptrdiff_t pitch, + __m256i *x /*x[8]*/) { + pack_pixels_with_format(src, PACK_8x2, pitch, x); +} + +static INLINE void pack_16x1_pixels(const uint16_t *src, const ptrdiff_t pitch, + __m256i *x /*x[8]*/) { + pack_pixels_with_format(src, PACK_16x1, pitch, x); +} + +// Note: +// Shared by horizontal and vertical filtering +static INLINE void pack_filters(const int16_t *filter, __m256i *f /*f[4]*/) { + const __m128i h = _mm_loadu_si128((const __m128i *)filter); + const __m256i hh = _mm256_insertf128_si256(_mm256_castsi128_si256(h), h, 1); + const __m256i p0 = _mm256_set1_epi32(0x03020100); + const __m256i p1 = _mm256_set1_epi32(0x07060504); + const __m256i p2 = _mm256_set1_epi32(0x0b0a0908); + const __m256i p3 = _mm256_set1_epi32(0x0f0e0d0c); + f[0] = _mm256_shuffle_epi8(hh, p0); + f[1] = _mm256_shuffle_epi8(hh, p1); + f[2] = _mm256_shuffle_epi8(hh, p2); + f[3] = _mm256_shuffle_epi8(hh, p3); +} + +static INLINE void filter_8x1_pixels(const __m256i *sig /*sig[4]*/, + const __m256i *fil /*fil[4]*/, + __m256i *y) { + __m256i a, a0, a1; + + a0 = _mm256_madd_epi16(fil[0], sig[0]); + a1 = _mm256_madd_epi16(fil[3], sig[3]); + a = _mm256_add_epi32(a0, a1); + + a0 = _mm256_madd_epi16(fil[1], sig[1]); + a1 = _mm256_madd_epi16(fil[2], sig[2]); + + const __m256i min = _mm256_min_epi32(a0, a1); + a = _mm256_add_epi32(a, min); + + const __m256i max = _mm256_max_epi32(a0, a1); + a = _mm256_add_epi32(a, max); + + const __m256i rounding = _mm256_set1_epi32(1 << (CONV8_ROUNDING_BITS - 1)); + a = _mm256_add_epi32(a, rounding); + *y = _mm256_srai_epi32(a, CONV8_ROUNDING_BITS); +} + +static void write_8x1_pixels(const __m256i *y, const __m256i *z, + const __m256i *mask, uint16_t *dst, + ptrdiff_t pitch) { + const __m128i a0 = _mm256_castsi256_si128(*y); + const __m128i a1 = _mm256_extractf128_si256(*y, 1); + __m128i res = _mm_packus_epi32(a0, a1); + (void)z; + (void)pitch; + res = _mm_min_epi16(res, _mm256_castsi256_si128(*mask)); + _mm_storeu_si128((__m128i *)dst, res); +} + +static void write_8x2_pixels(const __m256i *y0, const __m256i *y1, + const __m256i *mask, uint16_t *dst, + ptrdiff_t pitch) { + __m256i a = _mm256_packus_epi32(*y0, *y1); + a = _mm256_min_epi16(a, *mask); + _mm_storeu_si128((__m128i *)dst, _mm256_castsi256_si128(a)); + _mm_storeu_si128((__m128i *)(dst + pitch), _mm256_extractf128_si256(a, 1)); +} + +static void write_16x1_pixels(const __m256i *y0, const __m256i *y1, + const __m256i *mask, uint16_t *dst, + ptrdiff_t dst_pitch) { + (void)dst_pitch; + __m256i a = _mm256_packus_epi32(*y0, *y1); + a = _mm256_min_epi16(a, *mask); + _mm256_storeu_si256((__m256i *)dst, a); +} + +static void filter_block_width8_horiz( + const uint16_t *src_ptr, ptrdiff_t src_pitch, const WritePixels write_8x1, + const WritePixels write_8x2, uint16_t *dst_ptr, ptrdiff_t dst_pitch, + uint32_t height, const int16_t *filter, int bd) { + __m256i signal[8], res0, res1; + const __m256i max = _mm256_set1_epi16((1 << bd) - 1); + + __m256i ff[4]; + pack_filters(filter, ff); + + src_ptr -= 3; + do { + pack_8x2_pixels(src_ptr, src_pitch, signal); + filter_8x1_pixels(signal, ff, &res0); + filter_8x1_pixels(&signal[4], ff, &res1); + write_8x2(&res0, &res1, &max, dst_ptr, dst_pitch); + height -= 2; + src_ptr += src_pitch << 1; + dst_ptr += dst_pitch << 1; + } while (height > 1); + + if (height > 0) { + pack_8x1_pixels(src_ptr, src_pitch, signal); + filter_8x1_pixels(signal, ff, &res0); + write_8x1(&res0, &res1, &max, dst_ptr, dst_pitch); + } +} + +static void aom_highbd_filter_block1d8_h8_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width8_horiz(src, src_pitch, write_8x1_pixels, write_8x2_pixels, + dst, dst_pitch, height, filter, bd); +} + +static void filter_block_width16_horiz(const uint16_t *src_ptr, + ptrdiff_t src_pitch, + const WritePixels write_16x1, + uint16_t *dst_ptr, ptrdiff_t dst_pitch, + uint32_t height, const int16_t *filter, + int bd) { + __m256i signal[8], res0, res1; + const __m256i max = _mm256_set1_epi16((1 << bd) - 1); + + __m256i ff[4]; + pack_filters(filter, ff); + + src_ptr -= 3; + do { + pack_16x1_pixels(src_ptr, src_pitch, signal); + filter_8x1_pixels(signal, ff, &res0); + filter_8x1_pixels(&signal[4], ff, &res1); + write_16x1(&res0, &res1, &max, dst_ptr, dst_pitch); + height -= 1; + src_ptr += src_pitch; + dst_ptr += dst_pitch; + } while (height > 0); +} + +static void aom_highbd_filter_block1d16_h8_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width16_horiz(src, src_pitch, write_16x1_pixels, dst, dst_pitch, + height, filter, bd); +} + +// 2-tap horizontal filtering + +static INLINE void pack_2t_filter(const int16_t *filter, __m256i *f) { + const __m128i h = _mm_loadu_si128((const __m128i *)filter); + const __m256i hh = _mm256_insertf128_si256(_mm256_castsi128_si256(h), h, 1); + const __m256i p = _mm256_set1_epi32(0x09080706); + f[0] = _mm256_shuffle_epi8(hh, p); +} + +// can be used by pack_8x2_2t_pixels() and pack_16x1_2t_pixels() +// the difference is s0/s1 specifies first and second rows or, +// first 16 samples and 8-sample shifted 16 samples +static INLINE void pack_16_2t_pixels(const __m256i *s0, const __m256i *s1, + __m256i *sig) { + const __m256i idx = _mm256_loadu_si256((const __m256i *)signal_index); + const __m256i sf2 = _mm256_loadu_si256((const __m256i *)signal_pattern_2); + __m256i x0 = _mm256_shuffle_epi8(*s0, sf2); + __m256i x1 = _mm256_shuffle_epi8(*s1, sf2); + __m256i r0 = _mm256_permutevar8x32_epi32(*s0, idx); + __m256i r1 = _mm256_permutevar8x32_epi32(*s1, idx); + r0 = _mm256_shuffle_epi8(r0, sf2); + r1 = _mm256_shuffle_epi8(r1, sf2); + sig[0] = _mm256_permute2x128_si256(x0, x1, 0x20); + sig[1] = _mm256_permute2x128_si256(r0, r1, 0x20); +} + +static INLINE void pack_8x2_2t_pixels(const uint16_t *src, + const ptrdiff_t pitch, __m256i *sig) { + const __m256i r0 = _mm256_loadu_si256((const __m256i *)src); + const __m256i r1 = _mm256_loadu_si256((const __m256i *)(src + pitch)); + pack_16_2t_pixels(&r0, &r1, sig); +} + +static INLINE void pack_16x1_2t_pixels(const uint16_t *src, + __m256i *sig /*sig[2]*/) { + const __m256i r0 = _mm256_loadu_si256((const __m256i *)src); + const __m256i r1 = _mm256_loadu_si256((const __m256i *)(src + 8)); + pack_16_2t_pixels(&r0, &r1, sig); +} + +static INLINE void pack_8x1_2t_pixels(const uint16_t *src, + __m256i *sig /*sig[2]*/) { + const __m256i idx = _mm256_loadu_si256((const __m256i *)signal_index); + const __m256i sf2 = _mm256_loadu_si256((const __m256i *)signal_pattern_2); + __m256i r0 = _mm256_loadu_si256((const __m256i *)src); + __m256i x0 = _mm256_shuffle_epi8(r0, sf2); + r0 = _mm256_permutevar8x32_epi32(r0, idx); + r0 = _mm256_shuffle_epi8(r0, sf2); + sig[0] = _mm256_permute2x128_si256(x0, r0, 0x20); +} + +// can be used by filter_8x2_2t_pixels() and filter_16x1_2t_pixels() +static INLINE void filter_16_2t_pixels(const __m256i *sig, const __m256i *f, + __m256i *y0, __m256i *y1) { + const __m256i rounding = _mm256_set1_epi32(1 << (CONV8_ROUNDING_BITS - 1)); + __m256i x0 = _mm256_madd_epi16(sig[0], *f); + __m256i x1 = _mm256_madd_epi16(sig[1], *f); + x0 = _mm256_add_epi32(x0, rounding); + x1 = _mm256_add_epi32(x1, rounding); + *y0 = _mm256_srai_epi32(x0, CONV8_ROUNDING_BITS); + *y1 = _mm256_srai_epi32(x1, CONV8_ROUNDING_BITS); +} + +static INLINE void filter_8x2_2t_pixels(const __m256i *sig, const __m256i *f, + __m256i *y0, __m256i *y1) { + filter_16_2t_pixels(sig, f, y0, y1); +} + +static INLINE void filter_16x1_2t_pixels(const __m256i *sig, const __m256i *f, + __m256i *y0, __m256i *y1) { + filter_16_2t_pixels(sig, f, y0, y1); +} + +static INLINE void filter_8x1_2t_pixels(const __m256i *sig, const __m256i *f, + __m256i *y0) { + const __m256i rounding = _mm256_set1_epi32(1 << (CONV8_ROUNDING_BITS - 1)); + __m256i x0 = _mm256_madd_epi16(sig[0], *f); + x0 = _mm256_add_epi32(x0, rounding); + *y0 = _mm256_srai_epi32(x0, CONV8_ROUNDING_BITS); +} + +static void filter_block_width8_2t_horiz( + const uint16_t *src_ptr, ptrdiff_t src_pitch, const WritePixels write_8x1, + const WritePixels write_8x2, uint16_t *dst_ptr, ptrdiff_t dst_pitch, + uint32_t height, const int16_t *filter, int bd) { + __m256i signal[2], res0, res1; + const __m256i max = _mm256_set1_epi16((1 << bd) - 1); + + __m256i ff; + pack_2t_filter(filter, &ff); + + src_ptr -= 3; + do { + pack_8x2_2t_pixels(src_ptr, src_pitch, signal); + filter_8x2_2t_pixels(signal, &ff, &res0, &res1); + write_8x2(&res0, &res1, &max, dst_ptr, dst_pitch); + height -= 2; + src_ptr += src_pitch << 1; + dst_ptr += dst_pitch << 1; + } while (height > 1); + + if (height > 0) { + pack_8x1_2t_pixels(src_ptr, signal); + filter_8x1_2t_pixels(signal, &ff, &res0); + write_8x1(&res0, &res1, &max, dst_ptr, dst_pitch); + } +} + +static void aom_highbd_filter_block1d8_h2_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width8_2t_horiz(src, src_pitch, write_8x1_pixels, + write_8x2_pixels, dst, dst_pitch, height, filter, + bd); +} + +static void filter_block_width16_2t_horiz(const uint16_t *src_ptr, + ptrdiff_t src_pitch, + const WritePixels write_16x1, + uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, + const int16_t *filter, int bd) { + __m256i signal[2], res0, res1; + const __m256i max = _mm256_set1_epi16((1 << bd) - 1); + + __m256i ff; + pack_2t_filter(filter, &ff); + + src_ptr -= 3; + do { + pack_16x1_2t_pixels(src_ptr, signal); + filter_16x1_2t_pixels(signal, &ff, &res0, &res1); + write_16x1(&res0, &res1, &max, dst_ptr, dst_pitch); + height -= 1; + src_ptr += src_pitch; + dst_ptr += dst_pitch; + } while (height > 0); +} + +static void aom_highbd_filter_block1d16_h2_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width16_2t_horiz(src, src_pitch, write_16x1_pixels, dst, + dst_pitch, height, filter, bd); +} + +// Vertical Filtering + +static void pack_8x9_init(const uint16_t *src, ptrdiff_t pitch, __m256i *sig) { + __m256i s0 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)src)); + __m256i s1 = + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(src + pitch))); + __m256i s2 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src + 2 * pitch))); + __m256i s3 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src + 3 * pitch))); + __m256i s4 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src + 4 * pitch))); + __m256i s5 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src + 5 * pitch))); + __m256i s6 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src + 6 * pitch))); + + s0 = _mm256_inserti128_si256(s0, _mm256_castsi256_si128(s1), 1); + s1 = _mm256_inserti128_si256(s1, _mm256_castsi256_si128(s2), 1); + s2 = _mm256_inserti128_si256(s2, _mm256_castsi256_si128(s3), 1); + s3 = _mm256_inserti128_si256(s3, _mm256_castsi256_si128(s4), 1); + s4 = _mm256_inserti128_si256(s4, _mm256_castsi256_si128(s5), 1); + s5 = _mm256_inserti128_si256(s5, _mm256_castsi256_si128(s6), 1); + + sig[0] = _mm256_unpacklo_epi16(s0, s1); + sig[4] = _mm256_unpackhi_epi16(s0, s1); + sig[1] = _mm256_unpacklo_epi16(s2, s3); + sig[5] = _mm256_unpackhi_epi16(s2, s3); + sig[2] = _mm256_unpacklo_epi16(s4, s5); + sig[6] = _mm256_unpackhi_epi16(s4, s5); + sig[8] = s6; +} + +static INLINE void pack_8x9_pixels(const uint16_t *src, ptrdiff_t pitch, + __m256i *sig) { + // base + 7th row + __m256i s0 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src + 7 * pitch))); + // base + 8th row + __m256i s1 = _mm256_castsi128_si256( + _mm_loadu_si128((const __m128i *)(src + 8 * pitch))); + __m256i s2 = _mm256_inserti128_si256(sig[8], _mm256_castsi256_si128(s0), 1); + __m256i s3 = _mm256_inserti128_si256(s0, _mm256_castsi256_si128(s1), 1); + sig[3] = _mm256_unpacklo_epi16(s2, s3); + sig[7] = _mm256_unpackhi_epi16(s2, s3); + sig[8] = s1; +} + +static INLINE void filter_8x9_pixels(const __m256i *sig, const __m256i *f, + __m256i *y0, __m256i *y1) { + filter_8x1_pixels(sig, f, y0); + filter_8x1_pixels(&sig[4], f, y1); +} + +static INLINE void update_pixels(__m256i *sig) { + int i; + for (i = 0; i < 3; ++i) { + sig[i] = sig[i + 1]; + sig[i + 4] = sig[i + 5]; + } +} + +static INLINE void write_8x1_pixels_ver(const __m256i *y0, const __m256i *y1, + const __m256i *mask, uint16_t *dst, + ptrdiff_t pitch) { + (void)pitch; + const __m128i v0 = _mm256_castsi256_si128(*y0); + const __m128i v1 = _mm256_castsi256_si128(*y1); + __m128i p = _mm_packus_epi32(v0, v1); + p = _mm_min_epi16(p, _mm256_castsi256_si128(*mask)); + _mm_storeu_si128((__m128i *)dst, p); +} + +static void filter_block_width8_vert(const uint16_t *src_ptr, + ptrdiff_t src_pitch, WritePixels write_8x1, + WritePixels write_8x2, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, + const int16_t *filter, int bd) { + __m256i signal[9], res0, res1; + const __m256i max = _mm256_set1_epi16((1 << bd) - 1); + + __m256i ff[4]; + pack_filters(filter, ff); + + pack_8x9_init(src_ptr, src_pitch, signal); + + do { + pack_8x9_pixels(src_ptr, src_pitch, signal); + + filter_8x9_pixels(signal, ff, &res0, &res1); + write_8x2(&res0, &res1, &max, dst_ptr, dst_pitch); + update_pixels(signal); + + src_ptr += src_pitch << 1; + dst_ptr += dst_pitch << 1; + height -= 2; + } while (height > 1); + + if (height > 0) { + pack_8x9_pixels(src_ptr, src_pitch, signal); + filter_8x9_pixels(signal, ff, &res0, &res1); + write_8x1(&res0, &res1, &max, dst_ptr, dst_pitch); + } +} + +static void aom_highbd_filter_block1d8_v8_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width8_vert(src, src_pitch, write_8x1_pixels_ver, + write_8x2_pixels, dst, dst_pitch, height, filter, + bd); +} + +static void pack_16x9_init(const uint16_t *src, ptrdiff_t pitch, __m256i *sig) { + __m256i u0, u1, u2, u3; + // load 0-6 rows + const __m256i s0 = _mm256_loadu_si256((const __m256i *)src); + const __m256i s1 = _mm256_loadu_si256((const __m256i *)(src + pitch)); + const __m256i s2 = _mm256_loadu_si256((const __m256i *)(src + 2 * pitch)); + const __m256i s3 = _mm256_loadu_si256((const __m256i *)(src + 3 * pitch)); + const __m256i s4 = _mm256_loadu_si256((const __m256i *)(src + 4 * pitch)); + const __m256i s5 = _mm256_loadu_si256((const __m256i *)(src + 5 * pitch)); + const __m256i s6 = _mm256_loadu_si256((const __m256i *)(src + 6 * pitch)); + + u0 = _mm256_permute2x128_si256(s0, s1, 0x20); // 0, 1 low + u1 = _mm256_permute2x128_si256(s0, s1, 0x31); // 0, 1 high + + u2 = _mm256_permute2x128_si256(s1, s2, 0x20); // 1, 2 low + u3 = _mm256_permute2x128_si256(s1, s2, 0x31); // 1, 2 high + + sig[0] = _mm256_unpacklo_epi16(u0, u2); + sig[4] = _mm256_unpackhi_epi16(u0, u2); + + sig[8] = _mm256_unpacklo_epi16(u1, u3); + sig[12] = _mm256_unpackhi_epi16(u1, u3); + + u0 = _mm256_permute2x128_si256(s2, s3, 0x20); + u1 = _mm256_permute2x128_si256(s2, s3, 0x31); + + u2 = _mm256_permute2x128_si256(s3, s4, 0x20); + u3 = _mm256_permute2x128_si256(s3, s4, 0x31); + + sig[1] = _mm256_unpacklo_epi16(u0, u2); + sig[5] = _mm256_unpackhi_epi16(u0, u2); + + sig[9] = _mm256_unpacklo_epi16(u1, u3); + sig[13] = _mm256_unpackhi_epi16(u1, u3); + + u0 = _mm256_permute2x128_si256(s4, s5, 0x20); + u1 = _mm256_permute2x128_si256(s4, s5, 0x31); + + u2 = _mm256_permute2x128_si256(s5, s6, 0x20); + u3 = _mm256_permute2x128_si256(s5, s6, 0x31); + + sig[2] = _mm256_unpacklo_epi16(u0, u2); + sig[6] = _mm256_unpackhi_epi16(u0, u2); + + sig[10] = _mm256_unpacklo_epi16(u1, u3); + sig[14] = _mm256_unpackhi_epi16(u1, u3); + + sig[16] = s6; +} + +static void pack_16x9_pixels(const uint16_t *src, ptrdiff_t pitch, + __m256i *sig) { + // base + 7th row + const __m256i s7 = _mm256_loadu_si256((const __m256i *)(src + 7 * pitch)); + // base + 8th row + const __m256i s8 = _mm256_loadu_si256((const __m256i *)(src + 8 * pitch)); + + __m256i u0, u1, u2, u3; + u0 = _mm256_permute2x128_si256(sig[16], s7, 0x20); + u1 = _mm256_permute2x128_si256(sig[16], s7, 0x31); + + u2 = _mm256_permute2x128_si256(s7, s8, 0x20); + u3 = _mm256_permute2x128_si256(s7, s8, 0x31); + + sig[3] = _mm256_unpacklo_epi16(u0, u2); + sig[7] = _mm256_unpackhi_epi16(u0, u2); + + sig[11] = _mm256_unpacklo_epi16(u1, u3); + sig[15] = _mm256_unpackhi_epi16(u1, u3); + + sig[16] = s8; +} + +static INLINE void filter_16x9_pixels(const __m256i *sig, const __m256i *f, + __m256i *y0, __m256i *y1) { + __m256i res[4]; + int i; + for (i = 0; i < 4; ++i) { + filter_8x1_pixels(&sig[i << 2], f, &res[i]); + } + + const __m256i l0l1 = _mm256_packus_epi32(res[0], res[1]); + const __m256i h0h1 = _mm256_packus_epi32(res[2], res[3]); + *y0 = _mm256_permute2x128_si256(l0l1, h0h1, 0x20); + *y1 = _mm256_permute2x128_si256(l0l1, h0h1, 0x31); +} + +static INLINE void write_16x2_pixels(const __m256i *y0, const __m256i *y1, + const __m256i *mask, uint16_t *dst, + ptrdiff_t pitch) { + __m256i p = _mm256_min_epi16(*y0, *mask); + _mm256_storeu_si256((__m256i *)dst, p); + p = _mm256_min_epi16(*y1, *mask); + _mm256_storeu_si256((__m256i *)(dst + pitch), p); +} + +static INLINE void write_16x1_pixels_ver(const __m256i *y0, const __m256i *y1, + const __m256i *mask, uint16_t *dst, + ptrdiff_t pitch) { + (void)y1; + (void)pitch; + const __m256i p = _mm256_min_epi16(*y0, *mask); + _mm256_storeu_si256((__m256i *)dst, p); +} + +static void update_16x9_pixels(__m256i *sig) { + update_pixels(&sig[0]); + update_pixels(&sig[8]); +} + +static void filter_block_width16_vert(const uint16_t *src_ptr, + ptrdiff_t src_pitch, + WritePixels write_16x1, + WritePixels write_16x2, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, + const int16_t *filter, int bd) { + __m256i signal[17], res0, res1; + const __m256i max = _mm256_set1_epi16((1 << bd) - 1); + + __m256i ff[4]; + pack_filters(filter, ff); + + pack_16x9_init(src_ptr, src_pitch, signal); + + do { + pack_16x9_pixels(src_ptr, src_pitch, signal); + filter_16x9_pixels(signal, ff, &res0, &res1); + write_16x2(&res0, &res1, &max, dst_ptr, dst_pitch); + update_16x9_pixels(signal); + + src_ptr += src_pitch << 1; + dst_ptr += dst_pitch << 1; + height -= 2; + } while (height > 1); + + if (height > 0) { + pack_16x9_pixels(src_ptr, src_pitch, signal); + filter_16x9_pixels(signal, ff, &res0, &res1); + write_16x1(&res0, &res1, &max, dst_ptr, dst_pitch); + } +} + +static void aom_highbd_filter_block1d16_v8_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width16_vert(src, src_pitch, write_16x1_pixels_ver, + write_16x2_pixels, dst, dst_pitch, height, filter, + bd); +} + +// 2-tap vertical filtering + +static void pack_16x2_init(const uint16_t *src, __m256i *sig) { + sig[2] = _mm256_loadu_si256((const __m256i *)src); +} + +static INLINE void pack_16x2_2t_pixels(const uint16_t *src, ptrdiff_t pitch, + __m256i *sig) { + // load the next row + const __m256i u = _mm256_loadu_si256((const __m256i *)(src + pitch)); + sig[0] = _mm256_unpacklo_epi16(sig[2], u); + sig[1] = _mm256_unpackhi_epi16(sig[2], u); + sig[2] = u; +} + +static INLINE void filter_16x2_2t_pixels(const __m256i *sig, const __m256i *f, + __m256i *y0, __m256i *y1) { + filter_16_2t_pixels(sig, f, y0, y1); +} + +static void filter_block_width16_2t_vert(const uint16_t *src_ptr, + ptrdiff_t src_pitch, + WritePixels write_16x1, + uint16_t *dst_ptr, ptrdiff_t dst_pitch, + uint32_t height, const int16_t *filter, + int bd) { + __m256i signal[3], res0, res1; + const __m256i max = _mm256_set1_epi16((1 << bd) - 1); + __m256i ff; + + pack_2t_filter(filter, &ff); + pack_16x2_init(src_ptr, signal); + + do { + pack_16x2_2t_pixels(src_ptr, src_pitch, signal); + filter_16x2_2t_pixels(signal, &ff, &res0, &res1); + write_16x1(&res0, &res1, &max, dst_ptr, dst_pitch); + + src_ptr += src_pitch; + dst_ptr += dst_pitch; + height -= 1; + } while (height > 0); +} + +static void aom_highbd_filter_block1d16_v2_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width16_2t_vert(src, src_pitch, write_16x1_pixels, dst, + dst_pitch, height, filter, bd); +} + +static INLINE void pack_8x1_2t_filter(const int16_t *filter, __m128i *f) { + const __m128i h = _mm_loadu_si128((const __m128i *)filter); + const __m128i p = _mm_set1_epi32(0x09080706); + f[0] = _mm_shuffle_epi8(h, p); +} + +static void pack_8x2_init(const uint16_t *src, __m128i *sig) { + sig[2] = _mm_loadu_si128((const __m128i *)src); +} + +static INLINE void pack_8x2_2t_pixels_ver(const uint16_t *src, ptrdiff_t pitch, + __m128i *sig) { + // load the next row + const __m128i u = _mm_loadu_si128((const __m128i *)(src + pitch)); + sig[0] = _mm_unpacklo_epi16(sig[2], u); + sig[1] = _mm_unpackhi_epi16(sig[2], u); + sig[2] = u; +} + +static INLINE void filter_8_2t_pixels(const __m128i *sig, const __m128i *f, + __m128i *y0, __m128i *y1) { + const __m128i rounding = _mm_set1_epi32(1 << (CONV8_ROUNDING_BITS - 1)); + __m128i x0 = _mm_madd_epi16(sig[0], *f); + __m128i x1 = _mm_madd_epi16(sig[1], *f); + x0 = _mm_add_epi32(x0, rounding); + x1 = _mm_add_epi32(x1, rounding); + *y0 = _mm_srai_epi32(x0, CONV8_ROUNDING_BITS); + *y1 = _mm_srai_epi32(x1, CONV8_ROUNDING_BITS); +} + +static void write_8x1_2t_pixels_ver(const __m128i *y0, const __m128i *y1, + const __m128i *mask, uint16_t *dst) { + __m128i res = _mm_packus_epi32(*y0, *y1); + res = _mm_min_epi16(res, *mask); + _mm_storeu_si128((__m128i *)dst, res); +} + +typedef void (*Write8Pixels)(const __m128i *y0, const __m128i *y1, + const __m128i *mask, uint16_t *dst); + +static void filter_block_width8_2t_vert(const uint16_t *src_ptr, + ptrdiff_t src_pitch, + Write8Pixels write_8x1, + uint16_t *dst_ptr, ptrdiff_t dst_pitch, + uint32_t height, const int16_t *filter, + int bd) { + __m128i signal[3], res0, res1; + const __m128i max = _mm_set1_epi16((1 << bd) - 1); + __m128i ff; + + pack_8x1_2t_filter(filter, &ff); + pack_8x2_init(src_ptr, signal); + + do { + pack_8x2_2t_pixels_ver(src_ptr, src_pitch, signal); + filter_8_2t_pixels(signal, &ff, &res0, &res1); + write_8x1(&res0, &res1, &max, dst_ptr); + + src_ptr += src_pitch; + dst_ptr += dst_pitch; + height -= 1; + } while (height > 0); +} + +static void aom_highbd_filter_block1d8_v2_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width8_2t_vert(src, src_pitch, write_8x1_2t_pixels_ver, dst, + dst_pitch, height, filter, bd); +} + +// Calculation with averaging the input pixels + +static void write_8x1_avg_pixels(const __m256i *y0, const __m256i *y1, + const __m256i *mask, uint16_t *dst, + ptrdiff_t pitch) { + (void)y1; + (void)pitch; + const __m128i a0 = _mm256_castsi256_si128(*y0); + const __m128i a1 = _mm256_extractf128_si256(*y0, 1); + __m128i res = _mm_packus_epi32(a0, a1); + const __m128i pix = _mm_loadu_si128((const __m128i *)dst); + res = _mm_min_epi16(res, _mm256_castsi256_si128(*mask)); + res = _mm_avg_epu16(res, pix); + _mm_storeu_si128((__m128i *)dst, res); +} + +static void write_8x2_avg_pixels(const __m256i *y0, const __m256i *y1, + const __m256i *mask, uint16_t *dst, + ptrdiff_t pitch) { + __m256i a = _mm256_packus_epi32(*y0, *y1); + const __m128i pix0 = _mm_loadu_si128((const __m128i *)dst); + const __m128i pix1 = _mm_loadu_si128((const __m128i *)(dst + pitch)); + const __m256i pix = + _mm256_insertf128_si256(_mm256_castsi128_si256(pix0), pix1, 1); + a = _mm256_min_epi16(a, *mask); + a = _mm256_avg_epu16(a, pix); + _mm_storeu_si128((__m128i *)dst, _mm256_castsi256_si128(a)); + _mm_storeu_si128((__m128i *)(dst + pitch), _mm256_extractf128_si256(a, 1)); +} + +static void write_16x1_avg_pixels(const __m256i *y0, const __m256i *y1, + const __m256i *mask, uint16_t *dst, + ptrdiff_t pitch) { + (void)pitch; + __m256i a = _mm256_packus_epi32(*y0, *y1); + const __m256i pix = _mm256_loadu_si256((const __m256i *)dst); + a = _mm256_min_epi16(a, *mask); + a = _mm256_avg_epu16(a, pix); + _mm256_storeu_si256((__m256i *)dst, a); +} + +static INLINE void write_8x1_avg_pixels_ver(const __m256i *y0, + const __m256i *y1, + const __m256i *mask, uint16_t *dst, + ptrdiff_t pitch) { + (void)pitch; + const __m128i v0 = _mm256_castsi256_si128(*y0); + const __m128i v1 = _mm256_castsi256_si128(*y1); + __m128i p = _mm_packus_epi32(v0, v1); + const __m128i pix = _mm_loadu_si128((const __m128i *)dst); + p = _mm_min_epi16(p, _mm256_castsi256_si128(*mask)); + p = _mm_avg_epu16(p, pix); + _mm_storeu_si128((__m128i *)dst, p); +} + +static INLINE void write_16x2_avg_pixels(const __m256i *y0, const __m256i *y1, + const __m256i *mask, uint16_t *dst, + ptrdiff_t pitch) { + const __m256i pix0 = _mm256_loadu_si256((const __m256i *)dst); + const __m256i pix1 = _mm256_loadu_si256((const __m256i *)(dst + pitch)); + __m256i p = _mm256_min_epi16(*y0, *mask); + p = _mm256_avg_epu16(p, pix0); + _mm256_storeu_si256((__m256i *)dst, p); + + p = _mm256_min_epi16(*y1, *mask); + p = _mm256_avg_epu16(p, pix1); + _mm256_storeu_si256((__m256i *)(dst + pitch), p); +} + +static INLINE void write_16x1_avg_pixels_ver(const __m256i *y0, + const __m256i *y1, + const __m256i *mask, uint16_t *dst, + ptrdiff_t pitch) { + (void)y1; + (void)pitch; + __m256i p = _mm256_min_epi16(*y0, *mask); + const __m256i pix = _mm256_loadu_si256((const __m256i *)dst); + p = _mm256_avg_epu16(p, pix); + _mm256_storeu_si256((__m256i *)dst, p); +} + +static void write_8x1_2t_avg_pixels_ver(const __m128i *y0, const __m128i *y1, + const __m128i *mask, uint16_t *dst) { + __m128i res = _mm_packus_epi32(*y0, *y1); + const __m128i pix = _mm_loadu_si128((const __m128i *)dst); + res = _mm_min_epi16(res, *mask); + res = _mm_avg_epu16(res, pix); + _mm_storeu_si128((__m128i *)dst, res); +} + +static void aom_highbd_filter_block1d8_h8_avg_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width8_horiz(src, src_pitch, write_8x1_avg_pixels, + write_8x2_avg_pixels, dst, dst_pitch, height, + filter, bd); +} + +static void aom_highbd_filter_block1d16_h8_avg_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width16_horiz(src, src_pitch, write_16x1_avg_pixels, dst, + dst_pitch, height, filter, bd); +} + +static void aom_highbd_filter_block1d8_v8_avg_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width8_vert(src, src_pitch, write_8x1_avg_pixels_ver, + write_8x2_avg_pixels, dst, dst_pitch, height, filter, + bd); +} + +static void aom_highbd_filter_block1d16_v8_avg_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width16_vert(src, src_pitch, write_16x1_avg_pixels_ver, + write_16x2_avg_pixels, dst, dst_pitch, height, + filter, bd); +} + +// 2-tap averaging + +static void aom_highbd_filter_block1d8_h2_avg_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width8_2t_horiz(src, src_pitch, write_8x1_avg_pixels, + write_8x2_avg_pixels, dst, dst_pitch, height, + filter, bd); +} + +static void aom_highbd_filter_block1d16_h2_avg_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width16_2t_horiz(src, src_pitch, write_16x1_avg_pixels, dst, + dst_pitch, height, filter, bd); +} + +static void aom_highbd_filter_block1d16_v2_avg_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width16_2t_vert(src, src_pitch, write_16x1_avg_pixels, dst, + dst_pitch, height, filter, bd); +} + +static void aom_highbd_filter_block1d8_v2_avg_avx2( + const uint16_t *src, ptrdiff_t src_pitch, uint16_t *dst, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + filter_block_width8_2t_vert(src, src_pitch, write_8x1_2t_avg_pixels_ver, dst, + dst_pitch, height, filter, bd); +} + +typedef void HbdFilter1dFunc(const uint16_t *, ptrdiff_t, uint16_t *, ptrdiff_t, + uint32_t, const int16_t *, int); + +#define HIGHBD_FUNC(width, dir, avg, opt) \ + aom_highbd_filter_block1d##width##_##dir##_##avg##opt + +HbdFilter1dFunc HIGHBD_FUNC(4, h8, , sse2); +HbdFilter1dFunc HIGHBD_FUNC(4, h2, , sse2); +HbdFilter1dFunc HIGHBD_FUNC(4, v8, , sse2); +HbdFilter1dFunc HIGHBD_FUNC(4, v2, , sse2); + +#define aom_highbd_filter_block1d4_h8_avx2 HIGHBD_FUNC(4, h8, , sse2) +#define aom_highbd_filter_block1d4_h2_avx2 HIGHBD_FUNC(4, h2, , sse2) +#define aom_highbd_filter_block1d4_v8_avx2 HIGHBD_FUNC(4, v8, , sse2) +#define aom_highbd_filter_block1d4_v2_avx2 HIGHBD_FUNC(4, v2, , sse2) + +HIGH_FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , avx2); +HIGH_FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , avx2); +HIGH_FUN_CONV_2D(, avx2); + +HbdFilter1dFunc HIGHBD_FUNC(4, h8, avg_, sse2); +HbdFilter1dFunc HIGHBD_FUNC(4, h2, avg_, sse2); +HbdFilter1dFunc HIGHBD_FUNC(4, v8, avg_, sse2); +HbdFilter1dFunc HIGHBD_FUNC(4, v2, avg_, sse2); + +#define aom_highbd_filter_block1d4_h8_avg_avx2 HIGHBD_FUNC(4, h8, avg_, sse2) +#define aom_highbd_filter_block1d4_h2_avg_avx2 HIGHBD_FUNC(4, h2, avg_, sse2) +#define aom_highbd_filter_block1d4_v8_avg_avx2 HIGHBD_FUNC(4, v8, avg_, sse2) +#define aom_highbd_filter_block1d4_v2_avg_avx2 HIGHBD_FUNC(4, v2, avg_, sse2) + +HIGH_FUN_CONV_1D(avg_horiz, x_step_q4, filter_x, h, src, avg_, avx2); +HIGH_FUN_CONV_1D(avg_vert, y_step_q4, filter_y, v, src - src_stride * 3, avg_, + avx2); +HIGH_FUN_CONV_2D(avg_, avx2); + +#undef HIGHBD_FUNC diff --git a/third_party/aom/aom_dsp/x86/highbd_intrapred_sse2.asm b/third_party/aom/aom_dsp/x86/highbd_intrapred_sse2.asm new file mode 100644 index 0000000000..5d84ef8a75 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_intrapred_sse2.asm @@ -0,0 +1,456 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION_RODATA +pw_4: times 8 dw 4 +pw_8: times 8 dw 8 +pw_16: times 4 dd 16 +pw_32: times 4 dd 32 + +SECTION .text +INIT_XMM sse2 +cglobal highbd_dc_predictor_4x4, 4, 5, 4, dst, stride, above, left, goffset + GET_GOT goffsetq + + movq m0, [aboveq] + movq m2, [leftq] + paddw m0, m2 + pshuflw m1, m0, 0xe + paddw m0, m1 + pshuflw m1, m0, 0x1 + paddw m0, m1 + paddw m0, [GLOBAL(pw_4)] + psraw m0, 3 + pshuflw m0, m0, 0x0 + movq [dstq ], m0 + movq [dstq+strideq*2], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq*2], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal highbd_dc_predictor_8x8, 4, 5, 4, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [aboveq] + mova m2, [leftq] + DEFINE_ARGS dst, stride, stride3, one + mov oned, 0x00010001 + lea stride3q, [strideq*3] + movd m3, oned + pshufd m3, m3, 0x0 + paddw m0, m2 + pmaddwd m0, m3 + packssdw m0, m1 + pmaddwd m0, m3 + packssdw m0, m1 + pmaddwd m0, m3 + paddw m0, [GLOBAL(pw_8)] + psrlw m0, 4 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + mova [dstq ], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*4 ], m0 + mova [dstq+stride3q*2], m0 + lea dstq, [dstq+strideq*8] + mova [dstq ], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*4 ], m0 + mova [dstq+stride3q*2], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal highbd_dc_predictor_16x16, 4, 5, 5, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [aboveq] + mova m3, [aboveq+16] + mova m2, [leftq] + mova m4, [leftq+16] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 4 + paddw m0, m2 + paddw m0, m3 + paddw m0, m4 + movhlps m2, m0 + paddw m0, m2 + punpcklwd m0, m1 + movhlps m2, m0 + paddd m0, m2 + punpckldq m0, m1 + movhlps m2, m0 + paddd m0, m2 + paddd m0, [GLOBAL(pw_16)] + psrad m0, 5 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq +16], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2 +16], m0 + mova [dstq+strideq*4 ], m0 + mova [dstq+strideq*4 +16], m0 + mova [dstq+stride3q*2 ], m0 + mova [dstq+stride3q*2+16], m0 + lea dstq, [dstq+strideq*8] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal highbd_dc_predictor_32x32, 4, 5, 7, dst, stride, above, left, goffset + GET_GOT goffsetq + + mova m0, [aboveq] + mova m2, [aboveq+16] + mova m3, [aboveq+32] + mova m4, [aboveq+48] + paddw m0, m2 + paddw m3, m4 + mova m2, [leftq] + mova m4, [leftq+16] + mova m5, [leftq+32] + mova m6, [leftq+48] + paddw m2, m4 + paddw m5, m6 + paddw m0, m3 + paddw m2, m5 + pxor m1, m1 + paddw m0, m2 + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 8 + movhlps m2, m0 + paddw m0, m2 + punpcklwd m0, m1 + movhlps m2, m0 + paddd m0, m2 + punpckldq m0, m1 + movhlps m2, m0 + paddd m0, m2 + paddd m0, [GLOBAL(pw_32)] + psrad m0, 6 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq +16 ], m0 + mova [dstq +32 ], m0 + mova [dstq +48 ], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16 ], m0 + mova [dstq+strideq*2+32 ], m0 + mova [dstq+strideq*2+48 ], m0 + mova [dstq+strideq*4 ], m0 + mova [dstq+strideq*4+16 ], m0 + mova [dstq+strideq*4+32 ], m0 + mova [dstq+strideq*4+48 ], m0 + mova [dstq+stride3q*2 ], m0 + mova [dstq+stride3q*2 +16], m0 + mova [dstq+stride3q*2 +32], m0 + mova [dstq+stride3q*2 +48], m0 + lea dstq, [dstq+strideq*8] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal highbd_v_predictor_4x4, 3, 3, 1, dst, stride, above + movq m0, [aboveq] + movq [dstq ], m0 + movq [dstq+strideq*2], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq*2], m0 + RET + +INIT_XMM sse2 +cglobal highbd_v_predictor_8x8, 3, 3, 1, dst, stride, above + mova m0, [aboveq] + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + mova [dstq ], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*4 ], m0 + mova [dstq+stride3q*2], m0 + lea dstq, [dstq+strideq*8] + mova [dstq ], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*4 ], m0 + mova [dstq+stride3q*2], m0 + RET + +INIT_XMM sse2 +cglobal highbd_v_predictor_16x16, 3, 4, 2, dst, stride, above + mova m0, [aboveq] + mova m1, [aboveq+16] + DEFINE_ARGS dst, stride, stride3, nlines4 + lea stride3q, [strideq*3] + mov nlines4d, 4 +.loop: + mova [dstq ], m0 + mova [dstq +16], m1 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2 +16], m1 + mova [dstq+strideq*4 ], m0 + mova [dstq+strideq*4 +16], m1 + mova [dstq+stride3q*2 ], m0 + mova [dstq+stride3q*2+16], m1 + lea dstq, [dstq+strideq*8] + dec nlines4d + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal highbd_v_predictor_32x32, 3, 4, 4, dst, stride, above + mova m0, [aboveq] + mova m1, [aboveq+16] + mova m2, [aboveq+32] + mova m3, [aboveq+48] + DEFINE_ARGS dst, stride, stride3, nlines4 + lea stride3q, [strideq*3] + mov nlines4d, 8 +.loop: + mova [dstq ], m0 + mova [dstq +16], m1 + mova [dstq +32], m2 + mova [dstq +48], m3 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2 +16], m1 + mova [dstq+strideq*2 +32], m2 + mova [dstq+strideq*2 +48], m3 + mova [dstq+strideq*4 ], m0 + mova [dstq+strideq*4 +16], m1 + mova [dstq+strideq*4 +32], m2 + mova [dstq+strideq*4 +48], m3 + mova [dstq+stride3q*2 ], m0 + mova [dstq+stride3q*2 +16], m1 + mova [dstq+stride3q*2 +32], m2 + mova [dstq+stride3q*2 +48], m3 + lea dstq, [dstq+strideq*8] + dec nlines4d + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal highbd_tm_predictor_4x4, 5, 5, 6, dst, stride, above, left, bps + movd m1, [aboveq-2] + movq m0, [aboveq] + pshuflw m1, m1, 0x0 + movlhps m0, m0 ; t1 t2 t3 t4 t1 t2 t3 t4 + movlhps m1, m1 ; tl tl tl tl tl tl tl tl + ; Get the values to compute the maximum value at this bit depth + pcmpeqw m3, m3 + movd m4, bpsd + psubw m0, m1 ; t1-tl t2-tl t3-tl t4-tl + psllw m3, m4 + pcmpeqw m2, m2 + pxor m4, m4 ; min possible value + pxor m3, m2 ; max possible value + mova m1, [leftq] + pshuflw m2, m1, 0x0 + pshuflw m5, m1, 0x55 + movlhps m2, m5 ; l1 l1 l1 l1 l2 l2 l2 l2 + paddw m2, m0 + ;Clamp to the bit-depth + pminsw m2, m3 + pmaxsw m2, m4 + ;Store the values + movq [dstq ], m2 + movhpd [dstq+strideq*2], m2 + lea dstq, [dstq+strideq*4] + pshuflw m2, m1, 0xaa + pshuflw m5, m1, 0xff + movlhps m2, m5 + paddw m2, m0 + ;Clamp to the bit-depth + pminsw m2, m3 + pmaxsw m2, m4 + ;Store the values + movq [dstq ], m2 + movhpd [dstq+strideq*2], m2 + RET + +INIT_XMM sse2 +cglobal highbd_tm_predictor_8x8, 5, 6, 5, dst, stride, above, left, bps, one + movd m1, [aboveq-2] + mova m0, [aboveq] + pshuflw m1, m1, 0x0 + ; Get the values to compute the maximum value at this bit depth + mov oned, 1 + pxor m3, m3 + pxor m4, m4 + pinsrw m3, oned, 0 + pinsrw m4, bpsd, 0 + pshuflw m3, m3, 0x0 + DEFINE_ARGS dst, stride, line, left + punpcklqdq m3, m3 + mov lineq, -4 + mova m2, m3 + punpcklqdq m1, m1 + psllw m3, m4 + add leftq, 16 + psubw m3, m2 ; max possible value + pxor m4, m4 ; min possible value + psubw m0, m1 +.loop: + movd m1, [leftq+lineq*4] + movd m2, [leftq+lineq*4+2] + pshuflw m1, m1, 0x0 + pshuflw m2, m2, 0x0 + punpcklqdq m1, m1 + punpcklqdq m2, m2 + paddw m1, m0 + paddw m2, m0 + ;Clamp to the bit-depth + pminsw m1, m3 + pminsw m2, m3 + pmaxsw m1, m4 + pmaxsw m2, m4 + ;Store the values + mova [dstq ], m1 + mova [dstq+strideq*2], m2 + lea dstq, [dstq+strideq*4] + inc lineq + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal highbd_tm_predictor_16x16, 5, 5, 8, dst, stride, above, left, bps + movd m2, [aboveq-2] + mova m0, [aboveq] + mova m1, [aboveq+16] + pshuflw m2, m2, 0x0 + ; Get the values to compute the maximum value at this bit depth + pcmpeqw m3, m3 + movd m4, bpsd + punpcklqdq m2, m2 + psllw m3, m4 + pcmpeqw m5, m5 + pxor m4, m4 ; min possible value + pxor m3, m5 ; max possible value + DEFINE_ARGS dst, stride, line, left + mov lineq, -8 + psubw m0, m2 + psubw m1, m2 +.loop: + movd m7, [leftq] + pshuflw m5, m7, 0x0 + pshuflw m2, m7, 0x55 + punpcklqdq m5, m5 ; l1 l1 l1 l1 l1 l1 l1 l1 + punpcklqdq m2, m2 ; l2 l2 l2 l2 l2 l2 l2 l2 + paddw m6, m5, m0 ; t1-tl+l1 to t4-tl+l1 + paddw m5, m1 ; t5-tl+l1 to t8-tl+l1 + pminsw m6, m3 + pminsw m5, m3 + pmaxsw m6, m4 ; Clamp to the bit-depth + pmaxsw m5, m4 + mova [dstq ], m6 + mova [dstq +16], m5 + paddw m6, m2, m0 + paddw m2, m1 + pminsw m6, m3 + pminsw m2, m3 + pmaxsw m6, m4 + pmaxsw m2, m4 + mova [dstq+strideq*2 ], m6 + mova [dstq+strideq*2+16], m2 + lea dstq, [dstq+strideq*4] + inc lineq + lea leftq, [leftq+4] + + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal highbd_tm_predictor_32x32, 5, 5, 8, dst, stride, above, left, bps + movd m0, [aboveq-2] + mova m1, [aboveq] + mova m2, [aboveq+16] + mova m3, [aboveq+32] + mova m4, [aboveq+48] + pshuflw m0, m0, 0x0 + ; Get the values to compute the maximum value at this bit depth + pcmpeqw m5, m5 + movd m6, bpsd + psllw m5, m6 + pcmpeqw m7, m7 + pxor m6, m6 ; min possible value + pxor m5, m7 ; max possible value + punpcklqdq m0, m0 + DEFINE_ARGS dst, stride, line, left + mov lineq, -16 + psubw m1, m0 + psubw m2, m0 + psubw m3, m0 + psubw m4, m0 +.loop: + movd m7, [leftq] + pshuflw m7, m7, 0x0 + punpcklqdq m7, m7 ; l1 l1 l1 l1 l1 l1 l1 l1 + paddw m0, m7, m1 + pminsw m0, m5 + pmaxsw m0, m6 + mova [dstq ], m0 + paddw m0, m7, m2 + pminsw m0, m5 + pmaxsw m0, m6 + mova [dstq +16], m0 + paddw m0, m7, m3 + pminsw m0, m5 + pmaxsw m0, m6 + mova [dstq +32], m0 + paddw m0, m7, m4 + pminsw m0, m5 + pmaxsw m0, m6 + mova [dstq +48], m0 + movd m7, [leftq+2] + pshuflw m7, m7, 0x0 + punpcklqdq m7, m7 ; l2 l2 l2 l2 l2 l2 l2 l2 + paddw m0, m7, m1 + pminsw m0, m5 + pmaxsw m0, m6 + mova [dstq+strideq*2 ], m0 + paddw m0, m7, m2 + pminsw m0, m5 + pmaxsw m0, m6 + mova [dstq+strideq*2+16], m0 + paddw m0, m7, m3 + pminsw m0, m5 + pmaxsw m0, m6 + mova [dstq+strideq*2+32], m0 + paddw m0, m7, m4 + pminsw m0, m5 + pmaxsw m0, m6 + mova [dstq+strideq*2+48], m0 + lea dstq, [dstq+strideq*4] + lea leftq, [leftq+4] + inc lineq + jnz .loop + REP_RET diff --git a/third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c b/third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c new file mode 100644 index 0000000000..76369871ba --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c @@ -0,0 +1,1140 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // SSE2 + +#include "./aom_dsp_rtcd.h" +#include "aom_ports/mem.h" +#include "aom_ports/emmintrin_compat.h" + +static INLINE __m128i signed_char_clamp_bd_sse2(__m128i value, int bd) { + __m128i ubounded; + __m128i lbounded; + __m128i retval; + + const __m128i zero = _mm_set1_epi16(0); + const __m128i one = _mm_set1_epi16(1); + __m128i t80, max, min; + + if (bd == 8) { + t80 = _mm_set1_epi16(0x80); + max = _mm_subs_epi16(_mm_subs_epi16(_mm_slli_epi16(one, 8), one), t80); + } else if (bd == 10) { + t80 = _mm_set1_epi16(0x200); + max = _mm_subs_epi16(_mm_subs_epi16(_mm_slli_epi16(one, 10), one), t80); + } else { // bd == 12 + t80 = _mm_set1_epi16(0x800); + max = _mm_subs_epi16(_mm_subs_epi16(_mm_slli_epi16(one, 12), one), t80); + } + + min = _mm_subs_epi16(zero, t80); + + ubounded = _mm_cmpgt_epi16(value, max); + lbounded = _mm_cmplt_epi16(value, min); + retval = _mm_andnot_si128(_mm_or_si128(ubounded, lbounded), value); + ubounded = _mm_and_si128(ubounded, max); + lbounded = _mm_and_si128(lbounded, min); + retval = _mm_or_si128(retval, ubounded); + retval = _mm_or_si128(retval, lbounded); + return retval; +} + +// TODO(debargha, peter): Break up large functions into smaller ones +// in this file. +void aom_highbd_lpf_horizontal_edge_8_sse2(uint16_t *s, int p, + const uint8_t *_blimit, + const uint8_t *_limit, + const uint8_t *_thresh, int bd) { + const __m128i zero = _mm_set1_epi16(0); + const __m128i one = _mm_set1_epi16(1); + __m128i blimit, limit, thresh; + __m128i q7, p7, q6, p6, q5, p5, q4, p4, q3, p3, q2, p2, q1, p1, q0, p0; + __m128i mask, hev, flat, flat2, abs_p1p0, abs_q1q0; + __m128i ps1, qs1, ps0, qs0; + __m128i abs_p0q0, abs_p1q1, ffff, work; + __m128i filt, work_a, filter1, filter2; + __m128i flat2_q6, flat2_p6, flat2_q5, flat2_p5, flat2_q4, flat2_p4; + __m128i flat2_q3, flat2_p3, flat2_q2, flat2_p2, flat2_q1, flat2_p1; + __m128i flat2_q0, flat2_p0; + __m128i flat_q2, flat_p2, flat_q1, flat_p1, flat_q0, flat_p0; + __m128i pixelFilter_p, pixelFilter_q; + __m128i pixetFilter_p2p1p0, pixetFilter_q2q1q0; + __m128i sum_p7, sum_q7, sum_p3, sum_q3; + __m128i t4, t3, t80, t1; + __m128i eight, four; + + if (bd == 8) { + blimit = _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_blimit), zero); + limit = _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_limit), zero); + thresh = _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_thresh), zero); + } else if (bd == 10) { + blimit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_blimit), zero), 2); + limit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_limit), zero), 2); + thresh = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_thresh), zero), 2); + } else { // bd == 12 + blimit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_blimit), zero), 4); + limit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_limit), zero), 4); + thresh = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_thresh), zero), 4); + } + + q4 = _mm_load_si128((__m128i *)(s + 4 * p)); + p4 = _mm_load_si128((__m128i *)(s - 5 * p)); + q3 = _mm_load_si128((__m128i *)(s + 3 * p)); + p3 = _mm_load_si128((__m128i *)(s - 4 * p)); + q2 = _mm_load_si128((__m128i *)(s + 2 * p)); + p2 = _mm_load_si128((__m128i *)(s - 3 * p)); + q1 = _mm_load_si128((__m128i *)(s + 1 * p)); + p1 = _mm_load_si128((__m128i *)(s - 2 * p)); + q0 = _mm_load_si128((__m128i *)(s + 0 * p)); + p0 = _mm_load_si128((__m128i *)(s - 1 * p)); + + // highbd_filter_mask + abs_p1p0 = _mm_or_si128(_mm_subs_epu16(p1, p0), _mm_subs_epu16(p0, p1)); + abs_q1q0 = _mm_or_si128(_mm_subs_epu16(q1, q0), _mm_subs_epu16(q0, q1)); + + ffff = _mm_cmpeq_epi16(abs_p1p0, abs_p1p0); + + abs_p0q0 = _mm_or_si128(_mm_subs_epu16(p0, q0), _mm_subs_epu16(q0, p0)); + abs_p1q1 = _mm_or_si128(_mm_subs_epu16(p1, q1), _mm_subs_epu16(q1, p1)); + + // highbd_hev_mask (in C code this is actually called from highbd_filter4) + flat = _mm_max_epi16(abs_p1p0, abs_q1q0); + hev = _mm_subs_epu16(flat, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi16(hev, zero), ffff); + + abs_p0q0 = _mm_adds_epu16(abs_p0q0, abs_p0q0); // abs(p0 - q0) * 2 + abs_p1q1 = _mm_srli_epi16(abs_p1q1, 1); // abs(p1 - q1) / 2 + mask = _mm_subs_epu16(_mm_adds_epu16(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi16(mask, zero), ffff); + mask = _mm_and_si128(mask, _mm_adds_epu16(limit, one)); + work = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p1, p0), _mm_subs_epu16(p0, p1)), + _mm_or_si128(_mm_subs_epu16(q1, q0), _mm_subs_epu16(q0, q1))); + mask = _mm_max_epi16(work, mask); + work = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p2, p1), _mm_subs_epu16(p1, p2)), + _mm_or_si128(_mm_subs_epu16(q2, q1), _mm_subs_epu16(q1, q2))); + mask = _mm_max_epi16(work, mask); + work = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p3, p2), _mm_subs_epu16(p2, p3)), + _mm_or_si128(_mm_subs_epu16(q3, q2), _mm_subs_epu16(q2, q3))); + mask = _mm_max_epi16(work, mask); + + mask = _mm_subs_epu16(mask, limit); + mask = _mm_cmpeq_epi16(mask, zero); // return ~mask + + // lp filter + // highbd_filter4 + t4 = _mm_set1_epi16(4); + t3 = _mm_set1_epi16(3); + if (bd == 8) + t80 = _mm_set1_epi16(0x80); + else if (bd == 10) + t80 = _mm_set1_epi16(0x200); + else // bd == 12 + t80 = _mm_set1_epi16(0x800); + + t1 = _mm_set1_epi16(0x1); + + ps1 = _mm_subs_epi16(p1, t80); + qs1 = _mm_subs_epi16(q1, t80); + ps0 = _mm_subs_epi16(p0, t80); + qs0 = _mm_subs_epi16(q0, t80); + + filt = _mm_and_si128(signed_char_clamp_bd_sse2(_mm_subs_epi16(ps1, qs1), bd), + hev); + work_a = _mm_subs_epi16(qs0, ps0); + filt = _mm_adds_epi16(filt, work_a); + filt = _mm_adds_epi16(filt, work_a); + filt = signed_char_clamp_bd_sse2(_mm_adds_epi16(filt, work_a), bd); + filt = _mm_and_si128(filt, mask); + filter1 = signed_char_clamp_bd_sse2(_mm_adds_epi16(filt, t4), bd); + filter2 = signed_char_clamp_bd_sse2(_mm_adds_epi16(filt, t3), bd); + + // Filter1 >> 3 + filter1 = _mm_srai_epi16(filter1, 0x3); + filter2 = _mm_srai_epi16(filter2, 0x3); + + qs0 = _mm_adds_epi16( + signed_char_clamp_bd_sse2(_mm_subs_epi16(qs0, filter1), bd), t80); + ps0 = _mm_adds_epi16( + signed_char_clamp_bd_sse2(_mm_adds_epi16(ps0, filter2), bd), t80); + filt = _mm_adds_epi16(filter1, t1); + filt = _mm_srai_epi16(filt, 1); + filt = _mm_andnot_si128(hev, filt); + qs1 = _mm_adds_epi16(signed_char_clamp_bd_sse2(_mm_subs_epi16(qs1, filt), bd), + t80); + ps1 = _mm_adds_epi16(signed_char_clamp_bd_sse2(_mm_adds_epi16(ps1, filt), bd), + t80); + + // end highbd_filter4 + // loopfilter done + + // highbd_flat_mask4 + flat = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p2, p0), _mm_subs_epu16(p0, p2)), + _mm_or_si128(_mm_subs_epu16(p3, p0), _mm_subs_epu16(p0, p3))); + work = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(q2, q0), _mm_subs_epu16(q0, q2)), + _mm_or_si128(_mm_subs_epu16(q3, q0), _mm_subs_epu16(q0, q3))); + flat = _mm_max_epi16(work, flat); + work = _mm_max_epi16(abs_p1p0, abs_q1q0); + flat = _mm_max_epi16(work, flat); + + if (bd == 8) + flat = _mm_subs_epu16(flat, one); + else if (bd == 10) + flat = _mm_subs_epu16(flat, _mm_slli_epi16(one, 2)); + else // bd == 12 + flat = _mm_subs_epu16(flat, _mm_slli_epi16(one, 4)); + + flat = _mm_cmpeq_epi16(flat, zero); + // end flat_mask4 + + // flat & mask = flat && mask (as used in filter8) + // (because, in both vars, each block of 16 either all 1s or all 0s) + flat = _mm_and_si128(flat, mask); + + p5 = _mm_load_si128((__m128i *)(s - 6 * p)); + q5 = _mm_load_si128((__m128i *)(s + 5 * p)); + p6 = _mm_load_si128((__m128i *)(s - 7 * p)); + q6 = _mm_load_si128((__m128i *)(s + 6 * p)); + p7 = _mm_load_si128((__m128i *)(s - 8 * p)); + q7 = _mm_load_si128((__m128i *)(s + 7 * p)); + + // highbd_flat_mask5 (arguments passed in are p0, q0, p4-p7, q4-q7 + // but referred to as p0-p4 & q0-q4 in fn) + flat2 = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p4, p0), _mm_subs_epu16(p0, p4)), + _mm_or_si128(_mm_subs_epu16(q4, q0), _mm_subs_epu16(q0, q4))); + + work = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p5, p0), _mm_subs_epu16(p0, p5)), + _mm_or_si128(_mm_subs_epu16(q5, q0), _mm_subs_epu16(q0, q5))); + flat2 = _mm_max_epi16(work, flat2); + + work = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p6, p0), _mm_subs_epu16(p0, p6)), + _mm_or_si128(_mm_subs_epu16(q6, q0), _mm_subs_epu16(q0, q6))); + flat2 = _mm_max_epi16(work, flat2); + + work = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p7, p0), _mm_subs_epu16(p0, p7)), + _mm_or_si128(_mm_subs_epu16(q7, q0), _mm_subs_epu16(q0, q7))); + flat2 = _mm_max_epi16(work, flat2); + + if (bd == 8) + flat2 = _mm_subs_epu16(flat2, one); + else if (bd == 10) + flat2 = _mm_subs_epu16(flat2, _mm_slli_epi16(one, 2)); + else // bd == 12 + flat2 = _mm_subs_epu16(flat2, _mm_slli_epi16(one, 4)); + + flat2 = _mm_cmpeq_epi16(flat2, zero); + flat2 = _mm_and_si128(flat2, flat); // flat2 & flat & mask + // end highbd_flat_mask5 + + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // flat and wide flat calculations + eight = _mm_set1_epi16(8); + four = _mm_set1_epi16(4); + + pixelFilter_p = _mm_add_epi16(_mm_add_epi16(p6, p5), _mm_add_epi16(p4, p3)); + pixelFilter_q = _mm_add_epi16(_mm_add_epi16(q6, q5), _mm_add_epi16(q4, q3)); + + pixetFilter_p2p1p0 = _mm_add_epi16(p0, _mm_add_epi16(p2, p1)); + pixelFilter_p = _mm_add_epi16(pixelFilter_p, pixetFilter_p2p1p0); + + pixetFilter_q2q1q0 = _mm_add_epi16(q0, _mm_add_epi16(q2, q1)); + pixelFilter_q = _mm_add_epi16(pixelFilter_q, pixetFilter_q2q1q0); + pixelFilter_p = + _mm_add_epi16(eight, _mm_add_epi16(pixelFilter_p, pixelFilter_q)); + pixetFilter_p2p1p0 = _mm_add_epi16( + four, _mm_add_epi16(pixetFilter_p2p1p0, pixetFilter_q2q1q0)); + flat2_p0 = + _mm_srli_epi16(_mm_add_epi16(pixelFilter_p, _mm_add_epi16(p7, p0)), 4); + flat2_q0 = + _mm_srli_epi16(_mm_add_epi16(pixelFilter_p, _mm_add_epi16(q7, q0)), 4); + flat_p0 = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_p2p1p0, _mm_add_epi16(p3, p0)), 3); + flat_q0 = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_p2p1p0, _mm_add_epi16(q3, q0)), 3); + + sum_p7 = _mm_add_epi16(p7, p7); + sum_q7 = _mm_add_epi16(q7, q7); + sum_p3 = _mm_add_epi16(p3, p3); + sum_q3 = _mm_add_epi16(q3, q3); + + pixelFilter_q = _mm_sub_epi16(pixelFilter_p, p6); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q6); + flat2_p1 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p1)), 4); + flat2_q1 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q1)), 4); + + pixetFilter_q2q1q0 = _mm_sub_epi16(pixetFilter_p2p1p0, p2); + pixetFilter_p2p1p0 = _mm_sub_epi16(pixetFilter_p2p1p0, q2); + flat_p1 = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_p2p1p0, _mm_add_epi16(sum_p3, p1)), 3); + flat_q1 = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_q2q1q0, _mm_add_epi16(sum_q3, q1)), 3); + + sum_p7 = _mm_add_epi16(sum_p7, p7); + sum_q7 = _mm_add_epi16(sum_q7, q7); + sum_p3 = _mm_add_epi16(sum_p3, p3); + sum_q3 = _mm_add_epi16(sum_q3, q3); + + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q5); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p5); + flat2_p2 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p2)), 4); + flat2_q2 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q2)), 4); + + pixetFilter_p2p1p0 = _mm_sub_epi16(pixetFilter_p2p1p0, q1); + pixetFilter_q2q1q0 = _mm_sub_epi16(pixetFilter_q2q1q0, p1); + flat_p2 = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_p2p1p0, _mm_add_epi16(sum_p3, p2)), 3); + flat_q2 = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_q2q1q0, _mm_add_epi16(sum_q3, q2)), 3); + + sum_p7 = _mm_add_epi16(sum_p7, p7); + sum_q7 = _mm_add_epi16(sum_q7, q7); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q4); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p4); + flat2_p3 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p3)), 4); + flat2_q3 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q3)), 4); + + sum_p7 = _mm_add_epi16(sum_p7, p7); + sum_q7 = _mm_add_epi16(sum_q7, q7); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q3); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p3); + flat2_p4 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p4)), 4); + flat2_q4 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q4)), 4); + + sum_p7 = _mm_add_epi16(sum_p7, p7); + sum_q7 = _mm_add_epi16(sum_q7, q7); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q2); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p2); + flat2_p5 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p5)), 4); + flat2_q5 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q5)), 4); + + sum_p7 = _mm_add_epi16(sum_p7, p7); + sum_q7 = _mm_add_epi16(sum_q7, q7); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q1); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p1); + flat2_p6 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p6)), 4); + flat2_q6 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q6)), 4); + + // wide flat + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + // highbd_filter8 + p2 = _mm_andnot_si128(flat, p2); + // p2 remains unchanged if !(flat && mask) + flat_p2 = _mm_and_si128(flat, flat_p2); + // when (flat && mask) + p2 = _mm_or_si128(p2, flat_p2); // full list of p2 values + q2 = _mm_andnot_si128(flat, q2); + flat_q2 = _mm_and_si128(flat, flat_q2); + q2 = _mm_or_si128(q2, flat_q2); // full list of q2 values + + ps1 = _mm_andnot_si128(flat, ps1); + // p1 takes the value assigned to in in filter4 if !(flat && mask) + flat_p1 = _mm_and_si128(flat, flat_p1); + // when (flat && mask) + p1 = _mm_or_si128(ps1, flat_p1); // full list of p1 values + qs1 = _mm_andnot_si128(flat, qs1); + flat_q1 = _mm_and_si128(flat, flat_q1); + q1 = _mm_or_si128(qs1, flat_q1); // full list of q1 values + + ps0 = _mm_andnot_si128(flat, ps0); + // p0 takes the value assigned to in in filter4 if !(flat && mask) + flat_p0 = _mm_and_si128(flat, flat_p0); + // when (flat && mask) + p0 = _mm_or_si128(ps0, flat_p0); // full list of p0 values + qs0 = _mm_andnot_si128(flat, qs0); + flat_q0 = _mm_and_si128(flat, flat_q0); + q0 = _mm_or_si128(qs0, flat_q0); // full list of q0 values + // end highbd_filter8 + + // highbd_filter16 + p6 = _mm_andnot_si128(flat2, p6); + // p6 remains unchanged if !(flat2 && flat && mask) + flat2_p6 = _mm_and_si128(flat2, flat2_p6); + // get values for when (flat2 && flat && mask) + p6 = _mm_or_si128(p6, flat2_p6); // full list of p6 values + q6 = _mm_andnot_si128(flat2, q6); + // q6 remains unchanged if !(flat2 && flat && mask) + flat2_q6 = _mm_and_si128(flat2, flat2_q6); + // get values for when (flat2 && flat && mask) + q6 = _mm_or_si128(q6, flat2_q6); // full list of q6 values + _mm_store_si128((__m128i *)(s - 7 * p), p6); + _mm_store_si128((__m128i *)(s + 6 * p), q6); + + p5 = _mm_andnot_si128(flat2, p5); + // p5 remains unchanged if !(flat2 && flat && mask) + flat2_p5 = _mm_and_si128(flat2, flat2_p5); + // get values for when (flat2 && flat && mask) + p5 = _mm_or_si128(p5, flat2_p5); + // full list of p5 values + q5 = _mm_andnot_si128(flat2, q5); + // q5 remains unchanged if !(flat2 && flat && mask) + flat2_q5 = _mm_and_si128(flat2, flat2_q5); + // get values for when (flat2 && flat && mask) + q5 = _mm_or_si128(q5, flat2_q5); + // full list of q5 values + _mm_store_si128((__m128i *)(s - 6 * p), p5); + _mm_store_si128((__m128i *)(s + 5 * p), q5); + + p4 = _mm_andnot_si128(flat2, p4); + // p4 remains unchanged if !(flat2 && flat && mask) + flat2_p4 = _mm_and_si128(flat2, flat2_p4); + // get values for when (flat2 && flat && mask) + p4 = _mm_or_si128(p4, flat2_p4); // full list of p4 values + q4 = _mm_andnot_si128(flat2, q4); + // q4 remains unchanged if !(flat2 && flat && mask) + flat2_q4 = _mm_and_si128(flat2, flat2_q4); + // get values for when (flat2 && flat && mask) + q4 = _mm_or_si128(q4, flat2_q4); // full list of q4 values + _mm_store_si128((__m128i *)(s - 5 * p), p4); + _mm_store_si128((__m128i *)(s + 4 * p), q4); + + p3 = _mm_andnot_si128(flat2, p3); + // p3 takes value from highbd_filter8 if !(flat2 && flat && mask) + flat2_p3 = _mm_and_si128(flat2, flat2_p3); + // get values for when (flat2 && flat && mask) + p3 = _mm_or_si128(p3, flat2_p3); // full list of p3 values + q3 = _mm_andnot_si128(flat2, q3); + // q3 takes value from highbd_filter8 if !(flat2 && flat && mask) + flat2_q3 = _mm_and_si128(flat2, flat2_q3); + // get values for when (flat2 && flat && mask) + q3 = _mm_or_si128(q3, flat2_q3); // full list of q3 values + _mm_store_si128((__m128i *)(s - 4 * p), p3); + _mm_store_si128((__m128i *)(s + 3 * p), q3); + + p2 = _mm_andnot_si128(flat2, p2); + // p2 takes value from highbd_filter8 if !(flat2 && flat && mask) + flat2_p2 = _mm_and_si128(flat2, flat2_p2); + // get values for when (flat2 && flat && mask) + p2 = _mm_or_si128(p2, flat2_p2); + // full list of p2 values + q2 = _mm_andnot_si128(flat2, q2); + // q2 takes value from highbd_filter8 if !(flat2 && flat && mask) + flat2_q2 = _mm_and_si128(flat2, flat2_q2); + // get values for when (flat2 && flat && mask) + q2 = _mm_or_si128(q2, flat2_q2); // full list of q2 values + _mm_store_si128((__m128i *)(s - 3 * p), p2); + _mm_store_si128((__m128i *)(s + 2 * p), q2); + + p1 = _mm_andnot_si128(flat2, p1); + // p1 takes value from highbd_filter8 if !(flat2 && flat && mask) + flat2_p1 = _mm_and_si128(flat2, flat2_p1); + // get values for when (flat2 && flat && mask) + p1 = _mm_or_si128(p1, flat2_p1); // full list of p1 values + q1 = _mm_andnot_si128(flat2, q1); + // q1 takes value from highbd_filter8 if !(flat2 && flat && mask) + flat2_q1 = _mm_and_si128(flat2, flat2_q1); + // get values for when (flat2 && flat && mask) + q1 = _mm_or_si128(q1, flat2_q1); // full list of q1 values + _mm_store_si128((__m128i *)(s - 2 * p), p1); + _mm_store_si128((__m128i *)(s + 1 * p), q1); + + p0 = _mm_andnot_si128(flat2, p0); + // p0 takes value from highbd_filter8 if !(flat2 && flat && mask) + flat2_p0 = _mm_and_si128(flat2, flat2_p0); + // get values for when (flat2 && flat && mask) + p0 = _mm_or_si128(p0, flat2_p0); // full list of p0 values + q0 = _mm_andnot_si128(flat2, q0); + // q0 takes value from highbd_filter8 if !(flat2 && flat && mask) + flat2_q0 = _mm_and_si128(flat2, flat2_q0); + // get values for when (flat2 && flat && mask) + q0 = _mm_or_si128(q0, flat2_q0); // full list of q0 values + _mm_store_si128((__m128i *)(s - 1 * p), p0); + _mm_store_si128((__m128i *)(s - 0 * p), q0); +} + +void aom_highbd_lpf_horizontal_edge_16_sse2(uint16_t *s, int p, + const uint8_t *_blimit, + const uint8_t *_limit, + const uint8_t *_thresh, int bd) { + aom_highbd_lpf_horizontal_edge_8_sse2(s, p, _blimit, _limit, _thresh, bd); + aom_highbd_lpf_horizontal_edge_8_sse2(s + 8, p, _blimit, _limit, _thresh, bd); +} + +void aom_highbd_lpf_horizontal_8_sse2(uint16_t *s, int p, + const uint8_t *_blimit, + const uint8_t *_limit, + const uint8_t *_thresh, int bd) { + DECLARE_ALIGNED(16, uint16_t, flat_op2[16]); + DECLARE_ALIGNED(16, uint16_t, flat_op1[16]); + DECLARE_ALIGNED(16, uint16_t, flat_op0[16]); + DECLARE_ALIGNED(16, uint16_t, flat_oq2[16]); + DECLARE_ALIGNED(16, uint16_t, flat_oq1[16]); + DECLARE_ALIGNED(16, uint16_t, flat_oq0[16]); + const __m128i zero = _mm_set1_epi16(0); + __m128i blimit, limit, thresh; + __m128i mask, hev, flat; + __m128i p3 = _mm_load_si128((__m128i *)(s - 4 * p)); + __m128i q3 = _mm_load_si128((__m128i *)(s + 3 * p)); + __m128i p2 = _mm_load_si128((__m128i *)(s - 3 * p)); + __m128i q2 = _mm_load_si128((__m128i *)(s + 2 * p)); + __m128i p1 = _mm_load_si128((__m128i *)(s - 2 * p)); + __m128i q1 = _mm_load_si128((__m128i *)(s + 1 * p)); + __m128i p0 = _mm_load_si128((__m128i *)(s - 1 * p)); + __m128i q0 = _mm_load_si128((__m128i *)(s + 0 * p)); + const __m128i one = _mm_set1_epi16(1); + const __m128i ffff = _mm_cmpeq_epi16(one, one); + __m128i abs_p1q1, abs_p0q0, abs_q1q0, abs_p1p0, work; + const __m128i four = _mm_set1_epi16(4); + __m128i workp_a, workp_b, workp_shft; + + const __m128i t4 = _mm_set1_epi16(4); + const __m128i t3 = _mm_set1_epi16(3); + __m128i t80; + const __m128i t1 = _mm_set1_epi16(0x1); + __m128i ps1, ps0, qs0, qs1; + __m128i filt; + __m128i work_a; + __m128i filter1, filter2; + + if (bd == 8) { + blimit = _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_blimit), zero); + limit = _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_limit), zero); + thresh = _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_thresh), zero); + t80 = _mm_set1_epi16(0x80); + } else if (bd == 10) { + blimit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_blimit), zero), 2); + limit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_limit), zero), 2); + thresh = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_thresh), zero), 2); + t80 = _mm_set1_epi16(0x200); + } else { // bd == 12 + blimit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_blimit), zero), 4); + limit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_limit), zero), 4); + thresh = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_thresh), zero), 4); + t80 = _mm_set1_epi16(0x800); + } + + ps1 = _mm_subs_epi16(p1, t80); + ps0 = _mm_subs_epi16(p0, t80); + qs0 = _mm_subs_epi16(q0, t80); + qs1 = _mm_subs_epi16(q1, t80); + + // filter_mask and hev_mask + abs_p1p0 = _mm_or_si128(_mm_subs_epu16(p1, p0), _mm_subs_epu16(p0, p1)); + abs_q1q0 = _mm_or_si128(_mm_subs_epu16(q1, q0), _mm_subs_epu16(q0, q1)); + + abs_p0q0 = _mm_or_si128(_mm_subs_epu16(p0, q0), _mm_subs_epu16(q0, p0)); + abs_p1q1 = _mm_or_si128(_mm_subs_epu16(p1, q1), _mm_subs_epu16(q1, p1)); + flat = _mm_max_epi16(abs_p1p0, abs_q1q0); + hev = _mm_subs_epu16(flat, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi16(hev, zero), ffff); + + abs_p0q0 = _mm_adds_epu16(abs_p0q0, abs_p0q0); + abs_p1q1 = _mm_srli_epi16(abs_p1q1, 1); + mask = _mm_subs_epu16(_mm_adds_epu16(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi16(mask, zero), ffff); + // mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + // So taking maximums continues to work: + mask = _mm_and_si128(mask, _mm_adds_epu16(limit, one)); + mask = _mm_max_epi16(abs_p1p0, mask); + // mask |= (abs(p1 - p0) > limit) * -1; + mask = _mm_max_epi16(abs_q1q0, mask); + // mask |= (abs(q1 - q0) > limit) * -1; + + work = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p2, p1), _mm_subs_epu16(p1, p2)), + _mm_or_si128(_mm_subs_epu16(q2, q1), _mm_subs_epu16(q1, q2))); + mask = _mm_max_epi16(work, mask); + work = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p3, p2), _mm_subs_epu16(p2, p3)), + _mm_or_si128(_mm_subs_epu16(q3, q2), _mm_subs_epu16(q2, q3))); + mask = _mm_max_epi16(work, mask); + mask = _mm_subs_epu16(mask, limit); + mask = _mm_cmpeq_epi16(mask, zero); + + // flat_mask4 + flat = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p2, p0), _mm_subs_epu16(p0, p2)), + _mm_or_si128(_mm_subs_epu16(q2, q0), _mm_subs_epu16(q0, q2))); + work = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p3, p0), _mm_subs_epu16(p0, p3)), + _mm_or_si128(_mm_subs_epu16(q3, q0), _mm_subs_epu16(q0, q3))); + flat = _mm_max_epi16(work, flat); + flat = _mm_max_epi16(abs_p1p0, flat); + flat = _mm_max_epi16(abs_q1q0, flat); + + if (bd == 8) + flat = _mm_subs_epu16(flat, one); + else if (bd == 10) + flat = _mm_subs_epu16(flat, _mm_slli_epi16(one, 2)); + else // bd == 12 + flat = _mm_subs_epu16(flat, _mm_slli_epi16(one, 4)); + + flat = _mm_cmpeq_epi16(flat, zero); + flat = _mm_and_si128(flat, mask); // flat & mask + + // Added before shift for rounding part of ROUND_POWER_OF_TWO + + workp_a = _mm_add_epi16(_mm_add_epi16(p3, p3), _mm_add_epi16(p2, p1)); + workp_a = _mm_add_epi16(_mm_add_epi16(workp_a, four), p0); + workp_b = _mm_add_epi16(_mm_add_epi16(q0, p2), p3); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_store_si128((__m128i *)&flat_op2[0], workp_shft); + + workp_b = _mm_add_epi16(_mm_add_epi16(q0, q1), p1); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_store_si128((__m128i *)&flat_op1[0], workp_shft); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p3), q2); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, p1), p0); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_store_si128((__m128i *)&flat_op0[0], workp_shft); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p3), q3); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, p0), q0); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_store_si128((__m128i *)&flat_oq0[0], workp_shft); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p2), q3); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, q0), q1); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_store_si128((__m128i *)&flat_oq1[0], workp_shft); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p1), q3); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, q1), q2); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_store_si128((__m128i *)&flat_oq2[0], workp_shft); + + // lp filter + filt = signed_char_clamp_bd_sse2(_mm_subs_epi16(ps1, qs1), bd); + filt = _mm_and_si128(filt, hev); + work_a = _mm_subs_epi16(qs0, ps0); + filt = _mm_adds_epi16(filt, work_a); + filt = _mm_adds_epi16(filt, work_a); + filt = _mm_adds_epi16(filt, work_a); + // (aom_filter + 3 * (qs0 - ps0)) & mask + filt = signed_char_clamp_bd_sse2(filt, bd); + filt = _mm_and_si128(filt, mask); + + filter1 = _mm_adds_epi16(filt, t4); + filter2 = _mm_adds_epi16(filt, t3); + + // Filter1 >> 3 + filter1 = signed_char_clamp_bd_sse2(filter1, bd); + filter1 = _mm_srai_epi16(filter1, 3); + + // Filter2 >> 3 + filter2 = signed_char_clamp_bd_sse2(filter2, bd); + filter2 = _mm_srai_epi16(filter2, 3); + + // filt >> 1 + filt = _mm_adds_epi16(filter1, t1); + filt = _mm_srai_epi16(filt, 1); + // filter = ROUND_POWER_OF_TWO(filter1, 1) & ~hev; + filt = _mm_andnot_si128(hev, filt); + + work_a = signed_char_clamp_bd_sse2(_mm_subs_epi16(qs0, filter1), bd); + work_a = _mm_adds_epi16(work_a, t80); + q0 = _mm_load_si128((__m128i *)flat_oq0); + work_a = _mm_andnot_si128(flat, work_a); + q0 = _mm_and_si128(flat, q0); + q0 = _mm_or_si128(work_a, q0); + + work_a = signed_char_clamp_bd_sse2(_mm_subs_epi16(qs1, filt), bd); + work_a = _mm_adds_epi16(work_a, t80); + q1 = _mm_load_si128((__m128i *)flat_oq1); + work_a = _mm_andnot_si128(flat, work_a); + q1 = _mm_and_si128(flat, q1); + q1 = _mm_or_si128(work_a, q1); + + work_a = _mm_loadu_si128((__m128i *)(s + 2 * p)); + q2 = _mm_load_si128((__m128i *)flat_oq2); + work_a = _mm_andnot_si128(flat, work_a); + q2 = _mm_and_si128(flat, q2); + q2 = _mm_or_si128(work_a, q2); + + work_a = signed_char_clamp_bd_sse2(_mm_adds_epi16(ps0, filter2), bd); + work_a = _mm_adds_epi16(work_a, t80); + p0 = _mm_load_si128((__m128i *)flat_op0); + work_a = _mm_andnot_si128(flat, work_a); + p0 = _mm_and_si128(flat, p0); + p0 = _mm_or_si128(work_a, p0); + + work_a = signed_char_clamp_bd_sse2(_mm_adds_epi16(ps1, filt), bd); + work_a = _mm_adds_epi16(work_a, t80); + p1 = _mm_load_si128((__m128i *)flat_op1); + work_a = _mm_andnot_si128(flat, work_a); + p1 = _mm_and_si128(flat, p1); + p1 = _mm_or_si128(work_a, p1); + + work_a = _mm_loadu_si128((__m128i *)(s - 3 * p)); + p2 = _mm_load_si128((__m128i *)flat_op2); + work_a = _mm_andnot_si128(flat, work_a); + p2 = _mm_and_si128(flat, p2); + p2 = _mm_or_si128(work_a, p2); + + _mm_store_si128((__m128i *)(s - 3 * p), p2); + _mm_store_si128((__m128i *)(s - 2 * p), p1); + _mm_store_si128((__m128i *)(s - 1 * p), p0); + _mm_store_si128((__m128i *)(s + 0 * p), q0); + _mm_store_si128((__m128i *)(s + 1 * p), q1); + _mm_store_si128((__m128i *)(s + 2 * p), q2); +} + +void aom_highbd_lpf_horizontal_8_dual_sse2( + uint16_t *s, int p, const uint8_t *_blimit0, const uint8_t *_limit0, + const uint8_t *_thresh0, const uint8_t *_blimit1, const uint8_t *_limit1, + const uint8_t *_thresh1, int bd) { + aom_highbd_lpf_horizontal_8_sse2(s, p, _blimit0, _limit0, _thresh0, bd); + aom_highbd_lpf_horizontal_8_sse2(s + 8, p, _blimit1, _limit1, _thresh1, bd); +} + +void aom_highbd_lpf_horizontal_4_sse2(uint16_t *s, int p, + const uint8_t *_blimit, + const uint8_t *_limit, + const uint8_t *_thresh, int bd) { + const __m128i zero = _mm_set1_epi16(0); + __m128i blimit, limit, thresh; + __m128i mask, hev, flat; + __m128i p3 = _mm_loadu_si128((__m128i *)(s - 4 * p)); + __m128i p2 = _mm_loadu_si128((__m128i *)(s - 3 * p)); + __m128i p1 = _mm_loadu_si128((__m128i *)(s - 2 * p)); + __m128i p0 = _mm_loadu_si128((__m128i *)(s - 1 * p)); + __m128i q0 = _mm_loadu_si128((__m128i *)(s - 0 * p)); + __m128i q1 = _mm_loadu_si128((__m128i *)(s + 1 * p)); + __m128i q2 = _mm_loadu_si128((__m128i *)(s + 2 * p)); + __m128i q3 = _mm_loadu_si128((__m128i *)(s + 3 * p)); + const __m128i abs_p1p0 = + _mm_or_si128(_mm_subs_epu16(p1, p0), _mm_subs_epu16(p0, p1)); + const __m128i abs_q1q0 = + _mm_or_si128(_mm_subs_epu16(q1, q0), _mm_subs_epu16(q0, q1)); + const __m128i ffff = _mm_cmpeq_epi16(abs_p1p0, abs_p1p0); + const __m128i one = _mm_set1_epi16(1); + __m128i abs_p0q0 = + _mm_or_si128(_mm_subs_epu16(p0, q0), _mm_subs_epu16(q0, p0)); + __m128i abs_p1q1 = + _mm_or_si128(_mm_subs_epu16(p1, q1), _mm_subs_epu16(q1, p1)); + __m128i work; + const __m128i t4 = _mm_set1_epi16(4); + const __m128i t3 = _mm_set1_epi16(3); + __m128i t80; + __m128i tff80; + __m128i tffe0; + __m128i t1f; + // equivalent to shifting 0x1f left by bitdepth - 8 + // and setting new bits to 1 + const __m128i t1 = _mm_set1_epi16(0x1); + __m128i t7f; + // equivalent to shifting 0x7f left by bitdepth - 8 + // and setting new bits to 1 + __m128i ps1, ps0, qs0, qs1; + __m128i filt; + __m128i work_a; + __m128i filter1, filter2; + + if (bd == 8) { + blimit = _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_blimit), zero); + limit = _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_limit), zero); + thresh = _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_thresh), zero); + t80 = _mm_set1_epi16(0x80); + tff80 = _mm_set1_epi16(0xff80); + tffe0 = _mm_set1_epi16(0xffe0); + t1f = _mm_srli_epi16(_mm_set1_epi16(0x1fff), 8); + t7f = _mm_srli_epi16(_mm_set1_epi16(0x7fff), 8); + } else if (bd == 10) { + blimit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_blimit), zero), 2); + limit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_limit), zero), 2); + thresh = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_thresh), zero), 2); + t80 = _mm_slli_epi16(_mm_set1_epi16(0x80), 2); + tff80 = _mm_slli_epi16(_mm_set1_epi16(0xff80), 2); + tffe0 = _mm_slli_epi16(_mm_set1_epi16(0xffe0), 2); + t1f = _mm_srli_epi16(_mm_set1_epi16(0x1fff), 6); + t7f = _mm_srli_epi16(_mm_set1_epi16(0x7fff), 6); + } else { // bd == 12 + blimit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_blimit), zero), 4); + limit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_limit), zero), 4); + thresh = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_thresh), zero), 4); + t80 = _mm_slli_epi16(_mm_set1_epi16(0x80), 4); + tff80 = _mm_slli_epi16(_mm_set1_epi16(0xff80), 4); + tffe0 = _mm_slli_epi16(_mm_set1_epi16(0xffe0), 4); + t1f = _mm_srli_epi16(_mm_set1_epi16(0x1fff), 4); + t7f = _mm_srli_epi16(_mm_set1_epi16(0x7fff), 4); + } + + ps1 = _mm_subs_epi16(_mm_loadu_si128((__m128i *)(s - 2 * p)), t80); + ps0 = _mm_subs_epi16(_mm_loadu_si128((__m128i *)(s - 1 * p)), t80); + qs0 = _mm_subs_epi16(_mm_loadu_si128((__m128i *)(s + 0 * p)), t80); + qs1 = _mm_subs_epi16(_mm_loadu_si128((__m128i *)(s + 1 * p)), t80); + + // filter_mask and hev_mask + flat = _mm_max_epi16(abs_p1p0, abs_q1q0); + hev = _mm_subs_epu16(flat, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi16(hev, zero), ffff); + + abs_p0q0 = _mm_adds_epu16(abs_p0q0, abs_p0q0); + abs_p1q1 = _mm_srli_epi16(abs_p1q1, 1); + mask = _mm_subs_epu16(_mm_adds_epu16(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi16(mask, zero), ffff); + // mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + // So taking maximums continues to work: + mask = _mm_and_si128(mask, _mm_adds_epu16(limit, one)); + mask = _mm_max_epi16(flat, mask); + // mask |= (abs(p1 - p0) > limit) * -1; + // mask |= (abs(q1 - q0) > limit) * -1; + work = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(p2, p1), _mm_subs_epu16(p1, p2)), + _mm_or_si128(_mm_subs_epu16(p3, p2), _mm_subs_epu16(p2, p3))); + mask = _mm_max_epi16(work, mask); + work = _mm_max_epi16( + _mm_or_si128(_mm_subs_epu16(q2, q1), _mm_subs_epu16(q1, q2)), + _mm_or_si128(_mm_subs_epu16(q3, q2), _mm_subs_epu16(q2, q3))); + mask = _mm_max_epi16(work, mask); + mask = _mm_subs_epu16(mask, limit); + mask = _mm_cmpeq_epi16(mask, zero); + + // filter4 + filt = signed_char_clamp_bd_sse2(_mm_subs_epi16(ps1, qs1), bd); + filt = _mm_and_si128(filt, hev); + work_a = _mm_subs_epi16(qs0, ps0); + filt = _mm_adds_epi16(filt, work_a); + filt = _mm_adds_epi16(filt, work_a); + filt = signed_char_clamp_bd_sse2(_mm_adds_epi16(filt, work_a), bd); + + // (aom_filter + 3 * (qs0 - ps0)) & mask + filt = _mm_and_si128(filt, mask); + + filter1 = signed_char_clamp_bd_sse2(_mm_adds_epi16(filt, t4), bd); + filter2 = signed_char_clamp_bd_sse2(_mm_adds_epi16(filt, t3), bd); + + // Filter1 >> 3 + work_a = _mm_cmpgt_epi16(zero, filter1); // get the values that are <0 + filter1 = _mm_srli_epi16(filter1, 3); + work_a = _mm_and_si128(work_a, tffe0); // sign bits for the values < 0 + filter1 = _mm_and_si128(filter1, t1f); // clamp the range + filter1 = _mm_or_si128(filter1, work_a); // reinsert the sign bits + + // Filter2 >> 3 + work_a = _mm_cmpgt_epi16(zero, filter2); + filter2 = _mm_srli_epi16(filter2, 3); + work_a = _mm_and_si128(work_a, tffe0); + filter2 = _mm_and_si128(filter2, t1f); + filter2 = _mm_or_si128(filter2, work_a); + + // filt >> 1 + filt = _mm_adds_epi16(filter1, t1); + work_a = _mm_cmpgt_epi16(zero, filt); + filt = _mm_srli_epi16(filt, 1); + work_a = _mm_and_si128(work_a, tff80); + filt = _mm_and_si128(filt, t7f); + filt = _mm_or_si128(filt, work_a); + + filt = _mm_andnot_si128(hev, filt); + + q0 = _mm_adds_epi16( + signed_char_clamp_bd_sse2(_mm_subs_epi16(qs0, filter1), bd), t80); + q1 = _mm_adds_epi16(signed_char_clamp_bd_sse2(_mm_subs_epi16(qs1, filt), bd), + t80); + p0 = _mm_adds_epi16( + signed_char_clamp_bd_sse2(_mm_adds_epi16(ps0, filter2), bd), t80); + p1 = _mm_adds_epi16(signed_char_clamp_bd_sse2(_mm_adds_epi16(ps1, filt), bd), + t80); + + _mm_storeu_si128((__m128i *)(s - 2 * p), p1); + _mm_storeu_si128((__m128i *)(s - 1 * p), p0); + _mm_storeu_si128((__m128i *)(s + 0 * p), q0); + _mm_storeu_si128((__m128i *)(s + 1 * p), q1); +} + +void aom_highbd_lpf_horizontal_4_dual_sse2( + uint16_t *s, int p, const uint8_t *_blimit0, const uint8_t *_limit0, + const uint8_t *_thresh0, const uint8_t *_blimit1, const uint8_t *_limit1, + const uint8_t *_thresh1, int bd) { + aom_highbd_lpf_horizontal_4_sse2(s, p, _blimit0, _limit0, _thresh0, bd); + aom_highbd_lpf_horizontal_4_sse2(s + 8, p, _blimit1, _limit1, _thresh1, bd); +} + +static INLINE void highbd_transpose(uint16_t *src[], int in_p, uint16_t *dst[], + int out_p, int num_8x8_to_transpose) { + int idx8x8 = 0; + __m128i p0, p1, p2, p3, p4, p5, p6, p7, x0, x1, x2, x3, x4, x5, x6, x7; + do { + uint16_t *in = src[idx8x8]; + uint16_t *out = dst[idx8x8]; + + p0 = + _mm_loadu_si128((__m128i *)(in + 0 * in_p)); // 00 01 02 03 04 05 06 07 + p1 = + _mm_loadu_si128((__m128i *)(in + 1 * in_p)); // 10 11 12 13 14 15 16 17 + p2 = + _mm_loadu_si128((__m128i *)(in + 2 * in_p)); // 20 21 22 23 24 25 26 27 + p3 = + _mm_loadu_si128((__m128i *)(in + 3 * in_p)); // 30 31 32 33 34 35 36 37 + p4 = + _mm_loadu_si128((__m128i *)(in + 4 * in_p)); // 40 41 42 43 44 45 46 47 + p5 = + _mm_loadu_si128((__m128i *)(in + 5 * in_p)); // 50 51 52 53 54 55 56 57 + p6 = + _mm_loadu_si128((__m128i *)(in + 6 * in_p)); // 60 61 62 63 64 65 66 67 + p7 = + _mm_loadu_si128((__m128i *)(in + 7 * in_p)); // 70 71 72 73 74 75 76 77 + // 00 10 01 11 02 12 03 13 + x0 = _mm_unpacklo_epi16(p0, p1); + // 20 30 21 31 22 32 23 33 + x1 = _mm_unpacklo_epi16(p2, p3); + // 40 50 41 51 42 52 43 53 + x2 = _mm_unpacklo_epi16(p4, p5); + // 60 70 61 71 62 72 63 73 + x3 = _mm_unpacklo_epi16(p6, p7); + // 00 10 20 30 01 11 21 31 + x4 = _mm_unpacklo_epi32(x0, x1); + // 40 50 60 70 41 51 61 71 + x5 = _mm_unpacklo_epi32(x2, x3); + // 00 10 20 30 40 50 60 70 + x6 = _mm_unpacklo_epi64(x4, x5); + // 01 11 21 31 41 51 61 71 + x7 = _mm_unpackhi_epi64(x4, x5); + + _mm_storeu_si128((__m128i *)(out + 0 * out_p), x6); + // 00 10 20 30 40 50 60 70 + _mm_storeu_si128((__m128i *)(out + 1 * out_p), x7); + // 01 11 21 31 41 51 61 71 + + // 02 12 22 32 03 13 23 33 + x4 = _mm_unpackhi_epi32(x0, x1); + // 42 52 62 72 43 53 63 73 + x5 = _mm_unpackhi_epi32(x2, x3); + // 02 12 22 32 42 52 62 72 + x6 = _mm_unpacklo_epi64(x4, x5); + // 03 13 23 33 43 53 63 73 + x7 = _mm_unpackhi_epi64(x4, x5); + + _mm_storeu_si128((__m128i *)(out + 2 * out_p), x6); + // 02 12 22 32 42 52 62 72 + _mm_storeu_si128((__m128i *)(out + 3 * out_p), x7); + // 03 13 23 33 43 53 63 73 + + // 04 14 05 15 06 16 07 17 + x0 = _mm_unpackhi_epi16(p0, p1); + // 24 34 25 35 26 36 27 37 + x1 = _mm_unpackhi_epi16(p2, p3); + // 44 54 45 55 46 56 47 57 + x2 = _mm_unpackhi_epi16(p4, p5); + // 64 74 65 75 66 76 67 77 + x3 = _mm_unpackhi_epi16(p6, p7); + // 04 14 24 34 05 15 25 35 + x4 = _mm_unpacklo_epi32(x0, x1); + // 44 54 64 74 45 55 65 75 + x5 = _mm_unpacklo_epi32(x2, x3); + // 04 14 24 34 44 54 64 74 + x6 = _mm_unpacklo_epi64(x4, x5); + // 05 15 25 35 45 55 65 75 + x7 = _mm_unpackhi_epi64(x4, x5); + + _mm_storeu_si128((__m128i *)(out + 4 * out_p), x6); + // 04 14 24 34 44 54 64 74 + _mm_storeu_si128((__m128i *)(out + 5 * out_p), x7); + // 05 15 25 35 45 55 65 75 + + // 06 16 26 36 07 17 27 37 + x4 = _mm_unpackhi_epi32(x0, x1); + // 46 56 66 76 47 57 67 77 + x5 = _mm_unpackhi_epi32(x2, x3); + // 06 16 26 36 46 56 66 76 + x6 = _mm_unpacklo_epi64(x4, x5); + // 07 17 27 37 47 57 67 77 + x7 = _mm_unpackhi_epi64(x4, x5); + + _mm_storeu_si128((__m128i *)(out + 6 * out_p), x6); + // 06 16 26 36 46 56 66 76 + _mm_storeu_si128((__m128i *)(out + 7 * out_p), x7); + // 07 17 27 37 47 57 67 77 + } while (++idx8x8 < num_8x8_to_transpose); +} + +static INLINE void highbd_transpose8x16(uint16_t *in0, uint16_t *in1, int in_p, + uint16_t *out, int out_p) { + uint16_t *src0[1]; + uint16_t *src1[1]; + uint16_t *dest0[1]; + uint16_t *dest1[1]; + src0[0] = in0; + src1[0] = in1; + dest0[0] = out; + dest1[0] = out + 8; + highbd_transpose(src0, in_p, dest0, out_p, 1); + highbd_transpose(src1, in_p, dest1, out_p, 1); +} + +void aom_highbd_lpf_vertical_4_sse2(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int bd) { + DECLARE_ALIGNED(16, uint16_t, t_dst[8 * 8]); + uint16_t *src[1]; + uint16_t *dst[1]; + + // Transpose 8x8 + src[0] = s - 4; + dst[0] = t_dst; + + highbd_transpose(src, p, dst, 8, 1); + + // Loop filtering + aom_highbd_lpf_horizontal_4_sse2(t_dst + 4 * 8, 8, blimit, limit, thresh, bd); + + src[0] = t_dst; + dst[0] = s - 4; + + // Transpose back + highbd_transpose(src, 8, dst, p, 1); +} + +void aom_highbd_lpf_vertical_4_dual_sse2( + uint16_t *s, int p, const uint8_t *blimit0, const uint8_t *limit0, + const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1, int bd) { + DECLARE_ALIGNED(16, uint16_t, t_dst[16 * 8]); + uint16_t *src[2]; + uint16_t *dst[2]; + + // Transpose 8x16 + highbd_transpose8x16(s - 4, s - 4 + p * 8, p, t_dst, 16); + + // Loop filtering + aom_highbd_lpf_horizontal_4_dual_sse2(t_dst + 4 * 16, 16, blimit0, limit0, + thresh0, blimit1, limit1, thresh1, bd); + src[0] = t_dst; + src[1] = t_dst + 8; + dst[0] = s - 4; + dst[1] = s - 4 + p * 8; + + // Transpose back + highbd_transpose(src, 16, dst, p, 2); +} + +void aom_highbd_lpf_vertical_8_sse2(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int bd) { + DECLARE_ALIGNED(16, uint16_t, t_dst[8 * 8]); + uint16_t *src[1]; + uint16_t *dst[1]; + + // Transpose 8x8 + src[0] = s - 4; + dst[0] = t_dst; + + highbd_transpose(src, p, dst, 8, 1); + + // Loop filtering + aom_highbd_lpf_horizontal_8_sse2(t_dst + 4 * 8, 8, blimit, limit, thresh, bd); + + src[0] = t_dst; + dst[0] = s - 4; + + // Transpose back + highbd_transpose(src, 8, dst, p, 1); +} + +void aom_highbd_lpf_vertical_8_dual_sse2( + uint16_t *s, int p, const uint8_t *blimit0, const uint8_t *limit0, + const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1, int bd) { + DECLARE_ALIGNED(16, uint16_t, t_dst[16 * 8]); + uint16_t *src[2]; + uint16_t *dst[2]; + + // Transpose 8x16 + highbd_transpose8x16(s - 4, s - 4 + p * 8, p, t_dst, 16); + + // Loop filtering + aom_highbd_lpf_horizontal_8_dual_sse2(t_dst + 4 * 16, 16, blimit0, limit0, + thresh0, blimit1, limit1, thresh1, bd); + src[0] = t_dst; + src[1] = t_dst + 8; + + dst[0] = s - 4; + dst[1] = s - 4 + p * 8; + + // Transpose back + highbd_transpose(src, 16, dst, p, 2); +} + +void aom_highbd_lpf_vertical_16_sse2(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, int bd) { + DECLARE_ALIGNED(16, uint16_t, t_dst[8 * 16]); + uint16_t *src[2]; + uint16_t *dst[2]; + + src[0] = s - 8; + src[1] = s; + dst[0] = t_dst; + dst[1] = t_dst + 8 * 8; + + // Transpose 16x8 + highbd_transpose(src, p, dst, 8, 2); + + // Loop filtering + aom_highbd_lpf_horizontal_edge_8_sse2(t_dst + 8 * 8, 8, blimit, limit, thresh, + bd); + src[0] = t_dst; + src[1] = t_dst + 8 * 8; + dst[0] = s - 8; + dst[1] = s; + + // Transpose back + highbd_transpose(src, 8, dst, p, 2); +} + +void aom_highbd_lpf_vertical_16_dual_sse2(uint16_t *s, int p, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, int bd) { + DECLARE_ALIGNED(16, uint16_t, t_dst[256]); + + // Transpose 16x16 + highbd_transpose8x16(s - 8, s - 8 + 8 * p, p, t_dst, 16); + highbd_transpose8x16(s, s + 8 * p, p, t_dst + 8 * 16, 16); + + // Loop filtering + aom_highbd_lpf_horizontal_edge_16_sse2(t_dst + 8 * 16, 16, blimit, limit, + thresh, bd); + + // Transpose back + highbd_transpose8x16(t_dst, t_dst + 8 * 16, 16, s - 8, p); + highbd_transpose8x16(t_dst + 8, t_dst + 8 + 8 * 16, 16, s - 8 + 8 * p, p); +} diff --git a/third_party/aom/aom_dsp/x86/highbd_quantize_intrin_sse2.c b/third_party/aom/aom_dsp/x86/highbd_quantize_intrin_sse2.c new file mode 100644 index 0000000000..3ee24ab161 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_quantize_intrin_sse2.c @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +#if CONFIG_HIGHBITDEPTH +void aom_highbd_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t count, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, + const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + int i, j, non_zero_regs = (int)count / 4, eob_i = -1; + __m128i zbins[2]; + __m128i nzbins[2]; + + zbins[0] = _mm_set_epi32((int)zbin_ptr[1], (int)zbin_ptr[1], (int)zbin_ptr[1], + (int)zbin_ptr[0]); + zbins[1] = _mm_set1_epi32((int)zbin_ptr[1]); + + nzbins[0] = _mm_setzero_si128(); + nzbins[1] = _mm_setzero_si128(); + nzbins[0] = _mm_sub_epi32(nzbins[0], zbins[0]); + nzbins[1] = _mm_sub_epi32(nzbins[1], zbins[1]); + + (void)scan; + + memset(qcoeff_ptr, 0, count * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, count * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + // Pre-scan pass + for (i = ((int)count / 4) - 1; i >= 0; i--) { + __m128i coeffs, cmp1, cmp2; + int test; + coeffs = _mm_load_si128((const __m128i *)(coeff_ptr + i * 4)); + cmp1 = _mm_cmplt_epi32(coeffs, zbins[i != 0]); + cmp2 = _mm_cmpgt_epi32(coeffs, nzbins[i != 0]); + cmp1 = _mm_and_si128(cmp1, cmp2); + test = _mm_movemask_epi8(cmp1); + if (test == 0xffff) + non_zero_regs--; + else + break; + } + + // Quantization pass: + for (i = 0; i < non_zero_regs; i++) { + __m128i coeffs, coeffs_sign, tmp1, tmp2; + int test; + int abs_coeff[4]; + int coeff_sign[4]; + + coeffs = _mm_load_si128((const __m128i *)(coeff_ptr + i * 4)); + coeffs_sign = _mm_srai_epi32(coeffs, 31); + coeffs = _mm_sub_epi32(_mm_xor_si128(coeffs, coeffs_sign), coeffs_sign); + tmp1 = _mm_cmpgt_epi32(coeffs, zbins[i != 0]); + tmp2 = _mm_cmpeq_epi32(coeffs, zbins[i != 0]); + tmp1 = _mm_or_si128(tmp1, tmp2); + test = _mm_movemask_epi8(tmp1); + _mm_storeu_si128((__m128i *)abs_coeff, coeffs); + _mm_storeu_si128((__m128i *)coeff_sign, coeffs_sign); + + for (j = 0; j < 4; j++) { + if (test & (1 << (4 * j))) { + int k = 4 * i + j; + const int64_t tmp3 = abs_coeff[j] + round_ptr[k != 0]; + const int64_t tmp4 = ((tmp3 * quant_ptr[k != 0]) >> 16) + tmp3; + const uint32_t abs_qcoeff = + (uint32_t)((tmp4 * quant_shift_ptr[k != 0]) >> 16); + qcoeff_ptr[k] = (int)(abs_qcoeff ^ coeff_sign[j]) - coeff_sign[j]; + dqcoeff_ptr[k] = qcoeff_ptr[k] * dequant_ptr[k != 0]; + if (abs_qcoeff) eob_i = iscan[k] > eob_i ? iscan[k] : eob_i; + } + } + } + } + *eob_ptr = eob_i + 1; +} + +void aom_highbd_quantize_b_32x32_sse2( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + __m128i zbins[2]; + __m128i nzbins[2]; + int idx = 0; + int idx_arr[1024]; + int i, eob = -1; + const int zbin0_tmp = ROUND_POWER_OF_TWO(zbin_ptr[0], 1); + const int zbin1_tmp = ROUND_POWER_OF_TWO(zbin_ptr[1], 1); + (void)scan; + zbins[0] = _mm_set_epi32(zbin1_tmp, zbin1_tmp, zbin1_tmp, zbin0_tmp); + zbins[1] = _mm_set1_epi32(zbin1_tmp); + + nzbins[0] = _mm_setzero_si128(); + nzbins[1] = _mm_setzero_si128(); + nzbins[0] = _mm_sub_epi32(nzbins[0], zbins[0]); + nzbins[1] = _mm_sub_epi32(nzbins[1], zbins[1]); + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + // Pre-scan pass + for (i = 0; i < n_coeffs / 4; i++) { + __m128i coeffs, cmp1, cmp2; + int test; + coeffs = _mm_load_si128((const __m128i *)(coeff_ptr + i * 4)); + cmp1 = _mm_cmplt_epi32(coeffs, zbins[i != 0]); + cmp2 = _mm_cmpgt_epi32(coeffs, nzbins[i != 0]); + cmp1 = _mm_and_si128(cmp1, cmp2); + test = _mm_movemask_epi8(cmp1); + if (!(test & 0xf)) idx_arr[idx++] = i * 4; + if (!(test & 0xf0)) idx_arr[idx++] = i * 4 + 1; + if (!(test & 0xf00)) idx_arr[idx++] = i * 4 + 2; + if (!(test & 0xf000)) idx_arr[idx++] = i * 4 + 3; + } + + // Quantization pass: only process the coefficients selected in + // pre-scan pass. Note: idx can be zero. + for (i = 0; i < idx; i++) { + const int rc = idx_arr[i]; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp1 = + abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1); + const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; + const uint32_t abs_qcoeff = + (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15); + qcoeff_ptr[rc] = (int)(abs_qcoeff ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2; + if (abs_qcoeff) eob = iscan[idx_arr[i]] > eob ? iscan[idx_arr[i]] : eob; + } + } + *eob_ptr = eob + 1; +} +#endif diff --git a/third_party/aom/aom_dsp/x86/highbd_sad4d_sse2.asm b/third_party/aom/aom_dsp/x86/highbd_sad4d_sse2.asm new file mode 100644 index 0000000000..0c7cb3998b --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_sad4d_sse2.asm @@ -0,0 +1,290 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION .text + +; HIGH_PROCESS_4x2x4 first, off_{first,second}_{src,ref}, advance_at_end +%macro HIGH_PROCESS_4x2x4 5-6 0 + movh m0, [srcq +%2*2] +%if %1 == 1 + movu m4, [ref1q+%3*2] + movu m5, [ref2q+%3*2] + movu m6, [ref3q+%3*2] + movu m7, [ref4q+%3*2] + movhps m0, [srcq +%4*2] + movhps m4, [ref1q+%5*2] + movhps m5, [ref2q+%5*2] + movhps m6, [ref3q+%5*2] + movhps m7, [ref4q+%5*2] + mova m3, m0 + mova m2, m0 + psubusw m3, m4 + psubusw m2, m5 + psubusw m4, m0 + psubusw m5, m0 + por m4, m3 + por m5, m2 + pmaddwd m4, m1 + pmaddwd m5, m1 + mova m3, m0 + mova m2, m0 + psubusw m3, m6 + psubusw m2, m7 + psubusw m6, m0 + psubusw m7, m0 + por m6, m3 + por m7, m2 + pmaddwd m6, m1 + pmaddwd m7, m1 +%else + movu m2, [ref1q+%3*2] + movhps m0, [srcq +%4*2] + movhps m2, [ref1q+%5*2] + mova m3, m0 + psubusw m3, m2 + psubusw m2, m0 + por m2, m3 + pmaddwd m2, m1 + paddd m4, m2 + + movu m2, [ref2q+%3*2] + mova m3, m0 + movhps m2, [ref2q+%5*2] + psubusw m3, m2 + psubusw m2, m0 + por m2, m3 + pmaddwd m2, m1 + paddd m5, m2 + + movu m2, [ref3q+%3*2] + mova m3, m0 + movhps m2, [ref3q+%5*2] + psubusw m3, m2 + psubusw m2, m0 + por m2, m3 + pmaddwd m2, m1 + paddd m6, m2 + + movu m2, [ref4q+%3*2] + mova m3, m0 + movhps m2, [ref4q+%5*2] + psubusw m3, m2 + psubusw m2, m0 + por m2, m3 + pmaddwd m2, m1 + paddd m7, m2 +%endif +%if %6 == 1 + lea srcq, [srcq +src_strideq*4] + lea ref1q, [ref1q+ref_strideq*4] + lea ref2q, [ref2q+ref_strideq*4] + lea ref3q, [ref3q+ref_strideq*4] + lea ref4q, [ref4q+ref_strideq*4] +%endif +%endmacro + +; PROCESS_8x2x4 first, off_{first,second}_{src,ref}, advance_at_end +%macro HIGH_PROCESS_8x2x4 5-6 0 + ; 1st 8 px + mova m0, [srcq +%2*2] +%if %1 == 1 + movu m4, [ref1q+%3*2] + movu m5, [ref2q+%3*2] + movu m6, [ref3q+%3*2] + movu m7, [ref4q+%3*2] + mova m3, m0 + mova m2, m0 + psubusw m3, m4 + psubusw m2, m5 + psubusw m4, m0 + psubusw m5, m0 + por m4, m3 + por m5, m2 + pmaddwd m4, m1 + pmaddwd m5, m1 + mova m3, m0 + mova m2, m0 + psubusw m3, m6 + psubusw m2, m7 + psubusw m6, m0 + psubusw m7, m0 + por m6, m3 + por m7, m2 + pmaddwd m6, m1 + pmaddwd m7, m1 +%else + mova m3, m0 + movu m2, [ref1q+%3*2] + psubusw m3, m2 + psubusw m2, m0 + por m2, m3 + mova m3, m0 + pmaddwd m2, m1 + paddd m4, m2 + movu m2, [ref2q+%3*2] + psubusw m3, m2 + psubusw m2, m0 + por m2, m3 + mova m3, m0 + pmaddwd m2, m1 + paddd m5, m2 + movu m2, [ref3q+%3*2] + psubusw m3, m2 + psubusw m2, m0 + por m2, m3 + mova m3, m0 + pmaddwd m2, m1 + paddd m6, m2 + movu m2, [ref4q+%3*2] + psubusw m3, m2 + psubusw m2, m0 + por m2, m3 + pmaddwd m2, m1 + paddd m7, m2 +%endif + + ; 2nd 8 px + mova m0, [srcq +(%4)*2] + mova m3, m0 + movu m2, [ref1q+(%5)*2] + psubusw m3, m2 + psubusw m2, m0 + por m2, m3 + mova m3, m0 + pmaddwd m2, m1 + paddd m4, m2 + movu m2, [ref2q+(%5)*2] + psubusw m3, m2 + psubusw m2, m0 + por m2, m3 + mova m3, m0 + pmaddwd m2, m1 + paddd m5, m2 + movu m2, [ref3q+(%5)*2] + psubusw m3, m2 + psubusw m2, m0 + por m2, m3 + mova m3, m0 + pmaddwd m2, m1 + paddd m6, m2 + movu m2, [ref4q+(%5)*2] + psubusw m3, m2 + psubusw m2, m0 +%if %6 == 1 + lea srcq, [srcq +src_strideq*4] + lea ref1q, [ref1q+ref_strideq*4] + lea ref2q, [ref2q+ref_strideq*4] + lea ref3q, [ref3q+ref_strideq*4] + lea ref4q, [ref4q+ref_strideq*4] +%endif + por m2, m3 + pmaddwd m2, m1 + paddd m7, m2 +%endmacro + +; HIGH_PROCESS_16x2x4 first, off_{first,second}_{src,ref}, advance_at_end +%macro HIGH_PROCESS_16x2x4 5-6 0 + HIGH_PROCESS_8x2x4 %1, %2, %3, (%2 + 8), (%3 + 8) + HIGH_PROCESS_8x2x4 0, %4, %5, (%4 + 8), (%5 + 8), %6 +%endmacro + +; HIGH_PROCESS_32x2x4 first, off_{first,second}_{src,ref}, advance_at_end +%macro HIGH_PROCESS_32x2x4 5-6 0 + HIGH_PROCESS_16x2x4 %1, %2, %3, (%2 + 16), (%3 + 16) + HIGH_PROCESS_16x2x4 0, %4, %5, (%4 + 16), (%5 + 16), %6 +%endmacro + +; HIGH_PROCESS_64x2x4 first, off_{first,second}_{src,ref}, advance_at_end +%macro HIGH_PROCESS_64x2x4 5-6 0 + HIGH_PROCESS_32x2x4 %1, %2, %3, (%2 + 32), (%3 + 32) + HIGH_PROCESS_32x2x4 0, %4, %5, (%4 + 32), (%5 + 32), %6 +%endmacro + +; void aom_highbd_sadNxNx4d_sse2(uint8_t *src, int src_stride, +; uint8_t *ref[4], int ref_stride, +; uint32_t res[4]); +; where NxN = 64x64, 32x32, 16x16, 16x8, 8x16 or 8x8 +%macro HIGH_SADNXN4D 2 +%if UNIX64 +cglobal highbd_sad%1x%2x4d, 5, 8, 8, src, src_stride, ref1, ref_stride, \ + res, ref2, ref3, ref4 +%else +cglobal highbd_sad%1x%2x4d, 4, 7, 8, src, src_stride, ref1, ref_stride, \ + ref2, ref3, ref4 +%endif + +; set m1 + push srcq + mov srcd, 0x00010001 + movd m1, srcd + pshufd m1, m1, 0x0 + pop srcq + + movsxdifnidn src_strideq, src_strided + movsxdifnidn ref_strideq, ref_strided + mov ref2q, [ref1q+gprsize*1] + mov ref3q, [ref1q+gprsize*2] + mov ref4q, [ref1q+gprsize*3] + mov ref1q, [ref1q+gprsize*0] + +; convert byte pointers to short pointers + shl srcq, 1 + shl ref2q, 1 + shl ref3q, 1 + shl ref4q, 1 + shl ref1q, 1 + + HIGH_PROCESS_%1x2x4 1, 0, 0, src_strideq, ref_strideq, 1 +%rep (%2-4)/2 + HIGH_PROCESS_%1x2x4 0, 0, 0, src_strideq, ref_strideq, 1 +%endrep + HIGH_PROCESS_%1x2x4 0, 0, 0, src_strideq, ref_strideq, 0 + ; N.B. HIGH_PROCESS outputs dwords (32 bits) + ; so in high bit depth even the smallest width (4) needs 128bits i.e. XMM + movhlps m0, m4 + movhlps m1, m5 + movhlps m2, m6 + movhlps m3, m7 + paddd m4, m0 + paddd m5, m1 + paddd m6, m2 + paddd m7, m3 + punpckldq m4, m5 + punpckldq m6, m7 + movhlps m0, m4 + movhlps m1, m6 + paddd m4, m0 + paddd m6, m1 + punpcklqdq m4, m6 + movifnidn r4, r4mp + movu [r4], m4 + RET +%endmacro + + +INIT_XMM sse2 +HIGH_SADNXN4D 64, 64 +HIGH_SADNXN4D 64, 32 +HIGH_SADNXN4D 32, 64 +HIGH_SADNXN4D 32, 32 +HIGH_SADNXN4D 32, 16 +HIGH_SADNXN4D 16, 32 +HIGH_SADNXN4D 16, 16 +HIGH_SADNXN4D 16, 8 +HIGH_SADNXN4D 8, 16 +HIGH_SADNXN4D 8, 8 +HIGH_SADNXN4D 8, 4 +HIGH_SADNXN4D 4, 8 +HIGH_SADNXN4D 4, 4 diff --git a/third_party/aom/aom_dsp/x86/highbd_sad_sse2.asm b/third_party/aom/aom_dsp/x86/highbd_sad_sse2.asm new file mode 100644 index 0000000000..8427b891cc --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_sad_sse2.asm @@ -0,0 +1,366 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION .text + +%macro HIGH_SAD_FN 4 +%if %4 == 0 +%if %3 == 5 +cglobal highbd_sad%1x%2, 4, %3, 7, src, src_stride, ref, ref_stride, n_rows +%else ; %3 == 7 +cglobal highbd_sad%1x%2, 4, %3, 7, src, src_stride, ref, ref_stride, \ + src_stride3, ref_stride3, n_rows +%endif ; %3 == 5/7 +%else ; avg +%if %3 == 5 +cglobal highbd_sad%1x%2_avg, 5, 1 + %3, 7, src, src_stride, ref, ref_stride, \ + second_pred, n_rows +%else ; %3 == 7 +cglobal highbd_sad%1x%2_avg, 5, ARCH_X86_64 + %3, 7, src, src_stride, \ + ref, ref_stride, \ + second_pred, \ + src_stride3, ref_stride3 +%if ARCH_X86_64 +%define n_rowsd r7d +%else ; x86-32 +%define n_rowsd dword r0m +%endif ; x86-32/64 +%endif ; %3 == 5/7 +%endif ; avg/sad + movsxdifnidn src_strideq, src_strided + movsxdifnidn ref_strideq, ref_strided +%if %3 == 7 + lea src_stride3q, [src_strideq*3] + lea ref_stride3q, [ref_strideq*3] +%endif ; %3 == 7 +; convert src, ref & second_pred to short ptrs (from byte ptrs) + shl srcq, 1 + shl refq, 1 +%if %4 == 1 + shl second_predq, 1 +%endif +%endmacro + +; unsigned int aom_highbd_sad64x{16,32,64}_sse2(uint8_t *src, int src_stride, +; uint8_t *ref, int ref_stride); +%macro HIGH_SAD64XN 1-2 0 + HIGH_SAD_FN 64, %1, 5, %2 + mov n_rowsd, %1 + pxor m0, m0 + pxor m6, m6 + +.loop: + ; first half of each row + movu m1, [refq] + movu m2, [refq+16] + movu m3, [refq+32] + movu m4, [refq+48] +%if %2 == 1 + pavgw m1, [second_predq+mmsize*0] + pavgw m2, [second_predq+mmsize*1] + pavgw m3, [second_predq+mmsize*2] + pavgw m4, [second_predq+mmsize*3] + lea second_predq, [second_predq+mmsize*4] +%endif + mova m5, [srcq] + psubusw m5, m1 + psubusw m1, [srcq] + por m1, m5 + mova m5, [srcq+16] + psubusw m5, m2 + psubusw m2, [srcq+16] + por m2, m5 + mova m5, [srcq+32] + psubusw m5, m3 + psubusw m3, [srcq+32] + por m3, m5 + mova m5, [srcq+48] + psubusw m5, m4 + psubusw m4, [srcq+48] + por m4, m5 + paddw m1, m2 + paddw m3, m4 + movhlps m2, m1 + movhlps m4, m3 + paddw m1, m2 + paddw m3, m4 + punpcklwd m1, m6 + punpcklwd m3, m6 + paddd m0, m1 + paddd m0, m3 + ; second half of each row + movu m1, [refq+64] + movu m2, [refq+80] + movu m3, [refq+96] + movu m4, [refq+112] +%if %2 == 1 + pavgw m1, [second_predq+mmsize*0] + pavgw m2, [second_predq+mmsize*1] + pavgw m3, [second_predq+mmsize*2] + pavgw m4, [second_predq+mmsize*3] + lea second_predq, [second_predq+mmsize*4] +%endif + mova m5, [srcq+64] + psubusw m5, m1 + psubusw m1, [srcq+64] + por m1, m5 + mova m5, [srcq+80] + psubusw m5, m2 + psubusw m2, [srcq+80] + por m2, m5 + mova m5, [srcq+96] + psubusw m5, m3 + psubusw m3, [srcq+96] + por m3, m5 + mova m5, [srcq+112] + psubusw m5, m4 + psubusw m4, [srcq+112] + por m4, m5 + paddw m1, m2 + paddw m3, m4 + movhlps m2, m1 + movhlps m4, m3 + paddw m1, m2 + paddw m3, m4 + punpcklwd m1, m6 + punpcklwd m3, m6 + lea refq, [refq+ref_strideq*2] + paddd m0, m1 + lea srcq, [srcq+src_strideq*2] + paddd m0, m3 + + dec n_rowsd + jg .loop + + movhlps m1, m0 + paddd m0, m1 + punpckldq m0, m6 + movhlps m1, m0 + paddd m0, m1 + movd eax, m0 + RET +%endmacro + +INIT_XMM sse2 +HIGH_SAD64XN 64 ; highbd_sad64x64_sse2 +HIGH_SAD64XN 32 ; highbd_sad64x32_sse2 +HIGH_SAD64XN 64, 1 ; highbd_sad64x64_avg_sse2 +HIGH_SAD64XN 32, 1 ; highbd_sad64x32_avg_sse2 + + +; unsigned int aom_highbd_sad32x{16,32,64}_sse2(uint8_t *src, int src_stride, +; uint8_t *ref, int ref_stride); +%macro HIGH_SAD32XN 1-2 0 + HIGH_SAD_FN 32, %1, 5, %2 + mov n_rowsd, %1 + pxor m0, m0 + pxor m6, m6 + +.loop: + movu m1, [refq] + movu m2, [refq+16] + movu m3, [refq+32] + movu m4, [refq+48] +%if %2 == 1 + pavgw m1, [second_predq+mmsize*0] + pavgw m2, [second_predq+mmsize*1] + pavgw m3, [second_predq+mmsize*2] + pavgw m4, [second_predq+mmsize*3] + lea second_predq, [second_predq+mmsize*4] +%endif + mova m5, [srcq] + psubusw m5, m1 + psubusw m1, [srcq] + por m1, m5 + mova m5, [srcq+16] + psubusw m5, m2 + psubusw m2, [srcq+16] + por m2, m5 + mova m5, [srcq+32] + psubusw m5, m3 + psubusw m3, [srcq+32] + por m3, m5 + mova m5, [srcq+48] + psubusw m5, m4 + psubusw m4, [srcq+48] + por m4, m5 + paddw m1, m2 + paddw m3, m4 + movhlps m2, m1 + movhlps m4, m3 + paddw m1, m2 + paddw m3, m4 + punpcklwd m1, m6 + punpcklwd m3, m6 + lea refq, [refq+ref_strideq*2] + paddd m0, m1 + lea srcq, [srcq+src_strideq*2] + paddd m0, m3 + dec n_rowsd + jg .loop + + movhlps m1, m0 + paddd m0, m1 + punpckldq m0, m6 + movhlps m1, m0 + paddd m0, m1 + movd eax, m0 + RET +%endmacro + +INIT_XMM sse2 +HIGH_SAD32XN 64 ; highbd_sad32x64_sse2 +HIGH_SAD32XN 32 ; highbd_sad32x32_sse2 +HIGH_SAD32XN 16 ; highbd_sad32x16_sse2 +HIGH_SAD32XN 64, 1 ; highbd_sad32x64_avg_sse2 +HIGH_SAD32XN 32, 1 ; highbd_sad32x32_avg_sse2 +HIGH_SAD32XN 16, 1 ; highbd_sad32x16_avg_sse2 + +; unsigned int aom_highbd_sad16x{8,16,32}_sse2(uint8_t *src, int src_stride, +; uint8_t *ref, int ref_stride); +%macro HIGH_SAD16XN 1-2 0 + HIGH_SAD_FN 16, %1, 5, %2 + mov n_rowsd, %1/2 + pxor m0, m0 + pxor m6, m6 + +.loop: + movu m1, [refq] + movu m2, [refq+16] + movu m3, [refq+ref_strideq*2] + movu m4, [refq+ref_strideq*2+16] +%if %2 == 1 + pavgw m1, [second_predq+mmsize*0] + pavgw m2, [second_predq+16] + pavgw m3, [second_predq+mmsize*2] + pavgw m4, [second_predq+mmsize*2+16] + lea second_predq, [second_predq+mmsize*4] +%endif + mova m5, [srcq] + psubusw m5, m1 + psubusw m1, [srcq] + por m1, m5 + mova m5, [srcq+16] + psubusw m5, m2 + psubusw m2, [srcq+16] + por m2, m5 + mova m5, [srcq+src_strideq*2] + psubusw m5, m3 + psubusw m3, [srcq+src_strideq*2] + por m3, m5 + mova m5, [srcq+src_strideq*2+16] + psubusw m5, m4 + psubusw m4, [srcq+src_strideq*2+16] + por m4, m5 + paddw m1, m2 + paddw m3, m4 + movhlps m2, m1 + movhlps m4, m3 + paddw m1, m2 + paddw m3, m4 + punpcklwd m1, m6 + punpcklwd m3, m6 + lea refq, [refq+ref_strideq*4] + paddd m0, m1 + lea srcq, [srcq+src_strideq*4] + paddd m0, m3 + dec n_rowsd + jg .loop + + movhlps m1, m0 + paddd m0, m1 + punpckldq m0, m6 + movhlps m1, m0 + paddd m0, m1 + movd eax, m0 + RET +%endmacro + +INIT_XMM sse2 +HIGH_SAD16XN 32 ; highbd_sad16x32_sse2 +HIGH_SAD16XN 16 ; highbd_sad16x16_sse2 +HIGH_SAD16XN 8 ; highbd_sad16x8_sse2 +HIGH_SAD16XN 32, 1 ; highbd_sad16x32_avg_sse2 +HIGH_SAD16XN 16, 1 ; highbd_sad16x16_avg_sse2 +HIGH_SAD16XN 8, 1 ; highbd_sad16x8_avg_sse2 + + +; unsigned int aom_highbd_sad8x{4,8,16}_sse2(uint8_t *src, int src_stride, +; uint8_t *ref, int ref_stride); +%macro HIGH_SAD8XN 1-2 0 + HIGH_SAD_FN 8, %1, 7, %2 + mov n_rowsd, %1/4 + pxor m0, m0 + pxor m6, m6 + +.loop: + movu m1, [refq] + movu m2, [refq+ref_strideq*2] + movu m3, [refq+ref_strideq*4] + movu m4, [refq+ref_stride3q*2] +%if %2 == 1 + pavgw m1, [second_predq+mmsize*0] + pavgw m2, [second_predq+mmsize*1] + pavgw m3, [second_predq+mmsize*2] + pavgw m4, [second_predq+mmsize*3] + lea second_predq, [second_predq+mmsize*4] +%endif + mova m5, [srcq] + psubusw m5, m1 + psubusw m1, [srcq] + por m1, m5 + mova m5, [srcq+src_strideq*2] + psubusw m5, m2 + psubusw m2, [srcq+src_strideq*2] + por m2, m5 + mova m5, [srcq+src_strideq*4] + psubusw m5, m3 + psubusw m3, [srcq+src_strideq*4] + por m3, m5 + mova m5, [srcq+src_stride3q*2] + psubusw m5, m4 + psubusw m4, [srcq+src_stride3q*2] + por m4, m5 + paddw m1, m2 + paddw m3, m4 + movhlps m2, m1 + movhlps m4, m3 + paddw m1, m2 + paddw m3, m4 + punpcklwd m1, m6 + punpcklwd m3, m6 + lea refq, [refq+ref_strideq*8] + paddd m0, m1 + lea srcq, [srcq+src_strideq*8] + paddd m0, m3 + dec n_rowsd + jg .loop + + movhlps m1, m0 + paddd m0, m1 + punpckldq m0, m6 + movhlps m1, m0 + paddd m0, m1 + movd eax, m0 + RET +%endmacro + +INIT_XMM sse2 +HIGH_SAD8XN 16 ; highbd_sad8x16_sse2 +HIGH_SAD8XN 8 ; highbd_sad8x8_sse2 +HIGH_SAD8XN 4 ; highbd_sad8x4_sse2 +HIGH_SAD8XN 16, 1 ; highbd_sad8x16_avg_sse2 +HIGH_SAD8XN 8, 1 ; highbd_sad8x8_avg_sse2 +HIGH_SAD8XN 4, 1 ; highbd_sad8x4_avg_sse2 diff --git a/third_party/aom/aom_dsp/x86/highbd_subpel_variance_impl_sse2.asm b/third_party/aom/aom_dsp/x86/highbd_subpel_variance_impl_sse2.asm new file mode 100644 index 0000000000..797e9c1d44 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_subpel_variance_impl_sse2.asm @@ -0,0 +1,1040 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION_RODATA +pw_8: times 8 dw 8 +bilin_filter_m_sse2: times 8 dw 16 + times 8 dw 0 + times 8 dw 14 + times 8 dw 2 + times 8 dw 12 + times 8 dw 4 + times 8 dw 10 + times 8 dw 6 + times 16 dw 8 + times 8 dw 6 + times 8 dw 10 + times 8 dw 4 + times 8 dw 12 + times 8 dw 2 + times 8 dw 14 + +SECTION .text + +; int aom_sub_pixel_varianceNxh(const uint8_t *src, ptrdiff_t src_stride, +; int x_offset, int y_offset, +; const uint8_t *dst, ptrdiff_t dst_stride, +; int height, unsigned int *sse); +; +; This function returns the SE and stores SSE in the given pointer. + +%macro SUM_SSE 6 ; src1, dst1, src2, dst2, sum, sse + psubw %3, %4 + psubw %1, %2 + mova %4, %3 ; make copies to manipulate to calc sum + mova %2, %1 ; use originals for calc sse + pmaddwd %3, %3 + paddw %4, %2 + pmaddwd %1, %1 + movhlps %2, %4 + paddd %6, %3 + paddw %4, %2 + pxor %2, %2 + pcmpgtw %2, %4 ; mask for 0 > %4 (sum) + punpcklwd %4, %2 ; sign-extend word to dword + paddd %6, %1 + paddd %5, %4 + +%endmacro + +%macro STORE_AND_RET 0 +%if mmsize == 16 + ; if H=64 and W=16, we have 8 words of each 2(1bit)x64(6bit)x9bit=16bit + ; in m6, i.e. it _exactly_ fits in a signed word per word in the xmm reg. + ; We have to sign-extend it before adding the words within the register + ; and outputing to a dword. + movhlps m3, m7 + movhlps m4, m6 + paddd m7, m3 + paddd m6, m4 + pshufd m3, m7, 0x1 + pshufd m4, m6, 0x1 + paddd m7, m3 + paddd m6, m4 + mov r1, ssem ; r1 = unsigned int *sse + movd [r1], m7 ; store sse + movd rax, m6 ; store sum as return value +%endif + RET +%endmacro + +%macro INC_SRC_BY_SRC_STRIDE 0 +%if ARCH_X86=1 && CONFIG_PIC=1 + add srcq, src_stridemp + add srcq, src_stridemp +%else + lea srcq, [srcq + src_strideq*2] +%endif +%endmacro + +%macro SUBPEL_VARIANCE 1-2 0 ; W +%define bilin_filter_m bilin_filter_m_sse2 +%define filter_idx_shift 5 + + +%ifdef PIC ; 64bit PIC + %if %2 == 1 ; avg + cglobal highbd_sub_pixel_avg_variance%1xh, 9, 10, 13, src, src_stride, \ + x_offset, y_offset, \ + dst, dst_stride, \ + sec, sec_stride, height, sse + %define sec_str sec_strideq + %else + cglobal highbd_sub_pixel_variance%1xh, 7, 8, 13, src, src_stride, x_offset, \ + y_offset, dst, dst_stride, height, sse + %endif + %define block_height heightd + %define bilin_filter sseq +%else + %if ARCH_X86=1 && CONFIG_PIC=1 + %if %2 == 1 ; avg + cglobal highbd_sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \ + x_offset, y_offset, \ + dst, dst_stride, \ + sec, sec_stride, \ + height, sse, g_bilin_filter, g_pw_8 + %define block_height dword heightm + %define sec_str sec_stridemp + + ; Store bilin_filter and pw_8 location in stack + %if GET_GOT_DEFINED == 1 + GET_GOT eax + add esp, 4 ; restore esp + %endif + + lea ecx, [GLOBAL(bilin_filter_m)] + mov g_bilin_filterm, ecx + + lea ecx, [GLOBAL(pw_8)] + mov g_pw_8m, ecx + + LOAD_IF_USED 0, 1 ; load eax, ecx back + %else + cglobal highbd_sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, \ + x_offset, y_offset, dst, dst_stride, height, \ + sse, g_bilin_filter, g_pw_8 + %define block_height heightd + + ; Store bilin_filter and pw_8 location in stack + %if GET_GOT_DEFINED == 1 + GET_GOT eax + add esp, 4 ; restore esp + %endif + + lea ecx, [GLOBAL(bilin_filter_m)] + mov g_bilin_filterm, ecx + + lea ecx, [GLOBAL(pw_8)] + mov g_pw_8m, ecx + + LOAD_IF_USED 0, 1 ; load eax, ecx back + %endif + %else + %if %2 == 1 ; avg + cglobal highbd_sub_pixel_avg_variance%1xh, 7 + 2 * ARCH_X86_64, \ + 7 + 2 * ARCH_X86_64, 13, src, src_stride, \ + x_offset, y_offset, \ + dst, dst_stride, \ + sec, sec_stride, \ + height, sse + %if ARCH_X86_64 + %define block_height heightd + %define sec_str sec_strideq + %else + %define block_height dword heightm + %define sec_str sec_stridemp + %endif + %else + cglobal highbd_sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, \ + x_offset, y_offset, dst, dst_stride, height, sse + %define block_height heightd + %endif + + %define bilin_filter bilin_filter_m + %endif +%endif + + ASSERT %1 <= 16 ; m6 overflows if w > 16 + pxor m6, m6 ; sum + pxor m7, m7 ; sse + +%if %1 < 16 + sar block_height, 1 +%endif +%if %2 == 1 ; avg + shl sec_str, 1 +%endif + + ; FIXME(rbultje) replace by jumptable? + test x_offsetd, x_offsetd + jnz .x_nonzero + ; x_offset == 0 + test y_offsetd, y_offsetd + jnz .x_zero_y_nonzero + + ; x_offset == 0 && y_offset == 0 +.x_zero_y_zero_loop: +%if %1 == 16 + movu m0, [srcq] + movu m2, [srcq + 16] + mova m1, [dstq] + mova m3, [dstq + 16] +%if %2 == 1 ; avg + pavgw m0, [secq] + pavgw m2, [secq+16] +%endif + SUM_SSE m0, m1, m2, m3, m6, m7 + + lea srcq, [srcq + src_strideq*2] + lea dstq, [dstq + dst_strideq*2] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%else ; %1 < 16 + movu m0, [srcq] + movu m2, [srcq + src_strideq*2] + mova m1, [dstq] + mova m3, [dstq + dst_strideq*2] +%if %2 == 1 ; avg + pavgw m0, [secq] + add secq, sec_str + pavgw m2, [secq] +%endif + SUM_SSE m0, m1, m2, m3, m6, m7 + + lea srcq, [srcq + src_strideq*4] + lea dstq, [dstq + dst_strideq*4] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%endif + dec block_height + jg .x_zero_y_zero_loop + STORE_AND_RET + +.x_zero_y_nonzero: + cmp y_offsetd, 8 + jne .x_zero_y_nonhalf + + ; x_offset == 0 && y_offset == 0.5 +.x_zero_y_half_loop: +%if %1 == 16 + movu m0, [srcq] + movu m1, [srcq+16] + movu m4, [srcq+src_strideq*2] + movu m5, [srcq+src_strideq*2+16] + mova m2, [dstq] + mova m3, [dstq+16] + pavgw m0, m4 + pavgw m1, m5 +%if %2 == 1 ; avg + pavgw m0, [secq] + pavgw m1, [secq+16] +%endif + SUM_SSE m0, m2, m1, m3, m6, m7 + + lea srcq, [srcq + src_strideq*2] + lea dstq, [dstq + dst_strideq*2] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%else ; %1 < 16 + movu m0, [srcq] + movu m1, [srcq+src_strideq*2] + movu m5, [srcq+src_strideq*4] + mova m2, [dstq] + mova m3, [dstq+dst_strideq*2] + pavgw m0, m1 + pavgw m1, m5 +%if %2 == 1 ; avg + pavgw m0, [secq] + add secq, sec_str + pavgw m1, [secq] +%endif + SUM_SSE m0, m2, m1, m3, m6, m7 + + lea srcq, [srcq + src_strideq*4] + lea dstq, [dstq + dst_strideq*4] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%endif + dec block_height + jg .x_zero_y_half_loop + STORE_AND_RET + +.x_zero_y_nonhalf: + ; x_offset == 0 && y_offset == bilin interpolation +%ifdef PIC + lea bilin_filter, [bilin_filter_m] +%endif + shl y_offsetd, filter_idx_shift +%if ARCH_X86_64 && mmsize == 16 + mova m8, [bilin_filter+y_offsetq] + mova m9, [bilin_filter+y_offsetq+16] + mova m10, [pw_8] +%define filter_y_a m8 +%define filter_y_b m9 +%define filter_rnd m10 +%else ; x86-32 or mmx +%if ARCH_X86=1 && CONFIG_PIC=1 +; x_offset == 0, reuse x_offset reg +%define tempq x_offsetq + add y_offsetq, g_bilin_filterm +%define filter_y_a [y_offsetq] +%define filter_y_b [y_offsetq+16] + mov tempq, g_pw_8m +%define filter_rnd [tempq] +%else + add y_offsetq, bilin_filter +%define filter_y_a [y_offsetq] +%define filter_y_b [y_offsetq+16] +%define filter_rnd [pw_8] +%endif +%endif + +.x_zero_y_other_loop: +%if %1 == 16 + movu m0, [srcq] + movu m1, [srcq + 16] + movu m4, [srcq+src_strideq*2] + movu m5, [srcq+src_strideq*2+16] + mova m2, [dstq] + mova m3, [dstq+16] + ; FIXME(rbultje) instead of out=((num-x)*in1+x*in2+rnd)>>log2(num), we can + ; also do out=in1+(((num-x)*(in2-in1)+rnd)>>log2(num)). Total number of + ; instructions is the same (5), but it is 1 mul instead of 2, so might be + ; slightly faster because of pmullw latency. It would also cut our rodata + ; tables in half for this function, and save 1-2 registers on x86-64. + pmullw m1, filter_y_a + pmullw m5, filter_y_b + paddw m1, filter_rnd + pmullw m0, filter_y_a + pmullw m4, filter_y_b + paddw m0, filter_rnd + paddw m1, m5 + paddw m0, m4 + psrlw m1, 4 + psrlw m0, 4 +%if %2 == 1 ; avg + pavgw m0, [secq] + pavgw m1, [secq+16] +%endif + SUM_SSE m0, m2, m1, m3, m6, m7 + + lea srcq, [srcq + src_strideq*2] + lea dstq, [dstq + dst_strideq*2] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%else ; %1 < 16 + movu m0, [srcq] + movu m1, [srcq+src_strideq*2] + movu m5, [srcq+src_strideq*4] + mova m4, m1 + mova m2, [dstq] + mova m3, [dstq+dst_strideq*2] + pmullw m1, filter_y_a + pmullw m5, filter_y_b + paddw m1, filter_rnd + pmullw m0, filter_y_a + pmullw m4, filter_y_b + paddw m0, filter_rnd + paddw m1, m5 + paddw m0, m4 + psrlw m1, 4 + psrlw m0, 4 +%if %2 == 1 ; avg + pavgw m0, [secq] + add secq, sec_str + pavgw m1, [secq] +%endif + SUM_SSE m0, m2, m1, m3, m6, m7 + + lea srcq, [srcq + src_strideq*4] + lea dstq, [dstq + dst_strideq*4] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%endif + dec block_height + jg .x_zero_y_other_loop +%undef filter_y_a +%undef filter_y_b +%undef filter_rnd + STORE_AND_RET + +.x_nonzero: + cmp x_offsetd, 8 + jne .x_nonhalf + ; x_offset == 0.5 + test y_offsetd, y_offsetd + jnz .x_half_y_nonzero + + ; x_offset == 0.5 && y_offset == 0 +.x_half_y_zero_loop: +%if %1 == 16 + movu m0, [srcq] + movu m1, [srcq + 16] + movu m4, [srcq + 2] + movu m5, [srcq + 18] + mova m2, [dstq] + mova m3, [dstq + 16] + pavgw m0, m4 + pavgw m1, m5 +%if %2 == 1 ; avg + pavgw m0, [secq] + pavgw m1, [secq+16] +%endif + SUM_SSE m0, m2, m1, m3, m6, m7 + + lea srcq, [srcq + src_strideq*2] + lea dstq, [dstq + dst_strideq*2] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%else ; %1 < 16 + movu m0, [srcq] + movu m1, [srcq + src_strideq*2] + movu m4, [srcq + 2] + movu m5, [srcq + src_strideq*2 + 2] + mova m2, [dstq] + mova m3, [dstq + dst_strideq*2] + pavgw m0, m4 + pavgw m1, m5 +%if %2 == 1 ; avg + pavgw m0, [secq] + add secq, sec_str + pavgw m1, [secq] +%endif + SUM_SSE m0, m2, m1, m3, m6, m7 + + lea srcq, [srcq + src_strideq*4] + lea dstq, [dstq + dst_strideq*4] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%endif + dec block_height + jg .x_half_y_zero_loop + STORE_AND_RET + +.x_half_y_nonzero: + cmp y_offsetd, 8 + jne .x_half_y_nonhalf + + ; x_offset == 0.5 && y_offset == 0.5 +%if %1 == 16 + movu m0, [srcq] + movu m1, [srcq+16] + movu m2, [srcq+2] + movu m3, [srcq+18] + lea srcq, [srcq + src_strideq*2] + pavgw m0, m2 + pavgw m1, m3 +.x_half_y_half_loop: + movu m2, [srcq] + movu m3, [srcq + 16] + movu m4, [srcq + 2] + movu m5, [srcq + 18] + pavgw m2, m4 + pavgw m3, m5 + pavgw m0, m2 + pavgw m1, m3 + mova m4, [dstq] + mova m5, [dstq + 16] +%if %2 == 1 ; avg + pavgw m0, [secq] + pavgw m1, [secq+16] +%endif + SUM_SSE m0, m4, m1, m5, m6, m7 + mova m0, m2 + mova m1, m3 + + lea srcq, [srcq + src_strideq*2] + lea dstq, [dstq + dst_strideq*2] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%else ; %1 < 16 + movu m0, [srcq] + movu m2, [srcq+2] + lea srcq, [srcq + src_strideq*2] + pavgw m0, m2 +.x_half_y_half_loop: + movu m2, [srcq] + movu m3, [srcq + src_strideq*2] + movu m4, [srcq + 2] + movu m5, [srcq + src_strideq*2 + 2] + pavgw m2, m4 + pavgw m3, m5 + pavgw m0, m2 + pavgw m2, m3 + mova m4, [dstq] + mova m5, [dstq + dst_strideq*2] +%if %2 == 1 ; avg + pavgw m0, [secq] + add secq, sec_str + pavgw m2, [secq] +%endif + SUM_SSE m0, m4, m2, m5, m6, m7 + mova m0, m3 + + lea srcq, [srcq + src_strideq*4] + lea dstq, [dstq + dst_strideq*4] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%endif + dec block_height + jg .x_half_y_half_loop + STORE_AND_RET + +.x_half_y_nonhalf: + ; x_offset == 0.5 && y_offset == bilin interpolation +%ifdef PIC + lea bilin_filter, [bilin_filter_m] +%endif + shl y_offsetd, filter_idx_shift +%if ARCH_X86_64 && mmsize == 16 + mova m8, [bilin_filter+y_offsetq] + mova m9, [bilin_filter+y_offsetq+16] + mova m10, [pw_8] +%define filter_y_a m8 +%define filter_y_b m9 +%define filter_rnd m10 +%else ; x86_32 +%if ARCH_X86=1 && CONFIG_PIC=1 +; x_offset == 0.5. We can reuse x_offset reg +%define tempq x_offsetq + add y_offsetq, g_bilin_filterm +%define filter_y_a [y_offsetq] +%define filter_y_b [y_offsetq+16] + mov tempq, g_pw_8m +%define filter_rnd [tempq] +%else + add y_offsetq, bilin_filter +%define filter_y_a [y_offsetq] +%define filter_y_b [y_offsetq+16] +%define filter_rnd [pw_8] +%endif +%endif + +%if %1 == 16 + movu m0, [srcq] + movu m1, [srcq+16] + movu m2, [srcq+2] + movu m3, [srcq+18] + lea srcq, [srcq + src_strideq*2] + pavgw m0, m2 + pavgw m1, m3 +.x_half_y_other_loop: + movu m2, [srcq] + movu m3, [srcq+16] + movu m4, [srcq+2] + movu m5, [srcq+18] + pavgw m2, m4 + pavgw m3, m5 + mova m4, m2 + mova m5, m3 + pmullw m1, filter_y_a + pmullw m3, filter_y_b + paddw m1, filter_rnd + paddw m1, m3 + pmullw m0, filter_y_a + pmullw m2, filter_y_b + paddw m0, filter_rnd + psrlw m1, 4 + paddw m0, m2 + mova m2, [dstq] + psrlw m0, 4 + mova m3, [dstq+16] +%if %2 == 1 ; avg + pavgw m0, [secq] + pavgw m1, [secq+16] +%endif + SUM_SSE m0, m2, m1, m3, m6, m7 + mova m0, m4 + mova m1, m5 + + lea srcq, [srcq + src_strideq*2] + lea dstq, [dstq + dst_strideq*2] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%else ; %1 < 16 + movu m0, [srcq] + movu m2, [srcq+2] + lea srcq, [srcq + src_strideq*2] + pavgw m0, m2 +.x_half_y_other_loop: + movu m2, [srcq] + movu m3, [srcq+src_strideq*2] + movu m4, [srcq+2] + movu m5, [srcq+src_strideq*2+2] + pavgw m2, m4 + pavgw m3, m5 + mova m4, m2 + mova m5, m3 + pmullw m4, filter_y_a + pmullw m3, filter_y_b + paddw m4, filter_rnd + paddw m4, m3 + pmullw m0, filter_y_a + pmullw m2, filter_y_b + paddw m0, filter_rnd + psrlw m4, 4 + paddw m0, m2 + mova m2, [dstq] + psrlw m0, 4 + mova m3, [dstq+dst_strideq*2] +%if %2 == 1 ; avg + pavgw m0, [secq] + add secq, sec_str + pavgw m4, [secq] +%endif + SUM_SSE m0, m2, m4, m3, m6, m7 + mova m0, m5 + + lea srcq, [srcq + src_strideq*4] + lea dstq, [dstq + dst_strideq*4] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%endif + dec block_height + jg .x_half_y_other_loop +%undef filter_y_a +%undef filter_y_b +%undef filter_rnd + STORE_AND_RET + +.x_nonhalf: + test y_offsetd, y_offsetd + jnz .x_nonhalf_y_nonzero + + ; x_offset == bilin interpolation && y_offset == 0 +%ifdef PIC + lea bilin_filter, [bilin_filter_m] +%endif + shl x_offsetd, filter_idx_shift +%if ARCH_X86_64 && mmsize == 16 + mova m8, [bilin_filter+x_offsetq] + mova m9, [bilin_filter+x_offsetq+16] + mova m10, [pw_8] +%define filter_x_a m8 +%define filter_x_b m9 +%define filter_rnd m10 +%else ; x86-32 +%if ARCH_X86=1 && CONFIG_PIC=1 +; y_offset == 0. We can reuse y_offset reg. +%define tempq y_offsetq + add x_offsetq, g_bilin_filterm +%define filter_x_a [x_offsetq] +%define filter_x_b [x_offsetq+16] + mov tempq, g_pw_8m +%define filter_rnd [tempq] +%else + add x_offsetq, bilin_filter +%define filter_x_a [x_offsetq] +%define filter_x_b [x_offsetq+16] +%define filter_rnd [pw_8] +%endif +%endif + +.x_other_y_zero_loop: +%if %1 == 16 + movu m0, [srcq] + movu m1, [srcq+16] + movu m2, [srcq+2] + movu m3, [srcq+18] + mova m4, [dstq] + mova m5, [dstq+16] + pmullw m1, filter_x_a + pmullw m3, filter_x_b + paddw m1, filter_rnd + pmullw m0, filter_x_a + pmullw m2, filter_x_b + paddw m0, filter_rnd + paddw m1, m3 + paddw m0, m2 + psrlw m1, 4 + psrlw m0, 4 +%if %2 == 1 ; avg + pavgw m0, [secq] + pavgw m1, [secq+16] +%endif + SUM_SSE m0, m4, m1, m5, m6, m7 + + lea srcq, [srcq+src_strideq*2] + lea dstq, [dstq+dst_strideq*2] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%else ; %1 < 16 + movu m0, [srcq] + movu m1, [srcq+src_strideq*2] + movu m2, [srcq+2] + movu m3, [srcq+src_strideq*2+2] + mova m4, [dstq] + mova m5, [dstq+dst_strideq*2] + pmullw m1, filter_x_a + pmullw m3, filter_x_b + paddw m1, filter_rnd + pmullw m0, filter_x_a + pmullw m2, filter_x_b + paddw m0, filter_rnd + paddw m1, m3 + paddw m0, m2 + psrlw m1, 4 + psrlw m0, 4 +%if %2 == 1 ; avg + pavgw m0, [secq] + add secq, sec_str + pavgw m1, [secq] +%endif + SUM_SSE m0, m4, m1, m5, m6, m7 + + lea srcq, [srcq+src_strideq*4] + lea dstq, [dstq+dst_strideq*4] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%endif + dec block_height + jg .x_other_y_zero_loop +%undef filter_x_a +%undef filter_x_b +%undef filter_rnd + STORE_AND_RET + +.x_nonhalf_y_nonzero: + cmp y_offsetd, 8 + jne .x_nonhalf_y_nonhalf + + ; x_offset == bilin interpolation && y_offset == 0.5 +%ifdef PIC + lea bilin_filter, [bilin_filter_m] +%endif + shl x_offsetd, filter_idx_shift +%if ARCH_X86_64 && mmsize == 16 + mova m8, [bilin_filter+x_offsetq] + mova m9, [bilin_filter+x_offsetq+16] + mova m10, [pw_8] +%define filter_x_a m8 +%define filter_x_b m9 +%define filter_rnd m10 +%else ; x86-32 +%if ARCH_X86=1 && CONFIG_PIC=1 +; y_offset == 0.5. We can reuse y_offset reg. +%define tempq y_offsetq + add x_offsetq, g_bilin_filterm +%define filter_x_a [x_offsetq] +%define filter_x_b [x_offsetq+16] + mov tempq, g_pw_8m +%define filter_rnd [tempq] +%else + add x_offsetq, bilin_filter +%define filter_x_a [x_offsetq] +%define filter_x_b [x_offsetq+16] +%define filter_rnd [pw_8] +%endif +%endif + +%if %1 == 16 + movu m0, [srcq] + movu m1, [srcq+16] + movu m2, [srcq+2] + movu m3, [srcq+18] + pmullw m0, filter_x_a + pmullw m2, filter_x_b + paddw m0, filter_rnd + pmullw m1, filter_x_a + pmullw m3, filter_x_b + paddw m1, filter_rnd + paddw m0, m2 + paddw m1, m3 + psrlw m0, 4 + psrlw m1, 4 + lea srcq, [srcq+src_strideq*2] +.x_other_y_half_loop: + movu m2, [srcq] + movu m3, [srcq+16] + movu m4, [srcq+2] + movu m5, [srcq+18] + pmullw m2, filter_x_a + pmullw m4, filter_x_b + paddw m2, filter_rnd + pmullw m3, filter_x_a + pmullw m5, filter_x_b + paddw m3, filter_rnd + paddw m2, m4 + paddw m3, m5 + mova m4, [dstq] + mova m5, [dstq+16] + psrlw m2, 4 + psrlw m3, 4 + pavgw m0, m2 + pavgw m1, m3 +%if %2 == 1 ; avg + pavgw m0, [secq] + pavgw m1, [secq+16] +%endif + SUM_SSE m0, m4, m1, m5, m6, m7 + mova m0, m2 + mova m1, m3 + + lea srcq, [srcq+src_strideq*2] + lea dstq, [dstq+dst_strideq*2] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%else ; %1 < 16 + movu m0, [srcq] + movu m2, [srcq+2] + pmullw m0, filter_x_a + pmullw m2, filter_x_b + paddw m0, filter_rnd + paddw m0, m2 + psrlw m0, 4 + lea srcq, [srcq+src_strideq*2] +.x_other_y_half_loop: + movu m2, [srcq] + movu m3, [srcq+src_strideq*2] + movu m4, [srcq+2] + movu m5, [srcq+src_strideq*2+2] + pmullw m2, filter_x_a + pmullw m4, filter_x_b + paddw m2, filter_rnd + pmullw m3, filter_x_a + pmullw m5, filter_x_b + paddw m3, filter_rnd + paddw m2, m4 + paddw m3, m5 + mova m4, [dstq] + mova m5, [dstq+dst_strideq*2] + psrlw m2, 4 + psrlw m3, 4 + pavgw m0, m2 + pavgw m2, m3 +%if %2 == 1 ; avg + pavgw m0, [secq] + add secq, sec_str + pavgw m2, [secq] +%endif + SUM_SSE m0, m4, m2, m5, m6, m7 + mova m0, m3 + + lea srcq, [srcq+src_strideq*4] + lea dstq, [dstq+dst_strideq*4] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%endif + dec block_height + jg .x_other_y_half_loop +%undef filter_x_a +%undef filter_x_b +%undef filter_rnd + STORE_AND_RET + +.x_nonhalf_y_nonhalf: +; loading filter - this is same as in 8-bit depth +%ifdef PIC + lea bilin_filter, [bilin_filter_m] +%endif + shl x_offsetd, filter_idx_shift ; filter_idx_shift = 5 + shl y_offsetd, filter_idx_shift +%if ARCH_X86_64 && mmsize == 16 + mova m8, [bilin_filter+x_offsetq] + mova m9, [bilin_filter+x_offsetq+16] + mova m10, [bilin_filter+y_offsetq] + mova m11, [bilin_filter+y_offsetq+16] + mova m12, [pw_8] +%define filter_x_a m8 +%define filter_x_b m9 +%define filter_y_a m10 +%define filter_y_b m11 +%define filter_rnd m12 +%else ; x86-32 +%if ARCH_X86=1 && CONFIG_PIC=1 +; In this case, there is NO unused register. Used src_stride register. Later, +; src_stride has to be loaded from stack when it is needed. +%define tempq src_strideq + mov tempq, g_bilin_filterm + add x_offsetq, tempq + add y_offsetq, tempq +%define filter_x_a [x_offsetq] +%define filter_x_b [x_offsetq+16] +%define filter_y_a [y_offsetq] +%define filter_y_b [y_offsetq+16] + + mov tempq, g_pw_8m +%define filter_rnd [tempq] +%else + add x_offsetq, bilin_filter + add y_offsetq, bilin_filter +%define filter_x_a [x_offsetq] +%define filter_x_b [x_offsetq+16] +%define filter_y_a [y_offsetq] +%define filter_y_b [y_offsetq+16] +%define filter_rnd [pw_8] +%endif +%endif +; end of load filter + + ; x_offset == bilin interpolation && y_offset == bilin interpolation +%if %1 == 16 + movu m0, [srcq] + movu m2, [srcq+2] + movu m1, [srcq+16] + movu m3, [srcq+18] + pmullw m0, filter_x_a + pmullw m2, filter_x_b + paddw m0, filter_rnd + pmullw m1, filter_x_a + pmullw m3, filter_x_b + paddw m1, filter_rnd + paddw m0, m2 + paddw m1, m3 + psrlw m0, 4 + psrlw m1, 4 + + INC_SRC_BY_SRC_STRIDE + +.x_other_y_other_loop: + movu m2, [srcq] + movu m4, [srcq+2] + movu m3, [srcq+16] + movu m5, [srcq+18] + pmullw m2, filter_x_a + pmullw m4, filter_x_b + paddw m2, filter_rnd + pmullw m3, filter_x_a + pmullw m5, filter_x_b + paddw m3, filter_rnd + paddw m2, m4 + paddw m3, m5 + psrlw m2, 4 + psrlw m3, 4 + mova m4, m2 + mova m5, m3 + pmullw m0, filter_y_a + pmullw m2, filter_y_b + paddw m0, filter_rnd + pmullw m1, filter_y_a + pmullw m3, filter_y_b + paddw m0, m2 + paddw m1, filter_rnd + mova m2, [dstq] + paddw m1, m3 + psrlw m0, 4 + psrlw m1, 4 + mova m3, [dstq+16] +%if %2 == 1 ; avg + pavgw m0, [secq] + pavgw m1, [secq+16] +%endif + SUM_SSE m0, m2, m1, m3, m6, m7 + mova m0, m4 + mova m1, m5 + + INC_SRC_BY_SRC_STRIDE + lea dstq, [dstq + dst_strideq * 2] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%else ; %1 < 16 + movu m0, [srcq] + movu m2, [srcq+2] + pmullw m0, filter_x_a + pmullw m2, filter_x_b + paddw m0, filter_rnd + paddw m0, m2 + psrlw m0, 4 + + INC_SRC_BY_SRC_STRIDE + +.x_other_y_other_loop: + movu m2, [srcq] + movu m4, [srcq+2] + INC_SRC_BY_SRC_STRIDE + movu m3, [srcq] + movu m5, [srcq+2] + pmullw m2, filter_x_a + pmullw m4, filter_x_b + paddw m2, filter_rnd + pmullw m3, filter_x_a + pmullw m5, filter_x_b + paddw m3, filter_rnd + paddw m2, m4 + paddw m3, m5 + psrlw m2, 4 + psrlw m3, 4 + mova m4, m2 + mova m5, m3 + pmullw m0, filter_y_a + pmullw m2, filter_y_b + paddw m0, filter_rnd + pmullw m4, filter_y_a + pmullw m3, filter_y_b + paddw m0, m2 + paddw m4, filter_rnd + mova m2, [dstq] + paddw m4, m3 + psrlw m0, 4 + psrlw m4, 4 + mova m3, [dstq+dst_strideq*2] +%if %2 == 1 ; avg + pavgw m0, [secq] + add secq, sec_str + pavgw m4, [secq] +%endif + SUM_SSE m0, m2, m4, m3, m6, m7 + mova m0, m5 + + INC_SRC_BY_SRC_STRIDE + lea dstq, [dstq + dst_strideq * 4] +%if %2 == 1 ; avg + add secq, sec_str +%endif +%endif + dec block_height + jg .x_other_y_other_loop +%undef filter_x_a +%undef filter_x_b +%undef filter_y_a +%undef filter_y_b +%undef filter_rnd + STORE_AND_RET +%endmacro + +INIT_XMM sse2 +SUBPEL_VARIANCE 8 +SUBPEL_VARIANCE 16 + +INIT_XMM sse2 +SUBPEL_VARIANCE 8, 1 +SUBPEL_VARIANCE 16, 1 diff --git a/third_party/aom/aom_dsp/x86/highbd_subtract_sse2.c b/third_party/aom/aom_dsp/x86/highbd_subtract_sse2.c new file mode 100644 index 0000000000..7bc8a0df32 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_subtract_sse2.c @@ -0,0 +1,364 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +typedef void (*SubtractWxHFuncType)(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, + ptrdiff_t pred_stride); + +static void subtract_4x4(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + __m128i u0, u1, u2, u3; + __m128i v0, v1, v2, v3; + __m128i x0, x1, x2, x3; + int64_t *store_diff = (int64_t *)(diff + 0 * diff_stride); + + u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride)); + u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride)); + u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride)); + u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride)); + + v0 = _mm_loadu_si128((__m128i const *)(pred + 0 * pred_stride)); + v1 = _mm_loadu_si128((__m128i const *)(pred + 1 * pred_stride)); + v2 = _mm_loadu_si128((__m128i const *)(pred + 2 * pred_stride)); + v3 = _mm_loadu_si128((__m128i const *)(pred + 3 * pred_stride)); + + x0 = _mm_sub_epi16(u0, v0); + x1 = _mm_sub_epi16(u1, v1); + x2 = _mm_sub_epi16(u2, v2); + x3 = _mm_sub_epi16(u3, v3); + + _mm_storel_epi64((__m128i *)store_diff, x0); + store_diff = (int64_t *)(diff + 1 * diff_stride); + _mm_storel_epi64((__m128i *)store_diff, x1); + store_diff = (int64_t *)(diff + 2 * diff_stride); + _mm_storel_epi64((__m128i *)store_diff, x2); + store_diff = (int64_t *)(diff + 3 * diff_stride); + _mm_storel_epi64((__m128i *)store_diff, x3); +} + +static void subtract_4x8(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + __m128i u0, u1, u2, u3, u4, u5, u6, u7; + __m128i v0, v1, v2, v3, v4, v5, v6, v7; + __m128i x0, x1, x2, x3, x4, x5, x6, x7; + int64_t *store_diff = (int64_t *)(diff + 0 * diff_stride); + + u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride)); + u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride)); + u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride)); + u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride)); + u4 = _mm_loadu_si128((__m128i const *)(src + 4 * src_stride)); + u5 = _mm_loadu_si128((__m128i const *)(src + 5 * src_stride)); + u6 = _mm_loadu_si128((__m128i const *)(src + 6 * src_stride)); + u7 = _mm_loadu_si128((__m128i const *)(src + 7 * src_stride)); + + v0 = _mm_loadu_si128((__m128i const *)(pred + 0 * pred_stride)); + v1 = _mm_loadu_si128((__m128i const *)(pred + 1 * pred_stride)); + v2 = _mm_loadu_si128((__m128i const *)(pred + 2 * pred_stride)); + v3 = _mm_loadu_si128((__m128i const *)(pred + 3 * pred_stride)); + v4 = _mm_loadu_si128((__m128i const *)(pred + 4 * pred_stride)); + v5 = _mm_loadu_si128((__m128i const *)(pred + 5 * pred_stride)); + v6 = _mm_loadu_si128((__m128i const *)(pred + 6 * pred_stride)); + v7 = _mm_loadu_si128((__m128i const *)(pred + 7 * pred_stride)); + + x0 = _mm_sub_epi16(u0, v0); + x1 = _mm_sub_epi16(u1, v1); + x2 = _mm_sub_epi16(u2, v2); + x3 = _mm_sub_epi16(u3, v3); + x4 = _mm_sub_epi16(u4, v4); + x5 = _mm_sub_epi16(u5, v5); + x6 = _mm_sub_epi16(u6, v6); + x7 = _mm_sub_epi16(u7, v7); + + _mm_storel_epi64((__m128i *)store_diff, x0); + store_diff = (int64_t *)(diff + 1 * diff_stride); + _mm_storel_epi64((__m128i *)store_diff, x1); + store_diff = (int64_t *)(diff + 2 * diff_stride); + _mm_storel_epi64((__m128i *)store_diff, x2); + store_diff = (int64_t *)(diff + 3 * diff_stride); + _mm_storel_epi64((__m128i *)store_diff, x3); + store_diff = (int64_t *)(diff + 4 * diff_stride); + _mm_storel_epi64((__m128i *)store_diff, x4); + store_diff = (int64_t *)(diff + 5 * diff_stride); + _mm_storel_epi64((__m128i *)store_diff, x5); + store_diff = (int64_t *)(diff + 6 * diff_stride); + _mm_storel_epi64((__m128i *)store_diff, x6); + store_diff = (int64_t *)(diff + 7 * diff_stride); + _mm_storel_epi64((__m128i *)store_diff, x7); +} + +static void subtract_8x4(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + __m128i u0, u1, u2, u3; + __m128i v0, v1, v2, v3; + __m128i x0, x1, x2, x3; + + u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride)); + u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride)); + u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride)); + u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride)); + + v0 = _mm_loadu_si128((__m128i const *)(pred + 0 * pred_stride)); + v1 = _mm_loadu_si128((__m128i const *)(pred + 1 * pred_stride)); + v2 = _mm_loadu_si128((__m128i const *)(pred + 2 * pred_stride)); + v3 = _mm_loadu_si128((__m128i const *)(pred + 3 * pred_stride)); + + x0 = _mm_sub_epi16(u0, v0); + x1 = _mm_sub_epi16(u1, v1); + x2 = _mm_sub_epi16(u2, v2); + x3 = _mm_sub_epi16(u3, v3); + + _mm_storeu_si128((__m128i *)(diff + 0 * diff_stride), x0); + _mm_storeu_si128((__m128i *)(diff + 1 * diff_stride), x1); + _mm_storeu_si128((__m128i *)(diff + 2 * diff_stride), x2); + _mm_storeu_si128((__m128i *)(diff + 3 * diff_stride), x3); +} + +static void subtract_8x8(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + __m128i u0, u1, u2, u3, u4, u5, u6, u7; + __m128i v0, v1, v2, v3, v4, v5, v6, v7; + __m128i x0, x1, x2, x3, x4, x5, x6, x7; + + u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride)); + u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride)); + u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride)); + u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride)); + u4 = _mm_loadu_si128((__m128i const *)(src + 4 * src_stride)); + u5 = _mm_loadu_si128((__m128i const *)(src + 5 * src_stride)); + u6 = _mm_loadu_si128((__m128i const *)(src + 6 * src_stride)); + u7 = _mm_loadu_si128((__m128i const *)(src + 7 * src_stride)); + + v0 = _mm_loadu_si128((__m128i const *)(pred + 0 * pred_stride)); + v1 = _mm_loadu_si128((__m128i const *)(pred + 1 * pred_stride)); + v2 = _mm_loadu_si128((__m128i const *)(pred + 2 * pred_stride)); + v3 = _mm_loadu_si128((__m128i const *)(pred + 3 * pred_stride)); + v4 = _mm_loadu_si128((__m128i const *)(pred + 4 * pred_stride)); + v5 = _mm_loadu_si128((__m128i const *)(pred + 5 * pred_stride)); + v6 = _mm_loadu_si128((__m128i const *)(pred + 6 * pred_stride)); + v7 = _mm_loadu_si128((__m128i const *)(pred + 7 * pred_stride)); + + x0 = _mm_sub_epi16(u0, v0); + x1 = _mm_sub_epi16(u1, v1); + x2 = _mm_sub_epi16(u2, v2); + x3 = _mm_sub_epi16(u3, v3); + x4 = _mm_sub_epi16(u4, v4); + x5 = _mm_sub_epi16(u5, v5); + x6 = _mm_sub_epi16(u6, v6); + x7 = _mm_sub_epi16(u7, v7); + + _mm_storeu_si128((__m128i *)(diff + 0 * diff_stride), x0); + _mm_storeu_si128((__m128i *)(diff + 1 * diff_stride), x1); + _mm_storeu_si128((__m128i *)(diff + 2 * diff_stride), x2); + _mm_storeu_si128((__m128i *)(diff + 3 * diff_stride), x3); + _mm_storeu_si128((__m128i *)(diff + 4 * diff_stride), x4); + _mm_storeu_si128((__m128i *)(diff + 5 * diff_stride), x5); + _mm_storeu_si128((__m128i *)(diff + 6 * diff_stride), x6); + _mm_storeu_si128((__m128i *)(diff + 7 * diff_stride), x7); +} + +static void subtract_8x16(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + subtract_8x8(diff, diff_stride, src, src_stride, pred, pred_stride); + diff += diff_stride << 3; + src += src_stride << 3; + pred += pred_stride << 3; + subtract_8x8(diff, diff_stride, src, src_stride, pred, pred_stride); +} + +static void subtract_16x8(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + subtract_8x8(diff, diff_stride, src, src_stride, pred, pred_stride); + diff += 8; + src += 8; + pred += 8; + subtract_8x8(diff, diff_stride, src, src_stride, pred, pred_stride); +} + +static void subtract_16x16(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + subtract_16x8(diff, diff_stride, src, src_stride, pred, pred_stride); + diff += diff_stride << 3; + src += src_stride << 3; + pred += pred_stride << 3; + subtract_16x8(diff, diff_stride, src, src_stride, pred, pred_stride); +} + +static void subtract_16x32(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + subtract_16x16(diff, diff_stride, src, src_stride, pred, pred_stride); + diff += diff_stride << 4; + src += src_stride << 4; + pred += pred_stride << 4; + subtract_16x16(diff, diff_stride, src, src_stride, pred, pred_stride); +} + +static void subtract_32x16(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + subtract_16x16(diff, diff_stride, src, src_stride, pred, pred_stride); + diff += 16; + src += 16; + pred += 16; + subtract_16x16(diff, diff_stride, src, src_stride, pred, pred_stride); +} + +static void subtract_32x32(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + subtract_32x16(diff, diff_stride, src, src_stride, pred, pred_stride); + diff += diff_stride << 4; + src += src_stride << 4; + pred += pred_stride << 4; + subtract_32x16(diff, diff_stride, src, src_stride, pred, pred_stride); +} + +static void subtract_32x64(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + subtract_32x32(diff, diff_stride, src, src_stride, pred, pred_stride); + diff += diff_stride << 5; + src += src_stride << 5; + pred += pred_stride << 5; + subtract_32x32(diff, diff_stride, src, src_stride, pred, pred_stride); +} + +static void subtract_64x32(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + subtract_32x32(diff, diff_stride, src, src_stride, pred, pred_stride); + diff += 32; + src += 32; + pred += 32; + subtract_32x32(diff, diff_stride, src, src_stride, pred, pred_stride); +} + +static void subtract_64x64(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + subtract_64x32(diff, diff_stride, src, src_stride, pred, pred_stride); + diff += diff_stride << 5; + src += src_stride << 5; + pred += pred_stride << 5; + subtract_64x32(diff, diff_stride, src, src_stride, pred, pred_stride); +} + +static void subtract_64x128(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + subtract_64x64(diff, diff_stride, src, src_stride, pred, pred_stride); + diff += diff_stride << 6; + src += src_stride << 6; + pred += pred_stride << 6; + subtract_64x64(diff, diff_stride, src, src_stride, pred, pred_stride); +} + +static void subtract_128x64(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + subtract_64x64(diff, diff_stride, src, src_stride, pred, pred_stride); + diff += 64; + src += 64; + pred += 64; + subtract_64x64(diff, diff_stride, src, src_stride, pred, pred_stride); +} + +static void subtract_128x128(int16_t *diff, ptrdiff_t diff_stride, + const uint16_t *src, ptrdiff_t src_stride, + const uint16_t *pred, ptrdiff_t pred_stride) { + subtract_128x64(diff, diff_stride, src, src_stride, pred, pred_stride); + diff += diff_stride << 6; + src += src_stride << 6; + pred += pred_stride << 6; + subtract_128x64(diff, diff_stride, src, src_stride, pred, pred_stride); +} + +static SubtractWxHFuncType getSubtractFunc(int rows, int cols) { + SubtractWxHFuncType ret_func_ptr = NULL; + if (rows == 4) { + if (cols == 4) { + ret_func_ptr = subtract_4x4; + } else if (cols == 8) { + ret_func_ptr = subtract_8x4; + } + } else if (rows == 8) { + if (cols == 4) { + ret_func_ptr = subtract_4x8; + } else if (cols == 8) { + ret_func_ptr = subtract_8x8; + } else if (cols == 16) { + ret_func_ptr = subtract_16x8; + } + } else if (rows == 16) { + if (cols == 8) { + ret_func_ptr = subtract_8x16; + } else if (cols == 16) { + ret_func_ptr = subtract_16x16; + } else if (cols == 32) { + ret_func_ptr = subtract_32x16; + } + } else if (rows == 32) { + if (cols == 16) { + ret_func_ptr = subtract_16x32; + } else if (cols == 32) { + ret_func_ptr = subtract_32x32; + } else if (cols == 64) { + ret_func_ptr = subtract_64x32; + } + } else if (rows == 64) { + if (cols == 32) { + ret_func_ptr = subtract_32x64; + } else if (cols == 64) { + ret_func_ptr = subtract_64x64; + } else if (cols == 128) { + ret_func_ptr = subtract_128x64; + } + } else if (rows == 128) { + if (cols == 64) { + ret_func_ptr = subtract_64x128; + } else if (cols == 128) { + ret_func_ptr = subtract_128x128; + } + } + if (!ret_func_ptr) { + assert(0); + } + return ret_func_ptr; +} + +void aom_highbd_subtract_block_sse2(int rows, int cols, int16_t *diff, + ptrdiff_t diff_stride, const uint8_t *src8, + ptrdiff_t src_stride, const uint8_t *pred8, + ptrdiff_t pred_stride, int bd) { + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *pred = CONVERT_TO_SHORTPTR(pred8); + SubtractWxHFuncType func; + (void)bd; + + func = getSubtractFunc(rows, cols); + func(diff, diff_stride, src, src_stride, pred, pred_stride); +} diff --git a/third_party/aom/aom_dsp/x86/highbd_variance_impl_sse2.asm b/third_party/aom/aom_dsp/x86/highbd_variance_impl_sse2.asm new file mode 100644 index 0000000000..cf8ea498c7 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_variance_impl_sse2.asm @@ -0,0 +1,316 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + +%include "aom_ports/x86_abi_support.asm" + +;unsigned int aom_highbd_calc16x16var_sse2 +;( +; unsigned char * src_ptr, +; int source_stride, +; unsigned char * ref_ptr, +; int recon_stride, +; unsigned int * SSE, +; int * Sum +;) +global sym(aom_highbd_calc16x16var_sse2) PRIVATE +sym(aom_highbd_calc16x16var_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rbx + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;[src_ptr] + mov rdi, arg(2) ;[ref_ptr] + + movsxd rax, DWORD PTR arg(1) ;[source_stride] + movsxd rdx, DWORD PTR arg(3) ;[recon_stride] + add rax, rax ; source stride in bytes + add rdx, rdx ; recon stride in bytes + + ; Prefetch data + prefetcht0 [rsi] + prefetcht0 [rsi+16] + prefetcht0 [rsi+rax] + prefetcht0 [rsi+rax+16] + lea rbx, [rsi+rax*2] + prefetcht0 [rbx] + prefetcht0 [rbx+16] + prefetcht0 [rbx+rax] + prefetcht0 [rbx+rax+16] + + prefetcht0 [rdi] + prefetcht0 [rdi+16] + prefetcht0 [rdi+rdx] + prefetcht0 [rdi+rdx+16] + lea rbx, [rdi+rdx*2] + prefetcht0 [rbx] + prefetcht0 [rbx+16] + prefetcht0 [rbx+rdx] + prefetcht0 [rbx+rdx+16] + + pxor xmm0, xmm0 ; clear xmm0 for unpack + pxor xmm7, xmm7 ; clear xmm7 for accumulating diffs + + pxor xmm6, xmm6 ; clear xmm6 for accumulating sse + mov rcx, 16 + +.var16loop: + movdqu xmm1, XMMWORD PTR [rsi] + movdqu xmm2, XMMWORD PTR [rdi] + + lea rbx, [rsi+rax*2] + prefetcht0 [rbx] + prefetcht0 [rbx+16] + prefetcht0 [rbx+rax] + prefetcht0 [rbx+rax+16] + lea rbx, [rdi+rdx*2] + prefetcht0 [rbx] + prefetcht0 [rbx+16] + prefetcht0 [rbx+rdx] + prefetcht0 [rbx+rdx+16] + + pxor xmm5, xmm5 + + psubw xmm1, xmm2 + movdqu xmm3, XMMWORD PTR [rsi+16] + paddw xmm5, xmm1 + pmaddwd xmm1, xmm1 + movdqu xmm2, XMMWORD PTR [rdi+16] + paddd xmm6, xmm1 + + psubw xmm3, xmm2 + movdqu xmm1, XMMWORD PTR [rsi+rax] + paddw xmm5, xmm3 + pmaddwd xmm3, xmm3 + movdqu xmm2, XMMWORD PTR [rdi+rdx] + paddd xmm6, xmm3 + + psubw xmm1, xmm2 + movdqu xmm3, XMMWORD PTR [rsi+rax+16] + paddw xmm5, xmm1 + pmaddwd xmm1, xmm1 + movdqu xmm2, XMMWORD PTR [rdi+rdx+16] + paddd xmm6, xmm1 + + psubw xmm3, xmm2 + paddw xmm5, xmm3 + pmaddwd xmm3, xmm3 + paddd xmm6, xmm3 + + movdqa xmm1, xmm5 + movdqa xmm2, xmm5 + pcmpgtw xmm1, xmm0 + pcmpeqw xmm2, xmm0 + por xmm1, xmm2 + pcmpeqw xmm1, xmm0 + movdqa xmm2, xmm5 + punpcklwd xmm5, xmm1 + punpckhwd xmm2, xmm1 + paddd xmm7, xmm5 + paddd xmm7, xmm2 + + lea rsi, [rsi + 2*rax] + lea rdi, [rdi + 2*rdx] + sub rcx, 2 + jnz .var16loop + + movdqa xmm4, xmm6 + punpckldq xmm6, xmm0 + + punpckhdq xmm4, xmm0 + movdqa xmm5, xmm7 + + paddd xmm6, xmm4 + punpckldq xmm7, xmm0 + + punpckhdq xmm5, xmm0 + paddd xmm7, xmm5 + + movdqa xmm4, xmm6 + movdqa xmm5, xmm7 + + psrldq xmm4, 8 + psrldq xmm5, 8 + + paddd xmm6, xmm4 + paddd xmm7, xmm5 + + mov rdi, arg(4) ; [SSE] + mov rax, arg(5) ; [Sum] + + movd DWORD PTR [rdi], xmm6 + movd DWORD PTR [rax], xmm7 + + + ; begin epilog + pop rdi + pop rsi + pop rbx + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + + +;unsigned int aom_highbd_calc8x8var_sse2 +;( +; unsigned char * src_ptr, +; int source_stride, +; unsigned char * ref_ptr, +; int recon_stride, +; unsigned int * SSE, +; int * Sum +;) +global sym(aom_highbd_calc8x8var_sse2) PRIVATE +sym(aom_highbd_calc8x8var_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 6 + SAVE_XMM 7 + push rbx + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;[src_ptr] + mov rdi, arg(2) ;[ref_ptr] + + movsxd rax, DWORD PTR arg(1) ;[source_stride] + movsxd rdx, DWORD PTR arg(3) ;[recon_stride] + add rax, rax ; source stride in bytes + add rdx, rdx ; recon stride in bytes + + ; Prefetch data + prefetcht0 [rsi] + prefetcht0 [rsi+rax] + lea rbx, [rsi+rax*2] + prefetcht0 [rbx] + prefetcht0 [rbx+rax] + + prefetcht0 [rdi] + prefetcht0 [rdi+rdx] + lea rbx, [rdi+rdx*2] + prefetcht0 [rbx] + prefetcht0 [rbx+rdx] + + pxor xmm0, xmm0 ; clear xmm0 for unpack + pxor xmm7, xmm7 ; clear xmm7 for accumulating diffs + + pxor xmm6, xmm6 ; clear xmm6 for accumulating sse + mov rcx, 8 + +.var8loop: + movdqu xmm1, XMMWORD PTR [rsi] + movdqu xmm2, XMMWORD PTR [rdi] + + lea rbx, [rsi+rax*4] + prefetcht0 [rbx] + prefetcht0 [rbx+rax] + lea rbx, [rbx+rax*2] + prefetcht0 [rbx] + prefetcht0 [rbx+rax] + lea rbx, [rdi+rdx*4] + prefetcht0 [rbx] + prefetcht0 [rbx+rdx] + lea rbx, [rbx+rdx*2] + prefetcht0 [rbx] + prefetcht0 [rbx+rdx] + + pxor xmm5, xmm5 + + psubw xmm1, xmm2 + movdqu xmm3, XMMWORD PTR [rsi+rax] + paddw xmm5, xmm1 + pmaddwd xmm1, xmm1 + movdqu xmm2, XMMWORD PTR [rdi+rdx] + paddd xmm6, xmm1 + + lea rsi, [rsi + 2*rax] + lea rdi, [rdi + 2*rdx] + + psubw xmm3, xmm2 + movdqu xmm1, XMMWORD PTR [rsi] + paddw xmm5, xmm3 + pmaddwd xmm3, xmm3 + movdqu xmm2, XMMWORD PTR [rdi] + paddd xmm6, xmm3 + + psubw xmm1, xmm2 + movdqu xmm3, XMMWORD PTR [rsi+rax] + paddw xmm5, xmm1 + pmaddwd xmm1, xmm1 + movdqu xmm2, XMMWORD PTR [rdi+rdx] + paddd xmm6, xmm1 + + psubw xmm3, xmm2 + paddw xmm5, xmm3 + pmaddwd xmm3, xmm3 + paddd xmm6, xmm3 + + movdqa xmm1, xmm5 + movdqa xmm2, xmm5 + pcmpgtw xmm1, xmm0 + pcmpeqw xmm2, xmm0 + por xmm1, xmm2 + pcmpeqw xmm1, xmm0 + movdqa xmm2, xmm5 + punpcklwd xmm5, xmm1 + punpckhwd xmm2, xmm1 + paddd xmm7, xmm5 + paddd xmm7, xmm2 + + lea rsi, [rsi + 2*rax] + lea rdi, [rdi + 2*rdx] + sub rcx, 4 + jnz .var8loop + + movdqa xmm4, xmm6 + punpckldq xmm6, xmm0 + + punpckhdq xmm4, xmm0 + movdqa xmm5, xmm7 + + paddd xmm6, xmm4 + punpckldq xmm7, xmm0 + + punpckhdq xmm5, xmm0 + paddd xmm7, xmm5 + + movdqa xmm4, xmm6 + movdqa xmm5, xmm7 + + psrldq xmm4, 8 + psrldq xmm5, 8 + + paddd xmm6, xmm4 + paddd xmm7, xmm5 + + mov rdi, arg(4) ; [SSE] + mov rax, arg(5) ; [Sum] + + movd DWORD PTR [rdi], xmm6 + movd DWORD PTR [rax], xmm7 + + ; begin epilog + pop rdi + pop rsi + pop rbx + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret diff --git a/third_party/aom/aom_dsp/x86/highbd_variance_sse2.c b/third_party/aom/aom_dsp/x86/highbd_variance_sse2.c new file mode 100644 index 0000000000..29f96ce248 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_variance_sse2.c @@ -0,0 +1,695 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // SSE2 + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "aom_ports/mem.h" + +typedef uint32_t (*high_variance_fn_t)(const uint16_t *src, int src_stride, + const uint16_t *ref, int ref_stride, + uint32_t *sse, int *sum); + +uint32_t aom_highbd_calc8x8var_sse2(const uint16_t *src, int src_stride, + const uint16_t *ref, int ref_stride, + uint32_t *sse, int *sum); + +uint32_t aom_highbd_calc16x16var_sse2(const uint16_t *src, int src_stride, + const uint16_t *ref, int ref_stride, + uint32_t *sse, int *sum); + +static void highbd_8_variance_sse2(const uint16_t *src, int src_stride, + const uint16_t *ref, int ref_stride, int w, + int h, uint32_t *sse, int *sum, + high_variance_fn_t var_fn, int block_size) { + int i, j; + + *sse = 0; + *sum = 0; + + for (i = 0; i < h; i += block_size) { + for (j = 0; j < w; j += block_size) { + unsigned int sse0; + int sum0; + var_fn(src + src_stride * i + j, src_stride, ref + ref_stride * i + j, + ref_stride, &sse0, &sum0); + *sse += sse0; + *sum += sum0; + } + } +} + +static void highbd_10_variance_sse2(const uint16_t *src, int src_stride, + const uint16_t *ref, int ref_stride, int w, + int h, uint32_t *sse, int *sum, + high_variance_fn_t var_fn, int block_size) { + int i, j; + uint64_t sse_long = 0; + int32_t sum_long = 0; + + for (i = 0; i < h; i += block_size) { + for (j = 0; j < w; j += block_size) { + unsigned int sse0; + int sum0; + var_fn(src + src_stride * i + j, src_stride, ref + ref_stride * i + j, + ref_stride, &sse0, &sum0); + sse_long += sse0; + sum_long += sum0; + } + } + *sum = ROUND_POWER_OF_TWO(sum_long, 2); + *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 4); +} + +static void highbd_12_variance_sse2(const uint16_t *src, int src_stride, + const uint16_t *ref, int ref_stride, int w, + int h, uint32_t *sse, int *sum, + high_variance_fn_t var_fn, int block_size) { + int i, j; + uint64_t sse_long = 0; + int32_t sum_long = 0; + + for (i = 0; i < h; i += block_size) { + for (j = 0; j < w; j += block_size) { + unsigned int sse0; + int sum0; + var_fn(src + src_stride * i + j, src_stride, ref + ref_stride * i + j, + ref_stride, &sse0, &sum0); + sse_long += sse0; + sum_long += sum0; + } + } + *sum = ROUND_POWER_OF_TWO(sum_long, 4); + *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 8); +} + +#define HIGH_GET_VAR(S) \ + void aom_highbd_get##S##x##S##var_sse2(const uint8_t *src8, int src_stride, \ + const uint8_t *ref8, int ref_stride, \ + uint32_t *sse, int *sum) { \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ + aom_highbd_calc##S##x##S##var_sse2(src, src_stride, ref, ref_stride, sse, \ + sum); \ + } \ + \ + void aom_highbd_10_get##S##x##S##var_sse2( \ + const uint8_t *src8, int src_stride, const uint8_t *ref8, \ + int ref_stride, uint32_t *sse, int *sum) { \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ + aom_highbd_calc##S##x##S##var_sse2(src, src_stride, ref, ref_stride, sse, \ + sum); \ + *sum = ROUND_POWER_OF_TWO(*sum, 2); \ + *sse = ROUND_POWER_OF_TWO(*sse, 4); \ + } \ + \ + void aom_highbd_12_get##S##x##S##var_sse2( \ + const uint8_t *src8, int src_stride, const uint8_t *ref8, \ + int ref_stride, uint32_t *sse, int *sum) { \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ + aom_highbd_calc##S##x##S##var_sse2(src, src_stride, ref, ref_stride, sse, \ + sum); \ + *sum = ROUND_POWER_OF_TWO(*sum, 4); \ + *sse = ROUND_POWER_OF_TWO(*sse, 8); \ + } + +HIGH_GET_VAR(16); +HIGH_GET_VAR(8); + +#undef HIGH_GET_VAR + +#define VAR_FN(w, h, block_size, shift) \ + uint32_t aom_highbd_8_variance##w##x##h##_sse2( \ + const uint8_t *src8, int src_stride, const uint8_t *ref8, \ + int ref_stride, uint32_t *sse) { \ + int sum; \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ + highbd_8_variance_sse2( \ + src, src_stride, ref, ref_stride, w, h, sse, &sum, \ + aom_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ + return *sse - (uint32_t)(((int64_t)sum * sum) >> shift); \ + } \ + \ + uint32_t aom_highbd_10_variance##w##x##h##_sse2( \ + const uint8_t *src8, int src_stride, const uint8_t *ref8, \ + int ref_stride, uint32_t *sse) { \ + int sum; \ + int64_t var; \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ + highbd_10_variance_sse2( \ + src, src_stride, ref, ref_stride, w, h, sse, &sum, \ + aom_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ + var = (int64_t)(*sse) - (((int64_t)sum * sum) >> shift); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } \ + \ + uint32_t aom_highbd_12_variance##w##x##h##_sse2( \ + const uint8_t *src8, int src_stride, const uint8_t *ref8, \ + int ref_stride, uint32_t *sse) { \ + int sum; \ + int64_t var; \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ + highbd_12_variance_sse2( \ + src, src_stride, ref, ref_stride, w, h, sse, &sum, \ + aom_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ + var = (int64_t)(*sse) - (((int64_t)sum * sum) >> shift); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } + +VAR_FN(64, 64, 16, 12); +VAR_FN(64, 32, 16, 11); +VAR_FN(32, 64, 16, 11); +VAR_FN(32, 32, 16, 10); +VAR_FN(32, 16, 16, 9); +VAR_FN(16, 32, 16, 9); +VAR_FN(16, 16, 16, 8); +VAR_FN(16, 8, 8, 7); +VAR_FN(8, 16, 8, 7); +VAR_FN(8, 8, 8, 6); + +#undef VAR_FN + +unsigned int aom_highbd_8_mse16x16_sse2(const uint8_t *src8, int src_stride, + const uint8_t *ref8, int ref_stride, + unsigned int *sse) { + int sum; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + highbd_8_variance_sse2(src, src_stride, ref, ref_stride, 16, 16, sse, &sum, + aom_highbd_calc16x16var_sse2, 16); + return *sse; +} + +unsigned int aom_highbd_10_mse16x16_sse2(const uint8_t *src8, int src_stride, + const uint8_t *ref8, int ref_stride, + unsigned int *sse) { + int sum; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + highbd_10_variance_sse2(src, src_stride, ref, ref_stride, 16, 16, sse, &sum, + aom_highbd_calc16x16var_sse2, 16); + return *sse; +} + +unsigned int aom_highbd_12_mse16x16_sse2(const uint8_t *src8, int src_stride, + const uint8_t *ref8, int ref_stride, + unsigned int *sse) { + int sum; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + highbd_12_variance_sse2(src, src_stride, ref, ref_stride, 16, 16, sse, &sum, + aom_highbd_calc16x16var_sse2, 16); + return *sse; +} + +unsigned int aom_highbd_8_mse8x8_sse2(const uint8_t *src8, int src_stride, + const uint8_t *ref8, int ref_stride, + unsigned int *sse) { + int sum; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + highbd_8_variance_sse2(src, src_stride, ref, ref_stride, 8, 8, sse, &sum, + aom_highbd_calc8x8var_sse2, 8); + return *sse; +} + +unsigned int aom_highbd_10_mse8x8_sse2(const uint8_t *src8, int src_stride, + const uint8_t *ref8, int ref_stride, + unsigned int *sse) { + int sum; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + highbd_10_variance_sse2(src, src_stride, ref, ref_stride, 8, 8, sse, &sum, + aom_highbd_calc8x8var_sse2, 8); + return *sse; +} + +unsigned int aom_highbd_12_mse8x8_sse2(const uint8_t *src8, int src_stride, + const uint8_t *ref8, int ref_stride, + unsigned int *sse) { + int sum; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + highbd_12_variance_sse2(src, src_stride, ref, ref_stride, 8, 8, sse, &sum, + aom_highbd_calc8x8var_sse2, 8); + return *sse; +} + +// The 2 unused parameters are place holders for PIC enabled build. +// These definitions are for functions defined in +// highbd_subpel_variance_impl_sse2.asm +#define DECL(w, opt) \ + int aom_highbd_sub_pixel_variance##w##xh_##opt( \ + const uint16_t *src, ptrdiff_t src_stride, int x_offset, int y_offset, \ + const uint16_t *dst, ptrdiff_t dst_stride, int height, \ + unsigned int *sse, void *unused0, void *unused); +#define DECLS(opt) \ + DECL(8, opt); \ + DECL(16, opt) + +DECLS(sse2); + +#undef DECLS +#undef DECL + +#define FN(w, h, wf, wlog2, hlog2, opt, cast) \ + uint32_t aom_highbd_8_sub_pixel_variance##w##x##h##_##opt( \ + const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ + const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr) { \ + uint32_t sse; \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ + int se = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ + src, src_stride, x_offset, y_offset, dst, dst_stride, h, &sse, NULL, \ + NULL); \ + if (w > wf) { \ + unsigned int sse2; \ + int se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ + src + 16, src_stride, x_offset, y_offset, dst + 16, dst_stride, h, \ + &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + if (w > wf * 2) { \ + se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ + src + 32, src_stride, x_offset, y_offset, dst + 32, dst_stride, h, \ + &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ + src + 48, src_stride, x_offset, y_offset, dst + 48, dst_stride, h, \ + &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + } \ + } \ + *sse_ptr = sse; \ + return sse - (uint32_t)((cast se * se) >> (wlog2 + hlog2)); \ + } \ + \ + uint32_t aom_highbd_10_sub_pixel_variance##w##x##h##_##opt( \ + const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ + const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr) { \ + int64_t var; \ + uint32_t sse; \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ + int se = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ + src, src_stride, x_offset, y_offset, dst, dst_stride, h, &sse, NULL, \ + NULL); \ + if (w > wf) { \ + uint32_t sse2; \ + int se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ + src + 16, src_stride, x_offset, y_offset, dst + 16, dst_stride, h, \ + &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + if (w > wf * 2) { \ + se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ + src + 32, src_stride, x_offset, y_offset, dst + 32, dst_stride, h, \ + &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ + src + 48, src_stride, x_offset, y_offset, dst + 48, dst_stride, h, \ + &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + } \ + } \ + se = ROUND_POWER_OF_TWO(se, 2); \ + sse = ROUND_POWER_OF_TWO(sse, 4); \ + *sse_ptr = sse; \ + var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } \ + \ + uint32_t aom_highbd_12_sub_pixel_variance##w##x##h##_##opt( \ + const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ + const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr) { \ + int start_row; \ + uint32_t sse; \ + int se = 0; \ + int64_t var; \ + uint64_t long_sse = 0; \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ + for (start_row = 0; start_row < h; start_row += 16) { \ + uint32_t sse2; \ + int height = h - start_row < 16 ? h - start_row : 16; \ + int se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ + src + (start_row * src_stride), src_stride, x_offset, y_offset, \ + dst + (start_row * dst_stride), dst_stride, height, &sse2, NULL, \ + NULL); \ + se += se2; \ + long_sse += sse2; \ + if (w > wf) { \ + se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ + src + 16 + (start_row * src_stride), src_stride, x_offset, \ + y_offset, dst + 16 + (start_row * dst_stride), dst_stride, height, \ + &sse2, NULL, NULL); \ + se += se2; \ + long_sse += sse2; \ + if (w > wf * 2) { \ + se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ + src + 32 + (start_row * src_stride), src_stride, x_offset, \ + y_offset, dst + 32 + (start_row * dst_stride), dst_stride, \ + height, &sse2, NULL, NULL); \ + se += se2; \ + long_sse += sse2; \ + se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ + src + 48 + (start_row * src_stride), src_stride, x_offset, \ + y_offset, dst + 48 + (start_row * dst_stride), dst_stride, \ + height, &sse2, NULL, NULL); \ + se += se2; \ + long_sse += sse2; \ + } \ + } \ + } \ + se = ROUND_POWER_OF_TWO(se, 4); \ + sse = (uint32_t)ROUND_POWER_OF_TWO(long_sse, 8); \ + *sse_ptr = sse; \ + var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } + +#define FNS(opt) \ + FN(64, 64, 16, 6, 6, opt, (int64_t)); \ + FN(64, 32, 16, 6, 5, opt, (int64_t)); \ + FN(32, 64, 16, 5, 6, opt, (int64_t)); \ + FN(32, 32, 16, 5, 5, opt, (int64_t)); \ + FN(32, 16, 16, 5, 4, opt, (int64_t)); \ + FN(16, 32, 16, 4, 5, opt, (int64_t)); \ + FN(16, 16, 16, 4, 4, opt, (int64_t)); \ + FN(16, 8, 16, 4, 3, opt, (int64_t)); \ + FN(8, 16, 8, 3, 4, opt, (int64_t)); \ + FN(8, 8, 8, 3, 3, opt, (int64_t)); \ + FN(8, 4, 8, 3, 2, opt, (int64_t)); + +FNS(sse2); + +#undef FNS +#undef FN + +// The 2 unused parameters are place holders for PIC enabled build. +#define DECL(w, opt) \ + int aom_highbd_sub_pixel_avg_variance##w##xh_##opt( \ + const uint16_t *src, ptrdiff_t src_stride, int x_offset, int y_offset, \ + const uint16_t *dst, ptrdiff_t dst_stride, const uint16_t *sec, \ + ptrdiff_t sec_stride, int height, unsigned int *sse, void *unused0, \ + void *unused); +#define DECLS(opt1) \ + DECL(16, opt1) \ + DECL(8, opt1) + +DECLS(sse2); +#undef DECL +#undef DECLS + +#define FN(w, h, wf, wlog2, hlog2, opt, cast) \ + uint32_t aom_highbd_8_sub_pixel_avg_variance##w##x##h##_##opt( \ + const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ + const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr, \ + const uint8_t *sec8) { \ + uint32_t sse; \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ + uint16_t *sec = CONVERT_TO_SHORTPTR(sec8); \ + int se = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ + src, src_stride, x_offset, y_offset, dst, dst_stride, sec, w, h, &sse, \ + NULL, NULL); \ + if (w > wf) { \ + uint32_t sse2; \ + int se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ + src + 16, src_stride, x_offset, y_offset, dst + 16, dst_stride, \ + sec + 16, w, h, &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + if (w > wf * 2) { \ + se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ + src + 32, src_stride, x_offset, y_offset, dst + 32, dst_stride, \ + sec + 32, w, h, &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ + src + 48, src_stride, x_offset, y_offset, dst + 48, dst_stride, \ + sec + 48, w, h, &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + } \ + } \ + *sse_ptr = sse; \ + return sse - (uint32_t)((cast se * se) >> (wlog2 + hlog2)); \ + } \ + \ + uint32_t aom_highbd_10_sub_pixel_avg_variance##w##x##h##_##opt( \ + const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ + const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr, \ + const uint8_t *sec8) { \ + int64_t var; \ + uint32_t sse; \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ + uint16_t *sec = CONVERT_TO_SHORTPTR(sec8); \ + int se = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ + src, src_stride, x_offset, y_offset, dst, dst_stride, sec, w, h, &sse, \ + NULL, NULL); \ + if (w > wf) { \ + uint32_t sse2; \ + int se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ + src + 16, src_stride, x_offset, y_offset, dst + 16, dst_stride, \ + sec + 16, w, h, &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + if (w > wf * 2) { \ + se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ + src + 32, src_stride, x_offset, y_offset, dst + 32, dst_stride, \ + sec + 32, w, h, &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ + src + 48, src_stride, x_offset, y_offset, dst + 48, dst_stride, \ + sec + 48, w, h, &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + } \ + } \ + se = ROUND_POWER_OF_TWO(se, 2); \ + sse = ROUND_POWER_OF_TWO(sse, 4); \ + *sse_ptr = sse; \ + var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } \ + \ + uint32_t aom_highbd_12_sub_pixel_avg_variance##w##x##h##_##opt( \ + const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ + const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr, \ + const uint8_t *sec8) { \ + int start_row; \ + int64_t var; \ + uint32_t sse; \ + int se = 0; \ + uint64_t long_sse = 0; \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ + uint16_t *sec = CONVERT_TO_SHORTPTR(sec8); \ + for (start_row = 0; start_row < h; start_row += 16) { \ + uint32_t sse2; \ + int height = h - start_row < 16 ? h - start_row : 16; \ + int se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ + src + (start_row * src_stride), src_stride, x_offset, y_offset, \ + dst + (start_row * dst_stride), dst_stride, sec + (start_row * w), \ + w, height, &sse2, NULL, NULL); \ + se += se2; \ + long_sse += sse2; \ + if (w > wf) { \ + se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ + src + 16 + (start_row * src_stride), src_stride, x_offset, \ + y_offset, dst + 16 + (start_row * dst_stride), dst_stride, \ + sec + 16 + (start_row * w), w, height, &sse2, NULL, NULL); \ + se += se2; \ + long_sse += sse2; \ + if (w > wf * 2) { \ + se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ + src + 32 + (start_row * src_stride), src_stride, x_offset, \ + y_offset, dst + 32 + (start_row * dst_stride), dst_stride, \ + sec + 32 + (start_row * w), w, height, &sse2, NULL, NULL); \ + se += se2; \ + long_sse += sse2; \ + se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ + src + 48 + (start_row * src_stride), src_stride, x_offset, \ + y_offset, dst + 48 + (start_row * dst_stride), dst_stride, \ + sec + 48 + (start_row * w), w, height, &sse2, NULL, NULL); \ + se += se2; \ + long_sse += sse2; \ + } \ + } \ + } \ + se = ROUND_POWER_OF_TWO(se, 4); \ + sse = (uint32_t)ROUND_POWER_OF_TWO(long_sse, 8); \ + *sse_ptr = sse; \ + var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } + +#define FNS(opt1) \ + FN(64, 64, 16, 6, 6, opt1, (int64_t)); \ + FN(64, 32, 16, 6, 5, opt1, (int64_t)); \ + FN(32, 64, 16, 5, 6, opt1, (int64_t)); \ + FN(32, 32, 16, 5, 5, opt1, (int64_t)); \ + FN(32, 16, 16, 5, 4, opt1, (int64_t)); \ + FN(16, 32, 16, 4, 5, opt1, (int64_t)); \ + FN(16, 16, 16, 4, 4, opt1, (int64_t)); \ + FN(16, 8, 16, 4, 3, opt1, (int64_t)); \ + FN(8, 16, 8, 4, 3, opt1, (int64_t)); \ + FN(8, 8, 8, 3, 3, opt1, (int64_t)); \ + FN(8, 4, 8, 3, 2, opt1, (int64_t)); + +FNS(sse2); + +#undef FNS +#undef FN + +void aom_highbd_upsampled_pred_sse2(uint16_t *comp_pred, int width, int height, + const uint8_t *ref8, int ref_stride) { + int i, j; + int stride = ref_stride << 3; + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + + if (width >= 8) { + // read 8 points at one time + for (i = 0; i < height; i++) { + for (j = 0; j < width; j += 8) { + __m128i s0 = _mm_cvtsi32_si128(*(const uint32_t *)ref); + __m128i s1 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 8)); + __m128i s2 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 16)); + __m128i s3 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 24)); + __m128i s4 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 32)); + __m128i s5 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 40)); + __m128i s6 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 48)); + __m128i s7 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 56)); + __m128i t0, t1, t2, t3; + + t0 = _mm_unpacklo_epi16(s0, s1); + t1 = _mm_unpacklo_epi16(s2, s3); + t2 = _mm_unpacklo_epi16(s4, s5); + t3 = _mm_unpacklo_epi16(s6, s7); + t0 = _mm_unpacklo_epi32(t0, t1); + t2 = _mm_unpacklo_epi32(t2, t3); + t0 = _mm_unpacklo_epi64(t0, t2); + + _mm_storeu_si128((__m128i *)(comp_pred), t0); + comp_pred += 8; + ref += 64; // 8 * 8; + } + ref += stride - (width << 3); + } + } else { + // read 4 points at one time + for (i = 0; i < height; i++) { + for (j = 0; j < width; j += 4) { + __m128i s0 = _mm_cvtsi32_si128(*(const uint32_t *)ref); + __m128i s1 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 8)); + __m128i s2 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 16)); + __m128i s3 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 24)); + __m128i t0, t1; + + t0 = _mm_unpacklo_epi16(s0, s1); + t1 = _mm_unpacklo_epi16(s2, s3); + t0 = _mm_unpacklo_epi32(t0, t1); + + _mm_storel_epi64((__m128i *)(comp_pred), t0); + comp_pred += 4; + ref += 4 * 8; + } + ref += stride - (width << 3); + } + } +} + +void aom_highbd_comp_avg_upsampled_pred_sse2(uint16_t *comp_pred, + const uint8_t *pred8, int width, + int height, const uint8_t *ref8, + int ref_stride) { + const __m128i one = _mm_set1_epi16(1); + int i, j; + int stride = ref_stride << 3; + uint16_t *pred = CONVERT_TO_SHORTPTR(pred8); + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + + if (width >= 8) { + // read 8 points at one time + for (i = 0; i < height; i++) { + for (j = 0; j < width; j += 8) { + __m128i s0 = _mm_cvtsi32_si128(*(const uint32_t *)ref); + __m128i s1 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 8)); + __m128i s2 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 16)); + __m128i s3 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 24)); + __m128i s4 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 32)); + __m128i s5 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 40)); + __m128i s6 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 48)); + __m128i s7 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 56)); + __m128i p0 = _mm_loadu_si128((const __m128i *)pred); + __m128i t0, t1, t2, t3; + + t0 = _mm_unpacklo_epi16(s0, s1); + t1 = _mm_unpacklo_epi16(s2, s3); + t2 = _mm_unpacklo_epi16(s4, s5); + t3 = _mm_unpacklo_epi16(s6, s7); + t0 = _mm_unpacklo_epi32(t0, t1); + t2 = _mm_unpacklo_epi32(t2, t3); + t0 = _mm_unpacklo_epi64(t0, t2); + + p0 = _mm_adds_epu16(t0, p0); + p0 = _mm_adds_epu16(p0, one); + p0 = _mm_srli_epi16(p0, 1); + + _mm_storeu_si128((__m128i *)(comp_pred), p0); + comp_pred += 8; + pred += 8; + ref += 8 * 8; + } + ref += stride - (width << 3); + } + } else { + // read 4 points at one time + for (i = 0; i < height; i++) { + for (j = 0; j < width; j += 4) { + __m128i s0 = _mm_cvtsi32_si128(*(const uint32_t *)ref); + __m128i s1 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 8)); + __m128i s2 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 16)); + __m128i s3 = _mm_cvtsi32_si128(*(const uint32_t *)(ref + 24)); + __m128i p0 = _mm_loadl_epi64((const __m128i *)pred); + __m128i t0, t1; + + t0 = _mm_unpacklo_epi16(s0, s1); + t1 = _mm_unpacklo_epi16(s2, s3); + t0 = _mm_unpacklo_epi32(t0, t1); + + p0 = _mm_adds_epu16(t0, p0); + p0 = _mm_adds_epu16(p0, one); + p0 = _mm_srli_epi16(p0, 1); + + _mm_storel_epi64((__m128i *)(comp_pred), p0); + comp_pred += 4; + pred += 4; + ref += 4 * 8; + } + ref += stride - (width << 3); + } + } +} diff --git a/third_party/aom/aom_dsp/x86/highbd_variance_sse4.c b/third_party/aom/aom_dsp/x86/highbd_variance_sse4.c new file mode 100644 index 0000000000..cc7f528113 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_variance_sse4.c @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include /* SSE4.1 */ + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "aom_dsp/variance.h" +#include "aom_dsp/aom_filter.h" + +static INLINE void variance4x4_64_sse4_1(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, + uint64_t *sse, int64_t *sum) { + __m128i u0, u1, u2, u3; + __m128i s0, s1, s2, s3; + __m128i t0, t1, x0, y0; + __m128i a0, a1, a2, a3; + __m128i b0, b1, b2, b3; + __m128i k_one_epi16 = _mm_set1_epi16((int16_t)1); + + uint16_t *a = CONVERT_TO_SHORTPTR(a8); + uint16_t *b = CONVERT_TO_SHORTPTR(b8); + + a0 = _mm_loadl_epi64((__m128i const *)(a + 0 * a_stride)); + a1 = _mm_loadl_epi64((__m128i const *)(a + 1 * a_stride)); + a2 = _mm_loadl_epi64((__m128i const *)(a + 2 * a_stride)); + a3 = _mm_loadl_epi64((__m128i const *)(a + 3 * a_stride)); + + b0 = _mm_loadl_epi64((__m128i const *)(b + 0 * b_stride)); + b1 = _mm_loadl_epi64((__m128i const *)(b + 1 * b_stride)); + b2 = _mm_loadl_epi64((__m128i const *)(b + 2 * b_stride)); + b3 = _mm_loadl_epi64((__m128i const *)(b + 3 * b_stride)); + + u0 = _mm_unpacklo_epi16(a0, a1); + u1 = _mm_unpacklo_epi16(a2, a3); + u2 = _mm_unpacklo_epi16(b0, b1); + u3 = _mm_unpacklo_epi16(b2, b3); + + s0 = _mm_sub_epi16(u0, u2); + s1 = _mm_sub_epi16(u1, u3); + + t0 = _mm_madd_epi16(s0, k_one_epi16); + t1 = _mm_madd_epi16(s1, k_one_epi16); + + s2 = _mm_hadd_epi32(t0, t1); + s3 = _mm_hadd_epi32(s2, s2); + y0 = _mm_hadd_epi32(s3, s3); + + t0 = _mm_madd_epi16(s0, s0); + t1 = _mm_madd_epi16(s1, s1); + + s2 = _mm_hadd_epi32(t0, t1); + s3 = _mm_hadd_epi32(s2, s2); + x0 = _mm_hadd_epi32(s3, s3); + + *sse = (uint64_t)_mm_extract_epi32(x0, 0); + *sum = (int64_t)_mm_extract_epi32(y0, 0); +} + +uint32_t aom_highbd_8_variance4x4_sse4_1(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + uint32_t *sse) { + int64_t sum, diff; + uint64_t local_sse; + + variance4x4_64_sse4_1(a, a_stride, b, b_stride, &local_sse, &sum); + *sse = (uint32_t)local_sse; + + diff = (int64_t)*sse - ((sum * sum) >> 4); + return (diff >= 0) ? (uint32_t)diff : 0; +} + +uint32_t aom_highbd_10_variance4x4_sse4_1(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + uint32_t *sse) { + int64_t sum, diff; + uint64_t local_sse; + + variance4x4_64_sse4_1(a, a_stride, b, b_stride, &local_sse, &sum); + *sse = (uint32_t)ROUND_POWER_OF_TWO(local_sse, 4); + sum = ROUND_POWER_OF_TWO(sum, 2); + + diff = (int64_t)*sse - ((sum * sum) >> 4); + return (diff >= 0) ? (uint32_t)diff : 0; +} + +uint32_t aom_highbd_12_variance4x4_sse4_1(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + uint32_t *sse) { + int64_t sum, diff; + uint64_t local_sse; + + variance4x4_64_sse4_1(a, a_stride, b, b_stride, &local_sse, &sum); + *sse = (uint32_t)ROUND_POWER_OF_TWO(local_sse, 8); + sum = ROUND_POWER_OF_TWO(sum, 4); + + diff = (int64_t)*sse - ((sum * sum) >> 4); + return diff >= 0 ? (uint32_t)diff : 0; +} + +// Sub-pixel +uint32_t aom_highbd_8_sub_pixel_variance4x4_sse4_1( + const uint8_t *src, int src_stride, int xoffset, int yoffset, + const uint8_t *dst, int dst_stride, uint32_t *sse) { + uint16_t fdata3[(4 + 1) * 4]; + uint16_t temp2[4 * 4]; + + aom_highbd_var_filter_block2d_bil_first_pass( + src, fdata3, src_stride, 1, 4 + 1, 4, bilinear_filters_2t[xoffset]); + aom_highbd_var_filter_block2d_bil_second_pass(fdata3, temp2, 4, 4, 4, 4, + bilinear_filters_2t[yoffset]); + + return aom_highbd_8_variance4x4(CONVERT_TO_BYTEPTR(temp2), 4, dst, dst_stride, + sse); +} + +uint32_t aom_highbd_10_sub_pixel_variance4x4_sse4_1( + const uint8_t *src, int src_stride, int xoffset, int yoffset, + const uint8_t *dst, int dst_stride, uint32_t *sse) { + uint16_t fdata3[(4 + 1) * 4]; + uint16_t temp2[4 * 4]; + + aom_highbd_var_filter_block2d_bil_first_pass( + src, fdata3, src_stride, 1, 4 + 1, 4, bilinear_filters_2t[xoffset]); + aom_highbd_var_filter_block2d_bil_second_pass(fdata3, temp2, 4, 4, 4, 4, + bilinear_filters_2t[yoffset]); + + return aom_highbd_10_variance4x4(CONVERT_TO_BYTEPTR(temp2), 4, dst, + dst_stride, sse); +} + +uint32_t aom_highbd_12_sub_pixel_variance4x4_sse4_1( + const uint8_t *src, int src_stride, int xoffset, int yoffset, + const uint8_t *dst, int dst_stride, uint32_t *sse) { + uint16_t fdata3[(4 + 1) * 4]; + uint16_t temp2[4 * 4]; + + aom_highbd_var_filter_block2d_bil_first_pass( + src, fdata3, src_stride, 1, 4 + 1, 4, bilinear_filters_2t[xoffset]); + aom_highbd_var_filter_block2d_bil_second_pass(fdata3, temp2, 4, 4, 4, 4, + bilinear_filters_2t[yoffset]); + + return aom_highbd_12_variance4x4(CONVERT_TO_BYTEPTR(temp2), 4, dst, + dst_stride, sse); +} + +// Sub-pixel average + +uint32_t aom_highbd_8_sub_pixel_avg_variance4x4_sse4_1( + const uint8_t *src, int src_stride, int xoffset, int yoffset, + const uint8_t *dst, int dst_stride, uint32_t *sse, + const uint8_t *second_pred) { + uint16_t fdata3[(4 + 1) * 4]; + uint16_t temp2[4 * 4]; + DECLARE_ALIGNED(16, uint16_t, temp3[4 * 4]); + + aom_highbd_var_filter_block2d_bil_first_pass( + src, fdata3, src_stride, 1, 4 + 1, 4, bilinear_filters_2t[xoffset]); + aom_highbd_var_filter_block2d_bil_second_pass(fdata3, temp2, 4, 4, 4, 4, + bilinear_filters_2t[yoffset]); + + aom_highbd_comp_avg_pred(temp3, second_pred, 4, 4, CONVERT_TO_BYTEPTR(temp2), + 4); + + return aom_highbd_8_variance4x4(CONVERT_TO_BYTEPTR(temp3), 4, dst, dst_stride, + sse); +} + +uint32_t aom_highbd_10_sub_pixel_avg_variance4x4_sse4_1( + const uint8_t *src, int src_stride, int xoffset, int yoffset, + const uint8_t *dst, int dst_stride, uint32_t *sse, + const uint8_t *second_pred) { + uint16_t fdata3[(4 + 1) * 4]; + uint16_t temp2[4 * 4]; + DECLARE_ALIGNED(16, uint16_t, temp3[4 * 4]); + + aom_highbd_var_filter_block2d_bil_first_pass( + src, fdata3, src_stride, 1, 4 + 1, 4, bilinear_filters_2t[xoffset]); + aom_highbd_var_filter_block2d_bil_second_pass(fdata3, temp2, 4, 4, 4, 4, + bilinear_filters_2t[yoffset]); + + aom_highbd_comp_avg_pred(temp3, second_pred, 4, 4, CONVERT_TO_BYTEPTR(temp2), + 4); + + return aom_highbd_10_variance4x4(CONVERT_TO_BYTEPTR(temp3), 4, dst, + dst_stride, sse); +} + +uint32_t aom_highbd_12_sub_pixel_avg_variance4x4_sse4_1( + const uint8_t *src, int src_stride, int xoffset, int yoffset, + const uint8_t *dst, int dst_stride, uint32_t *sse, + const uint8_t *second_pred) { + uint16_t fdata3[(4 + 1) * 4]; + uint16_t temp2[4 * 4]; + DECLARE_ALIGNED(16, uint16_t, temp3[4 * 4]); + + aom_highbd_var_filter_block2d_bil_first_pass( + src, fdata3, src_stride, 1, 4 + 1, 4, bilinear_filters_2t[xoffset]); + aom_highbd_var_filter_block2d_bil_second_pass(fdata3, temp2, 4, 4, 4, 4, + bilinear_filters_2t[yoffset]); + + aom_highbd_comp_avg_pred(temp3, second_pred, 4, 4, CONVERT_TO_BYTEPTR(temp2), + 4); + + return aom_highbd_12_variance4x4(CONVERT_TO_BYTEPTR(temp3), 4, dst, + dst_stride, sse); +} diff --git a/third_party/aom/aom_dsp/x86/intrapred_sse2.asm b/third_party/aom/aom_dsp/x86/intrapred_sse2.asm new file mode 100644 index 0000000000..02567db492 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/intrapred_sse2.asm @@ -0,0 +1,771 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION_RODATA +pb_1: times 16 db 1 +pw_4: times 8 dw 4 +pw_8: times 8 dw 8 +pw_16: times 8 dw 16 +pw_32: times 8 dw 32 +dc_128: times 16 db 128 +pw2_4: times 8 dw 2 +pw2_8: times 8 dw 4 +pw2_16: times 8 dw 8 +pw2_32: times 8 dw 16 + +SECTION .text + +; ------------------------------------------ +; input: x, y, z, result +; +; trick from pascal +; (x+2y+z+2)>>2 can be calculated as: +; result = avg(x,z) +; result -= xor(x,z) & 1 +; result = avg(result,y) +; ------------------------------------------ +%macro X_PLUS_2Y_PLUS_Z_PLUS_2_RSH_2 4 + pavgb %4, %1, %3 + pxor %3, %1 + pand %3, [GLOBAL(pb_1)] + psubb %4, %3 + pavgb %4, %2 +%endmacro + +INIT_XMM sse2 +cglobal dc_predictor_4x4, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + movd m2, [leftq] + movd m0, [aboveq] + pxor m1, m1 + punpckldq m0, m2 + psadbw m0, m1 + paddw m0, [GLOBAL(pw_4)] + psraw m0, 3 + pshuflw m0, m0, 0x0 + packuswb m0, m0 + movd [dstq ], m0 + movd [dstq+strideq], m0 + lea dstq, [dstq+strideq*2] + movd [dstq ], m0 + movd [dstq+strideq], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_left_predictor_4x4, 2, 5, 2, dst, stride, above, left, goffset + movifnidn leftq, leftmp + GET_GOT goffsetq + + pxor m1, m1 + movd m0, [leftq] + psadbw m0, m1 + paddw m0, [GLOBAL(pw2_4)] + psraw m0, 2 + pshuflw m0, m0, 0x0 + packuswb m0, m0 + movd [dstq ], m0 + movd [dstq+strideq], m0 + lea dstq, [dstq+strideq*2] + movd [dstq ], m0 + movd [dstq+strideq], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_top_predictor_4x4, 3, 5, 2, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + movd m0, [aboveq] + psadbw m0, m1 + paddw m0, [GLOBAL(pw2_4)] + psraw m0, 2 + pshuflw m0, m0, 0x0 + packuswb m0, m0 + movd [dstq ], m0 + movd [dstq+strideq], m0 + lea dstq, [dstq+strideq*2] + movd [dstq ], m0 + movd [dstq+strideq], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_predictor_8x8, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + movq m0, [aboveq] + movq m2, [leftq] + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + psadbw m0, m1 + psadbw m2, m1 + paddw m0, m2 + paddw m0, [GLOBAL(pw_8)] + psraw m0, 4 + punpcklbw m0, m0 + pshuflw m0, m0, 0x0 + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_top_predictor_8x8, 3, 5, 2, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + movq m0, [aboveq] + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + psadbw m0, m1 + paddw m0, [GLOBAL(pw2_8)] + psraw m0, 3 + punpcklbw m0, m0 + pshuflw m0, m0, 0x0 + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_left_predictor_8x8, 2, 5, 2, dst, stride, above, left, goffset + movifnidn leftq, leftmp + GET_GOT goffsetq + + pxor m1, m1 + movq m0, [leftq] + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + psadbw m0, m1 + paddw m0, [GLOBAL(pw2_8)] + psraw m0, 3 + punpcklbw m0, m0 + pshuflw m0, m0, 0x0 + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_128_predictor_4x4, 2, 5, 1, dst, stride, above, left, goffset + GET_GOT goffsetq + + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + movd m0, [GLOBAL(dc_128)] + movd [dstq ], m0 + movd [dstq+strideq ], m0 + movd [dstq+strideq*2], m0 + movd [dstq+stride3q ], m0 + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_128_predictor_8x8, 2, 5, 1, dst, stride, above, left, goffset + GET_GOT goffsetq + + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + movq m0, [GLOBAL(dc_128)] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [aboveq] + mova m2, [leftq] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 4 + psadbw m0, m1 + psadbw m2, m1 + paddw m0, m2 + movhlps m2, m0 + paddw m0, m2 + paddw m0, [GLOBAL(pw_16)] + psraw m0, 5 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + packuswb m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + + +INIT_XMM sse2 +cglobal dc_top_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [aboveq] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 4 + psadbw m0, m1 + movhlps m2, m0 + paddw m0, m2 + paddw m0, [GLOBAL(pw2_16)] + psraw m0, 4 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + packuswb m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal dc_left_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [leftq] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 4 + psadbw m0, m1 + movhlps m2, m0 + paddw m0, m2 + paddw m0, [GLOBAL(pw2_16)] + psraw m0, 4 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + packuswb m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal dc_128_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 4 + mova m0, [GLOBAL(dc_128)] +.loop: + mova [dstq ], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + RESTORE_GOT + RET + + +INIT_XMM sse2 +cglobal dc_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [aboveq] + mova m2, [aboveq+16] + mova m3, [leftq] + mova m4, [leftq+16] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 8 + psadbw m0, m1 + psadbw m2, m1 + psadbw m3, m1 + psadbw m4, m1 + paddw m0, m2 + paddw m0, m3 + paddw m0, m4 + movhlps m2, m0 + paddw m0, m2 + paddw m0, [GLOBAL(pw_32)] + psraw m0, 6 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + packuswb m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq +16], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq +16], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m0 + mova [dstq+stride3q ], m0 + mova [dstq+stride3q +16], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal dc_top_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [aboveq] + mova m2, [aboveq+16] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 8 + psadbw m0, m1 + psadbw m2, m1 + paddw m0, m2 + movhlps m2, m0 + paddw m0, m2 + paddw m0, [GLOBAL(pw2_32)] + psraw m0, 5 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + packuswb m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq +16], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq +16], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m0 + mova [dstq+stride3q ], m0 + mova [dstq+stride3q +16], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal dc_left_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [leftq] + mova m2, [leftq+16] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 8 + psadbw m0, m1 + psadbw m2, m1 + paddw m0, m2 + movhlps m2, m0 + paddw m0, m2 + paddw m0, [GLOBAL(pw2_32)] + psraw m0, 5 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + packuswb m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq +16], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq +16], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m0 + mova [dstq+stride3q ], m0 + mova [dstq+stride3q +16], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal dc_128_predictor_32x32, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 8 + mova m0, [GLOBAL(dc_128)] +.loop: + mova [dstq ], m0 + mova [dstq +16], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq +16], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m0 + mova [dstq+stride3q ], m0 + mova [dstq+stride3q +16], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal v_predictor_4x4, 3, 3, 1, dst, stride, above + movd m0, [aboveq] + movd [dstq ], m0 + movd [dstq+strideq], m0 + lea dstq, [dstq+strideq*2] + movd [dstq ], m0 + movd [dstq+strideq], m0 + RET + +INIT_XMM sse2 +cglobal v_predictor_8x8, 3, 3, 1, dst, stride, above + movq m0, [aboveq] + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + RET + +INIT_XMM sse2 +cglobal v_predictor_16x16, 3, 4, 1, dst, stride, above + mova m0, [aboveq] + DEFINE_ARGS dst, stride, stride3, nlines4 + lea stride3q, [strideq*3] + mov nlines4d, 4 +.loop: + mova [dstq ], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + dec nlines4d + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal v_predictor_32x32, 3, 4, 2, dst, stride, above + mova m0, [aboveq] + mova m1, [aboveq+16] + DEFINE_ARGS dst, stride, stride3, nlines4 + lea stride3q, [strideq*3] + mov nlines4d, 8 +.loop: + mova [dstq ], m0 + mova [dstq +16], m1 + mova [dstq+strideq ], m0 + mova [dstq+strideq +16], m1 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m1 + mova [dstq+stride3q ], m0 + mova [dstq+stride3q +16], m1 + lea dstq, [dstq+strideq*4] + dec nlines4d + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal h_predictor_4x4, 2, 4, 4, dst, stride, line, left + movifnidn leftq, leftmp + movd m0, [leftq] + punpcklbw m0, m0 + punpcklbw m0, m0 + pshufd m1, m0, 0x1 + movd [dstq ], m0 + movd [dstq+strideq], m1 + pshufd m2, m0, 0x2 + lea dstq, [dstq+strideq*2] + pshufd m3, m0, 0x3 + movd [dstq ], m2 + movd [dstq+strideq], m3 + RET + +INIT_XMM sse2 +cglobal h_predictor_8x8, 2, 5, 3, dst, stride, line, left + movifnidn leftq, leftmp + mov lineq, -2 + DEFINE_ARGS dst, stride, line, left, stride3 + lea stride3q, [strideq*3] + movq m0, [leftq ] + punpcklbw m0, m0 ; l1 l1 l2 l2 ... l8 l8 +.loop: + pshuflw m1, m0, 0x0 ; l1 l1 l1 l1 l1 l1 l1 l1 + pshuflw m2, m0, 0x55 ; l2 l2 l2 l2 l2 l2 l2 l2 + movq [dstq ], m1 + movq [dstq+strideq], m2 + pshuflw m1, m0, 0xaa + pshuflw m2, m0, 0xff + movq [dstq+strideq*2], m1 + movq [dstq+stride3q ], m2 + pshufd m0, m0, 0xe ; [63:0] l5 l5 l6 l6 l7 l7 l8 l8 + inc lineq + lea dstq, [dstq+strideq*4] + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal h_predictor_16x16, 2, 5, 3, dst, stride, line, left + movifnidn leftq, leftmp + mov lineq, -4 + DEFINE_ARGS dst, stride, line, left, stride3 + lea stride3q, [strideq*3] +.loop: + movd m0, [leftq] + punpcklbw m0, m0 + punpcklbw m0, m0 ; l1 to l4 each repeated 4 times + pshufd m1, m0, 0x0 ; l1 repeated 16 times + pshufd m2, m0, 0x55 ; l2 repeated 16 times + mova [dstq ], m1 + mova [dstq+strideq ], m2 + pshufd m1, m0, 0xaa + pshufd m2, m0, 0xff + mova [dstq+strideq*2], m1 + mova [dstq+stride3q ], m2 + inc lineq + lea leftq, [leftq+4 ] + lea dstq, [dstq+strideq*4] + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal h_predictor_32x32, 2, 5, 3, dst, stride, line, left + movifnidn leftq, leftmp + mov lineq, -8 + DEFINE_ARGS dst, stride, line, left, stride3 + lea stride3q, [strideq*3] +.loop: + movd m0, [leftq] + punpcklbw m0, m0 + punpcklbw m0, m0 ; l1 to l4 each repeated 4 times + pshufd m1, m0, 0x0 ; l1 repeated 16 times + pshufd m2, m0, 0x55 ; l2 repeated 16 times + mova [dstq ], m1 + mova [dstq+16 ], m1 + mova [dstq+strideq ], m2 + mova [dstq+strideq+16 ], m2 + pshufd m1, m0, 0xaa + pshufd m2, m0, 0xff + mova [dstq+strideq*2 ], m1 + mova [dstq+strideq*2+16], m1 + mova [dstq+stride3q ], m2 + mova [dstq+stride3q+16 ], m2 + inc lineq + lea leftq, [leftq+4 ] + lea dstq, [dstq+strideq*4] + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal tm_predictor_4x4, 4, 4, 5, dst, stride, above, left + pxor m1, m1 + movq m0, [aboveq-1]; [63:0] tl t1 t2 t3 t4 x x x + punpcklbw m0, m1 + pshuflw m2, m0, 0x0 ; [63:0] tl tl tl tl [word] + psrldq m0, 2 + psubw m0, m2 ; [63:0] t1-tl t2-tl t3-tl t4-tl [word] + movd m2, [leftq] + punpcklbw m2, m1 + pshuflw m4, m2, 0x0 ; [63:0] l1 l1 l1 l1 [word] + pshuflw m3, m2, 0x55 ; [63:0] l2 l2 l2 l2 [word] + paddw m4, m0 + paddw m3, m0 + packuswb m4, m4 + packuswb m3, m3 + movd [dstq ], m4 + movd [dstq+strideq], m3 + lea dstq, [dstq+strideq*2] + pshuflw m4, m2, 0xaa + pshuflw m3, m2, 0xff + paddw m4, m0 + paddw m3, m0 + packuswb m4, m4 + packuswb m3, m3 + movd [dstq ], m4 + movd [dstq+strideq], m3 + RET + +INIT_XMM sse2 +cglobal tm_predictor_8x8, 4, 4, 5, dst, stride, above, left + pxor m1, m1 + movd m2, [aboveq-1] + movq m0, [aboveq] + punpcklbw m2, m1 + punpcklbw m0, m1 ; t1 t2 t3 t4 t5 t6 t7 t8 [word] + pshuflw m2, m2, 0x0 ; [63:0] tl tl tl tl [word] + DEFINE_ARGS dst, stride, line, left + mov lineq, -4 + punpcklqdq m2, m2 ; tl tl tl tl tl tl tl tl [word] + psubw m0, m2 ; t1-tl t2-tl ... t8-tl [word] + movq m2, [leftq] + punpcklbw m2, m1 ; l1 l2 l3 l4 l5 l6 l7 l8 [word] +.loop: + pshuflw m4, m2, 0x0 ; [63:0] l1 l1 l1 l1 [word] + pshuflw m3, m2, 0x55 ; [63:0] l2 l2 l2 l2 [word] + punpcklqdq m4, m4 ; l1 l1 l1 l1 l1 l1 l1 l1 [word] + punpcklqdq m3, m3 ; l2 l2 l2 l2 l2 l2 l2 l2 [word] + paddw m4, m0 + paddw m3, m0 + packuswb m4, m3 + movq [dstq ], m4 + movhps [dstq+strideq], m4 + lea dstq, [dstq+strideq*2] + psrldq m2, 4 + inc lineq + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal tm_predictor_16x16, 4, 5, 8, dst, stride, above, left + pxor m1, m1 + mova m2, [aboveq-16]; + mova m0, [aboveq] ; t1 t2 ... t16 [byte] + punpckhbw m2, m1 ; [127:112] tl [word] + punpckhbw m4, m0, m1 + punpcklbw m0, m1 ; m0:m4 t1 t2 ... t16 [word] + DEFINE_ARGS dst, stride, line, left, stride8 + mov lineq, -8 + pshufhw m2, m2, 0xff + mova m3, [leftq] ; l1 l2 ... l16 [byte] + punpckhqdq m2, m2 ; tl repeated 8 times [word] + psubw m0, m2 + psubw m4, m2 ; m0:m4 t1-tl t2-tl ... t16-tl [word] + punpckhbw m5, m3, m1 + punpcklbw m3, m1 ; m3:m5 l1 l2 ... l16 [word] + lea stride8q, [strideq*8] +.loop: + pshuflw m6, m3, 0x0 + pshuflw m7, m5, 0x0 + punpcklqdq m6, m6 ; l1 repeated 8 times [word] + punpcklqdq m7, m7 ; l8 repeated 8 times [word] + paddw m1, m6, m0 + paddw m6, m4 ; m1:m6 ti-tl+l1 [i=1,15] [word] + psrldq m5, 2 + packuswb m1, m6 + mova [dstq ], m1 + paddw m1, m7, m0 + paddw m7, m4 ; m1:m7 ti-tl+l8 [i=1,15] [word] + psrldq m3, 2 + packuswb m1, m7 + mova [dstq+stride8q], m1 + inc lineq + lea dstq, [dstq+strideq] + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal tm_predictor_32x32, 4, 4, 8, dst, stride, above, left + pxor m1, m1 + movd m2, [aboveq-1] + mova m0, [aboveq] + mova m4, [aboveq+16] + punpcklbw m2, m1 + punpckhbw m3, m0, m1 + punpckhbw m5, m4, m1 + punpcklbw m0, m1 + punpcklbw m4, m1 + pshuflw m2, m2, 0x0 + DEFINE_ARGS dst, stride, line, left + mov lineq, -16 + punpcklqdq m2, m2 + add leftq, 32 + psubw m0, m2 + psubw m3, m2 + psubw m4, m2 + psubw m5, m2 +.loop: + movd m2, [leftq+lineq*2] + pxor m1, m1 + punpcklbw m2, m1 + pshuflw m7, m2, 0x55 + pshuflw m2, m2, 0x0 + punpcklqdq m2, m2 + punpcklqdq m7, m7 + paddw m6, m2, m3 + paddw m1, m2, m0 + packuswb m1, m6 + mova [dstq ], m1 + paddw m6, m2, m5 + paddw m1, m2, m4 + packuswb m1, m6 + mova [dstq+16 ], m1 + paddw m6, m7, m3 + paddw m1, m7, m0 + packuswb m1, m6 + mova [dstq+strideq ], m1 + paddw m6, m7, m5 + paddw m1, m7, m4 + packuswb m1, m6 + mova [dstq+strideq+16], m1 + lea dstq, [dstq+strideq*2] + inc lineq + jnz .loop + REP_RET diff --git a/third_party/aom/aom_dsp/x86/intrapred_ssse3.asm b/third_party/aom/aom_dsp/x86/intrapred_ssse3.asm new file mode 100644 index 0000000000..bc1bb2ff3d --- /dev/null +++ b/third_party/aom/aom_dsp/x86/intrapred_ssse3.asm @@ -0,0 +1,410 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION_RODATA + +pb_1: times 16 db 1 +sh_b12345677: db 1, 2, 3, 4, 5, 6, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0 +sh_b23456777: db 2, 3, 4, 5, 6, 7, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0 +sh_b0123456777777777: db 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7 +sh_b1234567777777777: db 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 +sh_b2345677777777777: db 2, 3, 4, 5, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 +sh_b123456789abcdeff: db 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 15 +sh_b23456789abcdefff: db 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 15, 15 +sh_b32104567: db 3, 2, 1, 0, 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0 +sh_b8091a2b345: db 8, 0, 9, 1, 10, 2, 11, 3, 4, 5, 0, 0, 0, 0, 0, 0 +sh_b76543210: db 7, 6, 5, 4, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 +sh_b65432108: db 6, 5, 4, 3, 2, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0 +sh_b54321089: db 5, 4, 3, 2, 1, 0, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0 +sh_b89abcdef: db 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0 +sh_bfedcba9876543210: db 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 + +SECTION .text + +; ------------------------------------------ +; input: x, y, z, result +; +; trick from pascal +; (x+2y+z+2)>>2 can be calculated as: +; result = avg(x,z) +; result -= xor(x,z) & 1 +; result = avg(result,y) +; ------------------------------------------ +%macro X_PLUS_2Y_PLUS_Z_PLUS_2_RSH_2 4 + pavgb %4, %1, %3 + pxor %3, %1 + pand %3, [GLOBAL(pb_1)] + psubb %4, %3 + pavgb %4, %2 +%endmacro + +INIT_XMM ssse3 +cglobal d63e_predictor_4x4, 3, 4, 5, dst, stride, above, goffset + GET_GOT goffsetq + + movq m3, [aboveq] + pshufb m1, m3, [GLOBAL(sh_b23456777)] + pshufb m2, m3, [GLOBAL(sh_b12345677)] + + X_PLUS_2Y_PLUS_Z_PLUS_2_RSH_2 m3, m2, m1, m4 + pavgb m3, m2 + + ; store 4 lines + movd [dstq ], m3 + movd [dstq+strideq], m4 + lea dstq, [dstq+strideq*2] + psrldq m3, 1 + psrldq m4, 1 + movd [dstq ], m3 + movd [dstq+strideq], m4 + RESTORE_GOT + RET + +INIT_XMM ssse3 +cglobal d153_predictor_4x4, 4, 5, 4, dst, stride, above, left, goffset + GET_GOT goffsetq + movd m0, [leftq] ; l1, l2, l3, l4 + movd m1, [aboveq-1] ; tl, t1, t2, t3 + punpckldq m0, m1 ; l1, l2, l3, l4, tl, t1, t2, t3 + pshufb m0, [GLOBAL(sh_b32104567)]; l4, l3, l2, l1, tl, t1, t2, t3 + psrldq m1, m0, 1 ; l3, l2, l1, tl, t1, t2, t3 + psrldq m2, m0, 2 ; l2, l1, tl, t1, t2, t3 + ; comments below are for a predictor like this + ; A1 B1 C1 D1 + ; A2 B2 A1 B1 + ; A3 B3 A2 B2 + ; A4 B4 A3 B3 + X_PLUS_2Y_PLUS_Z_PLUS_2_RSH_2 m0, m1, m2, m3 ; 3-tap avg B4 B3 B2 B1 C1 D1 + pavgb m1, m0 ; 2-tap avg A4 A3 A2 A1 + + punpcklqdq m3, m1 ; B4 B3 B2 B1 C1 D1 x x A4 A3 A2 A1 .. + + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + pshufb m3, [GLOBAL(sh_b8091a2b345)] ; A4 B4 A3 B3 A2 B2 A1 B1 C1 D1 .. + movd [dstq+stride3q ], m3 + psrldq m3, 2 ; A3 B3 A2 B2 A1 B1 C1 D1 .. + movd [dstq+strideq*2], m3 + psrldq m3, 2 ; A2 B2 A1 B1 C1 D1 .. + movd [dstq+strideq ], m3 + psrldq m3, 2 ; A1 B1 C1 D1 .. + movd [dstq ], m3 + RESTORE_GOT + RET + +INIT_XMM ssse3 +cglobal d153_predictor_8x8, 4, 5, 8, dst, stride, above, left, goffset + GET_GOT goffsetq + movq m0, [leftq] ; [0- 7] l1-8 [byte] + movhps m0, [aboveq-1] ; [8-15] tl, t1-7 [byte] + pshufb m1, m0, [GLOBAL(sh_b76543210)] ; l8-1 [word] + pshufb m2, m0, [GLOBAL(sh_b65432108)] ; l7-1,tl [word] + pshufb m3, m0, [GLOBAL(sh_b54321089)] ; l6-1,tl,t1 [word] + pshufb m0, [GLOBAL(sh_b89abcdef)] ; tl,t1-7 [word] + psrldq m4, m0, 1 ; t1-7 [word] + psrldq m5, m0, 2 ; t2-7 [word] + ; comments below are for a predictor like this + ; A1 B1 C1 D1 E1 F1 G1 H1 + ; A2 B2 A1 B1 C1 D1 E1 F1 + ; A3 B3 A2 B2 A1 B1 C1 D1 + ; A4 B4 A3 B3 A2 B2 A1 B1 + ; A5 B5 A4 B4 A3 B3 A2 B2 + ; A6 B6 A5 B5 A4 B4 A3 B3 + ; A7 B7 A6 B6 A5 B5 A4 B4 + ; A8 B8 A7 B7 A6 B6 A5 B5 + pavgb m6, m1, m2 ; 2-tap avg A8-A1 + + X_PLUS_2Y_PLUS_Z_PLUS_2_RSH_2 m0, m4, m5, m7 ; 3-tap avg C-H1 + + X_PLUS_2Y_PLUS_Z_PLUS_2_RSH_2 m1, m2, m3, m0 ; 3-tap avg B8-1 + + punpcklbw m6, m0 ; A-B8, A-B7 ... A-B2, A-B1 + + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + + movhps [dstq+stride3q], m6 ; A-B4, A-B3, A-B2, A-B1 + palignr m0, m7, m6, 10 ; A-B3, A-B2, A-B1, C-H1 + movq [dstq+strideq*2], m0 + psrldq m0, 2 ; A-B2, A-B1, C-H1 + movq [dstq+strideq ], m0 + psrldq m0, 2 ; A-H1 + movq [dstq ], m0 + lea dstq, [dstq+strideq*4] + movq [dstq+stride3q ], m6 ; A-B8, A-B7, A-B6, A-B5 + psrldq m6, 2 ; A-B7, A-B6, A-B5, A-B4 + movq [dstq+strideq*2], m6 + psrldq m6, 2 ; A-B6, A-B5, A-B4, A-B3 + movq [dstq+strideq ], m6 + psrldq m6, 2 ; A-B5, A-B4, A-B3, A-B2 + movq [dstq ], m6 + RESTORE_GOT + RET + +INIT_XMM ssse3 +cglobal d153_predictor_16x16, 4, 5, 8, dst, stride, above, left, goffset + GET_GOT goffsetq + mova m0, [leftq] + movu m7, [aboveq-1] + ; comments below are for a predictor like this + ; A1 B1 C1 D1 E1 F1 G1 H1 I1 J1 K1 L1 M1 N1 O1 P1 + ; A2 B2 A1 B1 C1 D1 E1 F1 G1 H1 I1 J1 K1 L1 M1 N1 + ; A3 B3 A2 B2 A1 B1 C1 D1 E1 F1 G1 H1 I1 J1 K1 L1 + ; A4 B4 A3 B3 A2 B2 A1 B1 C1 D1 E1 F1 G1 H1 I1 J1 + ; A5 B5 A4 B4 A3 B3 A2 B2 A1 B1 C1 D1 E1 F1 G1 H1 + ; A6 B6 A5 B5 A4 B4 A3 B3 A2 B2 A1 B1 C1 D1 E1 F1 + ; A7 B7 A6 B6 A5 B5 A4 B4 A3 B3 A2 B2 A1 B1 C1 D1 + ; A8 B8 A7 B7 A6 B6 A5 B5 A4 B4 A3 B3 A2 B2 A1 B1 + ; A9 B9 A8 B8 A7 B7 A6 B6 A5 B5 A4 B4 A3 B3 A2 B2 + ; Aa Ba A9 B9 A8 B8 A7 B7 A6 B6 A5 B5 A4 B4 A3 B3 + ; Ab Bb Aa Ba A9 B9 A8 B8 A7 B7 A6 B6 A5 B5 A4 B4 + ; Ac Bc Ab Bb Aa Ba A9 B9 A8 B8 A7 B7 A6 B6 A5 B5 + ; Ad Bd Ac Bc Ab Bb Aa Ba A9 B9 A8 B8 A7 B7 A6 B6 + ; Ae Be Ad Bd Ac Bc Ab Bb Aa Ba A9 B9 A8 B8 A7 B7 + ; Af Bf Ae Be Ad Bd Ac Bc Ab Bb Aa Ba A9 B9 A8 B8 + ; Ag Bg Af Bf Ae Be Ad Bd Ac Bc Ab Bb Aa Ba A9 B9 + pshufb m6, m7, [GLOBAL(sh_bfedcba9876543210)] + palignr m5, m0, m6, 15 + palignr m3, m0, m6, 14 + + X_PLUS_2Y_PLUS_Z_PLUS_2_RSH_2 m0, m5, m3, m4 ; 3-tap avg B3-Bg + pshufb m1, m0, [GLOBAL(sh_b123456789abcdeff)] + pavgb m5, m0 ; A1 - Ag + + punpcklbw m0, m4, m5 ; A-B8 ... A-B1 + punpckhbw m4, m5 ; A-B9 ... A-Bg + + pshufb m3, m7, [GLOBAL(sh_b123456789abcdeff)] + pshufb m5, m7, [GLOBAL(sh_b23456789abcdefff)] + + X_PLUS_2Y_PLUS_Z_PLUS_2_RSH_2 m7, m3, m5, m1 ; 3-tap avg C1-P1 + + pshufb m6, m0, [GLOBAL(sh_bfedcba9876543210)] + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + palignr m2, m1, m6, 14 + mova [dstq ], m2 + palignr m2, m1, m6, 12 + mova [dstq+strideq ], m2 + palignr m2, m1, m6, 10 + mova [dstq+strideq*2], m2 + palignr m2, m1, m6, 8 + mova [dstq+stride3q ], m2 + lea dstq, [dstq+strideq*4] + palignr m2, m1, m6, 6 + mova [dstq ], m2 + palignr m2, m1, m6, 4 + mova [dstq+strideq ], m2 + palignr m2, m1, m6, 2 + mova [dstq+strideq*2], m2 + pshufb m4, [GLOBAL(sh_bfedcba9876543210)] + mova [dstq+stride3q ], m6 + lea dstq, [dstq+strideq*4] + + palignr m2, m6, m4, 14 + mova [dstq ], m2 + palignr m2, m6, m4, 12 + mova [dstq+strideq ], m2 + palignr m2, m6, m4, 10 + mova [dstq+strideq*2], m2 + palignr m2, m6, m4, 8 + mova [dstq+stride3q ], m2 + lea dstq, [dstq+strideq*4] + palignr m2, m6, m4, 6 + mova [dstq ], m2 + palignr m2, m6, m4, 4 + mova [dstq+strideq ], m2 + palignr m2, m6, m4, 2 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m4 + RESTORE_GOT + RET + +INIT_XMM ssse3 +cglobal d153_predictor_32x32, 4, 5, 8, dst, stride, above, left, goffset + GET_GOT goffsetq + mova m0, [leftq] + movu m7, [aboveq-1] + movu m1, [aboveq+15] + + pshufb m4, m1, [GLOBAL(sh_b123456789abcdeff)] + pshufb m6, m1, [GLOBAL(sh_b23456789abcdefff)] + + X_PLUS_2Y_PLUS_Z_PLUS_2_RSH_2 m1, m4, m6, m2 ; 3-tap avg above [high] + + palignr m3, m1, m7, 1 + palignr m5, m1, m7, 2 + + X_PLUS_2Y_PLUS_Z_PLUS_2_RSH_2 m7, m3, m5, m1 ; 3-tap avg above [low] + + pshufb m7, [GLOBAL(sh_bfedcba9876543210)] + palignr m5, m0, m7, 15 + palignr m3, m0, m7, 14 + + X_PLUS_2Y_PLUS_Z_PLUS_2_RSH_2 m0, m5, m3, m4 ; 3-tap avg B3-Bg + pavgb m5, m0 ; A1 - Ag + punpcklbw m6, m4, m5 ; A-B8 ... A-B1 + punpckhbw m4, m5 ; A-B9 ... A-Bg + pshufb m6, [GLOBAL(sh_bfedcba9876543210)] + pshufb m4, [GLOBAL(sh_bfedcba9876543210)] + + DEFINE_ARGS dst, stride, stride3, left, line + lea stride3q, [strideq*3] + + palignr m5, m2, m1, 14 + palignr m7, m1, m6, 14 + mova [dstq ], m7 + mova [dstq+16 ], m5 + palignr m5, m2, m1, 12 + palignr m7, m1, m6, 12 + mova [dstq+strideq ], m7 + mova [dstq+strideq+16 ], m5 + palignr m5, m2, m1, 10 + palignr m7, m1, m6, 10 + mova [dstq+strideq*2 ], m7 + mova [dstq+strideq*2+16], m5 + palignr m5, m2, m1, 8 + palignr m7, m1, m6, 8 + mova [dstq+stride3q ], m7 + mova [dstq+stride3q+16 ], m5 + lea dstq, [dstq+strideq*4] + palignr m5, m2, m1, 6 + palignr m7, m1, m6, 6 + mova [dstq ], m7 + mova [dstq+16 ], m5 + palignr m5, m2, m1, 4 + palignr m7, m1, m6, 4 + mova [dstq+strideq ], m7 + mova [dstq+strideq+16 ], m5 + palignr m5, m2, m1, 2 + palignr m7, m1, m6, 2 + mova [dstq+strideq*2 ], m7 + mova [dstq+strideq*2+16], m5 + mova [dstq+stride3q ], m6 + mova [dstq+stride3q+16 ], m1 + lea dstq, [dstq+strideq*4] + + palignr m5, m1, m6, 14 + palignr m3, m6, m4, 14 + mova [dstq ], m3 + mova [dstq+16 ], m5 + palignr m5, m1, m6, 12 + palignr m3, m6, m4, 12 + mova [dstq+strideq ], m3 + mova [dstq+strideq+16 ], m5 + palignr m5, m1, m6, 10 + palignr m3, m6, m4, 10 + mova [dstq+strideq*2 ], m3 + mova [dstq+strideq*2+16], m5 + palignr m5, m1, m6, 8 + palignr m3, m6, m4, 8 + mova [dstq+stride3q ], m3 + mova [dstq+stride3q+16 ], m5 + lea dstq, [dstq+strideq*4] + palignr m5, m1, m6, 6 + palignr m3, m6, m4, 6 + mova [dstq ], m3 + mova [dstq+16 ], m5 + palignr m5, m1, m6, 4 + palignr m3, m6, m4, 4 + mova [dstq+strideq ], m3 + mova [dstq+strideq+16 ], m5 + palignr m5, m1, m6, 2 + palignr m3, m6, m4, 2 + mova [dstq+strideq*2 ], m3 + mova [dstq+strideq*2+16], m5 + mova [dstq+stride3q ], m4 + mova [dstq+stride3q+16 ], m6 + lea dstq, [dstq+strideq*4] + + mova m7, [leftq] + mova m3, [leftq+16] + palignr m5, m3, m7, 15 + palignr m0, m3, m7, 14 + + X_PLUS_2Y_PLUS_Z_PLUS_2_RSH_2 m3, m5, m0, m2 ; 3-tap avg Bh - + pavgb m5, m3 ; Ah - + punpcklbw m3, m2, m5 ; A-B8 ... A-B1 + punpckhbw m2, m5 ; A-B9 ... A-Bg + pshufb m3, [GLOBAL(sh_bfedcba9876543210)] + pshufb m2, [GLOBAL(sh_bfedcba9876543210)] + + palignr m7, m6, m4, 14 + palignr m0, m4, m3, 14 + mova [dstq ], m0 + mova [dstq+16 ], m7 + palignr m7, m6, m4, 12 + palignr m0, m4, m3, 12 + mova [dstq+strideq ], m0 + mova [dstq+strideq+16 ], m7 + palignr m7, m6, m4, 10 + palignr m0, m4, m3, 10 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m7 + palignr m7, m6, m4, 8 + palignr m0, m4, m3, 8 + mova [dstq+stride3q ], m0 + mova [dstq+stride3q+16 ], m7 + lea dstq, [dstq+strideq*4] + palignr m7, m6, m4, 6 + palignr m0, m4, m3, 6 + mova [dstq ], m0 + mova [dstq+16 ], m7 + palignr m7, m6, m4, 4 + palignr m0, m4, m3, 4 + mova [dstq+strideq ], m0 + mova [dstq+strideq+16 ], m7 + palignr m7, m6, m4, 2 + palignr m0, m4, m3, 2 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m7 + mova [dstq+stride3q ], m3 + mova [dstq+stride3q+16 ], m4 + lea dstq, [dstq+strideq*4] + + palignr m7, m4, m3, 14 + palignr m0, m3, m2, 14 + mova [dstq ], m0 + mova [dstq+16 ], m7 + palignr m7, m4, m3, 12 + palignr m0, m3, m2, 12 + mova [dstq+strideq ], m0 + mova [dstq+strideq+16 ], m7 + palignr m7, m4, m3, 10 + palignr m0, m3, m2, 10 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m7 + palignr m7, m4, m3, 8 + palignr m0, m3, m2, 8 + mova [dstq+stride3q ], m0 + mova [dstq+stride3q+16 ], m7 + lea dstq, [dstq+strideq*4] + palignr m7, m4, m3, 6 + palignr m0, m3, m2, 6 + mova [dstq ], m0 + mova [dstq+16 ], m7 + palignr m7, m4, m3, 4 + palignr m0, m3, m2, 4 + mova [dstq+strideq ], m0 + mova [dstq+strideq+16 ], m7 + palignr m7, m4, m3, 2 + palignr m0, m3, m2, 2 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m7 + mova [dstq+stride3q ], m2 + mova [dstq+stride3q+16 ], m3 + + RESTORE_GOT + RET diff --git a/third_party/aom/aom_dsp/x86/inv_txfm_sse2.c b/third_party/aom/aom_dsp/x86/inv_txfm_sse2.c new file mode 100644 index 0000000000..5795a1845c --- /dev/null +++ b/third_party/aom/aom_dsp/x86/inv_txfm_sse2.c @@ -0,0 +1,3631 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/x86/inv_txfm_sse2.h" +#include "aom_dsp/x86/txfm_common_sse2.h" + +#define RECON_AND_STORE4X4(dest, in_x) \ + { \ + __m128i d0 = _mm_cvtsi32_si128(*(const int *)(dest)); \ + d0 = _mm_unpacklo_epi8(d0, zero); \ + d0 = _mm_add_epi16(in_x, d0); \ + d0 = _mm_packus_epi16(d0, d0); \ + *(int *)(dest) = _mm_cvtsi128_si32(d0); \ + } + +void aom_idct4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride) { + const __m128i zero = _mm_setzero_si128(); + const __m128i eight = _mm_set1_epi16(8); + const __m128i cst = _mm_setr_epi16( + (int16_t)cospi_16_64, (int16_t)cospi_16_64, (int16_t)cospi_16_64, + (int16_t)-cospi_16_64, (int16_t)cospi_24_64, (int16_t)-cospi_8_64, + (int16_t)cospi_8_64, (int16_t)cospi_24_64); + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + __m128i input0, input1, input2, input3; + + // Rows + input0 = load_input_data(input); + input2 = load_input_data(input + 8); + + // Construct i3, i1, i3, i1, i2, i0, i2, i0 + input0 = _mm_shufflelo_epi16(input0, 0xd8); + input0 = _mm_shufflehi_epi16(input0, 0xd8); + input2 = _mm_shufflelo_epi16(input2, 0xd8); + input2 = _mm_shufflehi_epi16(input2, 0xd8); + + input1 = _mm_unpackhi_epi32(input0, input0); + input0 = _mm_unpacklo_epi32(input0, input0); + input3 = _mm_unpackhi_epi32(input2, input2); + input2 = _mm_unpacklo_epi32(input2, input2); + + // Stage 1 + input0 = _mm_madd_epi16(input0, cst); + input1 = _mm_madd_epi16(input1, cst); + input2 = _mm_madd_epi16(input2, cst); + input3 = _mm_madd_epi16(input3, cst); + + input0 = _mm_add_epi32(input0, rounding); + input1 = _mm_add_epi32(input1, rounding); + input2 = _mm_add_epi32(input2, rounding); + input3 = _mm_add_epi32(input3, rounding); + + input0 = _mm_srai_epi32(input0, DCT_CONST_BITS); + input1 = _mm_srai_epi32(input1, DCT_CONST_BITS); + input2 = _mm_srai_epi32(input2, DCT_CONST_BITS); + input3 = _mm_srai_epi32(input3, DCT_CONST_BITS); + + // Stage 2 + input0 = _mm_packs_epi32(input0, input1); + input1 = _mm_packs_epi32(input2, input3); + + // Transpose + input2 = _mm_unpacklo_epi16(input0, input1); + input3 = _mm_unpackhi_epi16(input0, input1); + input0 = _mm_unpacklo_epi32(input2, input3); + input1 = _mm_unpackhi_epi32(input2, input3); + + // Switch column2, column 3, and then, we got: + // input2: column1, column 0; input3: column2, column 3. + input1 = _mm_shuffle_epi32(input1, 0x4e); + input2 = _mm_add_epi16(input0, input1); + input3 = _mm_sub_epi16(input0, input1); + + // Columns + // Construct i3, i1, i3, i1, i2, i0, i2, i0 + input0 = _mm_unpacklo_epi32(input2, input2); + input1 = _mm_unpackhi_epi32(input2, input2); + input2 = _mm_unpackhi_epi32(input3, input3); + input3 = _mm_unpacklo_epi32(input3, input3); + + // Stage 1 + input0 = _mm_madd_epi16(input0, cst); + input1 = _mm_madd_epi16(input1, cst); + input2 = _mm_madd_epi16(input2, cst); + input3 = _mm_madd_epi16(input3, cst); + + input0 = _mm_add_epi32(input0, rounding); + input1 = _mm_add_epi32(input1, rounding); + input2 = _mm_add_epi32(input2, rounding); + input3 = _mm_add_epi32(input3, rounding); + + input0 = _mm_srai_epi32(input0, DCT_CONST_BITS); + input1 = _mm_srai_epi32(input1, DCT_CONST_BITS); + input2 = _mm_srai_epi32(input2, DCT_CONST_BITS); + input3 = _mm_srai_epi32(input3, DCT_CONST_BITS); + + // Stage 2 + input0 = _mm_packs_epi32(input0, input2); + input1 = _mm_packs_epi32(input1, input3); + + // Transpose + input2 = _mm_unpacklo_epi16(input0, input1); + input3 = _mm_unpackhi_epi16(input0, input1); + input0 = _mm_unpacklo_epi32(input2, input3); + input1 = _mm_unpackhi_epi32(input2, input3); + + // Switch column2, column 3, and then, we got: + // input2: column1, column 0; input3: column2, column 3. + input1 = _mm_shuffle_epi32(input1, 0x4e); + input2 = _mm_add_epi16(input0, input1); + input3 = _mm_sub_epi16(input0, input1); + + // Final round and shift + input2 = _mm_add_epi16(input2, eight); + input3 = _mm_add_epi16(input3, eight); + + input2 = _mm_srai_epi16(input2, 4); + input3 = _mm_srai_epi16(input3, 4); + + // Reconstruction and Store + { + __m128i d0 = _mm_cvtsi32_si128(*(const int *)(dest)); + __m128i d2 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 2)); + d0 = _mm_unpacklo_epi32(d0, + _mm_cvtsi32_si128(*(const int *)(dest + stride))); + d2 = _mm_unpacklo_epi32( + _mm_cvtsi32_si128(*(const int *)(dest + stride * 3)), d2); + d0 = _mm_unpacklo_epi8(d0, zero); + d2 = _mm_unpacklo_epi8(d2, zero); + d0 = _mm_add_epi16(d0, input2); + d2 = _mm_add_epi16(d2, input3); + d0 = _mm_packus_epi16(d0, d2); + // store input0 + *(int *)dest = _mm_cvtsi128_si32(d0); + // store input1 + d0 = _mm_srli_si128(d0, 4); + *(int *)(dest + stride) = _mm_cvtsi128_si32(d0); + // store input2 + d0 = _mm_srli_si128(d0, 4); + *(int *)(dest + stride * 3) = _mm_cvtsi128_si32(d0); + // store input3 + d0 = _mm_srli_si128(d0, 4); + *(int *)(dest + stride * 2) = _mm_cvtsi128_si32(d0); + } +} + +void aom_idct4x4_1_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride) { + __m128i dc_value; + const __m128i zero = _mm_setzero_si128(); + int a; + + a = (int)dct_const_round_shift(input[0] * cospi_16_64); + a = (int)dct_const_round_shift(a * cospi_16_64); + a = ROUND_POWER_OF_TWO(a, 4); + + if (a == 0) return; + + dc_value = _mm_set1_epi16(a); + + RECON_AND_STORE4X4(dest + 0 * stride, dc_value); + RECON_AND_STORE4X4(dest + 1 * stride, dc_value); + RECON_AND_STORE4X4(dest + 2 * stride, dc_value); + RECON_AND_STORE4X4(dest + 3 * stride, dc_value); +} + +void aom_idct4_sse2(__m128i *in) { + const __m128i k__cospi_p16_p16 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + __m128i u[8], v[8]; + + array_transpose_4x4(in); + // stage 1 + u[0] = _mm_unpacklo_epi16(in[0], in[1]); + u[1] = _mm_unpackhi_epi16(in[0], in[1]); + v[0] = _mm_madd_epi16(u[0], k__cospi_p16_p16); + v[1] = _mm_madd_epi16(u[0], k__cospi_p16_m16); + v[2] = _mm_madd_epi16(u[1], k__cospi_p24_m08); + v[3] = _mm_madd_epi16(u[1], k__cospi_p08_p24); + + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + + v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + + u[0] = _mm_packs_epi32(v[0], v[1]); + u[1] = _mm_packs_epi32(v[3], v[2]); + + // stage 2 + in[0] = _mm_add_epi16(u[0], u[1]); + in[1] = _mm_sub_epi16(u[0], u[1]); + in[1] = _mm_shuffle_epi32(in[1], 0x4E); +} + +void aom_iadst4_sse2(__m128i *in) { + const __m128i k__sinpi_p01_p04 = pair_set_epi16(sinpi_1_9, sinpi_4_9); + const __m128i k__sinpi_p03_p02 = pair_set_epi16(sinpi_3_9, sinpi_2_9); + const __m128i k__sinpi_p02_m01 = pair_set_epi16(sinpi_2_9, -sinpi_1_9); + const __m128i k__sinpi_p03_m04 = pair_set_epi16(sinpi_3_9, -sinpi_4_9); + const __m128i k__sinpi_p03_p03 = _mm_set1_epi16((int16_t)sinpi_3_9); + const __m128i kZero = _mm_set1_epi16(0); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + __m128i u[8], v[8], in7; + + array_transpose_4x4(in); + in7 = _mm_srli_si128(in[1], 8); + in7 = _mm_add_epi16(in7, in[0]); + in7 = _mm_sub_epi16(in7, in[1]); + + u[0] = _mm_unpacklo_epi16(in[0], in[1]); + u[1] = _mm_unpackhi_epi16(in[0], in[1]); + u[2] = _mm_unpacklo_epi16(in7, kZero); + u[3] = _mm_unpackhi_epi16(in[0], kZero); + + v[0] = _mm_madd_epi16(u[0], k__sinpi_p01_p04); // s0 + s3 + v[1] = _mm_madd_epi16(u[1], k__sinpi_p03_p02); // s2 + s5 + v[2] = _mm_madd_epi16(u[2], k__sinpi_p03_p03); // x2 + v[3] = _mm_madd_epi16(u[0], k__sinpi_p02_m01); // s1 - s4 + v[4] = _mm_madd_epi16(u[1], k__sinpi_p03_m04); // s2 - s6 + v[5] = _mm_madd_epi16(u[3], k__sinpi_p03_p03); // s2 + + u[0] = _mm_add_epi32(v[0], v[1]); + u[1] = _mm_add_epi32(v[3], v[4]); + u[2] = v[2]; + u[3] = _mm_add_epi32(u[0], u[1]); + u[4] = _mm_slli_epi32(v[5], 2); + u[5] = _mm_add_epi32(u[3], v[5]); + u[6] = _mm_sub_epi32(u[5], u[4]); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + + in[0] = _mm_packs_epi32(u[0], u[1]); + in[1] = _mm_packs_epi32(u[2], u[3]); +} + +// Define Macro for multiplying elements by constants and adding them together. +#define MULTIPLICATION_AND_ADD(lo_0, hi_0, lo_1, hi_1, cst0, cst1, cst2, cst3, \ + res0, res1, res2, res3) \ + { \ + tmp0 = _mm_madd_epi16(lo_0, cst0); \ + tmp1 = _mm_madd_epi16(hi_0, cst0); \ + tmp2 = _mm_madd_epi16(lo_0, cst1); \ + tmp3 = _mm_madd_epi16(hi_0, cst1); \ + tmp4 = _mm_madd_epi16(lo_1, cst2); \ + tmp5 = _mm_madd_epi16(hi_1, cst2); \ + tmp6 = _mm_madd_epi16(lo_1, cst3); \ + tmp7 = _mm_madd_epi16(hi_1, cst3); \ + \ + tmp0 = _mm_add_epi32(tmp0, rounding); \ + tmp1 = _mm_add_epi32(tmp1, rounding); \ + tmp2 = _mm_add_epi32(tmp2, rounding); \ + tmp3 = _mm_add_epi32(tmp3, rounding); \ + tmp4 = _mm_add_epi32(tmp4, rounding); \ + tmp5 = _mm_add_epi32(tmp5, rounding); \ + tmp6 = _mm_add_epi32(tmp6, rounding); \ + tmp7 = _mm_add_epi32(tmp7, rounding); \ + \ + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \ + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \ + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \ + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \ + tmp4 = _mm_srai_epi32(tmp4, DCT_CONST_BITS); \ + tmp5 = _mm_srai_epi32(tmp5, DCT_CONST_BITS); \ + tmp6 = _mm_srai_epi32(tmp6, DCT_CONST_BITS); \ + tmp7 = _mm_srai_epi32(tmp7, DCT_CONST_BITS); \ + \ + res0 = _mm_packs_epi32(tmp0, tmp1); \ + res1 = _mm_packs_epi32(tmp2, tmp3); \ + res2 = _mm_packs_epi32(tmp4, tmp5); \ + res3 = _mm_packs_epi32(tmp6, tmp7); \ + } + +#define MULTIPLICATION_AND_ADD_2(lo_0, hi_0, cst0, cst1, res0, res1) \ + { \ + tmp0 = _mm_madd_epi16(lo_0, cst0); \ + tmp1 = _mm_madd_epi16(hi_0, cst0); \ + tmp2 = _mm_madd_epi16(lo_0, cst1); \ + tmp3 = _mm_madd_epi16(hi_0, cst1); \ + \ + tmp0 = _mm_add_epi32(tmp0, rounding); \ + tmp1 = _mm_add_epi32(tmp1, rounding); \ + tmp2 = _mm_add_epi32(tmp2, rounding); \ + tmp3 = _mm_add_epi32(tmp3, rounding); \ + \ + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \ + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \ + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \ + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \ + \ + res0 = _mm_packs_epi32(tmp0, tmp1); \ + res1 = _mm_packs_epi32(tmp2, tmp3); \ + } + +#define IDCT8(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3, \ + out4, out5, out6, out7) \ + { \ + /* Stage1 */ \ + { \ + const __m128i lo_17 = _mm_unpacklo_epi16(in1, in7); \ + const __m128i hi_17 = _mm_unpackhi_epi16(in1, in7); \ + const __m128i lo_35 = _mm_unpacklo_epi16(in3, in5); \ + const __m128i hi_35 = _mm_unpackhi_epi16(in3, in5); \ + \ + MULTIPLICATION_AND_ADD(lo_17, hi_17, lo_35, hi_35, stg1_0, stg1_1, \ + stg1_2, stg1_3, stp1_4, stp1_7, stp1_5, stp1_6) \ + } \ + \ + /* Stage2 */ \ + { \ + const __m128i lo_04 = _mm_unpacklo_epi16(in0, in4); \ + const __m128i hi_04 = _mm_unpackhi_epi16(in0, in4); \ + const __m128i lo_26 = _mm_unpacklo_epi16(in2, in6); \ + const __m128i hi_26 = _mm_unpackhi_epi16(in2, in6); \ + \ + MULTIPLICATION_AND_ADD(lo_04, hi_04, lo_26, hi_26, stg2_0, stg2_1, \ + stg2_2, stg2_3, stp2_0, stp2_1, stp2_2, stp2_3) \ + \ + stp2_4 = _mm_adds_epi16(stp1_4, stp1_5); \ + stp2_5 = _mm_subs_epi16(stp1_4, stp1_5); \ + stp2_6 = _mm_subs_epi16(stp1_7, stp1_6); \ + stp2_7 = _mm_adds_epi16(stp1_7, stp1_6); \ + } \ + \ + /* Stage3 */ \ + { \ + const __m128i lo_56 = _mm_unpacklo_epi16(stp2_6, stp2_5); \ + const __m128i hi_56 = _mm_unpackhi_epi16(stp2_6, stp2_5); \ + \ + stp1_0 = _mm_adds_epi16(stp2_0, stp2_3); \ + stp1_1 = _mm_adds_epi16(stp2_1, stp2_2); \ + stp1_2 = _mm_subs_epi16(stp2_1, stp2_2); \ + stp1_3 = _mm_subs_epi16(stp2_0, stp2_3); \ + \ + tmp0 = _mm_madd_epi16(lo_56, stg2_1); \ + tmp1 = _mm_madd_epi16(hi_56, stg2_1); \ + tmp2 = _mm_madd_epi16(lo_56, stg2_0); \ + tmp3 = _mm_madd_epi16(hi_56, stg2_0); \ + \ + tmp0 = _mm_add_epi32(tmp0, rounding); \ + tmp1 = _mm_add_epi32(tmp1, rounding); \ + tmp2 = _mm_add_epi32(tmp2, rounding); \ + tmp3 = _mm_add_epi32(tmp3, rounding); \ + \ + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \ + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \ + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \ + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \ + \ + stp1_5 = _mm_packs_epi32(tmp0, tmp1); \ + stp1_6 = _mm_packs_epi32(tmp2, tmp3); \ + } \ + \ + /* Stage4 */ \ + out0 = _mm_adds_epi16(stp1_0, stp2_7); \ + out1 = _mm_adds_epi16(stp1_1, stp1_6); \ + out2 = _mm_adds_epi16(stp1_2, stp1_5); \ + out3 = _mm_adds_epi16(stp1_3, stp2_4); \ + out4 = _mm_subs_epi16(stp1_3, stp2_4); \ + out5 = _mm_subs_epi16(stp1_2, stp1_5); \ + out6 = _mm_subs_epi16(stp1_1, stp1_6); \ + out7 = _mm_subs_epi16(stp1_0, stp2_7); \ + } + +void aom_idct8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride) { + const __m128i zero = _mm_setzero_si128(); + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i final_rounding = _mm_set1_epi16(1 << 4); + const __m128i stg1_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i stg1_1 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i stg1_2 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i stg1_3 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i stg2_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg2_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg2_2 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i stg2_3 = pair_set_epi16(cospi_8_64, cospi_24_64); + + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7; + __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + int i; + + // Load input data. + in0 = load_input_data(input); + in1 = load_input_data(input + 8 * 1); + in2 = load_input_data(input + 8 * 2); + in3 = load_input_data(input + 8 * 3); + in4 = load_input_data(input + 8 * 4); + in5 = load_input_data(input + 8 * 5); + in6 = load_input_data(input + 8 * 6); + in7 = load_input_data(input + 8 * 7); + + // 2-D + for (i = 0; i < 2; i++) { + // 8x8 Transpose is copied from aom_fdct8x8_sse2() + TRANSPOSE_8X8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + + // 4-stage 1D idct8x8 + IDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, in5, + in6, in7); + } + + // Final rounding and shift + in0 = _mm_adds_epi16(in0, final_rounding); + in1 = _mm_adds_epi16(in1, final_rounding); + in2 = _mm_adds_epi16(in2, final_rounding); + in3 = _mm_adds_epi16(in3, final_rounding); + in4 = _mm_adds_epi16(in4, final_rounding); + in5 = _mm_adds_epi16(in5, final_rounding); + in6 = _mm_adds_epi16(in6, final_rounding); + in7 = _mm_adds_epi16(in7, final_rounding); + + in0 = _mm_srai_epi16(in0, 5); + in1 = _mm_srai_epi16(in1, 5); + in2 = _mm_srai_epi16(in2, 5); + in3 = _mm_srai_epi16(in3, 5); + in4 = _mm_srai_epi16(in4, 5); + in5 = _mm_srai_epi16(in5, 5); + in6 = _mm_srai_epi16(in6, 5); + in7 = _mm_srai_epi16(in7, 5); + + RECON_AND_STORE(dest + 0 * stride, in0); + RECON_AND_STORE(dest + 1 * stride, in1); + RECON_AND_STORE(dest + 2 * stride, in2); + RECON_AND_STORE(dest + 3 * stride, in3); + RECON_AND_STORE(dest + 4 * stride, in4); + RECON_AND_STORE(dest + 5 * stride, in5); + RECON_AND_STORE(dest + 6 * stride, in6); + RECON_AND_STORE(dest + 7 * stride, in7); +} + +void aom_idct8x8_1_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride) { + __m128i dc_value; + const __m128i zero = _mm_setzero_si128(); + int a; + + a = (int)dct_const_round_shift(input[0] * cospi_16_64); + a = (int)dct_const_round_shift(a * cospi_16_64); + a = ROUND_POWER_OF_TWO(a, 5); + + if (a == 0) return; + + dc_value = _mm_set1_epi16(a); + + RECON_AND_STORE(dest + 0 * stride, dc_value); + RECON_AND_STORE(dest + 1 * stride, dc_value); + RECON_AND_STORE(dest + 2 * stride, dc_value); + RECON_AND_STORE(dest + 3 * stride, dc_value); + RECON_AND_STORE(dest + 4 * stride, dc_value); + RECON_AND_STORE(dest + 5 * stride, dc_value); + RECON_AND_STORE(dest + 6 * stride, dc_value); + RECON_AND_STORE(dest + 7 * stride, dc_value); +} + +void aom_idct8_sse2(__m128i *in) { + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i stg1_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i stg1_1 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i stg1_2 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i stg1_3 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i stg2_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg2_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg2_2 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i stg2_3 = pair_set_epi16(cospi_8_64, cospi_24_64); + + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7; + __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + // 8x8 Transpose is copied from aom_fdct8x8_sse2() + TRANSPOSE_8X8(in[0], in[1], in[2], in[3], in[4], in[5], in[6], in[7], in0, + in1, in2, in3, in4, in5, in6, in7); + + // 4-stage 1D idct8x8 + IDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in[0], in[1], in[2], in[3], + in[4], in[5], in[6], in[7]); +} + +void aom_iadst8_sse2(__m128i *in) { + const __m128i k__cospi_p02_p30 = pair_set_epi16(cospi_2_64, cospi_30_64); + const __m128i k__cospi_p30_m02 = pair_set_epi16(cospi_30_64, -cospi_2_64); + const __m128i k__cospi_p10_p22 = pair_set_epi16(cospi_10_64, cospi_22_64); + const __m128i k__cospi_p22_m10 = pair_set_epi16(cospi_22_64, -cospi_10_64); + const __m128i k__cospi_p18_p14 = pair_set_epi16(cospi_18_64, cospi_14_64); + const __m128i k__cospi_p14_m18 = pair_set_epi16(cospi_14_64, -cospi_18_64); + const __m128i k__cospi_p26_p06 = pair_set_epi16(cospi_26_64, cospi_6_64); + const __m128i k__cospi_p06_m26 = pair_set_epi16(cospi_6_64, -cospi_26_64); + const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64); + const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i k__cospi_m24_p08 = pair_set_epi16(-cospi_24_64, cospi_8_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__const_0 = _mm_set1_epi16(0); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + + __m128i u0, u1, u2, u3, u4, u5, u6, u7, u8, u9, u10, u11, u12, u13, u14, u15; + __m128i v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15; + __m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; + __m128i s0, s1, s2, s3, s4, s5, s6, s7; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + + // transpose + array_transpose_8x8(in, in); + + // properly aligned for butterfly input + in0 = in[7]; + in1 = in[0]; + in2 = in[5]; + in3 = in[2]; + in4 = in[3]; + in5 = in[4]; + in6 = in[1]; + in7 = in[6]; + + // column transformation + // stage 1 + // interleave and multiply/add into 32-bit integer + s0 = _mm_unpacklo_epi16(in0, in1); + s1 = _mm_unpackhi_epi16(in0, in1); + s2 = _mm_unpacklo_epi16(in2, in3); + s3 = _mm_unpackhi_epi16(in2, in3); + s4 = _mm_unpacklo_epi16(in4, in5); + s5 = _mm_unpackhi_epi16(in4, in5); + s6 = _mm_unpacklo_epi16(in6, in7); + s7 = _mm_unpackhi_epi16(in6, in7); + + u0 = _mm_madd_epi16(s0, k__cospi_p02_p30); + u1 = _mm_madd_epi16(s1, k__cospi_p02_p30); + u2 = _mm_madd_epi16(s0, k__cospi_p30_m02); + u3 = _mm_madd_epi16(s1, k__cospi_p30_m02); + u4 = _mm_madd_epi16(s2, k__cospi_p10_p22); + u5 = _mm_madd_epi16(s3, k__cospi_p10_p22); + u6 = _mm_madd_epi16(s2, k__cospi_p22_m10); + u7 = _mm_madd_epi16(s3, k__cospi_p22_m10); + u8 = _mm_madd_epi16(s4, k__cospi_p18_p14); + u9 = _mm_madd_epi16(s5, k__cospi_p18_p14); + u10 = _mm_madd_epi16(s4, k__cospi_p14_m18); + u11 = _mm_madd_epi16(s5, k__cospi_p14_m18); + u12 = _mm_madd_epi16(s6, k__cospi_p26_p06); + u13 = _mm_madd_epi16(s7, k__cospi_p26_p06); + u14 = _mm_madd_epi16(s6, k__cospi_p06_m26); + u15 = _mm_madd_epi16(s7, k__cospi_p06_m26); + + // addition + w0 = _mm_add_epi32(u0, u8); + w1 = _mm_add_epi32(u1, u9); + w2 = _mm_add_epi32(u2, u10); + w3 = _mm_add_epi32(u3, u11); + w4 = _mm_add_epi32(u4, u12); + w5 = _mm_add_epi32(u5, u13); + w6 = _mm_add_epi32(u6, u14); + w7 = _mm_add_epi32(u7, u15); + w8 = _mm_sub_epi32(u0, u8); + w9 = _mm_sub_epi32(u1, u9); + w10 = _mm_sub_epi32(u2, u10); + w11 = _mm_sub_epi32(u3, u11); + w12 = _mm_sub_epi32(u4, u12); + w13 = _mm_sub_epi32(u5, u13); + w14 = _mm_sub_epi32(u6, u14); + w15 = _mm_sub_epi32(u7, u15); + + // shift and rounding + v0 = _mm_add_epi32(w0, k__DCT_CONST_ROUNDING); + v1 = _mm_add_epi32(w1, k__DCT_CONST_ROUNDING); + v2 = _mm_add_epi32(w2, k__DCT_CONST_ROUNDING); + v3 = _mm_add_epi32(w3, k__DCT_CONST_ROUNDING); + v4 = _mm_add_epi32(w4, k__DCT_CONST_ROUNDING); + v5 = _mm_add_epi32(w5, k__DCT_CONST_ROUNDING); + v6 = _mm_add_epi32(w6, k__DCT_CONST_ROUNDING); + v7 = _mm_add_epi32(w7, k__DCT_CONST_ROUNDING); + v8 = _mm_add_epi32(w8, k__DCT_CONST_ROUNDING); + v9 = _mm_add_epi32(w9, k__DCT_CONST_ROUNDING); + v10 = _mm_add_epi32(w10, k__DCT_CONST_ROUNDING); + v11 = _mm_add_epi32(w11, k__DCT_CONST_ROUNDING); + v12 = _mm_add_epi32(w12, k__DCT_CONST_ROUNDING); + v13 = _mm_add_epi32(w13, k__DCT_CONST_ROUNDING); + v14 = _mm_add_epi32(w14, k__DCT_CONST_ROUNDING); + v15 = _mm_add_epi32(w15, k__DCT_CONST_ROUNDING); + + u0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + u1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + u2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + u3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + u4 = _mm_srai_epi32(v4, DCT_CONST_BITS); + u5 = _mm_srai_epi32(v5, DCT_CONST_BITS); + u6 = _mm_srai_epi32(v6, DCT_CONST_BITS); + u7 = _mm_srai_epi32(v7, DCT_CONST_BITS); + u8 = _mm_srai_epi32(v8, DCT_CONST_BITS); + u9 = _mm_srai_epi32(v9, DCT_CONST_BITS); + u10 = _mm_srai_epi32(v10, DCT_CONST_BITS); + u11 = _mm_srai_epi32(v11, DCT_CONST_BITS); + u12 = _mm_srai_epi32(v12, DCT_CONST_BITS); + u13 = _mm_srai_epi32(v13, DCT_CONST_BITS); + u14 = _mm_srai_epi32(v14, DCT_CONST_BITS); + u15 = _mm_srai_epi32(v15, DCT_CONST_BITS); + + // back to 16-bit and pack 8 integers into __m128i + in[0] = _mm_packs_epi32(u0, u1); + in[1] = _mm_packs_epi32(u2, u3); + in[2] = _mm_packs_epi32(u4, u5); + in[3] = _mm_packs_epi32(u6, u7); + in[4] = _mm_packs_epi32(u8, u9); + in[5] = _mm_packs_epi32(u10, u11); + in[6] = _mm_packs_epi32(u12, u13); + in[7] = _mm_packs_epi32(u14, u15); + + // stage 2 + s0 = _mm_add_epi16(in[0], in[2]); + s1 = _mm_add_epi16(in[1], in[3]); + s2 = _mm_sub_epi16(in[0], in[2]); + s3 = _mm_sub_epi16(in[1], in[3]); + u0 = _mm_unpacklo_epi16(in[4], in[5]); + u1 = _mm_unpackhi_epi16(in[4], in[5]); + u2 = _mm_unpacklo_epi16(in[6], in[7]); + u3 = _mm_unpackhi_epi16(in[6], in[7]); + + v0 = _mm_madd_epi16(u0, k__cospi_p08_p24); + v1 = _mm_madd_epi16(u1, k__cospi_p08_p24); + v2 = _mm_madd_epi16(u0, k__cospi_p24_m08); + v3 = _mm_madd_epi16(u1, k__cospi_p24_m08); + v4 = _mm_madd_epi16(u2, k__cospi_m24_p08); + v5 = _mm_madd_epi16(u3, k__cospi_m24_p08); + v6 = _mm_madd_epi16(u2, k__cospi_p08_p24); + v7 = _mm_madd_epi16(u3, k__cospi_p08_p24); + + w0 = _mm_add_epi32(v0, v4); + w1 = _mm_add_epi32(v1, v5); + w2 = _mm_add_epi32(v2, v6); + w3 = _mm_add_epi32(v3, v7); + w4 = _mm_sub_epi32(v0, v4); + w5 = _mm_sub_epi32(v1, v5); + w6 = _mm_sub_epi32(v2, v6); + w7 = _mm_sub_epi32(v3, v7); + + v0 = _mm_add_epi32(w0, k__DCT_CONST_ROUNDING); + v1 = _mm_add_epi32(w1, k__DCT_CONST_ROUNDING); + v2 = _mm_add_epi32(w2, k__DCT_CONST_ROUNDING); + v3 = _mm_add_epi32(w3, k__DCT_CONST_ROUNDING); + v4 = _mm_add_epi32(w4, k__DCT_CONST_ROUNDING); + v5 = _mm_add_epi32(w5, k__DCT_CONST_ROUNDING); + v6 = _mm_add_epi32(w6, k__DCT_CONST_ROUNDING); + v7 = _mm_add_epi32(w7, k__DCT_CONST_ROUNDING); + + u0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + u1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + u2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + u3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + u4 = _mm_srai_epi32(v4, DCT_CONST_BITS); + u5 = _mm_srai_epi32(v5, DCT_CONST_BITS); + u6 = _mm_srai_epi32(v6, DCT_CONST_BITS); + u7 = _mm_srai_epi32(v7, DCT_CONST_BITS); + + // back to 16-bit intergers + s4 = _mm_packs_epi32(u0, u1); + s5 = _mm_packs_epi32(u2, u3); + s6 = _mm_packs_epi32(u4, u5); + s7 = _mm_packs_epi32(u6, u7); + + // stage 3 + u0 = _mm_unpacklo_epi16(s2, s3); + u1 = _mm_unpackhi_epi16(s2, s3); + u2 = _mm_unpacklo_epi16(s6, s7); + u3 = _mm_unpackhi_epi16(s6, s7); + + v0 = _mm_madd_epi16(u0, k__cospi_p16_p16); + v1 = _mm_madd_epi16(u1, k__cospi_p16_p16); + v2 = _mm_madd_epi16(u0, k__cospi_p16_m16); + v3 = _mm_madd_epi16(u1, k__cospi_p16_m16); + v4 = _mm_madd_epi16(u2, k__cospi_p16_p16); + v5 = _mm_madd_epi16(u3, k__cospi_p16_p16); + v6 = _mm_madd_epi16(u2, k__cospi_p16_m16); + v7 = _mm_madd_epi16(u3, k__cospi_p16_m16); + + u0 = _mm_add_epi32(v0, k__DCT_CONST_ROUNDING); + u1 = _mm_add_epi32(v1, k__DCT_CONST_ROUNDING); + u2 = _mm_add_epi32(v2, k__DCT_CONST_ROUNDING); + u3 = _mm_add_epi32(v3, k__DCT_CONST_ROUNDING); + u4 = _mm_add_epi32(v4, k__DCT_CONST_ROUNDING); + u5 = _mm_add_epi32(v5, k__DCT_CONST_ROUNDING); + u6 = _mm_add_epi32(v6, k__DCT_CONST_ROUNDING); + u7 = _mm_add_epi32(v7, k__DCT_CONST_ROUNDING); + + v0 = _mm_srai_epi32(u0, DCT_CONST_BITS); + v1 = _mm_srai_epi32(u1, DCT_CONST_BITS); + v2 = _mm_srai_epi32(u2, DCT_CONST_BITS); + v3 = _mm_srai_epi32(u3, DCT_CONST_BITS); + v4 = _mm_srai_epi32(u4, DCT_CONST_BITS); + v5 = _mm_srai_epi32(u5, DCT_CONST_BITS); + v6 = _mm_srai_epi32(u6, DCT_CONST_BITS); + v7 = _mm_srai_epi32(u7, DCT_CONST_BITS); + + s2 = _mm_packs_epi32(v0, v1); + s3 = _mm_packs_epi32(v2, v3); + s6 = _mm_packs_epi32(v4, v5); + s7 = _mm_packs_epi32(v6, v7); + + in[0] = s0; + in[1] = _mm_sub_epi16(k__const_0, s4); + in[2] = s6; + in[3] = _mm_sub_epi16(k__const_0, s2); + in[4] = s3; + in[5] = _mm_sub_epi16(k__const_0, s7); + in[6] = s5; + in[7] = _mm_sub_epi16(k__const_0, s1); +} + +void aom_idct8x8_12_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride) { + const __m128i zero = _mm_setzero_si128(); + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i final_rounding = _mm_set1_epi16(1 << 4); + const __m128i stg1_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i stg1_1 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i stg1_2 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i stg1_3 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i stg2_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg2_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg2_2 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i stg2_3 = pair_set_epi16(cospi_8_64, cospi_24_64); + const __m128i stg3_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7; + __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + // Rows. Load 4-row input data. + in0 = load_input_data(input); + in1 = load_input_data(input + 8 * 1); + in2 = load_input_data(input + 8 * 2); + in3 = load_input_data(input + 8 * 3); + + // 8x4 Transpose + TRANSPOSE_8X8_10(in0, in1, in2, in3, in0, in1); + // Stage1 + { + const __m128i lo_17 = _mm_unpackhi_epi16(in0, zero); + const __m128i lo_35 = _mm_unpackhi_epi16(in1, zero); + + tmp0 = _mm_madd_epi16(lo_17, stg1_0); + tmp2 = _mm_madd_epi16(lo_17, stg1_1); + tmp4 = _mm_madd_epi16(lo_35, stg1_2); + tmp6 = _mm_madd_epi16(lo_35, stg1_3); + + tmp0 = _mm_add_epi32(tmp0, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp4 = _mm_add_epi32(tmp4, rounding); + tmp6 = _mm_add_epi32(tmp6, rounding); + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + tmp4 = _mm_srai_epi32(tmp4, DCT_CONST_BITS); + tmp6 = _mm_srai_epi32(tmp6, DCT_CONST_BITS); + + stp1_4 = _mm_packs_epi32(tmp0, tmp2); + stp1_5 = _mm_packs_epi32(tmp4, tmp6); + } + + // Stage2 + { + const __m128i lo_04 = _mm_unpacklo_epi16(in0, zero); + const __m128i lo_26 = _mm_unpacklo_epi16(in1, zero); + + tmp0 = _mm_madd_epi16(lo_04, stg2_0); + tmp2 = _mm_madd_epi16(lo_04, stg2_1); + tmp4 = _mm_madd_epi16(lo_26, stg2_2); + tmp6 = _mm_madd_epi16(lo_26, stg2_3); + + tmp0 = _mm_add_epi32(tmp0, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp4 = _mm_add_epi32(tmp4, rounding); + tmp6 = _mm_add_epi32(tmp6, rounding); + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + tmp4 = _mm_srai_epi32(tmp4, DCT_CONST_BITS); + tmp6 = _mm_srai_epi32(tmp6, DCT_CONST_BITS); + + stp2_0 = _mm_packs_epi32(tmp0, tmp2); + stp2_2 = _mm_packs_epi32(tmp6, tmp4); + + tmp0 = _mm_adds_epi16(stp1_4, stp1_5); + tmp1 = _mm_subs_epi16(stp1_4, stp1_5); + + stp2_4 = tmp0; + stp2_5 = _mm_unpacklo_epi64(tmp1, zero); + stp2_6 = _mm_unpackhi_epi64(tmp1, zero); + } + + // Stage3 + { + const __m128i lo_56 = _mm_unpacklo_epi16(stp2_5, stp2_6); + + tmp4 = _mm_adds_epi16(stp2_0, stp2_2); + tmp6 = _mm_subs_epi16(stp2_0, stp2_2); + + stp1_2 = _mm_unpackhi_epi64(tmp6, tmp4); + stp1_3 = _mm_unpacklo_epi64(tmp6, tmp4); + + tmp0 = _mm_madd_epi16(lo_56, stg3_0); + tmp2 = _mm_madd_epi16(lo_56, stg2_0); // stg3_1 = stg2_0 + + tmp0 = _mm_add_epi32(tmp0, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + + stp1_5 = _mm_packs_epi32(tmp0, tmp2); + } + + // Stage4 + tmp0 = _mm_adds_epi16(stp1_3, stp2_4); + tmp1 = _mm_adds_epi16(stp1_2, stp1_5); + tmp2 = _mm_subs_epi16(stp1_3, stp2_4); + tmp3 = _mm_subs_epi16(stp1_2, stp1_5); + + TRANSPOSE_4X8_10(tmp0, tmp1, tmp2, tmp3, in0, in1, in2, in3) + + IDCT8(in0, in1, in2, in3, zero, zero, zero, zero, in0, in1, in2, in3, in4, + in5, in6, in7); + // Final rounding and shift + in0 = _mm_adds_epi16(in0, final_rounding); + in1 = _mm_adds_epi16(in1, final_rounding); + in2 = _mm_adds_epi16(in2, final_rounding); + in3 = _mm_adds_epi16(in3, final_rounding); + in4 = _mm_adds_epi16(in4, final_rounding); + in5 = _mm_adds_epi16(in5, final_rounding); + in6 = _mm_adds_epi16(in6, final_rounding); + in7 = _mm_adds_epi16(in7, final_rounding); + + in0 = _mm_srai_epi16(in0, 5); + in1 = _mm_srai_epi16(in1, 5); + in2 = _mm_srai_epi16(in2, 5); + in3 = _mm_srai_epi16(in3, 5); + in4 = _mm_srai_epi16(in4, 5); + in5 = _mm_srai_epi16(in5, 5); + in6 = _mm_srai_epi16(in6, 5); + in7 = _mm_srai_epi16(in7, 5); + + RECON_AND_STORE(dest + 0 * stride, in0); + RECON_AND_STORE(dest + 1 * stride, in1); + RECON_AND_STORE(dest + 2 * stride, in2); + RECON_AND_STORE(dest + 3 * stride, in3); + RECON_AND_STORE(dest + 4 * stride, in4); + RECON_AND_STORE(dest + 5 * stride, in5); + RECON_AND_STORE(dest + 6 * stride, in6); + RECON_AND_STORE(dest + 7 * stride, in7); +} + +#define IDCT16 \ + /* Stage2 */ \ + { \ + const __m128i lo_1_15 = _mm_unpacklo_epi16(in[1], in[15]); \ + const __m128i hi_1_15 = _mm_unpackhi_epi16(in[1], in[15]); \ + const __m128i lo_9_7 = _mm_unpacklo_epi16(in[9], in[7]); \ + const __m128i hi_9_7 = _mm_unpackhi_epi16(in[9], in[7]); \ + const __m128i lo_5_11 = _mm_unpacklo_epi16(in[5], in[11]); \ + const __m128i hi_5_11 = _mm_unpackhi_epi16(in[5], in[11]); \ + const __m128i lo_13_3 = _mm_unpacklo_epi16(in[13], in[3]); \ + const __m128i hi_13_3 = _mm_unpackhi_epi16(in[13], in[3]); \ + \ + MULTIPLICATION_AND_ADD(lo_1_15, hi_1_15, lo_9_7, hi_9_7, stg2_0, stg2_1, \ + stg2_2, stg2_3, stp2_8, stp2_15, stp2_9, stp2_14) \ + \ + MULTIPLICATION_AND_ADD(lo_5_11, hi_5_11, lo_13_3, hi_13_3, stg2_4, stg2_5, \ + stg2_6, stg2_7, stp2_10, stp2_13, stp2_11, stp2_12) \ + } \ + \ + /* Stage3 */ \ + { \ + const __m128i lo_2_14 = _mm_unpacklo_epi16(in[2], in[14]); \ + const __m128i hi_2_14 = _mm_unpackhi_epi16(in[2], in[14]); \ + const __m128i lo_10_6 = _mm_unpacklo_epi16(in[10], in[6]); \ + const __m128i hi_10_6 = _mm_unpackhi_epi16(in[10], in[6]); \ + \ + MULTIPLICATION_AND_ADD(lo_2_14, hi_2_14, lo_10_6, hi_10_6, stg3_0, stg3_1, \ + stg3_2, stg3_3, stp1_4, stp1_7, stp1_5, stp1_6) \ + \ + stp1_8_0 = _mm_add_epi16(stp2_8, stp2_9); \ + stp1_9 = _mm_sub_epi16(stp2_8, stp2_9); \ + stp1_10 = _mm_sub_epi16(stp2_11, stp2_10); \ + stp1_11 = _mm_add_epi16(stp2_11, stp2_10); \ + \ + stp1_12_0 = _mm_add_epi16(stp2_12, stp2_13); \ + stp1_13 = _mm_sub_epi16(stp2_12, stp2_13); \ + stp1_14 = _mm_sub_epi16(stp2_15, stp2_14); \ + stp1_15 = _mm_add_epi16(stp2_15, stp2_14); \ + } \ + \ + /* Stage4 */ \ + { \ + const __m128i lo_0_8 = _mm_unpacklo_epi16(in[0], in[8]); \ + const __m128i hi_0_8 = _mm_unpackhi_epi16(in[0], in[8]); \ + const __m128i lo_4_12 = _mm_unpacklo_epi16(in[4], in[12]); \ + const __m128i hi_4_12 = _mm_unpackhi_epi16(in[4], in[12]); \ + \ + const __m128i lo_9_14 = _mm_unpacklo_epi16(stp1_9, stp1_14); \ + const __m128i hi_9_14 = _mm_unpackhi_epi16(stp1_9, stp1_14); \ + const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); \ + const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); \ + \ + MULTIPLICATION_AND_ADD(lo_0_8, hi_0_8, lo_4_12, hi_4_12, stg4_0, stg4_1, \ + stg4_2, stg4_3, stp2_0, stp2_1, stp2_2, stp2_3) \ + \ + stp2_4 = _mm_add_epi16(stp1_4, stp1_5); \ + stp2_5 = _mm_sub_epi16(stp1_4, stp1_5); \ + stp2_6 = _mm_sub_epi16(stp1_7, stp1_6); \ + stp2_7 = _mm_add_epi16(stp1_7, stp1_6); \ + \ + MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, \ + stg4_5, stg4_6, stg4_7, stp2_9, stp2_14, stp2_10, \ + stp2_13) \ + } \ + \ + /* Stage5 */ \ + { \ + const __m128i lo_6_5 = _mm_unpacklo_epi16(stp2_6, stp2_5); \ + const __m128i hi_6_5 = _mm_unpackhi_epi16(stp2_6, stp2_5); \ + \ + stp1_0 = _mm_add_epi16(stp2_0, stp2_3); \ + stp1_1 = _mm_add_epi16(stp2_1, stp2_2); \ + stp1_2 = _mm_sub_epi16(stp2_1, stp2_2); \ + stp1_3 = _mm_sub_epi16(stp2_0, stp2_3); \ + \ + tmp0 = _mm_madd_epi16(lo_6_5, stg4_1); \ + tmp1 = _mm_madd_epi16(hi_6_5, stg4_1); \ + tmp2 = _mm_madd_epi16(lo_6_5, stg4_0); \ + tmp3 = _mm_madd_epi16(hi_6_5, stg4_0); \ + \ + tmp0 = _mm_add_epi32(tmp0, rounding); \ + tmp1 = _mm_add_epi32(tmp1, rounding); \ + tmp2 = _mm_add_epi32(tmp2, rounding); \ + tmp3 = _mm_add_epi32(tmp3, rounding); \ + \ + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \ + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \ + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \ + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \ + \ + stp1_5 = _mm_packs_epi32(tmp0, tmp1); \ + stp1_6 = _mm_packs_epi32(tmp2, tmp3); \ + \ + stp1_8 = _mm_add_epi16(stp1_8_0, stp1_11); \ + stp1_9 = _mm_add_epi16(stp2_9, stp2_10); \ + stp1_10 = _mm_sub_epi16(stp2_9, stp2_10); \ + stp1_11 = _mm_sub_epi16(stp1_8_0, stp1_11); \ + \ + stp1_12 = _mm_sub_epi16(stp1_15, stp1_12_0); \ + stp1_13 = _mm_sub_epi16(stp2_14, stp2_13); \ + stp1_14 = _mm_add_epi16(stp2_14, stp2_13); \ + stp1_15 = _mm_add_epi16(stp1_15, stp1_12_0); \ + } \ + \ + /* Stage6 */ \ + { \ + const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); \ + const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); \ + const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1_11, stp1_12); \ + const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1_11, stp1_12); \ + \ + stp2_0 = _mm_add_epi16(stp1_0, stp2_7); \ + stp2_1 = _mm_add_epi16(stp1_1, stp1_6); \ + stp2_2 = _mm_add_epi16(stp1_2, stp1_5); \ + stp2_3 = _mm_add_epi16(stp1_3, stp2_4); \ + stp2_4 = _mm_sub_epi16(stp1_3, stp2_4); \ + stp2_5 = _mm_sub_epi16(stp1_2, stp1_5); \ + stp2_6 = _mm_sub_epi16(stp1_1, stp1_6); \ + stp2_7 = _mm_sub_epi16(stp1_0, stp2_7); \ + \ + MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0, \ + stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11, \ + stp2_12) \ + } + +#define IDCT16_10 \ + /* Stage2 */ \ + { \ + const __m128i lo_1_15 = _mm_unpacklo_epi16(in[1], zero); \ + const __m128i hi_1_15 = _mm_unpackhi_epi16(in[1], zero); \ + const __m128i lo_13_3 = _mm_unpacklo_epi16(zero, in[3]); \ + const __m128i hi_13_3 = _mm_unpackhi_epi16(zero, in[3]); \ + \ + MULTIPLICATION_AND_ADD(lo_1_15, hi_1_15, lo_13_3, hi_13_3, stg2_0, stg2_1, \ + stg2_6, stg2_7, stp1_8_0, stp1_15, stp1_11, \ + stp1_12_0) \ + } \ + \ + /* Stage3 */ \ + { \ + const __m128i lo_2_14 = _mm_unpacklo_epi16(in[2], zero); \ + const __m128i hi_2_14 = _mm_unpackhi_epi16(in[2], zero); \ + \ + MULTIPLICATION_AND_ADD_2(lo_2_14, hi_2_14, stg3_0, stg3_1, stp2_4, stp2_7) \ + \ + stp1_9 = stp1_8_0; \ + stp1_10 = stp1_11; \ + \ + stp1_13 = stp1_12_0; \ + stp1_14 = stp1_15; \ + } \ + \ + /* Stage4 */ \ + { \ + const __m128i lo_0_8 = _mm_unpacklo_epi16(in[0], zero); \ + const __m128i hi_0_8 = _mm_unpackhi_epi16(in[0], zero); \ + \ + const __m128i lo_9_14 = _mm_unpacklo_epi16(stp1_9, stp1_14); \ + const __m128i hi_9_14 = _mm_unpackhi_epi16(stp1_9, stp1_14); \ + const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); \ + const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); \ + \ + MULTIPLICATION_AND_ADD_2(lo_0_8, hi_0_8, stg4_0, stg4_1, stp1_0, stp1_1) \ + stp2_5 = stp2_4; \ + stp2_6 = stp2_7; \ + \ + MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, \ + stg4_5, stg4_6, stg4_7, stp2_9, stp2_14, stp2_10, \ + stp2_13) \ + } \ + \ + /* Stage5 */ \ + { \ + const __m128i lo_6_5 = _mm_unpacklo_epi16(stp2_6, stp2_5); \ + const __m128i hi_6_5 = _mm_unpackhi_epi16(stp2_6, stp2_5); \ + \ + stp1_2 = stp1_1; \ + stp1_3 = stp1_0; \ + \ + tmp0 = _mm_madd_epi16(lo_6_5, stg4_1); \ + tmp1 = _mm_madd_epi16(hi_6_5, stg4_1); \ + tmp2 = _mm_madd_epi16(lo_6_5, stg4_0); \ + tmp3 = _mm_madd_epi16(hi_6_5, stg4_0); \ + \ + tmp0 = _mm_add_epi32(tmp0, rounding); \ + tmp1 = _mm_add_epi32(tmp1, rounding); \ + tmp2 = _mm_add_epi32(tmp2, rounding); \ + tmp3 = _mm_add_epi32(tmp3, rounding); \ + \ + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \ + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \ + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \ + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \ + \ + stp1_5 = _mm_packs_epi32(tmp0, tmp1); \ + stp1_6 = _mm_packs_epi32(tmp2, tmp3); \ + \ + stp1_8 = _mm_add_epi16(stp1_8_0, stp1_11); \ + stp1_9 = _mm_add_epi16(stp2_9, stp2_10); \ + stp1_10 = _mm_sub_epi16(stp2_9, stp2_10); \ + stp1_11 = _mm_sub_epi16(stp1_8_0, stp1_11); \ + \ + stp1_12 = _mm_sub_epi16(stp1_15, stp1_12_0); \ + stp1_13 = _mm_sub_epi16(stp2_14, stp2_13); \ + stp1_14 = _mm_add_epi16(stp2_14, stp2_13); \ + stp1_15 = _mm_add_epi16(stp1_15, stp1_12_0); \ + } \ + \ + /* Stage6 */ \ + { \ + const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); \ + const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); \ + const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1_11, stp1_12); \ + const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1_11, stp1_12); \ + \ + stp2_0 = _mm_add_epi16(stp1_0, stp2_7); \ + stp2_1 = _mm_add_epi16(stp1_1, stp1_6); \ + stp2_2 = _mm_add_epi16(stp1_2, stp1_5); \ + stp2_3 = _mm_add_epi16(stp1_3, stp2_4); \ + stp2_4 = _mm_sub_epi16(stp1_3, stp2_4); \ + stp2_5 = _mm_sub_epi16(stp1_2, stp1_5); \ + stp2_6 = _mm_sub_epi16(stp1_1, stp1_6); \ + stp2_7 = _mm_sub_epi16(stp1_0, stp2_7); \ + \ + MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0, \ + stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11, \ + stp2_12) \ + } + +void aom_idct16x16_256_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride) { + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i final_rounding = _mm_set1_epi16(1 << 5); + const __m128i zero = _mm_setzero_si128(); + + const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64); + const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64); + const __m128i stg2_2 = pair_set_epi16(cospi_14_64, -cospi_18_64); + const __m128i stg2_3 = pair_set_epi16(cospi_18_64, cospi_14_64); + const __m128i stg2_4 = pair_set_epi16(cospi_22_64, -cospi_10_64); + const __m128i stg2_5 = pair_set_epi16(cospi_10_64, cospi_22_64); + const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64); + const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64); + + const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i stg3_2 = pair_set_epi16(cospi_12_64, -cospi_20_64); + const __m128i stg3_3 = pair_set_epi16(cospi_20_64, cospi_12_64); + + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg4_2 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i stg4_3 = pair_set_epi16(cospi_8_64, cospi_24_64); + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + const __m128i stg4_7 = pair_set_epi16(-cospi_8_64, cospi_24_64); + + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + + __m128i in[16], l[16], r[16], *curr1; + __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7, + stp1_8, stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15, + stp1_8_0, stp1_12_0; + __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7, + stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + int i; + + curr1 = l; + for (i = 0; i < 2; i++) { + // 1-D idct + + // Load input data. + in[0] = load_input_data(input); + in[8] = load_input_data(input + 8 * 1); + in[1] = load_input_data(input + 8 * 2); + in[9] = load_input_data(input + 8 * 3); + in[2] = load_input_data(input + 8 * 4); + in[10] = load_input_data(input + 8 * 5); + in[3] = load_input_data(input + 8 * 6); + in[11] = load_input_data(input + 8 * 7); + in[4] = load_input_data(input + 8 * 8); + in[12] = load_input_data(input + 8 * 9); + in[5] = load_input_data(input + 8 * 10); + in[13] = load_input_data(input + 8 * 11); + in[6] = load_input_data(input + 8 * 12); + in[14] = load_input_data(input + 8 * 13); + in[7] = load_input_data(input + 8 * 14); + in[15] = load_input_data(input + 8 * 15); + + array_transpose_8x8(in, in); + array_transpose_8x8(in + 8, in + 8); + + IDCT16 + + // Stage7 + curr1[0] = _mm_add_epi16(stp2_0, stp1_15); + curr1[1] = _mm_add_epi16(stp2_1, stp1_14); + curr1[2] = _mm_add_epi16(stp2_2, stp2_13); + curr1[3] = _mm_add_epi16(stp2_3, stp2_12); + curr1[4] = _mm_add_epi16(stp2_4, stp2_11); + curr1[5] = _mm_add_epi16(stp2_5, stp2_10); + curr1[6] = _mm_add_epi16(stp2_6, stp1_9); + curr1[7] = _mm_add_epi16(stp2_7, stp1_8); + curr1[8] = _mm_sub_epi16(stp2_7, stp1_8); + curr1[9] = _mm_sub_epi16(stp2_6, stp1_9); + curr1[10] = _mm_sub_epi16(stp2_5, stp2_10); + curr1[11] = _mm_sub_epi16(stp2_4, stp2_11); + curr1[12] = _mm_sub_epi16(stp2_3, stp2_12); + curr1[13] = _mm_sub_epi16(stp2_2, stp2_13); + curr1[14] = _mm_sub_epi16(stp2_1, stp1_14); + curr1[15] = _mm_sub_epi16(stp2_0, stp1_15); + + curr1 = r; + input += 128; + } + for (i = 0; i < 2; i++) { + int j; + // 1-D idct + array_transpose_8x8(l + i * 8, in); + array_transpose_8x8(r + i * 8, in + 8); + + IDCT16 + + // 2-D + in[0] = _mm_add_epi16(stp2_0, stp1_15); + in[1] = _mm_add_epi16(stp2_1, stp1_14); + in[2] = _mm_add_epi16(stp2_2, stp2_13); + in[3] = _mm_add_epi16(stp2_3, stp2_12); + in[4] = _mm_add_epi16(stp2_4, stp2_11); + in[5] = _mm_add_epi16(stp2_5, stp2_10); + in[6] = _mm_add_epi16(stp2_6, stp1_9); + in[7] = _mm_add_epi16(stp2_7, stp1_8); + in[8] = _mm_sub_epi16(stp2_7, stp1_8); + in[9] = _mm_sub_epi16(stp2_6, stp1_9); + in[10] = _mm_sub_epi16(stp2_5, stp2_10); + in[11] = _mm_sub_epi16(stp2_4, stp2_11); + in[12] = _mm_sub_epi16(stp2_3, stp2_12); + in[13] = _mm_sub_epi16(stp2_2, stp2_13); + in[14] = _mm_sub_epi16(stp2_1, stp1_14); + in[15] = _mm_sub_epi16(stp2_0, stp1_15); + + for (j = 0; j < 16; ++j) { + // Final rounding and shift + in[j] = _mm_adds_epi16(in[j], final_rounding); + in[j] = _mm_srai_epi16(in[j], 6); + RECON_AND_STORE(dest + j * stride, in[j]); + } + + dest += 8; + } +} + +void aom_idct16x16_1_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride) { + __m128i dc_value; + const __m128i zero = _mm_setzero_si128(); + int a, i; + + a = (int)dct_const_round_shift(input[0] * cospi_16_64); + a = (int)dct_const_round_shift(a * cospi_16_64); + a = ROUND_POWER_OF_TWO(a, 6); + + if (a == 0) return; + + dc_value = _mm_set1_epi16(a); + + for (i = 0; i < 16; ++i) { + RECON_AND_STORE(dest + 0, dc_value); + RECON_AND_STORE(dest + 8, dc_value); + dest += stride; + } +} + +void iadst16_8col(__m128i *in) { + // perform 16x16 1-D ADST for 8 columns + __m128i s[16], x[16], u[32], v[32]; + const __m128i k__cospi_p01_p31 = pair_set_epi16(cospi_1_64, cospi_31_64); + const __m128i k__cospi_p31_m01 = pair_set_epi16(cospi_31_64, -cospi_1_64); + const __m128i k__cospi_p05_p27 = pair_set_epi16(cospi_5_64, cospi_27_64); + const __m128i k__cospi_p27_m05 = pair_set_epi16(cospi_27_64, -cospi_5_64); + const __m128i k__cospi_p09_p23 = pair_set_epi16(cospi_9_64, cospi_23_64); + const __m128i k__cospi_p23_m09 = pair_set_epi16(cospi_23_64, -cospi_9_64); + const __m128i k__cospi_p13_p19 = pair_set_epi16(cospi_13_64, cospi_19_64); + const __m128i k__cospi_p19_m13 = pair_set_epi16(cospi_19_64, -cospi_13_64); + const __m128i k__cospi_p17_p15 = pair_set_epi16(cospi_17_64, cospi_15_64); + const __m128i k__cospi_p15_m17 = pair_set_epi16(cospi_15_64, -cospi_17_64); + const __m128i k__cospi_p21_p11 = pair_set_epi16(cospi_21_64, cospi_11_64); + const __m128i k__cospi_p11_m21 = pair_set_epi16(cospi_11_64, -cospi_21_64); + const __m128i k__cospi_p25_p07 = pair_set_epi16(cospi_25_64, cospi_7_64); + const __m128i k__cospi_p07_m25 = pair_set_epi16(cospi_7_64, -cospi_25_64); + const __m128i k__cospi_p29_p03 = pair_set_epi16(cospi_29_64, cospi_3_64); + const __m128i k__cospi_p03_m29 = pair_set_epi16(cospi_3_64, -cospi_29_64); + const __m128i k__cospi_p04_p28 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i k__cospi_p28_m04 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i k__cospi_p20_p12 = pair_set_epi16(cospi_20_64, cospi_12_64); + const __m128i k__cospi_p12_m20 = pair_set_epi16(cospi_12_64, -cospi_20_64); + const __m128i k__cospi_m28_p04 = pair_set_epi16(-cospi_28_64, cospi_4_64); + const __m128i k__cospi_m12_p20 = pair_set_epi16(-cospi_12_64, cospi_20_64); + const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64); + const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i k__cospi_m24_p08 = pair_set_epi16(-cospi_24_64, cospi_8_64); + const __m128i k__cospi_m16_m16 = _mm_set1_epi16((int16_t)-cospi_16_64); + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i kZero = _mm_set1_epi16(0); + + u[0] = _mm_unpacklo_epi16(in[15], in[0]); + u[1] = _mm_unpackhi_epi16(in[15], in[0]); + u[2] = _mm_unpacklo_epi16(in[13], in[2]); + u[3] = _mm_unpackhi_epi16(in[13], in[2]); + u[4] = _mm_unpacklo_epi16(in[11], in[4]); + u[5] = _mm_unpackhi_epi16(in[11], in[4]); + u[6] = _mm_unpacklo_epi16(in[9], in[6]); + u[7] = _mm_unpackhi_epi16(in[9], in[6]); + u[8] = _mm_unpacklo_epi16(in[7], in[8]); + u[9] = _mm_unpackhi_epi16(in[7], in[8]); + u[10] = _mm_unpacklo_epi16(in[5], in[10]); + u[11] = _mm_unpackhi_epi16(in[5], in[10]); + u[12] = _mm_unpacklo_epi16(in[3], in[12]); + u[13] = _mm_unpackhi_epi16(in[3], in[12]); + u[14] = _mm_unpacklo_epi16(in[1], in[14]); + u[15] = _mm_unpackhi_epi16(in[1], in[14]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_p01_p31); + v[1] = _mm_madd_epi16(u[1], k__cospi_p01_p31); + v[2] = _mm_madd_epi16(u[0], k__cospi_p31_m01); + v[3] = _mm_madd_epi16(u[1], k__cospi_p31_m01); + v[4] = _mm_madd_epi16(u[2], k__cospi_p05_p27); + v[5] = _mm_madd_epi16(u[3], k__cospi_p05_p27); + v[6] = _mm_madd_epi16(u[2], k__cospi_p27_m05); + v[7] = _mm_madd_epi16(u[3], k__cospi_p27_m05); + v[8] = _mm_madd_epi16(u[4], k__cospi_p09_p23); + v[9] = _mm_madd_epi16(u[5], k__cospi_p09_p23); + v[10] = _mm_madd_epi16(u[4], k__cospi_p23_m09); + v[11] = _mm_madd_epi16(u[5], k__cospi_p23_m09); + v[12] = _mm_madd_epi16(u[6], k__cospi_p13_p19); + v[13] = _mm_madd_epi16(u[7], k__cospi_p13_p19); + v[14] = _mm_madd_epi16(u[6], k__cospi_p19_m13); + v[15] = _mm_madd_epi16(u[7], k__cospi_p19_m13); + v[16] = _mm_madd_epi16(u[8], k__cospi_p17_p15); + v[17] = _mm_madd_epi16(u[9], k__cospi_p17_p15); + v[18] = _mm_madd_epi16(u[8], k__cospi_p15_m17); + v[19] = _mm_madd_epi16(u[9], k__cospi_p15_m17); + v[20] = _mm_madd_epi16(u[10], k__cospi_p21_p11); + v[21] = _mm_madd_epi16(u[11], k__cospi_p21_p11); + v[22] = _mm_madd_epi16(u[10], k__cospi_p11_m21); + v[23] = _mm_madd_epi16(u[11], k__cospi_p11_m21); + v[24] = _mm_madd_epi16(u[12], k__cospi_p25_p07); + v[25] = _mm_madd_epi16(u[13], k__cospi_p25_p07); + v[26] = _mm_madd_epi16(u[12], k__cospi_p07_m25); + v[27] = _mm_madd_epi16(u[13], k__cospi_p07_m25); + v[28] = _mm_madd_epi16(u[14], k__cospi_p29_p03); + v[29] = _mm_madd_epi16(u[15], k__cospi_p29_p03); + v[30] = _mm_madd_epi16(u[14], k__cospi_p03_m29); + v[31] = _mm_madd_epi16(u[15], k__cospi_p03_m29); + + u[0] = _mm_add_epi32(v[0], v[16]); + u[1] = _mm_add_epi32(v[1], v[17]); + u[2] = _mm_add_epi32(v[2], v[18]); + u[3] = _mm_add_epi32(v[3], v[19]); + u[4] = _mm_add_epi32(v[4], v[20]); + u[5] = _mm_add_epi32(v[5], v[21]); + u[6] = _mm_add_epi32(v[6], v[22]); + u[7] = _mm_add_epi32(v[7], v[23]); + u[8] = _mm_add_epi32(v[8], v[24]); + u[9] = _mm_add_epi32(v[9], v[25]); + u[10] = _mm_add_epi32(v[10], v[26]); + u[11] = _mm_add_epi32(v[11], v[27]); + u[12] = _mm_add_epi32(v[12], v[28]); + u[13] = _mm_add_epi32(v[13], v[29]); + u[14] = _mm_add_epi32(v[14], v[30]); + u[15] = _mm_add_epi32(v[15], v[31]); + u[16] = _mm_sub_epi32(v[0], v[16]); + u[17] = _mm_sub_epi32(v[1], v[17]); + u[18] = _mm_sub_epi32(v[2], v[18]); + u[19] = _mm_sub_epi32(v[3], v[19]); + u[20] = _mm_sub_epi32(v[4], v[20]); + u[21] = _mm_sub_epi32(v[5], v[21]); + u[22] = _mm_sub_epi32(v[6], v[22]); + u[23] = _mm_sub_epi32(v[7], v[23]); + u[24] = _mm_sub_epi32(v[8], v[24]); + u[25] = _mm_sub_epi32(v[9], v[25]); + u[26] = _mm_sub_epi32(v[10], v[26]); + u[27] = _mm_sub_epi32(v[11], v[27]); + u[28] = _mm_sub_epi32(v[12], v[28]); + u[29] = _mm_sub_epi32(v[13], v[29]); + u[30] = _mm_sub_epi32(v[14], v[30]); + u[31] = _mm_sub_epi32(v[15], v[31]); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + v[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); + v[16] = _mm_add_epi32(u[16], k__DCT_CONST_ROUNDING); + v[17] = _mm_add_epi32(u[17], k__DCT_CONST_ROUNDING); + v[18] = _mm_add_epi32(u[18], k__DCT_CONST_ROUNDING); + v[19] = _mm_add_epi32(u[19], k__DCT_CONST_ROUNDING); + v[20] = _mm_add_epi32(u[20], k__DCT_CONST_ROUNDING); + v[21] = _mm_add_epi32(u[21], k__DCT_CONST_ROUNDING); + v[22] = _mm_add_epi32(u[22], k__DCT_CONST_ROUNDING); + v[23] = _mm_add_epi32(u[23], k__DCT_CONST_ROUNDING); + v[24] = _mm_add_epi32(u[24], k__DCT_CONST_ROUNDING); + v[25] = _mm_add_epi32(u[25], k__DCT_CONST_ROUNDING); + v[26] = _mm_add_epi32(u[26], k__DCT_CONST_ROUNDING); + v[27] = _mm_add_epi32(u[27], k__DCT_CONST_ROUNDING); + v[28] = _mm_add_epi32(u[28], k__DCT_CONST_ROUNDING); + v[29] = _mm_add_epi32(u[29], k__DCT_CONST_ROUNDING); + v[30] = _mm_add_epi32(u[30], k__DCT_CONST_ROUNDING); + v[31] = _mm_add_epi32(u[31], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS); + u[8] = _mm_srai_epi32(v[8], DCT_CONST_BITS); + u[9] = _mm_srai_epi32(v[9], DCT_CONST_BITS); + u[10] = _mm_srai_epi32(v[10], DCT_CONST_BITS); + u[11] = _mm_srai_epi32(v[11], DCT_CONST_BITS); + u[12] = _mm_srai_epi32(v[12], DCT_CONST_BITS); + u[13] = _mm_srai_epi32(v[13], DCT_CONST_BITS); + u[14] = _mm_srai_epi32(v[14], DCT_CONST_BITS); + u[15] = _mm_srai_epi32(v[15], DCT_CONST_BITS); + u[16] = _mm_srai_epi32(v[16], DCT_CONST_BITS); + u[17] = _mm_srai_epi32(v[17], DCT_CONST_BITS); + u[18] = _mm_srai_epi32(v[18], DCT_CONST_BITS); + u[19] = _mm_srai_epi32(v[19], DCT_CONST_BITS); + u[20] = _mm_srai_epi32(v[20], DCT_CONST_BITS); + u[21] = _mm_srai_epi32(v[21], DCT_CONST_BITS); + u[22] = _mm_srai_epi32(v[22], DCT_CONST_BITS); + u[23] = _mm_srai_epi32(v[23], DCT_CONST_BITS); + u[24] = _mm_srai_epi32(v[24], DCT_CONST_BITS); + u[25] = _mm_srai_epi32(v[25], DCT_CONST_BITS); + u[26] = _mm_srai_epi32(v[26], DCT_CONST_BITS); + u[27] = _mm_srai_epi32(v[27], DCT_CONST_BITS); + u[28] = _mm_srai_epi32(v[28], DCT_CONST_BITS); + u[29] = _mm_srai_epi32(v[29], DCT_CONST_BITS); + u[30] = _mm_srai_epi32(v[30], DCT_CONST_BITS); + u[31] = _mm_srai_epi32(v[31], DCT_CONST_BITS); + + s[0] = _mm_packs_epi32(u[0], u[1]); + s[1] = _mm_packs_epi32(u[2], u[3]); + s[2] = _mm_packs_epi32(u[4], u[5]); + s[3] = _mm_packs_epi32(u[6], u[7]); + s[4] = _mm_packs_epi32(u[8], u[9]); + s[5] = _mm_packs_epi32(u[10], u[11]); + s[6] = _mm_packs_epi32(u[12], u[13]); + s[7] = _mm_packs_epi32(u[14], u[15]); + s[8] = _mm_packs_epi32(u[16], u[17]); + s[9] = _mm_packs_epi32(u[18], u[19]); + s[10] = _mm_packs_epi32(u[20], u[21]); + s[11] = _mm_packs_epi32(u[22], u[23]); + s[12] = _mm_packs_epi32(u[24], u[25]); + s[13] = _mm_packs_epi32(u[26], u[27]); + s[14] = _mm_packs_epi32(u[28], u[29]); + s[15] = _mm_packs_epi32(u[30], u[31]); + + // stage 2 + u[0] = _mm_unpacklo_epi16(s[8], s[9]); + u[1] = _mm_unpackhi_epi16(s[8], s[9]); + u[2] = _mm_unpacklo_epi16(s[10], s[11]); + u[3] = _mm_unpackhi_epi16(s[10], s[11]); + u[4] = _mm_unpacklo_epi16(s[12], s[13]); + u[5] = _mm_unpackhi_epi16(s[12], s[13]); + u[6] = _mm_unpacklo_epi16(s[14], s[15]); + u[7] = _mm_unpackhi_epi16(s[14], s[15]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_p04_p28); + v[1] = _mm_madd_epi16(u[1], k__cospi_p04_p28); + v[2] = _mm_madd_epi16(u[0], k__cospi_p28_m04); + v[3] = _mm_madd_epi16(u[1], k__cospi_p28_m04); + v[4] = _mm_madd_epi16(u[2], k__cospi_p20_p12); + v[5] = _mm_madd_epi16(u[3], k__cospi_p20_p12); + v[6] = _mm_madd_epi16(u[2], k__cospi_p12_m20); + v[7] = _mm_madd_epi16(u[3], k__cospi_p12_m20); + v[8] = _mm_madd_epi16(u[4], k__cospi_m28_p04); + v[9] = _mm_madd_epi16(u[5], k__cospi_m28_p04); + v[10] = _mm_madd_epi16(u[4], k__cospi_p04_p28); + v[11] = _mm_madd_epi16(u[5], k__cospi_p04_p28); + v[12] = _mm_madd_epi16(u[6], k__cospi_m12_p20); + v[13] = _mm_madd_epi16(u[7], k__cospi_m12_p20); + v[14] = _mm_madd_epi16(u[6], k__cospi_p20_p12); + v[15] = _mm_madd_epi16(u[7], k__cospi_p20_p12); + + u[0] = _mm_add_epi32(v[0], v[8]); + u[1] = _mm_add_epi32(v[1], v[9]); + u[2] = _mm_add_epi32(v[2], v[10]); + u[3] = _mm_add_epi32(v[3], v[11]); + u[4] = _mm_add_epi32(v[4], v[12]); + u[5] = _mm_add_epi32(v[5], v[13]); + u[6] = _mm_add_epi32(v[6], v[14]); + u[7] = _mm_add_epi32(v[7], v[15]); + u[8] = _mm_sub_epi32(v[0], v[8]); + u[9] = _mm_sub_epi32(v[1], v[9]); + u[10] = _mm_sub_epi32(v[2], v[10]); + u[11] = _mm_sub_epi32(v[3], v[11]); + u[12] = _mm_sub_epi32(v[4], v[12]); + u[13] = _mm_sub_epi32(v[5], v[13]); + u[14] = _mm_sub_epi32(v[6], v[14]); + u[15] = _mm_sub_epi32(v[7], v[15]); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + v[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS); + u[8] = _mm_srai_epi32(v[8], DCT_CONST_BITS); + u[9] = _mm_srai_epi32(v[9], DCT_CONST_BITS); + u[10] = _mm_srai_epi32(v[10], DCT_CONST_BITS); + u[11] = _mm_srai_epi32(v[11], DCT_CONST_BITS); + u[12] = _mm_srai_epi32(v[12], DCT_CONST_BITS); + u[13] = _mm_srai_epi32(v[13], DCT_CONST_BITS); + u[14] = _mm_srai_epi32(v[14], DCT_CONST_BITS); + u[15] = _mm_srai_epi32(v[15], DCT_CONST_BITS); + + x[0] = _mm_add_epi16(s[0], s[4]); + x[1] = _mm_add_epi16(s[1], s[5]); + x[2] = _mm_add_epi16(s[2], s[6]); + x[3] = _mm_add_epi16(s[3], s[7]); + x[4] = _mm_sub_epi16(s[0], s[4]); + x[5] = _mm_sub_epi16(s[1], s[5]); + x[6] = _mm_sub_epi16(s[2], s[6]); + x[7] = _mm_sub_epi16(s[3], s[7]); + x[8] = _mm_packs_epi32(u[0], u[1]); + x[9] = _mm_packs_epi32(u[2], u[3]); + x[10] = _mm_packs_epi32(u[4], u[5]); + x[11] = _mm_packs_epi32(u[6], u[7]); + x[12] = _mm_packs_epi32(u[8], u[9]); + x[13] = _mm_packs_epi32(u[10], u[11]); + x[14] = _mm_packs_epi32(u[12], u[13]); + x[15] = _mm_packs_epi32(u[14], u[15]); + + // stage 3 + u[0] = _mm_unpacklo_epi16(x[4], x[5]); + u[1] = _mm_unpackhi_epi16(x[4], x[5]); + u[2] = _mm_unpacklo_epi16(x[6], x[7]); + u[3] = _mm_unpackhi_epi16(x[6], x[7]); + u[4] = _mm_unpacklo_epi16(x[12], x[13]); + u[5] = _mm_unpackhi_epi16(x[12], x[13]); + u[6] = _mm_unpacklo_epi16(x[14], x[15]); + u[7] = _mm_unpackhi_epi16(x[14], x[15]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_p08_p24); + v[1] = _mm_madd_epi16(u[1], k__cospi_p08_p24); + v[2] = _mm_madd_epi16(u[0], k__cospi_p24_m08); + v[3] = _mm_madd_epi16(u[1], k__cospi_p24_m08); + v[4] = _mm_madd_epi16(u[2], k__cospi_m24_p08); + v[5] = _mm_madd_epi16(u[3], k__cospi_m24_p08); + v[6] = _mm_madd_epi16(u[2], k__cospi_p08_p24); + v[7] = _mm_madd_epi16(u[3], k__cospi_p08_p24); + v[8] = _mm_madd_epi16(u[4], k__cospi_p08_p24); + v[9] = _mm_madd_epi16(u[5], k__cospi_p08_p24); + v[10] = _mm_madd_epi16(u[4], k__cospi_p24_m08); + v[11] = _mm_madd_epi16(u[5], k__cospi_p24_m08); + v[12] = _mm_madd_epi16(u[6], k__cospi_m24_p08); + v[13] = _mm_madd_epi16(u[7], k__cospi_m24_p08); + v[14] = _mm_madd_epi16(u[6], k__cospi_p08_p24); + v[15] = _mm_madd_epi16(u[7], k__cospi_p08_p24); + + u[0] = _mm_add_epi32(v[0], v[4]); + u[1] = _mm_add_epi32(v[1], v[5]); + u[2] = _mm_add_epi32(v[2], v[6]); + u[3] = _mm_add_epi32(v[3], v[7]); + u[4] = _mm_sub_epi32(v[0], v[4]); + u[5] = _mm_sub_epi32(v[1], v[5]); + u[6] = _mm_sub_epi32(v[2], v[6]); + u[7] = _mm_sub_epi32(v[3], v[7]); + u[8] = _mm_add_epi32(v[8], v[12]); + u[9] = _mm_add_epi32(v[9], v[13]); + u[10] = _mm_add_epi32(v[10], v[14]); + u[11] = _mm_add_epi32(v[11], v[15]); + u[12] = _mm_sub_epi32(v[8], v[12]); + u[13] = _mm_sub_epi32(v[9], v[13]); + u[14] = _mm_sub_epi32(v[10], v[14]); + u[15] = _mm_sub_epi32(v[11], v[15]); + + u[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + u[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); + u[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); + u[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); + u[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); + u[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); + u[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); + u[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); + u[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + v[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS); + v[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS); + v[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS); + v[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS); + v[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS); + v[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS); + v[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS); + v[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS); + + s[0] = _mm_add_epi16(x[0], x[2]); + s[1] = _mm_add_epi16(x[1], x[3]); + s[2] = _mm_sub_epi16(x[0], x[2]); + s[3] = _mm_sub_epi16(x[1], x[3]); + s[4] = _mm_packs_epi32(v[0], v[1]); + s[5] = _mm_packs_epi32(v[2], v[3]); + s[6] = _mm_packs_epi32(v[4], v[5]); + s[7] = _mm_packs_epi32(v[6], v[7]); + s[8] = _mm_add_epi16(x[8], x[10]); + s[9] = _mm_add_epi16(x[9], x[11]); + s[10] = _mm_sub_epi16(x[8], x[10]); + s[11] = _mm_sub_epi16(x[9], x[11]); + s[12] = _mm_packs_epi32(v[8], v[9]); + s[13] = _mm_packs_epi32(v[10], v[11]); + s[14] = _mm_packs_epi32(v[12], v[13]); + s[15] = _mm_packs_epi32(v[14], v[15]); + + // stage 4 + u[0] = _mm_unpacklo_epi16(s[2], s[3]); + u[1] = _mm_unpackhi_epi16(s[2], s[3]); + u[2] = _mm_unpacklo_epi16(s[6], s[7]); + u[3] = _mm_unpackhi_epi16(s[6], s[7]); + u[4] = _mm_unpacklo_epi16(s[10], s[11]); + u[5] = _mm_unpackhi_epi16(s[10], s[11]); + u[6] = _mm_unpacklo_epi16(s[14], s[15]); + u[7] = _mm_unpackhi_epi16(s[14], s[15]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_m16_m16); + v[1] = _mm_madd_epi16(u[1], k__cospi_m16_m16); + v[2] = _mm_madd_epi16(u[0], k__cospi_p16_m16); + v[3] = _mm_madd_epi16(u[1], k__cospi_p16_m16); + v[4] = _mm_madd_epi16(u[2], k__cospi_p16_p16); + v[5] = _mm_madd_epi16(u[3], k__cospi_p16_p16); + v[6] = _mm_madd_epi16(u[2], k__cospi_m16_p16); + v[7] = _mm_madd_epi16(u[3], k__cospi_m16_p16); + v[8] = _mm_madd_epi16(u[4], k__cospi_p16_p16); + v[9] = _mm_madd_epi16(u[5], k__cospi_p16_p16); + v[10] = _mm_madd_epi16(u[4], k__cospi_m16_p16); + v[11] = _mm_madd_epi16(u[5], k__cospi_m16_p16); + v[12] = _mm_madd_epi16(u[6], k__cospi_m16_m16); + v[13] = _mm_madd_epi16(u[7], k__cospi_m16_m16); + v[14] = _mm_madd_epi16(u[6], k__cospi_p16_m16); + v[15] = _mm_madd_epi16(u[7], k__cospi_p16_m16); + + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); + u[8] = _mm_add_epi32(v[8], k__DCT_CONST_ROUNDING); + u[9] = _mm_add_epi32(v[9], k__DCT_CONST_ROUNDING); + u[10] = _mm_add_epi32(v[10], k__DCT_CONST_ROUNDING); + u[11] = _mm_add_epi32(v[11], k__DCT_CONST_ROUNDING); + u[12] = _mm_add_epi32(v[12], k__DCT_CONST_ROUNDING); + u[13] = _mm_add_epi32(v[13], k__DCT_CONST_ROUNDING); + u[14] = _mm_add_epi32(v[14], k__DCT_CONST_ROUNDING); + u[15] = _mm_add_epi32(v[15], k__DCT_CONST_ROUNDING); + + v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + v[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS); + v[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS); + v[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS); + v[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS); + v[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS); + v[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS); + v[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS); + v[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS); + + in[0] = s[0]; + in[1] = _mm_sub_epi16(kZero, s[8]); + in[2] = s[12]; + in[3] = _mm_sub_epi16(kZero, s[4]); + in[4] = _mm_packs_epi32(v[4], v[5]); + in[5] = _mm_packs_epi32(v[12], v[13]); + in[6] = _mm_packs_epi32(v[8], v[9]); + in[7] = _mm_packs_epi32(v[0], v[1]); + in[8] = _mm_packs_epi32(v[2], v[3]); + in[9] = _mm_packs_epi32(v[10], v[11]); + in[10] = _mm_packs_epi32(v[14], v[15]); + in[11] = _mm_packs_epi32(v[6], v[7]); + in[12] = s[5]; + in[13] = _mm_sub_epi16(kZero, s[13]); + in[14] = s[9]; + in[15] = _mm_sub_epi16(kZero, s[1]); +} + +void idct16_8col(__m128i *in) { + const __m128i k__cospi_p30_m02 = pair_set_epi16(cospi_30_64, -cospi_2_64); + const __m128i k__cospi_p02_p30 = pair_set_epi16(cospi_2_64, cospi_30_64); + const __m128i k__cospi_p14_m18 = pair_set_epi16(cospi_14_64, -cospi_18_64); + const __m128i k__cospi_p18_p14 = pair_set_epi16(cospi_18_64, cospi_14_64); + const __m128i k__cospi_p22_m10 = pair_set_epi16(cospi_22_64, -cospi_10_64); + const __m128i k__cospi_p10_p22 = pair_set_epi16(cospi_10_64, cospi_22_64); + const __m128i k__cospi_p06_m26 = pair_set_epi16(cospi_6_64, -cospi_26_64); + const __m128i k__cospi_p26_p06 = pair_set_epi16(cospi_26_64, cospi_6_64); + const __m128i k__cospi_p28_m04 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i k__cospi_p04_p28 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i k__cospi_p12_m20 = pair_set_epi16(cospi_12_64, -cospi_20_64); + const __m128i k__cospi_p20_p12 = pair_set_epi16(cospi_20_64, cospi_12_64); + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64); + const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + __m128i v[16], u[16], s[16], t[16]; + + // stage 1 + s[0] = in[0]; + s[1] = in[8]; + s[2] = in[4]; + s[3] = in[12]; + s[4] = in[2]; + s[5] = in[10]; + s[6] = in[6]; + s[7] = in[14]; + s[8] = in[1]; + s[9] = in[9]; + s[10] = in[5]; + s[11] = in[13]; + s[12] = in[3]; + s[13] = in[11]; + s[14] = in[7]; + s[15] = in[15]; + + // stage 2 + u[0] = _mm_unpacklo_epi16(s[8], s[15]); + u[1] = _mm_unpackhi_epi16(s[8], s[15]); + u[2] = _mm_unpacklo_epi16(s[9], s[14]); + u[3] = _mm_unpackhi_epi16(s[9], s[14]); + u[4] = _mm_unpacklo_epi16(s[10], s[13]); + u[5] = _mm_unpackhi_epi16(s[10], s[13]); + u[6] = _mm_unpacklo_epi16(s[11], s[12]); + u[7] = _mm_unpackhi_epi16(s[11], s[12]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_p30_m02); + v[1] = _mm_madd_epi16(u[1], k__cospi_p30_m02); + v[2] = _mm_madd_epi16(u[0], k__cospi_p02_p30); + v[3] = _mm_madd_epi16(u[1], k__cospi_p02_p30); + v[4] = _mm_madd_epi16(u[2], k__cospi_p14_m18); + v[5] = _mm_madd_epi16(u[3], k__cospi_p14_m18); + v[6] = _mm_madd_epi16(u[2], k__cospi_p18_p14); + v[7] = _mm_madd_epi16(u[3], k__cospi_p18_p14); + v[8] = _mm_madd_epi16(u[4], k__cospi_p22_m10); + v[9] = _mm_madd_epi16(u[5], k__cospi_p22_m10); + v[10] = _mm_madd_epi16(u[4], k__cospi_p10_p22); + v[11] = _mm_madd_epi16(u[5], k__cospi_p10_p22); + v[12] = _mm_madd_epi16(u[6], k__cospi_p06_m26); + v[13] = _mm_madd_epi16(u[7], k__cospi_p06_m26); + v[14] = _mm_madd_epi16(u[6], k__cospi_p26_p06); + v[15] = _mm_madd_epi16(u[7], k__cospi_p26_p06); + + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); + u[8] = _mm_add_epi32(v[8], k__DCT_CONST_ROUNDING); + u[9] = _mm_add_epi32(v[9], k__DCT_CONST_ROUNDING); + u[10] = _mm_add_epi32(v[10], k__DCT_CONST_ROUNDING); + u[11] = _mm_add_epi32(v[11], k__DCT_CONST_ROUNDING); + u[12] = _mm_add_epi32(v[12], k__DCT_CONST_ROUNDING); + u[13] = _mm_add_epi32(v[13], k__DCT_CONST_ROUNDING); + u[14] = _mm_add_epi32(v[14], k__DCT_CONST_ROUNDING); + u[15] = _mm_add_epi32(v[15], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + u[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + u[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + u[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + u[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + u[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS); + u[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS); + u[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS); + u[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS); + u[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS); + u[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS); + u[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS); + u[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS); + + s[8] = _mm_packs_epi32(u[0], u[1]); + s[15] = _mm_packs_epi32(u[2], u[3]); + s[9] = _mm_packs_epi32(u[4], u[5]); + s[14] = _mm_packs_epi32(u[6], u[7]); + s[10] = _mm_packs_epi32(u[8], u[9]); + s[13] = _mm_packs_epi32(u[10], u[11]); + s[11] = _mm_packs_epi32(u[12], u[13]); + s[12] = _mm_packs_epi32(u[14], u[15]); + + // stage 3 + t[0] = s[0]; + t[1] = s[1]; + t[2] = s[2]; + t[3] = s[3]; + u[0] = _mm_unpacklo_epi16(s[4], s[7]); + u[1] = _mm_unpackhi_epi16(s[4], s[7]); + u[2] = _mm_unpacklo_epi16(s[5], s[6]); + u[3] = _mm_unpackhi_epi16(s[5], s[6]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_p28_m04); + v[1] = _mm_madd_epi16(u[1], k__cospi_p28_m04); + v[2] = _mm_madd_epi16(u[0], k__cospi_p04_p28); + v[3] = _mm_madd_epi16(u[1], k__cospi_p04_p28); + v[4] = _mm_madd_epi16(u[2], k__cospi_p12_m20); + v[5] = _mm_madd_epi16(u[3], k__cospi_p12_m20); + v[6] = _mm_madd_epi16(u[2], k__cospi_p20_p12); + v[7] = _mm_madd_epi16(u[3], k__cospi_p20_p12); + + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + u[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + u[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + u[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + u[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + + t[4] = _mm_packs_epi32(u[0], u[1]); + t[7] = _mm_packs_epi32(u[2], u[3]); + t[5] = _mm_packs_epi32(u[4], u[5]); + t[6] = _mm_packs_epi32(u[6], u[7]); + t[8] = _mm_add_epi16(s[8], s[9]); + t[9] = _mm_sub_epi16(s[8], s[9]); + t[10] = _mm_sub_epi16(s[11], s[10]); + t[11] = _mm_add_epi16(s[10], s[11]); + t[12] = _mm_add_epi16(s[12], s[13]); + t[13] = _mm_sub_epi16(s[12], s[13]); + t[14] = _mm_sub_epi16(s[15], s[14]); + t[15] = _mm_add_epi16(s[14], s[15]); + + // stage 4 + u[0] = _mm_unpacklo_epi16(t[0], t[1]); + u[1] = _mm_unpackhi_epi16(t[0], t[1]); + u[2] = _mm_unpacklo_epi16(t[2], t[3]); + u[3] = _mm_unpackhi_epi16(t[2], t[3]); + u[4] = _mm_unpacklo_epi16(t[9], t[14]); + u[5] = _mm_unpackhi_epi16(t[9], t[14]); + u[6] = _mm_unpacklo_epi16(t[10], t[13]); + u[7] = _mm_unpackhi_epi16(t[10], t[13]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_p16_p16); + v[1] = _mm_madd_epi16(u[1], k__cospi_p16_p16); + v[2] = _mm_madd_epi16(u[0], k__cospi_p16_m16); + v[3] = _mm_madd_epi16(u[1], k__cospi_p16_m16); + v[4] = _mm_madd_epi16(u[2], k__cospi_p24_m08); + v[5] = _mm_madd_epi16(u[3], k__cospi_p24_m08); + v[6] = _mm_madd_epi16(u[2], k__cospi_p08_p24); + v[7] = _mm_madd_epi16(u[3], k__cospi_p08_p24); + v[8] = _mm_madd_epi16(u[4], k__cospi_m08_p24); + v[9] = _mm_madd_epi16(u[5], k__cospi_m08_p24); + v[10] = _mm_madd_epi16(u[4], k__cospi_p24_p08); + v[11] = _mm_madd_epi16(u[5], k__cospi_p24_p08); + v[12] = _mm_madd_epi16(u[6], k__cospi_m24_m08); + v[13] = _mm_madd_epi16(u[7], k__cospi_m24_m08); + v[14] = _mm_madd_epi16(u[6], k__cospi_m08_p24); + v[15] = _mm_madd_epi16(u[7], k__cospi_m08_p24); + + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); + u[8] = _mm_add_epi32(v[8], k__DCT_CONST_ROUNDING); + u[9] = _mm_add_epi32(v[9], k__DCT_CONST_ROUNDING); + u[10] = _mm_add_epi32(v[10], k__DCT_CONST_ROUNDING); + u[11] = _mm_add_epi32(v[11], k__DCT_CONST_ROUNDING); + u[12] = _mm_add_epi32(v[12], k__DCT_CONST_ROUNDING); + u[13] = _mm_add_epi32(v[13], k__DCT_CONST_ROUNDING); + u[14] = _mm_add_epi32(v[14], k__DCT_CONST_ROUNDING); + u[15] = _mm_add_epi32(v[15], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + u[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + u[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + u[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + u[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + u[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS); + u[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS); + u[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS); + u[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS); + u[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS); + u[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS); + u[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS); + u[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS); + + s[0] = _mm_packs_epi32(u[0], u[1]); + s[1] = _mm_packs_epi32(u[2], u[3]); + s[2] = _mm_packs_epi32(u[4], u[5]); + s[3] = _mm_packs_epi32(u[6], u[7]); + s[4] = _mm_add_epi16(t[4], t[5]); + s[5] = _mm_sub_epi16(t[4], t[5]); + s[6] = _mm_sub_epi16(t[7], t[6]); + s[7] = _mm_add_epi16(t[6], t[7]); + s[8] = t[8]; + s[15] = t[15]; + s[9] = _mm_packs_epi32(u[8], u[9]); + s[14] = _mm_packs_epi32(u[10], u[11]); + s[10] = _mm_packs_epi32(u[12], u[13]); + s[13] = _mm_packs_epi32(u[14], u[15]); + s[11] = t[11]; + s[12] = t[12]; + + // stage 5 + t[0] = _mm_add_epi16(s[0], s[3]); + t[1] = _mm_add_epi16(s[1], s[2]); + t[2] = _mm_sub_epi16(s[1], s[2]); + t[3] = _mm_sub_epi16(s[0], s[3]); + t[4] = s[4]; + t[7] = s[7]; + + u[0] = _mm_unpacklo_epi16(s[5], s[6]); + u[1] = _mm_unpackhi_epi16(s[5], s[6]); + v[0] = _mm_madd_epi16(u[0], k__cospi_m16_p16); + v[1] = _mm_madd_epi16(u[1], k__cospi_m16_p16); + v[2] = _mm_madd_epi16(u[0], k__cospi_p16_p16); + v[3] = _mm_madd_epi16(u[1], k__cospi_p16_p16); + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + u[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + t[5] = _mm_packs_epi32(u[0], u[1]); + t[6] = _mm_packs_epi32(u[2], u[3]); + + t[8] = _mm_add_epi16(s[8], s[11]); + t[9] = _mm_add_epi16(s[9], s[10]); + t[10] = _mm_sub_epi16(s[9], s[10]); + t[11] = _mm_sub_epi16(s[8], s[11]); + t[12] = _mm_sub_epi16(s[15], s[12]); + t[13] = _mm_sub_epi16(s[14], s[13]); + t[14] = _mm_add_epi16(s[13], s[14]); + t[15] = _mm_add_epi16(s[12], s[15]); + + // stage 6 + s[0] = _mm_add_epi16(t[0], t[7]); + s[1] = _mm_add_epi16(t[1], t[6]); + s[2] = _mm_add_epi16(t[2], t[5]); + s[3] = _mm_add_epi16(t[3], t[4]); + s[4] = _mm_sub_epi16(t[3], t[4]); + s[5] = _mm_sub_epi16(t[2], t[5]); + s[6] = _mm_sub_epi16(t[1], t[6]); + s[7] = _mm_sub_epi16(t[0], t[7]); + s[8] = t[8]; + s[9] = t[9]; + + u[0] = _mm_unpacklo_epi16(t[10], t[13]); + u[1] = _mm_unpackhi_epi16(t[10], t[13]); + u[2] = _mm_unpacklo_epi16(t[11], t[12]); + u[3] = _mm_unpackhi_epi16(t[11], t[12]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_m16_p16); + v[1] = _mm_madd_epi16(u[1], k__cospi_m16_p16); + v[2] = _mm_madd_epi16(u[0], k__cospi_p16_p16); + v[3] = _mm_madd_epi16(u[1], k__cospi_p16_p16); + v[4] = _mm_madd_epi16(u[2], k__cospi_m16_p16); + v[5] = _mm_madd_epi16(u[3], k__cospi_m16_p16); + v[6] = _mm_madd_epi16(u[2], k__cospi_p16_p16); + v[7] = _mm_madd_epi16(u[3], k__cospi_p16_p16); + + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + u[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + u[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + u[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + u[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + + s[10] = _mm_packs_epi32(u[0], u[1]); + s[13] = _mm_packs_epi32(u[2], u[3]); + s[11] = _mm_packs_epi32(u[4], u[5]); + s[12] = _mm_packs_epi32(u[6], u[7]); + s[14] = t[14]; + s[15] = t[15]; + + // stage 7 + in[0] = _mm_add_epi16(s[0], s[15]); + in[1] = _mm_add_epi16(s[1], s[14]); + in[2] = _mm_add_epi16(s[2], s[13]); + in[3] = _mm_add_epi16(s[3], s[12]); + in[4] = _mm_add_epi16(s[4], s[11]); + in[5] = _mm_add_epi16(s[5], s[10]); + in[6] = _mm_add_epi16(s[6], s[9]); + in[7] = _mm_add_epi16(s[7], s[8]); + in[8] = _mm_sub_epi16(s[7], s[8]); + in[9] = _mm_sub_epi16(s[6], s[9]); + in[10] = _mm_sub_epi16(s[5], s[10]); + in[11] = _mm_sub_epi16(s[4], s[11]); + in[12] = _mm_sub_epi16(s[3], s[12]); + in[13] = _mm_sub_epi16(s[2], s[13]); + in[14] = _mm_sub_epi16(s[1], s[14]); + in[15] = _mm_sub_epi16(s[0], s[15]); +} + +void aom_idct16_sse2(__m128i *in0, __m128i *in1) { + array_transpose_16x16(in0, in1); + idct16_8col(in0); + idct16_8col(in1); +} + +void aom_iadst16_sse2(__m128i *in0, __m128i *in1) { + array_transpose_16x16(in0, in1); + iadst16_8col(in0); + iadst16_8col(in1); +} + +void aom_idct16x16_10_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride) { + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i final_rounding = _mm_set1_epi16(1 << 5); + const __m128i zero = _mm_setzero_si128(); + + const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64); + const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64); + const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64); + const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64); + + const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64); + + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + const __m128i stg4_7 = pair_set_epi16(-cospi_8_64, cospi_24_64); + + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + __m128i in[16], l[16]; + __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_8, + stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15, stp1_8_0, + stp1_12_0; + __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7, + stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + int i; + // First 1-D inverse DCT + // Load input data. + in[0] = load_input_data(input); + in[1] = load_input_data(input + 8 * 2); + in[2] = load_input_data(input + 8 * 4); + in[3] = load_input_data(input + 8 * 6); + + TRANSPOSE_8X4(in[0], in[1], in[2], in[3], in[0], in[1]); + + // Stage2 + { + const __m128i lo_1_15 = _mm_unpackhi_epi16(in[0], zero); + const __m128i lo_13_3 = _mm_unpackhi_epi16(zero, in[1]); + + tmp0 = _mm_madd_epi16(lo_1_15, stg2_0); + tmp2 = _mm_madd_epi16(lo_1_15, stg2_1); + tmp5 = _mm_madd_epi16(lo_13_3, stg2_6); + tmp7 = _mm_madd_epi16(lo_13_3, stg2_7); + + tmp0 = _mm_add_epi32(tmp0, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp5 = _mm_add_epi32(tmp5, rounding); + tmp7 = _mm_add_epi32(tmp7, rounding); + + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + tmp5 = _mm_srai_epi32(tmp5, DCT_CONST_BITS); + tmp7 = _mm_srai_epi32(tmp7, DCT_CONST_BITS); + + stp2_8 = _mm_packs_epi32(tmp0, tmp2); + stp2_11 = _mm_packs_epi32(tmp5, tmp7); + } + + // Stage3 + { + const __m128i lo_2_14 = _mm_unpacklo_epi16(in[1], zero); + + tmp0 = _mm_madd_epi16(lo_2_14, stg3_0); + tmp2 = _mm_madd_epi16(lo_2_14, stg3_1); + + tmp0 = _mm_add_epi32(tmp0, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + + stp1_13 = _mm_unpackhi_epi64(stp2_11, zero); + stp1_14 = _mm_unpackhi_epi64(stp2_8, zero); + + stp1_4 = _mm_packs_epi32(tmp0, tmp2); + } + + // Stage4 + { + const __m128i lo_0_8 = _mm_unpacklo_epi16(in[0], zero); + const __m128i lo_9_14 = _mm_unpacklo_epi16(stp2_8, stp1_14); + const __m128i lo_10_13 = _mm_unpacklo_epi16(stp2_11, stp1_13); + + tmp0 = _mm_madd_epi16(lo_0_8, stg4_0); + tmp2 = _mm_madd_epi16(lo_0_8, stg4_1); + tmp1 = _mm_madd_epi16(lo_9_14, stg4_4); + tmp3 = _mm_madd_epi16(lo_9_14, stg4_5); + tmp5 = _mm_madd_epi16(lo_10_13, stg4_6); + tmp7 = _mm_madd_epi16(lo_10_13, stg4_7); + + tmp0 = _mm_add_epi32(tmp0, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp1 = _mm_add_epi32(tmp1, rounding); + tmp3 = _mm_add_epi32(tmp3, rounding); + tmp5 = _mm_add_epi32(tmp5, rounding); + tmp7 = _mm_add_epi32(tmp7, rounding); + + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); + tmp5 = _mm_srai_epi32(tmp5, DCT_CONST_BITS); + tmp7 = _mm_srai_epi32(tmp7, DCT_CONST_BITS); + + stp1_0 = _mm_packs_epi32(tmp0, tmp0); + stp1_1 = _mm_packs_epi32(tmp2, tmp2); + stp2_9 = _mm_packs_epi32(tmp1, tmp3); + stp2_10 = _mm_packs_epi32(tmp5, tmp7); + + stp2_6 = _mm_unpackhi_epi64(stp1_4, zero); + } + + // Stage5 and Stage6 + { + tmp0 = _mm_add_epi16(stp2_8, stp2_11); + tmp1 = _mm_sub_epi16(stp2_8, stp2_11); + tmp2 = _mm_add_epi16(stp2_9, stp2_10); + tmp3 = _mm_sub_epi16(stp2_9, stp2_10); + + stp1_9 = _mm_unpacklo_epi64(tmp2, zero); + stp1_10 = _mm_unpacklo_epi64(tmp3, zero); + stp1_8 = _mm_unpacklo_epi64(tmp0, zero); + stp1_11 = _mm_unpacklo_epi64(tmp1, zero); + + stp1_13 = _mm_unpackhi_epi64(tmp3, zero); + stp1_14 = _mm_unpackhi_epi64(tmp2, zero); + stp1_12 = _mm_unpackhi_epi64(tmp1, zero); + stp1_15 = _mm_unpackhi_epi64(tmp0, zero); + } + + // Stage6 + { + const __m128i lo_6_5 = _mm_unpacklo_epi16(stp2_6, stp1_4); + const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); + const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1_11, stp1_12); + + tmp1 = _mm_madd_epi16(lo_6_5, stg4_1); + tmp3 = _mm_madd_epi16(lo_6_5, stg4_0); + tmp0 = _mm_madd_epi16(lo_10_13, stg6_0); + tmp2 = _mm_madd_epi16(lo_10_13, stg4_0); + tmp4 = _mm_madd_epi16(lo_11_12, stg6_0); + tmp6 = _mm_madd_epi16(lo_11_12, stg4_0); + + tmp1 = _mm_add_epi32(tmp1, rounding); + tmp3 = _mm_add_epi32(tmp3, rounding); + tmp0 = _mm_add_epi32(tmp0, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp4 = _mm_add_epi32(tmp4, rounding); + tmp6 = _mm_add_epi32(tmp6, rounding); + + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + tmp4 = _mm_srai_epi32(tmp4, DCT_CONST_BITS); + tmp6 = _mm_srai_epi32(tmp6, DCT_CONST_BITS); + + stp1_6 = _mm_packs_epi32(tmp3, tmp1); + + stp2_10 = _mm_packs_epi32(tmp0, zero); + stp2_13 = _mm_packs_epi32(tmp2, zero); + stp2_11 = _mm_packs_epi32(tmp4, zero); + stp2_12 = _mm_packs_epi32(tmp6, zero); + + tmp0 = _mm_add_epi16(stp1_0, stp1_4); + tmp1 = _mm_sub_epi16(stp1_0, stp1_4); + tmp2 = _mm_add_epi16(stp1_1, stp1_6); + tmp3 = _mm_sub_epi16(stp1_1, stp1_6); + + stp2_0 = _mm_unpackhi_epi64(tmp0, zero); + stp2_1 = _mm_unpacklo_epi64(tmp2, zero); + stp2_2 = _mm_unpackhi_epi64(tmp2, zero); + stp2_3 = _mm_unpacklo_epi64(tmp0, zero); + stp2_4 = _mm_unpacklo_epi64(tmp1, zero); + stp2_5 = _mm_unpackhi_epi64(tmp3, zero); + stp2_6 = _mm_unpacklo_epi64(tmp3, zero); + stp2_7 = _mm_unpackhi_epi64(tmp1, zero); + } + + // Stage7. Left 8x16 only. + l[0] = _mm_add_epi16(stp2_0, stp1_15); + l[1] = _mm_add_epi16(stp2_1, stp1_14); + l[2] = _mm_add_epi16(stp2_2, stp2_13); + l[3] = _mm_add_epi16(stp2_3, stp2_12); + l[4] = _mm_add_epi16(stp2_4, stp2_11); + l[5] = _mm_add_epi16(stp2_5, stp2_10); + l[6] = _mm_add_epi16(stp2_6, stp1_9); + l[7] = _mm_add_epi16(stp2_7, stp1_8); + l[8] = _mm_sub_epi16(stp2_7, stp1_8); + l[9] = _mm_sub_epi16(stp2_6, stp1_9); + l[10] = _mm_sub_epi16(stp2_5, stp2_10); + l[11] = _mm_sub_epi16(stp2_4, stp2_11); + l[12] = _mm_sub_epi16(stp2_3, stp2_12); + l[13] = _mm_sub_epi16(stp2_2, stp2_13); + l[14] = _mm_sub_epi16(stp2_1, stp1_14); + l[15] = _mm_sub_epi16(stp2_0, stp1_15); + + // Second 1-D inverse transform, performed per 8x16 block + for (i = 0; i < 2; i++) { + int j; + array_transpose_4X8(l + 8 * i, in); + + IDCT16_10 + + // Stage7 + in[0] = _mm_add_epi16(stp2_0, stp1_15); + in[1] = _mm_add_epi16(stp2_1, stp1_14); + in[2] = _mm_add_epi16(stp2_2, stp2_13); + in[3] = _mm_add_epi16(stp2_3, stp2_12); + in[4] = _mm_add_epi16(stp2_4, stp2_11); + in[5] = _mm_add_epi16(stp2_5, stp2_10); + in[6] = _mm_add_epi16(stp2_6, stp1_9); + in[7] = _mm_add_epi16(stp2_7, stp1_8); + in[8] = _mm_sub_epi16(stp2_7, stp1_8); + in[9] = _mm_sub_epi16(stp2_6, stp1_9); + in[10] = _mm_sub_epi16(stp2_5, stp2_10); + in[11] = _mm_sub_epi16(stp2_4, stp2_11); + in[12] = _mm_sub_epi16(stp2_3, stp2_12); + in[13] = _mm_sub_epi16(stp2_2, stp2_13); + in[14] = _mm_sub_epi16(stp2_1, stp1_14); + in[15] = _mm_sub_epi16(stp2_0, stp1_15); + + for (j = 0; j < 16; ++j) { + // Final rounding and shift + in[j] = _mm_adds_epi16(in[j], final_rounding); + in[j] = _mm_srai_epi16(in[j], 6); + RECON_AND_STORE(dest + j * stride, in[j]); + } + + dest += 8; + } +} + +#define LOAD_DQCOEFF(reg, input) \ + { \ + reg = load_input_data(input); \ + input += 8; \ + } + +#define IDCT32_34 \ + /* Stage1 */ \ + { \ + const __m128i lo_1_31 = _mm_unpacklo_epi16(in[1], zero); \ + const __m128i hi_1_31 = _mm_unpackhi_epi16(in[1], zero); \ + \ + const __m128i lo_25_7 = _mm_unpacklo_epi16(zero, in[7]); \ + const __m128i hi_25_7 = _mm_unpackhi_epi16(zero, in[7]); \ + \ + const __m128i lo_5_27 = _mm_unpacklo_epi16(in[5], zero); \ + const __m128i hi_5_27 = _mm_unpackhi_epi16(in[5], zero); \ + \ + const __m128i lo_29_3 = _mm_unpacklo_epi16(zero, in[3]); \ + const __m128i hi_29_3 = _mm_unpackhi_epi16(zero, in[3]); \ + \ + MULTIPLICATION_AND_ADD_2(lo_1_31, hi_1_31, stg1_0, stg1_1, stp1_16, \ + stp1_31); \ + MULTIPLICATION_AND_ADD_2(lo_25_7, hi_25_7, stg1_6, stg1_7, stp1_19, \ + stp1_28); \ + MULTIPLICATION_AND_ADD_2(lo_5_27, hi_5_27, stg1_8, stg1_9, stp1_20, \ + stp1_27); \ + MULTIPLICATION_AND_ADD_2(lo_29_3, hi_29_3, stg1_14, stg1_15, stp1_23, \ + stp1_24); \ + } \ + \ + /* Stage2 */ \ + { \ + const __m128i lo_2_30 = _mm_unpacklo_epi16(in[2], zero); \ + const __m128i hi_2_30 = _mm_unpackhi_epi16(in[2], zero); \ + \ + const __m128i lo_26_6 = _mm_unpacklo_epi16(zero, in[6]); \ + const __m128i hi_26_6 = _mm_unpackhi_epi16(zero, in[6]); \ + \ + MULTIPLICATION_AND_ADD_2(lo_2_30, hi_2_30, stg2_0, stg2_1, stp2_8, \ + stp2_15); \ + MULTIPLICATION_AND_ADD_2(lo_26_6, hi_26_6, stg2_6, stg2_7, stp2_11, \ + stp2_12); \ + \ + stp2_16 = stp1_16; \ + stp2_19 = stp1_19; \ + \ + stp2_20 = stp1_20; \ + stp2_23 = stp1_23; \ + \ + stp2_24 = stp1_24; \ + stp2_27 = stp1_27; \ + \ + stp2_28 = stp1_28; \ + stp2_31 = stp1_31; \ + } \ + \ + /* Stage3 */ \ + { \ + const __m128i lo_4_28 = _mm_unpacklo_epi16(in[4], zero); \ + const __m128i hi_4_28 = _mm_unpackhi_epi16(in[4], zero); \ + \ + const __m128i lo_17_30 = _mm_unpacklo_epi16(stp1_16, stp1_31); \ + const __m128i hi_17_30 = _mm_unpackhi_epi16(stp1_16, stp1_31); \ + const __m128i lo_18_29 = _mm_unpacklo_epi16(stp1_19, stp1_28); \ + const __m128i hi_18_29 = _mm_unpackhi_epi16(stp1_19, stp1_28); \ + \ + const __m128i lo_21_26 = _mm_unpacklo_epi16(stp1_20, stp1_27); \ + const __m128i hi_21_26 = _mm_unpackhi_epi16(stp1_20, stp1_27); \ + const __m128i lo_22_25 = _mm_unpacklo_epi16(stp1_23, stp1_24); \ + const __m128i hi_22_25 = _mm_unpackhi_epi16(stp1_23, stp2_24); \ + \ + MULTIPLICATION_AND_ADD_2(lo_4_28, hi_4_28, stg3_0, stg3_1, stp1_4, \ + stp1_7); \ + \ + stp1_8 = stp2_8; \ + stp1_11 = stp2_11; \ + stp1_12 = stp2_12; \ + stp1_15 = stp2_15; \ + \ + MULTIPLICATION_AND_ADD(lo_17_30, hi_17_30, lo_18_29, hi_18_29, stg3_4, \ + stg3_5, stg3_6, stg3_4, stp1_17, stp1_30, stp1_18, \ + stp1_29) \ + MULTIPLICATION_AND_ADD(lo_21_26, hi_21_26, lo_22_25, hi_22_25, stg3_8, \ + stg3_9, stg3_10, stg3_8, stp1_21, stp1_26, stp1_22, \ + stp1_25) \ + \ + stp1_16 = stp2_16; \ + stp1_31 = stp2_31; \ + stp1_19 = stp2_19; \ + stp1_20 = stp2_20; \ + stp1_23 = stp2_23; \ + stp1_24 = stp2_24; \ + stp1_27 = stp2_27; \ + stp1_28 = stp2_28; \ + } \ + \ + /* Stage4 */ \ + { \ + const __m128i lo_0_16 = _mm_unpacklo_epi16(in[0], zero); \ + const __m128i hi_0_16 = _mm_unpackhi_epi16(in[0], zero); \ + \ + const __m128i lo_9_14 = _mm_unpacklo_epi16(stp2_8, stp2_15); \ + const __m128i hi_9_14 = _mm_unpackhi_epi16(stp2_8, stp2_15); \ + const __m128i lo_10_13 = _mm_unpacklo_epi16(stp2_11, stp2_12); \ + const __m128i hi_10_13 = _mm_unpackhi_epi16(stp2_11, stp2_12); \ + \ + MULTIPLICATION_AND_ADD_2(lo_0_16, hi_0_16, stg4_0, stg4_1, stp2_0, \ + stp2_1); \ + \ + stp2_4 = stp1_4; \ + stp2_5 = stp1_4; \ + stp2_6 = stp1_7; \ + stp2_7 = stp1_7; \ + \ + MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, \ + stg4_5, stg4_6, stg4_4, stp2_9, stp2_14, stp2_10, \ + stp2_13) \ + \ + stp2_8 = stp1_8; \ + stp2_15 = stp1_15; \ + stp2_11 = stp1_11; \ + stp2_12 = stp1_12; \ + \ + stp2_16 = _mm_add_epi16(stp1_16, stp1_19); \ + stp2_17 = _mm_add_epi16(stp1_17, stp1_18); \ + stp2_18 = _mm_sub_epi16(stp1_17, stp1_18); \ + stp2_19 = _mm_sub_epi16(stp1_16, stp1_19); \ + stp2_20 = _mm_sub_epi16(stp1_23, stp1_20); \ + stp2_21 = _mm_sub_epi16(stp1_22, stp1_21); \ + stp2_22 = _mm_add_epi16(stp1_22, stp1_21); \ + stp2_23 = _mm_add_epi16(stp1_23, stp1_20); \ + \ + stp2_24 = _mm_add_epi16(stp1_24, stp1_27); \ + stp2_25 = _mm_add_epi16(stp1_25, stp1_26); \ + stp2_26 = _mm_sub_epi16(stp1_25, stp1_26); \ + stp2_27 = _mm_sub_epi16(stp1_24, stp1_27); \ + stp2_28 = _mm_sub_epi16(stp1_31, stp1_28); \ + stp2_29 = _mm_sub_epi16(stp1_30, stp1_29); \ + stp2_30 = _mm_add_epi16(stp1_29, stp1_30); \ + stp2_31 = _mm_add_epi16(stp1_28, stp1_31); \ + } \ + \ + /* Stage5 */ \ + { \ + const __m128i lo_6_5 = _mm_unpacklo_epi16(stp2_6, stp2_5); \ + const __m128i hi_6_5 = _mm_unpackhi_epi16(stp2_6, stp2_5); \ + const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29); \ + const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29); \ + \ + const __m128i lo_19_28 = _mm_unpacklo_epi16(stp2_19, stp2_28); \ + const __m128i hi_19_28 = _mm_unpackhi_epi16(stp2_19, stp2_28); \ + const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); \ + const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); \ + \ + const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); \ + const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); \ + \ + stp1_0 = stp2_0; \ + stp1_1 = stp2_1; \ + stp1_2 = stp2_1; \ + stp1_3 = stp2_0; \ + \ + tmp0 = _mm_madd_epi16(lo_6_5, stg4_1); \ + tmp1 = _mm_madd_epi16(hi_6_5, stg4_1); \ + tmp2 = _mm_madd_epi16(lo_6_5, stg4_0); \ + tmp3 = _mm_madd_epi16(hi_6_5, stg4_0); \ + \ + tmp0 = _mm_add_epi32(tmp0, rounding); \ + tmp1 = _mm_add_epi32(tmp1, rounding); \ + tmp2 = _mm_add_epi32(tmp2, rounding); \ + tmp3 = _mm_add_epi32(tmp3, rounding); \ + \ + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \ + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \ + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \ + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \ + \ + stp1_5 = _mm_packs_epi32(tmp0, tmp1); \ + stp1_6 = _mm_packs_epi32(tmp2, tmp3); \ + \ + stp1_4 = stp2_4; \ + stp1_7 = stp2_7; \ + \ + stp1_8 = _mm_add_epi16(stp2_8, stp2_11); \ + stp1_9 = _mm_add_epi16(stp2_9, stp2_10); \ + stp1_10 = _mm_sub_epi16(stp2_9, stp2_10); \ + stp1_11 = _mm_sub_epi16(stp2_8, stp2_11); \ + stp1_12 = _mm_sub_epi16(stp2_15, stp2_12); \ + stp1_13 = _mm_sub_epi16(stp2_14, stp2_13); \ + stp1_14 = _mm_add_epi16(stp2_14, stp2_13); \ + stp1_15 = _mm_add_epi16(stp2_15, stp2_12); \ + \ + stp1_16 = stp2_16; \ + stp1_17 = stp2_17; \ + \ + MULTIPLICATION_AND_ADD(lo_18_29, hi_18_29, lo_19_28, hi_19_28, stg4_4, \ + stg4_5, stg4_4, stg4_5, stp1_18, stp1_29, stp1_19, \ + stp1_28) \ + MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg4_6, \ + stg4_4, stg4_6, stg4_4, stp1_20, stp1_27, stp1_21, \ + stp1_26) \ + \ + stp1_22 = stp2_22; \ + stp1_23 = stp2_23; \ + stp1_24 = stp2_24; \ + stp1_25 = stp2_25; \ + stp1_30 = stp2_30; \ + stp1_31 = stp2_31; \ + } \ + \ + /* Stage6 */ \ + { \ + const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); \ + const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); \ + const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1_11, stp1_12); \ + const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1_11, stp1_12); \ + \ + stp2_0 = _mm_add_epi16(stp1_0, stp1_7); \ + stp2_1 = _mm_add_epi16(stp1_1, stp1_6); \ + stp2_2 = _mm_add_epi16(stp1_2, stp1_5); \ + stp2_3 = _mm_add_epi16(stp1_3, stp1_4); \ + stp2_4 = _mm_sub_epi16(stp1_3, stp1_4); \ + stp2_5 = _mm_sub_epi16(stp1_2, stp1_5); \ + stp2_6 = _mm_sub_epi16(stp1_1, stp1_6); \ + stp2_7 = _mm_sub_epi16(stp1_0, stp1_7); \ + \ + stp2_8 = stp1_8; \ + stp2_9 = stp1_9; \ + stp2_14 = stp1_14; \ + stp2_15 = stp1_15; \ + \ + MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0, \ + stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11, \ + stp2_12) \ + \ + stp2_16 = _mm_add_epi16(stp1_16, stp1_23); \ + stp2_17 = _mm_add_epi16(stp1_17, stp1_22); \ + stp2_18 = _mm_add_epi16(stp1_18, stp1_21); \ + stp2_19 = _mm_add_epi16(stp1_19, stp1_20); \ + stp2_20 = _mm_sub_epi16(stp1_19, stp1_20); \ + stp2_21 = _mm_sub_epi16(stp1_18, stp1_21); \ + stp2_22 = _mm_sub_epi16(stp1_17, stp1_22); \ + stp2_23 = _mm_sub_epi16(stp1_16, stp1_23); \ + \ + stp2_24 = _mm_sub_epi16(stp1_31, stp1_24); \ + stp2_25 = _mm_sub_epi16(stp1_30, stp1_25); \ + stp2_26 = _mm_sub_epi16(stp1_29, stp1_26); \ + stp2_27 = _mm_sub_epi16(stp1_28, stp1_27); \ + stp2_28 = _mm_add_epi16(stp1_27, stp1_28); \ + stp2_29 = _mm_add_epi16(stp1_26, stp1_29); \ + stp2_30 = _mm_add_epi16(stp1_25, stp1_30); \ + stp2_31 = _mm_add_epi16(stp1_24, stp1_31); \ + } \ + \ + /* Stage7 */ \ + { \ + const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); \ + const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); \ + const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); \ + const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); \ + \ + const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25); \ + const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25); \ + const __m128i lo_23_24 = _mm_unpacklo_epi16(stp2_23, stp2_24); \ + const __m128i hi_23_24 = _mm_unpackhi_epi16(stp2_23, stp2_24); \ + \ + stp1_0 = _mm_add_epi16(stp2_0, stp2_15); \ + stp1_1 = _mm_add_epi16(stp2_1, stp2_14); \ + stp1_2 = _mm_add_epi16(stp2_2, stp2_13); \ + stp1_3 = _mm_add_epi16(stp2_3, stp2_12); \ + stp1_4 = _mm_add_epi16(stp2_4, stp2_11); \ + stp1_5 = _mm_add_epi16(stp2_5, stp2_10); \ + stp1_6 = _mm_add_epi16(stp2_6, stp2_9); \ + stp1_7 = _mm_add_epi16(stp2_7, stp2_8); \ + stp1_8 = _mm_sub_epi16(stp2_7, stp2_8); \ + stp1_9 = _mm_sub_epi16(stp2_6, stp2_9); \ + stp1_10 = _mm_sub_epi16(stp2_5, stp2_10); \ + stp1_11 = _mm_sub_epi16(stp2_4, stp2_11); \ + stp1_12 = _mm_sub_epi16(stp2_3, stp2_12); \ + stp1_13 = _mm_sub_epi16(stp2_2, stp2_13); \ + stp1_14 = _mm_sub_epi16(stp2_1, stp2_14); \ + stp1_15 = _mm_sub_epi16(stp2_0, stp2_15); \ + \ + stp1_16 = stp2_16; \ + stp1_17 = stp2_17; \ + stp1_18 = stp2_18; \ + stp1_19 = stp2_19; \ + \ + MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg6_0, \ + stg4_0, stg6_0, stg4_0, stp1_20, stp1_27, stp1_21, \ + stp1_26) \ + MULTIPLICATION_AND_ADD(lo_22_25, hi_22_25, lo_23_24, hi_23_24, stg6_0, \ + stg4_0, stg6_0, stg4_0, stp1_22, stp1_25, stp1_23, \ + stp1_24) \ + \ + stp1_28 = stp2_28; \ + stp1_29 = stp2_29; \ + stp1_30 = stp2_30; \ + stp1_31 = stp2_31; \ + } + +#define IDCT32(in0, in1) \ + /* Stage1 */ \ + { \ + const __m128i lo_1_31 = _mm_unpacklo_epi16((in0)[1], (in1)[15]); \ + const __m128i hi_1_31 = _mm_unpackhi_epi16((in0)[1], (in1)[15]); \ + const __m128i lo_17_15 = _mm_unpacklo_epi16((in1)[1], (in0)[15]); \ + const __m128i hi_17_15 = _mm_unpackhi_epi16((in1)[1], (in0)[15]); \ + \ + const __m128i lo_9_23 = _mm_unpacklo_epi16((in0)[9], (in1)[7]); \ + const __m128i hi_9_23 = _mm_unpackhi_epi16((in0)[9], (in1)[7]); \ + const __m128i lo_25_7 = _mm_unpacklo_epi16((in1)[9], (in0)[7]); \ + const __m128i hi_25_7 = _mm_unpackhi_epi16((in1)[9], (in0)[7]); \ + \ + const __m128i lo_5_27 = _mm_unpacklo_epi16((in0)[5], (in1)[11]); \ + const __m128i hi_5_27 = _mm_unpackhi_epi16((in0)[5], (in1)[11]); \ + const __m128i lo_21_11 = _mm_unpacklo_epi16((in1)[5], (in0)[11]); \ + const __m128i hi_21_11 = _mm_unpackhi_epi16((in1)[5], (in0)[11]); \ + \ + const __m128i lo_13_19 = _mm_unpacklo_epi16((in0)[13], (in1)[3]); \ + const __m128i hi_13_19 = _mm_unpackhi_epi16((in0)[13], (in1)[3]); \ + const __m128i lo_29_3 = _mm_unpacklo_epi16((in1)[13], (in0)[3]); \ + const __m128i hi_29_3 = _mm_unpackhi_epi16((in1)[13], (in0)[3]); \ + \ + MULTIPLICATION_AND_ADD(lo_1_31, hi_1_31, lo_17_15, hi_17_15, stg1_0, \ + stg1_1, stg1_2, stg1_3, stp1_16, stp1_31, stp1_17, \ + stp1_30) \ + MULTIPLICATION_AND_ADD(lo_9_23, hi_9_23, lo_25_7, hi_25_7, stg1_4, stg1_5, \ + stg1_6, stg1_7, stp1_18, stp1_29, stp1_19, stp1_28) \ + MULTIPLICATION_AND_ADD(lo_5_27, hi_5_27, lo_21_11, hi_21_11, stg1_8, \ + stg1_9, stg1_10, stg1_11, stp1_20, stp1_27, \ + stp1_21, stp1_26) \ + MULTIPLICATION_AND_ADD(lo_13_19, hi_13_19, lo_29_3, hi_29_3, stg1_12, \ + stg1_13, stg1_14, stg1_15, stp1_22, stp1_25, \ + stp1_23, stp1_24) \ + } \ + \ + /* Stage2 */ \ + { \ + const __m128i lo_2_30 = _mm_unpacklo_epi16((in0)[2], (in1)[14]); \ + const __m128i hi_2_30 = _mm_unpackhi_epi16((in0)[2], (in1)[14]); \ + const __m128i lo_18_14 = _mm_unpacklo_epi16((in1)[2], (in0)[14]); \ + const __m128i hi_18_14 = _mm_unpackhi_epi16((in1)[2], (in0)[14]); \ + \ + const __m128i lo_10_22 = _mm_unpacklo_epi16((in0)[10], (in1)[6]); \ + const __m128i hi_10_22 = _mm_unpackhi_epi16((in0)[10], (in1)[6]); \ + const __m128i lo_26_6 = _mm_unpacklo_epi16((in1)[10], (in0)[6]); \ + const __m128i hi_26_6 = _mm_unpackhi_epi16((in1)[10], (in0)[6]); \ + \ + MULTIPLICATION_AND_ADD(lo_2_30, hi_2_30, lo_18_14, hi_18_14, stg2_0, \ + stg2_1, stg2_2, stg2_3, stp2_8, stp2_15, stp2_9, \ + stp2_14) \ + MULTIPLICATION_AND_ADD(lo_10_22, hi_10_22, lo_26_6, hi_26_6, stg2_4, \ + stg2_5, stg2_6, stg2_7, stp2_10, stp2_13, stp2_11, \ + stp2_12) \ + \ + stp2_16 = _mm_add_epi16(stp1_16, stp1_17); \ + stp2_17 = _mm_sub_epi16(stp1_16, stp1_17); \ + stp2_18 = _mm_sub_epi16(stp1_19, stp1_18); \ + stp2_19 = _mm_add_epi16(stp1_19, stp1_18); \ + \ + stp2_20 = _mm_add_epi16(stp1_20, stp1_21); \ + stp2_21 = _mm_sub_epi16(stp1_20, stp1_21); \ + stp2_22 = _mm_sub_epi16(stp1_23, stp1_22); \ + stp2_23 = _mm_add_epi16(stp1_23, stp1_22); \ + \ + stp2_24 = _mm_add_epi16(stp1_24, stp1_25); \ + stp2_25 = _mm_sub_epi16(stp1_24, stp1_25); \ + stp2_26 = _mm_sub_epi16(stp1_27, stp1_26); \ + stp2_27 = _mm_add_epi16(stp1_27, stp1_26); \ + \ + stp2_28 = _mm_add_epi16(stp1_28, stp1_29); \ + stp2_29 = _mm_sub_epi16(stp1_28, stp1_29); \ + stp2_30 = _mm_sub_epi16(stp1_31, stp1_30); \ + stp2_31 = _mm_add_epi16(stp1_31, stp1_30); \ + } \ + \ + /* Stage3 */ \ + { \ + const __m128i lo_4_28 = _mm_unpacklo_epi16((in0)[4], (in1)[12]); \ + const __m128i hi_4_28 = _mm_unpackhi_epi16((in0)[4], (in1)[12]); \ + const __m128i lo_20_12 = _mm_unpacklo_epi16((in1)[4], (in0)[12]); \ + const __m128i hi_20_12 = _mm_unpackhi_epi16((in1)[4], (in0)[12]); \ + \ + const __m128i lo_17_30 = _mm_unpacklo_epi16(stp2_17, stp2_30); \ + const __m128i hi_17_30 = _mm_unpackhi_epi16(stp2_17, stp2_30); \ + const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29); \ + const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29); \ + \ + const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); \ + const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); \ + const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25); \ + const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25); \ + \ + MULTIPLICATION_AND_ADD(lo_4_28, hi_4_28, lo_20_12, hi_20_12, stg3_0, \ + stg3_1, stg3_2, stg3_3, stp1_4, stp1_7, stp1_5, \ + stp1_6) \ + \ + stp1_8 = _mm_add_epi16(stp2_8, stp2_9); \ + stp1_9 = _mm_sub_epi16(stp2_8, stp2_9); \ + stp1_10 = _mm_sub_epi16(stp2_11, stp2_10); \ + stp1_11 = _mm_add_epi16(stp2_11, stp2_10); \ + stp1_12 = _mm_add_epi16(stp2_12, stp2_13); \ + stp1_13 = _mm_sub_epi16(stp2_12, stp2_13); \ + stp1_14 = _mm_sub_epi16(stp2_15, stp2_14); \ + stp1_15 = _mm_add_epi16(stp2_15, stp2_14); \ + \ + MULTIPLICATION_AND_ADD(lo_17_30, hi_17_30, lo_18_29, hi_18_29, stg3_4, \ + stg3_5, stg3_6, stg3_4, stp1_17, stp1_30, stp1_18, \ + stp1_29) \ + MULTIPLICATION_AND_ADD(lo_21_26, hi_21_26, lo_22_25, hi_22_25, stg3_8, \ + stg3_9, stg3_10, stg3_8, stp1_21, stp1_26, stp1_22, \ + stp1_25) \ + \ + stp1_16 = stp2_16; \ + stp1_31 = stp2_31; \ + stp1_19 = stp2_19; \ + stp1_20 = stp2_20; \ + stp1_23 = stp2_23; \ + stp1_24 = stp2_24; \ + stp1_27 = stp2_27; \ + stp1_28 = stp2_28; \ + } \ + \ + /* Stage4 */ \ + { \ + const __m128i lo_0_16 = _mm_unpacklo_epi16((in0)[0], (in1)[0]); \ + const __m128i hi_0_16 = _mm_unpackhi_epi16((in0)[0], (in1)[0]); \ + const __m128i lo_8_24 = _mm_unpacklo_epi16((in0)[8], (in1)[8]); \ + const __m128i hi_8_24 = _mm_unpackhi_epi16((in0)[8], (in1)[8]); \ + \ + const __m128i lo_9_14 = _mm_unpacklo_epi16(stp1_9, stp1_14); \ + const __m128i hi_9_14 = _mm_unpackhi_epi16(stp1_9, stp1_14); \ + const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); \ + const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); \ + \ + MULTIPLICATION_AND_ADD(lo_0_16, hi_0_16, lo_8_24, hi_8_24, stg4_0, stg4_1, \ + stg4_2, stg4_3, stp2_0, stp2_1, stp2_2, stp2_3) \ + \ + stp2_4 = _mm_add_epi16(stp1_4, stp1_5); \ + stp2_5 = _mm_sub_epi16(stp1_4, stp1_5); \ + stp2_6 = _mm_sub_epi16(stp1_7, stp1_6); \ + stp2_7 = _mm_add_epi16(stp1_7, stp1_6); \ + \ + MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, \ + stg4_5, stg4_6, stg4_4, stp2_9, stp2_14, stp2_10, \ + stp2_13) \ + \ + stp2_8 = stp1_8; \ + stp2_15 = stp1_15; \ + stp2_11 = stp1_11; \ + stp2_12 = stp1_12; \ + \ + stp2_16 = _mm_add_epi16(stp1_16, stp1_19); \ + stp2_17 = _mm_add_epi16(stp1_17, stp1_18); \ + stp2_18 = _mm_sub_epi16(stp1_17, stp1_18); \ + stp2_19 = _mm_sub_epi16(stp1_16, stp1_19); \ + stp2_20 = _mm_sub_epi16(stp1_23, stp1_20); \ + stp2_21 = _mm_sub_epi16(stp1_22, stp1_21); \ + stp2_22 = _mm_add_epi16(stp1_22, stp1_21); \ + stp2_23 = _mm_add_epi16(stp1_23, stp1_20); \ + \ + stp2_24 = _mm_add_epi16(stp1_24, stp1_27); \ + stp2_25 = _mm_add_epi16(stp1_25, stp1_26); \ + stp2_26 = _mm_sub_epi16(stp1_25, stp1_26); \ + stp2_27 = _mm_sub_epi16(stp1_24, stp1_27); \ + stp2_28 = _mm_sub_epi16(stp1_31, stp1_28); \ + stp2_29 = _mm_sub_epi16(stp1_30, stp1_29); \ + stp2_30 = _mm_add_epi16(stp1_29, stp1_30); \ + stp2_31 = _mm_add_epi16(stp1_28, stp1_31); \ + } \ + \ + /* Stage5 */ \ + { \ + const __m128i lo_6_5 = _mm_unpacklo_epi16(stp2_6, stp2_5); \ + const __m128i hi_6_5 = _mm_unpackhi_epi16(stp2_6, stp2_5); \ + const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29); \ + const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29); \ + \ + const __m128i lo_19_28 = _mm_unpacklo_epi16(stp2_19, stp2_28); \ + const __m128i hi_19_28 = _mm_unpackhi_epi16(stp2_19, stp2_28); \ + const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); \ + const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); \ + \ + const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); \ + const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); \ + \ + stp1_0 = _mm_add_epi16(stp2_0, stp2_3); \ + stp1_1 = _mm_add_epi16(stp2_1, stp2_2); \ + stp1_2 = _mm_sub_epi16(stp2_1, stp2_2); \ + stp1_3 = _mm_sub_epi16(stp2_0, stp2_3); \ + \ + tmp0 = _mm_madd_epi16(lo_6_5, stg4_1); \ + tmp1 = _mm_madd_epi16(hi_6_5, stg4_1); \ + tmp2 = _mm_madd_epi16(lo_6_5, stg4_0); \ + tmp3 = _mm_madd_epi16(hi_6_5, stg4_0); \ + \ + tmp0 = _mm_add_epi32(tmp0, rounding); \ + tmp1 = _mm_add_epi32(tmp1, rounding); \ + tmp2 = _mm_add_epi32(tmp2, rounding); \ + tmp3 = _mm_add_epi32(tmp3, rounding); \ + \ + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \ + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \ + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \ + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \ + \ + stp1_5 = _mm_packs_epi32(tmp0, tmp1); \ + stp1_6 = _mm_packs_epi32(tmp2, tmp3); \ + \ + stp1_4 = stp2_4; \ + stp1_7 = stp2_7; \ + \ + stp1_8 = _mm_add_epi16(stp2_8, stp2_11); \ + stp1_9 = _mm_add_epi16(stp2_9, stp2_10); \ + stp1_10 = _mm_sub_epi16(stp2_9, stp2_10); \ + stp1_11 = _mm_sub_epi16(stp2_8, stp2_11); \ + stp1_12 = _mm_sub_epi16(stp2_15, stp2_12); \ + stp1_13 = _mm_sub_epi16(stp2_14, stp2_13); \ + stp1_14 = _mm_add_epi16(stp2_14, stp2_13); \ + stp1_15 = _mm_add_epi16(stp2_15, stp2_12); \ + \ + stp1_16 = stp2_16; \ + stp1_17 = stp2_17; \ + \ + MULTIPLICATION_AND_ADD(lo_18_29, hi_18_29, lo_19_28, hi_19_28, stg4_4, \ + stg4_5, stg4_4, stg4_5, stp1_18, stp1_29, stp1_19, \ + stp1_28) \ + MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg4_6, \ + stg4_4, stg4_6, stg4_4, stp1_20, stp1_27, stp1_21, \ + stp1_26) \ + \ + stp1_22 = stp2_22; \ + stp1_23 = stp2_23; \ + stp1_24 = stp2_24; \ + stp1_25 = stp2_25; \ + stp1_30 = stp2_30; \ + stp1_31 = stp2_31; \ + } \ + \ + /* Stage6 */ \ + { \ + const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); \ + const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); \ + const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1_11, stp1_12); \ + const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1_11, stp1_12); \ + \ + stp2_0 = _mm_add_epi16(stp1_0, stp1_7); \ + stp2_1 = _mm_add_epi16(stp1_1, stp1_6); \ + stp2_2 = _mm_add_epi16(stp1_2, stp1_5); \ + stp2_3 = _mm_add_epi16(stp1_3, stp1_4); \ + stp2_4 = _mm_sub_epi16(stp1_3, stp1_4); \ + stp2_5 = _mm_sub_epi16(stp1_2, stp1_5); \ + stp2_6 = _mm_sub_epi16(stp1_1, stp1_6); \ + stp2_7 = _mm_sub_epi16(stp1_0, stp1_7); \ + \ + stp2_8 = stp1_8; \ + stp2_9 = stp1_9; \ + stp2_14 = stp1_14; \ + stp2_15 = stp1_15; \ + \ + MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0, \ + stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11, \ + stp2_12) \ + \ + stp2_16 = _mm_add_epi16(stp1_16, stp1_23); \ + stp2_17 = _mm_add_epi16(stp1_17, stp1_22); \ + stp2_18 = _mm_add_epi16(stp1_18, stp1_21); \ + stp2_19 = _mm_add_epi16(stp1_19, stp1_20); \ + stp2_20 = _mm_sub_epi16(stp1_19, stp1_20); \ + stp2_21 = _mm_sub_epi16(stp1_18, stp1_21); \ + stp2_22 = _mm_sub_epi16(stp1_17, stp1_22); \ + stp2_23 = _mm_sub_epi16(stp1_16, stp1_23); \ + \ + stp2_24 = _mm_sub_epi16(stp1_31, stp1_24); \ + stp2_25 = _mm_sub_epi16(stp1_30, stp1_25); \ + stp2_26 = _mm_sub_epi16(stp1_29, stp1_26); \ + stp2_27 = _mm_sub_epi16(stp1_28, stp1_27); \ + stp2_28 = _mm_add_epi16(stp1_27, stp1_28); \ + stp2_29 = _mm_add_epi16(stp1_26, stp1_29); \ + stp2_30 = _mm_add_epi16(stp1_25, stp1_30); \ + stp2_31 = _mm_add_epi16(stp1_24, stp1_31); \ + } \ + \ + /* Stage7 */ \ + { \ + const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); \ + const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); \ + const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); \ + const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); \ + \ + const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25); \ + const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25); \ + const __m128i lo_23_24 = _mm_unpacklo_epi16(stp2_23, stp2_24); \ + const __m128i hi_23_24 = _mm_unpackhi_epi16(stp2_23, stp2_24); \ + \ + stp1_0 = _mm_add_epi16(stp2_0, stp2_15); \ + stp1_1 = _mm_add_epi16(stp2_1, stp2_14); \ + stp1_2 = _mm_add_epi16(stp2_2, stp2_13); \ + stp1_3 = _mm_add_epi16(stp2_3, stp2_12); \ + stp1_4 = _mm_add_epi16(stp2_4, stp2_11); \ + stp1_5 = _mm_add_epi16(stp2_5, stp2_10); \ + stp1_6 = _mm_add_epi16(stp2_6, stp2_9); \ + stp1_7 = _mm_add_epi16(stp2_7, stp2_8); \ + stp1_8 = _mm_sub_epi16(stp2_7, stp2_8); \ + stp1_9 = _mm_sub_epi16(stp2_6, stp2_9); \ + stp1_10 = _mm_sub_epi16(stp2_5, stp2_10); \ + stp1_11 = _mm_sub_epi16(stp2_4, stp2_11); \ + stp1_12 = _mm_sub_epi16(stp2_3, stp2_12); \ + stp1_13 = _mm_sub_epi16(stp2_2, stp2_13); \ + stp1_14 = _mm_sub_epi16(stp2_1, stp2_14); \ + stp1_15 = _mm_sub_epi16(stp2_0, stp2_15); \ + \ + stp1_16 = stp2_16; \ + stp1_17 = stp2_17; \ + stp1_18 = stp2_18; \ + stp1_19 = stp2_19; \ + \ + MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg6_0, \ + stg4_0, stg6_0, stg4_0, stp1_20, stp1_27, stp1_21, \ + stp1_26) \ + MULTIPLICATION_AND_ADD(lo_22_25, hi_22_25, lo_23_24, hi_23_24, stg6_0, \ + stg4_0, stg6_0, stg4_0, stp1_22, stp1_25, stp1_23, \ + stp1_24) \ + \ + stp1_28 = stp2_28; \ + stp1_29 = stp2_29; \ + stp1_30 = stp2_30; \ + stp1_31 = stp2_31; \ + } + +// Only upper-left 8x8 has non-zero coeff +void aom_idct32x32_34_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride) { + const __m128i zero = _mm_setzero_si128(); + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i final_rounding = _mm_set1_epi16(1 << 5); + + // idct constants for each stage + const __m128i stg1_0 = pair_set_epi16(cospi_31_64, -cospi_1_64); + const __m128i stg1_1 = pair_set_epi16(cospi_1_64, cospi_31_64); + const __m128i stg1_6 = pair_set_epi16(cospi_7_64, -cospi_25_64); + const __m128i stg1_7 = pair_set_epi16(cospi_25_64, cospi_7_64); + const __m128i stg1_8 = pair_set_epi16(cospi_27_64, -cospi_5_64); + const __m128i stg1_9 = pair_set_epi16(cospi_5_64, cospi_27_64); + const __m128i stg1_14 = pair_set_epi16(cospi_3_64, -cospi_29_64); + const __m128i stg1_15 = pair_set_epi16(cospi_29_64, cospi_3_64); + + const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64); + const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64); + const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64); + const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64); + + const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64); + const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64); + + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + + __m128i in[32], col[32]; + __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7, + stp1_8, stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15, + stp1_16, stp1_17, stp1_18, stp1_19, stp1_20, stp1_21, stp1_22, stp1_23, + stp1_24, stp1_25, stp1_26, stp1_27, stp1_28, stp1_29, stp1_30, stp1_31; + __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7, + stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15, + stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23, + stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + int i; + + // Load input data. Only need to load the top left 8x8 block. + in[0] = load_input_data(input); + in[1] = load_input_data(input + 32); + in[2] = load_input_data(input + 64); + in[3] = load_input_data(input + 96); + in[4] = load_input_data(input + 128); + in[5] = load_input_data(input + 160); + in[6] = load_input_data(input + 192); + in[7] = load_input_data(input + 224); + + for (i = 8; i < 32; ++i) { + in[i] = _mm_setzero_si128(); + } + + array_transpose_8x8(in, in); + // TODO(hkuang): Following transposes are unnecessary. But remove them will + // lead to performance drop on some devices. + array_transpose_8x8(in + 8, in + 8); + array_transpose_8x8(in + 16, in + 16); + array_transpose_8x8(in + 24, in + 24); + + IDCT32_34 + + // 1_D: Store 32 intermediate results for each 8x32 block. + col[0] = _mm_add_epi16(stp1_0, stp1_31); + col[1] = _mm_add_epi16(stp1_1, stp1_30); + col[2] = _mm_add_epi16(stp1_2, stp1_29); + col[3] = _mm_add_epi16(stp1_3, stp1_28); + col[4] = _mm_add_epi16(stp1_4, stp1_27); + col[5] = _mm_add_epi16(stp1_5, stp1_26); + col[6] = _mm_add_epi16(stp1_6, stp1_25); + col[7] = _mm_add_epi16(stp1_7, stp1_24); + col[8] = _mm_add_epi16(stp1_8, stp1_23); + col[9] = _mm_add_epi16(stp1_9, stp1_22); + col[10] = _mm_add_epi16(stp1_10, stp1_21); + col[11] = _mm_add_epi16(stp1_11, stp1_20); + col[12] = _mm_add_epi16(stp1_12, stp1_19); + col[13] = _mm_add_epi16(stp1_13, stp1_18); + col[14] = _mm_add_epi16(stp1_14, stp1_17); + col[15] = _mm_add_epi16(stp1_15, stp1_16); + col[16] = _mm_sub_epi16(stp1_15, stp1_16); + col[17] = _mm_sub_epi16(stp1_14, stp1_17); + col[18] = _mm_sub_epi16(stp1_13, stp1_18); + col[19] = _mm_sub_epi16(stp1_12, stp1_19); + col[20] = _mm_sub_epi16(stp1_11, stp1_20); + col[21] = _mm_sub_epi16(stp1_10, stp1_21); + col[22] = _mm_sub_epi16(stp1_9, stp1_22); + col[23] = _mm_sub_epi16(stp1_8, stp1_23); + col[24] = _mm_sub_epi16(stp1_7, stp1_24); + col[25] = _mm_sub_epi16(stp1_6, stp1_25); + col[26] = _mm_sub_epi16(stp1_5, stp1_26); + col[27] = _mm_sub_epi16(stp1_4, stp1_27); + col[28] = _mm_sub_epi16(stp1_3, stp1_28); + col[29] = _mm_sub_epi16(stp1_2, stp1_29); + col[30] = _mm_sub_epi16(stp1_1, stp1_30); + col[31] = _mm_sub_epi16(stp1_0, stp1_31); + for (i = 0; i < 4; i++) { + int j; + // Transpose 32x8 block to 8x32 block + array_transpose_8x8(col + i * 8, in); + IDCT32_34 + + // 2_D: Calculate the results and store them to destination. + in[0] = _mm_add_epi16(stp1_0, stp1_31); + in[1] = _mm_add_epi16(stp1_1, stp1_30); + in[2] = _mm_add_epi16(stp1_2, stp1_29); + in[3] = _mm_add_epi16(stp1_3, stp1_28); + in[4] = _mm_add_epi16(stp1_4, stp1_27); + in[5] = _mm_add_epi16(stp1_5, stp1_26); + in[6] = _mm_add_epi16(stp1_6, stp1_25); + in[7] = _mm_add_epi16(stp1_7, stp1_24); + in[8] = _mm_add_epi16(stp1_8, stp1_23); + in[9] = _mm_add_epi16(stp1_9, stp1_22); + in[10] = _mm_add_epi16(stp1_10, stp1_21); + in[11] = _mm_add_epi16(stp1_11, stp1_20); + in[12] = _mm_add_epi16(stp1_12, stp1_19); + in[13] = _mm_add_epi16(stp1_13, stp1_18); + in[14] = _mm_add_epi16(stp1_14, stp1_17); + in[15] = _mm_add_epi16(stp1_15, stp1_16); + in[16] = _mm_sub_epi16(stp1_15, stp1_16); + in[17] = _mm_sub_epi16(stp1_14, stp1_17); + in[18] = _mm_sub_epi16(stp1_13, stp1_18); + in[19] = _mm_sub_epi16(stp1_12, stp1_19); + in[20] = _mm_sub_epi16(stp1_11, stp1_20); + in[21] = _mm_sub_epi16(stp1_10, stp1_21); + in[22] = _mm_sub_epi16(stp1_9, stp1_22); + in[23] = _mm_sub_epi16(stp1_8, stp1_23); + in[24] = _mm_sub_epi16(stp1_7, stp1_24); + in[25] = _mm_sub_epi16(stp1_6, stp1_25); + in[26] = _mm_sub_epi16(stp1_5, stp1_26); + in[27] = _mm_sub_epi16(stp1_4, stp1_27); + in[28] = _mm_sub_epi16(stp1_3, stp1_28); + in[29] = _mm_sub_epi16(stp1_2, stp1_29); + in[30] = _mm_sub_epi16(stp1_1, stp1_30); + in[31] = _mm_sub_epi16(stp1_0, stp1_31); + + for (j = 0; j < 32; ++j) { + // Final rounding and shift + in[j] = _mm_adds_epi16(in[j], final_rounding); + in[j] = _mm_srai_epi16(in[j], 6); + RECON_AND_STORE(dest + j * stride, in[j]); + } + + dest += 8; + } +} + +void aom_idct32x32_1024_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride) { + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i final_rounding = _mm_set1_epi16(1 << 5); + const __m128i zero = _mm_setzero_si128(); + + // idct constants for each stage + const __m128i stg1_0 = pair_set_epi16(cospi_31_64, -cospi_1_64); + const __m128i stg1_1 = pair_set_epi16(cospi_1_64, cospi_31_64); + const __m128i stg1_2 = pair_set_epi16(cospi_15_64, -cospi_17_64); + const __m128i stg1_3 = pair_set_epi16(cospi_17_64, cospi_15_64); + const __m128i stg1_4 = pair_set_epi16(cospi_23_64, -cospi_9_64); + const __m128i stg1_5 = pair_set_epi16(cospi_9_64, cospi_23_64); + const __m128i stg1_6 = pair_set_epi16(cospi_7_64, -cospi_25_64); + const __m128i stg1_7 = pair_set_epi16(cospi_25_64, cospi_7_64); + const __m128i stg1_8 = pair_set_epi16(cospi_27_64, -cospi_5_64); + const __m128i stg1_9 = pair_set_epi16(cospi_5_64, cospi_27_64); + const __m128i stg1_10 = pair_set_epi16(cospi_11_64, -cospi_21_64); + const __m128i stg1_11 = pair_set_epi16(cospi_21_64, cospi_11_64); + const __m128i stg1_12 = pair_set_epi16(cospi_19_64, -cospi_13_64); + const __m128i stg1_13 = pair_set_epi16(cospi_13_64, cospi_19_64); + const __m128i stg1_14 = pair_set_epi16(cospi_3_64, -cospi_29_64); + const __m128i stg1_15 = pair_set_epi16(cospi_29_64, cospi_3_64); + + const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64); + const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64); + const __m128i stg2_2 = pair_set_epi16(cospi_14_64, -cospi_18_64); + const __m128i stg2_3 = pair_set_epi16(cospi_18_64, cospi_14_64); + const __m128i stg2_4 = pair_set_epi16(cospi_22_64, -cospi_10_64); + const __m128i stg2_5 = pair_set_epi16(cospi_10_64, cospi_22_64); + const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64); + const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64); + + const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i stg3_2 = pair_set_epi16(cospi_12_64, -cospi_20_64); + const __m128i stg3_3 = pair_set_epi16(cospi_20_64, cospi_12_64); + const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64); + const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64); + + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg4_2 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i stg4_3 = pair_set_epi16(cospi_8_64, cospi_24_64); + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + + __m128i in[32], col[128], zero_idx[16]; + __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7, + stp1_8, stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15, + stp1_16, stp1_17, stp1_18, stp1_19, stp1_20, stp1_21, stp1_22, stp1_23, + stp1_24, stp1_25, stp1_26, stp1_27, stp1_28, stp1_29, stp1_30, stp1_31; + __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7, + stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15, + stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23, + stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + int i, j, i32; + + for (i = 0; i < 4; i++) { + i32 = (i << 5); + // First 1-D idct + // Load input data. + LOAD_DQCOEFF(in[0], input); + LOAD_DQCOEFF(in[8], input); + LOAD_DQCOEFF(in[16], input); + LOAD_DQCOEFF(in[24], input); + LOAD_DQCOEFF(in[1], input); + LOAD_DQCOEFF(in[9], input); + LOAD_DQCOEFF(in[17], input); + LOAD_DQCOEFF(in[25], input); + LOAD_DQCOEFF(in[2], input); + LOAD_DQCOEFF(in[10], input); + LOAD_DQCOEFF(in[18], input); + LOAD_DQCOEFF(in[26], input); + LOAD_DQCOEFF(in[3], input); + LOAD_DQCOEFF(in[11], input); + LOAD_DQCOEFF(in[19], input); + LOAD_DQCOEFF(in[27], input); + + LOAD_DQCOEFF(in[4], input); + LOAD_DQCOEFF(in[12], input); + LOAD_DQCOEFF(in[20], input); + LOAD_DQCOEFF(in[28], input); + LOAD_DQCOEFF(in[5], input); + LOAD_DQCOEFF(in[13], input); + LOAD_DQCOEFF(in[21], input); + LOAD_DQCOEFF(in[29], input); + LOAD_DQCOEFF(in[6], input); + LOAD_DQCOEFF(in[14], input); + LOAD_DQCOEFF(in[22], input); + LOAD_DQCOEFF(in[30], input); + LOAD_DQCOEFF(in[7], input); + LOAD_DQCOEFF(in[15], input); + LOAD_DQCOEFF(in[23], input); + LOAD_DQCOEFF(in[31], input); + + // checking if all entries are zero + zero_idx[0] = _mm_or_si128(in[0], in[1]); + zero_idx[1] = _mm_or_si128(in[2], in[3]); + zero_idx[2] = _mm_or_si128(in[4], in[5]); + zero_idx[3] = _mm_or_si128(in[6], in[7]); + zero_idx[4] = _mm_or_si128(in[8], in[9]); + zero_idx[5] = _mm_or_si128(in[10], in[11]); + zero_idx[6] = _mm_or_si128(in[12], in[13]); + zero_idx[7] = _mm_or_si128(in[14], in[15]); + zero_idx[8] = _mm_or_si128(in[16], in[17]); + zero_idx[9] = _mm_or_si128(in[18], in[19]); + zero_idx[10] = _mm_or_si128(in[20], in[21]); + zero_idx[11] = _mm_or_si128(in[22], in[23]); + zero_idx[12] = _mm_or_si128(in[24], in[25]); + zero_idx[13] = _mm_or_si128(in[26], in[27]); + zero_idx[14] = _mm_or_si128(in[28], in[29]); + zero_idx[15] = _mm_or_si128(in[30], in[31]); + + zero_idx[0] = _mm_or_si128(zero_idx[0], zero_idx[1]); + zero_idx[1] = _mm_or_si128(zero_idx[2], zero_idx[3]); + zero_idx[2] = _mm_or_si128(zero_idx[4], zero_idx[5]); + zero_idx[3] = _mm_or_si128(zero_idx[6], zero_idx[7]); + zero_idx[4] = _mm_or_si128(zero_idx[8], zero_idx[9]); + zero_idx[5] = _mm_or_si128(zero_idx[10], zero_idx[11]); + zero_idx[6] = _mm_or_si128(zero_idx[12], zero_idx[13]); + zero_idx[7] = _mm_or_si128(zero_idx[14], zero_idx[15]); + + zero_idx[8] = _mm_or_si128(zero_idx[0], zero_idx[1]); + zero_idx[9] = _mm_or_si128(zero_idx[2], zero_idx[3]); + zero_idx[10] = _mm_or_si128(zero_idx[4], zero_idx[5]); + zero_idx[11] = _mm_or_si128(zero_idx[6], zero_idx[7]); + zero_idx[12] = _mm_or_si128(zero_idx[8], zero_idx[9]); + zero_idx[13] = _mm_or_si128(zero_idx[10], zero_idx[11]); + zero_idx[14] = _mm_or_si128(zero_idx[12], zero_idx[13]); + + if (_mm_movemask_epi8(_mm_cmpeq_epi32(zero_idx[14], zero)) == 0xFFFF) { + col[i32 + 0] = _mm_setzero_si128(); + col[i32 + 1] = _mm_setzero_si128(); + col[i32 + 2] = _mm_setzero_si128(); + col[i32 + 3] = _mm_setzero_si128(); + col[i32 + 4] = _mm_setzero_si128(); + col[i32 + 5] = _mm_setzero_si128(); + col[i32 + 6] = _mm_setzero_si128(); + col[i32 + 7] = _mm_setzero_si128(); + col[i32 + 8] = _mm_setzero_si128(); + col[i32 + 9] = _mm_setzero_si128(); + col[i32 + 10] = _mm_setzero_si128(); + col[i32 + 11] = _mm_setzero_si128(); + col[i32 + 12] = _mm_setzero_si128(); + col[i32 + 13] = _mm_setzero_si128(); + col[i32 + 14] = _mm_setzero_si128(); + col[i32 + 15] = _mm_setzero_si128(); + col[i32 + 16] = _mm_setzero_si128(); + col[i32 + 17] = _mm_setzero_si128(); + col[i32 + 18] = _mm_setzero_si128(); + col[i32 + 19] = _mm_setzero_si128(); + col[i32 + 20] = _mm_setzero_si128(); + col[i32 + 21] = _mm_setzero_si128(); + col[i32 + 22] = _mm_setzero_si128(); + col[i32 + 23] = _mm_setzero_si128(); + col[i32 + 24] = _mm_setzero_si128(); + col[i32 + 25] = _mm_setzero_si128(); + col[i32 + 26] = _mm_setzero_si128(); + col[i32 + 27] = _mm_setzero_si128(); + col[i32 + 28] = _mm_setzero_si128(); + col[i32 + 29] = _mm_setzero_si128(); + col[i32 + 30] = _mm_setzero_si128(); + col[i32 + 31] = _mm_setzero_si128(); + continue; + } + + // Transpose 32x8 block to 8x32 block + array_transpose_8x8(in, in); + array_transpose_8x8(in + 8, in + 8); + array_transpose_8x8(in + 16, in + 16); + array_transpose_8x8(in + 24, in + 24); + + IDCT32(in, in + 16) + + // 1_D: Store 32 intermediate results for each 8x32 block. + col[i32 + 0] = _mm_add_epi16(stp1_0, stp1_31); + col[i32 + 1] = _mm_add_epi16(stp1_1, stp1_30); + col[i32 + 2] = _mm_add_epi16(stp1_2, stp1_29); + col[i32 + 3] = _mm_add_epi16(stp1_3, stp1_28); + col[i32 + 4] = _mm_add_epi16(stp1_4, stp1_27); + col[i32 + 5] = _mm_add_epi16(stp1_5, stp1_26); + col[i32 + 6] = _mm_add_epi16(stp1_6, stp1_25); + col[i32 + 7] = _mm_add_epi16(stp1_7, stp1_24); + col[i32 + 8] = _mm_add_epi16(stp1_8, stp1_23); + col[i32 + 9] = _mm_add_epi16(stp1_9, stp1_22); + col[i32 + 10] = _mm_add_epi16(stp1_10, stp1_21); + col[i32 + 11] = _mm_add_epi16(stp1_11, stp1_20); + col[i32 + 12] = _mm_add_epi16(stp1_12, stp1_19); + col[i32 + 13] = _mm_add_epi16(stp1_13, stp1_18); + col[i32 + 14] = _mm_add_epi16(stp1_14, stp1_17); + col[i32 + 15] = _mm_add_epi16(stp1_15, stp1_16); + col[i32 + 16] = _mm_sub_epi16(stp1_15, stp1_16); + col[i32 + 17] = _mm_sub_epi16(stp1_14, stp1_17); + col[i32 + 18] = _mm_sub_epi16(stp1_13, stp1_18); + col[i32 + 19] = _mm_sub_epi16(stp1_12, stp1_19); + col[i32 + 20] = _mm_sub_epi16(stp1_11, stp1_20); + col[i32 + 21] = _mm_sub_epi16(stp1_10, stp1_21); + col[i32 + 22] = _mm_sub_epi16(stp1_9, stp1_22); + col[i32 + 23] = _mm_sub_epi16(stp1_8, stp1_23); + col[i32 + 24] = _mm_sub_epi16(stp1_7, stp1_24); + col[i32 + 25] = _mm_sub_epi16(stp1_6, stp1_25); + col[i32 + 26] = _mm_sub_epi16(stp1_5, stp1_26); + col[i32 + 27] = _mm_sub_epi16(stp1_4, stp1_27); + col[i32 + 28] = _mm_sub_epi16(stp1_3, stp1_28); + col[i32 + 29] = _mm_sub_epi16(stp1_2, stp1_29); + col[i32 + 30] = _mm_sub_epi16(stp1_1, stp1_30); + col[i32 + 31] = _mm_sub_epi16(stp1_0, stp1_31); + } + for (i = 0; i < 4; i++) { + // Second 1-D idct + j = i << 3; + + // Transpose 32x8 block to 8x32 block + array_transpose_8x8(col + j, in); + array_transpose_8x8(col + j + 32, in + 8); + array_transpose_8x8(col + j + 64, in + 16); + array_transpose_8x8(col + j + 96, in + 24); + + IDCT32(in, in + 16) + + // 2_D: Calculate the results and store them to destination. + in[0] = _mm_add_epi16(stp1_0, stp1_31); + in[1] = _mm_add_epi16(stp1_1, stp1_30); + in[2] = _mm_add_epi16(stp1_2, stp1_29); + in[3] = _mm_add_epi16(stp1_3, stp1_28); + in[4] = _mm_add_epi16(stp1_4, stp1_27); + in[5] = _mm_add_epi16(stp1_5, stp1_26); + in[6] = _mm_add_epi16(stp1_6, stp1_25); + in[7] = _mm_add_epi16(stp1_7, stp1_24); + in[8] = _mm_add_epi16(stp1_8, stp1_23); + in[9] = _mm_add_epi16(stp1_9, stp1_22); + in[10] = _mm_add_epi16(stp1_10, stp1_21); + in[11] = _mm_add_epi16(stp1_11, stp1_20); + in[12] = _mm_add_epi16(stp1_12, stp1_19); + in[13] = _mm_add_epi16(stp1_13, stp1_18); + in[14] = _mm_add_epi16(stp1_14, stp1_17); + in[15] = _mm_add_epi16(stp1_15, stp1_16); + in[16] = _mm_sub_epi16(stp1_15, stp1_16); + in[17] = _mm_sub_epi16(stp1_14, stp1_17); + in[18] = _mm_sub_epi16(stp1_13, stp1_18); + in[19] = _mm_sub_epi16(stp1_12, stp1_19); + in[20] = _mm_sub_epi16(stp1_11, stp1_20); + in[21] = _mm_sub_epi16(stp1_10, stp1_21); + in[22] = _mm_sub_epi16(stp1_9, stp1_22); + in[23] = _mm_sub_epi16(stp1_8, stp1_23); + in[24] = _mm_sub_epi16(stp1_7, stp1_24); + in[25] = _mm_sub_epi16(stp1_6, stp1_25); + in[26] = _mm_sub_epi16(stp1_5, stp1_26); + in[27] = _mm_sub_epi16(stp1_4, stp1_27); + in[28] = _mm_sub_epi16(stp1_3, stp1_28); + in[29] = _mm_sub_epi16(stp1_2, stp1_29); + in[30] = _mm_sub_epi16(stp1_1, stp1_30); + in[31] = _mm_sub_epi16(stp1_0, stp1_31); + + for (j = 0; j < 32; ++j) { + // Final rounding and shift + in[j] = _mm_adds_epi16(in[j], final_rounding); + in[j] = _mm_srai_epi16(in[j], 6); + RECON_AND_STORE(dest + j * stride, in[j]); + } + + dest += 8; + } +} + +void aom_idct32x32_1_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride) { + __m128i dc_value; + const __m128i zero = _mm_setzero_si128(); + int a, j; + + a = (int)dct_const_round_shift(input[0] * cospi_16_64); + a = (int)dct_const_round_shift(a * cospi_16_64); + a = ROUND_POWER_OF_TWO(a, 6); + + if (a == 0) return; + + dc_value = _mm_set1_epi16(a); + + for (j = 0; j < 32; ++j) { + RECON_AND_STORE(dest + 0 + j * stride, dc_value); + RECON_AND_STORE(dest + 8 + j * stride, dc_value); + RECON_AND_STORE(dest + 16 + j * stride, dc_value); + RECON_AND_STORE(dest + 24 + j * stride, dc_value); + } +} + +// Apply a 32-element IDCT to 8 columns. This does not do any transposition +// of its input - the caller is expected to have done that. +// The input buffers are the top and bottom halves of an 8x32 block. +void idct32_8col(__m128i *in0, __m128i *in1) { + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + + // idct constants for each stage + const __m128i stg1_0 = pair_set_epi16(cospi_31_64, -cospi_1_64); + const __m128i stg1_1 = pair_set_epi16(cospi_1_64, cospi_31_64); + const __m128i stg1_2 = pair_set_epi16(cospi_15_64, -cospi_17_64); + const __m128i stg1_3 = pair_set_epi16(cospi_17_64, cospi_15_64); + const __m128i stg1_4 = pair_set_epi16(cospi_23_64, -cospi_9_64); + const __m128i stg1_5 = pair_set_epi16(cospi_9_64, cospi_23_64); + const __m128i stg1_6 = pair_set_epi16(cospi_7_64, -cospi_25_64); + const __m128i stg1_7 = pair_set_epi16(cospi_25_64, cospi_7_64); + const __m128i stg1_8 = pair_set_epi16(cospi_27_64, -cospi_5_64); + const __m128i stg1_9 = pair_set_epi16(cospi_5_64, cospi_27_64); + const __m128i stg1_10 = pair_set_epi16(cospi_11_64, -cospi_21_64); + const __m128i stg1_11 = pair_set_epi16(cospi_21_64, cospi_11_64); + const __m128i stg1_12 = pair_set_epi16(cospi_19_64, -cospi_13_64); + const __m128i stg1_13 = pair_set_epi16(cospi_13_64, cospi_19_64); + const __m128i stg1_14 = pair_set_epi16(cospi_3_64, -cospi_29_64); + const __m128i stg1_15 = pair_set_epi16(cospi_29_64, cospi_3_64); + + const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64); + const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64); + const __m128i stg2_2 = pair_set_epi16(cospi_14_64, -cospi_18_64); + const __m128i stg2_3 = pair_set_epi16(cospi_18_64, cospi_14_64); + const __m128i stg2_4 = pair_set_epi16(cospi_22_64, -cospi_10_64); + const __m128i stg2_5 = pair_set_epi16(cospi_10_64, cospi_22_64); + const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64); + const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64); + + const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i stg3_2 = pair_set_epi16(cospi_12_64, -cospi_20_64); + const __m128i stg3_3 = pair_set_epi16(cospi_20_64, cospi_12_64); + const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64); + const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64); + + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg4_2 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i stg4_3 = pair_set_epi16(cospi_8_64, cospi_24_64); + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + + __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7, + stp1_8, stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15, + stp1_16, stp1_17, stp1_18, stp1_19, stp1_20, stp1_21, stp1_22, stp1_23, + stp1_24, stp1_25, stp1_26, stp1_27, stp1_28, stp1_29, stp1_30, stp1_31; + __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7, + stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15, + stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23, + stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + IDCT32(in0, in1) + + // 2_D: Calculate the results and store them to destination. + in0[0] = _mm_add_epi16(stp1_0, stp1_31); + in0[1] = _mm_add_epi16(stp1_1, stp1_30); + in0[2] = _mm_add_epi16(stp1_2, stp1_29); + in0[3] = _mm_add_epi16(stp1_3, stp1_28); + in0[4] = _mm_add_epi16(stp1_4, stp1_27); + in0[5] = _mm_add_epi16(stp1_5, stp1_26); + in0[6] = _mm_add_epi16(stp1_6, stp1_25); + in0[7] = _mm_add_epi16(stp1_7, stp1_24); + in0[8] = _mm_add_epi16(stp1_8, stp1_23); + in0[9] = _mm_add_epi16(stp1_9, stp1_22); + in0[10] = _mm_add_epi16(stp1_10, stp1_21); + in0[11] = _mm_add_epi16(stp1_11, stp1_20); + in0[12] = _mm_add_epi16(stp1_12, stp1_19); + in0[13] = _mm_add_epi16(stp1_13, stp1_18); + in0[14] = _mm_add_epi16(stp1_14, stp1_17); + in0[15] = _mm_add_epi16(stp1_15, stp1_16); + in1[0] = _mm_sub_epi16(stp1_15, stp1_16); + in1[1] = _mm_sub_epi16(stp1_14, stp1_17); + in1[2] = _mm_sub_epi16(stp1_13, stp1_18); + in1[3] = _mm_sub_epi16(stp1_12, stp1_19); + in1[4] = _mm_sub_epi16(stp1_11, stp1_20); + in1[5] = _mm_sub_epi16(stp1_10, stp1_21); + in1[6] = _mm_sub_epi16(stp1_9, stp1_22); + in1[7] = _mm_sub_epi16(stp1_8, stp1_23); + in1[8] = _mm_sub_epi16(stp1_7, stp1_24); + in1[9] = _mm_sub_epi16(stp1_6, stp1_25); + in1[10] = _mm_sub_epi16(stp1_5, stp1_26); + in1[11] = _mm_sub_epi16(stp1_4, stp1_27); + in1[12] = _mm_sub_epi16(stp1_3, stp1_28); + in1[13] = _mm_sub_epi16(stp1_2, stp1_29); + in1[14] = _mm_sub_epi16(stp1_1, stp1_30); + in1[15] = _mm_sub_epi16(stp1_0, stp1_31); +} + +#if CONFIG_HIGHBITDEPTH +static INLINE __m128i clamp_high_sse2(__m128i value, int bd) { + __m128i ubounded, retval; + const __m128i zero = _mm_set1_epi16(0); + const __m128i one = _mm_set1_epi16(1); + const __m128i max = _mm_subs_epi16(_mm_slli_epi16(one, bd), one); + ubounded = _mm_cmpgt_epi16(value, max); + retval = _mm_andnot_si128(ubounded, value); + ubounded = _mm_and_si128(ubounded, max); + retval = _mm_or_si128(retval, ubounded); + retval = _mm_and_si128(retval, _mm_cmpgt_epi16(retval, zero)); + return retval; +} + +void aom_highbd_idct4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest8, + int stride, int bd) { + tran_low_t out[4 * 4]; + tran_low_t *outptr = out; + int i, j; + __m128i inptr[4]; + __m128i sign_bits[2]; + __m128i temp_mm, min_input, max_input; + int test; + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + int optimised_cols = 0; + const __m128i zero = _mm_set1_epi16(0); + const __m128i eight = _mm_set1_epi16(8); + const __m128i max = _mm_set1_epi16(12043); + const __m128i min = _mm_set1_epi16(-12043); + // Load input into __m128i + inptr[0] = _mm_loadu_si128((const __m128i *)input); + inptr[1] = _mm_loadu_si128((const __m128i *)(input + 4)); + inptr[2] = _mm_loadu_si128((const __m128i *)(input + 8)); + inptr[3] = _mm_loadu_si128((const __m128i *)(input + 12)); + + // Pack to 16 bits + inptr[0] = _mm_packs_epi32(inptr[0], inptr[1]); + inptr[1] = _mm_packs_epi32(inptr[2], inptr[3]); + + max_input = _mm_max_epi16(inptr[0], inptr[1]); + min_input = _mm_min_epi16(inptr[0], inptr[1]); + max_input = _mm_cmpgt_epi16(max_input, max); + min_input = _mm_cmplt_epi16(min_input, min); + temp_mm = _mm_or_si128(max_input, min_input); + test = _mm_movemask_epi8(temp_mm); + + if (!test) { + // Do the row transform + aom_idct4_sse2(inptr); + + // Check the min & max values + max_input = _mm_max_epi16(inptr[0], inptr[1]); + min_input = _mm_min_epi16(inptr[0], inptr[1]); + max_input = _mm_cmpgt_epi16(max_input, max); + min_input = _mm_cmplt_epi16(min_input, min); + temp_mm = _mm_or_si128(max_input, min_input); + test = _mm_movemask_epi8(temp_mm); + + if (test) { + array_transpose_4x4(inptr); + sign_bits[0] = _mm_cmplt_epi16(inptr[0], zero); + sign_bits[1] = _mm_cmplt_epi16(inptr[1], zero); + inptr[3] = _mm_unpackhi_epi16(inptr[1], sign_bits[1]); + inptr[2] = _mm_unpacklo_epi16(inptr[1], sign_bits[1]); + inptr[1] = _mm_unpackhi_epi16(inptr[0], sign_bits[0]); + inptr[0] = _mm_unpacklo_epi16(inptr[0], sign_bits[0]); + _mm_storeu_si128((__m128i *)outptr, inptr[0]); + _mm_storeu_si128((__m128i *)(outptr + 4), inptr[1]); + _mm_storeu_si128((__m128i *)(outptr + 8), inptr[2]); + _mm_storeu_si128((__m128i *)(outptr + 12), inptr[3]); + } else { + // Set to use the optimised transform for the column + optimised_cols = 1; + } + } else { + // Run the un-optimised row transform + for (i = 0; i < 4; ++i) { + aom_highbd_idct4_c(input, outptr, bd); + input += 4; + outptr += 4; + } + } + + if (optimised_cols) { + aom_idct4_sse2(inptr); + + // Final round and shift + inptr[0] = _mm_add_epi16(inptr[0], eight); + inptr[1] = _mm_add_epi16(inptr[1], eight); + + inptr[0] = _mm_srai_epi16(inptr[0], 4); + inptr[1] = _mm_srai_epi16(inptr[1], 4); + + // Reconstruction and Store + { + __m128i d0 = _mm_loadl_epi64((const __m128i *)dest); + __m128i d2 = _mm_loadl_epi64((const __m128i *)(dest + stride * 2)); + d0 = _mm_unpacklo_epi64( + d0, _mm_loadl_epi64((const __m128i *)(dest + stride))); + d2 = _mm_unpacklo_epi64( + d2, _mm_loadl_epi64((const __m128i *)(dest + stride * 3))); + d0 = clamp_high_sse2(_mm_adds_epi16(d0, inptr[0]), bd); + d2 = clamp_high_sse2(_mm_adds_epi16(d2, inptr[1]), bd); + // store input0 + _mm_storel_epi64((__m128i *)dest, d0); + // store input1 + d0 = _mm_srli_si128(d0, 8); + _mm_storel_epi64((__m128i *)(dest + stride), d0); + // store input2 + _mm_storel_epi64((__m128i *)(dest + stride * 2), d2); + // store input3 + d2 = _mm_srli_si128(d2, 8); + _mm_storel_epi64((__m128i *)(dest + stride * 3), d2); + } + } else { + // Run the un-optimised column transform + tran_low_t temp_in[4], temp_out[4]; + // Columns + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) temp_in[j] = out[j * 4 + i]; + aom_highbd_idct4_c(temp_in, temp_out, bd); + for (j = 0; j < 4; ++j) { + dest[j * stride + i] = highbd_clip_pixel_add( + dest[j * stride + i], ROUND_POWER_OF_TWO(temp_out[j], 4), bd); + } + } + } +} + +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/x86/inv_txfm_sse2.h b/third_party/aom/aom_dsp/x86/inv_txfm_sse2.h new file mode 100644 index 0000000000..95d246c3c5 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/inv_txfm_sse2.h @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_X86_INV_TXFM_SSE2_H_ +#define AOM_DSP_X86_INV_TXFM_SSE2_H_ + +#include // SSE2 +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/inv_txfm.h" +#include "aom_dsp/x86/txfm_common_sse2.h" + +// perform 8x8 transpose +static INLINE void array_transpose_4x4(__m128i *res) { + const __m128i tr0_0 = _mm_unpacklo_epi16(res[0], res[1]); + const __m128i tr0_1 = _mm_unpackhi_epi16(res[0], res[1]); + + res[0] = _mm_unpacklo_epi16(tr0_0, tr0_1); + res[1] = _mm_unpackhi_epi16(tr0_0, tr0_1); +} + +static INLINE void array_transpose_8x8(__m128i *in, __m128i *res) { + const __m128i tr0_0 = _mm_unpacklo_epi16(in[0], in[1]); + const __m128i tr0_1 = _mm_unpacklo_epi16(in[2], in[3]); + const __m128i tr0_2 = _mm_unpackhi_epi16(in[0], in[1]); + const __m128i tr0_3 = _mm_unpackhi_epi16(in[2], in[3]); + const __m128i tr0_4 = _mm_unpacklo_epi16(in[4], in[5]); + const __m128i tr0_5 = _mm_unpacklo_epi16(in[6], in[7]); + const __m128i tr0_6 = _mm_unpackhi_epi16(in[4], in[5]); + const __m128i tr0_7 = _mm_unpackhi_epi16(in[6], in[7]); + + const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); + const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_4, tr0_5); + const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); + const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_4, tr0_5); + const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_2, tr0_3); + const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); + const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_2, tr0_3); + const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); + + res[0] = _mm_unpacklo_epi64(tr1_0, tr1_1); + res[1] = _mm_unpackhi_epi64(tr1_0, tr1_1); + res[2] = _mm_unpacklo_epi64(tr1_2, tr1_3); + res[3] = _mm_unpackhi_epi64(tr1_2, tr1_3); + res[4] = _mm_unpacklo_epi64(tr1_4, tr1_5); + res[5] = _mm_unpackhi_epi64(tr1_4, tr1_5); + res[6] = _mm_unpacklo_epi64(tr1_6, tr1_7); + res[7] = _mm_unpackhi_epi64(tr1_6, tr1_7); +} + +#define TRANSPOSE_8X8(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3, out4, out5, out6, out7) \ + { \ + const __m128i tr0_0 = _mm_unpacklo_epi16(in0, in1); \ + const __m128i tr0_1 = _mm_unpacklo_epi16(in2, in3); \ + const __m128i tr0_2 = _mm_unpackhi_epi16(in0, in1); \ + const __m128i tr0_3 = _mm_unpackhi_epi16(in2, in3); \ + const __m128i tr0_4 = _mm_unpacklo_epi16(in4, in5); \ + const __m128i tr0_5 = _mm_unpacklo_epi16(in6, in7); \ + const __m128i tr0_6 = _mm_unpackhi_epi16(in4, in5); \ + const __m128i tr0_7 = _mm_unpackhi_epi16(in6, in7); \ + \ + const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); \ + const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3); \ + const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); \ + const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3); \ + const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); \ + const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); \ + const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); \ + const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); \ + \ + out0 = _mm_unpacklo_epi64(tr1_0, tr1_4); \ + out1 = _mm_unpackhi_epi64(tr1_0, tr1_4); \ + out2 = _mm_unpacklo_epi64(tr1_2, tr1_6); \ + out3 = _mm_unpackhi_epi64(tr1_2, tr1_6); \ + out4 = _mm_unpacklo_epi64(tr1_1, tr1_5); \ + out5 = _mm_unpackhi_epi64(tr1_1, tr1_5); \ + out6 = _mm_unpacklo_epi64(tr1_3, tr1_7); \ + out7 = _mm_unpackhi_epi64(tr1_3, tr1_7); \ + } + +#define TRANSPOSE_8X4(in0, in1, in2, in3, out0, out1) \ + { \ + const __m128i tr0_0 = _mm_unpacklo_epi16(in0, in1); \ + const __m128i tr0_1 = _mm_unpacklo_epi16(in2, in3); \ + \ + in0 = _mm_unpacklo_epi32(tr0_0, tr0_1); /* i1 i0 */ \ + in1 = _mm_unpackhi_epi32(tr0_0, tr0_1); /* i3 i2 */ \ + } + +static INLINE void array_transpose_4X8(__m128i *in, __m128i *out) { + const __m128i tr0_0 = _mm_unpacklo_epi16(in[0], in[1]); + const __m128i tr0_1 = _mm_unpacklo_epi16(in[2], in[3]); + const __m128i tr0_4 = _mm_unpacklo_epi16(in[4], in[5]); + const __m128i tr0_5 = _mm_unpacklo_epi16(in[6], in[7]); + + const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); + const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); + const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); + const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); + + out[0] = _mm_unpacklo_epi64(tr1_0, tr1_4); + out[1] = _mm_unpackhi_epi64(tr1_0, tr1_4); + out[2] = _mm_unpacklo_epi64(tr1_2, tr1_6); + out[3] = _mm_unpackhi_epi64(tr1_2, tr1_6); +} + +static INLINE void array_transpose_16x16(__m128i *res0, __m128i *res1) { + __m128i tbuf[8]; + array_transpose_8x8(res0, res0); + array_transpose_8x8(res1, tbuf); + array_transpose_8x8(res0 + 8, res1); + array_transpose_8x8(res1 + 8, res1 + 8); + + res0[8] = tbuf[0]; + res0[9] = tbuf[1]; + res0[10] = tbuf[2]; + res0[11] = tbuf[3]; + res0[12] = tbuf[4]; + res0[13] = tbuf[5]; + res0[14] = tbuf[6]; + res0[15] = tbuf[7]; +} + +// Function to allow 8 bit optimisations to be used when profile 0 is used with +// highbitdepth enabled +static INLINE __m128i load_input_data(const tran_low_t *data) { +#if CONFIG_HIGHBITDEPTH + return octa_set_epi16(data[0], data[1], data[2], data[3], data[4], data[5], + data[6], data[7]); +#else + return _mm_load_si128((const __m128i *)data); +#endif +} + +static INLINE void load_buffer_8x16(const tran_low_t *input, __m128i *in) { + in[0] = load_input_data(input + 0 * 16); + in[1] = load_input_data(input + 1 * 16); + in[2] = load_input_data(input + 2 * 16); + in[3] = load_input_data(input + 3 * 16); + in[4] = load_input_data(input + 4 * 16); + in[5] = load_input_data(input + 5 * 16); + in[6] = load_input_data(input + 6 * 16); + in[7] = load_input_data(input + 7 * 16); + + in[8] = load_input_data(input + 8 * 16); + in[9] = load_input_data(input + 9 * 16); + in[10] = load_input_data(input + 10 * 16); + in[11] = load_input_data(input + 11 * 16); + in[12] = load_input_data(input + 12 * 16); + in[13] = load_input_data(input + 13 * 16); + in[14] = load_input_data(input + 14 * 16); + in[15] = load_input_data(input + 15 * 16); +} + +#define RECON_AND_STORE(dest, in_x) \ + { \ + __m128i d0 = _mm_loadl_epi64((__m128i *)(dest)); \ + d0 = _mm_unpacklo_epi8(d0, zero); \ + d0 = _mm_add_epi16(in_x, d0); \ + d0 = _mm_packus_epi16(d0, d0); \ + _mm_storel_epi64((__m128i *)(dest), d0); \ + } + +static INLINE void write_buffer_8x16(uint8_t *dest, __m128i *in, int stride) { + const __m128i final_rounding = _mm_set1_epi16(1 << 5); + const __m128i zero = _mm_setzero_si128(); + // Final rounding and shift + in[0] = _mm_adds_epi16(in[0], final_rounding); + in[1] = _mm_adds_epi16(in[1], final_rounding); + in[2] = _mm_adds_epi16(in[2], final_rounding); + in[3] = _mm_adds_epi16(in[3], final_rounding); + in[4] = _mm_adds_epi16(in[4], final_rounding); + in[5] = _mm_adds_epi16(in[5], final_rounding); + in[6] = _mm_adds_epi16(in[6], final_rounding); + in[7] = _mm_adds_epi16(in[7], final_rounding); + in[8] = _mm_adds_epi16(in[8], final_rounding); + in[9] = _mm_adds_epi16(in[9], final_rounding); + in[10] = _mm_adds_epi16(in[10], final_rounding); + in[11] = _mm_adds_epi16(in[11], final_rounding); + in[12] = _mm_adds_epi16(in[12], final_rounding); + in[13] = _mm_adds_epi16(in[13], final_rounding); + in[14] = _mm_adds_epi16(in[14], final_rounding); + in[15] = _mm_adds_epi16(in[15], final_rounding); + + in[0] = _mm_srai_epi16(in[0], 6); + in[1] = _mm_srai_epi16(in[1], 6); + in[2] = _mm_srai_epi16(in[2], 6); + in[3] = _mm_srai_epi16(in[3], 6); + in[4] = _mm_srai_epi16(in[4], 6); + in[5] = _mm_srai_epi16(in[5], 6); + in[6] = _mm_srai_epi16(in[6], 6); + in[7] = _mm_srai_epi16(in[7], 6); + in[8] = _mm_srai_epi16(in[8], 6); + in[9] = _mm_srai_epi16(in[9], 6); + in[10] = _mm_srai_epi16(in[10], 6); + in[11] = _mm_srai_epi16(in[11], 6); + in[12] = _mm_srai_epi16(in[12], 6); + in[13] = _mm_srai_epi16(in[13], 6); + in[14] = _mm_srai_epi16(in[14], 6); + in[15] = _mm_srai_epi16(in[15], 6); + + RECON_AND_STORE(dest + 0 * stride, in[0]); + RECON_AND_STORE(dest + 1 * stride, in[1]); + RECON_AND_STORE(dest + 2 * stride, in[2]); + RECON_AND_STORE(dest + 3 * stride, in[3]); + RECON_AND_STORE(dest + 4 * stride, in[4]); + RECON_AND_STORE(dest + 5 * stride, in[5]); + RECON_AND_STORE(dest + 6 * stride, in[6]); + RECON_AND_STORE(dest + 7 * stride, in[7]); + RECON_AND_STORE(dest + 8 * stride, in[8]); + RECON_AND_STORE(dest + 9 * stride, in[9]); + RECON_AND_STORE(dest + 10 * stride, in[10]); + RECON_AND_STORE(dest + 11 * stride, in[11]); + RECON_AND_STORE(dest + 12 * stride, in[12]); + RECON_AND_STORE(dest + 13 * stride, in[13]); + RECON_AND_STORE(dest + 14 * stride, in[14]); + RECON_AND_STORE(dest + 15 * stride, in[15]); +} + +#define TRANSPOSE_4X8_10(tmp0, tmp1, tmp2, tmp3, out0, out1, out2, out3) \ + { \ + const __m128i tr0_0 = _mm_unpackhi_epi16(tmp0, tmp1); \ + const __m128i tr0_1 = _mm_unpacklo_epi16(tmp1, tmp0); \ + const __m128i tr0_4 = _mm_unpacklo_epi16(tmp2, tmp3); \ + const __m128i tr0_5 = _mm_unpackhi_epi16(tmp3, tmp2); \ + \ + const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); \ + const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); \ + const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); \ + const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); \ + \ + out0 = _mm_unpacklo_epi64(tr1_0, tr1_4); \ + out1 = _mm_unpackhi_epi64(tr1_0, tr1_4); \ + out2 = _mm_unpacklo_epi64(tr1_2, tr1_6); \ + out3 = _mm_unpackhi_epi64(tr1_2, tr1_6); \ + } + +#define TRANSPOSE_8X8_10(in0, in1, in2, in3, out0, out1) \ + { \ + const __m128i tr0_0 = _mm_unpacklo_epi16(in0, in1); \ + const __m128i tr0_1 = _mm_unpacklo_epi16(in2, in3); \ + out0 = _mm_unpacklo_epi32(tr0_0, tr0_1); \ + out1 = _mm_unpackhi_epi32(tr0_0, tr0_1); \ + } + +void iadst16_8col(__m128i *in); +void idct16_8col(__m128i *in); +void aom_idct4_sse2(__m128i *in); +void aom_idct8_sse2(__m128i *in); +void aom_idct16_sse2(__m128i *in0, __m128i *in1); +void aom_iadst4_sse2(__m128i *in); +void aom_iadst8_sse2(__m128i *in); +void aom_iadst16_sse2(__m128i *in0, __m128i *in1); +void idct32_8col(__m128i *in0, __m128i *in1); + +#endif // AOM_DSP_X86_INV_TXFM_SSE2_H_ diff --git a/third_party/aom/aom_dsp/x86/inv_txfm_ssse3.c b/third_party/aom/aom_dsp/x86/inv_txfm_ssse3.c new file mode 100644 index 0000000000..9d006797be --- /dev/null +++ b/third_party/aom/aom_dsp/x86/inv_txfm_ssse3.c @@ -0,0 +1,1333 @@ +/* + * Copyright (c) 2017 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/x86/inv_txfm_sse2.h" +#include "aom_dsp/x86/txfm_common_sse2.h" + +void aom_idct8x8_64_add_ssse3(const tran_low_t *input, uint8_t *dest, + int stride) { + const __m128i zero = _mm_setzero_si128(); + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i final_rounding = _mm_set1_epi16(1 << 4); + const __m128i stg1_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i stg1_1 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i stg1_2 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i stg1_3 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i stk2_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stk2_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg2_2 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i stg2_3 = pair_set_epi16(cospi_8_64, cospi_24_64); + + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7; + __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + int i; + + // Load input data. + in0 = load_input_data(input); + in1 = load_input_data(input + 8 * 1); + in2 = load_input_data(input + 8 * 2); + in3 = load_input_data(input + 8 * 3); + in4 = load_input_data(input + 8 * 4); + in5 = load_input_data(input + 8 * 5); + in6 = load_input_data(input + 8 * 6); + in7 = load_input_data(input + 8 * 7); + + // 2-D + for (i = 0; i < 2; i++) { + // 8x8 Transpose is copied from vpx_fdct8x8_sse2() + TRANSPOSE_8X8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + + // 4-stage 1D idct8x8 + { + /* Stage1 */ + { + const __m128i lo_17 = _mm_unpacklo_epi16(in1, in7); + const __m128i hi_17 = _mm_unpackhi_epi16(in1, in7); + const __m128i lo_35 = _mm_unpacklo_epi16(in3, in5); + const __m128i hi_35 = _mm_unpackhi_epi16(in3, in5); + + { + tmp0 = _mm_madd_epi16(lo_17, stg1_0); + tmp1 = _mm_madd_epi16(hi_17, stg1_0); + tmp2 = _mm_madd_epi16(lo_17, stg1_1); + tmp3 = _mm_madd_epi16(hi_17, stg1_1); + tmp4 = _mm_madd_epi16(lo_35, stg1_2); + tmp5 = _mm_madd_epi16(hi_35, stg1_2); + tmp6 = _mm_madd_epi16(lo_35, stg1_3); + tmp7 = _mm_madd_epi16(hi_35, stg1_3); + + tmp0 = _mm_add_epi32(tmp0, rounding); + tmp1 = _mm_add_epi32(tmp1, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp3 = _mm_add_epi32(tmp3, rounding); + tmp4 = _mm_add_epi32(tmp4, rounding); + tmp5 = _mm_add_epi32(tmp5, rounding); + tmp6 = _mm_add_epi32(tmp6, rounding); + tmp7 = _mm_add_epi32(tmp7, rounding); + + tmp0 = _mm_srai_epi32(tmp0, 14); + tmp1 = _mm_srai_epi32(tmp1, 14); + tmp2 = _mm_srai_epi32(tmp2, 14); + tmp3 = _mm_srai_epi32(tmp3, 14); + tmp4 = _mm_srai_epi32(tmp4, 14); + tmp5 = _mm_srai_epi32(tmp5, 14); + tmp6 = _mm_srai_epi32(tmp6, 14); + tmp7 = _mm_srai_epi32(tmp7, 14); + + stp1_4 = _mm_packs_epi32(tmp0, tmp1); + stp1_7 = _mm_packs_epi32(tmp2, tmp3); + stp1_5 = _mm_packs_epi32(tmp4, tmp5); + stp1_6 = _mm_packs_epi32(tmp6, tmp7); + } + } + + /* Stage2 */ + { + const __m128i lo_26 = _mm_unpacklo_epi16(in2, in6); + const __m128i hi_26 = _mm_unpackhi_epi16(in2, in6); + + { + tmp0 = _mm_unpacklo_epi16(in0, in4); + tmp1 = _mm_unpackhi_epi16(in0, in4); + + tmp2 = _mm_madd_epi16(tmp0, stk2_0); + tmp3 = _mm_madd_epi16(tmp1, stk2_0); + tmp4 = _mm_madd_epi16(tmp0, stk2_1); + tmp5 = _mm_madd_epi16(tmp1, stk2_1); + + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp3 = _mm_add_epi32(tmp3, rounding); + tmp4 = _mm_add_epi32(tmp4, rounding); + tmp5 = _mm_add_epi32(tmp5, rounding); + + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); + tmp4 = _mm_srai_epi32(tmp4, DCT_CONST_BITS); + tmp5 = _mm_srai_epi32(tmp5, DCT_CONST_BITS); + + stp2_0 = _mm_packs_epi32(tmp2, tmp3); + stp2_1 = _mm_packs_epi32(tmp4, tmp5); + + tmp0 = _mm_madd_epi16(lo_26, stg2_2); + tmp1 = _mm_madd_epi16(hi_26, stg2_2); + tmp2 = _mm_madd_epi16(lo_26, stg2_3); + tmp3 = _mm_madd_epi16(hi_26, stg2_3); + + tmp0 = _mm_add_epi32(tmp0, rounding); + tmp1 = _mm_add_epi32(tmp1, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp3 = _mm_add_epi32(tmp3, rounding); + + tmp0 = _mm_srai_epi32(tmp0, 14); + tmp1 = _mm_srai_epi32(tmp1, 14); + tmp2 = _mm_srai_epi32(tmp2, 14); + tmp3 = _mm_srai_epi32(tmp3, 14); + + stp2_2 = _mm_packs_epi32(tmp0, tmp1); + stp2_3 = _mm_packs_epi32(tmp2, tmp3); + } + + stp2_4 = _mm_add_epi16(stp1_4, stp1_5); + stp2_5 = _mm_sub_epi16(stp1_4, stp1_5); + stp2_6 = _mm_sub_epi16(stp1_7, stp1_6); + stp2_7 = _mm_add_epi16(stp1_7, stp1_6); + } + + /* Stage3 */ + { + stp1_0 = _mm_add_epi16(stp2_0, stp2_3); + stp1_1 = _mm_add_epi16(stp2_1, stp2_2); + stp1_2 = _mm_sub_epi16(stp2_1, stp2_2); + stp1_3 = _mm_sub_epi16(stp2_0, stp2_3); + + tmp0 = _mm_unpacklo_epi16(stp2_6, stp2_5); + tmp1 = _mm_unpackhi_epi16(stp2_6, stp2_5); + + tmp2 = _mm_madd_epi16(tmp0, stk2_1); + tmp3 = _mm_madd_epi16(tmp1, stk2_1); + tmp4 = _mm_madd_epi16(tmp0, stk2_0); + tmp5 = _mm_madd_epi16(tmp1, stk2_0); + + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp3 = _mm_add_epi32(tmp3, rounding); + tmp4 = _mm_add_epi32(tmp4, rounding); + tmp5 = _mm_add_epi32(tmp5, rounding); + + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); + tmp4 = _mm_srai_epi32(tmp4, DCT_CONST_BITS); + tmp5 = _mm_srai_epi32(tmp5, DCT_CONST_BITS); + + stp1_5 = _mm_packs_epi32(tmp2, tmp3); + stp1_6 = _mm_packs_epi32(tmp4, tmp5); + } + + /* Stage4 */ + in0 = _mm_add_epi16(stp1_0, stp2_7); + in1 = _mm_add_epi16(stp1_1, stp1_6); + in2 = _mm_add_epi16(stp1_2, stp1_5); + in3 = _mm_add_epi16(stp1_3, stp2_4); + in4 = _mm_sub_epi16(stp1_3, stp2_4); + in5 = _mm_sub_epi16(stp1_2, stp1_5); + in6 = _mm_sub_epi16(stp1_1, stp1_6); + in7 = _mm_sub_epi16(stp1_0, stp2_7); + } + } + + // Final rounding and shift + in0 = _mm_adds_epi16(in0, final_rounding); + in1 = _mm_adds_epi16(in1, final_rounding); + in2 = _mm_adds_epi16(in2, final_rounding); + in3 = _mm_adds_epi16(in3, final_rounding); + in4 = _mm_adds_epi16(in4, final_rounding); + in5 = _mm_adds_epi16(in5, final_rounding); + in6 = _mm_adds_epi16(in6, final_rounding); + in7 = _mm_adds_epi16(in7, final_rounding); + + in0 = _mm_srai_epi16(in0, 5); + in1 = _mm_srai_epi16(in1, 5); + in2 = _mm_srai_epi16(in2, 5); + in3 = _mm_srai_epi16(in3, 5); + in4 = _mm_srai_epi16(in4, 5); + in5 = _mm_srai_epi16(in5, 5); + in6 = _mm_srai_epi16(in6, 5); + in7 = _mm_srai_epi16(in7, 5); + + RECON_AND_STORE(dest + 0 * stride, in0); + RECON_AND_STORE(dest + 1 * stride, in1); + RECON_AND_STORE(dest + 2 * stride, in2); + RECON_AND_STORE(dest + 3 * stride, in3); + RECON_AND_STORE(dest + 4 * stride, in4); + RECON_AND_STORE(dest + 5 * stride, in5); + RECON_AND_STORE(dest + 6 * stride, in6); + RECON_AND_STORE(dest + 7 * stride, in7); +} + +void aom_idct8x8_12_add_ssse3(const tran_low_t *input, uint8_t *dest, + int stride) { + const __m128i zero = _mm_setzero_si128(); + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i final_rounding = _mm_set1_epi16(1 << 4); + const __m128i stg1_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64); + const __m128i stg1_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64); + const __m128i stg1_2 = pair_set_epi16(-2 * cospi_20_64, -2 * cospi_20_64); + const __m128i stg1_3 = pair_set_epi16(2 * cospi_12_64, 2 * cospi_12_64); + const __m128i stg2_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64); + const __m128i stk2_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stk2_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg2_2 = pair_set_epi16(2 * cospi_24_64, 2 * cospi_24_64); + const __m128i stg2_3 = pair_set_epi16(2 * cospi_8_64, 2 * cospi_8_64); + const __m128i stg3_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7; + __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7; + __m128i tmp0, tmp1, tmp2, tmp3; + + // Rows. Load 4-row input data. + in0 = load_input_data(input); + in1 = load_input_data(input + 8 * 1); + in2 = load_input_data(input + 8 * 2); + in3 = load_input_data(input + 8 * 3); + + // 8x4 Transpose + TRANSPOSE_8X8_10(in0, in1, in2, in3, in0, in1); + + // Stage1 + tmp0 = _mm_mulhrs_epi16(in0, stg1_0); + tmp1 = _mm_mulhrs_epi16(in0, stg1_1); + tmp2 = _mm_mulhrs_epi16(in1, stg1_2); + tmp3 = _mm_mulhrs_epi16(in1, stg1_3); + + stp1_4 = _mm_unpackhi_epi64(tmp0, tmp1); + stp1_5 = _mm_unpackhi_epi64(tmp2, tmp3); + + // Stage2 + tmp0 = _mm_mulhrs_epi16(in0, stg2_0); + stp2_0 = _mm_unpacklo_epi64(tmp0, tmp0); + + tmp1 = _mm_mulhrs_epi16(in1, stg2_2); + tmp2 = _mm_mulhrs_epi16(in1, stg2_3); + stp2_2 = _mm_unpacklo_epi64(tmp2, tmp1); + + tmp0 = _mm_add_epi16(stp1_4, stp1_5); + tmp1 = _mm_sub_epi16(stp1_4, stp1_5); + + stp2_4 = tmp0; + stp2_5 = _mm_unpacklo_epi64(tmp1, zero); + stp2_6 = _mm_unpackhi_epi64(tmp1, zero); + + tmp0 = _mm_unpacklo_epi16(stp2_5, stp2_6); + tmp1 = _mm_madd_epi16(tmp0, stg3_0); + tmp2 = _mm_madd_epi16(tmp0, stk2_0); // stg3_1 = stk2_0 + + tmp1 = _mm_add_epi32(tmp1, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + + stp1_5 = _mm_packs_epi32(tmp1, tmp2); + + // Stage3 + tmp2 = _mm_add_epi16(stp2_0, stp2_2); + tmp3 = _mm_sub_epi16(stp2_0, stp2_2); + + stp1_2 = _mm_unpackhi_epi64(tmp3, tmp2); + stp1_3 = _mm_unpacklo_epi64(tmp3, tmp2); + + // Stage4 + tmp0 = _mm_add_epi16(stp1_3, stp2_4); + tmp1 = _mm_add_epi16(stp1_2, stp1_5); + tmp2 = _mm_sub_epi16(stp1_3, stp2_4); + tmp3 = _mm_sub_epi16(stp1_2, stp1_5); + + TRANSPOSE_4X8_10(tmp0, tmp1, tmp2, tmp3, in0, in1, in2, in3) + + /* Stage1 */ + stp1_4 = _mm_mulhrs_epi16(in1, stg1_0); + stp1_7 = _mm_mulhrs_epi16(in1, stg1_1); + stp1_5 = _mm_mulhrs_epi16(in3, stg1_2); + stp1_6 = _mm_mulhrs_epi16(in3, stg1_3); + + /* Stage2 */ + stp2_0 = _mm_mulhrs_epi16(in0, stg2_0); + stp2_1 = _mm_mulhrs_epi16(in0, stg2_0); + + stp2_2 = _mm_mulhrs_epi16(in2, stg2_2); + stp2_3 = _mm_mulhrs_epi16(in2, stg2_3); + + stp2_4 = _mm_add_epi16(stp1_4, stp1_5); + stp2_5 = _mm_sub_epi16(stp1_4, stp1_5); + stp2_6 = _mm_sub_epi16(stp1_7, stp1_6); + stp2_7 = _mm_add_epi16(stp1_7, stp1_6); + + /* Stage3 */ + stp1_0 = _mm_add_epi16(stp2_0, stp2_3); + stp1_1 = _mm_add_epi16(stp2_1, stp2_2); + stp1_2 = _mm_sub_epi16(stp2_1, stp2_2); + stp1_3 = _mm_sub_epi16(stp2_0, stp2_3); + + tmp0 = _mm_unpacklo_epi16(stp2_6, stp2_5); + tmp1 = _mm_unpackhi_epi16(stp2_6, stp2_5); + + tmp2 = _mm_madd_epi16(tmp0, stk2_0); + tmp3 = _mm_madd_epi16(tmp1, stk2_0); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp3 = _mm_add_epi32(tmp3, rounding); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); + stp1_6 = _mm_packs_epi32(tmp2, tmp3); + + tmp2 = _mm_madd_epi16(tmp0, stk2_1); + tmp3 = _mm_madd_epi16(tmp1, stk2_1); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp3 = _mm_add_epi32(tmp3, rounding); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); + stp1_5 = _mm_packs_epi32(tmp2, tmp3); + + /* Stage4 */ + in0 = _mm_add_epi16(stp1_0, stp2_7); + in1 = _mm_add_epi16(stp1_1, stp1_6); + in2 = _mm_add_epi16(stp1_2, stp1_5); + in3 = _mm_add_epi16(stp1_3, stp2_4); + in4 = _mm_sub_epi16(stp1_3, stp2_4); + in5 = _mm_sub_epi16(stp1_2, stp1_5); + in6 = _mm_sub_epi16(stp1_1, stp1_6); + in7 = _mm_sub_epi16(stp1_0, stp2_7); + + // Final rounding and shift + in0 = _mm_adds_epi16(in0, final_rounding); + in1 = _mm_adds_epi16(in1, final_rounding); + in2 = _mm_adds_epi16(in2, final_rounding); + in3 = _mm_adds_epi16(in3, final_rounding); + in4 = _mm_adds_epi16(in4, final_rounding); + in5 = _mm_adds_epi16(in5, final_rounding); + in6 = _mm_adds_epi16(in6, final_rounding); + in7 = _mm_adds_epi16(in7, final_rounding); + + in0 = _mm_srai_epi16(in0, 5); + in1 = _mm_srai_epi16(in1, 5); + in2 = _mm_srai_epi16(in2, 5); + in3 = _mm_srai_epi16(in3, 5); + in4 = _mm_srai_epi16(in4, 5); + in5 = _mm_srai_epi16(in5, 5); + in6 = _mm_srai_epi16(in6, 5); + in7 = _mm_srai_epi16(in7, 5); + + RECON_AND_STORE(dest + 0 * stride, in0); + RECON_AND_STORE(dest + 1 * stride, in1); + RECON_AND_STORE(dest + 2 * stride, in2); + RECON_AND_STORE(dest + 3 * stride, in3); + RECON_AND_STORE(dest + 4 * stride, in4); + RECON_AND_STORE(dest + 5 * stride, in5); + RECON_AND_STORE(dest + 6 * stride, in6); + RECON_AND_STORE(dest + 7 * stride, in7); +} + +// Only do addition and subtraction butterfly, size = 16, 32 +static INLINE void add_sub_butterfly(const __m128i *in, __m128i *out, + int size) { + int i = 0; + const int num = size >> 1; + const int bound = size - 1; + while (i < num) { + out[i] = _mm_add_epi16(in[i], in[bound - i]); + out[bound - i] = _mm_sub_epi16(in[i], in[bound - i]); + i++; + } +} + +#define BUTTERFLY_PAIR(x0, x1, co0, co1) \ + do { \ + tmp0 = _mm_madd_epi16(x0, co0); \ + tmp1 = _mm_madd_epi16(x1, co0); \ + tmp2 = _mm_madd_epi16(x0, co1); \ + tmp3 = _mm_madd_epi16(x1, co1); \ + tmp0 = _mm_add_epi32(tmp0, rounding); \ + tmp1 = _mm_add_epi32(tmp1, rounding); \ + tmp2 = _mm_add_epi32(tmp2, rounding); \ + tmp3 = _mm_add_epi32(tmp3, rounding); \ + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \ + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \ + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \ + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \ + } while (0) + +static INLINE void butterfly(const __m128i *x0, const __m128i *x1, + const __m128i *c0, const __m128i *c1, __m128i *y0, + __m128i *y1) { + __m128i tmp0, tmp1, tmp2, tmp3, u0, u1; + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + + u0 = _mm_unpacklo_epi16(*x0, *x1); + u1 = _mm_unpackhi_epi16(*x0, *x1); + BUTTERFLY_PAIR(u0, u1, *c0, *c1); + *y0 = _mm_packs_epi32(tmp0, tmp1); + *y1 = _mm_packs_epi32(tmp2, tmp3); +} + +static INLINE void butterfly_self(__m128i *x0, __m128i *x1, const __m128i *c0, + const __m128i *c1) { + __m128i tmp0, tmp1, tmp2, tmp3, u0, u1; + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + + u0 = _mm_unpacklo_epi16(*x0, *x1); + u1 = _mm_unpackhi_epi16(*x0, *x1); + BUTTERFLY_PAIR(u0, u1, *c0, *c1); + *x0 = _mm_packs_epi32(tmp0, tmp1); + *x1 = _mm_packs_epi32(tmp2, tmp3); +} + +static void idct32_34_first_half(const __m128i *in, __m128i *stp1) { + const __m128i stk2_0 = pair_set_epi16(2 * cospi_30_64, 2 * cospi_30_64); + const __m128i stk2_1 = pair_set_epi16(2 * cospi_2_64, 2 * cospi_2_64); + const __m128i stk2_6 = pair_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64); + const __m128i stk2_7 = pair_set_epi16(2 * cospi_6_64, 2 * cospi_6_64); + + const __m128i stk3_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64); + const __m128i stk3_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64); + + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64); + const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + __m128i u0, u1, u2, u3, u4, u5, u6, u7; + __m128i x0, x1, x4, x5, x6, x7; + __m128i v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15; + + // phase 1 + + // 0, 15 + u2 = _mm_mulhrs_epi16(in[2], stk2_1); // stp2_15 + u3 = _mm_mulhrs_epi16(in[6], stk2_7); // stp2_12 + v15 = _mm_add_epi16(u2, u3); + // in[0], in[4] + x0 = _mm_mulhrs_epi16(in[0], stk4_0); // stp1[0] + x7 = _mm_mulhrs_epi16(in[4], stk3_1); // stp1[7] + v0 = _mm_add_epi16(x0, x7); // stp2_0 + stp1[0] = _mm_add_epi16(v0, v15); + stp1[15] = _mm_sub_epi16(v0, v15); + + // in[2], in[6] + u0 = _mm_mulhrs_epi16(in[2], stk2_0); // stp2_8 + u1 = _mm_mulhrs_epi16(in[6], stk2_6); // stp2_11 + butterfly(&u0, &u2, &stg4_4, &stg4_5, &u4, &u5); // stp2_9, stp2_14 + butterfly(&u1, &u3, &stg4_6, &stg4_4, &u6, &u7); // stp2_10, stp2_13 + + v8 = _mm_add_epi16(u0, u1); + v9 = _mm_add_epi16(u4, u6); + v10 = _mm_sub_epi16(u4, u6); + v11 = _mm_sub_epi16(u0, u1); + v12 = _mm_sub_epi16(u2, u3); + v13 = _mm_sub_epi16(u5, u7); + v14 = _mm_add_epi16(u5, u7); + + butterfly_self(&v10, &v13, &stg6_0, &stg4_0); + butterfly_self(&v11, &v12, &stg6_0, &stg4_0); + + // 1, 14 + x1 = _mm_mulhrs_epi16(in[0], stk4_0); // stp1[1], stk4_1 = stk4_0 + // stp1[2] = stp1[0], stp1[3] = stp1[1] + x4 = _mm_mulhrs_epi16(in[4], stk3_0); // stp1[4] + butterfly(&x7, &x4, &stg4_1, &stg4_0, &x5, &x6); + v1 = _mm_add_epi16(x1, x6); // stp2_1 + v2 = _mm_add_epi16(x0, x5); // stp2_2 + stp1[1] = _mm_add_epi16(v1, v14); + stp1[14] = _mm_sub_epi16(v1, v14); + + stp1[2] = _mm_add_epi16(v2, v13); + stp1[13] = _mm_sub_epi16(v2, v13); + + v3 = _mm_add_epi16(x1, x4); // stp2_3 + v4 = _mm_sub_epi16(x1, x4); // stp2_4 + + v5 = _mm_sub_epi16(x0, x5); // stp2_5 + + v6 = _mm_sub_epi16(x1, x6); // stp2_6 + v7 = _mm_sub_epi16(x0, x7); // stp2_7 + stp1[3] = _mm_add_epi16(v3, v12); + stp1[12] = _mm_sub_epi16(v3, v12); + + stp1[6] = _mm_add_epi16(v6, v9); + stp1[9] = _mm_sub_epi16(v6, v9); + + stp1[7] = _mm_add_epi16(v7, v8); + stp1[8] = _mm_sub_epi16(v7, v8); + + stp1[4] = _mm_add_epi16(v4, v11); + stp1[11] = _mm_sub_epi16(v4, v11); + + stp1[5] = _mm_add_epi16(v5, v10); + stp1[10] = _mm_sub_epi16(v5, v10); +} + +static void idct32_34_second_half(const __m128i *in, __m128i *stp1) { + const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64); + const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64); + const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64); + const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64); + const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64); + const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64); + const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64); + const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64); + const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64); + const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64); + + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + __m128i v16, v17, v18, v19, v20, v21, v22, v23; + __m128i v24, v25, v26, v27, v28, v29, v30, v31; + __m128i u16, u17, u18, u19, u20, u21, u22, u23; + __m128i u24, u25, u26, u27, u28, u29, u30, u31; + + v16 = _mm_mulhrs_epi16(in[1], stk1_0); + v31 = _mm_mulhrs_epi16(in[1], stk1_1); + + v19 = _mm_mulhrs_epi16(in[7], stk1_6); + v28 = _mm_mulhrs_epi16(in[7], stk1_7); + + v20 = _mm_mulhrs_epi16(in[5], stk1_8); + v27 = _mm_mulhrs_epi16(in[5], stk1_9); + + v23 = _mm_mulhrs_epi16(in[3], stk1_14); + v24 = _mm_mulhrs_epi16(in[3], stk1_15); + + butterfly(&v16, &v31, &stg3_4, &stg3_5, &v17, &v30); + butterfly(&v19, &v28, &stg3_6, &stg3_4, &v18, &v29); + butterfly(&v20, &v27, &stg3_8, &stg3_9, &v21, &v26); + butterfly(&v23, &v24, &stg3_10, &stg3_8, &v22, &v25); + + u16 = _mm_add_epi16(v16, v19); + u17 = _mm_add_epi16(v17, v18); + u18 = _mm_sub_epi16(v17, v18); + u19 = _mm_sub_epi16(v16, v19); + u20 = _mm_sub_epi16(v23, v20); + u21 = _mm_sub_epi16(v22, v21); + u22 = _mm_add_epi16(v22, v21); + u23 = _mm_add_epi16(v23, v20); + u24 = _mm_add_epi16(v24, v27); + u27 = _mm_sub_epi16(v24, v27); + u25 = _mm_add_epi16(v25, v26); + u26 = _mm_sub_epi16(v25, v26); + u28 = _mm_sub_epi16(v31, v28); + u31 = _mm_add_epi16(v28, v31); + u29 = _mm_sub_epi16(v30, v29); + u30 = _mm_add_epi16(v29, v30); + + butterfly_self(&u18, &u29, &stg4_4, &stg4_5); + butterfly_self(&u19, &u28, &stg4_4, &stg4_5); + butterfly_self(&u20, &u27, &stg4_6, &stg4_4); + butterfly_self(&u21, &u26, &stg4_6, &stg4_4); + + stp1[16] = _mm_add_epi16(u16, u23); + stp1[23] = _mm_sub_epi16(u16, u23); + + stp1[17] = _mm_add_epi16(u17, u22); + stp1[22] = _mm_sub_epi16(u17, u22); + + stp1[18] = _mm_add_epi16(u18, u21); + stp1[21] = _mm_sub_epi16(u18, u21); + + stp1[19] = _mm_add_epi16(u19, u20); + stp1[20] = _mm_sub_epi16(u19, u20); + + stp1[24] = _mm_sub_epi16(u31, u24); + stp1[31] = _mm_add_epi16(u24, u31); + + stp1[25] = _mm_sub_epi16(u30, u25); + stp1[30] = _mm_add_epi16(u25, u30); + + stp1[26] = _mm_sub_epi16(u29, u26); + stp1[29] = _mm_add_epi16(u26, u29); + + stp1[27] = _mm_sub_epi16(u28, u27); + stp1[28] = _mm_add_epi16(u27, u28); + + butterfly_self(&stp1[20], &stp1[27], &stg6_0, &stg4_0); + butterfly_self(&stp1[21], &stp1[26], &stg6_0, &stg4_0); + butterfly_self(&stp1[22], &stp1[25], &stg6_0, &stg4_0); + butterfly_self(&stp1[23], &stp1[24], &stg6_0, &stg4_0); +} + +// Only upper-left 8x8 has non-zero coeff +void aom_idct32x32_34_add_ssse3(const tran_low_t *input, uint8_t *dest, + int stride) { + const __m128i zero = _mm_setzero_si128(); + const __m128i final_rounding = _mm_set1_epi16(1 << 5); + __m128i in[32], col[32]; + __m128i stp1[32]; + int i; + + // Load input data. Only need to load the top left 8x8 block. + in[0] = load_input_data(input); + in[1] = load_input_data(input + 32); + in[2] = load_input_data(input + 64); + in[3] = load_input_data(input + 96); + in[4] = load_input_data(input + 128); + in[5] = load_input_data(input + 160); + in[6] = load_input_data(input + 192); + in[7] = load_input_data(input + 224); + + array_transpose_8x8(in, in); + idct32_34_first_half(in, stp1); + idct32_34_second_half(in, stp1); + + // 1_D: Store 32 intermediate results for each 8x32 block. + add_sub_butterfly(stp1, col, 32); + for (i = 0; i < 4; i++) { + int j; + // Transpose 32x8 block to 8x32 block + array_transpose_8x8(col + i * 8, in); + idct32_34_first_half(in, stp1); + idct32_34_second_half(in, stp1); + + // 2_D: Calculate the results and store them to destination. + add_sub_butterfly(stp1, in, 32); + for (j = 0; j < 32; ++j) { + // Final rounding and shift + in[j] = _mm_adds_epi16(in[j], final_rounding); + in[j] = _mm_srai_epi16(in[j], 6); + RECON_AND_STORE(dest + j * stride, in[j]); + } + + dest += 8; + } +} + +// in0[16] represents the left 8x16 block +// in1[16] represents the right 8x16 block +static void load_buffer_16x16(const tran_low_t *input, __m128i *in0, + __m128i *in1) { + int i; + for (i = 0; i < 16; i++) { + in0[i] = load_input_data(input); + in1[i] = load_input_data(input + 8); + input += 32; + } +} + +static void array_transpose_16x16_2(__m128i *in0, __m128i *in1, __m128i *out0, + __m128i *out1) { + array_transpose_8x8(in0, out0); + array_transpose_8x8(&in0[8], out1); + array_transpose_8x8(in1, &out0[8]); + array_transpose_8x8(&in1[8], &out1[8]); +} + +// Group the coefficient calculation into smaller functions +// to prevent stack spillover: +// quarter_1: 0-7 +// quarter_2: 8-15 +// quarter_3_4: 16-23, 24-31 +static void idct32_8x32_135_quarter_1(const __m128i *in /*in[16]*/, + __m128i *out /*out[8]*/) { + __m128i u0, u1, u2, u3, u4, u5, u6, u7; + __m128i v0, v1, v2, v3, v4, v5, v6, v7; + + { + const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64); + const __m128i stk4_2 = pair_set_epi16(2 * cospi_24_64, 2 * cospi_24_64); + const __m128i stk4_3 = pair_set_epi16(2 * cospi_8_64, 2 * cospi_8_64); + u0 = _mm_mulhrs_epi16(in[0], stk4_0); + u2 = _mm_mulhrs_epi16(in[8], stk4_2); + u3 = _mm_mulhrs_epi16(in[8], stk4_3); + u1 = u0; + } + + v0 = _mm_add_epi16(u0, u3); + v1 = _mm_add_epi16(u1, u2); + v2 = _mm_sub_epi16(u1, u2); + v3 = _mm_sub_epi16(u0, u3); + + { + const __m128i stk3_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64); + const __m128i stk3_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64); + const __m128i stk3_2 = pair_set_epi16(-2 * cospi_20_64, -2 * cospi_20_64); + const __m128i stk3_3 = pair_set_epi16(2 * cospi_12_64, 2 * cospi_12_64); + u4 = _mm_mulhrs_epi16(in[4], stk3_0); + u7 = _mm_mulhrs_epi16(in[4], stk3_1); + u5 = _mm_mulhrs_epi16(in[12], stk3_2); + u6 = _mm_mulhrs_epi16(in[12], stk3_3); + } + + v4 = _mm_add_epi16(u4, u5); + v5 = _mm_sub_epi16(u4, u5); + v6 = _mm_sub_epi16(u7, u6); + v7 = _mm_add_epi16(u7, u6); + + { + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + butterfly(&v6, &v5, &stg4_1, &stg4_0, &v5, &v6); + } + + out[0] = _mm_add_epi16(v0, v7); + out[1] = _mm_add_epi16(v1, v6); + out[2] = _mm_add_epi16(v2, v5); + out[3] = _mm_add_epi16(v3, v4); + out[4] = _mm_sub_epi16(v3, v4); + out[5] = _mm_sub_epi16(v2, v5); + out[6] = _mm_sub_epi16(v1, v6); + out[7] = _mm_sub_epi16(v0, v7); +} + +static void idct32_8x32_135_quarter_2(const __m128i *in /*in[16]*/, + __m128i *out /*out[8]*/) { + __m128i u8, u9, u10, u11, u12, u13, u14, u15; + __m128i v8, v9, v10, v11, v12, v13, v14, v15; + + { + const __m128i stk2_0 = pair_set_epi16(2 * cospi_30_64, 2 * cospi_30_64); + const __m128i stk2_1 = pair_set_epi16(2 * cospi_2_64, 2 * cospi_2_64); + const __m128i stk2_2 = pair_set_epi16(-2 * cospi_18_64, -2 * cospi_18_64); + const __m128i stk2_3 = pair_set_epi16(2 * cospi_14_64, 2 * cospi_14_64); + const __m128i stk2_4 = pair_set_epi16(2 * cospi_22_64, 2 * cospi_22_64); + const __m128i stk2_5 = pair_set_epi16(2 * cospi_10_64, 2 * cospi_10_64); + const __m128i stk2_6 = pair_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64); + const __m128i stk2_7 = pair_set_epi16(2 * cospi_6_64, 2 * cospi_6_64); + u8 = _mm_mulhrs_epi16(in[2], stk2_0); + u15 = _mm_mulhrs_epi16(in[2], stk2_1); + u9 = _mm_mulhrs_epi16(in[14], stk2_2); + u14 = _mm_mulhrs_epi16(in[14], stk2_3); + u10 = _mm_mulhrs_epi16(in[10], stk2_4); + u13 = _mm_mulhrs_epi16(in[10], stk2_5); + u11 = _mm_mulhrs_epi16(in[6], stk2_6); + u12 = _mm_mulhrs_epi16(in[6], stk2_7); + } + + v8 = _mm_add_epi16(u8, u9); + v9 = _mm_sub_epi16(u8, u9); + v10 = _mm_sub_epi16(u11, u10); + v11 = _mm_add_epi16(u11, u10); + v12 = _mm_add_epi16(u12, u13); + v13 = _mm_sub_epi16(u12, u13); + v14 = _mm_sub_epi16(u15, u14); + v15 = _mm_add_epi16(u15, u14); + + { + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + butterfly_self(&v9, &v14, &stg4_4, &stg4_5); + butterfly_self(&v10, &v13, &stg4_6, &stg4_4); + } + + out[0] = _mm_add_epi16(v8, v11); + out[1] = _mm_add_epi16(v9, v10); + out[2] = _mm_sub_epi16(v9, v10); + out[3] = _mm_sub_epi16(v8, v11); + out[4] = _mm_sub_epi16(v15, v12); + out[5] = _mm_sub_epi16(v14, v13); + out[6] = _mm_add_epi16(v14, v13); + out[7] = _mm_add_epi16(v15, v12); + + { + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + butterfly_self(&out[2], &out[5], &stg6_0, &stg4_0); + butterfly_self(&out[3], &out[4], &stg6_0, &stg4_0); + } +} + +// 8x32 block even indexed 8 inputs of in[16], +// output first half 16 to out[32] +static void idct32_8x32_quarter_1_2(const __m128i *in /*in[16]*/, + __m128i *out /*out[32]*/) { + __m128i temp[16]; + idct32_8x32_135_quarter_1(in, temp); + idct32_8x32_135_quarter_2(in, &temp[8]); + add_sub_butterfly(temp, out, 16); +} + +// 8x32 block odd indexed 8 inputs of in[16], +// output second half 16 to out[32] +static void idct32_8x32_quarter_3_4(const __m128i *in /*in[16]*/, + __m128i *out /*out[32]*/) { + __m128i v16, v17, v18, v19, v20, v21, v22, v23; + __m128i v24, v25, v26, v27, v28, v29, v30, v31; + __m128i u16, u17, u18, u19, u20, u21, u22, u23; + __m128i u24, u25, u26, u27, u28, u29, u30, u31; + + { + const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64); + const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64); + const __m128i stk1_2 = pair_set_epi16(-2 * cospi_17_64, -2 * cospi_17_64); + const __m128i stk1_3 = pair_set_epi16(2 * cospi_15_64, 2 * cospi_15_64); + + const __m128i stk1_4 = pair_set_epi16(2 * cospi_23_64, 2 * cospi_23_64); + const __m128i stk1_5 = pair_set_epi16(2 * cospi_9_64, 2 * cospi_9_64); + const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64); + const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64); + const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64); + const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64); + const __m128i stk1_10 = pair_set_epi16(-2 * cospi_21_64, -2 * cospi_21_64); + const __m128i stk1_11 = pair_set_epi16(2 * cospi_11_64, 2 * cospi_11_64); + + const __m128i stk1_12 = pair_set_epi16(2 * cospi_19_64, 2 * cospi_19_64); + const __m128i stk1_13 = pair_set_epi16(2 * cospi_13_64, 2 * cospi_13_64); + const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64); + const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64); + u16 = _mm_mulhrs_epi16(in[1], stk1_0); + u31 = _mm_mulhrs_epi16(in[1], stk1_1); + u17 = _mm_mulhrs_epi16(in[15], stk1_2); + u30 = _mm_mulhrs_epi16(in[15], stk1_3); + + u18 = _mm_mulhrs_epi16(in[9], stk1_4); + u29 = _mm_mulhrs_epi16(in[9], stk1_5); + u19 = _mm_mulhrs_epi16(in[7], stk1_6); + u28 = _mm_mulhrs_epi16(in[7], stk1_7); + + u20 = _mm_mulhrs_epi16(in[5], stk1_8); + u27 = _mm_mulhrs_epi16(in[5], stk1_9); + u21 = _mm_mulhrs_epi16(in[11], stk1_10); + u26 = _mm_mulhrs_epi16(in[11], stk1_11); + + u22 = _mm_mulhrs_epi16(in[13], stk1_12); + u25 = _mm_mulhrs_epi16(in[13], stk1_13); + u23 = _mm_mulhrs_epi16(in[3], stk1_14); + u24 = _mm_mulhrs_epi16(in[3], stk1_15); + } + + v16 = _mm_add_epi16(u16, u17); + v17 = _mm_sub_epi16(u16, u17); + v18 = _mm_sub_epi16(u19, u18); + v19 = _mm_add_epi16(u19, u18); + + v20 = _mm_add_epi16(u20, u21); + v21 = _mm_sub_epi16(u20, u21); + v22 = _mm_sub_epi16(u23, u22); + v23 = _mm_add_epi16(u23, u22); + + v24 = _mm_add_epi16(u24, u25); + v25 = _mm_sub_epi16(u24, u25); + v26 = _mm_sub_epi16(u27, u26); + v27 = _mm_add_epi16(u27, u26); + + v28 = _mm_add_epi16(u28, u29); + v29 = _mm_sub_epi16(u28, u29); + v30 = _mm_sub_epi16(u31, u30); + v31 = _mm_add_epi16(u31, u30); + + { + const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64); + const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64); + + butterfly_self(&v17, &v30, &stg3_4, &stg3_5); + butterfly_self(&v18, &v29, &stg3_6, &stg3_4); + butterfly_self(&v21, &v26, &stg3_8, &stg3_9); + butterfly_self(&v22, &v25, &stg3_10, &stg3_8); + } + + u16 = _mm_add_epi16(v16, v19); + u17 = _mm_add_epi16(v17, v18); + u18 = _mm_sub_epi16(v17, v18); + u19 = _mm_sub_epi16(v16, v19); + u20 = _mm_sub_epi16(v23, v20); + u21 = _mm_sub_epi16(v22, v21); + u22 = _mm_add_epi16(v22, v21); + u23 = _mm_add_epi16(v23, v20); + + u24 = _mm_add_epi16(v24, v27); + u25 = _mm_add_epi16(v25, v26); + u26 = _mm_sub_epi16(v25, v26); + u27 = _mm_sub_epi16(v24, v27); + u28 = _mm_sub_epi16(v31, v28); + u29 = _mm_sub_epi16(v30, v29); + u30 = _mm_add_epi16(v29, v30); + u31 = _mm_add_epi16(v28, v31); + + { + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + butterfly_self(&u18, &u29, &stg4_4, &stg4_5); + butterfly_self(&u19, &u28, &stg4_4, &stg4_5); + butterfly_self(&u20, &u27, &stg4_6, &stg4_4); + butterfly_self(&u21, &u26, &stg4_6, &stg4_4); + } + + out[0] = _mm_add_epi16(u16, u23); + out[1] = _mm_add_epi16(u17, u22); + out[2] = _mm_add_epi16(u18, u21); + out[3] = _mm_add_epi16(u19, u20); + v20 = _mm_sub_epi16(u19, u20); + v21 = _mm_sub_epi16(u18, u21); + v22 = _mm_sub_epi16(u17, u22); + v23 = _mm_sub_epi16(u16, u23); + + v24 = _mm_sub_epi16(u31, u24); + v25 = _mm_sub_epi16(u30, u25); + v26 = _mm_sub_epi16(u29, u26); + v27 = _mm_sub_epi16(u28, u27); + out[12] = _mm_add_epi16(u27, u28); + out[13] = _mm_add_epi16(u26, u29); + out[14] = _mm_add_epi16(u25, u30); + out[15] = _mm_add_epi16(u24, u31); + + { + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + butterfly(&v20, &v27, &stg6_0, &stg4_0, &out[4], &out[11]); + butterfly(&v21, &v26, &stg6_0, &stg4_0, &out[5], &out[10]); + butterfly(&v22, &v25, &stg6_0, &stg4_0, &out[6], &out[9]); + butterfly(&v23, &v24, &stg6_0, &stg4_0, &out[7], &out[8]); + } +} + +// 8x16 block, input __m128i in[16], output __m128i in[32] +static void idct32_8x32_135(__m128i *in /*in[32]*/) { + __m128i out[32]; + idct32_8x32_quarter_1_2(in, out); + idct32_8x32_quarter_3_4(in, &out[16]); + add_sub_butterfly(out, in, 32); +} + +static INLINE void store_buffer_8x32(__m128i *in, uint8_t *dst, int stride) { + const __m128i final_rounding = _mm_set1_epi16(1 << 5); + const __m128i zero = _mm_setzero_si128(); + int j = 0; + while (j < 32) { + in[j] = _mm_adds_epi16(in[j], final_rounding); + in[j + 1] = _mm_adds_epi16(in[j + 1], final_rounding); + + in[j] = _mm_srai_epi16(in[j], 6); + in[j + 1] = _mm_srai_epi16(in[j + 1], 6); + + RECON_AND_STORE(dst, in[j]); + dst += stride; + RECON_AND_STORE(dst, in[j + 1]); + dst += stride; + j += 2; + } +} + +static INLINE void recon_and_store(__m128i *in0, __m128i *in1, uint8_t *dest, + int stride) { + store_buffer_8x32(in0, dest, stride); + store_buffer_8x32(in1, dest + 8, stride); +} + +static INLINE void idct32_135(__m128i *col0, __m128i *col1) { + idct32_8x32_135(col0); + idct32_8x32_135(col1); +} + +typedef enum { left_16, right_16 } ColsIndicator; + +static void transpose_and_copy_16x16(__m128i *in0, __m128i *in1, __m128i *store, + ColsIndicator cols) { + switch (cols) { + case left_16: { + int i; + array_transpose_16x16(in0, in1); + for (i = 0; i < 16; ++i) { + store[i] = in0[16 + i]; + store[16 + i] = in1[16 + i]; + } + break; + } + case right_16: { + array_transpose_16x16_2(store, &store[16], in0, in1); + break; + } + default: { assert(0); } + } +} + +// Only upper-left 16x16 has non-zero coeff +void aom_idct32x32_135_add_ssse3(const tran_low_t *input, uint8_t *dest, + int stride) { + // Each array represents an 8x32 block + __m128i col0[32], col1[32]; + // This array represents a 16x16 block + __m128i temp[32]; + + // Load input data. Only need to load the top left 16x16 block. + load_buffer_16x16(input, col0, col1); + + // columns + array_transpose_16x16(col0, col1); + idct32_135(col0, col1); + + // rows + transpose_and_copy_16x16(col0, col1, temp, left_16); + idct32_135(col0, col1); + recon_and_store(col0, col1, dest, stride); + + transpose_and_copy_16x16(col0, col1, temp, right_16); + idct32_135(col0, col1); + recon_and_store(col0, col1, dest + 16, stride); +} + +// For each 8x32 block __m128i in[32], +// Input with index, 2, 6, 10, 14, 18, 22, 26, 30 +// output pixels: 8-15 in __m128i in[32] +static void idct32_full_8x32_quarter_2(const __m128i *in /*in[32]*/, + __m128i *out /*out[16]*/) { + __m128i u8, u9, u10, u11, u12, u13, u14, u15; // stp2_ + __m128i v8, v9, v10, v11, v12, v13, v14, v15; // stp1_ + + { + const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64); + const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64); + const __m128i stg2_2 = pair_set_epi16(cospi_14_64, -cospi_18_64); + const __m128i stg2_3 = pair_set_epi16(cospi_18_64, cospi_14_64); + butterfly(&in[2], &in[30], &stg2_0, &stg2_1, &u8, &u15); + butterfly(&in[18], &in[14], &stg2_2, &stg2_3, &u9, &u14); + } + + v8 = _mm_add_epi16(u8, u9); + v9 = _mm_sub_epi16(u8, u9); + v14 = _mm_sub_epi16(u15, u14); + v15 = _mm_add_epi16(u15, u14); + + { + const __m128i stg2_4 = pair_set_epi16(cospi_22_64, -cospi_10_64); + const __m128i stg2_5 = pair_set_epi16(cospi_10_64, cospi_22_64); + const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64); + const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64); + butterfly(&in[10], &in[22], &stg2_4, &stg2_5, &u10, &u13); + butterfly(&in[26], &in[6], &stg2_6, &stg2_7, &u11, &u12); + } + + v10 = _mm_sub_epi16(u11, u10); + v11 = _mm_add_epi16(u11, u10); + v12 = _mm_add_epi16(u12, u13); + v13 = _mm_sub_epi16(u12, u13); + + { + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + butterfly_self(&v9, &v14, &stg4_4, &stg4_5); + butterfly_self(&v10, &v13, &stg4_6, &stg4_4); + } + + out[0] = _mm_add_epi16(v8, v11); + out[1] = _mm_add_epi16(v9, v10); + out[6] = _mm_add_epi16(v14, v13); + out[7] = _mm_add_epi16(v15, v12); + + out[2] = _mm_sub_epi16(v9, v10); + out[3] = _mm_sub_epi16(v8, v11); + out[4] = _mm_sub_epi16(v15, v12); + out[5] = _mm_sub_epi16(v14, v13); + + { + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + butterfly_self(&out[2], &out[5], &stg6_0, &stg4_0); + butterfly_self(&out[3], &out[4], &stg6_0, &stg4_0); + } +} + +// For each 8x32 block __m128i in[32], +// Input with index, 0, 4, 8, 12, 16, 20, 24, 28 +// output pixels: 0-7 in __m128i in[32] +static void idct32_full_8x32_quarter_1(const __m128i *in /*in[32]*/, + __m128i *out /*out[8]*/) { + __m128i u0, u1, u2, u3, u4, u5, u6, u7; // stp1_ + __m128i v0, v1, v2, v3, v4, v5, v6, v7; // stp2_ + + { + const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i stg3_2 = pair_set_epi16(cospi_12_64, -cospi_20_64); + const __m128i stg3_3 = pair_set_epi16(cospi_20_64, cospi_12_64); + butterfly(&in[4], &in[28], &stg3_0, &stg3_1, &u4, &u7); + butterfly(&in[20], &in[12], &stg3_2, &stg3_3, &u5, &u6); + } + + v4 = _mm_add_epi16(u4, u5); + v5 = _mm_sub_epi16(u4, u5); + v6 = _mm_sub_epi16(u7, u6); + v7 = _mm_add_epi16(u7, u6); + + { + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg4_2 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i stg4_3 = pair_set_epi16(cospi_8_64, cospi_24_64); + butterfly(&v6, &v5, &stg4_1, &stg4_0, &v5, &v6); + + butterfly(&in[0], &in[16], &stg4_0, &stg4_1, &u0, &u1); + butterfly(&in[8], &in[24], &stg4_2, &stg4_3, &u2, &u3); + } + + v0 = _mm_add_epi16(u0, u3); + v1 = _mm_add_epi16(u1, u2); + v2 = _mm_sub_epi16(u1, u2); + v3 = _mm_sub_epi16(u0, u3); + + out[0] = _mm_add_epi16(v0, v7); + out[1] = _mm_add_epi16(v1, v6); + out[2] = _mm_add_epi16(v2, v5); + out[3] = _mm_add_epi16(v3, v4); + out[4] = _mm_sub_epi16(v3, v4); + out[5] = _mm_sub_epi16(v2, v5); + out[6] = _mm_sub_epi16(v1, v6); + out[7] = _mm_sub_epi16(v0, v7); +} + +// For each 8x32 block __m128i in[32], +// Input with odd index, +// 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 +// output pixels: 16-23, 24-31 in __m128i in[32] +// We avoid hide an offset, 16, inside this function. So we output 0-15 into +// array out[16] +static void idct32_full_8x32_quarter_3_4(const __m128i *in /*in[32]*/, + __m128i *out /*out[16]*/) { + __m128i v16, v17, v18, v19, v20, v21, v22, v23; + __m128i v24, v25, v26, v27, v28, v29, v30, v31; + __m128i u16, u17, u18, u19, u20, u21, u22, u23; + __m128i u24, u25, u26, u27, u28, u29, u30, u31; + + { + const __m128i stg1_0 = pair_set_epi16(cospi_31_64, -cospi_1_64); + const __m128i stg1_1 = pair_set_epi16(cospi_1_64, cospi_31_64); + const __m128i stg1_2 = pair_set_epi16(cospi_15_64, -cospi_17_64); + const __m128i stg1_3 = pair_set_epi16(cospi_17_64, cospi_15_64); + const __m128i stg1_4 = pair_set_epi16(cospi_23_64, -cospi_9_64); + const __m128i stg1_5 = pair_set_epi16(cospi_9_64, cospi_23_64); + const __m128i stg1_6 = pair_set_epi16(cospi_7_64, -cospi_25_64); + const __m128i stg1_7 = pair_set_epi16(cospi_25_64, cospi_7_64); + const __m128i stg1_8 = pair_set_epi16(cospi_27_64, -cospi_5_64); + const __m128i stg1_9 = pair_set_epi16(cospi_5_64, cospi_27_64); + const __m128i stg1_10 = pair_set_epi16(cospi_11_64, -cospi_21_64); + const __m128i stg1_11 = pair_set_epi16(cospi_21_64, cospi_11_64); + const __m128i stg1_12 = pair_set_epi16(cospi_19_64, -cospi_13_64); + const __m128i stg1_13 = pair_set_epi16(cospi_13_64, cospi_19_64); + const __m128i stg1_14 = pair_set_epi16(cospi_3_64, -cospi_29_64); + const __m128i stg1_15 = pair_set_epi16(cospi_29_64, cospi_3_64); + butterfly(&in[1], &in[31], &stg1_0, &stg1_1, &u16, &u31); + butterfly(&in[17], &in[15], &stg1_2, &stg1_3, &u17, &u30); + butterfly(&in[9], &in[23], &stg1_4, &stg1_5, &u18, &u29); + butterfly(&in[25], &in[7], &stg1_6, &stg1_7, &u19, &u28); + + butterfly(&in[5], &in[27], &stg1_8, &stg1_9, &u20, &u27); + butterfly(&in[21], &in[11], &stg1_10, &stg1_11, &u21, &u26); + + butterfly(&in[13], &in[19], &stg1_12, &stg1_13, &u22, &u25); + butterfly(&in[29], &in[3], &stg1_14, &stg1_15, &u23, &u24); + } + + v16 = _mm_add_epi16(u16, u17); + v17 = _mm_sub_epi16(u16, u17); + v18 = _mm_sub_epi16(u19, u18); + v19 = _mm_add_epi16(u19, u18); + + v20 = _mm_add_epi16(u20, u21); + v21 = _mm_sub_epi16(u20, u21); + v22 = _mm_sub_epi16(u23, u22); + v23 = _mm_add_epi16(u23, u22); + + v24 = _mm_add_epi16(u24, u25); + v25 = _mm_sub_epi16(u24, u25); + v26 = _mm_sub_epi16(u27, u26); + v27 = _mm_add_epi16(u27, u26); + + v28 = _mm_add_epi16(u28, u29); + v29 = _mm_sub_epi16(u28, u29); + v30 = _mm_sub_epi16(u31, u30); + v31 = _mm_add_epi16(u31, u30); + + { + const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64); + const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64); + butterfly_self(&v17, &v30, &stg3_4, &stg3_5); + butterfly_self(&v18, &v29, &stg3_6, &stg3_4); + butterfly_self(&v21, &v26, &stg3_8, &stg3_9); + butterfly_self(&v22, &v25, &stg3_10, &stg3_8); + } + + u16 = _mm_add_epi16(v16, v19); + u17 = _mm_add_epi16(v17, v18); + u18 = _mm_sub_epi16(v17, v18); + u19 = _mm_sub_epi16(v16, v19); + u20 = _mm_sub_epi16(v23, v20); + u21 = _mm_sub_epi16(v22, v21); + u22 = _mm_add_epi16(v22, v21); + u23 = _mm_add_epi16(v23, v20); + + u24 = _mm_add_epi16(v24, v27); + u25 = _mm_add_epi16(v25, v26); + u26 = _mm_sub_epi16(v25, v26); + u27 = _mm_sub_epi16(v24, v27); + + u28 = _mm_sub_epi16(v31, v28); + u29 = _mm_sub_epi16(v30, v29); + u30 = _mm_add_epi16(v29, v30); + u31 = _mm_add_epi16(v28, v31); + + { + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + butterfly_self(&u18, &u29, &stg4_4, &stg4_5); + butterfly_self(&u19, &u28, &stg4_4, &stg4_5); + butterfly_self(&u20, &u27, &stg4_6, &stg4_4); + butterfly_self(&u21, &u26, &stg4_6, &stg4_4); + } + + out[0] = _mm_add_epi16(u16, u23); + out[1] = _mm_add_epi16(u17, u22); + out[2] = _mm_add_epi16(u18, u21); + out[3] = _mm_add_epi16(u19, u20); + out[4] = _mm_sub_epi16(u19, u20); + out[5] = _mm_sub_epi16(u18, u21); + out[6] = _mm_sub_epi16(u17, u22); + out[7] = _mm_sub_epi16(u16, u23); + + out[8] = _mm_sub_epi16(u31, u24); + out[9] = _mm_sub_epi16(u30, u25); + out[10] = _mm_sub_epi16(u29, u26); + out[11] = _mm_sub_epi16(u28, u27); + out[12] = _mm_add_epi16(u27, u28); + out[13] = _mm_add_epi16(u26, u29); + out[14] = _mm_add_epi16(u25, u30); + out[15] = _mm_add_epi16(u24, u31); + + { + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + butterfly_self(&out[4], &out[11], &stg6_0, &stg4_0); + butterfly_self(&out[5], &out[10], &stg6_0, &stg4_0); + butterfly_self(&out[6], &out[9], &stg6_0, &stg4_0); + butterfly_self(&out[7], &out[8], &stg6_0, &stg4_0); + } +} + +static void idct32_full_8x32_quarter_1_2(const __m128i *in /*in[32]*/, + __m128i *out /*out[32]*/) { + __m128i temp[16]; + idct32_full_8x32_quarter_1(in, temp); + idct32_full_8x32_quarter_2(in, &temp[8]); + add_sub_butterfly(temp, out, 16); +} + +static void idct32_full_8x32(const __m128i *in /*in[32]*/, + __m128i *out /*out[32]*/) { + __m128i temp[32]; + idct32_full_8x32_quarter_1_2(in, temp); + idct32_full_8x32_quarter_3_4(in, &temp[16]); + add_sub_butterfly(temp, out, 32); +} + +static void load_buffer_8x32(const tran_low_t *input, __m128i *in) { + int i; + for (i = 0; i < 8; ++i) { + in[i] = load_input_data(input); + in[i + 8] = load_input_data(input + 8); + in[i + 16] = load_input_data(input + 16); + in[i + 24] = load_input_data(input + 24); + input += 32; + } +} + +void aom_idct32x32_1024_add_ssse3(const tran_low_t *input, uint8_t *dest, + int stride) { + __m128i col[128], in[32]; + int i, j; + + // rows + for (i = 0; i < 4; ++i) { + load_buffer_8x32(input, in); + input += 32 << 3; + + // Transpose 32x8 block to 8x32 block + array_transpose_8x8(in, in); + array_transpose_8x8(in + 8, in + 8); + array_transpose_8x8(in + 16, in + 16); + array_transpose_8x8(in + 24, in + 24); + + idct32_full_8x32(in, col + (i << 5)); + } + + // columns + for (i = 0; i < 4; ++i) { + j = i << 3; + // Transpose 32x8 block to 8x32 block + array_transpose_8x8(col + j, in); + array_transpose_8x8(col + j + 32, in + 8); + array_transpose_8x8(col + j + 64, in + 16); + array_transpose_8x8(col + j + 96, in + 24); + + idct32_full_8x32(in, in); + store_buffer_8x32(in, dest, stride); + dest += 8; + } +} diff --git a/third_party/aom/aom_dsp/x86/inv_wht_sse2.asm b/third_party/aom/aom_dsp/x86/inv_wht_sse2.asm new file mode 100644 index 0000000000..f0668e6f34 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/inv_wht_sse2.asm @@ -0,0 +1,112 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION .text + +%macro REORDER_INPUTS 0 + ; a c d b to a b c d + SWAP 1, 3, 2 +%endmacro + +%macro TRANSFORM_COLS 0 + ; input: + ; m0 a + ; m1 b + ; m2 c + ; m3 d + paddw m0, m2 + psubw m3, m1 + + ; wide subtract + punpcklwd m4, m0 + punpcklwd m5, m3 + psrad m4, 16 + psrad m5, 16 + psubd m4, m5 + psrad m4, 1 + packssdw m4, m4 ; e + + psubw m5, m4, m1 ; b + psubw m4, m2 ; c + psubw m0, m5 + paddw m3, m4 + ; m0 a + SWAP 1, 5 ; m1 b + SWAP 2, 4 ; m2 c + ; m3 d +%endmacro + +%macro TRANSPOSE_4X4 0 + punpcklwd m0, m2 + punpcklwd m1, m3 + mova m2, m0 + punpcklwd m0, m1 + punpckhwd m2, m1 + pshufd m1, m0, 0x0e + pshufd m3, m2, 0x0e +%endmacro + +; transpose a 4x4 int16 matrix in xmm0 and xmm1 to the bottom half of xmm0-xmm3 +%macro TRANSPOSE_4X4_WIDE 0 + mova m3, m0 + punpcklwd m0, m1 + punpckhwd m3, m1 + mova m2, m0 + punpcklwd m0, m3 + punpckhwd m2, m3 + pshufd m1, m0, 0x0e + pshufd m3, m2, 0x0e +%endmacro + +%macro ADD_STORE_4P_2X 5 ; src1, src2, tmp1, tmp2, zero + movd m%3, [outputq] + movd m%4, [outputq + strideq] + punpcklbw m%3, m%5 + punpcklbw m%4, m%5 + paddw m%1, m%3 + paddw m%2, m%4 + packuswb m%1, m%5 + packuswb m%2, m%5 + movd [outputq], m%1 + movd [outputq + strideq], m%2 +%endmacro + +INIT_XMM sse2 +cglobal iwht4x4_16_add, 3, 3, 7, input, output, stride +%if CONFIG_HIGHBITDEPTH + mova m0, [inputq + 0] + packssdw m0, [inputq + 16] + mova m1, [inputq + 32] + packssdw m1, [inputq + 48] +%else + mova m0, [inputq + 0] + mova m1, [inputq + 16] +%endif + psraw m0, 2 + psraw m1, 2 + + TRANSPOSE_4X4_WIDE + REORDER_INPUTS + TRANSFORM_COLS + TRANSPOSE_4X4 + REORDER_INPUTS + TRANSFORM_COLS + + pxor m4, m4 + ADD_STORE_4P_2X 0, 1, 5, 6, 4 + lea outputq, [outputq + 2 * strideq] + ADD_STORE_4P_2X 2, 3, 5, 6, 4 + + RET diff --git a/third_party/aom/aom_dsp/x86/loopfilter_avx2.c b/third_party/aom/aom_dsp/x86/loopfilter_avx2.c new file mode 100644 index 0000000000..bf8150e2a3 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/loopfilter_avx2.c @@ -0,0 +1,915 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include /* AVX2 */ + +#include "./aom_dsp_rtcd.h" +#include "aom_ports/mem.h" + +void aom_lpf_horizontal_edge_8_avx2(unsigned char *s, int p, + const unsigned char *_blimit, + const unsigned char *_limit, + const unsigned char *_thresh) { + __m128i mask, hev, flat, flat2; + const __m128i zero = _mm_set1_epi16(0); + const __m128i one = _mm_set1_epi8(1); + __m128i q7p7, q6p6, q5p5, q4p4, q3p3, q2p2, q1p1, q0p0, p0q0, p1q1; + __m128i abs_p1p0; + + const __m128i thresh = + _mm_broadcastb_epi8(_mm_cvtsi32_si128((int)_thresh[0])); + const __m128i limit = _mm_broadcastb_epi8(_mm_cvtsi32_si128((int)_limit[0])); + const __m128i blimit = + _mm_broadcastb_epi8(_mm_cvtsi32_si128((int)_blimit[0])); + + q4p4 = _mm_loadl_epi64((__m128i *)(s - 5 * p)); + q4p4 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q4p4), (__m64 *)(s + 4 * p))); + q3p3 = _mm_loadl_epi64((__m128i *)(s - 4 * p)); + q3p3 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q3p3), (__m64 *)(s + 3 * p))); + q2p2 = _mm_loadl_epi64((__m128i *)(s - 3 * p)); + q2p2 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q2p2), (__m64 *)(s + 2 * p))); + q1p1 = _mm_loadl_epi64((__m128i *)(s - 2 * p)); + q1p1 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q1p1), (__m64 *)(s + 1 * p))); + p1q1 = _mm_shuffle_epi32(q1p1, 78); + q0p0 = _mm_loadl_epi64((__m128i *)(s - 1 * p)); + q0p0 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q0p0), (__m64 *)(s - 0 * p))); + p0q0 = _mm_shuffle_epi32(q0p0, 78); + + { + __m128i abs_p1q1, abs_p0q0, abs_q1q0, fe, ff, work; + abs_p1p0 = + _mm_or_si128(_mm_subs_epu8(q1p1, q0p0), _mm_subs_epu8(q0p0, q1p1)); + abs_q1q0 = _mm_srli_si128(abs_p1p0, 8); + fe = _mm_set1_epi8(0xfe); + ff = _mm_cmpeq_epi8(abs_p1p0, abs_p1p0); + abs_p0q0 = + _mm_or_si128(_mm_subs_epu8(q0p0, p0q0), _mm_subs_epu8(p0q0, q0p0)); + abs_p1q1 = + _mm_or_si128(_mm_subs_epu8(q1p1, p1q1), _mm_subs_epu8(p1q1, q1p1)); + flat = _mm_max_epu8(abs_p1p0, abs_q1q0); + hev = _mm_subs_epu8(flat, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi8(hev, zero), ff); + + abs_p0q0 = _mm_adds_epu8(abs_p0q0, abs_p0q0); + abs_p1q1 = _mm_srli_epi16(_mm_and_si128(abs_p1q1, fe), 1); + mask = _mm_subs_epu8(_mm_adds_epu8(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi8(mask, zero), ff); + // mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + mask = _mm_max_epu8(abs_p1p0, mask); + // mask |= (abs(p1 - p0) > limit) * -1; + // mask |= (abs(q1 - q0) > limit) * -1; + + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(q2p2, q1p1), _mm_subs_epu8(q1p1, q2p2)), + _mm_or_si128(_mm_subs_epu8(q3p3, q2p2), _mm_subs_epu8(q2p2, q3p3))); + mask = _mm_max_epu8(work, mask); + mask = _mm_max_epu8(mask, _mm_srli_si128(mask, 8)); + mask = _mm_subs_epu8(mask, limit); + mask = _mm_cmpeq_epi8(mask, zero); + } + + // lp filter + { + const __m128i t4 = _mm_set1_epi8(4); + const __m128i t3 = _mm_set1_epi8(3); + const __m128i t80 = _mm_set1_epi8(0x80); + const __m128i t1 = _mm_set1_epi16(0x1); + __m128i qs1ps1 = _mm_xor_si128(q1p1, t80); + __m128i qs0ps0 = _mm_xor_si128(q0p0, t80); + __m128i qs0 = _mm_xor_si128(p0q0, t80); + __m128i qs1 = _mm_xor_si128(p1q1, t80); + __m128i filt; + __m128i work_a; + __m128i filter1, filter2; + __m128i flat2_q6p6, flat2_q5p5, flat2_q4p4, flat2_q3p3, flat2_q2p2; + __m128i flat2_q1p1, flat2_q0p0, flat_q2p2, flat_q1p1, flat_q0p0; + + filt = _mm_and_si128(_mm_subs_epi8(qs1ps1, qs1), hev); + work_a = _mm_subs_epi8(qs0, qs0ps0); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + /* (aom_filter + 3 * (qs0 - ps0)) & mask */ + filt = _mm_and_si128(filt, mask); + + filter1 = _mm_adds_epi8(filt, t4); + filter2 = _mm_adds_epi8(filt, t3); + + filter1 = _mm_unpacklo_epi8(zero, filter1); + filter1 = _mm_srai_epi16(filter1, 0xB); + filter2 = _mm_unpacklo_epi8(zero, filter2); + filter2 = _mm_srai_epi16(filter2, 0xB); + + /* Filter1 >> 3 */ + filt = _mm_packs_epi16(filter2, _mm_subs_epi16(zero, filter1)); + qs0ps0 = _mm_xor_si128(_mm_adds_epi8(qs0ps0, filt), t80); + + /* filt >> 1 */ + filt = _mm_adds_epi16(filter1, t1); + filt = _mm_srai_epi16(filt, 1); + filt = _mm_andnot_si128(_mm_srai_epi16(_mm_unpacklo_epi8(zero, hev), 0x8), + filt); + filt = _mm_packs_epi16(filt, _mm_subs_epi16(zero, filt)); + qs1ps1 = _mm_xor_si128(_mm_adds_epi8(qs1ps1, filt), t80); + // loopfilter done + + { + __m128i work; + flat = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(q2p2, q0p0), _mm_subs_epu8(q0p0, q2p2)), + _mm_or_si128(_mm_subs_epu8(q3p3, q0p0), _mm_subs_epu8(q0p0, q3p3))); + flat = _mm_max_epu8(abs_p1p0, flat); + flat = _mm_max_epu8(flat, _mm_srli_si128(flat, 8)); + flat = _mm_subs_epu8(flat, one); + flat = _mm_cmpeq_epi8(flat, zero); + flat = _mm_and_si128(flat, mask); + + q5p5 = _mm_loadl_epi64((__m128i *)(s - 6 * p)); + q5p5 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q5p5), (__m64 *)(s + 5 * p))); + + q6p6 = _mm_loadl_epi64((__m128i *)(s - 7 * p)); + q6p6 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q6p6), (__m64 *)(s + 6 * p))); + + flat2 = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(q4p4, q0p0), _mm_subs_epu8(q0p0, q4p4)), + _mm_or_si128(_mm_subs_epu8(q5p5, q0p0), _mm_subs_epu8(q0p0, q5p5))); + + q7p7 = _mm_loadl_epi64((__m128i *)(s - 8 * p)); + q7p7 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q7p7), (__m64 *)(s + 7 * p))); + + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(q6p6, q0p0), _mm_subs_epu8(q0p0, q6p6)), + _mm_or_si128(_mm_subs_epu8(q7p7, q0p0), _mm_subs_epu8(q0p0, q7p7))); + + flat2 = _mm_max_epu8(work, flat2); + flat2 = _mm_max_epu8(flat2, _mm_srli_si128(flat2, 8)); + flat2 = _mm_subs_epu8(flat2, one); + flat2 = _mm_cmpeq_epi8(flat2, zero); + flat2 = _mm_and_si128(flat2, flat); // flat2 & flat & mask + } + + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // flat and wide flat calculations + { + const __m128i eight = _mm_set1_epi16(8); + const __m128i four = _mm_set1_epi16(4); + __m128i p7_16, p6_16, p5_16, p4_16, p3_16, p2_16, p1_16, p0_16; + __m128i q7_16, q6_16, q5_16, q4_16, q3_16, q2_16, q1_16, q0_16; + __m128i pixelFilter_p, pixelFilter_q; + __m128i pixetFilter_p2p1p0, pixetFilter_q2q1q0; + __m128i sum_p7, sum_q7, sum_p3, sum_q3, res_p, res_q; + + p7_16 = _mm_unpacklo_epi8(q7p7, zero); + p6_16 = _mm_unpacklo_epi8(q6p6, zero); + p5_16 = _mm_unpacklo_epi8(q5p5, zero); + p4_16 = _mm_unpacklo_epi8(q4p4, zero); + p3_16 = _mm_unpacklo_epi8(q3p3, zero); + p2_16 = _mm_unpacklo_epi8(q2p2, zero); + p1_16 = _mm_unpacklo_epi8(q1p1, zero); + p0_16 = _mm_unpacklo_epi8(q0p0, zero); + q0_16 = _mm_unpackhi_epi8(q0p0, zero); + q1_16 = _mm_unpackhi_epi8(q1p1, zero); + q2_16 = _mm_unpackhi_epi8(q2p2, zero); + q3_16 = _mm_unpackhi_epi8(q3p3, zero); + q4_16 = _mm_unpackhi_epi8(q4p4, zero); + q5_16 = _mm_unpackhi_epi8(q5p5, zero); + q6_16 = _mm_unpackhi_epi8(q6p6, zero); + q7_16 = _mm_unpackhi_epi8(q7p7, zero); + + pixelFilter_p = _mm_add_epi16(_mm_add_epi16(p6_16, p5_16), + _mm_add_epi16(p4_16, p3_16)); + pixelFilter_q = _mm_add_epi16(_mm_add_epi16(q6_16, q5_16), + _mm_add_epi16(q4_16, q3_16)); + + pixetFilter_p2p1p0 = _mm_add_epi16(p0_16, _mm_add_epi16(p2_16, p1_16)); + pixelFilter_p = _mm_add_epi16(pixelFilter_p, pixetFilter_p2p1p0); + + pixetFilter_q2q1q0 = _mm_add_epi16(q0_16, _mm_add_epi16(q2_16, q1_16)); + pixelFilter_q = _mm_add_epi16(pixelFilter_q, pixetFilter_q2q1q0); + pixelFilter_p = + _mm_add_epi16(eight, _mm_add_epi16(pixelFilter_p, pixelFilter_q)); + pixetFilter_p2p1p0 = _mm_add_epi16( + four, _mm_add_epi16(pixetFilter_p2p1p0, pixetFilter_q2q1q0)); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(p7_16, p0_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(q7_16, q0_16)), 4); + flat2_q0p0 = _mm_packus_epi16(res_p, res_q); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_p2p1p0, _mm_add_epi16(p3_16, p0_16)), 3); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_p2p1p0, _mm_add_epi16(q3_16, q0_16)), 3); + + flat_q0p0 = _mm_packus_epi16(res_p, res_q); + + sum_p7 = _mm_add_epi16(p7_16, p7_16); + sum_q7 = _mm_add_epi16(q7_16, q7_16); + sum_p3 = _mm_add_epi16(p3_16, p3_16); + sum_q3 = _mm_add_epi16(q3_16, q3_16); + + pixelFilter_q = _mm_sub_epi16(pixelFilter_p, p6_16); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q6_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p1_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q1_16)), 4); + flat2_q1p1 = _mm_packus_epi16(res_p, res_q); + + pixetFilter_q2q1q0 = _mm_sub_epi16(pixetFilter_p2p1p0, p2_16); + pixetFilter_p2p1p0 = _mm_sub_epi16(pixetFilter_p2p1p0, q2_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_p2p1p0, _mm_add_epi16(sum_p3, p1_16)), 3); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_q2q1q0, _mm_add_epi16(sum_q3, q1_16)), 3); + flat_q1p1 = _mm_packus_epi16(res_p, res_q); + + sum_p7 = _mm_add_epi16(sum_p7, p7_16); + sum_q7 = _mm_add_epi16(sum_q7, q7_16); + sum_p3 = _mm_add_epi16(sum_p3, p3_16); + sum_q3 = _mm_add_epi16(sum_q3, q3_16); + + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q5_16); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p5_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p2_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q2_16)), 4); + flat2_q2p2 = _mm_packus_epi16(res_p, res_q); + + pixetFilter_p2p1p0 = _mm_sub_epi16(pixetFilter_p2p1p0, q1_16); + pixetFilter_q2q1q0 = _mm_sub_epi16(pixetFilter_q2q1q0, p1_16); + + res_p = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_p2p1p0, _mm_add_epi16(sum_p3, p2_16)), 3); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_q2q1q0, _mm_add_epi16(sum_q3, q2_16)), 3); + flat_q2p2 = _mm_packus_epi16(res_p, res_q); + + sum_p7 = _mm_add_epi16(sum_p7, p7_16); + sum_q7 = _mm_add_epi16(sum_q7, q7_16); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q4_16); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p4_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p3_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q3_16)), 4); + flat2_q3p3 = _mm_packus_epi16(res_p, res_q); + + sum_p7 = _mm_add_epi16(sum_p7, p7_16); + sum_q7 = _mm_add_epi16(sum_q7, q7_16); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q3_16); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p3_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p4_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q4_16)), 4); + flat2_q4p4 = _mm_packus_epi16(res_p, res_q); + + sum_p7 = _mm_add_epi16(sum_p7, p7_16); + sum_q7 = _mm_add_epi16(sum_q7, q7_16); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q2_16); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p2_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p5_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q5_16)), 4); + flat2_q5p5 = _mm_packus_epi16(res_p, res_q); + + sum_p7 = _mm_add_epi16(sum_p7, p7_16); + sum_q7 = _mm_add_epi16(sum_q7, q7_16); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q1_16); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p1_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p6_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q6_16)), 4); + flat2_q6p6 = _mm_packus_epi16(res_p, res_q); + } + // wide flat + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + flat = _mm_shuffle_epi32(flat, 68); + flat2 = _mm_shuffle_epi32(flat2, 68); + + q2p2 = _mm_andnot_si128(flat, q2p2); + flat_q2p2 = _mm_and_si128(flat, flat_q2p2); + q2p2 = _mm_or_si128(q2p2, flat_q2p2); + + qs1ps1 = _mm_andnot_si128(flat, qs1ps1); + flat_q1p1 = _mm_and_si128(flat, flat_q1p1); + q1p1 = _mm_or_si128(qs1ps1, flat_q1p1); + + qs0ps0 = _mm_andnot_si128(flat, qs0ps0); + flat_q0p0 = _mm_and_si128(flat, flat_q0p0); + q0p0 = _mm_or_si128(qs0ps0, flat_q0p0); + + q6p6 = _mm_andnot_si128(flat2, q6p6); + flat2_q6p6 = _mm_and_si128(flat2, flat2_q6p6); + q6p6 = _mm_or_si128(q6p6, flat2_q6p6); + _mm_storel_epi64((__m128i *)(s - 7 * p), q6p6); + _mm_storeh_pi((__m64 *)(s + 6 * p), _mm_castsi128_ps(q6p6)); + + q5p5 = _mm_andnot_si128(flat2, q5p5); + flat2_q5p5 = _mm_and_si128(flat2, flat2_q5p5); + q5p5 = _mm_or_si128(q5p5, flat2_q5p5); + _mm_storel_epi64((__m128i *)(s - 6 * p), q5p5); + _mm_storeh_pi((__m64 *)(s + 5 * p), _mm_castsi128_ps(q5p5)); + + q4p4 = _mm_andnot_si128(flat2, q4p4); + flat2_q4p4 = _mm_and_si128(flat2, flat2_q4p4); + q4p4 = _mm_or_si128(q4p4, flat2_q4p4); + _mm_storel_epi64((__m128i *)(s - 5 * p), q4p4); + _mm_storeh_pi((__m64 *)(s + 4 * p), _mm_castsi128_ps(q4p4)); + + q3p3 = _mm_andnot_si128(flat2, q3p3); + flat2_q3p3 = _mm_and_si128(flat2, flat2_q3p3); + q3p3 = _mm_or_si128(q3p3, flat2_q3p3); + _mm_storel_epi64((__m128i *)(s - 4 * p), q3p3); + _mm_storeh_pi((__m64 *)(s + 3 * p), _mm_castsi128_ps(q3p3)); + + q2p2 = _mm_andnot_si128(flat2, q2p2); + flat2_q2p2 = _mm_and_si128(flat2, flat2_q2p2); + q2p2 = _mm_or_si128(q2p2, flat2_q2p2); + _mm_storel_epi64((__m128i *)(s - 3 * p), q2p2); + _mm_storeh_pi((__m64 *)(s + 2 * p), _mm_castsi128_ps(q2p2)); + + q1p1 = _mm_andnot_si128(flat2, q1p1); + flat2_q1p1 = _mm_and_si128(flat2, flat2_q1p1); + q1p1 = _mm_or_si128(q1p1, flat2_q1p1); + _mm_storel_epi64((__m128i *)(s - 2 * p), q1p1); + _mm_storeh_pi((__m64 *)(s + 1 * p), _mm_castsi128_ps(q1p1)); + + q0p0 = _mm_andnot_si128(flat2, q0p0); + flat2_q0p0 = _mm_and_si128(flat2, flat2_q0p0); + q0p0 = _mm_or_si128(q0p0, flat2_q0p0); + _mm_storel_epi64((__m128i *)(s - 1 * p), q0p0); + _mm_storeh_pi((__m64 *)(s - 0 * p), _mm_castsi128_ps(q0p0)); + } +} + +DECLARE_ALIGNED(32, static const uint8_t, filt_loopfilter_avx2[32]) = { + 0, 128, 1, 128, 2, 128, 3, 128, 4, 128, 5, 128, 6, 128, 7, 128, + 8, 128, 9, 128, 10, 128, 11, 128, 12, 128, 13, 128, 14, 128, 15, 128 +}; + +void aom_lpf_horizontal_edge_16_avx2(unsigned char *s, int p, + const unsigned char *_blimit, + const unsigned char *_limit, + const unsigned char *_thresh) { + __m128i mask, hev, flat, flat2; + const __m128i zero = _mm_set1_epi16(0); + const __m128i one = _mm_set1_epi8(1); + __m128i p7, p6, p5; + __m128i p4, p3, p2, p1, p0, q0, q1, q2, q3, q4; + __m128i q5, q6, q7; + __m256i p256_7, q256_7, p256_6, q256_6, p256_5, q256_5, p256_4, q256_4, + p256_3, q256_3, p256_2, q256_2, p256_1, q256_1, p256_0, q256_0; + + const __m128i thresh = + _mm_broadcastb_epi8(_mm_cvtsi32_si128((int)_thresh[0])); + const __m128i limit = _mm_broadcastb_epi8(_mm_cvtsi32_si128((int)_limit[0])); + const __m128i blimit = + _mm_broadcastb_epi8(_mm_cvtsi32_si128((int)_blimit[0])); + + p256_4 = + _mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s - 5 * p))); + p256_3 = + _mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s - 4 * p))); + p256_2 = + _mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s - 3 * p))); + p256_1 = + _mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s - 2 * p))); + p256_0 = + _mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s - 1 * p))); + q256_0 = + _mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s - 0 * p))); + q256_1 = + _mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s + 1 * p))); + q256_2 = + _mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s + 2 * p))); + q256_3 = + _mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s + 3 * p))); + q256_4 = + _mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s + 4 * p))); + + p4 = _mm256_castsi256_si128(p256_4); + p3 = _mm256_castsi256_si128(p256_3); + p2 = _mm256_castsi256_si128(p256_2); + p1 = _mm256_castsi256_si128(p256_1); + p0 = _mm256_castsi256_si128(p256_0); + q0 = _mm256_castsi256_si128(q256_0); + q1 = _mm256_castsi256_si128(q256_1); + q2 = _mm256_castsi256_si128(q256_2); + q3 = _mm256_castsi256_si128(q256_3); + q4 = _mm256_castsi256_si128(q256_4); + + { + const __m128i abs_p1p0 = + _mm_or_si128(_mm_subs_epu8(p1, p0), _mm_subs_epu8(p0, p1)); + const __m128i abs_q1q0 = + _mm_or_si128(_mm_subs_epu8(q1, q0), _mm_subs_epu8(q0, q1)); + const __m128i fe = _mm_set1_epi8(0xfe); + const __m128i ff = _mm_cmpeq_epi8(abs_p1p0, abs_p1p0); + __m128i abs_p0q0 = + _mm_or_si128(_mm_subs_epu8(p0, q0), _mm_subs_epu8(q0, p0)); + __m128i abs_p1q1 = + _mm_or_si128(_mm_subs_epu8(p1, q1), _mm_subs_epu8(q1, p1)); + __m128i work; + flat = _mm_max_epu8(abs_p1p0, abs_q1q0); + hev = _mm_subs_epu8(flat, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi8(hev, zero), ff); + + abs_p0q0 = _mm_adds_epu8(abs_p0q0, abs_p0q0); + abs_p1q1 = _mm_srli_epi16(_mm_and_si128(abs_p1q1, fe), 1); + mask = _mm_subs_epu8(_mm_adds_epu8(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi8(mask, zero), ff); + // mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + mask = _mm_max_epu8(flat, mask); + // mask |= (abs(p1 - p0) > limit) * -1; + // mask |= (abs(q1 - q0) > limit) * -1; + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(p2, p1), _mm_subs_epu8(p1, p2)), + _mm_or_si128(_mm_subs_epu8(p3, p2), _mm_subs_epu8(p2, p3))); + mask = _mm_max_epu8(work, mask); + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(q2, q1), _mm_subs_epu8(q1, q2)), + _mm_or_si128(_mm_subs_epu8(q3, q2), _mm_subs_epu8(q2, q3))); + mask = _mm_max_epu8(work, mask); + mask = _mm_subs_epu8(mask, limit); + mask = _mm_cmpeq_epi8(mask, zero); + } + + // lp filter + { + const __m128i t4 = _mm_set1_epi8(4); + const __m128i t3 = _mm_set1_epi8(3); + const __m128i t80 = _mm_set1_epi8(0x80); + const __m128i te0 = _mm_set1_epi8(0xe0); + const __m128i t1f = _mm_set1_epi8(0x1f); + const __m128i t1 = _mm_set1_epi8(0x1); + const __m128i t7f = _mm_set1_epi8(0x7f); + + __m128i ps1 = _mm_xor_si128(p1, t80); + __m128i ps0 = _mm_xor_si128(p0, t80); + __m128i qs0 = _mm_xor_si128(q0, t80); + __m128i qs1 = _mm_xor_si128(q1, t80); + __m128i filt; + __m128i work_a; + __m128i filter1, filter2; + __m128i flat2_p6, flat2_p5, flat2_p4, flat2_p3, flat2_p2, flat2_p1, + flat2_p0, flat2_q0, flat2_q1, flat2_q2, flat2_q3, flat2_q4, flat2_q5, + flat2_q6, flat_p2, flat_p1, flat_p0, flat_q0, flat_q1, flat_q2; + + filt = _mm_and_si128(_mm_subs_epi8(ps1, qs1), hev); + work_a = _mm_subs_epi8(qs0, ps0); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + /* (aom_filter + 3 * (qs0 - ps0)) & mask */ + filt = _mm_and_si128(filt, mask); + + filter1 = _mm_adds_epi8(filt, t4); + filter2 = _mm_adds_epi8(filt, t3); + + /* Filter1 >> 3 */ + work_a = _mm_cmpgt_epi8(zero, filter1); + filter1 = _mm_srli_epi16(filter1, 3); + work_a = _mm_and_si128(work_a, te0); + filter1 = _mm_and_si128(filter1, t1f); + filter1 = _mm_or_si128(filter1, work_a); + qs0 = _mm_xor_si128(_mm_subs_epi8(qs0, filter1), t80); + + /* Filter2 >> 3 */ + work_a = _mm_cmpgt_epi8(zero, filter2); + filter2 = _mm_srli_epi16(filter2, 3); + work_a = _mm_and_si128(work_a, te0); + filter2 = _mm_and_si128(filter2, t1f); + filter2 = _mm_or_si128(filter2, work_a); + ps0 = _mm_xor_si128(_mm_adds_epi8(ps0, filter2), t80); + + /* filt >> 1 */ + filt = _mm_adds_epi8(filter1, t1); + work_a = _mm_cmpgt_epi8(zero, filt); + filt = _mm_srli_epi16(filt, 1); + work_a = _mm_and_si128(work_a, t80); + filt = _mm_and_si128(filt, t7f); + filt = _mm_or_si128(filt, work_a); + filt = _mm_andnot_si128(hev, filt); + ps1 = _mm_xor_si128(_mm_adds_epi8(ps1, filt), t80); + qs1 = _mm_xor_si128(_mm_subs_epi8(qs1, filt), t80); + // loopfilter done + + { + __m128i work; + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(p2, p0), _mm_subs_epu8(p0, p2)), + _mm_or_si128(_mm_subs_epu8(q2, q0), _mm_subs_epu8(q0, q2))); + flat = _mm_max_epu8(work, flat); + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(p3, p0), _mm_subs_epu8(p0, p3)), + _mm_or_si128(_mm_subs_epu8(q3, q0), _mm_subs_epu8(q0, q3))); + flat = _mm_max_epu8(work, flat); + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(p4, p0), _mm_subs_epu8(p0, p4)), + _mm_or_si128(_mm_subs_epu8(q4, q0), _mm_subs_epu8(q0, q4))); + flat = _mm_subs_epu8(flat, one); + flat = _mm_cmpeq_epi8(flat, zero); + flat = _mm_and_si128(flat, mask); + + p256_5 = _mm256_castpd_si256( + _mm256_broadcast_pd((__m128d const *)(s - 6 * p))); + q256_5 = _mm256_castpd_si256( + _mm256_broadcast_pd((__m128d const *)(s + 5 * p))); + p5 = _mm256_castsi256_si128(p256_5); + q5 = _mm256_castsi256_si128(q256_5); + flat2 = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(p5, p0), _mm_subs_epu8(p0, p5)), + _mm_or_si128(_mm_subs_epu8(q5, q0), _mm_subs_epu8(q0, q5))); + + flat2 = _mm_max_epu8(work, flat2); + p256_6 = _mm256_castpd_si256( + _mm256_broadcast_pd((__m128d const *)(s - 7 * p))); + q256_6 = _mm256_castpd_si256( + _mm256_broadcast_pd((__m128d const *)(s + 6 * p))); + p6 = _mm256_castsi256_si128(p256_6); + q6 = _mm256_castsi256_si128(q256_6); + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(p6, p0), _mm_subs_epu8(p0, p6)), + _mm_or_si128(_mm_subs_epu8(q6, q0), _mm_subs_epu8(q0, q6))); + + flat2 = _mm_max_epu8(work, flat2); + + p256_7 = _mm256_castpd_si256( + _mm256_broadcast_pd((__m128d const *)(s - 8 * p))); + q256_7 = _mm256_castpd_si256( + _mm256_broadcast_pd((__m128d const *)(s + 7 * p))); + p7 = _mm256_castsi256_si128(p256_7); + q7 = _mm256_castsi256_si128(q256_7); + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(p7, p0), _mm_subs_epu8(p0, p7)), + _mm_or_si128(_mm_subs_epu8(q7, q0), _mm_subs_epu8(q0, q7))); + + flat2 = _mm_max_epu8(work, flat2); + flat2 = _mm_subs_epu8(flat2, one); + flat2 = _mm_cmpeq_epi8(flat2, zero); + flat2 = _mm_and_si128(flat2, flat); // flat2 & flat & mask + } + + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // flat and wide flat calculations + { + const __m256i eight = _mm256_set1_epi16(8); + const __m256i four = _mm256_set1_epi16(4); + __m256i pixelFilter_p, pixelFilter_q, pixetFilter_p2p1p0, + pixetFilter_q2q1q0, sum_p7, sum_q7, sum_p3, sum_q3, res_p, res_q; + + const __m256i filter = + _mm256_load_si256((__m256i const *)filt_loopfilter_avx2); + p256_7 = _mm256_shuffle_epi8(p256_7, filter); + p256_6 = _mm256_shuffle_epi8(p256_6, filter); + p256_5 = _mm256_shuffle_epi8(p256_5, filter); + p256_4 = _mm256_shuffle_epi8(p256_4, filter); + p256_3 = _mm256_shuffle_epi8(p256_3, filter); + p256_2 = _mm256_shuffle_epi8(p256_2, filter); + p256_1 = _mm256_shuffle_epi8(p256_1, filter); + p256_0 = _mm256_shuffle_epi8(p256_0, filter); + q256_0 = _mm256_shuffle_epi8(q256_0, filter); + q256_1 = _mm256_shuffle_epi8(q256_1, filter); + q256_2 = _mm256_shuffle_epi8(q256_2, filter); + q256_3 = _mm256_shuffle_epi8(q256_3, filter); + q256_4 = _mm256_shuffle_epi8(q256_4, filter); + q256_5 = _mm256_shuffle_epi8(q256_5, filter); + q256_6 = _mm256_shuffle_epi8(q256_6, filter); + q256_7 = _mm256_shuffle_epi8(q256_7, filter); + + pixelFilter_p = _mm256_add_epi16(_mm256_add_epi16(p256_6, p256_5), + _mm256_add_epi16(p256_4, p256_3)); + pixelFilter_q = _mm256_add_epi16(_mm256_add_epi16(q256_6, q256_5), + _mm256_add_epi16(q256_4, q256_3)); + + pixetFilter_p2p1p0 = + _mm256_add_epi16(p256_0, _mm256_add_epi16(p256_2, p256_1)); + pixelFilter_p = _mm256_add_epi16(pixelFilter_p, pixetFilter_p2p1p0); + + pixetFilter_q2q1q0 = + _mm256_add_epi16(q256_0, _mm256_add_epi16(q256_2, q256_1)); + pixelFilter_q = _mm256_add_epi16(pixelFilter_q, pixetFilter_q2q1q0); + + pixelFilter_p = _mm256_add_epi16( + eight, _mm256_add_epi16(pixelFilter_p, pixelFilter_q)); + + pixetFilter_p2p1p0 = _mm256_add_epi16( + four, _mm256_add_epi16(pixetFilter_p2p1p0, pixetFilter_q2q1q0)); + + res_p = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(p256_7, p256_0)), 4); + + flat2_p0 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168)); + + res_q = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(q256_7, q256_0)), 4); + + flat2_q0 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168)); + + res_p = + _mm256_srli_epi16(_mm256_add_epi16(pixetFilter_p2p1p0, + _mm256_add_epi16(p256_3, p256_0)), + 3); + + flat_p0 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168)); + + res_q = + _mm256_srli_epi16(_mm256_add_epi16(pixetFilter_p2p1p0, + _mm256_add_epi16(q256_3, q256_0)), + 3); + + flat_q0 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168)); + + sum_p7 = _mm256_add_epi16(p256_7, p256_7); + + sum_q7 = _mm256_add_epi16(q256_7, q256_7); + + sum_p3 = _mm256_add_epi16(p256_3, p256_3); + + sum_q3 = _mm256_add_epi16(q256_3, q256_3); + + pixelFilter_q = _mm256_sub_epi16(pixelFilter_p, p256_6); + + pixelFilter_p = _mm256_sub_epi16(pixelFilter_p, q256_6); + + res_p = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(sum_p7, p256_1)), 4); + + flat2_p1 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168)); + + res_q = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_q, _mm256_add_epi16(sum_q7, q256_1)), 4); + + flat2_q1 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168)); + + pixetFilter_q2q1q0 = _mm256_sub_epi16(pixetFilter_p2p1p0, p256_2); + + pixetFilter_p2p1p0 = _mm256_sub_epi16(pixetFilter_p2p1p0, q256_2); + + res_p = + _mm256_srli_epi16(_mm256_add_epi16(pixetFilter_p2p1p0, + _mm256_add_epi16(sum_p3, p256_1)), + 3); + + flat_p1 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168)); + + res_q = + _mm256_srli_epi16(_mm256_add_epi16(pixetFilter_q2q1q0, + _mm256_add_epi16(sum_q3, q256_1)), + 3); + + flat_q1 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168)); + + sum_p7 = _mm256_add_epi16(sum_p7, p256_7); + + sum_q7 = _mm256_add_epi16(sum_q7, q256_7); + + sum_p3 = _mm256_add_epi16(sum_p3, p256_3); + + sum_q3 = _mm256_add_epi16(sum_q3, q256_3); + + pixelFilter_p = _mm256_sub_epi16(pixelFilter_p, q256_5); + + pixelFilter_q = _mm256_sub_epi16(pixelFilter_q, p256_5); + + res_p = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(sum_p7, p256_2)), 4); + + flat2_p2 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168)); + + res_q = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_q, _mm256_add_epi16(sum_q7, q256_2)), 4); + + flat2_q2 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168)); + + pixetFilter_p2p1p0 = _mm256_sub_epi16(pixetFilter_p2p1p0, q256_1); + + pixetFilter_q2q1q0 = _mm256_sub_epi16(pixetFilter_q2q1q0, p256_1); + + res_p = + _mm256_srli_epi16(_mm256_add_epi16(pixetFilter_p2p1p0, + _mm256_add_epi16(sum_p3, p256_2)), + 3); + + flat_p2 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168)); + + res_q = + _mm256_srli_epi16(_mm256_add_epi16(pixetFilter_q2q1q0, + _mm256_add_epi16(sum_q3, q256_2)), + 3); + + flat_q2 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168)); + + sum_p7 = _mm256_add_epi16(sum_p7, p256_7); + + sum_q7 = _mm256_add_epi16(sum_q7, q256_7); + + pixelFilter_p = _mm256_sub_epi16(pixelFilter_p, q256_4); + + pixelFilter_q = _mm256_sub_epi16(pixelFilter_q, p256_4); + + res_p = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(sum_p7, p256_3)), 4); + + flat2_p3 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168)); + + res_q = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_q, _mm256_add_epi16(sum_q7, q256_3)), 4); + + flat2_q3 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168)); + + sum_p7 = _mm256_add_epi16(sum_p7, p256_7); + + sum_q7 = _mm256_add_epi16(sum_q7, q256_7); + + pixelFilter_p = _mm256_sub_epi16(pixelFilter_p, q256_3); + + pixelFilter_q = _mm256_sub_epi16(pixelFilter_q, p256_3); + + res_p = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(sum_p7, p256_4)), 4); + + flat2_p4 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168)); + + res_q = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_q, _mm256_add_epi16(sum_q7, q256_4)), 4); + + flat2_q4 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168)); + + sum_p7 = _mm256_add_epi16(sum_p7, p256_7); + + sum_q7 = _mm256_add_epi16(sum_q7, q256_7); + + pixelFilter_p = _mm256_sub_epi16(pixelFilter_p, q256_2); + + pixelFilter_q = _mm256_sub_epi16(pixelFilter_q, p256_2); + + res_p = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(sum_p7, p256_5)), 4); + + flat2_p5 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168)); + + res_q = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_q, _mm256_add_epi16(sum_q7, q256_5)), 4); + + flat2_q5 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168)); + + sum_p7 = _mm256_add_epi16(sum_p7, p256_7); + + sum_q7 = _mm256_add_epi16(sum_q7, q256_7); + + pixelFilter_p = _mm256_sub_epi16(pixelFilter_p, q256_1); + + pixelFilter_q = _mm256_sub_epi16(pixelFilter_q, p256_1); + + res_p = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(sum_p7, p256_6)), 4); + + flat2_p6 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168)); + + res_q = _mm256_srli_epi16( + _mm256_add_epi16(pixelFilter_q, _mm256_add_epi16(sum_q7, q256_6)), 4); + + flat2_q6 = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168)); + } + + // wide flat + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + p2 = _mm_andnot_si128(flat, p2); + flat_p2 = _mm_and_si128(flat, flat_p2); + p2 = _mm_or_si128(flat_p2, p2); + + p1 = _mm_andnot_si128(flat, ps1); + flat_p1 = _mm_and_si128(flat, flat_p1); + p1 = _mm_or_si128(flat_p1, p1); + + p0 = _mm_andnot_si128(flat, ps0); + flat_p0 = _mm_and_si128(flat, flat_p0); + p0 = _mm_or_si128(flat_p0, p0); + + q0 = _mm_andnot_si128(flat, qs0); + flat_q0 = _mm_and_si128(flat, flat_q0); + q0 = _mm_or_si128(flat_q0, q0); + + q1 = _mm_andnot_si128(flat, qs1); + flat_q1 = _mm_and_si128(flat, flat_q1); + q1 = _mm_or_si128(flat_q1, q1); + + q2 = _mm_andnot_si128(flat, q2); + flat_q2 = _mm_and_si128(flat, flat_q2); + q2 = _mm_or_si128(flat_q2, q2); + + p6 = _mm_andnot_si128(flat2, p6); + flat2_p6 = _mm_and_si128(flat2, flat2_p6); + p6 = _mm_or_si128(flat2_p6, p6); + _mm_storeu_si128((__m128i *)(s - 7 * p), p6); + + p5 = _mm_andnot_si128(flat2, p5); + flat2_p5 = _mm_and_si128(flat2, flat2_p5); + p5 = _mm_or_si128(flat2_p5, p5); + _mm_storeu_si128((__m128i *)(s - 6 * p), p5); + + p4 = _mm_andnot_si128(flat2, p4); + flat2_p4 = _mm_and_si128(flat2, flat2_p4); + p4 = _mm_or_si128(flat2_p4, p4); + _mm_storeu_si128((__m128i *)(s - 5 * p), p4); + + p3 = _mm_andnot_si128(flat2, p3); + flat2_p3 = _mm_and_si128(flat2, flat2_p3); + p3 = _mm_or_si128(flat2_p3, p3); + _mm_storeu_si128((__m128i *)(s - 4 * p), p3); + + p2 = _mm_andnot_si128(flat2, p2); + flat2_p2 = _mm_and_si128(flat2, flat2_p2); + p2 = _mm_or_si128(flat2_p2, p2); + _mm_storeu_si128((__m128i *)(s - 3 * p), p2); + + p1 = _mm_andnot_si128(flat2, p1); + flat2_p1 = _mm_and_si128(flat2, flat2_p1); + p1 = _mm_or_si128(flat2_p1, p1); + _mm_storeu_si128((__m128i *)(s - 2 * p), p1); + + p0 = _mm_andnot_si128(flat2, p0); + flat2_p0 = _mm_and_si128(flat2, flat2_p0); + p0 = _mm_or_si128(flat2_p0, p0); + _mm_storeu_si128((__m128i *)(s - 1 * p), p0); + + q0 = _mm_andnot_si128(flat2, q0); + flat2_q0 = _mm_and_si128(flat2, flat2_q0); + q0 = _mm_or_si128(flat2_q0, q0); + _mm_storeu_si128((__m128i *)(s - 0 * p), q0); + + q1 = _mm_andnot_si128(flat2, q1); + flat2_q1 = _mm_and_si128(flat2, flat2_q1); + q1 = _mm_or_si128(flat2_q1, q1); + _mm_storeu_si128((__m128i *)(s + 1 * p), q1); + + q2 = _mm_andnot_si128(flat2, q2); + flat2_q2 = _mm_and_si128(flat2, flat2_q2); + q2 = _mm_or_si128(flat2_q2, q2); + _mm_storeu_si128((__m128i *)(s + 2 * p), q2); + + q3 = _mm_andnot_si128(flat2, q3); + flat2_q3 = _mm_and_si128(flat2, flat2_q3); + q3 = _mm_or_si128(flat2_q3, q3); + _mm_storeu_si128((__m128i *)(s + 3 * p), q3); + + q4 = _mm_andnot_si128(flat2, q4); + flat2_q4 = _mm_and_si128(flat2, flat2_q4); + q4 = _mm_or_si128(flat2_q4, q4); + _mm_storeu_si128((__m128i *)(s + 4 * p), q4); + + q5 = _mm_andnot_si128(flat2, q5); + flat2_q5 = _mm_and_si128(flat2, flat2_q5); + q5 = _mm_or_si128(flat2_q5, q5); + _mm_storeu_si128((__m128i *)(s + 5 * p), q5); + + q6 = _mm_andnot_si128(flat2, q6); + flat2_q6 = _mm_and_si128(flat2, flat2_q6); + q6 = _mm_or_si128(flat2_q6, q6); + _mm_storeu_si128((__m128i *)(s + 6 * p), q6); + } + _mm256_zeroupper(); +} diff --git a/third_party/aom/aom_dsp/x86/loopfilter_sse2.c b/third_party/aom/aom_dsp/x86/loopfilter_sse2.c new file mode 100644 index 0000000000..7e134dc63f --- /dev/null +++ b/third_party/aom/aom_dsp/x86/loopfilter_sse2.c @@ -0,0 +1,1892 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // SSE2 + +#include "./aom_dsp_rtcd.h" +#include "aom_ports/mem.h" +#include "aom_ports/emmintrin_compat.h" + +static INLINE __m128i abs_diff(__m128i a, __m128i b) { + return _mm_or_si128(_mm_subs_epu8(a, b), _mm_subs_epu8(b, a)); +} + +#if CONFIG_PARALLEL_DEBLOCKING +// filter_mask and hev_mask +#define FILTER_HEV_MASK4 \ + do { \ + /* (abs(q1 - q0), abs(p1 - p0) */ \ + __m128i flat = abs_diff(q1p1, q0p0); \ + /* abs(p1 - q1), abs(p0 - q0) */ \ + const __m128i abs_p1q1p0q0 = abs_diff(p1p0, q1q0); \ + __m128i abs_p0q0, abs_p1q1; \ + \ + /* const uint8_t hev = hev_mask(thresh, *op1, *op0, *oq0, *oq1); */ \ + hev = \ + _mm_unpacklo_epi8(_mm_max_epu8(flat, _mm_srli_si128(flat, 8)), zero); \ + hev = _mm_cmpgt_epi16(hev, thresh); \ + hev = _mm_packs_epi16(hev, hev); \ + \ + /* const int8_t mask = filter_mask2(*limit, *blimit, */ \ + /* p1, p0, q0, q1); */ \ + abs_p0q0 = \ + _mm_adds_epu8(abs_p1q1p0q0, abs_p1q1p0q0); /* abs(p0 - q0) * 2 */ \ + abs_p1q1 = \ + _mm_unpackhi_epi8(abs_p1q1p0q0, abs_p1q1p0q0); /* abs(p1 - q1) */ \ + abs_p1q1 = _mm_srli_epi16(abs_p1q1, 9); \ + abs_p1q1 = _mm_packs_epi16(abs_p1q1, abs_p1q1); /* abs(p1 - q1) / 2 */ \ + /* abs(p0 - q0) * 2 + abs(p1 - q1) / 2 */ \ + mask = _mm_adds_epu8(abs_p0q0, abs_p1q1); \ + flat = _mm_max_epu8(flat, _mm_srli_si128(flat, 8)); \ + mask = _mm_unpacklo_epi64(mask, flat); \ + mask = _mm_subs_epu8(mask, limit); \ + mask = _mm_cmpeq_epi8(mask, zero); \ + mask = _mm_and_si128(mask, _mm_srli_si128(mask, 8)); \ + } while (0) +#endif // CONFIG_PARALLEL_DEBLOCKING + +// filter_mask and hev_mask +#define FILTER_HEV_MASK \ + do { \ + /* (abs(q1 - q0), abs(p1 - p0) */ \ + __m128i flat = abs_diff(q1p1, q0p0); \ + /* abs(p1 - q1), abs(p0 - q0) */ \ + const __m128i abs_p1q1p0q0 = abs_diff(p1p0, q1q0); \ + __m128i abs_p0q0, abs_p1q1, work; \ + \ + /* const uint8_t hev = hev_mask(thresh, *op1, *op0, *oq0, *oq1); */ \ + hev = \ + _mm_unpacklo_epi8(_mm_max_epu8(flat, _mm_srli_si128(flat, 8)), zero); \ + hev = _mm_cmpgt_epi16(hev, thresh); \ + hev = _mm_packs_epi16(hev, hev); \ + \ + /* const int8_t mask = filter_mask(*limit, *blimit, */ \ + /* p3, p2, p1, p0, q0, q1, q2, q3); */ \ + abs_p0q0 = \ + _mm_adds_epu8(abs_p1q1p0q0, abs_p1q1p0q0); /* abs(p0 - q0) * 2 */ \ + abs_p1q1 = \ + _mm_unpackhi_epi8(abs_p1q1p0q0, abs_p1q1p0q0); /* abs(p1 - q1) */ \ + abs_p1q1 = _mm_srli_epi16(abs_p1q1, 9); \ + abs_p1q1 = _mm_packs_epi16(abs_p1q1, abs_p1q1); /* abs(p1 - q1) / 2 */ \ + /* abs(p0 - q0) * 2 + abs(p1 - q1) / 2 */ \ + mask = _mm_adds_epu8(abs_p0q0, abs_p1q1); \ + /* abs(p3 - p2), abs(p2 - p1) */ \ + work = abs_diff(p3p2, p2p1); \ + flat = _mm_max_epu8(work, flat); \ + /* abs(q3 - q2), abs(q2 - q1) */ \ + work = abs_diff(q3q2, q2q1); \ + flat = _mm_max_epu8(work, flat); \ + flat = _mm_max_epu8(flat, _mm_srli_si128(flat, 8)); \ + mask = _mm_unpacklo_epi64(mask, flat); \ + mask = _mm_subs_epu8(mask, limit); \ + mask = _mm_cmpeq_epi8(mask, zero); \ + mask = _mm_and_si128(mask, _mm_srli_si128(mask, 8)); \ + } while (0) + +#define FILTER4 \ + do { \ + const __m128i t3t4 = \ + _mm_set_epi8(3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4); \ + const __m128i t80 = _mm_set1_epi8(0x80); \ + __m128i filter, filter2filter1, work; \ + \ + ps1ps0 = _mm_xor_si128(p1p0, t80); /* ^ 0x80 */ \ + qs1qs0 = _mm_xor_si128(q1q0, t80); \ + \ + /* int8_t filter = signed_char_clamp(ps1 - qs1) & hev; */ \ + work = _mm_subs_epi8(ps1ps0, qs1qs0); \ + filter = _mm_and_si128(_mm_srli_si128(work, 8), hev); \ + /* filter = signed_char_clamp(filter + 3 * (qs0 - ps0)) & mask; */ \ + filter = _mm_subs_epi8(filter, work); \ + filter = _mm_subs_epi8(filter, work); \ + filter = _mm_subs_epi8(filter, work); /* + 3 * (qs0 - ps0) */ \ + filter = _mm_and_si128(filter, mask); /* & mask */ \ + filter = _mm_unpacklo_epi64(filter, filter); \ + \ + /* filter1 = signed_char_clamp(filter + 4) >> 3; */ \ + /* filter2 = signed_char_clamp(filter + 3) >> 3; */ \ + filter2filter1 = _mm_adds_epi8(filter, t3t4); /* signed_char_clamp */ \ + filter = _mm_unpackhi_epi8(filter2filter1, filter2filter1); \ + filter2filter1 = _mm_unpacklo_epi8(filter2filter1, filter2filter1); \ + filter2filter1 = _mm_srai_epi16(filter2filter1, 11); /* >> 3 */ \ + filter = _mm_srai_epi16(filter, 11); /* >> 3 */ \ + filter2filter1 = _mm_packs_epi16(filter2filter1, filter); \ + \ + /* filter = ROUND_POWER_OF_TWO(filter1, 1) & ~hev; */ \ + filter = _mm_subs_epi8(filter2filter1, ff); /* + 1 */ \ + filter = _mm_unpacklo_epi8(filter, filter); \ + filter = _mm_srai_epi16(filter, 9); /* round */ \ + filter = _mm_packs_epi16(filter, filter); \ + filter = _mm_andnot_si128(hev, filter); \ + \ + hev = _mm_unpackhi_epi64(filter2filter1, filter); \ + filter2filter1 = _mm_unpacklo_epi64(filter2filter1, filter); \ + \ + /* signed_char_clamp(qs1 - filter), signed_char_clamp(qs0 - filter1) */ \ + qs1qs0 = _mm_subs_epi8(qs1qs0, filter2filter1); \ + /* signed_char_clamp(ps1 + filter), signed_char_clamp(ps0 + filter2) */ \ + ps1ps0 = _mm_adds_epi8(ps1ps0, hev); \ + qs1qs0 = _mm_xor_si128(qs1qs0, t80); /* ^ 0x80 */ \ + ps1ps0 = _mm_xor_si128(ps1ps0, t80); /* ^ 0x80 */ \ + } while (0) + +void aom_lpf_horizontal_4_sse2(uint8_t *s, int p /* pitch */, + const uint8_t *_blimit, const uint8_t *_limit, + const uint8_t *_thresh) { + const __m128i zero = _mm_set1_epi16(0); + const __m128i limit = + _mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i *)_blimit), + _mm_loadl_epi64((const __m128i *)_limit)); + const __m128i thresh = + _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)_thresh), zero); + const __m128i ff = _mm_cmpeq_epi8(zero, zero); +#if !CONFIG_PARALLEL_DEBLOCKING + __m128i p3p2, p2p1, q3q2, q2q1; +#endif // !CONFIG_PARALLEL_DEBLOCKING + __m128i q1p1, q0p0, p1p0, q1q0, ps1ps0, qs1qs0; + __m128i mask, hev; +#if !CONFIG_PARALLEL_DEBLOCKING + p3p2 = _mm_unpacklo_epi64(_mm_loadl_epi64((__m128i *)(s - 3 * p)), + _mm_loadl_epi64((__m128i *)(s - 4 * p))); +#endif // !CONFIG_PARALLEL_DEBLOCKING + q1p1 = _mm_unpacklo_epi64(_mm_loadl_epi64((__m128i *)(s - 2 * p)), + _mm_loadl_epi64((__m128i *)(s + 1 * p))); + q0p0 = _mm_unpacklo_epi64(_mm_loadl_epi64((__m128i *)(s - 1 * p)), + _mm_loadl_epi64((__m128i *)(s + 0 * p))); +#if !CONFIG_PARALLEL_DEBLOCKING + q3q2 = _mm_unpacklo_epi64(_mm_loadl_epi64((__m128i *)(s + 2 * p)), + _mm_loadl_epi64((__m128i *)(s + 3 * p))); +#endif // !CONFIG_PARALLEL_DEBLOCKING + p1p0 = _mm_unpacklo_epi64(q0p0, q1p1); + q1q0 = _mm_unpackhi_epi64(q0p0, q1p1); +#if !CONFIG_PARALLEL_DEBLOCKING + p2p1 = _mm_unpacklo_epi64(q1p1, p3p2); + q2q1 = _mm_unpacklo_epi64(_mm_srli_si128(q1p1, 8), q3q2); +#endif // !CONFIG_PARALLEL_DEBLOCKING +#if !CONFIG_PARALLEL_DEBLOCKING + FILTER_HEV_MASK; +#else // CONFIG_PARALLEL_DEBLOCKING + FILTER_HEV_MASK4; +#endif // !CONFIG_PARALLEL_DEBLOCKING + FILTER4; + + _mm_storeh_pi((__m64 *)(s - 2 * p), _mm_castsi128_ps(ps1ps0)); // *op1 + _mm_storel_epi64((__m128i *)(s - 1 * p), ps1ps0); // *op0 + _mm_storel_epi64((__m128i *)(s + 0 * p), qs1qs0); // *oq0 + _mm_storeh_pi((__m64 *)(s + 1 * p), _mm_castsi128_ps(qs1qs0)); // *oq1 +} + +void aom_lpf_vertical_4_sse2(uint8_t *s, int p /* pitch */, + const uint8_t *_blimit, const uint8_t *_limit, + const uint8_t *_thresh) { + const __m128i zero = _mm_set1_epi16(0); + const __m128i limit = + _mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i *)_blimit), + _mm_loadl_epi64((const __m128i *)_limit)); + const __m128i thresh = + _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)_thresh), zero); + const __m128i ff = _mm_cmpeq_epi8(zero, zero); + __m128i x0, x1, x2, x3; +#if !CONFIG_PARALLEL_DEBLOCKING + __m128i p3p2, p2p1, q3q2, q2q1; +#endif // !CONFIG_PARALLEL_DEBLOCKING + __m128i q1p1, q0p0, p1p0, q1q0, ps1ps0, qs1qs0; + __m128i mask, hev; + + // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 + q1q0 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(s + 0 * p - 4)), + _mm_loadl_epi64((__m128i *)(s + 1 * p - 4))); + + // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 + x1 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(s + 2 * p - 4)), + _mm_loadl_epi64((__m128i *)(s + 3 * p - 4))); + + // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57 + x2 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(s + 4 * p - 4)), + _mm_loadl_epi64((__m128i *)(s + 5 * p - 4))); + + // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77 + x3 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(s + 6 * p - 4)), + _mm_loadl_epi64((__m128i *)(s + 7 * p - 4))); + + // Transpose 8x8 + // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 + p1p0 = _mm_unpacklo_epi16(q1q0, x1); + // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73 + x0 = _mm_unpacklo_epi16(x2, x3); +#if !CONFIG_PARALLEL_DEBLOCKING + // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 + p3p2 = _mm_unpacklo_epi32(p1p0, x0); +#endif // !CONFIG_PARALLEL_DEBLOCKING + // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 + p1p0 = _mm_unpackhi_epi32(p1p0, x0); +#if !CONFIG_PARALLEL_DEBLOCKING + p3p2 = _mm_unpackhi_epi64(p3p2, _mm_slli_si128(p3p2, 8)); // swap lo and high +#endif // !CONFIG_PARALLEL_DEBLOCKING + p1p0 = _mm_unpackhi_epi64(p1p0, _mm_slli_si128(p1p0, 8)); // swap lo and high + + // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37 + q1q0 = _mm_unpackhi_epi16(q1q0, x1); + // 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77 + x2 = _mm_unpackhi_epi16(x2, x3); +#if !CONFIG_PARALLEL_DEBLOCKING + // 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77 + q3q2 = _mm_unpackhi_epi32(q1q0, x2); +#endif // !CONFIG_PARALLEL_DEBLOCKING + // 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75 + q1q0 = _mm_unpacklo_epi32(q1q0, x2); + + q0p0 = _mm_unpacklo_epi64(p1p0, q1q0); + q1p1 = _mm_unpackhi_epi64(p1p0, q1q0); + p1p0 = _mm_unpacklo_epi64(q0p0, q1p1); +#if !CONFIG_PARALLEL_DEBLOCKING + p2p1 = _mm_unpacklo_epi64(q1p1, p3p2); + q2q1 = _mm_unpacklo_epi64(_mm_srli_si128(q1p1, 8), q3q2); +#endif // !CONFIG_PARALLEL_DEBLOCKING +#if !CONFIG_PARALLEL_DEBLOCKING + FILTER_HEV_MASK; +#else // CONFIG_PARALLEL_DEBLOCKING + FILTER_HEV_MASK4; +#endif // !CONFIG_PARALLEL_DEBLOCKING + FILTER4; + + // Transpose 8x4 to 4x8 + // qs1qs0: 20 21 22 23 24 25 26 27 30 31 32 33 34 34 36 37 + // ps1ps0: 10 11 12 13 14 15 16 17 00 01 02 03 04 05 06 07 + // 00 01 02 03 04 05 06 07 10 11 12 13 14 15 16 17 + ps1ps0 = _mm_unpackhi_epi64(ps1ps0, _mm_slli_si128(ps1ps0, 8)); + // 10 30 11 31 12 32 13 33 14 34 15 35 16 36 17 37 + x0 = _mm_unpackhi_epi8(ps1ps0, qs1qs0); + // 00 20 01 21 02 22 03 23 04 24 05 25 06 26 07 27 + ps1ps0 = _mm_unpacklo_epi8(ps1ps0, qs1qs0); + // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37 + qs1qs0 = _mm_unpackhi_epi8(ps1ps0, x0); + // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 + ps1ps0 = _mm_unpacklo_epi8(ps1ps0, x0); + + *(int *)(s + 0 * p - 2) = _mm_cvtsi128_si32(ps1ps0); + ps1ps0 = _mm_srli_si128(ps1ps0, 4); + *(int *)(s + 1 * p - 2) = _mm_cvtsi128_si32(ps1ps0); + ps1ps0 = _mm_srli_si128(ps1ps0, 4); + *(int *)(s + 2 * p - 2) = _mm_cvtsi128_si32(ps1ps0); + ps1ps0 = _mm_srli_si128(ps1ps0, 4); + *(int *)(s + 3 * p - 2) = _mm_cvtsi128_si32(ps1ps0); + + *(int *)(s + 4 * p - 2) = _mm_cvtsi128_si32(qs1qs0); + qs1qs0 = _mm_srli_si128(qs1qs0, 4); + *(int *)(s + 5 * p - 2) = _mm_cvtsi128_si32(qs1qs0); + qs1qs0 = _mm_srli_si128(qs1qs0, 4); + *(int *)(s + 6 * p - 2) = _mm_cvtsi128_si32(qs1qs0); + qs1qs0 = _mm_srli_si128(qs1qs0, 4); + *(int *)(s + 7 * p - 2) = _mm_cvtsi128_si32(qs1qs0); +} + +void aom_lpf_horizontal_edge_8_sse2(unsigned char *s, int p, + const unsigned char *_blimit, + const unsigned char *_limit, + const unsigned char *_thresh) { + const __m128i zero = _mm_set1_epi16(0); + const __m128i one = _mm_set1_epi8(1); + const __m128i blimit = _mm_load_si128((const __m128i *)_blimit); + const __m128i limit = _mm_load_si128((const __m128i *)_limit); + const __m128i thresh = _mm_load_si128((const __m128i *)_thresh); + __m128i mask, hev, flat, flat2; + __m128i q7p7, q6p6, q5p5, q4p4, q3p3, q2p2, q1p1, q0p0, p0q0, p1q1; + __m128i abs_p1p0; + + q4p4 = _mm_loadl_epi64((__m128i *)(s - 5 * p)); + q4p4 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q4p4), (__m64 *)(s + 4 * p))); + q3p3 = _mm_loadl_epi64((__m128i *)(s - 4 * p)); + q3p3 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q3p3), (__m64 *)(s + 3 * p))); + q2p2 = _mm_loadl_epi64((__m128i *)(s - 3 * p)); + q2p2 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q2p2), (__m64 *)(s + 2 * p))); + q1p1 = _mm_loadl_epi64((__m128i *)(s - 2 * p)); + q1p1 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q1p1), (__m64 *)(s + 1 * p))); + p1q1 = _mm_shuffle_epi32(q1p1, 78); + q0p0 = _mm_loadl_epi64((__m128i *)(s - 1 * p)); + q0p0 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q0p0), (__m64 *)(s - 0 * p))); + p0q0 = _mm_shuffle_epi32(q0p0, 78); + + { + __m128i abs_p1q1, abs_p0q0, abs_q1q0, fe, ff, work; + abs_p1p0 = abs_diff(q1p1, q0p0); + abs_q1q0 = _mm_srli_si128(abs_p1p0, 8); + fe = _mm_set1_epi8(0xfe); + ff = _mm_cmpeq_epi8(abs_p1p0, abs_p1p0); + abs_p0q0 = abs_diff(q0p0, p0q0); + abs_p1q1 = abs_diff(q1p1, p1q1); + flat = _mm_max_epu8(abs_p1p0, abs_q1q0); + hev = _mm_subs_epu8(flat, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi8(hev, zero), ff); + + abs_p0q0 = _mm_adds_epu8(abs_p0q0, abs_p0q0); + abs_p1q1 = _mm_srli_epi16(_mm_and_si128(abs_p1q1, fe), 1); + mask = _mm_subs_epu8(_mm_adds_epu8(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi8(mask, zero), ff); + // mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + mask = _mm_max_epu8(abs_p1p0, mask); + // mask |= (abs(p1 - p0) > limit) * -1; + // mask |= (abs(q1 - q0) > limit) * -1; + + work = _mm_max_epu8(abs_diff(q2p2, q1p1), abs_diff(q3p3, q2p2)); + mask = _mm_max_epu8(work, mask); + mask = _mm_max_epu8(mask, _mm_srli_si128(mask, 8)); + mask = _mm_subs_epu8(mask, limit); + mask = _mm_cmpeq_epi8(mask, zero); + } + + // lp filter + { + const __m128i t4 = _mm_set1_epi8(4); + const __m128i t3 = _mm_set1_epi8(3); + const __m128i t80 = _mm_set1_epi8(0x80); + const __m128i t1 = _mm_set1_epi16(0x1); + __m128i qs1ps1 = _mm_xor_si128(q1p1, t80); + __m128i qs0ps0 = _mm_xor_si128(q0p0, t80); + __m128i qs0 = _mm_xor_si128(p0q0, t80); + __m128i qs1 = _mm_xor_si128(p1q1, t80); + __m128i filt; + __m128i work_a; + __m128i filter1, filter2; + __m128i flat2_q6p6, flat2_q5p5, flat2_q4p4, flat2_q3p3, flat2_q2p2; + __m128i flat2_q1p1, flat2_q0p0, flat_q2p2, flat_q1p1, flat_q0p0; + + filt = _mm_and_si128(_mm_subs_epi8(qs1ps1, qs1), hev); + work_a = _mm_subs_epi8(qs0, qs0ps0); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + // (aom_filter + 3 * (qs0 - ps0)) & mask + filt = _mm_and_si128(filt, mask); + + filter1 = _mm_adds_epi8(filt, t4); + filter2 = _mm_adds_epi8(filt, t3); + + filter1 = _mm_unpacklo_epi8(zero, filter1); + filter1 = _mm_srai_epi16(filter1, 0xB); + filter2 = _mm_unpacklo_epi8(zero, filter2); + filter2 = _mm_srai_epi16(filter2, 0xB); + + // Filter1 >> 3 + filt = _mm_packs_epi16(filter2, _mm_subs_epi16(zero, filter1)); + qs0ps0 = _mm_xor_si128(_mm_adds_epi8(qs0ps0, filt), t80); + + // filt >> 1 + filt = _mm_adds_epi16(filter1, t1); + filt = _mm_srai_epi16(filt, 1); + filt = _mm_andnot_si128(_mm_srai_epi16(_mm_unpacklo_epi8(zero, hev), 0x8), + filt); + filt = _mm_packs_epi16(filt, _mm_subs_epi16(zero, filt)); + qs1ps1 = _mm_xor_si128(_mm_adds_epi8(qs1ps1, filt), t80); + // loopfilter done + + { + __m128i work; + flat = _mm_max_epu8(abs_diff(q2p2, q0p0), abs_diff(q3p3, q0p0)); + flat = _mm_max_epu8(abs_p1p0, flat); + flat = _mm_max_epu8(flat, _mm_srli_si128(flat, 8)); + flat = _mm_subs_epu8(flat, one); + flat = _mm_cmpeq_epi8(flat, zero); + flat = _mm_and_si128(flat, mask); + + q5p5 = _mm_loadl_epi64((__m128i *)(s - 6 * p)); + q5p5 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q5p5), (__m64 *)(s + 5 * p))); + + q6p6 = _mm_loadl_epi64((__m128i *)(s - 7 * p)); + q6p6 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q6p6), (__m64 *)(s + 6 * p))); + flat2 = _mm_max_epu8(abs_diff(q4p4, q0p0), abs_diff(q5p5, q0p0)); + + q7p7 = _mm_loadl_epi64((__m128i *)(s - 8 * p)); + q7p7 = _mm_castps_si128( + _mm_loadh_pi(_mm_castsi128_ps(q7p7), (__m64 *)(s + 7 * p))); + work = _mm_max_epu8(abs_diff(q6p6, q0p0), abs_diff(q7p7, q0p0)); + flat2 = _mm_max_epu8(work, flat2); + flat2 = _mm_max_epu8(flat2, _mm_srli_si128(flat2, 8)); + flat2 = _mm_subs_epu8(flat2, one); + flat2 = _mm_cmpeq_epi8(flat2, zero); + flat2 = _mm_and_si128(flat2, flat); // flat2 & flat & mask + } + + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // flat and wide flat calculations + { + const __m128i eight = _mm_set1_epi16(8); + const __m128i four = _mm_set1_epi16(4); + __m128i p7_16, p6_16, p5_16, p4_16, p3_16, p2_16, p1_16, p0_16; + __m128i q7_16, q6_16, q5_16, q4_16, q3_16, q2_16, q1_16, q0_16; + __m128i pixelFilter_p, pixelFilter_q; + __m128i pixetFilter_p2p1p0, pixetFilter_q2q1q0; + __m128i sum_p7, sum_q7, sum_p3, sum_q3, res_p, res_q; + + p7_16 = _mm_unpacklo_epi8(q7p7, zero); + p6_16 = _mm_unpacklo_epi8(q6p6, zero); + p5_16 = _mm_unpacklo_epi8(q5p5, zero); + p4_16 = _mm_unpacklo_epi8(q4p4, zero); + p3_16 = _mm_unpacklo_epi8(q3p3, zero); + p2_16 = _mm_unpacklo_epi8(q2p2, zero); + p1_16 = _mm_unpacklo_epi8(q1p1, zero); + p0_16 = _mm_unpacklo_epi8(q0p0, zero); + q0_16 = _mm_unpackhi_epi8(q0p0, zero); + q1_16 = _mm_unpackhi_epi8(q1p1, zero); + q2_16 = _mm_unpackhi_epi8(q2p2, zero); + q3_16 = _mm_unpackhi_epi8(q3p3, zero); + q4_16 = _mm_unpackhi_epi8(q4p4, zero); + q5_16 = _mm_unpackhi_epi8(q5p5, zero); + q6_16 = _mm_unpackhi_epi8(q6p6, zero); + q7_16 = _mm_unpackhi_epi8(q7p7, zero); + + pixelFilter_p = _mm_add_epi16(_mm_add_epi16(p6_16, p5_16), + _mm_add_epi16(p4_16, p3_16)); + pixelFilter_q = _mm_add_epi16(_mm_add_epi16(q6_16, q5_16), + _mm_add_epi16(q4_16, q3_16)); + + pixetFilter_p2p1p0 = _mm_add_epi16(p0_16, _mm_add_epi16(p2_16, p1_16)); + pixelFilter_p = _mm_add_epi16(pixelFilter_p, pixetFilter_p2p1p0); + + pixetFilter_q2q1q0 = _mm_add_epi16(q0_16, _mm_add_epi16(q2_16, q1_16)); + pixelFilter_q = _mm_add_epi16(pixelFilter_q, pixetFilter_q2q1q0); + pixelFilter_p = + _mm_add_epi16(eight, _mm_add_epi16(pixelFilter_p, pixelFilter_q)); + pixetFilter_p2p1p0 = _mm_add_epi16( + four, _mm_add_epi16(pixetFilter_p2p1p0, pixetFilter_q2q1q0)); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(p7_16, p0_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(q7_16, q0_16)), 4); + flat2_q0p0 = _mm_packus_epi16(res_p, res_q); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_p2p1p0, _mm_add_epi16(p3_16, p0_16)), 3); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_p2p1p0, _mm_add_epi16(q3_16, q0_16)), 3); + + flat_q0p0 = _mm_packus_epi16(res_p, res_q); + + sum_p7 = _mm_add_epi16(p7_16, p7_16); + sum_q7 = _mm_add_epi16(q7_16, q7_16); + sum_p3 = _mm_add_epi16(p3_16, p3_16); + sum_q3 = _mm_add_epi16(q3_16, q3_16); + + pixelFilter_q = _mm_sub_epi16(pixelFilter_p, p6_16); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q6_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p1_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q1_16)), 4); + flat2_q1p1 = _mm_packus_epi16(res_p, res_q); + + pixetFilter_q2q1q0 = _mm_sub_epi16(pixetFilter_p2p1p0, p2_16); + pixetFilter_p2p1p0 = _mm_sub_epi16(pixetFilter_p2p1p0, q2_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_p2p1p0, _mm_add_epi16(sum_p3, p1_16)), 3); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_q2q1q0, _mm_add_epi16(sum_q3, q1_16)), 3); + flat_q1p1 = _mm_packus_epi16(res_p, res_q); + + sum_p7 = _mm_add_epi16(sum_p7, p7_16); + sum_q7 = _mm_add_epi16(sum_q7, q7_16); + sum_p3 = _mm_add_epi16(sum_p3, p3_16); + sum_q3 = _mm_add_epi16(sum_q3, q3_16); + + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q5_16); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p5_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p2_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q2_16)), 4); + flat2_q2p2 = _mm_packus_epi16(res_p, res_q); + + pixetFilter_p2p1p0 = _mm_sub_epi16(pixetFilter_p2p1p0, q1_16); + pixetFilter_q2q1q0 = _mm_sub_epi16(pixetFilter_q2q1q0, p1_16); + + res_p = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_p2p1p0, _mm_add_epi16(sum_p3, p2_16)), 3); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixetFilter_q2q1q0, _mm_add_epi16(sum_q3, q2_16)), 3); + flat_q2p2 = _mm_packus_epi16(res_p, res_q); + + sum_p7 = _mm_add_epi16(sum_p7, p7_16); + sum_q7 = _mm_add_epi16(sum_q7, q7_16); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q4_16); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p4_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p3_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q3_16)), 4); + flat2_q3p3 = _mm_packus_epi16(res_p, res_q); + + sum_p7 = _mm_add_epi16(sum_p7, p7_16); + sum_q7 = _mm_add_epi16(sum_q7, q7_16); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q3_16); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p3_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p4_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q4_16)), 4); + flat2_q4p4 = _mm_packus_epi16(res_p, res_q); + + sum_p7 = _mm_add_epi16(sum_p7, p7_16); + sum_q7 = _mm_add_epi16(sum_q7, q7_16); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q2_16); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p2_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p5_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q5_16)), 4); + flat2_q5p5 = _mm_packus_epi16(res_p, res_q); + + sum_p7 = _mm_add_epi16(sum_p7, p7_16); + sum_q7 = _mm_add_epi16(sum_q7, q7_16); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q1_16); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p1_16); + res_p = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p6_16)), 4); + res_q = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q6_16)), 4); + flat2_q6p6 = _mm_packus_epi16(res_p, res_q); + } + // wide flat + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + flat = _mm_shuffle_epi32(flat, 68); + flat2 = _mm_shuffle_epi32(flat2, 68); + + q2p2 = _mm_andnot_si128(flat, q2p2); + flat_q2p2 = _mm_and_si128(flat, flat_q2p2); + q2p2 = _mm_or_si128(q2p2, flat_q2p2); + + qs1ps1 = _mm_andnot_si128(flat, qs1ps1); + flat_q1p1 = _mm_and_si128(flat, flat_q1p1); + q1p1 = _mm_or_si128(qs1ps1, flat_q1p1); + + qs0ps0 = _mm_andnot_si128(flat, qs0ps0); + flat_q0p0 = _mm_and_si128(flat, flat_q0p0); + q0p0 = _mm_or_si128(qs0ps0, flat_q0p0); + + q6p6 = _mm_andnot_si128(flat2, q6p6); + flat2_q6p6 = _mm_and_si128(flat2, flat2_q6p6); + q6p6 = _mm_or_si128(q6p6, flat2_q6p6); + _mm_storel_epi64((__m128i *)(s - 7 * p), q6p6); + _mm_storeh_pi((__m64 *)(s + 6 * p), _mm_castsi128_ps(q6p6)); + + q5p5 = _mm_andnot_si128(flat2, q5p5); + flat2_q5p5 = _mm_and_si128(flat2, flat2_q5p5); + q5p5 = _mm_or_si128(q5p5, flat2_q5p5); + _mm_storel_epi64((__m128i *)(s - 6 * p), q5p5); + _mm_storeh_pi((__m64 *)(s + 5 * p), _mm_castsi128_ps(q5p5)); + + q4p4 = _mm_andnot_si128(flat2, q4p4); + flat2_q4p4 = _mm_and_si128(flat2, flat2_q4p4); + q4p4 = _mm_or_si128(q4p4, flat2_q4p4); + _mm_storel_epi64((__m128i *)(s - 5 * p), q4p4); + _mm_storeh_pi((__m64 *)(s + 4 * p), _mm_castsi128_ps(q4p4)); + + q3p3 = _mm_andnot_si128(flat2, q3p3); + flat2_q3p3 = _mm_and_si128(flat2, flat2_q3p3); + q3p3 = _mm_or_si128(q3p3, flat2_q3p3); + _mm_storel_epi64((__m128i *)(s - 4 * p), q3p3); + _mm_storeh_pi((__m64 *)(s + 3 * p), _mm_castsi128_ps(q3p3)); + + q2p2 = _mm_andnot_si128(flat2, q2p2); + flat2_q2p2 = _mm_and_si128(flat2, flat2_q2p2); + q2p2 = _mm_or_si128(q2p2, flat2_q2p2); + _mm_storel_epi64((__m128i *)(s - 3 * p), q2p2); + _mm_storeh_pi((__m64 *)(s + 2 * p), _mm_castsi128_ps(q2p2)); + + q1p1 = _mm_andnot_si128(flat2, q1p1); + flat2_q1p1 = _mm_and_si128(flat2, flat2_q1p1); + q1p1 = _mm_or_si128(q1p1, flat2_q1p1); + _mm_storel_epi64((__m128i *)(s - 2 * p), q1p1); + _mm_storeh_pi((__m64 *)(s + 1 * p), _mm_castsi128_ps(q1p1)); + + q0p0 = _mm_andnot_si128(flat2, q0p0); + flat2_q0p0 = _mm_and_si128(flat2, flat2_q0p0); + q0p0 = _mm_or_si128(q0p0, flat2_q0p0); + _mm_storel_epi64((__m128i *)(s - 1 * p), q0p0); + _mm_storeh_pi((__m64 *)(s - 0 * p), _mm_castsi128_ps(q0p0)); + } +} + +static INLINE __m128i filter_add2_sub2(const __m128i *const total, + const __m128i *const a1, + const __m128i *const a2, + const __m128i *const s1, + const __m128i *const s2) { + __m128i x = _mm_add_epi16(*a1, *total); + x = _mm_add_epi16(_mm_sub_epi16(x, _mm_add_epi16(*s1, *s2)), *a2); + return x; +} + +static INLINE __m128i filter8_mask(const __m128i *const flat, + const __m128i *const other_filt, + const __m128i *const f8_lo, + const __m128i *const f8_hi) { + const __m128i f8 = + _mm_packus_epi16(_mm_srli_epi16(*f8_lo, 3), _mm_srli_epi16(*f8_hi, 3)); + const __m128i result = _mm_and_si128(*flat, f8); + return _mm_or_si128(_mm_andnot_si128(*flat, *other_filt), result); +} + +static INLINE __m128i filter16_mask(const __m128i *const flat, + const __m128i *const other_filt, + const __m128i *const f_lo, + const __m128i *const f_hi) { + const __m128i f = + _mm_packus_epi16(_mm_srli_epi16(*f_lo, 4), _mm_srli_epi16(*f_hi, 4)); + const __m128i result = _mm_and_si128(*flat, f); + return _mm_or_si128(_mm_andnot_si128(*flat, *other_filt), result); +} + +void aom_lpf_horizontal_edge_16_sse2(unsigned char *s, int p, + const unsigned char *_blimit, + const unsigned char *_limit, + const unsigned char *_thresh) { + const __m128i zero = _mm_set1_epi16(0); + const __m128i one = _mm_set1_epi8(1); + const __m128i blimit = _mm_load_si128((const __m128i *)_blimit); + const __m128i limit = _mm_load_si128((const __m128i *)_limit); + const __m128i thresh = _mm_load_si128((const __m128i *)_thresh); + __m128i mask, hev, flat, flat2; + __m128i p7, p6, p5; + __m128i p4, p3, p2, p1, p0, q0, q1, q2, q3, q4; + __m128i q5, q6, q7; + + __m128i op2, op1, op0, oq0, oq1, oq2; + + __m128i max_abs_p1p0q1q0; + + p7 = _mm_loadu_si128((__m128i *)(s - 8 * p)); + p6 = _mm_loadu_si128((__m128i *)(s - 7 * p)); + p5 = _mm_loadu_si128((__m128i *)(s - 6 * p)); + p4 = _mm_loadu_si128((__m128i *)(s - 5 * p)); + p3 = _mm_loadu_si128((__m128i *)(s - 4 * p)); + p2 = _mm_loadu_si128((__m128i *)(s - 3 * p)); + p1 = _mm_loadu_si128((__m128i *)(s - 2 * p)); + p0 = _mm_loadu_si128((__m128i *)(s - 1 * p)); + q0 = _mm_loadu_si128((__m128i *)(s - 0 * p)); + q1 = _mm_loadu_si128((__m128i *)(s + 1 * p)); + q2 = _mm_loadu_si128((__m128i *)(s + 2 * p)); + q3 = _mm_loadu_si128((__m128i *)(s + 3 * p)); + q4 = _mm_loadu_si128((__m128i *)(s + 4 * p)); + q5 = _mm_loadu_si128((__m128i *)(s + 5 * p)); + q6 = _mm_loadu_si128((__m128i *)(s + 6 * p)); + q7 = _mm_loadu_si128((__m128i *)(s + 7 * p)); + + { + const __m128i abs_p1p0 = abs_diff(p1, p0); + const __m128i abs_q1q0 = abs_diff(q1, q0); + const __m128i fe = _mm_set1_epi8(0xfe); + const __m128i ff = _mm_cmpeq_epi8(zero, zero); + __m128i abs_p0q0 = abs_diff(p0, q0); + __m128i abs_p1q1 = abs_diff(p1, q1); + __m128i work; + max_abs_p1p0q1q0 = _mm_max_epu8(abs_p1p0, abs_q1q0); + + abs_p0q0 = _mm_adds_epu8(abs_p0q0, abs_p0q0); + abs_p1q1 = _mm_srli_epi16(_mm_and_si128(abs_p1q1, fe), 1); + mask = _mm_subs_epu8(_mm_adds_epu8(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi8(mask, zero), ff); + // mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + mask = _mm_max_epu8(max_abs_p1p0q1q0, mask); + // mask |= (abs(p1 - p0) > limit) * -1; + // mask |= (abs(q1 - q0) > limit) * -1; + work = _mm_max_epu8(abs_diff(p2, p1), abs_diff(p3, p2)); + mask = _mm_max_epu8(work, mask); + work = _mm_max_epu8(abs_diff(q2, q1), abs_diff(q3, q2)); + mask = _mm_max_epu8(work, mask); + mask = _mm_subs_epu8(mask, limit); + mask = _mm_cmpeq_epi8(mask, zero); + } + + { + __m128i work; + work = _mm_max_epu8(abs_diff(p2, p0), abs_diff(q2, q0)); + flat = _mm_max_epu8(work, max_abs_p1p0q1q0); + work = _mm_max_epu8(abs_diff(p3, p0), abs_diff(q3, q0)); + flat = _mm_max_epu8(work, flat); + work = _mm_max_epu8(abs_diff(p4, p0), abs_diff(q4, q0)); + flat = _mm_subs_epu8(flat, one); + flat = _mm_cmpeq_epi8(flat, zero); + flat = _mm_and_si128(flat, mask); + flat2 = _mm_max_epu8(abs_diff(p5, p0), abs_diff(q5, q0)); + flat2 = _mm_max_epu8(work, flat2); + work = _mm_max_epu8(abs_diff(p6, p0), abs_diff(q6, q0)); + flat2 = _mm_max_epu8(work, flat2); + work = _mm_max_epu8(abs_diff(p7, p0), abs_diff(q7, q0)); + flat2 = _mm_max_epu8(work, flat2); + flat2 = _mm_subs_epu8(flat2, one); + flat2 = _mm_cmpeq_epi8(flat2, zero); + flat2 = _mm_and_si128(flat2, flat); // flat2 & flat & mask + } + + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // filter4 + { + const __m128i t4 = _mm_set1_epi8(4); + const __m128i t3 = _mm_set1_epi8(3); + const __m128i t80 = _mm_set1_epi8(0x80); + const __m128i te0 = _mm_set1_epi8(0xe0); + const __m128i t1f = _mm_set1_epi8(0x1f); + const __m128i t1 = _mm_set1_epi8(0x1); + const __m128i t7f = _mm_set1_epi8(0x7f); + const __m128i ff = _mm_cmpeq_epi8(t4, t4); + + __m128i filt; + __m128i work_a; + __m128i filter1, filter2; + + op1 = _mm_xor_si128(p1, t80); + op0 = _mm_xor_si128(p0, t80); + oq0 = _mm_xor_si128(q0, t80); + oq1 = _mm_xor_si128(q1, t80); + + hev = _mm_subs_epu8(max_abs_p1p0q1q0, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi8(hev, zero), ff); + filt = _mm_and_si128(_mm_subs_epi8(op1, oq1), hev); + + work_a = _mm_subs_epi8(oq0, op0); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + // (aom_filter + 3 * (qs0 - ps0)) & mask + filt = _mm_and_si128(filt, mask); + filter1 = _mm_adds_epi8(filt, t4); + filter2 = _mm_adds_epi8(filt, t3); + + // Filter1 >> 3 + work_a = _mm_cmpgt_epi8(zero, filter1); + filter1 = _mm_srli_epi16(filter1, 3); + work_a = _mm_and_si128(work_a, te0); + filter1 = _mm_and_si128(filter1, t1f); + filter1 = _mm_or_si128(filter1, work_a); + oq0 = _mm_xor_si128(_mm_subs_epi8(oq0, filter1), t80); + + // Filter2 >> 3 + work_a = _mm_cmpgt_epi8(zero, filter2); + filter2 = _mm_srli_epi16(filter2, 3); + work_a = _mm_and_si128(work_a, te0); + filter2 = _mm_and_si128(filter2, t1f); + filter2 = _mm_or_si128(filter2, work_a); + op0 = _mm_xor_si128(_mm_adds_epi8(op0, filter2), t80); + + // filt >> 1 + filt = _mm_adds_epi8(filter1, t1); + work_a = _mm_cmpgt_epi8(zero, filt); + filt = _mm_srli_epi16(filt, 1); + work_a = _mm_and_si128(work_a, t80); + filt = _mm_and_si128(filt, t7f); + filt = _mm_or_si128(filt, work_a); + filt = _mm_andnot_si128(hev, filt); + op1 = _mm_xor_si128(_mm_adds_epi8(op1, filt), t80); + oq1 = _mm_xor_si128(_mm_subs_epi8(oq1, filt), t80); + // loopfilter done + + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // filter8 + { + const __m128i four = _mm_set1_epi16(4); + const __m128i p3_lo = _mm_unpacklo_epi8(p3, zero); + const __m128i p2_lo = _mm_unpacklo_epi8(p2, zero); + const __m128i p1_lo = _mm_unpacklo_epi8(p1, zero); + const __m128i p0_lo = _mm_unpacklo_epi8(p0, zero); + const __m128i q0_lo = _mm_unpacklo_epi8(q0, zero); + const __m128i q1_lo = _mm_unpacklo_epi8(q1, zero); + const __m128i q2_lo = _mm_unpacklo_epi8(q2, zero); + const __m128i q3_lo = _mm_unpacklo_epi8(q3, zero); + + const __m128i p3_hi = _mm_unpackhi_epi8(p3, zero); + const __m128i p2_hi = _mm_unpackhi_epi8(p2, zero); + const __m128i p1_hi = _mm_unpackhi_epi8(p1, zero); + const __m128i p0_hi = _mm_unpackhi_epi8(p0, zero); + const __m128i q0_hi = _mm_unpackhi_epi8(q0, zero); + const __m128i q1_hi = _mm_unpackhi_epi8(q1, zero); + const __m128i q2_hi = _mm_unpackhi_epi8(q2, zero); + const __m128i q3_hi = _mm_unpackhi_epi8(q3, zero); + __m128i f8_lo, f8_hi; + + f8_lo = _mm_add_epi16(_mm_add_epi16(p3_lo, four), + _mm_add_epi16(p3_lo, p2_lo)); + f8_lo = _mm_add_epi16(_mm_add_epi16(p3_lo, f8_lo), + _mm_add_epi16(p2_lo, p1_lo)); + f8_lo = _mm_add_epi16(_mm_add_epi16(p0_lo, q0_lo), f8_lo); + + f8_hi = _mm_add_epi16(_mm_add_epi16(p3_hi, four), + _mm_add_epi16(p3_hi, p2_hi)); + f8_hi = _mm_add_epi16(_mm_add_epi16(p3_hi, f8_hi), + _mm_add_epi16(p2_hi, p1_hi)); + f8_hi = _mm_add_epi16(_mm_add_epi16(p0_hi, q0_hi), f8_hi); + + op2 = filter8_mask(&flat, &p2, &f8_lo, &f8_hi); + + f8_lo = filter_add2_sub2(&f8_lo, &q1_lo, &p1_lo, &p2_lo, &p3_lo); + f8_hi = filter_add2_sub2(&f8_hi, &q1_hi, &p1_hi, &p2_hi, &p3_hi); + op1 = filter8_mask(&flat, &op1, &f8_lo, &f8_hi); + + f8_lo = filter_add2_sub2(&f8_lo, &q2_lo, &p0_lo, &p1_lo, &p3_lo); + f8_hi = filter_add2_sub2(&f8_hi, &q2_hi, &p0_hi, &p1_hi, &p3_hi); + op0 = filter8_mask(&flat, &op0, &f8_lo, &f8_hi); + + f8_lo = filter_add2_sub2(&f8_lo, &q3_lo, &q0_lo, &p0_lo, &p3_lo); + f8_hi = filter_add2_sub2(&f8_hi, &q3_hi, &q0_hi, &p0_hi, &p3_hi); + oq0 = filter8_mask(&flat, &oq0, &f8_lo, &f8_hi); + + f8_lo = filter_add2_sub2(&f8_lo, &q3_lo, &q1_lo, &q0_lo, &p2_lo); + f8_hi = filter_add2_sub2(&f8_hi, &q3_hi, &q1_hi, &q0_hi, &p2_hi); + oq1 = filter8_mask(&flat, &oq1, &f8_lo, &f8_hi); + + f8_lo = filter_add2_sub2(&f8_lo, &q3_lo, &q2_lo, &q1_lo, &p1_lo); + f8_hi = filter_add2_sub2(&f8_hi, &q3_hi, &q2_hi, &q1_hi, &p1_hi); + oq2 = filter8_mask(&flat, &q2, &f8_lo, &f8_hi); + } + + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // wide flat calculations + { + const __m128i eight = _mm_set1_epi16(8); + const __m128i p7_lo = _mm_unpacklo_epi8(p7, zero); + const __m128i p6_lo = _mm_unpacklo_epi8(p6, zero); + const __m128i p5_lo = _mm_unpacklo_epi8(p5, zero); + const __m128i p4_lo = _mm_unpacklo_epi8(p4, zero); + const __m128i p3_lo = _mm_unpacklo_epi8(p3, zero); + const __m128i p2_lo = _mm_unpacklo_epi8(p2, zero); + const __m128i p1_lo = _mm_unpacklo_epi8(p1, zero); + const __m128i p0_lo = _mm_unpacklo_epi8(p0, zero); + const __m128i q0_lo = _mm_unpacklo_epi8(q0, zero); + const __m128i q1_lo = _mm_unpacklo_epi8(q1, zero); + const __m128i q2_lo = _mm_unpacklo_epi8(q2, zero); + const __m128i q3_lo = _mm_unpacklo_epi8(q3, zero); + const __m128i q4_lo = _mm_unpacklo_epi8(q4, zero); + const __m128i q5_lo = _mm_unpacklo_epi8(q5, zero); + const __m128i q6_lo = _mm_unpacklo_epi8(q6, zero); + const __m128i q7_lo = _mm_unpacklo_epi8(q7, zero); + + const __m128i p7_hi = _mm_unpackhi_epi8(p7, zero); + const __m128i p6_hi = _mm_unpackhi_epi8(p6, zero); + const __m128i p5_hi = _mm_unpackhi_epi8(p5, zero); + const __m128i p4_hi = _mm_unpackhi_epi8(p4, zero); + const __m128i p3_hi = _mm_unpackhi_epi8(p3, zero); + const __m128i p2_hi = _mm_unpackhi_epi8(p2, zero); + const __m128i p1_hi = _mm_unpackhi_epi8(p1, zero); + const __m128i p0_hi = _mm_unpackhi_epi8(p0, zero); + const __m128i q0_hi = _mm_unpackhi_epi8(q0, zero); + const __m128i q1_hi = _mm_unpackhi_epi8(q1, zero); + const __m128i q2_hi = _mm_unpackhi_epi8(q2, zero); + const __m128i q3_hi = _mm_unpackhi_epi8(q3, zero); + const __m128i q4_hi = _mm_unpackhi_epi8(q4, zero); + const __m128i q5_hi = _mm_unpackhi_epi8(q5, zero); + const __m128i q6_hi = _mm_unpackhi_epi8(q6, zero); + const __m128i q7_hi = _mm_unpackhi_epi8(q7, zero); + + __m128i f_lo; + __m128i f_hi; + + f_lo = _mm_sub_epi16(_mm_slli_epi16(p7_lo, 3), p7_lo); // p7 * 7 + f_lo = + _mm_add_epi16(_mm_slli_epi16(p6_lo, 1), _mm_add_epi16(p4_lo, f_lo)); + f_lo = _mm_add_epi16(_mm_add_epi16(p3_lo, f_lo), + _mm_add_epi16(p2_lo, p1_lo)); + f_lo = _mm_add_epi16(_mm_add_epi16(p0_lo, q0_lo), f_lo); + f_lo = _mm_add_epi16(_mm_add_epi16(p5_lo, eight), f_lo); + + f_hi = _mm_sub_epi16(_mm_slli_epi16(p7_hi, 3), p7_hi); // p7 * 7 + f_hi = + _mm_add_epi16(_mm_slli_epi16(p6_hi, 1), _mm_add_epi16(p4_hi, f_hi)); + f_hi = _mm_add_epi16(_mm_add_epi16(p3_hi, f_hi), + _mm_add_epi16(p2_hi, p1_hi)); + f_hi = _mm_add_epi16(_mm_add_epi16(p0_hi, q0_hi), f_hi); + f_hi = _mm_add_epi16(_mm_add_epi16(p5_hi, eight), f_hi); + + p6 = filter16_mask(&flat2, &p6, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s - 7 * p), p6); + + f_lo = filter_add2_sub2(&f_lo, &q1_lo, &p5_lo, &p6_lo, &p7_lo); + f_hi = filter_add2_sub2(&f_hi, &q1_hi, &p5_hi, &p6_hi, &p7_hi); + p5 = filter16_mask(&flat2, &p5, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s - 6 * p), p5); + + f_lo = filter_add2_sub2(&f_lo, &q2_lo, &p4_lo, &p5_lo, &p7_lo); + f_hi = filter_add2_sub2(&f_hi, &q2_hi, &p4_hi, &p5_hi, &p7_hi); + p4 = filter16_mask(&flat2, &p4, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s - 5 * p), p4); + + f_lo = filter_add2_sub2(&f_lo, &q3_lo, &p3_lo, &p4_lo, &p7_lo); + f_hi = filter_add2_sub2(&f_hi, &q3_hi, &p3_hi, &p4_hi, &p7_hi); + p3 = filter16_mask(&flat2, &p3, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s - 4 * p), p3); + + f_lo = filter_add2_sub2(&f_lo, &q4_lo, &p2_lo, &p3_lo, &p7_lo); + f_hi = filter_add2_sub2(&f_hi, &q4_hi, &p2_hi, &p3_hi, &p7_hi); + op2 = filter16_mask(&flat2, &op2, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s - 3 * p), op2); + + f_lo = filter_add2_sub2(&f_lo, &q5_lo, &p1_lo, &p2_lo, &p7_lo); + f_hi = filter_add2_sub2(&f_hi, &q5_hi, &p1_hi, &p2_hi, &p7_hi); + op1 = filter16_mask(&flat2, &op1, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s - 2 * p), op1); + + f_lo = filter_add2_sub2(&f_lo, &q6_lo, &p0_lo, &p1_lo, &p7_lo); + f_hi = filter_add2_sub2(&f_hi, &q6_hi, &p0_hi, &p1_hi, &p7_hi); + op0 = filter16_mask(&flat2, &op0, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s - 1 * p), op0); + + f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q0_lo, &p0_lo, &p7_lo); + f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q0_hi, &p0_hi, &p7_hi); + oq0 = filter16_mask(&flat2, &oq0, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s - 0 * p), oq0); + + f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q1_lo, &p6_lo, &q0_lo); + f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q1_hi, &p6_hi, &q0_hi); + oq1 = filter16_mask(&flat2, &oq1, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s + 1 * p), oq1); + + f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q2_lo, &p5_lo, &q1_lo); + f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q2_hi, &p5_hi, &q1_hi); + oq2 = filter16_mask(&flat2, &oq2, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s + 2 * p), oq2); + + f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q3_lo, &p4_lo, &q2_lo); + f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q3_hi, &p4_hi, &q2_hi); + q3 = filter16_mask(&flat2, &q3, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s + 3 * p), q3); + + f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q4_lo, &p3_lo, &q3_lo); + f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q4_hi, &p3_hi, &q3_hi); + q4 = filter16_mask(&flat2, &q4, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s + 4 * p), q4); + + f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q5_lo, &p2_lo, &q4_lo); + f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q5_hi, &p2_hi, &q4_hi); + q5 = filter16_mask(&flat2, &q5, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s + 5 * p), q5); + + f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q6_lo, &p1_lo, &q5_lo); + f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q6_hi, &p1_hi, &q5_hi); + q6 = filter16_mask(&flat2, &q6, &f_lo, &f_hi); + _mm_storeu_si128((__m128i *)(s + 6 * p), q6); + } + // wide flat + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + } +} + +void aom_lpf_horizontal_8_sse2(unsigned char *s, int p, + const unsigned char *_blimit, + const unsigned char *_limit, + const unsigned char *_thresh) { + DECLARE_ALIGNED(16, unsigned char, flat_op2[16]); + DECLARE_ALIGNED(16, unsigned char, flat_op1[16]); + DECLARE_ALIGNED(16, unsigned char, flat_op0[16]); + DECLARE_ALIGNED(16, unsigned char, flat_oq2[16]); + DECLARE_ALIGNED(16, unsigned char, flat_oq1[16]); + DECLARE_ALIGNED(16, unsigned char, flat_oq0[16]); + const __m128i zero = _mm_set1_epi16(0); + const __m128i blimit = _mm_load_si128((const __m128i *)_blimit); + const __m128i limit = _mm_load_si128((const __m128i *)_limit); + const __m128i thresh = _mm_load_si128((const __m128i *)_thresh); + __m128i mask, hev, flat; + __m128i p3, p2, p1, p0, q0, q1, q2, q3; + __m128i q3p3, q2p2, q1p1, q0p0, p1q1, p0q0; + + q3p3 = _mm_unpacklo_epi64(_mm_loadl_epi64((__m128i *)(s - 4 * p)), + _mm_loadl_epi64((__m128i *)(s + 3 * p))); + q2p2 = _mm_unpacklo_epi64(_mm_loadl_epi64((__m128i *)(s - 3 * p)), + _mm_loadl_epi64((__m128i *)(s + 2 * p))); + q1p1 = _mm_unpacklo_epi64(_mm_loadl_epi64((__m128i *)(s - 2 * p)), + _mm_loadl_epi64((__m128i *)(s + 1 * p))); + q0p0 = _mm_unpacklo_epi64(_mm_loadl_epi64((__m128i *)(s - 1 * p)), + _mm_loadl_epi64((__m128i *)(s - 0 * p))); + p1q1 = _mm_shuffle_epi32(q1p1, 78); + p0q0 = _mm_shuffle_epi32(q0p0, 78); + + { + // filter_mask and hev_mask + const __m128i one = _mm_set1_epi8(1); + const __m128i fe = _mm_set1_epi8(0xfe); + const __m128i ff = _mm_cmpeq_epi8(fe, fe); + __m128i abs_p1q1, abs_p0q0, abs_q1q0, abs_p1p0, work; + abs_p1p0 = abs_diff(q1p1, q0p0); + abs_q1q0 = _mm_srli_si128(abs_p1p0, 8); + + abs_p0q0 = abs_diff(q0p0, p0q0); + abs_p1q1 = abs_diff(q1p1, p1q1); + flat = _mm_max_epu8(abs_p1p0, abs_q1q0); + hev = _mm_subs_epu8(flat, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi8(hev, zero), ff); + + abs_p0q0 = _mm_adds_epu8(abs_p0q0, abs_p0q0); + abs_p1q1 = _mm_srli_epi16(_mm_and_si128(abs_p1q1, fe), 1); + mask = _mm_subs_epu8(_mm_adds_epu8(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi8(mask, zero), ff); + // mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + mask = _mm_max_epu8(abs_p1p0, mask); + // mask |= (abs(p1 - p0) > limit) * -1; + // mask |= (abs(q1 - q0) > limit) * -1; + + work = _mm_max_epu8(abs_diff(q2p2, q1p1), abs_diff(q3p3, q2p2)); + mask = _mm_max_epu8(work, mask); + mask = _mm_max_epu8(mask, _mm_srli_si128(mask, 8)); + mask = _mm_subs_epu8(mask, limit); + mask = _mm_cmpeq_epi8(mask, zero); + + // flat_mask4 + + flat = _mm_max_epu8(abs_diff(q2p2, q0p0), abs_diff(q3p3, q0p0)); + flat = _mm_max_epu8(abs_p1p0, flat); + flat = _mm_max_epu8(flat, _mm_srli_si128(flat, 8)); + flat = _mm_subs_epu8(flat, one); + flat = _mm_cmpeq_epi8(flat, zero); + flat = _mm_and_si128(flat, mask); + } + + { + const __m128i four = _mm_set1_epi16(4); + unsigned char *src = s; + { + __m128i workp_a, workp_b, workp_shft; + p3 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src - 4 * p)), zero); + p2 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src - 3 * p)), zero); + p1 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src - 2 * p)), zero); + p0 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src - 1 * p)), zero); + q0 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src - 0 * p)), zero); + q1 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src + 1 * p)), zero); + q2 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src + 2 * p)), zero); + q3 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src + 3 * p)), zero); + + workp_a = _mm_add_epi16(_mm_add_epi16(p3, p3), _mm_add_epi16(p2, p1)); + workp_a = _mm_add_epi16(_mm_add_epi16(workp_a, four), p0); + workp_b = _mm_add_epi16(_mm_add_epi16(q0, p2), p3); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_storel_epi64((__m128i *)&flat_op2[0], + _mm_packus_epi16(workp_shft, workp_shft)); + + workp_b = _mm_add_epi16(_mm_add_epi16(q0, q1), p1); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_storel_epi64((__m128i *)&flat_op1[0], + _mm_packus_epi16(workp_shft, workp_shft)); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p3), q2); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, p1), p0); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_storel_epi64((__m128i *)&flat_op0[0], + _mm_packus_epi16(workp_shft, workp_shft)); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p3), q3); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, p0), q0); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_storel_epi64((__m128i *)&flat_oq0[0], + _mm_packus_epi16(workp_shft, workp_shft)); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p2), q3); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, q0), q1); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_storel_epi64((__m128i *)&flat_oq1[0], + _mm_packus_epi16(workp_shft, workp_shft)); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p1), q3); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, q1), q2); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_storel_epi64((__m128i *)&flat_oq2[0], + _mm_packus_epi16(workp_shft, workp_shft)); + } + } + // lp filter + { + const __m128i t4 = _mm_set1_epi8(4); + const __m128i t3 = _mm_set1_epi8(3); + const __m128i t80 = _mm_set1_epi8(0x80); + const __m128i t1 = _mm_set1_epi8(0x1); + const __m128i ps1 = + _mm_xor_si128(_mm_loadl_epi64((__m128i *)(s - 2 * p)), t80); + const __m128i ps0 = + _mm_xor_si128(_mm_loadl_epi64((__m128i *)(s - 1 * p)), t80); + const __m128i qs0 = + _mm_xor_si128(_mm_loadl_epi64((__m128i *)(s + 0 * p)), t80); + const __m128i qs1 = + _mm_xor_si128(_mm_loadl_epi64((__m128i *)(s + 1 * p)), t80); + __m128i filt; + __m128i work_a; + __m128i filter1, filter2; + + filt = _mm_and_si128(_mm_subs_epi8(ps1, qs1), hev); + work_a = _mm_subs_epi8(qs0, ps0); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + // (aom_filter + 3 * (qs0 - ps0)) & mask + filt = _mm_and_si128(filt, mask); + + filter1 = _mm_adds_epi8(filt, t4); + filter2 = _mm_adds_epi8(filt, t3); + + // Filter1 >> 3 + filter1 = _mm_unpacklo_epi8(zero, filter1); + filter1 = _mm_srai_epi16(filter1, 11); + filter1 = _mm_packs_epi16(filter1, filter1); + + // Filter2 >> 3 + filter2 = _mm_unpacklo_epi8(zero, filter2); + filter2 = _mm_srai_epi16(filter2, 11); + filter2 = _mm_packs_epi16(filter2, zero); + + // filt >> 1 + filt = _mm_adds_epi8(filter1, t1); + filt = _mm_unpacklo_epi8(zero, filt); + filt = _mm_srai_epi16(filt, 9); + filt = _mm_packs_epi16(filt, zero); + + filt = _mm_andnot_si128(hev, filt); + + work_a = _mm_xor_si128(_mm_subs_epi8(qs0, filter1), t80); + q0 = _mm_loadl_epi64((__m128i *)flat_oq0); + work_a = _mm_andnot_si128(flat, work_a); + q0 = _mm_and_si128(flat, q0); + q0 = _mm_or_si128(work_a, q0); + + work_a = _mm_xor_si128(_mm_subs_epi8(qs1, filt), t80); + q1 = _mm_loadl_epi64((__m128i *)flat_oq1); + work_a = _mm_andnot_si128(flat, work_a); + q1 = _mm_and_si128(flat, q1); + q1 = _mm_or_si128(work_a, q1); + + work_a = _mm_loadu_si128((__m128i *)(s + 2 * p)); + q2 = _mm_loadl_epi64((__m128i *)flat_oq2); + work_a = _mm_andnot_si128(flat, work_a); + q2 = _mm_and_si128(flat, q2); + q2 = _mm_or_si128(work_a, q2); + + work_a = _mm_xor_si128(_mm_adds_epi8(ps0, filter2), t80); + p0 = _mm_loadl_epi64((__m128i *)flat_op0); + work_a = _mm_andnot_si128(flat, work_a); + p0 = _mm_and_si128(flat, p0); + p0 = _mm_or_si128(work_a, p0); + + work_a = _mm_xor_si128(_mm_adds_epi8(ps1, filt), t80); + p1 = _mm_loadl_epi64((__m128i *)flat_op1); + work_a = _mm_andnot_si128(flat, work_a); + p1 = _mm_and_si128(flat, p1); + p1 = _mm_or_si128(work_a, p1); + + work_a = _mm_loadu_si128((__m128i *)(s - 3 * p)); + p2 = _mm_loadl_epi64((__m128i *)flat_op2); + work_a = _mm_andnot_si128(flat, work_a); + p2 = _mm_and_si128(flat, p2); + p2 = _mm_or_si128(work_a, p2); + + _mm_storel_epi64((__m128i *)(s - 3 * p), p2); + _mm_storel_epi64((__m128i *)(s - 2 * p), p1); + _mm_storel_epi64((__m128i *)(s - 1 * p), p0); + _mm_storel_epi64((__m128i *)(s + 0 * p), q0); + _mm_storel_epi64((__m128i *)(s + 1 * p), q1); + _mm_storel_epi64((__m128i *)(s + 2 * p), q2); + } +} + +void aom_lpf_horizontal_8_dual_sse2(uint8_t *s, int p, const uint8_t *_blimit0, + const uint8_t *_limit0, + const uint8_t *_thresh0, + const uint8_t *_blimit1, + const uint8_t *_limit1, + const uint8_t *_thresh1) { + DECLARE_ALIGNED(16, unsigned char, flat_op2[16]); + DECLARE_ALIGNED(16, unsigned char, flat_op1[16]); + DECLARE_ALIGNED(16, unsigned char, flat_op0[16]); + DECLARE_ALIGNED(16, unsigned char, flat_oq2[16]); + DECLARE_ALIGNED(16, unsigned char, flat_oq1[16]); + DECLARE_ALIGNED(16, unsigned char, flat_oq0[16]); + const __m128i zero = _mm_set1_epi16(0); + const __m128i blimit = + _mm_unpacklo_epi64(_mm_load_si128((const __m128i *)_blimit0), + _mm_load_si128((const __m128i *)_blimit1)); + const __m128i limit = + _mm_unpacklo_epi64(_mm_load_si128((const __m128i *)_limit0), + _mm_load_si128((const __m128i *)_limit1)); + const __m128i thresh = + _mm_unpacklo_epi64(_mm_load_si128((const __m128i *)_thresh0), + _mm_load_si128((const __m128i *)_thresh1)); + + __m128i mask, hev, flat; + __m128i p3, p2, p1, p0, q0, q1, q2, q3; + + p3 = _mm_loadu_si128((__m128i *)(s - 4 * p)); + p2 = _mm_loadu_si128((__m128i *)(s - 3 * p)); + p1 = _mm_loadu_si128((__m128i *)(s - 2 * p)); + p0 = _mm_loadu_si128((__m128i *)(s - 1 * p)); + q0 = _mm_loadu_si128((__m128i *)(s - 0 * p)); + q1 = _mm_loadu_si128((__m128i *)(s + 1 * p)); + q2 = _mm_loadu_si128((__m128i *)(s + 2 * p)); + q3 = _mm_loadu_si128((__m128i *)(s + 3 * p)); + { + const __m128i abs_p1p0 = + _mm_or_si128(_mm_subs_epu8(p1, p0), _mm_subs_epu8(p0, p1)); + const __m128i abs_q1q0 = + _mm_or_si128(_mm_subs_epu8(q1, q0), _mm_subs_epu8(q0, q1)); + const __m128i one = _mm_set1_epi8(1); + const __m128i fe = _mm_set1_epi8(0xfe); + const __m128i ff = _mm_cmpeq_epi8(abs_p1p0, abs_p1p0); + __m128i abs_p0q0 = + _mm_or_si128(_mm_subs_epu8(p0, q0), _mm_subs_epu8(q0, p0)); + __m128i abs_p1q1 = + _mm_or_si128(_mm_subs_epu8(p1, q1), _mm_subs_epu8(q1, p1)); + __m128i work; + + // filter_mask and hev_mask + flat = _mm_max_epu8(abs_p1p0, abs_q1q0); + hev = _mm_subs_epu8(flat, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi8(hev, zero), ff); + + abs_p0q0 = _mm_adds_epu8(abs_p0q0, abs_p0q0); + abs_p1q1 = _mm_srli_epi16(_mm_and_si128(abs_p1q1, fe), 1); + mask = _mm_subs_epu8(_mm_adds_epu8(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi8(mask, zero), ff); + // mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + mask = _mm_max_epu8(flat, mask); + // mask |= (abs(p1 - p0) > limit) * -1; + // mask |= (abs(q1 - q0) > limit) * -1; + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(p2, p1), _mm_subs_epu8(p1, p2)), + _mm_or_si128(_mm_subs_epu8(p3, p2), _mm_subs_epu8(p2, p3))); + mask = _mm_max_epu8(work, mask); + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(q2, q1), _mm_subs_epu8(q1, q2)), + _mm_or_si128(_mm_subs_epu8(q3, q2), _mm_subs_epu8(q2, q3))); + mask = _mm_max_epu8(work, mask); + mask = _mm_subs_epu8(mask, limit); + mask = _mm_cmpeq_epi8(mask, zero); + + // flat_mask4 + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(p2, p0), _mm_subs_epu8(p0, p2)), + _mm_or_si128(_mm_subs_epu8(q2, q0), _mm_subs_epu8(q0, q2))); + flat = _mm_max_epu8(work, flat); + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(p3, p0), _mm_subs_epu8(p0, p3)), + _mm_or_si128(_mm_subs_epu8(q3, q0), _mm_subs_epu8(q0, q3))); + flat = _mm_max_epu8(work, flat); + flat = _mm_subs_epu8(flat, one); + flat = _mm_cmpeq_epi8(flat, zero); + flat = _mm_and_si128(flat, mask); + } + { + const __m128i four = _mm_set1_epi16(4); + unsigned char *src = s; + int i = 0; + + do { + __m128i workp_a, workp_b, workp_shft; + p3 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src - 4 * p)), zero); + p2 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src - 3 * p)), zero); + p1 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src - 2 * p)), zero); + p0 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src - 1 * p)), zero); + q0 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src - 0 * p)), zero); + q1 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src + 1 * p)), zero); + q2 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src + 2 * p)), zero); + q3 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(src + 3 * p)), zero); + + workp_a = _mm_add_epi16(_mm_add_epi16(p3, p3), _mm_add_epi16(p2, p1)); + workp_a = _mm_add_epi16(_mm_add_epi16(workp_a, four), p0); + workp_b = _mm_add_epi16(_mm_add_epi16(q0, p2), p3); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_storel_epi64((__m128i *)&flat_op2[i * 8], + _mm_packus_epi16(workp_shft, workp_shft)); + + workp_b = _mm_add_epi16(_mm_add_epi16(q0, q1), p1); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_storel_epi64((__m128i *)&flat_op1[i * 8], + _mm_packus_epi16(workp_shft, workp_shft)); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p3), q2); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, p1), p0); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_storel_epi64((__m128i *)&flat_op0[i * 8], + _mm_packus_epi16(workp_shft, workp_shft)); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p3), q3); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, p0), q0); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_storel_epi64((__m128i *)&flat_oq0[i * 8], + _mm_packus_epi16(workp_shft, workp_shft)); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p2), q3); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, q0), q1); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_storel_epi64((__m128i *)&flat_oq1[i * 8], + _mm_packus_epi16(workp_shft, workp_shft)); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p1), q3); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, q1), q2); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_storel_epi64((__m128i *)&flat_oq2[i * 8], + _mm_packus_epi16(workp_shft, workp_shft)); + + src += 8; + } while (++i < 2); + } + // lp filter + { + const __m128i t4 = _mm_set1_epi8(4); + const __m128i t3 = _mm_set1_epi8(3); + const __m128i t80 = _mm_set1_epi8(0x80); + const __m128i te0 = _mm_set1_epi8(0xe0); + const __m128i t1f = _mm_set1_epi8(0x1f); + const __m128i t1 = _mm_set1_epi8(0x1); + const __m128i t7f = _mm_set1_epi8(0x7f); + + const __m128i ps1 = + _mm_xor_si128(_mm_loadu_si128((__m128i *)(s - 2 * p)), t80); + const __m128i ps0 = + _mm_xor_si128(_mm_loadu_si128((__m128i *)(s - 1 * p)), t80); + const __m128i qs0 = + _mm_xor_si128(_mm_loadu_si128((__m128i *)(s + 0 * p)), t80); + const __m128i qs1 = + _mm_xor_si128(_mm_loadu_si128((__m128i *)(s + 1 * p)), t80); + __m128i filt; + __m128i work_a; + __m128i filter1, filter2; + + filt = _mm_and_si128(_mm_subs_epi8(ps1, qs1), hev); + work_a = _mm_subs_epi8(qs0, ps0); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + // (aom_filter + 3 * (qs0 - ps0)) & mask + filt = _mm_and_si128(filt, mask); + + filter1 = _mm_adds_epi8(filt, t4); + filter2 = _mm_adds_epi8(filt, t3); + + // Filter1 >> 3 + work_a = _mm_cmpgt_epi8(zero, filter1); + filter1 = _mm_srli_epi16(filter1, 3); + work_a = _mm_and_si128(work_a, te0); + filter1 = _mm_and_si128(filter1, t1f); + filter1 = _mm_or_si128(filter1, work_a); + + // Filter2 >> 3 + work_a = _mm_cmpgt_epi8(zero, filter2); + filter2 = _mm_srli_epi16(filter2, 3); + work_a = _mm_and_si128(work_a, te0); + filter2 = _mm_and_si128(filter2, t1f); + filter2 = _mm_or_si128(filter2, work_a); + + // filt >> 1 + filt = _mm_adds_epi8(filter1, t1); + work_a = _mm_cmpgt_epi8(zero, filt); + filt = _mm_srli_epi16(filt, 1); + work_a = _mm_and_si128(work_a, t80); + filt = _mm_and_si128(filt, t7f); + filt = _mm_or_si128(filt, work_a); + + filt = _mm_andnot_si128(hev, filt); + + work_a = _mm_xor_si128(_mm_subs_epi8(qs0, filter1), t80); + q0 = _mm_load_si128((__m128i *)flat_oq0); + work_a = _mm_andnot_si128(flat, work_a); + q0 = _mm_and_si128(flat, q0); + q0 = _mm_or_si128(work_a, q0); + + work_a = _mm_xor_si128(_mm_subs_epi8(qs1, filt), t80); + q1 = _mm_load_si128((__m128i *)flat_oq1); + work_a = _mm_andnot_si128(flat, work_a); + q1 = _mm_and_si128(flat, q1); + q1 = _mm_or_si128(work_a, q1); + + work_a = _mm_loadu_si128((__m128i *)(s + 2 * p)); + q2 = _mm_load_si128((__m128i *)flat_oq2); + work_a = _mm_andnot_si128(flat, work_a); + q2 = _mm_and_si128(flat, q2); + q2 = _mm_or_si128(work_a, q2); + + work_a = _mm_xor_si128(_mm_adds_epi8(ps0, filter2), t80); + p0 = _mm_load_si128((__m128i *)flat_op0); + work_a = _mm_andnot_si128(flat, work_a); + p0 = _mm_and_si128(flat, p0); + p0 = _mm_or_si128(work_a, p0); + + work_a = _mm_xor_si128(_mm_adds_epi8(ps1, filt), t80); + p1 = _mm_load_si128((__m128i *)flat_op1); + work_a = _mm_andnot_si128(flat, work_a); + p1 = _mm_and_si128(flat, p1); + p1 = _mm_or_si128(work_a, p1); + + work_a = _mm_loadu_si128((__m128i *)(s - 3 * p)); + p2 = _mm_load_si128((__m128i *)flat_op2); + work_a = _mm_andnot_si128(flat, work_a); + p2 = _mm_and_si128(flat, p2); + p2 = _mm_or_si128(work_a, p2); + + _mm_storeu_si128((__m128i *)(s - 3 * p), p2); + _mm_storeu_si128((__m128i *)(s - 2 * p), p1); + _mm_storeu_si128((__m128i *)(s - 1 * p), p0); + _mm_storeu_si128((__m128i *)(s + 0 * p), q0); + _mm_storeu_si128((__m128i *)(s + 1 * p), q1); + _mm_storeu_si128((__m128i *)(s + 2 * p), q2); + } +} + +void aom_lpf_horizontal_4_dual_sse2(unsigned char *s, int p, + const unsigned char *_blimit0, + const unsigned char *_limit0, + const unsigned char *_thresh0, + const unsigned char *_blimit1, + const unsigned char *_limit1, + const unsigned char *_thresh1) { + const __m128i blimit = + _mm_unpacklo_epi64(_mm_load_si128((const __m128i *)_blimit0), + _mm_load_si128((const __m128i *)_blimit1)); + const __m128i limit = + _mm_unpacklo_epi64(_mm_load_si128((const __m128i *)_limit0), + _mm_load_si128((const __m128i *)_limit1)); + const __m128i thresh = + _mm_unpacklo_epi64(_mm_load_si128((const __m128i *)_thresh0), + _mm_load_si128((const __m128i *)_thresh1)); + const __m128i zero = _mm_set1_epi16(0); +#if !CONFIG_PARALLEL_DEBLOCKING + __m128i p3, p2, q2, q3; +#endif // !CONFIG_PARALLEL_DEBLOCKING + __m128i p1, p0, q0, q1; + __m128i mask, hev, flat; +#if !CONFIG_PARALLEL_DEBLOCKING + p3 = _mm_loadu_si128((__m128i *)(s - 4 * p)); + p2 = _mm_loadu_si128((__m128i *)(s - 3 * p)); +#endif // !CONFIG_PARALLEL_DEBLOCKING + p1 = _mm_loadu_si128((__m128i *)(s - 2 * p)); + p0 = _mm_loadu_si128((__m128i *)(s - 1 * p)); + q0 = _mm_loadu_si128((__m128i *)(s - 0 * p)); + q1 = _mm_loadu_si128((__m128i *)(s + 1 * p)); +#if !CONFIG_PARALLEL_DEBLOCKING + q2 = _mm_loadu_si128((__m128i *)(s + 2 * p)); + q3 = _mm_loadu_si128((__m128i *)(s + 3 * p)); +#endif // !CONFIG_PARALLEL_DEBLOCKING + // filter_mask and hev_mask + { + const __m128i abs_p1p0 = + _mm_or_si128(_mm_subs_epu8(p1, p0), _mm_subs_epu8(p0, p1)); + const __m128i abs_q1q0 = + _mm_or_si128(_mm_subs_epu8(q1, q0), _mm_subs_epu8(q0, q1)); + const __m128i fe = _mm_set1_epi8(0xfe); + const __m128i ff = _mm_cmpeq_epi8(abs_p1p0, abs_p1p0); + __m128i abs_p0q0 = + _mm_or_si128(_mm_subs_epu8(p0, q0), _mm_subs_epu8(q0, p0)); + __m128i abs_p1q1 = + _mm_or_si128(_mm_subs_epu8(p1, q1), _mm_subs_epu8(q1, p1)); +#if !CONFIG_PARALLEL_DEBLOCKING + __m128i work; +#endif // !CONFIG_PARALLEL_DEBLOCKING + flat = _mm_max_epu8(abs_p1p0, abs_q1q0); + hev = _mm_subs_epu8(flat, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi8(hev, zero), ff); + + abs_p0q0 = _mm_adds_epu8(abs_p0q0, abs_p0q0); + abs_p1q1 = _mm_srli_epi16(_mm_and_si128(abs_p1q1, fe), 1); + mask = _mm_subs_epu8(_mm_adds_epu8(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi8(mask, zero), ff); + // mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + mask = _mm_max_epu8(flat, mask); +#if !CONFIG_PARALLEL_DEBLOCKING + // mask |= (abs(p1 - p0) > limit) * -1; + // mask |= (abs(q1 - q0) > limit) * -1; + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(p2, p1), _mm_subs_epu8(p1, p2)), + _mm_or_si128(_mm_subs_epu8(p3, p2), _mm_subs_epu8(p2, p3))); + mask = _mm_max_epu8(work, mask); + work = _mm_max_epu8( + _mm_or_si128(_mm_subs_epu8(q2, q1), _mm_subs_epu8(q1, q2)), + _mm_or_si128(_mm_subs_epu8(q3, q2), _mm_subs_epu8(q2, q3))); + mask = _mm_max_epu8(work, mask); +#endif // !CONFIG_PARALLEL_DEBLOCKING + mask = _mm_subs_epu8(mask, limit); + mask = _mm_cmpeq_epi8(mask, zero); + } + + // filter4 + { + const __m128i t4 = _mm_set1_epi8(4); + const __m128i t3 = _mm_set1_epi8(3); + const __m128i t80 = _mm_set1_epi8(0x80); + const __m128i te0 = _mm_set1_epi8(0xe0); + const __m128i t1f = _mm_set1_epi8(0x1f); + const __m128i t1 = _mm_set1_epi8(0x1); + const __m128i t7f = _mm_set1_epi8(0x7f); + + const __m128i ps1 = + _mm_xor_si128(_mm_loadu_si128((__m128i *)(s - 2 * p)), t80); + const __m128i ps0 = + _mm_xor_si128(_mm_loadu_si128((__m128i *)(s - 1 * p)), t80); + const __m128i qs0 = + _mm_xor_si128(_mm_loadu_si128((__m128i *)(s + 0 * p)), t80); + const __m128i qs1 = + _mm_xor_si128(_mm_loadu_si128((__m128i *)(s + 1 * p)), t80); + __m128i filt; + __m128i work_a; + __m128i filter1, filter2; + + filt = _mm_and_si128(_mm_subs_epi8(ps1, qs1), hev); + work_a = _mm_subs_epi8(qs0, ps0); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + filt = _mm_adds_epi8(filt, work_a); + // (aom_filter + 3 * (qs0 - ps0)) & mask + filt = _mm_and_si128(filt, mask); + + filter1 = _mm_adds_epi8(filt, t4); + filter2 = _mm_adds_epi8(filt, t3); + + // Filter1 >> 3 + work_a = _mm_cmpgt_epi8(zero, filter1); + filter1 = _mm_srli_epi16(filter1, 3); + work_a = _mm_and_si128(work_a, te0); + filter1 = _mm_and_si128(filter1, t1f); + filter1 = _mm_or_si128(filter1, work_a); + + // Filter2 >> 3 + work_a = _mm_cmpgt_epi8(zero, filter2); + filter2 = _mm_srli_epi16(filter2, 3); + work_a = _mm_and_si128(work_a, te0); + filter2 = _mm_and_si128(filter2, t1f); + filter2 = _mm_or_si128(filter2, work_a); + + // filt >> 1 + filt = _mm_adds_epi8(filter1, t1); + work_a = _mm_cmpgt_epi8(zero, filt); + filt = _mm_srli_epi16(filt, 1); + work_a = _mm_and_si128(work_a, t80); + filt = _mm_and_si128(filt, t7f); + filt = _mm_or_si128(filt, work_a); + + filt = _mm_andnot_si128(hev, filt); + + q0 = _mm_xor_si128(_mm_subs_epi8(qs0, filter1), t80); + q1 = _mm_xor_si128(_mm_subs_epi8(qs1, filt), t80); + p0 = _mm_xor_si128(_mm_adds_epi8(ps0, filter2), t80); + p1 = _mm_xor_si128(_mm_adds_epi8(ps1, filt), t80); + + _mm_storeu_si128((__m128i *)(s - 2 * p), p1); + _mm_storeu_si128((__m128i *)(s - 1 * p), p0); + _mm_storeu_si128((__m128i *)(s + 0 * p), q0); + _mm_storeu_si128((__m128i *)(s + 1 * p), q1); + } +} + +static INLINE void transpose8x16(unsigned char *in0, unsigned char *in1, + int in_p, unsigned char *out, int out_p) { + __m128i x0, x1, x2, x3, x4, x5, x6, x7; + __m128i x8, x9, x10, x11, x12, x13, x14, x15; + + // 2-way interleave w/hoisting of unpacks + x0 = _mm_loadl_epi64((__m128i *)in0); // 1 + x1 = _mm_loadl_epi64((__m128i *)(in0 + in_p)); // 3 + x0 = _mm_unpacklo_epi8(x0, x1); // 1 + + x2 = _mm_loadl_epi64((__m128i *)(in0 + 2 * in_p)); // 5 + x3 = _mm_loadl_epi64((__m128i *)(in0 + 3 * in_p)); // 7 + x1 = _mm_unpacklo_epi8(x2, x3); // 2 + + x4 = _mm_loadl_epi64((__m128i *)(in0 + 4 * in_p)); // 9 + x5 = _mm_loadl_epi64((__m128i *)(in0 + 5 * in_p)); // 11 + x2 = _mm_unpacklo_epi8(x4, x5); // 3 + + x6 = _mm_loadl_epi64((__m128i *)(in0 + 6 * in_p)); // 13 + x7 = _mm_loadl_epi64((__m128i *)(in0 + 7 * in_p)); // 15 + x3 = _mm_unpacklo_epi8(x6, x7); // 4 + x4 = _mm_unpacklo_epi16(x0, x1); // 9 + + x8 = _mm_loadl_epi64((__m128i *)in1); // 2 + x9 = _mm_loadl_epi64((__m128i *)(in1 + in_p)); // 4 + x8 = _mm_unpacklo_epi8(x8, x9); // 5 + x5 = _mm_unpacklo_epi16(x2, x3); // 10 + + x10 = _mm_loadl_epi64((__m128i *)(in1 + 2 * in_p)); // 6 + x11 = _mm_loadl_epi64((__m128i *)(in1 + 3 * in_p)); // 8 + x9 = _mm_unpacklo_epi8(x10, x11); // 6 + + x12 = _mm_loadl_epi64((__m128i *)(in1 + 4 * in_p)); // 10 + x13 = _mm_loadl_epi64((__m128i *)(in1 + 5 * in_p)); // 12 + x10 = _mm_unpacklo_epi8(x12, x13); // 7 + x12 = _mm_unpacklo_epi16(x8, x9); // 11 + + x14 = _mm_loadl_epi64((__m128i *)(in1 + 6 * in_p)); // 14 + x15 = _mm_loadl_epi64((__m128i *)(in1 + 7 * in_p)); // 16 + x11 = _mm_unpacklo_epi8(x14, x15); // 8 + x13 = _mm_unpacklo_epi16(x10, x11); // 12 + + x6 = _mm_unpacklo_epi32(x4, x5); // 13 + x7 = _mm_unpackhi_epi32(x4, x5); // 14 + x14 = _mm_unpacklo_epi32(x12, x13); // 15 + x15 = _mm_unpackhi_epi32(x12, x13); // 16 + + // Store first 4-line result + _mm_storeu_si128((__m128i *)out, _mm_unpacklo_epi64(x6, x14)); + _mm_storeu_si128((__m128i *)(out + out_p), _mm_unpackhi_epi64(x6, x14)); + _mm_storeu_si128((__m128i *)(out + 2 * out_p), _mm_unpacklo_epi64(x7, x15)); + _mm_storeu_si128((__m128i *)(out + 3 * out_p), _mm_unpackhi_epi64(x7, x15)); + + x4 = _mm_unpackhi_epi16(x0, x1); + x5 = _mm_unpackhi_epi16(x2, x3); + x12 = _mm_unpackhi_epi16(x8, x9); + x13 = _mm_unpackhi_epi16(x10, x11); + + x6 = _mm_unpacklo_epi32(x4, x5); + x7 = _mm_unpackhi_epi32(x4, x5); + x14 = _mm_unpacklo_epi32(x12, x13); + x15 = _mm_unpackhi_epi32(x12, x13); + + // Store second 4-line result + _mm_storeu_si128((__m128i *)(out + 4 * out_p), _mm_unpacklo_epi64(x6, x14)); + _mm_storeu_si128((__m128i *)(out + 5 * out_p), _mm_unpackhi_epi64(x6, x14)); + _mm_storeu_si128((__m128i *)(out + 6 * out_p), _mm_unpacklo_epi64(x7, x15)); + _mm_storeu_si128((__m128i *)(out + 7 * out_p), _mm_unpackhi_epi64(x7, x15)); +} + +#if CONFIG_PARALLEL_DEBLOCKING +#define movq(p) _mm_loadl_epi64((const __m128i *)(p)) +#define punpcklbw(r0, r1) _mm_unpacklo_epi8(r0, r1) +#define punpcklwd(r0, r1) _mm_unpacklo_epi16(r0, r1) +#define punpckhwd(r0, r1) _mm_unpackhi_epi16(r0, r1) +#define movd(p, r) *((uint32_t *)(p)) = _mm_cvtsi128_si32(r) +#define pshufd(r, imm) _mm_shuffle_epi32(r, imm) +enum { ROTATE_DWORD_RIGHT = 0x39 }; +static INLINE void transpose16x4(uint8_t *pDst, const ptrdiff_t dstStride, + const uint8_t *pSrc, + const ptrdiff_t srcStride) { + for (uint32_t idx = 0; idx < 2; idx += 1) { + __m128i r0, r1, r2, r3; + // load data + r0 = movq(pSrc); + r1 = movq(pSrc + srcStride); + r2 = movq(pSrc + srcStride * 2); + r3 = movq(pSrc + srcStride * 3); + // transpose + r0 = punpcklbw(r0, r1); + r2 = punpcklbw(r2, r3); + r1 = punpckhwd(r0, r2); + r0 = punpcklwd(r0, r2); + // store data + movd(pDst, r0); + r0 = pshufd(r0, ROTATE_DWORD_RIGHT); + movd(pDst + dstStride, r0); + r0 = pshufd(r0, ROTATE_DWORD_RIGHT); + movd(pDst + dstStride * 2, r0); + r0 = pshufd(r0, ROTATE_DWORD_RIGHT); + movd(pDst + dstStride * 3, r0); + movd(pDst + dstStride * 4, r1); + r1 = pshufd(r1, ROTATE_DWORD_RIGHT); + movd(pDst + dstStride * 5, r1); + r1 = pshufd(r1, ROTATE_DWORD_RIGHT); + movd(pDst + dstStride * 6, r1); + r1 = pshufd(r1, ROTATE_DWORD_RIGHT); + movd(pDst + dstStride * 7, r1); + // advance the pointers + pDst += dstStride * 8; + pSrc += 8; + } +} + +#endif // CONFIG_PARALLEL_DEBLOCKING +static INLINE void transpose(unsigned char *src[], int in_p, + unsigned char *dst[], int out_p, + int num_8x8_to_transpose) { + int idx8x8 = 0; + __m128i x0, x1, x2, x3, x4, x5, x6, x7; + do { + unsigned char *in = src[idx8x8]; + unsigned char *out = dst[idx8x8]; + + x0 = + _mm_loadl_epi64((__m128i *)(in + 0 * in_p)); // 00 01 02 03 04 05 06 07 + x1 = + _mm_loadl_epi64((__m128i *)(in + 1 * in_p)); // 10 11 12 13 14 15 16 17 + // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 + x0 = _mm_unpacklo_epi8(x0, x1); + + x2 = + _mm_loadl_epi64((__m128i *)(in + 2 * in_p)); // 20 21 22 23 24 25 26 27 + x3 = + _mm_loadl_epi64((__m128i *)(in + 3 * in_p)); // 30 31 32 33 34 35 36 37 + // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 + x1 = _mm_unpacklo_epi8(x2, x3); + + x4 = + _mm_loadl_epi64((__m128i *)(in + 4 * in_p)); // 40 41 42 43 44 45 46 47 + x5 = + _mm_loadl_epi64((__m128i *)(in + 5 * in_p)); // 50 51 52 53 54 55 56 57 + // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57 + x2 = _mm_unpacklo_epi8(x4, x5); + + x6 = + _mm_loadl_epi64((__m128i *)(in + 6 * in_p)); // 60 61 62 63 64 65 66 67 + x7 = + _mm_loadl_epi64((__m128i *)(in + 7 * in_p)); // 70 71 72 73 74 75 76 77 + // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77 + x3 = _mm_unpacklo_epi8(x6, x7); + + // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 + x4 = _mm_unpacklo_epi16(x0, x1); + // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73 + x5 = _mm_unpacklo_epi16(x2, x3); + // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 + x6 = _mm_unpacklo_epi32(x4, x5); + _mm_storel_pd((double *)(out + 0 * out_p), + _mm_castsi128_pd(x6)); // 00 10 20 30 40 50 60 70 + _mm_storeh_pd((double *)(out + 1 * out_p), + _mm_castsi128_pd(x6)); // 01 11 21 31 41 51 61 71 + // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 + x7 = _mm_unpackhi_epi32(x4, x5); + _mm_storel_pd((double *)(out + 2 * out_p), + _mm_castsi128_pd(x7)); // 02 12 22 32 42 52 62 72 + _mm_storeh_pd((double *)(out + 3 * out_p), + _mm_castsi128_pd(x7)); // 03 13 23 33 43 53 63 73 + + // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37 + x4 = _mm_unpackhi_epi16(x0, x1); + // 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77 + x5 = _mm_unpackhi_epi16(x2, x3); + // 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75 + x6 = _mm_unpacklo_epi32(x4, x5); + _mm_storel_pd((double *)(out + 4 * out_p), + _mm_castsi128_pd(x6)); // 04 14 24 34 44 54 64 74 + _mm_storeh_pd((double *)(out + 5 * out_p), + _mm_castsi128_pd(x6)); // 05 15 25 35 45 55 65 75 + // 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77 + x7 = _mm_unpackhi_epi32(x4, x5); + + _mm_storel_pd((double *)(out + 6 * out_p), + _mm_castsi128_pd(x7)); // 06 16 26 36 46 56 66 76 + _mm_storeh_pd((double *)(out + 7 * out_p), + _mm_castsi128_pd(x7)); // 07 17 27 37 47 57 67 77 + } while (++idx8x8 < num_8x8_to_transpose); +} + +void aom_lpf_vertical_4_dual_sse2(uint8_t *s, int p, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, + const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1) { + DECLARE_ALIGNED(16, unsigned char, t_dst[16 * 8]); +#if !CONFIG_PARALLEL_DEBLOCKING + unsigned char *src[2]; + unsigned char *dst[2]; +#endif // !CONFIG_PARALLEL_DEBLOCKING + // Transpose 8x16 + transpose8x16(s - 4, s - 4 + p * 8, p, t_dst, 16); + + // Loop filtering + aom_lpf_horizontal_4_dual_sse2(t_dst + 4 * 16, 16, blimit0, limit0, thresh0, + blimit1, limit1, thresh1); +#if !CONFIG_PARALLEL_DEBLOCKING + src[0] = t_dst; + src[1] = t_dst + 8; + dst[0] = s - 4; + dst[1] = s - 4 + p * 8; + + // Transpose back + transpose(src, 16, dst, p, 2); +#else // CONFIG_PARALLEL_DEBLOCKING + transpose16x4(s - 2, p, t_dst + 16 * 2, 16); +#endif // !CONFIG_PARALLEL_DEBLOCKING +} + +void aom_lpf_vertical_8_sse2(unsigned char *s, int p, + const unsigned char *blimit, + const unsigned char *limit, + const unsigned char *thresh) { + DECLARE_ALIGNED(8, unsigned char, t_dst[8 * 8]); + unsigned char *src[1]; + unsigned char *dst[1]; + + // Transpose 8x8 + src[0] = s - 4; + dst[0] = t_dst; + + transpose(src, p, dst, 8, 1); + + // Loop filtering + aom_lpf_horizontal_8_sse2(t_dst + 4 * 8, 8, blimit, limit, thresh); + + src[0] = t_dst; + dst[0] = s - 4; + + // Transpose back + transpose(src, 8, dst, p, 1); +} + +void aom_lpf_vertical_8_dual_sse2(uint8_t *s, int p, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, + const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1) { + DECLARE_ALIGNED(16, unsigned char, t_dst[16 * 8]); + unsigned char *src[2]; + unsigned char *dst[2]; + + // Transpose 8x16 + transpose8x16(s - 4, s - 4 + p * 8, p, t_dst, 16); + + // Loop filtering + aom_lpf_horizontal_8_dual_sse2(t_dst + 4 * 16, 16, blimit0, limit0, thresh0, + blimit1, limit1, thresh1); + src[0] = t_dst; + src[1] = t_dst + 8; + + dst[0] = s - 4; + dst[1] = s - 4 + p * 8; + + // Transpose back + transpose(src, 16, dst, p, 2); +} + +void aom_lpf_vertical_16_sse2(unsigned char *s, int p, + const unsigned char *blimit, + const unsigned char *limit, + const unsigned char *thresh) { + DECLARE_ALIGNED(8, unsigned char, t_dst[8 * 16]); + unsigned char *src[2]; + unsigned char *dst[2]; + + src[0] = s - 8; + src[1] = s; + dst[0] = t_dst; + dst[1] = t_dst + 8 * 8; + + // Transpose 16x8 + transpose(src, p, dst, 8, 2); + + // Loop filtering + aom_lpf_horizontal_edge_8_sse2(t_dst + 8 * 8, 8, blimit, limit, thresh); + + src[0] = t_dst; + src[1] = t_dst + 8 * 8; + dst[0] = s - 8; + dst[1] = s; + + // Transpose back + transpose(src, 8, dst, p, 2); +} + +void aom_lpf_vertical_16_dual_sse2(unsigned char *s, int p, + const uint8_t *blimit, const uint8_t *limit, + const uint8_t *thresh) { + DECLARE_ALIGNED(16, unsigned char, t_dst[256]); + + // Transpose 16x16 + transpose8x16(s - 8, s - 8 + 8 * p, p, t_dst, 16); + transpose8x16(s, s + 8 * p, p, t_dst + 8 * 16, 16); + + // Loop filtering + aom_lpf_horizontal_edge_16_sse2(t_dst + 8 * 16, 16, blimit, limit, thresh); + + // Transpose back + transpose8x16(t_dst, t_dst + 8 * 16, 16, s - 8, p); + transpose8x16(t_dst + 8, t_dst + 8 + 8 * 16, 16, s - 8 + 8 * p, p); +} diff --git a/third_party/aom/aom_dsp/x86/masked_sad_intrin_ssse3.c b/third_party/aom/aom_dsp/x86/masked_sad_intrin_ssse3.c new file mode 100644 index 0000000000..5166e9e0af --- /dev/null +++ b/third_party/aom/aom_dsp/x86/masked_sad_intrin_ssse3.c @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "aom_ports/mem.h" +#include "./aom_config.h" +#include "aom/aom_integer.h" + +static INLINE __m128i width8_load_2rows(const uint8_t *ptr, int stride) { + __m128i temp1 = _mm_loadl_epi64((const __m128i *)ptr); + __m128i temp2 = _mm_loadl_epi64((const __m128i *)(ptr + stride)); + return _mm_unpacklo_epi64(temp1, temp2); +} + +static INLINE __m128i width4_load_4rows(const uint8_t *ptr, int stride) { + __m128i temp1 = _mm_cvtsi32_si128(*(const uint32_t *)ptr); + __m128i temp2 = _mm_cvtsi32_si128(*(const uint32_t *)(ptr + stride)); + __m128i temp3 = _mm_unpacklo_epi32(temp1, temp2); + temp1 = _mm_cvtsi32_si128(*(const uint32_t *)(ptr + stride * 2)); + temp2 = _mm_cvtsi32_si128(*(const uint32_t *)(ptr + stride * 3)); + temp1 = _mm_unpacklo_epi32(temp1, temp2); + return _mm_unpacklo_epi64(temp3, temp1); +} + +static INLINE unsigned int masked_sad_ssse3(const uint8_t *a_ptr, int a_stride, + const uint8_t *b_ptr, int b_stride, + const uint8_t *m_ptr, int m_stride, + int width, int height); + +static INLINE unsigned int masked_sad8xh_ssse3( + const uint8_t *a_ptr, int a_stride, const uint8_t *b_ptr, int b_stride, + const uint8_t *m_ptr, int m_stride, int height); + +static INLINE unsigned int masked_sad4xh_ssse3( + const uint8_t *a_ptr, int a_stride, const uint8_t *b_ptr, int b_stride, + const uint8_t *m_ptr, int m_stride, int height); + +#define MASKSADMXN_SSSE3(m, n) \ + unsigned int aom_masked_sad##m##x##n##_ssse3( \ + const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ + const uint8_t *msk, int msk_stride) { \ + return masked_sad_ssse3(src, src_stride, ref, ref_stride, msk, msk_stride, \ + m, n); \ + } + +#if CONFIG_EXT_PARTITION +MASKSADMXN_SSSE3(128, 128) +MASKSADMXN_SSSE3(128, 64) +MASKSADMXN_SSSE3(64, 128) +#endif // CONFIG_EXT_PARTITION +MASKSADMXN_SSSE3(64, 64) +MASKSADMXN_SSSE3(64, 32) +MASKSADMXN_SSSE3(32, 64) +MASKSADMXN_SSSE3(32, 32) +MASKSADMXN_SSSE3(32, 16) +MASKSADMXN_SSSE3(16, 32) +MASKSADMXN_SSSE3(16, 16) +MASKSADMXN_SSSE3(16, 8) + +#define MASKSAD8XN_SSSE3(n) \ + unsigned int aom_masked_sad8x##n##_ssse3( \ + const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ + const uint8_t *msk, int msk_stride) { \ + return masked_sad8xh_ssse3(src, src_stride, ref, ref_stride, msk, \ + msk_stride, n); \ + } + +MASKSAD8XN_SSSE3(16) +MASKSAD8XN_SSSE3(8) +MASKSAD8XN_SSSE3(4) + +#define MASKSAD4XN_SSSE3(n) \ + unsigned int aom_masked_sad4x##n##_ssse3( \ + const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ + const uint8_t *msk, int msk_stride) { \ + return masked_sad4xh_ssse3(src, src_stride, ref, ref_stride, msk, \ + msk_stride, n); \ + } + +MASKSAD4XN_SSSE3(8) +MASKSAD4XN_SSSE3(4) + +// For width a multiple of 16 +// Assumes values in m are <=64 +static INLINE unsigned int masked_sad_ssse3(const uint8_t *a_ptr, int a_stride, + const uint8_t *b_ptr, int b_stride, + const uint8_t *m_ptr, int m_stride, + int width, int height) { + int y, x; + __m128i a, b, m, temp1, temp2; + __m128i res = _mm_setzero_si128(); + __m128i one = _mm_set1_epi16(1); + // For each row + for (y = 0; y < height; y++) { + // Covering the full width + for (x = 0; x < width; x += 16) { + // Load a, b, m in xmm registers + a = _mm_loadu_si128((const __m128i *)(a_ptr + x)); + b = _mm_loadu_si128((const __m128i *)(b_ptr + x)); + m = _mm_loadu_si128((const __m128i *)(m_ptr + x)); + + // Calculate the difference between a & b + temp1 = _mm_subs_epu8(a, b); + temp2 = _mm_subs_epu8(b, a); + temp1 = _mm_or_si128(temp1, temp2); + + // Multiply by m and add together + temp2 = _mm_maddubs_epi16(temp1, m); + // Pad out row result to 32 bit integers & add to running total + res = _mm_add_epi32(res, _mm_madd_epi16(temp2, one)); + } + // Move onto the next row + a_ptr += a_stride; + b_ptr += b_stride; + m_ptr += m_stride; + } + res = _mm_hadd_epi32(res, _mm_setzero_si128()); + res = _mm_hadd_epi32(res, _mm_setzero_si128()); + // sad = (sad + 31) >> 6; + return (_mm_cvtsi128_si32(res) + 31) >> 6; +} + +static INLINE unsigned int masked_sad8xh_ssse3( + const uint8_t *a_ptr, int a_stride, const uint8_t *b_ptr, int b_stride, + const uint8_t *m_ptr, int m_stride, int height) { + int y; + __m128i a, b, m, temp1, temp2, row_res; + __m128i res = _mm_setzero_si128(); + __m128i one = _mm_set1_epi16(1); + // Add the masked SAD for 2 rows at a time + for (y = 0; y < height; y += 2) { + // Load a, b, m in xmm registers + a = width8_load_2rows(a_ptr, a_stride); + b = width8_load_2rows(b_ptr, b_stride); + m = width8_load_2rows(m_ptr, m_stride); + + // Calculate the difference between a & b + temp1 = _mm_subs_epu8(a, b); + temp2 = _mm_subs_epu8(b, a); + temp1 = _mm_or_si128(temp1, temp2); + + // Multiply by m and add together + row_res = _mm_maddubs_epi16(temp1, m); + + // Pad out row result to 32 bit integers & add to running total + res = _mm_add_epi32(res, _mm_madd_epi16(row_res, one)); + + // Move onto the next rows + a_ptr += a_stride * 2; + b_ptr += b_stride * 2; + m_ptr += m_stride * 2; + } + res = _mm_hadd_epi32(res, _mm_setzero_si128()); + res = _mm_hadd_epi32(res, _mm_setzero_si128()); + // sad = (sad + 31) >> 6; + return (_mm_cvtsi128_si32(res) + 31) >> 6; +} + +static INLINE unsigned int masked_sad4xh_ssse3( + const uint8_t *a_ptr, int a_stride, const uint8_t *b_ptr, int b_stride, + const uint8_t *m_ptr, int m_stride, int height) { + int y; + __m128i a, b, m, temp1, temp2, row_res; + __m128i res = _mm_setzero_si128(); + __m128i one = _mm_set1_epi16(1); + // Add the masked SAD for 4 rows at a time + for (y = 0; y < height; y += 4) { + // Load a, b, m in xmm registers + a = width4_load_4rows(a_ptr, a_stride); + b = width4_load_4rows(b_ptr, b_stride); + m = width4_load_4rows(m_ptr, m_stride); + + // Calculate the difference between a & b + temp1 = _mm_subs_epu8(a, b); + temp2 = _mm_subs_epu8(b, a); + temp1 = _mm_or_si128(temp1, temp2); + + // Multiply by m and add together + row_res = _mm_maddubs_epi16(temp1, m); + + // Pad out row result to 32 bit integers & add to running total + res = _mm_add_epi32(res, _mm_madd_epi16(row_res, one)); + + // Move onto the next rows + a_ptr += a_stride * 4; + b_ptr += b_stride * 4; + m_ptr += m_stride * 4; + } + // Pad out row result to 32 bit integers & add to running total + res = _mm_hadd_epi32(res, _mm_setzero_si128()); + res = _mm_hadd_epi32(res, _mm_setzero_si128()); + // sad = (sad + 31) >> 6; + return (_mm_cvtsi128_si32(res) + 31) >> 6; +} + +#if CONFIG_HIGHBITDEPTH +static INLINE __m128i highbd_width4_load_2rows(const uint16_t *ptr, + int stride) { + __m128i temp1 = _mm_loadl_epi64((const __m128i *)ptr); + __m128i temp2 = _mm_loadl_epi64((const __m128i *)(ptr + stride)); + return _mm_unpacklo_epi64(temp1, temp2); +} + +static INLINE unsigned int highbd_masked_sad_ssse3( + const uint8_t *a8_ptr, int a_stride, const uint8_t *b8_ptr, int b_stride, + const uint8_t *m_ptr, int m_stride, int width, int height); + +static INLINE unsigned int highbd_masked_sad4xh_ssse3( + const uint8_t *a8_ptr, int a_stride, const uint8_t *b8_ptr, int b_stride, + const uint8_t *m_ptr, int m_stride, int height); + +#define HIGHBD_MASKSADMXN_SSSE3(m, n) \ + unsigned int aom_highbd_masked_sad##m##x##n##_ssse3( \ + const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ + const uint8_t *msk, int msk_stride) { \ + return highbd_masked_sad_ssse3(src, src_stride, ref, ref_stride, msk, \ + msk_stride, m, n); \ + } + +#if CONFIG_EXT_PARTITION +HIGHBD_MASKSADMXN_SSSE3(128, 128) +HIGHBD_MASKSADMXN_SSSE3(128, 64) +HIGHBD_MASKSADMXN_SSSE3(64, 128) +#endif // CONFIG_EXT_PARTITION +HIGHBD_MASKSADMXN_SSSE3(64, 64) +HIGHBD_MASKSADMXN_SSSE3(64, 32) +HIGHBD_MASKSADMXN_SSSE3(32, 64) +HIGHBD_MASKSADMXN_SSSE3(32, 32) +HIGHBD_MASKSADMXN_SSSE3(32, 16) +HIGHBD_MASKSADMXN_SSSE3(16, 32) +HIGHBD_MASKSADMXN_SSSE3(16, 16) +HIGHBD_MASKSADMXN_SSSE3(16, 8) +HIGHBD_MASKSADMXN_SSSE3(8, 16) +HIGHBD_MASKSADMXN_SSSE3(8, 8) +HIGHBD_MASKSADMXN_SSSE3(8, 4) + +#define HIGHBD_MASKSAD4XN_SSSE3(n) \ + unsigned int aom_highbd_masked_sad4x##n##_ssse3( \ + const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ + const uint8_t *msk, int msk_stride) { \ + return highbd_masked_sad4xh_ssse3(src, src_stride, ref, ref_stride, msk, \ + msk_stride, n); \ + } + +HIGHBD_MASKSAD4XN_SSSE3(8) +HIGHBD_MASKSAD4XN_SSSE3(4) + +// For width a multiple of 8 +// Assumes values in m are <=64 +static INLINE unsigned int highbd_masked_sad_ssse3( + const uint8_t *a8_ptr, int a_stride, const uint8_t *b8_ptr, int b_stride, + const uint8_t *m_ptr, int m_stride, int width, int height) { + int y, x; + __m128i a, b, m, temp1, temp2; + const uint16_t *a_ptr = CONVERT_TO_SHORTPTR(a8_ptr); + const uint16_t *b_ptr = CONVERT_TO_SHORTPTR(b8_ptr); + __m128i res = _mm_setzero_si128(); + // For each row + for (y = 0; y < height; y++) { + // Covering the full width + for (x = 0; x < width; x += 8) { + // Load a, b, m in xmm registers + a = _mm_loadu_si128((const __m128i *)(a_ptr + x)); + b = _mm_loadu_si128((const __m128i *)(b_ptr + x)); + m = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(m_ptr + x)), + _mm_setzero_si128()); + + // Calculate the difference between a & b + temp1 = _mm_subs_epu16(a, b); + temp2 = _mm_subs_epu16(b, a); + temp1 = _mm_or_si128(temp1, temp2); + + // Add result of multiplying by m and add pairs together to running total + res = _mm_add_epi32(res, _mm_madd_epi16(temp1, m)); + } + // Move onto the next row + a_ptr += a_stride; + b_ptr += b_stride; + m_ptr += m_stride; + } + res = _mm_hadd_epi32(res, _mm_setzero_si128()); + res = _mm_hadd_epi32(res, _mm_setzero_si128()); + // sad = (sad + 31) >> 6; + return (_mm_cvtsi128_si32(res) + 31) >> 6; +} + +static INLINE unsigned int highbd_masked_sad4xh_ssse3( + const uint8_t *a8_ptr, int a_stride, const uint8_t *b8_ptr, int b_stride, + const uint8_t *m_ptr, int m_stride, int height) { + int y; + __m128i a, b, m, temp1, temp2; + const uint16_t *a_ptr = CONVERT_TO_SHORTPTR(a8_ptr); + const uint16_t *b_ptr = CONVERT_TO_SHORTPTR(b8_ptr); + __m128i res = _mm_setzero_si128(); + // Add the masked SAD for 2 rows at a time + for (y = 0; y < height; y += 2) { + // Load a, b, m in xmm registers + a = highbd_width4_load_2rows(a_ptr, a_stride); + b = highbd_width4_load_2rows(b_ptr, b_stride); + temp1 = _mm_loadl_epi64((const __m128i *)m_ptr); + temp2 = _mm_loadl_epi64((const __m128i *)(m_ptr + m_stride)); + m = _mm_unpacklo_epi8(_mm_unpacklo_epi32(temp1, temp2), + _mm_setzero_si128()); + + // Calculate the difference between a & b + temp1 = _mm_subs_epu16(a, b); + temp2 = _mm_subs_epu16(b, a); + temp1 = _mm_or_si128(temp1, temp2); + + // Multiply by m and add together + res = _mm_add_epi32(res, _mm_madd_epi16(temp1, m)); + + // Move onto the next rows + a_ptr += a_stride * 2; + b_ptr += b_stride * 2; + m_ptr += m_stride * 2; + } + res = _mm_hadd_epi32(res, _mm_setzero_si128()); + res = _mm_hadd_epi32(res, _mm_setzero_si128()); + // sad = (sad + 31) >> 6; + return (_mm_cvtsi128_si32(res) + 31) >> 6; +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/x86/masked_variance_intrin_ssse3.c b/third_party/aom/aom_dsp/x86/masked_variance_intrin_ssse3.c new file mode 100644 index 0000000000..fe14597f64 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/masked_variance_intrin_ssse3.c @@ -0,0 +1,1948 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include + +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" +#include "aom_dsp/aom_filter.h" + +// Half pixel shift +#define HALF_PIXEL_OFFSET (BIL_SUBPEL_SHIFTS / 2) + +/***************************************************************************** + * Horizontal additions + *****************************************************************************/ + +static INLINE int32_t hsum_epi32_si32(__m128i v_d) { + v_d = _mm_hadd_epi32(v_d, v_d); + v_d = _mm_hadd_epi32(v_d, v_d); + return _mm_cvtsi128_si32(v_d); +} + +static INLINE int64_t hsum_epi64_si64(__m128i v_q) { + v_q = _mm_add_epi64(v_q, _mm_srli_si128(v_q, 8)); +#if ARCH_X86_64 + return _mm_cvtsi128_si64(v_q); +#else + { + int64_t tmp; + _mm_storel_epi64((__m128i *)&tmp, v_q); + return tmp; + } +#endif +} + +#if CONFIG_HIGHBITDEPTH +static INLINE int64_t hsum_epi32_si64(__m128i v_d) { + const __m128i v_sign_d = _mm_cmplt_epi32(v_d, _mm_setzero_si128()); + const __m128i v_0_q = _mm_unpacklo_epi32(v_d, v_sign_d); + const __m128i v_1_q = _mm_unpackhi_epi32(v_d, v_sign_d); + return hsum_epi64_si64(_mm_add_epi64(v_0_q, v_1_q)); +} +#endif // CONFIG_HIGHBITDEPTH + +static INLINE uint32_t calc_masked_variance(__m128i v_sum_d, __m128i v_sse_q, + uint32_t *sse, int w, int h) { + int64_t sum64; + uint64_t sse64; + + // Horizontal sum + sum64 = hsum_epi32_si32(v_sum_d); + sse64 = hsum_epi64_si64(v_sse_q); + + sum64 = (sum64 >= 0) ? sum64 : -sum64; + + // Round + sum64 = ROUND_POWER_OF_TWO(sum64, 6); + sse64 = ROUND_POWER_OF_TWO(sse64, 12); + + // Store the SSE + *sse = (uint32_t)sse64; + // Compute the variance + return *sse - (uint32_t)((sum64 * sum64) / (w * h)); +} + +/***************************************************************************** + * n*16 Wide versions + *****************************************************************************/ + +static INLINE unsigned int masked_variancewxh_ssse3( + const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, + const uint8_t *m, int m_stride, int w, int h, unsigned int *sse) { + int ii, jj; + + const __m128i v_zero = _mm_setzero_si128(); + + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + + assert((w % 16) == 0); + + for (ii = 0; ii < h; ii++) { + for (jj = 0; jj < w; jj += 16) { + // Load inputs - 8 bits + const __m128i v_a_b = _mm_loadu_si128((const __m128i *)(a + jj)); + const __m128i v_b_b = _mm_loadu_si128((const __m128i *)(b + jj)); + const __m128i v_m_b = _mm_loadu_si128((const __m128i *)(m + jj)); + + // Unpack to 16 bits - still containing max 8 bits + const __m128i v_a0_w = _mm_unpacklo_epi8(v_a_b, v_zero); + const __m128i v_b0_w = _mm_unpacklo_epi8(v_b_b, v_zero); + const __m128i v_m0_w = _mm_unpacklo_epi8(v_m_b, v_zero); + const __m128i v_a1_w = _mm_unpackhi_epi8(v_a_b, v_zero); + const __m128i v_b1_w = _mm_unpackhi_epi8(v_b_b, v_zero); + const __m128i v_m1_w = _mm_unpackhi_epi8(v_m_b, v_zero); + + // Difference: [-255, 255] + const __m128i v_d0_w = _mm_sub_epi16(v_a0_w, v_b0_w); + const __m128i v_d1_w = _mm_sub_epi16(v_a1_w, v_b1_w); + + // Error - [-255, 255] * [0, 64] = [0xc040, 0x3fc0] => fits in 15 bits + const __m128i v_e0_w = _mm_mullo_epi16(v_d0_w, v_m0_w); + const __m128i v_e0_d = _mm_madd_epi16(v_d0_w, v_m0_w); + const __m128i v_e1_w = _mm_mullo_epi16(v_d1_w, v_m1_w); + const __m128i v_e1_d = _mm_madd_epi16(v_d1_w, v_m1_w); + + // Squared error - using madd it's max (15 bits * 15 bits) * 2 = 31 bits + const __m128i v_se0_d = _mm_madd_epi16(v_e0_w, v_e0_w); + const __m128i v_se1_d = _mm_madd_epi16(v_e1_w, v_e1_w); + + // Sum of v_se{0,1}_d - 31 bits + 31 bits = 32 bits + const __m128i v_se_d = _mm_add_epi32(v_se0_d, v_se1_d); + + // Unpack Squared error to 64 bits + const __m128i v_se_lo_q = _mm_unpacklo_epi32(v_se_d, v_zero); + const __m128i v_se_hi_q = _mm_unpackhi_epi32(v_se_d, v_zero); + + // Accumulate + v_sum_d = _mm_add_epi32(v_sum_d, v_e0_d); + v_sum_d = _mm_add_epi32(v_sum_d, v_e1_d); + v_sse_q = _mm_add_epi64(v_sse_q, v_se_lo_q); + v_sse_q = _mm_add_epi64(v_sse_q, v_se_hi_q); + } + + // Move on to next row + a += a_stride; + b += b_stride; + m += m_stride; + } + + return calc_masked_variance(v_sum_d, v_sse_q, sse, w, h); +} + +#define MASKED_VARWXH(W, H) \ + unsigned int aom_masked_variance##W##x##H##_ssse3( \ + const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, \ + const uint8_t *m, int m_stride, unsigned int *sse) { \ + return masked_variancewxh_ssse3(a, a_stride, b, b_stride, m, m_stride, W, \ + H, sse); \ + } + +MASKED_VARWXH(16, 8) +MASKED_VARWXH(16, 16) +MASKED_VARWXH(16, 32) +MASKED_VARWXH(32, 16) +MASKED_VARWXH(32, 32) +MASKED_VARWXH(32, 64) +MASKED_VARWXH(64, 32) +MASKED_VARWXH(64, 64) +#if CONFIG_EXT_PARTITION +MASKED_VARWXH(64, 128) +MASKED_VARWXH(128, 64) +MASKED_VARWXH(128, 128) +#endif // CONFIG_EXT_PARTITION + +/***************************************************************************** + * 8 Wide versions + *****************************************************************************/ + +static INLINE unsigned int masked_variance8xh_ssse3( + const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, + const uint8_t *m, int m_stride, int h, unsigned int *sse) { + int ii; + + const __m128i v_zero = _mm_setzero_si128(); + + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + + for (ii = 0; ii < h; ii++) { + // Load inputs - 8 bits + const __m128i v_a_b = _mm_loadl_epi64((const __m128i *)a); + const __m128i v_b_b = _mm_loadl_epi64((const __m128i *)b); + const __m128i v_m_b = _mm_loadl_epi64((const __m128i *)m); + + // Unpack to 16 bits - still containing max 8 bits + const __m128i v_a_w = _mm_unpacklo_epi8(v_a_b, v_zero); + const __m128i v_b_w = _mm_unpacklo_epi8(v_b_b, v_zero); + const __m128i v_m_w = _mm_unpacklo_epi8(v_m_b, v_zero); + + // Difference: [-255, 255] + const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w); + + // Error - [-255, 255] * [0, 64] = [0xc040, 0x3fc0] => fits in 15 bits + const __m128i v_e_w = _mm_mullo_epi16(v_d_w, v_m_w); + const __m128i v_e_d = _mm_madd_epi16(v_d_w, v_m_w); + + // Squared error - using madd it's max (15 bits * 15 bits) * 2 = 31 bits + const __m128i v_se_d = _mm_madd_epi16(v_e_w, v_e_w); + + // Unpack Squared error to 64 bits + const __m128i v_se_lo_q = _mm_unpacklo_epi32(v_se_d, v_zero); + const __m128i v_se_hi_q = _mm_unpackhi_epi32(v_se_d, v_zero); + + // Accumulate + v_sum_d = _mm_add_epi32(v_sum_d, v_e_d); + v_sse_q = _mm_add_epi64(v_sse_q, v_se_lo_q); + v_sse_q = _mm_add_epi64(v_sse_q, v_se_hi_q); + + // Move on to next row + a += a_stride; + b += b_stride; + m += m_stride; + } + + return calc_masked_variance(v_sum_d, v_sse_q, sse, 8, h); +} + +#define MASKED_VAR8XH(H) \ + unsigned int aom_masked_variance8x##H##_ssse3( \ + const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, \ + const uint8_t *m, int m_stride, unsigned int *sse) { \ + return masked_variance8xh_ssse3(a, a_stride, b, b_stride, m, m_stride, H, \ + sse); \ + } + +MASKED_VAR8XH(4) +MASKED_VAR8XH(8) +MASKED_VAR8XH(16) + +/***************************************************************************** + * 4 Wide versions + *****************************************************************************/ + +static INLINE unsigned int masked_variance4xh_ssse3( + const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, + const uint8_t *m, int m_stride, int h, unsigned int *sse) { + int ii; + + const __m128i v_zero = _mm_setzero_si128(); + + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + + assert((h % 2) == 0); + + for (ii = 0; ii < h / 2; ii++) { + // Load 2 input rows - 8 bits + const __m128i v_a0_b = _mm_cvtsi32_si128(*(const uint32_t *)a); + const __m128i v_b0_b = _mm_cvtsi32_si128(*(const uint32_t *)b); + const __m128i v_m0_b = _mm_cvtsi32_si128(*(const uint32_t *)m); + const __m128i v_a1_b = _mm_cvtsi32_si128(*(const uint32_t *)(a + a_stride)); + const __m128i v_b1_b = _mm_cvtsi32_si128(*(const uint32_t *)(b + b_stride)); + const __m128i v_m1_b = _mm_cvtsi32_si128(*(const uint32_t *)(m + m_stride)); + + // Interleave 2 rows into a single register + const __m128i v_a_b = _mm_unpacklo_epi32(v_a0_b, v_a1_b); + const __m128i v_b_b = _mm_unpacklo_epi32(v_b0_b, v_b1_b); + const __m128i v_m_b = _mm_unpacklo_epi32(v_m0_b, v_m1_b); + + // Unpack to 16 bits - still containing max 8 bits + const __m128i v_a_w = _mm_unpacklo_epi8(v_a_b, v_zero); + const __m128i v_b_w = _mm_unpacklo_epi8(v_b_b, v_zero); + const __m128i v_m_w = _mm_unpacklo_epi8(v_m_b, v_zero); + + // Difference: [-255, 255] + const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w); + + // Error - [-255, 255] * [0, 64] = [0xc040, 0x3fc0] => fits in 15 bits + const __m128i v_e_w = _mm_mullo_epi16(v_d_w, v_m_w); + const __m128i v_e_d = _mm_madd_epi16(v_d_w, v_m_w); + + // Squared error - using madd it's max (15 bits * 15 bits) * 2 = 31 bits + const __m128i v_se_d = _mm_madd_epi16(v_e_w, v_e_w); + + // Unpack Squared error to 64 bits + const __m128i v_se_lo_q = _mm_unpacklo_epi32(v_se_d, v_zero); + const __m128i v_se_hi_q = _mm_unpackhi_epi32(v_se_d, v_zero); + + // Accumulate + v_sum_d = _mm_add_epi32(v_sum_d, v_e_d); + v_sse_q = _mm_add_epi64(v_sse_q, v_se_lo_q); + v_sse_q = _mm_add_epi64(v_sse_q, v_se_hi_q); + + // Move on to next 2 row + a += a_stride * 2; + b += b_stride * 2; + m += m_stride * 2; + } + + return calc_masked_variance(v_sum_d, v_sse_q, sse, 4, h); +} + +#define MASKED_VAR4XH(H) \ + unsigned int aom_masked_variance4x##H##_ssse3( \ + const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, \ + const uint8_t *m, int m_stride, unsigned int *sse) { \ + return masked_variance4xh_ssse3(a, a_stride, b, b_stride, m, m_stride, H, \ + sse); \ + } + +MASKED_VAR4XH(4) +MASKED_VAR4XH(8) + +#if CONFIG_HIGHBITDEPTH + +// Main calculation for n*8 wide blocks +static INLINE void highbd_masked_variance64_ssse3( + const uint16_t *a, int a_stride, const uint16_t *b, int b_stride, + const uint8_t *m, int m_stride, int w, int h, int64_t *sum, uint64_t *sse) { + int ii, jj; + + const __m128i v_zero = _mm_setzero_si128(); + + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + + assert((w % 8) == 0); + + for (ii = 0; ii < h; ii++) { + for (jj = 0; jj < w; jj += 8) { + // Load inputs - 8 bits + const __m128i v_a_w = _mm_loadu_si128((const __m128i *)(a + jj)); + const __m128i v_b_w = _mm_loadu_si128((const __m128i *)(b + jj)); + const __m128i v_m_b = _mm_loadl_epi64((const __m128i *)(m + jj)); + + // Unpack m to 16 bits - still containing max 8 bits + const __m128i v_m_w = _mm_unpacklo_epi8(v_m_b, v_zero); + + // Difference: [-4095, 4095] + const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w); + + // Error - [-4095, 4095] * [0, 64] => sum of 2 of these fits in 19 bits + const __m128i v_e_d = _mm_madd_epi16(v_d_w, v_m_w); + + // Squared error - max (18 bits * 18 bits) = 36 bits (no sign bit) + const __m128i v_absd_w = _mm_abs_epi16(v_d_w); + const __m128i v_dlo_d = _mm_unpacklo_epi16(v_absd_w, v_zero); + const __m128i v_mlo_d = _mm_unpacklo_epi16(v_m_w, v_zero); + const __m128i v_elo_d = _mm_madd_epi16(v_dlo_d, v_mlo_d); + const __m128i v_dhi_d = _mm_unpackhi_epi16(v_absd_w, v_zero); + const __m128i v_mhi_d = _mm_unpackhi_epi16(v_m_w, v_zero); + const __m128i v_ehi_d = _mm_madd_epi16(v_dhi_d, v_mhi_d); + // Square and sum the errors -> 36bits * 4 = 38bits + __m128i v_se0_q, v_se1_q, v_se2_q, v_se3_q, v_se_q, v_elo1_d, v_ehi3_d; + v_se0_q = _mm_mul_epu32(v_elo_d, v_elo_d); + v_elo1_d = _mm_srli_si128(v_elo_d, 4); + v_se1_q = _mm_mul_epu32(v_elo1_d, v_elo1_d); + v_se0_q = _mm_add_epi64(v_se0_q, v_se1_q); + v_se2_q = _mm_mul_epu32(v_ehi_d, v_ehi_d); + v_ehi3_d = _mm_srli_si128(v_ehi_d, 4); + v_se3_q = _mm_mul_epu32(v_ehi3_d, v_ehi3_d); + v_se1_q = _mm_add_epi64(v_se2_q, v_se3_q); + v_se_q = _mm_add_epi64(v_se0_q, v_se1_q); + + // Accumulate + v_sum_d = _mm_add_epi32(v_sum_d, v_e_d); + v_sse_q = _mm_add_epi64(v_sse_q, v_se_q); + } + + // Move on to next row + a += a_stride; + b += b_stride; + m += m_stride; + } + + // Horizontal sum + *sum = hsum_epi32_si64(v_sum_d); + *sse = hsum_epi64_si64(v_sse_q); + + // Round + *sum = (*sum >= 0) ? *sum : -*sum; + *sum = ROUND_POWER_OF_TWO(*sum, 6); + *sse = ROUND_POWER_OF_TWO(*sse, 12); +} + +// Main calculation for 4 wide blocks +static INLINE void highbd_masked_variance64_4wide_ssse3( + const uint16_t *a, int a_stride, const uint16_t *b, int b_stride, + const uint8_t *m, int m_stride, int h, int64_t *sum, uint64_t *sse) { + int ii; + + const __m128i v_zero = _mm_setzero_si128(); + + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + + assert((h % 2) == 0); + + for (ii = 0; ii < h / 2; ii++) { + // Load 2 input rows - 8 bits + const __m128i v_a0_w = _mm_loadl_epi64((const __m128i *)a); + const __m128i v_b0_w = _mm_loadl_epi64((const __m128i *)b); + const __m128i v_m0_b = _mm_cvtsi32_si128(*(const uint32_t *)m); + const __m128i v_a1_w = _mm_loadl_epi64((const __m128i *)(a + a_stride)); + const __m128i v_b1_w = _mm_loadl_epi64((const __m128i *)(b + b_stride)); + const __m128i v_m1_b = _mm_cvtsi32_si128(*(const uint32_t *)(m + m_stride)); + + // Interleave 2 rows into a single register + const __m128i v_a_w = _mm_unpacklo_epi64(v_a0_w, v_a1_w); + const __m128i v_b_w = _mm_unpacklo_epi64(v_b0_w, v_b1_w); + const __m128i v_m_b = _mm_unpacklo_epi32(v_m0_b, v_m1_b); + + // Unpack to 16 bits - still containing max 8 bits + const __m128i v_m_w = _mm_unpacklo_epi8(v_m_b, v_zero); + + // Difference: [-4095, 4095] + const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w); + + // Error - [-4095, 4095] * [0, 64] => fits in 19 bits (incld sign bit) + const __m128i v_e_d = _mm_madd_epi16(v_d_w, v_m_w); + + // Squared error - max (18 bits * 18 bits) = 36 bits (no sign bit) + const __m128i v_absd_w = _mm_abs_epi16(v_d_w); + const __m128i v_dlo_d = _mm_unpacklo_epi16(v_absd_w, v_zero); + const __m128i v_mlo_d = _mm_unpacklo_epi16(v_m_w, v_zero); + const __m128i v_elo_d = _mm_madd_epi16(v_dlo_d, v_mlo_d); + const __m128i v_dhi_d = _mm_unpackhi_epi16(v_absd_w, v_zero); + const __m128i v_mhi_d = _mm_unpackhi_epi16(v_m_w, v_zero); + const __m128i v_ehi_d = _mm_madd_epi16(v_dhi_d, v_mhi_d); + // Square and sum the errors -> 36bits * 4 = 38bits + __m128i v_se0_q, v_se1_q, v_se2_q, v_se3_q, v_se_q, v_elo1_d, v_ehi3_d; + v_se0_q = _mm_mul_epu32(v_elo_d, v_elo_d); + v_elo1_d = _mm_srli_si128(v_elo_d, 4); + v_se1_q = _mm_mul_epu32(v_elo1_d, v_elo1_d); + v_se0_q = _mm_add_epi64(v_se0_q, v_se1_q); + v_se2_q = _mm_mul_epu32(v_ehi_d, v_ehi_d); + v_ehi3_d = _mm_srli_si128(v_ehi_d, 4); + v_se3_q = _mm_mul_epu32(v_ehi3_d, v_ehi3_d); + v_se1_q = _mm_add_epi64(v_se2_q, v_se3_q); + v_se_q = _mm_add_epi64(v_se0_q, v_se1_q); + + // Accumulate + v_sum_d = _mm_add_epi32(v_sum_d, v_e_d); + v_sse_q = _mm_add_epi64(v_sse_q, v_se_q); + + // Move on to next row + a += a_stride * 2; + b += b_stride * 2; + m += m_stride * 2; + } + + // Horizontal sum + *sum = hsum_epi32_si32(v_sum_d); + *sse = hsum_epi64_si64(v_sse_q); + + // Round + *sum = (*sum >= 0) ? *sum : -*sum; + *sum = ROUND_POWER_OF_TWO(*sum, 6); + *sse = ROUND_POWER_OF_TWO(*sse, 12); +} + +static INLINE unsigned int highbd_masked_variancewxh_ssse3( + const uint16_t *a, int a_stride, const uint16_t *b, int b_stride, + const uint8_t *m, int m_stride, int w, int h, unsigned int *sse) { + uint64_t sse64; + int64_t sum64; + + if (w == 4) + highbd_masked_variance64_4wide_ssse3(a, a_stride, b, b_stride, m, m_stride, + h, &sum64, &sse64); + else + highbd_masked_variance64_ssse3(a, a_stride, b, b_stride, m, m_stride, w, h, + &sum64, &sse64); + + // Store the SSE + *sse = (uint32_t)sse64; + // Compute and return variance + return *sse - (uint32_t)((sum64 * sum64) / (w * h)); +} + +static INLINE unsigned int highbd_10_masked_variancewxh_ssse3( + const uint16_t *a, int a_stride, const uint16_t *b, int b_stride, + const uint8_t *m, int m_stride, int w, int h, unsigned int *sse) { + uint64_t sse64; + int64_t sum64; + + if (w == 4) + highbd_masked_variance64_4wide_ssse3(a, a_stride, b, b_stride, m, m_stride, + h, &sum64, &sse64); + else + highbd_masked_variance64_ssse3(a, a_stride, b, b_stride, m, m_stride, w, h, + &sum64, &sse64); + + // Normalise + sum64 = ROUND_POWER_OF_TWO(sum64, 2); + sse64 = ROUND_POWER_OF_TWO(sse64, 4); + + // Store the SSE + *sse = (uint32_t)sse64; + // Compute and return variance + return *sse - (uint32_t)((sum64 * sum64) / (w * h)); +} + +static INLINE unsigned int highbd_12_masked_variancewxh_ssse3( + const uint16_t *a, int a_stride, const uint16_t *b, int b_stride, + const uint8_t *m, int m_stride, int w, int h, unsigned int *sse) { + uint64_t sse64; + int64_t sum64; + + if (w == 4) + highbd_masked_variance64_4wide_ssse3(a, a_stride, b, b_stride, m, m_stride, + h, &sum64, &sse64); + else + highbd_masked_variance64_ssse3(a, a_stride, b, b_stride, m, m_stride, w, h, + &sum64, &sse64); + + sum64 = ROUND_POWER_OF_TWO(sum64, 4); + sse64 = ROUND_POWER_OF_TWO(sse64, 8); + + // Store the SSE + *sse = (uint32_t)sse64; + // Compute and return variance + return *sse - (uint32_t)((sum64 * sum64) / (w * h)); +} + +#define HIGHBD_MASKED_VARWXH(W, H) \ + unsigned int aom_highbd_masked_variance##W##x##H##_ssse3( \ + const uint8_t *a8, int a_stride, const uint8_t *b8, int b_stride, \ + const uint8_t *m, int m_stride, unsigned int *sse) { \ + uint16_t *a = CONVERT_TO_SHORTPTR(a8); \ + uint16_t *b = CONVERT_TO_SHORTPTR(b8); \ + return highbd_masked_variancewxh_ssse3(a, a_stride, b, b_stride, m, \ + m_stride, W, H, sse); \ + } \ + \ + unsigned int aom_highbd_10_masked_variance##W##x##H##_ssse3( \ + const uint8_t *a8, int a_stride, const uint8_t *b8, int b_stride, \ + const uint8_t *m, int m_stride, unsigned int *sse) { \ + uint16_t *a = CONVERT_TO_SHORTPTR(a8); \ + uint16_t *b = CONVERT_TO_SHORTPTR(b8); \ + return highbd_10_masked_variancewxh_ssse3(a, a_stride, b, b_stride, m, \ + m_stride, W, H, sse); \ + } \ + \ + unsigned int aom_highbd_12_masked_variance##W##x##H##_ssse3( \ + const uint8_t *a8, int a_stride, const uint8_t *b8, int b_stride, \ + const uint8_t *m, int m_stride, unsigned int *sse) { \ + uint16_t *a = CONVERT_TO_SHORTPTR(a8); \ + uint16_t *b = CONVERT_TO_SHORTPTR(b8); \ + return highbd_12_masked_variancewxh_ssse3(a, a_stride, b, b_stride, m, \ + m_stride, W, H, sse); \ + } + +HIGHBD_MASKED_VARWXH(4, 4) +HIGHBD_MASKED_VARWXH(4, 8) +HIGHBD_MASKED_VARWXH(8, 4) +HIGHBD_MASKED_VARWXH(8, 8) +HIGHBD_MASKED_VARWXH(8, 16) +HIGHBD_MASKED_VARWXH(16, 8) +HIGHBD_MASKED_VARWXH(16, 16) +HIGHBD_MASKED_VARWXH(16, 32) +HIGHBD_MASKED_VARWXH(32, 16) +HIGHBD_MASKED_VARWXH(32, 32) +HIGHBD_MASKED_VARWXH(32, 64) +HIGHBD_MASKED_VARWXH(64, 32) +HIGHBD_MASKED_VARWXH(64, 64) +#if CONFIG_EXT_PARTITION +HIGHBD_MASKED_VARWXH(64, 128) +HIGHBD_MASKED_VARWXH(128, 64) +HIGHBD_MASKED_VARWXH(128, 128) +#endif // CONFIG_EXT_PARTITION + +#endif + +////////////////////////////////////////////////////////////////////////////// +// Sub pixel versions +////////////////////////////////////////////////////////////////////////////// + +typedef __m128i (*filter_fn_t)(__m128i v_a_b, __m128i v_b_b, + __m128i v_filter_b); + +static INLINE __m128i apply_filter_avg(const __m128i v_a_b, const __m128i v_b_b, + const __m128i v_filter_b) { + (void)v_filter_b; + return _mm_avg_epu8(v_a_b, v_b_b); +} + +static INLINE __m128i apply_filter(const __m128i v_a_b, const __m128i v_b_b, + const __m128i v_filter_b) { + const __m128i v_rounding_w = _mm_set1_epi16(1 << (FILTER_BITS - 1)); + __m128i v_input_lo_b = _mm_unpacklo_epi8(v_a_b, v_b_b); + __m128i v_input_hi_b = _mm_unpackhi_epi8(v_a_b, v_b_b); + __m128i v_temp0_w = _mm_maddubs_epi16(v_input_lo_b, v_filter_b); + __m128i v_temp1_w = _mm_maddubs_epi16(v_input_hi_b, v_filter_b); + __m128i v_res_lo_w = + _mm_srai_epi16(_mm_add_epi16(v_temp0_w, v_rounding_w), FILTER_BITS); + __m128i v_res_hi_w = + _mm_srai_epi16(_mm_add_epi16(v_temp1_w, v_rounding_w), FILTER_BITS); + return _mm_packus_epi16(v_res_lo_w, v_res_hi_w); +} + +// Apply the filter to the contents of the lower half of a and b +static INLINE void apply_filter_lo(const __m128i v_a_lo_b, + const __m128i v_b_lo_b, + const __m128i v_filter_b, __m128i *v_res_w) { + const __m128i v_rounding_w = _mm_set1_epi16(1 << (FILTER_BITS - 1)); + __m128i v_input_b = _mm_unpacklo_epi8(v_a_lo_b, v_b_lo_b); + __m128i v_temp0_w = _mm_maddubs_epi16(v_input_b, v_filter_b); + *v_res_w = + _mm_srai_epi16(_mm_add_epi16(v_temp0_w, v_rounding_w), FILTER_BITS); +} + +static void sum_and_sse(const __m128i v_a_b, const __m128i v_b_b, + const __m128i v_m_b, __m128i *v_sum_d, + __m128i *v_sse_q) { + const __m128i v_zero = _mm_setzero_si128(); + // Unpack to 16 bits - still containing max 8 bits + const __m128i v_a0_w = _mm_unpacklo_epi8(v_a_b, v_zero); + const __m128i v_b0_w = _mm_unpacklo_epi8(v_b_b, v_zero); + const __m128i v_m0_w = _mm_unpacklo_epi8(v_m_b, v_zero); + const __m128i v_a1_w = _mm_unpackhi_epi8(v_a_b, v_zero); + const __m128i v_b1_w = _mm_unpackhi_epi8(v_b_b, v_zero); + const __m128i v_m1_w = _mm_unpackhi_epi8(v_m_b, v_zero); + + // Difference: [-255, 255] + const __m128i v_d0_w = _mm_sub_epi16(v_a0_w, v_b0_w); + const __m128i v_d1_w = _mm_sub_epi16(v_a1_w, v_b1_w); + + // Error - [-255, 255] * [0, 64] = [0xc040, 0x3fc0] => fits in 15 bits + const __m128i v_e0_w = _mm_mullo_epi16(v_d0_w, v_m0_w); + const __m128i v_e0_d = _mm_madd_epi16(v_d0_w, v_m0_w); + const __m128i v_e1_w = _mm_mullo_epi16(v_d1_w, v_m1_w); + const __m128i v_e1_d = _mm_madd_epi16(v_d1_w, v_m1_w); + + // Squared error - using madd it's max (15 bits * 15 bits) * 2 = 31 bits + const __m128i v_se0_d = _mm_madd_epi16(v_e0_w, v_e0_w); + const __m128i v_se1_d = _mm_madd_epi16(v_e1_w, v_e1_w); + + // Sum of v_se{0,1}_d - 31 bits + 31 bits = 32 bits + const __m128i v_se_d = _mm_add_epi32(v_se0_d, v_se1_d); + + // Unpack Squared error to 64 bits + const __m128i v_se_lo_q = _mm_unpacklo_epi32(v_se_d, v_zero); + const __m128i v_se_hi_q = _mm_unpackhi_epi32(v_se_d, v_zero); + + // Accumulate + *v_sum_d = _mm_add_epi32(*v_sum_d, v_e0_d); + *v_sum_d = _mm_add_epi32(*v_sum_d, v_e1_d); + *v_sse_q = _mm_add_epi64(*v_sse_q, v_se_lo_q); + *v_sse_q = _mm_add_epi64(*v_sse_q, v_se_hi_q); +} + +// Functions for width (W) >= 16 +unsigned int aom_masked_subpel_varWxH_xzero(const uint8_t *src, int src_stride, + int yoffset, const uint8_t *dst, + int dst_stride, const uint8_t *msk, + int msk_stride, unsigned int *sse, + int w, int h, + filter_fn_t filter_fn) { + int i, j; + __m128i v_src0_b, v_src1_b, v_res_b, v_dst_b, v_msk_b; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + const __m128i v_filter_b = _mm_set1_epi16( + (bilinear_filters_2t[yoffset][1] << 8) + bilinear_filters_2t[yoffset][0]); + assert(yoffset < BIL_SUBPEL_SHIFTS); + for (j = 0; j < w; j += 16) { + // Load the first row ready + v_src0_b = _mm_loadu_si128((const __m128i *)(src + j)); + // Process 2 rows at a time + for (i = 0; i < h; i += 2) { + // Load the next row apply the filter + v_src1_b = _mm_loadu_si128((const __m128i *)(src + j + src_stride)); + v_res_b = filter_fn(v_src0_b, v_src1_b, v_filter_b); + // Load the dst and msk for the variance calculation + v_dst_b = _mm_loadu_si128((const __m128i *)(dst + j)); + v_msk_b = _mm_loadu_si128((const __m128i *)(msk + j)); + sum_and_sse(v_res_b, v_dst_b, v_msk_b, &v_sum_d, &v_sse_q); + + // Load the next row apply the filter + v_src0_b = _mm_loadu_si128((const __m128i *)(src + j + src_stride * 2)); + v_res_b = filter_fn(v_src1_b, v_src0_b, v_filter_b); + // Load the dst and msk for the variance calculation + v_dst_b = _mm_loadu_si128((const __m128i *)(dst + j + dst_stride)); + v_msk_b = _mm_loadu_si128((const __m128i *)(msk + j + msk_stride)); + sum_and_sse(v_res_b, v_dst_b, v_msk_b, &v_sum_d, &v_sse_q); + // Move onto the next block of rows + src += src_stride * 2; + dst += dst_stride * 2; + msk += msk_stride * 2; + } + // Reset to the top of the block + src -= src_stride * h; + dst -= dst_stride * h; + msk -= msk_stride * h; + } + return calc_masked_variance(v_sum_d, v_sse_q, sse, w, h); +} +unsigned int aom_masked_subpel_varWxH_yzero(const uint8_t *src, int src_stride, + int xoffset, const uint8_t *dst, + int dst_stride, const uint8_t *msk, + int msk_stride, unsigned int *sse, + int w, int h, + filter_fn_t filter_fn) { + int i, j; + __m128i v_src0_b, v_src1_b, v_res_b, v_dst_b, v_msk_b; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + const __m128i v_filter_b = _mm_set1_epi16( + (bilinear_filters_2t[xoffset][1] << 8) + bilinear_filters_2t[xoffset][0]); + assert(xoffset < BIL_SUBPEL_SHIFTS); + for (i = 0; i < h; i++) { + for (j = 0; j < w; j += 16) { + // Load this row and one below & apply the filter to them + v_src0_b = _mm_loadu_si128((const __m128i *)(src + j)); + v_src1_b = _mm_loadu_si128((const __m128i *)(src + j + 1)); + v_res_b = filter_fn(v_src0_b, v_src1_b, v_filter_b); + + // Load the dst and msk for the variance calculation + v_dst_b = _mm_loadu_si128((const __m128i *)(dst + j)); + v_msk_b = _mm_loadu_si128((const __m128i *)(msk + j)); + sum_and_sse(v_res_b, v_dst_b, v_msk_b, &v_sum_d, &v_sse_q); + } + src += src_stride; + dst += dst_stride; + msk += msk_stride; + } + return calc_masked_variance(v_sum_d, v_sse_q, sse, w, h); +} +unsigned int aom_masked_subpel_varWxH_xnonzero_ynonzero( + const uint8_t *src, int src_stride, int xoffset, int yoffset, + const uint8_t *dst, int dst_stride, const uint8_t *msk, int msk_stride, + unsigned int *sse, int w, int h, filter_fn_t xfilter_fn, + filter_fn_t yfilter_fn) { + int i, j; + __m128i v_src0_b, v_src1_b, v_src2_b, v_src3_b; + __m128i v_filtered0_b, v_filtered1_b, v_res_b, v_dst_b, v_msk_b; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + const __m128i v_filterx_b = _mm_set1_epi16( + (bilinear_filters_2t[xoffset][1] << 8) + bilinear_filters_2t[xoffset][0]); + const __m128i v_filtery_b = _mm_set1_epi16( + (bilinear_filters_2t[yoffset][1] << 8) + bilinear_filters_2t[yoffset][0]); + assert(yoffset < BIL_SUBPEL_SHIFTS); + assert(xoffset < BIL_SUBPEL_SHIFTS); + for (j = 0; j < w; j += 16) { + // Load the first row ready + v_src0_b = _mm_loadu_si128((const __m128i *)(src + j)); + v_src1_b = _mm_loadu_si128((const __m128i *)(src + j + 1)); + v_filtered0_b = xfilter_fn(v_src0_b, v_src1_b, v_filterx_b); + // Process 2 rows at a time + for (i = 0; i < h; i += 2) { + // Load the next row & apply the filter + v_src2_b = _mm_loadu_si128((const __m128i *)(src + src_stride + j)); + v_src3_b = _mm_loadu_si128((const __m128i *)(src + src_stride + j + 1)); + v_filtered1_b = xfilter_fn(v_src2_b, v_src3_b, v_filterx_b); + // Load the dst and msk for the variance calculation + v_dst_b = _mm_loadu_si128((const __m128i *)(dst + j)); + v_msk_b = _mm_loadu_si128((const __m128i *)(msk + j)); + // Complete the calculation for this row and add it to the running total + v_res_b = yfilter_fn(v_filtered0_b, v_filtered1_b, v_filtery_b); + sum_and_sse(v_res_b, v_dst_b, v_msk_b, &v_sum_d, &v_sse_q); + + // Load the next row & apply the filter + v_src0_b = _mm_loadu_si128((const __m128i *)(src + src_stride * 2 + j)); + v_src1_b = + _mm_loadu_si128((const __m128i *)(src + src_stride * 2 + j + 1)); + v_filtered0_b = xfilter_fn(v_src0_b, v_src1_b, v_filterx_b); + // Load the dst and msk for the variance calculation + v_dst_b = _mm_loadu_si128((const __m128i *)(dst + dst_stride + j)); + v_msk_b = _mm_loadu_si128((const __m128i *)(msk + msk_stride + j)); + // Complete the calculation for this row and add it to the running total + v_res_b = yfilter_fn(v_filtered1_b, v_filtered0_b, v_filtery_b); + sum_and_sse(v_res_b, v_dst_b, v_msk_b, &v_sum_d, &v_sse_q); + // Move onto the next block of rows + src += src_stride * 2; + dst += dst_stride * 2; + msk += msk_stride * 2; + } + // Reset to the top of the block + src -= src_stride * h; + dst -= dst_stride * h; + msk -= msk_stride * h; + } + return calc_masked_variance(v_sum_d, v_sse_q, sse, w, h); +} + +// Note order in which rows loaded xmm[127:96] = row 1, xmm[95:64] = row 2, +// xmm[63:32] = row 3, xmm[31:0] = row 4 +unsigned int aom_masked_subpel_var4xH_xzero(const uint8_t *src, int src_stride, + int yoffset, const uint8_t *dst, + int dst_stride, const uint8_t *msk, + int msk_stride, unsigned int *sse, + int h) { + int i; + __m128i v_src0_b, v_src1_b, v_src2_b, v_src3_b, v_filtered1_w, v_filtered2_w; + __m128i v_dst0_b, v_dst1_b, v_dst2_b, v_dst3_b; + __m128i v_msk0_b, v_msk1_b, v_msk2_b, v_msk3_b, v_res_b; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + __m128i v_filter_b = _mm_set1_epi16((bilinear_filters_2t[yoffset][1] << 8) + + bilinear_filters_2t[yoffset][0]); + assert(yoffset < BIL_SUBPEL_SHIFTS); + // Load the first row of src data ready + v_src0_b = _mm_loadl_epi64((const __m128i *)src); + for (i = 0; i < h; i += 4) { + // Load the rest of the source data for these rows + v_src1_b = _mm_loadl_epi64((const __m128i *)(src + src_stride * 1)); + v_src1_b = _mm_unpacklo_epi32(v_src1_b, v_src0_b); + v_src2_b = _mm_loadl_epi64((const __m128i *)(src + src_stride * 2)); + v_src3_b = _mm_loadl_epi64((const __m128i *)(src + src_stride * 3)); + v_src3_b = _mm_unpacklo_epi32(v_src3_b, v_src2_b); + v_src0_b = _mm_loadl_epi64((const __m128i *)(src + src_stride * 4)); + // Load the dst data + v_dst0_b = _mm_cvtsi32_si128(*(const uint32_t *)(dst + dst_stride * 0)); + v_dst1_b = _mm_cvtsi32_si128(*(const uint32_t *)(dst + dst_stride * 1)); + v_dst0_b = _mm_unpacklo_epi32(v_dst1_b, v_dst0_b); + v_dst2_b = _mm_cvtsi32_si128(*(const uint32_t *)(dst + dst_stride * 2)); + v_dst3_b = _mm_cvtsi32_si128(*(const uint32_t *)(dst + dst_stride * 3)); + v_dst2_b = _mm_unpacklo_epi32(v_dst3_b, v_dst2_b); + v_dst0_b = _mm_unpacklo_epi64(v_dst2_b, v_dst0_b); + // Load the mask data + v_msk0_b = _mm_cvtsi32_si128(*(const uint32_t *)(msk + msk_stride * 0)); + v_msk1_b = _mm_cvtsi32_si128(*(const uint32_t *)(msk + msk_stride * 1)); + v_msk0_b = _mm_unpacklo_epi32(v_msk1_b, v_msk0_b); + v_msk2_b = _mm_cvtsi32_si128(*(const uint32_t *)(msk + msk_stride * 2)); + v_msk3_b = _mm_cvtsi32_si128(*(const uint32_t *)(msk + msk_stride * 3)); + v_msk2_b = _mm_unpacklo_epi32(v_msk3_b, v_msk2_b); + v_msk0_b = _mm_unpacklo_epi64(v_msk2_b, v_msk0_b); + // Apply the y filter + if (yoffset == HALF_PIXEL_OFFSET) { + v_src1_b = _mm_unpacklo_epi64(v_src3_b, v_src1_b); + v_src2_b = + _mm_or_si128(_mm_slli_si128(v_src1_b, 4), + _mm_and_si128(v_src0_b, _mm_setr_epi32(-1, 0, 0, 0))); + v_res_b = _mm_avg_epu8(v_src1_b, v_src2_b); + } else { + v_src2_b = + _mm_or_si128(_mm_slli_si128(v_src1_b, 4), + _mm_and_si128(v_src2_b, _mm_setr_epi32(-1, 0, 0, 0))); + apply_filter_lo(v_src1_b, v_src2_b, v_filter_b, &v_filtered1_w); + v_src2_b = + _mm_or_si128(_mm_slli_si128(v_src3_b, 4), + _mm_and_si128(v_src0_b, _mm_setr_epi32(-1, 0, 0, 0))); + apply_filter_lo(v_src3_b, v_src2_b, v_filter_b, &v_filtered2_w); + v_res_b = _mm_packus_epi16(v_filtered2_w, v_filtered1_w); + } + // Compute the sum and SSE + sum_and_sse(v_res_b, v_dst0_b, v_msk0_b, &v_sum_d, &v_sse_q); + // Move onto the next set of rows + src += src_stride * 4; + dst += dst_stride * 4; + msk += msk_stride * 4; + } + return calc_masked_variance(v_sum_d, v_sse_q, sse, 4, h); +} + +// Note order in which rows loaded xmm[127:64] = row 1, xmm[63:0] = row 2 +unsigned int aom_masked_subpel_var8xH_xzero(const uint8_t *src, int src_stride, + int yoffset, const uint8_t *dst, + int dst_stride, const uint8_t *msk, + int msk_stride, unsigned int *sse, + int h) { + int i; + __m128i v_src0_b, v_src1_b, v_filtered0_w, v_filtered1_w, v_res_b; + __m128i v_dst_b = _mm_setzero_si128(); + __m128i v_msk_b = _mm_setzero_si128(); + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + __m128i v_filter_b = _mm_set1_epi16((bilinear_filters_2t[yoffset][1] << 8) + + bilinear_filters_2t[yoffset][0]); + assert(yoffset < BIL_SUBPEL_SHIFTS); + // Load the first row of src data ready + v_src0_b = _mm_loadl_epi64((const __m128i *)src); + for (i = 0; i < h; i += 2) { + if (yoffset == HALF_PIXEL_OFFSET) { + // Load the rest of the source data for these rows + v_src1_b = _mm_or_si128( + _mm_slli_si128(v_src0_b, 8), + _mm_loadl_epi64((const __m128i *)(src + src_stride * 1))); + v_src0_b = _mm_or_si128( + _mm_slli_si128(v_src1_b, 8), + _mm_loadl_epi64((const __m128i *)(src + src_stride * 2))); + // Apply the y filter + v_res_b = _mm_avg_epu8(v_src1_b, v_src0_b); + } else { + // Load the data and apply the y filter + v_src1_b = _mm_loadl_epi64((const __m128i *)(src + src_stride * 1)); + apply_filter_lo(v_src0_b, v_src1_b, v_filter_b, &v_filtered0_w); + v_src0_b = _mm_loadl_epi64((const __m128i *)(src + src_stride * 2)); + apply_filter_lo(v_src1_b, v_src0_b, v_filter_b, &v_filtered1_w); + v_res_b = _mm_packus_epi16(v_filtered1_w, v_filtered0_w); + } + // Load the dst data + v_dst_b = _mm_unpacklo_epi64( + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 1)), + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 0))); + // Load the mask data + v_msk_b = _mm_unpacklo_epi64( + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 1)), + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 0))); + // Compute the sum and SSE + sum_and_sse(v_res_b, v_dst_b, v_msk_b, &v_sum_d, &v_sse_q); + // Move onto the next set of rows + src += src_stride * 2; + dst += dst_stride * 2; + msk += msk_stride * 2; + } + return calc_masked_variance(v_sum_d, v_sse_q, sse, 8, h); +} + +// Note order in which rows loaded xmm[127:96] = row 1, xmm[95:64] = row 2, +// xmm[63:32] = row 3, xmm[31:0] = row 4 +unsigned int aom_masked_subpel_var4xH_yzero(const uint8_t *src, int src_stride, + int xoffset, const uint8_t *dst, + int dst_stride, const uint8_t *msk, + int msk_stride, unsigned int *sse, + int h) { + int i; + __m128i v_src0_b, v_src1_b, v_src2_b, v_src3_b, v_filtered0_w, v_filtered2_w; + __m128i v_src0_shift_b, v_src1_shift_b, v_src2_shift_b, v_src3_shift_b; + __m128i v_dst0_b, v_dst1_b, v_dst2_b, v_dst3_b; + __m128i v_msk0_b, v_msk1_b, v_msk2_b, v_msk3_b, v_res_b; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + __m128i v_filter_b = _mm_set1_epi16((bilinear_filters_2t[xoffset][1] << 8) + + bilinear_filters_2t[xoffset][0]); + assert(xoffset < BIL_SUBPEL_SHIFTS); + for (i = 0; i < h; i += 4) { + // Load the src data + v_src0_b = _mm_loadl_epi64((const __m128i *)src); + v_src0_shift_b = _mm_srli_si128(v_src0_b, 1); + v_src1_b = _mm_loadl_epi64((const __m128i *)(src + src_stride * 1)); + v_src0_b = _mm_unpacklo_epi32(v_src1_b, v_src0_b); + v_src1_shift_b = _mm_srli_si128(v_src1_b, 1); + v_src2_b = _mm_loadl_epi64((const __m128i *)(src + src_stride * 2)); + v_src0_shift_b = _mm_unpacklo_epi32(v_src1_shift_b, v_src0_shift_b); + v_src2_shift_b = _mm_srli_si128(v_src2_b, 1); + v_src3_b = _mm_loadl_epi64((const __m128i *)(src + src_stride * 3)); + v_src2_b = _mm_unpacklo_epi32(v_src3_b, v_src2_b); + v_src3_shift_b = _mm_srli_si128(v_src3_b, 1); + v_src2_shift_b = _mm_unpacklo_epi32(v_src3_shift_b, v_src2_shift_b); + // Load the dst data + v_dst0_b = _mm_cvtsi32_si128(*(const uint32_t *)(dst + dst_stride * 0)); + v_dst1_b = _mm_cvtsi32_si128(*(const uint32_t *)(dst + dst_stride * 1)); + v_dst0_b = _mm_unpacklo_epi32(v_dst1_b, v_dst0_b); + v_dst2_b = _mm_cvtsi32_si128(*(const uint32_t *)(dst + dst_stride * 2)); + v_dst3_b = _mm_cvtsi32_si128(*(const uint32_t *)(dst + dst_stride * 3)); + v_dst2_b = _mm_unpacklo_epi32(v_dst3_b, v_dst2_b); + v_dst0_b = _mm_unpacklo_epi64(v_dst2_b, v_dst0_b); + // Load the mask data + v_msk0_b = _mm_cvtsi32_si128(*(const uint32_t *)(msk + msk_stride * 0)); + v_msk1_b = _mm_cvtsi32_si128(*(const uint32_t *)(msk + msk_stride * 1)); + v_msk0_b = _mm_unpacklo_epi32(v_msk1_b, v_msk0_b); + v_msk2_b = _mm_cvtsi32_si128(*(const uint32_t *)(msk + msk_stride * 2)); + v_msk3_b = _mm_cvtsi32_si128(*(const uint32_t *)(msk + msk_stride * 3)); + v_msk2_b = _mm_unpacklo_epi32(v_msk3_b, v_msk2_b); + v_msk0_b = _mm_unpacklo_epi64(v_msk2_b, v_msk0_b); + // Apply the x filter + if (xoffset == HALF_PIXEL_OFFSET) { + v_src0_b = _mm_unpacklo_epi64(v_src2_b, v_src0_b); + v_src0_shift_b = _mm_unpacklo_epi64(v_src2_shift_b, v_src0_shift_b); + v_res_b = _mm_avg_epu8(v_src0_b, v_src0_shift_b); + } else { + apply_filter_lo(v_src0_b, v_src0_shift_b, v_filter_b, &v_filtered0_w); + apply_filter_lo(v_src2_b, v_src2_shift_b, v_filter_b, &v_filtered2_w); + v_res_b = _mm_packus_epi16(v_filtered2_w, v_filtered0_w); + } + // Compute the sum and SSE + sum_and_sse(v_res_b, v_dst0_b, v_msk0_b, &v_sum_d, &v_sse_q); + // Move onto the next set of rows + src += src_stride * 4; + dst += dst_stride * 4; + msk += msk_stride * 4; + } + return calc_masked_variance(v_sum_d, v_sse_q, sse, 4, h); +} + +unsigned int aom_masked_subpel_var8xH_yzero(const uint8_t *src, int src_stride, + int xoffset, const uint8_t *dst, + int dst_stride, const uint8_t *msk, + int msk_stride, unsigned int *sse, + int h) { + int i; + __m128i v_src0_b, v_src1_b, v_filtered0_w, v_filtered1_w; + __m128i v_src0_shift_b, v_src1_shift_b, v_res_b, v_dst_b, v_msk_b; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + __m128i v_filter_b = _mm_set1_epi16((bilinear_filters_2t[xoffset][1] << 8) + + bilinear_filters_2t[xoffset][0]); + assert(xoffset < BIL_SUBPEL_SHIFTS); + for (i = 0; i < h; i += 2) { + // Load the src data + v_src0_b = _mm_loadu_si128((const __m128i *)(src)); + v_src0_shift_b = _mm_srli_si128(v_src0_b, 1); + v_src1_b = _mm_loadu_si128((const __m128i *)(src + src_stride)); + v_src1_shift_b = _mm_srli_si128(v_src1_b, 1); + // Apply the x filter + if (xoffset == HALF_PIXEL_OFFSET) { + v_src1_b = _mm_unpacklo_epi64(v_src0_b, v_src1_b); + v_src1_shift_b = _mm_unpacklo_epi64(v_src0_shift_b, v_src1_shift_b); + v_res_b = _mm_avg_epu8(v_src1_b, v_src1_shift_b); + } else { + apply_filter_lo(v_src0_b, v_src0_shift_b, v_filter_b, &v_filtered0_w); + apply_filter_lo(v_src1_b, v_src1_shift_b, v_filter_b, &v_filtered1_w); + v_res_b = _mm_packus_epi16(v_filtered0_w, v_filtered1_w); + } + // Load the dst data + v_dst_b = _mm_unpacklo_epi64( + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 0)), + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 1))); + // Load the mask data + v_msk_b = _mm_unpacklo_epi64( + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 0)), + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 1))); + // Compute the sum and SSE + sum_and_sse(v_res_b, v_dst_b, v_msk_b, &v_sum_d, &v_sse_q); + // Move onto the next set of rows + src += src_stride * 2; + dst += dst_stride * 2; + msk += msk_stride * 2; + } + return calc_masked_variance(v_sum_d, v_sse_q, sse, 8, h); +} + +// Note order in which rows loaded xmm[127:96] = row 1, xmm[95:64] = row 2, +// xmm[63:32] = row 3, xmm[31:0] = row 4 +unsigned int aom_masked_subpel_var4xH_xnonzero_ynonzero( + const uint8_t *src, int src_stride, int xoffset, int yoffset, + const uint8_t *dst, int dst_stride, const uint8_t *msk, int msk_stride, + unsigned int *sse, int h) { + int i; + __m128i v_src0_b, v_src1_b, v_src2_b, v_src3_b, v_filtered0_w, v_filtered2_w; + __m128i v_src0_shift_b, v_src1_shift_b, v_src2_shift_b, v_src3_shift_b; + __m128i v_dst0_b, v_dst1_b, v_dst2_b, v_dst3_b, v_temp_b; + __m128i v_msk0_b, v_msk1_b, v_msk2_b, v_msk3_b, v_extra_row_b, v_res_b; + __m128i v_xres_b[2]; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + __m128i v_filterx_b = _mm_set1_epi16((bilinear_filters_2t[xoffset][1] << 8) + + bilinear_filters_2t[xoffset][0]); + __m128i v_filtery_b = _mm_set1_epi16((bilinear_filters_2t[yoffset][1] << 8) + + bilinear_filters_2t[yoffset][0]); + assert(xoffset < BIL_SUBPEL_SHIFTS); + assert(yoffset < BIL_SUBPEL_SHIFTS); + for (i = 0; i < h; i += 4) { + // Load the src data + v_src0_b = _mm_loadl_epi64((const __m128i *)src); + v_src0_shift_b = _mm_srli_si128(v_src0_b, 1); + v_src1_b = _mm_loadl_epi64((const __m128i *)(src + src_stride * 1)); + v_src0_b = _mm_unpacklo_epi32(v_src1_b, v_src0_b); + v_src1_shift_b = _mm_srli_si128(v_src1_b, 1); + v_src2_b = _mm_loadl_epi64((const __m128i *)(src + src_stride * 2)); + v_src0_shift_b = _mm_unpacklo_epi32(v_src1_shift_b, v_src0_shift_b); + v_src2_shift_b = _mm_srli_si128(v_src2_b, 1); + v_src3_b = _mm_loadl_epi64((const __m128i *)(src + src_stride * 3)); + v_src2_b = _mm_unpacklo_epi32(v_src3_b, v_src2_b); + v_src3_shift_b = _mm_srli_si128(v_src3_b, 1); + v_src2_shift_b = _mm_unpacklo_epi32(v_src3_shift_b, v_src2_shift_b); + // Apply the x filter + if (xoffset == HALF_PIXEL_OFFSET) { + v_src0_b = _mm_unpacklo_epi64(v_src2_b, v_src0_b); + v_src0_shift_b = _mm_unpacklo_epi64(v_src2_shift_b, v_src0_shift_b); + v_xres_b[i == 0 ? 0 : 1] = _mm_avg_epu8(v_src0_b, v_src0_shift_b); + } else { + apply_filter_lo(v_src0_b, v_src0_shift_b, v_filterx_b, &v_filtered0_w); + apply_filter_lo(v_src2_b, v_src2_shift_b, v_filterx_b, &v_filtered2_w); + v_xres_b[i == 0 ? 0 : 1] = _mm_packus_epi16(v_filtered2_w, v_filtered0_w); + } + // Move onto the next set of rows + src += src_stride * 4; + } + // Load one more row to be used in the y filter + v_src0_b = _mm_loadl_epi64((const __m128i *)src); + v_src0_shift_b = _mm_srli_si128(v_src0_b, 1); + // Apply the x filter + if (xoffset == HALF_PIXEL_OFFSET) { + v_extra_row_b = _mm_and_si128(_mm_avg_epu8(v_src0_b, v_src0_shift_b), + _mm_setr_epi32(-1, 0, 0, 0)); + } else { + apply_filter_lo(v_src0_b, v_src0_shift_b, v_filterx_b, &v_filtered0_w); + v_extra_row_b = + _mm_and_si128(_mm_packus_epi16(v_filtered0_w, _mm_setzero_si128()), + _mm_setr_epi32(-1, 0, 0, 0)); + } + + for (i = 0; i < h; i += 4) { + if (h == 8 && i == 0) { + v_temp_b = _mm_or_si128(_mm_slli_si128(v_xres_b[0], 4), + _mm_srli_si128(v_xres_b[1], 12)); + } else { + v_temp_b = _mm_or_si128(_mm_slli_si128(v_xres_b[i == 0 ? 0 : 1], 4), + v_extra_row_b); + } + // Apply the y filter + if (yoffset == HALF_PIXEL_OFFSET) { + v_res_b = _mm_avg_epu8(v_xres_b[i == 0 ? 0 : 1], v_temp_b); + } else { + v_res_b = apply_filter(v_xres_b[i == 0 ? 0 : 1], v_temp_b, v_filtery_b); + } + + // Load the dst data + v_dst0_b = _mm_cvtsi32_si128(*(const uint32_t *)(dst + dst_stride * 0)); + v_dst1_b = _mm_cvtsi32_si128(*(const uint32_t *)(dst + dst_stride * 1)); + v_dst0_b = _mm_unpacklo_epi32(v_dst1_b, v_dst0_b); + v_dst2_b = _mm_cvtsi32_si128(*(const uint32_t *)(dst + dst_stride * 2)); + v_dst3_b = _mm_cvtsi32_si128(*(const uint32_t *)(dst + dst_stride * 3)); + v_dst2_b = _mm_unpacklo_epi32(v_dst3_b, v_dst2_b); + v_dst0_b = _mm_unpacklo_epi64(v_dst2_b, v_dst0_b); + // Load the mask data + v_msk0_b = _mm_cvtsi32_si128(*(const uint32_t *)(msk + msk_stride * 0)); + v_msk1_b = _mm_cvtsi32_si128(*(const uint32_t *)(msk + msk_stride * 1)); + v_msk0_b = _mm_unpacklo_epi32(v_msk1_b, v_msk0_b); + v_msk2_b = _mm_cvtsi32_si128(*(const uint32_t *)(msk + msk_stride * 2)); + v_msk3_b = _mm_cvtsi32_si128(*(const uint32_t *)(msk + msk_stride * 3)); + v_msk2_b = _mm_unpacklo_epi32(v_msk3_b, v_msk2_b); + v_msk0_b = _mm_unpacklo_epi64(v_msk2_b, v_msk0_b); + // Compute the sum and SSE + sum_and_sse(v_res_b, v_dst0_b, v_msk0_b, &v_sum_d, &v_sse_q); + // Move onto the next set of rows + dst += dst_stride * 4; + msk += msk_stride * 4; + } + return calc_masked_variance(v_sum_d, v_sse_q, sse, 4, h); +} + +unsigned int aom_masked_subpel_var8xH_xnonzero_ynonzero( + const uint8_t *src, int src_stride, int xoffset, int yoffset, + const uint8_t *dst, int dst_stride, const uint8_t *msk, int msk_stride, + unsigned int *sse, int h) { + int i; + __m128i v_src0_b, v_src1_b, v_filtered0_w, v_filtered1_w, v_dst_b, v_msk_b; + __m128i v_src0_shift_b, v_src1_shift_b; + __m128i v_xres0_b, v_xres1_b, v_res_b, v_temp_b; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + __m128i v_filterx_b = _mm_set1_epi16((bilinear_filters_2t[xoffset][1] << 8) + + bilinear_filters_2t[xoffset][0]); + __m128i v_filtery_b = _mm_set1_epi16((bilinear_filters_2t[yoffset][1] << 8) + + bilinear_filters_2t[yoffset][0]); + assert(xoffset < BIL_SUBPEL_SHIFTS); + assert(yoffset < BIL_SUBPEL_SHIFTS); + // Load the first block of src data + v_src0_b = _mm_loadu_si128((const __m128i *)(src)); + v_src0_shift_b = _mm_srli_si128(v_src0_b, 1); + v_src1_b = _mm_loadu_si128((const __m128i *)(src + src_stride)); + v_src1_shift_b = _mm_srli_si128(v_src1_b, 1); + // Apply the x filter + if (xoffset == HALF_PIXEL_OFFSET) { + v_src1_b = _mm_unpacklo_epi64(v_src0_b, v_src1_b); + v_src1_shift_b = _mm_unpacklo_epi64(v_src0_shift_b, v_src1_shift_b); + v_xres0_b = _mm_avg_epu8(v_src1_b, v_src1_shift_b); + } else { + apply_filter_lo(v_src0_b, v_src0_shift_b, v_filterx_b, &v_filtered0_w); + apply_filter_lo(v_src1_b, v_src1_shift_b, v_filterx_b, &v_filtered1_w); + v_xres0_b = _mm_packus_epi16(v_filtered0_w, v_filtered1_w); + } + for (i = 0; i < h; i += 4) { + // Load the next block of src data + v_src0_b = _mm_loadu_si128((const __m128i *)(src + src_stride * 2)); + v_src0_shift_b = _mm_srli_si128(v_src0_b, 1); + v_src1_b = _mm_loadu_si128((const __m128i *)(src + src_stride * 3)); + v_src1_shift_b = _mm_srli_si128(v_src1_b, 1); + // Apply the x filter + if (xoffset == HALF_PIXEL_OFFSET) { + v_src1_b = _mm_unpacklo_epi64(v_src0_b, v_src1_b); + v_src1_shift_b = _mm_unpacklo_epi64(v_src0_shift_b, v_src1_shift_b); + v_xres1_b = _mm_avg_epu8(v_src1_b, v_src1_shift_b); + } else { + apply_filter_lo(v_src0_b, v_src0_shift_b, v_filterx_b, &v_filtered0_w); + apply_filter_lo(v_src1_b, v_src1_shift_b, v_filterx_b, &v_filtered1_w); + v_xres1_b = _mm_packus_epi16(v_filtered0_w, v_filtered1_w); + } + // Apply the y filter to the previous block + v_temp_b = _mm_or_si128(_mm_srli_si128(v_xres0_b, 8), + _mm_slli_si128(v_xres1_b, 8)); + if (yoffset == HALF_PIXEL_OFFSET) { + v_res_b = _mm_avg_epu8(v_xres0_b, v_temp_b); + } else { + v_res_b = apply_filter(v_xres0_b, v_temp_b, v_filtery_b); + } + // Load the dst data + v_dst_b = _mm_unpacklo_epi64( + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 0)), + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 1))); + // Load the mask data + v_msk_b = _mm_unpacklo_epi64( + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 0)), + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 1))); + // Compute the sum and SSE + sum_and_sse(v_res_b, v_dst_b, v_msk_b, &v_sum_d, &v_sse_q); + + // Load the next block of src data + v_src0_b = _mm_loadu_si128((const __m128i *)(src + src_stride * 4)); + v_src0_shift_b = _mm_srli_si128(v_src0_b, 1); + v_src1_b = _mm_loadu_si128((const __m128i *)(src + src_stride * 5)); + v_src1_shift_b = _mm_srli_si128(v_src1_b, 1); + // Apply the x filter + if (xoffset == HALF_PIXEL_OFFSET) { + v_src1_b = _mm_unpacklo_epi64(v_src0_b, v_src1_b); + v_src1_shift_b = _mm_unpacklo_epi64(v_src0_shift_b, v_src1_shift_b); + v_xres0_b = _mm_avg_epu8(v_src1_b, v_src1_shift_b); + } else { + apply_filter_lo(v_src0_b, v_src0_shift_b, v_filterx_b, &v_filtered0_w); + apply_filter_lo(v_src1_b, v_src1_shift_b, v_filterx_b, &v_filtered1_w); + v_xres0_b = _mm_packus_epi16(v_filtered0_w, v_filtered1_w); + } + // Apply the y filter to the previous block + v_temp_b = _mm_or_si128(_mm_srli_si128(v_xres1_b, 8), + _mm_slli_si128(v_xres0_b, 8)); + if (yoffset == HALF_PIXEL_OFFSET) { + v_res_b = _mm_avg_epu8(v_xres1_b, v_temp_b); + } else { + v_res_b = apply_filter(v_xres1_b, v_temp_b, v_filtery_b); + } + // Load the dst data + v_dst_b = _mm_unpacklo_epi64( + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 2)), + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 3))); + // Load the mask data + v_msk_b = _mm_unpacklo_epi64( + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 2)), + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 3))); + // Compute the sum and SSE + sum_and_sse(v_res_b, v_dst_b, v_msk_b, &v_sum_d, &v_sse_q); + // Move onto the next set of rows + src += src_stride * 4; + dst += dst_stride * 4; + msk += msk_stride * 4; + } + return calc_masked_variance(v_sum_d, v_sse_q, sse, 8, h); +} + +// For W >=16 +#define MASK_SUBPIX_VAR_LARGE(W, H) \ + unsigned int aom_masked_sub_pixel_variance##W##x##H##_ssse3( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, const uint8_t *msk, int msk_stride, \ + unsigned int *sse) { \ + assert(W % 16 == 0); \ + if (xoffset == 0) { \ + if (yoffset == 0) \ + return aom_masked_variance##W##x##H##_ssse3( \ + src, src_stride, dst, dst_stride, msk, msk_stride, sse); \ + else if (yoffset == HALF_PIXEL_OFFSET) \ + return aom_masked_subpel_varWxH_xzero( \ + src, src_stride, HALF_PIXEL_OFFSET, dst, dst_stride, msk, \ + msk_stride, sse, W, H, apply_filter_avg); \ + else \ + return aom_masked_subpel_varWxH_xzero(src, src_stride, yoffset, dst, \ + dst_stride, msk, msk_stride, \ + sse, W, H, apply_filter); \ + } else if (yoffset == 0) { \ + if (xoffset == HALF_PIXEL_OFFSET) \ + return aom_masked_subpel_varWxH_yzero( \ + src, src_stride, HALF_PIXEL_OFFSET, dst, dst_stride, msk, \ + msk_stride, sse, W, H, apply_filter_avg); \ + else \ + return aom_masked_subpel_varWxH_yzero(src, src_stride, xoffset, dst, \ + dst_stride, msk, msk_stride, \ + sse, W, H, apply_filter); \ + } else if (xoffset == HALF_PIXEL_OFFSET) { \ + if (yoffset == HALF_PIXEL_OFFSET) \ + return aom_masked_subpel_varWxH_xnonzero_ynonzero( \ + src, src_stride, HALF_PIXEL_OFFSET, HALF_PIXEL_OFFSET, dst, \ + dst_stride, msk, msk_stride, sse, W, H, apply_filter_avg, \ + apply_filter_avg); \ + else \ + return aom_masked_subpel_varWxH_xnonzero_ynonzero( \ + src, src_stride, HALF_PIXEL_OFFSET, yoffset, dst, dst_stride, msk, \ + msk_stride, sse, W, H, apply_filter_avg, apply_filter); \ + } else { \ + if (yoffset == HALF_PIXEL_OFFSET) \ + return aom_masked_subpel_varWxH_xnonzero_ynonzero( \ + src, src_stride, xoffset, HALF_PIXEL_OFFSET, dst, dst_stride, msk, \ + msk_stride, sse, W, H, apply_filter, apply_filter_avg); \ + else \ + return aom_masked_subpel_varWxH_xnonzero_ynonzero( \ + src, src_stride, xoffset, yoffset, dst, dst_stride, msk, \ + msk_stride, sse, W, H, apply_filter, apply_filter); \ + } \ + } + +// For W < 16 +#define MASK_SUBPIX_VAR_SMALL(W, H) \ + unsigned int aom_masked_sub_pixel_variance##W##x##H##_ssse3( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, const uint8_t *msk, int msk_stride, \ + unsigned int *sse) { \ + assert(W == 4 || W == 8); \ + if (xoffset == 0 && yoffset == 0) \ + return aom_masked_variance##W##x##H##_ssse3( \ + src, src_stride, dst, dst_stride, msk, msk_stride, sse); \ + else if (xoffset == 0) \ + return aom_masked_subpel_var##W##xH_xzero( \ + src, src_stride, yoffset, dst, dst_stride, msk, msk_stride, sse, H); \ + else if (yoffset == 0) \ + return aom_masked_subpel_var##W##xH_yzero( \ + src, src_stride, xoffset, dst, dst_stride, msk, msk_stride, sse, H); \ + else \ + return aom_masked_subpel_var##W##xH_xnonzero_ynonzero( \ + src, src_stride, xoffset, yoffset, dst, dst_stride, msk, msk_stride, \ + sse, H); \ + } + +MASK_SUBPIX_VAR_SMALL(4, 4) +MASK_SUBPIX_VAR_SMALL(4, 8) +MASK_SUBPIX_VAR_SMALL(8, 4) +MASK_SUBPIX_VAR_SMALL(8, 8) +MASK_SUBPIX_VAR_SMALL(8, 16) +MASK_SUBPIX_VAR_LARGE(16, 8) +MASK_SUBPIX_VAR_LARGE(16, 16) +MASK_SUBPIX_VAR_LARGE(16, 32) +MASK_SUBPIX_VAR_LARGE(32, 16) +MASK_SUBPIX_VAR_LARGE(32, 32) +MASK_SUBPIX_VAR_LARGE(32, 64) +MASK_SUBPIX_VAR_LARGE(64, 32) +MASK_SUBPIX_VAR_LARGE(64, 64) +#if CONFIG_EXT_PARTITION +MASK_SUBPIX_VAR_LARGE(64, 128) +MASK_SUBPIX_VAR_LARGE(128, 64) +MASK_SUBPIX_VAR_LARGE(128, 128) +#endif // CONFIG_EXT_PARTITION + +#if CONFIG_HIGHBITDEPTH +typedef uint32_t (*highbd_calc_masked_var_t)(__m128i v_sum_d, __m128i v_sse_q, + uint32_t *sse, int w, int h); +typedef unsigned int (*highbd_variance_fn_t)(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, + const uint8_t *m, int m_stride, + unsigned int *sse); +typedef __m128i (*highbd_filter_fn_t)(__m128i v_a_w, __m128i v_b_w, + __m128i v_filter_w); + +static INLINE __m128i highbd_apply_filter_avg(const __m128i v_a_w, + const __m128i v_b_w, + const __m128i v_filter_w) { + (void)v_filter_w; + return _mm_avg_epu16(v_a_w, v_b_w); +} + +static INLINE __m128i highbd_apply_filter(const __m128i v_a_w, + const __m128i v_b_w, + const __m128i v_filter_w) { + const __m128i v_rounding_d = _mm_set1_epi32(1 << (FILTER_BITS - 1)); + __m128i v_input_lo_w = _mm_unpacklo_epi16(v_a_w, v_b_w); + __m128i v_input_hi_w = _mm_unpackhi_epi16(v_a_w, v_b_w); + __m128i v_temp0_d = _mm_madd_epi16(v_input_lo_w, v_filter_w); + __m128i v_temp1_d = _mm_madd_epi16(v_input_hi_w, v_filter_w); + __m128i v_res_lo_d = + _mm_srai_epi32(_mm_add_epi32(v_temp0_d, v_rounding_d), FILTER_BITS); + __m128i v_res_hi_d = + _mm_srai_epi32(_mm_add_epi32(v_temp1_d, v_rounding_d), FILTER_BITS); + return _mm_packs_epi32(v_res_lo_d, v_res_hi_d); +} +// Apply the filter to the contents of the lower half of a and b +static INLINE void highbd_apply_filter_lo(const __m128i v_a_lo_w, + const __m128i v_b_lo_w, + const __m128i v_filter_w, + __m128i *v_res_d) { + const __m128i v_rounding_d = _mm_set1_epi32(1 << (FILTER_BITS - 1)); + __m128i v_input_w = _mm_unpacklo_epi16(v_a_lo_w, v_b_lo_w); + __m128i v_temp0_d = _mm_madd_epi16(v_input_w, v_filter_w); + *v_res_d = + _mm_srai_epi32(_mm_add_epi32(v_temp0_d, v_rounding_d), FILTER_BITS); +} + +static void highbd_sum_and_sse(const __m128i v_a_w, const __m128i v_b_w, + const __m128i v_m_b, __m128i *v_sum_d, + __m128i *v_sse_q) { + const __m128i v_zero = _mm_setzero_si128(); + const __m128i v_m_w = _mm_unpacklo_epi8(v_m_b, v_zero); + + // Difference: [-2^12, 2^12] => 13 bits (incld sign bit) + const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w); + + // Error - [-4095, 4095] * [0, 64] & sum pairs => fits in 19 + 1 bits + const __m128i v_e_d = _mm_madd_epi16(v_d_w, v_m_w); + + // Squared error - max (18 bits * 18 bits) = 36 bits (no sign bit) + const __m128i v_absd_w = _mm_abs_epi16(v_d_w); + const __m128i v_dlo_d = _mm_unpacklo_epi16(v_absd_w, v_zero); + const __m128i v_mlo_d = _mm_unpacklo_epi16(v_m_w, v_zero); + const __m128i v_elo_d = _mm_madd_epi16(v_dlo_d, v_mlo_d); + const __m128i v_dhi_d = _mm_unpackhi_epi16(v_absd_w, v_zero); + const __m128i v_mhi_d = _mm_unpackhi_epi16(v_m_w, v_zero); + const __m128i v_ehi_d = _mm_madd_epi16(v_dhi_d, v_mhi_d); + // Square and sum the errors -> 36bits * 4 = 38bits + __m128i v_se0_q, v_se1_q, v_se2_q, v_se3_q, v_se_q, v_elo1_d, v_ehi3_d; + v_se0_q = _mm_mul_epu32(v_elo_d, v_elo_d); + v_elo1_d = _mm_srli_si128(v_elo_d, 4); + v_se1_q = _mm_mul_epu32(v_elo1_d, v_elo1_d); + v_se0_q = _mm_add_epi64(v_se0_q, v_se1_q); + v_se2_q = _mm_mul_epu32(v_ehi_d, v_ehi_d); + v_ehi3_d = _mm_srli_si128(v_ehi_d, 4); + v_se3_q = _mm_mul_epu32(v_ehi3_d, v_ehi3_d); + v_se1_q = _mm_add_epi64(v_se2_q, v_se3_q); + v_se_q = _mm_add_epi64(v_se0_q, v_se1_q); + + // Accumulate + *v_sum_d = _mm_add_epi32(*v_sum_d, v_e_d); + *v_sse_q = _mm_add_epi64(*v_sse_q, v_se_q); +} + +static INLINE uint32_t highbd_10_calc_masked_variance(__m128i v_sum_d, + __m128i v_sse_q, + uint32_t *sse, int w, + int h) { + int64_t sum64; + uint64_t sse64; + + // Horizontal sum + sum64 = hsum_epi32_si32(v_sum_d); + sse64 = hsum_epi64_si64(v_sse_q); + + sum64 = (sum64 >= 0) ? sum64 : -sum64; + + // Round + sum64 = ROUND_POWER_OF_TWO(sum64, 6); + sse64 = ROUND_POWER_OF_TWO(sse64, 12); + + // Normalise + sum64 = ROUND_POWER_OF_TWO(sum64, 2); + sse64 = ROUND_POWER_OF_TWO(sse64, 4); + + // Store the SSE + *sse = (uint32_t)sse64; + // Compute the variance + return *sse - (uint32_t)((sum64 * sum64) / (w * h)); +} +static INLINE uint32_t highbd_12_calc_masked_variance(__m128i v_sum_d, + __m128i v_sse_q, + uint32_t *sse, int w, + int h) { + int64_t sum64; + uint64_t sse64; + + // Horizontal sum + sum64 = hsum_epi32_si64(v_sum_d); + sse64 = hsum_epi64_si64(v_sse_q); + + sum64 = (sum64 >= 0) ? sum64 : -sum64; + + // Round + sum64 = ROUND_POWER_OF_TWO(sum64, 6); + sse64 = ROUND_POWER_OF_TWO(sse64, 12); + + // Normalise + sum64 = ROUND_POWER_OF_TWO(sum64, 4); + sse64 = ROUND_POWER_OF_TWO(sse64, 8); + + // Store the SSE + *sse = (uint32_t)sse64; + // Compute the variance + return *sse - (uint32_t)((sum64 * sum64) / (w * h)); +} + +// High bit depth functions for width (W) >= 8 +unsigned int aom_highbd_masked_subpel_varWxH_xzero( + const uint16_t *src, int src_stride, int yoffset, const uint16_t *dst, + int dst_stride, const uint8_t *msk, int msk_stride, unsigned int *sse, + int w, int h, highbd_filter_fn_t filter_fn, + highbd_calc_masked_var_t calc_var) { + int i, j; + __m128i v_src0_w, v_src1_w, v_res_w, v_dst_w, v_msk_b; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + const __m128i v_filter_w = + _mm_set1_epi32((bilinear_filters_2t[yoffset][1] << 16) + + bilinear_filters_2t[yoffset][0]); + assert(yoffset < BIL_SUBPEL_SHIFTS); + for (j = 0; j < w; j += 8) { + // Load the first row ready + v_src0_w = _mm_loadu_si128((const __m128i *)(src + j)); + // Process 2 rows at a time + for (i = 0; i < h; i += 2) { + // Load the next row apply the filter + v_src1_w = _mm_loadu_si128((const __m128i *)(src + j + src_stride)); + v_res_w = filter_fn(v_src0_w, v_src1_w, v_filter_w); + // Load the dst and msk for the variance calculation + v_dst_w = _mm_loadu_si128((const __m128i *)(dst + j)); + v_msk_b = _mm_loadl_epi64((const __m128i *)(msk + j)); + highbd_sum_and_sse(v_res_w, v_dst_w, v_msk_b, &v_sum_d, &v_sse_q); + + // Load the next row apply the filter + v_src0_w = _mm_loadu_si128((const __m128i *)(src + j + src_stride * 2)); + v_res_w = filter_fn(v_src1_w, v_src0_w, v_filter_w); + // Load the dst and msk for the variance calculation + v_dst_w = _mm_loadu_si128((const __m128i *)(dst + j + dst_stride)); + v_msk_b = _mm_loadl_epi64((const __m128i *)(msk + j + msk_stride)); + highbd_sum_and_sse(v_res_w, v_dst_w, v_msk_b, &v_sum_d, &v_sse_q); + // Move onto the next block of rows + src += src_stride * 2; + dst += dst_stride * 2; + msk += msk_stride * 2; + } + // Reset to the top of the block + src -= src_stride * h; + dst -= dst_stride * h; + msk -= msk_stride * h; + } + return calc_var(v_sum_d, v_sse_q, sse, w, h); +} +unsigned int aom_highbd_masked_subpel_varWxH_yzero( + const uint16_t *src, int src_stride, int xoffset, const uint16_t *dst, + int dst_stride, const uint8_t *msk, int msk_stride, unsigned int *sse, + int w, int h, highbd_filter_fn_t filter_fn, + highbd_calc_masked_var_t calc_var) { + int i, j; + __m128i v_src0_w, v_src1_w, v_res_w, v_dst_w, v_msk_b; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + const __m128i v_filter_w = + _mm_set1_epi32((bilinear_filters_2t[xoffset][1] << 16) + + bilinear_filters_2t[xoffset][0]); + assert(xoffset < BIL_SUBPEL_SHIFTS); + for (i = 0; i < h; i++) { + for (j = 0; j < w; j += 8) { + // Load this row & apply the filter to them + v_src0_w = _mm_loadu_si128((const __m128i *)(src + j)); + v_src1_w = _mm_loadu_si128((const __m128i *)(src + j + 1)); + v_res_w = filter_fn(v_src0_w, v_src1_w, v_filter_w); + + // Load the dst and msk for the variance calculation + v_dst_w = _mm_loadu_si128((const __m128i *)(dst + j)); + v_msk_b = _mm_loadl_epi64((const __m128i *)(msk + j)); + highbd_sum_and_sse(v_res_w, v_dst_w, v_msk_b, &v_sum_d, &v_sse_q); + } + src += src_stride; + dst += dst_stride; + msk += msk_stride; + } + return calc_var(v_sum_d, v_sse_q, sse, w, h); +} + +unsigned int aom_highbd_masked_subpel_varWxH_xnonzero_ynonzero( + const uint16_t *src, int src_stride, int xoffset, int yoffset, + const uint16_t *dst, int dst_stride, const uint8_t *msk, int msk_stride, + unsigned int *sse, int w, int h, highbd_filter_fn_t xfilter_fn, + highbd_filter_fn_t yfilter_fn, highbd_calc_masked_var_t calc_var) { + int i, j; + __m128i v_src0_w, v_src1_w, v_src2_w, v_src3_w; + __m128i v_filtered0_w, v_filtered1_w, v_res_w, v_dst_w, v_msk_b; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + const __m128i v_filterx_w = + _mm_set1_epi32((bilinear_filters_2t[xoffset][1] << 16) + + bilinear_filters_2t[xoffset][0]); + const __m128i v_filtery_w = + _mm_set1_epi32((bilinear_filters_2t[yoffset][1] << 16) + + bilinear_filters_2t[yoffset][0]); + assert(xoffset < BIL_SUBPEL_SHIFTS); + assert(yoffset < BIL_SUBPEL_SHIFTS); + for (j = 0; j < w; j += 8) { + // Load the first row ready + v_src0_w = _mm_loadu_si128((const __m128i *)(src + j)); + v_src1_w = _mm_loadu_si128((const __m128i *)(src + j + 1)); + v_filtered0_w = xfilter_fn(v_src0_w, v_src1_w, v_filterx_w); + // Process 2 rows at a time + for (i = 0; i < h; i += 2) { + // Load the next row & apply the filter + v_src2_w = _mm_loadu_si128((const __m128i *)(src + src_stride + j)); + v_src3_w = _mm_loadu_si128((const __m128i *)(src + src_stride + j + 1)); + v_filtered1_w = xfilter_fn(v_src2_w, v_src3_w, v_filterx_w); + // Load the dst and msk for the variance calculation + v_dst_w = _mm_loadu_si128((const __m128i *)(dst + j)); + v_msk_b = _mm_loadl_epi64((const __m128i *)(msk + j)); + // Complete the calculation for this row and add it to the running total + v_res_w = yfilter_fn(v_filtered0_w, v_filtered1_w, v_filtery_w); + highbd_sum_and_sse(v_res_w, v_dst_w, v_msk_b, &v_sum_d, &v_sse_q); + + // Load the next row & apply the filter + v_src0_w = _mm_loadu_si128((const __m128i *)(src + src_stride * 2 + j)); + v_src1_w = + _mm_loadu_si128((const __m128i *)(src + src_stride * 2 + j + 1)); + v_filtered0_w = xfilter_fn(v_src0_w, v_src1_w, v_filterx_w); + // Load the dst and msk for the variance calculation + v_dst_w = _mm_loadu_si128((const __m128i *)(dst + dst_stride + j)); + v_msk_b = _mm_loadl_epi64((const __m128i *)(msk + msk_stride + j)); + // Complete the calculation for this row and add it to the running total + v_res_w = yfilter_fn(v_filtered1_w, v_filtered0_w, v_filtery_w); + highbd_sum_and_sse(v_res_w, v_dst_w, v_msk_b, &v_sum_d, &v_sse_q); + // Move onto the next block of rows + src += src_stride * 2; + dst += dst_stride * 2; + msk += msk_stride * 2; + } + // Reset to the top of the block + src -= src_stride * h; + dst -= dst_stride * h; + msk -= msk_stride * h; + } + return calc_var(v_sum_d, v_sse_q, sse, w, h); +} + +// Note order in which rows loaded xmm[127:64] = row 1, xmm[63:0] = row 2 +unsigned int aom_highbd_masked_subpel_var4xH_xzero( + const uint16_t *src, int src_stride, int yoffset, const uint16_t *dst, + int dst_stride, const uint8_t *msk, int msk_stride, unsigned int *sse, + int h, highbd_calc_masked_var_t calc_var) { + int i; + __m128i v_src0_w, v_src1_w, v_filtered0_d, v_filtered1_d, v_res_w; + __m128i v_dst_w, v_msk_b; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + __m128i v_filter_w = _mm_set1_epi32((bilinear_filters_2t[yoffset][1] << 16) + + bilinear_filters_2t[yoffset][0]); + assert(yoffset < BIL_SUBPEL_SHIFTS); + // Load the first row of src data ready + v_src0_w = _mm_loadl_epi64((const __m128i *)src); + for (i = 0; i < h; i += 2) { + if (yoffset == HALF_PIXEL_OFFSET) { + // Load the rest of the source data for these rows + v_src1_w = _mm_or_si128( + _mm_slli_si128(v_src0_w, 8), + _mm_loadl_epi64((const __m128i *)(src + src_stride * 1))); + v_src0_w = _mm_or_si128( + _mm_slli_si128(v_src1_w, 8), + _mm_loadl_epi64((const __m128i *)(src + src_stride * 2))); + // Apply the y filter + v_res_w = _mm_avg_epu16(v_src1_w, v_src0_w); + } else { + // Load the data and apply the y filter + v_src1_w = _mm_loadl_epi64((const __m128i *)(src + src_stride * 1)); + highbd_apply_filter_lo(v_src0_w, v_src1_w, v_filter_w, &v_filtered0_d); + v_src0_w = _mm_loadl_epi64((const __m128i *)(src + src_stride * 2)); + highbd_apply_filter_lo(v_src1_w, v_src0_w, v_filter_w, &v_filtered1_d); + v_res_w = _mm_packs_epi32(v_filtered1_d, v_filtered0_d); + } + // Load the dst data + v_dst_w = _mm_unpacklo_epi64( + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 1)), + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 0))); + // Load the mask data + v_msk_b = _mm_unpacklo_epi32( + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 1)), + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 0))); + // Compute the sum and SSE + highbd_sum_and_sse(v_res_w, v_dst_w, v_msk_b, &v_sum_d, &v_sse_q); + // Move onto the next set of rows + src += src_stride * 2; + dst += dst_stride * 2; + msk += msk_stride * 2; + } + return calc_var(v_sum_d, v_sse_q, sse, 4, h); +} + +unsigned int aom_highbd_masked_subpel_var4xH_yzero( + const uint16_t *src, int src_stride, int xoffset, const uint16_t *dst, + int dst_stride, const uint8_t *msk, int msk_stride, unsigned int *sse, + int h, highbd_calc_masked_var_t calc_var) { + int i; + __m128i v_src0_w, v_src1_w, v_filtered0_d, v_filtered1_d; + __m128i v_src0_shift_w, v_src1_shift_w, v_res_w, v_dst_w, v_msk_b; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + __m128i v_filter_w = _mm_set1_epi32((bilinear_filters_2t[xoffset][1] << 16) + + bilinear_filters_2t[xoffset][0]); + assert(xoffset < BIL_SUBPEL_SHIFTS); + for (i = 0; i < h; i += 2) { + // Load the src data + v_src0_w = _mm_loadu_si128((const __m128i *)(src)); + v_src0_shift_w = _mm_srli_si128(v_src0_w, 2); + v_src1_w = _mm_loadu_si128((const __m128i *)(src + src_stride)); + v_src1_shift_w = _mm_srli_si128(v_src1_w, 2); + // Apply the x filter + if (xoffset == HALF_PIXEL_OFFSET) { + v_src1_w = _mm_unpacklo_epi64(v_src0_w, v_src1_w); + v_src1_shift_w = _mm_unpacklo_epi64(v_src0_shift_w, v_src1_shift_w); + v_res_w = _mm_avg_epu16(v_src1_w, v_src1_shift_w); + } else { + highbd_apply_filter_lo(v_src0_w, v_src0_shift_w, v_filter_w, + &v_filtered0_d); + highbd_apply_filter_lo(v_src1_w, v_src1_shift_w, v_filter_w, + &v_filtered1_d); + v_res_w = _mm_packs_epi32(v_filtered0_d, v_filtered1_d); + } + // Load the dst data + v_dst_w = _mm_unpacklo_epi64( + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 0)), + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 1))); + // Load the mask data + v_msk_b = _mm_unpacklo_epi32( + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 0)), + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 1))); + // Compute the sum and SSE + highbd_sum_and_sse(v_res_w, v_dst_w, v_msk_b, &v_sum_d, &v_sse_q); + // Move onto the next set of rows + src += src_stride * 2; + dst += dst_stride * 2; + msk += msk_stride * 2; + } + return calc_var(v_sum_d, v_sse_q, sse, 4, h); +} + +unsigned int aom_highbd_masked_subpel_var4xH_xnonzero_ynonzero( + const uint16_t *src, int src_stride, int xoffset, int yoffset, + const uint16_t *dst, int dst_stride, const uint8_t *msk, int msk_stride, + unsigned int *sse, int h, highbd_calc_masked_var_t calc_var) { + int i; + __m128i v_src0_w, v_src1_w, v_filtered0_d, v_filtered1_d, v_dst_w, v_msk_b; + __m128i v_src0_shift_w, v_src1_shift_w; + __m128i v_xres0_w, v_xres1_w, v_res_w, v_temp_w; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_q = _mm_setzero_si128(); + __m128i v_filterx_w = _mm_set1_epi32((bilinear_filters_2t[xoffset][1] << 16) + + bilinear_filters_2t[xoffset][0]); + __m128i v_filtery_w = _mm_set1_epi32((bilinear_filters_2t[yoffset][1] << 16) + + bilinear_filters_2t[yoffset][0]); + assert(xoffset < BIL_SUBPEL_SHIFTS); + assert(yoffset < BIL_SUBPEL_SHIFTS); + // Load the first block of src data + v_src0_w = _mm_loadu_si128((const __m128i *)(src)); + v_src0_shift_w = _mm_srli_si128(v_src0_w, 2); + v_src1_w = _mm_loadu_si128((const __m128i *)(src + src_stride)); + v_src1_shift_w = _mm_srli_si128(v_src1_w, 2); + // Apply the x filter + if (xoffset == HALF_PIXEL_OFFSET) { + v_src1_w = _mm_unpacklo_epi64(v_src0_w, v_src1_w); + v_src1_shift_w = _mm_unpacklo_epi64(v_src0_shift_w, v_src1_shift_w); + v_xres0_w = _mm_avg_epu16(v_src1_w, v_src1_shift_w); + } else { + highbd_apply_filter_lo(v_src0_w, v_src0_shift_w, v_filterx_w, + &v_filtered0_d); + highbd_apply_filter_lo(v_src1_w, v_src1_shift_w, v_filterx_w, + &v_filtered1_d); + v_xres0_w = _mm_packs_epi32(v_filtered0_d, v_filtered1_d); + } + for (i = 0; i < h; i += 4) { + // Load the next block of src data + v_src0_w = _mm_loadu_si128((const __m128i *)(src + src_stride * 2)); + v_src0_shift_w = _mm_srli_si128(v_src0_w, 2); + v_src1_w = _mm_loadu_si128((const __m128i *)(src + src_stride * 3)); + v_src1_shift_w = _mm_srli_si128(v_src1_w, 2); + // Apply the x filter + if (xoffset == HALF_PIXEL_OFFSET) { + v_src1_w = _mm_unpacklo_epi64(v_src0_w, v_src1_w); + v_src1_shift_w = _mm_unpacklo_epi64(v_src0_shift_w, v_src1_shift_w); + v_xres1_w = _mm_avg_epu16(v_src1_w, v_src1_shift_w); + } else { + highbd_apply_filter_lo(v_src0_w, v_src0_shift_w, v_filterx_w, + &v_filtered0_d); + highbd_apply_filter_lo(v_src1_w, v_src1_shift_w, v_filterx_w, + &v_filtered1_d); + v_xres1_w = _mm_packs_epi32(v_filtered0_d, v_filtered1_d); + } + // Apply the y filter to the previous block + v_temp_w = _mm_or_si128(_mm_srli_si128(v_xres0_w, 8), + _mm_slli_si128(v_xres1_w, 8)); + if (yoffset == HALF_PIXEL_OFFSET) { + v_res_w = _mm_avg_epu16(v_xres0_w, v_temp_w); + } else { + v_res_w = highbd_apply_filter(v_xres0_w, v_temp_w, v_filtery_w); + } + // Load the dst data + v_dst_w = _mm_unpacklo_epi64( + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 0)), + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 1))); + // Load the mask data + v_msk_b = _mm_unpacklo_epi32( + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 0)), + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 1))); + // Compute the sum and SSE + highbd_sum_and_sse(v_res_w, v_dst_w, v_msk_b, &v_sum_d, &v_sse_q); + + // Load the next block of src data + v_src0_w = _mm_loadu_si128((const __m128i *)(src + src_stride * 4)); + v_src0_shift_w = _mm_srli_si128(v_src0_w, 2); + v_src1_w = _mm_loadu_si128((const __m128i *)(src + src_stride * 5)); + v_src1_shift_w = _mm_srli_si128(v_src1_w, 2); + // Apply the x filter + if (xoffset == HALF_PIXEL_OFFSET) { + v_src1_w = _mm_unpacklo_epi64(v_src0_w, v_src1_w); + v_src1_shift_w = _mm_unpacklo_epi64(v_src0_shift_w, v_src1_shift_w); + v_xres0_w = _mm_avg_epu16(v_src1_w, v_src1_shift_w); + } else { + highbd_apply_filter_lo(v_src0_w, v_src0_shift_w, v_filterx_w, + &v_filtered0_d); + highbd_apply_filter_lo(v_src1_w, v_src1_shift_w, v_filterx_w, + &v_filtered1_d); + v_xres0_w = _mm_packs_epi32(v_filtered0_d, v_filtered1_d); + } + // Apply the y filter to the previous block + v_temp_w = _mm_or_si128(_mm_srli_si128(v_xres1_w, 8), + _mm_slli_si128(v_xres0_w, 8)); + if (yoffset == HALF_PIXEL_OFFSET) { + v_res_w = _mm_avg_epu16(v_xres1_w, v_temp_w); + } else { + v_res_w = highbd_apply_filter(v_xres1_w, v_temp_w, v_filtery_w); + } + // Load the dst data + v_dst_w = _mm_unpacklo_epi64( + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 2)), + _mm_loadl_epi64((const __m128i *)(dst + dst_stride * 3))); + // Load the mask data + v_msk_b = _mm_unpacklo_epi32( + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 2)), + _mm_loadl_epi64((const __m128i *)(msk + msk_stride * 3))); + // Compute the sum and SSE + highbd_sum_and_sse(v_res_w, v_dst_w, v_msk_b, &v_sum_d, &v_sse_q); + // Move onto the next set of rows + src += src_stride * 4; + dst += dst_stride * 4; + msk += msk_stride * 4; + } + return calc_var(v_sum_d, v_sse_q, sse, 4, h); +} + +// For W >=8 +#define HIGHBD_MASK_SUBPIX_VAR_LARGE(W, H) \ + unsigned int highbd_masked_sub_pixel_variance##W##x##H##_ssse3( \ + const uint8_t *src8, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst8, int dst_stride, const uint8_t *msk, int msk_stride, \ + unsigned int *sse, highbd_calc_masked_var_t calc_var, \ + highbd_variance_fn_t full_variance_function) { \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ + assert(W % 8 == 0); \ + if (xoffset == 0) { \ + if (yoffset == 0) \ + return full_variance_function(src8, src_stride, dst8, dst_stride, msk, \ + msk_stride, sse); \ + else if (yoffset == HALF_PIXEL_OFFSET) \ + return aom_highbd_masked_subpel_varWxH_xzero( \ + src, src_stride, HALF_PIXEL_OFFSET, dst, dst_stride, msk, \ + msk_stride, sse, W, H, highbd_apply_filter_avg, calc_var); \ + else \ + return aom_highbd_masked_subpel_varWxH_xzero( \ + src, src_stride, yoffset, dst, dst_stride, msk, msk_stride, sse, \ + W, H, highbd_apply_filter, calc_var); \ + } else if (yoffset == 0) { \ + if (xoffset == HALF_PIXEL_OFFSET) \ + return aom_highbd_masked_subpel_varWxH_yzero( \ + src, src_stride, HALF_PIXEL_OFFSET, dst, dst_stride, msk, \ + msk_stride, sse, W, H, highbd_apply_filter_avg, calc_var); \ + else \ + return aom_highbd_masked_subpel_varWxH_yzero( \ + src, src_stride, xoffset, dst, dst_stride, msk, msk_stride, sse, \ + W, H, highbd_apply_filter, calc_var); \ + } else if (xoffset == HALF_PIXEL_OFFSET) { \ + if (yoffset == HALF_PIXEL_OFFSET) \ + return aom_highbd_masked_subpel_varWxH_xnonzero_ynonzero( \ + src, src_stride, HALF_PIXEL_OFFSET, HALF_PIXEL_OFFSET, dst, \ + dst_stride, msk, msk_stride, sse, W, H, highbd_apply_filter_avg, \ + highbd_apply_filter_avg, calc_var); \ + else \ + return aom_highbd_masked_subpel_varWxH_xnonzero_ynonzero( \ + src, src_stride, HALF_PIXEL_OFFSET, yoffset, dst, dst_stride, msk, \ + msk_stride, sse, W, H, highbd_apply_filter_avg, \ + highbd_apply_filter, calc_var); \ + } else { \ + if (yoffset == HALF_PIXEL_OFFSET) \ + return aom_highbd_masked_subpel_varWxH_xnonzero_ynonzero( \ + src, src_stride, xoffset, HALF_PIXEL_OFFSET, dst, dst_stride, msk, \ + msk_stride, sse, W, H, highbd_apply_filter, \ + highbd_apply_filter_avg, calc_var); \ + else \ + return aom_highbd_masked_subpel_varWxH_xnonzero_ynonzero( \ + src, src_stride, xoffset, yoffset, dst, dst_stride, msk, \ + msk_stride, sse, W, H, highbd_apply_filter, highbd_apply_filter, \ + calc_var); \ + } \ + } + +// For W < 8 +#define HIGHBD_MASK_SUBPIX_VAR_SMALL(W, H) \ + unsigned int highbd_masked_sub_pixel_variance##W##x##H##_ssse3( \ + const uint8_t *src8, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst8, int dst_stride, const uint8_t *msk, int msk_stride, \ + unsigned int *sse, highbd_calc_masked_var_t calc_var, \ + highbd_variance_fn_t full_variance_function) { \ + uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ + assert(W == 4); \ + if (xoffset == 0 && yoffset == 0) \ + return full_variance_function(src8, src_stride, dst8, dst_stride, msk, \ + msk_stride, sse); \ + else if (xoffset == 0) \ + return aom_highbd_masked_subpel_var4xH_xzero( \ + src, src_stride, yoffset, dst, dst_stride, msk, msk_stride, sse, H, \ + calc_var); \ + else if (yoffset == 0) \ + return aom_highbd_masked_subpel_var4xH_yzero( \ + src, src_stride, xoffset, dst, dst_stride, msk, msk_stride, sse, H, \ + calc_var); \ + else \ + return aom_highbd_masked_subpel_var4xH_xnonzero_ynonzero( \ + src, src_stride, xoffset, yoffset, dst, dst_stride, msk, msk_stride, \ + sse, H, calc_var); \ + } + +#define HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(W, H) \ + unsigned int aom_highbd_masked_sub_pixel_variance##W##x##H##_ssse3( \ + const uint8_t *src8, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst8, int dst_stride, const uint8_t *msk, int msk_stride, \ + unsigned int *sse) { \ + return highbd_masked_sub_pixel_variance##W##x##H##_ssse3( \ + src8, src_stride, xoffset, yoffset, dst8, dst_stride, msk, msk_stride, \ + sse, calc_masked_variance, \ + aom_highbd_masked_variance##W##x##H##_ssse3); \ + } \ + unsigned int aom_highbd_10_masked_sub_pixel_variance##W##x##H##_ssse3( \ + const uint8_t *src8, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst8, int dst_stride, const uint8_t *msk, int msk_stride, \ + unsigned int *sse) { \ + return highbd_masked_sub_pixel_variance##W##x##H##_ssse3( \ + src8, src_stride, xoffset, yoffset, dst8, dst_stride, msk, msk_stride, \ + sse, highbd_10_calc_masked_variance, \ + aom_highbd_10_masked_variance##W##x##H##_ssse3); \ + } \ + unsigned int aom_highbd_12_masked_sub_pixel_variance##W##x##H##_ssse3( \ + const uint8_t *src8, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst8, int dst_stride, const uint8_t *msk, int msk_stride, \ + unsigned int *sse) { \ + return highbd_masked_sub_pixel_variance##W##x##H##_ssse3( \ + src8, src_stride, xoffset, yoffset, dst8, dst_stride, msk, msk_stride, \ + sse, highbd_12_calc_masked_variance, \ + aom_highbd_12_masked_variance##W##x##H##_ssse3); \ + } + +HIGHBD_MASK_SUBPIX_VAR_SMALL(4, 4) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(4, 4) +HIGHBD_MASK_SUBPIX_VAR_SMALL(4, 8) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(4, 8) +HIGHBD_MASK_SUBPIX_VAR_LARGE(8, 4) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(8, 4) +HIGHBD_MASK_SUBPIX_VAR_LARGE(8, 8) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(8, 8) +HIGHBD_MASK_SUBPIX_VAR_LARGE(8, 16) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(8, 16) +HIGHBD_MASK_SUBPIX_VAR_LARGE(16, 8) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(16, 8) +HIGHBD_MASK_SUBPIX_VAR_LARGE(16, 16) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(16, 16) +HIGHBD_MASK_SUBPIX_VAR_LARGE(16, 32) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(16, 32) +HIGHBD_MASK_SUBPIX_VAR_LARGE(32, 16) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(32, 16) +HIGHBD_MASK_SUBPIX_VAR_LARGE(32, 32) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(32, 32) +HIGHBD_MASK_SUBPIX_VAR_LARGE(32, 64) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(32, 64) +HIGHBD_MASK_SUBPIX_VAR_LARGE(64, 32) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(64, 32) +HIGHBD_MASK_SUBPIX_VAR_LARGE(64, 64) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(64, 64) +#if CONFIG_EXT_PARTITION +HIGHBD_MASK_SUBPIX_VAR_LARGE(64, 128) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(64, 128) +HIGHBD_MASK_SUBPIX_VAR_LARGE(128, 64) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(128, 64) +HIGHBD_MASK_SUBPIX_VAR_LARGE(128, 128) +HIGHBD_MASK_SUBPIX_VAR_WRAPPERS(128, 128) +#endif // CONFIG_EXT_PARTITION +#endif diff --git a/third_party/aom/aom_dsp/x86/obmc_sad_sse4.c b/third_party/aom/aom_dsp/x86/obmc_sad_sse4.c new file mode 100644 index 0000000000..ad77f974c7 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/obmc_sad_sse4.c @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "aom_ports/mem.h" +#include "aom/aom_integer.h" + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/x86/synonyms.h" + +//////////////////////////////////////////////////////////////////////////////// +// 8 bit +//////////////////////////////////////////////////////////////////////////////// + +static INLINE unsigned int obmc_sad_w4(const uint8_t *pre, const int pre_stride, + const int32_t *wsrc, const int32_t *mask, + const int height) { + const int pre_step = pre_stride - 4; + int n = 0; + __m128i v_sad_d = _mm_setzero_si128(); + + do { + const __m128i v_p_b = xx_loadl_32(pre + n); + const __m128i v_m_d = xx_load_128(mask + n); + const __m128i v_w_d = xx_load_128(wsrc + n); + + const __m128i v_p_d = _mm_cvtepu8_epi32(v_p_b); + + // Values in both pre and mask fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_pm_d = _mm_madd_epi16(v_p_d, v_m_d); + + const __m128i v_diff_d = _mm_sub_epi32(v_w_d, v_pm_d); + const __m128i v_absdiff_d = _mm_abs_epi32(v_diff_d); + + // Rounded absolute difference + const __m128i v_rad_d = xx_roundn_epu32(v_absdiff_d, 12); + + v_sad_d = _mm_add_epi32(v_sad_d, v_rad_d); + + n += 4; + + if (n % 4 == 0) pre += pre_step; + } while (n < 4 * height); + + return xx_hsum_epi32_si32(v_sad_d); +} + +static INLINE unsigned int obmc_sad_w8n(const uint8_t *pre, + const int pre_stride, + const int32_t *wsrc, + const int32_t *mask, const int width, + const int height) { + const int pre_step = pre_stride - width; + int n = 0; + __m128i v_sad_d = _mm_setzero_si128(); + + assert(width >= 8); + assert(IS_POWER_OF_TWO(width)); + + do { + const __m128i v_p1_b = xx_loadl_32(pre + n + 4); + const __m128i v_m1_d = xx_load_128(mask + n + 4); + const __m128i v_w1_d = xx_load_128(wsrc + n + 4); + const __m128i v_p0_b = xx_loadl_32(pre + n); + const __m128i v_m0_d = xx_load_128(mask + n); + const __m128i v_w0_d = xx_load_128(wsrc + n); + + const __m128i v_p0_d = _mm_cvtepu8_epi32(v_p0_b); + const __m128i v_p1_d = _mm_cvtepu8_epi32(v_p1_b); + + // Values in both pre and mask fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_pm0_d = _mm_madd_epi16(v_p0_d, v_m0_d); + const __m128i v_pm1_d = _mm_madd_epi16(v_p1_d, v_m1_d); + + const __m128i v_diff0_d = _mm_sub_epi32(v_w0_d, v_pm0_d); + const __m128i v_diff1_d = _mm_sub_epi32(v_w1_d, v_pm1_d); + const __m128i v_absdiff0_d = _mm_abs_epi32(v_diff0_d); + const __m128i v_absdiff1_d = _mm_abs_epi32(v_diff1_d); + + // Rounded absolute difference + const __m128i v_rad0_d = xx_roundn_epu32(v_absdiff0_d, 12); + const __m128i v_rad1_d = xx_roundn_epu32(v_absdiff1_d, 12); + + v_sad_d = _mm_add_epi32(v_sad_d, v_rad0_d); + v_sad_d = _mm_add_epi32(v_sad_d, v_rad1_d); + + n += 8; + + if (n % width == 0) pre += pre_step; + } while (n < width * height); + + return xx_hsum_epi32_si32(v_sad_d); +} + +#define OBMCSADWXH(w, h) \ + unsigned int aom_obmc_sad##w##x##h##_sse4_1( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *msk) { \ + if (w == 4) { \ + return obmc_sad_w4(pre, pre_stride, wsrc, msk, h); \ + } else { \ + return obmc_sad_w8n(pre, pre_stride, wsrc, msk, w, h); \ + } \ + } + +#if CONFIG_EXT_PARTITION +OBMCSADWXH(128, 128) +OBMCSADWXH(128, 64) +OBMCSADWXH(64, 128) +#endif // CONFIG_EXT_PARTITION +OBMCSADWXH(64, 64) +OBMCSADWXH(64, 32) +OBMCSADWXH(32, 64) +OBMCSADWXH(32, 32) +OBMCSADWXH(32, 16) +OBMCSADWXH(16, 32) +OBMCSADWXH(16, 16) +OBMCSADWXH(16, 8) +OBMCSADWXH(8, 16) +OBMCSADWXH(8, 8) +OBMCSADWXH(8, 4) +OBMCSADWXH(4, 8) +OBMCSADWXH(4, 4) + +//////////////////////////////////////////////////////////////////////////////// +// High bit-depth +//////////////////////////////////////////////////////////////////////////////// + +#if CONFIG_HIGHBITDEPTH +static INLINE unsigned int hbd_obmc_sad_w4(const uint8_t *pre8, + const int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + const int height) { + const uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); + const int pre_step = pre_stride - 4; + int n = 0; + __m128i v_sad_d = _mm_setzero_si128(); + + do { + const __m128i v_p_w = xx_loadl_64(pre + n); + const __m128i v_m_d = xx_load_128(mask + n); + const __m128i v_w_d = xx_load_128(wsrc + n); + + const __m128i v_p_d = _mm_cvtepu16_epi32(v_p_w); + + // Values in both pre and mask fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_pm_d = _mm_madd_epi16(v_p_d, v_m_d); + + const __m128i v_diff_d = _mm_sub_epi32(v_w_d, v_pm_d); + const __m128i v_absdiff_d = _mm_abs_epi32(v_diff_d); + + // Rounded absolute difference + const __m128i v_rad_d = xx_roundn_epu32(v_absdiff_d, 12); + + v_sad_d = _mm_add_epi32(v_sad_d, v_rad_d); + + n += 4; + + if (n % 4 == 0) pre += pre_step; + } while (n < 4 * height); + + return xx_hsum_epi32_si32(v_sad_d); +} + +static INLINE unsigned int hbd_obmc_sad_w8n(const uint8_t *pre8, + const int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + const int width, const int height) { + const uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); + const int pre_step = pre_stride - width; + int n = 0; + __m128i v_sad_d = _mm_setzero_si128(); + + assert(width >= 8); + assert(IS_POWER_OF_TWO(width)); + + do { + const __m128i v_p1_w = xx_loadl_64(pre + n + 4); + const __m128i v_m1_d = xx_load_128(mask + n + 4); + const __m128i v_w1_d = xx_load_128(wsrc + n + 4); + const __m128i v_p0_w = xx_loadl_64(pre + n); + const __m128i v_m0_d = xx_load_128(mask + n); + const __m128i v_w0_d = xx_load_128(wsrc + n); + + const __m128i v_p0_d = _mm_cvtepu16_epi32(v_p0_w); + const __m128i v_p1_d = _mm_cvtepu16_epi32(v_p1_w); + + // Values in both pre and mask fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_pm0_d = _mm_madd_epi16(v_p0_d, v_m0_d); + const __m128i v_pm1_d = _mm_madd_epi16(v_p1_d, v_m1_d); + + const __m128i v_diff0_d = _mm_sub_epi32(v_w0_d, v_pm0_d); + const __m128i v_diff1_d = _mm_sub_epi32(v_w1_d, v_pm1_d); + const __m128i v_absdiff0_d = _mm_abs_epi32(v_diff0_d); + const __m128i v_absdiff1_d = _mm_abs_epi32(v_diff1_d); + + // Rounded absolute difference + const __m128i v_rad0_d = xx_roundn_epu32(v_absdiff0_d, 12); + const __m128i v_rad1_d = xx_roundn_epu32(v_absdiff1_d, 12); + + v_sad_d = _mm_add_epi32(v_sad_d, v_rad0_d); + v_sad_d = _mm_add_epi32(v_sad_d, v_rad1_d); + + n += 8; + + if (n % width == 0) pre += pre_step; + } while (n < width * height); + + return xx_hsum_epi32_si32(v_sad_d); +} + +#define HBD_OBMCSADWXH(w, h) \ + unsigned int aom_highbd_obmc_sad##w##x##h##_sse4_1( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask) { \ + if (w == 4) { \ + return hbd_obmc_sad_w4(pre, pre_stride, wsrc, mask, h); \ + } else { \ + return hbd_obmc_sad_w8n(pre, pre_stride, wsrc, mask, w, h); \ + } \ + } + +#if CONFIG_EXT_PARTITION +HBD_OBMCSADWXH(128, 128) +HBD_OBMCSADWXH(128, 64) +HBD_OBMCSADWXH(64, 128) +#endif // CONFIG_EXT_PARTITION +HBD_OBMCSADWXH(64, 64) +HBD_OBMCSADWXH(64, 32) +HBD_OBMCSADWXH(32, 64) +HBD_OBMCSADWXH(32, 32) +HBD_OBMCSADWXH(32, 16) +HBD_OBMCSADWXH(16, 32) +HBD_OBMCSADWXH(16, 16) +HBD_OBMCSADWXH(16, 8) +HBD_OBMCSADWXH(8, 16) +HBD_OBMCSADWXH(8, 8) +HBD_OBMCSADWXH(8, 4) +HBD_OBMCSADWXH(4, 8) +HBD_OBMCSADWXH(4, 4) +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/x86/obmc_variance_sse4.c b/third_party/aom/aom_dsp/x86/obmc_variance_sse4.c new file mode 100644 index 0000000000..efb3659cfc --- /dev/null +++ b/third_party/aom/aom_dsp/x86/obmc_variance_sse4.c @@ -0,0 +1,355 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "aom_ports/mem.h" +#include "aom/aom_integer.h" + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/x86/synonyms.h" +#include "aom_dsp/aom_filter.h" + +//////////////////////////////////////////////////////////////////////////////// +// 8 bit +//////////////////////////////////////////////////////////////////////////////// + +static INLINE void obmc_variance_w4(const uint8_t *pre, const int pre_stride, + const int32_t *wsrc, const int32_t *mask, + unsigned int *const sse, int *const sum, + const int h) { + const int pre_step = pre_stride - 4; + int n = 0; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_d = _mm_setzero_si128(); + + assert(IS_POWER_OF_TWO(h)); + + do { + const __m128i v_p_b = xx_loadl_32(pre + n); + const __m128i v_m_d = xx_load_128(mask + n); + const __m128i v_w_d = xx_load_128(wsrc + n); + + const __m128i v_p_d = _mm_cvtepu8_epi32(v_p_b); + + // Values in both pre and mask fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_pm_d = _mm_madd_epi16(v_p_d, v_m_d); + + const __m128i v_diff_d = _mm_sub_epi32(v_w_d, v_pm_d); + const __m128i v_rdiff_d = xx_roundn_epi32(v_diff_d, 12); + const __m128i v_sqrdiff_d = _mm_mullo_epi32(v_rdiff_d, v_rdiff_d); + + v_sum_d = _mm_add_epi32(v_sum_d, v_rdiff_d); + v_sse_d = _mm_add_epi32(v_sse_d, v_sqrdiff_d); + + n += 4; + + if (n % 4 == 0) pre += pre_step; + } while (n < 4 * h); + + *sum = xx_hsum_epi32_si32(v_sum_d); + *sse = xx_hsum_epi32_si32(v_sse_d); +} + +static INLINE void obmc_variance_w8n(const uint8_t *pre, const int pre_stride, + const int32_t *wsrc, const int32_t *mask, + unsigned int *const sse, int *const sum, + const int w, const int h) { + const int pre_step = pre_stride - w; + int n = 0; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_d = _mm_setzero_si128(); + + assert(w >= 8); + assert(IS_POWER_OF_TWO(w)); + assert(IS_POWER_OF_TWO(h)); + + do { + const __m128i v_p1_b = xx_loadl_32(pre + n + 4); + const __m128i v_m1_d = xx_load_128(mask + n + 4); + const __m128i v_w1_d = xx_load_128(wsrc + n + 4); + const __m128i v_p0_b = xx_loadl_32(pre + n); + const __m128i v_m0_d = xx_load_128(mask + n); + const __m128i v_w0_d = xx_load_128(wsrc + n); + + const __m128i v_p0_d = _mm_cvtepu8_epi32(v_p0_b); + const __m128i v_p1_d = _mm_cvtepu8_epi32(v_p1_b); + + // Values in both pre and mask fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_pm0_d = _mm_madd_epi16(v_p0_d, v_m0_d); + const __m128i v_pm1_d = _mm_madd_epi16(v_p1_d, v_m1_d); + + const __m128i v_diff0_d = _mm_sub_epi32(v_w0_d, v_pm0_d); + const __m128i v_diff1_d = _mm_sub_epi32(v_w1_d, v_pm1_d); + + const __m128i v_rdiff0_d = xx_roundn_epi32(v_diff0_d, 12); + const __m128i v_rdiff1_d = xx_roundn_epi32(v_diff1_d, 12); + const __m128i v_rdiff01_w = _mm_packs_epi32(v_rdiff0_d, v_rdiff1_d); + const __m128i v_sqrdiff_d = _mm_madd_epi16(v_rdiff01_w, v_rdiff01_w); + + v_sum_d = _mm_add_epi32(v_sum_d, v_rdiff0_d); + v_sum_d = _mm_add_epi32(v_sum_d, v_rdiff1_d); + v_sse_d = _mm_add_epi32(v_sse_d, v_sqrdiff_d); + + n += 8; + + if (n % w == 0) pre += pre_step; + } while (n < w * h); + + *sum = xx_hsum_epi32_si32(v_sum_d); + *sse = xx_hsum_epi32_si32(v_sse_d); +} + +#define OBMCVARWXH(W, H) \ + unsigned int aom_obmc_variance##W##x##H##_sse4_1( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask, unsigned int *sse) { \ + int sum; \ + if (W == 4) { \ + obmc_variance_w4(pre, pre_stride, wsrc, mask, sse, &sum, H); \ + } else { \ + obmc_variance_w8n(pre, pre_stride, wsrc, mask, sse, &sum, W, H); \ + } \ + return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \ + } + +#if CONFIG_EXT_PARTITION +OBMCVARWXH(128, 128) +OBMCVARWXH(128, 64) +OBMCVARWXH(64, 128) +#endif // CONFIG_EXT_PARTITION +OBMCVARWXH(64, 64) +OBMCVARWXH(64, 32) +OBMCVARWXH(32, 64) +OBMCVARWXH(32, 32) +OBMCVARWXH(32, 16) +OBMCVARWXH(16, 32) +OBMCVARWXH(16, 16) +OBMCVARWXH(16, 8) +OBMCVARWXH(8, 16) +OBMCVARWXH(8, 8) +OBMCVARWXH(8, 4) +OBMCVARWXH(4, 8) +OBMCVARWXH(4, 4) + +//////////////////////////////////////////////////////////////////////////////// +// High bit-depth +//////////////////////////////////////////////////////////////////////////////// + +#if CONFIG_HIGHBITDEPTH +static INLINE void hbd_obmc_variance_w4( + const uint8_t *pre8, const int pre_stride, const int32_t *wsrc, + const int32_t *mask, uint64_t *const sse, int64_t *const sum, const int h) { + const uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); + const int pre_step = pre_stride - 4; + int n = 0; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_d = _mm_setzero_si128(); + + assert(IS_POWER_OF_TWO(h)); + + do { + const __m128i v_p_w = xx_loadl_64(pre + n); + const __m128i v_m_d = xx_load_128(mask + n); + const __m128i v_w_d = xx_load_128(wsrc + n); + + const __m128i v_p_d = _mm_cvtepu16_epi32(v_p_w); + + // Values in both pre and mask fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_pm_d = _mm_madd_epi16(v_p_d, v_m_d); + + const __m128i v_diff_d = _mm_sub_epi32(v_w_d, v_pm_d); + const __m128i v_rdiff_d = xx_roundn_epi32(v_diff_d, 12); + const __m128i v_sqrdiff_d = _mm_mullo_epi32(v_rdiff_d, v_rdiff_d); + + v_sum_d = _mm_add_epi32(v_sum_d, v_rdiff_d); + v_sse_d = _mm_add_epi32(v_sse_d, v_sqrdiff_d); + + n += 4; + + if (n % 4 == 0) pre += pre_step; + } while (n < 4 * h); + + *sum = xx_hsum_epi32_si32(v_sum_d); + *sse = xx_hsum_epi32_si32(v_sse_d); +} + +static INLINE void hbd_obmc_variance_w8n( + const uint8_t *pre8, const int pre_stride, const int32_t *wsrc, + const int32_t *mask, uint64_t *const sse, int64_t *const sum, const int w, + const int h) { + const uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); + const int pre_step = pre_stride - w; + int n = 0; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_d = _mm_setzero_si128(); + + assert(w >= 8); + assert(IS_POWER_OF_TWO(w)); + assert(IS_POWER_OF_TWO(h)); + + do { + const __m128i v_p1_w = xx_loadl_64(pre + n + 4); + const __m128i v_m1_d = xx_load_128(mask + n + 4); + const __m128i v_w1_d = xx_load_128(wsrc + n + 4); + const __m128i v_p0_w = xx_loadl_64(pre + n); + const __m128i v_m0_d = xx_load_128(mask + n); + const __m128i v_w0_d = xx_load_128(wsrc + n); + + const __m128i v_p0_d = _mm_cvtepu16_epi32(v_p0_w); + const __m128i v_p1_d = _mm_cvtepu16_epi32(v_p1_w); + + // Values in both pre and mask fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_pm0_d = _mm_madd_epi16(v_p0_d, v_m0_d); + const __m128i v_pm1_d = _mm_madd_epi16(v_p1_d, v_m1_d); + + const __m128i v_diff0_d = _mm_sub_epi32(v_w0_d, v_pm0_d); + const __m128i v_diff1_d = _mm_sub_epi32(v_w1_d, v_pm1_d); + + const __m128i v_rdiff0_d = xx_roundn_epi32(v_diff0_d, 12); + const __m128i v_rdiff1_d = xx_roundn_epi32(v_diff1_d, 12); + const __m128i v_rdiff01_w = _mm_packs_epi32(v_rdiff0_d, v_rdiff1_d); + const __m128i v_sqrdiff_d = _mm_madd_epi16(v_rdiff01_w, v_rdiff01_w); + + v_sum_d = _mm_add_epi32(v_sum_d, v_rdiff0_d); + v_sum_d = _mm_add_epi32(v_sum_d, v_rdiff1_d); + v_sse_d = _mm_add_epi32(v_sse_d, v_sqrdiff_d); + + n += 8; + + if (n % w == 0) pre += pre_step; + } while (n < w * h); + + *sum += xx_hsum_epi32_si64(v_sum_d); + *sse += xx_hsum_epi32_si64(v_sse_d); +} + +static INLINE void highbd_obmc_variance(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, int w, int h, + unsigned int *sse, int *sum) { + int64_t sum64 = 0; + uint64_t sse64 = 0; + if (w == 4) { + hbd_obmc_variance_w4(pre8, pre_stride, wsrc, mask, &sse64, &sum64, h); + } else { + hbd_obmc_variance_w8n(pre8, pre_stride, wsrc, mask, &sse64, &sum64, w, h); + } + *sum = (int)sum64; + *sse = (unsigned int)sse64; +} + +static INLINE void highbd_10_obmc_variance(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, int w, int h, + unsigned int *sse, int *sum) { + int64_t sum64 = 0; + uint64_t sse64 = 0; + if (w == 4) { + hbd_obmc_variance_w4(pre8, pre_stride, wsrc, mask, &sse64, &sum64, h); + } else { + hbd_obmc_variance_w8n(pre8, pre_stride, wsrc, mask, &sse64, &sum64, w, h); + } + *sum = (int)ROUND_POWER_OF_TWO(sum64, 2); + *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 4); +} + +static INLINE void highbd_12_obmc_variance(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, int w, int h, + unsigned int *sse, int *sum) { + int64_t sum64 = 0; + uint64_t sse64 = 0; + if (w == 128) { + do { + hbd_obmc_variance_w8n(pre8, pre_stride, wsrc, mask, &sse64, &sum64, 128, + 32); + pre8 += 32 * pre_stride; + wsrc += 32 * 128; + mask += 32 * 128; + h -= 32; + } while (h > 0); + } else if (w == 64 && h >= 128) { + do { + hbd_obmc_variance_w8n(pre8, pre_stride, wsrc, mask, &sse64, &sum64, 64, + 64); + pre8 += 64 * pre_stride; + wsrc += 64 * 64; + mask += 64 * 64; + h -= 64; + } while (h > 0); + } else if (w == 4) { + hbd_obmc_variance_w4(pre8, pre_stride, wsrc, mask, &sse64, &sum64, h); + } else { + hbd_obmc_variance_w8n(pre8, pre_stride, wsrc, mask, &sse64, &sum64, w, h); + } + *sum = (int)ROUND_POWER_OF_TWO(sum64, 4); + *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 8); +} + +#define HBD_OBMCVARWXH(W, H) \ + unsigned int aom_highbd_obmc_variance##W##x##H##_sse4_1( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask, unsigned int *sse) { \ + int sum; \ + highbd_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ + return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \ + } \ + \ + unsigned int aom_highbd_10_obmc_variance##W##x##H##_sse4_1( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask, unsigned int *sse) { \ + int sum; \ + int64_t var; \ + highbd_10_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ + var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } \ + \ + unsigned int aom_highbd_12_obmc_variance##W##x##H##_sse4_1( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask, unsigned int *sse) { \ + int sum; \ + int64_t var; \ + highbd_12_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ + var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \ + return (var >= 0) ? (uint32_t)var : 0; \ + } + +#if CONFIG_EXT_PARTITION +HBD_OBMCVARWXH(128, 128) +HBD_OBMCVARWXH(128, 64) +HBD_OBMCVARWXH(64, 128) +#endif // CONFIG_EXT_PARTITION +HBD_OBMCVARWXH(64, 64) +HBD_OBMCVARWXH(64, 32) +HBD_OBMCVARWXH(32, 64) +HBD_OBMCVARWXH(32, 32) +HBD_OBMCVARWXH(32, 16) +HBD_OBMCVARWXH(16, 32) +HBD_OBMCVARWXH(16, 16) +HBD_OBMCVARWXH(16, 8) +HBD_OBMCVARWXH(8, 16) +HBD_OBMCVARWXH(8, 8) +HBD_OBMCVARWXH(8, 4) +HBD_OBMCVARWXH(4, 8) +HBD_OBMCVARWXH(4, 4) +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_dsp/x86/quantize_avx_x86_64.asm b/third_party/aom/aom_dsp/x86/quantize_avx_x86_64.asm new file mode 100644 index 0000000000..954a95b983 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/quantize_avx_x86_64.asm @@ -0,0 +1,547 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION .text + +%macro QUANTIZE_FN 2 +cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, skip, zbin, round, quant, \ + shift, qcoeff, dqcoeff, dequant, \ + eob, scan, iscan + + vzeroupper + + ; If we can skip this block, then just zero the output + cmp skipmp, 0 + jne .blank + +%ifnidn %1, b_32x32 + + ; Special case for ncoeff == 16, as it is frequent and we can save on + ; not setting up a loop. + cmp ncoeffmp, 16 + jne .generic + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Special case of ncoeff == 16 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.single: + + movifnidn coeffq, coeffmp + movifnidn zbinq, zbinmp + mova m0, [zbinq] ; m0 = zbin + + ; Get DC and first 15 AC coeffs - in this special case, that is all. +%if CONFIG_HIGHBITDEPTH + ; coeff stored as 32bit numbers but we process them as 16 bit numbers + mova m9, [coeffq] + packssdw m9, [coeffq+16] ; m9 = c[i] + mova m10, [coeffq+32] + packssdw m10, [coeffq+48] ; m10 = c[i] +%else + mova m9, [coeffq] ; m9 = c[i] + mova m10, [coeffq+16] ; m10 = c[i] +%endif + + mov r0, eobmp ; Output pointer + mov r1, qcoeffmp ; Output pointer + mov r2, dqcoeffmp ; Output pointer + + pxor m5, m5 ; m5 = dedicated zero + + pcmpeqw m4, m4 ; All word lanes -1 + paddw m0, m4 ; m0 = zbin - 1 + + pabsw m6, m9 ; m6 = abs(m9) + pabsw m11, m10 ; m11 = abs(m10) + pcmpgtw m7, m6, m0 ; m7 = c[i] >= zbin + punpckhqdq m0, m0 + pcmpgtw m12, m11, m0 ; m12 = c[i] >= zbin + + ; Check if all coeffs are less than zbin. If yes, we just write zeros + ; to the outputs and we are done. + por m14, m7, m12 + ptest m14, m14 + jnz .single_nonzero + +%if CONFIG_HIGHBITDEPTH + mova [r1 ], ymm5 + mova [r1+32], ymm5 + mova [r2 ], ymm5 + mova [r2+32], ymm5 +%else + mova [r1], ymm5 + mova [r2], ymm5 +%endif + mov [r0], word 0 + + vzeroupper + RET + +.single_nonzero: + + ; Actual quantization of size 16 block - setup pointers, rounders, etc. + movifnidn r4, roundmp + movifnidn r5, quantmp + mov r3, dequantmp + mov r6, shiftmp + mova m1, [r4] ; m1 = round + mova m2, [r5] ; m2 = quant + mova m3, [r3] ; m3 = dequant + mova m4, [r6] ; m4 = shift + + mov r3, iscanmp + + DEFINE_ARGS eob, qcoeff, dqcoeff, iscan + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + paddsw m6, m1 ; m6 += round + punpckhqdq m1, m1 + paddsw m11, m1 ; m11 += round + pmulhw m8, m6, m2 ; m8 = m6*q>>16 + punpckhqdq m2, m2 + pmulhw m13, m11, m2 ; m13 = m11*q>>16 + paddw m8, m6 ; m8 += m6 + paddw m13, m11 ; m13 += m11 + pmulhw m8, m4 ; m8 = m8*qsh>>16 + punpckhqdq m4, m4 + pmulhw m13, m4 ; m13 = m13*qsh>>16 + psignw m8, m9 ; m8 = reinsert sign + psignw m13, m10 ; m13 = reinsert sign + pand m8, m7 + pand m13, m12 + +%if CONFIG_HIGHBITDEPTH + ; Store 16bit numbers as 32bit numbers in array pointed to by qcoeff + pcmpgtw m6, m5, m8 + punpckhwd m6, m8, m6 + pmovsxwd m11, m8 + mova [qcoeffq ], m11 + mova [qcoeffq+16], m6 + pcmpgtw m6, m5, m13 + punpckhwd m6, m13, m6 + pmovsxwd m11, m13 + mova [qcoeffq+32], m11 + mova [qcoeffq+48], m6 +%else + mova [qcoeffq ], m8 + mova [qcoeffq+16], m13 +%endif + + pmullw m8, m3 ; dqc[i] = qc[i] * q + punpckhqdq m3, m3 + pmullw m13, m3 ; dqc[i] = qc[i] * q + +%if CONFIG_HIGHBITDEPTH + ; Store 16bit numbers as 32bit numbers in array pointed to by qcoeff + pcmpgtw m6, m5, m8 + punpckhwd m6, m8, m6 + pmovsxwd m11, m8 + mova [dqcoeffq ], m11 + mova [dqcoeffq+16], m6 + pcmpgtw m6, m5, m13 + punpckhwd m6, m13, m6 + pmovsxwd m11, m13 + mova [dqcoeffq+32], m11 + mova [dqcoeffq+48], m6 +%else + mova [dqcoeffq ], m8 + mova [dqcoeffq+16], m13 +%endif + + mova m6, [iscanq] ; m6 = scan[i] + mova m11, [iscanq+16] ; m11 = scan[i] + + pcmpeqw m8, m8, m5 ; m8 = c[i] == 0 + pcmpeqw m13, m13, m5 ; m13 = c[i] == 0 + psubw m6, m6, m7 ; m6 = scan[i] + 1 + psubw m11, m11, m12 ; m11 = scan[i] + 1 + pandn m8, m8, m6 ; m8 = max(eob) + pandn m13, m13, m11 ; m13 = max(eob) + pmaxsw m8, m8, m13 + + ; Horizontally accumulate/max eobs and write into [eob] memory pointer + pshufd m7, m8, 0xe + pmaxsw m8, m7 + pshuflw m7, m8, 0xe + pmaxsw m8, m7 + pshuflw m7, m8, 0x1 + pmaxsw m8, m7 + movq rax, m8 + mov [eobq], ax + + vzeroupper + RET + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Generic case of ncoeff != 16 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.generic: + +%endif ; %ifnidn %1, b_32x32 + +DEFINE_ARGS coeff, ncoeff, skip, zbin, round, quant, shift, \ + qcoeff, dqcoeff, dequant, eob, scan, iscan + + ; Actual quantization loop - setup pointers, rounders, etc. + movifnidn coeffq, coeffmp + movifnidn ncoeffq, ncoeffmp + mov r2, dequantmp + movifnidn zbinq, zbinmp + movifnidn roundq, roundmp + movifnidn quantq, quantmp + mova m0, [zbinq] ; m0 = zbin + mova m1, [roundq] ; m1 = round + mova m2, [quantq] ; m2 = quant + mova m3, [r2] ; m3 = dequant + pcmpeqw m4, m4 ; All lanes -1 +%ifidn %1, b_32x32 + psubw m0, m4 + psubw m1, m4 + psrlw m0, 1 ; m0 = (m0 + 1) / 2 + psrlw m1, 1 ; m1 = (m1 + 1) / 2 +%endif + paddw m0, m4 ; m0 = m0 + 1 + + mov r2, shiftmp + mov r3, qcoeffmp + mova m4, [r2] ; m4 = shift + mov r4, dqcoeffmp + mov r5, iscanmp +%ifidn %1, b_32x32 + psllw m4, 1 +%endif + pxor m5, m5 ; m5 = dedicated zero + + DEFINE_ARGS coeff, ncoeff, d1, qcoeff, dqcoeff, iscan, d2, d3, d4, d5, eob + +%if CONFIG_HIGHBITDEPTH + lea coeffq, [ coeffq+ncoeffq*4] + lea qcoeffq, [ qcoeffq+ncoeffq*4] + lea dqcoeffq, [dqcoeffq+ncoeffq*4] +%else + lea coeffq, [ coeffq+ncoeffq*2] + lea qcoeffq, [ qcoeffq+ncoeffq*2] + lea dqcoeffq, [dqcoeffq+ncoeffq*2] +%endif + lea iscanq, [ iscanq+ncoeffq*2] + neg ncoeffq + + ; get DC and first 15 AC coeffs +%if CONFIG_HIGHBITDEPTH + ; coeff stored as 32bit numbers & require 16bit numbers + mova m9, [coeffq+ncoeffq*4+ 0] + packssdw m9, [coeffq+ncoeffq*4+16] + mova m10, [coeffq+ncoeffq*4+32] + packssdw m10, [coeffq+ncoeffq*4+48] +%else + mova m9, [coeffq+ncoeffq*2+ 0] ; m9 = c[i] + mova m10, [coeffq+ncoeffq*2+16] ; m10 = c[i] +%endif + + pabsw m6, m9 ; m6 = abs(m9) + pabsw m11, m10 ; m11 = abs(m10) + pcmpgtw m7, m6, m0 ; m7 = c[i] >= zbin + punpckhqdq m0, m0 + pcmpgtw m12, m11, m0 ; m12 = c[i] >= zbin + + ; Check if all coeffs are less than zbin. If yes, skip forward quickly. + por m14, m7, m12 + ptest m14, m14 + jnz .first_nonzero + +%if CONFIG_HIGHBITDEPTH + mova [qcoeffq+ncoeffq*4 ], ymm5 + mova [qcoeffq+ncoeffq*4+32], ymm5 + mova [dqcoeffq+ncoeffq*4 ], ymm5 + mova [dqcoeffq+ncoeffq*4+32], ymm5 +%else + mova [qcoeffq+ncoeffq*2], ymm5 + mova [dqcoeffq+ncoeffq*2], ymm5 +%endif + + add ncoeffq, mmsize + + punpckhqdq m1, m1 + punpckhqdq m2, m2 + punpckhqdq m3, m3 + punpckhqdq m4, m4 + pxor m8, m8 + + jmp .ac_only_loop + +.first_nonzero: + + paddsw m6, m1 ; m6 += round + punpckhqdq m1, m1 + paddsw m11, m1 ; m11 += round + pmulhw m8, m6, m2 ; m8 = m6*q>>16 + punpckhqdq m2, m2 + pmulhw m13, m11, m2 ; m13 = m11*q>>16 + paddw m8, m6 ; m8 += m6 + paddw m13, m11 ; m13 += m11 + pmulhw m8, m4 ; m8 = m8*qsh>>16 + punpckhqdq m4, m4 + pmulhw m13, m4 ; m13 = m13*qsh>>16 + psignw m8, m9 ; m8 = reinsert sign + psignw m13, m10 ; m13 = reinsert sign + pand m8, m7 + pand m13, m12 + +%if CONFIG_HIGHBITDEPTH + ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff + pcmpgtw m6, m5, m8 + punpckhwd m6, m8, m6 + pmovsxwd m11, m8 + mova [qcoeffq+ncoeffq*4+ 0], m11 + mova [qcoeffq+ncoeffq*4+16], m6 + pcmpgtw m6, m5, m13 + punpckhwd m6, m13, m6 + pmovsxwd m11, m13 + mova [qcoeffq+ncoeffq*4+32], m11 + mova [qcoeffq+ncoeffq*4+48], m6 +%else + mova [qcoeffq+ncoeffq*2+ 0], m8 + mova [qcoeffq+ncoeffq*2+16], m13 +%endif + +%ifidn %1, b_32x32 + pabsw m8, m8 + pabsw m13, m13 +%endif + pmullw m8, m3 ; dqc[i] = qc[i] * q + punpckhqdq m3, m3 + pmullw m13, m3 ; dqc[i] = qc[i] * q +%ifidn %1, b_32x32 + psrlw m8, 1 + psrlw m13, 1 + psignw m8, m9 + psignw m13, m10 +%endif + +%if CONFIG_HIGHBITDEPTH + ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff + pcmpgtw m6, m5, m8 + punpckhwd m6, m8, m6 + pmovsxwd m11, m8 + mova [dqcoeffq+ncoeffq*4+ 0], m11 + mova [dqcoeffq+ncoeffq*4+16], m6 + pcmpgtw m6, m5, m13 + punpckhwd m6, m13, m6 + pmovsxwd m11, m13 + mova [dqcoeffq+ncoeffq*4+32], m11 + mova [dqcoeffq+ncoeffq*4+48], m6 +%else + mova [dqcoeffq+ncoeffq*2+ 0], m8 + mova [dqcoeffq+ncoeffq*2+16], m13 +%endif + + pcmpeqw m8, m5 ; m8 = c[i] == 0 + pcmpeqw m13, m5 ; m13 = c[i] == 0 + mova m6, [iscanq+ncoeffq*2] ; m6 = scan[i] + mova m11, [iscanq+ncoeffq*2+16] ; m11 = scan[i] + psubw m6, m7 ; m6 = scan[i] + 1 + psubw m11, m12 ; m11 = scan[i] + 1 + pandn m8, m6 ; m8 = max(eob) + pandn m13, m11 ; m13 = max(eob) + pmaxsw m8, m13 + add ncoeffq, mmsize + +.ac_only_loop: + +%if CONFIG_HIGHBITDEPTH + ; pack coeff from 32bit to 16bit array + mova m9, [coeffq+ncoeffq*4+ 0] + packssdw m9, [coeffq+ncoeffq*4+16] + mova m10, [coeffq+ncoeffq*4+32] + packssdw m10, [coeffq+ncoeffq*4+48] +%else + mova m9, [coeffq+ncoeffq*2+ 0] ; m9 = c[i] + mova m10, [coeffq+ncoeffq*2+16] ; m10 = c[i] +%endif + + pabsw m6, m9 ; m6 = abs(m9) + pabsw m11, m10 ; m11 = abs(m10) + pcmpgtw m7, m6, m0 ; m7 = c[i] >= zbin + pcmpgtw m12, m11, m0 ; m12 = c[i] >= zbin + + ; Check if all coeffs are less than zbin. If yes, skip this itertion. + ; And just write zeros as the result would be. + por m14, m7, m12 + ptest m14, m14 + jnz .rest_nonzero + +%if CONFIG_HIGHBITDEPTH + mova [qcoeffq+ncoeffq*4+ 0], ymm5 + mova [qcoeffq+ncoeffq*4+32], ymm5 + mova [dqcoeffq+ncoeffq*4+ 0], ymm5 + mova [dqcoeffq+ncoeffq*4+32], ymm5 +%else + mova [qcoeffq+ncoeffq*2+ 0], ymm5 + mova [dqcoeffq+ncoeffq*2+ 0], ymm5 +%endif + add ncoeffq, mmsize + jnz .ac_only_loop + + ; Horizontally accumulate/max eobs and write into [eob] memory pointer + mov r2, eobmp + pshufd m7, m8, 0xe + pmaxsw m8, m7 + pshuflw m7, m8, 0xe + pmaxsw m8, m7 + pshuflw m7, m8, 0x1 + pmaxsw m8, m7 + movq rax, m8 + mov [r2], ax + vzeroupper + RET + +.rest_nonzero: + paddsw m6, m1 ; m6 += round + paddsw m11, m1 ; m11 += round + pmulhw m14, m6, m2 ; m14 = m6*q>>16 + pmulhw m13, m11, m2 ; m13 = m11*q>>16 + paddw m14, m6 ; m14 += m6 + paddw m13, m11 ; m13 += m11 + pmulhw m14, m4 ; m14 = m14*qsh>>16 + pmulhw m13, m4 ; m13 = m13*qsh>>16 + psignw m14, m9 ; m14 = reinsert sign + psignw m13, m10 ; m13 = reinsert sign + pand m14, m7 + pand m13, m12 + +%if CONFIG_HIGHBITDEPTH + ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff + pcmpgtw m6, m5, m14 + punpckhwd m6, m14, m6 + pmovsxwd m11, m14 + mova [qcoeffq+ncoeffq*4+ 0], m11 + mova [qcoeffq+ncoeffq*4+16], m6 + pcmpgtw m6, m5, m13 + punpckhwd m6, m13, m6 + pmovsxwd m11, m13 + mova [qcoeffq+ncoeffq*4+32], m11 + mova [qcoeffq+ncoeffq*4+48], m6 +%else + mova [qcoeffq+ncoeffq*2+ 0], m14 + mova [qcoeffq+ncoeffq*2+16], m13 +%endif + +%ifidn %1, b_32x32 + pabsw m14, m14 + pabsw m13, m13 +%endif + pmullw m14, m3 ; dqc[i] = qc[i] * q + pmullw m13, m3 ; dqc[i] = qc[i] * q +%ifidn %1, b_32x32 + psrlw m14, 1 + psrlw m13, 1 + psignw m14, m9 + psignw m13, m10 +%endif + +%if CONFIG_HIGHBITDEPTH + ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff + pcmpgtw m6, m5, m14 + punpckhwd m6, m14, m6 + pmovsxwd m11, m14 + mova [dqcoeffq+ncoeffq*4+ 0], m11 + mova [dqcoeffq+ncoeffq*4+16], m6 + pcmpgtw m6, m5, m13 + punpckhwd m6, m13, m6 + pmovsxwd m11, m13 + mova [dqcoeffq+ncoeffq*4+32], m11 + mova [dqcoeffq+ncoeffq*4+48], m6 +%else + mova [dqcoeffq+ncoeffq*2+ 0], m14 + mova [dqcoeffq+ncoeffq*2+16], m13 +%endif + + pcmpeqw m14, m5 ; m14 = c[i] == 0 + pcmpeqw m13, m5 ; m13 = c[i] == 0 + mova m6, [iscanq+ncoeffq*2+ 0] ; m6 = scan[i] + mova m11, [iscanq+ncoeffq*2+16] ; m11 = scan[i] + psubw m6, m7 ; m6 = scan[i] + 1 + psubw m11, m12 ; m11 = scan[i] + 1 + pandn m14, m6 ; m14 = max(eob) + pandn m13, m11 ; m13 = max(eob) + pmaxsw m8, m14 + pmaxsw m8, m13 + add ncoeffq, mmsize + jnz .ac_only_loop + + ; Horizontally accumulate/max eobs and write into [eob] memory pointer + mov r2, eobmp + pshufd m7, m8, 0xe + pmaxsw m8, m7 + pshuflw m7, m8, 0xe + pmaxsw m8, m7 + pshuflw m7, m8, 0x1 + pmaxsw m8, m7 + movq rax, m8 + mov [r2], ax + vzeroupper + RET + + ; Skip-block, i.e. just write all zeroes +.blank: + +DEFINE_ARGS coeff, ncoeff, skip, zbin, round, quant, shift, \ + qcoeff, dqcoeff, dequant, eob, scan, iscan + + mov r0, dqcoeffmp + movifnidn ncoeffq, ncoeffmp + mov r2, qcoeffmp + mov r3, eobmp + +DEFINE_ARGS dqcoeff, ncoeff, qcoeff, eob + +%if CONFIG_HIGHBITDEPTH + lea dqcoeffq, [dqcoeffq+ncoeffq*4] + lea qcoeffq, [ qcoeffq+ncoeffq*4] +%else + lea dqcoeffq, [dqcoeffq+ncoeffq*2] + lea qcoeffq, [ qcoeffq+ncoeffq*2] +%endif + + neg ncoeffq + pxor m7, m7 + +.blank_loop: +%if CONFIG_HIGHBITDEPTH + mova [dqcoeffq+ncoeffq*4+ 0], ymm7 + mova [dqcoeffq+ncoeffq*4+32], ymm7 + mova [qcoeffq+ncoeffq*4+ 0], ymm7 + mova [qcoeffq+ncoeffq*4+32], ymm7 +%else + mova [dqcoeffq+ncoeffq*2+ 0], ymm7 + mova [qcoeffq+ncoeffq*2+ 0], ymm7 +%endif + add ncoeffq, mmsize + jl .blank_loop + + mov [eobq], word 0 + + vzeroupper + RET +%endmacro + +INIT_XMM avx +QUANTIZE_FN b, 7 +QUANTIZE_FN b_32x32, 7 + +END diff --git a/third_party/aom/aom_dsp/x86/quantize_sse2.c b/third_party/aom/aom_dsp/x86/quantize_sse2.c new file mode 100644 index 0000000000..890c1f01e8 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/quantize_sse2.c @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +static INLINE __m128i load_coefficients(const tran_low_t *coeff_ptr) { +#if CONFIG_HIGHBITDEPTH + return _mm_setr_epi16((int16_t)coeff_ptr[0], (int16_t)coeff_ptr[1], + (int16_t)coeff_ptr[2], (int16_t)coeff_ptr[3], + (int16_t)coeff_ptr[4], (int16_t)coeff_ptr[5], + (int16_t)coeff_ptr[6], (int16_t)coeff_ptr[7]); +#else + return _mm_load_si128((const __m128i *)coeff_ptr); +#endif +} + +static INLINE void store_coefficients(__m128i coeff_vals, + tran_low_t *coeff_ptr) { +#if CONFIG_HIGHBITDEPTH + __m128i one = _mm_set1_epi16(1); + __m128i coeff_vals_hi = _mm_mulhi_epi16(coeff_vals, one); + __m128i coeff_vals_lo = _mm_mullo_epi16(coeff_vals, one); + __m128i coeff_vals_1 = _mm_unpacklo_epi16(coeff_vals_lo, coeff_vals_hi); + __m128i coeff_vals_2 = _mm_unpackhi_epi16(coeff_vals_lo, coeff_vals_hi); + _mm_store_si128((__m128i *)(coeff_ptr), coeff_vals_1); + _mm_store_si128((__m128i *)(coeff_ptr + 4), coeff_vals_2); +#else + _mm_store_si128((__m128i *)(coeff_ptr), coeff_vals); +#endif +} + +void aom_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, + uint16_t *eob_ptr, const int16_t *scan_ptr, + const int16_t *iscan_ptr) { + __m128i zero; + (void)scan_ptr; + + coeff_ptr += n_coeffs; + iscan_ptr += n_coeffs; + qcoeff_ptr += n_coeffs; + dqcoeff_ptr += n_coeffs; + n_coeffs = -n_coeffs; + zero = _mm_setzero_si128(); + if (!skip_block) { + __m128i eob; + __m128i zbin; + __m128i round, quant, dequant, shift; + { + __m128i coeff0, coeff1; + + // Setup global values + { + __m128i pw_1; + zbin = _mm_load_si128((const __m128i *)zbin_ptr); + round = _mm_load_si128((const __m128i *)round_ptr); + quant = _mm_load_si128((const __m128i *)quant_ptr); + pw_1 = _mm_set1_epi16(1); + zbin = _mm_sub_epi16(zbin, pw_1); + dequant = _mm_load_si128((const __m128i *)dequant_ptr); + shift = _mm_load_si128((const __m128i *)quant_shift_ptr); + } + + { + __m128i coeff0_sign, coeff1_sign; + __m128i qcoeff0, qcoeff1; + __m128i qtmp0, qtmp1; + __m128i cmp_mask0, cmp_mask1; + // Do DC and first 15 AC + coeff0 = load_coefficients(coeff_ptr + n_coeffs); + coeff1 = load_coefficients(coeff_ptr + n_coeffs + 8); + + // Poor man's sign extract + coeff0_sign = _mm_srai_epi16(coeff0, 15); + coeff1_sign = _mm_srai_epi16(coeff1, 15); + qcoeff0 = _mm_xor_si128(coeff0, coeff0_sign); + qcoeff1 = _mm_xor_si128(coeff1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin); + zbin = _mm_unpackhi_epi64(zbin, zbin); // Switch DC to AC + cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin); + qcoeff0 = _mm_adds_epi16(qcoeff0, round); + round = _mm_unpackhi_epi64(round, round); + qcoeff1 = _mm_adds_epi16(qcoeff1, round); + qtmp0 = _mm_mulhi_epi16(qcoeff0, quant); + quant = _mm_unpackhi_epi64(quant, quant); + qtmp1 = _mm_mulhi_epi16(qcoeff1, quant); + qtmp0 = _mm_add_epi16(qtmp0, qcoeff0); + qtmp1 = _mm_add_epi16(qtmp1, qcoeff1); + qcoeff0 = _mm_mulhi_epi16(qtmp0, shift); + shift = _mm_unpackhi_epi64(shift, shift); + qcoeff1 = _mm_mulhi_epi16(qtmp1, shift); + + // Reinsert signs + qcoeff0 = _mm_xor_si128(qcoeff0, coeff0_sign); + qcoeff1 = _mm_xor_si128(qcoeff1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + // Mask out zbin threshold coeffs + qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0); + qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1); + + store_coefficients(qcoeff0, qcoeff_ptr + n_coeffs); + store_coefficients(qcoeff1, qcoeff_ptr + n_coeffs + 8); + + coeff0 = _mm_mullo_epi16(qcoeff0, dequant); + dequant = _mm_unpackhi_epi64(dequant, dequant); + coeff1 = _mm_mullo_epi16(qcoeff1, dequant); + + store_coefficients(coeff0, dqcoeff_ptr + n_coeffs); + store_coefficients(coeff1, dqcoeff_ptr + n_coeffs + 8); + } + + { + // Scan for eob + __m128i zero_coeff0, zero_coeff1; + __m128i nzero_coeff0, nzero_coeff1; + __m128i iscan0, iscan1; + __m128i eob1; + zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero); + zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero); + nzero_coeff0 = _mm_cmpeq_epi16(zero_coeff0, zero); + nzero_coeff1 = _mm_cmpeq_epi16(zero_coeff1, zero); + iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs)); + iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs) + 1); + // Add one to convert from indices to counts + iscan0 = _mm_sub_epi16(iscan0, nzero_coeff0); + iscan1 = _mm_sub_epi16(iscan1, nzero_coeff1); + eob = _mm_and_si128(iscan0, nzero_coeff0); + eob1 = _mm_and_si128(iscan1, nzero_coeff1); + eob = _mm_max_epi16(eob, eob1); + } + n_coeffs += 8 * 2; + } + + // AC only loop + while (n_coeffs < 0) { + __m128i coeff0, coeff1; + { + __m128i coeff0_sign, coeff1_sign; + __m128i qcoeff0, qcoeff1; + __m128i qtmp0, qtmp1; + __m128i cmp_mask0, cmp_mask1; + + coeff0 = load_coefficients(coeff_ptr + n_coeffs); + coeff1 = load_coefficients(coeff_ptr + n_coeffs + 8); + + // Poor man's sign extract + coeff0_sign = _mm_srai_epi16(coeff0, 15); + coeff1_sign = _mm_srai_epi16(coeff1, 15); + qcoeff0 = _mm_xor_si128(coeff0, coeff0_sign); + qcoeff1 = _mm_xor_si128(coeff1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin); + cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin); + qcoeff0 = _mm_adds_epi16(qcoeff0, round); + qcoeff1 = _mm_adds_epi16(qcoeff1, round); + qtmp0 = _mm_mulhi_epi16(qcoeff0, quant); + qtmp1 = _mm_mulhi_epi16(qcoeff1, quant); + qtmp0 = _mm_add_epi16(qtmp0, qcoeff0); + qtmp1 = _mm_add_epi16(qtmp1, qcoeff1); + qcoeff0 = _mm_mulhi_epi16(qtmp0, shift); + qcoeff1 = _mm_mulhi_epi16(qtmp1, shift); + + // Reinsert signs + qcoeff0 = _mm_xor_si128(qcoeff0, coeff0_sign); + qcoeff1 = _mm_xor_si128(qcoeff1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + // Mask out zbin threshold coeffs + qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0); + qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1); + + store_coefficients(qcoeff0, qcoeff_ptr + n_coeffs); + store_coefficients(qcoeff1, qcoeff_ptr + n_coeffs + 8); + + coeff0 = _mm_mullo_epi16(qcoeff0, dequant); + coeff1 = _mm_mullo_epi16(qcoeff1, dequant); + + store_coefficients(coeff0, dqcoeff_ptr + n_coeffs); + store_coefficients(coeff1, dqcoeff_ptr + n_coeffs + 8); + } + + { + // Scan for eob + __m128i zero_coeff0, zero_coeff1; + __m128i nzero_coeff0, nzero_coeff1; + __m128i iscan0, iscan1; + __m128i eob0, eob1; + zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero); + zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero); + nzero_coeff0 = _mm_cmpeq_epi16(zero_coeff0, zero); + nzero_coeff1 = _mm_cmpeq_epi16(zero_coeff1, zero); + iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs)); + iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs) + 1); + // Add one to convert from indices to counts + iscan0 = _mm_sub_epi16(iscan0, nzero_coeff0); + iscan1 = _mm_sub_epi16(iscan1, nzero_coeff1); + eob0 = _mm_and_si128(iscan0, nzero_coeff0); + eob1 = _mm_and_si128(iscan1, nzero_coeff1); + eob0 = _mm_max_epi16(eob0, eob1); + eob = _mm_max_epi16(eob, eob0); + } + n_coeffs += 8 * 2; + } + + // Accumulate EOB + { + __m128i eob_shuffled; + eob_shuffled = _mm_shuffle_epi32(eob, 0xe); + eob = _mm_max_epi16(eob, eob_shuffled); + eob_shuffled = _mm_shufflelo_epi16(eob, 0xe); + eob = _mm_max_epi16(eob, eob_shuffled); + eob_shuffled = _mm_shufflelo_epi16(eob, 0x1); + eob = _mm_max_epi16(eob, eob_shuffled); + *eob_ptr = _mm_extract_epi16(eob, 1); + } + } else { + do { + store_coefficients(zero, dqcoeff_ptr + n_coeffs); + store_coefficients(zero, dqcoeff_ptr + n_coeffs + 8); + store_coefficients(zero, qcoeff_ptr + n_coeffs); + store_coefficients(zero, qcoeff_ptr + n_coeffs + 8); + n_coeffs += 8 * 2; + } while (n_coeffs < 0); + *eob_ptr = 0; + } +} diff --git a/third_party/aom/aom_dsp/x86/quantize_ssse3_x86_64.asm b/third_party/aom/aom_dsp/x86/quantize_ssse3_x86_64.asm new file mode 100644 index 0000000000..36b4dddbdf --- /dev/null +++ b/third_party/aom/aom_dsp/x86/quantize_ssse3_x86_64.asm @@ -0,0 +1,349 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION_RODATA +pw_1: times 8 dw 1 + +SECTION .text + +; TODO(yunqingwang)fix quantize_b code for skip=1 case. +%macro QUANTIZE_FN 2 +cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, skip, zbin, round, quant, \ + shift, qcoeff, dqcoeff, dequant, \ + eob, scan, iscan + cmp dword skipm, 0 + jne .blank + + ; actual quantize loop - setup pointers, rounders, etc. + movifnidn coeffq, coeffmp + movifnidn ncoeffq, ncoeffmp + mov r2, dequantmp + movifnidn zbinq, zbinmp + movifnidn roundq, roundmp + movifnidn quantq, quantmp + mova m0, [zbinq] ; m0 = zbin + mova m1, [roundq] ; m1 = round + mova m2, [quantq] ; m2 = quant +%ifidn %1, b_32x32 + pcmpeqw m5, m5 + psrlw m5, 15 + paddw m0, m5 + paddw m1, m5 + psrlw m0, 1 ; m0 = (m0 + 1) / 2 + psrlw m1, 1 ; m1 = (m1 + 1) / 2 +%endif + mova m3, [r2q] ; m3 = dequant + psubw m0, [pw_1] + mov r2, shiftmp + mov r3, qcoeffmp + mova m4, [r2] ; m4 = shift + mov r4, dqcoeffmp + mov r5, iscanmp +%ifidn %1, b_32x32 + psllw m4, 1 +%endif + pxor m5, m5 ; m5 = dedicated zero + DEFINE_ARGS coeff, ncoeff, d1, qcoeff, dqcoeff, iscan, d2, d3, d4, d5, eob +%if CONFIG_HIGHBITDEPTH + lea coeffq, [ coeffq+ncoeffq*4] + lea qcoeffq, [ qcoeffq+ncoeffq*4] + lea dqcoeffq, [dqcoeffq+ncoeffq*4] +%else + lea coeffq, [ coeffq+ncoeffq*2] + lea qcoeffq, [ qcoeffq+ncoeffq*2] + lea dqcoeffq, [dqcoeffq+ncoeffq*2] +%endif + lea iscanq, [ iscanq+ncoeffq*2] + neg ncoeffq + + ; get DC and first 15 AC coeffs +%if CONFIG_HIGHBITDEPTH + ; coeff stored as 32bit numbers & require 16bit numbers + mova m9, [ coeffq+ncoeffq*4+ 0] + packssdw m9, [ coeffq+ncoeffq*4+16] + mova m10, [ coeffq+ncoeffq*4+32] + packssdw m10, [ coeffq+ncoeffq*4+48] +%else + mova m9, [ coeffq+ncoeffq*2+ 0] ; m9 = c[i] + mova m10, [ coeffq+ncoeffq*2+16] ; m10 = c[i] +%endif + pabsw m6, m9 ; m6 = abs(m9) + pabsw m11, m10 ; m11 = abs(m10) + pcmpgtw m7, m6, m0 ; m7 = c[i] >= zbin + punpckhqdq m0, m0 + pcmpgtw m12, m11, m0 ; m12 = c[i] >= zbin + paddsw m6, m1 ; m6 += round + punpckhqdq m1, m1 + paddsw m11, m1 ; m11 += round + pmulhw m8, m6, m2 ; m8 = m6*q>>16 + punpckhqdq m2, m2 + pmulhw m13, m11, m2 ; m13 = m11*q>>16 + paddw m8, m6 ; m8 += m6 + paddw m13, m11 ; m13 += m11 + pmulhw m8, m4 ; m8 = m8*qsh>>16 + punpckhqdq m4, m4 + pmulhw m13, m4 ; m13 = m13*qsh>>16 + psignw m8, m9 ; m8 = reinsert sign + psignw m13, m10 ; m13 = reinsert sign + pand m8, m7 + pand m13, m12 +%if CONFIG_HIGHBITDEPTH + ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff + mova m11, m8 + mova m6, m8 + pcmpgtw m5, m8 + punpcklwd m11, m5 + punpckhwd m6, m5 + mova [qcoeffq+ncoeffq*4+ 0], m11 + mova [qcoeffq+ncoeffq*4+16], m6 + pxor m5, m5 + mova m11, m13 + mova m6, m13 + pcmpgtw m5, m13 + punpcklwd m11, m5 + punpckhwd m6, m5 + mova [qcoeffq+ncoeffq*4+32], m11 + mova [qcoeffq+ncoeffq*4+48], m6 + pxor m5, m5 ; reset m5 to zero register +%else + mova [qcoeffq+ncoeffq*2+ 0], m8 + mova [qcoeffq+ncoeffq*2+16], m13 +%endif +%ifidn %1, b_32x32 + pabsw m8, m8 + pabsw m13, m13 +%endif + pmullw m8, m3 ; dqc[i] = qc[i] * q + punpckhqdq m3, m3 + pmullw m13, m3 ; dqc[i] = qc[i] * q +%ifidn %1, b_32x32 + psrlw m8, 1 + psrlw m13, 1 + psignw m8, m9 + psignw m13, m10 +%endif +%if CONFIG_HIGHBITDEPTH + ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff + mova m11, m8 + mova m6, m8 + pcmpgtw m5, m8 + punpcklwd m11, m5 + punpckhwd m6, m5 + mova [dqcoeffq+ncoeffq*4+ 0], m11 + mova [dqcoeffq+ncoeffq*4+16], m6 + pxor m5, m5 + mova m11, m13 + mova m6, m13 + pcmpgtw m5, m13 + punpcklwd m11, m5 + punpckhwd m6, m5 + mova [dqcoeffq+ncoeffq*4+32], m11 + mova [dqcoeffq+ncoeffq*4+48], m6 + pxor m5, m5 ; reset m5 to zero register +%else + mova [dqcoeffq+ncoeffq*2+ 0], m8 + mova [dqcoeffq+ncoeffq*2+16], m13 +%endif + pcmpeqw m8, m5 ; m8 = c[i] == 0 + pcmpeqw m13, m5 ; m13 = c[i] == 0 + mova m6, [ iscanq+ncoeffq*2+ 0] ; m6 = scan[i] + mova m11, [ iscanq+ncoeffq*2+16] ; m11 = scan[i] + psubw m6, m7 ; m6 = scan[i] + 1 + psubw m11, m12 ; m11 = scan[i] + 1 + pandn m8, m6 ; m8 = max(eob) + pandn m13, m11 ; m13 = max(eob) + pmaxsw m8, m13 + add ncoeffq, mmsize + jz .accumulate_eob + +.ac_only_loop: +%if CONFIG_HIGHBITDEPTH + ; pack coeff from 32bit to 16bit array + mova m9, [ coeffq+ncoeffq*4+ 0] + packssdw m9, [ coeffq+ncoeffq*4+16] + mova m10, [ coeffq+ncoeffq*4+32] + packssdw m10, [ coeffq+ncoeffq*4+48] +%else + mova m9, [ coeffq+ncoeffq*2+ 0] ; m9 = c[i] + mova m10, [ coeffq+ncoeffq*2+16] ; m10 = c[i] +%endif + pabsw m6, m9 ; m6 = abs(m9) + pabsw m11, m10 ; m11 = abs(m10) + pcmpgtw m7, m6, m0 ; m7 = c[i] >= zbin + pcmpgtw m12, m11, m0 ; m12 = c[i] >= zbin +%ifidn %1, b_32x32 + pmovmskb r6d, m7 + pmovmskb r2d, m12 + or r6, r2 + jz .skip_iter +%endif + paddsw m6, m1 ; m6 += round + paddsw m11, m1 ; m11 += round + pmulhw m14, m6, m2 ; m14 = m6*q>>16 + pmulhw m13, m11, m2 ; m13 = m11*q>>16 + paddw m14, m6 ; m14 += m6 + paddw m13, m11 ; m13 += m11 + pmulhw m14, m4 ; m14 = m14*qsh>>16 + pmulhw m13, m4 ; m13 = m13*qsh>>16 + psignw m14, m9 ; m14 = reinsert sign + psignw m13, m10 ; m13 = reinsert sign + pand m14, m7 + pand m13, m12 +%if CONFIG_HIGHBITDEPTH + ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff + pxor m11, m11 + mova m11, m14 + mova m6, m14 + pcmpgtw m5, m14 + punpcklwd m11, m5 + punpckhwd m6, m5 + mova [qcoeffq+ncoeffq*4+ 0], m11 + mova [qcoeffq+ncoeffq*4+16], m6 + pxor m5, m5 + mova m11, m13 + mova m6, m13 + pcmpgtw m5, m13 + punpcklwd m11, m5 + punpckhwd m6, m5 + mova [qcoeffq+ncoeffq*4+32], m11 + mova [qcoeffq+ncoeffq*4+48], m6 + pxor m5, m5 ; reset m5 to zero register +%else + mova [qcoeffq+ncoeffq*2+ 0], m14 + mova [qcoeffq+ncoeffq*2+16], m13 +%endif +%ifidn %1, b_32x32 + pabsw m14, m14 + pabsw m13, m13 +%endif + pmullw m14, m3 ; dqc[i] = qc[i] * q + pmullw m13, m3 ; dqc[i] = qc[i] * q +%ifidn %1, b_32x32 + psrlw m14, 1 + psrlw m13, 1 + psignw m14, m9 + psignw m13, m10 +%endif +%if CONFIG_HIGHBITDEPTH + ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff + mova m11, m14 + mova m6, m14 + pcmpgtw m5, m14 + punpcklwd m11, m5 + punpckhwd m6, m5 + mova [dqcoeffq+ncoeffq*4+ 0], m11 + mova [dqcoeffq+ncoeffq*4+16], m6 + pxor m5, m5 + mova m11, m13 + mova m6, m13 + pcmpgtw m5, m13 + punpcklwd m11, m5 + punpckhwd m6, m5 + mova [dqcoeffq+ncoeffq*4+32], m11 + mova [dqcoeffq+ncoeffq*4+48], m6 + pxor m5, m5 +%else + mova [dqcoeffq+ncoeffq*2+ 0], m14 + mova [dqcoeffq+ncoeffq*2+16], m13 +%endif + pcmpeqw m14, m5 ; m14 = c[i] == 0 + pcmpeqw m13, m5 ; m13 = c[i] == 0 + mova m6, [ iscanq+ncoeffq*2+ 0] ; m6 = scan[i] + mova m11, [ iscanq+ncoeffq*2+16] ; m11 = scan[i] + psubw m6, m7 ; m6 = scan[i] + 1 + psubw m11, m12 ; m11 = scan[i] + 1 + pandn m14, m6 ; m14 = max(eob) + pandn m13, m11 ; m13 = max(eob) + pmaxsw m8, m14 + pmaxsw m8, m13 + add ncoeffq, mmsize + jl .ac_only_loop + +%ifidn %1, b_32x32 + jmp .accumulate_eob +.skip_iter: +%if CONFIG_HIGHBITDEPTH + mova [qcoeffq+ncoeffq*4+ 0], m5 + mova [qcoeffq+ncoeffq*4+16], m5 + mova [qcoeffq+ncoeffq*4+32], m5 + mova [qcoeffq+ncoeffq*4+48], m5 + mova [dqcoeffq+ncoeffq*4+ 0], m5 + mova [dqcoeffq+ncoeffq*4+16], m5 + mova [dqcoeffq+ncoeffq*4+32], m5 + mova [dqcoeffq+ncoeffq*4+48], m5 +%else + mova [qcoeffq+ncoeffq*2+ 0], m5 + mova [qcoeffq+ncoeffq*2+16], m5 + mova [dqcoeffq+ncoeffq*2+ 0], m5 + mova [dqcoeffq+ncoeffq*2+16], m5 +%endif + add ncoeffq, mmsize + jl .ac_only_loop +%endif + +.accumulate_eob: + ; horizontally accumulate/max eobs and write into [eob] memory pointer + mov r2, eobmp + pshufd m7, m8, 0xe + pmaxsw m8, m7 + pshuflw m7, m8, 0xe + pmaxsw m8, m7 + pshuflw m7, m8, 0x1 + pmaxsw m8, m7 + pextrw r6, m8, 0 + mov [r2], r6 + RET + + ; skip-block, i.e. just write all zeroes +.blank: + mov r0, dqcoeffmp + movifnidn ncoeffq, ncoeffmp + mov r2, qcoeffmp + mov r3, eobmp + DEFINE_ARGS dqcoeff, ncoeff, qcoeff, eob +%if CONFIG_HIGHBITDEPTH + lea dqcoeffq, [dqcoeffq+ncoeffq*4] + lea qcoeffq, [ qcoeffq+ncoeffq*4] +%else + lea dqcoeffq, [dqcoeffq+ncoeffq*2] + lea qcoeffq, [ qcoeffq+ncoeffq*2] +%endif + neg ncoeffq + pxor m7, m7 +.blank_loop: +%if CONFIG_HIGHBITDEPTH + mova [dqcoeffq+ncoeffq*4+ 0], m7 + mova [dqcoeffq+ncoeffq*4+16], m7 + mova [dqcoeffq+ncoeffq*4+32], m7 + mova [dqcoeffq+ncoeffq*4+48], m7 + mova [qcoeffq+ncoeffq*4+ 0], m7 + mova [qcoeffq+ncoeffq*4+16], m7 + mova [qcoeffq+ncoeffq*4+32], m7 + mova [qcoeffq+ncoeffq*4+48], m7 +%else + mova [dqcoeffq+ncoeffq*2+ 0], m7 + mova [dqcoeffq+ncoeffq*2+16], m7 + mova [qcoeffq+ncoeffq*2+ 0], m7 + mova [qcoeffq+ncoeffq*2+16], m7 +%endif + add ncoeffq, mmsize + jl .blank_loop + mov word [eobq], 0 + RET +%endmacro + +INIT_XMM ssse3 +QUANTIZE_FN b, 7 +QUANTIZE_FN b_32x32, 7 diff --git a/third_party/aom/aom_dsp/x86/sad4d_avx2.c b/third_party/aom/aom_dsp/x86/sad4d_avx2.c new file mode 100644 index 0000000000..e60f518b4c --- /dev/null +++ b/third_party/aom/aom_dsp/x86/sad4d_avx2.c @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include // AVX2 +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +void aom_sad32x32x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref[4], int ref_stride, + uint32_t res[4]) { + __m256i src_reg, ref0_reg, ref1_reg, ref2_reg, ref3_reg; + __m256i sum_ref0, sum_ref1, sum_ref2, sum_ref3; + __m256i sum_mlow, sum_mhigh; + int i; + const uint8_t *ref0, *ref1, *ref2, *ref3; + + ref0 = ref[0]; + ref1 = ref[1]; + ref2 = ref[2]; + ref3 = ref[3]; + sum_ref0 = _mm256_set1_epi16(0); + sum_ref1 = _mm256_set1_epi16(0); + sum_ref2 = _mm256_set1_epi16(0); + sum_ref3 = _mm256_set1_epi16(0); + for (i = 0; i < 32; i++) { + // load src and all refs + src_reg = _mm256_loadu_si256((const __m256i *)src); + ref0_reg = _mm256_loadu_si256((const __m256i *)ref0); + ref1_reg = _mm256_loadu_si256((const __m256i *)ref1); + ref2_reg = _mm256_loadu_si256((const __m256i *)ref2); + ref3_reg = _mm256_loadu_si256((const __m256i *)ref3); + // sum of the absolute differences between every ref-i to src + ref0_reg = _mm256_sad_epu8(ref0_reg, src_reg); + ref1_reg = _mm256_sad_epu8(ref1_reg, src_reg); + ref2_reg = _mm256_sad_epu8(ref2_reg, src_reg); + ref3_reg = _mm256_sad_epu8(ref3_reg, src_reg); + // sum every ref-i + sum_ref0 = _mm256_add_epi32(sum_ref0, ref0_reg); + sum_ref1 = _mm256_add_epi32(sum_ref1, ref1_reg); + sum_ref2 = _mm256_add_epi32(sum_ref2, ref2_reg); + sum_ref3 = _mm256_add_epi32(sum_ref3, ref3_reg); + + src += src_stride; + ref0 += ref_stride; + ref1 += ref_stride; + ref2 += ref_stride; + ref3 += ref_stride; + } + { + __m128i sum; + // in sum_ref-i the result is saved in the first 4 bytes + // the other 4 bytes are zeroed. + // sum_ref1 and sum_ref3 are shifted left by 4 bytes + sum_ref1 = _mm256_slli_si256(sum_ref1, 4); + sum_ref3 = _mm256_slli_si256(sum_ref3, 4); + + // merge sum_ref0 and sum_ref1 also sum_ref2 and sum_ref3 + sum_ref0 = _mm256_or_si256(sum_ref0, sum_ref1); + sum_ref2 = _mm256_or_si256(sum_ref2, sum_ref3); + + // merge every 64 bit from each sum_ref-i + sum_mlow = _mm256_unpacklo_epi64(sum_ref0, sum_ref2); + sum_mhigh = _mm256_unpackhi_epi64(sum_ref0, sum_ref2); + + // add the low 64 bit to the high 64 bit + sum_mlow = _mm256_add_epi32(sum_mlow, sum_mhigh); + + // add the low 128 bit to the high 128 bit + sum = _mm_add_epi32(_mm256_castsi256_si128(sum_mlow), + _mm256_extractf128_si256(sum_mlow, 1)); + + _mm_storeu_si128((__m128i *)(res), sum); + } + _mm256_zeroupper(); +} + +void aom_sad64x64x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref[4], int ref_stride, + uint32_t res[4]) { + __m256i src_reg, srcnext_reg, ref0_reg, ref0next_reg; + __m256i ref1_reg, ref1next_reg, ref2_reg, ref2next_reg; + __m256i ref3_reg, ref3next_reg; + __m256i sum_ref0, sum_ref1, sum_ref2, sum_ref3; + __m256i sum_mlow, sum_mhigh; + int i; + const uint8_t *ref0, *ref1, *ref2, *ref3; + + ref0 = ref[0]; + ref1 = ref[1]; + ref2 = ref[2]; + ref3 = ref[3]; + sum_ref0 = _mm256_set1_epi16(0); + sum_ref1 = _mm256_set1_epi16(0); + sum_ref2 = _mm256_set1_epi16(0); + sum_ref3 = _mm256_set1_epi16(0); + for (i = 0; i < 64; i++) { + // load 64 bytes from src and all refs + src_reg = _mm256_loadu_si256((const __m256i *)src); + srcnext_reg = _mm256_loadu_si256((const __m256i *)(src + 32)); + ref0_reg = _mm256_loadu_si256((const __m256i *)ref0); + ref0next_reg = _mm256_loadu_si256((const __m256i *)(ref0 + 32)); + ref1_reg = _mm256_loadu_si256((const __m256i *)ref1); + ref1next_reg = _mm256_loadu_si256((const __m256i *)(ref1 + 32)); + ref2_reg = _mm256_loadu_si256((const __m256i *)ref2); + ref2next_reg = _mm256_loadu_si256((const __m256i *)(ref2 + 32)); + ref3_reg = _mm256_loadu_si256((const __m256i *)ref3); + ref3next_reg = _mm256_loadu_si256((const __m256i *)(ref3 + 32)); + // sum of the absolute differences between every ref-i to src + ref0_reg = _mm256_sad_epu8(ref0_reg, src_reg); + ref1_reg = _mm256_sad_epu8(ref1_reg, src_reg); + ref2_reg = _mm256_sad_epu8(ref2_reg, src_reg); + ref3_reg = _mm256_sad_epu8(ref3_reg, src_reg); + ref0next_reg = _mm256_sad_epu8(ref0next_reg, srcnext_reg); + ref1next_reg = _mm256_sad_epu8(ref1next_reg, srcnext_reg); + ref2next_reg = _mm256_sad_epu8(ref2next_reg, srcnext_reg); + ref3next_reg = _mm256_sad_epu8(ref3next_reg, srcnext_reg); + + // sum every ref-i + sum_ref0 = _mm256_add_epi32(sum_ref0, ref0_reg); + sum_ref1 = _mm256_add_epi32(sum_ref1, ref1_reg); + sum_ref2 = _mm256_add_epi32(sum_ref2, ref2_reg); + sum_ref3 = _mm256_add_epi32(sum_ref3, ref3_reg); + sum_ref0 = _mm256_add_epi32(sum_ref0, ref0next_reg); + sum_ref1 = _mm256_add_epi32(sum_ref1, ref1next_reg); + sum_ref2 = _mm256_add_epi32(sum_ref2, ref2next_reg); + sum_ref3 = _mm256_add_epi32(sum_ref3, ref3next_reg); + src += src_stride; + ref0 += ref_stride; + ref1 += ref_stride; + ref2 += ref_stride; + ref3 += ref_stride; + } + { + __m128i sum; + + // in sum_ref-i the result is saved in the first 4 bytes + // the other 4 bytes are zeroed. + // sum_ref1 and sum_ref3 are shifted left by 4 bytes + sum_ref1 = _mm256_slli_si256(sum_ref1, 4); + sum_ref3 = _mm256_slli_si256(sum_ref3, 4); + + // merge sum_ref0 and sum_ref1 also sum_ref2 and sum_ref3 + sum_ref0 = _mm256_or_si256(sum_ref0, sum_ref1); + sum_ref2 = _mm256_or_si256(sum_ref2, sum_ref3); + + // merge every 64 bit from each sum_ref-i + sum_mlow = _mm256_unpacklo_epi64(sum_ref0, sum_ref2); + sum_mhigh = _mm256_unpackhi_epi64(sum_ref0, sum_ref2); + + // add the low 64 bit to the high 64 bit + sum_mlow = _mm256_add_epi32(sum_mlow, sum_mhigh); + + // add the low 128 bit to the high 128 bit + sum = _mm_add_epi32(_mm256_castsi256_si128(sum_mlow), + _mm256_extractf128_si256(sum_mlow, 1)); + + _mm_storeu_si128((__m128i *)(res), sum); + } + _mm256_zeroupper(); +} + +void aom_sad32x64x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref[4], int ref_stride, + uint32_t res[4]) { + const uint8_t *rf[4]; + uint32_t sum0[4]; + uint32_t sum1[4]; + + rf[0] = ref[0]; + rf[1] = ref[1]; + rf[2] = ref[2]; + rf[3] = ref[3]; + aom_sad32x32x4d_avx2(src, src_stride, rf, ref_stride, sum0); + src += src_stride << 5; + rf[0] += ref_stride << 5; + rf[1] += ref_stride << 5; + rf[2] += ref_stride << 5; + rf[3] += ref_stride << 5; + aom_sad32x32x4d_avx2(src, src_stride, rf, ref_stride, sum1); + res[0] = sum0[0] + sum1[0]; + res[1] = sum0[1] + sum1[1]; + res[2] = sum0[2] + sum1[2]; + res[3] = sum0[3] + sum1[3]; +} + +void aom_sad64x32x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref[4], int ref_stride, + uint32_t res[4]) { + const uint8_t *rf[4]; + uint32_t sum0[4]; + uint32_t sum1[4]; + unsigned int half_width = 32; + + rf[0] = ref[0]; + rf[1] = ref[1]; + rf[2] = ref[2]; + rf[3] = ref[3]; + aom_sad32x32x4d_avx2(src, src_stride, rf, ref_stride, sum0); + src += half_width; + rf[0] += half_width; + rf[1] += half_width; + rf[2] += half_width; + rf[3] += half_width; + aom_sad32x32x4d_avx2(src, src_stride, rf, ref_stride, sum1); + res[0] = sum0[0] + sum1[0]; + res[1] = sum0[1] + sum1[1]; + res[2] = sum0[2] + sum1[2]; + res[3] = sum0[3] + sum1[3]; +} diff --git a/third_party/aom/aom_dsp/x86/sad4d_sse2.asm b/third_party/aom/aom_dsp/x86/sad4d_sse2.asm new file mode 100644 index 0000000000..8f04ef2f3c --- /dev/null +++ b/third_party/aom/aom_dsp/x86/sad4d_sse2.asm @@ -0,0 +1,253 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION .text + +; PROCESS_4x2x4 first, off_{first,second}_{src,ref}, advance_at_end +%macro PROCESS_4x2x4 5-6 0 + movd m0, [srcq +%2] +%if %1 == 1 + movd m6, [ref1q+%3] + movd m4, [ref2q+%3] + movd m7, [ref3q+%3] + movd m5, [ref4q+%3] + movd m1, [srcq +%4] + movd m2, [ref1q+%5] + punpckldq m0, m1 + punpckldq m6, m2 + movd m1, [ref2q+%5] + movd m2, [ref3q+%5] + movd m3, [ref4q+%5] + punpckldq m4, m1 + punpckldq m7, m2 + punpckldq m5, m3 + movlhps m0, m0 + movlhps m6, m4 + movlhps m7, m5 + psadbw m6, m0 + psadbw m7, m0 +%else + movd m1, [ref1q+%3] + movd m5, [ref1q+%5] + movd m2, [ref2q+%3] + movd m4, [ref2q+%5] + punpckldq m1, m5 + punpckldq m2, m4 + movd m3, [ref3q+%3] + movd m5, [ref3q+%5] + punpckldq m3, m5 + movd m4, [ref4q+%3] + movd m5, [ref4q+%5] + punpckldq m4, m5 + movd m5, [srcq +%4] + punpckldq m0, m5 + movlhps m0, m0 + movlhps m1, m2 + movlhps m3, m4 + psadbw m1, m0 + psadbw m3, m0 + paddd m6, m1 + paddd m7, m3 +%endif +%if %6 == 1 + lea srcq, [srcq +src_strideq*2] + lea ref1q, [ref1q+ref_strideq*2] + lea ref2q, [ref2q+ref_strideq*2] + lea ref3q, [ref3q+ref_strideq*2] + lea ref4q, [ref4q+ref_strideq*2] +%endif +%endmacro + +; PROCESS_8x2x4 first, off_{first,second}_{src,ref}, advance_at_end +%macro PROCESS_8x2x4 5-6 0 + movh m0, [srcq +%2] +%if %1 == 1 + movh m4, [ref1q+%3] + movh m5, [ref2q+%3] + movh m6, [ref3q+%3] + movh m7, [ref4q+%3] + movhps m0, [srcq +%4] + movhps m4, [ref1q+%5] + movhps m5, [ref2q+%5] + movhps m6, [ref3q+%5] + movhps m7, [ref4q+%5] + psadbw m4, m0 + psadbw m5, m0 + psadbw m6, m0 + psadbw m7, m0 +%else + movh m1, [ref1q+%3] + movh m2, [ref2q+%3] + movh m3, [ref3q+%3] + movhps m0, [srcq +%4] + movhps m1, [ref1q+%5] + movhps m2, [ref2q+%5] + movhps m3, [ref3q+%5] + psadbw m1, m0 + psadbw m2, m0 + psadbw m3, m0 + paddd m4, m1 + movh m1, [ref4q+%3] + movhps m1, [ref4q+%5] + paddd m5, m2 + paddd m6, m3 + psadbw m1, m0 + paddd m7, m1 +%endif +%if %6 == 1 + lea srcq, [srcq +src_strideq*2] + lea ref1q, [ref1q+ref_strideq*2] + lea ref2q, [ref2q+ref_strideq*2] + lea ref3q, [ref3q+ref_strideq*2] + lea ref4q, [ref4q+ref_strideq*2] +%endif +%endmacro + +; PROCESS_16x2x4 first, off_{first,second}_{src,ref}, advance_at_end +%macro PROCESS_16x2x4 5-6 0 + ; 1st 16 px + mova m0, [srcq +%2] +%if %1 == 1 + movu m4, [ref1q+%3] + movu m5, [ref2q+%3] + movu m6, [ref3q+%3] + movu m7, [ref4q+%3] + psadbw m4, m0 + psadbw m5, m0 + psadbw m6, m0 + psadbw m7, m0 +%else + movu m1, [ref1q+%3] + movu m2, [ref2q+%3] + movu m3, [ref3q+%3] + psadbw m1, m0 + psadbw m2, m0 + psadbw m3, m0 + paddd m4, m1 + movu m1, [ref4q+%3] + paddd m5, m2 + paddd m6, m3 + psadbw m1, m0 + paddd m7, m1 +%endif + + ; 2nd 16 px + mova m0, [srcq +%4] + movu m1, [ref1q+%5] + movu m2, [ref2q+%5] + movu m3, [ref3q+%5] + psadbw m1, m0 + psadbw m2, m0 + psadbw m3, m0 + paddd m4, m1 + movu m1, [ref4q+%5] + paddd m5, m2 + paddd m6, m3 +%if %6 == 1 + lea srcq, [srcq +src_strideq*2] + lea ref1q, [ref1q+ref_strideq*2] + lea ref2q, [ref2q+ref_strideq*2] + lea ref3q, [ref3q+ref_strideq*2] + lea ref4q, [ref4q+ref_strideq*2] +%endif + psadbw m1, m0 + paddd m7, m1 +%endmacro + +; PROCESS_32x2x4 first, off_{first,second}_{src,ref}, advance_at_end +%macro PROCESS_32x2x4 5-6 0 + PROCESS_16x2x4 %1, %2, %3, %2 + 16, %3 + 16 + PROCESS_16x2x4 0, %4, %5, %4 + 16, %5 + 16, %6 +%endmacro + +; PROCESS_64x2x4 first, off_{first,second}_{src,ref}, advance_at_end +%macro PROCESS_64x2x4 5-6 0 + PROCESS_32x2x4 %1, %2, %3, %2 + 32, %3 + 32 + PROCESS_32x2x4 0, %4, %5, %4 + 32, %5 + 32, %6 +%endmacro + +; PROCESS_128x2x4 first, off_{first,second}_{src,ref}, advance_at_end +%macro PROCESS_128x2x4 5-6 0 + PROCESS_64x2x4 %1, %2, %3, %2 + 64, %3 + 64 + PROCESS_64x2x4 0, %4, %5, %4 + 64, %5 + 64, %6 +%endmacro + +; void aom_sadNxNx4d_sse2(uint8_t *src, int src_stride, +; uint8_t *ref[4], int ref_stride, +; uint32_t res[4]); +; where NxN = 64x64, 32x32, 16x16, 16x8, 8x16, 8x8, 8x4, 4x8 and 4x4 +%macro SADNXN4D 2 +%if UNIX64 +cglobal sad%1x%2x4d, 5, 8, 8, src, src_stride, ref1, ref_stride, \ + res, ref2, ref3, ref4 +%else +cglobal sad%1x%2x4d, 4, 7, 8, src, src_stride, ref1, ref_stride, \ + ref2, ref3, ref4 +%endif + movsxdifnidn src_strideq, src_strided + movsxdifnidn ref_strideq, ref_strided + mov ref2q, [ref1q+gprsize*1] + mov ref3q, [ref1q+gprsize*2] + mov ref4q, [ref1q+gprsize*3] + mov ref1q, [ref1q+gprsize*0] + + PROCESS_%1x2x4 1, 0, 0, src_strideq, ref_strideq, 1 +%rep (%2-4)/2 + PROCESS_%1x2x4 0, 0, 0, src_strideq, ref_strideq, 1 +%endrep + PROCESS_%1x2x4 0, 0, 0, src_strideq, ref_strideq, 0 + +%if %1 > 4 + pslldq m5, 4 + pslldq m7, 4 + por m4, m5 + por m6, m7 + mova m5, m4 + mova m7, m6 + punpcklqdq m4, m6 + punpckhqdq m5, m7 + movifnidn r4, r4mp + paddd m4, m5 + movu [r4], m4 + RET +%else + movifnidn r4, r4mp + pshufd m6, m6, 0x08 + pshufd m7, m7, 0x08 + movq [r4+0], m6 + movq [r4+8], m7 + RET +%endif +%endmacro + +INIT_XMM sse2 +%if CONFIG_EXT_PARTITION +SADNXN4D 128, 128 +SADNXN4D 128, 64 +SADNXN4D 64, 128 +%endif +SADNXN4D 64, 64 +SADNXN4D 64, 32 +SADNXN4D 32, 64 +SADNXN4D 32, 32 +SADNXN4D 32, 16 +SADNXN4D 16, 32 +SADNXN4D 16, 16 +SADNXN4D 16, 8 +SADNXN4D 8, 16 +SADNXN4D 8, 8 +SADNXN4D 8, 4 +SADNXN4D 4, 8 +SADNXN4D 4, 4 diff --git a/third_party/aom/aom_dsp/x86/sad_avx2.c b/third_party/aom/aom_dsp/x86/sad_avx2.c new file mode 100644 index 0000000000..efba612896 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/sad_avx2.c @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include +#include "./aom_dsp_rtcd.h" +#include "aom_ports/mem.h" + +#define FSAD64_H(h) \ + unsigned int aom_sad64x##h##_avx2(const uint8_t *src_ptr, int src_stride, \ + const uint8_t *ref_ptr, int ref_stride) { \ + int i, res; \ + __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ + __m256i sum_sad = _mm256_setzero_si256(); \ + __m256i sum_sad_h; \ + __m128i sum_sad128; \ + for (i = 0; i < h; i++) { \ + ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); \ + ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + 32)); \ + sad1_reg = _mm256_sad_epu8( \ + ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); \ + sad2_reg = _mm256_sad_epu8( \ + ref2_reg, _mm256_loadu_si256((__m256i const *)(src_ptr + 32))); \ + sum_sad = \ + _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); \ + ref_ptr += ref_stride; \ + src_ptr += src_stride; \ + } \ + sum_sad_h = _mm256_srli_si256(sum_sad, 8); \ + sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); \ + sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); \ + sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ + res = _mm_cvtsi128_si32(sum_sad128); \ + _mm256_zeroupper(); \ + return res; \ + } + +#define FSAD32_H(h) \ + unsigned int aom_sad32x##h##_avx2(const uint8_t *src_ptr, int src_stride, \ + const uint8_t *ref_ptr, int ref_stride) { \ + int i, res; \ + __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ + __m256i sum_sad = _mm256_setzero_si256(); \ + __m256i sum_sad_h; \ + __m128i sum_sad128; \ + int ref2_stride = ref_stride << 1; \ + int src2_stride = src_stride << 1; \ + int max = h >> 1; \ + for (i = 0; i < max; i++) { \ + ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); \ + ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + ref_stride)); \ + sad1_reg = _mm256_sad_epu8( \ + ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); \ + sad2_reg = _mm256_sad_epu8( \ + ref2_reg, \ + _mm256_loadu_si256((__m256i const *)(src_ptr + src_stride))); \ + sum_sad = \ + _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); \ + ref_ptr += ref2_stride; \ + src_ptr += src2_stride; \ + } \ + sum_sad_h = _mm256_srli_si256(sum_sad, 8); \ + sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); \ + sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); \ + sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ + res = _mm_cvtsi128_si32(sum_sad128); \ + _mm256_zeroupper(); \ + return res; \ + } + +#define FSAD64 \ + FSAD64_H(64); \ + FSAD64_H(32); + +#define FSAD32 \ + FSAD32_H(64); \ + FSAD32_H(32); \ + FSAD32_H(16); + +/* clang-format off */ +FSAD64 +FSAD32 +/* clang-format on */ + +#undef FSAD64 +#undef FSAD32 +#undef FSAD64_H +#undef FSAD32_H + +#define FSADAVG64_H(h) \ + unsigned int aom_sad64x##h##_avg_avx2( \ + const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ + int ref_stride, const uint8_t *second_pred) { \ + int i, res; \ + __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ + __m256i sum_sad = _mm256_setzero_si256(); \ + __m256i sum_sad_h; \ + __m128i sum_sad128; \ + for (i = 0; i < h; i++) { \ + ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); \ + ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + 32)); \ + ref1_reg = _mm256_avg_epu8( \ + ref1_reg, _mm256_loadu_si256((__m256i const *)second_pred)); \ + ref2_reg = _mm256_avg_epu8( \ + ref2_reg, _mm256_loadu_si256((__m256i const *)(second_pred + 32))); \ + sad1_reg = _mm256_sad_epu8( \ + ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); \ + sad2_reg = _mm256_sad_epu8( \ + ref2_reg, _mm256_loadu_si256((__m256i const *)(src_ptr + 32))); \ + sum_sad = \ + _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); \ + ref_ptr += ref_stride; \ + src_ptr += src_stride; \ + second_pred += 64; \ + } \ + sum_sad_h = _mm256_srli_si256(sum_sad, 8); \ + sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); \ + sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); \ + sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ + res = _mm_cvtsi128_si32(sum_sad128); \ + _mm256_zeroupper(); \ + return res; \ + } + +#define FSADAVG32_H(h) \ + unsigned int aom_sad32x##h##_avg_avx2( \ + const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ + int ref_stride, const uint8_t *second_pred) { \ + int i, res; \ + __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ + __m256i sum_sad = _mm256_setzero_si256(); \ + __m256i sum_sad_h; \ + __m128i sum_sad128; \ + int ref2_stride = ref_stride << 1; \ + int src2_stride = src_stride << 1; \ + int max = h >> 1; \ + for (i = 0; i < max; i++) { \ + ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); \ + ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + ref_stride)); \ + ref1_reg = _mm256_avg_epu8( \ + ref1_reg, _mm256_loadu_si256((__m256i const *)second_pred)); \ + ref2_reg = _mm256_avg_epu8( \ + ref2_reg, _mm256_loadu_si256((__m256i const *)(second_pred + 32))); \ + sad1_reg = _mm256_sad_epu8( \ + ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); \ + sad2_reg = _mm256_sad_epu8( \ + ref2_reg, \ + _mm256_loadu_si256((__m256i const *)(src_ptr + src_stride))); \ + sum_sad = \ + _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); \ + ref_ptr += ref2_stride; \ + src_ptr += src2_stride; \ + second_pred += 64; \ + } \ + sum_sad_h = _mm256_srli_si256(sum_sad, 8); \ + sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); \ + sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); \ + sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ + res = _mm_cvtsi128_si32(sum_sad128); \ + _mm256_zeroupper(); \ + return res; \ + } + +#define FSADAVG64 \ + FSADAVG64_H(64); \ + FSADAVG64_H(32); + +#define FSADAVG32 \ + FSADAVG32_H(64); \ + FSADAVG32_H(32); \ + FSADAVG32_H(16); + +/* clang-format off */ +FSADAVG64 +FSADAVG32 +/* clang-format on */ + +#undef FSADAVG64 +#undef FSADAVG32 +#undef FSADAVG64_H +#undef FSADAVG32_H diff --git a/third_party/aom/aom_dsp/x86/sad_highbd_avx2.c b/third_party/aom/aom_dsp/x86/sad_highbd_avx2.c new file mode 100644 index 0000000000..196394379a --- /dev/null +++ b/third_party/aom/aom_dsp/x86/sad_highbd_avx2.c @@ -0,0 +1,1043 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" + +// SAD +static INLINE unsigned int get_sad_from_mm256_epi32(const __m256i *v) { + // input 8 32-bit summation + __m128i lo128, hi128; + __m256i u = _mm256_srli_si256(*v, 8); + u = _mm256_add_epi32(u, *v); + + // 4 32-bit summation + hi128 = _mm256_extracti128_si256(u, 1); + lo128 = _mm256_castsi256_si128(u); + lo128 = _mm_add_epi32(hi128, lo128); + + // 2 32-bit summation + hi128 = _mm_srli_si128(lo128, 4); + lo128 = _mm_add_epi32(lo128, hi128); + + return (unsigned int)_mm_cvtsi128_si32(lo128); +} + +unsigned int aom_highbd_sad16x8_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + const uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src); + const uint16_t *ref_ptr = CONVERT_TO_SHORTPTR(ref); + + // first 4 rows + __m256i s0 = _mm256_loadu_si256((const __m256i *)src_ptr); + __m256i s1 = _mm256_loadu_si256((const __m256i *)(src_ptr + src_stride)); + __m256i s2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 2 * src_stride)); + __m256i s3 = _mm256_loadu_si256((const __m256i *)(src_ptr + 3 * src_stride)); + + __m256i r0 = _mm256_loadu_si256((const __m256i *)ref_ptr); + __m256i r1 = _mm256_loadu_si256((const __m256i *)(ref_ptr + ref_stride)); + __m256i r2 = _mm256_loadu_si256((const __m256i *)(ref_ptr + 2 * ref_stride)); + __m256i r3 = _mm256_loadu_si256((const __m256i *)(ref_ptr + 3 * ref_stride)); + + __m256i u0 = _mm256_sub_epi16(s0, r0); + __m256i u1 = _mm256_sub_epi16(s1, r1); + __m256i u2 = _mm256_sub_epi16(s2, r2); + __m256i u3 = _mm256_sub_epi16(s3, r3); + __m256i zero = _mm256_setzero_si256(); + __m256i sum0, sum1; + + u0 = _mm256_abs_epi16(u0); + u1 = _mm256_abs_epi16(u1); + u2 = _mm256_abs_epi16(u2); + u3 = _mm256_abs_epi16(u3); + + sum0 = _mm256_add_epi16(u0, u1); + sum0 = _mm256_add_epi16(sum0, u2); + sum0 = _mm256_add_epi16(sum0, u3); + + // second 4 rows + src_ptr += src_stride << 2; + ref_ptr += ref_stride << 2; + s0 = _mm256_loadu_si256((const __m256i *)src_ptr); + s1 = _mm256_loadu_si256((const __m256i *)(src_ptr + src_stride)); + s2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 2 * src_stride)); + s3 = _mm256_loadu_si256((const __m256i *)(src_ptr + 3 * src_stride)); + + r0 = _mm256_loadu_si256((const __m256i *)ref_ptr); + r1 = _mm256_loadu_si256((const __m256i *)(ref_ptr + ref_stride)); + r2 = _mm256_loadu_si256((const __m256i *)(ref_ptr + 2 * ref_stride)); + r3 = _mm256_loadu_si256((const __m256i *)(ref_ptr + 3 * ref_stride)); + + u0 = _mm256_sub_epi16(s0, r0); + u1 = _mm256_sub_epi16(s1, r1); + u2 = _mm256_sub_epi16(s2, r2); + u3 = _mm256_sub_epi16(s3, r3); + + u0 = _mm256_abs_epi16(u0); + u1 = _mm256_abs_epi16(u1); + u2 = _mm256_abs_epi16(u2); + u3 = _mm256_abs_epi16(u3); + + sum1 = _mm256_add_epi16(u0, u1); + sum1 = _mm256_add_epi16(sum1, u2); + sum1 = _mm256_add_epi16(sum1, u3); + + // find out the SAD + s0 = _mm256_unpacklo_epi16(sum0, zero); + s1 = _mm256_unpackhi_epi16(sum0, zero); + r0 = _mm256_unpacklo_epi16(sum1, zero); + r1 = _mm256_unpackhi_epi16(sum1, zero); + s0 = _mm256_add_epi32(s0, s1); + r0 = _mm256_add_epi32(r0, r1); + sum0 = _mm256_add_epi32(s0, r0); + // 8 32-bit summation + + return (unsigned int)get_sad_from_mm256_epi32(&sum0); +} + +unsigned int aom_highbd_sad16x16_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + const uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src); + const uint16_t *ref_ptr = CONVERT_TO_SHORTPTR(ref); + __m256i s0, s1, s2, s3, r0, r1, r2, r3, u0, u1, u2, u3; + __m256i sum0; + __m256i sum = _mm256_setzero_si256(); + const __m256i zero = _mm256_setzero_si256(); + int row = 0; + + // Loop for every 4 rows + while (row < 16) { + s0 = _mm256_loadu_si256((const __m256i *)src_ptr); + s1 = _mm256_loadu_si256((const __m256i *)(src_ptr + src_stride)); + s2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 2 * src_stride)); + s3 = _mm256_loadu_si256((const __m256i *)(src_ptr + 3 * src_stride)); + + r0 = _mm256_loadu_si256((const __m256i *)ref_ptr); + r1 = _mm256_loadu_si256((const __m256i *)(ref_ptr + ref_stride)); + r2 = _mm256_loadu_si256((const __m256i *)(ref_ptr + 2 * ref_stride)); + r3 = _mm256_loadu_si256((const __m256i *)(ref_ptr + 3 * ref_stride)); + + u0 = _mm256_sub_epi16(s0, r0); + u1 = _mm256_sub_epi16(s1, r1); + u2 = _mm256_sub_epi16(s2, r2); + u3 = _mm256_sub_epi16(s3, r3); + + u0 = _mm256_abs_epi16(u0); + u1 = _mm256_abs_epi16(u1); + u2 = _mm256_abs_epi16(u2); + u3 = _mm256_abs_epi16(u3); + + sum0 = _mm256_add_epi16(u0, u1); + sum0 = _mm256_add_epi16(sum0, u2); + sum0 = _mm256_add_epi16(sum0, u3); + + s0 = _mm256_unpacklo_epi16(sum0, zero); + s1 = _mm256_unpackhi_epi16(sum0, zero); + sum = _mm256_add_epi32(sum, s0); + sum = _mm256_add_epi32(sum, s1); + // 8 32-bit summation + + row += 4; + src_ptr += src_stride << 2; + ref_ptr += ref_stride << 2; + } + return get_sad_from_mm256_epi32(&sum); +} + +static void sad32x4(const uint16_t *src_ptr, int src_stride, + const uint16_t *ref_ptr, int ref_stride, + const uint16_t *sec_ptr, __m256i *sad_acc) { + __m256i s0, s1, s2, s3, r0, r1, r2, r3; + const __m256i zero = _mm256_setzero_si256(); + int row_sections = 0; + + while (row_sections < 2) { + s0 = _mm256_loadu_si256((const __m256i *)src_ptr); + s1 = _mm256_loadu_si256((const __m256i *)(src_ptr + 16)); + s2 = _mm256_loadu_si256((const __m256i *)(src_ptr + src_stride)); + s3 = _mm256_loadu_si256((const __m256i *)(src_ptr + src_stride + 16)); + + r0 = _mm256_loadu_si256((const __m256i *)ref_ptr); + r1 = _mm256_loadu_si256((const __m256i *)(ref_ptr + 16)); + r2 = _mm256_loadu_si256((const __m256i *)(ref_ptr + ref_stride)); + r3 = _mm256_loadu_si256((const __m256i *)(ref_ptr + ref_stride + 16)); + + if (sec_ptr) { + r0 = _mm256_avg_epu16(r0, _mm256_loadu_si256((const __m256i *)sec_ptr)); + r1 = _mm256_avg_epu16( + r1, _mm256_loadu_si256((const __m256i *)(sec_ptr + 16))); + r2 = _mm256_avg_epu16( + r2, _mm256_loadu_si256((const __m256i *)(sec_ptr + 32))); + r3 = _mm256_avg_epu16( + r3, _mm256_loadu_si256((const __m256i *)(sec_ptr + 48))); + } + s0 = _mm256_sub_epi16(s0, r0); + s1 = _mm256_sub_epi16(s1, r1); + s2 = _mm256_sub_epi16(s2, r2); + s3 = _mm256_sub_epi16(s3, r3); + + s0 = _mm256_abs_epi16(s0); + s1 = _mm256_abs_epi16(s1); + s2 = _mm256_abs_epi16(s2); + s3 = _mm256_abs_epi16(s3); + + s0 = _mm256_add_epi16(s0, s1); + s0 = _mm256_add_epi16(s0, s2); + s0 = _mm256_add_epi16(s0, s3); + + r0 = _mm256_unpacklo_epi16(s0, zero); + r1 = _mm256_unpackhi_epi16(s0, zero); + + r0 = _mm256_add_epi32(r0, r1); + *sad_acc = _mm256_add_epi32(*sad_acc, r0); + + row_sections += 1; + src_ptr += src_stride << 1; + ref_ptr += ref_stride << 1; + if (sec_ptr) sec_ptr += 32 << 1; + } +} + +unsigned int aom_highbd_sad32x16_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + __m256i sad = _mm256_setzero_si256(); + uint16_t *srcp = CONVERT_TO_SHORTPTR(src); + uint16_t *refp = CONVERT_TO_SHORTPTR(ref); + const int left_shift = 2; + int row_section = 0; + + while (row_section < 4) { + sad32x4(srcp, src_stride, refp, ref_stride, NULL, &sad); + srcp += src_stride << left_shift; + refp += ref_stride << left_shift; + row_section += 1; + } + return get_sad_from_mm256_epi32(&sad); +} + +unsigned int aom_highbd_sad16x32_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + uint32_t sum = aom_highbd_sad16x16_avx2(src, src_stride, ref, ref_stride); + src += src_stride << 4; + ref += ref_stride << 4; + sum += aom_highbd_sad16x16_avx2(src, src_stride, ref, ref_stride); + return sum; +} + +unsigned int aom_highbd_sad32x32_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + uint32_t sum = aom_highbd_sad32x16_avx2(src, src_stride, ref, ref_stride); + src += src_stride << 4; + ref += ref_stride << 4; + sum += aom_highbd_sad32x16_avx2(src, src_stride, ref, ref_stride); + return sum; +} + +unsigned int aom_highbd_sad32x64_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + uint32_t sum = aom_highbd_sad32x32_avx2(src, src_stride, ref, ref_stride); + src += src_stride << 5; + ref += ref_stride << 5; + sum += aom_highbd_sad32x32_avx2(src, src_stride, ref, ref_stride); + return sum; +} + +static void sad64x2(const uint16_t *src_ptr, int src_stride, + const uint16_t *ref_ptr, int ref_stride, + const uint16_t *sec_ptr, __m256i *sad_acc) { + __m256i s[8], r[8]; + const __m256i zero = _mm256_setzero_si256(); + + s[0] = _mm256_loadu_si256((const __m256i *)src_ptr); + s[1] = _mm256_loadu_si256((const __m256i *)(src_ptr + 16)); + s[2] = _mm256_loadu_si256((const __m256i *)(src_ptr + 32)); + s[3] = _mm256_loadu_si256((const __m256i *)(src_ptr + 48)); + s[4] = _mm256_loadu_si256((const __m256i *)(src_ptr + src_stride)); + s[5] = _mm256_loadu_si256((const __m256i *)(src_ptr + src_stride + 16)); + s[6] = _mm256_loadu_si256((const __m256i *)(src_ptr + src_stride + 32)); + s[7] = _mm256_loadu_si256((const __m256i *)(src_ptr + src_stride + 48)); + + r[0] = _mm256_loadu_si256((const __m256i *)ref_ptr); + r[1] = _mm256_loadu_si256((const __m256i *)(ref_ptr + 16)); + r[2] = _mm256_loadu_si256((const __m256i *)(ref_ptr + 32)); + r[3] = _mm256_loadu_si256((const __m256i *)(ref_ptr + 48)); + r[4] = _mm256_loadu_si256((const __m256i *)(ref_ptr + ref_stride)); + r[5] = _mm256_loadu_si256((const __m256i *)(ref_ptr + ref_stride + 16)); + r[6] = _mm256_loadu_si256((const __m256i *)(ref_ptr + ref_stride + 32)); + r[7] = _mm256_loadu_si256((const __m256i *)(ref_ptr + ref_stride + 48)); + + if (sec_ptr) { + r[0] = _mm256_avg_epu16(r[0], _mm256_loadu_si256((const __m256i *)sec_ptr)); + r[1] = _mm256_avg_epu16( + r[1], _mm256_loadu_si256((const __m256i *)(sec_ptr + 16))); + r[2] = _mm256_avg_epu16( + r[2], _mm256_loadu_si256((const __m256i *)(sec_ptr + 32))); + r[3] = _mm256_avg_epu16( + r[3], _mm256_loadu_si256((const __m256i *)(sec_ptr + 48))); + r[4] = _mm256_avg_epu16( + r[4], _mm256_loadu_si256((const __m256i *)(sec_ptr + 64))); + r[5] = _mm256_avg_epu16( + r[5], _mm256_loadu_si256((const __m256i *)(sec_ptr + 80))); + r[6] = _mm256_avg_epu16( + r[6], _mm256_loadu_si256((const __m256i *)(sec_ptr + 96))); + r[7] = _mm256_avg_epu16( + r[7], _mm256_loadu_si256((const __m256i *)(sec_ptr + 112))); + } + + s[0] = _mm256_sub_epi16(s[0], r[0]); + s[1] = _mm256_sub_epi16(s[1], r[1]); + s[2] = _mm256_sub_epi16(s[2], r[2]); + s[3] = _mm256_sub_epi16(s[3], r[3]); + s[4] = _mm256_sub_epi16(s[4], r[4]); + s[5] = _mm256_sub_epi16(s[5], r[5]); + s[6] = _mm256_sub_epi16(s[6], r[6]); + s[7] = _mm256_sub_epi16(s[7], r[7]); + + s[0] = _mm256_abs_epi16(s[0]); + s[1] = _mm256_abs_epi16(s[1]); + s[2] = _mm256_abs_epi16(s[2]); + s[3] = _mm256_abs_epi16(s[3]); + s[4] = _mm256_abs_epi16(s[4]); + s[5] = _mm256_abs_epi16(s[5]); + s[6] = _mm256_abs_epi16(s[6]); + s[7] = _mm256_abs_epi16(s[7]); + + s[0] = _mm256_add_epi16(s[0], s[1]); + s[0] = _mm256_add_epi16(s[0], s[2]); + s[0] = _mm256_add_epi16(s[0], s[3]); + + s[4] = _mm256_add_epi16(s[4], s[5]); + s[4] = _mm256_add_epi16(s[4], s[6]); + s[4] = _mm256_add_epi16(s[4], s[7]); + + r[0] = _mm256_unpacklo_epi16(s[0], zero); + r[1] = _mm256_unpackhi_epi16(s[0], zero); + r[2] = _mm256_unpacklo_epi16(s[4], zero); + r[3] = _mm256_unpackhi_epi16(s[4], zero); + + r[0] = _mm256_add_epi32(r[0], r[1]); + r[0] = _mm256_add_epi32(r[0], r[2]); + r[0] = _mm256_add_epi32(r[0], r[3]); + *sad_acc = _mm256_add_epi32(*sad_acc, r[0]); +} + +unsigned int aom_highbd_sad64x32_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + __m256i sad = _mm256_setzero_si256(); + uint16_t *srcp = CONVERT_TO_SHORTPTR(src); + uint16_t *refp = CONVERT_TO_SHORTPTR(ref); + const int left_shift = 1; + int row_section = 0; + + while (row_section < 16) { + sad64x2(srcp, src_stride, refp, ref_stride, NULL, &sad); + srcp += src_stride << left_shift; + refp += ref_stride << left_shift; + row_section += 1; + } + return get_sad_from_mm256_epi32(&sad); +} + +unsigned int aom_highbd_sad64x64_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + uint32_t sum = aom_highbd_sad64x32_avx2(src, src_stride, ref, ref_stride); + src += src_stride << 5; + ref += ref_stride << 5; + sum += aom_highbd_sad64x32_avx2(src, src_stride, ref, ref_stride); + return sum; +} + +#if CONFIG_EXT_PARTITION +static void sad128x1(const uint16_t *src_ptr, const uint16_t *ref_ptr, + const uint16_t *sec_ptr, __m256i *sad_acc) { + __m256i s[8], r[8]; + const __m256i zero = _mm256_setzero_si256(); + + s[0] = _mm256_loadu_si256((const __m256i *)src_ptr); + s[1] = _mm256_loadu_si256((const __m256i *)(src_ptr + 16)); + s[2] = _mm256_loadu_si256((const __m256i *)(src_ptr + 32)); + s[3] = _mm256_loadu_si256((const __m256i *)(src_ptr + 48)); + s[4] = _mm256_loadu_si256((const __m256i *)(src_ptr + 64)); + s[5] = _mm256_loadu_si256((const __m256i *)(src_ptr + 80)); + s[6] = _mm256_loadu_si256((const __m256i *)(src_ptr + 96)); + s[7] = _mm256_loadu_si256((const __m256i *)(src_ptr + 112)); + + r[0] = _mm256_loadu_si256((const __m256i *)ref_ptr); + r[1] = _mm256_loadu_si256((const __m256i *)(ref_ptr + 16)); + r[2] = _mm256_loadu_si256((const __m256i *)(ref_ptr + 32)); + r[3] = _mm256_loadu_si256((const __m256i *)(ref_ptr + 48)); + r[4] = _mm256_loadu_si256((const __m256i *)(ref_ptr + 64)); + r[5] = _mm256_loadu_si256((const __m256i *)(ref_ptr + 80)); + r[6] = _mm256_loadu_si256((const __m256i *)(ref_ptr + 96)); + r[7] = _mm256_loadu_si256((const __m256i *)(ref_ptr + 112)); + + if (sec_ptr) { + r[0] = _mm256_avg_epu16(r[0], _mm256_loadu_si256((const __m256i *)sec_ptr)); + r[1] = _mm256_avg_epu16( + r[1], _mm256_loadu_si256((const __m256i *)(sec_ptr + 16))); + r[2] = _mm256_avg_epu16( + r[2], _mm256_loadu_si256((const __m256i *)(sec_ptr + 32))); + r[3] = _mm256_avg_epu16( + r[3], _mm256_loadu_si256((const __m256i *)(sec_ptr + 48))); + r[4] = _mm256_avg_epu16( + r[4], _mm256_loadu_si256((const __m256i *)(sec_ptr + 64))); + r[5] = _mm256_avg_epu16( + r[5], _mm256_loadu_si256((const __m256i *)(sec_ptr + 80))); + r[6] = _mm256_avg_epu16( + r[6], _mm256_loadu_si256((const __m256i *)(sec_ptr + 96))); + r[7] = _mm256_avg_epu16( + r[7], _mm256_loadu_si256((const __m256i *)(sec_ptr + 112))); + } + + s[0] = _mm256_sub_epi16(s[0], r[0]); + s[1] = _mm256_sub_epi16(s[1], r[1]); + s[2] = _mm256_sub_epi16(s[2], r[2]); + s[3] = _mm256_sub_epi16(s[3], r[3]); + s[4] = _mm256_sub_epi16(s[4], r[4]); + s[5] = _mm256_sub_epi16(s[5], r[5]); + s[6] = _mm256_sub_epi16(s[6], r[6]); + s[7] = _mm256_sub_epi16(s[7], r[7]); + + s[0] = _mm256_abs_epi16(s[0]); + s[1] = _mm256_abs_epi16(s[1]); + s[2] = _mm256_abs_epi16(s[2]); + s[3] = _mm256_abs_epi16(s[3]); + s[4] = _mm256_abs_epi16(s[4]); + s[5] = _mm256_abs_epi16(s[5]); + s[6] = _mm256_abs_epi16(s[6]); + s[7] = _mm256_abs_epi16(s[7]); + + s[0] = _mm256_add_epi16(s[0], s[1]); + s[0] = _mm256_add_epi16(s[0], s[2]); + s[0] = _mm256_add_epi16(s[0], s[3]); + + s[4] = _mm256_add_epi16(s[4], s[5]); + s[4] = _mm256_add_epi16(s[4], s[6]); + s[4] = _mm256_add_epi16(s[4], s[7]); + + r[0] = _mm256_unpacklo_epi16(s[0], zero); + r[1] = _mm256_unpackhi_epi16(s[0], zero); + r[2] = _mm256_unpacklo_epi16(s[4], zero); + r[3] = _mm256_unpackhi_epi16(s[4], zero); + + r[0] = _mm256_add_epi32(r[0], r[1]); + r[0] = _mm256_add_epi32(r[0], r[2]); + r[0] = _mm256_add_epi32(r[0], r[3]); + *sad_acc = _mm256_add_epi32(*sad_acc, r[0]); +} + +unsigned int aom_highbd_sad128x64_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + __m256i sad = _mm256_setzero_si256(); + uint16_t *srcp = CONVERT_TO_SHORTPTR(src); + uint16_t *refp = CONVERT_TO_SHORTPTR(ref); + int row = 0; + while (row < 64) { + sad128x1(srcp, refp, NULL, &sad); + srcp += src_stride; + refp += ref_stride; + row += 1; + } + return get_sad_from_mm256_epi32(&sad); +} + +unsigned int aom_highbd_sad64x128_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + uint32_t sum = aom_highbd_sad64x64_avx2(src, src_stride, ref, ref_stride); + src += src_stride << 6; + ref += ref_stride << 6; + sum += aom_highbd_sad64x64_avx2(src, src_stride, ref, ref_stride); + return sum; +} + +unsigned int aom_highbd_sad128x128_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride) { + uint32_t sum = aom_highbd_sad128x64_avx2(src, src_stride, ref, ref_stride); + src += src_stride << 6; + ref += ref_stride << 6; + sum += aom_highbd_sad128x64_avx2(src, src_stride, ref, ref_stride); + return sum; +} +#endif // CONFIG_EXT_PARTITION + +// If sec_ptr = 0, calculate regular SAD. Otherwise, calculate average SAD. +static INLINE void sad16x4(const uint16_t *src_ptr, int src_stride, + const uint16_t *ref_ptr, int ref_stride, + const uint16_t *sec_ptr, __m256i *sad_acc) { + __m256i s0, s1, s2, s3, r0, r1, r2, r3; + const __m256i zero = _mm256_setzero_si256(); + + s0 = _mm256_loadu_si256((const __m256i *)src_ptr); + s1 = _mm256_loadu_si256((const __m256i *)(src_ptr + src_stride)); + s2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 2 * src_stride)); + s3 = _mm256_loadu_si256((const __m256i *)(src_ptr + 3 * src_stride)); + + r0 = _mm256_loadu_si256((const __m256i *)ref_ptr); + r1 = _mm256_loadu_si256((const __m256i *)(ref_ptr + ref_stride)); + r2 = _mm256_loadu_si256((const __m256i *)(ref_ptr + 2 * ref_stride)); + r3 = _mm256_loadu_si256((const __m256i *)(ref_ptr + 3 * ref_stride)); + + if (sec_ptr) { + r0 = _mm256_avg_epu16(r0, _mm256_loadu_si256((const __m256i *)sec_ptr)); + r1 = _mm256_avg_epu16(r1, + _mm256_loadu_si256((const __m256i *)(sec_ptr + 16))); + r2 = _mm256_avg_epu16(r2, + _mm256_loadu_si256((const __m256i *)(sec_ptr + 32))); + r3 = _mm256_avg_epu16(r3, + _mm256_loadu_si256((const __m256i *)(sec_ptr + 48))); + } + + s0 = _mm256_sub_epi16(s0, r0); + s1 = _mm256_sub_epi16(s1, r1); + s2 = _mm256_sub_epi16(s2, r2); + s3 = _mm256_sub_epi16(s3, r3); + + s0 = _mm256_abs_epi16(s0); + s1 = _mm256_abs_epi16(s1); + s2 = _mm256_abs_epi16(s2); + s3 = _mm256_abs_epi16(s3); + + s0 = _mm256_add_epi16(s0, s1); + s0 = _mm256_add_epi16(s0, s2); + s0 = _mm256_add_epi16(s0, s3); + + r0 = _mm256_unpacklo_epi16(s0, zero); + r1 = _mm256_unpackhi_epi16(s0, zero); + + r0 = _mm256_add_epi32(r0, r1); + *sad_acc = _mm256_add_epi32(*sad_acc, r0); +} + +unsigned int aom_highbd_sad16x8_avg_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred) { + __m256i sad = _mm256_setzero_si256(); + uint16_t *srcp = CONVERT_TO_SHORTPTR(src); + uint16_t *refp = CONVERT_TO_SHORTPTR(ref); + uint16_t *secp = CONVERT_TO_SHORTPTR(second_pred); + + sad16x4(srcp, src_stride, refp, ref_stride, secp, &sad); + + // Next 4 rows + srcp += src_stride << 2; + refp += ref_stride << 2; + secp += 64; + sad16x4(srcp, src_stride, refp, ref_stride, secp, &sad); + return get_sad_from_mm256_epi32(&sad); +} + +unsigned int aom_highbd_sad16x16_avg_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred) { + const int left_shift = 3; + uint32_t sum = aom_highbd_sad16x8_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + src += src_stride << left_shift; + ref += ref_stride << left_shift; + second_pred += 16 << left_shift; + sum += aom_highbd_sad16x8_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + return sum; +} + +unsigned int aom_highbd_sad16x32_avg_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred) { + const int left_shift = 4; + uint32_t sum = aom_highbd_sad16x16_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + src += src_stride << left_shift; + ref += ref_stride << left_shift; + second_pred += 16 << left_shift; + sum += aom_highbd_sad16x16_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + return sum; +} + +unsigned int aom_highbd_sad32x16_avg_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred) { + __m256i sad = _mm256_setzero_si256(); + uint16_t *srcp = CONVERT_TO_SHORTPTR(src); + uint16_t *refp = CONVERT_TO_SHORTPTR(ref); + uint16_t *secp = CONVERT_TO_SHORTPTR(second_pred); + const int left_shift = 2; + int row_section = 0; + + while (row_section < 4) { + sad32x4(srcp, src_stride, refp, ref_stride, secp, &sad); + srcp += src_stride << left_shift; + refp += ref_stride << left_shift; + secp += 32 << left_shift; + row_section += 1; + } + return get_sad_from_mm256_epi32(&sad); +} + +unsigned int aom_highbd_sad32x32_avg_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred) { + const int left_shift = 4; + uint32_t sum = aom_highbd_sad32x16_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + src += src_stride << left_shift; + ref += ref_stride << left_shift; + second_pred += 32 << left_shift; + sum += aom_highbd_sad32x16_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + return sum; +} + +unsigned int aom_highbd_sad32x64_avg_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred) { + const int left_shift = 5; + uint32_t sum = aom_highbd_sad32x32_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + src += src_stride << left_shift; + ref += ref_stride << left_shift; + second_pred += 32 << left_shift; + sum += aom_highbd_sad32x32_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + return sum; +} + +unsigned int aom_highbd_sad64x32_avg_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred) { + __m256i sad = _mm256_setzero_si256(); + uint16_t *srcp = CONVERT_TO_SHORTPTR(src); + uint16_t *refp = CONVERT_TO_SHORTPTR(ref); + uint16_t *secp = CONVERT_TO_SHORTPTR(second_pred); + const int left_shift = 1; + int row_section = 0; + + while (row_section < 16) { + sad64x2(srcp, src_stride, refp, ref_stride, secp, &sad); + srcp += src_stride << left_shift; + refp += ref_stride << left_shift; + secp += 64 << left_shift; + row_section += 1; + } + return get_sad_from_mm256_epi32(&sad); +} + +unsigned int aom_highbd_sad64x64_avg_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred) { + const int left_shift = 5; + uint32_t sum = aom_highbd_sad64x32_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + src += src_stride << left_shift; + ref += ref_stride << left_shift; + second_pred += 64 << left_shift; + sum += aom_highbd_sad64x32_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + return sum; +} + +#if CONFIG_EXT_PARTITION +unsigned int aom_highbd_sad64x128_avg_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred) { + const int left_shift = 6; + uint32_t sum = aom_highbd_sad64x64_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + src += src_stride << left_shift; + ref += ref_stride << left_shift; + second_pred += 64 << left_shift; + sum += aom_highbd_sad64x64_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + return sum; +} + +unsigned int aom_highbd_sad128x64_avg_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred) { + __m256i sad = _mm256_setzero_si256(); + uint16_t *srcp = CONVERT_TO_SHORTPTR(src); + uint16_t *refp = CONVERT_TO_SHORTPTR(ref); + uint16_t *secp = CONVERT_TO_SHORTPTR(second_pred); + int row = 0; + while (row < 64) { + sad128x1(srcp, refp, secp, &sad); + srcp += src_stride; + refp += ref_stride; + secp += 16 << 3; + row += 1; + } + return get_sad_from_mm256_epi32(&sad); +} + +unsigned int aom_highbd_sad128x128_avg_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred) { + unsigned int sum; + const int left_shift = 6; + + sum = aom_highbd_sad128x64_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + src += src_stride << left_shift; + ref += ref_stride << left_shift; + second_pred += 128 << left_shift; + sum += aom_highbd_sad128x64_avg_avx2(src, src_stride, ref, ref_stride, + second_pred); + return sum; +} +#endif // CONFIG_EXT_PARTITION + +// SAD 4D +// Combine 4 __m256i vectors to uint32_t result[4] +static INLINE void get_4d_sad_from_mm256_epi32(const __m256i *v, + uint32_t *res) { + __m256i u0, u1, u2, u3; + const __m256i mask = _mm256_set1_epi64x(UINT32_MAX); + __m128i sad; + + // 8 32-bit summation + u0 = _mm256_srli_si256(v[0], 4); + u1 = _mm256_srli_si256(v[1], 4); + u2 = _mm256_srli_si256(v[2], 4); + u3 = _mm256_srli_si256(v[3], 4); + + u0 = _mm256_add_epi32(u0, v[0]); + u1 = _mm256_add_epi32(u1, v[1]); + u2 = _mm256_add_epi32(u2, v[2]); + u3 = _mm256_add_epi32(u3, v[3]); + + u0 = _mm256_and_si256(u0, mask); + u1 = _mm256_and_si256(u1, mask); + u2 = _mm256_and_si256(u2, mask); + u3 = _mm256_and_si256(u3, mask); + // 4 32-bit summation, evenly positioned + + u1 = _mm256_slli_si256(u1, 4); + u3 = _mm256_slli_si256(u3, 4); + + u0 = _mm256_or_si256(u0, u1); + u2 = _mm256_or_si256(u2, u3); + // 8 32-bit summation, interleaved + + u1 = _mm256_unpacklo_epi64(u0, u2); + u3 = _mm256_unpackhi_epi64(u0, u2); + + u0 = _mm256_add_epi32(u1, u3); + sad = _mm_add_epi32(_mm256_extractf128_si256(u0, 1), + _mm256_castsi256_si128(u0)); + _mm_storeu_si128((__m128i *)res, sad); +} + +static void convert_pointers(const uint8_t *const ref8[], + const uint16_t *ref[]) { + ref[0] = CONVERT_TO_SHORTPTR(ref8[0]); + ref[1] = CONVERT_TO_SHORTPTR(ref8[1]); + ref[2] = CONVERT_TO_SHORTPTR(ref8[2]); + ref[3] = CONVERT_TO_SHORTPTR(ref8[3]); +} + +static void init_sad(__m256i *s) { + s[0] = _mm256_setzero_si256(); + s[1] = _mm256_setzero_si256(); + s[2] = _mm256_setzero_si256(); + s[3] = _mm256_setzero_si256(); +} + +void aom_highbd_sad16x8x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref_array[], + int ref_stride, uint32_t *sad_array) { + __m256i sad_vec[4]; + const uint16_t *refp[4]; + const uint16_t *keep = CONVERT_TO_SHORTPTR(src); + const uint16_t *srcp; + const int shift_for_4_rows = 2; + int i; + + init_sad(sad_vec); + convert_pointers(ref_array, refp); + + for (i = 0; i < 4; ++i) { + srcp = keep; + sad16x4(srcp, src_stride, refp[i], ref_stride, 0, &sad_vec[i]); + srcp += src_stride << shift_for_4_rows; + refp[i] += ref_stride << shift_for_4_rows; + sad16x4(srcp, src_stride, refp[i], ref_stride, 0, &sad_vec[i]); + } + get_4d_sad_from_mm256_epi32(sad_vec, sad_array); +} + +void aom_highbd_sad16x16x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref_array[], + int ref_stride, uint32_t *sad_array) { + uint32_t first8rows[4]; + uint32_t second8rows[4]; + const uint8_t *ref[4]; + const int shift_for_8_rows = 3; + + ref[0] = ref_array[0]; + ref[1] = ref_array[1]; + ref[2] = ref_array[2]; + ref[3] = ref_array[3]; + + aom_highbd_sad16x8x4d_avx2(src, src_stride, ref, ref_stride, first8rows); + src += src_stride << shift_for_8_rows; + ref[0] += ref_stride << shift_for_8_rows; + ref[1] += ref_stride << shift_for_8_rows; + ref[2] += ref_stride << shift_for_8_rows; + ref[3] += ref_stride << shift_for_8_rows; + aom_highbd_sad16x8x4d_avx2(src, src_stride, ref, ref_stride, second8rows); + sad_array[0] = first8rows[0] + second8rows[0]; + sad_array[1] = first8rows[1] + second8rows[1]; + sad_array[2] = first8rows[2] + second8rows[2]; + sad_array[3] = first8rows[3] + second8rows[3]; +} + +void aom_highbd_sad16x32x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref_array[], + int ref_stride, uint32_t *sad_array) { + uint32_t first_half[4]; + uint32_t second_half[4]; + const uint8_t *ref[4]; + const int shift_for_rows = 4; + + ref[0] = ref_array[0]; + ref[1] = ref_array[1]; + ref[2] = ref_array[2]; + ref[3] = ref_array[3]; + + aom_highbd_sad16x16x4d_avx2(src, src_stride, ref, ref_stride, first_half); + src += src_stride << shift_for_rows; + ref[0] += ref_stride << shift_for_rows; + ref[1] += ref_stride << shift_for_rows; + ref[2] += ref_stride << shift_for_rows; + ref[3] += ref_stride << shift_for_rows; + aom_highbd_sad16x16x4d_avx2(src, src_stride, ref, ref_stride, second_half); + sad_array[0] = first_half[0] + second_half[0]; + sad_array[1] = first_half[1] + second_half[1]; + sad_array[2] = first_half[2] + second_half[2]; + sad_array[3] = first_half[3] + second_half[3]; +} + +void aom_highbd_sad32x16x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref_array[], + int ref_stride, uint32_t *sad_array) { + __m256i sad_vec[4]; + const uint16_t *refp[4]; + const uint16_t *keep = CONVERT_TO_SHORTPTR(src); + const uint16_t *srcp; + const int shift_for_4_rows = 2; + int i; + int rows_section; + + init_sad(sad_vec); + convert_pointers(ref_array, refp); + + for (i = 0; i < 4; ++i) { + srcp = keep; + rows_section = 0; + while (rows_section < 4) { + sad32x4(srcp, src_stride, refp[i], ref_stride, 0, &sad_vec[i]); + srcp += src_stride << shift_for_4_rows; + refp[i] += ref_stride << shift_for_4_rows; + rows_section++; + } + } + get_4d_sad_from_mm256_epi32(sad_vec, sad_array); +} + +void aom_highbd_sad32x32x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref_array[], + int ref_stride, uint32_t *sad_array) { + uint32_t first_half[4]; + uint32_t second_half[4]; + const uint8_t *ref[4]; + const int shift_for_rows = 4; + + ref[0] = ref_array[0]; + ref[1] = ref_array[1]; + ref[2] = ref_array[2]; + ref[3] = ref_array[3]; + + aom_highbd_sad32x16x4d_avx2(src, src_stride, ref, ref_stride, first_half); + src += src_stride << shift_for_rows; + ref[0] += ref_stride << shift_for_rows; + ref[1] += ref_stride << shift_for_rows; + ref[2] += ref_stride << shift_for_rows; + ref[3] += ref_stride << shift_for_rows; + aom_highbd_sad32x16x4d_avx2(src, src_stride, ref, ref_stride, second_half); + sad_array[0] = first_half[0] + second_half[0]; + sad_array[1] = first_half[1] + second_half[1]; + sad_array[2] = first_half[2] + second_half[2]; + sad_array[3] = first_half[3] + second_half[3]; +} + +void aom_highbd_sad32x64x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref_array[], + int ref_stride, uint32_t *sad_array) { + uint32_t first_half[4]; + uint32_t second_half[4]; + const uint8_t *ref[4]; + const int shift_for_rows = 5; + + ref[0] = ref_array[0]; + ref[1] = ref_array[1]; + ref[2] = ref_array[2]; + ref[3] = ref_array[3]; + + aom_highbd_sad32x32x4d_avx2(src, src_stride, ref, ref_stride, first_half); + src += src_stride << shift_for_rows; + ref[0] += ref_stride << shift_for_rows; + ref[1] += ref_stride << shift_for_rows; + ref[2] += ref_stride << shift_for_rows; + ref[3] += ref_stride << shift_for_rows; + aom_highbd_sad32x32x4d_avx2(src, src_stride, ref, ref_stride, second_half); + sad_array[0] = first_half[0] + second_half[0]; + sad_array[1] = first_half[1] + second_half[1]; + sad_array[2] = first_half[2] + second_half[2]; + sad_array[3] = first_half[3] + second_half[3]; +} + +void aom_highbd_sad64x32x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref_array[], + int ref_stride, uint32_t *sad_array) { + __m256i sad_vec[4]; + const uint16_t *refp[4]; + const uint16_t *keep = CONVERT_TO_SHORTPTR(src); + const uint16_t *srcp; + const int shift_for_rows = 1; + int i; + int rows_section; + + init_sad(sad_vec); + convert_pointers(ref_array, refp); + + for (i = 0; i < 4; ++i) { + srcp = keep; + rows_section = 0; + while (rows_section < 16) { + sad64x2(srcp, src_stride, refp[i], ref_stride, NULL, &sad_vec[i]); + srcp += src_stride << shift_for_rows; + refp[i] += ref_stride << shift_for_rows; + rows_section++; + } + } + get_4d_sad_from_mm256_epi32(sad_vec, sad_array); +} + +void aom_highbd_sad64x64x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref_array[], + int ref_stride, uint32_t *sad_array) { + uint32_t first_half[4]; + uint32_t second_half[4]; + const uint8_t *ref[4]; + const int shift_for_rows = 5; + + ref[0] = ref_array[0]; + ref[1] = ref_array[1]; + ref[2] = ref_array[2]; + ref[3] = ref_array[3]; + + aom_highbd_sad64x32x4d_avx2(src, src_stride, ref, ref_stride, first_half); + src += src_stride << shift_for_rows; + ref[0] += ref_stride << shift_for_rows; + ref[1] += ref_stride << shift_for_rows; + ref[2] += ref_stride << shift_for_rows; + ref[3] += ref_stride << shift_for_rows; + aom_highbd_sad64x32x4d_avx2(src, src_stride, ref, ref_stride, second_half); + sad_array[0] = first_half[0] + second_half[0]; + sad_array[1] = first_half[1] + second_half[1]; + sad_array[2] = first_half[2] + second_half[2]; + sad_array[3] = first_half[3] + second_half[3]; +} + +#if CONFIG_EXT_PARTITION +void aom_highbd_sad64x128x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref_array[], + int ref_stride, uint32_t *sad_array) { + uint32_t first_half[4]; + uint32_t second_half[4]; + const uint8_t *ref[4]; + const int shift_for_rows = 6; + + ref[0] = ref_array[0]; + ref[1] = ref_array[1]; + ref[2] = ref_array[2]; + ref[3] = ref_array[3]; + + aom_highbd_sad64x64x4d_avx2(src, src_stride, ref, ref_stride, first_half); + src += src_stride << shift_for_rows; + ref[0] += ref_stride << shift_for_rows; + ref[1] += ref_stride << shift_for_rows; + ref[2] += ref_stride << shift_for_rows; + ref[3] += ref_stride << shift_for_rows; + aom_highbd_sad64x64x4d_avx2(src, src_stride, ref, ref_stride, second_half); + sad_array[0] = first_half[0] + second_half[0]; + sad_array[1] = first_half[1] + second_half[1]; + sad_array[2] = first_half[2] + second_half[2]; + sad_array[3] = first_half[3] + second_half[3]; +} + +void aom_highbd_sad128x64x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref_array[], + int ref_stride, uint32_t *sad_array) { + __m256i sad_vec[4]; + const uint16_t *refp[4]; + const uint16_t *keep = CONVERT_TO_SHORTPTR(src); + const uint16_t *srcp; + int i; + int rows_section; + + init_sad(sad_vec); + convert_pointers(ref_array, refp); + + for (i = 0; i < 4; ++i) { + srcp = keep; + rows_section = 0; + while (rows_section < 64) { + sad128x1(srcp, refp[i], NULL, &sad_vec[i]); + srcp += src_stride; + refp[i] += ref_stride; + rows_section++; + } + } + get_4d_sad_from_mm256_epi32(sad_vec, sad_array); +} + +void aom_highbd_sad128x128x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref_array[], + int ref_stride, uint32_t *sad_array) { + uint32_t first_half[4]; + uint32_t second_half[4]; + const uint8_t *ref[4]; + const int shift_for_rows = 6; + + ref[0] = ref_array[0]; + ref[1] = ref_array[1]; + ref[2] = ref_array[2]; + ref[3] = ref_array[3]; + + aom_highbd_sad128x64x4d_avx2(src, src_stride, ref, ref_stride, first_half); + src += src_stride << shift_for_rows; + ref[0] += ref_stride << shift_for_rows; + ref[1] += ref_stride << shift_for_rows; + ref[2] += ref_stride << shift_for_rows; + ref[3] += ref_stride << shift_for_rows; + aom_highbd_sad128x64x4d_avx2(src, src_stride, ref, ref_stride, second_half); + sad_array[0] = first_half[0] + second_half[0]; + sad_array[1] = first_half[1] + second_half[1]; + sad_array[2] = first_half[2] + second_half[2]; + sad_array[3] = first_half[3] + second_half[3]; +} +#endif // CONFIG_EXT_PARTITION diff --git a/third_party/aom/aom_dsp/x86/sad_impl_avx2.c b/third_party/aom/aom_dsp/x86/sad_impl_avx2.c new file mode 100644 index 0000000000..4419c65b29 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/sad_impl_avx2.c @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./aom_dsp_rtcd.h" + +static unsigned int sad32x32(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride) { + __m256i s1, s2, r1, r2; + __m256i sum = _mm256_setzero_si256(); + __m128i sum_i128; + int i; + + for (i = 0; i < 16; ++i) { + r1 = _mm256_loadu_si256((__m256i const *)ref_ptr); + r2 = _mm256_loadu_si256((__m256i const *)(ref_ptr + ref_stride)); + s1 = _mm256_sad_epu8(r1, _mm256_loadu_si256((__m256i const *)src_ptr)); + s2 = _mm256_sad_epu8( + r2, _mm256_loadu_si256((__m256i const *)(src_ptr + src_stride))); + sum = _mm256_add_epi32(sum, _mm256_add_epi32(s1, s2)); + ref_ptr += ref_stride << 1; + src_ptr += src_stride << 1; + } + + sum = _mm256_add_epi32(sum, _mm256_srli_si256(sum, 8)); + sum_i128 = _mm_add_epi32(_mm256_extracti128_si256(sum, 1), + _mm256_castsi256_si128(sum)); + return _mm_cvtsi128_si32(sum_i128); +} + +static unsigned int sad64x32(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride) { + unsigned int half_width = 32; + uint32_t sum = sad32x32(src_ptr, src_stride, ref_ptr, ref_stride); + src_ptr += half_width; + ref_ptr += half_width; + sum += sad32x32(src_ptr, src_stride, ref_ptr, ref_stride); + return sum; +} + +static unsigned int sad64x64(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride) { + uint32_t sum = sad64x32(src_ptr, src_stride, ref_ptr, ref_stride); + src_ptr += src_stride << 5; + ref_ptr += ref_stride << 5; + sum += sad64x32(src_ptr, src_stride, ref_ptr, ref_stride); + return sum; +} + +unsigned int aom_sad128x64_avx2(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride) { + unsigned int half_width = 64; + uint32_t sum = sad64x64(src_ptr, src_stride, ref_ptr, ref_stride); + src_ptr += half_width; + ref_ptr += half_width; + sum += sad64x64(src_ptr, src_stride, ref_ptr, ref_stride); + return sum; +} + +unsigned int aom_sad64x128_avx2(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride) { + uint32_t sum = sad64x64(src_ptr, src_stride, ref_ptr, ref_stride); + src_ptr += src_stride << 6; + ref_ptr += ref_stride << 6; + sum += sad64x64(src_ptr, src_stride, ref_ptr, ref_stride); + return sum; +} + +unsigned int aom_sad128x128_avx2(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride) { + uint32_t sum = aom_sad128x64_avx2(src_ptr, src_stride, ref_ptr, ref_stride); + src_ptr += src_stride << 6; + ref_ptr += ref_stride << 6; + sum += aom_sad128x64_avx2(src_ptr, src_stride, ref_ptr, ref_stride); + return sum; +} + +static void sad64x64x4d(const uint8_t *src, int src_stride, + const uint8_t *const ref[4], int ref_stride, + __m128i *res) { + uint32_t sum[4]; + aom_sad64x64x4d_avx2(src, src_stride, ref, ref_stride, sum); + *res = _mm_loadu_si128((const __m128i *)sum); +} + +void aom_sad64x128x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref[4], int ref_stride, + uint32_t res[4]) { + __m128i sum0, sum1; + const uint8_t *rf[4]; + + rf[0] = ref[0]; + rf[1] = ref[1]; + rf[2] = ref[2]; + rf[3] = ref[3]; + sad64x64x4d(src, src_stride, rf, ref_stride, &sum0); + src += src_stride << 6; + rf[0] += ref_stride << 6; + rf[1] += ref_stride << 6; + rf[2] += ref_stride << 6; + rf[3] += ref_stride << 6; + sad64x64x4d(src, src_stride, rf, ref_stride, &sum1); + sum0 = _mm_add_epi32(sum0, sum1); + _mm_storeu_si128((__m128i *)res, sum0); +} + +void aom_sad128x64x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref[4], int ref_stride, + uint32_t res[4]) { + __m128i sum0, sum1; + unsigned int half_width = 64; + const uint8_t *rf[4]; + + rf[0] = ref[0]; + rf[1] = ref[1]; + rf[2] = ref[2]; + rf[3] = ref[3]; + sad64x64x4d(src, src_stride, rf, ref_stride, &sum0); + src += half_width; + rf[0] += half_width; + rf[1] += half_width; + rf[2] += half_width; + rf[3] += half_width; + sad64x64x4d(src, src_stride, rf, ref_stride, &sum1); + sum0 = _mm_add_epi32(sum0, sum1); + _mm_storeu_si128((__m128i *)res, sum0); +} + +void aom_sad128x128x4d_avx2(const uint8_t *src, int src_stride, + const uint8_t *const ref[4], int ref_stride, + uint32_t res[4]) { + const uint8_t *rf[4]; + uint32_t sum0[4]; + uint32_t sum1[4]; + + rf[0] = ref[0]; + rf[1] = ref[1]; + rf[2] = ref[2]; + rf[3] = ref[3]; + aom_sad128x64x4d_avx2(src, src_stride, rf, ref_stride, sum0); + src += src_stride << 6; + rf[0] += ref_stride << 6; + rf[1] += ref_stride << 6; + rf[2] += ref_stride << 6; + rf[3] += ref_stride << 6; + aom_sad128x64x4d_avx2(src, src_stride, rf, ref_stride, sum1); + res[0] = sum0[0] + sum1[0]; + res[1] = sum0[1] + sum1[1]; + res[2] = sum0[2] + sum1[2]; + res[3] = sum0[3] + sum1[3]; +} + +static unsigned int sad_w64_avg_avx2(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + const int h, const uint8_t *second_pred, + const int second_pred_stride) { + int i, res; + __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; + __m256i sum_sad = _mm256_setzero_si256(); + __m256i sum_sad_h; + __m128i sum_sad128; + for (i = 0; i < h; i++) { + ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); + ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + 32)); + ref1_reg = _mm256_avg_epu8( + ref1_reg, _mm256_loadu_si256((__m256i const *)second_pred)); + ref2_reg = _mm256_avg_epu8( + ref2_reg, _mm256_loadu_si256((__m256i const *)(second_pred + 32))); + sad1_reg = + _mm256_sad_epu8(ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); + sad2_reg = _mm256_sad_epu8( + ref2_reg, _mm256_loadu_si256((__m256i const *)(src_ptr + 32))); + sum_sad = _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); + ref_ptr += ref_stride; + src_ptr += src_stride; + second_pred += second_pred_stride; + } + sum_sad_h = _mm256_srli_si256(sum_sad, 8); + sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); + sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); + sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); + res = _mm_cvtsi128_si32(sum_sad128); + + return res; +} + +unsigned int aom_sad64x128_avg_avx2(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + const uint8_t *second_pred) { + uint32_t sum = sad_w64_avg_avx2(src_ptr, src_stride, ref_ptr, ref_stride, 64, + second_pred, 64); + src_ptr += src_stride << 6; + ref_ptr += ref_stride << 6; + second_pred += 64 << 6; + sum += sad_w64_avg_avx2(src_ptr, src_stride, ref_ptr, ref_stride, 64, + second_pred, 64); + return sum; +} + +unsigned int aom_sad128x64_avg_avx2(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + const uint8_t *second_pred) { + unsigned int half_width = 64; + uint32_t sum = sad_w64_avg_avx2(src_ptr, src_stride, ref_ptr, ref_stride, 64, + second_pred, 128); + src_ptr += half_width; + ref_ptr += half_width; + second_pred += half_width; + sum += sad_w64_avg_avx2(src_ptr, src_stride, ref_ptr, ref_stride, 64, + second_pred, 128); + return sum; +} + +unsigned int aom_sad128x128_avg_avx2(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + const uint8_t *second_pred) { + uint32_t sum = aom_sad128x64_avg_avx2(src_ptr, src_stride, ref_ptr, + ref_stride, second_pred); + src_ptr += src_stride << 6; + ref_ptr += ref_stride << 6; + second_pred += 128 << 6; + sum += aom_sad128x64_avg_avx2(src_ptr, src_stride, ref_ptr, ref_stride, + second_pred); + return sum; +} diff --git a/third_party/aom/aom_dsp/x86/sad_sse2.asm b/third_party/aom/aom_dsp/x86/sad_sse2.asm new file mode 100644 index 0000000000..e45457a572 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/sad_sse2.asm @@ -0,0 +1,345 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION .text + +%macro SAD_FN 4 +%if %4 == 0 +%if %3 == 5 +cglobal sad%1x%2, 4, %3, 5, src, src_stride, ref, ref_stride, n_rows +%else ; %3 == 7 +cglobal sad%1x%2, 4, %3, 6, src, src_stride, ref, ref_stride, \ + src_stride3, ref_stride3, n_rows +%endif ; %3 == 5/7 +%else ; avg +%if %3 == 5 +cglobal sad%1x%2_avg, 5, 1 + %3, 5, src, src_stride, ref, ref_stride, \ + second_pred, n_rows +%else ; %3 == 7 +cglobal sad%1x%2_avg, 5, ARCH_X86_64 + %3, 6, src, src_stride, \ + ref, ref_stride, \ + second_pred, \ + src_stride3, ref_stride3 +%if ARCH_X86_64 +%define n_rowsd r7d +%else ; x86-32 +%define n_rowsd dword r0m +%endif ; x86-32/64 +%endif ; %3 == 5/7 +%endif ; avg/sad + movsxdifnidn src_strideq, src_strided + movsxdifnidn ref_strideq, ref_strided +%if %3 == 7 + lea src_stride3q, [src_strideq*3] + lea ref_stride3q, [ref_strideq*3] +%endif ; %3 == 7 +%endmacro + +%if CONFIG_EXT_PARTITION +; unsigned int aom_sad128x128_sse2(uint8_t *src, int src_stride, +; uint8_t *ref, int ref_stride); +%macro SAD128XN 1-2 0 + SAD_FN 128, %1, 5, %2 + mov n_rowsd, %1 + pxor m0, m0 + +.loop: + movu m1, [refq] + movu m2, [refq+16] + movu m3, [refq+32] + movu m4, [refq+48] +%if %2 == 1 + pavgb m1, [second_predq+mmsize*0] + pavgb m2, [second_predq+mmsize*1] + pavgb m3, [second_predq+mmsize*2] + pavgb m4, [second_predq+mmsize*3] +%endif + psadbw m1, [srcq] + psadbw m2, [srcq+16] + psadbw m3, [srcq+32] + psadbw m4, [srcq+48] + + paddd m1, m2 + paddd m3, m4 + paddd m0, m1 + paddd m0, m3 + + movu m1, [refq+64] + movu m2, [refq+80] + movu m3, [refq+96] + movu m4, [refq+112] +%if %2 == 1 + pavgb m1, [second_predq+mmsize*4] + pavgb m2, [second_predq+mmsize*5] + pavgb m3, [second_predq+mmsize*6] + pavgb m4, [second_predq+mmsize*7] + lea second_predq, [second_predq+mmsize*8] +%endif + psadbw m1, [srcq+64] + psadbw m2, [srcq+80] + psadbw m3, [srcq+96] + psadbw m4, [srcq+112] + + add refq, ref_strideq + add srcq, src_strideq + + paddd m1, m2 + paddd m3, m4 + paddd m0, m1 + paddd m0, m3 + + sub n_rowsd, 1 + jg .loop + + movhlps m1, m0 + paddd m0, m1 + movd eax, m0 + RET +%endmacro + +INIT_XMM sse2 +SAD128XN 128 ; sad128x128_sse2 +SAD128XN 128, 1 ; sad128x128_avg_sse2 +SAD128XN 64 ; sad128x64_sse2 +SAD128XN 64, 1 ; sad128x64_avg_sse2 +%endif + + +; unsigned int aom_sad64x64_sse2(uint8_t *src, int src_stride, +; uint8_t *ref, int ref_stride); +%macro SAD64XN 1-2 0 + SAD_FN 64, %1, 5, %2 + mov n_rowsd, %1 + pxor m0, m0 +.loop: + movu m1, [refq] + movu m2, [refq+16] + movu m3, [refq+32] + movu m4, [refq+48] +%if %2 == 1 + pavgb m1, [second_predq+mmsize*0] + pavgb m2, [second_predq+mmsize*1] + pavgb m3, [second_predq+mmsize*2] + pavgb m4, [second_predq+mmsize*3] + lea second_predq, [second_predq+mmsize*4] +%endif + psadbw m1, [srcq] + psadbw m2, [srcq+16] + psadbw m3, [srcq+32] + psadbw m4, [srcq+48] + paddd m1, m2 + paddd m3, m4 + add refq, ref_strideq + paddd m0, m1 + add srcq, src_strideq + paddd m0, m3 + dec n_rowsd + jg .loop + + movhlps m1, m0 + paddd m0, m1 + movd eax, m0 + RET +%endmacro + +INIT_XMM sse2 +%if CONFIG_EXT_PARTITION +SAD64XN 128 ; sad64x128_sse2 +SAD64XN 128, 1 ; sad64x128_avg_sse2 +%endif +SAD64XN 64 ; sad64x64_sse2 +SAD64XN 32 ; sad64x32_sse2 +SAD64XN 64, 1 ; sad64x64_avg_sse2 +SAD64XN 32, 1 ; sad64x32_avg_sse2 + +; unsigned int aom_sad32x32_sse2(uint8_t *src, int src_stride, +; uint8_t *ref, int ref_stride); +%macro SAD32XN 1-2 0 + SAD_FN 32, %1, 5, %2 + mov n_rowsd, %1/2 + pxor m0, m0 +.loop: + movu m1, [refq] + movu m2, [refq+16] + movu m3, [refq+ref_strideq] + movu m4, [refq+ref_strideq+16] +%if %2 == 1 + pavgb m1, [second_predq+mmsize*0] + pavgb m2, [second_predq+mmsize*1] + pavgb m3, [second_predq+mmsize*2] + pavgb m4, [second_predq+mmsize*3] + lea second_predq, [second_predq+mmsize*4] +%endif + psadbw m1, [srcq] + psadbw m2, [srcq+16] + psadbw m3, [srcq+src_strideq] + psadbw m4, [srcq+src_strideq+16] + paddd m1, m2 + paddd m3, m4 + lea refq, [refq+ref_strideq*2] + paddd m0, m1 + lea srcq, [srcq+src_strideq*2] + paddd m0, m3 + dec n_rowsd + jg .loop + + movhlps m1, m0 + paddd m0, m1 + movd eax, m0 + RET +%endmacro + +INIT_XMM sse2 +SAD32XN 64 ; sad32x64_sse2 +SAD32XN 32 ; sad32x32_sse2 +SAD32XN 16 ; sad32x16_sse2 +SAD32XN 64, 1 ; sad32x64_avg_sse2 +SAD32XN 32, 1 ; sad32x32_avg_sse2 +SAD32XN 16, 1 ; sad32x16_avg_sse2 + +; unsigned int aom_sad16x{8,16}_sse2(uint8_t *src, int src_stride, +; uint8_t *ref, int ref_stride); +%macro SAD16XN 1-2 0 + SAD_FN 16, %1, 7, %2 + mov n_rowsd, %1/4 + pxor m0, m0 + +.loop: + movu m1, [refq] + movu m2, [refq+ref_strideq] + movu m3, [refq+ref_strideq*2] + movu m4, [refq+ref_stride3q] +%if %2 == 1 + pavgb m1, [second_predq+mmsize*0] + pavgb m2, [second_predq+mmsize*1] + pavgb m3, [second_predq+mmsize*2] + pavgb m4, [second_predq+mmsize*3] + lea second_predq, [second_predq+mmsize*4] +%endif + psadbw m1, [srcq] + psadbw m2, [srcq+src_strideq] + psadbw m3, [srcq+src_strideq*2] + psadbw m4, [srcq+src_stride3q] + paddd m1, m2 + paddd m3, m4 + lea refq, [refq+ref_strideq*4] + paddd m0, m1 + lea srcq, [srcq+src_strideq*4] + paddd m0, m3 + dec n_rowsd + jg .loop + + movhlps m1, m0 + paddd m0, m1 + movd eax, m0 + RET +%endmacro + +INIT_XMM sse2 +SAD16XN 32 ; sad16x32_sse2 +SAD16XN 16 ; sad16x16_sse2 +SAD16XN 8 ; sad16x8_sse2 +SAD16XN 32, 1 ; sad16x32_avg_sse2 +SAD16XN 16, 1 ; sad16x16_avg_sse2 +SAD16XN 8, 1 ; sad16x8_avg_sse2 + +; unsigned int aom_sad8x{8,16}_sse2(uint8_t *src, int src_stride, +; uint8_t *ref, int ref_stride); +%macro SAD8XN 1-2 0 + SAD_FN 8, %1, 7, %2 + mov n_rowsd, %1/4 + pxor m0, m0 + +.loop: + movh m1, [refq] + movhps m1, [refq+ref_strideq] + movh m2, [refq+ref_strideq*2] + movhps m2, [refq+ref_stride3q] +%if %2 == 1 + pavgb m1, [second_predq+mmsize*0] + pavgb m2, [second_predq+mmsize*1] + lea second_predq, [second_predq+mmsize*2] +%endif + movh m3, [srcq] + movhps m3, [srcq+src_strideq] + movh m4, [srcq+src_strideq*2] + movhps m4, [srcq+src_stride3q] + psadbw m1, m3 + psadbw m2, m4 + lea refq, [refq+ref_strideq*4] + paddd m0, m1 + lea srcq, [srcq+src_strideq*4] + paddd m0, m2 + dec n_rowsd + jg .loop + + movhlps m1, m0 + paddd m0, m1 + movd eax, m0 + RET +%endmacro + +INIT_XMM sse2 +SAD8XN 16 ; sad8x16_sse2 +SAD8XN 8 ; sad8x8_sse2 +SAD8XN 4 ; sad8x4_sse2 +SAD8XN 16, 1 ; sad8x16_avg_sse2 +SAD8XN 8, 1 ; sad8x8_avg_sse2 +SAD8XN 4, 1 ; sad8x4_avg_sse2 + +; unsigned int aom_sad4x{4, 8}_sse2(uint8_t *src, int src_stride, +; uint8_t *ref, int ref_stride); +%macro SAD4XN 1-2 0 + SAD_FN 4, %1, 7, %2 + mov n_rowsd, %1/4 + pxor m0, m0 + +.loop: + movd m1, [refq] + movd m2, [refq+ref_strideq] + movd m3, [refq+ref_strideq*2] + movd m4, [refq+ref_stride3q] + punpckldq m1, m2 + punpckldq m3, m4 + movlhps m1, m3 +%if %2 == 1 + pavgb m1, [second_predq+mmsize*0] + lea second_predq, [second_predq+mmsize*1] +%endif + movd m2, [srcq] + movd m5, [srcq+src_strideq] + movd m4, [srcq+src_strideq*2] + movd m3, [srcq+src_stride3q] + punpckldq m2, m5 + punpckldq m4, m3 + movlhps m2, m4 + psadbw m1, m2 + lea refq, [refq+ref_strideq*4] + paddd m0, m1 + lea srcq, [srcq+src_strideq*4] + dec n_rowsd + jg .loop + + movhlps m1, m0 + paddd m0, m1 + movd eax, m0 + RET +%endmacro + +INIT_XMM sse2 +SAD4XN 8 ; sad4x8_sse +SAD4XN 4 ; sad4x4_sse +SAD4XN 8, 1 ; sad4x8_avg_sse +SAD4XN 4, 1 ; sad4x4_avg_sse diff --git a/third_party/aom/aom_dsp/x86/sad_sse3.asm b/third_party/aom/aom_dsp/x86/sad_sse3.asm new file mode 100644 index 0000000000..f6c27c855a --- /dev/null +++ b/third_party/aom/aom_dsp/x86/sad_sse3.asm @@ -0,0 +1,377 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "aom_ports/x86_abi_support.asm" + +%macro STACK_FRAME_CREATE_X3 0 +%if ABI_IS_32BIT + %define src_ptr rsi + %define src_stride rax + %define ref_ptr rdi + %define ref_stride rdx + %define end_ptr rcx + %define ret_var rbx + %define result_ptr arg(4) + %define height dword ptr arg(4) + push rbp + mov rbp, rsp + push rsi + push rdi + push rbx + + mov rsi, arg(0) ; src_ptr + mov rdi, arg(2) ; ref_ptr + + movsxd rax, dword ptr arg(1) ; src_stride + movsxd rdx, dword ptr arg(3) ; ref_stride +%else + %if LIBAOM_YASM_WIN64 + SAVE_XMM 7, u + %define src_ptr rcx + %define src_stride rdx + %define ref_ptr r8 + %define ref_stride r9 + %define end_ptr r10 + %define ret_var r11 + %define result_ptr [rsp+xmm_stack_space+8+4*8] + %define height dword ptr [rsp+xmm_stack_space+8+4*8] + %else + %define src_ptr rdi + %define src_stride rsi + %define ref_ptr rdx + %define ref_stride rcx + %define end_ptr r9 + %define ret_var r10 + %define result_ptr r8 + %define height r8 + %endif +%endif + +%endmacro + +%macro STACK_FRAME_DESTROY_X3 0 + %define src_ptr + %define src_stride + %define ref_ptr + %define ref_stride + %define end_ptr + %define ret_var + %define result_ptr + %define height + +%if ABI_IS_32BIT + pop rbx + pop rdi + pop rsi + pop rbp +%else + %if LIBAOM_YASM_WIN64 + RESTORE_XMM + %endif +%endif + ret +%endmacro + +%macro PROCESS_16X2X3 5 +%if %1==0 + movdqa xmm0, XMMWORD PTR [%2] + lddqu xmm5, XMMWORD PTR [%3] + lddqu xmm6, XMMWORD PTR [%3+1] + lddqu xmm7, XMMWORD PTR [%3+2] + + psadbw xmm5, xmm0 + psadbw xmm6, xmm0 + psadbw xmm7, xmm0 +%else + movdqa xmm0, XMMWORD PTR [%2] + lddqu xmm1, XMMWORD PTR [%3] + lddqu xmm2, XMMWORD PTR [%3+1] + lddqu xmm3, XMMWORD PTR [%3+2] + + psadbw xmm1, xmm0 + psadbw xmm2, xmm0 + psadbw xmm3, xmm0 + + paddw xmm5, xmm1 + paddw xmm6, xmm2 + paddw xmm7, xmm3 +%endif + movdqa xmm0, XMMWORD PTR [%2+%4] + lddqu xmm1, XMMWORD PTR [%3+%5] + lddqu xmm2, XMMWORD PTR [%3+%5+1] + lddqu xmm3, XMMWORD PTR [%3+%5+2] + +%if %1==0 || %1==1 + lea %2, [%2+%4*2] + lea %3, [%3+%5*2] +%endif + + psadbw xmm1, xmm0 + psadbw xmm2, xmm0 + psadbw xmm3, xmm0 + + paddw xmm5, xmm1 + paddw xmm6, xmm2 + paddw xmm7, xmm3 +%endmacro + +%macro PROCESS_8X2X3 5 +%if %1==0 + movq mm0, QWORD PTR [%2] + movq mm5, QWORD PTR [%3] + movq mm6, QWORD PTR [%3+1] + movq mm7, QWORD PTR [%3+2] + + psadbw mm5, mm0 + psadbw mm6, mm0 + psadbw mm7, mm0 +%else + movq mm0, QWORD PTR [%2] + movq mm1, QWORD PTR [%3] + movq mm2, QWORD PTR [%3+1] + movq mm3, QWORD PTR [%3+2] + + psadbw mm1, mm0 + psadbw mm2, mm0 + psadbw mm3, mm0 + + paddw mm5, mm1 + paddw mm6, mm2 + paddw mm7, mm3 +%endif + movq mm0, QWORD PTR [%2+%4] + movq mm1, QWORD PTR [%3+%5] + movq mm2, QWORD PTR [%3+%5+1] + movq mm3, QWORD PTR [%3+%5+2] + +%if %1==0 || %1==1 + lea %2, [%2+%4*2] + lea %3, [%3+%5*2] +%endif + + psadbw mm1, mm0 + psadbw mm2, mm0 + psadbw mm3, mm0 + + paddw mm5, mm1 + paddw mm6, mm2 + paddw mm7, mm3 +%endmacro + +;void int aom_sad16x16x3_sse3( +; unsigned char *src_ptr, +; int src_stride, +; unsigned char *ref_ptr, +; int ref_stride, +; int *results) +global sym(aom_sad16x16x3_sse3) PRIVATE +sym(aom_sad16x16x3_sse3): + + STACK_FRAME_CREATE_X3 + + PROCESS_16X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_16X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride + + mov rcx, result_ptr + + movq xmm0, xmm5 + psrldq xmm5, 8 + + paddw xmm0, xmm5 + movd [rcx], xmm0 +;- + movq xmm0, xmm6 + psrldq xmm6, 8 + + paddw xmm0, xmm6 + movd [rcx+4], xmm0 +;- + movq xmm0, xmm7 + psrldq xmm7, 8 + + paddw xmm0, xmm7 + movd [rcx+8], xmm0 + + STACK_FRAME_DESTROY_X3 + +;void int aom_sad16x8x3_sse3( +; unsigned char *src_ptr, +; int src_stride, +; unsigned char *ref_ptr, +; int ref_stride, +; int *results) +global sym(aom_sad16x8x3_sse3) PRIVATE +sym(aom_sad16x8x3_sse3): + + STACK_FRAME_CREATE_X3 + + PROCESS_16X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_16X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride + + mov rcx, result_ptr + + movq xmm0, xmm5 + psrldq xmm5, 8 + + paddw xmm0, xmm5 + movd [rcx], xmm0 +;- + movq xmm0, xmm6 + psrldq xmm6, 8 + + paddw xmm0, xmm6 + movd [rcx+4], xmm0 +;- + movq xmm0, xmm7 + psrldq xmm7, 8 + + paddw xmm0, xmm7 + movd [rcx+8], xmm0 + + STACK_FRAME_DESTROY_X3 + +;void int aom_sad8x16x3_sse3( +; unsigned char *src_ptr, +; int src_stride, +; unsigned char *ref_ptr, +; int ref_stride, +; int *results) +global sym(aom_sad8x16x3_sse3) PRIVATE +sym(aom_sad8x16x3_sse3): + + STACK_FRAME_CREATE_X3 + + PROCESS_8X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_8X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride + + mov rcx, result_ptr + + punpckldq mm5, mm6 + + movq [rcx], mm5 + movd [rcx+8], mm7 + + STACK_FRAME_DESTROY_X3 + +;void int aom_sad8x8x3_sse3( +; unsigned char *src_ptr, +; int src_stride, +; unsigned char *ref_ptr, +; int ref_stride, +; int *results) +global sym(aom_sad8x8x3_sse3) PRIVATE +sym(aom_sad8x8x3_sse3): + + STACK_FRAME_CREATE_X3 + + PROCESS_8X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride + PROCESS_8X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride + + mov rcx, result_ptr + + punpckldq mm5, mm6 + + movq [rcx], mm5 + movd [rcx+8], mm7 + + STACK_FRAME_DESTROY_X3 + +;void int aom_sad4x4x3_sse3( +; unsigned char *src_ptr, +; int src_stride, +; unsigned char *ref_ptr, +; int ref_stride, +; int *results) +global sym(aom_sad4x4x3_sse3) PRIVATE +sym(aom_sad4x4x3_sse3): + + STACK_FRAME_CREATE_X3 + + movd mm0, DWORD PTR [src_ptr] + movd mm1, DWORD PTR [ref_ptr] + + movd mm2, DWORD PTR [src_ptr+src_stride] + movd mm3, DWORD PTR [ref_ptr+ref_stride] + + punpcklbw mm0, mm2 + punpcklbw mm1, mm3 + + movd mm4, DWORD PTR [ref_ptr+1] + movd mm5, DWORD PTR [ref_ptr+2] + + movd mm2, DWORD PTR [ref_ptr+ref_stride+1] + movd mm3, DWORD PTR [ref_ptr+ref_stride+2] + + psadbw mm1, mm0 + + punpcklbw mm4, mm2 + punpcklbw mm5, mm3 + + psadbw mm4, mm0 + psadbw mm5, mm0 + + lea src_ptr, [src_ptr+src_stride*2] + lea ref_ptr, [ref_ptr+ref_stride*2] + + movd mm0, DWORD PTR [src_ptr] + movd mm2, DWORD PTR [ref_ptr] + + movd mm3, DWORD PTR [src_ptr+src_stride] + movd mm6, DWORD PTR [ref_ptr+ref_stride] + + punpcklbw mm0, mm3 + punpcklbw mm2, mm6 + + movd mm3, DWORD PTR [ref_ptr+1] + movd mm7, DWORD PTR [ref_ptr+2] + + psadbw mm2, mm0 + + paddw mm1, mm2 + + movd mm2, DWORD PTR [ref_ptr+ref_stride+1] + movd mm6, DWORD PTR [ref_ptr+ref_stride+2] + + punpcklbw mm3, mm2 + punpcklbw mm7, mm6 + + psadbw mm3, mm0 + psadbw mm7, mm0 + + paddw mm3, mm4 + paddw mm7, mm5 + + mov rcx, result_ptr + + punpckldq mm1, mm3 + + movq [rcx], mm1 + movd [rcx+8], mm7 + + STACK_FRAME_DESTROY_X3 diff --git a/third_party/aom/aom_dsp/x86/sad_sse4.asm b/third_party/aom/aom_dsp/x86/sad_sse4.asm new file mode 100644 index 0000000000..5e9c758451 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/sad_sse4.asm @@ -0,0 +1,362 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + +%include "aom_ports/x86_abi_support.asm" + +%macro PROCESS_16X2X8 1 +%if %1 + movdqa xmm0, XMMWORD PTR [rsi] + movq xmm1, MMWORD PTR [rdi] + movq xmm3, MMWORD PTR [rdi+8] + movq xmm2, MMWORD PTR [rdi+16] + punpcklqdq xmm1, xmm3 + punpcklqdq xmm3, xmm2 + + movdqa xmm2, xmm1 + mpsadbw xmm1, xmm0, 0x0 + mpsadbw xmm2, xmm0, 0x5 + + psrldq xmm0, 8 + + movdqa xmm4, xmm3 + mpsadbw xmm3, xmm0, 0x0 + mpsadbw xmm4, xmm0, 0x5 + + paddw xmm1, xmm2 + paddw xmm1, xmm3 + paddw xmm1, xmm4 +%else + movdqa xmm0, XMMWORD PTR [rsi] + movq xmm5, MMWORD PTR [rdi] + movq xmm3, MMWORD PTR [rdi+8] + movq xmm2, MMWORD PTR [rdi+16] + punpcklqdq xmm5, xmm3 + punpcklqdq xmm3, xmm2 + + movdqa xmm2, xmm5 + mpsadbw xmm5, xmm0, 0x0 + mpsadbw xmm2, xmm0, 0x5 + + psrldq xmm0, 8 + + movdqa xmm4, xmm3 + mpsadbw xmm3, xmm0, 0x0 + mpsadbw xmm4, xmm0, 0x5 + + paddw xmm5, xmm2 + paddw xmm5, xmm3 + paddw xmm5, xmm4 + + paddw xmm1, xmm5 +%endif + movdqa xmm0, XMMWORD PTR [rsi + rax] + movq xmm5, MMWORD PTR [rdi+ rdx] + movq xmm3, MMWORD PTR [rdi+ rdx+8] + movq xmm2, MMWORD PTR [rdi+ rdx+16] + punpcklqdq xmm5, xmm3 + punpcklqdq xmm3, xmm2 + + lea rsi, [rsi+rax*2] + lea rdi, [rdi+rdx*2] + + movdqa xmm2, xmm5 + mpsadbw xmm5, xmm0, 0x0 + mpsadbw xmm2, xmm0, 0x5 + + psrldq xmm0, 8 + movdqa xmm4, xmm3 + mpsadbw xmm3, xmm0, 0x0 + mpsadbw xmm4, xmm0, 0x5 + + paddw xmm5, xmm2 + paddw xmm5, xmm3 + paddw xmm5, xmm4 + + paddw xmm1, xmm5 +%endmacro + +%macro PROCESS_8X2X8 1 +%if %1 + movq xmm0, MMWORD PTR [rsi] + movq xmm1, MMWORD PTR [rdi] + movq xmm3, MMWORD PTR [rdi+8] + punpcklqdq xmm1, xmm3 + + movdqa xmm2, xmm1 + mpsadbw xmm1, xmm0, 0x0 + mpsadbw xmm2, xmm0, 0x5 + paddw xmm1, xmm2 +%else + movq xmm0, MMWORD PTR [rsi] + movq xmm5, MMWORD PTR [rdi] + movq xmm3, MMWORD PTR [rdi+8] + punpcklqdq xmm5, xmm3 + + movdqa xmm2, xmm5 + mpsadbw xmm5, xmm0, 0x0 + mpsadbw xmm2, xmm0, 0x5 + paddw xmm5, xmm2 + + paddw xmm1, xmm5 +%endif + movq xmm0, MMWORD PTR [rsi + rax] + movq xmm5, MMWORD PTR [rdi+ rdx] + movq xmm3, MMWORD PTR [rdi+ rdx+8] + punpcklqdq xmm5, xmm3 + + lea rsi, [rsi+rax*2] + lea rdi, [rdi+rdx*2] + + movdqa xmm2, xmm5 + mpsadbw xmm5, xmm0, 0x0 + mpsadbw xmm2, xmm0, 0x5 + paddw xmm5, xmm2 + + paddw xmm1, xmm5 +%endmacro + +%macro PROCESS_4X2X8 1 +%if %1 + movd xmm0, [rsi] + movq xmm1, MMWORD PTR [rdi] + movq xmm3, MMWORD PTR [rdi+8] + punpcklqdq xmm1, xmm3 + + mpsadbw xmm1, xmm0, 0x0 +%else + movd xmm0, [rsi] + movq xmm5, MMWORD PTR [rdi] + movq xmm3, MMWORD PTR [rdi+8] + punpcklqdq xmm5, xmm3 + + mpsadbw xmm5, xmm0, 0x0 + + paddw xmm1, xmm5 +%endif + movd xmm0, [rsi + rax] + movq xmm5, MMWORD PTR [rdi+ rdx] + movq xmm3, MMWORD PTR [rdi+ rdx+8] + punpcklqdq xmm5, xmm3 + + lea rsi, [rsi+rax*2] + lea rdi, [rdi+rdx*2] + + mpsadbw xmm5, xmm0, 0x0 + + paddw xmm1, xmm5 +%endmacro + +%macro WRITE_AS_INTS 0 + mov rdi, arg(4) ;Results + pxor xmm0, xmm0 + movdqa xmm2, xmm1 + punpcklwd xmm1, xmm0 + punpckhwd xmm2, xmm0 + + movdqa [rdi], xmm1 + movdqa [rdi + 16], xmm2 +%endmacro + +;void aom_sad16x16x8_sse4_1( +; const unsigned char *src_ptr, +; int src_stride, +; const unsigned char *ref_ptr, +; int ref_stride, +; unsigned short *sad_array); +global sym(aom_sad16x16x8_sse4_1) PRIVATE +sym(aom_sad16x16x8_sse4_1): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 5 + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;ref_ptr + + movsxd rax, dword ptr arg(1) ;src_stride + movsxd rdx, dword ptr arg(3) ;ref_stride + + PROCESS_16X2X8 1 + PROCESS_16X2X8 0 + PROCESS_16X2X8 0 + PROCESS_16X2X8 0 + PROCESS_16X2X8 0 + PROCESS_16X2X8 0 + PROCESS_16X2X8 0 + PROCESS_16X2X8 0 + + WRITE_AS_INTS + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + + +;void aom_sad16x8x8_sse4_1( +; const unsigned char *src_ptr, +; int src_stride, +; const unsigned char *ref_ptr, +; int ref_stride, +; unsigned short *sad_array +;); +global sym(aom_sad16x8x8_sse4_1) PRIVATE +sym(aom_sad16x8x8_sse4_1): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 5 + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;ref_ptr + + movsxd rax, dword ptr arg(1) ;src_stride + movsxd rdx, dword ptr arg(3) ;ref_stride + + PROCESS_16X2X8 1 + PROCESS_16X2X8 0 + PROCESS_16X2X8 0 + PROCESS_16X2X8 0 + + WRITE_AS_INTS + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + + +;void aom_sad8x8x8_sse4_1( +; const unsigned char *src_ptr, +; int src_stride, +; const unsigned char *ref_ptr, +; int ref_stride, +; unsigned short *sad_array +;); +global sym(aom_sad8x8x8_sse4_1) PRIVATE +sym(aom_sad8x8x8_sse4_1): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 5 + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;ref_ptr + + movsxd rax, dword ptr arg(1) ;src_stride + movsxd rdx, dword ptr arg(3) ;ref_stride + + PROCESS_8X2X8 1 + PROCESS_8X2X8 0 + PROCESS_8X2X8 0 + PROCESS_8X2X8 0 + + WRITE_AS_INTS + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + + +;void aom_sad8x16x8_sse4_1( +; const unsigned char *src_ptr, +; int src_stride, +; const unsigned char *ref_ptr, +; int ref_stride, +; unsigned short *sad_array +;); +global sym(aom_sad8x16x8_sse4_1) PRIVATE +sym(aom_sad8x16x8_sse4_1): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 5 + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;ref_ptr + + movsxd rax, dword ptr arg(1) ;src_stride + movsxd rdx, dword ptr arg(3) ;ref_stride + + PROCESS_8X2X8 1 + PROCESS_8X2X8 0 + PROCESS_8X2X8 0 + PROCESS_8X2X8 0 + PROCESS_8X2X8 0 + PROCESS_8X2X8 0 + PROCESS_8X2X8 0 + PROCESS_8X2X8 0 + + WRITE_AS_INTS + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + + +;void aom_sad4x4x8_sse4_1( +; const unsigned char *src_ptr, +; int src_stride, +; const unsigned char *ref_ptr, +; int ref_stride, +; unsigned short *sad_array +;); +global sym(aom_sad4x4x8_sse4_1) PRIVATE +sym(aom_sad4x4x8_sse4_1): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 5 + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;ref_ptr + + movsxd rax, dword ptr arg(1) ;src_stride + movsxd rdx, dword ptr arg(3) ;ref_stride + + PROCESS_4X2X8 1 + PROCESS_4X2X8 0 + + WRITE_AS_INTS + + ; begin epilog + pop rdi + pop rsi + UNSHADOW_ARGS + pop rbp + ret + + + + diff --git a/third_party/aom/aom_dsp/x86/sad_ssse3.asm b/third_party/aom/aom_dsp/x86/sad_ssse3.asm new file mode 100644 index 0000000000..96b64b040b --- /dev/null +++ b/third_party/aom/aom_dsp/x86/sad_ssse3.asm @@ -0,0 +1,373 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + +%include "aom_ports/x86_abi_support.asm" + +%macro PROCESS_16X2X3 1 +%if %1 + movdqa xmm0, XMMWORD PTR [rsi] + lddqu xmm5, XMMWORD PTR [rdi] + lddqu xmm6, XMMWORD PTR [rdi+1] + lddqu xmm7, XMMWORD PTR [rdi+2] + + psadbw xmm5, xmm0 + psadbw xmm6, xmm0 + psadbw xmm7, xmm0 +%else + movdqa xmm0, XMMWORD PTR [rsi] + lddqu xmm1, XMMWORD PTR [rdi] + lddqu xmm2, XMMWORD PTR [rdi+1] + lddqu xmm3, XMMWORD PTR [rdi+2] + + psadbw xmm1, xmm0 + psadbw xmm2, xmm0 + psadbw xmm3, xmm0 + + paddw xmm5, xmm1 + paddw xmm6, xmm2 + paddw xmm7, xmm3 +%endif + movdqa xmm0, XMMWORD PTR [rsi+rax] + lddqu xmm1, XMMWORD PTR [rdi+rdx] + lddqu xmm2, XMMWORD PTR [rdi+rdx+1] + lddqu xmm3, XMMWORD PTR [rdi+rdx+2] + + lea rsi, [rsi+rax*2] + lea rdi, [rdi+rdx*2] + + psadbw xmm1, xmm0 + psadbw xmm2, xmm0 + psadbw xmm3, xmm0 + + paddw xmm5, xmm1 + paddw xmm6, xmm2 + paddw xmm7, xmm3 +%endmacro + +%macro PROCESS_16X2X3_OFFSET 2 +%if %1 + movdqa xmm0, XMMWORD PTR [rsi] + movdqa xmm4, XMMWORD PTR [rdi] + movdqa xmm7, XMMWORD PTR [rdi+16] + + movdqa xmm5, xmm7 + palignr xmm5, xmm4, %2 + + movdqa xmm6, xmm7 + palignr xmm6, xmm4, (%2+1) + + palignr xmm7, xmm4, (%2+2) + + psadbw xmm5, xmm0 + psadbw xmm6, xmm0 + psadbw xmm7, xmm0 +%else + movdqa xmm0, XMMWORD PTR [rsi] + movdqa xmm4, XMMWORD PTR [rdi] + movdqa xmm3, XMMWORD PTR [rdi+16] + + movdqa xmm1, xmm3 + palignr xmm1, xmm4, %2 + + movdqa xmm2, xmm3 + palignr xmm2, xmm4, (%2+1) + + palignr xmm3, xmm4, (%2+2) + + psadbw xmm1, xmm0 + psadbw xmm2, xmm0 + psadbw xmm3, xmm0 + + paddw xmm5, xmm1 + paddw xmm6, xmm2 + paddw xmm7, xmm3 +%endif + movdqa xmm0, XMMWORD PTR [rsi+rax] + movdqa xmm4, XMMWORD PTR [rdi+rdx] + movdqa xmm3, XMMWORD PTR [rdi+rdx+16] + + movdqa xmm1, xmm3 + palignr xmm1, xmm4, %2 + + movdqa xmm2, xmm3 + palignr xmm2, xmm4, (%2+1) + + palignr xmm3, xmm4, (%2+2) + + lea rsi, [rsi+rax*2] + lea rdi, [rdi+rdx*2] + + psadbw xmm1, xmm0 + psadbw xmm2, xmm0 + psadbw xmm3, xmm0 + + paddw xmm5, xmm1 + paddw xmm6, xmm2 + paddw xmm7, xmm3 +%endmacro + +%macro PROCESS_16X16X3_OFFSET 2 +%2_aligned_by_%1: + + sub rdi, %1 + + PROCESS_16X2X3_OFFSET 1, %1 + PROCESS_16X2X3_OFFSET 0, %1 + PROCESS_16X2X3_OFFSET 0, %1 + PROCESS_16X2X3_OFFSET 0, %1 + PROCESS_16X2X3_OFFSET 0, %1 + PROCESS_16X2X3_OFFSET 0, %1 + PROCESS_16X2X3_OFFSET 0, %1 + PROCESS_16X2X3_OFFSET 0, %1 + + jmp %2_store_off + +%endmacro + +%macro PROCESS_16X8X3_OFFSET 2 +%2_aligned_by_%1: + + sub rdi, %1 + + PROCESS_16X2X3_OFFSET 1, %1 + PROCESS_16X2X3_OFFSET 0, %1 + PROCESS_16X2X3_OFFSET 0, %1 + PROCESS_16X2X3_OFFSET 0, %1 + + jmp %2_store_off + +%endmacro + +;void int aom_sad16x16x3_ssse3( +; unsigned char *src_ptr, +; int src_stride, +; unsigned char *ref_ptr, +; int ref_stride, +; int *results) +global sym(aom_sad16x16x3_ssse3) PRIVATE +sym(aom_sad16x16x3_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 5 + SAVE_XMM 7 + push rsi + push rdi + push rcx + ; end prolog + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;ref_ptr + + mov rdx, 0xf + and rdx, rdi + + jmp .aom_sad16x16x3_ssse3_skiptable +.aom_sad16x16x3_ssse3_jumptable: + dd .aom_sad16x16x3_ssse3_aligned_by_0 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_1 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_2 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_3 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_4 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_5 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_6 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_7 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_8 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_9 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_10 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_11 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_12 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_13 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_14 - .aom_sad16x16x3_ssse3_do_jump + dd .aom_sad16x16x3_ssse3_aligned_by_15 - .aom_sad16x16x3_ssse3_do_jump +.aom_sad16x16x3_ssse3_skiptable: + + call .aom_sad16x16x3_ssse3_do_jump +.aom_sad16x16x3_ssse3_do_jump: + pop rcx ; get the address of do_jump + mov rax, .aom_sad16x16x3_ssse3_jumptable - .aom_sad16x16x3_ssse3_do_jump + add rax, rcx ; get the absolute address of aom_sad16x16x3_ssse3_jumptable + + movsxd rax, dword [rax + 4*rdx] ; get the 32 bit offset from the jumptable + add rcx, rax + + movsxd rax, dword ptr arg(1) ;src_stride + movsxd rdx, dword ptr arg(3) ;ref_stride + + jmp rcx + + PROCESS_16X16X3_OFFSET 0, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 1, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 2, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 3, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 4, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 5, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 6, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 7, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 8, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 9, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 10, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 11, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 12, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 13, .aom_sad16x16x3_ssse3 + PROCESS_16X16X3_OFFSET 14, .aom_sad16x16x3_ssse3 + +.aom_sad16x16x3_ssse3_aligned_by_15: + PROCESS_16X2X3 1 + PROCESS_16X2X3 0 + PROCESS_16X2X3 0 + PROCESS_16X2X3 0 + PROCESS_16X2X3 0 + PROCESS_16X2X3 0 + PROCESS_16X2X3 0 + PROCESS_16X2X3 0 + +.aom_sad16x16x3_ssse3_store_off: + mov rdi, arg(4) ;Results + + movq xmm0, xmm5 + psrldq xmm5, 8 + + paddw xmm0, xmm5 + movd [rdi], xmm0 +;- + movq xmm0, xmm6 + psrldq xmm6, 8 + + paddw xmm0, xmm6 + movd [rdi+4], xmm0 +;- + movq xmm0, xmm7 + psrldq xmm7, 8 + + paddw xmm0, xmm7 + movd [rdi+8], xmm0 + + ; begin epilog + pop rcx + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void int aom_sad16x8x3_ssse3( +; unsigned char *src_ptr, +; int src_stride, +; unsigned char *ref_ptr, +; int ref_stride, +; int *results) +global sym(aom_sad16x8x3_ssse3) PRIVATE +sym(aom_sad16x8x3_ssse3): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 5 + SAVE_XMM 7 + push rsi + push rdi + push rcx + ; end prolog + + mov rsi, arg(0) ;src_ptr + mov rdi, arg(2) ;ref_ptr + + mov rdx, 0xf + and rdx, rdi + + jmp .aom_sad16x8x3_ssse3_skiptable +.aom_sad16x8x3_ssse3_jumptable: + dd .aom_sad16x8x3_ssse3_aligned_by_0 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_1 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_2 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_3 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_4 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_5 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_6 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_7 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_8 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_9 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_10 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_11 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_12 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_13 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_14 - .aom_sad16x8x3_ssse3_do_jump + dd .aom_sad16x8x3_ssse3_aligned_by_15 - .aom_sad16x8x3_ssse3_do_jump +.aom_sad16x8x3_ssse3_skiptable: + + call .aom_sad16x8x3_ssse3_do_jump +.aom_sad16x8x3_ssse3_do_jump: + pop rcx ; get the address of do_jump + mov rax, .aom_sad16x8x3_ssse3_jumptable - .aom_sad16x8x3_ssse3_do_jump + add rax, rcx ; get the absolute address of aom_sad16x8x3_ssse3_jumptable + + movsxd rax, dword [rax + 4*rdx] ; get the 32 bit offset from the jumptable + add rcx, rax + + movsxd rax, dword ptr arg(1) ;src_stride + movsxd rdx, dword ptr arg(3) ;ref_stride + + jmp rcx + + PROCESS_16X8X3_OFFSET 0, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 1, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 2, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 3, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 4, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 5, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 6, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 7, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 8, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 9, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 10, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 11, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 12, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 13, .aom_sad16x8x3_ssse3 + PROCESS_16X8X3_OFFSET 14, .aom_sad16x8x3_ssse3 + +.aom_sad16x8x3_ssse3_aligned_by_15: + + PROCESS_16X2X3 1 + PROCESS_16X2X3 0 + PROCESS_16X2X3 0 + PROCESS_16X2X3 0 + +.aom_sad16x8x3_ssse3_store_off: + mov rdi, arg(4) ;Results + + movq xmm0, xmm5 + psrldq xmm5, 8 + + paddw xmm0, xmm5 + movd [rdi], xmm0 +;- + movq xmm0, xmm6 + psrldq xmm6, 8 + + paddw xmm0, xmm6 + movd [rdi+4], xmm0 +;- + movq xmm0, xmm7 + psrldq xmm7, 8 + + paddw xmm0, xmm7 + movd [rdi+8], xmm0 + + ; begin epilog + pop rcx + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret diff --git a/third_party/aom/aom_dsp/x86/ssim_opt_x86_64.asm b/third_party/aom/aom_dsp/x86/ssim_opt_x86_64.asm new file mode 100644 index 0000000000..aa70106c84 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/ssim_opt_x86_64.asm @@ -0,0 +1,219 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "aom_ports/x86_abi_support.asm" + +; tabulate_ssim - sums sum_s,sum_r,sum_sq_s,sum_sq_r, sum_sxr +%macro TABULATE_SSIM 0 + paddusw xmm15, xmm3 ; sum_s + paddusw xmm14, xmm4 ; sum_r + movdqa xmm1, xmm3 + pmaddwd xmm1, xmm1 + paddd xmm13, xmm1 ; sum_sq_s + movdqa xmm2, xmm4 + pmaddwd xmm2, xmm2 + paddd xmm12, xmm2 ; sum_sq_r + pmaddwd xmm3, xmm4 + paddd xmm11, xmm3 ; sum_sxr +%endmacro + +; Sum across the register %1 starting with q words +%macro SUM_ACROSS_Q 1 + movdqa xmm2,%1 + punpckldq %1,xmm0 + punpckhdq xmm2,xmm0 + paddq %1,xmm2 + movdqa xmm2,%1 + punpcklqdq %1,xmm0 + punpckhqdq xmm2,xmm0 + paddq %1,xmm2 +%endmacro + +; Sum across the register %1 starting with q words +%macro SUM_ACROSS_W 1 + movdqa xmm1, %1 + punpcklwd %1,xmm0 + punpckhwd xmm1,xmm0 + paddd %1, xmm1 + SUM_ACROSS_Q %1 +%endmacro +;void ssim_parms_sse2( +; unsigned char *s, +; int sp, +; unsigned char *r, +; int rp +; uint32_t *sum_s, +; uint32_t *sum_r, +; uint32_t *sum_sq_s, +; uint32_t *sum_sq_r, +; uint32_t *sum_sxr); +; +; TODO: Use parm passing through structure, probably don't need the pxors +; ( calling app will initialize to 0 ) could easily fit everything in sse2 +; without too much hastle, and can probably do better estimates with psadw +; or pavgb At this point this is just meant to be first pass for calculating +; all the parms needed for 16x16 ssim so we can play with dssim as distortion +; in mode selection code. +global sym(aom_ssim_parms_16x16_sse2) PRIVATE +sym(aom_ssim_parms_16x16_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 9 + SAVE_XMM 15 + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;s + mov rcx, arg(1) ;sp + mov rdi, arg(2) ;r + mov rax, arg(3) ;rp + + pxor xmm0, xmm0 + pxor xmm15,xmm15 ;sum_s + pxor xmm14,xmm14 ;sum_r + pxor xmm13,xmm13 ;sum_sq_s + pxor xmm12,xmm12 ;sum_sq_r + pxor xmm11,xmm11 ;sum_sxr + + mov rdx, 16 ;row counter +.NextRow: + + ;grab source and reference pixels + movdqu xmm5, [rsi] + movdqu xmm6, [rdi] + movdqa xmm3, xmm5 + movdqa xmm4, xmm6 + punpckhbw xmm3, xmm0 ; high_s + punpckhbw xmm4, xmm0 ; high_r + + TABULATE_SSIM + + movdqa xmm3, xmm5 + movdqa xmm4, xmm6 + punpcklbw xmm3, xmm0 ; low_s + punpcklbw xmm4, xmm0 ; low_r + + TABULATE_SSIM + + add rsi, rcx ; next s row + add rdi, rax ; next r row + + dec rdx ; counter + jnz .NextRow + + SUM_ACROSS_W xmm15 + SUM_ACROSS_W xmm14 + SUM_ACROSS_Q xmm13 + SUM_ACROSS_Q xmm12 + SUM_ACROSS_Q xmm11 + + mov rdi,arg(4) + movd [rdi], xmm15; + mov rdi,arg(5) + movd [rdi], xmm14; + mov rdi,arg(6) + movd [rdi], xmm13; + mov rdi,arg(7) + movd [rdi], xmm12; + mov rdi,arg(8) + movd [rdi], xmm11; + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void ssim_parms_sse2( +; unsigned char *s, +; int sp, +; unsigned char *r, +; int rp +; uint32_t *sum_s, +; uint32_t *sum_r, +; uint32_t *sum_sq_s, +; uint32_t *sum_sq_r, +; uint32_t *sum_sxr); +; +; TODO: Use parm passing through structure, probably don't need the pxors +; ( calling app will initialize to 0 ) could easily fit everything in sse2 +; without too much hastle, and can probably do better estimates with psadw +; or pavgb At this point this is just meant to be first pass for calculating +; all the parms needed for 16x16 ssim so we can play with dssim as distortion +; in mode selection code. +global sym(aom_ssim_parms_8x8_sse2) PRIVATE +sym(aom_ssim_parms_8x8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 9 + SAVE_XMM 15 + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;s + mov rcx, arg(1) ;sp + mov rdi, arg(2) ;r + mov rax, arg(3) ;rp + + pxor xmm0, xmm0 + pxor xmm15,xmm15 ;sum_s + pxor xmm14,xmm14 ;sum_r + pxor xmm13,xmm13 ;sum_sq_s + pxor xmm12,xmm12 ;sum_sq_r + pxor xmm11,xmm11 ;sum_sxr + + mov rdx, 8 ;row counter +.NextRow: + + ;grab source and reference pixels + movq xmm3, [rsi] + movq xmm4, [rdi] + punpcklbw xmm3, xmm0 ; low_s + punpcklbw xmm4, xmm0 ; low_r + + TABULATE_SSIM + + add rsi, rcx ; next s row + add rdi, rax ; next r row + + dec rdx ; counter + jnz .NextRow + + SUM_ACROSS_W xmm15 + SUM_ACROSS_W xmm14 + SUM_ACROSS_Q xmm13 + SUM_ACROSS_Q xmm12 + SUM_ACROSS_Q xmm11 + + mov rdi,arg(4) + movd [rdi], xmm15; + mov rdi,arg(5) + movd [rdi], xmm14; + mov rdi,arg(6) + movd [rdi], xmm13; + mov rdi,arg(7) + movd [rdi], xmm12; + mov rdi,arg(8) + movd [rdi], xmm11; + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret diff --git a/third_party/aom/aom_dsp/x86/subpel_variance_sse2.asm b/third_party/aom/aom_dsp/x86/subpel_variance_sse2.asm new file mode 100644 index 0000000000..d3feb7ec03 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/subpel_variance_sse2.asm @@ -0,0 +1,1489 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION_RODATA +pw_8: times 8 dw 8 +bilin_filter_m_sse2: times 8 dw 16 + times 8 dw 0 + times 8 dw 14 + times 8 dw 2 + times 8 dw 12 + times 8 dw 4 + times 8 dw 10 + times 8 dw 6 + times 16 dw 8 + times 8 dw 6 + times 8 dw 10 + times 8 dw 4 + times 8 dw 12 + times 8 dw 2 + times 8 dw 14 + +bilin_filter_m_ssse3: times 8 db 16, 0 + times 8 db 14, 2 + times 8 db 12, 4 + times 8 db 10, 6 + times 16 db 8 + times 8 db 6, 10 + times 8 db 4, 12 + times 8 db 2, 14 + +SECTION .text + +; int aom_sub_pixel_varianceNxh(const uint8_t *src, ptrdiff_t src_stride, +; int x_offset, int y_offset, +; const uint8_t *dst, ptrdiff_t dst_stride, +; int height, unsigned int *sse); +; +; This function returns the SE and stores SSE in the given pointer. + +%macro SUM_SSE 6 ; src1, dst1, src2, dst2, sum, sse + psubw %3, %4 + psubw %1, %2 + paddw %5, %3 + pmaddwd %3, %3 + paddw %5, %1 + pmaddwd %1, %1 + paddd %6, %3 + paddd %6, %1 +%endmacro + +%macro STORE_AND_RET 1 +%if %1 > 4 + ; if H=64 and W=16, we have 8 words of each 2(1bit)x64(6bit)x9bit=16bit + ; in m6, i.e. it _exactly_ fits in a signed word per word in the xmm reg. + ; We have to sign-extend it before adding the words within the register + ; and outputing to a dword. + pcmpgtw m5, m6 ; mask for 0 > x + movhlps m3, m7 + punpcklwd m4, m6, m5 + punpckhwd m6, m5 ; sign-extend m6 word->dword + paddd m7, m3 + paddd m6, m4 + pshufd m3, m7, 0x1 + movhlps m4, m6 + paddd m7, m3 + paddd m6, m4 + mov r1, ssem ; r1 = unsigned int *sse + pshufd m4, m6, 0x1 + movd [r1], m7 ; store sse + paddd m6, m4 + movd raxd, m6 ; store sum as return value +%else ; 4xh + pshuflw m4, m6, 0xe + pshuflw m3, m7, 0xe + paddw m6, m4 + paddd m7, m3 + pcmpgtw m5, m6 ; mask for 0 > x + mov r1, ssem ; r1 = unsigned int *sse + punpcklwd m6, m5 ; sign-extend m6 word->dword + movd [r1], m7 ; store sse + pshuflw m4, m6, 0xe + paddd m6, m4 + movd raxd, m6 ; store sum as return value +%endif + RET +%endmacro + +%macro INC_SRC_BY_SRC_STRIDE 0 +%if ARCH_X86=1 && CONFIG_PIC=1 + add srcq, src_stridemp +%else + add srcq, src_strideq +%endif +%endmacro + +%macro SUBPEL_VARIANCE 1-2 0 ; W +%if cpuflag(ssse3) +%define bilin_filter_m bilin_filter_m_ssse3 +%define filter_idx_shift 4 +%else +%define bilin_filter_m bilin_filter_m_sse2 +%define filter_idx_shift 5 +%endif +; FIXME(rbultje) only bilinear filters use >8 registers, and ssse3 only uses +; 11, not 13, if the registers are ordered correctly. May make a minor speed +; difference on Win64 + +%ifdef PIC ; 64bit PIC + %if %2 == 1 ; avg + cglobal sub_pixel_avg_variance%1xh, 9, 10, 13, src, src_stride, \ + x_offset, y_offset, \ + dst, dst_stride, \ + sec, sec_stride, height, sse + %define sec_str sec_strideq + %else + cglobal sub_pixel_variance%1xh, 7, 8, 13, src, src_stride, x_offset, \ + y_offset, dst, dst_stride, height, sse + %endif + %define block_height heightd + %define bilin_filter sseq +%else + %if ARCH_X86=1 && CONFIG_PIC=1 + %if %2 == 1 ; avg + cglobal sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \ + x_offset, y_offset, \ + dst, dst_stride, \ + sec, sec_stride, \ + height, sse, g_bilin_filter, g_pw_8 + %define block_height dword heightm + %define sec_str sec_stridemp + + ;Store bilin_filter and pw_8 location in stack + %if GET_GOT_DEFINED == 1 + GET_GOT eax + add esp, 4 ; restore esp + %endif + + lea ecx, [GLOBAL(bilin_filter_m)] + mov g_bilin_filterm, ecx + + lea ecx, [GLOBAL(pw_8)] + mov g_pw_8m, ecx + + LOAD_IF_USED 0, 1 ; load eax, ecx back + %else + cglobal sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, x_offset, \ + y_offset, dst, dst_stride, height, sse, \ + g_bilin_filter, g_pw_8 + %define block_height heightd + + ;Store bilin_filter and pw_8 location in stack + %if GET_GOT_DEFINED == 1 + GET_GOT eax + add esp, 4 ; restore esp + %endif + + lea ecx, [GLOBAL(bilin_filter_m)] + mov g_bilin_filterm, ecx + + lea ecx, [GLOBAL(pw_8)] + mov g_pw_8m, ecx + + LOAD_IF_USED 0, 1 ; load eax, ecx back + %endif + %else + %if %2 == 1 ; avg + cglobal sub_pixel_avg_variance%1xh, 7 + 2 * ARCH_X86_64, \ + 7 + 2 * ARCH_X86_64, 13, src, src_stride, \ + x_offset, y_offset, \ + dst, dst_stride, \ + sec, sec_stride, \ + height, sse + %if ARCH_X86_64 + %define block_height heightd + %define sec_str sec_strideq + %else + %define block_height dword heightm + %define sec_str sec_stridemp + %endif + %else + cglobal sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, x_offset, \ + y_offset, dst, dst_stride, height, sse + %define block_height heightd + %endif + + %define bilin_filter bilin_filter_m + %endif +%endif + +%if %1 == 4 + %define movx movd +%else + %define movx movh +%endif + + ASSERT %1 <= 16 ; m6 overflows if w > 16 + pxor m6, m6 ; sum + pxor m7, m7 ; sse + ; FIXME(rbultje) if both filters are bilinear, we don't actually use m5; we + ; could perhaps use it for something more productive then + pxor m5, m5 ; dedicated zero register +%if %1 < 16 + sar block_height, 1 +%if %2 == 1 ; avg + shl sec_str, 1 +%endif +%endif + + ; FIXME(rbultje) replace by jumptable? + test x_offsetd, x_offsetd + jnz .x_nonzero + ; x_offset == 0 + test y_offsetd, y_offsetd + jnz .x_zero_y_nonzero + + ; x_offset == 0 && y_offset == 0 +.x_zero_y_zero_loop: +%if %1 == 16 + movu m0, [srcq] + mova m1, [dstq] +%if %2 == 1 ; avg + pavgb m0, [secq] + punpckhbw m3, m1, m5 + punpcklbw m1, m5 +%endif + punpckhbw m2, m0, m5 + punpcklbw m0, m5 + +%if %2 == 0 ; !avg + punpckhbw m3, m1, m5 + punpcklbw m1, m5 +%endif + SUM_SSE m0, m1, m2, m3, m6, m7 + + add srcq, src_strideq + add dstq, dst_strideq +%else ; %1 < 16 + movx m0, [srcq] +%if %2 == 1 ; avg +%if %1 > 4 + movhps m0, [srcq+src_strideq] +%else ; 4xh + movx m1, [srcq+src_strideq] + punpckldq m0, m1 +%endif +%else ; !avg + movx m2, [srcq+src_strideq] +%endif + + movx m1, [dstq] + movx m3, [dstq+dst_strideq] + +%if %2 == 1 ; avg +%if %1 > 4 + pavgb m0, [secq] +%else + movh m2, [secq] + pavgb m0, m2 +%endif + punpcklbw m3, m5 + punpcklbw m1, m5 +%if %1 > 4 + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%else ; 4xh + punpcklbw m0, m5 + movhlps m2, m0 +%endif +%else ; !avg + punpcklbw m0, m5 + punpcklbw m2, m5 + punpcklbw m3, m5 + punpcklbw m1, m5 +%endif + SUM_SSE m0, m1, m2, m3, m6, m7 + + lea srcq, [srcq+src_strideq*2] + lea dstq, [dstq+dst_strideq*2] +%endif +%if %2 == 1 ; avg + add secq, sec_str +%endif + dec block_height + jg .x_zero_y_zero_loop + STORE_AND_RET %1 + +.x_zero_y_nonzero: + cmp y_offsetd, 4 + jne .x_zero_y_nonhalf + + ; x_offset == 0 && y_offset == 0.5 +.x_zero_y_half_loop: +%if %1 == 16 + movu m0, [srcq] + movu m4, [srcq+src_strideq] + mova m1, [dstq] + pavgb m0, m4 + punpckhbw m3, m1, m5 +%if %2 == 1 ; avg + pavgb m0, [secq] +%endif + punpcklbw m1, m5 + punpckhbw m2, m0, m5 + punpcklbw m0, m5 + SUM_SSE m0, m1, m2, m3, m6, m7 + + add srcq, src_strideq + add dstq, dst_strideq +%else ; %1 < 16 + movx m0, [srcq] + movx m2, [srcq+src_strideq] +%if %2 == 1 ; avg +%if %1 > 4 + movhps m2, [srcq+src_strideq*2] +%else ; 4xh + movx m1, [srcq+src_strideq*2] + punpckldq m2, m1 +%endif + movx m1, [dstq] +%if %1 > 4 + movlhps m0, m2 +%else ; 4xh + punpckldq m0, m2 +%endif + movx m3, [dstq+dst_strideq] + pavgb m0, m2 + punpcklbw m1, m5 +%if %1 > 4 + pavgb m0, [secq] + punpcklbw m3, m5 + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%else ; 4xh + movh m4, [secq] + pavgb m0, m4 + punpcklbw m3, m5 + punpcklbw m0, m5 + movhlps m2, m0 +%endif +%else ; !avg + movx m4, [srcq+src_strideq*2] + movx m1, [dstq] + pavgb m0, m2 + movx m3, [dstq+dst_strideq] + pavgb m2, m4 + punpcklbw m0, m5 + punpcklbw m2, m5 + punpcklbw m3, m5 + punpcklbw m1, m5 +%endif + SUM_SSE m0, m1, m2, m3, m6, m7 + + lea srcq, [srcq+src_strideq*2] + lea dstq, [dstq+dst_strideq*2] +%endif +%if %2 == 1 ; avg + add secq, sec_str +%endif + dec block_height + jg .x_zero_y_half_loop + STORE_AND_RET %1 + +.x_zero_y_nonhalf: + ; x_offset == 0 && y_offset == bilin interpolation +%ifdef PIC + lea bilin_filter, [bilin_filter_m] +%endif + shl y_offsetd, filter_idx_shift +%if ARCH_X86_64 && %1 > 4 + mova m8, [bilin_filter+y_offsetq] +%if notcpuflag(ssse3) ; FIXME(rbultje) don't scatter registers on x86-64 + mova m9, [bilin_filter+y_offsetq+16] +%endif + mova m10, [pw_8] +%define filter_y_a m8 +%define filter_y_b m9 +%define filter_rnd m10 +%else ; x86-32 or mmx +%if ARCH_X86=1 && CONFIG_PIC=1 +; x_offset == 0, reuse x_offset reg +%define tempq x_offsetq + add y_offsetq, g_bilin_filterm +%define filter_y_a [y_offsetq] +%define filter_y_b [y_offsetq+16] + mov tempq, g_pw_8m +%define filter_rnd [tempq] +%else + add y_offsetq, bilin_filter +%define filter_y_a [y_offsetq] +%define filter_y_b [y_offsetq+16] +%define filter_rnd [pw_8] +%endif +%endif + +.x_zero_y_other_loop: +%if %1 == 16 + movu m0, [srcq] + movu m4, [srcq+src_strideq] + mova m1, [dstq] +%if cpuflag(ssse3) + punpckhbw m2, m0, m4 + punpcklbw m0, m4 + pmaddubsw m2, filter_y_a + pmaddubsw m0, filter_y_a + paddw m2, filter_rnd + paddw m0, filter_rnd +%else + punpckhbw m2, m0, m5 + punpckhbw m3, m4, m5 + punpcklbw m0, m5 + punpcklbw m4, m5 + ; FIXME(rbultje) instead of out=((num-x)*in1+x*in2+rnd)>>log2(num), we can + ; also do out=in1+(((num-x)*(in2-in1)+rnd)>>log2(num)). Total number of + ; instructions is the same (5), but it is 1 mul instead of 2, so might be + ; slightly faster because of pmullw latency. It would also cut our rodata + ; tables in half for this function, and save 1-2 registers on x86-64. + pmullw m2, filter_y_a + pmullw m3, filter_y_b + paddw m2, filter_rnd + pmullw m0, filter_y_a + pmullw m4, filter_y_b + paddw m0, filter_rnd + paddw m2, m3 + paddw m0, m4 +%endif + psraw m2, 4 + psraw m0, 4 +%if %2 == 1 ; avg + ; FIXME(rbultje) pipeline + packuswb m0, m2 + pavgb m0, [secq] + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%endif + punpckhbw m3, m1, m5 + punpcklbw m1, m5 + SUM_SSE m0, m1, m2, m3, m6, m7 + + add srcq, src_strideq + add dstq, dst_strideq +%else ; %1 < 16 + movx m0, [srcq] + movx m2, [srcq+src_strideq] + movx m4, [srcq+src_strideq*2] + movx m3, [dstq+dst_strideq] +%if cpuflag(ssse3) + movx m1, [dstq] + punpcklbw m0, m2 + punpcklbw m2, m4 + pmaddubsw m0, filter_y_a + pmaddubsw m2, filter_y_a + punpcklbw m3, m5 + paddw m2, filter_rnd + paddw m0, filter_rnd +%else + punpcklbw m0, m5 + punpcklbw m2, m5 + punpcklbw m4, m5 + pmullw m0, filter_y_a + pmullw m1, m2, filter_y_b + punpcklbw m3, m5 + paddw m0, filter_rnd + pmullw m2, filter_y_a + pmullw m4, filter_y_b + paddw m0, m1 + paddw m2, filter_rnd + movx m1, [dstq] + paddw m2, m4 +%endif + psraw m0, 4 + psraw m2, 4 +%if %2 == 1 ; avg + ; FIXME(rbultje) pipeline +%if %1 == 4 + movlhps m0, m2 +%endif + packuswb m0, m2 +%if %1 > 4 + pavgb m0, [secq] + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%else ; 4xh + movh m2, [secq] + pavgb m0, m2 + punpcklbw m0, m5 + movhlps m2, m0 +%endif +%endif + punpcklbw m1, m5 + SUM_SSE m0, m1, m2, m3, m6, m7 + + lea srcq, [srcq+src_strideq*2] + lea dstq, [dstq+dst_strideq*2] +%endif +%if %2 == 1 ; avg + add secq, sec_str +%endif + dec block_height + jg .x_zero_y_other_loop +%undef filter_y_a +%undef filter_y_b +%undef filter_rnd + STORE_AND_RET %1 + +.x_nonzero: + cmp x_offsetd, 4 + jne .x_nonhalf + ; x_offset == 0.5 + test y_offsetd, y_offsetd + jnz .x_half_y_nonzero + + ; x_offset == 0.5 && y_offset == 0 +.x_half_y_zero_loop: +%if %1 == 16 + movu m0, [srcq] + movu m4, [srcq+1] + mova m1, [dstq] + pavgb m0, m4 + punpckhbw m3, m1, m5 +%if %2 == 1 ; avg + pavgb m0, [secq] +%endif + punpcklbw m1, m5 + punpckhbw m2, m0, m5 + punpcklbw m0, m5 + SUM_SSE m0, m1, m2, m3, m6, m7 + + add srcq, src_strideq + add dstq, dst_strideq +%else ; %1 < 16 + movx m0, [srcq] + movx m4, [srcq+1] +%if %2 == 1 ; avg +%if %1 > 4 + movhps m0, [srcq+src_strideq] + movhps m4, [srcq+src_strideq+1] +%else ; 4xh + movx m1, [srcq+src_strideq] + punpckldq m0, m1 + movx m2, [srcq+src_strideq+1] + punpckldq m4, m2 +%endif + movx m1, [dstq] + movx m3, [dstq+dst_strideq] + pavgb m0, m4 + punpcklbw m3, m5 +%if %1 > 4 + pavgb m0, [secq] + punpcklbw m1, m5 + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%else ; 4xh + movh m2, [secq] + pavgb m0, m2 + punpcklbw m1, m5 + punpcklbw m0, m5 + movhlps m2, m0 +%endif +%else ; !avg + movx m2, [srcq+src_strideq] + movx m1, [dstq] + pavgb m0, m4 + movx m4, [srcq+src_strideq+1] + movx m3, [dstq+dst_strideq] + pavgb m2, m4 + punpcklbw m0, m5 + punpcklbw m2, m5 + punpcklbw m3, m5 + punpcklbw m1, m5 +%endif + SUM_SSE m0, m1, m2, m3, m6, m7 + + lea srcq, [srcq+src_strideq*2] + lea dstq, [dstq+dst_strideq*2] +%endif +%if %2 == 1 ; avg + add secq, sec_str +%endif + dec block_height + jg .x_half_y_zero_loop + STORE_AND_RET %1 + +.x_half_y_nonzero: + cmp y_offsetd, 4 + jne .x_half_y_nonhalf + + ; x_offset == 0.5 && y_offset == 0.5 +%if %1 == 16 + movu m0, [srcq] + movu m3, [srcq+1] + add srcq, src_strideq + pavgb m0, m3 +.x_half_y_half_loop: + movu m4, [srcq] + movu m3, [srcq+1] + mova m1, [dstq] + pavgb m4, m3 + punpckhbw m3, m1, m5 + pavgb m0, m4 +%if %2 == 1 ; avg + punpcklbw m1, m5 + pavgb m0, [secq] + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%else + punpckhbw m2, m0, m5 + punpcklbw m0, m5 + punpcklbw m1, m5 +%endif + SUM_SSE m0, m1, m2, m3, m6, m7 + mova m0, m4 + + add srcq, src_strideq + add dstq, dst_strideq +%else ; %1 < 16 + movx m0, [srcq] + movx m3, [srcq+1] + add srcq, src_strideq + pavgb m0, m3 +.x_half_y_half_loop: + movx m2, [srcq] + movx m3, [srcq+1] +%if %2 == 1 ; avg +%if %1 > 4 + movhps m2, [srcq+src_strideq] + movhps m3, [srcq+src_strideq+1] +%else + movx m1, [srcq+src_strideq] + punpckldq m2, m1 + movx m1, [srcq+src_strideq+1] + punpckldq m3, m1 +%endif + pavgb m2, m3 +%if %1 > 4 + movlhps m0, m2 + movhlps m4, m2 +%else ; 4xh + punpckldq m0, m2 + pshuflw m4, m2, 0xe +%endif + movx m1, [dstq] + pavgb m0, m2 + movx m3, [dstq+dst_strideq] +%if %1 > 4 + pavgb m0, [secq] +%else + movh m2, [secq] + pavgb m0, m2 +%endif + punpcklbw m3, m5 + punpcklbw m1, m5 +%if %1 > 4 + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%else + punpcklbw m0, m5 + movhlps m2, m0 +%endif +%else ; !avg + movx m4, [srcq+src_strideq] + movx m1, [srcq+src_strideq+1] + pavgb m2, m3 + pavgb m4, m1 + pavgb m0, m2 + pavgb m2, m4 + movx m1, [dstq] + movx m3, [dstq+dst_strideq] + punpcklbw m0, m5 + punpcklbw m2, m5 + punpcklbw m3, m5 + punpcklbw m1, m5 +%endif + SUM_SSE m0, m1, m2, m3, m6, m7 + mova m0, m4 + + lea srcq, [srcq+src_strideq*2] + lea dstq, [dstq+dst_strideq*2] +%endif +%if %2 == 1 ; avg + add secq, sec_str +%endif + dec block_height + jg .x_half_y_half_loop + STORE_AND_RET %1 + +.x_half_y_nonhalf: + ; x_offset == 0.5 && y_offset == bilin interpolation +%ifdef PIC + lea bilin_filter, [bilin_filter_m] +%endif + shl y_offsetd, filter_idx_shift +%if ARCH_X86_64 && %1 > 4 + mova m8, [bilin_filter+y_offsetq] +%if notcpuflag(ssse3) ; FIXME(rbultje) don't scatter registers on x86-64 + mova m9, [bilin_filter+y_offsetq+16] +%endif + mova m10, [pw_8] +%define filter_y_a m8 +%define filter_y_b m9 +%define filter_rnd m10 +%else ;x86_32 +%if ARCH_X86=1 && CONFIG_PIC=1 +; x_offset == 0.5. We can reuse x_offset reg +%define tempq x_offsetq + add y_offsetq, g_bilin_filterm +%define filter_y_a [y_offsetq] +%define filter_y_b [y_offsetq+16] + mov tempq, g_pw_8m +%define filter_rnd [tempq] +%else + add y_offsetq, bilin_filter +%define filter_y_a [y_offsetq] +%define filter_y_b [y_offsetq+16] +%define filter_rnd [pw_8] +%endif +%endif + +%if %1 == 16 + movu m0, [srcq] + movu m3, [srcq+1] + add srcq, src_strideq + pavgb m0, m3 +.x_half_y_other_loop: + movu m4, [srcq] + movu m2, [srcq+1] + mova m1, [dstq] + pavgb m4, m2 +%if cpuflag(ssse3) + punpckhbw m2, m0, m4 + punpcklbw m0, m4 + pmaddubsw m2, filter_y_a + pmaddubsw m0, filter_y_a + paddw m2, filter_rnd + paddw m0, filter_rnd + psraw m2, 4 +%else + punpckhbw m2, m0, m5 + punpckhbw m3, m4, m5 + pmullw m2, filter_y_a + pmullw m3, filter_y_b + paddw m2, filter_rnd + punpcklbw m0, m5 + paddw m2, m3 + punpcklbw m3, m4, m5 + pmullw m0, filter_y_a + pmullw m3, filter_y_b + paddw m0, filter_rnd + psraw m2, 4 + paddw m0, m3 +%endif + punpckhbw m3, m1, m5 + psraw m0, 4 +%if %2 == 1 ; avg + ; FIXME(rbultje) pipeline + packuswb m0, m2 + pavgb m0, [secq] + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%endif + punpcklbw m1, m5 + SUM_SSE m0, m1, m2, m3, m6, m7 + mova m0, m4 + + add srcq, src_strideq + add dstq, dst_strideq +%else ; %1 < 16 + movx m0, [srcq] + movx m3, [srcq+1] + add srcq, src_strideq + pavgb m0, m3 +%if notcpuflag(ssse3) + punpcklbw m0, m5 +%endif +.x_half_y_other_loop: + movx m2, [srcq] + movx m1, [srcq+1] + movx m4, [srcq+src_strideq] + movx m3, [srcq+src_strideq+1] + pavgb m2, m1 + pavgb m4, m3 + movx m3, [dstq+dst_strideq] +%if cpuflag(ssse3) + movx m1, [dstq] + punpcklbw m0, m2 + punpcklbw m2, m4 + pmaddubsw m0, filter_y_a + pmaddubsw m2, filter_y_a + punpcklbw m3, m5 + paddw m0, filter_rnd + paddw m2, filter_rnd +%else + punpcklbw m2, m5 + punpcklbw m4, m5 + pmullw m0, filter_y_a + pmullw m1, m2, filter_y_b + punpcklbw m3, m5 + paddw m0, filter_rnd + pmullw m2, filter_y_a + paddw m0, m1 + pmullw m1, m4, filter_y_b + paddw m2, filter_rnd + paddw m2, m1 + movx m1, [dstq] +%endif + psraw m0, 4 + psraw m2, 4 +%if %2 == 1 ; avg + ; FIXME(rbultje) pipeline +%if %1 == 4 + movlhps m0, m2 +%endif + packuswb m0, m2 +%if %1 > 4 + pavgb m0, [secq] + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%else + movh m2, [secq] + pavgb m0, m2 + punpcklbw m0, m5 + movhlps m2, m0 +%endif +%endif + punpcklbw m1, m5 + SUM_SSE m0, m1, m2, m3, m6, m7 + mova m0, m4 + + lea srcq, [srcq+src_strideq*2] + lea dstq, [dstq+dst_strideq*2] +%endif +%if %2 == 1 ; avg + add secq, sec_str +%endif + dec block_height + jg .x_half_y_other_loop +%undef filter_y_a +%undef filter_y_b +%undef filter_rnd + STORE_AND_RET %1 + +.x_nonhalf: + test y_offsetd, y_offsetd + jnz .x_nonhalf_y_nonzero + + ; x_offset == bilin interpolation && y_offset == 0 +%ifdef PIC + lea bilin_filter, [bilin_filter_m] +%endif + shl x_offsetd, filter_idx_shift +%if ARCH_X86_64 && %1 > 4 + mova m8, [bilin_filter+x_offsetq] +%if notcpuflag(ssse3) ; FIXME(rbultje) don't scatter registers on x86-64 + mova m9, [bilin_filter+x_offsetq+16] +%endif + mova m10, [pw_8] +%define filter_x_a m8 +%define filter_x_b m9 +%define filter_rnd m10 +%else ; x86-32 +%if ARCH_X86=1 && CONFIG_PIC=1 +;y_offset == 0. We can reuse y_offset reg. +%define tempq y_offsetq + add x_offsetq, g_bilin_filterm +%define filter_x_a [x_offsetq] +%define filter_x_b [x_offsetq+16] + mov tempq, g_pw_8m +%define filter_rnd [tempq] +%else + add x_offsetq, bilin_filter +%define filter_x_a [x_offsetq] +%define filter_x_b [x_offsetq+16] +%define filter_rnd [pw_8] +%endif +%endif + +.x_other_y_zero_loop: +%if %1 == 16 + movu m0, [srcq] + movu m4, [srcq+1] + mova m1, [dstq] +%if cpuflag(ssse3) + punpckhbw m2, m0, m4 + punpcklbw m0, m4 + pmaddubsw m2, filter_x_a + pmaddubsw m0, filter_x_a + paddw m2, filter_rnd + paddw m0, filter_rnd +%else + punpckhbw m2, m0, m5 + punpckhbw m3, m4, m5 + punpcklbw m0, m5 + punpcklbw m4, m5 + pmullw m2, filter_x_a + pmullw m3, filter_x_b + paddw m2, filter_rnd + pmullw m0, filter_x_a + pmullw m4, filter_x_b + paddw m0, filter_rnd + paddw m2, m3 + paddw m0, m4 +%endif + psraw m2, 4 + psraw m0, 4 +%if %2 == 1 ; avg + ; FIXME(rbultje) pipeline + packuswb m0, m2 + pavgb m0, [secq] + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%endif + punpckhbw m3, m1, m5 + punpcklbw m1, m5 + SUM_SSE m0, m1, m2, m3, m6, m7 + + add srcq, src_strideq + add dstq, dst_strideq +%else ; %1 < 16 + movx m0, [srcq] + movx m1, [srcq+1] + movx m2, [srcq+src_strideq] + movx m4, [srcq+src_strideq+1] + movx m3, [dstq+dst_strideq] +%if cpuflag(ssse3) + punpcklbw m0, m1 + movx m1, [dstq] + punpcklbw m2, m4 + pmaddubsw m0, filter_x_a + pmaddubsw m2, filter_x_a + punpcklbw m3, m5 + paddw m0, filter_rnd + paddw m2, filter_rnd +%else + punpcklbw m0, m5 + punpcklbw m1, m5 + punpcklbw m2, m5 + punpcklbw m4, m5 + pmullw m0, filter_x_a + pmullw m1, filter_x_b + punpcklbw m3, m5 + paddw m0, filter_rnd + pmullw m2, filter_x_a + pmullw m4, filter_x_b + paddw m0, m1 + paddw m2, filter_rnd + movx m1, [dstq] + paddw m2, m4 +%endif + psraw m0, 4 + psraw m2, 4 +%if %2 == 1 ; avg + ; FIXME(rbultje) pipeline +%if %1 == 4 + movlhps m0, m2 +%endif + packuswb m0, m2 +%if %1 > 4 + pavgb m0, [secq] + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%else + movh m2, [secq] + pavgb m0, m2 + punpcklbw m0, m5 + movhlps m2, m0 +%endif +%endif + punpcklbw m1, m5 + SUM_SSE m0, m1, m2, m3, m6, m7 + + lea srcq, [srcq+src_strideq*2] + lea dstq, [dstq+dst_strideq*2] +%endif +%if %2 == 1 ; avg + add secq, sec_str +%endif + dec block_height + jg .x_other_y_zero_loop +%undef filter_x_a +%undef filter_x_b +%undef filter_rnd + STORE_AND_RET %1 + +.x_nonhalf_y_nonzero: + cmp y_offsetd, 4 + jne .x_nonhalf_y_nonhalf + + ; x_offset == bilin interpolation && y_offset == 0.5 +%ifdef PIC + lea bilin_filter, [bilin_filter_m] +%endif + shl x_offsetd, filter_idx_shift +%if ARCH_X86_64 && %1 > 4 + mova m8, [bilin_filter+x_offsetq] +%if notcpuflag(ssse3) ; FIXME(rbultje) don't scatter registers on x86-64 + mova m9, [bilin_filter+x_offsetq+16] +%endif + mova m10, [pw_8] +%define filter_x_a m8 +%define filter_x_b m9 +%define filter_rnd m10 +%else ; x86-32 +%if ARCH_X86=1 && CONFIG_PIC=1 +; y_offset == 0.5. We can reuse y_offset reg. +%define tempq y_offsetq + add x_offsetq, g_bilin_filterm +%define filter_x_a [x_offsetq] +%define filter_x_b [x_offsetq+16] + mov tempq, g_pw_8m +%define filter_rnd [tempq] +%else + add x_offsetq, bilin_filter +%define filter_x_a [x_offsetq] +%define filter_x_b [x_offsetq+16] +%define filter_rnd [pw_8] +%endif +%endif + +%if %1 == 16 + movu m0, [srcq] + movu m1, [srcq+1] +%if cpuflag(ssse3) + punpckhbw m2, m0, m1 + punpcklbw m0, m1 + pmaddubsw m2, filter_x_a + pmaddubsw m0, filter_x_a + paddw m2, filter_rnd + paddw m0, filter_rnd +%else + punpckhbw m2, m0, m5 + punpckhbw m3, m1, m5 + punpcklbw m0, m5 + punpcklbw m1, m5 + pmullw m0, filter_x_a + pmullw m1, filter_x_b + paddw m0, filter_rnd + pmullw m2, filter_x_a + pmullw m3, filter_x_b + paddw m2, filter_rnd + paddw m0, m1 + paddw m2, m3 +%endif + psraw m0, 4 + psraw m2, 4 + add srcq, src_strideq + packuswb m0, m2 +.x_other_y_half_loop: + movu m4, [srcq] + movu m3, [srcq+1] +%if cpuflag(ssse3) + mova m1, [dstq] + punpckhbw m2, m4, m3 + punpcklbw m4, m3 + pmaddubsw m2, filter_x_a + pmaddubsw m4, filter_x_a + paddw m2, filter_rnd + paddw m4, filter_rnd + psraw m2, 4 + psraw m4, 4 + packuswb m4, m2 + pavgb m0, m4 + punpckhbw m3, m1, m5 + punpcklbw m1, m5 +%else + punpckhbw m2, m4, m5 + punpckhbw m1, m3, m5 + punpcklbw m4, m5 + punpcklbw m3, m5 + pmullw m4, filter_x_a + pmullw m3, filter_x_b + paddw m4, filter_rnd + pmullw m2, filter_x_a + pmullw m1, filter_x_b + paddw m2, filter_rnd + paddw m4, m3 + paddw m2, m1 + mova m1, [dstq] + psraw m4, 4 + psraw m2, 4 + punpckhbw m3, m1, m5 + ; FIXME(rbultje) the repeated pack/unpack here around m0/m2 is because we + ; have a 1-register shortage to be able to store the backup of the bilin + ; filtered second line as words as cache for the next line. Packing into + ; a byte costs 1 pack and 2 unpacks, but saves a register. + packuswb m4, m2 + punpcklbw m1, m5 + pavgb m0, m4 +%endif +%if %2 == 1 ; avg + ; FIXME(rbultje) pipeline + pavgb m0, [secq] +%endif + punpckhbw m2, m0, m5 + punpcklbw m0, m5 + SUM_SSE m0, m1, m2, m3, m6, m7 + mova m0, m4 + + add srcq, src_strideq + add dstq, dst_strideq +%else ; %1 < 16 + movx m0, [srcq] + movx m1, [srcq+1] +%if cpuflag(ssse3) + punpcklbw m0, m1 + pmaddubsw m0, filter_x_a + paddw m0, filter_rnd +%else + punpcklbw m0, m5 + punpcklbw m1, m5 + pmullw m0, filter_x_a + pmullw m1, filter_x_b + paddw m0, filter_rnd + paddw m0, m1 +%endif + add srcq, src_strideq + psraw m0, 4 +.x_other_y_half_loop: + movx m2, [srcq] + movx m1, [srcq+1] + movx m4, [srcq+src_strideq] + movx m3, [srcq+src_strideq+1] +%if cpuflag(ssse3) + punpcklbw m2, m1 + punpcklbw m4, m3 + pmaddubsw m2, filter_x_a + pmaddubsw m4, filter_x_a + movx m1, [dstq] + movx m3, [dstq+dst_strideq] + paddw m2, filter_rnd + paddw m4, filter_rnd +%else + punpcklbw m2, m5 + punpcklbw m1, m5 + punpcklbw m4, m5 + punpcklbw m3, m5 + pmullw m2, filter_x_a + pmullw m1, filter_x_b + paddw m2, filter_rnd + pmullw m4, filter_x_a + pmullw m3, filter_x_b + paddw m4, filter_rnd + paddw m2, m1 + movx m1, [dstq] + paddw m4, m3 + movx m3, [dstq+dst_strideq] +%endif + psraw m2, 4 + psraw m4, 4 + pavgw m0, m2 + pavgw m2, m4 +%if %2 == 1 ; avg + ; FIXME(rbultje) pipeline - also consider going to bytes here +%if %1 == 4 + movlhps m0, m2 +%endif + packuswb m0, m2 +%if %1 > 4 + pavgb m0, [secq] + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%else + movh m2, [secq] + pavgb m0, m2 + punpcklbw m0, m5 + movhlps m2, m0 +%endif +%endif + punpcklbw m3, m5 + punpcklbw m1, m5 + SUM_SSE m0, m1, m2, m3, m6, m7 + mova m0, m4 + + lea srcq, [srcq+src_strideq*2] + lea dstq, [dstq+dst_strideq*2] +%endif +%if %2 == 1 ; avg + add secq, sec_str +%endif + dec block_height + jg .x_other_y_half_loop +%undef filter_x_a +%undef filter_x_b +%undef filter_rnd + STORE_AND_RET %1 + +.x_nonhalf_y_nonhalf: +%ifdef PIC + lea bilin_filter, [bilin_filter_m] +%endif + shl x_offsetd, filter_idx_shift + shl y_offsetd, filter_idx_shift +%if ARCH_X86_64 && %1 > 4 + mova m8, [bilin_filter+x_offsetq] +%if notcpuflag(ssse3) ; FIXME(rbultje) don't scatter registers on x86-64 + mova m9, [bilin_filter+x_offsetq+16] +%endif + mova m10, [bilin_filter+y_offsetq] +%if notcpuflag(ssse3) ; FIXME(rbultje) don't scatter registers on x86-64 + mova m11, [bilin_filter+y_offsetq+16] +%endif + mova m12, [pw_8] +%define filter_x_a m8 +%define filter_x_b m9 +%define filter_y_a m10 +%define filter_y_b m11 +%define filter_rnd m12 +%else ; x86-32 +%if ARCH_X86=1 && CONFIG_PIC=1 +; In this case, there is NO unused register. Used src_stride register. Later, +; src_stride has to be loaded from stack when it is needed. +%define tempq src_strideq + mov tempq, g_bilin_filterm + add x_offsetq, tempq + add y_offsetq, tempq +%define filter_x_a [x_offsetq] +%define filter_x_b [x_offsetq+16] +%define filter_y_a [y_offsetq] +%define filter_y_b [y_offsetq+16] + + mov tempq, g_pw_8m +%define filter_rnd [tempq] +%else + add x_offsetq, bilin_filter + add y_offsetq, bilin_filter +%define filter_x_a [x_offsetq] +%define filter_x_b [x_offsetq+16] +%define filter_y_a [y_offsetq] +%define filter_y_b [y_offsetq+16] +%define filter_rnd [pw_8] +%endif +%endif + + ; x_offset == bilin interpolation && y_offset == bilin interpolation +%if %1 == 16 + movu m0, [srcq] + movu m1, [srcq+1] +%if cpuflag(ssse3) + punpckhbw m2, m0, m1 + punpcklbw m0, m1 + pmaddubsw m2, filter_x_a + pmaddubsw m0, filter_x_a + paddw m2, filter_rnd + paddw m0, filter_rnd +%else + punpckhbw m2, m0, m5 + punpckhbw m3, m1, m5 + punpcklbw m0, m5 + punpcklbw m1, m5 + pmullw m0, filter_x_a + pmullw m1, filter_x_b + paddw m0, filter_rnd + pmullw m2, filter_x_a + pmullw m3, filter_x_b + paddw m2, filter_rnd + paddw m0, m1 + paddw m2, m3 +%endif + psraw m0, 4 + psraw m2, 4 + + INC_SRC_BY_SRC_STRIDE + + packuswb m0, m2 +.x_other_y_other_loop: +%if cpuflag(ssse3) + movu m4, [srcq] + movu m3, [srcq+1] + mova m1, [dstq] + punpckhbw m2, m4, m3 + punpcklbw m4, m3 + pmaddubsw m2, filter_x_a + pmaddubsw m4, filter_x_a + punpckhbw m3, m1, m5 + paddw m2, filter_rnd + paddw m4, filter_rnd + psraw m2, 4 + psraw m4, 4 + packuswb m4, m2 + punpckhbw m2, m0, m4 + punpcklbw m0, m4 + pmaddubsw m2, filter_y_a + pmaddubsw m0, filter_y_a + punpcklbw m1, m5 + paddw m2, filter_rnd + paddw m0, filter_rnd + psraw m2, 4 + psraw m0, 4 +%else + movu m3, [srcq] + movu m4, [srcq+1] + punpckhbw m1, m3, m5 + punpckhbw m2, m4, m5 + punpcklbw m3, m5 + punpcklbw m4, m5 + pmullw m3, filter_x_a + pmullw m4, filter_x_b + paddw m3, filter_rnd + pmullw m1, filter_x_a + pmullw m2, filter_x_b + paddw m1, filter_rnd + paddw m3, m4 + paddw m1, m2 + psraw m3, 4 + psraw m1, 4 + packuswb m4, m3, m1 + punpckhbw m2, m0, m5 + punpcklbw m0, m5 + pmullw m2, filter_y_a + pmullw m1, filter_y_b + paddw m2, filter_rnd + pmullw m0, filter_y_a + pmullw m3, filter_y_b + paddw m2, m1 + mova m1, [dstq] + paddw m0, filter_rnd + psraw m2, 4 + paddw m0, m3 + punpckhbw m3, m1, m5 + psraw m0, 4 + punpcklbw m1, m5 +%endif +%if %2 == 1 ; avg + ; FIXME(rbultje) pipeline + packuswb m0, m2 + pavgb m0, [secq] + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%endif + SUM_SSE m0, m1, m2, m3, m6, m7 + mova m0, m4 + + INC_SRC_BY_SRC_STRIDE + add dstq, dst_strideq +%else ; %1 < 16 + movx m0, [srcq] + movx m1, [srcq+1] +%if cpuflag(ssse3) + punpcklbw m0, m1 + pmaddubsw m0, filter_x_a + paddw m0, filter_rnd +%else + punpcklbw m0, m5 + punpcklbw m1, m5 + pmullw m0, filter_x_a + pmullw m1, filter_x_b + paddw m0, filter_rnd + paddw m0, m1 +%endif + psraw m0, 4 +%if cpuflag(ssse3) + packuswb m0, m0 +%endif + + INC_SRC_BY_SRC_STRIDE + +.x_other_y_other_loop: + movx m2, [srcq] + movx m1, [srcq+1] + + INC_SRC_BY_SRC_STRIDE + movx m4, [srcq] + movx m3, [srcq+1] + +%if cpuflag(ssse3) + punpcklbw m2, m1 + punpcklbw m4, m3 + pmaddubsw m2, filter_x_a + pmaddubsw m4, filter_x_a + movx m3, [dstq+dst_strideq] + movx m1, [dstq] + paddw m2, filter_rnd + paddw m4, filter_rnd + psraw m2, 4 + psraw m4, 4 + packuswb m2, m2 + packuswb m4, m4 + punpcklbw m0, m2 + punpcklbw m2, m4 + pmaddubsw m0, filter_y_a + pmaddubsw m2, filter_y_a + punpcklbw m3, m5 + paddw m0, filter_rnd + paddw m2, filter_rnd + psraw m0, 4 + psraw m2, 4 + punpcklbw m1, m5 +%else + punpcklbw m2, m5 + punpcklbw m1, m5 + punpcklbw m4, m5 + punpcklbw m3, m5 + pmullw m2, filter_x_a + pmullw m1, filter_x_b + paddw m2, filter_rnd + pmullw m4, filter_x_a + pmullw m3, filter_x_b + paddw m4, filter_rnd + paddw m2, m1 + paddw m4, m3 + psraw m2, 4 + psraw m4, 4 + pmullw m0, filter_y_a + pmullw m3, m2, filter_y_b + paddw m0, filter_rnd + pmullw m2, filter_y_a + pmullw m1, m4, filter_y_b + paddw m2, filter_rnd + paddw m0, m3 + movx m3, [dstq+dst_strideq] + paddw m2, m1 + movx m1, [dstq] + psraw m0, 4 + psraw m2, 4 + punpcklbw m3, m5 + punpcklbw m1, m5 +%endif +%if %2 == 1 ; avg + ; FIXME(rbultje) pipeline +%if %1 == 4 + movlhps m0, m2 +%endif + packuswb m0, m2 +%if %1 > 4 + pavgb m0, [secq] + punpckhbw m2, m0, m5 + punpcklbw m0, m5 +%else + movh m2, [secq] + pavgb m0, m2 + punpcklbw m0, m5 + movhlps m2, m0 +%endif +%endif + SUM_SSE m0, m1, m2, m3, m6, m7 + mova m0, m4 + + INC_SRC_BY_SRC_STRIDE + lea dstq, [dstq+dst_strideq*2] +%endif +%if %2 == 1 ; avg + add secq, sec_str +%endif + dec block_height + jg .x_other_y_other_loop +%undef filter_x_a +%undef filter_x_b +%undef filter_y_a +%undef filter_y_b +%undef filter_rnd +%undef movx + STORE_AND_RET %1 +%endmacro + +; FIXME(rbultje) the non-bilinear versions (i.e. x=0,8&&y=0,8) are identical +; between the ssse3 and non-ssse3 version. It may make sense to merge their +; code in the sense that the ssse3 version would jump to the appropriate +; location in the sse/2 version, rather than duplicating that code in the +; binary. + +INIT_XMM sse2 +SUBPEL_VARIANCE 4 +SUBPEL_VARIANCE 8 +SUBPEL_VARIANCE 16 + +INIT_XMM ssse3 +SUBPEL_VARIANCE 4 +SUBPEL_VARIANCE 8 +SUBPEL_VARIANCE 16 + +INIT_XMM sse2 +SUBPEL_VARIANCE 4, 1 +SUBPEL_VARIANCE 8, 1 +SUBPEL_VARIANCE 16, 1 + +INIT_XMM ssse3 +SUBPEL_VARIANCE 4, 1 +SUBPEL_VARIANCE 8, 1 +SUBPEL_VARIANCE 16, 1 diff --git a/third_party/aom/aom_dsp/x86/subtract_sse2.asm b/third_party/aom/aom_dsp/x86/subtract_sse2.asm new file mode 100644 index 0000000000..7bd5b23ad2 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/subtract_sse2.asm @@ -0,0 +1,150 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION .text + +; void aom_subtract_block(int rows, int cols, +; int16_t *diff, ptrdiff_t diff_stride, +; const uint8_t *src, ptrdiff_t src_stride, +; const uint8_t *pred, ptrdiff_t pred_stride) + +INIT_XMM sse2 +cglobal subtract_block, 7, 7, 8, \ + rows, cols, diff, diff_stride, src, src_stride, \ + pred, pred_stride +%define pred_str colsq + pxor m7, m7 ; dedicated zero register + cmp colsd, 4 + je .case_4 + cmp colsd, 8 + je .case_8 + cmp colsd, 16 + je .case_16 + cmp colsd, 32 + je .case_32 +%if CONFIG_EXT_PARTITION + cmp colsd, 64 + je .case_64 +%endif + +%macro loop16 6 + mova m0, [srcq+%1] + mova m4, [srcq+%2] + mova m1, [predq+%3] + mova m5, [predq+%4] + punpckhbw m2, m0, m7 + punpckhbw m3, m1, m7 + punpcklbw m0, m7 + punpcklbw m1, m7 + psubw m2, m3 + psubw m0, m1 + punpckhbw m1, m4, m7 + punpckhbw m3, m5, m7 + punpcklbw m4, m7 + punpcklbw m5, m7 + psubw m1, m3 + psubw m4, m5 + mova [diffq+mmsize*0+%5], m0 + mova [diffq+mmsize*1+%5], m2 + mova [diffq+mmsize*0+%6], m4 + mova [diffq+mmsize*1+%6], m1 +%endmacro + +%if CONFIG_EXT_PARTITION + mov pred_str, pred_stridemp +.loop_128: + loop16 0*mmsize, 1*mmsize, 0*mmsize, 1*mmsize, 0*mmsize, 2*mmsize + loop16 2*mmsize, 3*mmsize, 2*mmsize, 3*mmsize, 4*mmsize, 6*mmsize + loop16 4*mmsize, 5*mmsize, 4*mmsize, 5*mmsize, 8*mmsize, 10*mmsize + loop16 6*mmsize, 7*mmsize, 6*mmsize, 7*mmsize, 12*mmsize, 14*mmsize + lea diffq, [diffq+diff_strideq*2] + add predq, pred_str + add srcq, src_strideq + sub rowsd, 1 + jnz .loop_128 + RET + +.case_64: +%endif + mov pred_str, pred_stridemp +.loop_64: + loop16 0*mmsize, 1*mmsize, 0*mmsize, 1*mmsize, 0*mmsize, 2*mmsize + loop16 2*mmsize, 3*mmsize, 2*mmsize, 3*mmsize, 4*mmsize, 6*mmsize + lea diffq, [diffq+diff_strideq*2] + add predq, pred_str + add srcq, src_strideq + dec rowsd + jg .loop_64 + RET + +.case_32: + mov pred_str, pred_stridemp +.loop_32: + loop16 0, mmsize, 0, mmsize, 0, 2*mmsize + lea diffq, [diffq+diff_strideq*2] + add predq, pred_str + add srcq, src_strideq + dec rowsd + jg .loop_32 + RET + +.case_16: + mov pred_str, pred_stridemp +.loop_16: + loop16 0, src_strideq, 0, pred_str, 0, diff_strideq*2 + lea diffq, [diffq+diff_strideq*4] + lea predq, [predq+pred_str*2] + lea srcq, [srcq+src_strideq*2] + sub rowsd, 2 + jg .loop_16 + RET + +%macro loop_h 0 + movh m0, [srcq] + movh m2, [srcq+src_strideq] + movh m1, [predq] + movh m3, [predq+pred_str] + punpcklbw m0, m7 + punpcklbw m1, m7 + punpcklbw m2, m7 + punpcklbw m3, m7 + psubw m0, m1 + psubw m2, m3 + mova [diffq], m0 + mova [diffq+diff_strideq*2], m2 +%endmacro + +.case_8: + mov pred_str, pred_stridemp +.loop_8: + loop_h + lea diffq, [diffq+diff_strideq*4] + lea srcq, [srcq+src_strideq*2] + lea predq, [predq+pred_str*2] + sub rowsd, 2 + jg .loop_8 + RET + +INIT_MMX +.case_4: + mov pred_str, pred_stridemp +.loop_4: + loop_h + lea diffq, [diffq+diff_strideq*4] + lea srcq, [srcq+src_strideq*2] + lea predq, [predq+pred_str*2] + sub rowsd, 2 + jg .loop_4 + RET diff --git a/third_party/aom/aom_dsp/x86/sum_squares_sse2.c b/third_party/aom/aom_dsp/x86/sum_squares_sse2.c new file mode 100644 index 0000000000..6be99fbcac --- /dev/null +++ b/third_party/aom/aom_dsp/x86/sum_squares_sse2.c @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "aom_dsp/x86/synonyms.h" + +#include "./aom_dsp_rtcd.h" + +static uint64_t aom_sum_squares_2d_i16_4x4_sse2(const int16_t *src, + int stride) { + const __m128i v_val_0_w = + _mm_loadl_epi64((const __m128i *)(src + 0 * stride)); + const __m128i v_val_1_w = + _mm_loadl_epi64((const __m128i *)(src + 1 * stride)); + const __m128i v_val_2_w = + _mm_loadl_epi64((const __m128i *)(src + 2 * stride)); + const __m128i v_val_3_w = + _mm_loadl_epi64((const __m128i *)(src + 3 * stride)); + + const __m128i v_sq_0_d = _mm_madd_epi16(v_val_0_w, v_val_0_w); + const __m128i v_sq_1_d = _mm_madd_epi16(v_val_1_w, v_val_1_w); + const __m128i v_sq_2_d = _mm_madd_epi16(v_val_2_w, v_val_2_w); + const __m128i v_sq_3_d = _mm_madd_epi16(v_val_3_w, v_val_3_w); + + const __m128i v_sum_01_d = _mm_add_epi32(v_sq_0_d, v_sq_1_d); + const __m128i v_sum_23_d = _mm_add_epi32(v_sq_2_d, v_sq_3_d); + const __m128i v_sum_0123_d = _mm_add_epi32(v_sum_01_d, v_sum_23_d); + + const __m128i v_sum_d = + _mm_add_epi32(v_sum_0123_d, _mm_srli_epi64(v_sum_0123_d, 32)); + + return (uint64_t)_mm_cvtsi128_si32(v_sum_d); +} + +#ifdef __GNUC__ +// This prevents GCC/Clang from inlining this function into +// aom_sum_squares_2d_i16_sse2, which in turn saves some stack +// maintenance instructions in the common case of 4x4. +__attribute__((noinline)) +#endif +static uint64_t +aom_sum_squares_2d_i16_nxn_sse2(const int16_t *src, int stride, int width, + int height) { + int r, c; + + const __m128i v_zext_mask_q = _mm_set_epi32(0, 0xffffffff, 0, 0xffffffff); + __m128i v_acc_q = _mm_setzero_si128(); + + for (r = 0; r < height; r += 8) { + __m128i v_acc_d = _mm_setzero_si128(); + + for (c = 0; c < width; c += 8) { + const int16_t *b = src + c; + + const __m128i v_val_0_w = + _mm_load_si128((const __m128i *)(b + 0 * stride)); + const __m128i v_val_1_w = + _mm_load_si128((const __m128i *)(b + 1 * stride)); + const __m128i v_val_2_w = + _mm_load_si128((const __m128i *)(b + 2 * stride)); + const __m128i v_val_3_w = + _mm_load_si128((const __m128i *)(b + 3 * stride)); + const __m128i v_val_4_w = + _mm_load_si128((const __m128i *)(b + 4 * stride)); + const __m128i v_val_5_w = + _mm_load_si128((const __m128i *)(b + 5 * stride)); + const __m128i v_val_6_w = + _mm_load_si128((const __m128i *)(b + 6 * stride)); + const __m128i v_val_7_w = + _mm_load_si128((const __m128i *)(b + 7 * stride)); + + const __m128i v_sq_0_d = _mm_madd_epi16(v_val_0_w, v_val_0_w); + const __m128i v_sq_1_d = _mm_madd_epi16(v_val_1_w, v_val_1_w); + const __m128i v_sq_2_d = _mm_madd_epi16(v_val_2_w, v_val_2_w); + const __m128i v_sq_3_d = _mm_madd_epi16(v_val_3_w, v_val_3_w); + const __m128i v_sq_4_d = _mm_madd_epi16(v_val_4_w, v_val_4_w); + const __m128i v_sq_5_d = _mm_madd_epi16(v_val_5_w, v_val_5_w); + const __m128i v_sq_6_d = _mm_madd_epi16(v_val_6_w, v_val_6_w); + const __m128i v_sq_7_d = _mm_madd_epi16(v_val_7_w, v_val_7_w); + + const __m128i v_sum_01_d = _mm_add_epi32(v_sq_0_d, v_sq_1_d); + const __m128i v_sum_23_d = _mm_add_epi32(v_sq_2_d, v_sq_3_d); + const __m128i v_sum_45_d = _mm_add_epi32(v_sq_4_d, v_sq_5_d); + const __m128i v_sum_67_d = _mm_add_epi32(v_sq_6_d, v_sq_7_d); + + const __m128i v_sum_0123_d = _mm_add_epi32(v_sum_01_d, v_sum_23_d); + const __m128i v_sum_4567_d = _mm_add_epi32(v_sum_45_d, v_sum_67_d); + + v_acc_d = _mm_add_epi32(v_acc_d, v_sum_0123_d); + v_acc_d = _mm_add_epi32(v_acc_d, v_sum_4567_d); + } + + v_acc_q = _mm_add_epi64(v_acc_q, _mm_and_si128(v_acc_d, v_zext_mask_q)); + v_acc_q = _mm_add_epi64(v_acc_q, _mm_srli_epi64(v_acc_d, 32)); + + src += 8 * stride; + } + + v_acc_q = _mm_add_epi64(v_acc_q, _mm_srli_si128(v_acc_q, 8)); + +#if ARCH_X86_64 + return (uint64_t)_mm_cvtsi128_si64(v_acc_q); +#else + { + uint64_t tmp; + _mm_storel_epi64((__m128i *)&tmp, v_acc_q); + return tmp; + } +#endif +} + +uint64_t aom_sum_squares_2d_i16_sse2(const int16_t *src, int stride, int width, + int height) { + // 4 elements per row only requires half an XMM register, so this + // must be a special case, but also note that over 75% of all calls + // are with size == 4, so it is also the common case. + if (LIKELY(width == 4 && height == 4)) { + return aom_sum_squares_2d_i16_4x4_sse2(src, stride); + } else if (LIKELY(width % 8 == 0 && height % 8 == 0)) { + // Generic case + return aom_sum_squares_2d_i16_nxn_sse2(src, stride, width, height); + } else { + return aom_sum_squares_2d_i16_c(src, stride, width, height); + } +} + +////////////////////////////////////////////////////////////////////////////// +// 1D version +////////////////////////////////////////////////////////////////////////////// + +static uint64_t aom_sum_squares_i16_64n_sse2(const int16_t *src, uint32_t n) { + const __m128i v_zext_mask_q = _mm_set_epi32(0, 0xffffffff, 0, 0xffffffff); + __m128i v_acc0_q = _mm_setzero_si128(); + __m128i v_acc1_q = _mm_setzero_si128(); + + const int16_t *const end = src + n; + + assert(n % 64 == 0); + + while (src < end) { + const __m128i v_val_0_w = xx_load_128(src); + const __m128i v_val_1_w = xx_load_128(src + 8); + const __m128i v_val_2_w = xx_load_128(src + 16); + const __m128i v_val_3_w = xx_load_128(src + 24); + const __m128i v_val_4_w = xx_load_128(src + 32); + const __m128i v_val_5_w = xx_load_128(src + 40); + const __m128i v_val_6_w = xx_load_128(src + 48); + const __m128i v_val_7_w = xx_load_128(src + 56); + + const __m128i v_sq_0_d = _mm_madd_epi16(v_val_0_w, v_val_0_w); + const __m128i v_sq_1_d = _mm_madd_epi16(v_val_1_w, v_val_1_w); + const __m128i v_sq_2_d = _mm_madd_epi16(v_val_2_w, v_val_2_w); + const __m128i v_sq_3_d = _mm_madd_epi16(v_val_3_w, v_val_3_w); + const __m128i v_sq_4_d = _mm_madd_epi16(v_val_4_w, v_val_4_w); + const __m128i v_sq_5_d = _mm_madd_epi16(v_val_5_w, v_val_5_w); + const __m128i v_sq_6_d = _mm_madd_epi16(v_val_6_w, v_val_6_w); + const __m128i v_sq_7_d = _mm_madd_epi16(v_val_7_w, v_val_7_w); + + const __m128i v_sum_01_d = _mm_add_epi32(v_sq_0_d, v_sq_1_d); + const __m128i v_sum_23_d = _mm_add_epi32(v_sq_2_d, v_sq_3_d); + const __m128i v_sum_45_d = _mm_add_epi32(v_sq_4_d, v_sq_5_d); + const __m128i v_sum_67_d = _mm_add_epi32(v_sq_6_d, v_sq_7_d); + + const __m128i v_sum_0123_d = _mm_add_epi32(v_sum_01_d, v_sum_23_d); + const __m128i v_sum_4567_d = _mm_add_epi32(v_sum_45_d, v_sum_67_d); + + const __m128i v_sum_d = _mm_add_epi32(v_sum_0123_d, v_sum_4567_d); + + v_acc0_q = _mm_add_epi64(v_acc0_q, _mm_and_si128(v_sum_d, v_zext_mask_q)); + v_acc1_q = _mm_add_epi64(v_acc1_q, _mm_srli_epi64(v_sum_d, 32)); + + src += 64; + } + + v_acc0_q = _mm_add_epi64(v_acc0_q, v_acc1_q); + v_acc0_q = _mm_add_epi64(v_acc0_q, _mm_srli_si128(v_acc0_q, 8)); + +#if ARCH_X86_64 + return (uint64_t)_mm_cvtsi128_si64(v_acc0_q); +#else + { + uint64_t tmp; + _mm_storel_epi64((__m128i *)&tmp, v_acc0_q); + return tmp; + } +#endif +} + +uint64_t aom_sum_squares_i16_sse2(const int16_t *src, uint32_t n) { + if (n % 64 == 0) { + return aom_sum_squares_i16_64n_sse2(src, n); + } else if (n > 64) { + int k = n & ~(64 - 1); + return aom_sum_squares_i16_64n_sse2(src, k) + + aom_sum_squares_i16_c(src + k, n - k); + } else { + return aom_sum_squares_i16_c(src, n); + } +} diff --git a/third_party/aom/aom_dsp/x86/synonyms.h b/third_party/aom/aom_dsp/x86/synonyms.h new file mode 100644 index 0000000000..bef606dae7 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/synonyms.h @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_X86_SYNONYMS_H_ +#define AOM_DSP_X86_SYNONYMS_H_ + +#include + +#include "./aom_config.h" +#include "aom/aom_integer.h" + +/** + * Various reusable shorthands for x86 SIMD intrinsics. + * + * Intrinsics prefixed with xx_ operate on or return 128bit XMM registers. + * Intrinsics prefixed with yy_ operate on or return 256bit YMM registers. + */ + +// Loads and stores to do away with the tedium of casting the address +// to the right type. +static INLINE __m128i xx_loadl_32(const void *a) { + return _mm_cvtsi32_si128(*(const uint32_t *)a); +} + +static INLINE __m128i xx_loadl_64(const void *a) { + return _mm_loadl_epi64((const __m128i *)a); +} + +static INLINE __m128i xx_load_128(const void *a) { + return _mm_load_si128((const __m128i *)a); +} + +static INLINE __m128i xx_loadu_128(const void *a) { + return _mm_loadu_si128((const __m128i *)a); +} + +static INLINE void xx_storel_32(void *const a, const __m128i v) { + *(uint32_t *)a = _mm_cvtsi128_si32(v); +} + +static INLINE void xx_storel_64(void *const a, const __m128i v) { + _mm_storel_epi64((__m128i *)a, v); +} + +static INLINE void xx_store_128(void *const a, const __m128i v) { + _mm_store_si128((__m128i *)a, v); +} + +static INLINE void xx_storeu_128(void *const a, const __m128i v) { + _mm_storeu_si128((__m128i *)a, v); +} + +static INLINE __m128i xx_round_epu16(__m128i v_val_w) { + return _mm_avg_epu16(v_val_w, _mm_setzero_si128()); +} + +static INLINE __m128i xx_roundn_epu16(__m128i v_val_w, int bits) { + const __m128i v_s_w = _mm_srli_epi16(v_val_w, bits - 1); + return _mm_avg_epu16(v_s_w, _mm_setzero_si128()); +} + +static INLINE __m128i xx_roundn_epu32(__m128i v_val_d, int bits) { + const __m128i v_bias_d = _mm_set1_epi32((1 << bits) >> 1); + const __m128i v_tmp_d = _mm_add_epi32(v_val_d, v_bias_d); + return _mm_srli_epi32(v_tmp_d, bits); +} + +// This is equivalent to ROUND_POWER_OF_TWO(v_val_d, bits) +static INLINE __m128i xx_roundn_epi32_unsigned(__m128i v_val_d, int bits) { + const __m128i v_bias_d = _mm_set1_epi32((1 << bits) >> 1); + const __m128i v_tmp_d = _mm_add_epi32(v_val_d, v_bias_d); + return _mm_srai_epi32(v_tmp_d, bits); +} + +// This is equivalent to ROUND_POWER_OF_TWO_SIGNED(v_val_d, bits) +static INLINE __m128i xx_roundn_epi32(__m128i v_val_d, int bits) { + const __m128i v_bias_d = _mm_set1_epi32((1 << bits) >> 1); + const __m128i v_sign_d = _mm_srai_epi32(v_val_d, 31); + const __m128i v_tmp_d = + _mm_add_epi32(_mm_add_epi32(v_val_d, v_bias_d), v_sign_d); + return _mm_srai_epi32(v_tmp_d, bits); +} + +#ifdef __SSSE3__ +static INLINE int32_t xx_hsum_epi32_si32(__m128i v_d) { + v_d = _mm_hadd_epi32(v_d, v_d); + v_d = _mm_hadd_epi32(v_d, v_d); + return _mm_cvtsi128_si32(v_d); +} + +static INLINE int64_t xx_hsum_epi64_si64(__m128i v_q) { + v_q = _mm_add_epi64(v_q, _mm_srli_si128(v_q, 8)); +#if ARCH_X86_64 + return _mm_cvtsi128_si64(v_q); +#else + { + int64_t tmp; + _mm_storel_epi64((__m128i *)&tmp, v_q); + return tmp; + } +#endif +} + +static INLINE int64_t xx_hsum_epi32_si64(__m128i v_d) { + const __m128i v_sign_d = _mm_cmplt_epi32(v_d, _mm_setzero_si128()); + const __m128i v_0_q = _mm_unpacklo_epi32(v_d, v_sign_d); + const __m128i v_1_q = _mm_unpackhi_epi32(v_d, v_sign_d); + return xx_hsum_epi64_si64(_mm_add_epi64(v_0_q, v_1_q)); +} +#endif // __SSSE3__ + +#endif // AOM_DSP_X86_SYNONYMS_H_ diff --git a/third_party/aom/aom_dsp/x86/txfm_common_avx2.h b/third_party/aom/aom_dsp/x86/txfm_common_avx2.h new file mode 100644 index 0000000000..39e9b8e2ad --- /dev/null +++ b/third_party/aom/aom_dsp/x86/txfm_common_avx2.h @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_X86_TXFM_COMMON_AVX2_H +#define AOM_DSP_X86_TXFM_COMMON_AVX2_H + +#include + +#include "aom_dsp/txfm_common.h" + +#define pair256_set_epi16(a, b) \ + _mm256_set_epi16((int16_t)(b), (int16_t)(a), (int16_t)(b), (int16_t)(a), \ + (int16_t)(b), (int16_t)(a), (int16_t)(b), (int16_t)(a), \ + (int16_t)(b), (int16_t)(a), (int16_t)(b), (int16_t)(a), \ + (int16_t)(b), (int16_t)(a), (int16_t)(b), (int16_t)(a)) + +#define pair256_set_epi32(a, b) \ + _mm256_set_epi32((int)(b), (int)(a), (int)(b), (int)(a), (int)(b), (int)(a), \ + (int)(b), (int)(a)) + +static INLINE void mm256_reverse_epi16(__m256i *u) { + const __m256i control = _mm256_set_epi16( + 0x0100, 0x0302, 0x0504, 0x0706, 0x0908, 0x0B0A, 0x0D0C, 0x0F0E, 0x0100, + 0x0302, 0x0504, 0x0706, 0x0908, 0x0B0A, 0x0D0C, 0x0F0E); + __m256i v = _mm256_shuffle_epi8(*u, control); + *u = _mm256_permute2x128_si256(v, v, 1); +} + +static INLINE void mm256_transpose_16x16(__m256i *in) { + __m256i tr0_0 = _mm256_unpacklo_epi16(in[0], in[1]); + __m256i tr0_1 = _mm256_unpackhi_epi16(in[0], in[1]); + __m256i tr0_2 = _mm256_unpacklo_epi16(in[2], in[3]); + __m256i tr0_3 = _mm256_unpackhi_epi16(in[2], in[3]); + __m256i tr0_4 = _mm256_unpacklo_epi16(in[4], in[5]); + __m256i tr0_5 = _mm256_unpackhi_epi16(in[4], in[5]); + __m256i tr0_6 = _mm256_unpacklo_epi16(in[6], in[7]); + __m256i tr0_7 = _mm256_unpackhi_epi16(in[6], in[7]); + + __m256i tr0_8 = _mm256_unpacklo_epi16(in[8], in[9]); + __m256i tr0_9 = _mm256_unpackhi_epi16(in[8], in[9]); + __m256i tr0_a = _mm256_unpacklo_epi16(in[10], in[11]); + __m256i tr0_b = _mm256_unpackhi_epi16(in[10], in[11]); + __m256i tr0_c = _mm256_unpacklo_epi16(in[12], in[13]); + __m256i tr0_d = _mm256_unpackhi_epi16(in[12], in[13]); + __m256i tr0_e = _mm256_unpacklo_epi16(in[14], in[15]); + __m256i tr0_f = _mm256_unpackhi_epi16(in[14], in[15]); + + // 00 10 01 11 02 12 03 13 08 18 09 19 0a 1a 0b 1b + // 04 14 05 15 06 16 07 17 0c 1c 0d 1d 0e 1e 0f 1f + // 20 30 21 31 22 32 23 33 28 38 29 39 2a 3a 2b 3b + // 24 34 25 35 26 36 27 37 2c 3c 2d 3d 2e 3e 2f 3f + // 40 50 41 51 42 52 43 53 48 58 49 59 4a 5a 4b 5b + // 44 54 45 55 46 56 47 57 4c 5c 4d 5d 4e 5e 4f 5f + // 60 70 61 71 62 72 63 73 68 78 69 79 6a 7a 6b 7b + // 64 74 65 75 66 76 67 77 6c 7c 6d 7d 6e 7e 6f 7f + + // 80 90 81 91 82 92 83 93 88 98 89 99 8a 9a 8b 9b + // 84 94 85 95 86 96 87 97 8c 9c 8d 9d 8e 9e 8f 9f + // a0 b0 a1 b1 a2 b2 a3 b3 a8 b8 a9 b9 aa ba ab bb + // a4 b4 a5 b5 a6 b6 a7 b7 ac bc ad bd ae be af bf + // c0 d0 c1 d1 c2 d2 c3 d3 c8 d8 c9 d9 ca da cb db + // c4 d4 c5 d5 c6 d6 c7 d7 cc dc cd dd ce de cf df + // e0 f0 e1 f1 e2 f2 e3 f3 e8 f8 e9 f9 ea fa eb fb + // e4 f4 e5 f5 e6 f6 e7 f7 ec fc ed fd ee fe ef ff + + __m256i tr1_0 = _mm256_unpacklo_epi32(tr0_0, tr0_2); + __m256i tr1_1 = _mm256_unpackhi_epi32(tr0_0, tr0_2); + __m256i tr1_2 = _mm256_unpacklo_epi32(tr0_1, tr0_3); + __m256i tr1_3 = _mm256_unpackhi_epi32(tr0_1, tr0_3); + __m256i tr1_4 = _mm256_unpacklo_epi32(tr0_4, tr0_6); + __m256i tr1_5 = _mm256_unpackhi_epi32(tr0_4, tr0_6); + __m256i tr1_6 = _mm256_unpacklo_epi32(tr0_5, tr0_7); + __m256i tr1_7 = _mm256_unpackhi_epi32(tr0_5, tr0_7); + + __m256i tr1_8 = _mm256_unpacklo_epi32(tr0_8, tr0_a); + __m256i tr1_9 = _mm256_unpackhi_epi32(tr0_8, tr0_a); + __m256i tr1_a = _mm256_unpacklo_epi32(tr0_9, tr0_b); + __m256i tr1_b = _mm256_unpackhi_epi32(tr0_9, tr0_b); + __m256i tr1_c = _mm256_unpacklo_epi32(tr0_c, tr0_e); + __m256i tr1_d = _mm256_unpackhi_epi32(tr0_c, tr0_e); + __m256i tr1_e = _mm256_unpacklo_epi32(tr0_d, tr0_f); + __m256i tr1_f = _mm256_unpackhi_epi32(tr0_d, tr0_f); + + // 00 10 20 30 01 11 21 31 08 18 28 38 09 19 29 39 + // 02 12 22 32 03 13 23 33 0a 1a 2a 3a 0b 1b 2b 3b + // 04 14 24 34 05 15 25 35 0c 1c 2c 3c 0d 1d 2d 3d + // 06 16 26 36 07 17 27 37 0e 1e 2e 3e 0f 1f 2f 3f + // 40 50 60 70 41 51 61 71 48 58 68 78 49 59 69 79 + // 42 52 62 72 43 53 63 73 4a 5a 6a 7a 4b 5b 6b 7b + // 44 54 64 74 45 55 65 75 4c 5c 6c 7c 4d 5d 6d 7d + // 46 56 66 76 47 57 67 77 4e 5e 6e 7e 4f 5f 6f 7f + + // 80 90 a0 b0 81 91 a1 b1 88 98 a8 b8 89 99 a9 b9 + // 82 92 a2 b2 83 93 a3 b3 8a 9a aa ba 8b 9b ab bb + // 84 94 a4 b4 85 95 a5 b5 8c 9c ac bc 8d 9d ad bd + // 86 96 a6 b6 87 97 a7 b7 8e ae 9e be 8f 9f af bf + // c0 d0 e0 f0 c1 d1 e1 f1 c8 d8 e8 f8 c9 d9 e9 f9 + // c2 d2 e2 f2 c3 d3 e3 f3 ca da ea fa cb db eb fb + // c4 d4 e4 f4 c5 d5 e5 f5 cc dc ef fc cd dd ed fd + // c6 d6 e6 f6 c7 d7 e7 f7 ce de ee fe cf df ef ff + + tr0_0 = _mm256_unpacklo_epi64(tr1_0, tr1_4); + tr0_1 = _mm256_unpackhi_epi64(tr1_0, tr1_4); + tr0_2 = _mm256_unpacklo_epi64(tr1_1, tr1_5); + tr0_3 = _mm256_unpackhi_epi64(tr1_1, tr1_5); + tr0_4 = _mm256_unpacklo_epi64(tr1_2, tr1_6); + tr0_5 = _mm256_unpackhi_epi64(tr1_2, tr1_6); + tr0_6 = _mm256_unpacklo_epi64(tr1_3, tr1_7); + tr0_7 = _mm256_unpackhi_epi64(tr1_3, tr1_7); + + tr0_8 = _mm256_unpacklo_epi64(tr1_8, tr1_c); + tr0_9 = _mm256_unpackhi_epi64(tr1_8, tr1_c); + tr0_a = _mm256_unpacklo_epi64(tr1_9, tr1_d); + tr0_b = _mm256_unpackhi_epi64(tr1_9, tr1_d); + tr0_c = _mm256_unpacklo_epi64(tr1_a, tr1_e); + tr0_d = _mm256_unpackhi_epi64(tr1_a, tr1_e); + tr0_e = _mm256_unpacklo_epi64(tr1_b, tr1_f); + tr0_f = _mm256_unpackhi_epi64(tr1_b, tr1_f); + + // 00 10 20 30 40 50 60 70 08 18 28 38 48 58 68 78 + // 01 11 21 31 41 51 61 71 09 19 29 39 49 59 69 79 + // 02 12 22 32 42 52 62 72 0a 1a 2a 3a 4a 5a 6a 7a + // 03 13 23 33 43 53 63 73 0b 1b 2b 3b 4b 5b 6b 7b + // 04 14 24 34 44 54 64 74 0c 1c 2c 3c 4c 5c 6c 7c + // 05 15 25 35 45 55 65 75 0d 1d 2d 3d 4d 5d 6d 7d + // 06 16 26 36 46 56 66 76 0e 1e 2e 3e 4e 5e 6e 7e + // 07 17 27 37 47 57 67 77 0f 1f 2f 3f 4f 5f 6f 7f + + // 80 90 a0 b0 c0 d0 e0 f0 88 98 a8 b8 c8 d8 e8 f8 + // 81 91 a1 b1 c1 d1 e1 f1 89 99 a9 b9 c9 d9 e9 f9 + // 82 92 a2 b2 c2 d2 e2 f2 8a 9a aa ba ca da ea fa + // 83 93 a3 b3 c3 d3 e3 f3 8b 9b ab bb cb db eb fb + // 84 94 a4 b4 c4 d4 e4 f4 8c 9c ac bc cc dc ef fc + // 85 95 a5 b5 c5 d5 e5 f5 8d 9d ad bd cd dd ed fd + // 86 96 a6 b6 c6 d6 e6 f6 8e ae 9e be ce de ee fe + // 87 97 a7 b7 c7 d7 e7 f7 8f 9f af bf cf df ef ff + + in[0] = _mm256_permute2x128_si256(tr0_0, tr0_8, 0x20); // 0010 0000 + in[8] = _mm256_permute2x128_si256(tr0_0, tr0_8, 0x31); // 0011 0001 + in[1] = _mm256_permute2x128_si256(tr0_1, tr0_9, 0x20); + in[9] = _mm256_permute2x128_si256(tr0_1, tr0_9, 0x31); + in[2] = _mm256_permute2x128_si256(tr0_2, tr0_a, 0x20); + in[10] = _mm256_permute2x128_si256(tr0_2, tr0_a, 0x31); + in[3] = _mm256_permute2x128_si256(tr0_3, tr0_b, 0x20); + in[11] = _mm256_permute2x128_si256(tr0_3, tr0_b, 0x31); + + in[4] = _mm256_permute2x128_si256(tr0_4, tr0_c, 0x20); + in[12] = _mm256_permute2x128_si256(tr0_4, tr0_c, 0x31); + in[5] = _mm256_permute2x128_si256(tr0_5, tr0_d, 0x20); + in[13] = _mm256_permute2x128_si256(tr0_5, tr0_d, 0x31); + in[6] = _mm256_permute2x128_si256(tr0_6, tr0_e, 0x20); + in[14] = _mm256_permute2x128_si256(tr0_6, tr0_e, 0x31); + in[7] = _mm256_permute2x128_si256(tr0_7, tr0_f, 0x20); + in[15] = _mm256_permute2x128_si256(tr0_7, tr0_f, 0x31); +} + +static INLINE __m256i butter_fly(__m256i a0, __m256i a1, const __m256i cospi) { + const __m256i dct_rounding = _mm256_set1_epi32(DCT_CONST_ROUNDING); + __m256i y0 = _mm256_madd_epi16(a0, cospi); + __m256i y1 = _mm256_madd_epi16(a1, cospi); + + y0 = _mm256_add_epi32(y0, dct_rounding); + y1 = _mm256_add_epi32(y1, dct_rounding); + y0 = _mm256_srai_epi32(y0, DCT_CONST_BITS); + y1 = _mm256_srai_epi32(y1, DCT_CONST_BITS); + + return _mm256_packs_epi32(y0, y1); +} + +static INLINE void txfm_scaling16_avx2(const int16_t c, __m256i *in) { + const __m256i zero = _mm256_setzero_si256(); + const __m256i sqrt2_epi16 = _mm256_set1_epi16(c); + const __m256i dct_const_rounding = _mm256_set1_epi32(DCT_CONST_ROUNDING); + __m256i u0, u1; + int i = 0; + + while (i < 16) { + in[i] = _mm256_slli_epi16(in[i], 1); + + u0 = _mm256_unpacklo_epi16(zero, in[i]); + u1 = _mm256_unpackhi_epi16(zero, in[i]); + + u0 = _mm256_madd_epi16(u0, sqrt2_epi16); + u1 = _mm256_madd_epi16(u1, sqrt2_epi16); + + u0 = _mm256_add_epi32(u0, dct_const_rounding); + u1 = _mm256_add_epi32(u1, dct_const_rounding); + + u0 = _mm256_srai_epi32(u0, DCT_CONST_BITS); + u1 = _mm256_srai_epi32(u1, DCT_CONST_BITS); + in[i] = _mm256_packs_epi32(u0, u1); + i++; + } +} + +#endif // AOM_DSP_X86_TXFM_COMMON_AVX2_H diff --git a/third_party/aom/aom_dsp/x86/txfm_common_intrin.h b/third_party/aom/aom_dsp/x86/txfm_common_intrin.h new file mode 100644 index 0000000000..e4ac56339c --- /dev/null +++ b/third_party/aom/aom_dsp/x86/txfm_common_intrin.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _AOM_DSP_X86_TXFM_COMMON_INTRIN_H_ +#define _AOM_DSP_X86_TXFM_COMMON_INTRIN_H_ + +// Note: +// This header file should be put below any x86 intrinsics head file + +static INLINE void storeu_output(const __m128i *poutput, tran_low_t *dst_ptr) { +#if CONFIG_HIGHBITDEPTH + const __m128i zero = _mm_setzero_si128(); + const __m128i sign_bits = _mm_cmplt_epi16(*poutput, zero); + __m128i out0 = _mm_unpacklo_epi16(*poutput, sign_bits); + __m128i out1 = _mm_unpackhi_epi16(*poutput, sign_bits); + _mm_storeu_si128((__m128i *)(dst_ptr), out0); + _mm_storeu_si128((__m128i *)(dst_ptr + 4), out1); +#else + _mm_storeu_si128((__m128i *)(dst_ptr), *poutput); +#endif // CONFIG_HIGHBITDEPTH +} + +#endif // _AOM_DSP_X86_TXFM_COMMON_INTRIN_H_ diff --git a/third_party/aom/aom_dsp/x86/txfm_common_sse2.h b/third_party/aom/aom_dsp/x86/txfm_common_sse2.h new file mode 100644 index 0000000000..4257d8b9ca --- /dev/null +++ b/third_party/aom/aom_dsp/x86/txfm_common_sse2.h @@ -0,0 +1,326 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_DSP_X86_TXFM_COMMON_SSE2_H_ +#define AOM_DSP_X86_TXFM_COMMON_SSE2_H_ + +#include +#include "aom/aom_integer.h" +#include "aom_dsp/x86/synonyms.h" + +#define pair_set_epi16(a, b) \ + _mm_set_epi16((int16_t)(b), (int16_t)(a), (int16_t)(b), (int16_t)(a), \ + (int16_t)(b), (int16_t)(a), (int16_t)(b), (int16_t)(a)) + +#define dual_set_epi16(a, b) \ + _mm_set_epi16((int16_t)(b), (int16_t)(b), (int16_t)(b), (int16_t)(b), \ + (int16_t)(a), (int16_t)(a), (int16_t)(a), (int16_t)(a)) + +#define octa_set_epi16(a, b, c, d, e, f, g, h) \ + _mm_setr_epi16((int16_t)(a), (int16_t)(b), (int16_t)(c), (int16_t)(d), \ + (int16_t)(e), (int16_t)(f), (int16_t)(g), (int16_t)(h)) + +// Reverse the 8 16 bit words in __m128i +static INLINE __m128i mm_reverse_epi16(const __m128i x) { + const __m128i a = _mm_shufflelo_epi16(x, 0x1b); + const __m128i b = _mm_shufflehi_epi16(a, 0x1b); + return _mm_shuffle_epi32(b, 0x4e); +} + +#if CONFIG_EXT_TX +// Identity transform (both forward and inverse). +static INLINE void idtx16_8col(__m128i *in) { + const __m128i k__zero_epi16 = _mm_set1_epi16((int16_t)0); + const __m128i k__sqrt2_epi16 = _mm_set1_epi16((int16_t)Sqrt2); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + + __m128i v0, v1, v2, v3, v4, v5, v6, v7; + __m128i u0, u1, u2, u3, u4, u5, u6, u7; + __m128i x0, x1, x2, x3, x4, x5, x6, x7; + __m128i y0, y1, y2, y3, y4, y5, y6, y7; + + in[0] = _mm_slli_epi16(in[0], 1); + in[1] = _mm_slli_epi16(in[1], 1); + in[2] = _mm_slli_epi16(in[2], 1); + in[3] = _mm_slli_epi16(in[3], 1); + in[4] = _mm_slli_epi16(in[4], 1); + in[5] = _mm_slli_epi16(in[5], 1); + in[6] = _mm_slli_epi16(in[6], 1); + in[7] = _mm_slli_epi16(in[7], 1); + in[8] = _mm_slli_epi16(in[8], 1); + in[9] = _mm_slli_epi16(in[9], 1); + in[10] = _mm_slli_epi16(in[10], 1); + in[11] = _mm_slli_epi16(in[11], 1); + in[12] = _mm_slli_epi16(in[12], 1); + in[13] = _mm_slli_epi16(in[13], 1); + in[14] = _mm_slli_epi16(in[14], 1); + in[15] = _mm_slli_epi16(in[15], 1); + + v0 = _mm_unpacklo_epi16(in[0], k__zero_epi16); + v1 = _mm_unpacklo_epi16(in[1], k__zero_epi16); + v2 = _mm_unpacklo_epi16(in[2], k__zero_epi16); + v3 = _mm_unpacklo_epi16(in[3], k__zero_epi16); + v4 = _mm_unpacklo_epi16(in[4], k__zero_epi16); + v5 = _mm_unpacklo_epi16(in[5], k__zero_epi16); + v6 = _mm_unpacklo_epi16(in[6], k__zero_epi16); + v7 = _mm_unpacklo_epi16(in[7], k__zero_epi16); + + u0 = _mm_unpacklo_epi16(in[8], k__zero_epi16); + u1 = _mm_unpacklo_epi16(in[9], k__zero_epi16); + u2 = _mm_unpacklo_epi16(in[10], k__zero_epi16); + u3 = _mm_unpacklo_epi16(in[11], k__zero_epi16); + u4 = _mm_unpacklo_epi16(in[12], k__zero_epi16); + u5 = _mm_unpacklo_epi16(in[13], k__zero_epi16); + u6 = _mm_unpacklo_epi16(in[14], k__zero_epi16); + u7 = _mm_unpacklo_epi16(in[15], k__zero_epi16); + + x0 = _mm_unpackhi_epi16(in[0], k__zero_epi16); + x1 = _mm_unpackhi_epi16(in[1], k__zero_epi16); + x2 = _mm_unpackhi_epi16(in[2], k__zero_epi16); + x3 = _mm_unpackhi_epi16(in[3], k__zero_epi16); + x4 = _mm_unpackhi_epi16(in[4], k__zero_epi16); + x5 = _mm_unpackhi_epi16(in[5], k__zero_epi16); + x6 = _mm_unpackhi_epi16(in[6], k__zero_epi16); + x7 = _mm_unpackhi_epi16(in[7], k__zero_epi16); + + y0 = _mm_unpackhi_epi16(in[8], k__zero_epi16); + y1 = _mm_unpackhi_epi16(in[9], k__zero_epi16); + y2 = _mm_unpackhi_epi16(in[10], k__zero_epi16); + y3 = _mm_unpackhi_epi16(in[11], k__zero_epi16); + y4 = _mm_unpackhi_epi16(in[12], k__zero_epi16); + y5 = _mm_unpackhi_epi16(in[13], k__zero_epi16); + y6 = _mm_unpackhi_epi16(in[14], k__zero_epi16); + y7 = _mm_unpackhi_epi16(in[15], k__zero_epi16); + + v0 = _mm_madd_epi16(v0, k__sqrt2_epi16); + v1 = _mm_madd_epi16(v1, k__sqrt2_epi16); + v2 = _mm_madd_epi16(v2, k__sqrt2_epi16); + v3 = _mm_madd_epi16(v3, k__sqrt2_epi16); + v4 = _mm_madd_epi16(v4, k__sqrt2_epi16); + v5 = _mm_madd_epi16(v5, k__sqrt2_epi16); + v6 = _mm_madd_epi16(v6, k__sqrt2_epi16); + v7 = _mm_madd_epi16(v7, k__sqrt2_epi16); + + x0 = _mm_madd_epi16(x0, k__sqrt2_epi16); + x1 = _mm_madd_epi16(x1, k__sqrt2_epi16); + x2 = _mm_madd_epi16(x2, k__sqrt2_epi16); + x3 = _mm_madd_epi16(x3, k__sqrt2_epi16); + x4 = _mm_madd_epi16(x4, k__sqrt2_epi16); + x5 = _mm_madd_epi16(x5, k__sqrt2_epi16); + x6 = _mm_madd_epi16(x6, k__sqrt2_epi16); + x7 = _mm_madd_epi16(x7, k__sqrt2_epi16); + + u0 = _mm_madd_epi16(u0, k__sqrt2_epi16); + u1 = _mm_madd_epi16(u1, k__sqrt2_epi16); + u2 = _mm_madd_epi16(u2, k__sqrt2_epi16); + u3 = _mm_madd_epi16(u3, k__sqrt2_epi16); + u4 = _mm_madd_epi16(u4, k__sqrt2_epi16); + u5 = _mm_madd_epi16(u5, k__sqrt2_epi16); + u6 = _mm_madd_epi16(u6, k__sqrt2_epi16); + u7 = _mm_madd_epi16(u7, k__sqrt2_epi16); + + y0 = _mm_madd_epi16(y0, k__sqrt2_epi16); + y1 = _mm_madd_epi16(y1, k__sqrt2_epi16); + y2 = _mm_madd_epi16(y2, k__sqrt2_epi16); + y3 = _mm_madd_epi16(y3, k__sqrt2_epi16); + y4 = _mm_madd_epi16(y4, k__sqrt2_epi16); + y5 = _mm_madd_epi16(y5, k__sqrt2_epi16); + y6 = _mm_madd_epi16(y6, k__sqrt2_epi16); + y7 = _mm_madd_epi16(y7, k__sqrt2_epi16); + + v0 = _mm_add_epi32(v0, k__DCT_CONST_ROUNDING); + v1 = _mm_add_epi32(v1, k__DCT_CONST_ROUNDING); + v2 = _mm_add_epi32(v2, k__DCT_CONST_ROUNDING); + v3 = _mm_add_epi32(v3, k__DCT_CONST_ROUNDING); + v4 = _mm_add_epi32(v4, k__DCT_CONST_ROUNDING); + v5 = _mm_add_epi32(v5, k__DCT_CONST_ROUNDING); + v6 = _mm_add_epi32(v6, k__DCT_CONST_ROUNDING); + v7 = _mm_add_epi32(v7, k__DCT_CONST_ROUNDING); + + x0 = _mm_add_epi32(x0, k__DCT_CONST_ROUNDING); + x1 = _mm_add_epi32(x1, k__DCT_CONST_ROUNDING); + x2 = _mm_add_epi32(x2, k__DCT_CONST_ROUNDING); + x3 = _mm_add_epi32(x3, k__DCT_CONST_ROUNDING); + x4 = _mm_add_epi32(x4, k__DCT_CONST_ROUNDING); + x5 = _mm_add_epi32(x5, k__DCT_CONST_ROUNDING); + x6 = _mm_add_epi32(x6, k__DCT_CONST_ROUNDING); + x7 = _mm_add_epi32(x7, k__DCT_CONST_ROUNDING); + + u0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); + u1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); + u2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); + u3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); + u4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); + u5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); + u6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); + u7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); + + y0 = _mm_add_epi32(y0, k__DCT_CONST_ROUNDING); + y1 = _mm_add_epi32(y1, k__DCT_CONST_ROUNDING); + y2 = _mm_add_epi32(y2, k__DCT_CONST_ROUNDING); + y3 = _mm_add_epi32(y3, k__DCT_CONST_ROUNDING); + y4 = _mm_add_epi32(y4, k__DCT_CONST_ROUNDING); + y5 = _mm_add_epi32(y5, k__DCT_CONST_ROUNDING); + y6 = _mm_add_epi32(y6, k__DCT_CONST_ROUNDING); + y7 = _mm_add_epi32(y7, k__DCT_CONST_ROUNDING); + + v0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + v1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + v2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + v3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + v4 = _mm_srai_epi32(v4, DCT_CONST_BITS); + v5 = _mm_srai_epi32(v5, DCT_CONST_BITS); + v6 = _mm_srai_epi32(v6, DCT_CONST_BITS); + v7 = _mm_srai_epi32(v7, DCT_CONST_BITS); + + x0 = _mm_srai_epi32(x0, DCT_CONST_BITS); + x1 = _mm_srai_epi32(x1, DCT_CONST_BITS); + x2 = _mm_srai_epi32(x2, DCT_CONST_BITS); + x3 = _mm_srai_epi32(x3, DCT_CONST_BITS); + x4 = _mm_srai_epi32(x4, DCT_CONST_BITS); + x5 = _mm_srai_epi32(x5, DCT_CONST_BITS); + x6 = _mm_srai_epi32(x6, DCT_CONST_BITS); + x7 = _mm_srai_epi32(x7, DCT_CONST_BITS); + + u0 = _mm_srai_epi32(u0, DCT_CONST_BITS); + u1 = _mm_srai_epi32(u1, DCT_CONST_BITS); + u2 = _mm_srai_epi32(u2, DCT_CONST_BITS); + u3 = _mm_srai_epi32(u3, DCT_CONST_BITS); + u4 = _mm_srai_epi32(u4, DCT_CONST_BITS); + u5 = _mm_srai_epi32(u5, DCT_CONST_BITS); + u6 = _mm_srai_epi32(u6, DCT_CONST_BITS); + u7 = _mm_srai_epi32(u7, DCT_CONST_BITS); + + y0 = _mm_srai_epi32(y0, DCT_CONST_BITS); + y1 = _mm_srai_epi32(y1, DCT_CONST_BITS); + y2 = _mm_srai_epi32(y2, DCT_CONST_BITS); + y3 = _mm_srai_epi32(y3, DCT_CONST_BITS); + y4 = _mm_srai_epi32(y4, DCT_CONST_BITS); + y5 = _mm_srai_epi32(y5, DCT_CONST_BITS); + y6 = _mm_srai_epi32(y6, DCT_CONST_BITS); + y7 = _mm_srai_epi32(y7, DCT_CONST_BITS); + + in[0] = _mm_packs_epi32(v0, x0); + in[1] = _mm_packs_epi32(v1, x1); + in[2] = _mm_packs_epi32(v2, x2); + in[3] = _mm_packs_epi32(v3, x3); + in[4] = _mm_packs_epi32(v4, x4); + in[5] = _mm_packs_epi32(v5, x5); + in[6] = _mm_packs_epi32(v6, x6); + in[7] = _mm_packs_epi32(v7, x7); + + in[8] = _mm_packs_epi32(u0, y0); + in[9] = _mm_packs_epi32(u1, y1); + in[10] = _mm_packs_epi32(u2, y2); + in[11] = _mm_packs_epi32(u3, y3); + in[12] = _mm_packs_epi32(u4, y4); + in[13] = _mm_packs_epi32(u5, y5); + in[14] = _mm_packs_epi32(u6, y6); + in[15] = _mm_packs_epi32(u7, y7); +} +#endif // CONFIG_EXT_TX + +static INLINE void scale_sqrt2_8x4(__m128i *in) { + // Implements ROUND_POWER_OF_TWO(input * Sqrt2, DCT_CONST_BITS), for 32 + // consecutive elements. + const __m128i v_scale_w = _mm_set1_epi16((int16_t)Sqrt2); + + const __m128i v_p0l_w = _mm_mullo_epi16(in[0], v_scale_w); + const __m128i v_p0h_w = _mm_mulhi_epi16(in[0], v_scale_w); + const __m128i v_p1l_w = _mm_mullo_epi16(in[1], v_scale_w); + const __m128i v_p1h_w = _mm_mulhi_epi16(in[1], v_scale_w); + const __m128i v_p2l_w = _mm_mullo_epi16(in[2], v_scale_w); + const __m128i v_p2h_w = _mm_mulhi_epi16(in[2], v_scale_w); + const __m128i v_p3l_w = _mm_mullo_epi16(in[3], v_scale_w); + const __m128i v_p3h_w = _mm_mulhi_epi16(in[3], v_scale_w); + + const __m128i v_p0a_d = _mm_unpacklo_epi16(v_p0l_w, v_p0h_w); + const __m128i v_p0b_d = _mm_unpackhi_epi16(v_p0l_w, v_p0h_w); + const __m128i v_p1a_d = _mm_unpacklo_epi16(v_p1l_w, v_p1h_w); + const __m128i v_p1b_d = _mm_unpackhi_epi16(v_p1l_w, v_p1h_w); + const __m128i v_p2a_d = _mm_unpacklo_epi16(v_p2l_w, v_p2h_w); + const __m128i v_p2b_d = _mm_unpackhi_epi16(v_p2l_w, v_p2h_w); + const __m128i v_p3a_d = _mm_unpacklo_epi16(v_p3l_w, v_p3h_w); + const __m128i v_p3b_d = _mm_unpackhi_epi16(v_p3l_w, v_p3h_w); + + in[0] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p0a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p0b_d, DCT_CONST_BITS)); + in[1] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p1a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p1b_d, DCT_CONST_BITS)); + in[2] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p2a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p2b_d, DCT_CONST_BITS)); + in[3] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p3a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p3b_d, DCT_CONST_BITS)); +} + +static INLINE void scale_sqrt2_8x8(__m128i *in) { + // Implements 'ROUND_POWER_OF_TWO_SIGNED(input * Sqrt2, DCT_CONST_BITS)' + // for each element. + const __m128i v_scale_w = _mm_set1_epi16((int16_t)Sqrt2); + + const __m128i v_p0l_w = _mm_mullo_epi16(in[0], v_scale_w); + const __m128i v_p0h_w = _mm_mulhi_epi16(in[0], v_scale_w); + const __m128i v_p1l_w = _mm_mullo_epi16(in[1], v_scale_w); + const __m128i v_p1h_w = _mm_mulhi_epi16(in[1], v_scale_w); + const __m128i v_p2l_w = _mm_mullo_epi16(in[2], v_scale_w); + const __m128i v_p2h_w = _mm_mulhi_epi16(in[2], v_scale_w); + const __m128i v_p3l_w = _mm_mullo_epi16(in[3], v_scale_w); + const __m128i v_p3h_w = _mm_mulhi_epi16(in[3], v_scale_w); + const __m128i v_p4l_w = _mm_mullo_epi16(in[4], v_scale_w); + const __m128i v_p4h_w = _mm_mulhi_epi16(in[4], v_scale_w); + const __m128i v_p5l_w = _mm_mullo_epi16(in[5], v_scale_w); + const __m128i v_p5h_w = _mm_mulhi_epi16(in[5], v_scale_w); + const __m128i v_p6l_w = _mm_mullo_epi16(in[6], v_scale_w); + const __m128i v_p6h_w = _mm_mulhi_epi16(in[6], v_scale_w); + const __m128i v_p7l_w = _mm_mullo_epi16(in[7], v_scale_w); + const __m128i v_p7h_w = _mm_mulhi_epi16(in[7], v_scale_w); + + const __m128i v_p0a_d = _mm_unpacklo_epi16(v_p0l_w, v_p0h_w); + const __m128i v_p0b_d = _mm_unpackhi_epi16(v_p0l_w, v_p0h_w); + const __m128i v_p1a_d = _mm_unpacklo_epi16(v_p1l_w, v_p1h_w); + const __m128i v_p1b_d = _mm_unpackhi_epi16(v_p1l_w, v_p1h_w); + const __m128i v_p2a_d = _mm_unpacklo_epi16(v_p2l_w, v_p2h_w); + const __m128i v_p2b_d = _mm_unpackhi_epi16(v_p2l_w, v_p2h_w); + const __m128i v_p3a_d = _mm_unpacklo_epi16(v_p3l_w, v_p3h_w); + const __m128i v_p3b_d = _mm_unpackhi_epi16(v_p3l_w, v_p3h_w); + const __m128i v_p4a_d = _mm_unpacklo_epi16(v_p4l_w, v_p4h_w); + const __m128i v_p4b_d = _mm_unpackhi_epi16(v_p4l_w, v_p4h_w); + const __m128i v_p5a_d = _mm_unpacklo_epi16(v_p5l_w, v_p5h_w); + const __m128i v_p5b_d = _mm_unpackhi_epi16(v_p5l_w, v_p5h_w); + const __m128i v_p6a_d = _mm_unpacklo_epi16(v_p6l_w, v_p6h_w); + const __m128i v_p6b_d = _mm_unpackhi_epi16(v_p6l_w, v_p6h_w); + const __m128i v_p7a_d = _mm_unpacklo_epi16(v_p7l_w, v_p7h_w); + const __m128i v_p7b_d = _mm_unpackhi_epi16(v_p7l_w, v_p7h_w); + + in[0] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p0a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p0b_d, DCT_CONST_BITS)); + in[1] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p1a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p1b_d, DCT_CONST_BITS)); + in[2] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p2a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p2b_d, DCT_CONST_BITS)); + in[3] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p3a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p3b_d, DCT_CONST_BITS)); + in[4] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p4a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p4b_d, DCT_CONST_BITS)); + in[5] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p5a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p5b_d, DCT_CONST_BITS)); + in[6] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p6a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p6b_d, DCT_CONST_BITS)); + in[7] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p7a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p7b_d, DCT_CONST_BITS)); +} + +static INLINE void scale_sqrt2_8x16(__m128i *in) { + scale_sqrt2_8x8(in); + scale_sqrt2_8x8(in + 8); +} + +#endif // AOM_DSP_X86_TXFM_COMMON_SSE2_H_ diff --git a/third_party/aom/aom_dsp/x86/variance_avx2.c b/third_party/aom/aom_dsp/x86/variance_avx2.c new file mode 100644 index 0000000000..18a70dffe7 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/variance_avx2.c @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./aom_dsp_rtcd.h" + +typedef void (*get_var_avx2)(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse, int *sum); + +void aom_get32x32var_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, unsigned int *sse, + int *sum); + +static void variance_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, int w, int h, + unsigned int *sse, int *sum, get_var_avx2 var_fn, + int block_size) { + int i, j; + + *sse = 0; + *sum = 0; + + for (i = 0; i < h; i += 16) { + for (j = 0; j < w; j += block_size) { + unsigned int sse0; + int sum0; + var_fn(&src[src_stride * i + j], src_stride, &ref[ref_stride * i + j], + ref_stride, &sse0, &sum0); + *sse += sse0; + *sum += sum0; + } + } +} + +unsigned int aom_variance16x16_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + unsigned int variance; + variance_avx2(src, src_stride, ref, ref_stride, 16, 16, sse, &sum, + aom_get16x16var_avx2, 16); + + variance = *sse - (((uint32_t)((int64_t)sum * sum)) >> 8); + _mm256_zeroupper(); + return variance; +} + +unsigned int aom_mse16x16_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + aom_get16x16var_avx2(src, src_stride, ref, ref_stride, sse, &sum); + _mm256_zeroupper(); + return *sse; +} + +unsigned int aom_variance32x16_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + unsigned int variance; + variance_avx2(src, src_stride, ref, ref_stride, 32, 16, sse, &sum, + aom_get32x32var_avx2, 32); + + variance = *sse - (uint32_t)(((int64_t)sum * sum) >> 9); + _mm256_zeroupper(); + return variance; +} + +unsigned int aom_variance32x32_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + unsigned int variance; + variance_avx2(src, src_stride, ref, ref_stride, 32, 32, sse, &sum, + aom_get32x32var_avx2, 32); + + variance = *sse - (uint32_t)(((int64_t)sum * sum) >> 10); + _mm256_zeroupper(); + return variance; +} + +unsigned int aom_variance64x64_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + unsigned int variance; + variance_avx2(src, src_stride, ref, ref_stride, 64, 64, sse, &sum, + aom_get32x32var_avx2, 32); + + variance = *sse - (uint32_t)(((int64_t)sum * sum) >> 12); + _mm256_zeroupper(); + return variance; +} + +unsigned int aom_variance64x32_avx2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + unsigned int variance; + variance_avx2(src, src_stride, ref, ref_stride, 64, 32, sse, &sum, + aom_get32x32var_avx2, 32); + + variance = *sse - (uint32_t)(((int64_t)sum * sum) >> 11); + _mm256_zeroupper(); + return variance; +} + +unsigned int aom_sub_pixel_variance32xh_avx2(const uint8_t *src, int src_stride, + int x_offset, int y_offset, + const uint8_t *dst, int dst_stride, + int height, unsigned int *sse); + +unsigned int aom_sub_pixel_avg_variance32xh_avx2( + const uint8_t *src, int src_stride, int x_offset, int y_offset, + const uint8_t *dst, int dst_stride, const uint8_t *sec, int sec_stride, + int height, unsigned int *sseptr); + +unsigned int aom_sub_pixel_variance64x64_avx2(const uint8_t *src, + int src_stride, int x_offset, + int y_offset, const uint8_t *dst, + int dst_stride, + unsigned int *sse) { + unsigned int sse1; + const int se1 = aom_sub_pixel_variance32xh_avx2( + src, src_stride, x_offset, y_offset, dst, dst_stride, 64, &sse1); + unsigned int sse2; + const int se2 = + aom_sub_pixel_variance32xh_avx2(src + 32, src_stride, x_offset, y_offset, + dst + 32, dst_stride, 64, &sse2); + const int se = se1 + se2; + unsigned int variance; + *sse = sse1 + sse2; + + variance = *sse - (uint32_t)(((int64_t)se * se) >> 12); + _mm256_zeroupper(); + return variance; +} + +unsigned int aom_sub_pixel_variance32x32_avx2(const uint8_t *src, + int src_stride, int x_offset, + int y_offset, const uint8_t *dst, + int dst_stride, + unsigned int *sse) { + const int se = aom_sub_pixel_variance32xh_avx2( + src, src_stride, x_offset, y_offset, dst, dst_stride, 32, sse); + + const unsigned int variance = *sse - (uint32_t)(((int64_t)se * se) >> 10); + _mm256_zeroupper(); + return variance; +} + +unsigned int aom_sub_pixel_avg_variance64x64_avx2( + const uint8_t *src, int src_stride, int x_offset, int y_offset, + const uint8_t *dst, int dst_stride, unsigned int *sse, const uint8_t *sec) { + unsigned int sse1; + const int se1 = aom_sub_pixel_avg_variance32xh_avx2( + src, src_stride, x_offset, y_offset, dst, dst_stride, sec, 64, 64, &sse1); + unsigned int sse2; + const int se2 = aom_sub_pixel_avg_variance32xh_avx2( + src + 32, src_stride, x_offset, y_offset, dst + 32, dst_stride, sec + 32, + 64, 64, &sse2); + const int se = se1 + se2; + unsigned int variance; + + *sse = sse1 + sse2; + + variance = *sse - (uint32_t)(((int64_t)se * se) >> 12); + _mm256_zeroupper(); + return variance; +} + +unsigned int aom_sub_pixel_avg_variance32x32_avx2( + const uint8_t *src, int src_stride, int x_offset, int y_offset, + const uint8_t *dst, int dst_stride, unsigned int *sse, const uint8_t *sec) { + // Process 32 elements in parallel. + const int se = aom_sub_pixel_avg_variance32xh_avx2( + src, src_stride, x_offset, y_offset, dst, dst_stride, sec, 32, 32, sse); + + const unsigned int variance = *sse - (uint32_t)(((int64_t)se * se) >> 10); + _mm256_zeroupper(); + return variance; +} diff --git a/third_party/aom/aom_dsp/x86/variance_impl_avx2.c b/third_party/aom/aom_dsp/x86/variance_impl_avx2.c new file mode 100644 index 0000000000..999b541e35 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/variance_impl_avx2.c @@ -0,0 +1,713 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // AVX2 + +#include "./aom_dsp_rtcd.h" +#include "aom_ports/mem.h" + +/* clang-format off */ +DECLARE_ALIGNED(32, static const uint8_t, bilinear_filters_avx2[512]) = { + 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, + 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, + 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, + 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, + 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, + 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, + 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, + 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, + 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, + 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, + 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, + 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, + 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, +}; +/* clang-format on */ + +void aom_get16x16var_avx2(const unsigned char *src_ptr, int source_stride, + const unsigned char *ref_ptr, int recon_stride, + unsigned int *SSE, int *Sum) { + __m256i src, src_expand_low, src_expand_high, ref, ref_expand_low; + __m256i ref_expand_high, madd_low, madd_high; + unsigned int i, src_2strides, ref_2strides; + __m256i zero_reg = _mm256_set1_epi16(0); + __m256i sum_ref_src = _mm256_set1_epi16(0); + __m256i madd_ref_src = _mm256_set1_epi16(0); + + // processing two strides in a 256 bit register reducing the number + // of loop stride by half (comparing to the sse2 code) + src_2strides = source_stride << 1; + ref_2strides = recon_stride << 1; + for (i = 0; i < 8; i++) { + src = _mm256_castsi128_si256(_mm_loadu_si128((__m128i const *)(src_ptr))); + src = _mm256_inserti128_si256( + src, _mm_loadu_si128((__m128i const *)(src_ptr + source_stride)), 1); + + ref = _mm256_castsi128_si256(_mm_loadu_si128((__m128i const *)(ref_ptr))); + ref = _mm256_inserti128_si256( + ref, _mm_loadu_si128((__m128i const *)(ref_ptr + recon_stride)), 1); + + // expanding to 16 bit each lane + src_expand_low = _mm256_unpacklo_epi8(src, zero_reg); + src_expand_high = _mm256_unpackhi_epi8(src, zero_reg); + + ref_expand_low = _mm256_unpacklo_epi8(ref, zero_reg); + ref_expand_high = _mm256_unpackhi_epi8(ref, zero_reg); + + // src-ref + src_expand_low = _mm256_sub_epi16(src_expand_low, ref_expand_low); + src_expand_high = _mm256_sub_epi16(src_expand_high, ref_expand_high); + + // madd low (src - ref) + madd_low = _mm256_madd_epi16(src_expand_low, src_expand_low); + + // add high to low + src_expand_low = _mm256_add_epi16(src_expand_low, src_expand_high); + + // madd high (src - ref) + madd_high = _mm256_madd_epi16(src_expand_high, src_expand_high); + + sum_ref_src = _mm256_add_epi16(sum_ref_src, src_expand_low); + + // add high to low + madd_ref_src = + _mm256_add_epi32(madd_ref_src, _mm256_add_epi32(madd_low, madd_high)); + + src_ptr += src_2strides; + ref_ptr += ref_2strides; + } + + { + __m128i sum_res, madd_res; + __m128i expand_sum_low, expand_sum_high, expand_sum; + __m128i expand_madd_low, expand_madd_high, expand_madd; + __m128i ex_expand_sum_low, ex_expand_sum_high, ex_expand_sum; + + // extract the low lane and add it to the high lane + sum_res = _mm_add_epi16(_mm256_castsi256_si128(sum_ref_src), + _mm256_extractf128_si256(sum_ref_src, 1)); + + madd_res = _mm_add_epi32(_mm256_castsi256_si128(madd_ref_src), + _mm256_extractf128_si256(madd_ref_src, 1)); + + // padding each 2 bytes with another 2 zeroed bytes + expand_sum_low = + _mm_unpacklo_epi16(_mm256_castsi256_si128(zero_reg), sum_res); + expand_sum_high = + _mm_unpackhi_epi16(_mm256_castsi256_si128(zero_reg), sum_res); + + // shifting the sign 16 bits right + expand_sum_low = _mm_srai_epi32(expand_sum_low, 16); + expand_sum_high = _mm_srai_epi32(expand_sum_high, 16); + + expand_sum = _mm_add_epi32(expand_sum_low, expand_sum_high); + + // expand each 32 bits of the madd result to 64 bits + expand_madd_low = + _mm_unpacklo_epi32(madd_res, _mm256_castsi256_si128(zero_reg)); + expand_madd_high = + _mm_unpackhi_epi32(madd_res, _mm256_castsi256_si128(zero_reg)); + + expand_madd = _mm_add_epi32(expand_madd_low, expand_madd_high); + + ex_expand_sum_low = + _mm_unpacklo_epi32(expand_sum, _mm256_castsi256_si128(zero_reg)); + ex_expand_sum_high = + _mm_unpackhi_epi32(expand_sum, _mm256_castsi256_si128(zero_reg)); + + ex_expand_sum = _mm_add_epi32(ex_expand_sum_low, ex_expand_sum_high); + + // shift 8 bytes eight + madd_res = _mm_srli_si128(expand_madd, 8); + sum_res = _mm_srli_si128(ex_expand_sum, 8); + + madd_res = _mm_add_epi32(madd_res, expand_madd); + sum_res = _mm_add_epi32(sum_res, ex_expand_sum); + + *((int *)SSE) = _mm_cvtsi128_si32(madd_res); + + *((int *)Sum) = _mm_cvtsi128_si32(sum_res); + } + _mm256_zeroupper(); +} + +void aom_get32x32var_avx2(const unsigned char *src_ptr, int source_stride, + const unsigned char *ref_ptr, int recon_stride, + unsigned int *SSE, int *Sum) { + __m256i src, src_expand_low, src_expand_high, ref, ref_expand_low; + __m256i ref_expand_high, madd_low, madd_high; + unsigned int i; + __m256i zero_reg = _mm256_set1_epi16(0); + __m256i sum_ref_src = _mm256_set1_epi16(0); + __m256i madd_ref_src = _mm256_set1_epi16(0); + + // processing 32 elements in parallel + for (i = 0; i < 16; i++) { + src = _mm256_loadu_si256((__m256i const *)(src_ptr)); + + ref = _mm256_loadu_si256((__m256i const *)(ref_ptr)); + + // expanding to 16 bit each lane + src_expand_low = _mm256_unpacklo_epi8(src, zero_reg); + src_expand_high = _mm256_unpackhi_epi8(src, zero_reg); + + ref_expand_low = _mm256_unpacklo_epi8(ref, zero_reg); + ref_expand_high = _mm256_unpackhi_epi8(ref, zero_reg); + + // src-ref + src_expand_low = _mm256_sub_epi16(src_expand_low, ref_expand_low); + src_expand_high = _mm256_sub_epi16(src_expand_high, ref_expand_high); + + // madd low (src - ref) + madd_low = _mm256_madd_epi16(src_expand_low, src_expand_low); + + // add high to low + src_expand_low = _mm256_add_epi16(src_expand_low, src_expand_high); + + // madd high (src - ref) + madd_high = _mm256_madd_epi16(src_expand_high, src_expand_high); + + sum_ref_src = _mm256_add_epi16(sum_ref_src, src_expand_low); + + // add high to low + madd_ref_src = + _mm256_add_epi32(madd_ref_src, _mm256_add_epi32(madd_low, madd_high)); + + src_ptr += source_stride; + ref_ptr += recon_stride; + } + + { + __m256i expand_sum_low, expand_sum_high, expand_sum; + __m256i expand_madd_low, expand_madd_high, expand_madd; + __m256i ex_expand_sum_low, ex_expand_sum_high, ex_expand_sum; + + // padding each 2 bytes with another 2 zeroed bytes + expand_sum_low = _mm256_unpacklo_epi16(zero_reg, sum_ref_src); + expand_sum_high = _mm256_unpackhi_epi16(zero_reg, sum_ref_src); + + // shifting the sign 16 bits right + expand_sum_low = _mm256_srai_epi32(expand_sum_low, 16); + expand_sum_high = _mm256_srai_epi32(expand_sum_high, 16); + + expand_sum = _mm256_add_epi32(expand_sum_low, expand_sum_high); + + // expand each 32 bits of the madd result to 64 bits + expand_madd_low = _mm256_unpacklo_epi32(madd_ref_src, zero_reg); + expand_madd_high = _mm256_unpackhi_epi32(madd_ref_src, zero_reg); + + expand_madd = _mm256_add_epi32(expand_madd_low, expand_madd_high); + + ex_expand_sum_low = _mm256_unpacklo_epi32(expand_sum, zero_reg); + ex_expand_sum_high = _mm256_unpackhi_epi32(expand_sum, zero_reg); + + ex_expand_sum = _mm256_add_epi32(ex_expand_sum_low, ex_expand_sum_high); + + // shift 8 bytes eight + madd_ref_src = _mm256_srli_si256(expand_madd, 8); + sum_ref_src = _mm256_srli_si256(ex_expand_sum, 8); + + madd_ref_src = _mm256_add_epi32(madd_ref_src, expand_madd); + sum_ref_src = _mm256_add_epi32(sum_ref_src, ex_expand_sum); + + // extract the low lane and the high lane and add the results + *((int *)SSE) = + _mm_cvtsi128_si32(_mm256_castsi256_si128(madd_ref_src)) + + _mm_cvtsi128_si32(_mm256_extractf128_si256(madd_ref_src, 1)); + + *((int *)Sum) = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_ref_src)) + + _mm_cvtsi128_si32(_mm256_extractf128_si256(sum_ref_src, 1)); + } + _mm256_zeroupper(); +} + +#define FILTER_SRC(filter) \ + /* filter the source */ \ + exp_src_lo = _mm256_maddubs_epi16(exp_src_lo, filter); \ + exp_src_hi = _mm256_maddubs_epi16(exp_src_hi, filter); \ + \ + /* add 8 to source */ \ + exp_src_lo = _mm256_add_epi16(exp_src_lo, pw8); \ + exp_src_hi = _mm256_add_epi16(exp_src_hi, pw8); \ + \ + /* divide source by 16 */ \ + exp_src_lo = _mm256_srai_epi16(exp_src_lo, 4); \ + exp_src_hi = _mm256_srai_epi16(exp_src_hi, 4); + +#define MERGE_WITH_SRC(src_reg, reg) \ + exp_src_lo = _mm256_unpacklo_epi8(src_reg, reg); \ + exp_src_hi = _mm256_unpackhi_epi8(src_reg, reg); + +#define LOAD_SRC_DST \ + /* load source and destination */ \ + src_reg = _mm256_loadu_si256((__m256i const *)(src)); \ + dst_reg = _mm256_loadu_si256((__m256i const *)(dst)); + +#define AVG_NEXT_SRC(src_reg, size_stride) \ + src_next_reg = _mm256_loadu_si256((__m256i const *)(src + size_stride)); \ + /* average between current and next stride source */ \ + src_reg = _mm256_avg_epu8(src_reg, src_next_reg); + +#define MERGE_NEXT_SRC(src_reg, size_stride) \ + src_next_reg = _mm256_loadu_si256((__m256i const *)(src + size_stride)); \ + MERGE_WITH_SRC(src_reg, src_next_reg) + +#define CALC_SUM_SSE_INSIDE_LOOP \ + /* expand each byte to 2 bytes */ \ + exp_dst_lo = _mm256_unpacklo_epi8(dst_reg, zero_reg); \ + exp_dst_hi = _mm256_unpackhi_epi8(dst_reg, zero_reg); \ + /* source - dest */ \ + exp_src_lo = _mm256_sub_epi16(exp_src_lo, exp_dst_lo); \ + exp_src_hi = _mm256_sub_epi16(exp_src_hi, exp_dst_hi); \ + /* caculate sum */ \ + sum_reg = _mm256_add_epi16(sum_reg, exp_src_lo); \ + exp_src_lo = _mm256_madd_epi16(exp_src_lo, exp_src_lo); \ + sum_reg = _mm256_add_epi16(sum_reg, exp_src_hi); \ + exp_src_hi = _mm256_madd_epi16(exp_src_hi, exp_src_hi); \ + /* calculate sse */ \ + sse_reg = _mm256_add_epi32(sse_reg, exp_src_lo); \ + sse_reg = _mm256_add_epi32(sse_reg, exp_src_hi); + +// final calculation to sum and sse +#define CALC_SUM_AND_SSE \ + res_cmp = _mm256_cmpgt_epi16(zero_reg, sum_reg); \ + sse_reg_hi = _mm256_srli_si256(sse_reg, 8); \ + sum_reg_lo = _mm256_unpacklo_epi16(sum_reg, res_cmp); \ + sum_reg_hi = _mm256_unpackhi_epi16(sum_reg, res_cmp); \ + sse_reg = _mm256_add_epi32(sse_reg, sse_reg_hi); \ + sum_reg = _mm256_add_epi32(sum_reg_lo, sum_reg_hi); \ + \ + sse_reg_hi = _mm256_srli_si256(sse_reg, 4); \ + sum_reg_hi = _mm256_srli_si256(sum_reg, 8); \ + \ + sse_reg = _mm256_add_epi32(sse_reg, sse_reg_hi); \ + sum_reg = _mm256_add_epi32(sum_reg, sum_reg_hi); \ + *((int *)sse) = _mm_cvtsi128_si32(_mm256_castsi256_si128(sse_reg)) + \ + _mm_cvtsi128_si32(_mm256_extractf128_si256(sse_reg, 1)); \ + sum_reg_hi = _mm256_srli_si256(sum_reg, 4); \ + sum_reg = _mm256_add_epi32(sum_reg, sum_reg_hi); \ + sum = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_reg)) + \ + _mm_cvtsi128_si32(_mm256_extractf128_si256(sum_reg, 1)); + +unsigned int aom_sub_pixel_variance32xh_avx2(const uint8_t *src, int src_stride, + int x_offset, int y_offset, + const uint8_t *dst, int dst_stride, + int height, unsigned int *sse) { + __m256i src_reg, dst_reg, exp_src_lo, exp_src_hi, exp_dst_lo, exp_dst_hi; + __m256i sse_reg, sum_reg, sse_reg_hi, res_cmp, sum_reg_lo, sum_reg_hi; + __m256i zero_reg; + int i, sum; + sum_reg = _mm256_set1_epi16(0); + sse_reg = _mm256_set1_epi16(0); + zero_reg = _mm256_set1_epi16(0); + + // x_offset = 0 and y_offset = 0 + if (x_offset == 0) { + if (y_offset == 0) { + for (i = 0; i < height; i++) { + LOAD_SRC_DST + // expend each byte to 2 bytes + MERGE_WITH_SRC(src_reg, zero_reg) + CALC_SUM_SSE_INSIDE_LOOP + src += src_stride; + dst += dst_stride; + } + // x_offset = 0 and y_offset = 8 + } else if (y_offset == 8) { + __m256i src_next_reg; + for (i = 0; i < height; i++) { + LOAD_SRC_DST + AVG_NEXT_SRC(src_reg, src_stride) + // expend each byte to 2 bytes + MERGE_WITH_SRC(src_reg, zero_reg) + CALC_SUM_SSE_INSIDE_LOOP + src += src_stride; + dst += dst_stride; + } + // x_offset = 0 and y_offset = bilin interpolation + } else { + __m256i filter, pw8, src_next_reg; + + y_offset <<= 5; + filter = _mm256_load_si256( + (__m256i const *)(bilinear_filters_avx2 + y_offset)); + pw8 = _mm256_set1_epi16(8); + for (i = 0; i < height; i++) { + LOAD_SRC_DST + MERGE_NEXT_SRC(src_reg, src_stride) + FILTER_SRC(filter) + CALC_SUM_SSE_INSIDE_LOOP + src += src_stride; + dst += dst_stride; + } + } + // x_offset = 8 and y_offset = 0 + } else if (x_offset == 8) { + if (y_offset == 0) { + __m256i src_next_reg; + for (i = 0; i < height; i++) { + LOAD_SRC_DST + AVG_NEXT_SRC(src_reg, 1) + // expand each byte to 2 bytes + MERGE_WITH_SRC(src_reg, zero_reg) + CALC_SUM_SSE_INSIDE_LOOP + src += src_stride; + dst += dst_stride; + } + // x_offset = 8 and y_offset = 8 + } else if (y_offset == 8) { + __m256i src_next_reg, src_avg; + // load source and another source starting from the next + // following byte + src_reg = _mm256_loadu_si256((__m256i const *)(src)); + AVG_NEXT_SRC(src_reg, 1) + for (i = 0; i < height; i++) { + src_avg = src_reg; + src += src_stride; + LOAD_SRC_DST + AVG_NEXT_SRC(src_reg, 1) + // average between previous average to current average + src_avg = _mm256_avg_epu8(src_avg, src_reg); + // expand each byte to 2 bytes + MERGE_WITH_SRC(src_avg, zero_reg) + // save current source average + CALC_SUM_SSE_INSIDE_LOOP + dst += dst_stride; + } + // x_offset = 8 and y_offset = bilin interpolation + } else { + __m256i filter, pw8, src_next_reg, src_avg; + y_offset <<= 5; + filter = _mm256_load_si256( + (__m256i const *)(bilinear_filters_avx2 + y_offset)); + pw8 = _mm256_set1_epi16(8); + // load source and another source starting from the next + // following byte + src_reg = _mm256_loadu_si256((__m256i const *)(src)); + AVG_NEXT_SRC(src_reg, 1) + for (i = 0; i < height; i++) { + // save current source average + src_avg = src_reg; + src += src_stride; + LOAD_SRC_DST + AVG_NEXT_SRC(src_reg, 1) + MERGE_WITH_SRC(src_avg, src_reg) + FILTER_SRC(filter) + CALC_SUM_SSE_INSIDE_LOOP + dst += dst_stride; + } + } + // x_offset = bilin interpolation and y_offset = 0 + } else { + if (y_offset == 0) { + __m256i filter, pw8, src_next_reg; + x_offset <<= 5; + filter = _mm256_load_si256( + (__m256i const *)(bilinear_filters_avx2 + x_offset)); + pw8 = _mm256_set1_epi16(8); + for (i = 0; i < height; i++) { + LOAD_SRC_DST + MERGE_NEXT_SRC(src_reg, 1) + FILTER_SRC(filter) + CALC_SUM_SSE_INSIDE_LOOP + src += src_stride; + dst += dst_stride; + } + // x_offset = bilin interpolation and y_offset = 8 + } else if (y_offset == 8) { + __m256i filter, pw8, src_next_reg, src_pack; + x_offset <<= 5; + filter = _mm256_load_si256( + (__m256i const *)(bilinear_filters_avx2 + x_offset)); + pw8 = _mm256_set1_epi16(8); + src_reg = _mm256_loadu_si256((__m256i const *)(src)); + MERGE_NEXT_SRC(src_reg, 1) + FILTER_SRC(filter) + // convert each 16 bit to 8 bit to each low and high lane source + src_pack = _mm256_packus_epi16(exp_src_lo, exp_src_hi); + for (i = 0; i < height; i++) { + src += src_stride; + LOAD_SRC_DST + MERGE_NEXT_SRC(src_reg, 1) + FILTER_SRC(filter) + src_reg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); + // average between previous pack to the current + src_pack = _mm256_avg_epu8(src_pack, src_reg); + MERGE_WITH_SRC(src_pack, zero_reg) + CALC_SUM_SSE_INSIDE_LOOP + src_pack = src_reg; + dst += dst_stride; + } + // x_offset = bilin interpolation and y_offset = bilin interpolation + } else { + __m256i xfilter, yfilter, pw8, src_next_reg, src_pack; + x_offset <<= 5; + xfilter = _mm256_load_si256( + (__m256i const *)(bilinear_filters_avx2 + x_offset)); + y_offset <<= 5; + yfilter = _mm256_load_si256( + (__m256i const *)(bilinear_filters_avx2 + y_offset)); + pw8 = _mm256_set1_epi16(8); + // load source and another source starting from the next + // following byte + src_reg = _mm256_loadu_si256((__m256i const *)(src)); + MERGE_NEXT_SRC(src_reg, 1) + + FILTER_SRC(xfilter) + // convert each 16 bit to 8 bit to each low and high lane source + src_pack = _mm256_packus_epi16(exp_src_lo, exp_src_hi); + for (i = 0; i < height; i++) { + src += src_stride; + LOAD_SRC_DST + MERGE_NEXT_SRC(src_reg, 1) + FILTER_SRC(xfilter) + src_reg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); + // merge previous pack to current pack source + MERGE_WITH_SRC(src_pack, src_reg) + // filter the source + FILTER_SRC(yfilter) + src_pack = src_reg; + CALC_SUM_SSE_INSIDE_LOOP + dst += dst_stride; + } + } + } + CALC_SUM_AND_SSE + _mm256_zeroupper(); + return sum; +} + +unsigned int aom_sub_pixel_avg_variance32xh_avx2( + const uint8_t *src, int src_stride, int x_offset, int y_offset, + const uint8_t *dst, int dst_stride, const uint8_t *sec, int sec_stride, + int height, unsigned int *sse) { + __m256i sec_reg; + __m256i src_reg, dst_reg, exp_src_lo, exp_src_hi, exp_dst_lo, exp_dst_hi; + __m256i sse_reg, sum_reg, sse_reg_hi, res_cmp, sum_reg_lo, sum_reg_hi; + __m256i zero_reg; + int i, sum; + sum_reg = _mm256_set1_epi16(0); + sse_reg = _mm256_set1_epi16(0); + zero_reg = _mm256_set1_epi16(0); + + // x_offset = 0 and y_offset = 0 + if (x_offset == 0) { + if (y_offset == 0) { + for (i = 0; i < height; i++) { + LOAD_SRC_DST + sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); + src_reg = _mm256_avg_epu8(src_reg, sec_reg); + sec += sec_stride; + // expend each byte to 2 bytes + MERGE_WITH_SRC(src_reg, zero_reg) + CALC_SUM_SSE_INSIDE_LOOP + src += src_stride; + dst += dst_stride; + } + } else if (y_offset == 8) { + __m256i src_next_reg; + for (i = 0; i < height; i++) { + LOAD_SRC_DST + AVG_NEXT_SRC(src_reg, src_stride) + sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); + src_reg = _mm256_avg_epu8(src_reg, sec_reg); + sec += sec_stride; + // expend each byte to 2 bytes + MERGE_WITH_SRC(src_reg, zero_reg) + CALC_SUM_SSE_INSIDE_LOOP + src += src_stride; + dst += dst_stride; + } + // x_offset = 0 and y_offset = bilin interpolation + } else { + __m256i filter, pw8, src_next_reg; + + y_offset <<= 5; + filter = _mm256_load_si256( + (__m256i const *)(bilinear_filters_avx2 + y_offset)); + pw8 = _mm256_set1_epi16(8); + for (i = 0; i < height; i++) { + LOAD_SRC_DST + MERGE_NEXT_SRC(src_reg, src_stride) + FILTER_SRC(filter) + src_reg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); + sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); + src_reg = _mm256_avg_epu8(src_reg, sec_reg); + sec += sec_stride; + MERGE_WITH_SRC(src_reg, zero_reg) + CALC_SUM_SSE_INSIDE_LOOP + src += src_stride; + dst += dst_stride; + } + } + // x_offset = 8 and y_offset = 0 + } else if (x_offset == 8) { + if (y_offset == 0) { + __m256i src_next_reg; + for (i = 0; i < height; i++) { + LOAD_SRC_DST + AVG_NEXT_SRC(src_reg, 1) + sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); + src_reg = _mm256_avg_epu8(src_reg, sec_reg); + sec += sec_stride; + // expand each byte to 2 bytes + MERGE_WITH_SRC(src_reg, zero_reg) + CALC_SUM_SSE_INSIDE_LOOP + src += src_stride; + dst += dst_stride; + } + // x_offset = 8 and y_offset = 8 + } else if (y_offset == 8) { + __m256i src_next_reg, src_avg; + // load source and another source starting from the next + // following byte + src_reg = _mm256_loadu_si256((__m256i const *)(src)); + AVG_NEXT_SRC(src_reg, 1) + for (i = 0; i < height; i++) { + // save current source average + src_avg = src_reg; + src += src_stride; + LOAD_SRC_DST + AVG_NEXT_SRC(src_reg, 1) + // average between previous average to current average + src_avg = _mm256_avg_epu8(src_avg, src_reg); + sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); + src_avg = _mm256_avg_epu8(src_avg, sec_reg); + sec += sec_stride; + // expand each byte to 2 bytes + MERGE_WITH_SRC(src_avg, zero_reg) + CALC_SUM_SSE_INSIDE_LOOP + dst += dst_stride; + } + // x_offset = 8 and y_offset = bilin interpolation + } else { + __m256i filter, pw8, src_next_reg, src_avg; + y_offset <<= 5; + filter = _mm256_load_si256( + (__m256i const *)(bilinear_filters_avx2 + y_offset)); + pw8 = _mm256_set1_epi16(8); + // load source and another source starting from the next + // following byte + src_reg = _mm256_loadu_si256((__m256i const *)(src)); + AVG_NEXT_SRC(src_reg, 1) + for (i = 0; i < height; i++) { + // save current source average + src_avg = src_reg; + src += src_stride; + LOAD_SRC_DST + AVG_NEXT_SRC(src_reg, 1) + MERGE_WITH_SRC(src_avg, src_reg) + FILTER_SRC(filter) + src_avg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); + sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); + src_avg = _mm256_avg_epu8(src_avg, sec_reg); + // expand each byte to 2 bytes + MERGE_WITH_SRC(src_avg, zero_reg) + sec += sec_stride; + CALC_SUM_SSE_INSIDE_LOOP + dst += dst_stride; + } + } + // x_offset = bilin interpolation and y_offset = 0 + } else { + if (y_offset == 0) { + __m256i filter, pw8, src_next_reg; + x_offset <<= 5; + filter = _mm256_load_si256( + (__m256i const *)(bilinear_filters_avx2 + x_offset)); + pw8 = _mm256_set1_epi16(8); + for (i = 0; i < height; i++) { + LOAD_SRC_DST + MERGE_NEXT_SRC(src_reg, 1) + FILTER_SRC(filter) + src_reg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); + sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); + src_reg = _mm256_avg_epu8(src_reg, sec_reg); + MERGE_WITH_SRC(src_reg, zero_reg) + sec += sec_stride; + CALC_SUM_SSE_INSIDE_LOOP + src += src_stride; + dst += dst_stride; + } + // x_offset = bilin interpolation and y_offset = 8 + } else if (y_offset == 8) { + __m256i filter, pw8, src_next_reg, src_pack; + x_offset <<= 5; + filter = _mm256_load_si256( + (__m256i const *)(bilinear_filters_avx2 + x_offset)); + pw8 = _mm256_set1_epi16(8); + src_reg = _mm256_loadu_si256((__m256i const *)(src)); + MERGE_NEXT_SRC(src_reg, 1) + FILTER_SRC(filter) + // convert each 16 bit to 8 bit to each low and high lane source + src_pack = _mm256_packus_epi16(exp_src_lo, exp_src_hi); + for (i = 0; i < height; i++) { + src += src_stride; + LOAD_SRC_DST + MERGE_NEXT_SRC(src_reg, 1) + FILTER_SRC(filter) + src_reg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); + // average between previous pack to the current + src_pack = _mm256_avg_epu8(src_pack, src_reg); + sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); + src_pack = _mm256_avg_epu8(src_pack, sec_reg); + sec += sec_stride; + MERGE_WITH_SRC(src_pack, zero_reg) + src_pack = src_reg; + CALC_SUM_SSE_INSIDE_LOOP + dst += dst_stride; + } + // x_offset = bilin interpolation and y_offset = bilin interpolation + } else { + __m256i xfilter, yfilter, pw8, src_next_reg, src_pack; + x_offset <<= 5; + xfilter = _mm256_load_si256( + (__m256i const *)(bilinear_filters_avx2 + x_offset)); + y_offset <<= 5; + yfilter = _mm256_load_si256( + (__m256i const *)(bilinear_filters_avx2 + y_offset)); + pw8 = _mm256_set1_epi16(8); + // load source and another source starting from the next + // following byte + src_reg = _mm256_loadu_si256((__m256i const *)(src)); + MERGE_NEXT_SRC(src_reg, 1) + + FILTER_SRC(xfilter) + // convert each 16 bit to 8 bit to each low and high lane source + src_pack = _mm256_packus_epi16(exp_src_lo, exp_src_hi); + for (i = 0; i < height; i++) { + src += src_stride; + LOAD_SRC_DST + MERGE_NEXT_SRC(src_reg, 1) + FILTER_SRC(xfilter) + src_reg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); + // merge previous pack to current pack source + MERGE_WITH_SRC(src_pack, src_reg) + // filter the source + FILTER_SRC(yfilter) + src_pack = _mm256_packus_epi16(exp_src_lo, exp_src_hi); + sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); + src_pack = _mm256_avg_epu8(src_pack, sec_reg); + MERGE_WITH_SRC(src_pack, zero_reg) + src_pack = src_reg; + sec += sec_stride; + CALC_SUM_SSE_INSIDE_LOOP + dst += dst_stride; + } + } + } + CALC_SUM_AND_SSE + _mm256_zeroupper(); + return sum; +} diff --git a/third_party/aom/aom_dsp/x86/variance_sse2.c b/third_party/aom/aom_dsp/x86/variance_sse2.c new file mode 100644 index 0000000000..d9563aa7f1 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/variance_sse2.c @@ -0,0 +1,690 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include // SSE2 + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "aom_ports/mem.h" + +typedef void (*getNxMvar_fn_t)(const unsigned char *src, int src_stride, + const unsigned char *ref, int ref_stride, + unsigned int *sse, int *sum); + +unsigned int aom_get_mb_ss_sse2(const int16_t *src) { + __m128i vsum = _mm_setzero_si128(); + int i; + + for (i = 0; i < 32; ++i) { + const __m128i v = _mm_loadu_si128((const __m128i *)src); + vsum = _mm_add_epi32(vsum, _mm_madd_epi16(v, v)); + src += 8; + } + + vsum = _mm_add_epi32(vsum, _mm_srli_si128(vsum, 8)); + vsum = _mm_add_epi32(vsum, _mm_srli_si128(vsum, 4)); + return _mm_cvtsi128_si32(vsum); +} + +#define READ64(p, stride, i) \ + _mm_unpacklo_epi8( \ + _mm_cvtsi32_si128(*(const uint32_t *)(p + i * stride)), \ + _mm_cvtsi32_si128(*(const uint32_t *)(p + (i + 1) * stride))) + +static void get4x4var_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse, int *sum) { + const __m128i zero = _mm_setzero_si128(); + const __m128i src0 = _mm_unpacklo_epi8(READ64(src, src_stride, 0), zero); + const __m128i src1 = _mm_unpacklo_epi8(READ64(src, src_stride, 2), zero); + const __m128i ref0 = _mm_unpacklo_epi8(READ64(ref, ref_stride, 0), zero); + const __m128i ref1 = _mm_unpacklo_epi8(READ64(ref, ref_stride, 2), zero); + const __m128i diff0 = _mm_sub_epi16(src0, ref0); + const __m128i diff1 = _mm_sub_epi16(src1, ref1); + + // sum + __m128i vsum = _mm_add_epi16(diff0, diff1); + vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 8)); + vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 4)); + vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 2)); + *sum = (int16_t)_mm_extract_epi16(vsum, 0); + + // sse + vsum = + _mm_add_epi32(_mm_madd_epi16(diff0, diff0), _mm_madd_epi16(diff1, diff1)); + vsum = _mm_add_epi32(vsum, _mm_srli_si128(vsum, 8)); + vsum = _mm_add_epi32(vsum, _mm_srli_si128(vsum, 4)); + *sse = _mm_cvtsi128_si32(vsum); +} + +void aom_get8x8var_sse2(const uint8_t *src, int src_stride, const uint8_t *ref, + int ref_stride, unsigned int *sse, int *sum) { + const __m128i zero = _mm_setzero_si128(); + __m128i vsum = _mm_setzero_si128(); + __m128i vsse = _mm_setzero_si128(); + int i; + + for (i = 0; i < 8; i += 2) { + const __m128i src0 = _mm_unpacklo_epi8( + _mm_loadl_epi64((const __m128i *)(src + i * src_stride)), zero); + const __m128i ref0 = _mm_unpacklo_epi8( + _mm_loadl_epi64((const __m128i *)(ref + i * ref_stride)), zero); + const __m128i diff0 = _mm_sub_epi16(src0, ref0); + + const __m128i src1 = _mm_unpacklo_epi8( + _mm_loadl_epi64((const __m128i *)(src + (i + 1) * src_stride)), zero); + const __m128i ref1 = _mm_unpacklo_epi8( + _mm_loadl_epi64((const __m128i *)(ref + (i + 1) * ref_stride)), zero); + const __m128i diff1 = _mm_sub_epi16(src1, ref1); + + vsum = _mm_add_epi16(vsum, diff0); + vsum = _mm_add_epi16(vsum, diff1); + vsse = _mm_add_epi32(vsse, _mm_madd_epi16(diff0, diff0)); + vsse = _mm_add_epi32(vsse, _mm_madd_epi16(diff1, diff1)); + } + + // sum + vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 8)); + vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 4)); + vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 2)); + *sum = (int16_t)_mm_extract_epi16(vsum, 0); + + // sse + vsse = _mm_add_epi32(vsse, _mm_srli_si128(vsse, 8)); + vsse = _mm_add_epi32(vsse, _mm_srli_si128(vsse, 4)); + *sse = _mm_cvtsi128_si32(vsse); +} + +void aom_get16x16var_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, unsigned int *sse, + int *sum) { + const __m128i zero = _mm_setzero_si128(); + __m128i vsum = _mm_setzero_si128(); + __m128i vsse = _mm_setzero_si128(); + int i; + + for (i = 0; i < 16; ++i) { + const __m128i s = _mm_loadu_si128((const __m128i *)src); + const __m128i r = _mm_loadu_si128((const __m128i *)ref); + + const __m128i src0 = _mm_unpacklo_epi8(s, zero); + const __m128i ref0 = _mm_unpacklo_epi8(r, zero); + const __m128i diff0 = _mm_sub_epi16(src0, ref0); + + const __m128i src1 = _mm_unpackhi_epi8(s, zero); + const __m128i ref1 = _mm_unpackhi_epi8(r, zero); + const __m128i diff1 = _mm_sub_epi16(src1, ref1); + + vsum = _mm_add_epi16(vsum, diff0); + vsum = _mm_add_epi16(vsum, diff1); + vsse = _mm_add_epi32(vsse, _mm_madd_epi16(diff0, diff0)); + vsse = _mm_add_epi32(vsse, _mm_madd_epi16(diff1, diff1)); + + src += src_stride; + ref += ref_stride; + } + + // sum + vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 8)); + vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 4)); + *sum = + (int16_t)_mm_extract_epi16(vsum, 0) + (int16_t)_mm_extract_epi16(vsum, 1); + + // sse + vsse = _mm_add_epi32(vsse, _mm_srli_si128(vsse, 8)); + vsse = _mm_add_epi32(vsse, _mm_srli_si128(vsse, 4)); + *sse = _mm_cvtsi128_si32(vsse); +} + +static void variance_sse2(const unsigned char *src, int src_stride, + const unsigned char *ref, int ref_stride, int w, + int h, unsigned int *sse, int *sum, + getNxMvar_fn_t var_fn, int block_size) { + int i, j; + + *sse = 0; + *sum = 0; + + for (i = 0; i < h; i += block_size) { + for (j = 0; j < w; j += block_size) { + unsigned int sse0; + int sum0; + var_fn(src + src_stride * i + j, src_stride, ref + ref_stride * i + j, + ref_stride, &sse0, &sum0); + *sse += sse0; + *sum += sum0; + } + } +} + +unsigned int aom_variance4x4_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + get4x4var_sse2(src, src_stride, ref, ref_stride, sse, &sum); + assert(sum <= 255 * 4 * 4); + assert(sum >= -255 * 4 * 4); + return *sse - ((sum * sum) >> 4); +} + +unsigned int aom_variance8x4_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + variance_sse2(src, src_stride, ref, ref_stride, 8, 4, sse, &sum, + get4x4var_sse2, 4); + assert(sum <= 255 * 8 * 4); + assert(sum >= -255 * 8 * 4); + return *sse - ((sum * sum) >> 5); +} + +unsigned int aom_variance4x8_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + variance_sse2(src, src_stride, ref, ref_stride, 4, 8, sse, &sum, + get4x4var_sse2, 4); + assert(sum <= 255 * 8 * 4); + assert(sum >= -255 * 8 * 4); + return *sse - ((sum * sum) >> 5); +} + +unsigned int aom_variance8x8_sse2(const unsigned char *src, int src_stride, + const unsigned char *ref, int ref_stride, + unsigned int *sse) { + int sum; + aom_get8x8var_sse2(src, src_stride, ref, ref_stride, sse, &sum); + assert(sum <= 255 * 8 * 8); + assert(sum >= -255 * 8 * 8); + return *sse - ((sum * sum) >> 6); +} + +unsigned int aom_variance16x8_sse2(const unsigned char *src, int src_stride, + const unsigned char *ref, int ref_stride, + unsigned int *sse) { + int sum; + variance_sse2(src, src_stride, ref, ref_stride, 16, 8, sse, &sum, + aom_get8x8var_sse2, 8); + assert(sum <= 255 * 16 * 8); + assert(sum >= -255 * 16 * 8); + return *sse - ((sum * sum) >> 7); +} + +unsigned int aom_variance8x16_sse2(const unsigned char *src, int src_stride, + const unsigned char *ref, int ref_stride, + unsigned int *sse) { + int sum; + variance_sse2(src, src_stride, ref, ref_stride, 8, 16, sse, &sum, + aom_get8x8var_sse2, 8); + assert(sum <= 255 * 16 * 8); + assert(sum >= -255 * 16 * 8); + return *sse - ((sum * sum) >> 7); +} + +unsigned int aom_variance16x16_sse2(const unsigned char *src, int src_stride, + const unsigned char *ref, int ref_stride, + unsigned int *sse) { + int sum; + aom_get16x16var_sse2(src, src_stride, ref, ref_stride, sse, &sum); + assert(sum <= 255 * 16 * 16); + assert(sum >= -255 * 16 * 16); + return *sse - ((uint32_t)((int64_t)sum * sum) >> 8); +} + +unsigned int aom_variance32x32_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + variance_sse2(src, src_stride, ref, ref_stride, 32, 32, sse, &sum, + aom_get16x16var_sse2, 16); + assert(sum <= 255 * 32 * 32); + assert(sum >= -255 * 32 * 32); + return *sse - (unsigned int)(((int64_t)sum * sum) >> 10); +} + +unsigned int aom_variance32x16_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + variance_sse2(src, src_stride, ref, ref_stride, 32, 16, sse, &sum, + aom_get16x16var_sse2, 16); + assert(sum <= 255 * 32 * 16); + assert(sum >= -255 * 32 * 16); + return *sse - (unsigned int)(((int64_t)sum * sum) >> 9); +} + +unsigned int aom_variance16x32_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + variance_sse2(src, src_stride, ref, ref_stride, 16, 32, sse, &sum, + aom_get16x16var_sse2, 16); + assert(sum <= 255 * 32 * 16); + assert(sum >= -255 * 32 * 16); + return *sse - (unsigned int)(((int64_t)sum * sum) >> 9); +} + +unsigned int aom_variance64x64_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + variance_sse2(src, src_stride, ref, ref_stride, 64, 64, sse, &sum, + aom_get16x16var_sse2, 16); + assert(sum <= 255 * 64 * 64); + assert(sum >= -255 * 64 * 64); + return *sse - (unsigned int)(((int64_t)sum * sum) >> 12); +} + +unsigned int aom_variance64x32_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + variance_sse2(src, src_stride, ref, ref_stride, 64, 32, sse, &sum, + aom_get16x16var_sse2, 16); + assert(sum <= 255 * 64 * 32); + assert(sum >= -255 * 64 * 32); + return *sse - (unsigned int)(((int64_t)sum * sum) >> 11); +} + +unsigned int aom_variance32x64_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + int sum; + variance_sse2(src, src_stride, ref, ref_stride, 32, 64, sse, &sum, + aom_get16x16var_sse2, 16); + assert(sum <= 255 * 64 * 32); + assert(sum >= -255 * 64 * 32); + return *sse - (unsigned int)(((int64_t)sum * sum) >> 11); +} + +unsigned int aom_mse8x8_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + aom_variance8x8_sse2(src, src_stride, ref, ref_stride, sse); + return *sse; +} + +unsigned int aom_mse8x16_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + aom_variance8x16_sse2(src, src_stride, ref, ref_stride, sse); + return *sse; +} + +unsigned int aom_mse16x8_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + aom_variance16x8_sse2(src, src_stride, ref, ref_stride, sse); + return *sse; +} + +unsigned int aom_mse16x16_sse2(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + unsigned int *sse) { + aom_variance16x16_sse2(src, src_stride, ref, ref_stride, sse); + return *sse; +} + +// The 2 unused parameters are place holders for PIC enabled build. +// These definitions are for functions defined in subpel_variance.asm +#define DECL(w, opt) \ + int aom_sub_pixel_variance##w##xh_##opt( \ + const uint8_t *src, ptrdiff_t src_stride, int x_offset, int y_offset, \ + const uint8_t *dst, ptrdiff_t dst_stride, int height, unsigned int *sse, \ + void *unused0, void *unused) +#define DECLS(opt1, opt2) \ + DECL(4, opt1); \ + DECL(8, opt1); \ + DECL(16, opt1) + +DECLS(sse2, sse2); +DECLS(ssse3, ssse3); +#undef DECLS +#undef DECL + +#define FN(w, h, wf, wlog2, hlog2, opt, cast_prod, cast) \ + unsigned int aom_sub_pixel_variance##w##x##h##_##opt( \ + const uint8_t *src, int src_stride, int x_offset, int y_offset, \ + const uint8_t *dst, int dst_stride, unsigned int *sse_ptr) { \ + unsigned int sse; \ + int se = aom_sub_pixel_variance##wf##xh_##opt(src, src_stride, x_offset, \ + y_offset, dst, dst_stride, \ + h, &sse, NULL, NULL); \ + if (w > wf) { \ + unsigned int sse2; \ + int se2 = aom_sub_pixel_variance##wf##xh_##opt( \ + src + 16, src_stride, x_offset, y_offset, dst + 16, dst_stride, h, \ + &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + if (w > wf * 2) { \ + se2 = aom_sub_pixel_variance##wf##xh_##opt( \ + src + 32, src_stride, x_offset, y_offset, dst + 32, dst_stride, h, \ + &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + se2 = aom_sub_pixel_variance##wf##xh_##opt( \ + src + 48, src_stride, x_offset, y_offset, dst + 48, dst_stride, h, \ + &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + } \ + } \ + *sse_ptr = sse; \ + return sse - (unsigned int)(cast_prod(cast se * se) >> (wlog2 + hlog2)); \ + } + +#define FNS(opt1, opt2) \ + FN(64, 64, 16, 6, 6, opt1, (int64_t), (int64_t)); \ + FN(64, 32, 16, 6, 5, opt1, (int64_t), (int64_t)); \ + FN(32, 64, 16, 5, 6, opt1, (int64_t), (int64_t)); \ + FN(32, 32, 16, 5, 5, opt1, (int64_t), (int64_t)); \ + FN(32, 16, 16, 5, 4, opt1, (int64_t), (int64_t)); \ + FN(16, 32, 16, 4, 5, opt1, (int64_t), (int64_t)); \ + FN(16, 16, 16, 4, 4, opt1, (uint32_t), (int64_t)); \ + FN(16, 8, 16, 4, 3, opt1, (int32_t), (int32_t)); \ + FN(8, 16, 8, 3, 4, opt1, (int32_t), (int32_t)); \ + FN(8, 8, 8, 3, 3, opt1, (int32_t), (int32_t)); \ + FN(8, 4, 8, 3, 2, opt1, (int32_t), (int32_t)); \ + FN(4, 8, 4, 2, 3, opt1, (int32_t), (int32_t)); \ + FN(4, 4, 4, 2, 2, opt1, (int32_t), (int32_t)) + +FNS(sse2, sse2); +FNS(ssse3, ssse3); + +#undef FNS +#undef FN + +// The 2 unused parameters are place holders for PIC enabled build. +#define DECL(w, opt) \ + int aom_sub_pixel_avg_variance##w##xh_##opt( \ + const uint8_t *src, ptrdiff_t src_stride, int x_offset, int y_offset, \ + const uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *sec, \ + ptrdiff_t sec_stride, int height, unsigned int *sse, void *unused0, \ + void *unused) +#define DECLS(opt1, opt2) \ + DECL(4, opt1); \ + DECL(8, opt1); \ + DECL(16, opt1) + +DECLS(sse2, sse2); +DECLS(ssse3, ssse3); +#undef DECL +#undef DECLS + +#define FN(w, h, wf, wlog2, hlog2, opt, cast_prod, cast) \ + unsigned int aom_sub_pixel_avg_variance##w##x##h##_##opt( \ + const uint8_t *src, int src_stride, int x_offset, int y_offset, \ + const uint8_t *dst, int dst_stride, unsigned int *sseptr, \ + const uint8_t *sec) { \ + unsigned int sse; \ + int se = aom_sub_pixel_avg_variance##wf##xh_##opt( \ + src, src_stride, x_offset, y_offset, dst, dst_stride, sec, w, h, &sse, \ + NULL, NULL); \ + if (w > wf) { \ + unsigned int sse2; \ + int se2 = aom_sub_pixel_avg_variance##wf##xh_##opt( \ + src + 16, src_stride, x_offset, y_offset, dst + 16, dst_stride, \ + sec + 16, w, h, &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + if (w > wf * 2) { \ + se2 = aom_sub_pixel_avg_variance##wf##xh_##opt( \ + src + 32, src_stride, x_offset, y_offset, dst + 32, dst_stride, \ + sec + 32, w, h, &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + se2 = aom_sub_pixel_avg_variance##wf##xh_##opt( \ + src + 48, src_stride, x_offset, y_offset, dst + 48, dst_stride, \ + sec + 48, w, h, &sse2, NULL, NULL); \ + se += se2; \ + sse += sse2; \ + } \ + } \ + *sseptr = sse; \ + return sse - (unsigned int)(cast_prod(cast se * se) >> (wlog2 + hlog2)); \ + } + +#define FNS(opt1, opt2) \ + FN(64, 64, 16, 6, 6, opt1, (int64_t), (int64_t)); \ + FN(64, 32, 16, 6, 5, opt1, (int64_t), (int64_t)); \ + FN(32, 64, 16, 5, 6, opt1, (int64_t), (int64_t)); \ + FN(32, 32, 16, 5, 5, opt1, (int64_t), (int64_t)); \ + FN(32, 16, 16, 5, 4, opt1, (int64_t), (int64_t)); \ + FN(16, 32, 16, 4, 5, opt1, (int64_t), (int64_t)); \ + FN(16, 16, 16, 4, 4, opt1, (uint32_t), (int64_t)); \ + FN(16, 8, 16, 4, 3, opt1, (uint32_t), (int32_t)); \ + FN(8, 16, 8, 3, 4, opt1, (uint32_t), (int32_t)); \ + FN(8, 8, 8, 3, 3, opt1, (uint32_t), (int32_t)); \ + FN(8, 4, 8, 3, 2, opt1, (uint32_t), (int32_t)); \ + FN(4, 8, 4, 2, 3, opt1, (uint32_t), (int32_t)); \ + FN(4, 4, 4, 2, 2, opt1, (uint32_t), (int32_t)) + +FNS(sse2, sse); +FNS(ssse3, ssse3); + +#undef FNS +#undef FN + +void aom_upsampled_pred_sse2(uint8_t *comp_pred, int width, int height, + const uint8_t *ref, int ref_stride) { + int i, j; + int stride = ref_stride << 3; + + if (width >= 16) { + // read 16 points at one time + for (i = 0; i < height; i++) { + for (j = 0; j < width; j += 16) { + __m128i s0 = _mm_loadu_si128((const __m128i *)ref); + __m128i s1 = _mm_loadu_si128((const __m128i *)(ref + 16)); + __m128i s2 = _mm_loadu_si128((const __m128i *)(ref + 32)); + __m128i s3 = _mm_loadu_si128((const __m128i *)(ref + 48)); + __m128i s4 = _mm_loadu_si128((const __m128i *)(ref + 64)); + __m128i s5 = _mm_loadu_si128((const __m128i *)(ref + 80)); + __m128i s6 = _mm_loadu_si128((const __m128i *)(ref + 96)); + __m128i s7 = _mm_loadu_si128((const __m128i *)(ref + 112)); + __m128i t0, t1, t2, t3; + + t0 = _mm_unpacklo_epi8(s0, s1); + s1 = _mm_unpackhi_epi8(s0, s1); + t1 = _mm_unpacklo_epi8(s2, s3); + s3 = _mm_unpackhi_epi8(s2, s3); + t2 = _mm_unpacklo_epi8(s4, s5); + s5 = _mm_unpackhi_epi8(s4, s5); + t3 = _mm_unpacklo_epi8(s6, s7); + s7 = _mm_unpackhi_epi8(s6, s7); + + s0 = _mm_unpacklo_epi8(t0, s1); + s2 = _mm_unpacklo_epi8(t1, s3); + s4 = _mm_unpacklo_epi8(t2, s5); + s6 = _mm_unpacklo_epi8(t3, s7); + s0 = _mm_unpacklo_epi32(s0, s2); + s4 = _mm_unpacklo_epi32(s4, s6); + s0 = _mm_unpacklo_epi64(s0, s4); + + _mm_storeu_si128((__m128i *)(comp_pred), s0); + comp_pred += 16; + ref += 16 * 8; + } + ref += stride - (width << 3); + } + } else if (width >= 8) { + // read 8 points at one time + for (i = 0; i < height; i++) { + for (j = 0; j < width; j += 8) { + __m128i s0 = _mm_loadu_si128((const __m128i *)ref); + __m128i s1 = _mm_loadu_si128((const __m128i *)(ref + 16)); + __m128i s2 = _mm_loadu_si128((const __m128i *)(ref + 32)); + __m128i s3 = _mm_loadu_si128((const __m128i *)(ref + 48)); + __m128i t0, t1; + + t0 = _mm_unpacklo_epi8(s0, s1); + s1 = _mm_unpackhi_epi8(s0, s1); + t1 = _mm_unpacklo_epi8(s2, s3); + s3 = _mm_unpackhi_epi8(s2, s3); + + s0 = _mm_unpacklo_epi8(t0, s1); + s2 = _mm_unpacklo_epi8(t1, s3); + s0 = _mm_unpacklo_epi32(s0, s2); + + _mm_storel_epi64((__m128i *)(comp_pred), s0); + comp_pred += 8; + ref += 8 * 8; + } + ref += stride - (width << 3); + } + } else { + // read 4 points at one time + for (i = 0; i < height; i++) { + for (j = 0; j < width; j += 4) { + __m128i s0 = _mm_loadu_si128((const __m128i *)ref); + __m128i s1 = _mm_loadu_si128((const __m128i *)(ref + 16)); + __m128i t0; + + t0 = _mm_unpacklo_epi8(s0, s1); + s1 = _mm_unpackhi_epi8(s0, s1); + s0 = _mm_unpacklo_epi8(t0, s1); + + *(int *)comp_pred = _mm_cvtsi128_si32(s0); + comp_pred += 4; + ref += 4 * 8; + } + ref += stride - (width << 3); + } + } +} + +void aom_comp_avg_upsampled_pred_sse2(uint8_t *comp_pred, const uint8_t *pred, + int width, int height, const uint8_t *ref, + int ref_stride) { + const __m128i zero = _mm_set1_epi16(0); + const __m128i one = _mm_set1_epi16(1); + int i, j; + int stride = ref_stride << 3; + + if (width >= 16) { + // read 16 points at one time + for (i = 0; i < height; i++) { + for (j = 0; j < width; j += 16) { + __m128i s0 = _mm_loadu_si128((const __m128i *)ref); + __m128i s1 = _mm_loadu_si128((const __m128i *)(ref + 16)); + __m128i s2 = _mm_loadu_si128((const __m128i *)(ref + 32)); + __m128i s3 = _mm_loadu_si128((const __m128i *)(ref + 48)); + __m128i s4 = _mm_loadu_si128((const __m128i *)(ref + 64)); + __m128i s5 = _mm_loadu_si128((const __m128i *)(ref + 80)); + __m128i s6 = _mm_loadu_si128((const __m128i *)(ref + 96)); + __m128i s7 = _mm_loadu_si128((const __m128i *)(ref + 112)); + __m128i p0 = _mm_loadu_si128((const __m128i *)pred); + __m128i p1; + __m128i t0, t1, t2, t3; + + t0 = _mm_unpacklo_epi8(s0, s1); + s1 = _mm_unpackhi_epi8(s0, s1); + t1 = _mm_unpacklo_epi8(s2, s3); + s3 = _mm_unpackhi_epi8(s2, s3); + t2 = _mm_unpacklo_epi8(s4, s5); + s5 = _mm_unpackhi_epi8(s4, s5); + t3 = _mm_unpacklo_epi8(s6, s7); + s7 = _mm_unpackhi_epi8(s6, s7); + + s0 = _mm_unpacklo_epi8(t0, s1); + s2 = _mm_unpacklo_epi8(t1, s3); + s4 = _mm_unpacklo_epi8(t2, s5); + s6 = _mm_unpacklo_epi8(t3, s7); + + s0 = _mm_unpacklo_epi32(s0, s2); + s4 = _mm_unpacklo_epi32(s4, s6); + s0 = _mm_unpacklo_epi8(s0, zero); + s4 = _mm_unpacklo_epi8(s4, zero); + + p1 = _mm_unpackhi_epi8(p0, zero); + p0 = _mm_unpacklo_epi8(p0, zero); + p0 = _mm_adds_epu16(s0, p0); + p1 = _mm_adds_epu16(s4, p1); + p0 = _mm_adds_epu16(p0, one); + p1 = _mm_adds_epu16(p1, one); + + p0 = _mm_srli_epi16(p0, 1); + p1 = _mm_srli_epi16(p1, 1); + p0 = _mm_packus_epi16(p0, p1); + + _mm_storeu_si128((__m128i *)(comp_pred), p0); + comp_pred += 16; + pred += 16; + ref += 16 * 8; + } + ref += stride - (width << 3); + } + } else if (width >= 8) { + // read 8 points at one time + for (i = 0; i < height; i++) { + for (j = 0; j < width; j += 8) { + __m128i s0 = _mm_loadu_si128((const __m128i *)ref); + __m128i s1 = _mm_loadu_si128((const __m128i *)(ref + 16)); + __m128i s2 = _mm_loadu_si128((const __m128i *)(ref + 32)); + __m128i s3 = _mm_loadu_si128((const __m128i *)(ref + 48)); + __m128i p0 = _mm_loadl_epi64((const __m128i *)pred); + __m128i t0, t1; + + t0 = _mm_unpacklo_epi8(s0, s1); + s1 = _mm_unpackhi_epi8(s0, s1); + t1 = _mm_unpacklo_epi8(s2, s3); + s3 = _mm_unpackhi_epi8(s2, s3); + + s0 = _mm_unpacklo_epi8(t0, s1); + s2 = _mm_unpacklo_epi8(t1, s3); + s0 = _mm_unpacklo_epi32(s0, s2); + s0 = _mm_unpacklo_epi8(s0, zero); + + p0 = _mm_unpacklo_epi8(p0, zero); + p0 = _mm_adds_epu16(s0, p0); + p0 = _mm_adds_epu16(p0, one); + p0 = _mm_srli_epi16(p0, 1); + p0 = _mm_packus_epi16(p0, zero); + + _mm_storel_epi64((__m128i *)(comp_pred), p0); + comp_pred += 8; + pred += 8; + ref += 8 * 8; + } + ref += stride - (width << 3); + } + } else { + // read 4 points at one time + for (i = 0; i < height; i++) { + for (j = 0; j < width; j += 4) { + __m128i s0 = _mm_loadu_si128((const __m128i *)ref); + __m128i s1 = _mm_loadu_si128((const __m128i *)(ref + 16)); + __m128i p0 = _mm_cvtsi32_si128(*(const uint32_t *)pred); + __m128i t0; + + t0 = _mm_unpacklo_epi8(s0, s1); + s1 = _mm_unpackhi_epi8(s0, s1); + s0 = _mm_unpacklo_epi8(t0, s1); + s0 = _mm_unpacklo_epi8(s0, zero); + + p0 = _mm_unpacklo_epi8(p0, zero); + p0 = _mm_adds_epu16(s0, p0); + p0 = _mm_adds_epu16(p0, one); + p0 = _mm_srli_epi16(p0, 1); + p0 = _mm_packus_epi16(p0, zero); + + *(int *)comp_pred = _mm_cvtsi128_si32(p0); + comp_pred += 4; + pred += 4; + ref += 4 * 8; + } + ref += stride - (width << 3); + } + } +} diff --git a/third_party/aom/aom_mem/aom_mem.c b/third_party/aom/aom_mem/aom_mem.c new file mode 100644 index 0000000000..66a0c08dea --- /dev/null +++ b/third_party/aom/aom_mem/aom_mem.c @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#define __AOM_MEM_C__ + +#include "aom_mem.h" +#include +#include +#include +#include "include/aom_mem_intrnl.h" +#include "aom/aom_integer.h" + +static size_t GetAlignedMallocSize(size_t size, size_t align) { + return size + align - 1 + ADDRESS_STORAGE_SIZE; +} + +static size_t *GetMallocAddressLocation(void *const mem) { + return ((size_t *)mem) - 1; +} + +static void SetActualMallocAddress(void *const mem, + const void *const malloc_addr) { + size_t *const malloc_addr_location = GetMallocAddressLocation(mem); + *malloc_addr_location = (size_t)malloc_addr; +} + +static void *GetActualMallocAddress(void *const mem) { + const size_t *const malloc_addr_location = GetMallocAddressLocation(mem); + return (void *)(*malloc_addr_location); +} + +void *aom_memalign(size_t align, size_t size) { + void *x = NULL; + const size_t aligned_size = GetAlignedMallocSize(size, align); + void *const addr = malloc(aligned_size); + if (addr) { + x = align_addr((unsigned char *)addr + ADDRESS_STORAGE_SIZE, align); + SetActualMallocAddress(x, addr); + } + return x; +} + +void *aom_malloc(size_t size) { return aom_memalign(DEFAULT_ALIGNMENT, size); } + +void *aom_calloc(size_t num, size_t size) { + const size_t total_size = num * size; + void *const x = aom_malloc(total_size); + if (x) memset(x, 0, total_size); + return x; +} + +void aom_free(void *memblk) { + if (memblk) { + void *addr = GetActualMallocAddress(memblk); + free(addr); + } +} + +#if CONFIG_HIGHBITDEPTH +void *aom_memset16(void *dest, int val, size_t length) { + size_t i; + uint16_t *dest16 = (uint16_t *)dest; + for (i = 0; i < length; i++) *dest16++ = val; + return dest; +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/aom_mem/aom_mem.cmake b/third_party/aom/aom_mem/aom_mem.cmake new file mode 100644 index 0000000000..0375b09e06 --- /dev/null +++ b/third_party/aom/aom_mem/aom_mem.cmake @@ -0,0 +1,22 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +set(AOM_MEM_SOURCES + "${AOM_ROOT}/aom_mem/aom_mem.c" + "${AOM_ROOT}/aom_mem/aom_mem.h" + "${AOM_ROOT}/aom_mem/include/aom_mem_intrnl.h") + +# Creates the aom_mem build target and makes libaom depend on it. The libaom +# target must exist before this function is called. +function (setup_aom_mem_targets) + add_library(aom_mem OBJECT ${AOM_MEM_SOURCES}) + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_mem PARENT_SCOPE) + target_sources(aom PUBLIC $) +endfunction () diff --git a/third_party/aom/aom_mem/aom_mem.h b/third_party/aom/aom_mem/aom_mem.h new file mode 100644 index 0000000000..75bd4be65c --- /dev/null +++ b/third_party/aom/aom_mem/aom_mem.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_MEM_AOM_MEM_H_ +#define AOM_MEM_AOM_MEM_H_ + +#include "aom_config.h" +#if defined(__uClinux__) +#include +#endif + +#include +#include + +#if defined(__cplusplus) +extern "C" { +#endif + +void *aom_memalign(size_t align, size_t size); +void *aom_malloc(size_t size); +void *aom_calloc(size_t num, size_t size); +void aom_free(void *memblk); + +#if CONFIG_HIGHBITDEPTH +void *aom_memset16(void *dest, int val, size_t length); +#endif + +#include + +#ifdef AOM_MEM_PLTFRM +#include AOM_MEM_PLTFRM +#endif + +#if defined(__cplusplus) +} +#endif + +#endif // AOM_MEM_AOM_MEM_H_ diff --git a/third_party/aom/aom_mem/aom_mem.mk b/third_party/aom/aom_mem/aom_mem.mk new file mode 100644 index 0000000000..e9162c2842 --- /dev/null +++ b/third_party/aom/aom_mem/aom_mem.mk @@ -0,0 +1,4 @@ +MEM_SRCS-yes += aom_mem.mk +MEM_SRCS-yes += aom_mem.c +MEM_SRCS-yes += aom_mem.h +MEM_SRCS-yes += include/aom_mem_intrnl.h diff --git a/third_party/aom/aom_mem/include/aom_mem_intrnl.h b/third_party/aom/aom_mem/include/aom_mem_intrnl.h new file mode 100644 index 0000000000..3cdfbe08df --- /dev/null +++ b/third_party/aom/aom_mem/include/aom_mem_intrnl.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_MEM_INCLUDE_AOM_MEM_INTRNL_H_ +#define AOM_MEM_INCLUDE_AOM_MEM_INTRNL_H_ +#include "./aom_config.h" + +#define ADDRESS_STORAGE_SIZE sizeof(size_t) + +#ifndef DEFAULT_ALIGNMENT +#if defined(VXWORKS) +/*default addr alignment to use in calls to aom_* functions other than + aom_memalign*/ +#define DEFAULT_ALIGNMENT 32 +#else +#define DEFAULT_ALIGNMENT (2 * sizeof(void *)) /* NOLINT */ +#endif +#endif + +/*returns an addr aligned to the byte boundary specified by align*/ +#define align_addr(addr, align) \ + (void *)(((size_t)(addr) + ((align)-1)) & ~(size_t)((align)-1)) + +#endif // AOM_MEM_INCLUDE_AOM_MEM_INTRNL_H_ diff --git a/third_party/aom/aom_ports/aom_once.h b/third_party/aom/aom_ports/aom_once.h new file mode 100644 index 0000000000..3cfd2fd958 --- /dev/null +++ b/third_party/aom/aom_ports/aom_once.h @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_PORTS_AOM_ONCE_H_ +#define AOM_PORTS_AOM_ONCE_H_ + +#include "aom_config.h" + +/* Implement a function wrapper to guarantee initialization + * thread-safety for library singletons. + * + * NOTE: These functions use static locks, and can only be + * used with one common argument per compilation unit. So + * + * file1.c: + * aom_once(foo); + * ... + * aom_once(foo); + * + * file2.c: + * aom_once(bar); + * + * will ensure foo() and bar() are each called only once, but in + * + * file1.c: + * aom_once(foo); + * aom_once(bar): + * + * bar() will never be called because the lock is used up + * by the call to foo(). + */ + +#if CONFIG_MULTITHREAD && defined(_WIN32) +#include +#include +/* Declare a per-compilation-unit state variable to track the progress + * of calling func() only once. This must be at global scope because + * local initializers are not thread-safe in MSVC prior to Visual + * Studio 2015. + * + * As a static, once_state will be zero-initialized as program start. + */ +static LONG once_state; +static void once(void (*func)(void)) { + /* Try to advance once_state from its initial value of 0 to 1. + * Only one thread can succeed in doing so. + */ + if (InterlockedCompareExchange(&once_state, 1, 0) == 0) { + /* We're the winning thread, having set once_state to 1. + * Call our function. */ + func(); + /* Now advance once_state to 2, unblocking any other threads. */ + InterlockedIncrement(&once_state); + return; + } + + /* We weren't the winning thread, but we want to block on + * the state variable so we don't return before func() + * has finished executing elsewhere. + * + * Try to advance once_state from 2 to 2, which is only possible + * after the winning thead advances it from 1 to 2. + */ + while (InterlockedCompareExchange(&once_state, 2, 2) != 2) { + /* State isn't yet 2. Try again. + * + * We are used for singleton initialization functions, + * which should complete quickly. Contention will likewise + * be rare, so it's worthwhile to use a simple but cpu- + * intensive busy-wait instead of successive backoff, + * waiting on a kernel object, or another heavier-weight scheme. + * + * We can at least yield our timeslice. + */ + Sleep(0); + } + + /* We've seen once_state advance to 2, so we know func() + * has been called. And we've left once_state as we found it, + * so other threads will have the same experience. + * + * It's safe to return now. + */ + return; +} + +#elif CONFIG_MULTITHREAD && defined(__OS2__) +#define INCL_DOS +#include +static void once(void (*func)(void)) { + static int done; + + /* If the initialization is complete, return early. */ + if (done) return; + + /* Causes all other threads in the process to block themselves + * and give up their time slice. + */ + DosEnterCritSec(); + + if (!done) { + func(); + done = 1; + } + + /* Restores normal thread dispatching for the current process. */ + DosExitCritSec(); +} + +#elif CONFIG_MULTITHREAD && HAVE_PTHREAD_H +#include +static void once(void (*func)(void)) { + static pthread_once_t lock = PTHREAD_ONCE_INIT; + pthread_once(&lock, func); +} + +#else +/* No-op version that performs no synchronization. *_rtcd() is idempotent, + * so as long as your platform provides atomic loads/stores of pointers + * no synchronization is strictly necessary. + */ + +static void once(void (*func)(void)) { + static int done; + + if (!done) { + func(); + done = 1; + } +} +#endif + +#endif // AOM_PORTS_AOM_ONCE_H_ diff --git a/third_party/aom/aom_ports/aom_ports.cmake b/third_party/aom/aom_ports/aom_ports.cmake new file mode 100644 index 0000000000..3e8883d564 --- /dev/null +++ b/third_party/aom/aom_ports/aom_ports.cmake @@ -0,0 +1,66 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +set(AOM_PORTS_INCLUDES + "${AOM_ROOT}/aom_ports/aom_once.h" + "${AOM_ROOT}/aom_ports/aom_timer.h" + "${AOM_ROOT}/aom_ports/bitops.h" + "${AOM_ROOT}/aom_ports/emmintrin_compat.h" + "${AOM_ROOT}/aom_ports/mem.h" + "${AOM_ROOT}/aom_ports/mem_ops.h" + "${AOM_ROOT}/aom_ports/mem_ops_aligned.h" + "${AOM_ROOT}/aom_ports/msvc.h" + "${AOM_ROOT}/aom_ports/system_state.h") + +set(AOM_PORTS_INCLUDES_X86 + "${AOM_ROOT}/aom_ports/x86_abi_support.asm") + +set(AOM_PORTS_ASM_MMX "${AOM_ROOT}/aom_ports/emms.asm") + +set(AOM_PORTS_SOURCES_ARM + "${AOM_ROOT}/aom_ports/arm.h" + "${AOM_ROOT}/aom_ports/arm_cpudetect.c") + +# For arm targets and targets where HAVE_MMX is true: +# Creates the aom_ports build target, adds the includes in aom_ports to the +# target, and makes libaom depend on it. +# Otherwise: +# Adds the includes in aom_ports to the libaom target. +# For all target platforms: +# The libaom target must exist before this function is called. +function (setup_aom_ports_targets) + if (HAVE_MMX) + add_asm_library("aom_ports" "AOM_PORTS_ASM_MMX" "aom") + set(aom_ports_has_symbols 1) + elseif ("${AOM_TARGET_CPU}" MATCHES "arm") + add_library(aom_ports OBJECT ${AOM_PORTS_SOURCES_ARM}) + set(aom_ports_has_symbols 1) + list(APPEND AOM_LIB_TARGETS aom_ports) + target_sources(aom PRIVATE $) + endif () + + if (aom_ports_has_symbols) + target_sources(aom_ports PUBLIC ${AOM_PORTS_INCLUDES}) + + if ("${AOM_TARGET_CPU}" STREQUAL "x86" OR + "${AOM_TARGET_CPU}" STREQUAL "x86_64") + target_sources(aom_ports PUBLIC ${AOM_PORTS_INCLUDES_X86}) + endif () + + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} PARENT_SCOPE) + else () + target_sources(aom PUBLIC ${AOM_PORTS_INCLUDES}) + + if ("${AOM_TARGET_CPU}" STREQUAL "x86" OR + "${AOM_TARGET_CPU}" STREQUAL "x86_64") + target_sources(aom PUBLIC ${AOM_PORTS_INCLUDES_X86}) + endif () + endif () +endfunction () diff --git a/third_party/aom/aom_ports/aom_ports.mk b/third_party/aom/aom_ports/aom_ports.mk new file mode 100644 index 0000000000..1f18f6bd16 --- /dev/null +++ b/third_party/aom/aom_ports/aom_ports.mk @@ -0,0 +1,29 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + + +PORTS_SRCS-yes += aom_ports.mk + +PORTS_SRCS-yes += bitops.h +PORTS_SRCS-yes += mem.h +PORTS_SRCS-yes += msvc.h +PORTS_SRCS-yes += system_state.h +PORTS_SRCS-yes += aom_timer.h + +ifeq ($(ARCH_X86)$(ARCH_X86_64),yes) +PORTS_SRCS-yes += emms.asm +PORTS_SRCS-yes += x86.h +PORTS_SRCS-yes += x86_abi_support.asm +endif + +PORTS_SRCS-$(ARCH_ARM) += arm_cpudetect.c +PORTS_SRCS-$(ARCH_ARM) += arm.h diff --git a/third_party/aom/aom_ports/aom_timer.h b/third_party/aom/aom_ports/aom_timer.h new file mode 100644 index 0000000000..904f2fe51c --- /dev/null +++ b/third_party/aom/aom_ports/aom_timer.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_PORTS_AOM_TIMER_H_ +#define AOM_PORTS_AOM_TIMER_H_ + +#include "./aom_config.h" + +#include "aom/aom_integer.h" + +#if CONFIG_OS_SUPPORT + +#if defined(_WIN32) +/* + * Win32 specific includes + */ +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +/* + * POSIX specific includes + */ +#include + +/* timersub is not provided by msys at this time. */ +#ifndef timersub +#define timersub(a, b, result) \ + do { \ + (result)->tv_sec = (a)->tv_sec - (b)->tv_sec; \ + (result)->tv_usec = (a)->tv_usec - (b)->tv_usec; \ + if ((result)->tv_usec < 0) { \ + --(result)->tv_sec; \ + (result)->tv_usec += 1000000; \ + } \ + } while (0) +#endif +#endif + +struct aom_usec_timer { +#if defined(_WIN32) + LARGE_INTEGER begin, end; +#else + struct timeval begin, end; +#endif +}; + +static INLINE void aom_usec_timer_start(struct aom_usec_timer *t) { +#if defined(_WIN32) + QueryPerformanceCounter(&t->begin); +#else + gettimeofday(&t->begin, NULL); +#endif +} + +static INLINE void aom_usec_timer_mark(struct aom_usec_timer *t) { +#if defined(_WIN32) + QueryPerformanceCounter(&t->end); +#else + gettimeofday(&t->end, NULL); +#endif +} + +static INLINE int64_t aom_usec_timer_elapsed(struct aom_usec_timer *t) { +#if defined(_WIN32) + LARGE_INTEGER freq, diff; + + diff.QuadPart = t->end.QuadPart - t->begin.QuadPart; + + QueryPerformanceFrequency(&freq); + return diff.QuadPart * 1000000 / freq.QuadPart; +#else + struct timeval diff; + + timersub(&t->end, &t->begin, &diff); + return diff.tv_sec * 1000000 + diff.tv_usec; +#endif +} + +#else /* CONFIG_OS_SUPPORT = 0*/ + +/* Empty timer functions if CONFIG_OS_SUPPORT = 0 */ +#ifndef timersub +#define timersub(a, b, result) +#endif + +struct aom_usec_timer { + void *dummy; +}; + +static INLINE void aom_usec_timer_start(struct aom_usec_timer *t) { (void)t; } + +static INLINE void aom_usec_timer_mark(struct aom_usec_timer *t) { (void)t; } + +static INLINE int aom_usec_timer_elapsed(struct aom_usec_timer *t) { + (void)t; + return 0; +} + +#endif /* CONFIG_OS_SUPPORT */ + +#endif // AOM_PORTS_AOM_TIMER_H_ diff --git a/third_party/aom/aom_ports/arm.h b/third_party/aom/aom_ports/arm.h new file mode 100644 index 0000000000..448a70dcc6 --- /dev/null +++ b/third_party/aom/aom_ports/arm.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_PORTS_ARM_H_ +#define AOM_PORTS_ARM_H_ +#include +#include "aom_config.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*ARMv5TE "Enhanced DSP" instructions.*/ +#define HAS_EDSP 0x01 +/*ARMv6 "Parallel" or "Media" instructions.*/ +#define HAS_MEDIA 0x02 +/*ARMv7 optional NEON instructions.*/ +#define HAS_NEON 0x04 + +int arm_cpu_caps(void); + +// Earlier gcc compilers have issues with some neon intrinsics +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ == 4 && \ + __GNUC_MINOR__ <= 6 +#define AOM_INCOMPATIBLE_GCC +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_PORTS_ARM_H_ diff --git a/third_party/aom/aom_ports/arm_cpudetect.c b/third_party/aom/aom_ports/arm_cpudetect.c new file mode 100644 index 0000000000..65ba846c9c --- /dev/null +++ b/third_party/aom/aom_ports/arm_cpudetect.c @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include "aom_ports/arm.h" +#include "./aom_config.h" + +#ifdef WINAPI_FAMILY +#include +#if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +#define getenv(x) NULL +#endif +#endif + +static int arm_cpu_env_flags(int *flags) { + char *env; + env = getenv("AOM_SIMD_CAPS"); + if (env && *env) { + *flags = (int)strtol(env, NULL, 0); + return 0; + } + *flags = 0; + return -1; +} + +static int arm_cpu_env_mask(void) { + char *env; + env = getenv("AOM_SIMD_CAPS_MASK"); + return env && *env ? (int)strtol(env, NULL, 0) : ~0; +} + +#if !CONFIG_RUNTIME_CPU_DETECT + +int arm_cpu_caps(void) { + /* This function should actually be a no-op. There is no way to adjust any of + * these because the RTCD tables do not exist: the functions are called + * statically */ + int flags; + int mask; + if (!arm_cpu_env_flags(&flags)) { + return flags; + } + mask = arm_cpu_env_mask(); +#if HAVE_MEDIA + flags |= HAS_MEDIA; +#endif /* HAVE_MEDIA */ +#if HAVE_NEON || HAVE_NEON_ASM + flags |= HAS_NEON; +#endif /* HAVE_NEON || HAVE_NEON_ASM */ + return flags & mask; +} + +#elif defined(_MSC_VER) /* end !CONFIG_RUNTIME_CPU_DETECT */ +/*For GetExceptionCode() and EXCEPTION_ILLEGAL_INSTRUCTION.*/ +#define WIN32_LEAN_AND_MEAN +#define WIN32_EXTRA_LEAN +#include + +int arm_cpu_caps(void) { + int flags; + int mask; + if (!arm_cpu_env_flags(&flags)) { + return flags; + } + mask = arm_cpu_env_mask(); +/* MSVC has no inline __asm support for ARM, but it does let you __emit + * instructions via their assembled hex code. + * All of these instructions should be essentially nops. + */ +#if HAVE_MEDIA + if (mask & HAS_MEDIA) { + __try { + /*SHADD8 r3,r3,r3*/ + __emit(0xE6333F93); + flags |= HAS_MEDIA; + } __except (GetExceptionCode() == EXCEPTION_ILLEGAL_INSTRUCTION) { + /*Ignore exception.*/ + } + } +#endif /* HAVE_MEDIA */ +#if HAVE_NEON || HAVE_NEON_ASM + if (mask & HAS_NEON) { + __try { + /*VORR q0,q0,q0*/ + __emit(0xF2200150); + flags |= HAS_NEON; + } __except (GetExceptionCode() == EXCEPTION_ILLEGAL_INSTRUCTION) { + /*Ignore exception.*/ + } + } +#endif /* HAVE_NEON || HAVE_NEON_ASM */ + return flags & mask; +} + +#elif defined(__ANDROID__) /* end _MSC_VER */ +#include + +int arm_cpu_caps(void) { + int flags; + int mask; + uint64_t features; + if (!arm_cpu_env_flags(&flags)) { + return flags; + } + mask = arm_cpu_env_mask(); + features = android_getCpuFeatures(); + +#if HAVE_MEDIA + flags |= HAS_MEDIA; +#endif /* HAVE_MEDIA */ +#if HAVE_NEON || HAVE_NEON_ASM + if (features & ANDROID_CPU_ARM_FEATURE_NEON) flags |= HAS_NEON; +#endif /* HAVE_NEON || HAVE_NEON_ASM */ + return flags & mask; +} + +#elif defined(__linux__) /* end __ANDROID__ */ + +#include + +int arm_cpu_caps(void) { + FILE *fin; + int flags; + int mask; + if (!arm_cpu_env_flags(&flags)) { + return flags; + } + mask = arm_cpu_env_mask(); + /* Reading /proc/self/auxv would be easier, but that doesn't work reliably + * on Android. + * This also means that detection will fail in Scratchbox. + */ + fin = fopen("/proc/cpuinfo", "r"); + if (fin != NULL) { + /* 512 should be enough for anybody (it's even enough for all the flags + * that x86 has accumulated... so far). + */ + char buf[512]; + while (fgets(buf, 511, fin) != NULL) { +#if HAVE_NEON || HAVE_NEON_ASM + if (memcmp(buf, "Features", 8) == 0) { + char *p; + p = strstr(buf, " neon"); + if (p != NULL && (p[5] == ' ' || p[5] == '\n')) { + flags |= HAS_NEON; + } + } +#endif /* HAVE_NEON || HAVE_NEON_ASM */ +#if HAVE_MEDIA + if (memcmp(buf, "CPU architecture:", 17) == 0) { + int version; + version = atoi(buf + 17); + if (version >= 6) { + flags |= HAS_MEDIA; + } + } +#endif /* HAVE_MEDIA */ + } + fclose(fin); + } + return flags & mask; +} +#else /* end __linux__ */ +#error \ + "--enable-runtime-cpu-detect selected, but no CPU detection method " \ +"available for your platform. Reconfigure with --disable-runtime-cpu-detect." +#endif diff --git a/third_party/aom/aom_ports/bitops.h b/third_party/aom/aom_ports/bitops.h new file mode 100644 index 0000000000..36f5bd487b --- /dev/null +++ b/third_party/aom/aom_ports/bitops.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_PORTS_BITOPS_H_ +#define AOM_PORTS_BITOPS_H_ + +#include + +#include "aom_ports/msvc.h" + +#ifdef _MSC_VER +#if defined(_M_X64) || defined(_M_IX86) +#include +#define USE_MSC_INTRINSICS +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +// These versions of get_msb() are only valid when n != 0 because all +// of the optimized versions are undefined when n == 0: +// https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html + +// use GNU builtins where available. +#if defined(__GNUC__) && \ + ((__GNUC__ == 3 && __GNUC_MINOR__ >= 4) || __GNUC__ >= 4) +static INLINE int get_msb(unsigned int n) { + assert(n != 0); + return 31 ^ __builtin_clz(n); +} +#elif defined(USE_MSC_INTRINSICS) +#pragma intrinsic(_BitScanReverse) + +static INLINE int get_msb(unsigned int n) { + unsigned long first_set_bit; + assert(n != 0); + _BitScanReverse(&first_set_bit, n); + return first_set_bit; +} +#undef USE_MSC_INTRINSICS +#else +// Returns (int)floor(log2(n)). n must be > 0. +static INLINE int get_msb(unsigned int n) { + int log = 0; + unsigned int value = n; + int i; + + assert(n != 0); + + for (i = 4; i >= 0; --i) { + const int shift = (1 << i); + const unsigned int x = value >> shift; + if (x != 0) { + value = x; + log += shift; + } + } + return log; +} +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_PORTS_BITOPS_H_ diff --git a/third_party/aom/aom_ports/config.h b/third_party/aom/aom_ports/config.h new file mode 100644 index 0000000000..462ec66fb0 --- /dev/null +++ b/third_party/aom/aom_ports/config.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_PORTS_CONFIG_H_ +#define AOM_PORTS_CONFIG_H_ + +#include "aom_config.h" + +#endif // AOM_PORTS_CONFIG_H_ diff --git a/third_party/aom/aom_ports/emmintrin_compat.h b/third_party/aom/aom_ports/emmintrin_compat.h new file mode 100644 index 0000000000..f9d44c647e --- /dev/null +++ b/third_party/aom/aom_ports/emmintrin_compat.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_PORTS_EMMINTRIN_COMPAT_H_ +#define AOM_PORTS_EMMINTRIN_COMPAT_H_ + +#if defined(__GNUC__) && __GNUC__ < 4 +/* From emmintrin.h (gcc 4.5.3) */ +/* Casts between various SP, DP, INT vector types. Note that these do no + conversion of values, they just change the type. */ +extern __inline __m128 + __attribute__((__gnu_inline__, __always_inline__, __artificial__)) + _mm_castpd_ps(__m128d __A) { + return (__m128)__A; +} + +extern __inline __m128i + __attribute__((__gnu_inline__, __always_inline__, __artificial__)) + _mm_castpd_si128(__m128d __A) { + return (__m128i)__A; +} + +extern __inline __m128d + __attribute__((__gnu_inline__, __always_inline__, __artificial__)) + _mm_castps_pd(__m128 __A) { + return (__m128d)__A; +} + +extern __inline __m128i + __attribute__((__gnu_inline__, __always_inline__, __artificial__)) + _mm_castps_si128(__m128 __A) { + return (__m128i)__A; +} + +extern __inline __m128 + __attribute__((__gnu_inline__, __always_inline__, __artificial__)) + _mm_castsi128_ps(__m128i __A) { + return (__m128)__A; +} + +extern __inline __m128d + __attribute__((__gnu_inline__, __always_inline__, __artificial__)) + _mm_castsi128_pd(__m128i __A) { + return (__m128d)__A; +} +#endif + +#endif // AOM_PORTS_EMMINTRIN_COMPAT_H_ diff --git a/third_party/aom/aom_ports/emms.asm b/third_party/aom/aom_ports/emms.asm new file mode 100644 index 0000000000..90776bacb3 --- /dev/null +++ b/third_party/aom/aom_ports/emms.asm @@ -0,0 +1,41 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + +%include "aom_ports/x86_abi_support.asm" + +section .text +global sym(aom_reset_mmx_state) PRIVATE +sym(aom_reset_mmx_state): + emms + ret + + +%if LIBAOM_YASM_WIN64 +global sym(aom_winx64_fldcw) PRIVATE +sym(aom_winx64_fldcw): + sub rsp, 8 + mov [rsp], rcx ; win x64 specific + fldcw [rsp] + add rsp, 8 + ret + + +global sym(aom_winx64_fstcw) PRIVATE +sym(aom_winx64_fstcw): + sub rsp, 8 + fstcw [rsp] + mov rax, [rsp] + add rsp, 8 + ret +%endif diff --git a/third_party/aom/aom_ports/mem.h b/third_party/aom/aom_ports/mem.h new file mode 100644 index 0000000000..bd881cdc8c --- /dev/null +++ b/third_party/aom/aom_ports/mem.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_PORTS_MEM_H_ +#define AOM_PORTS_MEM_H_ + +#include "aom_config.h" +#include "aom/aom_integer.h" + +#if (defined(__GNUC__) && __GNUC__) || defined(__SUNPRO_C) +#define DECLARE_ALIGNED(n, typ, val) typ val __attribute__((aligned(n))) +#elif defined(_MSC_VER) +#define DECLARE_ALIGNED(n, typ, val) __declspec(align(n)) typ val +#else +#warning No alignment directives known for this compiler. +#define DECLARE_ALIGNED(n, typ, val) typ val +#endif + +/* Indicates that the usage of the specified variable has been audited to assure + * that it's safe to use uninitialized. Silences 'may be used uninitialized' + * warnings on gcc. + */ +#if defined(__GNUC__) && __GNUC__ +#define UNINITIALIZED_IS_SAFE(x) x = x +#else +#define UNINITIALIZED_IS_SAFE(x) x +#endif + +#if HAVE_NEON && defined(_MSC_VER) +#define __builtin_prefetch(x) +#endif + +/* Shift down with rounding for use when n >= 0, value >= 0 */ +#define ROUND_POWER_OF_TWO(value, n) (((value) + (((1 << (n)) >> 1))) >> (n)) + +/* Shift down with rounding for signed integers, for use when n >= 0 */ +#define ROUND_POWER_OF_TWO_SIGNED(value, n) \ + (((value) < 0) ? -ROUND_POWER_OF_TWO(-(value), (n)) \ + : ROUND_POWER_OF_TWO((value), (n))) + +/* Shift down with rounding for use when n >= 0, value >= 0 for (64 bit) */ +#define ROUND_POWER_OF_TWO_64(value, n) \ + (((value) + ((((int64_t)1 << (n)) >> 1))) >> (n)) +/* Shift down with rounding for signed integers, for use when n >= 0 (64 bit) */ +#define ROUND_POWER_OF_TWO_SIGNED_64(value, n) \ + (((value) < 0) ? -ROUND_POWER_OF_TWO_64(-(value), (n)) \ + : ROUND_POWER_OF_TWO_64((value), (n))) + +#define ALIGN_POWER_OF_TWO(value, n) \ + (((value) + ((1 << (n)) - 1)) & ~((1 << (n)) - 1)) + +#define CONVERT_TO_SHORTPTR(x) ((uint16_t *)(((uintptr_t)(x)) << 1)) +#if CONFIG_HIGHBITDEPTH +#define CONVERT_TO_BYTEPTR(x) ((uint8_t *)(((uintptr_t)(x)) >> 1)) +#endif // CONFIG_HIGHBITDEPTH + +#endif // AOM_PORTS_MEM_H_ diff --git a/third_party/aom/aom_ports/mem_ops.h b/third_party/aom/aom_ports/mem_ops.h new file mode 100644 index 0000000000..62126755c8 --- /dev/null +++ b/third_party/aom/aom_ports/mem_ops.h @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_PORTS_MEM_OPS_H_ +#define AOM_PORTS_MEM_OPS_H_ + +/* \file + * \brief Provides portable memory access primitives + * + * This function provides portable primitives for getting and setting of + * signed and unsigned integers in 16, 24, and 32 bit sizes. The operations + * can be performed on unaligned data regardless of hardware support for + * unaligned accesses. + * + * The type used to pass the integral values may be changed by defining + * MEM_VALUE_T with the appropriate type. The type given must be an integral + * numeric type. + * + * The actual functions instantiated have the MEM_VALUE_T type name pasted + * on to the symbol name. This allows the developer to instantiate these + * operations for multiple types within the same translation unit. This is + * of somewhat questionable utility, but the capability exists nonetheless. + * Users not making use of this functionality should call the functions + * without the type name appended, and the preprocessor will take care of + * it. + * + * NOTE: This code is not supported on platforms where char > 1 octet ATM. + */ + +#ifndef MAU_T +/* Minimum Access Unit for this target */ +#define MAU_T unsigned char +#endif + +#ifndef MEM_VALUE_T +#define MEM_VALUE_T int +#endif + +#undef MEM_VALUE_T_SZ_BITS +#define MEM_VALUE_T_SZ_BITS (sizeof(MEM_VALUE_T) << 3) + +#undef mem_ops_wrap_symbol +#define mem_ops_wrap_symbol(fn) mem_ops_wrap_symbol2(fn, MEM_VALUE_T) +#undef mem_ops_wrap_symbol2 +#define mem_ops_wrap_symbol2(fn, typ) mem_ops_wrap_symbol3(fn, typ) +#undef mem_ops_wrap_symbol3 +#define mem_ops_wrap_symbol3(fn, typ) fn##_as_##typ + +/* + * Include aligned access routines + */ +#define INCLUDED_BY_MEM_OPS_H +#include "mem_ops_aligned.h" +#undef INCLUDED_BY_MEM_OPS_H + +#undef mem_get_be16 +#define mem_get_be16 mem_ops_wrap_symbol(mem_get_be16) +static unsigned MEM_VALUE_T mem_get_be16(const void *vmem) { + unsigned MEM_VALUE_T val; + const MAU_T *mem = (const MAU_T *)vmem; + + val = mem[0] << 8; + val |= mem[1]; + return val; +} + +#undef mem_get_be24 +#define mem_get_be24 mem_ops_wrap_symbol(mem_get_be24) +static unsigned MEM_VALUE_T mem_get_be24(const void *vmem) { + unsigned MEM_VALUE_T val; + const MAU_T *mem = (const MAU_T *)vmem; + + val = mem[0] << 16; + val |= mem[1] << 8; + val |= mem[2]; + return val; +} + +#undef mem_get_be32 +#define mem_get_be32 mem_ops_wrap_symbol(mem_get_be32) +static unsigned MEM_VALUE_T mem_get_be32(const void *vmem) { + unsigned MEM_VALUE_T val; + const MAU_T *mem = (const MAU_T *)vmem; + + val = ((unsigned MEM_VALUE_T)mem[0]) << 24; + val |= mem[1] << 16; + val |= mem[2] << 8; + val |= mem[3]; + return val; +} + +#undef mem_get_le16 +#define mem_get_le16 mem_ops_wrap_symbol(mem_get_le16) +static unsigned MEM_VALUE_T mem_get_le16(const void *vmem) { + unsigned MEM_VALUE_T val; + const MAU_T *mem = (const MAU_T *)vmem; + + val = mem[1] << 8; + val |= mem[0]; + return val; +} + +#undef mem_get_le24 +#define mem_get_le24 mem_ops_wrap_symbol(mem_get_le24) +static unsigned MEM_VALUE_T mem_get_le24(const void *vmem) { + unsigned MEM_VALUE_T val; + const MAU_T *mem = (const MAU_T *)vmem; + + val = mem[2] << 16; + val |= mem[1] << 8; + val |= mem[0]; + return val; +} + +#undef mem_get_le32 +#define mem_get_le32 mem_ops_wrap_symbol(mem_get_le32) +static unsigned MEM_VALUE_T mem_get_le32(const void *vmem) { + unsigned MEM_VALUE_T val; + const MAU_T *mem = (const MAU_T *)vmem; + + val = ((unsigned MEM_VALUE_T)mem[3]) << 24; + val |= mem[2] << 16; + val |= mem[1] << 8; + val |= mem[0]; + return val; +} + +#define mem_get_s_generic(end, sz) \ + static AOM_INLINE signed MEM_VALUE_T mem_get_s##end##sz(const void *vmem) { \ + const MAU_T *mem = (const MAU_T *)vmem; \ + signed MEM_VALUE_T val = mem_get_##end##sz(mem); \ + return (val << (MEM_VALUE_T_SZ_BITS - sz)) >> (MEM_VALUE_T_SZ_BITS - sz); \ + } + +/* clang-format off */ +#undef mem_get_sbe16 +#define mem_get_sbe16 mem_ops_wrap_symbol(mem_get_sbe16) +mem_get_s_generic(be, 16) + +#undef mem_get_sbe24 +#define mem_get_sbe24 mem_ops_wrap_symbol(mem_get_sbe24) +mem_get_s_generic(be, 24) + +#undef mem_get_sbe32 +#define mem_get_sbe32 mem_ops_wrap_symbol(mem_get_sbe32) +mem_get_s_generic(be, 32) + +#undef mem_get_sle16 +#define mem_get_sle16 mem_ops_wrap_symbol(mem_get_sle16) +mem_get_s_generic(le, 16) + +#undef mem_get_sle24 +#define mem_get_sle24 mem_ops_wrap_symbol(mem_get_sle24) +mem_get_s_generic(le, 24) + +#undef mem_get_sle32 +#define mem_get_sle32 mem_ops_wrap_symbol(mem_get_sle32) +mem_get_s_generic(le, 32) + +#undef mem_put_be16 +#define mem_put_be16 mem_ops_wrap_symbol(mem_put_be16) +static AOM_INLINE void mem_put_be16(void *vmem, MEM_VALUE_T val) { + MAU_T *mem = (MAU_T *)vmem; + + mem[0] = (MAU_T)((val >> 8) & 0xff); + mem[1] = (MAU_T)((val >> 0) & 0xff); +} + +#undef mem_put_be24 +#define mem_put_be24 mem_ops_wrap_symbol(mem_put_be24) +static AOM_INLINE void mem_put_be24(void *vmem, MEM_VALUE_T val) { + MAU_T *mem = (MAU_T *)vmem; + + mem[0] = (MAU_T)((val >> 16) & 0xff); + mem[1] = (MAU_T)((val >> 8) & 0xff); + mem[2] = (MAU_T)((val >> 0) & 0xff); +} + +#undef mem_put_be32 +#define mem_put_be32 mem_ops_wrap_symbol(mem_put_be32) +static AOM_INLINE void mem_put_be32(void *vmem, MEM_VALUE_T val) { + MAU_T *mem = (MAU_T *)vmem; + + mem[0] = (MAU_T)((val >> 24) & 0xff); + mem[1] = (MAU_T)((val >> 16) & 0xff); + mem[2] = (MAU_T)((val >> 8) & 0xff); + mem[3] = (MAU_T)((val >> 0) & 0xff); +} + +#undef mem_put_le16 +#define mem_put_le16 mem_ops_wrap_symbol(mem_put_le16) +static AOM_INLINE void mem_put_le16(void *vmem, MEM_VALUE_T val) { + MAU_T *mem = (MAU_T *)vmem; + + mem[0] = (MAU_T)((val >> 0) & 0xff); + mem[1] = (MAU_T)((val >> 8) & 0xff); +} + +#undef mem_put_le24 +#define mem_put_le24 mem_ops_wrap_symbol(mem_put_le24) +static AOM_INLINE void mem_put_le24(void *vmem, MEM_VALUE_T val) { + MAU_T *mem = (MAU_T *)vmem; + + mem[0] = (MAU_T)((val >> 0) & 0xff); + mem[1] = (MAU_T)((val >> 8) & 0xff); + mem[2] = (MAU_T)((val >> 16) & 0xff); +} + +#undef mem_put_le32 +#define mem_put_le32 mem_ops_wrap_symbol(mem_put_le32) +static AOM_INLINE void mem_put_le32(void *vmem, MEM_VALUE_T val) { + MAU_T *mem = (MAU_T *)vmem; + + mem[0] = (MAU_T)((val >> 0) & 0xff); + mem[1] = (MAU_T)((val >> 8) & 0xff); + mem[2] = (MAU_T)((val >> 16) & 0xff); + mem[3] = (MAU_T)((val >> 24) & 0xff); +} +/* clang-format on */ + +#endif // AOM_PORTS_MEM_OPS_H_ diff --git a/third_party/aom/aom_ports/mem_ops_aligned.h b/third_party/aom/aom_ports/mem_ops_aligned.h new file mode 100644 index 0000000000..8c3ab1cb1a --- /dev/null +++ b/third_party/aom/aom_ports/mem_ops_aligned.h @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_PORTS_MEM_OPS_ALIGNED_H_ +#define AOM_PORTS_MEM_OPS_ALIGNED_H_ + +#include "aom/aom_integer.h" + +/* \file + * \brief Provides portable memory access primitives for operating on aligned + * data + * + * This file is split from mem_ops.h for easier maintenance. See mem_ops.h + * for a more detailed description of these primitives. + */ +#ifndef INCLUDED_BY_MEM_OPS_H +#error Include mem_ops.h, not mem_ops_aligned.h directly. +#endif + +/* Architectures that provide instructions for doing this byte swapping + * could redefine these macros. + */ +#define swap_endian_16(val, raw) \ + do { \ + val = (uint16_t)(((raw >> 8) & 0x00ff) | ((raw << 8) & 0xff00)); \ + } while (0) +#define swap_endian_32(val, raw) \ + do { \ + val = ((raw >> 24) & 0x000000ff) | ((raw >> 8) & 0x0000ff00) | \ + ((raw << 8) & 0x00ff0000) | ((raw << 24) & 0xff000000); \ + } while (0) +#define swap_endian_16_se(val, raw) \ + do { \ + swap_endian_16(val, raw); \ + val = ((val << 16) >> 16); \ + } while (0) +#define swap_endian_32_se(val, raw) swap_endian_32(val, raw) + +#define mem_get_ne_aligned_generic(end, sz) \ + static AOM_INLINE unsigned MEM_VALUE_T mem_get_##end##sz##_aligned( \ + const void *vmem) { \ + const uint##sz##_t *mem = (const uint##sz##_t *)vmem; \ + return *mem; \ + } + +#define mem_get_sne_aligned_generic(end, sz) \ + static AOM_INLINE signed MEM_VALUE_T mem_get_s##end##sz##_aligned( \ + const void *vmem) { \ + const int##sz##_t *mem = (const int##sz##_t *)vmem; \ + return *mem; \ + } + +#define mem_get_se_aligned_generic(end, sz) \ + static AOM_INLINE unsigned MEM_VALUE_T mem_get_##end##sz##_aligned( \ + const void *vmem) { \ + const uint##sz##_t *mem = (const uint##sz##_t *)vmem; \ + unsigned MEM_VALUE_T val, raw = *mem; \ + swap_endian_##sz(val, raw); \ + return val; \ + } + +#define mem_get_sse_aligned_generic(end, sz) \ + static AOM_INLINE signed MEM_VALUE_T mem_get_s##end##sz##_aligned( \ + const void *vmem) { \ + const int##sz##_t *mem = (const int##sz##_t *)vmem; \ + unsigned MEM_VALUE_T val, raw = *mem; \ + swap_endian_##sz##_se(val, raw); \ + return val; \ + } + +#define mem_put_ne_aligned_generic(end, sz) \ + static AOM_INLINE void mem_put_##end##sz##_aligned(void *vmem, \ + MEM_VALUE_T val) { \ + uint##sz##_t *mem = (uint##sz##_t *)vmem; \ + *mem = (uint##sz##_t)val; \ + } + +#define mem_put_se_aligned_generic(end, sz) \ + static AOM_INLINE void mem_put_##end##sz##_aligned(void *vmem, \ + MEM_VALUE_T val) { \ + uint##sz##_t *mem = (uint##sz##_t *)vmem, raw; \ + swap_endian_##sz(raw, val); \ + *mem = (uint##sz##_t)raw; \ + } + +#include "aom_config.h" +#if CONFIG_BIG_ENDIAN +#define mem_get_be_aligned_generic(sz) mem_get_ne_aligned_generic(be, sz) +#define mem_get_sbe_aligned_generic(sz) mem_get_sne_aligned_generic(be, sz) +#define mem_get_le_aligned_generic(sz) mem_get_se_aligned_generic(le, sz) +#define mem_get_sle_aligned_generic(sz) mem_get_sse_aligned_generic(le, sz) +#define mem_put_be_aligned_generic(sz) mem_put_ne_aligned_generic(be, sz) +#define mem_put_le_aligned_generic(sz) mem_put_se_aligned_generic(le, sz) +#else +#define mem_get_be_aligned_generic(sz) mem_get_se_aligned_generic(be, sz) +#define mem_get_sbe_aligned_generic(sz) mem_get_sse_aligned_generic(be, sz) +#define mem_get_le_aligned_generic(sz) mem_get_ne_aligned_generic(le, sz) +#define mem_get_sle_aligned_generic(sz) mem_get_sne_aligned_generic(le, sz) +#define mem_put_be_aligned_generic(sz) mem_put_se_aligned_generic(be, sz) +#define mem_put_le_aligned_generic(sz) mem_put_ne_aligned_generic(le, sz) +#endif + +/* clang-format off */ +#undef mem_get_be16_aligned +#define mem_get_be16_aligned mem_ops_wrap_symbol(mem_get_be16_aligned) +mem_get_be_aligned_generic(16) + +#undef mem_get_be32_aligned +#define mem_get_be32_aligned mem_ops_wrap_symbol(mem_get_be32_aligned) +mem_get_be_aligned_generic(32) + +#undef mem_get_le16_aligned +#define mem_get_le16_aligned mem_ops_wrap_symbol(mem_get_le16_aligned) +mem_get_le_aligned_generic(16) + +#undef mem_get_le32_aligned +#define mem_get_le32_aligned mem_ops_wrap_symbol(mem_get_le32_aligned) +mem_get_le_aligned_generic(32) + +#undef mem_get_sbe16_aligned +#define mem_get_sbe16_aligned mem_ops_wrap_symbol(mem_get_sbe16_aligned) +mem_get_sbe_aligned_generic(16) + +#undef mem_get_sbe32_aligned +#define mem_get_sbe32_aligned mem_ops_wrap_symbol(mem_get_sbe32_aligned) +mem_get_sbe_aligned_generic(32) + +#undef mem_get_sle16_aligned +#define mem_get_sle16_aligned mem_ops_wrap_symbol(mem_get_sle16_aligned) +mem_get_sle_aligned_generic(16) + +#undef mem_get_sle32_aligned +#define mem_get_sle32_aligned mem_ops_wrap_symbol(mem_get_sle32_aligned) +mem_get_sle_aligned_generic(32) + +#undef mem_put_be16_aligned +#define mem_put_be16_aligned mem_ops_wrap_symbol(mem_put_be16_aligned) +mem_put_be_aligned_generic(16) + +#undef mem_put_be32_aligned +#define mem_put_be32_aligned mem_ops_wrap_symbol(mem_put_be32_aligned) +mem_put_be_aligned_generic(32) + +#undef mem_put_le16_aligned +#define mem_put_le16_aligned mem_ops_wrap_symbol(mem_put_le16_aligned) +mem_put_le_aligned_generic(16) + +#undef mem_put_le32_aligned +#define mem_put_le32_aligned mem_ops_wrap_symbol(mem_put_le32_aligned) +mem_put_le_aligned_generic(32) + +#undef mem_get_ne_aligned_generic +#undef mem_get_se_aligned_generic +#undef mem_get_sne_aligned_generic +#undef mem_get_sse_aligned_generic +#undef mem_put_ne_aligned_generic +#undef mem_put_se_aligned_generic +#undef swap_endian_16 +#undef swap_endian_32 +#undef swap_endian_16_se +#undef swap_endian_32_se +/* clang-format on */ + +#endif // AOM_PORTS_MEM_OPS_ALIGNED_H_ diff --git a/third_party/aom/aom_ports/msvc.h b/third_party/aom/aom_ports/msvc.h new file mode 100644 index 0000000000..2d3ab9b653 --- /dev/null +++ b/third_party/aom/aom_ports/msvc.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_PORTS_MSVC_H_ +#define AOM_PORTS_MSVC_H_ +#ifdef _MSC_VER + +#include "./aom_config.h" + +#if _MSC_VER < 1900 // VS2015 provides snprintf +#define snprintf _snprintf +#endif // _MSC_VER < 1900 + +#if _MSC_VER < 1800 // VS2013 provides round +#include +static INLINE double round(double x) { + if (x < 0) + return ceil(x - 0.5); + else + return floor(x + 0.5); +} + +static INLINE float roundf(float x) { + if (x < 0) + return (float)ceil(x - 0.5f); + else + return (float)floor(x + 0.5f); +} + +static INLINE long lroundf(float x) { + if (x < 0) + return (long)(x - 0.5f); + else + return (long)(x + 0.5f); +} +#endif // _MSC_VER < 1800 + +#endif // _MSC_VER +#endif // AOM_PORTS_MSVC_H_ diff --git a/third_party/aom/aom_ports/system_state.h b/third_party/aom/aom_ports/system_state.h new file mode 100644 index 0000000000..5d40d4cb82 --- /dev/null +++ b/third_party/aom/aom_ports/system_state.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_PORTS_SYSTEM_STATE_H_ +#define AOM_PORTS_SYSTEM_STATE_H_ + +#include "./aom_config.h" + +#if ARCH_X86 || ARCH_X86_64 +void aom_reset_mmx_state(void); +#define aom_clear_system_state() aom_reset_mmx_state() +#else +#define aom_clear_system_state() +#endif // ARCH_X86 || ARCH_X86_64 +#endif // AOM_PORTS_SYSTEM_STATE_H_ diff --git a/third_party/aom/aom_ports/x86.h b/third_party/aom/aom_ports/x86.h new file mode 100644 index 0000000000..e5680ca934 --- /dev/null +++ b/third_party/aom/aom_ports/x86.h @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_PORTS_X86_H_ +#define AOM_PORTS_X86_H_ +#include + +#if defined(_MSC_VER) +#include /* For __cpuidex, __rdtsc */ +#endif + +#include "aom_config.h" +#include "aom/aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + AOM_CPU_UNKNOWN = -1, + AOM_CPU_AMD, + AOM_CPU_AMD_OLD, + AOM_CPU_CENTAUR, + AOM_CPU_CYRIX, + AOM_CPU_INTEL, + AOM_CPU_NEXGEN, + AOM_CPU_NSC, + AOM_CPU_RISE, + AOM_CPU_SIS, + AOM_CPU_TRANSMETA, + AOM_CPU_TRANSMETA_OLD, + AOM_CPU_UMC, + AOM_CPU_VIA, + + AOM_CPU_LAST +} aom_cpu_t; + +#if defined(__GNUC__) && __GNUC__ || defined(__ANDROID__) +#if ARCH_X86_64 +#define cpuid(func, func2, ax, bx, cx, dx) \ + __asm__ __volatile__("cpuid \n\t" \ + : "=a"(ax), "=b"(bx), "=c"(cx), "=d"(dx) \ + : "a"(func), "c"(func2)); +#else +#define cpuid(func, func2, ax, bx, cx, dx) \ + __asm__ __volatile__( \ + "mov %%ebx, %%edi \n\t" \ + "cpuid \n\t" \ + "xchg %%edi, %%ebx \n\t" \ + : "=a"(ax), "=D"(bx), "=c"(cx), "=d"(dx) \ + : "a"(func), "c"(func2)); +#endif +#elif defined(__SUNPRO_C) || \ + defined(__SUNPRO_CC) /* end __GNUC__ or __ANDROID__*/ +#if ARCH_X86_64 +#define cpuid(func, func2, ax, bx, cx, dx) \ + asm volatile( \ + "xchg %rsi, %rbx \n\t" \ + "cpuid \n\t" \ + "movl %ebx, %edi \n\t" \ + "xchg %rsi, %rbx \n\t" \ + : "=a"(ax), "=D"(bx), "=c"(cx), "=d"(dx) \ + : "a"(func), "c"(func2)); +#else +#define cpuid(func, func2, ax, bx, cx, dx) \ + asm volatile( \ + "pushl %ebx \n\t" \ + "cpuid \n\t" \ + "movl %ebx, %edi \n\t" \ + "popl %ebx \n\t" \ + : "=a"(ax), "=D"(bx), "=c"(cx), "=d"(dx) \ + : "a"(func), "c"(func2)); +#endif +#else /* end __SUNPRO__ */ +#if ARCH_X86_64 +#if defined(_MSC_VER) && _MSC_VER > 1500 +#define cpuid(func, func2, a, b, c, d) \ + do { \ + int regs[4]; \ + __cpuidex(regs, func, func2); \ + a = regs[0]; \ + b = regs[1]; \ + c = regs[2]; \ + d = regs[3]; \ + } while (0) +#else +#define cpuid(func, func2, a, b, c, d) \ + do { \ + int regs[4]; \ + __cpuid(regs, func); \ + a = regs[0]; \ + b = regs[1]; \ + c = regs[2]; \ + d = regs[3]; \ + } while (0) +#endif +#else +/* clang-format off */ +#define cpuid(func, func2, a, b, c, d) \ + __asm mov eax, func \ + __asm mov ecx, func2 \ + __asm cpuid \ + __asm mov a, eax \ + __asm mov b, ebx \ + __asm mov c, ecx \ + __asm mov d, edx +#endif +/* clang-format on */ +#endif /* end others */ + +// NaCl has no support for xgetbv or the raw opcode. +#if !defined(__native_client__) && (defined(__i386__) || defined(__x86_64__)) +static INLINE uint64_t xgetbv(void) { + const uint32_t ecx = 0; + uint32_t eax, edx; + // Use the raw opcode for xgetbv for compatibility with older toolchains. + __asm__ volatile(".byte 0x0f, 0x01, 0xd0\n" + : "=a"(eax), "=d"(edx) + : "c"(ecx)); + return ((uint64_t)edx << 32) | eax; +} +#elif (defined(_M_X64) || defined(_M_IX86)) && defined(_MSC_FULL_VER) && \ + _MSC_FULL_VER >= 160040219 // >= VS2010 SP1 +#include +#define xgetbv() _xgetbv(0) +#elif defined(_MSC_VER) && defined(_M_IX86) +static INLINE uint64_t xgetbv(void) { + uint32_t eax_, edx_; + __asm { + xor ecx, ecx // ecx = 0 + // Use the raw opcode for xgetbv for compatibility with older toolchains. + __asm _emit 0x0f __asm _emit 0x01 __asm _emit 0xd0 + mov eax_, eax + mov edx_, edx + } + return ((uint64_t)edx_ << 32) | eax_; +} +#else +#define xgetbv() 0U // no AVX for older x64 or unrecognized toolchains. +#endif + +#if defined(_MSC_VER) && _MSC_VER >= 1700 +#include +#if WINAPI_FAMILY_PARTITION(WINAPI_FAMILY_APP) +#define getenv(x) NULL +#endif +#endif + +#define HAS_MMX 0x01 +#define HAS_SSE 0x02 +#define HAS_SSE2 0x04 +#define HAS_SSE3 0x08 +#define HAS_SSSE3 0x10 +#define HAS_SSE4_1 0x20 +#define HAS_AVX 0x40 +#define HAS_AVX2 0x80 +#ifndef BIT +#define BIT(n) (1 << n) +#endif + +static INLINE int x86_simd_caps(void) { + unsigned int flags = 0; + unsigned int mask = ~0; + unsigned int max_cpuid_val, reg_eax, reg_ebx, reg_ecx, reg_edx; + char *env; + (void)reg_ebx; + + /* See if the CPU capabilities are being overridden by the environment */ + env = getenv("AOM_SIMD_CAPS"); + + if (env && *env) return (int)strtol(env, NULL, 0); + + env = getenv("AOM_SIMD_CAPS_MASK"); + + if (env && *env) mask = (unsigned int)strtoul(env, NULL, 0); + + /* Ensure that the CPUID instruction supports extended features */ + cpuid(0, 0, max_cpuid_val, reg_ebx, reg_ecx, reg_edx); + + if (max_cpuid_val < 1) return 0; + + /* Get the standard feature flags */ + cpuid(1, 0, reg_eax, reg_ebx, reg_ecx, reg_edx); + + if (reg_edx & BIT(23)) flags |= HAS_MMX; + + if (reg_edx & BIT(25)) flags |= HAS_SSE; /* aka xmm */ + + if (reg_edx & BIT(26)) flags |= HAS_SSE2; /* aka wmt */ + + if (reg_ecx & BIT(0)) flags |= HAS_SSE3; + + if (reg_ecx & BIT(9)) flags |= HAS_SSSE3; + + if (reg_ecx & BIT(19)) flags |= HAS_SSE4_1; + + // bits 27 (OSXSAVE) & 28 (256-bit AVX) + if ((reg_ecx & (BIT(27) | BIT(28))) == (BIT(27) | BIT(28))) { + if ((xgetbv() & 0x6) == 0x6) { + flags |= HAS_AVX; + + if (max_cpuid_val >= 7) { + /* Get the leaf 7 feature flags. Needed to check for AVX2 support */ + cpuid(7, 0, reg_eax, reg_ebx, reg_ecx, reg_edx); + + if (reg_ebx & BIT(5)) flags |= HAS_AVX2; + } + } + } + + return flags & mask; +} + +// Note: +// 32-bit CPU cycle counter is light-weighted for most function performance +// measurement. For large function (CPU time > a couple of seconds), 64-bit +// counter should be used. +// 32-bit CPU cycle counter +static INLINE unsigned int x86_readtsc(void) { +#if defined(__GNUC__) && __GNUC__ + unsigned int tsc; + __asm__ __volatile__("rdtsc\n\t" : "=a"(tsc) :); + return tsc; +#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) + unsigned int tsc; + asm volatile("rdtsc\n\t" : "=a"(tsc) :); + return tsc; +#else +#if ARCH_X86_64 + return (unsigned int)__rdtsc(); +#else + __asm rdtsc; +#endif +#endif +} +// 64-bit CPU cycle counter +static INLINE uint64_t x86_readtsc64(void) { +#if defined(__GNUC__) && __GNUC__ + uint32_t hi, lo; + __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); + return ((uint64_t)hi << 32) | lo; +#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) + uint_t hi, lo; + asm volatile("rdtsc\n\t" : "=a"(lo), "=d"(hi)); + return ((uint64_t)hi << 32) | lo; +#else +#if ARCH_X86_64 + return (uint64_t)__rdtsc(); +#else + __asm rdtsc; +#endif +#endif +} + +#if defined(__GNUC__) && __GNUC__ +#define x86_pause_hint() __asm__ __volatile__("pause \n\t") +#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) +#define x86_pause_hint() asm volatile("pause \n\t") +#else +#if ARCH_X86_64 +#define x86_pause_hint() _mm_pause(); +#else +#define x86_pause_hint() __asm pause +#endif +#endif + +#if defined(__GNUC__) && __GNUC__ +static void x87_set_control_word(unsigned short mode) { + __asm__ __volatile__("fldcw %0" : : "m"(*&mode)); +} +static unsigned short x87_get_control_word(void) { + unsigned short mode; + __asm__ __volatile__("fstcw %0\n\t" : "=m"(*&mode) :); + return mode; +} +#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) +static void x87_set_control_word(unsigned short mode) { + asm volatile("fldcw %0" : : "m"(*&mode)); +} +static unsigned short x87_get_control_word(void) { + unsigned short mode; + asm volatile("fstcw %0\n\t" : "=m"(*&mode) :); + return mode; +} +#elif ARCH_X86_64 +/* No fldcw intrinsics on Windows x64, punt to external asm */ +extern void aom_winx64_fldcw(unsigned short mode); +extern unsigned short aom_winx64_fstcw(void); +#define x87_set_control_word aom_winx64_fldcw +#define x87_get_control_word aom_winx64_fstcw +#else +static void x87_set_control_word(unsigned short mode) { + __asm { fldcw mode } +} +static unsigned short x87_get_control_word(void) { + unsigned short mode; + __asm { fstcw mode } + return mode; +} +#endif + +static INLINE unsigned int x87_set_double_precision(void) { + unsigned int mode = x87_get_control_word(); + x87_set_control_word((mode & ~0x300) | 0x200); + return mode; +} + +extern void aom_reset_mmx_state(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_PORTS_X86_H_ diff --git a/third_party/aom/aom_ports/x86_abi_support.asm b/third_party/aom/aom_ports/x86_abi_support.asm new file mode 100644 index 0000000000..6aeee60a06 --- /dev/null +++ b/third_party/aom/aom_ports/x86_abi_support.asm @@ -0,0 +1,395 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + +%include "aom_config.asm" + +; 32/64 bit compatibility macros +; +; In general, we make the source use 64 bit syntax, then twiddle with it using +; the preprocessor to get the 32 bit syntax on 32 bit platforms. +; +%ifidn __OUTPUT_FORMAT__,elf32 +%define ABI_IS_32BIT 1 +%elifidn __OUTPUT_FORMAT__,macho32 +%define ABI_IS_32BIT 1 +%elifidn __OUTPUT_FORMAT__,win32 +%define ABI_IS_32BIT 1 +%elifidn __OUTPUT_FORMAT__,aout +%define ABI_IS_32BIT 1 +%else +%define ABI_IS_32BIT 0 +%endif + +%if ABI_IS_32BIT +%define rax eax +%define rbx ebx +%define rcx ecx +%define rdx edx +%define rsi esi +%define rdi edi +%define rsp esp +%define rbp ebp +%define movsxd mov +%macro movq 2 + %ifidn %1,eax + movd %1,%2 + %elifidn %2,eax + movd %1,%2 + %elifidn %1,ebx + movd %1,%2 + %elifidn %2,ebx + movd %1,%2 + %elifidn %1,ecx + movd %1,%2 + %elifidn %2,ecx + movd %1,%2 + %elifidn %1,edx + movd %1,%2 + %elifidn %2,edx + movd %1,%2 + %elifidn %1,esi + movd %1,%2 + %elifidn %2,esi + movd %1,%2 + %elifidn %1,edi + movd %1,%2 + %elifidn %2,edi + movd %1,%2 + %elifidn %1,esp + movd %1,%2 + %elifidn %2,esp + movd %1,%2 + %elifidn %1,ebp + movd %1,%2 + %elifidn %2,ebp + movd %1,%2 + %else + movq %1,%2 + %endif +%endmacro +%endif + + +; LIBAOM_YASM_WIN64 +; Set LIBAOM_YASM_WIN64 if output is Windows 64bit so the code will work if x64 +; or win64 is defined on the Yasm command line. +%ifidn __OUTPUT_FORMAT__,win64 +%define LIBAOM_YASM_WIN64 1 +%elifidn __OUTPUT_FORMAT__,x64 +%define LIBAOM_YASM_WIN64 1 +%else +%define LIBAOM_YASM_WIN64 0 +%endif + +; sym() +; Return the proper symbol name for the target ABI. +; +; Certain ABIs, notably MS COFF and Darwin MACH-O, require that symbols +; with C linkage be prefixed with an underscore. +; +%ifidn __OUTPUT_FORMAT__,elf32 +%define sym(x) x +%elifidn __OUTPUT_FORMAT__,elf64 +%define sym(x) x +%elifidn __OUTPUT_FORMAT__,elfx32 +%define sym(x) x +%elif LIBAOM_YASM_WIN64 +%define sym(x) x +%else +%define sym(x) _ %+ x +%endif + +; PRIVATE +; Macro for the attribute to hide a global symbol for the target ABI. +; This is only active if CHROMIUM is defined. +; +; Chromium doesn't like exported global symbols due to symbol clashing with +; plugins among other things. +; +; Requires Chromium's patched copy of yasm: +; http://src.chromium.org/viewvc/chrome?view=rev&revision=73761 +; http://www.tortall.net/projects/yasm/ticket/236 +; +%ifdef CHROMIUM + %ifidn __OUTPUT_FORMAT__,elf32 + %define PRIVATE :hidden + %elifidn __OUTPUT_FORMAT__,elf64 + %define PRIVATE :hidden + %elifidn __OUTPUT_FORMAT__,elfx32 + %define PRIVATE :hidden + %elif LIBAOM_YASM_WIN64 + %define PRIVATE + %else + %define PRIVATE :private_extern + %endif +%else + %define PRIVATE +%endif + +; arg() +; Return the address specification of the given argument +; +%if ABI_IS_32BIT + %define arg(x) [ebp+8+4*x] +%else + ; 64 bit ABI passes arguments in registers. This is a workaround to get up + ; and running quickly. Relies on SHADOW_ARGS_TO_STACK + %if LIBAOM_YASM_WIN64 + %define arg(x) [rbp+16+8*x] + %else + %define arg(x) [rbp-8-8*x] + %endif +%endif + +; REG_SZ_BYTES, REG_SZ_BITS +; Size of a register +%if ABI_IS_32BIT +%define REG_SZ_BYTES 4 +%define REG_SZ_BITS 32 +%else +%define REG_SZ_BYTES 8 +%define REG_SZ_BITS 64 +%endif + + +; ALIGN_STACK +; This macro aligns the stack to the given alignment (in bytes). The stack +; is left such that the previous value of the stack pointer is the first +; argument on the stack (ie, the inverse of this macro is 'pop rsp.') +; This macro uses one temporary register, which is not preserved, and thus +; must be specified as an argument. +%macro ALIGN_STACK 2 + mov %2, rsp + and rsp, -%1 + lea rsp, [rsp - (%1 - REG_SZ_BYTES)] + push %2 +%endmacro + + +; +; The Microsoft assembler tries to impose a certain amount of type safety in +; its register usage. YASM doesn't recognize these directives, so we just +; %define them away to maintain as much compatibility as possible with the +; original inline assembler we're porting from. +; +%idefine PTR +%idefine XMMWORD +%idefine MMWORD + +; PIC macros +; +%if ABI_IS_32BIT + %if CONFIG_PIC=1 + %ifidn __OUTPUT_FORMAT__,elf32 + %define WRT_PLT wrt ..plt + %macro GET_GOT 1 + extern _GLOBAL_OFFSET_TABLE_ + push %1 + call %%get_got + %%sub_offset: + jmp %%exitGG + %%get_got: + mov %1, [esp] + add %1, _GLOBAL_OFFSET_TABLE_ + $$ - %%sub_offset wrt ..gotpc + ret + %%exitGG: + %undef GLOBAL + %define GLOBAL(x) x + %1 wrt ..gotoff + %undef RESTORE_GOT + %define RESTORE_GOT pop %1 + %endmacro + %elifidn __OUTPUT_FORMAT__,macho32 + %macro GET_GOT 1 + push %1 + call %%get_got + %%get_got: + pop %1 + %undef GLOBAL + %define GLOBAL(x) x + %1 - %%get_got + %undef RESTORE_GOT + %define RESTORE_GOT pop %1 + %endmacro + %endif + %endif + + %ifdef CHROMIUM + %ifidn __OUTPUT_FORMAT__,macho32 + %define HIDDEN_DATA(x) x:private_extern + %else + %define HIDDEN_DATA(x) x + %endif + %else + %define HIDDEN_DATA(x) x + %endif +%else + %macro GET_GOT 1 + %endmacro + %define GLOBAL(x) rel x + %ifidn __OUTPUT_FORMAT__,elf64 + %define WRT_PLT wrt ..plt + %define HIDDEN_DATA(x) x:data hidden + %elifidn __OUTPUT_FORMAT__,elfx32 + %define WRT_PLT wrt ..plt + %define HIDDEN_DATA(x) x:data hidden + %elifidn __OUTPUT_FORMAT__,macho64 + %ifdef CHROMIUM + %define HIDDEN_DATA(x) x:private_extern + %else + %define HIDDEN_DATA(x) x + %endif + %else + %define HIDDEN_DATA(x) x + %endif +%endif +%ifnmacro GET_GOT + %macro GET_GOT 1 + %endmacro + %define GLOBAL(x) x +%endif +%ifndef RESTORE_GOT +%define RESTORE_GOT +%endif +%ifndef WRT_PLT +%define WRT_PLT +%endif + +%if ABI_IS_32BIT + %macro SHADOW_ARGS_TO_STACK 1 + %endm + %define UNSHADOW_ARGS +%else +%if LIBAOM_YASM_WIN64 + %macro SHADOW_ARGS_TO_STACK 1 ; argc + %if %1 > 0 + mov arg(0),rcx + %endif + %if %1 > 1 + mov arg(1),rdx + %endif + %if %1 > 2 + mov arg(2),r8 + %endif + %if %1 > 3 + mov arg(3),r9 + %endif + %endm +%else + %macro SHADOW_ARGS_TO_STACK 1 ; argc + %if %1 > 0 + push rdi + %endif + %if %1 > 1 + push rsi + %endif + %if %1 > 2 + push rdx + %endif + %if %1 > 3 + push rcx + %endif + %if %1 > 4 + push r8 + %endif + %if %1 > 5 + push r9 + %endif + %if %1 > 6 + %assign i %1-6 + %assign off 16 + %rep i + mov rax,[rbp+off] + push rax + %assign off off+8 + %endrep + %endif + %endm +%endif + %define UNSHADOW_ARGS mov rsp, rbp +%endif + +; Win64 ABI requires that XMM6:XMM15 are callee saved +; SAVE_XMM n, [u] +; store registers 6-n on the stack +; if u is specified, use unaligned movs. +; Win64 ABI requires 16 byte stack alignment, but then pushes an 8 byte return +; value. Typically we follow this up with 'push rbp' - re-aligning the stack - +; but in some cases this is not done and unaligned movs must be used. +%if LIBAOM_YASM_WIN64 +%macro SAVE_XMM 1-2 a + %if %1 < 6 + %error Only xmm registers 6-15 must be preserved + %else + %assign last_xmm %1 + %define movxmm movdq %+ %2 + %assign xmm_stack_space ((last_xmm - 5) * 16) + sub rsp, xmm_stack_space + %assign i 6 + %rep (last_xmm - 5) + movxmm [rsp + ((i - 6) * 16)], xmm %+ i + %assign i i+1 + %endrep + %endif +%endmacro +%macro RESTORE_XMM 0 + %ifndef last_xmm + %error RESTORE_XMM must be paired with SAVE_XMM n + %else + %assign i last_xmm + %rep (last_xmm - 5) + movxmm xmm %+ i, [rsp +((i - 6) * 16)] + %assign i i-1 + %endrep + add rsp, xmm_stack_space + ; there are a couple functions which return from multiple places. + ; otherwise, we could uncomment these: + ; %undef last_xmm + ; %undef xmm_stack_space + ; %undef movxmm + %endif +%endmacro +%else +%macro SAVE_XMM 1-2 +%endmacro +%macro RESTORE_XMM 0 +%endmacro +%endif + +; Name of the rodata section +; +; .rodata seems to be an elf-ism, as it doesn't work on OSX. +; +%ifidn __OUTPUT_FORMAT__,macho64 +%define SECTION_RODATA section .text +%elifidn __OUTPUT_FORMAT__,macho32 +%macro SECTION_RODATA 0 +section .text +%endmacro +%elifidn __OUTPUT_FORMAT__,aout +%define SECTION_RODATA section .data +%else +%define SECTION_RODATA section .rodata +%endif + + +; Tell GNU ld that we don't require an executable stack. +%ifidn __OUTPUT_FORMAT__,elf32 +section .note.GNU-stack noalloc noexec nowrite progbits +section .text +%elifidn __OUTPUT_FORMAT__,elf64 +section .note.GNU-stack noalloc noexec nowrite progbits +section .text +%elifidn __OUTPUT_FORMAT__,elfx32 +section .note.GNU-stack noalloc noexec nowrite progbits +section .text +%endif diff --git a/third_party/aom/aom_scale/aom_scale.cmake b/third_party/aom/aom_scale/aom_scale.cmake new file mode 100644 index 0000000000..a6aa31afce --- /dev/null +++ b/third_party/aom/aom_scale/aom_scale.cmake @@ -0,0 +1,34 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +set(AOM_SCALE_SOURCES + "${AOM_ROOT}/aom_scale/aom_scale.h" + "${AOM_ROOT}/aom_scale/generic/aom_scale.c" + "${AOM_ROOT}/aom_scale/generic/gen_scalers.c" + "${AOM_ROOT}/aom_scale/generic/yv12config.c" + "${AOM_ROOT}/aom_scale/generic/yv12extend.c" + "${AOM_ROOT}/aom_scale/yv12config.h") + +set(AOM_SCALE_INTRIN_DSPR2 + "${AOM_ROOT}/aom_scale/mips/dspr2/yv12extend_dspr2.c") + +# Creates the aom_scale build target and makes libaom depend on it. The libaom +# target must exist before this function is called. +function (setup_aom_scale_targets) + add_library(aom_scale OBJECT ${AOM_SCALE_SOURCES}) + target_sources(aom PUBLIC $) + + if (HAVE_DSPR2) + add_intrinsics_object_library("" "dspr2" "aom_scale" + "AOM_SCALE_INTRIN_DSPR2") + endif () + + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_scale PARENT_SCOPE) +endfunction () diff --git a/third_party/aom/aom_scale/aom_scale.h b/third_party/aom/aom_scale/aom_scale.h new file mode 100644 index 0000000000..6e089f5aaa --- /dev/null +++ b/third_party/aom/aom_scale/aom_scale.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_SCALE_AOM_SCALE_H_ +#define AOM_SCALE_AOM_SCALE_H_ + +#include "aom_scale/yv12config.h" + +extern void aom_scale_frame(YV12_BUFFER_CONFIG *src, YV12_BUFFER_CONFIG *dst, + unsigned char *temp_area, unsigned char temp_height, + unsigned int hscale, unsigned int hratio, + unsigned int vscale, unsigned int vratio, + unsigned int interlaced); + +#endif // AOM_SCALE_AOM_SCALE_H_ diff --git a/third_party/aom/aom_scale/aom_scale.mk b/third_party/aom/aom_scale/aom_scale.mk new file mode 100644 index 0000000000..e3a68cfcf0 --- /dev/null +++ b/third_party/aom/aom_scale/aom_scale.mk @@ -0,0 +1,16 @@ +SCALE_SRCS-yes += aom_scale.mk +SCALE_SRCS-yes += yv12config.h +SCALE_SRCS-$(CONFIG_SPATIAL_RESAMPLING) += aom_scale.h +SCALE_SRCS-$(CONFIG_SPATIAL_RESAMPLING) += generic/aom_scale.c +SCALE_SRCS-yes += generic/yv12config.c +SCALE_SRCS-yes += generic/yv12extend.c +SCALE_SRCS-$(CONFIG_SPATIAL_RESAMPLING) += generic/gen_scalers.c +SCALE_SRCS-yes += aom_scale_rtcd.c +SCALE_SRCS-yes += aom_scale_rtcd.pl + +#mips(dspr2) +SCALE_SRCS-$(HAVE_DSPR2) += mips/dspr2/yv12extend_dspr2.c + +SCALE_SRCS-no += $(SCALE_SRCS_REMOVE-yes) + +$(eval $(call rtcd_h_template,aom_scale_rtcd,aom_scale/aom_scale_rtcd.pl)) diff --git a/third_party/aom/aom_scale/aom_scale_rtcd.c b/third_party/aom/aom_scale/aom_scale_rtcd.c new file mode 100644 index 0000000000..dec23735bf --- /dev/null +++ b/third_party/aom/aom_scale/aom_scale_rtcd.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include "./aom_config.h" +#define RTCD_C +#include "./aom_scale_rtcd.h" +#include "aom_ports/aom_once.h" + +void aom_scale_rtcd() { once(setup_rtcd_internal); } diff --git a/third_party/aom/aom_scale/aom_scale_rtcd.pl b/third_party/aom/aom_scale/aom_scale_rtcd.pl new file mode 100644 index 0000000000..9d9a1a29ae --- /dev/null +++ b/third_party/aom/aom_scale/aom_scale_rtcd.pl @@ -0,0 +1,38 @@ +sub aom_scale_forward_decls() { +print <> 4); + source += source_step; + dest += dest_step; + } +} + +/**************************************************************************** + * + * ROUTINE : scale1d_2t1_ps + * + * INPUTS : const unsigned char *source : Pointer to data to be scaled. + * int source_step : Number of pixels to step on + * in source. + * unsigned int source_scale : Scale for source (UNUSED). + * unsigned int source_length : Length of source (UNUSED). + * unsigned char *dest : Pointer to output data array. + * int dest_step : Number of pixels to step on + * in destination. + * unsigned int dest_scale : Scale for destination + * (UNUSED). + * unsigned int dest_length : Length of destination. + * + * OUTPUTS : None. + * + * RETURNS : void + * + * FUNCTION : Performs 2-to-1 point subsampled scaling. + * + * SPECIAL NOTES : None. + * + ****************************************************************************/ +static void scale1d_2t1_ps(const unsigned char *source, int source_step, + unsigned int source_scale, + unsigned int source_length, unsigned char *dest, + int dest_step, unsigned int dest_scale, + unsigned int dest_length) { + const unsigned char *const dest_end = dest + dest_length * dest_step; + (void)source_length; + (void)source_scale; + (void)dest_scale; + + source_step *= 2; // Every other row. + + while (dest < dest_end) { + *dest = *source; + source += source_step; + dest += dest_step; + } +} +/**************************************************************************** + * + * ROUTINE : scale1d_c + * + * INPUTS : const unsigned char *source : Pointer to data to be scaled. + * int source_step : Number of pixels to step on + * in source. + * unsigned int source_scale : Scale for source. + * unsigned int source_length : Length of source (UNUSED). + * unsigned char *dest : Pointer to output data array. + * int dest_step : Number of pixels to step on + * in destination. + * unsigned int dest_scale : Scale for destination. + * unsigned int dest_length : Length of destination. + * + * OUTPUTS : None. + * + * RETURNS : void + * + * FUNCTION : Performs linear interpolation in one dimension. + * + * SPECIAL NOTES : None. + * + ****************************************************************************/ +static void scale1d_c(const unsigned char *source, int source_step, + unsigned int source_scale, unsigned int source_length, + unsigned char *dest, int dest_step, + unsigned int dest_scale, unsigned int dest_length) { + const unsigned char *const dest_end = dest + dest_length * dest_step; + const unsigned int round_value = dest_scale / 2; + unsigned int left_modifier = dest_scale; + unsigned int right_modifier = 0; + unsigned char left_pixel = source[0]; + unsigned char right_pixel = source[source_step]; + + (void)source_length; + + /* These asserts are needed if there are boundary issues... */ + /* assert ( dest_scale > source_scale );*/ + /* assert ( (source_length - 1) * dest_scale >= (dest_length - 1) * + * source_scale);*/ + + while (dest < dest_end) { + *dest = (unsigned char)((left_modifier * left_pixel + + right_modifier * right_pixel + round_value) / + dest_scale); + + right_modifier += source_scale; + + while (right_modifier > dest_scale) { + right_modifier -= dest_scale; + source += source_step; + left_pixel = source[0]; + right_pixel = source[source_step]; + } + + left_modifier = dest_scale - right_modifier; + } +} + +/**************************************************************************** + * + * ROUTINE : Scale2D + * + * INPUTS : const unsigned char *source : Pointer to data to be + * scaled. + * int source_pitch : Stride of source image. + * unsigned int source_width : Width of input image. + * unsigned int source_height : Height of input image. + * unsigned char *dest : Pointer to output data + * array. + * int dest_pitch : Stride of destination + * image. + * unsigned int dest_width : Width of destination image. + * unsigned int dest_height : Height of destination + * image. + * unsigned char *temp_area : Pointer to temp work area. + * unsigned char temp_area_height : Height of temp work area. + * unsigned int hscale : Horizontal scale factor + * numerator. + * unsigned int hratio : Horizontal scale factor + * denominator. + * unsigned int vscale : Vertical scale factor + * numerator. + * unsigned int vratio : Vertical scale factor + * denominator. + * unsigned int interlaced : Interlace flag. + * + * OUTPUTS : None. + * + * RETURNS : void + * + * FUNCTION : Performs 2-tap linear interpolation in two dimensions. + * + * SPECIAL NOTES : Expansion is performed one band at a time to help with + * caching. + * + ****************************************************************************/ +static void Scale2D( + /*const*/ + unsigned char *source, int source_pitch, unsigned int source_width, + unsigned int source_height, unsigned char *dest, int dest_pitch, + unsigned int dest_width, unsigned int dest_height, unsigned char *temp_area, + unsigned char temp_area_height, unsigned int hscale, unsigned int hratio, + unsigned int vscale, unsigned int vratio, unsigned int interlaced) { + unsigned int i, j, k; + unsigned int bands; + unsigned int dest_band_height; + unsigned int source_band_height; + + typedef void (*Scale1D)(const unsigned char *source, int source_step, + unsigned int source_scale, unsigned int source_length, + unsigned char *dest, int dest_step, + unsigned int dest_scale, unsigned int dest_length); + + Scale1D Scale1Dv = scale1d_c; + Scale1D Scale1Dh = scale1d_c; + + void (*horiz_line_scale)(const unsigned char *, unsigned int, unsigned char *, + unsigned int) = NULL; + void (*vert_band_scale)(unsigned char *, int, unsigned char *, int, + unsigned int) = NULL; + + int ratio_scalable = 1; + int interpolation = 0; + + unsigned char *source_base; + unsigned char *line_src; + + source_base = (unsigned char *)source; + + if (source_pitch < 0) { + int offset; + + offset = (source_height - 1); + offset *= source_pitch; + + source_base += offset; + } + + /* find out the ratio for each direction */ + switch (hratio * 10 / hscale) { + case 8: + /* 4-5 Scale in Width direction */ + horiz_line_scale = aom_horizontal_line_5_4_scale; + break; + case 6: + /* 3-5 Scale in Width direction */ + horiz_line_scale = aom_horizontal_line_5_3_scale; + break; + case 5: + /* 1-2 Scale in Width direction */ + horiz_line_scale = aom_horizontal_line_2_1_scale; + break; + default: + /* The ratio is not acceptable now */ + /* throw("The ratio is not acceptable for now!"); */ + ratio_scalable = 0; + break; + } + + switch (vratio * 10 / vscale) { + case 8: + /* 4-5 Scale in vertical direction */ + vert_band_scale = aom_vertical_band_5_4_scale; + source_band_height = 5; + dest_band_height = 4; + break; + case 6: + /* 3-5 Scale in vertical direction */ + vert_band_scale = aom_vertical_band_5_3_scale; + source_band_height = 5; + dest_band_height = 3; + break; + case 5: + /* 1-2 Scale in vertical direction */ + + if (interlaced) { + /* if the content is interlaced, point sampling is used */ + vert_band_scale = aom_vertical_band_2_1_scale; + } else { + interpolation = 1; + /* if the content is progressive, interplo */ + vert_band_scale = aom_vertical_band_2_1_scale_i; + } + + source_band_height = 2; + dest_band_height = 1; + break; + default: + /* The ratio is not acceptable now */ + /* throw("The ratio is not acceptable for now!"); */ + ratio_scalable = 0; + break; + } + + if (ratio_scalable) { + if (source_height == dest_height) { + /* for each band of the image */ + for (k = 0; k < dest_height; ++k) { + horiz_line_scale(source, source_width, dest, dest_width); + source += source_pitch; + dest += dest_pitch; + } + + return; + } + + if (interpolation) { + if (source < source_base) source = source_base; + + horiz_line_scale(source, source_width, temp_area, dest_width); + } + + for (k = 0; k < (dest_height + dest_band_height - 1) / dest_band_height; + ++k) { + /* scale one band horizontally */ + for (i = 0; i < source_band_height; ++i) { + /* Trap case where we could read off the base of the source buffer */ + + line_src = source + i * source_pitch; + + if (line_src < source_base) line_src = source_base; + + horiz_line_scale(line_src, source_width, + temp_area + (i + 1) * dest_pitch, dest_width); + } + + /* Vertical scaling is in place */ + vert_band_scale(temp_area + dest_pitch, dest_pitch, dest, dest_pitch, + dest_width); + + if (interpolation) + memcpy(temp_area, temp_area + source_band_height * dest_pitch, + dest_width); + + /* Next band... */ + source += (unsigned long)source_band_height * source_pitch; + dest += (unsigned long)dest_band_height * dest_pitch; + } + + return; + } + + if (hscale == 2 && hratio == 1) Scale1Dh = scale1d_2t1_ps; + + if (vscale == 2 && vratio == 1) { + if (interlaced) + Scale1Dv = scale1d_2t1_ps; + else + Scale1Dv = scale1d_2t1_i; + } + + if (source_height == dest_height) { + /* for each band of the image */ + for (k = 0; k < dest_height; ++k) { + Scale1Dh(source, 1, hscale, source_width + 1, dest, 1, hratio, + dest_width); + source += source_pitch; + dest += dest_pitch; + } + + return; + } + + if (dest_height > source_height) { + dest_band_height = temp_area_height - 1; + source_band_height = dest_band_height * source_height / dest_height; + } else { + source_band_height = temp_area_height - 1; + dest_band_height = source_band_height * vratio / vscale; + } + + /* first row needs to be done so that we can stay one row ahead for vertical + * zoom */ + Scale1Dh(source, 1, hscale, source_width + 1, temp_area, 1, hratio, + dest_width); + + /* for each band of the image */ + bands = (dest_height + dest_band_height - 1) / dest_band_height; + + for (k = 0; k < bands; ++k) { + /* scale one band horizontally */ + for (i = 1; i < source_band_height + 1; ++i) { + if (k * source_band_height + i < source_height) { + Scale1Dh(source + i * source_pitch, 1, hscale, source_width + 1, + temp_area + i * dest_pitch, 1, hratio, dest_width); + } else { /* Duplicate the last row */ + /* copy temp_area row 0 over from last row in the past */ + memcpy(temp_area + i * dest_pitch, temp_area + (i - 1) * dest_pitch, + dest_pitch); + } + } + + /* scale one band vertically */ + for (j = 0; j < dest_width; ++j) { + Scale1Dv(&temp_area[j], dest_pitch, vscale, source_band_height + 1, + &dest[j], dest_pitch, vratio, dest_band_height); + } + + /* copy temp_area row 0 over from last row in the past */ + memcpy(temp_area, temp_area + source_band_height * dest_pitch, dest_pitch); + + /* move to the next band */ + source += source_band_height * source_pitch; + dest += dest_band_height * dest_pitch; + } +} + +/**************************************************************************** + * + * ROUTINE : aom_scale_frame + * + * INPUTS : YV12_BUFFER_CONFIG *src : Pointer to frame to be + * scaled. + * YV12_BUFFER_CONFIG *dst : Pointer to buffer to hold + * scaled frame. + * unsigned char *temp_area : Pointer to temp work area. + * unsigned char temp_area_height : Height of temp work area. + * unsigned int hscale : Horizontal scale factor + * numerator. + * unsigned int hratio : Horizontal scale factor + * denominator. + * unsigned int vscale : Vertical scale factor + * numerator. + * unsigned int vratio : Vertical scale factor + * denominator. + * unsigned int interlaced : Interlace flag. + * + * OUTPUTS : None. + * + * RETURNS : void + * + * FUNCTION : Performs 2-tap linear interpolation in two dimensions. + * + * SPECIAL NOTES : Expansion is performed one band at a time to help with + * caching. + * + ****************************************************************************/ +void aom_scale_frame(YV12_BUFFER_CONFIG *src, YV12_BUFFER_CONFIG *dst, + unsigned char *temp_area, unsigned char temp_height, + unsigned int hscale, unsigned int hratio, + unsigned int vscale, unsigned int vratio, + unsigned int interlaced) { + int i; + int dw = (hscale - 1 + src->y_width * hratio) / hscale; + int dh = (vscale - 1 + src->y_height * vratio) / vscale; + + /* call our internal scaling routines!! */ + Scale2D((unsigned char *)src->y_buffer, src->y_stride, src->y_width, + src->y_height, (unsigned char *)dst->y_buffer, dst->y_stride, dw, dh, + temp_area, temp_height, hscale, hratio, vscale, vratio, interlaced); + + if (dw < (int)dst->y_width) + for (i = 0; i < dh; ++i) + memset(dst->y_buffer + i * dst->y_stride + dw - 1, + dst->y_buffer[i * dst->y_stride + dw - 2], dst->y_width - dw + 1); + + if (dh < (int)dst->y_height) + for (i = dh - 1; i < (int)dst->y_height; ++i) + memcpy(dst->y_buffer + i * dst->y_stride, + dst->y_buffer + (dh - 2) * dst->y_stride, dst->y_width + 1); + + Scale2D((unsigned char *)src->u_buffer, src->uv_stride, src->uv_width, + src->uv_height, (unsigned char *)dst->u_buffer, dst->uv_stride, + dw / 2, dh / 2, temp_area, temp_height, hscale, hratio, vscale, + vratio, interlaced); + + if (dw / 2 < (int)dst->uv_width) + for (i = 0; i < dst->uv_height; ++i) + memset(dst->u_buffer + i * dst->uv_stride + dw / 2 - 1, + dst->u_buffer[i * dst->uv_stride + dw / 2 - 2], + dst->uv_width - dw / 2 + 1); + + if (dh / 2 < (int)dst->uv_height) + for (i = dh / 2 - 1; i < (int)dst->y_height / 2; ++i) + memcpy(dst->u_buffer + i * dst->uv_stride, + dst->u_buffer + (dh / 2 - 2) * dst->uv_stride, dst->uv_width); + + Scale2D((unsigned char *)src->v_buffer, src->uv_stride, src->uv_width, + src->uv_height, (unsigned char *)dst->v_buffer, dst->uv_stride, + dw / 2, dh / 2, temp_area, temp_height, hscale, hratio, vscale, + vratio, interlaced); + + if (dw / 2 < (int)dst->uv_width) + for (i = 0; i < dst->uv_height; ++i) + memset(dst->v_buffer + i * dst->uv_stride + dw / 2 - 1, + dst->v_buffer[i * dst->uv_stride + dw / 2 - 2], + dst->uv_width - dw / 2 + 1); + + if (dh / 2 < (int)dst->uv_height) + for (i = dh / 2 - 1; i < (int)dst->y_height / 2; ++i) + memcpy(dst->v_buffer + i * dst->uv_stride, + dst->v_buffer + (dh / 2 - 2) * dst->uv_stride, dst->uv_width); +} diff --git a/third_party/aom/aom_scale/generic/gen_scalers.c b/third_party/aom/aom_scale/generic/gen_scalers.c new file mode 100644 index 0000000000..71fa82f308 --- /dev/null +++ b/third_party/aom/aom_scale/generic/gen_scalers.c @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_scale_rtcd.h" +#include "aom_scale/aom_scale.h" +#include "aom_mem/aom_mem.h" +/**************************************************************************** +* Imports +****************************************************************************/ + +/**************************************************************************** + * + * + * INPUTS : const unsigned char *source : Pointer to source data. + * unsigned int source_width : Stride of source. + * unsigned char *dest : Pointer to destination data. + * unsigned int dest_width : Stride of destination + * (NOT USED). + * + * OUTPUTS : None. + * + * RETURNS : void + * + * FUNCTION : Copies horizontal line of pixels from source to + * destination scaling up by 4 to 5. + * + * SPECIAL NOTES : None. + * + ****************************************************************************/ +void aom_horizontal_line_5_4_scale_c(const unsigned char *source, + unsigned int source_width, + unsigned char *dest, + unsigned int dest_width) { + const unsigned char *const source_end = source + source_width; + (void)dest_width; + + while (source < source_end) { + const unsigned int a = source[0]; + const unsigned int b = source[1]; + const unsigned int c = source[2]; + const unsigned int d = source[3]; + const unsigned int e = source[4]; + + dest[0] = (unsigned char)a; + dest[1] = (unsigned char)((b * 192 + c * 64 + 128) >> 8); + dest[2] = (unsigned char)((c * 128 + d * 128 + 128) >> 8); + dest[3] = (unsigned char)((d * 64 + e * 192 + 128) >> 8); + + source += 5; + dest += 4; + } +} + +void aom_vertical_band_5_4_scale_c(unsigned char *source, int src_pitch, + unsigned char *dest, int dest_pitch, + unsigned int dest_width) { + const unsigned char *const dest_end = dest + dest_width; + while (dest < dest_end) { + const unsigned int a = source[0 * src_pitch]; + const unsigned int b = source[1 * src_pitch]; + const unsigned int c = source[2 * src_pitch]; + const unsigned int d = source[3 * src_pitch]; + const unsigned int e = source[4 * src_pitch]; + + dest[0 * dest_pitch] = (unsigned char)a; + dest[1 * dest_pitch] = (unsigned char)((b * 192 + c * 64 + 128) >> 8); + dest[2 * dest_pitch] = (unsigned char)((c * 128 + d * 128 + 128) >> 8); + dest[3 * dest_pitch] = (unsigned char)((d * 64 + e * 192 + 128) >> 8); + + ++source; + ++dest; + } +} + +/*7*************************************************************************** + * + * ROUTINE : aom_horizontal_line_3_5_scale_c + * + * INPUTS : const unsigned char *source : Pointer to source data. + * unsigned int source_width : Stride of source. + * unsigned char *dest : Pointer to destination data. + * unsigned int dest_width : Stride of destination + * (NOT USED). + * + * OUTPUTS : None. + * + * RETURNS : void + * + * FUNCTION : Copies horizontal line of pixels from source to + * destination scaling up by 3 to 5. + * + * SPECIAL NOTES : None. + * + * + ****************************************************************************/ +void aom_horizontal_line_5_3_scale_c(const unsigned char *source, + unsigned int source_width, + unsigned char *dest, + unsigned int dest_width) { + const unsigned char *const source_end = source + source_width; + (void)dest_width; + while (source < source_end) { + const unsigned int a = source[0]; + const unsigned int b = source[1]; + const unsigned int c = source[2]; + const unsigned int d = source[3]; + const unsigned int e = source[4]; + + dest[0] = (unsigned char)a; + dest[1] = (unsigned char)((b * 85 + c * 171 + 128) >> 8); + dest[2] = (unsigned char)((d * 171 + e * 85 + 128) >> 8); + + source += 5; + dest += 3; + } +} + +void aom_vertical_band_5_3_scale_c(unsigned char *source, int src_pitch, + unsigned char *dest, int dest_pitch, + unsigned int dest_width) { + const unsigned char *const dest_end = dest + dest_width; + while (dest < dest_end) { + const unsigned int a = source[0 * src_pitch]; + const unsigned int b = source[1 * src_pitch]; + const unsigned int c = source[2 * src_pitch]; + const unsigned int d = source[3 * src_pitch]; + const unsigned int e = source[4 * src_pitch]; + + dest[0 * dest_pitch] = (unsigned char)a; + dest[1 * dest_pitch] = (unsigned char)((b * 85 + c * 171 + 128) >> 8); + dest[2 * dest_pitch] = (unsigned char)((d * 171 + e * 85 + 128) >> 8); + + ++source; + ++dest; + } +} + +/**************************************************************************** + * + * ROUTINE : aom_horizontal_line_1_2_scale_c + * + * INPUTS : const unsigned char *source : Pointer to source data. + * unsigned int source_width : Stride of source. + * unsigned char *dest : Pointer to destination data. + * unsigned int dest_width : Stride of destination + * (NOT USED). + * + * OUTPUTS : None. + * + * RETURNS : void + * + * FUNCTION : Copies horizontal line of pixels from source to + * destination scaling up by 1 to 2. + * + * SPECIAL NOTES : None. + * + ****************************************************************************/ +void aom_horizontal_line_2_1_scale_c(const unsigned char *source, + unsigned int source_width, + unsigned char *dest, + unsigned int dest_width) { + const unsigned char *const source_end = source + source_width; + (void)dest_width; + while (source < source_end) { + dest[0] = source[0]; + source += 2; + ++dest; + } +} + +void aom_vertical_band_2_1_scale_c(unsigned char *source, int src_pitch, + unsigned char *dest, int dest_pitch, + unsigned int dest_width) { + (void)dest_pitch; + (void)src_pitch; + memcpy(dest, source, dest_width); +} + +void aom_vertical_band_2_1_scale_i_c(unsigned char *source, int src_pitch, + unsigned char *dest, int dest_pitch, + unsigned int dest_width) { + const unsigned char *const dest_end = dest + dest_width; + (void)dest_pitch; + while (dest < dest_end) { + const unsigned int a = source[-src_pitch] * 3; + const unsigned int b = source[0] * 10; + const unsigned int c = source[src_pitch] * 3; + dest[0] = (unsigned char)((8 + a + b + c) >> 4); + ++source; + ++dest; + } +} diff --git a/third_party/aom/aom_scale/generic/yv12config.c b/third_party/aom/aom_scale/generic/yv12config.c new file mode 100644 index 0000000000..ee15ae103c --- /dev/null +++ b/third_party/aom/aom_scale/generic/yv12config.c @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom_scale/yv12config.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +/**************************************************************************** +* Exports +****************************************************************************/ + +/**************************************************************************** + * + ****************************************************************************/ +#define yv12_align_addr(addr, align) \ + (void *)(((size_t)(addr) + ((align)-1)) & (size_t) - (align)) + +#if CONFIG_AV1 +// TODO(jkoleszar): Maybe replace this with struct aom_image + +int aom_free_frame_buffer(YV12_BUFFER_CONFIG *ybf) { + if (ybf) { + if (ybf->buffer_alloc_sz > 0) { + aom_free(ybf->buffer_alloc); + } + +#if CONFIG_HIGHBITDEPTH && CONFIG_GLOBAL_MOTION + if (ybf->y_buffer_8bit) free(ybf->y_buffer_8bit); +#endif + + /* buffer_alloc isn't accessed by most functions. Rather y_buffer, + u_buffer and v_buffer point to buffer_alloc and are used. Clear out + all of this so that a freed pointer isn't inadvertently used */ + memset(ybf, 0, sizeof(YV12_BUFFER_CONFIG)); + } else { + return -1; + } + + return 0; +} + +int aom_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height, + int ss_x, int ss_y, +#if CONFIG_HIGHBITDEPTH + int use_highbitdepth, +#endif + int border, int byte_alignment, + aom_codec_frame_buffer_t *fb, + aom_get_frame_buffer_cb_fn_t cb, void *cb_priv) { + if (ybf) { + const int aom_byte_align = (byte_alignment == 0) ? 1 : byte_alignment; + const int aligned_width = (width + 7) & ~7; + const int aligned_height = (height + 7) & ~7; + const int y_stride = ((aligned_width + 2 * border) + 31) & ~31; + const uint64_t yplane_size = + (aligned_height + 2 * border) * (uint64_t)y_stride + byte_alignment; + const int uv_width = aligned_width >> ss_x; + const int uv_height = aligned_height >> ss_y; + const int uv_stride = y_stride >> ss_x; + const int uv_border_w = border >> ss_x; + const int uv_border_h = border >> ss_y; + const uint64_t uvplane_size = + (uv_height + 2 * uv_border_h) * (uint64_t)uv_stride + byte_alignment; + +#if CONFIG_HIGHBITDEPTH + const uint64_t frame_size = + (1 + use_highbitdepth) * (yplane_size + 2 * uvplane_size); +#else + const uint64_t frame_size = yplane_size + 2 * uvplane_size; +#endif // CONFIG_HIGHBITDEPTH + + uint8_t *buf = NULL; + + if (cb != NULL) { + const int align_addr_extra_size = 31; + const uint64_t external_frame_size = frame_size + align_addr_extra_size; + + assert(fb != NULL); + + if (external_frame_size != (size_t)external_frame_size) return -1; + + // Allocation to hold larger frame, or first allocation. + if (cb(cb_priv, (size_t)external_frame_size, fb) < 0) return -1; + + if (fb->data == NULL || fb->size < external_frame_size) return -1; + + ybf->buffer_alloc = (uint8_t *)yv12_align_addr(fb->data, 32); + +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) + // This memset is needed for fixing the issue of using uninitialized + // value in msan test. It will cause a perf loss, so only do this for + // msan test. + memset(ybf->buffer_alloc, 0, (int)frame_size); +#endif +#endif + } else if (frame_size > (size_t)ybf->buffer_alloc_sz) { + // Allocation to hold larger frame, or first allocation. + aom_free(ybf->buffer_alloc); + ybf->buffer_alloc = NULL; + + if (frame_size != (size_t)frame_size) return -1; + + ybf->buffer_alloc = (uint8_t *)aom_memalign(32, (size_t)frame_size); + if (!ybf->buffer_alloc) return -1; + + ybf->buffer_alloc_sz = (size_t)frame_size; + + // This memset is needed for fixing valgrind error from C loop filter + // due to access uninitialized memory in frame border. It could be + // removed if border is totally removed. + memset(ybf->buffer_alloc, 0, ybf->buffer_alloc_sz); + } + + /* Only support allocating buffers that have a border that's a multiple + * of 32. The border restriction is required to get 16-byte alignment of + * the start of the chroma rows without introducing an arbitrary gap + * between planes, which would break the semantics of things like + * aom_img_set_rect(). */ + if (border & 0x1f) return -3; + + ybf->y_crop_width = width; + ybf->y_crop_height = height; + ybf->y_width = aligned_width; + ybf->y_height = aligned_height; + ybf->y_stride = y_stride; + + ybf->uv_crop_width = (width + ss_x) >> ss_x; + ybf->uv_crop_height = (height + ss_y) >> ss_y; + ybf->uv_width = uv_width; + ybf->uv_height = uv_height; + ybf->uv_stride = uv_stride; + + ybf->border = border; + ybf->frame_size = (size_t)frame_size; + ybf->subsampling_x = ss_x; + ybf->subsampling_y = ss_y; + + buf = ybf->buffer_alloc; +#if CONFIG_HIGHBITDEPTH + if (use_highbitdepth) { + // Store uint16 addresses when using 16bit framebuffers + buf = CONVERT_TO_BYTEPTR(ybf->buffer_alloc); + ybf->flags = YV12_FLAG_HIGHBITDEPTH; + } else { + ybf->flags = 0; + } +#endif // CONFIG_HIGHBITDEPTH + + ybf->y_buffer = (uint8_t *)yv12_align_addr( + buf + (border * y_stride) + border, aom_byte_align); + ybf->u_buffer = (uint8_t *)yv12_align_addr( + buf + yplane_size + (uv_border_h * uv_stride) + uv_border_w, + aom_byte_align); + ybf->v_buffer = + (uint8_t *)yv12_align_addr(buf + yplane_size + uvplane_size + + (uv_border_h * uv_stride) + uv_border_w, + aom_byte_align); + +#if CONFIG_HIGHBITDEPTH && CONFIG_GLOBAL_MOTION + if (ybf->y_buffer_8bit) { + free(ybf->y_buffer_8bit); + ybf->y_buffer_8bit = NULL; + } +#endif + + ybf->corrupted = 0; /* assume not corrupted by errors */ + return 0; + } + return -2; +} + +int aom_alloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height, + int ss_x, int ss_y, +#if CONFIG_HIGHBITDEPTH + int use_highbitdepth, +#endif + int border, int byte_alignment) { + if (ybf) { + aom_free_frame_buffer(ybf); + return aom_realloc_frame_buffer(ybf, width, height, ss_x, ss_y, +#if CONFIG_HIGHBITDEPTH + use_highbitdepth, +#endif + border, byte_alignment, NULL, NULL, NULL); + } + return -2; +} +#endif diff --git a/third_party/aom/aom_scale/generic/yv12extend.c b/third_party/aom/aom_scale/generic/yv12extend.c new file mode 100644 index 0000000000..05e4633625 --- /dev/null +++ b/third_party/aom/aom_scale/generic/yv12extend.c @@ -0,0 +1,382 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./aom_config.h" +#include "./aom_scale_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" +#include "aom_scale/yv12config.h" + +static void extend_plane(uint8_t *const src, int src_stride, int width, + int height, int extend_top, int extend_left, + int extend_bottom, int extend_right) { + int i; + const int linesize = extend_left + extend_right + width; + + /* copy the left and right most columns out */ + uint8_t *src_ptr1 = src; + uint8_t *src_ptr2 = src + width - 1; + uint8_t *dst_ptr1 = src - extend_left; + uint8_t *dst_ptr2 = src + width; + + for (i = 0; i < height; ++i) { + memset(dst_ptr1, src_ptr1[0], extend_left); + memset(dst_ptr2, src_ptr2[0], extend_right); + src_ptr1 += src_stride; + src_ptr2 += src_stride; + dst_ptr1 += src_stride; + dst_ptr2 += src_stride; + } + + /* Now copy the top and bottom lines into each line of the respective + * borders + */ + src_ptr1 = src - extend_left; + src_ptr2 = src + src_stride * (height - 1) - extend_left; + dst_ptr1 = src + src_stride * -extend_top - extend_left; + dst_ptr2 = src + src_stride * height - extend_left; + + for (i = 0; i < extend_top; ++i) { + memcpy(dst_ptr1, src_ptr1, linesize); + dst_ptr1 += src_stride; + } + + for (i = 0; i < extend_bottom; ++i) { + memcpy(dst_ptr2, src_ptr2, linesize); + dst_ptr2 += src_stride; + } +} + +#if CONFIG_HIGHBITDEPTH +static void extend_plane_high(uint8_t *const src8, int src_stride, int width, + int height, int extend_top, int extend_left, + int extend_bottom, int extend_right) { + int i; + const int linesize = extend_left + extend_right + width; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + + /* copy the left and right most columns out */ + uint16_t *src_ptr1 = src; + uint16_t *src_ptr2 = src + width - 1; + uint16_t *dst_ptr1 = src - extend_left; + uint16_t *dst_ptr2 = src + width; + + for (i = 0; i < height; ++i) { + aom_memset16(dst_ptr1, src_ptr1[0], extend_left); + aom_memset16(dst_ptr2, src_ptr2[0], extend_right); + src_ptr1 += src_stride; + src_ptr2 += src_stride; + dst_ptr1 += src_stride; + dst_ptr2 += src_stride; + } + + /* Now copy the top and bottom lines into each line of the respective + * borders + */ + src_ptr1 = src - extend_left; + src_ptr2 = src + src_stride * (height - 1) - extend_left; + dst_ptr1 = src + src_stride * -extend_top - extend_left; + dst_ptr2 = src + src_stride * height - extend_left; + + for (i = 0; i < extend_top; ++i) { + memcpy(dst_ptr1, src_ptr1, linesize * sizeof(uint16_t)); + dst_ptr1 += src_stride; + } + + for (i = 0; i < extend_bottom; ++i) { + memcpy(dst_ptr2, src_ptr2, linesize * sizeof(uint16_t)); + dst_ptr2 += src_stride; + } +} +#endif + +void aom_yv12_extend_frame_borders_c(YV12_BUFFER_CONFIG *ybf) { + const int uv_border = ybf->border / 2; + + assert(ybf->border % 2 == 0); + assert(ybf->y_height - ybf->y_crop_height < 16); + assert(ybf->y_width - ybf->y_crop_width < 16); + assert(ybf->y_height - ybf->y_crop_height >= 0); + assert(ybf->y_width - ybf->y_crop_width >= 0); + +#if CONFIG_HIGHBITDEPTH + if (ybf->flags & YV12_FLAG_HIGHBITDEPTH) { + extend_plane_high(ybf->y_buffer, ybf->y_stride, ybf->y_crop_width, + ybf->y_crop_height, ybf->border, ybf->border, + ybf->border + ybf->y_height - ybf->y_crop_height, + ybf->border + ybf->y_width - ybf->y_crop_width); + + extend_plane_high(ybf->u_buffer, ybf->uv_stride, ybf->uv_crop_width, + ybf->uv_crop_height, uv_border, uv_border, + uv_border + ybf->uv_height - ybf->uv_crop_height, + uv_border + ybf->uv_width - ybf->uv_crop_width); + + extend_plane_high(ybf->v_buffer, ybf->uv_stride, ybf->uv_crop_width, + ybf->uv_crop_height, uv_border, uv_border, + uv_border + ybf->uv_height - ybf->uv_crop_height, + uv_border + ybf->uv_width - ybf->uv_crop_width); + return; + } +#endif + extend_plane(ybf->y_buffer, ybf->y_stride, ybf->y_crop_width, + ybf->y_crop_height, ybf->border, ybf->border, + ybf->border + ybf->y_height - ybf->y_crop_height, + ybf->border + ybf->y_width - ybf->y_crop_width); + + extend_plane(ybf->u_buffer, ybf->uv_stride, ybf->uv_crop_width, + ybf->uv_crop_height, uv_border, uv_border, + uv_border + ybf->uv_height - ybf->uv_crop_height, + uv_border + ybf->uv_width - ybf->uv_crop_width); + + extend_plane(ybf->v_buffer, ybf->uv_stride, ybf->uv_crop_width, + ybf->uv_crop_height, uv_border, uv_border, + uv_border + ybf->uv_height - ybf->uv_crop_height, + uv_border + ybf->uv_width - ybf->uv_crop_width); +} + +#if CONFIG_AV1 +static void extend_frame(YV12_BUFFER_CONFIG *const ybf, int ext_size) { + const int c_w = ybf->uv_crop_width; + const int c_h = ybf->uv_crop_height; + const int ss_x = ybf->uv_width < ybf->y_width; + const int ss_y = ybf->uv_height < ybf->y_height; + const int c_et = ext_size >> ss_y; + const int c_el = ext_size >> ss_x; + const int c_eb = c_et + ybf->uv_height - ybf->uv_crop_height; + const int c_er = c_el + ybf->uv_width - ybf->uv_crop_width; + + assert(ybf->y_height - ybf->y_crop_height < 16); + assert(ybf->y_width - ybf->y_crop_width < 16); + assert(ybf->y_height - ybf->y_crop_height >= 0); + assert(ybf->y_width - ybf->y_crop_width >= 0); + +#if CONFIG_HIGHBITDEPTH + if (ybf->flags & YV12_FLAG_HIGHBITDEPTH) { + extend_plane_high(ybf->y_buffer, ybf->y_stride, ybf->y_crop_width, + ybf->y_crop_height, ext_size, ext_size, + ext_size + ybf->y_height - ybf->y_crop_height, + ext_size + ybf->y_width - ybf->y_crop_width); + extend_plane_high(ybf->u_buffer, ybf->uv_stride, c_w, c_h, c_et, c_el, c_eb, + c_er); + extend_plane_high(ybf->v_buffer, ybf->uv_stride, c_w, c_h, c_et, c_el, c_eb, + c_er); + return; + } +#endif + extend_plane(ybf->y_buffer, ybf->y_stride, ybf->y_crop_width, + ybf->y_crop_height, ext_size, ext_size, + ext_size + ybf->y_height - ybf->y_crop_height, + ext_size + ybf->y_width - ybf->y_crop_width); + + extend_plane(ybf->u_buffer, ybf->uv_stride, c_w, c_h, c_et, c_el, c_eb, c_er); + + extend_plane(ybf->v_buffer, ybf->uv_stride, c_w, c_h, c_et, c_el, c_eb, c_er); +} + +void aom_extend_frame_borders_c(YV12_BUFFER_CONFIG *ybf) { + extend_frame(ybf, ybf->border); +} + +void aom_extend_frame_inner_borders_c(YV12_BUFFER_CONFIG *ybf) { + const int inner_bw = (ybf->border > AOMINNERBORDERINPIXELS) + ? AOMINNERBORDERINPIXELS + : ybf->border; + extend_frame(ybf, inner_bw); +} + +void aom_extend_frame_borders_y_c(YV12_BUFFER_CONFIG *ybf) { + int ext_size = ybf->border; + assert(ybf->y_height - ybf->y_crop_height < 16); + assert(ybf->y_width - ybf->y_crop_width < 16); + assert(ybf->y_height - ybf->y_crop_height >= 0); + assert(ybf->y_width - ybf->y_crop_width >= 0); + +#if CONFIG_HIGHBITDEPTH + if (ybf->flags & YV12_FLAG_HIGHBITDEPTH) { + extend_plane_high(ybf->y_buffer, ybf->y_stride, ybf->y_crop_width, + ybf->y_crop_height, ext_size, ext_size, + ext_size + ybf->y_height - ybf->y_crop_height, + ext_size + ybf->y_width - ybf->y_crop_width); + return; + } +#endif + extend_plane(ybf->y_buffer, ybf->y_stride, ybf->y_crop_width, + ybf->y_crop_height, ext_size, ext_size, + ext_size + ybf->y_height - ybf->y_crop_height, + ext_size + ybf->y_width - ybf->y_crop_width); +} +#endif // CONFIG_AV1 + +#if CONFIG_HIGHBITDEPTH +static void memcpy_short_addr(uint8_t *dst8, const uint8_t *src8, int num) { + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + memcpy(dst, src, num * sizeof(uint16_t)); +} +#endif // CONFIG_HIGHBITDEPTH + +// Copies the source image into the destination image and updates the +// destination's UMV borders. +// Note: The frames are assumed to be identical in size. +void aom_yv12_copy_frame_c(const YV12_BUFFER_CONFIG *src_bc, + YV12_BUFFER_CONFIG *dst_bc) { + int row; + const uint8_t *src = src_bc->y_buffer; + uint8_t *dst = dst_bc->y_buffer; + +#if 0 + /* These assertions are valid in the codec, but the libaom-tester uses + * this code slightly differently. + */ + assert(src_bc->y_width == dst_bc->y_width); + assert(src_bc->y_height == dst_bc->y_height); +#endif + +#if CONFIG_HIGHBITDEPTH + if (src_bc->flags & YV12_FLAG_HIGHBITDEPTH) { + assert(dst_bc->flags & YV12_FLAG_HIGHBITDEPTH); + for (row = 0; row < src_bc->y_height; ++row) { + memcpy_short_addr(dst, src, src_bc->y_width); + src += src_bc->y_stride; + dst += dst_bc->y_stride; + } + + src = src_bc->u_buffer; + dst = dst_bc->u_buffer; + + for (row = 0; row < src_bc->uv_height; ++row) { + memcpy_short_addr(dst, src, src_bc->uv_width); + src += src_bc->uv_stride; + dst += dst_bc->uv_stride; + } + + src = src_bc->v_buffer; + dst = dst_bc->v_buffer; + + for (row = 0; row < src_bc->uv_height; ++row) { + memcpy_short_addr(dst, src, src_bc->uv_width); + src += src_bc->uv_stride; + dst += dst_bc->uv_stride; + } + + aom_yv12_extend_frame_borders_c(dst_bc); + return; + } else { + assert(!(dst_bc->flags & YV12_FLAG_HIGHBITDEPTH)); + } +#endif + + for (row = 0; row < src_bc->y_height; ++row) { + memcpy(dst, src, src_bc->y_width); + src += src_bc->y_stride; + dst += dst_bc->y_stride; + } + + src = src_bc->u_buffer; + dst = dst_bc->u_buffer; + + for (row = 0; row < src_bc->uv_height; ++row) { + memcpy(dst, src, src_bc->uv_width); + src += src_bc->uv_stride; + dst += dst_bc->uv_stride; + } + + src = src_bc->v_buffer; + dst = dst_bc->v_buffer; + + for (row = 0; row < src_bc->uv_height; ++row) { + memcpy(dst, src, src_bc->uv_width); + src += src_bc->uv_stride; + dst += dst_bc->uv_stride; + } + + aom_yv12_extend_frame_borders_c(dst_bc); +} + +void aom_yv12_copy_y_c(const YV12_BUFFER_CONFIG *src_ybc, + YV12_BUFFER_CONFIG *dst_ybc) { + int row; + const uint8_t *src = src_ybc->y_buffer; + uint8_t *dst = dst_ybc->y_buffer; + +#if CONFIG_HIGHBITDEPTH + if (src_ybc->flags & YV12_FLAG_HIGHBITDEPTH) { + const uint16_t *src16 = CONVERT_TO_SHORTPTR(src); + uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst); + for (row = 0; row < src_ybc->y_height; ++row) { + memcpy(dst16, src16, src_ybc->y_width * sizeof(uint16_t)); + src16 += src_ybc->y_stride; + dst16 += dst_ybc->y_stride; + } + return; + } +#endif // CONFIG_HIGHBITDEPTH + + for (row = 0; row < src_ybc->y_height; ++row) { + memcpy(dst, src, src_ybc->y_width); + src += src_ybc->y_stride; + dst += dst_ybc->y_stride; + } +} + +void aom_yv12_copy_u_c(const YV12_BUFFER_CONFIG *src_bc, + YV12_BUFFER_CONFIG *dst_bc) { + int row; + const uint8_t *src = src_bc->u_buffer; + uint8_t *dst = dst_bc->u_buffer; + +#if CONFIG_HIGHBITDEPTH + if (src_bc->flags & YV12_FLAG_HIGHBITDEPTH) { + const uint16_t *src16 = CONVERT_TO_SHORTPTR(src); + uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst); + for (row = 0; row < src_bc->uv_height; ++row) { + memcpy(dst16, src16, src_bc->uv_width * sizeof(uint16_t)); + src16 += src_bc->uv_stride; + dst16 += dst_bc->uv_stride; + } + return; + } +#endif // CONFIG_HIGHBITDEPTH + + for (row = 0; row < src_bc->uv_height; ++row) { + memcpy(dst, src, src_bc->uv_width); + src += src_bc->uv_stride; + dst += dst_bc->uv_stride; + } +} + +void aom_yv12_copy_v_c(const YV12_BUFFER_CONFIG *src_bc, + YV12_BUFFER_CONFIG *dst_bc) { + int row; + const uint8_t *src = src_bc->v_buffer; + uint8_t *dst = dst_bc->v_buffer; + +#if CONFIG_HIGHBITDEPTH + if (src_bc->flags & YV12_FLAG_HIGHBITDEPTH) { + const uint16_t *src16 = CONVERT_TO_SHORTPTR(src); + uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst); + for (row = 0; row < src_bc->uv_height; ++row) { + memcpy(dst16, src16, src_bc->uv_width * sizeof(uint16_t)); + src16 += src_bc->uv_stride; + dst16 += dst_bc->uv_stride; + } + return; + } +#endif // CONFIG_HIGHBITDEPTH + + for (row = 0; row < src_bc->uv_height; ++row) { + memcpy(dst, src, src_bc->uv_width); + src += src_bc->uv_stride; + dst += dst_bc->uv_stride; + } +} diff --git a/third_party/aom/aom_scale/mips/dspr2/yv12extend_dspr2.c b/third_party/aom/aom_scale/mips/dspr2/yv12extend_dspr2.c new file mode 100644 index 0000000000..51192f7b93 --- /dev/null +++ b/third_party/aom/aom_scale/mips/dspr2/yv12extend_dspr2.c @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "aom_scale/yv12config.h" +#include "aom_mem/aom_mem.h" +#include "aom_scale/aom_scale.h" + +#if HAVE_DSPR2 +static void extend_plane(uint8_t *const src, int src_stride, int width, + int height, int extend_top, int extend_left, + int extend_bottom, int extend_right) { + int i, j; + uint8_t *left_src, *right_src; + uint8_t *left_dst_start, *right_dst_start; + uint8_t *left_dst, *right_dst; + uint8_t *top_src, *bot_src; + uint8_t *top_dst, *bot_dst; + uint32_t left_pix; + uint32_t right_pix; + uint32_t linesize; + + /* copy the left and right most columns out */ + left_src = src; + right_src = src + width - 1; + left_dst_start = src - extend_left; + right_dst_start = src + width; + + for (i = height; i--;) { + left_dst = left_dst_start; + right_dst = right_dst_start; + + __asm__ __volatile__( + "lb %[left_pix], 0(%[left_src]) \n\t" + "lb %[right_pix], 0(%[right_src]) \n\t" + "replv.qb %[left_pix], %[left_pix] \n\t" + "replv.qb %[right_pix], %[right_pix] \n\t" + + : [left_pix] "=&r"(left_pix), [right_pix] "=&r"(right_pix) + : [left_src] "r"(left_src), [right_src] "r"(right_src)); + + for (j = extend_left / 4; j--;) { + __asm__ __volatile__( + "sw %[left_pix], 0(%[left_dst]) \n\t" + "sw %[right_pix], 0(%[right_dst]) \n\t" + + : + : [left_dst] "r"(left_dst), [left_pix] "r"(left_pix), + [right_dst] "r"(right_dst), [right_pix] "r"(right_pix)); + + left_dst += 4; + right_dst += 4; + } + + for (j = extend_left % 4; j--;) { + __asm__ __volatile__( + "sb %[left_pix], 0(%[left_dst]) \n\t" + "sb %[right_pix], 0(%[right_dst]) \n\t" + + : + : [left_dst] "r"(left_dst), [left_pix] "r"(left_pix), + [right_dst] "r"(right_dst), [right_pix] "r"(right_pix)); + + left_dst += 1; + right_dst += 1; + } + + left_src += src_stride; + right_src += src_stride; + left_dst_start += src_stride; + right_dst_start += src_stride; + } + + /* Now copy the top and bottom lines into each line of the respective + * borders + */ + top_src = src - extend_left; + bot_src = src + src_stride * (height - 1) - extend_left; + top_dst = src + src_stride * (-extend_top) - extend_left; + bot_dst = src + src_stride * (height)-extend_left; + linesize = extend_left + extend_right + width; + + for (i = 0; i < extend_top; i++) { + memcpy(top_dst, top_src, linesize); + top_dst += src_stride; + } + + for (i = 0; i < extend_bottom; i++) { + memcpy(bot_dst, bot_src, linesize); + bot_dst += src_stride; + } +} + +static void extend_frame(YV12_BUFFER_CONFIG *const ybf, int ext_size) { + const int c_w = ybf->uv_crop_width; + const int c_h = ybf->uv_crop_height; + const int ss_x = ybf->uv_width < ybf->y_width; + const int ss_y = ybf->uv_height < ybf->y_height; + const int c_et = ext_size >> ss_y; + const int c_el = ext_size >> ss_x; + const int c_eb = c_et + ybf->uv_height - ybf->uv_crop_height; + const int c_er = c_el + ybf->uv_width - ybf->uv_crop_width; + + assert(ybf->y_height - ybf->y_crop_height < 16); + assert(ybf->y_width - ybf->y_crop_width < 16); + assert(ybf->y_height - ybf->y_crop_height >= 0); + assert(ybf->y_width - ybf->y_crop_width >= 0); + + extend_plane(ybf->y_buffer, ybf->y_stride, ybf->y_crop_width, + ybf->y_crop_height, ext_size, ext_size, + ext_size + ybf->y_height - ybf->y_crop_height, + ext_size + ybf->y_width - ybf->y_crop_width); + + extend_plane(ybf->u_buffer, ybf->uv_stride, c_w, c_h, c_et, c_el, c_eb, c_er); + + extend_plane(ybf->v_buffer, ybf->uv_stride, c_w, c_h, c_et, c_el, c_eb, c_er); +} + +void aom_extend_frame_borders_dspr2(YV12_BUFFER_CONFIG *ybf) { + extend_frame(ybf, ybf->border); +} + +void aom_extend_frame_inner_borders_dspr2(YV12_BUFFER_CONFIG *ybf) { + const int inner_bw = (ybf->border > AOMINNERBORDERINPIXELS) + ? AOMINNERBORDERINPIXELS + : ybf->border; + extend_frame(ybf, inner_bw); +} +#endif diff --git a/third_party/aom/aom_scale/yv12config.h b/third_party/aom/aom_scale/yv12config.h new file mode 100644 index 0000000000..434dc7b4a3 --- /dev/null +++ b/third_party/aom/aom_scale/yv12config.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_SCALE_YV12CONFIG_H_ +#define AOM_SCALE_YV12CONFIG_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./aom_config.h" +#include "aom/aom_codec.h" +#include "aom/aom_frame_buffer.h" +#include "aom/aom_integer.h" + +#if CONFIG_EXT_PARTITION +#define AOMINNERBORDERINPIXELS 160 +#else +#define AOMINNERBORDERINPIXELS 96 +#endif // CONFIG_EXT_PARTITION +#define AOM_INTERP_EXTEND 4 + +// TODO(jingning): Use unified inter predictor for encoder and +// decoder during the development process. Revisit the frame border +// to improve the decoder performance. +#define AOM_BORDER_IN_PIXELS 160 + +typedef struct yv12_buffer_config { + int y_width; + int y_height; + int y_crop_width; + int y_crop_height; + int y_stride; + + int uv_width; + int uv_height; + int uv_crop_width; + int uv_crop_height; + int uv_stride; + + int alpha_width; + int alpha_height; + int alpha_stride; + + uint8_t *y_buffer; + uint8_t *u_buffer; + uint8_t *v_buffer; + uint8_t *alpha_buffer; + +#if CONFIG_HIGHBITDEPTH && CONFIG_GLOBAL_MOTION + // If the frame is stored in a 16-bit buffer, this stores an 8-bit version + // for use in global motion detection. It is allocated on-demand. + uint8_t *y_buffer_8bit; +#endif + + uint8_t *buffer_alloc; + size_t buffer_alloc_sz; + int border; + size_t frame_size; + int subsampling_x; + int subsampling_y; + unsigned int bit_depth; + aom_color_space_t color_space; + aom_color_range_t color_range; + int render_width; + int render_height; + + int corrupted; + int flags; +} YV12_BUFFER_CONFIG; + +#define YV12_FLAG_HIGHBITDEPTH 8 + +int aom_alloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height, + int ss_x, int ss_y, +#if CONFIG_HIGHBITDEPTH + int use_highbitdepth, +#endif + int border, int byte_alignment); + +// Updates the yv12 buffer config with the frame buffer. |byte_alignment| must +// be a power of 2, from 32 to 1024. 0 sets legacy alignment. If cb is not +// NULL, then libaom is using the frame buffer callbacks to handle memory. +// If cb is not NULL, libaom will call cb with minimum size in bytes needed +// to decode the current frame. If cb is NULL, libaom will allocate memory +// internally to decode the current frame. Returns 0 on success. Returns < 0 +// on failure. +int aom_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height, + int ss_x, int ss_y, +#if CONFIG_HIGHBITDEPTH + int use_highbitdepth, +#endif + int border, int byte_alignment, + aom_codec_frame_buffer_t *fb, + aom_get_frame_buffer_cb_fn_t cb, void *cb_priv); +int aom_free_frame_buffer(YV12_BUFFER_CONFIG *ybf); + +#ifdef __cplusplus +} +#endif + +#endif // AOM_SCALE_YV12CONFIG_H_ diff --git a/third_party/aom/aom_util/aom_thread.c b/third_party/aom/aom_util/aom_thread.c new file mode 100644 index 0000000000..954b8f99c3 --- /dev/null +++ b/third_party/aom/aom_util/aom_thread.c @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +// +// Multi-threaded worker +// +// Original source: +// https://chromium.googlesource.com/webm/libwebp + +#include +#include // for memset() +#include "./aom_thread.h" +#include "aom_mem/aom_mem.h" + +#if CONFIG_MULTITHREAD + +struct AVxWorkerImpl { + pthread_mutex_t mutex_; + pthread_cond_t condition_; + pthread_t thread_; +}; + +//------------------------------------------------------------------------------ + +static void execute(AVxWorker *const worker); // Forward declaration. + +static THREADFN thread_loop(void *ptr) { + AVxWorker *const worker = (AVxWorker *)ptr; + int done = 0; + while (!done) { + pthread_mutex_lock(&worker->impl_->mutex_); + while (worker->status_ == OK) { // wait in idling mode + pthread_cond_wait(&worker->impl_->condition_, &worker->impl_->mutex_); + } + if (worker->status_ == WORK) { + execute(worker); + worker->status_ = OK; + } else if (worker->status_ == NOT_OK) { // finish the worker + done = 1; + } + // signal to the main thread that we're done (for sync()) + pthread_cond_signal(&worker->impl_->condition_); + pthread_mutex_unlock(&worker->impl_->mutex_); + } + return THREAD_RETURN(NULL); // Thread is finished +} + +// main thread state control +static void change_state(AVxWorker *const worker, AVxWorkerStatus new_status) { + // No-op when attempting to change state on a thread that didn't come up. + // Checking status_ without acquiring the lock first would result in a data + // race. + if (worker->impl_ == NULL) return; + + pthread_mutex_lock(&worker->impl_->mutex_); + if (worker->status_ >= OK) { + // wait for the worker to finish + while (worker->status_ != OK) { + pthread_cond_wait(&worker->impl_->condition_, &worker->impl_->mutex_); + } + // assign new status and release the working thread if needed + if (new_status != OK) { + worker->status_ = new_status; + pthread_cond_signal(&worker->impl_->condition_); + } + } + pthread_mutex_unlock(&worker->impl_->mutex_); +} + +#endif // CONFIG_MULTITHREAD + +//------------------------------------------------------------------------------ + +static void init(AVxWorker *const worker) { + memset(worker, 0, sizeof(*worker)); + worker->status_ = NOT_OK; +} + +static int sync(AVxWorker *const worker) { +#if CONFIG_MULTITHREAD + change_state(worker, OK); +#endif + assert(worker->status_ <= OK); + return !worker->had_error; +} + +static int reset(AVxWorker *const worker) { + int ok = 1; + worker->had_error = 0; + if (worker->status_ < OK) { +#if CONFIG_MULTITHREAD + worker->impl_ = (AVxWorkerImpl *)aom_calloc(1, sizeof(*worker->impl_)); + if (worker->impl_ == NULL) { + return 0; + } + if (pthread_mutex_init(&worker->impl_->mutex_, NULL)) { + goto Error; + } + if (pthread_cond_init(&worker->impl_->condition_, NULL)) { + pthread_mutex_destroy(&worker->impl_->mutex_); + goto Error; + } + pthread_mutex_lock(&worker->impl_->mutex_); + ok = !pthread_create(&worker->impl_->thread_, NULL, thread_loop, worker); + if (ok) worker->status_ = OK; + pthread_mutex_unlock(&worker->impl_->mutex_); + if (!ok) { + pthread_mutex_destroy(&worker->impl_->mutex_); + pthread_cond_destroy(&worker->impl_->condition_); + Error: + aom_free(worker->impl_); + worker->impl_ = NULL; + return 0; + } +#else + worker->status_ = OK; +#endif + } else if (worker->status_ > OK) { + ok = sync(worker); + } + assert(!ok || (worker->status_ == OK)); + return ok; +} + +static void execute(AVxWorker *const worker) { + if (worker->hook != NULL) { + worker->had_error |= !worker->hook(worker->data1, worker->data2); + } +} + +static void launch(AVxWorker *const worker) { +#if CONFIG_MULTITHREAD + change_state(worker, WORK); +#else + execute(worker); +#endif +} + +static void end(AVxWorker *const worker) { +#if CONFIG_MULTITHREAD + if (worker->impl_ != NULL) { + change_state(worker, NOT_OK); + pthread_join(worker->impl_->thread_, NULL); + pthread_mutex_destroy(&worker->impl_->mutex_); + pthread_cond_destroy(&worker->impl_->condition_); + aom_free(worker->impl_); + worker->impl_ = NULL; + } +#else + worker->status_ = NOT_OK; + assert(worker->impl_ == NULL); +#endif + assert(worker->status_ == NOT_OK); +} + +//------------------------------------------------------------------------------ + +static AVxWorkerInterface g_worker_interface = { init, reset, sync, + launch, execute, end }; + +int aom_set_worker_interface(const AVxWorkerInterface *const winterface) { + if (winterface == NULL || winterface->init == NULL || + winterface->reset == NULL || winterface->sync == NULL || + winterface->launch == NULL || winterface->execute == NULL || + winterface->end == NULL) { + return 0; + } + g_worker_interface = *winterface; + return 1; +} + +const AVxWorkerInterface *aom_get_worker_interface(void) { + return &g_worker_interface; +} + +//------------------------------------------------------------------------------ diff --git a/third_party/aom/aom_util/aom_thread.h b/third_party/aom/aom_util/aom_thread.h new file mode 100644 index 0000000000..0ae8f2f49d --- /dev/null +++ b/third_party/aom/aom_util/aom_thread.h @@ -0,0 +1,412 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +// +// Multi-threaded worker +// +// Original source: +// https://chromium.googlesource.com/webm/libwebp + +#ifndef AOM_THREAD_H_ +#define AOM_THREAD_H_ + +#include "./aom_config.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Set maximum decode threads to be 8 due to the limit of frame buffers +// and not enough semaphores in the emulation layer on windows. +#define MAX_DECODE_THREADS 8 + +#if CONFIG_MULTITHREAD + +#if defined(_WIN32) && !HAVE_PTHREAD_H +#include // NOLINT +#include // NOLINT +#include // NOLINT +typedef HANDLE pthread_t; +typedef CRITICAL_SECTION pthread_mutex_t; + +#if _WIN32_WINNT >= 0x0600 // Windows Vista / Server 2008 or greater +#define USE_WINDOWS_CONDITION_VARIABLE +typedef CONDITION_VARIABLE pthread_cond_t; +#else +typedef struct { + HANDLE waiting_sem_; + HANDLE received_sem_; + HANDLE signal_event_; +} pthread_cond_t; +#endif // _WIN32_WINNT >= 0x600 + +#ifndef WINAPI_FAMILY_PARTITION +#define WINAPI_PARTITION_DESKTOP 1 +#define WINAPI_FAMILY_PARTITION(x) x +#endif + +#if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +#define USE_CREATE_THREAD +#endif + +//------------------------------------------------------------------------------ +// simplistic pthread emulation layer + +// _beginthreadex requires __stdcall +#define THREADFN unsigned int __stdcall +#define THREAD_RETURN(val) (unsigned int)((DWORD_PTR)val) + +#if _WIN32_WINNT >= 0x0501 // Windows XP or greater +#define WaitForSingleObject(obj, timeout) \ + WaitForSingleObjectEx(obj, timeout, FALSE /*bAlertable*/) +#endif + +static INLINE int pthread_create(pthread_t *const thread, const void *attr, + unsigned int(__stdcall *start)(void *), + void *arg) { + (void)attr; +#ifdef USE_CREATE_THREAD + *thread = CreateThread(NULL, /* lpThreadAttributes */ + 0, /* dwStackSize */ + start, arg, 0, /* dwStackSize */ + NULL); /* lpThreadId */ +#else + *thread = (pthread_t)_beginthreadex(NULL, /* void *security */ + 0, /* unsigned stack_size */ + start, arg, 0, /* unsigned initflag */ + NULL); /* unsigned *thrdaddr */ +#endif + if (*thread == NULL) return 1; + SetThreadPriority(*thread, THREAD_PRIORITY_ABOVE_NORMAL); + return 0; +} + +static INLINE int pthread_join(pthread_t thread, void **value_ptr) { + (void)value_ptr; + return (WaitForSingleObject(thread, INFINITE) != WAIT_OBJECT_0 || + CloseHandle(thread) == 0); +} + +// Mutex +static INLINE int pthread_mutex_init(pthread_mutex_t *const mutex, + void *mutexattr) { + (void)mutexattr; +#if _WIN32_WINNT >= 0x0600 // Windows Vista / Server 2008 or greater + InitializeCriticalSectionEx(mutex, 0 /*dwSpinCount*/, 0 /*Flags*/); +#else + InitializeCriticalSection(mutex); +#endif + return 0; +} + +static INLINE int pthread_mutex_trylock(pthread_mutex_t *const mutex) { + return TryEnterCriticalSection(mutex) ? 0 : EBUSY; +} + +static INLINE int pthread_mutex_lock(pthread_mutex_t *const mutex) { + EnterCriticalSection(mutex); + return 0; +} + +static INLINE int pthread_mutex_unlock(pthread_mutex_t *const mutex) { + LeaveCriticalSection(mutex); + return 0; +} + +static INLINE int pthread_mutex_destroy(pthread_mutex_t *const mutex) { + DeleteCriticalSection(mutex); + return 0; +} + +// Condition +static INLINE int pthread_cond_destroy(pthread_cond_t *const condition) { + int ok = 1; +#ifdef USE_WINDOWS_CONDITION_VARIABLE + (void)condition; +#else + ok &= (CloseHandle(condition->waiting_sem_) != 0); + ok &= (CloseHandle(condition->received_sem_) != 0); + ok &= (CloseHandle(condition->signal_event_) != 0); +#endif + return !ok; +} + +static INLINE int pthread_cond_init(pthread_cond_t *const condition, + void *cond_attr) { + (void)cond_attr; +#ifdef USE_WINDOWS_CONDITION_VARIABLE + InitializeConditionVariable(condition); +#else + condition->waiting_sem_ = CreateSemaphore(NULL, 0, MAX_DECODE_THREADS, NULL); + condition->received_sem_ = CreateSemaphore(NULL, 0, MAX_DECODE_THREADS, NULL); + condition->signal_event_ = CreateEvent(NULL, FALSE, FALSE, NULL); + if (condition->waiting_sem_ == NULL || condition->received_sem_ == NULL || + condition->signal_event_ == NULL) { + pthread_cond_destroy(condition); + return 1; + } +#endif + return 0; +} + +static INLINE int pthread_cond_signal(pthread_cond_t *const condition) { + int ok = 1; +#ifdef USE_WINDOWS_CONDITION_VARIABLE + WakeConditionVariable(condition); +#else + if (WaitForSingleObject(condition->waiting_sem_, 0) == WAIT_OBJECT_0) { + // a thread is waiting in pthread_cond_wait: allow it to be notified + ok = SetEvent(condition->signal_event_); + // wait until the event is consumed so the signaler cannot consume + // the event via its own pthread_cond_wait. + ok &= (WaitForSingleObject(condition->received_sem_, INFINITE) != + WAIT_OBJECT_0); + } +#endif + return !ok; +} + +static INLINE int pthread_cond_wait(pthread_cond_t *const condition, + pthread_mutex_t *const mutex) { + int ok; +#ifdef USE_WINDOWS_CONDITION_VARIABLE + ok = SleepConditionVariableCS(condition, mutex, INFINITE); +#else + // note that there is a consumer available so the signal isn't dropped in + // pthread_cond_signal + if (!ReleaseSemaphore(condition->waiting_sem_, 1, NULL)) return 1; + // now unlock the mutex so pthread_cond_signal may be issued + pthread_mutex_unlock(mutex); + ok = (WaitForSingleObject(condition->signal_event_, INFINITE) == + WAIT_OBJECT_0); + ok &= ReleaseSemaphore(condition->received_sem_, 1, NULL); + pthread_mutex_lock(mutex); +#endif + return !ok; +} +#elif defined(__OS2__) +#define INCL_DOS +#include // NOLINT + +#include // NOLINT +#include // NOLINT +#include // NOLINT + +#define pthread_t TID +#define pthread_mutex_t HMTX + +typedef struct { + HEV event_sem_; + HEV ack_sem_; + volatile unsigned wait_count_; +} pthread_cond_t; + +//------------------------------------------------------------------------------ +// simplistic pthread emulation layer + +#define THREADFN void * +#define THREAD_RETURN(val) (val) + +typedef struct { + void *(*start_)(void *); + void *arg_; +} thread_arg; + +static void thread_start(void *arg) { + thread_arg targ = *(thread_arg *)arg; + free(arg); + + targ.start_(targ.arg_); +} + +static INLINE int pthread_create(pthread_t *const thread, const void *attr, + void *(*start)(void *), void *arg) { + int tid; + thread_arg *targ = (thread_arg *)malloc(sizeof(*targ)); + if (targ == NULL) return 1; + + (void)attr; + + targ->start_ = start; + targ->arg_ = arg; + tid = (pthread_t)_beginthread(thread_start, NULL, 1024 * 1024, targ); + if (tid == -1) { + free(targ); + return 1; + } + + *thread = tid; + return 0; +} + +static INLINE int pthread_join(pthread_t thread, void **value_ptr) { + (void)value_ptr; + return DosWaitThread(&thread, DCWW_WAIT) != 0; +} + +// Mutex +static INLINE int pthread_mutex_init(pthread_mutex_t *const mutex, + void *mutexattr) { + (void)mutexattr; + return DosCreateMutexSem(NULL, mutex, 0, FALSE) != 0; +} + +static INLINE int pthread_mutex_trylock(pthread_mutex_t *const mutex) { + return DosRequestMutexSem(*mutex, SEM_IMMEDIATE_RETURN) == 0 ? 0 : EBUSY; +} + +static INLINE int pthread_mutex_lock(pthread_mutex_t *const mutex) { + return DosRequestMutexSem(*mutex, SEM_INDEFINITE_WAIT) != 0; +} + +static INLINE int pthread_mutex_unlock(pthread_mutex_t *const mutex) { + return DosReleaseMutexSem(*mutex) != 0; +} + +static INLINE int pthread_mutex_destroy(pthread_mutex_t *const mutex) { + return DosCloseMutexSem(*mutex) != 0; +} + +// Condition +static INLINE int pthread_cond_destroy(pthread_cond_t *const condition) { + int ok = 1; + ok &= DosCloseEventSem(condition->event_sem_) == 0; + ok &= DosCloseEventSem(condition->ack_sem_) == 0; + return !ok; +} + +static INLINE int pthread_cond_init(pthread_cond_t *const condition, + void *cond_attr) { + int ok = 1; + (void)cond_attr; + + ok &= + DosCreateEventSem(NULL, &condition->event_sem_, DCE_POSTONE, FALSE) == 0; + ok &= DosCreateEventSem(NULL, &condition->ack_sem_, DCE_POSTONE, FALSE) == 0; + if (!ok) { + pthread_cond_destroy(condition); + return 1; + } + condition->wait_count_ = 0; + return 0; +} + +static INLINE int pthread_cond_signal(pthread_cond_t *const condition) { + int ok = 1; + + if (!__atomic_cmpxchg32(&condition->wait_count_, 0, 0)) { + ok &= DosPostEventSem(condition->event_sem_) == 0; + ok &= DosWaitEventSem(condition->ack_sem_, SEM_INDEFINITE_WAIT) == 0; + } + + return !ok; +} + +static INLINE int pthread_cond_broadcast(pthread_cond_t *const condition) { + int ok = 1; + + while (!__atomic_cmpxchg32(&condition->wait_count_, 0, 0)) + ok &= pthread_cond_signal(condition) == 0; + + return !ok; +} + +static INLINE int pthread_cond_wait(pthread_cond_t *const condition, + pthread_mutex_t *const mutex) { + int ok = 1; + + __atomic_increment(&condition->wait_count_); + + ok &= pthread_mutex_unlock(mutex) == 0; + + ok &= DosWaitEventSem(condition->event_sem_, SEM_INDEFINITE_WAIT) == 0; + + __atomic_decrement(&condition->wait_count_); + + ok &= DosPostEventSem(condition->ack_sem_) == 0; + + pthread_mutex_lock(mutex); + + return !ok; +} +#else // _WIN32 +#include // NOLINT +#define THREADFN void * +#define THREAD_RETURN(val) val +#endif + +#endif // CONFIG_MULTITHREAD + +// State of the worker thread object +typedef enum { + NOT_OK = 0, // object is unusable + OK, // ready to work + WORK // busy finishing the current task +} AVxWorkerStatus; + +// Function to be called by the worker thread. Takes two opaque pointers as +// arguments (data1 and data2), and should return false in case of error. +typedef int (*AVxWorkerHook)(void *, void *); + +// Platform-dependent implementation details for the worker. +typedef struct AVxWorkerImpl AVxWorkerImpl; + +// Synchronization object used to launch job in the worker thread +typedef struct { + AVxWorkerImpl *impl_; + AVxWorkerStatus status_; + AVxWorkerHook hook; // hook to call + void *data1; // first argument passed to 'hook' + void *data2; // second argument passed to 'hook' + int had_error; // return value of the last call to 'hook' +} AVxWorker; + +// The interface for all thread-worker related functions. All these functions +// must be implemented. +typedef struct { + // Must be called first, before any other method. + void (*init)(AVxWorker *const worker); + // Must be called to initialize the object and spawn the thread. Re-entrant. + // Will potentially launch the thread. Returns false in case of error. + int (*reset)(AVxWorker *const worker); + // Makes sure the previous work is finished. Returns true if worker->had_error + // was not set and no error condition was triggered by the working thread. + int (*sync)(AVxWorker *const worker); + // Triggers the thread to call hook() with data1 and data2 arguments. These + // hook/data1/data2 values can be changed at any time before calling this + // function, but not be changed afterward until the next call to Sync(). + void (*launch)(AVxWorker *const worker); + // This function is similar to launch() except that it calls the + // hook directly instead of using a thread. Convenient to bypass the thread + // mechanism while still using the AVxWorker structs. sync() must + // still be called afterward (for error reporting). + void (*execute)(AVxWorker *const worker); + // Kill the thread and terminate the object. To use the object again, one + // must call reset() again. + void (*end)(AVxWorker *const worker); +} AVxWorkerInterface; + +// Install a new set of threading functions, overriding the defaults. This +// should be done before any workers are started, i.e., before any encoding or +// decoding takes place. The contents of the interface struct are copied, it +// is safe to free the corresponding memory after this call. This function is +// not thread-safe. Return false in case of invalid pointer or methods. +int aom_set_worker_interface(const AVxWorkerInterface *const winterface); + +// Retrieve the currently set thread worker interface. +const AVxWorkerInterface *aom_get_worker_interface(void); + +//------------------------------------------------------------------------------ + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_THREAD_H_ diff --git a/third_party/aom/aom_util/aom_util.cmake b/third_party/aom/aom_util/aom_util.cmake new file mode 100644 index 0000000000..484d9d9a04 --- /dev/null +++ b/third_party/aom/aom_util/aom_util.cmake @@ -0,0 +1,29 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +set(AOM_UTIL_SOURCES + "${AOM_ROOT}/aom_util/aom_thread.c" + "${AOM_ROOT}/aom_util/aom_thread.h" + "${AOM_ROOT}/aom_util/endian_inl.h") + +if (CONFIG_BITSTREAM_DEBUG) + set(AOM_UTIL_SOURCES + ${AOM_UTIL_SOURCES} + "${AOM_ROOT}/aom_util/debug_util.c" + "${AOM_ROOT}/aom_util/debug_util.h") +endif () + +# Creates the aom_util build target and makes libaom depend on it. The libaom +# target must exist before this function is called. +function (setup_aom_util_targets) + add_library(aom_util OBJECT ${AOM_UTIL_SOURCES}) + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_util PARENT_SCOPE) + target_sources(aom PUBLIC $) +endfunction () diff --git a/third_party/aom/aom_util/aom_util.mk b/third_party/aom/aom_util/aom_util.mk new file mode 100644 index 0000000000..14b484a153 --- /dev/null +++ b/third_party/aom/aom_util/aom_util.mk @@ -0,0 +1,18 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + +UTIL_SRCS-yes += aom_util.mk +UTIL_SRCS-yes += aom_thread.c +UTIL_SRCS-yes += aom_thread.h +UTIL_SRCS-$(CONFIG_BITSTREAM_DEBUG) += debug_util.c +UTIL_SRCS-$(CONFIG_BITSTREAM_DEBUG) += debug_util.h +UTIL_SRCS-yes += endian_inl.h diff --git a/third_party/aom/aom_util/debug_util.c b/third_party/aom/aom_util/debug_util.c new file mode 100644 index 0000000000..071d66976c --- /dev/null +++ b/third_party/aom/aom_util/debug_util.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include "aom_util/debug_util.h" + +#define QUEUE_MAX_SIZE 2000000 +static int result_queue[QUEUE_MAX_SIZE]; +#if CONFIG_DAALA_EC +static int nsymbs_queue[QUEUE_MAX_SIZE]; +static aom_cdf_prob cdf_queue[QUEUE_MAX_SIZE][16]; +#else +static int prob_queue[QUEUE_MAX_SIZE]; +#endif + +static int queue_r = 0; +static int queue_w = 0; +static int queue_prev_w = -1; +static int skip_r = 0; +static int skip_w = 0; + +static int frame_idx_w = 0; + +static int frame_idx_r = 0; + +void bitstream_queue_set_frame_write(int frame_idx) { frame_idx_w = frame_idx; } + +int bitstream_queue_get_frame_write(void) { return frame_idx_w; } + +void bitstream_queue_set_frame_read(int frame_idx) { frame_idx_r = frame_idx; } + +int bitstream_queue_get_frame_read(void) { return frame_idx_r; } + +void bitstream_queue_set_skip_write(int skip) { skip_w = skip; } + +void bitstream_queue_set_skip_read(int skip) { skip_r = skip; } + +void bitstream_queue_record_write(void) { queue_prev_w = queue_w; } + +void bitstream_queue_reset_write(void) { queue_w = queue_prev_w; } + +int bitstream_queue_get_write(void) { return queue_w; } + +int bitstream_queue_get_read(void) { return queue_r; } + +void bitstream_queue_pop(int *result, +#if CONFIG_DAALA_EC + aom_cdf_prob *cdf, int *nsymbs) { +#else + int *prob) { +#endif // CONFIG_DAALA_EC + if (!skip_r) { + if (queue_w == queue_r) { + printf("buffer underflow queue_w %d queue_r %d\n", queue_w, queue_r); + assert(0); + } + *result = result_queue[queue_r]; +#if CONFIG_DAALA_EC + *nsymbs = nsymbs_queue[queue_r]; + memcpy(cdf, cdf_queue[queue_r], *nsymbs * sizeof(*cdf)); +#else + *prob = prob_queue[queue_r]; +#endif // CONFIG_DAALA_EC + queue_r = (queue_r + 1) % QUEUE_MAX_SIZE; + } +} + +void bitstream_queue_push(int result, +#if CONFIG_DAALA_EC + const aom_cdf_prob *cdf, int nsymbs) { +#else + int prob) { +#endif // CONFIG_DAALA_EC + if (!skip_w) { + result_queue[queue_w] = result; +#if CONFIG_DAALA_EC + nsymbs_queue[queue_w] = nsymbs; + memcpy(cdf_queue[queue_w], cdf, nsymbs * sizeof(*cdf)); +#else + prob_queue[queue_w] = prob; +#endif // CONFIG_DAALA_EC + queue_w = (queue_w + 1) % QUEUE_MAX_SIZE; + if (queue_w == queue_r) { + printf("buffer overflow queue_w %d queue_r %d\n", queue_w, queue_r); + assert(0); + } + } +} diff --git a/third_party/aom/aom_util/debug_util.h b/third_party/aom/aom_util/debug_util.h new file mode 100644 index 0000000000..2ed56ea22b --- /dev/null +++ b/third_party/aom/aom_util/debug_util.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_UTIL_DEBUG_UTIL_H_ +#define AOM_UTIL_DEBUG_UTIL_H_ + +#include "./aom_config.h" +#if CONFIG_DAALA_EC +#include "aom_dsp/prob.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* This is a debug tool used to detect bitstream error. On encoder side, it + * pushes each bit and probability into a queue before the bit is written into + * the Arithmetic coder. On decoder side, whenever a bit is read out from the + * Arithmetic coder, it pops out the reference bit and probability from the + * queue as well. If the two results do not match, this debug tool will report + * an error. This tool can be used to pin down the bitstream error precisely. + * By combining gdb's backtrace method, we can detect which module causes the + * bitstream error. */ +int bitstream_queue_get_write(void); +int bitstream_queue_get_read(void); +void bitstream_queue_record_write(void); +void bitstream_queue_reset_write(void); +#if CONFIG_DAALA_EC +void bitstream_queue_pop(int *result, aom_cdf_prob *cdf, int *nsymbs); +void bitstream_queue_push(int result, const aom_cdf_prob *cdf, int nsymbs); +#else +void bitstream_queue_pop(int *result, int *prob); +void bitstream_queue_push(int result, int prob); +#endif +void bitstream_queue_set_skip_write(int skip); +void bitstream_queue_set_skip_read(int skip); +void bitstream_queue_set_frame_write(int frame_idx); +int bitstream_queue_get_frame_write(void); +void bitstream_queue_set_frame_read(int frame_idx); +int bitstream_queue_get_frame_read(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_UTIL_DEBUG_UTIL_H_ diff --git a/third_party/aom/aom_util/endian_inl.h b/third_party/aom/aom_util/endian_inl.h new file mode 100644 index 0000000000..17a238649c --- /dev/null +++ b/third_party/aom/aom_util/endian_inl.h @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +// +// Endian related functions. + +#ifndef AOM_UTIL_ENDIAN_INL_H_ +#define AOM_UTIL_ENDIAN_INL_H_ + +#include +#include "./aom_config.h" +#include "aom/aom_integer.h" + +#if defined(__GNUC__) +#define LOCAL_GCC_VERSION ((__GNUC__ << 8) | __GNUC_MINOR__) +#define LOCAL_GCC_PREREQ(maj, min) (LOCAL_GCC_VERSION >= (((maj) << 8) | (min))) +#else +#define LOCAL_GCC_VERSION 0 +#define LOCAL_GCC_PREREQ(maj, min) 0 +#endif + +// handle clang compatibility +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +// some endian fix (e.g.: mips-gcc doesn't define __BIG_ENDIAN__) +#if !defined(WORDS_BIGENDIAN) && \ + (defined(__BIG_ENDIAN__) || defined(_M_PPC) || \ + (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))) +#define WORDS_BIGENDIAN +#endif + +#if defined(WORDS_BIGENDIAN) +#define HToLE32 BSwap32 +#define HToLE16 BSwap16 +#define HToBE64(x) (x) +#define HToBE32(x) (x) +#else +#define HToLE32(x) (x) +#define HToLE16(x) (x) +#define HToBE64(X) BSwap64(X) +#define HToBE32(X) BSwap32(X) +#endif + +#if LOCAL_GCC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16) +#define HAVE_BUILTIN_BSWAP16 +#endif + +#if LOCAL_GCC_PREREQ(4, 3) || __has_builtin(__builtin_bswap32) +#define HAVE_BUILTIN_BSWAP32 +#endif + +#if LOCAL_GCC_PREREQ(4, 3) || __has_builtin(__builtin_bswap64) +#define HAVE_BUILTIN_BSWAP64 +#endif + +#if HAVE_MIPS32 && defined(__mips__) && !defined(__mips64) && \ + defined(__mips_isa_rev) && (__mips_isa_rev >= 2) && (__mips_isa_rev < 6) +#define AOM_USE_MIPS32_R2 +#endif + +static INLINE uint16_t BSwap16(uint16_t x) { +#if defined(HAVE_BUILTIN_BSWAP16) + return __builtin_bswap16(x); +#elif defined(_MSC_VER) + return _byteswap_ushort(x); +#else + // gcc will recognize a 'rorw $8, ...' here: + return (x >> 8) | ((x & 0xff) << 8); +#endif // HAVE_BUILTIN_BSWAP16 +} + +static INLINE uint32_t BSwap32(uint32_t x) { +#if defined(AOM_USE_MIPS32_R2) + uint32_t ret; + __asm__ volatile( + "wsbh %[ret], %[x] \n\t" + "rotr %[ret], %[ret], 16 \n\t" + : [ret] "=r"(ret) + : [x] "r"(x)); + return ret; +#elif defined(HAVE_BUILTIN_BSWAP32) + return __builtin_bswap32(x); +#elif defined(__i386__) || defined(__x86_64__) + uint32_t swapped_bytes; + __asm__ volatile("bswap %0" : "=r"(swapped_bytes) : "0"(x)); + return swapped_bytes; +#elif defined(_MSC_VER) + return (uint32_t)_byteswap_ulong(x); +#else + return (x >> 24) | ((x >> 8) & 0xff00) | ((x << 8) & 0xff0000) | (x << 24); +#endif // HAVE_BUILTIN_BSWAP32 +} + +static INLINE uint64_t BSwap64(uint64_t x) { +#if defined(HAVE_BUILTIN_BSWAP64) + return __builtin_bswap64(x); +#elif defined(__x86_64__) + uint64_t swapped_bytes; + __asm__ volatile("bswapq %0" : "=r"(swapped_bytes) : "0"(x)); + return swapped_bytes; +#elif defined(_MSC_VER) + return (uint64_t)_byteswap_uint64(x); +#else // generic code for swapping 64-bit values (suggested by bdb@) + x = ((x & 0xffffffff00000000ull) >> 32) | ((x & 0x00000000ffffffffull) << 32); + x = ((x & 0xffff0000ffff0000ull) >> 16) | ((x & 0x0000ffff0000ffffull) << 16); + x = ((x & 0xff00ff00ff00ff00ull) >> 8) | ((x & 0x00ff00ff00ff00ffull) << 8); + return x; +#endif // HAVE_BUILTIN_BSWAP64 +} + +#endif // AOM_UTIL_ENDIAN_INL_H_ diff --git a/third_party/aom/aomdec.c b/third_party/aom/aomdec.c new file mode 100644 index 0000000000..f74baddb7a --- /dev/null +++ b/third_party/aom/aomdec.c @@ -0,0 +1,1070 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include +#include +#include + +#include "./aom_config.h" + +#if CONFIG_OS_SUPPORT +#if HAVE_UNISTD_H +#include // NOLINT +#elif !defined(STDOUT_FILENO) +#define STDOUT_FILENO 1 +#endif +#endif + +#if CONFIG_LIBYUV +#include "third_party/libyuv/include/libyuv/scale.h" +#endif + +#include "./args.h" +#include "./ivfdec.h" + +#include "aom/aom_decoder.h" +#include "aom_ports/mem_ops.h" +#include "aom_ports/aom_timer.h" + +#if CONFIG_AV1_DECODER +#include "aom/aomdx.h" +#endif + +#include "./md5_utils.h" + +#include "./tools_common.h" +#if CONFIG_WEBM_IO +#include "./webmdec.h" +#endif +#include "./y4menc.h" + +static const char *exec_name; + +struct AvxDecInputContext { + struct AvxInputContext *aom_input_ctx; + struct WebmInputContext *webm_ctx; +}; + +static const arg_def_t looparg = + ARG_DEF(NULL, "loops", 1, "Number of times to decode the file"); +static const arg_def_t codecarg = ARG_DEF(NULL, "codec", 1, "Codec to use"); +static const arg_def_t use_yv12 = + ARG_DEF(NULL, "yv12", 0, "Output raw YV12 frames"); +static const arg_def_t use_i420 = + ARG_DEF(NULL, "i420", 0, "Output raw I420 frames"); +static const arg_def_t flipuvarg = + ARG_DEF(NULL, "flipuv", 0, "Flip the chroma planes in the output"); +static const arg_def_t rawvideo = + ARG_DEF(NULL, "rawvideo", 0, "Output raw YUV frames"); +static const arg_def_t noblitarg = + ARG_DEF(NULL, "noblit", 0, "Don't process the decoded frames"); +static const arg_def_t progressarg = + ARG_DEF(NULL, "progress", 0, "Show progress after each frame decodes"); +static const arg_def_t limitarg = + ARG_DEF(NULL, "limit", 1, "Stop decoding after n frames"); +static const arg_def_t skiparg = + ARG_DEF(NULL, "skip", 1, "Skip the first n input frames"); +static const arg_def_t postprocarg = + ARG_DEF(NULL, "postproc", 0, "Postprocess decoded frames"); +static const arg_def_t summaryarg = + ARG_DEF(NULL, "summary", 0, "Show timing summary"); +static const arg_def_t outputfile = + ARG_DEF("o", "output", 1, "Output file name pattern (see below)"); +static const arg_def_t threadsarg = + ARG_DEF("t", "threads", 1, "Max threads to use"); +static const arg_def_t frameparallelarg = + ARG_DEF(NULL, "frame-parallel", 0, "Frame parallel decode"); +static const arg_def_t verbosearg = + ARG_DEF("v", "verbose", 0, "Show version string"); +static const arg_def_t error_concealment = + ARG_DEF(NULL, "error-concealment", 0, "Enable decoder error-concealment"); +static const arg_def_t scalearg = + ARG_DEF("S", "scale", 0, "Scale output frames uniformly"); +static const arg_def_t continuearg = + ARG_DEF("k", "keep-going", 0, "(debug) Continue decoding after error"); +static const arg_def_t fb_arg = + ARG_DEF(NULL, "frame-buffers", 1, "Number of frame buffers to use"); +static const arg_def_t md5arg = + ARG_DEF(NULL, "md5", 0, "Compute the MD5 sum of the decoded frame"); +static const arg_def_t framestatsarg = + ARG_DEF(NULL, "framestats", 1, "Output per-frame stats (.csv format)"); +#if CONFIG_HIGHBITDEPTH +static const arg_def_t outbitdeptharg = + ARG_DEF(NULL, "output-bit-depth", 1, "Output bit-depth for decoded frames"); +#endif +#if CONFIG_EXT_TILE +static const arg_def_t tiler = ARG_DEF(NULL, "tile-row", 1, + "Row index of tile to decode " + "(-1 for all rows)"); +static const arg_def_t tilec = ARG_DEF(NULL, "tile-column", 1, + "Column index of tile to decode " + "(-1 for all columns)"); +#endif // CONFIG_EXT_TILE + +static const arg_def_t *all_args[] = { &codecarg, + &use_yv12, + &use_i420, + &flipuvarg, + &rawvideo, + &noblitarg, + &progressarg, + &limitarg, + &skiparg, + &postprocarg, + &summaryarg, + &outputfile, + &threadsarg, + &frameparallelarg, + &verbosearg, + &scalearg, + &fb_arg, + &md5arg, + &framestatsarg, + &error_concealment, + &continuearg, +#if CONFIG_HIGHBITDEPTH + &outbitdeptharg, +#endif +#if CONFIG_EXT_TILE + &tiler, + &tilec, +#endif // CONFIG_EXT_TILE + NULL }; + +#if CONFIG_LIBYUV +static INLINE int libyuv_scale(aom_image_t *src, aom_image_t *dst, + FilterModeEnum mode) { +#if CONFIG_HIGHBITDEPTH + if (src->fmt == AOM_IMG_FMT_I42016) { + assert(dst->fmt == AOM_IMG_FMT_I42016); + return I420Scale_16( + (uint16_t *)src->planes[AOM_PLANE_Y], src->stride[AOM_PLANE_Y] / 2, + (uint16_t *)src->planes[AOM_PLANE_U], src->stride[AOM_PLANE_U] / 2, + (uint16_t *)src->planes[AOM_PLANE_V], src->stride[AOM_PLANE_V] / 2, + src->d_w, src->d_h, (uint16_t *)dst->planes[AOM_PLANE_Y], + dst->stride[AOM_PLANE_Y] / 2, (uint16_t *)dst->planes[AOM_PLANE_U], + dst->stride[AOM_PLANE_U] / 2, (uint16_t *)dst->planes[AOM_PLANE_V], + dst->stride[AOM_PLANE_V] / 2, dst->d_w, dst->d_h, mode); + } +#endif + assert(src->fmt == AOM_IMG_FMT_I420); + assert(dst->fmt == AOM_IMG_FMT_I420); + return I420Scale(src->planes[AOM_PLANE_Y], src->stride[AOM_PLANE_Y], + src->planes[AOM_PLANE_U], src->stride[AOM_PLANE_U], + src->planes[AOM_PLANE_V], src->stride[AOM_PLANE_V], src->d_w, + src->d_h, dst->planes[AOM_PLANE_Y], dst->stride[AOM_PLANE_Y], + dst->planes[AOM_PLANE_U], dst->stride[AOM_PLANE_U], + dst->planes[AOM_PLANE_V], dst->stride[AOM_PLANE_V], dst->d_w, + dst->d_h, mode); +} +#endif + +void usage_exit(void) { + int i; + + fprintf(stderr, + "Usage: %s filename\n\n" + "Options:\n", + exec_name); + arg_show_usage(stderr, all_args); + fprintf(stderr, + "\nOutput File Patterns:\n\n" + " The -o argument specifies the name of the file(s) to " + "write to. If the\n argument does not include any escape " + "characters, the output will be\n written to a single file. " + "Otherwise, the filename will be calculated by\n expanding " + "the following escape characters:\n"); + fprintf(stderr, + "\n\t%%w - Frame width" + "\n\t%%h - Frame height" + "\n\t%% - Frame number, zero padded to places (1..9)" + "\n\n Pattern arguments are only supported in conjunction " + "with the --yv12 and\n --i420 options. If the -o option is " + "not specified, the output will be\n directed to stdout.\n"); + fprintf(stderr, "\nIncluded decoders:\n\n"); + + for (i = 0; i < get_aom_decoder_count(); ++i) { + const AvxInterface *const decoder = get_aom_decoder_by_index(i); + fprintf(stderr, " %-6s - %s\n", decoder->name, + aom_codec_iface_name(decoder->codec_interface())); + } + + exit(EXIT_FAILURE); +} + +static int raw_read_frame(FILE *infile, uint8_t **buffer, size_t *bytes_read, + size_t *buffer_size) { + char raw_hdr[RAW_FRAME_HDR_SZ]; + size_t frame_size = 0; + + if (fread(raw_hdr, RAW_FRAME_HDR_SZ, 1, infile) != 1) { + if (!feof(infile)) warn("Failed to read RAW frame size\n"); + } else { + const size_t kCorruptFrameThreshold = 256 * 1024 * 1024; + const size_t kFrameTooSmallThreshold = 256 * 1024; + frame_size = mem_get_le32(raw_hdr); + + if (frame_size > kCorruptFrameThreshold) { + warn("Read invalid frame size (%u)\n", (unsigned int)frame_size); + frame_size = 0; + } + + if (frame_size < kFrameTooSmallThreshold) { + warn("Warning: Read invalid frame size (%u) - not a raw file?\n", + (unsigned int)frame_size); + } + + if (frame_size > *buffer_size) { + uint8_t *new_buf = realloc(*buffer, 2 * frame_size); + if (new_buf) { + *buffer = new_buf; + *buffer_size = 2 * frame_size; + } else { + warn("Failed to allocate compressed data buffer\n"); + frame_size = 0; + } + } + } + + if (!feof(infile)) { + if (fread(*buffer, 1, frame_size, infile) != frame_size) { + warn("Failed to read full frame\n"); + return 1; + } + *bytes_read = frame_size; + } + + return 0; +} + +static int read_frame(struct AvxDecInputContext *input, uint8_t **buf, + size_t *bytes_in_buffer, size_t *buffer_size) { + switch (input->aom_input_ctx->file_type) { +#if CONFIG_WEBM_IO + case FILE_TYPE_WEBM: + return webm_read_frame(input->webm_ctx, buf, bytes_in_buffer); +#endif + case FILE_TYPE_RAW: + return raw_read_frame(input->aom_input_ctx->file, buf, bytes_in_buffer, + buffer_size); + case FILE_TYPE_IVF: + return ivf_read_frame(input->aom_input_ctx->file, buf, bytes_in_buffer, + buffer_size); + default: return 1; + } +} + +static void update_image_md5(const aom_image_t *img, const int planes[3], + MD5Context *md5) { + int i, y; + + for (i = 0; i < 3; ++i) { + const int plane = planes[i]; + const unsigned char *buf = img->planes[plane]; + const int stride = img->stride[plane]; + const int w = aom_img_plane_width(img, plane) * + ((img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1); + const int h = aom_img_plane_height(img, plane); + + for (y = 0; y < h; ++y) { + MD5Update(md5, buf, w); + buf += stride; + } + } +} + +static void write_image_file(const aom_image_t *img, const int planes[3], + FILE *file) { + int i, y; +#if CONFIG_HIGHBITDEPTH + const int bytes_per_sample = ((img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1); +#else + const int bytes_per_sample = 1; +#endif + + for (i = 0; i < 3; ++i) { + const int plane = planes[i]; + const unsigned char *buf = img->planes[plane]; + const int stride = img->stride[plane]; + const int w = aom_img_plane_width(img, plane); + const int h = aom_img_plane_height(img, plane); + + for (y = 0; y < h; ++y) { + fwrite(buf, bytes_per_sample, w, file); + buf += stride; + } + } +} + +static int file_is_raw(struct AvxInputContext *input) { + uint8_t buf[32]; + int is_raw = 0; + aom_codec_stream_info_t si; + + si.sz = sizeof(si); + + if (fread(buf, 1, 32, input->file) == 32) { + int i; + + if (mem_get_le32(buf) < 256 * 1024 * 1024) { + for (i = 0; i < get_aom_decoder_count(); ++i) { + const AvxInterface *const decoder = get_aom_decoder_by_index(i); + if (!aom_codec_peek_stream_info(decoder->codec_interface(), buf + 4, + 32 - 4, &si)) { + is_raw = 1; + input->fourcc = decoder->fourcc; + input->width = si.w; + input->height = si.h; + input->framerate.numerator = 30; + input->framerate.denominator = 1; + break; + } + } + } + } + + rewind(input->file); + return is_raw; +} + +static void show_progress(int frame_in, int frame_out, uint64_t dx_time) { + fprintf(stderr, + "%d decoded frames/%d showed frames in %" PRId64 " us (%.2f fps)\r", + frame_in, frame_out, dx_time, + (double)frame_out * 1000000.0 / (double)dx_time); +} + +struct ExternalFrameBuffer { + uint8_t *data; + size_t size; + int in_use; +}; + +struct ExternalFrameBufferList { + int num_external_frame_buffers; + struct ExternalFrameBuffer *ext_fb; +}; + +// Callback used by libaom to request an external frame buffer. |cb_priv| +// Application private data passed into the set function. |min_size| is the +// minimum size in bytes needed to decode the next frame. |fb| pointer to the +// frame buffer. +static int get_av1_frame_buffer(void *cb_priv, size_t min_size, + aom_codec_frame_buffer_t *fb) { + int i; + struct ExternalFrameBufferList *const ext_fb_list = + (struct ExternalFrameBufferList *)cb_priv; + if (ext_fb_list == NULL) return -1; + + // Find a free frame buffer. + for (i = 0; i < ext_fb_list->num_external_frame_buffers; ++i) { + if (!ext_fb_list->ext_fb[i].in_use) break; + } + + if (i == ext_fb_list->num_external_frame_buffers) return -1; + + if (ext_fb_list->ext_fb[i].size < min_size) { + free(ext_fb_list->ext_fb[i].data); + ext_fb_list->ext_fb[i].data = (uint8_t *)calloc(min_size, sizeof(uint8_t)); + if (!ext_fb_list->ext_fb[i].data) return -1; + + ext_fb_list->ext_fb[i].size = min_size; + } + + fb->data = ext_fb_list->ext_fb[i].data; + fb->size = ext_fb_list->ext_fb[i].size; + ext_fb_list->ext_fb[i].in_use = 1; + + // Set the frame buffer's private data to point at the external frame buffer. + fb->priv = &ext_fb_list->ext_fb[i]; + return 0; +} + +// Callback used by libaom when there are no references to the frame buffer. +// |cb_priv| user private data passed into the set function. |fb| pointer +// to the frame buffer. +static int release_av1_frame_buffer(void *cb_priv, + aom_codec_frame_buffer_t *fb) { + struct ExternalFrameBuffer *const ext_fb = + (struct ExternalFrameBuffer *)fb->priv; + (void)cb_priv; + ext_fb->in_use = 0; + return 0; +} + +static void generate_filename(const char *pattern, char *out, size_t q_len, + unsigned int d_w, unsigned int d_h, + unsigned int frame_in) { + const char *p = pattern; + char *q = out; + + do { + char *next_pat = strchr(p, '%'); + + if (p == next_pat) { + size_t pat_len; + + /* parse the pattern */ + q[q_len - 1] = '\0'; + switch (p[1]) { + case 'w': snprintf(q, q_len - 1, "%d", d_w); break; + case 'h': snprintf(q, q_len - 1, "%d", d_h); break; + case '1': snprintf(q, q_len - 1, "%d", frame_in); break; + case '2': snprintf(q, q_len - 1, "%02d", frame_in); break; + case '3': snprintf(q, q_len - 1, "%03d", frame_in); break; + case '4': snprintf(q, q_len - 1, "%04d", frame_in); break; + case '5': snprintf(q, q_len - 1, "%05d", frame_in); break; + case '6': snprintf(q, q_len - 1, "%06d", frame_in); break; + case '7': snprintf(q, q_len - 1, "%07d", frame_in); break; + case '8': snprintf(q, q_len - 1, "%08d", frame_in); break; + case '9': snprintf(q, q_len - 1, "%09d", frame_in); break; + default: die("Unrecognized pattern %%%c\n", p[1]); break; + } + + pat_len = strlen(q); + if (pat_len >= q_len - 1) die("Output filename too long.\n"); + q += pat_len; + p += 2; + q_len -= pat_len; + } else { + size_t copy_len; + + /* copy the next segment */ + if (!next_pat) + copy_len = strlen(p); + else + copy_len = next_pat - p; + + if (copy_len >= q_len - 1) die("Output filename too long.\n"); + + memcpy(q, p, copy_len); + q[copy_len] = '\0'; + q += copy_len; + p += copy_len; + q_len -= copy_len; + } + } while (*p); +} + +static int is_single_file(const char *outfile_pattern) { + const char *p = outfile_pattern; + + do { + p = strchr(p, '%'); + if (p && p[1] >= '1' && p[1] <= '9') + return 0; // pattern contains sequence number, so it's not unique + if (p) p++; + } while (p); + + return 1; +} + +static void print_md5(unsigned char digest[16], const char *filename) { + int i; + + for (i = 0; i < 16; ++i) printf("%02x", digest[i]); + printf(" %s\n", filename); +} + +static FILE *open_outfile(const char *name) { + if (strcmp("-", name) == 0) { + set_binary_mode(stdout); + return stdout; + } else { + FILE *file = fopen(name, "wb"); + if (!file) fatal("Failed to open output file '%s'", name); + return file; + } +} + +#if CONFIG_HIGHBITDEPTH +static int img_shifted_realloc_required(const aom_image_t *img, + const aom_image_t *shifted, + aom_img_fmt_t required_fmt) { + return img->d_w != shifted->d_w || img->d_h != shifted->d_h || + required_fmt != shifted->fmt; +} +#endif + +static int main_loop(int argc, const char **argv_) { + aom_codec_ctx_t decoder; + char *fn = NULL; + int i; + int ret = EXIT_FAILURE; + uint8_t *buf = NULL; + size_t bytes_in_buffer = 0, buffer_size = 0; + FILE *infile; + int frame_in = 0, frame_out = 0, flipuv = 0, noblit = 0; + int do_md5 = 0, progress = 0, frame_parallel = 0; + int stop_after = 0, postproc = 0, summary = 0, quiet = 1; + int arg_skip = 0; + int ec_enabled = 0; + int keep_going = 0; + const AvxInterface *interface = NULL; + const AvxInterface *fourcc_interface = NULL; + uint64_t dx_time = 0; + struct arg arg; + char **argv, **argi, **argj; + + int single_file; + int use_y4m = 1; + int opt_yv12 = 0; + int opt_i420 = 0; + aom_codec_dec_cfg_t cfg = { 0, 0, 0 }; +#if CONFIG_HIGHBITDEPTH + unsigned int output_bit_depth = 0; +#endif +#if CONFIG_EXT_TILE + int tile_row = -1; + int tile_col = -1; +#endif // CONFIG_EXT_TILE + int frames_corrupted = 0; + int dec_flags = 0; + int do_scale = 0; + aom_image_t *scaled_img = NULL; +#if CONFIG_HIGHBITDEPTH + aom_image_t *img_shifted = NULL; +#endif + int frame_avail, got_data, flush_decoder = 0; + int num_external_frame_buffers = 0; + struct ExternalFrameBufferList ext_fb_list = { 0, NULL }; + + const char *outfile_pattern = NULL; + char outfile_name[PATH_MAX] = { 0 }; + FILE *outfile = NULL; + + FILE *framestats_file = NULL; + + MD5Context md5_ctx; + unsigned char md5_digest[16]; + + struct AvxDecInputContext input = { NULL, NULL }; + struct AvxInputContext aom_input_ctx; +#if CONFIG_WEBM_IO + struct WebmInputContext webm_ctx; + memset(&(webm_ctx), 0, sizeof(webm_ctx)); + input.webm_ctx = &webm_ctx; +#endif + input.aom_input_ctx = &aom_input_ctx; + + /* Parse command line */ + exec_name = argv_[0]; + argv = argv_dup(argc - 1, argv_ + 1); + + for (argi = argj = argv; (*argj = *argi); argi += arg.argv_step) { + memset(&arg, 0, sizeof(arg)); + arg.argv_step = 1; + + if (arg_match(&arg, &codecarg, argi)) { + interface = get_aom_decoder_by_name(arg.val); + if (!interface) + die("Error: Unrecognized argument (%s) to --codec\n", arg.val); + } else if (arg_match(&arg, &looparg, argi)) { + // no-op + } else if (arg_match(&arg, &outputfile, argi)) { + outfile_pattern = arg.val; + } else if (arg_match(&arg, &use_yv12, argi)) { + use_y4m = 0; + flipuv = 1; + opt_yv12 = 1; + } else if (arg_match(&arg, &use_i420, argi)) { + use_y4m = 0; + flipuv = 0; + opt_i420 = 1; + } else if (arg_match(&arg, &rawvideo, argi)) { + use_y4m = 0; + } else if (arg_match(&arg, &flipuvarg, argi)) { + flipuv = 1; + } else if (arg_match(&arg, &noblitarg, argi)) { + noblit = 1; + } else if (arg_match(&arg, &progressarg, argi)) { + progress = 1; + } else if (arg_match(&arg, &limitarg, argi)) { + stop_after = arg_parse_uint(&arg); + } else if (arg_match(&arg, &skiparg, argi)) { + arg_skip = arg_parse_uint(&arg); + } else if (arg_match(&arg, &postprocarg, argi)) { + postproc = 1; + } else if (arg_match(&arg, &md5arg, argi)) { + do_md5 = 1; + } else if (arg_match(&arg, &framestatsarg, argi)) { + framestats_file = fopen(arg.val, "w"); + if (!framestats_file) { + die("Error: Could not open --framestats file (%s) for writing.\n", + arg.val); + } + } else if (arg_match(&arg, &summaryarg, argi)) { + summary = 1; + } else if (arg_match(&arg, &threadsarg, argi)) { + cfg.threads = arg_parse_uint(&arg); + } +#if CONFIG_AV1_DECODER + else if (arg_match(&arg, &frameparallelarg, argi)) + frame_parallel = 1; +#endif + else if (arg_match(&arg, &verbosearg, argi)) + quiet = 0; + else if (arg_match(&arg, &scalearg, argi)) + do_scale = 1; + else if (arg_match(&arg, &fb_arg, argi)) + num_external_frame_buffers = arg_parse_uint(&arg); + else if (arg_match(&arg, &continuearg, argi)) + keep_going = 1; +#if CONFIG_HIGHBITDEPTH + else if (arg_match(&arg, &outbitdeptharg, argi)) { + output_bit_depth = arg_parse_uint(&arg); + } +#endif +#if CONFIG_EXT_TILE + else if (arg_match(&arg, &tiler, argi)) + tile_row = arg_parse_int(&arg); + else if (arg_match(&arg, &tilec, argi)) + tile_col = arg_parse_int(&arg); +#endif // CONFIG_EXT_TILE + else + argj++; + } + + /* Check for unrecognized options */ + for (argi = argv; *argi; argi++) + if (argi[0][0] == '-' && strlen(argi[0]) > 1) + die("Error: Unrecognized option %s\n", *argi); + + /* Handle non-option arguments */ + fn = argv[0]; + + if (!fn) { + free(argv); + usage_exit(); + } + /* Open file */ + infile = strcmp(fn, "-") ? fopen(fn, "rb") : set_binary_mode(stdin); + + if (!infile) { + fatal("Failed to open input file '%s'", strcmp(fn, "-") ? fn : "stdin"); + } +#if CONFIG_OS_SUPPORT + /* Make sure we don't dump to the terminal, unless forced to with -o - */ + if (!outfile_pattern && isatty(STDOUT_FILENO) && !do_md5 && !noblit) { + fprintf(stderr, + "Not dumping raw video to your terminal. Use '-o -' to " + "override.\n"); + return EXIT_FAILURE; + } +#endif + input.aom_input_ctx->file = infile; + if (file_is_ivf(input.aom_input_ctx)) + input.aom_input_ctx->file_type = FILE_TYPE_IVF; +#if CONFIG_WEBM_IO + else if (file_is_webm(input.webm_ctx, input.aom_input_ctx)) + input.aom_input_ctx->file_type = FILE_TYPE_WEBM; +#endif + else if (file_is_raw(input.aom_input_ctx)) + input.aom_input_ctx->file_type = FILE_TYPE_RAW; + else { + fprintf(stderr, "Unrecognized input file type.\n"); +#if !CONFIG_WEBM_IO + fprintf(stderr, "aomdec was built without WebM container support.\n"); +#endif + return EXIT_FAILURE; + } + + outfile_pattern = outfile_pattern ? outfile_pattern : "-"; + single_file = is_single_file(outfile_pattern); + + if (!noblit && single_file) { + generate_filename(outfile_pattern, outfile_name, PATH_MAX, + aom_input_ctx.width, aom_input_ctx.height, 0); + if (do_md5) + MD5Init(&md5_ctx); + else + outfile = open_outfile(outfile_name); + } + + if (use_y4m && !noblit) { + if (!single_file) { + fprintf(stderr, + "YUV4MPEG2 not supported with output patterns," + " try --i420 or --yv12 or --rawvideo.\n"); + return EXIT_FAILURE; + } + +#if CONFIG_WEBM_IO + if (aom_input_ctx.file_type == FILE_TYPE_WEBM) { + if (webm_guess_framerate(input.webm_ctx, input.aom_input_ctx)) { + fprintf(stderr, + "Failed to guess framerate -- error parsing " + "webm file?\n"); + return EXIT_FAILURE; + } + } +#endif + } + + fourcc_interface = get_aom_decoder_by_fourcc(aom_input_ctx.fourcc); + if (interface && fourcc_interface && interface != fourcc_interface) + warn("Header indicates codec: %s\n", fourcc_interface->name); + else + interface = fourcc_interface; + + if (!interface) interface = get_aom_decoder_by_index(0); + + dec_flags = (postproc ? AOM_CODEC_USE_POSTPROC : 0) | + (ec_enabled ? AOM_CODEC_USE_ERROR_CONCEALMENT : 0) | + (frame_parallel ? AOM_CODEC_USE_FRAME_THREADING : 0); + if (aom_codec_dec_init(&decoder, interface->codec_interface(), &cfg, + dec_flags)) { + fprintf(stderr, "Failed to initialize decoder: %s\n", + aom_codec_error(&decoder)); + goto fail2; + } + + if (!quiet) fprintf(stderr, "%s\n", decoder.name); + +#if CONFIG_AV1_DECODER && CONFIG_EXT_TILE + if (aom_codec_control(&decoder, AV1_SET_DECODE_TILE_ROW, tile_row)) { + fprintf(stderr, "Failed to set decode_tile_row: %s\n", + aom_codec_error(&decoder)); + goto fail; + } + + if (aom_codec_control(&decoder, AV1_SET_DECODE_TILE_COL, tile_col)) { + fprintf(stderr, "Failed to set decode_tile_col: %s\n", + aom_codec_error(&decoder)); + goto fail; + } +#endif + + if (arg_skip) fprintf(stderr, "Skipping first %d frames.\n", arg_skip); + while (arg_skip) { + if (read_frame(&input, &buf, &bytes_in_buffer, &buffer_size)) break; + arg_skip--; + } + + if (num_external_frame_buffers > 0) { + ext_fb_list.num_external_frame_buffers = num_external_frame_buffers; + ext_fb_list.ext_fb = (struct ExternalFrameBuffer *)calloc( + num_external_frame_buffers, sizeof(*ext_fb_list.ext_fb)); + if (aom_codec_set_frame_buffer_functions(&decoder, get_av1_frame_buffer, + release_av1_frame_buffer, + &ext_fb_list)) { + fprintf(stderr, "Failed to configure external frame buffers: %s\n", + aom_codec_error(&decoder)); + goto fail; + } + } + + frame_avail = 1; + got_data = 0; + + if (framestats_file) fprintf(framestats_file, "bytes,qp\r\n"); + + /* Decode file */ + while (frame_avail || got_data) { + aom_codec_iter_t iter = NULL; + aom_image_t *img; + struct aom_usec_timer timer; + int corrupted = 0; + + frame_avail = 0; + if (!stop_after || frame_in < stop_after) { + if (!read_frame(&input, &buf, &bytes_in_buffer, &buffer_size)) { + frame_avail = 1; + frame_in++; + + aom_usec_timer_start(&timer); + + if (aom_codec_decode(&decoder, buf, (unsigned int)bytes_in_buffer, NULL, + 0)) { + const char *detail = aom_codec_error_detail(&decoder); + warn("Failed to decode frame %d: %s", frame_in, + aom_codec_error(&decoder)); + + if (detail) warn("Additional information: %s", detail); + if (!keep_going) goto fail; + } + + if (framestats_file) { + int qp; + if (aom_codec_control(&decoder, AOMD_GET_LAST_QUANTIZER, &qp)) { + warn("Failed AOMD_GET_LAST_QUANTIZER: %s", + aom_codec_error(&decoder)); + if (!keep_going) goto fail; + } + fprintf(framestats_file, "%d,%d\r\n", (int)bytes_in_buffer, qp); + } + + aom_usec_timer_mark(&timer); + dx_time += aom_usec_timer_elapsed(&timer); + } else { + flush_decoder = 1; + } + } else { + flush_decoder = 1; + } + + aom_usec_timer_start(&timer); + + if (flush_decoder) { + // Flush the decoder in frame parallel decode. + if (aom_codec_decode(&decoder, NULL, 0, NULL, 0)) { + warn("Failed to flush decoder: %s", aom_codec_error(&decoder)); + } + } + + got_data = 0; + if ((img = aom_codec_get_frame(&decoder, &iter))) { + ++frame_out; + got_data = 1; + } + + aom_usec_timer_mark(&timer); + dx_time += (unsigned int)aom_usec_timer_elapsed(&timer); + + if (!frame_parallel && + aom_codec_control(&decoder, AOMD_GET_FRAME_CORRUPTED, &corrupted)) { + warn("Failed AOM_GET_FRAME_CORRUPTED: %s", aom_codec_error(&decoder)); + if (!keep_going) goto fail; + } + frames_corrupted += corrupted; + + if (progress) show_progress(frame_in, frame_out, dx_time); + + if (!noblit && img) { + const int PLANES_YUV[] = { AOM_PLANE_Y, AOM_PLANE_U, AOM_PLANE_V }; + const int PLANES_YVU[] = { AOM_PLANE_Y, AOM_PLANE_V, AOM_PLANE_U }; + const int *planes = flipuv ? PLANES_YVU : PLANES_YUV; + + if (do_scale) { + if (frame_out == 1) { + // If the output frames are to be scaled to a fixed display size then + // use the width and height specified in the container. If either of + // these is set to 0, use the display size set in the first frame + // header. If that is unavailable, use the raw decoded size of the + // first decoded frame. + int render_width = aom_input_ctx.width; + int render_height = aom_input_ctx.height; + if (!render_width || !render_height) { + int render_size[2]; + if (aom_codec_control(&decoder, AV1D_GET_DISPLAY_SIZE, + render_size)) { + // As last resort use size of first frame as display size. + render_width = img->d_w; + render_height = img->d_h; + } else { + render_width = render_size[0]; + render_height = render_size[1]; + } + } + scaled_img = + aom_img_alloc(NULL, img->fmt, render_width, render_height, 16); + scaled_img->bit_depth = img->bit_depth; + } + + if (img->d_w != scaled_img->d_w || img->d_h != scaled_img->d_h) { +#if CONFIG_LIBYUV + libyuv_scale(img, scaled_img, kFilterBox); + img = scaled_img; +#else + fprintf(stderr, + "Failed to scale output frame: %s.\n" + "Scaling is disabled in this configuration. " + "To enable scaling, configure with --enable-libyuv\n", + aom_codec_error(&decoder)); + goto fail; +#endif + } + } +#if CONFIG_HIGHBITDEPTH + // Default to codec bit depth if output bit depth not set + if (!output_bit_depth && single_file && !do_md5) { + output_bit_depth = img->bit_depth; + } + // Shift up or down if necessary + if (output_bit_depth != 0 && output_bit_depth != img->bit_depth) { + const aom_img_fmt_t shifted_fmt = + output_bit_depth == 8 + ? img->fmt ^ (img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) + : img->fmt | AOM_IMG_FMT_HIGHBITDEPTH; + if (img_shifted && + img_shifted_realloc_required(img, img_shifted, shifted_fmt)) { + aom_img_free(img_shifted); + img_shifted = NULL; + } + if (!img_shifted) { + img_shifted = + aom_img_alloc(NULL, shifted_fmt, img->d_w, img->d_h, 16); + img_shifted->bit_depth = output_bit_depth; + } + if (output_bit_depth > img->bit_depth) { + aom_img_upshift(img_shifted, img, output_bit_depth - img->bit_depth); + } else { + aom_img_downshift(img_shifted, img, + img->bit_depth - output_bit_depth); + } + img = img_shifted; + } +#endif + +#if CONFIG_EXT_TILE + aom_input_ctx.width = img->d_w; + aom_input_ctx.height = img->d_h; +#endif // CONFIG_EXT_TILE + + if (single_file) { + if (use_y4m) { + char y4m_buf[Y4M_BUFFER_SIZE] = { 0 }; + size_t len = 0; + if (img->fmt == AOM_IMG_FMT_I440 || img->fmt == AOM_IMG_FMT_I44016) { + fprintf(stderr, "Cannot produce y4m output for 440 sampling.\n"); + goto fail; + } + if (frame_out == 1) { + // Y4M file header + len = y4m_write_file_header( + y4m_buf, sizeof(y4m_buf), aom_input_ctx.width, + aom_input_ctx.height, &aom_input_ctx.framerate, img->fmt, + img->bit_depth); + if (do_md5) { + MD5Update(&md5_ctx, (md5byte *)y4m_buf, (unsigned int)len); + } else { + fputs(y4m_buf, outfile); + } + } + + // Y4M frame header + len = y4m_write_frame_header(y4m_buf, sizeof(y4m_buf)); + if (do_md5) { + MD5Update(&md5_ctx, (md5byte *)y4m_buf, (unsigned int)len); + } else { + fputs(y4m_buf, outfile); + } + } else { + if (frame_out == 1) { + // Check if --yv12 or --i420 options are consistent with the + // bit-stream decoded + if (opt_i420) { + if (img->fmt != AOM_IMG_FMT_I420 && + img->fmt != AOM_IMG_FMT_I42016) { + fprintf(stderr, "Cannot produce i420 output for bit-stream.\n"); + goto fail; + } + } + if (opt_yv12) { + if ((img->fmt != AOM_IMG_FMT_I420 && + img->fmt != AOM_IMG_FMT_YV12) || + img->bit_depth != 8) { + fprintf(stderr, "Cannot produce yv12 output for bit-stream.\n"); + goto fail; + } + } + } + } + + if (do_md5) { + update_image_md5(img, planes, &md5_ctx); + } else { + write_image_file(img, planes, outfile); + } + } else { + generate_filename(outfile_pattern, outfile_name, PATH_MAX, img->d_w, + img->d_h, frame_in); + if (do_md5) { + MD5Init(&md5_ctx); + update_image_md5(img, planes, &md5_ctx); + MD5Final(md5_digest, &md5_ctx); + print_md5(md5_digest, outfile_name); + } else { + outfile = open_outfile(outfile_name); + write_image_file(img, planes, outfile); + fclose(outfile); + } + } + } + } + + if (summary || progress) { + show_progress(frame_in, frame_out, dx_time); + fprintf(stderr, "\n"); + } + + if (frames_corrupted) { + fprintf(stderr, "WARNING: %d frames corrupted.\n", frames_corrupted); + } else { + ret = EXIT_SUCCESS; + } + +fail: + + if (aom_codec_destroy(&decoder)) { + fprintf(stderr, "Failed to destroy decoder: %s\n", + aom_codec_error(&decoder)); + } + +fail2: + + if (!noblit && single_file) { + if (do_md5) { + MD5Final(md5_digest, &md5_ctx); + print_md5(md5_digest, outfile_name); + } else { + fclose(outfile); + } + } + +#if CONFIG_WEBM_IO + if (input.aom_input_ctx->file_type == FILE_TYPE_WEBM) + webm_free(input.webm_ctx); +#endif + + if (input.aom_input_ctx->file_type != FILE_TYPE_WEBM) free(buf); + + if (scaled_img) aom_img_free(scaled_img); +#if CONFIG_HIGHBITDEPTH + if (img_shifted) aom_img_free(img_shifted); +#endif + + for (i = 0; i < ext_fb_list.num_external_frame_buffers; ++i) { + free(ext_fb_list.ext_fb[i].data); + } + free(ext_fb_list.ext_fb); + + fclose(infile); + if (framestats_file) fclose(framestats_file); + + free(argv); + + return ret; +} + +int main(int argc, const char **argv_) { + unsigned int loops = 1, i; + char **argv, **argi, **argj; + struct arg arg; + int error = 0; + + argv = argv_dup(argc - 1, argv_ + 1); + for (argi = argj = argv; (*argj = *argi); argi += arg.argv_step) { + memset(&arg, 0, sizeof(arg)); + arg.argv_step = 1; + + if (arg_match(&arg, &looparg, argi)) { + loops = arg_parse_uint(&arg); + break; + } + } + free(argv); + for (i = 0; !error && i < loops; i++) error = main_loop(argc, argv_); + return error; +} diff --git a/third_party/aom/aomenc.c b/third_party/aom/aomenc.c new file mode 100644 index 0000000000..f4bf888ce5 --- /dev/null +++ b/third_party/aom/aomenc.c @@ -0,0 +1,2144 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aomenc.h" +#include "./aom_config.h" + +#include +#include +#include +#include +#include +#include +#include + +#if CONFIG_LIBYUV +#include "third_party/libyuv/include/libyuv/scale.h" +#endif + +#include "aom/aom_encoder.h" +#if CONFIG_DECODERS +#include "aom/aom_decoder.h" +#endif + +#include "./args.h" +#include "./ivfenc.h" +#include "./tools_common.h" +#include "examples/encoder_util.h" + +#if CONFIG_AV1_ENCODER +#include "aom/aomcx.h" +#endif +#if CONFIG_AV1_DECODER +#include "aom/aomdx.h" +#endif + +#include "./aomstats.h" +#include "./rate_hist.h" +#include "./warnings.h" +#include "aom/aom_integer.h" +#include "aom_ports/aom_timer.h" +#include "aom_ports/mem_ops.h" +#if CONFIG_WEBM_IO +#include "./webmenc.h" +#endif +#include "./y4minput.h" + +/* Swallow warnings about unused results of fread/fwrite */ +static size_t wrap_fread(void *ptr, size_t size, size_t nmemb, FILE *stream) { + return fread(ptr, size, nmemb, stream); +} +#define fread wrap_fread + +static size_t wrap_fwrite(const void *ptr, size_t size, size_t nmemb, + FILE *stream) { + return fwrite(ptr, size, nmemb, stream); +} +#define fwrite wrap_fwrite + +static const char *exec_name; + +static void warn_or_exit_on_errorv(aom_codec_ctx_t *ctx, int fatal, + const char *s, va_list ap) { + if (ctx->err) { + const char *detail = aom_codec_error_detail(ctx); + + vfprintf(stderr, s, ap); + fprintf(stderr, ": %s\n", aom_codec_error(ctx)); + + if (detail) fprintf(stderr, " %s\n", detail); + + if (fatal) exit(EXIT_FAILURE); + } +} + +static void ctx_exit_on_error(aom_codec_ctx_t *ctx, const char *s, ...) { + va_list ap; + + va_start(ap, s); + warn_or_exit_on_errorv(ctx, 1, s, ap); + va_end(ap); +} + +static void warn_or_exit_on_error(aom_codec_ctx_t *ctx, int fatal, + const char *s, ...) { + va_list ap; + + va_start(ap, s); + warn_or_exit_on_errorv(ctx, fatal, s, ap); + va_end(ap); +} + +static int read_frame(struct AvxInputContext *input_ctx, aom_image_t *img) { + FILE *f = input_ctx->file; + y4m_input *y4m = &input_ctx->y4m; + int shortread = 0; + + if (input_ctx->file_type == FILE_TYPE_Y4M) { + if (y4m_input_fetch_frame(y4m, f, img) < 1) return 0; + } else { + shortread = read_yuv_frame(input_ctx, img); + } + + return !shortread; +} + +static int file_is_y4m(const char detect[4]) { + if (memcmp(detect, "YUV4", 4) == 0) { + return 1; + } + return 0; +} + +static int fourcc_is_ivf(const char detect[4]) { + if (memcmp(detect, "DKIF", 4) == 0) { + return 1; + } + return 0; +} + +static const arg_def_t debugmode = + ARG_DEF("D", "debug", 0, "Debug mode (makes output deterministic)"); +static const arg_def_t outputfile = + ARG_DEF("o", "output", 1, "Output filename"); +static const arg_def_t use_yv12 = + ARG_DEF(NULL, "yv12", 0, "Input file is YV12 "); +static const arg_def_t use_i420 = + ARG_DEF(NULL, "i420", 0, "Input file is I420 (default)"); +static const arg_def_t use_i422 = + ARG_DEF(NULL, "i422", 0, "Input file is I422"); +static const arg_def_t use_i444 = + ARG_DEF(NULL, "i444", 0, "Input file is I444"); +static const arg_def_t use_i440 = + ARG_DEF(NULL, "i440", 0, "Input file is I440"); +static const arg_def_t codecarg = ARG_DEF(NULL, "codec", 1, "Codec to use"); +static const arg_def_t passes = + ARG_DEF("p", "passes", 1, "Number of passes (1/2)"); +static const arg_def_t pass_arg = + ARG_DEF(NULL, "pass", 1, "Pass to execute (1/2)"); +static const arg_def_t fpf_name = + ARG_DEF(NULL, "fpf", 1, "First pass statistics file name"); +#if CONFIG_FP_MB_STATS +static const arg_def_t fpmbf_name = + ARG_DEF(NULL, "fpmbf", 1, "First pass block statistics file name"); +#endif +static const arg_def_t limit = + ARG_DEF(NULL, "limit", 1, "Stop encoding after n input frames"); +static const arg_def_t skip = + ARG_DEF(NULL, "skip", 1, "Skip the first n input frames"); +static const arg_def_t deadline = + ARG_DEF("d", "deadline", 1, "Deadline per frame (usec)"); +static const arg_def_t good_dl = + ARG_DEF(NULL, "good", 0, "Use Good Quality Deadline"); +static const arg_def_t quietarg = + ARG_DEF("q", "quiet", 0, "Do not print encode progress"); +static const arg_def_t verbosearg = + ARG_DEF("v", "verbose", 0, "Show encoder parameters"); +static const arg_def_t psnrarg = + ARG_DEF(NULL, "psnr", 0, "Show PSNR in status line"); + +static const struct arg_enum_list test_decode_enum[] = { + { "off", TEST_DECODE_OFF }, + { "fatal", TEST_DECODE_FATAL }, + { "warn", TEST_DECODE_WARN }, + { NULL, 0 } +}; +static const arg_def_t recontest = ARG_DEF_ENUM( + NULL, "test-decode", 1, "Test encode/decode mismatch", test_decode_enum); +static const arg_def_t framerate = + ARG_DEF(NULL, "fps", 1, "Stream frame rate (rate/scale)"); +static const arg_def_t use_webm = + ARG_DEF(NULL, "webm", 0, "Output WebM (default when WebM IO is enabled)"); +static const arg_def_t use_ivf = ARG_DEF(NULL, "ivf", 0, "Output IVF"); +static const arg_def_t out_part = + ARG_DEF("P", "output-partitions", 0, + "Makes encoder output partitions. Requires IVF output!"); +static const arg_def_t q_hist_n = + ARG_DEF(NULL, "q-hist", 1, "Show quantizer histogram (n-buckets)"); +static const arg_def_t rate_hist_n = + ARG_DEF(NULL, "rate-hist", 1, "Show rate histogram (n-buckets)"); +static const arg_def_t disable_warnings = + ARG_DEF(NULL, "disable-warnings", 0, + "Disable warnings about potentially incorrect encode settings."); +static const arg_def_t disable_warning_prompt = + ARG_DEF("y", "disable-warning-prompt", 0, + "Display warnings, but do not prompt user to continue."); + +#if CONFIG_HIGHBITDEPTH +static const arg_def_t test16bitinternalarg = ARG_DEF( + NULL, "test-16bit-internal", 0, "Force use of 16 bit internal buffer"); + +static const struct arg_enum_list bitdepth_enum[] = { + { "8", AOM_BITS_8 }, { "10", AOM_BITS_10 }, { "12", AOM_BITS_12 }, { NULL, 0 } +}; + +static const arg_def_t bitdeptharg = ARG_DEF_ENUM( + "b", "bit-depth", 1, + "Bit depth for codec (8 for version <=1, 10 or 12 for version 2)", + bitdepth_enum); +static const arg_def_t inbitdeptharg = + ARG_DEF(NULL, "input-bit-depth", 1, "Bit depth of input"); +#endif + +static const arg_def_t *main_args[] = { &debugmode, + &outputfile, + &codecarg, + &passes, + &pass_arg, + &fpf_name, + &limit, + &skip, + &deadline, + &good_dl, + &quietarg, + &verbosearg, + &psnrarg, + &use_webm, + &use_ivf, + &out_part, + &q_hist_n, + &rate_hist_n, + &disable_warnings, + &disable_warning_prompt, + &recontest, + NULL }; + +static const arg_def_t usage = + ARG_DEF("u", "usage", 1, "Usage profile number to use"); +static const arg_def_t threads = + ARG_DEF("t", "threads", 1, "Max number of threads to use"); +static const arg_def_t profile = + ARG_DEF(NULL, "profile", 1, "Bitstream profile number to use"); +static const arg_def_t width = ARG_DEF("w", "width", 1, "Frame width"); +static const arg_def_t height = ARG_DEF("h", "height", 1, "Frame height"); +#if CONFIG_WEBM_IO +static const struct arg_enum_list stereo_mode_enum[] = { + { "mono", STEREO_FORMAT_MONO }, + { "left-right", STEREO_FORMAT_LEFT_RIGHT }, + { "bottom-top", STEREO_FORMAT_BOTTOM_TOP }, + { "top-bottom", STEREO_FORMAT_TOP_BOTTOM }, + { "right-left", STEREO_FORMAT_RIGHT_LEFT }, + { NULL, 0 } +}; +static const arg_def_t stereo_mode = ARG_DEF_ENUM( + NULL, "stereo-mode", 1, "Stereo 3D video format", stereo_mode_enum); +#endif +static const arg_def_t timebase = ARG_DEF( + NULL, "timebase", 1, "Output timestamp precision (fractional seconds)"); +static const arg_def_t error_resilient = + ARG_DEF(NULL, "error-resilient", 1, "Enable error resiliency features"); +static const arg_def_t lag_in_frames = + ARG_DEF(NULL, "lag-in-frames", 1, "Max number of frames to lag"); + +static const arg_def_t *global_args[] = { &use_yv12, + &use_i420, + &use_i422, + &use_i444, + &use_i440, + &usage, + &threads, + &profile, + &width, + &height, +#if CONFIG_WEBM_IO + &stereo_mode, +#endif + &timebase, + &framerate, + &error_resilient, +#if CONFIG_HIGHBITDEPTH + &test16bitinternalarg, + &bitdeptharg, +#endif + &lag_in_frames, + NULL }; + +static const arg_def_t dropframe_thresh = + ARG_DEF(NULL, "drop-frame", 1, "Temporal resampling threshold (buf %)"); +static const arg_def_t resize_allowed = + ARG_DEF(NULL, "resize-allowed", 1, "Spatial resampling enabled (bool)"); +static const arg_def_t resize_width = + ARG_DEF(NULL, "resize-width", 1, "Width of encoded frame"); +static const arg_def_t resize_height = + ARG_DEF(NULL, "resize-height", 1, "Height of encoded frame"); +static const arg_def_t resize_up_thresh = + ARG_DEF(NULL, "resize-up", 1, "Upscale threshold (buf %)"); +static const arg_def_t resize_down_thresh = + ARG_DEF(NULL, "resize-down", 1, "Downscale threshold (buf %)"); +static const struct arg_enum_list end_usage_enum[] = { { "vbr", AOM_VBR }, + { "cbr", AOM_CBR }, + { "cq", AOM_CQ }, + { "q", AOM_Q }, + { NULL, 0 } }; +static const arg_def_t end_usage = + ARG_DEF_ENUM(NULL, "end-usage", 1, "Rate control mode", end_usage_enum); +static const arg_def_t target_bitrate = + ARG_DEF(NULL, "target-bitrate", 1, "Bitrate (kbps)"); +static const arg_def_t min_quantizer = + ARG_DEF(NULL, "min-q", 1, "Minimum (best) quantizer"); +static const arg_def_t max_quantizer = + ARG_DEF(NULL, "max-q", 1, "Maximum (worst) quantizer"); +static const arg_def_t undershoot_pct = + ARG_DEF(NULL, "undershoot-pct", 1, "Datarate undershoot (min) target (%)"); +static const arg_def_t overshoot_pct = + ARG_DEF(NULL, "overshoot-pct", 1, "Datarate overshoot (max) target (%)"); +static const arg_def_t buf_sz = + ARG_DEF(NULL, "buf-sz", 1, "Client buffer size (ms)"); +static const arg_def_t buf_initial_sz = + ARG_DEF(NULL, "buf-initial-sz", 1, "Client initial buffer size (ms)"); +static const arg_def_t buf_optimal_sz = + ARG_DEF(NULL, "buf-optimal-sz", 1, "Client optimal buffer size (ms)"); +static const arg_def_t *rc_args[] = { + &dropframe_thresh, &resize_allowed, &resize_width, &resize_height, + &resize_up_thresh, &resize_down_thresh, &end_usage, &target_bitrate, + &min_quantizer, &max_quantizer, &undershoot_pct, &overshoot_pct, + &buf_sz, &buf_initial_sz, &buf_optimal_sz, NULL +}; + +static const arg_def_t bias_pct = + ARG_DEF(NULL, "bias-pct", 1, "CBR/VBR bias (0=CBR, 100=VBR)"); +static const arg_def_t minsection_pct = + ARG_DEF(NULL, "minsection-pct", 1, "GOP min bitrate (% of target)"); +static const arg_def_t maxsection_pct = + ARG_DEF(NULL, "maxsection-pct", 1, "GOP max bitrate (% of target)"); +static const arg_def_t *rc_twopass_args[] = { &bias_pct, &minsection_pct, + &maxsection_pct, NULL }; + +static const arg_def_t kf_min_dist = + ARG_DEF(NULL, "kf-min-dist", 1, "Minimum keyframe interval (frames)"); +static const arg_def_t kf_max_dist = + ARG_DEF(NULL, "kf-max-dist", 1, "Maximum keyframe interval (frames)"); +static const arg_def_t kf_disabled = + ARG_DEF(NULL, "disable-kf", 0, "Disable keyframe placement"); +static const arg_def_t *kf_args[] = { &kf_min_dist, &kf_max_dist, &kf_disabled, + NULL }; + +static const arg_def_t noise_sens = + ARG_DEF(NULL, "noise-sensitivity", 1, "Noise sensitivity (frames to blur)"); +static const arg_def_t sharpness = + ARG_DEF(NULL, "sharpness", 1, "Loop filter sharpness (0..7)"); +static const arg_def_t static_thresh = + ARG_DEF(NULL, "static-thresh", 1, "Motion detection threshold"); +static const arg_def_t auto_altref = + ARG_DEF(NULL, "auto-alt-ref", 1, "Enable automatic alt reference frames"); +static const arg_def_t arnr_maxframes = + ARG_DEF(NULL, "arnr-maxframes", 1, "AltRef max frames (0..15)"); +static const arg_def_t arnr_strength = + ARG_DEF(NULL, "arnr-strength", 1, "AltRef filter strength (0..6)"); +static const struct arg_enum_list tuning_enum[] = { + { "psnr", AOM_TUNE_PSNR }, { "ssim", AOM_TUNE_SSIM }, { NULL, 0 } +}; +static const arg_def_t tune_ssim = + ARG_DEF_ENUM(NULL, "tune", 1, "Material to favor", tuning_enum); +static const arg_def_t cq_level = + ARG_DEF(NULL, "cq-level", 1, "Constant/Constrained Quality level"); +static const arg_def_t max_intra_rate_pct = + ARG_DEF(NULL, "max-intra-rate", 1, "Max I-frame bitrate (pct)"); + +#if CONFIG_AV1_ENCODER +static const arg_def_t cpu_used_av1 = + ARG_DEF(NULL, "cpu-used", 1, "CPU Used (0..8)"); +static const arg_def_t tile_cols = + ARG_DEF(NULL, "tile-columns", 1, "Number of tile columns to use, log2"); +static const arg_def_t tile_rows = + ARG_DEF(NULL, "tile-rows", 1, + "Number of tile rows to use, log2 (set to 0 while threads > 1)"); +#if CONFIG_EXT_TILE +static const arg_def_t tile_encoding_mode = + ARG_DEF(NULL, "tile-encoding-mode", 1, + "Tile encoding mode (0: normal" + " (default), 1: vr)"); +#endif +#if CONFIG_DEPENDENT_HORZTILES +static const arg_def_t tile_dependent_rows = + ARG_DEF(NULL, "tile-dependent-rows", 1, "Enable dependent Tile rows"); +#endif +#if CONFIG_LOOPFILTERING_ACROSS_TILES +static const arg_def_t tile_loopfilter = ARG_DEF( + NULL, "tile-loopfilter", 1, "Enable loop filter across tile boundary"); +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES +static const arg_def_t lossless = + ARG_DEF(NULL, "lossless", 1, "Lossless mode (0: false (default), 1: true)"); +#if CONFIG_AOM_QM +static const arg_def_t enable_qm = + ARG_DEF(NULL, "enable-qm", 1, + "Enable quantisation matrices (0: false (default), 1: true)"); +static const arg_def_t qm_min = ARG_DEF( + NULL, "qm-min", 1, "Min quant matrix flatness (0..15), default is 8"); +static const arg_def_t qm_max = ARG_DEF( + NULL, "qm-max", 1, "Max quant matrix flatness (0..15), default is 16"); +#endif +#if CONFIG_TILE_GROUPS +static const arg_def_t num_tg = ARG_DEF( + NULL, "num-tile-groups", 1, "Maximum number of tile groups, default is 1"); +static const arg_def_t mtu_size = + ARG_DEF(NULL, "mtu-size", 1, + "MTU size for a tile group, default is 0 (no MTU targeting), " + "overrides maximum number of tile groups"); +#endif +#if CONFIG_TEMPMV_SIGNALING +static const arg_def_t disable_tempmv = ARG_DEF( + NULL, "disable-tempmv", 1, "Disable temporal mv prediction (default is 0)"); +#endif +static const arg_def_t frame_parallel_decoding = + ARG_DEF(NULL, "frame-parallel", 1, + "Enable frame parallel decodability features " + "(0: false (default), 1: true)"); +#if CONFIG_DELTA_Q && !CONFIG_EXT_DELTA_Q +static const arg_def_t aq_mode = ARG_DEF( + NULL, "aq-mode", 1, + "Adaptive quantization mode (0: off (default), 1: variance 2: complexity, " + "3: cyclic refresh, 4: delta quant)"); +#else +static const arg_def_t aq_mode = ARG_DEF( + NULL, "aq-mode", 1, + "Adaptive quantization mode (0: off (default), 1: variance 2: complexity, " + "3: cyclic refresh)"); +#endif +#if CONFIG_EXT_DELTA_Q +static const arg_def_t deltaq_mode = ARG_DEF( + NULL, "deltaq-mode", 1, + "Delta qindex mode (0: off (default), 1: deltaq 2: deltaq + deltalf)"); +#endif +static const arg_def_t frame_periodic_boost = + ARG_DEF(NULL, "frame-boost", 1, + "Enable frame periodic boost (0: off (default), 1: on)"); +static const arg_def_t gf_cbr_boost_pct = ARG_DEF( + NULL, "gf-cbr-boost", 1, "Boost for Golden Frame in CBR mode (pct)"); +static const arg_def_t max_inter_rate_pct = + ARG_DEF(NULL, "max-inter-rate", 1, "Max P-frame bitrate (pct)"); +static const arg_def_t min_gf_interval = ARG_DEF( + NULL, "min-gf-interval", 1, + "min gf/arf frame interval (default 0, indicating in-built behavior)"); +static const arg_def_t max_gf_interval = ARG_DEF( + NULL, "max-gf-interval", 1, + "max gf/arf frame interval (default 0, indicating in-built behavior)"); + +static const struct arg_enum_list color_space_enum[] = { + { "unknown", AOM_CS_UNKNOWN }, + { "bt601", AOM_CS_BT_601 }, + { "bt709", AOM_CS_BT_709 }, + { "smpte170", AOM_CS_SMPTE_170 }, + { "smpte240", AOM_CS_SMPTE_240 }, + { "bt2020", AOM_CS_BT_2020 }, + { "reserved", AOM_CS_RESERVED }, + { "sRGB", AOM_CS_SRGB }, + { NULL, 0 } +}; + +static const arg_def_t input_color_space = + ARG_DEF_ENUM(NULL, "color-space", 1, "The color space of input content:", + color_space_enum); + +static const struct arg_enum_list tune_content_enum[] = { + { "default", AOM_CONTENT_DEFAULT }, + { "screen", AOM_CONTENT_SCREEN }, + { NULL, 0 } +}; + +static const arg_def_t tune_content = ARG_DEF_ENUM( + NULL, "tune-content", 1, "Tune content type", tune_content_enum); +#endif + +#if CONFIG_AV1_ENCODER +#if CONFIG_EXT_PARTITION +static const struct arg_enum_list superblock_size_enum[] = { + { "dynamic", AOM_SUPERBLOCK_SIZE_DYNAMIC }, + { "64", AOM_SUPERBLOCK_SIZE_64X64 }, + { "128", AOM_SUPERBLOCK_SIZE_128X128 }, + { NULL, 0 } +}; +static const arg_def_t superblock_size = ARG_DEF_ENUM( + NULL, "sb-size", 1, "Superblock size to use", superblock_size_enum); +#endif // CONFIG_EXT_PARTITION + +static const arg_def_t *av1_args[] = { &cpu_used_av1, + &auto_altref, + &sharpness, + &static_thresh, + &tile_cols, + &tile_rows, +#if CONFIG_EXT_TILE + &tile_encoding_mode, +#endif +#if CONFIG_DEPENDENT_HORZTILES + &tile_dependent_rows, +#endif +#if CONFIG_LOOPFILTERING_ACROSS_TILES + &tile_loopfilter, +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + &arnr_maxframes, + &arnr_strength, + &tune_ssim, + &cq_level, + &max_intra_rate_pct, + &max_inter_rate_pct, + &gf_cbr_boost_pct, + &lossless, +#if CONFIG_AOM_QM + &enable_qm, + &qm_min, + &qm_max, +#endif + &frame_parallel_decoding, + &aq_mode, +#if CONFIG_EXT_DELTA_Q + &deltaq_mode, +#endif + &frame_periodic_boost, + &noise_sens, + &tune_content, + &input_color_space, + &min_gf_interval, + &max_gf_interval, +#if CONFIG_EXT_PARTITION + &superblock_size, +#endif // CONFIG_EXT_PARTITION +#if CONFIG_TILE_GROUPS + &num_tg, + &mtu_size, +#endif +#if CONFIG_TEMPMV_SIGNALING + &disable_tempmv, +#endif +#if CONFIG_HIGHBITDEPTH + &bitdeptharg, + &inbitdeptharg, +#endif // CONFIG_HIGHBITDEPTH + NULL }; +static const int av1_arg_ctrl_map[] = { AOME_SET_CPUUSED, + AOME_SET_ENABLEAUTOALTREF, + AOME_SET_SHARPNESS, + AOME_SET_STATIC_THRESHOLD, + AV1E_SET_TILE_COLUMNS, + AV1E_SET_TILE_ROWS, +#if CONFIG_EXT_TILE + AV1E_SET_TILE_ENCODING_MODE, +#endif +#if CONFIG_DEPENDENT_HORZTILES + AV1E_SET_TILE_DEPENDENT_ROWS, +#endif +#if CONFIG_LOOPFILTERING_ACROSS_TILES + AV1E_SET_TILE_LOOPFILTER, +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + AOME_SET_ARNR_MAXFRAMES, + AOME_SET_ARNR_STRENGTH, + AOME_SET_TUNING, + AOME_SET_CQ_LEVEL, + AOME_SET_MAX_INTRA_BITRATE_PCT, + AV1E_SET_MAX_INTER_BITRATE_PCT, + AV1E_SET_GF_CBR_BOOST_PCT, + AV1E_SET_LOSSLESS, +#if CONFIG_AOM_QM + AV1E_SET_ENABLE_QM, + AV1E_SET_QM_MIN, + AV1E_SET_QM_MAX, +#endif + AV1E_SET_FRAME_PARALLEL_DECODING, + AV1E_SET_AQ_MODE, +#if CONFIG_EXT_DELTA_Q + AV1E_SET_DELTAQ_MODE, +#endif + AV1E_SET_FRAME_PERIODIC_BOOST, + AV1E_SET_NOISE_SENSITIVITY, + AV1E_SET_TUNE_CONTENT, + AV1E_SET_COLOR_SPACE, + AV1E_SET_MIN_GF_INTERVAL, + AV1E_SET_MAX_GF_INTERVAL, +#if CONFIG_EXT_PARTITION + AV1E_SET_SUPERBLOCK_SIZE, +#endif // CONFIG_EXT_PARTITION +#if CONFIG_TILE_GROUPS + AV1E_SET_NUM_TG, + AV1E_SET_MTU, +#endif +#if CONFIG_TEMPMV_SIGNALING + AV1E_SET_DISABLE_TEMPMV, +#endif + 0 }; +#endif + +static const arg_def_t *no_args[] = { NULL }; + +void usage_exit(void) { + int i; + const int num_encoder = get_aom_encoder_count(); + + fprintf(stderr, "Usage: %s -o dst_filename src_filename \n", + exec_name); + + fprintf(stderr, "\nOptions:\n"); + arg_show_usage(stderr, main_args); + fprintf(stderr, "\nEncoder Global Options:\n"); + arg_show_usage(stderr, global_args); + fprintf(stderr, "\nRate Control Options:\n"); + arg_show_usage(stderr, rc_args); + fprintf(stderr, "\nTwopass Rate Control Options:\n"); + arg_show_usage(stderr, rc_twopass_args); + fprintf(stderr, "\nKeyframe Placement Options:\n"); + arg_show_usage(stderr, kf_args); +#if CONFIG_AV1_ENCODER + fprintf(stderr, "\nAV1 Specific Options:\n"); + arg_show_usage(stderr, av1_args); +#endif + fprintf(stderr, + "\nStream timebase (--timebase):\n" + " The desired precision of timestamps in the output, expressed\n" + " in fractional seconds. Default is 1/1000.\n"); + fprintf(stderr, "\nIncluded encoders:\n\n"); + + for (i = 0; i < num_encoder; ++i) { + const AvxInterface *const encoder = get_aom_encoder_by_index(i); + const char *defstr = (i == (num_encoder - 1)) ? "(default)" : ""; + fprintf(stderr, " %-6s - %s %s\n", encoder->name, + aom_codec_iface_name(encoder->codec_interface()), defstr); + } + fprintf(stderr, "\n "); + fprintf(stderr, "Use --codec to switch to a non-default encoder.\n\n"); + + exit(EXIT_FAILURE); +} + +#define NELEMENTS(x) (sizeof(x) / sizeof(x[0])) +#if CONFIG_AV1_ENCODER +#define ARG_CTRL_CNT_MAX NELEMENTS(av1_arg_ctrl_map) +#endif + +#if !CONFIG_WEBM_IO +typedef int stereo_format_t; +struct WebmOutputContext { + int debug; +}; +#endif + +/* Per-stream configuration */ +struct stream_config { + struct aom_codec_enc_cfg cfg; + const char *out_fn; + const char *stats_fn; +#if CONFIG_FP_MB_STATS + const char *fpmb_stats_fn; +#endif + stereo_format_t stereo_fmt; + int arg_ctrls[ARG_CTRL_CNT_MAX][2]; + int arg_ctrl_cnt; + int write_webm; +#if CONFIG_HIGHBITDEPTH + // whether to use 16bit internal buffers + int use_16bit_internal; +#endif +}; + +struct stream_state { + int index; + struct stream_state *next; + struct stream_config config; + FILE *file; + struct rate_hist *rate_hist; + struct WebmOutputContext webm_ctx; + uint64_t psnr_sse_total; + uint64_t psnr_samples_total; + double psnr_totals[4]; + int psnr_count; + int counts[64]; + aom_codec_ctx_t encoder; + unsigned int frames_out; + uint64_t cx_time; + size_t nbytes; + stats_io_t stats; +#if CONFIG_FP_MB_STATS + stats_io_t fpmb_stats; +#endif + struct aom_image *img; + aom_codec_ctx_t decoder; + int mismatch_seen; +}; + +static void validate_positive_rational(const char *msg, + struct aom_rational *rat) { + if (rat->den < 0) { + rat->num *= -1; + rat->den *= -1; + } + + if (rat->num < 0) die("Error: %s must be positive\n", msg); + + if (!rat->den) die("Error: %s has zero denominator\n", msg); +} + +static void parse_global_config(struct AvxEncoderConfig *global, char **argv) { + char **argi, **argj; + struct arg arg; + const int num_encoder = get_aom_encoder_count(); + + if (num_encoder < 1) die("Error: no valid encoder available\n"); + + /* Initialize default parameters */ + memset(global, 0, sizeof(*global)); + global->codec = get_aom_encoder_by_index(num_encoder - 1); + global->passes = 0; + global->color_type = I420; + /* Assign default deadline to good quality */ + global->deadline = AOM_DL_GOOD_QUALITY; + + for (argi = argj = argv; (*argj = *argi); argi += arg.argv_step) { + arg.argv_step = 1; + + if (arg_match(&arg, &codecarg, argi)) { + global->codec = get_aom_encoder_by_name(arg.val); + if (!global->codec) + die("Error: Unrecognized argument (%s) to --codec\n", arg.val); + } else if (arg_match(&arg, &passes, argi)) { + global->passes = arg_parse_uint(&arg); + + if (global->passes < 1 || global->passes > 2) + die("Error: Invalid number of passes (%d)\n", global->passes); + } else if (arg_match(&arg, &pass_arg, argi)) { + global->pass = arg_parse_uint(&arg); + + if (global->pass < 1 || global->pass > 2) + die("Error: Invalid pass selected (%d)\n", global->pass); + } else if (arg_match(&arg, &usage, argi)) + global->usage = arg_parse_uint(&arg); + else if (arg_match(&arg, &deadline, argi)) + global->deadline = arg_parse_uint(&arg); + else if (arg_match(&arg, &good_dl, argi)) + global->deadline = AOM_DL_GOOD_QUALITY; + else if (arg_match(&arg, &use_yv12, argi)) + global->color_type = YV12; + else if (arg_match(&arg, &use_i420, argi)) + global->color_type = I420; + else if (arg_match(&arg, &use_i422, argi)) + global->color_type = I422; + else if (arg_match(&arg, &use_i444, argi)) + global->color_type = I444; + else if (arg_match(&arg, &use_i440, argi)) + global->color_type = I440; + else if (arg_match(&arg, &quietarg, argi)) + global->quiet = 1; + else if (arg_match(&arg, &verbosearg, argi)) + global->verbose = 1; + else if (arg_match(&arg, &limit, argi)) + global->limit = arg_parse_uint(&arg); + else if (arg_match(&arg, &skip, argi)) + global->skip_frames = arg_parse_uint(&arg); + else if (arg_match(&arg, &psnrarg, argi)) + global->show_psnr = 1; + else if (arg_match(&arg, &recontest, argi)) + global->test_decode = arg_parse_enum_or_int(&arg); + else if (arg_match(&arg, &framerate, argi)) { + global->framerate = arg_parse_rational(&arg); + validate_positive_rational(arg.name, &global->framerate); + global->have_framerate = 1; + } else if (arg_match(&arg, &out_part, argi)) + global->out_part = 1; + else if (arg_match(&arg, &debugmode, argi)) + global->debug = 1; + else if (arg_match(&arg, &q_hist_n, argi)) + global->show_q_hist_buckets = arg_parse_uint(&arg); + else if (arg_match(&arg, &rate_hist_n, argi)) + global->show_rate_hist_buckets = arg_parse_uint(&arg); + else if (arg_match(&arg, &disable_warnings, argi)) + global->disable_warnings = 1; + else if (arg_match(&arg, &disable_warning_prompt, argi)) + global->disable_warning_prompt = 1; + else + argj++; + } + + if (global->pass) { + /* DWIM: Assume the user meant passes=2 if pass=2 is specified */ + if (global->pass > global->passes) { + warn("Assuming --pass=%d implies --passes=%d\n", global->pass, + global->pass); + global->passes = global->pass; + } + } + /* Validate global config */ + if (global->passes == 0) { +#if CONFIG_AV1_ENCODER + // Make default AV1 passes = 2 until there is a better quality 1-pass + // encoder + if (global->codec != NULL && global->codec->name != NULL) + global->passes = (strcmp(global->codec->name, "av1") == 0) ? 2 : 1; +#else + global->passes = 1; +#endif + } +} + +static void open_input_file(struct AvxInputContext *input) { + /* Parse certain options from the input file, if possible */ + input->file = strcmp(input->filename, "-") ? fopen(input->filename, "rb") + : set_binary_mode(stdin); + + if (!input->file) fatal("Failed to open input file"); + + if (!fseeko(input->file, 0, SEEK_END)) { + /* Input file is seekable. Figure out how long it is, so we can get + * progress info. + */ + input->length = ftello(input->file); + rewind(input->file); + } + + /* Default to 1:1 pixel aspect ratio. */ + input->pixel_aspect_ratio.numerator = 1; + input->pixel_aspect_ratio.denominator = 1; + + /* For RAW input sources, these bytes will applied on the first frame + * in read_frame(). + */ + input->detect.buf_read = fread(input->detect.buf, 1, 4, input->file); + input->detect.position = 0; + + if (input->detect.buf_read == 4 && file_is_y4m(input->detect.buf)) { + if (y4m_input_open(&input->y4m, input->file, input->detect.buf, 4, + input->only_i420) >= 0) { + input->file_type = FILE_TYPE_Y4M; + input->width = input->y4m.pic_w; + input->height = input->y4m.pic_h; + input->pixel_aspect_ratio.numerator = input->y4m.par_n; + input->pixel_aspect_ratio.denominator = input->y4m.par_d; + input->framerate.numerator = input->y4m.fps_n; + input->framerate.denominator = input->y4m.fps_d; + input->fmt = input->y4m.aom_fmt; + input->bit_depth = input->y4m.bit_depth; + } else + fatal("Unsupported Y4M stream."); + } else if (input->detect.buf_read == 4 && fourcc_is_ivf(input->detect.buf)) { + fatal("IVF is not supported as input."); + } else { + input->file_type = FILE_TYPE_RAW; + } +} + +static void close_input_file(struct AvxInputContext *input) { + fclose(input->file); + if (input->file_type == FILE_TYPE_Y4M) y4m_input_close(&input->y4m); +} + +static struct stream_state *new_stream(struct AvxEncoderConfig *global, + struct stream_state *prev) { + struct stream_state *stream; + + stream = calloc(1, sizeof(*stream)); + if (stream == NULL) { + fatal("Failed to allocate new stream."); + } + + if (prev) { + memcpy(stream, prev, sizeof(*stream)); + stream->index++; + prev->next = stream; + } else { + aom_codec_err_t res; + + /* Populate encoder configuration */ + res = aom_codec_enc_config_default(global->codec->codec_interface(), + &stream->config.cfg, global->usage); + if (res) fatal("Failed to get config: %s\n", aom_codec_err_to_string(res)); + + /* Change the default timebase to a high enough value so that the + * encoder will always create strictly increasing timestamps. + */ + stream->config.cfg.g_timebase.den = 1000; + + /* Never use the library's default resolution, require it be parsed + * from the file or set on the command line. + */ + stream->config.cfg.g_w = 0; + stream->config.cfg.g_h = 0; + + /* Initialize remaining stream parameters */ + stream->config.write_webm = 1; +#if CONFIG_WEBM_IO + stream->config.stereo_fmt = STEREO_FORMAT_MONO; + stream->webm_ctx.last_pts_ns = -1; + stream->webm_ctx.writer = NULL; + stream->webm_ctx.segment = NULL; +#endif + + /* Allows removal of the application version from the EBML tags */ + stream->webm_ctx.debug = global->debug; + } + + /* Output files must be specified for each stream */ + stream->config.out_fn = NULL; + + stream->next = NULL; + return stream; +} + +static int parse_stream_params(struct AvxEncoderConfig *global, + struct stream_state *stream, char **argv) { + char **argi, **argj; + struct arg arg; + static const arg_def_t **ctrl_args = no_args; + static const int *ctrl_args_map = NULL; + struct stream_config *config = &stream->config; + int eos_mark_found = 0; +#if CONFIG_HIGHBITDEPTH + int test_16bit_internal = 0; +#endif + + // Handle codec specific options + if (0) { +#if CONFIG_AV1_ENCODER + } else if (strcmp(global->codec->name, "av1") == 0) { + // TODO(jingning): Reuse AV1 specific encoder configuration parameters. + // Consider to expand this set for AV1 encoder control. + ctrl_args = av1_args; + ctrl_args_map = av1_arg_ctrl_map; +#endif + } + + for (argi = argj = argv; (*argj = *argi); argi += arg.argv_step) { + arg.argv_step = 1; + + /* Once we've found an end-of-stream marker (--) we want to continue + * shifting arguments but not consuming them. + */ + if (eos_mark_found) { + argj++; + continue; + } else if (!strcmp(*argj, "--")) { + eos_mark_found = 1; + continue; + } + + if (arg_match(&arg, &outputfile, argi)) { + config->out_fn = arg.val; + } else if (arg_match(&arg, &fpf_name, argi)) { + config->stats_fn = arg.val; +#if CONFIG_FP_MB_STATS + } else if (arg_match(&arg, &fpmbf_name, argi)) { + config->fpmb_stats_fn = arg.val; +#endif + } else if (arg_match(&arg, &use_webm, argi)) { +#if CONFIG_WEBM_IO + config->write_webm = 1; +#else + die("Error: --webm specified but webm is disabled."); +#endif + } else if (arg_match(&arg, &use_ivf, argi)) { + config->write_webm = 0; + } else if (arg_match(&arg, &threads, argi)) { + config->cfg.g_threads = arg_parse_uint(&arg); + } else if (arg_match(&arg, &profile, argi)) { + config->cfg.g_profile = arg_parse_uint(&arg); + } else if (arg_match(&arg, &width, argi)) { + config->cfg.g_w = arg_parse_uint(&arg); + } else if (arg_match(&arg, &height, argi)) { + config->cfg.g_h = arg_parse_uint(&arg); +#if CONFIG_HIGHBITDEPTH + } else if (arg_match(&arg, &bitdeptharg, argi)) { + config->cfg.g_bit_depth = arg_parse_enum_or_int(&arg); + } else if (arg_match(&arg, &inbitdeptharg, argi)) { + config->cfg.g_input_bit_depth = arg_parse_uint(&arg); +#endif +#if CONFIG_WEBM_IO + } else if (arg_match(&arg, &stereo_mode, argi)) { + config->stereo_fmt = arg_parse_enum_or_int(&arg); +#endif + } else if (arg_match(&arg, &timebase, argi)) { + config->cfg.g_timebase = arg_parse_rational(&arg); + validate_positive_rational(arg.name, &config->cfg.g_timebase); + } else if (arg_match(&arg, &error_resilient, argi)) { + config->cfg.g_error_resilient = arg_parse_uint(&arg); + } else if (arg_match(&arg, &lag_in_frames, argi)) { + config->cfg.g_lag_in_frames = arg_parse_uint(&arg); + } else if (arg_match(&arg, &dropframe_thresh, argi)) { + config->cfg.rc_dropframe_thresh = arg_parse_uint(&arg); + } else if (arg_match(&arg, &resize_allowed, argi)) { + config->cfg.rc_resize_allowed = arg_parse_uint(&arg); + } else if (arg_match(&arg, &resize_width, argi)) { + config->cfg.rc_scaled_width = arg_parse_uint(&arg); + } else if (arg_match(&arg, &resize_height, argi)) { + config->cfg.rc_scaled_height = arg_parse_uint(&arg); + } else if (arg_match(&arg, &resize_up_thresh, argi)) { + config->cfg.rc_resize_up_thresh = arg_parse_uint(&arg); + } else if (arg_match(&arg, &resize_down_thresh, argi)) { + config->cfg.rc_resize_down_thresh = arg_parse_uint(&arg); + } else if (arg_match(&arg, &end_usage, argi)) { + config->cfg.rc_end_usage = arg_parse_enum_or_int(&arg); + } else if (arg_match(&arg, &target_bitrate, argi)) { + config->cfg.rc_target_bitrate = arg_parse_uint(&arg); + } else if (arg_match(&arg, &min_quantizer, argi)) { + config->cfg.rc_min_quantizer = arg_parse_uint(&arg); + } else if (arg_match(&arg, &max_quantizer, argi)) { + config->cfg.rc_max_quantizer = arg_parse_uint(&arg); + } else if (arg_match(&arg, &undershoot_pct, argi)) { + config->cfg.rc_undershoot_pct = arg_parse_uint(&arg); + } else if (arg_match(&arg, &overshoot_pct, argi)) { + config->cfg.rc_overshoot_pct = arg_parse_uint(&arg); + } else if (arg_match(&arg, &buf_sz, argi)) { + config->cfg.rc_buf_sz = arg_parse_uint(&arg); + } else if (arg_match(&arg, &buf_initial_sz, argi)) { + config->cfg.rc_buf_initial_sz = arg_parse_uint(&arg); + } else if (arg_match(&arg, &buf_optimal_sz, argi)) { + config->cfg.rc_buf_optimal_sz = arg_parse_uint(&arg); + } else if (arg_match(&arg, &bias_pct, argi)) { + config->cfg.rc_2pass_vbr_bias_pct = arg_parse_uint(&arg); + if (global->passes < 2) + warn("option %s ignored in one-pass mode.\n", arg.name); + } else if (arg_match(&arg, &minsection_pct, argi)) { + config->cfg.rc_2pass_vbr_minsection_pct = arg_parse_uint(&arg); + + if (global->passes < 2) + warn("option %s ignored in one-pass mode.\n", arg.name); + } else if (arg_match(&arg, &maxsection_pct, argi)) { + config->cfg.rc_2pass_vbr_maxsection_pct = arg_parse_uint(&arg); + + if (global->passes < 2) + warn("option %s ignored in one-pass mode.\n", arg.name); + } else if (arg_match(&arg, &kf_min_dist, argi)) { + config->cfg.kf_min_dist = arg_parse_uint(&arg); + } else if (arg_match(&arg, &kf_max_dist, argi)) { + config->cfg.kf_max_dist = arg_parse_uint(&arg); + } else if (arg_match(&arg, &kf_disabled, argi)) { + config->cfg.kf_mode = AOM_KF_DISABLED; +#if CONFIG_HIGHBITDEPTH + } else if (arg_match(&arg, &test16bitinternalarg, argi)) { + if (strcmp(global->codec->name, "av1") == 0 || + strcmp(global->codec->name, "av1") == 0) { + test_16bit_internal = 1; + } +#endif + } else { + int i, match = 0; + for (i = 0; ctrl_args[i]; i++) { + if (arg_match(&arg, ctrl_args[i], argi)) { + int j; + match = 1; + + /* Point either to the next free element or the first + * instance of this control. + */ + for (j = 0; j < config->arg_ctrl_cnt; j++) + if (ctrl_args_map != NULL && + config->arg_ctrls[j][0] == ctrl_args_map[i]) + break; + + /* Update/insert */ + assert(j < (int)ARG_CTRL_CNT_MAX); + if (ctrl_args_map != NULL && j < (int)ARG_CTRL_CNT_MAX) { + config->arg_ctrls[j][0] = ctrl_args_map[i]; + config->arg_ctrls[j][1] = arg_parse_enum_or_int(&arg); + if (j == config->arg_ctrl_cnt) config->arg_ctrl_cnt++; + } + } + } + if (!match) argj++; + } + } +#if CONFIG_HIGHBITDEPTH +#if CONFIG_LOWBITDEPTH + if (strcmp(global->codec->name, "av1") == 0 || + strcmp(global->codec->name, "av1") == 0) { + config->use_16bit_internal = + test_16bit_internal | (config->cfg.g_profile > 1); + } +#else + config->use_16bit_internal = 1; +#endif +#endif + return eos_mark_found; +} + +#define FOREACH_STREAM(func) \ + do { \ + struct stream_state *stream; \ + for (stream = streams; stream; stream = stream->next) { \ + func; \ + } \ + } while (0) + +static void validate_stream_config(const struct stream_state *stream, + const struct AvxEncoderConfig *global) { + const struct stream_state *streami; + (void)global; + + if (!stream->config.cfg.g_w || !stream->config.cfg.g_h) + fatal( + "Stream %d: Specify stream dimensions with --width (-w) " + " and --height (-h)", + stream->index); + + // Check that the codec bit depth is greater than the input bit depth. + if (stream->config.cfg.g_input_bit_depth > + (unsigned int)stream->config.cfg.g_bit_depth) { + fatal("Stream %d: codec bit depth (%d) less than input bit depth (%d)", + stream->index, (int)stream->config.cfg.g_bit_depth, + stream->config.cfg.g_input_bit_depth); + } + + for (streami = stream; streami; streami = streami->next) { + /* All streams require output files */ + if (!streami->config.out_fn) + fatal("Stream %d: Output file is required (specify with -o)", + streami->index); + + /* Check for two streams outputting to the same file */ + if (streami != stream) { + const char *a = stream->config.out_fn; + const char *b = streami->config.out_fn; + if (!strcmp(a, b) && strcmp(a, "/dev/null") && strcmp(a, ":nul")) + fatal("Stream %d: duplicate output file (from stream %d)", + streami->index, stream->index); + } + + /* Check for two streams sharing a stats file. */ + if (streami != stream) { + const char *a = stream->config.stats_fn; + const char *b = streami->config.stats_fn; + if (a && b && !strcmp(a, b)) + fatal("Stream %d: duplicate stats file (from stream %d)", + streami->index, stream->index); + } + +#if CONFIG_FP_MB_STATS + /* Check for two streams sharing a mb stats file. */ + if (streami != stream) { + const char *a = stream->config.fpmb_stats_fn; + const char *b = streami->config.fpmb_stats_fn; + if (a && b && !strcmp(a, b)) + fatal("Stream %d: duplicate mb stats file (from stream %d)", + streami->index, stream->index); + } +#endif + } +} + +static void set_stream_dimensions(struct stream_state *stream, unsigned int w, + unsigned int h) { + if (!stream->config.cfg.g_w) { + if (!stream->config.cfg.g_h) + stream->config.cfg.g_w = w; + else + stream->config.cfg.g_w = w * stream->config.cfg.g_h / h; + } + if (!stream->config.cfg.g_h) { + stream->config.cfg.g_h = h * stream->config.cfg.g_w / w; + } +} + +static const char *file_type_to_string(enum VideoFileType t) { + switch (t) { + case FILE_TYPE_RAW: return "RAW"; + case FILE_TYPE_Y4M: return "Y4M"; + default: return "Other"; + } +} + +static const char *image_format_to_string(aom_img_fmt_t f) { + switch (f) { + case AOM_IMG_FMT_I420: return "I420"; + case AOM_IMG_FMT_I422: return "I422"; + case AOM_IMG_FMT_I444: return "I444"; + case AOM_IMG_FMT_I440: return "I440"; + case AOM_IMG_FMT_YV12: return "YV12"; + case AOM_IMG_FMT_I42016: return "I42016"; + case AOM_IMG_FMT_I42216: return "I42216"; + case AOM_IMG_FMT_I44416: return "I44416"; + case AOM_IMG_FMT_I44016: return "I44016"; + default: return "Other"; + } +} + +static void show_stream_config(struct stream_state *stream, + struct AvxEncoderConfig *global, + struct AvxInputContext *input) { +#define SHOW(field) \ + fprintf(stderr, " %-28s = %d\n", #field, stream->config.cfg.field) + + if (stream->index == 0) { + fprintf(stderr, "Codec: %s\n", + aom_codec_iface_name(global->codec->codec_interface())); + fprintf(stderr, "Source file: %s File Type: %s Format: %s\n", + input->filename, file_type_to_string(input->file_type), + image_format_to_string(input->fmt)); + } + if (stream->next || stream->index) + fprintf(stderr, "\nStream Index: %d\n", stream->index); + fprintf(stderr, "Destination file: %s\n", stream->config.out_fn); + fprintf(stderr, "Encoder parameters:\n"); + + SHOW(g_usage); + SHOW(g_threads); + SHOW(g_profile); + SHOW(g_w); + SHOW(g_h); + SHOW(g_bit_depth); + SHOW(g_input_bit_depth); + SHOW(g_timebase.num); + SHOW(g_timebase.den); + SHOW(g_error_resilient); + SHOW(g_pass); + SHOW(g_lag_in_frames); + SHOW(rc_dropframe_thresh); + SHOW(rc_resize_allowed); + SHOW(rc_scaled_width); + SHOW(rc_scaled_height); + SHOW(rc_resize_up_thresh); + SHOW(rc_resize_down_thresh); + SHOW(rc_end_usage); + SHOW(rc_target_bitrate); + SHOW(rc_min_quantizer); + SHOW(rc_max_quantizer); + SHOW(rc_undershoot_pct); + SHOW(rc_overshoot_pct); + SHOW(rc_buf_sz); + SHOW(rc_buf_initial_sz); + SHOW(rc_buf_optimal_sz); + SHOW(rc_2pass_vbr_bias_pct); + SHOW(rc_2pass_vbr_minsection_pct); + SHOW(rc_2pass_vbr_maxsection_pct); + SHOW(kf_mode); + SHOW(kf_min_dist); + SHOW(kf_max_dist); +} + +static void open_output_file(struct stream_state *stream, + struct AvxEncoderConfig *global, + const struct AvxRational *pixel_aspect_ratio) { + const char *fn = stream->config.out_fn; + const struct aom_codec_enc_cfg *const cfg = &stream->config.cfg; + + if (cfg->g_pass == AOM_RC_FIRST_PASS) return; + + stream->file = strcmp(fn, "-") ? fopen(fn, "wb") : set_binary_mode(stdout); + + if (!stream->file) fatal("Failed to open output file"); + + if (stream->config.write_webm && fseek(stream->file, 0, SEEK_CUR)) + fatal("WebM output to pipes not supported."); + +#if CONFIG_WEBM_IO + if (stream->config.write_webm) { + stream->webm_ctx.stream = stream->file; + write_webm_file_header(&stream->webm_ctx, cfg, stream->config.stereo_fmt, + global->codec->fourcc, pixel_aspect_ratio); + } +#else + (void)pixel_aspect_ratio; +#endif + + if (!stream->config.write_webm) { + ivf_write_file_header(stream->file, cfg, global->codec->fourcc, 0); + } +} + +static void close_output_file(struct stream_state *stream, + unsigned int fourcc) { + const struct aom_codec_enc_cfg *const cfg = &stream->config.cfg; + + if (cfg->g_pass == AOM_RC_FIRST_PASS) return; + +#if CONFIG_WEBM_IO + if (stream->config.write_webm) { + write_webm_file_footer(&stream->webm_ctx); + } +#endif + + if (!stream->config.write_webm) { + if (!fseek(stream->file, 0, SEEK_SET)) + ivf_write_file_header(stream->file, &stream->config.cfg, fourcc, + stream->frames_out); + } + + fclose(stream->file); +} + +static void setup_pass(struct stream_state *stream, + struct AvxEncoderConfig *global, int pass) { + if (stream->config.stats_fn) { + if (!stats_open_file(&stream->stats, stream->config.stats_fn, pass)) + fatal("Failed to open statistics store"); + } else { + if (!stats_open_mem(&stream->stats, pass)) + fatal("Failed to open statistics store"); + } + +#if CONFIG_FP_MB_STATS + if (stream->config.fpmb_stats_fn) { + if (!stats_open_file(&stream->fpmb_stats, stream->config.fpmb_stats_fn, + pass)) + fatal("Failed to open mb statistics store"); + } else { + if (!stats_open_mem(&stream->fpmb_stats, pass)) + fatal("Failed to open mb statistics store"); + } +#endif + + stream->config.cfg.g_pass = global->passes == 2 + ? pass ? AOM_RC_LAST_PASS : AOM_RC_FIRST_PASS + : AOM_RC_ONE_PASS; + if (pass) { + stream->config.cfg.rc_twopass_stats_in = stats_get(&stream->stats); +#if CONFIG_FP_MB_STATS + stream->config.cfg.rc_firstpass_mb_stats_in = + stats_get(&stream->fpmb_stats); +#endif + } + + stream->cx_time = 0; + stream->nbytes = 0; + stream->frames_out = 0; +} + +static void initialize_encoder(struct stream_state *stream, + struct AvxEncoderConfig *global) { + int i; + int flags = 0; + + flags |= global->show_psnr ? AOM_CODEC_USE_PSNR : 0; + flags |= global->out_part ? AOM_CODEC_USE_OUTPUT_PARTITION : 0; +#if CONFIG_HIGHBITDEPTH + flags |= stream->config.use_16bit_internal ? AOM_CODEC_USE_HIGHBITDEPTH : 0; +#endif + + /* Construct Encoder Context */ + aom_codec_enc_init(&stream->encoder, global->codec->codec_interface(), + &stream->config.cfg, flags); + ctx_exit_on_error(&stream->encoder, "Failed to initialize encoder"); + + /* Note that we bypass the aom_codec_control wrapper macro because + * we're being clever to store the control IDs in an array. Real + * applications will want to make use of the enumerations directly + */ + for (i = 0; i < stream->config.arg_ctrl_cnt; i++) { + int ctrl = stream->config.arg_ctrls[i][0]; + int value = stream->config.arg_ctrls[i][1]; + if (aom_codec_control_(&stream->encoder, ctrl, value)) + fprintf(stderr, "Error: Tried to set control %d = %d\n", ctrl, value); + + ctx_exit_on_error(&stream->encoder, "Failed to control codec"); + } + +#if CONFIG_DECODERS + if (global->test_decode != TEST_DECODE_OFF) { + const AvxInterface *decoder = get_aom_decoder_by_name(global->codec->name); + aom_codec_dec_cfg_t cfg = { 0, 0, 0 }; + aom_codec_dec_init(&stream->decoder, decoder->codec_interface(), &cfg, 0); + +#if CONFIG_AV1_DECODER && CONFIG_EXT_TILE + if (strcmp(global->codec->name, "av1") == 0) { + aom_codec_control(&stream->decoder, AV1_SET_DECODE_TILE_ROW, -1); + ctx_exit_on_error(&stream->decoder, "Failed to set decode_tile_row"); + + aom_codec_control(&stream->decoder, AV1_SET_DECODE_TILE_COL, -1); + ctx_exit_on_error(&stream->decoder, "Failed to set decode_tile_col"); + } +#endif + } +#endif +} + +static void encode_frame(struct stream_state *stream, + struct AvxEncoderConfig *global, struct aom_image *img, + unsigned int frames_in) { + aom_codec_pts_t frame_start, next_frame_start; + struct aom_codec_enc_cfg *cfg = &stream->config.cfg; + struct aom_usec_timer timer; + + frame_start = + (cfg->g_timebase.den * (int64_t)(frames_in - 1) * global->framerate.den) / + cfg->g_timebase.num / global->framerate.num; + next_frame_start = + (cfg->g_timebase.den * (int64_t)(frames_in)*global->framerate.den) / + cfg->g_timebase.num / global->framerate.num; + +/* Scale if necessary */ +#if CONFIG_HIGHBITDEPTH + if (img) { + if ((img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) && + (img->d_w != cfg->g_w || img->d_h != cfg->g_h)) { + if (img->fmt != AOM_IMG_FMT_I42016) { + fprintf(stderr, "%s can only scale 4:2:0 inputs\n", exec_name); + exit(EXIT_FAILURE); + } +#if CONFIG_LIBYUV + if (!stream->img) { + stream->img = + aom_img_alloc(NULL, AOM_IMG_FMT_I42016, cfg->g_w, cfg->g_h, 16); + } + I420Scale_16( + (uint16 *)img->planes[AOM_PLANE_Y], img->stride[AOM_PLANE_Y] / 2, + (uint16 *)img->planes[AOM_PLANE_U], img->stride[AOM_PLANE_U] / 2, + (uint16 *)img->planes[AOM_PLANE_V], img->stride[AOM_PLANE_V] / 2, + img->d_w, img->d_h, (uint16 *)stream->img->planes[AOM_PLANE_Y], + stream->img->stride[AOM_PLANE_Y] / 2, + (uint16 *)stream->img->planes[AOM_PLANE_U], + stream->img->stride[AOM_PLANE_U] / 2, + (uint16 *)stream->img->planes[AOM_PLANE_V], + stream->img->stride[AOM_PLANE_V] / 2, stream->img->d_w, + stream->img->d_h, kFilterBox); + img = stream->img; +#else + stream->encoder.err = 1; + ctx_exit_on_error(&stream->encoder, + "Stream %d: Failed to encode frame.\n" + "Scaling disabled in this configuration. \n" + "To enable, configure with --enable-libyuv\n", + stream->index); +#endif + } + } +#endif + if (img && (img->d_w != cfg->g_w || img->d_h != cfg->g_h)) { + if (img->fmt != AOM_IMG_FMT_I420 && img->fmt != AOM_IMG_FMT_YV12) { + fprintf(stderr, "%s can only scale 4:2:0 8bpp inputs\n", exec_name); + exit(EXIT_FAILURE); + } +#if CONFIG_LIBYUV + if (!stream->img) + stream->img = + aom_img_alloc(NULL, AOM_IMG_FMT_I420, cfg->g_w, cfg->g_h, 16); + I420Scale( + img->planes[AOM_PLANE_Y], img->stride[AOM_PLANE_Y], + img->planes[AOM_PLANE_U], img->stride[AOM_PLANE_U], + img->planes[AOM_PLANE_V], img->stride[AOM_PLANE_V], img->d_w, img->d_h, + stream->img->planes[AOM_PLANE_Y], stream->img->stride[AOM_PLANE_Y], + stream->img->planes[AOM_PLANE_U], stream->img->stride[AOM_PLANE_U], + stream->img->planes[AOM_PLANE_V], stream->img->stride[AOM_PLANE_V], + stream->img->d_w, stream->img->d_h, kFilterBox); + img = stream->img; +#else + stream->encoder.err = 1; + ctx_exit_on_error(&stream->encoder, + "Stream %d: Failed to encode frame.\n" + "Scaling disabled in this configuration. \n" + "To enable, configure with --enable-libyuv\n", + stream->index); +#endif + } + + aom_usec_timer_start(&timer); + aom_codec_encode(&stream->encoder, img, frame_start, + (unsigned long)(next_frame_start - frame_start), 0, + global->deadline); + aom_usec_timer_mark(&timer); + stream->cx_time += aom_usec_timer_elapsed(&timer); + ctx_exit_on_error(&stream->encoder, "Stream %d: Failed to encode frame", + stream->index); +} + +static void update_quantizer_histogram(struct stream_state *stream) { + if (stream->config.cfg.g_pass != AOM_RC_FIRST_PASS) { + int q; + + aom_codec_control(&stream->encoder, AOME_GET_LAST_QUANTIZER_64, &q); + ctx_exit_on_error(&stream->encoder, "Failed to read quantizer"); + stream->counts[q]++; + } +} + +static void get_cx_data(struct stream_state *stream, + struct AvxEncoderConfig *global, int *got_data) { + const aom_codec_cx_pkt_t *pkt; + const struct aom_codec_enc_cfg *cfg = &stream->config.cfg; + aom_codec_iter_t iter = NULL; + + *got_data = 0; + while ((pkt = aom_codec_get_cx_data(&stream->encoder, &iter))) { + static size_t fsize = 0; + static FileOffset ivf_header_pos = 0; + + switch (pkt->kind) { + case AOM_CODEC_CX_FRAME_PKT: + if (!(pkt->data.frame.flags & AOM_FRAME_IS_FRAGMENT)) { + stream->frames_out++; + } + if (!global->quiet) + fprintf(stderr, " %6luF", (unsigned long)pkt->data.frame.sz); + + update_rate_histogram(stream->rate_hist, cfg, pkt); +#if CONFIG_WEBM_IO + if (stream->config.write_webm) { + write_webm_block(&stream->webm_ctx, cfg, pkt); + } +#endif + if (!stream->config.write_webm) { + if (pkt->data.frame.partition_id <= 0) { + ivf_header_pos = ftello(stream->file); + fsize = pkt->data.frame.sz; + + ivf_write_frame_header(stream->file, pkt->data.frame.pts, fsize); + } else { + fsize += pkt->data.frame.sz; + + if (!(pkt->data.frame.flags & AOM_FRAME_IS_FRAGMENT)) { + const FileOffset currpos = ftello(stream->file); + fseeko(stream->file, ivf_header_pos, SEEK_SET); + ivf_write_frame_size(stream->file, fsize); + fseeko(stream->file, currpos, SEEK_SET); + } + } + + (void)fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, + stream->file); + } + stream->nbytes += pkt->data.raw.sz; + + *got_data = 1; +#if CONFIG_DECODERS + if (global->test_decode != TEST_DECODE_OFF && !stream->mismatch_seen) { + aom_codec_decode(&stream->decoder, pkt->data.frame.buf, + (unsigned int)pkt->data.frame.sz, NULL, 0); + if (stream->decoder.err) { + warn_or_exit_on_error(&stream->decoder, + global->test_decode == TEST_DECODE_FATAL, + "Failed to decode frame %d in stream %d", + stream->frames_out + 1, stream->index); + stream->mismatch_seen = stream->frames_out + 1; + } + } +#endif + break; + case AOM_CODEC_STATS_PKT: + stream->frames_out++; + stats_write(&stream->stats, pkt->data.twopass_stats.buf, + pkt->data.twopass_stats.sz); + stream->nbytes += pkt->data.raw.sz; + break; +#if CONFIG_FP_MB_STATS + case AOM_CODEC_FPMB_STATS_PKT: + stats_write(&stream->fpmb_stats, pkt->data.firstpass_mb_stats.buf, + pkt->data.firstpass_mb_stats.sz); + stream->nbytes += pkt->data.raw.sz; + break; +#endif + case AOM_CODEC_PSNR_PKT: + + if (global->show_psnr) { + int i; + + stream->psnr_sse_total += pkt->data.psnr.sse[0]; + stream->psnr_samples_total += pkt->data.psnr.samples[0]; + for (i = 0; i < 4; i++) { + if (!global->quiet) + fprintf(stderr, "%.3f ", pkt->data.psnr.psnr[i]); + stream->psnr_totals[i] += pkt->data.psnr.psnr[i]; + } + stream->psnr_count++; + } + + break; + default: break; + } + } +} + +static void show_psnr(struct stream_state *stream, double peak) { + int i; + double ovpsnr; + + if (!stream->psnr_count) return; + + fprintf(stderr, "Stream %d PSNR (Overall/Avg/Y/U/V)", stream->index); + ovpsnr = sse_to_psnr((double)stream->psnr_samples_total, peak, + (double)stream->psnr_sse_total); + fprintf(stderr, " %.3f", ovpsnr); + + for (i = 0; i < 4; i++) { + fprintf(stderr, " %.3f", stream->psnr_totals[i] / stream->psnr_count); + } + fprintf(stderr, "\n"); +} + +static float usec_to_fps(uint64_t usec, unsigned int frames) { + return (float)(usec > 0 ? frames * 1000000.0 / (float)usec : 0); +} + +static void test_decode(struct stream_state *stream, + enum TestDecodeFatality fatal, + const AvxInterface *codec) { + aom_image_t enc_img, dec_img; + + if (stream->mismatch_seen) return; + + /* Get the internal reference frame */ + if (strcmp(codec->name, "vp8") == 0) { + struct aom_ref_frame ref_enc, ref_dec; + const unsigned int frame_width = (stream->config.cfg.g_w + 15) & ~15; + const unsigned int frame_height = (stream->config.cfg.g_h + 15) & ~15; + aom_img_alloc(&ref_enc.img, AOM_IMG_FMT_I420, frame_width, frame_height, 1); + enc_img = ref_enc.img; + aom_img_alloc(&ref_dec.img, AOM_IMG_FMT_I420, frame_width, frame_height, 1); + dec_img = ref_dec.img; + + ref_enc.frame_type = AOM_LAST_FRAME; + ref_dec.frame_type = AOM_LAST_FRAME; + aom_codec_control(&stream->encoder, AOM_COPY_REFERENCE, &ref_enc); + aom_codec_control(&stream->decoder, AOM_COPY_REFERENCE, &ref_dec); + } else { + aom_codec_control(&stream->encoder, AV1_GET_NEW_FRAME_IMAGE, &enc_img); + aom_codec_control(&stream->decoder, AV1_GET_NEW_FRAME_IMAGE, &dec_img); + +#if CONFIG_HIGHBITDEPTH + if ((enc_img.fmt & AOM_IMG_FMT_HIGHBITDEPTH) != + (dec_img.fmt & AOM_IMG_FMT_HIGHBITDEPTH)) { + if (enc_img.fmt & AOM_IMG_FMT_HIGHBITDEPTH) { + aom_image_t enc_hbd_img; + aom_img_alloc(&enc_hbd_img, enc_img.fmt - AOM_IMG_FMT_HIGHBITDEPTH, + enc_img.d_w, enc_img.d_h, 16); + aom_img_truncate_16_to_8(&enc_hbd_img, &enc_img); + enc_img = enc_hbd_img; + } + if (dec_img.fmt & AOM_IMG_FMT_HIGHBITDEPTH) { + aom_image_t dec_hbd_img; + aom_img_alloc(&dec_hbd_img, dec_img.fmt - AOM_IMG_FMT_HIGHBITDEPTH, + dec_img.d_w, dec_img.d_h, 16); + aom_img_truncate_16_to_8(&dec_hbd_img, &dec_img); + dec_img = dec_hbd_img; + } + } +#endif + } + ctx_exit_on_error(&stream->encoder, "Failed to get encoder reference frame"); + ctx_exit_on_error(&stream->decoder, "Failed to get decoder reference frame"); + + if (!aom_compare_img(&enc_img, &dec_img)) { + int y[4], u[4], v[4]; +#if CONFIG_HIGHBITDEPTH + if (enc_img.fmt & AOM_IMG_FMT_HIGHBITDEPTH) { + aom_find_mismatch_high(&enc_img, &dec_img, y, u, v); + } else { + aom_find_mismatch(&enc_img, &dec_img, y, u, v); + } +#else + aom_find_mismatch(&enc_img, &dec_img, y, u, v); +#endif + stream->decoder.err = 1; + warn_or_exit_on_error(&stream->decoder, fatal == TEST_DECODE_FATAL, + "Stream %d: Encode/decode mismatch on frame %d at" + " Y[%d, %d] {%d/%d}," + " U[%d, %d] {%d/%d}," + " V[%d, %d] {%d/%d}", + stream->index, stream->frames_out, y[0], y[1], y[2], + y[3], u[0], u[1], u[2], u[3], v[0], v[1], v[2], v[3]); + stream->mismatch_seen = stream->frames_out; + } + + aom_img_free(&enc_img); + aom_img_free(&dec_img); +} + +static void print_time(const char *label, int64_t etl) { + int64_t hours; + int64_t mins; + int64_t secs; + + if (etl >= 0) { + hours = etl / 3600; + etl -= hours * 3600; + mins = etl / 60; + etl -= mins * 60; + secs = etl; + + fprintf(stderr, "[%3s %2" PRId64 ":%02" PRId64 ":%02" PRId64 "] ", label, + hours, mins, secs); + } else { + fprintf(stderr, "[%3s unknown] ", label); + } +} + +int main(int argc, const char **argv_) { + int pass; + aom_image_t raw; +#if CONFIG_HIGHBITDEPTH + aom_image_t raw_shift; + int allocated_raw_shift = 0; + int use_16bit_internal = 0; + int input_shift = 0; +#endif + int frame_avail, got_data; + + struct AvxInputContext input; + struct AvxEncoderConfig global; + struct stream_state *streams = NULL; + char **argv, **argi; + uint64_t cx_time = 0; + int stream_cnt = 0; + int res = 0; + int profile_updated = 0; + + memset(&input, 0, sizeof(input)); + exec_name = argv_[0]; + + if (argc < 3) usage_exit(); + + /* Setup default input stream settings */ + input.framerate.numerator = 30; + input.framerate.denominator = 1; + input.only_i420 = 1; + input.bit_depth = 0; + + /* First parse the global configuration values, because we want to apply + * other parameters on top of the default configuration provided by the + * codec. + */ + argv = argv_dup(argc - 1, argv_ + 1); + parse_global_config(&global, argv); + + switch (global.color_type) { + case I420: input.fmt = AOM_IMG_FMT_I420; break; + case I422: input.fmt = AOM_IMG_FMT_I422; break; + case I444: input.fmt = AOM_IMG_FMT_I444; break; + case I440: input.fmt = AOM_IMG_FMT_I440; break; + case YV12: input.fmt = AOM_IMG_FMT_YV12; break; + } + + { + /* Now parse each stream's parameters. Using a local scope here + * due to the use of 'stream' as loop variable in FOREACH_STREAM + * loops + */ + struct stream_state *stream = NULL; + + do { + stream = new_stream(&global, stream); + stream_cnt++; + if (!streams) streams = stream; + } while (parse_stream_params(&global, stream, argv)); + } + + /* Check for unrecognized options */ + for (argi = argv; *argi; argi++) + if (argi[0][0] == '-' && argi[0][1]) + die("Error: Unrecognized option %s\n", *argi); + + FOREACH_STREAM(check_encoder_config(global.disable_warning_prompt, &global, + &stream->config.cfg);); + + /* Handle non-option arguments */ + input.filename = argv[0]; + + if (!input.filename) usage_exit(); + + /* Decide if other chroma subsamplings than 4:2:0 are supported */ + if (global.codec->fourcc == AV1_FOURCC) input.only_i420 = 0; + + for (pass = global.pass ? global.pass - 1 : 0; pass < global.passes; pass++) { + int frames_in = 0, seen_frames = 0; + int64_t estimated_time_left = -1; + int64_t average_rate = -1; + int64_t lagged_count = 0; + + open_input_file(&input); + + /* If the input file doesn't specify its w/h (raw files), try to get + * the data from the first stream's configuration. + */ + if (!input.width || !input.height) { + FOREACH_STREAM({ + if (stream->config.cfg.g_w && stream->config.cfg.g_h) { + input.width = stream->config.cfg.g_w; + input.height = stream->config.cfg.g_h; + break; + } + }); + } + + /* Update stream configurations from the input file's parameters */ + if (!input.width || !input.height) + fatal( + "Specify stream dimensions with --width (-w) " + " and --height (-h)"); + + /* If input file does not specify bit-depth but input-bit-depth parameter + * exists, assume that to be the input bit-depth. However, if the + * input-bit-depth paramter does not exist, assume the input bit-depth + * to be the same as the codec bit-depth. + */ + if (!input.bit_depth) { + FOREACH_STREAM({ + if (stream->config.cfg.g_input_bit_depth) + input.bit_depth = stream->config.cfg.g_input_bit_depth; + else + input.bit_depth = stream->config.cfg.g_input_bit_depth = + (int)stream->config.cfg.g_bit_depth; + }); + if (input.bit_depth > 8) input.fmt |= AOM_IMG_FMT_HIGHBITDEPTH; + } else { + FOREACH_STREAM( + { stream->config.cfg.g_input_bit_depth = input.bit_depth; }); + } + +#if CONFIG_HIGHBITDEPTH + FOREACH_STREAM({ + if (input.fmt != AOM_IMG_FMT_I420 && input.fmt != AOM_IMG_FMT_I42016) { + /* Automatically upgrade if input is non-4:2:0 but a 4:2:0 profile + was selected. */ + switch (stream->config.cfg.g_profile) { + case 0: + stream->config.cfg.g_profile = 1; + profile_updated = 1; + break; + case 2: + stream->config.cfg.g_profile = 3; + profile_updated = 1; + break; + default: break; + } + } + /* Automatically set the codec bit depth to match the input bit depth. + * Upgrade the profile if required. */ + if (stream->config.cfg.g_input_bit_depth > + (unsigned int)stream->config.cfg.g_bit_depth) { + stream->config.cfg.g_bit_depth = stream->config.cfg.g_input_bit_depth; + } + if (stream->config.cfg.g_bit_depth > 8) { + switch (stream->config.cfg.g_profile) { + case 0: + stream->config.cfg.g_profile = 2; + profile_updated = 1; + break; + case 1: + stream->config.cfg.g_profile = 3; + profile_updated = 1; + break; + default: break; + } + } + if (stream->config.cfg.g_profile > 1) { + stream->config.use_16bit_internal = 1; + } + if (profile_updated && !global.quiet) { + fprintf(stderr, + "Warning: automatically upgrading to profile %d to " + "match input format.\n", + stream->config.cfg.g_profile); + } + }); +#else + FOREACH_STREAM({ + if (input.fmt != AOM_IMG_FMT_I420 && input.fmt != AOM_IMG_FMT_I42016) { + /* Automatically upgrade if input is non-4:2:0 but a 4:2:0 profile + was selected. */ + switch (stream->config.cfg.g_profile) { + case 0: + stream->config.cfg.g_profile = 1; + profile_updated = 1; + break; + case 2: + stream->config.cfg.g_profile = 3; + profile_updated = 1; + break; + default: break; + } + } + if (profile_updated && !global.quiet) { + fprintf(stderr, + "Warning: automatically upgrading to profile %d to " + "match input format.\n", + stream->config.cfg.g_profile); + } + }); +#endif + + FOREACH_STREAM(set_stream_dimensions(stream, input.width, input.height)); + FOREACH_STREAM(validate_stream_config(stream, &global)); + + /* Ensure that --passes and --pass are consistent. If --pass is set and + * --passes=2, ensure --fpf was set. + */ + if (global.pass && global.passes == 2) + FOREACH_STREAM({ + if (!stream->config.stats_fn) + die("Stream %d: Must specify --fpf when --pass=%d" + " and --passes=2\n", + stream->index, global.pass); + }); + +#if !CONFIG_WEBM_IO + FOREACH_STREAM({ + if (stream->config.write_webm) { + stream->config.write_webm = 0; + warn( + "aomenc was compiled without WebM container support." + "Producing IVF output"); + } + }); +#endif + + /* Use the frame rate from the file only if none was specified + * on the command-line. + */ + if (!global.have_framerate) { + global.framerate.num = input.framerate.numerator; + global.framerate.den = input.framerate.denominator; + FOREACH_STREAM(stream->config.cfg.g_timebase.den = global.framerate.num; + stream->config.cfg.g_timebase.num = global.framerate.den); + } + + /* Show configuration */ + if (global.verbose && pass == 0) + FOREACH_STREAM(show_stream_config(stream, &global, &input)); + + if (pass == (global.pass ? global.pass - 1 : 0)) { + if (input.file_type == FILE_TYPE_Y4M) + /*The Y4M reader does its own allocation. + Just initialize this here to avoid problems if we never read any + frames.*/ + memset(&raw, 0, sizeof(raw)); + else + aom_img_alloc(&raw, input.fmt, input.width, input.height, 32); + + FOREACH_STREAM(stream->rate_hist = init_rate_histogram( + &stream->config.cfg, &global.framerate)); + } + + FOREACH_STREAM(setup_pass(stream, &global, pass)); + FOREACH_STREAM( + open_output_file(stream, &global, &input.pixel_aspect_ratio)); + FOREACH_STREAM(initialize_encoder(stream, &global)); + +#if CONFIG_HIGHBITDEPTH + if (strcmp(global.codec->name, "av1") == 0 || + strcmp(global.codec->name, "av1") == 0) { + // Check to see if at least one stream uses 16 bit internal. + // Currently assume that the bit_depths for all streams using + // highbitdepth are the same. + FOREACH_STREAM({ + if (stream->config.use_16bit_internal) { + use_16bit_internal = 1; + } + if (stream->config.cfg.g_profile == 0) { + input_shift = 0; + } else { + input_shift = (int)stream->config.cfg.g_bit_depth - + stream->config.cfg.g_input_bit_depth; + } + }); + } +#endif + + frame_avail = 1; + got_data = 0; + + while (frame_avail || got_data) { + struct aom_usec_timer timer; + + if (!global.limit || frames_in < global.limit) { + frame_avail = read_frame(&input, &raw); + + if (frame_avail) frames_in++; + seen_frames = + frames_in > global.skip_frames ? frames_in - global.skip_frames : 0; + + if (!global.quiet) { + float fps = usec_to_fps(cx_time, seen_frames); + fprintf(stderr, "\rPass %d/%d ", pass + 1, global.passes); + + if (stream_cnt == 1) + fprintf(stderr, "frame %4d/%-4d %7" PRId64 "B ", frames_in, + streams->frames_out, (int64_t)streams->nbytes); + else + fprintf(stderr, "frame %4d ", frames_in); + + fprintf(stderr, "%7" PRId64 " %s %.2f %s ", + cx_time > 9999999 ? cx_time / 1000 : cx_time, + cx_time > 9999999 ? "ms" : "us", fps >= 1.0 ? fps : fps * 60, + fps >= 1.0 ? "fps" : "fpm"); + print_time("ETA", estimated_time_left); + } + + } else { + frame_avail = 0; + } + + if (frames_in > global.skip_frames) { +#if CONFIG_HIGHBITDEPTH + aom_image_t *frame_to_encode; + if (input_shift || (use_16bit_internal && input.bit_depth == 8)) { + assert(use_16bit_internal); + // Input bit depth and stream bit depth do not match, so up + // shift frame to stream bit depth + if (!allocated_raw_shift) { + aom_img_alloc(&raw_shift, raw.fmt | AOM_IMG_FMT_HIGHBITDEPTH, + input.width, input.height, 32); + allocated_raw_shift = 1; + } + aom_img_upshift(&raw_shift, &raw, input_shift); + frame_to_encode = &raw_shift; + } else { + frame_to_encode = &raw; + } + aom_usec_timer_start(&timer); + if (use_16bit_internal) { + assert(frame_to_encode->fmt & AOM_IMG_FMT_HIGHBITDEPTH); + FOREACH_STREAM({ + if (stream->config.use_16bit_internal) + encode_frame(stream, &global, + frame_avail ? frame_to_encode : NULL, frames_in); + else + assert(0); + }); + } else { + assert((frame_to_encode->fmt & AOM_IMG_FMT_HIGHBITDEPTH) == 0); + FOREACH_STREAM(encode_frame(stream, &global, + frame_avail ? frame_to_encode : NULL, + frames_in)); + } +#else + aom_usec_timer_start(&timer); + FOREACH_STREAM(encode_frame(stream, &global, frame_avail ? &raw : NULL, + frames_in)); +#endif + aom_usec_timer_mark(&timer); + cx_time += aom_usec_timer_elapsed(&timer); + + FOREACH_STREAM(update_quantizer_histogram(stream)); + + got_data = 0; + FOREACH_STREAM(get_cx_data(stream, &global, &got_data)); + + if (!got_data && input.length && streams != NULL && + !streams->frames_out) { + lagged_count = global.limit ? seen_frames : ftello(input.file); + } else if (input.length) { + int64_t remaining; + int64_t rate; + + if (global.limit) { + const int64_t frame_in_lagged = (seen_frames - lagged_count) * 1000; + + rate = cx_time ? frame_in_lagged * (int64_t)1000000 / cx_time : 0; + remaining = 1000 * (global.limit - global.skip_frames - + seen_frames + lagged_count); + } else { + const int64_t input_pos = ftello(input.file); + const int64_t input_pos_lagged = input_pos - lagged_count; + const int64_t input_limit = input.length; + + rate = cx_time ? input_pos_lagged * (int64_t)1000000 / cx_time : 0; + remaining = input_limit - input_pos + lagged_count; + } + + average_rate = + (average_rate <= 0) ? rate : (average_rate * 7 + rate) / 8; + estimated_time_left = average_rate ? remaining / average_rate : -1; + } + + if (got_data && global.test_decode != TEST_DECODE_OFF) + FOREACH_STREAM(test_decode(stream, global.test_decode, global.codec)); + } + + fflush(stdout); + if (!global.quiet) fprintf(stderr, "\033[K"); + } + + if (stream_cnt > 1) fprintf(stderr, "\n"); + + if (!global.quiet) { + FOREACH_STREAM(fprintf( + stderr, "\rPass %d/%d frame %4d/%-4d %7" PRId64 "B %7" PRId64 + "b/f %7" PRId64 "b/s" + " %7" PRId64 " %s (%.2f fps)\033[K\n", + pass + 1, global.passes, frames_in, stream->frames_out, + (int64_t)stream->nbytes, + seen_frames ? (int64_t)(stream->nbytes * 8 / seen_frames) : 0, + seen_frames + ? (int64_t)stream->nbytes * 8 * (int64_t)global.framerate.num / + global.framerate.den / seen_frames + : 0, + stream->cx_time > 9999999 ? stream->cx_time / 1000 : stream->cx_time, + stream->cx_time > 9999999 ? "ms" : "us", + usec_to_fps(stream->cx_time, seen_frames))); + } + + if (global.show_psnr) { + if (global.codec->fourcc == AV1_FOURCC) { + FOREACH_STREAM( + show_psnr(stream, (1 << stream->config.cfg.g_input_bit_depth) - 1)); + } else { + FOREACH_STREAM(show_psnr(stream, 255.0)); + } + } + + FOREACH_STREAM(aom_codec_destroy(&stream->encoder)); + + if (global.test_decode != TEST_DECODE_OFF) { + FOREACH_STREAM(aom_codec_destroy(&stream->decoder)); + } + + close_input_file(&input); + + if (global.test_decode == TEST_DECODE_FATAL) { + FOREACH_STREAM(res |= stream->mismatch_seen); + } + FOREACH_STREAM(close_output_file(stream, global.codec->fourcc)); + + FOREACH_STREAM(stats_close(&stream->stats, global.passes - 1)); + +#if CONFIG_FP_MB_STATS + FOREACH_STREAM(stats_close(&stream->fpmb_stats, global.passes - 1)); +#endif + + if (global.pass) break; + } + + if (global.show_q_hist_buckets) + FOREACH_STREAM( + show_q_histogram(stream->counts, global.show_q_hist_buckets)); + + if (global.show_rate_hist_buckets) + FOREACH_STREAM(show_rate_histogram(stream->rate_hist, &stream->config.cfg, + global.show_rate_hist_buckets)); + FOREACH_STREAM(destroy_rate_histogram(stream->rate_hist)); + +#if CONFIG_INTERNAL_STATS + /* TODO(jkoleszar): This doesn't belong in this executable. Do it for now, + * to match some existing utilities. + */ + if (!(global.pass == 1 && global.passes == 2)) + FOREACH_STREAM({ + FILE *f = fopen("opsnr.stt", "a"); + if (stream->mismatch_seen) { + fprintf(f, "First mismatch occurred in frame %d\n", + stream->mismatch_seen); + } else { + fprintf(f, "No mismatch detected in recon buffers\n"); + } + fclose(f); + }); +#endif + +#if CONFIG_HIGHBITDEPTH + if (allocated_raw_shift) aom_img_free(&raw_shift); +#endif + aom_img_free(&raw); + free(argv); + free(streams); + return res ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/third_party/aom/aomenc.h b/third_party/aom/aomenc.h new file mode 100644 index 0000000000..248e58356e --- /dev/null +++ b/third_party/aom/aomenc.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOMENC_H_ +#define AOMENC_H_ + +#include "aom/aom_encoder.h" + +#ifdef __cplusplus +extern "C" { +#endif + +enum TestDecodeFatality { + TEST_DECODE_OFF, + TEST_DECODE_FATAL, + TEST_DECODE_WARN, +}; + +typedef enum { + I420, // 4:2:0 8+ bit-depth + I422, // 4:2:2 8+ bit-depth + I444, // 4:4:4 8+ bit-depth + I440, // 4:4:0 8+ bit-depth + YV12, // 4:2:0 with uv flipped, only 8-bit depth +} ColorInputType; + +struct AvxInterface; + +/* Configuration elements common to all streams. */ +struct AvxEncoderConfig { + const struct AvxInterface *codec; + int passes; + int pass; + int usage; + int deadline; + ColorInputType color_type; + int quiet; + int verbose; + int limit; + int skip_frames; + int show_psnr; + enum TestDecodeFatality test_decode; + int have_framerate; + struct aom_rational framerate; + int out_part; + int debug; + int show_q_hist_buckets; + int show_rate_hist_buckets; + int disable_warnings; + int disable_warning_prompt; + int experimental_bitstream; +}; + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOMENC_H_ diff --git a/third_party/aom/aomstats.c b/third_party/aom/aomstats.c new file mode 100644 index 0000000000..0cfeea2f14 --- /dev/null +++ b/third_party/aom/aomstats.c @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aomstats.h" + +#include +#include +#include + +#include "./tools_common.h" + +int stats_open_file(stats_io_t *stats, const char *fpf, int pass) { + int res; + stats->pass = pass; + + if (pass == 0) { + stats->file = fopen(fpf, "wb"); + stats->buf.sz = 0; + stats->buf.buf = NULL; + res = (stats->file != NULL); + } else { + size_t nbytes; + + stats->file = fopen(fpf, "rb"); + + if (stats->file == NULL) fatal("First-pass stats file does not exist!"); + + if (fseek(stats->file, 0, SEEK_END)) + fatal("First-pass stats file must be seekable!"); + + stats->buf.sz = stats->buf_alloc_sz = ftell(stats->file); + rewind(stats->file); + + stats->buf.buf = malloc(stats->buf_alloc_sz); + + if (!stats->buf.buf) + fatal("Failed to allocate first-pass stats buffer (%lu bytes)", + (unsigned int)stats->buf_alloc_sz); + + nbytes = fread(stats->buf.buf, 1, stats->buf.sz, stats->file); + res = (nbytes == stats->buf.sz); + } + + return res; +} + +int stats_open_mem(stats_io_t *stats, int pass) { + int res; + stats->pass = pass; + + if (!pass) { + stats->buf.sz = 0; + stats->buf_alloc_sz = 64 * 1024; + stats->buf.buf = malloc(stats->buf_alloc_sz); + } + + stats->buf_ptr = stats->buf.buf; + res = (stats->buf.buf != NULL); + return res; +} + +void stats_close(stats_io_t *stats, int last_pass) { + if (stats->file) { + if (stats->pass == last_pass) { + free(stats->buf.buf); + } + + fclose(stats->file); + stats->file = NULL; + } else { + if (stats->pass == last_pass) free(stats->buf.buf); + } +} + +void stats_write(stats_io_t *stats, const void *pkt, size_t len) { + if (stats->file) { + (void)fwrite(pkt, 1, len, stats->file); + } else { + if (stats->buf.sz + len > stats->buf_alloc_sz) { + size_t new_sz = stats->buf_alloc_sz + 64 * 1024; + char *new_ptr = realloc(stats->buf.buf, new_sz); + + if (new_ptr) { + stats->buf_ptr = new_ptr + (stats->buf_ptr - (char *)stats->buf.buf); + stats->buf.buf = new_ptr; + stats->buf_alloc_sz = new_sz; + } else { + fatal("Failed to realloc firstpass stats buffer."); + } + } + + memcpy(stats->buf_ptr, pkt, len); + stats->buf.sz += len; + stats->buf_ptr += len; + } +} + +aom_fixed_buf_t stats_get(stats_io_t *stats) { return stats->buf; } diff --git a/third_party/aom/aomstats.h b/third_party/aom/aomstats.h new file mode 100644 index 0000000000..6438093447 --- /dev/null +++ b/third_party/aom/aomstats.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOMSTATS_H_ +#define AOMSTATS_H_ + +#include + +#include "aom/aom_encoder.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* This structure is used to abstract the different ways of handling + * first pass statistics + */ +typedef struct { + aom_fixed_buf_t buf; + int pass; + FILE *file; + char *buf_ptr; + size_t buf_alloc_sz; +} stats_io_t; + +int stats_open_file(stats_io_t *stats, const char *fpf, int pass); +int stats_open_mem(stats_io_t *stats, int pass); +void stats_close(stats_io_t *stats, int last_pass); +void stats_write(stats_io_t *stats, const void *pkt, size_t len); +aom_fixed_buf_t stats_get(stats_io_t *stats); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOMSTATS_H_ diff --git a/third_party/aom/args.c b/third_party/aom/args.c new file mode 100644 index 0000000000..571103595f --- /dev/null +++ b/third_party/aom/args.c @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include "args.h" + +#include "aom/aom_integer.h" +#include "aom_ports/msvc.h" + +#if defined(__GNUC__) && __GNUC__ +extern void die(const char *fmt, ...) __attribute__((noreturn)); +#else +extern void die(const char *fmt, ...); +#endif + +struct arg arg_init(char **argv) { + struct arg a; + + a.argv = argv; + a.argv_step = 1; + a.name = NULL; + a.val = NULL; + a.def = NULL; + return a; +} + +int arg_match(struct arg *arg_, const struct arg_def *def, char **argv) { + struct arg arg; + + if (!argv[0] || argv[0][0] != '-') return 0; + + arg = arg_init(argv); + + if (def->short_name && strlen(arg.argv[0]) == strlen(def->short_name) + 1 && + !strcmp(arg.argv[0] + 1, def->short_name)) { + arg.name = arg.argv[0] + 1; + arg.val = def->has_val ? arg.argv[1] : NULL; + arg.argv_step = def->has_val ? 2 : 1; + } else if (def->long_name) { + const size_t name_len = strlen(def->long_name); + + if (strlen(arg.argv[0]) >= name_len + 2 && arg.argv[0][1] == '-' && + !strncmp(arg.argv[0] + 2, def->long_name, name_len) && + (arg.argv[0][name_len + 2] == '=' || + arg.argv[0][name_len + 2] == '\0')) { + arg.name = arg.argv[0] + 2; + arg.val = arg.name[name_len] == '=' ? arg.name + name_len + 1 : NULL; + arg.argv_step = 1; + } + } + + if (arg.name && !arg.val && def->has_val) + die("Error: option %s requires argument.\n", arg.name); + + if (arg.name && arg.val && !def->has_val) + die("Error: option %s requires no argument.\n", arg.name); + + if (arg.name && (arg.val || !def->has_val)) { + arg.def = def; + *arg_ = arg; + return 1; + } + + return 0; +} + +const char *arg_next(struct arg *arg) { + if (arg->argv[0]) arg->argv += arg->argv_step; + + return *arg->argv; +} + +char **argv_dup(int argc, const char **argv) { + char **new_argv = malloc((argc + 1) * sizeof(*argv)); + + memcpy(new_argv, argv, argc * sizeof(*argv)); + new_argv[argc] = NULL; + return new_argv; +} + +void arg_show_usage(FILE *fp, const struct arg_def *const *defs) { + char option_text[40] = { 0 }; + + for (; *defs; defs++) { + const struct arg_def *def = *defs; + char *short_val = def->has_val ? " " : ""; + char *long_val = def->has_val ? "=" : ""; + + if (def->short_name && def->long_name) { + char *comma = def->has_val ? "," : ", "; + + snprintf(option_text, 37, "-%s%s%s --%s%6s", def->short_name, short_val, + comma, def->long_name, long_val); + } else if (def->short_name) + snprintf(option_text, 37, "-%s%s", def->short_name, short_val); + else if (def->long_name) + snprintf(option_text, 37, " --%s%s", def->long_name, long_val); + + fprintf(fp, " %-37s\t%s\n", option_text, def->desc); + + if (def->enums) { + const struct arg_enum_list *listptr; + + fprintf(fp, " %-37s\t ", ""); + + for (listptr = def->enums; listptr->name; listptr++) + fprintf(fp, "%s%s", listptr->name, listptr[1].name ? ", " : "\n"); + } + } +} + +unsigned int arg_parse_uint(const struct arg *arg) { + char *endptr; + const unsigned long rawval = strtoul(arg->val, &endptr, 10); // NOLINT + + if (arg->val[0] != '\0' && endptr[0] == '\0') { + if (rawval <= UINT_MAX) return (unsigned int)rawval; + + die("Option %s: Value %lu out of range for unsigned int\n", arg->name, + rawval); + } + + die("Option %s: Invalid character '%c'\n", arg->name, *endptr); + return 0; +} + +int arg_parse_int(const struct arg *arg) { + char *endptr; + const long rawval = strtol(arg->val, &endptr, 10); // NOLINT + + if (arg->val[0] != '\0' && endptr[0] == '\0') { + if (rawval >= INT_MIN && rawval <= INT_MAX) return (int)rawval; + + die("Option %s: Value %ld out of range for signed int\n", arg->name, + rawval); + } + + die("Option %s: Invalid character '%c'\n", arg->name, *endptr); + return 0; +} + +struct aom_rational { + int num; /**< fraction numerator */ + int den; /**< fraction denominator */ +}; +struct aom_rational arg_parse_rational(const struct arg *arg) { + long int rawval; + char *endptr; + struct aom_rational rat; + + /* parse numerator */ + rawval = strtol(arg->val, &endptr, 10); + + if (arg->val[0] != '\0' && endptr[0] == '/') { + if (rawval >= INT_MIN && rawval <= INT_MAX) + rat.num = (int)rawval; + else + die("Option %s: Value %ld out of range for signed int\n", arg->name, + rawval); + } else + die("Option %s: Expected / at '%c'\n", arg->name, *endptr); + + /* parse denominator */ + rawval = strtol(endptr + 1, &endptr, 10); + + if (arg->val[0] != '\0' && endptr[0] == '\0') { + if (rawval >= INT_MIN && rawval <= INT_MAX) + rat.den = (int)rawval; + else + die("Option %s: Value %ld out of range for signed int\n", arg->name, + rawval); + } else + die("Option %s: Invalid character '%c'\n", arg->name, *endptr); + + return rat; +} + +int arg_parse_enum(const struct arg *arg) { + const struct arg_enum_list *listptr; + long int rawval; + char *endptr; + + /* First see if the value can be parsed as a raw value */ + rawval = strtol(arg->val, &endptr, 10); + if (arg->val[0] != '\0' && endptr[0] == '\0') { + /* Got a raw value, make sure it's valid */ + for (listptr = arg->def->enums; listptr->name; listptr++) + if (listptr->val == rawval) return (int)rawval; + } + + /* Next see if it can be parsed as a string */ + for (listptr = arg->def->enums; listptr->name; listptr++) + if (!strcmp(arg->val, listptr->name)) return listptr->val; + + die("Option %s: Invalid value '%s'\n", arg->name, arg->val); + return 0; +} + +int arg_parse_enum_or_int(const struct arg *arg) { + if (arg->def->enums) return arg_parse_enum(arg); + return arg_parse_int(arg); +} diff --git a/third_party/aom/args.h b/third_party/aom/args.h new file mode 100644 index 0000000000..e7841fc646 --- /dev/null +++ b/third_party/aom/args.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef ARGS_H_ +#define ARGS_H_ +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct arg { + char **argv; + const char *name; + const char *val; + unsigned int argv_step; + const struct arg_def *def; +}; + +struct arg_enum_list { + const char *name; + int val; +}; +#define ARG_ENUM_LIST_END \ + { 0 } + +typedef struct arg_def { + const char *short_name; + const char *long_name; + int has_val; + const char *desc; + const struct arg_enum_list *enums; +} arg_def_t; +#define ARG_DEF(s, l, v, d) \ + { s, l, v, d, NULL } +#define ARG_DEF_ENUM(s, l, v, d, e) \ + { s, l, v, d, e } +#define ARG_DEF_LIST_END \ + { 0 } + +struct arg arg_init(char **argv); +int arg_match(struct arg *arg_, const struct arg_def *def, char **argv); +const char *arg_next(struct arg *arg); +void arg_show_usage(FILE *fp, const struct arg_def *const *defs); +char **argv_dup(int argc, const char **argv); + +unsigned int arg_parse_uint(const struct arg *arg); +int arg_parse_int(const struct arg *arg); +struct aom_rational arg_parse_rational(const struct arg *arg); +int arg_parse_enum(const struct arg *arg); +int arg_parse_enum_or_int(const struct arg *arg); +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // ARGS_H_ diff --git a/third_party/aom/av1/av1.cmake b/third_party/aom/av1/av1.cmake new file mode 100644 index 0000000000..00f687a0df --- /dev/null +++ b/third_party/aom/av1/av1.cmake @@ -0,0 +1,518 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +set(AOM_AV1_COMMON_SOURCES + "${AOM_ROOT}/av1/av1_iface_common.h" + "${AOM_ROOT}/av1/common/alloccommon.c" + "${AOM_ROOT}/av1/common/alloccommon.h" + # TODO(tomfinegan): Foward transform belongs in encoder. + "${AOM_ROOT}/av1/common/av1_fwd_txfm1d.c" + "${AOM_ROOT}/av1/common/av1_fwd_txfm1d.h" + "${AOM_ROOT}/av1/common/av1_fwd_txfm2d.c" + "${AOM_ROOT}/av1/common/av1_fwd_txfm2d_cfg.h" + "${AOM_ROOT}/av1/common/av1_inv_txfm1d.c" + "${AOM_ROOT}/av1/common/av1_inv_txfm1d.h" + "${AOM_ROOT}/av1/common/av1_inv_txfm2d.c" + "${AOM_ROOT}/av1/common/av1_inv_txfm2d_cfg.h" + "${AOM_ROOT}/av1/common/av1_loopfilter.c" + "${AOM_ROOT}/av1/common/av1_loopfilter.h" + "${AOM_ROOT}/av1/common/av1_txfm.h" + "${AOM_ROOT}/av1/common/blockd.c" + "${AOM_ROOT}/av1/common/blockd.h" + "${AOM_ROOT}/av1/common/common.h" + "${AOM_ROOT}/av1/common/common_data.h" + "${AOM_ROOT}/av1/common/convolve.c" + "${AOM_ROOT}/av1/common/convolve.h" + "${AOM_ROOT}/av1/common/debugmodes.c" + "${AOM_ROOT}/av1/common/entropy.c" + "${AOM_ROOT}/av1/common/entropy.h" + "${AOM_ROOT}/av1/common/entropymode.c" + "${AOM_ROOT}/av1/common/entropymode.h" + "${AOM_ROOT}/av1/common/entropymv.c" + "${AOM_ROOT}/av1/common/entropymv.h" + "${AOM_ROOT}/av1/common/enums.h" + "${AOM_ROOT}/av1/common/filter.c" + "${AOM_ROOT}/av1/common/filter.h" + "${AOM_ROOT}/av1/common/frame_buffers.c" + "${AOM_ROOT}/av1/common/frame_buffers.h" + "${AOM_ROOT}/av1/common/idct.c" + "${AOM_ROOT}/av1/common/idct.h" + "${AOM_ROOT}/av1/common/mv.h" + "${AOM_ROOT}/av1/common/mvref_common.c" + "${AOM_ROOT}/av1/common/mvref_common.h" + "${AOM_ROOT}/av1/common/odintrin.c" + "${AOM_ROOT}/av1/common/odintrin.h" + "${AOM_ROOT}/av1/common/onyxc_int.h" + "${AOM_ROOT}/av1/common/pred_common.c" + "${AOM_ROOT}/av1/common/pred_common.h" + "${AOM_ROOT}/av1/common/quant_common.c" + "${AOM_ROOT}/av1/common/quant_common.h" + "${AOM_ROOT}/av1/common/reconinter.c" + "${AOM_ROOT}/av1/common/reconinter.h" + "${AOM_ROOT}/av1/common/reconintra.c" + "${AOM_ROOT}/av1/common/reconintra.h" + "${AOM_ROOT}/av1/common/resize.c" + "${AOM_ROOT}/av1/common/resize.h" + "${AOM_ROOT}/av1/common/restoration.h" + "${AOM_ROOT}/av1/common/scale.c" + "${AOM_ROOT}/av1/common/scale.h" + "${AOM_ROOT}/av1/common/scan.c" + "${AOM_ROOT}/av1/common/scan.h" + "${AOM_ROOT}/av1/common/seg_common.c" + "${AOM_ROOT}/av1/common/seg_common.h" + "${AOM_ROOT}/av1/common/thread_common.c" + "${AOM_ROOT}/av1/common/thread_common.h" + "${AOM_ROOT}/av1/common/tile_common.c" + "${AOM_ROOT}/av1/common/tile_common.h") + +set(AOM_AV1_DECODER_SOURCES + "${AOM_ROOT}/av1/av1_dx_iface.c" + "${AOM_ROOT}/av1/decoder/decodeframe.c" + "${AOM_ROOT}/av1/decoder/decodeframe.h" + "${AOM_ROOT}/av1/decoder/decodemv.c" + "${AOM_ROOT}/av1/decoder/decodemv.h" + "${AOM_ROOT}/av1/decoder/decoder.c" + "${AOM_ROOT}/av1/decoder/decoder.h" + "${AOM_ROOT}/av1/decoder/detokenize.c" + "${AOM_ROOT}/av1/decoder/detokenize.h" + "${AOM_ROOT}/av1/decoder/dsubexp.c" + "${AOM_ROOT}/av1/decoder/dsubexp.h" + "${AOM_ROOT}/av1/decoder/dthread.c" + "${AOM_ROOT}/av1/decoder/dthread.h") + +set(AOM_AV1_ENCODER_SOURCES + "${AOM_ROOT}/av1/av1_cx_iface.c" + "${AOM_ROOT}/av1/encoder/aq_complexity.c" + "${AOM_ROOT}/av1/encoder/aq_complexity.h" + "${AOM_ROOT}/av1/encoder/aq_cyclicrefresh.c" + "${AOM_ROOT}/av1/encoder/aq_cyclicrefresh.h" + "${AOM_ROOT}/av1/encoder/aq_variance.c" + "${AOM_ROOT}/av1/encoder/aq_variance.h" + "${AOM_ROOT}/av1/encoder/av1_quantize.c" + "${AOM_ROOT}/av1/encoder/av1_quantize.h" + "${AOM_ROOT}/av1/encoder/bitstream.c" + "${AOM_ROOT}/av1/encoder/bitstream.h" + "${AOM_ROOT}/av1/encoder/block.h" + "${AOM_ROOT}/av1/encoder/context_tree.c" + "${AOM_ROOT}/av1/encoder/context_tree.h" + "${AOM_ROOT}/av1/encoder/cost.c" + "${AOM_ROOT}/av1/encoder/cost.h" + "${AOM_ROOT}/av1/encoder/dct.c" + "${AOM_ROOT}/av1/encoder/encodeframe.c" + "${AOM_ROOT}/av1/encoder/encodeframe.h" + "${AOM_ROOT}/av1/encoder/encodemb.c" + "${AOM_ROOT}/av1/encoder/encodemb.h" + "${AOM_ROOT}/av1/encoder/encodemv.c" + "${AOM_ROOT}/av1/encoder/encodemv.h" + "${AOM_ROOT}/av1/encoder/encoder.c" + "${AOM_ROOT}/av1/encoder/encoder.h" + "${AOM_ROOT}/av1/encoder/ethread.c" + "${AOM_ROOT}/av1/encoder/ethread.h" + "${AOM_ROOT}/av1/encoder/extend.c" + "${AOM_ROOT}/av1/encoder/extend.h" + "${AOM_ROOT}/av1/encoder/firstpass.c" + "${AOM_ROOT}/av1/encoder/firstpass.h" + "${AOM_ROOT}/av1/encoder/hybrid_fwd_txfm.c" + "${AOM_ROOT}/av1/encoder/hybrid_fwd_txfm.h" + "${AOM_ROOT}/av1/encoder/lookahead.c" + "${AOM_ROOT}/av1/encoder/lookahead.h" + "${AOM_ROOT}/av1/encoder/mbgraph.c" + "${AOM_ROOT}/av1/encoder/mbgraph.h" + "${AOM_ROOT}/av1/encoder/mcomp.c" + "${AOM_ROOT}/av1/encoder/mcomp.h" + "${AOM_ROOT}/av1/encoder/picklpf.c" + "${AOM_ROOT}/av1/encoder/picklpf.h" + "${AOM_ROOT}/av1/encoder/ratectrl.c" + "${AOM_ROOT}/av1/encoder/ratectrl.h" + "${AOM_ROOT}/av1/encoder/rd.c" + "${AOM_ROOT}/av1/encoder/rd.h" + "${AOM_ROOT}/av1/encoder/rdopt.c" + "${AOM_ROOT}/av1/encoder/rdopt.h" + "${AOM_ROOT}/av1/encoder/segmentation.c" + "${AOM_ROOT}/av1/encoder/segmentation.h" + "${AOM_ROOT}/av1/encoder/speed_features.c" + "${AOM_ROOT}/av1/encoder/speed_features.h" + "${AOM_ROOT}/av1/encoder/subexp.c" + "${AOM_ROOT}/av1/encoder/subexp.h" + "${AOM_ROOT}/av1/encoder/temporal_filter.c" + "${AOM_ROOT}/av1/encoder/temporal_filter.h" + "${AOM_ROOT}/av1/encoder/tokenize.c" + "${AOM_ROOT}/av1/encoder/tokenize.h" + "${AOM_ROOT}/av1/encoder/treewriter.c" + "${AOM_ROOT}/av1/encoder/treewriter.h" + "${AOM_ROOT}/av1/encoder/variance_tree.c" + "${AOM_ROOT}/av1/encoder/variance_tree.h") + +set(AOM_AV1_COMMON_INTRIN_SSE2 + # Requires CONFIG_GLOBAL_MOTION or CONFIG_WARPED_MOTION + #"${AOM_ROOT}/av1/common/x86/warp_plane_sse2.c" + "${AOM_ROOT}/av1/common/x86/idct_intrin_sse2.c") + +set(AOM_AV1_COMMON_INTRIN_SSSE3 + "${AOM_ROOT}/av1/common/x86/av1_convolve_ssse3.c") + +set(AOM_AV1_COMMON_INTRIN_SSE4_1 + "${AOM_ROOT}/av1/common/x86/av1_fwd_txfm1d_sse4.c" + "${AOM_ROOT}/av1/common/x86/av1_fwd_txfm2d_sse4.c") + +set(AOM_AV1_COMMON_INTRIN_AVX2 + "${AOM_ROOT}/av1/common/x86/hybrid_inv_txfm_avx2.c") + +set(AOM_AV1_COMMON_INTRIN_DSPR2 + "${AOM_ROOT}/av1/common/mips/dspr2/av1_itrans16_dspr2.c" + "${AOM_ROOT}/av1/common/mips/dspr2/av1_itrans4_dspr2.c" + "${AOM_ROOT}/av1/common/mips/dspr2/av1_itrans8_dspr2.c") + +set(AOM_AV1_COMMON_INTRIN_MSA + "${AOM_ROOT}/av1/common/mips/msa/av1_idct16x16_msa.c" + "${AOM_ROOT}/av1/common/mips/msa/av1_idct4x4_msa.c" + "${AOM_ROOT}/av1/common/mips/msa/av1_idct8x8_msa.c") + +set(AOM_AV1_ENCODER_ASM_SSE2 + "${AOM_ROOT}/av1/encoder/x86/dct_sse2.asm" + "${AOM_ROOT}/av1/encoder/x86/error_sse2.asm" + "${AOM_ROOT}/av1/encoder/x86/temporal_filter_apply_sse2.asm") + +set(AOM_AV1_ENCODER_INTRIN_SSE2 + "${AOM_ROOT}/av1/encoder/x86/dct_intrin_sse2.c" + "${AOM_ROOT}/av1/encoder/x86/highbd_block_error_intrin_sse2.c" + "${AOM_ROOT}/av1/encoder/x86/av1_quantize_sse2.c") + +set(AOM_AV1_ENCODER_ASM_SSSE3_X86_64 + "${AOM_ROOT}/av1/encoder/x86/av1_quantize_ssse3_x86_64.asm") + +set(AOM_AV1_ENCODER_INTRIN_SSSE3 + "${AOM_ROOT}/av1/encoder/x86/dct_ssse3.c") + +set(AOM_AV1_ENCODER_INTRIN_AVX2 + "${AOM_ROOT}/av1/encoder/x86/error_intrin_avx2.c" + "${AOM_ROOT}/av1/encoder/x86/hybrid_fwd_txfm_avx2.c") + +set(AOM_AV1_ENCODER_INTRIN_NEON + "${AOM_ROOT}/av1/encoder/arm/neon/quantize_neon.c") + +set(AOM_AV1_ENCODER_INTRIN_MSA + "${AOM_ROOT}/av1/encoder/mips/msa/error_msa.c" + "${AOM_ROOT}/av1/encoder/mips/msa/fdct16x16_msa.c" + "${AOM_ROOT}/av1/encoder/mips/msa/fdct4x4_msa.c" + "${AOM_ROOT}/av1/encoder/mips/msa/fdct8x8_msa.c" + "${AOM_ROOT}/av1/encoder/mips/msa/fdct_msa.h" + "${AOM_ROOT}/av1/encoder/mips/msa/temporal_filter_msa.c") + +if (CONFIG_HIGHBITDEPTH) + set(AOM_AV1_COMMON_INTRIN_SSE4_1 + ${AOM_AV1_COMMON_INTRIN_SSE4_1} + "${AOM_ROOT}/av1/common/x86/av1_highbd_convolve_sse4.c" + "${AOM_ROOT}/av1/common/x86/highbd_inv_txfm_sse4.c") + + set(AOM_AV1_COMMON_INTRIN_AVX2 + ${AOM_AV1_COMMON_INTRIN_AVX2} + "${AOM_ROOT}/av1/common/x86/highbd_inv_txfm_avx2.c") + + set(AOM_AV1_ENCODER_INTRIN_SSE4_1 + ${AOM_AV1_ENCODER_INTRIN_SSE4_1} + "${AOM_ROOT}/av1/encoder/x86/av1_highbd_quantize_sse4.c" + "${AOM_ROOT}/av1/encoder/x86/highbd_fwd_txfm_sse4.c") +else () + set(AOM_AV1_COMMON_INTRIN_NEON + ${AOM_AV1_COMMON_INTRIN_NEON} + "${AOM_ROOT}/av1/encoder/arm/neon/dct_neon.c" + "${AOM_ROOT}/av1/common/arm/neon/iht4x4_add_neon.c" + "${AOM_ROOT}/av1/common/arm/neon/iht8x8_add_neon.c") + + set(AOM_AV1_ENCODER_INTRIN_NEON + ${AOM_AV1_ENCODER_INTRIN_NEON} + "${AOM_ROOT}/av1/encoder/arm/neon/error_neon.c") +endif () + +if (CONFIG_CDEF) + set(AOM_AV1_COMMON_SOURCES + ${AOM_AV1_COMMON_SOURCES} + "${AOM_ROOT}/av1/common/clpf.c" + "${AOM_ROOT}/av1/common/clpf.h" + "${AOM_ROOT}/av1/common/clpf_simd.h" + "${AOM_ROOT}/av1/common/cdef_simd.h" + "${AOM_ROOT}/av1/common/cdef.c" + "${AOM_ROOT}/av1/common/cdef.h" + "${AOM_ROOT}/av1/common/od_dering.c" + "${AOM_ROOT}/av1/common/od_dering.h" + "${AOM_ROOT}/av1/common/od_dering_simd.h") + + set(AOM_AV1_ENCODER_SOURCES + ${AOM_AV1_ENCODER_SOURCES} + "${AOM_ROOT}/av1/encoder/pickcdef.c") + + set(AOM_AV1_COMMON_INTRIN_SSE2 + ${AOM_AV1_COMMON_INTRIN_SSE2} + "${AOM_ROOT}/av1/common/clpf_sse2.c" + "${AOM_ROOT}/av1/common/od_dering_sse2.c") + + set(AOM_AV1_COMMON_INTRIN_SSSE3 + ${AOM_AV1_COMMON_INTRIN_SSSE3} + "${AOM_ROOT}/av1/common/clpf_ssse3.c" + "${AOM_ROOT}/av1/common/od_dering_ssse3.c") + + set(AOM_AV1_COMMON_INTRIN_SSE4_1 + ${AOM_AV1_COMMON_INTRIN_SSE4_1} + "${AOM_ROOT}/av1/common/clpf_sse4.c" + "${AOM_ROOT}/av1/common/od_dering_sse4.c") + + set(AOM_AV1_COMMON_INTRIN_NEON + ${AOM_AV1_COMMON_INTRIN_NEON} + "${AOM_ROOT}/av1/common/clpf_neon.c" + "${AOM_ROOT}/av1/common/od_dering_neon.c") +endif () + +if (CONFIG_EXT_INTER) + set(AOM_AV1_ENCODER_SOURCES + ${AOM_AV1_ENCODER_SOURCES} + "${AOM_ROOT}/av1/encoder/wedge_utils.c") + + set(AOM_AV1_ENCODER_INTRIN_SSE2 + ${AOM_AV1_ENCODER_INTRIN_SSE2} + "${AOM_ROOT}/av1/encoder/x86/wedge_utils_sse2.c") +endif () + +if (CONFIG_FILTER_INTRA) + set(AOM_AV1_COMMON_INTRIN_SSE4_1 + ${AOM_AV1_COMMON_INTRIN_SSE4_1} + "${AOM_ROOT}/av1/common/x86/filterintra_sse4.c") +endif () + +if (CONFIG_ACCOUNTING) + set(AOM_AV1_DECODER_SOURCES + ${AOM_AV1_DECODER_SOURCES} + "${AOM_ROOT}/av1/decoder/accounting.c" + "${AOM_ROOT}/av1/decoder/accounting.h") +endif () + +if (CONFIG_INSPECTION) + set(AOM_AV1_DECODER_SOURCES + ${AOM_AV1_DECODER_SOURCES} + "${AOM_ROOT}/av1/decoder/inspection.c" + "${AOM_ROOT}/av1/decoder/inspection.h") +endif () + +if (CONFIG_INTERNAL_STATS) + set(AOM_AV1_ENCODER_SOURCES + ${AOM_AV1_ENCODER_SOURCES} + "${AOM_ROOT}/av1/encoder/blockiness.c") +endif () + +if (CONFIG_PALETTE) + set(AOM_AV1_ENCODER_SOURCES + ${AOM_AV1_ENCODER_SOURCES} + "${AOM_ROOT}/av1/encoder/palette.c" + "${AOM_ROOT}/av1/encoder/palette.h") +endif () + +if (CONFIG_CFL) + set(AOM_AV1_COMMON_SOURCES + ${AOM_AV1_COMMON_SOURCES} + "${AOM_ROOT}/av1/common/cfl.c" + "${AOM_ROOT}/av1/common/cfl.h") +endif () + +if (CONFIG_PVQ) + set(AOM_AV1_COMMON_SOURCES + ${AOM_AV1_COMMON_SOURCES} + "${AOM_ROOT}/av1/common/laplace_tables.c" + "${AOM_ROOT}/av1/common/pvq.c" + "${AOM_ROOT}/av1/common/pvq.h" + "${AOM_ROOT}/av1/common/pvq_state.c" + "${AOM_ROOT}/av1/common/pvq_state.h" + "${AOM_ROOT}/av1/common/partition.c" + "${AOM_ROOT}/av1/common/partition.h" + "${AOM_ROOT}/av1/common/generic_code.c" + "${AOM_ROOT}/av1/common/generic_code.h" + "${AOM_ROOT}/av1/common/zigzag4.c" + "${AOM_ROOT}/av1/common/zigzag8.c" + "${AOM_ROOT}/av1/common/zigzag16.c" + "${AOM_ROOT}/av1/common/zigzag32.c") + + set(AOM_AV1_DECODER_SOURCES + ${AOM_AV1_DECODER_SOURCES} + "${AOM_ROOT}/av1/decoder/decint.h" + "${AOM_ROOT}/av1/decoder/pvq_decoder.c" + "${AOM_ROOT}/av1/decoder/pvq_decoder.h" + "${AOM_ROOT}/av1/decoder/generic_decoder.c" + "${AOM_ROOT}/av1/decoder/laplace_decoder.c") + + set(AOM_AV1_ENCODER_SOURCES + ${AOM_AV1_ENCODER_SOURCES} + "${AOM_ROOT}/av1/encoder/daala_compat_enc.c" + "${AOM_ROOT}/av1/encoder/encint.h" + "${AOM_ROOT}/av1/encoder/pvq_encoder.c" + "${AOM_ROOT}/av1/encoder/pvq_encoder.h" + "${AOM_ROOT}/av1/encoder/generic_encoder.c" + "${AOM_ROOT}/av1/encoder/laplace_encoder.c") + + set(AOM_AV1_COMMON_INTRIN_SSE4_1 + ${AOM_AV1_COMMON_INTRIN_SSE4_1} + "${AOM_ROOT}/av1/common/x86/pvq_sse4.c" + "${AOM_ROOT}/av1/common/x86/pvq_sse4.h") + + if (NOT CONFIG_AV1_ENCODER) + # TODO(tomfinegan): These should probably be in av1/common, and in a + # common source list. For now this mirrors the original build system. + set(AOM_AV1_DECODER_SOURCES + ${AOM_AV1_DECODER_SOURCES} + "${AOM_ROOT}/av1/encoder/dct.c" + "${AOM_ROOT}/av1/encoder/hybrid_fwd_txfm.c" + "${AOM_ROOT}/av1/encoder/hybrid_fwd_txfm.h") + + set(AOM_AV1_DECODER_ASM_SSE2 + ${AOM_AV1_DECODER_ASM_SSE2} + "${AOM_ROOT}/av1/encoder/x86/dct_sse2.asm") + + set(AOM_AV1_DECODER_INTRIN_SSE2 + ${AOM_AV1_DECODER_INTRIN_SSE2} + "${AOM_ROOT}/av1/encoder/x86/dct_intrin_sse2.c") + + set(AOM_AV1_DECODER_INTRIN_SSSE3 + ${AOM_AV1_DECODER_INTRIN_SSSE3} + "${AOM_ROOT}/av1/encoder/x86/dct_ssse3.c") + endif () +endif () + +if (CONFIG_WARPED_MOTION) + set(AOM_AV1_COMMON_SOURCES + ${AOM_AV1_COMMON_SOURCES} + "${AOM_ROOT}/av1/common/warped_motion.c" + "${AOM_ROOT}/av1/common/warped_motion.h") + + set(AOM_AV1_COMMON_INTRIN_SSE2 + ${AOM_AV1_COMMON_INTRIN_SSE2} + "${AOM_ROOT}/av1/common/x86/warp_plane_sse2.c") +endif () + +# Setup AV1 common/decoder/encoder targets. The libaom target must exist before +# this function is called. +function (setup_av1_targets) + add_library(aom_av1_common OBJECT ${AOM_AV1_COMMON_SOURCES}) + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_av1_common) + target_sources(aom PUBLIC $) + + if (CONFIG_AV1_DECODER) + add_library(aom_av1_decoder OBJECT ${AOM_AV1_DECODER_SOURCES}) + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_av1_decoder) + target_sources(aom PUBLIC $) + endif () + + if (CONFIG_AV1_ENCODER) + add_library(aom_av1_encoder OBJECT ${AOM_AV1_ENCODER_SOURCES}) + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_av1_encoder) + target_sources(aom PUBLIC $) + endif () + + if (HAVE_SSE2) + require_flag_nomsvc("-msse2" NO) + add_intrinsics_object_library("-msse2" "sse2" "aom_av1_common" + "AOM_AV1_COMMON_INTRIN_SSE2") + if (CONFIG_AV1_DECODER) + if (AOM_AV1_DECODER_ASM_SSE2) + add_asm_library("aom_av1_decoder_sse2" "AOM_AV1_DECODER_ASM_SSE2" "aom") + endif () + + if (AOM_AV1_DECODER_INTRIN_SSE2) + add_intrinsics_object_library("-msse2" "sse2" "aom_av1_decoder" + "AOM_AV1_DECODER_INTRIN_SSE2") + endif () + endif () + + if (CONFIG_AV1_ENCODER) + add_asm_library("aom_av1_encoder_sse2" "AOM_AV1_ENCODER_ASM_SSE2" "aom") + add_intrinsics_object_library("-msse2" "sse2" "aom_av1_encoder" + "AOM_AV1_ENCODER_INTRIN_SSE2") + endif () + endif () + + if (HAVE_SSSE3) + require_flag_nomsvc("-mssse3" NO) + add_intrinsics_object_library("-mssse3" "ssse3" "aom_av1_common" + "AOM_AV1_COMMON_INTRIN_SSSE3") + + if (CONFIG_AV1_DECODER) + if (AOM_AV1_DECODER_INTRIN_SSSE3) + add_intrinsics_object_library("-mssse3" "ssse3" "aom_av1_decoder" + "AOM_AV1_DECODER_INTRIN_SSSE3") + endif () + endif () + + if (CONFIG_AV1_ENCODER) + add_intrinsics_object_library("-mssse3" "ssse3" "aom_av1_encoder" + "AOM_AV1_ENCODER_INTRIN_SSSE3") + endif () + endif () + + if (HAVE_SSE4_1) + require_flag_nomsvc("-msse4.1" NO) + add_intrinsics_object_library("-msse4.1" "sse4" "aom_av1_common" + "AOM_AV1_COMMON_INTRIN_SSE4_1") + + if (CONFIG_AV1_ENCODER) + if ("${AOM_TARGET_CPU}" STREQUAL "x86_64") + add_asm_library("aom_av1_encoder_ssse3" + "AOM_AV1_ENCODER_ASM_SSSE3_X86_64" "aom") + endif () + + if (AOM_AV1_ENCODER_INTRIN_SSE4_1) + add_intrinsics_object_library("-msse4.1" "sse4" "aom_av1_encoder" + "AOM_AV1_ENCODER_INTRIN_SSE4_1") + endif () + endif () + endif () + + if (HAVE_AVX2) + require_flag_nomsvc("-mavx2" NO) + add_intrinsics_object_library("-mavx2" "avx2" "aom_av1_common" + "AOM_AV1_COMMON_INTRIN_AVX2") + + if (CONFIG_AV1_ENCODER) + add_intrinsics_object_library("-mavx2" "avx2" "aom_av1_encoder" + "AOM_AV1_ENCODER_INTRIN_AVX2") + endif () + endif () + + if (HAVE_NEON) + if (AOM_AV1_COMMON_INTRIN_NEON) + add_intrinsics_object_library("${AOM_INTRIN_NEON_FLAG}" + "neon" + "aom_av1_common" + "AOM_AV1_COMMON_INTRIN_NEON") + endif () + + if (AOM_AV1_ENCODER_INTRIN_NEON) + add_intrinsics_object_library("${AOM_INTRIN_NEON_FLAG}" + "neon" + "aom_av1_encoder" + "AOM_AV1_ENCODER_INTRIN_NEON") + endif () + endif () + + if (HAVE_DSPR2) + add_intrinsics_object_library("" "dspr2" "aom_av1_common" + "AOM_AV1_COMMON_INTRIN_DSPR2") + endif () + + if (HAVE_MSA) + add_intrinsics_object_library("" "msa" "aom_av1_common" + "AOM_AV1_COMMON_INTRIN_MSA") + add_intrinsics_object_library("" "msa" "aom_av1_encoder" + "AOM_AV1_ENCODER_INTRIN_MSA") + endif () + + # Pass the new lib targets up to the parent scope instance of + # $AOM_LIB_TARGETS. + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} PARENT_SCOPE) +endfunction () + +function (setup_av1_test_targets) +endfunction () diff --git a/third_party/aom/av1/av1_common.mk b/third_party/aom/av1/av1_common.mk new file mode 100644 index 0000000000..6b9a289aff --- /dev/null +++ b/third_party/aom/av1/av1_common.mk @@ -0,0 +1,180 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +AV1_COMMON_SRCS-yes += av1_common.mk +AV1_COMMON_SRCS-yes += av1_iface_common.h +AV1_COMMON_SRCS-yes += common/alloccommon.c +AV1_COMMON_SRCS-yes += common/av1_loopfilter.c +AV1_COMMON_SRCS-yes += common/av1_loopfilter.h +AV1_COMMON_SRCS-yes += common/blockd.c +AV1_COMMON_SRCS-yes += common/debugmodes.c +AV1_COMMON_SRCS-yes += common/entropy.c +AV1_COMMON_SRCS-yes += common/entropymode.c +AV1_COMMON_SRCS-yes += common/entropymv.c +AV1_COMMON_SRCS-yes += common/frame_buffers.c +AV1_COMMON_SRCS-yes += common/frame_buffers.h +AV1_COMMON_SRCS-yes += common/alloccommon.h +AV1_COMMON_SRCS-yes += common/blockd.h +AV1_COMMON_SRCS-yes += common/common.h +AV1_COMMON_SRCS-yes += common/entropy.h +AV1_COMMON_SRCS-yes += common/entropymode.h +AV1_COMMON_SRCS-yes += common/entropymv.h +AV1_COMMON_SRCS-yes += common/enums.h +AV1_COMMON_SRCS-yes += common/filter.h +AV1_COMMON_SRCS-yes += common/filter.c +AV1_COMMON_SRCS-yes += common/idct.h +AV1_COMMON_SRCS-yes += common/idct.c +AV1_COMMON_SRCS-yes += common/thread_common.h +AV1_COMMON_SRCS-$(CONFIG_LV_MAP) += common/txb_common.h +AV1_COMMON_SRCS-$(CONFIG_LV_MAP) += common/txb_common.c +AV1_COMMON_SRCS-yes += common/mv.h +AV1_COMMON_SRCS-yes += common/onyxc_int.h +AV1_COMMON_SRCS-yes += common/pred_common.h +AV1_COMMON_SRCS-yes += common/pred_common.c +AV1_COMMON_SRCS-yes += common/quant_common.h +AV1_COMMON_SRCS-yes += common/reconinter.h +AV1_COMMON_SRCS-yes += common/reconintra.h +AV1_COMMON_SRCS-yes += common/av1_rtcd.c +AV1_COMMON_SRCS-yes += common/av1_rtcd_defs.pl +AV1_COMMON_SRCS-yes += common/scale.h +AV1_COMMON_SRCS-yes += common/scale.c +AV1_COMMON_SRCS-yes += common/seg_common.h +AV1_COMMON_SRCS-yes += common/seg_common.c +AV1_COMMON_SRCS-yes += common/tile_common.h +AV1_COMMON_SRCS-yes += common/tile_common.c +AV1_COMMON_SRCS-yes += common/thread_common.c +AV1_COMMON_SRCS-yes += common/mvref_common.c +AV1_COMMON_SRCS-yes += common/mvref_common.h +AV1_COMMON_SRCS-yes += common/quant_common.c +AV1_COMMON_SRCS-yes += common/reconinter.c +AV1_COMMON_SRCS-yes += common/reconintra.c +AV1_COMMON_SRCS-yes += common/resize.c +AV1_COMMON_SRCS-yes += common/resize.h +AV1_COMMON_SRCS-yes += common/restoration.h +AV1_COMMON_SRCS-yes += common/common_data.h +AV1_COMMON_SRCS-yes += common/scan.c +AV1_COMMON_SRCS-yes += common/scan.h +# TODO(angiebird) the forward transform belongs under encoder/ +AV1_COMMON_SRCS-yes += common/av1_txfm.h +AV1_COMMON_SRCS-yes += common/av1_fwd_txfm1d.h +AV1_COMMON_SRCS-yes += common/av1_fwd_txfm1d.c +AV1_COMMON_SRCS-yes += common/av1_inv_txfm1d.h +AV1_COMMON_SRCS-yes += common/av1_inv_txfm1d.c +AV1_COMMON_SRCS-yes += common/av1_fwd_txfm2d.c +AV1_COMMON_SRCS-yes += common/av1_fwd_txfm2d_cfg.h +AV1_COMMON_SRCS-yes += common/av1_inv_txfm2d.c +AV1_COMMON_SRCS-yes += common/av1_inv_txfm2d_cfg.h +AV1_COMMON_SRCS-$(HAVE_SSSE3) += common/x86/av1_convolve_ssse3.c +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/av1_highbd_convolve_sse4.c +endif +AV1_COMMON_SRCS-yes += common/convolve.c +AV1_COMMON_SRCS-yes += common/convolve.h +ifeq ($(CONFIG_LOOP_RESTORATION),yes) +AV1_COMMON_SRCS-yes += common/restoration.h +AV1_COMMON_SRCS-yes += common/restoration.c +AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/selfguided_sse4.c +endif +ifeq (yes,$(filter $(CONFIG_GLOBAL_MOTION) $(CONFIG_WARPED_MOTION),yes)) +AV1_COMMON_SRCS-yes += common/warped_motion.h +AV1_COMMON_SRCS-yes += common/warped_motion.c +endif +ifeq ($(CONFIG_CDEF),yes) +AV1_COMMON_SRCS-yes += common/clpf.c +AV1_COMMON_SRCS-yes += common/clpf.h +AV1_COMMON_SRCS-yes += common/clpf_simd.h +AV1_COMMON_SRCS-yes += common/cdef_simd.h +AV1_COMMON_SRCS-$(HAVE_SSE2) += common/clpf_sse2.c +AV1_COMMON_SRCS-$(HAVE_SSSE3) += common/clpf_ssse3.c +AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/clpf_sse4.c +AV1_COMMON_SRCS-$(HAVE_NEON) += common/clpf_neon.c +AV1_COMMON_SRCS-$(HAVE_SSE2) += common/od_dering_sse2.c +AV1_COMMON_SRCS-$(HAVE_SSSE3) += common/od_dering_ssse3.c +AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/od_dering_sse4.c +AV1_COMMON_SRCS-$(HAVE_NEON) += common/od_dering_neon.c +AV1_COMMON_SRCS-yes += common/od_dering.c +AV1_COMMON_SRCS-yes += common/od_dering.h +AV1_COMMON_SRCS-yes += common/od_dering_simd.h +AV1_COMMON_SRCS-yes += common/cdef.c +AV1_COMMON_SRCS-yes += common/cdef.h +endif +AV1_COMMON_SRCS-yes += common/odintrin.c +AV1_COMMON_SRCS-yes += common/odintrin.h + +ifeq ($(CONFIG_CFL),yes) +AV1_COMMON_SRCS-yes += common/cfl.h +AV1_COMMON_SRCS-yes += common/cfl.c +endif + +ifeq ($(CONFIG_PVQ),yes) +# PVQ from daala +AV1_COMMON_SRCS-yes += common/pvq.c +AV1_COMMON_SRCS-yes += common/partition.c +AV1_COMMON_SRCS-yes += common/partition.h +AV1_COMMON_SRCS-yes += common/zigzag4.c +AV1_COMMON_SRCS-yes += common/zigzag8.c +AV1_COMMON_SRCS-yes += common/zigzag16.c +AV1_COMMON_SRCS-yes += common/zigzag32.c +AV1_COMMON_SRCS-yes += common/zigzag.h +AV1_COMMON_SRCS-yes += common/generic_code.c +AV1_COMMON_SRCS-yes += common/pvq_state.c +AV1_COMMON_SRCS-yes += common/laplace_tables.c +AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/pvq_sse4.c +AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/pvq_sse4.h +endif +ifneq ($(findstring yes,$(CONFIG_PVQ)$(CONFIG_DAALA_DIST)$(CONFIG_XIPHRC)),) +AV1_COMMON_SRCS-yes += common/pvq.h +AV1_COMMON_SRCS-yes += common/pvq_state.h +AV1_COMMON_SRCS-yes += common/generic_code.h +endif + +ifneq ($(CONFIG_HIGHBITDEPTH),yes) +AV1_COMMON_SRCS-$(HAVE_DSPR2) += common/mips/dspr2/av1_itrans4_dspr2.c +AV1_COMMON_SRCS-$(HAVE_DSPR2) += common/mips/dspr2/av1_itrans8_dspr2.c +AV1_COMMON_SRCS-$(HAVE_DSPR2) += common/mips/dspr2/av1_itrans16_dspr2.c +endif + +# common (msa) +AV1_COMMON_SRCS-$(HAVE_MSA) += common/mips/msa/av1_idct4x4_msa.c +AV1_COMMON_SRCS-$(HAVE_MSA) += common/mips/msa/av1_idct8x8_msa.c +AV1_COMMON_SRCS-$(HAVE_MSA) += common/mips/msa/av1_idct16x16_msa.c + +AV1_COMMON_SRCS-$(HAVE_SSE2) += common/x86/idct_intrin_sse2.c +AV1_COMMON_SRCS-$(HAVE_AVX2) += common/x86/hybrid_inv_txfm_avx2.c + +ifeq ($(CONFIG_AV1_ENCODER),yes) +AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/av1_txfm1d_sse4.h +AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/av1_fwd_txfm1d_sse4.c +AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/av1_fwd_txfm2d_sse4.c +endif +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/highbd_txfm_utility_sse4.h +AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/highbd_inv_txfm_sse4.c +AV1_COMMON_SRCS-$(HAVE_AVX2) += common/x86/highbd_inv_txfm_avx2.c +endif + +ifneq ($(CONFIG_HIGHBITDEPTH),yes) +AV1_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/iht4x4_add_neon.c +AV1_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/iht8x8_add_neon.c +endif + +ifeq ($(CONFIG_FILTER_INTRA),yes) +AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/filterintra_sse4.c +endif + +ifneq ($(findstring yes,$(CONFIG_GLOBAL_MOTION) $(CONFIG_WARPED_MOTION)),) +AV1_COMMON_SRCS-$(HAVE_SSE2) += common/x86/warp_plane_sse2.c +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +AV1_COMMON_SRCS-$(HAVE_SSSE3) += common/x86/highbd_warp_plane_ssse3.c +endif +endif + +$(eval $(call rtcd_h_template,av1_rtcd,av1/common/av1_rtcd_defs.pl)) diff --git a/third_party/aom/av1/av1_cx.mk b/third_party/aom/av1/av1_cx.mk new file mode 100644 index 0000000000..0a0d770ce2 --- /dev/null +++ b/third_party/aom/av1/av1_cx.mk @@ -0,0 +1,165 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +AV1_CX_EXPORTS += exports_enc + +AV1_CX_SRCS-yes += $(AV1_COMMON_SRCS-yes) +AV1_CX_SRCS-no += $(AV1_COMMON_SRCS-no) +AV1_CX_SRCS_REMOVE-yes += $(AV1_COMMON_SRCS_REMOVE-yes) +AV1_CX_SRCS_REMOVE-no += $(AV1_COMMON_SRCS_REMOVE-no) + +AV1_CX_SRCS-yes += av1_cx_iface.c + +AV1_CX_SRCS-yes += encoder/av1_quantize.c +AV1_CX_SRCS-yes += encoder/av1_quantize.h +AV1_CX_SRCS-yes += encoder/bitstream.c +AV1_CX_SRCS-yes += encoder/context_tree.c +AV1_CX_SRCS-yes += encoder/context_tree.h +AV1_CX_SRCS-yes += encoder/variance_tree.c +AV1_CX_SRCS-yes += encoder/variance_tree.h +AV1_CX_SRCS-yes += encoder/cost.h +AV1_CX_SRCS-yes += encoder/cost.c +AV1_CX_SRCS-yes += encoder/dct.c +AV1_CX_SRCS-yes += encoder/hybrid_fwd_txfm.c +AV1_CX_SRCS-yes += encoder/hybrid_fwd_txfm.h +AV1_CX_SRCS-yes += encoder/encodeframe.c +AV1_CX_SRCS-yes += encoder/encodeframe.h +AV1_CX_SRCS-yes += encoder/encodemb.c +AV1_CX_SRCS-yes += encoder/encodemv.c +AV1_CX_SRCS-yes += encoder/ethread.h +AV1_CX_SRCS-yes += encoder/ethread.c +AV1_CX_SRCS-yes += encoder/extend.c +AV1_CX_SRCS-yes += encoder/firstpass.c +AV1_CX_SRCS-$(CONFIG_GLOBAL_MOTION) += ../third_party/fastfeat/fast.h +AV1_CX_SRCS-$(CONFIG_GLOBAL_MOTION) += ../third_party/fastfeat/nonmax.c +AV1_CX_SRCS-$(CONFIG_GLOBAL_MOTION) += ../third_party/fastfeat/fast_9.c +AV1_CX_SRCS-$(CONFIG_GLOBAL_MOTION) += ../third_party/fastfeat/fast.c +AV1_CX_SRCS-$(CONFIG_GLOBAL_MOTION) += encoder/corner_match.c +AV1_CX_SRCS-$(CONFIG_GLOBAL_MOTION) += encoder/corner_match.h +AV1_CX_SRCS-$(CONFIG_GLOBAL_MOTION) += encoder/corner_detect.c +AV1_CX_SRCS-$(CONFIG_GLOBAL_MOTION) += encoder/corner_detect.h +AV1_CX_SRCS-$(CONFIG_GLOBAL_MOTION) += encoder/global_motion.c +AV1_CX_SRCS-$(CONFIG_GLOBAL_MOTION) += encoder/global_motion.h +AV1_CX_SRCS-$(CONFIG_GLOBAL_MOTION) += encoder/ransac.c +AV1_CX_SRCS-$(CONFIG_GLOBAL_MOTION) += encoder/ransac.h +AV1_CX_SRCS-yes += encoder/block.h +AV1_CX_SRCS-yes += encoder/bitstream.h +AV1_CX_SRCS-yes += encoder/encodemb.h +AV1_CX_SRCS-yes += encoder/encodemv.h +AV1_CX_SRCS-$(CONFIG_LV_MAP) += encoder/encodetxb.c +AV1_CX_SRCS-$(CONFIG_LV_MAP) += encoder/encodetxb.h +AV1_CX_SRCS-yes += encoder/extend.h +AV1_CX_SRCS-yes += encoder/firstpass.h +AV1_CX_SRCS-yes += encoder/lookahead.c +AV1_CX_SRCS-yes += encoder/lookahead.h +AV1_CX_SRCS-yes += encoder/mcomp.h +AV1_CX_SRCS-yes += encoder/encoder.h +AV1_CX_SRCS-yes += encoder/ratectrl.h +ifeq ($(CONFIG_XIPHRC),yes) +AV1_CX_SRCS-yes += encoder/ratectrl_xiph.h +endif +AV1_CX_SRCS-yes += encoder/rd.h +AV1_CX_SRCS-yes += encoder/rdopt.h +AV1_CX_SRCS-yes += encoder/tokenize.h +AV1_CX_SRCS-yes += encoder/treewriter.h +AV1_CX_SRCS-yes += encoder/mcomp.c +AV1_CX_SRCS-yes += encoder/encoder.c +ifeq ($(CONFIG_PALETTE),yes) +AV1_CX_SRCS-yes += encoder/palette.h +AV1_CX_SRCS-yes += encoder/palette.c +endif +AV1_CX_SRCS-yes += encoder/picklpf.c +AV1_CX_SRCS-yes += encoder/picklpf.h +AV1_CX_SRCS-$(CONFIG_LOOP_RESTORATION) += encoder/pickrst.c +AV1_CX_SRCS-$(CONFIG_LOOP_RESTORATION) += encoder/pickrst.h +AV1_CX_SRCS-yes += encoder/ratectrl.c +ifeq ($(CONFIG_XIPHRC),yes) +AV1_CX_SRCS-yes += encoder/ratectrl_xiph.c +endif +AV1_CX_SRCS-yes += encoder/rd.c +AV1_CX_SRCS-yes += encoder/rdopt.c +AV1_CX_SRCS-yes += encoder/segmentation.c +AV1_CX_SRCS-yes += encoder/segmentation.h +AV1_CX_SRCS-yes += encoder/speed_features.c +AV1_CX_SRCS-yes += encoder/speed_features.h +AV1_CX_SRCS-yes += encoder/subexp.c +AV1_CX_SRCS-yes += encoder/subexp.h +AV1_CX_SRCS-$(CONFIG_INTERNAL_STATS) += encoder/blockiness.c + +AV1_CX_SRCS-yes += encoder/tokenize.c +AV1_CX_SRCS-yes += encoder/treewriter.c +AV1_CX_SRCS-yes += encoder/aq_variance.c +AV1_CX_SRCS-yes += encoder/aq_variance.h +AV1_CX_SRCS-yes += encoder/aq_cyclicrefresh.c +AV1_CX_SRCS-yes += encoder/aq_cyclicrefresh.h +AV1_CX_SRCS-yes += encoder/aq_complexity.c +AV1_CX_SRCS-yes += encoder/aq_complexity.h +AV1_CX_SRCS-yes += encoder/temporal_filter.c +AV1_CX_SRCS-yes += encoder/temporal_filter.h +AV1_CX_SRCS-yes += encoder/mbgraph.c +AV1_CX_SRCS-yes += encoder/mbgraph.h +ifeq ($(CONFIG_CDEF),yes) +AV1_CX_SRCS-yes += encoder/pickcdef.c +endif +ifeq ($(CONFIG_PVQ),yes) +# PVQ from daala +AV1_CX_SRCS-yes += encoder/daala_compat_enc.c +AV1_CX_SRCS-yes += encoder/pvq_encoder.c +AV1_CX_SRCS-yes += encoder/pvq_encoder.h +AV1_CX_SRCS-yes += encoder/generic_encoder.c +AV1_CX_SRCS-yes += encoder/laplace_encoder.c +endif +ifneq ($(findstring yes,$(CONFIG_XIPHRC)$(CONFIG_PVQ)),) +AV1_CX_SRCS-yes += encoder/encint.h +endif + +AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/av1_quantize_sse2.c +AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/temporal_filter_apply_sse2.asm +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/highbd_block_error_intrin_sse2.c +endif + +AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/dct_sse2.asm +AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/error_sse2.asm + +ifeq ($(ARCH_X86_64),yes) +AV1_CX_SRCS-$(HAVE_SSSE3) += encoder/x86/av1_quantize_ssse3_x86_64.asm +endif + +AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/dct_intrin_sse2.c +AV1_CX_SRCS-$(HAVE_SSSE3) += encoder/x86/dct_ssse3.c +AV1_CX_SRCS-$(HAVE_AVX2) += encoder/x86/hybrid_fwd_txfm_avx2.c +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +AV1_CX_SRCS-$(HAVE_SSE4_1) += encoder/x86/av1_highbd_quantize_sse4.c +AV1_CX_SRCS-$(HAVE_SSE4_1) += encoder/x86/highbd_fwd_txfm_sse4.c +endif + +ifeq ($(CONFIG_EXT_INTER),yes) +AV1_CX_SRCS-yes += encoder/wedge_utils.c +AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/wedge_utils_sse2.c +endif + +AV1_CX_SRCS-$(HAVE_AVX2) += encoder/x86/error_intrin_avx2.c + +ifneq ($(CONFIG_HIGHBITDEPTH),yes) +AV1_CX_SRCS-$(HAVE_NEON) += encoder/arm/neon/dct_neon.c +AV1_CX_SRCS-$(HAVE_NEON) += encoder/arm/neon/error_neon.c +endif +AV1_CX_SRCS-$(HAVE_NEON) += encoder/arm/neon/quantize_neon.c + +AV1_CX_SRCS-$(HAVE_MSA) += encoder/mips/msa/error_msa.c +AV1_CX_SRCS-$(HAVE_MSA) += encoder/mips/msa/fdct4x4_msa.c +AV1_CX_SRCS-$(HAVE_MSA) += encoder/mips/msa/fdct8x8_msa.c +AV1_CX_SRCS-$(HAVE_MSA) += encoder/mips/msa/fdct16x16_msa.c +AV1_CX_SRCS-$(HAVE_MSA) += encoder/mips/msa/fdct_msa.h +AV1_CX_SRCS-$(HAVE_MSA) += encoder/mips/msa/temporal_filter_msa.c + +AV1_CX_SRCS-yes := $(filter-out $(AV1_CX_SRCS_REMOVE-yes),$(AV1_CX_SRCS-yes)) diff --git a/third_party/aom/av1/av1_cx_iface.c b/third_party/aom/av1/av1_cx_iface.c new file mode 100644 index 0000000000..d4832a15c2 --- /dev/null +++ b/third_party/aom/av1/av1_cx_iface.c @@ -0,0 +1,1605 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "aom/aom_encoder.h" +#include "aom_ports/aom_once.h" +#include "aom_ports/system_state.h" +#include "aom/internal/aom_codec_internal.h" +#include "./aom_version.h" +#include "av1/encoder/encoder.h" +#include "aom/aomcx.h" +#include "av1/encoder/firstpass.h" +#include "av1/av1_iface_common.h" + +struct av1_extracfg { + int cpu_used; // available cpu percentage in 1/16 + unsigned int enable_auto_alt_ref; +#if CONFIG_EXT_REFS + unsigned int enable_auto_bwd_ref; +#endif // CONFIG_EXT_REFS + unsigned int noise_sensitivity; + unsigned int sharpness; + unsigned int static_thresh; + unsigned int tile_columns; + unsigned int tile_rows; +#if CONFIG_DEPENDENT_HORZTILES + unsigned int dependent_horz_tiles; +#endif +#if CONFIG_LOOPFILTERING_ACROSS_TILES + unsigned int loop_filter_across_tiles_enabled; +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + unsigned int arnr_max_frames; + unsigned int arnr_strength; + unsigned int min_gf_interval; + unsigned int max_gf_interval; + aom_tune_metric tuning; + unsigned int cq_level; // constrained quality level + unsigned int rc_max_intra_bitrate_pct; + unsigned int rc_max_inter_bitrate_pct; + unsigned int gf_cbr_boost_pct; + unsigned int lossless; +#if CONFIG_AOM_QM + unsigned int enable_qm; + unsigned int qm_min; + unsigned int qm_max; +#endif +#if CONFIG_TILE_GROUPS + unsigned int num_tg; + unsigned int mtu_size; +#endif +#if CONFIG_TEMPMV_SIGNALING + unsigned int disable_tempmv; +#endif + unsigned int frame_parallel_decoding_mode; + AQ_MODE aq_mode; +#if CONFIG_EXT_DELTA_Q + DELTAQ_MODE deltaq_mode; +#endif + unsigned int frame_periodic_boost; + aom_bit_depth_t bit_depth; + aom_tune_content content; + aom_color_space_t color_space; + int color_range; + int render_width; + int render_height; + aom_superblock_size_t superblock_size; +#if CONFIG_ANS && ANS_MAX_SYMBOLS + int ans_window_size_log2; +#endif +#if CONFIG_EXT_TILE + unsigned int tile_encoding_mode; +#endif // CONFIG_EXT_TILE + + unsigned int motion_vector_unit_test; +}; + +static struct av1_extracfg default_extra_cfg = { + 0, // cpu_used + 1, // enable_auto_alt_ref +#if CONFIG_EXT_REFS + 0, // enable_auto_bwd_ref +#endif // CONFIG_EXT_REFS + 0, // noise_sensitivity + 0, // sharpness + 0, // static_thresh +#if CONFIG_EXT_TILE + UINT_MAX, // tile_columns + UINT_MAX, // tile_rows +#else + 0, // tile_columns + 0, // tile_rows +#endif // CONFIG_EXT_TILE +#if CONFIG_DEPENDENT_HORZTILES + 0, // Depdendent Horizontal tiles +#endif +#if CONFIG_LOOPFILTERING_ACROSS_TILES + 1, // loop_filter_across_tiles_enabled +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + 7, // arnr_max_frames + 5, // arnr_strength + 0, // min_gf_interval; 0 -> default decision + 0, // max_gf_interval; 0 -> default decision + AOM_TUNE_PSNR, // tuning + 10, // cq_level + 0, // rc_max_intra_bitrate_pct + 0, // rc_max_inter_bitrate_pct + 0, // gf_cbr_boost_pct + 0, // lossless +#if CONFIG_AOM_QM + 0, // enable_qm + DEFAULT_QM_FIRST, // qm_min + DEFAULT_QM_LAST, // qm_max +#endif +#if CONFIG_TILE_GROUPS + 1, // max number of tile groups + 0, // mtu_size +#endif +#if CONFIG_TEMPMV_SIGNALING + 0, // disable temporal mv prediction +#endif + 1, // frame_parallel_decoding_mode + NO_AQ, // aq_mode +#if CONFIG_EXT_DELTA_Q + NO_DELTA_Q, // deltaq_mode +#endif + CONFIG_XIPHRC, // frame_periodic_delta_q + AOM_BITS_8, // Bit depth + AOM_CONTENT_DEFAULT, // content + AOM_CS_UNKNOWN, // color space + 0, // color range + 0, // render width + 0, // render height + AOM_SUPERBLOCK_SIZE_DYNAMIC, // superblock_size +#if CONFIG_ANS && ANS_MAX_SYMBOLS + 23, // ans_window_size_log2 +#endif +#if CONFIG_EXT_TILE + 0, // Tile encoding mode is TILE_NORMAL by default. +#endif // CONFIG_EXT_TILE + + 0, // motion_vector_unit_test +}; + +struct aom_codec_alg_priv { + aom_codec_priv_t base; + aom_codec_enc_cfg_t cfg; + struct av1_extracfg extra_cfg; + AV1EncoderConfig oxcf; + AV1_COMP *cpi; + unsigned char *cx_data; + size_t cx_data_sz; + unsigned char *pending_cx_data; + size_t pending_cx_data_sz; + int pending_frame_count; + size_t pending_frame_sizes[8]; + aom_image_t preview_img; + aom_enc_frame_flags_t next_frame_flags; + aom_postproc_cfg_t preview_ppcfg; + aom_codec_pkt_list_decl(256) pkt_list; + unsigned int fixed_kf_cntr; + // BufferPool that holds all reference frames. + BufferPool *buffer_pool; +}; + +static aom_codec_err_t update_error_state( + aom_codec_alg_priv_t *ctx, const struct aom_internal_error_info *error) { + const aom_codec_err_t res = error->error_code; + + if (res != AOM_CODEC_OK) + ctx->base.err_detail = error->has_detail ? error->detail : NULL; + + return res; +} + +#undef ERROR +#define ERROR(str) \ + do { \ + ctx->base.err_detail = str; \ + return AOM_CODEC_INVALID_PARAM; \ + } while (0) + +#define RANGE_CHECK(p, memb, lo, hi) \ + do { \ + if (!((p)->memb >= (lo) && (p)->memb <= (hi))) \ + ERROR(#memb " out of range [" #lo ".." #hi "]"); \ + } while (0) + +#define RANGE_CHECK_HI(p, memb, hi) \ + do { \ + if (!((p)->memb <= (hi))) ERROR(#memb " out of range [.." #hi "]"); \ + } while (0) + +#define RANGE_CHECK_LO(p, memb, lo) \ + do { \ + if (!((p)->memb >= (lo))) ERROR(#memb " out of range [" #lo "..]"); \ + } while (0) + +#define RANGE_CHECK_BOOL(p, memb) \ + do { \ + if (!!((p)->memb) != (p)->memb) ERROR(#memb " expected boolean"); \ + } while (0) + +static aom_codec_err_t validate_config(aom_codec_alg_priv_t *ctx, + const aom_codec_enc_cfg_t *cfg, + const struct av1_extracfg *extra_cfg) { + RANGE_CHECK(cfg, g_w, 1, 65535); // 16 bits available + RANGE_CHECK(cfg, g_h, 1, 65535); // 16 bits available + RANGE_CHECK(cfg, g_timebase.den, 1, 1000000000); + RANGE_CHECK(cfg, g_timebase.num, 1, cfg->g_timebase.den); + RANGE_CHECK_HI(cfg, g_profile, 3); + + RANGE_CHECK_HI(cfg, rc_max_quantizer, 63); + RANGE_CHECK_HI(cfg, rc_min_quantizer, cfg->rc_max_quantizer); + RANGE_CHECK_BOOL(extra_cfg, lossless); + RANGE_CHECK(extra_cfg, aq_mode, 0, AQ_MODE_COUNT - 1); +#if CONFIG_EXT_DELTA_Q + RANGE_CHECK(extra_cfg, deltaq_mode, 0, DELTAQ_MODE_COUNT - 1); +#endif + RANGE_CHECK_HI(extra_cfg, frame_periodic_boost, 1); + RANGE_CHECK_HI(cfg, g_threads, 64); + RANGE_CHECK_HI(cfg, g_lag_in_frames, MAX_LAG_BUFFERS); + RANGE_CHECK(cfg, rc_end_usage, AOM_VBR, AOM_Q); + RANGE_CHECK_HI(cfg, rc_undershoot_pct, 100); + RANGE_CHECK_HI(cfg, rc_overshoot_pct, 100); + RANGE_CHECK_HI(cfg, rc_2pass_vbr_bias_pct, 100); + RANGE_CHECK(cfg, kf_mode, AOM_KF_DISABLED, AOM_KF_AUTO); + RANGE_CHECK_BOOL(cfg, rc_resize_allowed); + RANGE_CHECK_HI(cfg, rc_dropframe_thresh, 100); + RANGE_CHECK_HI(cfg, rc_resize_up_thresh, 100); + RANGE_CHECK_HI(cfg, rc_resize_down_thresh, 100); + RANGE_CHECK(cfg, g_pass, AOM_RC_ONE_PASS, AOM_RC_LAST_PASS); + RANGE_CHECK_HI(extra_cfg, min_gf_interval, MAX_LAG_BUFFERS - 1); + RANGE_CHECK_HI(extra_cfg, max_gf_interval, MAX_LAG_BUFFERS - 1); + if (extra_cfg->max_gf_interval > 0) { + RANGE_CHECK(extra_cfg, max_gf_interval, 2, (MAX_LAG_BUFFERS - 1)); + } + if (extra_cfg->min_gf_interval > 0 && extra_cfg->max_gf_interval > 0) { + RANGE_CHECK(extra_cfg, max_gf_interval, extra_cfg->min_gf_interval, + (MAX_LAG_BUFFERS - 1)); + } + + if (cfg->rc_resize_allowed == 1) { + RANGE_CHECK_HI(cfg, rc_scaled_width, cfg->g_w); + RANGE_CHECK_HI(cfg, rc_scaled_height, cfg->g_h); + } + + // AV1 does not support a lower bound on the keyframe interval in + // automatic keyframe placement mode. + if (cfg->kf_mode != AOM_KF_DISABLED && cfg->kf_min_dist != cfg->kf_max_dist && + cfg->kf_min_dist > 0) + ERROR( + "kf_min_dist not supported in auto mode, use 0 " + "or kf_max_dist instead."); + + RANGE_CHECK_HI(extra_cfg, motion_vector_unit_test, 2); + RANGE_CHECK_HI(extra_cfg, enable_auto_alt_ref, 2); +#if CONFIG_EXT_REFS + RANGE_CHECK_HI(extra_cfg, enable_auto_bwd_ref, 2); +#endif // CONFIG_EXT_REFS + RANGE_CHECK(extra_cfg, cpu_used, 0, 8); + RANGE_CHECK_HI(extra_cfg, noise_sensitivity, 6); + RANGE_CHECK(extra_cfg, superblock_size, AOM_SUPERBLOCK_SIZE_64X64, + AOM_SUPERBLOCK_SIZE_DYNAMIC); +#if CONFIG_EXT_TILE +// TODO(any): Waring. If CONFIG_EXT_TILE is true, tile_columns really +// means tile_width, and tile_rows really means tile_hight. The interface +// should be sanitized. +#if CONFIG_EXT_PARTITION + if (extra_cfg->superblock_size != AOM_SUPERBLOCK_SIZE_64X64) { + if (extra_cfg->tile_columns != UINT_MAX) + RANGE_CHECK(extra_cfg, tile_columns, 1, 32); + if (extra_cfg->tile_rows != UINT_MAX) + RANGE_CHECK(extra_cfg, tile_rows, 1, 32); + } else +#endif // CONFIG_EXT_PARTITION + { + if (extra_cfg->tile_columns != UINT_MAX) + RANGE_CHECK(extra_cfg, tile_columns, 1, 64); + if (extra_cfg->tile_rows != UINT_MAX) + RANGE_CHECK(extra_cfg, tile_rows, 1, 64); + } + RANGE_CHECK_HI(extra_cfg, tile_encoding_mode, 1); +#else + RANGE_CHECK_HI(extra_cfg, tile_columns, 6); + RANGE_CHECK_HI(extra_cfg, tile_rows, 2); +#endif // CONFIG_EXT_TILE +#if CONFIG_DEPENDENT_HORZTILES + RANGE_CHECK_HI(extra_cfg, dependent_horz_tiles, 1); +#endif +#if CONFIG_LOOPFILTERING_ACROSS_TILES + RANGE_CHECK_HI(extra_cfg, loop_filter_across_tiles_enabled, 1); +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + RANGE_CHECK_HI(extra_cfg, sharpness, 7); + RANGE_CHECK_HI(extra_cfg, arnr_max_frames, 15); + RANGE_CHECK_HI(extra_cfg, arnr_strength, 6); + RANGE_CHECK_HI(extra_cfg, cq_level, 63); + RANGE_CHECK(cfg, g_bit_depth, AOM_BITS_8, AOM_BITS_12); + RANGE_CHECK(cfg, g_input_bit_depth, 8, 12); + RANGE_CHECK(extra_cfg, content, AOM_CONTENT_DEFAULT, AOM_CONTENT_INVALID - 1); + + // TODO(yaowu): remove this when ssim tuning is implemented for av1 + if (extra_cfg->tuning == AOM_TUNE_SSIM) + ERROR("Option --tune=ssim is not currently supported in AV1."); + + if (cfg->g_pass == AOM_RC_LAST_PASS) { +#if !CONFIG_XIPHRC + const size_t packet_sz = sizeof(FIRSTPASS_STATS); + const int n_packets = (int)(cfg->rc_twopass_stats_in.sz / packet_sz); + const FIRSTPASS_STATS *stats; +#endif + + if (cfg->rc_twopass_stats_in.buf == NULL) + ERROR("rc_twopass_stats_in.buf not set."); + +#if !CONFIG_XIPHRC + if (cfg->rc_twopass_stats_in.sz % packet_sz) + ERROR("rc_twopass_stats_in.sz indicates truncated packet."); + + if (cfg->rc_twopass_stats_in.sz < 2 * packet_sz) + ERROR("rc_twopass_stats_in requires at least two packets."); + + stats = + (const FIRSTPASS_STATS *)cfg->rc_twopass_stats_in.buf + n_packets - 1; + + if ((int)(stats->count + 0.5) != n_packets - 1) + ERROR("rc_twopass_stats_in missing EOS stats packet"); +#endif + } + +#if !CONFIG_HIGHBITDEPTH + if (cfg->g_profile > (unsigned int)PROFILE_1) { + ERROR("Profile > 1 not supported in this build configuration"); + } +#endif + if (cfg->g_profile <= (unsigned int)PROFILE_1 && + cfg->g_bit_depth > AOM_BITS_8) { + ERROR("Codec high bit-depth not supported in profile < 2"); + } + if (cfg->g_profile <= (unsigned int)PROFILE_1 && cfg->g_input_bit_depth > 8) { + ERROR("Source high bit-depth not supported in profile < 2"); + } + if (cfg->g_profile > (unsigned int)PROFILE_1 && + cfg->g_bit_depth == AOM_BITS_8) { + ERROR("Codec bit-depth 8 not supported in profile > 1"); + } + RANGE_CHECK(extra_cfg, color_space, AOM_CS_UNKNOWN, AOM_CS_SRGB); + RANGE_CHECK(extra_cfg, color_range, 0, 1); +#if CONFIG_ANS && ANS_MAX_SYMBOLS + RANGE_CHECK(extra_cfg, ans_window_size_log2, 8, 23); +#endif + return AOM_CODEC_OK; +} + +static aom_codec_err_t validate_img(aom_codec_alg_priv_t *ctx, + const aom_image_t *img) { + switch (img->fmt) { + case AOM_IMG_FMT_YV12: + case AOM_IMG_FMT_I420: + case AOM_IMG_FMT_I42016: break; + case AOM_IMG_FMT_I422: + case AOM_IMG_FMT_I444: + case AOM_IMG_FMT_I440: + if (ctx->cfg.g_profile != (unsigned int)PROFILE_1) { + ERROR( + "Invalid image format. I422, I444, I440 images are " + "not supported in profile."); + } + break; + case AOM_IMG_FMT_I42216: + case AOM_IMG_FMT_I44416: + case AOM_IMG_FMT_I44016: + if (ctx->cfg.g_profile != (unsigned int)PROFILE_1 && + ctx->cfg.g_profile != (unsigned int)PROFILE_3) { + ERROR( + "Invalid image format. 16-bit I422, I444, I440 images are " + "not supported in profile."); + } + break; + default: + ERROR( + "Invalid image format. Only YV12, I420, I422, I444 images are " + "supported."); + break; + } + + if (img->d_w != ctx->cfg.g_w || img->d_h != ctx->cfg.g_h) + ERROR("Image size must match encoder init configuration size"); + + return AOM_CODEC_OK; +} + +static int get_image_bps(const aom_image_t *img) { + switch (img->fmt) { + case AOM_IMG_FMT_YV12: + case AOM_IMG_FMT_I420: return 12; + case AOM_IMG_FMT_I422: return 16; + case AOM_IMG_FMT_I444: return 24; + case AOM_IMG_FMT_I440: return 16; + case AOM_IMG_FMT_I42016: return 24; + case AOM_IMG_FMT_I42216: return 32; + case AOM_IMG_FMT_I44416: return 48; + case AOM_IMG_FMT_I44016: return 32; + default: assert(0 && "Invalid image format"); break; + } + return 0; +} + +static aom_codec_err_t set_encoder_config( + AV1EncoderConfig *oxcf, const aom_codec_enc_cfg_t *cfg, + const struct av1_extracfg *extra_cfg) { + const int is_vbr = cfg->rc_end_usage == AOM_VBR; + oxcf->profile = cfg->g_profile; + oxcf->max_threads = (int)cfg->g_threads; + oxcf->width = cfg->g_w; + oxcf->height = cfg->g_h; + oxcf->bit_depth = cfg->g_bit_depth; + oxcf->input_bit_depth = cfg->g_input_bit_depth; + // guess a frame rate if out of whack, use 30 + oxcf->init_framerate = (double)cfg->g_timebase.den / cfg->g_timebase.num; + if (oxcf->init_framerate > 180) oxcf->init_framerate = 30; + + oxcf->mode = GOOD; + + switch (cfg->g_pass) { + case AOM_RC_ONE_PASS: oxcf->pass = 0; break; + case AOM_RC_FIRST_PASS: oxcf->pass = 1; break; + case AOM_RC_LAST_PASS: oxcf->pass = 2; break; + } + + oxcf->lag_in_frames = + cfg->g_pass == AOM_RC_FIRST_PASS ? 0 : cfg->g_lag_in_frames; + oxcf->rc_mode = cfg->rc_end_usage; + + // Convert target bandwidth from Kbit/s to Bit/s + oxcf->target_bandwidth = 1000 * cfg->rc_target_bitrate; + oxcf->rc_max_intra_bitrate_pct = extra_cfg->rc_max_intra_bitrate_pct; + oxcf->rc_max_inter_bitrate_pct = extra_cfg->rc_max_inter_bitrate_pct; + oxcf->gf_cbr_boost_pct = extra_cfg->gf_cbr_boost_pct; + + oxcf->best_allowed_q = + extra_cfg->lossless ? 0 : av1_quantizer_to_qindex(cfg->rc_min_quantizer); + oxcf->worst_allowed_q = + extra_cfg->lossless ? 0 : av1_quantizer_to_qindex(cfg->rc_max_quantizer); + oxcf->cq_level = av1_quantizer_to_qindex(extra_cfg->cq_level); + oxcf->fixed_q = -1; + +#if CONFIG_AOM_QM + oxcf->using_qm = extra_cfg->enable_qm; + oxcf->qm_minlevel = extra_cfg->qm_min; + oxcf->qm_maxlevel = extra_cfg->qm_max; +#endif + +#if CONFIG_TILE_GROUPS + oxcf->num_tile_groups = extra_cfg->num_tg; + oxcf->mtu = extra_cfg->mtu_size; +#endif + +#if CONFIG_TEMPMV_SIGNALING + oxcf->disable_tempmv = extra_cfg->disable_tempmv; +#endif + oxcf->under_shoot_pct = cfg->rc_undershoot_pct; + oxcf->over_shoot_pct = cfg->rc_overshoot_pct; + + oxcf->scaled_frame_width = cfg->rc_scaled_width; + oxcf->scaled_frame_height = cfg->rc_scaled_height; + if (cfg->rc_resize_allowed == 1) { + oxcf->resize_mode = + (oxcf->scaled_frame_width == 0 || oxcf->scaled_frame_height == 0) + ? RESIZE_DYNAMIC + : RESIZE_FIXED; + } else { + oxcf->resize_mode = RESIZE_NONE; + } + + oxcf->maximum_buffer_size_ms = is_vbr ? 240000 : cfg->rc_buf_sz; + oxcf->starting_buffer_level_ms = is_vbr ? 60000 : cfg->rc_buf_initial_sz; + oxcf->optimal_buffer_level_ms = is_vbr ? 60000 : cfg->rc_buf_optimal_sz; + + oxcf->drop_frames_water_mark = cfg->rc_dropframe_thresh; + + oxcf->two_pass_vbrbias = cfg->rc_2pass_vbr_bias_pct; + oxcf->two_pass_vbrmin_section = cfg->rc_2pass_vbr_minsection_pct; + oxcf->two_pass_vbrmax_section = cfg->rc_2pass_vbr_maxsection_pct; + + oxcf->auto_key = + cfg->kf_mode == AOM_KF_AUTO && cfg->kf_min_dist != cfg->kf_max_dist; + + oxcf->key_freq = cfg->kf_max_dist; + + oxcf->speed = extra_cfg->cpu_used; + oxcf->enable_auto_arf = extra_cfg->enable_auto_alt_ref; +#if CONFIG_EXT_REFS + oxcf->enable_auto_brf = extra_cfg->enable_auto_bwd_ref; +#endif // CONFIG_EXT_REFS + oxcf->noise_sensitivity = extra_cfg->noise_sensitivity; + oxcf->sharpness = extra_cfg->sharpness; + + oxcf->two_pass_stats_in = cfg->rc_twopass_stats_in; + +#if CONFIG_FP_MB_STATS + oxcf->firstpass_mb_stats_in = cfg->rc_firstpass_mb_stats_in; +#endif + + oxcf->color_space = extra_cfg->color_space; + oxcf->color_range = extra_cfg->color_range; + oxcf->render_width = extra_cfg->render_width; + oxcf->render_height = extra_cfg->render_height; + oxcf->arnr_max_frames = extra_cfg->arnr_max_frames; + oxcf->arnr_strength = extra_cfg->arnr_strength; + oxcf->min_gf_interval = extra_cfg->min_gf_interval; + oxcf->max_gf_interval = extra_cfg->max_gf_interval; + + oxcf->tuning = extra_cfg->tuning; + oxcf->content = extra_cfg->content; + +#if CONFIG_EXT_PARTITION + oxcf->superblock_size = extra_cfg->superblock_size; +#endif // CONFIG_EXT_PARTITION +#if CONFIG_ANS && ANS_MAX_SYMBOLS + oxcf->ans_window_size_log2 = extra_cfg->ans_window_size_log2; +#endif // CONFIG_ANS && ANS_MAX_SYMBOLS + +#if CONFIG_EXT_TILE + { +#if CONFIG_EXT_PARTITION + const unsigned int max = + extra_cfg->superblock_size == AOM_SUPERBLOCK_SIZE_64X64 ? 64 : 32; +#else + const unsigned int max = 64; +#endif // CONFIG_EXT_PARTITION + oxcf->tile_columns = AOMMIN(extra_cfg->tile_columns, max); + oxcf->tile_rows = AOMMIN(extra_cfg->tile_rows, max); + oxcf->tile_encoding_mode = extra_cfg->tile_encoding_mode; + } +#else + oxcf->tile_columns = extra_cfg->tile_columns; + oxcf->tile_rows = extra_cfg->tile_rows; +#endif // CONFIG_EXT_TILE +#if CONFIG_DEPENDENT_HORZTILES + oxcf->dependent_horz_tiles = extra_cfg->dependent_horz_tiles; +#endif +#if CONFIG_LOOPFILTERING_ACROSS_TILES + oxcf->loop_filter_across_tiles_enabled = + extra_cfg->loop_filter_across_tiles_enabled; +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + oxcf->error_resilient_mode = cfg->g_error_resilient; + oxcf->frame_parallel_decoding_mode = extra_cfg->frame_parallel_decoding_mode; + + oxcf->aq_mode = extra_cfg->aq_mode; +#if CONFIG_EXT_DELTA_Q + oxcf->deltaq_mode = extra_cfg->deltaq_mode; +#endif + + oxcf->frame_periodic_boost = extra_cfg->frame_periodic_boost; + + oxcf->motion_vector_unit_test = extra_cfg->motion_vector_unit_test; + /* + printf("Current AV1 Settings: \n"); + printf("target_bandwidth: %d\n", oxcf->target_bandwidth); + printf("noise_sensitivity: %d\n", oxcf->noise_sensitivity); + printf("sharpness: %d\n", oxcf->sharpness); + printf("cpu_used: %d\n", oxcf->cpu_used); + printf("Mode: %d\n", oxcf->mode); + printf("auto_key: %d\n", oxcf->auto_key); + printf("key_freq: %d\n", oxcf->key_freq); + printf("end_usage: %d\n", oxcf->end_usage); + printf("under_shoot_pct: %d\n", oxcf->under_shoot_pct); + printf("over_shoot_pct: %d\n", oxcf->over_shoot_pct); + printf("starting_buffer_level: %d\n", oxcf->starting_buffer_level); + printf("optimal_buffer_level: %d\n", oxcf->optimal_buffer_level); + printf("maximum_buffer_size: %d\n", oxcf->maximum_buffer_size); + printf("fixed_q: %d\n", oxcf->fixed_q); + printf("worst_allowed_q: %d\n", oxcf->worst_allowed_q); + printf("best_allowed_q: %d\n", oxcf->best_allowed_q); + printf("allow_spatial_resampling: %d\n", oxcf->allow_spatial_resampling); + printf("scaled_frame_width: %d\n", oxcf->scaled_frame_width); + printf("scaled_frame_height: %d\n", oxcf->scaled_frame_height); + printf("two_pass_vbrbias: %d\n", oxcf->two_pass_vbrbias); + printf("two_pass_vbrmin_section: %d\n", oxcf->two_pass_vbrmin_section); + printf("two_pass_vbrmax_section: %d\n", oxcf->two_pass_vbrmax_section); + printf("lag_in_frames: %d\n", oxcf->lag_in_frames); + printf("enable_auto_arf: %d\n", oxcf->enable_auto_arf); + printf("Version: %d\n", oxcf->Version); + printf("error resilient: %d\n", oxcf->error_resilient_mode); + printf("frame parallel detokenization: %d\n", + oxcf->frame_parallel_decoding_mode); + */ + return AOM_CODEC_OK; +} + +static aom_codec_err_t encoder_set_config(aom_codec_alg_priv_t *ctx, + const aom_codec_enc_cfg_t *cfg) { + aom_codec_err_t res; + int force_key = 0; + + if (cfg->g_w != ctx->cfg.g_w || cfg->g_h != ctx->cfg.g_h) { + if (cfg->g_lag_in_frames > 1 || cfg->g_pass != AOM_RC_ONE_PASS) + ERROR("Cannot change width or height after initialization"); + if (!valid_ref_frame_size(ctx->cfg.g_w, ctx->cfg.g_h, cfg->g_w, cfg->g_h) || + (ctx->cpi->initial_width && (int)cfg->g_w > ctx->cpi->initial_width) || + (ctx->cpi->initial_height && (int)cfg->g_h > ctx->cpi->initial_height)) + force_key = 1; + } + + // Prevent increasing lag_in_frames. This check is stricter than it needs + // to be -- the limit is not increasing past the first lag_in_frames + // value, but we don't track the initial config, only the last successful + // config. + if (cfg->g_lag_in_frames > ctx->cfg.g_lag_in_frames) + ERROR("Cannot increase lag_in_frames"); + + res = validate_config(ctx, cfg, &ctx->extra_cfg); + + if (res == AOM_CODEC_OK) { + ctx->cfg = *cfg; + set_encoder_config(&ctx->oxcf, &ctx->cfg, &ctx->extra_cfg); + // On profile change, request a key frame + force_key |= ctx->cpi->common.profile != ctx->oxcf.profile; + av1_change_config(ctx->cpi, &ctx->oxcf); + } + + if (force_key) ctx->next_frame_flags |= AOM_EFLAG_FORCE_KF; + + return res; +} + +static aom_codec_err_t ctrl_get_quantizer(aom_codec_alg_priv_t *ctx, + va_list args) { + int *const arg = va_arg(args, int *); + if (arg == NULL) return AOM_CODEC_INVALID_PARAM; + *arg = av1_get_quantizer(ctx->cpi); + return AOM_CODEC_OK; +} + +static aom_codec_err_t ctrl_get_quantizer64(aom_codec_alg_priv_t *ctx, + va_list args) { + int *const arg = va_arg(args, int *); + if (arg == NULL) return AOM_CODEC_INVALID_PARAM; + *arg = av1_qindex_to_quantizer(av1_get_quantizer(ctx->cpi)); + return AOM_CODEC_OK; +} + +static aom_codec_err_t update_extra_cfg(aom_codec_alg_priv_t *ctx, + const struct av1_extracfg *extra_cfg) { + const aom_codec_err_t res = validate_config(ctx, &ctx->cfg, extra_cfg); + if (res == AOM_CODEC_OK) { + ctx->extra_cfg = *extra_cfg; + set_encoder_config(&ctx->oxcf, &ctx->cfg, &ctx->extra_cfg); + av1_change_config(ctx->cpi, &ctx->oxcf); + } + return res; +} + +static aom_codec_err_t ctrl_set_cpuused(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.cpu_used = CAST(AOME_SET_CPUUSED, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_auto_alt_ref(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_auto_alt_ref = CAST(AOME_SET_ENABLEAUTOALTREF, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +#if CONFIG_EXT_REFS +static aom_codec_err_t ctrl_set_enable_auto_bwd_ref(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_auto_bwd_ref = CAST(AOME_SET_ENABLEAUTOBWDREF, args); + return update_extra_cfg(ctx, &extra_cfg); +} +#endif // CONFIG_EXT_REFS + +static aom_codec_err_t ctrl_set_noise_sensitivity(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.noise_sensitivity = CAST(AV1E_SET_NOISE_SENSITIVITY, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_sharpness(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.sharpness = CAST(AOME_SET_SHARPNESS, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_static_thresh(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.static_thresh = CAST(AOME_SET_STATIC_THRESHOLD, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_tile_columns(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.tile_columns = CAST(AV1E_SET_TILE_COLUMNS, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_tile_rows(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.tile_rows = CAST(AV1E_SET_TILE_ROWS, args); + return update_extra_cfg(ctx, &extra_cfg); +} +#if CONFIG_DEPENDENT_HORZTILES +static aom_codec_err_t ctrl_set_tile_dependent_rows(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.dependent_horz_tiles = CAST(AV1E_SET_TILE_DEPENDENT_ROWS, args); + return update_extra_cfg(ctx, &extra_cfg); +} +#endif +#if CONFIG_LOOPFILTERING_ACROSS_TILES +static aom_codec_err_t ctrl_set_tile_loopfilter(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.loop_filter_across_tiles_enabled = + CAST(AV1E_SET_TILE_LOOPFILTER, args); + return update_extra_cfg(ctx, &extra_cfg); +} +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + +static aom_codec_err_t ctrl_set_arnr_max_frames(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.arnr_max_frames = CAST(AOME_SET_ARNR_MAXFRAMES, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_arnr_strength(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.arnr_strength = CAST(AOME_SET_ARNR_STRENGTH, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_tuning(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.tuning = CAST(AOME_SET_TUNING, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_cq_level(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.cq_level = CAST(AOME_SET_CQ_LEVEL, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_rc_max_intra_bitrate_pct( + aom_codec_alg_priv_t *ctx, va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.rc_max_intra_bitrate_pct = + CAST(AOME_SET_MAX_INTRA_BITRATE_PCT, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_rc_max_inter_bitrate_pct( + aom_codec_alg_priv_t *ctx, va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.rc_max_inter_bitrate_pct = + CAST(AOME_SET_MAX_INTER_BITRATE_PCT, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_rc_gf_cbr_boost_pct(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.gf_cbr_boost_pct = CAST(AV1E_SET_GF_CBR_BOOST_PCT, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_lossless(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.lossless = CAST(AV1E_SET_LOSSLESS, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +#if CONFIG_AOM_QM +static aom_codec_err_t ctrl_set_enable_qm(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_qm = CAST(AV1E_SET_ENABLE_QM, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_qm_min(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.qm_min = CAST(AV1E_SET_QM_MIN, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_qm_max(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.qm_max = CAST(AV1E_SET_QM_MAX, args); + return update_extra_cfg(ctx, &extra_cfg); +} +#endif + +#if CONFIG_TILE_GROUPS +static aom_codec_err_t ctrl_set_num_tg(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.num_tg = CAST(AV1E_SET_NUM_TG, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_mtu(aom_codec_alg_priv_t *ctx, va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.mtu_size = CAST(AV1E_SET_MTU, args); + return update_extra_cfg(ctx, &extra_cfg); +} +#endif +#if CONFIG_TEMPMV_SIGNALING +static aom_codec_err_t ctrl_set_disable_tempmv(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.disable_tempmv = CAST(AV1E_SET_DISABLE_TEMPMV, args); + return update_extra_cfg(ctx, &extra_cfg); +} +#endif +static aom_codec_err_t ctrl_set_frame_parallel_decoding_mode( + aom_codec_alg_priv_t *ctx, va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.frame_parallel_decoding_mode = + CAST(AV1E_SET_FRAME_PARALLEL_DECODING, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +#if CONFIG_EXT_TILE +static aom_codec_err_t ctrl_set_tile_encoding_mode(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.tile_encoding_mode = CAST(AV1E_SET_TILE_ENCODING_MODE, args); + return update_extra_cfg(ctx, &extra_cfg); +} +#endif // CONFIG_EXT_TILE + +static aom_codec_err_t ctrl_set_aq_mode(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.aq_mode = CAST(AV1E_SET_AQ_MODE, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +#if CONFIG_EXT_DELTA_Q +static aom_codec_err_t ctrl_set_deltaq_mode(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.deltaq_mode = CAST(AV1E_SET_DELTAQ_MODE, args); + return update_extra_cfg(ctx, &extra_cfg); +} +#endif +static aom_codec_err_t ctrl_set_min_gf_interval(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.min_gf_interval = CAST(AV1E_SET_MIN_GF_INTERVAL, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_max_gf_interval(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.max_gf_interval = CAST(AV1E_SET_MAX_GF_INTERVAL, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_frame_periodic_boost(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.frame_periodic_boost = CAST(AV1E_SET_FRAME_PERIODIC_BOOST, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_enable_motion_vector_unit_test( + aom_codec_alg_priv_t *ctx, va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.motion_vector_unit_test = + CAST(AV1E_ENABLE_MOTION_VECTOR_UNIT_TEST, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t encoder_init(aom_codec_ctx_t *ctx, + aom_codec_priv_enc_mr_cfg_t *data) { + aom_codec_err_t res = AOM_CODEC_OK; + (void)data; + + if (ctx->priv == NULL) { + aom_codec_alg_priv_t *const priv = aom_calloc(1, sizeof(*priv)); + if (priv == NULL) return AOM_CODEC_MEM_ERROR; + + ctx->priv = (aom_codec_priv_t *)priv; + ctx->priv->init_flags = ctx->init_flags; + ctx->priv->enc.total_encoders = 1; + priv->buffer_pool = (BufferPool *)aom_calloc(1, sizeof(BufferPool)); + if (priv->buffer_pool == NULL) return AOM_CODEC_MEM_ERROR; + +#if CONFIG_MULTITHREAD + if (pthread_mutex_init(&priv->buffer_pool->pool_mutex, NULL)) { + return AOM_CODEC_MEM_ERROR; + } +#endif + + if (ctx->config.enc) { + // Update the reference to the config structure to an internal copy. + priv->cfg = *ctx->config.enc; + ctx->config.enc = &priv->cfg; + } + + priv->extra_cfg = default_extra_cfg; + once(av1_initialize_enc); + + res = validate_config(priv, &priv->cfg, &priv->extra_cfg); + + if (res == AOM_CODEC_OK) { + set_encoder_config(&priv->oxcf, &priv->cfg, &priv->extra_cfg); +#if CONFIG_HIGHBITDEPTH + priv->oxcf.use_highbitdepth = + (ctx->init_flags & AOM_CODEC_USE_HIGHBITDEPTH) ? 1 : 0; +#endif + priv->cpi = av1_create_compressor(&priv->oxcf, priv->buffer_pool); + if (priv->cpi == NULL) + res = AOM_CODEC_MEM_ERROR; + else + priv->cpi->output_pkt_list = &priv->pkt_list.head; + } + } + + return res; +} + +static aom_codec_err_t encoder_destroy(aom_codec_alg_priv_t *ctx) { + free(ctx->cx_data); + av1_remove_compressor(ctx->cpi); +#if CONFIG_MULTITHREAD + pthread_mutex_destroy(&ctx->buffer_pool->pool_mutex); +#endif + aom_free(ctx->buffer_pool); + aom_free(ctx); + return AOM_CODEC_OK; +} + +static void pick_quickcompress_mode(aom_codec_alg_priv_t *ctx, + unsigned long deadline) { + MODE new_mode = GOOD; + + switch (ctx->cfg.g_pass) { + case AOM_RC_ONE_PASS: + switch (deadline) { + default: new_mode = GOOD; break; + } + break; + case AOM_RC_FIRST_PASS: break; + case AOM_RC_LAST_PASS: new_mode = GOOD; + } + + if (ctx->oxcf.mode != new_mode) { + ctx->oxcf.mode = new_mode; + av1_change_config(ctx->cpi, &ctx->oxcf); + } +} + +// Turn on to test if supplemental superframe data breaks decoding +// #define TEST_SUPPLEMENTAL_SUPERFRAME_DATA +static int write_superframe_index(aom_codec_alg_priv_t *ctx) { + uint8_t marker = 0xc0; + unsigned int mask; + int mag, index_sz; + int i; + size_t max_frame_sz = 0; + + assert(ctx->pending_frame_count); + assert(ctx->pending_frame_count <= 8); + + // Add the number of frames to the marker byte + marker |= ctx->pending_frame_count - 1; + for (i = 0; i < ctx->pending_frame_count - 1; i++) { + const size_t frame_sz = (unsigned int)ctx->pending_frame_sizes[i] - 1; + max_frame_sz = frame_sz > max_frame_sz ? frame_sz : max_frame_sz; + } + + // Choose the magnitude + for (mag = 0, mask = 0xff; mag < 4; mag++) { + if (max_frame_sz <= mask) break; + mask <<= 8; + mask |= 0xff; + } + marker |= mag << 3; + + // Write the index + index_sz = 2 + (mag + 1) * (ctx->pending_frame_count - 1); + if (ctx->pending_cx_data_sz + index_sz < ctx->cx_data_sz) { + uint8_t *x = ctx->pending_cx_data + ctx->pending_cx_data_sz; +#ifdef TEST_SUPPLEMENTAL_SUPERFRAME_DATA + uint8_t marker_test = 0xc0; + int mag_test = 2; // 1 - 4 + int frames_test = 4; // 1 - 8 + int index_sz_test = 2 + mag_test * frames_test; + marker_test |= frames_test - 1; + marker_test |= (mag_test - 1) << 3; + *x++ = marker_test; + for (i = 0; i < mag_test * frames_test; ++i) + *x++ = 0; // fill up with arbitrary data + *x++ = marker_test; + ctx->pending_cx_data_sz += index_sz_test; + printf("Added supplemental superframe data\n"); +#endif + + *x++ = marker; + for (i = 0; i < ctx->pending_frame_count - 1; i++) { + unsigned int this_sz; + int j; + + assert(ctx->pending_frame_sizes[i] > 0); + this_sz = (unsigned int)ctx->pending_frame_sizes[i] - 1; + for (j = 0; j <= mag; j++) { + *x++ = this_sz & 0xff; + this_sz >>= 8; + } + } + *x++ = marker; + ctx->pending_cx_data_sz += index_sz; +#ifdef TEST_SUPPLEMENTAL_SUPERFRAME_DATA + index_sz += index_sz_test; +#endif + } + return index_sz; +} + +// av1 uses 10,000,000 ticks/second as time stamp +#define TICKS_PER_SEC 10000000LL + +static int64_t timebase_units_to_ticks(const aom_rational_t *timebase, + int64_t n) { + return n * TICKS_PER_SEC * timebase->num / timebase->den; +} + +static int64_t ticks_to_timebase_units(const aom_rational_t *timebase, + int64_t n) { + const int64_t round = TICKS_PER_SEC * timebase->num / 2 - 1; + return (n * timebase->den + round) / timebase->num / TICKS_PER_SEC; +} + +static aom_codec_frame_flags_t get_frame_pkt_flags(const AV1_COMP *cpi, + unsigned int lib_flags) { + aom_codec_frame_flags_t flags = lib_flags << 16; + + if (lib_flags & FRAMEFLAGS_KEY) flags |= AOM_FRAME_IS_KEY; + + if (cpi->droppable) flags |= AOM_FRAME_IS_DROPPABLE; + + return flags; +} + +static aom_codec_err_t encoder_encode(aom_codec_alg_priv_t *ctx, + const aom_image_t *img, + aom_codec_pts_t pts, + unsigned long duration, + aom_enc_frame_flags_t enc_flags, + unsigned long deadline) { + const size_t kMinCompressedSize = 8192; + volatile aom_codec_err_t res = AOM_CODEC_OK; + volatile aom_enc_frame_flags_t flags = enc_flags; + AV1_COMP *const cpi = ctx->cpi; + const aom_rational_t *const timebase = &ctx->cfg.g_timebase; + size_t data_sz; + + if (cpi == NULL) return AOM_CODEC_INVALID_PARAM; + + if (img != NULL) { + res = validate_img(ctx, img); + // TODO(jzern) the checks related to cpi's validity should be treated as a + // failure condition, encoder setup is done fully in init() currently. + if (res == AOM_CODEC_OK) { +#if CONFIG_EXT_REFS + data_sz = ALIGN_POWER_OF_TWO(ctx->cfg.g_w, 5) * + ALIGN_POWER_OF_TWO(ctx->cfg.g_h, 5) * get_image_bps(img); +#else + // There's no codec control for multiple alt-refs so check the encoder + // instance for its status to determine the compressed data size. + data_sz = ALIGN_POWER_OF_TWO(ctx->cfg.g_w, 5) * + ALIGN_POWER_OF_TWO(ctx->cfg.g_h, 5) * get_image_bps(img) / 8 * + (cpi->multi_arf_allowed ? 8 : 2); +#endif // CONFIG_EXT_REFS + if (data_sz < kMinCompressedSize) data_sz = kMinCompressedSize; + if (ctx->cx_data == NULL || ctx->cx_data_sz < data_sz) { + ctx->cx_data_sz = data_sz; + free(ctx->cx_data); + ctx->cx_data = (unsigned char *)malloc(ctx->cx_data_sz); + if (ctx->cx_data == NULL) { + return AOM_CODEC_MEM_ERROR; + } + } + } + } + + pick_quickcompress_mode(ctx, deadline); + aom_codec_pkt_list_init(&ctx->pkt_list); + + // Handle Flags + if (((flags & AOM_EFLAG_NO_UPD_GF) && (flags & AOM_EFLAG_FORCE_GF)) || + ((flags & AOM_EFLAG_NO_UPD_ARF) && (flags & AOM_EFLAG_FORCE_ARF))) { + ctx->base.err_detail = "Conflicting flags."; + return AOM_CODEC_INVALID_PARAM; + } + + if (setjmp(cpi->common.error.jmp)) { + cpi->common.error.setjmp = 0; + res = update_error_state(ctx, &cpi->common.error); + aom_clear_system_state(); + return res; + } + cpi->common.error.setjmp = 1; + + av1_apply_encoding_flags(cpi, flags); + + // Handle fixed keyframe intervals + if (ctx->cfg.kf_mode == AOM_KF_AUTO && + ctx->cfg.kf_min_dist == ctx->cfg.kf_max_dist) { + if (++ctx->fixed_kf_cntr > ctx->cfg.kf_min_dist) { + flags |= AOM_EFLAG_FORCE_KF; + ctx->fixed_kf_cntr = 1; + } + } + + if (res == AOM_CODEC_OK) { + unsigned int lib_flags = 0; + YV12_BUFFER_CONFIG sd; + int64_t dst_time_stamp = timebase_units_to_ticks(timebase, pts); + int64_t dst_end_time_stamp = + timebase_units_to_ticks(timebase, pts + duration); + size_t size, cx_data_sz; + unsigned char *cx_data; + + // Set up internal flags + if (ctx->base.init_flags & AOM_CODEC_USE_PSNR) cpi->b_calculate_psnr = 1; + + if (img != NULL) { + res = image2yuvconfig(img, &sd); + + // Store the original flags in to the frame buffer. Will extract the + // key frame flag when we actually encode this frame. + if (av1_receive_raw_frame(cpi, flags | ctx->next_frame_flags, &sd, + dst_time_stamp, dst_end_time_stamp)) { + res = update_error_state(ctx, &cpi->common.error); + } + ctx->next_frame_flags = 0; + } + + cx_data = ctx->cx_data; + cx_data_sz = ctx->cx_data_sz; + + /* Any pending invisible frames? */ + if (ctx->pending_cx_data) { + memmove(cx_data, ctx->pending_cx_data, ctx->pending_cx_data_sz); + ctx->pending_cx_data = cx_data; + cx_data += ctx->pending_cx_data_sz; + cx_data_sz -= ctx->pending_cx_data_sz; + + /* TODO: this is a minimal check, the underlying codec doesn't respect + * the buffer size anyway. + */ + if (cx_data_sz < ctx->cx_data_sz / 2) { + aom_internal_error(&cpi->common.error, AOM_CODEC_ERROR, + "Compressed data buffer too small"); + return AOM_CODEC_ERROR; + } + } + + while (cx_data_sz >= ctx->cx_data_sz / 2 && + -1 != av1_get_compressed_data(cpi, &lib_flags, &size, cx_data, + &dst_time_stamp, &dst_end_time_stamp, + !img)) { +#if CONFIG_REFERENCE_BUFFER + if (cpi->common.invalid_delta_frame_id_minus1) { + ctx->base.err_detail = "Invalid delta_frame_id_minus1"; + return AOM_CODEC_ERROR; + } +#endif + if (size) { + aom_codec_cx_pkt_t pkt; + + // Pack invisible frames with the next visible frame + if (!cpi->common.show_frame) { + if (ctx->pending_cx_data == 0) ctx->pending_cx_data = cx_data; + ctx->pending_cx_data_sz += size; + ctx->pending_frame_sizes[ctx->pending_frame_count++] = size; + cx_data += size; + cx_data_sz -= size; + + continue; + } + + // Add the frame packet to the list of returned packets. + pkt.kind = AOM_CODEC_CX_FRAME_PKT; + pkt.data.frame.pts = ticks_to_timebase_units(timebase, dst_time_stamp); + pkt.data.frame.duration = (unsigned long)ticks_to_timebase_units( + timebase, dst_end_time_stamp - dst_time_stamp); + pkt.data.frame.flags = get_frame_pkt_flags(cpi, lib_flags); + + if (ctx->pending_cx_data) { + ctx->pending_frame_sizes[ctx->pending_frame_count++] = size; + ctx->pending_cx_data_sz += size; + size += write_superframe_index(ctx); + pkt.data.frame.buf = ctx->pending_cx_data; + pkt.data.frame.sz = ctx->pending_cx_data_sz; + ctx->pending_cx_data = NULL; + ctx->pending_cx_data_sz = 0; + ctx->pending_frame_count = 0; + } else { + pkt.data.frame.buf = cx_data; + pkt.data.frame.sz = size; + } + pkt.data.frame.partition_id = -1; + + aom_codec_pkt_list_add(&ctx->pkt_list.head, &pkt); + + cx_data += size; + cx_data_sz -= size; + } + } + } + + cpi->common.error.setjmp = 0; + return res; +} + +static const aom_codec_cx_pkt_t *encoder_get_cxdata(aom_codec_alg_priv_t *ctx, + aom_codec_iter_t *iter) { + return aom_codec_pkt_list_get(&ctx->pkt_list.head, iter); +} + +static aom_codec_err_t ctrl_set_reference(aom_codec_alg_priv_t *ctx, + va_list args) { + aom_ref_frame_t *const frame = va_arg(args, aom_ref_frame_t *); + + if (frame != NULL) { + YV12_BUFFER_CONFIG sd; + + image2yuvconfig(&frame->img, &sd); + av1_set_reference_enc(ctx->cpi, ref_frame_to_av1_reframe(frame->frame_type), + &sd); + return AOM_CODEC_OK; + } else { + return AOM_CODEC_INVALID_PARAM; + } +} + +static aom_codec_err_t ctrl_copy_reference(aom_codec_alg_priv_t *ctx, + va_list args) { + aom_ref_frame_t *const frame = va_arg(args, aom_ref_frame_t *); + + if (frame != NULL) { + YV12_BUFFER_CONFIG sd; + + image2yuvconfig(&frame->img, &sd); + av1_copy_reference_enc(ctx->cpi, + ref_frame_to_av1_reframe(frame->frame_type), &sd); + return AOM_CODEC_OK; + } else { + return AOM_CODEC_INVALID_PARAM; + } +} + +static aom_codec_err_t ctrl_get_reference(aom_codec_alg_priv_t *ctx, + va_list args) { + av1_ref_frame_t *const frame = va_arg(args, av1_ref_frame_t *); + + if (frame != NULL) { + YV12_BUFFER_CONFIG *fb = get_ref_frame(&ctx->cpi->common, frame->idx); + if (fb == NULL) return AOM_CODEC_ERROR; + + yuvconfig2image(&frame->img, fb, NULL); + return AOM_CODEC_OK; + } else { + return AOM_CODEC_INVALID_PARAM; + } +} + +static aom_codec_err_t ctrl_get_new_frame_image(aom_codec_alg_priv_t *ctx, + va_list args) { + aom_image_t *const new_img = va_arg(args, aom_image_t *); + + if (new_img != NULL) { + YV12_BUFFER_CONFIG new_frame; + + if (av1_get_last_show_frame(ctx->cpi, &new_frame) == 0) { + yuvconfig2image(new_img, &new_frame, NULL); + return AOM_CODEC_OK; + } else { + return AOM_CODEC_ERROR; + } + } else { + return AOM_CODEC_INVALID_PARAM; + } +} + +static aom_codec_err_t ctrl_set_previewpp(aom_codec_alg_priv_t *ctx, + va_list args) { + (void)ctx; + (void)args; + return AOM_CODEC_INCAPABLE; +} + +static aom_image_t *encoder_get_preview(aom_codec_alg_priv_t *ctx) { + YV12_BUFFER_CONFIG sd; + + if (av1_get_preview_raw_frame(ctx->cpi, &sd) == 0) { + yuvconfig2image(&ctx->preview_img, &sd, NULL); + return &ctx->preview_img; + } else { + return NULL; + } +} + +static aom_codec_err_t ctrl_use_reference(aom_codec_alg_priv_t *ctx, + va_list args) { + const int reference_flag = va_arg(args, int); + + av1_use_as_reference(ctx->cpi, reference_flag); + return AOM_CODEC_OK; +} + +static aom_codec_err_t ctrl_set_roi_map(aom_codec_alg_priv_t *ctx, + va_list args) { + (void)ctx; + (void)args; + + // TODO(yaowu): Need to re-implement and test for AV1. + return AOM_CODEC_INVALID_PARAM; +} + +static aom_codec_err_t ctrl_set_active_map(aom_codec_alg_priv_t *ctx, + va_list args) { + aom_active_map_t *const map = va_arg(args, aom_active_map_t *); + + if (map) { + if (!av1_set_active_map(ctx->cpi, map->active_map, (int)map->rows, + (int)map->cols)) + return AOM_CODEC_OK; + else + return AOM_CODEC_INVALID_PARAM; + } else { + return AOM_CODEC_INVALID_PARAM; + } +} + +static aom_codec_err_t ctrl_get_active_map(aom_codec_alg_priv_t *ctx, + va_list args) { + aom_active_map_t *const map = va_arg(args, aom_active_map_t *); + + if (map) { + if (!av1_get_active_map(ctx->cpi, map->active_map, (int)map->rows, + (int)map->cols)) + return AOM_CODEC_OK; + else + return AOM_CODEC_INVALID_PARAM; + } else { + return AOM_CODEC_INVALID_PARAM; + } +} + +static aom_codec_err_t ctrl_set_scale_mode(aom_codec_alg_priv_t *ctx, + va_list args) { + aom_scaling_mode_t *const mode = va_arg(args, aom_scaling_mode_t *); + + if (mode) { + const int res = + av1_set_internal_size(ctx->cpi, (AOM_SCALING)mode->h_scaling_mode, + (AOM_SCALING)mode->v_scaling_mode); + return (res == 0) ? AOM_CODEC_OK : AOM_CODEC_INVALID_PARAM; + } else { + return AOM_CODEC_INVALID_PARAM; + } +} + +static aom_codec_err_t ctrl_set_tune_content(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.content = CAST(AV1E_SET_TUNE_CONTENT, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_color_space(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.color_space = CAST(AV1E_SET_COLOR_SPACE, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_color_range(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.color_range = CAST(AV1E_SET_COLOR_RANGE, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_render_size(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + int *const render_size = va_arg(args, int *); + extra_cfg.render_width = render_size[0]; + extra_cfg.render_height = render_size[1]; + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_superblock_size(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.superblock_size = CAST(AV1E_SET_SUPERBLOCK_SIZE, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +#if CONFIG_ANS && ANS_MAX_SYMBOLS +static aom_codec_err_t ctrl_set_ans_window_size_log2(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.ans_window_size_log2 = CAST(AV1E_SET_ANS_WINDOW_SIZE_LOG2, args); + return update_extra_cfg(ctx, &extra_cfg); +} +#endif + +static aom_codec_ctrl_fn_map_t encoder_ctrl_maps[] = { + { AOM_COPY_REFERENCE, ctrl_copy_reference }, + { AOME_USE_REFERENCE, ctrl_use_reference }, + + // Setters + { AOM_SET_REFERENCE, ctrl_set_reference }, + { AOM_SET_POSTPROC, ctrl_set_previewpp }, + { AOME_SET_ROI_MAP, ctrl_set_roi_map }, + { AOME_SET_ACTIVEMAP, ctrl_set_active_map }, + { AOME_SET_SCALEMODE, ctrl_set_scale_mode }, + { AOME_SET_CPUUSED, ctrl_set_cpuused }, + { AOME_SET_ENABLEAUTOALTREF, ctrl_set_enable_auto_alt_ref }, +#if CONFIG_EXT_REFS + { AOME_SET_ENABLEAUTOBWDREF, ctrl_set_enable_auto_bwd_ref }, +#endif // CONFIG_EXT_REFS + { AOME_SET_SHARPNESS, ctrl_set_sharpness }, + { AOME_SET_STATIC_THRESHOLD, ctrl_set_static_thresh }, + { AV1E_SET_TILE_COLUMNS, ctrl_set_tile_columns }, + { AV1E_SET_TILE_ROWS, ctrl_set_tile_rows }, +#if CONFIG_DEPENDENT_HORZTILES + { AV1E_SET_TILE_DEPENDENT_ROWS, ctrl_set_tile_dependent_rows }, +#endif +#if CONFIG_LOOPFILTERING_ACROSS_TILES + { AV1E_SET_TILE_LOOPFILTER, ctrl_set_tile_loopfilter }, +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + { AOME_SET_ARNR_MAXFRAMES, ctrl_set_arnr_max_frames }, + { AOME_SET_ARNR_STRENGTH, ctrl_set_arnr_strength }, + { AOME_SET_TUNING, ctrl_set_tuning }, + { AOME_SET_CQ_LEVEL, ctrl_set_cq_level }, + { AOME_SET_MAX_INTRA_BITRATE_PCT, ctrl_set_rc_max_intra_bitrate_pct }, + { AV1E_SET_MAX_INTER_BITRATE_PCT, ctrl_set_rc_max_inter_bitrate_pct }, + { AV1E_SET_GF_CBR_BOOST_PCT, ctrl_set_rc_gf_cbr_boost_pct }, + { AV1E_SET_LOSSLESS, ctrl_set_lossless }, +#if CONFIG_AOM_QM + { AV1E_SET_ENABLE_QM, ctrl_set_enable_qm }, + { AV1E_SET_QM_MIN, ctrl_set_qm_min }, + { AV1E_SET_QM_MAX, ctrl_set_qm_max }, +#endif +#if CONFIG_TILE_GROUPS + { AV1E_SET_NUM_TG, ctrl_set_num_tg }, + { AV1E_SET_MTU, ctrl_set_mtu }, +#endif +#if CONFIG_TEMPMV_SIGNALING + { AV1E_SET_DISABLE_TEMPMV, ctrl_set_disable_tempmv }, +#endif + { AV1E_SET_FRAME_PARALLEL_DECODING, ctrl_set_frame_parallel_decoding_mode }, + { AV1E_SET_AQ_MODE, ctrl_set_aq_mode }, +#if CONFIG_EXT_DELTA_Q + { AV1E_SET_DELTAQ_MODE, ctrl_set_deltaq_mode }, +#endif + { AV1E_SET_FRAME_PERIODIC_BOOST, ctrl_set_frame_periodic_boost }, + { AV1E_SET_TUNE_CONTENT, ctrl_set_tune_content }, + { AV1E_SET_COLOR_SPACE, ctrl_set_color_space }, + { AV1E_SET_COLOR_RANGE, ctrl_set_color_range }, + { AV1E_SET_NOISE_SENSITIVITY, ctrl_set_noise_sensitivity }, + { AV1E_SET_MIN_GF_INTERVAL, ctrl_set_min_gf_interval }, + { AV1E_SET_MAX_GF_INTERVAL, ctrl_set_max_gf_interval }, + { AV1E_SET_RENDER_SIZE, ctrl_set_render_size }, + { AV1E_SET_SUPERBLOCK_SIZE, ctrl_set_superblock_size }, +#if CONFIG_ANS && ANS_MAX_SYMBOLS + { AV1E_SET_ANS_WINDOW_SIZE_LOG2, ctrl_set_ans_window_size_log2 }, +#endif +#if CONFIG_EXT_TILE + { AV1E_SET_TILE_ENCODING_MODE, ctrl_set_tile_encoding_mode }, +#endif // CONFIG_EXT_TILE + { AV1E_ENABLE_MOTION_VECTOR_UNIT_TEST, ctrl_enable_motion_vector_unit_test }, + + // Getters + { AOME_GET_LAST_QUANTIZER, ctrl_get_quantizer }, + { AOME_GET_LAST_QUANTIZER_64, ctrl_get_quantizer64 }, + { AV1_GET_REFERENCE, ctrl_get_reference }, + { AV1E_GET_ACTIVEMAP, ctrl_get_active_map }, + { AV1_GET_NEW_FRAME_IMAGE, ctrl_get_new_frame_image }, + + { -1, NULL }, +}; + +static aom_codec_enc_cfg_map_t encoder_usage_cfg_map[] = { + { 0, + { + // NOLINT + 0, // g_usage + 8, // g_threads + 0, // g_profile + + 320, // g_width + 240, // g_height + AOM_BITS_8, // g_bit_depth + 8, // g_input_bit_depth + + { 1, 30 }, // g_timebase + + 0, // g_error_resilient + + AOM_RC_ONE_PASS, // g_pass + + 25, // g_lag_in_frames + + 0, // rc_dropframe_thresh + 0, // rc_resize_allowed + 0, // rc_scaled_width + 0, // rc_scaled_height + 60, // rc_resize_down_thresold + 30, // rc_resize_up_thresold + + AOM_VBR, // rc_end_usage + { NULL, 0 }, // rc_twopass_stats_in + { NULL, 0 }, // rc_firstpass_mb_stats_in + 256, // rc_target_bandwidth + 0, // rc_min_quantizer + 63, // rc_max_quantizer + 25, // rc_undershoot_pct + 25, // rc_overshoot_pct + + 6000, // rc_max_buffer_size + 4000, // rc_buffer_initial_size + 5000, // rc_buffer_optimal_size + + 50, // rc_two_pass_vbrbias + 0, // rc_two_pass_vbrmin_section + 2000, // rc_two_pass_vbrmax_section + + // keyframing settings (kf) + AOM_KF_AUTO, // g_kfmode + 0, // kf_min_dist + 9999, // kf_max_dist + } }, +}; + +#ifndef VERSION_STRING +#define VERSION_STRING +#endif +CODEC_INTERFACE(aom_codec_av1_cx) = { + "AOMedia Project AV1 Encoder" VERSION_STRING, + AOM_CODEC_INTERNAL_ABI_VERSION, +#if CONFIG_HIGHBITDEPTH + AOM_CODEC_CAP_HIGHBITDEPTH | +#endif + AOM_CODEC_CAP_ENCODER | AOM_CODEC_CAP_PSNR, // aom_codec_caps_t + encoder_init, // aom_codec_init_fn_t + encoder_destroy, // aom_codec_destroy_fn_t + encoder_ctrl_maps, // aom_codec_ctrl_fn_map_t + { + // NOLINT + NULL, // aom_codec_peek_si_fn_t + NULL, // aom_codec_get_si_fn_t + NULL, // aom_codec_decode_fn_t + NULL, // aom_codec_frame_get_fn_t + NULL // aom_codec_set_fb_fn_t + }, + { + // NOLINT + 1, // 1 cfg map + encoder_usage_cfg_map, // aom_codec_enc_cfg_map_t + encoder_encode, // aom_codec_encode_fn_t + encoder_get_cxdata, // aom_codec_get_cx_data_fn_t + encoder_set_config, // aom_codec_enc_config_set_fn_t + NULL, // aom_codec_get_global_headers_fn_t + encoder_get_preview, // aom_codec_get_preview_frame_fn_t + NULL // aom_codec_enc_mr_get_mem_loc_fn_t + } +}; diff --git a/third_party/aom/av1/av1_dx.mk b/third_party/aom/av1/av1_dx.mk new file mode 100644 index 0000000000..1a54ea22a2 --- /dev/null +++ b/third_party/aom/av1/av1_dx.mk @@ -0,0 +1,71 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +AV1_DX_EXPORTS += exports_dec + +AV1_DX_SRCS-yes += $(AV1_COMMON_SRCS-yes) +AV1_DX_SRCS-no += $(AV1_COMMON_SRCS-no) +AV1_DX_SRCS_REMOVE-yes += $(AV1_COMMON_SRCS_REMOVE-yes) +AV1_DX_SRCS_REMOVE-no += $(AV1_COMMON_SRCS_REMOVE-no) + +AV1_DX_SRCS-yes += av1_dx_iface.c + +AV1_DX_SRCS-yes += decoder/decodemv.c +AV1_DX_SRCS-yes += decoder/decodeframe.c +AV1_DX_SRCS-yes += decoder/decodeframe.h +AV1_DX_SRCS-yes += decoder/detokenize.c +AV1_DX_SRCS-yes += decoder/decodemv.h +AV1_DX_SRCS-$(CONFIG_LV_MAP) += decoder/decodetxb.c +AV1_DX_SRCS-$(CONFIG_LV_MAP) += decoder/decodetxb.h +AV1_DX_SRCS-yes += decoder/detokenize.h +AV1_DX_SRCS-yes += decoder/dthread.c +AV1_DX_SRCS-yes += decoder/dthread.h +AV1_DX_SRCS-yes += decoder/decoder.c +AV1_DX_SRCS-yes += decoder/decoder.h +AV1_DX_SRCS-yes += decoder/dsubexp.c +AV1_DX_SRCS-yes += decoder/dsubexp.h + +ifeq ($(CONFIG_ACCOUNTING),yes) +AV1_DX_SRCS-yes += decoder/accounting.h +AV1_DX_SRCS-yes += decoder/accounting.c +endif + +ifeq ($(CONFIG_INSPECTION),yes) +AV1_DX_SRCS-yes += decoder/inspection.c +AV1_DX_SRCS-yes += decoder/inspection.h +endif + +ifeq ($(CONFIG_PVQ),yes) +# PVQ from daala +AV1_DX_SRCS-yes += decoder/pvq_decoder.c +AV1_DX_SRCS-yes += decoder/pvq_decoder.h +AV1_DX_SRCS-yes += decoder/decint.h +AV1_DX_SRCS-yes += decoder/generic_decoder.c +AV1_DX_SRCS-yes += decoder/laplace_decoder.c +AV1_DX_SRCS-yes += encoder/hybrid_fwd_txfm.c +AV1_DX_SRCS-yes += encoder/hybrid_fwd_txfm.h + +AV1_DX_SRCS-yes += encoder/dct.c +AV1_DX_SRCS-$(HAVE_SSE2) += encoder/x86/dct_sse2.asm +AV1_DX_SRCS-$(HAVE_SSE2) += encoder/x86/dct_intrin_sse2.c +AV1_DX_SRCS-$(HAVE_SSSE3) += encoder/x86/dct_ssse3.c + +ifneq ($(CONFIG_HIGHBITDEPTH),yes) +AV1_DX_SRCS-$(HAVE_NEON) += encoder/arm/neon/dct_neon.c +endif + +AV1_DX_SRCS-$(HAVE_MSA) += encoder/mips/msa/fdct4x4_msa.c +AV1_DX_SRCS-$(HAVE_MSA) += encoder/mips/msa/fdct8x8_msa.c +AV1_DX_SRCS-$(HAVE_MSA) += encoder/mips/msa/fdct16x16_msa.c +AV1_DX_SRCS-$(HAVE_MSA) += encoder/mips/msa/fdct_msa.h +endif + +AV1_DX_SRCS-yes := $(filter-out $(AV1_DX_SRCS_REMOVE-yes),$(AV1_DX_SRCS-yes)) diff --git a/third_party/aom/av1/av1_dx_iface.c b/third_party/aom/av1/av1_dx_iface.c new file mode 100644 index 0000000000..f20ea4815a --- /dev/null +++ b/third_party/aom/av1/av1_dx_iface.c @@ -0,0 +1,1223 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./aom_version.h" + +#include "aom/internal/aom_codec_internal.h" +#include "aom/aomdx.h" +#include "aom/aom_decoder.h" +#include "aom_dsp/bitreader_buffer.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_util/aom_thread.h" + +#include "av1/common/alloccommon.h" +#include "av1/common/frame_buffers.h" +#include "av1/common/enums.h" + +#include "av1/decoder/decoder.h" +#include "av1/decoder/decodeframe.h" + +#include "av1/av1_iface_common.h" + +typedef aom_codec_stream_info_t av1_stream_info_t; + +// This limit is due to framebuffer numbers. +// TODO(hkuang): Remove this limit after implementing ondemand framebuffers. +#define FRAME_CACHE_SIZE 6 // Cache maximum 6 decoded frames. + +typedef struct cache_frame { + int fb_idx; + aom_image_t img; +} cache_frame; + +struct aom_codec_alg_priv { + aom_codec_priv_t base; + aom_codec_dec_cfg_t cfg; + av1_stream_info_t si; + int postproc_cfg_set; + aom_postproc_cfg_t postproc_cfg; + aom_decrypt_cb decrypt_cb; + void *decrypt_state; + aom_image_t img; + int img_avail; + int flushed; + int invert_tile_order; + int last_show_frame; // Index of last output frame. + int byte_alignment; + int skip_loop_filter; + int decode_tile_row; + int decode_tile_col; + + // Frame parallel related. + int frame_parallel_decode; // frame-based threading. + AVxWorker *frame_workers; + int num_frame_workers; + int next_submit_worker_id; + int last_submit_worker_id; + int next_output_worker_id; + int available_threads; + cache_frame frame_cache[FRAME_CACHE_SIZE]; + int frame_cache_write; + int frame_cache_read; + int num_cache_frames; + int need_resync; // wait for key/intra-only frame + // BufferPool that holds all reference frames. Shared by all the FrameWorkers. + BufferPool *buffer_pool; + + // External frame buffer info to save for AV1 common. + void *ext_priv; // Private data associated with the external frame buffers. + aom_get_frame_buffer_cb_fn_t get_ext_fb_cb; + aom_release_frame_buffer_cb_fn_t release_ext_fb_cb; + +#if CONFIG_INSPECTION + aom_inspect_cb inspect_cb; + void *inspect_ctx; +#endif +}; + +static aom_codec_err_t decoder_init(aom_codec_ctx_t *ctx, + aom_codec_priv_enc_mr_cfg_t *data) { + // This function only allocates space for the aom_codec_alg_priv_t + // structure. More memory may be required at the time the stream + // information becomes known. + (void)data; + + if (!ctx->priv) { + aom_codec_alg_priv_t *const priv = + (aom_codec_alg_priv_t *)aom_calloc(1, sizeof(*priv)); + if (priv == NULL) return AOM_CODEC_MEM_ERROR; + + ctx->priv = (aom_codec_priv_t *)priv; + ctx->priv->init_flags = ctx->init_flags; + priv->si.sz = sizeof(priv->si); + priv->flushed = 0; + // Only do frame parallel decode when threads > 1. + priv->frame_parallel_decode = + (ctx->config.dec && (ctx->config.dec->threads > 1) && + (ctx->init_flags & AOM_CODEC_USE_FRAME_THREADING)) + ? 1 + : 0; + if (ctx->config.dec) { + priv->cfg = *ctx->config.dec; + ctx->config.dec = &priv->cfg; + } + } + + return AOM_CODEC_OK; +} + +static aom_codec_err_t decoder_destroy(aom_codec_alg_priv_t *ctx) { + if (ctx->frame_workers != NULL) { + int i; + for (i = 0; i < ctx->num_frame_workers; ++i) { + AVxWorker *const worker = &ctx->frame_workers[i]; + FrameWorkerData *const frame_worker_data = + (FrameWorkerData *)worker->data1; + aom_get_worker_interface()->end(worker); + av1_remove_common(&frame_worker_data->pbi->common); +#if CONFIG_LOOP_RESTORATION + av1_free_restoration_buffers(&frame_worker_data->pbi->common); +#endif // CONFIG_LOOP_RESTORATION + av1_decoder_remove(frame_worker_data->pbi); + aom_free(frame_worker_data->scratch_buffer); +#if CONFIG_MULTITHREAD + pthread_mutex_destroy(&frame_worker_data->stats_mutex); + pthread_cond_destroy(&frame_worker_data->stats_cond); +#endif + aom_free(frame_worker_data); + } +#if CONFIG_MULTITHREAD + pthread_mutex_destroy(&ctx->buffer_pool->pool_mutex); +#endif + } + + if (ctx->buffer_pool) { + av1_free_ref_frame_buffers(ctx->buffer_pool); + av1_free_internal_frame_buffers(&ctx->buffer_pool->int_frame_buffers); + } + + aom_free(ctx->frame_workers); + aom_free(ctx->buffer_pool); + aom_free(ctx); + return AOM_CODEC_OK; +} + +static int parse_bitdepth_colorspace_sampling(BITSTREAM_PROFILE profile, + struct aom_read_bit_buffer *rb) { + aom_color_space_t color_space; + if (profile >= PROFILE_2) rb->bit_offset += 1; // Bit-depth 10 or 12. + color_space = (aom_color_space_t)aom_rb_read_literal(rb, 3); + if (color_space != AOM_CS_SRGB) { + rb->bit_offset += 1; // [16,235] (including xvycc) vs [0,255] range. + if (profile == PROFILE_1 || profile == PROFILE_3) { + rb->bit_offset += 2; // subsampling x/y. + rb->bit_offset += 1; // unused. + } + } else { + if (profile == PROFILE_1 || profile == PROFILE_3) { + rb->bit_offset += 1; // unused + } else { + // RGB is only available in version 1. + return 0; + } + } + return 1; +} + +static aom_codec_err_t decoder_peek_si_internal( + const uint8_t *data, unsigned int data_sz, aom_codec_stream_info_t *si, + int *is_intra_only, aom_decrypt_cb decrypt_cb, void *decrypt_state) { + int intra_only_flag = 0; + uint8_t clear_buffer[9]; + + if (data + data_sz <= data) return AOM_CODEC_INVALID_PARAM; + + si->is_kf = 0; + si->w = si->h = 0; + + if (decrypt_cb) { + data_sz = AOMMIN(sizeof(clear_buffer), data_sz); + decrypt_cb(decrypt_state, data, clear_buffer, data_sz); + data = clear_buffer; + } + + { + int show_frame; + int error_resilient; + struct aom_read_bit_buffer rb = { data, data + data_sz, 0, NULL, NULL }; + const int frame_marker = aom_rb_read_literal(&rb, 2); + const BITSTREAM_PROFILE profile = av1_read_profile(&rb); + + if (frame_marker != AOM_FRAME_MARKER) return AOM_CODEC_UNSUP_BITSTREAM; + + if (profile >= MAX_PROFILES) return AOM_CODEC_UNSUP_BITSTREAM; + + if ((profile >= 2 && data_sz <= 1) || data_sz < 1) + return AOM_CODEC_UNSUP_BITSTREAM; + + if (aom_rb_read_bit(&rb)) { // show an existing frame + aom_rb_read_literal(&rb, 3); // Frame buffer to show. + return AOM_CODEC_OK; + } + + if (data_sz <= 8) return AOM_CODEC_UNSUP_BITSTREAM; + + si->is_kf = !aom_rb_read_bit(&rb); + show_frame = aom_rb_read_bit(&rb); + error_resilient = aom_rb_read_bit(&rb); +#if CONFIG_REFERENCE_BUFFER + { + /* TODO: Move outside frame loop or inside key-frame branch */ + int frame_id_len; + SequenceHeader seq_params; + read_sequence_header(&seq_params); + if (seq_params.frame_id_numbers_present_flag) { + frame_id_len = seq_params.frame_id_length_minus7 + 7; + aom_rb_read_literal(&rb, frame_id_len); + } + } +#endif + if (si->is_kf) { + if (!av1_read_sync_code(&rb)) return AOM_CODEC_UNSUP_BITSTREAM; + + if (!parse_bitdepth_colorspace_sampling(profile, &rb)) + return AOM_CODEC_UNSUP_BITSTREAM; + av1_read_frame_size(&rb, (int *)&si->w, (int *)&si->h); + } else { + intra_only_flag = show_frame ? 0 : aom_rb_read_bit(&rb); + + rb.bit_offset += error_resilient ? 0 : 2; // reset_frame_context + + if (intra_only_flag) { + if (!av1_read_sync_code(&rb)) return AOM_CODEC_UNSUP_BITSTREAM; + if (profile > PROFILE_0) { + if (!parse_bitdepth_colorspace_sampling(profile, &rb)) + return AOM_CODEC_UNSUP_BITSTREAM; + } + rb.bit_offset += REF_FRAMES; // refresh_frame_flags + av1_read_frame_size(&rb, (int *)&si->w, (int *)&si->h); + } + } + } + if (is_intra_only != NULL) *is_intra_only = intra_only_flag; + return AOM_CODEC_OK; +} + +static aom_codec_err_t decoder_peek_si(const uint8_t *data, + unsigned int data_sz, + aom_codec_stream_info_t *si) { + return decoder_peek_si_internal(data, data_sz, si, NULL, NULL, NULL); +} + +static aom_codec_err_t decoder_get_si(aom_codec_alg_priv_t *ctx, + aom_codec_stream_info_t *si) { + const size_t sz = (si->sz >= sizeof(av1_stream_info_t)) + ? sizeof(av1_stream_info_t) + : sizeof(aom_codec_stream_info_t); + memcpy(si, &ctx->si, sz); + si->sz = (unsigned int)sz; + + return AOM_CODEC_OK; +} + +static void set_error_detail(aom_codec_alg_priv_t *ctx, + const char *const error) { + ctx->base.err_detail = error; +} + +static aom_codec_err_t update_error_state( + aom_codec_alg_priv_t *ctx, const struct aom_internal_error_info *error) { + if (error->error_code) + set_error_detail(ctx, error->has_detail ? error->detail : NULL); + + return error->error_code; +} + +static void init_buffer_callbacks(aom_codec_alg_priv_t *ctx) { + int i; + + for (i = 0; i < ctx->num_frame_workers; ++i) { + AVxWorker *const worker = &ctx->frame_workers[i]; + FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; + AV1_COMMON *const cm = &frame_worker_data->pbi->common; + BufferPool *const pool = cm->buffer_pool; + + cm->new_fb_idx = INVALID_IDX; + cm->byte_alignment = ctx->byte_alignment; + cm->skip_loop_filter = ctx->skip_loop_filter; + + if (ctx->get_ext_fb_cb != NULL && ctx->release_ext_fb_cb != NULL) { + pool->get_fb_cb = ctx->get_ext_fb_cb; + pool->release_fb_cb = ctx->release_ext_fb_cb; + pool->cb_priv = ctx->ext_priv; + } else { + pool->get_fb_cb = av1_get_frame_buffer; + pool->release_fb_cb = av1_release_frame_buffer; + + if (av1_alloc_internal_frame_buffers(&pool->int_frame_buffers)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to initialize internal frame buffers"); + + pool->cb_priv = &pool->int_frame_buffers; + } + } +} + +static void set_default_ppflags(aom_postproc_cfg_t *cfg) { + cfg->post_proc_flag = AOM_DEBLOCK | AOM_DEMACROBLOCK; + cfg->deblocking_level = 4; + cfg->noise_level = 0; +} + +static int frame_worker_hook(void *arg1, void *arg2) { + FrameWorkerData *const frame_worker_data = (FrameWorkerData *)arg1; + const uint8_t *data = frame_worker_data->data; + (void)arg2; + + frame_worker_data->result = av1_receive_compressed_data( + frame_worker_data->pbi, frame_worker_data->data_size, &data); + frame_worker_data->data_end = data; + + if (frame_worker_data->pbi->common.frame_parallel_decode) { + // In frame parallel decoding, a worker thread must successfully decode all + // the compressed data. + if (frame_worker_data->result != 0 || + frame_worker_data->data + frame_worker_data->data_size - 1 > data) { + AVxWorker *const worker = frame_worker_data->pbi->frame_worker_owner; + BufferPool *const pool = frame_worker_data->pbi->common.buffer_pool; + // Signal all the other threads that are waiting for this frame. + av1_frameworker_lock_stats(worker); + frame_worker_data->frame_context_ready = 1; + lock_buffer_pool(pool); + frame_worker_data->pbi->cur_buf->buf.corrupted = 1; + unlock_buffer_pool(pool); + frame_worker_data->pbi->need_resync = 1; + av1_frameworker_signal_stats(worker); + av1_frameworker_unlock_stats(worker); + return 0; + } + } else if (frame_worker_data->result != 0) { + // Check decode result in serial decode. + frame_worker_data->pbi->cur_buf->buf.corrupted = 1; + frame_worker_data->pbi->need_resync = 1; + } + return !frame_worker_data->result; +} + +static aom_codec_err_t init_decoder(aom_codec_alg_priv_t *ctx) { + int i; + const AVxWorkerInterface *const winterface = aom_get_worker_interface(); + + ctx->last_show_frame = -1; + ctx->next_submit_worker_id = 0; + ctx->last_submit_worker_id = 0; + ctx->next_output_worker_id = 0; + ctx->frame_cache_read = 0; + ctx->frame_cache_write = 0; + ctx->num_cache_frames = 0; + ctx->need_resync = 1; + ctx->num_frame_workers = + (ctx->frame_parallel_decode == 1) ? ctx->cfg.threads : 1; + if (ctx->num_frame_workers > MAX_DECODE_THREADS) + ctx->num_frame_workers = MAX_DECODE_THREADS; + ctx->available_threads = ctx->num_frame_workers; + ctx->flushed = 0; + + ctx->buffer_pool = (BufferPool *)aom_calloc(1, sizeof(BufferPool)); + if (ctx->buffer_pool == NULL) return AOM_CODEC_MEM_ERROR; + +#if CONFIG_MULTITHREAD + if (pthread_mutex_init(&ctx->buffer_pool->pool_mutex, NULL)) { + set_error_detail(ctx, "Failed to allocate buffer pool mutex"); + return AOM_CODEC_MEM_ERROR; + } +#endif + + ctx->frame_workers = (AVxWorker *)aom_malloc(ctx->num_frame_workers * + sizeof(*ctx->frame_workers)); + if (ctx->frame_workers == NULL) { + set_error_detail(ctx, "Failed to allocate frame_workers"); + return AOM_CODEC_MEM_ERROR; + } + + for (i = 0; i < ctx->num_frame_workers; ++i) { + AVxWorker *const worker = &ctx->frame_workers[i]; + FrameWorkerData *frame_worker_data = NULL; + winterface->init(worker); + worker->data1 = aom_memalign(32, sizeof(FrameWorkerData)); + if (worker->data1 == NULL) { + set_error_detail(ctx, "Failed to allocate frame_worker_data"); + return AOM_CODEC_MEM_ERROR; + } + frame_worker_data = (FrameWorkerData *)worker->data1; + frame_worker_data->pbi = av1_decoder_create(ctx->buffer_pool); + if (frame_worker_data->pbi == NULL) { + set_error_detail(ctx, "Failed to allocate frame_worker_data"); + return AOM_CODEC_MEM_ERROR; + } + frame_worker_data->pbi->frame_worker_owner = worker; + frame_worker_data->worker_id = i; + frame_worker_data->scratch_buffer = NULL; + frame_worker_data->scratch_buffer_size = 0; + frame_worker_data->frame_context_ready = 0; + frame_worker_data->received_frame = 0; +#if CONFIG_MULTITHREAD + if (pthread_mutex_init(&frame_worker_data->stats_mutex, NULL)) { + set_error_detail(ctx, "Failed to allocate frame_worker_data mutex"); + return AOM_CODEC_MEM_ERROR; + } + + if (pthread_cond_init(&frame_worker_data->stats_cond, NULL)) { + set_error_detail(ctx, "Failed to allocate frame_worker_data cond"); + return AOM_CODEC_MEM_ERROR; + } +#endif + // If decoding in serial mode, FrameWorker thread could create tile worker + // thread or loopfilter thread. + frame_worker_data->pbi->max_threads = + (ctx->frame_parallel_decode == 0) ? ctx->cfg.threads : 0; + + frame_worker_data->pbi->inv_tile_order = ctx->invert_tile_order; + frame_worker_data->pbi->common.frame_parallel_decode = + ctx->frame_parallel_decode; + worker->hook = (AVxWorkerHook)frame_worker_hook; + if (!winterface->reset(worker)) { + set_error_detail(ctx, "Frame Worker thread creation failed"); + return AOM_CODEC_MEM_ERROR; + } + } + + // If postprocessing was enabled by the application and a + // configuration has not been provided, default it. + if (!ctx->postproc_cfg_set && (ctx->base.init_flags & AOM_CODEC_USE_POSTPROC)) + set_default_ppflags(&ctx->postproc_cfg); + + init_buffer_callbacks(ctx); + + return AOM_CODEC_OK; +} + +static INLINE void check_resync(aom_codec_alg_priv_t *const ctx, + const AV1Decoder *const pbi) { + // Clear resync flag if worker got a key frame or intra only frame. + if (ctx->need_resync == 1 && pbi->need_resync == 0 && + (pbi->common.intra_only || pbi->common.frame_type == KEY_FRAME)) + ctx->need_resync = 0; +} + +static aom_codec_err_t decode_one(aom_codec_alg_priv_t *ctx, + const uint8_t **data, unsigned int data_sz, + void *user_priv, int64_t deadline) { + const AVxWorkerInterface *const winterface = aom_get_worker_interface(); + (void)deadline; + + // Determine the stream parameters. Note that we rely on peek_si to + // validate that we have a buffer that does not wrap around the top + // of the heap. + if (!ctx->si.h) { + int is_intra_only = 0; + const aom_codec_err_t res = + decoder_peek_si_internal(*data, data_sz, &ctx->si, &is_intra_only, + ctx->decrypt_cb, ctx->decrypt_state); + if (res != AOM_CODEC_OK) return res; + + if (!ctx->si.is_kf && !is_intra_only) return AOM_CODEC_ERROR; + } + + if (!ctx->frame_parallel_decode) { + AVxWorker *const worker = ctx->frame_workers; + FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; + frame_worker_data->data = *data; + frame_worker_data->data_size = data_sz; + frame_worker_data->user_priv = user_priv; + frame_worker_data->received_frame = 1; + + // Set these even if already initialized. The caller may have changed the + // decrypt config between frames. + frame_worker_data->pbi->decrypt_cb = ctx->decrypt_cb; + frame_worker_data->pbi->decrypt_state = ctx->decrypt_state; +#if CONFIG_INSPECTION + frame_worker_data->pbi->inspect_cb = ctx->inspect_cb; + frame_worker_data->pbi->inspect_ctx = ctx->inspect_ctx; +#endif + +#if CONFIG_EXT_TILE + frame_worker_data->pbi->dec_tile_row = ctx->decode_tile_row; + frame_worker_data->pbi->dec_tile_col = ctx->decode_tile_col; +#endif // CONFIG_EXT_TILE + + worker->had_error = 0; + winterface->execute(worker); + + // Update data pointer after decode. + *data = frame_worker_data->data_end; + + if (worker->had_error) + return update_error_state(ctx, &frame_worker_data->pbi->common.error); + + check_resync(ctx, frame_worker_data->pbi); + } else { + AVxWorker *const worker = &ctx->frame_workers[ctx->next_submit_worker_id]; + FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; + // Copy context from last worker thread to next worker thread. + if (ctx->next_submit_worker_id != ctx->last_submit_worker_id) + av1_frameworker_copy_context( + &ctx->frame_workers[ctx->next_submit_worker_id], + &ctx->frame_workers[ctx->last_submit_worker_id]); + + frame_worker_data->pbi->ready_for_new_data = 0; + // Copy the compressed data into worker's internal buffer. + // TODO(hkuang): Will all the workers allocate the same size + // as the size of the first intra frame be better? This will + // avoid too many deallocate and allocate. + if (frame_worker_data->scratch_buffer_size < data_sz) { + aom_free(frame_worker_data->scratch_buffer); + frame_worker_data->scratch_buffer = (uint8_t *)aom_malloc(data_sz); + if (frame_worker_data->scratch_buffer == NULL) { + set_error_detail(ctx, "Failed to reallocate scratch buffer"); + return AOM_CODEC_MEM_ERROR; + } + frame_worker_data->scratch_buffer_size = data_sz; + } + frame_worker_data->data_size = data_sz; + memcpy(frame_worker_data->scratch_buffer, *data, data_sz); + + frame_worker_data->frame_decoded = 0; + frame_worker_data->frame_context_ready = 0; + frame_worker_data->received_frame = 1; + frame_worker_data->data = frame_worker_data->scratch_buffer; + frame_worker_data->user_priv = user_priv; + + if (ctx->next_submit_worker_id != ctx->last_submit_worker_id) + ctx->last_submit_worker_id = + (ctx->last_submit_worker_id + 1) % ctx->num_frame_workers; + + ctx->next_submit_worker_id = + (ctx->next_submit_worker_id + 1) % ctx->num_frame_workers; + --ctx->available_threads; + worker->had_error = 0; + winterface->launch(worker); + } + + return AOM_CODEC_OK; +} + +static void wait_worker_and_cache_frame(aom_codec_alg_priv_t *ctx) { + YV12_BUFFER_CONFIG sd; + const AVxWorkerInterface *const winterface = aom_get_worker_interface(); + AVxWorker *const worker = &ctx->frame_workers[ctx->next_output_worker_id]; + FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; + ctx->next_output_worker_id = + (ctx->next_output_worker_id + 1) % ctx->num_frame_workers; + // TODO(hkuang): Add worker error handling here. + winterface->sync(worker); + frame_worker_data->received_frame = 0; + ++ctx->available_threads; + + check_resync(ctx, frame_worker_data->pbi); + + if (av1_get_raw_frame(frame_worker_data->pbi, &sd) == 0) { + AV1_COMMON *const cm = &frame_worker_data->pbi->common; + RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; + ctx->frame_cache[ctx->frame_cache_write].fb_idx = cm->new_fb_idx; + yuvconfig2image(&ctx->frame_cache[ctx->frame_cache_write].img, &sd, + frame_worker_data->user_priv); + ctx->frame_cache[ctx->frame_cache_write].img.fb_priv = + frame_bufs[cm->new_fb_idx].raw_frame_buffer.priv; + ctx->frame_cache_write = (ctx->frame_cache_write + 1) % FRAME_CACHE_SIZE; + ++ctx->num_cache_frames; + } +} + +static aom_codec_err_t decoder_decode(aom_codec_alg_priv_t *ctx, + const uint8_t *data, unsigned int data_sz, + void *user_priv, long deadline) { + const uint8_t *data_start = data; + const uint8_t *const data_end = data + data_sz; + aom_codec_err_t res; + uint32_t frame_sizes[8]; + int frame_count; + + if (data == NULL && data_sz == 0) { + ctx->flushed = 1; + return AOM_CODEC_OK; + } + + // Reset flushed when receiving a valid frame. + ctx->flushed = 0; + + // Initialize the decoder workers on the first frame. + if (ctx->frame_workers == NULL) { + res = init_decoder(ctx); + if (res != AOM_CODEC_OK) return res; + } + + res = av1_parse_superframe_index(data, data_sz, frame_sizes, &frame_count, + ctx->decrypt_cb, ctx->decrypt_state); + if (res != AOM_CODEC_OK) return res; + + if (ctx->frame_parallel_decode) { + // Decode in frame parallel mode. When decoding in this mode, the frame + // passed to the decoder must be either a normal frame or a superframe with + // superframe index so the decoder could get each frame's start position + // in the superframe. + if (frame_count > 0) { + int i; + + for (i = 0; i < frame_count; ++i) { + const uint8_t *data_start_copy = data_start; + const uint32_t frame_size = frame_sizes[i]; + if (data_start < data || + frame_size > (uint32_t)(data_end - data_start)) { + set_error_detail(ctx, "Invalid frame size in index"); + return AOM_CODEC_CORRUPT_FRAME; + } + + if (ctx->available_threads == 0) { + // No more threads for decoding. Wait until the next output worker + // finishes decoding. Then copy the decoded frame into cache. + if (ctx->num_cache_frames < FRAME_CACHE_SIZE) { + wait_worker_and_cache_frame(ctx); + } else { + // TODO(hkuang): Add unit test to test this path. + set_error_detail(ctx, "Frame output cache is full."); + return AOM_CODEC_ERROR; + } + } + + res = + decode_one(ctx, &data_start_copy, frame_size, user_priv, deadline); + if (res != AOM_CODEC_OK) return res; + data_start += frame_size; + } + } else { + if (ctx->available_threads == 0) { + // No more threads for decoding. Wait until the next output worker + // finishes decoding. Then copy the decoded frame into cache. + if (ctx->num_cache_frames < FRAME_CACHE_SIZE) { + wait_worker_and_cache_frame(ctx); + } else { + // TODO(hkuang): Add unit test to test this path. + set_error_detail(ctx, "Frame output cache is full."); + return AOM_CODEC_ERROR; + } + } + + res = decode_one(ctx, &data, data_sz, user_priv, deadline); + if (res != AOM_CODEC_OK) return res; + } + } else { + // Decode in serial mode. + if (frame_count > 0) { + int i; + + for (i = 0; i < frame_count; ++i) { + const uint8_t *data_start_copy = data_start; + const uint32_t frame_size = frame_sizes[i]; + if (data_start < data || + frame_size > (uint32_t)(data_end - data_start)) { + set_error_detail(ctx, "Invalid frame size in index"); + return AOM_CODEC_CORRUPT_FRAME; + } + + res = + decode_one(ctx, &data_start_copy, frame_size, user_priv, deadline); + if (res != AOM_CODEC_OK) return res; + + data_start += frame_size; + } + } else { + while (data_start < data_end) { + const uint32_t frame_size = (uint32_t)(data_end - data_start); + res = decode_one(ctx, &data_start, frame_size, user_priv, deadline); + if (res != AOM_CODEC_OK) return res; + + // Account for suboptimal termination by the encoder. + while (data_start < data_end) { + const uint8_t marker = + read_marker(ctx->decrypt_cb, ctx->decrypt_state, data_start); + if (marker) break; + ++data_start; + } + } + } + } + + return res; +} + +static void release_last_output_frame(aom_codec_alg_priv_t *ctx) { + RefCntBuffer *const frame_bufs = ctx->buffer_pool->frame_bufs; + // Decrease reference count of last output frame in frame parallel mode. + if (ctx->frame_parallel_decode && ctx->last_show_frame >= 0) { + BufferPool *const pool = ctx->buffer_pool; + lock_buffer_pool(pool); + decrease_ref_count(ctx->last_show_frame, frame_bufs, pool); + unlock_buffer_pool(pool); + } +} + +static aom_image_t *decoder_get_frame(aom_codec_alg_priv_t *ctx, + aom_codec_iter_t *iter) { + aom_image_t *img = NULL; + + // Only return frame when all the cpu are busy or + // application fluhsed the decoder in frame parallel decode. + if (ctx->frame_parallel_decode && ctx->available_threads > 0 && + !ctx->flushed) { + return NULL; + } + + // Output the frames in the cache first. + if (ctx->num_cache_frames > 0) { + release_last_output_frame(ctx); + ctx->last_show_frame = ctx->frame_cache[ctx->frame_cache_read].fb_idx; + if (ctx->need_resync) return NULL; + img = &ctx->frame_cache[ctx->frame_cache_read].img; + ctx->frame_cache_read = (ctx->frame_cache_read + 1) % FRAME_CACHE_SIZE; + --ctx->num_cache_frames; + return img; + } + + // iter acts as a flip flop, so an image is only returned on the first + // call to get_frame. + if (*iter == NULL && ctx->frame_workers != NULL) { + do { + YV12_BUFFER_CONFIG sd; + const AVxWorkerInterface *const winterface = aom_get_worker_interface(); + AVxWorker *const worker = &ctx->frame_workers[ctx->next_output_worker_id]; + FrameWorkerData *const frame_worker_data = + (FrameWorkerData *)worker->data1; + ctx->next_output_worker_id = + (ctx->next_output_worker_id + 1) % ctx->num_frame_workers; + // Wait for the frame from worker thread. + if (winterface->sync(worker)) { + // Check if worker has received any frames. + if (frame_worker_data->received_frame == 1) { + ++ctx->available_threads; + frame_worker_data->received_frame = 0; + check_resync(ctx, frame_worker_data->pbi); + } + if (av1_get_raw_frame(frame_worker_data->pbi, &sd) == 0) { + AV1_COMMON *const cm = &frame_worker_data->pbi->common; + RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; + release_last_output_frame(ctx); + ctx->last_show_frame = frame_worker_data->pbi->common.new_fb_idx; + if (ctx->need_resync) return NULL; + yuvconfig2image(&ctx->img, &sd, frame_worker_data->user_priv); + +#if CONFIG_EXT_TILE + if (cm->tile_encoding_mode && + frame_worker_data->pbi->dec_tile_row >= 0) { + const int tile_row = + AOMMIN(frame_worker_data->pbi->dec_tile_row, cm->tile_rows - 1); + const int mi_row = tile_row * cm->tile_height; + const int ssy = ctx->img.y_chroma_shift; + int plane; + ctx->img.planes[0] += mi_row * MI_SIZE * ctx->img.stride[0]; + for (plane = 1; plane < MAX_MB_PLANE; ++plane) { + ctx->img.planes[plane] += + mi_row * (MI_SIZE >> ssy) * ctx->img.stride[plane]; + } + ctx->img.d_h = + AOMMIN(cm->tile_height, cm->mi_rows - mi_row) * MI_SIZE; + } + + if (cm->tile_encoding_mode && + frame_worker_data->pbi->dec_tile_col >= 0) { + const int tile_col = + AOMMIN(frame_worker_data->pbi->dec_tile_col, cm->tile_cols - 1); + const int mi_col = tile_col * cm->tile_width; + const int ssx = ctx->img.x_chroma_shift; + int plane; + ctx->img.planes[0] += mi_col * MI_SIZE; + for (plane = 1; plane < MAX_MB_PLANE; ++plane) { + ctx->img.planes[plane] += mi_col * (MI_SIZE >> ssx); + } + ctx->img.d_w = + AOMMIN(cm->tile_width, cm->mi_cols - mi_col) * MI_SIZE; + } +#endif // CONFIG_EXT_TILE + + ctx->img.fb_priv = frame_bufs[cm->new_fb_idx].raw_frame_buffer.priv; + img = &ctx->img; + return img; + } + } else { + // Decoding failed. Release the worker thread. + frame_worker_data->received_frame = 0; + ++ctx->available_threads; + ctx->need_resync = 1; + if (ctx->flushed != 1) return NULL; + } + } while (ctx->next_output_worker_id != ctx->next_submit_worker_id); + } + return NULL; +} + +static aom_codec_err_t decoder_set_fb_fn( + aom_codec_alg_priv_t *ctx, aom_get_frame_buffer_cb_fn_t cb_get, + aom_release_frame_buffer_cb_fn_t cb_release, void *cb_priv) { + if (cb_get == NULL || cb_release == NULL) { + return AOM_CODEC_INVALID_PARAM; + } else if (ctx->frame_workers == NULL) { + // If the decoder has already been initialized, do not accept changes to + // the frame buffer functions. + ctx->get_ext_fb_cb = cb_get; + ctx->release_ext_fb_cb = cb_release; + ctx->ext_priv = cb_priv; + return AOM_CODEC_OK; + } + + return AOM_CODEC_ERROR; +} + +static aom_codec_err_t ctrl_set_reference(aom_codec_alg_priv_t *ctx, + va_list args) { + aom_ref_frame_t *const data = va_arg(args, aom_ref_frame_t *); + + // Only support this function in serial decode. + if (ctx->frame_parallel_decode) { + set_error_detail(ctx, "Not supported in frame parallel decode"); + return AOM_CODEC_INCAPABLE; + } + + if (data) { + aom_ref_frame_t *const frame = (aom_ref_frame_t *)data; + YV12_BUFFER_CONFIG sd; + AVxWorker *const worker = ctx->frame_workers; + FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; + image2yuvconfig(&frame->img, &sd); + return av1_set_reference_dec(&frame_worker_data->pbi->common, + ref_frame_to_av1_reframe(frame->frame_type), + &sd); + } else { + return AOM_CODEC_INVALID_PARAM; + } +} + +static aom_codec_err_t ctrl_copy_reference(aom_codec_alg_priv_t *ctx, + va_list args) { + const aom_ref_frame_t *const frame = va_arg(args, aom_ref_frame_t *); + + // Only support this function in serial decode. + if (ctx->frame_parallel_decode) { + set_error_detail(ctx, "Not supported in frame parallel decode"); + return AOM_CODEC_INCAPABLE; + } + + if (frame) { + YV12_BUFFER_CONFIG sd; + AVxWorker *const worker = ctx->frame_workers; + FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; + image2yuvconfig(&frame->img, &sd); + return av1_copy_reference_dec(frame_worker_data->pbi, + (AOM_REFFRAME)frame->frame_type, &sd); + } else { + return AOM_CODEC_INVALID_PARAM; + } +} + +static aom_codec_err_t ctrl_get_reference(aom_codec_alg_priv_t *ctx, + va_list args) { + av1_ref_frame_t *data = va_arg(args, av1_ref_frame_t *); + + // Only support this function in serial decode. + if (ctx->frame_parallel_decode) { + set_error_detail(ctx, "Not supported in frame parallel decode"); + return AOM_CODEC_INCAPABLE; + } + + if (data) { + YV12_BUFFER_CONFIG *fb; + AVxWorker *const worker = ctx->frame_workers; + FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; + fb = get_ref_frame(&frame_worker_data->pbi->common, data->idx); + if (fb == NULL) return AOM_CODEC_ERROR; + yuvconfig2image(&data->img, fb, NULL); + return AOM_CODEC_OK; + } else { + return AOM_CODEC_INVALID_PARAM; + } +} + +static aom_codec_err_t ctrl_get_new_frame_image(aom_codec_alg_priv_t *ctx, + va_list args) { + aom_image_t *new_img = va_arg(args, aom_image_t *); + + // Only support this function in serial decode. + if (ctx->frame_parallel_decode) { + set_error_detail(ctx, "Not supported in frame parallel decode"); + return AOM_CODEC_INCAPABLE; + } + + if (new_img) { + YV12_BUFFER_CONFIG new_frame; + AVxWorker *const worker = ctx->frame_workers; + FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; + + if (av1_get_frame_to_show(frame_worker_data->pbi, &new_frame) == 0) { + yuvconfig2image(new_img, &new_frame, NULL); + return AOM_CODEC_OK; + } else { + return AOM_CODEC_ERROR; + } + } else { + return AOM_CODEC_INVALID_PARAM; + } +} + +static aom_codec_err_t ctrl_set_postproc(aom_codec_alg_priv_t *ctx, + va_list args) { + (void)ctx; + (void)args; + return AOM_CODEC_INCAPABLE; +} + +static aom_codec_err_t ctrl_set_dbg_options(aom_codec_alg_priv_t *ctx, + va_list args) { + (void)ctx; + (void)args; + return AOM_CODEC_INCAPABLE; +} + +static aom_codec_err_t ctrl_get_last_ref_updates(aom_codec_alg_priv_t *ctx, + va_list args) { + int *const update_info = va_arg(args, int *); + + // Only support this function in serial decode. + if (ctx->frame_parallel_decode) { + set_error_detail(ctx, "Not supported in frame parallel decode"); + return AOM_CODEC_INCAPABLE; + } + + if (update_info) { + if (ctx->frame_workers) { + AVxWorker *const worker = ctx->frame_workers; + FrameWorkerData *const frame_worker_data = + (FrameWorkerData *)worker->data1; + *update_info = frame_worker_data->pbi->refresh_frame_flags; + return AOM_CODEC_OK; + } else { + return AOM_CODEC_ERROR; + } + } + + return AOM_CODEC_INVALID_PARAM; +} + +static aom_codec_err_t ctrl_get_last_quantizer(aom_codec_alg_priv_t *ctx, + va_list args) { + int *const arg = va_arg(args, int *); + if (arg == NULL) return AOM_CODEC_INVALID_PARAM; + *arg = + ((FrameWorkerData *)ctx->frame_workers[0].data1)->pbi->common.base_qindex; + return AOM_CODEC_OK; +} + +static aom_codec_err_t ctrl_get_frame_corrupted(aom_codec_alg_priv_t *ctx, + va_list args) { + int *corrupted = va_arg(args, int *); + + if (corrupted) { + if (ctx->frame_workers) { + AVxWorker *const worker = ctx->frame_workers; + FrameWorkerData *const frame_worker_data = + (FrameWorkerData *)worker->data1; + RefCntBuffer *const frame_bufs = + frame_worker_data->pbi->common.buffer_pool->frame_bufs; + if (frame_worker_data->pbi->common.frame_to_show == NULL) + return AOM_CODEC_ERROR; + if (ctx->last_show_frame >= 0) + *corrupted = frame_bufs[ctx->last_show_frame].buf.corrupted; + return AOM_CODEC_OK; + } else { + return AOM_CODEC_ERROR; + } + } + + return AOM_CODEC_INVALID_PARAM; +} + +static aom_codec_err_t ctrl_get_frame_size(aom_codec_alg_priv_t *ctx, + va_list args) { + int *const frame_size = va_arg(args, int *); + + // Only support this function in serial decode. + if (ctx->frame_parallel_decode) { + set_error_detail(ctx, "Not supported in frame parallel decode"); + return AOM_CODEC_INCAPABLE; + } + + if (frame_size) { + if (ctx->frame_workers) { + AVxWorker *const worker = ctx->frame_workers; + FrameWorkerData *const frame_worker_data = + (FrameWorkerData *)worker->data1; + const AV1_COMMON *const cm = &frame_worker_data->pbi->common; + frame_size[0] = cm->width; + frame_size[1] = cm->height; + return AOM_CODEC_OK; + } else { + return AOM_CODEC_ERROR; + } + } + + return AOM_CODEC_INVALID_PARAM; +} + +static aom_codec_err_t ctrl_get_render_size(aom_codec_alg_priv_t *ctx, + va_list args) { + int *const render_size = va_arg(args, int *); + + // Only support this function in serial decode. + if (ctx->frame_parallel_decode) { + set_error_detail(ctx, "Not supported in frame parallel decode"); + return AOM_CODEC_INCAPABLE; + } + + if (render_size) { + if (ctx->frame_workers) { + AVxWorker *const worker = ctx->frame_workers; + FrameWorkerData *const frame_worker_data = + (FrameWorkerData *)worker->data1; + const AV1_COMMON *const cm = &frame_worker_data->pbi->common; + render_size[0] = cm->render_width; + render_size[1] = cm->render_height; + return AOM_CODEC_OK; + } else { + return AOM_CODEC_ERROR; + } + } + + return AOM_CODEC_INVALID_PARAM; +} + +static aom_codec_err_t ctrl_get_bit_depth(aom_codec_alg_priv_t *ctx, + va_list args) { + unsigned int *const bit_depth = va_arg(args, unsigned int *); + AVxWorker *const worker = &ctx->frame_workers[ctx->next_output_worker_id]; + + if (bit_depth) { + if (worker) { + FrameWorkerData *const frame_worker_data = + (FrameWorkerData *)worker->data1; + const AV1_COMMON *const cm = &frame_worker_data->pbi->common; + *bit_depth = cm->bit_depth; + return AOM_CODEC_OK; + } else { + return AOM_CODEC_ERROR; + } + } + + return AOM_CODEC_INVALID_PARAM; +} + +static aom_codec_err_t ctrl_set_invert_tile_order(aom_codec_alg_priv_t *ctx, + va_list args) { + ctx->invert_tile_order = va_arg(args, int); + return AOM_CODEC_OK; +} + +static aom_codec_err_t ctrl_set_decryptor(aom_codec_alg_priv_t *ctx, + va_list args) { + aom_decrypt_init *init = va_arg(args, aom_decrypt_init *); + ctx->decrypt_cb = init ? init->decrypt_cb : NULL; + ctx->decrypt_state = init ? init->decrypt_state : NULL; + return AOM_CODEC_OK; +} + +static aom_codec_err_t ctrl_set_byte_alignment(aom_codec_alg_priv_t *ctx, + va_list args) { + const int legacy_byte_alignment = 0; + const int min_byte_alignment = 32; + const int max_byte_alignment = 1024; + const int byte_alignment = va_arg(args, int); + + if (byte_alignment != legacy_byte_alignment && + (byte_alignment < min_byte_alignment || + byte_alignment > max_byte_alignment || + (byte_alignment & (byte_alignment - 1)) != 0)) + return AOM_CODEC_INVALID_PARAM; + + ctx->byte_alignment = byte_alignment; + if (ctx->frame_workers) { + AVxWorker *const worker = ctx->frame_workers; + FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; + frame_worker_data->pbi->common.byte_alignment = byte_alignment; + } + return AOM_CODEC_OK; +} + +static aom_codec_err_t ctrl_set_skip_loop_filter(aom_codec_alg_priv_t *ctx, + va_list args) { + ctx->skip_loop_filter = va_arg(args, int); + + if (ctx->frame_workers) { + AVxWorker *const worker = ctx->frame_workers; + FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; + frame_worker_data->pbi->common.skip_loop_filter = ctx->skip_loop_filter; + } + + return AOM_CODEC_OK; +} + +static aom_codec_err_t ctrl_get_accounting(aom_codec_alg_priv_t *ctx, + va_list args) { +#if !CONFIG_ACCOUNTING + (void)ctx; + (void)args; + return AOM_CODEC_INCAPABLE; +#else + if (ctx->frame_workers) { + AVxWorker *const worker = ctx->frame_workers; + FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; + AV1Decoder *pbi = frame_worker_data->pbi; + Accounting **acct = va_arg(args, Accounting **); + *acct = &pbi->accounting; + return AOM_CODEC_OK; + } + return AOM_CODEC_ERROR; +#endif +} +static aom_codec_err_t ctrl_set_decode_tile_row(aom_codec_alg_priv_t *ctx, + va_list args) { + ctx->decode_tile_row = va_arg(args, int); + return AOM_CODEC_OK; +} + +static aom_codec_err_t ctrl_set_decode_tile_col(aom_codec_alg_priv_t *ctx, + va_list args) { + ctx->decode_tile_col = va_arg(args, int); + return AOM_CODEC_OK; +} + +static aom_codec_err_t ctrl_set_inspection_callback(aom_codec_alg_priv_t *ctx, + va_list args) { +#if !CONFIG_INSPECTION + (void)ctx; + (void)args; + return AOM_CODEC_INCAPABLE; +#else + aom_inspect_init *init = va_arg(args, aom_inspect_init *); + ctx->inspect_cb = init->inspect_cb; + ctx->inspect_ctx = init->inspect_ctx; + return AOM_CODEC_OK; +#endif +} + +static aom_codec_ctrl_fn_map_t decoder_ctrl_maps[] = { + { AOM_COPY_REFERENCE, ctrl_copy_reference }, + + // Setters + { AOM_SET_REFERENCE, ctrl_set_reference }, + { AOM_SET_POSTPROC, ctrl_set_postproc }, + { AOM_SET_DBG_COLOR_REF_FRAME, ctrl_set_dbg_options }, + { AOM_SET_DBG_COLOR_MB_MODES, ctrl_set_dbg_options }, + { AOM_SET_DBG_COLOR_B_MODES, ctrl_set_dbg_options }, + { AOM_SET_DBG_DISPLAY_MV, ctrl_set_dbg_options }, + { AV1_INVERT_TILE_DECODE_ORDER, ctrl_set_invert_tile_order }, + { AOMD_SET_DECRYPTOR, ctrl_set_decryptor }, + { AV1_SET_BYTE_ALIGNMENT, ctrl_set_byte_alignment }, + { AV1_SET_SKIP_LOOP_FILTER, ctrl_set_skip_loop_filter }, + { AV1_SET_DECODE_TILE_ROW, ctrl_set_decode_tile_row }, + { AV1_SET_DECODE_TILE_COL, ctrl_set_decode_tile_col }, + { AV1_SET_INSPECTION_CALLBACK, ctrl_set_inspection_callback }, + + // Getters + { AOMD_GET_FRAME_CORRUPTED, ctrl_get_frame_corrupted }, + { AOMD_GET_LAST_QUANTIZER, ctrl_get_last_quantizer }, + { AOMD_GET_LAST_REF_UPDATES, ctrl_get_last_ref_updates }, + { AV1D_GET_BIT_DEPTH, ctrl_get_bit_depth }, + { AV1D_GET_DISPLAY_SIZE, ctrl_get_render_size }, + { AV1D_GET_FRAME_SIZE, ctrl_get_frame_size }, + { AV1_GET_ACCOUNTING, ctrl_get_accounting }, + { AV1_GET_NEW_FRAME_IMAGE, ctrl_get_new_frame_image }, + { AV1_GET_REFERENCE, ctrl_get_reference }, + + { -1, NULL }, +}; + +#ifndef VERSION_STRING +#define VERSION_STRING +#endif +CODEC_INTERFACE(aom_codec_av1_dx) = { + "AOMedia Project AV1 Decoder" VERSION_STRING, + AOM_CODEC_INTERNAL_ABI_VERSION, + AOM_CODEC_CAP_DECODER | + AOM_CODEC_CAP_EXTERNAL_FRAME_BUFFER, // aom_codec_caps_t + decoder_init, // aom_codec_init_fn_t + decoder_destroy, // aom_codec_destroy_fn_t + decoder_ctrl_maps, // aom_codec_ctrl_fn_map_t + { + // NOLINT + decoder_peek_si, // aom_codec_peek_si_fn_t + decoder_get_si, // aom_codec_get_si_fn_t + decoder_decode, // aom_codec_decode_fn_t + decoder_get_frame, // aom_codec_frame_get_fn_t + decoder_set_fb_fn, // aom_codec_set_fb_fn_t + }, + { + // NOLINT + 0, + NULL, // aom_codec_enc_cfg_map_t + NULL, // aom_codec_encode_fn_t + NULL, // aom_codec_get_cx_data_fn_t + NULL, // aom_codec_enc_config_set_fn_t + NULL, // aom_codec_get_global_headers_fn_t + NULL, // aom_codec_get_preview_frame_fn_t + NULL // aom_codec_enc_mr_get_mem_loc_fn_t + } +}; diff --git a/third_party/aom/av1/av1_iface_common.h b/third_party/aom/av1/av1_iface_common.h new file mode 100644 index 0000000000..df3614212b --- /dev/null +++ b/third_party/aom/av1/av1_iface_common.h @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AV1_AV1_IFACE_COMMON_H_ +#define AV1_AV1_IFACE_COMMON_H_ + +#include "aom_ports/mem.h" + +static void yuvconfig2image(aom_image_t *img, const YV12_BUFFER_CONFIG *yv12, + void *user_priv) { + /** aom_img_wrap() doesn't allow specifying independent strides for + * the Y, U, and V planes, nor other alignment adjustments that + * might be representable by a YV12_BUFFER_CONFIG, so we just + * initialize all the fields.*/ + int bps; + if (!yv12->subsampling_y) { + if (!yv12->subsampling_x) { + img->fmt = AOM_IMG_FMT_I444; + bps = 24; + } else { + img->fmt = AOM_IMG_FMT_I422; + bps = 16; + } + } else { + if (!yv12->subsampling_x) { + img->fmt = AOM_IMG_FMT_I440; + bps = 16; + } else { + img->fmt = AOM_IMG_FMT_I420; + bps = 12; + } + } + img->cs = yv12->color_space; + img->range = yv12->color_range; + img->bit_depth = 8; + img->w = yv12->y_stride; + img->h = ALIGN_POWER_OF_TWO(yv12->y_height + 2 * AOM_BORDER_IN_PIXELS, 3); + img->d_w = yv12->y_crop_width; + img->d_h = yv12->y_crop_height; + img->r_w = yv12->render_width; + img->r_h = yv12->render_height; + img->x_chroma_shift = yv12->subsampling_x; + img->y_chroma_shift = yv12->subsampling_y; + img->planes[AOM_PLANE_Y] = yv12->y_buffer; + img->planes[AOM_PLANE_U] = yv12->u_buffer; + img->planes[AOM_PLANE_V] = yv12->v_buffer; + img->planes[AOM_PLANE_ALPHA] = NULL; + img->stride[AOM_PLANE_Y] = yv12->y_stride; + img->stride[AOM_PLANE_U] = yv12->uv_stride; + img->stride[AOM_PLANE_V] = yv12->uv_stride; + img->stride[AOM_PLANE_ALPHA] = yv12->y_stride; +#if CONFIG_HIGHBITDEPTH + if (yv12->flags & YV12_FLAG_HIGHBITDEPTH) { + // aom_image_t uses byte strides and a pointer to the first byte + // of the image. + img->fmt = (aom_img_fmt_t)(img->fmt | AOM_IMG_FMT_HIGHBITDEPTH); + img->bit_depth = yv12->bit_depth; + img->planes[AOM_PLANE_Y] = (uint8_t *)CONVERT_TO_SHORTPTR(yv12->y_buffer); + img->planes[AOM_PLANE_U] = (uint8_t *)CONVERT_TO_SHORTPTR(yv12->u_buffer); + img->planes[AOM_PLANE_V] = (uint8_t *)CONVERT_TO_SHORTPTR(yv12->v_buffer); + img->planes[AOM_PLANE_ALPHA] = NULL; + img->stride[AOM_PLANE_Y] = 2 * yv12->y_stride; + img->stride[AOM_PLANE_U] = 2 * yv12->uv_stride; + img->stride[AOM_PLANE_V] = 2 * yv12->uv_stride; + img->stride[AOM_PLANE_ALPHA] = 2 * yv12->y_stride; + } +#endif // CONFIG_HIGHBITDEPTH + img->bps = bps; + img->user_priv = user_priv; + img->img_data = yv12->buffer_alloc; + img->img_data_owner = 0; + img->self_allocd = 0; +} + +static aom_codec_err_t image2yuvconfig(const aom_image_t *img, + YV12_BUFFER_CONFIG *yv12) { + yv12->y_buffer = img->planes[AOM_PLANE_Y]; + yv12->u_buffer = img->planes[AOM_PLANE_U]; + yv12->v_buffer = img->planes[AOM_PLANE_V]; + + yv12->y_crop_width = img->d_w; + yv12->y_crop_height = img->d_h; + yv12->render_width = img->r_w; + yv12->render_height = img->r_h; + yv12->y_width = img->d_w; + yv12->y_height = img->d_h; + + yv12->uv_width = + img->x_chroma_shift == 1 ? (1 + yv12->y_width) / 2 : yv12->y_width; + yv12->uv_height = + img->y_chroma_shift == 1 ? (1 + yv12->y_height) / 2 : yv12->y_height; + yv12->uv_crop_width = yv12->uv_width; + yv12->uv_crop_height = yv12->uv_height; + + yv12->y_stride = img->stride[AOM_PLANE_Y]; + yv12->uv_stride = img->stride[AOM_PLANE_U]; + yv12->color_space = img->cs; + yv12->color_range = img->range; + +#if CONFIG_HIGHBITDEPTH + if (img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) { + // In aom_image_t + // planes point to uint8 address of start of data + // stride counts uint8s to reach next row + // In YV12_BUFFER_CONFIG + // y_buffer, u_buffer, v_buffer point to uint16 address of data + // stride and border counts in uint16s + // This means that all the address calculations in the main body of code + // should work correctly. + // However, before we do any pixel operations we need to cast the address + // to a uint16 ponter and double its value. + yv12->y_buffer = CONVERT_TO_BYTEPTR(yv12->y_buffer); + yv12->u_buffer = CONVERT_TO_BYTEPTR(yv12->u_buffer); + yv12->v_buffer = CONVERT_TO_BYTEPTR(yv12->v_buffer); + yv12->y_stride >>= 1; + yv12->uv_stride >>= 1; + yv12->flags = YV12_FLAG_HIGHBITDEPTH; + } else { + yv12->flags = 0; + } + yv12->border = (yv12->y_stride - img->w) / 2; +#else + yv12->border = (img->stride[AOM_PLANE_Y] - img->w) / 2; +#endif // CONFIG_HIGHBITDEPTH + yv12->subsampling_x = img->x_chroma_shift; + yv12->subsampling_y = img->y_chroma_shift; + return AOM_CODEC_OK; +} + +static AOM_REFFRAME ref_frame_to_av1_reframe(aom_ref_frame_type_t frame) { + switch (frame) { + case AOM_LAST_FRAME: return AOM_LAST_FLAG; + case AOM_GOLD_FRAME: return AOM_GOLD_FLAG; + case AOM_ALTR_FRAME: return AOM_ALT_FLAG; + } + assert(0 && "Invalid Reference Frame"); + return AOM_LAST_FLAG; +} +#endif // AV1_AV1_IFACE_COMMON_H_ diff --git a/third_party/aom/av1/common/alloccommon.c b/third_party/aom/av1/common/alloccommon.c new file mode 100644 index 0000000000..79d41a9c82 --- /dev/null +++ b/third_party/aom/av1/common/alloccommon.c @@ -0,0 +1,209 @@ +/* + * + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" +#include "aom_mem/aom_mem.h" + +#include "av1/common/alloccommon.h" +#include "av1/common/blockd.h" +#include "av1/common/entropymode.h" +#include "av1/common/entropymv.h" +#include "av1/common/onyxc_int.h" + +void av1_set_mb_mi(AV1_COMMON *cm, int width, int height) { + // TODO(jingning): Fine tune the loop filter operations and bring this + // back to integer multiple of 4 for cb4x4. + const int aligned_width = ALIGN_POWER_OF_TWO(width, 3); + const int aligned_height = ALIGN_POWER_OF_TWO(height, 3); + + cm->mi_cols = aligned_width >> MI_SIZE_LOG2; + cm->mi_rows = aligned_height >> MI_SIZE_LOG2; + cm->mi_stride = calc_mi_size(cm->mi_cols); + +#if CONFIG_CB4X4 + cm->mb_cols = (cm->mi_cols + 2) >> 2; + cm->mb_rows = (cm->mi_rows + 2) >> 2; +#else + cm->mb_cols = (cm->mi_cols + 1) >> 1; + cm->mb_rows = (cm->mi_rows + 1) >> 1; +#endif + cm->MBs = cm->mb_rows * cm->mb_cols; +} + +static int alloc_seg_map(AV1_COMMON *cm, int seg_map_size) { + int i; + + for (i = 0; i < NUM_PING_PONG_BUFFERS; ++i) { + cm->seg_map_array[i] = (uint8_t *)aom_calloc(seg_map_size, 1); + if (cm->seg_map_array[i] == NULL) return 1; + } + cm->seg_map_alloc_size = seg_map_size; + + // Init the index. + cm->seg_map_idx = 0; + cm->prev_seg_map_idx = 1; + + cm->current_frame_seg_map = cm->seg_map_array[cm->seg_map_idx]; + if (!cm->frame_parallel_decode) + cm->last_frame_seg_map = cm->seg_map_array[cm->prev_seg_map_idx]; + + return 0; +} + +static void free_seg_map(AV1_COMMON *cm) { + int i; + + for (i = 0; i < NUM_PING_PONG_BUFFERS; ++i) { + aom_free(cm->seg_map_array[i]); + cm->seg_map_array[i] = NULL; + } + + cm->current_frame_seg_map = NULL; + + if (!cm->frame_parallel_decode) { + cm->last_frame_seg_map = NULL; + } +} + +void av1_free_ref_frame_buffers(BufferPool *pool) { + int i; + + for (i = 0; i < FRAME_BUFFERS; ++i) { + if (pool->frame_bufs[i].ref_count > 0 && + pool->frame_bufs[i].raw_frame_buffer.data != NULL) { + pool->release_fb_cb(pool->cb_priv, &pool->frame_bufs[i].raw_frame_buffer); + pool->frame_bufs[i].ref_count = 0; + } + aom_free(pool->frame_bufs[i].mvs); + pool->frame_bufs[i].mvs = NULL; + aom_free_frame_buffer(&pool->frame_bufs[i].buf); + } +} + +#if CONFIG_LOOP_RESTORATION +// Assumes cm->rst_info[p].restoration_tilesize is already initialized +void av1_alloc_restoration_buffers(AV1_COMMON *cm) { + int p; + av1_alloc_restoration_struct(cm, &cm->rst_info[0], cm->width, cm->height); + for (p = 1; p < MAX_MB_PLANE; ++p) + av1_alloc_restoration_struct( + cm, &cm->rst_info[p], ROUND_POWER_OF_TWO(cm->width, cm->subsampling_x), + ROUND_POWER_OF_TWO(cm->height, cm->subsampling_y)); + aom_free(cm->rst_internal.tmpbuf); + CHECK_MEM_ERROR(cm, cm->rst_internal.tmpbuf, + (int32_t *)aom_memalign(16, RESTORATION_TMPBUF_SIZE)); +} + +void av1_free_restoration_buffers(AV1_COMMON *cm) { + int p; + for (p = 0; p < MAX_MB_PLANE; ++p) + av1_free_restoration_struct(&cm->rst_info[p]); + aom_free(cm->rst_internal.tmpbuf); + cm->rst_internal.tmpbuf = NULL; +} +#endif // CONFIG_LOOP_RESTORATION + +void av1_free_context_buffers(AV1_COMMON *cm) { + int i; + cm->free_mi(cm); + free_seg_map(cm); + for (i = 0; i < MAX_MB_PLANE; i++) { + aom_free(cm->above_context[i]); + cm->above_context[i] = NULL; + } + aom_free(cm->above_seg_context); + cm->above_seg_context = NULL; +#if CONFIG_VAR_TX + aom_free(cm->above_txfm_context); + cm->above_txfm_context = NULL; +#endif +} + +int av1_alloc_context_buffers(AV1_COMMON *cm, int width, int height) { + int new_mi_size; + + av1_set_mb_mi(cm, width, height); + new_mi_size = cm->mi_stride * calc_mi_size(cm->mi_rows); + if (cm->mi_alloc_size < new_mi_size) { + cm->free_mi(cm); + if (cm->alloc_mi(cm, new_mi_size)) goto fail; + } + + if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) { + // Create the segmentation map structure and set to 0. + free_seg_map(cm); + if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail; + } + + if (cm->above_context_alloc_cols < cm->mi_cols) { + // TODO(geza.lore): These are bigger than they need to be. + // cm->tile_width would be enough but it complicates indexing a + // little elsewhere. + const int aligned_mi_cols = + ALIGN_POWER_OF_TWO(cm->mi_cols, MAX_MIB_SIZE_LOG2); + int i; + + for (i = 0; i < MAX_MB_PLANE; i++) { + aom_free(cm->above_context[i]); + cm->above_context[i] = (ENTROPY_CONTEXT *)aom_calloc( + 2 * aligned_mi_cols, sizeof(*cm->above_context[0])); + if (!cm->above_context[i]) goto fail; + } + + aom_free(cm->above_seg_context); + cm->above_seg_context = (PARTITION_CONTEXT *)aom_calloc( + aligned_mi_cols, sizeof(*cm->above_seg_context)); + if (!cm->above_seg_context) goto fail; + +#if CONFIG_VAR_TX + aom_free(cm->above_txfm_context); + cm->above_txfm_context = (TXFM_CONTEXT *)aom_calloc( + aligned_mi_cols, sizeof(*cm->above_txfm_context)); + if (!cm->above_txfm_context) goto fail; +#endif + + cm->above_context_alloc_cols = aligned_mi_cols; + } + + return 0; + +fail: + // clear the mi_* values to force a realloc on resync + av1_set_mb_mi(cm, 0, 0); + av1_free_context_buffers(cm); + return 1; +} + +void av1_remove_common(AV1_COMMON *cm) { + av1_free_context_buffers(cm); + + aom_free(cm->fc); + cm->fc = NULL; + aom_free(cm->frame_contexts); + cm->frame_contexts = NULL; +} + +void av1_init_context_buffers(AV1_COMMON *cm) { + cm->setup_mi(cm); + if (cm->last_frame_seg_map && !cm->frame_parallel_decode) + memset(cm->last_frame_seg_map, 0, cm->mi_rows * cm->mi_cols); +} + +void av1_swap_current_and_last_seg_map(AV1_COMMON *cm) { + // Swap indices. + const int tmp = cm->seg_map_idx; + cm->seg_map_idx = cm->prev_seg_map_idx; + cm->prev_seg_map_idx = tmp; + + cm->current_frame_seg_map = cm->seg_map_array[cm->seg_map_idx]; + cm->last_frame_seg_map = cm->seg_map_array[cm->prev_seg_map_idx]; +} diff --git a/third_party/aom/av1/common/alloccommon.h b/third_party/aom/av1/common/alloccommon.h new file mode 100644 index 0000000000..51863cd045 --- /dev/null +++ b/third_party/aom/av1/common/alloccommon.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_ALLOCCOMMON_H_ +#define AV1_COMMON_ALLOCCOMMON_H_ + +#define INVALID_IDX -1 // Invalid buffer index. + +#ifdef __cplusplus +extern "C" { +#endif + +struct AV1Common; +struct BufferPool; + +void av1_remove_common(struct AV1Common *cm); + +int av1_alloc_context_buffers(struct AV1Common *cm, int width, int height); +void av1_init_context_buffers(struct AV1Common *cm); +void av1_free_context_buffers(struct AV1Common *cm); + +void av1_free_ref_frame_buffers(struct BufferPool *pool); +#if CONFIG_LOOP_RESTORATION +void av1_alloc_restoration_buffers(struct AV1Common *cm); +void av1_free_restoration_buffers(struct AV1Common *cm); +#endif // CONFIG_LOOP_RESTORATION + +int av1_alloc_state_buffers(struct AV1Common *cm, int width, int height); +void av1_free_state_buffers(struct AV1Common *cm); + +void av1_set_mb_mi(struct AV1Common *cm, int width, int height); + +void av1_swap_current_and_last_seg_map(struct AV1Common *cm); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_ALLOCCOMMON_H_ diff --git a/third_party/aom/av1/common/arm/neon/iht4x4_add_neon.c b/third_party/aom/av1/common/arm/neon/iht4x4_add_neon.c new file mode 100644 index 0000000000..02572d4059 --- /dev/null +++ b/third_party/aom/av1/common/arm/neon/iht4x4_add_neon.c @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "aom_dsp/txfm_common.h" +#include "av1/common/common.h" + +static INLINE void TRANSPOSE4X4(int16x8_t *q8s16, int16x8_t *q9s16) { + int32x4_t q8s32, q9s32; + int16x4x2_t d0x2s16, d1x2s16; + int32x4x2_t q0x2s32; + + d0x2s16 = vtrn_s16(vget_low_s16(*q8s16), vget_high_s16(*q8s16)); + d1x2s16 = vtrn_s16(vget_low_s16(*q9s16), vget_high_s16(*q9s16)); + + q8s32 = vreinterpretq_s32_s16(vcombine_s16(d0x2s16.val[0], d0x2s16.val[1])); + q9s32 = vreinterpretq_s32_s16(vcombine_s16(d1x2s16.val[0], d1x2s16.val[1])); + q0x2s32 = vtrnq_s32(q8s32, q9s32); + + *q8s16 = vreinterpretq_s16_s32(q0x2s32.val[0]); + *q9s16 = vreinterpretq_s16_s32(q0x2s32.val[1]); + return; +} + +static INLINE void GENERATE_COSINE_CONSTANTS(int16x4_t *d0s16, int16x4_t *d1s16, + int16x4_t *d2s16) { + *d0s16 = vdup_n_s16((int16_t)cospi_8_64); + *d1s16 = vdup_n_s16((int16_t)cospi_16_64); + *d2s16 = vdup_n_s16((int16_t)cospi_24_64); + return; +} + +static INLINE void GENERATE_SINE_CONSTANTS(int16x4_t *d3s16, int16x4_t *d4s16, + int16x4_t *d5s16, int16x8_t *q3s16) { + *d3s16 = vdup_n_s16((int16_t)sinpi_1_9); + *d4s16 = vdup_n_s16((int16_t)sinpi_2_9); + *q3s16 = vdupq_n_s16((int16_t)sinpi_3_9); + *d5s16 = vdup_n_s16((int16_t)sinpi_4_9); + return; +} + +static INLINE void IDCT4x4_1D(int16x4_t *d0s16, int16x4_t *d1s16, + int16x4_t *d2s16, int16x8_t *q8s16, + int16x8_t *q9s16) { + int16x4_t d16s16, d17s16, d18s16, d19s16, d23s16, d24s16; + int16x4_t d26s16, d27s16, d28s16, d29s16; + int32x4_t q10s32, q13s32, q14s32, q15s32; + int16x8_t q13s16, q14s16; + + d16s16 = vget_low_s16(*q8s16); + d17s16 = vget_high_s16(*q8s16); + d18s16 = vget_low_s16(*q9s16); + d19s16 = vget_high_s16(*q9s16); + + d23s16 = vadd_s16(d16s16, d18s16); + d24s16 = vsub_s16(d16s16, d18s16); + + q15s32 = vmull_s16(d17s16, *d2s16); + q10s32 = vmull_s16(d17s16, *d0s16); + q13s32 = vmull_s16(d23s16, *d1s16); + q14s32 = vmull_s16(d24s16, *d1s16); + q15s32 = vmlsl_s16(q15s32, d19s16, *d0s16); + q10s32 = vmlal_s16(q10s32, d19s16, *d2s16); + + d26s16 = vqrshrn_n_s32(q13s32, 14); + d27s16 = vqrshrn_n_s32(q14s32, 14); + d29s16 = vqrshrn_n_s32(q15s32, 14); + d28s16 = vqrshrn_n_s32(q10s32, 14); + + q13s16 = vcombine_s16(d26s16, d27s16); + q14s16 = vcombine_s16(d28s16, d29s16); + *q8s16 = vaddq_s16(q13s16, q14s16); + *q9s16 = vsubq_s16(q13s16, q14s16); + *q9s16 = vcombine_s16(vget_high_s16(*q9s16), vget_low_s16(*q9s16)); // vswp + return; +} + +static INLINE void IADST4x4_1D(int16x4_t *d3s16, int16x4_t *d4s16, + int16x4_t *d5s16, int16x8_t *q3s16, + int16x8_t *q8s16, int16x8_t *q9s16) { + int16x4_t d6s16, d16s16, d17s16, d18s16, d19s16; + int32x4_t q8s32, q9s32, q10s32, q11s32, q12s32, q13s32, q14s32, q15s32; + + d6s16 = vget_low_s16(*q3s16); + + d16s16 = vget_low_s16(*q8s16); + d17s16 = vget_high_s16(*q8s16); + d18s16 = vget_low_s16(*q9s16); + d19s16 = vget_high_s16(*q9s16); + + q10s32 = vmull_s16(*d3s16, d16s16); + q11s32 = vmull_s16(*d4s16, d16s16); + q12s32 = vmull_s16(d6s16, d17s16); + q13s32 = vmull_s16(*d5s16, d18s16); + q14s32 = vmull_s16(*d3s16, d18s16); + q15s32 = vmovl_s16(d16s16); + q15s32 = vaddw_s16(q15s32, d19s16); + q8s32 = vmull_s16(*d4s16, d19s16); + q15s32 = vsubw_s16(q15s32, d18s16); + q9s32 = vmull_s16(*d5s16, d19s16); + + q10s32 = vaddq_s32(q10s32, q13s32); + q10s32 = vaddq_s32(q10s32, q8s32); + q11s32 = vsubq_s32(q11s32, q14s32); + q8s32 = vdupq_n_s32((int32_t)sinpi_3_9); + q11s32 = vsubq_s32(q11s32, q9s32); + q15s32 = vmulq_s32(q15s32, q8s32); + + q13s32 = vaddq_s32(q10s32, q12s32); + q10s32 = vaddq_s32(q10s32, q11s32); + q14s32 = vaddq_s32(q11s32, q12s32); + q10s32 = vsubq_s32(q10s32, q12s32); + + d16s16 = vqrshrn_n_s32(q13s32, 14); + d17s16 = vqrshrn_n_s32(q14s32, 14); + d18s16 = vqrshrn_n_s32(q15s32, 14); + d19s16 = vqrshrn_n_s32(q10s32, 14); + + *q8s16 = vcombine_s16(d16s16, d17s16); + *q9s16 = vcombine_s16(d18s16, d19s16); + return; +} + +void av1_iht4x4_16_add_neon(const tran_low_t *input, uint8_t *dest, + int dest_stride, int tx_type) { + uint8x8_t d26u8, d27u8; + int16x4_t d0s16, d1s16, d2s16, d3s16, d4s16, d5s16; + uint32x2_t d26u32, d27u32; + int16x8_t q3s16, q8s16, q9s16; + uint16x8_t q8u16, q9u16; + + d26u32 = d27u32 = vdup_n_u32(0); + + q8s16 = vld1q_s16(input); + q9s16 = vld1q_s16(input + 8); + + TRANSPOSE4X4(&q8s16, &q9s16); + + switch (tx_type) { + case 0: // idct_idct is not supported. Fall back to C + av1_iht4x4_16_add_c(input, dest, dest_stride, tx_type); + return; + break; + case 1: // iadst_idct + // generate constants + GENERATE_COSINE_CONSTANTS(&d0s16, &d1s16, &d2s16); + GENERATE_SINE_CONSTANTS(&d3s16, &d4s16, &d5s16, &q3s16); + + // first transform rows + IDCT4x4_1D(&d0s16, &d1s16, &d2s16, &q8s16, &q9s16); + + // transpose the matrix + TRANSPOSE4X4(&q8s16, &q9s16); + + // then transform columns + IADST4x4_1D(&d3s16, &d4s16, &d5s16, &q3s16, &q8s16, &q9s16); + break; + case 2: // idct_iadst + // generate constantsyy + GENERATE_COSINE_CONSTANTS(&d0s16, &d1s16, &d2s16); + GENERATE_SINE_CONSTANTS(&d3s16, &d4s16, &d5s16, &q3s16); + + // first transform rows + IADST4x4_1D(&d3s16, &d4s16, &d5s16, &q3s16, &q8s16, &q9s16); + + // transpose the matrix + TRANSPOSE4X4(&q8s16, &q9s16); + + // then transform columns + IDCT4x4_1D(&d0s16, &d1s16, &d2s16, &q8s16, &q9s16); + break; + case 3: // iadst_iadst + // generate constants + GENERATE_SINE_CONSTANTS(&d3s16, &d4s16, &d5s16, &q3s16); + + // first transform rows + IADST4x4_1D(&d3s16, &d4s16, &d5s16, &q3s16, &q8s16, &q9s16); + + // transpose the matrix + TRANSPOSE4X4(&q8s16, &q9s16); + + // then transform columns + IADST4x4_1D(&d3s16, &d4s16, &d5s16, &q3s16, &q8s16, &q9s16); + break; + default: // iadst_idct + assert(0); + break; + } + + q8s16 = vrshrq_n_s16(q8s16, 4); + q9s16 = vrshrq_n_s16(q9s16, 4); + + d26u32 = vld1_lane_u32((const uint32_t *)dest, d26u32, 0); + dest += dest_stride; + d26u32 = vld1_lane_u32((const uint32_t *)dest, d26u32, 1); + dest += dest_stride; + d27u32 = vld1_lane_u32((const uint32_t *)dest, d27u32, 0); + dest += dest_stride; + d27u32 = vld1_lane_u32((const uint32_t *)dest, d27u32, 1); + + q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16), vreinterpret_u8_u32(d26u32)); + q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16), vreinterpret_u8_u32(d27u32)); + + d26u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16)); + d27u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16)); + + vst1_lane_u32((uint32_t *)dest, vreinterpret_u32_u8(d27u8), 1); + dest -= dest_stride; + vst1_lane_u32((uint32_t *)dest, vreinterpret_u32_u8(d27u8), 0); + dest -= dest_stride; + vst1_lane_u32((uint32_t *)dest, vreinterpret_u32_u8(d26u8), 1); + dest -= dest_stride; + vst1_lane_u32((uint32_t *)dest, vreinterpret_u32_u8(d26u8), 0); + return; +} diff --git a/third_party/aom/av1/common/arm/neon/iht8x8_add_neon.c b/third_party/aom/av1/common/arm/neon/iht8x8_add_neon.c new file mode 100644 index 0000000000..86798ccf14 --- /dev/null +++ b/third_party/aom/av1/common/arm/neon/iht8x8_add_neon.c @@ -0,0 +1,593 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "aom_dsp/txfm_common.h" +#include "av1/common/common.h" + +static INLINE void TRANSPOSE8X8(int16x8_t *q8s16, int16x8_t *q9s16, + int16x8_t *q10s16, int16x8_t *q11s16, + int16x8_t *q12s16, int16x8_t *q13s16, + int16x8_t *q14s16, int16x8_t *q15s16) { + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16, d23s16; + int16x4_t d24s16, d25s16, d26s16, d27s16, d28s16, d29s16, d30s16, d31s16; + int32x4x2_t q0x2s32, q1x2s32, q2x2s32, q3x2s32; + int16x8x2_t q0x2s16, q1x2s16, q2x2s16, q3x2s16; + + d16s16 = vget_low_s16(*q8s16); + d17s16 = vget_high_s16(*q8s16); + d18s16 = vget_low_s16(*q9s16); + d19s16 = vget_high_s16(*q9s16); + d20s16 = vget_low_s16(*q10s16); + d21s16 = vget_high_s16(*q10s16); + d22s16 = vget_low_s16(*q11s16); + d23s16 = vget_high_s16(*q11s16); + d24s16 = vget_low_s16(*q12s16); + d25s16 = vget_high_s16(*q12s16); + d26s16 = vget_low_s16(*q13s16); + d27s16 = vget_high_s16(*q13s16); + d28s16 = vget_low_s16(*q14s16); + d29s16 = vget_high_s16(*q14s16); + d30s16 = vget_low_s16(*q15s16); + d31s16 = vget_high_s16(*q15s16); + + *q8s16 = vcombine_s16(d16s16, d24s16); // vswp d17, d24 + *q9s16 = vcombine_s16(d18s16, d26s16); // vswp d19, d26 + *q10s16 = vcombine_s16(d20s16, d28s16); // vswp d21, d28 + *q11s16 = vcombine_s16(d22s16, d30s16); // vswp d23, d30 + *q12s16 = vcombine_s16(d17s16, d25s16); + *q13s16 = vcombine_s16(d19s16, d27s16); + *q14s16 = vcombine_s16(d21s16, d29s16); + *q15s16 = vcombine_s16(d23s16, d31s16); + + q0x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(*q8s16), vreinterpretq_s32_s16(*q10s16)); + q1x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(*q9s16), vreinterpretq_s32_s16(*q11s16)); + q2x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(*q12s16), vreinterpretq_s32_s16(*q14s16)); + q3x2s32 = + vtrnq_s32(vreinterpretq_s32_s16(*q13s16), vreinterpretq_s32_s16(*q15s16)); + + q0x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q0x2s32.val[0]), // q8 + vreinterpretq_s16_s32(q1x2s32.val[0])); // q9 + q1x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q0x2s32.val[1]), // q10 + vreinterpretq_s16_s32(q1x2s32.val[1])); // q11 + q2x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q2x2s32.val[0]), // q12 + vreinterpretq_s16_s32(q3x2s32.val[0])); // q13 + q3x2s16 = vtrnq_s16(vreinterpretq_s16_s32(q2x2s32.val[1]), // q14 + vreinterpretq_s16_s32(q3x2s32.val[1])); // q15 + + *q8s16 = q0x2s16.val[0]; + *q9s16 = q0x2s16.val[1]; + *q10s16 = q1x2s16.val[0]; + *q11s16 = q1x2s16.val[1]; + *q12s16 = q2x2s16.val[0]; + *q13s16 = q2x2s16.val[1]; + *q14s16 = q3x2s16.val[0]; + *q15s16 = q3x2s16.val[1]; + return; +} + +static INLINE void IDCT8x8_1D(int16x8_t *q8s16, int16x8_t *q9s16, + int16x8_t *q10s16, int16x8_t *q11s16, + int16x8_t *q12s16, int16x8_t *q13s16, + int16x8_t *q14s16, int16x8_t *q15s16) { + int16x4_t d0s16, d1s16, d2s16, d3s16; + int16x4_t d8s16, d9s16, d10s16, d11s16, d12s16, d13s16, d14s16, d15s16; + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16, d23s16; + int16x4_t d24s16, d25s16, d26s16, d27s16, d28s16, d29s16, d30s16, d31s16; + int16x8_t q0s16, q1s16, q2s16, q3s16, q4s16, q5s16, q6s16, q7s16; + int32x4_t q2s32, q3s32, q5s32, q6s32, q8s32, q9s32; + int32x4_t q10s32, q11s32, q12s32, q13s32, q15s32; + + d0s16 = vdup_n_s16((int16_t)cospi_28_64); + d1s16 = vdup_n_s16((int16_t)cospi_4_64); + d2s16 = vdup_n_s16((int16_t)cospi_12_64); + d3s16 = vdup_n_s16((int16_t)cospi_20_64); + + d16s16 = vget_low_s16(*q8s16); + d17s16 = vget_high_s16(*q8s16); + d18s16 = vget_low_s16(*q9s16); + d19s16 = vget_high_s16(*q9s16); + d20s16 = vget_low_s16(*q10s16); + d21s16 = vget_high_s16(*q10s16); + d22s16 = vget_low_s16(*q11s16); + d23s16 = vget_high_s16(*q11s16); + d24s16 = vget_low_s16(*q12s16); + d25s16 = vget_high_s16(*q12s16); + d26s16 = vget_low_s16(*q13s16); + d27s16 = vget_high_s16(*q13s16); + d28s16 = vget_low_s16(*q14s16); + d29s16 = vget_high_s16(*q14s16); + d30s16 = vget_low_s16(*q15s16); + d31s16 = vget_high_s16(*q15s16); + + q2s32 = vmull_s16(d18s16, d0s16); + q3s32 = vmull_s16(d19s16, d0s16); + q5s32 = vmull_s16(d26s16, d2s16); + q6s32 = vmull_s16(d27s16, d2s16); + + q2s32 = vmlsl_s16(q2s32, d30s16, d1s16); + q3s32 = vmlsl_s16(q3s32, d31s16, d1s16); + q5s32 = vmlsl_s16(q5s32, d22s16, d3s16); + q6s32 = vmlsl_s16(q6s32, d23s16, d3s16); + + d8s16 = vqrshrn_n_s32(q2s32, 14); + d9s16 = vqrshrn_n_s32(q3s32, 14); + d10s16 = vqrshrn_n_s32(q5s32, 14); + d11s16 = vqrshrn_n_s32(q6s32, 14); + q4s16 = vcombine_s16(d8s16, d9s16); + q5s16 = vcombine_s16(d10s16, d11s16); + + q2s32 = vmull_s16(d18s16, d1s16); + q3s32 = vmull_s16(d19s16, d1s16); + q9s32 = vmull_s16(d26s16, d3s16); + q13s32 = vmull_s16(d27s16, d3s16); + + q2s32 = vmlal_s16(q2s32, d30s16, d0s16); + q3s32 = vmlal_s16(q3s32, d31s16, d0s16); + q9s32 = vmlal_s16(q9s32, d22s16, d2s16); + q13s32 = vmlal_s16(q13s32, d23s16, d2s16); + + d14s16 = vqrshrn_n_s32(q2s32, 14); + d15s16 = vqrshrn_n_s32(q3s32, 14); + d12s16 = vqrshrn_n_s32(q9s32, 14); + d13s16 = vqrshrn_n_s32(q13s32, 14); + q6s16 = vcombine_s16(d12s16, d13s16); + q7s16 = vcombine_s16(d14s16, d15s16); + + d0s16 = vdup_n_s16((int16_t)cospi_16_64); + + q2s32 = vmull_s16(d16s16, d0s16); + q3s32 = vmull_s16(d17s16, d0s16); + q13s32 = vmull_s16(d16s16, d0s16); + q15s32 = vmull_s16(d17s16, d0s16); + + q2s32 = vmlal_s16(q2s32, d24s16, d0s16); + q3s32 = vmlal_s16(q3s32, d25s16, d0s16); + q13s32 = vmlsl_s16(q13s32, d24s16, d0s16); + q15s32 = vmlsl_s16(q15s32, d25s16, d0s16); + + d0s16 = vdup_n_s16((int16_t)cospi_24_64); + d1s16 = vdup_n_s16((int16_t)cospi_8_64); + + d18s16 = vqrshrn_n_s32(q2s32, 14); + d19s16 = vqrshrn_n_s32(q3s32, 14); + d22s16 = vqrshrn_n_s32(q13s32, 14); + d23s16 = vqrshrn_n_s32(q15s32, 14); + *q9s16 = vcombine_s16(d18s16, d19s16); + *q11s16 = vcombine_s16(d22s16, d23s16); + + q2s32 = vmull_s16(d20s16, d0s16); + q3s32 = vmull_s16(d21s16, d0s16); + q8s32 = vmull_s16(d20s16, d1s16); + q12s32 = vmull_s16(d21s16, d1s16); + + q2s32 = vmlsl_s16(q2s32, d28s16, d1s16); + q3s32 = vmlsl_s16(q3s32, d29s16, d1s16); + q8s32 = vmlal_s16(q8s32, d28s16, d0s16); + q12s32 = vmlal_s16(q12s32, d29s16, d0s16); + + d26s16 = vqrshrn_n_s32(q2s32, 14); + d27s16 = vqrshrn_n_s32(q3s32, 14); + d30s16 = vqrshrn_n_s32(q8s32, 14); + d31s16 = vqrshrn_n_s32(q12s32, 14); + *q13s16 = vcombine_s16(d26s16, d27s16); + *q15s16 = vcombine_s16(d30s16, d31s16); + + q0s16 = vaddq_s16(*q9s16, *q15s16); + q1s16 = vaddq_s16(*q11s16, *q13s16); + q2s16 = vsubq_s16(*q11s16, *q13s16); + q3s16 = vsubq_s16(*q9s16, *q15s16); + + *q13s16 = vsubq_s16(q4s16, q5s16); + q4s16 = vaddq_s16(q4s16, q5s16); + *q14s16 = vsubq_s16(q7s16, q6s16); + q7s16 = vaddq_s16(q7s16, q6s16); + d26s16 = vget_low_s16(*q13s16); + d27s16 = vget_high_s16(*q13s16); + d28s16 = vget_low_s16(*q14s16); + d29s16 = vget_high_s16(*q14s16); + + d16s16 = vdup_n_s16((int16_t)cospi_16_64); + + q9s32 = vmull_s16(d28s16, d16s16); + q10s32 = vmull_s16(d29s16, d16s16); + q11s32 = vmull_s16(d28s16, d16s16); + q12s32 = vmull_s16(d29s16, d16s16); + + q9s32 = vmlsl_s16(q9s32, d26s16, d16s16); + q10s32 = vmlsl_s16(q10s32, d27s16, d16s16); + q11s32 = vmlal_s16(q11s32, d26s16, d16s16); + q12s32 = vmlal_s16(q12s32, d27s16, d16s16); + + d10s16 = vqrshrn_n_s32(q9s32, 14); + d11s16 = vqrshrn_n_s32(q10s32, 14); + d12s16 = vqrshrn_n_s32(q11s32, 14); + d13s16 = vqrshrn_n_s32(q12s32, 14); + q5s16 = vcombine_s16(d10s16, d11s16); + q6s16 = vcombine_s16(d12s16, d13s16); + + *q8s16 = vaddq_s16(q0s16, q7s16); + *q9s16 = vaddq_s16(q1s16, q6s16); + *q10s16 = vaddq_s16(q2s16, q5s16); + *q11s16 = vaddq_s16(q3s16, q4s16); + *q12s16 = vsubq_s16(q3s16, q4s16); + *q13s16 = vsubq_s16(q2s16, q5s16); + *q14s16 = vsubq_s16(q1s16, q6s16); + *q15s16 = vsubq_s16(q0s16, q7s16); + return; +} + +static INLINE void IADST8X8_1D(int16x8_t *q8s16, int16x8_t *q9s16, + int16x8_t *q10s16, int16x8_t *q11s16, + int16x8_t *q12s16, int16x8_t *q13s16, + int16x8_t *q14s16, int16x8_t *q15s16) { + int16x4_t d0s16, d1s16, d2s16, d3s16, d4s16, d5s16, d6s16, d7s16; + int16x4_t d8s16, d9s16, d10s16, d11s16, d12s16, d13s16, d14s16, d15s16; + int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16, d23s16; + int16x4_t d24s16, d25s16, d26s16, d27s16, d28s16, d29s16, d30s16, d31s16; + int16x8_t q2s16, q4s16, q5s16, q6s16; + int32x4_t q0s32, q1s32, q2s32, q3s32, q4s32, q5s32, q6s32, q7s32, q8s32; + int32x4_t q9s32, q10s32, q11s32, q12s32, q13s32, q14s32, q15s32; + + d16s16 = vget_low_s16(*q8s16); + d17s16 = vget_high_s16(*q8s16); + d18s16 = vget_low_s16(*q9s16); + d19s16 = vget_high_s16(*q9s16); + d20s16 = vget_low_s16(*q10s16); + d21s16 = vget_high_s16(*q10s16); + d22s16 = vget_low_s16(*q11s16); + d23s16 = vget_high_s16(*q11s16); + d24s16 = vget_low_s16(*q12s16); + d25s16 = vget_high_s16(*q12s16); + d26s16 = vget_low_s16(*q13s16); + d27s16 = vget_high_s16(*q13s16); + d28s16 = vget_low_s16(*q14s16); + d29s16 = vget_high_s16(*q14s16); + d30s16 = vget_low_s16(*q15s16); + d31s16 = vget_high_s16(*q15s16); + + d14s16 = vdup_n_s16((int16_t)cospi_2_64); + d15s16 = vdup_n_s16((int16_t)cospi_30_64); + + q1s32 = vmull_s16(d30s16, d14s16); + q2s32 = vmull_s16(d31s16, d14s16); + q3s32 = vmull_s16(d30s16, d15s16); + q4s32 = vmull_s16(d31s16, d15s16); + + d30s16 = vdup_n_s16((int16_t)cospi_18_64); + d31s16 = vdup_n_s16((int16_t)cospi_14_64); + + q1s32 = vmlal_s16(q1s32, d16s16, d15s16); + q2s32 = vmlal_s16(q2s32, d17s16, d15s16); + q3s32 = vmlsl_s16(q3s32, d16s16, d14s16); + q4s32 = vmlsl_s16(q4s32, d17s16, d14s16); + + q5s32 = vmull_s16(d22s16, d30s16); + q6s32 = vmull_s16(d23s16, d30s16); + q7s32 = vmull_s16(d22s16, d31s16); + q8s32 = vmull_s16(d23s16, d31s16); + + q5s32 = vmlal_s16(q5s32, d24s16, d31s16); + q6s32 = vmlal_s16(q6s32, d25s16, d31s16); + q7s32 = vmlsl_s16(q7s32, d24s16, d30s16); + q8s32 = vmlsl_s16(q8s32, d25s16, d30s16); + + q11s32 = vaddq_s32(q1s32, q5s32); + q12s32 = vaddq_s32(q2s32, q6s32); + q1s32 = vsubq_s32(q1s32, q5s32); + q2s32 = vsubq_s32(q2s32, q6s32); + + d22s16 = vqrshrn_n_s32(q11s32, 14); + d23s16 = vqrshrn_n_s32(q12s32, 14); + *q11s16 = vcombine_s16(d22s16, d23s16); + + q12s32 = vaddq_s32(q3s32, q7s32); + q15s32 = vaddq_s32(q4s32, q8s32); + q3s32 = vsubq_s32(q3s32, q7s32); + q4s32 = vsubq_s32(q4s32, q8s32); + + d2s16 = vqrshrn_n_s32(q1s32, 14); + d3s16 = vqrshrn_n_s32(q2s32, 14); + d24s16 = vqrshrn_n_s32(q12s32, 14); + d25s16 = vqrshrn_n_s32(q15s32, 14); + d6s16 = vqrshrn_n_s32(q3s32, 14); + d7s16 = vqrshrn_n_s32(q4s32, 14); + *q12s16 = vcombine_s16(d24s16, d25s16); + + d0s16 = vdup_n_s16((int16_t)cospi_10_64); + d1s16 = vdup_n_s16((int16_t)cospi_22_64); + q4s32 = vmull_s16(d26s16, d0s16); + q5s32 = vmull_s16(d27s16, d0s16); + q2s32 = vmull_s16(d26s16, d1s16); + q6s32 = vmull_s16(d27s16, d1s16); + + d30s16 = vdup_n_s16((int16_t)cospi_26_64); + d31s16 = vdup_n_s16((int16_t)cospi_6_64); + + q4s32 = vmlal_s16(q4s32, d20s16, d1s16); + q5s32 = vmlal_s16(q5s32, d21s16, d1s16); + q2s32 = vmlsl_s16(q2s32, d20s16, d0s16); + q6s32 = vmlsl_s16(q6s32, d21s16, d0s16); + + q0s32 = vmull_s16(d18s16, d30s16); + q13s32 = vmull_s16(d19s16, d30s16); + + q0s32 = vmlal_s16(q0s32, d28s16, d31s16); + q13s32 = vmlal_s16(q13s32, d29s16, d31s16); + + q10s32 = vmull_s16(d18s16, d31s16); + q9s32 = vmull_s16(d19s16, d31s16); + + q10s32 = vmlsl_s16(q10s32, d28s16, d30s16); + q9s32 = vmlsl_s16(q9s32, d29s16, d30s16); + + q14s32 = vaddq_s32(q2s32, q10s32); + q15s32 = vaddq_s32(q6s32, q9s32); + q2s32 = vsubq_s32(q2s32, q10s32); + q6s32 = vsubq_s32(q6s32, q9s32); + + d28s16 = vqrshrn_n_s32(q14s32, 14); + d29s16 = vqrshrn_n_s32(q15s32, 14); + d4s16 = vqrshrn_n_s32(q2s32, 14); + d5s16 = vqrshrn_n_s32(q6s32, 14); + *q14s16 = vcombine_s16(d28s16, d29s16); + + q9s32 = vaddq_s32(q4s32, q0s32); + q10s32 = vaddq_s32(q5s32, q13s32); + q4s32 = vsubq_s32(q4s32, q0s32); + q5s32 = vsubq_s32(q5s32, q13s32); + + d30s16 = vdup_n_s16((int16_t)cospi_8_64); + d31s16 = vdup_n_s16((int16_t)cospi_24_64); + + d18s16 = vqrshrn_n_s32(q9s32, 14); + d19s16 = vqrshrn_n_s32(q10s32, 14); + d8s16 = vqrshrn_n_s32(q4s32, 14); + d9s16 = vqrshrn_n_s32(q5s32, 14); + *q9s16 = vcombine_s16(d18s16, d19s16); + + q5s32 = vmull_s16(d2s16, d30s16); + q6s32 = vmull_s16(d3s16, d30s16); + q7s32 = vmull_s16(d2s16, d31s16); + q0s32 = vmull_s16(d3s16, d31s16); + + q5s32 = vmlal_s16(q5s32, d6s16, d31s16); + q6s32 = vmlal_s16(q6s32, d7s16, d31s16); + q7s32 = vmlsl_s16(q7s32, d6s16, d30s16); + q0s32 = vmlsl_s16(q0s32, d7s16, d30s16); + + q1s32 = vmull_s16(d4s16, d30s16); + q3s32 = vmull_s16(d5s16, d30s16); + q10s32 = vmull_s16(d4s16, d31s16); + q2s32 = vmull_s16(d5s16, d31s16); + + q1s32 = vmlsl_s16(q1s32, d8s16, d31s16); + q3s32 = vmlsl_s16(q3s32, d9s16, d31s16); + q10s32 = vmlal_s16(q10s32, d8s16, d30s16); + q2s32 = vmlal_s16(q2s32, d9s16, d30s16); + + *q8s16 = vaddq_s16(*q11s16, *q9s16); + *q11s16 = vsubq_s16(*q11s16, *q9s16); + q4s16 = vaddq_s16(*q12s16, *q14s16); + *q12s16 = vsubq_s16(*q12s16, *q14s16); + + q14s32 = vaddq_s32(q5s32, q1s32); + q15s32 = vaddq_s32(q6s32, q3s32); + q5s32 = vsubq_s32(q5s32, q1s32); + q6s32 = vsubq_s32(q6s32, q3s32); + + d18s16 = vqrshrn_n_s32(q14s32, 14); + d19s16 = vqrshrn_n_s32(q15s32, 14); + d10s16 = vqrshrn_n_s32(q5s32, 14); + d11s16 = vqrshrn_n_s32(q6s32, 14); + *q9s16 = vcombine_s16(d18s16, d19s16); + + q1s32 = vaddq_s32(q7s32, q10s32); + q3s32 = vaddq_s32(q0s32, q2s32); + q7s32 = vsubq_s32(q7s32, q10s32); + q0s32 = vsubq_s32(q0s32, q2s32); + + d28s16 = vqrshrn_n_s32(q1s32, 14); + d29s16 = vqrshrn_n_s32(q3s32, 14); + d14s16 = vqrshrn_n_s32(q7s32, 14); + d15s16 = vqrshrn_n_s32(q0s32, 14); + *q14s16 = vcombine_s16(d28s16, d29s16); + + d30s16 = vdup_n_s16((int16_t)cospi_16_64); + + d22s16 = vget_low_s16(*q11s16); + d23s16 = vget_high_s16(*q11s16); + q2s32 = vmull_s16(d22s16, d30s16); + q3s32 = vmull_s16(d23s16, d30s16); + q13s32 = vmull_s16(d22s16, d30s16); + q1s32 = vmull_s16(d23s16, d30s16); + + d24s16 = vget_low_s16(*q12s16); + d25s16 = vget_high_s16(*q12s16); + q2s32 = vmlal_s16(q2s32, d24s16, d30s16); + q3s32 = vmlal_s16(q3s32, d25s16, d30s16); + q13s32 = vmlsl_s16(q13s32, d24s16, d30s16); + q1s32 = vmlsl_s16(q1s32, d25s16, d30s16); + + d4s16 = vqrshrn_n_s32(q2s32, 14); + d5s16 = vqrshrn_n_s32(q3s32, 14); + d24s16 = vqrshrn_n_s32(q13s32, 14); + d25s16 = vqrshrn_n_s32(q1s32, 14); + q2s16 = vcombine_s16(d4s16, d5s16); + *q12s16 = vcombine_s16(d24s16, d25s16); + + q13s32 = vmull_s16(d10s16, d30s16); + q1s32 = vmull_s16(d11s16, d30s16); + q11s32 = vmull_s16(d10s16, d30s16); + q0s32 = vmull_s16(d11s16, d30s16); + + q13s32 = vmlal_s16(q13s32, d14s16, d30s16); + q1s32 = vmlal_s16(q1s32, d15s16, d30s16); + q11s32 = vmlsl_s16(q11s32, d14s16, d30s16); + q0s32 = vmlsl_s16(q0s32, d15s16, d30s16); + + d20s16 = vqrshrn_n_s32(q13s32, 14); + d21s16 = vqrshrn_n_s32(q1s32, 14); + d12s16 = vqrshrn_n_s32(q11s32, 14); + d13s16 = vqrshrn_n_s32(q0s32, 14); + *q10s16 = vcombine_s16(d20s16, d21s16); + q6s16 = vcombine_s16(d12s16, d13s16); + + q5s16 = vdupq_n_s16(0); + + *q9s16 = vsubq_s16(q5s16, *q9s16); + *q11s16 = vsubq_s16(q5s16, q2s16); + *q13s16 = vsubq_s16(q5s16, q6s16); + *q15s16 = vsubq_s16(q5s16, q4s16); + return; +} + +void av1_iht8x8_64_add_neon(const tran_low_t *input, uint8_t *dest, + int dest_stride, int tx_type) { + int i; + uint8_t *d1, *d2; + uint8x8_t d0u8, d1u8, d2u8, d3u8; + uint64x1_t d0u64, d1u64, d2u64, d3u64; + int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16, q13s16, q14s16, q15s16; + uint16x8_t q8u16, q9u16, q10u16, q11u16; + + q8s16 = vld1q_s16(input); + q9s16 = vld1q_s16(input + 8); + q10s16 = vld1q_s16(input + 8 * 2); + q11s16 = vld1q_s16(input + 8 * 3); + q12s16 = vld1q_s16(input + 8 * 4); + q13s16 = vld1q_s16(input + 8 * 5); + q14s16 = vld1q_s16(input + 8 * 6); + q15s16 = vld1q_s16(input + 8 * 7); + + TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + switch (tx_type) { + case 0: // idct_idct is not supported. Fall back to C + av1_iht8x8_64_add_c(input, dest, dest_stride, tx_type); + return; + break; + case 1: // iadst_idct + // generate IDCT constants + // GENERATE_IDCT_CONSTANTS + + // first transform rows + IDCT8x8_1D(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + // transpose the matrix + TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + // generate IADST constants + // GENERATE_IADST_CONSTANTS + + // then transform columns + IADST8X8_1D(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + break; + case 2: // idct_iadst + // generate IADST constants + // GENERATE_IADST_CONSTANTS + + // first transform rows + IADST8X8_1D(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + // transpose the matrix + TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + // generate IDCT constants + // GENERATE_IDCT_CONSTANTS + + // then transform columns + IDCT8x8_1D(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + break; + case 3: // iadst_iadst + // generate IADST constants + // GENERATE_IADST_CONSTANTS + + // first transform rows + IADST8X8_1D(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + // transpose the matrix + TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + + // then transform columns + IADST8X8_1D(&q8s16, &q9s16, &q10s16, &q11s16, &q12s16, &q13s16, &q14s16, + &q15s16); + break; + default: // iadst_idct + assert(0); + break; + } + + q8s16 = vrshrq_n_s16(q8s16, 5); + q9s16 = vrshrq_n_s16(q9s16, 5); + q10s16 = vrshrq_n_s16(q10s16, 5); + q11s16 = vrshrq_n_s16(q11s16, 5); + q12s16 = vrshrq_n_s16(q12s16, 5); + q13s16 = vrshrq_n_s16(q13s16, 5); + q14s16 = vrshrq_n_s16(q14s16, 5); + q15s16 = vrshrq_n_s16(q15s16, 5); + + for (d1 = d2 = dest, i = 0; i < 2; i++) { + if (i != 0) { + q8s16 = q12s16; + q9s16 = q13s16; + q10s16 = q14s16; + q11s16 = q15s16; + } + + d0u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d1u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d2u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + d3u64 = vld1_u64((uint64_t *)d1); + d1 += dest_stride; + + q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16), vreinterpret_u8_u64(d0u64)); + q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16), vreinterpret_u8_u64(d1u64)); + q10u16 = + vaddw_u8(vreinterpretq_u16_s16(q10s16), vreinterpret_u8_u64(d2u64)); + q11u16 = + vaddw_u8(vreinterpretq_u16_s16(q11s16), vreinterpret_u8_u64(d3u64)); + + d0u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16)); + d1u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16)); + d2u8 = vqmovun_s16(vreinterpretq_s16_u16(q10u16)); + d3u8 = vqmovun_s16(vreinterpretq_s16_u16(q11u16)); + + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d0u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d1u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d2u8)); + d2 += dest_stride; + vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d3u8)); + d2 += dest_stride; + } + return; +} diff --git a/third_party/aom/av1/common/av1_fwd_txfm1d.c b/third_party/aom/av1/common/av1_fwd_txfm1d.c new file mode 100644 index 0000000000..7a691e03f5 --- /dev/null +++ b/third_party/aom/av1/common/av1_fwd_txfm1d.c @@ -0,0 +1,2312 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "av1/common/av1_fwd_txfm1d.h" +#if CONFIG_COEFFICIENT_RANGE_CHECKING + +void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf, + int32_t size, int8_t bit); + +#define range_check(stage, input, buf, size, bit) \ + range_check_func(stage, input, buf, size, bit) +#else +#define range_check(stage, input, buf, size, bit) \ + { \ + (void)stage; \ + (void)input; \ + (void)buf; \ + (void)size; \ + (void)bit; \ + } +#endif + +// TODO(angiebird): Make 1-d txfm functions static +void av1_fdct4_new(const int32_t *input, int32_t *output, const int8_t *cos_bit, + const int8_t *stage_range) { + const int32_t size = 4; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[4]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[0] + input[3]; + bf1[1] = input[1] + input[2]; + bf1[2] = -input[2] + input[1]; + bf1[3] = -input[3] + input[0]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[32], bf0[0], cospi[32], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(-cospi[32], bf0[1], cospi[32], bf0[0], cos_bit[stage]); + bf1[2] = half_btf(cospi[48], bf0[2], cospi[16], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[48], bf0[3], -cospi[16], bf0[2], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[2]; + bf1[2] = bf0[1]; + bf1[3] = bf0[3]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_fdct8_new(const int32_t *input, int32_t *output, const int8_t *cos_bit, + const int8_t *stage_range) { + const int32_t size = 8; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[8]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[0] + input[7]; + bf1[1] = input[1] + input[6]; + bf1[2] = input[2] + input[5]; + bf1[3] = input[3] + input[4]; + bf1[4] = -input[4] + input[3]; + bf1[5] = -input[5] + input[2]; + bf1[6] = -input[6] + input[1]; + bf1[7] = -input[7] + input[0]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0] + bf0[3]; + bf1[1] = bf0[1] + bf0[2]; + bf1[2] = -bf0[2] + bf0[1]; + bf1[3] = -bf0[3] + bf0[0]; + bf1[4] = bf0[4]; + bf1[5] = half_btf(-cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[32], bf0[6], cospi[32], bf0[5], cos_bit[stage]); + bf1[7] = bf0[7]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = half_btf(cospi[32], bf0[0], cospi[32], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(-cospi[32], bf0[1], cospi[32], bf0[0], cos_bit[stage]); + bf1[2] = half_btf(cospi[48], bf0[2], cospi[16], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[48], bf0[3], -cospi[16], bf0[2], cos_bit[stage]); + bf1[4] = bf0[4] + bf0[5]; + bf1[5] = -bf0[5] + bf0[4]; + bf1[6] = -bf0[6] + bf0[7]; + bf1[7] = bf0[7] + bf0[6]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[56], bf0[4], cospi[8], bf0[7], cos_bit[stage]); + bf1[5] = half_btf(cospi[24], bf0[5], cospi[40], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[24], bf0[6], -cospi[40], bf0[5], cos_bit[stage]); + bf1[7] = half_btf(cospi[56], bf0[7], -cospi[8], bf0[4], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[4]; + bf1[2] = bf0[2]; + bf1[3] = bf0[6]; + bf1[4] = bf0[1]; + bf1[5] = bf0[5]; + bf1[6] = bf0[3]; + bf1[7] = bf0[7]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_fdct16_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 16; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[16]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[0] + input[15]; + bf1[1] = input[1] + input[14]; + bf1[2] = input[2] + input[13]; + bf1[3] = input[3] + input[12]; + bf1[4] = input[4] + input[11]; + bf1[5] = input[5] + input[10]; + bf1[6] = input[6] + input[9]; + bf1[7] = input[7] + input[8]; + bf1[8] = -input[8] + input[7]; + bf1[9] = -input[9] + input[6]; + bf1[10] = -input[10] + input[5]; + bf1[11] = -input[11] + input[4]; + bf1[12] = -input[12] + input[3]; + bf1[13] = -input[13] + input[2]; + bf1[14] = -input[14] + input[1]; + bf1[15] = -input[15] + input[0]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0] + bf0[7]; + bf1[1] = bf0[1] + bf0[6]; + bf1[2] = bf0[2] + bf0[5]; + bf1[3] = bf0[3] + bf0[4]; + bf1[4] = -bf0[4] + bf0[3]; + bf1[5] = -bf0[5] + bf0[2]; + bf1[6] = -bf0[6] + bf0[1]; + bf1[7] = -bf0[7] + bf0[0]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = half_btf(-cospi[32], bf0[10], cospi[32], bf0[13], cos_bit[stage]); + bf1[11] = half_btf(-cospi[32], bf0[11], cospi[32], bf0[12], cos_bit[stage]); + bf1[12] = half_btf(cospi[32], bf0[12], cospi[32], bf0[11], cos_bit[stage]); + bf1[13] = half_btf(cospi[32], bf0[13], cospi[32], bf0[10], cos_bit[stage]); + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[3]; + bf1[1] = bf0[1] + bf0[2]; + bf1[2] = -bf0[2] + bf0[1]; + bf1[3] = -bf0[3] + bf0[0]; + bf1[4] = bf0[4]; + bf1[5] = half_btf(-cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[32], bf0[6], cospi[32], bf0[5], cos_bit[stage]); + bf1[7] = bf0[7]; + bf1[8] = bf0[8] + bf0[11]; + bf1[9] = bf0[9] + bf0[10]; + bf1[10] = -bf0[10] + bf0[9]; + bf1[11] = -bf0[11] + bf0[8]; + bf1[12] = -bf0[12] + bf0[15]; + bf1[13] = -bf0[13] + bf0[14]; + bf1[14] = bf0[14] + bf0[13]; + bf1[15] = bf0[15] + bf0[12]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[32], bf0[0], cospi[32], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(-cospi[32], bf0[1], cospi[32], bf0[0], cos_bit[stage]); + bf1[2] = half_btf(cospi[48], bf0[2], cospi[16], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[48], bf0[3], -cospi[16], bf0[2], cos_bit[stage]); + bf1[4] = bf0[4] + bf0[5]; + bf1[5] = -bf0[5] + bf0[4]; + bf1[6] = -bf0[6] + bf0[7]; + bf1[7] = bf0[7] + bf0[6]; + bf1[8] = bf0[8]; + bf1[9] = half_btf(-cospi[16], bf0[9], cospi[48], bf0[14], cos_bit[stage]); + bf1[10] = half_btf(-cospi[48], bf0[10], -cospi[16], bf0[13], cos_bit[stage]); + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = half_btf(cospi[48], bf0[13], -cospi[16], bf0[10], cos_bit[stage]); + bf1[14] = half_btf(cospi[16], bf0[14], cospi[48], bf0[9], cos_bit[stage]); + bf1[15] = bf0[15]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[56], bf0[4], cospi[8], bf0[7], cos_bit[stage]); + bf1[5] = half_btf(cospi[24], bf0[5], cospi[40], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[24], bf0[6], -cospi[40], bf0[5], cos_bit[stage]); + bf1[7] = half_btf(cospi[56], bf0[7], -cospi[8], bf0[4], cos_bit[stage]); + bf1[8] = bf0[8] + bf0[9]; + bf1[9] = -bf0[9] + bf0[8]; + bf1[10] = -bf0[10] + bf0[11]; + bf1[11] = bf0[11] + bf0[10]; + bf1[12] = bf0[12] + bf0[13]; + bf1[13] = -bf0[13] + bf0[12]; + bf1[14] = -bf0[14] + bf0[15]; + bf1[15] = bf0[15] + bf0[14]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 6 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = half_btf(cospi[60], bf0[8], cospi[4], bf0[15], cos_bit[stage]); + bf1[9] = half_btf(cospi[28], bf0[9], cospi[36], bf0[14], cos_bit[stage]); + bf1[10] = half_btf(cospi[44], bf0[10], cospi[20], bf0[13], cos_bit[stage]); + bf1[11] = half_btf(cospi[12], bf0[11], cospi[52], bf0[12], cos_bit[stage]); + bf1[12] = half_btf(cospi[12], bf0[12], -cospi[52], bf0[11], cos_bit[stage]); + bf1[13] = half_btf(cospi[44], bf0[13], -cospi[20], bf0[10], cos_bit[stage]); + bf1[14] = half_btf(cospi[28], bf0[14], -cospi[36], bf0[9], cos_bit[stage]); + bf1[15] = half_btf(cospi[60], bf0[15], -cospi[4], bf0[8], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 7 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[8]; + bf1[2] = bf0[4]; + bf1[3] = bf0[12]; + bf1[4] = bf0[2]; + bf1[5] = bf0[10]; + bf1[6] = bf0[6]; + bf1[7] = bf0[14]; + bf1[8] = bf0[1]; + bf1[9] = bf0[9]; + bf1[10] = bf0[5]; + bf1[11] = bf0[13]; + bf1[12] = bf0[3]; + bf1[13] = bf0[11]; + bf1[14] = bf0[7]; + bf1[15] = bf0[15]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_fdct32_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 32; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[32]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[0] + input[31]; + bf1[1] = input[1] + input[30]; + bf1[2] = input[2] + input[29]; + bf1[3] = input[3] + input[28]; + bf1[4] = input[4] + input[27]; + bf1[5] = input[5] + input[26]; + bf1[6] = input[6] + input[25]; + bf1[7] = input[7] + input[24]; + bf1[8] = input[8] + input[23]; + bf1[9] = input[9] + input[22]; + bf1[10] = input[10] + input[21]; + bf1[11] = input[11] + input[20]; + bf1[12] = input[12] + input[19]; + bf1[13] = input[13] + input[18]; + bf1[14] = input[14] + input[17]; + bf1[15] = input[15] + input[16]; + bf1[16] = -input[16] + input[15]; + bf1[17] = -input[17] + input[14]; + bf1[18] = -input[18] + input[13]; + bf1[19] = -input[19] + input[12]; + bf1[20] = -input[20] + input[11]; + bf1[21] = -input[21] + input[10]; + bf1[22] = -input[22] + input[9]; + bf1[23] = -input[23] + input[8]; + bf1[24] = -input[24] + input[7]; + bf1[25] = -input[25] + input[6]; + bf1[26] = -input[26] + input[5]; + bf1[27] = -input[27] + input[4]; + bf1[28] = -input[28] + input[3]; + bf1[29] = -input[29] + input[2]; + bf1[30] = -input[30] + input[1]; + bf1[31] = -input[31] + input[0]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0] + bf0[15]; + bf1[1] = bf0[1] + bf0[14]; + bf1[2] = bf0[2] + bf0[13]; + bf1[3] = bf0[3] + bf0[12]; + bf1[4] = bf0[4] + bf0[11]; + bf1[5] = bf0[5] + bf0[10]; + bf1[6] = bf0[6] + bf0[9]; + bf1[7] = bf0[7] + bf0[8]; + bf1[8] = -bf0[8] + bf0[7]; + bf1[9] = -bf0[9] + bf0[6]; + bf1[10] = -bf0[10] + bf0[5]; + bf1[11] = -bf0[11] + bf0[4]; + bf1[12] = -bf0[12] + bf0[3]; + bf1[13] = -bf0[13] + bf0[2]; + bf1[14] = -bf0[14] + bf0[1]; + bf1[15] = -bf0[15] + bf0[0]; + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = bf0[18]; + bf1[19] = bf0[19]; + bf1[20] = half_btf(-cospi[32], bf0[20], cospi[32], bf0[27], cos_bit[stage]); + bf1[21] = half_btf(-cospi[32], bf0[21], cospi[32], bf0[26], cos_bit[stage]); + bf1[22] = half_btf(-cospi[32], bf0[22], cospi[32], bf0[25], cos_bit[stage]); + bf1[23] = half_btf(-cospi[32], bf0[23], cospi[32], bf0[24], cos_bit[stage]); + bf1[24] = half_btf(cospi[32], bf0[24], cospi[32], bf0[23], cos_bit[stage]); + bf1[25] = half_btf(cospi[32], bf0[25], cospi[32], bf0[22], cos_bit[stage]); + bf1[26] = half_btf(cospi[32], bf0[26], cospi[32], bf0[21], cos_bit[stage]); + bf1[27] = half_btf(cospi[32], bf0[27], cospi[32], bf0[20], cos_bit[stage]); + bf1[28] = bf0[28]; + bf1[29] = bf0[29]; + bf1[30] = bf0[30]; + bf1[31] = bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[7]; + bf1[1] = bf0[1] + bf0[6]; + bf1[2] = bf0[2] + bf0[5]; + bf1[3] = bf0[3] + bf0[4]; + bf1[4] = -bf0[4] + bf0[3]; + bf1[5] = -bf0[5] + bf0[2]; + bf1[6] = -bf0[6] + bf0[1]; + bf1[7] = -bf0[7] + bf0[0]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = half_btf(-cospi[32], bf0[10], cospi[32], bf0[13], cos_bit[stage]); + bf1[11] = half_btf(-cospi[32], bf0[11], cospi[32], bf0[12], cos_bit[stage]); + bf1[12] = half_btf(cospi[32], bf0[12], cospi[32], bf0[11], cos_bit[stage]); + bf1[13] = half_btf(cospi[32], bf0[13], cospi[32], bf0[10], cos_bit[stage]); + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = bf0[16] + bf0[23]; + bf1[17] = bf0[17] + bf0[22]; + bf1[18] = bf0[18] + bf0[21]; + bf1[19] = bf0[19] + bf0[20]; + bf1[20] = -bf0[20] + bf0[19]; + bf1[21] = -bf0[21] + bf0[18]; + bf1[22] = -bf0[22] + bf0[17]; + bf1[23] = -bf0[23] + bf0[16]; + bf1[24] = -bf0[24] + bf0[31]; + bf1[25] = -bf0[25] + bf0[30]; + bf1[26] = -bf0[26] + bf0[29]; + bf1[27] = -bf0[27] + bf0[28]; + bf1[28] = bf0[28] + bf0[27]; + bf1[29] = bf0[29] + bf0[26]; + bf1[30] = bf0[30] + bf0[25]; + bf1[31] = bf0[31] + bf0[24]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0] + bf0[3]; + bf1[1] = bf0[1] + bf0[2]; + bf1[2] = -bf0[2] + bf0[1]; + bf1[3] = -bf0[3] + bf0[0]; + bf1[4] = bf0[4]; + bf1[5] = half_btf(-cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[32], bf0[6], cospi[32], bf0[5], cos_bit[stage]); + bf1[7] = bf0[7]; + bf1[8] = bf0[8] + bf0[11]; + bf1[9] = bf0[9] + bf0[10]; + bf1[10] = -bf0[10] + bf0[9]; + bf1[11] = -bf0[11] + bf0[8]; + bf1[12] = -bf0[12] + bf0[15]; + bf1[13] = -bf0[13] + bf0[14]; + bf1[14] = bf0[14] + bf0[13]; + bf1[15] = bf0[15] + bf0[12]; + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = half_btf(-cospi[16], bf0[18], cospi[48], bf0[29], cos_bit[stage]); + bf1[19] = half_btf(-cospi[16], bf0[19], cospi[48], bf0[28], cos_bit[stage]); + bf1[20] = half_btf(-cospi[48], bf0[20], -cospi[16], bf0[27], cos_bit[stage]); + bf1[21] = half_btf(-cospi[48], bf0[21], -cospi[16], bf0[26], cos_bit[stage]); + bf1[22] = bf0[22]; + bf1[23] = bf0[23]; + bf1[24] = bf0[24]; + bf1[25] = bf0[25]; + bf1[26] = half_btf(cospi[48], bf0[26], -cospi[16], bf0[21], cos_bit[stage]); + bf1[27] = half_btf(cospi[48], bf0[27], -cospi[16], bf0[20], cos_bit[stage]); + bf1[28] = half_btf(cospi[16], bf0[28], cospi[48], bf0[19], cos_bit[stage]); + bf1[29] = half_btf(cospi[16], bf0[29], cospi[48], bf0[18], cos_bit[stage]); + bf1[30] = bf0[30]; + bf1[31] = bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = half_btf(cospi[32], bf0[0], cospi[32], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(-cospi[32], bf0[1], cospi[32], bf0[0], cos_bit[stage]); + bf1[2] = half_btf(cospi[48], bf0[2], cospi[16], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[48], bf0[3], -cospi[16], bf0[2], cos_bit[stage]); + bf1[4] = bf0[4] + bf0[5]; + bf1[5] = -bf0[5] + bf0[4]; + bf1[6] = -bf0[6] + bf0[7]; + bf1[7] = bf0[7] + bf0[6]; + bf1[8] = bf0[8]; + bf1[9] = half_btf(-cospi[16], bf0[9], cospi[48], bf0[14], cos_bit[stage]); + bf1[10] = half_btf(-cospi[48], bf0[10], -cospi[16], bf0[13], cos_bit[stage]); + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = half_btf(cospi[48], bf0[13], -cospi[16], bf0[10], cos_bit[stage]); + bf1[14] = half_btf(cospi[16], bf0[14], cospi[48], bf0[9], cos_bit[stage]); + bf1[15] = bf0[15]; + bf1[16] = bf0[16] + bf0[19]; + bf1[17] = bf0[17] + bf0[18]; + bf1[18] = -bf0[18] + bf0[17]; + bf1[19] = -bf0[19] + bf0[16]; + bf1[20] = -bf0[20] + bf0[23]; + bf1[21] = -bf0[21] + bf0[22]; + bf1[22] = bf0[22] + bf0[21]; + bf1[23] = bf0[23] + bf0[20]; + bf1[24] = bf0[24] + bf0[27]; + bf1[25] = bf0[25] + bf0[26]; + bf1[26] = -bf0[26] + bf0[25]; + bf1[27] = -bf0[27] + bf0[24]; + bf1[28] = -bf0[28] + bf0[31]; + bf1[29] = -bf0[29] + bf0[30]; + bf1[30] = bf0[30] + bf0[29]; + bf1[31] = bf0[31] + bf0[28]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 6 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[56], bf0[4], cospi[8], bf0[7], cos_bit[stage]); + bf1[5] = half_btf(cospi[24], bf0[5], cospi[40], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[24], bf0[6], -cospi[40], bf0[5], cos_bit[stage]); + bf1[7] = half_btf(cospi[56], bf0[7], -cospi[8], bf0[4], cos_bit[stage]); + bf1[8] = bf0[8] + bf0[9]; + bf1[9] = -bf0[9] + bf0[8]; + bf1[10] = -bf0[10] + bf0[11]; + bf1[11] = bf0[11] + bf0[10]; + bf1[12] = bf0[12] + bf0[13]; + bf1[13] = -bf0[13] + bf0[12]; + bf1[14] = -bf0[14] + bf0[15]; + bf1[15] = bf0[15] + bf0[14]; + bf1[16] = bf0[16]; + bf1[17] = half_btf(-cospi[8], bf0[17], cospi[56], bf0[30], cos_bit[stage]); + bf1[18] = half_btf(-cospi[56], bf0[18], -cospi[8], bf0[29], cos_bit[stage]); + bf1[19] = bf0[19]; + bf1[20] = bf0[20]; + bf1[21] = half_btf(-cospi[40], bf0[21], cospi[24], bf0[26], cos_bit[stage]); + bf1[22] = half_btf(-cospi[24], bf0[22], -cospi[40], bf0[25], cos_bit[stage]); + bf1[23] = bf0[23]; + bf1[24] = bf0[24]; + bf1[25] = half_btf(cospi[24], bf0[25], -cospi[40], bf0[22], cos_bit[stage]); + bf1[26] = half_btf(cospi[40], bf0[26], cospi[24], bf0[21], cos_bit[stage]); + bf1[27] = bf0[27]; + bf1[28] = bf0[28]; + bf1[29] = half_btf(cospi[56], bf0[29], -cospi[8], bf0[18], cos_bit[stage]); + bf1[30] = half_btf(cospi[8], bf0[30], cospi[56], bf0[17], cos_bit[stage]); + bf1[31] = bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 7 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = half_btf(cospi[60], bf0[8], cospi[4], bf0[15], cos_bit[stage]); + bf1[9] = half_btf(cospi[28], bf0[9], cospi[36], bf0[14], cos_bit[stage]); + bf1[10] = half_btf(cospi[44], bf0[10], cospi[20], bf0[13], cos_bit[stage]); + bf1[11] = half_btf(cospi[12], bf0[11], cospi[52], bf0[12], cos_bit[stage]); + bf1[12] = half_btf(cospi[12], bf0[12], -cospi[52], bf0[11], cos_bit[stage]); + bf1[13] = half_btf(cospi[44], bf0[13], -cospi[20], bf0[10], cos_bit[stage]); + bf1[14] = half_btf(cospi[28], bf0[14], -cospi[36], bf0[9], cos_bit[stage]); + bf1[15] = half_btf(cospi[60], bf0[15], -cospi[4], bf0[8], cos_bit[stage]); + bf1[16] = bf0[16] + bf0[17]; + bf1[17] = -bf0[17] + bf0[16]; + bf1[18] = -bf0[18] + bf0[19]; + bf1[19] = bf0[19] + bf0[18]; + bf1[20] = bf0[20] + bf0[21]; + bf1[21] = -bf0[21] + bf0[20]; + bf1[22] = -bf0[22] + bf0[23]; + bf1[23] = bf0[23] + bf0[22]; + bf1[24] = bf0[24] + bf0[25]; + bf1[25] = -bf0[25] + bf0[24]; + bf1[26] = -bf0[26] + bf0[27]; + bf1[27] = bf0[27] + bf0[26]; + bf1[28] = bf0[28] + bf0[29]; + bf1[29] = -bf0[29] + bf0[28]; + bf1[30] = -bf0[30] + bf0[31]; + bf1[31] = bf0[31] + bf0[30]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 8 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = bf0[10]; + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = bf0[13]; + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = half_btf(cospi[62], bf0[16], cospi[2], bf0[31], cos_bit[stage]); + bf1[17] = half_btf(cospi[30], bf0[17], cospi[34], bf0[30], cos_bit[stage]); + bf1[18] = half_btf(cospi[46], bf0[18], cospi[18], bf0[29], cos_bit[stage]); + bf1[19] = half_btf(cospi[14], bf0[19], cospi[50], bf0[28], cos_bit[stage]); + bf1[20] = half_btf(cospi[54], bf0[20], cospi[10], bf0[27], cos_bit[stage]); + bf1[21] = half_btf(cospi[22], bf0[21], cospi[42], bf0[26], cos_bit[stage]); + bf1[22] = half_btf(cospi[38], bf0[22], cospi[26], bf0[25], cos_bit[stage]); + bf1[23] = half_btf(cospi[6], bf0[23], cospi[58], bf0[24], cos_bit[stage]); + bf1[24] = half_btf(cospi[6], bf0[24], -cospi[58], bf0[23], cos_bit[stage]); + bf1[25] = half_btf(cospi[38], bf0[25], -cospi[26], bf0[22], cos_bit[stage]); + bf1[26] = half_btf(cospi[22], bf0[26], -cospi[42], bf0[21], cos_bit[stage]); + bf1[27] = half_btf(cospi[54], bf0[27], -cospi[10], bf0[20], cos_bit[stage]); + bf1[28] = half_btf(cospi[14], bf0[28], -cospi[50], bf0[19], cos_bit[stage]); + bf1[29] = half_btf(cospi[46], bf0[29], -cospi[18], bf0[18], cos_bit[stage]); + bf1[30] = half_btf(cospi[30], bf0[30], -cospi[34], bf0[17], cos_bit[stage]); + bf1[31] = half_btf(cospi[62], bf0[31], -cospi[2], bf0[16], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 9 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[16]; + bf1[2] = bf0[8]; + bf1[3] = bf0[24]; + bf1[4] = bf0[4]; + bf1[5] = bf0[20]; + bf1[6] = bf0[12]; + bf1[7] = bf0[28]; + bf1[8] = bf0[2]; + bf1[9] = bf0[18]; + bf1[10] = bf0[10]; + bf1[11] = bf0[26]; + bf1[12] = bf0[6]; + bf1[13] = bf0[22]; + bf1[14] = bf0[14]; + bf1[15] = bf0[30]; + bf1[16] = bf0[1]; + bf1[17] = bf0[17]; + bf1[18] = bf0[9]; + bf1[19] = bf0[25]; + bf1[20] = bf0[5]; + bf1[21] = bf0[21]; + bf1[22] = bf0[13]; + bf1[23] = bf0[29]; + bf1[24] = bf0[3]; + bf1[25] = bf0[19]; + bf1[26] = bf0[11]; + bf1[27] = bf0[27]; + bf1[28] = bf0[7]; + bf1[29] = bf0[23]; + bf1[30] = bf0[15]; + bf1[31] = bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_fadst4_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 4; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[4]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[3]; + bf1[1] = input[0]; + bf1[2] = input[1]; + bf1[3] = input[2]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[8], bf0[0], cospi[56], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(-cospi[8], bf0[1], cospi[56], bf0[0], cos_bit[stage]); + bf1[2] = half_btf(cospi[40], bf0[2], cospi[24], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(-cospi[40], bf0[3], cospi[24], bf0[2], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[2]; + bf1[1] = bf0[1] + bf0[3]; + bf1[2] = -bf0[2] + bf0[0]; + bf1[3] = -bf0[3] + bf0[1]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = half_btf(cospi[32], bf0[2], cospi[32], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(-cospi[32], bf0[3], cospi[32], bf0[2], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = -bf0[2]; + bf1[2] = bf0[3]; + bf1[3] = -bf0[1]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_fadst8_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 8; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[8]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[7]; + bf1[1] = input[0]; + bf1[2] = input[5]; + bf1[3] = input[2]; + bf1[4] = input[3]; + bf1[5] = input[4]; + bf1[6] = input[1]; + bf1[7] = input[6]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[4], bf0[0], cospi[60], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(-cospi[4], bf0[1], cospi[60], bf0[0], cos_bit[stage]); + bf1[2] = half_btf(cospi[20], bf0[2], cospi[44], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(-cospi[20], bf0[3], cospi[44], bf0[2], cos_bit[stage]); + bf1[4] = half_btf(cospi[36], bf0[4], cospi[28], bf0[5], cos_bit[stage]); + bf1[5] = half_btf(-cospi[36], bf0[5], cospi[28], bf0[4], cos_bit[stage]); + bf1[6] = half_btf(cospi[52], bf0[6], cospi[12], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(-cospi[52], bf0[7], cospi[12], bf0[6], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[4]; + bf1[1] = bf0[1] + bf0[5]; + bf1[2] = bf0[2] + bf0[6]; + bf1[3] = bf0[3] + bf0[7]; + bf1[4] = -bf0[4] + bf0[0]; + bf1[5] = -bf0[5] + bf0[1]; + bf1[6] = -bf0[6] + bf0[2]; + bf1[7] = -bf0[7] + bf0[3]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[16], bf0[4], cospi[48], bf0[5], cos_bit[stage]); + bf1[5] = half_btf(-cospi[16], bf0[5], cospi[48], bf0[4], cos_bit[stage]); + bf1[6] = half_btf(-cospi[48], bf0[6], cospi[16], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(cospi[48], bf0[7], cospi[16], bf0[6], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[2]; + bf1[1] = bf0[1] + bf0[3]; + bf1[2] = -bf0[2] + bf0[0]; + bf1[3] = -bf0[3] + bf0[1]; + bf1[4] = bf0[4] + bf0[6]; + bf1[5] = bf0[5] + bf0[7]; + bf1[6] = -bf0[6] + bf0[4]; + bf1[7] = -bf0[7] + bf0[5]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 6 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = half_btf(cospi[32], bf0[2], cospi[32], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(-cospi[32], bf0[3], cospi[32], bf0[2], cos_bit[stage]); + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = half_btf(cospi[32], bf0[6], cospi[32], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(-cospi[32], bf0[7], cospi[32], bf0[6], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 7 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = -bf0[4]; + bf1[2] = bf0[6]; + bf1[3] = -bf0[2]; + bf1[4] = bf0[3]; + bf1[5] = -bf0[7]; + bf1[6] = bf0[5]; + bf1[7] = -bf0[1]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_fadst16_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 16; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[16]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[15]; + bf1[1] = input[0]; + bf1[2] = input[13]; + bf1[3] = input[2]; + bf1[4] = input[11]; + bf1[5] = input[4]; + bf1[6] = input[9]; + bf1[7] = input[6]; + bf1[8] = input[7]; + bf1[9] = input[8]; + bf1[10] = input[5]; + bf1[11] = input[10]; + bf1[12] = input[3]; + bf1[13] = input[12]; + bf1[14] = input[1]; + bf1[15] = input[14]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[2], bf0[0], cospi[62], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(-cospi[2], bf0[1], cospi[62], bf0[0], cos_bit[stage]); + bf1[2] = half_btf(cospi[10], bf0[2], cospi[54], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(-cospi[10], bf0[3], cospi[54], bf0[2], cos_bit[stage]); + bf1[4] = half_btf(cospi[18], bf0[4], cospi[46], bf0[5], cos_bit[stage]); + bf1[5] = half_btf(-cospi[18], bf0[5], cospi[46], bf0[4], cos_bit[stage]); + bf1[6] = half_btf(cospi[26], bf0[6], cospi[38], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(-cospi[26], bf0[7], cospi[38], bf0[6], cos_bit[stage]); + bf1[8] = half_btf(cospi[34], bf0[8], cospi[30], bf0[9], cos_bit[stage]); + bf1[9] = half_btf(-cospi[34], bf0[9], cospi[30], bf0[8], cos_bit[stage]); + bf1[10] = half_btf(cospi[42], bf0[10], cospi[22], bf0[11], cos_bit[stage]); + bf1[11] = half_btf(-cospi[42], bf0[11], cospi[22], bf0[10], cos_bit[stage]); + bf1[12] = half_btf(cospi[50], bf0[12], cospi[14], bf0[13], cos_bit[stage]); + bf1[13] = half_btf(-cospi[50], bf0[13], cospi[14], bf0[12], cos_bit[stage]); + bf1[14] = half_btf(cospi[58], bf0[14], cospi[6], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(-cospi[58], bf0[15], cospi[6], bf0[14], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[8]; + bf1[1] = bf0[1] + bf0[9]; + bf1[2] = bf0[2] + bf0[10]; + bf1[3] = bf0[3] + bf0[11]; + bf1[4] = bf0[4] + bf0[12]; + bf1[5] = bf0[5] + bf0[13]; + bf1[6] = bf0[6] + bf0[14]; + bf1[7] = bf0[7] + bf0[15]; + bf1[8] = -bf0[8] + bf0[0]; + bf1[9] = -bf0[9] + bf0[1]; + bf1[10] = -bf0[10] + bf0[2]; + bf1[11] = -bf0[11] + bf0[3]; + bf1[12] = -bf0[12] + bf0[4]; + bf1[13] = -bf0[13] + bf0[5]; + bf1[14] = -bf0[14] + bf0[6]; + bf1[15] = -bf0[15] + bf0[7]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = half_btf(cospi[8], bf0[8], cospi[56], bf0[9], cos_bit[stage]); + bf1[9] = half_btf(-cospi[8], bf0[9], cospi[56], bf0[8], cos_bit[stage]); + bf1[10] = half_btf(cospi[40], bf0[10], cospi[24], bf0[11], cos_bit[stage]); + bf1[11] = half_btf(-cospi[40], bf0[11], cospi[24], bf0[10], cos_bit[stage]); + bf1[12] = half_btf(-cospi[56], bf0[12], cospi[8], bf0[13], cos_bit[stage]); + bf1[13] = half_btf(cospi[56], bf0[13], cospi[8], bf0[12], cos_bit[stage]); + bf1[14] = half_btf(-cospi[24], bf0[14], cospi[40], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(cospi[24], bf0[15], cospi[40], bf0[14], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[4]; + bf1[1] = bf0[1] + bf0[5]; + bf1[2] = bf0[2] + bf0[6]; + bf1[3] = bf0[3] + bf0[7]; + bf1[4] = -bf0[4] + bf0[0]; + bf1[5] = -bf0[5] + bf0[1]; + bf1[6] = -bf0[6] + bf0[2]; + bf1[7] = -bf0[7] + bf0[3]; + bf1[8] = bf0[8] + bf0[12]; + bf1[9] = bf0[9] + bf0[13]; + bf1[10] = bf0[10] + bf0[14]; + bf1[11] = bf0[11] + bf0[15]; + bf1[12] = -bf0[12] + bf0[8]; + bf1[13] = -bf0[13] + bf0[9]; + bf1[14] = -bf0[14] + bf0[10]; + bf1[15] = -bf0[15] + bf0[11]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 6 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[16], bf0[4], cospi[48], bf0[5], cos_bit[stage]); + bf1[5] = half_btf(-cospi[16], bf0[5], cospi[48], bf0[4], cos_bit[stage]); + bf1[6] = half_btf(-cospi[48], bf0[6], cospi[16], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(cospi[48], bf0[7], cospi[16], bf0[6], cos_bit[stage]); + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = bf0[10]; + bf1[11] = bf0[11]; + bf1[12] = half_btf(cospi[16], bf0[12], cospi[48], bf0[13], cos_bit[stage]); + bf1[13] = half_btf(-cospi[16], bf0[13], cospi[48], bf0[12], cos_bit[stage]); + bf1[14] = half_btf(-cospi[48], bf0[14], cospi[16], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(cospi[48], bf0[15], cospi[16], bf0[14], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 7 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[2]; + bf1[1] = bf0[1] + bf0[3]; + bf1[2] = -bf0[2] + bf0[0]; + bf1[3] = -bf0[3] + bf0[1]; + bf1[4] = bf0[4] + bf0[6]; + bf1[5] = bf0[5] + bf0[7]; + bf1[6] = -bf0[6] + bf0[4]; + bf1[7] = -bf0[7] + bf0[5]; + bf1[8] = bf0[8] + bf0[10]; + bf1[9] = bf0[9] + bf0[11]; + bf1[10] = -bf0[10] + bf0[8]; + bf1[11] = -bf0[11] + bf0[9]; + bf1[12] = bf0[12] + bf0[14]; + bf1[13] = bf0[13] + bf0[15]; + bf1[14] = -bf0[14] + bf0[12]; + bf1[15] = -bf0[15] + bf0[13]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 8 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = half_btf(cospi[32], bf0[2], cospi[32], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(-cospi[32], bf0[3], cospi[32], bf0[2], cos_bit[stage]); + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = half_btf(cospi[32], bf0[6], cospi[32], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(-cospi[32], bf0[7], cospi[32], bf0[6], cos_bit[stage]); + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = half_btf(cospi[32], bf0[10], cospi[32], bf0[11], cos_bit[stage]); + bf1[11] = half_btf(-cospi[32], bf0[11], cospi[32], bf0[10], cos_bit[stage]); + bf1[12] = bf0[12]; + bf1[13] = bf0[13]; + bf1[14] = half_btf(cospi[32], bf0[14], cospi[32], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(-cospi[32], bf0[15], cospi[32], bf0[14], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 9 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = -bf0[8]; + bf1[2] = bf0[12]; + bf1[3] = -bf0[4]; + bf1[4] = bf0[6]; + bf1[5] = -bf0[14]; + bf1[6] = bf0[10]; + bf1[7] = -bf0[2]; + bf1[8] = bf0[3]; + bf1[9] = -bf0[11]; + bf1[10] = bf0[15]; + bf1[11] = -bf0[7]; + bf1[12] = bf0[5]; + bf1[13] = -bf0[13]; + bf1[14] = bf0[9]; + bf1[15] = -bf0[1]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_fadst32_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 32; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[32]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[31]; + bf1[1] = input[0]; + bf1[2] = input[29]; + bf1[3] = input[2]; + bf1[4] = input[27]; + bf1[5] = input[4]; + bf1[6] = input[25]; + bf1[7] = input[6]; + bf1[8] = input[23]; + bf1[9] = input[8]; + bf1[10] = input[21]; + bf1[11] = input[10]; + bf1[12] = input[19]; + bf1[13] = input[12]; + bf1[14] = input[17]; + bf1[15] = input[14]; + bf1[16] = input[15]; + bf1[17] = input[16]; + bf1[18] = input[13]; + bf1[19] = input[18]; + bf1[20] = input[11]; + bf1[21] = input[20]; + bf1[22] = input[9]; + bf1[23] = input[22]; + bf1[24] = input[7]; + bf1[25] = input[24]; + bf1[26] = input[5]; + bf1[27] = input[26]; + bf1[28] = input[3]; + bf1[29] = input[28]; + bf1[30] = input[1]; + bf1[31] = input[30]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[1], bf0[0], cospi[63], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(-cospi[1], bf0[1], cospi[63], bf0[0], cos_bit[stage]); + bf1[2] = half_btf(cospi[5], bf0[2], cospi[59], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(-cospi[5], bf0[3], cospi[59], bf0[2], cos_bit[stage]); + bf1[4] = half_btf(cospi[9], bf0[4], cospi[55], bf0[5], cos_bit[stage]); + bf1[5] = half_btf(-cospi[9], bf0[5], cospi[55], bf0[4], cos_bit[stage]); + bf1[6] = half_btf(cospi[13], bf0[6], cospi[51], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(-cospi[13], bf0[7], cospi[51], bf0[6], cos_bit[stage]); + bf1[8] = half_btf(cospi[17], bf0[8], cospi[47], bf0[9], cos_bit[stage]); + bf1[9] = half_btf(-cospi[17], bf0[9], cospi[47], bf0[8], cos_bit[stage]); + bf1[10] = half_btf(cospi[21], bf0[10], cospi[43], bf0[11], cos_bit[stage]); + bf1[11] = half_btf(-cospi[21], bf0[11], cospi[43], bf0[10], cos_bit[stage]); + bf1[12] = half_btf(cospi[25], bf0[12], cospi[39], bf0[13], cos_bit[stage]); + bf1[13] = half_btf(-cospi[25], bf0[13], cospi[39], bf0[12], cos_bit[stage]); + bf1[14] = half_btf(cospi[29], bf0[14], cospi[35], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(-cospi[29], bf0[15], cospi[35], bf0[14], cos_bit[stage]); + bf1[16] = half_btf(cospi[33], bf0[16], cospi[31], bf0[17], cos_bit[stage]); + bf1[17] = half_btf(-cospi[33], bf0[17], cospi[31], bf0[16], cos_bit[stage]); + bf1[18] = half_btf(cospi[37], bf0[18], cospi[27], bf0[19], cos_bit[stage]); + bf1[19] = half_btf(-cospi[37], bf0[19], cospi[27], bf0[18], cos_bit[stage]); + bf1[20] = half_btf(cospi[41], bf0[20], cospi[23], bf0[21], cos_bit[stage]); + bf1[21] = half_btf(-cospi[41], bf0[21], cospi[23], bf0[20], cos_bit[stage]); + bf1[22] = half_btf(cospi[45], bf0[22], cospi[19], bf0[23], cos_bit[stage]); + bf1[23] = half_btf(-cospi[45], bf0[23], cospi[19], bf0[22], cos_bit[stage]); + bf1[24] = half_btf(cospi[49], bf0[24], cospi[15], bf0[25], cos_bit[stage]); + bf1[25] = half_btf(-cospi[49], bf0[25], cospi[15], bf0[24], cos_bit[stage]); + bf1[26] = half_btf(cospi[53], bf0[26], cospi[11], bf0[27], cos_bit[stage]); + bf1[27] = half_btf(-cospi[53], bf0[27], cospi[11], bf0[26], cos_bit[stage]); + bf1[28] = half_btf(cospi[57], bf0[28], cospi[7], bf0[29], cos_bit[stage]); + bf1[29] = half_btf(-cospi[57], bf0[29], cospi[7], bf0[28], cos_bit[stage]); + bf1[30] = half_btf(cospi[61], bf0[30], cospi[3], bf0[31], cos_bit[stage]); + bf1[31] = half_btf(-cospi[61], bf0[31], cospi[3], bf0[30], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[16]; + bf1[1] = bf0[1] + bf0[17]; + bf1[2] = bf0[2] + bf0[18]; + bf1[3] = bf0[3] + bf0[19]; + bf1[4] = bf0[4] + bf0[20]; + bf1[5] = bf0[5] + bf0[21]; + bf1[6] = bf0[6] + bf0[22]; + bf1[7] = bf0[7] + bf0[23]; + bf1[8] = bf0[8] + bf0[24]; + bf1[9] = bf0[9] + bf0[25]; + bf1[10] = bf0[10] + bf0[26]; + bf1[11] = bf0[11] + bf0[27]; + bf1[12] = bf0[12] + bf0[28]; + bf1[13] = bf0[13] + bf0[29]; + bf1[14] = bf0[14] + bf0[30]; + bf1[15] = bf0[15] + bf0[31]; + bf1[16] = -bf0[16] + bf0[0]; + bf1[17] = -bf0[17] + bf0[1]; + bf1[18] = -bf0[18] + bf0[2]; + bf1[19] = -bf0[19] + bf0[3]; + bf1[20] = -bf0[20] + bf0[4]; + bf1[21] = -bf0[21] + bf0[5]; + bf1[22] = -bf0[22] + bf0[6]; + bf1[23] = -bf0[23] + bf0[7]; + bf1[24] = -bf0[24] + bf0[8]; + bf1[25] = -bf0[25] + bf0[9]; + bf1[26] = -bf0[26] + bf0[10]; + bf1[27] = -bf0[27] + bf0[11]; + bf1[28] = -bf0[28] + bf0[12]; + bf1[29] = -bf0[29] + bf0[13]; + bf1[30] = -bf0[30] + bf0[14]; + bf1[31] = -bf0[31] + bf0[15]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = bf0[10]; + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = bf0[13]; + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = half_btf(cospi[4], bf0[16], cospi[60], bf0[17], cos_bit[stage]); + bf1[17] = half_btf(-cospi[4], bf0[17], cospi[60], bf0[16], cos_bit[stage]); + bf1[18] = half_btf(cospi[20], bf0[18], cospi[44], bf0[19], cos_bit[stage]); + bf1[19] = half_btf(-cospi[20], bf0[19], cospi[44], bf0[18], cos_bit[stage]); + bf1[20] = half_btf(cospi[36], bf0[20], cospi[28], bf0[21], cos_bit[stage]); + bf1[21] = half_btf(-cospi[36], bf0[21], cospi[28], bf0[20], cos_bit[stage]); + bf1[22] = half_btf(cospi[52], bf0[22], cospi[12], bf0[23], cos_bit[stage]); + bf1[23] = half_btf(-cospi[52], bf0[23], cospi[12], bf0[22], cos_bit[stage]); + bf1[24] = half_btf(-cospi[60], bf0[24], cospi[4], bf0[25], cos_bit[stage]); + bf1[25] = half_btf(cospi[60], bf0[25], cospi[4], bf0[24], cos_bit[stage]); + bf1[26] = half_btf(-cospi[44], bf0[26], cospi[20], bf0[27], cos_bit[stage]); + bf1[27] = half_btf(cospi[44], bf0[27], cospi[20], bf0[26], cos_bit[stage]); + bf1[28] = half_btf(-cospi[28], bf0[28], cospi[36], bf0[29], cos_bit[stage]); + bf1[29] = half_btf(cospi[28], bf0[29], cospi[36], bf0[28], cos_bit[stage]); + bf1[30] = half_btf(-cospi[12], bf0[30], cospi[52], bf0[31], cos_bit[stage]); + bf1[31] = half_btf(cospi[12], bf0[31], cospi[52], bf0[30], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[8]; + bf1[1] = bf0[1] + bf0[9]; + bf1[2] = bf0[2] + bf0[10]; + bf1[3] = bf0[3] + bf0[11]; + bf1[4] = bf0[4] + bf0[12]; + bf1[5] = bf0[5] + bf0[13]; + bf1[6] = bf0[6] + bf0[14]; + bf1[7] = bf0[7] + bf0[15]; + bf1[8] = -bf0[8] + bf0[0]; + bf1[9] = -bf0[9] + bf0[1]; + bf1[10] = -bf0[10] + bf0[2]; + bf1[11] = -bf0[11] + bf0[3]; + bf1[12] = -bf0[12] + bf0[4]; + bf1[13] = -bf0[13] + bf0[5]; + bf1[14] = -bf0[14] + bf0[6]; + bf1[15] = -bf0[15] + bf0[7]; + bf1[16] = bf0[16] + bf0[24]; + bf1[17] = bf0[17] + bf0[25]; + bf1[18] = bf0[18] + bf0[26]; + bf1[19] = bf0[19] + bf0[27]; + bf1[20] = bf0[20] + bf0[28]; + bf1[21] = bf0[21] + bf0[29]; + bf1[22] = bf0[22] + bf0[30]; + bf1[23] = bf0[23] + bf0[31]; + bf1[24] = -bf0[24] + bf0[16]; + bf1[25] = -bf0[25] + bf0[17]; + bf1[26] = -bf0[26] + bf0[18]; + bf1[27] = -bf0[27] + bf0[19]; + bf1[28] = -bf0[28] + bf0[20]; + bf1[29] = -bf0[29] + bf0[21]; + bf1[30] = -bf0[30] + bf0[22]; + bf1[31] = -bf0[31] + bf0[23]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 6 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = half_btf(cospi[8], bf0[8], cospi[56], bf0[9], cos_bit[stage]); + bf1[9] = half_btf(-cospi[8], bf0[9], cospi[56], bf0[8], cos_bit[stage]); + bf1[10] = half_btf(cospi[40], bf0[10], cospi[24], bf0[11], cos_bit[stage]); + bf1[11] = half_btf(-cospi[40], bf0[11], cospi[24], bf0[10], cos_bit[stage]); + bf1[12] = half_btf(-cospi[56], bf0[12], cospi[8], bf0[13], cos_bit[stage]); + bf1[13] = half_btf(cospi[56], bf0[13], cospi[8], bf0[12], cos_bit[stage]); + bf1[14] = half_btf(-cospi[24], bf0[14], cospi[40], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(cospi[24], bf0[15], cospi[40], bf0[14], cos_bit[stage]); + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = bf0[18]; + bf1[19] = bf0[19]; + bf1[20] = bf0[20]; + bf1[21] = bf0[21]; + bf1[22] = bf0[22]; + bf1[23] = bf0[23]; + bf1[24] = half_btf(cospi[8], bf0[24], cospi[56], bf0[25], cos_bit[stage]); + bf1[25] = half_btf(-cospi[8], bf0[25], cospi[56], bf0[24], cos_bit[stage]); + bf1[26] = half_btf(cospi[40], bf0[26], cospi[24], bf0[27], cos_bit[stage]); + bf1[27] = half_btf(-cospi[40], bf0[27], cospi[24], bf0[26], cos_bit[stage]); + bf1[28] = half_btf(-cospi[56], bf0[28], cospi[8], bf0[29], cos_bit[stage]); + bf1[29] = half_btf(cospi[56], bf0[29], cospi[8], bf0[28], cos_bit[stage]); + bf1[30] = half_btf(-cospi[24], bf0[30], cospi[40], bf0[31], cos_bit[stage]); + bf1[31] = half_btf(cospi[24], bf0[31], cospi[40], bf0[30], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 7 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[4]; + bf1[1] = bf0[1] + bf0[5]; + bf1[2] = bf0[2] + bf0[6]; + bf1[3] = bf0[3] + bf0[7]; + bf1[4] = -bf0[4] + bf0[0]; + bf1[5] = -bf0[5] + bf0[1]; + bf1[6] = -bf0[6] + bf0[2]; + bf1[7] = -bf0[7] + bf0[3]; + bf1[8] = bf0[8] + bf0[12]; + bf1[9] = bf0[9] + bf0[13]; + bf1[10] = bf0[10] + bf0[14]; + bf1[11] = bf0[11] + bf0[15]; + bf1[12] = -bf0[12] + bf0[8]; + bf1[13] = -bf0[13] + bf0[9]; + bf1[14] = -bf0[14] + bf0[10]; + bf1[15] = -bf0[15] + bf0[11]; + bf1[16] = bf0[16] + bf0[20]; + bf1[17] = bf0[17] + bf0[21]; + bf1[18] = bf0[18] + bf0[22]; + bf1[19] = bf0[19] + bf0[23]; + bf1[20] = -bf0[20] + bf0[16]; + bf1[21] = -bf0[21] + bf0[17]; + bf1[22] = -bf0[22] + bf0[18]; + bf1[23] = -bf0[23] + bf0[19]; + bf1[24] = bf0[24] + bf0[28]; + bf1[25] = bf0[25] + bf0[29]; + bf1[26] = bf0[26] + bf0[30]; + bf1[27] = bf0[27] + bf0[31]; + bf1[28] = -bf0[28] + bf0[24]; + bf1[29] = -bf0[29] + bf0[25]; + bf1[30] = -bf0[30] + bf0[26]; + bf1[31] = -bf0[31] + bf0[27]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 8 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[16], bf0[4], cospi[48], bf0[5], cos_bit[stage]); + bf1[5] = half_btf(-cospi[16], bf0[5], cospi[48], bf0[4], cos_bit[stage]); + bf1[6] = half_btf(-cospi[48], bf0[6], cospi[16], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(cospi[48], bf0[7], cospi[16], bf0[6], cos_bit[stage]); + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = bf0[10]; + bf1[11] = bf0[11]; + bf1[12] = half_btf(cospi[16], bf0[12], cospi[48], bf0[13], cos_bit[stage]); + bf1[13] = half_btf(-cospi[16], bf0[13], cospi[48], bf0[12], cos_bit[stage]); + bf1[14] = half_btf(-cospi[48], bf0[14], cospi[16], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(cospi[48], bf0[15], cospi[16], bf0[14], cos_bit[stage]); + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = bf0[18]; + bf1[19] = bf0[19]; + bf1[20] = half_btf(cospi[16], bf0[20], cospi[48], bf0[21], cos_bit[stage]); + bf1[21] = half_btf(-cospi[16], bf0[21], cospi[48], bf0[20], cos_bit[stage]); + bf1[22] = half_btf(-cospi[48], bf0[22], cospi[16], bf0[23], cos_bit[stage]); + bf1[23] = half_btf(cospi[48], bf0[23], cospi[16], bf0[22], cos_bit[stage]); + bf1[24] = bf0[24]; + bf1[25] = bf0[25]; + bf1[26] = bf0[26]; + bf1[27] = bf0[27]; + bf1[28] = half_btf(cospi[16], bf0[28], cospi[48], bf0[29], cos_bit[stage]); + bf1[29] = half_btf(-cospi[16], bf0[29], cospi[48], bf0[28], cos_bit[stage]); + bf1[30] = half_btf(-cospi[48], bf0[30], cospi[16], bf0[31], cos_bit[stage]); + bf1[31] = half_btf(cospi[48], bf0[31], cospi[16], bf0[30], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 9 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[2]; + bf1[1] = bf0[1] + bf0[3]; + bf1[2] = -bf0[2] + bf0[0]; + bf1[3] = -bf0[3] + bf0[1]; + bf1[4] = bf0[4] + bf0[6]; + bf1[5] = bf0[5] + bf0[7]; + bf1[6] = -bf0[6] + bf0[4]; + bf1[7] = -bf0[7] + bf0[5]; + bf1[8] = bf0[8] + bf0[10]; + bf1[9] = bf0[9] + bf0[11]; + bf1[10] = -bf0[10] + bf0[8]; + bf1[11] = -bf0[11] + bf0[9]; + bf1[12] = bf0[12] + bf0[14]; + bf1[13] = bf0[13] + bf0[15]; + bf1[14] = -bf0[14] + bf0[12]; + bf1[15] = -bf0[15] + bf0[13]; + bf1[16] = bf0[16] + bf0[18]; + bf1[17] = bf0[17] + bf0[19]; + bf1[18] = -bf0[18] + bf0[16]; + bf1[19] = -bf0[19] + bf0[17]; + bf1[20] = bf0[20] + bf0[22]; + bf1[21] = bf0[21] + bf0[23]; + bf1[22] = -bf0[22] + bf0[20]; + bf1[23] = -bf0[23] + bf0[21]; + bf1[24] = bf0[24] + bf0[26]; + bf1[25] = bf0[25] + bf0[27]; + bf1[26] = -bf0[26] + bf0[24]; + bf1[27] = -bf0[27] + bf0[25]; + bf1[28] = bf0[28] + bf0[30]; + bf1[29] = bf0[29] + bf0[31]; + bf1[30] = -bf0[30] + bf0[28]; + bf1[31] = -bf0[31] + bf0[29]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 10 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = half_btf(cospi[32], bf0[2], cospi[32], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(-cospi[32], bf0[3], cospi[32], bf0[2], cos_bit[stage]); + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = half_btf(cospi[32], bf0[6], cospi[32], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(-cospi[32], bf0[7], cospi[32], bf0[6], cos_bit[stage]); + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = half_btf(cospi[32], bf0[10], cospi[32], bf0[11], cos_bit[stage]); + bf1[11] = half_btf(-cospi[32], bf0[11], cospi[32], bf0[10], cos_bit[stage]); + bf1[12] = bf0[12]; + bf1[13] = bf0[13]; + bf1[14] = half_btf(cospi[32], bf0[14], cospi[32], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(-cospi[32], bf0[15], cospi[32], bf0[14], cos_bit[stage]); + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = half_btf(cospi[32], bf0[18], cospi[32], bf0[19], cos_bit[stage]); + bf1[19] = half_btf(-cospi[32], bf0[19], cospi[32], bf0[18], cos_bit[stage]); + bf1[20] = bf0[20]; + bf1[21] = bf0[21]; + bf1[22] = half_btf(cospi[32], bf0[22], cospi[32], bf0[23], cos_bit[stage]); + bf1[23] = half_btf(-cospi[32], bf0[23], cospi[32], bf0[22], cos_bit[stage]); + bf1[24] = bf0[24]; + bf1[25] = bf0[25]; + bf1[26] = half_btf(cospi[32], bf0[26], cospi[32], bf0[27], cos_bit[stage]); + bf1[27] = half_btf(-cospi[32], bf0[27], cospi[32], bf0[26], cos_bit[stage]); + bf1[28] = bf0[28]; + bf1[29] = bf0[29]; + bf1[30] = half_btf(cospi[32], bf0[30], cospi[32], bf0[31], cos_bit[stage]); + bf1[31] = half_btf(-cospi[32], bf0[31], cospi[32], bf0[30], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 11 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = -bf0[16]; + bf1[2] = bf0[24]; + bf1[3] = -bf0[8]; + bf1[4] = bf0[12]; + bf1[5] = -bf0[28]; + bf1[6] = bf0[20]; + bf1[7] = -bf0[4]; + bf1[8] = bf0[6]; + bf1[9] = -bf0[22]; + bf1[10] = bf0[30]; + bf1[11] = -bf0[14]; + bf1[12] = bf0[10]; + bf1[13] = -bf0[26]; + bf1[14] = bf0[18]; + bf1[15] = -bf0[2]; + bf1[16] = bf0[3]; + bf1[17] = -bf0[19]; + bf1[18] = bf0[27]; + bf1[19] = -bf0[11]; + bf1[20] = bf0[15]; + bf1[21] = -bf0[31]; + bf1[22] = bf0[23]; + bf1[23] = -bf0[7]; + bf1[24] = bf0[5]; + bf1[25] = -bf0[21]; + bf1[26] = bf0[29]; + bf1[27] = -bf0[13]; + bf1[28] = bf0[9]; + bf1[29] = -bf0[25]; + bf1[30] = bf0[17]; + bf1[31] = -bf0[1]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +#if CONFIG_TX64X64 +void av1_fdct64_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 64; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[64]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf1 = output; + bf1[0] = input[0] + input[63]; + bf1[1] = input[1] + input[62]; + bf1[2] = input[2] + input[61]; + bf1[3] = input[3] + input[60]; + bf1[4] = input[4] + input[59]; + bf1[5] = input[5] + input[58]; + bf1[6] = input[6] + input[57]; + bf1[7] = input[7] + input[56]; + bf1[8] = input[8] + input[55]; + bf1[9] = input[9] + input[54]; + bf1[10] = input[10] + input[53]; + bf1[11] = input[11] + input[52]; + bf1[12] = input[12] + input[51]; + bf1[13] = input[13] + input[50]; + bf1[14] = input[14] + input[49]; + bf1[15] = input[15] + input[48]; + bf1[16] = input[16] + input[47]; + bf1[17] = input[17] + input[46]; + bf1[18] = input[18] + input[45]; + bf1[19] = input[19] + input[44]; + bf1[20] = input[20] + input[43]; + bf1[21] = input[21] + input[42]; + bf1[22] = input[22] + input[41]; + bf1[23] = input[23] + input[40]; + bf1[24] = input[24] + input[39]; + bf1[25] = input[25] + input[38]; + bf1[26] = input[26] + input[37]; + bf1[27] = input[27] + input[36]; + bf1[28] = input[28] + input[35]; + bf1[29] = input[29] + input[34]; + bf1[30] = input[30] + input[33]; + bf1[31] = input[31] + input[32]; + bf1[32] = -input[32] + input[31]; + bf1[33] = -input[33] + input[30]; + bf1[34] = -input[34] + input[29]; + bf1[35] = -input[35] + input[28]; + bf1[36] = -input[36] + input[27]; + bf1[37] = -input[37] + input[26]; + bf1[38] = -input[38] + input[25]; + bf1[39] = -input[39] + input[24]; + bf1[40] = -input[40] + input[23]; + bf1[41] = -input[41] + input[22]; + bf1[42] = -input[42] + input[21]; + bf1[43] = -input[43] + input[20]; + bf1[44] = -input[44] + input[19]; + bf1[45] = -input[45] + input[18]; + bf1[46] = -input[46] + input[17]; + bf1[47] = -input[47] + input[16]; + bf1[48] = -input[48] + input[15]; + bf1[49] = -input[49] + input[14]; + bf1[50] = -input[50] + input[13]; + bf1[51] = -input[51] + input[12]; + bf1[52] = -input[52] + input[11]; + bf1[53] = -input[53] + input[10]; + bf1[54] = -input[54] + input[9]; + bf1[55] = -input[55] + input[8]; + bf1[56] = -input[56] + input[7]; + bf1[57] = -input[57] + input[6]; + bf1[58] = -input[58] + input[5]; + bf1[59] = -input[59] + input[4]; + bf1[60] = -input[60] + input[3]; + bf1[61] = -input[61] + input[2]; + bf1[62] = -input[62] + input[1]; + bf1[63] = -input[63] + input[0]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0] + bf0[31]; + bf1[1] = bf0[1] + bf0[30]; + bf1[2] = bf0[2] + bf0[29]; + bf1[3] = bf0[3] + bf0[28]; + bf1[4] = bf0[4] + bf0[27]; + bf1[5] = bf0[5] + bf0[26]; + bf1[6] = bf0[6] + bf0[25]; + bf1[7] = bf0[7] + bf0[24]; + bf1[8] = bf0[8] + bf0[23]; + bf1[9] = bf0[9] + bf0[22]; + bf1[10] = bf0[10] + bf0[21]; + bf1[11] = bf0[11] + bf0[20]; + bf1[12] = bf0[12] + bf0[19]; + bf1[13] = bf0[13] + bf0[18]; + bf1[14] = bf0[14] + bf0[17]; + bf1[15] = bf0[15] + bf0[16]; + bf1[16] = -bf0[16] + bf0[15]; + bf1[17] = -bf0[17] + bf0[14]; + bf1[18] = -bf0[18] + bf0[13]; + bf1[19] = -bf0[19] + bf0[12]; + bf1[20] = -bf0[20] + bf0[11]; + bf1[21] = -bf0[21] + bf0[10]; + bf1[22] = -bf0[22] + bf0[9]; + bf1[23] = -bf0[23] + bf0[8]; + bf1[24] = -bf0[24] + bf0[7]; + bf1[25] = -bf0[25] + bf0[6]; + bf1[26] = -bf0[26] + bf0[5]; + bf1[27] = -bf0[27] + bf0[4]; + bf1[28] = -bf0[28] + bf0[3]; + bf1[29] = -bf0[29] + bf0[2]; + bf1[30] = -bf0[30] + bf0[1]; + bf1[31] = -bf0[31] + bf0[0]; + bf1[32] = bf0[32]; + bf1[33] = bf0[33]; + bf1[34] = bf0[34]; + bf1[35] = bf0[35]; + bf1[36] = bf0[36]; + bf1[37] = bf0[37]; + bf1[38] = bf0[38]; + bf1[39] = bf0[39]; + bf1[40] = half_btf(-cospi[32], bf0[40], cospi[32], bf0[55], cos_bit[stage]); + bf1[41] = half_btf(-cospi[32], bf0[41], cospi[32], bf0[54], cos_bit[stage]); + bf1[42] = half_btf(-cospi[32], bf0[42], cospi[32], bf0[53], cos_bit[stage]); + bf1[43] = half_btf(-cospi[32], bf0[43], cospi[32], bf0[52], cos_bit[stage]); + bf1[44] = half_btf(-cospi[32], bf0[44], cospi[32], bf0[51], cos_bit[stage]); + bf1[45] = half_btf(-cospi[32], bf0[45], cospi[32], bf0[50], cos_bit[stage]); + bf1[46] = half_btf(-cospi[32], bf0[46], cospi[32], bf0[49], cos_bit[stage]); + bf1[47] = half_btf(-cospi[32], bf0[47], cospi[32], bf0[48], cos_bit[stage]); + bf1[48] = half_btf(cospi[32], bf0[48], cospi[32], bf0[47], cos_bit[stage]); + bf1[49] = half_btf(cospi[32], bf0[49], cospi[32], bf0[46], cos_bit[stage]); + bf1[50] = half_btf(cospi[32], bf0[50], cospi[32], bf0[45], cos_bit[stage]); + bf1[51] = half_btf(cospi[32], bf0[51], cospi[32], bf0[44], cos_bit[stage]); + bf1[52] = half_btf(cospi[32], bf0[52], cospi[32], bf0[43], cos_bit[stage]); + bf1[53] = half_btf(cospi[32], bf0[53], cospi[32], bf0[42], cos_bit[stage]); + bf1[54] = half_btf(cospi[32], bf0[54], cospi[32], bf0[41], cos_bit[stage]); + bf1[55] = half_btf(cospi[32], bf0[55], cospi[32], bf0[40], cos_bit[stage]); + bf1[56] = bf0[56]; + bf1[57] = bf0[57]; + bf1[58] = bf0[58]; + bf1[59] = bf0[59]; + bf1[60] = bf0[60]; + bf1[61] = bf0[61]; + bf1[62] = bf0[62]; + bf1[63] = bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[15]; + bf1[1] = bf0[1] + bf0[14]; + bf1[2] = bf0[2] + bf0[13]; + bf1[3] = bf0[3] + bf0[12]; + bf1[4] = bf0[4] + bf0[11]; + bf1[5] = bf0[5] + bf0[10]; + bf1[6] = bf0[6] + bf0[9]; + bf1[7] = bf0[7] + bf0[8]; + bf1[8] = -bf0[8] + bf0[7]; + bf1[9] = -bf0[9] + bf0[6]; + bf1[10] = -bf0[10] + bf0[5]; + bf1[11] = -bf0[11] + bf0[4]; + bf1[12] = -bf0[12] + bf0[3]; + bf1[13] = -bf0[13] + bf0[2]; + bf1[14] = -bf0[14] + bf0[1]; + bf1[15] = -bf0[15] + bf0[0]; + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = bf0[18]; + bf1[19] = bf0[19]; + bf1[20] = half_btf(-cospi[32], bf0[20], cospi[32], bf0[27], cos_bit[stage]); + bf1[21] = half_btf(-cospi[32], bf0[21], cospi[32], bf0[26], cos_bit[stage]); + bf1[22] = half_btf(-cospi[32], bf0[22], cospi[32], bf0[25], cos_bit[stage]); + bf1[23] = half_btf(-cospi[32], bf0[23], cospi[32], bf0[24], cos_bit[stage]); + bf1[24] = half_btf(cospi[32], bf0[24], cospi[32], bf0[23], cos_bit[stage]); + bf1[25] = half_btf(cospi[32], bf0[25], cospi[32], bf0[22], cos_bit[stage]); + bf1[26] = half_btf(cospi[32], bf0[26], cospi[32], bf0[21], cos_bit[stage]); + bf1[27] = half_btf(cospi[32], bf0[27], cospi[32], bf0[20], cos_bit[stage]); + bf1[28] = bf0[28]; + bf1[29] = bf0[29]; + bf1[30] = bf0[30]; + bf1[31] = bf0[31]; + bf1[32] = bf0[32] + bf0[47]; + bf1[33] = bf0[33] + bf0[46]; + bf1[34] = bf0[34] + bf0[45]; + bf1[35] = bf0[35] + bf0[44]; + bf1[36] = bf0[36] + bf0[43]; + bf1[37] = bf0[37] + bf0[42]; + bf1[38] = bf0[38] + bf0[41]; + bf1[39] = bf0[39] + bf0[40]; + bf1[40] = -bf0[40] + bf0[39]; + bf1[41] = -bf0[41] + bf0[38]; + bf1[42] = -bf0[42] + bf0[37]; + bf1[43] = -bf0[43] + bf0[36]; + bf1[44] = -bf0[44] + bf0[35]; + bf1[45] = -bf0[45] + bf0[34]; + bf1[46] = -bf0[46] + bf0[33]; + bf1[47] = -bf0[47] + bf0[32]; + bf1[48] = -bf0[48] + bf0[63]; + bf1[49] = -bf0[49] + bf0[62]; + bf1[50] = -bf0[50] + bf0[61]; + bf1[51] = -bf0[51] + bf0[60]; + bf1[52] = -bf0[52] + bf0[59]; + bf1[53] = -bf0[53] + bf0[58]; + bf1[54] = -bf0[54] + bf0[57]; + bf1[55] = -bf0[55] + bf0[56]; + bf1[56] = bf0[56] + bf0[55]; + bf1[57] = bf0[57] + bf0[54]; + bf1[58] = bf0[58] + bf0[53]; + bf1[59] = bf0[59] + bf0[52]; + bf1[60] = bf0[60] + bf0[51]; + bf1[61] = bf0[61] + bf0[50]; + bf1[62] = bf0[62] + bf0[49]; + bf1[63] = bf0[63] + bf0[48]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0] + bf0[7]; + bf1[1] = bf0[1] + bf0[6]; + bf1[2] = bf0[2] + bf0[5]; + bf1[3] = bf0[3] + bf0[4]; + bf1[4] = -bf0[4] + bf0[3]; + bf1[5] = -bf0[5] + bf0[2]; + bf1[6] = -bf0[6] + bf0[1]; + bf1[7] = -bf0[7] + bf0[0]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = half_btf(-cospi[32], bf0[10], cospi[32], bf0[13], cos_bit[stage]); + bf1[11] = half_btf(-cospi[32], bf0[11], cospi[32], bf0[12], cos_bit[stage]); + bf1[12] = half_btf(cospi[32], bf0[12], cospi[32], bf0[11], cos_bit[stage]); + bf1[13] = half_btf(cospi[32], bf0[13], cospi[32], bf0[10], cos_bit[stage]); + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = bf0[16] + bf0[23]; + bf1[17] = bf0[17] + bf0[22]; + bf1[18] = bf0[18] + bf0[21]; + bf1[19] = bf0[19] + bf0[20]; + bf1[20] = -bf0[20] + bf0[19]; + bf1[21] = -bf0[21] + bf0[18]; + bf1[22] = -bf0[22] + bf0[17]; + bf1[23] = -bf0[23] + bf0[16]; + bf1[24] = -bf0[24] + bf0[31]; + bf1[25] = -bf0[25] + bf0[30]; + bf1[26] = -bf0[26] + bf0[29]; + bf1[27] = -bf0[27] + bf0[28]; + bf1[28] = bf0[28] + bf0[27]; + bf1[29] = bf0[29] + bf0[26]; + bf1[30] = bf0[30] + bf0[25]; + bf1[31] = bf0[31] + bf0[24]; + bf1[32] = bf0[32]; + bf1[33] = bf0[33]; + bf1[34] = bf0[34]; + bf1[35] = bf0[35]; + bf1[36] = half_btf(-cospi[16], bf0[36], cospi[48], bf0[59], cos_bit[stage]); + bf1[37] = half_btf(-cospi[16], bf0[37], cospi[48], bf0[58], cos_bit[stage]); + bf1[38] = half_btf(-cospi[16], bf0[38], cospi[48], bf0[57], cos_bit[stage]); + bf1[39] = half_btf(-cospi[16], bf0[39], cospi[48], bf0[56], cos_bit[stage]); + bf1[40] = half_btf(-cospi[48], bf0[40], -cospi[16], bf0[55], cos_bit[stage]); + bf1[41] = half_btf(-cospi[48], bf0[41], -cospi[16], bf0[54], cos_bit[stage]); + bf1[42] = half_btf(-cospi[48], bf0[42], -cospi[16], bf0[53], cos_bit[stage]); + bf1[43] = half_btf(-cospi[48], bf0[43], -cospi[16], bf0[52], cos_bit[stage]); + bf1[44] = bf0[44]; + bf1[45] = bf0[45]; + bf1[46] = bf0[46]; + bf1[47] = bf0[47]; + bf1[48] = bf0[48]; + bf1[49] = bf0[49]; + bf1[50] = bf0[50]; + bf1[51] = bf0[51]; + bf1[52] = half_btf(cospi[48], bf0[52], -cospi[16], bf0[43], cos_bit[stage]); + bf1[53] = half_btf(cospi[48], bf0[53], -cospi[16], bf0[42], cos_bit[stage]); + bf1[54] = half_btf(cospi[48], bf0[54], -cospi[16], bf0[41], cos_bit[stage]); + bf1[55] = half_btf(cospi[48], bf0[55], -cospi[16], bf0[40], cos_bit[stage]); + bf1[56] = half_btf(cospi[16], bf0[56], cospi[48], bf0[39], cos_bit[stage]); + bf1[57] = half_btf(cospi[16], bf0[57], cospi[48], bf0[38], cos_bit[stage]); + bf1[58] = half_btf(cospi[16], bf0[58], cospi[48], bf0[37], cos_bit[stage]); + bf1[59] = half_btf(cospi[16], bf0[59], cospi[48], bf0[36], cos_bit[stage]); + bf1[60] = bf0[60]; + bf1[61] = bf0[61]; + bf1[62] = bf0[62]; + bf1[63] = bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[3]; + bf1[1] = bf0[1] + bf0[2]; + bf1[2] = -bf0[2] + bf0[1]; + bf1[3] = -bf0[3] + bf0[0]; + bf1[4] = bf0[4]; + bf1[5] = half_btf(-cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[32], bf0[6], cospi[32], bf0[5], cos_bit[stage]); + bf1[7] = bf0[7]; + bf1[8] = bf0[8] + bf0[11]; + bf1[9] = bf0[9] + bf0[10]; + bf1[10] = -bf0[10] + bf0[9]; + bf1[11] = -bf0[11] + bf0[8]; + bf1[12] = -bf0[12] + bf0[15]; + bf1[13] = -bf0[13] + bf0[14]; + bf1[14] = bf0[14] + bf0[13]; + bf1[15] = bf0[15] + bf0[12]; + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = half_btf(-cospi[16], bf0[18], cospi[48], bf0[29], cos_bit[stage]); + bf1[19] = half_btf(-cospi[16], bf0[19], cospi[48], bf0[28], cos_bit[stage]); + bf1[20] = half_btf(-cospi[48], bf0[20], -cospi[16], bf0[27], cos_bit[stage]); + bf1[21] = half_btf(-cospi[48], bf0[21], -cospi[16], bf0[26], cos_bit[stage]); + bf1[22] = bf0[22]; + bf1[23] = bf0[23]; + bf1[24] = bf0[24]; + bf1[25] = bf0[25]; + bf1[26] = half_btf(cospi[48], bf0[26], -cospi[16], bf0[21], cos_bit[stage]); + bf1[27] = half_btf(cospi[48], bf0[27], -cospi[16], bf0[20], cos_bit[stage]); + bf1[28] = half_btf(cospi[16], bf0[28], cospi[48], bf0[19], cos_bit[stage]); + bf1[29] = half_btf(cospi[16], bf0[29], cospi[48], bf0[18], cos_bit[stage]); + bf1[30] = bf0[30]; + bf1[31] = bf0[31]; + bf1[32] = bf0[32] + bf0[39]; + bf1[33] = bf0[33] + bf0[38]; + bf1[34] = bf0[34] + bf0[37]; + bf1[35] = bf0[35] + bf0[36]; + bf1[36] = -bf0[36] + bf0[35]; + bf1[37] = -bf0[37] + bf0[34]; + bf1[38] = -bf0[38] + bf0[33]; + bf1[39] = -bf0[39] + bf0[32]; + bf1[40] = -bf0[40] + bf0[47]; + bf1[41] = -bf0[41] + bf0[46]; + bf1[42] = -bf0[42] + bf0[45]; + bf1[43] = -bf0[43] + bf0[44]; + bf1[44] = bf0[44] + bf0[43]; + bf1[45] = bf0[45] + bf0[42]; + bf1[46] = bf0[46] + bf0[41]; + bf1[47] = bf0[47] + bf0[40]; + bf1[48] = bf0[48] + bf0[55]; + bf1[49] = bf0[49] + bf0[54]; + bf1[50] = bf0[50] + bf0[53]; + bf1[51] = bf0[51] + bf0[52]; + bf1[52] = -bf0[52] + bf0[51]; + bf1[53] = -bf0[53] + bf0[50]; + bf1[54] = -bf0[54] + bf0[49]; + bf1[55] = -bf0[55] + bf0[48]; + bf1[56] = -bf0[56] + bf0[63]; + bf1[57] = -bf0[57] + bf0[62]; + bf1[58] = -bf0[58] + bf0[61]; + bf1[59] = -bf0[59] + bf0[60]; + bf1[60] = bf0[60] + bf0[59]; + bf1[61] = bf0[61] + bf0[58]; + bf1[62] = bf0[62] + bf0[57]; + bf1[63] = bf0[63] + bf0[56]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 6 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[32], bf0[0], cospi[32], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(-cospi[32], bf0[1], cospi[32], bf0[0], cos_bit[stage]); + bf1[2] = half_btf(cospi[48], bf0[2], cospi[16], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[48], bf0[3], -cospi[16], bf0[2], cos_bit[stage]); + bf1[4] = bf0[4] + bf0[5]; + bf1[5] = -bf0[5] + bf0[4]; + bf1[6] = -bf0[6] + bf0[7]; + bf1[7] = bf0[7] + bf0[6]; + bf1[8] = bf0[8]; + bf1[9] = half_btf(-cospi[16], bf0[9], cospi[48], bf0[14], cos_bit[stage]); + bf1[10] = half_btf(-cospi[48], bf0[10], -cospi[16], bf0[13], cos_bit[stage]); + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = half_btf(cospi[48], bf0[13], -cospi[16], bf0[10], cos_bit[stage]); + bf1[14] = half_btf(cospi[16], bf0[14], cospi[48], bf0[9], cos_bit[stage]); + bf1[15] = bf0[15]; + bf1[16] = bf0[16] + bf0[19]; + bf1[17] = bf0[17] + bf0[18]; + bf1[18] = -bf0[18] + bf0[17]; + bf1[19] = -bf0[19] + bf0[16]; + bf1[20] = -bf0[20] + bf0[23]; + bf1[21] = -bf0[21] + bf0[22]; + bf1[22] = bf0[22] + bf0[21]; + bf1[23] = bf0[23] + bf0[20]; + bf1[24] = bf0[24] + bf0[27]; + bf1[25] = bf0[25] + bf0[26]; + bf1[26] = -bf0[26] + bf0[25]; + bf1[27] = -bf0[27] + bf0[24]; + bf1[28] = -bf0[28] + bf0[31]; + bf1[29] = -bf0[29] + bf0[30]; + bf1[30] = bf0[30] + bf0[29]; + bf1[31] = bf0[31] + bf0[28]; + bf1[32] = bf0[32]; + bf1[33] = bf0[33]; + bf1[34] = half_btf(-cospi[8], bf0[34], cospi[56], bf0[61], cos_bit[stage]); + bf1[35] = half_btf(-cospi[8], bf0[35], cospi[56], bf0[60], cos_bit[stage]); + bf1[36] = half_btf(-cospi[56], bf0[36], -cospi[8], bf0[59], cos_bit[stage]); + bf1[37] = half_btf(-cospi[56], bf0[37], -cospi[8], bf0[58], cos_bit[stage]); + bf1[38] = bf0[38]; + bf1[39] = bf0[39]; + bf1[40] = bf0[40]; + bf1[41] = bf0[41]; + bf1[42] = half_btf(-cospi[40], bf0[42], cospi[24], bf0[53], cos_bit[stage]); + bf1[43] = half_btf(-cospi[40], bf0[43], cospi[24], bf0[52], cos_bit[stage]); + bf1[44] = half_btf(-cospi[24], bf0[44], -cospi[40], bf0[51], cos_bit[stage]); + bf1[45] = half_btf(-cospi[24], bf0[45], -cospi[40], bf0[50], cos_bit[stage]); + bf1[46] = bf0[46]; + bf1[47] = bf0[47]; + bf1[48] = bf0[48]; + bf1[49] = bf0[49]; + bf1[50] = half_btf(cospi[24], bf0[50], -cospi[40], bf0[45], cos_bit[stage]); + bf1[51] = half_btf(cospi[24], bf0[51], -cospi[40], bf0[44], cos_bit[stage]); + bf1[52] = half_btf(cospi[40], bf0[52], cospi[24], bf0[43], cos_bit[stage]); + bf1[53] = half_btf(cospi[40], bf0[53], cospi[24], bf0[42], cos_bit[stage]); + bf1[54] = bf0[54]; + bf1[55] = bf0[55]; + bf1[56] = bf0[56]; + bf1[57] = bf0[57]; + bf1[58] = half_btf(cospi[56], bf0[58], -cospi[8], bf0[37], cos_bit[stage]); + bf1[59] = half_btf(cospi[56], bf0[59], -cospi[8], bf0[36], cos_bit[stage]); + bf1[60] = half_btf(cospi[8], bf0[60], cospi[56], bf0[35], cos_bit[stage]); + bf1[61] = half_btf(cospi[8], bf0[61], cospi[56], bf0[34], cos_bit[stage]); + bf1[62] = bf0[62]; + bf1[63] = bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 7 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[56], bf0[4], cospi[8], bf0[7], cos_bit[stage]); + bf1[5] = half_btf(cospi[24], bf0[5], cospi[40], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[24], bf0[6], -cospi[40], bf0[5], cos_bit[stage]); + bf1[7] = half_btf(cospi[56], bf0[7], -cospi[8], bf0[4], cos_bit[stage]); + bf1[8] = bf0[8] + bf0[9]; + bf1[9] = -bf0[9] + bf0[8]; + bf1[10] = -bf0[10] + bf0[11]; + bf1[11] = bf0[11] + bf0[10]; + bf1[12] = bf0[12] + bf0[13]; + bf1[13] = -bf0[13] + bf0[12]; + bf1[14] = -bf0[14] + bf0[15]; + bf1[15] = bf0[15] + bf0[14]; + bf1[16] = bf0[16]; + bf1[17] = half_btf(-cospi[8], bf0[17], cospi[56], bf0[30], cos_bit[stage]); + bf1[18] = half_btf(-cospi[56], bf0[18], -cospi[8], bf0[29], cos_bit[stage]); + bf1[19] = bf0[19]; + bf1[20] = bf0[20]; + bf1[21] = half_btf(-cospi[40], bf0[21], cospi[24], bf0[26], cos_bit[stage]); + bf1[22] = half_btf(-cospi[24], bf0[22], -cospi[40], bf0[25], cos_bit[stage]); + bf1[23] = bf0[23]; + bf1[24] = bf0[24]; + bf1[25] = half_btf(cospi[24], bf0[25], -cospi[40], bf0[22], cos_bit[stage]); + bf1[26] = half_btf(cospi[40], bf0[26], cospi[24], bf0[21], cos_bit[stage]); + bf1[27] = bf0[27]; + bf1[28] = bf0[28]; + bf1[29] = half_btf(cospi[56], bf0[29], -cospi[8], bf0[18], cos_bit[stage]); + bf1[30] = half_btf(cospi[8], bf0[30], cospi[56], bf0[17], cos_bit[stage]); + bf1[31] = bf0[31]; + bf1[32] = bf0[32] + bf0[35]; + bf1[33] = bf0[33] + bf0[34]; + bf1[34] = -bf0[34] + bf0[33]; + bf1[35] = -bf0[35] + bf0[32]; + bf1[36] = -bf0[36] + bf0[39]; + bf1[37] = -bf0[37] + bf0[38]; + bf1[38] = bf0[38] + bf0[37]; + bf1[39] = bf0[39] + bf0[36]; + bf1[40] = bf0[40] + bf0[43]; + bf1[41] = bf0[41] + bf0[42]; + bf1[42] = -bf0[42] + bf0[41]; + bf1[43] = -bf0[43] + bf0[40]; + bf1[44] = -bf0[44] + bf0[47]; + bf1[45] = -bf0[45] + bf0[46]; + bf1[46] = bf0[46] + bf0[45]; + bf1[47] = bf0[47] + bf0[44]; + bf1[48] = bf0[48] + bf0[51]; + bf1[49] = bf0[49] + bf0[50]; + bf1[50] = -bf0[50] + bf0[49]; + bf1[51] = -bf0[51] + bf0[48]; + bf1[52] = -bf0[52] + bf0[55]; + bf1[53] = -bf0[53] + bf0[54]; + bf1[54] = bf0[54] + bf0[53]; + bf1[55] = bf0[55] + bf0[52]; + bf1[56] = bf0[56] + bf0[59]; + bf1[57] = bf0[57] + bf0[58]; + bf1[58] = -bf0[58] + bf0[57]; + bf1[59] = -bf0[59] + bf0[56]; + bf1[60] = -bf0[60] + bf0[63]; + bf1[61] = -bf0[61] + bf0[62]; + bf1[62] = bf0[62] + bf0[61]; + bf1[63] = bf0[63] + bf0[60]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 8 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = half_btf(cospi[60], bf0[8], cospi[4], bf0[15], cos_bit[stage]); + bf1[9] = half_btf(cospi[28], bf0[9], cospi[36], bf0[14], cos_bit[stage]); + bf1[10] = half_btf(cospi[44], bf0[10], cospi[20], bf0[13], cos_bit[stage]); + bf1[11] = half_btf(cospi[12], bf0[11], cospi[52], bf0[12], cos_bit[stage]); + bf1[12] = half_btf(cospi[12], bf0[12], -cospi[52], bf0[11], cos_bit[stage]); + bf1[13] = half_btf(cospi[44], bf0[13], -cospi[20], bf0[10], cos_bit[stage]); + bf1[14] = half_btf(cospi[28], bf0[14], -cospi[36], bf0[9], cos_bit[stage]); + bf1[15] = half_btf(cospi[60], bf0[15], -cospi[4], bf0[8], cos_bit[stage]); + bf1[16] = bf0[16] + bf0[17]; + bf1[17] = -bf0[17] + bf0[16]; + bf1[18] = -bf0[18] + bf0[19]; + bf1[19] = bf0[19] + bf0[18]; + bf1[20] = bf0[20] + bf0[21]; + bf1[21] = -bf0[21] + bf0[20]; + bf1[22] = -bf0[22] + bf0[23]; + bf1[23] = bf0[23] + bf0[22]; + bf1[24] = bf0[24] + bf0[25]; + bf1[25] = -bf0[25] + bf0[24]; + bf1[26] = -bf0[26] + bf0[27]; + bf1[27] = bf0[27] + bf0[26]; + bf1[28] = bf0[28] + bf0[29]; + bf1[29] = -bf0[29] + bf0[28]; + bf1[30] = -bf0[30] + bf0[31]; + bf1[31] = bf0[31] + bf0[30]; + bf1[32] = bf0[32]; + bf1[33] = half_btf(-cospi[4], bf0[33], cospi[60], bf0[62], cos_bit[stage]); + bf1[34] = half_btf(-cospi[60], bf0[34], -cospi[4], bf0[61], cos_bit[stage]); + bf1[35] = bf0[35]; + bf1[36] = bf0[36]; + bf1[37] = half_btf(-cospi[36], bf0[37], cospi[28], bf0[58], cos_bit[stage]); + bf1[38] = half_btf(-cospi[28], bf0[38], -cospi[36], bf0[57], cos_bit[stage]); + bf1[39] = bf0[39]; + bf1[40] = bf0[40]; + bf1[41] = half_btf(-cospi[20], bf0[41], cospi[44], bf0[54], cos_bit[stage]); + bf1[42] = half_btf(-cospi[44], bf0[42], -cospi[20], bf0[53], cos_bit[stage]); + bf1[43] = bf0[43]; + bf1[44] = bf0[44]; + bf1[45] = half_btf(-cospi[52], bf0[45], cospi[12], bf0[50], cos_bit[stage]); + bf1[46] = half_btf(-cospi[12], bf0[46], -cospi[52], bf0[49], cos_bit[stage]); + bf1[47] = bf0[47]; + bf1[48] = bf0[48]; + bf1[49] = half_btf(cospi[12], bf0[49], -cospi[52], bf0[46], cos_bit[stage]); + bf1[50] = half_btf(cospi[52], bf0[50], cospi[12], bf0[45], cos_bit[stage]); + bf1[51] = bf0[51]; + bf1[52] = bf0[52]; + bf1[53] = half_btf(cospi[44], bf0[53], -cospi[20], bf0[42], cos_bit[stage]); + bf1[54] = half_btf(cospi[20], bf0[54], cospi[44], bf0[41], cos_bit[stage]); + bf1[55] = bf0[55]; + bf1[56] = bf0[56]; + bf1[57] = half_btf(cospi[28], bf0[57], -cospi[36], bf0[38], cos_bit[stage]); + bf1[58] = half_btf(cospi[36], bf0[58], cospi[28], bf0[37], cos_bit[stage]); + bf1[59] = bf0[59]; + bf1[60] = bf0[60]; + bf1[61] = half_btf(cospi[60], bf0[61], -cospi[4], bf0[34], cos_bit[stage]); + bf1[62] = half_btf(cospi[4], bf0[62], cospi[60], bf0[33], cos_bit[stage]); + bf1[63] = bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 9 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = bf0[10]; + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = bf0[13]; + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = half_btf(cospi[62], bf0[16], cospi[2], bf0[31], cos_bit[stage]); + bf1[17] = half_btf(cospi[30], bf0[17], cospi[34], bf0[30], cos_bit[stage]); + bf1[18] = half_btf(cospi[46], bf0[18], cospi[18], bf0[29], cos_bit[stage]); + bf1[19] = half_btf(cospi[14], bf0[19], cospi[50], bf0[28], cos_bit[stage]); + bf1[20] = half_btf(cospi[54], bf0[20], cospi[10], bf0[27], cos_bit[stage]); + bf1[21] = half_btf(cospi[22], bf0[21], cospi[42], bf0[26], cos_bit[stage]); + bf1[22] = half_btf(cospi[38], bf0[22], cospi[26], bf0[25], cos_bit[stage]); + bf1[23] = half_btf(cospi[6], bf0[23], cospi[58], bf0[24], cos_bit[stage]); + bf1[24] = half_btf(cospi[6], bf0[24], -cospi[58], bf0[23], cos_bit[stage]); + bf1[25] = half_btf(cospi[38], bf0[25], -cospi[26], bf0[22], cos_bit[stage]); + bf1[26] = half_btf(cospi[22], bf0[26], -cospi[42], bf0[21], cos_bit[stage]); + bf1[27] = half_btf(cospi[54], bf0[27], -cospi[10], bf0[20], cos_bit[stage]); + bf1[28] = half_btf(cospi[14], bf0[28], -cospi[50], bf0[19], cos_bit[stage]); + bf1[29] = half_btf(cospi[46], bf0[29], -cospi[18], bf0[18], cos_bit[stage]); + bf1[30] = half_btf(cospi[30], bf0[30], -cospi[34], bf0[17], cos_bit[stage]); + bf1[31] = half_btf(cospi[62], bf0[31], -cospi[2], bf0[16], cos_bit[stage]); + bf1[32] = bf0[32] + bf0[33]; + bf1[33] = -bf0[33] + bf0[32]; + bf1[34] = -bf0[34] + bf0[35]; + bf1[35] = bf0[35] + bf0[34]; + bf1[36] = bf0[36] + bf0[37]; + bf1[37] = -bf0[37] + bf0[36]; + bf1[38] = -bf0[38] + bf0[39]; + bf1[39] = bf0[39] + bf0[38]; + bf1[40] = bf0[40] + bf0[41]; + bf1[41] = -bf0[41] + bf0[40]; + bf1[42] = -bf0[42] + bf0[43]; + bf1[43] = bf0[43] + bf0[42]; + bf1[44] = bf0[44] + bf0[45]; + bf1[45] = -bf0[45] + bf0[44]; + bf1[46] = -bf0[46] + bf0[47]; + bf1[47] = bf0[47] + bf0[46]; + bf1[48] = bf0[48] + bf0[49]; + bf1[49] = -bf0[49] + bf0[48]; + bf1[50] = -bf0[50] + bf0[51]; + bf1[51] = bf0[51] + bf0[50]; + bf1[52] = bf0[52] + bf0[53]; + bf1[53] = -bf0[53] + bf0[52]; + bf1[54] = -bf0[54] + bf0[55]; + bf1[55] = bf0[55] + bf0[54]; + bf1[56] = bf0[56] + bf0[57]; + bf1[57] = -bf0[57] + bf0[56]; + bf1[58] = -bf0[58] + bf0[59]; + bf1[59] = bf0[59] + bf0[58]; + bf1[60] = bf0[60] + bf0[61]; + bf1[61] = -bf0[61] + bf0[60]; + bf1[62] = -bf0[62] + bf0[63]; + bf1[63] = bf0[63] + bf0[62]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 10 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = bf0[10]; + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = bf0[13]; + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = bf0[18]; + bf1[19] = bf0[19]; + bf1[20] = bf0[20]; + bf1[21] = bf0[21]; + bf1[22] = bf0[22]; + bf1[23] = bf0[23]; + bf1[24] = bf0[24]; + bf1[25] = bf0[25]; + bf1[26] = bf0[26]; + bf1[27] = bf0[27]; + bf1[28] = bf0[28]; + bf1[29] = bf0[29]; + bf1[30] = bf0[30]; + bf1[31] = bf0[31]; + bf1[32] = half_btf(cospi[63], bf0[32], cospi[1], bf0[63], cos_bit[stage]); + bf1[33] = half_btf(cospi[31], bf0[33], cospi[33], bf0[62], cos_bit[stage]); + bf1[34] = half_btf(cospi[47], bf0[34], cospi[17], bf0[61], cos_bit[stage]); + bf1[35] = half_btf(cospi[15], bf0[35], cospi[49], bf0[60], cos_bit[stage]); + bf1[36] = half_btf(cospi[55], bf0[36], cospi[9], bf0[59], cos_bit[stage]); + bf1[37] = half_btf(cospi[23], bf0[37], cospi[41], bf0[58], cos_bit[stage]); + bf1[38] = half_btf(cospi[39], bf0[38], cospi[25], bf0[57], cos_bit[stage]); + bf1[39] = half_btf(cospi[7], bf0[39], cospi[57], bf0[56], cos_bit[stage]); + bf1[40] = half_btf(cospi[59], bf0[40], cospi[5], bf0[55], cos_bit[stage]); + bf1[41] = half_btf(cospi[27], bf0[41], cospi[37], bf0[54], cos_bit[stage]); + bf1[42] = half_btf(cospi[43], bf0[42], cospi[21], bf0[53], cos_bit[stage]); + bf1[43] = half_btf(cospi[11], bf0[43], cospi[53], bf0[52], cos_bit[stage]); + bf1[44] = half_btf(cospi[51], bf0[44], cospi[13], bf0[51], cos_bit[stage]); + bf1[45] = half_btf(cospi[19], bf0[45], cospi[45], bf0[50], cos_bit[stage]); + bf1[46] = half_btf(cospi[35], bf0[46], cospi[29], bf0[49], cos_bit[stage]); + bf1[47] = half_btf(cospi[3], bf0[47], cospi[61], bf0[48], cos_bit[stage]); + bf1[48] = half_btf(cospi[3], bf0[48], -cospi[61], bf0[47], cos_bit[stage]); + bf1[49] = half_btf(cospi[35], bf0[49], -cospi[29], bf0[46], cos_bit[stage]); + bf1[50] = half_btf(cospi[19], bf0[50], -cospi[45], bf0[45], cos_bit[stage]); + bf1[51] = half_btf(cospi[51], bf0[51], -cospi[13], bf0[44], cos_bit[stage]); + bf1[52] = half_btf(cospi[11], bf0[52], -cospi[53], bf0[43], cos_bit[stage]); + bf1[53] = half_btf(cospi[43], bf0[53], -cospi[21], bf0[42], cos_bit[stage]); + bf1[54] = half_btf(cospi[27], bf0[54], -cospi[37], bf0[41], cos_bit[stage]); + bf1[55] = half_btf(cospi[59], bf0[55], -cospi[5], bf0[40], cos_bit[stage]); + bf1[56] = half_btf(cospi[7], bf0[56], -cospi[57], bf0[39], cos_bit[stage]); + bf1[57] = half_btf(cospi[39], bf0[57], -cospi[25], bf0[38], cos_bit[stage]); + bf1[58] = half_btf(cospi[23], bf0[58], -cospi[41], bf0[37], cos_bit[stage]); + bf1[59] = half_btf(cospi[55], bf0[59], -cospi[9], bf0[36], cos_bit[stage]); + bf1[60] = half_btf(cospi[15], bf0[60], -cospi[49], bf0[35], cos_bit[stage]); + bf1[61] = half_btf(cospi[47], bf0[61], -cospi[17], bf0[34], cos_bit[stage]); + bf1[62] = half_btf(cospi[31], bf0[62], -cospi[33], bf0[33], cos_bit[stage]); + bf1[63] = half_btf(cospi[63], bf0[63], -cospi[1], bf0[32], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 11 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[32]; + bf1[2] = bf0[16]; + bf1[3] = bf0[48]; + bf1[4] = bf0[8]; + bf1[5] = bf0[40]; + bf1[6] = bf0[24]; + bf1[7] = bf0[56]; + bf1[8] = bf0[4]; + bf1[9] = bf0[36]; + bf1[10] = bf0[20]; + bf1[11] = bf0[52]; + bf1[12] = bf0[12]; + bf1[13] = bf0[44]; + bf1[14] = bf0[28]; + bf1[15] = bf0[60]; + bf1[16] = bf0[2]; + bf1[17] = bf0[34]; + bf1[18] = bf0[18]; + bf1[19] = bf0[50]; + bf1[20] = bf0[10]; + bf1[21] = bf0[42]; + bf1[22] = bf0[26]; + bf1[23] = bf0[58]; + bf1[24] = bf0[6]; + bf1[25] = bf0[38]; + bf1[26] = bf0[22]; + bf1[27] = bf0[54]; + bf1[28] = bf0[14]; + bf1[29] = bf0[46]; + bf1[30] = bf0[30]; + bf1[31] = bf0[62]; + bf1[32] = bf0[1]; + bf1[33] = bf0[33]; + bf1[34] = bf0[17]; + bf1[35] = bf0[49]; + bf1[36] = bf0[9]; + bf1[37] = bf0[41]; + bf1[38] = bf0[25]; + bf1[39] = bf0[57]; + bf1[40] = bf0[5]; + bf1[41] = bf0[37]; + bf1[42] = bf0[21]; + bf1[43] = bf0[53]; + bf1[44] = bf0[13]; + bf1[45] = bf0[45]; + bf1[46] = bf0[29]; + bf1[47] = bf0[61]; + bf1[48] = bf0[3]; + bf1[49] = bf0[35]; + bf1[50] = bf0[19]; + bf1[51] = bf0[51]; + bf1[52] = bf0[11]; + bf1[53] = bf0[43]; + bf1[54] = bf0[27]; + bf1[55] = bf0[59]; + bf1[56] = bf0[7]; + bf1[57] = bf0[39]; + bf1[58] = bf0[23]; + bf1[59] = bf0[55]; + bf1[60] = bf0[15]; + bf1[61] = bf0[47]; + bf1[62] = bf0[31]; + bf1[63] = bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); +} +#endif // CONFIG_TX64X64 diff --git a/third_party/aom/av1/common/av1_fwd_txfm1d.h b/third_party/aom/av1/common/av1_fwd_txfm1d.h new file mode 100644 index 0000000000..9f246717e6 --- /dev/null +++ b/third_party/aom/av1/common/av1_fwd_txfm1d.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_FWD_TXFM1D_H_ +#define AV1_FWD_TXFM1D_H_ + +#include "av1/common/av1_txfm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_fdct4_new(const int32_t *input, int32_t *output, const int8_t *cos_bit, + const int8_t *stage_range); +void av1_fdct8_new(const int32_t *input, int32_t *output, const int8_t *cos_bit, + const int8_t *stage_range); +void av1_fdct16_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_fdct32_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_fdct64_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); + +void av1_fadst4_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_fadst8_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_fadst16_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_fadst32_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); + +#ifdef __cplusplus +} +#endif + +#endif // AV1_FWD_TXFM1D_H_ diff --git a/third_party/aom/av1/common/av1_fwd_txfm2d.c b/third_party/aom/av1/common/av1_fwd_txfm2d.c new file mode 100644 index 0000000000..d1dba82ca1 --- /dev/null +++ b/third_party/aom/av1/common/av1_fwd_txfm2d.c @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./av1_rtcd.h" +#include "av1/common/enums.h" +#include "av1/common/av1_fwd_txfm1d.h" +#include "av1/common/av1_fwd_txfm2d_cfg.h" +#include "av1/common/av1_txfm.h" + +static INLINE TxfmFunc fwd_txfm_type_to_func(TXFM_TYPE txfm_type) { + switch (txfm_type) { + case TXFM_TYPE_DCT4: return av1_fdct4_new; + case TXFM_TYPE_DCT8: return av1_fdct8_new; + case TXFM_TYPE_DCT16: return av1_fdct16_new; + case TXFM_TYPE_DCT32: return av1_fdct32_new; + case TXFM_TYPE_ADST4: return av1_fadst4_new; + case TXFM_TYPE_ADST8: return av1_fadst8_new; + case TXFM_TYPE_ADST16: return av1_fadst16_new; + case TXFM_TYPE_ADST32: return av1_fadst32_new; + default: assert(0); return NULL; + } +} + +static INLINE void fwd_txfm2d_c(const int16_t *input, int32_t *output, + const int stride, const TXFM_2D_FLIP_CFG *cfg, + int32_t *buf) { + int c, r; + const int txfm_size = cfg->cfg->txfm_size; + const int8_t *shift = cfg->cfg->shift; + const int8_t *stage_range_col = cfg->cfg->stage_range_col; + const int8_t *stage_range_row = cfg->cfg->stage_range_row; + const int8_t *cos_bit_col = cfg->cfg->cos_bit_col; + const int8_t *cos_bit_row = cfg->cfg->cos_bit_row; + const TxfmFunc txfm_func_col = fwd_txfm_type_to_func(cfg->cfg->txfm_type_col); + const TxfmFunc txfm_func_row = fwd_txfm_type_to_func(cfg->cfg->txfm_type_row); + + // use output buffer as temp buffer + int32_t *temp_in = output; + int32_t *temp_out = output + txfm_size; + + // Columns + for (c = 0; c < txfm_size; ++c) { + if (cfg->ud_flip == 0) { + for (r = 0; r < txfm_size; ++r) temp_in[r] = input[r * stride + c]; + } else { + for (r = 0; r < txfm_size; ++r) + // flip upside down + temp_in[r] = input[(txfm_size - r - 1) * stride + c]; + } + round_shift_array(temp_in, txfm_size, -shift[0]); + txfm_func_col(temp_in, temp_out, cos_bit_col, stage_range_col); + round_shift_array(temp_out, txfm_size, -shift[1]); + if (cfg->lr_flip == 0) { + for (r = 0; r < txfm_size; ++r) buf[r * txfm_size + c] = temp_out[r]; + } else { + for (r = 0; r < txfm_size; ++r) + // flip from left to right + buf[r * txfm_size + (txfm_size - c - 1)] = temp_out[r]; + } + } + + // Rows + for (r = 0; r < txfm_size; ++r) { + txfm_func_row(buf + r * txfm_size, output + r * txfm_size, cos_bit_row, + stage_range_row); + round_shift_array(output + r * txfm_size, txfm_size, -shift[2]); + } +} + +void av1_fwd_txfm2d_4x4_c(const int16_t *input, int32_t *output, int stride, + int tx_type, int bd) { + int32_t txfm_buf[4 * 4]; + TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_4X4); + (void)bd; + fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf); +} + +void av1_fwd_txfm2d_8x8_c(const int16_t *input, int32_t *output, int stride, + int tx_type, int bd) { + int32_t txfm_buf[8 * 8]; + TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_8X8); + (void)bd; + fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf); +} + +void av1_fwd_txfm2d_16x16_c(const int16_t *input, int32_t *output, int stride, + int tx_type, int bd) { + int32_t txfm_buf[16 * 16]; + TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_16X16); + (void)bd; + fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf); +} + +void av1_fwd_txfm2d_32x32_c(const int16_t *input, int32_t *output, int stride, + int tx_type, int bd) { + int32_t txfm_buf[32 * 32]; + TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_32X32); + (void)bd; + fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf); +} + +void av1_fwd_txfm2d_64x64_c(const int16_t *input, int32_t *output, int stride, + int tx_type, int bd) { + int32_t txfm_buf[64 * 64]; + TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_64x64_cfg(tx_type); + (void)bd; + fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf); +} + +#if CONFIG_EXT_TX +static const TXFM_2D_CFG *fwd_txfm_cfg_ls[FLIPADST_ADST + 1][TX_SIZES] = { + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_dct_dct_4, &fwd_txfm_2d_cfg_dct_dct_8, + &fwd_txfm_2d_cfg_dct_dct_16, &fwd_txfm_2d_cfg_dct_dct_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_adst_dct_4, &fwd_txfm_2d_cfg_adst_dct_8, + &fwd_txfm_2d_cfg_adst_dct_16, &fwd_txfm_2d_cfg_adst_dct_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_dct_adst_4, &fwd_txfm_2d_cfg_dct_adst_8, + &fwd_txfm_2d_cfg_dct_adst_16, &fwd_txfm_2d_cfg_dct_adst_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_adst_adst_4, &fwd_txfm_2d_cfg_adst_adst_8, + &fwd_txfm_2d_cfg_adst_adst_16, &fwd_txfm_2d_cfg_adst_adst_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_adst_dct_4, &fwd_txfm_2d_cfg_adst_dct_8, + &fwd_txfm_2d_cfg_adst_dct_16, &fwd_txfm_2d_cfg_adst_dct_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_dct_adst_4, &fwd_txfm_2d_cfg_dct_adst_8, + &fwd_txfm_2d_cfg_dct_adst_16, &fwd_txfm_2d_cfg_dct_adst_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_adst_adst_4, &fwd_txfm_2d_cfg_adst_adst_8, + &fwd_txfm_2d_cfg_adst_adst_16, &fwd_txfm_2d_cfg_adst_adst_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_adst_adst_4, &fwd_txfm_2d_cfg_adst_adst_8, + &fwd_txfm_2d_cfg_adst_adst_16, &fwd_txfm_2d_cfg_adst_adst_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_adst_adst_4, &fwd_txfm_2d_cfg_adst_adst_8, + &fwd_txfm_2d_cfg_adst_adst_16, &fwd_txfm_2d_cfg_adst_adst_32 }, +}; +#else // CONFIG_EXT_TX +static const TXFM_2D_CFG *fwd_txfm_cfg_ls[TX_TYPES][TX_SIZES] = { + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_dct_dct_4, &fwd_txfm_2d_cfg_dct_dct_8, + &fwd_txfm_2d_cfg_dct_dct_16, &fwd_txfm_2d_cfg_dct_dct_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_adst_dct_4, &fwd_txfm_2d_cfg_adst_dct_8, + &fwd_txfm_2d_cfg_adst_dct_16, &fwd_txfm_2d_cfg_adst_dct_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_dct_adst_4, &fwd_txfm_2d_cfg_dct_adst_8, + &fwd_txfm_2d_cfg_dct_adst_16, &fwd_txfm_2d_cfg_dct_adst_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_adst_adst_4, &fwd_txfm_2d_cfg_adst_adst_8, + &fwd_txfm_2d_cfg_adst_adst_16, &fwd_txfm_2d_cfg_adst_adst_32 }, +}; +#endif // CONFIG_EXT_TX + +TXFM_2D_FLIP_CFG av1_get_fwd_txfm_cfg(int tx_type, int tx_size) { + TXFM_2D_FLIP_CFG cfg; + set_flip_cfg(tx_type, &cfg); + cfg.cfg = fwd_txfm_cfg_ls[tx_type][tx_size]; + return cfg; +} + +TXFM_2D_FLIP_CFG av1_get_fwd_txfm_64x64_cfg(int tx_type) { + TXFM_2D_FLIP_CFG cfg; + switch (tx_type) { + case DCT_DCT: + cfg.cfg = &fwd_txfm_2d_cfg_dct_dct_64; + cfg.ud_flip = 0; + cfg.lr_flip = 0; + break; + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + default: + cfg.ud_flip = 0; + cfg.lr_flip = 0; + assert(0); + } + return cfg; +} diff --git a/third_party/aom/av1/common/av1_fwd_txfm2d_cfg.h b/third_party/aom/av1/common/av1_fwd_txfm2d_cfg.h new file mode 100644 index 0000000000..b5c828286b --- /dev/null +++ b/third_party/aom/av1/common/av1_fwd_txfm2d_cfg.h @@ -0,0 +1,444 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_FWD_TXFM2D_CFG_H_ +#define AV1_FWD_TXFM2D_CFG_H_ +#include "av1/common/enums.h" +#include "av1/common/av1_fwd_txfm1d.h" +// ---------------- config fwd_dct_dct_4 ---------------- +static const int8_t fwd_shift_dct_dct_4[3] = { 2, 0, 0 }; +static const int8_t fwd_stage_range_col_dct_dct_4[4] = { 15, 16, 17, 17 }; +static const int8_t fwd_stage_range_row_dct_dct_4[4] = { 17, 18, 18, 18 }; +static const int8_t fwd_cos_bit_col_dct_dct_4[4] = { 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_dct_dct_4[4] = { 13, 13, 13, 13 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_4 = { + 4, // .txfm_size + 4, // .stage_num_col + 4, // .stage_num_row + // 0, // .log_scale + fwd_shift_dct_dct_4, // .shift + fwd_stage_range_col_dct_dct_4, // .stage_range_col + fwd_stage_range_row_dct_dct_4, // .stage_range_row + fwd_cos_bit_col_dct_dct_4, // .cos_bit_col + fwd_cos_bit_row_dct_dct_4, // .cos_bit_row + TXFM_TYPE_DCT4, // .txfm_type_col + TXFM_TYPE_DCT4 +}; // .txfm_type_row + +// ---------------- config fwd_dct_dct_8 ---------------- +static const int8_t fwd_shift_dct_dct_8[3] = { 2, -1, 0 }; +static const int8_t fwd_stage_range_col_dct_dct_8[6] = { + 15, 16, 17, 18, 18, 18 +}; +static const int8_t fwd_stage_range_row_dct_dct_8[6] = { + 17, 18, 19, 19, 19, 19 +}; +static const int8_t fwd_cos_bit_col_dct_dct_8[6] = { 13, 13, 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_dct_dct_8[6] = { 13, 13, 13, 13, 13, 13 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_8 = { + 8, // .txfm_size + 6, // .stage_num_col + 6, // .stage_num_row + // 0, // .log_scale + fwd_shift_dct_dct_8, // .shift + fwd_stage_range_col_dct_dct_8, // .stage_range_col + fwd_stage_range_row_dct_dct_8, // .stage_range_row + fwd_cos_bit_col_dct_dct_8, // .cos_bit_col + fwd_cos_bit_row_dct_dct_8, // .cos_bit_row + TXFM_TYPE_DCT8, // .txfm_type_col + TXFM_TYPE_DCT8 +}; // .txfm_type_row + +// ---------------- config fwd_dct_dct_16 ---------------- +static const int8_t fwd_shift_dct_dct_16[3] = { 2, -2, 0 }; +static const int8_t fwd_stage_range_col_dct_dct_16[8] = { 15, 16, 17, 18, + 19, 19, 19, 19 }; +static const int8_t fwd_stage_range_row_dct_dct_16[8] = { 17, 18, 19, 20, + 20, 20, 20, 20 }; +static const int8_t fwd_cos_bit_col_dct_dct_16[8] = { 13, 13, 13, 13, + 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_dct_dct_16[8] = { 12, 12, 12, 12, + 12, 12, 12, 12 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_16 = { + 16, // .txfm_size + 8, // .stage_num_col + 8, // .stage_num_row + // 0, // .log_scale + fwd_shift_dct_dct_16, // .shift + fwd_stage_range_col_dct_dct_16, // .stage_range_col + fwd_stage_range_row_dct_dct_16, // .stage_range_row + fwd_cos_bit_col_dct_dct_16, // .cos_bit_col + fwd_cos_bit_row_dct_dct_16, // .cos_bit_row + TXFM_TYPE_DCT16, // .txfm_type_col + TXFM_TYPE_DCT16 +}; // .txfm_type_row + +// ---------------- config fwd_dct_dct_32 ---------------- +static const int8_t fwd_shift_dct_dct_32[3] = { 2, -4, 0 }; +static const int8_t fwd_stage_range_col_dct_dct_32[10] = { 15, 16, 17, 18, 19, + 20, 20, 20, 20, 20 }; +static const int8_t fwd_stage_range_row_dct_dct_32[10] = { 16, 17, 18, 19, 20, + 20, 20, 20, 20, 20 }; +static const int8_t fwd_cos_bit_col_dct_dct_32[10] = { 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12 }; +static const int8_t fwd_cos_bit_row_dct_dct_32[10] = { 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_32 = { + 32, // .txfm_size + 10, // .stage_num_col + 10, // .stage_num_row + // 1, // .log_scale + fwd_shift_dct_dct_32, // .shift + fwd_stage_range_col_dct_dct_32, // .stage_range_col + fwd_stage_range_row_dct_dct_32, // .stage_range_row + fwd_cos_bit_col_dct_dct_32, // .cos_bit_col + fwd_cos_bit_row_dct_dct_32, // .cos_bit_row + TXFM_TYPE_DCT32, // .txfm_type_col + TXFM_TYPE_DCT32 +}; // .txfm_type_row + +// ---------------- config fwd_dct_dct_64 ---------------- +static const int8_t fwd_shift_dct_dct_64[3] = { 0, -2, -2 }; +static const int8_t fwd_stage_range_col_dct_dct_64[12] = { + 13, 14, 15, 16, 17, 18, 19, 19, 19, 19, 19, 19 +}; +static const int8_t fwd_stage_range_row_dct_dct_64[12] = { + 17, 18, 19, 20, 21, 22, 22, 22, 22, 22, 22, 22 +}; +static const int8_t fwd_cos_bit_col_dct_dct_64[12] = { 15, 15, 15, 15, 15, 14, + 13, 13, 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_dct_dct_64[12] = { 15, 14, 13, 12, 11, 10, + 10, 10, 10, 10, 10, 10 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_64 = { + 64, // .txfm_size + 12, // .stage_num_col + 12, // .stage_num_row + fwd_shift_dct_dct_64, // .shift + fwd_stage_range_col_dct_dct_64, // .stage_range_col + fwd_stage_range_row_dct_dct_64, // .stage_range_row + fwd_cos_bit_col_dct_dct_64, // .cos_bit_col + fwd_cos_bit_row_dct_dct_64, // .cos_bit_row + TXFM_TYPE_DCT64, // .txfm_type_col + TXFM_TYPE_DCT64 +}; // .txfm_type_row + +// ---------------- config fwd_dct_adst_4 ---------------- +static const int8_t fwd_shift_dct_adst_4[3] = { 2, 0, 0 }; +static const int8_t fwd_stage_range_col_dct_adst_4[4] = { 15, 16, 17, 17 }; +static const int8_t fwd_stage_range_row_dct_adst_4[6] = { + 17, 17, 17, 18, 18, 18 +}; +static const int8_t fwd_cos_bit_col_dct_adst_4[4] = { 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_dct_adst_4[6] = { 13, 13, 13, 13, 13, 13 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_adst_4 = { + 4, // .txfm_size + 4, // .stage_num_col + 6, // .stage_num_row + // 0, // .log_scale + fwd_shift_dct_adst_4, // .shift + fwd_stage_range_col_dct_adst_4, // .stage_range_col + fwd_stage_range_row_dct_adst_4, // .stage_range_row + fwd_cos_bit_col_dct_adst_4, // .cos_bit_col + fwd_cos_bit_row_dct_adst_4, // .cos_bit_row + TXFM_TYPE_DCT4, // .txfm_type_col + TXFM_TYPE_ADST4 +}; // .txfm_type_row + +// ---------------- config fwd_dct_adst_8 ---------------- +static const int8_t fwd_shift_dct_adst_8[3] = { 2, -1, 0 }; +static const int8_t fwd_stage_range_col_dct_adst_8[6] = { + 15, 16, 17, 18, 18, 18 +}; +static const int8_t fwd_stage_range_row_dct_adst_8[8] = { 17, 17, 17, 18, + 18, 19, 19, 19 }; +static const int8_t fwd_cos_bit_col_dct_adst_8[6] = { 13, 13, 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_dct_adst_8[8] = { 13, 13, 13, 13, + 13, 13, 13, 13 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_adst_8 = { + 8, // .txfm_size + 6, // .stage_num_col + 8, // .stage_num_row + // 0, // .log_scale + fwd_shift_dct_adst_8, // .shift + fwd_stage_range_col_dct_adst_8, // .stage_range_col + fwd_stage_range_row_dct_adst_8, // .stage_range_row + fwd_cos_bit_col_dct_adst_8, // .cos_bit_col + fwd_cos_bit_row_dct_adst_8, // .cos_bit_row + TXFM_TYPE_DCT8, // .txfm_type_col + TXFM_TYPE_ADST8 +}; // .txfm_type_row + +// ---------------- config fwd_dct_adst_16 ---------------- +static const int8_t fwd_shift_dct_adst_16[3] = { 2, -2, 0 }; +static const int8_t fwd_stage_range_col_dct_adst_16[8] = { 15, 16, 17, 18, + 19, 19, 19, 19 }; +static const int8_t fwd_stage_range_row_dct_adst_16[10] = { + 17, 17, 17, 18, 18, 19, 19, 20, 20, 20 +}; +static const int8_t fwd_cos_bit_col_dct_adst_16[8] = { 13, 13, 13, 13, + 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_dct_adst_16[10] = { 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_adst_16 = { + 16, // .txfm_size + 8, // .stage_num_col + 10, // .stage_num_row + // 0, // .log_scale + fwd_shift_dct_adst_16, // .shift + fwd_stage_range_col_dct_adst_16, // .stage_range_col + fwd_stage_range_row_dct_adst_16, // .stage_range_row + fwd_cos_bit_col_dct_adst_16, // .cos_bit_col + fwd_cos_bit_row_dct_adst_16, // .cos_bit_row + TXFM_TYPE_DCT16, // .txfm_type_col + TXFM_TYPE_ADST16 +}; // .txfm_type_row + +// ---------------- config fwd_dct_adst_32 ---------------- +static const int8_t fwd_shift_dct_adst_32[3] = { 2, -4, 0 }; +static const int8_t fwd_stage_range_col_dct_adst_32[10] = { + 15, 16, 17, 18, 19, 20, 20, 20, 20, 20 +}; +static const int8_t fwd_stage_range_row_dct_adst_32[12] = { + 16, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 20 +}; +static const int8_t fwd_cos_bit_col_dct_adst_32[10] = { 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12 }; +static const int8_t fwd_cos_bit_row_dct_adst_32[12] = { + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12 +}; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_adst_32 = { + 32, // .txfm_size + 10, // .stage_num_col + 12, // .stage_num_row + // 1, // .log_scale + fwd_shift_dct_adst_32, // .shift + fwd_stage_range_col_dct_adst_32, // .stage_range_col + fwd_stage_range_row_dct_adst_32, // .stage_range_row + fwd_cos_bit_col_dct_adst_32, // .cos_bit_col + fwd_cos_bit_row_dct_adst_32, // .cos_bit_row + TXFM_TYPE_DCT32, // .txfm_type_col + TXFM_TYPE_ADST32 +}; // .txfm_type_row +// ---------------- config fwd_adst_adst_4 ---------------- +static const int8_t fwd_shift_adst_adst_4[3] = { 2, 0, 0 }; +static const int8_t fwd_stage_range_col_adst_adst_4[6] = { 15, 15, 16, + 17, 17, 17 }; +static const int8_t fwd_stage_range_row_adst_adst_4[6] = { 17, 17, 17, + 18, 18, 18 }; +static const int8_t fwd_cos_bit_col_adst_adst_4[6] = { 13, 13, 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_adst_adst_4[6] = { 13, 13, 13, 13, 13, 13 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_adst_adst_4 = { + 4, // .txfm_size + 6, // .stage_num_col + 6, // .stage_num_row + // 0, // .log_scale + fwd_shift_adst_adst_4, // .shift + fwd_stage_range_col_adst_adst_4, // .stage_range_col + fwd_stage_range_row_adst_adst_4, // .stage_range_row + fwd_cos_bit_col_adst_adst_4, // .cos_bit_col + fwd_cos_bit_row_adst_adst_4, // .cos_bit_row + TXFM_TYPE_ADST4, // .txfm_type_col + TXFM_TYPE_ADST4 +}; // .txfm_type_row + +// ---------------- config fwd_adst_adst_8 ---------------- +static const int8_t fwd_shift_adst_adst_8[3] = { 2, -1, 0 }; +static const int8_t fwd_stage_range_col_adst_adst_8[8] = { 15, 15, 16, 17, + 17, 18, 18, 18 }; +static const int8_t fwd_stage_range_row_adst_adst_8[8] = { 17, 17, 17, 18, + 18, 19, 19, 19 }; +static const int8_t fwd_cos_bit_col_adst_adst_8[8] = { 13, 13, 13, 13, + 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_adst_adst_8[8] = { 13, 13, 13, 13, + 13, 13, 13, 13 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_adst_adst_8 = { + 8, // .txfm_size + 8, // .stage_num_col + 8, // .stage_num_row + // 0, // .log_scale + fwd_shift_adst_adst_8, // .shift + fwd_stage_range_col_adst_adst_8, // .stage_range_col + fwd_stage_range_row_adst_adst_8, // .stage_range_row + fwd_cos_bit_col_adst_adst_8, // .cos_bit_col + fwd_cos_bit_row_adst_adst_8, // .cos_bit_row + TXFM_TYPE_ADST8, // .txfm_type_col + TXFM_TYPE_ADST8 +}; // .txfm_type_row + +// ---------------- config fwd_adst_adst_16 ---------------- +static const int8_t fwd_shift_adst_adst_16[3] = { 2, -2, 0 }; +static const int8_t fwd_stage_range_col_adst_adst_16[10] = { + 15, 15, 16, 17, 17, 18, 18, 19, 19, 19 +}; +static const int8_t fwd_stage_range_row_adst_adst_16[10] = { + 17, 17, 17, 18, 18, 19, 19, 20, 20, 20 +}; +static const int8_t fwd_cos_bit_col_adst_adst_16[10] = { 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_adst_adst_16[10] = { 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_adst_adst_16 = { + 16, // .txfm_size + 10, // .stage_num_col + 10, // .stage_num_row + // 0, // .log_scale + fwd_shift_adst_adst_16, // .shift + fwd_stage_range_col_adst_adst_16, // .stage_range_col + fwd_stage_range_row_adst_adst_16, // .stage_range_row + fwd_cos_bit_col_adst_adst_16, // .cos_bit_col + fwd_cos_bit_row_adst_adst_16, // .cos_bit_row + TXFM_TYPE_ADST16, // .txfm_type_col + TXFM_TYPE_ADST16 +}; // .txfm_type_row + +// ---------------- config fwd_adst_adst_32 ---------------- +static const int8_t fwd_shift_adst_adst_32[3] = { 2, -4, 0 }; +static const int8_t fwd_stage_range_col_adst_adst_32[12] = { + 15, 15, 16, 17, 17, 18, 18, 19, 19, 20, 20, 20 +}; +static const int8_t fwd_stage_range_row_adst_adst_32[12] = { + 16, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 20 +}; +static const int8_t fwd_cos_bit_col_adst_adst_32[12] = { + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12 +}; +static const int8_t fwd_cos_bit_row_adst_adst_32[12] = { + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12 +}; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_adst_adst_32 = { + 32, // .txfm_size + 12, // .stage_num_col + 12, // .stage_num_row + // 1, // .log_scale + fwd_shift_adst_adst_32, // .shift + fwd_stage_range_col_adst_adst_32, // .stage_range_col + fwd_stage_range_row_adst_adst_32, // .stage_range_row + fwd_cos_bit_col_adst_adst_32, // .cos_bit_col + fwd_cos_bit_row_adst_adst_32, // .cos_bit_row + TXFM_TYPE_ADST32, // .txfm_type_col + TXFM_TYPE_ADST32 +}; // .txfm_type_row + +// ---------------- config fwd_adst_dct_4 ---------------- +static const int8_t fwd_shift_adst_dct_4[3] = { 2, 0, 0 }; +static const int8_t fwd_stage_range_col_adst_dct_4[6] = { + 15, 15, 16, 17, 17, 17 +}; +static const int8_t fwd_stage_range_row_adst_dct_4[4] = { 17, 18, 18, 18 }; +static const int8_t fwd_cos_bit_col_adst_dct_4[6] = { 13, 13, 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_adst_dct_4[4] = { 13, 13, 13, 13 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_adst_dct_4 = { + 4, // .txfm_size + 6, // .stage_num_col + 4, // .stage_num_row + // 0, // .log_scale + fwd_shift_adst_dct_4, // .shift + fwd_stage_range_col_adst_dct_4, // .stage_range_col + fwd_stage_range_row_adst_dct_4, // .stage_range_row + fwd_cos_bit_col_adst_dct_4, // .cos_bit_col + fwd_cos_bit_row_adst_dct_4, // .cos_bit_row + TXFM_TYPE_ADST4, // .txfm_type_col + TXFM_TYPE_DCT4 +}; // .txfm_type_row + +// ---------------- config fwd_adst_dct_8 ---------------- +static const int8_t fwd_shift_adst_dct_8[3] = { 2, -1, 0 }; +static const int8_t fwd_stage_range_col_adst_dct_8[8] = { 15, 15, 16, 17, + 17, 18, 18, 18 }; +static const int8_t fwd_stage_range_row_adst_dct_8[6] = { + 17, 18, 19, 19, 19, 19 +}; +static const int8_t fwd_cos_bit_col_adst_dct_8[8] = { 13, 13, 13, 13, + 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_adst_dct_8[6] = { 13, 13, 13, 13, 13, 13 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_adst_dct_8 = { + 8, // .txfm_size + 8, // .stage_num_col + 6, // .stage_num_row + // 0, // .log_scale + fwd_shift_adst_dct_8, // .shift + fwd_stage_range_col_adst_dct_8, // .stage_range_col + fwd_stage_range_row_adst_dct_8, // .stage_range_row + fwd_cos_bit_col_adst_dct_8, // .cos_bit_col + fwd_cos_bit_row_adst_dct_8, // .cos_bit_row + TXFM_TYPE_ADST8, // .txfm_type_col + TXFM_TYPE_DCT8 +}; // .txfm_type_row + +// ---------------- config fwd_adst_dct_16 ---------------- +static const int8_t fwd_shift_adst_dct_16[3] = { 2, -2, 0 }; +static const int8_t fwd_stage_range_col_adst_dct_16[10] = { + 15, 15, 16, 17, 17, 18, 18, 19, 19, 19 +}; +static const int8_t fwd_stage_range_row_adst_dct_16[8] = { 17, 18, 19, 20, + 20, 20, 20, 20 }; +static const int8_t fwd_cos_bit_col_adst_dct_16[10] = { 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13 }; +static const int8_t fwd_cos_bit_row_adst_dct_16[8] = { 12, 12, 12, 12, + 12, 12, 12, 12 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_adst_dct_16 = { + 16, // .txfm_size + 10, // .stage_num_col + 8, // .stage_num_row + // 0, // .log_scale + fwd_shift_adst_dct_16, // .shift + fwd_stage_range_col_adst_dct_16, // .stage_range_col + fwd_stage_range_row_adst_dct_16, // .stage_range_row + fwd_cos_bit_col_adst_dct_16, // .cos_bit_col + fwd_cos_bit_row_adst_dct_16, // .cos_bit_row + TXFM_TYPE_ADST16, // .txfm_type_col + TXFM_TYPE_DCT16 +}; // .txfm_type_row + +// ---------------- config fwd_adst_dct_32 ---------------- +static const int8_t fwd_shift_adst_dct_32[3] = { 2, -4, 0 }; +static const int8_t fwd_stage_range_col_adst_dct_32[12] = { + 15, 15, 16, 17, 17, 18, 18, 19, 19, 20, 20, 20 +}; +static const int8_t fwd_stage_range_row_adst_dct_32[10] = { + 16, 17, 18, 19, 20, 20, 20, 20, 20, 20 +}; +static const int8_t fwd_cos_bit_col_adst_dct_32[12] = { + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12 +}; +static const int8_t fwd_cos_bit_row_adst_dct_32[10] = { 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12 }; + +static const TXFM_2D_CFG fwd_txfm_2d_cfg_adst_dct_32 = { + 32, // .txfm_size + 12, // .stage_num_col + 10, // .stage_num_row + // 1, // .log_scale + fwd_shift_adst_dct_32, // .shift + fwd_stage_range_col_adst_dct_32, // .stage_range_col + fwd_stage_range_row_adst_dct_32, // .stage_range_row + fwd_cos_bit_col_adst_dct_32, // .cos_bit_col + fwd_cos_bit_row_adst_dct_32, // .cos_bit_row + TXFM_TYPE_ADST32, // .txfm_type_col + TXFM_TYPE_DCT32 +}; // .txfm_type_row +#endif // AV1_FWD_TXFM2D_CFG_H_ diff --git a/third_party/aom/av1/common/av1_inv_txfm1d.c b/third_party/aom/av1/common/av1_inv_txfm1d.c new file mode 100644 index 0000000000..54bbe9adf1 --- /dev/null +++ b/third_party/aom/av1/common/av1_inv_txfm1d.c @@ -0,0 +1,2334 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "av1/common/av1_inv_txfm1d.h" +#if CONFIG_COEFFICIENT_RANGE_CHECKING + +void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf, + int32_t size, int8_t bit) { + const int64_t maxValue = (1LL << (bit - 1)) - 1; + const int64_t minValue = -(1LL << (bit - 1)); + + for (int i = 0; i < size; ++i) { + if (buf[i] < minValue || buf[i] > maxValue) { + fprintf(stderr, "Error: coeffs contain out-of-range values\n"); + fprintf(stderr, "stage: %d\n", stage); + fprintf(stderr, "node: %d\n", i); + fprintf(stderr, "allowed range: [%" PRId64 ";%" PRId64 "]\n", minValue, + maxValue); + fprintf(stderr, "coeffs: "); + + fprintf(stderr, "["); + for (int j = 0; j < size; j++) { + if (j > 0) fprintf(stderr, ", "); + fprintf(stderr, "%d", input[j]); + } + fprintf(stderr, "]\n"); + assert(0); + } + } +} + +#define range_check(stage, input, buf, size, bit) \ + range_check_func(stage, input, buf, size, bit) +#else +#define range_check(stage, input, buf, size, bit) \ + { \ + (void)stage; \ + (void)input; \ + (void)buf; \ + (void)size; \ + (void)bit; \ + } +#endif + +// TODO(angiebird): Make 1-d txfm functions static +void av1_idct4_new(const int32_t *input, int32_t *output, const int8_t *cos_bit, + const int8_t *stage_range) { + const int32_t size = 4; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[4]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[0]; + bf1[1] = input[2]; + bf1[2] = input[1]; + bf1[3] = input[3]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[32], bf0[0], cospi[32], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(cospi[32], bf0[0], -cospi[32], bf0[1], cos_bit[stage]); + bf1[2] = half_btf(cospi[48], bf0[2], -cospi[16], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[16], bf0[2], cospi[48], bf0[3], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[3]; + bf1[1] = bf0[1] + bf0[2]; + bf1[2] = bf0[1] - bf0[2]; + bf1[3] = bf0[0] - bf0[3]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_idct8_new(const int32_t *input, int32_t *output, const int8_t *cos_bit, + const int8_t *stage_range) { + const int32_t size = 8; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[8]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[0]; + bf1[1] = input[4]; + bf1[2] = input[2]; + bf1[3] = input[6]; + bf1[4] = input[1]; + bf1[5] = input[5]; + bf1[6] = input[3]; + bf1[7] = input[7]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[56], bf0[4], -cospi[8], bf0[7], cos_bit[stage]); + bf1[5] = half_btf(cospi[24], bf0[5], -cospi[40], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[40], bf0[5], cospi[24], bf0[6], cos_bit[stage]); + bf1[7] = half_btf(cospi[8], bf0[4], cospi[56], bf0[7], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = half_btf(cospi[32], bf0[0], cospi[32], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(cospi[32], bf0[0], -cospi[32], bf0[1], cos_bit[stage]); + bf1[2] = half_btf(cospi[48], bf0[2], -cospi[16], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[16], bf0[2], cospi[48], bf0[3], cos_bit[stage]); + bf1[4] = bf0[4] + bf0[5]; + bf1[5] = bf0[4] - bf0[5]; + bf1[6] = -bf0[6] + bf0[7]; + bf1[7] = bf0[6] + bf0[7]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0] + bf0[3]; + bf1[1] = bf0[1] + bf0[2]; + bf1[2] = bf0[1] - bf0[2]; + bf1[3] = bf0[0] - bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = half_btf(-cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]); + bf1[7] = bf0[7]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[7]; + bf1[1] = bf0[1] + bf0[6]; + bf1[2] = bf0[2] + bf0[5]; + bf1[3] = bf0[3] + bf0[4]; + bf1[4] = bf0[3] - bf0[4]; + bf1[5] = bf0[2] - bf0[5]; + bf1[6] = bf0[1] - bf0[6]; + bf1[7] = bf0[0] - bf0[7]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_idct16_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 16; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[16]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[0]; + bf1[1] = input[8]; + bf1[2] = input[4]; + bf1[3] = input[12]; + bf1[4] = input[2]; + bf1[5] = input[10]; + bf1[6] = input[6]; + bf1[7] = input[14]; + bf1[8] = input[1]; + bf1[9] = input[9]; + bf1[10] = input[5]; + bf1[11] = input[13]; + bf1[12] = input[3]; + bf1[13] = input[11]; + bf1[14] = input[7]; + bf1[15] = input[15]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = half_btf(cospi[60], bf0[8], -cospi[4], bf0[15], cos_bit[stage]); + bf1[9] = half_btf(cospi[28], bf0[9], -cospi[36], bf0[14], cos_bit[stage]); + bf1[10] = half_btf(cospi[44], bf0[10], -cospi[20], bf0[13], cos_bit[stage]); + bf1[11] = half_btf(cospi[12], bf0[11], -cospi[52], bf0[12], cos_bit[stage]); + bf1[12] = half_btf(cospi[52], bf0[11], cospi[12], bf0[12], cos_bit[stage]); + bf1[13] = half_btf(cospi[20], bf0[10], cospi[44], bf0[13], cos_bit[stage]); + bf1[14] = half_btf(cospi[36], bf0[9], cospi[28], bf0[14], cos_bit[stage]); + bf1[15] = half_btf(cospi[4], bf0[8], cospi[60], bf0[15], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[56], bf0[4], -cospi[8], bf0[7], cos_bit[stage]); + bf1[5] = half_btf(cospi[24], bf0[5], -cospi[40], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[40], bf0[5], cospi[24], bf0[6], cos_bit[stage]); + bf1[7] = half_btf(cospi[8], bf0[4], cospi[56], bf0[7], cos_bit[stage]); + bf1[8] = bf0[8] + bf0[9]; + bf1[9] = bf0[8] - bf0[9]; + bf1[10] = -bf0[10] + bf0[11]; + bf1[11] = bf0[10] + bf0[11]; + bf1[12] = bf0[12] + bf0[13]; + bf1[13] = bf0[12] - bf0[13]; + bf1[14] = -bf0[14] + bf0[15]; + bf1[15] = bf0[14] + bf0[15]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[32], bf0[0], cospi[32], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(cospi[32], bf0[0], -cospi[32], bf0[1], cos_bit[stage]); + bf1[2] = half_btf(cospi[48], bf0[2], -cospi[16], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[16], bf0[2], cospi[48], bf0[3], cos_bit[stage]); + bf1[4] = bf0[4] + bf0[5]; + bf1[5] = bf0[4] - bf0[5]; + bf1[6] = -bf0[6] + bf0[7]; + bf1[7] = bf0[6] + bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = half_btf(-cospi[16], bf0[9], cospi[48], bf0[14], cos_bit[stage]); + bf1[10] = half_btf(-cospi[48], bf0[10], -cospi[16], bf0[13], cos_bit[stage]); + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = half_btf(-cospi[16], bf0[10], cospi[48], bf0[13], cos_bit[stage]); + bf1[14] = half_btf(cospi[48], bf0[9], cospi[16], bf0[14], cos_bit[stage]); + bf1[15] = bf0[15]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[3]; + bf1[1] = bf0[1] + bf0[2]; + bf1[2] = bf0[1] - bf0[2]; + bf1[3] = bf0[0] - bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = half_btf(-cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]); + bf1[7] = bf0[7]; + bf1[8] = bf0[8] + bf0[11]; + bf1[9] = bf0[9] + bf0[10]; + bf1[10] = bf0[9] - bf0[10]; + bf1[11] = bf0[8] - bf0[11]; + bf1[12] = -bf0[12] + bf0[15]; + bf1[13] = -bf0[13] + bf0[14]; + bf1[14] = bf0[13] + bf0[14]; + bf1[15] = bf0[12] + bf0[15]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 6 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0] + bf0[7]; + bf1[1] = bf0[1] + bf0[6]; + bf1[2] = bf0[2] + bf0[5]; + bf1[3] = bf0[3] + bf0[4]; + bf1[4] = bf0[3] - bf0[4]; + bf1[5] = bf0[2] - bf0[5]; + bf1[6] = bf0[1] - bf0[6]; + bf1[7] = bf0[0] - bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = half_btf(-cospi[32], bf0[10], cospi[32], bf0[13], cos_bit[stage]); + bf1[11] = half_btf(-cospi[32], bf0[11], cospi[32], bf0[12], cos_bit[stage]); + bf1[12] = half_btf(cospi[32], bf0[11], cospi[32], bf0[12], cos_bit[stage]); + bf1[13] = half_btf(cospi[32], bf0[10], cospi[32], bf0[13], cos_bit[stage]); + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 7 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[15]; + bf1[1] = bf0[1] + bf0[14]; + bf1[2] = bf0[2] + bf0[13]; + bf1[3] = bf0[3] + bf0[12]; + bf1[4] = bf0[4] + bf0[11]; + bf1[5] = bf0[5] + bf0[10]; + bf1[6] = bf0[6] + bf0[9]; + bf1[7] = bf0[7] + bf0[8]; + bf1[8] = bf0[7] - bf0[8]; + bf1[9] = bf0[6] - bf0[9]; + bf1[10] = bf0[5] - bf0[10]; + bf1[11] = bf0[4] - bf0[11]; + bf1[12] = bf0[3] - bf0[12]; + bf1[13] = bf0[2] - bf0[13]; + bf1[14] = bf0[1] - bf0[14]; + bf1[15] = bf0[0] - bf0[15]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_idct32_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 32; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[32]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[0]; + bf1[1] = input[16]; + bf1[2] = input[8]; + bf1[3] = input[24]; + bf1[4] = input[4]; + bf1[5] = input[20]; + bf1[6] = input[12]; + bf1[7] = input[28]; + bf1[8] = input[2]; + bf1[9] = input[18]; + bf1[10] = input[10]; + bf1[11] = input[26]; + bf1[12] = input[6]; + bf1[13] = input[22]; + bf1[14] = input[14]; + bf1[15] = input[30]; + bf1[16] = input[1]; + bf1[17] = input[17]; + bf1[18] = input[9]; + bf1[19] = input[25]; + bf1[20] = input[5]; + bf1[21] = input[21]; + bf1[22] = input[13]; + bf1[23] = input[29]; + bf1[24] = input[3]; + bf1[25] = input[19]; + bf1[26] = input[11]; + bf1[27] = input[27]; + bf1[28] = input[7]; + bf1[29] = input[23]; + bf1[30] = input[15]; + bf1[31] = input[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = bf0[10]; + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = bf0[13]; + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = half_btf(cospi[62], bf0[16], -cospi[2], bf0[31], cos_bit[stage]); + bf1[17] = half_btf(cospi[30], bf0[17], -cospi[34], bf0[30], cos_bit[stage]); + bf1[18] = half_btf(cospi[46], bf0[18], -cospi[18], bf0[29], cos_bit[stage]); + bf1[19] = half_btf(cospi[14], bf0[19], -cospi[50], bf0[28], cos_bit[stage]); + bf1[20] = half_btf(cospi[54], bf0[20], -cospi[10], bf0[27], cos_bit[stage]); + bf1[21] = half_btf(cospi[22], bf0[21], -cospi[42], bf0[26], cos_bit[stage]); + bf1[22] = half_btf(cospi[38], bf0[22], -cospi[26], bf0[25], cos_bit[stage]); + bf1[23] = half_btf(cospi[6], bf0[23], -cospi[58], bf0[24], cos_bit[stage]); + bf1[24] = half_btf(cospi[58], bf0[23], cospi[6], bf0[24], cos_bit[stage]); + bf1[25] = half_btf(cospi[26], bf0[22], cospi[38], bf0[25], cos_bit[stage]); + bf1[26] = half_btf(cospi[42], bf0[21], cospi[22], bf0[26], cos_bit[stage]); + bf1[27] = half_btf(cospi[10], bf0[20], cospi[54], bf0[27], cos_bit[stage]); + bf1[28] = half_btf(cospi[50], bf0[19], cospi[14], bf0[28], cos_bit[stage]); + bf1[29] = half_btf(cospi[18], bf0[18], cospi[46], bf0[29], cos_bit[stage]); + bf1[30] = half_btf(cospi[34], bf0[17], cospi[30], bf0[30], cos_bit[stage]); + bf1[31] = half_btf(cospi[2], bf0[16], cospi[62], bf0[31], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = half_btf(cospi[60], bf0[8], -cospi[4], bf0[15], cos_bit[stage]); + bf1[9] = half_btf(cospi[28], bf0[9], -cospi[36], bf0[14], cos_bit[stage]); + bf1[10] = half_btf(cospi[44], bf0[10], -cospi[20], bf0[13], cos_bit[stage]); + bf1[11] = half_btf(cospi[12], bf0[11], -cospi[52], bf0[12], cos_bit[stage]); + bf1[12] = half_btf(cospi[52], bf0[11], cospi[12], bf0[12], cos_bit[stage]); + bf1[13] = half_btf(cospi[20], bf0[10], cospi[44], bf0[13], cos_bit[stage]); + bf1[14] = half_btf(cospi[36], bf0[9], cospi[28], bf0[14], cos_bit[stage]); + bf1[15] = half_btf(cospi[4], bf0[8], cospi[60], bf0[15], cos_bit[stage]); + bf1[16] = bf0[16] + bf0[17]; + bf1[17] = bf0[16] - bf0[17]; + bf1[18] = -bf0[18] + bf0[19]; + bf1[19] = bf0[18] + bf0[19]; + bf1[20] = bf0[20] + bf0[21]; + bf1[21] = bf0[20] - bf0[21]; + bf1[22] = -bf0[22] + bf0[23]; + bf1[23] = bf0[22] + bf0[23]; + bf1[24] = bf0[24] + bf0[25]; + bf1[25] = bf0[24] - bf0[25]; + bf1[26] = -bf0[26] + bf0[27]; + bf1[27] = bf0[26] + bf0[27]; + bf1[28] = bf0[28] + bf0[29]; + bf1[29] = bf0[28] - bf0[29]; + bf1[30] = -bf0[30] + bf0[31]; + bf1[31] = bf0[30] + bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[56], bf0[4], -cospi[8], bf0[7], cos_bit[stage]); + bf1[5] = half_btf(cospi[24], bf0[5], -cospi[40], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[40], bf0[5], cospi[24], bf0[6], cos_bit[stage]); + bf1[7] = half_btf(cospi[8], bf0[4], cospi[56], bf0[7], cos_bit[stage]); + bf1[8] = bf0[8] + bf0[9]; + bf1[9] = bf0[8] - bf0[9]; + bf1[10] = -bf0[10] + bf0[11]; + bf1[11] = bf0[10] + bf0[11]; + bf1[12] = bf0[12] + bf0[13]; + bf1[13] = bf0[12] - bf0[13]; + bf1[14] = -bf0[14] + bf0[15]; + bf1[15] = bf0[14] + bf0[15]; + bf1[16] = bf0[16]; + bf1[17] = half_btf(-cospi[8], bf0[17], cospi[56], bf0[30], cos_bit[stage]); + bf1[18] = half_btf(-cospi[56], bf0[18], -cospi[8], bf0[29], cos_bit[stage]); + bf1[19] = bf0[19]; + bf1[20] = bf0[20]; + bf1[21] = half_btf(-cospi[40], bf0[21], cospi[24], bf0[26], cos_bit[stage]); + bf1[22] = half_btf(-cospi[24], bf0[22], -cospi[40], bf0[25], cos_bit[stage]); + bf1[23] = bf0[23]; + bf1[24] = bf0[24]; + bf1[25] = half_btf(-cospi[40], bf0[22], cospi[24], bf0[25], cos_bit[stage]); + bf1[26] = half_btf(cospi[24], bf0[21], cospi[40], bf0[26], cos_bit[stage]); + bf1[27] = bf0[27]; + bf1[28] = bf0[28]; + bf1[29] = half_btf(-cospi[8], bf0[18], cospi[56], bf0[29], cos_bit[stage]); + bf1[30] = half_btf(cospi[56], bf0[17], cospi[8], bf0[30], cos_bit[stage]); + bf1[31] = bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = half_btf(cospi[32], bf0[0], cospi[32], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(cospi[32], bf0[0], -cospi[32], bf0[1], cos_bit[stage]); + bf1[2] = half_btf(cospi[48], bf0[2], -cospi[16], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[16], bf0[2], cospi[48], bf0[3], cos_bit[stage]); + bf1[4] = bf0[4] + bf0[5]; + bf1[5] = bf0[4] - bf0[5]; + bf1[6] = -bf0[6] + bf0[7]; + bf1[7] = bf0[6] + bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = half_btf(-cospi[16], bf0[9], cospi[48], bf0[14], cos_bit[stage]); + bf1[10] = half_btf(-cospi[48], bf0[10], -cospi[16], bf0[13], cos_bit[stage]); + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = half_btf(-cospi[16], bf0[10], cospi[48], bf0[13], cos_bit[stage]); + bf1[14] = half_btf(cospi[48], bf0[9], cospi[16], bf0[14], cos_bit[stage]); + bf1[15] = bf0[15]; + bf1[16] = bf0[16] + bf0[19]; + bf1[17] = bf0[17] + bf0[18]; + bf1[18] = bf0[17] - bf0[18]; + bf1[19] = bf0[16] - bf0[19]; + bf1[20] = -bf0[20] + bf0[23]; + bf1[21] = -bf0[21] + bf0[22]; + bf1[22] = bf0[21] + bf0[22]; + bf1[23] = bf0[20] + bf0[23]; + bf1[24] = bf0[24] + bf0[27]; + bf1[25] = bf0[25] + bf0[26]; + bf1[26] = bf0[25] - bf0[26]; + bf1[27] = bf0[24] - bf0[27]; + bf1[28] = -bf0[28] + bf0[31]; + bf1[29] = -bf0[29] + bf0[30]; + bf1[30] = bf0[29] + bf0[30]; + bf1[31] = bf0[28] + bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 6 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0] + bf0[3]; + bf1[1] = bf0[1] + bf0[2]; + bf1[2] = bf0[1] - bf0[2]; + bf1[3] = bf0[0] - bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = half_btf(-cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]); + bf1[7] = bf0[7]; + bf1[8] = bf0[8] + bf0[11]; + bf1[9] = bf0[9] + bf0[10]; + bf1[10] = bf0[9] - bf0[10]; + bf1[11] = bf0[8] - bf0[11]; + bf1[12] = -bf0[12] + bf0[15]; + bf1[13] = -bf0[13] + bf0[14]; + bf1[14] = bf0[13] + bf0[14]; + bf1[15] = bf0[12] + bf0[15]; + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = half_btf(-cospi[16], bf0[18], cospi[48], bf0[29], cos_bit[stage]); + bf1[19] = half_btf(-cospi[16], bf0[19], cospi[48], bf0[28], cos_bit[stage]); + bf1[20] = half_btf(-cospi[48], bf0[20], -cospi[16], bf0[27], cos_bit[stage]); + bf1[21] = half_btf(-cospi[48], bf0[21], -cospi[16], bf0[26], cos_bit[stage]); + bf1[22] = bf0[22]; + bf1[23] = bf0[23]; + bf1[24] = bf0[24]; + bf1[25] = bf0[25]; + bf1[26] = half_btf(-cospi[16], bf0[21], cospi[48], bf0[26], cos_bit[stage]); + bf1[27] = half_btf(-cospi[16], bf0[20], cospi[48], bf0[27], cos_bit[stage]); + bf1[28] = half_btf(cospi[48], bf0[19], cospi[16], bf0[28], cos_bit[stage]); + bf1[29] = half_btf(cospi[48], bf0[18], cospi[16], bf0[29], cos_bit[stage]); + bf1[30] = bf0[30]; + bf1[31] = bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 7 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[7]; + bf1[1] = bf0[1] + bf0[6]; + bf1[2] = bf0[2] + bf0[5]; + bf1[3] = bf0[3] + bf0[4]; + bf1[4] = bf0[3] - bf0[4]; + bf1[5] = bf0[2] - bf0[5]; + bf1[6] = bf0[1] - bf0[6]; + bf1[7] = bf0[0] - bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = half_btf(-cospi[32], bf0[10], cospi[32], bf0[13], cos_bit[stage]); + bf1[11] = half_btf(-cospi[32], bf0[11], cospi[32], bf0[12], cos_bit[stage]); + bf1[12] = half_btf(cospi[32], bf0[11], cospi[32], bf0[12], cos_bit[stage]); + bf1[13] = half_btf(cospi[32], bf0[10], cospi[32], bf0[13], cos_bit[stage]); + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = bf0[16] + bf0[23]; + bf1[17] = bf0[17] + bf0[22]; + bf1[18] = bf0[18] + bf0[21]; + bf1[19] = bf0[19] + bf0[20]; + bf1[20] = bf0[19] - bf0[20]; + bf1[21] = bf0[18] - bf0[21]; + bf1[22] = bf0[17] - bf0[22]; + bf1[23] = bf0[16] - bf0[23]; + bf1[24] = -bf0[24] + bf0[31]; + bf1[25] = -bf0[25] + bf0[30]; + bf1[26] = -bf0[26] + bf0[29]; + bf1[27] = -bf0[27] + bf0[28]; + bf1[28] = bf0[27] + bf0[28]; + bf1[29] = bf0[26] + bf0[29]; + bf1[30] = bf0[25] + bf0[30]; + bf1[31] = bf0[24] + bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 8 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0] + bf0[15]; + bf1[1] = bf0[1] + bf0[14]; + bf1[2] = bf0[2] + bf0[13]; + bf1[3] = bf0[3] + bf0[12]; + bf1[4] = bf0[4] + bf0[11]; + bf1[5] = bf0[5] + bf0[10]; + bf1[6] = bf0[6] + bf0[9]; + bf1[7] = bf0[7] + bf0[8]; + bf1[8] = bf0[7] - bf0[8]; + bf1[9] = bf0[6] - bf0[9]; + bf1[10] = bf0[5] - bf0[10]; + bf1[11] = bf0[4] - bf0[11]; + bf1[12] = bf0[3] - bf0[12]; + bf1[13] = bf0[2] - bf0[13]; + bf1[14] = bf0[1] - bf0[14]; + bf1[15] = bf0[0] - bf0[15]; + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = bf0[18]; + bf1[19] = bf0[19]; + bf1[20] = half_btf(-cospi[32], bf0[20], cospi[32], bf0[27], cos_bit[stage]); + bf1[21] = half_btf(-cospi[32], bf0[21], cospi[32], bf0[26], cos_bit[stage]); + bf1[22] = half_btf(-cospi[32], bf0[22], cospi[32], bf0[25], cos_bit[stage]); + bf1[23] = half_btf(-cospi[32], bf0[23], cospi[32], bf0[24], cos_bit[stage]); + bf1[24] = half_btf(cospi[32], bf0[23], cospi[32], bf0[24], cos_bit[stage]); + bf1[25] = half_btf(cospi[32], bf0[22], cospi[32], bf0[25], cos_bit[stage]); + bf1[26] = half_btf(cospi[32], bf0[21], cospi[32], bf0[26], cos_bit[stage]); + bf1[27] = half_btf(cospi[32], bf0[20], cospi[32], bf0[27], cos_bit[stage]); + bf1[28] = bf0[28]; + bf1[29] = bf0[29]; + bf1[30] = bf0[30]; + bf1[31] = bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 9 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[31]; + bf1[1] = bf0[1] + bf0[30]; + bf1[2] = bf0[2] + bf0[29]; + bf1[3] = bf0[3] + bf0[28]; + bf1[4] = bf0[4] + bf0[27]; + bf1[5] = bf0[5] + bf0[26]; + bf1[6] = bf0[6] + bf0[25]; + bf1[7] = bf0[7] + bf0[24]; + bf1[8] = bf0[8] + bf0[23]; + bf1[9] = bf0[9] + bf0[22]; + bf1[10] = bf0[10] + bf0[21]; + bf1[11] = bf0[11] + bf0[20]; + bf1[12] = bf0[12] + bf0[19]; + bf1[13] = bf0[13] + bf0[18]; + bf1[14] = bf0[14] + bf0[17]; + bf1[15] = bf0[15] + bf0[16]; + bf1[16] = bf0[15] - bf0[16]; + bf1[17] = bf0[14] - bf0[17]; + bf1[18] = bf0[13] - bf0[18]; + bf1[19] = bf0[12] - bf0[19]; + bf1[20] = bf0[11] - bf0[20]; + bf1[21] = bf0[10] - bf0[21]; + bf1[22] = bf0[9] - bf0[22]; + bf1[23] = bf0[8] - bf0[23]; + bf1[24] = bf0[7] - bf0[24]; + bf1[25] = bf0[6] - bf0[25]; + bf1[26] = bf0[5] - bf0[26]; + bf1[27] = bf0[4] - bf0[27]; + bf1[28] = bf0[3] - bf0[28]; + bf1[29] = bf0[2] - bf0[29]; + bf1[30] = bf0[1] - bf0[30]; + bf1[31] = bf0[0] - bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_iadst4_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 4; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[4]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[0]; + bf1[1] = -input[3]; + bf1[2] = -input[1]; + bf1[3] = input[2]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = half_btf(cospi[32], bf0[2], cospi[32], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[32], bf0[2], -cospi[32], bf0[3], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[2]; + bf1[1] = bf0[1] + bf0[3]; + bf1[2] = bf0[0] - bf0[2]; + bf1[3] = bf0[1] - bf0[3]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[8], bf0[0], cospi[56], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(cospi[56], bf0[0], -cospi[8], bf0[1], cos_bit[stage]); + bf1[2] = half_btf(cospi[40], bf0[2], cospi[24], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[24], bf0[2], -cospi[40], bf0[3], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[1]; + bf1[1] = bf0[2]; + bf1[2] = bf0[3]; + bf1[3] = bf0[0]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_iadst8_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 8; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[8]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[0]; + bf1[1] = -input[7]; + bf1[2] = -input[3]; + bf1[3] = input[4]; + bf1[4] = -input[1]; + bf1[5] = input[6]; + bf1[6] = input[2]; + bf1[7] = -input[5]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = half_btf(cospi[32], bf0[2], cospi[32], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[32], bf0[2], -cospi[32], bf0[3], cos_bit[stage]); + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = half_btf(cospi[32], bf0[6], cospi[32], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(cospi[32], bf0[6], -cospi[32], bf0[7], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[2]; + bf1[1] = bf0[1] + bf0[3]; + bf1[2] = bf0[0] - bf0[2]; + bf1[3] = bf0[1] - bf0[3]; + bf1[4] = bf0[4] + bf0[6]; + bf1[5] = bf0[5] + bf0[7]; + bf1[6] = bf0[4] - bf0[6]; + bf1[7] = bf0[5] - bf0[7]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[16], bf0[4], cospi[48], bf0[5], cos_bit[stage]); + bf1[5] = half_btf(cospi[48], bf0[4], -cospi[16], bf0[5], cos_bit[stage]); + bf1[6] = half_btf(-cospi[48], bf0[6], cospi[16], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(cospi[16], bf0[6], cospi[48], bf0[7], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[4]; + bf1[1] = bf0[1] + bf0[5]; + bf1[2] = bf0[2] + bf0[6]; + bf1[3] = bf0[3] + bf0[7]; + bf1[4] = bf0[0] - bf0[4]; + bf1[5] = bf0[1] - bf0[5]; + bf1[6] = bf0[2] - bf0[6]; + bf1[7] = bf0[3] - bf0[7]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 6 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[4], bf0[0], cospi[60], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(cospi[60], bf0[0], -cospi[4], bf0[1], cos_bit[stage]); + bf1[2] = half_btf(cospi[20], bf0[2], cospi[44], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[44], bf0[2], -cospi[20], bf0[3], cos_bit[stage]); + bf1[4] = half_btf(cospi[36], bf0[4], cospi[28], bf0[5], cos_bit[stage]); + bf1[5] = half_btf(cospi[28], bf0[4], -cospi[36], bf0[5], cos_bit[stage]); + bf1[6] = half_btf(cospi[52], bf0[6], cospi[12], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(cospi[12], bf0[6], -cospi[52], bf0[7], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 7 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[1]; + bf1[1] = bf0[6]; + bf1[2] = bf0[3]; + bf1[3] = bf0[4]; + bf1[4] = bf0[5]; + bf1[5] = bf0[2]; + bf1[6] = bf0[7]; + bf1[7] = bf0[0]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_iadst16_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 16; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[16]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[0]; + bf1[1] = -input[15]; + bf1[2] = -input[7]; + bf1[3] = input[8]; + bf1[4] = -input[3]; + bf1[5] = input[12]; + bf1[6] = input[4]; + bf1[7] = -input[11]; + bf1[8] = -input[1]; + bf1[9] = input[14]; + bf1[10] = input[6]; + bf1[11] = -input[9]; + bf1[12] = input[2]; + bf1[13] = -input[13]; + bf1[14] = -input[5]; + bf1[15] = input[10]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = half_btf(cospi[32], bf0[2], cospi[32], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[32], bf0[2], -cospi[32], bf0[3], cos_bit[stage]); + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = half_btf(cospi[32], bf0[6], cospi[32], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(cospi[32], bf0[6], -cospi[32], bf0[7], cos_bit[stage]); + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = half_btf(cospi[32], bf0[10], cospi[32], bf0[11], cos_bit[stage]); + bf1[11] = half_btf(cospi[32], bf0[10], -cospi[32], bf0[11], cos_bit[stage]); + bf1[12] = bf0[12]; + bf1[13] = bf0[13]; + bf1[14] = half_btf(cospi[32], bf0[14], cospi[32], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(cospi[32], bf0[14], -cospi[32], bf0[15], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[2]; + bf1[1] = bf0[1] + bf0[3]; + bf1[2] = bf0[0] - bf0[2]; + bf1[3] = bf0[1] - bf0[3]; + bf1[4] = bf0[4] + bf0[6]; + bf1[5] = bf0[5] + bf0[7]; + bf1[6] = bf0[4] - bf0[6]; + bf1[7] = bf0[5] - bf0[7]; + bf1[8] = bf0[8] + bf0[10]; + bf1[9] = bf0[9] + bf0[11]; + bf1[10] = bf0[8] - bf0[10]; + bf1[11] = bf0[9] - bf0[11]; + bf1[12] = bf0[12] + bf0[14]; + bf1[13] = bf0[13] + bf0[15]; + bf1[14] = bf0[12] - bf0[14]; + bf1[15] = bf0[13] - bf0[15]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[16], bf0[4], cospi[48], bf0[5], cos_bit[stage]); + bf1[5] = half_btf(cospi[48], bf0[4], -cospi[16], bf0[5], cos_bit[stage]); + bf1[6] = half_btf(-cospi[48], bf0[6], cospi[16], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(cospi[16], bf0[6], cospi[48], bf0[7], cos_bit[stage]); + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = bf0[10]; + bf1[11] = bf0[11]; + bf1[12] = half_btf(cospi[16], bf0[12], cospi[48], bf0[13], cos_bit[stage]); + bf1[13] = half_btf(cospi[48], bf0[12], -cospi[16], bf0[13], cos_bit[stage]); + bf1[14] = half_btf(-cospi[48], bf0[14], cospi[16], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(cospi[16], bf0[14], cospi[48], bf0[15], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[4]; + bf1[1] = bf0[1] + bf0[5]; + bf1[2] = bf0[2] + bf0[6]; + bf1[3] = bf0[3] + bf0[7]; + bf1[4] = bf0[0] - bf0[4]; + bf1[5] = bf0[1] - bf0[5]; + bf1[6] = bf0[2] - bf0[6]; + bf1[7] = bf0[3] - bf0[7]; + bf1[8] = bf0[8] + bf0[12]; + bf1[9] = bf0[9] + bf0[13]; + bf1[10] = bf0[10] + bf0[14]; + bf1[11] = bf0[11] + bf0[15]; + bf1[12] = bf0[8] - bf0[12]; + bf1[13] = bf0[9] - bf0[13]; + bf1[14] = bf0[10] - bf0[14]; + bf1[15] = bf0[11] - bf0[15]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 6 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = half_btf(cospi[8], bf0[8], cospi[56], bf0[9], cos_bit[stage]); + bf1[9] = half_btf(cospi[56], bf0[8], -cospi[8], bf0[9], cos_bit[stage]); + bf1[10] = half_btf(cospi[40], bf0[10], cospi[24], bf0[11], cos_bit[stage]); + bf1[11] = half_btf(cospi[24], bf0[10], -cospi[40], bf0[11], cos_bit[stage]); + bf1[12] = half_btf(-cospi[56], bf0[12], cospi[8], bf0[13], cos_bit[stage]); + bf1[13] = half_btf(cospi[8], bf0[12], cospi[56], bf0[13], cos_bit[stage]); + bf1[14] = half_btf(-cospi[24], bf0[14], cospi[40], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(cospi[40], bf0[14], cospi[24], bf0[15], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 7 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[8]; + bf1[1] = bf0[1] + bf0[9]; + bf1[2] = bf0[2] + bf0[10]; + bf1[3] = bf0[3] + bf0[11]; + bf1[4] = bf0[4] + bf0[12]; + bf1[5] = bf0[5] + bf0[13]; + bf1[6] = bf0[6] + bf0[14]; + bf1[7] = bf0[7] + bf0[15]; + bf1[8] = bf0[0] - bf0[8]; + bf1[9] = bf0[1] - bf0[9]; + bf1[10] = bf0[2] - bf0[10]; + bf1[11] = bf0[3] - bf0[11]; + bf1[12] = bf0[4] - bf0[12]; + bf1[13] = bf0[5] - bf0[13]; + bf1[14] = bf0[6] - bf0[14]; + bf1[15] = bf0[7] - bf0[15]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 8 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[2], bf0[0], cospi[62], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(cospi[62], bf0[0], -cospi[2], bf0[1], cos_bit[stage]); + bf1[2] = half_btf(cospi[10], bf0[2], cospi[54], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[54], bf0[2], -cospi[10], bf0[3], cos_bit[stage]); + bf1[4] = half_btf(cospi[18], bf0[4], cospi[46], bf0[5], cos_bit[stage]); + bf1[5] = half_btf(cospi[46], bf0[4], -cospi[18], bf0[5], cos_bit[stage]); + bf1[6] = half_btf(cospi[26], bf0[6], cospi[38], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(cospi[38], bf0[6], -cospi[26], bf0[7], cos_bit[stage]); + bf1[8] = half_btf(cospi[34], bf0[8], cospi[30], bf0[9], cos_bit[stage]); + bf1[9] = half_btf(cospi[30], bf0[8], -cospi[34], bf0[9], cos_bit[stage]); + bf1[10] = half_btf(cospi[42], bf0[10], cospi[22], bf0[11], cos_bit[stage]); + bf1[11] = half_btf(cospi[22], bf0[10], -cospi[42], bf0[11], cos_bit[stage]); + bf1[12] = half_btf(cospi[50], bf0[12], cospi[14], bf0[13], cos_bit[stage]); + bf1[13] = half_btf(cospi[14], bf0[12], -cospi[50], bf0[13], cos_bit[stage]); + bf1[14] = half_btf(cospi[58], bf0[14], cospi[6], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(cospi[6], bf0[14], -cospi[58], bf0[15], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 9 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[1]; + bf1[1] = bf0[14]; + bf1[2] = bf0[3]; + bf1[3] = bf0[12]; + bf1[4] = bf0[5]; + bf1[5] = bf0[10]; + bf1[6] = bf0[7]; + bf1[7] = bf0[8]; + bf1[8] = bf0[9]; + bf1[9] = bf0[6]; + bf1[10] = bf0[11]; + bf1[11] = bf0[4]; + bf1[12] = bf0[13]; + bf1[13] = bf0[2]; + bf1[14] = bf0[15]; + bf1[15] = bf0[0]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +void av1_iadst32_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 32; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[32]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + bf1 = output; + bf1[0] = input[0]; + bf1[1] = -input[31]; + bf1[2] = -input[15]; + bf1[3] = input[16]; + bf1[4] = -input[7]; + bf1[5] = input[24]; + bf1[6] = input[8]; + bf1[7] = -input[23]; + bf1[8] = -input[3]; + bf1[9] = input[28]; + bf1[10] = input[12]; + bf1[11] = -input[19]; + bf1[12] = input[4]; + bf1[13] = -input[27]; + bf1[14] = -input[11]; + bf1[15] = input[20]; + bf1[16] = -input[1]; + bf1[17] = input[30]; + bf1[18] = input[14]; + bf1[19] = -input[17]; + bf1[20] = input[6]; + bf1[21] = -input[25]; + bf1[22] = -input[9]; + bf1[23] = input[22]; + bf1[24] = input[2]; + bf1[25] = -input[29]; + bf1[26] = -input[13]; + bf1[27] = input[18]; + bf1[28] = -input[5]; + bf1[29] = input[26]; + bf1[30] = input[10]; + bf1[31] = -input[21]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = half_btf(cospi[32], bf0[2], cospi[32], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[32], bf0[2], -cospi[32], bf0[3], cos_bit[stage]); + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = half_btf(cospi[32], bf0[6], cospi[32], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(cospi[32], bf0[6], -cospi[32], bf0[7], cos_bit[stage]); + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = half_btf(cospi[32], bf0[10], cospi[32], bf0[11], cos_bit[stage]); + bf1[11] = half_btf(cospi[32], bf0[10], -cospi[32], bf0[11], cos_bit[stage]); + bf1[12] = bf0[12]; + bf1[13] = bf0[13]; + bf1[14] = half_btf(cospi[32], bf0[14], cospi[32], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(cospi[32], bf0[14], -cospi[32], bf0[15], cos_bit[stage]); + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = half_btf(cospi[32], bf0[18], cospi[32], bf0[19], cos_bit[stage]); + bf1[19] = half_btf(cospi[32], bf0[18], -cospi[32], bf0[19], cos_bit[stage]); + bf1[20] = bf0[20]; + bf1[21] = bf0[21]; + bf1[22] = half_btf(cospi[32], bf0[22], cospi[32], bf0[23], cos_bit[stage]); + bf1[23] = half_btf(cospi[32], bf0[22], -cospi[32], bf0[23], cos_bit[stage]); + bf1[24] = bf0[24]; + bf1[25] = bf0[25]; + bf1[26] = half_btf(cospi[32], bf0[26], cospi[32], bf0[27], cos_bit[stage]); + bf1[27] = half_btf(cospi[32], bf0[26], -cospi[32], bf0[27], cos_bit[stage]); + bf1[28] = bf0[28]; + bf1[29] = bf0[29]; + bf1[30] = half_btf(cospi[32], bf0[30], cospi[32], bf0[31], cos_bit[stage]); + bf1[31] = half_btf(cospi[32], bf0[30], -cospi[32], bf0[31], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[2]; + bf1[1] = bf0[1] + bf0[3]; + bf1[2] = bf0[0] - bf0[2]; + bf1[3] = bf0[1] - bf0[3]; + bf1[4] = bf0[4] + bf0[6]; + bf1[5] = bf0[5] + bf0[7]; + bf1[6] = bf0[4] - bf0[6]; + bf1[7] = bf0[5] - bf0[7]; + bf1[8] = bf0[8] + bf0[10]; + bf1[9] = bf0[9] + bf0[11]; + bf1[10] = bf0[8] - bf0[10]; + bf1[11] = bf0[9] - bf0[11]; + bf1[12] = bf0[12] + bf0[14]; + bf1[13] = bf0[13] + bf0[15]; + bf1[14] = bf0[12] - bf0[14]; + bf1[15] = bf0[13] - bf0[15]; + bf1[16] = bf0[16] + bf0[18]; + bf1[17] = bf0[17] + bf0[19]; + bf1[18] = bf0[16] - bf0[18]; + bf1[19] = bf0[17] - bf0[19]; + bf1[20] = bf0[20] + bf0[22]; + bf1[21] = bf0[21] + bf0[23]; + bf1[22] = bf0[20] - bf0[22]; + bf1[23] = bf0[21] - bf0[23]; + bf1[24] = bf0[24] + bf0[26]; + bf1[25] = bf0[25] + bf0[27]; + bf1[26] = bf0[24] - bf0[26]; + bf1[27] = bf0[25] - bf0[27]; + bf1[28] = bf0[28] + bf0[30]; + bf1[29] = bf0[29] + bf0[31]; + bf1[30] = bf0[28] - bf0[30]; + bf1[31] = bf0[29] - bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[16], bf0[4], cospi[48], bf0[5], cos_bit[stage]); + bf1[5] = half_btf(cospi[48], bf0[4], -cospi[16], bf0[5], cos_bit[stage]); + bf1[6] = half_btf(-cospi[48], bf0[6], cospi[16], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(cospi[16], bf0[6], cospi[48], bf0[7], cos_bit[stage]); + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = bf0[10]; + bf1[11] = bf0[11]; + bf1[12] = half_btf(cospi[16], bf0[12], cospi[48], bf0[13], cos_bit[stage]); + bf1[13] = half_btf(cospi[48], bf0[12], -cospi[16], bf0[13], cos_bit[stage]); + bf1[14] = half_btf(-cospi[48], bf0[14], cospi[16], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(cospi[16], bf0[14], cospi[48], bf0[15], cos_bit[stage]); + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = bf0[18]; + bf1[19] = bf0[19]; + bf1[20] = half_btf(cospi[16], bf0[20], cospi[48], bf0[21], cos_bit[stage]); + bf1[21] = half_btf(cospi[48], bf0[20], -cospi[16], bf0[21], cos_bit[stage]); + bf1[22] = half_btf(-cospi[48], bf0[22], cospi[16], bf0[23], cos_bit[stage]); + bf1[23] = half_btf(cospi[16], bf0[22], cospi[48], bf0[23], cos_bit[stage]); + bf1[24] = bf0[24]; + bf1[25] = bf0[25]; + bf1[26] = bf0[26]; + bf1[27] = bf0[27]; + bf1[28] = half_btf(cospi[16], bf0[28], cospi[48], bf0[29], cos_bit[stage]); + bf1[29] = half_btf(cospi[48], bf0[28], -cospi[16], bf0[29], cos_bit[stage]); + bf1[30] = half_btf(-cospi[48], bf0[30], cospi[16], bf0[31], cos_bit[stage]); + bf1[31] = half_btf(cospi[16], bf0[30], cospi[48], bf0[31], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[4]; + bf1[1] = bf0[1] + bf0[5]; + bf1[2] = bf0[2] + bf0[6]; + bf1[3] = bf0[3] + bf0[7]; + bf1[4] = bf0[0] - bf0[4]; + bf1[5] = bf0[1] - bf0[5]; + bf1[6] = bf0[2] - bf0[6]; + bf1[7] = bf0[3] - bf0[7]; + bf1[8] = bf0[8] + bf0[12]; + bf1[9] = bf0[9] + bf0[13]; + bf1[10] = bf0[10] + bf0[14]; + bf1[11] = bf0[11] + bf0[15]; + bf1[12] = bf0[8] - bf0[12]; + bf1[13] = bf0[9] - bf0[13]; + bf1[14] = bf0[10] - bf0[14]; + bf1[15] = bf0[11] - bf0[15]; + bf1[16] = bf0[16] + bf0[20]; + bf1[17] = bf0[17] + bf0[21]; + bf1[18] = bf0[18] + bf0[22]; + bf1[19] = bf0[19] + bf0[23]; + bf1[20] = bf0[16] - bf0[20]; + bf1[21] = bf0[17] - bf0[21]; + bf1[22] = bf0[18] - bf0[22]; + bf1[23] = bf0[19] - bf0[23]; + bf1[24] = bf0[24] + bf0[28]; + bf1[25] = bf0[25] + bf0[29]; + bf1[26] = bf0[26] + bf0[30]; + bf1[27] = bf0[27] + bf0[31]; + bf1[28] = bf0[24] - bf0[28]; + bf1[29] = bf0[25] - bf0[29]; + bf1[30] = bf0[26] - bf0[30]; + bf1[31] = bf0[27] - bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 6 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = half_btf(cospi[8], bf0[8], cospi[56], bf0[9], cos_bit[stage]); + bf1[9] = half_btf(cospi[56], bf0[8], -cospi[8], bf0[9], cos_bit[stage]); + bf1[10] = half_btf(cospi[40], bf0[10], cospi[24], bf0[11], cos_bit[stage]); + bf1[11] = half_btf(cospi[24], bf0[10], -cospi[40], bf0[11], cos_bit[stage]); + bf1[12] = half_btf(-cospi[56], bf0[12], cospi[8], bf0[13], cos_bit[stage]); + bf1[13] = half_btf(cospi[8], bf0[12], cospi[56], bf0[13], cos_bit[stage]); + bf1[14] = half_btf(-cospi[24], bf0[14], cospi[40], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(cospi[40], bf0[14], cospi[24], bf0[15], cos_bit[stage]); + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = bf0[18]; + bf1[19] = bf0[19]; + bf1[20] = bf0[20]; + bf1[21] = bf0[21]; + bf1[22] = bf0[22]; + bf1[23] = bf0[23]; + bf1[24] = half_btf(cospi[8], bf0[24], cospi[56], bf0[25], cos_bit[stage]); + bf1[25] = half_btf(cospi[56], bf0[24], -cospi[8], bf0[25], cos_bit[stage]); + bf1[26] = half_btf(cospi[40], bf0[26], cospi[24], bf0[27], cos_bit[stage]); + bf1[27] = half_btf(cospi[24], bf0[26], -cospi[40], bf0[27], cos_bit[stage]); + bf1[28] = half_btf(-cospi[56], bf0[28], cospi[8], bf0[29], cos_bit[stage]); + bf1[29] = half_btf(cospi[8], bf0[28], cospi[56], bf0[29], cos_bit[stage]); + bf1[30] = half_btf(-cospi[24], bf0[30], cospi[40], bf0[31], cos_bit[stage]); + bf1[31] = half_btf(cospi[40], bf0[30], cospi[24], bf0[31], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 7 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[8]; + bf1[1] = bf0[1] + bf0[9]; + bf1[2] = bf0[2] + bf0[10]; + bf1[3] = bf0[3] + bf0[11]; + bf1[4] = bf0[4] + bf0[12]; + bf1[5] = bf0[5] + bf0[13]; + bf1[6] = bf0[6] + bf0[14]; + bf1[7] = bf0[7] + bf0[15]; + bf1[8] = bf0[0] - bf0[8]; + bf1[9] = bf0[1] - bf0[9]; + bf1[10] = bf0[2] - bf0[10]; + bf1[11] = bf0[3] - bf0[11]; + bf1[12] = bf0[4] - bf0[12]; + bf1[13] = bf0[5] - bf0[13]; + bf1[14] = bf0[6] - bf0[14]; + bf1[15] = bf0[7] - bf0[15]; + bf1[16] = bf0[16] + bf0[24]; + bf1[17] = bf0[17] + bf0[25]; + bf1[18] = bf0[18] + bf0[26]; + bf1[19] = bf0[19] + bf0[27]; + bf1[20] = bf0[20] + bf0[28]; + bf1[21] = bf0[21] + bf0[29]; + bf1[22] = bf0[22] + bf0[30]; + bf1[23] = bf0[23] + bf0[31]; + bf1[24] = bf0[16] - bf0[24]; + bf1[25] = bf0[17] - bf0[25]; + bf1[26] = bf0[18] - bf0[26]; + bf1[27] = bf0[19] - bf0[27]; + bf1[28] = bf0[20] - bf0[28]; + bf1[29] = bf0[21] - bf0[29]; + bf1[30] = bf0[22] - bf0[30]; + bf1[31] = bf0[23] - bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 8 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = bf0[10]; + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = bf0[13]; + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = half_btf(cospi[4], bf0[16], cospi[60], bf0[17], cos_bit[stage]); + bf1[17] = half_btf(cospi[60], bf0[16], -cospi[4], bf0[17], cos_bit[stage]); + bf1[18] = half_btf(cospi[20], bf0[18], cospi[44], bf0[19], cos_bit[stage]); + bf1[19] = half_btf(cospi[44], bf0[18], -cospi[20], bf0[19], cos_bit[stage]); + bf1[20] = half_btf(cospi[36], bf0[20], cospi[28], bf0[21], cos_bit[stage]); + bf1[21] = half_btf(cospi[28], bf0[20], -cospi[36], bf0[21], cos_bit[stage]); + bf1[22] = half_btf(cospi[52], bf0[22], cospi[12], bf0[23], cos_bit[stage]); + bf1[23] = half_btf(cospi[12], bf0[22], -cospi[52], bf0[23], cos_bit[stage]); + bf1[24] = half_btf(-cospi[60], bf0[24], cospi[4], bf0[25], cos_bit[stage]); + bf1[25] = half_btf(cospi[4], bf0[24], cospi[60], bf0[25], cos_bit[stage]); + bf1[26] = half_btf(-cospi[44], bf0[26], cospi[20], bf0[27], cos_bit[stage]); + bf1[27] = half_btf(cospi[20], bf0[26], cospi[44], bf0[27], cos_bit[stage]); + bf1[28] = half_btf(-cospi[28], bf0[28], cospi[36], bf0[29], cos_bit[stage]); + bf1[29] = half_btf(cospi[36], bf0[28], cospi[28], bf0[29], cos_bit[stage]); + bf1[30] = half_btf(-cospi[12], bf0[30], cospi[52], bf0[31], cos_bit[stage]); + bf1[31] = half_btf(cospi[52], bf0[30], cospi[12], bf0[31], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 9 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[16]; + bf1[1] = bf0[1] + bf0[17]; + bf1[2] = bf0[2] + bf0[18]; + bf1[3] = bf0[3] + bf0[19]; + bf1[4] = bf0[4] + bf0[20]; + bf1[5] = bf0[5] + bf0[21]; + bf1[6] = bf0[6] + bf0[22]; + bf1[7] = bf0[7] + bf0[23]; + bf1[8] = bf0[8] + bf0[24]; + bf1[9] = bf0[9] + bf0[25]; + bf1[10] = bf0[10] + bf0[26]; + bf1[11] = bf0[11] + bf0[27]; + bf1[12] = bf0[12] + bf0[28]; + bf1[13] = bf0[13] + bf0[29]; + bf1[14] = bf0[14] + bf0[30]; + bf1[15] = bf0[15] + bf0[31]; + bf1[16] = bf0[0] - bf0[16]; + bf1[17] = bf0[1] - bf0[17]; + bf1[18] = bf0[2] - bf0[18]; + bf1[19] = bf0[3] - bf0[19]; + bf1[20] = bf0[4] - bf0[20]; + bf1[21] = bf0[5] - bf0[21]; + bf1[22] = bf0[6] - bf0[22]; + bf1[23] = bf0[7] - bf0[23]; + bf1[24] = bf0[8] - bf0[24]; + bf1[25] = bf0[9] - bf0[25]; + bf1[26] = bf0[10] - bf0[26]; + bf1[27] = bf0[11] - bf0[27]; + bf1[28] = bf0[12] - bf0[28]; + bf1[29] = bf0[13] - bf0[29]; + bf1[30] = bf0[14] - bf0[30]; + bf1[31] = bf0[15] - bf0[31]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 10 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[1], bf0[0], cospi[63], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(cospi[63], bf0[0], -cospi[1], bf0[1], cos_bit[stage]); + bf1[2] = half_btf(cospi[5], bf0[2], cospi[59], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[59], bf0[2], -cospi[5], bf0[3], cos_bit[stage]); + bf1[4] = half_btf(cospi[9], bf0[4], cospi[55], bf0[5], cos_bit[stage]); + bf1[5] = half_btf(cospi[55], bf0[4], -cospi[9], bf0[5], cos_bit[stage]); + bf1[6] = half_btf(cospi[13], bf0[6], cospi[51], bf0[7], cos_bit[stage]); + bf1[7] = half_btf(cospi[51], bf0[6], -cospi[13], bf0[7], cos_bit[stage]); + bf1[8] = half_btf(cospi[17], bf0[8], cospi[47], bf0[9], cos_bit[stage]); + bf1[9] = half_btf(cospi[47], bf0[8], -cospi[17], bf0[9], cos_bit[stage]); + bf1[10] = half_btf(cospi[21], bf0[10], cospi[43], bf0[11], cos_bit[stage]); + bf1[11] = half_btf(cospi[43], bf0[10], -cospi[21], bf0[11], cos_bit[stage]); + bf1[12] = half_btf(cospi[25], bf0[12], cospi[39], bf0[13], cos_bit[stage]); + bf1[13] = half_btf(cospi[39], bf0[12], -cospi[25], bf0[13], cos_bit[stage]); + bf1[14] = half_btf(cospi[29], bf0[14], cospi[35], bf0[15], cos_bit[stage]); + bf1[15] = half_btf(cospi[35], bf0[14], -cospi[29], bf0[15], cos_bit[stage]); + bf1[16] = half_btf(cospi[33], bf0[16], cospi[31], bf0[17], cos_bit[stage]); + bf1[17] = half_btf(cospi[31], bf0[16], -cospi[33], bf0[17], cos_bit[stage]); + bf1[18] = half_btf(cospi[37], bf0[18], cospi[27], bf0[19], cos_bit[stage]); + bf1[19] = half_btf(cospi[27], bf0[18], -cospi[37], bf0[19], cos_bit[stage]); + bf1[20] = half_btf(cospi[41], bf0[20], cospi[23], bf0[21], cos_bit[stage]); + bf1[21] = half_btf(cospi[23], bf0[20], -cospi[41], bf0[21], cos_bit[stage]); + bf1[22] = half_btf(cospi[45], bf0[22], cospi[19], bf0[23], cos_bit[stage]); + bf1[23] = half_btf(cospi[19], bf0[22], -cospi[45], bf0[23], cos_bit[stage]); + bf1[24] = half_btf(cospi[49], bf0[24], cospi[15], bf0[25], cos_bit[stage]); + bf1[25] = half_btf(cospi[15], bf0[24], -cospi[49], bf0[25], cos_bit[stage]); + bf1[26] = half_btf(cospi[53], bf0[26], cospi[11], bf0[27], cos_bit[stage]); + bf1[27] = half_btf(cospi[11], bf0[26], -cospi[53], bf0[27], cos_bit[stage]); + bf1[28] = half_btf(cospi[57], bf0[28], cospi[7], bf0[29], cos_bit[stage]); + bf1[29] = half_btf(cospi[7], bf0[28], -cospi[57], bf0[29], cos_bit[stage]); + bf1[30] = half_btf(cospi[61], bf0[30], cospi[3], bf0[31], cos_bit[stage]); + bf1[31] = half_btf(cospi[3], bf0[30], -cospi[61], bf0[31], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 11 + stage++; + bf0 = step; + bf1 = output; + bf1[0] = bf0[1]; + bf1[1] = bf0[30]; + bf1[2] = bf0[3]; + bf1[3] = bf0[28]; + bf1[4] = bf0[5]; + bf1[5] = bf0[26]; + bf1[6] = bf0[7]; + bf1[7] = bf0[24]; + bf1[8] = bf0[9]; + bf1[9] = bf0[22]; + bf1[10] = bf0[11]; + bf1[11] = bf0[20]; + bf1[12] = bf0[13]; + bf1[13] = bf0[18]; + bf1[14] = bf0[15]; + bf1[15] = bf0[16]; + bf1[16] = bf0[17]; + bf1[17] = bf0[14]; + bf1[18] = bf0[19]; + bf1[19] = bf0[12]; + bf1[20] = bf0[21]; + bf1[21] = bf0[10]; + bf1[22] = bf0[23]; + bf1[23] = bf0[8]; + bf1[24] = bf0[25]; + bf1[25] = bf0[6]; + bf1[26] = bf0[27]; + bf1[27] = bf0[4]; + bf1[28] = bf0[29]; + bf1[29] = bf0[2]; + bf1[30] = bf0[31]; + bf1[31] = bf0[0]; + range_check(stage, input, bf1, size, stage_range[stage]); +} + +#if CONFIG_TX64X64 +void av1_idct64_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int32_t size = 64; + const int32_t *cospi; + + int32_t stage = 0; + int32_t *bf0, *bf1; + int32_t step[64]; + + // stage 0; + range_check(stage, input, input, size, stage_range[stage]); + + // stage 1; + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf1 = output; + bf1[0] = input[0]; + bf1[1] = input[32]; + bf1[2] = input[16]; + bf1[3] = input[48]; + bf1[4] = input[8]; + bf1[5] = input[40]; + bf1[6] = input[24]; + bf1[7] = input[56]; + bf1[8] = input[4]; + bf1[9] = input[36]; + bf1[10] = input[20]; + bf1[11] = input[52]; + bf1[12] = input[12]; + bf1[13] = input[44]; + bf1[14] = input[28]; + bf1[15] = input[60]; + bf1[16] = input[2]; + bf1[17] = input[34]; + bf1[18] = input[18]; + bf1[19] = input[50]; + bf1[20] = input[10]; + bf1[21] = input[42]; + bf1[22] = input[26]; + bf1[23] = input[58]; + bf1[24] = input[6]; + bf1[25] = input[38]; + bf1[26] = input[22]; + bf1[27] = input[54]; + bf1[28] = input[14]; + bf1[29] = input[46]; + bf1[30] = input[30]; + bf1[31] = input[62]; + bf1[32] = input[1]; + bf1[33] = input[33]; + bf1[34] = input[17]; + bf1[35] = input[49]; + bf1[36] = input[9]; + bf1[37] = input[41]; + bf1[38] = input[25]; + bf1[39] = input[57]; + bf1[40] = input[5]; + bf1[41] = input[37]; + bf1[42] = input[21]; + bf1[43] = input[53]; + bf1[44] = input[13]; + bf1[45] = input[45]; + bf1[46] = input[29]; + bf1[47] = input[61]; + bf1[48] = input[3]; + bf1[49] = input[35]; + bf1[50] = input[19]; + bf1[51] = input[51]; + bf1[52] = input[11]; + bf1[53] = input[43]; + bf1[54] = input[27]; + bf1[55] = input[59]; + bf1[56] = input[7]; + bf1[57] = input[39]; + bf1[58] = input[23]; + bf1[59] = input[55]; + bf1[60] = input[15]; + bf1[61] = input[47]; + bf1[62] = input[31]; + bf1[63] = input[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 2 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = bf0[10]; + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = bf0[13]; + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = bf0[18]; + bf1[19] = bf0[19]; + bf1[20] = bf0[20]; + bf1[21] = bf0[21]; + bf1[22] = bf0[22]; + bf1[23] = bf0[23]; + bf1[24] = bf0[24]; + bf1[25] = bf0[25]; + bf1[26] = bf0[26]; + bf1[27] = bf0[27]; + bf1[28] = bf0[28]; + bf1[29] = bf0[29]; + bf1[30] = bf0[30]; + bf1[31] = bf0[31]; + bf1[32] = half_btf(cospi[63], bf0[32], -cospi[1], bf0[63], cos_bit[stage]); + bf1[33] = half_btf(cospi[31], bf0[33], -cospi[33], bf0[62], cos_bit[stage]); + bf1[34] = half_btf(cospi[47], bf0[34], -cospi[17], bf0[61], cos_bit[stage]); + bf1[35] = half_btf(cospi[15], bf0[35], -cospi[49], bf0[60], cos_bit[stage]); + bf1[36] = half_btf(cospi[55], bf0[36], -cospi[9], bf0[59], cos_bit[stage]); + bf1[37] = half_btf(cospi[23], bf0[37], -cospi[41], bf0[58], cos_bit[stage]); + bf1[38] = half_btf(cospi[39], bf0[38], -cospi[25], bf0[57], cos_bit[stage]); + bf1[39] = half_btf(cospi[7], bf0[39], -cospi[57], bf0[56], cos_bit[stage]); + bf1[40] = half_btf(cospi[59], bf0[40], -cospi[5], bf0[55], cos_bit[stage]); + bf1[41] = half_btf(cospi[27], bf0[41], -cospi[37], bf0[54], cos_bit[stage]); + bf1[42] = half_btf(cospi[43], bf0[42], -cospi[21], bf0[53], cos_bit[stage]); + bf1[43] = half_btf(cospi[11], bf0[43], -cospi[53], bf0[52], cos_bit[stage]); + bf1[44] = half_btf(cospi[51], bf0[44], -cospi[13], bf0[51], cos_bit[stage]); + bf1[45] = half_btf(cospi[19], bf0[45], -cospi[45], bf0[50], cos_bit[stage]); + bf1[46] = half_btf(cospi[35], bf0[46], -cospi[29], bf0[49], cos_bit[stage]); + bf1[47] = half_btf(cospi[3], bf0[47], -cospi[61], bf0[48], cos_bit[stage]); + bf1[48] = half_btf(cospi[61], bf0[47], cospi[3], bf0[48], cos_bit[stage]); + bf1[49] = half_btf(cospi[29], bf0[46], cospi[35], bf0[49], cos_bit[stage]); + bf1[50] = half_btf(cospi[45], bf0[45], cospi[19], bf0[50], cos_bit[stage]); + bf1[51] = half_btf(cospi[13], bf0[44], cospi[51], bf0[51], cos_bit[stage]); + bf1[52] = half_btf(cospi[53], bf0[43], cospi[11], bf0[52], cos_bit[stage]); + bf1[53] = half_btf(cospi[21], bf0[42], cospi[43], bf0[53], cos_bit[stage]); + bf1[54] = half_btf(cospi[37], bf0[41], cospi[27], bf0[54], cos_bit[stage]); + bf1[55] = half_btf(cospi[5], bf0[40], cospi[59], bf0[55], cos_bit[stage]); + bf1[56] = half_btf(cospi[57], bf0[39], cospi[7], bf0[56], cos_bit[stage]); + bf1[57] = half_btf(cospi[25], bf0[38], cospi[39], bf0[57], cos_bit[stage]); + bf1[58] = half_btf(cospi[41], bf0[37], cospi[23], bf0[58], cos_bit[stage]); + bf1[59] = half_btf(cospi[9], bf0[36], cospi[55], bf0[59], cos_bit[stage]); + bf1[60] = half_btf(cospi[49], bf0[35], cospi[15], bf0[60], cos_bit[stage]); + bf1[61] = half_btf(cospi[17], bf0[34], cospi[47], bf0[61], cos_bit[stage]); + bf1[62] = half_btf(cospi[33], bf0[33], cospi[31], bf0[62], cos_bit[stage]); + bf1[63] = half_btf(cospi[1], bf0[32], cospi[63], bf0[63], cos_bit[stage]); + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 3 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = bf0[10]; + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = bf0[13]; + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = half_btf(cospi[62], bf0[16], -cospi[2], bf0[31], cos_bit[stage]); + bf1[17] = half_btf(cospi[30], bf0[17], -cospi[34], bf0[30], cos_bit[stage]); + bf1[18] = half_btf(cospi[46], bf0[18], -cospi[18], bf0[29], cos_bit[stage]); + bf1[19] = half_btf(cospi[14], bf0[19], -cospi[50], bf0[28], cos_bit[stage]); + bf1[20] = half_btf(cospi[54], bf0[20], -cospi[10], bf0[27], cos_bit[stage]); + bf1[21] = half_btf(cospi[22], bf0[21], -cospi[42], bf0[26], cos_bit[stage]); + bf1[22] = half_btf(cospi[38], bf0[22], -cospi[26], bf0[25], cos_bit[stage]); + bf1[23] = half_btf(cospi[6], bf0[23], -cospi[58], bf0[24], cos_bit[stage]); + bf1[24] = half_btf(cospi[58], bf0[23], cospi[6], bf0[24], cos_bit[stage]); + bf1[25] = half_btf(cospi[26], bf0[22], cospi[38], bf0[25], cos_bit[stage]); + bf1[26] = half_btf(cospi[42], bf0[21], cospi[22], bf0[26], cos_bit[stage]); + bf1[27] = half_btf(cospi[10], bf0[20], cospi[54], bf0[27], cos_bit[stage]); + bf1[28] = half_btf(cospi[50], bf0[19], cospi[14], bf0[28], cos_bit[stage]); + bf1[29] = half_btf(cospi[18], bf0[18], cospi[46], bf0[29], cos_bit[stage]); + bf1[30] = half_btf(cospi[34], bf0[17], cospi[30], bf0[30], cos_bit[stage]); + bf1[31] = half_btf(cospi[2], bf0[16], cospi[62], bf0[31], cos_bit[stage]); + bf1[32] = bf0[32] + bf0[33]; + bf1[33] = bf0[32] - bf0[33]; + bf1[34] = -bf0[34] + bf0[35]; + bf1[35] = bf0[34] + bf0[35]; + bf1[36] = bf0[36] + bf0[37]; + bf1[37] = bf0[36] - bf0[37]; + bf1[38] = -bf0[38] + bf0[39]; + bf1[39] = bf0[38] + bf0[39]; + bf1[40] = bf0[40] + bf0[41]; + bf1[41] = bf0[40] - bf0[41]; + bf1[42] = -bf0[42] + bf0[43]; + bf1[43] = bf0[42] + bf0[43]; + bf1[44] = bf0[44] + bf0[45]; + bf1[45] = bf0[44] - bf0[45]; + bf1[46] = -bf0[46] + bf0[47]; + bf1[47] = bf0[46] + bf0[47]; + bf1[48] = bf0[48] + bf0[49]; + bf1[49] = bf0[48] - bf0[49]; + bf1[50] = -bf0[50] + bf0[51]; + bf1[51] = bf0[50] + bf0[51]; + bf1[52] = bf0[52] + bf0[53]; + bf1[53] = bf0[52] - bf0[53]; + bf1[54] = -bf0[54] + bf0[55]; + bf1[55] = bf0[54] + bf0[55]; + bf1[56] = bf0[56] + bf0[57]; + bf1[57] = bf0[56] - bf0[57]; + bf1[58] = -bf0[58] + bf0[59]; + bf1[59] = bf0[58] + bf0[59]; + bf1[60] = bf0[60] + bf0[61]; + bf1[61] = bf0[60] - bf0[61]; + bf1[62] = -bf0[62] + bf0[63]; + bf1[63] = bf0[62] + bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 4 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = half_btf(cospi[60], bf0[8], -cospi[4], bf0[15], cos_bit[stage]); + bf1[9] = half_btf(cospi[28], bf0[9], -cospi[36], bf0[14], cos_bit[stage]); + bf1[10] = half_btf(cospi[44], bf0[10], -cospi[20], bf0[13], cos_bit[stage]); + bf1[11] = half_btf(cospi[12], bf0[11], -cospi[52], bf0[12], cos_bit[stage]); + bf1[12] = half_btf(cospi[52], bf0[11], cospi[12], bf0[12], cos_bit[stage]); + bf1[13] = half_btf(cospi[20], bf0[10], cospi[44], bf0[13], cos_bit[stage]); + bf1[14] = half_btf(cospi[36], bf0[9], cospi[28], bf0[14], cos_bit[stage]); + bf1[15] = half_btf(cospi[4], bf0[8], cospi[60], bf0[15], cos_bit[stage]); + bf1[16] = bf0[16] + bf0[17]; + bf1[17] = bf0[16] - bf0[17]; + bf1[18] = -bf0[18] + bf0[19]; + bf1[19] = bf0[18] + bf0[19]; + bf1[20] = bf0[20] + bf0[21]; + bf1[21] = bf0[20] - bf0[21]; + bf1[22] = -bf0[22] + bf0[23]; + bf1[23] = bf0[22] + bf0[23]; + bf1[24] = bf0[24] + bf0[25]; + bf1[25] = bf0[24] - bf0[25]; + bf1[26] = -bf0[26] + bf0[27]; + bf1[27] = bf0[26] + bf0[27]; + bf1[28] = bf0[28] + bf0[29]; + bf1[29] = bf0[28] - bf0[29]; + bf1[30] = -bf0[30] + bf0[31]; + bf1[31] = bf0[30] + bf0[31]; + bf1[32] = bf0[32]; + bf1[33] = half_btf(-cospi[4], bf0[33], cospi[60], bf0[62], cos_bit[stage]); + bf1[34] = half_btf(-cospi[60], bf0[34], -cospi[4], bf0[61], cos_bit[stage]); + bf1[35] = bf0[35]; + bf1[36] = bf0[36]; + bf1[37] = half_btf(-cospi[36], bf0[37], cospi[28], bf0[58], cos_bit[stage]); + bf1[38] = half_btf(-cospi[28], bf0[38], -cospi[36], bf0[57], cos_bit[stage]); + bf1[39] = bf0[39]; + bf1[40] = bf0[40]; + bf1[41] = half_btf(-cospi[20], bf0[41], cospi[44], bf0[54], cos_bit[stage]); + bf1[42] = half_btf(-cospi[44], bf0[42], -cospi[20], bf0[53], cos_bit[stage]); + bf1[43] = bf0[43]; + bf1[44] = bf0[44]; + bf1[45] = half_btf(-cospi[52], bf0[45], cospi[12], bf0[50], cos_bit[stage]); + bf1[46] = half_btf(-cospi[12], bf0[46], -cospi[52], bf0[49], cos_bit[stage]); + bf1[47] = bf0[47]; + bf1[48] = bf0[48]; + bf1[49] = half_btf(-cospi[52], bf0[46], cospi[12], bf0[49], cos_bit[stage]); + bf1[50] = half_btf(cospi[12], bf0[45], cospi[52], bf0[50], cos_bit[stage]); + bf1[51] = bf0[51]; + bf1[52] = bf0[52]; + bf1[53] = half_btf(-cospi[20], bf0[42], cospi[44], bf0[53], cos_bit[stage]); + bf1[54] = half_btf(cospi[44], bf0[41], cospi[20], bf0[54], cos_bit[stage]); + bf1[55] = bf0[55]; + bf1[56] = bf0[56]; + bf1[57] = half_btf(-cospi[36], bf0[38], cospi[28], bf0[57], cos_bit[stage]); + bf1[58] = half_btf(cospi[28], bf0[37], cospi[36], bf0[58], cos_bit[stage]); + bf1[59] = bf0[59]; + bf1[60] = bf0[60]; + bf1[61] = half_btf(-cospi[4], bf0[34], cospi[60], bf0[61], cos_bit[stage]); + bf1[62] = half_btf(cospi[60], bf0[33], cospi[4], bf0[62], cos_bit[stage]); + bf1[63] = bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 5 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = half_btf(cospi[56], bf0[4], -cospi[8], bf0[7], cos_bit[stage]); + bf1[5] = half_btf(cospi[24], bf0[5], -cospi[40], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[40], bf0[5], cospi[24], bf0[6], cos_bit[stage]); + bf1[7] = half_btf(cospi[8], bf0[4], cospi[56], bf0[7], cos_bit[stage]); + bf1[8] = bf0[8] + bf0[9]; + bf1[9] = bf0[8] - bf0[9]; + bf1[10] = -bf0[10] + bf0[11]; + bf1[11] = bf0[10] + bf0[11]; + bf1[12] = bf0[12] + bf0[13]; + bf1[13] = bf0[12] - bf0[13]; + bf1[14] = -bf0[14] + bf0[15]; + bf1[15] = bf0[14] + bf0[15]; + bf1[16] = bf0[16]; + bf1[17] = half_btf(-cospi[8], bf0[17], cospi[56], bf0[30], cos_bit[stage]); + bf1[18] = half_btf(-cospi[56], bf0[18], -cospi[8], bf0[29], cos_bit[stage]); + bf1[19] = bf0[19]; + bf1[20] = bf0[20]; + bf1[21] = half_btf(-cospi[40], bf0[21], cospi[24], bf0[26], cos_bit[stage]); + bf1[22] = half_btf(-cospi[24], bf0[22], -cospi[40], bf0[25], cos_bit[stage]); + bf1[23] = bf0[23]; + bf1[24] = bf0[24]; + bf1[25] = half_btf(-cospi[40], bf0[22], cospi[24], bf0[25], cos_bit[stage]); + bf1[26] = half_btf(cospi[24], bf0[21], cospi[40], bf0[26], cos_bit[stage]); + bf1[27] = bf0[27]; + bf1[28] = bf0[28]; + bf1[29] = half_btf(-cospi[8], bf0[18], cospi[56], bf0[29], cos_bit[stage]); + bf1[30] = half_btf(cospi[56], bf0[17], cospi[8], bf0[30], cos_bit[stage]); + bf1[31] = bf0[31]; + bf1[32] = bf0[32] + bf0[35]; + bf1[33] = bf0[33] + bf0[34]; + bf1[34] = bf0[33] - bf0[34]; + bf1[35] = bf0[32] - bf0[35]; + bf1[36] = -bf0[36] + bf0[39]; + bf1[37] = -bf0[37] + bf0[38]; + bf1[38] = bf0[37] + bf0[38]; + bf1[39] = bf0[36] + bf0[39]; + bf1[40] = bf0[40] + bf0[43]; + bf1[41] = bf0[41] + bf0[42]; + bf1[42] = bf0[41] - bf0[42]; + bf1[43] = bf0[40] - bf0[43]; + bf1[44] = -bf0[44] + bf0[47]; + bf1[45] = -bf0[45] + bf0[46]; + bf1[46] = bf0[45] + bf0[46]; + bf1[47] = bf0[44] + bf0[47]; + bf1[48] = bf0[48] + bf0[51]; + bf1[49] = bf0[49] + bf0[50]; + bf1[50] = bf0[49] - bf0[50]; + bf1[51] = bf0[48] - bf0[51]; + bf1[52] = -bf0[52] + bf0[55]; + bf1[53] = -bf0[53] + bf0[54]; + bf1[54] = bf0[53] + bf0[54]; + bf1[55] = bf0[52] + bf0[55]; + bf1[56] = bf0[56] + bf0[59]; + bf1[57] = bf0[57] + bf0[58]; + bf1[58] = bf0[57] - bf0[58]; + bf1[59] = bf0[56] - bf0[59]; + bf1[60] = -bf0[60] + bf0[63]; + bf1[61] = -bf0[61] + bf0[62]; + bf1[62] = bf0[61] + bf0[62]; + bf1[63] = bf0[60] + bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 6 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = half_btf(cospi[32], bf0[0], cospi[32], bf0[1], cos_bit[stage]); + bf1[1] = half_btf(cospi[32], bf0[0], -cospi[32], bf0[1], cos_bit[stage]); + bf1[2] = half_btf(cospi[48], bf0[2], -cospi[16], bf0[3], cos_bit[stage]); + bf1[3] = half_btf(cospi[16], bf0[2], cospi[48], bf0[3], cos_bit[stage]); + bf1[4] = bf0[4] + bf0[5]; + bf1[5] = bf0[4] - bf0[5]; + bf1[6] = -bf0[6] + bf0[7]; + bf1[7] = bf0[6] + bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = half_btf(-cospi[16], bf0[9], cospi[48], bf0[14], cos_bit[stage]); + bf1[10] = half_btf(-cospi[48], bf0[10], -cospi[16], bf0[13], cos_bit[stage]); + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = half_btf(-cospi[16], bf0[10], cospi[48], bf0[13], cos_bit[stage]); + bf1[14] = half_btf(cospi[48], bf0[9], cospi[16], bf0[14], cos_bit[stage]); + bf1[15] = bf0[15]; + bf1[16] = bf0[16] + bf0[19]; + bf1[17] = bf0[17] + bf0[18]; + bf1[18] = bf0[17] - bf0[18]; + bf1[19] = bf0[16] - bf0[19]; + bf1[20] = -bf0[20] + bf0[23]; + bf1[21] = -bf0[21] + bf0[22]; + bf1[22] = bf0[21] + bf0[22]; + bf1[23] = bf0[20] + bf0[23]; + bf1[24] = bf0[24] + bf0[27]; + bf1[25] = bf0[25] + bf0[26]; + bf1[26] = bf0[25] - bf0[26]; + bf1[27] = bf0[24] - bf0[27]; + bf1[28] = -bf0[28] + bf0[31]; + bf1[29] = -bf0[29] + bf0[30]; + bf1[30] = bf0[29] + bf0[30]; + bf1[31] = bf0[28] + bf0[31]; + bf1[32] = bf0[32]; + bf1[33] = bf0[33]; + bf1[34] = half_btf(-cospi[8], bf0[34], cospi[56], bf0[61], cos_bit[stage]); + bf1[35] = half_btf(-cospi[8], bf0[35], cospi[56], bf0[60], cos_bit[stage]); + bf1[36] = half_btf(-cospi[56], bf0[36], -cospi[8], bf0[59], cos_bit[stage]); + bf1[37] = half_btf(-cospi[56], bf0[37], -cospi[8], bf0[58], cos_bit[stage]); + bf1[38] = bf0[38]; + bf1[39] = bf0[39]; + bf1[40] = bf0[40]; + bf1[41] = bf0[41]; + bf1[42] = half_btf(-cospi[40], bf0[42], cospi[24], bf0[53], cos_bit[stage]); + bf1[43] = half_btf(-cospi[40], bf0[43], cospi[24], bf0[52], cos_bit[stage]); + bf1[44] = half_btf(-cospi[24], bf0[44], -cospi[40], bf0[51], cos_bit[stage]); + bf1[45] = half_btf(-cospi[24], bf0[45], -cospi[40], bf0[50], cos_bit[stage]); + bf1[46] = bf0[46]; + bf1[47] = bf0[47]; + bf1[48] = bf0[48]; + bf1[49] = bf0[49]; + bf1[50] = half_btf(-cospi[40], bf0[45], cospi[24], bf0[50], cos_bit[stage]); + bf1[51] = half_btf(-cospi[40], bf0[44], cospi[24], bf0[51], cos_bit[stage]); + bf1[52] = half_btf(cospi[24], bf0[43], cospi[40], bf0[52], cos_bit[stage]); + bf1[53] = half_btf(cospi[24], bf0[42], cospi[40], bf0[53], cos_bit[stage]); + bf1[54] = bf0[54]; + bf1[55] = bf0[55]; + bf1[56] = bf0[56]; + bf1[57] = bf0[57]; + bf1[58] = half_btf(-cospi[8], bf0[37], cospi[56], bf0[58], cos_bit[stage]); + bf1[59] = half_btf(-cospi[8], bf0[36], cospi[56], bf0[59], cos_bit[stage]); + bf1[60] = half_btf(cospi[56], bf0[35], cospi[8], bf0[60], cos_bit[stage]); + bf1[61] = half_btf(cospi[56], bf0[34], cospi[8], bf0[61], cos_bit[stage]); + bf1[62] = bf0[62]; + bf1[63] = bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 7 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[3]; + bf1[1] = bf0[1] + bf0[2]; + bf1[2] = bf0[1] - bf0[2]; + bf1[3] = bf0[0] - bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = half_btf(-cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]); + bf1[6] = half_btf(cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]); + bf1[7] = bf0[7]; + bf1[8] = bf0[8] + bf0[11]; + bf1[9] = bf0[9] + bf0[10]; + bf1[10] = bf0[9] - bf0[10]; + bf1[11] = bf0[8] - bf0[11]; + bf1[12] = -bf0[12] + bf0[15]; + bf1[13] = -bf0[13] + bf0[14]; + bf1[14] = bf0[13] + bf0[14]; + bf1[15] = bf0[12] + bf0[15]; + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = half_btf(-cospi[16], bf0[18], cospi[48], bf0[29], cos_bit[stage]); + bf1[19] = half_btf(-cospi[16], bf0[19], cospi[48], bf0[28], cos_bit[stage]); + bf1[20] = half_btf(-cospi[48], bf0[20], -cospi[16], bf0[27], cos_bit[stage]); + bf1[21] = half_btf(-cospi[48], bf0[21], -cospi[16], bf0[26], cos_bit[stage]); + bf1[22] = bf0[22]; + bf1[23] = bf0[23]; + bf1[24] = bf0[24]; + bf1[25] = bf0[25]; + bf1[26] = half_btf(-cospi[16], bf0[21], cospi[48], bf0[26], cos_bit[stage]); + bf1[27] = half_btf(-cospi[16], bf0[20], cospi[48], bf0[27], cos_bit[stage]); + bf1[28] = half_btf(cospi[48], bf0[19], cospi[16], bf0[28], cos_bit[stage]); + bf1[29] = half_btf(cospi[48], bf0[18], cospi[16], bf0[29], cos_bit[stage]); + bf1[30] = bf0[30]; + bf1[31] = bf0[31]; + bf1[32] = bf0[32] + bf0[39]; + bf1[33] = bf0[33] + bf0[38]; + bf1[34] = bf0[34] + bf0[37]; + bf1[35] = bf0[35] + bf0[36]; + bf1[36] = bf0[35] - bf0[36]; + bf1[37] = bf0[34] - bf0[37]; + bf1[38] = bf0[33] - bf0[38]; + bf1[39] = bf0[32] - bf0[39]; + bf1[40] = -bf0[40] + bf0[47]; + bf1[41] = -bf0[41] + bf0[46]; + bf1[42] = -bf0[42] + bf0[45]; + bf1[43] = -bf0[43] + bf0[44]; + bf1[44] = bf0[43] + bf0[44]; + bf1[45] = bf0[42] + bf0[45]; + bf1[46] = bf0[41] + bf0[46]; + bf1[47] = bf0[40] + bf0[47]; + bf1[48] = bf0[48] + bf0[55]; + bf1[49] = bf0[49] + bf0[54]; + bf1[50] = bf0[50] + bf0[53]; + bf1[51] = bf0[51] + bf0[52]; + bf1[52] = bf0[51] - bf0[52]; + bf1[53] = bf0[50] - bf0[53]; + bf1[54] = bf0[49] - bf0[54]; + bf1[55] = bf0[48] - bf0[55]; + bf1[56] = -bf0[56] + bf0[63]; + bf1[57] = -bf0[57] + bf0[62]; + bf1[58] = -bf0[58] + bf0[61]; + bf1[59] = -bf0[59] + bf0[60]; + bf1[60] = bf0[59] + bf0[60]; + bf1[61] = bf0[58] + bf0[61]; + bf1[62] = bf0[57] + bf0[62]; + bf1[63] = bf0[56] + bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 8 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0] + bf0[7]; + bf1[1] = bf0[1] + bf0[6]; + bf1[2] = bf0[2] + bf0[5]; + bf1[3] = bf0[3] + bf0[4]; + bf1[4] = bf0[3] - bf0[4]; + bf1[5] = bf0[2] - bf0[5]; + bf1[6] = bf0[1] - bf0[6]; + bf1[7] = bf0[0] - bf0[7]; + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = half_btf(-cospi[32], bf0[10], cospi[32], bf0[13], cos_bit[stage]); + bf1[11] = half_btf(-cospi[32], bf0[11], cospi[32], bf0[12], cos_bit[stage]); + bf1[12] = half_btf(cospi[32], bf0[11], cospi[32], bf0[12], cos_bit[stage]); + bf1[13] = half_btf(cospi[32], bf0[10], cospi[32], bf0[13], cos_bit[stage]); + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = bf0[16] + bf0[23]; + bf1[17] = bf0[17] + bf0[22]; + bf1[18] = bf0[18] + bf0[21]; + bf1[19] = bf0[19] + bf0[20]; + bf1[20] = bf0[19] - bf0[20]; + bf1[21] = bf0[18] - bf0[21]; + bf1[22] = bf0[17] - bf0[22]; + bf1[23] = bf0[16] - bf0[23]; + bf1[24] = -bf0[24] + bf0[31]; + bf1[25] = -bf0[25] + bf0[30]; + bf1[26] = -bf0[26] + bf0[29]; + bf1[27] = -bf0[27] + bf0[28]; + bf1[28] = bf0[27] + bf0[28]; + bf1[29] = bf0[26] + bf0[29]; + bf1[30] = bf0[25] + bf0[30]; + bf1[31] = bf0[24] + bf0[31]; + bf1[32] = bf0[32]; + bf1[33] = bf0[33]; + bf1[34] = bf0[34]; + bf1[35] = bf0[35]; + bf1[36] = half_btf(-cospi[16], bf0[36], cospi[48], bf0[59], cos_bit[stage]); + bf1[37] = half_btf(-cospi[16], bf0[37], cospi[48], bf0[58], cos_bit[stage]); + bf1[38] = half_btf(-cospi[16], bf0[38], cospi[48], bf0[57], cos_bit[stage]); + bf1[39] = half_btf(-cospi[16], bf0[39], cospi[48], bf0[56], cos_bit[stage]); + bf1[40] = half_btf(-cospi[48], bf0[40], -cospi[16], bf0[55], cos_bit[stage]); + bf1[41] = half_btf(-cospi[48], bf0[41], -cospi[16], bf0[54], cos_bit[stage]); + bf1[42] = half_btf(-cospi[48], bf0[42], -cospi[16], bf0[53], cos_bit[stage]); + bf1[43] = half_btf(-cospi[48], bf0[43], -cospi[16], bf0[52], cos_bit[stage]); + bf1[44] = bf0[44]; + bf1[45] = bf0[45]; + bf1[46] = bf0[46]; + bf1[47] = bf0[47]; + bf1[48] = bf0[48]; + bf1[49] = bf0[49]; + bf1[50] = bf0[50]; + bf1[51] = bf0[51]; + bf1[52] = half_btf(-cospi[16], bf0[43], cospi[48], bf0[52], cos_bit[stage]); + bf1[53] = half_btf(-cospi[16], bf0[42], cospi[48], bf0[53], cos_bit[stage]); + bf1[54] = half_btf(-cospi[16], bf0[41], cospi[48], bf0[54], cos_bit[stage]); + bf1[55] = half_btf(-cospi[16], bf0[40], cospi[48], bf0[55], cos_bit[stage]); + bf1[56] = half_btf(cospi[48], bf0[39], cospi[16], bf0[56], cos_bit[stage]); + bf1[57] = half_btf(cospi[48], bf0[38], cospi[16], bf0[57], cos_bit[stage]); + bf1[58] = half_btf(cospi[48], bf0[37], cospi[16], bf0[58], cos_bit[stage]); + bf1[59] = half_btf(cospi[48], bf0[36], cospi[16], bf0[59], cos_bit[stage]); + bf1[60] = bf0[60]; + bf1[61] = bf0[61]; + bf1[62] = bf0[62]; + bf1[63] = bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 9 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[15]; + bf1[1] = bf0[1] + bf0[14]; + bf1[2] = bf0[2] + bf0[13]; + bf1[3] = bf0[3] + bf0[12]; + bf1[4] = bf0[4] + bf0[11]; + bf1[5] = bf0[5] + bf0[10]; + bf1[6] = bf0[6] + bf0[9]; + bf1[7] = bf0[7] + bf0[8]; + bf1[8] = bf0[7] - bf0[8]; + bf1[9] = bf0[6] - bf0[9]; + bf1[10] = bf0[5] - bf0[10]; + bf1[11] = bf0[4] - bf0[11]; + bf1[12] = bf0[3] - bf0[12]; + bf1[13] = bf0[2] - bf0[13]; + bf1[14] = bf0[1] - bf0[14]; + bf1[15] = bf0[0] - bf0[15]; + bf1[16] = bf0[16]; + bf1[17] = bf0[17]; + bf1[18] = bf0[18]; + bf1[19] = bf0[19]; + bf1[20] = half_btf(-cospi[32], bf0[20], cospi[32], bf0[27], cos_bit[stage]); + bf1[21] = half_btf(-cospi[32], bf0[21], cospi[32], bf0[26], cos_bit[stage]); + bf1[22] = half_btf(-cospi[32], bf0[22], cospi[32], bf0[25], cos_bit[stage]); + bf1[23] = half_btf(-cospi[32], bf0[23], cospi[32], bf0[24], cos_bit[stage]); + bf1[24] = half_btf(cospi[32], bf0[23], cospi[32], bf0[24], cos_bit[stage]); + bf1[25] = half_btf(cospi[32], bf0[22], cospi[32], bf0[25], cos_bit[stage]); + bf1[26] = half_btf(cospi[32], bf0[21], cospi[32], bf0[26], cos_bit[stage]); + bf1[27] = half_btf(cospi[32], bf0[20], cospi[32], bf0[27], cos_bit[stage]); + bf1[28] = bf0[28]; + bf1[29] = bf0[29]; + bf1[30] = bf0[30]; + bf1[31] = bf0[31]; + bf1[32] = bf0[32] + bf0[47]; + bf1[33] = bf0[33] + bf0[46]; + bf1[34] = bf0[34] + bf0[45]; + bf1[35] = bf0[35] + bf0[44]; + bf1[36] = bf0[36] + bf0[43]; + bf1[37] = bf0[37] + bf0[42]; + bf1[38] = bf0[38] + bf0[41]; + bf1[39] = bf0[39] + bf0[40]; + bf1[40] = bf0[39] - bf0[40]; + bf1[41] = bf0[38] - bf0[41]; + bf1[42] = bf0[37] - bf0[42]; + bf1[43] = bf0[36] - bf0[43]; + bf1[44] = bf0[35] - bf0[44]; + bf1[45] = bf0[34] - bf0[45]; + bf1[46] = bf0[33] - bf0[46]; + bf1[47] = bf0[32] - bf0[47]; + bf1[48] = -bf0[48] + bf0[63]; + bf1[49] = -bf0[49] + bf0[62]; + bf1[50] = -bf0[50] + bf0[61]; + bf1[51] = -bf0[51] + bf0[60]; + bf1[52] = -bf0[52] + bf0[59]; + bf1[53] = -bf0[53] + bf0[58]; + bf1[54] = -bf0[54] + bf0[57]; + bf1[55] = -bf0[55] + bf0[56]; + bf1[56] = bf0[55] + bf0[56]; + bf1[57] = bf0[54] + bf0[57]; + bf1[58] = bf0[53] + bf0[58]; + bf1[59] = bf0[52] + bf0[59]; + bf1[60] = bf0[51] + bf0[60]; + bf1[61] = bf0[50] + bf0[61]; + bf1[62] = bf0[49] + bf0[62]; + bf1[63] = bf0[48] + bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 10 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = output; + bf1 = step; + bf1[0] = bf0[0] + bf0[31]; + bf1[1] = bf0[1] + bf0[30]; + bf1[2] = bf0[2] + bf0[29]; + bf1[3] = bf0[3] + bf0[28]; + bf1[4] = bf0[4] + bf0[27]; + bf1[5] = bf0[5] + bf0[26]; + bf1[6] = bf0[6] + bf0[25]; + bf1[7] = bf0[7] + bf0[24]; + bf1[8] = bf0[8] + bf0[23]; + bf1[9] = bf0[9] + bf0[22]; + bf1[10] = bf0[10] + bf0[21]; + bf1[11] = bf0[11] + bf0[20]; + bf1[12] = bf0[12] + bf0[19]; + bf1[13] = bf0[13] + bf0[18]; + bf1[14] = bf0[14] + bf0[17]; + bf1[15] = bf0[15] + bf0[16]; + bf1[16] = bf0[15] - bf0[16]; + bf1[17] = bf0[14] - bf0[17]; + bf1[18] = bf0[13] - bf0[18]; + bf1[19] = bf0[12] - bf0[19]; + bf1[20] = bf0[11] - bf0[20]; + bf1[21] = bf0[10] - bf0[21]; + bf1[22] = bf0[9] - bf0[22]; + bf1[23] = bf0[8] - bf0[23]; + bf1[24] = bf0[7] - bf0[24]; + bf1[25] = bf0[6] - bf0[25]; + bf1[26] = bf0[5] - bf0[26]; + bf1[27] = bf0[4] - bf0[27]; + bf1[28] = bf0[3] - bf0[28]; + bf1[29] = bf0[2] - bf0[29]; + bf1[30] = bf0[1] - bf0[30]; + bf1[31] = bf0[0] - bf0[31]; + bf1[32] = bf0[32]; + bf1[33] = bf0[33]; + bf1[34] = bf0[34]; + bf1[35] = bf0[35]; + bf1[36] = bf0[36]; + bf1[37] = bf0[37]; + bf1[38] = bf0[38]; + bf1[39] = bf0[39]; + bf1[40] = half_btf(-cospi[32], bf0[40], cospi[32], bf0[55], cos_bit[stage]); + bf1[41] = half_btf(-cospi[32], bf0[41], cospi[32], bf0[54], cos_bit[stage]); + bf1[42] = half_btf(-cospi[32], bf0[42], cospi[32], bf0[53], cos_bit[stage]); + bf1[43] = half_btf(-cospi[32], bf0[43], cospi[32], bf0[52], cos_bit[stage]); + bf1[44] = half_btf(-cospi[32], bf0[44], cospi[32], bf0[51], cos_bit[stage]); + bf1[45] = half_btf(-cospi[32], bf0[45], cospi[32], bf0[50], cos_bit[stage]); + bf1[46] = half_btf(-cospi[32], bf0[46], cospi[32], bf0[49], cos_bit[stage]); + bf1[47] = half_btf(-cospi[32], bf0[47], cospi[32], bf0[48], cos_bit[stage]); + bf1[48] = half_btf(cospi[32], bf0[47], cospi[32], bf0[48], cos_bit[stage]); + bf1[49] = half_btf(cospi[32], bf0[46], cospi[32], bf0[49], cos_bit[stage]); + bf1[50] = half_btf(cospi[32], bf0[45], cospi[32], bf0[50], cos_bit[stage]); + bf1[51] = half_btf(cospi[32], bf0[44], cospi[32], bf0[51], cos_bit[stage]); + bf1[52] = half_btf(cospi[32], bf0[43], cospi[32], bf0[52], cos_bit[stage]); + bf1[53] = half_btf(cospi[32], bf0[42], cospi[32], bf0[53], cos_bit[stage]); + bf1[54] = half_btf(cospi[32], bf0[41], cospi[32], bf0[54], cos_bit[stage]); + bf1[55] = half_btf(cospi[32], bf0[40], cospi[32], bf0[55], cos_bit[stage]); + bf1[56] = bf0[56]; + bf1[57] = bf0[57]; + bf1[58] = bf0[58]; + bf1[59] = bf0[59]; + bf1[60] = bf0[60]; + bf1[61] = bf0[61]; + bf1[62] = bf0[62]; + bf1[63] = bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); + + // stage 11 + stage++; + cospi = cospi_arr[cos_bit[stage] - cos_bit_min]; + bf0 = step; + bf1 = output; + bf1[0] = bf0[0] + bf0[63]; + bf1[1] = bf0[1] + bf0[62]; + bf1[2] = bf0[2] + bf0[61]; + bf1[3] = bf0[3] + bf0[60]; + bf1[4] = bf0[4] + bf0[59]; + bf1[5] = bf0[5] + bf0[58]; + bf1[6] = bf0[6] + bf0[57]; + bf1[7] = bf0[7] + bf0[56]; + bf1[8] = bf0[8] + bf0[55]; + bf1[9] = bf0[9] + bf0[54]; + bf1[10] = bf0[10] + bf0[53]; + bf1[11] = bf0[11] + bf0[52]; + bf1[12] = bf0[12] + bf0[51]; + bf1[13] = bf0[13] + bf0[50]; + bf1[14] = bf0[14] + bf0[49]; + bf1[15] = bf0[15] + bf0[48]; + bf1[16] = bf0[16] + bf0[47]; + bf1[17] = bf0[17] + bf0[46]; + bf1[18] = bf0[18] + bf0[45]; + bf1[19] = bf0[19] + bf0[44]; + bf1[20] = bf0[20] + bf0[43]; + bf1[21] = bf0[21] + bf0[42]; + bf1[22] = bf0[22] + bf0[41]; + bf1[23] = bf0[23] + bf0[40]; + bf1[24] = bf0[24] + bf0[39]; + bf1[25] = bf0[25] + bf0[38]; + bf1[26] = bf0[26] + bf0[37]; + bf1[27] = bf0[27] + bf0[36]; + bf1[28] = bf0[28] + bf0[35]; + bf1[29] = bf0[29] + bf0[34]; + bf1[30] = bf0[30] + bf0[33]; + bf1[31] = bf0[31] + bf0[32]; + bf1[32] = bf0[31] - bf0[32]; + bf1[33] = bf0[30] - bf0[33]; + bf1[34] = bf0[29] - bf0[34]; + bf1[35] = bf0[28] - bf0[35]; + bf1[36] = bf0[27] - bf0[36]; + bf1[37] = bf0[26] - bf0[37]; + bf1[38] = bf0[25] - bf0[38]; + bf1[39] = bf0[24] - bf0[39]; + bf1[40] = bf0[23] - bf0[40]; + bf1[41] = bf0[22] - bf0[41]; + bf1[42] = bf0[21] - bf0[42]; + bf1[43] = bf0[20] - bf0[43]; + bf1[44] = bf0[19] - bf0[44]; + bf1[45] = bf0[18] - bf0[45]; + bf1[46] = bf0[17] - bf0[46]; + bf1[47] = bf0[16] - bf0[47]; + bf1[48] = bf0[15] - bf0[48]; + bf1[49] = bf0[14] - bf0[49]; + bf1[50] = bf0[13] - bf0[50]; + bf1[51] = bf0[12] - bf0[51]; + bf1[52] = bf0[11] - bf0[52]; + bf1[53] = bf0[10] - bf0[53]; + bf1[54] = bf0[9] - bf0[54]; + bf1[55] = bf0[8] - bf0[55]; + bf1[56] = bf0[7] - bf0[56]; + bf1[57] = bf0[6] - bf0[57]; + bf1[58] = bf0[5] - bf0[58]; + bf1[59] = bf0[4] - bf0[59]; + bf1[60] = bf0[3] - bf0[60]; + bf1[61] = bf0[2] - bf0[61]; + bf1[62] = bf0[1] - bf0[62]; + bf1[63] = bf0[0] - bf0[63]; + range_check(stage, input, bf1, size, stage_range[stage]); +} +#endif // CONFIG_TX64X64 diff --git a/third_party/aom/av1/common/av1_inv_txfm1d.h b/third_party/aom/av1/common/av1_inv_txfm1d.h new file mode 100644 index 0000000000..9e7a2323ba --- /dev/null +++ b/third_party/aom/av1/common/av1_inv_txfm1d.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_INV_TXFM1D_H_ +#define AV1_INV_TXFM1D_H_ + +#include "av1/common/av1_txfm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_idct4_new(const int32_t *input, int32_t *output, const int8_t *cos_bit, + const int8_t *stage_range); +void av1_idct8_new(const int32_t *input, int32_t *output, const int8_t *cos_bit, + const int8_t *stage_range); +void av1_idct16_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_idct32_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_idct64_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); + +void av1_iadst4_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_iadst8_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_iadst16_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_iadst32_new(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); + +#ifdef __cplusplus +} +#endif + +#endif // AV1_INV_TXFM1D_H_ diff --git a/third_party/aom/av1/common/av1_inv_txfm2d.c b/third_party/aom/av1/common/av1_inv_txfm2d.c new file mode 100644 index 0000000000..d56c7d11f9 --- /dev/null +++ b/third_party/aom/av1/common/av1_inv_txfm2d.c @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./av1_rtcd.h" +#include "av1/common/enums.h" +#include "av1/common/av1_txfm.h" +#include "av1/common/av1_inv_txfm1d.h" +#include "av1/common/av1_inv_txfm2d_cfg.h" + +static INLINE TxfmFunc inv_txfm_type_to_func(TXFM_TYPE txfm_type) { + switch (txfm_type) { + case TXFM_TYPE_DCT4: return av1_idct4_new; + case TXFM_TYPE_DCT8: return av1_idct8_new; + case TXFM_TYPE_DCT16: return av1_idct16_new; + case TXFM_TYPE_DCT32: return av1_idct32_new; + case TXFM_TYPE_ADST4: return av1_iadst4_new; + case TXFM_TYPE_ADST8: return av1_iadst8_new; + case TXFM_TYPE_ADST16: return av1_iadst16_new; + case TXFM_TYPE_ADST32: return av1_iadst32_new; + default: assert(0); return NULL; + } +} + +const TXFM_2D_CFG *inv_txfm_cfg_ls[TX_TYPES][TX_SIZES] = { + // DCT_DCT + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_dct_dct_4, &inv_txfm_2d_cfg_dct_dct_8, + &inv_txfm_2d_cfg_dct_dct_16, &inv_txfm_2d_cfg_dct_dct_32 }, + // ADST_DCT + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_dct_4, &inv_txfm_2d_cfg_adst_dct_8, + &inv_txfm_2d_cfg_adst_dct_16, &inv_txfm_2d_cfg_adst_dct_32 }, + // DCT_ADST + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_dct_adst_4, &inv_txfm_2d_cfg_dct_adst_8, + &inv_txfm_2d_cfg_dct_adst_16, &inv_txfm_2d_cfg_dct_adst_32 }, + // ADST_ADST + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_adst_4, &inv_txfm_2d_cfg_adst_adst_8, + &inv_txfm_2d_cfg_adst_adst_16, &inv_txfm_2d_cfg_adst_adst_32 }, +#if CONFIG_EXT_TX + // FLIPADST_DCT + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_dct_4, &inv_txfm_2d_cfg_adst_dct_8, + &inv_txfm_2d_cfg_adst_dct_16, &inv_txfm_2d_cfg_adst_dct_32 }, + // DCT_FLIPADST + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_dct_adst_4, &inv_txfm_2d_cfg_dct_adst_8, + &inv_txfm_2d_cfg_dct_adst_16, &inv_txfm_2d_cfg_dct_adst_32 }, + // FLIPADST_FLIPADST + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_adst_4, &inv_txfm_2d_cfg_adst_adst_8, + &inv_txfm_2d_cfg_adst_adst_16, &inv_txfm_2d_cfg_adst_adst_32 }, + // ADST_FLIPADST + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_adst_4, &inv_txfm_2d_cfg_adst_adst_8, + &inv_txfm_2d_cfg_adst_adst_16, &inv_txfm_2d_cfg_adst_adst_32 }, + // FLIPADST_ADST + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_adst_4, &inv_txfm_2d_cfg_adst_adst_8, + &inv_txfm_2d_cfg_adst_adst_16, &inv_txfm_2d_cfg_adst_adst_32 }, + { // IDTX +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_adst_4, &inv_txfm_2d_cfg_adst_adst_8, + &inv_txfm_2d_cfg_adst_adst_16, &inv_txfm_2d_cfg_adst_adst_32 }, + { // V_DCT +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_dct_adst_4, &inv_txfm_2d_cfg_dct_adst_8, + &inv_txfm_2d_cfg_dct_adst_16, &inv_txfm_2d_cfg_dct_adst_32 }, + { // H_DCT +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_dct_4, &inv_txfm_2d_cfg_adst_dct_8, + &inv_txfm_2d_cfg_adst_dct_16, &inv_txfm_2d_cfg_adst_dct_32 }, + { // V_ADST +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_adst_4, &inv_txfm_2d_cfg_adst_adst_8, + &inv_txfm_2d_cfg_adst_adst_16, &inv_txfm_2d_cfg_adst_adst_32 }, + { // H_ADST +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_adst_4, &inv_txfm_2d_cfg_adst_adst_8, + &inv_txfm_2d_cfg_adst_adst_16, &inv_txfm_2d_cfg_adst_adst_32 }, + { // V_FLIP_ADST +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_adst_4, &inv_txfm_2d_cfg_adst_adst_8, + &inv_txfm_2d_cfg_adst_adst_16, &inv_txfm_2d_cfg_adst_adst_32 }, + { // H_FLIP_ADST +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_adst_4, &inv_txfm_2d_cfg_adst_adst_8, + &inv_txfm_2d_cfg_adst_adst_16, &inv_txfm_2d_cfg_adst_adst_32 }, +#endif // CONFIG_EXT_TX +}; + +TXFM_2D_FLIP_CFG av1_get_inv_txfm_cfg(int tx_type, int tx_size) { + TXFM_2D_FLIP_CFG cfg; + set_flip_cfg(tx_type, &cfg); + cfg.cfg = inv_txfm_cfg_ls[tx_type][tx_size]; + return cfg; +} + +TXFM_2D_FLIP_CFG av1_get_inv_txfm_64x64_cfg(int tx_type) { + TXFM_2D_FLIP_CFG cfg = { 0, 0, NULL }; + switch (tx_type) { + case DCT_DCT: + cfg.cfg = &inv_txfm_2d_cfg_dct_dct_64; + set_flip_cfg(tx_type, &cfg); + break; + default: assert(0); + } + return cfg; +} + +static INLINE void inv_txfm2d_add_c(const int32_t *input, int16_t *output, + int stride, TXFM_2D_FLIP_CFG *cfg, + int32_t *txfm_buf) { + const int txfm_size = cfg->cfg->txfm_size; + const int8_t *shift = cfg->cfg->shift; + const int8_t *stage_range_col = cfg->cfg->stage_range_col; + const int8_t *stage_range_row = cfg->cfg->stage_range_row; + const int8_t *cos_bit_col = cfg->cfg->cos_bit_col; + const int8_t *cos_bit_row = cfg->cfg->cos_bit_row; + const TxfmFunc txfm_func_col = inv_txfm_type_to_func(cfg->cfg->txfm_type_col); + const TxfmFunc txfm_func_row = inv_txfm_type_to_func(cfg->cfg->txfm_type_row); + + // txfm_buf's length is txfm_size * txfm_size + 2 * txfm_size + // it is used for intermediate data buffering + int32_t *temp_in = txfm_buf; + int32_t *temp_out = temp_in + txfm_size; + int32_t *buf = temp_out + txfm_size; + int32_t *buf_ptr = buf; + int c, r; + + // Rows + for (r = 0; r < txfm_size; ++r) { + txfm_func_row(input, buf_ptr, cos_bit_row, stage_range_row); + round_shift_array(buf_ptr, txfm_size, -shift[0]); + input += txfm_size; + buf_ptr += txfm_size; + } + + // Columns + for (c = 0; c < txfm_size; ++c) { + if (cfg->lr_flip == 0) { + for (r = 0; r < txfm_size; ++r) temp_in[r] = buf[r * txfm_size + c]; + } else { + // flip left right + for (r = 0; r < txfm_size; ++r) + temp_in[r] = buf[r * txfm_size + (txfm_size - c - 1)]; + } + txfm_func_col(temp_in, temp_out, cos_bit_col, stage_range_col); + round_shift_array(temp_out, txfm_size, -shift[1]); + if (cfg->ud_flip == 0) { + for (r = 0; r < txfm_size; ++r) output[r * stride + c] += temp_out[r]; + } else { + // flip upside down + for (r = 0; r < txfm_size; ++r) + output[r * stride + c] += temp_out[txfm_size - r - 1]; + } + } +} + +static INLINE void inv_txfm2d_add_facade(const int32_t *input, uint16_t *output, + int stride, int32_t *txfm_buf, + int tx_type, int tx_size, int bd) { + // output contains the prediction signal which is always positive and smaller + // than (1 << bd) - 1 + // since bd < 16-1, therefore we can treat the uint16_t* output buffer as an + // int16_t* + TXFM_2D_FLIP_CFG cfg = av1_get_inv_txfm_cfg(tx_type, tx_size); + inv_txfm2d_add_c(input, (int16_t *)output, stride, &cfg, txfm_buf); + clamp_block((int16_t *)output, cfg.cfg->txfm_size, stride, 0, (1 << bd) - 1); +} + +void av1_inv_txfm2d_add_4x4_c(const int32_t *input, uint16_t *output, + int stride, int tx_type, int bd) { + int txfm_buf[4 * 4 + 4 + 4]; + inv_txfm2d_add_facade(input, output, stride, txfm_buf, tx_type, TX_4X4, bd); +} + +void av1_inv_txfm2d_add_8x8_c(const int32_t *input, uint16_t *output, + int stride, int tx_type, int bd) { + int txfm_buf[8 * 8 + 8 + 8]; + inv_txfm2d_add_facade(input, output, stride, txfm_buf, tx_type, TX_8X8, bd); +} + +void av1_inv_txfm2d_add_16x16_c(const int32_t *input, uint16_t *output, + int stride, int tx_type, int bd) { + int txfm_buf[16 * 16 + 16 + 16]; + inv_txfm2d_add_facade(input, output, stride, txfm_buf, tx_type, TX_16X16, bd); +} + +void av1_inv_txfm2d_add_32x32_c(const int32_t *input, uint16_t *output, + int stride, int tx_type, int bd) { + int txfm_buf[32 * 32 + 32 + 32]; + inv_txfm2d_add_facade(input, output, stride, txfm_buf, tx_type, TX_32X32, bd); +} + +void av1_inv_txfm2d_add_64x64_c(const int32_t *input, uint16_t *output, + int stride, int tx_type, int bd) { + int txfm_buf[64 * 64 + 64 + 64]; + // output contains the prediction signal which is always positive and smaller + // than (1 << bd) - 1 + // since bd < 16-1, therefore we can treat the uint16_t* output buffer as an + // int16_t* + TXFM_2D_FLIP_CFG cfg = av1_get_inv_txfm_64x64_cfg(tx_type); + inv_txfm2d_add_c(input, (int16_t *)output, stride, &cfg, txfm_buf); + clamp_block((int16_t *)output, 64, stride, 0, (1 << bd) - 1); +} diff --git a/third_party/aom/av1/common/av1_inv_txfm2d_cfg.h b/third_party/aom/av1/common/av1_inv_txfm2d_cfg.h new file mode 100644 index 0000000000..9eabc2e5a2 --- /dev/null +++ b/third_party/aom/av1/common/av1_inv_txfm2d_cfg.h @@ -0,0 +1,447 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_INV_TXFM2D_CFG_H_ +#define AV1_INV_TXFM2D_CFG_H_ +#include "av1/common/av1_inv_txfm1d.h" +// ---------------- config inv_dct_dct_4 ---------------- +static const int8_t inv_shift_dct_dct_4[2] = { 0, -4 }; +static const int8_t inv_stage_range_col_dct_dct_4[4] = { 18, 18, 17, 17 }; +static const int8_t inv_stage_range_row_dct_dct_4[4] = { 18, 18, 18, 18 }; +static const int8_t inv_cos_bit_col_dct_dct_4[4] = { 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_dct_dct_4[4] = { 13, 13, 13, 13 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_4 = { + 4, // .txfm_size + 4, // .stage_num_col + 4, // .stage_num_row + // 0, // .log_scale + inv_shift_dct_dct_4, // .shift + inv_stage_range_col_dct_dct_4, // .stage_range_col + inv_stage_range_row_dct_dct_4, // .stage_range_row + inv_cos_bit_col_dct_dct_4, // .cos_bit_col + inv_cos_bit_row_dct_dct_4, // .cos_bit_row + TXFM_TYPE_DCT4, // .txfm_type_col + TXFM_TYPE_DCT4 +}; // .txfm_type_row + +// ---------------- config inv_dct_dct_8 ---------------- +static const int8_t inv_shift_dct_dct_8[2] = { 0, -5 }; +static const int8_t inv_stage_range_col_dct_dct_8[6] = { + 19, 19, 19, 19, 18, 18 +}; +static const int8_t inv_stage_range_row_dct_dct_8[6] = { + 19, 19, 19, 19, 19, 19 +}; +static const int8_t inv_cos_bit_col_dct_dct_8[6] = { 13, 13, 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_dct_dct_8[6] = { 13, 13, 13, 13, 13, 13 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_8 = { + 8, // .txfm_size + 6, // .stage_num_col + 6, // .stage_num_row + // 0, // .log_scale + inv_shift_dct_dct_8, // .shift + inv_stage_range_col_dct_dct_8, // .stage_range_col + inv_stage_range_row_dct_dct_8, // .stage_range_row + inv_cos_bit_col_dct_dct_8, // .cos_bit_col + inv_cos_bit_row_dct_dct_8, // .cos_bit_row + TXFM_TYPE_DCT8, // .txfm_type_col + TXFM_TYPE_DCT8 +}; // .txfm_type_row + +// ---------------- config inv_dct_dct_16 ---------------- +static const int8_t inv_shift_dct_dct_16[2] = { -1, -5 }; +static const int8_t inv_stage_range_col_dct_dct_16[8] = { 19, 19, 19, 19, + 19, 19, 18, 18 }; +static const int8_t inv_stage_range_row_dct_dct_16[8] = { 20, 20, 20, 20, + 20, 20, 20, 20 }; +static const int8_t inv_cos_bit_col_dct_dct_16[8] = { 13, 13, 13, 13, + 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_dct_dct_16[8] = { 12, 12, 12, 12, + 12, 12, 12, 12 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_16 = { + 16, // .txfm_size + 8, // .stage_num_col + 8, // .stage_num_row + // 0, // .log_scale + inv_shift_dct_dct_16, // .shift + inv_stage_range_col_dct_dct_16, // .stage_range_col + inv_stage_range_row_dct_dct_16, // .stage_range_row + inv_cos_bit_col_dct_dct_16, // .cos_bit_col + inv_cos_bit_row_dct_dct_16, // .cos_bit_row + TXFM_TYPE_DCT16, // .txfm_type_col + TXFM_TYPE_DCT16 +}; // .txfm_type_row + +// ---------------- config inv_dct_dct_32 ---------------- +static const int8_t inv_shift_dct_dct_32[2] = { -1, -5 }; +static const int8_t inv_stage_range_col_dct_dct_32[10] = { 19, 19, 19, 19, 19, + 19, 19, 19, 18, 18 }; +static const int8_t inv_stage_range_row_dct_dct_32[10] = { 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20 }; +static const int8_t inv_cos_bit_col_dct_dct_32[10] = { 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_dct_dct_32[10] = { 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_32 = { + 32, // .txfm_size + 10, // .stage_num_col + 10, // .stage_num_row + // 1, // .log_scale + inv_shift_dct_dct_32, // .shift + inv_stage_range_col_dct_dct_32, // .stage_range_col + inv_stage_range_row_dct_dct_32, // .stage_range_row + inv_cos_bit_col_dct_dct_32, // .cos_bit_col + inv_cos_bit_row_dct_dct_32, // .cos_bit_row + TXFM_TYPE_DCT32, // .txfm_type_col + TXFM_TYPE_DCT32 +}; // .txfm_type_row + +// ---------------- config inv_dct_dct_64 ---------------- +static const int8_t inv_shift_dct_dct_64[2] = { -1, -7 }; +static const int8_t inv_stage_range_col_dct_dct_64[12] = { + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 18, 18 +}; +static const int8_t inv_stage_range_row_dct_dct_64[12] = { + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20 +}; +static const int8_t inv_cos_bit_col_dct_dct_64[12] = { 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_dct_dct_64[12] = { 12, 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_64 = { + 64, // .txfm_size + 12, // .stage_num_col + 12, // .stage_num_row + inv_shift_dct_dct_64, // .shift + inv_stage_range_col_dct_dct_64, // .stage_range_col + inv_stage_range_row_dct_dct_64, // .stage_range_row + inv_cos_bit_col_dct_dct_64, // .cos_bit_col + inv_cos_bit_row_dct_dct_64, // .cos_bit_row + TXFM_TYPE_DCT64, // .txfm_type_col + TXFM_TYPE_DCT64 +}; // .txfm_type_row + +// ---------------- config inv_dct_adst_4 ---------------- +static const int8_t inv_shift_dct_adst_4[2] = { 0, -4 }; +static const int8_t inv_stage_range_col_dct_adst_4[4] = { 18, 18, 17, 17 }; +static const int8_t inv_stage_range_row_dct_adst_4[6] = { + 18, 18, 18, 18, 18, 18 +}; +static const int8_t inv_cos_bit_col_dct_adst_4[4] = { 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_dct_adst_4[6] = { 13, 13, 13, 13, 13, 13 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_adst_4 = { + 4, // .txfm_size + 4, // .stage_num_col + 6, // .stage_num_row + // 0, // .log_scale + inv_shift_dct_adst_4, // .shift + inv_stage_range_col_dct_adst_4, // .stage_range_col + inv_stage_range_row_dct_adst_4, // .stage_range_row + inv_cos_bit_col_dct_adst_4, // .cos_bit_col + inv_cos_bit_row_dct_adst_4, // .cos_bit_row + TXFM_TYPE_DCT4, // .txfm_type_col + TXFM_TYPE_ADST4 +}; // .txfm_type_row + +// ---------------- config inv_dct_adst_8 ---------------- +static const int8_t inv_shift_dct_adst_8[2] = { 0, -5 }; +static const int8_t inv_stage_range_col_dct_adst_8[6] = { + 19, 19, 19, 19, 18, 18 +}; +static const int8_t inv_stage_range_row_dct_adst_8[8] = { 19, 19, 19, 19, + 19, 19, 19, 19 }; +static const int8_t inv_cos_bit_col_dct_adst_8[6] = { 13, 13, 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_dct_adst_8[8] = { 13, 13, 13, 13, + 13, 13, 13, 13 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_adst_8 = { + 8, // .txfm_size + 6, // .stage_num_col + 8, // .stage_num_row + // 0, // .log_scale + inv_shift_dct_adst_8, // .shift + inv_stage_range_col_dct_adst_8, // .stage_range_col + inv_stage_range_row_dct_adst_8, // .stage_range_row + inv_cos_bit_col_dct_adst_8, // .cos_bit_col + inv_cos_bit_row_dct_adst_8, // .cos_bit_row + TXFM_TYPE_DCT8, // .txfm_type_col + TXFM_TYPE_ADST8 +}; // .txfm_type_row + +// ---------------- config inv_dct_adst_16 ---------------- +static const int8_t inv_shift_dct_adst_16[2] = { -1, -5 }; +static const int8_t inv_stage_range_col_dct_adst_16[8] = { 19, 19, 19, 19, + 19, 19, 18, 18 }; +static const int8_t inv_stage_range_row_dct_adst_16[10] = { + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20 +}; +static const int8_t inv_cos_bit_col_dct_adst_16[8] = { 13, 13, 13, 13, + 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_dct_adst_16[10] = { 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_adst_16 = { + 16, // .txfm_size + 8, // .stage_num_col + 10, // .stage_num_row + // 0, // .log_scale + inv_shift_dct_adst_16, // .shift + inv_stage_range_col_dct_adst_16, // .stage_range_col + inv_stage_range_row_dct_adst_16, // .stage_range_row + inv_cos_bit_col_dct_adst_16, // .cos_bit_col + inv_cos_bit_row_dct_adst_16, // .cos_bit_row + TXFM_TYPE_DCT16, // .txfm_type_col + TXFM_TYPE_ADST16 +}; // .txfm_type_row + +// ---------------- config inv_dct_adst_32 ---------------- +static const int8_t inv_shift_dct_adst_32[2] = { -1, -5 }; +static const int8_t inv_stage_range_col_dct_adst_32[10] = { + 19, 19, 19, 19, 19, 19, 19, 19, 18, 18 +}; +static const int8_t inv_stage_range_row_dct_adst_32[12] = { + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20 +}; +static const int8_t inv_cos_bit_col_dct_adst_32[10] = { 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_dct_adst_32[12] = { + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12 +}; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_adst_32 = { + 32, // .txfm_size + 10, // .stage_num_col + 12, // .stage_num_row + // 1, // .log_scale + inv_shift_dct_adst_32, // .shift + inv_stage_range_col_dct_adst_32, // .stage_range_col + inv_stage_range_row_dct_adst_32, // .stage_range_row + inv_cos_bit_col_dct_adst_32, // .cos_bit_col + inv_cos_bit_row_dct_adst_32, // .cos_bit_row + TXFM_TYPE_DCT32, // .txfm_type_col + TXFM_TYPE_ADST32 +}; // .txfm_type_row + +// ---------------- config inv_adst_adst_4 ---------------- +static const int8_t inv_shift_adst_adst_4[2] = { 0, -4 }; +static const int8_t inv_stage_range_col_adst_adst_4[6] = { 18, 18, 18, + 18, 17, 17 }; +static const int8_t inv_stage_range_row_adst_adst_4[6] = { 18, 18, 18, + 18, 18, 18 }; +static const int8_t inv_cos_bit_col_adst_adst_4[6] = { 13, 13, 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_adst_adst_4[6] = { 13, 13, 13, 13, 13, 13 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_adst_adst_4 = { + 4, // .txfm_size + 6, // .stage_num_col + 6, // .stage_num_row + // 0, // .log_scale + inv_shift_adst_adst_4, // .shift + inv_stage_range_col_adst_adst_4, // .stage_range_col + inv_stage_range_row_adst_adst_4, // .stage_range_row + inv_cos_bit_col_adst_adst_4, // .cos_bit_col + inv_cos_bit_row_adst_adst_4, // .cos_bit_row + TXFM_TYPE_ADST4, // .txfm_type_col + TXFM_TYPE_ADST4 +}; // .txfm_type_row + +// ---------------- config inv_adst_adst_8 ---------------- +static const int8_t inv_shift_adst_adst_8[2] = { 0, -5 }; +static const int8_t inv_stage_range_col_adst_adst_8[8] = { 19, 19, 19, 19, + 19, 19, 18, 18 }; +static const int8_t inv_stage_range_row_adst_adst_8[8] = { 19, 19, 19, 19, + 19, 19, 19, 19 }; +static const int8_t inv_cos_bit_col_adst_adst_8[8] = { 13, 13, 13, 13, + 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_adst_adst_8[8] = { 13, 13, 13, 13, + 13, 13, 13, 13 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_adst_adst_8 = { + 8, // .txfm_size + 8, // .stage_num_col + 8, // .stage_num_row + // 0, // .log_scale + inv_shift_adst_adst_8, // .shift + inv_stage_range_col_adst_adst_8, // .stage_range_col + inv_stage_range_row_adst_adst_8, // .stage_range_row + inv_cos_bit_col_adst_adst_8, // .cos_bit_col + inv_cos_bit_row_adst_adst_8, // .cos_bit_row + TXFM_TYPE_ADST8, // .txfm_type_col + TXFM_TYPE_ADST8 +}; // .txfm_type_row + +// ---------------- config inv_adst_adst_16 ---------------- +static const int8_t inv_shift_adst_adst_16[2] = { -1, -5 }; +static const int8_t inv_stage_range_col_adst_adst_16[10] = { + 19, 19, 19, 19, 19, 19, 19, 19, 18, 18 +}; +static const int8_t inv_stage_range_row_adst_adst_16[10] = { + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20 +}; +static const int8_t inv_cos_bit_col_adst_adst_16[10] = { 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_adst_adst_16[10] = { 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_adst_adst_16 = { + 16, // .txfm_size + 10, // .stage_num_col + 10, // .stage_num_row + // 0, // .log_scale + inv_shift_adst_adst_16, // .shift + inv_stage_range_col_adst_adst_16, // .stage_range_col + inv_stage_range_row_adst_adst_16, // .stage_range_row + inv_cos_bit_col_adst_adst_16, // .cos_bit_col + inv_cos_bit_row_adst_adst_16, // .cos_bit_row + TXFM_TYPE_ADST16, // .txfm_type_col + TXFM_TYPE_ADST16 +}; // .txfm_type_row + +// ---------------- config inv_adst_adst_32 ---------------- +static const int8_t inv_shift_adst_adst_32[2] = { -1, -5 }; +static const int8_t inv_stage_range_col_adst_adst_32[12] = { + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 18, 18 +}; +static const int8_t inv_stage_range_row_adst_adst_32[12] = { + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20 +}; +static const int8_t inv_cos_bit_col_adst_adst_32[12] = { + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13 +}; +static const int8_t inv_cos_bit_row_adst_adst_32[12] = { + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12 +}; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_adst_adst_32 = { + 32, // .txfm_size + 12, // .stage_num_col + 12, // .stage_num_row + // 1, // .log_scale + inv_shift_adst_adst_32, // .shift + inv_stage_range_col_adst_adst_32, // .stage_range_col + inv_stage_range_row_adst_adst_32, // .stage_range_row + inv_cos_bit_col_adst_adst_32, // .cos_bit_col + inv_cos_bit_row_adst_adst_32, // .cos_bit_row + TXFM_TYPE_ADST32, // .txfm_type_col + TXFM_TYPE_ADST32 +}; // .txfm_type_row + +// ---------------- config inv_adst_dct_4 ---------------- +static const int8_t inv_shift_adst_dct_4[2] = { 0, -4 }; +static const int8_t inv_stage_range_col_adst_dct_4[6] = { + 18, 18, 18, 18, 17, 17 +}; +static const int8_t inv_stage_range_row_adst_dct_4[4] = { 18, 18, 18, 18 }; +static const int8_t inv_cos_bit_col_adst_dct_4[6] = { 13, 13, 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_adst_dct_4[4] = { 13, 13, 13, 13 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_adst_dct_4 = { + 4, // .txfm_size + 6, // .stage_num_col + 4, // .stage_num_row + // 0, // .log_scale + inv_shift_adst_dct_4, // .shift + inv_stage_range_col_adst_dct_4, // .stage_range_col + inv_stage_range_row_adst_dct_4, // .stage_range_row + inv_cos_bit_col_adst_dct_4, // .cos_bit_col + inv_cos_bit_row_adst_dct_4, // .cos_bit_row + TXFM_TYPE_ADST4, // .txfm_type_col + TXFM_TYPE_DCT4 +}; // .txfm_type_row + +// ---------------- config inv_adst_dct_8 ---------------- +static const int8_t inv_shift_adst_dct_8[2] = { 0, -5 }; +static const int8_t inv_stage_range_col_adst_dct_8[8] = { 19, 19, 19, 19, + 19, 19, 18, 18 }; +static const int8_t inv_stage_range_row_adst_dct_8[6] = { + 19, 19, 19, 19, 19, 19 +}; +static const int8_t inv_cos_bit_col_adst_dct_8[8] = { 13, 13, 13, 13, + 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_adst_dct_8[6] = { 13, 13, 13, 13, 13, 13 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_adst_dct_8 = { + 8, // .txfm_size + 8, // .stage_num_col + 6, // .stage_num_row + // 0, // .log_scale + inv_shift_adst_dct_8, // .shift + inv_stage_range_col_adst_dct_8, // .stage_range_col + inv_stage_range_row_adst_dct_8, // .stage_range_row + inv_cos_bit_col_adst_dct_8, // .cos_bit_col + inv_cos_bit_row_adst_dct_8, // .cos_bit_row + TXFM_TYPE_ADST8, // .txfm_type_col + TXFM_TYPE_DCT8 +}; // .txfm_type_row + +// ---------------- config inv_adst_dct_16 ---------------- +static const int8_t inv_shift_adst_dct_16[2] = { -1, -5 }; +static const int8_t inv_stage_range_col_adst_dct_16[10] = { + 19, 19, 19, 19, 19, 19, 19, 19, 18, 18 +}; +static const int8_t inv_stage_range_row_adst_dct_16[8] = { 20, 20, 20, 20, + 20, 20, 20, 20 }; +static const int8_t inv_cos_bit_col_adst_dct_16[10] = { 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13 }; +static const int8_t inv_cos_bit_row_adst_dct_16[8] = { 12, 12, 12, 12, + 12, 12, 12, 12 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_adst_dct_16 = { + 16, // .txfm_size + 10, // .stage_num_col + 8, // .stage_num_row + // 0, // .log_scale + inv_shift_adst_dct_16, // .shift + inv_stage_range_col_adst_dct_16, // .stage_range_col + inv_stage_range_row_adst_dct_16, // .stage_range_row + inv_cos_bit_col_adst_dct_16, // .cos_bit_col + inv_cos_bit_row_adst_dct_16, // .cos_bit_row + TXFM_TYPE_ADST16, // .txfm_type_col + TXFM_TYPE_DCT16 +}; // .txfm_type_row + +// ---------------- config inv_adst_dct_32 ---------------- +static const int8_t inv_shift_adst_dct_32[2] = { -1, -5 }; +static const int8_t inv_stage_range_col_adst_dct_32[12] = { + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 18, 18 +}; +static const int8_t inv_stage_range_row_adst_dct_32[10] = { + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20 +}; +static const int8_t inv_cos_bit_col_adst_dct_32[12] = { + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13 +}; +static const int8_t inv_cos_bit_row_adst_dct_32[10] = { 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12 }; + +static const TXFM_2D_CFG inv_txfm_2d_cfg_adst_dct_32 = { + 32, // .txfm_size + 12, // .stage_num_col + 10, // .stage_num_row + // 1, // .log_scale + inv_shift_adst_dct_32, // .shift + inv_stage_range_col_adst_dct_32, // .stage_range_col + inv_stage_range_row_adst_dct_32, // .stage_range_row + inv_cos_bit_col_adst_dct_32, // .cos_bit_col + inv_cos_bit_row_adst_dct_32, // .cos_bit_row + TXFM_TYPE_ADST32, // .txfm_type_col + TXFM_TYPE_DCT32 +}; // .txfm_type_row + +extern const TXFM_2D_CFG *inv_txfm_cfg_ls[TX_TYPES][TX_SIZES]; + +#endif // AV1_INV_TXFM2D_CFG_H_ diff --git a/third_party/aom/av1/common/av1_loopfilter.c b/third_party/aom/av1/common/av1_loopfilter.c new file mode 100644 index 0000000000..5308717954 --- /dev/null +++ b/third_party/aom/av1/common/av1_loopfilter.c @@ -0,0 +1,2336 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "av1/common/av1_loopfilter.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/reconinter.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +#include "av1/common/seg_common.h" + +#define CONFIG_PARALLEL_DEBLOCKING_15TAPLUMAONLY 0 + +// 64 bit masks for left transform size. Each 1 represents a position where +// we should apply a loop filter across the left border of an 8x8 block +// boundary. +// +// In the case of TX_16X16-> ( in low order byte first we end up with +// a mask that looks like this +// +// 10101010 +// 10101010 +// 10101010 +// 10101010 +// 10101010 +// 10101010 +// 10101010 +// 10101010 +// +// A loopfilter should be applied to every other 8x8 horizontally. +static const uint64_t left_64x64_txform_mask[TX_SIZES] = { +#if CONFIG_CB4X4 + 0xffffffffffffffffULL, // TX_2X2 +#endif + 0xffffffffffffffffULL, // TX_4X4 + 0xffffffffffffffffULL, // TX_8x8 + 0x5555555555555555ULL, // TX_16x16 + 0x1111111111111111ULL, // TX_32x32 +#if CONFIG_TX64X64 + 0x0101010101010101ULL, // TX_64x64 +#endif // CONFIG_TX64X64 +}; + +// 64 bit masks for above transform size. Each 1 represents a position where +// we should apply a loop filter across the top border of an 8x8 block +// boundary. +// +// In the case of TX_32x32 -> ( in low order byte first we end up with +// a mask that looks like this +// +// 11111111 +// 00000000 +// 00000000 +// 00000000 +// 11111111 +// 00000000 +// 00000000 +// 00000000 +// +// A loopfilter should be applied to every other 4 the row vertically. +static const uint64_t above_64x64_txform_mask[TX_SIZES] = { +#if CONFIG_CB4X4 + 0xffffffffffffffffULL, // TX_4X4 +#endif + 0xffffffffffffffffULL, // TX_4X4 + 0xffffffffffffffffULL, // TX_8x8 + 0x00ff00ff00ff00ffULL, // TX_16x16 + 0x000000ff000000ffULL, // TX_32x32 +#if CONFIG_TX64X64 + 0x00000000000000ffULL, // TX_64x64 +#endif // CONFIG_TX64X64 +}; + +// 64 bit masks for prediction sizes (left). Each 1 represents a position +// where left border of an 8x8 block. These are aligned to the right most +// appropriate bit, and then shifted into place. +// +// In the case of TX_16x32 -> ( low order byte first ) we end up with +// a mask that looks like this : +// +// 10000000 +// 10000000 +// 10000000 +// 10000000 +// 00000000 +// 00000000 +// 00000000 +// 00000000 +static const uint64_t left_prediction_mask[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 0x0000000000000001ULL, // BLOCK_2X2, + 0x0000000000000001ULL, // BLOCK_2X4, + 0x0000000000000001ULL, // BLOCK_4X2, +#endif + 0x0000000000000001ULL, // BLOCK_4X4, + 0x0000000000000001ULL, // BLOCK_4X8, + 0x0000000000000001ULL, // BLOCK_8X4, + 0x0000000000000001ULL, // BLOCK_8X8, + 0x0000000000000101ULL, // BLOCK_8X16, + 0x0000000000000001ULL, // BLOCK_16X8, + 0x0000000000000101ULL, // BLOCK_16X16, + 0x0000000001010101ULL, // BLOCK_16X32, + 0x0000000000000101ULL, // BLOCK_32X16, + 0x0000000001010101ULL, // BLOCK_32X32, + 0x0101010101010101ULL, // BLOCK_32X64, + 0x0000000001010101ULL, // BLOCK_64X32, + 0x0101010101010101ULL, // BLOCK_64X64 +}; + +// 64 bit mask to shift and set for each prediction size. +static const uint64_t above_prediction_mask[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 0x0000000000000001ULL, // BLOCK_2X2 + 0x0000000000000001ULL, // BLOCK_2X4 + 0x0000000000000001ULL, // BLOCK_4X2 +#endif + 0x0000000000000001ULL, // BLOCK_4X4 + 0x0000000000000001ULL, // BLOCK_4X8 + 0x0000000000000001ULL, // BLOCK_8X4 + 0x0000000000000001ULL, // BLOCK_8X8 + 0x0000000000000001ULL, // BLOCK_8X16, + 0x0000000000000003ULL, // BLOCK_16X8 + 0x0000000000000003ULL, // BLOCK_16X16 + 0x0000000000000003ULL, // BLOCK_16X32, + 0x000000000000000fULL, // BLOCK_32X16, + 0x000000000000000fULL, // BLOCK_32X32, + 0x000000000000000fULL, // BLOCK_32X64, + 0x00000000000000ffULL, // BLOCK_64X32, + 0x00000000000000ffULL, // BLOCK_64X64 +}; +// 64 bit mask to shift and set for each prediction size. A bit is set for +// each 8x8 block that would be in the left most block of the given block +// size in the 64x64 block. +static const uint64_t size_mask[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 0x0000000000000001ULL, // BLOCK_2X2 + 0x0000000000000001ULL, // BLOCK_2X4 + 0x0000000000000001ULL, // BLOCK_4X2 +#endif + 0x0000000000000001ULL, // BLOCK_4X4 + 0x0000000000000001ULL, // BLOCK_4X8 + 0x0000000000000001ULL, // BLOCK_8X4 + 0x0000000000000001ULL, // BLOCK_8X8 + 0x0000000000000101ULL, // BLOCK_8X16, + 0x0000000000000003ULL, // BLOCK_16X8 + 0x0000000000000303ULL, // BLOCK_16X16 + 0x0000000003030303ULL, // BLOCK_16X32, + 0x0000000000000f0fULL, // BLOCK_32X16, + 0x000000000f0f0f0fULL, // BLOCK_32X32, + 0x0f0f0f0f0f0f0f0fULL, // BLOCK_32X64, + 0x00000000ffffffffULL, // BLOCK_64X32, + 0xffffffffffffffffULL, // BLOCK_64X64 +}; + +// These are used for masking the left and above 32x32 borders. +static const uint64_t left_border = 0x1111111111111111ULL; +static const uint64_t above_border = 0x000000ff000000ffULL; + +// 16 bit masks for uv transform sizes. +static const uint16_t left_64x64_txform_mask_uv[TX_SIZES] = { +#if CONFIG_CB4X4 + 0xffff, // TX_2X2 +#endif + 0xffff, // TX_4X4 + 0xffff, // TX_8x8 + 0x5555, // TX_16x16 + 0x1111, // TX_32x32 +#if CONFIG_TX64X64 + 0x0101, // TX_64x64, never used +#endif // CONFIG_TX64X64 +}; + +static const uint16_t above_64x64_txform_mask_uv[TX_SIZES] = { +#if CONFIG_CB4X4 + 0xffff, // TX_2X2 +#endif + 0xffff, // TX_4X4 + 0xffff, // TX_8x8 + 0x0f0f, // TX_16x16 + 0x000f, // TX_32x32 +#if CONFIG_TX64X64 + 0x0003, // TX_64x64, never used +#endif // CONFIG_TX64X64 +}; + +// 16 bit left mask to shift and set for each uv prediction size. +static const uint16_t left_prediction_mask_uv[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 0x0001, // BLOCK_2X2, + 0x0001, // BLOCK_2X4, + 0x0001, // BLOCK_4X2, +#endif + 0x0001, // BLOCK_4X4, + 0x0001, // BLOCK_4X8, + 0x0001, // BLOCK_8X4, + 0x0001, // BLOCK_8X8, + 0x0001, // BLOCK_8X16, + 0x0001, // BLOCK_16X8, + 0x0001, // BLOCK_16X16, + 0x0011, // BLOCK_16X32, + 0x0001, // BLOCK_32X16, + 0x0011, // BLOCK_32X32, + 0x1111, // BLOCK_32X64 + 0x0011, // BLOCK_64X32, + 0x1111, // BLOCK_64X64 +}; +// 16 bit above mask to shift and set for uv each prediction size. +static const uint16_t above_prediction_mask_uv[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 0x0001, // BLOCK_2X2 + 0x0001, // BLOCK_2X4 + 0x0001, // BLOCK_4X2 +#endif + 0x0001, // BLOCK_4X4 + 0x0001, // BLOCK_4X8 + 0x0001, // BLOCK_8X4 + 0x0001, // BLOCK_8X8 + 0x0001, // BLOCK_8X16, + 0x0001, // BLOCK_16X8 + 0x0001, // BLOCK_16X16 + 0x0001, // BLOCK_16X32, + 0x0003, // BLOCK_32X16, + 0x0003, // BLOCK_32X32, + 0x0003, // BLOCK_32X64, + 0x000f, // BLOCK_64X32, + 0x000f, // BLOCK_64X64 +}; + +// 64 bit mask to shift and set for each uv prediction size +static const uint16_t size_mask_uv[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 0x0001, // BLOCK_2X2 + 0x0001, // BLOCK_2X4 + 0x0001, // BLOCK_4X2 +#endif + 0x0001, // BLOCK_4X4 + 0x0001, // BLOCK_4X8 + 0x0001, // BLOCK_8X4 + 0x0001, // BLOCK_8X8 + 0x0001, // BLOCK_8X16, + 0x0001, // BLOCK_16X8 + 0x0001, // BLOCK_16X16 + 0x0011, // BLOCK_16X32, + 0x0003, // BLOCK_32X16, + 0x0033, // BLOCK_32X32, + 0x3333, // BLOCK_32X64, + 0x00ff, // BLOCK_64X32, + 0xffff, // BLOCK_64X64 +}; +static const uint16_t left_border_uv = 0x1111; +static const uint16_t above_border_uv = 0x000f; + +static const int mode_lf_lut[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // INTRA_MODES +#if CONFIG_ALT_INTRA + 0, +#endif + 1, 1, 0, 1, // INTER_MODES (ZEROMV == 0) +#if CONFIG_EXT_INTER + 1, 1, 1, 1, 1, 1, 1, 1, 0, 1 // INTER_COMPOUND_MODES (ZERO_ZEROMV == 0) +#endif // CONFIG_EXT_INTER +}; + +static void update_sharpness(loop_filter_info_n *lfi, int sharpness_lvl) { + int lvl; + + // For each possible value for the loop filter fill out limits + for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++) { + // Set loop filter parameters that control sharpness. + int block_inside_limit = lvl >> ((sharpness_lvl > 0) + (sharpness_lvl > 4)); + + if (sharpness_lvl > 0) { + if (block_inside_limit > (9 - sharpness_lvl)) + block_inside_limit = (9 - sharpness_lvl); + } + + if (block_inside_limit < 1) block_inside_limit = 1; + + memset(lfi->lfthr[lvl].lim, block_inside_limit, SIMD_WIDTH); + memset(lfi->lfthr[lvl].mblim, (2 * (lvl + 2) + block_inside_limit), + SIMD_WIDTH); + } +} +#if CONFIG_EXT_DELTA_Q +static uint8_t get_filter_level(const AV1_COMMON *cm, + const loop_filter_info_n *lfi_n, + const MB_MODE_INFO *mbmi) { +#if CONFIG_SUPERTX + const int segment_id = AOMMIN(mbmi->segment_id, mbmi->segment_id_supertx); + assert( + IMPLIES(supertx_enabled(mbmi), mbmi->segment_id_supertx != MAX_SEGMENTS)); + assert(IMPLIES(supertx_enabled(mbmi), + mbmi->segment_id_supertx <= mbmi->segment_id)); +#else + const int segment_id = mbmi->segment_id; +#endif // CONFIG_SUPERTX + if (cm->delta_lf_present_flag) { + int lvl_seg = clamp(mbmi->current_delta_lf_from_base + cm->lf.filter_level, + 0, MAX_LOOP_FILTER); + const int scale = 1 << (lvl_seg >> 5); + if (segfeature_active(&cm->seg, segment_id, SEG_LVL_ALT_LF)) { + const int data = get_segdata(&cm->seg, segment_id, SEG_LVL_ALT_LF); + lvl_seg = + clamp(cm->seg.abs_delta == SEGMENT_ABSDATA ? data : lvl_seg + data, 0, + MAX_LOOP_FILTER); + } + + if (cm->lf.mode_ref_delta_enabled) { + lvl_seg += cm->lf.ref_deltas[mbmi->ref_frame[0]] * scale; + if (mbmi->ref_frame[0] > INTRA_FRAME) + lvl_seg += cm->lf.mode_deltas[mode_lf_lut[mbmi->mode]] * scale; + lvl_seg = clamp(lvl_seg, 0, MAX_LOOP_FILTER); + } + return lvl_seg; + } else { + return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]]; + } +} +#else +static uint8_t get_filter_level(const loop_filter_info_n *lfi_n, + const MB_MODE_INFO *mbmi) { +#if CONFIG_SUPERTX + const int segment_id = AOMMIN(mbmi->segment_id, mbmi->segment_id_supertx); + assert( + IMPLIES(supertx_enabled(mbmi), mbmi->segment_id_supertx != MAX_SEGMENTS)); + assert(IMPLIES(supertx_enabled(mbmi), + mbmi->segment_id_supertx <= mbmi->segment_id)); +#else + const int segment_id = mbmi->segment_id; +#endif // CONFIG_SUPERTX + return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]]; +} +#endif + +#define NELEMENTS(x) (sizeof((x)) / sizeof((x)[0])) + +void av1_loop_filter_init(AV1_COMMON *cm) { + assert(MB_MODE_COUNT == NELEMENTS(mode_lf_lut)); + loop_filter_info_n *lfi = &cm->lf_info; + struct loopfilter *lf = &cm->lf; + int lvl; + + // init limits for given sharpness + update_sharpness(lfi, lf->sharpness_level); + lf->last_sharpness_level = lf->sharpness_level; + + // init hev threshold const vectors + for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++) + memset(lfi->lfthr[lvl].hev_thr, (lvl >> 4), SIMD_WIDTH); +} + +void av1_loop_filter_frame_init(AV1_COMMON *cm, int default_filt_lvl) { + int seg_id; + // n_shift is the multiplier for lf_deltas + // the multiplier is 1 for when filter_lvl is between 0 and 31; + // 2 when filter_lvl is between 32 and 63 + const int scale = 1 << (default_filt_lvl >> 5); + loop_filter_info_n *const lfi = &cm->lf_info; + struct loopfilter *const lf = &cm->lf; + const struct segmentation *const seg = &cm->seg; + + // update limits if sharpness has changed + if (lf->last_sharpness_level != lf->sharpness_level) { + update_sharpness(lfi, lf->sharpness_level); + lf->last_sharpness_level = lf->sharpness_level; + } + + for (seg_id = 0; seg_id < MAX_SEGMENTS; seg_id++) { + int lvl_seg = default_filt_lvl; + if (segfeature_active(seg, seg_id, SEG_LVL_ALT_LF)) { + const int data = get_segdata(seg, seg_id, SEG_LVL_ALT_LF); + lvl_seg = clamp( + seg->abs_delta == SEGMENT_ABSDATA ? data : default_filt_lvl + data, 0, + MAX_LOOP_FILTER); + } + + if (!lf->mode_ref_delta_enabled) { + // we could get rid of this if we assume that deltas are set to + // zero when not in use; encoder always uses deltas + memset(lfi->lvl[seg_id], lvl_seg, sizeof(lfi->lvl[seg_id])); + } else { + int ref, mode; + const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale; + lfi->lvl[seg_id][INTRA_FRAME][0] = clamp(intra_lvl, 0, MAX_LOOP_FILTER); + + for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) { + for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) { + const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale + + lf->mode_deltas[mode] * scale; + lfi->lvl[seg_id][ref][mode] = clamp(inter_lvl, 0, MAX_LOOP_FILTER); + } + } + } + } +} + +static void filter_selectively_vert_row2(int subsampling_factor, uint8_t *s, + int pitch, unsigned int mask_16x16_l, + unsigned int mask_8x8_l, + unsigned int mask_4x4_l, + unsigned int mask_4x4_int_l, + const loop_filter_info_n *lfi_n, + const uint8_t *lfl) { + const int mask_shift = subsampling_factor ? 4 : 8; + const int mask_cutoff = subsampling_factor ? 0xf : 0xff; + const int lfl_forward = subsampling_factor ? 4 : 8; + + unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff; + unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff; + unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff; + unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff; + unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff; + unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff; + unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff; + unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff; + unsigned int mask; + + for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 | + mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1; + mask; mask >>= 1) { + const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl; + const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward); + + if (mask & 1) { + if ((mask_16x16_0 | mask_16x16_1) & 1) { + if ((mask_16x16_0 & mask_16x16_1) & 1) { + aom_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr); + } else if (mask_16x16_0 & 1) { + aom_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr); + } else { + aom_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim, + lfi1->hev_thr); + } + } + + if ((mask_8x8_0 | mask_8x8_1) & 1) { + if ((mask_8x8_0 & mask_8x8_1) & 1) { + aom_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, lfi1->mblim, lfi1->lim, + lfi1->hev_thr); + } else if (mask_8x8_0 & 1) { + aom_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr); + } else { + aom_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim, + lfi1->hev_thr); + } + } + + if ((mask_4x4_0 | mask_4x4_1) & 1) { + if ((mask_4x4_0 & mask_4x4_1) & 1) { + aom_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, lfi1->mblim, lfi1->lim, + lfi1->hev_thr); + } else if (mask_4x4_0 & 1) { + aom_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr); + } else { + aom_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim, + lfi1->hev_thr); + } + } + + if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) { + if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) { + aom_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, lfi1->mblim, lfi1->lim, + lfi1->hev_thr); + } else if (mask_4x4_int_0 & 1) { + aom_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr); + } else { + aom_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim, lfi1->lim, + lfi1->hev_thr); + } + } + } + + s += 8; + lfl += 1; + mask_16x16_0 >>= 1; + mask_8x8_0 >>= 1; + mask_4x4_0 >>= 1; + mask_4x4_int_0 >>= 1; + mask_16x16_1 >>= 1; + mask_8x8_1 >>= 1; + mask_4x4_1 >>= 1; + mask_4x4_int_1 >>= 1; + } +} + +#if CONFIG_HIGHBITDEPTH +static void highbd_filter_selectively_vert_row2( + int subsampling_factor, uint16_t *s, int pitch, unsigned int mask_16x16_l, + unsigned int mask_8x8_l, unsigned int mask_4x4_l, + unsigned int mask_4x4_int_l, const loop_filter_info_n *lfi_n, + const uint8_t *lfl, int bd) { + const int mask_shift = subsampling_factor ? 4 : 8; + const int mask_cutoff = subsampling_factor ? 0xf : 0xff; + const int lfl_forward = subsampling_factor ? 4 : 8; + + unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff; + unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff; + unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff; + unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff; + unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff; + unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff; + unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff; + unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff; + unsigned int mask; + + for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 | + mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1; + mask; mask >>= 1) { + const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl; + const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward); + + if (mask & 1) { + if ((mask_16x16_0 | mask_16x16_1) & 1) { + if ((mask_16x16_0 & mask_16x16_1) & 1) { + aom_highbd_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, bd); + } else if (mask_16x16_0 & 1) { + aom_highbd_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, bd); + } else { + aom_highbd_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim, + lfi1->lim, lfi1->hev_thr, bd); + } + } + + if ((mask_8x8_0 | mask_8x8_1) & 1) { + if ((mask_8x8_0 & mask_8x8_1) & 1) { + aom_highbd_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, lfi1->mblim, lfi1->lim, + lfi1->hev_thr, bd); + } else if (mask_8x8_0 & 1) { + aom_highbd_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, bd); + } else { + aom_highbd_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim, + lfi1->lim, lfi1->hev_thr, bd); + } + } + + if ((mask_4x4_0 | mask_4x4_1) & 1) { + if ((mask_4x4_0 & mask_4x4_1) & 1) { + aom_highbd_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, lfi1->mblim, lfi1->lim, + lfi1->hev_thr, bd); + } else if (mask_4x4_0 & 1) { + aom_highbd_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, bd); + } else { + aom_highbd_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim, + lfi1->lim, lfi1->hev_thr, bd); + } + } + + if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) { + if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) { + aom_highbd_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, lfi1->mblim, lfi1->lim, + lfi1->hev_thr, bd); + } else if (mask_4x4_int_0 & 1) { + aom_highbd_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, bd); + } else { + aom_highbd_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim, + lfi1->lim, lfi1->hev_thr, bd); + } + } + } + + s += 8; + lfl += 1; + mask_16x16_0 >>= 1; + mask_8x8_0 >>= 1; + mask_4x4_0 >>= 1; + mask_4x4_int_0 >>= 1; + mask_16x16_1 >>= 1; + mask_8x8_1 >>= 1; + mask_4x4_1 >>= 1; + mask_4x4_int_1 >>= 1; + } +} +#endif // CONFIG_HIGHBITDEPTH + +static void filter_selectively_horiz( + uint8_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8, + unsigned int mask_4x4, unsigned int mask_4x4_int, + const loop_filter_info_n *lfi_n, const uint8_t *lfl) { + unsigned int mask; + int count; + + for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask; + mask >>= count) { + const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl; + + count = 1; + if (mask & 1) { + if (mask_16x16 & 1) { + if ((mask_16x16 & 3) == 3) { + aom_lpf_horizontal_edge_16(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr); + count = 2; + } else { + aom_lpf_horizontal_edge_8(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr); + } + } else if (mask_8x8 & 1) { + if ((mask_8x8 & 3) == 3) { + // Next block's thresholds. + const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1); + + aom_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, lfin->mblim, lfin->lim, + lfin->hev_thr); + + if ((mask_4x4_int & 3) == 3) { + aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim, + lfi->lim, lfi->hev_thr, lfin->mblim, + lfin->lim, lfin->hev_thr); + } else { + if (mask_4x4_int & 1) + aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr); + else if (mask_4x4_int & 2) + aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim, + lfin->lim, lfin->hev_thr); + } + count = 2; + } else { + aom_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr); + + if (mask_4x4_int & 1) + aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr); + } + } else if (mask_4x4 & 1) { + if ((mask_4x4 & 3) == 3) { + // Next block's thresholds. + const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1); + + aom_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, lfin->mblim, lfin->lim, + lfin->hev_thr); + if ((mask_4x4_int & 3) == 3) { + aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim, + lfi->lim, lfi->hev_thr, lfin->mblim, + lfin->lim, lfin->hev_thr); + } else { + if (mask_4x4_int & 1) + aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr); + else if (mask_4x4_int & 2) + aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim, + lfin->lim, lfin->hev_thr); + } + count = 2; + } else { + aom_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr); + + if (mask_4x4_int & 1) + aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr); + } + } else if (mask_4x4_int & 1) { + aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr); + } + } + s += 8 * count; + lfl += count; + mask_16x16 >>= count; + mask_8x8 >>= count; + mask_4x4 >>= count; + mask_4x4_int >>= count; + } +} + +#if CONFIG_HIGHBITDEPTH +static void highbd_filter_selectively_horiz( + uint16_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8, + unsigned int mask_4x4, unsigned int mask_4x4_int, + const loop_filter_info_n *lfi_n, const uint8_t *lfl, int bd) { + unsigned int mask; + int count; + + for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask; + mask >>= count) { + const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl; + + count = 1; + if (mask & 1) { + if (mask_16x16 & 1) { + if ((mask_16x16 & 3) == 3) { + aom_highbd_lpf_horizontal_edge_16(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, bd); + count = 2; + } else { + aom_highbd_lpf_horizontal_edge_8(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, bd); + } + } else if (mask_8x8 & 1) { + if ((mask_8x8 & 3) == 3) { + // Next block's thresholds. + const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1); + + aom_highbd_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, lfin->mblim, lfin->lim, + lfin->hev_thr, bd); + + if ((mask_4x4_int & 3) == 3) { + aom_highbd_lpf_horizontal_4_dual( + s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr, + lfin->mblim, lfin->lim, lfin->hev_thr, bd); + } else { + if (mask_4x4_int & 1) { + aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, + lfi->lim, lfi->hev_thr, bd); + } else if (mask_4x4_int & 2) { + aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim, + lfin->lim, lfin->hev_thr, bd); + } + } + count = 2; + } else { + aom_highbd_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, bd); + + if (mask_4x4_int & 1) { + aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, + lfi->lim, lfi->hev_thr, bd); + } + } + } else if (mask_4x4 & 1) { + if ((mask_4x4 & 3) == 3) { + // Next block's thresholds. + const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1); + + aom_highbd_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, lfin->mblim, lfin->lim, + lfin->hev_thr, bd); + if ((mask_4x4_int & 3) == 3) { + aom_highbd_lpf_horizontal_4_dual( + s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr, + lfin->mblim, lfin->lim, lfin->hev_thr, bd); + } else { + if (mask_4x4_int & 1) { + aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, + lfi->lim, lfi->hev_thr, bd); + } else if (mask_4x4_int & 2) { + aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim, + lfin->lim, lfin->hev_thr, bd); + } + } + count = 2; + } else { + aom_highbd_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, bd); + + if (mask_4x4_int & 1) { + aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, + lfi->lim, lfi->hev_thr, bd); + } + } + } else if (mask_4x4_int & 1) { + aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, bd); + } + } + s += 8 * count; + lfl += count; + mask_16x16 >>= count; + mask_8x8 >>= count; + mask_4x4 >>= count; + mask_4x4_int >>= count; + } +} +#endif // CONFIG_HIGHBITDEPTH + +// This function ors into the current lfm structure, where to do loop +// filters for the specific mi we are looking at. It uses information +// including the block_size_type (32x16, 32x32, etc.), the transform size, +// whether there were any coefficients encoded, and the loop filter strength +// block we are currently looking at. Shift is used to position the +// 1's we produce. +// TODO(JBB) Need another function for different resolution color.. +static void build_masks(AV1_COMMON *const cm, + const loop_filter_info_n *const lfi_n, + const MODE_INFO *mi, const int shift_y, + const int shift_uv, LOOP_FILTER_MASK *lfm) { + const MB_MODE_INFO *mbmi = &mi->mbmi; + const BLOCK_SIZE block_size = mbmi->sb_type; + // TODO(debargha): Check if masks can be setup correctly when + // rectangular transfroms are used with the EXT_TX expt. + const TX_SIZE tx_size_y = txsize_sqr_map[mbmi->tx_size]; + const TX_SIZE tx_size_y_left = txsize_horz_map[mbmi->tx_size]; + const TX_SIZE tx_size_y_above = txsize_vert_map[mbmi->tx_size]; + const TX_SIZE tx_size_uv = + txsize_sqr_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]]; + const TX_SIZE tx_size_uv_left = + txsize_horz_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]]; + const TX_SIZE tx_size_uv_above = + txsize_vert_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]]; +#if CONFIG_EXT_DELTA_Q + const int filter_level = get_filter_level(cm, lfi_n, mbmi); +#else + const int filter_level = get_filter_level(lfi_n, mbmi); + (void)cm; +#endif + uint64_t *const left_y = &lfm->left_y[tx_size_y_left]; + uint64_t *const above_y = &lfm->above_y[tx_size_y_above]; + uint64_t *const int_4x4_y = &lfm->int_4x4_y; + uint16_t *const left_uv = &lfm->left_uv[tx_size_uv_left]; + uint16_t *const above_uv = &lfm->above_uv[tx_size_uv_above]; + uint16_t *const int_4x4_uv = &lfm->left_int_4x4_uv; + int i; + + // If filter level is 0 we don't loop filter. + if (!filter_level) { + return; + } else { + const int w = num_8x8_blocks_wide_lookup[block_size]; + const int h = num_8x8_blocks_high_lookup[block_size]; + const int row = (shift_y >> MAX_MIB_SIZE_LOG2); + const int col = shift_y - (row << MAX_MIB_SIZE_LOG2); + + for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w); + } + + // These set 1 in the current block size for the block size edges. + // For instance if the block size is 32x16, we'll set: + // above = 1111 + // 0000 + // and + // left = 1000 + // = 1000 + // NOTE : In this example the low bit is left most ( 1000 ) is stored as + // 1, not 8... + // + // U and V set things on a 16 bit scale. + // + *above_y |= above_prediction_mask[block_size] << shift_y; + *above_uv |= above_prediction_mask_uv[block_size] << shift_uv; + *left_y |= left_prediction_mask[block_size] << shift_y; + *left_uv |= left_prediction_mask_uv[block_size] << shift_uv; + + // If the block has no coefficients and is not intra we skip applying + // the loop filter on block edges. + if (mbmi->skip && is_inter_block(mbmi)) return; + + // Here we are adding a mask for the transform size. The transform + // size mask is set to be correct for a 64x64 prediction block size. We + // mask to match the size of the block we are working on and then shift it + // into place.. + *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y_above]) + << shift_y; + *above_uv |= + (size_mask_uv[block_size] & above_64x64_txform_mask_uv[tx_size_uv_above]) + << shift_uv; + + *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y_left]) + << shift_y; + *left_uv |= + (size_mask_uv[block_size] & left_64x64_txform_mask_uv[tx_size_uv_left]) + << shift_uv; + + // Here we are trying to determine what to do with the internal 4x4 block + // boundaries. These differ from the 4x4 boundaries on the outside edge of + // an 8x8 in that the internal ones can be skipped and don't depend on + // the prediction block size. + if (tx_size_y == TX_4X4) + *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y; + + if (tx_size_uv == TX_4X4) + *int_4x4_uv |= (size_mask_uv[block_size] & 0xffff) << shift_uv; +} + +// This function does the same thing as the one above with the exception that +// it only affects the y masks. It exists because for blocks < 16x16 in size, +// we only update u and v masks on the first block. +static void build_y_mask(AV1_COMMON *const cm, + const loop_filter_info_n *const lfi_n, + const MODE_INFO *mi, const int shift_y, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif // CONFIG_SUPERTX + LOOP_FILTER_MASK *lfm) { + const MB_MODE_INFO *mbmi = &mi->mbmi; + const TX_SIZE tx_size_y = txsize_sqr_map[mbmi->tx_size]; + const TX_SIZE tx_size_y_left = txsize_horz_map[mbmi->tx_size]; + const TX_SIZE tx_size_y_above = txsize_vert_map[mbmi->tx_size]; +#if CONFIG_SUPERTX + const BLOCK_SIZE block_size = + supertx_enabled ? (BLOCK_SIZE)(3 * tx_size_y) : mbmi->sb_type; +#else + const BLOCK_SIZE block_size = mbmi->sb_type; +#endif +#if CONFIG_EXT_DELTA_Q + const int filter_level = get_filter_level(cm, lfi_n, mbmi); +#else + const int filter_level = get_filter_level(lfi_n, mbmi); + (void)cm; +#endif + uint64_t *const left_y = &lfm->left_y[tx_size_y_left]; + uint64_t *const above_y = &lfm->above_y[tx_size_y_above]; + uint64_t *const int_4x4_y = &lfm->int_4x4_y; + int i; + + if (!filter_level) { + return; + } else { + const int w = num_8x8_blocks_wide_lookup[block_size]; + const int h = num_8x8_blocks_high_lookup[block_size]; + const int row = (shift_y >> MAX_MIB_SIZE_LOG2); + const int col = shift_y - (row << MAX_MIB_SIZE_LOG2); + + for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w); + } + + *above_y |= above_prediction_mask[block_size] << shift_y; + *left_y |= left_prediction_mask[block_size] << shift_y; + + if (mbmi->skip && is_inter_block(mbmi)) return; + + *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y_above]) + << shift_y; + + *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y_left]) + << shift_y; + + if (tx_size_y == TX_4X4) + *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y; +} + +#if CONFIG_LOOPFILTERING_ACROSS_TILES +// This function update the bit masks for the entire 64x64 region represented +// by mi_row, mi_col. In case one of the edge is a tile boundary, loop filtering +// for that edge is disabled. This function only check the tile boundary info +// for the top left corner mi to determine the boundary information for the +// top and left edge of the whole super block +static void update_tile_boundary_filter_mask(AV1_COMMON *const cm, + const int mi_row, const int mi_col, + LOOP_FILTER_MASK *lfm) { + int i; + MODE_INFO *const mi = cm->mi + mi_row * cm->mi_stride + mi_col; + + if (mi->mbmi.boundary_info & TILE_LEFT_BOUNDARY) { + for (i = 0; i <= TX_32X32; i++) { + lfm->left_y[i] &= 0xfefefefefefefefeULL; + lfm->left_uv[i] &= 0xeeee; + } + } + + if (mi->mbmi.boundary_info & TILE_ABOVE_BOUNDARY) { + for (i = 0; i <= TX_32X32; i++) { + lfm->above_y[i] &= 0xffffffffffffff00ULL; + lfm->above_uv[i] &= 0xfff0; + } + } +} +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + +// This function sets up the bit masks for the entire 64x64 region represented +// by mi_row, mi_col. +// TODO(JBB): This function only works for yv12. +void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col, + MODE_INFO **mi, const int mode_info_stride, + LOOP_FILTER_MASK *lfm) { + int idx_32, idx_16, idx_8; + const loop_filter_info_n *const lfi_n = &cm->lf_info; + MODE_INFO **mip = mi; + MODE_INFO **mip2 = mi; + + // These are offsets to the next mi in the 64x64 block. It is what gets + // added to the mi ptr as we go through each loop. It helps us to avoid + // setting up special row and column counters for each index. The last step + // brings us out back to the starting position. + const int offset_32[] = { 4, (mode_info_stride << 2) - 4, 4, + -(mode_info_stride << 2) - 4 }; + const int offset_16[] = { 2, (mode_info_stride << 1) - 2, 2, + -(mode_info_stride << 1) - 2 }; + const int offset[] = { 1, mode_info_stride - 1, 1, -mode_info_stride - 1 }; + + // Following variables represent shifts to position the current block + // mask over the appropriate block. A shift of 36 to the left will move + // the bits for the final 32 by 32 block in the 64x64 up 4 rows and left + // 4 rows to the appropriate spot. + const int shift_32_y[] = { 0, 4, 32, 36 }; + const int shift_16_y[] = { 0, 2, 16, 18 }; + const int shift_8_y[] = { 0, 1, 8, 9 }; + const int shift_32_uv[] = { 0, 2, 8, 10 }; + const int shift_16_uv[] = { 0, 1, 4, 5 }; + int i; + const int max_rows = AOMMIN(cm->mi_rows - mi_row, MAX_MIB_SIZE); + const int max_cols = AOMMIN(cm->mi_cols - mi_col, MAX_MIB_SIZE); +#if CONFIG_EXT_PARTITION + assert(0 && "Not yet updated"); +#endif // CONFIG_EXT_PARTITION + + av1_zero(*lfm); + assert(mip[0] != NULL); + + // TODO(jimbankoski): Try moving most of the following code into decode + // loop and storing lfm in the mbmi structure so that we don't have to go + // through the recursive loop structure multiple times. + switch (mip[0]->mbmi.sb_type) { + case BLOCK_64X64: build_masks(cm, lfi_n, mip[0], 0, 0, lfm); break; + case BLOCK_64X32: build_masks(cm, lfi_n, mip[0], 0, 0, lfm); +#if CONFIG_SUPERTX && CONFIG_TX64X64 + if (supertx_enabled(&mip[0]->mbmi)) break; +#endif // CONFIG_SUPERTX && CONFIG_TX64X64 + mip2 = mip + mode_info_stride * 4; + if (4 >= max_rows) break; + build_masks(cm, lfi_n, mip2[0], 32, 8, lfm); + break; + case BLOCK_32X64: build_masks(cm, lfi_n, mip[0], 0, 0, lfm); +#if CONFIG_SUPERTX && CONFIG_TX64X64 + if (supertx_enabled(&mip[0]->mbmi)) break; +#endif // CONFIG_SUPERTX && CONFIG_TX64X64 + mip2 = mip + 4; + if (4 >= max_cols) break; + build_masks(cm, lfi_n, mip2[0], 4, 2, lfm); + break; + default: +#if CONFIG_SUPERTX && CONFIG_TX64X64 + if (mip[0]->mbmi.tx_size == TX_64X64) { + build_masks(cm, lfi_n, mip[0], 0, 0, lfm); + } else { +#endif // CONFIG_SUPERTX && CONFIG_TX64X64 + for (idx_32 = 0; idx_32 < 4; mip += offset_32[idx_32], ++idx_32) { + const int shift_y_32 = shift_32_y[idx_32]; + const int shift_uv_32 = shift_32_uv[idx_32]; + const int mi_32_col_offset = ((idx_32 & 1) << 2); + const int mi_32_row_offset = ((idx_32 >> 1) << 2); + if (mi_32_col_offset >= max_cols || mi_32_row_offset >= max_rows) + continue; + switch (mip[0]->mbmi.sb_type) { + case BLOCK_32X32: + build_masks(cm, lfi_n, mip[0], shift_y_32, shift_uv_32, lfm); + break; + case BLOCK_32X16: + build_masks(cm, lfi_n, mip[0], shift_y_32, shift_uv_32, lfm); +#if CONFIG_SUPERTX + if (supertx_enabled(&mip[0]->mbmi)) break; +#endif + if (mi_32_row_offset + 2 >= max_rows) continue; + mip2 = mip + mode_info_stride * 2; + build_masks(cm, lfi_n, mip2[0], shift_y_32 + 16, shift_uv_32 + 4, + lfm); + break; + case BLOCK_16X32: + build_masks(cm, lfi_n, mip[0], shift_y_32, shift_uv_32, lfm); +#if CONFIG_SUPERTX + if (supertx_enabled(&mip[0]->mbmi)) break; +#endif + if (mi_32_col_offset + 2 >= max_cols) continue; + mip2 = mip + 2; + build_masks(cm, lfi_n, mip2[0], shift_y_32 + 2, shift_uv_32 + 1, + lfm); + break; + default: +#if CONFIG_SUPERTX + if (mip[0]->mbmi.tx_size == TX_32X32) { + build_masks(cm, lfi_n, mip[0], shift_y_32, shift_uv_32, lfm); + break; + } +#endif + for (idx_16 = 0; idx_16 < 4; mip += offset_16[idx_16], ++idx_16) { + const int shift_y_32_16 = shift_y_32 + shift_16_y[idx_16]; + const int shift_uv_32_16 = shift_uv_32 + shift_16_uv[idx_16]; + const int mi_16_col_offset = + mi_32_col_offset + ((idx_16 & 1) << 1); + const int mi_16_row_offset = + mi_32_row_offset + ((idx_16 >> 1) << 1); + + if (mi_16_col_offset >= max_cols || + mi_16_row_offset >= max_rows) + continue; + + switch (mip[0]->mbmi.sb_type) { + case BLOCK_16X16: + build_masks(cm, lfi_n, mip[0], shift_y_32_16, + shift_uv_32_16, lfm); + break; + case BLOCK_16X8: +#if CONFIG_SUPERTX + if (supertx_enabled(&mip[0]->mbmi)) break; +#endif + build_masks(cm, lfi_n, mip[0], shift_y_32_16, + shift_uv_32_16, lfm); + if (mi_16_row_offset + 1 >= max_rows) continue; + mip2 = mip + mode_info_stride; + build_y_mask(cm, lfi_n, mip2[0], shift_y_32_16 + 8, +#if CONFIG_SUPERTX + 0, +#endif + lfm); + break; + case BLOCK_8X16: +#if CONFIG_SUPERTX + if (supertx_enabled(&mip[0]->mbmi)) break; +#endif + build_masks(cm, lfi_n, mip[0], shift_y_32_16, + shift_uv_32_16, lfm); + if (mi_16_col_offset + 1 >= max_cols) continue; + mip2 = mip + 1; + build_y_mask(cm, lfi_n, mip2[0], shift_y_32_16 + 1, +#if CONFIG_SUPERTX + 0, +#endif + lfm); + break; + default: { + const int shift_y_32_16_8_zero = + shift_y_32_16 + shift_8_y[0]; +#if CONFIG_SUPERTX + if (mip[0]->mbmi.tx_size == TX_16X16) { + build_masks(cm, lfi_n, mip[0], shift_y_32_16_8_zero, + shift_uv_32_16, lfm); + break; + } +#endif + build_masks(cm, lfi_n, mip[0], shift_y_32_16_8_zero, + shift_uv_32_16, lfm); + mip += offset[0]; + for (idx_8 = 1; idx_8 < 4; mip += offset[idx_8], ++idx_8) { + const int shift_y_32_16_8 = + shift_y_32_16 + shift_8_y[idx_8]; + const int mi_8_col_offset = + mi_16_col_offset + ((idx_8 & 1)); + const int mi_8_row_offset = + mi_16_row_offset + ((idx_8 >> 1)); + + if (mi_8_col_offset >= max_cols || + mi_8_row_offset >= max_rows) + continue; + build_y_mask(cm, lfi_n, mip[0], shift_y_32_16_8, +#if CONFIG_SUPERTX + supertx_enabled(&mip[0]->mbmi), +#endif + lfm); + } + break; + } + } + } + break; + } + } +#if CONFIG_SUPERTX && CONFIG_TX64X64 + } +#endif // CONFIG_SUPERTX && CONFIG_TX64X64 + break; + } + // The largest loopfilter we have is 16x16 so we use the 16x16 mask + // for 32x32 transforms also. + lfm->left_y[TX_16X16] |= lfm->left_y[TX_32X32]; + lfm->above_y[TX_16X16] |= lfm->above_y[TX_32X32]; + lfm->left_uv[TX_16X16] |= lfm->left_uv[TX_32X32]; + lfm->above_uv[TX_16X16] |= lfm->above_uv[TX_32X32]; + + // We do at least 8 tap filter on every 32x32 even if the transform size + // is 4x4. So if the 4x4 is set on a border pixel add it to the 8x8 and + // remove it from the 4x4. + lfm->left_y[TX_8X8] |= lfm->left_y[TX_4X4] & left_border; + lfm->left_y[TX_4X4] &= ~left_border; + lfm->above_y[TX_8X8] |= lfm->above_y[TX_4X4] & above_border; + lfm->above_y[TX_4X4] &= ~above_border; + lfm->left_uv[TX_8X8] |= lfm->left_uv[TX_4X4] & left_border_uv; + lfm->left_uv[TX_4X4] &= ~left_border_uv; + lfm->above_uv[TX_8X8] |= lfm->above_uv[TX_4X4] & above_border_uv; + lfm->above_uv[TX_4X4] &= ~above_border_uv; + + // We do some special edge handling. + if (mi_row + MAX_MIB_SIZE > cm->mi_rows) { + const uint64_t rows = cm->mi_rows - mi_row; + + // Each pixel inside the border gets a 1, + const uint64_t mask_y = (((uint64_t)1 << (rows << MAX_MIB_SIZE_LOG2)) - 1); + const uint16_t mask_uv = + (((uint16_t)1 << (((rows + 1) >> 1) << (MAX_MIB_SIZE_LOG2 - 1))) - 1); + + // Remove values completely outside our border. + for (i = 0; i < TX_32X32; i++) { + lfm->left_y[i] &= mask_y; + lfm->above_y[i] &= mask_y; + lfm->left_uv[i] &= mask_uv; + lfm->above_uv[i] &= mask_uv; + } + lfm->int_4x4_y &= mask_y; + lfm->above_int_4x4_uv = lfm->left_int_4x4_uv & mask_uv; + + // We don't apply a wide loop filter on the last uv block row. If set + // apply the shorter one instead. + if (rows == 1) { + lfm->above_uv[TX_8X8] |= lfm->above_uv[TX_16X16]; + lfm->above_uv[TX_16X16] = 0; + } + if (rows == 5) { + lfm->above_uv[TX_8X8] |= lfm->above_uv[TX_16X16] & 0xff00; + lfm->above_uv[TX_16X16] &= ~(lfm->above_uv[TX_16X16] & 0xff00); + } + } else { + lfm->above_int_4x4_uv = lfm->left_int_4x4_uv; + } + + if (mi_col + MAX_MIB_SIZE > cm->mi_cols) { + const uint64_t columns = cm->mi_cols - mi_col; + + // Each pixel inside the border gets a 1, the multiply copies the border + // to where we need it. + const uint64_t mask_y = (((1 << columns) - 1)) * 0x0101010101010101ULL; + const uint16_t mask_uv = ((1 << ((columns + 1) >> 1)) - 1) * 0x1111; + + // Internal edges are not applied on the last column of the image so + // we mask 1 more for the internal edges + const uint16_t mask_uv_int = ((1 << (columns >> 1)) - 1) * 0x1111; + + // Remove the bits outside the image edge. + for (i = 0; i < TX_32X32; i++) { + lfm->left_y[i] &= mask_y; + lfm->above_y[i] &= mask_y; + lfm->left_uv[i] &= mask_uv; + lfm->above_uv[i] &= mask_uv; + } + lfm->int_4x4_y &= mask_y; + lfm->left_int_4x4_uv &= mask_uv_int; + + // We don't apply a wide loop filter on the last uv column. If set + // apply the shorter one instead. + if (columns == 1) { + lfm->left_uv[TX_8X8] |= lfm->left_uv[TX_16X16]; + lfm->left_uv[TX_16X16] = 0; + } + if (columns == 5) { + lfm->left_uv[TX_8X8] |= (lfm->left_uv[TX_16X16] & 0xcccc); + lfm->left_uv[TX_16X16] &= ~(lfm->left_uv[TX_16X16] & 0xcccc); + } + } + // We don't apply a loop filter on the first column in the image, mask that + // out. + if (mi_col == 0) { + for (i = 0; i < TX_32X32; i++) { + lfm->left_y[i] &= 0xfefefefefefefefeULL; + lfm->left_uv[i] &= 0xeeee; + } + } + +#if CONFIG_LOOPFILTERING_ACROSS_TILES + if (av1_disable_loopfilter_on_tile_boundary(cm)) { + update_tile_boundary_filter_mask(cm, mi_row, mi_col, lfm); + } +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + + // Assert if we try to apply 2 different loop filters at the same position. + assert(!(lfm->left_y[TX_16X16] & lfm->left_y[TX_8X8])); + assert(!(lfm->left_y[TX_16X16] & lfm->left_y[TX_4X4])); + assert(!(lfm->left_y[TX_8X8] & lfm->left_y[TX_4X4])); + assert(!(lfm->int_4x4_y & lfm->left_y[TX_16X16])); + assert(!(lfm->left_uv[TX_16X16] & lfm->left_uv[TX_8X8])); + assert(!(lfm->left_uv[TX_16X16] & lfm->left_uv[TX_4X4])); + assert(!(lfm->left_uv[TX_8X8] & lfm->left_uv[TX_4X4])); + assert(!(lfm->left_int_4x4_uv & lfm->left_uv[TX_16X16])); + assert(!(lfm->above_y[TX_16X16] & lfm->above_y[TX_8X8])); + assert(!(lfm->above_y[TX_16X16] & lfm->above_y[TX_4X4])); + assert(!(lfm->above_y[TX_8X8] & lfm->above_y[TX_4X4])); + assert(!(lfm->int_4x4_y & lfm->above_y[TX_16X16])); + assert(!(lfm->above_uv[TX_16X16] & lfm->above_uv[TX_8X8])); + assert(!(lfm->above_uv[TX_16X16] & lfm->above_uv[TX_4X4])); + assert(!(lfm->above_uv[TX_8X8] & lfm->above_uv[TX_4X4])); + assert(!(lfm->above_int_4x4_uv & lfm->above_uv[TX_16X16])); +} + +static void filter_selectively_vert( + uint8_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8, + unsigned int mask_4x4, unsigned int mask_4x4_int, + const loop_filter_info_n *lfi_n, const uint8_t *lfl) { + unsigned int mask; + + for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask; + mask >>= 1) { + const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl; + + if (mask & 1) { + if (mask_16x16 & 1) { + aom_lpf_vertical_16(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr); + } else if (mask_8x8 & 1) { + aom_lpf_vertical_8(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr); + } else if (mask_4x4 & 1) { + aom_lpf_vertical_4(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr); + } + } + if (mask_4x4_int & 1) + aom_lpf_vertical_4(s + 4, pitch, lfi->mblim, lfi->lim, lfi->hev_thr); + s += 8; + lfl += 1; + mask_16x16 >>= 1; + mask_8x8 >>= 1; + mask_4x4 >>= 1; + mask_4x4_int >>= 1; + } +} + +#if CONFIG_HIGHBITDEPTH +static void highbd_filter_selectively_vert( + uint16_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8, + unsigned int mask_4x4, unsigned int mask_4x4_int, + const loop_filter_info_n *lfi_n, const uint8_t *lfl, int bd) { + unsigned int mask; + + for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask; + mask >>= 1) { + const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl; + + if (mask & 1) { + if (mask_16x16 & 1) { + aom_highbd_lpf_vertical_16(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr, + bd); + } else if (mask_8x8 & 1) { + aom_highbd_lpf_vertical_8(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr, + bd); + } else if (mask_4x4 & 1) { + aom_highbd_lpf_vertical_4(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr, + bd); + } + } + if (mask_4x4_int & 1) + aom_highbd_lpf_vertical_4(s + 4, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, bd); + s += 8; + lfl += 1; + mask_16x16 >>= 1; + mask_8x8 >>= 1; + mask_4x4 >>= 1; + mask_4x4_int >>= 1; + } +} +#endif // CONFIG_HIGHBITDEPTH + +typedef struct { + unsigned int m16x16; + unsigned int m8x8; + unsigned int m4x4; +} FilterMasks; + +// Get filter level and masks for the given row index 'idx_r'. (Only used for +// the non420 case). +// Note: 'row_masks_ptr' and/or 'col_masks_ptr' can be passed NULL. +static void get_filter_level_and_masks_non420( + AV1_COMMON *const cm, const struct macroblockd_plane *const plane, + MODE_INFO **mib, int mi_row, int mi_col, int idx_r, uint8_t *const lfl_r, + unsigned int *const mask_4x4_int_r, FilterMasks *const row_masks_ptr, + FilterMasks *const col_masks_ptr) { + const int ss_x = plane->subsampling_x; + const int ss_y = plane->subsampling_y; + const int col_step = mi_size_wide[BLOCK_8X8] << ss_x; + FilterMasks row_masks, col_masks; + memset(&row_masks, 0, sizeof(row_masks)); + memset(&col_masks, 0, sizeof(col_masks)); + *mask_4x4_int_r = 0; + const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8]; + + // Determine the vertical edges that need filtering + int idx_c; + for (idx_c = 0; idx_c < cm->mib_size && mi_col + idx_c < cm->mi_cols; + idx_c += col_step) { + const MODE_INFO *mi = mib[idx_r * cm->mi_stride + idx_c]; + const MB_MODE_INFO *mbmi = &mi[0].mbmi; + const BLOCK_SIZE sb_type = mbmi->sb_type; + const int skip_this = mbmi->skip && is_inter_block(mbmi); + // Map index to 8x8 unit + const int c = idx_c >> mi_width_log2_lookup[BLOCK_8X8]; + + const int blk_row = r & (num_8x8_blocks_high_lookup[sb_type] - 1); + const int blk_col = c & (num_8x8_blocks_wide_lookup[sb_type] - 1); + + // left edge of current unit is block/partition edge -> no skip + const int block_edge_left = + (num_4x4_blocks_wide_lookup[sb_type] > 1) ? !blk_col : 1; + const int skip_this_c = skip_this && !block_edge_left; + // top edge of current unit is block/partition edge -> no skip + const int block_edge_above = + (num_4x4_blocks_high_lookup[sb_type] > 1) ? !blk_row : 1; + const int skip_this_r = skip_this && !block_edge_above; + +#if CONFIG_VAR_TX + const TX_SIZE mb_tx_size = mbmi->inter_tx_size[blk_row][blk_col]; +#endif + + TX_SIZE tx_size = (plane->plane_type == PLANE_TYPE_UV) + ? get_uv_tx_size(mbmi, plane) + : mbmi->tx_size; + + const int skip_border_4x4_c = + ss_x && mi_col + idx_c >= cm->mi_cols - mi_size_wide[BLOCK_8X8]; + const int skip_border_4x4_r = + ss_y && mi_row + idx_r >= cm->mi_rows - mi_size_high[BLOCK_8X8]; + + TX_SIZE tx_size_c = txsize_horz_map[tx_size]; + TX_SIZE tx_size_r = txsize_vert_map[tx_size]; + + int tx_size_mask = 0; + const int c_step = (c >> ss_x); + const int r_step = (r >> ss_y); + const int col_mask = 1 << c_step; + +#if CONFIG_VAR_TX + if (is_inter_block(mbmi) && !mbmi->skip) { + tx_size = (plane->plane_type == PLANE_TYPE_UV) + ? uv_txsize_lookup[sb_type][mb_tx_size][ss_x][ss_y] + : mb_tx_size; + } +#endif + +// Filter level can vary per MI +#if CONFIG_EXT_DELTA_Q + if (!(lfl_r[c_step] = get_filter_level(cm, &cm->lf_info, mbmi))) continue; +#else + if (!(lfl_r[c_step] = get_filter_level(&cm->lf_info, mbmi))) continue; +#endif + +#if CONFIG_VAR_TX + tx_size_r = AOMMIN(tx_size, cm->above_txfm_context[mi_col + c]); + tx_size_c = + AOMMIN(tx_size, cm->left_txfm_context[(mi_row + r) & MAX_MIB_MASK]); + + cm->above_txfm_context[mi_col + c] = tx_size; + cm->left_txfm_context[(mi_row + r) & MAX_MIB_MASK] = tx_size; +#endif // CONFIG_VAR_TX + + if (tx_size_c == TX_32X32) + tx_size_mask = 3; + else if (tx_size_c == TX_16X16) + tx_size_mask = 1; + else + tx_size_mask = 0; + + // Build masks based on the transform size of each block + // handle vertical mask + if (tx_size_c == TX_32X32) { + if (!skip_this_c && (c_step & tx_size_mask) == 0) { + if (!skip_border_4x4_c) + col_masks.m16x16 |= col_mask; + else + col_masks.m8x8 |= col_mask; + } + } else if (tx_size_c == TX_16X16) { + if (!skip_this_c && (c_step & tx_size_mask) == 0) { + if (!skip_border_4x4_c) + col_masks.m16x16 |= col_mask; + else + col_masks.m8x8 |= col_mask; + } + } else { + // force 8x8 filtering on 32x32 boundaries + if (!skip_this_c && (c_step & tx_size_mask) == 0) { + if (tx_size_c == TX_8X8 || ((c >> ss_x) & 3) == 0) + col_masks.m8x8 |= col_mask; + else + col_masks.m4x4 |= col_mask; + } + + if (!skip_this && tx_size_c < TX_8X8 && !skip_border_4x4_c && + (c_step & tx_size_mask) == 0) + *mask_4x4_int_r |= col_mask; + } + + if (tx_size_r == TX_32X32) + tx_size_mask = 3; + else if (tx_size_r == TX_16X16) + tx_size_mask = 1; + else + tx_size_mask = 0; + + // set horizontal mask + if (tx_size_r == TX_32X32) { + if (!skip_this_r && (r_step & tx_size_mask) == 0) { + if (!skip_border_4x4_r) + row_masks.m16x16 |= col_mask; + else + row_masks.m8x8 |= col_mask; + } + } else if (tx_size_r == TX_16X16) { + if (!skip_this_r && (r_step & tx_size_mask) == 0) { + if (!skip_border_4x4_r) + row_masks.m16x16 |= col_mask; + else + row_masks.m8x8 |= col_mask; + } + } else { + // force 8x8 filtering on 32x32 boundaries + if (!skip_this_r && (r_step & tx_size_mask) == 0) { + if (tx_size_r == TX_8X8 || (r_step & 3) == 0) + row_masks.m8x8 |= col_mask; + else + row_masks.m4x4 |= col_mask; + } + + if (!skip_this && tx_size_r < TX_8X8 && !skip_border_4x4_c && + ((r >> ss_y) & tx_size_mask) == 0) + *mask_4x4_int_r |= col_mask; + } + } + + if (row_masks_ptr) *row_masks_ptr = row_masks; + if (col_masks_ptr) *col_masks_ptr = col_masks; +} + +void av1_filter_block_plane_non420_ver(AV1_COMMON *const cm, + struct macroblockd_plane *plane, + MODE_INFO **mib, int mi_row, + int mi_col) { + const int ss_y = plane->subsampling_y; + const int row_step = mi_size_high[BLOCK_8X8] << ss_y; + struct buf_2d *const dst = &plane->dst; + uint8_t *const dst0 = dst->buf; + uint8_t lfl[MAX_MIB_SIZE][MAX_MIB_SIZE] = { { 0 } }; + + int idx_r; + for (idx_r = 0; idx_r < cm->mib_size && mi_row + idx_r < cm->mi_rows; + idx_r += row_step) { + unsigned int mask_4x4_int; + FilterMasks col_masks; + const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8]; + get_filter_level_and_masks_non420(cm, plane, mib, mi_row, mi_col, idx_r, + &lfl[r][0], &mask_4x4_int, NULL, + &col_masks); + + // Disable filtering on the leftmost column or tile boundary + unsigned int border_mask = ~(mi_col == 0); +#if CONFIG_LOOPFILTERING_ACROSS_TILES + if (av1_disable_loopfilter_on_tile_boundary(cm) && + ((mib[0]->mbmi.boundary_info & TILE_LEFT_BOUNDARY) != 0)) { + border_mask = 0xfffffffe; + } +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + highbd_filter_selectively_vert( + CONVERT_TO_SHORTPTR(dst->buf), dst->stride, + col_masks.m16x16 & border_mask, col_masks.m8x8 & border_mask, + col_masks.m4x4 & border_mask, mask_4x4_int, &cm->lf_info, &lfl[r][0], + (int)cm->bit_depth); + else +#endif // CONFIG_HIGHBITDEPTH + filter_selectively_vert( + dst->buf, dst->stride, col_masks.m16x16 & border_mask, + col_masks.m8x8 & border_mask, col_masks.m4x4 & border_mask, + mask_4x4_int, &cm->lf_info, &lfl[r][0]); + dst->buf += 8 * dst->stride; + } + + // Now do horizontal pass + dst->buf = dst0; +} + +void av1_filter_block_plane_non420_hor(AV1_COMMON *const cm, + struct macroblockd_plane *plane, + MODE_INFO **mib, int mi_row, + int mi_col) { + const int ss_y = plane->subsampling_y; + const int row_step = mi_size_high[BLOCK_8X8] << ss_y; + struct buf_2d *const dst = &plane->dst; + uint8_t *const dst0 = dst->buf; + FilterMasks row_masks_array[MAX_MIB_SIZE]; + unsigned int mask_4x4_int[MAX_MIB_SIZE] = { 0 }; + uint8_t lfl[MAX_MIB_SIZE][MAX_MIB_SIZE] = { { 0 } }; + int idx_r; + for (idx_r = 0; idx_r < cm->mib_size && mi_row + idx_r < cm->mi_rows; + idx_r += row_step) { + const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8]; + get_filter_level_and_masks_non420(cm, plane, mib, mi_row, mi_col, idx_r, + &lfl[r][0], mask_4x4_int + r, + row_masks_array + r, NULL); + } + for (idx_r = 0; idx_r < cm->mib_size && mi_row + idx_r < cm->mi_rows; + idx_r += row_step) { + const int skip_border_4x4_r = + ss_y && mi_row + idx_r >= cm->mi_rows - mi_size_wide[BLOCK_8X8]; + const int r = idx_r >> mi_width_log2_lookup[BLOCK_8X8]; + const unsigned int mask_4x4_int_r = skip_border_4x4_r ? 0 : mask_4x4_int[r]; + FilterMasks row_masks; + +#if CONFIG_LOOPFILTERING_ACROSS_TILES + // Disable filtering on the abovemost row or tile boundary + const MODE_INFO *mi = cm->mi + (mi_row + r) * cm->mi_stride; + if ((av1_disable_loopfilter_on_tile_boundary(cm) && + (mi->mbmi.boundary_info & TILE_ABOVE_BOUNDARY)) || + (mi_row + idx_r == 0)) { + memset(&row_masks, 0, sizeof(row_masks)); +#else + if (mi_row + idx_r == 0) { + memset(&row_masks, 0, sizeof(row_masks)); +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + } else { + memcpy(&row_masks, row_masks_array + r, sizeof(row_masks)); + } +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + highbd_filter_selectively_horiz( + CONVERT_TO_SHORTPTR(dst->buf), dst->stride, row_masks.m16x16, + row_masks.m8x8, row_masks.m4x4, mask_4x4_int_r, &cm->lf_info, + &lfl[r][0], (int)cm->bit_depth); + else +#endif // CONFIG_HIGHBITDEPTH + filter_selectively_horiz(dst->buf, dst->stride, row_masks.m16x16, + row_masks.m8x8, row_masks.m4x4, mask_4x4_int_r, + &cm->lf_info, &lfl[r][0]); + dst->buf += 8 * dst->stride; + } + dst->buf = dst0; +} + +void av1_filter_block_plane_ss00_ver(AV1_COMMON *const cm, + struct macroblockd_plane *const plane, + int mi_row, LOOP_FILTER_MASK *lfm) { + struct buf_2d *const dst = &plane->dst; + uint8_t *const dst0 = dst->buf; + int r; + uint64_t mask_16x16 = lfm->left_y[TX_16X16]; + uint64_t mask_8x8 = lfm->left_y[TX_8X8]; + uint64_t mask_4x4 = lfm->left_y[TX_4X4]; + uint64_t mask_4x4_int = lfm->int_4x4_y; + + assert(plane->subsampling_x == 0 && plane->subsampling_y == 0); + + // Vertical pass: do 2 rows at one time + for (r = 0; r < cm->mib_size && mi_row + r < cm->mi_rows; r += 2) { + unsigned int mask_16x16_l = mask_16x16 & 0xffff; + unsigned int mask_8x8_l = mask_8x8 & 0xffff; + unsigned int mask_4x4_l = mask_4x4 & 0xffff; + unsigned int mask_4x4_int_l = mask_4x4_int & 0xffff; + +// Disable filtering on the leftmost column. +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + highbd_filter_selectively_vert_row2( + plane->subsampling_x, CONVERT_TO_SHORTPTR(dst->buf), dst->stride, + mask_16x16_l, mask_8x8_l, mask_4x4_l, mask_4x4_int_l, &cm->lf_info, + &lfm->lfl_y[r][0], (int)cm->bit_depth); + else +#endif // CONFIG_HIGHBITDEPTH + filter_selectively_vert_row2( + plane->subsampling_x, dst->buf, dst->stride, mask_16x16_l, mask_8x8_l, + mask_4x4_l, mask_4x4_int_l, &cm->lf_info, &lfm->lfl_y[r][0]); + + dst->buf += 2 * MI_SIZE * dst->stride; + mask_16x16 >>= 2 * MI_SIZE; + mask_8x8 >>= 2 * MI_SIZE; + mask_4x4 >>= 2 * MI_SIZE; + mask_4x4_int >>= 2 * MI_SIZE; + } + + // Horizontal pass + dst->buf = dst0; +} + +void av1_filter_block_plane_ss00_hor(AV1_COMMON *const cm, + struct macroblockd_plane *const plane, + int mi_row, LOOP_FILTER_MASK *lfm) { + struct buf_2d *const dst = &plane->dst; + uint8_t *const dst0 = dst->buf; + int r; + uint64_t mask_16x16 = lfm->above_y[TX_16X16]; + uint64_t mask_8x8 = lfm->above_y[TX_8X8]; + uint64_t mask_4x4 = lfm->above_y[TX_4X4]; + uint64_t mask_4x4_int = lfm->int_4x4_y; + + assert(plane->subsampling_x == 0 && plane->subsampling_y == 0); + + for (r = 0; r < cm->mib_size && mi_row + r < cm->mi_rows; r++) { + unsigned int mask_16x16_r; + unsigned int mask_8x8_r; + unsigned int mask_4x4_r; + + if (mi_row + r == 0) { + mask_16x16_r = 0; + mask_8x8_r = 0; + mask_4x4_r = 0; + } else { + mask_16x16_r = mask_16x16 & 0xff; + mask_8x8_r = mask_8x8 & 0xff; + mask_4x4_r = mask_4x4 & 0xff; + } + +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + highbd_filter_selectively_horiz( + CONVERT_TO_SHORTPTR(dst->buf), dst->stride, mask_16x16_r, mask_8x8_r, + mask_4x4_r, mask_4x4_int & 0xff, &cm->lf_info, &lfm->lfl_y[r][0], + (int)cm->bit_depth); + else +#endif // CONFIG_HIGHBITDEPTH + filter_selectively_horiz(dst->buf, dst->stride, mask_16x16_r, mask_8x8_r, + mask_4x4_r, mask_4x4_int & 0xff, &cm->lf_info, + &lfm->lfl_y[r][0]); + + dst->buf += MI_SIZE * dst->stride; + mask_16x16 >>= MI_SIZE; + mask_8x8 >>= MI_SIZE; + mask_4x4 >>= MI_SIZE; + mask_4x4_int >>= MI_SIZE; + } + // restore the buf pointer in case there is additional filter pass. + dst->buf = dst0; +} + +void av1_filter_block_plane_ss11_ver(AV1_COMMON *const cm, + struct macroblockd_plane *const plane, + int mi_row, LOOP_FILTER_MASK *lfm) { + struct buf_2d *const dst = &plane->dst; + uint8_t *const dst0 = dst->buf; + int r, c; + + uint16_t mask_16x16 = lfm->left_uv[TX_16X16]; + uint16_t mask_8x8 = lfm->left_uv[TX_8X8]; + uint16_t mask_4x4 = lfm->left_uv[TX_4X4]; + uint16_t mask_4x4_int = lfm->left_int_4x4_uv; + + assert(plane->subsampling_x == 1 && plane->subsampling_y == 1); + assert(plane->plane_type == PLANE_TYPE_UV); + memset(lfm->lfl_uv, 0, sizeof(lfm->lfl_uv)); + + // Vertical pass: do 2 rows at one time + for (r = 0; r < cm->mib_size && mi_row + r < cm->mi_rows; r += 4) { + for (c = 0; c < (cm->mib_size >> 1); c++) { + lfm->lfl_uv[r >> 1][c] = lfm->lfl_y[r][c << 1]; + lfm->lfl_uv[(r + 2) >> 1][c] = lfm->lfl_y[r + 2][c << 1]; + } + + { + unsigned int mask_16x16_l = mask_16x16 & 0xff; + unsigned int mask_8x8_l = mask_8x8 & 0xff; + unsigned int mask_4x4_l = mask_4x4 & 0xff; + unsigned int mask_4x4_int_l = mask_4x4_int & 0xff; + +// Disable filtering on the leftmost column. +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + highbd_filter_selectively_vert_row2( + plane->subsampling_x, CONVERT_TO_SHORTPTR(dst->buf), dst->stride, + mask_16x16_l, mask_8x8_l, mask_4x4_l, mask_4x4_int_l, &cm->lf_info, + &lfm->lfl_uv[r >> 1][0], (int)cm->bit_depth); + else +#endif // CONFIG_HIGHBITDEPTH + filter_selectively_vert_row2(plane->subsampling_x, dst->buf, + dst->stride, mask_16x16_l, mask_8x8_l, + mask_4x4_l, mask_4x4_int_l, &cm->lf_info, + &lfm->lfl_uv[r >> 1][0]); + + dst->buf += 2 * MI_SIZE * dst->stride; + mask_16x16 >>= MI_SIZE; + mask_8x8 >>= MI_SIZE; + mask_4x4 >>= MI_SIZE; + mask_4x4_int >>= MI_SIZE; + } + } + + // Horizontal pass + dst->buf = dst0; +} + +void av1_filter_block_plane_ss11_hor(AV1_COMMON *const cm, + struct macroblockd_plane *const plane, + int mi_row, LOOP_FILTER_MASK *lfm) { + struct buf_2d *const dst = &plane->dst; + uint8_t *const dst0 = dst->buf; + int r, c; + uint64_t mask_16x16 = lfm->above_uv[TX_16X16]; + uint64_t mask_8x8 = lfm->above_uv[TX_8X8]; + uint64_t mask_4x4 = lfm->above_uv[TX_4X4]; + uint64_t mask_4x4_int = lfm->above_int_4x4_uv; + + assert(plane->subsampling_x == 1 && plane->subsampling_y == 1); + memset(lfm->lfl_uv, 0, sizeof(lfm->lfl_uv)); + + // re-porpulate the filter level for uv, same as the code for vertical + // filter in av1_filter_block_plane_ss11_ver + for (r = 0; r < cm->mib_size && mi_row + r < cm->mi_rows; r += 4) { + for (c = 0; c < (cm->mib_size >> 1); c++) { + lfm->lfl_uv[r >> 1][c] = lfm->lfl_y[r][c << 1]; + lfm->lfl_uv[(r + 2) >> 1][c] = lfm->lfl_y[r + 2][c << 1]; + } + } + + for (r = 0; r < cm->mib_size && mi_row + r < cm->mi_rows; r += 2) { + const int skip_border_4x4_r = mi_row + r == cm->mi_rows - 1; + const unsigned int mask_4x4_int_r = + skip_border_4x4_r ? 0 : (mask_4x4_int & 0xf); + unsigned int mask_16x16_r; + unsigned int mask_8x8_r; + unsigned int mask_4x4_r; + + if (mi_row + r == 0) { + mask_16x16_r = 0; + mask_8x8_r = 0; + mask_4x4_r = 0; + } else { + mask_16x16_r = mask_16x16 & 0xf; + mask_8x8_r = mask_8x8 & 0xf; + mask_4x4_r = mask_4x4 & 0xf; + } + +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + highbd_filter_selectively_horiz( + CONVERT_TO_SHORTPTR(dst->buf), dst->stride, mask_16x16_r, mask_8x8_r, + mask_4x4_r, mask_4x4_int_r, &cm->lf_info, &lfm->lfl_uv[r >> 1][0], + (int)cm->bit_depth); + else +#endif // CONFIG_HIGHBITDEPTH + filter_selectively_horiz(dst->buf, dst->stride, mask_16x16_r, mask_8x8_r, + mask_4x4_r, mask_4x4_int_r, &cm->lf_info, + &lfm->lfl_uv[r >> 1][0]); + + dst->buf += MI_SIZE * dst->stride; + mask_16x16 >>= MI_SIZE / 2; + mask_8x8 >>= MI_SIZE / 2; + mask_4x4 >>= MI_SIZE / 2; + mask_4x4_int >>= MI_SIZE / 2; + } + // restore the buf pointer in case there is additional filter pass. + dst->buf = dst0; +} + +#if !(CONFIG_VAR_TX || CONFIG_EXT_PARTITION || CONFIG_EXT_PARTITION_TYPES || \ + CONFIG_CB4X4) +#if CONFIG_PARALLEL_DEBLOCKING +typedef enum EDGE_DIR { VERT_EDGE = 0, HORZ_EDGE = 1, NUM_EDGE_DIRS } EDGE_DIR; +static const uint32_t av1_prediction_masks[NUM_EDGE_DIRS][BLOCK_SIZES] = { + // mask for vertical edges filtering + { +#if CONFIG_CB4X4 + 2 - 1, // BLOCK_2X2 + 2 - 1, // BLOCK_2X4 + 4 - 1, // BLOCK_4X2 +#endif // CONFIG_CB4X4 + 4 - 1, // BLOCK_4X4 + 4 - 1, // BLOCK_4X8 + 8 - 1, // BLOCK_8X4 + 8 - 1, // BLOCK_8X8 + 8 - 1, // BLOCK_8X16 + 16 - 1, // BLOCK_16X8 + 16 - 1, // BLOCK_16X16 + 16 - 1, // BLOCK_16X32 + 32 - 1, // BLOCK_32X16 + 32 - 1, // BLOCK_32X32 + 32 - 1, // BLOCK_32X64 + 64 - 1, // BLOCK_64X32 + 64 - 1, // BLOCK_64X64 +#if CONFIG_EXT_PARTITION + 64 - 1, // BLOCK_64X128 + 128 - 1, // BLOCK_128X64 + 128 - 1 // BLOCK_128X128 +#endif // CONFIG_EXT_PARTITION + }, + // mask for horizontal edges filtering + { +#if CONFIG_CB4X4 + 2 - 1, // BLOCK_2X2 + 4 - 1, // BLOCK_2X4 + 2 - 1, // BLOCK_4X2 +#endif // CONFIG_CB4X4 + 4 - 1, // BLOCK_4X4 + 8 - 1, // BLOCK_4X8 + 4 - 1, // BLOCK_8X4 + 8 - 1, // BLOCK_8X8 + 16 - 1, // BLOCK_8X16 + 8 - 1, // BLOCK_16X8 + 16 - 1, // BLOCK_16X16 + 32 - 1, // BLOCK_16X32 + 16 - 1, // BLOCK_32X16 + 32 - 1, // BLOCK_32X32 + 64 - 1, // BLOCK_32X64 + 32 - 1, // BLOCK_64X32 + 64 - 1, // BLOCK_64X64 +#if CONFIG_EXT_PARTITION + 128 - 1, // BLOCK_64X128 + 64 - 1, // BLOCK_128X64 + 128 - 1 // BLOCK_128X128 +#endif // CONFIG_EXT_PARTITION + }, +}; + +static const uint32_t av1_transform_masks[NUM_EDGE_DIRS][TX_SIZES_ALL] = { + { +#if CONFIG_CB4X4 + 2 - 1, // TX_2X2 +#endif + 4 - 1, // TX_4X4 + 8 - 1, // TX_8X8 + 16 - 1, // TX_16X16 + 32 - 1, // TX_32X32 +#if CONFIG_TX64X64 + 64 - 1, // TX_64X64 +#endif // CONFIG_TX64X64 + 4 - 1, // TX_4X8 + 8 - 1, // TX_8X4 + 8 - 1, // TX_8X16 + 16 - 1, // TX_16X8 + 16 - 1, // TX_16X32 + 32 - 1, // TX_32X16 + 4 - 1, // TX_4X16 + 16 - 1, // TX_16X4 + 8 - 1, // TX_8X32 + 32 - 1 // TX_32X8 + }, + { +#if CONFIG_CB4X4 + 2 - 1, // TX_2X2 +#endif + 4 - 1, // TX_4X4 + 8 - 1, // TX_8X8 + 16 - 1, // TX_16X16 + 32 - 1, // TX_32X32 +#if CONFIG_TX64X64 + 64 - 1, // TX_64X64 +#endif // CONFIG_TX64X64 + 8 - 1, // TX_4X8 + 4 - 1, // TX_8X4 + 16 - 1, // TX_8X16 + 8 - 1, // TX_16X8 + 32 - 1, // TX_16X32 + 16 - 1, // TX_32X16 + 16 - 1, // TX_4X16 + 4 - 1, // TX_16X4 + 32 - 1, // TX_8X32 + 8 - 1 // TX_32X8 + } +}; + +static TX_SIZE av1_get_transform_size(const MODE_INFO *const pCurr, + const EDGE_DIR edgeDir, + const uint32_t scaleHorz, + const uint32_t scaleVert) { + const BLOCK_SIZE bs = pCurr->mbmi.sb_type; + TX_SIZE txSize; + // since in case of chrominance or non-square transorm need to convert + // transform size into transform size in particular direction. + txSize = uv_txsize_lookup[bs][pCurr->mbmi.tx_size][scaleHorz][scaleVert]; + if (VERT_EDGE == edgeDir) { + txSize = txsize_horz_map[txSize]; + } else { + txSize = txsize_vert_map[txSize]; + } + return txSize; +} + +typedef struct AV1_DEBLOCKING_PARAMETERS { + // length of the filter applied to the outer edge + uint32_t filterLength; + // length of the filter applied to the inner edge + uint32_t filterLengthInternal; + // deblocking limits + const uint8_t *lim; + const uint8_t *mblim; + const uint8_t *hev_thr; +} AV1_DEBLOCKING_PARAMETERS; + +static void set_lpf_parameters(AV1_DEBLOCKING_PARAMETERS *const pParams, + const MODE_INFO **const ppCurr, + const ptrdiff_t modeStep, + const AV1_COMMON *const cm, + const EDGE_DIR edgeDir, const uint32_t x, + const uint32_t y, const uint32_t width, + const uint32_t height, const uint32_t scaleHorz, + const uint32_t scaleVert) { + // reset to initial values + pParams->filterLength = 0; + pParams->filterLengthInternal = 0; + // no deblocking is required + if ((width <= x) || (height <= y)) { + return; + } +#if CONFIG_EXT_PARTITION + // not sure if changes are required. + assert(0 && "Not yet updated"); +#endif // CONFIG_EXT_PARTITION + { + const TX_SIZE ts = + av1_get_transform_size(ppCurr[0], edgeDir, scaleHorz, scaleVert); + const uint32_t currLevel = get_filter_level(&cm->lf_info, &ppCurr[0]->mbmi); + const int currSkipped = + ppCurr[0]->mbmi.skip && is_inter_block(&ppCurr[0]->mbmi); + const uint32_t coord = (VERT_EDGE == edgeDir) ? (x) : (y); + uint32_t level = currLevel; + // prepare outer edge parameters. deblock the edge if it's an edge of a TU + if (coord) { +#if CONFIG_LOOPFILTERING_ACROSS_TILES + if (!av1_disable_loopfilter_on_tile_boundary(cm) || + ((VERT_EDGE == edgeDir) && + (0 == (ppCurr[0]->mbmi.boundary_info & TILE_LEFT_BOUNDARY))) || + ((HORZ_EDGE == edgeDir) && + (0 == (ppCurr[0]->mbmi.boundary_info & TILE_ABOVE_BOUNDARY)))) +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + { + const int32_t tuEdge = + (coord & av1_transform_masks[edgeDir][ts]) ? (0) : (1); + if (tuEdge) { + const MODE_INFO *const pPrev = *(ppCurr - modeStep); + const TX_SIZE pvTs = + av1_get_transform_size(pPrev, edgeDir, scaleHorz, scaleVert); + const uint32_t pvLvl = get_filter_level(&cm->lf_info, &pPrev->mbmi); + const int pvSkip = pPrev->mbmi.skip && is_inter_block(&pPrev->mbmi); + const int32_t puEdge = + (coord & + av1_prediction_masks[edgeDir] + [ss_size_lookup[ppCurr[0]->mbmi.sb_type] + [scaleHorz][scaleVert]]) + ? (0) + : (1); + // if the current and the previous blocks are skipped, + // deblock the edge if the edge belongs to a PU's edge only. + if ((currLevel || pvLvl) && (!pvSkip || !currSkipped || puEdge)) { +#if CONFIG_PARALLEL_DEBLOCKING_15TAP || CONFIG_PARALLEL_DEBLOCKING_15TAPLUMAONLY + const TX_SIZE minTs = AOMMIN(ts, pvTs); + if (TX_4X4 >= minTs) { + pParams->filterLength = 4; + } else if (TX_8X8 == minTs) { + pParams->filterLength = 8; + } else { + pParams->filterLength = 16; +#if CONFIG_PARALLEL_DEBLOCKING_15TAPLUMAONLY + // No wide filtering for chroma plane + if (scaleHorz || scaleVert) { + pParams->filterLength = 8; + } +#endif + } +#else + pParams->filterLength = (TX_4X4 >= AOMMIN(ts, pvTs)) ? (4) : (8); + +#endif // CONFIG_PARALLEL_DEBLOCKING_15TAP + + // update the level if the current block is skipped, + // but the previous one is not + level = (currLevel) ? (currLevel) : (pvLvl); + } + } + } + // prepare internal edge parameters + if (currLevel && !currSkipped) { + pParams->filterLengthInternal = (TX_4X4 >= ts) ? (4) : (0); + } + // prepare common parameters + if (pParams->filterLength || pParams->filterLengthInternal) { + const loop_filter_thresh *const limits = cm->lf_info.lfthr + level; + pParams->lim = limits->lim; + pParams->mblim = limits->mblim; + pParams->hev_thr = limits->hev_thr; + } + } + } +} + +static void av1_filter_block_plane_vert(const AV1_COMMON *const cm, + const MACROBLOCKD_PLANE *const pPlane, + const MODE_INFO **ppModeInfo, + const ptrdiff_t modeStride, + const uint32_t cuX, + const uint32_t cuY) { + const uint32_t scaleHorz = pPlane->subsampling_x; + const uint32_t scaleVert = pPlane->subsampling_y; + const uint32_t width = pPlane->dst.width; + const uint32_t height = pPlane->dst.height; + uint8_t *const pDst = pPlane->dst.buf; + const int dstStride = pPlane->dst.stride; + for (int y = 0; y < (MAX_MIB_SIZE >> scaleVert); y += 1) { + uint8_t *p = pDst + y * MI_SIZE * dstStride; + for (int x = 0; x < (MAX_MIB_SIZE >> scaleHorz); x += 1) { + const MODE_INFO **const pCurr = + ppModeInfo + (y << scaleVert) * modeStride + (x << scaleHorz); + AV1_DEBLOCKING_PARAMETERS params; + memset(¶ms, 0, sizeof(params)); + set_lpf_parameters(¶ms, pCurr, ((ptrdiff_t)1 << scaleHorz), cm, + VERT_EDGE, cuX + x * MI_SIZE, cuY + y * MI_SIZE, width, + height, scaleHorz, scaleVert); + switch (params.filterLength) { + // apply 4-tap filtering + case 4: + aom_lpf_vertical_4(p, dstStride, params.mblim, params.lim, + params.hev_thr); + break; + // apply 8-tap filtering + case 8: + aom_lpf_vertical_8(p, dstStride, params.mblim, params.lim, + params.hev_thr); + break; +#if CONFIG_PARALLEL_DEBLOCKING_15TAP || CONFIG_PARALLEL_DEBLOCKING_15TAPLUMAONLY + // apply 16-tap filtering + case 16: + aom_lpf_vertical_16(p, dstStride, params.mblim, params.lim, + params.hev_thr); + break; +#endif // CONFIG_PARALLEL_DEBLOCKING_15TAP + // no filtering + default: break; + } + // process the internal edge + if (params.filterLengthInternal) { + aom_lpf_vertical_4(p + 4, dstStride, params.mblim, params.lim, + params.hev_thr); + } + // advance the destination pointer + p += 8; + } + } +} + +static void av1_filter_block_plane_horz(const AV1_COMMON *const cm, + const MACROBLOCKD_PLANE *const pPlane, + const MODE_INFO **ppModeInfo, + const ptrdiff_t modeStride, + const uint32_t cuX, + const uint32_t cuY) { + const uint32_t scaleHorz = pPlane->subsampling_x; + const uint32_t scaleVert = pPlane->subsampling_y; + const uint32_t width = pPlane->dst.width; + const uint32_t height = pPlane->dst.height; + uint8_t *const pDst = pPlane->dst.buf; + const int dstStride = pPlane->dst.stride; + for (int y = 0; y < (MAX_MIB_SIZE >> scaleVert); y += 1) { + uint8_t *p = pDst + y * MI_SIZE * dstStride; + for (int x = 0; x < (MAX_MIB_SIZE >> scaleHorz); x += 1) { + const MODE_INFO **const pCurr = + ppModeInfo + (y << scaleVert) * modeStride + (x << scaleHorz); + AV1_DEBLOCKING_PARAMETERS params; + memset(¶ms, 0, sizeof(params)); + set_lpf_parameters(¶ms, pCurr, (modeStride << scaleVert), cm, + HORZ_EDGE, cuX + x * MI_SIZE, cuY + y * MI_SIZE, width, + height, scaleHorz, scaleVert); + switch (params.filterLength) { + // apply 4-tap filtering + case 4: + aom_lpf_horizontal_4(p, dstStride, params.mblim, params.lim, + params.hev_thr); + break; + // apply 8-tap filtering + case 8: + aom_lpf_horizontal_8(p, dstStride, params.mblim, params.lim, + params.hev_thr); + break; +#if CONFIG_PARALLEL_DEBLOCKING_15TAP || CONFIG_PARALLEL_DEBLOCKING_15TAPLUMAONLY + // apply 16-tap filtering + case 16: + aom_lpf_horizontal_edge_16(p, dstStride, params.mblim, params.lim, + params.hev_thr); + break; +#endif // CONFIG_PARALLEL_DEBLOCKING_15TAP + // no filtering + default: break; + } + // process the internal edge + if (params.filterLengthInternal) { + aom_lpf_horizontal_4(p + 4 * dstStride, dstStride, params.mblim, + params.lim, params.hev_thr); + } + // advance the destination pointer + p += 8; + } + } +} +#endif // CONFIG_PARALLEL_DEBLOCKING +#endif + +void av1_loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, AV1_COMMON *cm, + struct macroblockd_plane planes[MAX_MB_PLANE], + int start, int stop, int y_only) { +#if CONFIG_VAR_TX || CONFIG_EXT_PARTITION || CONFIG_EXT_PARTITION_TYPES || \ + CONFIG_CB4X4 + const int num_planes = y_only ? 1 : MAX_MB_PLANE; + int mi_row, mi_col; + +#if CONFIG_VAR_TX + memset(cm->above_txfm_context, TX_SIZES, cm->mi_cols); +#endif // CONFIG_VAR_TX + for (mi_row = start; mi_row < stop; mi_row += cm->mib_size) { + MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride; +#if CONFIG_VAR_TX + memset(cm->left_txfm_context, TX_SIZES, MAX_MIB_SIZE); +#endif // CONFIG_VAR_TX + for (mi_col = 0; mi_col < cm->mi_cols; mi_col += cm->mib_size) { + int plane; + + av1_setup_dst_planes(planes, cm->sb_size, frame_buffer, mi_row, mi_col); + + for (plane = 0; plane < num_planes; ++plane) { + av1_filter_block_plane_non420_ver(cm, &planes[plane], mi + mi_col, + mi_row, mi_col); + av1_filter_block_plane_non420_hor(cm, &planes[plane], mi + mi_col, + mi_row, mi_col); + } + } + } +#else // CONFIG_VAR_TX || CONFIG_EXT_PARTITION || CONFIG_EXT_PARTITION_TYPES + const int num_planes = y_only ? 1 : MAX_MB_PLANE; + int mi_row, mi_col; +#if !CONFIG_PARALLEL_DEBLOCKING + enum lf_path path; + LOOP_FILTER_MASK lfm; + + if (y_only) + path = LF_PATH_444; + else if (planes[1].subsampling_y == 1 && planes[1].subsampling_x == 1) + path = LF_PATH_420; + else if (planes[1].subsampling_y == 0 && planes[1].subsampling_x == 0) + path = LF_PATH_444; + else + path = LF_PATH_SLOW; +#endif +#if CONFIG_PARALLEL_DEBLOCKING + for (mi_row = start; mi_row < stop; mi_row += MAX_MIB_SIZE) { + MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride; + for (mi_col = 0; mi_col < cm->mi_cols; mi_col += MAX_MIB_SIZE) { + av1_setup_dst_planes(planes, cm->sb_size, frame_buffer, mi_row, mi_col); + for (int planeIdx = 0; planeIdx < num_planes; planeIdx += 1) { + const int32_t scaleHorz = planes[planeIdx].subsampling_x; + const int32_t scaleVert = planes[planeIdx].subsampling_y; + av1_filter_block_plane_vert( + cm, planes + planeIdx, (const MODE_INFO **)(mi + mi_col), + cm->mi_stride, (mi_col * MI_SIZE) >> scaleHorz, + (mi_row * MI_SIZE) >> scaleVert); + } + } + } + for (mi_row = start; mi_row < stop; mi_row += MAX_MIB_SIZE) { + MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride; + for (mi_col = 0; mi_col < cm->mi_cols; mi_col += MAX_MIB_SIZE) { + av1_setup_dst_planes(planes, cm->sb_size, frame_buffer, mi_row, mi_col); + for (int planeIdx = 0; planeIdx < num_planes; planeIdx += 1) { + const int32_t scaleHorz = planes[planeIdx].subsampling_x; + const int32_t scaleVert = planes[planeIdx].subsampling_y; + av1_filter_block_plane_horz( + cm, planes + planeIdx, (const MODE_INFO **)(mi + mi_col), + cm->mi_stride, (mi_col * MI_SIZE) >> scaleHorz, + (mi_row * MI_SIZE) >> scaleVert); + } + } + } +#else // CONFIG_PARALLEL_DEBLOCKING + for (mi_row = start; mi_row < stop; mi_row += MAX_MIB_SIZE) { + MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride; + for (mi_col = 0; mi_col < cm->mi_cols; mi_col += MAX_MIB_SIZE) { + int plane; + + av1_setup_dst_planes(planes, cm->sb_size, frame_buffer, mi_row, mi_col); + + // TODO(JBB): Make setup_mask work for non 420. + av1_setup_mask(cm, mi_row, mi_col, mi + mi_col, cm->mi_stride, &lfm); + + av1_filter_block_plane_ss00_ver(cm, &planes[0], mi_row, &lfm); + av1_filter_block_plane_ss00_hor(cm, &planes[0], mi_row, &lfm); + for (plane = 1; plane < num_planes; ++plane) { + switch (path) { + case LF_PATH_420: + av1_filter_block_plane_ss11_ver(cm, &planes[plane], mi_row, &lfm); + av1_filter_block_plane_ss11_hor(cm, &planes[plane], mi_row, &lfm); + break; + case LF_PATH_444: + av1_filter_block_plane_ss00_ver(cm, &planes[plane], mi_row, &lfm); + av1_filter_block_plane_ss00_hor(cm, &planes[plane], mi_row, &lfm); + break; + case LF_PATH_SLOW: + av1_filter_block_plane_non420_ver(cm, &planes[plane], mi + mi_col, + mi_row, mi_col); + av1_filter_block_plane_non420_hor(cm, &planes[plane], mi + mi_col, + mi_row, mi_col); + + break; + } + } + } + } +#endif // CONFIG_PARALLEL_DEBLOCKING +#endif // CONFIG_VAR_TX || CONFIG_EXT_PARTITION || CONFIG_EXT_PARTITION_TYPES +} + +void av1_loop_filter_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, + MACROBLOCKD *xd, int frame_filter_level, int y_only, + int partial_frame) { + int start_mi_row, end_mi_row, mi_rows_to_filter; +#if CONFIG_EXT_DELTA_Q + int orig_filter_level = cm->lf.filter_level; +#endif + if (!frame_filter_level) return; + start_mi_row = 0; + mi_rows_to_filter = cm->mi_rows; + if (partial_frame && cm->mi_rows > 8) { + start_mi_row = cm->mi_rows >> 1; + start_mi_row &= 0xfffffff8; + mi_rows_to_filter = AOMMAX(cm->mi_rows / 8, 8); + } + end_mi_row = start_mi_row + mi_rows_to_filter; + av1_loop_filter_frame_init(cm, frame_filter_level); +#if CONFIG_EXT_DELTA_Q + cm->lf.filter_level = frame_filter_level; +#endif + av1_loop_filter_rows(frame, cm, xd->plane, start_mi_row, end_mi_row, y_only); +#if CONFIG_EXT_DELTA_Q + cm->lf.filter_level = orig_filter_level; +#endif +} + +void av1_loop_filter_data_reset( + LFWorkerData *lf_data, YV12_BUFFER_CONFIG *frame_buffer, + struct AV1Common *cm, const struct macroblockd_plane planes[MAX_MB_PLANE]) { + lf_data->frame_buffer = frame_buffer; + lf_data->cm = cm; + lf_data->start = 0; + lf_data->stop = 0; + lf_data->y_only = 0; + memcpy(lf_data->planes, planes, sizeof(lf_data->planes)); +} + +int av1_loop_filter_worker(LFWorkerData *const lf_data, void *unused) { + (void)unused; + av1_loop_filter_rows(lf_data->frame_buffer, lf_data->cm, lf_data->planes, + lf_data->start, lf_data->stop, lf_data->y_only); + return 1; +} diff --git a/third_party/aom/av1/common/av1_loopfilter.h b/third_party/aom/av1/common/av1_loopfilter.h new file mode 100644 index 0000000000..8ac5d99e64 --- /dev/null +++ b/third_party/aom/av1/common/av1_loopfilter.h @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_LOOPFILTER_H_ +#define AV1_COMMON_LOOPFILTER_H_ + +#include "aom_ports/mem.h" +#include "./aom_config.h" + +#include "av1/common/blockd.h" +#include "av1/common/seg_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MAX_LOOP_FILTER 63 +#define MAX_SHARPNESS 7 + +#define SIMD_WIDTH 16 + +#define MAX_MODE_LF_DELTAS 2 + +enum lf_path { + LF_PATH_420, + LF_PATH_444, + LF_PATH_SLOW, +}; + +struct loopfilter { + int filter_level; + + int sharpness_level; + int last_sharpness_level; + + uint8_t mode_ref_delta_enabled; + uint8_t mode_ref_delta_update; + + // 0 = Intra, Last, Last2+Last3(CONFIG_EXT_REFS), + // GF, BRF(CONFIG_EXT_REFS), ARF + signed char ref_deltas[TOTAL_REFS_PER_FRAME]; + signed char last_ref_deltas[TOTAL_REFS_PER_FRAME]; + + // 0 = ZERO_MV, MV + signed char mode_deltas[MAX_MODE_LF_DELTAS]; + signed char last_mode_deltas[MAX_MODE_LF_DELTAS]; +}; + +// Need to align this structure so when it is declared and +// passed it can be loaded into vector registers. +typedef struct { + DECLARE_ALIGNED(SIMD_WIDTH, uint8_t, mblim[SIMD_WIDTH]); + DECLARE_ALIGNED(SIMD_WIDTH, uint8_t, lim[SIMD_WIDTH]); + DECLARE_ALIGNED(SIMD_WIDTH, uint8_t, hev_thr[SIMD_WIDTH]); +} loop_filter_thresh; + +typedef struct { + loop_filter_thresh lfthr[MAX_LOOP_FILTER + 1]; + uint8_t lvl[MAX_SEGMENTS][TOTAL_REFS_PER_FRAME][MAX_MODE_LF_DELTAS]; +} loop_filter_info_n; + +// This structure holds bit masks for all 8x8 blocks in a 64x64 region. +// Each 1 bit represents a position in which we want to apply the loop filter. +// Left_ entries refer to whether we apply a filter on the border to the +// left of the block. Above_ entries refer to whether or not to apply a +// filter on the above border. Int_ entries refer to whether or not to +// apply borders on the 4x4 edges within the 8x8 block that each bit +// represents. +// Since each transform is accompanied by a potentially different type of +// loop filter there is a different entry in the array for each transform size. +typedef struct { + uint64_t left_y[TX_SIZES]; + uint64_t above_y[TX_SIZES]; + uint64_t int_4x4_y; + uint16_t left_uv[TX_SIZES]; + uint16_t above_uv[TX_SIZES]; + uint16_t left_int_4x4_uv; + uint16_t above_int_4x4_uv; + uint8_t lfl_y[MAX_MIB_SIZE][MAX_MIB_SIZE]; + uint8_t lfl_uv[MAX_MIB_SIZE / 2][MAX_MIB_SIZE / 2]; +} LOOP_FILTER_MASK; + +/* assorted loopfilter functions which get used elsewhere */ +struct AV1Common; +struct macroblockd; +struct AV1LfSyncData; + +// This function sets up the bit masks for the entire 64x64 region represented +// by mi_row, mi_col. +void av1_setup_mask(struct AV1Common *const cm, const int mi_row, + const int mi_col, MODE_INFO **mi_8x8, + const int mode_info_stride, LOOP_FILTER_MASK *lfm); + +void av1_filter_block_plane_ss00_ver(struct AV1Common *const cm, + struct macroblockd_plane *const plane, + int mi_row, LOOP_FILTER_MASK *lfm); +void av1_filter_block_plane_ss00_hor(struct AV1Common *const cm, + struct macroblockd_plane *const plane, + int mi_row, LOOP_FILTER_MASK *lfm); +void av1_filter_block_plane_ss11_ver(struct AV1Common *const cm, + struct macroblockd_plane *const plane, + int mi_row, LOOP_FILTER_MASK *lfm); +void av1_filter_block_plane_ss11_hor(struct AV1Common *const cm, + struct macroblockd_plane *const plane, + int mi_row, LOOP_FILTER_MASK *lfm); + +void av1_filter_block_plane_non420_ver(struct AV1Common *const cm, + struct macroblockd_plane *plane, + MODE_INFO **mi_8x8, int mi_row, + int mi_col); +void av1_filter_block_plane_non420_hor(struct AV1Common *const cm, + struct macroblockd_plane *plane, + MODE_INFO **mi_8x8, int mi_row, + int mi_col); + +void av1_loop_filter_init(struct AV1Common *cm); + +// Update the loop filter for the current frame. +// This should be called before av1_loop_filter_rows(), +// av1_loop_filter_frame() +// calls this function directly. +void av1_loop_filter_frame_init(struct AV1Common *cm, int default_filt_lvl); + +void av1_loop_filter_frame(YV12_BUFFER_CONFIG *frame, struct AV1Common *cm, + struct macroblockd *mbd, int filter_level, + int y_only, int partial_frame); + +// Apply the loop filter to [start, stop) macro block rows in frame_buffer. +void av1_loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, + struct AV1Common *cm, + struct macroblockd_plane planes[MAX_MB_PLANE], + int start, int stop, int y_only); + +typedef struct LoopFilterWorkerData { + YV12_BUFFER_CONFIG *frame_buffer; + struct AV1Common *cm; + struct macroblockd_plane planes[MAX_MB_PLANE]; + + int start; + int stop; + int y_only; +} LFWorkerData; + +void av1_loop_filter_data_reset( + LFWorkerData *lf_data, YV12_BUFFER_CONFIG *frame_buffer, + struct AV1Common *cm, const struct macroblockd_plane planes[MAX_MB_PLANE]); + +// Operates on the rows described by 'lf_data'. +int av1_loop_filter_worker(LFWorkerData *const lf_data, void *unused); +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_LOOPFILTER_H_ diff --git a/third_party/aom/av1/common/av1_rtcd.c b/third_party/aom/av1/common/av1_rtcd.c new file mode 100644 index 0000000000..f9ccd19795 --- /dev/null +++ b/third_party/aom/av1/common/av1_rtcd.c @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include "./aom_config.h" +#define RTCD_C +#include "./av1_rtcd.h" +#include "aom_ports/aom_once.h" + +void av1_rtcd() { + // TODO(JBB): Remove this once, by insuring that both the encoder and + // decoder setup functions are protected by once(); + once(setup_rtcd_internal); +} diff --git a/third_party/aom/av1/common/av1_rtcd_defs.pl b/third_party/aom/av1/common/av1_rtcd_defs.pl new file mode 100755 index 0000000000..1dca10c526 --- /dev/null +++ b/third_party/aom/av1/common/av1_rtcd_defs.pl @@ -0,0 +1,644 @@ +sub av1_common_forward_decls() { +print < +#include +#include + +#include "av1/common/enums.h" +#include "aom/aom_integer.h" +#include "aom_dsp/aom_dsp_common.h" + +static const int cos_bit_min = 10; +static const int cos_bit_max = 16; + +// cospi_arr[i][j] = (int)round(cos(M_PI*j/128) * (1<<(cos_bit_min+i))); +static const int32_t cospi_arr[7][64] = { + { 1024, 1024, 1023, 1021, 1019, 1016, 1013, 1009, 1004, 999, 993, 987, 980, + 972, 964, 955, 946, 936, 926, 915, 903, 891, 878, 865, 851, 837, + 822, 807, 792, 775, 759, 742, 724, 706, 688, 669, 650, 630, 610, + 590, 569, 548, 526, 505, 483, 460, 438, 415, 392, 369, 345, 321, + 297, 273, 249, 224, 200, 175, 150, 125, 100, 75, 50, 25 }, + { 2048, 2047, 2046, 2042, 2038, 2033, 2026, 2018, 2009, 1998, 1987, + 1974, 1960, 1945, 1928, 1911, 1892, 1872, 1851, 1829, 1806, 1782, + 1757, 1730, 1703, 1674, 1645, 1615, 1583, 1551, 1517, 1483, 1448, + 1412, 1375, 1338, 1299, 1260, 1220, 1179, 1138, 1096, 1053, 1009, + 965, 921, 876, 830, 784, 737, 690, 642, 595, 546, 498, + 449, 400, 350, 301, 251, 201, 151, 100, 50 }, + { 4096, 4095, 4091, 4085, 4076, 4065, 4052, 4036, 4017, 3996, 3973, + 3948, 3920, 3889, 3857, 3822, 3784, 3745, 3703, 3659, 3612, 3564, + 3513, 3461, 3406, 3349, 3290, 3229, 3166, 3102, 3035, 2967, 2896, + 2824, 2751, 2675, 2598, 2520, 2440, 2359, 2276, 2191, 2106, 2019, + 1931, 1842, 1751, 1660, 1567, 1474, 1380, 1285, 1189, 1092, 995, + 897, 799, 700, 601, 501, 401, 301, 201, 101 }, + { 8192, 8190, 8182, 8170, 8153, 8130, 8103, 8071, 8035, 7993, 7946, + 7895, 7839, 7779, 7713, 7643, 7568, 7489, 7405, 7317, 7225, 7128, + 7027, 6921, 6811, 6698, 6580, 6458, 6333, 6203, 6070, 5933, 5793, + 5649, 5501, 5351, 5197, 5040, 4880, 4717, 4551, 4383, 4212, 4038, + 3862, 3683, 3503, 3320, 3135, 2948, 2760, 2570, 2378, 2185, 1990, + 1795, 1598, 1401, 1202, 1003, 803, 603, 402, 201 }, + { 16384, 16379, 16364, 16340, 16305, 16261, 16207, 16143, 16069, 15986, 15893, + 15791, 15679, 15557, 15426, 15286, 15137, 14978, 14811, 14635, 14449, 14256, + 14053, 13842, 13623, 13395, 13160, 12916, 12665, 12406, 12140, 11866, 11585, + 11297, 11003, 10702, 10394, 10080, 9760, 9434, 9102, 8765, 8423, 8076, + 7723, 7366, 7005, 6639, 6270, 5897, 5520, 5139, 4756, 4370, 3981, + 3590, 3196, 2801, 2404, 2006, 1606, 1205, 804, 402 }, + { 32768, 32758, 32729, 32679, 32610, 32522, 32413, 32286, 32138, 31972, 31786, + 31581, 31357, 31114, 30853, 30572, 30274, 29957, 29622, 29269, 28899, 28511, + 28106, 27684, 27246, 26791, 26320, 25833, 25330, 24812, 24279, 23732, 23170, + 22595, 22006, 21403, 20788, 20160, 19520, 18868, 18205, 17531, 16846, 16151, + 15447, 14733, 14010, 13279, 12540, 11793, 11039, 10279, 9512, 8740, 7962, + 7180, 6393, 5602, 4808, 4011, 3212, 2411, 1608, 804 }, + { 65536, 65516, 65457, 65358, 65220, 65043, 64827, 64571, 64277, 63944, 63572, + 63162, 62714, 62228, 61705, 61145, 60547, 59914, 59244, 58538, 57798, 57022, + 56212, 55368, 54491, 53581, 52639, 51665, 50660, 49624, 48559, 47464, 46341, + 45190, 44011, 42806, 41576, 40320, 39040, 37736, 36410, 35062, 33692, 32303, + 30893, 29466, 28020, 26558, 25080, 23586, 22078, 20557, 19024, 17479, 15924, + 14359, 12785, 11204, 9616, 8022, 6424, 4821, 3216, 1608 } +}; + +static INLINE int32_t round_shift(int32_t value, int bit) { + assert(bit >= 1); + return (value + (1 << (bit - 1))) >> bit; +} + +static INLINE void round_shift_array(int32_t *arr, int size, int bit) { + int i; + if (bit == 0) { + return; + } else { + if (bit > 0) { + for (i = 0; i < size; i++) { + arr[i] = round_shift(arr[i], bit); + } + } else { + for (i = 0; i < size; i++) { + arr[i] = arr[i] * (1 << (-bit)); + } + } + } +} + +static INLINE int32_t half_btf(int32_t w0, int32_t in0, int32_t w1, int32_t in1, + int bit) { + int32_t result_32 = w0 * in0 + w1 * in1; +#if CONFIG_COEFFICIENT_RANGE_CHECKING + int64_t result_64 = (int64_t)w0 * (int64_t)in0 + (int64_t)w1 * (int64_t)in1; + if (result_64 < INT32_MIN || result_64 > INT32_MAX) { + printf("%s %d overflow result_32: %d result_64: %" PRId64 + " w0: %d in0: %d w1: %d in1: " + "%d\n", + __FILE__, __LINE__, result_32, result_64, w0, in0, w1, in1); + assert(0 && "half_btf overflow"); + } +#endif + return round_shift(result_32, bit); +} + +static INLINE int get_max_bit(int x) { + int max_bit = -1; + while (x) { + x = x >> 1; + max_bit++; + } + return max_bit; +} + +// TODO(angiebird): implement SSE +static INLINE void clamp_block(int16_t *block, int block_size, int stride, + int low, int high) { + int i, j; + for (i = 0; i < block_size; ++i) { + for (j = 0; j < block_size; ++j) { + block[i * stride + j] = clamp(block[i * stride + j], low, high); + } + } +} + +typedef void (*TxfmFunc)(const int32_t *input, int32_t *output, + const int8_t *cos_bit, const int8_t *stage_range); + +typedef enum TXFM_TYPE { + TXFM_TYPE_DCT4, + TXFM_TYPE_DCT8, + TXFM_TYPE_DCT16, + TXFM_TYPE_DCT32, + TXFM_TYPE_DCT64, + TXFM_TYPE_ADST4, + TXFM_TYPE_ADST8, + TXFM_TYPE_ADST16, + TXFM_TYPE_ADST32, +} TXFM_TYPE; + +typedef struct TXFM_2D_CFG { + const int txfm_size; + const int stage_num_col; + const int stage_num_row; + + const int8_t *shift; + const int8_t *stage_range_col; + const int8_t *stage_range_row; + const int8_t *cos_bit_col; + const int8_t *cos_bit_row; + const TXFM_TYPE txfm_type_col; + const TXFM_TYPE txfm_type_row; +} TXFM_2D_CFG; + +typedef struct TXFM_2D_FLIP_CFG { + int ud_flip; // flip upside down + int lr_flip; // flip left to right + const TXFM_2D_CFG *cfg; +} TXFM_2D_FLIP_CFG; + +static INLINE void set_flip_cfg(int tx_type, TXFM_2D_FLIP_CFG *cfg) { + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + cfg->ud_flip = 0; + cfg->lr_flip = 0; + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + cfg->ud_flip = 1; + cfg->lr_flip = 0; + break; + case DCT_FLIPADST: + cfg->ud_flip = 0; + cfg->lr_flip = 1; + break; + case FLIPADST_FLIPADST: + cfg->ud_flip = 1; + cfg->lr_flip = 1; + break; + case ADST_FLIPADST: + cfg->ud_flip = 0; + cfg->lr_flip = 1; + break; + case FLIPADST_ADST: + cfg->ud_flip = 1; + cfg->lr_flip = 0; + break; +#endif // CONFIG_EXT_TX + default: + cfg->ud_flip = 0; + cfg->lr_flip = 0; + assert(0); + } +} + +#ifdef __cplusplus +extern "C" { +#endif +TXFM_2D_FLIP_CFG av1_get_fwd_txfm_cfg(int tx_type, int tx_size); +TXFM_2D_FLIP_CFG av1_get_fwd_txfm_64x64_cfg(int tx_type); +#ifdef __cplusplus +} +#endif // __cplusplus + +#endif // AV1_TXFM_H_ diff --git a/third_party/aom/av1/common/blockd.c b/third_party/aom/av1/common/blockd.c new file mode 100644 index 0000000000..4eb6f01eaf --- /dev/null +++ b/third_party/aom/av1/common/blockd.c @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom_ports/system_state.h" + +#include "av1/common/blockd.h" +#include "av1/common/onyxc_int.h" + +PREDICTION_MODE av1_left_block_mode(const MODE_INFO *cur_mi, + const MODE_INFO *left_mi, int b) { + if (b == 0 || b == 2) { + if (!left_mi || is_inter_block(&left_mi->mbmi)) return DC_PRED; + + return get_y_mode(left_mi, b + 1); + } else { + assert(b == 1 || b == 3); + return cur_mi->bmi[b - 1].as_mode; + } +} + +PREDICTION_MODE av1_above_block_mode(const MODE_INFO *cur_mi, + const MODE_INFO *above_mi, int b) { + if (b == 0 || b == 1) { + if (!above_mi || is_inter_block(&above_mi->mbmi)) return DC_PRED; + + return get_y_mode(above_mi, b + 2); + } else { + assert(b == 2 || b == 3); + return cur_mi->bmi[b - 2].as_mode; + } +} + +#if CONFIG_COEF_INTERLEAVE +void av1_foreach_transformed_block_interleave( + const MACROBLOCKD *const xd, BLOCK_SIZE bsize, + foreach_transformed_block_visitor visit, void *arg) { + const struct macroblockd_plane *const pd_y = &xd->plane[0]; + const struct macroblockd_plane *const pd_c = &xd->plane[1]; + const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + + const TX_SIZE tx_log2_y = mbmi->tx_size; + const TX_SIZE tx_log2_c = get_uv_tx_size(mbmi, pd_c); + const int tx_sz_y = (1 << tx_log2_y); + const int tx_sz_c = (1 << tx_log2_c); + + const BLOCK_SIZE plane_bsize_y = get_plane_block_size(bsize, pd_y); + const BLOCK_SIZE plane_bsize_c = get_plane_block_size(bsize, pd_c); + + const int num_4x4_w_y = num_4x4_blocks_wide_lookup[plane_bsize_y]; + const int num_4x4_w_c = num_4x4_blocks_wide_lookup[plane_bsize_c]; + const int num_4x4_h_y = num_4x4_blocks_high_lookup[plane_bsize_y]; + const int num_4x4_h_c = num_4x4_blocks_high_lookup[plane_bsize_c]; + + const int step_y = 1 << (tx_log2_y << 1); + const int step_c = 1 << (tx_log2_c << 1); + + const int max_4x4_w_y = + get_max_4x4_size(num_4x4_w_y, xd->mb_to_right_edge, pd_y->subsampling_x); + const int max_4x4_h_y = + get_max_4x4_size(num_4x4_h_y, xd->mb_to_bottom_edge, pd_y->subsampling_y); + + const int extra_step_y = ((num_4x4_w_y - max_4x4_w_y) >> tx_log2_y) * step_y; + + const int max_4x4_w_c = + get_max_4x4_size(num_4x4_w_c, xd->mb_to_right_edge, pd_c->subsampling_x); + const int max_4x4_h_c = + get_max_4x4_size(num_4x4_h_c, xd->mb_to_bottom_edge, pd_c->subsampling_y); + + const int extra_step_c = ((num_4x4_w_c - max_4x4_w_c) >> tx_log2_c) * step_c; + + // The max_4x4_w/h may be smaller than tx_sz under some corner cases, + // i.e. when the SB is splitted by tile boundaries. + const int tu_num_w_y = (max_4x4_w_y + tx_sz_y - 1) / tx_sz_y; + const int tu_num_h_y = (max_4x4_h_y + tx_sz_y - 1) / tx_sz_y; + const int tu_num_w_c = (max_4x4_w_c + tx_sz_c - 1) / tx_sz_c; + const int tu_num_h_c = (max_4x4_h_c + tx_sz_c - 1) / tx_sz_c; + const int tu_num_c = tu_num_w_c * tu_num_h_c; + + int tu_idx_c = 0; + int offset_y, row_y, col_y; + int offset_c, row_c, col_c; + + for (row_y = 0; row_y < tu_num_h_y; row_y++) { + for (col_y = 0; col_y < tu_num_w_y; col_y++) { + // luma + offset_y = (row_y * tu_num_w_y + col_y) * step_y + row_y * extra_step_y; + visit(0, offset_y, row_y * tx_sz_y, col_y * tx_sz_y, plane_bsize_y, + tx_log2_y, arg); + // chroma + if (tu_idx_c < tu_num_c) { + row_c = (tu_idx_c / tu_num_w_c) * tx_sz_c; + col_c = (tu_idx_c % tu_num_w_c) * tx_sz_c; + offset_c = tu_idx_c * step_c + (tu_idx_c / tu_num_w_c) * extra_step_c; + visit(1, offset_c, row_c, col_c, plane_bsize_c, tx_log2_c, arg); + visit(2, offset_c, row_c, col_c, plane_bsize_c, tx_log2_c, arg); + tu_idx_c++; + } + } + } + + // In 422 case, it's possible that Chroma has more TUs than Luma + while (tu_idx_c < tu_num_c) { + row_c = (tu_idx_c / tu_num_w_c) * tx_sz_c; + col_c = (tu_idx_c % tu_num_w_c) * tx_sz_c; + offset_c = tu_idx_c * step_c + row_c * extra_step_c; + visit(1, offset_c, row_c, col_c, plane_bsize_c, tx_log2_c, arg); + visit(2, offset_c, row_c, col_c, plane_bsize_c, tx_log2_c, arg); + tu_idx_c++; + } +} +#endif + +void av1_foreach_transformed_block_in_plane( + const MACROBLOCKD *const xd, BLOCK_SIZE bsize, int plane, + foreach_transformed_block_visitor visit, void *arg) { + const struct macroblockd_plane *const pd = &xd->plane[plane]; + // block and transform sizes, in number of 4x4 blocks log 2 ("*_b") + // 4x4=0, 8x8=2, 16x16=4, 32x32=6, 64x64=8 + // transform size varies per plane, look it up in a common way. + const TX_SIZE tx_size = get_tx_size(plane, xd); +#if CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + const BLOCK_SIZE plane_bsize = + AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd)); +#else + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); +#endif + const uint8_t txw_unit = tx_size_wide_unit[tx_size]; + const uint8_t txh_unit = tx_size_high_unit[tx_size]; + const int step = txw_unit * txh_unit; + int i = 0, r, c; + + // If mb_to_right_edge is < 0 we are in a situation in which + // the current block size extends into the UMV and we won't + // visit the sub blocks that are wholly within the UMV. + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + const int max_blocks_high = max_block_high(xd, plane_bsize, plane); + + // Keep track of the row and column of the blocks we use so that we know + // if we are in the unrestricted motion border. + for (r = 0; r < max_blocks_high; r += txh_unit) { + // Skip visiting the sub blocks that are wholly within the UMV. + for (c = 0; c < max_blocks_wide; c += txw_unit) { + visit(plane, i, r, c, plane_bsize, tx_size, arg); + i += step; + } + } +} + +#if CONFIG_LV_MAP +void av1_foreach_transformed_block(const MACROBLOCKD *const xd, + BLOCK_SIZE bsize, int mi_row, int mi_col, + foreach_transformed_block_visitor visit, + void *arg) { + int plane; + + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { +#if CONFIG_CB4X4 + if (!is_chroma_reference(mi_row, mi_col, bsize, + xd->plane[plane].subsampling_x, + xd->plane[plane].subsampling_y)) + continue; +#else + (void)mi_row; + (void)mi_col; +#endif + av1_foreach_transformed_block_in_plane(xd, bsize, plane, visit, arg); + } +} +#endif + +#if CONFIG_DAALA_DIST +void av1_foreach_8x8_transformed_block_in_plane( + const MACROBLOCKD *const xd, BLOCK_SIZE bsize, int plane, + foreach_transformed_block_visitor visit, + foreach_transformed_block_visitor mi_visit, void *arg) { + const struct macroblockd_plane *const pd = &xd->plane[plane]; + // block and transform sizes, in number of 4x4 blocks log 2 ("*_b") + // 4x4=0, 8x8=2, 16x16=4, 32x32=6, 64x64=8 + // transform size varies per plane, look it up in a common way. + const TX_SIZE tx_size = get_tx_size(plane, xd); + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); + const uint8_t txw_unit = tx_size_wide_unit[tx_size]; + const uint8_t txh_unit = tx_size_high_unit[tx_size]; + const int step = txw_unit * txh_unit; + int i = 0, r, c; + + // If mb_to_right_edge is < 0 we are in a situation in which + // the current block size extends into the UMV and we won't + // visit the sub blocks that are wholly within the UMV. + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + const int max_blocks_high = max_block_high(xd, plane_bsize, plane); + + // Keep track of the row and column of the blocks we use so that we know + // if we are in the unrestricted motion border. + for (r = 0; r < max_blocks_high; r += txh_unit) { + // Skip visiting the sub blocks that are wholly within the UMV. + for (c = 0; c < max_blocks_wide; c += txw_unit) { + visit(plane, i, r, c, plane_bsize, tx_size, arg); + // Call whenever each 8x8 block is done + if ((r & 1) && (c & 1)) + mi_visit(plane, i, r - 1, c - 1, plane_bsize, TX_8X8, arg); + i += step; + } + } +} +#endif + +#if !CONFIG_PVQ || CONFIG_VAR_TX +void av1_set_contexts(const MACROBLOCKD *xd, struct macroblockd_plane *pd, + int plane, TX_SIZE tx_size, int has_eob, int aoff, + int loff) { + ENTROPY_CONTEXT *const a = pd->above_context + aoff; + ENTROPY_CONTEXT *const l = pd->left_context + loff; + const int txs_wide = tx_size_wide_unit[tx_size]; + const int txs_high = tx_size_high_unit[tx_size]; +#if CONFIG_CB4X4 + const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; +#else + const BLOCK_SIZE bsize = AOMMAX(xd->mi[0]->mbmi.sb_type, BLOCK_8X8); +#endif + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); + + // above + if (has_eob && xd->mb_to_right_edge < 0) { + int i; + const int blocks_wide = max_block_wide(xd, plane_bsize, plane); + int above_contexts = txs_wide; + if (above_contexts + aoff > blocks_wide) + above_contexts = blocks_wide - aoff; + + for (i = 0; i < above_contexts; ++i) a[i] = has_eob; + for (i = above_contexts; i < txs_wide; ++i) a[i] = 0; + } else { + memset(a, has_eob, sizeof(ENTROPY_CONTEXT) * txs_wide); + } + + // left + if (has_eob && xd->mb_to_bottom_edge < 0) { + int i; + const int blocks_high = max_block_high(xd, plane_bsize, plane); + int left_contexts = txs_high; + if (left_contexts + loff > blocks_high) left_contexts = blocks_high - loff; + + for (i = 0; i < left_contexts; ++i) l[i] = has_eob; + for (i = left_contexts; i < txs_high; ++i) l[i] = 0; + } else { + memset(l, has_eob, sizeof(ENTROPY_CONTEXT) * txs_high); + } +} +#endif + +void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y) { + int i; + + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].plane_type = get_plane_type(i); + xd->plane[i].subsampling_x = i ? ss_x : 0; + xd->plane[i].subsampling_y = i ? ss_y : 0; + } +} + +#if CONFIG_EXT_INTRA +const int16_t dr_intra_derivative[90] = { + 1, 14666, 7330, 4884, 3660, 2926, 2435, 2084, 1821, 1616, 1451, 1317, 1204, + 1108, 1026, 955, 892, 837, 787, 743, 703, 666, 633, 603, 574, 548, + 524, 502, 481, 461, 443, 426, 409, 394, 379, 365, 352, 339, 327, + 316, 305, 294, 284, 274, 265, 256, 247, 238, 230, 222, 214, 207, + 200, 192, 185, 179, 172, 166, 159, 153, 147, 141, 136, 130, 124, + 119, 113, 108, 103, 98, 93, 88, 83, 78, 73, 68, 63, 59, + 54, 49, 45, 40, 35, 31, 26, 22, 17, 13, 8, 4, +}; + +#if CONFIG_INTRA_INTERP +int av1_is_intra_filter_switchable(int angle) { + assert(angle > 0 && angle < 270); + if (angle % 45 == 0) return 0; + if (angle > 90 && angle < 180) { + return 1; + } else { + return ((angle < 90 ? dr_intra_derivative[angle] + : dr_intra_derivative[270 - angle]) & + 0xFF) > 0; + } +} +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA diff --git a/third_party/aom/av1/common/blockd.h b/third_party/aom/av1/common/blockd.h new file mode 100644 index 0000000000..0acab965d8 --- /dev/null +++ b/third_party/aom/av1/common/blockd.h @@ -0,0 +1,1371 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_BLOCKD_H_ +#define AV1_COMMON_BLOCKD_H_ + +#include "./aom_config.h" + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" +#include "aom_scale/yv12config.h" + +#include "av1/common/common_data.h" +#include "av1/common/quant_common.h" +#include "av1/common/entropy.h" +#include "av1/common/entropymode.h" +#include "av1/common/mv.h" +#include "av1/common/scale.h" +#include "av1/common/seg_common.h" +#include "av1/common/tile_common.h" +#if CONFIG_PVQ +#include "av1/common/pvq.h" +#include "av1/common/pvq_state.h" +#include "av1/decoder/decint.h" +#endif +#if CONFIG_CFL +#include "av1/common/cfl.h" +#endif +#ifdef __cplusplus +extern "C" { +#endif + +#define SUB8X8_COMP_REF (!(CONFIG_CB4X4 && CONFIG_CHROMA_2X2)) + +#define MAX_MB_PLANE 3 + +#if CONFIG_EXT_INTER + +#if CONFIG_COMPOUND_SEGMENT +// Set COMPOUND_SEGMENT_TYPE to one of the three +// 0: Uniform +// 1: Difference weighted +#define COMPOUND_SEGMENT_TYPE 1 + +#if COMPOUND_SEGMENT_TYPE == 0 +#define MAX_SEG_MASK_BITS 1 +// SEG_MASK_TYPES should not surpass 1 << MAX_SEG_MASK_BITS +typedef enum { + UNIFORM_45 = 0, + UNIFORM_45_INV, + SEG_MASK_TYPES, +} SEG_MASK_TYPE; + +#elif COMPOUND_SEGMENT_TYPE == 1 +#define MAX_SEG_MASK_BITS 1 +// SEG_MASK_TYPES should not surpass 1 << MAX_SEG_MASK_BITS +typedef enum { + DIFFWTD_42 = 0, + DIFFWTD_42_INV, + SEG_MASK_TYPES, +} SEG_MASK_TYPE; + +#endif // COMPOUND_SEGMENT_TYPE +#endif // CONFIG_COMPOUND_SEGMENT +#endif // CONFIG_EXT_INTER + +typedef enum { + KEY_FRAME = 0, + INTER_FRAME = 1, + FRAME_TYPES, +} FRAME_TYPE; + +static INLINE int is_inter_mode(PREDICTION_MODE mode) { +#if CONFIG_EXT_INTER + return mode >= NEARESTMV && mode <= NEW_NEWMV; +#else + return mode >= NEARESTMV && mode <= NEWMV; +#endif // CONFIG_EXT_INTER +} + +#if CONFIG_PVQ +typedef struct PVQ_INFO { + int theta[PVQ_MAX_PARTITIONS]; + int qg[PVQ_MAX_PARTITIONS]; + int k[PVQ_MAX_PARTITIONS]; + od_coeff y[OD_TXSIZE_MAX * OD_TXSIZE_MAX]; + int nb_bands; + int off[PVQ_MAX_PARTITIONS]; + int size[PVQ_MAX_PARTITIONS]; + int skip_rest; + int skip_dir; + int bs; // log of the block size minus two, + // i.e. equivalent to aom's TX_SIZE + // Block skip info, indicating whether DC/AC, is coded. + PVQ_SKIP_TYPE ac_dc_coded; // bit0: DC coded, bit1 : AC coded (1 means coded) + tran_low_t dq_dc_residue; +} PVQ_INFO; + +typedef struct PVQ_QUEUE { + PVQ_INFO *buf; // buffer for pvq info, stored in encoding order + int curr_pos; // curr position to write PVQ_INFO + int buf_len; // allocated buffer length + int last_pos; // last written position of PVQ_INFO in a tile +} PVQ_QUEUE; +#endif + +typedef struct { + uint8_t *plane[MAX_MB_PLANE]; + int stride[MAX_MB_PLANE]; +} BUFFER_SET; + +#if CONFIG_EXT_INTER +static INLINE int is_inter_singleref_mode(PREDICTION_MODE mode) { + return mode >= NEARESTMV && mode <= NEWMV; +} +#if CONFIG_COMPOUND_SINGLEREF +static INLINE int is_inter_singleref_comp_mode(PREDICTION_MODE mode) { + return mode >= SR_NEAREST_NEARMV && mode <= SR_NEW_NEWMV; +} +#endif // CONFIG_COMPOUND_SINGLEREF +static INLINE int is_inter_compound_mode(PREDICTION_MODE mode) { + return mode >= NEAREST_NEARESTMV && mode <= NEW_NEWMV; +} + +static INLINE PREDICTION_MODE compound_ref0_mode(PREDICTION_MODE mode) { + static PREDICTION_MODE lut[MB_MODE_COUNT] = { + MB_MODE_COUNT, // DC_PRED + MB_MODE_COUNT, // V_PRED + MB_MODE_COUNT, // H_PRED + MB_MODE_COUNT, // D45_PRED + MB_MODE_COUNT, // D135_PRED + MB_MODE_COUNT, // D117_PRED + MB_MODE_COUNT, // D153_PRED + MB_MODE_COUNT, // D207_PRED + MB_MODE_COUNT, // D63_PRED +#if CONFIG_ALT_INTRA + MB_MODE_COUNT, // SMOOTH_PRED +#endif // CONFIG_ALT_INTRA + MB_MODE_COUNT, // TM_PRED + MB_MODE_COUNT, // NEARESTMV + MB_MODE_COUNT, // NEARMV + MB_MODE_COUNT, // ZEROMV + MB_MODE_COUNT, // NEWMV +#if CONFIG_COMPOUND_SINGLEREF + NEARESTMV, // SR_NEAREST_NEARMV + NEARESTMV, // SR_NEAREST_NEWMV + NEARMV, // SR_NEAR_NEWMV + ZEROMV, // SR_ZERO_NEWMV + NEWMV, // SR_NEW_NEWMV +#endif // CONFIG_COMPOUND_SINGLEREF + NEARESTMV, // NEAREST_NEARESTMV + NEARESTMV, // NEAREST_NEARMV + NEARMV, // NEAR_NEARESTMV + NEARMV, // NEAR_NEARMV + NEARESTMV, // NEAREST_NEWMV + NEWMV, // NEW_NEARESTMV + NEARMV, // NEAR_NEWMV + NEWMV, // NEW_NEARMV + ZEROMV, // ZERO_ZEROMV + NEWMV, // NEW_NEWMV + }; + assert(is_inter_compound_mode(mode)); + return lut[mode]; +} + +static INLINE PREDICTION_MODE compound_ref1_mode(PREDICTION_MODE mode) { + static PREDICTION_MODE lut[MB_MODE_COUNT] = { + MB_MODE_COUNT, // DC_PRED + MB_MODE_COUNT, // V_PRED + MB_MODE_COUNT, // H_PRED + MB_MODE_COUNT, // D45_PRED + MB_MODE_COUNT, // D135_PRED + MB_MODE_COUNT, // D117_PRED + MB_MODE_COUNT, // D153_PRED + MB_MODE_COUNT, // D207_PRED + MB_MODE_COUNT, // D63_PRED +#if CONFIG_ALT_INTRA + MB_MODE_COUNT, // SMOOTH_PRED +#endif // CONFIG_ALT_INTRA + MB_MODE_COUNT, // TM_PRED + MB_MODE_COUNT, // NEARESTMV + MB_MODE_COUNT, // NEARMV + MB_MODE_COUNT, // ZEROMV + MB_MODE_COUNT, // NEWMV +#if CONFIG_COMPOUND_SINGLEREF + NEARMV, // SR_NEAREST_NEARMV + NEWMV, // SR_NEAREST_NEWMV + NEWMV, // SR_NEAR_NEWMV + NEWMV, // SR_ZERO_NEWMV + NEWMV, // SR_NEW_NEWMV +#endif // CONFIG_COMPOUND_SINGLEREF + NEARESTMV, // NEAREST_NEARESTMV + NEARMV, // NEAREST_NEARMV + NEARESTMV, // NEAR_NEARESTMV + NEARMV, // NEAR_NEARMV + NEWMV, // NEAREST_NEWMV + NEARESTMV, // NEW_NEARESTMV + NEWMV, // NEAR_NEWMV + NEARMV, // NEW_NEARMV + ZEROMV, // ZERO_ZEROMV + NEWMV, // NEW_NEWMV + }; + assert(is_inter_compound_mode(mode)); + return lut[mode]; +} + +static INLINE int have_nearmv_in_inter_mode(PREDICTION_MODE mode) { + return (mode == NEARMV || mode == NEAR_NEARMV || mode == NEAREST_NEARMV || + mode == NEAR_NEARESTMV || mode == NEAR_NEWMV || mode == NEW_NEARMV); +} + +static INLINE int have_newmv_in_inter_mode(PREDICTION_MODE mode) { + return (mode == NEWMV || mode == NEW_NEWMV || mode == NEAREST_NEWMV || + mode == NEW_NEARESTMV || mode == NEAR_NEWMV || mode == NEW_NEARMV); +} + +static INLINE int use_masked_motion_search(COMPOUND_TYPE type) { +#if CONFIG_WEDGE + return (type == COMPOUND_WEDGE); +#else + (void)type; + return 0; +#endif +} + +static INLINE int is_masked_compound_type(COMPOUND_TYPE type) { +#if CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE + return (type == COMPOUND_WEDGE || type == COMPOUND_SEG); +#elif !CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE + return (type == COMPOUND_WEDGE); +#elif CONFIG_COMPOUND_SEGMENT && !CONFIG_WEDGE + return (type == COMPOUND_SEG); +#endif // CONFIG_COMPOUND_SEGMENT + (void)type; + return 0; +} +#else + +static INLINE int have_nearmv_in_inter_mode(PREDICTION_MODE mode) { + return (mode == NEARMV); +} + +static INLINE int have_newmv_in_inter_mode(PREDICTION_MODE mode) { + return (mode == NEWMV); +} +#endif // CONFIG_EXT_INTER + +/* For keyframes, intra block modes are predicted by the (already decoded) + modes for the Y blocks to the left and above us; for interframes, there + is a single probability table. */ + +typedef struct { + PREDICTION_MODE as_mode; + int_mv as_mv[2]; // first, second inter predictor motion vectors +#if CONFIG_REF_MV + int_mv pred_mv[2]; +#endif +#if CONFIG_EXT_INTER + int_mv ref_mv[2]; +#endif // CONFIG_EXT_INTER +} b_mode_info; + +typedef int8_t MV_REFERENCE_FRAME; + +#if CONFIG_PALETTE +typedef struct { + // Number of base colors for Y (0) and UV (1) + uint8_t palette_size[2]; +// Value of base colors for Y, U, and V +#if CONFIG_HIGHBITDEPTH + uint16_t palette_colors[3 * PALETTE_MAX_SIZE]; +#else + uint8_t palette_colors[3 * PALETTE_MAX_SIZE]; +#endif // CONFIG_HIGHBITDEPTH + // Only used by encoder to store the color index of the top left pixel. + // TODO(huisu): move this to encoder + uint8_t palette_first_color_idx[2]; +} PALETTE_MODE_INFO; +#endif // CONFIG_PALETTE + +#if CONFIG_FILTER_INTRA +#define USE_3TAP_INTRA_FILTER 1 // 0: 4-tap; 1: 3-tap +typedef struct { + // 1: an ext intra mode is used; 0: otherwise. + uint8_t use_filter_intra_mode[PLANE_TYPES]; + FILTER_INTRA_MODE filter_intra_mode[PLANE_TYPES]; +} FILTER_INTRA_MODE_INFO; +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_VAR_TX +#if CONFIG_RD_DEBUG +#define TXB_COEFF_COST_MAP_SIZE (2 * MAX_MIB_SIZE) +#endif +#endif + +typedef struct RD_STATS { + int rate; + int64_t dist; + // Please be careful of using rdcost, it's not guaranteed to be set all the + // time. + // TODO(angiebird): Create a set of functions to manipulate the RD_STATS. In + // these functions, make sure rdcost is always up-to-date according to + // rate/dist. + int64_t rdcost; + int64_t sse; + int skip; // sse should equal to dist when skip == 1 +#if CONFIG_RD_DEBUG + int txb_coeff_cost[MAX_MB_PLANE]; +#if CONFIG_VAR_TX + int txb_coeff_cost_map[MAX_MB_PLANE][TXB_COEFF_COST_MAP_SIZE] + [TXB_COEFF_COST_MAP_SIZE]; +#endif // CONFIG_VAR_TX +#endif // CONFIG_RD_DEBUG +} RD_STATS; + +#if CONFIG_EXT_INTER +// This struct is used to group function args that are commonly +// sent together in functions related to interinter compound modes +typedef struct { +#if CONFIG_WEDGE + int wedge_index; + int wedge_sign; +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + SEG_MASK_TYPE mask_type; + uint8_t *seg_mask; +#endif // CONFIG_COMPOUND_SEGMENT + COMPOUND_TYPE interinter_compound_type; +} INTERINTER_COMPOUND_DATA; +#endif // CONFIG_EXT_INTER + +// This structure now relates to 8x8 block regions. +typedef struct { + // Common for both INTER and INTRA blocks + BLOCK_SIZE sb_type; + PREDICTION_MODE mode; + TX_SIZE tx_size; +#if CONFIG_VAR_TX + // TODO(jingning): This effectively assigned a separate entry for each + // 8x8 block. Apparently it takes much more space than needed. + TX_SIZE inter_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE]; + TX_SIZE min_tx_size; +#endif + int8_t skip; + int8_t segment_id; +#if CONFIG_SUPERTX + // Minimum of all segment IDs under the current supertx block. + int8_t segment_id_supertx; +#endif // CONFIG_SUPERTX + int8_t seg_id_predicted; // valid only when temporal_update is enabled + + // Only for INTRA blocks + PREDICTION_MODE uv_mode; +#if CONFIG_PALETTE + PALETTE_MODE_INFO palette_mode_info; +#endif // CONFIG_PALETTE +#if CONFIG_INTRABC + uint8_t use_intrabc; +#endif // CONFIG_INTRABC + +// Only for INTER blocks +#if CONFIG_DUAL_FILTER + InterpFilter interp_filter[4]; +#else + InterpFilter interp_filter; +#endif + MV_REFERENCE_FRAME ref_frame[2]; + TX_TYPE tx_type; +#if CONFIG_TXK_SEL + TX_TYPE txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)]; +#endif + +#if CONFIG_FILTER_INTRA + FILTER_INTRA_MODE_INFO filter_intra_mode_info; +#endif // CONFIG_FILTER_INTRA +#if CONFIG_EXT_INTRA + // The actual prediction angle is the base angle + (angle_delta * step). + int8_t angle_delta[2]; +#if CONFIG_INTRA_INTERP + // To-Do (huisu): this may be replaced by interp_filter + INTRA_FILTER intra_filter; +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA + +#if CONFIG_EXT_INTER + // interintra members + INTERINTRA_MODE interintra_mode; + // TODO(debargha): Consolidate these flags + int use_wedge_interintra; + int interintra_wedge_index; + int interintra_wedge_sign; + // interinter members + COMPOUND_TYPE interinter_compound_type; +#if CONFIG_WEDGE + int wedge_index; + int wedge_sign; +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + SEG_MASK_TYPE mask_type; +#endif // CONFIG_COMPOUND_SEGMENT +#endif // CONFIG_EXT_INTER + MOTION_MODE motion_mode; +#if CONFIG_MOTION_VAR + int overlappable_neighbors[2]; +#endif // CONFIG_MOTION_VAR + int_mv mv[2]; + int_mv pred_mv[2]; +#if CONFIG_REF_MV + uint8_t ref_mv_idx; +#endif +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_TYPE partition; +#endif +#if CONFIG_NEW_QUANT + int dq_off_index; + int send_dq_bit; +#endif // CONFIG_NEW_QUANT + /* deringing gain *per-superblock* */ + int8_t cdef_strength; +#if CONFIG_DELTA_Q + int current_q_index; +#if CONFIG_EXT_DELTA_Q + int current_delta_lf_from_base; +#endif +#endif +#if CONFIG_RD_DEBUG + RD_STATS rd_stats; + int mi_row; + int mi_col; +#endif +#if CONFIG_WARPED_MOTION + int num_proj_ref[2]; + WarpedMotionParams wm_params[2]; +#endif // CONFIG_WARPED_MOTION + + BOUNDARY_TYPE boundary_info; +} MB_MODE_INFO; + +typedef struct MODE_INFO { + MB_MODE_INFO mbmi; + b_mode_info bmi[4]; +} MODE_INFO; + +#if CONFIG_INTRABC +static INLINE int is_intrabc_block(const MB_MODE_INFO *mbmi) { + return mbmi->use_intrabc; +} +#endif + +static INLINE PREDICTION_MODE get_y_mode(const MODE_INFO *mi, int block) { +#if CONFIG_CB4X4 + (void)block; + return mi->mbmi.mode; +#else + return mi->mbmi.sb_type < BLOCK_8X8 ? mi->bmi[block].as_mode : mi->mbmi.mode; +#endif +} + +static INLINE int is_inter_block(const MB_MODE_INFO *mbmi) { +#if CONFIG_INTRABC + if (is_intrabc_block(mbmi)) return 1; +#endif + return mbmi->ref_frame[0] > INTRA_FRAME; +} + +static INLINE int has_second_ref(const MB_MODE_INFO *mbmi) { + return mbmi->ref_frame[1] > INTRA_FRAME; +} + +PREDICTION_MODE av1_left_block_mode(const MODE_INFO *cur_mi, + const MODE_INFO *left_mi, int b); + +PREDICTION_MODE av1_above_block_mode(const MODE_INFO *cur_mi, + const MODE_INFO *above_mi, int b); + +#if CONFIG_GLOBAL_MOTION +static INLINE int is_global_mv_block(const MODE_INFO *mi, int block, + TransformationType type) { + PREDICTION_MODE mode = get_y_mode(mi, block); +#if GLOBAL_SUB8X8_USED + const int block_size_allowed = 1; +#else + const BLOCK_SIZE bsize = mi->mbmi.sb_type; + const int block_size_allowed = (bsize >= BLOCK_8X8); +#endif // GLOBAL_SUB8X8_USED +#if CONFIG_EXT_INTER + return (mode == ZEROMV || mode == ZERO_ZEROMV) && type > TRANSLATION && + block_size_allowed; +#else + return mode == ZEROMV && type > TRANSLATION && block_size_allowed; +#endif // CONFIG_EXT_INTER +} +#endif // CONFIG_GLOBAL_MOTION + +enum mv_precision { MV_PRECISION_Q3, MV_PRECISION_Q4 }; + +struct buf_2d { + uint8_t *buf; + uint8_t *buf0; + int width; + int height; + int stride; +}; + +typedef struct macroblockd_plane { + tran_low_t *dqcoeff; + PLANE_TYPE plane_type; + int subsampling_x; + int subsampling_y; + struct buf_2d dst; + struct buf_2d pre[2]; + ENTROPY_CONTEXT *above_context; + ENTROPY_CONTEXT *left_context; + int16_t seg_dequant[MAX_SEGMENTS][2]; +#if CONFIG_NEW_QUANT + dequant_val_type_nuq seg_dequant_nuq[MAX_SEGMENTS][QUANT_PROFILES] + [COEF_BANDS]; +#endif +#if CONFIG_PALETTE + uint8_t *color_index_map; +#endif // CONFIG_PALETTE + + // number of 4x4s in current block + uint16_t n4_w, n4_h; + // log2 of n4_w, n4_h + uint8_t n4_wl, n4_hl; + // block size in pixels + uint8_t width, height; + +#if CONFIG_AOM_QM + const qm_val_t *seg_iqmatrix[MAX_SEGMENTS][2][TX_SIZES]; +#endif + // encoder + const int16_t *dequant; +#if CONFIG_NEW_QUANT + const dequant_val_type_nuq *dequant_val_nuq[QUANT_PROFILES]; +#endif // CONFIG_NEW_QUANT +#if CONFIG_AOM_QM + const qm_val_t *seg_qmatrix[MAX_SEGMENTS][2][TX_SIZES]; +#endif + +#if CONFIG_PVQ || CONFIG_DAALA_DIST + DECLARE_ALIGNED(16, int16_t, pred[MAX_SB_SQUARE]); + // PVQ: forward transformed predicted image, a reference for PVQ. + tran_low_t *pvq_ref_coeff; +#endif +} MACROBLOCKD_PLANE; + +#define BLOCK_OFFSET(x, i) \ + ((x) + (i) * (1 << (tx_size_wide_log2[0] + tx_size_high_log2[0]))) + +typedef struct RefBuffer { + // TODO(dkovalev): idx is not really required and should be removed, now it + // is used in av1_onyxd_if.c + int idx; + YV12_BUFFER_CONFIG *buf; + struct scale_factors sf; +} RefBuffer; + +typedef int16_t EobThresholdMD[TX_SIZES_ALL][TX_TYPES]; + +typedef struct macroblockd { + struct macroblockd_plane plane[MAX_MB_PLANE]; + uint8_t bmode_blocks_wl; + uint8_t bmode_blocks_hl; + + FRAME_COUNTS *counts; + TileInfo tile; + + int mi_stride; + + MODE_INFO **mi; + MODE_INFO *left_mi; + MODE_INFO *above_mi; + MB_MODE_INFO *left_mbmi; + MB_MODE_INFO *above_mbmi; + + int up_available; + int left_available; +#if CONFIG_CHROMA_SUB8X8 + int chroma_up_available; + int chroma_left_available; +#endif + + const aom_prob (*partition_probs)[PARTITION_TYPES - 1]; + + /* Distance of MB away from frame edges */ + int mb_to_left_edge; + int mb_to_right_edge; + int mb_to_top_edge; + int mb_to_bottom_edge; + + FRAME_CONTEXT *fc; + + /* pointers to reference frames */ + const RefBuffer *block_refs[2]; + + /* pointer to current frame */ + const YV12_BUFFER_CONFIG *cur_buf; + + ENTROPY_CONTEXT *above_context[MAX_MB_PLANE]; + ENTROPY_CONTEXT left_context[MAX_MB_PLANE][2 * MAX_MIB_SIZE]; + + PARTITION_CONTEXT *above_seg_context; + PARTITION_CONTEXT left_seg_context[MAX_MIB_SIZE]; + +#if CONFIG_VAR_TX + TXFM_CONTEXT *above_txfm_context; + TXFM_CONTEXT *left_txfm_context; + TXFM_CONTEXT left_txfm_context_buffer[MAX_MIB_SIZE]; + + TX_SIZE max_tx_size; +#if CONFIG_SUPERTX + TX_SIZE supertx_size; +#endif +#endif + + // block dimension in the unit of mode_info. + uint8_t n8_w, n8_h; + +#if CONFIG_REF_MV + uint8_t ref_mv_count[MODE_CTX_REF_FRAMES]; + CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE]; + uint8_t is_sec_rect; +#endif + +#if CONFIG_PVQ + daala_dec_ctx daala_dec; +#endif +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *tile_ctx; +#endif +#if CONFIG_HIGHBITDEPTH + /* Bit depth: 8, 10, 12 */ + int bd; +#endif + + int qindex[MAX_SEGMENTS]; + int lossless[MAX_SEGMENTS]; + int corrupted; + + struct aom_internal_error_info *error_info; +#if CONFIG_GLOBAL_MOTION + WarpedMotionParams *global_motion; +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_DELTA_Q + int prev_qindex; + int delta_qindex; + int current_qindex; +#if CONFIG_EXT_DELTA_Q + // Since actual frame level loop filtering level value is not available + // at the beginning of the tile (only available during actual filtering) + // at encoder side.we record the delta_lf (against the frame level loop + // filtering level) and code the delta between previous superblock's delta + // lf and current delta lf. It is equivalent to the delta between previous + // superblock's actual lf and current lf. + int prev_delta_lf_from_base; + int current_delta_lf_from_base; +#endif +#endif +#if CONFIG_ADAPT_SCAN + const EobThresholdMD *eob_threshold_md; +#endif + +#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SEGMENT + DECLARE_ALIGNED(16, uint8_t, seg_mask[2 * MAX_SB_SQUARE]); +#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SEGMENT + +#if CONFIG_CFL + CFL_CTX *cfl; +#endif +} MACROBLOCKD; + +static INLINE BLOCK_SIZE get_subsize(BLOCK_SIZE bsize, + PARTITION_TYPE partition) { + if (partition == PARTITION_INVALID) + return BLOCK_INVALID; + else + return subsize_lookup[partition][bsize]; +} + +static const TX_TYPE intra_mode_to_tx_type_context[INTRA_MODES] = { + DCT_DCT, // DC + ADST_DCT, // V + DCT_ADST, // H + DCT_DCT, // D45 + ADST_ADST, // D135 + ADST_DCT, // D117 + DCT_ADST, // D153 + DCT_ADST, // D207 + ADST_DCT, // D63 +#if CONFIG_ALT_INTRA + ADST_ADST, // SMOOTH +#endif // CONFIG_ALT_INTRA + ADST_ADST, // TM +}; + +#if CONFIG_SUPERTX +static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) { + TX_SIZE max_tx_size = txsize_sqr_map[mbmi->tx_size]; + return tx_size_wide[max_tx_size] > + AOMMIN(block_size_wide[mbmi->sb_type], block_size_high[mbmi->sb_type]); +} +#endif // CONFIG_SUPERTX + +#define USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 1 + +#if CONFIG_RECT_TX +static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; } +#endif // CONFIG_RECT_TX + +#if CONFIG_EXT_TX +#define ALLOW_INTRA_EXT_TX 1 + +typedef enum { + // DCT only + EXT_TX_SET_DCTONLY = 0, + // DCT + Identity only + EXT_TX_SET_DCT_IDTX = 1, + // Discrete Trig transforms w/o flip (4) + Identity (1) + EXT_TX_SET_DTT4_IDTX = 2, + // Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2) + EXT_TX_SET_DTT4_IDTX_1DDCT = 3, + // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2) + EXT_TX_SET_DTT9_IDTX_1DDCT = 4, + // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6) + EXT_TX_SET_ALL16 = 5, + EXT_TX_SET_TYPES +} TxSetType; + +// Number of transform types in each set type +static const int num_ext_tx_set[EXT_TX_SET_TYPES] = { 1, 2, 5, 7, 12, 16 }; + +// Maps intra set index to the set type +static const int ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = { + EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX +}; + +// Maps inter set index to the set type +static const int ext_tx_set_type_inter[EXT_TX_SETS_INTER] = { + EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT, + EXT_TX_SET_DCT_IDTX +}; + +// Maps set types above to the indices used for intra +static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = { 0, -1, 2, + 1, -1, -1 }; + +// Maps set types above to the indices used for inter +static const int ext_tx_set_index_inter[EXT_TX_SET_TYPES] = { + 0, 3, -1, -1, 2, 1 +}; + +static INLINE TxSetType get_ext_tx_set_type(TX_SIZE tx_size, BLOCK_SIZE bs, + int is_inter, int use_reduced_set) { + const TX_SIZE tx_size2 = txsize_sqr_up_map[tx_size]; + tx_size = txsize_sqr_map[tx_size]; +#if CONFIG_CB4X4 && USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 + (void)bs; + if (tx_size > TX_32X32) return EXT_TX_SET_DCTONLY; +#else + if (tx_size > TX_32X32 || bs < BLOCK_8X8) return EXT_TX_SET_DCTONLY; +#endif + if (use_reduced_set) + return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX; + if (tx_size2 == TX_32X32) + return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY; + if (is_inter) + return (tx_size == TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT + : EXT_TX_SET_ALL16); + else + return (tx_size == TX_16X16 ? EXT_TX_SET_DTT4_IDTX + : EXT_TX_SET_DTT4_IDTX_1DDCT); +} + +static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter, + int use_reduced_set) { + const TxSetType set_type = + get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set); + return is_inter ? ext_tx_set_index_inter[set_type] + : ext_tx_set_index_intra[set_type]; +} + +static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] = + { +#if CONFIG_CB4X4 + { 1, 1, 1, 1, 1 }, // unused + { 0, 1, 1, 0, 0 }, + { 0, 0, 0, 1, 0 }, +#else + { 1, 1, 1, 1 }, // unused + { 1, 1, 0, 0 }, + { 0, 0, 1, 0 }, +#endif // CONFIG_CB4X4 + }; + +static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] = + { +#if CONFIG_CB4X4 + { 1, 1, 1, 1, 1 }, // unused + { 0, 1, 1, 0, 0 }, + { 0, 0, 0, 1, 0 }, + { 0, 0, 0, 0, 1 }, +#else + { 1, 1, 1, 1 }, // unused + { 1, 1, 0, 0 }, + { 0, 0, 1, 0 }, + { 0, 0, 0, 1 }, +#endif // CONFIG_CB4X4 + }; + +// Transform types used in each intra set +static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = { + { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0 }, + { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 }, +}; + +// Numbers of transform types used in each intra set +static const int ext_tx_cnt_intra[EXT_TX_SETS_INTRA] = { 1, 7, 5 }; + +// Transform types used in each inter set +static const int ext_tx_used_inter[EXT_TX_SETS_INTER][TX_TYPES] = { + { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0 }, + { 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 }, +}; + +// Numbers of transform types used in each inter set +static const int ext_tx_cnt_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2 }; + +// 1D Transforms used in inter set, this needs to be changed if +// ext_tx_used_inter is changed +static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = { + { 1, 0, 0, 0 }, { 1, 1, 1, 1 }, { 1, 1, 1, 1 }, { 1, 0, 0, 1 }, +}; + +static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter, + int use_reduced_set) { + const int set_type = + get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set); + return num_ext_tx_set[set_type]; +} + +#if CONFIG_RECT_TX +static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) { + static const char LUT[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 0, // BLOCK_2X2 + 0, // BLOCK_2X4 + 0, // BLOCK_4X2 +#endif + 0, // BLOCK_4X4 + 1, // BLOCK_4X8 + 1, // BLOCK_8X4 + 0, // BLOCK_8X8 + 1, // BLOCK_8X16 + 1, // BLOCK_16X8 + 0, // BLOCK_16X16 + 1, // BLOCK_16X32 + 1, // BLOCK_32X16 + 0, // BLOCK_32X32 + 0, // BLOCK_32X64 + 0, // BLOCK_64X32 + 0, // BLOCK_64X64 +#if CONFIG_EXT_PARTITION + 0, // BLOCK_64X128 + 0, // BLOCK_128X64 + 0, // BLOCK_128X128 +#endif // CONFIG_EXT_PARTITION + }; + + return LUT[bsize]; +} + +static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd, + const MB_MODE_INFO *mbmi) { + return is_rect_tx_allowed_bsize(mbmi->sb_type) && + !xd->lossless[mbmi->segment_id]; +} +#endif // CONFIG_RECT_TX +#endif // CONFIG_EXT_TX + +static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode, + int is_inter) { + const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[tx_mode]; +#if (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX + const TX_SIZE max_rect_tx_size = max_txsize_rect_lookup[bsize]; +#else + const TX_SIZE max_tx_size = max_txsize_lookup[bsize]; +#endif // (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX + (void)is_inter; +#if CONFIG_VAR_TX && CONFIG_RECT_TX +#if CONFIG_CB4X4 + if (bsize == BLOCK_4X4) + return AOMMIN(max_txsize_lookup[bsize], largest_tx_size); +#else + if (bsize < BLOCK_8X8) + return AOMMIN(max_txsize_lookup[bsize], largest_tx_size); +#endif + if (txsize_sqr_map[max_rect_tx_size] <= largest_tx_size) + return max_rect_tx_size; + else + return largest_tx_size; +#elif CONFIG_EXT_TX && CONFIG_RECT_TX + if (txsize_sqr_up_map[max_rect_tx_size] <= largest_tx_size) { + return max_rect_tx_size; + } else { + return largest_tx_size; + } +#else + return AOMMIN(max_tx_size, largest_tx_size); +#endif // CONFIG_VAR_TX && CONFIG_RECT_TX +} + +#if CONFIG_EXT_INTRA +#define MAX_ANGLE_DELTA 3 +#define ANGLE_STEP 3 +extern const int16_t dr_intra_derivative[90]; +static const uint8_t mode_to_angle_map[INTRA_MODES] = { + 0, 90, 180, 45, 135, 111, 157, 203, 67, 0, +}; +#if CONFIG_INTRA_INTERP +// Returns whether filter selection is needed for a given +// intra prediction angle. +int av1_is_intra_filter_switchable(int angle); +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA + +#define FIXED_TX_TYPE 0 + +// Converts block_index for given transform size to index of the block in raster +// order. +static INLINE int av1_block_index_to_raster_order(TX_SIZE tx_size, + int block_idx) { + // For transform size 4x8, the possible block_idx values are 0 & 2, because + // block_idx values are incremented in steps of size 'tx_width_unit x + // tx_height_unit'. But, for this transform size, block_idx = 2 corresponds to + // block number 1 in raster order, inside an 8x8 MI block. + // For any other transform size, the two indices are equivalent. + return (tx_size == TX_4X8 && block_idx == 2) ? 1 : block_idx; +} + +// Inverse of above function. +// Note: only implemented for transform sizes 4x4, 4x8 and 8x4 right now. +static INLINE int av1_raster_order_to_block_index(TX_SIZE tx_size, + int raster_order) { + assert(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4); + // We ensure that block indices are 0 & 2 if tx size is 4x8 or 8x4. + return (tx_size == TX_4X4) ? raster_order : (raster_order > 0) ? 2 : 0; +} + +static INLINE TX_TYPE get_default_tx_type(PLANE_TYPE plane_type, + const MACROBLOCKD *xd, int block_idx, + TX_SIZE tx_size) { + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + + if (is_inter_block(mbmi) || plane_type != PLANE_TYPE_Y || + xd->lossless[mbmi->segment_id] || tx_size >= TX_32X32) + return DCT_DCT; + + return intra_mode_to_tx_type_context[plane_type == PLANE_TYPE_Y + ? get_y_mode(xd->mi[0], block_idx) + : mbmi->uv_mode]; +} + +static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type, const MACROBLOCKD *xd, + int block, TX_SIZE tx_size) { + const MODE_INFO *const mi = xd->mi[0]; + const MB_MODE_INFO *const mbmi = &mi->mbmi; +#if CONFIG_INTRABC + // TODO(aconverse@google.com): Revisit this decision + if (is_intrabc_block(mbmi)) return DCT_DCT; +#endif // CONFIG_INTRABC +#if !CONFIG_TXK_SEL +#if FIXED_TX_TYPE + const int block_raster_idx = av1_block_index_to_raster_order(tx_size, block); + return get_default_tx_type(plane_type, xd, block_raster_idx, tx_size); +#elif CONFIG_EXT_TX +#if !CONFIG_CB4X4 + const int block_raster_idx = av1_block_index_to_raster_order(tx_size, block); +#endif // !CONFIG_CB4X4 + if (xd->lossless[mbmi->segment_id] || txsize_sqr_map[tx_size] > TX_32X32 || + (txsize_sqr_map[tx_size] >= TX_32X32 && !is_inter_block(mbmi))) + return DCT_DCT; + if (mbmi->sb_type >= BLOCK_8X8 || CONFIG_CB4X4) { + if (plane_type == PLANE_TYPE_Y) { +#if !ALLOW_INTRA_EXT_TX + if (is_inter_block(mbmi)) +#endif // ALLOW_INTRA_EXT_TX + return mbmi->tx_type; + } + + if (is_inter_block(mbmi)) { +// UV Inter only +#if CONFIG_CB4X4 + if (tx_size < TX_4X4) return DCT_DCT; +#endif + return (mbmi->tx_type == IDTX && txsize_sqr_map[tx_size] >= TX_32X32) + ? DCT_DCT + : mbmi->tx_type; + } + } + +#if CONFIG_CB4X4 + (void)block; + if (tx_size < TX_4X4) + return DCT_DCT; + else + return intra_mode_to_tx_type_context[mbmi->uv_mode]; +#else + + // Sub8x8-Inter/Intra OR UV-Intra + if (is_inter_block(mbmi)) // Sub8x8-Inter + return DCT_DCT; + else // Sub8x8 Intra OR UV-Intra + return intra_mode_to_tx_type_context[plane_type == PLANE_TYPE_Y + ? get_y_mode(mi, block_raster_idx) + : mbmi->uv_mode]; +#endif // CONFIG_CB4X4 +#else // CONFIG_EXT_TX + (void)block; + if (plane_type != PLANE_TYPE_Y || xd->lossless[mbmi->segment_id] || + txsize_sqr_map[tx_size] >= TX_32X32) + return DCT_DCT; + return mbmi->tx_type; +#endif // CONFIG_EXT_TX +#else // !CONFIG_TXK_SEL + (void)tx_size; + TX_TYPE tx_type; + if (plane_type != PLANE_TYPE_Y || xd->lossless[mbmi->segment_id] || + mbmi->tx_size >= TX_32X32) { + tx_type = DCT_DCT; + } else { + tx_type = mbmi->txk_type[block]; + } + assert(tx_type >= DCT_DCT && tx_type < TX_TYPES); + return tx_type; +#endif // !CONFIG_TXK_SEL +} + +void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y); + +static INLINE int tx_size_to_depth(TX_SIZE tx_size) { + return (int)(tx_size - TX_4X4); +} + +static INLINE TX_SIZE depth_to_tx_size(int depth) { + return (TX_SIZE)(depth + TX_4X4); +} + +static INLINE TX_SIZE get_uv_tx_size(const MB_MODE_INFO *mbmi, + const struct macroblockd_plane *pd) { + TX_SIZE uv_txsize; +#if CONFIG_CB4X4 + assert(mbmi->tx_size > TX_2X2); +#endif + +#if CONFIG_SUPERTX + if (supertx_enabled(mbmi)) + return uvsupertx_size_lookup[txsize_sqr_map[mbmi->tx_size]] + [pd->subsampling_x][pd->subsampling_y]; +#endif // CONFIG_SUPERTX + + uv_txsize = uv_txsize_lookup[mbmi->sb_type][mbmi->tx_size][pd->subsampling_x] + [pd->subsampling_y]; +#if CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + uv_txsize = AOMMAX(uv_txsize, TX_4X4); +#endif + assert(uv_txsize != TX_INVALID); + return uv_txsize; +} + +static INLINE TX_SIZE get_tx_size(int plane, const MACROBLOCKD *xd) { + const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const MACROBLOCKD_PLANE *pd = &xd->plane[plane]; + const TX_SIZE tx_size = plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size; + return tx_size; +} + +static INLINE BLOCK_SIZE +get_plane_block_size(BLOCK_SIZE bsize, const struct macroblockd_plane *pd) { + return ss_size_lookup[bsize][pd->subsampling_x][pd->subsampling_y]; +} + +static INLINE void reset_skip_context(MACROBLOCKD *xd, BLOCK_SIZE bsize) { + int i; + for (i = 0; i < MAX_MB_PLANE; i++) { + struct macroblockd_plane *const pd = &xd->plane[i]; + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); + const int txs_wide = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; + const int txs_high = block_size_high[plane_bsize] >> tx_size_high_log2[0]; + memset(pd->above_context, 0, sizeof(ENTROPY_CONTEXT) * txs_wide); + memset(pd->left_context, 0, sizeof(ENTROPY_CONTEXT) * txs_high); + } +} + +typedef void (*foreach_transformed_block_visitor)(int plane, int block, + int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, + TX_SIZE tx_size, void *arg); + +void av1_foreach_transformed_block_in_plane( + const MACROBLOCKD *const xd, BLOCK_SIZE bsize, int plane, + foreach_transformed_block_visitor visit, void *arg); + +#if CONFIG_LV_MAP +void av1_foreach_transformed_block(const MACROBLOCKD *const xd, + BLOCK_SIZE bsize, int mi_row, int mi_col, + foreach_transformed_block_visitor visit, + void *arg); +#endif + +#if CONFIG_DAALA_DIST +void av1_foreach_8x8_transformed_block_in_plane( + const MACROBLOCKD *const xd, BLOCK_SIZE bsize, int plane, + foreach_transformed_block_visitor visit, + foreach_transformed_block_visitor mi_visit, void *arg); +#endif + +#if CONFIG_COEF_INTERLEAVE +static INLINE int get_max_4x4_size(int num_4x4, int mb_to_edge, + int subsampling) { + return num_4x4 + (mb_to_edge >= 0 ? 0 : mb_to_edge >> (5 + subsampling)); +} + +void av1_foreach_transformed_block_interleave( + const MACROBLOCKD *const xd, BLOCK_SIZE bsize, + foreach_transformed_block_visitor visit, void *arg); +#endif + +void av1_set_contexts(const MACROBLOCKD *xd, struct macroblockd_plane *pd, + int plane, TX_SIZE tx_size, int has_eob, int aoff, + int loff); + +#if CONFIG_EXT_INTER +static INLINE int is_interintra_allowed_bsize(const BLOCK_SIZE bsize) { +#if CONFIG_INTERINTRA + // TODO(debargha): Should this be bsize < BLOCK_LARGEST? + return (bsize >= BLOCK_8X8) && (bsize < BLOCK_64X64); +#else + (void)bsize; + return 0; +#endif // CONFIG_INTERINTRA +} + +static INLINE int is_interintra_allowed_mode(const PREDICTION_MODE mode) { +#if CONFIG_INTERINTRA + return (mode >= NEARESTMV) && (mode <= NEWMV); +#else + (void)mode; + return 0; +#endif // CONFIG_INTERINTRA +} + +static INLINE int is_interintra_allowed_ref(const MV_REFERENCE_FRAME rf[2]) { +#if CONFIG_INTERINTRA + return (rf[0] > INTRA_FRAME) && (rf[1] <= INTRA_FRAME); +#else + (void)rf; + return 0; +#endif // CONFIG_INTERINTRA +} + +static INLINE int is_interintra_allowed(const MB_MODE_INFO *mbmi) { + return is_interintra_allowed_bsize(mbmi->sb_type) && + is_interintra_allowed_mode(mbmi->mode) && + is_interintra_allowed_ref(mbmi->ref_frame); +} + +static INLINE int is_interintra_allowed_bsize_group(int group) { + int i; + for (i = 0; i < BLOCK_SIZES; i++) { + if (size_group_lookup[i] == group && + is_interintra_allowed_bsize((BLOCK_SIZE)i)) { + return 1; + } + } + return 0; +} + +static INLINE int is_interintra_pred(const MB_MODE_INFO *mbmi) { + return (mbmi->ref_frame[1] == INTRA_FRAME) && is_interintra_allowed(mbmi); +} +#endif // CONFIG_EXT_INTER + +#if CONFIG_VAR_TX +static INLINE int get_vartx_max_txsize(const MB_MODE_INFO *const mbmi, + BLOCK_SIZE bsize) { +#if CONFIG_CB4X4 + (void)mbmi; + return max_txsize_rect_lookup[bsize]; +#endif // CONFIG_C4X4 + return mbmi->sb_type < BLOCK_8X8 ? max_txsize_rect_lookup[mbmi->sb_type] + : max_txsize_rect_lookup[bsize]; +} +#endif // CONFIG_VAR_TX + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +static INLINE int is_motion_variation_allowed_bsize(BLOCK_SIZE bsize) { + return (bsize >= BLOCK_8X8); +} + +static INLINE int is_motion_variation_allowed_compound( + const MB_MODE_INFO *mbmi) { + if (!has_second_ref(mbmi)) + return 1; + else + return 0; +} + +#if CONFIG_MOTION_VAR +// input: log2 of length, 0(4), 1(8), ... +static const int max_neighbor_obmc[6] = { 0, 1, 2, 3, 4, 4 }; + +static INLINE int check_num_overlappable_neighbors(const MB_MODE_INFO *mbmi) { + return !(mbmi->overlappable_neighbors[0] == 0 && + mbmi->overlappable_neighbors[1] == 0); +} +#endif + +static INLINE MOTION_MODE motion_mode_allowed( +#if CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + int block, const WarpedMotionParams *gm_params, +#endif // CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + const MODE_INFO *mi) { + const MB_MODE_INFO *mbmi = &mi->mbmi; +#if CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + const TransformationType gm_type = gm_params[mbmi->ref_frame[0]].wmtype; + if (is_global_mv_block(mi, block, gm_type)) return SIMPLE_TRANSLATION; +#endif // CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION +#if CONFIG_EXT_INTER + if (is_motion_variation_allowed_bsize(mbmi->sb_type) && + is_inter_mode(mbmi->mode) && mbmi->ref_frame[1] != INTRA_FRAME && + is_motion_variation_allowed_compound(mbmi)) { +#else + if (is_motion_variation_allowed_bsize(mbmi->sb_type) && + is_inter_mode(mbmi->mode) && is_motion_variation_allowed_compound(mbmi)) { +#endif // CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR + if (!check_num_overlappable_neighbors(mbmi)) return SIMPLE_TRANSLATION; +#endif +#if CONFIG_WARPED_MOTION + if (!has_second_ref(mbmi) && mbmi->num_proj_ref[0] >= 1) + return WARPED_CAUSAL; + else +#endif // CONFIG_WARPED_MOTION +#if CONFIG_MOTION_VAR + return OBMC_CAUSAL; +#else + return SIMPLE_TRANSLATION; +#endif // CONFIG_MOTION_VAR + } else { + return SIMPLE_TRANSLATION; + } +} + +static INLINE void assert_motion_mode_valid(MOTION_MODE mode, +#if CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + int block, + const WarpedMotionParams *gm_params, +#endif // CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + const MODE_INFO *mi) { + const MOTION_MODE last_motion_mode_allowed = motion_mode_allowed( +#if CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + block, gm_params, +#endif // CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + mi); + // Check that the input mode is not illegal + if (last_motion_mode_allowed < mode) + assert(0 && "Illegal motion mode selected"); +} + +#if CONFIG_MOTION_VAR +static INLINE int is_neighbor_overlappable(const MB_MODE_INFO *mbmi) { + return (is_inter_block(mbmi)); +} +#endif // CONFIG_MOTION_VAR +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + +// Returns sub-sampled dimensions of the given block. +// The output values for 'rows_within_bounds' and 'cols_within_bounds' will +// differ from 'height' and 'width' when part of the block is outside the right +// and/or bottom image boundary. +static INLINE void av1_get_block_dimensions(BLOCK_SIZE bsize, int plane, + const MACROBLOCKD *xd, int *width, + int *height, + int *rows_within_bounds, + int *cols_within_bounds) { + const int block_height = block_size_high[bsize]; + const int block_width = block_size_wide[bsize]; + const int block_rows = (xd->mb_to_bottom_edge >= 0) + ? block_height + : (xd->mb_to_bottom_edge >> 3) + block_height; + const int block_cols = (xd->mb_to_right_edge >= 0) + ? block_width + : (xd->mb_to_right_edge >> 3) + block_width; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + assert(IMPLIES(plane == PLANE_TYPE_Y, pd->subsampling_x == 0)); + assert(IMPLIES(plane == PLANE_TYPE_Y, pd->subsampling_y == 0)); + assert(block_width >= block_cols); + assert(block_height >= block_rows); + if (width) *width = block_width >> pd->subsampling_x; + if (height) *height = block_height >> pd->subsampling_y; + if (rows_within_bounds) *rows_within_bounds = block_rows >> pd->subsampling_y; + if (cols_within_bounds) *cols_within_bounds = block_cols >> pd->subsampling_x; +} + +#if CONFIG_GLOBAL_MOTION +static INLINE int is_nontrans_global_motion(const MACROBLOCKD *xd) { + const MODE_INFO *mi = xd->mi[0]; + const MB_MODE_INFO *const mbmi = &mi->mbmi; + int ref; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + + // First check if all modes are ZEROMV + if (mbmi->sb_type >= BLOCK_8X8 || unify_bsize) { +#if CONFIG_EXT_INTER + if (mbmi->mode != ZEROMV && mbmi->mode != ZERO_ZEROMV) return 0; +#else + if (mbmi->mode != ZEROMV) return 0; +#endif // CONFIG_EXT_INTER + } else { +#if CONFIG_EXT_INTER + if (mi->bmi[0].as_mode != ZEROMV || mi->bmi[1].as_mode != ZEROMV || + mi->bmi[2].as_mode != ZEROMV || mi->bmi[3].as_mode != ZEROMV || + mi->bmi[0].as_mode != ZERO_ZEROMV || + mi->bmi[1].as_mode != ZERO_ZEROMV || + mi->bmi[2].as_mode != ZERO_ZEROMV || mi->bmi[3].as_mode != ZERO_ZEROMV) + return 0; +#else + if (mi->bmi[0].as_mode != ZEROMV || mi->bmi[1].as_mode != ZEROMV || + mi->bmi[2].as_mode != ZEROMV || mi->bmi[3].as_mode != ZEROMV) + return 0; +#endif // CONFIG_EXT_INTER + } + +#if !GLOBAL_SUB8X8_USED + if (mbmi->sb_type < BLOCK_8X8) return 0; +#endif + + // Now check if all global motion is non translational + for (ref = 0; ref < 1 + has_second_ref(mbmi); ++ref) { + if (xd->global_motion[mbmi->ref_frame[ref]].wmtype <= TRANSLATION) return 0; + } + return 1; +} +#endif // CONFIG_GLOBAL_MOTION + +static INLINE PLANE_TYPE get_plane_type(int plane) { + return (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV; +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_BLOCKD_H_ diff --git a/third_party/aom/av1/common/cdef.c b/third_party/aom/av1/common/cdef.c new file mode 100644 index 0000000000..53dff98b78 --- /dev/null +++ b/third_party/aom/av1/common/cdef.c @@ -0,0 +1,445 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "./aom_scale_rtcd.h" +#include "aom/aom_integer.h" +#include "av1/common/cdef.h" +#include "av1/common/od_dering.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/reconinter.h" + +int sb_all_skip(const AV1_COMMON *const cm, int mi_row, int mi_col) { + int r, c; + int maxc, maxr; + int skip = 1; + maxc = cm->mi_cols - mi_col; + maxr = cm->mi_rows - mi_row; +#if CONFIG_EXT_PARTITION + if (maxr > cm->mib_size_log2) maxr = cm->mib_size_log2; + if (maxc > cm->mib_size_log2) maxc = cm->mib_size_log2; +#else + if (maxr > MAX_MIB_SIZE) maxr = MAX_MIB_SIZE; + if (maxc > MAX_MIB_SIZE) maxc = MAX_MIB_SIZE; +#endif + + for (r = 0; r < maxr; r++) { + for (c = 0; c < maxc; c++) { + skip = skip && + cm->mi_grid_visible[(mi_row + r) * cm->mi_stride + mi_col + c] + ->mbmi.skip; + } + } + return skip; +} + +static int is_8x8_block_skip(MODE_INFO **grid, int mi_row, int mi_col, + int mi_stride) { + int is_skip = 1; + for (int r = 0; r < mi_size_high[BLOCK_8X8]; ++r) + for (int c = 0; c < mi_size_wide[BLOCK_8X8]; ++c) + is_skip &= grid[(mi_row + r) * mi_stride + (mi_col + c)]->mbmi.skip; + + return is_skip; +} + +int sb_compute_dering_list(const AV1_COMMON *const cm, int mi_row, int mi_col, + dering_list *dlist, int filter_skip) { + int r, c; + int maxc, maxr; + MODE_INFO **grid; + int count = 0; + grid = cm->mi_grid_visible; + maxc = cm->mi_cols - mi_col; + maxr = cm->mi_rows - mi_row; +#if CONFIG_EXT_PARTITION + if (maxr > cm->mib_size_log2) maxr = cm->mib_size_log2; + if (maxc > cm->mib_size_log2) maxc = cm->mib_size_log2; +#else + if (maxr > MAX_MIB_SIZE) maxr = MAX_MIB_SIZE; + if (maxc > MAX_MIB_SIZE) maxc = MAX_MIB_SIZE; +#endif + + const int r_step = mi_size_high[BLOCK_8X8]; + const int c_step = mi_size_wide[BLOCK_8X8]; + const int r_shift = (r_step == 2); + const int c_shift = (c_step == 2); + + assert(r_step == 1 || r_step == 2); + assert(c_step == 1 || c_step == 2); + + if (filter_skip) { + for (r = 0; r < maxr; r += r_step) { + for (c = 0; c < maxc; c += c_step) { + dlist[count].by = r >> r_shift; + dlist[count].bx = c >> c_shift; + dlist[count].skip = + is_8x8_block_skip(grid, mi_row + r, mi_col + c, cm->mi_stride); + count++; + } + } + } else { + for (r = 0; r < maxr; r += r_step) { + for (c = 0; c < maxc; c += c_step) { + if (!is_8x8_block_skip(grid, mi_row + r, mi_col + c, cm->mi_stride)) { + dlist[count].by = r >> r_shift; + dlist[count].bx = c >> c_shift; + dlist[count].skip = 0; + count++; + } + } + } + } + return count; +} + +void copy_rect8_8bit_to_16bit_c(uint16_t *dst, int dstride, const uint8_t *src, + int sstride, int v, int h) { + int i, j; + for (i = 0; i < v; i++) { + for (j = 0; j < h; j++) { + dst[i * dstride + j] = src[i * sstride + j]; + } + } +} + +void copy_rect8_16bit_to_16bit_c(uint16_t *dst, int dstride, + const uint16_t *src, int sstride, int v, + int h) { + int i, j; + for (i = 0; i < v; i++) { + for (j = 0; j < h; j++) { + dst[i * dstride + j] = src[i * sstride + j]; + } + } +} + +void copy_sb8_16(UNUSED AV1_COMMON *cm, uint16_t *dst, int dstride, + const uint8_t *src, int src_voffset, int src_hoffset, + int sstride, int vsize, int hsize) { +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) { + const uint16_t *base = + &CONVERT_TO_SHORTPTR(src)[src_voffset * sstride + src_hoffset]; + copy_rect8_16bit_to_16bit(dst, dstride, base, sstride, vsize, hsize); + } else { +#endif + const uint8_t *base = &src[src_voffset * sstride + src_hoffset]; + copy_rect8_8bit_to_16bit(dst, dstride, base, sstride, vsize, hsize); +#if CONFIG_HIGHBITDEPTH + } +#endif +} + +static INLINE void fill_rect(uint16_t *dst, int dstride, int v, int h, + uint16_t x) { + int i, j; + for (i = 0; i < v; i++) { + for (j = 0; j < h; j++) { + dst[i * dstride + j] = x; + } + } +} + +static INLINE void copy_rect(uint16_t *dst, int dstride, const uint16_t *src, + int sstride, int v, int h) { + int i, j; + for (i = 0; i < v; i++) { + for (j = 0; j < h; j++) { + dst[i * dstride + j] = src[i * sstride + j]; + } + } +} + +void av1_cdef_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, + MACROBLOCKD *xd) { + int sbr, sbc; + int nhsb, nvsb; + uint16_t src[OD_DERING_INBUF_SIZE]; + uint16_t *linebuf[3]; + uint16_t *colbuf[3]; + dering_list dlist[MAX_MIB_SIZE * MAX_MIB_SIZE]; + unsigned char *row_dering, *prev_row_dering, *curr_row_dering; + int dering_count; + int dir[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS] = { { 0 } }; + int var[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS] = { { 0 } }; + int stride; + int mi_wide_l2[3]; + int mi_high_l2[3]; + int xdec[3]; + int ydec[3]; + int pli; + int dering_left; + int coeff_shift = AOMMAX(cm->bit_depth - 8, 0); + int nplanes = 3; + int chroma_dering = + xd->plane[1].subsampling_x == xd->plane[1].subsampling_y && + xd->plane[2].subsampling_x == xd->plane[2].subsampling_y; + nvsb = (cm->mi_rows + MAX_MIB_SIZE - 1) / MAX_MIB_SIZE; + nhsb = (cm->mi_cols + MAX_MIB_SIZE - 1) / MAX_MIB_SIZE; + av1_setup_dst_planes(xd->plane, cm->sb_size, frame, 0, 0); + row_dering = aom_malloc(sizeof(*row_dering) * (nhsb + 2) * 2); + memset(row_dering, 1, sizeof(*row_dering) * (nhsb + 2) * 2); + prev_row_dering = row_dering + 1; + curr_row_dering = prev_row_dering + nhsb + 2; + for (pli = 0; pli < nplanes; pli++) { + xdec[pli] = xd->plane[pli].subsampling_x; + ydec[pli] = xd->plane[pli].subsampling_y; + mi_wide_l2[pli] = MI_SIZE_LOG2 - xd->plane[pli].subsampling_x; + mi_high_l2[pli] = MI_SIZE_LOG2 - xd->plane[pli].subsampling_y; + } + stride = (cm->mi_cols << MI_SIZE_LOG2) + 2 * OD_FILT_HBORDER; + for (pli = 0; pli < nplanes; pli++) { + linebuf[pli] = aom_malloc(sizeof(*linebuf) * OD_FILT_VBORDER * stride); + colbuf[pli] = + aom_malloc(sizeof(*colbuf) * + ((MAX_SB_SIZE << mi_high_l2[pli]) + 2 * OD_FILT_VBORDER) * + OD_FILT_HBORDER); + } + for (sbr = 0; sbr < nvsb; sbr++) { + for (pli = 0; pli < nplanes; pli++) { + const int block_height = + (MAX_MIB_SIZE << mi_high_l2[pli]) + 2 * OD_FILT_VBORDER; + fill_rect(colbuf[pli], OD_FILT_HBORDER, block_height, OD_FILT_HBORDER, + OD_DERING_VERY_LARGE); + } + dering_left = 1; + for (sbc = 0; sbc < nhsb; sbc++) { + int level, clpf_strength; + int uv_level, uv_clpf_strength; + int nhb, nvb; + int cstart = 0; + curr_row_dering[sbc] = 0; + if (cm->mi_grid_visible[MAX_MIB_SIZE * sbr * cm->mi_stride + + MAX_MIB_SIZE * sbc] == NULL || + cm->mi_grid_visible[MAX_MIB_SIZE * sbr * cm->mi_stride + + MAX_MIB_SIZE * sbc] + ->mbmi.cdef_strength == -1) { + dering_left = 0; + continue; + } + if (!dering_left) cstart = -OD_FILT_HBORDER; + nhb = AOMMIN(MAX_MIB_SIZE, cm->mi_cols - MAX_MIB_SIZE * sbc); + nvb = AOMMIN(MAX_MIB_SIZE, cm->mi_rows - MAX_MIB_SIZE * sbr); + int tile_top, tile_left, tile_bottom, tile_right; + int mi_idx = MAX_MIB_SIZE * sbr * cm->mi_stride + MAX_MIB_SIZE * sbc; + BOUNDARY_TYPE boundary_tl = + cm->mi_grid_visible[MAX_MIB_SIZE * sbr * cm->mi_stride + + MAX_MIB_SIZE * sbc] + ->mbmi.boundary_info; + tile_top = boundary_tl & TILE_ABOVE_BOUNDARY; + tile_left = boundary_tl & TILE_LEFT_BOUNDARY; + /* Right and bottom information appear unreliable, so we use the top + and left flags for the next superblocks. */ + if (sbr != nvsb - 1 && + cm->mi_grid_visible[mi_idx + MAX_MIB_SIZE * cm->mi_stride]) + tile_bottom = cm->mi_grid_visible[mi_idx + MAX_MIB_SIZE * cm->mi_stride] + ->mbmi.boundary_info & + TILE_ABOVE_BOUNDARY; + else + tile_bottom = 1; + if (sbc != nhsb - 1 && cm->mi_grid_visible[mi_idx + MAX_MIB_SIZE]) + tile_right = + cm->mi_grid_visible[mi_idx + MAX_MIB_SIZE]->mbmi.boundary_info & + TILE_LEFT_BOUNDARY; + else + tile_right = 1; + const int mbmi_cdef_strength = + cm->mi_grid_visible[MAX_MIB_SIZE * sbr * cm->mi_stride + + MAX_MIB_SIZE * sbc] + ->mbmi.cdef_strength; + level = cm->cdef_strengths[mbmi_cdef_strength] / CLPF_STRENGTHS; + clpf_strength = cm->cdef_strengths[mbmi_cdef_strength] % CLPF_STRENGTHS; + clpf_strength += clpf_strength == 3; + uv_level = cm->cdef_uv_strengths[mbmi_cdef_strength] / CLPF_STRENGTHS; + uv_clpf_strength = + cm->cdef_uv_strengths[mbmi_cdef_strength] % CLPF_STRENGTHS; + uv_clpf_strength += uv_clpf_strength == 3; + if ((level == 0 && clpf_strength == 0 && uv_level == 0 && + uv_clpf_strength == 0) || + (dering_count = sb_compute_dering_list( + cm, sbr * MAX_MIB_SIZE, sbc * MAX_MIB_SIZE, dlist, + get_filter_skip(level) || get_filter_skip(uv_level))) == 0) { + dering_left = 0; + continue; + } + + curr_row_dering[sbc] = 1; + for (pli = 0; pli < nplanes; pli++) { + uint16_t dst[MAX_SB_SIZE * MAX_SB_SIZE]; + int coffset; + int rend, cend; + int clpf_damping = cm->cdef_clpf_damping; + int dering_damping = cm->cdef_dering_damping; + int hsize = nhb << mi_wide_l2[pli]; + int vsize = nvb << mi_high_l2[pli]; + + if (pli) { + if (chroma_dering) + level = uv_level; + else + level = 0; + clpf_strength = uv_clpf_strength; + } + + if (sbc == nhsb - 1) + cend = hsize; + else + cend = hsize + OD_FILT_HBORDER; + + if (sbr == nvsb - 1) + rend = vsize; + else + rend = vsize + OD_FILT_VBORDER; + + coffset = sbc * MAX_MIB_SIZE << mi_wide_l2[pli]; + if (sbc == nhsb - 1) { + /* On the last superblock column, fill in the right border with + OD_DERING_VERY_LARGE to avoid filtering with the outside. */ + fill_rect(&src[cend + OD_FILT_HBORDER], OD_FILT_BSTRIDE, + rend + OD_FILT_VBORDER, hsize + OD_FILT_HBORDER - cend, + OD_DERING_VERY_LARGE); + } + if (sbr == nvsb - 1) { + /* On the last superblock row, fill in the bottom border with + OD_DERING_VERY_LARGE to avoid filtering with the outside. */ + fill_rect(&src[(rend + OD_FILT_VBORDER) * OD_FILT_BSTRIDE], + OD_FILT_BSTRIDE, OD_FILT_VBORDER, + hsize + 2 * OD_FILT_HBORDER, OD_DERING_VERY_LARGE); + } + /* Copy in the pixels we need from the current superblock for + deringing.*/ + copy_sb8_16( + cm, + &src[OD_FILT_VBORDER * OD_FILT_BSTRIDE + OD_FILT_HBORDER + cstart], + OD_FILT_BSTRIDE, xd->plane[pli].dst.buf, + (MAX_MIB_SIZE << mi_high_l2[pli]) * sbr, coffset + cstart, + xd->plane[pli].dst.stride, rend, cend - cstart); + if (!prev_row_dering[sbc]) { + copy_sb8_16(cm, &src[OD_FILT_HBORDER], OD_FILT_BSTRIDE, + xd->plane[pli].dst.buf, + (MAX_MIB_SIZE << mi_high_l2[pli]) * sbr - OD_FILT_VBORDER, + coffset, xd->plane[pli].dst.stride, OD_FILT_VBORDER, + hsize); + } else if (sbr > 0) { + copy_rect(&src[OD_FILT_HBORDER], OD_FILT_BSTRIDE, + &linebuf[pli][coffset], stride, OD_FILT_VBORDER, hsize); + } else { + fill_rect(&src[OD_FILT_HBORDER], OD_FILT_BSTRIDE, OD_FILT_VBORDER, + hsize, OD_DERING_VERY_LARGE); + } + if (!prev_row_dering[sbc - 1]) { + copy_sb8_16(cm, src, OD_FILT_BSTRIDE, xd->plane[pli].dst.buf, + (MAX_MIB_SIZE << mi_high_l2[pli]) * sbr - OD_FILT_VBORDER, + coffset - OD_FILT_HBORDER, xd->plane[pli].dst.stride, + OD_FILT_VBORDER, OD_FILT_HBORDER); + } else if (sbr > 0 && sbc > 0) { + copy_rect(src, OD_FILT_BSTRIDE, + &linebuf[pli][coffset - OD_FILT_HBORDER], stride, + OD_FILT_VBORDER, OD_FILT_HBORDER); + } else { + fill_rect(src, OD_FILT_BSTRIDE, OD_FILT_VBORDER, OD_FILT_HBORDER, + OD_DERING_VERY_LARGE); + } + if (!prev_row_dering[sbc + 1]) { + copy_sb8_16(cm, &src[OD_FILT_HBORDER + (nhb << mi_wide_l2[pli])], + OD_FILT_BSTRIDE, xd->plane[pli].dst.buf, + (MAX_MIB_SIZE << mi_high_l2[pli]) * sbr - OD_FILT_VBORDER, + coffset + hsize, xd->plane[pli].dst.stride, + OD_FILT_VBORDER, OD_FILT_HBORDER); + } else if (sbr > 0 && sbc < nhsb - 1) { + copy_rect(&src[hsize + OD_FILT_HBORDER], OD_FILT_BSTRIDE, + &linebuf[pli][coffset + hsize], stride, OD_FILT_VBORDER, + OD_FILT_HBORDER); + } else { + fill_rect(&src[hsize + OD_FILT_HBORDER], OD_FILT_BSTRIDE, + OD_FILT_VBORDER, OD_FILT_HBORDER, OD_DERING_VERY_LARGE); + } + if (dering_left) { + /* If we deringed the superblock on the left then we need to copy in + saved pixels. */ + copy_rect(src, OD_FILT_BSTRIDE, colbuf[pli], OD_FILT_HBORDER, + rend + OD_FILT_VBORDER, OD_FILT_HBORDER); + } + /* Saving pixels in case we need to dering the superblock on the + right. */ + copy_rect(colbuf[pli], OD_FILT_HBORDER, src + hsize, OD_FILT_BSTRIDE, + rend + OD_FILT_VBORDER, OD_FILT_HBORDER); + copy_sb8_16( + cm, &linebuf[pli][coffset], stride, xd->plane[pli].dst.buf, + (MAX_MIB_SIZE << mi_high_l2[pli]) * (sbr + 1) - OD_FILT_VBORDER, + coffset, xd->plane[pli].dst.stride, OD_FILT_VBORDER, hsize); + + if (tile_top) { + fill_rect(src, OD_FILT_BSTRIDE, OD_FILT_VBORDER, + hsize + 2 * OD_FILT_HBORDER, OD_DERING_VERY_LARGE); + } + if (tile_left) { + fill_rect(src, OD_FILT_BSTRIDE, vsize + 2 * OD_FILT_VBORDER, + OD_FILT_HBORDER, OD_DERING_VERY_LARGE); + } + if (tile_bottom) { + fill_rect(&src[(vsize + OD_FILT_VBORDER) * OD_FILT_BSTRIDE], + OD_FILT_BSTRIDE, OD_FILT_VBORDER, + hsize + 2 * OD_FILT_HBORDER, OD_DERING_VERY_LARGE); + } + if (tile_right) { + fill_rect(&src[hsize + OD_FILT_HBORDER], OD_FILT_BSTRIDE, + vsize + 2 * OD_FILT_VBORDER, OD_FILT_HBORDER, + OD_DERING_VERY_LARGE); + } +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) { + od_dering( + (uint8_t *)&CONVERT_TO_SHORTPTR( + xd->plane[pli] + .dst.buf)[xd->plane[pli].dst.stride * + (MAX_MIB_SIZE * sbr << mi_high_l2[pli]) + + (sbc * MAX_MIB_SIZE << mi_wide_l2[pli])], + xd->plane[pli].dst.stride, dst, + &src[OD_FILT_VBORDER * OD_FILT_BSTRIDE + OD_FILT_HBORDER], + xdec[pli], ydec[pli], dir, NULL, var, pli, dlist, dering_count, + level, clpf_strength, clpf_damping, dering_damping, coeff_shift, + 0, 1); + } else { +#endif + od_dering(&xd->plane[pli] + .dst.buf[xd->plane[pli].dst.stride * + (MAX_MIB_SIZE * sbr << mi_high_l2[pli]) + + (sbc * MAX_MIB_SIZE << mi_wide_l2[pli])], + xd->plane[pli].dst.stride, dst, + &src[OD_FILT_VBORDER * OD_FILT_BSTRIDE + OD_FILT_HBORDER], + xdec[pli], ydec[pli], dir, NULL, var, pli, dlist, + dering_count, level, clpf_strength, clpf_damping, + dering_damping, coeff_shift, 0, 0); + +#if CONFIG_HIGHBITDEPTH + } +#endif + } + dering_left = 1; + } + { + unsigned char *tmp; + tmp = prev_row_dering; + prev_row_dering = curr_row_dering; + curr_row_dering = tmp; + } + } + aom_free(row_dering); + for (pli = 0; pli < nplanes; pli++) { + aom_free(linebuf[pli]); + aom_free(colbuf[pli]); + } +} diff --git a/third_party/aom/av1/common/cdef.h b/third_party/aom/av1/common/cdef.h new file mode 100644 index 0000000000..08c438de6e --- /dev/null +++ b/third_party/aom/av1/common/cdef.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AV1_COMMON_DERING_H_ +#define AV1_COMMON_DERING_H_ + +#define CDEF_STRENGTH_BITS 7 + +#define DERING_STRENGTHS 32 +#define CLPF_STRENGTHS 4 + +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" +#include "av1/common/od_dering.h" +#include "av1/common/onyxc_int.h" +#include "./od_dering.h" + +static INLINE int sign(int i) { return i < 0 ? -1 : 1; } + +static INLINE int constrain(int diff, int threshold, unsigned int damping) { + return threshold + ? sign(diff) * + AOMMIN( + abs(diff), + AOMMAX(0, threshold - (abs(diff) >> + (damping - get_msb(threshold))))) + : 0; +} + +#ifdef __cplusplus +extern "C" { +#endif + +int sb_all_skip(const AV1_COMMON *const cm, int mi_row, int mi_col); +int sb_compute_dering_list(const AV1_COMMON *const cm, int mi_row, int mi_col, + dering_list *dlist, int filter_skip); +void av1_cdef_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, MACROBLOCKD *xd); + +void av1_cdef_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref, + AV1_COMMON *cm, MACROBLOCKD *xd); + +#ifdef __cplusplus +} // extern "C" +#endif +#endif // AV1_COMMON_DERING_H_ diff --git a/third_party/aom/av1/common/cdef_simd.h b/third_party/aom/av1/common/cdef_simd.h new file mode 100644 index 0000000000..2649099a21 --- /dev/null +++ b/third_party/aom/av1/common/cdef_simd.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AV1_COMMON_CDEF_SIMD_H_ +#define AV1_COMMON_CDEF_SIMD_H_ + +#include "aom_dsp/aom_simd.h" + +// sign(a-b) * min(abs(a-b), max(0, threshold - (abs(a-b) >> adjdamp))) +SIMD_INLINE v128 constrain16(v128 a, v128 b, unsigned int threshold, + unsigned int adjdamp) { + v128 diff = v128_sub_16(a, b); + const v128 sign = v128_shr_n_s16(diff, 15); + diff = v128_abs_s16(diff); + const v128 s = + v128_ssub_u16(v128_dup_16(threshold), v128_shr_u16(diff, adjdamp)); + return v128_xor(v128_add_16(sign, v128_min_s16(diff, s)), sign); +} + +#endif // AV1_COMMON_CDEF_SIMD_H_ diff --git a/third_party/aom/av1/common/cfl.c b/third_party/aom/av1/common/cfl.c new file mode 100644 index 0000000000..d66a989ad6 --- /dev/null +++ b/third_party/aom/av1/common/cfl.c @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/common/cfl.h" +#include "av1/common/common_data.h" +#include "av1/common/onyxc_int.h" + +#include "aom/internal/aom_codec_internal.h" + +void cfl_init(CFL_CTX *cfl, AV1_COMMON *cm, int subsampling_x, + int subsampling_y) { + if (!((subsampling_x == 0 && subsampling_y == 0) || + (subsampling_x == 1 && subsampling_y == 1))) { + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "Only 4:4:4 and 4:2:0 are currently supported by CfL"); + } + memset(&cfl->y_pix, 0, sizeof(uint8_t) * MAX_SB_SQUARE); + cfl->subsampling_x = subsampling_x; + cfl->subsampling_y = subsampling_y; +} + +// CfL computes its own block-level DC_PRED. This is required to compute both +// alpha_cb and alpha_cr before the prediction are computed. +void cfl_dc_pred(MACROBLOCKD *xd, BLOCK_SIZE plane_bsize, TX_SIZE tx_size) { + const struct macroblockd_plane *const pd_u = &xd->plane[AOM_PLANE_U]; + const struct macroblockd_plane *const pd_v = &xd->plane[AOM_PLANE_V]; + + const uint8_t *const dst_u = pd_u->dst.buf; + const uint8_t *const dst_v = pd_v->dst.buf; + + const int dst_u_stride = pd_u->dst.stride; + const int dst_v_stride = pd_v->dst.stride; + + const int block_width = (plane_bsize != BLOCK_INVALID) + ? block_size_wide[plane_bsize] + : tx_size_wide[tx_size]; + const int block_height = (plane_bsize != BLOCK_INVALID) + ? block_size_high[plane_bsize] + : tx_size_high[tx_size]; + + // Number of pixel on the top and left borders. + const int num_pel = block_width + block_height; + + int sum_u = 0; + int sum_v = 0; + + // Match behavior of build_intra_predictors (reconintra.c) at superblock + // boundaries: + // + // 127 127 127 .. 127 127 127 127 127 127 + // 129 A B .. Y Z + // 129 C D .. W X + // 129 E F .. U V + // 129 G H .. S T T T T T + // .. + + // TODO(ltrudeau) replace this with DC_PRED assembly + if (xd->up_available && xd->mb_to_right_edge >= 0) { + for (int i = 0; i < block_width; i++) { + sum_u += dst_u[-dst_u_stride + i]; + sum_v += dst_v[-dst_v_stride + i]; + } + } else { + sum_u = block_width * 127; + sum_v = block_width * 127; + } + + if (xd->left_available && xd->mb_to_bottom_edge >= 0) { + for (int i = 0; i < block_height; i++) { + sum_u += dst_u[i * dst_u_stride - 1]; + sum_v += dst_v[i * dst_v_stride - 1]; + } + } else { + sum_u += block_height * 129; + sum_v += block_height * 129; + } + + xd->cfl->dc_pred[CFL_PRED_U] = (sum_u + (num_pel >> 1)) / num_pel; + xd->cfl->dc_pred[CFL_PRED_V] = (sum_v + (num_pel >> 1)) / num_pel; +} + +// Predict the current transform block using CfL. +// it is assumed that dst points at the start of the transform block +void cfl_predict_block(const CFL_CTX *cfl, uint8_t *dst, int dst_stride, + int row, int col, TX_SIZE tx_size, int dc_pred) { + const int tx_block_width = tx_size_wide[tx_size]; + const int tx_block_height = tx_size_high[tx_size]; + + // TODO(ltrudeau) implement alpha + // Place holder for alpha + const double alpha = 0; + const double y_avg = cfl_load(cfl, dst, dst_stride, row, col, tx_size); + + for (int j = 0; j < tx_block_height; j++) { + for (int i = 0; i < tx_block_width; i++) { + dst[i] = (uint8_t)(alpha * y_avg + dc_pred + 0.5); + } + dst += dst_stride; + } +} + +void cfl_store(CFL_CTX *cfl, const uint8_t *input, int input_stride, int row, + int col, TX_SIZE tx_size) { + const int tx_width = tx_size_wide[tx_size]; + const int tx_height = tx_size_high[tx_size]; + const int tx_off_log2 = tx_size_wide_log2[0]; + + // Store the input into the CfL pixel buffer + uint8_t *y_pix = &cfl->y_pix[(row * MAX_SB_SIZE + col) << tx_off_log2]; + + // Check that we remain inside the pixel buffer. + assert(MAX_SB_SIZE * (row + tx_height - 1) + col + tx_width - 1 < + MAX_SB_SQUARE); + + for (int j = 0; j < tx_height; j++) { + for (int i = 0; i < tx_width; i++) { + y_pix[i] = input[i]; + } + y_pix += MAX_SB_SIZE; + input += input_stride; + } + + // Store the surface of the pixel buffer that was written to, this way we + // can manage chroma overrun (e.g. when the chroma surfaces goes beyond the + // frame boundary) + if (col == 0 && row == 0) { + cfl->y_width = tx_width; + cfl->y_height = tx_height; + } else { + cfl->y_width = OD_MAXI((col << tx_off_log2) + tx_width, cfl->y_width); + cfl->y_height = OD_MAXI((row << tx_off_log2) + tx_height, cfl->y_height); + } +} + +// Load from the CfL pixel buffer into output +double cfl_load(const CFL_CTX *cfl, uint8_t *output, int output_stride, int row, + int col, TX_SIZE tx_size) { + const int tx_width = tx_size_wide[tx_size]; + const int tx_height = tx_size_high[tx_size]; + const int sub_x = cfl->subsampling_x; + const int sub_y = cfl->subsampling_y; + const int tx_off_log2 = tx_size_wide_log2[0]; + + const uint8_t *y_pix; + + int diff_width = 0; + int diff_height = 0; + + int pred_row_offset = 0; + int output_row_offset = 0; + int top_left, bot_left; + + // TODO(ltrudeau) add support for 4:2:2 + if (sub_y == 0 && sub_x == 0) { + y_pix = &cfl->y_pix[(row * MAX_SB_SIZE + col) << tx_off_log2]; + int uv_width = (col << tx_off_log2) + tx_width; + diff_width = uv_width - cfl->y_width; + int uv_height = (row << tx_off_log2) + tx_width; + diff_height = uv_height - cfl->y_height; + for (int j = 0; j < tx_height; j++) { + for (int i = 0; i < tx_width; i++) { + // In 4:4:4, pixels match 1 to 1 + output[output_row_offset + i] = y_pix[pred_row_offset + i]; + } + pred_row_offset += MAX_SB_SIZE; + output_row_offset += output_stride; + } + } else if (sub_y == 1 && sub_x == 1) { + y_pix = &cfl->y_pix[(row * MAX_SB_SIZE + col) << (tx_off_log2 + sub_y)]; + int uv_width = ((col << tx_off_log2) + tx_width) << sub_x; + diff_width = (uv_width - cfl->y_width) >> sub_x; + int uv_height = ((row << tx_off_log2) + tx_width) << sub_y; + diff_height = (uv_height - cfl->y_height) >> sub_y; + for (int j = 0; j < tx_height; j++) { + for (int i = 0; i < tx_width; i++) { + top_left = (pred_row_offset + i) << sub_y; + bot_left = top_left + MAX_SB_SIZE; + // In 4:2:0, average pixels in 2x2 grid + output[output_row_offset + i] = OD_SHR_ROUND( + y_pix[top_left] + y_pix[top_left + 1] // Top row + + y_pix[bot_left] + y_pix[bot_left + 1] // Bottom row + , + 2); + } + pred_row_offset += MAX_SB_SIZE; + output_row_offset += output_stride; + } + } else { + assert(0); // Unsupported chroma subsampling + } + // Due to frame boundary issues, it is possible that the total area of + // covered by Chroma exceeds that of Luma. When this happens, we write over + // the broken data by repeating the last columns and/or rows. + // + // Note that in order to manage the case where both rows and columns + // overrun, + // we apply rows first. This way, when the rows overrun the bottom of the + // frame, the columns will be copied over them. + if (diff_width > 0) { + int last_pixel; + output_row_offset = tx_width - diff_width; + + for (int j = 0; j < tx_height; j++) { + last_pixel = output_row_offset - 1; + for (int i = 0; i < diff_width; i++) { + output[output_row_offset + i] = output[last_pixel]; + } + output_row_offset += output_stride; + } + } + + if (diff_height > 0) { + output_row_offset = diff_height * output_stride; + const int last_row_offset = output_row_offset - output_stride; + for (int j = 0; j < diff_height; j++) { + for (int i = 0; i < tx_width; i++) { + output[output_row_offset + i] = output[last_row_offset + i]; + } + output_row_offset += output_stride; + } + } + + int avg = 0; + output_row_offset = 0; + for (int j = 0; j < tx_height; j++) { + for (int i = 0; i < tx_width; i++) { + avg += output[output_row_offset + i]; + } + output_row_offset += output_stride; + } + return avg / (double)(tx_width * tx_height); +} diff --git a/third_party/aom/av1/common/cfl.h b/third_party/aom/av1/common/cfl.h new file mode 100644 index 0000000000..371df70bea --- /dev/null +++ b/third_party/aom/av1/common/cfl.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_CFL_H_ +#define AV1_COMMON_CFL_H_ + +#include "av1/common/enums.h" + +// Forward declaration of AV1_COMMON, in order to avoid creating a cyclic +// dependency by importing av1/common/onyxc_int.h +typedef struct AV1Common AV1_COMMON; + +// Forward declaration of MACROBLOCK, in order to avoid creating a cyclic +// dependency by importing av1/common/blockd.h +typedef struct macroblockd MACROBLOCKD; + +typedef struct { + // Pixel buffer containing the luma pixels used as prediction for chroma + uint8_t y_pix[MAX_SB_SQUARE]; + + // Height and width of the luma prediction block currently in the pixel buffer + int y_height, y_width; + + // Chroma subsampling + int subsampling_x, subsampling_y; + + // CfL Performs its own block level DC_PRED for each chromatic plane + int dc_pred[CFL_PRED_PLANES]; +} CFL_CTX; + +void cfl_init(CFL_CTX *cfl, AV1_COMMON *cm, int subsampling_x, + int subsampling_y); + +void cfl_dc_pred(MACROBLOCKD *xd, BLOCK_SIZE plane_bsize, TX_SIZE tx_size); + +void cfl_predict_block(const CFL_CTX *cfl, uint8_t *dst, int dst_stride, + int row, int col, TX_SIZE tx_size, int dc_pred); + +void cfl_store(CFL_CTX *cfl, const uint8_t *input, int input_stride, int row, + int col, TX_SIZE tx_size); + +double cfl_load(const CFL_CTX *cfl, uint8_t *output, int output_stride, int row, + int col, TX_SIZE tx_size); +#endif // AV1_COMMON_CFL_H_ diff --git a/third_party/aom/av1/common/clpf.c b/third_party/aom/av1/common/clpf.c new file mode 100644 index 0000000000..3637deeea9 --- /dev/null +++ b/third_party/aom/av1/common/clpf.c @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./clpf.h" +#include "./av1_rtcd.h" +#include "./cdef.h" +#include "aom/aom_image.h" +#include "aom_dsp/aom_dsp_common.h" + +int av1_clpf_sample(int X, int A, int B, int C, int D, int E, int F, int G, + int H, int s, unsigned int dmp) { + int delta = 1 * constrain(A - X, s, dmp) + 3 * constrain(B - X, s, dmp) + + 1 * constrain(C - X, s, dmp) + 3 * constrain(D - X, s, dmp) + + 3 * constrain(E - X, s, dmp) + 1 * constrain(F - X, s, dmp) + + 3 * constrain(G - X, s, dmp) + 1 * constrain(H - X, s, dmp); + return (8 + delta - (delta < 0)) >> 4; +} + +int av1_clpf_hsample(int X, int A, int B, int C, int D, int s, + unsigned int dmp) { + int delta = 1 * constrain(A - X, s, dmp) + 3 * constrain(B - X, s, dmp) + + 3 * constrain(C - X, s, dmp) + 1 * constrain(D - X, s, dmp); + return (4 + delta - (delta < 0)) >> 3; +} + +void aom_clpf_block_c(uint8_t *dst, const uint16_t *src, int dstride, + int sstride, int sizex, int sizey, unsigned int strength, + unsigned int damping) { + int x, y; + + for (y = 0; y < sizey; y++) { + for (x = 0; x < sizex; x++) { + const int X = src[y * sstride + x]; + const int A = src[(y - 2) * sstride + x]; + const int B = src[(y - 1) * sstride + x]; + const int C = src[y * sstride + x - 2]; + const int D = src[y * sstride + x - 1]; + const int E = src[y * sstride + x + 1]; + const int F = src[y * sstride + x + 2]; + const int G = src[(y + 1) * sstride + x]; + const int H = src[(y + 2) * sstride + x]; + const int delta = + av1_clpf_sample(X, A, B, C, D, E, F, G, H, strength, damping); + dst[y * dstride + x] = X + delta; + } + } +} + +// Identical to aom_clpf_block_c() apart from "dst". +void aom_clpf_block_hbd_c(uint16_t *dst, const uint16_t *src, int dstride, + int sstride, int sizex, int sizey, + unsigned int strength, unsigned int damping) { + int x, y; + + for (y = 0; y < sizey; y++) { + for (x = 0; x < sizex; x++) { + const int X = src[y * sstride + x]; + const int A = src[(y - 2) * sstride + x]; + const int B = src[(y - 1) * sstride + x]; + const int C = src[y * sstride + x - 2]; + const int D = src[y * sstride + x - 1]; + const int E = src[y * sstride + x + 1]; + const int F = src[y * sstride + x + 2]; + const int G = src[(y + 1) * sstride + x]; + const int H = src[(y + 2) * sstride + x]; + const int delta = + av1_clpf_sample(X, A, B, C, D, E, F, G, H, strength, damping); + dst[y * dstride + x] = X + delta; + } + } +} + +// Vertically restricted filter +void aom_clpf_hblock_c(uint8_t *dst, const uint16_t *src, int dstride, + int sstride, int sizex, int sizey, unsigned int strength, + unsigned int damping) { + int x, y; + + for (y = 0; y < sizey; y++) { + for (x = 0; x < sizex; x++) { + const int X = src[y * sstride + x]; + const int A = src[y * sstride + x - 2]; + const int B = src[y * sstride + x - 1]; + const int C = src[y * sstride + x + 1]; + const int D = src[y * sstride + x + 2]; + const int delta = av1_clpf_hsample(X, A, B, C, D, strength, damping); + dst[y * dstride + x] = X + delta; + } + } +} + +void aom_clpf_hblock_hbd_c(uint16_t *dst, const uint16_t *src, int dstride, + int sstride, int sizex, int sizey, + unsigned int strength, unsigned int damping) { + int x, y; + + for (y = 0; y < sizey; y++) { + for (x = 0; x < sizex; x++) { + const int X = src[y * sstride + x]; + const int A = src[y * sstride + x - 2]; + const int B = src[y * sstride + x - 1]; + const int C = src[y * sstride + x + 1]; + const int D = src[y * sstride + x + 2]; + const int delta = av1_clpf_hsample(X, A, B, C, D, strength, damping); + dst[y * dstride + x] = X + delta; + } + } +} diff --git a/third_party/aom/av1/common/clpf.h b/third_party/aom/av1/common/clpf.h new file mode 100644 index 0000000000..d6348deb0f --- /dev/null +++ b/third_party/aom/av1/common/clpf.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AV1_COMMON_CLPF_H_ +#define AV1_COMMON_CLPF_H_ + +#include "av1/common/reconinter.h" + +int av1_clpf_sample(int X, int A, int B, int C, int D, int E, int F, int G, + int H, int b, unsigned int dmp); +#endif diff --git a/third_party/aom/av1/common/clpf_neon.c b/third_party/aom/av1/common/clpf_neon.c new file mode 100644 index 0000000000..f1a004c2c6 --- /dev/null +++ b/third_party/aom/av1/common/clpf_neon.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/aom_simd.h" +#define SIMD_FUNC(name) name##_neon +#include "./clpf_simd.h" diff --git a/third_party/aom/av1/common/clpf_simd.h b/third_party/aom/av1/common/clpf_simd.h new file mode 100644 index 0000000000..a615b5ed38 --- /dev/null +++ b/third_party/aom/av1/common/clpf_simd.h @@ -0,0 +1,446 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./av1_rtcd.h" +#include "./cdef_simd.h" +#include "aom_ports/bitops.h" +#include "aom_ports/mem.h" + +// sign(a - b) * min(abs(a - b), max(0, strength - (abs(a - b) >> adjdamp))) +SIMD_INLINE v128 constrain(v256 a, v256 b, unsigned int strength, + unsigned int adjdamp) { + const v256 diff16 = v256_sub_16(a, b); + v128 diff = v128_pack_s16_s8(v256_high_v128(diff16), v256_low_v128(diff16)); + const v128 sign = v128_cmplt_s8(diff, v128_zero()); + diff = v128_abs_s8(diff); + return v128_xor( + v128_add_8(sign, + v128_min_u8(diff, v128_ssub_u8(v128_dup_8(strength), + v128_shr_u8(diff, adjdamp)))), + sign); +} + +// delta = 1/16 * constrain(a, x, s, d) + 3/16 * constrain(b, x, s, d) + +// 1/16 * constrain(c, x, s, d) + 3/16 * constrain(d, x, s, d) + +// 3/16 * constrain(e, x, s, d) + 1/16 * constrain(f, x, s, d) + +// 3/16 * constrain(g, x, s, d) + 1/16 * constrain(h, x, s, d) +SIMD_INLINE v128 calc_delta(v256 x, v256 a, v256 b, v256 c, v256 d, v256 e, + v256 f, v256 g, v256 h, unsigned int s, + unsigned int dmp) { + const v128 bdeg = + v128_add_8(v128_add_8(constrain(b, x, s, dmp), constrain(d, x, s, dmp)), + v128_add_8(constrain(e, x, s, dmp), constrain(g, x, s, dmp))); + const v128 delta = v128_add_8( + v128_add_8(v128_add_8(constrain(a, x, s, dmp), constrain(c, x, s, dmp)), + v128_add_8(constrain(f, x, s, dmp), constrain(h, x, s, dmp))), + v128_add_8(v128_add_8(bdeg, bdeg), bdeg)); + return v128_add_8( + v128_pack_s16_u8(v256_high_v128(x), v256_low_v128(x)), + v128_shr_s8( + v128_add_8(v128_dup_8(8), + v128_add_8(delta, v128_cmplt_s8(delta, v128_zero()))), + 4)); +} + +// delta = 1/8 * constrain(a, x, s, d) + 3/8 * constrain(b, x, s, d) + +// 3/8 * constrain(c, x, s, d) + 1/8 * constrain(d, x, s, d) + +SIMD_INLINE v128 calc_hdelta(v256 x, v256 a, v256 b, v256 c, v256 d, + unsigned int s, unsigned int dmp) { + const v128 bc = v128_add_8(constrain(b, x, s, dmp), constrain(c, x, s, dmp)); + const v128 delta = + v128_add_8(v128_add_8(constrain(a, x, s, dmp), constrain(d, x, s, dmp)), + v128_add_8(v128_add_8(bc, bc), bc)); + return v128_add_8( + v128_pack_s16_u8(v256_high_v128(x), v256_low_v128(x)), + v128_shr_s8( + v128_add_8(v128_dup_8(4), + v128_add_8(delta, v128_cmplt_s8(delta, v128_zero()))), + 3)); +} + +// Process blocks of width 8, two lines at a time, 8 bit. +static void SIMD_FUNC(clpf_block8)(uint8_t *dst, const uint16_t *src, + int dstride, int sstride, int sizey, + unsigned int strength, + unsigned int adjdamp) { + int y; + + for (y = 0; y < sizey; y += 2) { + const v128 l1 = v128_load_aligned(src); + const v128 l2 = v128_load_aligned(src + sstride); + const v128 l3 = v128_load_aligned(src - sstride); + const v128 l4 = v128_load_aligned(src + 2 * sstride); + const v256 a = v256_from_v128(v128_load_aligned(src - 2 * sstride), l3); + const v256 b = v256_from_v128(l3, l1); + const v256 g = v256_from_v128(l2, l4); + const v256 h = v256_from_v128(l4, v128_load_aligned(src + 3 * sstride)); + const v256 c = v256_from_v128(v128_load_unaligned(src - 2), + v128_load_unaligned(src - 2 + sstride)); + const v256 d = v256_from_v128(v128_load_unaligned(src - 1), + v128_load_unaligned(src - 1 + sstride)); + const v256 e = v256_from_v128(v128_load_unaligned(src + 1), + v128_load_unaligned(src + 1 + sstride)); + const v256 f = v256_from_v128(v128_load_unaligned(src + 2), + v128_load_unaligned(src + 2 + sstride)); + const v128 o = calc_delta(v256_from_v128(l1, l2), a, b, c, d, e, f, g, h, + strength, adjdamp); + + v64_store_aligned(dst, v128_high_v64(o)); + v64_store_aligned(dst + dstride, v128_low_v64(o)); + src += sstride * 2; + dst += dstride * 2; + } +} + +// Process blocks of width 4, four lines at a time, 8 bit. +static void SIMD_FUNC(clpf_block4)(uint8_t *dst, const uint16_t *src, + int dstride, int sstride, int sizey, + unsigned int strength, + unsigned int adjdamp) { + int y; + + for (y = 0; y < sizey; y += 4) { + const v64 l0 = v64_load_aligned(src - 2 * sstride); + const v64 l1 = v64_load_aligned(src - sstride); + const v64 l2 = v64_load_aligned(src); + const v64 l3 = v64_load_aligned(src + sstride); + const v64 l4 = v64_load_aligned(src + 2 * sstride); + const v64 l5 = v64_load_aligned(src + 3 * sstride); + const v64 l6 = v64_load_aligned(src + 4 * sstride); + const v64 l7 = v64_load_aligned(src + 5 * sstride); + const v128 o = + calc_delta(v256_from_v64(l2, l3, l4, l5), v256_from_v64(l0, l1, l2, l3), + v256_from_v64(l1, l2, l3, l4), + v256_from_v64(v64_load_unaligned(src - 2), + v64_load_unaligned(src + sstride - 2), + v64_load_unaligned(src + 2 * sstride - 2), + v64_load_unaligned(src + 3 * sstride - 2)), + v256_from_v64(v64_load_unaligned(src - 1), + v64_load_unaligned(src + sstride - 1), + v64_load_unaligned(src + 2 * sstride - 1), + v64_load_unaligned(src + 3 * sstride - 1)), + v256_from_v64(v64_load_unaligned(src + 1), + v64_load_unaligned(src + sstride + 1), + v64_load_unaligned(src + 2 * sstride + 1), + v64_load_unaligned(src + 3 * sstride + 1)), + v256_from_v64(v64_load_unaligned(src + 2), + v64_load_unaligned(src + sstride + 2), + v64_load_unaligned(src + 2 * sstride + 2), + v64_load_unaligned(src + 3 * sstride + 2)), + v256_from_v64(l3, l4, l5, l6), v256_from_v64(l4, l5, l6, l7), + strength, adjdamp); + + u32_store_aligned(dst, v128_low_u32(v128_shr_n_byte(o, 12))); + u32_store_aligned(dst + dstride, v128_low_u32(v128_shr_n_byte(o, 8))); + u32_store_aligned(dst + 2 * dstride, v128_low_u32(v128_shr_n_byte(o, 4))); + u32_store_aligned(dst + 3 * dstride, v128_low_u32(o)); + + dst += 4 * dstride; + src += 4 * sstride; + } +} + +static void SIMD_FUNC(clpf_hblock8)(uint8_t *dst, const uint16_t *src, + int dstride, int sstride, int sizey, + unsigned int strength, + unsigned int adjdamp) { + int y; + + for (y = 0; y < sizey; y += 2) { + const v256 x = v256_from_v128(v128_load_aligned(src), + v128_load_aligned(src + sstride)); + const v256 a = v256_from_v128(v128_load_unaligned(src - 2), + v128_load_unaligned(src - 2 + sstride)); + const v256 b = v256_from_v128(v128_load_unaligned(src - 1), + v128_load_unaligned(src - 1 + sstride)); + const v256 c = v256_from_v128(v128_load_unaligned(src + 1), + v128_load_unaligned(src + 1 + sstride)); + const v256 d = v256_from_v128(v128_load_unaligned(src + 2), + v128_load_unaligned(src + 2 + sstride)); + const v128 o = calc_hdelta(x, a, b, c, d, strength, adjdamp); + + v64_store_aligned(dst, v128_high_v64(o)); + v64_store_aligned(dst + dstride, v128_low_v64(o)); + src += sstride * 2; + dst += dstride * 2; + } +} + +// Process blocks of width 4, four lines at a time, 8 bit. +static void SIMD_FUNC(clpf_hblock4)(uint8_t *dst, const uint16_t *src, + int dstride, int sstride, int sizey, + unsigned int strength, + unsigned int adjdamp) { + int y; + + for (y = 0; y < sizey; y += 4) { + const v256 a = v256_from_v64(v64_load_unaligned(src - 2), + v64_load_unaligned(src + sstride - 2), + v64_load_unaligned(src + 2 * sstride - 2), + v64_load_unaligned(src + 3 * sstride - 2)); + const v256 b = v256_from_v64(v64_load_unaligned(src - 1), + v64_load_unaligned(src + sstride - 1), + v64_load_unaligned(src + 2 * sstride - 1), + v64_load_unaligned(src + 3 * sstride - 1)); + const v256 c = v256_from_v64(v64_load_unaligned(src + 1), + v64_load_unaligned(src + sstride + 1), + v64_load_unaligned(src + 2 * sstride + 1), + v64_load_unaligned(src + 3 * sstride + 1)); + const v256 d = v256_from_v64(v64_load_unaligned(src + 2), + v64_load_unaligned(src + sstride + 2), + v64_load_unaligned(src + 2 * sstride + 2), + v64_load_unaligned(src + 3 * sstride + 2)); + + const v128 o = calc_hdelta( + v256_from_v64(v64_load_aligned(src), v64_load_aligned(src + sstride), + v64_load_aligned(src + 2 * sstride), + v64_load_aligned(src + 3 * sstride)), + a, b, c, d, strength, adjdamp); + + u32_store_aligned(dst, v128_low_u32(v128_shr_n_byte(o, 12))); + u32_store_aligned(dst + dstride, v128_low_u32(v128_shr_n_byte(o, 8))); + u32_store_aligned(dst + 2 * dstride, v128_low_u32(v128_shr_n_byte(o, 4))); + u32_store_aligned(dst + 3 * dstride, v128_low_u32(o)); + + dst += 4 * dstride; + src += 4 * sstride; + } +} + +void SIMD_FUNC(aom_clpf_block)(uint8_t *dst, const uint16_t *src, int dstride, + int sstride, int sizex, int sizey, + unsigned int strength, unsigned int dmp) { + if ((sizex != 4 && sizex != 8) || ((sizey & 3) && sizex == 4)) { + // Fallback to C for odd sizes: + // * block widths not 4 or 8 + // * block heights not a multiple of 4 if the block width is 4 + aom_clpf_block_c(dst, src, dstride, sstride, sizex, sizey, strength, dmp); + } else { + (sizex == 4 ? SIMD_FUNC(clpf_block4) : SIMD_FUNC(clpf_block8))( + dst, src, dstride, sstride, sizey, strength, dmp - get_msb(strength)); + } +} + +void SIMD_FUNC(aom_clpf_hblock)(uint8_t *dst, const uint16_t *src, int dstride, + int sstride, int sizex, int sizey, + unsigned int strength, unsigned int dmp) { + if ((sizex != 4 && sizex != 8) || ((sizey & 3) && sizex == 4)) { + // Fallback to C for odd sizes: + // * block widths not 4 or 8 + // * block heights not a multiple of 4 if the block width is 4 + aom_clpf_hblock_c(dst, src, dstride, sstride, sizex, sizey, strength, dmp); + } else { + (sizex == 4 ? SIMD_FUNC(clpf_hblock4) : SIMD_FUNC(clpf_hblock8))( + dst, src, dstride, sstride, sizey, strength, dmp - get_msb(strength)); + } +} + +// delta = 1/16 * constrain(a, x, s, d) + 3/16 * constrain(b, x, s, d) + +// 1/16 * constrain(c, x, s, d) + 3/16 * constrain(d, x, s, d) + +// 3/16 * constrain(e, x, s, d) + 1/16 * constrain(f, x, s, d) + +// 3/16 * constrain(g, x, s, d) + 1/16 * constrain(h, x, s, d) +SIMD_INLINE v128 calc_delta_hbd(v128 x, v128 a, v128 b, v128 c, v128 d, v128 e, + v128 f, v128 g, v128 h, unsigned int s, + unsigned int dmp) { + const v128 bdeg = v128_add_16( + v128_add_16(constrain16(b, x, s, dmp), constrain16(d, x, s, dmp)), + v128_add_16(constrain16(e, x, s, dmp), constrain16(g, x, s, dmp))); + const v128 delta = v128_add_16( + v128_add_16( + v128_add_16(constrain16(a, x, s, dmp), constrain16(c, x, s, dmp)), + v128_add_16(constrain16(f, x, s, dmp), constrain16(h, x, s, dmp))), + v128_add_16(v128_add_16(bdeg, bdeg), bdeg)); + return v128_add_16( + x, + v128_shr_s16( + v128_add_16(v128_dup_16(8), + v128_add_16(delta, v128_cmplt_s16(delta, v128_zero()))), + 4)); +} + +static void calc_delta_hbd4(v128 o, v128 a, v128 b, v128 c, v128 d, v128 e, + v128 f, v128 g, v128 h, uint16_t *dst, + unsigned int s, unsigned int dmp, int dstride) { + o = calc_delta_hbd(o, a, b, c, d, e, f, g, h, s, dmp); + v64_store_aligned(dst, v128_high_v64(o)); + v64_store_aligned(dst + dstride, v128_low_v64(o)); +} + +static void calc_delta_hbd8(v128 o, v128 a, v128 b, v128 c, v128 d, v128 e, + v128 f, v128 g, v128 h, uint16_t *dst, + unsigned int s, unsigned int adjdamp) { + v128_store_aligned(dst, + calc_delta_hbd(o, a, b, c, d, e, f, g, h, s, adjdamp)); +} + +// delta = 1/16 * constrain(a, x, s, dmp) + 3/16 * constrain(b, x, s, dmp) + +// 3/16 * constrain(c, x, s, dmp) + 1/16 * constrain(d, x, s, dmp) +SIMD_INLINE v128 calc_hdelta_hbd(v128 x, v128 a, v128 b, v128 c, v128 d, + unsigned int s, unsigned int dmp) { + const v128 bc = + v128_add_16(constrain16(b, x, s, dmp), constrain16(c, x, s, dmp)); + const v128 delta = v128_add_16( + v128_add_16(constrain16(a, x, s, dmp), constrain16(d, x, s, dmp)), + v128_add_16(v128_add_16(bc, bc), bc)); + return v128_add_16( + x, + v128_shr_s16( + v128_add_16(v128_dup_16(4), + v128_add_16(delta, v128_cmplt_s16(delta, v128_zero()))), + 3)); +} + +static void calc_hdelta_hbd4(v128 o, v128 a, v128 b, v128 c, v128 d, + uint16_t *dst, unsigned int s, + unsigned int adjdamp, int dstride) { + o = calc_hdelta_hbd(o, a, b, c, d, s, adjdamp); + v64_store_aligned(dst, v128_high_v64(o)); + v64_store_aligned(dst + dstride, v128_low_v64(o)); +} + +static void calc_hdelta_hbd8(v128 o, v128 a, v128 b, v128 c, v128 d, + uint16_t *dst, unsigned int s, + unsigned int adjdamp) { + v128_store_aligned(dst, calc_hdelta_hbd(o, a, b, c, d, s, adjdamp)); +} + +// Process blocks of width 4, two lines at time. +static void SIMD_FUNC(clpf_block_hbd4)(uint16_t *dst, const uint16_t *src, + int dstride, int sstride, int sizey, + unsigned int strength, + unsigned int adjdamp) { + int y; + + for (y = 0; y < sizey; y += 2) { + const v64 l1 = v64_load_aligned(src); + const v64 l2 = v64_load_aligned(src + sstride); + const v64 l3 = v64_load_aligned(src - sstride); + const v64 l4 = v64_load_aligned(src + 2 * sstride); + const v128 a = v128_from_v64(v64_load_aligned(src - 2 * sstride), l3); + const v128 b = v128_from_v64(l3, l1); + const v128 g = v128_from_v64(l2, l4); + const v128 h = v128_from_v64(l4, v64_load_aligned(src + 3 * sstride)); + const v128 c = v128_from_v64(v64_load_unaligned(src - 2), + v64_load_unaligned(src - 2 + sstride)); + const v128 d = v128_from_v64(v64_load_unaligned(src - 1), + v64_load_unaligned(src - 1 + sstride)); + const v128 e = v128_from_v64(v64_load_unaligned(src + 1), + v64_load_unaligned(src + 1 + sstride)); + const v128 f = v128_from_v64(v64_load_unaligned(src + 2), + v64_load_unaligned(src + 2 + sstride)); + + calc_delta_hbd4(v128_from_v64(l1, l2), a, b, c, d, e, f, g, h, dst, + strength, adjdamp, dstride); + src += sstride * 2; + dst += dstride * 2; + } +} + +// The most simple case. Start here if you need to understand the functions. +static void SIMD_FUNC(clpf_block_hbd)(uint16_t *dst, const uint16_t *src, + int dstride, int sstride, int sizey, + unsigned int strength, + unsigned int adjdamp) { + int y; + + for (y = 0; y < sizey; y++) { + const v128 o = v128_load_aligned(src); + const v128 a = v128_load_aligned(src - 2 * sstride); + const v128 b = v128_load_aligned(src - 1 * sstride); + const v128 g = v128_load_aligned(src + sstride); + const v128 h = v128_load_aligned(src + 2 * sstride); + const v128 c = v128_load_unaligned(src - 2); + const v128 d = v128_load_unaligned(src - 1); + const v128 e = v128_load_unaligned(src + 1); + const v128 f = v128_load_unaligned(src + 2); + + calc_delta_hbd8(o, a, b, c, d, e, f, g, h, dst, strength, adjdamp); + src += sstride; + dst += dstride; + } +} + +// Process blocks of width 4, horizontal filter, two lines at time. +static void SIMD_FUNC(clpf_hblock_hbd4)(uint16_t *dst, const uint16_t *src, + int dstride, int sstride, int sizey, + unsigned int strength, + unsigned int adjdamp) { + int y; + + for (y = 0; y < sizey; y += 2) { + const v128 a = v128_from_v64(v64_load_unaligned(src - 2), + v64_load_unaligned(src - 2 + sstride)); + const v128 b = v128_from_v64(v64_load_unaligned(src - 1), + v64_load_unaligned(src - 1 + sstride)); + const v128 c = v128_from_v64(v64_load_unaligned(src + 1), + v64_load_unaligned(src + 1 + sstride)); + const v128 d = v128_from_v64(v64_load_unaligned(src + 2), + v64_load_unaligned(src + 2 + sstride)); + + calc_hdelta_hbd4(v128_from_v64(v64_load_unaligned(src), + v64_load_unaligned(src + sstride)), + a, b, c, d, dst, strength, adjdamp, dstride); + src += sstride * 2; + dst += dstride * 2; + } +} + +// Process blocks of width 8, horizontal filter, two lines at time. +static void SIMD_FUNC(clpf_hblock_hbd)(uint16_t *dst, const uint16_t *src, + int dstride, int sstride, int sizey, + unsigned int strength, + unsigned int adjdamp) { + int y; + + for (y = 0; y < sizey; y++) { + const v128 o = v128_load_aligned(src); + const v128 a = v128_load_unaligned(src - 2); + const v128 b = v128_load_unaligned(src - 1); + const v128 c = v128_load_unaligned(src + 1); + const v128 d = v128_load_unaligned(src + 2); + + calc_hdelta_hbd8(o, a, b, c, d, dst, strength, adjdamp); + src += sstride; + dst += dstride; + } +} + +void SIMD_FUNC(aom_clpf_block_hbd)(uint16_t *dst, const uint16_t *src, + int dstride, int sstride, int sizex, + int sizey, unsigned int strength, + unsigned int dmp) { + if ((sizex != 4 && sizex != 8) || ((sizey & 1) && sizex == 4)) { + // Fallback to C for odd sizes: + // * block width not 4 or 8 + // * block heights not a multiple of 2 if the block width is 4 + aom_clpf_block_hbd_c(dst, src, dstride, sstride, sizex, sizey, strength, + dmp); + } else { + (sizex == 4 ? SIMD_FUNC(clpf_block_hbd4) : SIMD_FUNC(clpf_block_hbd))( + dst, src, dstride, sstride, sizey, strength, dmp - get_msb(strength)); + } +} + +void SIMD_FUNC(aom_clpf_hblock_hbd)(uint16_t *dst, const uint16_t *src, + int dstride, int sstride, int sizex, + int sizey, unsigned int strength, + unsigned int dmp) { + if ((sizex != 4 && sizex != 8) || ((sizey & 1) && sizex == 4)) { + // Fallback to C for odd sizes: + // * block width not 4 or 8 + // * block heights not a multiple of 2 if the block width is 4 + aom_clpf_hblock_hbd_c(dst, src, dstride, sstride, sizex, sizey, strength, + dmp); + } else { + (sizex == 4 ? SIMD_FUNC(clpf_hblock_hbd4) : SIMD_FUNC(clpf_hblock_hbd))( + dst, src, dstride, sstride, sizey, strength, dmp - get_msb(strength)); + } +} diff --git a/third_party/aom/av1/common/clpf_sse2.c b/third_party/aom/av1/common/clpf_sse2.c new file mode 100644 index 0000000000..e29c2ab7ea --- /dev/null +++ b/third_party/aom/av1/common/clpf_sse2.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/aom_simd.h" +#define SIMD_FUNC(name) name##_sse2 +#include "./clpf_simd.h" diff --git a/third_party/aom/av1/common/clpf_sse4.c b/third_party/aom/av1/common/clpf_sse4.c new file mode 100644 index 0000000000..537139f17a --- /dev/null +++ b/third_party/aom/av1/common/clpf_sse4.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/aom_simd.h" +#define SIMD_FUNC(name) name##_sse4_1 +#include "./clpf_simd.h" diff --git a/third_party/aom/av1/common/clpf_ssse3.c b/third_party/aom/av1/common/clpf_ssse3.c new file mode 100644 index 0000000000..d7ed8dec5d --- /dev/null +++ b/third_party/aom/av1/common/clpf_ssse3.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/aom_simd.h" +#define SIMD_FUNC(name) name##_ssse3 +#include "./clpf_simd.h" diff --git a/third_party/aom/av1/common/common.h b/third_party/aom/av1/common/common.h new file mode 100644 index 0000000000..551055a76d --- /dev/null +++ b/third_party/aom/av1/common/common.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_COMMON_H_ +#define AV1_COMMON_COMMON_H_ + +/* Interface header for common constant data structures and lookup tables */ + +#include + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom/aom_integer.h" +#include "aom_ports/bitops.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define PI 3.141592653589793238462643383279502884 + +// Only need this for fixed-size arrays, for structs just assign. +#define av1_copy(dest, src) \ + { \ + assert(sizeof(dest) == sizeof(src)); \ + memcpy(dest, src, sizeof(src)); \ + } + +// Use this for variably-sized arrays. +#define av1_copy_array(dest, src, n) \ + { \ + assert(sizeof(*(dest)) == sizeof(*(src))); \ + memcpy(dest, src, n * sizeof(*(src))); \ + } + +#define av1_zero(dest) memset(&(dest), 0, sizeof(dest)) +#define av1_zero_array(dest, n) memset(dest, 0, n * sizeof(*(dest))) + +static INLINE int get_unsigned_bits(unsigned int num_values) { + return num_values > 0 ? get_msb(num_values) + 1 : 0; +} + +#define CHECK_MEM_ERROR(cm, lval, expr) \ + AOM_CHECK_MEM_ERROR(&cm->error, lval, expr) +// TODO(yaowu: validate the usage of these codes or develop new ones.) +#define AV1_SYNC_CODE_0 0x49 +#define AV1_SYNC_CODE_1 0x83 +#define AV1_SYNC_CODE_2 0x43 + +#define AOM_FRAME_MARKER 0x2 + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_COMMON_H_ diff --git a/third_party/aom/av1/common/common_data.h b/third_party/aom/av1/common/common_data.h new file mode 100644 index 0000000000..415d5cf73e --- /dev/null +++ b/third_party/aom/av1/common/common_data.h @@ -0,0 +1,1405 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_COMMON_DATA_H_ +#define AV1_COMMON_COMMON_DATA_H_ + +#include "av1/common/enums.h" +#include "aom/aom_integer.h" +#include "aom_dsp/aom_dsp_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if CONFIG_EXT_PARTITION +#define IF_EXT_PARTITION(...) __VA_ARGS__ +#else +#define IF_EXT_PARTITION(...) +#endif + +// Log 2 conversion lookup tables for block width and height +static const uint8_t b_width_log2_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 0, 0, 0, +#endif + 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, IF_EXT_PARTITION(4, 5, 5) +}; +static const uint8_t b_height_log2_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 0, 0, 0, +#endif + 0, 1, 0, 1, 2, 1, 2, 3, 2, 3, 4, 3, 4, IF_EXT_PARTITION(5, 4, 5) +}; +// Log 2 conversion lookup tables for modeinfo width and height +static const uint8_t mi_width_log2_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, IF_EXT_PARTITION(4, 5, 5) +#else + 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, IF_EXT_PARTITION(3, 4, 4) +#endif +}; +static const uint8_t mi_height_log2_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 0, 0, 0, 0, 1, 0, 1, 2, 1, 2, 3, 2, 3, 4, 3, 4, IF_EXT_PARTITION(5, 4, 5) +#else + 0, 0, 0, 0, 1, 0, 1, 2, 1, 2, 3, 2, 3, IF_EXT_PARTITION(4, 3, 4) +#endif +}; + +static const uint8_t mi_size_wide[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 1, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8, 8, 16, 16, IF_EXT_PARTITION(16, 32, 32) +#else + 1, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8, IF_EXT_PARTITION(8, 16, 16) +#endif +}; +static const uint8_t mi_size_high[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 1, 1, 1, 1, 2, 1, 2, 4, 2, 4, 8, 4, 8, 16, 8, 16, IF_EXT_PARTITION(32, 16, 32) +#else + 1, 1, 1, 1, 2, 1, 2, 4, 2, 4, 8, 4, 8, IF_EXT_PARTITION(16, 8, 16) +#endif +}; + +// Width/height lookup tables in units of various block sizes +static const uint8_t block_size_wide[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 2, 2, 4, +#endif + 4, 4, 8, 8, 8, 16, 16, 16, 32, 32, 32, 64, 64, IF_EXT_PARTITION(64, 128, 128) +}; + +static const uint8_t block_size_high[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 2, 4, 2, +#endif + 4, 8, 4, 8, 16, 8, 16, 32, 16, 32, 64, 32, 64, IF_EXT_PARTITION(128, 64, 128) +}; + +static const uint8_t num_4x4_blocks_wide_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 1, 1, 1, +#endif + 1, 1, 2, 2, 2, 4, 4, 4, 8, 8, 8, 16, 16, IF_EXT_PARTITION(16, 32, 32) +}; +static const uint8_t num_4x4_blocks_high_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 1, 1, 1, +#endif + 1, 2, 1, 2, 4, 2, 4, 8, 4, 8, 16, 8, 16, IF_EXT_PARTITION(32, 16, 32) +}; +static const uint8_t num_8x8_blocks_wide_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 1, 1, 1, +#endif + 1, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8, IF_EXT_PARTITION(8, 16, 16) +}; +static const uint8_t num_8x8_blocks_high_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 1, 1, 1, +#endif + 1, 1, 1, 1, 2, 1, 2, 4, 2, 4, 8, 4, 8, IF_EXT_PARTITION(16, 8, 16) +}; +static const uint8_t num_16x16_blocks_wide_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 1, 1, 1, +#endif + 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 4, 4, IF_EXT_PARTITION(4, 8, 8) +}; +static const uint8_t num_16x16_blocks_high_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 1, 1, 1, +#endif + 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 4, 2, 4, IF_EXT_PARTITION(8, 4, 8) +}; + +// AOMMIN(3, AOMMIN(b_width_log2(bsize), b_height_log2(bsize))) +static const uint8_t size_group_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 0, 0, 0, +#endif + 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, IF_EXT_PARTITION(3, 3, 3) +}; + +static const uint8_t num_pels_log2_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 2, 3, 3, +#endif + 4, 5, 5, 6, 7, 7, 8, 9, 9, 10, 11, 11, 12, IF_EXT_PARTITION(13, 13, 14) +}; + +/* clang-format off */ +static const PARTITION_TYPE + partition_lookup[MAX_SB_SIZE_LOG2 - 1][BLOCK_SIZES] = { + { // 4X4 -> +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#endif + // 4X4 + PARTITION_NONE, + // 4X8, 8X4, 8X8 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, + // 8X16, 16X8, 16X16 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, + // 16X32, 32X16, 32X32 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, + // 32X64, 64X32, 64X64 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#if CONFIG_EXT_PARTITION + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#endif // CONFIG_EXT_PARTITION + }, { // 8X8 -> +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#endif + // 4X4 + PARTITION_SPLIT, + // 4X8, 8X4, 8X8 + PARTITION_VERT, PARTITION_HORZ, PARTITION_NONE, + // 8X16, 16X8, 16X16 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, + // 16X32, 32X16, 32X32 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, + // 32X64, 64X32, 64X64 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#endif // CONFIG_EXT_PARTITION + }, { // 16X16 -> +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#endif + // 4X4 + PARTITION_SPLIT, + // 4X8, 8X4, 8X8 + PARTITION_SPLIT, PARTITION_SPLIT, PARTITION_SPLIT, + // 8X16, 16X8, 16X16 + PARTITION_VERT, PARTITION_HORZ, PARTITION_NONE, + // 16X32, 32X16, 32X32 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, + // 32X64, 64X32, 64X64 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#endif // CONFIG_EXT_PARTITION + }, { // 32X32 -> +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#endif + // 4X4 + PARTITION_SPLIT, + // 4X8, 8X4, 8X8 + PARTITION_SPLIT, PARTITION_SPLIT, PARTITION_SPLIT, + // 8X16, 16X8, 16X16 + PARTITION_SPLIT, PARTITION_SPLIT, PARTITION_SPLIT, + // 16X32, 32X16, 32X32 + PARTITION_VERT, PARTITION_HORZ, PARTITION_NONE, + // 32X64, 64X32, 64X64 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#endif // CONFIG_EXT_PARTITION + }, { // 64X64 -> +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#endif + // 4X4 + PARTITION_SPLIT, + // 4X8, 8X4, 8X8 + PARTITION_SPLIT, PARTITION_SPLIT, PARTITION_SPLIT, + // 8X16, 16X8, 16X16 + PARTITION_SPLIT, PARTITION_SPLIT, PARTITION_SPLIT, + // 16X32, 32X16, 32X32 + PARTITION_SPLIT, PARTITION_SPLIT, PARTITION_SPLIT, + // 32X64, 64X32, 64X64 + PARTITION_VERT, PARTITION_HORZ, PARTITION_NONE, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, + }, { // 128x128 -> +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + PARTITION_INVALID, PARTITION_INVALID, PARTITION_INVALID, +#endif + // 4X4 + PARTITION_SPLIT, + // 4X8, 8X4, 8X8 + PARTITION_SPLIT, PARTITION_SPLIT, PARTITION_SPLIT, + // 8X16, 16X8, 16X16 + PARTITION_SPLIT, PARTITION_SPLIT, PARTITION_SPLIT, + // 16X32, 32X16, 32X32 + PARTITION_SPLIT, PARTITION_SPLIT, PARTITION_SPLIT, + // 32X64, 64X32, 64X64 + PARTITION_SPLIT, PARTITION_SPLIT, PARTITION_SPLIT, + // 64x128, 128x64, 128x128 + PARTITION_VERT, PARTITION_HORZ, PARTITION_NONE, +#endif // CONFIG_EXT_PARTITION + } +}; + +#if CONFIG_EXT_PARTITION_TYPES +static const BLOCK_SIZE subsize_lookup[EXT_PARTITION_TYPES][BLOCK_SIZES] = +#else +static const BLOCK_SIZE subsize_lookup[PARTITION_TYPES][BLOCK_SIZES] = +#endif // CONFIG_EXT_PARTITION_TYPES +{ + { // PARTITION_NONE +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + BLOCK_2X2, BLOCK_2X4, BLOCK_4X2, +#endif + // 4X4 + BLOCK_4X4, + // 4X8, 8X4, 8X8 + BLOCK_4X8, BLOCK_8X4, BLOCK_8X8, + // 8X16, 16X8, 16X16 + BLOCK_8X16, BLOCK_16X8, BLOCK_16X16, + // 16X32, 32X16, 32X32 + BLOCK_16X32, BLOCK_32X16, BLOCK_32X32, + // 32X64, 64X32, 64X64 + BLOCK_32X64, BLOCK_64X32, BLOCK_64X64, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + BLOCK_64X128, BLOCK_128X64, BLOCK_128X128, +#endif // CONFIG_EXT_PARTITION + }, { // PARTITION_HORZ +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + BLOCK_INVALID, BLOCK_INVALID, BLOCK_INVALID, + // 4X4 + BLOCK_4X2, +#else + // 4X4 + BLOCK_INVALID, +#endif + // 4X8, 8X4, 8X8 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_8X4, + // 8X16, 16X8, 16X16 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_16X8, + // 16X32, 32X16, 32X32 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_32X16, + // 32X64, 64X32, 64X64 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_64X32, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_128X64, +#endif // CONFIG_EXT_PARTITION + }, { // PARTITION_VERT +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + BLOCK_INVALID, BLOCK_INVALID, BLOCK_INVALID, + // 4X4 + BLOCK_2X4, +#else + // 4X4 + BLOCK_INVALID, +#endif + // 4X8, 8X4, 8X8 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_4X8, + // 8X16, 16X8, 16X16 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_8X16, + // 16X32, 32X16, 32X32 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_16X32, + // 32X64, 64X32, 64X64 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_32X64, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_64X128, +#endif // CONFIG_EXT_PARTITION + }, { // PARTITION_SPLIT +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + BLOCK_INVALID, BLOCK_INVALID, BLOCK_INVALID, +#endif + // 4X4 + BLOCK_INVALID, + // 4X8, 8X4, 8X8 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_4X4, + // 8X16, 16X8, 16X16 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_8X8, + // 16X32, 32X16, 32X32 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_16X16, + // 32X64, 64X32, 64X64 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_32X32, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_64X64, +#endif // CONFIG_EXT_PARTITION +#if CONFIG_EXT_PARTITION_TYPES + }, { // PARTITION_HORZ_A +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + BLOCK_INVALID, BLOCK_INVALID, BLOCK_INVALID, +#endif + // 4X4 + BLOCK_INVALID, + // 4X8, 8X4, 8X8 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_8X4, + // 8X16, 16X8, 16X16 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_16X8, + // 16X32, 32X16, 32X32 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_32X16, + // 32X64, 64X32, 64X64 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_64X32, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_128X64, +#endif // CONFIG_EXT_PARTITION + }, { // PARTITION_HORZ_B +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + BLOCK_INVALID, BLOCK_INVALID, BLOCK_INVALID, +#endif + // 4X4 + BLOCK_INVALID, + // 4X8, 8X4, 8X8 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_8X4, + // 8X16, 16X8, 16X16 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_16X8, + // 16X32, 32X16, 32X32 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_32X16, + // 32X64, 64X32, 64X64 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_64X32, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_128X64, +#endif // CONFIG_EXT_PARTITION + }, { // PARTITION_VERT_A +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + BLOCK_INVALID, BLOCK_INVALID, BLOCK_INVALID, +#endif + // 4X4 + BLOCK_INVALID, + // 4X8, 8X4, 8X8 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_4X8, + // 8X16, 16X8, 16X16 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_8X16, + // 16X32, 32X16, 32X32 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_16X32, + // 32X64, 64X32, 64X64 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_32X64, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_64X128, +#endif // CONFIG_EXT_PARTITION + }, { // PARTITION_VERT_B +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + BLOCK_INVALID, BLOCK_INVALID, BLOCK_INVALID, +#endif + // 4X4 + BLOCK_INVALID, + // 4X8, 8X4, 8X8 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_4X8, + // 8X16, 16X8, 16X16 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_8X16, + // 16X32, 32X16, 32X32 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_16X32, + // 32X64, 64X32, 64X64 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_32X64, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + BLOCK_INVALID, BLOCK_INVALID, BLOCK_64X128, +#endif // CONFIG_EXT_PARTITION +#endif // CONFIG_EXT_PARTITION_TYPES + } +}; + +static const TX_SIZE max_txsize_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + TX_2X2, TX_2X2, TX_2X2, +#endif + // 4X4 + TX_4X4, + // 4X8, 8X4, 8X8 + TX_4X4, TX_4X4, TX_8X8, + // 8X16, 16X8, 16X16 + TX_8X8, TX_8X8, TX_16X16, + // 16X32, 32X16, 32X32 + TX_16X16, TX_16X16, TX_32X32, + // 32X64, 64X32, + TX_32X32, TX_32X32, +#if CONFIG_TX64X64 + // 64X64 + TX_64X64, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + TX_64X64, TX_64X64, TX_64X64, +#endif // CONFIG_EXT_PARTITION +#else + // 64X64 + TX_32X32, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + TX_32X32, TX_32X32, TX_32X32, +#endif // CONFIG_EXT_PARTITION +#endif // CONFIG_TX64X64 +}; + +#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) +static const TX_SIZE max_txsize_rect_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + TX_2X2, TX_2X2, TX_2X2, +#endif // CONFIG_CB4X4 + // 4X4 + TX_4X4, + // 4X8, 8X4, 8X8 + TX_4X8, TX_8X4, TX_8X8, + // 8X16, 16X8, 16X16 + TX_8X16, TX_16X8, TX_16X16, + // 16X32, 32X16, 32X32 + TX_16X32, TX_32X16, TX_32X32, + // 32X64, 64X32, + TX_32X32, TX_32X32, +#if CONFIG_TX64X64 + // 64X64 + TX_64X64, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + TX_64X64, TX_64X64, TX_64X64, +#endif // CONFIG_EXT_PARTITION +#else + // 64X64 + TX_32X32, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + TX_32X32, TX_32X32, TX_32X32, +#endif // CONFIG_EXT_PARTITION +#endif // CONFIG_TX64X64 +}; +#else +#define max_txsize_rect_lookup max_txsize_lookup +#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + +#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) +// Same as "max_txsize_lookup[bsize] - TX_8X8", except for rectangular +// block which may use a rectangular transform, in which case it is +// "(max_txsize_lookup[bsize] + 1) - TX_8X8", invalid for bsize < 8X8 +static const int32_t intra_tx_size_cat_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + INT32_MIN, INT32_MIN, INT32_MIN, + // 4X4, + INT32_MIN, + // 4X8, 8X4, 8X8, + TX_8X8 - TX_8X8, TX_8X8 - TX_8X8, TX_8X8 - TX_8X8, +#else + // 4X4 + INT32_MIN, + // 4X8, 8X4, 8X8 + INT32_MIN, INT32_MIN, TX_8X8 - TX_8X8, +#endif // CONFIG_CB4X4 + // 8X16, 16X8, 16X16 + TX_16X16 - TX_8X8, TX_16X16 - TX_8X8, TX_16X16 - TX_8X8, + // 16X32, 32X16, 32X32 + TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, + // 32X64, 64X32, + TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, +#if CONFIG_TX64X64 + // 64X64 + TX_64X64 - TX_8X8, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + TX_64X64 - TX_8X8, TX_64X64 - TX_8X8, TX_64X64 - TX_8X8, +#endif // CONFIG_EXT_PARTITION +#else + // 64X64 + TX_32X32 - TX_8X8, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, +#endif // CONFIG_EXT_PARTITION +#endif // CONFIG_TX64X64 +}; +#else +// Same as "max_txsize_lookup[bsize] - TX_8X8", invalid for bsize < 8X8 +static const int32_t intra_tx_size_cat_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2, + INT32_MIN, INT32_MIN, INT32_MIN, +#endif + // 4X4 + INT32_MIN, + // 4X8, 8X4, 8X8 + INT32_MIN, INT32_MIN, TX_8X8 - TX_8X8, + // 8X16, 16X8, 16X16 + TX_8X8 - TX_8X8, TX_8X8 - TX_8X8, TX_16X16 - TX_8X8, + // 16X32, 32X16, 32X32 + TX_16X16 - TX_8X8, TX_16X16 - TX_8X8, TX_32X32 - TX_8X8, + // 32X64, 64X32, + TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, +#if CONFIG_TX64X64 + // 64X64 + TX_64X64 - TX_8X8, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + TX_64X64 - TX_8X8, TX_64X64 - TX_8X8, TX_64X64 - TX_8X8, +#endif // CONFIG_EXT_PARTITION +#else + // 64X64 + TX_32X32 - TX_8X8, +#if CONFIG_EXT_PARTITION + // 64x128, 128x64, 128x128 + TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, +#endif // CONFIG_EXT_PARTITION +#endif // CONFIG_TX64X64 +}; +#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + +#define inter_tx_size_cat_lookup intra_tx_size_cat_lookup + +/* clang-format on */ + +static const TX_SIZE sub_tx_size_map[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + TX_2X2, // TX_2X2 +#endif + TX_4X4, // TX_4X4 + TX_4X4, // TX_8X8 + TX_8X8, // TX_16X16 + TX_16X16, // TX_32X32 +#if CONFIG_TX64X64 + TX_32X32, // TX_64X64 +#endif // CONFIG_TX64X64 + TX_4X4, // TX_4X8 + TX_4X4, // TX_8X4 + TX_8X8, // TX_8X16 + TX_8X8, // TX_16X8 + TX_16X16, // TX_16X32 + TX_16X16, // TX_32X16 + TX_4X4, // TX_4X16 + TX_4X4, // TX_16X4 + TX_8X8, // TX_8X32 + TX_8X8, // TX_32X8 +}; + +static const TX_SIZE txsize_horz_map[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + TX_2X2, // TX_2X2 +#endif + TX_4X4, // TX_4X4 + TX_8X8, // TX_8X8 + TX_16X16, // TX_16X16 + TX_32X32, // TX_32X32 +#if CONFIG_TX64X64 + TX_64X64, // TX_64X64 +#endif // CONFIG_TX64X64 + TX_4X4, // TX_4X8 + TX_8X8, // TX_8X4 + TX_8X8, // TX_8X16 + TX_16X16, // TX_16X8 + TX_16X16, // TX_16X32 + TX_32X32, // TX_32X16 + TX_4X4, // TX_4X16 + TX_16X16, // TX_16X4 + TX_8X8, // TX_8X32 + TX_32X32, // TX_32X8 +}; + +static const TX_SIZE txsize_vert_map[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + TX_2X2, // TX_2X2 +#endif + TX_4X4, // TX_4X4 + TX_8X8, // TX_8X8 + TX_16X16, // TX_16X16 + TX_32X32, // TX_32X32 +#if CONFIG_TX64X64 + TX_64X64, // TX_64X64 +#endif // CONFIG_TX64X64 + TX_8X8, // TX_4X8 + TX_4X4, // TX_8X4 + TX_16X16, // TX_8X16 + TX_8X8, // TX_16X8 + TX_32X32, // TX_16X32 + TX_16X16, // TX_32X16 + TX_16X16, // TX_4X16 + TX_4X4, // TX_16X4 + TX_32X32, // TX_8X32 + TX_8X8, // TX_32X8 +}; + +#if CONFIG_CB4X4 +#define TX_SIZE_W_MIN 2 +#else +#define TX_SIZE_W_MIN 4 +#endif + +// Transform block width in pixels +static const int tx_size_wide[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + 2, +#endif + 4, 8, 16, 32, +#if CONFIG_TX64X64 + 64, +#endif // CONFIG_TX64X64 + 4, 8, 8, 16, 16, 32, 4, 16, 8, 32 +}; + +#if CONFIG_CB4X4 +#define TX_SIZE_H_MIN 2 +#else +#define TX_SIZE_H_MIN 4 +#endif + +// Transform block height in pixels +static const int tx_size_high[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + 2, +#endif + 4, 8, 16, 32, +#if CONFIG_TX64X64 + 64, +#endif // CONFIG_TX64X64 + 8, 4, 16, 8, 32, 16, 16, 4, 32, 8 +}; + +// Transform block width in unit +static const int tx_size_wide_unit[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + 1, 2, 4, 8, 16, +#if CONFIG_TX64X64 + 32, +#endif // CONFIG_TX64X64 + 2, 4, 4, 8, 8, 16, 2, 8, 4, 16 +#else // CONFIG_CB4X4 + 1, 2, 4, 8, +#if CONFIG_TX64X64 + 16, +#endif // CONFIG_TX64X64 + 1, 2, 2, 4, 4, 8, 1, 4, 2, 8 +#endif // CONFIG_CB4X4 +}; + +// Transform block height in unit +static const int tx_size_high_unit[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + 1, 2, 4, 8, 16, +#if CONFIG_TX64X64 + 32, +#endif // CONFIG_TX64X64 + 4, 2, 8, 4, 16, 8, 8, 2, 16, 4 +#else // CONFIG_CB4X4 + 1, 2, 4, 8, +#if CONFIG_TX64X64 + 16, +#endif // CONFIG_TX64X64 + 2, 1, 4, 2, 8, 4, 4, 1, 8, 2 +#endif // CONFIG_CB4X4 +}; + +// Transform block width in log2 +static const int tx_size_wide_log2[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + 1, +#endif + 2, 3, 4, 5, +#if CONFIG_TX64X64 + 6, +#endif // CONFIG_TX64X64 + 2, 3, 3, 4, 4, 5, 2, 4, 3, 5 +}; + +// Transform block height in log2 +static const int tx_size_high_log2[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + 1, +#endif + 2, 3, 4, 5, +#if CONFIG_TX64X64 + 6, +#endif // CONFIG_TX64X64 + 3, 2, 4, 3, 5, 4, 4, 2, 5, 3 +}; + +static const int tx_size_2d[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + 4, +#endif + 16, 64, 256, 1024, +#if CONFIG_TX64X64 + 4096, +#endif // CONFIG_TX64X64 + 32, 32, 128, 128, 512, 512, 64, 64, 256, 256 +}; + +static const BLOCK_SIZE txsize_to_bsize[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + BLOCK_2X2, // TX_2X2 +#endif + BLOCK_4X4, // TX_4X4 + BLOCK_8X8, // TX_8X8 + BLOCK_16X16, // TX_16X16 + BLOCK_32X32, // TX_32X32 +#if CONFIG_TX64X64 + BLOCK_64X64, // TX_64X64 +#endif // CONFIG_TX64X64 + BLOCK_4X8, // TX_4X8 + BLOCK_8X4, // TX_8X4 + BLOCK_8X16, // TX_8X16 + BLOCK_16X8, // TX_16X8 + BLOCK_16X32, // TX_16X32 + BLOCK_32X16, // TX_32X16 + BLOCK_INVALID, // TX_4X16 + BLOCK_INVALID, // TX_16X4 + BLOCK_INVALID, // TX_8X32 + BLOCK_INVALID, // TX_32X8 +}; + +static const TX_SIZE txsize_sqr_map[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + TX_2X2, // TX_2X2 +#endif + TX_4X4, // TX_4X4 + TX_8X8, // TX_8X8 + TX_16X16, // TX_16X16 + TX_32X32, // TX_32X32 +#if CONFIG_TX64X64 + TX_64X64, // TX_64X64 +#endif // CONFIG_TX64X64 + TX_4X4, // TX_4X8 + TX_4X4, // TX_8X4 + TX_8X8, // TX_8X16 + TX_8X8, // TX_16X8 + TX_16X16, // TX_16X32 + TX_16X16, // TX_32X16 + TX_4X4, // TX_4X16 + TX_4X4, // TX_16X4 + TX_8X8, // TX_8X32 + TX_8X8, // TX_32X8 +}; + +static const TX_SIZE txsize_sqr_up_map[TX_SIZES_ALL] = { +#if CONFIG_CB4X4 + TX_2X2, // TX_2X2 +#endif + TX_4X4, // TX_4X4 + TX_8X8, // TX_8X8 + TX_16X16, // TX_16X16 + TX_32X32, // TX_32X32 +#if CONFIG_TX64X64 + TX_64X64, // TX_64X64 +#endif // CONFIG_TX64X64 + TX_8X8, // TX_4X8 + TX_8X8, // TX_8X4 + TX_16X16, // TX_8X16 + TX_16X16, // TX_16X8 + TX_32X32, // TX_16X32 + TX_32X32, // TX_32X16 + TX_16X16, // TX_4X16 + TX_16X16, // TX_16X4 + TX_32X32, // TX_8X32 + TX_32X32, // TX_32X8 +}; + +/* clang-format off */ +static const TX_SIZE tx_mode_to_biggest_tx_size[TX_MODES] = { + TX_4X4, // ONLY_4X4 + TX_8X8, // ALLOW_8X8 + TX_16X16, // ALLOW_16X16 + TX_32X32, // ALLOW_32X32 +#if CONFIG_TX64X64 + TX_64X64, // ALLOW_64X64 + TX_64X64, // TX_MODE_SELECT +#else + TX_32X32, // TX_MODE_SELECT +#endif // CONFIG_TX64X64 +}; +/* clang-format on */ + +static const BLOCK_SIZE ss_size_lookup[BLOCK_SIZES][2][2] = { +// ss_x == 0 ss_x == 0 ss_x == 1 ss_x == 1 +// ss_y == 0 ss_y == 1 ss_y == 0 ss_y == 1 +#if CONFIG_CB4X4 + { { BLOCK_2X2, BLOCK_INVALID }, { BLOCK_INVALID, BLOCK_INVALID } }, + { { BLOCK_2X4, BLOCK_INVALID }, { BLOCK_INVALID, BLOCK_INVALID } }, + { { BLOCK_4X2, BLOCK_INVALID }, { BLOCK_INVALID, BLOCK_INVALID } }, + { { BLOCK_4X4, BLOCK_4X2 }, { BLOCK_2X4, BLOCK_2X2 } }, + { { BLOCK_4X8, BLOCK_4X4 }, { BLOCK_INVALID, BLOCK_2X4 } }, + { { BLOCK_8X4, BLOCK_INVALID }, { BLOCK_4X4, BLOCK_4X2 } }, +#else + { { BLOCK_4X4, BLOCK_INVALID }, { BLOCK_INVALID, BLOCK_INVALID } }, + { { BLOCK_4X8, BLOCK_4X4 }, { BLOCK_INVALID, BLOCK_INVALID } }, + { { BLOCK_8X4, BLOCK_INVALID }, { BLOCK_4X4, BLOCK_INVALID } }, +#endif + { { BLOCK_8X8, BLOCK_8X4 }, { BLOCK_4X8, BLOCK_4X4 } }, + { { BLOCK_8X16, BLOCK_8X8 }, { BLOCK_INVALID, BLOCK_4X8 } }, + { { BLOCK_16X8, BLOCK_INVALID }, { BLOCK_8X8, BLOCK_8X4 } }, + { { BLOCK_16X16, BLOCK_16X8 }, { BLOCK_8X16, BLOCK_8X8 } }, + { { BLOCK_16X32, BLOCK_16X16 }, { BLOCK_INVALID, BLOCK_8X16 } }, + { { BLOCK_32X16, BLOCK_INVALID }, { BLOCK_16X16, BLOCK_16X8 } }, + { { BLOCK_32X32, BLOCK_32X16 }, { BLOCK_16X32, BLOCK_16X16 } }, + { { BLOCK_32X64, BLOCK_32X32 }, { BLOCK_INVALID, BLOCK_16X32 } }, + { { BLOCK_64X32, BLOCK_INVALID }, { BLOCK_32X32, BLOCK_32X16 } }, + { { BLOCK_64X64, BLOCK_64X32 }, { BLOCK_32X64, BLOCK_32X32 } }, +#if CONFIG_EXT_PARTITION + { { BLOCK_64X128, BLOCK_64X64 }, { BLOCK_INVALID, BLOCK_32X64 } }, + { { BLOCK_128X64, BLOCK_INVALID }, { BLOCK_64X64, BLOCK_64X32 } }, + { { BLOCK_128X128, BLOCK_128X64 }, { BLOCK_64X128, BLOCK_64X64 } }, +#endif // CONFIG_EXT_PARTITION +}; + +static const TX_SIZE uv_txsize_lookup[BLOCK_SIZES][TX_SIZES_ALL][2][2] = { +// ss_x == 0 ss_x == 0 ss_x == 1 ss_x == 1 +// ss_y == 0 ss_y == 1 ss_y == 0 ss_y == 1 +#if CONFIG_CB4X4 + { + // BLOCK_2X2 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, +#if CONFIG_TX64X64 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, +#endif // CONFIG_TX64X64 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { + // BLOCK_2X4 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, +#if CONFIG_TX64X64 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, +#endif // CONFIG_TX64X64 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { + // BLOCK_2X4 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, +#if CONFIG_TX64X64 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, +#endif // CONFIG_TX64X64 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, +#endif + { +// BLOCK_4X4 +#if CONFIG_CB4X4 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_4X4, TX_2X2 }, { TX_2X2, TX_2X2 } }, +#else + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif // CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#if CONFIG_TX64X64 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif // CONFIG_TX64X64 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_4X8 +#if CONFIG_CB4X4 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_4X4, TX_2X2 }, { TX_2X2, TX_2X2 } }, +#else + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#if CONFIG_TX64X64 + { { TX_4X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif // CONFIG_TX64X64 +#if CONFIG_CB4X4 + { { TX_4X8, TX_4X4 }, { TX_2X2, TX_2X2 } }, // used +#else + { { TX_4X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, // used +#endif + { { TX_4X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_4X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_8X4 +#if CONFIG_CB4X4 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, + { { TX_4X4, TX_2X2 }, { TX_2X2, TX_2X2 } }, +#else + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_8X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#if CONFIG_TX64X64 + { { TX_8X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif // CONFIG_TX64X64 + { { TX_8X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#if CONFIG_CB4X4 + { { TX_8X4, TX_2X2 }, { TX_4X4, TX_2X2 } }, // used +#else + { { TX_8X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, // used +#endif + { { TX_8X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_8X8 +#if CONFIG_CB4X4 + { { TX_2X2, TX_2X2 }, { TX_2X2, TX_2X2 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#if CONFIG_TX64X64 + { { TX_8X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X4 }, { TX_4X8, TX_4X4 } }, + { { TX_8X4, TX_8X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X4 }, { TX_4X8, TX_4X4 } }, + { { TX_8X8, TX_8X4 }, { TX_4X8, TX_4X4 } }, + { { TX_8X8, TX_8X4 }, { TX_4X8, TX_4X4 } }, + { { TX_8X8, TX_8X4 }, { TX_4X8, TX_4X4 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_8X16 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_4X4, TX_4X4 } }, +#if CONFIG_TX64X64 + { { TX_8X8, TX_8X8 }, { TX_4X4, TX_4X4 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X4, TX_8X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X16, TX_8X8 }, { TX_4X8, TX_4X8 } }, // used + { { TX_8X16, TX_8X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X16, TX_8X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X16, TX_8X8 }, { TX_4X8, TX_4X8 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_16X8 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_4X4 }, { TX_8X8, TX_4X4 } }, + { { TX_8X8, TX_4X4 }, { TX_8X8, TX_4X4 } }, + { { TX_8X8, TX_4X4 }, { TX_8X8, TX_4X4 } }, +#if CONFIG_TX64X64 + { { TX_8X8, TX_4X4 }, { TX_8X8, TX_4X4 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X4 }, { TX_4X8, TX_4X4 } }, + { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } }, + { { TX_16X8, TX_8X4 }, { TX_8X8, TX_8X4 } }, + { { TX_16X8, TX_8X4 }, { TX_8X8, TX_8X4 } }, // used + { { TX_16X8, TX_8X4 }, { TX_8X8, TX_8X4 } }, + { { TX_16X8, TX_8X4 }, { TX_8X8, TX_8X4 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_16X16 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_8X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_8X8 }, { TX_8X8, TX_8X8 } }, +#if CONFIG_TX64X64 + { { TX_16X16, TX_8X8 }, { TX_8X8, TX_8X8 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } }, + { { TX_8X16, TX_8X8 }, { TX_8X16, TX_8X8 } }, + { { TX_16X8, TX_16X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_16X8 }, { TX_8X16, TX_8X8 } }, + { { TX_16X16, TX_16X8 }, { TX_8X16, TX_8X8 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_16X32 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_16X16 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_16X16 }, { TX_8X8, TX_8X8 } }, +#if CONFIG_TX64X64 + { { TX_16X16, TX_16X16 }, { TX_8X8, TX_8X8 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } }, + { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } }, + { { TX_16X8, TX_16X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X32, TX_16X16 }, { TX_8X16, TX_8X16 } }, // used + { { TX_16X32, TX_16X16 }, { TX_8X16, TX_8X16 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_32X16 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_8X8 }, { TX_16X16, TX_8X8 } }, + { { TX_16X16, TX_8X8 }, { TX_16X16, TX_8X8 } }, +#if CONFIG_TX64X64 + { { TX_16X16, TX_8X8 }, { TX_16X16, TX_8X8 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } }, + { { TX_8X16, TX_8X8 }, { TX_8X16, TX_8X8 } }, + { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } }, + { { TX_32X16, TX_16X8 }, { TX_16X16, TX_16X8 } }, + { { TX_32X16, TX_16X8 }, { TX_16X16, TX_16X8 } }, // used + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_32X32 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } }, + { { TX_32X32, TX_16X16 }, { TX_16X16, TX_16X16 } }, +#if CONFIG_TX64X64 + { { TX_32X32, TX_16X16 }, { TX_16X16, TX_16X16 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } }, + { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } }, + { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } }, + { { TX_16X32, TX_16X16 }, { TX_16X32, TX_16X16 } }, + { { TX_32X16, TX_32X16 }, { TX_16X16, TX_16X16 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_32X64 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } }, + { { TX_32X32, TX_32X32 }, { TX_16X16, TX_16X16 } }, +#if CONFIG_TX64X64 + { { TX_32X32, TX_32X32 }, { TX_16X16, TX_16X16 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } }, + { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } }, + { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } }, + { { TX_16X32, TX_16X32 }, { TX_16X16, TX_16X16 } }, + { { TX_32X16, TX_32X16 }, { TX_16X16, TX_16X16 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_64X32 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } }, + { { TX_32X32, TX_16X16 }, { TX_32X32, TX_16X16 } }, +#if CONFIG_TX64X64 + { { TX_32X32, TX_16X16 }, { TX_32X32, TX_16X16 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } }, + { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } }, + { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } }, + { { TX_16X32, TX_16X16 }, { TX_16X32, TX_16X16 } }, + { { TX_32X16, TX_16X16 }, { TX_32X16, TX_16X16 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_64X64 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } }, + { { TX_32X32, TX_32X32 }, { TX_32X32, TX_32X32 } }, +#if CONFIG_TX64X64 + { { TX_64X64, TX_32X32 }, { TX_32X32, TX_32X32 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } }, + { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } }, + { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } }, + { { TX_16X32, TX_16X32 }, { TX_16X32, TX_16X32 } }, + { { TX_32X16, TX_32X16 }, { TX_32X16, TX_16X16 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, +#if CONFIG_EXT_PARTITION + { +// BLOCK_64X128 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } }, + { { TX_32X32, TX_32X32 }, { TX_32X32, TX_32X32 } }, +#if CONFIG_TX64X64 + { { TX_64X64, TX_64X64 }, { TX_32X32, TX_32X32 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } }, + { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } }, + { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } }, + { { TX_16X32, TX_16X32 }, { TX_16X32, TX_16X32 } }, + { { TX_32X16, TX_32X16 }, { TX_32X16, TX_32X16 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_128X64 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } }, + { { TX_32X32, TX_32X32 }, { TX_32X32, TX_32X32 } }, +#if CONFIG_TX64X64 + { { TX_64X64, TX_32X32 }, { TX_64X64, TX_32X32 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } }, + { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } }, + { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } }, + { { TX_16X32, TX_16X32 }, { TX_16X32, TX_16X32 } }, + { { TX_32X16, TX_32X16 }, { TX_32X16, TX_32X16 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, + { +// BLOCK_128X128 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } }, + { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } }, + { { TX_32X32, TX_32X32 }, { TX_32X32, TX_32X32 } }, +#if CONFIG_TX64X64 + { { TX_64X64, TX_64X64 }, { TX_64X64, TX_64X64 } }, +#endif // CONFIG_TX64X64 + { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } }, + { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } }, + { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } }, + { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } }, + { { TX_16X32, TX_16X32 }, { TX_16X32, TX_16X32 } }, + { { TX_32X16, TX_32X16 }, { TX_32X16, TX_32X16 } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } }, + }, +#endif // CONFIG_EXT_PARTITION +}; + +// Generates 4 bit field in which each bit set to 1 represents +// a blocksize partition 1111 means we split 64x64, 32x32, 16x16 +// and 8x8. 1000 means we just split the 64x64 to 32x32 +/* clang-format off */ +static const struct { + PARTITION_CONTEXT above; + PARTITION_CONTEXT left; +} partition_context_lookup[BLOCK_SIZES] = { +#if CONFIG_EXT_PARTITION +#if CONFIG_CB4X4 + { 31, 31 }, // 2X2 - {0b11111, 0b11111} + { 31, 31 }, // 2X4 - {0b11111, 0b11111} + { 31, 31 }, // 4X2 - {0b11111, 0b11111} +#endif + { 31, 31 }, // 4X4 - {0b11111, 0b11111} + { 31, 30 }, // 4X8 - {0b11111, 0b11110} + { 30, 31 }, // 8X4 - {0b11110, 0b11111} + { 30, 30 }, // 8X8 - {0b11110, 0b11110} + { 30, 28 }, // 8X16 - {0b11110, 0b11100} + { 28, 30 }, // 16X8 - {0b11100, 0b11110} + { 28, 28 }, // 16X16 - {0b11100, 0b11100} + { 28, 24 }, // 16X32 - {0b11100, 0b11000} + { 24, 28 }, // 32X16 - {0b11000, 0b11100} + { 24, 24 }, // 32X32 - {0b11000, 0b11000} + { 24, 16 }, // 32X64 - {0b11000, 0b10000} + { 16, 24 }, // 64X32 - {0b10000, 0b11000} + { 16, 16 }, // 64X64 - {0b10000, 0b10000} + { 16, 0 }, // 64X128- {0b10000, 0b00000} + { 0, 16 }, // 128X64- {0b00000, 0b10000} + { 0, 0 }, // 128X128-{0b00000, 0b00000} +#else +#if CONFIG_CB4X4 + { 15, 15 }, // 2X2 - {0b1111, 0b1111} + { 15, 15 }, // 2X4 - {0b1111, 0b1111} + { 15, 15 }, // 4X2 - {0b1111, 0b1111} +#endif + { 15, 15 }, // 4X4 - {0b1111, 0b1111} + { 15, 14 }, // 4X8 - {0b1111, 0b1110} + { 14, 15 }, // 8X4 - {0b1110, 0b1111} + { 14, 14 }, // 8X8 - {0b1110, 0b1110} + { 14, 12 }, // 8X16 - {0b1110, 0b1100} + { 12, 14 }, // 16X8 - {0b1100, 0b1110} + { 12, 12 }, // 16X16 - {0b1100, 0b1100} + { 12, 8 }, // 16X32 - {0b1100, 0b1000} + { 8, 12 }, // 32X16 - {0b1000, 0b1100} + { 8, 8 }, // 32X32 - {0b1000, 0b1000} + { 8, 0 }, // 32X64 - {0b1000, 0b0000} + { 0, 8 }, // 64X32 - {0b0000, 0b1000} + { 0, 0 }, // 64X64 - {0b0000, 0b0000} +#endif // CONFIG_EXT_PARTITION +}; +/* clang-format on */ + +#if CONFIG_SUPERTX +static const TX_SIZE uvsupertx_size_lookup[TX_SIZES][2][2] = { +// ss_x == 0 ss_x == 0 ss_x == 1 ss_x == 1 +// ss_y == 0 ss_y == 1 ss_y == 0 ss_y == 1 +#if CONFIG_CB4X4 + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, +#endif + { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_8X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, + { { TX_16X16, TX_8X8 }, { TX_8X8, TX_8X8 } }, + { { TX_32X32, TX_16X16 }, { TX_16X16, TX_16X16 } }, +#if CONFIG_TX64X64 + { { TX_64X64, TX_32X32 }, { TX_32X32, TX_32X32 } }, +#endif // CONFIG_TX64X64 +}; + +#if CONFIG_EXT_PARTITION_TYPES +static const int partition_supertx_context_lookup[EXT_PARTITION_TYPES] = { + -1, 0, 0, 1, 0, 0, 0, 0 +}; + +#else +static const int partition_supertx_context_lookup[PARTITION_TYPES] = { -1, 0, 0, + 1 }; +#endif // CONFIG_EXT_PARTITION_TYPES +#endif // CONFIG_SUPERTX + +#if CONFIG_ADAPT_SCAN +#define EOB_THRESHOLD_NUM 2 +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_COMMON_DATA_H_ diff --git a/third_party/aom/av1/common/convolve.c b/third_party/aom/av1/common/convolve.c new file mode 100644 index 0000000000..eab6fe7a35 --- /dev/null +++ b/third_party/aom/av1/common/convolve.c @@ -0,0 +1,775 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" +#include "av1/common/convolve.h" +#include "av1/common/filter.h" +#include "av1/common/onyxc_int.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" + +#define MAX_BLOCK_WIDTH (MAX_SB_SIZE) +#define MAX_BLOCK_HEIGHT (MAX_SB_SIZE) +#define MAX_STEP (32) + +void av1_convolve_horiz_c(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, + const InterpFilterParams filter_params, + const int subpel_x_q4, int x_step_q4, + ConvolveParams *conv_params) { + int x, y; + int filter_size = filter_params.taps; + assert(conv_params->round == CONVOLVE_OPT_ROUND); + src -= filter_size / 2 - 1; + for (y = 0; y < h; ++y) { + int x_q4 = subpel_x_q4; + for (x = 0; x < w; ++x) { + const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS]; + const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( + filter_params, x_q4 & SUBPEL_MASK); + int k, sum = 0; + for (k = 0; k < filter_size; ++k) sum += src_x[k] * x_filter[k]; + + sum = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); + if (conv_params->ref) + dst[x] = ROUND_POWER_OF_TWO(dst[x] + sum, 1); + else + dst[x] = sum; + + x_q4 += x_step_q4; + } + src += src_stride; + dst += dst_stride; + } +} + +void av1_convolve_vert_c(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, + const InterpFilterParams filter_params, + const int subpel_y_q4, int y_step_q4, + ConvolveParams *conv_params) { + int x, y; + int filter_size = filter_params.taps; + assert(conv_params->round == CONVOLVE_OPT_ROUND); + src -= src_stride * (filter_size / 2 - 1); + for (x = 0; x < w; ++x) { + int y_q4 = subpel_y_q4; + for (y = 0; y < h; ++y) { + const uint8_t *const src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; + const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( + filter_params, y_q4 & SUBPEL_MASK); + int k, sum = 0; + for (k = 0; k < filter_size; ++k) + sum += src_y[k * src_stride] * y_filter[k]; + + sum = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); + if (conv_params->ref) + dst[y * dst_stride] = ROUND_POWER_OF_TWO(dst[y * dst_stride] + sum, 1); + else + dst[y * dst_stride] = sum; + + y_q4 += y_step_q4; + } + ++src; + ++dst; + } +} + +static void convolve_copy(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, + ConvolveParams *conv_params) { + assert(conv_params->round == CONVOLVE_OPT_ROUND); + if (conv_params->ref == 0) { + int r; + for (r = 0; r < h; ++r) { + memcpy(dst, src, w); + src += src_stride; + dst += dst_stride; + } + } else { + int r, c; + for (r = 0; r < h; ++r) { + for (c = 0; c < w; ++c) { + dst[c] = clip_pixel(ROUND_POWER_OF_TWO(dst[c] + src[c], 1)); + } + src += src_stride; + dst += dst_stride; + } + } +} + +void av1_convolve_horiz_facade(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, + const InterpFilterParams filter_params, + const int subpel_x_q4, int x_step_q4, + ConvolveParams *conv_params) { + assert(conv_params->round == CONVOLVE_OPT_ROUND); + if (filter_params.taps == SUBPEL_TAPS) { + const int16_t *filter_x = + av1_get_interp_filter_subpel_kernel(filter_params, subpel_x_q4); + if (conv_params->ref == 0) + aom_convolve8_horiz(src, src_stride, dst, dst_stride, filter_x, x_step_q4, + NULL, -1, w, h); + else + aom_convolve8_avg_horiz(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, NULL, -1, w, h); + } else { + av1_convolve_horiz(src, src_stride, dst, dst_stride, w, h, filter_params, + subpel_x_q4, x_step_q4, conv_params); + } +} + +void av1_convolve_horiz_facade_c(const uint8_t *src, int src_stride, + uint8_t *dst, int dst_stride, int w, int h, + const InterpFilterParams filter_params, + const int subpel_x_q4, int x_step_q4, + ConvolveParams *conv_params) { + assert(conv_params->round == CONVOLVE_OPT_ROUND); + if (filter_params.taps == SUBPEL_TAPS) { + const int16_t *filter_x = + av1_get_interp_filter_subpel_kernel(filter_params, subpel_x_q4); + if (conv_params->ref == 0) + aom_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, NULL, -1, w, h); + else + aom_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter_x, + x_step_q4, NULL, -1, w, h); + } else { + av1_convolve_horiz_c(src, src_stride, dst, dst_stride, w, h, filter_params, + subpel_x_q4, x_step_q4, conv_params); + } +} + +void av1_convolve_vert_facade(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, + const InterpFilterParams filter_params, + const int subpel_y_q4, int y_step_q4, + ConvolveParams *conv_params) { + assert(conv_params->round == CONVOLVE_OPT_ROUND); + if (filter_params.taps == SUBPEL_TAPS) { + const int16_t *filter_y = + av1_get_interp_filter_subpel_kernel(filter_params, subpel_y_q4); + if (conv_params->ref == 0) { + aom_convolve8_vert(src, src_stride, dst, dst_stride, NULL, -1, filter_y, + y_step_q4, w, h); + } else { + aom_convolve8_avg_vert(src, src_stride, dst, dst_stride, NULL, -1, + filter_y, y_step_q4, w, h); + } + } else { + av1_convolve_vert(src, src_stride, dst, dst_stride, w, h, filter_params, + subpel_y_q4, y_step_q4, conv_params); + } +} + +void av1_convolve_vert_facade_c(const uint8_t *src, int src_stride, + uint8_t *dst, int dst_stride, int w, int h, + const InterpFilterParams filter_params, + const int subpel_y_q4, int y_step_q4, + ConvolveParams *conv_params) { + assert(conv_params->round == CONVOLVE_OPT_ROUND); + if (filter_params.taps == SUBPEL_TAPS) { + const int16_t *filter_y = + av1_get_interp_filter_subpel_kernel(filter_params, subpel_y_q4); + if (conv_params->ref == 0) { + aom_convolve8_vert_c(src, src_stride, dst, dst_stride, NULL, -1, filter_y, + y_step_q4, w, h); + } else { + aom_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, NULL, -1, + filter_y, y_step_q4, w, h); + } + } else { + av1_convolve_vert_c(src, src_stride, dst, dst_stride, w, h, filter_params, + subpel_y_q4, y_step_q4, conv_params); + } +} + +#if CONFIG_CONVOLVE_ROUND +void av1_convolve_rounding(const int32_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, int bits) { + int r, c; + for (r = 0; r < h; ++r) { + for (c = 0; c < w; ++c) { + dst[r * dst_stride + c] = + clip_pixel(ROUND_POWER_OF_TWO_SIGNED(src[r * src_stride + c], bits)); + } + } +} + +void av1_convolve_2d(const uint8_t *src, int src_stride, CONV_BUF_TYPE *dst, + int dst_stride, int w, int h, + InterpFilterParams *filter_params_x, + InterpFilterParams *filter_params_y, const int subpel_x_q4, + const int subpel_y_q4, ConvolveParams *conv_params) { + int x, y, k; + CONV_BUF_TYPE im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]; + int im_h = h + filter_params_y->taps - 1; + int im_stride = w; + const int fo_vert = filter_params_y->taps / 2 - 1; + const int fo_horiz = filter_params_x->taps / 2 - 1; + (void)conv_params; + // horizontal filter + const uint8_t *src_horiz = src - fo_vert * src_stride; + const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( + *filter_params_x, subpel_x_q4 & SUBPEL_MASK); + for (y = 0; y < im_h; ++y) { + for (x = 0; x < w; ++x) { + CONV_BUF_TYPE sum = 0; + for (k = 0; k < filter_params_x->taps; ++k) { + sum += x_filter[k] * src_horiz[y * src_stride + x - fo_horiz + k]; + } +#if CONFIG_COMPOUND_ROUND + im_block[y * im_stride + x] = + clip_pixel(ROUND_POWER_OF_TWO_SIGNED(sum, conv_params->round_0)); +#else + im_block[y * im_stride + x] = + ROUND_POWER_OF_TWO_SIGNED(sum, conv_params->round_0); +#endif + } + } + + // vertical filter + CONV_BUF_TYPE *src_vert = im_block + fo_vert * im_stride; + const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( + *filter_params_y, subpel_y_q4 & SUBPEL_MASK); + for (y = 0; y < h; ++y) { + for (x = 0; x < w; ++x) { + CONV_BUF_TYPE sum = 0; + for (k = 0; k < filter_params_y->taps; ++k) { + sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x]; + } + dst[y * dst_stride + x] += + ROUND_POWER_OF_TWO_SIGNED(sum, conv_params->round_1); + } + } +} + +static INLINE void transpose_uint8(uint8_t *dst, int dst_stride, + const uint8_t *src, int src_stride, int w, + int h) { + int r, c; + for (r = 0; r < h; ++r) + for (c = 0; c < w; ++c) + dst[c * (dst_stride) + r] = src[r * (src_stride) + c]; +} + +static INLINE void transpose_int32(int32_t *dst, int dst_stride, + const int32_t *src, int src_stride, int w, + int h) { + int r, c; + for (r = 0; r < h; ++r) + for (c = 0; c < w; ++c) + dst[c * (dst_stride) + r] = src[r * (src_stride) + c]; +} + +void av1_convolve_2d_facade(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, + const InterpFilter *interp_filter, + const int subpel_x_q4, int x_step_q4, + const int subpel_y_q4, int y_step_q4, + ConvolveParams *conv_params) { + (void)x_step_q4; + (void)y_step_q4; + (void)dst; + (void)dst_stride; +#if CONFIG_DUAL_FILTER + InterpFilterParams filter_params_x = + av1_get_interp_filter_params(interp_filter[1 + 2 * conv_params->ref]); + InterpFilterParams filter_params_y = + av1_get_interp_filter_params(interp_filter[0 + 2 * conv_params->ref]); + + if (filter_params_x.interp_filter == MULTITAP_SHARP && + filter_params_y.interp_filter == MULTITAP_SHARP) { + // Avoid two directions both using 12-tap filter. + // This will reduce hardware implementation cost. + filter_params_y = av1_get_interp_filter_params(EIGHTTAP_SHARP); + } +#else + InterpFilterParams filter_params_x = + av1_get_interp_filter_params(*interp_filter); + InterpFilterParams filter_params_y = + av1_get_interp_filter_params(*interp_filter); +#endif + + if (filter_params_y.taps < filter_params_x.taps) { + uint8_t tr_src[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * + (MAX_SB_SIZE + MAX_FILTER_TAP - 1)]; + int tr_src_stride = MAX_SB_SIZE + MAX_FILTER_TAP - 1; + CONV_BUF_TYPE tr_dst[MAX_SB_SIZE * MAX_SB_SIZE]; + int tr_dst_stride = MAX_SB_SIZE; + int fo_vert = filter_params_y.taps / 2 - 1; + int fo_horiz = filter_params_x.taps / 2 - 1; + + transpose_uint8(tr_src, tr_src_stride, + src - fo_vert * src_stride - fo_horiz, src_stride, + w + filter_params_x.taps - 1, h + filter_params_y.taps - 1); + transpose_int32(tr_dst, tr_dst_stride, conv_params->dst, + conv_params->dst_stride, w, h); + + // horizontal and vertical parameters are swapped because of the transpose + av1_convolve_2d(tr_src + fo_horiz * tr_src_stride + fo_vert, tr_src_stride, + tr_dst, tr_dst_stride, h, w, &filter_params_y, + &filter_params_x, subpel_y_q4, subpel_x_q4, conv_params); + transpose_int32(conv_params->dst, conv_params->dst_stride, tr_dst, + tr_dst_stride, h, w); + } else { + av1_convolve_2d(src, src_stride, conv_params->dst, conv_params->dst_stride, + w, h, &filter_params_x, &filter_params_y, subpel_x_q4, + subpel_y_q4, conv_params); + } +} + +#endif // CONFIG_CONVOLVE_ROUND + +typedef void (*ConvolveFunc)(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, + const InterpFilterParams filter_params, + const int subpel_q4, int step_q4, + ConvolveParams *conv_params); + +static void convolve_helper(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif + const int subpel_x_q4, int x_step_q4, + const int subpel_y_q4, int y_step_q4, + ConvolveParams *conv_params, + ConvolveFunc convolve_horiz, + ConvolveFunc convolve_vert) { + int ignore_horiz = x_step_q4 == 16 && subpel_x_q4 == 0; + int ignore_vert = y_step_q4 == 16 && subpel_y_q4 == 0; +#if CONFIG_DUAL_FILTER + InterpFilterParams filter_params_x = + av1_get_interp_filter_params(interp_filter[1 + 2 * conv_params->ref]); + InterpFilterParams filter_params_y = + av1_get_interp_filter_params(interp_filter[0 + 2 * conv_params->ref]); + InterpFilterParams filter_params; +#else + InterpFilterParams filter_params = + av1_get_interp_filter_params(interp_filter); +#endif + assert(conv_params->round == CONVOLVE_OPT_ROUND); + + assert(w <= MAX_BLOCK_WIDTH); + assert(h <= MAX_BLOCK_HEIGHT); + assert(y_step_q4 <= MAX_STEP); + assert(x_step_q4 <= MAX_STEP); + + if (ignore_horiz && ignore_vert) { + convolve_copy(src, src_stride, dst, dst_stride, w, h, conv_params); + } else if (ignore_vert) { +#if CONFIG_DUAL_FILTER + filter_params = filter_params_x; +#endif + assert(filter_params.taps <= MAX_FILTER_TAP); + convolve_horiz(src, src_stride, dst, dst_stride, w, h, filter_params, + subpel_x_q4, x_step_q4, conv_params); + } else if (ignore_horiz) { +#if CONFIG_DUAL_FILTER + filter_params = filter_params_y; +#endif + assert(filter_params.taps <= MAX_FILTER_TAP); + convolve_vert(src, src_stride, dst, dst_stride, w, h, filter_params, + subpel_y_q4, y_step_q4, conv_params); + } else { + // temp's size is set to a 256 aligned value to facilitate SIMD + // implementation. The value is greater than (maximum possible intermediate + // height or width) * MAX_SB_SIZE + DECLARE_ALIGNED(16, uint8_t, + temp[((MAX_SB_SIZE * 2 + 16) + 16) * MAX_SB_SIZE]); + int max_intermediate_size = ((MAX_SB_SIZE * 2 + 16) + 16); + int filter_size; +#if CONFIG_DUAL_FILTER + if (interp_filter[0 + 2 * conv_params->ref] == MULTITAP_SHARP && + interp_filter[1 + 2 * conv_params->ref] == MULTITAP_SHARP) { + // Avoid two directions both using 12-tap filter. + // This will reduce hardware implementation cost. + filter_params_y = av1_get_interp_filter_params(EIGHTTAP_SHARP); + } + + // we do filter with fewer taps first to reduce hardware implementation + // complexity + if (filter_params_y.taps < filter_params_x.taps) { + int intermediate_width; + int temp_stride = max_intermediate_size; + ConvolveParams temp_conv_params; + temp_conv_params.ref = 0; + temp_conv_params.round = CONVOLVE_OPT_ROUND; + filter_params = filter_params_y; + filter_size = filter_params_x.taps; + intermediate_width = + (((w - 1) * x_step_q4 + subpel_x_q4) >> SUBPEL_BITS) + filter_size; + assert(intermediate_width <= max_intermediate_size); + + assert(filter_params.taps <= MAX_FILTER_TAP); + + convolve_vert(src - (filter_size / 2 - 1), src_stride, temp, temp_stride, + intermediate_width, h, filter_params, subpel_y_q4, + y_step_q4, &temp_conv_params); + + filter_params = filter_params_x; + assert(filter_params.taps <= MAX_FILTER_TAP); + convolve_horiz(temp + (filter_size / 2 - 1), temp_stride, dst, dst_stride, + w, h, filter_params, subpel_x_q4, x_step_q4, conv_params); + } else +#endif // CONFIG_DUAL_FILTER + { + int intermediate_height; + int temp_stride = MAX_SB_SIZE; + ConvolveParams temp_conv_params; + temp_conv_params.ref = 0; + temp_conv_params.round = CONVOLVE_OPT_ROUND; +#if CONFIG_DUAL_FILTER + filter_params = filter_params_x; + filter_size = filter_params_y.taps; +#else + filter_size = filter_params.taps; +#endif + intermediate_height = + (((h - 1) * y_step_q4 + subpel_y_q4) >> SUBPEL_BITS) + filter_size; + assert(intermediate_height <= max_intermediate_size); + (void)max_intermediate_size; + + assert(filter_params.taps <= MAX_FILTER_TAP); + + convolve_horiz(src - src_stride * (filter_size / 2 - 1), src_stride, temp, + temp_stride, w, intermediate_height, filter_params, + subpel_x_q4, x_step_q4, &temp_conv_params); + +#if CONFIG_DUAL_FILTER + filter_params = filter_params_y; +#endif + assert(filter_params.taps <= MAX_FILTER_TAP); + + convolve_vert(temp + temp_stride * (filter_size / 2 - 1), temp_stride, + dst, dst_stride, w, h, filter_params, subpel_y_q4, + y_step_q4, conv_params); + } + } +} + +void av1_convolve(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif + const int subpel_x_q4, int x_step_q4, const int subpel_y_q4, + int y_step_q4, ConvolveParams *conv_params) { + convolve_helper(src, src_stride, dst, dst_stride, w, h, interp_filter, + subpel_x_q4, x_step_q4, subpel_y_q4, y_step_q4, conv_params, + av1_convolve_horiz_facade, av1_convolve_vert_facade); +} + +void av1_convolve_c(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif + const int subpel_x_q4, int x_step_q4, const int subpel_y_q4, + int y_step_q4, ConvolveParams *conv_params) { + convolve_helper(src, src_stride, dst, dst_stride, w, h, interp_filter, + subpel_x_q4, x_step_q4, subpel_y_q4, y_step_q4, conv_params, + av1_convolve_horiz_facade_c, av1_convolve_vert_facade_c); +} + +void av1_lowbd_convolve_init_c(void) { + // A placeholder for SIMD initialization + return; +} + +void av1_highbd_convolve_init_c(void) { + // A placeholder for SIMD initialization + return; +} + +void av1_convolve_init(AV1_COMMON *cm) { +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + av1_highbd_convolve_init(); + else + av1_lowbd_convolve_init(); +#else + (void)cm; + av1_lowbd_convolve_init(); +#endif + return; +} + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_convolve_horiz_c(const uint16_t *src, int src_stride, + uint16_t *dst, int dst_stride, int w, int h, + const InterpFilterParams filter_params, + const int subpel_x_q4, int x_step_q4, int avg, + int bd) { + int x, y; + int filter_size = filter_params.taps; + src -= filter_size / 2 - 1; + for (y = 0; y < h; ++y) { + int x_q4 = subpel_x_q4; + for (x = 0; x < w; ++x) { + const uint16_t *const src_x = &src[x_q4 >> SUBPEL_BITS]; + const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( + filter_params, x_q4 & SUBPEL_MASK); + int k, sum = 0; + for (k = 0; k < filter_size; ++k) sum += src_x[k] * x_filter[k]; + if (avg) + dst[x] = ROUND_POWER_OF_TWO( + dst[x] + + clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd), + 1); + else + dst[x] = clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd); + x_q4 += x_step_q4; + } + src += src_stride; + dst += dst_stride; + } +} + +void av1_highbd_convolve_vert_c(const uint16_t *src, int src_stride, + uint16_t *dst, int dst_stride, int w, int h, + const InterpFilterParams filter_params, + const int subpel_y_q4, int y_step_q4, int avg, + int bd) { + int x, y; + int filter_size = filter_params.taps; + src -= src_stride * (filter_size / 2 - 1); + + for (x = 0; x < w; ++x) { + int y_q4 = subpel_y_q4; + for (y = 0; y < h; ++y) { + const uint16_t *const src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; + const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( + filter_params, y_q4 & SUBPEL_MASK); + int k, sum = 0; + for (k = 0; k < filter_size; ++k) + sum += src_y[k * src_stride] * y_filter[k]; + if (avg) { + dst[y * dst_stride] = ROUND_POWER_OF_TWO( + dst[y * dst_stride] + + clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd), + 1); + } else { + dst[y * dst_stride] = + clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd); + } + y_q4 += y_step_q4; + } + ++src; + ++dst; + } +} + +static void highbd_convolve_copy(const uint16_t *src, int src_stride, + uint16_t *dst, int dst_stride, int w, int h, + int avg, int bd) { + if (avg == 0) { + int r; + for (r = 0; r < h; ++r) { + memcpy(dst, src, w * sizeof(*src)); + src += src_stride; + dst += dst_stride; + } + } else { + int r, c; + for (r = 0; r < h; ++r) { + for (c = 0; c < w; ++c) { + dst[c] = clip_pixel_highbd(ROUND_POWER_OF_TWO(dst[c] + src[c], 1), bd); + } + src += src_stride; + dst += dst_stride; + } + } +} + +void av1_highbd_convolve_horiz_facade(const uint8_t *src8, int src_stride, + uint8_t *dst8, int dst_stride, int w, + int h, + const InterpFilterParams filter_params, + const int subpel_x_q4, int x_step_q4, + int avg, int bd) { + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + if (filter_params.taps == SUBPEL_TAPS) { + const int16_t *filter_x = + av1_get_interp_filter_subpel_kernel(filter_params, subpel_x_q4); + if (avg == 0) + aom_highbd_convolve8_horiz(src8, src_stride, dst8, dst_stride, filter_x, + x_step_q4, NULL, -1, w, h, bd); + else + aom_highbd_convolve8_avg_horiz(src8, src_stride, dst8, dst_stride, + filter_x, x_step_q4, NULL, -1, w, h, bd); + } else { + av1_highbd_convolve_horiz(src, src_stride, dst, dst_stride, w, h, + filter_params, subpel_x_q4, x_step_q4, avg, bd); + } +} + +void av1_highbd_convolve_vert_facade(const uint8_t *src8, int src_stride, + uint8_t *dst8, int dst_stride, int w, + int h, + const InterpFilterParams filter_params, + const int subpel_y_q4, int y_step_q4, + int avg, int bd) { + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + + if (filter_params.taps == SUBPEL_TAPS) { + const int16_t *filter_y = + av1_get_interp_filter_subpel_kernel(filter_params, subpel_y_q4); + if (avg == 0) { + aom_highbd_convolve8_vert(src8, src_stride, dst8, dst_stride, NULL, -1, + filter_y, y_step_q4, w, h, bd); + } else { + aom_highbd_convolve8_avg_vert(src8, src_stride, dst8, dst_stride, NULL, + -1, filter_y, y_step_q4, w, h, bd); + } + } else { + av1_highbd_convolve_vert(src, src_stride, dst, dst_stride, w, h, + filter_params, subpel_y_q4, y_step_q4, avg, bd); + } +} + +void av1_highbd_convolve(const uint8_t *src8, int src_stride, uint8_t *dst8, + int dst_stride, int w, int h, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif + const int subpel_x_q4, int x_step_q4, + const int subpel_y_q4, int y_step_q4, int ref_idx, + int bd) { + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + int ignore_horiz = x_step_q4 == 16 && subpel_x_q4 == 0; + int ignore_vert = y_step_q4 == 16 && subpel_y_q4 == 0; + + assert(w <= MAX_BLOCK_WIDTH); + assert(h <= MAX_BLOCK_HEIGHT); + assert(y_step_q4 <= MAX_STEP); + assert(x_step_q4 <= MAX_STEP); + + if (ignore_horiz && ignore_vert) { + highbd_convolve_copy(src, src_stride, dst, dst_stride, w, h, ref_idx, bd); + } else if (ignore_vert) { +#if CONFIG_DUAL_FILTER + InterpFilterParams filter_params = + av1_get_interp_filter_params(interp_filter[1 + 2 * ref_idx]); +#else + InterpFilterParams filter_params = + av1_get_interp_filter_params(interp_filter); +#endif + av1_highbd_convolve_horiz_facade(src8, src_stride, dst8, dst_stride, w, h, + filter_params, subpel_x_q4, x_step_q4, + ref_idx, bd); + } else if (ignore_horiz) { +#if CONFIG_DUAL_FILTER + InterpFilterParams filter_params = + av1_get_interp_filter_params(interp_filter[0 + 2 * ref_idx]); +#else + InterpFilterParams filter_params = + av1_get_interp_filter_params(interp_filter); +#endif + av1_highbd_convolve_vert_facade(src8, src_stride, dst8, dst_stride, w, h, + filter_params, subpel_y_q4, y_step_q4, + ref_idx, bd); + } else { + // temp's size is set to a 256 aligned value to facilitate SIMD + // implementation. The value is greater than (maximum possible intermediate + // height or width) * MAX_SB_SIZE + DECLARE_ALIGNED(16, uint16_t, + temp[((MAX_SB_SIZE * 2 + 16) + 16) * MAX_SB_SIZE]); + uint8_t *temp8 = CONVERT_TO_BYTEPTR(temp); + int max_intermediate_size = ((MAX_SB_SIZE * 2 + 16) + 16); + int filter_size; + InterpFilterParams filter_params; +#if CONFIG_DUAL_FILTER + InterpFilterParams filter_params_x = + av1_get_interp_filter_params(interp_filter[1 + 2 * ref_idx]); + InterpFilterParams filter_params_y = + av1_get_interp_filter_params(interp_filter[0 + 2 * ref_idx]); + if (interp_filter[0 + 2 * ref_idx] == MULTITAP_SHARP && + interp_filter[1 + 2 * ref_idx] == MULTITAP_SHARP) { + // Avoid two directions both using 12-tap filter. + // This will reduce hardware implementation cost. + filter_params_y = av1_get_interp_filter_params(EIGHTTAP_SHARP); + } +#endif + +#if CONFIG_DUAL_FILTER + if (filter_params_y.taps < filter_params_x.taps) { + int intermediate_width; + int temp_stride = max_intermediate_size; + filter_params = filter_params_y; + filter_size = filter_params_x.taps; + intermediate_width = + (((w - 1) * x_step_q4 + subpel_x_q4) >> SUBPEL_BITS) + filter_size; + assert(intermediate_width <= max_intermediate_size); + + assert(filter_params.taps <= MAX_FILTER_TAP); + + av1_highbd_convolve_vert_facade( + src8 - (filter_size / 2 - 1), src_stride, temp8, temp_stride, + intermediate_width, h, filter_params, subpel_y_q4, y_step_q4, 0, bd); + + filter_params = filter_params_x; + assert(filter_params.taps <= MAX_FILTER_TAP); + + av1_highbd_convolve_horiz_facade( + temp8 + (filter_size / 2 - 1), temp_stride, dst8, dst_stride, w, h, + filter_params, subpel_x_q4, x_step_q4, ref_idx, bd); + } else +#endif // CONFIG_DUAL_FILTER + { + int intermediate_height; + int temp_stride = MAX_SB_SIZE; +#if CONFIG_DUAL_FILTER + filter_params = filter_params_x; + filter_size = filter_params_y.taps; +#else + filter_params = av1_get_interp_filter_params(interp_filter); + filter_size = filter_params.taps; +#endif + intermediate_height = + (((h - 1) * y_step_q4 + subpel_y_q4) >> SUBPEL_BITS) + filter_size; + assert(intermediate_height <= max_intermediate_size); + (void)max_intermediate_size; + + av1_highbd_convolve_horiz_facade( + src8 - src_stride * (filter_size / 2 - 1), src_stride, temp8, + temp_stride, w, intermediate_height, filter_params, subpel_x_q4, + x_step_q4, 0, bd); + +#if CONFIG_DUAL_FILTER + filter_params = filter_params_y; +#endif + filter_size = filter_params.taps; + assert(filter_params.taps <= MAX_FILTER_TAP); + + av1_highbd_convolve_vert_facade( + temp8 + temp_stride * (filter_size / 2 - 1), temp_stride, dst8, + dst_stride, w, h, filter_params, subpel_y_q4, y_step_q4, ref_idx, bd); + } + } +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/av1/common/convolve.h b/third_party/aom/av1/common/convolve.h new file mode 100644 index 0000000000..4a4dd8cdb1 --- /dev/null +++ b/third_party/aom/av1/common/convolve.h @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_AV1_CONVOLVE_H_ +#define AV1_COMMON_AV1_CONVOLVE_H_ +#include "av1/common/filter.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum CONVOLVE_OPT { + // indicate the results in dst buf is rounded by FILTER_BITS or not + CONVOLVE_OPT_ROUND, + CONVOLVE_OPT_NO_ROUND, +} CONVOLVE_OPT; + +typedef int32_t CONV_BUF_TYPE; + +typedef struct ConvolveParams { + int ref; + CONVOLVE_OPT round; + CONV_BUF_TYPE *dst; + int dst_stride; + int round_0; + int round_1; + int plane; +} ConvolveParams; + +static INLINE ConvolveParams get_conv_params(int ref, int plane) { + ConvolveParams conv_params; + conv_params.ref = ref; + conv_params.round = CONVOLVE_OPT_ROUND; + conv_params.plane = plane; + return conv_params; +} +struct AV1Common; +void av1_convolve_init(struct AV1Common *cm); +#if CONFIG_CONVOLVE_ROUND +void av1_convolve_2d(const uint8_t *src, int src_stride, CONV_BUF_TYPE *dst, + int dst_stride, int w, int h, + InterpFilterParams *filter_params_x, + InterpFilterParams *filter_params_y, const int subpel_x_q4, + const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_convolve_2d_facade(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, + const InterpFilter *interp_filter, + const int subpel_x_q4, int x_step_q4, + const int subpel_y_q4, int y_step_q4, + ConvolveParams *conv_params); + +static INLINE ConvolveParams get_conv_params_no_round(int ref, int plane, + int32_t *dst, + int dst_stride) { + ConvolveParams conv_params; + conv_params.ref = ref; + conv_params.round = CONVOLVE_OPT_NO_ROUND; +#if CONFIG_COMPOUND_ROUND + conv_params.round_0 = FILTER_BITS; +#else + conv_params.round_0 = 5; +#endif + conv_params.round_1 = 0; + conv_params.dst = dst; + conv_params.dst_stride = dst_stride; + conv_params.plane = plane; + return conv_params; +} + +void av1_convolve_rounding(const int32_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, int bits); +#endif // CONFIG_CONVOLVE_ROUND + +void av1_convolve(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif + const int subpel_x, int xstep, const int subpel_y, int ystep, + ConvolveParams *conv_params); + +void av1_convolve_c(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif + const int subpel_x, int xstep, const int subpel_y, + int ystep, ConvolveParams *conv_params); + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_convolve(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif + const int subpel_x, int xstep, const int subpel_y, + int ystep, int avg, int bd); +#endif // CONFIG_HIGHBITDEPTH + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_AV1_CONVOLVE_H_ diff --git a/third_party/aom/av1/common/debugmodes.c b/third_party/aom/av1/common/debugmodes.c new file mode 100644 index 0000000000..d7b31c1e4c --- /dev/null +++ b/third_party/aom/av1/common/debugmodes.c @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/blockd.h" +#include "av1/common/onyxc_int.h" + +static void log_frame_info(AV1_COMMON *cm, const char *str, FILE *f) { + fprintf(f, "%s", str); + fprintf(f, "(Frame %d, Show:%d, Q:%d): \n", cm->current_video_frame, + cm->show_frame, cm->base_qindex); +} +/* This function dereferences a pointer to the mbmi structure + * and uses the passed in member offset to print out the value of an integer + * for each mbmi member value in the mi structure. + */ +static void print_mi_data(AV1_COMMON *cm, FILE *file, const char *descriptor, + size_t member_offset) { + int mi_row, mi_col; + MODE_INFO **mi = cm->mi_grid_visible; + int rows = cm->mi_rows; + int cols = cm->mi_cols; + char prefix = descriptor[0]; + + log_frame_info(cm, descriptor, file); + for (mi_row = 0; mi_row < rows; mi_row++) { + fprintf(file, "%c ", prefix); + for (mi_col = 0; mi_col < cols; mi_col++) { + fprintf(file, "%2d ", *((int *)((char *)(&mi[0]->mbmi) + member_offset))); + mi++; + } + fprintf(file, "\n"); + mi += 8; + } + fprintf(file, "\n"); +} + +void av1_print_modes_and_motion_vectors(AV1_COMMON *cm, const char *file) { + int mi_row; + int mi_col; + FILE *mvs = fopen(file, "a"); + MODE_INFO **mi = cm->mi_grid_visible; + int rows = cm->mi_rows; + int cols = cm->mi_cols; + + print_mi_data(cm, mvs, "Partitions:", offsetof(MB_MODE_INFO, sb_type)); + print_mi_data(cm, mvs, "Modes:", offsetof(MB_MODE_INFO, mode)); + print_mi_data(cm, mvs, "Ref frame:", offsetof(MB_MODE_INFO, ref_frame[0])); + print_mi_data(cm, mvs, "Transform:", offsetof(MB_MODE_INFO, tx_size)); + print_mi_data(cm, mvs, "UV Modes:", offsetof(MB_MODE_INFO, uv_mode)); + + // output skip infomation. + log_frame_info(cm, "Skips:", mvs); + for (mi_row = 0; mi_row < rows; mi_row++) { + fprintf(mvs, "S "); + for (mi_col = 0; mi_col < cols; mi_col++) { + fprintf(mvs, "%2d ", mi[0]->mbmi.skip); + mi++; + } + fprintf(mvs, "\n"); + mi += 8; + } + fprintf(mvs, "\n"); + + // output motion vectors. + log_frame_info(cm, "Vectors ", mvs); + mi = cm->mi_grid_visible; + for (mi_row = 0; mi_row < rows; mi_row++) { + fprintf(mvs, "V "); + for (mi_col = 0; mi_col < cols; mi_col++) { + fprintf(mvs, "%4d:%4d ", mi[0]->mbmi.mv[0].as_mv.row, + mi[0]->mbmi.mv[0].as_mv.col); + mi++; + } + fprintf(mvs, "\n"); + mi += 8; + } + fprintf(mvs, "\n"); + + fclose(mvs); +} diff --git a/third_party/aom/av1/common/entropy.c b/third_party/aom/av1/common/entropy.c new file mode 100644 index 0000000000..14ab53ca00 --- /dev/null +++ b/third_party/aom/av1/common/entropy.c @@ -0,0 +1,6438 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_mem/aom_mem.h" +#include "av1/common/blockd.h" +#include "av1/common/entropy.h" +#include "av1/common/entropymode.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/scan.h" +#if CONFIG_LV_MAP +#include "av1/common/txb_common.h" +#endif + +// Unconstrained Node Tree +/* clang-format off */ +const aom_tree_index av1_coef_con_tree[TREE_SIZE(ENTROPY_TOKENS)] = { + 2, 6, // 0 = LOW_VAL + -TWO_TOKEN, 4, // 1 = TWO + -THREE_TOKEN, -FOUR_TOKEN, // 2 = THREE + 8, 10, // 3 = HIGH_LOW + -CATEGORY1_TOKEN, -CATEGORY2_TOKEN, // 4 = CAT_ONE + 12, 14, // 5 = CAT_THREEFOUR + -CATEGORY3_TOKEN, -CATEGORY4_TOKEN, // 6 = CAT_THREE + -CATEGORY5_TOKEN, -CATEGORY6_TOKEN // 7 = CAT_FIVE +}; +/* clang-format on */ + +#if CONFIG_NEW_MULTISYMBOL +/* Extra bits coded from LSB to MSB */ +const aom_cdf_prob av1_cat1_cdf0[CDF_SIZE(2)] = { AOM_ICDF(20352), + AOM_ICDF(32768), 0 }; +const aom_cdf_prob *av1_cat1_cdf[] = { av1_cat1_cdf0 }; + +const aom_cdf_prob av1_cat2_cdf0[CDF_SIZE(4)] = { + AOM_ICDF(11963), AOM_ICDF(21121), AOM_ICDF(27719), AOM_ICDF(32768), 0 +}; +const aom_cdf_prob *av1_cat2_cdf[] = { av1_cat2_cdf0 }; +const aom_cdf_prob av1_cat3_cdf0[CDF_SIZE(8)] = { + AOM_ICDF(7001), AOM_ICDF(12802), AOM_ICDF(17911), + AOM_ICDF(22144), AOM_ICDF(25503), AOM_ICDF(28286), + AOM_ICDF(30737), AOM_ICDF(32768), 0 +}; +const aom_cdf_prob *av1_cat3_cdf[] = { av1_cat3_cdf0 }; + +const aom_cdf_prob av1_cat4_cdf0[CDF_SIZE(16)] = { AOM_ICDF(3934), + AOM_ICDF(7460), + AOM_ICDF(10719), + AOM_ICDF(13640), + AOM_ICDF(16203), + AOM_ICDF(18500), + AOM_ICDF(20624), + AOM_ICDF(22528), + AOM_ICDF(24316), + AOM_ICDF(25919), + AOM_ICDF(27401), + AOM_ICDF(28729), + AOM_ICDF(29894), + AOM_ICDF(30938), + AOM_ICDF(31903), + AOM_ICDF(32768), + 0 }; +const aom_cdf_prob *av1_cat4_cdf[] = { av1_cat4_cdf0 }; + +const aom_cdf_prob av1_cat5_cdf0[CDF_SIZE(16)] = { AOM_ICDF(2942), + AOM_ICDF(5794), + AOM_ICDF(8473), + AOM_ICDF(11069), + AOM_ICDF(13469), + AOM_ICDF(15795), + AOM_ICDF(17980), + AOM_ICDF(20097), + AOM_ICDF(21952), + AOM_ICDF(23750), + AOM_ICDF(25439), + AOM_ICDF(27076), + AOM_ICDF(28589), + AOM_ICDF(30056), + AOM_ICDF(31434), + AOM_ICDF(32768), + 0 }; +const aom_cdf_prob av1_cat5_cdf1[CDF_SIZE(2)] = { AOM_ICDF(23040), + AOM_ICDF(32768), 0 }; +const aom_cdf_prob *av1_cat5_cdf[] = { av1_cat5_cdf0, av1_cat5_cdf1 }; + +const aom_cdf_prob av1_cat6_cdf0[CDF_SIZE(16)] = { + AOM_ICDF(2382), AOM_ICDF(4727), AOM_ICDF(7036), AOM_ICDF(9309), + AOM_ICDF(11512), AOM_ICDF(13681), AOM_ICDF(15816), AOM_ICDF(17918), + AOM_ICDF(19892), AOM_ICDF(21835), AOM_ICDF(23748), AOM_ICDF(25632), + AOM_ICDF(27458), AOM_ICDF(29255), AOM_ICDF(31024), AOM_ICDF(32768) +}; +const aom_cdf_prob av1_cat6_cdf1[CDF_SIZE(16)] = { + AOM_ICDF(9314), AOM_ICDF(15584), AOM_ICDF(19741), AOM_ICDF(22540), + AOM_ICDF(25391), AOM_ICDF(27310), AOM_ICDF(28583), AOM_ICDF(29440), + AOM_ICDF(30493), AOM_ICDF(31202), AOM_ICDF(31672), AOM_ICDF(31988), + AOM_ICDF(32310), AOM_ICDF(32527), AOM_ICDF(32671), AOM_ICDF(32768) +}; +const aom_cdf_prob av1_cat6_cdf2[CDF_SIZE(16)] = { + AOM_ICDF(29548), AOM_ICDF(31129), AOM_ICDF(31960), AOM_ICDF(32004), + AOM_ICDF(32473), AOM_ICDF(32498), AOM_ICDF(32511), AOM_ICDF(32512), + AOM_ICDF(32745), AOM_ICDF(32757), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768) +}; +const aom_cdf_prob av1_cat6_cdf3[CDF_SIZE(16)] = { + AOM_ICDF(32006), AOM_ICDF(32258), AOM_ICDF(32510), AOM_ICDF(32512), + AOM_ICDF(32638), AOM_ICDF(32639), AOM_ICDF(32640), AOM_ICDF(32641), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768) +}; +const aom_cdf_prob av1_cat6_cdf4[CDF_SIZE(4)] = { + AOM_ICDF(32513), AOM_ICDF(32641), AOM_ICDF(32767), AOM_ICDF(32768) +}; +const aom_cdf_prob *av1_cat6_cdf[] = { + av1_cat6_cdf0, av1_cat6_cdf1, av1_cat6_cdf2, av1_cat6_cdf3, av1_cat6_cdf4 +}; +#endif +/* Extra bits coded from MSB to LSB */ +const aom_prob av1_cat1_prob[] = { 159 }; +const aom_prob av1_cat2_prob[] = { 165, 145 }; +const aom_prob av1_cat3_prob[] = { 173, 148, 140 }; +const aom_prob av1_cat4_prob[] = { 176, 155, 140, 135 }; +const aom_prob av1_cat5_prob[] = { 180, 157, 141, 134, 130 }; +const aom_prob av1_cat6_prob[] = { + 255, 255, 255, 255, 254, 254, 254, 252, 249, + 243, 230, 196, 177, 153, 140, 133, 130, 129 +}; + +const uint16_t band_count_table[TX_SIZES_ALL][8] = { +#if CONFIG_CB4X4 + { 1, 2, 2, 3, 0, 0, 0 }, +#endif + { 1, 2, 3, 4, 3, 16 - 13, 0 }, { 1, 2, 3, 4, 11, 64 - 21, 0 }, + { 1, 2, 3, 4, 11, 256 - 21, 0 }, { 1, 2, 3, 4, 11, 1024 - 21, 0 }, +#if CONFIG_TX64X64 + { 1, 2, 3, 4, 11, 4096 - 21, 0 }, +#endif // CONFIG_TX64X64 + { 1, 2, 3, 4, 8, 32 - 18, 0 }, { 1, 2, 3, 4, 8, 32 - 18, 0 }, + { 1, 2, 3, 4, 11, 128 - 21, 0 }, { 1, 2, 3, 4, 11, 128 - 21, 0 }, + { 1, 2, 3, 4, 11, 512 - 21, 0 }, { 1, 2, 3, 4, 11, 512 - 21, 0 }, + { 1, 2, 3, 4, 11, 64 - 21, 0 }, { 1, 2, 3, 4, 11, 64 - 21, 0 }, + { 1, 2, 3, 4, 11, 256 - 21, 0 }, { 1, 2, 3, 4, 11, 256 - 21, 0 }, +}; + +const uint16_t band_cum_count_table[TX_SIZES_ALL][8] = { +#if CONFIG_CB4X4 + { 0, 1, 3, 6, 10, 13, 16, 0 }, +#endif + { 0, 1, 3, 6, 10, 13, 16, 0 }, { 0, 1, 3, 6, 10, 21, 64, 0 }, + { 0, 1, 3, 6, 10, 21, 256, 0 }, { 0, 1, 3, 6, 10, 21, 1024, 0 }, +#if CONFIG_TX64X64 + { 0, 1, 3, 6, 10, 21, 4096, 0 }, +#endif // CONFIG_TX64X64 + { 0, 1, 3, 6, 10, 18, 32, 0 }, { 0, 1, 3, 6, 10, 18, 32, 0 }, + { 0, 1, 3, 6, 10, 21, 128, 0 }, { 0, 1, 3, 6, 10, 21, 128, 0 }, + { 0, 1, 3, 6, 10, 21, 512, 0 }, { 0, 1, 3, 6, 10, 21, 512, 0 }, + { 0, 1, 3, 6, 10, 21, 64, 0 }, { 0, 1, 3, 6, 10, 21, 64, 0 }, + { 0, 1, 3, 6, 10, 21, 256, 0 }, { 0, 1, 3, 6, 10, 21, 256, 0 }, +}; + +const uint8_t av1_coefband_trans_8x8plus[MAX_TX_SQUARE] = { + 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, + // beyond MAXBAND_INDEX+1 all values are filled as 5 + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, +#if CONFIG_TX64X64 + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5 +#endif // CONFIG_TX64X64 +}; + +const uint8_t av1_coefband_trans_4x8_8x4[32] = { + 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, + 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, +}; + +const uint8_t av1_coefband_trans_4x4[16] = { + 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, +}; + +const uint8_t av1_pt_energy_class[ENTROPY_TOKENS] = { 0, 1, 2, 3, 3, 4, + 4, 5, 5, 5, 5, 5 }; + +// Model obtained from a 2-sided zero-centered distribution derived +// from a Pareto distribution. The cdf of the distribution is: +// cdf(x) = 0.5 + 0.5 * sgn(x) * [1 - {alpha/(alpha + |x|)} ^ beta] +// +// For a given beta and a given probablity of the 1-node, the alpha +// is first solved, and then the {alpha, beta} pair is used to generate +// the probabilities for the rest of the nodes. + +// beta = 8 + +// Every odd line in this table can be generated from the even lines +// by averaging : +// av1_pareto8_full[l][node] = (av1_pareto8_full[l-1][node] + +// av1_pareto8_full[l+1][node] ) >> 1; +// Values for tokens ONE_TOKEN through CATEGORY6_TOKEN included here. +const aom_prob av1_pareto8_full[COEFF_PROB_MODELS][MODEL_NODES] = { + { 3, 86, 128, 6, 86, 23, 88, 29 }, + { 6, 86, 128, 11, 87, 42, 91, 52 }, + { 9, 86, 129, 17, 88, 61, 94, 76 }, + { 12, 86, 129, 22, 88, 77, 97, 93 }, + { 15, 87, 129, 28, 89, 93, 100, 110 }, + { 17, 87, 129, 33, 90, 105, 103, 123 }, + { 20, 88, 130, 38, 91, 118, 106, 136 }, + { 23, 88, 130, 43, 91, 128, 108, 146 }, + { 26, 89, 131, 48, 92, 139, 111, 156 }, + { 28, 89, 131, 53, 93, 147, 114, 163 }, + { 31, 90, 131, 58, 94, 156, 117, 171 }, + { 34, 90, 131, 62, 94, 163, 119, 177 }, + { 37, 90, 132, 66, 95, 171, 122, 184 }, + { 39, 90, 132, 70, 96, 177, 124, 189 }, + { 42, 91, 132, 75, 97, 183, 127, 194 }, + { 44, 91, 132, 79, 97, 188, 129, 198 }, + { 47, 92, 133, 83, 98, 193, 132, 202 }, + { 49, 92, 133, 86, 99, 197, 134, 205 }, + { 52, 93, 133, 90, 100, 201, 137, 208 }, + { 54, 93, 133, 94, 100, 204, 139, 211 }, + { 57, 94, 134, 98, 101, 208, 142, 214 }, + { 59, 94, 134, 101, 102, 211, 144, 216 }, + { 62, 94, 135, 105, 103, 214, 146, 218 }, + { 64, 94, 135, 108, 103, 216, 148, 220 }, + { 66, 95, 135, 111, 104, 219, 151, 222 }, + { 68, 95, 135, 114, 105, 221, 153, 223 }, + { 71, 96, 136, 117, 106, 224, 155, 225 }, + { 73, 96, 136, 120, 106, 225, 157, 226 }, + { 76, 97, 136, 123, 107, 227, 159, 228 }, + { 78, 97, 136, 126, 108, 229, 160, 229 }, + { 80, 98, 137, 129, 109, 231, 162, 231 }, + { 82, 98, 137, 131, 109, 232, 164, 232 }, + { 84, 98, 138, 134, 110, 234, 166, 233 }, + { 86, 98, 138, 137, 111, 235, 168, 234 }, + { 89, 99, 138, 140, 112, 236, 170, 235 }, + { 91, 99, 138, 142, 112, 237, 171, 235 }, + { 93, 100, 139, 145, 113, 238, 173, 236 }, + { 95, 100, 139, 147, 114, 239, 174, 237 }, + { 97, 101, 140, 149, 115, 240, 176, 238 }, + { 99, 101, 140, 151, 115, 241, 177, 238 }, + { 101, 102, 140, 154, 116, 242, 179, 239 }, + { 103, 102, 140, 156, 117, 242, 180, 239 }, + { 105, 103, 141, 158, 118, 243, 182, 240 }, + { 107, 103, 141, 160, 118, 243, 183, 240 }, + { 109, 104, 141, 162, 119, 244, 185, 241 }, + { 111, 104, 141, 164, 119, 244, 186, 241 }, + { 113, 104, 142, 166, 120, 245, 187, 242 }, + { 114, 104, 142, 168, 121, 245, 188, 242 }, + { 116, 105, 143, 170, 122, 246, 190, 243 }, + { 118, 105, 143, 171, 122, 246, 191, 243 }, + { 120, 106, 143, 173, 123, 247, 192, 244 }, + { 121, 106, 143, 175, 124, 247, 193, 244 }, + { 123, 107, 144, 177, 125, 248, 195, 244 }, + { 125, 107, 144, 178, 125, 248, 196, 244 }, + { 127, 108, 145, 180, 126, 249, 197, 245 }, + { 128, 108, 145, 181, 127, 249, 198, 245 }, + { 130, 109, 145, 183, 128, 249, 199, 245 }, + { 132, 109, 145, 184, 128, 249, 200, 245 }, + { 134, 110, 146, 186, 129, 250, 201, 246 }, + { 135, 110, 146, 187, 130, 250, 202, 246 }, + { 137, 111, 147, 189, 131, 251, 203, 246 }, + { 138, 111, 147, 190, 131, 251, 204, 246 }, + { 140, 112, 147, 192, 132, 251, 205, 247 }, + { 141, 112, 147, 193, 132, 251, 206, 247 }, + { 143, 113, 148, 194, 133, 251, 207, 247 }, + { 144, 113, 148, 195, 134, 251, 207, 247 }, + { 146, 114, 149, 197, 135, 252, 208, 248 }, + { 147, 114, 149, 198, 135, 252, 209, 248 }, + { 149, 115, 149, 199, 136, 252, 210, 248 }, + { 150, 115, 149, 200, 137, 252, 210, 248 }, + { 152, 115, 150, 201, 138, 252, 211, 248 }, + { 153, 115, 150, 202, 138, 252, 212, 248 }, + { 155, 116, 151, 204, 139, 253, 213, 249 }, + { 156, 116, 151, 205, 139, 253, 213, 249 }, + { 158, 117, 151, 206, 140, 253, 214, 249 }, + { 159, 117, 151, 207, 141, 253, 215, 249 }, + { 161, 118, 152, 208, 142, 253, 216, 249 }, + { 162, 118, 152, 209, 142, 253, 216, 249 }, + { 163, 119, 153, 210, 143, 253, 217, 249 }, + { 164, 119, 153, 211, 143, 253, 217, 249 }, + { 166, 120, 153, 212, 144, 254, 218, 250 }, + { 167, 120, 153, 212, 145, 254, 219, 250 }, + { 168, 121, 154, 213, 146, 254, 220, 250 }, + { 169, 121, 154, 214, 146, 254, 220, 250 }, + { 171, 122, 155, 215, 147, 254, 221, 250 }, + { 172, 122, 155, 216, 147, 254, 221, 250 }, + { 173, 123, 155, 217, 148, 254, 222, 250 }, + { 174, 123, 155, 217, 149, 254, 222, 250 }, + { 176, 124, 156, 218, 150, 254, 223, 250 }, + { 177, 124, 156, 219, 150, 254, 223, 250 }, + { 178, 125, 157, 220, 151, 254, 224, 251 }, + { 179, 125, 157, 220, 151, 254, 224, 251 }, + { 180, 126, 157, 221, 152, 254, 225, 251 }, + { 181, 126, 157, 221, 152, 254, 225, 251 }, + { 183, 127, 158, 222, 153, 254, 226, 251 }, + { 184, 127, 158, 223, 154, 254, 226, 251 }, + { 185, 128, 159, 224, 155, 255, 227, 251 }, + { 186, 128, 159, 224, 155, 255, 227, 251 }, + { 187, 129, 160, 225, 156, 255, 228, 251 }, + { 188, 130, 160, 225, 156, 255, 228, 251 }, + { 189, 131, 160, 226, 157, 255, 228, 251 }, + { 190, 131, 160, 226, 158, 255, 228, 251 }, + { 191, 132, 161, 227, 159, 255, 229, 251 }, + { 192, 132, 161, 227, 159, 255, 229, 251 }, + { 193, 133, 162, 228, 160, 255, 230, 252 }, + { 194, 133, 162, 229, 160, 255, 230, 252 }, + { 195, 134, 163, 230, 161, 255, 231, 252 }, + { 196, 134, 163, 230, 161, 255, 231, 252 }, + { 197, 135, 163, 231, 162, 255, 231, 252 }, + { 198, 135, 163, 231, 162, 255, 231, 252 }, + { 199, 136, 164, 232, 163, 255, 232, 252 }, + { 200, 136, 164, 232, 164, 255, 232, 252 }, + { 201, 137, 165, 233, 165, 255, 233, 252 }, + { 201, 137, 165, 233, 165, 255, 233, 252 }, + { 202, 138, 166, 233, 166, 255, 233, 252 }, + { 203, 138, 166, 233, 166, 255, 233, 252 }, + { 204, 139, 166, 234, 167, 255, 234, 252 }, + { 205, 139, 166, 234, 167, 255, 234, 252 }, + { 206, 140, 167, 235, 168, 255, 235, 252 }, + { 206, 140, 167, 235, 168, 255, 235, 252 }, + { 207, 141, 168, 236, 169, 255, 235, 252 }, + { 208, 141, 168, 236, 170, 255, 235, 252 }, + { 209, 142, 169, 237, 171, 255, 236, 252 }, + { 209, 143, 169, 237, 171, 255, 236, 252 }, + { 210, 144, 169, 237, 172, 255, 236, 252 }, + { 211, 144, 169, 237, 172, 255, 236, 252 }, + { 212, 145, 170, 238, 173, 255, 237, 252 }, + { 213, 145, 170, 238, 173, 255, 237, 252 }, + { 214, 146, 171, 239, 174, 255, 237, 253 }, + { 214, 146, 171, 239, 174, 255, 237, 253 }, + { 215, 147, 172, 240, 175, 255, 238, 253 }, + { 215, 147, 172, 240, 175, 255, 238, 253 }, + { 216, 148, 173, 240, 176, 255, 238, 253 }, + { 217, 148, 173, 240, 176, 255, 238, 253 }, + { 218, 149, 173, 241, 177, 255, 239, 253 }, + { 218, 149, 173, 241, 178, 255, 239, 253 }, + { 219, 150, 174, 241, 179, 255, 239, 253 }, + { 219, 151, 174, 241, 179, 255, 239, 253 }, + { 220, 152, 175, 242, 180, 255, 240, 253 }, + { 221, 152, 175, 242, 180, 255, 240, 253 }, + { 222, 153, 176, 242, 181, 255, 240, 253 }, + { 222, 153, 176, 242, 181, 255, 240, 253 }, + { 223, 154, 177, 243, 182, 255, 240, 253 }, + { 223, 154, 177, 243, 182, 255, 240, 253 }, + { 224, 155, 178, 244, 183, 255, 241, 253 }, + { 224, 155, 178, 244, 183, 255, 241, 253 }, + { 225, 156, 178, 244, 184, 255, 241, 253 }, + { 225, 157, 178, 244, 184, 255, 241, 253 }, + { 226, 158, 179, 244, 185, 255, 242, 253 }, + { 227, 158, 179, 244, 185, 255, 242, 253 }, + { 228, 159, 180, 245, 186, 255, 242, 253 }, + { 228, 159, 180, 245, 186, 255, 242, 253 }, + { 229, 160, 181, 245, 187, 255, 242, 253 }, + { 229, 160, 181, 245, 187, 255, 242, 253 }, + { 230, 161, 182, 246, 188, 255, 243, 253 }, + { 230, 162, 182, 246, 188, 255, 243, 253 }, + { 231, 163, 183, 246, 189, 255, 243, 253 }, + { 231, 163, 183, 246, 189, 255, 243, 253 }, + { 232, 164, 184, 247, 190, 255, 243, 253 }, + { 232, 164, 184, 247, 190, 255, 243, 253 }, + { 233, 165, 185, 247, 191, 255, 244, 253 }, + { 233, 165, 185, 247, 191, 255, 244, 253 }, + { 234, 166, 185, 247, 192, 255, 244, 253 }, + { 234, 167, 185, 247, 192, 255, 244, 253 }, + { 235, 168, 186, 248, 193, 255, 244, 253 }, + { 235, 168, 186, 248, 193, 255, 244, 253 }, + { 236, 169, 187, 248, 194, 255, 244, 253 }, + { 236, 169, 187, 248, 194, 255, 244, 253 }, + { 236, 170, 188, 248, 195, 255, 245, 253 }, + { 236, 170, 188, 248, 195, 255, 245, 253 }, + { 237, 171, 189, 249, 196, 255, 245, 254 }, + { 237, 172, 189, 249, 196, 255, 245, 254 }, + { 238, 173, 190, 249, 197, 255, 245, 254 }, + { 238, 173, 190, 249, 197, 255, 245, 254 }, + { 239, 174, 191, 249, 198, 255, 245, 254 }, + { 239, 174, 191, 249, 198, 255, 245, 254 }, + { 240, 175, 192, 249, 199, 255, 246, 254 }, + { 240, 176, 192, 249, 199, 255, 246, 254 }, + { 240, 177, 193, 250, 200, 255, 246, 254 }, + { 240, 177, 193, 250, 200, 255, 246, 254 }, + { 241, 178, 194, 250, 201, 255, 246, 254 }, + { 241, 178, 194, 250, 201, 255, 246, 254 }, + { 242, 179, 195, 250, 202, 255, 246, 254 }, + { 242, 180, 195, 250, 202, 255, 246, 254 }, + { 242, 181, 196, 250, 203, 255, 247, 254 }, + { 242, 181, 196, 250, 203, 255, 247, 254 }, + { 243, 182, 197, 251, 204, 255, 247, 254 }, + { 243, 183, 197, 251, 204, 255, 247, 254 }, + { 244, 184, 198, 251, 205, 255, 247, 254 }, + { 244, 184, 198, 251, 205, 255, 247, 254 }, + { 244, 185, 199, 251, 206, 255, 247, 254 }, + { 244, 185, 199, 251, 206, 255, 247, 254 }, + { 245, 186, 200, 251, 207, 255, 247, 254 }, + { 245, 187, 200, 251, 207, 255, 247, 254 }, + { 246, 188, 201, 252, 207, 255, 248, 254 }, + { 246, 188, 201, 252, 207, 255, 248, 254 }, + { 246, 189, 202, 252, 208, 255, 248, 254 }, + { 246, 190, 202, 252, 208, 255, 248, 254 }, + { 247, 191, 203, 252, 209, 255, 248, 254 }, + { 247, 191, 203, 252, 209, 255, 248, 254 }, + { 247, 192, 204, 252, 210, 255, 248, 254 }, + { 247, 193, 204, 252, 210, 255, 248, 254 }, + { 248, 194, 205, 252, 211, 255, 248, 254 }, + { 248, 194, 205, 252, 211, 255, 248, 254 }, + { 248, 195, 206, 252, 212, 255, 249, 254 }, + { 248, 196, 206, 252, 212, 255, 249, 254 }, + { 249, 197, 207, 253, 213, 255, 249, 254 }, + { 249, 197, 207, 253, 213, 255, 249, 254 }, + { 249, 198, 208, 253, 214, 255, 249, 254 }, + { 249, 199, 209, 253, 214, 255, 249, 254 }, + { 250, 200, 210, 253, 215, 255, 249, 254 }, + { 250, 200, 210, 253, 215, 255, 249, 254 }, + { 250, 201, 211, 253, 215, 255, 249, 254 }, + { 250, 202, 211, 253, 215, 255, 249, 254 }, + { 250, 203, 212, 253, 216, 255, 249, 254 }, + { 250, 203, 212, 253, 216, 255, 249, 254 }, + { 251, 204, 213, 253, 217, 255, 250, 254 }, + { 251, 205, 213, 253, 217, 255, 250, 254 }, + { 251, 206, 214, 254, 218, 255, 250, 254 }, + { 251, 206, 215, 254, 218, 255, 250, 254 }, + { 252, 207, 216, 254, 219, 255, 250, 254 }, + { 252, 208, 216, 254, 219, 255, 250, 254 }, + { 252, 209, 217, 254, 220, 255, 250, 254 }, + { 252, 210, 217, 254, 220, 255, 250, 254 }, + { 252, 211, 218, 254, 221, 255, 250, 254 }, + { 252, 212, 218, 254, 221, 255, 250, 254 }, + { 253, 213, 219, 254, 222, 255, 250, 254 }, + { 253, 213, 220, 254, 222, 255, 250, 254 }, + { 253, 214, 221, 254, 223, 255, 250, 254 }, + { 253, 215, 221, 254, 223, 255, 250, 254 }, + { 253, 216, 222, 254, 224, 255, 251, 254 }, + { 253, 217, 223, 254, 224, 255, 251, 254 }, + { 253, 218, 224, 254, 225, 255, 251, 254 }, + { 253, 219, 224, 254, 225, 255, 251, 254 }, + { 254, 220, 225, 254, 225, 255, 251, 254 }, + { 254, 221, 226, 254, 225, 255, 251, 254 }, + { 254, 222, 227, 255, 226, 255, 251, 254 }, + { 254, 223, 227, 255, 226, 255, 251, 254 }, + { 254, 224, 228, 255, 227, 255, 251, 254 }, + { 254, 225, 229, 255, 227, 255, 251, 254 }, + { 254, 226, 230, 255, 228, 255, 251, 254 }, + { 254, 227, 230, 255, 229, 255, 251, 254 }, + { 255, 228, 231, 255, 230, 255, 251, 254 }, + { 255, 229, 232, 255, 230, 255, 251, 254 }, + { 255, 230, 233, 255, 231, 255, 252, 254 }, + { 255, 231, 234, 255, 231, 255, 252, 254 }, + { 255, 232, 235, 255, 232, 255, 252, 254 }, + { 255, 233, 236, 255, 232, 255, 252, 254 }, + { 255, 235, 237, 255, 233, 255, 252, 254 }, + { 255, 236, 238, 255, 234, 255, 252, 254 }, + { 255, 238, 240, 255, 235, 255, 252, 255 }, + { 255, 239, 241, 255, 235, 255, 252, 254 }, + { 255, 241, 243, 255, 236, 255, 252, 254 }, + { 255, 243, 245, 255, 237, 255, 252, 254 }, + { 255, 246, 247, 255, 239, 255, 253, 255 }, +}; + +// Model obtained from a 2-sided zero-centered distribution derived +// from a Pareto distribution. The cdf of the distribution is: +// cdf(x) = 0.5 + 0.5 * sgn(x) * [1 - {alpha/(alpha + |x|)} ^ beta] +// +// For a given beta and a given probability of the 1-node, the alpha +// is first solved, and then the {alpha, beta} pair is used to generate +// the probabilities for the rest of the nodes. +// +// The full source code of the generating program is available in: +// tools/gen_constrained_tokenset.py +// +#if CONFIG_NEW_TOKENSET +// Values for tokens TWO_TOKEN through CATEGORY6_TOKEN included +// in the table here : the ONE_TOKEN probability is +// removed and the probabilities rescaled. +// +// ZERO_TOKEN and ONE_TOKEN are coded as one CDF, +// and EOB_TOKEN is coded as flags outside this coder. +const aom_cdf_prob av1_pareto8_tail_probs[COEFF_PROB_MODELS][TAIL_NODES] = { + { 128, 127, 127, 252, 497, 969, 1839, 3318, 25511 }, + { 256, 254, 251, 496, 966, 1834, 3308, 5408, 19995 }, + { 383, 378, 373, 732, 1408, 2605, 4470, 6646, 15773 }, + { 511, 502, 493, 961, 1824, 3289, 5373, 7298, 12517 }, + { 638, 625, 611, 1182, 2215, 3894, 6064, 7548, 9991 }, + { 766, 746, 726, 1396, 2582, 4428, 6578, 7529, 8017 }, + { 893, 866, 839, 1603, 2927, 4896, 6945, 7332, 6467 }, + { 1020, 984, 950, 1803, 3250, 5305, 7191, 7022, 5243 }, + { 1147, 1102, 1059, 1996, 3552, 5659, 7338, 6646, 4269 }, + { 1274, 1218, 1166, 2183, 3835, 5963, 7403, 6234, 3492 }, + { 1400, 1334, 1270, 2363, 4099, 6223, 7401, 5809, 2869 }, + { 1527, 1447, 1372, 2537, 4345, 6442, 7346, 5386, 2366 }, + { 1654, 1560, 1473, 2704, 4574, 6624, 7247, 4973, 1959 }, + { 1780, 1672, 1571, 2866, 4787, 6771, 7114, 4579, 1628 }, + { 1906, 1782, 1667, 3022, 4984, 6889, 6954, 4206, 1358 }, + { 2032, 1891, 1762, 3172, 5167, 6979, 6773, 3856, 1136 }, + { 2158, 2000, 1854, 3316, 5335, 7044, 6577, 3530, 954 }, + { 2284, 2106, 1944, 3455, 5490, 7087, 6370, 3229, 803 }, + { 2410, 2212, 2032, 3588, 5632, 7109, 6155, 2951, 679 }, + { 2535, 2317, 2119, 3717, 5761, 7113, 5936, 2695, 575 }, + { 2661, 2420, 2203, 3840, 5880, 7101, 5714, 2461, 488 }, + { 2786, 2522, 2286, 3958, 5987, 7074, 5493, 2246, 416 }, + { 2911, 2624, 2367, 4072, 6083, 7033, 5273, 2050, 355 }, + { 3037, 2724, 2446, 4180, 6170, 6981, 5055, 1871, 304 }, + { 3162, 2822, 2523, 4284, 6247, 6919, 4842, 1708, 261 }, + { 3286, 2920, 2599, 4384, 6315, 6848, 4633, 1559, 224 }, + { 3411, 3017, 2672, 4478, 6374, 6768, 4430, 1424, 194 }, + { 3536, 3112, 2745, 4569, 6426, 6681, 4232, 1300, 167 }, + { 3660, 3207, 2815, 4656, 6469, 6588, 4040, 1188, 145 }, + { 3785, 3300, 2883, 4738, 6505, 6490, 3855, 1086, 126 }, + { 3909, 3392, 2950, 4817, 6534, 6387, 3677, 993, 109 }, + { 4033, 3483, 3015, 4891, 6557, 6281, 3505, 908, 95 }, + { 4157, 3573, 3079, 4962, 6573, 6170, 3340, 831, 83 }, + { 4281, 3662, 3141, 5029, 6584, 6058, 3181, 760, 72 }, + { 4405, 3750, 3201, 5093, 6588, 5943, 3029, 696, 63 }, + { 4529, 3837, 3260, 5152, 6587, 5826, 2883, 638, 56 }, + { 4652, 3922, 3317, 5209, 6582, 5709, 2744, 584, 49 }, + { 4775, 4007, 3373, 5262, 6572, 5590, 2610, 536, 43 }, + { 4899, 4090, 3427, 5312, 6557, 5470, 2483, 492, 38 }, + { 5022, 4173, 3480, 5359, 6538, 5351, 2361, 451, 33 }, + { 5145, 4254, 3531, 5403, 6515, 5231, 2246, 414, 29 }, + { 5268, 4334, 3581, 5443, 6489, 5112, 2135, 380, 26 }, + { 5391, 4414, 3629, 5481, 6458, 4993, 2029, 350, 23 }, + { 5514, 4492, 3676, 5515, 6425, 4875, 1929, 321, 21 }, + { 5637, 4569, 3721, 5548, 6388, 4758, 1833, 296, 18 }, + { 5759, 4645, 3766, 5577, 6349, 4642, 1742, 272, 16 }, + { 5881, 4720, 3808, 5604, 6307, 4528, 1656, 250, 14 }, + { 6004, 4794, 3849, 5628, 6262, 4414, 1573, 231, 13 }, + { 6126, 4867, 3890, 5649, 6215, 4302, 1495, 213, 11 }, + { 6248, 4939, 3928, 5669, 6166, 4192, 1420, 196, 10 }, + { 6370, 5010, 3966, 5686, 6114, 4083, 1349, 181, 9 }, + { 6492, 5080, 4002, 5700, 6061, 3976, 1282, 167, 8 }, + { 6614, 5149, 4037, 5712, 6006, 3871, 1218, 154, 7 }, + { 6735, 5217, 4070, 5723, 5950, 3767, 1157, 142, 7 }, + { 6857, 5284, 4103, 5731, 5891, 3666, 1099, 131, 6 }, + { 6978, 5351, 4134, 5737, 5832, 3566, 1044, 121, 5 }, + { 7099, 5415, 4164, 5741, 5771, 3469, 992, 112, 5 }, + { 7221, 5479, 4192, 5743, 5709, 3373, 943, 104, 4 }, + { 7342, 5542, 4220, 5743, 5646, 3279, 896, 96, 4 }, + { 7462, 5604, 4246, 5742, 5583, 3187, 851, 89, 4 }, + { 7584, 5665, 4272, 5739, 5518, 3097, 808, 82, 3 }, + { 7704, 5725, 4296, 5734, 5453, 3009, 768, 76, 3 }, + { 7825, 5784, 4318, 5727, 5386, 2924, 730, 71, 3 }, + { 7945, 5843, 4341, 5719, 5320, 2840, 693, 65, 2 }, + { 8066, 5900, 4361, 5709, 5252, 2758, 659, 61, 2 }, + { 8186, 5956, 4381, 5698, 5185, 2678, 626, 56, 2 }, + { 8306, 6011, 4400, 5685, 5117, 2600, 595, 52, 2 }, + { 8426, 6066, 4418, 5671, 5049, 2523, 565, 48, 2 }, + { 8547, 6119, 4434, 5655, 4981, 2449, 537, 45, 1 }, + { 8666, 6171, 4450, 5638, 4912, 2377, 511, 42, 1 }, + { 8786, 6223, 4465, 5620, 4843, 2306, 485, 39, 1 }, + { 8906, 6274, 4478, 5600, 4775, 2237, 461, 36, 1 }, + { 9025, 6323, 4491, 5580, 4706, 2170, 438, 34, 1 }, + { 9144, 6372, 4503, 5558, 4637, 2105, 417, 31, 1 }, + { 9264, 6420, 4514, 5535, 4568, 2041, 396, 29, 1 }, + { 9383, 6467, 4524, 5511, 4500, 1979, 376, 27, 1 }, + { 9502, 6513, 4532, 5486, 4432, 1919, 358, 25, 1 }, + { 9621, 6558, 4541, 5460, 4364, 1860, 340, 23, 1 }, + { 9740, 6602, 4548, 5433, 4296, 1803, 323, 22, 1 }, + { 9859, 6645, 4554, 5405, 4229, 1748, 307, 20, 1 }, + { 9978, 6688, 4559, 5376, 4161, 1694, 292, 19, 1 }, + { 10096, 6729, 4564, 5347, 4094, 1641, 278, 18, 1 }, + { 10215, 6770, 4568, 5316, 4028, 1590, 264, 16, 1 }, + { 10333, 6809, 4571, 5285, 3962, 1541, 251, 15, 1 }, + { 10452, 6848, 4573, 5253, 3896, 1492, 239, 14, 1 }, + { 10570, 6886, 4574, 5220, 3831, 1446, 227, 13, 1 }, + { 10688, 6923, 4575, 5186, 3767, 1400, 216, 12, 1 }, + { 10806, 6959, 4575, 5152, 3702, 1356, 205, 12, 1 }, + { 10924, 6994, 4574, 5117, 3639, 1313, 195, 11, 1 }, + { 11041, 7029, 4572, 5082, 3576, 1271, 186, 10, 1 }, + { 11159, 7062, 4570, 5046, 3513, 1231, 177, 9, 1 }, + { 11277, 7095, 4566, 5009, 3451, 1192, 168, 9, 1 }, + { 11394, 7127, 4563, 4972, 3390, 1153, 160, 8, 1 }, + { 11512, 7158, 4558, 4934, 3329, 1116, 152, 8, 1 }, + { 11629, 7188, 4553, 4896, 3269, 1080, 145, 7, 1 }, + { 11746, 7217, 4547, 4857, 3210, 1045, 138, 7, 1 }, + { 11864, 7245, 4540, 4818, 3151, 1012, 131, 6, 1 }, + { 11980, 7273, 4533, 4779, 3093, 979, 124, 6, 1 }, + { 12097, 7300, 4525, 4739, 3035, 947, 118, 6, 1 }, + { 12215, 7326, 4516, 4698, 2978, 916, 113, 5, 1 }, + { 12331, 7351, 4507, 4658, 2922, 886, 107, 5, 1 }, + { 12448, 7375, 4497, 4617, 2866, 857, 102, 5, 1 }, + { 12564, 7398, 4487, 4576, 2812, 829, 97, 4, 1 }, + { 12681, 7421, 4476, 4534, 2757, 802, 92, 4, 1 }, + { 12797, 7443, 4464, 4492, 2704, 775, 88, 4, 1 }, + { 12914, 7464, 4452, 4450, 2651, 749, 84, 3, 1 }, + { 13030, 7484, 4439, 4408, 2599, 725, 79, 3, 1 }, + { 13147, 7503, 4426, 4365, 2547, 700, 76, 3, 1 }, + { 13262, 7522, 4412, 4322, 2497, 677, 72, 3, 1 }, + { 13378, 7539, 4398, 4280, 2447, 654, 68, 3, 1 }, + { 13494, 7556, 4383, 4237, 2397, 632, 65, 3, 1 }, + { 13610, 7573, 4368, 4193, 2348, 611, 62, 2, 1 }, + { 13726, 7588, 4352, 4150, 2300, 590, 59, 2, 1 }, + { 13841, 7602, 4335, 4107, 2253, 571, 56, 2, 1 }, + { 13957, 7616, 4318, 4063, 2207, 551, 53, 2, 1 }, + { 14072, 7629, 4301, 4019, 2161, 532, 51, 2, 1 }, + { 14188, 7641, 4283, 3976, 2115, 514, 48, 2, 1 }, + { 14302, 7652, 4265, 3932, 2071, 497, 46, 2, 1 }, + { 14418, 7663, 4246, 3888, 2027, 480, 44, 1, 1 }, + { 14533, 7673, 4227, 3844, 1984, 463, 42, 1, 1 }, + { 14649, 7682, 4207, 3800, 1941, 447, 40, 1, 1 }, + { 14763, 7690, 4187, 3757, 1899, 432, 38, 1, 1 }, + { 14878, 7698, 4166, 3713, 1858, 417, 36, 1, 1 }, + { 14993, 7705, 4146, 3669, 1817, 402, 34, 1, 1 }, + { 15109, 7711, 4124, 3625, 1777, 388, 32, 1, 1 }, + { 15223, 7715, 4103, 3581, 1738, 375, 31, 1, 1 }, + { 15337, 7720, 4081, 3538, 1699, 362, 29, 1, 1 }, + { 15452, 7724, 4058, 3494, 1661, 349, 28, 1, 1 }, + { 15567, 7727, 4035, 3450, 1624, 337, 26, 1, 1 }, + { 15681, 7729, 4012, 3407, 1587, 325, 25, 1, 1 }, + { 15795, 7730, 3989, 3364, 1551, 313, 24, 1, 1 }, + { 15909, 7731, 3965, 3320, 1516, 302, 23, 1, 1 }, + { 16024, 7731, 3940, 3277, 1481, 291, 22, 1, 1 }, + { 16138, 7730, 3916, 3234, 1446, 281, 21, 1, 1 }, + { 16252, 7728, 3891, 3191, 1413, 271, 20, 1, 1 }, + { 16366, 7726, 3866, 3148, 1380, 261, 19, 1, 1 }, + { 16480, 7723, 3840, 3106, 1347, 252, 18, 1, 1 }, + { 16594, 7720, 3814, 3063, 1315, 243, 17, 1, 1 }, + { 16708, 7715, 3788, 3021, 1284, 234, 16, 1, 1 }, + { 16822, 7710, 3762, 2979, 1253, 225, 15, 1, 1 }, + { 16936, 7704, 3735, 2937, 1223, 217, 14, 1, 1 }, + { 17050, 7697, 3708, 2895, 1193, 209, 14, 1, 1 }, + { 17162, 7690, 3681, 2854, 1164, 202, 13, 1, 1 }, + { 17276, 7682, 3654, 2812, 1136, 194, 12, 1, 1 }, + { 17389, 7673, 3626, 2771, 1108, 187, 12, 1, 1 }, + { 17504, 7663, 3598, 2730, 1080, 180, 11, 1, 1 }, + { 17617, 7653, 3570, 2689, 1053, 173, 11, 1, 1 }, + { 17730, 7642, 3541, 2649, 1027, 167, 10, 1, 1 }, + { 17843, 7630, 3513, 2608, 1001, 161, 10, 1, 1 }, + { 17957, 7618, 3484, 2569, 975, 154, 9, 1, 1 }, + { 18069, 7605, 3455, 2529, 950, 149, 9, 1, 1 }, + { 18183, 7591, 3426, 2489, 926, 143, 8, 1, 1 }, + { 18296, 7576, 3396, 2450, 902, 138, 8, 1, 1 }, + { 18410, 7562, 3366, 2411, 878, 132, 7, 1, 1 }, + { 18523, 7545, 3337, 2372, 855, 127, 7, 1, 1 }, + { 18636, 7529, 3306, 2333, 833, 122, 7, 1, 1 }, + { 18749, 7511, 3276, 2295, 811, 118, 6, 1, 1 }, + { 18862, 7493, 3246, 2257, 789, 113, 6, 1, 1 }, + { 18975, 7474, 3215, 2219, 768, 109, 6, 1, 1 }, + { 19088, 7455, 3185, 2182, 747, 104, 5, 1, 1 }, + { 19201, 7435, 3154, 2144, 727, 100, 5, 1, 1 }, + { 19314, 7414, 3123, 2107, 707, 96, 5, 1, 1 }, + { 19427, 7392, 3092, 2071, 687, 92, 5, 1, 1 }, + { 19541, 7370, 3060, 2034, 668, 89, 4, 1, 1 }, + { 19654, 7347, 3029, 1998, 649, 85, 4, 1, 1 }, + { 19766, 7323, 2997, 1963, 631, 82, 4, 1, 1 }, + { 19878, 7299, 2966, 1927, 613, 79, 4, 1, 1 }, + { 19991, 7274, 2934, 1892, 596, 75, 4, 1, 1 }, + { 20105, 7248, 2902, 1857, 579, 72, 3, 1, 1 }, + { 20218, 7222, 2870, 1822, 562, 69, 3, 1, 1 }, + { 20331, 7195, 2838, 1788, 545, 66, 3, 1, 1 }, + { 20443, 7167, 2806, 1754, 529, 64, 3, 1, 1 }, + { 20556, 7138, 2774, 1720, 514, 61, 3, 1, 1 }, + { 20670, 7109, 2741, 1687, 498, 58, 3, 1, 1 }, + { 20783, 7079, 2709, 1654, 483, 56, 2, 1, 1 }, + { 20895, 7049, 2676, 1621, 469, 54, 2, 1, 1 }, + { 21008, 7017, 2644, 1589, 455, 51, 2, 1, 1 }, + { 21121, 6985, 2611, 1557, 441, 49, 2, 1, 1 }, + { 21234, 6953, 2578, 1525, 427, 47, 2, 1, 1 }, + { 21347, 6919, 2545, 1494, 414, 45, 2, 1, 1 }, + { 21460, 6885, 2513, 1462, 401, 43, 2, 1, 1 }, + { 21573, 6850, 2480, 1432, 388, 41, 2, 1, 1 }, + { 21687, 6815, 2447, 1401, 375, 39, 2, 1, 1 }, + { 21801, 6778, 2414, 1371, 363, 38, 1, 1, 1 }, + { 21914, 6741, 2381, 1341, 352, 36, 1, 1, 1 }, + { 22028, 6704, 2348, 1311, 340, 34, 1, 1, 1 }, + { 22141, 6665, 2315, 1282, 329, 33, 1, 1, 1 }, + { 22255, 6626, 2282, 1253, 318, 31, 1, 1, 1 }, + { 22368, 6586, 2249, 1225, 307, 30, 1, 1, 1 }, + { 22482, 6546, 2216, 1196, 297, 28, 1, 1, 1 }, + { 22595, 6505, 2183, 1169, 286, 27, 1, 1, 1 }, + { 22709, 6463, 2149, 1141, 277, 26, 1, 1, 1 }, + { 22823, 6420, 2116, 1114, 267, 25, 1, 1, 1 }, + { 22938, 6377, 2083, 1087, 257, 23, 1, 1, 1 }, + { 23053, 6332, 2050, 1060, 248, 22, 1, 1, 1 }, + { 23167, 6287, 2017, 1034, 239, 21, 1, 1, 1 }, + { 23280, 6242, 1984, 1008, 231, 20, 1, 1, 1 }, + { 23396, 6195, 1951, 982, 222, 19, 1, 1, 1 }, + { 23510, 6148, 1918, 957, 214, 18, 1, 1, 1 }, + { 23625, 6100, 1885, 932, 206, 17, 1, 1, 1 }, + { 23741, 6051, 1852, 907, 198, 16, 1, 1, 1 }, + { 23855, 6002, 1819, 883, 190, 16, 1, 1, 1 }, + { 23971, 5951, 1786, 859, 183, 15, 1, 1, 1 }, + { 24087, 5900, 1753, 835, 176, 14, 1, 1, 1 }, + { 24203, 5848, 1720, 812, 169, 13, 1, 1, 1 }, + { 24318, 5796, 1687, 789, 162, 13, 1, 1, 1 }, + { 24435, 5742, 1655, 766, 155, 12, 1, 1, 1 }, + { 24552, 5688, 1622, 743, 149, 11, 1, 1, 1 }, + { 24669, 5632, 1589, 721, 143, 11, 1, 1, 1 }, + { 24786, 5576, 1557, 699, 137, 10, 1, 1, 1 }, + { 24903, 5519, 1524, 678, 131, 10, 1, 1, 1 }, + { 25021, 5462, 1491, 657, 125, 9, 1, 1, 1 }, + { 25139, 5403, 1459, 636, 120, 8, 1, 1, 1 }, + { 25258, 5343, 1427, 615, 114, 8, 1, 1, 1 }, + { 25376, 5283, 1394, 595, 109, 8, 1, 1, 1 }, + { 25496, 5221, 1362, 575, 104, 7, 1, 1, 1 }, + { 25614, 5159, 1330, 556, 99, 7, 1, 1, 1 }, + { 25735, 5096, 1298, 536, 94, 6, 1, 1, 1 }, + { 25856, 5031, 1265, 517, 90, 6, 1, 1, 1 }, + { 25977, 4966, 1233, 499, 85, 5, 1, 1, 1 }, + { 26098, 4899, 1202, 480, 81, 5, 1, 1, 1 }, + { 26220, 4831, 1170, 462, 77, 5, 1, 1, 1 }, + { 26343, 4763, 1138, 444, 73, 4, 1, 1, 1 }, + { 26466, 4693, 1106, 427, 69, 4, 1, 1, 1 }, + { 26589, 4622, 1075, 410, 65, 4, 1, 1, 1 }, + { 26713, 4550, 1043, 393, 62, 4, 1, 1, 1 }, + { 26840, 4476, 1012, 376, 58, 3, 1, 1, 1 }, + { 26966, 4401, 980, 360, 55, 3, 1, 1, 1 }, + { 27092, 4325, 949, 344, 52, 3, 1, 1, 1 }, + { 27220, 4248, 918, 328, 48, 3, 1, 1, 1 }, + { 27350, 4169, 886, 313, 45, 2, 1, 1, 1 }, + { 27480, 4088, 855, 298, 42, 2, 1, 1, 1 }, + { 27610, 4006, 824, 283, 40, 2, 1, 1, 1 }, + { 27743, 3922, 793, 268, 37, 2, 1, 1, 1 }, + { 27876, 3837, 762, 254, 34, 2, 1, 1, 1 }, + { 28011, 3749, 731, 240, 32, 2, 1, 1, 1 }, + { 28147, 3659, 701, 227, 30, 1, 1, 1, 1 }, + { 28286, 3568, 670, 213, 27, 1, 1, 1, 1 }, + { 28426, 3474, 639, 200, 25, 1, 1, 1, 1 }, + { 28569, 3377, 608, 187, 23, 1, 1, 1, 1 }, + { 28714, 3278, 577, 174, 21, 1, 1, 1, 1 }, + { 28860, 3176, 547, 162, 19, 1, 1, 1, 1 }, + { 29010, 3071, 516, 150, 17, 1, 1, 1, 1 }, + { 29163, 2962, 485, 138, 16, 1, 1, 1, 1 }, + { 29320, 2849, 454, 127, 14, 1, 1, 1, 1 }, + { 29483, 2731, 423, 115, 12, 1, 1, 1, 1 }, + { 29650, 2608, 391, 104, 11, 1, 1, 1, 1 }, + { 29823, 2479, 360, 93, 9, 1, 1, 1, 1 }, + { 30002, 2343, 328, 83, 8, 1, 1, 1, 1 }, + { 30192, 2198, 295, 72, 7, 1, 1, 1, 1 }, + { 30393, 2041, 262, 62, 6, 1, 1, 1, 1 }, + { 30612, 1869, 227, 52, 4, 1, 1, 1, 1 }, + { 30853, 1676, 191, 41, 3, 1, 1, 1, 1 }, + { 31131, 1448, 152, 31, 2, 1, 1, 1, 1 }, + { 31486, 1150, 107, 20, 1, 1, 1, 1, 1 }, +}; +#elif CONFIG_EC_MULTISYMBOL +// Values for tokens ONE_TOKEN through CATEGORY6_TOKEN included here. +// ZERO_TOKEN and EOB_TOKEN are coded as flags outside this coder. +const aom_cdf_prob + av1_pareto8_token_probs[COEFF_PROB_MODELS][ENTROPY_TOKENS - 2] = { + { 128, 127, 127, 126, 251, 495, 965, 1832, 3305, 25412 }, + { 256, 254, 252, 249, 492, 959, 1820, 3283, 5365, 19838 }, + { 384, 379, 374, 369, 724, 1392, 2574, 4417, 6568, 15587 }, + { 512, 503, 494, 486, 946, 1795, 3238, 5289, 7184, 12321 }, + { 640, 626, 612, 599, 1159, 2172, 3818, 5946, 7401, 9795 }, + { 768, 748, 728, 709, 1363, 2522, 4324, 6424, 7352, 7830 }, + { 896, 869, 842, 816, 1559, 2847, 4762, 6755, 7131, 6291 }, + { 1024, 988, 954, 921, 1747, 3148, 5139, 6966, 6803, 5078 }, + { 1152, 1107, 1063, 1022, 1926, 3427, 5460, 7080, 6412, 4119 }, + { 1280, 1224, 1171, 1120, 2098, 3685, 5730, 7113, 5991, 3356 }, + { 1408, 1340, 1276, 1216, 2261, 3923, 5955, 7083, 5560, 2746 }, + { 1536, 1455, 1380, 1308, 2418, 4142, 6140, 7001, 5133, 2255 }, + { 1664, 1569, 1481, 1398, 2567, 4342, 6287, 6879, 4721, 1860 }, + { 1792, 1683, 1580, 1485, 2709, 4525, 6401, 6725, 4329, 1539 }, + { 1920, 1794, 1678, 1570, 2845, 4692, 6486, 6546, 3959, 1278 }, + { 2048, 1905, 1773, 1651, 2974, 4844, 6543, 6350, 3615, 1065 }, + { 2176, 2015, 1867, 1731, 3096, 4980, 6576, 6140, 3296, 891 }, + { 2304, 2123, 1958, 1807, 3212, 5104, 6589, 5922, 3002, 747 }, + { 2432, 2231, 2048, 1882, 3322, 5214, 6581, 5698, 2732, 628 }, + { 2560, 2337, 2136, 1953, 3427, 5311, 6557, 5472, 2485, 530 }, + { 2688, 2442, 2222, 2023, 3525, 5397, 6518, 5246, 2259, 448 }, + { 2816, 2547, 2306, 2090, 3618, 5472, 6465, 5021, 2053, 380 }, + { 2944, 2650, 2388, 2154, 3706, 5537, 6401, 4799, 1866, 323 }, + { 3072, 2752, 2468, 2217, 3788, 5591, 6327, 4581, 1696, 276 }, + { 3200, 2853, 2547, 2277, 3866, 5637, 6243, 4369, 1541, 235 }, + { 3328, 2952, 2624, 2335, 3938, 5673, 6152, 4163, 1401, 202 }, + { 3456, 3051, 2699, 2391, 4006, 5702, 6054, 3962, 1274, 173 }, + { 3584, 3149, 2772, 2444, 4070, 5723, 5950, 3769, 1158, 149 }, + { 3712, 3246, 2843, 2496, 4128, 5736, 5842, 3583, 1054, 128 }, + { 3840, 3341, 2913, 2545, 4183, 5743, 5729, 3404, 959, 111 }, + { 3968, 3436, 2981, 2593, 4233, 5743, 5614, 3232, 872, 96 }, + { 4096, 3529, 3048, 2638, 4280, 5737, 5496, 3067, 794, 83 }, + { 4224, 3621, 3113, 2682, 4322, 5726, 5375, 2909, 724, 72 }, + { 4352, 3712, 3176, 2724, 4361, 5709, 5253, 2759, 659, 63 }, + { 4480, 3803, 3237, 2764, 4396, 5687, 5130, 2615, 601, 55 }, + { 4608, 3892, 3297, 2801, 4428, 5661, 5007, 2478, 548, 48 }, + { 4736, 3980, 3355, 2838, 4456, 5631, 4883, 2347, 500, 42 }, + { 4864, 4067, 3412, 2872, 4481, 5596, 4760, 2223, 456, 37 }, + { 4992, 4152, 3467, 2905, 4503, 5558, 4637, 2105, 417, 32 }, + { 5120, 4237, 3521, 2936, 4521, 5516, 4515, 1993, 381, 28 }, + { 5248, 4321, 3573, 2966, 4537, 5471, 4393, 1886, 348, 25 }, + { 5376, 4404, 3623, 2993, 4550, 5424, 4273, 1785, 318, 22 }, + { 5504, 4486, 3672, 3020, 4560, 5373, 4155, 1688, 291, 19 }, + { 5632, 4566, 3720, 3044, 4568, 5321, 4037, 1597, 266, 17 }, + { 5760, 4646, 3766, 3067, 4572, 5265, 3922, 1511, 244, 15 }, + { 5888, 4724, 3811, 3089, 4575, 5208, 3808, 1429, 223, 13 }, + { 6016, 4802, 3854, 3109, 4575, 5148, 3696, 1352, 204, 12 }, + { 6144, 4878, 3895, 3128, 4573, 5088, 3587, 1278, 187, 10 }, + { 6272, 4953, 3936, 3145, 4568, 5025, 3479, 1209, 172, 9 }, + { 6400, 5028, 3975, 3161, 4561, 4961, 3373, 1143, 158, 8 }, + { 6528, 5101, 4012, 3175, 4553, 4896, 3270, 1081, 145, 7 }, + { 6656, 5173, 4048, 3189, 4542, 4830, 3168, 1022, 133, 7 }, + { 6784, 5244, 4083, 3201, 4530, 4763, 3069, 966, 122, 6 }, + { 6912, 5314, 4117, 3212, 4516, 4694, 2973, 913, 112, 5 }, + { 7040, 5383, 4149, 3221, 4500, 4626, 2878, 863, 103, 5 }, + { 7168, 5452, 4180, 3229, 4482, 4556, 2786, 816, 95, 4 }, + { 7296, 5519, 4210, 3236, 4463, 4486, 2696, 771, 87, 4 }, + { 7424, 5585, 4238, 3242, 4442, 4416, 2609, 729, 80, 3 }, + { 7552, 5650, 4265, 3247, 4420, 4345, 2523, 689, 74, 3 }, + { 7680, 5714, 4291, 3251, 4396, 4274, 2440, 651, 68, 3 }, + { 7808, 5777, 4315, 3254, 4371, 4203, 2359, 616, 63, 2 }, + { 7936, 5838, 4339, 3255, 4345, 4132, 2281, 582, 58, 2 }, + { 8064, 5899, 4361, 3256, 4318, 4061, 2204, 550, 53, 2 }, + { 8192, 5959, 4382, 3255, 4289, 3990, 2130, 520, 49, 2 }, + { 8320, 6018, 4402, 3254, 4259, 3919, 2057, 492, 45, 2 }, + { 8448, 6075, 4421, 3252, 4229, 3848, 1987, 465, 42, 1 }, + { 8576, 6133, 4438, 3248, 4197, 3778, 1919, 439, 39, 1 }, + { 8704, 6188, 4455, 3244, 4164, 3708, 1853, 415, 36, 1 }, + { 8832, 6243, 4470, 3239, 4131, 3638, 1789, 392, 33, 1 }, + { 8960, 6297, 4484, 3233, 4096, 3569, 1727, 371, 30, 1 }, + { 9088, 6349, 4497, 3226, 4061, 3500, 1667, 351, 28, 1 }, + { 9216, 6401, 4509, 3219, 4025, 3432, 1608, 331, 26, 1 }, + { 9344, 6452, 4520, 3210, 3989, 3364, 1551, 313, 24, 1 }, + { 9472, 6501, 4530, 3201, 3952, 3297, 1496, 296, 22, 1 }, + { 9600, 6550, 4539, 3191, 3914, 3230, 1443, 280, 20, 1 }, + { 9728, 6597, 4547, 3180, 3875, 3164, 1392, 265, 19, 1 }, + { 9856, 6644, 4554, 3169, 3836, 3098, 1342, 250, 18, 1 }, + { 9984, 6690, 4560, 3157, 3796, 3034, 1293, 237, 16, 1 }, + { 10112, 6734, 4565, 3144, 3756, 2970, 1247, 224, 15, 1 }, + { 10240, 6778, 4568, 3131, 3716, 2907, 1202, 211, 14, 1 }, + { 10368, 6821, 4571, 3117, 3675, 2844, 1158, 200, 13, 1 }, + { 10496, 6862, 4573, 3102, 3634, 2783, 1116, 189, 12, 1 }, + { 10624, 6903, 4574, 3087, 3592, 2722, 1075, 179, 11, 1 }, + { 10752, 6942, 4575, 3071, 3551, 2662, 1035, 169, 10, 1 }, + { 10880, 6981, 4574, 3054, 3508, 2603, 997, 160, 10, 1 }, + { 11008, 7019, 4572, 3038, 3466, 2544, 960, 151, 9, 1 }, + { 11136, 7055, 4570, 3020, 3424, 2487, 924, 143, 8, 1 }, + { 11264, 7091, 4566, 3002, 3381, 2430, 890, 135, 8, 1 }, + { 11392, 7126, 4563, 2984, 3338, 2374, 856, 127, 7, 1 }, + { 11520, 7159, 4557, 2965, 3295, 2319, 824, 121, 7, 1 }, + { 11648, 7193, 4552, 2945, 3252, 2264, 793, 114, 6, 1 }, + { 11776, 7224, 4545, 2925, 3209, 2211, 763, 108, 6, 1 }, + { 11904, 7255, 4538, 2905, 3165, 2159, 734, 102, 5, 1 }, + { 12032, 7285, 4530, 2884, 3122, 2107, 706, 96, 5, 1 }, + { 12160, 7314, 4520, 2863, 3079, 2056, 679, 91, 5, 1 }, + { 12288, 7341, 4511, 2842, 3036, 2006, 653, 86, 4, 1 }, + { 12416, 7368, 4500, 2820, 2993, 1957, 628, 81, 4, 1 }, + { 12544, 7394, 4489, 2797, 2949, 1909, 604, 77, 4, 1 }, + { 12672, 7419, 4477, 2775, 2906, 1861, 581, 73, 3, 1 }, + { 12800, 7443, 4464, 2752, 2863, 1815, 558, 69, 3, 1 }, + { 12928, 7466, 4451, 2729, 2820, 1769, 536, 65, 3, 1 }, + { 13056, 7488, 4437, 2705, 2777, 1724, 516, 61, 3, 1 }, + { 13184, 7509, 4422, 2682, 2734, 1680, 495, 58, 3, 1 }, + { 13312, 7529, 4406, 2658, 2692, 1637, 476, 55, 2, 1 }, + { 13440, 7548, 4390, 2633, 2650, 1595, 457, 52, 2, 1 }, + { 13568, 7567, 4373, 2609, 2607, 1553, 439, 49, 2, 1 }, + { 13696, 7583, 4356, 2584, 2565, 1513, 422, 46, 2, 1 }, + { 13824, 7600, 4337, 2559, 2523, 1473, 405, 44, 2, 1 }, + { 13952, 7615, 4319, 2533, 2482, 1434, 389, 41, 2, 1 }, + { 14080, 7629, 4300, 2508, 2441, 1395, 373, 39, 2, 1 }, + { 14208, 7643, 4280, 2482, 2400, 1358, 358, 37, 1, 1 }, + { 14336, 7655, 4259, 2457, 2359, 1321, 344, 35, 1, 1 }, + { 14464, 7667, 4238, 2431, 2318, 1285, 330, 33, 1, 1 }, + { 14592, 7677, 4217, 2405, 2278, 1250, 316, 31, 1, 1 }, + { 14720, 7687, 4195, 2378, 2238, 1215, 304, 29, 1, 1 }, + { 14848, 7696, 4172, 2352, 2198, 1181, 291, 28, 1, 1 }, + { 14976, 7703, 4149, 2326, 2159, 1148, 279, 26, 1, 1 }, + { 15104, 7710, 4125, 2299, 2119, 1116, 268, 25, 1, 1 }, + { 15232, 7715, 4101, 2272, 2081, 1085, 257, 23, 1, 1 }, + { 15360, 7721, 4076, 2245, 2042, 1054, 246, 22, 1, 1 }, + { 15488, 7724, 4051, 2219, 2004, 1023, 236, 21, 1, 1 }, + { 15616, 7727, 4025, 2192, 1966, 994, 226, 20, 1, 1 }, + { 15744, 7729, 3999, 2164, 1929, 965, 217, 19, 1, 1 }, + { 15872, 7731, 3972, 2137, 1892, 937, 207, 18, 1, 1 }, + { 16000, 7731, 3945, 2110, 1855, 909, 199, 17, 1, 1 }, + { 16128, 7730, 3918, 2083, 1819, 882, 190, 16, 1, 1 }, + { 16256, 7728, 3890, 2056, 1783, 856, 182, 15, 1, 1 }, + { 16384, 7725, 3862, 2029, 1747, 831, 174, 14, 1, 1 }, + { 16512, 7721, 3833, 2002, 1712, 806, 167, 13, 1, 1 }, + { 16640, 7717, 3804, 1975, 1677, 781, 160, 12, 1, 1 }, + { 16768, 7712, 3775, 1947, 1642, 757, 153, 12, 1, 1 }, + { 16896, 7706, 3745, 1920, 1608, 734, 146, 11, 1, 1 }, + { 17024, 7699, 3714, 1893, 1575, 711, 140, 10, 1, 1 }, + { 17152, 7690, 3684, 1866, 1541, 689, 134, 10, 1, 1 }, + { 17280, 7681, 3653, 1839, 1508, 668, 128, 9, 1, 1 }, + { 17408, 7671, 3621, 1812, 1476, 647, 122, 9, 1, 1 }, + { 17536, 7660, 3590, 1785, 1444, 626, 117, 8, 1, 1 }, + { 17664, 7648, 3558, 1758, 1412, 606, 112, 8, 1, 1 }, + { 17792, 7635, 3526, 1731, 1381, 587, 107, 7, 1, 1 }, + { 17920, 7622, 3493, 1704, 1350, 568, 102, 7, 1, 1 }, + { 18048, 7607, 3461, 1678, 1319, 549, 98, 6, 1, 1 }, + { 18176, 7592, 3428, 1651, 1289, 531, 93, 6, 1, 1 }, + { 18304, 7575, 3394, 1625, 1259, 514, 89, 6, 1, 1 }, + { 18432, 7558, 3361, 1598, 1230, 497, 85, 5, 1, 1 }, + { 18560, 7540, 3327, 1572, 1201, 480, 81, 5, 1, 1 }, + { 18688, 7520, 3293, 1546, 1173, 464, 77, 5, 1, 1 }, + { 18816, 7500, 3258, 1520, 1145, 448, 74, 5, 1, 1 }, + { 18944, 7480, 3224, 1494, 1117, 433, 70, 4, 1, 1 }, + { 19072, 7458, 3189, 1468, 1090, 418, 67, 4, 1, 1 }, + { 19200, 7435, 3154, 1442, 1063, 404, 64, 4, 1, 1 }, + { 19328, 7410, 3119, 1417, 1037, 390, 61, 4, 1, 1 }, + { 19456, 7386, 3084, 1392, 1011, 376, 58, 3, 1, 1 }, + { 19584, 7361, 3048, 1366, 986, 363, 55, 3, 1, 1 }, + { 19712, 7335, 3012, 1341, 960, 350, 53, 3, 1, 1 }, + { 19840, 7307, 2977, 1316, 936, 337, 50, 3, 1, 1 }, + { 19968, 7279, 2941, 1291, 911, 325, 48, 3, 1, 1 }, + { 20096, 7251, 2905, 1267, 887, 313, 45, 2, 1, 1 }, + { 20224, 7220, 2868, 1243, 864, 302, 43, 2, 1, 1 }, + { 20352, 7189, 2832, 1218, 841, 291, 41, 2, 1, 1 }, + { 20480, 7158, 2795, 1194, 818, 280, 39, 2, 1, 1 }, + { 20608, 7124, 2759, 1170, 796, 270, 37, 2, 1, 1 }, + { 20736, 7091, 2722, 1147, 774, 259, 35, 2, 1, 1 }, + { 20864, 7056, 2685, 1123, 752, 250, 34, 2, 1, 1 }, + { 20992, 7021, 2648, 1100, 731, 240, 32, 2, 1, 1 }, + { 21120, 6985, 2612, 1077, 710, 231, 30, 1, 1, 1 }, + { 21248, 6948, 2574, 1054, 690, 222, 29, 1, 1, 1 }, + { 21376, 6911, 2537, 1031, 670, 213, 27, 1, 1, 1 }, + { 21504, 6872, 2500, 1008, 650, 205, 26, 1, 1, 1 }, + { 21632, 6831, 2463, 986, 631, 197, 25, 1, 1, 1 }, + { 21760, 6791, 2426, 964, 612, 189, 23, 1, 1, 1 }, + { 21888, 6749, 2389, 942, 594, 181, 22, 1, 1, 1 }, + { 22016, 6707, 2351, 921, 575, 174, 21, 1, 1, 1 }, + { 22144, 6663, 2314, 899, 558, 167, 20, 1, 1, 1 }, + { 22272, 6619, 2277, 878, 540, 160, 19, 1, 1, 1 }, + { 22400, 6574, 2240, 857, 523, 153, 18, 1, 1, 1 }, + { 22528, 6529, 2202, 836, 507, 146, 17, 1, 1, 1 }, + { 22656, 6482, 2165, 816, 490, 140, 16, 1, 1, 1 }, + { 22784, 6435, 2128, 795, 474, 134, 15, 1, 1, 1 }, + { 22912, 6386, 2091, 775, 459, 128, 14, 1, 1, 1 }, + { 23040, 6336, 2054, 756, 443, 123, 13, 1, 1, 1 }, + { 23168, 6286, 2017, 736, 428, 117, 13, 1, 1, 1 }, + { 23296, 6234, 1980, 717, 414, 112, 12, 1, 1, 1 }, + { 23424, 6183, 1943, 698, 399, 107, 11, 1, 1, 1 }, + { 23552, 6130, 1906, 679, 385, 102, 11, 1, 1, 1 }, + { 23680, 6077, 1869, 660, 372, 97, 10, 1, 1, 1 }, + { 23808, 6022, 1833, 642, 358, 93, 9, 1, 1, 1 }, + { 23936, 5966, 1796, 624, 345, 89, 9, 1, 1, 1 }, + { 24064, 5910, 1760, 606, 333, 84, 8, 1, 1, 1 }, + { 24192, 5853, 1724, 588, 320, 80, 8, 1, 1, 1 }, + { 24320, 5796, 1687, 571, 308, 76, 7, 1, 1, 1 }, + { 24448, 5735, 1651, 554, 297, 73, 7, 1, 1, 1 }, + { 24576, 5677, 1615, 537, 285, 69, 6, 1, 1, 1 }, + { 24704, 5615, 1579, 521, 274, 66, 6, 1, 1, 1 }, + { 24832, 5554, 1544, 504, 263, 62, 6, 1, 1, 1 }, + { 24960, 5492, 1508, 488, 253, 59, 5, 1, 1, 1 }, + { 25088, 5428, 1473, 473, 242, 56, 5, 1, 1, 1 }, + { 25216, 5364, 1438, 457, 232, 53, 5, 1, 1, 1 }, + { 25344, 5300, 1403, 442, 222, 50, 4, 1, 1, 1 }, + { 25472, 5233, 1368, 427, 213, 48, 4, 1, 1, 1 }, + { 25600, 5166, 1334, 412, 204, 45, 4, 1, 1, 1 }, + { 25728, 5098, 1299, 398, 195, 43, 4, 1, 1, 1 }, + { 25856, 5030, 1266, 384, 186, 40, 3, 1, 1, 1 }, + { 25984, 4960, 1232, 370, 178, 38, 3, 1, 1, 1 }, + { 26112, 4890, 1198, 356, 170, 36, 3, 1, 1, 1 }, + { 26240, 4819, 1164, 343, 162, 34, 3, 1, 1, 1 }, + { 26368, 4748, 1132, 329, 154, 32, 2, 1, 1, 1 }, + { 26496, 4675, 1098, 317, 147, 30, 2, 1, 1, 1 }, + { 26624, 4602, 1066, 304, 139, 28, 2, 1, 1, 1 }, + { 26752, 4527, 1034, 292, 132, 26, 2, 1, 1, 1 }, + { 26880, 4451, 1001, 280, 126, 25, 2, 1, 1, 1 }, + { 27008, 4375, 970, 268, 119, 23, 2, 1, 1, 1 }, + { 27136, 4299, 938, 256, 113, 21, 2, 1, 1, 1 }, + { 27264, 4221, 907, 245, 107, 20, 1, 1, 1, 1 }, + { 27392, 4142, 876, 234, 101, 19, 1, 1, 1, 1 }, + { 27520, 4063, 846, 223, 95, 17, 1, 1, 1, 1 }, + { 27648, 3982, 815, 213, 90, 16, 1, 1, 1, 1 }, + { 27776, 3900, 786, 202, 85, 15, 1, 1, 1, 1 }, + { 27904, 3818, 756, 192, 80, 14, 1, 1, 1, 1 }, + { 28032, 3734, 727, 183, 75, 13, 1, 1, 1, 1 }, + { 28160, 3651, 698, 173, 70, 12, 1, 1, 1, 1 }, + { 28288, 3566, 669, 164, 66, 11, 1, 1, 1, 1 }, + { 28416, 3481, 641, 155, 61, 10, 1, 1, 1, 1 }, + { 28544, 3393, 614, 147, 57, 9, 1, 1, 1, 1 }, + { 28672, 3306, 586, 138, 53, 9, 1, 1, 1, 1 }, + { 28800, 3217, 559, 130, 50, 8, 1, 1, 1, 1 }, + { 28928, 3128, 533, 122, 46, 7, 1, 1, 1, 1 }, + { 29056, 3037, 507, 114, 43, 7, 1, 1, 1, 1 }, + { 29184, 2947, 481, 107, 39, 6, 1, 1, 1, 1 }, + { 29312, 2855, 456, 100, 36, 5, 1, 1, 1, 1 }, + { 29440, 2762, 431, 93, 33, 5, 1, 1, 1, 1 }, + { 29568, 2668, 407, 86, 31, 4, 1, 1, 1, 1 }, + { 29696, 2573, 383, 80, 28, 4, 1, 1, 1, 1 }, + { 29824, 2478, 359, 74, 25, 4, 1, 1, 1, 1 }, + { 29952, 2381, 337, 68, 23, 3, 1, 1, 1, 1 }, + { 30080, 2284, 314, 62, 21, 3, 1, 1, 1, 1 }, + { 30208, 2185, 293, 57, 19, 2, 1, 1, 1, 1 }, + { 30336, 2086, 271, 52, 17, 2, 1, 1, 1, 1 }, + { 30464, 1986, 250, 47, 15, 2, 1, 1, 1, 1 }, + { 30592, 1885, 230, 42, 13, 2, 1, 1, 1, 1 }, + { 30720, 1782, 211, 38, 12, 1, 1, 1, 1, 1 }, + { 30848, 1679, 192, 34, 10, 1, 1, 1, 1, 1 }, + { 30976, 1575, 173, 30, 9, 1, 1, 1, 1, 1 }, + { 31104, 1469, 156, 26, 8, 1, 1, 1, 1, 1 }, + { 31232, 1364, 138, 23, 6, 1, 1, 1, 1, 1 }, + { 31360, 1257, 122, 19, 5, 1, 1, 1, 1, 1 }, + { 31488, 1149, 106, 16, 4, 1, 1, 1, 1, 1 }, + { 31616, 1038, 91, 14, 4, 1, 1, 1, 1, 1 }, + { 31744, 928, 77, 11, 3, 1, 1, 1, 1, 1 }, + { 31872, 816, 64, 9, 2, 1, 1, 1, 1, 1 }, + { 32000, 703, 51, 7, 2, 1, 1, 1, 1, 1 }, + { 32128, 589, 40, 5, 1, 1, 1, 1, 1, 1 }, + { 32256, 473, 29, 4, 1, 1, 1, 1, 1, 1 }, + { 32384, 357, 19, 2, 1, 1, 1, 1, 1, 1 }, + { 32512, 238, 11, 1, 1, 1, 1, 1, 1, 1 }, + { 32640, 117, 4, 1, 1, 1, 1, 1, 1, 1 }, + }; +#endif // CONFIG_NEW_TOKENSET + +/* clang-format off */ +#if CONFIG_Q_ADAPT_PROBS +const av1_coeff_probs_model +default_qctx_coef_probs[QCTX_BINS][TX_SIZES][PLANE_TYPES] = { + { // Q_Index 0 +#if CONFIG_CB4X4 + { // TX_SIZE 0 + { // Y plane + { // Intra + { // band 0 + {182, 34, 137}, { 79, 39, 103}, { 10, 28, 51}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 45, 88, 147}, { 46, 80, 140}, { 25, 69, 119}, + { 12, 57, 96}, { 4, 41, 65}, { 1, 20, 31}, + }, + { // band 2 + { 58, 124, 190}, { 39, 106, 178}, { 16, 86, 147}, + { 7, 69, 114}, { 3, 50, 80}, { 1, 25, 42}, + }, + { // band 3 + { 90, 138, 215}, { 54, 116, 198}, { 18, 86, 155}, + { 5, 62, 112}, { 1, 38, 68}, { 1, 17, 30}, + }, + { // band 4 + {126, 149, 231}, { 82, 114, 211}, { 21, 80, 157}, + { 6, 56, 105}, { 1, 36, 64}, { 1, 17, 31}, + }, + { // band 5 + {171, 56, 236}, {140, 54, 219}, { 57, 45, 167}, + { 26, 36, 113}, { 11, 29, 72}, { 3, 18, 39}, + }, + }, + { // Intra + { // band 0 + {153, 122, 186}, {106, 109, 171}, { 36, 84, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 27, 151, 201}, { 34, 131, 199}, { 23, 102, 161}, + { 10, 80, 120}, { 4, 52, 78}, { 1, 24, 37}, + }, + { // band 2 + { 43, 158, 213}, { 35, 133, 203}, { 8, 92, 151}, + { 2, 64, 106}, { 1, 36, 60}, { 1, 13, 24}, + }, + { // band 3 + { 68, 167, 223}, { 36, 135, 211}, { 9, 94, 157}, + { 2, 67, 112}, { 1, 40, 68}, { 1, 17, 31}, + }, + { // band 4 + {131, 146, 237}, { 72, 119, 223}, { 17, 82, 164}, + { 4, 55, 107}, { 1, 34, 63}, { 1, 16, 29}, + }, + { // band 5 + {184, 68, 244}, {153, 59, 232}, { 68, 51, 179}, + { 31, 40, 123}, { 13, 29, 77}, { 4, 17, 37}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {203, 41, 203}, {127, 56, 174}, { 49, 56, 127}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {110, 121, 217}, {119, 113, 213}, { 64, 95, 185}, + { 30, 72, 144}, { 8, 42, 76}, { 2, 17, 25}, + }, + { // band 2 + {127, 159, 229}, {115, 134, 223}, { 36, 100, 189}, + { 11, 75, 142}, { 3, 48, 83}, { 1, 19, 33}, + }, + { // band 3 + {150, 172, 241}, { 90, 133, 231}, { 28, 102, 192}, + { 7, 81, 147}, { 1, 53, 91}, { 1, 25, 42}, + }, + { // band 4 + {184, 144, 248}, {114, 117, 237}, { 37, 89, 192}, + { 10, 63, 130}, { 4, 42, 76}, { 1, 19, 38}, + }, + { // band 5 + {207, 79, 250}, {179, 74, 241}, { 83, 67, 199}, + { 38, 51, 142}, { 17, 37, 97}, { 10, 14, 55}, + }, + }, + { // Inter + { // band 0 + {220, 82, 232}, {150, 93, 214}, { 66, 95, 177}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {116, 160, 227}, {136, 141, 227}, { 67, 114, 190}, + { 40, 94, 148}, { 21, 70, 107}, { 10, 43, 63}, + }, + { // band 2 + {124, 173, 235}, {105, 147, 226}, { 27, 107, 184}, + { 10, 80, 142}, { 3, 50, 86}, { 1, 16, 32}, + }, + { // band 3 + {149, 179, 243}, { 89, 147, 234}, { 29, 112, 193}, + { 9, 94, 157}, { 1, 64, 111}, { 1, 25, 43}, + }, + { // band 4 + {187, 153, 248}, {127, 130, 241}, { 52, 99, 202}, + { 20, 79, 152}, { 4, 50, 93}, { 1, 19, 32}, + }, + { // band 5 + {215, 82, 251}, {195, 80, 246}, { 93, 70, 204}, + { 39, 54, 147}, { 14, 33, 88}, { 6, 14, 39}, + }, + }, + }, + }, +#endif + { // TX_SIZE 0 + { // Y plane + { // Intra + { // band 0 + {182, 34, 137}, { 79, 39, 103}, { 10, 28, 51}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 45, 88, 147}, { 46, 80, 140}, { 25, 69, 119}, + { 12, 57, 96}, { 4, 41, 65}, { 1, 20, 31}, + }, + { // band 2 + { 58, 124, 190}, { 39, 106, 178}, { 16, 86, 147}, + { 7, 69, 114}, { 3, 50, 80}, { 1, 25, 42}, + }, + { // band 3 + { 90, 138, 215}, { 54, 116, 198}, { 18, 86, 155}, + { 5, 62, 112}, { 1, 38, 68}, { 1, 17, 30}, + }, + { // band 4 + {126, 149, 231}, { 82, 114, 211}, { 21, 80, 157}, + { 6, 56, 105}, { 1, 36, 64}, { 1, 17, 31}, + }, + { // band 5 + {171, 56, 236}, {140, 54, 219}, { 57, 45, 167}, + { 26, 36, 113}, { 11, 29, 72}, { 3, 18, 39}, + }, + }, + { // Intra + { // band 0 + {153, 122, 186}, {106, 109, 171}, { 36, 84, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 27, 151, 201}, { 34, 131, 199}, { 23, 102, 161}, + { 10, 80, 120}, { 4, 52, 78}, { 1, 24, 37}, + }, + { // band 2 + { 43, 158, 213}, { 35, 133, 203}, { 8, 92, 151}, + { 2, 64, 106}, { 1, 36, 60}, { 1, 13, 24}, + }, + { // band 3 + { 68, 167, 223}, { 36, 135, 211}, { 9, 94, 157}, + { 2, 67, 112}, { 1, 40, 68}, { 1, 17, 31}, + }, + { // band 4 + {131, 146, 237}, { 72, 119, 223}, { 17, 82, 164}, + { 4, 55, 107}, { 1, 34, 63}, { 1, 16, 29}, + }, + { // band 5 + {184, 68, 244}, {153, 59, 232}, { 68, 51, 179}, + { 31, 40, 123}, { 13, 29, 77}, { 4, 17, 37}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {203, 41, 203}, {127, 56, 174}, { 49, 56, 127}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {110, 121, 217}, {119, 113, 213}, { 64, 95, 185}, + { 30, 72, 144}, { 8, 42, 76}, { 2, 17, 25}, + }, + { // band 2 + {127, 159, 229}, {115, 134, 223}, { 36, 100, 189}, + { 11, 75, 142}, { 3, 48, 83}, { 1, 19, 33}, + }, + { // band 3 + {150, 172, 241}, { 90, 133, 231}, { 28, 102, 192}, + { 7, 81, 147}, { 1, 53, 91}, { 1, 25, 42}, + }, + { // band 4 + {184, 144, 248}, {114, 117, 237}, { 37, 89, 192}, + { 10, 63, 130}, { 4, 42, 76}, { 1, 19, 38}, + }, + { // band 5 + {207, 79, 250}, {179, 74, 241}, { 83, 67, 199}, + { 38, 51, 142}, { 17, 37, 97}, { 10, 14, 55}, + }, + }, + { // Inter + { // band 0 + {220, 82, 232}, {150, 93, 214}, { 66, 95, 177}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {116, 160, 227}, {136, 141, 227}, { 67, 114, 190}, + { 40, 94, 148}, { 21, 70, 107}, { 10, 43, 63}, + }, + { // band 2 + {124, 173, 235}, {105, 147, 226}, { 27, 107, 184}, + { 10, 80, 142}, { 3, 50, 86}, { 1, 16, 32}, + }, + { // band 3 + {149, 179, 243}, { 89, 147, 234}, { 29, 112, 193}, + { 9, 94, 157}, { 1, 64, 111}, { 1, 25, 43}, + }, + { // band 4 + {187, 153, 248}, {127, 130, 241}, { 52, 99, 202}, + { 20, 79, 152}, { 4, 50, 93}, { 1, 19, 32}, + }, + { // band 5 + {215, 82, 251}, {195, 80, 246}, { 93, 70, 204}, + { 39, 54, 147}, { 14, 33, 88}, { 6, 14, 39}, + }, + }, + }, + }, + { // TX_SIZE 1 + { // Y plane + { // Intra + { // band 0 + {116, 43, 131}, { 39, 41, 94}, { 4, 28, 47}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 28, 101, 141}, { 27, 95, 140}, { 18, 80, 121}, + { 10, 61, 95}, { 4, 39, 60}, { 1, 19, 26}, + }, + { // band 2 + { 29, 150, 183}, { 19, 127, 175}, { 8, 98, 147}, + { 3, 76, 115}, { 1, 55, 84}, { 1, 29, 43}, + }, + { // band 3 + { 26, 168, 202}, { 12, 138, 188}, { 2, 98, 149}, + { 1, 69, 110}, { 1, 40, 65}, { 1, 17, 25}, + }, + { // band 4 + { 33, 188, 225}, { 12, 155, 207}, { 2, 101, 155}, + { 1, 65, 106}, { 1, 36, 60}, { 1, 18, 26}, + }, + { // band 5 + { 79, 205, 242}, { 30, 168, 224}, { 5, 106, 164}, + { 1, 68, 110}, { 1, 39, 65}, { 1, 18, 28}, + }, + }, + { // Intra + { // band 0 + { 96, 80, 201}, { 51, 88, 168}, { 14, 78, 116}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 6, 167, 216}, { 32, 152, 211}, { 24, 121, 182}, + { 13, 98, 149}, { 12, 76, 108}, { 8, 48, 62}, + }, + { // band 2 + { 17, 176, 225}, { 13, 147, 209}, { 3, 96, 155}, + { 1, 65, 108}, { 2, 43, 63}, { 2, 23, 25}, + }, + { // band 3 + { 18, 183, 232}, { 10, 153, 214}, { 1, 96, 154}, + { 1, 63, 105}, { 1, 39, 59}, { 1, 21, 24}, + }, + { // band 4 + { 23, 191, 239}, { 8, 159, 221}, { 1, 97, 158}, + { 1, 61, 105}, { 1, 37, 60}, { 1, 20, 26}, + }, + { // band 5 + { 70, 201, 243}, { 29, 163, 228}, { 4, 102, 169}, + { 1, 67, 114}, { 1, 39, 66}, { 1, 17, 29}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {181, 38, 192}, { 95, 47, 151}, { 29, 49, 102}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 72, 131, 202}, { 93, 120, 205}, { 50, 103, 179}, + { 24, 79, 143}, { 11, 47, 78}, { 7, 19, 25}, + }, + { // band 2 + { 84, 176, 221}, { 56, 144, 214}, { 21, 108, 182}, + { 8, 83, 139}, { 3, 55, 90}, { 2, 27, 41}, + }, + { // band 3 + { 84, 195, 234}, { 42, 156, 222}, { 10, 109, 180}, + { 4, 77, 133}, { 1, 48, 80}, { 1, 23, 35}, + }, + { // band 4 + { 89, 210, 238}, { 35, 165, 221}, { 6, 106, 172}, + { 2, 70, 123}, { 1, 44, 74}, { 1, 21, 30}, + }, + { // band 5 + {114, 221, 247}, { 49, 170, 234}, { 7, 113, 184}, + { 2, 77, 132}, { 1, 48, 79}, { 1, 25, 33}, + }, + }, + { // Inter + { // band 0 + {192, 66, 237}, {113, 84, 211}, { 35, 84, 154}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 81, 180, 234}, {127, 165, 229}, { 58, 137, 204}, + { 41, 114, 174}, { 44, 94, 136}, { 29, 66, 86}, + }, + { // band 2 + { 82, 193, 240}, { 39, 162, 223}, { 8, 113, 179}, + { 3, 83, 136}, { 6, 62, 84}, { 5, 45, 45}, + }, + { // band 3 + { 78, 203, 242}, { 31, 170, 227}, { 4, 115, 181}, + { 1, 82, 135}, { 2, 59, 82}, { 1, 45, 47}, + }, + { // band 4 + { 76, 210, 239}, { 25, 170, 213}, { 2, 99, 152}, + { 1, 69, 115}, { 1, 49, 80}, { 1, 47, 57}, + }, + { // band 5 + {103, 217, 250}, { 42, 180, 237}, { 3, 124, 191}, + { 1, 90, 150}, { 1, 69, 116}, { 1, 52, 46}, + }, + }, + }, + }, + { // TX_SIZE 2 + { // Y plane + { // Intra + { // band 0 + { 58, 38, 99}, { 9, 26, 51}, { 1, 14, 22}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 14, 78, 109}, { 16, 73, 105}, { 11, 62, 92}, + { 6, 47, 72}, { 2, 29, 45}, { 1, 12, 18}, + }, + { // band 2 + { 17, 131, 148}, { 11, 112, 140}, { 5, 87, 118}, + { 2, 63, 90}, { 1, 42, 63}, { 1, 19, 31}, + }, + { // band 3 + { 12, 151, 168}, { 6, 116, 152}, { 1, 76, 115}, + { 1, 50, 81}, { 1, 32, 52}, { 1, 14, 23}, + }, + { // band 4 + { 10, 174, 191}, { 3, 130, 172}, { 1, 80, 126}, + { 1, 53, 88}, { 1, 32, 55}, { 1, 14, 24}, + }, + { // band 5 + { 19, 219, 237}, { 3, 168, 211}, { 1, 90, 142}, + { 1, 53, 91}, { 1, 29, 51}, { 1, 12, 21}, + }, + }, + { // Intra + { // band 0 + { 21, 46, 184}, { 10, 53, 130}, { 2, 49, 78}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 3, 169, 198}, { 37, 165, 196}, { 26, 134, 176}, + { 11, 108, 149}, { 5, 81, 112}, { 3, 47, 64}, + }, + { // band 2 + { 11, 183, 215}, { 8, 142, 192}, { 2, 91, 141}, + { 1, 62, 100}, { 1, 38, 62}, { 1, 17, 28}, + }, + { // band 3 + { 12, 190, 223}, { 6, 149, 199}, { 1, 88, 139}, + { 1, 56, 93}, { 1, 31, 54}, { 1, 13, 21}, + }, + { // band 4 + { 11, 197, 230}, { 3, 154, 204}, { 1, 83, 134}, + { 1, 50, 86}, { 1, 28, 49}, { 1, 12, 21}, + }, + { // band 5 + { 17, 211, 240}, { 2, 167, 217}, { 1, 88, 143}, + { 1, 53, 91}, { 1, 30, 53}, { 1, 14, 24}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {151, 30, 151}, { 50, 36, 105}, { 8, 34, 66}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 39, 111, 160}, { 62, 111, 165}, { 37, 99, 147}, + { 15, 77, 118}, { 3, 47, 73}, { 1, 17, 27}, + }, + { // band 2 + { 48, 170, 190}, { 32, 135, 180}, { 11, 100, 149}, + { 4, 76, 116}, { 1, 51, 80}, { 1, 22, 36}, + }, + { // band 3 + { 39, 191, 208}, { 18, 141, 191}, { 3, 96, 150}, + { 1, 66, 110}, { 1, 41, 69}, { 1, 17, 28}, + }, + { // band 4 + { 32, 209, 219}, { 8, 152, 201}, { 1, 96, 153}, + { 1, 63, 106}, { 1, 38, 66}, { 1, 17, 29}, + }, + { // band 5 + { 33, 230, 237}, { 5, 173, 214}, { 1, 100, 155}, + { 1, 62, 105}, { 1, 38, 66}, { 1, 18, 32}, + }, + }, + { // Inter + { // band 0 + {149, 38, 231}, { 59, 51, 186}, { 12, 54, 117}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 53, 179, 226}, {126, 176, 223}, { 58, 147, 202}, + { 28, 118, 174}, { 15, 94, 138}, { 14, 63, 87}, + }, + { // band 2 + { 58, 196, 232}, { 26, 158, 213}, { 5, 106, 166}, + { 1, 75, 124}, { 1, 46, 79}, { 1, 23, 39}, + }, + { // band 3 + { 46, 203, 235}, { 17, 162, 213}, { 2, 104, 165}, + { 1, 72, 120}, { 1, 44, 74}, { 1, 22, 33}, + }, + { // band 4 + { 37, 213, 238}, { 8, 167, 216}, { 1, 104, 168}, + { 1, 68, 119}, { 1, 40, 67}, { 1, 17, 29}, + }, + { // band 5 + { 30, 228, 239}, { 4, 181, 213}, { 1, 103, 153}, + { 1, 65, 110}, { 1, 43, 79}, { 1, 27, 56}, + }, + }, + }, + }, + { // TX_SIZE 3 + { // Y plane + { // Intra + { // band 0 + { 76, 25, 53}, { 9, 18, 32}, { 1, 12, 18}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 29, 55, 91}, { 19, 58, 95}, { 15, 57, 89}, + { 12, 49, 77}, { 3, 29, 44}, { 1, 8, 12}, + }, + { // band 2 + { 32, 160, 148}, { 33, 143, 146}, { 19, 122, 132}, + { 6, 90, 102}, { 1, 58, 70}, { 1, 17, 24}, + }, + { // band 3 + { 16, 181, 181}, { 6, 142, 165}, { 1, 90, 120}, + { 1, 50, 71}, { 1, 25, 38}, { 1, 9, 14}, + }, + { // band 4 + { 13, 203, 203}, { 3, 154, 176}, { 1, 80, 108}, + { 1, 41, 61}, { 1, 24, 37}, { 1, 11, 17}, + }, + { // band 5 + { 6, 234, 240}, { 1, 178, 204}, { 1, 80, 119}, + { 1, 45, 71}, { 1, 26, 42}, { 1, 12, 19}, + }, + }, + { // Intra + { // band 0 + { 78, 20, 135}, { 25, 18, 101}, { 5, 19, 57}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 7, 144, 183}, {117, 151, 195}, {109, 151, 187}, + { 39, 130, 168}, { 11, 100, 125}, { 4, 59, 64}, + }, + { // band 2 + { 20, 184, 212}, { 12, 148, 191}, { 2, 98, 141}, + { 1, 65, 100}, { 1, 39, 61}, { 1, 14, 22}, + }, + { // band 3 + { 15, 194, 222}, { 6, 153, 198}, { 1, 92, 138}, + { 1, 58, 91}, { 1, 32, 52}, { 1, 12, 18}, + }, + { // band 4 + { 14, 206, 232}, { 3, 162, 206}, { 1, 89, 134}, + { 1, 52, 83}, { 1, 28, 46}, { 1, 11, 17}, + }, + { // band 5 + { 6, 225, 241}, { 1, 175, 210}, { 1, 81, 125}, + { 1, 48, 78}, { 1, 28, 46}, { 1, 13, 21}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {124, 23, 93}, { 31, 24, 63}, { 6, 24, 46}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 23, 86, 126}, { 45, 90, 145}, { 31, 91, 133}, + { 19, 80, 114}, { 7, 53, 72}, { 1, 20, 27}, + }, + { // band 2 + { 51, 186, 189}, { 48, 159, 182}, { 33, 128, 156}, + { 15, 92, 124}, { 2, 62, 83}, { 1, 29, 43}, + }, + { // band 3 + { 36, 198, 211}, { 15, 156, 187}, { 3, 97, 137}, + { 1, 61, 93}, { 1, 35, 57}, { 1, 15, 23}, + }, + { // band 4 + { 34, 219, 223}, { 9, 162, 193}, { 1, 91, 136}, + { 1, 58, 92}, { 1, 35, 54}, { 1, 14, 23}, + }, + { // band 5 + { 19, 243, 243}, { 3, 191, 208}, { 1, 91, 137}, + { 1, 56, 90}, { 1, 34, 55}, { 1, 16, 24}, + }, + }, + { // Inter + { // band 0 + {119, 20, 197}, { 19, 29, 156}, { 3, 30, 107}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 24, 192, 226}, {161, 193, 227}, { 97, 185, 222}, + { 31, 158, 204}, { 16, 122, 165}, { 17, 84, 112}, + }, + { // band 2 + { 26, 202, 229}, { 11, 165, 210}, { 2, 103, 152}, + { 1, 68, 104}, { 1, 42, 70}, { 1, 16, 36}, + }, + { // band 3 + { 24, 209, 237}, { 6, 169, 214}, { 1, 102, 154}, + { 1, 65, 107}, { 1, 45, 68}, { 1, 17, 24}, + }, + { // band 4 + { 19, 219, 243}, { 4, 183, 226}, { 1, 115, 172}, + { 1, 73, 119}, { 1, 43, 77}, { 1, 15, 37}, + }, + { // band 5 + { 11, 237, 241}, { 2, 190, 216}, { 1, 108, 146}, + { 1, 59, 94}, { 1, 40, 67}, { 1, 30, 53}, + }, + }, + }, + }, +#if CONFIG_TX64X64 + { // TX_SIZE 4 + { // Y plane + { // Intra + { // band 0 + { 76, 25, 53}, { 9, 18, 32}, { 1, 12, 18}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 29, 55, 91}, { 19, 58, 95}, { 15, 57, 89}, + { 12, 49, 77}, { 3, 29, 44}, { 1, 8, 12}, + }, + { // band 2 + { 32, 160, 148}, { 33, 143, 146}, { 19, 122, 132}, + { 6, 90, 102}, { 1, 58, 70}, { 1, 17, 24}, + }, + { // band 3 + { 16, 181, 181}, { 6, 142, 165}, { 1, 90, 120}, + { 1, 50, 71}, { 1, 25, 38}, { 1, 9, 14}, + }, + { // band 4 + { 13, 203, 203}, { 3, 154, 176}, { 1, 80, 108}, + { 1, 41, 61}, { 1, 24, 37}, { 1, 11, 17}, + }, + { // band 5 + { 6, 234, 240}, { 1, 178, 204}, { 1, 80, 119}, + { 1, 45, 71}, { 1, 26, 42}, { 1, 12, 19}, + }, + }, + { // Intra + { // band 0 + { 78, 20, 135}, { 25, 18, 101}, { 5, 19, 57}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 7, 144, 183}, {117, 151, 195}, {109, 151, 187}, + { 39, 130, 168}, { 11, 100, 125}, { 4, 59, 64}, + }, + { // band 2 + { 20, 184, 212}, { 12, 148, 191}, { 2, 98, 141}, + { 1, 65, 100}, { 1, 39, 61}, { 1, 14, 22}, + }, + { // band 3 + { 15, 194, 222}, { 6, 153, 198}, { 1, 92, 138}, + { 1, 58, 91}, { 1, 32, 52}, { 1, 12, 18}, + }, + { // band 4 + { 14, 206, 232}, { 3, 162, 206}, { 1, 89, 134}, + { 1, 52, 83}, { 1, 28, 46}, { 1, 11, 17}, + }, + { // band 5 + { 6, 225, 241}, { 1, 175, 210}, { 1, 81, 125}, + { 1, 48, 78}, { 1, 28, 46}, { 1, 13, 21}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {124, 23, 93}, { 31, 24, 63}, { 6, 24, 46}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 23, 86, 126}, { 45, 90, 145}, { 31, 91, 133}, + { 19, 80, 114}, { 7, 53, 72}, { 1, 20, 27}, + }, + { // band 2 + { 51, 186, 189}, { 48, 159, 182}, { 33, 128, 156}, + { 15, 92, 124}, { 2, 62, 83}, { 1, 29, 43}, + }, + { // band 3 + { 36, 198, 211}, { 15, 156, 187}, { 3, 97, 137}, + { 1, 61, 93}, { 1, 35, 57}, { 1, 15, 23}, + }, + { // band 4 + { 34, 219, 223}, { 9, 162, 193}, { 1, 91, 136}, + { 1, 58, 92}, { 1, 35, 54}, { 1, 14, 23}, + }, + { // band 5 + { 19, 243, 243}, { 3, 191, 208}, { 1, 91, 137}, + { 1, 56, 90}, { 1, 34, 55}, { 1, 16, 24}, + }, + }, + { // Inter + { // band 0 + {119, 20, 197}, { 19, 29, 156}, { 3, 30, 107}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 24, 192, 226}, {161, 193, 227}, { 97, 185, 222}, + { 31, 158, 204}, { 16, 122, 165}, { 17, 84, 112}, + }, + { // band 2 + { 26, 202, 229}, { 11, 165, 210}, { 2, 103, 152}, + { 1, 68, 104}, { 1, 42, 70}, { 1, 16, 36}, + }, + { // band 3 + { 24, 209, 237}, { 6, 169, 214}, { 1, 102, 154}, + { 1, 65, 107}, { 1, 45, 68}, { 1, 17, 24}, + }, + { // band 4 + { 19, 219, 243}, { 4, 183, 226}, { 1, 115, 172}, + { 1, 73, 119}, { 1, 43, 77}, { 1, 15, 37}, + }, + { // band 5 + { 11, 237, 241}, { 2, 190, 216}, { 1, 108, 146}, + { 1, 59, 94}, { 1, 40, 67}, { 1, 30, 53}, + }, + }, + }, + }, +#endif // CONFIG_TX64X64 + }, + { // Q_Index 1 +#if CONFIG_CB4X4 + { // TX_SIZE 0 + { // Y plane + { // Intra + { // band 0 + {174, 30, 159}, { 76, 38, 115}, { 15, 33, 65}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 60, 80, 153}, { 72, 75, 147}, { 36, 68, 129}, + { 15, 59, 104}, { 4, 45, 74}, { 1, 28, 45}, + }, + { // band 2 + { 70, 122, 186}, { 55, 104, 175}, { 21, 83, 144}, + { 8, 67, 112}, { 2, 51, 82}, { 1, 34, 57}, + }, + { // band 3 + { 97, 144, 207}, { 52, 109, 195}, { 16, 77, 153}, + { 4, 58, 113}, { 1, 43, 77}, { 1, 27, 48}, + }, + { // band 4 + {128, 148, 229}, { 76, 104, 210}, { 18, 77, 159}, + { 4, 65, 110}, { 1, 52, 82}, { 1, 31, 55}, + }, + { // band 5 + {165, 51, 238}, {128, 50, 230}, { 57, 49, 185}, + { 28, 47, 130}, { 12, 44, 96}, { 3, 36, 60}, + }, + }, + { // Intra + { // band 0 + {169, 103, 203}, {117, 96, 176}, { 56, 81, 137}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 31, 150, 224}, { 49, 128, 212}, { 19, 92, 165}, + { 6, 67, 116}, { 2, 43, 71}, { 1, 21, 36}, + }, + { // band 2 + { 58, 156, 230}, { 47, 130, 215}, { 7, 87, 158}, + { 2, 63, 114}, { 1, 39, 71}, { 1, 18, 36}, + }, + { // band 3 + { 85, 176, 240}, { 43, 138, 226}, { 8, 93, 172}, + { 2, 70, 127}, { 1, 46, 81}, { 1, 26, 47}, + }, + { // band 4 + {155, 144, 248}, { 93, 116, 235}, { 21, 83, 180}, + { 4, 59, 119}, { 1, 43, 80}, { 1, 25, 50}, + }, + { // band 5 + {203, 61, 250}, {171, 57, 243}, { 71, 57, 199}, + { 31, 49, 144}, { 13, 42, 96}, { 7, 30, 52}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {204, 44, 204}, {137, 57, 184}, { 72, 62, 152}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {145, 117, 236}, {151, 112, 231}, { 87, 95, 208}, + { 31, 77, 165}, { 5, 49, 98}, { 1, 24, 39}, + }, + { // band 2 + {146, 152, 241}, {140, 132, 236}, { 41, 103, 209}, + { 10, 86, 165}, { 2, 55, 106}, { 1, 25, 58}, + }, + { // band 3 + {154, 181, 249}, { 84, 143, 240}, { 23, 114, 210}, + { 6, 102, 182}, { 2, 71, 137}, { 1, 35, 90}, + }, + { // band 4 + {184, 150, 251}, {115, 130, 244}, { 34, 105, 215}, + { 15, 89, 173}, { 1, 51, 141}, {128, 128, 128}, + }, + { // band 5 + {211, 71, 253}, {193, 78, 249}, {106, 91, 232}, + { 61, 87, 198}, { 85, 153, 254}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {232, 104, 242}, {165, 114, 227}, { 96, 120, 206}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {137, 178, 250}, {146, 153, 245}, { 74, 108, 205}, + { 41, 81, 149}, { 24, 55, 104}, { 13, 36, 68}, + }, + { // band 2 + {147, 185, 252}, {127, 161, 246}, { 30, 104, 208}, + { 11, 74, 154}, { 6, 54, 100}, { 2, 29, 63}, + }, + { // band 3 + {163, 191, 254}, {101, 161, 249}, { 22, 114, 215}, + { 6, 89, 173}, { 1, 65, 120}, { 1, 1, 170}, + }, + { // band 4 + {197, 160, 254}, {142, 141, 251}, { 39, 102, 218}, + { 10, 76, 158}, { 1, 56, 122}, {128, 128, 128}, + }, + { // band 5 + {224, 76, 254}, {215, 84, 253}, {107, 85, 232}, + { 43, 71, 177}, { 1, 1, 254}, {128, 128, 128}, + }, + }, + }, + }, +#endif + { // TX_SIZE 0 + { // Y plane + { // Intra + { // band 0 + {174, 30, 159}, { 76, 38, 115}, { 15, 33, 65}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 60, 80, 153}, { 72, 75, 147}, { 36, 68, 129}, + { 15, 59, 104}, { 4, 45, 74}, { 1, 28, 45}, + }, + { // band 2 + { 70, 122, 186}, { 55, 104, 175}, { 21, 83, 144}, + { 8, 67, 112}, { 2, 51, 82}, { 1, 34, 57}, + }, + { // band 3 + { 97, 144, 207}, { 52, 109, 195}, { 16, 77, 153}, + { 4, 58, 113}, { 1, 43, 77}, { 1, 27, 48}, + }, + { // band 4 + {128, 148, 229}, { 76, 104, 210}, { 18, 77, 159}, + { 4, 65, 110}, { 1, 52, 82}, { 1, 31, 55}, + }, + { // band 5 + {165, 51, 238}, {128, 50, 230}, { 57, 49, 185}, + { 28, 47, 130}, { 12, 44, 96}, { 3, 36, 60}, + }, + }, + { // Intra + { // band 0 + {169, 103, 203}, {117, 96, 176}, { 56, 81, 137}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 31, 150, 224}, { 49, 128, 212}, { 19, 92, 165}, + { 6, 67, 116}, { 2, 43, 71}, { 1, 21, 36}, + }, + { // band 2 + { 58, 156, 230}, { 47, 130, 215}, { 7, 87, 158}, + { 2, 63, 114}, { 1, 39, 71}, { 1, 18, 36}, + }, + { // band 3 + { 85, 176, 240}, { 43, 138, 226}, { 8, 93, 172}, + { 2, 70, 127}, { 1, 46, 81}, { 1, 26, 47}, + }, + { // band 4 + {155, 144, 248}, { 93, 116, 235}, { 21, 83, 180}, + { 4, 59, 119}, { 1, 43, 80}, { 1, 25, 50}, + }, + { // band 5 + {203, 61, 250}, {171, 57, 243}, { 71, 57, 199}, + { 31, 49, 144}, { 13, 42, 96}, { 7, 30, 52}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {204, 44, 204}, {137, 57, 184}, { 72, 62, 152}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {145, 117, 236}, {151, 112, 231}, { 87, 95, 208}, + { 31, 77, 165}, { 5, 49, 98}, { 1, 24, 39}, + }, + { // band 2 + {146, 152, 241}, {140, 132, 236}, { 41, 103, 209}, + { 10, 86, 165}, { 2, 55, 106}, { 1, 25, 58}, + }, + { // band 3 + {154, 181, 249}, { 84, 143, 240}, { 23, 114, 210}, + { 6, 102, 182}, { 2, 71, 137}, { 1, 35, 90}, + }, + { // band 4 + {184, 150, 251}, {115, 130, 244}, { 34, 105, 215}, + { 15, 89, 173}, { 1, 51, 141}, {128, 128, 128}, + }, + { // band 5 + {211, 71, 253}, {193, 78, 249}, {106, 91, 232}, + { 61, 87, 198}, { 85, 153, 254}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {232, 104, 242}, {165, 114, 227}, { 96, 120, 206}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {137, 178, 250}, {146, 153, 245}, { 74, 108, 205}, + { 41, 81, 149}, { 24, 55, 104}, { 13, 36, 68}, + }, + { // band 2 + {147, 185, 252}, {127, 161, 246}, { 30, 104, 208}, + { 11, 74, 154}, { 6, 54, 100}, { 2, 29, 63}, + }, + { // band 3 + {163, 191, 254}, {101, 161, 249}, { 22, 114, 215}, + { 6, 89, 173}, { 1, 65, 120}, { 1, 1, 170}, + }, + { // band 4 + {197, 160, 254}, {142, 141, 251}, { 39, 102, 218}, + { 10, 76, 158}, { 1, 56, 122}, {128, 128, 128}, + }, + { // band 5 + {224, 76, 254}, {215, 84, 253}, {107, 85, 232}, + { 43, 71, 177}, { 1, 1, 254}, {128, 128, 128}, + }, + }, + }, + }, + { // TX_SIZE 1 + { // Y plane + { // Intra + { // band 0 + { 68, 37, 120}, { 21, 34, 82}, { 5, 26, 49}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 41, 89, 138}, { 56, 83, 132}, { 31, 73, 115}, + { 16, 62, 92}, { 5, 45, 62}, { 1, 24, 32}, + }, + { // band 2 + { 48, 139, 165}, { 30, 114, 160}, { 13, 92, 132}, + { 6, 72, 103}, { 3, 49, 72}, { 1, 26, 41}, + }, + { // band 3 + { 44, 162, 191}, { 20, 127, 175}, { 5, 90, 137}, + { 1, 62, 100}, { 1, 38, 63}, { 1, 20, 32}, + }, + { // band 4 + { 51, 184, 213}, { 16, 137, 193}, { 2, 89, 143}, + { 1, 60, 102}, { 1, 39, 66}, { 1, 23, 37}, + }, + { // band 5 + { 76, 200, 235}, { 27, 150, 216}, { 3, 99, 164}, + { 1, 70, 119}, { 1, 45, 77}, { 1, 22, 38}, + }, + }, + { // Intra + { // band 0 + { 81, 112, 199}, { 49, 101, 164}, { 19, 80, 119}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 12, 181, 217}, { 48, 151, 212}, { 38, 118, 180}, + { 22, 95, 140}, { 11, 67, 92}, { 13, 46, 44}, + }, + { // band 2 + { 29, 188, 226}, { 19, 147, 210}, { 5, 95, 154}, + { 4, 68, 106}, { 3, 44, 60}, { 1, 24, 27}, + }, + { // band 3 + { 30, 195, 234}, { 15, 153, 216}, { 3, 95, 156}, + { 2, 66, 108}, { 2, 44, 62}, { 1, 24, 29}, + }, + { // band 4 + { 36, 203, 243}, { 12, 162, 225}, { 2, 98, 163}, + { 2, 67, 113}, { 2, 45, 68}, { 1, 24, 34}, + }, + { // band 5 + { 86, 207, 248}, { 35, 165, 236}, { 3, 107, 180}, + { 1, 73, 128}, { 1, 45, 78}, { 1, 20, 34}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {188, 37, 205}, {118, 51, 172}, { 56, 57, 135}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {116, 135, 225}, {144, 123, 221}, { 72, 103, 197}, + { 35, 77, 153}, { 15, 47, 82}, { 6, 25, 34}, + }, + { // band 2 + {128, 171, 233}, { 82, 142, 226}, { 31, 106, 191}, + { 16, 82, 146}, { 9, 59, 98}, { 4, 33, 54}, + }, + { // band 3 + {126, 197, 241}, { 66, 155, 230}, { 18, 108, 190}, + { 7, 82, 148}, { 3, 58, 98}, { 1, 25, 50}, + }, + { // band 4 + {117, 207, 244}, { 44, 163, 233}, { 9, 112, 191}, + { 5, 84, 148}, { 3, 61, 87}, { 1, 28, 38}, + }, + { // band 5 + {112, 214, 249}, { 39, 174, 240}, { 6, 125, 205}, + { 4, 96, 163}, { 5, 66, 100}, { 1, 128, 254}, + }, + }, + { // Inter + { // band 0 + {227, 70, 234}, {145, 91, 213}, { 61, 100, 173}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {108, 198, 243}, {171, 172, 240}, {118, 130, 210}, + {104, 107, 165}, { 64, 85, 114}, { 55, 64, 60}, + }, + { // band 2 + {110, 208, 247}, { 64, 175, 237}, { 24, 112, 187}, + { 24, 81, 133}, { 24, 63, 83}, { 21, 47, 53}, + }, + { // band 3 + { 91, 218, 249}, { 46, 188, 238}, { 8, 113, 184}, + { 5, 83, 137}, { 6, 62, 95}, { 17, 44, 94}, + }, + { // band 4 + { 84, 216, 248}, { 30, 187, 237}, { 2, 117, 188}, + { 1, 88, 141}, { 3, 63, 98}, { 1, 1, 1}, + }, + { // band 5 + {116, 218, 252}, { 47, 186, 242}, { 2, 132, 204}, + { 1, 106, 175}, { 1, 88, 104}, { 1, 254, 128}, + }, + }, + }, + }, + { // TX_SIZE 2 + { // Y plane + { // Intra + { // band 0 + { 35, 41, 129}, { 12, 30, 70}, { 2, 19, 32}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 30, 77, 116}, { 39, 70, 110}, { 20, 58, 96}, + { 8, 47, 77}, { 2, 33, 52}, { 1, 17, 26}, + }, + { // band 2 + { 31, 123, 146}, { 18, 103, 140}, { 7, 81, 119}, + { 2, 62, 95}, { 1, 44, 70}, { 1, 26, 42}, + }, + { // band 3 + { 21, 149, 170}, { 9, 114, 158}, { 2, 80, 126}, + { 1, 57, 94}, { 1, 36, 61}, { 1, 18, 31}, + }, + { // band 4 + { 20, 178, 199}, { 6, 134, 183}, { 1, 87, 139}, + { 1, 60, 100}, { 1, 37, 64}, { 1, 18, 31}, + }, + { // band 5 + { 36, 218, 233}, { 6, 160, 207}, { 1, 92, 147}, + { 1, 59, 101}, { 1, 35, 62}, { 1, 18, 31}, + }, + }, + { // Intra + { // band 0 + { 17, 62, 211}, { 14, 62, 153}, { 5, 50, 84}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 11, 180, 205}, { 87, 160, 205}, { 53, 128, 184}, + { 27, 106, 156}, { 13, 79, 115}, { 6, 46, 67}, + }, + { // band 2 + { 32, 194, 220}, { 20, 145, 202}, { 4, 96, 152}, + { 1, 67, 111}, { 1, 42, 70}, { 1, 21, 37}, + }, + { // band 3 + { 30, 204, 228}, { 14, 152, 207}, { 1, 92, 149}, + { 1, 61, 103}, { 1, 34, 59}, { 1, 16, 28}, + }, + { // band 4 + { 27, 213, 235}, { 7, 159, 210}, { 1, 88, 143}, + { 1, 55, 94}, { 1, 31, 53}, { 1, 16, 27}, + }, + { // band 5 + { 28, 223, 243}, { 4, 173, 217}, { 1, 91, 146}, + { 1, 58, 98}, { 1, 35, 60}, { 1, 19, 33}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {172, 37, 202}, { 83, 51, 156}, { 24, 53, 110}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 76, 134, 206}, {110, 124, 200}, { 47, 106, 180}, + { 15, 82, 145}, { 3, 48, 83}, { 1, 19, 32}, + }, + { // band 2 + { 80, 176, 220}, { 49, 145, 212}, { 17, 112, 180}, + { 7, 84, 140}, { 1, 53, 89}, { 1, 27, 43}, + }, + { // band 3 + { 74, 201, 232}, { 38, 158, 221}, { 8, 112, 179}, + { 2, 79, 132}, { 1, 47, 82}, { 1, 26, 42}, + }, + { // band 4 + { 73, 215, 239}, { 28, 169, 227}, { 3, 112, 176}, + { 1, 74, 126}, { 1, 48, 79}, { 1, 27, 44}, + }, + { // band 5 + { 71, 233, 244}, { 18, 180, 230}, { 1, 114, 180}, + { 1, 80, 134}, { 1, 51, 85}, { 1, 26, 36}, + }, + }, + { // Inter + { // band 0 + {213, 34, 244}, {126, 57, 212}, { 46, 67, 151}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {120, 202, 245}, {198, 173, 241}, {119, 146, 224}, + { 76, 126, 195}, { 44, 102, 159}, { 40, 76, 115}, + }, + { // band 2 + {120, 215, 248}, { 69, 171, 237}, { 23, 119, 194}, + { 10, 86, 147}, { 2, 56, 94}, { 1, 25, 44}, + }, + { // band 3 + {102, 226, 250}, { 53, 183, 239}, { 9, 118, 188}, + { 2, 78, 131}, { 1, 48, 89}, { 1, 17, 36}, + }, + { // band 4 + { 86, 235, 252}, { 34, 194, 240}, { 2, 109, 173}, + { 1, 68, 118}, { 1, 44, 79}, { 1, 1, 38}, + }, + { // band 5 + { 59, 236, 243}, { 11, 189, 228}, { 1, 112, 187}, + { 1, 88, 145}, { 1, 55, 92}, { 1, 1, 128}, + }, + }, + }, + }, + { // TX_SIZE 3 + { // Y plane + { // Intra + { // band 0 + { 41, 40, 104}, { 12, 31, 64}, { 2, 16, 28}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 65, 58, 132}, { 50, 61, 130}, { 40, 57, 116}, + { 22, 46, 87}, { 2, 28, 44}, { 1, 11, 17}, + }, + { // band 2 + { 55, 139, 135}, { 46, 122, 132}, { 21, 89, 110}, + { 6, 60, 78}, { 1, 38, 54}, { 1, 17, 27}, + }, + { // band 3 + { 29, 167, 161}, { 10, 120, 141}, { 1, 69, 98}, + { 1, 42, 66}, { 1, 28, 44}, { 1, 15, 24}, + }, + { // band 4 + { 19, 191, 180}, { 4, 125, 154}, { 1, 70, 107}, + { 1, 48, 77}, { 1, 33, 53}, { 1, 17, 28}, + }, + { // band 5 + { 16, 238, 231}, { 2, 163, 198}, { 1, 85, 134}, + { 1, 54, 90}, { 1, 34, 57}, { 1, 17, 29}, + }, + }, + { // Intra + { // band 0 + { 70, 15, 216}, { 40, 18, 164}, { 14, 17, 83}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 25, 150, 200}, {185, 154, 211}, {123, 137, 199}, + { 67, 119, 177}, { 31, 96, 137}, { 18, 63, 86}, + }, + { // band 2 + { 57, 187, 223}, { 35, 148, 207}, { 7, 104, 159}, + { 2, 72, 113}, { 1, 44, 71}, { 1, 20, 34}, + }, + { // band 3 + { 44, 203, 233}, { 18, 157, 212}, { 1, 98, 150}, + { 1, 61, 102}, { 1, 38, 62}, { 1, 19, 31}, + }, + { // band 4 + { 41, 215, 238}, { 11, 166, 215}, { 1, 94, 146}, + { 1, 60, 101}, { 1, 37, 63}, { 1, 17, 28}, + }, + { // band 5 + { 19, 236, 246}, { 3, 188, 223}, { 1, 95, 146}, + { 1, 58, 95}, { 1, 34, 56}, { 1, 17, 27}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {146, 27, 156}, { 49, 32, 116}, { 10, 39, 77}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 47, 101, 172}, { 93, 100, 178}, { 58, 91, 165}, + { 26, 75, 134}, { 4, 49, 82}, { 2, 22, 33}, + }, + { // band 2 + { 60, 158, 196}, { 44, 135, 186}, { 25, 106, 157}, + { 8, 81, 124}, { 2, 56, 86}, { 1, 28, 45}, + }, + { // band 3 + { 44, 169, 212}, { 15, 138, 196}, { 2, 100, 157}, + { 1, 74, 119}, { 1, 49, 76}, { 1, 20, 34}, + }, + { // band 4 + { 38, 199, 231}, { 11, 158, 214}, { 1, 111, 167}, + { 1, 76, 122}, { 1, 44, 76}, { 1, 17, 39}, + }, + { // band 5 + { 40, 236, 246}, { 10, 187, 230}, { 1, 115, 175}, + { 1, 74, 122}, { 1, 42, 71}, { 1, 14, 59}, + }, + }, + { // Inter + { // band 0 + {161, 26, 237}, { 65, 46, 209}, { 21, 46, 161}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 87, 229, 245}, {206, 214, 244}, {148, 186, 236}, + { 89, 165, 221}, { 41, 132, 186}, { 37, 93, 141}, + }, + { // band 2 + { 93, 231, 246}, { 47, 181, 231}, { 8, 117, 188}, + { 2, 84, 138}, { 1, 43, 87}, { 1, 27, 41}, + }, + { // band 3 + { 80, 239, 250}, { 28, 190, 236}, { 1, 119, 183}, + { 1, 84, 135}, { 1, 81, 69}, { 1, 102, 1}, + }, + { // band 4 + { 67, 245, 252}, { 22, 206, 242}, { 1, 130, 195}, + { 1, 77, 136}, { 1, 35, 88}, {128, 128, 128}, + }, + { // band 5 + { 43, 250, 228}, { 31, 185, 204}, { 6, 101, 183}, + { 1, 92, 151}, { 1, 84, 137}, {128, 128, 128}, + }, + }, + }, + }, +#if CONFIG_TX64X64 + { // TX_SIZE 4 + { // Y plane + { // Intra + { // band 0 + { 41, 40, 104}, { 12, 31, 64}, { 2, 16, 28}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 65, 58, 132}, { 50, 61, 130}, { 40, 57, 116}, + { 22, 46, 87}, { 2, 28, 44}, { 1, 11, 17}, + }, + { // band 2 + { 55, 139, 135}, { 46, 122, 132}, { 21, 89, 110}, + { 6, 60, 78}, { 1, 38, 54}, { 1, 17, 27}, + }, + { // band 3 + { 29, 167, 161}, { 10, 120, 141}, { 1, 69, 98}, + { 1, 42, 66}, { 1, 28, 44}, { 1, 15, 24}, + }, + { // band 4 + { 19, 191, 180}, { 4, 125, 154}, { 1, 70, 107}, + { 1, 48, 77}, { 1, 33, 53}, { 1, 17, 28}, + }, + { // band 5 + { 16, 238, 231}, { 2, 163, 198}, { 1, 85, 134}, + { 1, 54, 90}, { 1, 34, 57}, { 1, 17, 29}, + }, + }, + { // Intra + { // band 0 + { 70, 15, 216}, { 40, 18, 164}, { 14, 17, 83}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 25, 150, 200}, {185, 154, 211}, {123, 137, 199}, + { 67, 119, 177}, { 31, 96, 137}, { 18, 63, 86}, + }, + { // band 2 + { 57, 187, 223}, { 35, 148, 207}, { 7, 104, 159}, + { 2, 72, 113}, { 1, 44, 71}, { 1, 20, 34}, + }, + { // band 3 + { 44, 203, 233}, { 18, 157, 212}, { 1, 98, 150}, + { 1, 61, 102}, { 1, 38, 62}, { 1, 19, 31}, + }, + { // band 4 + { 41, 215, 238}, { 11, 166, 215}, { 1, 94, 146}, + { 1, 60, 101}, { 1, 37, 63}, { 1, 17, 28}, + }, + { // band 5 + { 19, 236, 246}, { 3, 188, 223}, { 1, 95, 146}, + { 1, 58, 95}, { 1, 34, 56}, { 1, 17, 27}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {146, 27, 156}, { 49, 32, 116}, { 10, 39, 77}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 47, 101, 172}, { 93, 100, 178}, { 58, 91, 165}, + { 26, 75, 134}, { 4, 49, 82}, { 2, 22, 33}, + }, + { // band 2 + { 60, 158, 196}, { 44, 135, 186}, { 25, 106, 157}, + { 8, 81, 124}, { 2, 56, 86}, { 1, 28, 45}, + }, + { // band 3 + { 44, 169, 212}, { 15, 138, 196}, { 2, 100, 157}, + { 1, 74, 119}, { 1, 49, 76}, { 1, 20, 34}, + }, + { // band 4 + { 38, 199, 231}, { 11, 158, 214}, { 1, 111, 167}, + { 1, 76, 122}, { 1, 44, 76}, { 1, 17, 39}, + }, + { // band 5 + { 40, 236, 246}, { 10, 187, 230}, { 1, 115, 175}, + { 1, 74, 122}, { 1, 42, 71}, { 1, 14, 59}, + }, + }, + { // Inter + { // band 0 + {161, 26, 237}, { 65, 46, 209}, { 21, 46, 161}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 87, 229, 245}, {206, 214, 244}, {148, 186, 236}, + { 89, 165, 221}, { 41, 132, 186}, { 37, 93, 141}, + }, + { // band 2 + { 93, 231, 246}, { 47, 181, 231}, { 8, 117, 188}, + { 2, 84, 138}, { 1, 43, 87}, { 1, 27, 41}, + }, + { // band 3 + { 80, 239, 250}, { 28, 190, 236}, { 1, 119, 183}, + { 1, 84, 135}, { 1, 81, 69}, { 1, 102, 1}, + }, + { // band 4 + { 67, 245, 252}, { 22, 206, 242}, { 1, 130, 195}, + { 1, 77, 136}, { 1, 35, 88}, {128, 128, 128}, + }, + { // band 5 + { 43, 250, 228}, { 31, 185, 204}, { 6, 101, 183}, + { 1, 92, 151}, { 1, 84, 137}, {128, 128, 128}, + }, + }, + }, + }, +#endif // CONFIG_TX64X64 + }, + { // Q_Index 2 +#if CONFIG_CB4X4 + { // TX_SIZE 0 + { // Y plane + { // Intra + { // band 0 + {181, 22, 175}, { 96, 37, 147}, { 35, 41, 105}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 80, 95, 197}, {111, 92, 193}, { 59, 87, 175}, + { 29, 79, 150}, { 10, 65, 118}, { 2, 47, 82}, + }, + { // band 2 + { 90, 141, 216}, { 77, 120, 210}, { 23, 95, 184}, + { 11, 81, 151}, { 6, 75, 130}, { 2, 58, 113}, + }, + { // band 3 + {122, 167, 231}, { 66, 119, 225}, { 26, 87, 189}, + { 7, 76, 151}, { 2, 63, 125}, { 1, 59, 77}, + }, + { // band 4 + {162, 147, 244}, {110, 97, 236}, { 32, 88, 204}, + { 11, 89, 174}, { 5, 78, 151}, {128, 128, 128}, + }, + { // band 5 + {205, 59, 251}, {176, 68, 248}, { 90, 71, 223}, + { 49, 72, 188}, { 17, 74, 203}, {128, 128, 128}, + }, + }, + { // Intra + { // band 0 + {188, 70, 207}, {140, 73, 189}, { 85, 73, 163}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 59, 144, 239}, { 79, 126, 237}, { 31, 102, 202}, + { 10, 81, 153}, { 3, 56, 102}, { 2, 33, 59}, + }, + { // band 2 + {100, 152, 243}, { 80, 129, 236}, { 14, 94, 194}, + { 4, 72, 150}, { 1, 50, 103}, { 1, 35, 60}, + }, + { // band 3 + {130, 183, 247}, { 70, 139, 242}, { 19, 100, 203}, + { 4, 83, 159}, { 1, 59, 119}, { 1, 44, 72}, + }, + { // band 4 + {197, 138, 252}, {135, 107, 247}, { 31, 86, 210}, + { 7, 74, 160}, { 1, 53, 107}, {128, 128, 128}, + }, + { // band 5 + {229, 54, 254}, {200, 51, 251}, { 83, 61, 226}, + { 33, 55, 177}, { 12, 74, 145}, {128, 128, 128}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {229, 20, 235}, {183, 37, 221}, {127, 47, 198}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {188, 115, 251}, {208, 110, 250}, {101, 99, 235}, + { 38, 81, 197}, { 9, 56, 132}, { 9, 52, 63}, + }, + { // band 2 + {189, 150, 252}, {186, 137, 251}, { 54, 107, 236}, + { 14, 90, 195}, { 1, 89, 104}, {128, 128, 128}, + }, + { // band 3 + {209, 180, 254}, {142, 145, 253}, { 51, 130, 236}, + { 6, 128, 214}, { 1, 128, 254}, {128, 128, 128}, + }, + { // band 4 + {231, 140, 254}, {194, 128, 254}, { 75, 119, 233}, + {128, 23, 230}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {244, 59, 254}, {239, 81, 254}, {128, 85, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {246, 55, 247}, {197, 64, 235}, {141, 74, 218}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {178, 163, 254}, {192, 138, 252}, { 85, 103, 231}, + { 49, 81, 179}, { 32, 54, 133}, { 12, 26, 98}, + }, + { // band 2 + {189, 173, 254}, {179, 150, 253}, { 60, 94, 237}, + { 34, 81, 198}, { 20, 53, 187}, {128, 128, 128}, + }, + { // band 3 + {202, 191, 254}, {157, 160, 254}, { 57, 117, 240}, + { 28, 105, 211}, { 1, 128, 1}, {128, 128, 128}, + }, + { // band 4 + {231, 146, 254}, {208, 133, 254}, { 66, 78, 233}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {246, 49, 254}, {246, 63, 254}, { 85, 142, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + }, +#endif + { // TX_SIZE 0 + { // Y plane + { // Intra + { // band 0 + {181, 22, 175}, { 96, 37, 147}, { 35, 41, 105}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 80, 95, 197}, {111, 92, 193}, { 59, 87, 175}, + { 29, 79, 150}, { 10, 65, 118}, { 2, 47, 82}, + }, + { // band 2 + { 90, 141, 216}, { 77, 120, 210}, { 23, 95, 184}, + { 11, 81, 151}, { 6, 75, 130}, { 2, 58, 113}, + }, + { // band 3 + {122, 167, 231}, { 66, 119, 225}, { 26, 87, 189}, + { 7, 76, 151}, { 2, 63, 125}, { 1, 59, 77}, + }, + { // band 4 + {162, 147, 244}, {110, 97, 236}, { 32, 88, 204}, + { 11, 89, 174}, { 5, 78, 151}, {128, 128, 128}, + }, + { // band 5 + {205, 59, 251}, {176, 68, 248}, { 90, 71, 223}, + { 49, 72, 188}, { 17, 74, 203}, {128, 128, 128}, + }, + }, + { // Intra + { // band 0 + {188, 70, 207}, {140, 73, 189}, { 85, 73, 163}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 59, 144, 239}, { 79, 126, 237}, { 31, 102, 202}, + { 10, 81, 153}, { 3, 56, 102}, { 2, 33, 59}, + }, + { // band 2 + {100, 152, 243}, { 80, 129, 236}, { 14, 94, 194}, + { 4, 72, 150}, { 1, 50, 103}, { 1, 35, 60}, + }, + { // band 3 + {130, 183, 247}, { 70, 139, 242}, { 19, 100, 203}, + { 4, 83, 159}, { 1, 59, 119}, { 1, 44, 72}, + }, + { // band 4 + {197, 138, 252}, {135, 107, 247}, { 31, 86, 210}, + { 7, 74, 160}, { 1, 53, 107}, {128, 128, 128}, + }, + { // band 5 + {229, 54, 254}, {200, 51, 251}, { 83, 61, 226}, + { 33, 55, 177}, { 12, 74, 145}, {128, 128, 128}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {229, 20, 235}, {183, 37, 221}, {127, 47, 198}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {188, 115, 251}, {208, 110, 250}, {101, 99, 235}, + { 38, 81, 197}, { 9, 56, 132}, { 9, 52, 63}, + }, + { // band 2 + {189, 150, 252}, {186, 137, 251}, { 54, 107, 236}, + { 14, 90, 195}, { 1, 89, 104}, {128, 128, 128}, + }, + { // band 3 + {209, 180, 254}, {142, 145, 253}, { 51, 130, 236}, + { 6, 128, 214}, { 1, 128, 254}, {128, 128, 128}, + }, + { // band 4 + {231, 140, 254}, {194, 128, 254}, { 75, 119, 233}, + {128, 23, 230}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {244, 59, 254}, {239, 81, 254}, {128, 85, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {246, 55, 247}, {197, 64, 235}, {141, 74, 218}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {178, 163, 254}, {192, 138, 252}, { 85, 103, 231}, + { 49, 81, 179}, { 32, 54, 133}, { 12, 26, 98}, + }, + { // band 2 + {189, 173, 254}, {179, 150, 253}, { 60, 94, 237}, + { 34, 81, 198}, { 20, 53, 187}, {128, 128, 128}, + }, + { // band 3 + {202, 191, 254}, {157, 160, 254}, { 57, 117, 240}, + { 28, 105, 211}, { 1, 128, 1}, {128, 128, 128}, + }, + { // band 4 + {231, 146, 254}, {208, 133, 254}, { 66, 78, 233}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {246, 49, 254}, {246, 63, 254}, { 85, 142, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + }, + { // TX_SIZE 1 + { // Y plane + { // Intra + { // band 0 + { 45, 28, 124}, { 23, 35, 107}, { 10, 34, 78}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 53, 99, 177}, { 82, 96, 174}, { 46, 89, 158}, + { 21, 76, 133}, { 6, 56, 94}, { 1, 33, 54}, + }, + { // band 2 + { 68, 147, 201}, { 42, 124, 195}, { 17, 98, 166}, + { 7, 75, 131}, { 2, 53, 93}, { 1, 33, 59}, + }, + { // band 3 + { 65, 176, 217}, { 30, 137, 206}, { 6, 97, 167}, + { 2, 70, 128}, { 1, 47, 88}, { 1, 29, 46}, + }, + { // band 4 + { 69, 195, 232}, { 24, 146, 218}, { 4, 100, 175}, + { 2, 72, 134}, { 1, 51, 93}, { 1, 29, 52}, + }, + { // band 5 + { 96, 212, 246}, { 39, 158, 234}, { 6, 109, 192}, + { 2, 77, 144}, { 1, 50, 95}, { 1, 20, 45}, + }, + }, + { // Intra + { // band 0 + { 71, 80, 213}, { 53, 73, 181}, { 25, 66, 141}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 35, 168, 231}, { 91, 150, 229}, { 49, 122, 202}, + { 22, 97, 162}, { 10, 68, 108}, { 9, 48, 57}, + }, + { // band 2 + { 56, 178, 236}, { 32, 148, 225}, { 9, 99, 176}, + { 4, 69, 127}, { 2, 44, 78}, { 1, 25, 41}, + }, + { // band 3 + { 57, 191, 242}, { 27, 155, 230}, { 5, 102, 180}, + { 2, 71, 133}, { 1, 44, 78}, { 1, 27, 41}, + }, + { // band 4 + { 67, 201, 247}, { 24, 162, 237}, { 3, 106, 188}, + { 3, 74, 137}, { 1, 46, 85}, { 1, 34, 48}, + }, + { // band 5 + {111, 210, 251}, { 47, 166, 244}, { 3, 113, 199}, + { 2, 77, 146}, { 1, 48, 93}, { 1, 38, 22}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {206, 21, 221}, {150, 36, 195}, { 94, 44, 164}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {147, 128, 239}, {194, 122, 238}, { 95, 104, 220}, + { 39, 81, 183}, { 13, 53, 111}, { 3, 24, 49}, + }, + { // band 2 + {164, 163, 244}, {106, 142, 239}, { 50, 112, 215}, + { 26, 90, 177}, { 12, 67, 130}, { 1, 1, 64}, + }, + { // band 3 + {155, 193, 249}, { 88, 158, 244}, { 26, 124, 220}, + { 10, 98, 173}, { 1, 77, 126}, {128, 128, 128}, + }, + { // band 4 + {141, 205, 252}, { 64, 174, 248}, { 17, 124, 221}, + { 12, 92, 176}, { 1, 29, 148}, {128, 128, 128}, + }, + { // band 5 + {150, 217, 254}, { 74, 191, 252}, { 30, 144, 215}, + { 1, 106, 137}, {128, 1, 128}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {241, 37, 242}, {175, 48, 223}, { 99, 53, 189}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {153, 183, 248}, {212, 156, 247}, {134, 124, 221}, + { 88, 103, 184}, { 59, 86, 132}, { 29, 61, 67}, + }, + { // band 2 + {162, 199, 250}, {106, 167, 247}, { 56, 110, 207}, + { 32, 85, 165}, { 16, 71, 130}, { 1, 93, 254}, + }, + { // band 3 + {143, 213, 252}, { 86, 187, 250}, { 23, 124, 220}, + { 7, 95, 176}, { 1, 109, 102}, {128, 128, 128}, + }, + { // band 4 + {130, 219, 254}, { 70, 201, 253}, { 15, 128, 215}, + { 1, 101, 201}, { 1, 64, 170}, {128, 128, 128}, + }, + { // band 5 + {155, 219, 254}, {105, 207, 254}, { 28, 155, 229}, + { 1, 153, 191}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + }, + { // TX_SIZE 2 + { // Y plane + { // Intra + { // band 0 + { 18, 26, 117}, { 10, 29, 82}, { 3, 25, 52}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 35, 88, 152}, { 62, 85, 150}, { 36, 77, 137}, + { 16, 66, 116}, { 4, 47, 81}, { 1, 26, 44}, + }, + { // band 2 + { 55, 141, 182}, { 32, 119, 177}, { 12, 93, 154}, + { 4, 71, 123}, { 1, 51, 89}, { 1, 32, 56}, + }, + { // band 3 + { 46, 171, 202}, { 21, 130, 191}, { 5, 91, 154}, + { 1, 64, 115}, { 1, 42, 77}, { 1, 25, 41}, + }, + { // band 4 + { 43, 195, 219}, { 12, 142, 203}, { 1, 91, 156}, + { 1, 63, 115}, { 1, 41, 77}, { 1, 22, 43}, + }, + { // band 5 + { 42, 221, 238}, { 8, 162, 219}, { 1, 98, 167}, + { 1, 67, 123}, { 1, 43, 83}, { 1, 25, 38}, + }, + }, + { // Intra + { // band 0 + { 16, 51, 216}, { 20, 48, 168}, { 9, 44, 109}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 34, 164, 226}, {124, 148, 226}, { 72, 127, 207}, + { 36, 107, 175}, { 15, 81, 129}, { 6, 51, 79}, + }, + { // band 2 + { 61, 182, 234}, { 35, 148, 220}, { 9, 101, 178}, + { 4, 71, 134}, { 1, 46, 90}, { 1, 24, 51}, + }, + { // band 3 + { 54, 198, 239}, { 25, 156, 224}, { 3, 98, 173}, + { 1, 66, 124}, { 1, 41, 78}, { 1, 15, 37}, + }, + { // band 4 + { 48, 209, 242}, { 12, 162, 226}, { 1, 96, 169}, + { 1, 63, 119}, { 1, 40, 78}, { 1, 18, 45}, + }, + { // band 5 + { 44, 223, 247}, { 6, 173, 232}, { 1, 105, 178}, + { 1, 71, 131}, { 1, 44, 84}, { 1, 13, 46}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {188, 26, 214}, {121, 42, 181}, { 66, 49, 149}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {136, 128, 233}, {172, 124, 230}, { 80, 106, 211}, + { 27, 81, 174}, { 6, 49, 98}, { 8, 28, 49}, + }, + { // band 2 + {145, 166, 239}, { 92, 141, 229}, { 28, 108, 196}, + { 8, 87, 154}, { 1, 58, 105}, { 1, 27, 59}, + }, + { // band 3 + {131, 193, 242}, { 66, 151, 231}, { 13, 112, 192}, + { 2, 81, 152}, { 1, 66, 121}, { 1, 23, 64}, + }, + { // band 4 + {112, 211, 246}, { 41, 164, 235}, { 5, 117, 202}, + { 1, 83, 162}, { 1, 64, 111}, {128, 128, 128}, + }, + { // band 5 + { 96, 230, 250}, { 28, 185, 243}, { 2, 132, 204}, + { 1, 91, 166}, { 1, 85, 46}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {238, 23, 242}, {157, 29, 215}, { 73, 27, 162}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {165, 173, 250}, {222, 151, 247}, {152, 134, 235}, + {114, 120, 210}, { 86, 109, 176}, { 53, 88, 145}, + }, + { // band 2 + {164, 194, 249}, {100, 158, 241}, { 35, 111, 212}, + { 17, 85, 167}, { 1, 52, 112}, { 1, 73, 1}, + }, + { // band 3 + {151, 215, 252}, { 83, 172, 245}, { 16, 122, 208}, + { 6, 101, 165}, { 1, 74, 113}, { 1, 1, 1}, + }, + { // band 4 + {138, 230, 253}, { 65, 184, 248}, { 8, 128, 212}, + { 1, 111, 182}, {128, 1, 1}, {128, 128, 128}, + }, + { // band 5 + {123, 240, 253}, { 36, 201, 250}, { 3, 127, 211}, + { 1, 68, 204}, {128, 1, 1}, {128, 128, 128}, + }, + }, + }, + }, + { // TX_SIZE 3 + { // Y plane + { // Intra + { // band 0 + { 51, 21, 156}, { 30, 23, 86}, { 4, 18, 37}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 38, 77, 129}, { 79, 76, 129}, { 40, 66, 117}, + { 12, 54, 95}, { 1, 36, 60}, { 1, 17, 29}, + }, + { // band 2 + { 44, 133, 149}, { 24, 107, 143}, { 8, 78, 121}, + { 3, 59, 97}, { 1, 42, 71}, { 1, 22, 37}, + }, + { // band 3 + { 29, 160, 171}, { 9, 114, 158}, { 1, 76, 125}, + { 1, 54, 93}, { 1, 36, 63}, { 1, 20, 35}, + }, + { // band 4 + { 22, 188, 205}, { 6, 132, 186}, { 1, 87, 144}, + { 1, 62, 107}, { 1, 41, 72}, { 1, 23, 41}, + }, + { // band 5 + { 25, 233, 236}, { 5, 165, 214}, { 1, 96, 158}, + { 1, 63, 112}, { 1, 40, 73}, { 1, 23, 40}, + }, + }, + { // Intra + { // band 0 + { 48, 20, 231}, { 37, 21, 179}, { 15, 18, 109}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 41, 154, 216}, {196, 142, 221}, {131, 125, 207}, + { 84, 111, 181}, { 45, 91, 142}, { 27, 62, 89}, + }, + { // band 2 + { 72, 181, 230}, { 41, 147, 215}, { 10, 102, 173}, + { 3, 73, 132}, { 1, 47, 89}, { 1, 23, 50}, + }, + { // band 3 + { 60, 201, 236}, { 23, 157, 219}, { 2, 99, 167}, + { 1, 69, 124}, { 1, 43, 80}, { 1, 22, 39}, + }, + { // band 4 + { 53, 214, 242}, { 15, 165, 224}, { 1, 101, 173}, + { 1, 70, 131}, { 1, 44, 83}, { 1, 23, 49}, + }, + { // band 5 + { 39, 239, 248}, { 7, 186, 233}, { 1, 108, 174}, + { 1, 70, 123}, { 1, 43, 77}, { 1, 16, 42}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {161, 26, 204}, { 77, 40, 160}, { 26, 50, 117}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 80, 140, 218}, {136, 133, 215}, { 63, 117, 197}, + { 20, 93, 170}, { 7, 55, 102}, { 13, 32, 52}, + }, + { // band 2 + { 86, 173, 231}, { 46, 150, 220}, { 18, 118, 190}, + { 8, 90, 150}, { 2, 60, 95}, { 1, 39, 41}, + }, + { // band 3 + { 80, 183, 242}, { 37, 160, 231}, { 6, 120, 182}, + { 1, 86, 137}, { 1, 46, 78}, { 1, 15, 24}, + }, + { // band 4 + { 88, 215, 247}, { 42, 179, 235}, { 4, 116, 182}, + { 2, 80, 133}, { 1, 46, 85}, { 1, 64, 43}, + }, + { // band 5 + {100, 236, 250}, { 31, 186, 234}, { 1, 114, 181}, + { 1, 85, 135}, { 1, 78, 64}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {213, 13, 245}, {106, 16, 211}, { 32, 11, 156}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {140, 214, 247}, {241, 186, 243}, {177, 172, 235}, + {128, 156, 219}, {106, 130, 191}, { 99, 105, 152}, + }, + { // band 2 + {125, 218, 248}, { 75, 167, 239}, { 29, 111, 212}, + { 6, 66, 152}, { 1, 42, 96}, { 1, 85, 128}, + }, + { // band 3 + {120, 232, 252}, { 60, 189, 247}, { 8, 141, 200}, + { 1, 89, 134}, { 1, 32, 128}, {128, 128, 128}, + }, + { // band 4 + {111, 238, 253}, { 56, 198, 245}, { 1, 123, 208}, + { 1, 93, 176}, { 1, 1, 73}, {128, 128, 128}, + }, + { // band 5 + { 98, 251, 249}, { 56, 189, 244}, { 17, 113, 220}, + { 1, 109, 179}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + }, +#if CONFIG_TX64X64 + { // TX_SIZE 4 + { // Y plane + { // Intra + { // band 0 + { 51, 21, 156}, { 30, 23, 86}, { 4, 18, 37}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 38, 77, 129}, { 79, 76, 129}, { 40, 66, 117}, + { 12, 54, 95}, { 1, 36, 60}, { 1, 17, 29}, + }, + { // band 2 + { 44, 133, 149}, { 24, 107, 143}, { 8, 78, 121}, + { 3, 59, 97}, { 1, 42, 71}, { 1, 22, 37}, + }, + { // band 3 + { 29, 160, 171}, { 9, 114, 158}, { 1, 76, 125}, + { 1, 54, 93}, { 1, 36, 63}, { 1, 20, 35}, + }, + { // band 4 + { 22, 188, 205}, { 6, 132, 186}, { 1, 87, 144}, + { 1, 62, 107}, { 1, 41, 72}, { 1, 23, 41}, + }, + { // band 5 + { 25, 233, 236}, { 5, 165, 214}, { 1, 96, 158}, + { 1, 63, 112}, { 1, 40, 73}, { 1, 23, 40}, + }, + }, + { // Intra + { // band 0 + { 48, 20, 231}, { 37, 21, 179}, { 15, 18, 109}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 41, 154, 216}, {196, 142, 221}, {131, 125, 207}, + { 84, 111, 181}, { 45, 91, 142}, { 27, 62, 89}, + }, + { // band 2 + { 72, 181, 230}, { 41, 147, 215}, { 10, 102, 173}, + { 3, 73, 132}, { 1, 47, 89}, { 1, 23, 50}, + }, + { // band 3 + { 60, 201, 236}, { 23, 157, 219}, { 2, 99, 167}, + { 1, 69, 124}, { 1, 43, 80}, { 1, 22, 39}, + }, + { // band 4 + { 53, 214, 242}, { 15, 165, 224}, { 1, 101, 173}, + { 1, 70, 131}, { 1, 44, 83}, { 1, 23, 49}, + }, + { // band 5 + { 39, 239, 248}, { 7, 186, 233}, { 1, 108, 174}, + { 1, 70, 123}, { 1, 43, 77}, { 1, 16, 42}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {161, 26, 204}, { 77, 40, 160}, { 26, 50, 117}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 80, 140, 218}, {136, 133, 215}, { 63, 117, 197}, + { 20, 93, 170}, { 7, 55, 102}, { 13, 32, 52}, + }, + { // band 2 + { 86, 173, 231}, { 46, 150, 220}, { 18, 118, 190}, + { 8, 90, 150}, { 2, 60, 95}, { 1, 39, 41}, + }, + { // band 3 + { 80, 183, 242}, { 37, 160, 231}, { 6, 120, 182}, + { 1, 86, 137}, { 1, 46, 78}, { 1, 15, 24}, + }, + { // band 4 + { 88, 215, 247}, { 42, 179, 235}, { 4, 116, 182}, + { 2, 80, 133}, { 1, 46, 85}, { 1, 64, 43}, + }, + { // band 5 + {100, 236, 250}, { 31, 186, 234}, { 1, 114, 181}, + { 1, 85, 135}, { 1, 78, 64}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {213, 13, 245}, {106, 16, 211}, { 32, 11, 156}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {140, 214, 247}, {241, 186, 243}, {177, 172, 235}, + {128, 156, 219}, {106, 130, 191}, { 99, 105, 152}, + }, + { // band 2 + {125, 218, 248}, { 75, 167, 239}, { 29, 111, 212}, + { 6, 66, 152}, { 1, 42, 96}, { 1, 85, 128}, + }, + { // band 3 + {120, 232, 252}, { 60, 189, 247}, { 8, 141, 200}, + { 1, 89, 134}, { 1, 32, 128}, {128, 128, 128}, + }, + { // band 4 + {111, 238, 253}, { 56, 198, 245}, { 1, 123, 208}, + { 1, 93, 176}, { 1, 1, 73}, {128, 128, 128}, + }, + { // band 5 + { 98, 251, 249}, { 56, 189, 244}, { 17, 113, 220}, + { 1, 109, 179}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + }, +#endif // CONFIG_TX64X64 + }, + { // Q_Index 3 +#if CONFIG_CB4X4 + { // TX_SIZE 0 + { // Y plane + { // Intra + { // band 0 + {186, 16, 200}, {122, 31, 187}, { 78, 40, 161}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {146, 119, 245}, {182, 115, 244}, {130, 113, 238}, + { 88, 110, 225}, { 47, 103, 208}, { 5, 102, 188}, + }, + { // band 2 + {164, 157, 248}, {155, 141, 250}, { 71, 116, 243}, + { 88, 129, 233}, { 50, 99, 228}, { 26, 148, 191}, + }, + { // band 3 + {200, 158, 253}, {177, 118, 252}, { 99, 113, 245}, + { 77, 120, 210}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {231, 104, 254}, {209, 82, 254}, {143, 112, 252}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {250, 36, 254}, {243, 55, 254}, {223, 170, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + { // Intra + { // band 0 + {207, 37, 226}, {164, 46, 218}, {122, 58, 201}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {149, 154, 253}, {170, 137, 253}, { 94, 123, 247}, + { 42, 113, 222}, { 16, 97, 174}, { 49, 98, 159}, + }, + { // band 2 + {177, 162, 253}, {165, 142, 252}, { 51, 108, 243}, + { 18, 108, 213}, { 1, 98, 254}, {128, 128, 128}, + }, + { // band 3 + {211, 152, 254}, {184, 116, 254}, { 70, 110, 244}, + { 8, 108, 237}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {236, 89, 254}, {210, 67, 254}, {112, 111, 248}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {246, 26, 254}, {233, 35, 254}, {128, 1, 254}, + {254, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {247, 2, 247}, {226, 8, 242}, {191, 14, 235}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {231, 94, 254}, {248, 91, 254}, {186, 89, 252}, + {128, 92, 244}, { 79, 112, 254}, {128, 128, 128}, + }, + { // band 2 + {228, 145, 253}, {240, 130, 254}, {223, 105, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 3 + {245, 153, 253}, {240, 120, 254}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {254, 128, 254}, {204, 128, 254}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {253, 7, 249}, {224, 9, 244}, {182, 13, 231}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {234, 109, 254}, {242, 104, 254}, {160, 98, 254}, + {123, 85, 243}, { 82, 43, 217}, {128, 128, 128}, + }, + { // band 2 + {243, 137, 254}, {240, 118, 254}, {136, 53, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 3 + {251, 173, 254}, {229, 129, 250}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {254, 119, 254}, {254, 128, 128}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + }, +#endif + { // TX_SIZE 0 + { // Y plane + { // Intra + { // band 0 + {186, 16, 200}, {122, 31, 187}, { 78, 40, 161}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {146, 119, 245}, {182, 115, 244}, {130, 113, 238}, + { 88, 110, 225}, { 47, 103, 208}, { 5, 102, 188}, + }, + { // band 2 + {164, 157, 248}, {155, 141, 250}, { 71, 116, 243}, + { 88, 129, 233}, { 50, 99, 228}, { 26, 148, 191}, + }, + { // band 3 + {200, 158, 253}, {177, 118, 252}, { 99, 113, 245}, + { 77, 120, 210}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {231, 104, 254}, {209, 82, 254}, {143, 112, 252}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {250, 36, 254}, {243, 55, 254}, {223, 170, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + { // Intra + { // band 0 + {207, 37, 226}, {164, 46, 218}, {122, 58, 201}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {149, 154, 253}, {170, 137, 253}, { 94, 123, 247}, + { 42, 113, 222}, { 16, 97, 174}, { 49, 98, 159}, + }, + { // band 2 + {177, 162, 253}, {165, 142, 252}, { 51, 108, 243}, + { 18, 108, 213}, { 1, 98, 254}, {128, 128, 128}, + }, + { // band 3 + {211, 152, 254}, {184, 116, 254}, { 70, 110, 244}, + { 8, 108, 237}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {236, 89, 254}, {210, 67, 254}, {112, 111, 248}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {246, 26, 254}, {233, 35, 254}, {128, 1, 254}, + {254, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {247, 2, 247}, {226, 8, 242}, {191, 14, 235}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {231, 94, 254}, {248, 91, 254}, {186, 89, 252}, + {128, 92, 244}, { 79, 112, 254}, {128, 128, 128}, + }, + { // band 2 + {228, 145, 253}, {240, 130, 254}, {223, 105, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 3 + {245, 153, 253}, {240, 120, 254}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {254, 128, 254}, {204, 128, 254}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {253, 7, 249}, {224, 9, 244}, {182, 13, 231}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {234, 109, 254}, {242, 104, 254}, {160, 98, 254}, + {123, 85, 243}, { 82, 43, 217}, {128, 128, 128}, + }, + { // band 2 + {243, 137, 254}, {240, 118, 254}, {136, 53, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 3 + {251, 173, 254}, {229, 129, 250}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {254, 119, 254}, {254, 128, 128}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + }, + { // TX_SIZE 1 + { // Y plane + { // Intra + { // band 0 + { 49, 26, 159}, { 36, 34, 150}, { 26, 38, 124}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 99, 122, 226}, {143, 119, 225}, { 90, 113, 213}, + { 46, 102, 193}, { 14, 84, 157}, { 3, 59, 107}, + }, + { // band 2 + {109, 164, 237}, { 74, 142, 233}, { 29, 112, 216}, + { 14, 92, 184}, { 10, 80, 156}, { 1, 52, 137}, + }, + { // band 3 + {110, 191, 245}, { 59, 156, 240}, { 18, 121, 220}, + { 8, 97, 184}, { 3, 84, 150}, {128, 128, 128}, + }, + { // band 4 + {115, 203, 250}, { 59, 167, 246}, { 16, 130, 226}, + { 7, 97, 192}, { 1, 71, 99}, {128, 128, 128}, + }, + { // band 5 + {149, 218, 253}, { 93, 171, 251}, { 28, 125, 233}, + { 28, 99, 192}, {128, 85, 85}, {128, 128, 128}, + }, + }, + { // Intra + { // band 0 + { 97, 45, 229}, { 79, 52, 205}, { 46, 58, 171}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 99, 180, 249}, {156, 165, 249}, { 73, 141, 237}, + { 31, 116, 208}, { 13, 81, 153}, { 5, 42, 86}, + }, + { // band 2 + {113, 188, 251}, { 68, 161, 244}, { 16, 108, 216}, + { 6, 81, 168}, { 2, 65, 118}, {128, 1, 1}, + }, + { // band 3 + {117, 201, 252}, { 62, 171, 248}, { 12, 119, 221}, + { 5, 90, 182}, { 4, 66, 116}, {128, 128, 128}, + }, + { // band 4 + {128, 207, 253}, { 70, 176, 251}, { 11, 126, 228}, + { 6, 89, 189}, { 1, 44, 148}, {128, 128, 128}, + }, + { // band 5 + {162, 218, 254}, {107, 170, 253}, { 22, 131, 238}, + { 1, 77, 182}, { 1, 254, 128}, {128, 128, 128}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {235, 5, 238}, {194, 14, 223}, {152, 22, 205}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {200, 121, 251}, {241, 115, 252}, {167, 108, 248}, + { 93, 93, 233}, { 36, 66, 189}, {128, 128, 128}, + }, + { // band 2 + {220, 151, 253}, {176, 135, 252}, { 95, 124, 254}, + { 64, 105, 217}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 3 + {225, 189, 254}, {175, 155, 254}, {102, 119, 254}, + { 1, 1, 1}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {218, 195, 254}, {125, 157, 253}, {128, 128, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {221, 197, 254}, { 85, 210, 254}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {250, 9, 246}, {204, 13, 234}, {144, 18, 211}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {213, 157, 253}, {243, 138, 253}, {170, 117, 250}, + {109, 91, 233}, { 66, 77, 163}, { 64, 85, 254}, + }, + { // band 2 + {221, 169, 254}, {182, 141, 253}, {112, 120, 239}, + { 85, 165, 254}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 3 + {226, 192, 254}, {189, 174, 251}, {153, 128, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {232, 192, 254}, {195, 187, 247}, { 1, 191, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {247, 185, 254}, {254, 93, 254}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + }, + { // TX_SIZE 2 + { // Y plane + { // Intra + { // band 0 + { 14, 30, 136}, { 15, 33, 120}, { 10, 33, 90}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 92, 109, 209}, {113, 108, 207}, { 77, 102, 193}, + { 39, 91, 171}, { 11, 70, 129}, { 2, 44, 77}, + }, + { // band 2 + { 99, 158, 223}, { 66, 135, 217}, { 23, 109, 194}, + { 9, 85, 160}, { 3, 66, 124}, { 1, 51, 100}, + }, + { // band 3 + { 89, 189, 234}, { 46, 149, 225}, { 10, 110, 194}, + { 2, 83, 156}, { 1, 57, 113}, { 1, 47, 73}, + }, + { // band 4 + { 78, 206, 242}, { 28, 161, 232}, { 3, 114, 200}, + { 1, 86, 161}, { 1, 62, 118}, { 1, 1, 1}, + }, + { // band 5 + { 72, 227, 250}, { 20, 182, 242}, { 3, 126, 210}, + { 2, 91, 166}, { 1, 64, 126}, {128, 128, 128}, + }, + }, + { // Intra + { // band 0 + { 23, 42, 227}, { 41, 43, 195}, { 25, 45, 146}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {100, 172, 245}, {165, 158, 246}, { 88, 137, 234}, + { 44, 116, 203}, { 18, 85, 149}, { 7, 56, 92}, + }, + { // band 2 + {117, 188, 247}, { 70, 155, 239}, { 18, 105, 204}, + { 7, 78, 158}, { 2, 50, 111}, { 1, 38, 77}, + }, + { // band 3 + {104, 207, 250}, { 54, 166, 241}, { 6, 110, 199}, + { 1, 78, 155}, { 1, 45, 100}, { 1, 1, 1}, + }, + { // band 4 + { 87, 216, 251}, { 30, 177, 243}, { 1, 114, 203}, + { 1, 85, 157}, { 1, 53, 108}, {128, 128, 128}, + }, + { // band 5 + { 80, 230, 253}, { 23, 193, 248}, { 1, 127, 215}, + { 1, 94, 170}, { 1, 71, 59}, {128, 128, 128}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {222, 9, 234}, {161, 20, 210}, {113, 30, 185}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {195, 120, 248}, {231, 124, 247}, {148, 116, 238}, + { 64, 98, 207}, { 20, 70, 147}, { 87, 68, 100}, + }, + { // band 2 + {186, 161, 250}, {124, 148, 245}, { 44, 123, 230}, + { 23, 107, 205}, { 1, 80, 131}, {128, 128, 128}, + }, + { // band 3 + {172, 196, 252}, {110, 160, 248}, { 37, 134, 235}, + { 23, 125, 200}, {128, 254, 128}, {128, 128, 128}, + }, + { // band 4 + {173, 209, 253}, {103, 175, 250}, { 1, 120, 240}, + { 1, 146, 254}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {184, 235, 254}, { 81, 186, 251}, {128, 109, 254}, + {128, 254, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {248, 8, 243}, {185, 11, 225}, {108, 11, 189}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {208, 158, 254}, {244, 147, 252}, {195, 132, 248}, + {161, 122, 224}, {129, 114, 188}, { 59, 119, 159}, + }, + { // band 2 + {202, 182, 253}, {143, 161, 251}, { 73, 115, 247}, + {146, 175, 204}, {128, 1, 254}, {128, 128, 128}, + }, + { // band 3 + {202, 204, 254}, {131, 174, 251}, { 18, 153, 207}, + {128, 254, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {192, 221, 254}, {114, 190, 254}, {128, 170, 254}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {166, 236, 254}, {119, 200, 254}, {128, 128, 128}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + }, + { // TX_SIZE 3 + { // Y plane + { // Intra + { // band 0 + { 30, 32, 144}, { 21, 35, 96}, { 4, 27, 55}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 35, 107, 172}, { 61, 104, 170}, { 33, 94, 160}, + { 13, 80, 139}, { 2, 55, 97}, { 1, 28, 49}, + }, + { // band 2 + { 51, 153, 195}, { 29, 129, 189}, { 9, 99, 163}, + { 3, 75, 129}, { 1, 49, 88}, { 1, 29, 50}, + }, + { // band 3 + { 53, 164, 210}, { 21, 134, 201}, { 3, 97, 164}, + { 1, 69, 124}, { 1, 45, 82}, { 1, 31, 58}, + }, + { // band 4 + { 47, 205, 234}, { 18, 158, 220}, { 2, 109, 177}, + { 1, 78, 137}, { 1, 53, 101}, { 1, 34, 70}, + }, + { // band 5 + { 55, 233, 245}, { 16, 179, 233}, { 1, 116, 191}, + { 1, 79, 145}, { 1, 53, 101}, { 1, 37, 58}, + }, + }, + { // Intra + { // band 0 + { 36, 33, 227}, { 39, 28, 190}, { 18, 27, 134}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 76, 156, 235}, {184, 147, 235}, {114, 130, 220}, + { 72, 112, 191}, { 42, 87, 144}, { 21, 65, 93}, + }, + { // band 2 + { 96, 179, 240}, { 51, 149, 228}, { 12, 105, 191}, + { 6, 74, 148}, { 1, 47, 100}, { 1, 29, 53}, + }, + { // band 3 + { 88, 191, 242}, { 35, 154, 231}, { 3, 106, 187}, + { 1, 74, 140}, { 1, 41, 84}, { 1, 25, 38}, + }, + { // band 4 + { 77, 212, 249}, { 28, 171, 239}, { 2, 117, 199}, + { 1, 79, 151}, { 1, 45, 99}, { 1, 1, 1}, + }, + { // band 5 + { 77, 236, 252}, { 27, 190, 246}, { 2, 120, 203}, + { 1, 78, 147}, { 1, 42, 72}, {128, 128, 128}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {185, 11, 227}, {113, 30, 182}, { 57, 44, 144}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {151, 139, 244}, {212, 139, 241}, {124, 126, 231}, + { 59, 104, 213}, { 26, 73, 158}, { 20, 45, 95}, + }, + { // band 2 + {155, 163, 247}, {108, 152, 239}, { 39, 124, 214}, + { 7, 109, 162}, { 29, 57, 128}, {128, 128, 128}, + }, + { // band 3 + {158, 176, 250}, { 89, 164, 243}, { 11, 114, 196}, + { 1, 96, 141}, { 1, 81, 118}, {128, 1, 1}, + }, + { // band 4 + {148, 212, 251}, { 59, 174, 240}, { 2, 130, 203}, + { 1, 70, 168}, { 1, 51, 106}, {128, 128, 128}, + }, + { // band 5 + {104, 237, 252}, { 39, 190, 246}, { 1, 154, 220}, + {128, 102, 1}, {128, 128, 128}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {236, 6, 242}, {111, 6, 206}, { 36, 5, 161}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {193, 193, 252}, {248, 182, 251}, {218, 150, 246}, + {182, 134, 244}, {151, 137, 227}, { 45, 102, 195}, + }, + { // band 2 + {188, 202, 251}, {125, 165, 249}, { 64, 75, 218}, + { 1, 128, 254}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 3 + {178, 225, 254}, {107, 188, 231}, { 21, 135, 233}, + {128, 1, 254}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {164, 227, 253}, { 55, 193, 251}, { 1, 111, 225}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {151, 243, 254}, { 50, 203, 254}, {128, 179, 254}, + {128, 1, 254}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + }, +#if CONFIG_TX64X64 + { // TX_SIZE 4 + { // Y plane + { // Intra + { // band 0 + { 30, 32, 144}, { 21, 35, 96}, { 4, 27, 55}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 35, 107, 172}, { 61, 104, 170}, { 33, 94, 160}, + { 13, 80, 139}, { 2, 55, 97}, { 1, 28, 49}, + }, + { // band 2 + { 51, 153, 195}, { 29, 129, 189}, { 9, 99, 163}, + { 3, 75, 129}, { 1, 49, 88}, { 1, 29, 50}, + }, + { // band 3 + { 53, 164, 210}, { 21, 134, 201}, { 3, 97, 164}, + { 1, 69, 124}, { 1, 45, 82}, { 1, 31, 58}, + }, + { // band 4 + { 47, 205, 234}, { 18, 158, 220}, { 2, 109, 177}, + { 1, 78, 137}, { 1, 53, 101}, { 1, 34, 70}, + }, + { // band 5 + { 55, 233, 245}, { 16, 179, 233}, { 1, 116, 191}, + { 1, 79, 145}, { 1, 53, 101}, { 1, 37, 58}, + }, + }, + { // Intra + { // band 0 + { 36, 33, 227}, { 39, 28, 190}, { 18, 27, 134}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + { 76, 156, 235}, {184, 147, 235}, {114, 130, 220}, + { 72, 112, 191}, { 42, 87, 144}, { 21, 65, 93}, + }, + { // band 2 + { 96, 179, 240}, { 51, 149, 228}, { 12, 105, 191}, + { 6, 74, 148}, { 1, 47, 100}, { 1, 29, 53}, + }, + { // band 3 + { 88, 191, 242}, { 35, 154, 231}, { 3, 106, 187}, + { 1, 74, 140}, { 1, 41, 84}, { 1, 25, 38}, + }, + { // band 4 + { 77, 212, 249}, { 28, 171, 239}, { 2, 117, 199}, + { 1, 79, 151}, { 1, 45, 99}, { 1, 1, 1}, + }, + { // band 5 + { 77, 236, 252}, { 27, 190, 246}, { 2, 120, 203}, + { 1, 78, 147}, { 1, 42, 72}, {128, 128, 128}, + }, + }, + }, + { // UV plane + { // Inter + { // band 0 + {185, 11, 227}, {113, 30, 182}, { 57, 44, 144}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {151, 139, 244}, {212, 139, 241}, {124, 126, 231}, + { 59, 104, 213}, { 26, 73, 158}, { 20, 45, 95}, + }, + { // band 2 + {155, 163, 247}, {108, 152, 239}, { 39, 124, 214}, + { 7, 109, 162}, { 29, 57, 128}, {128, 128, 128}, + }, + { // band 3 + {158, 176, 250}, { 89, 164, 243}, { 11, 114, 196}, + { 1, 96, 141}, { 1, 81, 118}, {128, 1, 1}, + }, + { // band 4 + {148, 212, 251}, { 59, 174, 240}, { 2, 130, 203}, + { 1, 70, 168}, { 1, 51, 106}, {128, 128, 128}, + }, + { // band 5 + {104, 237, 252}, { 39, 190, 246}, { 1, 154, 220}, + {128, 102, 1}, {128, 128, 128}, {128, 128, 128}, + }, + }, + { // Inter + { // band 0 + {236, 6, 242}, {111, 6, 206}, { 36, 5, 161}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 1 + {193, 193, 252}, {248, 182, 251}, {218, 150, 246}, + {182, 134, 244}, {151, 137, 227}, { 45, 102, 195}, + }, + { // band 2 + {188, 202, 251}, {125, 165, 249}, { 64, 75, 218}, + { 1, 128, 254}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 3 + {178, 225, 254}, {107, 188, 231}, { 21, 135, 233}, + {128, 1, 254}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 4 + {164, 227, 253}, { 55, 193, 251}, { 1, 111, 225}, + {128, 128, 128}, {128, 128, 128}, {128, 128, 128}, + }, + { // band 5 + {151, 243, 254}, { 50, 203, 254}, {128, 179, 254}, + {128, 1, 254}, {128, 128, 128}, {128, 128, 128}, + }, + }, + }, + }, +#endif // CONFIG_TX64X64 + }, +}; +#else +#if CONFIG_NEW_TOKENSET +static const av1_coeff_probs_model default_coef_probs_4x4[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + {97, 27, 144}, {81, 38, 128}, {51, 42, 99} + }, + { // Band 1 + {74, 113, 204}, {68, 101, 199}, {50, 87, 173}, + {31, 76, 133}, {13, 55, 86}, {3, 30, 39} + }, + { // Band 2 + {83, 156, 222}, {74, 127, 215}, {46, 101, 179}, + {30, 80, 129}, {14, 57, 81}, {3, 27, 37} + }, + { // Band 3 + {105, 164, 233}, {84, 128, 224}, {49, 92, 175}, + {28, 60, 114}, {12, 34, 53}, {20, 59, 98} + }, + { // Band 4 + {131, 159, 243}, {98, 123, 228}, {40, 78, 151}, + {19, 46, 97}, {13, 47, 19}, {19, 16, 19} + }, + { // Band 5 + {192, 71, 241}, {174, 70, 226}, {125, 46, 153}, + {108, 49, 116}, {82, 24, 46}, {60, 14, 30} + } + }, + { // Inter + { // Band 0 + {111, 66, 218}, {107, 87, 211}, {93, 99, 207} + }, + { // Band 1 + {107, 166, 250}, {107, 143, 247}, {73, 119, 221}, + {43, 91, 166}, {17, 74, 102}, {3, 70, 53} + }, + { // Band 2 + {126, 177, 251}, {109, 148, 246}, {64, 99, 204}, + {42, 68, 140}, {28, 52, 84}, {20, 34, 1} + }, + { // Band 3 + {143, 178, 252}, {114, 144, 245}, {46, 92, 188}, + {45, 65, 104}, {40, 44, 76}, {1, 1, 1} + }, + { // Band 4 + {163, 159, 251}, {120, 131, 243}, {47, 81, 182}, + {32, 39, 128}, {33, 44, 56}, {1, 17, 34} + }, + { // Band 5 + {209, 94, 251}, {190, 81, 241}, {139, 45, 147}, + {123, 35, 73}, {118, 1, 118}, {3, 16, 42} + } + } + }, + { // UV plane + { // Intra + { // Band 0 + {189, 37, 229}, {145, 68, 205}, {99, 74, 171} + }, + { // Band 1 + {153, 139, 242}, {135, 125, 235}, {84, 100, 200}, + {49, 75, 162}, {9, 21, 84}, {3, 31, 69} + }, + { // Band 2 + {165, 165, 244}, {128, 144, 240}, {68, 94, 204}, + {39, 72, 132}, {22, 44, 93}, {26, 73, 26} + }, + { // Band 3 + {181, 174, 246}, {142, 132, 241}, {81, 96, 212}, + {41, 70, 166}, {9, 48, 92}, {1, 19, 38} + }, + { // Band 4 + {197, 159, 251}, {168, 121, 245}, {107, 75, 218}, + {70, 43, 158}, {1, 128, 1}, {1, 18, 37} + }, + { // Band 5 + {231, 79, 255}, {211, 74, 249}, {157, 104, 210}, + {128, 102, 213}, {12, 34, 96}, {2, 20, 47} + } + }, + { // Inter + { // Band 0 + {220, 53, 252}, {191, 80, 248}, {154, 100, 245} + }, + { // Band 1 + {205, 153, 255}, {182, 147, 254}, {110, 131, 231}, + {68, 114, 161}, {50, 114, 140}, {1, 33, 57} + }, + { // Band 2 + {213, 171, 255}, {184, 163, 254}, {116, 104, 235}, + {79, 71, 207}, {1, 41, 79}, {1, 20, 39} + }, + { // Band 3 + {223, 158, 255}, {203, 137, 255}, {111, 142, 244}, + {2, 255, 133}, {1, 44, 85}, {1, 22, 47} + }, + { // Band 4 + {232, 148, 255}, {222, 123, 255}, {255, 128, 255}, + {3, 61, 124}, {1, 41, 84}, {1, 21, 52} + }, + { // Band 5 + {248, 92, 255}, {248, 96, 255}, {69, 58, 184}, + {31, 44, 137}, {14, 38, 105}, {8, 23, 61} + } + } + } +}; +static const av1_coeff_probs_model default_coef_probs_8x8[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + {112, 31, 159}, {72, 37, 119}, {22, 35, 68} + }, + { // Band 1 + {42, 109, 174}, {45, 99, 172}, {32, 84, 149}, + {18, 69, 119}, {6, 46, 76}, {1, 19, 31} + }, + { // Band 2 + {40, 154, 202}, {35, 126, 191}, {19, 98, 160}, + {10, 75, 122}, {5, 53, 82}, {1, 23, 39} + }, + { // Band 3 + {39, 176, 215}, {28, 135, 200}, {11, 93, 156}, + {5, 63, 109}, {1, 36, 64}, {1, 14, 26} + }, + { // Band 4 + {41, 191, 230}, {25, 147, 212}, {9, 97, 160}, + {3, 65, 109}, {1, 33, 58}, {1, 14, 20} + }, + { // Band 5 + {68, 203, 242}, {40, 159, 220}, {12, 97, 153}, + {5, 58, 97}, {1, 29, 55}, {1, 11, 18} + } + }, + { // Inter + { // Band 0 + {99, 67, 221}, {86, 80, 204}, {60, 87, 184} + }, + { // Band 1 + {73, 169, 246}, {79, 158, 242}, {50, 135, 220}, + {30, 113, 181}, {18, 76, 126}, {5, 54, 85} + }, + { // Band 2 + {90, 184, 250}, {78, 162, 243}, {47, 118, 214}, + {35, 85, 171}, {32, 53, 115}, {20, 28, 76} + }, + { // Band 3 + {109, 197, 252}, {89, 172, 247}, {52, 119, 217}, + {37, 80, 161}, {23, 44, 100}, {1, 18, 34} + }, + { // Band 4 + {132, 202, 254}, {110, 175, 251}, {63, 128, 228}, + {37, 86, 168}, {64, 91, 102}, {1, 17, 34} + }, + { // Band 5 + {126, 204, 253}, {100, 174, 250}, {50, 148, 237}, + {25, 90, 133}, {1, 64, 85}, {3, 16, 42} + } + } + }, + { // UV plane + { // Intra + { // Band 0 + {195, 35, 235}, {137, 63, 201}, {62, 70, 145} + }, + { // Band 1 + {110, 158, 233}, {102, 143, 227}, {60, 120, 199}, + {30, 85, 156}, {9, 50, 90}, {1, 16, 33} + }, + { // Band 2 + {102, 185, 233}, {71, 152, 224}, {29, 111, 187}, + {18, 74, 138}, {4, 56, 87}, {1, 18, 46} + }, + { // Band 3 + {101, 205, 239}, {66, 161, 229}, {23, 109, 183}, + {9, 85, 135}, {5, 71, 142}, {1, 1, 102} + }, + { // Band 4 + {109, 216, 243}, {69, 168, 233}, {23, 119, 191}, + {8, 137, 115}, {1, 54, 98}, {1, 1, 255} + }, + { // Band 5 + {139, 224, 249}, {98, 176, 238}, {55, 129, 187}, + {25, 101, 131}, {26, 59, 154}, {2, 20, 47} + } + }, + { // Inter + { // Band 0 + {220, 72, 254}, {176, 108, 251}, {114, 132, 247} + }, + { // Band 1 + {161, 185, 255}, {141, 185, 254}, {131, 180, 249}, + {111, 164, 186}, {50, 98, 142}, {1, 128, 1} + }, + { // Band 2 + {171, 195, 255}, {133, 184, 254}, {68, 140, 231}, + {102, 96, 205}, {1, 1, 128}, {1, 20, 39} + }, + { // Band 3 + {180, 206, 255}, {148, 191, 254}, {83, 157, 241}, + {128, 171, 128}, {1, 44, 85}, {1, 22, 47} + }, + { // Band 4 + {194, 214, 255}, {159, 188, 255}, {122, 148, 250}, + {3, 255, 124}, {1, 41, 84}, {1, 21, 52} + }, + { // Band 5 + {231, 217, 255}, {209, 149, 255}, {205, 145, 205}, + {31, 44, 137}, {14, 38, 105}, {8, 23, 61} + } + } + } +}; +static const av1_coeff_probs_model default_coef_probs_16x16[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + {91, 31, 117}, {49, 31, 89}, {14, 25, 48} + }, + { // Band 1 + {31, 97, 151}, {33, 89, 148}, {28, 76, 133}, + {17, 60, 106}, {7, 42, 72}, {1, 19, 32} + }, + { // Band 2 + {28, 152, 182}, {28, 120, 174}, {15, 93, 146}, + {9, 72, 116}, {5, 47, 82}, {1, 21, 37} + }, + { // Band 3 + {29, 174, 203}, {23, 127, 187}, {9, 89, 145}, + {2, 56, 100}, {1, 31, 56}, {1, 12, 25} + }, + { // Band 4 + {28, 193, 220}, {17, 141, 197}, {4, 87, 142}, + {1, 54, 95}, {1, 31, 56}, {1, 12, 26} + }, + { // Band 5 + {29, 221, 240}, {11, 167, 215}, {2, 93, 149}, + {1, 58, 100}, {1, 35, 61}, {1, 16, 28} + } + }, + { // Inter + { // Band 0 + {108, 52, 214}, {84, 60, 186}, {45, 69, 161} + }, + { // Band 1 + {43, 164, 236}, {57, 161, 233}, {38, 146, 214}, + {24, 120, 182}, {15, 80, 126}, {5, 28, 66} + }, + { // Band 2 + {58, 187, 242}, {47, 163, 234}, {28, 118, 204}, + {26, 82, 165}, {21, 54, 112}, {4, 28, 55} + }, + { // Band 3 + {65, 201, 248}, {51, 170, 239}, {22, 117, 204}, + {11, 81, 159}, {10, 43, 102}, {1, 1, 1} + }, + { // Band 4 + {80, 206, 252}, {57, 179, 245}, {25, 129, 214}, + {16, 97, 170}, {6, 60, 130}, {1, 128, 1} + }, + { // Band 5 + {97, 217, 253}, {68, 186, 250}, {26, 138, 216}, + {20, 105, 166}, {11, 78, 111}, {3, 16, 42} + } + } + }, + { // UV plane + { // Intra + { // Band 0 + {181, 37, 233}, {121, 55, 192}, {46, 52, 124} + }, + { // Band 1 + {108, 157, 221}, {98, 140, 215}, {59, 124, 187}, + {34, 92, 158}, {9, 68, 112}, {1, 41, 70} + }, + { // Band 2 + {80, 188, 223}, {46, 153, 204}, {25, 91, 173}, + {11, 73, 131}, {5, 43, 82}, {1, 17, 91} + }, + { // Band 3 + {63, 209, 228}, {31, 157, 206}, {8, 104, 167}, + {3, 63, 122}, {1, 44, 87}, {1, 43, 51} + }, + { // Band 4 + {52, 220, 234}, {22, 165, 216}, {4, 104, 163}, + {2, 62, 129}, {1, 33, 50}, {1, 26, 28} + }, + { // Band 5 + {58, 238, 242}, {24, 183, 224}, {4, 109, 172}, + {2, 87, 141}, {1, 52, 79}, {1, 51, 64} + } + }, + { // Inter + { // Band 0 + {224, 52, 250}, {188, 81, 239}, {138, 114, 228} + }, + { // Band 1 + {131, 206, 255}, {128, 193, 254}, {119, 173, 247}, + {106, 127, 187}, {50, 100, 124}, {1, 96, 1} + }, + { // Band 2 + {123, 214, 254}, {86, 194, 254}, {64, 119, 221}, + {43, 51, 128}, {1, 32, 110}, {1, 20, 39} + }, + { // Band 3 + {115, 223, 255}, {78, 200, 254}, {75, 164, 203}, + {128, 85, 255}, {1, 44, 85}, {1, 22, 47} + }, + { // Band 4 + {132, 226, 255}, {88, 207, 254}, {20, 140, 225}, + {3, 61, 124}, {1, 41, 84}, {1, 21, 52} + }, + { // Band 5 + {180, 236, 255}, {138, 223, 254}, {73, 166, 238}, + {31, 255, 137}, {14, 38, 105}, {8, 23, 61} + } + } + } +}; +static const av1_coeff_probs_model default_coef_probs_32x32[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + {163, 26, 188}, {78, 29, 105}, {22, 22, 48} + }, + { // Band 1 + {72, 93, 168}, {74, 91, 170}, {62, 72, 151}, + {37, 55, 112}, {10, 33, 63}, {1, 14, 23} + }, + { // Band 2 + {41, 163, 182}, {36, 136, 177}, {20, 102, 153}, + {10, 76, 114}, {5, 45, 71}, {1, 17, 27} + }, + { // Band 3 + {43, 202, 213}, {28, 142, 193}, {10, 90, 141}, + {2, 51, 93}, {1, 24, 48}, {1, 10, 19} + }, + { // Band 4 + {46, 216, 220}, {26, 150, 199}, {7, 87, 136}, + {2, 49, 86}, {1, 28, 47}, {1, 12, 24} + }, + { // Band 5 + {19, 241, 237}, {5, 172, 200}, {1, 82, 126}, + {1, 47, 79}, {1, 29, 47}, {1, 14, 25} + } + }, + { // Inter + { // Band 0 + {185, 20, 226}, {151, 26, 187}, {109, 34, 144} + }, + { // Band 1 + {56, 151, 227}, {76, 165, 232}, {62, 161, 222}, + {47, 139, 201}, {29, 106, 150}, {14, 61, 98} + }, + { // Band 2 + {57, 200, 237}, {43, 164, 227}, {22, 106, 190}, + {14, 68, 140}, {10, 48, 90}, {1, 15, 40} + }, + { // Band 3 + {46, 209, 238}, {28, 165, 225}, {7, 107, 180}, + {2, 69, 125}, {2, 36, 94}, {1, 1, 1} + }, + { // Band 4 + {55, 225, 248}, {28, 181, 237}, {7, 117, 198}, + {6, 77, 144}, {3, 60, 90}, {1, 1, 1} + }, + { // Band 5 + {63, 243, 251}, {27, 193, 242}, {4, 124, 200}, + {1, 58, 153}, {1, 59, 124}, {3, 16, 42} + } + } + }, + { // UV plane + { // Intra + { // Band 0 + {208, 28, 218}, {183, 32, 188}, {169, 21, 189} + }, + { // Band 1 + {205, 124, 247}, {190, 96, 240}, {233, 89, 233}, + {177, 44, 212}, {59, 58, 59}, {32, 33, 38} + }, + { // Band 2 + {194, 195, 250}, {179, 190, 226}, {32, 174, 128}, + {32, 85, 128}, {12, 64, 122}, {1, 85, 90} + }, + { // Band 3 + {149, 232, 249}, {95, 159, 227}, {28, 91, 171}, + {28, 102, 114}, {1, 1, 73}, {1, 19, 38} + }, + { // Band 4 + {154, 239, 246}, {138, 151, 235}, {1, 123, 138}, + {128, 183, 255}, {1, 128, 1}, {1, 18, 37} + }, + { // Band 5 + {157, 255, 253}, {75, 171, 241}, {43, 102, 171}, + {30, 44, 136}, {12, 34, 96}, {2, 20, 47} + } + }, + { // Inter + { // Band 0 + {249, 13, 248}, {238, 14, 220}, {225, 16, 174} + }, + { // Band 1 + {190, 189, 254}, {169, 134, 253}, {124, 179, 248}, + {138, 131, 223}, {64, 133, 192}, {1, 85, 128} + }, + { // Band 2 + {139, 212, 254}, {126, 177, 255}, {93, 39, 186}, + {1, 1, 171}, {1, 41, 79}, {1, 20, 39} + }, + { // Band 3 + {153, 216, 255}, {165, 204, 255}, {1, 1, 255}, + {2, 73, 133}, {1, 1, 1}, {1, 22, 47} + }, + { // Band 4 + {147, 226, 254}, {119, 196, 255}, {1, 128, 255}, + {1, 1, 171}, {1, 1, 1}, {1, 21, 52} + }, + { // Band 5 + {168, 240, 255}, {95, 179, 255}, {1, 171, 1}, + {31, 44, 137}, {14, 38, 105}, {8, 23, 61} + } + } + } +}; +#else // CONFIG_NEW_TOKENSET +static const av1_coeff_probs_model default_coef_probs_4x4[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + { 195, 29, 183 }, { 84, 49, 136 }, { 8, 42, 71 } + }, { // Band 1 + { 31, 107, 169 }, { 35, 99, 159 }, { 17, 82, 140 }, + { 8, 66, 114 }, { 2, 44, 76 }, { 1, 19, 32 } + }, { // Band 2 + { 40, 132, 201 }, { 29, 114, 187 }, { 13, 91, 157 }, + { 7, 75, 127 }, { 3, 58, 95 }, { 1, 28, 47 } + }, { // Band 3 + { 69, 142, 221 }, { 42, 122, 201 }, { 15, 91, 159 }, + { 6, 67, 121 }, { 1, 42, 77 }, { 1, 17, 31 } + }, { // Band 4 + { 102, 148, 228 }, { 67, 117, 204 }, { 17, 82, 154 }, + { 6, 59, 114 }, { 2, 39, 75 }, { 1, 15, 29 } + }, { // Band 5 + { 156, 57, 233 }, { 119, 57, 212 }, { 58, 48, 163 }, + { 29, 40, 124 }, { 12, 30, 81 }, { 3, 12, 31 } + } + }, { // Inter + { // Band 0 + { 191, 107, 226 }, { 124, 117, 204 }, { 25, 99, 155 } + }, { // Band 1 + { 29, 148, 210 }, { 37, 126, 194 }, { 8, 93, 157 }, + { 2, 68, 118 }, { 1, 39, 69 }, { 1, 17, 33 } + }, { // Band 2 + { 41, 151, 213 }, { 27, 123, 193 }, { 3, 82, 144 }, + { 1, 58, 105 }, { 1, 32, 60 }, { 1, 13, 26 } + }, { // Band 3 + { 59, 159, 220 }, { 23, 126, 198 }, { 4, 88, 151 }, + { 1, 66, 114 }, { 1, 38, 71 }, { 1, 18, 34 } + }, { // Band 4 + { 114, 136, 232 }, { 51, 114, 207 }, { 11, 83, 155 }, + { 3, 56, 105 }, { 1, 33, 65 }, { 1, 17, 34 } + }, { // Band 5 + { 149, 65, 234 }, { 121, 57, 215 }, { 61, 49, 166 }, + { 28, 36, 114 }, { 12, 25, 76 }, { 3, 16, 42 } + } + } + }, { // UV plane + { // Intra + { // Band 0 + { 214, 49, 220 }, { 132, 63, 188 }, { 42, 65, 137 } + }, { // Band 1 + { 85, 137, 221 }, { 104, 131, 216 }, { 49, 111, 192 }, + { 21, 87, 155 }, { 2, 49, 87 }, { 1, 16, 28 } + }, { // Band 2 + { 89, 163, 230 }, { 90, 137, 220 }, { 29, 100, 183 }, + { 10, 70, 135 }, { 2, 42, 81 }, { 1, 17, 33 } + }, { // Band 3 + { 108, 167, 237 }, { 55, 133, 222 }, { 15, 97, 179 }, + { 4, 72, 135 }, { 1, 45, 85 }, { 1, 19, 38 } + }, { // Band 4 + { 124, 146, 240 }, { 66, 124, 224 }, { 17, 88, 175 }, + { 4, 58, 122 }, { 1, 36, 75 }, { 1, 18, 37 } + }, { // Band 5 + { 141, 79, 241 }, { 126, 70, 227 }, { 66, 58, 182 }, + { 30, 44, 136 }, { 12, 34, 96 }, { 2, 20, 47 } + } + }, { // Inter + { // Band 0 + { 229, 99, 249 }, { 143, 111, 235 }, { 46, 109, 192 } + }, { // Band 1 + { 82, 158, 236 }, { 94, 146, 224 }, { 25, 117, 191 }, + { 9, 87, 149 }, { 3, 56, 99 }, { 1, 33, 57 } + }, { // Band 2 + { 83, 167, 237 }, { 68, 145, 222 }, { 10, 103, 177 }, + { 2, 72, 131 }, { 1, 41, 79 }, { 1, 20, 39 } + }, { // Band 3 + { 99, 167, 239 }, { 47, 141, 224 }, { 10, 104, 178 }, + { 2, 73, 133 }, { 1, 44, 85 }, { 1, 22, 47 } + }, { // Band 4 + { 127, 145, 243 }, { 71, 129, 228 }, { 17, 93, 177 }, + { 3, 61, 124 }, { 1, 41, 84 }, { 1, 21, 52 } + }, { // Band 5 + { 157, 78, 244 }, { 140, 72, 231 }, { 69, 58, 184 }, + { 31, 44, 137 }, { 14, 38, 105 }, { 8, 23, 61 } + } + } + } +}; + +static const av1_coeff_probs_model default_coef_probs_8x8[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + { 125, 34, 187 }, { 52, 41, 133 }, { 6, 31, 56 } + }, { // Band 1 + { 37, 109, 153 }, { 51, 102, 147 }, { 23, 87, 128 }, + { 8, 67, 101 }, { 1, 41, 63 }, { 1, 19, 29 } + }, { // Band 2 + { 31, 154, 185 }, { 17, 127, 175 }, { 6, 96, 145 }, + { 2, 73, 114 }, { 1, 51, 82 }, { 1, 28, 45 } + }, { // Band 3 + { 23, 163, 200 }, { 10, 131, 185 }, { 2, 93, 148 }, + { 1, 67, 111 }, { 1, 41, 69 }, { 1, 14, 24 } + }, { // Band 4 + { 29, 176, 217 }, { 12, 145, 201 }, { 3, 101, 156 }, + { 1, 69, 111 }, { 1, 39, 63 }, { 1, 14, 23 } + }, { // Band 5 + { 57, 192, 233 }, { 25, 154, 215 }, { 6, 109, 167 }, + { 3, 78, 118 }, { 1, 48, 69 }, { 1, 21, 29 } + } + }, { // Inter + { // Band 0 + { 202, 105, 245 }, { 108, 106, 216 }, { 18, 90, 144 } + }, { // Band 1 + { 33, 172, 219 }, { 64, 149, 206 }, { 14, 117, 177 }, + { 5, 90, 141 }, { 2, 61, 95 }, { 1, 37, 57 } + }, { // Band 2 + { 33, 179, 220 }, { 11, 140, 198 }, { 1, 89, 148 }, + { 1, 60, 104 }, { 1, 33, 57 }, { 1, 12, 21 } + }, { // Band 3 + { 30, 181, 221 }, { 8, 141, 198 }, { 1, 87, 145 }, + { 1, 58, 100 }, { 1, 31, 55 }, { 1, 12, 20 } + }, { // Band 4 + { 32, 186, 224 }, { 7, 142, 198 }, { 1, 86, 143 }, + { 1, 58, 100 }, { 1, 31, 55 }, { 1, 12, 22 } + }, { // Band 5 + { 57, 192, 227 }, { 20, 143, 204 }, { 3, 96, 154 }, + { 1, 68, 112 }, { 1, 42, 69 }, { 1, 19, 32 } + } + } + }, { // UV plane + { // Intra + { // Band 0 + { 212, 35, 215 }, { 113, 47, 169 }, { 29, 48, 105 } + }, { // Band 1 + { 74, 129, 203 }, { 106, 120, 203 }, { 49, 107, 178 }, + { 19, 84, 144 }, { 4, 50, 84 }, { 1, 15, 25 } + }, { // Band 2 + { 71, 172, 217 }, { 44, 141, 209 }, { 15, 102, 173 }, + { 6, 76, 133 }, { 2, 51, 89 }, { 1, 24, 42 } + }, { // Band 3 + { 64, 185, 231 }, { 31, 148, 216 }, { 8, 103, 175 }, + { 3, 74, 131 }, { 1, 46, 81 }, { 1, 18, 30 } + }, { // Band 4 + { 65, 196, 235 }, { 25, 157, 221 }, { 5, 105, 174 }, + { 1, 67, 120 }, { 1, 38, 69 }, { 1, 15, 30 } + }, { // Band 5 + { 65, 204, 238 }, { 30, 156, 224 }, { 7, 107, 177 }, + { 2, 70, 124 }, { 1, 42, 73 }, { 1, 18, 34 } + } + }, { // Inter + { // Band 0 + { 225, 86, 251 }, { 144, 104, 235 }, { 42, 99, 181 } + }, { // Band 1 + { 85, 175, 239 }, { 112, 165, 229 }, { 29, 136, 200 }, + { 12, 103, 162 }, { 6, 77, 123 }, { 2, 53, 84 } + }, { // Band 2 + { 75, 183, 239 }, { 30, 155, 221 }, { 3, 106, 171 }, + { 1, 74, 128 }, { 1, 44, 76 }, { 1, 17, 28 } + }, { // Band 3 + { 73, 185, 240 }, { 27, 159, 222 }, { 2, 107, 172 }, + { 1, 75, 127 }, { 1, 42, 73 }, { 1, 17, 29 } + }, { // Band 4 + { 62, 190, 238 }, { 21, 159, 222 }, { 2, 107, 172 }, + { 1, 72, 122 }, { 1, 40, 71 }, { 1, 18, 32 } + }, { // Band 5 + { 61, 199, 240 }, { 27, 161, 226 }, { 4, 113, 180 }, + { 1, 76, 129 }, { 1, 46, 80 }, { 1, 23, 41 } + } + } + } +}; + +static const av1_coeff_probs_model default_coef_probs_16x16[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + { 7, 27, 153 }, { 5, 30, 95 }, { 1, 16, 30 } + }, { // Band 1 + { 50, 75, 127 }, { 57, 75, 124 }, { 27, 67, 108 }, + { 10, 54, 86 }, { 1, 33, 52 }, { 1, 12, 18 } + }, { // Band 2 + { 43, 125, 151 }, { 26, 108, 148 }, { 7, 83, 122 }, + { 2, 59, 89 }, { 1, 38, 60 }, { 1, 17, 27 } + }, { // Band 3 + { 23, 144, 163 }, { 13, 112, 154 }, { 2, 75, 117 }, + { 1, 50, 81 }, { 1, 31, 51 }, { 1, 14, 23 } + }, { // Band 4 + { 18, 162, 185 }, { 6, 123, 171 }, { 1, 78, 125 }, + { 1, 51, 86 }, { 1, 31, 54 }, { 1, 14, 23 } + }, { // Band 5 + { 15, 199, 227 }, { 3, 150, 204 }, { 1, 91, 146 }, + { 1, 55, 95 }, { 1, 30, 53 }, { 1, 11, 20 } + } + }, { // Inter + { // Band 0 + { 19, 55, 240 }, { 19, 59, 196 }, { 3, 52, 105 } + }, { // Band 1 + { 41, 166, 207 }, { 104, 153, 199 }, { 31, 123, 181 }, + { 14, 101, 152 }, { 5, 72, 106 }, { 1, 36, 52 } + }, { // Band 2 + { 35, 176, 211 }, { 12, 131, 190 }, { 2, 88, 144 }, + { 1, 60, 101 }, { 1, 36, 60 }, { 1, 16, 28 } + }, { // Band 3 + { 28, 183, 213 }, { 8, 134, 191 }, { 1, 86, 142 }, + { 1, 56, 96 }, { 1, 30, 53 }, { 1, 12, 20 } + }, { // Band 4 + { 20, 190, 215 }, { 4, 135, 192 }, { 1, 84, 139 }, + { 1, 53, 91 }, { 1, 28, 49 }, { 1, 11, 20 } + }, { // Band 5 + { 13, 196, 216 }, { 2, 137, 192 }, { 1, 86, 143 }, + { 1, 57, 99 }, { 1, 32, 56 }, { 1, 13, 24 } + } + } + }, { // UV plane + { // Intra + { // Band 0 + { 211, 29, 217 }, { 96, 47, 156 }, { 22, 43, 87 } + }, { // Band 1 + { 78, 120, 193 }, { 111, 116, 186 }, { 46, 102, 164 }, + { 15, 80, 128 }, { 2, 49, 76 }, { 1, 18, 28 } + }, { // Band 2 + { 71, 161, 203 }, { 42, 132, 192 }, { 10, 98, 150 }, + { 3, 69, 109 }, { 1, 44, 70 }, { 1, 18, 29 } + }, { // Band 3 + { 57, 186, 211 }, { 30, 140, 196 }, { 4, 93, 146 }, + { 1, 62, 102 }, { 1, 38, 65 }, { 1, 16, 27 } + }, { // Band 4 + { 47, 199, 217 }, { 14, 145, 196 }, { 1, 88, 142 }, + { 1, 57, 98 }, { 1, 36, 62 }, { 1, 15, 26 } + }, { // Band 5 + { 26, 219, 229 }, { 5, 155, 207 }, { 1, 94, 151 }, + { 1, 60, 104 }, { 1, 36, 62 }, { 1, 16, 28 } + } + }, { // Inter + { // Band 0 + { 233, 29, 248 }, { 146, 47, 220 }, { 43, 52, 140 } + }, { // Band 1 + { 100, 163, 232 }, { 179, 161, 222 }, { 63, 142, 204 }, + { 37, 113, 174 }, { 26, 89, 137 }, { 18, 68, 97 } + }, { // Band 2 + { 85, 181, 230 }, { 32, 146, 209 }, { 7, 100, 164 }, + { 3, 71, 121 }, { 1, 45, 77 }, { 1, 18, 30 } + }, { // Band 3 + { 65, 187, 230 }, { 20, 148, 207 }, { 2, 97, 159 }, + { 1, 68, 116 }, { 1, 40, 70 }, { 1, 14, 29 } + }, { // Band 4 + { 40, 194, 227 }, { 8, 147, 204 }, { 1, 94, 155 }, + { 1, 65, 112 }, { 1, 39, 66 }, { 1, 14, 26 } + }, { // Band 5 + { 16, 208, 228 }, { 3, 151, 207 }, { 1, 98, 160 }, + { 1, 67, 117 }, { 1, 41, 74 }, { 1, 17, 31 } + } + } + } +}; + +static const av1_coeff_probs_model default_coef_probs_32x32[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + { 17, 38, 140 }, { 7, 34, 80 }, { 1, 17, 29 } + }, { // Band 1 + { 37, 75, 128 }, { 41, 76, 128 }, { 26, 66, 116 }, + { 12, 52, 94 }, { 2, 32, 55 }, { 1, 10, 16 } + }, { // Band 2 + { 50, 127, 154 }, { 37, 109, 152 }, { 16, 82, 121 }, + { 5, 59, 85 }, { 1, 35, 54 }, { 1, 13, 20 } + }, { // Band 3 + { 40, 142, 167 }, { 17, 110, 157 }, { 2, 71, 112 }, + { 1, 44, 72 }, { 1, 27, 45 }, { 1, 11, 17 } + }, { // Band 4 + { 30, 175, 188 }, { 9, 124, 169 }, { 1, 74, 116 }, + { 1, 48, 78 }, { 1, 30, 49 }, { 1, 11, 18 } + }, { // Band 5 + { 10, 222, 223 }, { 2, 150, 194 }, { 1, 83, 128 }, + { 1, 48, 79 }, { 1, 27, 45 }, { 1, 11, 17 } + } + }, { // Inter + { // Band 0 + { 36, 41, 235 }, { 29, 36, 193 }, { 10, 27, 111 } + }, { // Band 1 + { 85, 165, 222 }, { 177, 162, 215 }, { 110, 135, 195 }, + { 57, 113, 168 }, { 23, 83, 120 }, { 10, 49, 61 } + }, { // Band 2 + { 85, 190, 223 }, { 36, 139, 200 }, { 5, 90, 146 }, + { 1, 60, 103 }, { 1, 38, 65 }, { 1, 18, 30 } + }, { // Band 3 + { 72, 202, 223 }, { 23, 141, 199 }, { 2, 86, 140 }, + { 1, 56, 97 }, { 1, 36, 61 }, { 1, 16, 27 } + }, { // Band 4 + { 55, 218, 225 }, { 13, 145, 200 }, { 1, 86, 141 }, + { 1, 57, 99 }, { 1, 35, 61 }, { 1, 13, 22 } + }, { // Band 5 + { 15, 235, 212 }, { 1, 132, 184 }, { 1, 84, 139 }, + { 1, 57, 97 }, { 1, 34, 56 }, { 1, 14, 23 } + } + } + }, { // UV plane + { // Intra + { // Band 0 + { 181, 21, 201 }, { 61, 37, 123 }, { 10, 38, 71 } + }, { // Band 1 + { 47, 106, 172 }, { 95, 104, 173 }, { 42, 93, 159 }, + { 18, 77, 131 }, { 4, 50, 81 }, { 1, 17, 23 } + }, { // Band 2 + { 62, 147, 199 }, { 44, 130, 189 }, { 28, 102, 154 }, + { 18, 75, 115 }, { 2, 44, 65 }, { 1, 12, 19 } + }, { // Band 3 + { 55, 153, 210 }, { 24, 130, 194 }, { 3, 93, 146 }, + { 1, 61, 97 }, { 1, 31, 50 }, { 1, 10, 16 } + }, { // Band 4 + { 49, 186, 223 }, { 17, 148, 204 }, { 1, 96, 142 }, + { 1, 53, 83 }, { 1, 26, 44 }, { 1, 11, 17 } + }, { // Band 5 + { 13, 217, 212 }, { 2, 136, 180 }, { 1, 78, 124 }, + { 1, 50, 83 }, { 1, 29, 49 }, { 1, 14, 23 } + } + }, { // Inter + { // Band 0 + { 197, 13, 247 }, { 82, 17, 222 }, { 25, 17, 162 } + }, { // Band 1 + { 126, 186, 247 }, { 234, 191, 243 }, { 176, 177, 234 }, + { 104, 158, 220 }, { 66, 128, 186 }, { 55, 90, 137 } + }, { // Band 2 + { 111, 197, 242 }, { 46, 158, 219 }, { 9, 104, 171 }, + { 2, 65, 125 }, { 1, 44, 80 }, { 1, 17, 91 } + }, { // Band 3 + { 104, 208, 245 }, { 39, 168, 224 }, { 3, 109, 162 }, + { 1, 79, 124 }, { 1, 50, 102 }, { 1, 43, 102 } + }, { // Band 4 + { 84, 220, 246 }, { 31, 177, 231 }, { 2, 115, 180 }, + { 1, 79, 134 }, { 1, 55, 77 }, { 1, 60, 79 } + }, { // Band 5 + { 43, 243, 240 }, { 8, 180, 217 }, { 1, 115, 166 }, + { 1, 84, 121 }, { 1, 51, 67 }, { 1, 16, 6 } + } + } + } +}; +#endif // CONFIG_NEW_TOKENSET + +#if CONFIG_TX64X64 +// FIXME. Optimize for EC_MULTISYMBOL +static const av1_coeff_probs_model default_coef_probs_64x64[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + { 17, 38, 140 }, { 7, 34, 80 }, { 1, 17, 29 } + }, { // Band 1 + { 37, 75, 128 }, { 41, 76, 128 }, { 26, 66, 116 }, + { 12, 52, 94 }, { 2, 32, 55 }, { 1, 10, 16 } + }, { // Band 2 + { 50, 127, 154 }, { 37, 109, 152 }, { 16, 82, 121 }, + { 5, 59, 85 }, { 1, 35, 54 }, { 1, 13, 20 } + }, { // Band 3 + { 40, 142, 167 }, { 17, 110, 157 }, { 2, 71, 112 }, + { 1, 44, 72 }, { 1, 27, 45 }, { 1, 11, 17 } + }, { // Band 4 + { 30, 175, 188 }, { 9, 124, 169 }, { 1, 74, 116 }, + { 1, 48, 78 }, { 1, 30, 49 }, { 1, 11, 18 } + }, { // Band 5 + { 10, 222, 223 }, { 2, 150, 194 }, { 1, 83, 128 }, + { 1, 48, 79 }, { 1, 27, 45 }, { 1, 11, 17 } + } + }, { // Inter + { // Band 0 + { 36, 41, 235 }, { 29, 36, 193 }, { 10, 27, 111 } + }, { // Band 1 + { 85, 165, 222 }, { 177, 162, 215 }, { 110, 135, 195 }, + { 57, 113, 168 }, { 23, 83, 120 }, { 10, 49, 61 } + }, { // Band 2 + { 85, 190, 223 }, { 36, 139, 200 }, { 5, 90, 146 }, + { 1, 60, 103 }, { 1, 38, 65 }, { 1, 18, 30 } + }, { // Band 3 + { 72, 202, 223 }, { 23, 141, 199 }, { 2, 86, 140 }, + { 1, 56, 97 }, { 1, 36, 61 }, { 1, 16, 27 } + }, { // Band 4 + { 55, 218, 225 }, { 13, 145, 200 }, { 1, 86, 141 }, + { 1, 57, 99 }, { 1, 35, 61 }, { 1, 13, 22 } + }, { // Band 5 + { 15, 235, 212 }, { 1, 132, 184 }, { 1, 84, 139 }, + { 1, 57, 97 }, { 1, 34, 56 }, { 1, 14, 23 } + } + } + }, { // UV plane + { // Intra + { // Band 0 + { 181, 21, 201 }, { 61, 37, 123 }, { 10, 38, 71 } + }, { // Band 1 + { 47, 106, 172 }, { 95, 104, 173 }, { 42, 93, 159 }, + { 18, 77, 131 }, { 4, 50, 81 }, { 1, 17, 23 } + }, { // Band 2 + { 62, 147, 199 }, { 44, 130, 189 }, { 28, 102, 154 }, + { 18, 75, 115 }, { 2, 44, 65 }, { 1, 12, 19 } + }, { // Band 3 + { 55, 153, 210 }, { 24, 130, 194 }, { 3, 93, 146 }, + { 1, 61, 97 }, { 1, 31, 50 }, { 1, 10, 16 } + }, { // Band 4 + { 49, 186, 223 }, { 17, 148, 204 }, { 1, 96, 142 }, + { 1, 53, 83 }, { 1, 26, 44 }, { 1, 11, 17 } + }, { // Band 5 + { 13, 217, 212 }, { 2, 136, 180 }, { 1, 78, 124 }, + { 1, 50, 83 }, { 1, 29, 49 }, { 1, 14, 23 } + } + }, { // Inter + { // Band 0 + { 197, 13, 247 }, { 82, 17, 222 }, { 25, 17, 162 } + }, { // Band 1 + { 126, 186, 247 }, { 234, 191, 243 }, { 176, 177, 234 }, + { 104, 158, 220 }, { 66, 128, 186 }, { 55, 90, 137 } + }, { // Band 2 + { 111, 197, 242 }, { 46, 158, 219 }, { 9, 104, 171 }, + { 2, 65, 125 }, { 1, 44, 80 }, { 1, 17, 91 } + }, { // Band 3 + { 104, 208, 245 }, { 39, 168, 224 }, { 3, 109, 162 }, + { 1, 79, 124 }, { 1, 50, 102 }, { 1, 43, 102 } + }, { // Band 4 + { 84, 220, 246 }, { 31, 177, 231 }, { 2, 115, 180 }, + { 1, 79, 134 }, { 1, 55, 77 }, { 1, 60, 79 } + }, { // Band 5 + { 43, 243, 240 }, { 8, 180, 217 }, { 1, 115, 166 }, + { 1, 84, 121 }, { 1, 51, 67 }, { 1, 16, 6 } + } + } + } +}; +#endif // CONFIG_TX64X64 +#endif // CONFIG_Q_ADAPT_PROBS +#if CONFIG_NEW_TOKENSET +static const aom_prob av1_default_blockzero_probs[TX_SIZES][PLANE_TYPES] + [REF_TYPES][BLOCKZ_CONTEXTS] = { + { // TX_4x4 + { // Y plane + { 195, 84, 8, }, // Intra + { 191, 124, 25, }, // Inter + }, + { // UV plane + { 214, 132, 42, }, // Intra + { 229, 143, 46, }, // Inter + }, + }, + { // TX_8x8 + { // Y plane + { 125, 52, 6, }, // Intra + { 202, 108, 18, }, // Inter + }, + { // UV plane + { 212, 113, 29, }, // Intra + { 225, 144, 42, }, // Inter + }, + }, + { // TX_16x16 + { // Y plane + { 7, 5, 1, }, // Intra + { 19, 19, 3, }, // Inter + }, + { // UV plane + { 211, 96, 22, }, // Intra + { 233, 146, 43, }, // Inter + }, + }, + { // TX_32x32 + { // Y plane + { 17, 7, 1, }, // Intra + { 36, 29, 10, }, // Inter + }, + { // UV plane + { 181, 61, 10, }, // Intra + { 197, 82, 25, }, // Inter + }, + }, +#if CONFIG_TX64X64 + { // TX_64x64 FIXME: currently the same as 32x32 + { // Y plane + { 17, 7, 1, }, // Intra + { 36, 29, 10, }, // Inter + }, + { // UV plane + { 181, 61, 10, }, // Intra + { 197, 82, 25, }, // Inter + }, + }, +#endif +}; + +static const coeff_cdf_model default_coef_head_cdf_4x4[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + { AOM_ICDF(25024), AOM_ICDF(25863), AOM_ICDF(27361), AOM_ICDF(29796), + AOM_ICDF(30374), AOM_ICDF(32768) }, + { AOM_ICDF(10816), AOM_ICDF(14127), AOM_ICDF(17116), AOM_ICDF(23516), + AOM_ICDF(24999), AOM_ICDF(32768) }, + { AOM_ICDF(1088), AOM_ICDF(6358), AOM_ICDF(8428), AOM_ICDF(16648), + AOM_ICDF(18276), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(14529), AOM_ICDF(18769), AOM_ICDF(29100), AOM_ICDF(29634), + AOM_ICDF(32768) }, + {AOM_ICDF(12993), AOM_ICDF(17117), AOM_ICDF(28404), AOM_ICDF(28988), + AOM_ICDF(32768) }, + {AOM_ICDF(11201), AOM_ICDF(14084), AOM_ICDF(25818), AOM_ICDF(26504), + AOM_ICDF(32768) }, + {AOM_ICDF(9793), AOM_ICDF(11267), AOM_ICDF(21775), AOM_ICDF(22451), + AOM_ICDF(32768) }, + {AOM_ICDF(7105), AOM_ICDF(7562), AOM_ICDF(15777), AOM_ICDF(16225), + AOM_ICDF(32768) }, + {AOM_ICDF(3905), AOM_ICDF(3966), AOM_ICDF(8359), AOM_ICDF(8526), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(20033), AOM_ICDF(23643), AOM_ICDF(31102), AOM_ICDF(31374), + AOM_ICDF(32768) }, + {AOM_ICDF(16321), AOM_ICDF(20350), AOM_ICDF(30167), AOM_ICDF(30546), + AOM_ICDF(32768) }, + {AOM_ICDF(12993), AOM_ICDF(15512), AOM_ICDF(26859), AOM_ICDF(27396), + AOM_ICDF(32768) }, + {AOM_ICDF(10305), AOM_ICDF(11659), AOM_ICDF(21669), AOM_ICDF(22330), + AOM_ICDF(32768) }, + {AOM_ICDF(7361), AOM_ICDF(7819), AOM_ICDF(15450), AOM_ICDF(15940), + AOM_ICDF(32768) }, + {AOM_ICDF(3521), AOM_ICDF(3580), AOM_ICDF(7805), AOM_ICDF(7976), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(21057), AOM_ICDF(25460), AOM_ICDF(31740), AOM_ICDF(31952), + AOM_ICDF(32768) }, + {AOM_ICDF(16449), AOM_ICDF(21173), AOM_ICDF(30761), AOM_ICDF(31092), + AOM_ICDF(32768) }, + {AOM_ICDF(11841), AOM_ICDF(14615), AOM_ICDF(26188), AOM_ICDF(26824), + AOM_ICDF(32768) }, + {AOM_ICDF(7745), AOM_ICDF(8991), AOM_ICDF(18937), AOM_ICDF(19707), + AOM_ICDF(32768) }, + {AOM_ICDF(4417), AOM_ICDF(4706), AOM_ICDF(10342), AOM_ICDF(10890), + AOM_ICDF(32768) }, + {AOM_ICDF(7617), AOM_ICDF(8392), AOM_ICDF(17295), AOM_ICDF(17915), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(20417), AOM_ICDF(26452), AOM_ICDF(32166), AOM_ICDF(32321), + AOM_ICDF(32768) }, + {AOM_ICDF(15809), AOM_ICDF(21634), AOM_ICDF(30947), AOM_ICDF(31298), + AOM_ICDF(32768) }, + {AOM_ICDF(10049), AOM_ICDF(12176), AOM_ICDF(23495), AOM_ICDF(24229), + AOM_ICDF(32768) }, + {AOM_ICDF(5953), AOM_ICDF(6731), AOM_ICDF(16166), AOM_ICDF(16798), + AOM_ICDF(32768) }, + {AOM_ICDF(6081), AOM_ICDF(6188), AOM_ICDF(8114), AOM_ICDF(8764), + AOM_ICDF(32768) }, + {AOM_ICDF(2113), AOM_ICDF(2291), AOM_ICDF(4448), AOM_ICDF(5527), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(9153), AOM_ICDF(25905), AOM_ICDF(31431), AOM_ICDF(31934), + AOM_ICDF(32768) }, + {AOM_ICDF(9025), AOM_ICDF(23345), AOM_ICDF(30033), AOM_ICDF(30965), + AOM_ICDF(32768) }, + {AOM_ICDF(5953), AOM_ICDF(13835), AOM_ICDF(22032), AOM_ICDF(24664), + AOM_ICDF(32768) }, + {AOM_ICDF(6337), AOM_ICDF(11435), AOM_ICDF(18366), AOM_ICDF(21418), + AOM_ICDF(32768) }, + {AOM_ICDF(3137), AOM_ICDF(4871), AOM_ICDF(8519), AOM_ICDF(12426), + AOM_ICDF(32768) }, + {AOM_ICDF(1857), AOM_ICDF(2727), AOM_ICDF(5540), AOM_ICDF(8757), + AOM_ICDF(32768) } } }, + { // Intra + { // Band 0 + { AOM_ICDF(24512), AOM_ICDF(26673), AOM_ICDF(28962), AOM_ICDF(31929), + AOM_ICDF(32126), AOM_ICDF(32768) }, + { AOM_ICDF(15936), AOM_ICDF(21711), AOM_ICDF(25569), AOM_ICDF(30899), + AOM_ICDF(31305), AOM_ICDF(32768) }, + { AOM_ICDF(3264), AOM_ICDF(14756), AOM_ICDF(20107), AOM_ICDF(29407), + AOM_ICDF(30032), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(21313), AOM_ICDF(26020), AOM_ICDF(32523), AOM_ICDF(32575), + AOM_ICDF(32768) }, + {AOM_ICDF(18369), AOM_ICDF(24215), AOM_ICDF(32291), AOM_ICDF(32391), + AOM_ICDF(32768) }, + {AOM_ICDF(15297), AOM_ICDF(19637), AOM_ICDF(30414), AOM_ICDF(30752), + AOM_ICDF(32768) }, + {AOM_ICDF(11713), AOM_ICDF(14040), AOM_ICDF(25408), AOM_ICDF(26033), + AOM_ICDF(32768) }, + {AOM_ICDF(9537), AOM_ICDF(10173), AOM_ICDF(18839), AOM_ICDF(19315), + AOM_ICDF(32768) }, + {AOM_ICDF(9025), AOM_ICDF(9093), AOM_ICDF(13987), AOM_ICDF(14115), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(22721), AOM_ICDF(27599), AOM_ICDF(32592), AOM_ICDF(32636), + AOM_ICDF(32768) }, + {AOM_ICDF(19009), AOM_ICDF(24676), AOM_ICDF(32258), AOM_ICDF(32367), + AOM_ICDF(32768) }, + {AOM_ICDF(12737), AOM_ICDF(16769), AOM_ICDF(28739), AOM_ICDF(29247), + AOM_ICDF(32768) }, + {AOM_ICDF(8769), AOM_ICDF(10956), AOM_ICDF(21941), AOM_ICDF(22840), + AOM_ICDF(32768) }, + {AOM_ICDF(6721), AOM_ICDF(7678), AOM_ICDF(15319), AOM_ICDF(16290), + AOM_ICDF(32768) }, + {AOM_ICDF(4417), AOM_ICDF(4430), AOM_ICDF(4583), AOM_ICDF(5712), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(22849), AOM_ICDF(28333), AOM_ICDF(32633), AOM_ICDF(32671), + AOM_ICDF(32768) }, + {AOM_ICDF(18497), AOM_ICDF(24619), AOM_ICDF(32184), AOM_ICDF(32315), + AOM_ICDF(32768) }, + {AOM_ICDF(11841), AOM_ICDF(14640), AOM_ICDF(27251), AOM_ICDF(27752), + AOM_ICDF(32768) }, + {AOM_ICDF(8385), AOM_ICDF(10154), AOM_ICDF(18339), AOM_ICDF(19621), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(6977), AOM_ICDF(13787), AOM_ICDF(15289), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(194), AOM_ICDF(384), AOM_ICDF(479), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(20417), AOM_ICDF(28167), AOM_ICDF(32552), AOM_ICDF(32621), + AOM_ICDF(32768) }, + {AOM_ICDF(16833), AOM_ICDF(23968), AOM_ICDF(31991), AOM_ICDF(32174), + AOM_ICDF(32768) }, + {AOM_ICDF(10433), AOM_ICDF(13387), AOM_ICDF(26356), AOM_ICDF(26951), + AOM_ICDF(32768) }, + {AOM_ICDF(5057), AOM_ICDF(6823), AOM_ICDF(18967), AOM_ICDF(19843), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(6479), AOM_ICDF(11672), AOM_ICDF(13052), + AOM_ICDF(32768) }, + {AOM_ICDF(2241), AOM_ICDF(2265), AOM_ICDF(6355), AOM_ICDF(6432), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(12097), AOM_ICDF(28717), AOM_ICDF(32406), AOM_ICDF(32555), + AOM_ICDF(32768) }, + {AOM_ICDF(10433), AOM_ICDF(26113), AOM_ICDF(31504), AOM_ICDF(31975), + AOM_ICDF(32768) }, + {AOM_ICDF(5825), AOM_ICDF(14284), AOM_ICDF(21349), AOM_ICDF(24461), + AOM_ICDF(32768) }, + {AOM_ICDF(4545), AOM_ICDF(8454), AOM_ICDF(12648), AOM_ICDF(17501), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(7173), AOM_ICDF(15272), AOM_ICDF(19322), + AOM_ICDF(32768) }, + {AOM_ICDF(2113), AOM_ICDF(2183), AOM_ICDF(7202), AOM_ICDF(7377), + AOM_ICDF(32768) } } } }, + { // UV plane + { // Inter + { // Band 0 + { AOM_ICDF(27456), AOM_ICDF(28244), AOM_ICDF(31289), AOM_ICDF(32358), + AOM_ICDF(32534), AOM_ICDF(32768) }, + { AOM_ICDF(16960), AOM_ICDF(21207), AOM_ICDF(26511), AOM_ICDF(30539), + AOM_ICDF(31190), AOM_ICDF(32768) }, + { AOM_ICDF(5440), AOM_ICDF(13412), AOM_ICDF(18469), AOM_ICDF(26423), + AOM_ICDF(27669), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(17857), AOM_ICDF(26327), AOM_ICDF(31983), AOM_ICDF(32219), + AOM_ICDF(32768) }, + {AOM_ICDF(16065), AOM_ICDF(24198), AOM_ICDF(31431), AOM_ICDF(31785), + AOM_ICDF(32768) }, + {AOM_ICDF(12865), AOM_ICDF(18011), AOM_ICDF(28454), AOM_ICDF(29166), + AOM_ICDF(32768) }, + {AOM_ICDF(9665), AOM_ICDF(12501), AOM_ICDF(24331), AOM_ICDF(25147), + AOM_ICDF(32768) }, + {AOM_ICDF(2753), AOM_ICDF(3121), AOM_ICDF(12661), AOM_ICDF(13034), + AOM_ICDF(32768) }, + {AOM_ICDF(4033), AOM_ICDF(4140), AOM_ICDF(11834), AOM_ICDF(11977), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(21185), AOM_ICDF(28338), AOM_ICDF(32249), AOM_ICDF(32417), + AOM_ICDF(32768) }, + {AOM_ICDF(18497), AOM_ICDF(25227), AOM_ICDF(31905), AOM_ICDF(32122), + AOM_ICDF(32768) }, + {AOM_ICDF(12097), AOM_ICDF(16516), AOM_ICDF(28610), AOM_ICDF(29166), + AOM_ICDF(32768) }, + {AOM_ICDF(9281), AOM_ICDF(11157), AOM_ICDF(21438), AOM_ICDF(22312), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(6566), AOM_ICDF(15585), AOM_ICDF(16340), + AOM_ICDF(32768) }, + {AOM_ICDF(9409), AOM_ICDF(9659), AOM_ICDF(11827), AOM_ICDF(12911), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(22337), AOM_ICDF(29459), AOM_ICDF(32382), AOM_ICDF(32519), + AOM_ICDF(32768) }, + {AOM_ICDF(16961), AOM_ICDF(25262), AOM_ICDF(31874), AOM_ICDF(32123), + AOM_ICDF(32768) }, + {AOM_ICDF(12353), AOM_ICDF(17748), AOM_ICDF(29300), AOM_ICDF(29852), + AOM_ICDF(32768) }, + {AOM_ICDF(9025), AOM_ICDF(11528), AOM_ICDF(24468), AOM_ICDF(25141), + AOM_ICDF(32768) }, + {AOM_ICDF(6209), AOM_ICDF(6565), AOM_ICDF(15806), AOM_ICDF(16121), + AOM_ICDF(32768) }, + {AOM_ICDF(2497), AOM_ICDF(2524), AOM_ICDF(7050), AOM_ICDF(7125), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(20417), AOM_ICDF(29779), AOM_ICDF(32552), AOM_ICDF(32636), + AOM_ICDF(32768) }, + {AOM_ICDF(15553), AOM_ICDF(26420), AOM_ICDF(32063), AOM_ICDF(32295), + AOM_ICDF(32768) }, + {AOM_ICDF(9665), AOM_ICDF(17946), AOM_ICDF(29385), AOM_ICDF(30096), + AOM_ICDF(32768) }, + {AOM_ICDF(5569), AOM_ICDF(10207), AOM_ICDF(22410), AOM_ICDF(23836), + AOM_ICDF(32768) }, + {AOM_ICDF(16449), AOM_ICDF(16450), AOM_ICDF(16545), AOM_ICDF(16593), + AOM_ICDF(32768) }, + {AOM_ICDF(2369), AOM_ICDF(2395), AOM_ICDF(6822), AOM_ICDF(6898), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(10177), AOM_ICDF(30567), AOM_ICDF(32725), AOM_ICDF(32745), + AOM_ICDF(32768) }, + {AOM_ICDF(9537), AOM_ICDF(28243), AOM_ICDF(32179), AOM_ICDF(32423), + AOM_ICDF(32768) }, + {AOM_ICDF(13377), AOM_ICDF(23187), AOM_ICDF(29322), AOM_ICDF(30382), + AOM_ICDF(32768) }, + {AOM_ICDF(13121), AOM_ICDF(21346), AOM_ICDF(29507), AOM_ICDF(30326), + AOM_ICDF(32768) }, + {AOM_ICDF(4417), AOM_ICDF(4939), AOM_ICDF(15104), AOM_ICDF(15535), + AOM_ICDF(32768) }, + {AOM_ICDF(2625), AOM_ICDF(2680), AOM_ICDF(8218), AOM_ICDF(8338), + AOM_ICDF(32768) } } }, + { // Inter + { // Band 0 + { AOM_ICDF(29376), AOM_ICDF(30098), AOM_ICDF(32421), AOM_ICDF(32766), + AOM_ICDF(32767), AOM_ICDF(32768) }, + { AOM_ICDF(18368), AOM_ICDF(22916), AOM_ICDF(30116), AOM_ICDF(32541), + AOM_ICDF(32650), AOM_ICDF(32768) }, + { AOM_ICDF(5952), AOM_ICDF(16505), AOM_ICDF(25955), AOM_ICDF(32163), + AOM_ICDF(32365), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(19649), AOM_ICDF(30160), AOM_ICDF(32743), AOM_ICDF(32753), + AOM_ICDF(32768) }, + {AOM_ICDF(18881), AOM_ICDF(28724), AOM_ICDF(32688), AOM_ICDF(32717), + AOM_ICDF(32768) }, + {AOM_ICDF(16833), AOM_ICDF(23053), AOM_ICDF(31244), AOM_ICDF(31573), + AOM_ICDF(32768) }, + {AOM_ICDF(14657), AOM_ICDF(17714), AOM_ICDF(26083), AOM_ICDF(26978), + AOM_ICDF(32768) }, + {AOM_ICDF(14657), AOM_ICDF(16618), AOM_ICDF(24597), AOM_ICDF(25403), + AOM_ICDF(32768) }, + {AOM_ICDF(4289), AOM_ICDF(4326), AOM_ICDF(10686), AOM_ICDF(10751), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(21953), AOM_ICDF(30956), AOM_ICDF(32748), AOM_ICDF(32757), + AOM_ICDF(32768) }, + {AOM_ICDF(20929), AOM_ICDF(29412), AOM_ICDF(32700), AOM_ICDF(32725), + AOM_ICDF(32768) }, + {AOM_ICDF(13377), AOM_ICDF(21495), AOM_ICDF(31216), AOM_ICDF(31569), + AOM_ICDF(32768) }, + {AOM_ICDF(9153), AOM_ICDF(15097), AOM_ICDF(28295), AOM_ICDF(28990), + AOM_ICDF(32768) }, + {AOM_ICDF(5313), AOM_ICDF(5363), AOM_ICDF(13839), AOM_ICDF(13894), + AOM_ICDF(32768) }, + {AOM_ICDF(2625), AOM_ICDF(2652), AOM_ICDF(7276), AOM_ICDF(7351), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(20289), AOM_ICDF(31164), AOM_ICDF(32745), AOM_ICDF(32755), + AOM_ICDF(32768) }, + {AOM_ICDF(17601), AOM_ICDF(29635), AOM_ICDF(32739), AOM_ICDF(32751), + AOM_ICDF(32768) }, + {AOM_ICDF(18241), AOM_ICDF(24284), AOM_ICDF(32116), AOM_ICDF(32258), + AOM_ICDF(32768) }, + {AOM_ICDF(32705), AOM_ICDF(32706), AOM_ICDF(32739), AOM_ICDF(32740), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(5750), AOM_ICDF(14739), AOM_ICDF(14792), + AOM_ICDF(32768) }, + {AOM_ICDF(2881), AOM_ICDF(2913), AOM_ICDF(8427), AOM_ICDF(8498), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(19009), AOM_ICDF(31481), AOM_ICDF(32742), AOM_ICDF(32754), + AOM_ICDF(32768) }, + {AOM_ICDF(15809), AOM_ICDF(30521), AOM_ICDF(32736), AOM_ICDF(32750), + AOM_ICDF(32768) }, + {AOM_ICDF(16449), AOM_ICDF(32705), AOM_ICDF(32737), AOM_ICDF(32753), + AOM_ICDF(32768) }, + {AOM_ICDF(7873), AOM_ICDF(8039), AOM_ICDF(19981), AOM_ICDF(20068), + AOM_ICDF(32768) }, + {AOM_ICDF(5313), AOM_ICDF(5366), AOM_ICDF(14376), AOM_ICDF(14430), + AOM_ICDF(32768) }, + {AOM_ICDF(2753), AOM_ICDF(2789), AOM_ICDF(8909), AOM_ICDF(8979), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(11841), AOM_ICDF(32116), AOM_ICDF(32728), AOM_ICDF(32748), + AOM_ICDF(32768) }, + {AOM_ICDF(12353), AOM_ICDF(32132), AOM_ICDF(32729), AOM_ICDF(32748), + AOM_ICDF(32768) }, + {AOM_ICDF(7489), AOM_ICDF(12435), AOM_ICDF(25708), AOM_ICDF(26666), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(7486), AOM_ICDF(20238), AOM_ICDF(21009), + AOM_ICDF(32768) }, + {AOM_ICDF(4929), AOM_ICDF(5579), AOM_ICDF(16402), AOM_ICDF(16866), + AOM_ICDF(32768) }, + {AOM_ICDF(3009), AOM_ICDF(3246), AOM_ICDF(10158), AOM_ICDF(10533), + AOM_ICDF(32768) } } } } +}; +static const coeff_cdf_model default_coef_head_cdf_8x8[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + { AOM_ICDF(16064), AOM_ICDF(18127), AOM_ICDF(22153), AOM_ICDF(27289), + AOM_ICDF(28507), AOM_ICDF(32768) }, + { AOM_ICDF(6720), AOM_ICDF(10545), AOM_ICDF(13491), AOM_ICDF(20948), + AOM_ICDF(22631), AOM_ICDF(32768) }, + { AOM_ICDF(832), AOM_ICDF(5270), AOM_ICDF(5918), AOM_ICDF(12645), + AOM_ICDF(13532), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(14017), AOM_ICDF(16139), AOM_ICDF(26799), AOM_ICDF(27295), + AOM_ICDF(32768) }, + {AOM_ICDF(12737), AOM_ICDF(15136), AOM_ICDF(26235), AOM_ICDF(26816), + AOM_ICDF(32768) }, + {AOM_ICDF(10817), AOM_ICDF(12445), AOM_ICDF(23637), AOM_ICDF(24217), + AOM_ICDF(32768) }, + {AOM_ICDF(8897), AOM_ICDF(9702), AOM_ICDF(20040), AOM_ICDF(20500), + AOM_ICDF(32768) }, + {AOM_ICDF(5953), AOM_ICDF(6156), AOM_ICDF(13966), AOM_ICDF(14205), + AOM_ICDF(32768) }, + {AOM_ICDF(2497), AOM_ICDF(2519), AOM_ICDF(6222), AOM_ICDF(6300), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(19777), AOM_ICDF(21403), AOM_ICDF(30054), AOM_ICDF(30269), + AOM_ICDF(32768) }, + {AOM_ICDF(16193), AOM_ICDF(17913), AOM_ICDF(28593), AOM_ICDF(28883), + AOM_ICDF(32768) }, + {AOM_ICDF(12609), AOM_ICDF(13572), AOM_ICDF(25248), AOM_ICDF(25534), + AOM_ICDF(32768) }, + {AOM_ICDF(9665), AOM_ICDF(10118), AOM_ICDF(20721), AOM_ICDF(20968), + AOM_ICDF(32768) }, + {AOM_ICDF(6849), AOM_ICDF(7028), AOM_ICDF(15202), AOM_ICDF(15391), + AOM_ICDF(32768) }, + {AOM_ICDF(3009), AOM_ICDF(3036), AOM_ICDF(7601), AOM_ICDF(7675), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(22593), AOM_ICDF(23915), AOM_ICDF(31159), AOM_ICDF(31283), + AOM_ICDF(32768) }, + {AOM_ICDF(17345), AOM_ICDF(18690), AOM_ICDF(29425), AOM_ICDF(29611), + AOM_ICDF(32768) }, + {AOM_ICDF(11969), AOM_ICDF(12540), AOM_ICDF(24685), AOM_ICDF(24867), + AOM_ICDF(32768) }, + {AOM_ICDF(8129), AOM_ICDF(8355), AOM_ICDF(18668), AOM_ICDF(18819), + AOM_ICDF(32768) }, + {AOM_ICDF(4673), AOM_ICDF(4714), AOM_ICDF(11752), AOM_ICDF(11814), + AOM_ICDF(32768) }, + {AOM_ICDF(1857), AOM_ICDF(1876), AOM_ICDF(5057), AOM_ICDF(5138), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(24513), AOM_ICDF(25718), AOM_ICDF(31947), AOM_ICDF(32014), + AOM_ICDF(32768) }, + {AOM_ICDF(18881), AOM_ICDF(20029), AOM_ICDF(30409), AOM_ICDF(30527), + AOM_ICDF(32768) }, + {AOM_ICDF(12481), AOM_ICDF(12953), AOM_ICDF(25201), AOM_ICDF(25341), + AOM_ICDF(32768) }, + {AOM_ICDF(8385), AOM_ICDF(8528), AOM_ICDF(18815), AOM_ICDF(18910), + AOM_ICDF(32768) }, + {AOM_ICDF(4289), AOM_ICDF(4327), AOM_ICDF(10797), AOM_ICDF(10861), + AOM_ICDF(32768) }, + {AOM_ICDF(1857), AOM_ICDF(1872), AOM_ICDF(4332), AOM_ICDF(4415), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(26049), AOM_ICDF(27752), AOM_ICDF(32415), AOM_ICDF(32462), + AOM_ICDF(32768) }, + {AOM_ICDF(20417), AOM_ICDF(22100), AOM_ICDF(31056), AOM_ICDF(31192), + AOM_ICDF(32768) }, + {AOM_ICDF(12481), AOM_ICDF(13075), AOM_ICDF(24646), AOM_ICDF(24844), + AOM_ICDF(32768) }, + {AOM_ICDF(7489), AOM_ICDF(7696), AOM_ICDF(17117), AOM_ICDF(17285), + AOM_ICDF(32768) }, + {AOM_ICDF(3777), AOM_ICDF(3814), AOM_ICDF(10062), AOM_ICDF(10129), + AOM_ICDF(32768) }, + {AOM_ICDF(1473), AOM_ICDF(1486), AOM_ICDF(3735), AOM_ICDF(3820), + AOM_ICDF(32768) } } }, + { // Intra + { // Band 0 + { AOM_ICDF(25920), AOM_ICDF(27743), AOM_ICDF(29455), AOM_ICDF(32147), + AOM_ICDF(32280), AOM_ICDF(32768) }, + { AOM_ICDF(13888), AOM_ICDF(19845), AOM_ICDF(23350), AOM_ICDF(30219), + AOM_ICDF(30660), AOM_ICDF(32768) }, + { AOM_ICDF(2368), AOM_ICDF(12781), AOM_ICDF(16196), AOM_ICDF(27232), + AOM_ICDF(27894), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(21697), AOM_ICDF(24758), AOM_ICDF(32358), AOM_ICDF(32417), + AOM_ICDF(32768) }, + {AOM_ICDF(20289), AOM_ICDF(23960), AOM_ICDF(32111), AOM_ICDF(32213), + AOM_ICDF(32768) }, + {AOM_ICDF(17345), AOM_ICDF(19966), AOM_ICDF(30630), AOM_ICDF(30841), + AOM_ICDF(32768) }, + {AOM_ICDF(14529), AOM_ICDF(16070), AOM_ICDF(27461), AOM_ICDF(27777), + AOM_ICDF(32768) }, + {AOM_ICDF(9793), AOM_ICDF(10613), AOM_ICDF(21146), AOM_ICDF(21566), + AOM_ICDF(32768) }, + {AOM_ICDF(6977), AOM_ICDF(7162), AOM_ICDF(15591), AOM_ICDF(15776), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(23617), AOM_ICDF(26783), AOM_ICDF(32572), AOM_ICDF(32607), + AOM_ICDF(32768) }, + {AOM_ICDF(20801), AOM_ICDF(24292), AOM_ICDF(32185), AOM_ICDF(32275), + AOM_ICDF(32768) }, + {AOM_ICDF(15169), AOM_ICDF(17905), AOM_ICDF(29916), AOM_ICDF(30181), + AOM_ICDF(32768) }, + {AOM_ICDF(10945), AOM_ICDF(12972), AOM_ICDF(25565), AOM_ICDF(26064), + AOM_ICDF(32768) }, + {AOM_ICDF(6849), AOM_ICDF(8334), AOM_ICDF(18543), AOM_ICDF(19446), + AOM_ICDF(32768) }, + {AOM_ICDF(3649), AOM_ICDF(4346), AOM_ICDF(12351), AOM_ICDF(13169), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(25281), AOM_ICDF(28440), AOM_ICDF(32667), AOM_ICDF(32689), + AOM_ICDF(32768) }, + {AOM_ICDF(22081), AOM_ICDF(25694), AOM_ICDF(32414), AOM_ICDF(32476), + AOM_ICDF(32768) }, + {AOM_ICDF(15297), AOM_ICDF(18341), AOM_ICDF(30141), AOM_ICDF(30410), + AOM_ICDF(32768) }, + {AOM_ICDF(10305), AOM_ICDF(12381), AOM_ICDF(24477), AOM_ICDF(25084), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(6673), AOM_ICDF(16325), AOM_ICDF(17080), + AOM_ICDF(32768) }, + {AOM_ICDF(2369), AOM_ICDF(2393), AOM_ICDF(6466), AOM_ICDF(6543), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(25921), AOM_ICDF(29445), AOM_ICDF(32729), AOM_ICDF(32739), + AOM_ICDF(32768) }, + {AOM_ICDF(22465), AOM_ICDF(26834), AOM_ICDF(32588), AOM_ICDF(32627), + AOM_ICDF(32768) }, + {AOM_ICDF(16449), AOM_ICDF(20062), AOM_ICDF(31016), AOM_ICDF(31233), + AOM_ICDF(32768) }, + {AOM_ICDF(11073), AOM_ICDF(13165), AOM_ICDF(25353), AOM_ICDF(25896), + AOM_ICDF(32768) }, + {AOM_ICDF(11713), AOM_ICDF(13837), AOM_ICDF(20144), AOM_ICDF(21734), + AOM_ICDF(32768) }, + {AOM_ICDF(2241), AOM_ICDF(2265), AOM_ICDF(6355), AOM_ICDF(6432), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(26177), AOM_ICDF(29403), AOM_ICDF(32705), AOM_ICDF(32721), + AOM_ICDF(32768) }, + {AOM_ICDF(22337), AOM_ICDF(26344), AOM_ICDF(32545), AOM_ICDF(32589), + AOM_ICDF(32768) }, + {AOM_ICDF(19009), AOM_ICDF(21527), AOM_ICDF(31775), AOM_ICDF(31873), + AOM_ICDF(32768) }, + {AOM_ICDF(11585), AOM_ICDF(12685), AOM_ICDF(22632), AOM_ICDF(23137), + AOM_ICDF(32768) }, + {AOM_ICDF(8257), AOM_ICDF(8305), AOM_ICDF(16444), AOM_ICDF(16492), + AOM_ICDF(32768) }, + {AOM_ICDF(2113), AOM_ICDF(2183), AOM_ICDF(7202), AOM_ICDF(7377), + AOM_ICDF(32768) } } } }, + { // UV plane + { // Inter + { // Band 0 + { AOM_ICDF(27200), AOM_ICDF(27981), AOM_ICDF(31389), AOM_ICDF(32444), + AOM_ICDF(32592), AOM_ICDF(32768) }, + { AOM_ICDF(14528), AOM_ICDF(19068), AOM_ICDF(24887), AOM_ICDF(29901), + AOM_ICDF(30688), AOM_ICDF(32768) }, + { AOM_ICDF(3776), AOM_ICDF(11778), AOM_ICDF(14700), AOM_ICDF(23745), + AOM_ICDF(24854), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(20289), AOM_ICDF(25202), AOM_ICDF(31672), AOM_ICDF(31909), + AOM_ICDF(32768) }, + {AOM_ICDF(18369), AOM_ICDF(23493), AOM_ICDF(31166), AOM_ICDF(31487), + AOM_ICDF(32768) }, + {AOM_ICDF(15425), AOM_ICDF(18619), AOM_ICDF(28941), AOM_ICDF(29393), + AOM_ICDF(32768) }, + {AOM_ICDF(10945), AOM_ICDF(12535), AOM_ICDF(24287), AOM_ICDF(24792), + AOM_ICDF(32768) }, + {AOM_ICDF(6465), AOM_ICDF(6810), AOM_ICDF(15764), AOM_ICDF(16080), + AOM_ICDF(32768) }, + {AOM_ICDF(2113), AOM_ICDF(2137), AOM_ICDF(6125), AOM_ICDF(6203), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(23745), AOM_ICDF(27041), AOM_ICDF(31976), AOM_ICDF(32135), + AOM_ICDF(32768) }, + {AOM_ICDF(19521), AOM_ICDF(22766), AOM_ICDF(31139), AOM_ICDF(31367), + AOM_ICDF(32768) }, + {AOM_ICDF(14273), AOM_ICDF(15834), AOM_ICDF(27820), AOM_ICDF(28105), + AOM_ICDF(32768) }, + {AOM_ICDF(9537), AOM_ICDF(10445), AOM_ICDF(22106), AOM_ICDF(22491), + AOM_ICDF(32768) }, + {AOM_ICDF(7233), AOM_ICDF(7386), AOM_ICDF(15961), AOM_ICDF(16109), + AOM_ICDF(32768) }, + {AOM_ICDF(2369), AOM_ICDF(2401), AOM_ICDF(7891), AOM_ICDF(7964), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(26305), AOM_ICDF(28703), AOM_ICDF(32352), AOM_ICDF(32435), + AOM_ICDF(32768) }, + {AOM_ICDF(20673), AOM_ICDF(23490), AOM_ICDF(31517), AOM_ICDF(31680), + AOM_ICDF(32768) }, + {AOM_ICDF(14017), AOM_ICDF(15251), AOM_ICDF(27458), AOM_ICDF(27702), + AOM_ICDF(32768) }, + {AOM_ICDF(10945), AOM_ICDF(11374), AOM_ICDF(22496), AOM_ICDF(22687), + AOM_ICDF(32768) }, + {AOM_ICDF(9153), AOM_ICDF(9435), AOM_ICDF(22299), AOM_ICDF(22411), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(269), AOM_ICDF(13236), AOM_ICDF(13293), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(27713), AOM_ICDF(29770), AOM_ICDF(32522), AOM_ICDF(32575), + AOM_ICDF(32768) }, + {AOM_ICDF(21569), AOM_ICDF(24342), AOM_ICDF(31785), AOM_ICDF(31919), + AOM_ICDF(32768) }, + {AOM_ICDF(15297), AOM_ICDF(16497), AOM_ICDF(28367), AOM_ICDF(28569), + AOM_ICDF(32768) }, + {AOM_ICDF(17601), AOM_ICDF(17828), AOM_ICDF(24444), AOM_ICDF(24582), + AOM_ICDF(32768) }, + {AOM_ICDF(6977), AOM_ICDF(7035), AOM_ICDF(16901), AOM_ICDF(16947), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(384), AOM_ICDF(32706), AOM_ICDF(32707), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(28737), AOM_ICDF(30879), AOM_ICDF(32667), AOM_ICDF(32695), + AOM_ICDF(32768) }, + {AOM_ICDF(22593), AOM_ICDF(26241), AOM_ICDF(32073), AOM_ICDF(32207), + AOM_ICDF(32768) }, + {AOM_ICDF(16577), AOM_ICDF(19148), AOM_ICDF(28436), AOM_ICDF(28906), + AOM_ICDF(32768) }, + {AOM_ICDF(12993), AOM_ICDF(14005), AOM_ICDF(23151), AOM_ICDF(23630), + AOM_ICDF(32768) }, + {AOM_ICDF(7617), AOM_ICDF(9188), AOM_ICDF(22797), AOM_ICDF(23313), + AOM_ICDF(32768) }, + {AOM_ICDF(2625), AOM_ICDF(2680), AOM_ICDF(8218), AOM_ICDF(8338), + AOM_ICDF(32768) } } }, + { // Inter + { // Band 0 + { AOM_ICDF(28864), AOM_ICDF(29988), AOM_ICDF(32423), AOM_ICDF(32766), + AOM_ICDF(32767), AOM_ICDF(32768) }, + { AOM_ICDF(18496), AOM_ICDF(24572), AOM_ICDF(30167), AOM_ICDF(32687), + AOM_ICDF(32737), AOM_ICDF(32768) }, + { AOM_ICDF(5440), AOM_ICDF(19618), AOM_ICDF(25332), AOM_ICDF(32393), + AOM_ICDF(32491), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(23745), AOM_ICDF(29427), AOM_ICDF(32751), AOM_ICDF(32757), + AOM_ICDF(32768) }, + {AOM_ICDF(23745), AOM_ICDF(28704), AOM_ICDF(32716), AOM_ICDF(32731), + AOM_ICDF(32768) }, + {AOM_ICDF(23105), AOM_ICDF(27943), AOM_ICDF(32524), AOM_ICDF(32587), + AOM_ICDF(32768) }, + {AOM_ICDF(21057), AOM_ICDF(24773), AOM_ICDF(29589), AOM_ICDF(30282), + AOM_ICDF(32768) }, + {AOM_ICDF(12609), AOM_ICDF(14823), AOM_ICDF(23831), AOM_ICDF(24713), + AOM_ICDF(32768) }, + {AOM_ICDF(16449), AOM_ICDF(16450), AOM_ICDF(16545), AOM_ICDF(16593), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(25025), AOM_ICDF(30203), AOM_ICDF(32754), AOM_ICDF(32759), + AOM_ICDF(32768) }, + {AOM_ICDF(23617), AOM_ICDF(28361), AOM_ICDF(32715), AOM_ICDF(32729), + AOM_ICDF(32768) }, + {AOM_ICDF(17985), AOM_ICDF(21562), AOM_ICDF(31354), AOM_ICDF(31543), + AOM_ICDF(32768) }, + {AOM_ICDF(12353), AOM_ICDF(18915), AOM_ICDF(28742), AOM_ICDF(29548), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(289), AOM_ICDF(16545), AOM_ICDF(16593), + AOM_ICDF(32768) }, + {AOM_ICDF(2625), AOM_ICDF(2652), AOM_ICDF(7276), AOM_ICDF(7351), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(26433), AOM_ICDF(30892), AOM_ICDF(32757), AOM_ICDF(32761), + AOM_ICDF(32768) }, + {AOM_ICDF(24513), AOM_ICDF(29274), AOM_ICDF(32721), AOM_ICDF(32735), + AOM_ICDF(32768) }, + {AOM_ICDF(20161), AOM_ICDF(24040), AOM_ICDF(32055), AOM_ICDF(32171), + AOM_ICDF(32768) }, + {AOM_ICDF(21953), AOM_ICDF(24678), AOM_ICDF(27382), AOM_ICDF(28734), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(5750), AOM_ICDF(14739), AOM_ICDF(14792), + AOM_ICDF(32768) }, + {AOM_ICDF(2881), AOM_ICDF(2913), AOM_ICDF(8427), AOM_ICDF(8498), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(27457), AOM_ICDF(31485), AOM_ICDF(32759), AOM_ICDF(32763), + AOM_ICDF(32768) }, + {AOM_ICDF(24129), AOM_ICDF(29502), AOM_ICDF(32752), AOM_ICDF(32757), + AOM_ICDF(32768) }, + {AOM_ICDF(19009), AOM_ICDF(25452), AOM_ICDF(32473), AOM_ICDF(32544), + AOM_ICDF(32768) }, + {AOM_ICDF(32705), AOM_ICDF(32706), AOM_ICDF(32737), AOM_ICDF(32738), + AOM_ICDF(32768) }, + {AOM_ICDF(5313), AOM_ICDF(5366), AOM_ICDF(14376), AOM_ICDF(14430), + AOM_ICDF(32768) }, + {AOM_ICDF(2753), AOM_ICDF(2789), AOM_ICDF(8909), AOM_ICDF(8979), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(27841), AOM_ICDF(32288), AOM_ICDF(32759), AOM_ICDF(32764), + AOM_ICDF(32768) }, + {AOM_ICDF(19137), AOM_ICDF(30271), AOM_ICDF(32742), AOM_ICDF(32753), + AOM_ICDF(32768) }, + {AOM_ICDF(18625), AOM_ICDF(27739), AOM_ICDF(29979), AOM_ICDF(31099), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(7486), AOM_ICDF(20238), AOM_ICDF(21009), + AOM_ICDF(32768) }, + {AOM_ICDF(4929), AOM_ICDF(5579), AOM_ICDF(16402), AOM_ICDF(16866), + AOM_ICDF(32768) }, + {AOM_ICDF(3009), AOM_ICDF(3246), AOM_ICDF(10158), AOM_ICDF(10533), + AOM_ICDF(32768) } } } } +}; +static const coeff_cdf_model default_coef_head_cdf_16x16[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + { AOM_ICDF(960), AOM_ICDF(4882), AOM_ICDF(9467), AOM_ICDF(17710), + AOM_ICDF(20412), AOM_ICDF(32768) }, + { AOM_ICDF(704), AOM_ICDF(4657), AOM_ICDF(6561), AOM_ICDF(14507), + AOM_ICDF(16279), AOM_ICDF(32768) }, + { AOM_ICDF(192), AOM_ICDF(3443), AOM_ICDF(3759), AOM_ICDF(9011), + AOM_ICDF(9685), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(12481), AOM_ICDF(13958), AOM_ICDF(24487), AOM_ICDF(24997), + AOM_ICDF(32768) }, + {AOM_ICDF(11457), AOM_ICDF(13075), AOM_ICDF(23820), AOM_ICDF(24406), + AOM_ICDF(32768) }, + {AOM_ICDF(9793), AOM_ICDF(11127), AOM_ICDF(21775), AOM_ICDF(22387), + AOM_ICDF(32768) }, + {AOM_ICDF(7745), AOM_ICDF(8457), AOM_ICDF(18155), AOM_ICDF(18655), + AOM_ICDF(32768) }, + {AOM_ICDF(5441), AOM_ICDF(5668), AOM_ICDF(13180), AOM_ICDF(13467), + AOM_ICDF(32768) }, + {AOM_ICDF(2497), AOM_ICDF(2520), AOM_ICDF(6340), AOM_ICDF(6417), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(19521), AOM_ICDF(20572), AOM_ICDF(28965), AOM_ICDF(29177), + AOM_ICDF(32768) }, + {AOM_ICDF(15425), AOM_ICDF(16741), AOM_ICDF(27247), AOM_ICDF(27554), + AOM_ICDF(32768) }, + {AOM_ICDF(11969), AOM_ICDF(12690), AOM_ICDF(23872), AOM_ICDF(24141), + AOM_ICDF(32768) }, + {AOM_ICDF(9281), AOM_ICDF(9678), AOM_ICDF(19970), AOM_ICDF(20207), + AOM_ICDF(32768) }, + {AOM_ICDF(6081), AOM_ICDF(6266), AOM_ICDF(14682), AOM_ICDF(14876), + AOM_ICDF(32768) }, + {AOM_ICDF(2753), AOM_ICDF(2779), AOM_ICDF(7150), AOM_ICDF(7225), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(22337), AOM_ICDF(23293), AOM_ICDF(30630), AOM_ICDF(30753), + AOM_ICDF(32768) }, + {AOM_ICDF(16321), AOM_ICDF(17427), AOM_ICDF(28368), AOM_ICDF(28570), + AOM_ICDF(32768) }, + {AOM_ICDF(11457), AOM_ICDF(11907), AOM_ICDF(23570), AOM_ICDF(23741), + AOM_ICDF(32768) }, + {AOM_ICDF(7233), AOM_ICDF(7331), AOM_ICDF(17258), AOM_ICDF(17334), + AOM_ICDF(32768) }, + {AOM_ICDF(4033), AOM_ICDF(4070), AOM_ICDF(10375), AOM_ICDF(10441), + AOM_ICDF(32768) }, + {AOM_ICDF(1601), AOM_ICDF(1619), AOM_ICDF(4706), AOM_ICDF(4788), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(24769), AOM_ICDF(25536), AOM_ICDF(31660), AOM_ICDF(31722), + AOM_ICDF(32768) }, + {AOM_ICDF(18113), AOM_ICDF(18886), AOM_ICDF(29420), AOM_ICDF(29534), + AOM_ICDF(32768) }, + {AOM_ICDF(11201), AOM_ICDF(11412), AOM_ICDF(23207), AOM_ICDF(23291), + AOM_ICDF(32768) }, + {AOM_ICDF(6977), AOM_ICDF(7033), AOM_ICDF(16599), AOM_ICDF(16646), + AOM_ICDF(32768) }, + {AOM_ICDF(4033), AOM_ICDF(4070), AOM_ICDF(10375), AOM_ICDF(10441), + AOM_ICDF(32768) }, + {AOM_ICDF(1601), AOM_ICDF(1620), AOM_ICDF(4827), AOM_ICDF(4909), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(28353), AOM_ICDF(28831), AOM_ICDF(32502), AOM_ICDF(32517), + AOM_ICDF(32768) }, + {AOM_ICDF(21441), AOM_ICDF(21869), AOM_ICDF(30977), AOM_ICDF(31017), + AOM_ICDF(32768) }, + {AOM_ICDF(11969), AOM_ICDF(12088), AOM_ICDF(24116), AOM_ICDF(24158), + AOM_ICDF(32768) }, + {AOM_ICDF(7489), AOM_ICDF(7547), AOM_ICDF(17413), AOM_ICDF(17458), + AOM_ICDF(32768) }, + {AOM_ICDF(4545), AOM_ICDF(4585), AOM_ICDF(11325), AOM_ICDF(11388), + AOM_ICDF(32768) }, + {AOM_ICDF(2113), AOM_ICDF(2133), AOM_ICDF(5526), AOM_ICDF(5606), + AOM_ICDF(32768) } } }, + { // Intra + { // Band 0 + { AOM_ICDF(2496), AOM_ICDF(8717), AOM_ICDF(17280), AOM_ICDF(28922), + AOM_ICDF(29751), AOM_ICDF(32768) }, + { AOM_ICDF(2496), AOM_ICDF(9665), AOM_ICDF(15235), AOM_ICDF(26542), + AOM_ICDF(27580), AOM_ICDF(32768) }, + { AOM_ICDF(448), AOM_ICDF(9240), AOM_ICDF(11886), AOM_ICDF(24124), + AOM_ICDF(24898), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(21057), AOM_ICDF(22896), AOM_ICDF(31877), AOM_ICDF(31953), + AOM_ICDF(32768) }, + {AOM_ICDF(20673), AOM_ICDF(23151), AOM_ICDF(31706), AOM_ICDF(31825), + AOM_ICDF(32768) }, + {AOM_ICDF(18753), AOM_ICDF(20519), AOM_ICDF(30497), AOM_ICDF(30668), + AOM_ICDF(32768) }, + {AOM_ICDF(15425), AOM_ICDF(16608), AOM_ICDF(27789), AOM_ICDF(28027), + AOM_ICDF(32768) }, + {AOM_ICDF(10305), AOM_ICDF(10977), AOM_ICDF(21405), AOM_ICDF(21749), + AOM_ICDF(32768) }, + {AOM_ICDF(3649), AOM_ICDF(3812), AOM_ICDF(11213), AOM_ICDF(11445), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(24001), AOM_ICDF(25899), AOM_ICDF(32307), AOM_ICDF(32360), + AOM_ICDF(32768) }, + {AOM_ICDF(20929), AOM_ICDF(22941), AOM_ICDF(31775), AOM_ICDF(31867), + AOM_ICDF(32768) }, + {AOM_ICDF(15169), AOM_ICDF(16734), AOM_ICDF(29228), AOM_ICDF(29425), + AOM_ICDF(32768) }, + {AOM_ICDF(10561), AOM_ICDF(12047), AOM_ICDF(24918), AOM_ICDF(25324), + AOM_ICDF(32768) }, + {AOM_ICDF(6977), AOM_ICDF(7929), AOM_ICDF(18311), AOM_ICDF(18918), + AOM_ICDF(32768) }, + {AOM_ICDF(3649), AOM_ICDF(3760), AOM_ICDF(9962), AOM_ICDF(10162), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(25793), AOM_ICDF(27526), AOM_ICDF(32565), AOM_ICDF(32591), + AOM_ICDF(32768) }, + {AOM_ICDF(21825), AOM_ICDF(23885), AOM_ICDF(32064), AOM_ICDF(32135), + AOM_ICDF(32768) }, + {AOM_ICDF(15041), AOM_ICDF(16286), AOM_ICDF(29203), AOM_ICDF(29360), + AOM_ICDF(32768) }, + {AOM_ICDF(10433), AOM_ICDF(11058), AOM_ICDF(24349), AOM_ICDF(24538), + AOM_ICDF(32768) }, + {AOM_ICDF(5569), AOM_ICDF(6016), AOM_ICDF(16460), AOM_ICDF(16794), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(194), AOM_ICDF(384), AOM_ICDF(479), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(26433), AOM_ICDF(28398), AOM_ICDF(32682), AOM_ICDF(32696), + AOM_ICDF(32768) }, + {AOM_ICDF(22977), AOM_ICDF(25086), AOM_ICDF(32367), AOM_ICDF(32412), + AOM_ICDF(32768) }, + {AOM_ICDF(16577), AOM_ICDF(17928), AOM_ICDF(30144), AOM_ICDF(30275), + AOM_ICDF(32768) }, + {AOM_ICDF(12481), AOM_ICDF(13352), AOM_ICDF(25993), AOM_ICDF(26211), + AOM_ICDF(32768) }, + {AOM_ICDF(7745), AOM_ICDF(8069), AOM_ICDF(20501), AOM_ICDF(20657), + AOM_ICDF(32768) }, + {AOM_ICDF(16449), AOM_ICDF(16450), AOM_ICDF(16545), AOM_ICDF(16593), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(27841), AOM_ICDF(29700), AOM_ICDF(32721), AOM_ICDF(32730), + AOM_ICDF(32768) }, + {AOM_ICDF(23873), AOM_ICDF(26202), AOM_ICDF(32578), AOM_ICDF(32604), + AOM_ICDF(32768) }, + {AOM_ICDF(17729), AOM_ICDF(19046), AOM_ICDF(30448), AOM_ICDF(30568), + AOM_ICDF(32768) }, + {AOM_ICDF(13505), AOM_ICDF(14508), AOM_ICDF(26034), AOM_ICDF(26304), + AOM_ICDF(32768) }, + {AOM_ICDF(10049), AOM_ICDF(10494), AOM_ICDF(19945), AOM_ICDF(20233), + AOM_ICDF(32768) }, + {AOM_ICDF(2113), AOM_ICDF(2183), AOM_ICDF(7202), AOM_ICDF(7377), + AOM_ICDF(32768) } } } }, + { // UV plane + { // Inter + { // Band 0 + { AOM_ICDF(27072), AOM_ICDF(27916), AOM_ICDF(31095), AOM_ICDF(32400), + AOM_ICDF(32553), AOM_ICDF(32768) }, + { AOM_ICDF(12352), AOM_ICDF(16792), AOM_ICDF(22516), AOM_ICDF(28853), + AOM_ICDF(29797), AOM_ICDF(32768) }, + { AOM_ICDF(2880), AOM_ICDF(9023), AOM_ICDF(11126), AOM_ICDF(20602), + AOM_ICDF(21713), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(20161), AOM_ICDF(24785), AOM_ICDF(31070), AOM_ICDF(31430), + AOM_ICDF(32768) }, + {AOM_ICDF(17985), AOM_ICDF(22773), AOM_ICDF(30430), AOM_ICDF(30880), + AOM_ICDF(32768) }, + {AOM_ICDF(15937), AOM_ICDF(18802), AOM_ICDF(28265), AOM_ICDF(28788), + AOM_ICDF(32768) }, + {AOM_ICDF(11841), AOM_ICDF(13587), AOM_ICDF(24798), AOM_ICDF(25335), + AOM_ICDF(32768) }, + {AOM_ICDF(8769), AOM_ICDF(9160), AOM_ICDF(19316), AOM_ICDF(19566), + AOM_ICDF(32768) }, + {AOM_ICDF(5313), AOM_ICDF(5357), AOM_ICDF(12874), AOM_ICDF(12932), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(24129), AOM_ICDF(26501), AOM_ICDF(31672), AOM_ICDF(31844), + AOM_ICDF(32768) }, + {AOM_ICDF(19649), AOM_ICDF(21553), AOM_ICDF(30130), AOM_ICDF(30370), + AOM_ICDF(32768) }, + {AOM_ICDF(11713), AOM_ICDF(13134), AOM_ICDF(25983), AOM_ICDF(26321), + AOM_ICDF(32768) }, + {AOM_ICDF(9409), AOM_ICDF(9948), AOM_ICDF(21408), AOM_ICDF(21663), + AOM_ICDF(32768) }, + {AOM_ICDF(5569), AOM_ICDF(5757), AOM_ICDF(14335), AOM_ICDF(14533), + AOM_ICDF(32768) }, + {AOM_ICDF(2241), AOM_ICDF(2305), AOM_ICDF(13152), AOM_ICDF(13209), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(26817), AOM_ICDF(28135), AOM_ICDF(32130), AOM_ICDF(32209), + AOM_ICDF(32768) }, + {AOM_ICDF(20161), AOM_ICDF(21412), AOM_ICDF(30331), AOM_ICDF(30481), + AOM_ICDF(32768) }, + {AOM_ICDF(13377), AOM_ICDF(13798), AOM_ICDF(26065), AOM_ICDF(26176), + AOM_ICDF(32768) }, + {AOM_ICDF(8129), AOM_ICDF(8290), AOM_ICDF(19920), AOM_ICDF(20008), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(5751), AOM_ICDF(14950), AOM_ICDF(15002), + AOM_ICDF(32768) }, + {AOM_ICDF(5569), AOM_ICDF(5601), AOM_ICDF(11041), AOM_ICDF(11105), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(28225), AOM_ICDF(29079), AOM_ICDF(32387), AOM_ICDF(32426), + AOM_ICDF(32768) }, + {AOM_ICDF(21185), AOM_ICDF(22046), AOM_ICDF(30982), AOM_ICDF(31061), + AOM_ICDF(32768) }, + {AOM_ICDF(13377), AOM_ICDF(13595), AOM_ICDF(25762), AOM_ICDF(25824), + AOM_ICDF(32768) }, + {AOM_ICDF(8001), AOM_ICDF(8123), AOM_ICDF(20530), AOM_ICDF(20590), + AOM_ICDF(32768) }, + {AOM_ICDF(4289), AOM_ICDF(4322), AOM_ICDF(9907), AOM_ICDF(9974), + AOM_ICDF(32768) }, + {AOM_ICDF(3393), AOM_ICDF(3412), AOM_ICDF(6663), AOM_ICDF(6739), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(30529), AOM_ICDF(31014), AOM_ICDF(32651), AOM_ICDF(32664), + AOM_ICDF(32768) }, + {AOM_ICDF(23489), AOM_ICDF(24268), AOM_ICDF(31627), AOM_ICDF(31682), + AOM_ICDF(32768) }, + {AOM_ICDF(14017), AOM_ICDF(14239), AOM_ICDF(26653), AOM_ICDF(26707), + AOM_ICDF(32768) }, + {AOM_ICDF(11201), AOM_ICDF(11317), AOM_ICDF(23122), AOM_ICDF(23169), + AOM_ICDF(32768) }, + {AOM_ICDF(6721), AOM_ICDF(6768), AOM_ICDF(14810), AOM_ICDF(14863), + AOM_ICDF(32768) }, + {AOM_ICDF(6593), AOM_ICDF(6632), AOM_ICDF(13188), AOM_ICDF(13245), + AOM_ICDF(32768) } } }, + { // Inter + { // Band 0 + { AOM_ICDF(29888), AOM_ICDF(30492), AOM_ICDF(32500), AOM_ICDF(32766), + AOM_ICDF(32767), AOM_ICDF(32768) }, + { AOM_ICDF(18752), AOM_ICDF(23235), AOM_ICDF(29846), AOM_ICDF(32214), + AOM_ICDF(32442), AOM_ICDF(32768) }, + { AOM_ICDF(5568), AOM_ICDF(17762), AOM_ICDF(25039), AOM_ICDF(31213), + AOM_ICDF(31651), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(26433), AOM_ICDF(29681), AOM_ICDF(32757), AOM_ICDF(32760), + AOM_ICDF(32768) }, + {AOM_ICDF(24769), AOM_ICDF(28761), AOM_ICDF(32722), AOM_ICDF(32734), + AOM_ICDF(32768) }, + {AOM_ICDF(22209), AOM_ICDF(26975), AOM_ICDF(32418), AOM_ICDF(32500), + AOM_ICDF(32768) }, + {AOM_ICDF(16321), AOM_ICDF(21333), AOM_ICDF(28368), AOM_ICDF(29283), + AOM_ICDF(32768) }, + {AOM_ICDF(12865), AOM_ICDF(14775), AOM_ICDF(22545), AOM_ICDF(23553), + AOM_ICDF(32768) }, + {AOM_ICDF(12353), AOM_ICDF(12354), AOM_ICDF(12473), AOM_ICDF(12532), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(27457), AOM_ICDF(30005), AOM_ICDF(32738), AOM_ICDF(32745), + AOM_ICDF(32768) }, + {AOM_ICDF(24897), AOM_ICDF(27541), AOM_ICDF(32723), AOM_ICDF(32731), + AOM_ICDF(32768) }, + {AOM_ICDF(15297), AOM_ICDF(19106), AOM_ICDF(30414), AOM_ICDF(30711), + AOM_ICDF(32768) }, + {AOM_ICDF(6593), AOM_ICDF(8826), AOM_ICDF(19732), AOM_ICDF(20840), + AOM_ICDF(32768) }, + {AOM_ICDF(4161), AOM_ICDF(4233), AOM_ICDF(16509), AOM_ICDF(16557), + AOM_ICDF(32768) }, + {AOM_ICDF(2625), AOM_ICDF(2652), AOM_ICDF(7276), AOM_ICDF(7351), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(28609), AOM_ICDF(30482), AOM_ICDF(32761), AOM_ICDF(32763), + AOM_ICDF(32768) }, + {AOM_ICDF(25665), AOM_ICDF(27830), AOM_ICDF(32727), AOM_ICDF(32733), + AOM_ICDF(32768) }, + {AOM_ICDF(21057), AOM_ICDF(23803), AOM_ICDF(30367), AOM_ICDF(30721), + AOM_ICDF(32768) }, + {AOM_ICDF(10945), AOM_ICDF(21878), AOM_ICDF(32726), AOM_ICDF(32737), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(5750), AOM_ICDF(14739), AOM_ICDF(14792), + AOM_ICDF(32768) }, + {AOM_ICDF(2881), AOM_ICDF(2913), AOM_ICDF(8427), AOM_ICDF(8498), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(28993), AOM_ICDF(30944), AOM_ICDF(32762), AOM_ICDF(32764), + AOM_ICDF(32768) }, + {AOM_ICDF(26561), AOM_ICDF(28695), AOM_ICDF(32733), AOM_ICDF(32739), + AOM_ICDF(32768) }, + {AOM_ICDF(17985), AOM_ICDF(19028), AOM_ICDF(31008), AOM_ICDF(31079), + AOM_ICDF(32768) }, + {AOM_ICDF(7873), AOM_ICDF(8039), AOM_ICDF(19981), AOM_ICDF(20068), + AOM_ICDF(32768) }, + {AOM_ICDF(5313), AOM_ICDF(5366), AOM_ICDF(14376), AOM_ICDF(14430), + AOM_ICDF(32768) }, + {AOM_ICDF(2753), AOM_ICDF(2789), AOM_ICDF(8909), AOM_ICDF(8979), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(30273), AOM_ICDF(32029), AOM_ICDF(32764), AOM_ICDF(32766), + AOM_ICDF(32768) }, + {AOM_ICDF(28609), AOM_ICDF(30847), AOM_ICDF(32745), AOM_ICDF(32751), + AOM_ICDF(32768) }, + {AOM_ICDF(21313), AOM_ICDF(24377), AOM_ICDF(31986), AOM_ICDF(32098), + AOM_ICDF(32768) }, + {AOM_ICDF(32705), AOM_ICDF(32709), AOM_ICDF(32739), AOM_ICDF(32741), + AOM_ICDF(32768) }, + {AOM_ICDF(4929), AOM_ICDF(5579), AOM_ICDF(16402), AOM_ICDF(16866), + AOM_ICDF(32768) }, + {AOM_ICDF(3009), AOM_ICDF(3246), AOM_ICDF(10158), AOM_ICDF(10533), + AOM_ICDF(32768) } } } } +}; +static const coeff_cdf_model default_coef_head_cdf_32x32[PLANE_TYPES] = { + { // Y plane + { // Intra + { // Band 0 + { AOM_ICDF(2240), AOM_ICDF(5407), AOM_ICDF(18304), AOM_ICDF(25601), + AOM_ICDF(27911), AOM_ICDF(32768) }, + { AOM_ICDF(960), AOM_ICDF(4633), AOM_ICDF(8197), AOM_ICDF(16254), + AOM_ICDF(18796), AOM_ICDF(32768) }, + { AOM_ICDF(192), AOM_ICDF(3061), AOM_ICDF(3557), AOM_ICDF(8701), + AOM_ICDF(9762), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(11969), AOM_ICDF(15846), AOM_ICDF(25660), AOM_ICDF(26667), + AOM_ICDF(32768) }, + {AOM_ICDF(11713), AOM_ICDF(15794), AOM_ICDF(25737), AOM_ICDF(26760), + AOM_ICDF(32768) }, + {AOM_ICDF(9281), AOM_ICDF(12675), AOM_ICDF(23181), AOM_ICDF(24351), + AOM_ICDF(32768) }, + {AOM_ICDF(7105), AOM_ICDF(8757), AOM_ICDF(18383), AOM_ICDF(19437), + AOM_ICDF(32768) }, + {AOM_ICDF(4289), AOM_ICDF(4579), AOM_ICDF(11353), AOM_ICDF(11792), + AOM_ICDF(32768) }, + {AOM_ICDF(1857), AOM_ICDF(1874), AOM_ICDF(4695), AOM_ICDF(4777), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(20929), AOM_ICDF(22297), AOM_ICDF(29370), AOM_ICDF(29646), + AOM_ICDF(32768) }, + {AOM_ICDF(17473), AOM_ICDF(18985), AOM_ICDF(28079), AOM_ICDF(28413), + AOM_ICDF(32768) }, + {AOM_ICDF(13121), AOM_ICDF(14064), AOM_ICDF(24902), AOM_ICDF(25217), + AOM_ICDF(32768) }, + {AOM_ICDF(9793), AOM_ICDF(10214), AOM_ICDF(20069), AOM_ICDF(20329), + AOM_ICDF(32768) }, + {AOM_ICDF(5825), AOM_ICDF(5987), AOM_ICDF(13350), AOM_ICDF(13559), + AOM_ICDF(32768) }, + {AOM_ICDF(2241), AOM_ICDF(2260), AOM_ICDF(5520), AOM_ICDF(5600), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(25921), AOM_ICDF(26891), AOM_ICDF(31632), AOM_ICDF(31729), + AOM_ICDF(32768) }, + {AOM_ICDF(18241), AOM_ICDF(19463), AOM_ICDF(29222), AOM_ICDF(29419), + AOM_ICDF(32768) }, + {AOM_ICDF(11585), AOM_ICDF(12065), AOM_ICDF(23294), AOM_ICDF(23488), + AOM_ICDF(32768) }, + {AOM_ICDF(6593), AOM_ICDF(6686), AOM_ICDF(16153), AOM_ICDF(16234), + AOM_ICDF(32768) }, + {AOM_ICDF(3137), AOM_ICDF(3170), AOM_ICDF(8751), AOM_ICDF(8821), + AOM_ICDF(32768) }, + {AOM_ICDF(1345), AOM_ICDF(1359), AOM_ICDF(3739), AOM_ICDF(3824), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(27713), AOM_ICDF(28504), AOM_ICDF(32068), AOM_ICDF(32132), + AOM_ICDF(32768) }, + {AOM_ICDF(19265), AOM_ICDF(20354), AOM_ICDF(29789), AOM_ICDF(29943), + AOM_ICDF(32768) }, + {AOM_ICDF(11201), AOM_ICDF(11538), AOM_ICDF(22701), AOM_ICDF(22848), + AOM_ICDF(32768) }, + {AOM_ICDF(6337), AOM_ICDF(6424), AOM_ICDF(15268), AOM_ICDF(15353), + AOM_ICDF(32768) }, + {AOM_ICDF(3649), AOM_ICDF(3681), AOM_ICDF(9052), AOM_ICDF(9121), + AOM_ICDF(32768) }, + {AOM_ICDF(1601), AOM_ICDF(1618), AOM_ICDF(4584), AOM_ICDF(4667), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(30913), AOM_ICDF(31044), AOM_ICDF(32635), AOM_ICDF(32640), + AOM_ICDF(32768) }, + {AOM_ICDF(22081), AOM_ICDF(22261), AOM_ICDF(30452), AOM_ICDF(30477), + AOM_ICDF(32768) }, + {AOM_ICDF(10561), AOM_ICDF(10625), AOM_ICDF(21535), AOM_ICDF(21568), + AOM_ICDF(32768) }, + {AOM_ICDF(6081), AOM_ICDF(6130), AOM_ICDF(14369), AOM_ICDF(14423), + AOM_ICDF(32768) }, + {AOM_ICDF(3777), AOM_ICDF(3809), AOM_ICDF(9156), AOM_ICDF(9225), + AOM_ICDF(32768) }, + {AOM_ICDF(1857), AOM_ICDF(1875), AOM_ICDF(4936), AOM_ICDF(5018), + AOM_ICDF(32768) } } }, + { // Intra + { // Band 0 + { AOM_ICDF(4672), AOM_ICDF(6927), AOM_ICDF(23534), AOM_ICDF(29846), + AOM_ICDF(30928), AOM_ICDF(32768) }, + { AOM_ICDF(3776), AOM_ICDF(6784), AOM_ICDF(18075), AOM_ICDF(25863), + AOM_ICDF(27926), AOM_ICDF(32768) }, + { AOM_ICDF(1344), AOM_ICDF(5588), AOM_ICDF(12166), AOM_ICDF(20966), + AOM_ICDF(23504), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(19393), AOM_ICDF(22016), AOM_ICDF(31280), AOM_ICDF(31444), + AOM_ICDF(32768) }, + {AOM_ICDF(21185), AOM_ICDF(24329), AOM_ICDF(31706), AOM_ICDF(31865), + AOM_ICDF(32768) }, + {AOM_ICDF(20673), AOM_ICDF(23240), AOM_ICDF(31186), AOM_ICDF(31379), + AOM_ICDF(32768) }, + {AOM_ICDF(17857), AOM_ICDF(20035), AOM_ICDF(29594), AOM_ICDF(29889), + AOM_ICDF(32768) }, + {AOM_ICDF(13633), AOM_ICDF(14929), AOM_ICDF(24883), AOM_ICDF(25337), + AOM_ICDF(32768) }, + {AOM_ICDF(7873), AOM_ICDF(8416), AOM_ICDF(17452), AOM_ICDF(17886), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(25665), AOM_ICDF(27145), AOM_ICDF(32256), AOM_ICDF(32314), + AOM_ICDF(32768) }, + {AOM_ICDF(21057), AOM_ICDF(22826), AOM_ICDF(31465), AOM_ICDF(31576), + AOM_ICDF(32768) }, + {AOM_ICDF(13633), AOM_ICDF(14885), AOM_ICDF(27873), AOM_ICDF(28088), + AOM_ICDF(32768) }, + {AOM_ICDF(8769), AOM_ICDF(9515), AOM_ICDF(21941), AOM_ICDF(22248), + AOM_ICDF(32768) }, + {AOM_ICDF(6209), AOM_ICDF(6594), AOM_ICDF(15598), AOM_ICDF(15950), + AOM_ICDF(32768) }, + {AOM_ICDF(1985), AOM_ICDF(2014), AOM_ICDF(6855), AOM_ICDF(6931), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(26817), AOM_ICDF(27824), AOM_ICDF(32362), AOM_ICDF(32399), + AOM_ICDF(32768) }, + {AOM_ICDF(21185), AOM_ICDF(22321), AOM_ICDF(31389), AOM_ICDF(31466), + AOM_ICDF(32768) }, + {AOM_ICDF(13761), AOM_ICDF(14154), AOM_ICDF(27163), AOM_ICDF(27245), + AOM_ICDF(32768) }, + {AOM_ICDF(8897), AOM_ICDF(9011), AOM_ICDF(20600), AOM_ICDF(20659), + AOM_ICDF(32768) }, + {AOM_ICDF(4673), AOM_ICDF(4774), AOM_ICDF(15044), AOM_ICDF(15131), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(194), AOM_ICDF(384), AOM_ICDF(479), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(28865), AOM_ICDF(29687), AOM_ICDF(32655), AOM_ICDF(32667), + AOM_ICDF(32768) }, + {AOM_ICDF(23233), AOM_ICDF(24218), AOM_ICDF(32080), AOM_ICDF(32118), + AOM_ICDF(32768) }, + {AOM_ICDF(15041), AOM_ICDF(15444), AOM_ICDF(28787), AOM_ICDF(28845), + AOM_ICDF(32768) }, + {AOM_ICDF(9921), AOM_ICDF(10248), AOM_ICDF(22818), AOM_ICDF(22944), + AOM_ICDF(32768) }, + {AOM_ICDF(7745), AOM_ICDF(7866), AOM_ICDF(16591), AOM_ICDF(16702), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(194), AOM_ICDF(384), AOM_ICDF(479), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(31169), AOM_ICDF(31559), AOM_ICDF(32741), AOM_ICDF(32744), + AOM_ICDF(32768) }, + {AOM_ICDF(24769), AOM_ICDF(25583), AOM_ICDF(32347), AOM_ICDF(32370), + AOM_ICDF(32768) }, + {AOM_ICDF(15937), AOM_ICDF(16169), AOM_ICDF(29120), AOM_ICDF(29152), + AOM_ICDF(32768) }, + {AOM_ICDF(7489), AOM_ICDF(7578), AOM_ICDF(22647), AOM_ICDF(22677), + AOM_ICDF(32768) }, + {AOM_ICDF(7617), AOM_ICDF(7689), AOM_ICDF(19849), AOM_ICDF(19887), + AOM_ICDF(32768) }, + {AOM_ICDF(2113), AOM_ICDF(2183), AOM_ICDF(7202), AOM_ICDF(7377), + AOM_ICDF(32768) } } } }, + { // UV plane + { // Inter + { // Band 0 + { AOM_ICDF(23232), AOM_ICDF(24301), AOM_ICDF(30231), AOM_ICDF(31582), + AOM_ICDF(32091), AOM_ICDF(32768) }, + { AOM_ICDF(7872), AOM_ICDF(11041), AOM_ICDF(22542), AOM_ICDF(27086), + AOM_ICDF(29145), AOM_ICDF(32768) }, + { AOM_ICDF(1344), AOM_ICDF(3989), AOM_ICDF(18125), AOM_ICDF(25340), + AOM_ICDF(27820), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(15937), AOM_ICDF(29000), AOM_ICDF(32210), AOM_ICDF(32434), + AOM_ICDF(32768) }, + {AOM_ICDF(12353), AOM_ICDF(26626), AOM_ICDF(31533), AOM_ICDF(31993), + AOM_ICDF(32768) }, + {AOM_ICDF(11457), AOM_ICDF(29187), AOM_ICDF(30896), AOM_ICDF(31750), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(21278), AOM_ICDF(28169), AOM_ICDF(29764), + AOM_ICDF(32768) }, + {AOM_ICDF(7489), AOM_ICDF(8855), AOM_ICDF(13365), AOM_ICDF(15620), + AOM_ICDF(32768) }, + {AOM_ICDF(4289), AOM_ICDF(4833), AOM_ICDF(8572), AOM_ICDF(10108), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(25025), AOM_ICDF(30783), AOM_ICDF(32603), AOM_ICDF(32666), + AOM_ICDF(32768) }, + {AOM_ICDF(24385), AOM_ICDF(29586), AOM_ICDF(31803), AOM_ICDF(32142), + AOM_ICDF(32768) }, + {AOM_ICDF(22337), AOM_ICDF(23002), AOM_ICDF(27573), AOM_ICDF(27903), + AOM_ICDF(32768) }, + {AOM_ICDF(10945), AOM_ICDF(12336), AOM_ICDF(21900), AOM_ICDF(22590), + AOM_ICDF(32768) }, + {AOM_ICDF(8257), AOM_ICDF(8830), AOM_ICDF(19986), AOM_ICDF(20298), + AOM_ICDF(32768) }, + {AOM_ICDF(10945), AOM_ICDF(10990), AOM_ICDF(18660), AOM_ICDF(18701), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(29761), AOM_ICDF(31473), AOM_ICDF(32693), AOM_ICDF(32715), + AOM_ICDF(32768) }, + {AOM_ICDF(20417), AOM_ICDF(24512), AOM_ICDF(31394), AOM_ICDF(31650), + AOM_ICDF(32768) }, + {AOM_ICDF(11713), AOM_ICDF(13283), AOM_ICDF(25819), AOM_ICDF(26206), + AOM_ICDF(32768) }, + {AOM_ICDF(13121), AOM_ICDF(14099), AOM_ICDF(21909), AOM_ICDF(22514), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(248), AOM_ICDF(9546), AOM_ICDF(9614), + AOM_ICDF(32768) }, + {AOM_ICDF(2497), AOM_ICDF(2524), AOM_ICDF(7050), AOM_ICDF(7125), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(30657), AOM_ICDF(31885), AOM_ICDF(32691), AOM_ICDF(32715), + AOM_ICDF(32768) }, + {AOM_ICDF(19393), AOM_ICDF(26050), AOM_ICDF(31698), AOM_ICDF(31988), + AOM_ICDF(32768) }, + {AOM_ICDF(15809), AOM_ICDF(15863), AOM_ICDF(24985), AOM_ICDF(25008), + AOM_ICDF(32768) }, + {AOM_ICDF(23489), AOM_ICDF(28138), AOM_ICDF(32751), AOM_ICDF(32756), + AOM_ICDF(32768) }, + {AOM_ICDF(16449), AOM_ICDF(16450), AOM_ICDF(16545), AOM_ICDF(16593), + AOM_ICDF(32768) }, + {AOM_ICDF(2369), AOM_ICDF(2395), AOM_ICDF(6822), AOM_ICDF(6898), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(32705), AOM_ICDF(32744), AOM_ICDF(32766), AOM_ICDF(32767), + AOM_ICDF(32768) }, + {AOM_ICDF(21953), AOM_ICDF(24962), AOM_ICDF(32156), AOM_ICDF(32246), + AOM_ICDF(32768) }, + {AOM_ICDF(13121), AOM_ICDF(15358), AOM_ICDF(26284), AOM_ICDF(26835), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(7417), AOM_ICDF(20132), AOM_ICDF(20885), + AOM_ICDF(32768) }, + {AOM_ICDF(4417), AOM_ICDF(4939), AOM_ICDF(15104), AOM_ICDF(15535), + AOM_ICDF(32768) }, + {AOM_ICDF(2625), AOM_ICDF(2680), AOM_ICDF(8218), AOM_ICDF(8338), + AOM_ICDF(32768) } } }, + { // Inter + { // Band 0 + { AOM_ICDF(25280), AOM_ICDF(25678), AOM_ICDF(32446), AOM_ICDF(32622), + AOM_ICDF(32724), AOM_ICDF(32768) }, + { AOM_ICDF(10560), AOM_ICDF(11822), AOM_ICDF(28682), AOM_ICDF(29919), + AOM_ICDF(31276), AOM_ICDF(32768) }, + { AOM_ICDF(3264), AOM_ICDF(5170), AOM_ICDF(21779), AOM_ICDF(24026), + AOM_ICDF(27905), AOM_ICDF(32768) } }, + { // Band 1 + {AOM_ICDF(24257), AOM_ICDF(30554), AOM_ICDF(32719), AOM_ICDF(32738), + AOM_ICDF(32768) }, + {AOM_ICDF(17217), AOM_ICDF(27413), AOM_ICDF(32617), AOM_ICDF(32667), + AOM_ICDF(32768) }, + {AOM_ICDF(22977), AOM_ICDF(27600), AOM_ICDF(32482), AOM_ICDF(32552), + AOM_ICDF(32768) }, + {AOM_ICDF(16833), AOM_ICDF(24360), AOM_ICDF(30746), AOM_ICDF(31293), + AOM_ICDF(32768) }, + {AOM_ICDF(17089), AOM_ICDF(20060), AOM_ICDF(28880), AOM_ICDF(29370), + AOM_ICDF(32768) }, + {AOM_ICDF(10945), AOM_ICDF(11009), AOM_ICDF(21900), AOM_ICDF(21932), + AOM_ICDF(32768) } }, + { // Band 2 + {AOM_ICDF(27201), AOM_ICDF(30217), AOM_ICDF(32736), AOM_ICDF(32745), + AOM_ICDF(32768) }, + {AOM_ICDF(22721), AOM_ICDF(27676), AOM_ICDF(32749), AOM_ICDF(32754), + AOM_ICDF(32768) }, + {AOM_ICDF(5057), AOM_ICDF(12431), AOM_ICDF(25246), AOM_ICDF(26620), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(321), AOM_ICDF(22016), AOM_ICDF(22048), + AOM_ICDF(32768) }, + {AOM_ICDF(5313), AOM_ICDF(5363), AOM_ICDF(13839), AOM_ICDF(13894), + AOM_ICDF(32768) }, + {AOM_ICDF(2625), AOM_ICDF(2652), AOM_ICDF(7276), AOM_ICDF(7351), + AOM_ICDF(32768) } }, + { // Band 3 + {AOM_ICDF(27713), AOM_ICDF(30739), AOM_ICDF(32759), AOM_ICDF(32762), + AOM_ICDF(32768) }, + {AOM_ICDF(26177), AOM_ICDF(30430), AOM_ICDF(32756), AOM_ICDF(32760), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(384), AOM_ICDF(32706), AOM_ICDF(32707), + AOM_ICDF(32768) }, + {AOM_ICDF(9409), AOM_ICDF(9528), AOM_ICDF(21591), AOM_ICDF(21646), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(194), AOM_ICDF(384), AOM_ICDF(479), + AOM_ICDF(32768) }, + {AOM_ICDF(2881), AOM_ICDF(2913), AOM_ICDF(8427), AOM_ICDF(8498), + AOM_ICDF(32768) } }, + { // Band 4 + {AOM_ICDF(28993), AOM_ICDF(31156), AOM_ICDF(32747), AOM_ICDF(32753), + AOM_ICDF(32768) }, + {AOM_ICDF(25153), AOM_ICDF(28701), AOM_ICDF(32754), AOM_ICDF(32758), + AOM_ICDF(32768) }, + {AOM_ICDF(16449), AOM_ICDF(16544), AOM_ICDF(32737), AOM_ICDF(32738), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(321), AOM_ICDF(22016), AOM_ICDF(22048), + AOM_ICDF(32768) }, + {AOM_ICDF(193), AOM_ICDF(194), AOM_ICDF(384), AOM_ICDF(479), + AOM_ICDF(32768) }, + {AOM_ICDF(2753), AOM_ICDF(2789), AOM_ICDF(8909), AOM_ICDF(8979), + AOM_ICDF(32768) } }, + { // Band 5 + {AOM_ICDF(30785), AOM_ICDF(32088), AOM_ICDF(32765), AOM_ICDF(32766), + AOM_ICDF(32768) }, + {AOM_ICDF(22977), AOM_ICDF(26623), AOM_ICDF(32750), AOM_ICDF(32754), + AOM_ICDF(32768) }, + {AOM_ICDF(21953), AOM_ICDF(21954), AOM_ICDF(22017), AOM_ICDF(22049), + AOM_ICDF(32768) }, + {AOM_ICDF(5697), AOM_ICDF(7486), AOM_ICDF(20238), AOM_ICDF(21009), + AOM_ICDF(32768) }, + {AOM_ICDF(4929), AOM_ICDF(5579), AOM_ICDF(16402), AOM_ICDF(16866), + AOM_ICDF(32768) }, + {AOM_ICDF(3009), AOM_ICDF(3246), AOM_ICDF(10158), AOM_ICDF(10533), + AOM_ICDF(32768) } } } } +}; +#endif // CONFIG_NEW_TOKENSET + +/* clang-format on */ + +static void extend_to_full_distribution(aom_prob *probs, aom_prob p) { + assert(p != 0); + memcpy(probs, av1_pareto8_full[p - 1], MODEL_NODES * sizeof(aom_prob)); +} + +void av1_model_to_full_probs(const aom_prob *model, aom_prob *full) { + if (full != model) + memcpy(full, model, sizeof(aom_prob) * UNCONSTRAINED_NODES); + extend_to_full_distribution(&full[UNCONSTRAINED_NODES], model[PIVOT_NODE]); +} + +#if CONFIG_NEW_TOKENSET + +static void build_tail_cdfs(aom_cdf_prob cdf_tail[CDF_SIZE(ENTROPY_TOKENS)], + aom_cdf_prob cdf_head[CDF_SIZE(ENTROPY_TOKENS)], + int band_zero) { + int probNZ, prob1, prob_idx, i; + int phead[HEAD_TOKENS + 1], sum; + const int is_dc = !!band_zero; + aom_cdf_prob prev_cdf; + prev_cdf = 0; + for (i = 0; i < HEAD_TOKENS + is_dc; ++i) { + phead[i] = AOM_ICDF(cdf_head[i]) - prev_cdf; + prev_cdf = AOM_ICDF(cdf_head[i]); + } + // Do the tail + probNZ = CDF_PROB_TOP - phead[ZERO_TOKEN + is_dc] - (is_dc ? phead[0] : 0); + prob1 = phead[is_dc + ONE_TOKEN_EOB] + phead[is_dc + ONE_TOKEN_NEOB]; + prob_idx = + AOMMIN(COEFF_PROB_MODELS - 1, AOMMAX(0, ((256 * prob1) / probNZ) - 1)); + + sum = 0; + for (i = 0; i < TAIL_TOKENS; ++i) { + sum += av1_pareto8_tail_probs[prob_idx][i]; + cdf_tail[i] = AOM_ICDF(sum); + } +} + +static void build_head_cdfs(const aom_prob *pdf_model, + const aom_prob *blockz_model, + aom_cdf_prob cdf_head[ENTROPY_TOKENS + 1]) { + int i, p, p1, p2, phead[6], prob_NZ, prob_EOB_1, prob_EOB_2p, prob_NEOB_1, + prob_NEOB_2p; + int prob8_blocknz; + // We have the first coefficient position and so an extended CDF + const int is_dc = blockz_model != NULL; + const int last_head_val = HEAD_TOKENS - 1 + is_dc; + + assert(pdf_model != NULL); + assert(pdf_model[2] != 0); + + /* FIXME: maintain true CDF counts. */ + + /* Values are 0=BLOCK_ZERO 1=ZERO_TOKEN, 2=ONE_TOKEN_EOB + 3=ONE_TOKEN_NEOB, 4=TWO_TOKEN_PLUS_EOB, 5=TWO_TOKEN_PLUS_NEOB + */ + // Block zero probability + if (is_dc) { + phead[0] = + ((*blockz_model) << (CDF_PROB_BITS - 8)) + (1 << (CDF_PROB_BITS - 9)); + phead[0] = AOMMIN(CDF_PROB_TOP - (HEAD_TOKENS + 1), AOMMAX(1, phead[0])); + } + + // Will scale the remaining probabilities by the probability of the block + // being non-zero + prob8_blocknz = is_dc ? (256 - *blockz_model) : 256; + + // Probability of zero + phead[is_dc + ZERO_TOKEN] = + (pdf_model[1] << (CDF_PROB_BITS - 8)) + (1 << (CDF_PROB_BITS - 9)); + + // Will scale the non-zero values + prob_NZ = CDF_PROB_TOP - phead[is_dc + ZERO_TOKEN]; + + // Will scale the EOBs by the probability of and EOB_TOKEN .. + prob_EOB_1 = + (pdf_model[0] << (CDF_PROB_BITS - 8)) + (1 << (CDF_PROB_BITS - 9)); + // .. use a lower probability of EOB for larger values + prob_EOB_2p = prob_EOB_1 / 2; + + prob_NEOB_1 = CDF_PROB_TOP - prob_EOB_1; + prob_NEOB_2p = CDF_PROB_TOP - prob_EOB_2p; + if (prob_NZ == 0 || prob_NZ == CDF_PROB_TOP) abort(); + if (prob_EOB_1 == 0 || prob_EOB_1 == CDF_PROB_TOP) abort(); + if (prob_EOB_2p == 0 || prob_EOB_2p == CDF_PROB_TOP) abort(); + + // ONE_CONTEXT_NODE prob + p = (pdf_model[2] << (CDF_PROB_BITS - 8)) + (1 << (CDF_PROB_BITS - 9)); + // Scale by the non-zero factor to get the probability of token = 1 + p1 = ROUND_POWER_OF_TWO(prob_NZ * p, 15); + + // Scale by the EOB factors + phead[is_dc + ONE_TOKEN_EOB] = ROUND_POWER_OF_TWO(p1 * prob_EOB_1, 15); + phead[is_dc + ONE_TOKEN_NEOB] = ROUND_POWER_OF_TWO(p1 * prob_NEOB_1, 15); + + // Probability token is 2 or more + p2 = CDF_PROB_TOP - p1 - phead[is_dc + ZERO_TOKEN]; + + phead[is_dc + TWO_TOKEN_PLUS_EOB] = ROUND_POWER_OF_TWO(p2 * prob_EOB_2p, 15); + phead[is_dc + TWO_TOKEN_PLUS_NEOB] = + ROUND_POWER_OF_TWO(p2 * prob_NEOB_2p, 15); + + // Now use block non-zerp prob to scale the values + for (i = is_dc; i < last_head_val; ++i) { + phead[i] = (prob8_blocknz * phead[i] + 128) >> 8; + } + + for (i = 0; i < last_head_val; ++i) { + int c0; + c0 = i > 0 ? AOM_ICDF(cdf_head[i - 1]) : 0; + p = AOMMAX(1, AOMMIN(CDF_PROB_TOP - (last_head_val - i) - c0, phead[i])); + cdf_head[i] = AOM_ICDF(c0 + p); + } + cdf_head[last_head_val] = AOM_ICDF(CDF_PROB_TOP); +} + +static void av1_default_coef_cdfs(FRAME_CONTEXT *fc) { + int i, j, k, l; + for (i = 0; i < PLANE_TYPES; ++i) + for (j = 0; j < REF_TYPES; ++j) + for (k = 0; k < COEF_BANDS; ++k) + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { +#if CONFIG_CB4X4 + av1_copy(fc->coef_head_cdfs[TX_2X2][i][j][k][l], + default_coef_head_cdf_4x4[i][j][k][l]); +#endif + av1_copy(fc->coef_head_cdfs[TX_4X4][i][j][k][l], + default_coef_head_cdf_4x4[i][j][k][l]); + av1_copy(fc->coef_head_cdfs[TX_8X8][i][j][k][l], + default_coef_head_cdf_8x8[i][j][k][l]); + av1_copy(fc->coef_head_cdfs[TX_16X16][i][j][k][l], + default_coef_head_cdf_16x16[i][j][k][l]); + av1_copy(fc->coef_head_cdfs[TX_32X32][i][j][k][l], + default_coef_head_cdf_32x32[i][j][k][l]); +#if CONFIG_TX64X64 + av1_copy(fc->coef_head_cdfs[TX_64X64][i][j][k][l], + default_coef_head_cdf_32x32[i][j][k][l]); +#endif + } +} + +void av1_coef_head_cdfs(FRAME_CONTEXT *fc) { + TX_SIZE t; + int i, j, k, l; + for (t = 0; t < TX_SIZES; ++t) + for (i = 0; i < PLANE_TYPES; ++i) + for (j = 0; j < REF_TYPES; ++j) + for (k = 0; k < COEF_BANDS; ++k) + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { + build_head_cdfs(fc->coef_probs[t][i][j][k][l], + k == 0 ? &fc->blockzero_probs[t][i][j][l] : NULL, + fc->coef_head_cdfs[t][i][j][k][l]); + } +} + +#elif CONFIG_EC_MULTISYMBOL +static void build_token_cdfs(const aom_prob *pdf_model, + aom_cdf_prob cdf[ENTROPY_TOKENS + 1]) { + int i, sum = 0; + assert(pdf_model[2] != 0); + for (i = 0; i < ENTROPY_TOKENS - 2; ++i) { + sum += av1_pareto8_token_probs[pdf_model[2] - 1][i]; + cdf[i] = AOM_ICDF(sum); + } +} +#endif // CONFIG_NEW_TOKENSET + +#if CONFIG_EC_MULTISYMBOL +void av1_coef_pareto_cdfs(FRAME_CONTEXT *fc) { + /* Build the tail based on a Pareto distribution */ + TX_SIZE t; + int i, j, k, l; + for (t = 0; t < TX_SIZES; ++t) + for (i = 0; i < PLANE_TYPES; ++i) + for (j = 0; j < REF_TYPES; ++j) + for (k = 0; k < COEF_BANDS; ++k) + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) +#if CONFIG_NEW_TOKENSET + build_tail_cdfs(fc->coef_tail_cdfs[t][i][j][k][l], + fc->coef_head_cdfs[t][i][j][k][l], k == 0); +#else + build_token_cdfs(fc->coef_probs[t][i][j][k][l], + fc->coef_cdfs[t][i][j][k][l]); +#endif +} +#endif + +void av1_default_coef_probs(AV1_COMMON *cm) { +#if CONFIG_Q_ADAPT_PROBS + const int index = AOMMIN( + ROUND_POWER_OF_TWO(cm->base_qindex, 8 - QCTX_BIN_BITS), QCTX_BINS - 1); + av1_copy(cm->fc->coef_probs, default_qctx_coef_probs[index]); +#else +#if CONFIG_CB4X4 + av1_copy(cm->fc->coef_probs[TX_2X2], default_coef_probs_4x4); +#endif + av1_copy(cm->fc->coef_probs[TX_4X4], default_coef_probs_4x4); + av1_copy(cm->fc->coef_probs[TX_8X8], default_coef_probs_8x8); + av1_copy(cm->fc->coef_probs[TX_16X16], default_coef_probs_16x16); + av1_copy(cm->fc->coef_probs[TX_32X32], default_coef_probs_32x32); +#if CONFIG_TX64X64 + av1_copy(cm->fc->coef_probs[TX_64X64], default_coef_probs_64x64); +#endif // CONFIG_TX64X64 +#endif // CONFIG_Q_ADAPT_PROBS +#if CONFIG_NEW_TOKENSET + av1_copy(cm->fc->blockzero_probs, av1_default_blockzero_probs); +#endif +#if CONFIG_NEW_TOKENSET + /* Load the head tokens */ + av1_default_coef_cdfs(cm->fc); +#endif +#if CONFIG_EC_MULTISYMBOL + av1_coef_pareto_cdfs(cm->fc); +#endif // CONFIG_EC_MULTISYMBOL +} + +#if !CONFIG_LV_MAP +static void adapt_coef_probs(AV1_COMMON *cm, TX_SIZE tx_size, + unsigned int count_sat, + unsigned int update_factor) { + const FRAME_CONTEXT *pre_fc = &cm->frame_contexts[cm->frame_context_idx]; + av1_coeff_probs_model *const probs = cm->fc->coef_probs[tx_size]; +#if CONFIG_SUBFRAME_PROB_UPDATE + const av1_coeff_probs_model *const pre_probs = + cm->partial_prob_update + ? (const av1_coeff_probs_model *)cm->starting_coef_probs[tx_size] + : pre_fc->coef_probs[tx_size]; +#else + const av1_coeff_probs_model *const pre_probs = pre_fc->coef_probs[tx_size]; +#endif // CONFIG_SUBFRAME_PROB_UPDATE + const av1_coeff_count_model *const counts = + (const av1_coeff_count_model *)cm->counts.coef[tx_size]; + const unsigned int(*eob_counts)[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS] = + (const unsigned int(*)[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS]) + cm->counts.eob_branch[tx_size]; +#if CONFIG_NEW_TOKENSET + const av1_blockz_probs_model *const pre_blockz_probs = + pre_fc->blockzero_probs[tx_size]; + av1_blockz_probs_model *const blockz_probs = cm->fc->blockzero_probs[tx_size]; + const av1_blockz_count_model *const blockz_counts = + (const av1_blockz_count_model *)&cm->counts.blockz_count[tx_size][0]; +#endif + int i, j, k, l, m; +#if CONFIG_RECT_TX + assert(!is_rect_tx(tx_size)); +#endif // CONFIG_RECT_TX + + for (i = 0; i < PLANE_TYPES; ++i) + for (j = 0; j < REF_TYPES; ++j) + for (k = 0; k < COEF_BANDS; ++k) + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { + const int n0 = counts[i][j][k][l][ZERO_TOKEN]; + const int n1 = counts[i][j][k][l][ONE_TOKEN]; + const int n2 = counts[i][j][k][l][TWO_TOKEN]; + const int neob = counts[i][j][k][l][EOB_MODEL_TOKEN]; + const unsigned int branch_ct[UNCONSTRAINED_NODES][2] = { + { neob, eob_counts[i][j][k][l] - neob }, { n0, n1 + n2 }, { n1, n2 } + }; + for (m = 0; m < UNCONSTRAINED_NODES; ++m) + probs[i][j][k][l][m] = + av1_merge_probs(pre_probs[i][j][k][l][m], branch_ct[m], + count_sat, update_factor); + } + +#if CONFIG_NEW_TOKENSET + for (i = 0; i < PLANE_TYPES; ++i) { + for (j = 0; j < REF_TYPES; ++j) { + for (k = 0; k < BLOCKZ_CONTEXTS; ++k) { + const int n0 = blockz_counts[i][j][k][0]; + const int n1 = blockz_counts[i][j][k][1]; + const unsigned int branch_ct[2] = { n0, n1 }; + blockz_probs[i][j][k] = av1_merge_probs( + pre_blockz_probs[i][j][k], branch_ct, count_sat, update_factor); + } + } + } +#endif +} +#endif // !CONFIG_LV_MAP + +void av1_adapt_coef_probs(AV1_COMMON *cm) { + unsigned int count_sat, update_factor; + + if (!frame_is_intra_only(cm) && cm->last_frame_type == KEY_FRAME) { + update_factor = COEF_MAX_UPDATE_FACTOR_AFTER_KEY; /* adapt quickly */ + count_sat = COEF_COUNT_SAT_AFTER_KEY; + } else { + update_factor = COEF_MAX_UPDATE_FACTOR; + count_sat = COEF_COUNT_SAT; + } +#if CONFIG_SUBFRAME_PROB_UPDATE + if (cm->partial_prob_update == 1) update_factor = COEF_MAX_UPDATE_FACTOR; +#endif // CONFIG_SUBFRAME_PROB_UPDATE + +#if CONFIG_LV_MAP + av1_adapt_txb_probs(cm, count_sat, update_factor); +#else + TX_SIZE tx_size; + for (tx_size = 0; tx_size < TX_SIZES; tx_size++) + adapt_coef_probs(cm, tx_size, count_sat, update_factor); +#endif +} + +#if CONFIG_SUBFRAME_PROB_UPDATE +void av1_partial_adapt_probs(AV1_COMMON *cm, int mi_row, int mi_col) { + (void)mi_row; + (void)mi_col; + + if (cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { + cm->partial_prob_update = 1; + av1_adapt_coef_probs(cm); + } +} +#endif // CONFIG_SUBFRAME_PROB_UPDATE + +#if CONFIG_EC_ADAPT +static void av1_average_cdf(aom_cdf_prob *cdf_ptr[], aom_cdf_prob *fc_cdf_ptr, + int cdf_size, const int num_tiles) { + int i; + for (i = 0; i < cdf_size;) { + do { + int sum = 0; + int j; + assert(i < cdf_size); + for (j = 0; j < num_tiles; ++j) sum += AOM_ICDF(cdf_ptr[j][i]); + fc_cdf_ptr[i] = AOM_ICDF(sum / num_tiles); + } while (fc_cdf_ptr[i++] != AOM_ICDF(CDF_PROB_TOP)); + // Zero symbol counts for the next frame + assert(i < cdf_size); + fc_cdf_ptr[i++] = 0; + // Skip trailing zeros until the start of the next CDF. + for (; i < cdf_size && fc_cdf_ptr[i] == 0; ++i) { + } + } +} + +#define AVERAGE_TILE_CDFS(cname) \ + for (i = 0; i < num_tiles; ++i) \ + cdf_ptr[i] = (aom_cdf_prob *)&ec_ctxs[i]->cname; \ + fc_cdf_ptr = (aom_cdf_prob *)&fc->cname; \ + cdf_size = (int)sizeof(fc->cname) / sizeof(aom_cdf_prob); \ + av1_average_cdf(cdf_ptr, fc_cdf_ptr, cdf_size, num_tiles); + +void av1_average_tile_coef_cdfs(FRAME_CONTEXT *fc, FRAME_CONTEXT *ec_ctxs[], + aom_cdf_prob *cdf_ptr[], const int num_tiles) { + int i, cdf_size; + + aom_cdf_prob *fc_cdf_ptr; + +#if CONFIG_NEW_TOKENSET + AVERAGE_TILE_CDFS(coef_head_cdfs) + AVERAGE_TILE_CDFS(coef_tail_cdfs) +#else + AVERAGE_TILE_CDFS(coef_cdfs) +#endif +} + +void av1_average_tile_mv_cdfs(FRAME_CONTEXT *fc, FRAME_CONTEXT *ec_ctxs[], + aom_cdf_prob *cdf_ptr[], const int num_tiles) { + int i, k, cdf_size; + + aom_cdf_prob *fc_cdf_ptr; + +#if CONFIG_REF_MV + int j; + for (j = 0; j < NMV_CONTEXTS; ++j) { + AVERAGE_TILE_CDFS(nmvc[j].joint_cdf) + + for (k = 0; k < 2; ++k) { + AVERAGE_TILE_CDFS(nmvc[j].comps[k].class_cdf); + AVERAGE_TILE_CDFS(nmvc[j].comps[k].class0_fp_cdf); + AVERAGE_TILE_CDFS(nmvc[j].comps[k].fp_cdf); + } + } +#else + AVERAGE_TILE_CDFS(nmvc.joint_cdf) + + for (k = 0; k < 2; ++k) { + AVERAGE_TILE_CDFS(nmvc.comps[k].class_cdf) + AVERAGE_TILE_CDFS(nmvc.comps[k].class0_fp_cdf) + AVERAGE_TILE_CDFS(nmvc.comps[k].fp_cdf) + } +#endif +} + +void av1_average_tile_intra_cdfs(FRAME_CONTEXT *fc, FRAME_CONTEXT *ec_ctxs[], + aom_cdf_prob *cdf_ptr[], const int num_tiles) { + int i, cdf_size; + + aom_cdf_prob *fc_cdf_ptr; + + AVERAGE_TILE_CDFS(tx_size_cdf); + +#if CONFIG_VAR_TX +// FIXME: txfm_partition probs +#endif + + // FIXME: skip probs + + AVERAGE_TILE_CDFS(intra_ext_tx_cdf) + AVERAGE_TILE_CDFS(inter_ext_tx_cdf); + + AVERAGE_TILE_CDFS(seg.tree_cdf) + AVERAGE_TILE_CDFS(uv_mode_cdf) + + AVERAGE_TILE_CDFS(partition_cdf) + +#if CONFIG_DELTA_Q + AVERAGE_TILE_CDFS(delta_q_cdf) +#if CONFIG_EXT_DELTA_Q + AVERAGE_TILE_CDFS(delta_lf_cdf) +#endif +#endif +#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP + AVERAGE_TILE_CDFS(intra_filter_cdf) +#endif // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP +#if CONFIG_FILTER_INTRA +#endif // CONFIG_FILTER_INTRA +} + +void av1_average_tile_inter_cdfs(AV1_COMMON *cm, FRAME_CONTEXT *fc, + FRAME_CONTEXT *ec_ctxs[], + aom_cdf_prob *cdf_ptr[], const int num_tiles) { + int i, cdf_size; + + aom_cdf_prob *fc_cdf_ptr; + +// FIXME: comp_inter_cdf not defined + +// FIXME: comp_ref_cdf and comp_bwd_ref not defined + +// FIXME: single_ref_cdf not defined + +#if CONFIG_REF_MV +// FIXME: cdfs not defined for newmv_mode, zeromv_mode, drl_mode, new2mv_mode +#else + AVERAGE_TILE_CDFS(inter_mode_cdf) +#endif + + // FIXME: cdfs not defined for motion_mode_prob, obmc_prob + + // FIXME: cdfs not defined for super_tx + + // FIXME: CONFIG_EXT_INTER cdfs not defined for inter_compound_mode, + // interintra_mode etc + + AVERAGE_TILE_CDFS(y_mode_cdf) + + if (cm->interp_filter == SWITCHABLE) { + AVERAGE_TILE_CDFS(switchable_interp_cdf) + } +} + +#if CONFIG_PVQ +// Averaging PVQ's expected values for symbol coding +static void av1_average_pvq_ex(int *cxt_ptr[], int *fc_cxt_ptr, int cxt_size, + const int num_tiles) { + int i, j; + for (i = 0; i < cxt_size; ++i) { + int sum = 0; + for (j = 0; j < num_tiles; ++j) sum += cxt_ptr[j][i]; + fc_cxt_ptr[i] = sum / num_tiles; + } +} + +#define AVERAGE_TILE_PVQ_EX(cname) \ + for (i = 0; i < num_tiles; ++i) cxt_ptr[i] = (int *)&ec_ctxs[i]->cname; \ + fc_cxt_ptr = (int *)&fc->cname; \ + cxt_size = (int)sizeof(fc->cname) / sizeof(int); \ + av1_average_pvq_ex(cxt_ptr, fc_cxt_ptr, cxt_size, num_tiles); + +void av1_default_pvq_probs(AV1_COMMON *cm) { + od_adapt_ctx *adapt = &cm->fc->pvq_context; + + // Init with flat probabilities. + od_adapt_ctx_reset(adapt, 0); + + // TODO(yushin): Prepare offline cdf and context table for PVQ, + // i.e. od_adapt_ctx, then load them from table, + // for example od_adapt_ctx default_pvq_context. + // Then do sth like this: + // av1_copy(cm->fc->pvq_context, default_pvq_context); +} + +void av1_average_tile_pvq_cdfs(FRAME_CONTEXT *fc, FRAME_CONTEXT *ec_ctxs[], + const int num_tiles) { + int i, j, cdf_size, cxt_size; + + aom_cdf_prob *cdf_ptr[MAX_TILE_ROWS * MAX_TILE_COLS]; + aom_cdf_prob *fc_cdf_ptr; + int *cxt_ptr[MAX_TILE_ROWS * MAX_TILE_COLS]; + int *fc_cxt_ptr; + + AVERAGE_TILE_PVQ_EX(pvq_context.ex_dc) + AVERAGE_TILE_PVQ_EX(pvq_context.ex_g) + + for (j = 0; j < OD_NPLANES_MAX; j++) { + AVERAGE_TILE_CDFS(pvq_context.model_dc[j].cdf) + } + + AVERAGE_TILE_CDFS(pvq_context.skip_cdf) + + AVERAGE_TILE_PVQ_EX(pvq_context.pvq.pvq_codeword_ctx.pvq_adapt) + AVERAGE_TILE_CDFS(pvq_context.pvq.pvq_codeword_ctx.pvq_k1_cdf) + AVERAGE_TILE_CDFS(pvq_context.pvq.pvq_codeword_ctx.pvq_split_cdf) + + for (j = 0; j < 3; j++) { + AVERAGE_TILE_CDFS(pvq_context.pvq.pvq_param_model[j].cdf) + } + + AVERAGE_TILE_PVQ_EX(pvq_context.pvq.pvq_ext) + AVERAGE_TILE_PVQ_EX(pvq_context.pvq.pvq_exg) + AVERAGE_TILE_CDFS(pvq_context.pvq.pvq_gaintheta_cdf) + AVERAGE_TILE_CDFS(pvq_context.pvq.pvq_skip_dir_cdf) +} +#endif // CONFIG_PVQ +#endif // CONFIG_EC_ADAPT diff --git a/third_party/aom/av1/common/entropy.h b/third_party/aom/av1/common/entropy.h new file mode 100644 index 0000000000..b02d41bff1 --- /dev/null +++ b/third_party/aom/av1/common/entropy.h @@ -0,0 +1,428 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_ENTROPY_H_ +#define AV1_COMMON_ENTROPY_H_ + +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/prob.h" + +#include "av1/common/common.h" +#include "av1/common/common_data.h" +#include "av1/common/enums.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define DIFF_UPDATE_PROB 252 +#define GROUP_DIFF_UPDATE_PROB 252 + +#if CONFIG_Q_ADAPT_PROBS +#define QCTX_BIN_BITS 2 +#define QCTX_BINS (1 << QCTX_BIN_BITS) +#endif // CONFIG_Q_ADAPT_PROBS + +#if CONFIG_SUBFRAME_PROB_UPDATE +#define COEF_PROBS_BUFS 16 +#endif // CONFIG_SUBFRAME_PROB_UPDATE + +// Coefficient token alphabet +#define ZERO_TOKEN 0 // 0 Extra Bits 0+0 +#define ONE_TOKEN 1 // 1 Extra Bits 0+1 +#define TWO_TOKEN 2 // 2 Extra Bits 0+1 +#define THREE_TOKEN 3 // 3 Extra Bits 0+1 +#define FOUR_TOKEN 4 // 4 Extra Bits 0+1 +#define CATEGORY1_TOKEN 5 // 5-6 Extra Bits 1+1 +#define CATEGORY2_TOKEN 6 // 7-10 Extra Bits 2+1 +#define CATEGORY3_TOKEN 7 // 11-18 Extra Bits 3+1 +#define CATEGORY4_TOKEN 8 // 19-34 Extra Bits 4+1 +#define CATEGORY5_TOKEN 9 // 35-66 Extra Bits 5+1 +#define CATEGORY6_TOKEN 10 // 67+ Extra Bits 14+1 +#define EOB_TOKEN 11 // EOB Extra Bits 0+0 +#if CONFIG_NEW_TOKENSET +#define NO_EOB 0 // Not an end-of-block +#define EARLY_EOB 1 // End of block before the last position +#define LAST_EOB 2 // End of block in the last position (implicit) +#define BLOCK_Z_TOKEN 255 // block zero +#define HEAD_TOKENS 5 +#define TAIL_TOKENS 9 +#define ONE_TOKEN_EOB 1 +#define ONE_TOKEN_NEOB 2 +#define TWO_TOKEN_PLUS_EOB 3 +#define TWO_TOKEN_PLUS_NEOB 4 +#endif +#define ENTROPY_TOKENS 12 + +#define ENTROPY_NODES 11 + +#if CONFIG_LV_MAP +#define TXB_SKIP_CONTEXTS 13 +#define SIG_COEF_CONTEXTS 20 +#define EOB_COEF_CONTEXTS 25 +#define COEFF_BASE_CONTEXTS 42 +#define DC_SIGN_CONTEXTS 3 + +#define BR_TMP_OFFSET 12 +#define BR_REF_CAT 4 +#define LEVEL_CONTEXTS (BR_TMP_OFFSET * BR_REF_CAT) + +#define NUM_BASE_LEVELS 2 +#define COEFF_BASE_RANGE (15 - NUM_BASE_LEVELS) + +#define COEFF_CONTEXT_BITS 6 +#define COEFF_CONTEXT_MASK ((1 << COEFF_CONTEXT_BITS) - 1) +#endif + +DECLARE_ALIGNED(16, extern const uint8_t, av1_pt_energy_class[ENTROPY_TOKENS]); + +#define CAT1_MIN_VAL 5 +#define CAT2_MIN_VAL 7 +#define CAT3_MIN_VAL 11 +#define CAT4_MIN_VAL 19 +#define CAT5_MIN_VAL 35 +#define CAT6_MIN_VAL 67 + +// Extra bit probabilities. +DECLARE_ALIGNED(16, extern const uint8_t, av1_cat1_prob[1]); +DECLARE_ALIGNED(16, extern const uint8_t, av1_cat2_prob[2]); +DECLARE_ALIGNED(16, extern const uint8_t, av1_cat3_prob[3]); +DECLARE_ALIGNED(16, extern const uint8_t, av1_cat4_prob[4]); +DECLARE_ALIGNED(16, extern const uint8_t, av1_cat5_prob[5]); +DECLARE_ALIGNED(16, extern const uint8_t, av1_cat6_prob[18]); +#if CONFIG_NEW_MULTISYMBOL +extern const aom_cdf_prob *av1_cat1_cdf[]; +extern const aom_cdf_prob *av1_cat2_cdf[]; +extern const aom_cdf_prob *av1_cat3_cdf[]; +extern const aom_cdf_prob *av1_cat4_cdf[]; +extern const aom_cdf_prob *av1_cat5_cdf[]; +extern const aom_cdf_prob *av1_cat6_cdf[]; +#endif + +#define EOB_MODEL_TOKEN 3 + +typedef struct { +#if CONFIG_NEW_MULTISYMBOL + const aom_cdf_prob **cdf; +#else + const aom_prob *prob; +#endif + int len; + int base_val; + const int16_t *cost; +} av1_extra_bit; + +// indexed by token value +extern const av1_extra_bit av1_extra_bits[ENTROPY_TOKENS]; + +static INLINE int av1_get_cat6_extrabits_size(TX_SIZE tx_size, + aom_bit_depth_t bit_depth) { + tx_size = txsize_sqr_up_map[tx_size]; +#if CONFIG_TX64X64 + // TODO(debargha): Does TX_64X64 require an additional extrabit? + if (tx_size > TX_32X32) tx_size = TX_32X32; +#endif +#if CONFIG_CB4X4 + int tx_offset = (tx_size < TX_4X4) ? 0 : (int)(tx_size - TX_4X4); +#else + int tx_offset = (int)(tx_size - TX_4X4); +#endif + int bits = (int)bit_depth + 3 + tx_offset; +#if CONFIG_NEW_MULTISYMBOL + // Round up + bits = AOMMIN((int)sizeof(av1_cat6_prob), ((bits + 3) & ~3)); +#endif + assert(bits <= (int)sizeof(av1_cat6_prob)); + return bits; +} + +#define DCT_MAX_VALUE 16384 +#if CONFIG_HIGHBITDEPTH +#define DCT_MAX_VALUE_HIGH10 65536 +#define DCT_MAX_VALUE_HIGH12 262144 +#endif // CONFIG_HIGHBITDEPTH + +/* Coefficients are predicted via a 3-dimensional probability table. */ + +#define REF_TYPES 2 // intra=0, inter=1 + +/* Middle dimension reflects the coefficient position within the transform. */ +#define COEF_BANDS 6 + +/* Inside dimension is measure of nearby complexity, that reflects the energy + of nearby coefficients are nonzero. For the first coefficient (DC, unless + block type is 0), we look at the (already encoded) blocks above and to the + left of the current block. The context index is then the number (0,1,or 2) + of these blocks having nonzero coefficients. + After decoding a coefficient, the measure is determined by the size of the + most recently decoded coefficient. + Note that the intuitive meaning of this measure changes as coefficients + are decoded, e.g., prior to the first token, a zero means that my neighbors + are empty while, after the first token, because of the use of end-of-block, + a zero means we just decoded a zero and hence guarantees that a non-zero + coefficient will appear later in this block. However, this shift + in meaning is perfectly OK because our context depends also on the + coefficient band (and since zigzag positions 0, 1, and 2 are in + distinct bands). */ + +#define COEFF_CONTEXTS 6 +#if CONFIG_EC_MULTISYMBOL +#define BLOCKZ_CONTEXTS 3 +#endif +#define COEFF_CONTEXTS0 3 // for band 0 +#define BAND_COEFF_CONTEXTS(band) \ + ((band) == 0 ? COEFF_CONTEXTS0 : COEFF_CONTEXTS) + +// #define ENTROPY_STATS + +typedef unsigned int av1_coeff_count[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS] + [ENTROPY_TOKENS]; +typedef unsigned int av1_coeff_stats[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS] + [ENTROPY_NODES][2]; + +#define SUBEXP_PARAM 4 /* Subexponential code parameter */ +#define MODULUS_PARAM 13 /* Modulus parameter */ + +struct AV1Common; +struct frame_contexts; +void av1_default_coef_probs(struct AV1Common *cm); +void av1_adapt_coef_probs(struct AV1Common *cm); +#if CONFIG_EC_ADAPT +void av1_adapt_coef_cdfs(struct AV1Common *cm, struct frame_contexts *pre_fc); +#endif +#if CONFIG_SUBFRAME_PROB_UPDATE +void av1_partial_adapt_probs(struct AV1Common *cm, int mi_row, int mi_col); +#endif // CONFIG_SUBFRAME_PROB_UPDATE + +// This is the index in the scan order beyond which all coefficients for +// 8x8 transform and above are in the top band. +// This macro is currently unused but may be used by certain implementations +#define MAXBAND_INDEX 21 + +DECLARE_ALIGNED(16, extern const uint8_t, + av1_coefband_trans_8x8plus[MAX_TX_SQUARE]); +DECLARE_ALIGNED(16, extern const uint8_t, av1_coefband_trans_4x8_8x4[32]); +DECLARE_ALIGNED(16, extern const uint8_t, av1_coefband_trans_4x4[16]); + +DECLARE_ALIGNED(16, extern const uint16_t, band_count_table[TX_SIZES_ALL][8]); +DECLARE_ALIGNED(16, extern const uint16_t, + band_cum_count_table[TX_SIZES_ALL][8]); + +static INLINE const uint8_t *get_band_translate(TX_SIZE tx_size) { + switch (tx_size) { + case TX_4X4: return av1_coefband_trans_4x4; + case TX_8X4: + case TX_4X8: return av1_coefband_trans_4x8_8x4; + default: return av1_coefband_trans_8x8plus; + } +} + +// 128 lists of probabilities are stored for the following ONE node probs: +// 1, 3, 5, 7, ..., 253, 255 +// In between probabilities are interpolated linearly + +#define COEFF_PROB_MODELS 255 + +#define UNCONSTRAINED_NODES 3 + +#define PIVOT_NODE 2 // which node is pivot + +#define MODEL_NODES (ENTROPY_NODES - UNCONSTRAINED_NODES) +#define TAIL_NODES (MODEL_NODES + 1) +extern const aom_tree_index av1_coef_con_tree[TREE_SIZE(ENTROPY_TOKENS)]; +extern const aom_prob av1_pareto8_full[COEFF_PROB_MODELS][MODEL_NODES]; + +typedef aom_prob av1_coeff_probs_model[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS] + [UNCONSTRAINED_NODES]; + +typedef unsigned int av1_coeff_count_model[REF_TYPES][COEF_BANDS] + [COEFF_CONTEXTS] + [UNCONSTRAINED_NODES + 1]; + +void av1_model_to_full_probs(const aom_prob *model, aom_prob *full); + +#if CONFIG_EC_MULTISYMBOL +typedef aom_cdf_prob coeff_cdf_model[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS] + [CDF_SIZE(ENTROPY_TOKENS)]; +typedef aom_prob av1_blockz_probs_model[REF_TYPES][BLOCKZ_CONTEXTS]; +typedef unsigned int av1_blockz_count_model[REF_TYPES][BLOCKZ_CONTEXTS][2]; +extern const aom_cdf_prob av1_pareto8_token_probs[COEFF_PROB_MODELS] + [ENTROPY_TOKENS - 2]; +extern const aom_cdf_prob av1_pareto8_tail_probs[COEFF_PROB_MODELS] + [ENTROPY_TOKENS - 3]; +struct frame_contexts; +#if CONFIG_NEW_TOKENSET +void av1_coef_head_cdfs(struct frame_contexts *fc); +#endif +void av1_coef_pareto_cdfs(struct frame_contexts *fc); +#endif // CONFIG_EC_MULTISYMBOL + +typedef char ENTROPY_CONTEXT; + +static INLINE int combine_entropy_contexts(ENTROPY_CONTEXT a, + ENTROPY_CONTEXT b) { + return (a != 0) + (b != 0); +} + +static INLINE int get_entropy_context(TX_SIZE tx_size, const ENTROPY_CONTEXT *a, + const ENTROPY_CONTEXT *l) { + ENTROPY_CONTEXT above_ec = 0, left_ec = 0; + +#if CONFIG_CB4X4 + switch (tx_size) { + case TX_2X2: + above_ec = a[0] != 0; + left_ec = l[0] != 0; + break; + case TX_4X4: + above_ec = !!*(const uint16_t *)a; + left_ec = !!*(const uint16_t *)l; + break; + case TX_4X8: + above_ec = !!*(const uint16_t *)a; + left_ec = !!*(const uint32_t *)l; + break; + case TX_8X4: + above_ec = !!*(const uint32_t *)a; + left_ec = !!*(const uint16_t *)l; + break; + case TX_8X8: + above_ec = !!*(const uint32_t *)a; + left_ec = !!*(const uint32_t *)l; + break; + case TX_8X16: + above_ec = !!*(const uint32_t *)a; + left_ec = !!*(const uint64_t *)l; + break; + case TX_16X8: + above_ec = !!*(const uint64_t *)a; + left_ec = !!*(const uint32_t *)l; + break; + case TX_16X16: + above_ec = !!*(const uint64_t *)a; + left_ec = !!*(const uint64_t *)l; + break; + case TX_16X32: + above_ec = !!*(const uint64_t *)a; + left_ec = !!(*(const uint64_t *)l | *(const uint64_t *)(l + 8)); + break; + case TX_32X16: + above_ec = !!(*(const uint64_t *)a | *(const uint64_t *)(a + 8)); + left_ec = !!*(const uint64_t *)l; + break; + case TX_32X32: + above_ec = !!(*(const uint64_t *)a | *(const uint64_t *)(a + 8)); + left_ec = !!(*(const uint64_t *)l | *(const uint64_t *)(l + 8)); + break; + default: assert(0 && "Invalid transform size."); break; + } + return combine_entropy_contexts(above_ec, left_ec); +#endif + + switch (tx_size) { + case TX_4X4: + above_ec = a[0] != 0; + left_ec = l[0] != 0; + break; + case TX_4X8: + above_ec = a[0] != 0; + left_ec = !!*(const uint16_t *)l; + break; + case TX_8X4: + above_ec = !!*(const uint16_t *)a; + left_ec = l[0] != 0; + break; + case TX_8X16: + above_ec = !!*(const uint16_t *)a; + left_ec = !!*(const uint32_t *)l; + break; + case TX_16X8: + above_ec = !!*(const uint32_t *)a; + left_ec = !!*(const uint16_t *)l; + break; + case TX_16X32: + above_ec = !!*(const uint32_t *)a; + left_ec = !!*(const uint64_t *)l; + break; + case TX_32X16: + above_ec = !!*(const uint64_t *)a; + left_ec = !!*(const uint32_t *)l; + break; + case TX_8X8: + above_ec = !!*(const uint16_t *)a; + left_ec = !!*(const uint16_t *)l; + break; + case TX_16X16: + above_ec = !!*(const uint32_t *)a; + left_ec = !!*(const uint32_t *)l; + break; + case TX_32X32: + above_ec = !!*(const uint64_t *)a; + left_ec = !!*(const uint64_t *)l; + break; +#if CONFIG_TX64X64 + case TX_64X64: + above_ec = !!(*(const uint64_t *)a | *(const uint64_t *)(a + 8)); + left_ec = !!(*(const uint64_t *)l | *(const uint64_t *)(l + 8)); + break; +#endif // CONFIG_TX64X64 + default: assert(0 && "Invalid transform size."); break; + } + return combine_entropy_contexts(above_ec, left_ec); +} + +#define COEF_COUNT_SAT 24 +#define COEF_MAX_UPDATE_FACTOR 112 +#define COEF_COUNT_SAT_AFTER_KEY 24 +#define COEF_MAX_UPDATE_FACTOR_AFTER_KEY 128 + +#if CONFIG_ADAPT_SCAN +#define ADAPT_SCAN_UPDATE_RATE_16 (1 << 13) +#endif + +static INLINE aom_prob av1_merge_probs(aom_prob pre_prob, + const unsigned int ct[2], + unsigned int count_sat, + unsigned int max_update_factor) { + return merge_probs(pre_prob, ct, count_sat, max_update_factor); +} + +static INLINE aom_prob av1_mode_mv_merge_probs(aom_prob pre_prob, + const unsigned int ct[2]) { + return mode_mv_merge_probs(pre_prob, ct); +} + +#if CONFIG_EC_ADAPT +void av1_average_tile_coef_cdfs(struct frame_contexts *fc, + struct frame_contexts *ec_ctxs[], + aom_cdf_prob *cdf_ptrs[], int num_tiles); +void av1_average_tile_mv_cdfs(struct frame_contexts *fc, + struct frame_contexts *ec_ctxs[], + aom_cdf_prob *cdf_ptrs[], int num_tiles); +void av1_average_tile_intra_cdfs(struct frame_contexts *fc, + struct frame_contexts *ec_ctxs[], + aom_cdf_prob *cdf_ptrs[], int num_tiles); +void av1_average_tile_inter_cdfs(struct AV1Common *cm, + struct frame_contexts *fc, + struct frame_contexts *ec_ctxs[], + aom_cdf_prob *cdf_ptrs[], int num_tiles); +#if CONFIG_PVQ +void av1_default_pvq_probs(struct AV1Common *cm); +void av1_average_tile_pvq_cdfs(struct frame_contexts *fc, + struct frame_contexts *ec_ctxs[], int num_tiles); +#endif // CONFIG_PVQ +#endif // CONFIG_EC_ADAPT +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_ENTROPY_H_ diff --git a/third_party/aom/av1/common/entropymode.c b/third_party/aom/av1/common/entropymode.c new file mode 100644 index 0000000000..0fcf762d16 --- /dev/null +++ b/third_party/aom/av1/common/entropymode.c @@ -0,0 +1,3792 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_mem/aom_mem.h" + +#include "av1/common/reconinter.h" +#include "av1/common/scan.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/seg_common.h" + +#if CONFIG_LV_MAP +const aom_prob default_txb_skip[TX_SIZES][TXB_SKIP_CONTEXTS] = { +#if CONFIG_CB4X4 + { 252, 71, 126, 184, 178, 218, 251, 49, 133, 221, 27, 92, 197 }, +#endif + { 252, 71, 126, 184, 178, 218, 251, 49, 133, 221, 27, 92, 197 }, + { 252, 71, 126, 184, 178, 218, 251, 49, 133, 221, 27, 92, 197 }, + { 252, 71, 126, 184, 178, 218, 251, 49, 133, 221, 27, 92, 197 }, + { 252, 71, 126, 184, 178, 218, 251, 49, 133, 221, 27, 92, 197 }, +}; +const aom_prob default_dc_sign[PLANE_TYPES][DC_SIGN_CONTEXTS] = { + { 125, 102, 147 }, { 119, 101, 135 }, +}; + +const aom_prob default_coeff_base + [TX_SIZES][PLANE_TYPES][NUM_BASE_LEVELS][COEFF_BASE_CONTEXTS] = { +#if CONFIG_CB4X4 + { // TX_2X2 + { + { 73, 128, 131, 204, 165, 226, 169, 236, 18, 128, 51, + 153, 97, 179, 123, 201, 145, 226, 20, 128, 59, 153, + 107, 181, 129, 201, 142, 226, 3, 128, 19, 99, 46, + 135, 92, 166, 129, 190, 157, 217, 128, 128 }, + + { 128, 128, 178, 218, 192, 236, 186, 243, 55, 128, 110, + 183, 151, 205, 168, 221, 180, 238, 65, 128, 116, 178, + 157, 206, 172, 222, 183, 238, 24, 128, 65, 127, 104, + 164, 154, 195, 187, 216, 205, 230, 128, 128 }, + }, + { + { 73, 128, 131, 204, 165, 226, 169, 236, 18, 128, 51, + 153, 97, 179, 123, 201, 145, 226, 20, 128, 59, 153, + 107, 181, 129, 201, 142, 226, 3, 128, 19, 99, 46, + 135, 92, 166, 129, 190, 157, 217, 128, 128 }, + + { 128, 128, 178, 218, 192, 236, 186, 243, 55, 128, 110, + 183, 151, 205, 168, 221, 180, 238, 65, 128, 116, 178, + 157, 206, 172, 222, 183, 238, 24, 128, 65, 127, 104, + 164, 154, 195, 187, 216, 205, 230, 128, 128 }, + } }, +#endif + { // TX_4X4 + { + // PLANE_Y + { 73, 128, 131, 204, 165, 226, 169, 236, 18, 128, 51, + 153, 97, 179, 123, 201, 145, 226, 20, 128, 59, 153, + 107, 181, 129, 201, 142, 226, 3, 128, 19, 99, 46, + 135, 92, 166, 129, 190, 157, 217, 128, 128 }, + + { 128, 128, 178, 218, 192, 236, 186, 243, 55, 128, 110, + 183, 151, 205, 168, 221, 180, 238, 65, 128, 116, 178, + 157, 206, 172, 222, 183, 238, 24, 128, 65, 127, 104, + 164, 154, 195, 187, 216, 205, 230, 128, 128 }, + }, + { + // PLANE_UV + { 47, 128, 100, 176, 140, 207, 150, 223, 11, 128, 35, + 133, 79, 165, 115, 186, 129, 210, 8, 128, 30, 114, + 80, 159, 116, 187, 146, 214, 2, 128, 9, 59, 28, + 86, 71, 131, 117, 165, 149, 188, 128, 128 }, + + { 83, 128, 152, 205, 168, 227, 192, 238, 42, 128, 92, + 169, 138, 193, 165, 209, 128, 206, 36, 128, 86, 159, + 141, 198, 181, 213, 102, 223, 18, 128, 50, 132, 90, + 144, 141, 169, 180, 191, 128, 217, 128, 128 }, + } }, + { + // TX_8X8 + { + // PLANE_Y + { 82, 128, 143, 203, 177, 225, 186, 237, 7, 128, 37, + 109, 78, 151, 110, 182, 139, 213, 25, 128, 51, 115, + 86, 146, 111, 175, 125, 205, 3, 128, 12, 55, 32, + 78, 63, 111, 96, 148, 123, 185, 146, 206 }, + + { 136, 128, 182, 220, 201, 236, 205, 243, 46, 128, 101, + 164, 147, 194, 170, 218, 177, 234, 62, 128, 104, 146, + 143, 183, 165, 207, 183, 228, 30, 128, 60, 95, 95, + 128, 135, 163, 166, 196, 175, 219, 192, 231 }, + }, + { + // PLANE_UV + { 47, 128, 112, 189, 164, 202, 163, 218, 8, 128, 32, + 110, 68, 151, 102, 179, 134, 195, 5, 128, 22, 76, + 54, 103, 80, 146, 101, 182, 1, 128, 5, 39, 17, + 53, 46, 93, 79, 127, 112, 161, 64, 195 }, + + { 90, 128, 156, 210, 183, 225, 128, 236, 39, 128, 98, + 164, 146, 201, 209, 219, 171, 208, 32, 128, 68, 123, + 119, 169, 154, 184, 128, 213, 15, 128, 38, 111, 83, + 112, 120, 163, 180, 170, 154, 213, 128, 205 }, + }, + }, + + { + // TX_16X16 + { + // PLANE_Y + { 96, 128, 169, 218, 208, 233, 187, 244, 10, 128, 34, + 101, 82, 153, 113, 184, 137, 212, 6, 128, 34, 104, + 81, 145, 109, 176, 147, 202, 1, 128, 3, 43, 15, + 53, 43, 89, 79, 129, 108, 168, 110, 194 }, + + { 156, 128, 206, 232, 218, 240, 128, 251, 39, 128, 108, + 161, 156, 202, 187, 216, 179, 234, 40, 128, 103, 152, + 144, 185, 159, 208, 205, 227, 14, 128, 39, 84, 76, + 110, 121, 151, 157, 187, 201, 206, 64, 216 }, + }, + { + // PLANE_UV + { 42, 128, 139, 211, 180, 230, 199, 238, 3, 128, 32, + 96, 69, 145, 102, 186, 117, 212, 4, 128, 25, 72, + 55, 111, 81, 159, 116, 198, 1, 128, 4, 22, 16, + 34, 35, 68, 63, 116, 89, 165, 102, 199 }, + + { 135, 128, 193, 227, 182, 239, 128, 246, 42, 128, 115, + 156, 146, 203, 188, 216, 128, 229, 32, 128, 82, 127, + 120, 178, 165, 203, 213, 229, 11, 128, 32, 73, 79, + 111, 129, 158, 162, 187, 156, 209, 85, 222 }, + }, + }, + + { + // TX_32X32 + { + // PLANE_Y + { 97, 128, 163, 232, 191, 246, 219, 252, 3, 128, 41, + 108, 91, 147, 104, 183, 118, 225, 6, 128, 45, 91, + 83, 125, 92, 160, 99, 215, 1, 128, 11, 36, 28, + 46, 43, 59, 57, 86, 73, 145, 91, 210 }, + + { 127, 128, 201, 239, 247, 248, 128, 254, 40, 128, 103, + 152, 158, 199, 186, 225, 181, 242, 38, 128, 92, 112, + 146, 189, 162, 217, 112, 239, 17, 128, 30, 47, 63, + 89, 113, 146, 147, 187, 168, 217, 150, 233 }, + }, + { + // PLANE_UV + { 65, 128, 155, 223, 166, 235, 154, 244, 15, 128, 57, + 154, 110, 199, 159, 224, 149, 239, 9, 128, 57, 140, + 97, 185, 148, 218, 176, 236, 1, 128, 3, 43, 19, + 42, 64, 98, 117, 167, 154, 199, 128, 158 }, + + { 130, 128, 189, 231, 171, 247, 128, 246, 63, 128, 132, + 222, 186, 224, 199, 244, 128, 247, 55, 128, 113, 211, + 164, 230, 225, 243, 128, 239, 7, 128, 31, 102, 106, + 138, 147, 183, 171, 223, 171, 224, 128, 128 }, + }, + }, + }; + +const aom_prob default_nz_map[TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS] = { +#if CONFIG_CB4X4 + { + { 34, 103, 61, 106, 62, 160, 112, 54, 173, 121, + 75, 157, 92, 75, 157, 129, 94, 65, 52, 37 }, + { 52, 124, 84, 136, 107, 197, 161, 82, 183, 151, + 109, 153, 140, 103, 152, 134, 109, 81, 69, 50 }, + }, +#endif + { + { 34, 103, 61, 106, 62, 160, 112, 54, 173, 121, + 75, 157, 92, 75, 157, 129, 94, 65, 52, 37 }, + { 52, 124, 84, 136, 107, 197, 161, 82, 183, 151, + 109, 153, 140, 103, 152, 134, 109, 81, 69, 50 }, + }, + { + { 34, 127, 74, 124, 74, 204, 153, 76, 226, 162, + 92, 207, 126, 91, 227, 192, 149, 108, 85, 55 }, + { 43, 136, 115, 158, 130, 212, 187, 112, 231, 180, + 130, 202, 164, 130, 236, 204, 168, 139, 112, 114 }, + }, + { + { 25, 117, 70, 120, 77, 215, 171, 102, 234, 156, + 105, 235, 155, 109, 247, 220, 176, 127, 92, 72 }, + { 24, 88, 49, 100, 62, 202, 148, 62, 237, 178, + 102, 233, 168, 105, 244, 198, 162, 127, 103, 71 }, + }, + { + { 11, 54, 17, 69, 26, 128, 125, 56, 232, 130, + 60, 237, 121, 66, 250, 168, 134, 114, 93, 53 }, + { 21, 52, 32, 95, 64, 171, 152, 70, 247, 159, + 81, 252, 177, 100, 252, 221, 192, 143, 195, 146 }, + }, +}; + +const aom_prob default_eob_flag[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS] = { +#if CONFIG_CB4X4 + { + { 229, 236, 231, 222, 239, 236, 214, 201, 236, 226, 195, 134, 228, + 210, 150, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }, + { 182, 186, 172, 176, 207, 213, 152, 122, 187, 171, 131, 65, 170, + 134, 101, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }, + }, +#endif + { + { 229, 236, 231, 222, 239, 236, 214, 201, 236, 226, 195, 134, 228, + 210, 150, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }, + { 182, 186, 172, 176, 207, 213, 152, 122, 187, 171, 131, 65, 170, + 134, 101, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }, + }, + { + { 225, 234, 244, 236, 205, 242, 246, 247, 246, 234, 191, 242, 237, + 215, 142, 224, 206, 142, 73, 128, 128, 128, 128, 128, 128 }, + { 154, 171, 187, 175, 62, 199, 202, 206, 215, 200, 111, 197, 199, + 174, 100, 135, 105, 104, 45, 128, 128, 128, 128, 128, 128 }, + }, + { + { 180, 213, 216, 229, 233, 232, 240, 235, 220, 178, 239, 238, 225, + 187, 229, 214, 226, 200, 183, 141, 158, 179, 128, 128, 128 }, + { 190, 225, 234, 248, 249, 248, 253, 251, 232, 110, 254, 252, 236, + 57, 253, 248, 232, 85, 244, 189, 112, 64, 128, 128, 128 }, + }, + { + { 248, 224, 246, 244, 239, 245, 251, 246, 251, 255, 255, 255, 249, + 255, 255, 255, 229, 255, 255, 255, 228, 255, 255, 247, 137 }, + { 204, 207, 233, 215, 193, 228, 239, 221, 227, 250, 236, 207, 135, + 236, 186, 182, 57, 209, 140, 128, 85, 184, 110, 128, 128 }, + }, +}; + +const aom_prob default_coeff_lps[TX_SIZES][PLANE_TYPES][LEVEL_CONTEXTS] = { +#if CONFIG_CB4X4 + { + { 164, 128, 134, 165, 128, 137, 168, 128, 97, 136, 167, 128, + 182, 205, 143, 172, 200, 145, 173, 193, 103, 137, 170, 191, + 198, 214, 162, 187, 209, 162, 187, 207, 128, 156, 183, 201, + 219, 230, 204, 210, 225, 201, 209, 225, 187, 190, 203, 214 }, + { 106, 128, 98, 126, 128, 87, 122, 128, 54, 89, 131, 128, + 142, 180, 123, 154, 189, 115, 149, 175, 79, 115, 157, 182, + 175, 197, 147, 174, 199, 145, 174, 201, 89, 135, 173, 194, + 212, 222, 206, 203, 223, 188, 201, 220, 128, 144, 202, 206 }, + }, +#endif + { + { 164, 128, 134, 165, 128, 137, 168, 128, 97, 136, 167, 128, + 182, 205, 143, 172, 200, 145, 173, 193, 103, 137, 170, 191, + 198, 214, 162, 187, 209, 162, 187, 207, 128, 156, 183, 201, + 219, 230, 204, 210, 225, 201, 209, 225, 187, 190, 203, 214 }, + { 106, 128, 98, 126, 128, 87, 122, 128, 54, 89, 131, 128, + 142, 180, 123, 154, 189, 115, 149, 175, 79, 115, 157, 182, + 175, 197, 147, 174, 199, 145, 174, 201, 89, 135, 173, 194, + 212, 222, 206, 203, 223, 188, 201, 220, 128, 144, 202, 206 }, + }, + { + { 171, 128, 123, 169, 128, 121, 165, 128, 82, 125, 168, 128, + 191, 213, 143, 177, 199, 136, 170, 194, 95, 135, 171, 195, + 206, 222, 166, 191, 212, 154, 184, 207, 115, 149, 180, 204, + 223, 237, 196, 215, 231, 186, 209, 228, 158, 178, 201, 222 }, + { 115, 128, 115, 146, 128, 91, 147, 128, 55, 93, 139, 128, + 147, 190, 141, 176, 201, 123, 156, 173, 68, 114, 156, 195, + 186, 205, 153, 191, 214, 141, 179, 205, 107, 132, 166, 184, + 215, 225, 200, 212, 230, 102, 207, 222, 128, 119, 200, 212 }, + }, + { + { 185, 128, 134, 198, 128, 128, 195, 128, 58, 110, 162, 128, + 208, 227, 154, 196, 206, 144, 188, 209, 83, 130, 168, 198, + 219, 232, 167, 205, 222, 158, 196, 216, 107, 143, 178, 204, + 233, 244, 202, 226, 238, 191, 217, 234, 153, 178, 200, 223 }, + { 160, 128, 154, 197, 128, 129, 178, 128, 53, 112, 157, 128, + 185, 214, 169, 196, 221, 134, 179, 186, 82, 131, 168, 194, + 204, 220, 176, 209, 221, 173, 194, 209, 107, 154, 181, 203, + 230, 241, 202, 226, 237, 185, 223, 234, 162, 187, 203, 222 }, + }, + { + { 177, 128, 165, 226, 128, 152, 219, 128, 45, 129, 188, 128, + 198, 218, 179, 220, 228, 163, 214, 220, 72, 134, 181, 206, + 216, 225, 177, 218, 231, 158, 213, 223, 112, 150, 185, 210, + 245, 251, 204, 234, 247, 195, 231, 243, 163, 186, 213, 235 }, + { 161, 128, 174, 205, 128, 146, 182, 128, 59, 125, 179, 128, + 183, 208, 199, 220, 239, 184, 213, 217, 71, 141, 196, 217, + 213, 219, 215, 230, 237, 171, 224, 238, 112, 173, 193, 221, + 239, 246, 168, 243, 249, 93, 241, 247, 128, 195, 216, 233 }, + }, +}; +#endif // CONFIG_LV_MAP + +#if CONFIG_ALT_INTRA + +const aom_prob av1_kf_y_mode_prob[INTRA_MODES][INTRA_MODES][INTRA_MODES - 1] = { + { + // above = dc + { 121, 30, 54, 128, 164, 158, 45, 41, 57, 91 }, // left = dc + { 91, 38, 101, 102, 124, 141, 49, 48, 45, 73 }, // left = v + { 66, 28, 27, 177, 225, 178, 32, 27, 52, 114 }, // left = h + { 106, 23, 50, 101, 134, 148, 64, 50, 49, 107 }, // left = d45 + { 75, 24, 32, 118, 66, 143, 42, 28, 57, 74 }, // left = d135 + { 95, 24, 40, 142, 56, 141, 72, 121, 129, 255 }, // left = d117 + { 71, 14, 25, 126, 117, 201, 28, 21, 117, 89 }, // left = d153 + { 85, 16, 37, 110, 163, 178, 41, 28, 48, 134 }, // left = d207 + { 86, 25, 32, 83, 105, 133, 58, 81, 46, 95 }, // left = d63 + { 79, 25, 38, 75, 150, 255, 30, 49, 34, 51 }, // left = smooth + { 68, 59, 48, 122, 193, 158, 43, 46, 46, 112 }, // left = paeth + }, + { + // above = v + { 66, 21, 118, 111, 145, 107, 27, 50, 27, 54 }, // left = dc + { 52, 25, 167, 81, 120, 101, 34, 55, 19, 32 }, // left = v + { 56, 18, 72, 134, 208, 139, 31, 34, 27, 89 }, // left = h + { 75, 21, 94, 88, 134, 123, 49, 57, 30, 68 }, // left = d45 + { 54, 18, 95, 96, 78, 107, 33, 49, 28, 65 }, // left = d135 + { 61, 19, 121, 131, 58, 101, 56, 143, 120, 255 }, // left = d117 + { 53, 13, 78, 103, 110, 147, 31, 41, 64, 77 }, // left = d153 + { 69, 14, 78, 93, 167, 121, 31, 39, 25, 113 }, // left = d207 + { 64, 18, 103, 79, 90, 108, 34, 73, 27, 69 }, // left = d63 + { 52, 20, 103, 61, 161, 255, 22, 42, 16, 35 }, // left = smooth + { 50, 31, 124, 92, 161, 120, 50, 53, 23, 60 }, // left = paeth + }, + { + // above = h + { 94, 29, 31, 158, 214, 178, 35, 31, 72, 111 }, // left = dc + { 72, 37, 72, 149, 184, 177, 43, 40, 53, 105 }, // left = v + { 53, 21, 14, 196, 242, 209, 29, 19, 55, 145 }, // left = h + { 93, 36, 36, 104, 176, 166, 56, 37, 49, 141 }, // left = d45 + { 84, 32, 27, 124, 108, 143, 38, 36, 76, 134 }, // left = d135 + { 82, 31, 47, 142, 122, 161, 83, 73, 126, 255 }, // left = d117 + { 66, 16, 20, 133, 148, 210, 30, 17, 113, 104 }, // left = d153 + { 76, 16, 17, 129, 207, 181, 41, 20, 46, 163 }, // left = d207 + { 72, 38, 21, 100, 142, 171, 37, 70, 49, 111 }, // left = d63 + { 61, 30, 27, 115, 208, 255, 27, 31, 44, 63 }, // left = smooth + { 53, 45, 29, 157, 222, 185, 49, 37, 55, 102 }, // left = paeth + }, + { + // above = d45 + { 96, 18, 37, 98, 138, 154, 68, 56, 59, 96 }, // left = dc + { 73, 18, 92, 81, 125, 132, 75, 64, 27, 67 }, // left = v + { 73, 17, 27, 128, 213, 154, 56, 44, 32, 105 }, // left = h + { 101, 20, 21, 75, 138, 138, 82, 56, 23, 154 }, // left = d45 + { 71, 15, 33, 91, 70, 150, 62, 55, 38, 118 }, // left = d135 + { 80, 19, 38, 116, 69, 122, 88, 132, 92, 255 }, // left = d117 + { 68, 11, 22, 101, 116, 179, 52, 44, 85, 96 }, // left = d153 + { 101, 8, 59, 77, 151, 170, 53, 41, 35, 172 }, // left = d207 + { 82, 19, 24, 81, 172, 129, 82, 128, 43, 108 }, // left = d63 + { 66, 18, 42, 64, 143, 255, 52, 52, 25, 83 }, // left = smooth + { 57, 24, 42, 85, 169, 145, 104, 71, 34, 86 }, // left = paeth + }, + { + // above = d135 + { 85, 15, 29, 113, 83, 176, 26, 29, 70, 110 }, // left = dc + { 78, 28, 49, 111, 91, 141, 30, 42, 48, 75 }, // left = v + { 56, 21, 16, 146, 190, 178, 23, 31, 49, 92 }, // left = h + { 70, 19, 20, 65, 90, 173, 97, 36, 57, 98 }, // left = d45 + { 77, 14, 26, 110, 51, 156, 34, 35, 54, 74 }, // left = d135 + { 78, 18, 36, 153, 47, 131, 62, 102, 155, 255 }, // left = d117 + { 56, 11, 15, 115, 85, 196, 32, 45, 81, 96 }, // left = d153 + { 90, 18, 24, 95, 126, 159, 34, 31, 46, 136 }, // left = d207 + { 80, 23, 28, 90, 75, 141, 39, 50, 46, 87 }, // left = d63 + { 63, 22, 31, 91, 110, 255, 26, 43, 51, 51 }, // left = smooth + { 66, 32, 31, 122, 145, 165, 40, 43, 56, 79 }, // left = paeth + }, + { + // above = d117 + { 81, 16, 61, 170, 74, 105, 54, 105, 113, 255 }, // left = dc + { 74, 20, 86, 163, 64, 97, 65, 129, 101, 255 }, // left = v + { 63, 15, 47, 168, 141, 176, 69, 77, 77, 255 }, // left = h + { 70, 17, 59, 97, 78, 114, 74, 122, 80, 255 }, // left = d45 + { 78, 13, 50, 153, 34, 126, 75, 114, 120, 255 }, // left = d135 + { 72, 16, 69, 159, 28, 108, 63, 134, 107, 255 }, // left = d117 + { 66, 9, 47, 131, 79, 148, 41, 88, 105, 255 }, // left = d153 + { 78, 12, 60, 119, 105, 133, 47, 95, 63, 255 }, // left = d207 + { 82, 21, 58, 128, 61, 98, 64, 136, 91, 255 }, // left = d63 + { 23, 26, 28, 96, 85, 128, 51, 64, 85, 128 }, // left = smooth + { 58, 27, 62, 162, 109, 151, 75, 106, 78, 255 }, // left = paeth + }, + { + // above = d153 + { 91, 18, 25, 121, 166, 173, 25, 25, 128, 102 }, // left = dc + { 80, 27, 51, 111, 141, 147, 45, 38, 70, 85 }, // left = v + { 53, 12, 11, 154, 197, 225, 17, 17, 74, 145 }, // left = h + { 93, 27, 23, 111, 143, 188, 43, 39, 69, 112 }, // left = d45 + { 83, 15, 21, 118, 67, 178, 40, 33, 73, 92 }, // left = d135 + { 94, 13, 31, 132, 66, 110, 61, 82, 148, 255 }, // left = d117 + { 76, 9, 11, 96, 105, 201, 16, 13, 157, 97 }, // left = d153 + { 70, 10, 12, 100, 172, 201, 23, 17, 53, 158 }, // left = d207 + { 114, 25, 21, 104, 108, 163, 30, 47, 53, 111 }, // left = d63 + { 70, 16, 21, 80, 157, 255, 25, 30, 81, 69 }, // left = smooth + { 87, 32, 26, 120, 191, 168, 32, 33, 70, 118 }, // left = paeth + }, + { + // above = d207 + { 98, 20, 39, 122, 168, 188, 38, 36, 54, 132 }, // left = dc + { 81, 37, 62, 97, 122, 153, 38, 43, 36, 118 }, // left = v + { 71, 21, 22, 154, 227, 183, 37, 31, 46, 140 }, // left = h + { 90, 34, 19, 93, 144, 194, 65, 47, 41, 163 }, // left = d45 + { 78, 20, 27, 91, 93, 173, 57, 52, 49, 113 }, // left = d135 + { 79, 25, 45, 121, 101, 147, 69, 56, 122, 255 }, // left = d117 + { 73, 13, 19, 105, 122, 206, 40, 28, 91, 126 }, // left = d153 + { 101, 14, 22, 87, 153, 169, 33, 25, 26, 175 }, // left = d207 + { 81, 28, 23, 86, 115, 169, 48, 56, 41, 111 }, // left = d63 + { 70, 24, 30, 90, 180, 255, 38, 26, 36, 82 }, // left = smooth + { 61, 37, 30, 94, 189, 163, 76, 50, 36, 127 }, // left = paeth + }, + { + // above = d63 + { 77, 13, 46, 86, 138, 117, 55, 88, 34, 68 }, // left = dc + { 68, 17, 80, 64, 105, 108, 66, 115, 32, 45 }, // left = v + { 62, 13, 37, 124, 210, 131, 46, 57, 28, 103 }, // left = h + { 88, 15, 45, 73, 134, 145, 73, 101, 37, 87 }, // left = d45 + { 68, 16, 35, 78, 81, 133, 54, 71, 33, 67 }, // left = d135 + { 71, 16, 57, 108, 61, 135, 71, 184, 113, 255 }, // left = d117 + { 55, 10, 27, 69, 107, 158, 39, 76, 82, 95 }, // left = d153 + { 80, 9, 38, 78, 153, 145, 50, 63, 28, 123 }, // left = d207 + { 86, 12, 33, 49, 107, 135, 64, 134, 57, 89 }, // left = d63 + { 56, 19, 55, 60, 163, 255, 38, 84, 22, 36 }, // left = smooth + { 53, 17, 60, 69, 151, 126, 73, 113, 26, 80 }, // left = paeth + }, + { + // above = smooth + { 79, 16, 46, 89, 167, 255, 22, 36, 29, 42 }, // left = dc + { 63, 22, 88, 71, 131, 255, 26, 41, 21, 35 }, // left = v + { 51, 18, 28, 142, 232, 255, 26, 25, 25, 75 }, // left = h + { 75, 18, 43, 70, 140, 255, 37, 49, 34, 89 }, // left = d45 + { 70, 14, 35, 87, 83, 255, 30, 36, 34, 50 }, // left = d135 + { 23, 26, 28, 96, 85, 128, 51, 64, 85, 128 }, // left = d117 + { 74, 12, 33, 83, 128, 255, 27, 33, 58, 68 }, // left = d153 + { 66, 11, 30, 77, 179, 255, 21, 27, 23, 113 }, // left = d207 + { 68, 22, 40, 65, 118, 255, 28, 61, 30, 50 }, // left = d63 + { 60, 18, 44, 69, 141, 255, 18, 32, 22, 40 }, // left = smooth + { 52, 32, 54, 96, 194, 255, 33, 37, 25, 53 }, // left = paeth + }, + { + // above = paeth + { 76, 47, 67, 123, 182, 150, 41, 52, 55, 97 }, // left = dc + { 69, 40, 125, 102, 138, 138, 42, 55, 32, 70 }, // left = v + { 46, 28, 27, 160, 232, 169, 34, 21, 32, 122 }, // left = h + { 78, 35, 41, 99, 128, 124, 49, 43, 35, 111 }, // left = d45 + { 66, 28, 47, 100, 113, 145, 37, 40, 72, 93 }, // left = d135 + { 77, 37, 76, 134, 124, 124, 65, 122, 88, 255 }, // left = d117 + { 53, 23, 38, 108, 128, 204, 26, 32, 115, 114 }, // left = d153 + { 65, 20, 29, 101, 202, 186, 29, 24, 29, 188 }, // left = d207 + { 71, 24, 49, 81, 126, 151, 36, 65, 28, 93 }, // left = d63 + { 54, 36, 53, 94, 193, 255, 25, 38, 20, 64 }, // left = smooth + { 52, 54, 60, 108, 176, 168, 47, 44, 50, 105 }, // left = paeth + }, +}; + +static const aom_prob default_if_y_probs[BLOCK_SIZE_GROUPS][INTRA_MODES - 1] = { + { 88, 16, 47, 133, 143, 150, 70, 48, 84, 122 }, // block_size < 8x8 + { 75, 26, 51, 120, 158, 157, 44, 45, 56, 102 }, // block_size < 16x16 + { 73, 24, 60, 115, 184, 164, 26, 36, 32, 63 }, // block_size < 32x32 + { 96, 27, 50, 107, 221, 148, 16, 22, 14, 39 }, // block_size >= 32x32 +}; + +static const aom_prob default_uv_probs[INTRA_MODES][INTRA_MODES - 1] = { + { 199, 3, 79, 179, 220, 109, 38, 50, 68, 138 }, // y = dc + { 17, 2, 219, 136, 131, 58, 21, 106, 23, 41 }, // y = v + { 26, 1, 5, 244, 253, 138, 16, 21, 68, 205 }, // y = h + { 183, 3, 66, 94, 195, 97, 101, 104, 41, 178 }, // y = d45 + { 178, 2, 36, 158, 99, 175, 21, 29, 105, 77 }, // y = d135 + { 154, 3, 65, 219, 40, 48, 45, 95, 146, 255 }, // y = d117 + { 167, 1, 16, 160, 214, 187, 10, 10, 200, 155 }, // y = d153 + { 154, 2, 18, 178, 238, 132, 25, 21, 34, 221 }, // y = d207 + { 153, 4, 76, 85, 157, 90, 38, 165, 46, 104 }, // y = d63 + { 163, 3, 68, 87, 190, 255, 19, 27, 25, 46 }, // y = smooth + { 185, 7, 113, 171, 203, 57, 18, 69, 49, 104 }, // y = paeth +}; + +#else + +const aom_prob av1_kf_y_mode_prob[INTRA_MODES][INTRA_MODES][INTRA_MODES - 1] = { + { + // above = dc + { 137, 30, 42, 148, 151, 207, 70, 52, 91 }, // left = dc + { 92, 45, 102, 136, 116, 180, 74, 90, 100 }, // left = v + { 73, 32, 19, 187, 222, 215, 46, 34, 100 }, // left = h + { 91, 30, 32, 116, 121, 186, 93, 86, 94 }, // left = d45 + { 72, 35, 36, 149, 68, 206, 68, 63, 105 }, // left = d135 + { 73, 31, 28, 138, 57, 124, 55, 122, 151 }, // left = d117 + { 67, 23, 21, 140, 126, 197, 40, 37, 171 }, // left = d153 + { 86, 27, 28, 128, 154, 212, 45, 43, 53 }, // left = d207 + { 74, 32, 27, 107, 86, 160, 63, 134, 102 }, // left = d63 + { 59, 67, 44, 140, 161, 202, 78, 67, 119 } // left = tm + }, + { + // above = v + { 63, 36, 126, 146, 123, 158, 60, 90, 96 }, // left = dc + { 43, 46, 168, 134, 107, 128, 69, 142, 92 }, // left = v + { 44, 29, 68, 159, 201, 177, 50, 57, 77 }, // left = h + { 58, 38, 76, 114, 97, 172, 78, 133, 92 }, // left = d45 + { 46, 41, 76, 140, 63, 184, 69, 112, 57 }, // left = d135 + { 38, 32, 85, 140, 46, 112, 54, 151, 133 }, // left = d117 + { 39, 27, 61, 131, 110, 175, 44, 75, 136 }, // left = d153 + { 52, 30, 74, 113, 130, 175, 51, 64, 58 }, // left = d207 + { 47, 35, 80, 100, 74, 143, 64, 163, 74 }, // left = d63 + { 36, 61, 116, 114, 128, 162, 80, 125, 82 } // left = tm + }, + { + // above = h + { 82, 26, 26, 171, 208, 204, 44, 32, 105 }, // left = dc + { 55, 44, 68, 166, 179, 192, 57, 57, 108 }, // left = v + { 42, 26, 11, 199, 241, 228, 23, 15, 85 }, // left = h + { 68, 42, 19, 131, 160, 199, 55, 52, 83 }, // left = d45 + { 58, 50, 25, 139, 115, 232, 39, 52, 118 }, // left = d135 + { 50, 35, 33, 153, 104, 162, 64, 59, 131 }, // left = d117 + { 44, 24, 16, 150, 177, 202, 33, 19, 156 }, // left = d153 + { 55, 27, 12, 153, 203, 218, 26, 27, 49 }, // left = d207 + { 53, 49, 21, 110, 116, 168, 59, 80, 76 }, // left = d63 + { 38, 72, 19, 168, 203, 212, 50, 50, 107 } // left = tm + }, + { + // above = d45 + { 103, 26, 36, 129, 132, 201, 83, 80, 93 }, // left = dc + { 59, 38, 83, 112, 103, 162, 98, 136, 90 }, // left = v + { 62, 30, 23, 158, 200, 207, 59, 57, 50 }, // left = h + { 67, 30, 29, 84, 86, 191, 102, 91, 59 }, // left = d45 + { 60, 32, 33, 112, 71, 220, 64, 89, 104 }, // left = d135 + { 53, 26, 34, 130, 56, 149, 84, 120, 103 }, // left = d117 + { 53, 21, 23, 133, 109, 210, 56, 77, 172 }, // left = d153 + { 77, 19, 29, 112, 142, 228, 55, 66, 36 }, // left = d207 + { 61, 29, 29, 93, 97, 165, 83, 175, 162 }, // left = d63 + { 47, 47, 43, 114, 137, 181, 100, 99, 95 } // left = tm + }, + { + // above = d135 + { 69, 23, 29, 128, 83, 199, 46, 44, 101 }, // left = dc + { 53, 40, 55, 139, 69, 183, 61, 80, 110 }, // left = v + { 40, 29, 19, 161, 180, 207, 43, 24, 91 }, // left = h + { 60, 34, 19, 105, 61, 198, 53, 64, 89 }, // left = d45 + { 52, 31, 22, 158, 40, 209, 58, 62, 89 }, // left = d135 + { 44, 31, 29, 147, 46, 158, 56, 102, 198 }, // left = d117 + { 35, 19, 12, 135, 87, 209, 41, 45, 167 }, // left = d153 + { 55, 25, 21, 118, 95, 215, 38, 39, 66 }, // left = d207 + { 51, 38, 25, 113, 58, 164, 70, 93, 97 }, // left = d63 + { 47, 54, 34, 146, 108, 203, 72, 103, 151 } // left = tm + }, + { + // above = d117 + { 64, 19, 37, 156, 66, 138, 49, 95, 133 }, // left = dc + { 46, 27, 80, 150, 55, 124, 55, 121, 135 }, // left = v + { 36, 23, 27, 165, 149, 166, 54, 64, 118 }, // left = h + { 53, 21, 36, 131, 63, 163, 60, 109, 81 }, // left = d45 + { 40, 26, 35, 154, 40, 185, 51, 97, 123 }, // left = d135 + { 35, 19, 34, 179, 19, 97, 48, 129, 124 }, // left = d117 + { 36, 20, 26, 136, 62, 164, 33, 77, 154 }, // left = d153 + { 45, 18, 32, 130, 90, 157, 40, 79, 91 }, // left = d207 + { 45, 26, 28, 129, 45, 129, 49, 147, 123 }, // left = d63 + { 38, 44, 51, 136, 74, 162, 57, 97, 121 } // left = tm + }, + { + // above = d153 + { 75, 17, 22, 136, 138, 185, 32, 34, 166 }, // left = dc + { 56, 39, 58, 133, 117, 173, 48, 53, 187 }, // left = v + { 35, 21, 12, 161, 212, 207, 20, 23, 145 }, // left = h + { 56, 29, 19, 117, 109, 181, 55, 68, 112 }, // left = d45 + { 47, 29, 17, 153, 64, 220, 59, 51, 114 }, // left = d135 + { 46, 16, 24, 136, 76, 147, 41, 64, 172 }, // left = d117 + { 34, 17, 11, 108, 152, 187, 13, 15, 209 }, // left = d153 + { 51, 24, 14, 115, 133, 209, 32, 26, 104 }, // left = d207 + { 55, 30, 18, 122, 79, 179, 44, 88, 116 }, // left = d63 + { 37, 49, 25, 129, 168, 164, 41, 54, 148 } // left = tm + }, + { + // above = d207 + { 82, 22, 32, 127, 143, 213, 39, 41, 70 }, // left = dc + { 62, 44, 61, 123, 105, 189, 48, 57, 64 }, // left = v + { 47, 25, 17, 175, 222, 220, 24, 30, 86 }, // left = h + { 68, 36, 17, 106, 102, 206, 59, 74, 74 }, // left = d45 + { 57, 39, 23, 151, 68, 216, 55, 63, 58 }, // left = d135 + { 49, 30, 35, 141, 70, 168, 82, 40, 115 }, // left = d117 + { 51, 25, 15, 136, 129, 202, 38, 35, 139 }, // left = d153 + { 68, 26, 16, 111, 141, 215, 29, 28, 28 }, // left = d207 + { 59, 39, 19, 114, 75, 180, 77, 104, 42 }, // left = d63 + { 40, 61, 26, 126, 152, 206, 61, 59, 93 } // left = tm + }, + { + // above = d63 + { 78, 23, 39, 111, 117, 170, 74, 124, 94 }, // left = dc + { 48, 34, 86, 101, 92, 146, 78, 179, 134 }, // left = v + { 47, 22, 24, 138, 187, 178, 68, 69, 59 }, // left = h + { 56, 25, 33, 105, 112, 187, 95, 177, 129 }, // left = d45 + { 48, 31, 27, 114, 63, 183, 82, 116, 56 }, // left = d135 + { 43, 28, 37, 121, 63, 123, 61, 192, 169 }, // left = d117 + { 42, 17, 24, 109, 97, 177, 56, 76, 122 }, // left = d153 + { 58, 18, 28, 105, 139, 182, 70, 92, 63 }, // left = d207 + { 46, 23, 32, 74, 86, 150, 67, 183, 88 }, // left = d63 + { 36, 38, 48, 92, 122, 165, 88, 137, 91 } // left = tm + }, + { + // above = tm + { 65, 70, 60, 155, 159, 199, 61, 60, 81 }, // left = dc + { 44, 78, 115, 132, 119, 173, 71, 112, 93 }, // left = v + { 39, 38, 21, 184, 227, 206, 42, 32, 64 }, // left = h + { 58, 47, 36, 124, 137, 193, 80, 82, 78 }, // left = d45 + { 49, 50, 35, 144, 95, 205, 63, 78, 59 }, // left = d135 + { 41, 53, 52, 148, 71, 142, 65, 128, 51 }, // left = d117 + { 40, 36, 28, 143, 143, 202, 40, 55, 137 }, // left = d153 + { 52, 34, 29, 129, 183, 227, 42, 35, 43 }, // left = d207 + { 42, 44, 44, 104, 105, 164, 64, 130, 80 }, // left = d63 + { 43, 81, 53, 140, 169, 204, 68, 84, 72 } // left = tm + } +}; + +// Default probabilities for signaling Intra mode for Y plane -- used only for +// inter frames. ('av1_kf_y_mode_prob' is used for intra-only frames). +// Context used: block size group. +static const aom_prob default_if_y_probs[BLOCK_SIZE_GROUPS][INTRA_MODES - 1] = { + { 65, 32, 18, 144, 162, 194, 41, 51, 98 }, // block_size < 8x8 + { 132, 68, 18, 165, 217, 196, 45, 40, 78 }, // block_size < 16x16 + { 173, 80, 19, 176, 240, 193, 64, 35, 46 }, // block_size < 32x32 + { 221, 135, 38, 194, 248, 121, 96, 85, 29 } // block_size >= 32x32 +}; + +// Default probabilities for signaling Intra mode for UV plane -- common for +// both intra and inter frames. +// Context used: Intra mode used by Y plane of the same block. +static const aom_prob default_uv_probs[INTRA_MODES][INTRA_MODES - 1] = { + { 120, 7, 76, 176, 208, 126, 28, 54, 103 }, // y = dc + { 48, 12, 154, 155, 139, 90, 34, 117, 119 }, // y = v + { 67, 6, 25, 204, 243, 158, 13, 21, 96 }, // y = h + { 97, 5, 44, 131, 176, 139, 48, 68, 97 }, // y = d45 + { 83, 5, 42, 156, 111, 152, 26, 49, 152 }, // y = d135 + { 80, 5, 58, 178, 74, 83, 33, 62, 145 }, // y = d117 + { 86, 5, 32, 154, 192, 168, 14, 22, 163 }, // y = d153 + { 85, 5, 32, 156, 216, 148, 19, 29, 73 }, // y = d207 + { 77, 7, 64, 116, 132, 122, 37, 126, 120 }, // y = d63 + { 101, 21, 107, 181, 192, 103, 19, 67, 125 } // y = tm +}; + +#endif // CONFIG_ALT_INTRA + +#if CONFIG_EXT_PARTITION_TYPES +static const aom_prob + default_partition_probs[PARTITION_CONTEXTS][EXT_PARTITION_TYPES - 1] = { + // 8x8 -> 4x4 + { 199, 122, 141, 128, 128, 128, 128 }, // a/l both not split + { 147, 63, 159, 128, 128, 128, 128 }, // a split, l not split + { 148, 133, 118, 128, 128, 128, 128 }, // l split, a not split + { 121, 104, 114, 128, 128, 128, 128 }, // a/l both split + // 16x16 -> 8x8 + { 174, 73, 87, 128, 128, 128, 128 }, // a/l both not split + { 92, 41, 83, 128, 128, 128, 128 }, // a split, l not split + { 82, 99, 50, 128, 128, 128, 128 }, // l split, a not split + { 53, 39, 39, 128, 128, 128, 128 }, // a/l both split + // 32x32 -> 16x16 + { 177, 58, 59, 128, 128, 128, 128 }, // a/l both not split + { 68, 26, 63, 128, 128, 128, 128 }, // a split, l not split + { 52, 79, 25, 128, 128, 128, 128 }, // l split, a not split + { 17, 14, 12, 128, 128, 128, 128 }, // a/l both split + // 64x64 -> 32x32 + { 222, 34, 30, 128, 128, 128, 128 }, // a/l both not split + { 72, 16, 44, 128, 128, 128, 128 }, // a split, l not split + { 58, 32, 12, 128, 128, 128, 128 }, // l split, a not split + { 10, 7, 6, 128, 128, 128, 128 }, // a/l both split +#if CONFIG_EXT_PARTITION + // 128x128 -> 64x64 + { 222, 34, 30, 128, 128, 128, 128 }, // a/l both not split + { 72, 16, 44, 128, 128, 128, 128 }, // a split, l not split + { 58, 32, 12, 128, 128, 128, 128 }, // l split, a not split + { 10, 7, 6, 128, 128, 128, 128 }, // a/l both split +#endif // CONFIG_EXT_PARTITION +#if CONFIG_UNPOISON_PARTITION_CTX + { 0, 0, 141, 0, 0, 0, 0 }, // 8x8 -> 4x4 + { 0, 0, 87, 0, 0, 0, 0 }, // 16x16 -> 8x8 + { 0, 0, 59, 0, 0, 0, 0 }, // 32x32 -> 16x16 + { 0, 0, 30, 0, 0, 0, 0 }, // 64x64 -> 32x32 +#if CONFIG_EXT_PARTITION + { 0, 0, 30, 0, 0, 0, 0 }, // 128x128 -> 64x64 +#endif // CONFIG_EXT_PARTITION + { 0, 122, 0, 0, 0, 0, 0 }, // 8x8 -> 4x4 + { 0, 73, 0, 0, 0, 0, 0 }, // 16x16 -> 8x8 + { 0, 58, 0, 0, 0, 0, 0 }, // 32x32 -> 16x16 + { 0, 34, 0, 0, 0, 0, 0 }, // 64x64 -> 32x32 +#if CONFIG_EXT_PARTITION + { 0, 34, 0, 0, 0, 0, 0 }, // 128x128 -> 64x64 +#endif // CONFIG_EXT_PARTITION +#endif // CONFIG_UNPOISON_PARTITION_CTX + }; +#else +static const aom_prob + default_partition_probs[PARTITION_CONTEXTS][PARTITION_TYPES - 1] = { + // 8x8 -> 4x4 + { 199, 122, 141 }, // a/l both not split + { 147, 63, 159 }, // a split, l not split + { 148, 133, 118 }, // l split, a not split + { 121, 104, 114 }, // a/l both split + // 16x16 -> 8x8 + { 174, 73, 87 }, // a/l both not split + { 92, 41, 83 }, // a split, l not split + { 82, 99, 50 }, // l split, a not split + { 53, 39, 39 }, // a/l both split + // 32x32 -> 16x16 + { 177, 58, 59 }, // a/l both not split + { 68, 26, 63 }, // a split, l not split + { 52, 79, 25 }, // l split, a not split + { 17, 14, 12 }, // a/l both split + // 64x64 -> 32x32 + { 222, 34, 30 }, // a/l both not split + { 72, 16, 44 }, // a split, l not split + { 58, 32, 12 }, // l split, a not split + { 10, 7, 6 }, // a/l both split +#if CONFIG_EXT_PARTITION + // 128x128 -> 64x64 + { 222, 34, 30 }, // a/l both not split + { 72, 16, 44 }, // a split, l not split + { 58, 32, 12 }, // l split, a not split + { 10, 7, 6 }, // a/l both split +#endif // CONFIG_EXT_PARTITION +#if CONFIG_UNPOISON_PARTITION_CTX + { 0, 0, 141 }, // 8x8 -> 4x4 + { 0, 0, 87 }, // 16x16 -> 8x8 + { 0, 0, 59 }, // 32x32 -> 16x16 + { 0, 0, 30 }, // 64x64 -> 32x32 +#if CONFIG_EXT_PARTITION + { 0, 0, 30 }, // 128x128 -> 64x64 +#endif // CONFIG_EXT_PARTITION + { 0, 122, 0 }, // 8x8 -> 4x4 + { 0, 73, 0 }, // 16x16 -> 8x8 + { 0, 58, 0 }, // 32x32 -> 16x16 + { 0, 34, 0 }, // 64x64 -> 32x32 +#if CONFIG_EXT_PARTITION + { 0, 34, 0 }, // 128x128 -> 64x64 +#endif // CONFIG_EXT_PARTITION +#endif // CONFIG_UNPOISON_PARTITION_CTX + }; +#endif // CONFIG_EXT_PARTITION_TYPES + +#if CONFIG_REF_MV +static const aom_prob default_newmv_prob[NEWMV_MODE_CONTEXTS] = { + 200, 180, 150, 150, 110, 70, 60, +}; + +static const aom_prob default_zeromv_prob[ZEROMV_MODE_CONTEXTS] = { + 192, 64, +}; + +static const aom_prob default_refmv_prob[REFMV_MODE_CONTEXTS] = { + 220, 220, 200, 200, 180, 128, 30, 220, 30, +}; + +static const aom_prob default_drl_prob[DRL_MODE_CONTEXTS] = { 128, 160, 180, + 128, 160 }; +#endif // CONFIG_REF_MV + +static const aom_prob + default_inter_mode_probs[INTER_MODE_CONTEXTS][INTER_MODES - 1] = { + { 2, 173, 34 }, // 0 = both zero mv + { 7, 145, 85 }, // 1 = one zero mv + one a predicted mv + { 7, 166, 63 }, // 2 = two predicted mvs + { 7, 94, 66 }, // 3 = one predicted/zero and one new mv + { 8, 64, 46 }, // 4 = two new mvs + { 17, 81, 31 }, // 5 = one intra neighbour + x + { 25, 29, 30 }, // 6 = two intra neighbours + }; + +#if CONFIG_EXT_INTER +static const aom_prob default_inter_compound_mode_probs + [INTER_MODE_CONTEXTS][INTER_COMPOUND_MODES - 1] = { + { 2, 173, 68, 192, 64, 192, 128, 180, 180 }, // 0 = both zero mv + { 7, 145, 160, 192, 64, 192, 128, 180, 180 }, // 1 = 1 zero + 1 predicted + { 7, 166, 126, 192, 64, 192, 128, 180, 180 }, // 2 = two predicted mvs + { 7, 94, 132, 192, 64, 192, 128, 180, 180 }, // 3 = 1 pred/zero, 1 new + { 8, 64, 64, 192, 64, 192, 128, 180, 180 }, // 4 = two new mvs + { 17, 81, 52, 192, 64, 192, 128, 180, 180 }, // 5 = one intra neighbour + { 25, 29, 50, 192, 64, 192, 128, 180, 180 }, // 6 = two intra neighbours + }; + +#if CONFIG_COMPOUND_SINGLEREF +// TODO(zoeliu): Default values to be further adjusted based on the collected +// stats. +static const aom_prob default_inter_singleref_comp_mode_probs + [INTER_MODE_CONTEXTS][INTER_SINGLEREF_COMP_MODES - 1] = { + { 2, 173, 68, 180 }, // 0 = both zero mv + { 7, 145, 160, 180 }, // 1 = 1 zero + 1 predicted + { 7, 166, 126, 180 }, // 2 = two predicted mvs + { 7, 94, 132, 180 }, // 3 = 1 pred/zero, 1 new + { 8, 64, 64, 180 }, // 4 = two new mvs + { 17, 81, 52, 180 }, // 5 = one intra neighbour + { 25, 29, 50, 180 }, // 6 = two intra neighbours + }; +#endif // CONFIG_COMPOUND_SINGLEREF + +#if CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE +static const aom_prob + default_compound_type_probs[BLOCK_SIZES][COMPOUND_TYPES - 1] = { +#if CONFIG_CB4X4 + { 255, 255 }, { 255, 255 }, { 255, 255 }, +#endif + { 208, 200 }, { 208, 200 }, { 208, 200 }, { 208, 200 }, { 208, 200 }, + { 208, 200 }, { 216, 200 }, { 216, 200 }, { 216, 200 }, { 224, 200 }, + { 224, 200 }, { 240, 200 }, { 240, 200 }, +#if CONFIG_EXT_PARTITION + { 255, 200 }, { 255, 200 }, { 255, 200 }, +#endif // CONFIG_EXT_PARTITION + }; +#elif !CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE +static const aom_prob + default_compound_type_probs[BLOCK_SIZES][COMPOUND_TYPES - 1] = { +#if CONFIG_CB4X4 + { 208 }, { 208 }, { 208 }, +#endif + { 208 }, { 208 }, { 208 }, { 208 }, { 208 }, { 208 }, { 216 }, + { 216 }, { 216 }, { 224 }, { 224 }, { 240 }, { 240 }, +#if CONFIG_EXT_PARTITION + { 255 }, { 255 }, { 255 }, +#endif // CONFIG_EXT_PARTITION + }; +#elif CONFIG_COMPOUND_SEGMENT && !CONFIG_WEDGE +static const aom_prob + default_compound_type_probs[BLOCK_SIZES][COMPOUND_TYPES - 1] = { +#if CONFIG_CB4X4 + { 208 }, { 208 }, { 208 }, +#endif + { 208 }, { 208 }, { 208 }, { 208 }, { 208 }, { 208 }, { 216 }, + { 216 }, { 216 }, { 224 }, { 224 }, { 240 }, { 240 }, +#if CONFIG_EXT_PARTITION + { 255 }, { 255 }, { 255 }, +#endif // CONFIG_EXT_PARTITION + }; +#else +static const aom_prob default_compound_type_probs[BLOCK_SIZES] + [COMPOUND_TYPES - 1]; +#endif // CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE + +static const aom_prob default_interintra_prob[BLOCK_SIZE_GROUPS] = { + 208, 208, 208, 208, +}; + +static const aom_prob + default_interintra_mode_prob[BLOCK_SIZE_GROUPS][INTERINTRA_MODES - 1] = { + { 65, 32, 18, 144, 162, 194, 41, 51, 98 }, // block_size < 8x8 + { 132, 68, 18, 165, 217, 196, 45, 40, 78 }, // block_size < 16x16 + { 173, 80, 19, 176, 240, 193, 64, 35, 46 }, // block_size < 32x32 + { 221, 135, 38, 194, 248, 121, 96, 85, 29 } // block_size >= 32x32 + }; + +static const aom_prob default_wedge_interintra_prob[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 208, 208, 208, +#endif + 208, 208, 208, 208, 208, 208, 216, 216, 216, 224, 224, 224, 240, +#if CONFIG_EXT_PARTITION + 208, 208, 208 +#endif // CONFIG_EXT_PARTITION +}; +#endif // CONFIG_EXT_INTER + +// Change this section appropriately once warped motion is supported +#if CONFIG_MOTION_VAR && !CONFIG_WARPED_MOTION +const aom_tree_index av1_motion_mode_tree[TREE_SIZE(MOTION_MODES)] = { + -SIMPLE_TRANSLATION, -OBMC_CAUSAL +}; +static const aom_prob default_motion_mode_prob[BLOCK_SIZES][MOTION_MODES - 1] = + { +#if CONFIG_CB4X4 + { 255 }, { 255 }, { 255 }, +#endif + { 255 }, { 255 }, { 255 }, { 151 }, { 153 }, { 144 }, { 178 }, + { 165 }, { 160 }, { 207 }, { 195 }, { 168 }, { 244 }, +#if CONFIG_EXT_PARTITION + { 252 }, { 252 }, { 252 }, +#endif // CONFIG_EXT_PARTITION + }; + +#elif !CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + +const aom_tree_index av1_motion_mode_tree[TREE_SIZE(MOTION_MODES)] = { + -SIMPLE_TRANSLATION, -WARPED_CAUSAL +}; + +static const aom_prob default_motion_mode_prob[BLOCK_SIZES][MOTION_MODES - 1] = + { +#if CONFIG_CB4X4 + { 255 }, { 255 }, { 255 }, +#endif + { 255 }, { 255 }, { 255 }, { 151 }, { 153 }, { 144 }, { 178 }, + { 165 }, { 160 }, { 207 }, { 195 }, { 168 }, { 244 }, +#if CONFIG_EXT_PARTITION + { 252 }, { 252 }, { 252 }, +#endif // CONFIG_EXT_PARTITION + }; + +#elif CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + +const aom_tree_index av1_motion_mode_tree[TREE_SIZE(MOTION_MODES)] = { + -SIMPLE_TRANSLATION, 2, -OBMC_CAUSAL, -WARPED_CAUSAL, +}; +static const aom_prob default_motion_mode_prob[BLOCK_SIZES][MOTION_MODES - 1] = + { +#if CONFIG_CB4X4 + { 255, 200 }, { 255, 200 }, { 255, 200 }, +#endif + { 255, 200 }, { 255, 200 }, { 255, 200 }, { 151, 200 }, { 153, 200 }, + { 144, 200 }, { 178, 200 }, { 165, 200 }, { 160, 200 }, { 207, 200 }, + { 195, 200 }, { 168, 200 }, { 244, 200 }, +#if CONFIG_EXT_PARTITION + { 252, 200 }, { 252, 200 }, { 252, 200 }, +#endif // CONFIG_EXT_PARTITION + }; + +// Probability for the case that only 1 additional motion mode is allowed +static const aom_prob default_obmc_prob[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 255, 255, 255, +#endif + 255, 255, 255, 151, 153, 144, 178, 165, 160, 207, 195, 168, 244, +#if CONFIG_EXT_PARTITION + 252, 252, 252, +#endif // CONFIG_EXT_PARTITION +}; +#endif + +#if CONFIG_DELTA_Q +static const aom_prob default_delta_q_probs[DELTA_Q_PROBS] = { 220, 220, 220 }; +#if CONFIG_EC_MULTISYMBOL +static const aom_cdf_prob default_delta_q_cdf[CDF_SIZE(DELTA_Q_PROBS + 1)] = { + AOM_ICDF(28160), AOM_ICDF(32120), AOM_ICDF(32677), AOM_ICDF(32768), 0 +}; +#endif +#if CONFIG_EXT_DELTA_Q +static const aom_prob default_delta_lf_probs[DELTA_LF_PROBS] = { 220, 220, + 220 }; +#if CONFIG_EC_MULTISYMBOL +static const aom_cdf_prob default_delta_lf_cdf[CDF_SIZE(DELTA_LF_PROBS + 1)] = { + 28160, 32120, 32677, 32768, 0 +}; +#endif +#endif +#endif +#if CONFIG_EC_MULTISYMBOL +int av1_intra_mode_ind[INTRA_MODES]; +int av1_intra_mode_inv[INTRA_MODES]; +int av1_inter_mode_ind[INTER_MODES]; +int av1_inter_mode_inv[INTER_MODES]; +#if CONFIG_EXT_TX +int av1_ext_tx_intra_ind[EXT_TX_SETS_INTRA][TX_TYPES]; +int av1_ext_tx_intra_inv[EXT_TX_SETS_INTRA][TX_TYPES]; +int av1_ext_tx_inter_ind[EXT_TX_SETS_INTER][TX_TYPES]; +int av1_ext_tx_inter_inv[EXT_TX_SETS_INTER][TX_TYPES]; +#endif +#endif + +#if CONFIG_ALT_INTRA +const aom_tree_index av1_intra_mode_tree[TREE_SIZE(INTRA_MODES)] = { + -DC_PRED, 2, /* 0 = DC_NODE */ + -TM_PRED, 4, /* 1 = TM_NODE */ + -V_PRED, 6, /* 2 = V_NODE */ + 8, 12, /* 3 = COM_NODE */ + -H_PRED, 10, /* 4 = H_NODE */ + -D135_PRED, -D117_PRED, /* 5 = D135_NODE */ + -D45_PRED, 14, /* 6 = D45_NODE */ + -D63_PRED, 16, /* 7 = D63_NODE */ + -D153_PRED, 18, /* 8 = D153_NODE */ + -D207_PRED, -SMOOTH_PRED, /* 9 = D207_NODE */ +}; +#else +const aom_tree_index av1_intra_mode_tree[TREE_SIZE(INTRA_MODES)] = { + -DC_PRED, 2, /* 0 = DC_NODE */ + -TM_PRED, 4, /* 1 = TM_NODE */ + -V_PRED, 6, /* 2 = V_NODE */ + 8, 12, /* 3 = COM_NODE */ + -H_PRED, 10, /* 4 = H_NODE */ + -D135_PRED, -D117_PRED, /* 5 = D135_NODE */ + -D45_PRED, 14, /* 6 = D45_NODE */ + -D63_PRED, 16, /* 7 = D63_NODE */ + -D153_PRED, -D207_PRED /* 8 = D153_NODE */ +}; +#endif // CONFIG_ALT_INTRA + +const aom_tree_index av1_inter_mode_tree[TREE_SIZE(INTER_MODES)] = { + -INTER_OFFSET(ZEROMV), 2, -INTER_OFFSET(NEARESTMV), 4, -INTER_OFFSET(NEARMV), + -INTER_OFFSET(NEWMV) +}; + +#if CONFIG_EXT_INTER +/* clang-format off */ +const aom_tree_index av1_interintra_mode_tree[TREE_SIZE(INTERINTRA_MODES)] = { + -II_DC_PRED, 2, /* 0 = II_DC_NODE */ + -II_TM_PRED, 4, /* 1 = II_TM_NODE */ + -II_V_PRED, 6, /* 2 = II_V_NODE */ + 8, 12, /* 3 = II_COM_NODE */ + -II_H_PRED, 10, /* 4 = II_H_NODE */ + -II_D135_PRED, -II_D117_PRED, /* 5 = II_D135_NODE */ + -II_D45_PRED, 14, /* 6 = II_D45_NODE */ + -II_D63_PRED, 16, /* 7 = II_D63_NODE */ + -II_D153_PRED, -II_D207_PRED /* 8 = II_D153_NODE */ +}; + +const aom_tree_index av1_inter_compound_mode_tree + [TREE_SIZE(INTER_COMPOUND_MODES)] = { + -INTER_COMPOUND_OFFSET(ZERO_ZEROMV), 2, + -INTER_COMPOUND_OFFSET(NEAREST_NEARESTMV), 4, + 6, -INTER_COMPOUND_OFFSET(NEW_NEWMV), + 8, 12, + -INTER_COMPOUND_OFFSET(NEAR_NEARMV), 10, + -INTER_COMPOUND_OFFSET(NEAREST_NEARMV), + -INTER_COMPOUND_OFFSET(NEAR_NEARESTMV), + 14, 16, + -INTER_COMPOUND_OFFSET(NEAREST_NEWMV), -INTER_COMPOUND_OFFSET(NEW_NEARESTMV), + -INTER_COMPOUND_OFFSET(NEAR_NEWMV), -INTER_COMPOUND_OFFSET(NEW_NEARMV) +}; + +#if CONFIG_COMPOUND_SINGLEREF +const aom_tree_index av1_inter_singleref_comp_mode_tree + [TREE_SIZE(INTER_SINGLEREF_COMP_MODES)] = { + -INTER_SINGLEREF_COMP_OFFSET(SR_ZERO_NEWMV), 2, + -INTER_SINGLEREF_COMP_OFFSET(SR_NEAREST_NEARMV), 4, + 6, -INTER_SINGLEREF_COMP_OFFSET(SR_NEW_NEWMV), + -INTER_SINGLEREF_COMP_OFFSET(SR_NEAREST_NEWMV), + -INTER_SINGLEREF_COMP_OFFSET(SR_NEAR_NEWMV) +}; +#endif // CONFIG_COMPOUND_SINGLEREF + +#if CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE +const aom_tree_index av1_compound_type_tree[TREE_SIZE(COMPOUND_TYPES)] = { + -COMPOUND_AVERAGE, 2, -COMPOUND_WEDGE, -COMPOUND_SEG +}; +#elif !CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE +const aom_tree_index av1_compound_type_tree[TREE_SIZE(COMPOUND_TYPES)] = { + -COMPOUND_AVERAGE, -COMPOUND_WEDGE +}; +#elif CONFIG_COMPOUND_SEGMENT && !CONFIG_WEDGE +const aom_tree_index av1_compound_type_tree[TREE_SIZE(COMPOUND_TYPES)] = { + -COMPOUND_AVERAGE, -COMPOUND_SEG +}; +#else +const aom_tree_index av1_compound_type_tree[TREE_SIZE(COMPOUND_TYPES)] = {}; +#endif // CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE +/* clang-format on */ +#endif // CONFIG_EXT_INTER + +const aom_tree_index av1_partition_tree[TREE_SIZE(PARTITION_TYPES)] = { + -PARTITION_NONE, 2, -PARTITION_HORZ, 4, -PARTITION_VERT, -PARTITION_SPLIT +}; + +#if CONFIG_EXT_PARTITION_TYPES +/* clang-format off */ +const aom_tree_index av1_ext_partition_tree[TREE_SIZE(EXT_PARTITION_TYPES)] = { + -PARTITION_NONE, 2, + 6, 4, + 8, -PARTITION_SPLIT, + -PARTITION_HORZ, 10, + -PARTITION_VERT, 12, + -PARTITION_HORZ_A, -PARTITION_HORZ_B, + -PARTITION_VERT_A, -PARTITION_VERT_B +}; +/* clang-format on */ +#endif // CONFIG_EXT_PARTITION_TYPES + +static const aom_prob default_intra_inter_p[INTRA_INTER_CONTEXTS] = { + 9, 102, 187, 225 +}; + +static const aom_prob default_comp_inter_p[COMP_INTER_CONTEXTS] = { + 239, 183, 119, 96, 41 +}; + +#if CONFIG_EXT_REFS +static const aom_prob default_comp_ref_p[REF_CONTEXTS][FWD_REFS - 1] = { + // TODO(zoeliu): To adjust the initial prob values. + { 33, 16, 16 }, + { 77, 74, 74 }, + { 142, 142, 142 }, + { 172, 170, 170 }, + { 238, 247, 247 } +}; +static const aom_prob default_comp_bwdref_p[REF_CONTEXTS][BWD_REFS - 1] = { + { 16 }, { 74 }, { 142 }, { 170 }, { 247 } +}; +#else +static const aom_prob default_comp_ref_p[REF_CONTEXTS][COMP_REFS - 1] = { + { 50 }, { 126 }, { 123 }, { 221 }, { 226 } +}; +#endif // CONFIG_EXT_REFS + +static const aom_prob default_single_ref_p[REF_CONTEXTS][SINGLE_REFS - 1] = { +#if CONFIG_EXT_REFS + { 33, 16, 16, 16, 16 }, + { 77, 74, 74, 74, 74 }, + { 142, 142, 142, 142, 142 }, + { 172, 170, 170, 170, 170 }, + { 238, 247, 247, 247, 247 } +#else + { 33, 16 }, { 77, 74 }, { 142, 142 }, { 172, 170 }, { 238, 247 } +#endif // CONFIG_EXT_REFS +}; + +#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF +// TODO(zoeliu): Default values to be further adjusted based on the collected +// stats. +static const aom_prob default_comp_inter_mode_p[COMP_INTER_MODE_CONTEXTS] = { + 41, 119, 187, 225 +}; +#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF + +#if CONFIG_PALETTE + +// Tree to code palette size (number of colors in a palette) and the +// corresponding probabilities for Y and UV planes. +const aom_tree_index av1_palette_size_tree[TREE_SIZE(PALETTE_SIZES)] = { + -TWO_COLORS, 2, -THREE_COLORS, 4, -FOUR_COLORS, 6, + -FIVE_COLORS, 8, -SIX_COLORS, 10, -SEVEN_COLORS, -EIGHT_COLORS, +}; + +// TODO(huisu): tune these probs +const aom_prob + av1_default_palette_y_size_prob[PALETTE_BLOCK_SIZES][PALETTE_SIZES - 1] = { + { 96, 89, 100, 64, 77, 130 }, { 22, 15, 44, 16, 34, 82 }, + { 30, 19, 57, 18, 38, 86 }, { 94, 36, 104, 23, 43, 92 }, + { 116, 76, 107, 46, 65, 105 }, { 112, 82, 94, 40, 70, 112 }, + { 147, 124, 123, 58, 69, 103 }, { 180, 113, 136, 49, 45, 114 }, + { 107, 70, 87, 49, 154, 156 }, { 98, 105, 142, 63, 64, 152 }, +#if CONFIG_EXT_PARTITION + { 98, 105, 142, 63, 64, 152 }, { 98, 105, 142, 63, 64, 152 }, + { 98, 105, 142, 63, 64, 152 }, +#endif // CONFIG_EXT_PARTITION + }; + +const aom_prob + av1_default_palette_uv_size_prob[PALETTE_BLOCK_SIZES][PALETTE_SIZES - 1] = { + { 160, 196, 228, 213, 175, 230 }, { 87, 148, 208, 141, 166, 163 }, + { 72, 151, 204, 139, 155, 161 }, { 78, 135, 171, 104, 120, 173 }, + { 59, 92, 131, 78, 92, 142 }, { 75, 118, 149, 84, 90, 128 }, + { 89, 87, 92, 66, 66, 128 }, { 67, 53, 54, 55, 66, 93 }, + { 120, 130, 83, 171, 75, 214 }, { 72, 55, 66, 68, 79, 107 }, +#if CONFIG_EXT_PARTITION + { 72, 55, 66, 68, 79, 107 }, { 72, 55, 66, 68, 79, 107 }, + { 72, 55, 66, 68, 79, 107 }, +#endif // CONFIG_EXT_PARTITION + }; + +// When palette mode is enabled, following probability tables indicate the +// probabilities to code the "is_palette" bit (i.e. the bit that indicates +// if this block uses palette mode or DC_PRED mode). +const aom_prob av1_default_palette_y_mode_prob + [PALETTE_BLOCK_SIZES][PALETTE_Y_MODE_CONTEXTS] = { + { 240, 180, 100 }, { 240, 180, 100 }, { 240, 180, 100 }, + { 240, 180, 100 }, { 240, 180, 100 }, { 240, 180, 100 }, + { 240, 180, 100 }, { 240, 180, 100 }, { 240, 180, 100 }, + { 240, 180, 100 }, +#if CONFIG_EXT_PARTITION + { 240, 180, 100 }, { 240, 180, 100 }, { 240, 180, 100 }, +#endif // CONFIG_EXT_PARTITION + }; + +const aom_prob av1_default_palette_uv_mode_prob[PALETTE_UV_MODE_CONTEXTS] = { + 253, 229 +}; + +// Trees to code palette color indices (for various palette sizes), and the +// corresponding probability tables for Y and UV planes. +const aom_tree_index + av1_palette_color_index_tree[PALETTE_SIZES][TREE_SIZE(PALETTE_COLORS)] = { + { // 2 colors + -PALETTE_COLOR_ONE, -PALETTE_COLOR_TWO }, + { // 3 colors + -PALETTE_COLOR_ONE, 2, -PALETTE_COLOR_TWO, -PALETTE_COLOR_THREE }, + { // 4 colors + -PALETTE_COLOR_ONE, 2, -PALETTE_COLOR_TWO, 4, -PALETTE_COLOR_THREE, + -PALETTE_COLOR_FOUR }, + { // 5 colors + -PALETTE_COLOR_ONE, 2, -PALETTE_COLOR_TWO, 4, -PALETTE_COLOR_THREE, 6, + -PALETTE_COLOR_FOUR, -PALETTE_COLOR_FIVE }, + { // 6 colors + -PALETTE_COLOR_ONE, 2, -PALETTE_COLOR_TWO, 4, -PALETTE_COLOR_THREE, 6, + -PALETTE_COLOR_FOUR, 8, -PALETTE_COLOR_FIVE, -PALETTE_COLOR_SIX }, + { // 7 colors + -PALETTE_COLOR_ONE, 2, -PALETTE_COLOR_TWO, 4, -PALETTE_COLOR_THREE, 6, + -PALETTE_COLOR_FOUR, 8, -PALETTE_COLOR_FIVE, 10, -PALETTE_COLOR_SIX, + -PALETTE_COLOR_SEVEN }, + { // 8 colors + -PALETTE_COLOR_ONE, 2, -PALETTE_COLOR_TWO, 4, -PALETTE_COLOR_THREE, 6, + -PALETTE_COLOR_FOUR, 8, -PALETTE_COLOR_FIVE, 10, -PALETTE_COLOR_SIX, 12, + -PALETTE_COLOR_SEVEN, -PALETTE_COLOR_EIGHT }, + }; + +// Note: Has to be non-zero to avoid any asserts triggering. +#define UNUSED_PROB 128 + +const aom_prob av1_default_palette_y_color_index_prob + [PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS][PALETTE_COLORS - 1] = { + { + // 2 colors + { 231, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB, UNUSED_PROB }, + { UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB, UNUSED_PROB }, + { 69, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB }, + { 224, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB, UNUSED_PROB }, + { 249, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB, UNUSED_PROB }, + }, + { + // 3 colors + { 219, 124, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB }, + { 91, 191, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB }, + { 34, 237, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB }, + { 184, 118, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB }, + { 252, 124, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB }, + }, + { + // 4 colors + { 204, 87, 97, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 74, 144, 129, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 52, 191, 134, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 151, 85, 147, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 248, 60, 115, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + }, + { + // 5 colors + { 218, 69, 62, 106, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 76, 143, 89, 127, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 21, 233, 94, 131, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 172, 72, 89, 112, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 253, 66, 65, 128, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + }, + { + // 6 colors + { 190, 60, 47, 54, 74, UNUSED_PROB, UNUSED_PROB }, + { 62, 106, 51, 95, 110, UNUSED_PROB, UNUSED_PROB }, + { 52, 180, 69, 72, 107, UNUSED_PROB, UNUSED_PROB }, + { 156, 83, 72, 83, 101, UNUSED_PROB, UNUSED_PROB }, + { 245, 45, 37, 52, 91, UNUSED_PROB, UNUSED_PROB }, + }, + { + // 7 colors + { 206, 56, 42, 42, 53, 85, UNUSED_PROB }, + { 70, 100, 45, 68, 77, 94, UNUSED_PROB }, + { 57, 169, 51, 62, 74, 119, UNUSED_PROB }, + { 172, 76, 71, 40, 59, 76, UNUSED_PROB }, + { 248, 47, 36, 53, 61, 110, UNUSED_PROB }, + }, + { + // 8 colors + { 208, 52, 38, 34, 34, 44, 66 }, + { 52, 107, 34, 73, 69, 82, 87 }, + { 28, 208, 53, 43, 62, 70, 102 }, + { 184, 64, 45, 37, 37, 69, 105 }, + { 251, 18, 31, 45, 47, 61, 104 }, + }, + }; + +const aom_prob av1_default_palette_uv_color_index_prob + [PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS][PALETTE_COLORS - 1] = { + { + // 2 colors + { 233, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB, UNUSED_PROB }, + { UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB, UNUSED_PROB }, + { 69, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB }, + { 240, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB, UNUSED_PROB }, + { 248, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB, UNUSED_PROB }, + }, + { + // 3 colors + { 216, 128, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB }, + { 110, 171, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB }, + { 40, 239, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB }, + { 191, 104, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB }, + { 247, 134, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, + UNUSED_PROB }, + }, + { + // 4 colors + { 202, 89, 132, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 90, 132, 136, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 63, 195, 149, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 152, 84, 152, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 241, 87, 136, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + }, + { + // 5 colors + { 209, 54, 82, 134, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 94, 173, 180, 93, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 10, 251, 127, 84, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 183, 20, 150, 47, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + { 252, 73, 111, 150, UNUSED_PROB, UNUSED_PROB, UNUSED_PROB }, + }, + { + // 6 colors + { 192, 67, 59, 46, 184, UNUSED_PROB, UNUSED_PROB }, + { 59, 92, 61, 100, 130, UNUSED_PROB, UNUSED_PROB }, + { 49, 162, 68, 91, 150, UNUSED_PROB, UNUSED_PROB }, + { 133, 29, 36, 153, 101, UNUSED_PROB, UNUSED_PROB }, + { 247, 71, 44, 90, 129, UNUSED_PROB, UNUSED_PROB }, + }, + { + // 7 colors + { 182, 62, 80, 78, 46, 116, UNUSED_PROB }, + { 59, 62, 39, 81, 65, 99, UNUSED_PROB }, + { 54, 177, 48, 58, 93, 104, UNUSED_PROB }, + { 137, 79, 54, 55, 44, 134, UNUSED_PROB }, + { 239, 82, 79, 44, 69, 71, UNUSED_PROB }, + }, + { + // 8 colors + { 172, 53, 27, 67, 30, 79, 113 }, + { 63, 57, 45, 81, 62, 35, 47 }, + { 51, 200, 36, 47, 82, 165, 129 }, + { 141, 100, 47, 29, 33, 37, 129 }, + { 236, 42, 50, 91, 24, 154, 65 }, + }, + }; + +#undef UNUSED_PROB + +#define MAX_COLOR_CONTEXT_HASH 8 +// Negative values are invalid +static const int palette_color_index_context_lookup[MAX_COLOR_CONTEXT_HASH + + 1] = { -1, -1, 0, -1, -1, + 4, 3, 2, 1 }; + +#endif // CONFIG_PALETTE + +// The transform size is coded as an offset to the smallest transform +// block size. +const aom_tree_index av1_tx_size_tree[MAX_TX_DEPTH][TREE_SIZE(TX_SIZES)] = { + { + // Max tx_size is 8X8 + -0, -1, + }, + { + // Max tx_size is 16X16 + -0, 2, -1, -2, + }, + { + // Max tx_size is 32X32 + -0, 2, -1, 4, -2, -3, + }, +#if CONFIG_TX64X64 + { + // Max tx_size is 64X64 + -0, 2, -1, 4, -2, 6, -3, -4, + }, +#endif // CONFIG_TX64X64 +}; + +static const aom_prob default_tx_size_prob[MAX_TX_DEPTH][TX_SIZE_CONTEXTS] + [MAX_TX_DEPTH] = { + { + // Max tx_size is 8X8 + { 100 }, + { 66 }, + }, + { + // Max tx_size is 16X16 + { 20, 152 }, + { 15, 101 }, + }, + { + // Max tx_size is 32X32 + { 3, 136, 37 }, + { 5, 52, 13 }, + }, +#if CONFIG_TX64X64 + { + // Max tx_size is 64X64 + { 1, 64, 136, 127 }, + { 1, 32, 52, 67 }, + }, +#endif // CONFIG_TX64X64 + }; + +#if CONFIG_LOOP_RESTORATION +const aom_tree_index + av1_switchable_restore_tree[TREE_SIZE(RESTORE_SWITCHABLE_TYPES)] = { + -RESTORE_NONE, 2, -RESTORE_WIENER, -RESTORE_SGRPROJ, + }; + +static const aom_prob + default_switchable_restore_prob[RESTORE_SWITCHABLE_TYPES - 1] = { + 32, 128, + }; +#endif // CONFIG_LOOP_RESTORATION + +#if CONFIG_PALETTE +#define NUM_PALETTE_NEIGHBORS 3 // left, top-left and top. +int av1_get_palette_color_index_context(const uint8_t *color_map, int stride, + int r, int c, int palette_size, + uint8_t *color_order, int *color_idx) { + int i; + // The +10 below should not be needed. But we get a warning "array subscript + // is above array bounds [-Werror=array-bounds]" without it, possibly due to + // this (or similar) bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 + int scores[PALETTE_MAX_SIZE + 10]; + const int weights[NUM_PALETTE_NEIGHBORS] = { 2, 1, 2 }; + const int hash_multipliers[NUM_PALETTE_NEIGHBORS] = { 1, 2, 2 }; + int color_index_ctx_hash; + int color_index_ctx; + int color_neighbors[NUM_PALETTE_NEIGHBORS]; + int inverse_color_order[PALETTE_MAX_SIZE]; + assert(palette_size <= PALETTE_MAX_SIZE); + assert(r > 0 || c > 0); + + // Get color indices of neighbors. + color_neighbors[0] = (c - 1 >= 0) ? color_map[r * stride + c - 1] : -1; + color_neighbors[1] = + (c - 1 >= 0 && r - 1 >= 0) ? color_map[(r - 1) * stride + c - 1] : -1; + color_neighbors[2] = (r - 1 >= 0) ? color_map[(r - 1) * stride + c] : -1; + + for (i = 0; i < PALETTE_MAX_SIZE; ++i) { + color_order[i] = i; + inverse_color_order[i] = i; + } + memset(scores, 0, PALETTE_MAX_SIZE * sizeof(scores[0])); + for (i = 0; i < NUM_PALETTE_NEIGHBORS; ++i) { + if (color_neighbors[i] >= 0) { + scores[color_neighbors[i]] += weights[i]; + } + } + + // Get the top NUM_PALETTE_NEIGHBORS scores (sorted from large to small). + for (i = 0; i < NUM_PALETTE_NEIGHBORS; ++i) { + int max = scores[i]; + int max_idx = i; + int j; + for (j = i + 1; j < palette_size; ++j) { + if (scores[j] > max) { + max = scores[j]; + max_idx = j; + } + } + if (max_idx != i) { + // Move the score at index 'max_idx' to index 'i', and shift the scores + // from 'i' to 'max_idx - 1' by 1. + const int max_score = scores[max_idx]; + const uint8_t max_color_order = color_order[max_idx]; + int k; + for (k = max_idx; k > i; --k) { + scores[k] = scores[k - 1]; + color_order[k] = color_order[k - 1]; + inverse_color_order[color_order[k]] = k; + } + scores[i] = max_score; + color_order[i] = max_color_order; + inverse_color_order[color_order[i]] = i; + } + } + + // Get hash value of context. + color_index_ctx_hash = 0; + for (i = 0; i < NUM_PALETTE_NEIGHBORS; ++i) { + color_index_ctx_hash += scores[i] * hash_multipliers[i]; + } + assert(color_index_ctx_hash > 0); + assert(color_index_ctx_hash <= MAX_COLOR_CONTEXT_HASH); + + // Lookup context from hash. + color_index_ctx = palette_color_index_context_lookup[color_index_ctx_hash]; + assert(color_index_ctx >= 0); + assert(color_index_ctx < PALETTE_COLOR_INDEX_CONTEXTS); + + if (color_idx != NULL) { + *color_idx = inverse_color_order[color_map[r * stride + c]]; + } + return color_index_ctx; +} +#undef NUM_PALETTE_NEIGHBORS +#undef MAX_COLOR_CONTEXT_HASH + +#endif // CONFIG_PALETTE + +#if CONFIG_VAR_TX +static const aom_prob default_txfm_partition_probs[TXFM_PARTITION_CONTEXTS] = { + 250, 231, 212, 241, 166, 66, 241, 230, 135, 243, 154, 64, 248, 161, 63, 128, +}; +#endif + +static const aom_prob default_skip_probs[SKIP_CONTEXTS] = { 192, 128, 64 }; + +#if CONFIG_DUAL_FILTER +static const aom_prob default_switchable_interp_prob + [SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS - 1] = { + { 235, 192, 128 }, { 36, 243, 48 }, { 34, 16, 128 }, + { 34, 16, 128 }, { 149, 160, 128 }, { 235, 192, 128 }, + { 36, 243, 48 }, { 34, 16, 128 }, { 34, 16, 128 }, + { 149, 160, 128 }, { 235, 192, 128 }, { 36, 243, 48 }, + { 34, 16, 128 }, { 34, 16, 128 }, { 149, 160, 128 }, + { 235, 192, 128 }, { 36, 243, 48 }, { 34, 16, 128 }, + { 34, 16, 128 }, { 149, 160, 128 }, + }; +#else // CONFIG_DUAL_FILTER +static const aom_prob default_switchable_interp_prob[SWITCHABLE_FILTER_CONTEXTS] + [SWITCHABLE_FILTERS - 1] = { + { 235, 162 }, + { 36, 255 }, + { 34, 3 }, + { 149, 144 }, + }; +#endif // CONFIG_DUAL_FILTER + +#if CONFIG_EXT_TX +/* clang-format off */ +const aom_tree_index av1_ext_tx_inter_tree[EXT_TX_SETS_INTER] + [TREE_SIZE(TX_TYPES)] = { + { // ToDo(yaowu): remove used entry 0. + 0 + }, { + -IDTX, 2, + 4, 14, + 6, 8, + -V_DCT, -H_DCT, + 10, 12, + -V_ADST, -H_ADST, + -V_FLIPADST, -H_FLIPADST, + -DCT_DCT, 16, + 18, 24, + 20, 22, + -ADST_DCT, -DCT_ADST, + -FLIPADST_DCT, -DCT_FLIPADST, + 26, 28, + -ADST_ADST, -FLIPADST_FLIPADST, + -ADST_FLIPADST, -FLIPADST_ADST + }, { + -IDTX, 2, + 4, 6, + -V_DCT, -H_DCT, + -DCT_DCT, 8, + 10, 16, + 12, 14, + -ADST_DCT, -DCT_ADST, + -FLIPADST_DCT, -DCT_FLIPADST, + 18, 20, + -ADST_ADST, -FLIPADST_FLIPADST, + -ADST_FLIPADST, -FLIPADST_ADST + }, { + -IDTX, -DCT_DCT, + } +}; + +const aom_tree_index av1_ext_tx_intra_tree[EXT_TX_SETS_INTRA] + [TREE_SIZE(TX_TYPES)] = { + { // ToDo(yaowu): remove unused entry 0. + 0 + }, { + -IDTX, 2, + -DCT_DCT, 4, + 6, 8, + -V_DCT, -H_DCT, + -ADST_ADST, 10, + -ADST_DCT, -DCT_ADST, + }, { + -IDTX, 2, + -DCT_DCT, 4, + -ADST_ADST, 6, + -ADST_DCT, -DCT_ADST, + } +}; +/* clang-format on */ + +static const aom_prob + default_inter_ext_tx_prob[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES - 1] = { + { +// ToDo(yaowu): remove unused entry 0. +#if CONFIG_CB4X4 + { 0 }, +#endif + { 0 }, + { 0 }, + { 0 }, + { 0 }, + }, + { +#if CONFIG_CB4X4 + { 0 }, +#endif + { 10, 24, 30, 128, 128, 128, 128, 112, 160, 128, 128, 128, 128, 128, + 128 }, + { 10, 24, 30, 128, 128, 128, 128, 112, 160, 128, 128, 128, 128, 128, + 128 }, + { 10, 24, 30, 128, 128, 128, 128, 112, 160, 128, 128, 128, 128, 128, + 128 }, + { 10, 24, 30, 128, 128, 128, 128, 112, 160, 128, 128, 128, 128, 128, + 128 }, + }, + { +#if CONFIG_CB4X4 + { 0 }, +#endif + { 10, 30, 128, 112, 160, 128, 128, 128, 128, 128, 128 }, + { 10, 30, 128, 112, 160, 128, 128, 128, 128, 128, 128 }, + { 10, 30, 128, 112, 160, 128, 128, 128, 128, 128, 128 }, + { 10, 30, 128, 112, 160, 128, 128, 128, 128, 128, 128 }, + }, + { +#if CONFIG_CB4X4 + { 0 }, +#endif + { 12 }, + { 12 }, + { 12 }, + { 12 }, + } + }; + +// TODO(urvang): 3rd context should be tx_type instead of intra mode just like +// the baseline. +static const aom_prob + default_intra_ext_tx_prob[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES] + [TX_TYPES - 1] = { + { +// ToDo(yaowu): remove unused entry 0. +#if CONFIG_CB4X4 + { + { 0 }, + }, +#endif + { + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, +#if CONFIG_ALT_INTRA + { 0 }, +#endif // CONFIG_ALT_INTRA + }, + { + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, +#if CONFIG_ALT_INTRA + { 0 }, +#endif // CONFIG_ALT_INTRA + }, + { + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, +#if CONFIG_ALT_INTRA + { 0 }, +#endif // CONFIG_ALT_INTRA + }, + { + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, +#if CONFIG_ALT_INTRA + { 0 }, +#endif // CONFIG_ALT_INTRA + }, + }, + { +#if CONFIG_CB4X4 + { + { 0 }, + }, +#endif + { + { 8, 224, 32, 128, 64, 128 }, + { 10, 32, 32, 128, 16, 192 }, + { 10, 32, 32, 128, 16, 64 }, + { 9, 200, 32, 128, 64, 128 }, + { 8, 8, 32, 128, 224, 128 }, + { 10, 32, 32, 128, 16, 192 }, + { 10, 32, 32, 128, 16, 64 }, + { 10, 23, 32, 128, 80, 176 }, + { 10, 23, 32, 128, 80, 176 }, + { 10, 32, 32, 128, 16, 64 }, +#if CONFIG_ALT_INTRA + { 10, 32, 32, 128, 16, 64 }, +#endif // CONFIG_ALT_INTRA + }, + { + { 8, 224, 32, 128, 64, 128 }, + { 10, 32, 32, 128, 16, 192 }, + { 10, 32, 32, 128, 16, 64 }, + { 9, 200, 32, 128, 64, 128 }, + { 8, 8, 32, 128, 224, 128 }, + { 10, 32, 32, 128, 16, 192 }, + { 10, 32, 32, 128, 16, 64 }, + { 10, 23, 32, 128, 80, 176 }, + { 10, 23, 32, 128, 80, 176 }, + { 10, 32, 32, 128, 16, 64 }, +#if CONFIG_ALT_INTRA + { 10, 32, 32, 128, 16, 64 }, +#endif // CONFIG_ALT_INTRA + }, + { + { 8, 224, 32, 128, 64, 128 }, + { 10, 32, 32, 128, 16, 192 }, + { 10, 32, 32, 128, 16, 64 }, + { 9, 200, 32, 128, 64, 128 }, + { 8, 8, 32, 128, 224, 128 }, + { 10, 32, 32, 128, 16, 192 }, + { 10, 32, 32, 128, 16, 64 }, + { 10, 23, 32, 128, 80, 176 }, + { 10, 23, 32, 128, 80, 176 }, + { 10, 32, 32, 128, 16, 64 }, +#if CONFIG_ALT_INTRA + { 10, 32, 32, 128, 16, 64 }, +#endif // CONFIG_ALT_INTRA + }, + { + { 8, 224, 32, 128, 64, 128 }, + { 10, 32, 32, 128, 16, 192 }, + { 10, 32, 32, 128, 16, 64 }, + { 9, 200, 32, 128, 64, 128 }, + { 8, 8, 32, 128, 224, 128 }, + { 10, 32, 32, 128, 16, 192 }, + { 10, 32, 32, 128, 16, 64 }, + { 10, 23, 32, 128, 80, 176 }, + { 10, 23, 32, 128, 80, 176 }, + { 10, 32, 32, 128, 16, 64 }, +#if CONFIG_ALT_INTRA + { 10, 32, 32, 128, 16, 64 }, +#endif // CONFIG_ALT_INTRA + }, + }, + { +#if CONFIG_CB4X4 + { + { 0 }, + }, +#endif + { + { 8, 224, 64, 128 }, + { 10, 32, 16, 192 }, + { 10, 32, 16, 64 }, + { 9, 200, 64, 128 }, + { 8, 8, 224, 128 }, + { 10, 32, 16, 192 }, + { 10, 32, 16, 64 }, + { 10, 23, 80, 176 }, + { 10, 23, 80, 176 }, + { 10, 32, 16, 64 }, +#if CONFIG_ALT_INTRA + { 10, 32, 16, 64 }, +#endif // CONFIG_ALT_INTRA + }, + { + { 8, 224, 64, 128 }, + { 10, 32, 16, 192 }, + { 10, 32, 16, 64 }, + { 9, 200, 64, 128 }, + { 8, 8, 224, 128 }, + { 10, 32, 16, 192 }, + { 10, 32, 16, 64 }, + { 10, 23, 80, 176 }, + { 10, 23, 80, 176 }, + { 10, 32, 16, 64 }, +#if CONFIG_ALT_INTRA + { 10, 32, 16, 64 }, +#endif // CONFIG_ALT_INTRA + }, + { + { 8, 224, 64, 128 }, + { 10, 32, 16, 192 }, + { 10, 32, 16, 64 }, + { 9, 200, 64, 128 }, + { 8, 8, 224, 128 }, + { 10, 32, 16, 192 }, + { 10, 32, 16, 64 }, + { 10, 23, 80, 176 }, + { 10, 23, 80, 176 }, + { 10, 32, 16, 64 }, +#if CONFIG_ALT_INTRA + { 10, 32, 16, 64 }, +#endif // CONFIG_ALT_INTRA + }, + { + { 8, 224, 64, 128 }, + { 10, 32, 16, 192 }, + { 10, 32, 16, 64 }, + { 9, 200, 64, 128 }, + { 8, 8, 224, 128 }, + { 10, 32, 16, 192 }, + { 10, 32, 16, 64 }, + { 10, 23, 80, 176 }, + { 10, 23, 80, 176 }, + { 10, 32, 16, 64 }, +#if CONFIG_ALT_INTRA + { 10, 32, 16, 64 }, +#endif // CONFIG_ALT_INTRA + }, + }, + }; +#else // !CONFIG_EXT_TX + +/* clang-format off */ +const aom_tree_index av1_ext_tx_tree[TREE_SIZE(TX_TYPES)] = { + -DCT_DCT, 2, + -ADST_ADST, 4, + -ADST_DCT, -DCT_ADST +}; +/* clang-format on */ + +int av1_ext_tx_ind[TX_TYPES]; +int av1_ext_tx_inv[TX_TYPES]; + +static const aom_prob + default_intra_ext_tx_prob[EXT_TX_SIZES][TX_TYPES][TX_TYPES - 1] = { +#if CONFIG_CB4X4 + { { 240, 85, 128 }, { 4, 1, 248 }, { 4, 1, 8 }, { 4, 248, 128 } }, +#endif + { { 240, 85, 128 }, { 4, 1, 248 }, { 4, 1, 8 }, { 4, 248, 128 } }, + { { 244, 85, 128 }, { 8, 2, 248 }, { 8, 2, 8 }, { 8, 248, 128 } }, + { { 248, 85, 128 }, { 16, 4, 248 }, { 16, 4, 8 }, { 16, 248, 128 } }, + }; + +static const aom_prob default_inter_ext_tx_prob[EXT_TX_SIZES][TX_TYPES - 1] = { +#if CONFIG_CB4X4 + { 160, 85, 128 }, +#endif + { 160, 85, 128 }, + { 176, 85, 128 }, + { 192, 85, 128 }, +}; +#endif // CONFIG_EXT_TX + +#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP +static const aom_prob + default_intra_filter_probs[INTRA_FILTERS + 1][INTRA_FILTERS - 1] = { + { 98, 63, 60 }, { 98, 82, 80 }, { 94, 65, 103 }, + { 49, 25, 24 }, { 72, 38, 50 }, + }; +const aom_tree_index av1_intra_filter_tree[TREE_SIZE(INTRA_FILTERS)] = { + -INTRA_FILTER_LINEAR, 2, -INTRA_FILTER_8TAP, 4, -INTRA_FILTER_8TAP_SHARP, + -INTRA_FILTER_8TAP_SMOOTH, +}; +#if CONFIG_EC_MULTISYMBOL +int av1_intra_filter_ind[INTRA_FILTERS]; +int av1_intra_filter_inv[INTRA_FILTERS]; +#endif // CONFIG_EC_MULTISYMBOL +#endif // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP + +#if CONFIG_FILTER_INTRA +static const aom_prob default_filter_intra_probs[2] = { 230, 230 }; +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_SUPERTX +static const aom_prob + default_supertx_prob[PARTITION_SUPERTX_CONTEXTS][TX_SIZES] = { +#if CONFIG_CB4X4 +#if CONFIG_TX64X64 + { 1, 1, 160, 160, 170, 180 }, { 1, 1, 200, 200, 210, 220 }, +#else + { 1, 1, 160, 160, 170 }, { 1, 1, 200, 200, 210 }, +#endif // CONFIG_TX64X64 +#else +#if CONFIG_TX64X64 + { 1, 160, 160, 170, 180 }, { 1, 200, 200, 210, 220 }, +#else + { 1, 160, 160, 170 }, { 1, 200, 200, 210 }, +#endif // CONFIG_CB4X4 +#endif // CONFIG_TX64X64 + }; +#endif // CONFIG_SUPERTX + +// FIXME(someone) need real defaults here +static const aom_prob default_segment_tree_probs[SEG_TREE_PROBS] = { + 128, 128, 128, 128, 128, 128, 128 +}; +// clang-format off +static const aom_prob default_segment_pred_probs[PREDICTION_PROBS] = { + 128, 128, 128 +}; +// clang-format on + +#if CONFIG_EC_MULTISYMBOL +#if CONFIG_DUAL_FILTER +static const aom_cdf_prob + default_switchable_interp_cdf[SWITCHABLE_FILTER_CONTEXTS][CDF_SIZE( + SWITCHABLE_FILTERS)] = { + { AOM_ICDF(30080), AOM_ICDF(31088), AOM_ICDF(32096), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4608), AOM_ICDF(9620), AOM_ICDF(31338), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4352), AOM_ICDF(5240), AOM_ICDF(6128), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4352), AOM_ICDF(5240), AOM_ICDF(6128), AOM_ICDF(32768), 0 }, + { AOM_ICDF(19072), AOM_ICDF(23352), AOM_ICDF(27632), AOM_ICDF(32768), 0 }, + { AOM_ICDF(30080), AOM_ICDF(31088), AOM_ICDF(32096), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4608), AOM_ICDF(9620), AOM_ICDF(31338), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4352), AOM_ICDF(5240), AOM_ICDF(6128), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4352), AOM_ICDF(5240), AOM_ICDF(6128), AOM_ICDF(32768), 0 }, + { AOM_ICDF(19072), AOM_ICDF(23352), AOM_ICDF(27632), AOM_ICDF(32768), 0 }, + { AOM_ICDF(30080), AOM_ICDF(31088), AOM_ICDF(32096), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4608), AOM_ICDF(9620), AOM_ICDF(31338), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4352), AOM_ICDF(5240), AOM_ICDF(6128), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4352), AOM_ICDF(5240), AOM_ICDF(6128), AOM_ICDF(32768), 0 }, + { AOM_ICDF(19072), AOM_ICDF(23352), AOM_ICDF(27632), AOM_ICDF(32768), 0 }, + { AOM_ICDF(30080), AOM_ICDF(31088), AOM_ICDF(32096), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4608), AOM_ICDF(9620), AOM_ICDF(31338), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4352), AOM_ICDF(5240), AOM_ICDF(6128), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4352), AOM_ICDF(5240), AOM_ICDF(6128), AOM_ICDF(32768), 0 }, + { AOM_ICDF(19072), AOM_ICDF(23352), AOM_ICDF(27632), AOM_ICDF(32768), 0 } + }; +#else +static const aom_cdf_prob + default_switchable_interp_cdf[SWITCHABLE_FILTER_CONTEXTS][CDF_SIZE( + SWITCHABLE_FILTERS)] = { + { AOM_ICDF(30080), AOM_ICDF(31781), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4608), AOM_ICDF(32658), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4352), AOM_ICDF(4685), AOM_ICDF(32768), 0 }, + { AOM_ICDF(19072), AOM_ICDF(26776), AOM_ICDF(32768), 0 }, + }; +#endif + +static const aom_cdf_prob default_seg_tree_cdf[CDF_SIZE(MAX_SEGMENTS)] = { + AOM_ICDF(4096), AOM_ICDF(8192), AOM_ICDF(12288), + AOM_ICDF(16384), AOM_ICDF(20480), AOM_ICDF(24576), + AOM_ICDF(28672), AOM_ICDF(32768), 0 +}; + +static const aom_cdf_prob + default_tx_size_cdf[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][CDF_SIZE(MAX_TX_DEPTH + + 1)] = { + { { AOM_ICDF(12800), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8448), AOM_ICDF(32768), 0 } }, + { { AOM_ICDF(2560), AOM_ICDF(20496), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1920), AOM_ICDF(14091), AOM_ICDF(32768), 0 } }, + { { AOM_ICDF(384), AOM_ICDF(17588), AOM_ICDF(19782), AOM_ICDF(32768), 0 }, + { AOM_ICDF(640), AOM_ICDF(7166), AOM_ICDF(8466), AOM_ICDF(32768), 0 } }, +#if CONFIG_TX64X64 + { { AOM_ICDF(128), AOM_ICDF(8288), AOM_ICDF(21293), AOM_ICDF(26986), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(128), AOM_ICDF(4208), AOM_ICDF(10009), AOM_ICDF(15965), + AOM_ICDF(32768), 0 } }, +#endif + }; + +#if CONFIG_ALT_INTRA +static const aom_cdf_prob + default_if_y_mode_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(INTRA_MODES)] = { + { AOM_ICDF(11264), AOM_ICDF(12608), AOM_ICDF(16309), AOM_ICDF(21086), + AOM_ICDF(23297), AOM_ICDF(24860), AOM_ICDF(27022), AOM_ICDF(28099), + AOM_ICDF(29631), AOM_ICDF(31126), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9600), AOM_ICDF(11953), AOM_ICDF(16100), AOM_ICDF(20922), + AOM_ICDF(22756), AOM_ICDF(23913), AOM_ICDF(25435), AOM_ICDF(26724), + AOM_ICDF(28046), AOM_ICDF(29927), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9344), AOM_ICDF(11540), AOM_ICDF(16515), AOM_ICDF(21763), + AOM_ICDF(23078), AOM_ICDF(23816), AOM_ICDF(24725), AOM_ICDF(25856), + AOM_ICDF(26720), AOM_ICDF(28208), AOM_ICDF(32768), 0 }, + { AOM_ICDF(12288), AOM_ICDF(14448), AOM_ICDF(18026), AOM_ICDF(23346), + AOM_ICDF(23833), AOM_ICDF(24188), AOM_ICDF(24724), AOM_ICDF(25415), + AOM_ICDF(25817), AOM_ICDF(26876), AOM_ICDF(32768), 0 }, + }; + +static const aom_cdf_prob + default_uv_mode_cdf[INTRA_MODES][CDF_SIZE(INTRA_MODES)] = { + { AOM_ICDF(25472), AOM_ICDF(25558), AOM_ICDF(27783), AOM_ICDF(30779), + AOM_ICDF(30988), AOM_ICDF(31269), AOM_ICDF(31492), AOM_ICDF(31741), + AOM_ICDF(32014), AOM_ICDF(32420), AOM_ICDF(32768), 0 }, + { AOM_ICDF(2176), AOM_ICDF(2415), AOM_ICDF(28381), AOM_ICDF(29574), + AOM_ICDF(29832), AOM_ICDF(30712), AOM_ICDF(30881), AOM_ICDF(31662), + AOM_ICDF(31761), AOM_ICDF(31922), AOM_ICDF(32768), 0 }, + { AOM_ICDF(3328), AOM_ICDF(3443), AOM_ICDF(4016), AOM_ICDF(31099), + AOM_ICDF(31272), AOM_ICDF(31420), AOM_ICDF(31504), AOM_ICDF(31608), + AOM_ICDF(31916), AOM_ICDF(32598), AOM_ICDF(32768), 0 }, + { AOM_ICDF(23424), AOM_ICDF(23534), AOM_ICDF(25915), AOM_ICDF(27831), + AOM_ICDF(28058), AOM_ICDF(28431), AOM_ICDF(30142), AOM_ICDF(31209), + AOM_ICDF(31459), AOM_ICDF(32369), AOM_ICDF(32768), 0 }, + { AOM_ICDF(22784), AOM_ICDF(22862), AOM_ICDF(24255), AOM_ICDF(26287), + AOM_ICDF(28490), AOM_ICDF(29509), AOM_ICDF(29776), AOM_ICDF(30115), + AOM_ICDF(31203), AOM_ICDF(31674), AOM_ICDF(32768), 0 }, + { AOM_ICDF(19712), AOM_ICDF(19865), AOM_ICDF(23141), AOM_ICDF(24428), + AOM_ICDF(25731), AOM_ICDF(31377), AOM_ICDF(31622), AOM_ICDF(32047), + AOM_ICDF(32458), AOM_ICDF(32767), AOM_ICDF(32768), 0 }, + { AOM_ICDF(21376), AOM_ICDF(21421), AOM_ICDF(22130), AOM_ICDF(27688), + AOM_ICDF(28485), AOM_ICDF(28779), AOM_ICDF(28935), AOM_ICDF(29085), + AOM_ICDF(31962), AOM_ICDF(32450), AOM_ICDF(32768), 0 }, + { AOM_ICDF(19712), AOM_ICDF(19814), AOM_ICDF(20725), AOM_ICDF(28510), + AOM_ICDF(28814), AOM_ICDF(29099), AOM_ICDF(29457), AOM_ICDF(29729), + AOM_ICDF(30133), AOM_ICDF(32408), AOM_ICDF(32768), 0 }, + { AOM_ICDF(19584), AOM_ICDF(19790), AOM_ICDF(23643), AOM_ICDF(25501), + AOM_ICDF(25913), AOM_ICDF(26673), AOM_ICDF(27578), AOM_ICDF(30923), + AOM_ICDF(31255), AOM_ICDF(31870), AOM_ICDF(32768), 0 }, + { AOM_ICDF(20864), AOM_ICDF(21004), AOM_ICDF(24129), AOM_ICDF(26308), + AOM_ICDF(27062), AOM_ICDF(27065), AOM_ICDF(27488), AOM_ICDF(28045), + AOM_ICDF(28506), AOM_ICDF(29272), AOM_ICDF(32768), 0 }, + { AOM_ICDF(23680), AOM_ICDF(23929), AOM_ICDF(27831), AOM_ICDF(30446), + AOM_ICDF(30598), AOM_ICDF(31129), AOM_ICDF(31244), AOM_ICDF(31655), + AOM_ICDF(31868), AOM_ICDF(32234), AOM_ICDF(32768), 0 }, + }; +#else // !CONFIG_ALT_INTRA +static const aom_cdf_prob + default_if_y_mode_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(INTRA_MODES)] = { + { AOM_ICDF(8320), AOM_ICDF(11376), AOM_ICDF(12880), AOM_ICDF(19959), + AOM_ICDF(23072), AOM_ICDF(24067), AOM_ICDF(25461), AOM_ICDF(26917), + AOM_ICDF(29157), AOM_ICDF(32768), 0 }, + { AOM_ICDF(16896), AOM_ICDF(21112), AOM_ICDF(21932), AOM_ICDF(27852), + AOM_ICDF(28667), AOM_ICDF(28916), AOM_ICDF(29593), AOM_ICDF(30089), + AOM_ICDF(30905), AOM_ICDF(32768), 0 }, + { AOM_ICDF(22144), AOM_ICDF(25464), AOM_ICDF(26006), AOM_ICDF(30364), + AOM_ICDF(30583), AOM_ICDF(30655), AOM_ICDF(31183), AOM_ICDF(31400), + AOM_ICDF(31646), AOM_ICDF(32768), 0 }, + { AOM_ICDF(28288), AOM_ICDF(30650), AOM_ICDF(30964), AOM_ICDF(32288), + AOM_ICDF(32308), AOM_ICDF(32331), AOM_ICDF(32495), AOM_ICDF(32586), + AOM_ICDF(32607), AOM_ICDF(32768), 0 }, + }; + +static const aom_cdf_prob + default_uv_mode_cdf[INTRA_MODES][CDF_SIZE(INTRA_MODES)] = { + { AOM_ICDF(15360), AOM_ICDF(15836), AOM_ICDF(20863), AOM_ICDF(27513), + AOM_ICDF(28269), AOM_ICDF(29048), AOM_ICDF(29455), AOM_ICDF(30154), + AOM_ICDF(31206), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6144), AOM_ICDF(7392), AOM_ICDF(22657), AOM_ICDF(25981), + AOM_ICDF(26965), AOM_ICDF(28779), AOM_ICDF(29309), AOM_ICDF(30890), + AOM_ICDF(31763), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8576), AOM_ICDF(9143), AOM_ICDF(11450), AOM_ICDF(27575), + AOM_ICDF(28108), AOM_ICDF(28438), AOM_ICDF(28658), AOM_ICDF(28995), + AOM_ICDF(30410), AOM_ICDF(32768), 0 }, + { AOM_ICDF(12416), AOM_ICDF(12814), AOM_ICDF(16244), AOM_ICDF(22057), + AOM_ICDF(23492), AOM_ICDF(24700), AOM_ICDF(26213), AOM_ICDF(27954), + AOM_ICDF(29778), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10624), AOM_ICDF(11057), AOM_ICDF(14619), AOM_ICDF(19415), + AOM_ICDF(23134), AOM_ICDF(25679), AOM_ICDF(26399), AOM_ICDF(27618), + AOM_ICDF(30676), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10240), AOM_ICDF(10680), AOM_ICDF(15684), AOM_ICDF(19118), + AOM_ICDF(21856), AOM_ICDF(27563), AOM_ICDF(28234), AOM_ICDF(29332), + AOM_ICDF(31278), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11008), AOM_ICDF(11433), AOM_ICDF(14100), AOM_ICDF(22522), + AOM_ICDF(24365), AOM_ICDF(25330), AOM_ICDF(25737), AOM_ICDF(26341), + AOM_ICDF(30433), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10880), AOM_ICDF(11308), AOM_ICDF(13991), AOM_ICDF(23645), + AOM_ICDF(24679), AOM_ICDF(25433), AOM_ICDF(25977), AOM_ICDF(26746), + AOM_ICDF(28463), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9856), AOM_ICDF(10483), AOM_ICDF(16054), AOM_ICDF(19959), + AOM_ICDF(21708), AOM_ICDF(23628), AOM_ICDF(24949), AOM_ICDF(28797), + AOM_ICDF(30658), AOM_ICDF(32768), 0 }, + { AOM_ICDF(12928), AOM_ICDF(14556), AOM_ICDF(22168), AOM_ICDF(27789), + AOM_ICDF(28543), AOM_ICDF(29663), AOM_ICDF(29893), AOM_ICDF(30645), + AOM_ICDF(31682), AOM_ICDF(32768), 0 }, + }; +#endif // CONFIG_ALT_INTRA + +#if CONFIG_EXT_PARTITION_TYPES +static const aom_cdf_prob + default_partition_cdf[PARTITION_CONTEXTS][CDF_SIZE(EXT_PARTITION_TYPES)] = { + // 8x8 -> 4x4 only supports the four legacy partition types + { AOM_ICDF(25472), AOM_ICDF(28949), AOM_ICDF(31052), AOM_ICDF(32768), 0, + 0, 0, 0, 0 }, + { AOM_ICDF(18816), AOM_ICDF(22250), AOM_ICDF(28783), AOM_ICDF(32768), 0, + 0, 0, 0, 0 }, + { AOM_ICDF(18944), AOM_ICDF(26126), AOM_ICDF(29188), AOM_ICDF(32768), 0, + 0, 0, 0, 0 }, + { AOM_ICDF(15488), AOM_ICDF(22508), AOM_ICDF(27077), AOM_ICDF(32768), 0, + 0, 0, 0, 0 }, + { AOM_ICDF(22272), AOM_ICDF(23768), AOM_ICDF(25043), AOM_ICDF(29996), + AOM_ICDF(30744), AOM_ICDF(31493), AOM_ICDF(32130), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11776), AOM_ICDF(13457), AOM_ICDF(16315), AOM_ICDF(28229), + AOM_ICDF(29069), AOM_ICDF(29910), AOM_ICDF(31339), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10496), AOM_ICDF(14802), AOM_ICDF(16136), AOM_ICDF(27127), + AOM_ICDF(29280), AOM_ICDF(31434), AOM_ICDF(32101), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(8763), AOM_ICDF(10440), AOM_ICDF(29110), + AOM_ICDF(30100), AOM_ICDF(31090), AOM_ICDF(31929), AOM_ICDF(32768), 0 }, + { AOM_ICDF(22656), AOM_ICDF(23801), AOM_ICDF(24702), AOM_ICDF(30721), + AOM_ICDF(31294), AOM_ICDF(31867), AOM_ICDF(32317), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8704), AOM_ICDF(9926), AOM_ICDF(12586), AOM_ICDF(28885), + AOM_ICDF(29496), AOM_ICDF(30107), AOM_ICDF(31437), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6656), AOM_ICDF(10685), AOM_ICDF(11566), AOM_ICDF(27857), + AOM_ICDF(29871), AOM_ICDF(31886), AOM_ICDF(32327), AOM_ICDF(32768), 0 }, + { AOM_ICDF(2176), AOM_ICDF(3012), AOM_ICDF(3690), AOM_ICDF(31253), + AOM_ICDF(31671), AOM_ICDF(32090), AOM_ICDF(32429), AOM_ICDF(32768), 0 }, + { AOM_ICDF(28416), AOM_ICDF(28705), AOM_ICDF(28926), AOM_ICDF(32258), + AOM_ICDF(32402), AOM_ICDF(32547), AOM_ICDF(32657), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9216), AOM_ICDF(9952), AOM_ICDF(11849), AOM_ICDF(30134), + AOM_ICDF(30502), AOM_ICDF(30870), AOM_ICDF(31819), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7424), AOM_ICDF(9008), AOM_ICDF(9528), AOM_ICDF(30664), + AOM_ICDF(31456), AOM_ICDF(32248), AOM_ICDF(32508), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(1710), AOM_ICDF(2069), AOM_ICDF(31978), + AOM_ICDF(32193), AOM_ICDF(32409), AOM_ICDF(32588), AOM_ICDF(32768), 0 }, +#if CONFIG_EXT_PARTITION + { AOM_ICDF(28416), AOM_ICDF(28705), AOM_ICDF(28926), AOM_ICDF(32258), + AOM_ICDF(32402), AOM_ICDF(32547), AOM_ICDF(32657), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9216), AOM_ICDF(9952), AOM_ICDF(11849), AOM_ICDF(30134), + AOM_ICDF(30502), AOM_ICDF(30870), AOM_ICDF(31819), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7424), AOM_ICDF(9008), AOM_ICDF(9528), AOM_ICDF(30664), + AOM_ICDF(31456), AOM_ICDF(32248), AOM_ICDF(32508), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(1710), AOM_ICDF(2069), AOM_ICDF(31978), + AOM_ICDF(32193), AOM_ICDF(32409), AOM_ICDF(32588), AOM_ICDF(32768), 0 }, +#endif + }; +#else +static const aom_cdf_prob + default_partition_cdf[PARTITION_CONTEXTS][CDF_SIZE(PARTITION_TYPES)] = { + { AOM_ICDF(25472), AOM_ICDF(28949), AOM_ICDF(31052), AOM_ICDF(32768), 0 }, + { AOM_ICDF(18816), AOM_ICDF(22250), AOM_ICDF(28783), AOM_ICDF(32768), 0 }, + { AOM_ICDF(18944), AOM_ICDF(26126), AOM_ICDF(29188), AOM_ICDF(32768), 0 }, + { AOM_ICDF(15488), AOM_ICDF(22508), AOM_ICDF(27077), AOM_ICDF(32768), 0 }, + { AOM_ICDF(22272), AOM_ICDF(25265), AOM_ICDF(27815), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11776), AOM_ICDF(15138), AOM_ICDF(20854), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10496), AOM_ICDF(19109), AOM_ICDF(21777), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(10743), AOM_ICDF(14098), AOM_ICDF(32768), 0 }, + { AOM_ICDF(22656), AOM_ICDF(24947), AOM_ICDF(26749), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8704), AOM_ICDF(11148), AOM_ICDF(16469), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6656), AOM_ICDF(14714), AOM_ICDF(16477), AOM_ICDF(32768), 0 }, + { AOM_ICDF(2176), AOM_ICDF(3849), AOM_ICDF(5205), AOM_ICDF(32768), 0 }, + { AOM_ICDF(28416), AOM_ICDF(28994), AOM_ICDF(29436), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9216), AOM_ICDF(10688), AOM_ICDF(14483), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7424), AOM_ICDF(10592), AOM_ICDF(11632), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(2141), AOM_ICDF(2859), AOM_ICDF(32768), 0 }, +#if CONFIG_EXT_PARTITION + { AOM_ICDF(28416), AOM_ICDF(28994), AOM_ICDF(29436), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9216), AOM_ICDF(10688), AOM_ICDF(14483), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7424), AOM_ICDF(10592), AOM_ICDF(11632), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(2141), AOM_ICDF(2859), AOM_ICDF(32768), 0 }, +#endif + }; +#endif + +static const aom_cdf_prob + default_inter_mode_cdf[INTER_MODE_CONTEXTS][CDF_SIZE(INTER_MODES)] = { + { AOM_ICDF(256), AOM_ICDF(22227), AOM_ICDF(23627), AOM_ICDF(32768), 0 }, + { AOM_ICDF(896), AOM_ICDF(18948), AOM_ICDF(23537), AOM_ICDF(32768), 0 }, + { AOM_ICDF(896), AOM_ICDF(21563), AOM_ICDF(24320), AOM_ICDF(32768), 0 }, + { AOM_ICDF(896), AOM_ICDF(12599), AOM_ICDF(17799), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1024), AOM_ICDF(8960), AOM_ICDF(13238), AOM_ICDF(32768), 0 }, + { AOM_ICDF(2176), AOM_ICDF(11856), AOM_ICDF(14388), AOM_ICDF(32768), 0 }, + { AOM_ICDF(3200), AOM_ICDF(6550), AOM_ICDF(9622), AOM_ICDF(32768), 0 }, + }; + +#if CONFIG_EXT_TX +static const aom_cdf_prob default_intra_ext_tx_cdf + [EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES][CDF_SIZE(TX_TYPES)] = { + { +// FIXME: unused zero positions, from uncoded trivial transform set +#if CONFIG_CB4X4 + { + { 0 }, + }, +#endif + { { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, +#if CONFIG_ALT_INTRA + { 0 } +#endif + }, + { { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, +#if CONFIG_ALT_INTRA + { 0 } +#endif + }, + { { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, +#if CONFIG_ALT_INTRA + { 0 } +#endif + }, + { { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, +#if CONFIG_ALT_INTRA + { 0 } +#endif + }, + }, + { + { { AOM_ICDF(1024), AOM_ICDF(28800), AOM_ICDF(29048), AOM_ICDF(29296), + AOM_ICDF(30164), AOM_ICDF(31466), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(27118), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1152), AOM_ICDF(25852), AOM_ICDF(26284), AOM_ICDF(26717), + AOM_ICDF(28230), AOM_ICDF(30499), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1024), AOM_ICDF(2016), AOM_ICDF(3938), AOM_ICDF(5860), + AOM_ICDF(29404), AOM_ICDF(31086), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(27118), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(5900), AOM_ICDF(7691), + AOM_ICDF(15528), AOM_ICDF(27380), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(5900), AOM_ICDF(7691), + AOM_ICDF(15528), AOM_ICDF(27380), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 }, +#if CONFIG_ALT_INTRA + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 } +#endif + }, + { { AOM_ICDF(1024), AOM_ICDF(28800), AOM_ICDF(29048), AOM_ICDF(29296), + AOM_ICDF(30164), AOM_ICDF(31466), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(27118), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1152), AOM_ICDF(25852), AOM_ICDF(26284), AOM_ICDF(26717), + AOM_ICDF(28230), AOM_ICDF(30499), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1024), AOM_ICDF(2016), AOM_ICDF(3938), AOM_ICDF(5860), + AOM_ICDF(29404), AOM_ICDF(31086), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(27118), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(5900), AOM_ICDF(7691), + AOM_ICDF(15528), AOM_ICDF(27380), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(5900), AOM_ICDF(7691), + AOM_ICDF(15528), AOM_ICDF(27380), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 }, +#if CONFIG_ALT_INTRA + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 } +#endif + }, + { { AOM_ICDF(1024), AOM_ICDF(28800), AOM_ICDF(29048), AOM_ICDF(29296), + AOM_ICDF(30164), AOM_ICDF(31466), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(27118), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1152), AOM_ICDF(25852), AOM_ICDF(26284), AOM_ICDF(26717), + AOM_ICDF(28230), AOM_ICDF(30499), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1024), AOM_ICDF(2016), AOM_ICDF(3938), AOM_ICDF(5860), + AOM_ICDF(29404), AOM_ICDF(31086), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(27118), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(5900), AOM_ICDF(7691), + AOM_ICDF(15528), AOM_ICDF(27380), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(5900), AOM_ICDF(7691), + AOM_ICDF(15528), AOM_ICDF(27380), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 }, +#if CONFIG_ALT_INTRA + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 } +#endif + }, + { { AOM_ICDF(1024), AOM_ICDF(28800), AOM_ICDF(29048), AOM_ICDF(29296), + AOM_ICDF(30164), AOM_ICDF(31466), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(27118), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1152), AOM_ICDF(25852), AOM_ICDF(26284), AOM_ICDF(26717), + AOM_ICDF(28230), AOM_ICDF(30499), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1024), AOM_ICDF(2016), AOM_ICDF(3938), AOM_ICDF(5860), + AOM_ICDF(29404), AOM_ICDF(31086), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(27118), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(5900), AOM_ICDF(7691), + AOM_ICDF(15528), AOM_ICDF(27380), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(5900), AOM_ICDF(7691), + AOM_ICDF(15528), AOM_ICDF(27380), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 }, +#if CONFIG_ALT_INTRA + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(8660), + AOM_ICDF(10167), AOM_ICDF(15817), AOM_ICDF(32768), 0 } +#endif + }, + }, + { + { { AOM_ICDF(1024), AOM_ICDF(28800), AOM_ICDF(29792), AOM_ICDF(31280), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(26310), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1152), AOM_ICDF(25852), AOM_ICDF(27581), AOM_ICDF(30174), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1024), AOM_ICDF(2016), AOM_ICDF(28924), AOM_ICDF(30846), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(26310), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(13065), AOM_ICDF(26611), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(13065), AOM_ICDF(26611), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 }, +#if CONFIG_ALT_INTRA + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 } +#endif + }, + { { AOM_ICDF(1024), AOM_ICDF(28800), AOM_ICDF(29792), AOM_ICDF(31280), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(26310), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1152), AOM_ICDF(25852), AOM_ICDF(27581), AOM_ICDF(30174), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1024), AOM_ICDF(2016), AOM_ICDF(28924), AOM_ICDF(30846), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(26310), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(13065), AOM_ICDF(26611), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(13065), AOM_ICDF(26611), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 }, +#if CONFIG_ALT_INTRA + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 } +#endif + }, + { { AOM_ICDF(1024), AOM_ICDF(28800), AOM_ICDF(29792), AOM_ICDF(31280), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(26310), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1152), AOM_ICDF(25852), AOM_ICDF(27581), AOM_ICDF(30174), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1024), AOM_ICDF(2016), AOM_ICDF(28924), AOM_ICDF(30846), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(26310), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(13065), AOM_ICDF(26611), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(13065), AOM_ICDF(26611), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 }, +#if CONFIG_ALT_INTRA + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 } +#endif + }, + { { AOM_ICDF(1024), AOM_ICDF(28800), AOM_ICDF(29792), AOM_ICDF(31280), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(26310), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1152), AOM_ICDF(25852), AOM_ICDF(27581), AOM_ICDF(30174), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1024), AOM_ICDF(2016), AOM_ICDF(28924), AOM_ICDF(30846), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(26310), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(13065), AOM_ICDF(26611), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(4109), AOM_ICDF(13065), AOM_ICDF(26611), + AOM_ICDF(32768), 0 }, + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 }, +#if CONFIG_ALT_INTRA + { AOM_ICDF(1280), AOM_ICDF(5216), AOM_ICDF(6938), AOM_ICDF(13396), + AOM_ICDF(32768), 0 } +#endif + }, + } + }; +static const aom_cdf_prob + default_inter_ext_tx_cdf[EXT_TX_SETS_INTER][EXT_TX_SIZES][CDF_SIZE( + TX_TYPES)] = { + { +#if CONFIG_CB4X4 + { 0 }, +#endif + { 0 }, + { 0 }, + { 0 }, + { 0 } }, + { +#if CONFIG_CB4X4 + { 0 }, +#endif + { AOM_ICDF(1280), AOM_ICDF(1453), AOM_ICDF(1626), AOM_ICDF(2277), + AOM_ICDF(2929), AOM_ICDF(3580), AOM_ICDF(4232), AOM_ICDF(16717), + AOM_ICDF(19225), AOM_ICDF(21733), AOM_ICDF(24241), AOM_ICDF(26749), + AOM_ICDF(28253), AOM_ICDF(29758), AOM_ICDF(31263), AOM_ICDF(32768), + 0 }, + { AOM_ICDF(1280), AOM_ICDF(1453), AOM_ICDF(1626), AOM_ICDF(2277), + AOM_ICDF(2929), AOM_ICDF(3580), AOM_ICDF(4232), AOM_ICDF(16717), + AOM_ICDF(19225), AOM_ICDF(21733), AOM_ICDF(24241), AOM_ICDF(26749), + AOM_ICDF(28253), AOM_ICDF(29758), AOM_ICDF(31263), AOM_ICDF(32768), + 0 }, + { AOM_ICDF(1280), AOM_ICDF(1453), AOM_ICDF(1626), AOM_ICDF(2277), + AOM_ICDF(2929), AOM_ICDF(3580), AOM_ICDF(4232), AOM_ICDF(16717), + AOM_ICDF(19225), AOM_ICDF(21733), AOM_ICDF(24241), AOM_ICDF(26749), + AOM_ICDF(28253), AOM_ICDF(29758), AOM_ICDF(31263), AOM_ICDF(32768), + 0 }, + { AOM_ICDF(1280), AOM_ICDF(1453), AOM_ICDF(1626), AOM_ICDF(2277), + AOM_ICDF(2929), AOM_ICDF(3580), AOM_ICDF(4232), AOM_ICDF(16717), + AOM_ICDF(19225), AOM_ICDF(21733), AOM_ICDF(24241), AOM_ICDF(26749), + AOM_ICDF(28253), AOM_ICDF(29758), AOM_ICDF(31263), AOM_ICDF(32768), + 0 } }, + { +#if CONFIG_CB4X4 + { 0 }, +#endif + { AOM_ICDF(1280), AOM_ICDF(3125), AOM_ICDF(4970), AOM_ICDF(17132), + AOM_ICDF(19575), AOM_ICDF(22018), AOM_ICDF(24461), AOM_ICDF(26904), + AOM_ICDF(28370), AOM_ICDF(29836), AOM_ICDF(31302), AOM_ICDF(32768), + 0 }, + { AOM_ICDF(1280), AOM_ICDF(3125), AOM_ICDF(4970), AOM_ICDF(17132), + AOM_ICDF(19575), AOM_ICDF(22018), AOM_ICDF(24461), AOM_ICDF(26904), + AOM_ICDF(28370), AOM_ICDF(29836), AOM_ICDF(31302), AOM_ICDF(32768), + 0 }, + { AOM_ICDF(1280), AOM_ICDF(3125), AOM_ICDF(4970), AOM_ICDF(17132), + AOM_ICDF(19575), AOM_ICDF(22018), AOM_ICDF(24461), AOM_ICDF(26904), + AOM_ICDF(28370), AOM_ICDF(29836), AOM_ICDF(31302), AOM_ICDF(32768), + 0 }, + { AOM_ICDF(1280), AOM_ICDF(3125), AOM_ICDF(4970), AOM_ICDF(17132), + AOM_ICDF(19575), AOM_ICDF(22018), AOM_ICDF(24461), AOM_ICDF(26904), + AOM_ICDF(28370), AOM_ICDF(29836), AOM_ICDF(31302), AOM_ICDF(32768), + 0 } }, + { +#if CONFIG_CB4X4 + { 0 }, +#endif + { AOM_ICDF(1536), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1536), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1536), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1536), AOM_ICDF(32768), 0 } } + }; +#else +static const aom_cdf_prob + default_intra_ext_tx_cdf[EXT_TX_SIZES][TX_TYPES][CDF_SIZE(TX_TYPES)] = { +#if CONFIG_CB4X4 + { { AOM_ICDF(30720), AOM_ICDF(31400), AOM_ICDF(32084), AOM_ICDF(32768), + 0 }, + { AOM_ICDF(512), AOM_ICDF(638), AOM_ICDF(31764), AOM_ICDF(32768), 0 }, + { AOM_ICDF(512), AOM_ICDF(638), AOM_ICDF(1642), AOM_ICDF(32768), 0 }, + { AOM_ICDF(512), AOM_ICDF(31760), AOM_ICDF(32264), AOM_ICDF(32768), + 0 } }, +#endif + { { AOM_ICDF(30720), AOM_ICDF(31400), AOM_ICDF(32084), AOM_ICDF(32768), + 0 }, + { AOM_ICDF(512), AOM_ICDF(638), AOM_ICDF(31764), AOM_ICDF(32768), 0 }, + { AOM_ICDF(512), AOM_ICDF(638), AOM_ICDF(1642), AOM_ICDF(32768), 0 }, + { AOM_ICDF(512), AOM_ICDF(31760), AOM_ICDF(32264), AOM_ICDF(32768), + 0 } }, + + { { AOM_ICDF(31232), AOM_ICDF(31742), AOM_ICDF(32255), AOM_ICDF(32768), + 0 }, + { AOM_ICDF(1024), AOM_ICDF(1272), AOM_ICDF(31784), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1024), AOM_ICDF(1272), AOM_ICDF(2256), AOM_ICDF(32768), 0 }, + { AOM_ICDF(1024), AOM_ICDF(31776), AOM_ICDF(32272), AOM_ICDF(32768), + 0 } }, + { { AOM_ICDF(31744), AOM_ICDF(32084), AOM_ICDF(32426), AOM_ICDF(32768), + 0 }, + { AOM_ICDF(2048), AOM_ICDF(2528), AOM_ICDF(31823), AOM_ICDF(32768), 0 }, + { AOM_ICDF(2048), AOM_ICDF(2528), AOM_ICDF(3473), AOM_ICDF(32768), 0 }, + { AOM_ICDF(2048), AOM_ICDF(31808), AOM_ICDF(32288), AOM_ICDF(32768), + 0 } }, + }; + +static const aom_cdf_prob + default_inter_ext_tx_cdf[EXT_TX_SIZES][CDF_SIZE(TX_TYPES)] = { +#if CONFIG_CB4X4 + { AOM_ICDF(20480), AOM_ICDF(24560), AOM_ICDF(28664), AOM_ICDF(32768), 0 }, +#endif + { AOM_ICDF(20480), AOM_ICDF(24560), AOM_ICDF(28664), AOM_ICDF(32768), 0 }, + { AOM_ICDF(22528), AOM_ICDF(25928), AOM_ICDF(29348), AOM_ICDF(32768), 0 }, + { AOM_ICDF(24576), AOM_ICDF(27296), AOM_ICDF(30032), AOM_ICDF(32768), 0 }, + }; +#endif // !CONFIG_EXT_TX + +#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP +static const aom_cdf_prob + default_intra_filter_cdf[INTRA_FILTERS + 1][CDF_SIZE(INTRA_FILTERS)] = { + { AOM_ICDF(12544), AOM_ICDF(17521), AOM_ICDF(21095), AOM_ICDF(32768), 0 }, + { AOM_ICDF(12544), AOM_ICDF(19022), AOM_ICDF(23318), AOM_ICDF(32768), 0 }, + { AOM_ICDF(12032), AOM_ICDF(17297), AOM_ICDF(23522), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6272), AOM_ICDF(8860), AOM_ICDF(11101), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9216), AOM_ICDF(12712), AOM_ICDF(16629), AOM_ICDF(32768), 0 }, + }; +#endif // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP + +// CDF version of 'av1_kf_y_mode_prob'. +const aom_cdf_prob + av1_kf_y_mode_cdf[INTRA_MODES][INTRA_MODES][CDF_SIZE(INTRA_MODES)] = { +#if CONFIG_ALT_INTRA + { + { AOM_ICDF(15488), AOM_ICDF(17513), AOM_ICDF(20731), AOM_ICDF(24586), + AOM_ICDF(25921), AOM_ICDF(26749), AOM_ICDF(27807), AOM_ICDF(28602), + AOM_ICDF(29530), AOM_ICDF(30681), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11648), AOM_ICDF(14783), AOM_ICDF(21879), AOM_ICDF(23981), + AOM_ICDF(25213), AOM_ICDF(26218), AOM_ICDF(27472), AOM_ICDF(28465), + AOM_ICDF(29221), AOM_ICDF(30232), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8448), AOM_ICDF(11108), AOM_ICDF(13392), AOM_ICDF(25167), + AOM_ICDF(26295), AOM_ICDF(26789), AOM_ICDF(27536), AOM_ICDF(28088), + AOM_ICDF(29039), AOM_ICDF(30700), AOM_ICDF(32768), 0 }, + { AOM_ICDF(13568), AOM_ICDF(15293), AOM_ICDF(18706), AOM_ICDF(21610), + AOM_ICDF(23139), AOM_ICDF(24254), AOM_ICDF(26383), AOM_ICDF(27630), + AOM_ICDF(28613), AOM_ICDF(30350), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9600), AOM_ICDF(11772), AOM_ICDF(14397), AOM_ICDF(16580), + AOM_ICDF(20091), AOM_ICDF(22865), AOM_ICDF(24490), AOM_ICDF(25395), + AOM_ICDF(27037), AOM_ICDF(28694), AOM_ICDF(32768), 0 }, + { AOM_ICDF(12160), AOM_ICDF(14092), AOM_ICDF(17010), AOM_ICDF(18922), + AOM_ICDF(22683), AOM_ICDF(25751), AOM_ICDF(27725), AOM_ICDF(30109), + AOM_ICDF(31449), AOM_ICDF(32763), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9088), AOM_ICDF(10383), AOM_ICDF(12569), AOM_ICDF(17113), + AOM_ICDF(21351), AOM_ICDF(22511), AOM_ICDF(23633), AOM_ICDF(24382), + AOM_ICDF(28215), AOM_ICDF(29798), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10880), AOM_ICDF(12248), AOM_ICDF(15214), AOM_ICDF(20017), + AOM_ICDF(21922), AOM_ICDF(22757), AOM_ICDF(24360), AOM_ICDF(25280), + AOM_ICDF(26684), AOM_ICDF(29869), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11008), AOM_ICDF(13133), AOM_ICDF(15587), AOM_ICDF(17872), + AOM_ICDF(19579), AOM_ICDF(21157), AOM_ICDF(23788), AOM_ICDF(26629), + AOM_ICDF(27732), AOM_ICDF(29601), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10112), AOM_ICDF(12325), AOM_ICDF(15360), AOM_ICDF(18348), + AOM_ICDF(20452), AOM_ICDF(20460), AOM_ICDF(21902), AOM_ICDF(23982), + AOM_ICDF(25149), AOM_ICDF(26667), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8704), AOM_ICDF(14250), AOM_ICDF(17722), AOM_ICDF(23128), + AOM_ICDF(24217), AOM_ICDF(24892), AOM_ICDF(26215), AOM_ICDF(27392), + AOM_ICDF(28358), AOM_ICDF(30287), AOM_ICDF(32768), 0 }, + }, + { + { AOM_ICDF(8448), AOM_ICDF(10443), AOM_ICDF(20733), AOM_ICDF(23689), + AOM_ICDF(24634), AOM_ICDF(25951), AOM_ICDF(26670), AOM_ICDF(27861), + AOM_ICDF(28379), AOM_ICDF(29305), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6656), AOM_ICDF(9206), AOM_ICDF(24577), AOM_ICDF(25792), + AOM_ICDF(26335), AOM_ICDF(27169), AOM_ICDF(27913), AOM_ICDF(28956), + AOM_ICDF(29239), AOM_ICDF(29680), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7168), AOM_ICDF(8968), AOM_ICDF(15662), AOM_ICDF(22937), + AOM_ICDF(23849), AOM_ICDF(24616), AOM_ICDF(25603), AOM_ICDF(26555), + AOM_ICDF(27210), AOM_ICDF(29142), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9600), AOM_ICDF(11501), AOM_ICDF(19310), AOM_ICDF(21731), + AOM_ICDF(22790), AOM_ICDF(23936), AOM_ICDF(25627), AOM_ICDF(27217), + AOM_ICDF(27868), AOM_ICDF(29170), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6912), AOM_ICDF(8730), AOM_ICDF(17650), AOM_ICDF(19377), + AOM_ICDF(21025), AOM_ICDF(23319), AOM_ICDF(24537), AOM_ICDF(26112), + AOM_ICDF(26840), AOM_ICDF(28345), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7808), AOM_ICDF(9661), AOM_ICDF(20583), AOM_ICDF(21996), + AOM_ICDF(23898), AOM_ICDF(26818), AOM_ICDF(28120), AOM_ICDF(30716), + AOM_ICDF(31678), AOM_ICDF(32764), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(8104), AOM_ICDF(15619), AOM_ICDF(18584), + AOM_ICDF(20844), AOM_ICDF(22519), AOM_ICDF(23760), AOM_ICDF(25203), + AOM_ICDF(27094), AOM_ICDF(28801), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8832), AOM_ICDF(10141), AOM_ICDF(17035), AOM_ICDF(20764), + AOM_ICDF(21703), AOM_ICDF(22751), AOM_ICDF(23964), AOM_ICDF(25305), + AOM_ICDF(26034), AOM_ICDF(29006), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8192), AOM_ICDF(9920), AOM_ICDF(19113), AOM_ICDF(20594), + AOM_ICDF(21747), AOM_ICDF(23327), AOM_ICDF(24581), AOM_ICDF(26916), + AOM_ICDF(27533), AOM_ICDF(28944), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6656), AOM_ICDF(8696), AOM_ICDF(18381), AOM_ICDF(20537), + AOM_ICDF(21804), AOM_ICDF(21809), AOM_ICDF(22751), AOM_ICDF(24394), + AOM_ICDF(24917), AOM_ICDF(25990), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6400), AOM_ICDF(9593), AOM_ICDF(20818), AOM_ICDF(23519), + AOM_ICDF(24266), AOM_ICDF(25113), AOM_ICDF(26608), AOM_ICDF(27883), + AOM_ICDF(28322), AOM_ICDF(29364), AOM_ICDF(32768), 0 }, + }, + { + { AOM_ICDF(12032), AOM_ICDF(14381), AOM_ICDF(16608), AOM_ICDF(24946), + AOM_ICDF(26084), AOM_ICDF(26582), AOM_ICDF(27428), AOM_ICDF(28075), + AOM_ICDF(29395), AOM_ICDF(30858), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9216), AOM_ICDF(12620), AOM_ICDF(18287), AOM_ICDF(24345), + AOM_ICDF(25984), AOM_ICDF(26715), AOM_ICDF(27732), AOM_ICDF(28519), + AOM_ICDF(29399), AOM_ICDF(30781), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(8916), AOM_ICDF(10220), AOM_ICDF(26539), + AOM_ICDF(27310), AOM_ICDF(27483), AOM_ICDF(28082), AOM_ICDF(28430), + AOM_ICDF(29362), AOM_ICDF(31291), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11904), AOM_ICDF(14838), AOM_ICDF(17359), AOM_ICDF(21663), + AOM_ICDF(22931), AOM_ICDF(23619), AOM_ICDF(25620), AOM_ICDF(26653), + AOM_ICDF(27823), AOM_ICDF(30547), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10752), AOM_ICDF(13504), AOM_ICDF(15536), AOM_ICDF(19057), + AOM_ICDF(21753), AOM_ICDF(23883), AOM_ICDF(25202), AOM_ICDF(26266), + AOM_ICDF(28196), AOM_ICDF(30589), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10496), AOM_ICDF(13193), AOM_ICDF(16787), AOM_ICDF(21011), + AOM_ICDF(23929), AOM_ICDF(25651), AOM_ICDF(27958), AOM_ICDF(29330), + AOM_ICDF(31022), AOM_ICDF(32761), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8448), AOM_ICDF(9968), AOM_ICDF(11749), AOM_ICDF(18062), + AOM_ICDF(21841), AOM_ICDF(22669), AOM_ICDF(23852), AOM_ICDF(24444), + AOM_ICDF(28118), AOM_ICDF(30007), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9728), AOM_ICDF(11168), AOM_ICDF(12602), AOM_ICDF(20819), + AOM_ICDF(22194), AOM_ICDF(22764), AOM_ICDF(24366), AOM_ICDF(25022), + AOM_ICDF(26414), AOM_ICDF(30460), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9216), AOM_ICDF(12712), AOM_ICDF(14357), AOM_ICDF(18346), + AOM_ICDF(20486), AOM_ICDF(21549), AOM_ICDF(23170), AOM_ICDF(25794), + AOM_ICDF(27129), AOM_ICDF(29574), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7808), AOM_ICDF(10733), AOM_ICDF(13057), AOM_ICDF(20252), + AOM_ICDF(21906), AOM_ICDF(21912), AOM_ICDF(23057), AOM_ICDF(24233), + AOM_ICDF(25700), AOM_ICDF(27439), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(11352), AOM_ICDF(13778), AOM_ICDF(23877), + AOM_ICDF(24995), AOM_ICDF(25424), AOM_ICDF(26830), AOM_ICDF(27688), + AOM_ICDF(28779), AOM_ICDF(30368), AOM_ICDF(32768), 0 }, + }, + { + { AOM_ICDF(12288), AOM_ICDF(13728), AOM_ICDF(16480), AOM_ICDF(19841), + AOM_ICDF(21570), AOM_ICDF(22715), AOM_ICDF(25385), AOM_ICDF(27000), + AOM_ICDF(28329), AOM_ICDF(29994), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9344), AOM_ICDF(10991), AOM_ICDF(18817), AOM_ICDF(20972), + AOM_ICDF(22137), AOM_ICDF(23231), AOM_ICDF(26025), AOM_ICDF(27711), + AOM_ICDF(28244), AOM_ICDF(29428), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9344), AOM_ICDF(10900), AOM_ICDF(13206), AOM_ICDF(21344), + AOM_ICDF(22332), AOM_ICDF(22987), AOM_ICDF(25127), AOM_ICDF(26440), + AOM_ICDF(27231), AOM_ICDF(29502), AOM_ICDF(32768), 0 }, + { AOM_ICDF(12928), AOM_ICDF(14478), AOM_ICDF(15978), AOM_ICDF(18630), + AOM_ICDF(19852), AOM_ICDF(20897), AOM_ICDF(24699), AOM_ICDF(26464), + AOM_ICDF(27030), AOM_ICDF(30482), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9088), AOM_ICDF(10476), AOM_ICDF(13350), AOM_ICDF(15237), + AOM_ICDF(18175), AOM_ICDF(20252), AOM_ICDF(23283), AOM_ICDF(25321), + AOM_ICDF(26426), AOM_ICDF(29349), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10240), AOM_ICDF(11912), AOM_ICDF(15008), AOM_ICDF(17177), + AOM_ICDF(19979), AOM_ICDF(23056), AOM_ICDF(26395), AOM_ICDF(29681), + AOM_ICDF(30790), AOM_ICDF(32760), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8704), AOM_ICDF(9738), AOM_ICDF(11717), AOM_ICDF(15480), + AOM_ICDF(18656), AOM_ICDF(20022), AOM_ICDF(22611), AOM_ICDF(24357), + AOM_ICDF(27150), AOM_ICDF(29257), AOM_ICDF(32768), 0 }, + { AOM_ICDF(12928), AOM_ICDF(13548), AOM_ICDF(17978), AOM_ICDF(20602), + AOM_ICDF(21814), AOM_ICDF(22427), AOM_ICDF(24568), AOM_ICDF(25881), + AOM_ICDF(26823), AOM_ICDF(30817), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10496), AOM_ICDF(12149), AOM_ICDF(14082), AOM_ICDF(18054), + AOM_ICDF(19032), AOM_ICDF(19994), AOM_ICDF(24086), AOM_ICDF(28427), + AOM_ICDF(29156), AOM_ICDF(30680), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8448), AOM_ICDF(10158), AOM_ICDF(13867), AOM_ICDF(16506), + AOM_ICDF(18584), AOM_ICDF(18592), AOM_ICDF(21472), AOM_ICDF(23767), + AOM_ICDF(24646), AOM_ICDF(27279), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7296), AOM_ICDF(9684), AOM_ICDF(13471), AOM_ICDF(17701), + AOM_ICDF(18934), AOM_ICDF(19878), AOM_ICDF(25115), AOM_ICDF(27238), + AOM_ICDF(27972), AOM_ICDF(29583), AOM_ICDF(32768), 0 }, + }, + { + { AOM_ICDF(10880), AOM_ICDF(12163), AOM_ICDF(14497), AOM_ICDF(17112), + AOM_ICDF(20859), AOM_ICDF(22562), AOM_ICDF(23599), AOM_ICDF(24638), + AOM_ICDF(26861), AOM_ICDF(29399), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9984), AOM_ICDF(12476), AOM_ICDF(16360), AOM_ICDF(18889), + AOM_ICDF(21414), AOM_ICDF(23474), AOM_ICDF(24563), AOM_ICDF(25909), + AOM_ICDF(27195), AOM_ICDF(28828), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7168), AOM_ICDF(9268), AOM_ICDF(10737), AOM_ICDF(20063), + AOM_ICDF(22315), AOM_ICDF(23302), AOM_ICDF(24152), AOM_ICDF(25195), + AOM_ICDF(26645), AOM_ICDF(28845), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8960), AOM_ICDF(10727), AOM_ICDF(12449), AOM_ICDF(14263), + AOM_ICDF(16523), AOM_ICDF(17608), AOM_ICDF(23352), AOM_ICDF(24676), + AOM_ICDF(26478), AOM_ICDF(28886), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9856), AOM_ICDF(11109), AOM_ICDF(13309), AOM_ICDF(14975), + AOM_ICDF(19055), AOM_ICDF(21670), AOM_ICDF(23144), AOM_ICDF(24460), + AOM_ICDF(26212), AOM_ICDF(28107), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9984), AOM_ICDF(11586), AOM_ICDF(14565), AOM_ICDF(16562), + AOM_ICDF(21107), AOM_ICDF(25444), AOM_ICDF(27218), AOM_ICDF(29429), + AOM_ICDF(31451), AOM_ICDF(32763), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7168), AOM_ICDF(8268), AOM_ICDF(9704), AOM_ICDF(13144), + AOM_ICDF(18443), AOM_ICDF(20065), AOM_ICDF(21653), AOM_ICDF(23607), + AOM_ICDF(26506), AOM_ICDF(28854), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11520), AOM_ICDF(13014), AOM_ICDF(14866), AOM_ICDF(18136), + AOM_ICDF(20231), AOM_ICDF(21509), AOM_ICDF(23004), AOM_ICDF(24186), + AOM_ICDF(25728), AOM_ICDF(29468), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10240), AOM_ICDF(12264), AOM_ICDF(14507), AOM_ICDF(16388), + AOM_ICDF(18888), AOM_ICDF(20927), AOM_ICDF(22731), AOM_ICDF(24691), + AOM_ICDF(26142), AOM_ICDF(28394), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8064), AOM_ICDF(10187), AOM_ICDF(12921), AOM_ICDF(15952), + AOM_ICDF(19960), AOM_ICDF(19976), AOM_ICDF(21275), AOM_ICDF(23205), + AOM_ICDF(25110), AOM_ICDF(26636), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8448), AOM_ICDF(11488), AOM_ICDF(14065), AOM_ICDF(19113), + AOM_ICDF(21604), AOM_ICDF(22978), AOM_ICDF(24508), AOM_ICDF(25895), + AOM_ICDF(27398), AOM_ICDF(29055), AOM_ICDF(32768), 0 }, + }, + { + { AOM_ICDF(10368), AOM_ICDF(11768), AOM_ICDF(16772), AOM_ICDF(19842), + AOM_ICDF(22940), AOM_ICDF(27394), AOM_ICDF(28528), AOM_ICDF(30267), + AOM_ICDF(31371), AOM_ICDF(32763), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9472), AOM_ICDF(11292), AOM_ICDF(18507), AOM_ICDF(20777), + AOM_ICDF(23357), AOM_ICDF(27587), AOM_ICDF(28902), AOM_ICDF(30850), + AOM_ICDF(31607), AOM_ICDF(32763), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8064), AOM_ICDF(9512), AOM_ICDF(13782), AOM_ICDF(20645), + AOM_ICDF(24493), AOM_ICDF(26242), AOM_ICDF(28001), AOM_ICDF(29435), + AOM_ICDF(30438), AOM_ICDF(32759), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8960), AOM_ICDF(10541), AOM_ICDF(15664), AOM_ICDF(17639), + AOM_ICDF(19646), AOM_ICDF(22145), AOM_ICDF(25216), AOM_ICDF(28815), + AOM_ICDF(30050), AOM_ICDF(32757), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9984), AOM_ICDF(11141), AOM_ICDF(15365), AOM_ICDF(16746), + AOM_ICDF(21186), AOM_ICDF(25766), AOM_ICDF(27817), AOM_ICDF(30022), + AOM_ICDF(31309), AOM_ICDF(32762), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9216), AOM_ICDF(10688), AOM_ICDF(16639), AOM_ICDF(17735), + AOM_ICDF(21499), AOM_ICDF(26657), AOM_ICDF(28161), AOM_ICDF(30572), + AOM_ICDF(31490), AOM_ICDF(32763), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8448), AOM_ICDF(9303), AOM_ICDF(13611), AOM_ICDF(16636), + AOM_ICDF(20555), AOM_ICDF(23414), AOM_ICDF(24912), AOM_ICDF(27613), + AOM_ICDF(29727), AOM_ICDF(32756), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9984), AOM_ICDF(11052), AOM_ICDF(16142), AOM_ICDF(19312), + AOM_ICDF(21680), AOM_ICDF(23870), AOM_ICDF(25504), AOM_ICDF(28200), + AOM_ICDF(29324), AOM_ICDF(32755), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10496), AOM_ICDF(12323), AOM_ICDF(16955), AOM_ICDF(18839), + AOM_ICDF(21144), AOM_ICDF(24861), AOM_ICDF(26838), AOM_ICDF(29988), + AOM_ICDF(30976), AOM_ICDF(32761), AOM_ICDF(32768), 0 }, + { AOM_ICDF(2944), AOM_ICDF(5973), AOM_ICDF(8904), AOM_ICDF(11875), + AOM_ICDF(14864), AOM_ICDF(17853), AOM_ICDF(20824), AOM_ICDF(23810), + AOM_ICDF(26784), AOM_ICDF(29776), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7424), AOM_ICDF(10097), AOM_ICDF(15588), AOM_ICDF(20217), + AOM_ICDF(23899), AOM_ICDF(26460), AOM_ICDF(28308), AOM_ICDF(30155), + AOM_ICDF(30951), AOM_ICDF(32761), AOM_ICDF(32768), 0 }, + }, + { + { AOM_ICDF(11648), AOM_ICDF(13133), AOM_ICDF(15050), AOM_ICDF(20481), + AOM_ICDF(22470), AOM_ICDF(23425), AOM_ICDF(24337), AOM_ICDF(25160), + AOM_ICDF(28964), AOM_ICDF(30480), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10240), AOM_ICDF(12616), AOM_ICDF(16631), AOM_ICDF(20485), + AOM_ICDF(22290), AOM_ICDF(23628), AOM_ICDF(25235), AOM_ICDF(26353), + AOM_ICDF(28107), AOM_ICDF(29655), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(8002), AOM_ICDF(9066), AOM_ICDF(20038), + AOM_ICDF(22926), AOM_ICDF(23324), AOM_ICDF(23951), AOM_ICDF(24537), + AOM_ICDF(26916), AOM_ICDF(30231), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11904), AOM_ICDF(14105), AOM_ICDF(15782), AOM_ICDF(19896), + AOM_ICDF(22283), AOM_ICDF(23147), AOM_ICDF(24763), AOM_ICDF(25983), + AOM_ICDF(27812), AOM_ICDF(29980), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10624), AOM_ICDF(11922), AOM_ICDF(13632), AOM_ICDF(15941), + AOM_ICDF(20469), AOM_ICDF(22453), AOM_ICDF(24065), AOM_ICDF(25187), + AOM_ICDF(27349), AOM_ICDF(29296), AOM_ICDF(32768), 0 }, + { AOM_ICDF(12032), AOM_ICDF(13085), AOM_ICDF(15468), AOM_ICDF(17768), + AOM_ICDF(20613), AOM_ICDF(24388), AOM_ICDF(26385), AOM_ICDF(28430), + AOM_ICDF(30938), AOM_ICDF(32761), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9728), AOM_ICDF(10538), AOM_ICDF(11493), AOM_ICDF(14765), + AOM_ICDF(18460), AOM_ICDF(19471), AOM_ICDF(20302), AOM_ICDF(20935), + AOM_ICDF(28192), AOM_ICDF(29926), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8960), AOM_ICDF(9890), AOM_ICDF(10962), AOM_ICDF(16685), + AOM_ICDF(18880), AOM_ICDF(19480), AOM_ICDF(20674), AOM_ICDF(21477), + AOM_ICDF(23815), AOM_ICDF(29341), AOM_ICDF(32768), 0 }, + { AOM_ICDF(14592), AOM_ICDF(16367), AOM_ICDF(17712), AOM_ICDF(20293), + AOM_ICDF(22544), AOM_ICDF(23829), AOM_ICDF(24877), AOM_ICDF(26326), + AOM_ICDF(27660), AOM_ICDF(29875), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8960), AOM_ICDF(10448), AOM_ICDF(12279), AOM_ICDF(16206), + AOM_ICDF(18672), AOM_ICDF(18682), AOM_ICDF(20058), AOM_ICDF(21547), + AOM_ICDF(25097), AOM_ICDF(27165), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11136), AOM_ICDF(13840), AOM_ICDF(15762), AOM_ICDF(21710), + AOM_ICDF(23038), AOM_ICDF(23734), AOM_ICDF(24863), AOM_ICDF(25882), + AOM_ICDF(27765), AOM_ICDF(30071), AOM_ICDF(32768), 0 }, + }, + { + { AOM_ICDF(12544), AOM_ICDF(14124), AOM_ICDF(16964), AOM_ICDF(21907), + AOM_ICDF(23808), AOM_ICDF(24496), AOM_ICDF(25724), AOM_ICDF(26715), + AOM_ICDF(27992), AOM_ICDF(30455), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10368), AOM_ICDF(13606), AOM_ICDF(18247), AOM_ICDF(20869), + AOM_ICDF(22590), AOM_ICDF(23749), AOM_ICDF(25088), AOM_ICDF(26378), + AOM_ICDF(27277), AOM_ICDF(29808), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9088), AOM_ICDF(11031), AOM_ICDF(12899), AOM_ICDF(23497), + AOM_ICDF(24465), AOM_ICDF(24851), AOM_ICDF(25995), AOM_ICDF(26815), + AOM_ICDF(27885), AOM_ICDF(30555), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11520), AOM_ICDF(14342), AOM_ICDF(15710), AOM_ICDF(19196), + AOM_ICDF(21250), AOM_ICDF(21907), AOM_ICDF(24665), AOM_ICDF(26153), + AOM_ICDF(27212), AOM_ICDF(30750), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9984), AOM_ICDF(11764), AOM_ICDF(13979), AOM_ICDF(16405), + AOM_ICDF(19279), AOM_ICDF(20658), AOM_ICDF(23354), AOM_ICDF(25266), + AOM_ICDF(26702), AOM_ICDF(29380), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10112), AOM_ICDF(12325), AOM_ICDF(15918), AOM_ICDF(19060), + AOM_ICDF(21829), AOM_ICDF(23882), AOM_ICDF(26277), AOM_ICDF(27697), + AOM_ICDF(30114), AOM_ICDF(32758), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9344), AOM_ICDF(10534), AOM_ICDF(12184), AOM_ICDF(16208), + AOM_ICDF(19764), AOM_ICDF(20627), AOM_ICDF(22524), AOM_ICDF(23644), + AOM_ICDF(26887), AOM_ICDF(29782), AOM_ICDF(32768), 0 }, + { AOM_ICDF(12928), AOM_ICDF(14013), AOM_ICDF(15625), AOM_ICDF(19107), + AOM_ICDF(20654), AOM_ICDF(21451), AOM_ICDF(22910), AOM_ICDF(23873), + AOM_ICDF(24776), AOM_ICDF(30239), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10368), AOM_ICDF(12818), AOM_ICDF(14610), AOM_ICDF(17350), + AOM_ICDF(19568), AOM_ICDF(20710), AOM_ICDF(22971), AOM_ICDF(25114), + AOM_ICDF(26340), AOM_ICDF(29127), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8960), AOM_ICDF(11192), AOM_ICDF(13720), AOM_ICDF(18429), + AOM_ICDF(20409), AOM_ICDF(20417), AOM_ICDF(22250), AOM_ICDF(23318), + AOM_ICDF(24647), AOM_ICDF(27248), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7808), AOM_ICDF(11416), AOM_ICDF(13918), AOM_ICDF(19028), + AOM_ICDF(20181), AOM_ICDF(20839), AOM_ICDF(24380), AOM_ICDF(26018), + AOM_ICDF(26967), AOM_ICDF(29845), AOM_ICDF(32768), 0 }, + }, + { + { AOM_ICDF(9856), AOM_ICDF(11020), AOM_ICDF(14928), AOM_ICDF(18159), + AOM_ICDF(19421), AOM_ICDF(20921), AOM_ICDF(23466), AOM_ICDF(26664), + AOM_ICDF(27475), AOM_ICDF(28881), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8704), AOM_ICDF(10302), AOM_ICDF(17323), AOM_ICDF(18907), + AOM_ICDF(19868), AOM_ICDF(21184), AOM_ICDF(24171), AOM_ICDF(28033), + AOM_ICDF(28625), AOM_ICDF(29353), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7936), AOM_ICDF(9197), AOM_ICDF(12604), AOM_ICDF(20616), + AOM_ICDF(21514), AOM_ICDF(22371), AOM_ICDF(24239), AOM_ICDF(26138), + AOM_ICDF(26863), AOM_ICDF(29239), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11264), AOM_ICDF(12524), AOM_ICDF(16083), AOM_ICDF(18574), + AOM_ICDF(19858), AOM_ICDF(20841), AOM_ICDF(24242), AOM_ICDF(27606), + AOM_ICDF(28352), AOM_ICDF(29853), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8704), AOM_ICDF(10208), AOM_ICDF(13292), AOM_ICDF(15170), + AOM_ICDF(17277), AOM_ICDF(19226), AOM_ICDF(22083), AOM_ICDF(25046), + AOM_ICDF(26041), AOM_ICDF(27802), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9088), AOM_ICDF(10568), AOM_ICDF(15511), AOM_ICDF(17246), + AOM_ICDF(20170), AOM_ICDF(22791), AOM_ICDF(25558), AOM_ICDF(30740), + AOM_ICDF(31635), AOM_ICDF(32764), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7040), AOM_ICDF(8045), AOM_ICDF(10653), AOM_ICDF(13145), + AOM_ICDF(15286), AOM_ICDF(16614), AOM_ICDF(19075), AOM_ICDF(23140), + AOM_ICDF(26224), AOM_ICDF(28652), AOM_ICDF(32768), 0 }, + { AOM_ICDF(10240), AOM_ICDF(11032), AOM_ICDF(14258), AOM_ICDF(17629), + AOM_ICDF(18914), AOM_ICDF(19898), AOM_ICDF(22412), AOM_ICDF(24961), + AOM_ICDF(25815), AOM_ICDF(29156), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11008), AOM_ICDF(12028), AOM_ICDF(14702), AOM_ICDF(16147), + AOM_ICDF(17209), AOM_ICDF(18160), AOM_ICDF(21812), AOM_ICDF(27547), + AOM_ICDF(28709), AOM_ICDF(30120), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7168), AOM_ICDF(9068), AOM_ICDF(14160), AOM_ICDF(16937), + AOM_ICDF(18515), AOM_ICDF(18521), AOM_ICDF(20636), AOM_ICDF(24617), + AOM_ICDF(25317), AOM_ICDF(26365), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(8510), AOM_ICDF(14195), AOM_ICDF(17148), + AOM_ICDF(18158), AOM_ICDF(19201), AOM_ICDF(23070), AOM_ICDF(27351), + AOM_ICDF(27901), AOM_ICDF(29422), AOM_ICDF(32768), 0 }, + }, + { + { AOM_ICDF(10112), AOM_ICDF(11528), AOM_ICDF(15345), AOM_ICDF(19296), + AOM_ICDF(21394), AOM_ICDF(21402), AOM_ICDF(22379), AOM_ICDF(23840), + AOM_ICDF(24851), AOM_ICDF(26150), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8064), AOM_ICDF(10187), AOM_ICDF(17949), AOM_ICDF(20052), + AOM_ICDF(22051), AOM_ICDF(22059), AOM_ICDF(23147), AOM_ICDF(24688), + AOM_ICDF(25351), AOM_ICDF(26365), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6528), AOM_ICDF(8373), AOM_ICDF(11041), AOM_ICDF(21963), + AOM_ICDF(23089), AOM_ICDF(23093), AOM_ICDF(24076), AOM_ICDF(24925), + AOM_ICDF(25691), AOM_ICDF(27764), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9600), AOM_ICDF(11229), AOM_ICDF(14847), AOM_ICDF(17527), + AOM_ICDF(19738), AOM_ICDF(19747), AOM_ICDF(21629), AOM_ICDF(23761), + AOM_ICDF(24957), AOM_ICDF(27673), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8960), AOM_ICDF(10262), AOM_ICDF(13339), AOM_ICDF(15480), + AOM_ICDF(19925), AOM_ICDF(19942), AOM_ICDF(21445), AOM_ICDF(23037), + AOM_ICDF(24329), AOM_ICDF(25977), AOM_ICDF(32768), 0 }, + { AOM_ICDF(2944), AOM_ICDF(5973), AOM_ICDF(8904), AOM_ICDF(11875), + AOM_ICDF(14864), AOM_ICDF(17853), AOM_ICDF(20824), AOM_ICDF(23810), + AOM_ICDF(26784), AOM_ICDF(29776), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9472), AOM_ICDF(10564), AOM_ICDF(13426), AOM_ICDF(16561), + AOM_ICDF(19685), AOM_ICDF(19697), AOM_ICDF(21076), AOM_ICDF(22583), + AOM_ICDF(24891), AOM_ICDF(26983), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8448), AOM_ICDF(9493), AOM_ICDF(12221), AOM_ICDF(16542), + AOM_ICDF(18394), AOM_ICDF(18401), AOM_ICDF(19580), AOM_ICDF(20971), + AOM_ICDF(22031), AOM_ICDF(26770), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8704), AOM_ICDF(10772), AOM_ICDF(14209), AOM_ICDF(16381), + AOM_ICDF(18911), AOM_ICDF(18921), AOM_ICDF(20436), AOM_ICDF(23374), + AOM_ICDF(24475), AOM_ICDF(26095), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7680), AOM_ICDF(9444), AOM_ICDF(13453), AOM_ICDF(16320), + AOM_ICDF(18650), AOM_ICDF(18659), AOM_ICDF(19651), AOM_ICDF(21291), + AOM_ICDF(22277), AOM_ICDF(23916), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6656), AOM_ICDF(9920), AOM_ICDF(14740), AOM_ICDF(19864), + AOM_ICDF(21495), AOM_ICDF(21501), AOM_ICDF(22953), AOM_ICDF(24372), + AOM_ICDF(25192), AOM_ICDF(26760), AOM_ICDF(32768), 0 }, + }, + { + { AOM_ICDF(9728), AOM_ICDF(13958), AOM_ICDF(18881), AOM_ICDF(23624), + AOM_ICDF(24754), AOM_ICDF(25553), AOM_ICDF(26709), AOM_ICDF(27940), + AOM_ICDF(28977), AOM_ICDF(30413), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8832), AOM_ICDF(12572), AOM_ICDF(22433), AOM_ICDF(24653), + AOM_ICDF(25676), AOM_ICDF(26551), AOM_ICDF(27571), AOM_ICDF(28688), + AOM_ICDF(29198), AOM_ICDF(30174), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5888), AOM_ICDF(8828), AOM_ICDF(11353), AOM_ICDF(23482), + AOM_ICDF(24310), AOM_ICDF(24737), AOM_ICDF(25804), AOM_ICDF(26375), + AOM_ICDF(27174), AOM_ICDF(29840), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9984), AOM_ICDF(13099), AOM_ICDF(16249), AOM_ICDF(19443), + AOM_ICDF(20990), AOM_ICDF(22637), AOM_ICDF(24576), AOM_ICDF(25952), + AOM_ICDF(26884), AOM_ICDF(29435), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8448), AOM_ICDF(11108), AOM_ICDF(15085), AOM_ICDF(18134), + AOM_ICDF(20319), AOM_ICDF(21992), AOM_ICDF(23549), AOM_ICDF(24989), + AOM_ICDF(27177), AOM_ICDF(29208), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9856), AOM_ICDF(13168), AOM_ICDF(18987), AOM_ICDF(22481), + AOM_ICDF(24282), AOM_ICDF(26200), AOM_ICDF(27868), AOM_ICDF(30203), + AOM_ICDF(31085), AOM_ICDF(32761), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(9119), AOM_ICDF(12629), AOM_ICDF(16877), + AOM_ICDF(20262), AOM_ICDF(21125), AOM_ICDF(22307), AOM_ICDF(23615), + AOM_ICDF(27727), AOM_ICDF(29972), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8320), AOM_ICDF(10230), AOM_ICDF(12783), AOM_ICDF(19005), + AOM_ICDF(20213), AOM_ICDF(20668), AOM_ICDF(22039), AOM_ICDF(23045), + AOM_ICDF(24146), AOM_ICDF(30478), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9088), AOM_ICDF(11308), AOM_ICDF(15416), AOM_ICDF(18118), + AOM_ICDF(19762), AOM_ICDF(20906), AOM_ICDF(22574), AOM_ICDF(25162), + AOM_ICDF(25994), AOM_ICDF(28455), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6912), AOM_ICDF(10548), AOM_ICDF(15148), AOM_ICDF(20026), + AOM_ICDF(21612), AOM_ICDF(21618), AOM_ICDF(22707), AOM_ICDF(24200), + AOM_ICDF(24869), AOM_ICDF(26844), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6656), AOM_ICDF(12164), AOM_ICDF(16993), AOM_ICDF(21568), + AOM_ICDF(22933), AOM_ICDF(23648), AOM_ICDF(25322), AOM_ICDF(26602), + AOM_ICDF(27806), AOM_ICDF(29841), AOM_ICDF(32768), 0 }, + }, +#else // !CONFIG_ALT_INTRA + { { AOM_ICDF(17536), AOM_ICDF(19321), AOM_ICDF(21527), AOM_ICDF(25360), + AOM_ICDF(27516), AOM_ICDF(28026), AOM_ICDF(29323), AOM_ICDF(30023), + AOM_ICDF(30999), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11776), AOM_ICDF(15466), AOM_ICDF(22360), AOM_ICDF(24865), + AOM_ICDF(26991), AOM_ICDF(27889), AOM_ICDF(29299), AOM_ICDF(30519), + AOM_ICDF(31398), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9344), AOM_ICDF(12272), AOM_ICDF(13793), AOM_ICDF(25813), + AOM_ICDF(27359), AOM_ICDF(27654), AOM_ICDF(28573), AOM_ICDF(29130), + AOM_ICDF(30551), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11648), AOM_ICDF(14123), AOM_ICDF(16454), AOM_ICDF(19948), + AOM_ICDF(22780), AOM_ICDF(23846), AOM_ICDF(27087), AOM_ICDF(28995), + AOM_ICDF(30380), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9216), AOM_ICDF(12436), AOM_ICDF(15295), AOM_ICDF(17996), + AOM_ICDF(24006), AOM_ICDF(25465), AOM_ICDF(27405), AOM_ICDF(28725), + AOM_ICDF(30383), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9344), AOM_ICDF(12181), AOM_ICDF(14433), AOM_ICDF(16634), + AOM_ICDF(20355), AOM_ICDF(24317), AOM_ICDF(26133), AOM_ICDF(29295), + AOM_ICDF(31344), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8576), AOM_ICDF(10750), AOM_ICDF(12556), AOM_ICDF(17996), + AOM_ICDF(22315), AOM_ICDF(23609), AOM_ICDF(25040), AOM_ICDF(26157), + AOM_ICDF(30573), AOM_ICDF(32768), 0 }, + { AOM_ICDF(11008), AOM_ICDF(13303), AOM_ICDF(15432), AOM_ICDF(20646), + AOM_ICDF(23506), AOM_ICDF(24100), AOM_ICDF(25624), AOM_ICDF(26824), + AOM_ICDF(28055), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9472), AOM_ICDF(12384), AOM_ICDF(14534), AOM_ICDF(17094), + AOM_ICDF(20257), AOM_ICDF(22155), AOM_ICDF(24767), AOM_ICDF(28955), + AOM_ICDF(30474), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7552), AOM_ICDF(14152), AOM_ICDF(17352), AOM_ICDF(22654), + AOM_ICDF(25123), AOM_ICDF(25783), AOM_ICDF(27911), AOM_ICDF(29182), + AOM_ICDF(30849), AOM_ICDF(32768), 0 } }, + { { AOM_ICDF(8064), AOM_ICDF(11538), AOM_ICDF(21987), AOM_ICDF(24941), + AOM_ICDF(26913), AOM_ICDF(28136), AOM_ICDF(29222), AOM_ICDF(30469), + AOM_ICDF(31331), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5504), AOM_ICDF(10403), AOM_ICDF(25080), AOM_ICDF(26762), + AOM_ICDF(27933), AOM_ICDF(29104), AOM_ICDF(30092), AOM_ICDF(31576), + AOM_ICDF(32004), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5632), AOM_ICDF(8706), AOM_ICDF(15097), AOM_ICDF(23714), + AOM_ICDF(25344), AOM_ICDF(26072), AOM_ICDF(27380), AOM_ICDF(28580), + AOM_ICDF(29840), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7424), AOM_ICDF(11186), AOM_ICDF(17593), AOM_ICDF(20154), + AOM_ICDF(22974), AOM_ICDF(24351), AOM_ICDF(26916), AOM_ICDF(29956), + AOM_ICDF(30967), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5888), AOM_ICDF(10193), AOM_ICDF(16895), AOM_ICDF(19031), + AOM_ICDF(23735), AOM_ICDF(25576), AOM_ICDF(27514), AOM_ICDF(29813), + AOM_ICDF(30471), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4864), AOM_ICDF(8352), AOM_ICDF(16459), AOM_ICDF(18062), + AOM_ICDF(21263), AOM_ICDF(25378), AOM_ICDF(26937), AOM_ICDF(30376), + AOM_ICDF(31619), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4992), AOM_ICDF(7922), AOM_ICDF(13842), AOM_ICDF(18004), + AOM_ICDF(21779), AOM_ICDF(23527), AOM_ICDF(25115), AOM_ICDF(27357), + AOM_ICDF(30232), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6656), AOM_ICDF(9716), AOM_ICDF(16379), AOM_ICDF(20053), + AOM_ICDF(22487), AOM_ICDF(23613), AOM_ICDF(25437), AOM_ICDF(27270), + AOM_ICDF(28516), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6016), AOM_ICDF(9674), AOM_ICDF(16891), AOM_ICDF(18684), + AOM_ICDF(21147), AOM_ICDF(23093), AOM_ICDF(25512), AOM_ICDF(30132), + AOM_ICDF(30894), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4608), AOM_ICDF(11318), AOM_ICDF(21038), AOM_ICDF(23650), + AOM_ICDF(25303), AOM_ICDF(26262), AOM_ICDF(28295), AOM_ICDF(30479), + AOM_ICDF(31212), AOM_ICDF(32768), 0 } }, + { { AOM_ICDF(10496), AOM_ICDF(12758), AOM_ICDF(14790), AOM_ICDF(24547), + AOM_ICDF(26342), AOM_ICDF(26799), AOM_ICDF(27825), AOM_ICDF(28443), + AOM_ICDF(30217), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7040), AOM_ICDF(11462), AOM_ICDF(17121), AOM_ICDF(24215), + AOM_ICDF(26504), AOM_ICDF(27267), AOM_ICDF(28492), AOM_ICDF(29444), + AOM_ICDF(30846), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5376), AOM_ICDF(8158), AOM_ICDF(9215), AOM_ICDF(26451), + AOM_ICDF(27407), AOM_ICDF(27524), AOM_ICDF(27995), AOM_ICDF(28275), + AOM_ICDF(29767), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8704), AOM_ICDF(12652), AOM_ICDF(14145), AOM_ICDF(20101), + AOM_ICDF(22879), AOM_ICDF(23675), AOM_ICDF(25629), AOM_ICDF(27079), + AOM_ICDF(28923), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7424), AOM_ICDF(12374), AOM_ICDF(14366), AOM_ICDF(18855), + AOM_ICDF(23842), AOM_ICDF(24358), AOM_ICDF(25639), AOM_ICDF(27087), + AOM_ICDF(29706), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6400), AOM_ICDF(10005), AOM_ICDF(12939), AOM_ICDF(17753), + AOM_ICDF(22206), AOM_ICDF(24790), AOM_ICDF(26785), AOM_ICDF(28164), + AOM_ICDF(30520), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5632), AOM_ICDF(8176), AOM_ICDF(9713), AOM_ICDF(19053), + AOM_ICDF(22343), AOM_ICDF(23222), AOM_ICDF(24453), AOM_ICDF(25070), + AOM_ICDF(29761), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7040), AOM_ICDF(9754), AOM_ICDF(10833), AOM_ICDF(21229), + AOM_ICDF(23540), AOM_ICDF(23943), AOM_ICDF(24839), AOM_ICDF(25675), + AOM_ICDF(27033), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(11758), AOM_ICDF(13481), AOM_ICDF(17236), + AOM_ICDF(20210), AOM_ICDF(21768), AOM_ICDF(24303), AOM_ICDF(26948), + AOM_ICDF(28676), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4864), AOM_ICDF(12712), AOM_ICDF(14201), AOM_ICDF(23863), + AOM_ICDF(25952), AOM_ICDF(26386), AOM_ICDF(27632), AOM_ICDF(28635), + AOM_ICDF(30362), AOM_ICDF(32768), 0 } }, + { { AOM_ICDF(13184), AOM_ICDF(15173), AOM_ICDF(17647), AOM_ICDF(21576), + AOM_ICDF(24474), AOM_ICDF(25267), AOM_ICDF(27699), AOM_ICDF(29283), + AOM_ICDF(30549), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7552), AOM_ICDF(11295), AOM_ICDF(18257), AOM_ICDF(20811), + AOM_ICDF(23213), AOM_ICDF(24606), AOM_ICDF(27731), AOM_ICDF(30407), + AOM_ICDF(31237), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7936), AOM_ICDF(10846), AOM_ICDF(12816), AOM_ICDF(22436), + AOM_ICDF(24614), AOM_ICDF(25130), AOM_ICDF(26890), AOM_ICDF(28199), + AOM_ICDF(29091), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8576), AOM_ICDF(11411), AOM_ICDF(13830), AOM_ICDF(15918), + AOM_ICDF(18996), AOM_ICDF(20044), AOM_ICDF(25114), AOM_ICDF(27835), + AOM_ICDF(28972), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7680), AOM_ICDF(10816), AOM_ICDF(13646), AOM_ICDF(15966), + AOM_ICDF(21162), AOM_ICDF(22012), AOM_ICDF(24701), AOM_ICDF(27506), + AOM_ICDF(29644), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(9423), AOM_ICDF(12524), AOM_ICDF(14773), + AOM_ICDF(19447), AOM_ICDF(22804), AOM_ICDF(26073), AOM_ICDF(29211), + AOM_ICDF(30642), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(8916), AOM_ICDF(11059), AOM_ICDF(15861), + AOM_ICDF(21174), AOM_ICDF(22338), AOM_ICDF(24620), AOM_ICDF(27071), + AOM_ICDF(30899), AOM_ICDF(32768), 0 }, + { AOM_ICDF(9856), AOM_ICDF(11557), AOM_ICDF(13960), AOM_ICDF(18525), + AOM_ICDF(21788), AOM_ICDF(22189), AOM_ICDF(24462), AOM_ICDF(26603), + AOM_ICDF(27470), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7808), AOM_ICDF(10636), AOM_ICDF(13143), AOM_ICDF(15844), + AOM_ICDF(18698), AOM_ICDF(20272), AOM_ICDF(24323), AOM_ICDF(30096), + AOM_ICDF(31787), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6016), AOM_ICDF(10928), AOM_ICDF(14596), AOM_ICDF(18926), + AOM_ICDF(21586), AOM_ICDF(22688), AOM_ICDF(26626), AOM_ICDF(29001), + AOM_ICDF(30399), AOM_ICDF(32768), 0 } }, + { { AOM_ICDF(8832), AOM_ICDF(10983), AOM_ICDF(13451), AOM_ICDF(16582), + AOM_ICDF(21656), AOM_ICDF(23109), AOM_ICDF(24845), AOM_ICDF(26207), + AOM_ICDF(28796), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(10844), AOM_ICDF(15554), AOM_ICDF(18073), + AOM_ICDF(22954), AOM_ICDF(24901), AOM_ICDF(26776), AOM_ICDF(28649), + AOM_ICDF(30419), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5120), AOM_ICDF(8252), AOM_ICDF(10072), AOM_ICDF(20108), + AOM_ICDF(23535), AOM_ICDF(24346), AOM_ICDF(25761), AOM_ICDF(26418), + AOM_ICDF(28675), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7680), AOM_ICDF(11012), AOM_ICDF(12627), AOM_ICDF(14595), + AOM_ICDF(19462), AOM_ICDF(20888), AOM_ICDF(23348), AOM_ICDF(25703), + AOM_ICDF(28159), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6656), AOM_ICDF(9818), AOM_ICDF(11790), AOM_ICDF(13813), + AOM_ICDF(22731), AOM_ICDF(24737), AOM_ICDF(26557), AOM_ICDF(28061), + AOM_ICDF(29697), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5632), AOM_ICDF(8918), AOM_ICDF(11620), AOM_ICDF(13802), + AOM_ICDF(19950), AOM_ICDF(23764), AOM_ICDF(25734), AOM_ICDF(28537), + AOM_ICDF(31809), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4480), AOM_ICDF(6580), AOM_ICDF(7808), AOM_ICDF(12281), + AOM_ICDF(19375), AOM_ICDF(20970), AOM_ICDF(22860), AOM_ICDF(24602), + AOM_ICDF(29929), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7040), AOM_ICDF(9553), AOM_ICDF(11457), AOM_ICDF(15102), + AOM_ICDF(20291), AOM_ICDF(21280), AOM_ICDF(22985), AOM_ICDF(24475), + AOM_ICDF(26613), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6528), AOM_ICDF(10423), AOM_ICDF(12605), AOM_ICDF(14621), + AOM_ICDF(19031), AOM_ICDF(21505), AOM_ICDF(24585), AOM_ICDF(27558), + AOM_ICDF(29532), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6016), AOM_ICDF(11659), AOM_ICDF(14463), AOM_ICDF(18867), + AOM_ICDF(23653), AOM_ICDF(24903), AOM_ICDF(27115), AOM_ICDF(29389), + AOM_ICDF(31382), AOM_ICDF(32768), 0 } }, + { { AOM_ICDF(8192), AOM_ICDF(10016), AOM_ICDF(13304), AOM_ICDF(16362), + AOM_ICDF(21107), AOM_ICDF(25165), AOM_ICDF(26620), AOM_ICDF(28901), + AOM_ICDF(30910), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5888), AOM_ICDF(8723), AOM_ICDF(16237), AOM_ICDF(18318), + AOM_ICDF(22002), AOM_ICDF(25923), AOM_ICDF(27394), AOM_ICDF(29934), + AOM_ICDF(31428), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4608), AOM_ICDF(7138), AOM_ICDF(9841), AOM_ICDF(18442), + AOM_ICDF(22447), AOM_ICDF(24618), AOM_ICDF(26337), AOM_ICDF(27945), + AOM_ICDF(30168), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6784), AOM_ICDF(8916), AOM_ICDF(12270), AOM_ICDF(14851), + AOM_ICDF(19886), AOM_ICDF(22759), AOM_ICDF(25105), AOM_ICDF(28368), + AOM_ICDF(29760), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5120), AOM_ICDF(7928), AOM_ICDF(11324), AOM_ICDF(13340), + AOM_ICDF(21205), AOM_ICDF(24224), AOM_ICDF(25926), AOM_ICDF(28518), + AOM_ICDF(30560), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4480), AOM_ICDF(6580), AOM_ICDF(10058), AOM_ICDF(11237), + AOM_ICDF(16807), AOM_ICDF(25937), AOM_ICDF(27218), AOM_ICDF(30015), + AOM_ICDF(31348), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4608), AOM_ICDF(6808), AOM_ICDF(9445), AOM_ICDF(12446), + AOM_ICDF(18461), AOM_ICDF(21835), AOM_ICDF(23244), AOM_ICDF(26109), + AOM_ICDF(30115), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5760), AOM_ICDF(7659), AOM_ICDF(10798), AOM_ICDF(14720), + AOM_ICDF(19157), AOM_ICDF(21955), AOM_ICDF(23645), AOM_ICDF(26460), + AOM_ICDF(28702), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5760), AOM_ICDF(8503), AOM_ICDF(11157), AOM_ICDF(13071), + AOM_ICDF(17594), AOM_ICDF(22047), AOM_ICDF(24099), AOM_ICDF(29077), + AOM_ICDF(30850), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4864), AOM_ICDF(9660), AOM_ICDF(14264), AOM_ICDF(17105), + AOM_ICDF(21528), AOM_ICDF(24094), AOM_ICDF(26025), AOM_ICDF(28580), + AOM_ICDF(30559), AOM_ICDF(32768), 0 } }, + { { AOM_ICDF(9600), AOM_ICDF(11139), AOM_ICDF(12998), AOM_ICDF(18660), + AOM_ICDF(22158), AOM_ICDF(23501), AOM_ICDF(24659), AOM_ICDF(25736), + AOM_ICDF(30296), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7168), AOM_ICDF(11068), AOM_ICDF(15984), AOM_ICDF(19969), + AOM_ICDF(23169), AOM_ICDF(24704), AOM_ICDF(26216), AOM_ICDF(27572), + AOM_ICDF(31368), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4480), AOM_ICDF(6801), AOM_ICDF(8018), AOM_ICDF(20908), + AOM_ICDF(23071), AOM_ICDF(23583), AOM_ICDF(24301), AOM_ICDF(25062), + AOM_ICDF(29427), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7168), AOM_ICDF(10068), AOM_ICDF(11753), AOM_ICDF(15843), + AOM_ICDF(19742), AOM_ICDF(21358), AOM_ICDF(23809), AOM_ICDF(26189), + AOM_ICDF(29067), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6016), AOM_ICDF(9047), AOM_ICDF(10622), AOM_ICDF(13931), + AOM_ICDF(22462), AOM_ICDF(23858), AOM_ICDF(25911), AOM_ICDF(27277), + AOM_ICDF(29722), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5888), AOM_ICDF(7568), AOM_ICDF(9931), AOM_ICDF(13533), + AOM_ICDF(18431), AOM_ICDF(22063), AOM_ICDF(23777), AOM_ICDF(26025), + AOM_ICDF(30555), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4352), AOM_ICDF(6239), AOM_ICDF(7379), AOM_ICDF(13739), + AOM_ICDF(16917), AOM_ICDF(18090), AOM_ICDF(18835), AOM_ICDF(19651), + AOM_ICDF(30360), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6528), AOM_ICDF(8988), AOM_ICDF(10288), AOM_ICDF(15534), + AOM_ICDF(19495), AOM_ICDF(20386), AOM_ICDF(21934), AOM_ICDF(23034), + AOM_ICDF(26988), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7040), AOM_ICDF(10055), AOM_ICDF(11652), AOM_ICDF(14757), + AOM_ICDF(19622), AOM_ICDF(21715), AOM_ICDF(23615), AOM_ICDF(26761), + AOM_ICDF(29483), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4736), AOM_ICDF(10102), AOM_ICDF(12315), AOM_ICDF(19078), + AOM_ICDF(21348), AOM_ICDF(22621), AOM_ICDF(24246), AOM_ICDF(26044), + AOM_ICDF(29931), AOM_ICDF(32768), 0 } }, + { { AOM_ICDF(10496), AOM_ICDF(12410), AOM_ICDF(14955), AOM_ICDF(19891), + AOM_ICDF(23137), AOM_ICDF(23792), AOM_ICDF(25159), AOM_ICDF(26378), + AOM_ICDF(28125), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7936), AOM_ICDF(12204), AOM_ICDF(17104), AOM_ICDF(20191), + AOM_ICDF(23468), AOM_ICDF(24630), AOM_ICDF(26156), AOM_ICDF(27628), + AOM_ICDF(28913), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6016), AOM_ICDF(8629), AOM_ICDF(10232), AOM_ICDF(23591), + AOM_ICDF(25349), AOM_ICDF(25637), AOM_ICDF(26306), AOM_ICDF(27063), + AOM_ICDF(28980), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8704), AOM_ICDF(12088), AOM_ICDF(13461), AOM_ICDF(16646), + AOM_ICDF(20516), AOM_ICDF(21455), AOM_ICDF(24062), AOM_ICDF(26579), + AOM_ICDF(28368), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7296), AOM_ICDF(11177), AOM_ICDF(13117), AOM_ICDF(16196), + AOM_ICDF(23378), AOM_ICDF(24708), AOM_ICDF(26440), AOM_ICDF(27997), + AOM_ICDF(29078), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6272), AOM_ICDF(9377), AOM_ICDF(12575), AOM_ICDF(15616), + AOM_ICDF(20919), AOM_ICDF(23697), AOM_ICDF(26603), AOM_ICDF(27566), + AOM_ICDF(29903), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6528), AOM_ICDF(9091), AOM_ICDF(10478), AOM_ICDF(16445), + AOM_ICDF(21081), AOM_ICDF(22320), AOM_ICDF(23871), AOM_ICDF(25087), + AOM_ICDF(29258), AOM_ICDF(32768), 0 }, + { AOM_ICDF(8704), AOM_ICDF(11148), AOM_ICDF(12499), AOM_ICDF(17340), + AOM_ICDF(20656), AOM_ICDF(21288), AOM_ICDF(22588), AOM_ICDF(23701), + AOM_ICDF(24693), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7552), AOM_ICDF(11394), AOM_ICDF(12980), AOM_ICDF(15562), + AOM_ICDF(19942), AOM_ICDF(21792), AOM_ICDF(25093), AOM_ICDF(28211), + AOM_ICDF(28959), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5120), AOM_ICDF(11708), AOM_ICDF(13847), AOM_ICDF(19377), + AOM_ICDF(22421), AOM_ICDF(23160), AOM_ICDF(25449), AOM_ICDF(27136), + AOM_ICDF(29182), AOM_ICDF(32768), 0 } }, + { { AOM_ICDF(9984), AOM_ICDF(12031), AOM_ICDF(15190), AOM_ICDF(18673), + AOM_ICDF(21422), AOM_ICDF(22812), AOM_ICDF(25690), AOM_ICDF(29118), + AOM_ICDF(30458), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6144), AOM_ICDF(9680), AOM_ICDF(17436), AOM_ICDF(19610), + AOM_ICDF(21820), AOM_ICDF(23485), AOM_ICDF(26313), AOM_ICDF(30826), + AOM_ICDF(31843), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6016), AOM_ICDF(8315), AOM_ICDF(10607), AOM_ICDF(19333), + AOM_ICDF(21572), AOM_ICDF(22553), AOM_ICDF(25266), AOM_ICDF(27288), + AOM_ICDF(28551), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7168), AOM_ICDF(9668), AOM_ICDF(12646), AOM_ICDF(16257), + AOM_ICDF(19648), AOM_ICDF(20899), AOM_ICDF(25304), AOM_ICDF(30465), + AOM_ICDF(31625), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6144), AOM_ICDF(9368), AOM_ICDF(11836), AOM_ICDF(14130), + AOM_ICDF(19153), AOM_ICDF(21157), AOM_ICDF(24876), AOM_ICDF(28452), + AOM_ICDF(29396), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5504), AOM_ICDF(8486), AOM_ICDF(11996), AOM_ICDF(14412), + AOM_ICDF(17968), AOM_ICDF(21814), AOM_ICDF(24424), AOM_ICDF(30682), + AOM_ICDF(32059), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5376), AOM_ICDF(7195), AOM_ICDF(9592), AOM_ICDF(13331), + AOM_ICDF(17569), AOM_ICDF(19460), AOM_ICDF(22371), AOM_ICDF(25458), + AOM_ICDF(28942), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7424), AOM_ICDF(9206), AOM_ICDF(11783), AOM_ICDF(16456), + AOM_ICDF(19253), AOM_ICDF(20390), AOM_ICDF(23775), AOM_ICDF(27007), + AOM_ICDF(28425), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5888), AOM_ICDF(8303), AOM_ICDF(11361), AOM_ICDF(13440), + AOM_ICDF(15848), AOM_ICDF(17549), AOM_ICDF(21532), AOM_ICDF(29564), + AOM_ICDF(30665), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4608), AOM_ICDF(8788), AOM_ICDF(13284), AOM_ICDF(16621), + AOM_ICDF(18983), AOM_ICDF(20286), AOM_ICDF(24577), AOM_ICDF(28960), + AOM_ICDF(30314), AOM_ICDF(32768), 0 } }, + { { AOM_ICDF(8320), AOM_ICDF(15005), AOM_ICDF(19168), AOM_ICDF(24282), + AOM_ICDF(26707), AOM_ICDF(27402), AOM_ICDF(28681), AOM_ICDF(29639), + AOM_ICDF(30629), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5632), AOM_ICDF(13900), AOM_ICDF(22376), AOM_ICDF(24867), + AOM_ICDF(26804), AOM_ICDF(27734), AOM_ICDF(29130), AOM_ICDF(30722), + AOM_ICDF(31465), AOM_ICDF(32768), 0 }, + { AOM_ICDF(4992), AOM_ICDF(9115), AOM_ICDF(11055), AOM_ICDF(24893), + AOM_ICDF(26316), AOM_ICDF(26661), AOM_ICDF(27663), AOM_ICDF(28301), + AOM_ICDF(29418), AOM_ICDF(32768), 0 }, + { AOM_ICDF(7424), AOM_ICDF(12077), AOM_ICDF(14987), AOM_ICDF(19596), + AOM_ICDF(22615), AOM_ICDF(23600), AOM_ICDF(26465), AOM_ICDF(28484), + AOM_ICDF(29789), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6272), AOM_ICDF(11447), AOM_ICDF(14362), AOM_ICDF(18204), + AOM_ICDF(23418), AOM_ICDF(24715), AOM_ICDF(26697), AOM_ICDF(28547), + AOM_ICDF(29520), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5248), AOM_ICDF(10946), AOM_ICDF(15379), AOM_ICDF(18167), + AOM_ICDF(22197), AOM_ICDF(25432), AOM_ICDF(27295), AOM_ICDF(30031), + AOM_ICDF(30576), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5120), AOM_ICDF(9008), AOM_ICDF(11607), AOM_ICDF(18210), + AOM_ICDF(22327), AOM_ICDF(23427), AOM_ICDF(24887), AOM_ICDF(26580), + AOM_ICDF(29892), AOM_ICDF(32768), 0 }, + { AOM_ICDF(6656), AOM_ICDF(10124), AOM_ICDF(12689), AOM_ICDF(19922), + AOM_ICDF(22480), AOM_ICDF(22807), AOM_ICDF(24441), AOM_ICDF(25579), + AOM_ICDF(26787), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5376), AOM_ICDF(10084), AOM_ICDF(13983), AOM_ICDF(17113), + AOM_ICDF(19996), AOM_ICDF(21614), AOM_ICDF(24403), AOM_ICDF(28651), + AOM_ICDF(29938), AOM_ICDF(32768), 0 }, + { AOM_ICDF(5504), AOM_ICDF(14131), AOM_ICDF(17989), AOM_ICDF(23324), + AOM_ICDF(25513), AOM_ICDF(26071), AOM_ICDF(27850), AOM_ICDF(29464), + AOM_ICDF(30393), AOM_ICDF(32768), 0 } }, +#endif // CONFIG_ALT_INTRA + }; +#endif // CONFIG_EC_MULTISYMBOL + +static void init_mode_probs(FRAME_CONTEXT *fc) { + av1_copy(fc->uv_mode_prob, default_uv_probs); + av1_copy(fc->y_mode_prob, default_if_y_probs); + av1_copy(fc->switchable_interp_prob, default_switchable_interp_prob); + av1_copy(fc->partition_prob, default_partition_probs); + av1_copy(fc->intra_inter_prob, default_intra_inter_p); + av1_copy(fc->comp_inter_prob, default_comp_inter_p); + av1_copy(fc->comp_ref_prob, default_comp_ref_p); +#if CONFIG_LV_MAP + av1_copy(fc->txb_skip, default_txb_skip); + av1_copy(fc->nz_map, default_nz_map); + av1_copy(fc->eob_flag, default_eob_flag); + av1_copy(fc->dc_sign, default_dc_sign); + av1_copy(fc->coeff_base, default_coeff_base); + av1_copy(fc->coeff_lps, default_coeff_lps); +#endif +#if CONFIG_EXT_REFS + av1_copy(fc->comp_bwdref_prob, default_comp_bwdref_p); +#endif // CONFIG_EXT_REFS + av1_copy(fc->single_ref_prob, default_single_ref_p); +#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF + av1_copy(fc->comp_inter_mode_prob, default_comp_inter_mode_p); +#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF + av1_copy(fc->tx_size_probs, default_tx_size_prob); +#if CONFIG_VAR_TX + av1_copy(fc->txfm_partition_prob, default_txfm_partition_probs); +#endif + av1_copy(fc->skip_probs, default_skip_probs); +#if CONFIG_REF_MV + av1_copy(fc->newmv_prob, default_newmv_prob); + av1_copy(fc->zeromv_prob, default_zeromv_prob); + av1_copy(fc->refmv_prob, default_refmv_prob); + av1_copy(fc->drl_prob, default_drl_prob); +#endif // CONFIG_REF_MV + av1_copy(fc->inter_mode_probs, default_inter_mode_probs); +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + av1_copy(fc->motion_mode_prob, default_motion_mode_prob); +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + av1_copy(fc->obmc_prob, default_obmc_prob); +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +#if CONFIG_EXT_INTER + av1_copy(fc->inter_compound_mode_probs, default_inter_compound_mode_probs); +#if CONFIG_COMPOUND_SINGLEREF + av1_copy(fc->inter_singleref_comp_mode_probs, + default_inter_singleref_comp_mode_probs); +#endif // CONFIG_COMPOUND_SINGLEREF + av1_copy(fc->compound_type_prob, default_compound_type_probs); + av1_copy(fc->interintra_prob, default_interintra_prob); + av1_copy(fc->interintra_mode_prob, default_interintra_mode_prob); + av1_copy(fc->wedge_interintra_prob, default_wedge_interintra_prob); +#endif // CONFIG_EXT_INTER +#if CONFIG_SUPERTX + av1_copy(fc->supertx_prob, default_supertx_prob); +#endif // CONFIG_SUPERTX + av1_copy(fc->seg.tree_probs, default_segment_tree_probs); + av1_copy(fc->seg.pred_probs, default_segment_pred_probs); +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP + av1_copy(fc->intra_filter_probs, default_intra_filter_probs); +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + av1_copy(fc->filter_intra_probs, default_filter_intra_probs); +#endif // CONFIG_FILTER_INTRA + av1_copy(fc->inter_ext_tx_prob, default_inter_ext_tx_prob); + av1_copy(fc->intra_ext_tx_prob, default_intra_ext_tx_prob); +#if CONFIG_LOOP_RESTORATION + av1_copy(fc->switchable_restore_prob, default_switchable_restore_prob); +#endif // CONFIG_LOOP_RESTORATION +#if CONFIG_EC_MULTISYMBOL + av1_copy(fc->y_mode_cdf, default_if_y_mode_cdf); + av1_copy(fc->uv_mode_cdf, default_uv_mode_cdf); + av1_copy(fc->switchable_interp_cdf, default_switchable_interp_cdf); + av1_copy(fc->partition_cdf, default_partition_cdf); + av1_copy(fc->inter_mode_cdf, default_inter_mode_cdf); + av1_copy(fc->intra_ext_tx_cdf, default_intra_ext_tx_cdf); + av1_copy(fc->inter_ext_tx_cdf, default_inter_ext_tx_cdf); +#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP + av1_copy(fc->intra_filter_cdf, default_intra_filter_cdf); +#endif // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP + av1_copy(fc->seg.tree_cdf, default_seg_tree_cdf); + av1_copy(fc->tx_size_cdf, default_tx_size_cdf); +#endif // CONFIG_EC_MULTISYMBOL +#if CONFIG_DELTA_Q + av1_copy(fc->delta_q_prob, default_delta_q_probs); +#if CONFIG_EC_MULTISYMBOL + av1_copy(fc->delta_q_cdf, default_delta_q_cdf); +#endif // CONFIG_EC_MULTISYMBOL +#if CONFIG_EXT_DELTA_Q + av1_copy(fc->delta_lf_prob, default_delta_lf_probs); +#if CONFIG_EC_MULTISYMBOL + av1_copy(fc->delta_lf_cdf, default_delta_lf_cdf); +#endif // CONFIG_EC_MULTISYMBOL +#endif +#endif // CONFIG_DELTA_Q +} + +#if CONFIG_EC_MULTISYMBOL +int av1_switchable_interp_ind[SWITCHABLE_FILTERS]; +int av1_switchable_interp_inv[SWITCHABLE_FILTERS]; + +#if !CONFIG_EC_ADAPT +void av1_set_mode_cdfs(struct AV1Common *cm) { + FRAME_CONTEXT *fc = cm->fc; + int i, j; + if (cm->seg.enabled && cm->seg.update_map) { + av1_tree_to_cdf(av1_segment_tree, cm->fc->seg.tree_probs, + cm->fc->seg.tree_cdf); + } + + for (i = 0; i < INTRA_MODES; ++i) + av1_tree_to_cdf(av1_intra_mode_tree, fc->uv_mode_prob[i], + fc->uv_mode_cdf[i]); +#if CONFIG_EXT_PARTITION_TYPES + for (i = 0; i < PARTITION_PLOFFSET; ++i) + av1_tree_to_cdf(av1_partition_tree, fc->partition_prob[i], + fc->partition_cdf[i]); + // Logical index (enum value) to inorder index (tree_to_cdf order) + aom_cdf_prob inorder_partition_cdf[CDF_SIZE(EXT_PARTITION_TYPES)] = {}; + // TODO(aconverse): Generate this dynamically. The assumptions that + // av1_indices_from_tree() makes don't hold for this tree. + static const uint8_t av1_ext_partition_index_map[EXT_PARTITION_TYPES] = { + 0, 1, 4, 7, 2, 3, 5, 6, + }; + for (; i < PARTITION_CONTEXTS; ++i) { + av1_tree_to_cdf(av1_ext_partition_tree, fc->partition_prob[i], + inorder_partition_cdf); + aom_cdf_prob cum_prob = 0; + for (j = 0; j < EXT_PARTITION_TYPES; ++j) { + int inorder_idx = av1_ext_partition_index_map[j]; + aom_cdf_prob prob = + AOM_ICDF(inorder_partition_cdf[inorder_idx]) - + (inorder_idx > 0 ? AOM_ICDF(inorder_partition_cdf[inorder_idx - 1]) + : 0); + cum_prob += prob; + fc->partition_cdf[i][j] = AOM_ICDF(cum_prob); + } + assert(cum_prob == CDF_PROB_TOP); + } +#else + for (i = 0; i < PARTITION_CONTEXTS; ++i) + av1_tree_to_cdf(av1_partition_tree, fc->partition_prob[i], + fc->partition_cdf[i]); +#endif + + for (i = 0; i < INTRA_MODES; ++i) + for (j = 0; j < INTRA_MODES; ++j) + av1_tree_to_cdf(av1_intra_mode_tree, cm->kf_y_prob[i][j], + cm->fc->kf_y_cdf[i][j]); + + for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) + av1_tree_to_cdf(av1_switchable_interp_tree, fc->switchable_interp_prob[j], + fc->switchable_interp_cdf[j]); + + for (i = 0; i < INTER_MODE_CONTEXTS; ++i) + av1_tree_to_cdf(av1_inter_mode_tree, fc->inter_mode_probs[i], + fc->inter_mode_cdf[i]); + + for (i = 0; i < BLOCK_SIZE_GROUPS; ++i) + av1_tree_to_cdf(av1_intra_mode_tree, fc->y_mode_prob[i], fc->y_mode_cdf[i]); + +#if CONFIG_EXT_TX + int s; + for (s = 0; s < EXT_TX_SETS_INTRA; ++s) + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) + for (j = 0; j < INTRA_MODES; ++j) + av1_tree_to_cdf(av1_ext_tx_intra_tree[s], + fc->intra_ext_tx_prob[s][i][j], + fc->intra_ext_tx_cdf[s][i][j]); + + for (s = 0; s < EXT_TX_SETS_INTER; ++s) + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) + av1_tree_to_cdf(av1_ext_tx_inter_tree[s], fc->inter_ext_tx_prob[s][i], + fc->inter_ext_tx_cdf[s][i]); +#else + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) + for (j = 0; j < TX_TYPES; ++j) + av1_tree_to_cdf(av1_ext_tx_tree, fc->intra_ext_tx_prob[i][j], + fc->intra_ext_tx_cdf[i][j]); + + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) + av1_tree_to_cdf(av1_ext_tx_tree, fc->inter_ext_tx_prob[i], + fc->inter_ext_tx_cdf[i]); +#endif + for (i = 0; i < MAX_TX_DEPTH; i++) { + for (j = 0; j < TX_SIZE_CONTEXTS; j++) { + av1_tree_to_cdf(av1_tx_size_tree[i], fc->tx_size_probs[i][j], + fc->tx_size_cdf[i][j]); + } + } +#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP + for (i = 0; i < INTRA_FILTERS + 1; ++i) { + av1_tree_to_cdf(av1_intra_filter_tree, fc->intra_filter_probs[i], + fc->intra_filter_cdf[i]); + } +#endif // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP +} +#endif // !CONFIG_EC_ADAPT +#endif // CONFIG_EC_MULTISYMBOL + +#if CONFIG_DUAL_FILTER +const aom_tree_index av1_switchable_interp_tree[TREE_SIZE(SWITCHABLE_FILTERS)] = + { + -EIGHTTAP_REGULAR, 2, 4, -MULTITAP_SHARP, -EIGHTTAP_SMOOTH, + -EIGHTTAP_SMOOTH2, + }; +#else +const aom_tree_index av1_switchable_interp_tree[TREE_SIZE(SWITCHABLE_FILTERS)] = + { -EIGHTTAP_REGULAR, 2, -EIGHTTAP_SMOOTH, -MULTITAP_SHARP }; +#endif // CONFIG_DUAL_FILTER + +void av1_adapt_inter_frame_probs(AV1_COMMON *cm) { + int i, j; + FRAME_CONTEXT *fc = cm->fc; + const FRAME_CONTEXT *pre_fc = &cm->frame_contexts[cm->frame_context_idx]; + const FRAME_COUNTS *counts = &cm->counts; + + for (i = 0; i < INTRA_INTER_CONTEXTS; i++) + fc->intra_inter_prob[i] = av1_mode_mv_merge_probs( + pre_fc->intra_inter_prob[i], counts->intra_inter[i]); + + for (i = 0; i < COMP_INTER_CONTEXTS; i++) + fc->comp_inter_prob[i] = av1_mode_mv_merge_probs(pre_fc->comp_inter_prob[i], + counts->comp_inter[i]); + +#if CONFIG_EXT_REFS + for (i = 0; i < REF_CONTEXTS; i++) + for (j = 0; j < (FWD_REFS - 1); j++) + fc->comp_ref_prob[i][j] = mode_mv_merge_probs(pre_fc->comp_ref_prob[i][j], + counts->comp_ref[i][j]); + for (i = 0; i < REF_CONTEXTS; i++) + for (j = 0; j < (BWD_REFS - 1); j++) + fc->comp_bwdref_prob[i][j] = mode_mv_merge_probs( + pre_fc->comp_bwdref_prob[i][j], counts->comp_bwdref[i][j]); +#else + for (i = 0; i < REF_CONTEXTS; i++) + for (j = 0; j < (COMP_REFS - 1); j++) + fc->comp_ref_prob[i][j] = mode_mv_merge_probs(pre_fc->comp_ref_prob[i][j], + counts->comp_ref[i][j]); +#endif // CONFIG_EXT_REFS + + for (i = 0; i < REF_CONTEXTS; i++) + for (j = 0; j < (SINGLE_REFS - 1); j++) + fc->single_ref_prob[i][j] = av1_mode_mv_merge_probs( + pre_fc->single_ref_prob[i][j], counts->single_ref[i][j]); + +#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF + for (i = 0; i < COMP_INTER_MODE_CONTEXTS; i++) + fc->comp_inter_mode_prob[i] = av1_mode_mv_merge_probs( + pre_fc->comp_inter_mode_prob[i], counts->comp_inter_mode[i]); + +#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF + +#if CONFIG_REF_MV + for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i) + fc->newmv_prob[i] = + av1_mode_mv_merge_probs(pre_fc->newmv_prob[i], counts->newmv_mode[i]); + for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i) + fc->zeromv_prob[i] = + av1_mode_mv_merge_probs(pre_fc->zeromv_prob[i], counts->zeromv_mode[i]); + for (i = 0; i < REFMV_MODE_CONTEXTS; ++i) + fc->refmv_prob[i] = + av1_mode_mv_merge_probs(pre_fc->refmv_prob[i], counts->refmv_mode[i]); + + for (i = 0; i < DRL_MODE_CONTEXTS; ++i) + fc->drl_prob[i] = + av1_mode_mv_merge_probs(pre_fc->drl_prob[i], counts->drl_mode[i]); +#else + for (i = 0; i < INTER_MODE_CONTEXTS; i++) + aom_tree_merge_probs(av1_inter_mode_tree, pre_fc->inter_mode_probs[i], + counts->inter_mode[i], fc->inter_mode_probs[i]); +#endif + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + for (i = BLOCK_8X8; i < BLOCK_SIZES; ++i) + aom_tree_merge_probs(av1_motion_mode_tree, pre_fc->motion_mode_prob[i], + counts->motion_mode[i], fc->motion_mode_prob[i]); +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + for (i = BLOCK_8X8; i < BLOCK_SIZES; ++i) + fc->obmc_prob[i] = + av1_mode_mv_merge_probs(pre_fc->obmc_prob[i], counts->obmc[i]); +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + +#if CONFIG_SUPERTX + for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) { + for (j = TX_8X8; j < TX_SIZES; ++j) { + fc->supertx_prob[i][j] = av1_mode_mv_merge_probs( + pre_fc->supertx_prob[i][j], counts->supertx[i][j]); + } + } +#endif // CONFIG_SUPERTX + +#if CONFIG_EXT_INTER + for (i = 0; i < INTER_MODE_CONTEXTS; i++) + aom_tree_merge_probs( + av1_inter_compound_mode_tree, pre_fc->inter_compound_mode_probs[i], + counts->inter_compound_mode[i], fc->inter_compound_mode_probs[i]); +#if CONFIG_COMPOUND_SINGLEREF + for (i = 0; i < INTER_MODE_CONTEXTS; i++) + aom_tree_merge_probs(av1_inter_singleref_comp_mode_tree, + pre_fc->inter_singleref_comp_mode_probs[i], + counts->inter_singleref_comp_mode[i], + fc->inter_singleref_comp_mode_probs[i]); +#endif // CONFIG_COMPOUND_SINGLEREF + for (i = 0; i < BLOCK_SIZE_GROUPS; ++i) { + if (is_interintra_allowed_bsize_group(i)) + fc->interintra_prob[i] = av1_mode_mv_merge_probs( + pre_fc->interintra_prob[i], counts->interintra[i]); + } + for (i = 0; i < BLOCK_SIZE_GROUPS; i++) { + aom_tree_merge_probs( + av1_interintra_mode_tree, pre_fc->interintra_mode_prob[i], + counts->interintra_mode[i], fc->interintra_mode_prob[i]); + } + for (i = 0; i < BLOCK_SIZES; ++i) { + if (is_interintra_allowed_bsize(i) && is_interintra_wedge_used(i)) + fc->wedge_interintra_prob[i] = av1_mode_mv_merge_probs( + pre_fc->wedge_interintra_prob[i], counts->wedge_interintra[i]); + } + +#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE + for (i = 0; i < BLOCK_SIZES; ++i) { + aom_tree_merge_probs(av1_compound_type_tree, pre_fc->compound_type_prob[i], + counts->compound_interinter[i], + fc->compound_type_prob[i]); + } +#endif // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE +#endif // CONFIG_EXT_INTER + + for (i = 0; i < BLOCK_SIZE_GROUPS; i++) + aom_tree_merge_probs(av1_intra_mode_tree, pre_fc->y_mode_prob[i], + counts->y_mode[i], fc->y_mode_prob[i]); + + if (cm->interp_filter == SWITCHABLE) { + for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++) + aom_tree_merge_probs( + av1_switchable_interp_tree, pre_fc->switchable_interp_prob[i], + counts->switchable_interp[i], fc->switchable_interp_prob[i]); + } +} + +void av1_adapt_intra_frame_probs(AV1_COMMON *cm) { + int i, j; + FRAME_CONTEXT *fc = cm->fc; + const FRAME_CONTEXT *pre_fc = &cm->frame_contexts[cm->frame_context_idx]; + const FRAME_COUNTS *counts = &cm->counts; + + if (cm->tx_mode == TX_MODE_SELECT) { + for (i = 0; i < MAX_TX_DEPTH; ++i) { + for (j = 0; j < TX_SIZE_CONTEXTS; ++j) + aom_tree_merge_probs(av1_tx_size_tree[i], pre_fc->tx_size_probs[i][j], + counts->tx_size[i][j], fc->tx_size_probs[i][j]); + } + } + +#if CONFIG_VAR_TX + if (cm->tx_mode == TX_MODE_SELECT) { + for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i) + fc->txfm_partition_prob[i] = av1_mode_mv_merge_probs( + pre_fc->txfm_partition_prob[i], counts->txfm_partition[i]); + } +#endif + + for (i = 0; i < SKIP_CONTEXTS; ++i) + fc->skip_probs[i] = + av1_mode_mv_merge_probs(pre_fc->skip_probs[i], counts->skip[i]); + +#if CONFIG_EXT_TX + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + int s; + for (s = 1; s < EXT_TX_SETS_INTER; ++s) { + if (use_inter_ext_tx_for_txsize[s][i]) { + aom_tree_merge_probs( + av1_ext_tx_inter_tree[s], pre_fc->inter_ext_tx_prob[s][i], + counts->inter_ext_tx[s][i], fc->inter_ext_tx_prob[s][i]); + } + } + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { + if (use_intra_ext_tx_for_txsize[s][i]) { + for (j = 0; j < INTRA_MODES; ++j) + aom_tree_merge_probs( + av1_ext_tx_intra_tree[s], pre_fc->intra_ext_tx_prob[s][i][j], + counts->intra_ext_tx[s][i][j], fc->intra_ext_tx_prob[s][i][j]); + } + } + } +#else + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + for (j = 0; j < TX_TYPES; ++j) { + aom_tree_merge_probs(av1_ext_tx_tree, pre_fc->intra_ext_tx_prob[i][j], + counts->intra_ext_tx[i][j], + fc->intra_ext_tx_prob[i][j]); + } + } + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + aom_tree_merge_probs(av1_ext_tx_tree, pre_fc->inter_ext_tx_prob[i], + counts->inter_ext_tx[i], fc->inter_ext_tx_prob[i]); + } +#endif // CONFIG_EXT_TX + + if (cm->seg.temporal_update) { + for (i = 0; i < PREDICTION_PROBS; i++) + fc->seg.pred_probs[i] = av1_mode_mv_merge_probs(pre_fc->seg.pred_probs[i], + counts->seg.pred[i]); + + aom_tree_merge_probs(av1_segment_tree, pre_fc->seg.tree_probs, + counts->seg.tree_mispred, fc->seg.tree_probs); + } else { + aom_tree_merge_probs(av1_segment_tree, pre_fc->seg.tree_probs, + counts->seg.tree_total, fc->seg.tree_probs); + } + + for (i = 0; i < INTRA_MODES; ++i) + aom_tree_merge_probs(av1_intra_mode_tree, pre_fc->uv_mode_prob[i], + counts->uv_mode[i], fc->uv_mode_prob[i]); + +#if CONFIG_EXT_PARTITION_TYPES + for (i = 0; i < PARTITION_PLOFFSET; ++i) + aom_tree_merge_probs(av1_partition_tree, pre_fc->partition_prob[i], + counts->partition[i], fc->partition_prob[i]); + for (; i < PARTITION_CONTEXTS_PRIMARY; ++i) + aom_tree_merge_probs(av1_ext_partition_tree, pre_fc->partition_prob[i], + counts->partition[i], fc->partition_prob[i]); +#else + for (i = 0; i < PARTITION_CONTEXTS_PRIMARY; ++i) { + aom_tree_merge_probs(av1_partition_tree, pre_fc->partition_prob[i], + counts->partition[i], fc->partition_prob[i]); + } +#endif // CONFIG_EXT_PARTITION_TYPES +#if CONFIG_UNPOISON_PARTITION_CTX + for (i = PARTITION_CONTEXTS_PRIMARY; + i < PARTITION_CONTEXTS_PRIMARY + PARTITION_BLOCK_SIZES; ++i) { + unsigned int ct[2] = { counts->partition[i][PARTITION_VERT], + counts->partition[i][PARTITION_SPLIT] }; + assert(counts->partition[i][PARTITION_NONE] == 0); + assert(counts->partition[i][PARTITION_HORZ] == 0); + assert(fc->partition_prob[i][PARTITION_NONE] == 0); + assert(fc->partition_prob[i][PARTITION_HORZ] == 0); + fc->partition_prob[i][PARTITION_VERT] = + av1_mode_mv_merge_probs(pre_fc->partition_prob[i][PARTITION_VERT], ct); + } + for (i = PARTITION_CONTEXTS_PRIMARY + PARTITION_BLOCK_SIZES; + i < PARTITION_CONTEXTS_PRIMARY + 2 * PARTITION_BLOCK_SIZES; ++i) { + unsigned int ct[2] = { counts->partition[i][PARTITION_HORZ], + counts->partition[i][PARTITION_SPLIT] }; + assert(counts->partition[i][PARTITION_NONE] == 0); + assert(counts->partition[i][PARTITION_VERT] == 0); + assert(fc->partition_prob[i][PARTITION_NONE] == 0); + assert(fc->partition_prob[i][PARTITION_VERT] == 0); + fc->partition_prob[i][PARTITION_HORZ] = + av1_mode_mv_merge_probs(pre_fc->partition_prob[i][PARTITION_HORZ], ct); + } +#endif +#if CONFIG_DELTA_Q + for (i = 0; i < DELTA_Q_PROBS; ++i) + fc->delta_q_prob[i] = + mode_mv_merge_probs(pre_fc->delta_q_prob[i], counts->delta_q[i]); +#if CONFIG_EXT_DELTA_Q + for (i = 0; i < DELTA_LF_PROBS; ++i) + fc->delta_lf_prob[i] = + mode_mv_merge_probs(pre_fc->delta_lf_prob[i], counts->delta_lf[i]); +#endif // CONFIG_EXT_DELTA_Q +#endif +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP + for (i = 0; i < INTRA_FILTERS + 1; ++i) { + aom_tree_merge_probs(av1_intra_filter_tree, pre_fc->intra_filter_probs[i], + counts->intra_filter[i], fc->intra_filter_probs[i]); + } +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + for (i = 0; i < PLANE_TYPES; ++i) { + fc->filter_intra_probs[i] = av1_mode_mv_merge_probs( + pre_fc->filter_intra_probs[i], counts->filter_intra[i]); + } +#endif // CONFIG_FILTER_INTRA +} + +static void set_default_lf_deltas(struct loopfilter *lf) { + lf->mode_ref_delta_enabled = 1; + lf->mode_ref_delta_update = 1; + + lf->ref_deltas[INTRA_FRAME] = 1; + lf->ref_deltas[LAST_FRAME] = 0; +#if CONFIG_EXT_REFS + lf->ref_deltas[LAST2_FRAME] = lf->ref_deltas[LAST_FRAME]; + lf->ref_deltas[LAST3_FRAME] = lf->ref_deltas[LAST_FRAME]; + lf->ref_deltas[BWDREF_FRAME] = lf->ref_deltas[LAST_FRAME]; +#endif // CONFIG_EXT_REFS + lf->ref_deltas[GOLDEN_FRAME] = -1; + lf->ref_deltas[ALTREF_FRAME] = -1; + + lf->mode_deltas[0] = 0; + lf->mode_deltas[1] = 0; +} + +void av1_setup_past_independence(AV1_COMMON *cm) { + // Reset the segment feature data to the default stats: + // Features disabled, 0, with delta coding (Default state). + struct loopfilter *const lf = &cm->lf; + + int i; + av1_clearall_segfeatures(&cm->seg); + cm->seg.abs_delta = SEGMENT_DELTADATA; + + if (cm->last_frame_seg_map && !cm->frame_parallel_decode) + memset(cm->last_frame_seg_map, 0, (cm->mi_rows * cm->mi_cols)); + + if (cm->current_frame_seg_map) + memset(cm->current_frame_seg_map, 0, (cm->mi_rows * cm->mi_cols)); + + // Reset the mode ref deltas for loop filter + av1_zero(lf->last_ref_deltas); + av1_zero(lf->last_mode_deltas); + set_default_lf_deltas(lf); + + // To force update of the sharpness + lf->last_sharpness_level = -1; + + av1_default_coef_probs(cm); + init_mode_probs(cm->fc); + av1_init_mv_probs(cm); +#if CONFIG_PVQ + av1_default_pvq_probs(cm); +#endif // CONFIG_PVQ +#if CONFIG_ADAPT_SCAN + av1_init_scan_order(cm); +#endif + av1_convolve_init(cm); + cm->fc->initialized = 1; + + if (cm->frame_type == KEY_FRAME || cm->error_resilient_mode || + cm->reset_frame_context == RESET_FRAME_CONTEXT_ALL) { + // Reset all frame contexts. + for (i = 0; i < FRAME_CONTEXTS; ++i) cm->frame_contexts[i] = *cm->fc; + } else if (cm->reset_frame_context == RESET_FRAME_CONTEXT_CURRENT) { + // Reset only the frame context specified in the frame header. + cm->frame_contexts[cm->frame_context_idx] = *cm->fc; + } + + // prev_mip will only be allocated in encoder. + if (frame_is_intra_only(cm) && cm->prev_mip && !cm->frame_parallel_decode) + memset(cm->prev_mip, 0, + cm->mi_stride * (cm->mi_rows + 1) * sizeof(*cm->prev_mip)); + + cm->frame_context_idx = 0; +} diff --git a/third_party/aom/av1/common/entropymode.h b/third_party/aom/av1/common/entropymode.h new file mode 100644 index 0000000000..9c3a78d611 --- /dev/null +++ b/third_party/aom/av1/common/entropymode.h @@ -0,0 +1,575 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_ENTROPYMODE_H_ +#define AV1_COMMON_ENTROPYMODE_H_ + +#include "av1/common/entropy.h" +#include "av1/common/entropymv.h" +#include "av1/common/filter.h" +#include "av1/common/seg_common.h" +#include "aom_dsp/aom_filter.h" + +#if CONFIG_PVQ +#include "av1/common/pvq.h" +#include "av1/common/pvq_state.h" +#include "av1/common/generic_code.h" +#endif // CONFIG_PVQ + +#ifdef __cplusplus +extern "C" { +#endif + +#define BLOCK_SIZE_GROUPS 4 + +#define TX_SIZE_CONTEXTS 2 + +#define INTER_OFFSET(mode) ((mode)-NEARESTMV) +#if CONFIG_EXT_INTER +#if CONFIG_COMPOUND_SINGLEREF +#define INTER_SINGLEREF_COMP_OFFSET(mode) ((mode)-SR_NEAREST_NEARMV) +#endif // CONFIG_COMPOUND_SINGLEREF +#define INTER_COMPOUND_OFFSET(mode) ((mode)-NEAREST_NEARESTMV) +#endif // CONFIG_EXT_INTER + +#if CONFIG_PALETTE +// Number of possible contexts for a color index. +// As can be seen from av1_get_palette_color_index_context(), the possible +// contexts are (2,0,0), (2,2,1), (3,2,0), (4,1,0), (5,0,0). These are mapped to +// a value from 0 to 4 using 'palette_color_index_context_lookup' table. +#define PALETTE_COLOR_INDEX_CONTEXTS 5 + +// Maximum number of colors in a palette. +#define PALETTE_MAX_SIZE 8 +// Minimum number of colors in a palette. +#define PALETTE_MIN_SIZE 2 + +// Palette mode is available for block sizes >= 8x8. +#define PALETTE_BLOCK_SIZES (BLOCK_LARGEST - BLOCK_8X8 + 1) + +// Palette Y mode context for a block is determined by number of neighboring +// blocks (top and/or left) using a palette for Y plane. So, possible Y mode' +// context values are: +// 0 if neither left nor top block uses palette for Y plane, +// 1 if exactly one of left or top block uses palette for Y plane, and +// 2 if both left and top blocks use palette for Y plane. +#define PALETTE_Y_MODE_CONTEXTS 3 + +// Palette UV mode context for a block is determined by whether this block uses +// palette for the Y plane. So, possible values are: +// 0 if this block doesn't use palette for Y plane. +// 1 if this block uses palette for Y plane (i.e. Y palette size > 0). +#define PALETTE_UV_MODE_CONTEXTS 2 + +#define PALETTE_MAX_BLOCK_SIZE (64 * 64) +#endif // CONFIG_PALETTE + +#if CONFIG_INTRABC +#define INTRABC_PROB 192 +#endif // CONFIG_INTRABC + +struct AV1Common; + +typedef struct { + const int16_t *scan; + const int16_t *iscan; + const int16_t *neighbors; +} SCAN_ORDER; + +struct seg_counts { + unsigned int tree_total[MAX_SEGMENTS]; + unsigned int tree_mispred[MAX_SEGMENTS]; + unsigned int pred[PREDICTION_PROBS][2]; +}; + +typedef struct frame_contexts { + aom_prob y_mode_prob[BLOCK_SIZE_GROUPS][INTRA_MODES - 1]; + aom_prob uv_mode_prob[INTRA_MODES][INTRA_MODES - 1]; +#if CONFIG_EXT_PARTITION_TYPES + aom_prob partition_prob[PARTITION_CONTEXTS][EXT_PARTITION_TYPES - 1]; +#else + aom_prob partition_prob[PARTITION_CONTEXTS][PARTITION_TYPES - 1]; +#endif + av1_coeff_probs_model coef_probs[TX_SIZES][PLANE_TYPES]; +#if CONFIG_NEW_TOKENSET + coeff_cdf_model coef_tail_cdfs[TX_SIZES][PLANE_TYPES]; + coeff_cdf_model coef_head_cdfs[TX_SIZES][PLANE_TYPES]; + aom_prob blockzero_probs[TX_SIZES][PLANE_TYPES][REF_TYPES][BLOCKZ_CONTEXTS]; +#elif CONFIG_EC_MULTISYMBOL + coeff_cdf_model coef_cdfs[TX_SIZES][PLANE_TYPES]; +#endif // CONFIG_NEW_TOKENSET + aom_prob switchable_interp_prob[SWITCHABLE_FILTER_CONTEXTS] + [SWITCHABLE_FILTERS - 1]; +#if CONFIG_ADAPT_SCAN +// TODO(angiebird): try aom_prob +#if CONFIG_CB4X4 + uint32_t non_zero_prob_2x2[TX_TYPES][4]; +#endif + uint32_t non_zero_prob_4X4[TX_TYPES][16]; + uint32_t non_zero_prob_8X8[TX_TYPES][64]; + uint32_t non_zero_prob_16X16[TX_TYPES][256]; + uint32_t non_zero_prob_32X32[TX_TYPES][1024]; + + uint32_t non_zero_prob_4X8[TX_TYPES][32]; + uint32_t non_zero_prob_8X4[TX_TYPES][32]; + uint32_t non_zero_prob_16X8[TX_TYPES][128]; + uint32_t non_zero_prob_8X16[TX_TYPES][128]; + uint32_t non_zero_prob_32X16[TX_TYPES][512]; + uint32_t non_zero_prob_16X32[TX_TYPES][512]; + +#if CONFIG_CB4X4 + DECLARE_ALIGNED(16, int16_t, scan_2x2[TX_TYPES][4]); +#endif + DECLARE_ALIGNED(16, int16_t, scan_4X4[TX_TYPES][16]); + DECLARE_ALIGNED(16, int16_t, scan_8X8[TX_TYPES][64]); + DECLARE_ALIGNED(16, int16_t, scan_16X16[TX_TYPES][256]); + DECLARE_ALIGNED(16, int16_t, scan_32X32[TX_TYPES][1024]); + + DECLARE_ALIGNED(16, int16_t, scan_4X8[TX_TYPES][32]); + DECLARE_ALIGNED(16, int16_t, scan_8X4[TX_TYPES][32]); + DECLARE_ALIGNED(16, int16_t, scan_8X16[TX_TYPES][128]); + DECLARE_ALIGNED(16, int16_t, scan_16X8[TX_TYPES][128]); + DECLARE_ALIGNED(16, int16_t, scan_16X32[TX_TYPES][512]); + DECLARE_ALIGNED(16, int16_t, scan_32X16[TX_TYPES][512]); + +#if CONFIG_CB4X4 + DECLARE_ALIGNED(16, int16_t, iscan_2x2[TX_TYPES][4]); +#endif + DECLARE_ALIGNED(16, int16_t, iscan_4X4[TX_TYPES][16]); + DECLARE_ALIGNED(16, int16_t, iscan_8X8[TX_TYPES][64]); + DECLARE_ALIGNED(16, int16_t, iscan_16X16[TX_TYPES][256]); + DECLARE_ALIGNED(16, int16_t, iscan_32X32[TX_TYPES][1024]); + + DECLARE_ALIGNED(16, int16_t, iscan_4X8[TX_TYPES][32]); + DECLARE_ALIGNED(16, int16_t, iscan_8X4[TX_TYPES][32]); + DECLARE_ALIGNED(16, int16_t, iscan_8X16[TX_TYPES][128]); + DECLARE_ALIGNED(16, int16_t, iscan_16X8[TX_TYPES][128]); + DECLARE_ALIGNED(16, int16_t, iscan_16X32[TX_TYPES][512]); + DECLARE_ALIGNED(16, int16_t, iscan_32X16[TX_TYPES][512]); + +#if CONFIG_CB4X4 + int16_t nb_2x2[TX_TYPES][(4 + 1) * 2]; +#endif + int16_t nb_4X4[TX_TYPES][(16 + 1) * 2]; + int16_t nb_8X8[TX_TYPES][(64 + 1) * 2]; + int16_t nb_16X16[TX_TYPES][(256 + 1) * 2]; + int16_t nb_32X32[TX_TYPES][(1024 + 1) * 2]; + + int16_t nb_4X8[TX_TYPES][(32 + 1) * 2]; + int16_t nb_8X4[TX_TYPES][(32 + 1) * 2]; + int16_t nb_8X16[TX_TYPES][(128 + 1) * 2]; + int16_t nb_16X8[TX_TYPES][(128 + 1) * 2]; + int16_t nb_16X32[TX_TYPES][(512 + 1) * 2]; + int16_t nb_32X16[TX_TYPES][(512 + 1) * 2]; + + SCAN_ORDER sc[TX_SIZES_ALL][TX_TYPES]; + + int16_t eob_threshold[TX_SIZES_ALL][TX_TYPES][EOB_THRESHOLD_NUM]; +#endif // CONFIG_ADAPT_SCAN + +#if CONFIG_LV_MAP + aom_prob txb_skip[TX_SIZES][TXB_SKIP_CONTEXTS]; + aom_prob nz_map[TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS]; + aom_prob eob_flag[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS]; + aom_prob dc_sign[PLANE_TYPES][DC_SIGN_CONTEXTS]; + aom_prob coeff_base[TX_SIZES][PLANE_TYPES][NUM_BASE_LEVELS] + [COEFF_BASE_CONTEXTS]; + aom_prob coeff_lps[TX_SIZES][PLANE_TYPES][LEVEL_CONTEXTS]; +#endif + +#if CONFIG_REF_MV + aom_prob newmv_prob[NEWMV_MODE_CONTEXTS]; + aom_prob zeromv_prob[ZEROMV_MODE_CONTEXTS]; + aom_prob refmv_prob[REFMV_MODE_CONTEXTS]; + aom_prob drl_prob[DRL_MODE_CONTEXTS]; +#endif // CONFIG_REF_MV + + aom_prob inter_mode_probs[INTER_MODE_CONTEXTS][INTER_MODES - 1]; +#if CONFIG_EXT_INTER + aom_prob inter_compound_mode_probs[INTER_MODE_CONTEXTS] + [INTER_COMPOUND_MODES - 1]; +#if CONFIG_COMPOUND_SINGLEREF + aom_prob inter_singleref_comp_mode_probs[INTER_MODE_CONTEXTS] + [INTER_SINGLEREF_COMP_MODES - 1]; +#endif // CONFIG_COMPOUND_SINGLEREF + aom_prob compound_type_prob[BLOCK_SIZES][COMPOUND_TYPES - 1]; + aom_prob interintra_prob[BLOCK_SIZE_GROUPS]; + aom_prob interintra_mode_prob[BLOCK_SIZE_GROUPS][INTERINTRA_MODES - 1]; + aom_prob wedge_interintra_prob[BLOCK_SIZES]; +#endif // CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + aom_prob motion_mode_prob[BLOCK_SIZES][MOTION_MODES - 1]; +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + aom_prob obmc_prob[BLOCK_SIZES]; +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + aom_prob intra_inter_prob[INTRA_INTER_CONTEXTS]; + aom_prob comp_inter_prob[COMP_INTER_CONTEXTS]; + aom_prob single_ref_prob[REF_CONTEXTS][SINGLE_REFS - 1]; +#if CONFIG_EXT_REFS + aom_prob comp_ref_prob[REF_CONTEXTS][FWD_REFS - 1]; + aom_prob comp_bwdref_prob[REF_CONTEXTS][BWD_REFS - 1]; +#else + aom_prob comp_ref_prob[REF_CONTEXTS][COMP_REFS - 1]; +#endif // CONFIG_EXT_REFS +#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF + aom_prob comp_inter_mode_prob[COMP_INTER_MODE_CONTEXTS]; +#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF + aom_prob tx_size_probs[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH]; +#if CONFIG_VAR_TX + aom_prob txfm_partition_prob[TXFM_PARTITION_CONTEXTS]; +#endif + aom_prob skip_probs[SKIP_CONTEXTS]; +#if CONFIG_REF_MV + nmv_context nmvc[NMV_CONTEXTS]; +#else + nmv_context nmvc; +#endif +#if CONFIG_INTRABC + nmv_context ndvc; +#endif + int initialized; +#if CONFIG_EXT_TX + aom_prob inter_ext_tx_prob[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES - 1]; + aom_prob intra_ext_tx_prob[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES] + [TX_TYPES - 1]; +#else + aom_prob intra_ext_tx_prob[EXT_TX_SIZES][TX_TYPES][TX_TYPES - 1]; + aom_prob inter_ext_tx_prob[EXT_TX_SIZES][TX_TYPES - 1]; +#endif // CONFIG_EXT_TX +#if CONFIG_SUPERTX + aom_prob supertx_prob[PARTITION_SUPERTX_CONTEXTS][TX_SIZES]; +#endif // CONFIG_SUPERTX + struct segmentation_probs seg; +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP + aom_prob intra_filter_probs[INTRA_FILTERS + 1][INTRA_FILTERS - 1]; +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + aom_prob filter_intra_probs[PLANE_TYPES]; +#endif // CONFIG_FILTER_INTRA +#if CONFIG_GLOBAL_MOTION + aom_prob global_motion_types_prob[GLOBAL_TRANS_TYPES - 1]; +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_LOOP_RESTORATION + aom_prob switchable_restore_prob[RESTORE_SWITCHABLE_TYPES - 1]; +#endif // CONFIG_LOOP_RESTORATION +#if CONFIG_EC_MULTISYMBOL + aom_cdf_prob y_mode_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(INTRA_MODES)]; + aom_cdf_prob uv_mode_cdf[INTRA_MODES][CDF_SIZE(INTRA_MODES)]; +#if CONFIG_EXT_PARTITION_TYPES + aom_cdf_prob partition_cdf[PARTITION_CONTEXTS][CDF_SIZE(EXT_PARTITION_TYPES)]; +#else + aom_cdf_prob partition_cdf[PARTITION_CONTEXTS][CDF_SIZE(PARTITION_TYPES)]; +#endif + aom_cdf_prob switchable_interp_cdf[SWITCHABLE_FILTER_CONTEXTS] + [CDF_SIZE(SWITCHABLE_FILTERS)]; + aom_cdf_prob inter_mode_cdf[INTER_MODE_CONTEXTS][CDF_SIZE(INTER_MODES)]; + /* Keep track of kf_y_cdf here, as this makes handling + multiple copies for adaptation in tiles easier */ + aom_cdf_prob kf_y_cdf[INTRA_MODES][INTRA_MODES][CDF_SIZE(INTRA_MODES)]; + aom_cdf_prob tx_size_cdf[MAX_TX_DEPTH][TX_SIZE_CONTEXTS] + [CDF_SIZE(MAX_TX_DEPTH + 1)]; +#if CONFIG_DELTA_Q + aom_cdf_prob delta_q_cdf[CDF_SIZE(DELTA_Q_PROBS + 1)]; +#if CONFIG_EXT_DELTA_Q + aom_cdf_prob delta_lf_cdf[CDF_SIZE(DELTA_LF_PROBS + 1)]; +#endif +#endif // CONFIG_DELTA_Q +#if CONFIG_EXT_TX + aom_cdf_prob intra_ext_tx_cdf[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES] + [CDF_SIZE(TX_TYPES)]; + aom_cdf_prob inter_ext_tx_cdf[EXT_TX_SETS_INTER][EXT_TX_SIZES] + [CDF_SIZE(TX_TYPES)]; +#else + aom_cdf_prob intra_ext_tx_cdf[EXT_TX_SIZES][TX_TYPES][CDF_SIZE(TX_TYPES)]; + aom_cdf_prob inter_ext_tx_cdf[EXT_TX_SIZES][CDF_SIZE(TX_TYPES)]; +#endif // CONFIG_EXT_TX +#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP + aom_cdf_prob intra_filter_cdf[INTRA_FILTERS + 1][CDF_SIZE(INTRA_FILTERS)]; +#endif // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP +#endif // CONFIG_EC_MULTISYMBOL +#if CONFIG_DELTA_Q + aom_prob delta_q_prob[DELTA_Q_PROBS]; +#if CONFIG_EXT_DELTA_Q + aom_prob delta_lf_prob[DELTA_LF_PROBS]; +#endif +#endif +#if CONFIG_PVQ + // TODO(any): If PVQ is enabled, most of coefficient related cdf, + // such as coef_cdfs[], coef_tail_cdfs[], and coef_heaf_cdfs[] can be removed. + od_adapt_ctx pvq_context; +#endif // CONFIG_PVQ +} FRAME_CONTEXT; + +typedef struct FRAME_COUNTS { + // Note: This structure should only contain 'unsigned int' fields, or + // aggregates built solely from 'unsigned int' fields/elements + unsigned int kf_y_mode[INTRA_MODES][INTRA_MODES][INTRA_MODES]; + unsigned int y_mode[BLOCK_SIZE_GROUPS][INTRA_MODES]; + unsigned int uv_mode[INTRA_MODES][INTRA_MODES]; +#if CONFIG_EXT_PARTITION_TYPES + unsigned int partition[PARTITION_CONTEXTS][EXT_PARTITION_TYPES]; +#else + unsigned int partition[PARTITION_CONTEXTS][PARTITION_TYPES]; +#endif + av1_coeff_count_model coef[TX_SIZES][PLANE_TYPES]; + unsigned int eob_branch[TX_SIZES][PLANE_TYPES][REF_TYPES][COEF_BANDS] + [COEFF_CONTEXTS]; + unsigned int switchable_interp[SWITCHABLE_FILTER_CONTEXTS] + [SWITCHABLE_FILTERS]; +#if CONFIG_ADAPT_SCAN +#if CONFIG_CB4X4 + unsigned int non_zero_count_2x2[TX_TYPES][4]; +#endif // CONFIG_CB4X4 + unsigned int non_zero_count_4X4[TX_TYPES][16]; + unsigned int non_zero_count_8X8[TX_TYPES][64]; + unsigned int non_zero_count_16X16[TX_TYPES][256]; + unsigned int non_zero_count_32X32[TX_TYPES][1024]; + + unsigned int non_zero_count_4x8[TX_TYPES][32]; + unsigned int non_zero_count_8x4[TX_TYPES][32]; + unsigned int non_zero_count_8x16[TX_TYPES][128]; + unsigned int non_zero_count_16x8[TX_TYPES][128]; + unsigned int non_zero_count_16x32[TX_TYPES][512]; + unsigned int non_zero_count_32x16[TX_TYPES][512]; + + unsigned int txb_count[TX_SIZES_ALL][TX_TYPES]; +#endif // CONFIG_ADAPT_SCAN + +#if CONFIG_LV_MAP + unsigned int txb_skip[TX_SIZES][TXB_SKIP_CONTEXTS][2]; + unsigned int nz_map[TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS][2]; + unsigned int eob_flag[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS][2]; + unsigned int dc_sign[PLANE_TYPES][DC_SIGN_CONTEXTS][2]; + unsigned int coeff_base[TX_SIZES][PLANE_TYPES][NUM_BASE_LEVELS] + [COEFF_BASE_CONTEXTS][2]; + unsigned int coeff_lps[TX_SIZES][PLANE_TYPES][LEVEL_CONTEXTS][2]; +#endif // CONFIG_LV_MAP + +#if CONFIG_EC_MULTISYMBOL + av1_blockz_count_model blockz_count[TX_SIZES][PLANE_TYPES]; +#endif + +#if CONFIG_REF_MV + unsigned int newmv_mode[NEWMV_MODE_CONTEXTS][2]; + unsigned int zeromv_mode[ZEROMV_MODE_CONTEXTS][2]; + unsigned int refmv_mode[REFMV_MODE_CONTEXTS][2]; + unsigned int drl_mode[DRL_MODE_CONTEXTS][2]; +#endif + + unsigned int inter_mode[INTER_MODE_CONTEXTS][INTER_MODES]; +#if CONFIG_EXT_INTER + unsigned int inter_compound_mode[INTER_MODE_CONTEXTS][INTER_COMPOUND_MODES]; +#if CONFIG_COMPOUND_SINGLEREF + unsigned int inter_singleref_comp_mode[INTER_MODE_CONTEXTS] + [INTER_SINGLEREF_COMP_MODES]; +#endif // CONFIG_COMPOUND_SINGLEREF + unsigned int interintra[BLOCK_SIZE_GROUPS][2]; + unsigned int interintra_mode[BLOCK_SIZE_GROUPS][INTERINTRA_MODES]; + unsigned int wedge_interintra[BLOCK_SIZES][2]; + unsigned int compound_interinter[BLOCK_SIZES][COMPOUND_TYPES]; +#endif // CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + unsigned int motion_mode[BLOCK_SIZES][MOTION_MODES]; +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + unsigned int obmc[BLOCK_SIZES][2]; +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + unsigned int intra_inter[INTRA_INTER_CONTEXTS][2]; + unsigned int comp_inter[COMP_INTER_CONTEXTS][2]; + unsigned int single_ref[REF_CONTEXTS][SINGLE_REFS - 1][2]; +#if CONFIG_EXT_REFS + unsigned int comp_ref[REF_CONTEXTS][FWD_REFS - 1][2]; + unsigned int comp_bwdref[REF_CONTEXTS][BWD_REFS - 1][2]; +#else + unsigned int comp_ref[REF_CONTEXTS][COMP_REFS - 1][2]; +#endif // CONFIG_EXT_REFS +#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF + unsigned int comp_inter_mode[COMP_INTER_MODE_CONTEXTS][2]; +#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF + // TODO(any): tx_size_totals is only used by the encoder to decide whether + // to use forward updates for the coeff probs, and as such it does not really + // belong into this structure. + unsigned int tx_size_totals[TX_SIZES]; + unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][TX_SIZES]; +#if CONFIG_VAR_TX + unsigned int txfm_partition[TXFM_PARTITION_CONTEXTS][2]; +#endif + unsigned int skip[SKIP_CONTEXTS][2]; +#if CONFIG_REF_MV + nmv_context_counts mv[NMV_CONTEXTS]; +#else + nmv_context_counts mv; +#endif +#if CONFIG_INTRABC + nmv_context_counts dv; +#endif +#if CONFIG_DELTA_Q + unsigned int delta_q[DELTA_Q_PROBS][2]; +#if CONFIG_EXT_DELTA_Q + unsigned int delta_lf[DELTA_LF_PROBS][2]; +#endif +#endif +#if CONFIG_EXT_TX +#if CONFIG_RECT_TX + unsigned int tx_size_implied[TX_SIZES][TX_SIZES]; +#endif // CONFIG_RECT_TX + unsigned int inter_ext_tx[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES]; + unsigned int intra_ext_tx[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES] + [TX_TYPES]; +#else + unsigned int intra_ext_tx[EXT_TX_SIZES][TX_TYPES][TX_TYPES]; + unsigned int inter_ext_tx[EXT_TX_SIZES][TX_TYPES]; +#endif // CONFIG_EXT_TX +#if CONFIG_SUPERTX + unsigned int supertx[PARTITION_SUPERTX_CONTEXTS][TX_SIZES][2]; + unsigned int supertx_size[TX_SIZES]; +#endif // CONFIG_SUPERTX + struct seg_counts seg; +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP + unsigned int intra_filter[INTRA_FILTERS + 1][INTRA_FILTERS]; +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + unsigned int filter_intra[PLANE_TYPES][2]; +#endif // CONFIG_FILTER_INTRA +} FRAME_COUNTS; + +// Default probabilities for signaling Intra mode for Y plane -- used only for +// intra-only frames. ('default_if_y_probs' is used for inter frames). +// Contexts used: Intra mode (Y plane) of 'above' and 'left' blocks. +extern const aom_prob av1_kf_y_mode_prob[INTRA_MODES][INTRA_MODES] + [INTRA_MODES - 1]; +#if CONFIG_EC_MULTISYMBOL +// CDF version of 'av1_kf_y_mode_prob'. +extern const aom_cdf_prob av1_kf_y_mode_cdf[INTRA_MODES][INTRA_MODES] + [CDF_SIZE(INTRA_MODES)]; +#endif + +#if CONFIG_PALETTE +extern const aom_prob av1_default_palette_y_mode_prob[PALETTE_BLOCK_SIZES] + [PALETTE_Y_MODE_CONTEXTS]; +extern const aom_prob + av1_default_palette_uv_mode_prob[PALETTE_UV_MODE_CONTEXTS]; +extern const aom_prob av1_default_palette_y_size_prob[PALETTE_BLOCK_SIZES] + [PALETTE_SIZES - 1]; +extern const aom_prob av1_default_palette_uv_size_prob[PALETTE_BLOCK_SIZES] + [PALETTE_SIZES - 1]; +extern const aom_prob av1_default_palette_y_color_index_prob + [PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS][PALETTE_COLORS - 1]; +extern const aom_prob av1_default_palette_uv_color_index_prob + [PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS][PALETTE_COLORS - 1]; +#endif // CONFIG_PALETTE + +extern const aom_tree_index av1_intra_mode_tree[TREE_SIZE(INTRA_MODES)]; +extern const aom_tree_index av1_inter_mode_tree[TREE_SIZE(INTER_MODES)]; +#if CONFIG_EC_MULTISYMBOL +extern int av1_intra_mode_ind[INTRA_MODES]; +extern int av1_intra_mode_inv[INTRA_MODES]; +extern int av1_inter_mode_ind[INTER_MODES]; +extern int av1_inter_mode_inv[INTER_MODES]; +#if CONFIG_EXT_TX +extern int av1_ext_tx_intra_ind[EXT_TX_SETS_INTRA][TX_TYPES]; +extern int av1_ext_tx_intra_inv[EXT_TX_SETS_INTRA][TX_TYPES]; +extern int av1_ext_tx_inter_ind[EXT_TX_SETS_INTER][TX_TYPES]; +extern int av1_ext_tx_inter_inv[EXT_TX_SETS_INTER][TX_TYPES]; +#endif +#endif + +#if CONFIG_EXT_INTER +extern const aom_tree_index + av1_interintra_mode_tree[TREE_SIZE(INTERINTRA_MODES)]; +extern const aom_tree_index + av1_inter_compound_mode_tree[TREE_SIZE(INTER_COMPOUND_MODES)]; +#if CONFIG_COMPOUND_SINGLEREF +extern const aom_tree_index + av1_inter_singleref_comp_mode_tree[TREE_SIZE(INTER_SINGLEREF_COMP_MODES)]; +#endif // CONFIG_COMPOUND_SINGLEREF +extern const aom_tree_index av1_compound_type_tree[TREE_SIZE(COMPOUND_TYPES)]; +#endif // CONFIG_EXT_INTER +extern const aom_tree_index av1_partition_tree[TREE_SIZE(PARTITION_TYPES)]; +#if CONFIG_EXT_PARTITION_TYPES +extern const aom_tree_index + av1_ext_partition_tree[TREE_SIZE(EXT_PARTITION_TYPES)]; +#endif +extern const aom_tree_index + av1_switchable_interp_tree[TREE_SIZE(SWITCHABLE_FILTERS)]; +#if CONFIG_PALETTE +extern const aom_tree_index av1_palette_size_tree[TREE_SIZE(PALETTE_SIZES)]; +extern const aom_tree_index + av1_palette_color_index_tree[PALETTE_SIZES][TREE_SIZE(PALETTE_COLORS)]; +#endif // CONFIG_PALETTE +extern const aom_tree_index av1_tx_size_tree[MAX_TX_DEPTH][TREE_SIZE(TX_SIZES)]; +#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP +extern const aom_tree_index av1_intra_filter_tree[TREE_SIZE(INTRA_FILTERS)]; +#endif // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP +#if CONFIG_EXT_TX +extern const aom_tree_index av1_ext_tx_inter_tree[EXT_TX_SETS_INTER] + [TREE_SIZE(TX_TYPES)]; +extern const aom_tree_index av1_ext_tx_intra_tree[EXT_TX_SETS_INTRA] + [TREE_SIZE(TX_TYPES)]; +#else +extern const aom_tree_index av1_ext_tx_tree[TREE_SIZE(TX_TYPES)]; +#endif // CONFIG_EXT_TX +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +extern const aom_tree_index av1_motion_mode_tree[TREE_SIZE(MOTION_MODES)]; +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + +#if CONFIG_LOOP_RESTORATION +#define RESTORE_NONE_SGRPROJ_PROB 64 +#define RESTORE_NONE_BILATERAL_PROB 16 +#define RESTORE_NONE_WIENER_PROB 64 +#define RESTORE_NONE_DOMAINTXFMRF_PROB 64 +extern const aom_tree_index + av1_switchable_restore_tree[TREE_SIZE(RESTORE_SWITCHABLE_TYPES)]; +#endif // CONFIG_LOOP_RESTORATION +#if CONFIG_EC_MULTISYMBOL +extern int av1_switchable_interp_ind[SWITCHABLE_FILTERS]; +extern int av1_switchable_interp_inv[SWITCHABLE_FILTERS]; + +void av1_set_mode_cdfs(struct AV1Common *cm); +#endif + +void av1_setup_past_independence(struct AV1Common *cm); + +void av1_adapt_intra_frame_probs(struct AV1Common *cm); +void av1_adapt_inter_frame_probs(struct AV1Common *cm); +#if CONFIG_EC_MULTISYMBOL && !CONFIG_EXT_TX +extern int av1_ext_tx_ind[TX_TYPES]; +extern int av1_ext_tx_inv[TX_TYPES]; +#endif + +static INLINE int av1_ceil_log2(int n) { + int i = 1, p = 2; + while (p < n) { + i++; + p = p << 1; + } + return i; +} + +#if CONFIG_PALETTE +// Returns the context for palette color index at row 'r' and column 'c', +// along with the 'color_order' of neighbors and the 'color_idx'. +// The 'color_map' is a 2D array with the given 'stride'. +int av1_get_palette_color_index_context(const uint8_t *color_map, int stride, + int r, int c, int palette_size, + uint8_t *color_order, int *color_idx); +#endif // CONFIG_PALETTE + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_ENTROPYMODE_H_ diff --git a/third_party/aom/av1/common/entropymv.c b/third_party/aom/av1/common/entropymv.c new file mode 100644 index 0000000000..9c162d2c50 --- /dev/null +++ b/third_party/aom/av1/common/entropymv.c @@ -0,0 +1,315 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/common/onyxc_int.h" +#include "av1/common/entropymv.h" + +// Integer pel reference mv threshold for use of high-precision 1/8 mv +#define COMPANDED_MVREF_THRESH 8 + +const aom_tree_index av1_mv_joint_tree[TREE_SIZE(MV_JOINTS)] = { + -MV_JOINT_ZERO, 2, -MV_JOINT_HNZVZ, 4, -MV_JOINT_HZVNZ, -MV_JOINT_HNZVNZ +}; + +/* clang-format off */ +const aom_tree_index av1_mv_class_tree[TREE_SIZE(MV_CLASSES)] = { + -MV_CLASS_0, 2, + -MV_CLASS_1, 4, + 6, 8, + -MV_CLASS_2, -MV_CLASS_3, + 10, 12, + -MV_CLASS_4, -MV_CLASS_5, + -MV_CLASS_6, 14, + 16, 18, + -MV_CLASS_7, -MV_CLASS_8, + -MV_CLASS_9, -MV_CLASS_10, +}; +/* clang-format on */ + +const aom_tree_index av1_mv_class0_tree[TREE_SIZE(CLASS0_SIZE)] = { + -0, -1, +}; + +const aom_tree_index av1_mv_fp_tree[TREE_SIZE(MV_FP_SIZE)] = { -0, 2, -1, + 4, -2, -3 }; + +static const nmv_context default_nmv_context = { + { 32, 64, 96 }, // joints +#if CONFIG_EC_MULTISYMBOL + { AOM_ICDF(4096), AOM_ICDF(11264), AOM_ICDF(19328), AOM_ICDF(32768), + 0 }, // joint_cdf +#endif + { { + // Vertical component + 128, // sign + { 224, 144, 192, 168, 192, 176, 192, 198, 198, 245 }, // class +#if CONFIG_EC_MULTISYMBOL + { AOM_ICDF(28672), AOM_ICDF(30976), AOM_ICDF(31858), AOM_ICDF(32320), + AOM_ICDF(32551), AOM_ICDF(32656), AOM_ICDF(32740), AOM_ICDF(32757), + AOM_ICDF(32762), AOM_ICDF(32767), AOM_ICDF(32768), 0 }, // class_cdf +#endif + { 216 }, // class0 + { 136, 140, 148, 160, 176, 192, 224, 234, 234, 240 }, // bits + { { 128, 128, 64 }, { 96, 112, 64 } }, // class0_fp + { 64, 96, 64 }, // fp +#if CONFIG_EC_MULTISYMBOL + { { AOM_ICDF(16384), AOM_ICDF(24576), AOM_ICDF(26624), AOM_ICDF(32768), + 0 }, + { AOM_ICDF(12288), AOM_ICDF(21248), AOM_ICDF(24128), AOM_ICDF(32768), + 0 } }, // class0_fp_cdf + { AOM_ICDF(8192), AOM_ICDF(17408), AOM_ICDF(21248), AOM_ICDF(32768), + 0 }, // fp_cdf +#endif + 160, // class0_hp bit + 128, // hp + }, + { + // Horizontal component + 128, // sign + { 216, 128, 176, 160, 176, 176, 192, 198, 198, 208 }, // class +#if CONFIG_EC_MULTISYMBOL + { AOM_ICDF(28672), AOM_ICDF(30976), AOM_ICDF(31858), AOM_ICDF(32320), + AOM_ICDF(32551), AOM_ICDF(32656), AOM_ICDF(32740), AOM_ICDF(32757), + AOM_ICDF(32762), AOM_ICDF(32767), AOM_ICDF(32768), 0 }, // class_cdf +#endif + { 208 }, // class0 + { 136, 140, 148, 160, 176, 192, 224, 234, 234, 240 }, // bits + { { 128, 128, 64 }, { 96, 112, 64 } }, // class0_fp + { 64, 96, 64 }, // fp +#if CONFIG_EC_MULTISYMBOL + { { AOM_ICDF(16384), AOM_ICDF(24576), AOM_ICDF(26624), AOM_ICDF(32768), + 0 }, + { AOM_ICDF(12288), AOM_ICDF(21248), AOM_ICDF(24128), AOM_ICDF(32768), + 0 } }, // class0_fp_cdf + { AOM_ICDF(8192), AOM_ICDF(17408), AOM_ICDF(21248), AOM_ICDF(32768), + 0 }, // fp_cdf +#endif + 160, // class0_hp bit + 128, // hp + } }, +}; + +static const uint8_t log_in_base_2[] = { + 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10 +}; + +#if CONFIG_GLOBAL_MOTION +#if GLOBAL_TRANS_TYPES == 7 // All models +const aom_tree_index av1_global_motion_types_tree[TREE_SIZE( + GLOBAL_TRANS_TYPES)] = { -IDENTITY, 2, -TRANSLATION, 4, + -ROTZOOM, 6, -AFFINE, 8, + -HOMOGRAPHY, 10, -HORTRAPEZOID, -VERTRAPEZOID }; + +static const aom_prob default_global_motion_types_prob[GLOBAL_TRANS_TYPES - 1] = + { 224, 128, 192, 192, 32, 128 }; + +#elif GLOBAL_TRANS_TYPES == 6 // Do not allow full homography +const aom_tree_index + av1_global_motion_types_tree[TREE_SIZE(GLOBAL_TRANS_TYPES)] = { + -IDENTITY, 2, -TRANSLATION, 4, -ROTZOOM, 6, -AFFINE, 8, -HORTRAPEZOID, + -VERTRAPEZOID + }; + +static const aom_prob default_global_motion_types_prob[GLOBAL_TRANS_TYPES - 1] = + { 224, 128, 192, 192, 128 }; + +#elif GLOBAL_TRANS_TYPES == 4 // Upto Affine +const aom_tree_index av1_global_motion_types_tree[TREE_SIZE( + GLOBAL_TRANS_TYPES)] = { -IDENTITY, 2, -TRANSLATION, 4, -ROTZOOM, -AFFINE }; + +static const aom_prob default_global_motion_types_prob[GLOBAL_TRANS_TYPES - 1] = + { 224, 128, 240 }; + +#elif GLOBAL_TRANS_TYPES == 3 // Upto rotation-zoom + +const aom_tree_index av1_global_motion_types_tree[TREE_SIZE( + GLOBAL_TRANS_TYPES)] = { -IDENTITY, 2, -TRANSLATION, -ROTZOOM }; + +static const aom_prob default_global_motion_types_prob[GLOBAL_TRANS_TYPES - 1] = + { 224, 128 }; +#endif // GLOBAL_TRANS_TYPES +#endif // CONFIG_GLOBAL_MOTION + +static INLINE int mv_class_base(MV_CLASS_TYPE c) { + return c ? CLASS0_SIZE << (c + 2) : 0; +} + +MV_CLASS_TYPE av1_get_mv_class(int z, int *offset) { + const MV_CLASS_TYPE c = (z >= CLASS0_SIZE * 4096) + ? MV_CLASS_10 + : (MV_CLASS_TYPE)log_in_base_2[z >> 3]; + if (offset) *offset = z - mv_class_base(c); + return c; +} + +static void inc_mv_component(int v, nmv_component_counts *comp_counts, int incr, + int usehp) { + int s, z, c, o, d, e, f; + assert(v != 0); /* should not be zero */ + s = v < 0; + comp_counts->sign[s] += incr; + z = (s ? -v : v) - 1; /* magnitude - 1 */ + + c = av1_get_mv_class(z, &o); + comp_counts->classes[c] += incr; + + d = (o >> 3); /* int mv data */ + f = (o >> 1) & 3; /* fractional pel mv data */ + e = (o & 1); /* high precision mv data */ + + if (c == MV_CLASS_0) { + comp_counts->class0[d] += incr; + comp_counts->class0_fp[d][f] += incr; + if (usehp) comp_counts->class0_hp[e] += incr; + } else { + int i; + int b = c + CLASS0_BITS - 1; // number of bits + for (i = 0; i < b; ++i) comp_counts->bits[i][((d >> i) & 1)] += incr; + comp_counts->fp[f] += incr; + if (usehp) comp_counts->hp[e] += incr; + } +} + +void av1_inc_mv(const MV *mv, nmv_context_counts *counts, const int usehp) { + if (counts != NULL) { + const MV_JOINT_TYPE j = av1_get_mv_joint(mv); + ++counts->joints[j]; + + if (mv_joint_vertical(j)) + inc_mv_component(mv->row, &counts->comps[0], 1, usehp); + + if (mv_joint_horizontal(j)) + inc_mv_component(mv->col, &counts->comps[1], 1, usehp); + } +} + +void av1_adapt_mv_probs(AV1_COMMON *cm, int allow_hp) { + int i, j; +#if CONFIG_REF_MV + int idx; + for (idx = 0; idx < NMV_CONTEXTS; ++idx) { + nmv_context *fc = &cm->fc->nmvc[idx]; + const nmv_context *pre_fc = + &cm->frame_contexts[cm->frame_context_idx].nmvc[idx]; + const nmv_context_counts *counts = &cm->counts.mv[idx]; +#else + nmv_context *fc = &cm->fc->nmvc; + const nmv_context *pre_fc = &cm->frame_contexts[cm->frame_context_idx].nmvc; + const nmv_context_counts *counts = &cm->counts.mv; +#endif // CONFIG_REF_MV + aom_tree_merge_probs(av1_mv_joint_tree, pre_fc->joints, counts->joints, + fc->joints); + for (i = 0; i < 2; ++i) { + nmv_component *comp = &fc->comps[i]; + const nmv_component *pre_comp = &pre_fc->comps[i]; + const nmv_component_counts *c = &counts->comps[i]; + + comp->sign = av1_mode_mv_merge_probs(pre_comp->sign, c->sign); + aom_tree_merge_probs(av1_mv_class_tree, pre_comp->classes, c->classes, + comp->classes); + aom_tree_merge_probs(av1_mv_class0_tree, pre_comp->class0, c->class0, + comp->class0); + + for (j = 0; j < MV_OFFSET_BITS; ++j) + comp->bits[j] = av1_mode_mv_merge_probs(pre_comp->bits[j], c->bits[j]); + + for (j = 0; j < CLASS0_SIZE; ++j) + aom_tree_merge_probs(av1_mv_fp_tree, pre_comp->class0_fp[j], + c->class0_fp[j], comp->class0_fp[j]); + + aom_tree_merge_probs(av1_mv_fp_tree, pre_comp->fp, c->fp, comp->fp); + + if (allow_hp) { + comp->class0_hp = + av1_mode_mv_merge_probs(pre_comp->class0_hp, c->class0_hp); + comp->hp = av1_mode_mv_merge_probs(pre_comp->hp, c->hp); + } + } +#if CONFIG_REF_MV + } +#endif // CONFIG_REF_MV +} + +#if CONFIG_EC_MULTISYMBOL && !CONFIG_EC_ADAPT +void av1_set_mv_cdfs(nmv_context *ctx) { + int i; + int j; + av1_tree_to_cdf(av1_mv_joint_tree, ctx->joints, ctx->joint_cdf); + + for (i = 0; i < 2; ++i) { + nmv_component *const comp_ctx = &ctx->comps[i]; + av1_tree_to_cdf(av1_mv_class_tree, comp_ctx->classes, comp_ctx->class_cdf); + + for (j = 0; j < CLASS0_SIZE; ++j) { + av1_tree_to_cdf(av1_mv_fp_tree, comp_ctx->class0_fp[j], + comp_ctx->class0_fp_cdf[j]); + } + av1_tree_to_cdf(av1_mv_fp_tree, comp_ctx->fp, comp_ctx->fp_cdf); + } +} +#endif + +void av1_init_mv_probs(AV1_COMMON *cm) { +#if CONFIG_REF_MV + int i; + for (i = 0; i < NMV_CONTEXTS; ++i) { + // NB: this sets CDFs too + cm->fc->nmvc[i] = default_nmv_context; + } +#else + cm->fc->nmvc = default_nmv_context; +#endif // CONFIG_REF_MV +#if CONFIG_INTRABC + cm->fc->ndvc = default_nmv_context; +#endif // CONFIG_INTRABC +#if CONFIG_GLOBAL_MOTION + av1_copy(cm->fc->global_motion_types_prob, default_global_motion_types_prob); +#endif // CONFIG_GLOBAL_MOTION +} diff --git a/third_party/aom/av1/common/entropymv.h b/third_party/aom/av1/common/entropymv.h new file mode 100644 index 0000000000..2c79d447a5 --- /dev/null +++ b/third_party/aom/av1/common/entropymv.h @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_ENTROPYMV_H_ +#define AV1_COMMON_ENTROPYMV_H_ + +#include "./aom_config.h" + +#include "aom_dsp/prob.h" + +#include "av1/common/mv.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct AV1Common; + +void av1_init_mv_probs(struct AV1Common *cm); + +void av1_adapt_mv_probs(struct AV1Common *cm, int usehp); + +#define MV_UPDATE_PROB 252 + +/* Symbols for coding which components are zero jointly */ +#define MV_JOINTS 4 +typedef enum { + MV_JOINT_ZERO = 0, /* Zero vector */ + MV_JOINT_HNZVZ = 1, /* Vert zero, hor nonzero */ + MV_JOINT_HZVNZ = 2, /* Hor zero, vert nonzero */ + MV_JOINT_HNZVNZ = 3, /* Both components nonzero */ +} MV_JOINT_TYPE; + +static INLINE int mv_joint_vertical(MV_JOINT_TYPE type) { + return type == MV_JOINT_HZVNZ || type == MV_JOINT_HNZVNZ; +} + +static INLINE int mv_joint_horizontal(MV_JOINT_TYPE type) { + return type == MV_JOINT_HNZVZ || type == MV_JOINT_HNZVNZ; +} + +/* Symbols for coding magnitude class of nonzero components */ +#define MV_CLASSES 11 +typedef enum { + MV_CLASS_0 = 0, /* (0, 2] integer pel */ + MV_CLASS_1 = 1, /* (2, 4] integer pel */ + MV_CLASS_2 = 2, /* (4, 8] integer pel */ + MV_CLASS_3 = 3, /* (8, 16] integer pel */ + MV_CLASS_4 = 4, /* (16, 32] integer pel */ + MV_CLASS_5 = 5, /* (32, 64] integer pel */ + MV_CLASS_6 = 6, /* (64, 128] integer pel */ + MV_CLASS_7 = 7, /* (128, 256] integer pel */ + MV_CLASS_8 = 8, /* (256, 512] integer pel */ + MV_CLASS_9 = 9, /* (512, 1024] integer pel */ + MV_CLASS_10 = 10, /* (1024,2048] integer pel */ +} MV_CLASS_TYPE; + +#define CLASS0_BITS 1 /* bits at integer precision for class 0 */ +#define CLASS0_SIZE (1 << CLASS0_BITS) +#define MV_OFFSET_BITS (MV_CLASSES + CLASS0_BITS - 2) +#define MV_FP_SIZE 4 + +#define MV_MAX_BITS (MV_CLASSES + CLASS0_BITS + 2) +#define MV_MAX ((1 << MV_MAX_BITS) - 1) +#define MV_VALS ((MV_MAX << 1) + 1) + +#define MV_IN_USE_BITS 14 +#define MV_UPP ((1 << MV_IN_USE_BITS) - 1) +#define MV_LOW (-(1 << MV_IN_USE_BITS)) + +extern const aom_tree_index av1_mv_joint_tree[]; +extern const aom_tree_index av1_mv_class_tree[]; +extern const aom_tree_index av1_mv_class0_tree[]; +extern const aom_tree_index av1_mv_fp_tree[]; + +typedef struct { + aom_prob sign; + aom_prob classes[MV_CLASSES - 1]; +#if CONFIG_EC_MULTISYMBOL + aom_cdf_prob class_cdf[CDF_SIZE(MV_CLASSES)]; +#endif + aom_prob class0[CLASS0_SIZE - 1]; + aom_prob bits[MV_OFFSET_BITS]; + aom_prob class0_fp[CLASS0_SIZE][MV_FP_SIZE - 1]; + aom_prob fp[MV_FP_SIZE - 1]; +#if CONFIG_EC_MULTISYMBOL + aom_cdf_prob class0_fp_cdf[CLASS0_SIZE][CDF_SIZE(MV_FP_SIZE)]; + aom_cdf_prob fp_cdf[CDF_SIZE(MV_FP_SIZE)]; +#endif + aom_prob class0_hp; + aom_prob hp; +} nmv_component; + +typedef struct { + aom_prob joints[MV_JOINTS - 1]; +#if CONFIG_EC_MULTISYMBOL + aom_cdf_prob joint_cdf[CDF_SIZE(MV_JOINTS)]; +#endif + nmv_component comps[2]; +} nmv_context; + +static INLINE MV_JOINT_TYPE av1_get_mv_joint(const MV *mv) { + if (mv->row == 0) { + return mv->col == 0 ? MV_JOINT_ZERO : MV_JOINT_HNZVZ; + } else { + return mv->col == 0 ? MV_JOINT_HZVNZ : MV_JOINT_HNZVNZ; + } +} + +MV_CLASS_TYPE av1_get_mv_class(int z, int *offset); + +typedef struct { + unsigned int sign[2]; + unsigned int classes[MV_CLASSES]; + unsigned int class0[CLASS0_SIZE]; + unsigned int bits[MV_OFFSET_BITS][2]; + unsigned int class0_fp[CLASS0_SIZE][MV_FP_SIZE]; + unsigned int fp[MV_FP_SIZE]; + unsigned int class0_hp[2]; + unsigned int hp[2]; +} nmv_component_counts; + +typedef struct { + unsigned int joints[MV_JOINTS]; + nmv_component_counts comps[2]; +} nmv_context_counts; + +void av1_inc_mv(const MV *mv, nmv_context_counts *mvctx, const int usehp); +#if CONFIG_GLOBAL_MOTION +extern const aom_tree_index + av1_global_motion_types_tree[TREE_SIZE(GLOBAL_TRANS_TYPES)]; +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_EC_MULTISYMBOL +void av1_set_mv_cdfs(nmv_context *ctx); +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_ENTROPYMV_H_ diff --git a/third_party/aom/av1/common/enums.h b/third_party/aom/av1/common/enums.h new file mode 100644 index 0000000000..054bd40be1 --- /dev/null +++ b/third_party/aom/av1/common/enums.h @@ -0,0 +1,543 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_ENUMS_H_ +#define AV1_COMMON_ENUMS_H_ + +#include "./aom_config.h" +#include "aom/aom_codec.h" +#include "aom/aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#undef MAX_SB_SIZE + +// Max superblock size +#if CONFIG_EXT_PARTITION +#define MAX_SB_SIZE_LOG2 7 +#else +#define MAX_SB_SIZE_LOG2 6 +#endif // CONFIG_EXT_PARTITION +#define MAX_SB_SIZE (1 << MAX_SB_SIZE_LOG2) +#define MAX_SB_SQUARE (MAX_SB_SIZE * MAX_SB_SIZE) + +// Min superblock size +#define MIN_SB_SIZE_LOG2 6 + +// Pixels per Mode Info (MI) unit +#if CONFIG_CB4X4 +#define MI_SIZE_LOG2 2 +#else +#define MI_SIZE_LOG2 3 +#endif +#define MI_SIZE (1 << MI_SIZE_LOG2) + +// MI-units per max superblock (MI Block - MIB) +#define MAX_MIB_SIZE_LOG2 (MAX_SB_SIZE_LOG2 - MI_SIZE_LOG2) +#define MAX_MIB_SIZE (1 << MAX_MIB_SIZE_LOG2) + +// MI-units per min superblock +#define MIN_MIB_SIZE_LOG2 (MIN_SB_SIZE_LOG2 - MI_SIZE_LOG2) + +// Mask to extract MI offset within max MIB +#define MAX_MIB_MASK (MAX_MIB_SIZE - 1) +#define MAX_MIB_MASK_2 (MAX_MIB_SIZE * 2 - 1) + +// Maximum number of tile rows and tile columns +#if CONFIG_EXT_TILE +#define TILE_NORMAL 0 +#define TILE_VR 1 + +#define MAX_TILE_ROWS 1024 +#define MAX_TILE_COLS 1024 +#else +#define MAX_TILE_ROWS 4 +#define MAX_TILE_COLS 64 +#endif // CONFIG_EXT_TILE + +#if CONFIG_VAR_TX +#define MAX_VARTX_DEPTH 2 +#endif + +// Bitstream profiles indicated by 2-3 bits in the uncompressed header. +// 00: Profile 0. 8-bit 4:2:0 only. +// 10: Profile 1. 8-bit 4:4:4, 4:2:2, and 4:4:0. +// 01: Profile 2. 10-bit and 12-bit color only, with 4:2:0 sampling. +// 110: Profile 3. 10-bit and 12-bit color only, with 4:2:2/4:4:4/4:4:0 +// sampling. +// 111: Undefined profile. +typedef enum BITSTREAM_PROFILE { + PROFILE_0, + PROFILE_1, + PROFILE_2, + PROFILE_3, + MAX_PROFILES +} BITSTREAM_PROFILE; + +// Note: Some enums use the attribute 'packed' to use smallest possible integer +// type, so that we can save memory when they are used in structs/arrays. + +typedef enum ATTRIBUTE_PACKED { +#if CONFIG_CB4X4 + BLOCK_2X2, + BLOCK_2X4, + BLOCK_4X2, +#endif + BLOCK_4X4, + BLOCK_4X8, + BLOCK_8X4, + BLOCK_8X8, + BLOCK_8X16, + BLOCK_16X8, + BLOCK_16X16, + BLOCK_16X32, + BLOCK_32X16, + BLOCK_32X32, + BLOCK_32X64, + BLOCK_64X32, + BLOCK_64X64, +#if CONFIG_EXT_PARTITION + BLOCK_64X128, + BLOCK_128X64, + BLOCK_128X128, +#endif // CONFIG_EXT_PARTITION + BLOCK_SIZES, + BLOCK_INVALID = BLOCK_SIZES, + BLOCK_LARGEST = (BLOCK_SIZES - 1) +} BLOCK_SIZE; + +typedef enum { + PARTITION_NONE, + PARTITION_HORZ, + PARTITION_VERT, + PARTITION_SPLIT, +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_HORZ_A, // HORZ split and the left partition is split again + PARTITION_HORZ_B, // HORZ split and the right partition is split again + PARTITION_VERT_A, // VERT split and the top partition is split again + PARTITION_VERT_B, // VERT split and the bottom partition is split again + EXT_PARTITION_TYPES, +#endif // CONFIG_EXT_PARTITION_TYPES + PARTITION_TYPES = PARTITION_SPLIT + 1, + PARTITION_INVALID = 255 +} PARTITION_TYPE; + +typedef char PARTITION_CONTEXT; +#define PARTITION_PLOFFSET 4 // number of probability models per block size +#define PARTITION_BLOCK_SIZES (4 + CONFIG_EXT_PARTITION) +#define PARTITION_CONTEXTS_PRIMARY (PARTITION_BLOCK_SIZES * PARTITION_PLOFFSET) +#if CONFIG_UNPOISON_PARTITION_CTX +#define PARTITION_CONTEXTS \ + (PARTITION_CONTEXTS_PRIMARY + 2 * PARTITION_BLOCK_SIZES) +#else +#define PARTITION_CONTEXTS PARTITION_CONTEXTS_PRIMARY +#endif + +// block transform size +typedef enum ATTRIBUTE_PACKED { +#if CONFIG_CB4X4 + TX_2X2, // 2x2 transform +#endif + TX_4X4, // 4x4 transform + TX_8X8, // 8x8 transform + TX_16X16, // 16x16 transform + TX_32X32, // 32x32 transform +#if CONFIG_TX64X64 + TX_64X64, // 64x64 transform +#endif // CONFIG_TX64X64 + TX_4X8, // 4x8 transform + TX_8X4, // 8x4 transform + TX_8X16, // 8x16 transform + TX_16X8, // 16x8 transform + TX_16X32, // 16x32 transform + TX_32X16, // 32x16 transform + TX_4X16, // 4x16 transform + TX_16X4, // 16x4 transform + TX_8X32, // 8x32 transform + TX_32X8, // 32x8 transform + TX_SIZES_ALL, // Includes rectangular transforms + TX_SIZES = TX_4X8, // Does NOT include rectangular transforms + TX_INVALID = 255 // Invalid transform size +} TX_SIZE; + +#define MAX_TX_DEPTH (TX_SIZES - 1 - TX_4X4) + +#define MAX_TX_SIZE_LOG2 (5 + CONFIG_TX64X64) +#define MAX_TX_SIZE (1 << MAX_TX_SIZE_LOG2) +#define MIN_TX_SIZE_LOG2 2 +#define MIN_TX_SIZE (1 << MIN_TX_SIZE_LOG2) +#define MAX_TX_SQUARE (MAX_TX_SIZE * MAX_TX_SIZE) + +// Number of maxium size transform blocks in the maximum size superblock +#define MAX_TX_BLOCKS_IN_MAX_SB_LOG2 ((MAX_SB_SIZE_LOG2 - MAX_TX_SIZE_LOG2) * 2) +#define MAX_TX_BLOCKS_IN_MAX_SB (1 << MAX_TX_BLOCKS_IN_MAX_SB_LOG2) + +#define MAX_NUM_TXB (1 << (MAX_SB_SIZE_LOG2 - MIN_TX_SIZE_LOG2)) + +// frame transform mode +typedef enum { + ONLY_4X4 = 0, // only 4x4 transform used + ALLOW_8X8 = 1, // allow block transform size up to 8x8 + ALLOW_16X16 = 2, // allow block transform size up to 16x16 + ALLOW_32X32 = 3, // allow block transform size up to 32x32 +#if CONFIG_TX64X64 + ALLOW_64X64 = 4, // allow block transform size up to 64x64 +#endif + TX_MODE_SELECT, // transform specified for each block + TX_MODES, +} TX_MODE; + +// 1D tx types +typedef enum { + DCT_1D = 0, + ADST_1D = 1, + FLIPADST_1D = 2, + IDTX_1D = 3, + TX_TYPES_1D = 4, +} TX_TYPE_1D; + +typedef enum { + DCT_DCT = 0, // DCT in both horizontal and vertical + ADST_DCT = 1, // ADST in vertical, DCT in horizontal + DCT_ADST = 2, // DCT in vertical, ADST in horizontal + ADST_ADST = 3, // ADST in both directions +#if CONFIG_EXT_TX + FLIPADST_DCT = 4, + DCT_FLIPADST = 5, + FLIPADST_FLIPADST = 6, + ADST_FLIPADST = 7, + FLIPADST_ADST = 8, + IDTX = 9, + V_DCT = 10, + H_DCT = 11, + V_ADST = 12, + H_ADST = 13, + V_FLIPADST = 14, + H_FLIPADST = 15, +#endif // CONFIG_EXT_TX + TX_TYPES, +} TX_TYPE; + +typedef enum { + TILE_LEFT_BOUNDARY = 1, + TILE_RIGHT_BOUNDARY = 2, + TILE_ABOVE_BOUNDARY = 4, + TILE_BOTTOM_BOUNDARY = 8, + FRAME_LEFT_BOUNDARY = 16, + FRAME_RIGHT_BOUNDARY = 32, + FRAME_ABOVE_BOUNDARY = 64, + FRAME_BOTTOM_BOUNDARY = 128, +} BOUNDARY_TYPE; + +#if CONFIG_EXT_TX +#if CONFIG_CB4X4 +#define EXT_TX_SIZES 5 // number of sizes that use extended transforms +#else +#define EXT_TX_SIZES 4 // number of sizes that use extended transforms +#endif // CONFIG_CB4X4 +#define EXT_TX_SETS_INTER 4 // Sets of transform selections for INTER +#define EXT_TX_SETS_INTRA 3 // Sets of transform selections for INTRA +#else +#if CONFIG_CB4X4 +#define EXT_TX_SIZES 4 // number of sizes that use extended transforms +#else +#define EXT_TX_SIZES 3 // number of sizes that use extended transforms +#endif +#endif // CONFIG_EXT_TX + +typedef enum { + AOM_LAST_FLAG = 1 << 0, +#if CONFIG_EXT_REFS + AOM_LAST2_FLAG = 1 << 1, + AOM_LAST3_FLAG = 1 << 2, + AOM_GOLD_FLAG = 1 << 3, + AOM_BWD_FLAG = 1 << 4, + AOM_ALT_FLAG = 1 << 5, + AOM_REFFRAME_ALL = (1 << 6) - 1 +#else + AOM_GOLD_FLAG = 1 << 1, + AOM_ALT_FLAG = 1 << 2, + AOM_REFFRAME_ALL = (1 << 3) - 1 +#endif // CONFIG_EXT_REFS +} AOM_REFFRAME; + +typedef enum { PLANE_TYPE_Y = 0, PLANE_TYPE_UV = 1, PLANE_TYPES } PLANE_TYPE; + +#if CONFIG_CFL +typedef enum { CFL_PRED_U = 0, CFL_PRED_V = 1, CFL_PRED_PLANES } CFL_PRED_TYPE; +#endif + +#if CONFIG_PALETTE +typedef enum { + TWO_COLORS, + THREE_COLORS, + FOUR_COLORS, + FIVE_COLORS, + SIX_COLORS, + SEVEN_COLORS, + EIGHT_COLORS, + PALETTE_SIZES +} PALETTE_SIZE; + +typedef enum { + PALETTE_COLOR_ONE, + PALETTE_COLOR_TWO, + PALETTE_COLOR_THREE, + PALETTE_COLOR_FOUR, + PALETTE_COLOR_FIVE, + PALETTE_COLOR_SIX, + PALETTE_COLOR_SEVEN, + PALETTE_COLOR_EIGHT, + PALETTE_COLORS +} PALETTE_COLOR; +#endif // CONFIG_PALETTE + +typedef enum ATTRIBUTE_PACKED { + DC_PRED, // Average of above and left pixels + V_PRED, // Vertical + H_PRED, // Horizontal + D45_PRED, // Directional 45 deg = round(arctan(1/1) * 180/pi) + D135_PRED, // Directional 135 deg = 180 - 45 + D117_PRED, // Directional 117 deg = 180 - 63 + D153_PRED, // Directional 153 deg = 180 - 27 + D207_PRED, // Directional 207 deg = 180 + 27 + D63_PRED, // Directional 63 deg = round(arctan(2/1) * 180/pi) +#if CONFIG_ALT_INTRA + SMOOTH_PRED, // Combination of horizontal and vertical interpolation +#endif // CONFIG_ALT_INTRA + TM_PRED, // True-motion + NEARESTMV, + NEARMV, + ZEROMV, + NEWMV, +#if CONFIG_EXT_INTER +#if CONFIG_COMPOUND_SINGLEREF + // Single ref compound modes + SR_NEAREST_NEARMV, + SR_NEAREST_NEWMV, + SR_NEAR_NEWMV, + SR_ZERO_NEWMV, + SR_NEW_NEWMV, +#endif // CONFIG_COMPOUND_SINGLEREF + // Compound ref compound modes + NEAREST_NEARESTMV, + NEAREST_NEARMV, + NEAR_NEARESTMV, + NEAR_NEARMV, + NEAREST_NEWMV, + NEW_NEARESTMV, + NEAR_NEWMV, + NEW_NEARMV, + ZERO_ZEROMV, + NEW_NEWMV, +#endif // CONFIG_EXT_INTER + MB_MODE_COUNT, + INTRA_MODES = TM_PRED + 1, + INTRA_INVALID = MB_MODE_COUNT // For uv_mode in inter blocks +} PREDICTION_MODE; + +typedef enum { + SIMPLE_TRANSLATION = 0, +#if CONFIG_MOTION_VAR + OBMC_CAUSAL, // 2-sided OBMC +#endif // CONFIG_MOTION_VAR +#if CONFIG_WARPED_MOTION + WARPED_CAUSAL, // 2-sided WARPED +#endif // CONFIG_WARPED_MOTION + MOTION_MODES +} MOTION_MODE; + +// TODO(urvang): Consider adding II_SMOOTH_PRED if it's helpful. + +#if CONFIG_EXT_INTER +typedef enum { + II_DC_PRED = 0, + II_V_PRED, + II_H_PRED, + II_D45_PRED, + II_D135_PRED, + II_D117_PRED, + II_D153_PRED, + II_D207_PRED, + II_D63_PRED, + II_TM_PRED, + INTERINTRA_MODES +} INTERINTRA_MODE; + +typedef enum { + COMPOUND_AVERAGE = 0, +#if CONFIG_WEDGE + COMPOUND_WEDGE, +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + COMPOUND_SEG, +#endif // CONFIG_COMPOUND_SEGMENT + COMPOUND_TYPES, +} COMPOUND_TYPE; +#endif // CONFIG_EXT_INTER + +// TODO(huisu): Consider adding FILTER_SMOOTH_PRED to "FILTER_INTRA_MODE". +#if CONFIG_FILTER_INTRA +typedef enum { + FILTER_DC_PRED, + FILTER_V_PRED, + FILTER_H_PRED, + FILTER_D45_PRED, + FILTER_D135_PRED, + FILTER_D117_PRED, + FILTER_D153_PRED, + FILTER_D207_PRED, + FILTER_D63_PRED, + FILTER_TM_PRED, + FILTER_INTRA_MODES, +} FILTER_INTRA_MODE; +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_EXT_INTRA +#define DIRECTIONAL_MODES (INTRA_MODES - 2) +#endif // CONFIG_EXT_INTRA + +#define INTER_MODES (1 + NEWMV - NEARESTMV) + +#if CONFIG_EXT_INTER +#if CONFIG_COMPOUND_SINGLEREF +#define INTER_SINGLEREF_COMP_MODES (1 + SR_NEW_NEWMV - SR_NEAREST_NEARMV) +#endif // CONFIG_COMPOUND_SINGLEREF + +#define INTER_COMPOUND_MODES (1 + NEW_NEWMV - NEAREST_NEARESTMV) +#endif // CONFIG_EXT_INTER + +#define SKIP_CONTEXTS 3 + +#if CONFIG_REF_MV +#define NMV_CONTEXTS 3 + +#define NEWMV_MODE_CONTEXTS 7 +#define ZEROMV_MODE_CONTEXTS 2 +#define REFMV_MODE_CONTEXTS 9 +#define DRL_MODE_CONTEXTS 5 + +#define ZEROMV_OFFSET 3 +#define REFMV_OFFSET 4 + +#define NEWMV_CTX_MASK ((1 << ZEROMV_OFFSET) - 1) +#define ZEROMV_CTX_MASK ((1 << (REFMV_OFFSET - ZEROMV_OFFSET)) - 1) +#define REFMV_CTX_MASK ((1 << (8 - REFMV_OFFSET)) - 1) + +#define ALL_ZERO_FLAG_OFFSET 8 +#define SKIP_NEARESTMV_OFFSET 9 +#define SKIP_NEARMV_OFFSET 10 +#define SKIP_NEARESTMV_SUB8X8_OFFSET 11 +#endif + +#define INTER_MODE_CONTEXTS 7 +#if CONFIG_DELTA_Q +#define DELTA_Q_SMALL 3 +#define DELTA_Q_PROBS (DELTA_Q_SMALL) +#define DEFAULT_DELTA_Q_RES 4 +#if CONFIG_EXT_DELTA_Q +#define DELTA_LF_SMALL 3 +#define DELTA_LF_PROBS (DELTA_LF_SMALL) +#define DEFAULT_DELTA_LF_RES 2 +#endif +#endif + +/* Segment Feature Masks */ +#define MAX_MV_REF_CANDIDATES 2 + +#if CONFIG_REF_MV +#define MAX_REF_MV_STACK_SIZE 16 +#if CONFIG_EXT_PARTITION +#define REF_CAT_LEVEL 640 +#else +#define REF_CAT_LEVEL 255 +#endif // CONFIG_EXT_PARTITION +#endif // CONFIG_REF_MV + +#define INTRA_INTER_CONTEXTS 4 +#define COMP_INTER_CONTEXTS 5 +#define REF_CONTEXTS 5 +#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF +#define COMP_INTER_MODE_CONTEXTS 4 +#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF + +#if CONFIG_VAR_TX +#define TXFM_PARTITION_CONTEXTS ((TX_SIZES - TX_8X8) * 6 - 2) +typedef uint8_t TXFM_CONTEXT; +#endif + +#define NONE_FRAME -1 +#define INTRA_FRAME 0 +#define LAST_FRAME 1 + +#if CONFIG_EXT_REFS +#define LAST2_FRAME 2 +#define LAST3_FRAME 3 +#define GOLDEN_FRAME 4 +#define BWDREF_FRAME 5 +#define ALTREF_FRAME 6 +#define LAST_REF_FRAMES (LAST3_FRAME - LAST_FRAME + 1) +#else +#define GOLDEN_FRAME 2 +#define ALTREF_FRAME 3 +#endif // CONFIG_EXT_REFS + +#define INTER_REFS_PER_FRAME (ALTREF_FRAME - LAST_FRAME + 1) +#define TOTAL_REFS_PER_FRAME (ALTREF_FRAME - INTRA_FRAME + 1) + +#define FWD_REFS (GOLDEN_FRAME - LAST_FRAME + 1) +#define FWD_RF_OFFSET(ref) (ref - LAST_FRAME) +#if CONFIG_EXT_REFS +#define BWD_REFS (ALTREF_FRAME - BWDREF_FRAME + 1) +#define BWD_RF_OFFSET(ref) (ref - BWDREF_FRAME) +#else +#define BWD_REFS 1 +#define BWD_RF_OFFSET(ref) (ref - ALTREF_FRAME) +#endif // CONFIG_EXT_REFS + +#define SINGLE_REFS (FWD_REFS + BWD_REFS) +#define COMP_REFS (FWD_REFS * BWD_REFS) + +#if CONFIG_REF_MV +#define MODE_CTX_REF_FRAMES (TOTAL_REFS_PER_FRAME + COMP_REFS) +#else +#define MODE_CTX_REF_FRAMES TOTAL_REFS_PER_FRAME +#endif + +#if CONFIG_SUPERTX +#define PARTITION_SUPERTX_CONTEXTS 2 +#define MAX_SUPERTX_BLOCK_SIZE BLOCK_32X32 +#endif // CONFIG_SUPERTX + +#if CONFIG_LOOP_RESTORATION +typedef enum { + RESTORE_NONE = 0, + RESTORE_WIENER = 1, + RESTORE_SGRPROJ = 2, + RESTORE_SWITCHABLE, + RESTORE_SWITCHABLE_TYPES = RESTORE_SWITCHABLE, + RESTORE_TYPES, +} RestorationType; +#endif // CONFIG_LOOP_RESTORATION + +#if CONFIG_FRAME_SUPERRES +#define SUPERRES_SCALE_DENOMINATOR 16 +#define SUPERRES_SCALE_BITS 3 +#define SUPERRES_SCALE_NUMERATOR_MIN 8 +#endif // CONFIG_FRAME_SUPERRES + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_ENUMS_H_ diff --git a/third_party/aom/av1/common/filter.c b/third_party/aom/av1/common/filter.c new file mode 100644 index 0000000000..9f0c58866c --- /dev/null +++ b/third_party/aom/av1/common/filter.c @@ -0,0 +1,360 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/filter.h" + +DECLARE_ALIGNED(256, static const InterpKernel, + bilinear_filters[SUBPEL_SHIFTS]) = { + { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, 0, 0, 120, 8, 0, 0, 0 }, + { 0, 0, 0, 112, 16, 0, 0, 0 }, { 0, 0, 0, 104, 24, 0, 0, 0 }, + { 0, 0, 0, 96, 32, 0, 0, 0 }, { 0, 0, 0, 88, 40, 0, 0, 0 }, + { 0, 0, 0, 80, 48, 0, 0, 0 }, { 0, 0, 0, 72, 56, 0, 0, 0 }, + { 0, 0, 0, 64, 64, 0, 0, 0 }, { 0, 0, 0, 56, 72, 0, 0, 0 }, + { 0, 0, 0, 48, 80, 0, 0, 0 }, { 0, 0, 0, 40, 88, 0, 0, 0 }, + { 0, 0, 0, 32, 96, 0, 0, 0 }, { 0, 0, 0, 24, 104, 0, 0, 0 }, + { 0, 0, 0, 16, 112, 0, 0, 0 }, { 0, 0, 0, 8, 120, 0, 0, 0 } +}; + +#if USE_TEMPORALFILTER_12TAP +DECLARE_ALIGNED(16, static const int16_t, + sub_pel_filters_temporalfilter_12[SUBPEL_SHIFTS][12]) = { + // intfilt 0.8 + { 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0 }, + { 0, 1, -1, 3, -7, 127, 8, -4, 2, -1, 0, 0 }, + { 0, 1, -3, 5, -12, 124, 18, -8, 4, -2, 1, 0 }, + { -1, 2, -4, 8, -17, 120, 28, -11, 6, -3, 1, -1 }, + { -1, 2, -4, 10, -21, 114, 38, -15, 8, -4, 2, -1 }, + { -1, 3, -5, 11, -23, 107, 49, -18, 9, -5, 2, -1 }, + { -1, 3, -6, 12, -25, 99, 60, -21, 11, -6, 3, -1 }, + { -1, 3, -6, 12, -25, 90, 70, -23, 12, -6, 3, -1 }, + { -1, 3, -6, 12, -24, 80, 80, -24, 12, -6, 3, -1 }, + { -1, 3, -6, 12, -23, 70, 90, -25, 12, -6, 3, -1 }, + { -1, 3, -6, 11, -21, 60, 99, -25, 12, -6, 3, -1 }, + { -1, 2, -5, 9, -18, 49, 107, -23, 11, -5, 3, -1 }, + { -1, 2, -4, 8, -15, 38, 114, -21, 10, -4, 2, -1 }, + { -1, 1, -3, 6, -11, 28, 120, -17, 8, -4, 2, -1 }, + { 0, 1, -2, 4, -8, 18, 124, -12, 5, -3, 1, 0 }, + { 0, 0, -1, 2, -4, 8, 127, -7, 3, -1, 1, 0 }, +}; +#endif // USE_TEMPORALFILTER_12TAP + +#if CONFIG_DUAL_FILTER +DECLARE_ALIGNED(256, static const InterpKernel, + sub_pel_filters_8[SUBPEL_SHIFTS]) = { + // intfilt 0.575 + { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, 1, -5, 126, 8, -3, 1, 0 }, + { -1, 3, -10, 123, 18, -6, 2, -1 }, { -1, 4, -14, 118, 27, -9, 3, 0 }, + { -1, 5, -16, 112, 37, -12, 4, -1 }, { -1, 5, -18, 105, 48, -14, 4, -1 }, + { -1, 6, -19, 97, 58, -17, 5, -1 }, { -1, 6, -20, 88, 68, -18, 6, -1 }, + { -1, 6, -19, 78, 78, -19, 6, -1 }, { -1, 6, -18, 68, 88, -20, 6, -1 }, + { -1, 5, -17, 58, 97, -19, 6, -1 }, { -1, 4, -14, 48, 105, -18, 5, -1 }, + { -1, 4, -12, 37, 112, -16, 5, -1 }, { 0, 3, -9, 27, 118, -14, 4, -1 }, + { -1, 2, -6, 18, 123, -10, 3, -1 }, { 0, 1, -3, 8, 126, -5, 1, 0 }, +}; + +DECLARE_ALIGNED(256, static const InterpKernel, + sub_pel_filters_regular_uv[SUBPEL_SHIFTS]) = { + // intfilt 0.575 + { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, 1, -5, 126, 8, -3, 1, 0 }, + { -1, 3, -10, 123, 18, -6, 2, -1 }, { -1, 4, -14, 118, 27, -9, 3, 0 }, + { -1, 5, -16, 112, 37, -12, 4, -1 }, { -1, 5, -18, 105, 48, -14, 4, -1 }, + { -1, 6, -19, 97, 58, -17, 5, -1 }, { -1, 6, -20, 88, 68, -18, 6, -1 }, + { -1, 6, -19, 78, 78, -19, 6, -1 }, { -1, 6, -18, 68, 88, -20, 6, -1 }, + { -1, 5, -17, 58, 97, -19, 6, -1 }, { -1, 4, -14, 48, 105, -18, 5, -1 }, + { -1, 4, -12, 37, 112, -16, 5, -1 }, { 0, 3, -9, 27, 118, -14, 4, -1 }, + { -1, 2, -6, 18, 123, -10, 3, -1 }, { 0, 1, -3, 8, 126, -5, 1, 0 }, +}; + +#if USE_12TAP_FILTER +DECLARE_ALIGNED(256, static const InterpKernel, + sub_pel_filters_8sharp[SUBPEL_SHIFTS]) = { + // intfilt 0.8 + { 0, 0, 0, 128, 0, 0, 0, 0 }, { -1, 2, -6, 127, 9, -4, 2, -1 }, + { -2, 5, -12, 124, 18, -7, 4, -2 }, { -2, 7, -16, 119, 28, -11, 5, -2 }, + { -3, 8, -19, 114, 38, -14, 7, -3 }, { -3, 9, -22, 107, 49, -17, 8, -3 }, + { -4, 10, -23, 99, 60, -20, 10, -4 }, { -4, 11, -23, 90, 70, -22, 10, -4 }, + { -4, 11, -23, 80, 80, -23, 11, -4 }, { -4, 10, -22, 70, 90, -23, 11, -4 }, + { -4, 10, -20, 60, 99, -23, 10, -4 }, { -3, 8, -17, 49, 107, -22, 9, -3 }, + { -3, 7, -14, 38, 114, -19, 8, -3 }, { -2, 5, -11, 28, 119, -16, 7, -2 }, + { -2, 4, -7, 18, 124, -12, 5, -2 }, { -1, 2, -4, 9, 127, -6, 2, -1 }, +}; + +DECLARE_ALIGNED(256, static const int16_t, + sub_pel_filters_10sharp[SUBPEL_SHIFTS][12]) = { + // intfilt 0.85 + { 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0 }, + { 0, 1, -2, 3, -7, 127, 8, -4, 2, -1, 1, 0 }, + { 0, 1, -3, 6, -13, 124, 18, -8, 4, -2, 1, 0 }, + { 0, 2, -4, 8, -18, 120, 28, -12, 6, -4, 2, 0 }, + { 0, 2, -5, 10, -21, 114, 38, -15, 8, -5, 2, 0 }, + { 0, 3, -6, 11, -24, 107, 49, -19, 10, -6, 3, 0 }, + { 0, 3, -7, 12, -25, 99, 59, -21, 11, -6, 3, 0 }, + { 0, 3, -7, 12, -25, 90, 70, -23, 12, -7, 3, 0 }, + { 0, 3, -7, 12, -25, 81, 81, -25, 12, -7, 3, 0 }, + { 0, 3, -7, 12, -23, 70, 90, -25, 12, -7, 3, 0 }, + { 0, 3, -6, 11, -21, 59, 99, -25, 12, -7, 3, 0 }, + { 0, 3, -6, 10, -19, 49, 107, -24, 11, -6, 3, 0 }, + { 0, 2, -5, 8, -15, 38, 114, -21, 10, -5, 2, 0 }, + { 0, 2, -4, 6, -12, 28, 120, -18, 8, -4, 2, 0 }, + { 0, 1, -2, 4, -8, 18, 124, -13, 6, -3, 1, 0 }, + { 0, 1, -1, 2, -4, 8, 127, -7, 3, -2, 1, 0 }, +}; +#else +DECLARE_ALIGNED(256, static const InterpKernel, + sub_pel_filters_8sharp[SUBPEL_SHIFTS]) = { +#if CONFIG_FILTER_7BIT + { 0, 0, 0, 128, 0, 0, 0, 0 }, { -2, 2, -6, 126, 8, -2, 2, 0 }, + { -2, 6, -12, 124, 16, -6, 4, -2 }, { -2, 8, -18, 120, 26, -10, 6, -2 }, + { -4, 10, -22, 116, 38, -14, 6, -2 }, { -4, 10, -22, 108, 48, -18, 8, -2 }, + { -4, 10, -24, 100, 60, -20, 8, -2 }, { -4, 10, -24, 90, 70, -22, 10, -2 }, + { -4, 12, -24, 80, 80, -24, 12, -4 }, { -2, 10, -22, 70, 90, -24, 10, -4 }, + { -2, 8, -20, 60, 100, -24, 10, -4 }, { -2, 8, -18, 48, 108, -22, 10, -4 }, + { -2, 6, -14, 38, 116, -22, 10, -4 }, { -2, 6, -10, 26, 120, -18, 8, -2 }, + { -2, 4, -6, 16, 124, -12, 6, -2 }, { 0, 2, -2, 8, 126, -6, 2, -2 } +#else + { 0, 0, 0, 128, 0, 0, 0, 0 }, { -1, 3, -7, 127, 8, -3, 1, 0 }, + { -2, 5, -13, 125, 17, -6, 3, -1 }, { -3, 7, -17, 121, 27, -10, 5, -2 }, + { -4, 9, -20, 115, 37, -13, 6, -2 }, { -4, 10, -23, 108, 48, -16, 8, -3 }, + { -4, 10, -24, 100, 59, -19, 9, -3 }, { -4, 11, -24, 90, 70, -21, 10, -4 }, + { -4, 11, -23, 80, 80, -23, 11, -4 }, { -4, 10, -21, 70, 90, -24, 11, -4 }, + { -3, 9, -19, 59, 100, -24, 10, -4 }, { -3, 8, -16, 48, 108, -23, 10, -4 }, + { -2, 6, -13, 37, 115, -20, 9, -4 }, { -2, 5, -10, 27, 121, -17, 7, -3 }, + { -1, 3, -6, 17, 125, -13, 5, -2 }, { 0, 1, -3, 8, 127, -7, 3, -1 } +#endif +}; +#endif + +DECLARE_ALIGNED(256, static const InterpKernel, + sub_pel_filters_8smooth2[SUBPEL_SHIFTS]) = { + // freqmultiplier = 0.2 + { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, 9, 30, 44, 32, 11, 2, 0 }, + { 0, 8, 28, 44, 34, 12, 2, 0 }, { 0, 7, 27, 44, 35, 13, 2, 0 }, + { 0, 6, 26, 43, 37, 14, 2, 0 }, { 0, 5, 24, 43, 38, 16, 2, 0 }, + { 0, 5, 23, 42, 38, 17, 3, 0 }, { 0, 4, 21, 41, 40, 19, 3, 0 }, + { 0, 4, 20, 40, 40, 20, 4, 0 }, { 0, 3, 19, 40, 41, 21, 4, 0 }, + { 0, 3, 17, 38, 42, 23, 5, 0 }, { 0, 2, 16, 38, 43, 24, 5, 0 }, + { 0, 2, 14, 37, 43, 26, 6, 0 }, { 0, 2, 13, 35, 44, 27, 7, 0 }, + { 0, 2, 12, 34, 44, 28, 8, 0 }, { 0, 2, 11, 32, 44, 30, 9, 0 }, +}; + +DECLARE_ALIGNED(256, static const InterpKernel, + sub_pel_filters_smooth2_uv[SUBPEL_SHIFTS]) = { + // freqmultiplier = 0.2 + { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, 9, 30, 44, 32, 11, 2, 0 }, + { 0, 8, 28, 44, 34, 12, 2, 0 }, { 0, 7, 27, 44, 35, 13, 2, 0 }, + { 0, 6, 26, 43, 37, 14, 2, 0 }, { 0, 5, 24, 43, 38, 16, 2, 0 }, + { 0, 5, 23, 42, 38, 17, 3, 0 }, { 0, 4, 21, 41, 40, 19, 3, 0 }, + { 0, 4, 20, 40, 40, 20, 4, 0 }, { 0, 3, 19, 40, 41, 21, 4, 0 }, + { 0, 3, 17, 38, 42, 23, 5, 0 }, { 0, 2, 16, 38, 43, 24, 5, 0 }, + { 0, 2, 14, 37, 43, 26, 6, 0 }, { 0, 2, 13, 35, 44, 27, 7, 0 }, + { 0, 2, 12, 34, 44, 28, 8, 0 }, { 0, 2, 11, 32, 44, 30, 9, 0 }, +}; + +DECLARE_ALIGNED(256, static const InterpKernel, + sub_pel_filters_8smooth[SUBPEL_SHIFTS]) = { + // freqmultiplier = 0.8 + { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, -5, 13, 102, 24, -7, 1, 0 }, + { 0, -4, 8, 100, 31, -8, 1, 0 }, { 0, -3, 4, 97, 37, -8, 1, 0 }, + { 0, -2, 0, 94, 44, -9, 1, 0 }, { 0, -2, -3, 90, 51, -9, 1, 0 }, + { 0, -1, -5, 84, 59, -9, 0, 0 }, { 0, 0, -7, 79, 65, -9, 0, 0 }, + { 0, 0, -8, 72, 72, -8, 0, 0 }, { 0, 0, -9, 65, 79, -7, 0, 0 }, + { 0, 0, -9, 59, 84, -5, -1, 0 }, { 0, 1, -9, 51, 90, -3, -2, 0 }, + { 0, 1, -9, 44, 94, 0, -2, 0 }, { 0, 1, -8, 37, 97, 4, -3, 0 }, + { 0, 1, -8, 31, 100, 8, -4, 0 }, { 0, 1, -7, 24, 102, 13, -5, 0 }, +}; + +DECLARE_ALIGNED(256, static const InterpKernel, + sub_pel_filters_smooth_uv[SUBPEL_SHIFTS]) = { + // freqmultiplier = 0.8 + { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, -5, 13, 102, 24, -7, 1, 0 }, + { 0, -4, 8, 100, 31, -8, 1, 0 }, { 0, -3, 4, 97, 37, -8, 1, 0 }, + { 0, -2, 0, 94, 44, -9, 1, 0 }, { 0, -2, -3, 90, 51, -9, 1, 0 }, + { 0, -1, -5, 84, 59, -9, 0, 0 }, { 0, 0, -7, 79, 65, -9, 0, 0 }, + { 0, 0, -8, 72, 72, -8, 0, 0 }, { 0, 0, -9, 65, 79, -7, 0, 0 }, + { 0, 0, -9, 59, 84, -5, -1, 0 }, { 0, 1, -9, 51, 90, -3, -2, 0 }, + { 0, 1, -9, 44, 94, 0, -2, 0 }, { 0, 1, -8, 37, 97, 4, -3, 0 }, + { 0, 1, -8, 31, 100, 8, -4, 0 }, { 0, 1, -7, 24, 102, 13, -5, 0 }, +}; +#else // CONFIG_DUAL_FILTER + +DECLARE_ALIGNED(256, static const InterpKernel, + sub_pel_filters_8[SUBPEL_SHIFTS]) = { +#if CONFIG_FILTER_7BIT + { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, 2, -6, 126, 8, -2, 0, 0 }, + { 0, 2, -10, 122, 18, -4, 0, 0 }, { 0, 2, -12, 116, 28, -8, 2, 0 }, + { 0, 2, -14, 110, 38, -10, 2, 0 }, { 0, 2, -14, 102, 48, -12, 2, 0 }, + { 0, 2, -16, 94, 58, -12, 2, 0 }, { 0, 2, -14, 84, 66, -12, 2, 0 }, + { 0, 2, -14, 76, 76, -14, 2, 0 }, { 0, 2, -12, 66, 84, -14, 2, 0 }, + { 0, 2, -12, 58, 94, -16, 2, 0 }, { 0, 2, -12, 48, 102, -14, 2, 0 }, + { 0, 2, -10, 38, 110, -14, 2, 0 }, { 0, 2, -8, 28, 116, -12, 2, 0 }, + { 0, 0, -4, 18, 122, -10, 2, 0 }, { 0, 0, -2, 8, 126, -6, 2, 0 } +#else + { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, 1, -5, 126, 8, -3, 1, 0 }, + { -1, 3, -10, 122, 18, -6, 2, 0 }, { -1, 4, -13, 118, 27, -9, 3, -1 }, + { -1, 4, -16, 112, 37, -11, 4, -1 }, { -1, 5, -18, 105, 48, -14, 4, -1 }, + { -1, 5, -19, 97, 58, -16, 5, -1 }, { -1, 6, -19, 88, 68, -18, 5, -1 }, + { -1, 6, -19, 78, 78, -19, 6, -1 }, { -1, 5, -18, 68, 88, -19, 6, -1 }, + { -1, 5, -16, 58, 97, -19, 5, -1 }, { -1, 4, -14, 48, 105, -18, 5, -1 }, + { -1, 4, -11, 37, 112, -16, 4, -1 }, { -1, 3, -9, 27, 118, -13, 4, -1 }, + { 0, 2, -6, 18, 122, -10, 3, -1 }, { 0, 1, -3, 8, 126, -5, 1, 0 } +#endif +}; + +DECLARE_ALIGNED(256, static const InterpKernel, + sub_pel_filters_8sharp[SUBPEL_SHIFTS]) = { +#if CONFIG_FILTER_7BIT + { 0, 0, 0, 128, 0, 0, 0, 0 }, { -2, 2, -6, 126, 8, -2, 2, 0 }, + { -2, 6, -12, 124, 16, -6, 4, -2 }, { -2, 8, -18, 120, 26, -10, 6, -2 }, + { -4, 10, -22, 116, 38, -14, 6, -2 }, { -4, 10, -22, 108, 48, -18, 8, -2 }, + { -4, 10, -24, 100, 60, -20, 8, -2 }, { -4, 10, -24, 90, 70, -22, 10, -2 }, + { -4, 12, -24, 80, 80, -24, 12, -4 }, { -2, 10, -22, 70, 90, -24, 10, -4 }, + { -2, 8, -20, 60, 100, -24, 10, -4 }, { -2, 8, -18, 48, 108, -22, 10, -4 }, + { -2, 6, -14, 38, 116, -22, 10, -4 }, { -2, 6, -10, 26, 120, -18, 8, -2 }, + { -2, 4, -6, 16, 124, -12, 6, -2 }, { 0, 2, -2, 8, 126, -6, 2, -2 } +#else + { 0, 0, 0, 128, 0, 0, 0, 0 }, { -1, 3, -7, 127, 8, -3, 1, 0 }, + { -2, 5, -13, 125, 17, -6, 3, -1 }, { -3, 7, -17, 121, 27, -10, 5, -2 }, + { -4, 9, -20, 115, 37, -13, 6, -2 }, { -4, 10, -23, 108, 48, -16, 8, -3 }, + { -4, 10, -24, 100, 59, -19, 9, -3 }, { -4, 11, -24, 90, 70, -21, 10, -4 }, + { -4, 11, -23, 80, 80, -23, 11, -4 }, { -4, 10, -21, 70, 90, -24, 11, -4 }, + { -3, 9, -19, 59, 100, -24, 10, -4 }, { -3, 8, -16, 48, 108, -23, 10, -4 }, + { -2, 6, -13, 37, 115, -20, 9, -4 }, { -2, 5, -10, 27, 121, -17, 7, -3 }, + { -1, 3, -6, 17, 125, -13, 5, -2 }, { 0, 1, -3, 8, 127, -7, 3, -1 } +#endif +}; + +DECLARE_ALIGNED(256, static const InterpKernel, + sub_pel_filters_8smooth[SUBPEL_SHIFTS]) = { +#if CONFIG_FILTER_7BIT + { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, 2, 28, 62, 34, 2, 0, 0 }, + { 0, 0, 26, 62, 36, 4, 0, 0 }, { 0, 0, 22, 62, 40, 4, 0, 0 }, + { 0, 0, 20, 60, 42, 6, 0, 0 }, { 0, 0, 18, 58, 44, 8, 0, 0 }, + { 0, 0, 16, 56, 46, 10, 0, 0 }, { 0, -2, 16, 54, 48, 12, 0, 0 }, + { 0, -2, 14, 52, 52, 14, -2, 0 }, { 0, 0, 12, 48, 54, 16, -2, 0 }, + { 0, 0, 10, 46, 56, 16, 0, 0 }, { 0, 0, 8, 44, 58, 18, 0, 0 }, + { 0, 0, 6, 42, 60, 20, 0, 0 }, { 0, 0, 4, 40, 62, 22, 0, 0 }, + { 0, 0, 4, 36, 62, 26, 0, 0 }, { 0, 0, 2, 34, 62, 28, 2, 0 } +#else + { 0, 0, 0, 128, 0, 0, 0, 0 }, { -3, -1, 32, 64, 38, 1, -3, 0 }, + { -2, -2, 29, 63, 41, 2, -3, 0 }, { -2, -2, 26, 63, 43, 4, -4, 0 }, + { -2, -3, 24, 62, 46, 5, -4, 0 }, { -2, -3, 21, 60, 49, 7, -4, 0 }, + { -1, -4, 18, 59, 51, 9, -4, 0 }, { -1, -4, 16, 57, 53, 12, -4, -1 }, + { -1, -4, 14, 55, 55, 14, -4, -1 }, { -1, -4, 12, 53, 57, 16, -4, -1 }, + { 0, -4, 9, 51, 59, 18, -4, -1 }, { 0, -4, 7, 49, 60, 21, -3, -2 }, + { 0, -4, 5, 46, 62, 24, -3, -2 }, { 0, -4, 4, 43, 63, 26, -2, -2 }, + { 0, -3, 2, 41, 63, 29, -2, -2 }, { 0, -3, 1, 38, 64, 32, -1, -3 } +#endif +}; +#endif // CONFIG_DUAL_FILTER + +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP +const InterpKernel *av1_intra_filter_kernels[INTRA_FILTERS] = { + bilinear_filters, // INTRA_FILTER_LINEAR + sub_pel_filters_8, // INTRA_FILTER_8TAP + sub_pel_filters_8sharp, // INTRA_FILTER_8TAP_SHARP + sub_pel_filters_8smooth, // INTRA_FILTER_8TAP_SMOOTH +}; +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA + +#if CONFIG_DUAL_FILTER +static const InterpFilterParams + av1_interp_filter_params_list[SWITCHABLE_FILTERS + EXTRA_FILTERS] = { + { (const int16_t *)sub_pel_filters_8, SUBPEL_TAPS, SUBPEL_SHIFTS, + EIGHTTAP_REGULAR }, + { (const int16_t *)sub_pel_filters_8smooth, SUBPEL_TAPS, SUBPEL_SHIFTS, + EIGHTTAP_SMOOTH }, +#if USE_12TAP_FILTER + { (const int16_t *)sub_pel_filters_10sharp, 12, SUBPEL_SHIFTS, + MULTITAP_SHARP }, +#else + { (const int16_t *)sub_pel_filters_8sharp, SUBPEL_TAPS, SUBPEL_SHIFTS, + EIGHTTAP_SHARP }, +#endif + { (const int16_t *)sub_pel_filters_8smooth2, SUBPEL_TAPS, SUBPEL_SHIFTS, + EIGHTTAP_SMOOTH2 }, + { (const int16_t *)bilinear_filters, SUBPEL_TAPS, SUBPEL_SHIFTS, + BILINEAR }, + { (const int16_t *)sub_pel_filters_8sharp, SUBPEL_TAPS, SUBPEL_SHIFTS, + EIGHTTAP_SHARP }, + { (const int16_t *)sub_pel_filters_regular_uv, SUBPEL_TAPS, SUBPEL_SHIFTS, + FILTER_REGULAR_UV }, + { (const int16_t *)sub_pel_filters_smooth_uv, SUBPEL_TAPS, SUBPEL_SHIFTS, + FILTER_SMOOTH_UV }, + { (const int16_t *)sub_pel_filters_8sharp, SUBPEL_TAPS, SUBPEL_SHIFTS, + FILTER_SHARP_UV }, + { (const int16_t *)sub_pel_filters_smooth2_uv, SUBPEL_TAPS, SUBPEL_SHIFTS, + FILTER_SMOOTH2_UV }, + }; +#else +static const InterpFilterParams + av1_interp_filter_params_list[SWITCHABLE_FILTERS + 1] = { + { (const int16_t *)sub_pel_filters_8, SUBPEL_TAPS, SUBPEL_SHIFTS, + EIGHTTAP_REGULAR }, + { (const int16_t *)sub_pel_filters_8smooth, SUBPEL_TAPS, SUBPEL_SHIFTS, + EIGHTTAP_SMOOTH }, + { (const int16_t *)sub_pel_filters_8sharp, SUBPEL_TAPS, SUBPEL_SHIFTS, + MULTITAP_SHARP }, + { (const int16_t *)bilinear_filters, SUBPEL_TAPS, SUBPEL_SHIFTS, + BILINEAR } + }; +#endif // CONFIG_DUAL_FILTER + +#if USE_TEMPORALFILTER_12TAP +static const InterpFilterParams av1_interp_temporalfilter_12tap = { + (const int16_t *)sub_pel_filters_temporalfilter_12, 12, SUBPEL_SHIFTS, + TEMPORALFILTER_12TAP +}; +#endif // USE_TEMPORALFILTER_12TAP + +InterpFilterParams av1_get_interp_filter_params( + const InterpFilter interp_filter) { +#if USE_TEMPORALFILTER_12TAP + if (interp_filter == TEMPORALFILTER_12TAP) + return av1_interp_temporalfilter_12tap; +#endif // USE_TEMPORALFILTER_12TAP + return av1_interp_filter_params_list[interp_filter]; +} + +const int16_t *av1_get_interp_filter_kernel(const InterpFilter interp_filter) { +#if USE_TEMPORALFILTER_12TAP + if (interp_filter == TEMPORALFILTER_12TAP) + return av1_interp_temporalfilter_12tap.filter_ptr; +#endif // USE_TEMPORALFILTER_12TAP + return (const int16_t *)av1_interp_filter_params_list[interp_filter] + .filter_ptr; +} + +#if CONFIG_DUAL_FILTER +InterpFilter av1_get_plane_interp_filter(InterpFilter interp_filter, + int plane) { +#if USE_TEMPORALFILTER_12TAP + assert(interp_filter <= EIGHTTAP_SHARP || + interp_filter == TEMPORALFILTER_12TAP); +#else + assert(interp_filter <= EIGHTTAP_SHARP); +#endif + if (plane == 0) { + return interp_filter; + } else { + switch (interp_filter) { + case EIGHTTAP_REGULAR: return FILTER_REGULAR_UV; + case EIGHTTAP_SMOOTH: return FILTER_SMOOTH_UV; + case MULTITAP_SHARP: return FILTER_SHARP_UV; + case EIGHTTAP_SMOOTH2: return FILTER_SMOOTH2_UV; + default: return interp_filter; + } + } +} +#endif diff --git a/third_party/aom/av1/common/filter.h b/third_party/aom/av1/common/filter.h new file mode 100644 index 0000000000..693a46902d --- /dev/null +++ b/third_party/aom/av1/common/filter.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_FILTER_H_ +#define AV1_COMMON_FILTER_H_ + +#include "./aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/aom_filter.h" +#include "aom_ports/mem.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define USE_TEMPORALFILTER_12TAP 1 +#define MAX_FILTER_TAP 12 + +#define USE_12TAP_FILTER 0 + +typedef enum { + EIGHTTAP_REGULAR, + EIGHTTAP_SMOOTH, + MULTITAP_SHARP, +#if CONFIG_DUAL_FILTER + EIGHTTAP_SMOOTH2, +#endif // CONFIG_DUAL_FILTER + BILINEAR, +#if CONFIG_DUAL_FILTER + EIGHTTAP_SHARP, + FILTER_REGULAR_UV, + FILTER_SMOOTH_UV, + FILTER_SHARP_UV, + FILTER_SMOOTH2_UV, +#endif // CONFIG_DUAL_FILTER + INTERP_FILTERS_ALL, + SWITCHABLE_FILTERS = BILINEAR, + SWITCHABLE = SWITCHABLE_FILTERS + 1, /* the last switchable one */ + EXTRA_FILTERS = INTERP_FILTERS_ALL - SWITCHABLE_FILTERS, +#if USE_TEMPORALFILTER_12TAP + TEMPORALFILTER_12TAP = SWITCHABLE_FILTERS + EXTRA_FILTERS, +#endif +} InterpFilter; + +#if CONFIG_DUAL_FILTER +#define MAX_SUBPEL_TAPS 12 +#define LOG_SWITCHABLE_FILTERS \ + 3 /* (1 << LOG_SWITCHABLE_FILTERS) > SWITCHABLE_FILTERS */ +#define SWITCHABLE_FILTER_CONTEXTS ((SWITCHABLE_FILTERS + 1) * 4) +#define INTER_FILTER_COMP_OFFSET (SWITCHABLE_FILTERS + 1) +#define INTER_FILTER_DIR_OFFSET ((SWITCHABLE_FILTERS + 1) * 2) +#else // CONFIG_DUAL_FILTER +#define LOG_SWITCHABLE_FILTERS \ + 2 /* (1 << LOG_SWITCHABLE_FILTERS) > SWITCHABLE_FILTERS */ +#define SWITCHABLE_FILTER_CONTEXTS (SWITCHABLE_FILTERS + 1) +#endif // CONFIG_DUAL_FILTER + +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP +typedef enum { + INTRA_FILTER_LINEAR, + INTRA_FILTER_8TAP, + INTRA_FILTER_8TAP_SHARP, + INTRA_FILTER_8TAP_SMOOTH, + INTRA_FILTERS, +} INTRA_FILTER; + +extern const InterpKernel *av1_intra_filter_kernels[INTRA_FILTERS]; +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA + +typedef struct InterpFilterParams { + const int16_t *filter_ptr; + uint16_t taps; + uint16_t subpel_shifts; + InterpFilter interp_filter; +} InterpFilterParams; + +InterpFilterParams av1_get_interp_filter_params( + const InterpFilter interp_filter); + +const int16_t *av1_get_interp_filter_kernel(const InterpFilter interp_filter); + +static INLINE const int16_t *av1_get_interp_filter_subpel_kernel( + const InterpFilterParams filter_params, const int subpel) { + return filter_params.filter_ptr + filter_params.taps * subpel; +} + +static INLINE int av1_is_interpolating_filter( + const InterpFilter interp_filter) { + const InterpFilterParams ip = av1_get_interp_filter_params(interp_filter); + return (ip.filter_ptr[ip.taps / 2 - 1] == 128); +} + +#if CONFIG_DUAL_FILTER +InterpFilter av1_get_plane_interp_filter(InterpFilter interp_filter, int plane); +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_FILTER_H_ diff --git a/third_party/aom/av1/common/frame_buffers.c b/third_party/aom/av1/common/frame_buffers.c new file mode 100644 index 0000000000..0b6b78e3d4 --- /dev/null +++ b/third_party/aom/av1/common/frame_buffers.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/frame_buffers.h" +#include "aom_mem/aom_mem.h" + +int av1_alloc_internal_frame_buffers(InternalFrameBufferList *list) { + assert(list != NULL); + av1_free_internal_frame_buffers(list); + + list->num_internal_frame_buffers = + AOM_MAXIMUM_REF_BUFFERS + AOM_MAXIMUM_WORK_BUFFERS; + list->int_fb = (InternalFrameBuffer *)aom_calloc( + list->num_internal_frame_buffers, sizeof(*list->int_fb)); + return (list->int_fb == NULL); +} + +void av1_free_internal_frame_buffers(InternalFrameBufferList *list) { + int i; + + assert(list != NULL); + + for (i = 0; i < list->num_internal_frame_buffers; ++i) { + aom_free(list->int_fb[i].data); + list->int_fb[i].data = NULL; + } + aom_free(list->int_fb); + list->int_fb = NULL; +} + +int av1_get_frame_buffer(void *cb_priv, size_t min_size, + aom_codec_frame_buffer_t *fb) { + int i; + InternalFrameBufferList *const int_fb_list = + (InternalFrameBufferList *)cb_priv; + if (int_fb_list == NULL) return -1; + + // Find a free frame buffer. + for (i = 0; i < int_fb_list->num_internal_frame_buffers; ++i) { + if (!int_fb_list->int_fb[i].in_use) break; + } + + if (i == int_fb_list->num_internal_frame_buffers) return -1; + + if (int_fb_list->int_fb[i].size < min_size) { + aom_free(int_fb_list->int_fb[i].data); + // The data must be zeroed to fix a valgrind error from the C loop filter + // due to access uninitialized memory in frame border. It could be + // skipped if border were totally removed. + int_fb_list->int_fb[i].data = (uint8_t *)aom_calloc(1, min_size); + if (!int_fb_list->int_fb[i].data) return -1; + int_fb_list->int_fb[i].size = min_size; + } + + fb->data = int_fb_list->int_fb[i].data; + fb->size = int_fb_list->int_fb[i].size; + int_fb_list->int_fb[i].in_use = 1; + + // Set the frame buffer's private data to point at the internal frame buffer. + fb->priv = &int_fb_list->int_fb[i]; + return 0; +} + +int av1_release_frame_buffer(void *cb_priv, aom_codec_frame_buffer_t *fb) { + InternalFrameBuffer *const int_fb = (InternalFrameBuffer *)fb->priv; + (void)cb_priv; + if (int_fb) int_fb->in_use = 0; + return 0; +} diff --git a/third_party/aom/av1/common/frame_buffers.h b/third_party/aom/av1/common/frame_buffers.h new file mode 100644 index 0000000000..e7341cfdd3 --- /dev/null +++ b/third_party/aom/av1/common/frame_buffers.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_FRAME_BUFFERS_H_ +#define AV1_COMMON_FRAME_BUFFERS_H_ + +#include "aom/aom_frame_buffer.h" +#include "aom/aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct InternalFrameBuffer { + uint8_t *data; + size_t size; + int in_use; +} InternalFrameBuffer; + +typedef struct InternalFrameBufferList { + int num_internal_frame_buffers; + InternalFrameBuffer *int_fb; +} InternalFrameBufferList; + +// Initializes |list|. Returns 0 on success. +int av1_alloc_internal_frame_buffers(InternalFrameBufferList *list); + +// Free any data allocated to the frame buffers. +void av1_free_internal_frame_buffers(InternalFrameBufferList *list); + +// Callback used by libaom to request an external frame buffer. |cb_priv| +// Callback private data, which points to an InternalFrameBufferList. +// |min_size| is the minimum size in bytes needed to decode the next frame. +// |fb| pointer to the frame buffer. +int av1_get_frame_buffer(void *cb_priv, size_t min_size, + aom_codec_frame_buffer_t *fb); + +// Callback used by libaom when there are no references to the frame buffer. +// |cb_priv| is not used. |fb| pointer to the frame buffer. +int av1_release_frame_buffer(void *cb_priv, aom_codec_frame_buffer_t *fb); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_FRAME_BUFFERS_H_ diff --git a/third_party/aom/av1/common/generic_code.c b/third_party/aom/av1/common/generic_code.c new file mode 100644 index 0000000000..2955a695f5 --- /dev/null +++ b/third_party/aom/av1/common/generic_code.c @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "generic_code.h" + +void aom_cdf_init_q15_1D(uint16_t *cdf, int nsyms, int cdf_size) { + int i; + for (i = 0; i < nsyms; i++) + cdf[i] = AOM_ICDF((i + 1)*CDF_PROB_TOP/nsyms); + +#if CONFIG_EC_ADAPT + cdf[cdf_size - 1] = 0; +#endif +} + +/** Adapts a Q15 cdf after encoding/decoding a symbol. */ +void aom_cdf_adapt_q15(int val, uint16_t *cdf, int n, int *count, int rate) { + int i; + *count = OD_MINI(*count + 1, 1 << rate); + OD_ASSERT(AOM_ICDF(cdf[n - 1]) == 32768); + if (*count >= 1 << rate) { + /* Steady-state adaptation based on a simple IIR with dyadic rate. */ + for (i = 0; i < n; i++) { + int tmp; + /* When (i < val), we want the adjustment ((cdf[i] - tmp) >> rate) to be + positive so long as (cdf[i] > i + 1), and 0 when (cdf[i] == i + 1), + to ensure we don't drive any probabilities to 0. Replacing cdf[i] with + (i + 2) and solving ((i + 2 - tmp) >> rate == 1) for tmp produces + tmp == i + 2 - (1 << rate). Using this value of tmp with + cdf[i] == i + 1 instead gives an adjustment of 0 as desired. + + When (i >= val), we want ((cdf[i] - tmp) >> rate) to be negative so + long as cdf[i] < 32768 - (n - 1 - i), and 0 when + cdf[i] == 32768 - (n - 1 - i), again to ensure we don't drive any + probabilities to 0. Since right-shifting any negative value is still + negative, we can solve (32768 - (n - 1 - i) - tmp == 0) for tmp, + producing tmp = 32769 - n + i. Using this value of tmp with smaller + values of cdf[i] instead gives negative adjustments, as desired. + + Combining the two cases gives the expression below. These could be + stored in a lookup table indexed by n and rate to avoid the + arithmetic. */ + tmp = 2 - (1<= val); + cdf[i] = AOM_ICDF(AOM_ICDF(cdf[i]) - ((AOM_ICDF(cdf[i]) - tmp) >> rate)); + } + } + else { + int alpha; + /* Initial adaptation for the first symbols. The adaptation rate is + computed to be equivalent to what od_{en,de}code_cdf_adapt() does + when the initial cdf is set to increment/4. */ + alpha = 4*32768/(n + 4**count); + for (i = 0; i < n; i++) { + int tmp; + tmp = (32768 - n)*(i >= val) + i + 1; + cdf[i] = AOM_ICDF(AOM_ICDF(cdf[i]) + - (((AOM_ICDF(cdf[i]) - tmp)*alpha) >> 15)); + } + } + OD_ASSERT(AOM_ICDF(cdf[n - 1]) == 32768); +} + +/** Takes the base-2 log of E(x) in Q1. + * + * @param [in] ExQ16 expectation of x in Q16 + * + * @retval 2*log2(ExQ16/2^16) + */ +int log_ex(int ex_q16) { + int lg; + int lg_q1; + int odd; + lg = OD_ILOG(ex_q16); + if (lg < 15) { + odd = ex_q16*ex_q16 > 2 << 2*lg; + } + else { + int tmp; + tmp = ex_q16 >> (lg - 8); + odd = tmp*tmp > (1 << 15); + } + lg_q1 = OD_MAXI(0, 2*lg - 33 + odd); + return lg_q1; +} + +/** Updates the probability model based on the encoded/decoded value + * + * @param [in,out] model generic prob model + * @param [in,out] ExQ16 expectation of x + * @param [in] x variable encoded/decoded (used for ExQ16) + * @param [in] xs variable x after shift (used for the model) + * @param [in] id id of the icdf to adapt + * @param [in] integration integration period of ExQ16 (leaky average over + * 1<> (shift)) + +void generic_model_init(generic_encoder *model); + +/* Initialize a CDF for use by aom_write_symbol_pvq()/aom_read_symbol_pvq(). + This is used for CDFs whose size might not match the declared array size. + The only real requirement is that the first value of every CDF be zero. + Then aom_cdf_init_q15_1D() will be called with the real size the first time + the CDF is used. */ +#define OD_CDFS_INIT_DYNAMIC(cdf) (memset(cdf, 0, sizeof(cdf))) + +// WARNING: DO NOT USE this init function, +// if the size of cdf is different from what is declared by code. +#define OD_CDFS_INIT_Q15(cdfs) \ + { int n_cdfs = sizeof(cdfs)/sizeof(cdfs[0]); \ + int cdf_size = sizeof(cdfs[0])/sizeof(cdfs[0][0]); \ + int nsyms = cdf_size - CONFIG_EC_ADAPT; \ + int i_; \ + for (i_ = 0; i_ < n_cdfs; i_++) \ + aom_cdf_init_q15_1D(cdfs[i_], nsyms, cdf_size); \ + } + +void aom_cdf_init(uint16_t *cdf, int ncdfs, int nsyms, int val, int first); + +void aom_cdf_init_q15_1D(uint16_t *cdf, int nsyms, int cdf_size); + +void aom_cdf_adapt_q15(int val, uint16_t *cdf, int n, int *count, int rate); + +void aom_encode_cdf_adapt_q15(aom_writer *w, int val, uint16_t *cdf, int n, + int *count, int rate); + +void generic_encode(aom_writer *w, generic_encoder *model, int x, + int *ex_q16, int integration); +double generic_encode_cost(generic_encoder *model, int x, int *ex_q16); + +double od_encode_cdf_cost(int val, uint16_t *cdf, int n); + +int aom_decode_cdf_adapt_q15_(aom_reader *r, uint16_t *cdf, int n, + int *count, int rate ACCT_STR_PARAM); + +int generic_decode_(aom_reader *r, generic_encoder *model, + int *ex_q16, int integration ACCT_STR_PARAM); + +int log_ex(int ex_q16); + +void generic_model_update(int *ex_q16, int x, int integration); + +#endif diff --git a/third_party/aom/av1/common/idct.c b/third_party/aom/av1/common/idct.c new file mode 100644 index 0000000000..0ea58bfe68 --- /dev/null +++ b/third_party/aom/av1/common/idct.c @@ -0,0 +1,3067 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" +#include "aom_dsp/inv_txfm.h" +#include "aom_ports/mem.h" +#include "av1/common/av1_inv_txfm2d_cfg.h" +#include "av1/common/blockd.h" +#include "av1/common/enums.h" +#include "av1/common/idct.h" + +int av1_get_tx_scale(const TX_SIZE tx_size) { + if (txsize_sqr_up_map[tx_size] == TX_32X32) return 1; +#if CONFIG_TX64X64 + else if (txsize_sqr_up_map[tx_size] == TX_64X64) + return 2; +#endif // CONFIG_TX64X64 + else + return 0; +} + +// NOTE: The implementation of all inverses need to be aware of the fact +// that input and output could be the same buffer. + +#if CONFIG_EXT_TX +static void iidtx4_c(const tran_low_t *input, tran_low_t *output) { + int i; + for (i = 0; i < 4; ++i) + output[i] = (tran_low_t)dct_const_round_shift(input[i] * Sqrt2); +} + +static void iidtx8_c(const tran_low_t *input, tran_low_t *output) { + int i; + for (i = 0; i < 8; ++i) output[i] = input[i] * 2; +} + +static void iidtx16_c(const tran_low_t *input, tran_low_t *output) { + int i; + for (i = 0; i < 16; ++i) + output[i] = (tran_low_t)dct_const_round_shift(input[i] * 2 * Sqrt2); +} + +static void iidtx32_c(const tran_low_t *input, tran_low_t *output) { + int i; + for (i = 0; i < 32; ++i) output[i] = input[i] * 4; +} + +#if CONFIG_TX64X64 +static void iidtx64_c(const tran_low_t *input, tran_low_t *output) { + int i; + for (i = 0; i < 64; ++i) + output[i] = (tran_low_t)dct_const_round_shift(input[i] * 4 * Sqrt2); +} +#endif // CONFIG_TX64X64 +#endif // CONFIG_EXT_TX + +// For use in lieu of ADST +static void ihalfright32_c(const tran_low_t *input, tran_low_t *output) { + int i; + tran_low_t inputhalf[16]; + // Multiply input by sqrt(2) + for (i = 0; i < 16; ++i) { + inputhalf[i] = (tran_low_t)dct_const_round_shift(input[i] * Sqrt2); + } + for (i = 0; i < 16; ++i) { + output[i] = input[16 + i] * 4; + } + aom_idct16_c(inputhalf, output + 16); + // Note overall scaling factor is 4 times orthogonal +} + +#if CONFIG_TX64X64 +static void idct64_col_c(const tran_low_t *input, tran_low_t *output) { + int32_t in[64], out[64]; + int i; + for (i = 0; i < 64; ++i) in[i] = (int32_t)input[i]; + av1_idct64_new(in, out, inv_cos_bit_col_dct_dct_64, + inv_stage_range_col_dct_dct_64); + for (i = 0; i < 64; ++i) output[i] = (tran_low_t)out[i]; +} + +static void idct64_row_c(const tran_low_t *input, tran_low_t *output) { + int32_t in[64], out[64]; + int i; + for (i = 0; i < 64; ++i) in[i] = (int32_t)input[i]; + av1_idct64_new(in, out, inv_cos_bit_row_dct_dct_64, + inv_stage_range_row_dct_dct_64); + for (i = 0; i < 64; ++i) output[i] = (tran_low_t)out[i]; +} + +// For use in lieu of ADST +static void ihalfright64_c(const tran_low_t *input, tran_low_t *output) { + int i; + tran_low_t inputhalf[32]; + // Multiply input by sqrt(2) + for (i = 0; i < 32; ++i) { + inputhalf[i] = (tran_low_t)dct_const_round_shift(input[i] * Sqrt2); + } + for (i = 0; i < 32; ++i) { + output[i] = (tran_low_t)dct_const_round_shift(input[32 + i] * 4 * Sqrt2); + } + aom_idct32_c(inputhalf, output + 32); + // Note overall scaling factor is 4 * sqrt(2) times orthogonal +} +#endif // CONFIG_TX64X64 + +#if CONFIG_HIGHBITDEPTH +static void highbd_idct4(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, const int8_t *stage_range, + int bd) { + (void)bd; + av1_idct4_new(input, output, cos_bit, stage_range); +} + +static void highbd_idct8(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, const int8_t *stage_range, + int bd) { + (void)bd; + av1_idct8_new(input, output, cos_bit, stage_range); +} + +static void highbd_idct16(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, const int8_t *stage_range, + int bd) { + (void)bd; + av1_idct16_new(input, output, cos_bit, stage_range); +} + +static void highbd_idct32(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, const int8_t *stage_range, + int bd) { + (void)bd; + av1_idct32_new(input, output, cos_bit, stage_range); +} + +static void highbd_iadst4(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, const int8_t *stage_range, + int bd) { + (void)bd; + av1_iadst4_new(input, output, cos_bit, stage_range); +} + +static void highbd_iadst8(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, const int8_t *stage_range, + int bd) { + (void)bd; + av1_iadst8_new(input, output, cos_bit, stage_range); +} + +static void highbd_iadst16(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, const int8_t *stage_range, + int bd) { + (void)bd; + av1_iadst16_new(input, output, cos_bit, stage_range); +} + +#if CONFIG_EXT_TX +static void highbd_iidtx4_c(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, const int8_t *stage_range, + int bd) { + int i; + (void)cos_bit; + (void)stage_range; + for (i = 0; i < 4; ++i) + output[i] = HIGHBD_WRAPLOW(dct_const_round_shift(input[i] * Sqrt2), bd); +} + +static void highbd_iidtx8_c(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, const int8_t *stage_range, + int bd) { + int i; + (void)bd; + (void)cos_bit; + (void)stage_range; + for (i = 0; i < 8; ++i) output[i] = input[i] * 2; +} + +static void highbd_iidtx16_c(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, const int8_t *stage_range, + int bd) { + int i; + (void)cos_bit; + (void)stage_range; + for (i = 0; i < 16; ++i) + output[i] = HIGHBD_WRAPLOW(dct_const_round_shift(input[i] * 2 * Sqrt2), bd); +} + +static void highbd_iidtx32_c(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, const int8_t *stage_range, + int bd) { + int i; + (void)bd; + (void)cos_bit; + (void)stage_range; + for (i = 0; i < 32; ++i) output[i] = input[i] * 4; +} +#endif // CONFIG_EXT_TX + +static void highbd_ihalfright32_c(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, + const int8_t *stage_range, int bd) { + int i; + tran_low_t inputhalf[16]; + // Multiply input by sqrt(2) + for (i = 0; i < 16; ++i) { + inputhalf[i] = HIGHBD_WRAPLOW(dct_const_round_shift(input[i] * Sqrt2), bd); + } + for (i = 0; i < 16; ++i) { + output[i] = input[16 + i] * 4; + } + highbd_idct16(inputhalf, output + 16, cos_bit, stage_range, bd); + // Note overall scaling factor is 4 times orthogonal +} + +#if CONFIG_EXT_TX +#if CONFIG_TX64X64 +static void highbd_iidtx64_c(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, const int8_t *stage_range, + int bd) { + (void)cos_bit; + (void)stage_range; + int i; + for (i = 0; i < 64; ++i) + output[i] = HIGHBD_WRAPLOW(dct_const_round_shift(input[i] * 4 * Sqrt2), bd); +} +#endif // CONFIG_TX64X64 +#endif // CONFIG_EXT_TX + +#if CONFIG_TX64X64 +// For use in lieu of ADST +static void highbd_ihalfright64_c(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, + const int8_t *stage_range, int bd) { + int i; + tran_low_t inputhalf[32]; + // Multiply input by sqrt(2) + for (i = 0; i < 32; ++i) { + inputhalf[i] = HIGHBD_WRAPLOW(dct_const_round_shift(input[i] * Sqrt2), bd); + } + for (i = 0; i < 32; ++i) { + output[i] = + HIGHBD_WRAPLOW(dct_const_round_shift(input[32 + i] * 4 * Sqrt2), bd); + } + highbd_idct32(inputhalf, output + 32, cos_bit, stage_range, bd); + // Note overall scaling factor is 4 * sqrt(2) times orthogonal +} + +static void highbd_idct64_col_c(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, + const int8_t *stage_range, int bd) { + int32_t in[64], out[64]; + int i; + (void)cos_bit; + (void)stage_range; + (void)bd; + for (i = 0; i < 64; ++i) in[i] = (int32_t)input[i]; + av1_idct64_new(in, out, inv_cos_bit_col_dct_dct_64, + inv_stage_range_col_dct_dct_64); + for (i = 0; i < 64; ++i) output[i] = (tran_low_t)out[i]; +} + +static void highbd_idct64_row_c(const tran_low_t *input, tran_low_t *output, + const int8_t *cos_bit, + const int8_t *stage_range, int bd) { + int32_t in[64], out[64]; + int i; + (void)cos_bit; + (void)stage_range; + (void)bd; + for (i = 0; i < 64; ++i) in[i] = (int32_t)input[i]; + av1_idct64_new(in, out, inv_cos_bit_row_dct_dct_64, + inv_stage_range_row_dct_dct_64); + for (i = 0; i < 64; ++i) output[i] = (tran_low_t)out[i]; +} +#endif // CONFIG_TX64X64 +#endif // CONFIG_HIGHBITDEPTH + +// Inverse identity transform and add. +#if CONFIG_EXT_TX +static void inv_idtx_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int bs, int tx_type) { + int r, c; + const int shift = bs < 32 ? 3 : (bs < 64 ? 2 : 1); + if (tx_type == IDTX) { + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) + dest[c] = clip_pixel_add(dest[c], input[c] >> shift); + dest += stride; + input += bs; + } + } +} +#endif // CONFIG_EXT_TX + +#define FLIPUD_PTR(dest, stride, size) \ + do { \ + (dest) = (dest) + ((size)-1) * (stride); \ + (stride) = -(stride); \ + } while (0) + +#if CONFIG_EXT_TX +static void maybe_flip_strides(uint8_t **dst, int *dstride, tran_low_t **src, + int *sstride, int tx_type, int sizey, + int sizex) { + // Note that the transpose of src will be added to dst. In order to LR + // flip the addends (in dst coordinates), we UD flip the src. To UD flip + // the addends, we UD flip the dst. + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + case IDTX: + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: break; + case FLIPADST_DCT: + case FLIPADST_ADST: + case V_FLIPADST: + // flip UD + FLIPUD_PTR(*dst, *dstride, sizey); + break; + case DCT_FLIPADST: + case ADST_FLIPADST: + case H_FLIPADST: + // flip LR + FLIPUD_PTR(*src, *sstride, sizex); + break; + case FLIPADST_FLIPADST: + // flip UD + FLIPUD_PTR(*dst, *dstride, sizey); + // flip LR + FLIPUD_PTR(*src, *sstride, sizex); + break; + default: assert(0); break; + } +} +#endif // CONFIG_EXT_TX + +#if CONFIG_HIGHBITDEPTH +#if CONFIG_EXT_TX +static void highbd_inv_idtx_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int bs, int tx_type, int bd) { + int r, c; + const int shift = bs < 32 ? 3 : 2; + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + if (tx_type == IDTX) { + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) + dest[c] = highbd_clip_pixel_add(dest[c], input[c] >> shift, bd); + dest += stride; + input += bs; + } + } +} + +static void maybe_flip_strides16(uint16_t **dst, int *dstride, tran_low_t **src, + int *sstride, int tx_type, int sizey, + int sizex) { + // Note that the transpose of src will be added to dst. In order to LR + // flip the addends (in dst coordinates), we UD flip the src. To UD flip + // the addends, we UD flip the dst. + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + case IDTX: + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: break; + case FLIPADST_DCT: + case FLIPADST_ADST: + case V_FLIPADST: + // flip UD + FLIPUD_PTR(*dst, *dstride, sizey); + break; + case DCT_FLIPADST: + case ADST_FLIPADST: + case H_FLIPADST: + // flip LR + FLIPUD_PTR(*src, *sstride, sizex); + break; + case FLIPADST_FLIPADST: + // flip UD + FLIPUD_PTR(*dst, *dstride, sizey); + // flip LR + FLIPUD_PTR(*src, *sstride, sizex); + break; + default: assert(0); break; + } +} +#endif // CONFIG_EXT_TX +#endif // CONFIG_HIGHBITDEPTH + +void av1_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_4[] = { + { aom_idct4_c, aom_idct4_c }, // DCT_DCT = 0 + { aom_iadst4_c, aom_idct4_c }, // ADST_DCT = 1 + { aom_idct4_c, aom_iadst4_c }, // DCT_ADST = 2 + { aom_iadst4_c, aom_iadst4_c }, // ADST_ADST = 3 +#if CONFIG_EXT_TX + { aom_iadst4_c, aom_idct4_c }, // FLIPADST_DCT + { aom_idct4_c, aom_iadst4_c }, // DCT_FLIPADST + { aom_iadst4_c, aom_iadst4_c }, // FLIPADST_FLIPADST + { aom_iadst4_c, aom_iadst4_c }, // ADST_FLIPADST + { aom_iadst4_c, aom_iadst4_c }, // FLIPADST_ADST + { iidtx4_c, iidtx4_c }, // IDTX + { aom_idct4_c, iidtx4_c }, // V_DCT + { iidtx4_c, aom_idct4_c }, // H_DCT + { aom_iadst4_c, iidtx4_c }, // V_ADST + { iidtx4_c, aom_iadst4_c }, // H_ADST + { aom_iadst4_c, iidtx4_c }, // V_FLIPADST + { iidtx4_c, aom_iadst4_c }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + + int i, j; + tran_low_t tmp; + tran_low_t out[4][4]; + tran_low_t *outp = &out[0][0]; + int outstride = 4; + + // inverse transform row vectors + for (i = 0; i < 4; ++i) { + IHT_4[tx_type].rows(input, out[i]); + input += 4; + } + + // transpose + for (i = 1; i < 4; i++) { + for (j = 0; j < i; j++) { + tmp = out[i][j]; + out[i][j] = out[j][i]; + out[j][i] = tmp; + } + } + + // inverse transform column vectors + for (i = 0; i < 4; ++i) { + IHT_4[tx_type].cols(out[i], out[i]); + } + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, 4, 4); +#endif + + // Sum with the destination + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 4)); + } + } +} + +void av1_iht4x8_32_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_4x8[] = { + { aom_idct8_c, aom_idct4_c }, // DCT_DCT + { aom_iadst8_c, aom_idct4_c }, // ADST_DCT + { aom_idct8_c, aom_iadst4_c }, // DCT_ADST + { aom_iadst8_c, aom_iadst4_c }, // ADST_ADST +#if CONFIG_EXT_TX + { aom_iadst8_c, aom_idct4_c }, // FLIPADST_DCT + { aom_idct8_c, aom_iadst4_c }, // DCT_FLIPADST + { aom_iadst8_c, aom_iadst4_c }, // FLIPADST_FLIPADST + { aom_iadst8_c, aom_iadst4_c }, // ADST_FLIPADST + { aom_iadst8_c, aom_iadst4_c }, // FLIPADST_ADST + { iidtx8_c, iidtx4_c }, // IDTX + { aom_idct8_c, iidtx4_c }, // V_DCT + { iidtx8_c, aom_idct4_c }, // H_DCT + { aom_iadst8_c, iidtx4_c }, // V_ADST + { iidtx8_c, aom_iadst4_c }, // H_ADST + { aom_iadst8_c, iidtx4_c }, // V_FLIPADST + { iidtx8_c, aom_iadst4_c }, // H_FLIPADST +#endif + }; + + const int n = 4; + const int n2 = 8; + int i, j; + tran_low_t out[4][8], outtmp[4]; + tran_low_t *outp = &out[0][0]; + int outstride = n2; + + // inverse transform row vectors and transpose + for (i = 0; i < n2; ++i) { + IHT_4x8[tx_type].rows(input, outtmp); + for (j = 0; j < n; ++j) + out[j][i] = (tran_low_t)dct_const_round_shift(outtmp[j] * Sqrt2); + input += n; + } + + // inverse transform column vectors + for (i = 0; i < n; ++i) { + IHT_4x8[tx_type].cols(out[i], out[i]); + } + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, n2, n); +#endif + + // Sum with the destination + for (i = 0; i < n2; ++i) { + for (j = 0; j < n; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5)); + } + } +} + +void av1_iht8x4_32_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_8x4[] = { + { aom_idct4_c, aom_idct8_c }, // DCT_DCT + { aom_iadst4_c, aom_idct8_c }, // ADST_DCT + { aom_idct4_c, aom_iadst8_c }, // DCT_ADST + { aom_iadst4_c, aom_iadst8_c }, // ADST_ADST +#if CONFIG_EXT_TX + { aom_iadst4_c, aom_idct8_c }, // FLIPADST_DCT + { aom_idct4_c, aom_iadst8_c }, // DCT_FLIPADST + { aom_iadst4_c, aom_iadst8_c }, // FLIPADST_FLIPADST + { aom_iadst4_c, aom_iadst8_c }, // ADST_FLIPADST + { aom_iadst4_c, aom_iadst8_c }, // FLIPADST_ADST + { iidtx4_c, iidtx8_c }, // IDTX + { aom_idct4_c, iidtx8_c }, // V_DCT + { iidtx4_c, aom_idct8_c }, // H_DCT + { aom_iadst4_c, iidtx8_c }, // V_ADST + { iidtx4_c, aom_iadst8_c }, // H_ADST + { aom_iadst4_c, iidtx8_c }, // V_FLIPADST + { iidtx4_c, aom_iadst8_c }, // H_FLIPADST +#endif + }; + const int n = 4; + const int n2 = 8; + + int i, j; + tran_low_t out[8][4], outtmp[8]; + tran_low_t *outp = &out[0][0]; + int outstride = n; + + // inverse transform row vectors and transpose + for (i = 0; i < n; ++i) { + IHT_8x4[tx_type].rows(input, outtmp); + for (j = 0; j < n2; ++j) + out[j][i] = (tran_low_t)dct_const_round_shift(outtmp[j] * Sqrt2); + input += n2; + } + + // inverse transform column vectors + for (i = 0; i < n2; ++i) { + IHT_8x4[tx_type].cols(out[i], out[i]); + } + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, n, n2); +#endif + + // Sum with the destination + for (i = 0; i < n; ++i) { + for (j = 0; j < n2; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5)); + } + } +} + +void av1_iht4x16_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_4x16[] = { + { aom_idct16_c, aom_idct4_c }, // DCT_DCT + { aom_iadst16_c, aom_idct4_c }, // ADST_DCT + { aom_idct16_c, aom_iadst4_c }, // DCT_ADST + { aom_iadst16_c, aom_iadst4_c }, // ADST_ADST +#if CONFIG_EXT_TX + { aom_iadst16_c, aom_idct4_c }, // FLIPADST_DCT + { aom_idct16_c, aom_iadst4_c }, // DCT_FLIPADST + { aom_iadst16_c, aom_iadst4_c }, // FLIPADST_FLIPADST + { aom_iadst16_c, aom_iadst4_c }, // ADST_FLIPADST + { aom_iadst16_c, aom_iadst4_c }, // FLIPADST_ADST + { iidtx16_c, iidtx4_c }, // IDTX + { aom_idct16_c, iidtx4_c }, // V_DCT + { iidtx16_c, aom_idct4_c }, // H_DCT + { aom_iadst16_c, iidtx4_c }, // V_ADST + { iidtx16_c, aom_iadst4_c }, // H_ADST + { aom_iadst16_c, iidtx4_c }, // V_FLIPADST + { iidtx16_c, aom_iadst4_c }, // H_FLIPADST +#endif + }; + + const int n = 4; + const int n4 = 16; + int i, j; + tran_low_t out[4][16], outtmp[4]; + tran_low_t *outp = &out[0][0]; + int outstride = n4; + + // inverse transform row vectors and transpose + for (i = 0; i < n4; ++i) { + IHT_4x16[tx_type].rows(input, outtmp); + for (j = 0; j < n; ++j) out[j][i] = outtmp[j]; + input += n; + } + + // inverse transform column vectors + for (i = 0; i < n; ++i) IHT_4x16[tx_type].cols(out[i], out[i]); + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, n4, n); +#endif + + // Sum with the destination + for (i = 0; i < n4; ++i) { + for (j = 0; j < n; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5)); + } + } +} + +void av1_iht16x4_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_16x4[] = { + { aom_idct4_c, aom_idct16_c }, // DCT_DCT + { aom_iadst4_c, aom_idct16_c }, // ADST_DCT + { aom_idct4_c, aom_iadst16_c }, // DCT_ADST + { aom_iadst4_c, aom_iadst16_c }, // ADST_ADST +#if CONFIG_EXT_TX + { aom_iadst4_c, aom_idct16_c }, // FLIPADST_DCT + { aom_idct4_c, aom_iadst16_c }, // DCT_FLIPADST + { aom_iadst4_c, aom_iadst16_c }, // FLIPADST_FLIPADST + { aom_iadst4_c, aom_iadst16_c }, // ADST_FLIPADST + { aom_iadst4_c, aom_iadst16_c }, // FLIPADST_ADST + { iidtx4_c, iidtx16_c }, // IDTX + { aom_idct4_c, iidtx16_c }, // V_DCT + { iidtx4_c, aom_idct16_c }, // H_DCT + { aom_iadst4_c, iidtx16_c }, // V_ADST + { iidtx4_c, aom_iadst16_c }, // H_ADST + { aom_iadst4_c, iidtx16_c }, // V_FLIPADST + { iidtx4_c, aom_iadst16_c }, // H_FLIPADST +#endif + }; + const int n = 4; + const int n4 = 16; + + int i, j; + tran_low_t out[16][4], outtmp[16]; + tran_low_t *outp = &out[0][0]; + int outstride = n; + + // inverse transform row vectors and transpose + for (i = 0; i < n; ++i) { + IHT_16x4[tx_type].rows(input, outtmp); + for (j = 0; j < n4; ++j) out[j][i] = outtmp[j]; + input += n4; + } + + // inverse transform column vectors + for (i = 0; i < n4; ++i) IHT_16x4[tx_type].cols(out[i], out[i]); + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, n, n4); +#endif + + // Sum with the destination + for (i = 0; i < n; ++i) { + for (j = 0; j < n4; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5)); + } + } +} + +void av1_iht8x16_128_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_8x16[] = { + { aom_idct16_c, aom_idct8_c }, // DCT_DCT + { aom_iadst16_c, aom_idct8_c }, // ADST_DCT + { aom_idct16_c, aom_iadst8_c }, // DCT_ADST + { aom_iadst16_c, aom_iadst8_c }, // ADST_ADST +#if CONFIG_EXT_TX + { aom_iadst16_c, aom_idct8_c }, // FLIPADST_DCT + { aom_idct16_c, aom_iadst8_c }, // DCT_FLIPADST + { aom_iadst16_c, aom_iadst8_c }, // FLIPADST_FLIPADST + { aom_iadst16_c, aom_iadst8_c }, // ADST_FLIPADST + { aom_iadst16_c, aom_iadst8_c }, // FLIPADST_ADST + { iidtx16_c, iidtx8_c }, // IDTX + { aom_idct16_c, iidtx8_c }, // V_DCT + { iidtx16_c, aom_idct8_c }, // H_DCT + { aom_iadst16_c, iidtx8_c }, // V_ADST + { iidtx16_c, aom_iadst8_c }, // H_ADST + { aom_iadst16_c, iidtx8_c }, // V_FLIPADST + { iidtx16_c, aom_iadst8_c }, // H_FLIPADST +#endif + }; + + const int n = 8; + const int n2 = 16; + int i, j; + tran_low_t out[8][16], outtmp[8]; + tran_low_t *outp = &out[0][0]; + int outstride = n2; + + // inverse transform row vectors and transpose + for (i = 0; i < n2; ++i) { + IHT_8x16[tx_type].rows(input, outtmp); + for (j = 0; j < n; ++j) + out[j][i] = (tran_low_t)dct_const_round_shift(outtmp[j] * Sqrt2); + input += n; + } + + // inverse transform column vectors + for (i = 0; i < n; ++i) { + IHT_8x16[tx_type].cols(out[i], out[i]); + } + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, n2, n); +#endif + + // Sum with the destination + for (i = 0; i < n2; ++i) { + for (j = 0; j < n; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6)); + } + } +} + +void av1_iht16x8_128_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_16x8[] = { + { aom_idct8_c, aom_idct16_c }, // DCT_DCT + { aom_iadst8_c, aom_idct16_c }, // ADST_DCT + { aom_idct8_c, aom_iadst16_c }, // DCT_ADST + { aom_iadst8_c, aom_iadst16_c }, // ADST_ADST +#if CONFIG_EXT_TX + { aom_iadst8_c, aom_idct16_c }, // FLIPADST_DCT + { aom_idct8_c, aom_iadst16_c }, // DCT_FLIPADST + { aom_iadst8_c, aom_iadst16_c }, // FLIPADST_FLIPADST + { aom_iadst8_c, aom_iadst16_c }, // ADST_FLIPADST + { aom_iadst8_c, aom_iadst16_c }, // FLIPADST_ADST + { iidtx8_c, iidtx16_c }, // IDTX + { aom_idct8_c, iidtx16_c }, // V_DCT + { iidtx8_c, aom_idct16_c }, // H_DCT + { aom_iadst8_c, iidtx16_c }, // V_ADST + { iidtx8_c, aom_iadst16_c }, // H_ADST + { aom_iadst8_c, iidtx16_c }, // V_FLIPADST + { iidtx8_c, aom_iadst16_c }, // H_FLIPADST +#endif + }; + const int n = 8; + const int n2 = 16; + + int i, j; + tran_low_t out[16][8], outtmp[16]; + tran_low_t *outp = &out[0][0]; + int outstride = n; + + // inverse transform row vectors and transpose + for (i = 0; i < n; ++i) { + IHT_16x8[tx_type].rows(input, outtmp); + for (j = 0; j < n2; ++j) + out[j][i] = (tran_low_t)dct_const_round_shift(outtmp[j] * Sqrt2); + input += n2; + } + + // inverse transform column vectors + for (i = 0; i < n2; ++i) { + IHT_16x8[tx_type].cols(out[i], out[i]); + } + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, n, n2); +#endif + + // Sum with the destination + for (i = 0; i < n; ++i) { + for (j = 0; j < n2; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6)); + } + } +} + +void av1_iht8x32_256_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_8x32[] = { + { aom_idct32_c, aom_idct8_c }, // DCT_DCT + { ihalfright32_c, aom_idct8_c }, // ADST_DCT + { aom_idct32_c, aom_iadst8_c }, // DCT_ADST + { ihalfright32_c, aom_iadst8_c }, // ADST_ADST +#if CONFIG_EXT_TX + { ihalfright32_c, aom_idct8_c }, // FLIPADST_DCT + { aom_idct32_c, aom_iadst8_c }, // DCT_FLIPADST + { ihalfright32_c, aom_iadst8_c }, // FLIPADST_FLIPADST + { ihalfright32_c, aom_iadst8_c }, // ADST_FLIPADST + { ihalfright32_c, aom_iadst8_c }, // FLIPADST_ADST + { iidtx32_c, iidtx8_c }, // IDTX + { aom_idct32_c, iidtx8_c }, // V_DCT + { iidtx32_c, aom_idct8_c }, // H_DCT + { ihalfright32_c, iidtx8_c }, // V_ADST + { iidtx32_c, aom_iadst8_c }, // H_ADST + { ihalfright32_c, iidtx8_c }, // V_FLIPADST + { iidtx32_c, aom_iadst8_c }, // H_FLIPADST +#endif + }; + + const int n = 8; + const int n4 = 32; + int i, j; + tran_low_t out[8][32], outtmp[8]; + tran_low_t *outp = &out[0][0]; + int outstride = n4; + + // inverse transform row vectors and transpose + for (i = 0; i < n4; ++i) { + IHT_8x32[tx_type].rows(input, outtmp); + for (j = 0; j < n; ++j) out[j][i] = outtmp[j]; + input += n; + } + + // inverse transform column vectors + for (i = 0; i < n; ++i) IHT_8x32[tx_type].cols(out[i], out[i]); + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, n4, n); +#endif + + // Sum with the destination + for (i = 0; i < n4; ++i) { + for (j = 0; j < n; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6)); + } + } +} + +void av1_iht32x8_256_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_32x8[] = { + { aom_idct8_c, aom_idct32_c }, // DCT_DCT + { aom_iadst8_c, aom_idct32_c }, // ADST_DCT + { aom_idct8_c, ihalfright32_c }, // DCT_ADST + { aom_iadst8_c, ihalfright32_c }, // ADST_ADST +#if CONFIG_EXT_TX + { aom_iadst8_c, aom_idct32_c }, // FLIPADST_DCT + { aom_idct8_c, ihalfright32_c }, // DCT_FLIPADST + { aom_iadst8_c, ihalfright32_c }, // FLIPADST_FLIPADST + { aom_iadst8_c, ihalfright32_c }, // ADST_FLIPADST + { aom_iadst8_c, ihalfright32_c }, // FLIPADST_ADST + { iidtx8_c, iidtx32_c }, // IDTX + { aom_idct8_c, iidtx32_c }, // V_DCT + { iidtx8_c, aom_idct32_c }, // H_DCT + { aom_iadst8_c, iidtx32_c }, // V_ADST + { iidtx8_c, ihalfright32_c }, // H_ADST + { aom_iadst8_c, iidtx32_c }, // V_FLIPADST + { iidtx8_c, ihalfright32_c }, // H_FLIPADST +#endif + }; + const int n = 8; + const int n4 = 32; + + int i, j; + tran_low_t out[32][8], outtmp[32]; + tran_low_t *outp = &out[0][0]; + int outstride = n; + + // inverse transform row vectors and transpose + for (i = 0; i < n; ++i) { + IHT_32x8[tx_type].rows(input, outtmp); + for (j = 0; j < n4; ++j) out[j][i] = outtmp[j]; + input += n4; + } + + // inverse transform column vectors + for (i = 0; i < n4; ++i) IHT_32x8[tx_type].cols(out[i], out[i]); + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, n, n4); +#endif + + // Sum with the destination + for (i = 0; i < n; ++i) { + for (j = 0; j < n4; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6)); + } + } +} + +void av1_iht16x32_512_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_16x32[] = { + { aom_idct32_c, aom_idct16_c }, // DCT_DCT + { ihalfright32_c, aom_idct16_c }, // ADST_DCT + { aom_idct32_c, aom_iadst16_c }, // DCT_ADST + { ihalfright32_c, aom_iadst16_c }, // ADST_ADST +#if CONFIG_EXT_TX + { ihalfright32_c, aom_idct16_c }, // FLIPADST_DCT + { aom_idct32_c, aom_iadst16_c }, // DCT_FLIPADST + { ihalfright32_c, aom_iadst16_c }, // FLIPADST_FLIPADST + { ihalfright32_c, aom_iadst16_c }, // ADST_FLIPADST + { ihalfright32_c, aom_iadst16_c }, // FLIPADST_ADST + { iidtx32_c, iidtx16_c }, // IDTX + { aom_idct32_c, iidtx16_c }, // V_DCT + { iidtx32_c, aom_idct16_c }, // H_DCT + { ihalfright32_c, iidtx16_c }, // V_ADST + { iidtx32_c, aom_iadst16_c }, // H_ADST + { ihalfright32_c, iidtx16_c }, // V_FLIPADST + { iidtx32_c, aom_iadst16_c }, // H_FLIPADST +#endif + }; + + const int n = 16; + const int n2 = 32; + int i, j; + tran_low_t out[16][32], outtmp[16]; + tran_low_t *outp = &out[0][0]; + int outstride = n2; + + // inverse transform row vectors and transpose + for (i = 0; i < n2; ++i) { + IHT_16x32[tx_type].rows(input, outtmp); + for (j = 0; j < n; ++j) + out[j][i] = (tran_low_t)dct_const_round_shift(outtmp[j] * Sqrt2); + input += n; + } + + // inverse transform column vectors + for (i = 0; i < n; ++i) { + IHT_16x32[tx_type].cols(out[i], out[i]); + } + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, n2, n); +#endif + + // Sum with the destination + for (i = 0; i < n2; ++i) { + for (j = 0; j < n; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6)); + } + } +} + +void av1_iht32x16_512_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_32x16[] = { + { aom_idct16_c, aom_idct32_c }, // DCT_DCT + { aom_iadst16_c, aom_idct32_c }, // ADST_DCT + { aom_idct16_c, ihalfright32_c }, // DCT_ADST + { aom_iadst16_c, ihalfright32_c }, // ADST_ADST +#if CONFIG_EXT_TX + { aom_iadst16_c, aom_idct32_c }, // FLIPADST_DCT + { aom_idct16_c, ihalfright32_c }, // DCT_FLIPADST + { aom_iadst16_c, ihalfright32_c }, // FLIPADST_FLIPADST + { aom_iadst16_c, ihalfright32_c }, // ADST_FLIPADST + { aom_iadst16_c, ihalfright32_c }, // FLIPADST_ADST + { iidtx16_c, iidtx32_c }, // IDTX + { aom_idct16_c, iidtx32_c }, // V_DCT + { iidtx16_c, aom_idct32_c }, // H_DCT + { aom_iadst16_c, iidtx32_c }, // V_ADST + { iidtx16_c, ihalfright32_c }, // H_ADST + { aom_iadst16_c, iidtx32_c }, // V_FLIPADST + { iidtx16_c, ihalfright32_c }, // H_FLIPADST +#endif + }; + const int n = 16; + const int n2 = 32; + + int i, j; + tran_low_t out[32][16], outtmp[32]; + tran_low_t *outp = &out[0][0]; + int outstride = n; + + // inverse transform row vectors and transpose + for (i = 0; i < n; ++i) { + IHT_32x16[tx_type].rows(input, outtmp); + for (j = 0; j < n2; ++j) + out[j][i] = (tran_low_t)dct_const_round_shift(outtmp[j] * Sqrt2); + input += n2; + } + + // inverse transform column vectors + for (i = 0; i < n2; ++i) { + IHT_32x16[tx_type].cols(out[i], out[i]); + } + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, n, n2); +#endif + + // Sum with the destination + for (i = 0; i < n; ++i) { + for (j = 0; j < n2; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6)); + } + } +} + +void av1_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_8[] = { + { aom_idct8_c, aom_idct8_c }, // DCT_DCT = 0 + { aom_iadst8_c, aom_idct8_c }, // ADST_DCT = 1 + { aom_idct8_c, aom_iadst8_c }, // DCT_ADST = 2 + { aom_iadst8_c, aom_iadst8_c }, // ADST_ADST = 3 +#if CONFIG_EXT_TX + { aom_iadst8_c, aom_idct8_c }, // FLIPADST_DCT + { aom_idct8_c, aom_iadst8_c }, // DCT_FLIPADST + { aom_iadst8_c, aom_iadst8_c }, // FLIPADST_FLIPADST + { aom_iadst8_c, aom_iadst8_c }, // ADST_FLIPADST + { aom_iadst8_c, aom_iadst8_c }, // FLIPADST_ADST + { iidtx8_c, iidtx8_c }, // IDTX + { aom_idct8_c, iidtx8_c }, // V_DCT + { iidtx8_c, aom_idct8_c }, // H_DCT + { aom_iadst8_c, iidtx8_c }, // V_ADST + { iidtx8_c, aom_iadst8_c }, // H_ADST + { aom_iadst8_c, iidtx8_c }, // V_FLIPADST + { iidtx8_c, aom_iadst8_c }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + + int i, j; + tran_low_t tmp; + tran_low_t out[8][8]; + tran_low_t *outp = &out[0][0]; + int outstride = 8; + + // inverse transform row vectors + for (i = 0; i < 8; ++i) { + IHT_8[tx_type].rows(input, out[i]); + input += 8; + } + + // transpose + for (i = 1; i < 8; i++) { + for (j = 0; j < i; j++) { + tmp = out[i][j]; + out[i][j] = out[j][i]; + out[j][i] = tmp; + } + } + + // inverse transform column vectors + for (i = 0; i < 8; ++i) { + IHT_8[tx_type].cols(out[i], out[i]); + } + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, 8, 8); +#endif + + // Sum with the destination + for (i = 0; i < 8; ++i) { + for (j = 0; j < 8; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5)); + } + } +} + +void av1_iht16x16_256_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_16[] = { + { aom_idct16_c, aom_idct16_c }, // DCT_DCT = 0 + { aom_iadst16_c, aom_idct16_c }, // ADST_DCT = 1 + { aom_idct16_c, aom_iadst16_c }, // DCT_ADST = 2 + { aom_iadst16_c, aom_iadst16_c }, // ADST_ADST = 3 +#if CONFIG_EXT_TX + { aom_iadst16_c, aom_idct16_c }, // FLIPADST_DCT + { aom_idct16_c, aom_iadst16_c }, // DCT_FLIPADST + { aom_iadst16_c, aom_iadst16_c }, // FLIPADST_FLIPADST + { aom_iadst16_c, aom_iadst16_c }, // ADST_FLIPADST + { aom_iadst16_c, aom_iadst16_c }, // FLIPADST_ADST + { iidtx16_c, iidtx16_c }, // IDTX + { aom_idct16_c, iidtx16_c }, // V_DCT + { iidtx16_c, aom_idct16_c }, // H_DCT + { aom_iadst16_c, iidtx16_c }, // V_ADST + { iidtx16_c, aom_iadst16_c }, // H_ADST + { aom_iadst16_c, iidtx16_c }, // V_FLIPADST + { iidtx16_c, aom_iadst16_c }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + + int i, j; + tran_low_t tmp; + tran_low_t out[16][16]; + tran_low_t *outp = &out[0][0]; + int outstride = 16; + + // inverse transform row vectors + for (i = 0; i < 16; ++i) { + IHT_16[tx_type].rows(input, out[i]); + input += 16; + } + + // transpose + for (i = 1; i < 16; i++) { + for (j = 0; j < i; j++) { + tmp = out[i][j]; + out[i][j] = out[j][i]; + out[j][i] = tmp; + } + } + + // inverse transform column vectors + for (i = 0; i < 16; ++i) { + IHT_16[tx_type].cols(out[i], out[i]); + } + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, 16, 16); +#endif + + // Sum with the destination + for (i = 0; i < 16; ++i) { + for (j = 0; j < 16; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6)); + } + } +} + +#if CONFIG_EXT_TX +void av1_iht32x32_1024_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_32[] = { + { aom_idct32_c, aom_idct32_c }, // DCT_DCT + { ihalfright32_c, aom_idct32_c }, // ADST_DCT + { aom_idct32_c, ihalfright32_c }, // DCT_ADST + { ihalfright32_c, ihalfright32_c }, // ADST_ADST + { ihalfright32_c, aom_idct32_c }, // FLIPADST_DCT + { aom_idct32_c, ihalfright32_c }, // DCT_FLIPADST + { ihalfright32_c, ihalfright32_c }, // FLIPADST_FLIPADST + { ihalfright32_c, ihalfright32_c }, // ADST_FLIPADST + { ihalfright32_c, ihalfright32_c }, // FLIPADST_ADST + { iidtx32_c, iidtx32_c }, // IDTX + { aom_idct32_c, iidtx32_c }, // V_DCT + { iidtx32_c, aom_idct32_c }, // H_DCT + { ihalfright32_c, iidtx32_c }, // V_ADST + { iidtx32_c, ihalfright32_c }, // H_ADST + { ihalfright32_c, iidtx32_c }, // V_FLIPADST + { iidtx32_c, ihalfright32_c }, // H_FLIPADST + }; + + int i, j; + tran_low_t tmp; + tran_low_t out[32][32]; + tran_low_t *outp = &out[0][0]; + int outstride = 32; + + // inverse transform row vectors + for (i = 0; i < 32; ++i) { + IHT_32[tx_type].rows(input, out[i]); + input += 32; + } + + // transpose + for (i = 1; i < 32; i++) { + for (j = 0; j < i; j++) { + tmp = out[i][j]; + out[i][j] = out[j][i]; + out[j][i] = tmp; + } + } + + // inverse transform column vectors + for (i = 0; i < 32; ++i) { + IHT_32[tx_type].cols(out[i], out[i]); + } + + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, 32, 32); + + // Sum with the destination + for (i = 0; i < 32; ++i) { + for (j = 0; j < 32; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6)); + } + } +} +#endif // CONFIG_EXT_TX + +#if CONFIG_TX64X64 +void av1_iht64x64_4096_add_c(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + static const transform_2d IHT_64[] = { + { idct64_col_c, idct64_row_c }, // DCT_DCT + { ihalfright64_c, idct64_row_c }, // ADST_DCT + { idct64_col_c, ihalfright64_c }, // DCT_ADST + { ihalfright64_c, ihalfright64_c }, // ADST_ADST +#if CONFIG_EXT_TX + { ihalfright64_c, idct64_row_c }, // FLIPADST_DCT + { idct64_col_c, ihalfright64_c }, // DCT_FLIPADST + { ihalfright64_c, ihalfright64_c }, // FLIPADST_FLIPADST + { ihalfright64_c, ihalfright64_c }, // ADST_FLIPADST + { ihalfright64_c, ihalfright64_c }, // FLIPADST_ADST + { iidtx64_c, iidtx64_c }, // IDTX + { idct64_col_c, iidtx64_c }, // V_DCT + { iidtx64_c, idct64_row_c }, // H_DCT + { ihalfright64_c, iidtx64_c }, // V_ADST + { iidtx64_c, ihalfright64_c }, // H_ADST + { ihalfright64_c, iidtx64_c }, // V_FLIPADST + { iidtx64_c, ihalfright64_c }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + + int i, j; + tran_low_t tmp; + tran_low_t out[64][64]; + tran_low_t *outp = &out[0][0]; + int outstride = 64; + + // inverse transform row vectors + for (i = 0; i < 64; ++i) { + IHT_64[tx_type].rows(input, out[i]); + for (j = 0; j < 64; ++j) out[i][j] = ROUND_POWER_OF_TWO(out[i][j], 1); + input += 64; + } + + // transpose + for (i = 1; i < 64; i++) { + for (j = 0; j < i; j++) { + tmp = out[i][j]; + out[i][j] = out[j][i]; + out[j][i] = tmp; + } + } + + // inverse transform column vectors + for (i = 0; i < 64; ++i) { + IHT_64[tx_type].cols(out[i], out[i]); + } + +#if CONFIG_EXT_TX + maybe_flip_strides(&dest, &stride, &outp, &outstride, tx_type, 64, 64); +#endif // CONFIG_EXT_TX + + // Sum with the destination + for (i = 0; i < 64; ++i) { + for (j = 0; j < 64; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5)); + } + } +} +#endif // CONFIG_TX64X64 + +// idct +void av1_idct4x4_add(const tran_low_t *input, uint8_t *dest, int stride, + int eob) { + if (eob > 1) + aom_idct4x4_16_add(input, dest, stride); + else + aom_idct4x4_1_add(input, dest, stride); +} + +void av1_iwht4x4_add(const tran_low_t *input, uint8_t *dest, int stride, + int eob) { + if (eob > 1) + aom_iwht4x4_16_add(input, dest, stride); + else + aom_iwht4x4_1_add(input, dest, stride); +} + +static void idct8x8_add(const tran_low_t *input, uint8_t *dest, int stride, + int eob) { + // If dc is 1, then input[0] is the reconstructed value, do not need + // dequantization. Also, when dc is 1, dc is counted in eobs, namely eobs >=1. + + // The calculation can be simplified if there are not many non-zero dct + // coefficients. Use eobs to decide what to do. + // TODO(yunqingwang): "eobs = 1" case is also handled in av1_short_idct8x8_c. + // Combine that with code here. + if (eob == 1) + // DC only DCT coefficient + aom_idct8x8_1_add(input, dest, stride); +#if !CONFIG_ADAPT_SCAN + else if (eob <= 12) + aom_idct8x8_12_add(input, dest, stride); +#endif + else + aom_idct8x8_64_add(input, dest, stride); +} + +static void idct16x16_add(const tran_low_t *input, uint8_t *dest, int stride, + int eob) { + /* The calculation can be simplified if there are not many non-zero dct + * coefficients. Use eobs to separate different cases. */ + if (eob == 1) /* DC only DCT coefficient. */ + aom_idct16x16_1_add(input, dest, stride); +#if !CONFIG_ADAPT_SCAN + else if (eob <= 10) + aom_idct16x16_10_add(input, dest, stride); +#endif + else + aom_idct16x16_256_add(input, dest, stride); +} + +static void idct32x32_add(const tran_low_t *input, uint8_t *dest, int stride, + int eob) { + if (eob == 1) aom_idct32x32_1_add(input, dest, stride); +#if !CONFIG_ADAPT_SCAN + else if (eob <= 34) + // non-zero coeff only in upper-left 8x8 + aom_idct32x32_34_add(input, dest, stride); +#endif + else + aom_idct32x32_1024_add(input, dest, stride); +} + +#if CONFIG_TX64X64 +static void idct64x64_add(const tran_low_t *input, uint8_t *dest, int stride, + int eob) { + (void)eob; + av1_iht64x64_4096_add(input, dest, stride, DCT_DCT); +} +#endif // CONFIG_TX64X64 + +#if CONFIG_CB4X4 +static void inv_txfm_add_2x2(const tran_low_t *input, uint8_t *dest, int stride, + int eob, TX_TYPE tx_type, int lossless) { + tran_high_t a1 = input[0] >> UNIT_QUANT_SHIFT; + tran_high_t b1 = input[1] >> UNIT_QUANT_SHIFT; + tran_high_t c1 = input[2] >> UNIT_QUANT_SHIFT; + tran_high_t d1 = input[3] >> UNIT_QUANT_SHIFT; + + tran_high_t a2 = a1 + c1; + tran_high_t b2 = b1 + d1; + tran_high_t c2 = a1 - c1; + tran_high_t d2 = b1 - d1; + + (void)tx_type; + (void)lossless; + (void)eob; + + a1 = (a2 + b2) >> 2; + b1 = (a2 - b2) >> 2; + c1 = (c2 + d2) >> 2; + d1 = (c2 - d2) >> 2; + + dest[0] = clip_pixel_add(dest[0], WRAPLOW(a1)); + dest[1] = clip_pixel_add(dest[1], WRAPLOW(b1)); + dest[stride] = clip_pixel_add(dest[stride], WRAPLOW(c1)); + dest[stride + 1] = clip_pixel_add(dest[stride + 1], WRAPLOW(d1)); +} +#endif + +void av1_inv_txfm_add_4x4(const tran_low_t *input, uint8_t *dest, int stride, + int eob, TX_TYPE tx_type, int lossless) { + if (lossless) { + assert(tx_type == DCT_DCT); + av1_iwht4x4_add(input, dest, stride, eob); + return; + } + + switch (tx_type) { + case DCT_DCT: av1_idct4x4_add(input, dest, stride, eob); break; + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: av1_iht4x4_16_add(input, dest, stride, tx_type); break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: av1_iht4x4_16_add(input, dest, stride, tx_type); break; + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + // Use C version since DST only exists in C code + av1_iht4x4_16_add_c(input, dest, stride, tx_type); + break; + case IDTX: inv_idtx_add_c(input, dest, stride, 4, tx_type); break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} + +void av1_inv_txfm_add_4x8(const tran_low_t *input, uint8_t *dest, int stride, + int eob, TX_TYPE tx_type) { + (void)eob; + av1_iht4x8_32_add(input, dest, stride, tx_type); +} + +void av1_inv_txfm_add_8x4(const tran_low_t *input, uint8_t *dest, int stride, + int eob, TX_TYPE tx_type) { + (void)eob; + av1_iht8x4_32_add(input, dest, stride, tx_type); +} + +// These will be used by the masked-tx experiment in the future. +#if CONFIG_MASKED_TX && 0 +static void inv_txfm_add_4x16(const tran_low_t *input, uint8_t *dest, + int stride, int eob, TX_TYPE tx_type) { + (void)eob; + av1_iht4x16_64_add(input, dest, stride, tx_type); +} + +static void inv_txfm_add_16x4(const tran_low_t *input, uint8_t *dest, + int stride, int eob, TX_TYPE tx_type) { + (void)eob; + av1_iht16x4_64_add(input, dest, stride, tx_type); +} + +static void inv_txfm_add_8x32(const tran_low_t *input, uint8_t *dest, + int stride, int eob, TX_TYPE tx_type) { + (void)eob; + av1_iht8x32_256_add(input, dest, stride, tx_type); +} + +static void inv_txfm_add_32x8(const tran_low_t *input, uint8_t *dest, + int stride, int eob, TX_TYPE tx_type) { + (void)eob; + av1_iht32x8_256_add(input, dest, stride, tx_type); +} +#endif // CONFIG_MASKED_TX + +static void inv_txfm_add_8x16(const tran_low_t *input, uint8_t *dest, + int stride, int eob, TX_TYPE tx_type) { + (void)eob; + av1_iht8x16_128_add(input, dest, stride, tx_type); +} + +static void inv_txfm_add_16x8(const tran_low_t *input, uint8_t *dest, + int stride, int eob, TX_TYPE tx_type) { + (void)eob; + av1_iht16x8_128_add(input, dest, stride, tx_type); +} + +static void inv_txfm_add_16x32(const tran_low_t *input, uint8_t *dest, + int stride, int eob, TX_TYPE tx_type) { + (void)eob; + av1_iht16x32_512_add(input, dest, stride, tx_type); +} + +static void inv_txfm_add_32x16(const tran_low_t *input, uint8_t *dest, + int stride, int eob, TX_TYPE tx_type) { + (void)eob; + av1_iht32x16_512_add(input, dest, stride, tx_type); +} + +static void inv_txfm_add_8x8(const tran_low_t *input, uint8_t *dest, int stride, + int eob, TX_TYPE tx_type) { + switch (tx_type) { + case DCT_DCT: idct8x8_add(input, dest, stride, eob); break; + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: av1_iht8x8_64_add(input, dest, stride, tx_type); break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: av1_iht8x8_64_add(input, dest, stride, tx_type); break; + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + // Use C version since DST only exists in C code + av1_iht8x8_64_add_c(input, dest, stride, tx_type); + break; + case IDTX: inv_idtx_add_c(input, dest, stride, 8, tx_type); break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} + +static void inv_txfm_add_16x16(const tran_low_t *input, uint8_t *dest, + int stride, int eob, TX_TYPE tx_type) { + switch (tx_type) { + case DCT_DCT: idct16x16_add(input, dest, stride, eob); break; + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: av1_iht16x16_256_add(input, dest, stride, tx_type); break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: av1_iht16x16_256_add(input, dest, stride, tx_type); break; + case IDTX: inv_idtx_add_c(input, dest, stride, 16, tx_type); break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} + +static void inv_txfm_add_32x32(const tran_low_t *input, uint8_t *dest, + int stride, int eob, TX_TYPE tx_type) { + switch (tx_type) { + case DCT_DCT: idct32x32_add(input, dest, stride, eob); break; +#if CONFIG_EXT_TX + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + av1_iht32x32_1024_add_c(input, dest, stride, tx_type); + break; + case IDTX: inv_idtx_add_c(input, dest, stride, 32, tx_type); break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} + +#if CONFIG_TX64X64 +static void inv_txfm_add_64x64(const tran_low_t *input, uint8_t *dest, + int stride, int eob, TX_TYPE tx_type) { + switch (tx_type) { + case DCT_DCT: idct64x64_add(input, dest, stride, eob); break; +#if CONFIG_EXT_TX + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + av1_iht64x64_4096_add_c(input, dest, stride, tx_type); + break; + case IDTX: inv_idtx_add_c(input, dest, stride, 64, tx_type); break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} +#endif // CONFIG_TX64X64 + +#if CONFIG_HIGHBITDEPTH + +const TXFM_2D_CFG *inv_txfm_cfg_ls[TX_TYPES][TX_SIZES]; + +typedef struct { + const int8_t *cos_bit; + const int8_t *stage_range; +} tx_1d_cfg; + +typedef struct { + tx_1d_cfg row; + tx_1d_cfg col; +} tx_2d_cfg; + +tx_2d_cfg inv_tx_cfg(int tx_type, int tx_size_row, int tx_size_col) { + const TXFM_2D_CFG *cfg_row = inv_txfm_cfg_ls[tx_type][tx_size_row]; + const int8_t *stage_range_row = cfg_row->stage_range_row; + const int8_t *cos_bit_row = cfg_row->cos_bit_row; + + const TXFM_2D_CFG *cfg_col = inv_txfm_cfg_ls[tx_type][tx_size_col]; + const int8_t *stage_range_col = cfg_col->stage_range_col; + const int8_t *cos_bit_col = cfg_col->cos_bit_col; + + tx_2d_cfg cfg = { + { cos_bit_row, stage_range_row }, { cos_bit_col, stage_range_col }, + }; + return cfg; +} + +void av1_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_4[] = { + { highbd_idct4, highbd_idct4 }, // DCT_DCT + { highbd_iadst4, highbd_idct4 }, // ADST_DCT + { highbd_idct4, highbd_iadst4 }, // DCT_ADST + { highbd_iadst4, highbd_iadst4 }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_iadst4, highbd_idct4 }, // FLIPADST_DCT + { highbd_idct4, highbd_iadst4 }, // DCT_FLIPADST + { highbd_iadst4, highbd_iadst4 }, // FLIPADST_FLIPADST + { highbd_iadst4, highbd_iadst4 }, // ADST_FLIPADST + { highbd_iadst4, highbd_iadst4 }, // FLIPADST_ADST + { highbd_iidtx4_c, highbd_iidtx4_c }, // IDTX + { highbd_idct4, highbd_iidtx4_c }, // V_DCT + { highbd_iidtx4_c, highbd_idct4 }, // H_DCT + { highbd_iadst4, highbd_iidtx4_c }, // V_ADST + { highbd_iidtx4_c, highbd_iadst4 }, // H_ADST + { highbd_iadst4, highbd_iidtx4_c }, // V_FLIPADST + { highbd_iidtx4_c, highbd_iadst4 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t tmp; + tran_low_t out[4][4]; + tran_low_t *outp = &out[0][0]; + int outstride = 4; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_4X4, TX_4X4); + + // inverse transform row vectors + for (i = 0; i < 4; ++i) { + HIGH_IHT_4[tx_type].rows(input, out[i], cfg.row.cos_bit, + cfg.row.stage_range, bd); + input += 4; + } + + // transpose + for (i = 1; i < 4; i++) { + for (j = 0; j < i; j++) { + tmp = out[i][j]; + out[i][j] = out[j][i]; + out[j][i] = tmp; + } + } + + // inverse transform column vectors + for (i = 0; i < 4; ++i) { + HIGH_IHT_4[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + } + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, 4, 4); +#endif + + // Sum with the destination + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 4), bd); + } + } +} + +void av1_highbd_iht4x8_32_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_4x8[] = { + { highbd_idct8, highbd_idct4 }, // DCT_DCT + { highbd_iadst8, highbd_idct4 }, // ADST_DCT + { highbd_idct8, highbd_iadst4 }, // DCT_ADST + { highbd_iadst8, highbd_iadst4 }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_iadst8, highbd_idct4 }, // FLIPADST_DCT + { highbd_idct8, highbd_iadst4 }, // DCT_FLIPADST + { highbd_iadst8, highbd_iadst4 }, // FLIPADST_FLIPADST + { highbd_iadst8, highbd_iadst4 }, // ADST_FLIPADST + { highbd_iadst8, highbd_iadst4 }, // FLIPADST_ADST + { highbd_iidtx8_c, highbd_iidtx4_c }, // IDTX + { highbd_idct8, highbd_iidtx4_c }, // V_DCT + { highbd_iidtx8_c, highbd_idct4 }, // H_DCT + { highbd_iadst8, highbd_iidtx4_c }, // V_ADST + { highbd_iidtx8_c, highbd_iadst4 }, // H_ADST + { highbd_iadst8, highbd_iidtx4_c }, // V_FLIPADST + { highbd_iidtx8_c, highbd_iadst4 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + const int n = 4; + const int n2 = 8; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t out[4][8], outtmp[4]; + tran_low_t *outp = &out[0][0]; + int outstride = n2; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_4X4, TX_8X8); + + // inverse transform row vectors, and transpose + for (i = 0; i < n2; ++i) { + HIGH_IHT_4x8[tx_type].rows(input, outtmp, cfg.row.cos_bit, + cfg.row.stage_range, bd); + for (j = 0; j < n; ++j) { + out[j][i] = HIGHBD_WRAPLOW(dct_const_round_shift(outtmp[j] * Sqrt2), bd); + } + input += n; + } + + // inverse transform column vectors + for (i = 0; i < n; ++i) { + HIGH_IHT_4x8[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + } + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, n2, n); +#endif // CONFIG_EXT_TX + + // Sum with the destination + for (i = 0; i < n2; ++i) { + for (j = 0; j < n; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5), bd); + } + } +} + +void av1_highbd_iht8x4_32_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_8x4[] = { + { highbd_idct4, highbd_idct8 }, // DCT_DCT + { highbd_iadst4, highbd_idct8 }, // ADST_DCT + { highbd_idct4, highbd_iadst8 }, // DCT_ADST + { highbd_iadst4, highbd_iadst8 }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_iadst4, highbd_idct8 }, // FLIPADST_DCT + { highbd_idct4, highbd_iadst8 }, // DCT_FLIPADST + { highbd_iadst4, highbd_iadst8 }, // FLIPADST_FLIPADST + { highbd_iadst4, highbd_iadst8 }, // ADST_FLIPADST + { highbd_iadst4, highbd_iadst8 }, // FLIPADST_ADST + { highbd_iidtx4_c, highbd_iidtx8_c }, // IDTX + { highbd_idct4, highbd_iidtx8_c }, // V_DCT + { highbd_iidtx4_c, highbd_idct8 }, // H_DCT + { highbd_iadst4, highbd_iidtx8_c }, // V_ADST + { highbd_iidtx4_c, highbd_iadst8 }, // H_ADST + { highbd_iadst4, highbd_iidtx8_c }, // V_FLIPADST + { highbd_iidtx4_c, highbd_iadst8 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + const int n = 4; + const int n2 = 8; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t out[8][4], outtmp[8]; + tran_low_t *outp = &out[0][0]; + int outstride = n; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_8X8, TX_4X4); + + // inverse transform row vectors, and transpose + for (i = 0; i < n; ++i) { + HIGH_IHT_8x4[tx_type].rows(input, outtmp, cfg.row.cos_bit, + cfg.row.stage_range, bd); + for (j = 0; j < n2; ++j) { + out[j][i] = HIGHBD_WRAPLOW(dct_const_round_shift(outtmp[j] * Sqrt2), bd); + } + input += n2; + } + + // inverse transform column vectors + for (i = 0; i < n2; ++i) { + HIGH_IHT_8x4[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + } + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, n, n2); +#endif // CONFIG_EXT_TX + + // Sum with the destination + for (i = 0; i < n; ++i) { + for (j = 0; j < n2; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5), bd); + } + } +} + +void av1_highbd_iht4x16_64_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_4x16[] = { + { highbd_idct16, highbd_idct4 }, // DCT_DCT + { highbd_iadst16, highbd_idct4 }, // ADST_DCT + { highbd_idct16, highbd_iadst4 }, // DCT_ADST + { highbd_iadst16, highbd_iadst4 }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_iadst16, highbd_idct4 }, // FLIPADST_DCT + { highbd_idct16, highbd_iadst4 }, // DCT_FLIPADST + { highbd_iadst16, highbd_iadst4 }, // FLIPADST_FLIPADST + { highbd_iadst16, highbd_iadst4 }, // ADST_FLIPADST + { highbd_iadst16, highbd_iadst4 }, // FLIPADST_ADST + { highbd_iidtx16_c, highbd_iidtx4_c }, // IDTX + { highbd_idct16, highbd_iidtx4_c }, // V_DCT + { highbd_iidtx16_c, highbd_idct4 }, // H_DCT + { highbd_iadst16, highbd_iidtx4_c }, // V_ADST + { highbd_iidtx16_c, highbd_iadst4 }, // H_ADST + { highbd_iadst16, highbd_iidtx4_c }, // V_FLIPADST + { highbd_iidtx16_c, highbd_iadst4 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + const int n = 4; + const int n4 = 16; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t out[4][16], outtmp[4]; + tran_low_t *outp = &out[0][0]; + int outstride = n4; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_4X4, TX_16X16); + + // inverse transform row vectors, and transpose + for (i = 0; i < n4; ++i) { + HIGH_IHT_4x16[tx_type].rows(input, outtmp, cfg.row.cos_bit, + cfg.row.stage_range, bd); + for (j = 0; j < n; ++j) out[j][i] = outtmp[j]; + input += n; + } + + // inverse transform column vectors + for (i = 0; i < n; ++i) + HIGH_IHT_4x16[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, n4, n); +#endif // CONFIG_EXT_TX + + // Sum with the destination + for (i = 0; i < n4; ++i) { + for (j = 0; j < n; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5), bd); + } + } +} + +void av1_highbd_iht16x4_64_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_16x4[] = { + { highbd_idct4, highbd_idct16 }, // DCT_DCT + { highbd_iadst4, highbd_idct16 }, // ADST_DCT + { highbd_idct4, highbd_iadst16 }, // DCT_ADST + { highbd_iadst4, highbd_iadst16 }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_iadst4, highbd_idct16 }, // FLIPADST_DCT + { highbd_idct4, highbd_iadst16 }, // DCT_FLIPADST + { highbd_iadst4, highbd_iadst16 }, // FLIPADST_FLIPADST + { highbd_iadst4, highbd_iadst16 }, // ADST_FLIPADST + { highbd_iadst4, highbd_iadst16 }, // FLIPADST_ADST + { highbd_iidtx4_c, highbd_iidtx16_c }, // IDTX + { highbd_idct4, highbd_iidtx16_c }, // V_DCT + { highbd_iidtx4_c, highbd_idct16 }, // H_DCT + { highbd_iadst4, highbd_iidtx16_c }, // V_ADST + { highbd_iidtx4_c, highbd_iadst16 }, // H_ADST + { highbd_iadst4, highbd_iidtx16_c }, // V_FLIPADST + { highbd_iidtx4_c, highbd_iadst16 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + const int n = 4; + const int n4 = 16; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t out[16][4], outtmp[16]; + tran_low_t *outp = &out[0][0]; + int outstride = n; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_16X16, TX_4X4); + + // inverse transform row vectors, and transpose + for (i = 0; i < n; ++i) { + HIGH_IHT_16x4[tx_type].rows(input, outtmp, cfg.row.cos_bit, + cfg.row.stage_range, bd); + for (j = 0; j < n4; ++j) out[j][i] = outtmp[j]; + input += n4; + } + + // inverse transform column vectors + for (i = 0; i < n4; ++i) { + HIGH_IHT_16x4[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + } + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, n, n4); +#endif // CONFIG_EXT_TX + + // Sum with the destination + for (i = 0; i < n; ++i) { + for (j = 0; j < n4; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5), bd); + } + } +} + +void av1_highbd_iht8x16_128_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_8x16[] = { + { highbd_idct16, highbd_idct8 }, // DCT_DCT + { highbd_iadst16, highbd_idct8 }, // ADST_DCT + { highbd_idct16, highbd_iadst8 }, // DCT_ADST + { highbd_iadst16, highbd_iadst8 }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_iadst16, highbd_idct8 }, // FLIPADST_DCT + { highbd_idct16, highbd_iadst8 }, // DCT_FLIPADST + { highbd_iadst16, highbd_iadst8 }, // FLIPADST_FLIPADST + { highbd_iadst16, highbd_iadst8 }, // ADST_FLIPADST + { highbd_iadst16, highbd_iadst8 }, // FLIPADST_ADST + { highbd_iidtx16_c, highbd_iidtx8_c }, // IDTX + { highbd_idct16, highbd_iidtx8_c }, // V_DCT + { highbd_iidtx16_c, highbd_idct8 }, // H_DCT + { highbd_iadst16, highbd_iidtx8_c }, // V_ADST + { highbd_iidtx16_c, highbd_iadst8 }, // H_ADST + { highbd_iadst16, highbd_iidtx8_c }, // V_FLIPADST + { highbd_iidtx16_c, highbd_iadst8 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + const int n = 8; + const int n2 = 16; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t out[8][16], outtmp[8]; + tran_low_t *outp = &out[0][0]; + int outstride = n2; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_8X8, TX_16X16); + + // inverse transform row vectors, and transpose + for (i = 0; i < n2; ++i) { + HIGH_IHT_8x16[tx_type].rows(input, outtmp, cfg.row.cos_bit, + cfg.row.stage_range, bd); + for (j = 0; j < n; ++j) + out[j][i] = HIGHBD_WRAPLOW(dct_const_round_shift(outtmp[j] * Sqrt2), bd); + input += n; + } + + // inverse transform column vectors + for (i = 0; i < n; ++i) { + HIGH_IHT_8x16[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + } + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, n2, n); +#endif // CONFIG_EXT_TX + + // Sum with the destination + for (i = 0; i < n2; ++i) { + for (j = 0; j < n; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6), bd); + } + } +} + +void av1_highbd_iht16x8_128_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_16x8[] = { + { highbd_idct8, highbd_idct16 }, // DCT_DCT + { highbd_iadst8, highbd_idct16 }, // ADST_DCT + { highbd_idct8, highbd_iadst16 }, // DCT_ADST + { highbd_iadst8, highbd_iadst16 }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_iadst8, highbd_idct16 }, // FLIPADST_DCT + { highbd_idct8, highbd_iadst16 }, // DCT_FLIPADST + { highbd_iadst8, highbd_iadst16 }, // FLIPADST_FLIPADST + { highbd_iadst8, highbd_iadst16 }, // ADST_FLIPADST + { highbd_iadst8, highbd_iadst16 }, // FLIPADST_ADST + { highbd_iidtx8_c, highbd_iidtx16_c }, // IDTX + { highbd_idct8, highbd_iidtx16_c }, // V_DCT + { highbd_iidtx8_c, highbd_idct16 }, // H_DCT + { highbd_iadst8, highbd_iidtx16_c }, // V_ADST + { highbd_iidtx8_c, highbd_iadst16 }, // H_ADST + { highbd_iadst8, highbd_iidtx16_c }, // V_FLIPADST + { highbd_iidtx8_c, highbd_iadst16 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + const int n = 8; + const int n2 = 16; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t out[16][8], outtmp[16]; + tran_low_t *outp = &out[0][0]; + int outstride = n; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_16X16, TX_8X8); + + // inverse transform row vectors, and transpose + for (i = 0; i < n; ++i) { + HIGH_IHT_16x8[tx_type].rows(input, outtmp, cfg.row.cos_bit, + cfg.row.stage_range, bd); + for (j = 0; j < n2; ++j) + out[j][i] = HIGHBD_WRAPLOW(dct_const_round_shift(outtmp[j] * Sqrt2), bd); + input += n2; + } + + // inverse transform column vectors + for (i = 0; i < n2; ++i) { + HIGH_IHT_16x8[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + } + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, n, n2); +#endif // CONFIG_EXT_TX + + // Sum with the destination + for (i = 0; i < n; ++i) { + for (j = 0; j < n2; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6), bd); + } + } +} + +void av1_highbd_iht8x32_256_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_8x32[] = { + { highbd_idct32, highbd_idct8 }, // DCT_DCT + { highbd_ihalfright32_c, highbd_idct8 }, // ADST_DCT + { highbd_idct32, highbd_iadst8 }, // DCT_ADST + { highbd_ihalfright32_c, highbd_iadst8 }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_ihalfright32_c, highbd_idct8 }, // FLIPADST_DCT + { highbd_idct32, highbd_iadst8 }, // DCT_FLIPADST + { highbd_ihalfright32_c, highbd_iadst8 }, // FLIPADST_FLIPADST + { highbd_ihalfright32_c, highbd_iadst8 }, // ADST_FLIPADST + { highbd_ihalfright32_c, highbd_iadst8 }, // FLIPADST_ADST + { highbd_iidtx32_c, highbd_iidtx8_c }, // IDTX + { highbd_idct32, highbd_iidtx8_c }, // V_DCT + { highbd_iidtx32_c, highbd_idct8 }, // H_DCT + { highbd_ihalfright32_c, highbd_iidtx8_c }, // V_ADST + { highbd_iidtx32_c, highbd_iadst8 }, // H_ADST + { highbd_ihalfright32_c, highbd_iidtx8_c }, // V_FLIPADST + { highbd_iidtx32_c, highbd_iadst8 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + const int n = 8; + const int n4 = 32; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t out[8][32], outtmp[8]; + tran_low_t *outp = &out[0][0]; + int outstride = n4; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_8X8, TX_32X32); + + // inverse transform row vectors, and transpose + for (i = 0; i < n4; ++i) { + HIGH_IHT_8x32[tx_type].rows(input, outtmp, cfg.row.cos_bit, + cfg.row.stage_range, bd); + for (j = 0; j < n; ++j) out[j][i] = outtmp[j]; + input += n; + } + + // inverse transform column vectors + for (i = 0; i < n; ++i) + HIGH_IHT_8x32[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, n4, n); +#endif // CONFIG_EXT_TX + + // Sum with the destination + for (i = 0; i < n4; ++i) { + for (j = 0; j < n; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6), bd); + } + } +} + +void av1_highbd_iht32x8_256_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_32x8[] = { + { highbd_idct8, highbd_idct32 }, // DCT_DCT + { highbd_iadst8, highbd_idct32 }, // ADST_DCT + { highbd_idct8, highbd_ihalfright32_c }, // DCT_ADST + { highbd_iadst8, highbd_ihalfright32_c }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_iadst8, highbd_idct32 }, // FLIPADST_DCT + { highbd_idct8, highbd_ihalfright32_c }, // DCT_FLIPADST + { highbd_iadst8, highbd_ihalfright32_c }, // FLIPADST_FLIPADST + { highbd_iadst8, highbd_ihalfright32_c }, // ADST_FLIPADST + { highbd_iadst8, highbd_ihalfright32_c }, // FLIPADST_ADST + { highbd_iidtx8_c, highbd_iidtx32_c }, // IDTX + { highbd_idct8, highbd_iidtx32_c }, // V_DCT + { highbd_iidtx8_c, highbd_idct32 }, // H_DCT + { highbd_iadst8, highbd_iidtx32_c }, // V_ADST + { highbd_iidtx8_c, highbd_ihalfright32_c }, // H_ADST + { highbd_iadst8, highbd_iidtx32_c }, // V_FLIPADST + { highbd_iidtx8_c, highbd_ihalfright32_c }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + const int n = 8; + const int n4 = 32; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t out[32][8], outtmp[32]; + tran_low_t *outp = &out[0][0]; + int outstride = n; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_32X32, TX_8X8); + + // inverse transform row vectors, and transpose + for (i = 0; i < n; ++i) { + HIGH_IHT_32x8[tx_type].rows(input, outtmp, cfg.row.cos_bit, + cfg.row.stage_range, bd); + for (j = 0; j < n4; ++j) out[j][i] = outtmp[j]; + input += n4; + } + + // inverse transform column vectors + for (i = 0; i < n4; ++i) + HIGH_IHT_32x8[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, n, n4); +#endif // CONFIG_EXT_TX + + // Sum with the destination + for (i = 0; i < n; ++i) { + for (j = 0; j < n4; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6), bd); + } + } +} + +void av1_highbd_iht16x32_512_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_16x32[] = { + { highbd_idct32, highbd_idct16 }, // DCT_DCT + { highbd_ihalfright32_c, highbd_idct16 }, // ADST_DCT + { highbd_idct32, highbd_iadst16 }, // DCT_ADST + { highbd_ihalfright32_c, highbd_iadst16 }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_ihalfright32_c, highbd_idct16 }, // FLIPADST_DCT + { highbd_idct32, highbd_iadst16 }, // DCT_FLIPADST + { highbd_ihalfright32_c, highbd_iadst16 }, // FLIPADST_FLIPADST + { highbd_ihalfright32_c, highbd_iadst16 }, // ADST_FLIPADST + { highbd_ihalfright32_c, highbd_iadst16 }, // FLIPADST_ADST + { highbd_iidtx32_c, highbd_iidtx16_c }, // IDTX + { highbd_idct32, highbd_iidtx16_c }, // V_DCT + { highbd_iidtx32_c, highbd_idct16 }, // H_DCT + { highbd_ihalfright32_c, highbd_iidtx16_c }, // V_ADST + { highbd_iidtx32_c, highbd_iadst16 }, // H_ADST + { highbd_ihalfright32_c, highbd_iidtx16_c }, // V_FLIPADST + { highbd_iidtx32_c, highbd_iadst16 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + const int n = 16; + const int n2 = 32; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t out[16][32], outtmp[16]; + tran_low_t *outp = &out[0][0]; + int outstride = n2; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_16X16, TX_32X32); + + // inverse transform row vectors, and transpose + for (i = 0; i < n2; ++i) { + HIGH_IHT_16x32[tx_type].rows(input, outtmp, cfg.row.cos_bit, + cfg.row.stage_range, bd); + for (j = 0; j < n; ++j) + out[j][i] = HIGHBD_WRAPLOW(dct_const_round_shift(outtmp[j] * Sqrt2), bd); + input += n; + } + + // inverse transform column vectors + for (i = 0; i < n; ++i) { + HIGH_IHT_16x32[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + } + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, n2, n); +#endif // CONFIG_EXT_TX + + // Sum with the destination + for (i = 0; i < n2; ++i) { + for (j = 0; j < n; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6), bd); + } + } +} + +void av1_highbd_iht32x16_512_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_32x16[] = { + { highbd_idct16, highbd_idct32 }, // DCT_DCT + { highbd_iadst16, highbd_idct32 }, // ADST_DCT + { highbd_idct16, highbd_ihalfright32_c }, // DCT_ADST + { highbd_iadst16, highbd_ihalfright32_c }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_iadst16, highbd_idct32 }, // FLIPADST_DCT + { highbd_idct16, highbd_ihalfright32_c }, // DCT_FLIPADST + { highbd_iadst16, highbd_ihalfright32_c }, // FLIPADST_FLIPADST + { highbd_iadst16, highbd_ihalfright32_c }, // ADST_FLIPADST + { highbd_iadst16, highbd_ihalfright32_c }, // FLIPADST_ADST + { highbd_iidtx16_c, highbd_iidtx32_c }, // IDTX + { highbd_idct16, highbd_iidtx32_c }, // V_DCT + { highbd_iidtx16_c, highbd_idct32 }, // H_DCT + { highbd_iadst16, highbd_iidtx32_c }, // V_ADST + { highbd_iidtx16_c, highbd_ihalfright32_c }, // H_ADST + { highbd_iadst16, highbd_iidtx32_c }, // V_FLIPADST + { highbd_iidtx16_c, highbd_ihalfright32_c }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + const int n = 16; + const int n2 = 32; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t out[32][16], outtmp[32]; + tran_low_t *outp = &out[0][0]; + int outstride = n; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_32X32, TX_16X16); + + // inverse transform row vectors, and transpose + for (i = 0; i < n; ++i) { + HIGH_IHT_32x16[tx_type].rows(input, outtmp, cfg.row.cos_bit, + cfg.row.stage_range, bd); + for (j = 0; j < n2; ++j) + out[j][i] = HIGHBD_WRAPLOW(dct_const_round_shift(outtmp[j] * Sqrt2), bd); + input += n2; + } + + // inverse transform column vectors + for (i = 0; i < n2; ++i) { + HIGH_IHT_32x16[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + } + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, n, n2); +#endif // CONFIG_EXT_TX + + // Sum with the destination + for (i = 0; i < n; ++i) { + for (j = 0; j < n2; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6), bd); + } + } +} + +void av1_highbd_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_8[] = { + { highbd_idct8, highbd_idct8 }, // DCT_DCT + { highbd_iadst8, highbd_idct8 }, // ADST_DCT + { highbd_idct8, highbd_iadst8 }, // DCT_ADST + { highbd_iadst8, highbd_iadst8 }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_iadst8, highbd_idct8 }, // FLIPADST_DCT + { highbd_idct8, highbd_iadst8 }, // DCT_FLIPADST + { highbd_iadst8, highbd_iadst8 }, // FLIPADST_FLIPADST + { highbd_iadst8, highbd_iadst8 }, // ADST_FLIPADST + { highbd_iadst8, highbd_iadst8 }, // FLIPADST_ADST + { highbd_iidtx8_c, highbd_iidtx8_c }, // IDTX + { highbd_idct8, highbd_iidtx8_c }, // V_DCT + { highbd_iidtx8_c, highbd_idct8 }, // H_DCT + { highbd_iadst8, highbd_iidtx8_c }, // V_ADST + { highbd_iidtx8_c, highbd_iadst8 }, // H_ADST + { highbd_iadst8, highbd_iidtx8_c }, // V_FLIPADST + { highbd_iidtx8_c, highbd_iadst8 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t tmp; + tran_low_t out[8][8]; + tran_low_t *outp = &out[0][0]; + int outstride = 8; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_8X8, TX_8X8); + + // inverse transform row vectors + for (i = 0; i < 8; ++i) { + HIGH_IHT_8[tx_type].rows(input, out[i], cfg.row.cos_bit, + cfg.row.stage_range, bd); + input += 8; + } + + // transpose + for (i = 1; i < 8; i++) { + for (j = 0; j < i; j++) { + tmp = out[i][j]; + out[i][j] = out[j][i]; + out[j][i] = tmp; + } + } + + // inverse transform column vectors + for (i = 0; i < 8; ++i) { + HIGH_IHT_8[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + } + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, 8, 8); +#endif + + // Sum with the destination + for (i = 0; i < 8; ++i) { + for (j = 0; j < 8; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5), bd); + } + } +} + +void av1_highbd_iht16x16_256_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_16[] = { + { highbd_idct16, highbd_idct16 }, // DCT_DCT + { highbd_iadst16, highbd_idct16 }, // ADST_DCT + { highbd_idct16, highbd_iadst16 }, // DCT_ADST + { highbd_iadst16, highbd_iadst16 }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_iadst16, highbd_idct16 }, // FLIPADST_DCT + { highbd_idct16, highbd_iadst16 }, // DCT_FLIPADST + { highbd_iadst16, highbd_iadst16 }, // FLIPADST_FLIPADST + { highbd_iadst16, highbd_iadst16 }, // ADST_FLIPADST + { highbd_iadst16, highbd_iadst16 }, // FLIPADST_ADST + { highbd_iidtx16_c, highbd_iidtx16_c }, // IDTX + { highbd_idct16, highbd_iidtx16_c }, // V_DCT + { highbd_iidtx16_c, highbd_idct16 }, // H_DCT + { highbd_iadst16, highbd_iidtx16_c }, // V_ADST + { highbd_iidtx16_c, highbd_iadst16 }, // H_ADST + { highbd_iadst16, highbd_iidtx16_c }, // V_FLIPADST + { highbd_iidtx16_c, highbd_iadst16 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t tmp; + tran_low_t out[16][16]; + tran_low_t *outp = &out[0][0]; + int outstride = 16; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_16X16, TX_16X16); + + // inverse transform row vectors + for (i = 0; i < 16; ++i) { + HIGH_IHT_16[tx_type].rows(input, out[i], cfg.row.cos_bit, + cfg.row.stage_range, bd); + input += 16; + } + + // transpose + for (i = 1; i < 16; i++) { + for (j = 0; j < i; j++) { + tmp = out[i][j]; + out[i][j] = out[j][i]; + out[j][i] = tmp; + } + } + + // inverse transform column vectors + for (i = 0; i < 16; ++i) { + HIGH_IHT_16[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + } + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, 16, 16); +#endif + + // Sum with the destination + for (i = 0; i < 16; ++i) { + for (j = 0; j < 16; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6), bd); + } + } +} + +#if CONFIG_EXT_TX +static void highbd_iht32x32_1024_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_32[] = { + { highbd_idct32, highbd_idct32 }, // DCT_DCT + { highbd_ihalfright32_c, highbd_idct32 }, // ADST_DCT + { highbd_idct32, highbd_ihalfright32_c }, // DCT_ADST + { highbd_ihalfright32_c, highbd_ihalfright32_c }, // ADST_ADST + { highbd_ihalfright32_c, highbd_idct32 }, // FLIPADST_DCT + { highbd_idct32, highbd_ihalfright32_c }, // DCT_FLIPADST + { highbd_ihalfright32_c, highbd_ihalfright32_c }, // FLIPADST_FLIPADST + { highbd_ihalfright32_c, highbd_ihalfright32_c }, // ADST_FLIPADST + { highbd_ihalfright32_c, highbd_ihalfright32_c }, // FLIPADST_ADST + { highbd_iidtx32_c, highbd_iidtx32_c }, // IDTX + { highbd_idct32, highbd_iidtx32_c }, // V_DCT + { highbd_iidtx32_c, highbd_idct32 }, // H_DCT + { highbd_ihalfright32_c, highbd_iidtx32_c }, // V_ADST + { highbd_iidtx32_c, highbd_ihalfright32_c }, // H_ADST + { highbd_ihalfright32_c, highbd_iidtx32_c }, // V_FLIPADST + { highbd_iidtx32_c, highbd_ihalfright32_c }, // H_FLIPADST + }; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t tmp; + tran_low_t out[32][32]; + tran_low_t *outp = &out[0][0]; + int outstride = 32; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_32X32, TX_32X32); + + // inverse transform row vectors + for (i = 0; i < 32; ++i) { + HIGH_IHT_32[tx_type].rows(input, out[i], cfg.row.cos_bit, + cfg.row.stage_range, bd); + input += 32; + } + + // transpose + for (i = 1; i < 32; i++) { + for (j = 0; j < i; j++) { + tmp = out[i][j]; + out[i][j] = out[j][i]; + out[j][i] = tmp; + } + } + + // inverse transform column vectors + for (i = 0; i < 32; ++i) { + HIGH_IHT_32[tx_type].cols(out[i], out[i], cfg.col.cos_bit, + cfg.col.stage_range, bd); + } + + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, 32, 32); + + // Sum with the destination + for (i = 0; i < 32; ++i) { + for (j = 0; j < 32; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6), bd); + } + } +} +#endif // CONFIG_EXT_TX + +#if CONFIG_TX64X64 +static void highbd_iht64x64_4096_add_c(const tran_low_t *input, uint8_t *dest8, + int stride, int tx_type, int bd) { + static const highbd_transform_2d HIGH_IHT_64[] = { + { highbd_idct64_col_c, highbd_idct64_row_c }, // DCT_DCT + { highbd_ihalfright64_c, highbd_idct64_row_c }, // ADST_DCT + { highbd_idct64_col_c, highbd_ihalfright64_c }, // DCT_ADST + { highbd_ihalfright64_c, highbd_ihalfright64_c }, // ADST_ADST +#if CONFIG_EXT_TX + { highbd_ihalfright64_c, highbd_idct64_row_c }, // FLIPADST_DCT + { highbd_idct64_col_c, highbd_ihalfright64_c }, // DCT_FLIPADST + { highbd_ihalfright64_c, highbd_ihalfright64_c }, // FLIPADST_FLIPADST + { highbd_ihalfright64_c, highbd_ihalfright64_c }, // ADST_FLIPADST + { highbd_ihalfright64_c, highbd_ihalfright64_c }, // FLIPADST_ADST + { highbd_iidtx64_c, highbd_iidtx64_c }, // IDTX + { highbd_idct64_col_c, highbd_iidtx64_c }, // V_DCT + { highbd_iidtx64_c, highbd_idct64_row_c }, // H_DCT + { highbd_ihalfright64_c, highbd_iidtx64_c }, // V_ADST + { highbd_iidtx64_c, highbd_ihalfright64_c }, // H_ADST + { highbd_ihalfright64_c, highbd_iidtx64_c }, // V_FLIPADST + { highbd_iidtx64_c, highbd_ihalfright64_c }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + + uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); + + int i, j; + tran_low_t tmp; + tran_low_t out[64][64]; + tran_low_t *outp = &out[0][0]; + int outstride = 64; + + tx_2d_cfg cfg = inv_tx_cfg(tx_type, TX_64X64, TX_64X64); + + // inverse transform row vectors + for (i = 0; i < 64; ++i) { + HIGH_IHT_64[tx_type].rows(input, out[i], cfg.row.cos_bit, + cfg.row.stage_range, bd); + for (j = 0; j < 64; ++j) out[i][j] = ROUND_POWER_OF_TWO(out[i][j], 1); + input += 64; + } + + // transpose + for (i = 1; i < 64; i++) { + for (j = 0; j < i; j++) { + tmp = out[i][j]; + out[i][j] = out[j][i]; + out[j][i] = tmp; + } + } + + // inverse transform column vectors + for (i = 0; i < 64; ++i) { + HIGH_IHT_64[tx_type].cols(out[i], out[i], cfg.col.cos_bit_col, + cfg.col.stage_range, bd); + } + +#if CONFIG_EXT_TX + maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, 64, 64); +#endif // CONFIG_EXT_TX + + // Sum with the destination + for (i = 0; i < 64; ++i) { + for (j = 0; j < 64; ++j) { + int d = i * stride + j; + int s = j * outstride + i; + dest[d] = + highbd_clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5), bd); + } + } +} +#endif // CONFIG_TX64X64 + +// idct +void av1_highbd_idct4x4_add(const tran_low_t *input, uint8_t *dest, int stride, + int eob, int bd) { + if (eob > 1) + aom_highbd_idct4x4_16_add(input, dest, stride, bd); + else + aom_highbd_idct4x4_1_add(input, dest, stride, bd); +} + +void av1_highbd_iwht4x4_add(const tran_low_t *input, uint8_t *dest, int stride, + int eob, int bd) { + if (eob > 1) + aom_highbd_iwht4x4_16_add(input, dest, stride, bd); + else + aom_highbd_iwht4x4_1_add(input, dest, stride, bd); +} + +#if CONFIG_CB4X4 +static void highbd_inv_txfm_add_2x2(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type, int lossless) { + tran_high_t a1 = input[0] >> UNIT_QUANT_SHIFT; + tran_high_t b1 = input[1] >> UNIT_QUANT_SHIFT; + tran_high_t c1 = input[2] >> UNIT_QUANT_SHIFT; + tran_high_t d1 = input[3] >> UNIT_QUANT_SHIFT; + + tran_high_t a2 = a1 + c1; + tran_high_t b2 = b1 + d1; + tran_high_t c2 = a1 - c1; + tran_high_t d2 = b1 - d1; + + uint16_t *dst = CONVERT_TO_SHORTPTR(dest); + + (void)tx_type; + (void)lossless; + (void)eob; + + a1 = (a2 + b2) >> 2; + b1 = (a2 - b2) >> 2; + c1 = (c2 + d2) >> 2; + d1 = (c2 - d2) >> 2; + + dst[0] = highbd_clip_pixel_add(dst[0], a1, bd); + dst[1] = highbd_clip_pixel_add(dst[1], b1, bd); + dst[stride] = highbd_clip_pixel_add(dst[stride], c1, bd); + dst[stride + 1] = highbd_clip_pixel_add(dst[stride + 1], d1, bd); +} +#endif + +void av1_highbd_inv_txfm_add_4x4(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, TX_TYPE tx_type, + int lossless) { + if (lossless) { + assert(tx_type == DCT_DCT); + av1_highbd_iwht4x4_add(input, dest, stride, eob, bd); + return; + } + + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: +#endif // CONFIG_EXT_TX + av1_inv_txfm2d_add_4x4(input, CONVERT_TO_SHORTPTR(dest), stride, tx_type, + bd); + break; +#if CONFIG_EXT_TX + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + // Use C version since DST only exists in C code + av1_highbd_iht4x4_16_add_c(input, dest, stride, tx_type, bd); + break; + case IDTX: + highbd_inv_idtx_add_c(input, dest, stride, 4, tx_type, bd); + break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} + +void av1_highbd_inv_txfm_add_4x8(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, TX_TYPE tx_type) { + (void)eob; + av1_highbd_iht4x8_32_add_c(input, dest, stride, tx_type, bd); +} + +void av1_highbd_inv_txfm_add_8x4(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, TX_TYPE tx_type) { + (void)eob; + av1_highbd_iht8x4_32_add_c(input, dest, stride, tx_type, bd); +} + +void av1_highbd_inv_txfm_add_4x16(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type) { + (void)eob; + av1_highbd_iht4x16_64_add_c(input, dest, stride, tx_type, bd); +} + +void av1_highbd_inv_txfm_add_16x4(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type) { + (void)eob; + av1_highbd_iht16x4_64_add_c(input, dest, stride, tx_type, bd); +} + +static void highbd_inv_txfm_add_8x16(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type) { + (void)eob; + av1_highbd_iht8x16_128_add_c(input, dest, stride, tx_type, bd); +} + +static void highbd_inv_txfm_add_16x8(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type) { + (void)eob; + av1_highbd_iht16x8_128_add_c(input, dest, stride, tx_type, bd); +} + +void av1_highbd_inv_txfm_add_8x32(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type) { + (void)eob; + av1_highbd_iht8x32_256_add_c(input, dest, stride, tx_type, bd); +} + +void av1_highbd_inv_txfm_add_32x8(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type) { + (void)eob; + av1_highbd_iht32x8_256_add_c(input, dest, stride, tx_type, bd); +} + +static void highbd_inv_txfm_add_16x32(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type) { + (void)eob; + av1_highbd_iht16x32_512_add_c(input, dest, stride, tx_type, bd); +} + +static void highbd_inv_txfm_add_32x16(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type) { + (void)eob; + av1_highbd_iht32x16_512_add_c(input, dest, stride, tx_type, bd); +} + +static void highbd_inv_txfm_add_8x8(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type) { + (void)eob; + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: +#endif // CONFIG_EXT_TX + av1_inv_txfm2d_add_8x8(input, CONVERT_TO_SHORTPTR(dest), stride, tx_type, + bd); + break; +#if CONFIG_EXT_TX + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + // Use C version since DST only exists in C code + av1_highbd_iht8x8_64_add_c(input, dest, stride, tx_type, bd); + break; + case IDTX: + highbd_inv_idtx_add_c(input, dest, stride, 8, tx_type, bd); + break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} + +static void highbd_inv_txfm_add_16x16(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type) { + (void)eob; + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: +#endif // CONFIG_EXT_TX + av1_inv_txfm2d_add_16x16(input, CONVERT_TO_SHORTPTR(dest), stride, + tx_type, bd); + break; +#if CONFIG_EXT_TX + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + // Use C version since DST only exists in C code + av1_highbd_iht16x16_256_add_c(input, dest, stride, tx_type, bd); + break; + case IDTX: + highbd_inv_idtx_add_c(input, dest, stride, 16, tx_type, bd); + break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} + +static void highbd_inv_txfm_add_32x32(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type) { + (void)eob; + switch (tx_type) { + case DCT_DCT: + av1_inv_txfm2d_add_32x32(input, CONVERT_TO_SHORTPTR(dest), stride, + DCT_DCT, bd); + break; +#if CONFIG_EXT_TX + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + highbd_iht32x32_1024_add_c(input, dest, stride, tx_type, bd); + break; + case IDTX: + highbd_inv_idtx_add_c(input, dest, stride, 32, tx_type, bd); + break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} + +#if CONFIG_TX64X64 +static void highbd_inv_txfm_add_64x64(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, + TX_TYPE tx_type) { + (void)eob; + switch (tx_type) { + case DCT_DCT: + av1_inv_txfm2d_add_64x64(input, CONVERT_TO_SHORTPTR(dest), stride, + DCT_DCT, bd); + break; +#if CONFIG_EXT_TX + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + highbd_iht64x64_4096_add_c(input, dest, stride, tx_type, bd); + break; + case IDTX: + highbd_inv_idtx_add_c(input, dest, stride, 64, tx_type, bd); + break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} +#endif // CONFIG_TX64X64 +#endif // CONFIG_HIGHBITDEPTH + +void av1_inv_txfm_add(const tran_low_t *input, uint8_t *dest, int stride, + INV_TXFM_PARAM *inv_txfm_param) { + const TX_TYPE tx_type = inv_txfm_param->tx_type; + const TX_SIZE tx_size = inv_txfm_param->tx_size; + const int eob = inv_txfm_param->eob; + const int lossless = inv_txfm_param->lossless; + + switch (tx_size) { +#if CONFIG_TX64X64 + case TX_64X64: inv_txfm_add_64x64(input, dest, stride, eob, tx_type); break; +#endif // CONFIG_TX64X64 + case TX_32X32: inv_txfm_add_32x32(input, dest, stride, eob, tx_type); break; + case TX_16X16: inv_txfm_add_16x16(input, dest, stride, eob, tx_type); break; + case TX_8X8: inv_txfm_add_8x8(input, dest, stride, eob, tx_type); break; + case TX_4X8: av1_inv_txfm_add_4x8(input, dest, stride, eob, tx_type); break; + case TX_8X4: av1_inv_txfm_add_8x4(input, dest, stride, eob, tx_type); break; + case TX_8X16: inv_txfm_add_8x16(input, dest, stride, eob, tx_type); break; + case TX_16X8: inv_txfm_add_16x8(input, dest, stride, eob, tx_type); break; + case TX_16X32: inv_txfm_add_16x32(input, dest, stride, eob, tx_type); break; + case TX_32X16: inv_txfm_add_32x16(input, dest, stride, eob, tx_type); break; + case TX_4X4: + // this is like av1_short_idct4x4 but has a special case around eob<=1 + // which is significant (not just an optimization) for the lossless + // case. + av1_inv_txfm_add_4x4(input, dest, stride, eob, tx_type, lossless); + break; +#if CONFIG_CB4X4 + case TX_2X2: + inv_txfm_add_2x2(input, dest, stride, eob, tx_type, lossless); + break; +#endif + default: assert(0 && "Invalid transform size"); break; + } +} + +static void init_inv_txfm_param(const MACROBLOCKD *xd, TX_SIZE tx_size, + TX_TYPE tx_type, int eob, INV_TXFM_PARAM *inv) { + inv->tx_type = tx_type; + inv->tx_size = tx_size; + inv->eob = eob; + inv->lossless = xd->lossless[xd->mi[0]->mbmi.segment_id]; +#if CONFIG_HIGHBITDEPTH + inv->bd = xd->bd; +#endif +#if CONFIG_ADAPT_SCAN + inv->eob_threshold = &xd->eob_threshold_md[tx_size][tx_type][0]; +#endif +} + +void av1_inverse_transform_block(const MACROBLOCKD *xd, + const tran_low_t *dqcoeff, TX_TYPE tx_type, + TX_SIZE tx_size, uint8_t *dst, int stride, + int eob) { + if (!eob) return; +#if CONFIG_PVQ + const BLOCK_SIZE tx_bsize = txsize_to_bsize[tx_size]; + const int txb_width = block_size_wide[tx_bsize]; + const int txb_height = block_size_high[tx_bsize]; + int r, c; +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst); + for (r = 0; r < txb_height; r++) + for (c = 0; c < txb_width; c++) + CONVERT_TO_SHORTPTR(dst)[r * stride + c] = 0; + } else { +#endif // CONFIG_HIGHBITDEPTH + for (r = 0; r < txb_height; r++) + for (c = 0; c < txb_width; c++) dst[r * stride + c] = 0; +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_PVQ + INV_TXFM_PARAM inv_txfm_param; + init_inv_txfm_param(xd, tx_size, tx_type, eob, &inv_txfm_param); + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + av1_highbd_inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param); + } else { +#endif // CONFIG_HIGHBITDEPTH + av1_inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param); +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH +} + +void av1_inverse_transform_block_facade(MACROBLOCKD *xd, int plane, int block, + int blk_row, int blk_col, int eob) { + struct macroblockd_plane *const pd = &xd->plane[plane]; + tran_low_t *dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); + const PLANE_TYPE plane_type = get_plane_type(plane); + const TX_SIZE tx_size = get_tx_size(plane, xd); + const TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + const int dst_stride = pd->dst.stride; + uint8_t *dst = + &pd->dst.buf[(blk_row * dst_stride + blk_col) << tx_size_wide_log2[0]]; + av1_inverse_transform_block(xd, dqcoeff, tx_type, tx_size, dst, dst_stride, + eob); +} + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_inv_txfm_add(const tran_low_t *input, uint8_t *dest, int stride, + INV_TXFM_PARAM *inv_txfm_param) { + const TX_TYPE tx_type = inv_txfm_param->tx_type; + const TX_SIZE tx_size = inv_txfm_param->tx_size; + const int eob = inv_txfm_param->eob; + const int bd = inv_txfm_param->bd; + const int lossless = inv_txfm_param->lossless; + + switch (tx_size) { +#if CONFIG_TX64X64 + case TX_64X64: + highbd_inv_txfm_add_64x64(input, dest, stride, eob, bd, tx_type); + break; +#endif // CONFIG_TX64X64 + case TX_32X32: + highbd_inv_txfm_add_32x32(input, dest, stride, eob, bd, tx_type); + break; + case TX_16X16: + highbd_inv_txfm_add_16x16(input, dest, stride, eob, bd, tx_type); + break; + case TX_8X8: + highbd_inv_txfm_add_8x8(input, dest, stride, eob, bd, tx_type); + break; + case TX_4X8: + av1_highbd_inv_txfm_add_4x8(input, dest, stride, eob, bd, tx_type); + break; + case TX_8X4: + av1_highbd_inv_txfm_add_8x4(input, dest, stride, eob, bd, tx_type); + break; + case TX_8X16: + highbd_inv_txfm_add_8x16(input, dest, stride, eob, bd, tx_type); + break; + case TX_16X8: + highbd_inv_txfm_add_16x8(input, dest, stride, eob, bd, tx_type); + break; + case TX_16X32: + highbd_inv_txfm_add_16x32(input, dest, stride, eob, bd, tx_type); + break; + case TX_32X16: + highbd_inv_txfm_add_32x16(input, dest, stride, eob, bd, tx_type); + break; + case TX_4X4: + // this is like av1_short_idct4x4 but has a special case around eob<=1 + // which is significant (not just an optimization) for the lossless + // case. + av1_highbd_inv_txfm_add_4x4(input, dest, stride, eob, bd, tx_type, + lossless); + break; +#if CONFIG_CB4X4 + case TX_2X2: + highbd_inv_txfm_add_2x2(input, dest, stride, eob, bd, tx_type, lossless); + break; +#endif + default: assert(0 && "Invalid transform size"); break; + } +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/av1/common/idct.h b/third_party/aom/av1/common/idct.h new file mode 100644 index 0000000000..e3a192187f --- /dev/null +++ b/third_party/aom/av1/common/idct.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_IDCT_H_ +#define AV1_COMMON_IDCT_H_ + +#include + +#include "./aom_config.h" +#include "av1/common/blockd.h" +#include "av1/common/common.h" +#include "av1/common/enums.h" +#include "aom_dsp/inv_txfm.h" +#include "aom_dsp/txfm_common.h" +#include "aom_ports/mem.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct INV_TXFM_PARAM { +#if CONFIG_ADAPT_SCAN + const int16_t *eob_threshold; +#endif + TX_TYPE tx_type; + TX_SIZE tx_size; + int eob; + int lossless; +#if CONFIG_HIGHBITDEPTH + int bd; +#endif +} INV_TXFM_PARAM; + +typedef void (*transform_1d)(const tran_low_t *, tran_low_t *); + +typedef struct { + transform_1d cols, rows; // vertical and horizontal +} transform_2d; + +#if CONFIG_HIGHBITDEPTH +typedef void (*highbd_transform_1d)(const tran_low_t *, tran_low_t *, + const int8_t *cos_bit, + const int8_t *stage_range, int bd); + +typedef struct { + highbd_transform_1d cols, rows; // vertical and horizontal +} highbd_transform_2d; +#endif // CONFIG_HIGHBITDEPTH + +#define MAX_TX_SCALE 1 +int av1_get_tx_scale(const TX_SIZE tx_size); + +void av1_iwht4x4_add(const tran_low_t *input, uint8_t *dest, int stride, + int eob); +void av1_idct4x4_add(const tran_low_t *input, uint8_t *dest, int stride, + int eob); + +void av1_inv_txfm_add_4x4(const tran_low_t *input, uint8_t *dest, int stride, + int eob, TX_TYPE tx_type, int lossless); +void av1_inv_txfm_add_8x4(const tran_low_t *input, uint8_t *dest, int stride, + int eob, TX_TYPE tx_type); +void av1_inv_txfm_add_4x8(const tran_low_t *input, uint8_t *dest, int stride, + int eob, TX_TYPE tx_type); +void av1_inv_txfm_add(const tran_low_t *input, uint8_t *dest, int stride, + INV_TXFM_PARAM *inv_txfm_param); +void av1_inverse_transform_block(const MACROBLOCKD *xd, + const tran_low_t *dqcoeff, TX_TYPE tx_type, + TX_SIZE tx_size, uint8_t *dst, int stride, + int eob); +void av1_inverse_transform_block_facade(MACROBLOCKD *xd, int plane, int block, + int blk_row, int blk_col, int eob); +#if CONFIG_HIGHBITDEPTH +void av1_highbd_iwht4x4_add(const tran_low_t *input, uint8_t *dest, int stride, + int eob, int bd); +void av1_highbd_idct4x4_add(const tran_low_t *input, uint8_t *dest, int stride, + int eob, int bd); +void av1_highbd_inv_txfm_add_4x4(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, TX_TYPE tx_type, + int lossless); +void av1_highbd_inv_txfm_add_4x8(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, TX_TYPE tx_type); +void av1_highbd_inv_txfm_add_8x4(const tran_low_t *input, uint8_t *dest, + int stride, int eob, int bd, TX_TYPE tx_type); +void av1_highbd_inv_txfm_add(const tran_low_t *input, uint8_t *dest, int stride, + INV_TXFM_PARAM *inv_txfm_param); +#endif // CONFIG_HIGHBITDEPTH +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_IDCT_H_ diff --git a/third_party/aom/av1/common/laplace_tables.c b/third_party/aom/av1/common/laplace_tables.c new file mode 100644 index 0000000000..ab87848955 --- /dev/null +++ b/third_party/aom/av1/common/laplace_tables.c @@ -0,0 +1,657 @@ +/* This file is auto-generated using "gen_laplace_tables 128 7" */ + +/* clang-format off */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "aom_dsp/prob.h" +#include "pvq.h" + +const uint16_t EXP_CDF_TABLE[128][16] = { + {AOM_ICDF(32753), AOM_ICDF(32754), AOM_ICDF(32755), AOM_ICDF(32756), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(32499), AOM_ICDF(32753), AOM_ICDF(32755), AOM_ICDF(32756), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(32243), AOM_ICDF(32747), AOM_ICDF(32755), AOM_ICDF(32756), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(31987), AOM_ICDF(32737), AOM_ICDF(32755), AOM_ICDF(32756), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(31732), AOM_ICDF(32724), AOM_ICDF(32755), AOM_ICDF(32756), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(31476), AOM_ICDF(32706), AOM_ICDF(32754), AOM_ICDF(32756), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(31220), AOM_ICDF(32684), AOM_ICDF(32753), AOM_ICDF(32756), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(30964), AOM_ICDF(32658), AOM_ICDF(32751), AOM_ICDF(32756), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(30708), AOM_ICDF(32628), AOM_ICDF(32748), AOM_ICDF(32756), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(30452), AOM_ICDF(32594), AOM_ICDF(32745), AOM_ICDF(32756), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(30198), AOM_ICDF(32558), AOM_ICDF(32742), AOM_ICDF(32756), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(29941), AOM_ICDF(32515), AOM_ICDF(32736), AOM_ICDF(32755), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(29686), AOM_ICDF(32470), AOM_ICDF(32731), AOM_ICDF(32755), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(29429), AOM_ICDF(32419), AOM_ICDF(32723), AOM_ICDF(32754), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(29174), AOM_ICDF(32366), AOM_ICDF(32715), AOM_ICDF(32753), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(28918), AOM_ICDF(32308), AOM_ICDF(32705), AOM_ICDF(32752), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(28662), AOM_ICDF(32246), AOM_ICDF(32694), AOM_ICDF(32750), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(28406), AOM_ICDF(32180), AOM_ICDF(32681), AOM_ICDF(32748), + AOM_ICDF(32757), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(28150), AOM_ICDF(32110), AOM_ICDF(32667), AOM_ICDF(32745), + AOM_ICDF(32756), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(27894), AOM_ICDF(32036), AOM_ICDF(32651), AOM_ICDF(32742), + AOM_ICDF(32756), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(27639), AOM_ICDF(31959), AOM_ICDF(32634), AOM_ICDF(32739), + AOM_ICDF(32755), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(27383), AOM_ICDF(31877), AOM_ICDF(32614), AOM_ICDF(32735), + AOM_ICDF(32755), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(27126), AOM_ICDF(31790), AOM_ICDF(32592), AOM_ICDF(32730), + AOM_ICDF(32754), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(26871), AOM_ICDF(31701), AOM_ICDF(32569), AOM_ICDF(32725), + AOM_ICDF(32753), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(26615), AOM_ICDF(31607), AOM_ICDF(32543), AOM_ICDF(32719), + AOM_ICDF(32752), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(26361), AOM_ICDF(31511), AOM_ICDF(32517), AOM_ICDF(32713), + AOM_ICDF(32751), AOM_ICDF(32758), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(26104), AOM_ICDF(31408), AOM_ICDF(32485), AOM_ICDF(32704), + AOM_ICDF(32748), AOM_ICDF(32757), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(25848), AOM_ICDF(31302), AOM_ICDF(32452), AOM_ICDF(32695), + AOM_ICDF(32746), AOM_ICDF(32757), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(25591), AOM_ICDF(31191), AOM_ICDF(32416), AOM_ICDF(32684), + AOM_ICDF(32743), AOM_ICDF(32756), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(25336), AOM_ICDF(31078), AOM_ICDF(32379), AOM_ICDF(32674), + AOM_ICDF(32741), AOM_ICDF(32756), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(25080), AOM_ICDF(30960), AOM_ICDF(32338), AOM_ICDF(32661), + AOM_ICDF(32737), AOM_ICDF(32755), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(24824), AOM_ICDF(30838), AOM_ICDF(32295), AOM_ICDF(32648), + AOM_ICDF(32733), AOM_ICDF(32754), AOM_ICDF(32759), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(24568), AOM_ICDF(30712), AOM_ICDF(32248), AOM_ICDF(32632), + AOM_ICDF(32728), AOM_ICDF(32752), AOM_ICDF(32758), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(24313), AOM_ICDF(30583), AOM_ICDF(32199), AOM_ICDF(32616), + AOM_ICDF(32723), AOM_ICDF(32751), AOM_ICDF(32758), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(24057), AOM_ICDF(30449), AOM_ICDF(32147), AOM_ICDF(32598), + AOM_ICDF(32718), AOM_ICDF(32750), AOM_ICDF(32758), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(23801), AOM_ICDF(30311), AOM_ICDF(32091), AOM_ICDF(32578), + AOM_ICDF(32711), AOM_ICDF(32747), AOM_ICDF(32757), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(23546), AOM_ICDF(30170), AOM_ICDF(32033), AOM_ICDF(32557), + AOM_ICDF(32704), AOM_ICDF(32745), AOM_ICDF(32757), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(23288), AOM_ICDF(30022), AOM_ICDF(31969), AOM_ICDF(32532), + AOM_ICDF(32695), AOM_ICDF(32742), AOM_ICDF(32756), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(23033), AOM_ICDF(29873), AOM_ICDF(31904), AOM_ICDF(32507), + AOM_ICDF(32686), AOM_ICDF(32739), AOM_ICDF(32755), AOM_ICDF(32760), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(22778), AOM_ICDF(29720), AOM_ICDF(31835), AOM_ICDF(32479), + AOM_ICDF(32675), AOM_ICDF(32735), AOM_ICDF(32753), AOM_ICDF(32759), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(22521), AOM_ICDF(29561), AOM_ICDF(31761), AOM_ICDF(32449), + AOM_ICDF(32664), AOM_ICDF(32731), AOM_ICDF(32752), AOM_ICDF(32759), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(22267), AOM_ICDF(29401), AOM_ICDF(31686), AOM_ICDF(32418), + AOM_ICDF(32652), AOM_ICDF(32727), AOM_ICDF(32751), AOM_ICDF(32759), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(22011), AOM_ICDF(29235), AOM_ICDF(31605), AOM_ICDF(32383), + AOM_ICDF(32638), AOM_ICDF(32722), AOM_ICDF(32749), AOM_ICDF(32758), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(21754), AOM_ICDF(29064), AOM_ICDF(31520), AOM_ICDF(32345), + AOM_ICDF(32622), AOM_ICDF(32715), AOM_ICDF(32746), AOM_ICDF(32757), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(21501), AOM_ICDF(28893), AOM_ICDF(31434), AOM_ICDF(32307), + AOM_ICDF(32607), AOM_ICDF(32710), AOM_ICDF(32745), AOM_ICDF(32757), + AOM_ICDF(32761), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(21243), AOM_ICDF(28713), AOM_ICDF(31339), AOM_ICDF(32262), + AOM_ICDF(32587), AOM_ICDF(32701), AOM_ICDF(32741), AOM_ICDF(32755), + AOM_ICDF(32760), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(20988), AOM_ICDF(28532), AOM_ICDF(31243), AOM_ICDF(32217), + AOM_ICDF(32567), AOM_ICDF(32693), AOM_ICDF(32738), AOM_ICDF(32754), + AOM_ICDF(32760), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(20730), AOM_ICDF(28344), AOM_ICDF(31140), AOM_ICDF(32167), + AOM_ICDF(32544), AOM_ICDF(32682), AOM_ICDF(32733), AOM_ICDF(32752), + AOM_ICDF(32759), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(20476), AOM_ICDF(28156), AOM_ICDF(31036), AOM_ICDF(32116), + AOM_ICDF(32521), AOM_ICDF(32673), AOM_ICDF(32730), AOM_ICDF(32751), + AOM_ICDF(32759), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(20220), AOM_ICDF(27962), AOM_ICDF(30926), AOM_ICDF(32061), + AOM_ICDF(32495), AOM_ICDF(32661), AOM_ICDF(32725), AOM_ICDF(32749), + AOM_ICDF(32758), AOM_ICDF(32762), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(19963), AOM_ICDF(27763), AOM_ICDF(30810), AOM_ICDF(32000), + AOM_ICDF(32465), AOM_ICDF(32647), AOM_ICDF(32718), AOM_ICDF(32746), + AOM_ICDF(32757), AOM_ICDF(32761), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(19708), AOM_ICDF(27562), AOM_ICDF(30691), AOM_ICDF(31938), + AOM_ICDF(32435), AOM_ICDF(32633), AOM_ICDF(32712), AOM_ICDF(32743), + AOM_ICDF(32756), AOM_ICDF(32761), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(19454), AOM_ICDF(27358), AOM_ICDF(30569), AOM_ICDF(31873), + AOM_ICDF(32403), AOM_ICDF(32618), AOM_ICDF(32705), AOM_ICDF(32741), + AOM_ICDF(32755), AOM_ICDF(32761), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(19196), AOM_ICDF(27146), AOM_ICDF(30438), AOM_ICDF(31801), + AOM_ICDF(32365), AOM_ICDF(32599), AOM_ICDF(32696), AOM_ICDF(32736), + AOM_ICDF(32753), AOM_ICDF(32760), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(18942), AOM_ICDF(26934), AOM_ICDF(30306), AOM_ICDF(31728), + AOM_ICDF(32328), AOM_ICDF(32581), AOM_ICDF(32688), AOM_ICDF(32733), + AOM_ICDF(32752), AOM_ICDF(32760), AOM_ICDF(32763), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(18684), AOM_ICDF(26714), AOM_ICDF(30164), AOM_ICDF(31647), + AOM_ICDF(32284), AOM_ICDF(32558), AOM_ICDF(32676), AOM_ICDF(32727), + AOM_ICDF(32749), AOM_ICDF(32758), AOM_ICDF(32762), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(18429), AOM_ICDF(26493), AOM_ICDF(30021), AOM_ICDF(31565), + AOM_ICDF(32240), AOM_ICDF(32535), AOM_ICDF(32664), AOM_ICDF(32721), + AOM_ICDF(32746), AOM_ICDF(32757), AOM_ICDF(32762), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(18174), AOM_ICDF(26268), AOM_ICDF(29872), AOM_ICDF(31477), + AOM_ICDF(32192), AOM_ICDF(32510), AOM_ICDF(32652), AOM_ICDF(32715), + AOM_ICDF(32743), AOM_ICDF(32756), AOM_ICDF(32762), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(17920), AOM_ICDF(26040), AOM_ICDF(29719), AOM_ICDF(31386), + AOM_ICDF(32141), AOM_ICDF(32483), AOM_ICDF(32638), AOM_ICDF(32708), + AOM_ICDF(32740), AOM_ICDF(32754), AOM_ICDF(32761), AOM_ICDF(32764), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(17661), AOM_ICDF(25803), AOM_ICDF(29556), AOM_ICDF(31286), + AOM_ICDF(32083), AOM_ICDF(32451), AOM_ICDF(32620), AOM_ICDF(32698), + AOM_ICDF(32734), AOM_ICDF(32751), AOM_ICDF(32759), AOM_ICDF(32763), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(17406), AOM_ICDF(25566), AOM_ICDF(29391), AOM_ICDF(31184), + AOM_ICDF(32024), AOM_ICDF(32418), AOM_ICDF(32603), AOM_ICDF(32690), + AOM_ICDF(32731), AOM_ICDF(32750), AOM_ICDF(32759), AOM_ICDF(32763), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(17151), AOM_ICDF(25325), AOM_ICDF(29220), AOM_ICDF(31076), + AOM_ICDF(31961), AOM_ICDF(32383), AOM_ICDF(32584), AOM_ICDF(32680), + AOM_ICDF(32726), AOM_ICDF(32748), AOM_ICDF(32758), AOM_ICDF(32763), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(16896), AOM_ICDF(25080), AOM_ICDF(29044), AOM_ICDF(30964), + AOM_ICDF(31894), AOM_ICDF(32344), AOM_ICDF(32562), AOM_ICDF(32668), + AOM_ICDF(32719), AOM_ICDF(32744), AOM_ICDF(32756), AOM_ICDF(32762), + AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(16639), AOM_ICDF(24829), AOM_ICDF(28860), AOM_ICDF(30844), + AOM_ICDF(31821), AOM_ICDF(32302), AOM_ICDF(32539), AOM_ICDF(32655), + AOM_ICDF(32712), AOM_ICDF(32740), AOM_ICDF(32754), AOM_ICDF(32761), + AOM_ICDF(32764), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(16384), AOM_ICDF(24576), AOM_ICDF(28672), AOM_ICDF(30720), + AOM_ICDF(31744), AOM_ICDF(32256), AOM_ICDF(32512), AOM_ICDF(32640), + AOM_ICDF(32704), AOM_ICDF(32736), AOM_ICDF(32752), AOM_ICDF(32760), + AOM_ICDF(32764), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(16130), AOM_ICDF(24320), AOM_ICDF(28479), AOM_ICDF(30591), + AOM_ICDF(31663), AOM_ICDF(32208), AOM_ICDF(32485), AOM_ICDF(32625), + AOM_ICDF(32696), AOM_ICDF(32732), AOM_ICDF(32750), AOM_ICDF(32759), + AOM_ICDF(32764), AOM_ICDF(32766), AOM_ICDF(32767), AOM_ICDF(32768)}, + {AOM_ICDF(15872), AOM_ICDF(24056), AOM_ICDF(28276), AOM_ICDF(30452), + AOM_ICDF(31574), AOM_ICDF(32152), AOM_ICDF(32450), AOM_ICDF(32604), + AOM_ICDF(32683), AOM_ICDF(32724), AOM_ICDF(32745), AOM_ICDF(32756), + AOM_ICDF(32762), AOM_ICDF(32765), AOM_ICDF(32766), AOM_ICDF(32768)}, + {AOM_ICDF(15615), AOM_ICDF(23789), AOM_ICDF(28068), AOM_ICDF(30308), + AOM_ICDF(31480), AOM_ICDF(32094), AOM_ICDF(32415), AOM_ICDF(32583), + AOM_ICDF(32671), AOM_ICDF(32717), AOM_ICDF(32741), AOM_ICDF(32754), + AOM_ICDF(32761), AOM_ICDF(32764), AOM_ICDF(32766), AOM_ICDF(32768)}, + {AOM_ICDF(15361), AOM_ICDF(23521), AOM_ICDF(27856), AOM_ICDF(30159), + AOM_ICDF(31382), AOM_ICDF(32032), AOM_ICDF(32377), AOM_ICDF(32560), + AOM_ICDF(32657), AOM_ICDF(32709), AOM_ICDF(32737), AOM_ICDF(32752), + AOM_ICDF(32760), AOM_ICDF(32764), AOM_ICDF(32766), AOM_ICDF(32768)}, + {AOM_ICDF(15103), AOM_ICDF(23245), AOM_ICDF(27634), AOM_ICDF(30000), + AOM_ICDF(31275), AOM_ICDF(31963), AOM_ICDF(32334), AOM_ICDF(32534), + AOM_ICDF(32642), AOM_ICDF(32700), AOM_ICDF(32731), AOM_ICDF(32748), + AOM_ICDF(32757), AOM_ICDF(32762), AOM_ICDF(32765), AOM_ICDF(32768)}, + {AOM_ICDF(14848), AOM_ICDF(22968), AOM_ICDF(27409), AOM_ICDF(29837), + AOM_ICDF(31165), AOM_ICDF(31891), AOM_ICDF(32288), AOM_ICDF(32505), + AOM_ICDF(32624), AOM_ICDF(32689), AOM_ICDF(32725), AOM_ICDF(32744), + AOM_ICDF(32755), AOM_ICDF(32761), AOM_ICDF(32764), AOM_ICDF(32768)}, + {AOM_ICDF(14592), AOM_ICDF(22686), AOM_ICDF(27176), AOM_ICDF(29666), + AOM_ICDF(31047), AOM_ICDF(31813), AOM_ICDF(32238), AOM_ICDF(32474), + AOM_ICDF(32605), AOM_ICDF(32678), AOM_ICDF(32718), AOM_ICDF(32740), + AOM_ICDF(32752), AOM_ICDF(32759), AOM_ICDF(32763), AOM_ICDF(32768)}, + {AOM_ICDF(14336), AOM_ICDF(22400), AOM_ICDF(26936), AOM_ICDF(29488), + AOM_ICDF(30923), AOM_ICDF(31730), AOM_ICDF(32184), AOM_ICDF(32439), + AOM_ICDF(32583), AOM_ICDF(32664), AOM_ICDF(32709), AOM_ICDF(32735), + AOM_ICDF(32749), AOM_ICDF(32757), AOM_ICDF(32762), AOM_ICDF(32768)}, + {AOM_ICDF(14079), AOM_ICDF(22109), AOM_ICDF(26689), AOM_ICDF(29301), + AOM_ICDF(30791), AOM_ICDF(31641), AOM_ICDF(32125), AOM_ICDF(32401), + AOM_ICDF(32559), AOM_ICDF(32649), AOM_ICDF(32700), AOM_ICDF(32729), + AOM_ICDF(32746), AOM_ICDF(32756), AOM_ICDF(32761), AOM_ICDF(32768)}, + {AOM_ICDF(13825), AOM_ICDF(21817), AOM_ICDF(26437), AOM_ICDF(29108), + AOM_ICDF(30652), AOM_ICDF(31545), AOM_ICDF(32061), AOM_ICDF(32359), + AOM_ICDF(32532), AOM_ICDF(32632), AOM_ICDF(32690), AOM_ICDF(32723), + AOM_ICDF(32742), AOM_ICDF(32753), AOM_ICDF(32759), AOM_ICDF(32768)}, + {AOM_ICDF(13568), AOM_ICDF(21518), AOM_ICDF(26176), AOM_ICDF(28905), + AOM_ICDF(30504), AOM_ICDF(31441), AOM_ICDF(31990), AOM_ICDF(32312), + AOM_ICDF(32501), AOM_ICDF(32611), AOM_ICDF(32676), AOM_ICDF(32714), + AOM_ICDF(32736), AOM_ICDF(32749), AOM_ICDF(32757), AOM_ICDF(32768)}, + {AOM_ICDF(13314), AOM_ICDF(21218), AOM_ICDF(25911), AOM_ICDF(28697), + AOM_ICDF(30351), AOM_ICDF(31333), AOM_ICDF(31916), AOM_ICDF(32262), + AOM_ICDF(32468), AOM_ICDF(32590), AOM_ICDF(32662), AOM_ICDF(32705), + AOM_ICDF(32731), AOM_ICDF(32746), AOM_ICDF(32755), AOM_ICDF(32768)}, + {AOM_ICDF(13054), AOM_ICDF(20908), AOM_ICDF(25633), AOM_ICDF(28475), + AOM_ICDF(30185), AOM_ICDF(31214), AOM_ICDF(31833), AOM_ICDF(32205), + AOM_ICDF(32429), AOM_ICDF(32564), AOM_ICDF(32645), AOM_ICDF(32694), + AOM_ICDF(32723), AOM_ICDF(32741), AOM_ICDF(32752), AOM_ICDF(32768)}, + {AOM_ICDF(12803), AOM_ICDF(20603), AOM_ICDF(25356), AOM_ICDF(28252), + AOM_ICDF(30017), AOM_ICDF(31093), AOM_ICDF(31748), AOM_ICDF(32147), + AOM_ICDF(32390), AOM_ICDF(32538), AOM_ICDF(32628), AOM_ICDF(32683), + AOM_ICDF(32717), AOM_ICDF(32737), AOM_ICDF(32749), AOM_ICDF(32768)}, + {AOM_ICDF(12544), AOM_ICDF(20286), AOM_ICDF(25064), AOM_ICDF(28013), + AOM_ICDF(29833), AOM_ICDF(30956), AOM_ICDF(31649), AOM_ICDF(32077), + AOM_ICDF(32341), AOM_ICDF(32504), AOM_ICDF(32605), AOM_ICDF(32667), + AOM_ICDF(32705), AOM_ICDF(32729), AOM_ICDF(32744), AOM_ICDF(32768)}, + {AOM_ICDF(12288), AOM_ICDF(19968), AOM_ICDF(24768), AOM_ICDF(27768), + AOM_ICDF(29643), AOM_ICDF(30815), AOM_ICDF(31547), AOM_ICDF(32005), + AOM_ICDF(32291), AOM_ICDF(32470), AOM_ICDF(32582), AOM_ICDF(32652), + AOM_ICDF(32696), AOM_ICDF(32723), AOM_ICDF(32740), AOM_ICDF(32768)}, + {AOM_ICDF(12033), AOM_ICDF(19647), AOM_ICDF(24465), AOM_ICDF(27514), + AOM_ICDF(29443), AOM_ICDF(30664), AOM_ICDF(31437), AOM_ICDF(31926), + AOM_ICDF(32235), AOM_ICDF(32431), AOM_ICDF(32555), AOM_ICDF(32633), + AOM_ICDF(32683), AOM_ICDF(32714), AOM_ICDF(32734), AOM_ICDF(32768)}, + {AOM_ICDF(11777), AOM_ICDF(19321), AOM_ICDF(24154), AOM_ICDF(27250), + AOM_ICDF(29233), AOM_ICDF(30504), AOM_ICDF(31318), AOM_ICDF(31839), + AOM_ICDF(32173), AOM_ICDF(32387), AOM_ICDF(32524), AOM_ICDF(32612), + AOM_ICDF(32668), AOM_ICDF(32704), AOM_ICDF(32727), AOM_ICDF(32768)}, + {AOM_ICDF(11521), AOM_ICDF(18991), AOM_ICDF(23835), AOM_ICDF(26976), + AOM_ICDF(29013), AOM_ICDF(30334), AOM_ICDF(31190), AOM_ICDF(31745), + AOM_ICDF(32105), AOM_ICDF(32338), AOM_ICDF(32489), AOM_ICDF(32587), + AOM_ICDF(32651), AOM_ICDF(32692), AOM_ICDF(32719), AOM_ICDF(32768)}, + {AOM_ICDF(11265), AOM_ICDF(18657), AOM_ICDF(23508), AOM_ICDF(26691), + AOM_ICDF(28780), AOM_ICDF(30151), AOM_ICDF(31051), AOM_ICDF(31641), + AOM_ICDF(32028), AOM_ICDF(32282), AOM_ICDF(32449), AOM_ICDF(32559), + AOM_ICDF(32631), AOM_ICDF(32678), AOM_ICDF(32709), AOM_ICDF(32768)}, + {AOM_ICDF(11006), AOM_ICDF(18316), AOM_ICDF(23170), AOM_ICDF(26394), + AOM_ICDF(28535), AOM_ICDF(29957), AOM_ICDF(30901), AOM_ICDF(31528), + AOM_ICDF(31944), AOM_ICDF(32220), AOM_ICDF(32404), AOM_ICDF(32526), + AOM_ICDF(32607), AOM_ICDF(32661), AOM_ICDF(32697), AOM_ICDF(32768)}, + {AOM_ICDF(10752), AOM_ICDF(17976), AOM_ICDF(22830), AOM_ICDF(26091), + AOM_ICDF(28282), AOM_ICDF(29754), AOM_ICDF(30743), AOM_ICDF(31408), + AOM_ICDF(31854), AOM_ICDF(32154), AOM_ICDF(32356), AOM_ICDF(32491), + AOM_ICDF(32582), AOM_ICDF(32643), AOM_ICDF(32684), AOM_ICDF(32768)}, + {AOM_ICDF(10496), AOM_ICDF(17630), AOM_ICDF(22479), AOM_ICDF(25775), + AOM_ICDF(28015), AOM_ICDF(29538), AOM_ICDF(30573), AOM_ICDF(31276), + AOM_ICDF(31754), AOM_ICDF(32079), AOM_ICDF(32300), AOM_ICDF(32450), + AOM_ICDF(32552), AOM_ICDF(32621), AOM_ICDF(32668), AOM_ICDF(32768)}, + {AOM_ICDF(10240), AOM_ICDF(17280), AOM_ICDF(22120), AOM_ICDF(25448), + AOM_ICDF(27736), AOM_ICDF(29309), AOM_ICDF(30390), AOM_ICDF(31133), + AOM_ICDF(31644), AOM_ICDF(31995), AOM_ICDF(32237), AOM_ICDF(32403), + AOM_ICDF(32517), AOM_ICDF(32595), AOM_ICDF(32649), AOM_ICDF(32768)}, + { AOM_ICDF(9984), AOM_ICDF(16926), AOM_ICDF(21753), AOM_ICDF(25109), + AOM_ICDF(27443), AOM_ICDF(29066), AOM_ICDF(30194), AOM_ICDF(30978), + AOM_ICDF(31523), AOM_ICDF(31902), AOM_ICDF(32166), AOM_ICDF(32349), + AOM_ICDF(32476), AOM_ICDF(32565), AOM_ICDF(32627), AOM_ICDF(32768)}, + { AOM_ICDF(9728), AOM_ICDF(16568), AOM_ICDF(21377), AOM_ICDF(24759), + AOM_ICDF(27137), AOM_ICDF(28809), AOM_ICDF(29984), AOM_ICDF(30811), + AOM_ICDF(31392), AOM_ICDF(31801), AOM_ICDF(32088), AOM_ICDF(32290), + AOM_ICDF(32432), AOM_ICDF(32532), AOM_ICDF(32602), AOM_ICDF(32768)}, + { AOM_ICDF(9474), AOM_ICDF(16208), AOM_ICDF(20995), AOM_ICDF(24399), + AOM_ICDF(26819), AOM_ICDF(28539), AOM_ICDF(29762), AOM_ICDF(30631), + AOM_ICDF(31249), AOM_ICDF(31688), AOM_ICDF(32000), AOM_ICDF(32222), + AOM_ICDF(32380), AOM_ICDF(32492), AOM_ICDF(32572), AOM_ICDF(32768)}, + { AOM_ICDF(9216), AOM_ICDF(15840), AOM_ICDF(20601), AOM_ICDF(24023), + AOM_ICDF(26483), AOM_ICDF(28251), AOM_ICDF(29522), AOM_ICDF(30435), + AOM_ICDF(31091), AOM_ICDF(31563), AOM_ICDF(31902), AOM_ICDF(32146), + AOM_ICDF(32321), AOM_ICDF(32447), AOM_ICDF(32537), AOM_ICDF(32768)}, + { AOM_ICDF(8959), AOM_ICDF(15469), AOM_ICDF(20199), AOM_ICDF(23636), + AOM_ICDF(26133), AOM_ICDF(27947), AOM_ICDF(29265), AOM_ICDF(30223), + AOM_ICDF(30919), AOM_ICDF(31425), AOM_ICDF(31792), AOM_ICDF(32059), + AOM_ICDF(32253), AOM_ICDF(32394), AOM_ICDF(32496), AOM_ICDF(32768)}, + { AOM_ICDF(8705), AOM_ICDF(15097), AOM_ICDF(19791), AOM_ICDF(23238), + AOM_ICDF(25770), AOM_ICDF(27629), AOM_ICDF(28994), AOM_ICDF(29997), + AOM_ICDF(30733), AOM_ICDF(31274), AOM_ICDF(31671), AOM_ICDF(31963), + AOM_ICDF(32177), AOM_ICDF(32334), AOM_ICDF(32449), AOM_ICDF(32768)}, + { AOM_ICDF(8449), AOM_ICDF(14719), AOM_ICDF(19373), AOM_ICDF(22827), + AOM_ICDF(25390), AOM_ICDF(27292), AOM_ICDF(28704), AOM_ICDF(29752), + AOM_ICDF(30530), AOM_ICDF(31107), AOM_ICDF(31535), AOM_ICDF(31853), + AOM_ICDF(32089), AOM_ICDF(32264), AOM_ICDF(32394), AOM_ICDF(32768)}, + { AOM_ICDF(8192), AOM_ICDF(14336), AOM_ICDF(18944), AOM_ICDF(22400), + AOM_ICDF(24992), AOM_ICDF(26936), AOM_ICDF(28394), AOM_ICDF(29488), + AOM_ICDF(30308), AOM_ICDF(30923), AOM_ICDF(31384), AOM_ICDF(31730), + AOM_ICDF(31989), AOM_ICDF(32184), AOM_ICDF(32330), AOM_ICDF(32768)}, + { AOM_ICDF(7936), AOM_ICDF(13950), AOM_ICDF(18507), AOM_ICDF(21961), + AOM_ICDF(24578), AOM_ICDF(26561), AOM_ICDF(28064), AOM_ICDF(29203), + AOM_ICDF(30066), AOM_ICDF(30720), AOM_ICDF(31216), AOM_ICDF(31592), + AOM_ICDF(31877), AOM_ICDF(32093), AOM_ICDF(32256), AOM_ICDF(32768)}, + { AOM_ICDF(7678), AOM_ICDF(13558), AOM_ICDF(18060), AOM_ICDF(21507), + AOM_ICDF(24146), AOM_ICDF(26166), AOM_ICDF(27713), AOM_ICDF(28897), + AOM_ICDF(29804), AOM_ICDF(30498), AOM_ICDF(31030), AOM_ICDF(31437), + AOM_ICDF(31749), AOM_ICDF(31988), AOM_ICDF(32171), AOM_ICDF(32768)}, + { AOM_ICDF(7423), AOM_ICDF(13165), AOM_ICDF(17606), AOM_ICDF(21041), + AOM_ICDF(23698), AOM_ICDF(25753), AOM_ICDF(27342), AOM_ICDF(28571), + AOM_ICDF(29522), AOM_ICDF(30257), AOM_ICDF(30826), AOM_ICDF(31266), + AOM_ICDF(31606), AOM_ICDF(31869), AOM_ICDF(32073), AOM_ICDF(32768)}, + { AOM_ICDF(7168), AOM_ICDF(12768), AOM_ICDF(17143), AOM_ICDF(20561), + AOM_ICDF(23231), AOM_ICDF(25317), AOM_ICDF(26947), AOM_ICDF(28220), + AOM_ICDF(29215), AOM_ICDF(29992), AOM_ICDF(30599), AOM_ICDF(31073), + AOM_ICDF(31444), AOM_ICDF(31734), AOM_ICDF(31960), AOM_ICDF(32768)}, + { AOM_ICDF(6911), AOM_ICDF(12365), AOM_ICDF(16669), AOM_ICDF(20065), + AOM_ICDF(22744), AOM_ICDF(24858), AOM_ICDF(26526), AOM_ICDF(27842), + AOM_ICDF(28881), AOM_ICDF(29701), AOM_ICDF(30348), AOM_ICDF(30858), + AOM_ICDF(31261), AOM_ICDF(31579), AOM_ICDF(31830), AOM_ICDF(32768)}, + { AOM_ICDF(6657), AOM_ICDF(11961), AOM_ICDF(16188), AOM_ICDF(19556), + AOM_ICDF(22240), AOM_ICDF(24379), AOM_ICDF(26083), AOM_ICDF(27441), + AOM_ICDF(28523), AOM_ICDF(29385), AOM_ICDF(30072), AOM_ICDF(30620), + AOM_ICDF(31056), AOM_ICDF(31404), AOM_ICDF(31681), AOM_ICDF(32768)}, + { AOM_ICDF(6400), AOM_ICDF(11550), AOM_ICDF(15694), AOM_ICDF(19029), + AOM_ICDF(21712), AOM_ICDF(23871), AOM_ICDF(25609), AOM_ICDF(27007), + AOM_ICDF(28132), AOM_ICDF(29037), AOM_ICDF(29766), AOM_ICDF(30352), + AOM_ICDF(30824), AOM_ICDF(31204), AOM_ICDF(31509), AOM_ICDF(32768)}, + { AOM_ICDF(6142), AOM_ICDF(11134), AOM_ICDF(15190), AOM_ICDF(18486), + AOM_ICDF(21164), AOM_ICDF(23340), AOM_ICDF(25108), AOM_ICDF(26544), + AOM_ICDF(27711), AOM_ICDF(28659), AOM_ICDF(29429), AOM_ICDF(30055), + AOM_ICDF(30564), AOM_ICDF(30977), AOM_ICDF(31313), AOM_ICDF(32768)}, + { AOM_ICDF(5890), AOM_ICDF(10720), AOM_ICDF(14682), AOM_ICDF(17932), + AOM_ICDF(20598), AOM_ICDF(22785), AOM_ICDF(24579), AOM_ICDF(26051), + AOM_ICDF(27258), AOM_ICDF(28248), AOM_ICDF(29060), AOM_ICDF(29726), + AOM_ICDF(30273), AOM_ICDF(30721), AOM_ICDF(31089), AOM_ICDF(32768)}, + { AOM_ICDF(5631), AOM_ICDF(10295), AOM_ICDF(14157), AOM_ICDF(17356), + AOM_ICDF(20005), AOM_ICDF(22199), AOM_ICDF(24016), AOM_ICDF(25520), + AOM_ICDF(26766), AOM_ICDF(27798), AOM_ICDF(28652), AOM_ICDF(29359), + AOM_ICDF(29945), AOM_ICDF(30430), AOM_ICDF(30832), AOM_ICDF(32768)}, + { AOM_ICDF(5377), AOM_ICDF(9871), AOM_ICDF(13628), AOM_ICDF(16768), + AOM_ICDF(19393), AOM_ICDF(21587), AOM_ICDF(23421), AOM_ICDF(24954), + AOM_ICDF(26236), AOM_ICDF(27308), AOM_ICDF(28204), AOM_ICDF(28953), + AOM_ICDF(29579), AOM_ICDF(30102), AOM_ICDF(30539), AOM_ICDF(32768)}, + { AOM_ICDF(5121), AOM_ICDF(9441), AOM_ICDF(13086), AOM_ICDF(16161), + AOM_ICDF(18756), AOM_ICDF(20945), AOM_ICDF(22792), AOM_ICDF(24351), + AOM_ICDF(25666), AOM_ICDF(26776), AOM_ICDF(27712), AOM_ICDF(28502), + AOM_ICDF(29169), AOM_ICDF(29731), AOM_ICDF(30206), AOM_ICDF(32768)}, + { AOM_ICDF(4865), AOM_ICDF(9007), AOM_ICDF(12534), AOM_ICDF(15538), + AOM_ICDF(18096), AOM_ICDF(20274), AOM_ICDF(22129), AOM_ICDF(23708), + AOM_ICDF(25053), AOM_ICDF(26198), AOM_ICDF(27173), AOM_ICDF(28004), + AOM_ICDF(28711), AOM_ICDF(29313), AOM_ICDF(29826), AOM_ICDF(32768)}, + { AOM_ICDF(4608), AOM_ICDF(8568), AOM_ICDF(11971), AOM_ICDF(14896), + AOM_ICDF(17409), AOM_ICDF(19569), AOM_ICDF(21425), AOM_ICDF(23020), + AOM_ICDF(24391), AOM_ICDF(25569), AOM_ICDF(26581), AOM_ICDF(27451), + AOM_ICDF(28199), AOM_ICDF(28842), AOM_ICDF(29394), AOM_ICDF(32768)}, + { AOM_ICDF(4351), AOM_ICDF(8125), AOM_ICDF(11398), AOM_ICDF(14236), + AOM_ICDF(16697), AOM_ICDF(18831), AOM_ICDF(20682), AOM_ICDF(22287), + AOM_ICDF(23679), AOM_ICDF(24886), AOM_ICDF(25933), AOM_ICDF(26841), + AOM_ICDF(27628), AOM_ICDF(28311), AOM_ICDF(28903), AOM_ICDF(32768)}, + { AOM_ICDF(4096), AOM_ICDF(7680), AOM_ICDF(10816), AOM_ICDF(13560), + AOM_ICDF(15961), AOM_ICDF(18062), AOM_ICDF(19900), AOM_ICDF(21508), + AOM_ICDF(22915), AOM_ICDF(24146), AOM_ICDF(25224), AOM_ICDF(26167), + AOM_ICDF(26992), AOM_ICDF(27714), AOM_ICDF(28346), AOM_ICDF(32768)}, + { AOM_ICDF(3840), AOM_ICDF(7230), AOM_ICDF(10223), AOM_ICDF(12865), + AOM_ICDF(15197), AOM_ICDF(17256), AOM_ICDF(19074), AOM_ICDF(20679), + AOM_ICDF(22096), AOM_ICDF(23347), AOM_ICDF(24451), AOM_ICDF(25426), + AOM_ICDF(26287), AOM_ICDF(27047), AOM_ICDF(27718), AOM_ICDF(32768)}, + { AOM_ICDF(3584), AOM_ICDF(6776), AOM_ICDF(9619), AOM_ICDF(12151), + AOM_ICDF(14406), AOM_ICDF(16414), AOM_ICDF(18203), AOM_ICDF(19796), + AOM_ICDF(21215), AOM_ICDF(22479), AOM_ICDF(23604), AOM_ICDF(24606), + AOM_ICDF(25499), AOM_ICDF(26294), AOM_ICDF(27002), AOM_ICDF(32768)}, + { AOM_ICDF(3328), AOM_ICDF(6318), AOM_ICDF(9004), AOM_ICDF(11417), + AOM_ICDF(13585), AOM_ICDF(15533), AOM_ICDF(17283), AOM_ICDF(18856), + AOM_ICDF(20269), AOM_ICDF(21538), AOM_ICDF(22678), AOM_ICDF(23703), + AOM_ICDF(24624), AOM_ICDF(25451), AOM_ICDF(26194), AOM_ICDF(32768)}, + { AOM_ICDF(3072), AOM_ICDF(5856), AOM_ICDF(8379), AOM_ICDF(10665), + AOM_ICDF(12737), AOM_ICDF(14615), AOM_ICDF(16317), AOM_ICDF(17859), + AOM_ICDF(19257), AOM_ICDF(20524), AOM_ICDF(21672), AOM_ICDF(22712), + AOM_ICDF(23655), AOM_ICDF(24509), AOM_ICDF(25283), AOM_ICDF(32768)}, + { AOM_ICDF(2816), AOM_ICDF(5390), AOM_ICDF(7743), AOM_ICDF(9894), + AOM_ICDF(11860), AOM_ICDF(13657), AOM_ICDF(15299), AOM_ICDF(16800), + AOM_ICDF(18172), AOM_ICDF(19426), AOM_ICDF(20573), AOM_ICDF(21621), + AOM_ICDF(22579), AOM_ICDF(23455), AOM_ICDF(24255), AOM_ICDF(32768)}, + { AOM_ICDF(2560), AOM_ICDF(4920), AOM_ICDF(7096), AOM_ICDF(9102), + AOM_ICDF(10951), AOM_ICDF(12656), AOM_ICDF(14227), AOM_ICDF(15676), + AOM_ICDF(17011), AOM_ICDF(18242), AOM_ICDF(19377), AOM_ICDF(20423), + AOM_ICDF(21388), AOM_ICDF(22277), AOM_ICDF(23097), AOM_ICDF(32768)}, + { AOM_ICDF(2304), AOM_ICDF(4446), AOM_ICDF(6437), AOM_ICDF(8288), + AOM_ICDF(10009), AOM_ICDF(11609), AOM_ICDF(13097), AOM_ICDF(14480), + AOM_ICDF(15766), AOM_ICDF(16961), AOM_ICDF(18072), AOM_ICDF(19105), + AOM_ICDF(20066), AOM_ICDF(20959), AOM_ICDF(21789), AOM_ICDF(32768)}, + { AOM_ICDF(2048), AOM_ICDF(3968), AOM_ICDF(5768), AOM_ICDF(7456), + AOM_ICDF(9038), AOM_ICDF(10521), AOM_ICDF(11911), AOM_ICDF(13215), + AOM_ICDF(14437), AOM_ICDF(15583), AOM_ICDF(16657), AOM_ICDF(17664), + AOM_ICDF(18608), AOM_ICDF(19493), AOM_ICDF(20323), AOM_ICDF(32768)}, + { AOM_ICDF(1792), AOM_ICDF(3486), AOM_ICDF(5087), AOM_ICDF(6601), + AOM_ICDF(8032), AOM_ICDF(9385), AOM_ICDF(10664), AOM_ICDF(11873), + AOM_ICDF(13016), AOM_ICDF(14096), AOM_ICDF(15117), AOM_ICDF(16082), + AOM_ICDF(16995), AOM_ICDF(17858), AOM_ICDF(18673), AOM_ICDF(32768)}, + { AOM_ICDF(1536), AOM_ICDF(3000), AOM_ICDF(4395), AOM_ICDF(5725), + AOM_ICDF(6993), AOM_ICDF(8201), AOM_ICDF(9353), AOM_ICDF(10451), + AOM_ICDF(11497), AOM_ICDF(12494), AOM_ICDF(13444), AOM_ICDF(14350), + AOM_ICDF(15213), AOM_ICDF(16036), AOM_ICDF(16820), AOM_ICDF(32768)}, + { AOM_ICDF(1280), AOM_ICDF(2510), AOM_ICDF(3692), AOM_ICDF(4828), + AOM_ICDF(5919), AOM_ICDF(6968), AOM_ICDF(7976), AOM_ICDF(8944), + AOM_ICDF(9875), AOM_ICDF(10769), AOM_ICDF(11628), AOM_ICDF(12454), + AOM_ICDF(13248), AOM_ICDF(14011), AOM_ICDF(14744), AOM_ICDF(32768)}, + { AOM_ICDF(1024), AOM_ICDF(2016), AOM_ICDF(2977), AOM_ICDF(3908), + AOM_ICDF(4810), AOM_ICDF(5684), AOM_ICDF(6530), AOM_ICDF(7350), + AOM_ICDF(8144), AOM_ICDF(8913), AOM_ICDF(9658), AOM_ICDF(10380), + AOM_ICDF(11080), AOM_ICDF(11758), AOM_ICDF(12415), AOM_ICDF(32768)}, + { AOM_ICDF(768), AOM_ICDF(1518), AOM_ICDF(2250), AOM_ICDF(2965), + AOM_ICDF(3663), AOM_ICDF(4345), AOM_ICDF(5011), AOM_ICDF(5662), + AOM_ICDF(6297), AOM_ICDF(6917), AOM_ICDF(7523), AOM_ICDF(8115), + AOM_ICDF(8693), AOM_ICDF(9257), AOM_ICDF(9808), AOM_ICDF(32768)}, + { AOM_ICDF(512), AOM_ICDF(1016), AOM_ICDF(1512), AOM_ICDF(2000), + AOM_ICDF(2481), AOM_ICDF(2954), AOM_ICDF(3420), AOM_ICDF(3879), + AOM_ICDF(4330), AOM_ICDF(4774), AOM_ICDF(5211), AOM_ICDF(5642), + AOM_ICDF(6066), AOM_ICDF(6483), AOM_ICDF(6894), AOM_ICDF(32768)}, + { AOM_ICDF(256), AOM_ICDF(510), AOM_ICDF(762), AOM_ICDF(1012), + AOM_ICDF(1260), AOM_ICDF(1506), AOM_ICDF(1750), AOM_ICDF(1992), + AOM_ICDF(2232), AOM_ICDF(2471), AOM_ICDF(2708), AOM_ICDF(2943), + AOM_ICDF(3176), AOM_ICDF(3407), AOM_ICDF(3636), AOM_ICDF(32768)}, +}; + + +const uint16_t LAPLACE_OFFSET[128] = { + 0, + 29871, + 28672, + 27751, + 26975, + 26291, + 25673, + 25105, + 24576, + 24079, + 23609, + 23162, + 22734, + 22325, + 21931, + 21550, + 21182, + 20826, + 20480, + 20143, + 19815, + 19495, + 19183, + 18877, + 18579, + 18286, + 17999, + 17718, + 17442, + 17170, + 16904, + 16642, + 16384, + 16129, + 15879, + 15633, + 15390, + 15150, + 14913, + 14680, + 14450, + 14222, + 13997, + 13775, + 13556, + 13338, + 13124, + 12911, + 12701, + 12493, + 12288, + 12084, + 11882, + 11682, + 11484, + 11288, + 11094, + 10901, + 10710, + 10521, + 10333, + 10147, + 9962, + 9779, + 9597, + 9417, + 9238, + 9060, + 8884, + 8709, + 8535, + 8363, + 8192, + 8021, + 7853, + 7685, + 7518, + 7352, + 7188, + 7025, + 6862, + 6701, + 6540, + 6381, + 6222, + 6065, + 5908, + 5753, + 5598, + 5444, + 5291, + 5138, + 4987, + 4837, + 4687, + 4538, + 4390, + 4242, + 4096, + 3950, + 3804, + 3660, + 3516, + 3373, + 3231, + 3089, + 2948, + 2808, + 2668, + 2529, + 2391, + 2253, + 2116, + 1979, + 1843, + 1708, + 1573, + 1439, + 1306, + 1172, + 1040, + 908, + 777, + 646, + 516, + 386, + 257, + 128, +}; diff --git a/third_party/aom/av1/common/mips/dspr2/av1_itrans16_dspr2.c b/third_party/aom/av1/common/mips/dspr2/av1_itrans16_dspr2.c new file mode 100644 index 0000000000..79f9338bd1 --- /dev/null +++ b/third_party/aom/av1/common/mips/dspr2/av1_itrans16_dspr2.c @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "av1/common/common.h" +#include "av1/common/blockd.h" +#include "av1/common/idct.h" +#include "aom_dsp/mips/inv_txfm_dspr2.h" +#include "aom_dsp/txfm_common.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +void av1_iht16x16_256_add_dspr2(const int16_t *input, uint8_t *dest, int pitch, + int tx_type) { + int i, j; + DECLARE_ALIGNED(32, int16_t, out[16 * 16]); + int16_t *outptr = out; + int16_t temp_out[16]; + uint32_t pos = 45; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" : : [pos] "r"(pos)); + + switch (tx_type) { + case DCT_DCT: // DCT in both horizontal and vertical + idct16_rows_dspr2(input, outptr, 16); + idct16_cols_add_blk_dspr2(out, dest, pitch); + break; + case ADST_DCT: // ADST in vertical, DCT in horizontal + idct16_rows_dspr2(input, outptr, 16); + + outptr = out; + + for (i = 0; i < 16; ++i) { + iadst16_dspr2(outptr, temp_out); + + for (j = 0; j < 16; ++j) + dest[j * pitch + i] = clip_pixel(ROUND_POWER_OF_TWO(temp_out[j], 6) + + dest[j * pitch + i]); + outptr += 16; + } + break; + case DCT_ADST: // DCT in vertical, ADST in horizontal + { + int16_t temp_in[16 * 16]; + + for (i = 0; i < 16; ++i) { + /* prefetch row */ + prefetch_load((const uint8_t *)(input + 16)); + + iadst16_dspr2(input, outptr); + input += 16; + outptr += 16; + } + + for (i = 0; i < 16; ++i) + for (j = 0; j < 16; ++j) temp_in[j * 16 + i] = out[i * 16 + j]; + + idct16_cols_add_blk_dspr2(temp_in, dest, pitch); + } break; + case ADST_ADST: // ADST in both directions + { + int16_t temp_in[16]; + + for (i = 0; i < 16; ++i) { + /* prefetch row */ + prefetch_load((const uint8_t *)(input + 16)); + + iadst16_dspr2(input, outptr); + input += 16; + outptr += 16; + } + + for (i = 0; i < 16; ++i) { + for (j = 0; j < 16; ++j) temp_in[j] = out[j * 16 + i]; + iadst16_dspr2(temp_in, temp_out); + for (j = 0; j < 16; ++j) + dest[j * pitch + i] = clip_pixel(ROUND_POWER_OF_TWO(temp_out[j], 6) + + dest[j * pitch + i]); + } + } break; + default: printf("av1_short_iht16x16_add_dspr2 : Invalid tx_type\n"); break; + } +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/av1/common/mips/dspr2/av1_itrans4_dspr2.c b/third_party/aom/av1/common/mips/dspr2/av1_itrans4_dspr2.c new file mode 100644 index 0000000000..0a9552376d --- /dev/null +++ b/third_party/aom/av1/common/mips/dspr2/av1_itrans4_dspr2.c @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "av1/common/common.h" +#include "av1/common/blockd.h" +#include "av1/common/idct.h" +#include "aom_dsp/mips/inv_txfm_dspr2.h" +#include "aom_dsp/txfm_common.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +void av1_iht4x4_16_add_dspr2(const int16_t *input, uint8_t *dest, + int dest_stride, int tx_type) { + int i, j; + DECLARE_ALIGNED(32, int16_t, out[4 * 4]); + int16_t *outptr = out; + int16_t temp_in[4 * 4], temp_out[4]; + uint32_t pos = 45; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" + : + : [pos] "r"(pos)); + + switch (tx_type) { + case DCT_DCT: // DCT in both horizontal and vertical + aom_idct4_rows_dspr2(input, outptr); + aom_idct4_columns_add_blk_dspr2(&out[0], dest, dest_stride); + break; + case ADST_DCT: // ADST in vertical, DCT in horizontal + aom_idct4_rows_dspr2(input, outptr); + + outptr = out; + + for (i = 0; i < 4; ++i) { + iadst4_dspr2(outptr, temp_out); + + for (j = 0; j < 4; ++j) + dest[j * dest_stride + i] = clip_pixel( + ROUND_POWER_OF_TWO(temp_out[j], 4) + dest[j * dest_stride + i]); + + outptr += 4; + } + break; + case DCT_ADST: // DCT in vertical, ADST in horizontal + for (i = 0; i < 4; ++i) { + iadst4_dspr2(input, outptr); + input += 4; + outptr += 4; + } + + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) { + temp_in[i * 4 + j] = out[j * 4 + i]; + } + } + aom_idct4_columns_add_blk_dspr2(&temp_in[0], dest, dest_stride); + break; + case ADST_ADST: // ADST in both directions + for (i = 0; i < 4; ++i) { + iadst4_dspr2(input, outptr); + input += 4; + outptr += 4; + } + + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) temp_in[j] = out[j * 4 + i]; + iadst4_dspr2(temp_in, temp_out); + + for (j = 0; j < 4; ++j) + dest[j * dest_stride + i] = clip_pixel( + ROUND_POWER_OF_TWO(temp_out[j], 4) + dest[j * dest_stride + i]); + } + break; + default: printf("av1_short_iht4x4_add_dspr2 : Invalid tx_type\n"); break; + } +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/av1/common/mips/dspr2/av1_itrans8_dspr2.c b/third_party/aom/av1/common/mips/dspr2/av1_itrans8_dspr2.c new file mode 100644 index 0000000000..8bf5b4f0e5 --- /dev/null +++ b/third_party/aom/av1/common/mips/dspr2/av1_itrans8_dspr2.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "av1/common/common.h" +#include "av1/common/blockd.h" +#include "aom_dsp/mips/inv_txfm_dspr2.h" +#include "aom_dsp/txfm_common.h" +#include "aom_ports/mem.h" + +#if HAVE_DSPR2 +void av1_iht8x8_64_add_dspr2(const int16_t *input, uint8_t *dest, + int dest_stride, int tx_type) { + int i, j; + DECLARE_ALIGNED(32, int16_t, out[8 * 8]); + int16_t *outptr = out; + int16_t temp_in[8 * 8], temp_out[8]; + uint32_t pos = 45; + + /* bit positon for extract from acc */ + __asm__ __volatile__("wrdsp %[pos], 1 \n\t" : : [pos] "r"(pos)); + + switch (tx_type) { + case DCT_DCT: // DCT in both horizontal and vertical + idct8_rows_dspr2(input, outptr, 8); + idct8_columns_add_blk_dspr2(&out[0], dest, dest_stride); + break; + case ADST_DCT: // ADST in vertical, DCT in horizontal + idct8_rows_dspr2(input, outptr, 8); + + for (i = 0; i < 8; ++i) { + iadst8_dspr2(&out[i * 8], temp_out); + + for (j = 0; j < 8; ++j) + dest[j * dest_stride + i] = clip_pixel( + ROUND_POWER_OF_TWO(temp_out[j], 5) + dest[j * dest_stride + i]); + } + break; + case DCT_ADST: // DCT in vertical, ADST in horizontal + for (i = 0; i < 8; ++i) { + iadst8_dspr2(input, outptr); + input += 8; + outptr += 8; + } + + for (i = 0; i < 8; ++i) { + for (j = 0; j < 8; ++j) { + temp_in[i * 8 + j] = out[j * 8 + i]; + } + } + idct8_columns_add_blk_dspr2(&temp_in[0], dest, dest_stride); + break; + case ADST_ADST: // ADST in both directions + for (i = 0; i < 8; ++i) { + iadst8_dspr2(input, outptr); + input += 8; + outptr += 8; + } + + for (i = 0; i < 8; ++i) { + for (j = 0; j < 8; ++j) temp_in[j] = out[j * 8 + i]; + + iadst8_dspr2(temp_in, temp_out); + + for (j = 0; j < 8; ++j) + dest[j * dest_stride + i] = clip_pixel( + ROUND_POWER_OF_TWO(temp_out[j], 5) + dest[j * dest_stride + i]); + } + break; + default: printf("av1_short_iht8x8_add_dspr2 : Invalid tx_type\n"); break; + } +} +#endif // #if HAVE_DSPR2 diff --git a/third_party/aom/av1/common/mips/msa/av1_idct16x16_msa.c b/third_party/aom/av1/common/mips/msa/av1_idct16x16_msa.c new file mode 100644 index 0000000000..4bd0a16356 --- /dev/null +++ b/third_party/aom/av1/common/mips/msa/av1_idct16x16_msa.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/enums.h" +#include "aom_dsp/mips/inv_txfm_msa.h" + +void av1_iht16x16_256_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride, int32_t tx_type) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, out[16 * 16]); + int16_t *out_ptr = &out[0]; + + switch (tx_type) { + case DCT_DCT: + /* transform rows */ + for (i = 0; i < 2; ++i) { + /* process 16 * 8 block */ + aom_idct16_1d_rows_msa((input + (i << 7)), (out_ptr + (i << 7))); + } + + /* transform columns */ + for (i = 0; i < 2; ++i) { + /* process 8 * 16 block */ + aom_idct16_1d_columns_addblk_msa((out_ptr + (i << 3)), (dst + (i << 3)), + dst_stride); + } + break; + case ADST_DCT: + /* transform rows */ + for (i = 0; i < 2; ++i) { + /* process 16 * 8 block */ + aom_idct16_1d_rows_msa((input + (i << 7)), (out_ptr + (i << 7))); + } + + /* transform columns */ + for (i = 0; i < 2; ++i) { + aom_iadst16_1d_columns_addblk_msa((out_ptr + (i << 3)), + (dst + (i << 3)), dst_stride); + } + break; + case DCT_ADST: + /* transform rows */ + for (i = 0; i < 2; ++i) { + /* process 16 * 8 block */ + aom_iadst16_1d_rows_msa((input + (i << 7)), (out_ptr + (i << 7))); + } + + /* transform columns */ + for (i = 0; i < 2; ++i) { + /* process 8 * 16 block */ + aom_idct16_1d_columns_addblk_msa((out_ptr + (i << 3)), (dst + (i << 3)), + dst_stride); + } + break; + case ADST_ADST: + /* transform rows */ + for (i = 0; i < 2; ++i) { + /* process 16 * 8 block */ + aom_iadst16_1d_rows_msa((input + (i << 7)), (out_ptr + (i << 7))); + } + + /* transform columns */ + for (i = 0; i < 2; ++i) { + aom_iadst16_1d_columns_addblk_msa((out_ptr + (i << 3)), + (dst + (i << 3)), dst_stride); + } + break; + default: assert(0); break; + } +} diff --git a/third_party/aom/av1/common/mips/msa/av1_idct4x4_msa.c b/third_party/aom/av1/common/mips/msa/av1_idct4x4_msa.c new file mode 100644 index 0000000000..8364f8dc44 --- /dev/null +++ b/third_party/aom/av1/common/mips/msa/av1_idct4x4_msa.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/enums.h" +#include "aom_dsp/mips/inv_txfm_msa.h" + +void av1_iht4x4_16_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride, int32_t tx_type) { + v8i16 in0, in1, in2, in3; + + /* load vector elements of 4x4 block */ + LD4x4_SH(input, in0, in1, in2, in3); + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + + switch (tx_type) { + case DCT_DCT: + /* DCT in horizontal */ + AOM_IDCT4x4(in0, in1, in2, in3, in0, in1, in2, in3); + /* DCT in vertical */ + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + AOM_IDCT4x4(in0, in1, in2, in3, in0, in1, in2, in3); + break; + case ADST_DCT: + /* DCT in horizontal */ + AOM_IDCT4x4(in0, in1, in2, in3, in0, in1, in2, in3); + /* ADST in vertical */ + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + AOM_IADST4x4(in0, in1, in2, in3, in0, in1, in2, in3); + break; + case DCT_ADST: + /* ADST in horizontal */ + AOM_IADST4x4(in0, in1, in2, in3, in0, in1, in2, in3); + /* DCT in vertical */ + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + AOM_IDCT4x4(in0, in1, in2, in3, in0, in1, in2, in3); + break; + case ADST_ADST: + /* ADST in horizontal */ + AOM_IADST4x4(in0, in1, in2, in3, in0, in1, in2, in3); + /* ADST in vertical */ + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + AOM_IADST4x4(in0, in1, in2, in3, in0, in1, in2, in3); + break; + default: assert(0); break; + } + + /* final rounding (add 2^3, divide by 2^4) and shift */ + SRARI_H4_SH(in0, in1, in2, in3, 4); + /* add block and store 4x4 */ + ADDBLK_ST4x4_UB(in0, in1, in2, in3, dst, dst_stride); +} diff --git a/third_party/aom/av1/common/mips/msa/av1_idct8x8_msa.c b/third_party/aom/av1/common/mips/msa/av1_idct8x8_msa.c new file mode 100644 index 0000000000..71117051bc --- /dev/null +++ b/third_party/aom/av1/common/mips/msa/av1_idct8x8_msa.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/enums.h" +#include "aom_dsp/mips/inv_txfm_msa.h" + +void av1_iht8x8_64_add_msa(const int16_t *input, uint8_t *dst, + int32_t dst_stride, int32_t tx_type) { + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + + /* load vector elements of 8x8 block */ + LD_SH8(input, 8, in0, in1, in2, in3, in4, in5, in6, in7); + + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + + switch (tx_type) { + case DCT_DCT: + /* DCT in horizontal */ + AOM_IDCT8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + /* DCT in vertical */ + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, + in3, in4, in5, in6, in7); + AOM_IDCT8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + break; + case ADST_DCT: + /* DCT in horizontal */ + AOM_IDCT8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + /* ADST in vertical */ + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, + in3, in4, in5, in6, in7); + AOM_ADST8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + break; + case DCT_ADST: + /* ADST in horizontal */ + AOM_ADST8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + /* DCT in vertical */ + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, + in3, in4, in5, in6, in7); + AOM_IDCT8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + break; + case ADST_ADST: + /* ADST in horizontal */ + AOM_ADST8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + /* ADST in vertical */ + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, + in3, in4, in5, in6, in7); + AOM_ADST8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + break; + default: assert(0); break; + } + + /* final rounding (add 2^4, divide by 2^5) and shift */ + SRARI_H4_SH(in0, in1, in2, in3, 5); + SRARI_H4_SH(in4, in5, in6, in7, 5); + + /* add block and store 8x8 */ + AOM_ADDBLK_ST8x4_UB(dst, dst_stride, in0, in1, in2, in3); + dst += (4 * dst_stride); + AOM_ADDBLK_ST8x4_UB(dst, dst_stride, in4, in5, in6, in7); +} diff --git a/third_party/aom/av1/common/mv.h b/third_party/aom/av1/common/mv.h new file mode 100644 index 0000000000..d4df3790f3 --- /dev/null +++ b/third_party/aom/av1/common/mv.h @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_MV_H_ +#define AV1_COMMON_MV_H_ + +#include "av1/common/common.h" +#include "av1/common/common_data.h" +#include "aom_dsp/aom_filter.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct mv { + int16_t row; + int16_t col; +} MV; + +typedef union int_mv { + uint32_t as_int; + MV as_mv; +} int_mv; /* facilitates faster equality tests and copies */ + +typedef struct mv32 { + int32_t row; + int32_t col; +} MV32; + +#if (CONFIG_WARPED_MOTION || CONFIG_MOTION_VAR) && CONFIG_GLOBAL_MOTION +#define SEPARATE_GLOBAL_MOTION 1 +#endif // (CONFIG_WARPED_MOTION || CONFIG_MOTION_VAR) && CONFIG_GLOBAL_MOTION +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +// Bits of precision used for the model +#define WARPEDMODEL_PREC_BITS 16 +#define WARPEDMODEL_ROW3HOMO_PREC_BITS 16 + +#define WARPEDMODEL_TRANS_CLAMP (128 << WARPEDMODEL_PREC_BITS) +#define WARPEDMODEL_DIAGAFFINE_CLAMP (1 << (WARPEDMODEL_PREC_BITS + 1)) +#define WARPEDMODEL_NONDIAGAFFINE_CLAMP (1 << (WARPEDMODEL_PREC_BITS - 1)) +#define WARPEDMODEL_ROW3HOMO_CLAMP (1 << (WARPEDMODEL_PREC_BITS - 1)) + +// Bits of subpel precision for warped interpolation +#define WARPEDPIXEL_PREC_BITS 6 +#define WARPEDPIXEL_PREC_SHIFTS (1 << WARPEDPIXEL_PREC_BITS) + +// Taps for ntap filter +#define WARPEDPIXEL_FILTER_TAPS 6 + +// Precision of filter taps +#define WARPEDPIXEL_FILTER_BITS 7 + +// Precision bits reduction after horizontal shear +#define HORSHEAR_REDUCE_PREC_BITS 5 +#define VERSHEAR_REDUCE_PREC_BITS \ + (2 * WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS) + +#define WARPEDDIFF_PREC_BITS (WARPEDMODEL_PREC_BITS - WARPEDPIXEL_PREC_BITS) + +/* clang-format off */ +typedef enum { + IDENTITY = 0, // identity transformation, 0-parameter + TRANSLATION = 1, // translational motion 2-parameter + ROTZOOM = 2, // simplified affine with rotation + zoom only, 4-parameter + AFFINE = 3, // affine, 6-parameter + HORTRAPEZOID = 4, // constrained homography, hor trapezoid, 6-parameter + VERTRAPEZOID = 5, // constrained homography, ver trapezoid, 6-parameter + HOMOGRAPHY = 6, // homography, 8-parameter + TRANS_TYPES = 7, +} TransformationType; +/* clang-format on */ + +// Number of types used for global motion (must be >= 3 and <= TRANS_TYPES) +// The following can be useful: +// GLOBAL_TRANS_TYPES 3 - up to rotation-zoom +// GLOBAL_TRANS_TYPES 4 - up to affine +// GLOBAL_TRANS_TYPES 6 - up to hor/ver trapezoids +// GLOBAL_TRANS_TYPES 7 - up to full homography +#define GLOBAL_TRANS_TYPES 4 + +typedef struct { +#if CONFIG_GLOBAL_MOTION + int global_warp_allowed; +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_WARPED_MOTION + int local_warp_allowed; +#endif // CONFIG_WARPED_MOTION +} WarpTypesAllowed; + +// number of parameters used by each transformation in TransformationTypes +static const int trans_model_params[TRANS_TYPES] = { 0, 2, 4, 6, 6, 6, 8 }; + +// The order of values in the wmmat matrix below is best described +// by the homography: +// [x' (m2 m3 m0 [x +// z . y' = m4 m5 m1 * y +// 1] m6 m7 1) 1] +typedef struct { + TransformationType wmtype; + int32_t wmmat[8]; + int16_t alpha, beta, gamma, delta; +} WarpedMotionParams; + +static INLINE void set_default_warp_params(WarpedMotionParams *wm) { + static const int32_t default_wm_mat[8] = { + 0, 0, (1 << WARPEDMODEL_PREC_BITS), 0, 0, (1 << WARPEDMODEL_PREC_BITS), 0, 0 + }; + memset(wm, 0, sizeof(*wm)); + memcpy(wm->wmmat, default_wm_mat, sizeof(wm->wmmat)); + wm->wmtype = IDENTITY; +} +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + +#if CONFIG_GLOBAL_MOTION +// The following constants describe the various precisions +// of different parameters in the global motion experiment. +// +// Given the general homography: +// [x' (a b c [x +// z . y' = d e f * y +// 1] g h i) 1] +// +// Constants using the name ALPHA here are related to parameters +// a, b, d, e. Constants using the name TRANS are related +// to parameters c and f. +// +// Anything ending in PREC_BITS is the number of bits of precision +// to maintain when converting from double to integer. +// +// The ABS parameters are used to create an upper and lower bound +// for each parameter. In other words, after a parameter is integerized +// it is clamped between -(1 << ABS_XXX_BITS) and (1 << ABS_XXX_BITS). +// +// XXX_PREC_DIFF and XXX_DECODE_FACTOR +// are computed once here to prevent repetitive +// computation on the decoder side. These are +// to allow the global motion parameters to be encoded in a lower +// precision than the warped model precision. This means that they +// need to be changed to warped precision when they are decoded. +// +// XX_MIN, XX_MAX are also computed to avoid repeated computation + +#define SUBEXPFIN_K 3 +#define GM_TRANS_PREC_BITS 6 +#define GM_ABS_TRANS_BITS 12 +#define GM_ABS_TRANS_ONLY_BITS (GM_ABS_TRANS_BITS - GM_TRANS_PREC_BITS + 3) +#define GM_TRANS_PREC_DIFF (WARPEDMODEL_PREC_BITS - GM_TRANS_PREC_BITS) +#define GM_TRANS_ONLY_PREC_DIFF (WARPEDMODEL_PREC_BITS - 3) +#define GM_TRANS_DECODE_FACTOR (1 << GM_TRANS_PREC_DIFF) +#define GM_TRANS_ONLY_DECODE_FACTOR (1 << GM_TRANS_ONLY_PREC_DIFF) + +#define GM_ALPHA_PREC_BITS 15 +#define GM_ABS_ALPHA_BITS 12 +#define GM_ALPHA_PREC_DIFF (WARPEDMODEL_PREC_BITS - GM_ALPHA_PREC_BITS) +#define GM_ALPHA_DECODE_FACTOR (1 << GM_ALPHA_PREC_DIFF) + +#define GM_ROW3HOMO_PREC_BITS 16 +#define GM_ABS_ROW3HOMO_BITS 11 +#define GM_ROW3HOMO_PREC_DIFF \ + (WARPEDMODEL_ROW3HOMO_PREC_BITS - GM_ROW3HOMO_PREC_BITS) +#define GM_ROW3HOMO_DECODE_FACTOR (1 << GM_ROW3HOMO_PREC_DIFF) + +#define GM_TRANS_MAX (1 << GM_ABS_TRANS_BITS) +#define GM_ALPHA_MAX (1 << GM_ABS_ALPHA_BITS) +#define GM_ROW3HOMO_MAX (1 << GM_ABS_ROW3HOMO_BITS) + +#define GM_TRANS_MIN -GM_TRANS_MAX +#define GM_ALPHA_MIN -GM_ALPHA_MAX +#define GM_ROW3HOMO_MIN -GM_ROW3HOMO_MAX + +// Use global motion parameters for sub8x8 blocks +#define GLOBAL_SUB8X8_USED 0 + +static INLINE int block_center_x(int mi_col, BLOCK_SIZE bs) { + const int bw = block_size_wide[bs]; + return mi_col * MI_SIZE + bw / 2 - 1; +} + +static INLINE int block_center_y(int mi_row, BLOCK_SIZE bs) { + const int bh = block_size_high[bs]; + return mi_row * MI_SIZE + bh / 2 - 1; +} + +static INLINE int convert_to_trans_prec(int allow_hp, int coor) { + if (allow_hp) + return ROUND_POWER_OF_TWO_SIGNED(coor, WARPEDMODEL_PREC_BITS - 3); + else + return ROUND_POWER_OF_TWO_SIGNED(coor, WARPEDMODEL_PREC_BITS - 2) * 2; +} + +// Convert a global motion translation vector (which may have more bits than a +// regular motion vector) into a motion vector +static INLINE int_mv gm_get_motion_vector(const WarpedMotionParams *gm, + int allow_hp, BLOCK_SIZE bsize, + int mi_col, int mi_row, + int block_idx) { + const int unify_bsize = CONFIG_CB4X4; + int_mv res; + const int32_t *mat = gm->wmmat; + int x, y, tx, ty; + + if (gm->wmtype == TRANSLATION) { + res.as_mv.row = gm->wmmat[0] >> GM_TRANS_ONLY_PREC_DIFF; + res.as_mv.col = gm->wmmat[1] >> GM_TRANS_ONLY_PREC_DIFF; + return res; + } + + if (bsize >= BLOCK_8X8 || unify_bsize) { + x = block_center_x(mi_col, bsize); + y = block_center_y(mi_row, bsize); + } else { + x = block_center_x(mi_col, bsize); + y = block_center_y(mi_row, bsize); + x += (block_idx & 1) * MI_SIZE / 2; + y += (block_idx & 2) * MI_SIZE / 4; + } + + if (gm->wmtype == ROTZOOM) { + assert(gm->wmmat[5] == gm->wmmat[2]); + assert(gm->wmmat[4] == -gm->wmmat[3]); + } + if (gm->wmtype > AFFINE) { + int xc = (int)((int64_t)mat[2] * x + (int64_t)mat[3] * y + mat[0]); + int yc = (int)((int64_t)mat[4] * x + (int64_t)mat[5] * y + mat[1]); + const int Z = (int)((int64_t)mat[6] * x + (int64_t)mat[7] * y + + (1 << WARPEDMODEL_ROW3HOMO_PREC_BITS)); + xc *= 1 << (WARPEDMODEL_ROW3HOMO_PREC_BITS - WARPEDMODEL_PREC_BITS); + yc *= 1 << (WARPEDMODEL_ROW3HOMO_PREC_BITS - WARPEDMODEL_PREC_BITS); + xc = (int)(xc > 0 ? ((int64_t)xc + Z / 2) / Z : ((int64_t)xc - Z / 2) / Z); + yc = (int)(yc > 0 ? ((int64_t)yc + Z / 2) / Z : ((int64_t)yc - Z / 2) / Z); + tx = convert_to_trans_prec(allow_hp, xc) - (x << 3); + ty = convert_to_trans_prec(allow_hp, yc) - (y << 3); + } else { + const int xc = + (mat[2] - (1 << WARPEDMODEL_PREC_BITS)) * x + mat[3] * y + mat[0]; + const int yc = + mat[4] * x + (mat[5] - (1 << WARPEDMODEL_PREC_BITS)) * y + mat[1]; + tx = convert_to_trans_prec(allow_hp, xc); + ty = convert_to_trans_prec(allow_hp, yc); + } + + res.as_mv.row = ty; + res.as_mv.col = tx; + return res; +} + +static INLINE TransformationType get_gmtype(const WarpedMotionParams *gm) { + if (gm->wmmat[6] != 0 || gm->wmmat[7] != 0) { + if (!gm->wmmat[6] && !gm->wmmat[4]) return HORTRAPEZOID; + if (!gm->wmmat[7] && !gm->wmmat[3]) return VERTRAPEZOID; + return HOMOGRAPHY; + } + if (gm->wmmat[5] == (1 << WARPEDMODEL_PREC_BITS) && !gm->wmmat[4] && + gm->wmmat[2] == (1 << WARPEDMODEL_PREC_BITS) && !gm->wmmat[3]) { + return ((!gm->wmmat[1] && !gm->wmmat[0]) ? IDENTITY : TRANSLATION); + } + if (gm->wmmat[2] == gm->wmmat[5] && gm->wmmat[3] == -gm->wmmat[4]) + return ROTZOOM; + else + return AFFINE; +} +#endif // CONFIG_GLOBAL_MOTION + +#if CONFIG_REF_MV +typedef struct candidate_mv { + int_mv this_mv; + int_mv comp_mv; + uint8_t pred_diff[2]; + int weight; +} CANDIDATE_MV; +#endif + +static INLINE int is_zero_mv(const MV *mv) { + return *((const uint32_t *)mv) == 0; +} + +static INLINE int is_equal_mv(const MV *a, const MV *b) { + return *((const uint32_t *)a) == *((const uint32_t *)b); +} + +static INLINE void clamp_mv(MV *mv, int min_col, int max_col, int min_row, + int max_row) { + mv->col = clamp(mv->col, min_col, max_col); + mv->row = clamp(mv->row, min_row, max_row); +} + +static INLINE int mv_has_subpel(const MV *mv) { + return (mv->row & SUBPEL_MASK) || (mv->col & SUBPEL_MASK); +} +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_MV_H_ diff --git a/third_party/aom/av1/common/mvref_common.c b/third_party/aom/av1/common/mvref_common.c new file mode 100644 index 0000000000..5222948c88 --- /dev/null +++ b/third_party/aom/av1/common/mvref_common.c @@ -0,0 +1,1164 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/common/mvref_common.h" +#if CONFIG_WARPED_MOTION +#include "av1/common/warped_motion.h" +#endif // CONFIG_WARPED_MOTION + +#if CONFIG_REF_MV + +static uint8_t add_ref_mv_candidate( + const MODE_INFO *const candidate_mi, const MB_MODE_INFO *const candidate, + const MV_REFERENCE_FRAME rf[2], uint8_t *refmv_count, + CANDIDATE_MV *ref_mv_stack, const int use_hp, int len, int block, int col) { + int index = 0, ref; + int newmv_count = 0; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + + if (rf[1] == NONE_FRAME) { + // single reference frame + for (ref = 0; ref < 2; ++ref) { + if (candidate->ref_frame[ref] == rf[0]) { + int_mv this_refmv = get_sub_block_mv(candidate_mi, ref, col, block); + lower_mv_precision(&this_refmv.as_mv, use_hp); + + for (index = 0; index < *refmv_count; ++index) + if (ref_mv_stack[index].this_mv.as_int == this_refmv.as_int) break; + + if (index < *refmv_count) ref_mv_stack[index].weight += 2 * len; + + // Add a new item to the list. + if (index == *refmv_count) { + ref_mv_stack[index].this_mv = this_refmv; + ref_mv_stack[index].pred_diff[0] = av1_get_pred_diff_ctx( + get_sub_block_pred_mv(candidate_mi, ref, col, block), this_refmv); + ref_mv_stack[index].weight = 2 * len; + ++(*refmv_count); + + if (candidate->mode == NEWMV) ++newmv_count; + } + + if (candidate_mi->mbmi.sb_type < BLOCK_8X8 && block >= 0 && + !unify_bsize) { + int alt_block = 3 - block; + this_refmv = get_sub_block_mv(candidate_mi, ref, col, alt_block); + lower_mv_precision(&this_refmv.as_mv, use_hp); + + for (index = 0; index < *refmv_count; ++index) + if (ref_mv_stack[index].this_mv.as_int == this_refmv.as_int) break; + + if (index < *refmv_count) ref_mv_stack[index].weight += len; + + // Add a new item to the list. + if (index == *refmv_count) { + ref_mv_stack[index].this_mv = this_refmv; + ref_mv_stack[index].pred_diff[0] = av1_get_pred_diff_ctx( + get_sub_block_pred_mv(candidate_mi, ref, col, alt_block), + this_refmv); + ref_mv_stack[index].weight = len; + ++(*refmv_count); + + if (candidate->mode == NEWMV) ++newmv_count; + } + } + } + } + } else { + // compound reference frame + if (candidate->ref_frame[0] == rf[0] && candidate->ref_frame[1] == rf[1]) { + int_mv this_refmv[2]; + + for (ref = 0; ref < 2; ++ref) { + this_refmv[ref] = get_sub_block_mv(candidate_mi, ref, col, block); + lower_mv_precision(&this_refmv[ref].as_mv, use_hp); + } + + for (index = 0; index < *refmv_count; ++index) + if ((ref_mv_stack[index].this_mv.as_int == this_refmv[0].as_int) && + (ref_mv_stack[index].comp_mv.as_int == this_refmv[1].as_int)) + break; + + if (index < *refmv_count) ref_mv_stack[index].weight += 2 * len; + + // Add a new item to the list. + if (index == *refmv_count) { + ref_mv_stack[index].this_mv = this_refmv[0]; + ref_mv_stack[index].comp_mv = this_refmv[1]; + ref_mv_stack[index].pred_diff[0] = av1_get_pred_diff_ctx( + get_sub_block_pred_mv(candidate_mi, 0, col, block), this_refmv[0]); + ref_mv_stack[index].pred_diff[1] = av1_get_pred_diff_ctx( + get_sub_block_pred_mv(candidate_mi, 1, col, block), this_refmv[1]); + ref_mv_stack[index].weight = 2 * len; + ++(*refmv_count); + +#if CONFIG_EXT_INTER + if (candidate->mode == NEW_NEWMV) +#else + if (candidate->mode == NEWMV) +#endif // CONFIG_EXT_INTER + ++newmv_count; + } + + if (candidate_mi->mbmi.sb_type < BLOCK_8X8 && block >= 0 && + !unify_bsize) { + int alt_block = 3 - block; + this_refmv[0] = get_sub_block_mv(candidate_mi, 0, col, alt_block); + this_refmv[1] = get_sub_block_mv(candidate_mi, 1, col, alt_block); + + for (ref = 0; ref < 2; ++ref) + lower_mv_precision(&this_refmv[ref].as_mv, use_hp); + + for (index = 0; index < *refmv_count; ++index) + if (ref_mv_stack[index].this_mv.as_int == this_refmv[0].as_int && + ref_mv_stack[index].comp_mv.as_int == this_refmv[1].as_int) + break; + + if (index < *refmv_count) ref_mv_stack[index].weight += len; + + // Add a new item to the list. + if (index == *refmv_count) { + ref_mv_stack[index].this_mv = this_refmv[0]; + ref_mv_stack[index].comp_mv = this_refmv[1]; + ref_mv_stack[index].pred_diff[0] = av1_get_pred_diff_ctx( + get_sub_block_pred_mv(candidate_mi, 0, col, block), + this_refmv[0]); + ref_mv_stack[index].pred_diff[0] = av1_get_pred_diff_ctx( + get_sub_block_pred_mv(candidate_mi, 1, col, block), + this_refmv[1]); + ref_mv_stack[index].weight = len; + ++(*refmv_count); + +#if CONFIG_EXT_INTER + if (candidate->mode == NEW_NEWMV) +#else + if (candidate->mode == NEWMV) +#endif // CONFIG_EXT_INTER + ++newmv_count; + } + } + } + } + return newmv_count; +} + +static uint8_t scan_row_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, + const int mi_row, const int mi_col, int block, + const MV_REFERENCE_FRAME rf[2], int row_offset, + CANDIDATE_MV *ref_mv_stack, uint8_t *refmv_count) { + const TileInfo *const tile = &xd->tile; + int i; + uint8_t newmv_count = 0; +#if CONFIG_CB4X4 + const int bsize = xd->mi[0]->mbmi.sb_type; + const int mi_offset = + bsize < BLOCK_8X8 ? mi_size_wide[BLOCK_4X4] : mi_size_wide[BLOCK_8X8]; + // TODO(jingning): Revisit this part after cb4x4 is stable. + if (bsize >= BLOCK_8X8) row_offset *= 2; +#else + const int mi_offset = mi_size_wide[BLOCK_8X8]; +#endif + + for (i = 0; i < xd->n8_w && *refmv_count < MAX_REF_MV_STACK_SIZE;) { + POSITION mi_pos; +#if CONFIG_CB4X4 + const int use_step_16 = (xd->n8_w >= 16); +#else + const int use_step_16 = (xd->n8_w >= 8); +#endif + + mi_pos.row = row_offset; + mi_pos.col = i; + if (is_inside(tile, mi_col, mi_row, cm->mi_rows, cm, &mi_pos)) { + const MODE_INFO *const candidate_mi = + xd->mi[mi_pos.row * xd->mi_stride + mi_pos.col]; + const MB_MODE_INFO *const candidate = &candidate_mi->mbmi; + int len = AOMMIN(xd->n8_w, mi_size_wide[candidate->sb_type]); + if (use_step_16) len = AOMMAX(mi_size_wide[BLOCK_16X16], len); + newmv_count += add_ref_mv_candidate( + candidate_mi, candidate, rf, refmv_count, ref_mv_stack, + cm->allow_high_precision_mv, len, block, mi_pos.col); + i += len; + } else { + if (use_step_16) + i += (mi_offset << 1); + else + i += mi_offset; + } + } + + return newmv_count; +} + +static uint8_t scan_col_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, + const int mi_row, const int mi_col, int block, + const MV_REFERENCE_FRAME rf[2], int col_offset, + CANDIDATE_MV *ref_mv_stack, uint8_t *refmv_count) { + const TileInfo *const tile = &xd->tile; + int i; + uint8_t newmv_count = 0; +#if CONFIG_CB4X4 + const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + const int mi_offset = + (bsize < BLOCK_8X8) ? mi_size_high[BLOCK_4X4] : mi_size_high[BLOCK_8X8]; + if (bsize >= BLOCK_8X8) col_offset *= 2; +#else + const int mi_offset = mi_size_wide[BLOCK_8X8]; +#endif + + for (i = 0; i < xd->n8_h && *refmv_count < MAX_REF_MV_STACK_SIZE;) { + POSITION mi_pos; +#if CONFIG_CB4X4 + const int use_step_16 = (xd->n8_h >= 16); +#else + const int use_step_16 = (xd->n8_h >= 8); +#endif + + mi_pos.row = i; + mi_pos.col = col_offset; + if (is_inside(tile, mi_col, mi_row, cm->mi_rows, cm, &mi_pos)) { + const MODE_INFO *const candidate_mi = + xd->mi[mi_pos.row * xd->mi_stride + mi_pos.col]; + const MB_MODE_INFO *const candidate = &candidate_mi->mbmi; + int len = AOMMIN(xd->n8_h, mi_size_high[candidate->sb_type]); + if (use_step_16) len = AOMMAX(mi_size_high[BLOCK_16X16], len); + newmv_count += add_ref_mv_candidate( + candidate_mi, candidate, rf, refmv_count, ref_mv_stack, + cm->allow_high_precision_mv, len, block, mi_pos.col); + i += len; + } else { + if (use_step_16) + i += (mi_offset << 1); + else + i += mi_offset; + } + } + + return newmv_count; +} + +static uint8_t scan_blk_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, + const int mi_row, const int mi_col, int block, + const MV_REFERENCE_FRAME rf[2], int row_offset, + int col_offset, CANDIDATE_MV *ref_mv_stack, + uint8_t *refmv_count) { + const TileInfo *const tile = &xd->tile; + POSITION mi_pos; + uint8_t newmv_count = 0; + + mi_pos.row = row_offset; + mi_pos.col = col_offset; + + if (is_inside(tile, mi_col, mi_row, cm->mi_rows, cm, &mi_pos) && + *refmv_count < MAX_REF_MV_STACK_SIZE) { + const MODE_INFO *const candidate_mi = + xd->mi[mi_pos.row * xd->mi_stride + mi_pos.col]; + const MB_MODE_INFO *const candidate = &candidate_mi->mbmi; + const int len = mi_size_wide[BLOCK_8X8]; + + newmv_count += add_ref_mv_candidate( + candidate_mi, candidate, rf, refmv_count, ref_mv_stack, + cm->allow_high_precision_mv, len, block, mi_pos.col); + } // Analyze a single 8x8 block motion information. + + return newmv_count; +} + +static int has_top_right(const MACROBLOCKD *xd, int mi_row, int mi_col, + int bs) { + const int mask_row = mi_row & MAX_MIB_MASK; + const int mask_col = mi_col & MAX_MIB_MASK; + + // In a split partition all apart from the bottom right has a top right + int has_tr = !((mask_row & bs) && (mask_col & bs)); + + // bs > 0 and bs is a power of 2 + assert(bs > 0 && !(bs & (bs - 1))); + + // For each 4x4 group of blocks, when the bottom right is decoded the blocks + // to the right have not been decoded therefore the bottom right does + // not have a top right + while (bs < MAX_MIB_SIZE) { + if (mask_col & bs) { + if ((mask_col & (2 * bs)) && (mask_row & (2 * bs))) { + has_tr = 0; + break; + } + } else { + break; + } + bs <<= 1; + } + + // The left hand of two vertical rectangles always has a top right (as the + // block above will have been decoded) + if (xd->n8_w < xd->n8_h) + if (!xd->is_sec_rect) has_tr = 1; + + // The bottom of two horizontal rectangles never has a top right (as the block + // to the right won't have been decoded) + if (xd->n8_w > xd->n8_h) + if (xd->is_sec_rect) has_tr = 0; + +#if CONFIG_EXT_PARTITION_TYPES + // The bottom left square of a Vertical A does not have a top right as it is + // decoded before the right hand rectangle of the partition + if (xd->mi[0]->mbmi.partition == PARTITION_VERT_A) + if ((mask_row & bs) && !(mask_col & bs)) has_tr = 0; +#endif // CONFIG_EXT_PARTITION_TYPES + + return has_tr; +} + +static int add_col_ref_mv(const AV1_COMMON *cm, + const MV_REF *prev_frame_mvs_base, + const MACROBLOCKD *xd, int mi_row, int mi_col, + MV_REFERENCE_FRAME ref_frame, int blk_row, + int blk_col, uint8_t *refmv_count, + CANDIDATE_MV *ref_mv_stack, int16_t *mode_context) { + const MV_REF *prev_frame_mvs = + prev_frame_mvs_base + blk_row * cm->mi_cols + blk_col; + POSITION mi_pos; + int ref, idx; + int coll_blk_count = 0; + const int weight_unit = mi_size_wide[BLOCK_8X8]; + +#if CONFIG_MV_COMPRESS + mi_pos.row = (mi_row & 0x01) ? blk_row : blk_row + 1; + mi_pos.col = (mi_col & 0x01) ? blk_col : blk_col + 1; +#else + mi_pos.row = blk_row; + mi_pos.col = blk_col; +#endif + + if (!is_inside(&xd->tile, mi_col, mi_row, cm->mi_rows, cm, &mi_pos)) + return coll_blk_count; + for (ref = 0; ref < 2; ++ref) { + if (prev_frame_mvs->ref_frame[ref] == ref_frame) { + int_mv this_refmv = prev_frame_mvs->mv[ref]; + lower_mv_precision(&this_refmv.as_mv, cm->allow_high_precision_mv); + + if (abs(this_refmv.as_mv.row) >= 16 || abs(this_refmv.as_mv.col) >= 16) + mode_context[ref_frame] |= (1 << ZEROMV_OFFSET); + + for (idx = 0; idx < *refmv_count; ++idx) + if (this_refmv.as_int == ref_mv_stack[idx].this_mv.as_int) break; + + if (idx < *refmv_count) ref_mv_stack[idx].weight += 2 * weight_unit; + + if (idx == *refmv_count && *refmv_count < MAX_REF_MV_STACK_SIZE) { + ref_mv_stack[idx].this_mv.as_int = this_refmv.as_int; + ref_mv_stack[idx].pred_diff[0] = + av1_get_pred_diff_ctx(prev_frame_mvs->pred_mv[ref], this_refmv); + ref_mv_stack[idx].weight = 2 * weight_unit; + ++(*refmv_count); + } + + ++coll_blk_count; + } + } + + return coll_blk_count; +} + +static void setup_ref_mv_list(const AV1_COMMON *cm, const MACROBLOCKD *xd, + MV_REFERENCE_FRAME ref_frame, + uint8_t *refmv_count, CANDIDATE_MV *ref_mv_stack, + int_mv *mv_ref_list, int block, int mi_row, + int mi_col, int16_t *mode_context) { + int idx, nearest_refmv_count = 0; + uint8_t newmv_count = 0; + CANDIDATE_MV tmp_mv; + int len, nr_len; + +#if CONFIG_MV_COMPRESS + const MV_REF *const prev_frame_mvs_base = + cm->use_prev_frame_mvs + ? cm->prev_frame->mvs + (((mi_row >> 1) << 1) + 1) * cm->mi_cols + + ((mi_col >> 1) << 1) + 1 + : NULL; +#else + const MV_REF *const prev_frame_mvs_base = + cm->use_prev_frame_mvs + ? cm->prev_frame->mvs + mi_row * cm->mi_cols + mi_col + : NULL; +#endif + + const int bs = AOMMAX(xd->n8_w, xd->n8_h); + const int has_tr = has_top_right(xd, mi_row, mi_col, bs); + MV_REFERENCE_FRAME rf[2]; + + av1_set_ref_frame(rf, ref_frame); + mode_context[ref_frame] = 0; + *refmv_count = 0; + + // Scan the first above row mode info. + newmv_count += scan_row_mbmi(cm, xd, mi_row, mi_col, block, rf, -1, + ref_mv_stack, refmv_count); + // Scan the first left column mode info. + newmv_count += scan_col_mbmi(cm, xd, mi_row, mi_col, block, rf, -1, + ref_mv_stack, refmv_count); + + // Check top-right boundary + if (has_tr) + newmv_count += scan_blk_mbmi(cm, xd, mi_row, mi_col, block, rf, -1, + xd->n8_w, ref_mv_stack, refmv_count); + + nearest_refmv_count = *refmv_count; + + for (idx = 0; idx < nearest_refmv_count; ++idx) + ref_mv_stack[idx].weight += REF_CAT_LEVEL; +#if CONFIG_TEMPMV_SIGNALING + if (cm->use_prev_frame_mvs && rf[1] == NONE_FRAME) { +#else + if (prev_frame_mvs_base && cm->show_frame && cm->last_show_frame && + rf[1] == NONE_FRAME) { +#endif + int blk_row, blk_col; + int coll_blk_count = 0; +#if CONFIG_CB4X4 + const int mi_step = (xd->n8_w == 1 || xd->n8_h == 1) + ? mi_size_wide[BLOCK_8X8] + : mi_size_wide[BLOCK_16X16]; +#else + const int mi_step = mi_size_wide[BLOCK_16X16]; +#endif + +#if CONFIG_TPL_MV + int tpl_sample_pos[5][2] = { { -1, xd->n8_w }, + { 0, xd->n8_w }, + { xd->n8_h, xd->n8_w }, + { xd->n8_h, 0 }, + { xd->n8_h, -1 } }; + int i; +#endif + + for (blk_row = 0; blk_row < xd->n8_h; blk_row += mi_step) { + for (blk_col = 0; blk_col < xd->n8_w; blk_col += mi_step) { + coll_blk_count += add_col_ref_mv( + cm, prev_frame_mvs_base, xd, mi_row, mi_col, ref_frame, blk_row, + blk_col, refmv_count, ref_mv_stack, mode_context); + } + } + +#if CONFIG_TPL_MV + for (i = 0; i < 5; ++i) { + blk_row = tpl_sample_pos[i][0]; + blk_col = tpl_sample_pos[i][1]; + coll_blk_count += add_col_ref_mv(cm, prev_frame_mvs_base, xd, mi_row, + mi_col, ref_frame, blk_row, blk_col, + refmv_count, ref_mv_stack, mode_context); + } +#endif + + if (coll_blk_count == 0) mode_context[ref_frame] |= (1 << ZEROMV_OFFSET); + } else { + mode_context[ref_frame] |= (1 << ZEROMV_OFFSET); + } + + // Scan the second outer area. + scan_blk_mbmi(cm, xd, mi_row, mi_col, block, rf, -1, -1, ref_mv_stack, + refmv_count); + for (idx = 2; idx <= 3; ++idx) { + scan_row_mbmi(cm, xd, mi_row, mi_col, block, rf, -idx, ref_mv_stack, + refmv_count); + scan_col_mbmi(cm, xd, mi_row, mi_col, block, rf, -idx, ref_mv_stack, + refmv_count); + } + scan_col_mbmi(cm, xd, mi_row, mi_col, block, rf, -4, ref_mv_stack, + refmv_count); + + switch (nearest_refmv_count) { + case 0: + mode_context[ref_frame] |= 0; + if (*refmv_count >= 1) mode_context[ref_frame] |= 1; + + if (*refmv_count == 1) + mode_context[ref_frame] |= (1 << REFMV_OFFSET); + else if (*refmv_count >= 2) + mode_context[ref_frame] |= (2 << REFMV_OFFSET); + break; + case 1: + mode_context[ref_frame] |= (newmv_count > 0) ? 2 : 3; + + if (*refmv_count == 1) + mode_context[ref_frame] |= (3 << REFMV_OFFSET); + else if (*refmv_count >= 2) + mode_context[ref_frame] |= (4 << REFMV_OFFSET); + break; + + case 2: + default: + if (newmv_count >= 2) + mode_context[ref_frame] |= 4; + else if (newmv_count == 1) + mode_context[ref_frame] |= 5; + else + mode_context[ref_frame] |= 6; + + mode_context[ref_frame] |= (5 << REFMV_OFFSET); + break; + } + + // Rank the likelihood and assign nearest and near mvs. + len = nearest_refmv_count; + while (len > 0) { + nr_len = 0; + for (idx = 1; idx < len; ++idx) { + if (ref_mv_stack[idx - 1].weight < ref_mv_stack[idx].weight) { + tmp_mv = ref_mv_stack[idx - 1]; + ref_mv_stack[idx - 1] = ref_mv_stack[idx]; + ref_mv_stack[idx] = tmp_mv; + nr_len = idx; + } + } + len = nr_len; + } + + len = *refmv_count; + while (len > nearest_refmv_count) { + nr_len = nearest_refmv_count; + for (idx = nearest_refmv_count + 1; idx < len; ++idx) { + if (ref_mv_stack[idx - 1].weight < ref_mv_stack[idx].weight) { + tmp_mv = ref_mv_stack[idx - 1]; + ref_mv_stack[idx - 1] = ref_mv_stack[idx]; + ref_mv_stack[idx] = tmp_mv; + nr_len = idx; + } + } + len = nr_len; + } + + if (rf[1] > NONE_FRAME) { + for (idx = 0; idx < *refmv_count; ++idx) { + clamp_mv_ref(&ref_mv_stack[idx].this_mv.as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + clamp_mv_ref(&ref_mv_stack[idx].comp_mv.as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + } + } else { + for (idx = 0; idx < AOMMIN(MAX_MV_REF_CANDIDATES, *refmv_count); ++idx) { + mv_ref_list[idx].as_int = ref_mv_stack[idx].this_mv.as_int; + clamp_mv_ref(&mv_ref_list[idx].as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + } + } +} +#endif + +// This function searches the neighbourhood of a given MB/SB +// to try and find candidate reference vectors. +static void find_mv_refs_idx(const AV1_COMMON *cm, const MACROBLOCKD *xd, + MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame, + int_mv *mv_ref_list, int block, int mi_row, + int mi_col, find_mv_refs_sync sync, + void *const data, int16_t *mode_context, + int_mv zeromv) { + const int *ref_sign_bias = cm->ref_frame_sign_bias; + int i, refmv_count = 0; +#if !CONFIG_REF_MV + const POSITION *const mv_ref_search = mv_ref_blocks[mi->mbmi.sb_type]; +#endif + int different_ref_found = 0; + int context_counter = 0; +#if CONFIG_MV_COMPRESS + const TileInfo *const tile_ = &xd->tile; + int mi_row_end = tile_->mi_row_end; + int mi_col_end = tile_->mi_col_end; + const MV_REF *const prev_frame_mvs = + cm->use_prev_frame_mvs + ? cm->prev_frame->mvs + + AOMMIN(((mi_row >> 1) << 1) + 1 + (((xd->n8_h - 1) >> 1) << 1), + mi_row_end - 1) * + cm->mi_cols + + AOMMIN(((mi_col >> 1) << 1) + 1 + (((xd->n8_w - 1) >> 1) << 1), + mi_col_end - 1) + : NULL; +#else + const MV_REF *const prev_frame_mvs = + cm->use_prev_frame_mvs + ? cm->prev_frame->mvs + mi_row * cm->mi_cols + mi_col + : NULL; +#endif + const TileInfo *const tile = &xd->tile; + const BLOCK_SIZE bsize = mi->mbmi.sb_type; + const int bw = block_size_wide[AOMMAX(bsize, BLOCK_8X8)]; + const int bh = block_size_high[AOMMAX(bsize, BLOCK_8X8)]; +#if CONFIG_REF_MV + POSITION mv_ref_search[MVREF_NEIGHBOURS]; + const int num_8x8_blocks_wide = num_8x8_blocks_wide_lookup[bsize]; + const int num_8x8_blocks_high = num_8x8_blocks_high_lookup[bsize]; + mv_ref_search[0].row = num_8x8_blocks_high - 1; + mv_ref_search[0].col = -1; + mv_ref_search[1].row = -1; + mv_ref_search[1].col = num_8x8_blocks_wide - 1; + mv_ref_search[2].row = -1; + mv_ref_search[2].col = (num_8x8_blocks_wide - 1) >> 1; + mv_ref_search[3].row = (num_8x8_blocks_high - 1) >> 1; + mv_ref_search[3].col = -1; + mv_ref_search[4].row = -1; + mv_ref_search[4].col = -1; +#if CONFIG_EXT_PARTITION_TYPES + if (num_8x8_blocks_wide == num_8x8_blocks_high) { + mv_ref_search[5].row = -1; + mv_ref_search[5].col = 0; + mv_ref_search[6].row = 0; + mv_ref_search[6].col = -1; + } else { + mv_ref_search[5].row = -1; + mv_ref_search[5].col = num_8x8_blocks_wide; + mv_ref_search[6].row = num_8x8_blocks_high; + mv_ref_search[6].col = -1; + } +#else + mv_ref_search[5].row = -1; + mv_ref_search[5].col = num_8x8_blocks_wide; + mv_ref_search[6].row = num_8x8_blocks_high; + mv_ref_search[6].col = -1; +#endif // CONFIG_EXT_PARTITION_TYPES + mv_ref_search[7].row = -1; + mv_ref_search[7].col = -3; + mv_ref_search[8].row = num_8x8_blocks_high - 1; + mv_ref_search[8].col = -3; + +#if CONFIG_CB4X4 + for (i = 0; i < MVREF_NEIGHBOURS; ++i) { + mv_ref_search[i].row *= 2; + mv_ref_search[i].col *= 2; + } +#endif // CONFIG_CB4X4 +#endif // CONFIG_REF_MV + + // The nearest 2 blocks are treated differently + // if the size < 8x8 we get the mv from the bmi substructure, + // and we also need to keep a mode count. + for (i = 0; i < 2; ++i) { + const POSITION *const mv_ref = &mv_ref_search[i]; + if (is_inside(tile, mi_col, mi_row, cm->mi_rows, cm, mv_ref)) { + const MODE_INFO *const candidate_mi = + xd->mi[mv_ref->col + mv_ref->row * xd->mi_stride]; + const MB_MODE_INFO *const candidate = &candidate_mi->mbmi; + // Keep counts for entropy encoding. + context_counter += mode_2_counter[candidate->mode]; + different_ref_found = 1; + + if (candidate->ref_frame[0] == ref_frame) + ADD_MV_REF_LIST(get_sub_block_mv(candidate_mi, 0, mv_ref->col, block), + refmv_count, mv_ref_list, bw, bh, xd, Done); + else if (candidate->ref_frame[1] == ref_frame) + ADD_MV_REF_LIST(get_sub_block_mv(candidate_mi, 1, mv_ref->col, block), + refmv_count, mv_ref_list, bw, bh, xd, Done); + } + } + + // Check the rest of the neighbors in much the same way + // as before except we don't need to keep track of sub blocks or + // mode counts. + for (; i < MVREF_NEIGHBOURS; ++i) { + const POSITION *const mv_ref = &mv_ref_search[i]; + if (is_inside(tile, mi_col, mi_row, cm->mi_rows, cm, mv_ref)) { + const MB_MODE_INFO *const candidate = + !xd->mi[mv_ref->col + mv_ref->row * xd->mi_stride] + ? NULL + : &xd->mi[mv_ref->col + mv_ref->row * xd->mi_stride]->mbmi; +#if CONFIG_REF_MV + if (candidate == NULL) continue; + if ((mi_row % MAX_MIB_SIZE) + mv_ref->row >= MAX_MIB_SIZE || + (mi_col % MAX_MIB_SIZE) + mv_ref->col >= MAX_MIB_SIZE) + continue; +#endif + different_ref_found = 1; + + if (candidate->ref_frame[0] == ref_frame) + ADD_MV_REF_LIST(candidate->mv[0], refmv_count, mv_ref_list, bw, bh, xd, + Done); + else if (candidate->ref_frame[1] == ref_frame) + ADD_MV_REF_LIST(candidate->mv[1], refmv_count, mv_ref_list, bw, bh, xd, + Done); + } + } + +// TODO(hkuang): Remove this sync after fixing pthread_cond_broadcast +// on windows platform. The sync here is unncessary if use_perv_frame_mvs +// is 0. But after removing it, there will be hang in the unit test on windows +// due to several threads waiting for a thread's signal. +#if defined(_WIN32) && !HAVE_PTHREAD_H + if (cm->frame_parallel_decode && sync != NULL) { + sync(data, mi_row); + } +#endif + + // Check the last frame's mode and mv info. + if (cm->use_prev_frame_mvs) { + // Synchronize here for frame parallel decode if sync function is provided. + if (cm->frame_parallel_decode && sync != NULL) { + sync(data, mi_row); + } + + if (prev_frame_mvs->ref_frame[0] == ref_frame) { + ADD_MV_REF_LIST(prev_frame_mvs->mv[0], refmv_count, mv_ref_list, bw, bh, + xd, Done); + } else if (prev_frame_mvs->ref_frame[1] == ref_frame) { + ADD_MV_REF_LIST(prev_frame_mvs->mv[1], refmv_count, mv_ref_list, bw, bh, + xd, Done); + } + } + + // Since we couldn't find 2 mvs from the same reference frame + // go back through the neighbors and find motion vectors from + // different reference frames. + if (different_ref_found) { + for (i = 0; i < MVREF_NEIGHBOURS; ++i) { + const POSITION *mv_ref = &mv_ref_search[i]; + if (is_inside(tile, mi_col, mi_row, cm->mi_rows, cm, mv_ref)) { + const MB_MODE_INFO *const candidate = + !xd->mi[mv_ref->col + mv_ref->row * xd->mi_stride] + ? NULL + : &xd->mi[mv_ref->col + mv_ref->row * xd->mi_stride]->mbmi; +#if CONFIG_REF_MV + if (candidate == NULL) continue; + if ((mi_row % MAX_MIB_SIZE) + mv_ref->row >= MAX_MIB_SIZE || + (mi_col % MAX_MIB_SIZE) + mv_ref->col >= MAX_MIB_SIZE) + continue; +#endif + + // If the candidate is INTRA we don't want to consider its mv. + IF_DIFF_REF_FRAME_ADD_MV(candidate, ref_frame, ref_sign_bias, + refmv_count, mv_ref_list, bw, bh, xd, Done); + } + } + } + + // Since we still don't have a candidate we'll try the last frame. + if (cm->use_prev_frame_mvs) { + if (prev_frame_mvs->ref_frame[0] != ref_frame && + prev_frame_mvs->ref_frame[0] > INTRA_FRAME) { + int_mv mv = prev_frame_mvs->mv[0]; + if (ref_sign_bias[prev_frame_mvs->ref_frame[0]] != + ref_sign_bias[ref_frame]) { + mv.as_mv.row *= -1; + mv.as_mv.col *= -1; + } + ADD_MV_REF_LIST(mv, refmv_count, mv_ref_list, bw, bh, xd, Done); + } + + if (prev_frame_mvs->ref_frame[1] > INTRA_FRAME && + prev_frame_mvs->ref_frame[1] != ref_frame) { + int_mv mv = prev_frame_mvs->mv[1]; + if (ref_sign_bias[prev_frame_mvs->ref_frame[1]] != + ref_sign_bias[ref_frame]) { + mv.as_mv.row *= -1; + mv.as_mv.col *= -1; + } + ADD_MV_REF_LIST(mv, refmv_count, mv_ref_list, bw, bh, xd, Done); + } + } + +Done: + if (mode_context) + mode_context[ref_frame] = counter_to_context[context_counter]; + for (i = refmv_count; i < MAX_MV_REF_CANDIDATES; ++i) + mv_ref_list[i].as_int = zeromv.as_int; +} + +#if CONFIG_EXT_INTER +// This function keeps a mode count for a given MB/SB +void av1_update_mv_context(const AV1_COMMON *cm, const MACROBLOCKD *xd, + MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame, + int_mv *mv_ref_list, int block, int mi_row, + int mi_col, int16_t *mode_context) { + int i, refmv_count = 0; +#if !CONFIG_REF_MV + const POSITION *const mv_ref_search = mv_ref_blocks[mi->mbmi.sb_type]; +#endif + int context_counter = 0; + const int bw = block_size_wide[mi->mbmi.sb_type]; + const int bh = block_size_high[mi->mbmi.sb_type]; + const TileInfo *const tile = &xd->tile; +#if CONFIG_REF_MV + POSITION mv_ref_search[MVREF_NEIGHBOURS]; + const int num_8x8_blocks_wide = mi_size_wide[mi->mbmi.sb_type]; + const int num_8x8_blocks_high = mi_size_high[mi->mbmi.sb_type]; + mv_ref_search[0].row = num_8x8_blocks_high - 1; + mv_ref_search[0].col = -1; + mv_ref_search[1].row = -1; + mv_ref_search[1].col = num_8x8_blocks_wide - 1; + mv_ref_search[2].row = -1; + mv_ref_search[2].col = (num_8x8_blocks_wide - 1) >> 1; + mv_ref_search[3].row = (num_8x8_blocks_high - 1) >> 1; + mv_ref_search[3].col = -1; + mv_ref_search[4].row = -1; + mv_ref_search[4].col = -1; +#if CONFIG_EXT_PARTITION_TYPES + if (num_8x8_blocks_wide == num_8x8_blocks_high) { + mv_ref_search[5].row = -1; + mv_ref_search[5].col = 0; + mv_ref_search[6].row = 0; + mv_ref_search[6].col = -1; + } else { + mv_ref_search[5].row = -1; + mv_ref_search[5].col = num_8x8_blocks_wide; + mv_ref_search[6].row = num_8x8_blocks_high; + mv_ref_search[6].col = -1; + } +#else + mv_ref_search[5].row = -1; + mv_ref_search[5].col = num_8x8_blocks_wide; + mv_ref_search[6].row = num_8x8_blocks_high; + mv_ref_search[6].col = -1; +#endif // CONFIG_EXT_PARTITION_TYPES + mv_ref_search[7].row = -1; + mv_ref_search[7].col = -3; + mv_ref_search[8].row = num_8x8_blocks_high - 1; + mv_ref_search[8].col = -3; +#endif + + // Blank the reference vector list + memset(mv_ref_list, 0, sizeof(*mv_ref_list) * MAX_MV_REF_CANDIDATES); + + // The nearest 2 blocks are examined only. + // If the size < 8x8, we get the mv from the bmi substructure; + for (i = 0; i < 2; ++i) { + const POSITION *const mv_ref = &mv_ref_search[i]; + if (is_inside(tile, mi_col, mi_row, cm->mi_rows, cm, mv_ref)) { + const MODE_INFO *const candidate_mi = + xd->mi[mv_ref->col + mv_ref->row * xd->mi_stride]; + const MB_MODE_INFO *const candidate = &candidate_mi->mbmi; + + // Keep counts for entropy encoding. + context_counter += mode_2_counter[candidate->mode]; + + if (candidate->ref_frame[0] == ref_frame) { + ADD_MV_REF_LIST(get_sub_block_mv(candidate_mi, 0, mv_ref->col, block), + refmv_count, mv_ref_list, bw, bh, xd, Done); + } else if (candidate->ref_frame[1] == ref_frame) { + ADD_MV_REF_LIST(get_sub_block_mv(candidate_mi, 1, mv_ref->col, block), + refmv_count, mv_ref_list, bw, bh, xd, Done); + } + } + } + +Done: + + if (mode_context) + mode_context[ref_frame] = counter_to_context[context_counter]; +} +#endif // CONFIG_EXT_INTER + +void av1_find_mv_refs(const AV1_COMMON *cm, const MACROBLOCKD *xd, + MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame, +#if CONFIG_REF_MV + uint8_t *ref_mv_count, CANDIDATE_MV *ref_mv_stack, +#if CONFIG_EXT_INTER + int16_t *compound_mode_context, +#endif // CONFIG_EXT_INTER +#endif + int_mv *mv_ref_list, int mi_row, int mi_col, + find_mv_refs_sync sync, void *const data, + int16_t *mode_context) { + int_mv zeromv[2]; +#if CONFIG_GLOBAL_MOTION + BLOCK_SIZE bsize = mi->mbmi.sb_type; +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_REF_MV + int idx, all_zero = 1; +#if CONFIG_GLOBAL_MOTION + MV_REFERENCE_FRAME rf[2]; +#endif // CONFIG_GLOBAL_MOTION +#endif // CONFIG_REF_MV +#if CONFIG_EXT_INTER + av1_update_mv_context(cm, xd, mi, ref_frame, mv_ref_list, -1, mi_row, mi_col, +#if CONFIG_REF_MV + compound_mode_context); +#else + mode_context); +#endif // CONFIG_REF_MV +#endif // CONFIG_EXT_INTER + +#if CONFIG_GLOBAL_MOTION +#if CONFIG_REF_MV + av1_set_ref_frame(rf, ref_frame); + zeromv[0].as_int = gm_get_motion_vector(&cm->global_motion[rf[0]], + cm->allow_high_precision_mv, bsize, + mi_col, mi_row, 0) + .as_int; + zeromv[1].as_int = (rf[1] != NONE_FRAME) + ? gm_get_motion_vector(&cm->global_motion[rf[1]], + cm->allow_high_precision_mv, + bsize, mi_col, mi_row, 0) + .as_int + : 0; +#else + zeromv[0].as_int = gm_get_motion_vector(&cm->global_motion[ref_frame], + cm->allow_high_precision_mv, bsize, + mi_col, mi_row, 0) + .as_int; + zeromv[1].as_int = 0; +#endif // CONFIG_REF_MV +#else + zeromv[0].as_int = zeromv[1].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION + +#if CONFIG_REF_MV + if (ref_frame <= ALTREF_FRAME) +#endif // CONFIG_REF_MV + find_mv_refs_idx(cm, xd, mi, ref_frame, mv_ref_list, -1, mi_row, mi_col, + sync, data, mode_context, zeromv[0]); + +#if CONFIG_REF_MV + setup_ref_mv_list(cm, xd, ref_frame, ref_mv_count, ref_mv_stack, mv_ref_list, + -1, mi_row, mi_col, mode_context); + /* Note: If global motion is enabled, then we want to set the ALL_ZERO flag + iff all of the MVs we could generate with NEARMV/NEARESTMV are equivalent + to the global motion vector. + Note: For the following to work properly, the encoder can't throw away + any global motion models after calling this function, even if they are + unused. Instead we rely on the recode loop: If any non-IDENTITY model + is unused, the whole frame will be re-encoded without it. + The problem is that, otherwise, we can end up in the following situation: + * Encoder has a global motion model with nonzero translational part, + and all candidate MVs are zero. So the ALL_ZERO flag is unset. + * Encoder throws away global motion because it is never used. + * Decoder sees that there is no global motion and all candidate MVs are + zero, so sets the ALL_ZERO flag. + * This leads to an encode/decode mismatch. + */ + if (*ref_mv_count >= 2) { + for (idx = 0; idx < AOMMIN(3, *ref_mv_count); ++idx) { + if (ref_mv_stack[idx].this_mv.as_int != zeromv[0].as_int) all_zero = 0; + if (ref_frame > ALTREF_FRAME) + if (ref_mv_stack[idx].comp_mv.as_int != zeromv[1].as_int) all_zero = 0; + } + } else if (ref_frame <= ALTREF_FRAME) { + for (idx = 0; idx < MAX_MV_REF_CANDIDATES; ++idx) + if (mv_ref_list[idx].as_int != zeromv[0].as_int) all_zero = 0; + } + + if (all_zero) mode_context[ref_frame] |= (1 << ALL_ZERO_FLAG_OFFSET); +#endif +} + +void av1_find_best_ref_mvs(int allow_hp, int_mv *mvlist, int_mv *nearest_mv, + int_mv *near_mv) { + int i; + // Make sure all the candidates are properly clamped etc + for (i = 0; i < MAX_MV_REF_CANDIDATES; ++i) { + lower_mv_precision(&mvlist[i].as_mv, allow_hp); + } + *nearest_mv = mvlist[0]; + *near_mv = mvlist[1]; +} + +void av1_append_sub8x8_mvs_for_idx(const AV1_COMMON *cm, MACROBLOCKD *xd, + int block, int ref, int mi_row, int mi_col, +#if CONFIG_REF_MV + CANDIDATE_MV *ref_mv_stack, + uint8_t *ref_mv_count, +#endif +#if CONFIG_EXT_INTER + int_mv *mv_list, +#endif // CONFIG_EXT_INTER + int_mv *nearest_mv, int_mv *near_mv) { +#if !CONFIG_EXT_INTER + int_mv mv_list[MAX_MV_REF_CANDIDATES]; +#endif // !CONFIG_EXT_INTER + MODE_INFO *const mi = xd->mi[0]; + b_mode_info *bmi = mi->bmi; + int n; + int_mv zeromv; +#if CONFIG_REF_MV + CANDIDATE_MV tmp_mv; + uint8_t idx; + uint8_t above_count = 0, left_count = 0; + MV_REFERENCE_FRAME rf[2] = { mi->mbmi.ref_frame[ref], NONE_FRAME }; + *ref_mv_count = 0; +#endif + + assert(MAX_MV_REF_CANDIDATES == 2); + +#if CONFIG_GLOBAL_MOTION + zeromv.as_int = + gm_get_motion_vector(&cm->global_motion[ref], cm->allow_high_precision_mv, + mi->mbmi.sb_type, mi_col, mi_row, block) + .as_int; +#else + zeromv.as_int = 0; +#endif + find_mv_refs_idx(cm, xd, mi, mi->mbmi.ref_frame[ref], mv_list, block, mi_row, + mi_col, NULL, NULL, NULL, zeromv); + +#if CONFIG_REF_MV + scan_blk_mbmi(cm, xd, mi_row, mi_col, block, rf, -1, 0, ref_mv_stack, + ref_mv_count); + above_count = *ref_mv_count; + + scan_blk_mbmi(cm, xd, mi_row, mi_col, block, rf, 0, -1, ref_mv_stack, + ref_mv_count); + left_count = *ref_mv_count - above_count; + + if (above_count > 1 && left_count > 0) { + tmp_mv = ref_mv_stack[1]; + ref_mv_stack[1] = ref_mv_stack[above_count]; + ref_mv_stack[above_count] = tmp_mv; + } + + for (idx = 0; idx < *ref_mv_count; ++idx) + clamp_mv_ref(&ref_mv_stack[idx].this_mv.as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + + for (idx = 0; idx < AOMMIN(MAX_MV_REF_CANDIDATES, *ref_mv_count); ++idx) + mv_list[idx].as_int = ref_mv_stack[idx].this_mv.as_int; +#endif + + near_mv->as_int = 0; + switch (block) { + case 0: + nearest_mv->as_int = mv_list[0].as_int; + near_mv->as_int = mv_list[1].as_int; + break; + case 1: + case 2: + nearest_mv->as_int = bmi[0].as_mv[ref].as_int; + for (n = 0; n < MAX_MV_REF_CANDIDATES; ++n) + if (nearest_mv->as_int != mv_list[n].as_int) { + near_mv->as_int = mv_list[n].as_int; + break; + } + break; + case 3: { + int_mv candidates[2 + MAX_MV_REF_CANDIDATES]; + candidates[0] = bmi[1].as_mv[ref]; + candidates[1] = bmi[0].as_mv[ref]; + candidates[2] = mv_list[0]; + candidates[3] = mv_list[1]; + + nearest_mv->as_int = bmi[2].as_mv[ref].as_int; + for (n = 0; n < 2 + MAX_MV_REF_CANDIDATES; ++n) + if (nearest_mv->as_int != candidates[n].as_int) { + near_mv->as_int = candidates[n].as_int; + break; + } + break; + } + default: assert(0 && "Invalid block index."); + } +} + +#if CONFIG_WARPED_MOTION +void calc_projection_samples(MB_MODE_INFO *const mbmi, int x, int y, + int *pts_inref) { + pts_inref[0] = (x * 8) + mbmi->mv[0].as_mv.col; + pts_inref[1] = (y * 8) + mbmi->mv[0].as_mv.row; +} + +// Note: Samples returned are at 1/8-pel precision +int findSamples(const AV1_COMMON *cm, MACROBLOCKD *xd, int mi_row, int mi_col, + int *pts, int *pts_inref) { + MB_MODE_INFO *const mbmi0 = &(xd->mi[0]->mbmi); + int ref_frame = mbmi0->ref_frame[0]; + int up_available = xd->up_available; + int left_available = xd->left_available; + int i, mi_step, np = 0; + int global_offset_c = mi_col * MI_SIZE; + int global_offset_r = mi_row * MI_SIZE; + + // scan the above row + if (up_available) { + for (i = 0; i < AOMMIN(xd->n8_w, cm->mi_cols - mi_col); i += mi_step) { + int mi_row_offset = -1; + int mi_col_offset = i; + + MODE_INFO *mi = xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]; + MB_MODE_INFO *mbmi = &mi->mbmi; + + mi_step = AOMMIN(xd->n8_w, mi_size_wide[mbmi->sb_type]); + + if (mbmi->ref_frame[0] == ref_frame && mbmi->ref_frame[1] == NONE_FRAME) { + int bw = block_size_wide[mbmi->sb_type]; + int bh = block_size_high[mbmi->sb_type]; + int cr_offset = -AOMMAX(bh, MI_SIZE) / 2 - 1; + int cc_offset = i * MI_SIZE + AOMMAX(bw, MI_SIZE) / 2 - 1; + int x = cc_offset + global_offset_c; + int y = cr_offset + global_offset_r; + + pts[0] = (x * 8); + pts[1] = (y * 8); + calc_projection_samples(mbmi, x, y, pts_inref); + pts += 2; + pts_inref += 2; + np++; + if (np >= LEAST_SQUARES_SAMPLES_MAX) return LEAST_SQUARES_SAMPLES_MAX; + } + } + } + assert(2 * np <= SAMPLES_ARRAY_SIZE); + + // scan the left column + if (left_available) { + for (i = 0; i < AOMMIN(xd->n8_h, cm->mi_rows - mi_row); i += mi_step) { + int mi_row_offset = i; + int mi_col_offset = -1; + + MODE_INFO *mi = xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]; + MB_MODE_INFO *mbmi = &mi->mbmi; + + mi_step = AOMMIN(xd->n8_h, mi_size_high[mbmi->sb_type]); + + if (mbmi->ref_frame[0] == ref_frame && mbmi->ref_frame[1] == NONE_FRAME) { + int bw = block_size_wide[mbmi->sb_type]; + int bh = block_size_high[mbmi->sb_type]; + int cr_offset = i * MI_SIZE + AOMMAX(bh, MI_SIZE) / 2 - 1; + int cc_offset = -AOMMAX(bw, MI_SIZE) / 2 - 1; + int x = cc_offset + global_offset_c; + int y = cr_offset + global_offset_r; + + pts[0] = (x * 8); + pts[1] = (y * 8); + calc_projection_samples(mbmi, x, y, pts_inref); + pts += 2; + pts_inref += 2; + np++; + if (np >= LEAST_SQUARES_SAMPLES_MAX) return LEAST_SQUARES_SAMPLES_MAX; + } + } + } + assert(2 * np <= SAMPLES_ARRAY_SIZE); + + if (left_available && up_available) { + int mi_row_offset = -1; + int mi_col_offset = -1; + + MODE_INFO *mi = xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]; + MB_MODE_INFO *mbmi = &mi->mbmi; + + if (mbmi->ref_frame[0] == ref_frame && mbmi->ref_frame[1] == NONE_FRAME) { + int bw = block_size_wide[mbmi->sb_type]; + int bh = block_size_high[mbmi->sb_type]; + int cr_offset = -AOMMAX(bh, MI_SIZE) / 2 - 1; + int cc_offset = -AOMMAX(bw, MI_SIZE) / 2 - 1; + int x = cc_offset + global_offset_c; + int y = cr_offset + global_offset_r; + + pts[0] = (x * 8); + pts[1] = (y * 8); + calc_projection_samples(mbmi, x, y, pts_inref); + np++; + } + } + assert(2 * np <= SAMPLES_ARRAY_SIZE); + + return np; +} +#endif // CONFIG_WARPED_MOTION diff --git a/third_party/aom/av1/common/mvref_common.h b/third_party/aom/av1/common/mvref_common.h new file mode 100644 index 0000000000..01f74b77a7 --- /dev/null +++ b/third_party/aom/av1/common/mvref_common.h @@ -0,0 +1,580 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AV1_COMMON_MVREF_COMMON_H_ +#define AV1_COMMON_MVREF_COMMON_H_ + +#include "av1/common/onyxc_int.h" +#include "av1/common/blockd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if CONFIG_REF_MV +#define MVREF_NEIGHBOURS 9 +#else +#define MVREF_NEIGHBOURS 8 +#endif + +typedef struct position { + int row; + int col; +} POSITION; + +typedef enum { + BOTH_ZERO = 0, + ZERO_PLUS_PREDICTED = 1, + BOTH_PREDICTED = 2, + NEW_PLUS_NON_INTRA = 3, + BOTH_NEW = 4, + INTRA_PLUS_NON_INTRA = 5, + BOTH_INTRA = 6, + INVALID_CASE = 9 +} motion_vector_context; + +// This is used to figure out a context for the ref blocks. The code flattens +// an array that would have 3 possible counts (0, 1 & 2) for 3 choices by +// adding 9 for each intra block, 3 for each zero mv and 1 for each new +// motion vector. This single number is then converted into a context +// with a single lookup ( counter_to_context ). +static const int mode_2_counter[MB_MODE_COUNT] = { + 9, // DC_PRED + 9, // V_PRED + 9, // H_PRED + 9, // D45_PRED + 9, // D135_PRED + 9, // D117_PRED + 9, // D153_PRED + 9, // D207_PRED + 9, // D63_PRED +#if CONFIG_ALT_INTRA + 9, // SMOOTH_PRED +#endif // CONFIG_ALT_INTRA + 9, // TM_PRED + 0, // NEARESTMV + 0, // NEARMV + 3, // ZEROMV + 1, // NEWMV +#if CONFIG_EXT_INTER +#if CONFIG_COMPOUND_SINGLEREF + 0, // SR_NEAREST_NEARMV + 1, // SR_NEAREST_NEWMV + 1, // SR_NEAR_NEWMV + 3, // SR_ZERO_NEWMV + 1, // SR_NEW_NEWMV +#endif // CONFIG_COMPOUND_SINGLEREF + 0, // NEAREST_NEARESTMV + 0, // NEAREST_NEARMV + 0, // NEAR_NEARESTMV + 0, // NEAR_NEARMV + 1, // NEAREST_NEWMV + 1, // NEW_NEARESTMV + 1, // NEAR_NEWMV + 1, // NEW_NEARMV + 3, // ZERO_ZEROMV + 1, // NEW_NEWMV +#endif // CONFIG_EXT_INTER +}; + +// There are 3^3 different combinations of 3 counts that can be either 0,1 or +// 2. However the actual count can never be greater than 2 so the highest +// counter we need is 18. 9 is an invalid counter that's never used. +static const int counter_to_context[19] = { + BOTH_PREDICTED, // 0 + NEW_PLUS_NON_INTRA, // 1 + BOTH_NEW, // 2 + ZERO_PLUS_PREDICTED, // 3 + NEW_PLUS_NON_INTRA, // 4 + INVALID_CASE, // 5 + BOTH_ZERO, // 6 + INVALID_CASE, // 7 + INVALID_CASE, // 8 + INTRA_PLUS_NON_INTRA, // 9 + INTRA_PLUS_NON_INTRA, // 10 + INVALID_CASE, // 11 + INTRA_PLUS_NON_INTRA, // 12 + INVALID_CASE, // 13 + INVALID_CASE, // 14 + INVALID_CASE, // 15 + INVALID_CASE, // 16 + INVALID_CASE, // 17 + BOTH_INTRA // 18 +}; + +#if !CONFIG_REF_MV +static const POSITION mv_ref_blocks[BLOCK_SIZES][MVREF_NEIGHBOURS] = { + // 4X4 + { { -1, 0 }, + { 0, -1 }, + { -1, -1 }, + { -2, 0 }, + { 0, -2 }, + { -2, -1 }, + { -1, -2 }, + { -2, -2 } }, + // 4X8 + { { -1, 0 }, + { 0, -1 }, + { -1, -1 }, + { -2, 0 }, + { 0, -2 }, + { -2, -1 }, + { -1, -2 }, + { -2, -2 } }, + // 8X4 + { { -1, 0 }, + { 0, -1 }, + { -1, -1 }, + { -2, 0 }, + { 0, -2 }, + { -2, -1 }, + { -1, -2 }, + { -2, -2 } }, + // 8X8 + { { -1, 0 }, + { 0, -1 }, + { -1, -1 }, + { -2, 0 }, + { 0, -2 }, + { -2, -1 }, + { -1, -2 }, + { -2, -2 } }, + // 8X16 + { { 0, -1 }, + { -1, 0 }, + { 1, -1 }, + { -1, -1 }, + { 0, -2 }, + { -2, 0 }, + { -2, -1 }, + { -1, -2 } }, + // 16X8 + { { -1, 0 }, + { 0, -1 }, + { -1, 1 }, + { -1, -1 }, + { -2, 0 }, + { 0, -2 }, + { -1, -2 }, + { -2, -1 } }, + // 16X16 + { { -1, 0 }, + { 0, -1 }, + { -1, 1 }, + { 1, -1 }, + { -1, -1 }, + { -3, 0 }, + { 0, -3 }, + { -3, -3 } }, + // 16X32 + { { 0, -1 }, + { -1, 0 }, + { 2, -1 }, + { -1, -1 }, + { -1, 1 }, + { 0, -3 }, + { -3, 0 }, + { -3, -3 } }, + // 32X16 + { { -1, 0 }, + { 0, -1 }, + { -1, 2 }, + { -1, -1 }, + { 1, -1 }, + { -3, 0 }, + { 0, -3 }, + { -3, -3 } }, + // 32X32 + { { -1, 1 }, + { 1, -1 }, + { -1, 2 }, + { 2, -1 }, + { -1, -1 }, + { -3, 0 }, + { 0, -3 }, + { -3, -3 } }, + // 32X64 + { { 0, -1 }, + { -1, 0 }, + { 4, -1 }, + { -1, 2 }, + { -1, -1 }, + { 0, -3 }, + { -3, 0 }, + { 2, -1 } }, + // 64X32 + { { -1, 0 }, + { 0, -1 }, + { -1, 4 }, + { 2, -1 }, + { -1, -1 }, + { -3, 0 }, + { 0, -3 }, + { -1, 2 } }, + // 64X64 + { { -1, 3 }, + { 3, -1 }, + { -1, 4 }, + { 4, -1 }, + { -1, -1 }, + { -1, 0 }, + { 0, -1 }, + { -1, 6 } }, +#if CONFIG_EXT_PARTITION + // TODO(debargha/jingning) Making them twice the 32x64, .. ones above + // 64x128 + { { 0, -2 }, + { -2, 0 }, + { 8, -2 }, + { -2, 4 }, + { -2, -2 }, + { 0, -6 }, + { -6, 0 }, + { 4, -2 } }, + // 128x64 + { { -2, 0 }, + { 0, -2 }, + { -2, 8 }, + { 4, -2 }, + { -2, -2 }, + { -6, 0 }, + { 0, -6 }, + { -2, 4 } }, + // 128x128 + { { -2, 6 }, + { 6, -2 }, + { -2, 8 }, + { 8, -2 }, + { -2, -2 }, + { -2, 0 }, + { 0, -2 }, + { -2, 12 } }, +#endif // CONFIG_EXT_PARTITION +}; +#endif + +static const int idx_n_column_to_subblock[4][2] = { + { 1, 2 }, { 1, 3 }, { 3, 2 }, { 3, 3 } +}; + +// clamp_mv_ref +#if CONFIG_EXT_PARTITION +#define MV_BORDER (16 << 3) // Allow 16 pels in 1/8th pel units +#else +#define MV_BORDER (8 << 3) // Allow 8 pels in 1/8th pel units +#endif // CONFIG_EXT_PARTITION + +static INLINE void clamp_mv_ref(MV *mv, int bw, int bh, const MACROBLOCKD *xd) { + clamp_mv(mv, xd->mb_to_left_edge - bw * 8 - MV_BORDER, + xd->mb_to_right_edge + bw * 8 + MV_BORDER, + xd->mb_to_top_edge - bh * 8 - MV_BORDER, + xd->mb_to_bottom_edge + bh * 8 + MV_BORDER); +} + +// This function returns either the appropriate sub block or block's mv +// on whether the block_size < 8x8 and we have check_sub_blocks set. +static INLINE int_mv get_sub_block_mv(const MODE_INFO *candidate, int which_mv, + int search_col, int block_idx) { +#if CONFIG_REF_MV + (void)search_col; + (void)block_idx; + return candidate->mbmi.mv[which_mv]; +#else + return block_idx >= 0 && candidate->mbmi.sb_type < BLOCK_8X8 + ? candidate + ->bmi[idx_n_column_to_subblock[block_idx][search_col == 0]] + .as_mv[which_mv] + : candidate->mbmi.mv[which_mv]; +#endif +} + +#if CONFIG_REF_MV +static INLINE int_mv get_sub_block_pred_mv(const MODE_INFO *candidate, + int which_mv, int search_col, + int block_idx) { + (void)search_col; + (void)block_idx; + return candidate->mbmi.mv[which_mv]; +} +#endif + +// Performs mv sign inversion if indicated by the reference frame combination. +static INLINE int_mv scale_mv(const MB_MODE_INFO *mbmi, int ref, + const MV_REFERENCE_FRAME this_ref_frame, + const int *ref_sign_bias) { + int_mv mv = mbmi->mv[ref]; + if (ref_sign_bias[mbmi->ref_frame[ref]] != ref_sign_bias[this_ref_frame]) { + mv.as_mv.row *= -1; + mv.as_mv.col *= -1; + } + return mv; +} + +#define CLIP_IN_ADD(mv, bw, bh, xd) clamp_mv_ref(mv, bw, bh, xd) + +// This macro is used to add a motion vector mv_ref list if it isn't +// already in the list. If it's the second motion vector it will also +// skip all additional processing and jump to done! +#define ADD_MV_REF_LIST(mv, refmv_count, mv_ref_list, bw, bh, xd, Done) \ + do { \ + (mv_ref_list)[(refmv_count)] = (mv); \ + CLIP_IN_ADD(&(mv_ref_list)[(refmv_count)].as_mv, (bw), (bh), (xd)); \ + if (refmv_count && (mv_ref_list)[1].as_int != (mv_ref_list)[0].as_int) { \ + (refmv_count) = 2; \ + goto Done; \ + } \ + (refmv_count) = 1; \ + } while (0) + +// If either reference frame is different, not INTRA, and they +// are different from each other scale and add the mv to our list. +#define IF_DIFF_REF_FRAME_ADD_MV(mbmi, ref_frame, ref_sign_bias, refmv_count, \ + mv_ref_list, bw, bh, xd, Done) \ + do { \ + if (is_inter_block(mbmi)) { \ + if ((mbmi)->ref_frame[0] != ref_frame) \ + ADD_MV_REF_LIST(scale_mv((mbmi), 0, ref_frame, ref_sign_bias), \ + refmv_count, mv_ref_list, bw, bh, xd, Done); \ + if (has_second_ref(mbmi) && (mbmi)->ref_frame[1] != ref_frame) \ + ADD_MV_REF_LIST(scale_mv((mbmi), 1, ref_frame, ref_sign_bias), \ + refmv_count, mv_ref_list, bw, bh, xd, Done); \ + } \ + } while (0) + +// Checks that the given mi_row, mi_col and search point +// are inside the borders of the tile. +static INLINE int is_inside(const TileInfo *const tile, int mi_col, int mi_row, + int mi_rows, const AV1_COMMON *cm, + const POSITION *mi_pos) { +#if CONFIG_DEPENDENT_HORZTILES + const int dependent_horz_tile_flag = cm->dependent_horz_tiles; +#else + const int dependent_horz_tile_flag = 0; + (void)cm; +#endif +#if CONFIG_TILE_GROUPS + if (dependent_horz_tile_flag && !tile->tg_horz_boundary) { +#else + if (dependent_horz_tile_flag) { +#endif + return !(mi_row + mi_pos->row < 0 || + mi_col + mi_pos->col < tile->mi_col_start || + mi_row + mi_pos->row >= mi_rows || + mi_col + mi_pos->col >= tile->mi_col_end); + } else { + return !(mi_row + mi_pos->row < tile->mi_row_start || + mi_col + mi_pos->col < tile->mi_col_start || + mi_row + mi_pos->row >= tile->mi_row_end || + mi_col + mi_pos->col >= tile->mi_col_end); + } +} + +static INLINE void lower_mv_precision(MV *mv, int allow_hp) { + if (!allow_hp) { + if (mv->row & 1) mv->row += (mv->row > 0 ? -1 : 1); + if (mv->col & 1) mv->col += (mv->col > 0 ? -1 : 1); + } +} + +#if CONFIG_REF_MV +static INLINE uint8_t av1_get_pred_diff_ctx(const int_mv pred_mv, + const int_mv this_mv) { + if (abs(this_mv.as_mv.row - pred_mv.as_mv.row) <= 4 && + abs(this_mv.as_mv.col - pred_mv.as_mv.col) <= 4) + return 2; + else + return 1; +} + +static INLINE int av1_nmv_ctx(const uint8_t ref_mv_count, + const CANDIDATE_MV *ref_mv_stack, int ref, + int ref_mv_idx) { + if (ref_mv_stack[ref_mv_idx].weight >= REF_CAT_LEVEL && ref_mv_count > 0) + return ref_mv_stack[ref_mv_idx].pred_diff[ref]; + + return 0; +} + +static INLINE int8_t av1_ref_frame_type(const MV_REFERENCE_FRAME *const rf) { + if (rf[1] > INTRA_FRAME) { + return TOTAL_REFS_PER_FRAME + FWD_RF_OFFSET(rf[0]) + + BWD_RF_OFFSET(rf[1]) * FWD_REFS; + } + + return rf[0]; +} + +// clang-format off +static MV_REFERENCE_FRAME ref_frame_map[COMP_REFS][2] = { +#if CONFIG_EXT_REFS + { LAST_FRAME, BWDREF_FRAME }, { LAST2_FRAME, BWDREF_FRAME }, + { LAST3_FRAME, BWDREF_FRAME }, { GOLDEN_FRAME, BWDREF_FRAME }, + + { LAST_FRAME, ALTREF_FRAME }, { LAST2_FRAME, ALTREF_FRAME }, + { LAST3_FRAME, ALTREF_FRAME }, { GOLDEN_FRAME, ALTREF_FRAME } +#else + { LAST_FRAME, ALTREF_FRAME }, { GOLDEN_FRAME, ALTREF_FRAME } +#endif +}; +// clang-format on + +static INLINE void av1_set_ref_frame(MV_REFERENCE_FRAME *rf, + int8_t ref_frame_type) { + if (ref_frame_type >= TOTAL_REFS_PER_FRAME) { + rf[0] = ref_frame_map[ref_frame_type - TOTAL_REFS_PER_FRAME][0]; + rf[1] = ref_frame_map[ref_frame_type - TOTAL_REFS_PER_FRAME][1]; + } else { + rf[0] = ref_frame_type; + rf[1] = NONE_FRAME; + assert(ref_frame_type > INTRA_FRAME && + ref_frame_type < TOTAL_REFS_PER_FRAME); + } +} + +static INLINE int16_t av1_mode_context_analyzer( + const int16_t *const mode_context, const MV_REFERENCE_FRAME *const rf, + BLOCK_SIZE bsize, int block) { + int16_t mode_ctx = 0; + int8_t ref_frame_type = av1_ref_frame_type(rf); + + if (block >= 0) { + mode_ctx = mode_context[rf[0]] & 0x00ff; +#if !CONFIG_CB4X4 + if (block > 0 && bsize < BLOCK_8X8 && bsize > BLOCK_4X4) + mode_ctx |= (1 << SKIP_NEARESTMV_SUB8X8_OFFSET); +#else + (void)block; + (void)bsize; +#endif + + return mode_ctx; + } + + return mode_context[ref_frame_type]; +} + +static INLINE uint8_t av1_drl_ctx(const CANDIDATE_MV *ref_mv_stack, + int ref_idx) { + if (ref_mv_stack[ref_idx].weight >= REF_CAT_LEVEL && + ref_mv_stack[ref_idx + 1].weight >= REF_CAT_LEVEL) + return 0; + + if (ref_mv_stack[ref_idx].weight >= REF_CAT_LEVEL && + ref_mv_stack[ref_idx + 1].weight < REF_CAT_LEVEL) + return 2; + + if (ref_mv_stack[ref_idx].weight < REF_CAT_LEVEL && + ref_mv_stack[ref_idx + 1].weight < REF_CAT_LEVEL) + return 3; + + return 0; +} +#endif + +typedef void (*find_mv_refs_sync)(void *const data, int mi_row); +void av1_find_mv_refs(const AV1_COMMON *cm, const MACROBLOCKD *xd, + MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame, +#if CONFIG_REF_MV + uint8_t *ref_mv_count, CANDIDATE_MV *ref_mv_stack, +#if CONFIG_EXT_INTER + int16_t *compound_mode_context, +#endif // CONFIG_EXT_INTER +#endif + int_mv *mv_ref_list, int mi_row, int mi_col, + find_mv_refs_sync sync, void *const data, + int16_t *mode_context); + +// check a list of motion vectors by sad score using a number rows of pixels +// above and a number cols of pixels in the left to select the one with best +// score to use as ref motion vector +void av1_find_best_ref_mvs(int allow_hp, int_mv *mvlist, int_mv *nearest_mv, + int_mv *near_mv); + +void av1_append_sub8x8_mvs_for_idx(const AV1_COMMON *cm, MACROBLOCKD *xd, + int block, int ref, int mi_row, int mi_col, +#if CONFIG_REF_MV + CANDIDATE_MV *ref_mv_stack, + uint8_t *ref_mv_count, +#endif +#if CONFIG_EXT_INTER + int_mv *mv_list, +#endif // CONFIG_EXT_INTER + int_mv *nearest_mv, int_mv *near_mv); + +#if CONFIG_EXT_INTER +// This function keeps a mode count for a given MB/SB +void av1_update_mv_context(const AV1_COMMON *cm, const MACROBLOCKD *xd, + MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame, + int_mv *mv_ref_list, int block, int mi_row, + int mi_col, int16_t *mode_context); +#endif // CONFIG_EXT_INTER + +#if CONFIG_WARPED_MOTION +int findSamples(const AV1_COMMON *cm, MACROBLOCKD *xd, int mi_row, int mi_col, + int *pts, int *pts_inref); +#endif // CONFIG_WARPED_MOTION + +#if CONFIG_INTRABC +static INLINE void av1_find_ref_dv(int_mv *ref_dv, int mi_row, int mi_col) { + // TODO(aconverse@google.com): Handle tiles and such + (void)mi_col; + if (mi_row < MAX_MIB_SIZE) { + ref_dv->as_mv.row = 0; + ref_dv->as_mv.col = -MI_SIZE * MAX_MIB_SIZE; + } else { + ref_dv->as_mv.row = -MI_SIZE * MAX_MIB_SIZE; + ref_dv->as_mv.col = 0; + } +} + +static INLINE int is_dv_valid(const MV dv, const TileInfo *const tile, + int mi_row, int mi_col, BLOCK_SIZE bsize) { + const int bw = block_size_wide[bsize]; + const int bh = block_size_high[bsize]; + const int SCALE_PX_TO_MV = 8; + // Disallow subpixel for now + // SUBPEL_MASK is not the correct scale + if ((dv.row & (SCALE_PX_TO_MV - 1) || dv.col & (SCALE_PX_TO_MV - 1))) + return 0; + // Is the source top-left inside the current tile? + const int src_top_edge = mi_row * MI_SIZE * SCALE_PX_TO_MV + dv.row; + const int tile_top_edge = tile->mi_row_start * MI_SIZE * SCALE_PX_TO_MV; + if (src_top_edge < tile_top_edge) return 0; + const int src_left_edge = mi_col * MI_SIZE * SCALE_PX_TO_MV + dv.col; + const int tile_left_edge = tile->mi_col_start * MI_SIZE * SCALE_PX_TO_MV; + if (src_left_edge < tile_left_edge) return 0; + // Is the bottom right inside the current tile? + const int src_bottom_edge = (mi_row * MI_SIZE + bh) * SCALE_PX_TO_MV + dv.row; + const int tile_bottom_edge = tile->mi_row_end * MI_SIZE * SCALE_PX_TO_MV; + if (src_bottom_edge > tile_bottom_edge) return 0; + const int src_right_edge = (mi_col * MI_SIZE + bw) * SCALE_PX_TO_MV + dv.col; + const int tile_right_edge = tile->mi_col_end * MI_SIZE * SCALE_PX_TO_MV; + if (src_right_edge > tile_right_edge) return 0; + // Is the bottom right within an already coded SB? + const int active_sb_top_edge = + (mi_row & ~MAX_MIB_MASK) * MI_SIZE * SCALE_PX_TO_MV; + const int active_sb_bottom_edge = + ((mi_row & ~MAX_MIB_MASK) + MAX_MIB_SIZE) * MI_SIZE * SCALE_PX_TO_MV; + const int active_sb_left_edge = + (mi_col & ~MAX_MIB_MASK) * MI_SIZE * SCALE_PX_TO_MV; + if (src_bottom_edge > active_sb_bottom_edge) return 0; + if (src_bottom_edge > active_sb_top_edge && + src_right_edge > active_sb_left_edge) + return 0; + return 1; +} +#endif // CONFIG_INTRABC + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_MVREF_COMMON_H_ diff --git a/third_party/aom/av1/common/od_dering.c b/third_party/aom/av1/common/od_dering.c new file mode 100644 index 0000000000..f54f337ef5 --- /dev/null +++ b/third_party/aom/av1/common/od_dering.c @@ -0,0 +1,416 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#ifdef HAVE_CONFIG_H +#include "./config.h" +#endif + +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" +#include "./cdef.h" + +/* Generated from gen_filter_tables.c. */ +const int OD_DIRECTION_OFFSETS_TABLE[8][3] = { + { -1 * OD_FILT_BSTRIDE + 1, -2 * OD_FILT_BSTRIDE + 2, + -3 * OD_FILT_BSTRIDE + 3 }, + { 0 * OD_FILT_BSTRIDE + 1, -1 * OD_FILT_BSTRIDE + 2, + -1 * OD_FILT_BSTRIDE + 3 }, + { 0 * OD_FILT_BSTRIDE + 1, 0 * OD_FILT_BSTRIDE + 2, 0 * OD_FILT_BSTRIDE + 3 }, + { 0 * OD_FILT_BSTRIDE + 1, 1 * OD_FILT_BSTRIDE + 2, 1 * OD_FILT_BSTRIDE + 3 }, + { 1 * OD_FILT_BSTRIDE + 1, 2 * OD_FILT_BSTRIDE + 2, 3 * OD_FILT_BSTRIDE + 3 }, + { 1 * OD_FILT_BSTRIDE + 0, 2 * OD_FILT_BSTRIDE + 1, 3 * OD_FILT_BSTRIDE + 1 }, + { 1 * OD_FILT_BSTRIDE + 0, 2 * OD_FILT_BSTRIDE + 0, 3 * OD_FILT_BSTRIDE + 0 }, + { 1 * OD_FILT_BSTRIDE + 0, 2 * OD_FILT_BSTRIDE - 1, 3 * OD_FILT_BSTRIDE - 1 }, +}; + +/* Detect direction. 0 means 45-degree up-right, 2 is horizontal, and so on. + The search minimizes the weighted variance along all the lines in a + particular direction, i.e. the squared error between the input and a + "predicted" block where each pixel is replaced by the average along a line + in a particular direction. Since each direction have the same sum(x^2) term, + that term is never computed. See Section 2, step 2, of: + http://jmvalin.ca/notes/intra_paint.pdf */ +int od_dir_find8_c(const uint16_t *img, int stride, int32_t *var, + int coeff_shift) { + int i; + int32_t cost[8] = { 0 }; + int partial[8][15] = { { 0 } }; + int32_t best_cost = 0; + int best_dir = 0; + /* Instead of dividing by n between 2 and 8, we multiply by 3*5*7*8/n. + The output is then 840 times larger, but we don't care for finding + the max. */ + static const int div_table[] = { 0, 840, 420, 280, 210, 168, 140, 120, 105 }; + for (i = 0; i < 8; i++) { + int j; + for (j = 0; j < 8; j++) { + int x; + /* We subtract 128 here to reduce the maximum range of the squared + partial sums. */ + x = (img[i * stride + j] >> coeff_shift) - 128; + partial[0][i + j] += x; + partial[1][i + j / 2] += x; + partial[2][i] += x; + partial[3][3 + i - j / 2] += x; + partial[4][7 + i - j] += x; + partial[5][3 - i / 2 + j] += x; + partial[6][j] += x; + partial[7][i / 2 + j] += x; + } + } + for (i = 0; i < 8; i++) { + cost[2] += partial[2][i] * partial[2][i]; + cost[6] += partial[6][i] * partial[6][i]; + } + cost[2] *= div_table[8]; + cost[6] *= div_table[8]; + for (i = 0; i < 7; i++) { + cost[0] += (partial[0][i] * partial[0][i] + + partial[0][14 - i] * partial[0][14 - i]) * + div_table[i + 1]; + cost[4] += (partial[4][i] * partial[4][i] + + partial[4][14 - i] * partial[4][14 - i]) * + div_table[i + 1]; + } + cost[0] += partial[0][7] * partial[0][7] * div_table[8]; + cost[4] += partial[4][7] * partial[4][7] * div_table[8]; + for (i = 1; i < 8; i += 2) { + int j; + for (j = 0; j < 4 + 1; j++) { + cost[i] += partial[i][3 + j] * partial[i][3 + j]; + } + cost[i] *= div_table[8]; + for (j = 0; j < 4 - 1; j++) { + cost[i] += (partial[i][j] * partial[i][j] + + partial[i][10 - j] * partial[i][10 - j]) * + div_table[2 * j + 2]; + } + } + for (i = 0; i < 8; i++) { + if (cost[i] > best_cost) { + best_cost = cost[i]; + best_dir = i; + } + } + /* Difference between the optimal variance and the variance along the + orthogonal direction. Again, the sum(x^2) terms cancel out. */ + *var = best_cost - cost[(best_dir + 4) & 7]; + /* We'd normally divide by 840, but dividing by 1024 is close enough + for what we're going to do with this. */ + *var >>= 10; + return best_dir; +} + +/* Smooth in the direction detected. */ +void od_filter_dering_direction_8x8_c(uint16_t *y, int ystride, + const uint16_t *in, int threshold, + int dir, int damping) { + int i; + int j; + int k; + static const int taps[3] = { 3, 2, 1 }; + for (i = 0; i < 8; i++) { + for (j = 0; j < 8; j++) { + int16_t sum; + int16_t xx; + int16_t yy; + xx = in[i * OD_FILT_BSTRIDE + j]; + sum = 0; + for (k = 0; k < 3; k++) { + int16_t p0; + int16_t p1; + p0 = in[i * OD_FILT_BSTRIDE + j + OD_DIRECTION_OFFSETS_TABLE[dir][k]] - + xx; + p1 = in[i * OD_FILT_BSTRIDE + j - OD_DIRECTION_OFFSETS_TABLE[dir][k]] - + xx; + sum += taps[k] * constrain(p0, threshold, damping); + sum += taps[k] * constrain(p1, threshold, damping); + } + sum = (sum + 8) >> 4; + yy = xx + sum; + y[i * ystride + j] = yy; + } + } +} + +/* Smooth in the direction detected. */ +void od_filter_dering_direction_4x4_c(uint16_t *y, int ystride, + const uint16_t *in, int threshold, + int dir, int damping) { + int i; + int j; + int k; + static const int taps[2] = { 4, 1 }; + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + int16_t sum; + int16_t xx; + int16_t yy; + xx = in[i * OD_FILT_BSTRIDE + j]; + sum = 0; + for (k = 0; k < 2; k++) { + int16_t p0; + int16_t p1; + p0 = in[i * OD_FILT_BSTRIDE + j + OD_DIRECTION_OFFSETS_TABLE[dir][k]] - + xx; + p1 = in[i * OD_FILT_BSTRIDE + j - OD_DIRECTION_OFFSETS_TABLE[dir][k]] - + xx; + sum += taps[k] * constrain(p0, threshold, damping); + sum += taps[k] * constrain(p1, threshold, damping); + } + sum = (sum + 8) >> 4; + yy = xx + sum; + y[i * ystride + j] = yy; + } + } +} + +/* Compute deringing filter threshold for an 8x8 block based on the + directional variance difference. A high variance difference means that we + have a highly directional pattern (e.g. a high contrast edge), so we can + apply more deringing. A low variance means that we either have a low + contrast edge, or a non-directional texture, so we want to be careful not + to blur. */ +static INLINE int od_adjust_thresh(int threshold, int32_t var) { + const int i = var >> 6 ? AOMMIN(get_msb(var >> 6), 12) : 0; + /* We use the variance of 8x8 blocks to adjust the threshold. */ + return var ? (threshold * (4 + i) + 8) >> 4 : 0; +} + +void copy_8x8_16bit_to_16bit_c(uint16_t *dst, int dstride, const uint16_t *src, + int sstride) { + int i, j; + for (i = 0; i < 8; i++) + for (j = 0; j < 8; j++) dst[i * dstride + j] = src[i * sstride + j]; +} + +void copy_4x4_16bit_to_16bit_c(uint16_t *dst, int dstride, const uint16_t *src, + int sstride) { + int i, j; + for (i = 0; i < 4; i++) + for (j = 0; j < 4; j++) dst[i * dstride + j] = src[i * sstride + j]; +} + +void copy_dering_16bit_to_16bit(uint16_t *dst, int dstride, uint16_t *src, + dering_list *dlist, int dering_count, + int bsize) { + int bi, bx, by; + + if (bsize == BLOCK_8X8) { + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + copy_8x8_16bit_to_16bit(&dst[(by << 3) * dstride + (bx << 3)], dstride, + &src[bi << (3 + 3)], 8); + } + } else if (bsize == BLOCK_4X8) { + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + copy_4x4_16bit_to_16bit(&dst[(by << 3) * dstride + (bx << 2)], dstride, + &src[bi << (3 + 2)], 4); + copy_4x4_16bit_to_16bit(&dst[((by << 3) + 4) * dstride + (bx << 2)], + dstride, &src[(bi << (3 + 2)) + 4 * 4], 4); + } + } else if (bsize == BLOCK_8X4) { + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + copy_4x4_16bit_to_16bit(&dst[(by << 2) * dstride + (bx << 3)], dstride, + &src[bi << (2 + 3)], 8); + copy_4x4_16bit_to_16bit(&dst[(by << 2) * dstride + (bx << 3) + 4], + dstride, &src[(bi << (2 + 3)) + 4], 8); + } + } else { + assert(bsize == BLOCK_4X4); + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + copy_4x4_16bit_to_16bit(&dst[(by << 2) * dstride + (bx << 2)], dstride, + &src[bi << (2 + 2)], 4); + } + } +} + +void copy_8x8_16bit_to_8bit_c(uint8_t *dst, int dstride, const uint16_t *src, + int sstride) { + int i, j; + for (i = 0; i < 8; i++) + for (j = 0; j < 8; j++) + dst[i * dstride + j] = (uint8_t)src[i * sstride + j]; +} + +void copy_4x4_16bit_to_8bit_c(uint8_t *dst, int dstride, const uint16_t *src, + int sstride) { + int i, j; + for (i = 0; i < 4; i++) + for (j = 0; j < 4; j++) + dst[i * dstride + j] = (uint8_t)src[i * sstride + j]; +} + +static void copy_dering_16bit_to_8bit(uint8_t *dst, int dstride, + const uint16_t *src, dering_list *dlist, + int dering_count, int bsize) { + int bi, bx, by; + if (bsize == BLOCK_8X8) { + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + copy_8x8_16bit_to_8bit(&dst[(by << 3) * dstride + (bx << 3)], dstride, + &src[bi << (3 + 3)], 8); + } + } else if (bsize == BLOCK_4X8) { + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + copy_4x4_16bit_to_8bit(&dst[(by << 3) * dstride + (bx << 2)], dstride, + &src[bi << (3 + 2)], 4); + copy_4x4_16bit_to_8bit(&dst[((by << 3) + 4) * dstride + (bx << 2)], + dstride, &src[(bi << (3 + 2)) + 4 * 4], 4); + } + } else if (bsize == BLOCK_8X4) { + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + copy_4x4_16bit_to_8bit(&dst[(by << 2) * dstride + (bx << 3)], dstride, + &src[bi << (2 + 3)], 8); + copy_4x4_16bit_to_8bit(&dst[(by << 2) * dstride + (bx << 3) + 4], dstride, + &src[(bi << (2 + 3)) + 4], 8); + } + } else { + assert(bsize == BLOCK_4X4); + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + copy_4x4_16bit_to_8bit(&dst[(by << 2) * dstride + (bx << 2)], dstride, + &src[bi << (2 * 2)], 4); + } + } +} + +int get_filter_skip(int level) { + int filter_skip = level & 1; + if (level == 1) filter_skip = 0; + return filter_skip; +} + +void od_dering(uint8_t *dst, int dstride, uint16_t *y, uint16_t *in, int xdec, + int ydec, int dir[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS], + int *dirinit, int var[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS], + int pli, dering_list *dlist, int dering_count, int level, + int clpf_strength, int clpf_damping, int dering_damping, + int coeff_shift, int skip_dering, int hbd) { + int bi; + int bx; + int by; + int bsize, bsizex, bsizey; + + int threshold = (level >> 1) << coeff_shift; + int filter_skip = get_filter_skip(level); + if (level == 1) threshold = 31 << coeff_shift; + + od_filter_dering_direction_func filter_dering_direction[] = { + od_filter_dering_direction_4x4, od_filter_dering_direction_8x8 + }; + clpf_damping += coeff_shift - (pli != AOM_PLANE_Y); + dering_damping += coeff_shift - (pli != AOM_PLANE_Y); + bsize = + ydec ? (xdec ? BLOCK_4X4 : BLOCK_8X4) : (xdec ? BLOCK_4X8 : BLOCK_8X8); + bsizex = 3 - xdec; + bsizey = 3 - ydec; + + if (!skip_dering) { + if (pli == 0) { + if (!dirinit || !*dirinit) { + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + dir[by][bx] = + od_dir_find8(&in[8 * by * OD_FILT_BSTRIDE + 8 * bx], + OD_FILT_BSTRIDE, &var[by][bx], coeff_shift); + } + if (dirinit) *dirinit = 1; + } + } + // Only run dering for non-zero threshold (which is always the case for + // 4:2:2 or 4:4:0). If we don't dering, we still need to eventually write + // something out in y[] later. + if (threshold != 0) { + assert(bsize == BLOCK_8X8 || bsize == BLOCK_4X4); + for (bi = 0; bi < dering_count; bi++) { + int t = !filter_skip && dlist[bi].skip ? 0 : threshold; + by = dlist[bi].by; + bx = dlist[bi].bx; + (filter_dering_direction[bsize == BLOCK_8X8])( + &y[bi << (bsizex + bsizey)], 1 << bsizex, + &in[(by * OD_FILT_BSTRIDE << bsizey) + (bx << bsizex)], + pli ? t : od_adjust_thresh(t, var[by][bx]), dir[by][bx], + dering_damping); + } + } + } + + if (clpf_strength) { + if (threshold && !skip_dering) + copy_dering_16bit_to_16bit(in, OD_FILT_BSTRIDE, y, dlist, dering_count, + bsize); + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + int py = by << bsizey; + int px = bx << bsizex; + + if (!filter_skip && dlist[bi].skip) continue; + if (!dst || hbd) { + // 16 bit destination if high bitdepth or 8 bit destination not given + (!threshold || (dir[by][bx] < 4 && dir[by][bx]) ? aom_clpf_block_hbd + : aom_clpf_hblock_hbd)( + dst ? (uint16_t *)dst + py * dstride + px + : &y[bi << (bsizex + bsizey)], + in + py * OD_FILT_BSTRIDE + px, dst && hbd ? dstride : 1 << bsizex, + OD_FILT_BSTRIDE, 1 << bsizex, 1 << bsizey, + clpf_strength << coeff_shift, clpf_damping); + } else { + // Do clpf and write the result to an 8 bit destination + (!threshold || (dir[by][bx] < 4 && dir[by][bx]) ? aom_clpf_block + : aom_clpf_hblock)( + dst + py * dstride + px, in + py * OD_FILT_BSTRIDE + px, dstride, + OD_FILT_BSTRIDE, 1 << bsizex, 1 << bsizey, + clpf_strength << coeff_shift, clpf_damping); + } + } + } else if (threshold != 0) { + // No clpf, so copy instead + if (hbd) { + copy_dering_16bit_to_16bit((uint16_t *)dst, dstride, y, dlist, + dering_count, bsize); + } else { + copy_dering_16bit_to_8bit(dst, dstride, y, dlist, dering_count, bsize); + } + } else if (dirinit) { + // If we're here, both dering and clpf are off, and we still haven't written + // anything to y[] yet, so we just copy the input to y[]. This is necessary + // only for av1_cdef_search() and only av1_cdef_search() sets dirinit. + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + int iy, ix; + // TODO(stemidts/jmvalin): SIMD optimisations + for (iy = 0; iy < 1 << bsizey; iy++) + for (ix = 0; ix < 1 << bsizex; ix++) + y[(bi << (bsizex + bsizey)) + (iy << bsizex) + ix] = + in[((by << bsizey) + iy) * OD_FILT_BSTRIDE + (bx << bsizex) + ix]; + } + } +} diff --git a/third_party/aom/av1/common/od_dering.h b/third_party/aom/av1/common/od_dering.h new file mode 100644 index 0000000000..4362001b45 --- /dev/null +++ b/third_party/aom/av1/common/od_dering.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if !defined(_dering_H) +#define _dering_H (1) + +#include "odintrin.h" + +#define OD_DERING_NBLOCKS (MAX_SB_SIZE / 8) + +/* We need to buffer three vertical lines. */ +#define OD_FILT_VBORDER (3) +/* We only need to buffer three horizontal pixels too, but let's align to + 16 bytes (8 x 16 bits) to make vectorization easier. */ +#define OD_FILT_HBORDER (8) +#define OD_FILT_BSTRIDE ALIGN_POWER_OF_TWO(MAX_SB_SIZE + 2 * OD_FILT_HBORDER, 3) + +#define OD_DERING_VERY_LARGE (30000) +#define OD_DERING_INBUF_SIZE \ + (OD_FILT_BSTRIDE * (MAX_SB_SIZE + 2 * OD_FILT_VBORDER)) + +extern const int OD_DIRECTION_OFFSETS_TABLE[8][3]; + +typedef struct { + uint8_t by; + uint8_t bx; + uint8_t skip; +} dering_list; + +typedef void (*od_filter_dering_direction_func)(uint16_t *y, int ystride, + const uint16_t *in, + int threshold, int dir, + int damping); +void copy_dering_16bit_to_16bit(uint16_t *dst, int dstride, uint16_t *src, + dering_list *dlist, int dering_count, + int bsize); + +int get_filter_skip(int level); + +void od_dering(uint8_t *dst, int dstride, uint16_t *y, uint16_t *in, int xdec, + int ydec, int dir[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS], + int *dirinit, int var[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS], + int pli, dering_list *dlist, int dering_count, int level, + int clpf_strength, int clpf_damping, int dering_damping, + int coeff_shift, int skip_dering, int hbd); +#endif diff --git a/third_party/aom/av1/common/od_dering_neon.c b/third_party/aom/av1/common/od_dering_neon.c new file mode 100644 index 0000000000..99441050ab --- /dev/null +++ b/third_party/aom/av1/common/od_dering_neon.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/aom_simd.h" +#define SIMD_FUNC(name) name##_neon +#include "./od_dering_simd.h" diff --git a/third_party/aom/av1/common/od_dering_simd.h b/third_party/aom/av1/common/od_dering_simd.h new file mode 100644 index 0000000000..4074e7e509 --- /dev/null +++ b/third_party/aom/av1/common/od_dering_simd.h @@ -0,0 +1,390 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./av1_rtcd.h" +#include "./cdef_simd.h" +#include "./od_dering.h" + +/* partial A is a 16-bit vector of the form: + [x8 x7 x6 x5 x4 x3 x2 x1] and partial B has the form: + [0 y1 y2 y3 y4 y5 y6 y7]. + This function computes (x1^2+y1^2)*C1 + (x2^2+y2^2)*C2 + ... + (x7^2+y2^7)*C7 + (x8^2+0^2)*C8 where the C1..C8 constants are in const1 + and const2. */ +static INLINE v128 fold_mul_and_sum(v128 partiala, v128 partialb, v128 const1, + v128 const2) { + v128 tmp; + /* Reverse partial B. */ + partialb = v128_shuffle_8( + partialb, v128_from_32(0x0f0e0100, 0x03020504, 0x07060908, 0x0b0a0d0c)); + /* Interleave the x and y values of identical indices and pair x8 with 0. */ + tmp = partiala; + partiala = v128_ziplo_16(partialb, partiala); + partialb = v128_ziphi_16(partialb, tmp); + /* Square and add the corresponding x and y values. */ + partiala = v128_madd_s16(partiala, partiala); + partialb = v128_madd_s16(partialb, partialb); + /* Multiply by constant. */ + partiala = v128_mullo_s32(partiala, const1); + partialb = v128_mullo_s32(partialb, const2); + /* Sum all results. */ + partiala = v128_add_32(partiala, partialb); + return partiala; +} + +static INLINE v128 hsum4(v128 x0, v128 x1, v128 x2, v128 x3) { + v128 t0, t1, t2, t3; + t0 = v128_ziplo_32(x1, x0); + t1 = v128_ziplo_32(x3, x2); + t2 = v128_ziphi_32(x1, x0); + t3 = v128_ziphi_32(x3, x2); + x0 = v128_ziplo_64(t1, t0); + x1 = v128_ziphi_64(t1, t0); + x2 = v128_ziplo_64(t3, t2); + x3 = v128_ziphi_64(t3, t2); + return v128_add_32(v128_add_32(x0, x1), v128_add_32(x2, x3)); +} + +/* Computes cost for directions 0, 5, 6 and 7. We can call this function again + to compute the remaining directions. */ +static INLINE v128 compute_directions(v128 lines[8], int32_t tmp_cost1[4]) { + v128 partial4a, partial4b, partial5a, partial5b, partial7a, partial7b; + v128 partial6; + v128 tmp; + /* Partial sums for lines 0 and 1. */ + partial4a = v128_shl_n_byte(lines[0], 14); + partial4b = v128_shr_n_byte(lines[0], 2); + partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[1], 12)); + partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[1], 4)); + tmp = v128_add_16(lines[0], lines[1]); + partial5a = v128_shl_n_byte(tmp, 10); + partial5b = v128_shr_n_byte(tmp, 6); + partial7a = v128_shl_n_byte(tmp, 4); + partial7b = v128_shr_n_byte(tmp, 12); + partial6 = tmp; + + /* Partial sums for lines 2 and 3. */ + partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[2], 10)); + partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[2], 6)); + partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[3], 8)); + partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[3], 8)); + tmp = v128_add_16(lines[2], lines[3]); + partial5a = v128_add_16(partial5a, v128_shl_n_byte(tmp, 8)); + partial5b = v128_add_16(partial5b, v128_shr_n_byte(tmp, 8)); + partial7a = v128_add_16(partial7a, v128_shl_n_byte(tmp, 6)); + partial7b = v128_add_16(partial7b, v128_shr_n_byte(tmp, 10)); + partial6 = v128_add_16(partial6, tmp); + + /* Partial sums for lines 4 and 5. */ + partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[4], 6)); + partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[4], 10)); + partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[5], 4)); + partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[5], 12)); + tmp = v128_add_16(lines[4], lines[5]); + partial5a = v128_add_16(partial5a, v128_shl_n_byte(tmp, 6)); + partial5b = v128_add_16(partial5b, v128_shr_n_byte(tmp, 10)); + partial7a = v128_add_16(partial7a, v128_shl_n_byte(tmp, 8)); + partial7b = v128_add_16(partial7b, v128_shr_n_byte(tmp, 8)); + partial6 = v128_add_16(partial6, tmp); + + /* Partial sums for lines 6 and 7. */ + partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[6], 2)); + partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[6], 14)); + partial4a = v128_add_16(partial4a, lines[7]); + tmp = v128_add_16(lines[6], lines[7]); + partial5a = v128_add_16(partial5a, v128_shl_n_byte(tmp, 4)); + partial5b = v128_add_16(partial5b, v128_shr_n_byte(tmp, 12)); + partial7a = v128_add_16(partial7a, v128_shl_n_byte(tmp, 10)); + partial7b = v128_add_16(partial7b, v128_shr_n_byte(tmp, 6)); + partial6 = v128_add_16(partial6, tmp); + + /* Compute costs in terms of partial sums. */ + partial4a = + fold_mul_and_sum(partial4a, partial4b, v128_from_32(210, 280, 420, 840), + v128_from_32(105, 120, 140, 168)); + partial7a = + fold_mul_and_sum(partial7a, partial7b, v128_from_32(210, 420, 0, 0), + v128_from_32(105, 105, 105, 140)); + partial5a = + fold_mul_and_sum(partial5a, partial5b, v128_from_32(210, 420, 0, 0), + v128_from_32(105, 105, 105, 140)); + partial6 = v128_madd_s16(partial6, partial6); + partial6 = v128_mullo_s32(partial6, v128_dup_32(105)); + + partial4a = hsum4(partial4a, partial5a, partial6, partial7a); + v128_store_unaligned(tmp_cost1, partial4a); + return partial4a; +} + +/* transpose and reverse the order of the lines -- equivalent to a 90-degree + counter-clockwise rotation of the pixels. */ +static INLINE void array_reverse_transpose_8x8(v128 *in, v128 *res) { + const v128 tr0_0 = v128_ziplo_16(in[1], in[0]); + const v128 tr0_1 = v128_ziplo_16(in[3], in[2]); + const v128 tr0_2 = v128_ziphi_16(in[1], in[0]); + const v128 tr0_3 = v128_ziphi_16(in[3], in[2]); + const v128 tr0_4 = v128_ziplo_16(in[5], in[4]); + const v128 tr0_5 = v128_ziplo_16(in[7], in[6]); + const v128 tr0_6 = v128_ziphi_16(in[5], in[4]); + const v128 tr0_7 = v128_ziphi_16(in[7], in[6]); + + const v128 tr1_0 = v128_ziplo_32(tr0_1, tr0_0); + const v128 tr1_1 = v128_ziplo_32(tr0_5, tr0_4); + const v128 tr1_2 = v128_ziphi_32(tr0_1, tr0_0); + const v128 tr1_3 = v128_ziphi_32(tr0_5, tr0_4); + const v128 tr1_4 = v128_ziplo_32(tr0_3, tr0_2); + const v128 tr1_5 = v128_ziplo_32(tr0_7, tr0_6); + const v128 tr1_6 = v128_ziphi_32(tr0_3, tr0_2); + const v128 tr1_7 = v128_ziphi_32(tr0_7, tr0_6); + + res[7] = v128_ziplo_64(tr1_1, tr1_0); + res[6] = v128_ziphi_64(tr1_1, tr1_0); + res[5] = v128_ziplo_64(tr1_3, tr1_2); + res[4] = v128_ziphi_64(tr1_3, tr1_2); + res[3] = v128_ziplo_64(tr1_5, tr1_4); + res[2] = v128_ziphi_64(tr1_5, tr1_4); + res[1] = v128_ziplo_64(tr1_7, tr1_6); + res[0] = v128_ziphi_64(tr1_7, tr1_6); +} + +int SIMD_FUNC(od_dir_find8)(const od_dering_in *img, int stride, int32_t *var, + int coeff_shift) { + int i; + int32_t cost[8]; + int32_t best_cost = 0; + int best_dir = 0; + v128 lines[8]; + for (i = 0; i < 8; i++) { + lines[i] = v128_load_unaligned(&img[i * stride]); + lines[i] = + v128_sub_16(v128_shr_s16(lines[i], coeff_shift), v128_dup_16(128)); + } + +#if defined(__SSE4_1__) + /* Compute "mostly vertical" directions. */ + __m128i dir47 = compute_directions(lines, cost + 4); + + array_reverse_transpose_8x8(lines, lines); + + /* Compute "mostly horizontal" directions. */ + __m128i dir03 = compute_directions(lines, cost); + + __m128i max = _mm_max_epi32(dir03, dir47); + max = _mm_max_epi32(max, _mm_shuffle_epi32(max, _MM_SHUFFLE(1, 0, 3, 2))); + max = _mm_max_epi32(max, _mm_shuffle_epi32(max, _MM_SHUFFLE(2, 3, 0, 1))); + best_cost = _mm_cvtsi128_si32(max); + __m128i t = + _mm_packs_epi32(_mm_cmpeq_epi32(max, dir03), _mm_cmpeq_epi32(max, dir47)); + best_dir = _mm_movemask_epi8(_mm_packs_epi16(t, t)); + best_dir = get_msb(best_dir ^ (best_dir - 1)); // Count trailing zeros +#else + /* Compute "mostly vertical" directions. */ + compute_directions(lines, cost + 4); + + array_reverse_transpose_8x8(lines, lines); + + /* Compute "mostly horizontal" directions. */ + compute_directions(lines, cost); + + for (i = 0; i < 8; i++) { + if (cost[i] > best_cost) { + best_cost = cost[i]; + best_dir = i; + } + } +#endif + + /* Difference between the optimal variance and the variance along the + orthogonal direction. Again, the sum(x^2) terms cancel out. */ + *var = best_cost - cost[(best_dir + 4) & 7]; + /* We'd normally divide by 840, but dividing by 1024 is close enough + for what we're going to do with this. */ + *var >>= 10; + return best_dir; +} + +void SIMD_FUNC(od_filter_dering_direction_4x4)(uint16_t *y, int ystride, + const uint16_t *in, + int threshold, int dir, + int damping) { + int i; + v128 p0, p1, sum, row, res; + int o1 = OD_DIRECTION_OFFSETS_TABLE[dir][0]; + int o2 = OD_DIRECTION_OFFSETS_TABLE[dir][1]; + + if (threshold) damping -= get_msb(threshold); + for (i = 0; i < 4; i += 2) { + sum = v128_zero(); + row = v128_from_v64(v64_load_aligned(&in[i * OD_FILT_BSTRIDE]), + v64_load_aligned(&in[(i + 1) * OD_FILT_BSTRIDE])); + + // p0 = constrain16(in[i*OD_FILT_BSTRIDE + offset], row, threshold, damping) + p0 = v128_from_v64(v64_load_unaligned(&in[i * OD_FILT_BSTRIDE + o1]), + v64_load_unaligned(&in[(i + 1) * OD_FILT_BSTRIDE + o1])); + p0 = constrain16(p0, row, threshold, damping); + + // p1 = constrain16(in[i*OD_FILT_BSTRIDE - offset], row, threshold, damping) + p1 = v128_from_v64(v64_load_unaligned(&in[i * OD_FILT_BSTRIDE - o1]), + v64_load_unaligned(&in[(i + 1) * OD_FILT_BSTRIDE - o1])); + p1 = constrain16(p1, row, threshold, damping); + + // sum += 4 * (p0 + p1) + sum = v128_add_16(sum, v128_shl_n_16(v128_add_16(p0, p1), 2)); + + // p0 = constrain16(in[i*OD_FILT_BSTRIDE + offset], row, threshold, damping) + p0 = v128_from_v64(v64_load_unaligned(&in[i * OD_FILT_BSTRIDE + o2]), + v64_load_unaligned(&in[(i + 1) * OD_FILT_BSTRIDE + o2])); + p0 = constrain16(p0, row, threshold, damping); + + // p1 = constrain16(in[i*OD_FILT_BSTRIDE - offset], row, threshold, damping) + p1 = v128_from_v64(v64_load_unaligned(&in[i * OD_FILT_BSTRIDE - o2]), + v64_load_unaligned(&in[(i + 1) * OD_FILT_BSTRIDE - o2])); + p1 = constrain16(p1, row, threshold, damping); + + // sum += 1 * (p0 + p1) + sum = v128_add_16(sum, v128_add_16(p0, p1)); + + // res = row + ((sum + 8) >> 4) + res = v128_add_16(sum, v128_dup_16(8)); + res = v128_shr_n_s16(res, 4); + res = v128_add_16(row, res); + v64_store_aligned(&y[i * ystride], v128_high_v64(res)); + v64_store_aligned(&y[(i + 1) * ystride], v128_low_v64(res)); + } +} + +void SIMD_FUNC(od_filter_dering_direction_8x8)(uint16_t *y, int ystride, + const uint16_t *in, + int threshold, int dir, + int damping) { + int i; + v128 sum, p0, p1, row, res; + int o1 = OD_DIRECTION_OFFSETS_TABLE[dir][0]; + int o2 = OD_DIRECTION_OFFSETS_TABLE[dir][1]; + int o3 = OD_DIRECTION_OFFSETS_TABLE[dir][2]; + + if (threshold) damping -= get_msb(threshold); + for (i = 0; i < 8; i++) { + sum = v128_zero(); + row = v128_load_aligned(&in[i * OD_FILT_BSTRIDE]); + + // p0 = constrain16(in[i*OD_FILT_BSTRIDE + offset], row, threshold, damping) + p0 = v128_load_unaligned(&in[i * OD_FILT_BSTRIDE + o1]); + p0 = constrain16(p0, row, threshold, damping); + + // p1 = constrain16(in[i*OD_FILT_BSTRIDE - offset], row, threshold, damping) + p1 = v128_load_unaligned(&in[i * OD_FILT_BSTRIDE - o1]); + p1 = constrain16(p1, row, threshold, damping); + + // sum += 3 * (p0 + p1) + p0 = v128_add_16(p0, p1); + p0 = v128_add_16(p0, v128_shl_n_16(p0, 1)); + sum = v128_add_16(sum, p0); + + // p0 = constrain16(in[i*OD_FILT_BSTRIDE + offset], row, threshold, damping) + p0 = v128_load_unaligned(&in[i * OD_FILT_BSTRIDE + o2]); + p0 = constrain16(p0, row, threshold, damping); + + // p1 = constrain16(in[i*OD_FILT_BSTRIDE - offset], row, threshold, damping) + p1 = v128_load_unaligned(&in[i * OD_FILT_BSTRIDE - o2]); + p1 = constrain16(p1, row, threshold, damping); + + // sum += 2 * (p0 + p1) + p0 = v128_shl_n_16(v128_add_16(p0, p1), 1); + sum = v128_add_16(sum, p0); + + // p0 = constrain16(in[i*OD_FILT_BSTRIDE + offset], row, threshold, damping) + p0 = v128_load_unaligned(&in[i * OD_FILT_BSTRIDE + o3]); + p0 = constrain16(p0, row, threshold, damping); + + // p1 = constrain16(in[i*OD_FILT_BSTRIDE - offset], row, threshold, damping) + p1 = v128_load_unaligned(&in[i * OD_FILT_BSTRIDE - o3]); + p1 = constrain16(p1, row, threshold, damping); + + // sum += (p0 + p1) + p0 = v128_add_16(p0, p1); + sum = v128_add_16(sum, p0); + + // res = row + ((sum + 8) >> 4) + res = v128_add_16(sum, v128_dup_16(8)); + res = v128_shr_n_s16(res, 4); + res = v128_add_16(row, res); + v128_store_unaligned(&y[i * ystride], res); + } +} + +void SIMD_FUNC(copy_8x8_16bit_to_8bit)(uint8_t *dst, int dstride, + const uint16_t *src, int sstride) { + int i; + for (i = 0; i < 8; i++) { + v128 row = v128_load_unaligned(&src[i * sstride]); + row = v128_pack_s16_u8(row, row); + v64_store_unaligned(&dst[i * dstride], v128_low_v64(row)); + } +} + +void SIMD_FUNC(copy_4x4_16bit_to_8bit)(uint8_t *dst, int dstride, + const uint16_t *src, int sstride) { + int i; + for (i = 0; i < 4; i++) { + v128 row = v128_load_unaligned(&src[i * sstride]); + row = v128_pack_s16_u8(row, row); + u32_store_unaligned(&dst[i * dstride], v128_low_u32(row)); + } +} + +void SIMD_FUNC(copy_8x8_16bit_to_16bit)(uint16_t *dst, int dstride, + const uint16_t *src, int sstride) { + int i; + for (i = 0; i < 8; i++) { + v128 row = v128_load_unaligned(&src[i * sstride]); + v128_store_unaligned(&dst[i * dstride], row); + } +} + +void SIMD_FUNC(copy_4x4_16bit_to_16bit)(uint16_t *dst, int dstride, + const uint16_t *src, int sstride) { + int i; + for (i = 0; i < 4; i++) { + v64 row = v64_load_unaligned(&src[i * sstride]); + v64_store_unaligned(&dst[i * dstride], row); + } +} + +void SIMD_FUNC(copy_rect8_8bit_to_16bit)(uint16_t *dst, int dstride, + const uint8_t *src, int sstride, int v, + int h) { + int i, j; + for (i = 0; i < v; i++) { + for (j = 0; j < (h & ~0x7); j += 8) { + v64 row = v64_load_unaligned(&src[i * sstride + j]); + v128_store_unaligned(&dst[i * dstride + j], v128_unpack_u8_s16(row)); + } + for (; j < h; j++) { + dst[i * dstride + j] = src[i * sstride + j]; + } + } +} + +void SIMD_FUNC(copy_rect8_16bit_to_16bit)(uint16_t *dst, int dstride, + const uint16_t *src, int sstride, + int v, int h) { + int i, j; + for (i = 0; i < v; i++) { + for (j = 0; j < (h & ~0x7); j += 8) { + v128 row = v128_load_unaligned(&src[i * sstride + j]); + v128_store_unaligned(&dst[i * dstride + j], row); + } + for (; j < h; j++) { + dst[i * dstride + j] = src[i * sstride + j]; + } + } +} diff --git a/third_party/aom/av1/common/od_dering_sse2.c b/third_party/aom/av1/common/od_dering_sse2.c new file mode 100644 index 0000000000..8a2a62f6c5 --- /dev/null +++ b/third_party/aom/av1/common/od_dering_sse2.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/aom_simd.h" +#define SIMD_FUNC(name) name##_sse2 +#include "./od_dering_simd.h" diff --git a/third_party/aom/av1/common/od_dering_sse4.c b/third_party/aom/av1/common/od_dering_sse4.c new file mode 100644 index 0000000000..0769db9fd9 --- /dev/null +++ b/third_party/aom/av1/common/od_dering_sse4.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/aom_simd.h" +#define SIMD_FUNC(name) name##_sse4_1 +#include "./od_dering_simd.h" diff --git a/third_party/aom/av1/common/od_dering_ssse3.c b/third_party/aom/av1/common/od_dering_ssse3.c new file mode 100644 index 0000000000..99df62b6b5 --- /dev/null +++ b/third_party/aom/av1/common/od_dering_ssse3.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/aom_simd.h" +#define SIMD_FUNC(name) name##_ssse3 +#include "./od_dering_simd.h" diff --git a/third_party/aom/av1/common/odintrin.c b/third_party/aom/av1/common/odintrin.c new file mode 100644 index 0000000000..868efacc99 --- /dev/null +++ b/third_party/aom/av1/common/odintrin.c @@ -0,0 +1,551 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#include "av1/common/odintrin.h" + +#if defined(OD_ENABLE_ASSERTIONS) +# include + +void od_fatal_impl(const char *_str, const char *_file, int _line) { + fprintf(stderr, "Fatal (internal) error in %s, line %d: %s\n", + _file, _line, _str); + abort(); +} +#endif + +/*Constants for use with OD_DIVU_SMALL(). + See \cite{Rob05} for details on computing these constants. + @INPROCEEDINGS{Rob05, + author="Arch D. Robison", + title="{N}-bit Unsigned Division via {N}-bit Multiply-Add", + booktitle="Proc. of the 17th IEEE Symposium on Computer Arithmetic + (ARITH'05)", + pages="131--139", + address="Cape Cod, MA", + month=Jun, + year=2005 + }*/ +uint32_t OD_DIVU_SMALL_CONSTS[OD_DIVU_DMAX][2] = { + { 0xFFFFFFFF, 0xFFFFFFFF }, { 0xFFFFFFFF, 0xFFFFFFFF }, + { 0xAAAAAAAB, 0 }, { 0xFFFFFFFF, 0xFFFFFFFF }, + { 0xCCCCCCCD, 0 }, { 0xAAAAAAAB, 0 }, + { 0x92492492, 0x92492492 }, { 0xFFFFFFFF, 0xFFFFFFFF }, + { 0xE38E38E4, 0 }, { 0xCCCCCCCD, 0 }, + { 0xBA2E8BA3, 0 }, { 0xAAAAAAAB, 0 }, + { 0x9D89D89E, 0 }, { 0x92492492, 0x92492492 }, + { 0x88888889, 0 }, { 0xFFFFFFFF, 0xFFFFFFFF }, + { 0xF0F0F0F1, 0 }, { 0xE38E38E4, 0 }, + { 0xD79435E5, 0xD79435E5 }, { 0xCCCCCCCD, 0 }, + { 0xC30C30C3, 0xC30C30C3 }, { 0xBA2E8BA3, 0 }, + { 0xB21642C9, 0 }, { 0xAAAAAAAB, 0 }, + { 0xA3D70A3E, 0 }, { 0x9D89D89E, 0 }, + { 0x97B425ED, 0x97B425ED }, { 0x92492492, 0x92492492 }, + { 0x8D3DCB09, 0 }, { 0x88888889, 0 }, + { 0x84210842, 0x84210842 }, { 0xFFFFFFFF, 0xFFFFFFFF }, + { 0xF83E0F84, 0 }, { 0xF0F0F0F1, 0 }, + { 0xEA0EA0EA, 0xEA0EA0EA }, { 0xE38E38E4, 0 }, + { 0xDD67C8A6, 0xDD67C8A6 }, { 0xD79435E5, 0xD79435E5 }, + { 0xD20D20D2, 0xD20D20D2 }, { 0xCCCCCCCD, 0 }, + { 0xC7CE0C7D, 0 }, { 0xC30C30C3, 0xC30C30C3 }, + { 0xBE82FA0C, 0 }, { 0xBA2E8BA3, 0 }, + { 0xB60B60B6, 0xB60B60B6 }, { 0xB21642C9, 0 }, + { 0xAE4C415D, 0 }, { 0xAAAAAAAB, 0 }, + { 0xA72F053A, 0 }, { 0xA3D70A3E, 0 }, + { 0xA0A0A0A1, 0 }, { 0x9D89D89E, 0 }, + { 0x9A90E7D9, 0x9A90E7D9 }, { 0x97B425ED, 0x97B425ED }, + { 0x94F2094F, 0x94F2094F }, { 0x92492492, 0x92492492 }, + { 0x8FB823EE, 0x8FB823EE }, { 0x8D3DCB09, 0 }, + { 0x8AD8F2FC, 0 }, { 0x88888889, 0 }, + { 0x864B8A7E, 0 }, { 0x84210842, 0x84210842 }, + { 0x82082082, 0x82082082 }, { 0xFFFFFFFF, 0xFFFFFFFF }, + { 0xFC0FC0FD, 0 }, { 0xF83E0F84, 0 }, + { 0xF4898D60, 0 }, { 0xF0F0F0F1, 0 }, + { 0xED7303B6, 0 }, { 0xEA0EA0EA, 0xEA0EA0EA }, + { 0xE6C2B449, 0 }, { 0xE38E38E4, 0 }, + { 0xE070381C, 0xE070381C }, { 0xDD67C8A6, 0xDD67C8A6 }, + { 0xDA740DA8, 0 }, { 0xD79435E5, 0xD79435E5 }, + { 0xD4C77B04, 0 }, { 0xD20D20D2, 0xD20D20D2 }, + { 0xCF6474A9, 0 }, { 0xCCCCCCCD, 0 }, + { 0xCA4587E7, 0 }, { 0xC7CE0C7D, 0 }, + { 0xC565C87C, 0 }, { 0xC30C30C3, 0xC30C30C3 }, + { 0xC0C0C0C1, 0 }, { 0xBE82FA0C, 0 }, + { 0xBC52640C, 0 }, { 0xBA2E8BA3, 0 }, + { 0xB81702E1, 0 }, { 0xB60B60B6, 0xB60B60B6 }, + { 0xB40B40B4, 0xB40B40B4 }, { 0xB21642C9, 0 }, + { 0xB02C0B03, 0 }, { 0xAE4C415D, 0 }, + { 0xAC769184, 0xAC769184 }, { 0xAAAAAAAB, 0 }, + { 0xA8E83F57, 0xA8E83F57 }, { 0xA72F053A, 0 }, + { 0xA57EB503, 0 }, { 0xA3D70A3E, 0 }, + { 0xA237C32B, 0xA237C32B }, { 0xA0A0A0A1, 0 }, + { 0x9F1165E7, 0x9F1165E7 }, { 0x9D89D89E, 0 }, + { 0x9C09C09C, 0x9C09C09C }, { 0x9A90E7D9, 0x9A90E7D9 }, + { 0x991F1A51, 0x991F1A51 }, { 0x97B425ED, 0x97B425ED }, + { 0x964FDA6C, 0x964FDA6C }, { 0x94F2094F, 0x94F2094F }, + { 0x939A85C4, 0x939A85C4 }, { 0x92492492, 0x92492492 }, + { 0x90FDBC09, 0x90FDBC09 }, { 0x8FB823EE, 0x8FB823EE }, + { 0x8E78356D, 0x8E78356D }, { 0x8D3DCB09, 0 }, + { 0x8C08C08C, 0x8C08C08C }, { 0x8AD8F2FC, 0 }, + { 0x89AE408A, 0 }, { 0x88888889, 0 }, + { 0x8767AB5F, 0x8767AB5F }, { 0x864B8A7E, 0 }, + { 0x85340853, 0x85340853 }, { 0x84210842, 0x84210842 }, + { 0x83126E98, 0 }, { 0x82082082, 0x82082082 }, + { 0x81020408, 0x81020408 }, { 0xFFFFFFFF, 0xFFFFFFFF }, + { 0xFE03F810, 0 }, { 0xFC0FC0FD, 0 }, + { 0xFA232CF3, 0 }, { 0xF83E0F84, 0 }, + { 0xF6603D99, 0 }, { 0xF4898D60, 0 }, + { 0xF2B9D649, 0 }, { 0xF0F0F0F1, 0 }, + { 0xEF2EB720, 0 }, { 0xED7303B6, 0 }, + { 0xEBBDB2A6, 0 }, { 0xEA0EA0EA, 0xEA0EA0EA }, + { 0xE865AC7C, 0 }, { 0xE6C2B449, 0 }, + { 0xE525982B, 0 }, { 0xE38E38E4, 0 }, + { 0xE1FC780F, 0 }, { 0xE070381C, 0xE070381C }, + { 0xDEE95C4D, 0 }, { 0xDD67C8A6, 0xDD67C8A6 }, + { 0xDBEB61EF, 0 }, { 0xDA740DA8, 0 }, + { 0xD901B204, 0 }, { 0xD79435E5, 0xD79435E5 }, + { 0xD62B80D7, 0 }, { 0xD4C77B04, 0 }, + { 0xD3680D37, 0 }, { 0xD20D20D2, 0xD20D20D2 }, + { 0xD0B69FCC, 0 }, { 0xCF6474A9, 0 }, + { 0xCE168A77, 0xCE168A77 }, { 0xCCCCCCCD, 0 }, + { 0xCB8727C1, 0 }, { 0xCA4587E7, 0 }, + { 0xC907DA4F, 0 }, { 0xC7CE0C7D, 0 }, + { 0xC6980C6A, 0 }, { 0xC565C87C, 0 }, + { 0xC4372F86, 0 }, { 0xC30C30C3, 0xC30C30C3 }, + { 0xC1E4BBD6, 0 }, { 0xC0C0C0C1, 0 }, + { 0xBFA02FE8, 0xBFA02FE8 }, { 0xBE82FA0C, 0 }, + { 0xBD691047, 0xBD691047 }, { 0xBC52640C, 0 }, + { 0xBB3EE722, 0 }, { 0xBA2E8BA3, 0 }, + { 0xB92143FA, 0xB92143FA }, { 0xB81702E1, 0 }, + { 0xB70FBB5A, 0xB70FBB5A }, { 0xB60B60B6, 0xB60B60B6 }, + { 0xB509E68B, 0 }, { 0xB40B40B4, 0xB40B40B4 }, + { 0xB30F6353, 0 }, { 0xB21642C9, 0 }, + { 0xB11FD3B8, 0xB11FD3B8 }, { 0xB02C0B03, 0 }, + { 0xAF3ADDC7, 0 }, { 0xAE4C415D, 0 }, + { 0xAD602B58, 0xAD602B58 }, { 0xAC769184, 0xAC769184 }, + { 0xAB8F69E3, 0 }, { 0xAAAAAAAB, 0 }, + { 0xA9C84A48, 0 }, { 0xA8E83F57, 0xA8E83F57 }, + { 0xA80A80A8, 0xA80A80A8 }, { 0xA72F053A, 0 }, + { 0xA655C439, 0xA655C439 }, { 0xA57EB503, 0 }, + { 0xA4A9CF1E, 0 }, { 0xA3D70A3E, 0 }, + { 0xA3065E40, 0 }, { 0xA237C32B, 0xA237C32B }, + { 0xA16B312F, 0 }, { 0xA0A0A0A1, 0 }, + { 0x9FD809FE, 0 }, { 0x9F1165E7, 0x9F1165E7 }, + { 0x9E4CAD24, 0 }, { 0x9D89D89E, 0 }, + { 0x9CC8E161, 0 }, { 0x9C09C09C, 0x9C09C09C }, + { 0x9B4C6F9F, 0 }, { 0x9A90E7D9, 0x9A90E7D9 }, + { 0x99D722DB, 0 }, { 0x991F1A51, 0x991F1A51 }, + { 0x9868C80A, 0 }, { 0x97B425ED, 0x97B425ED }, + { 0x97012E02, 0x97012E02 }, { 0x964FDA6C, 0x964FDA6C }, + { 0x95A02568, 0x95A02568 }, { 0x94F2094F, 0x94F2094F }, + { 0x94458094, 0x94458094 }, { 0x939A85C4, 0x939A85C4 }, + { 0x92F11384, 0x92F11384 }, { 0x92492492, 0x92492492 }, + { 0x91A2B3C5, 0 }, { 0x90FDBC09, 0x90FDBC09 }, + { 0x905A3863, 0x905A3863 }, { 0x8FB823EE, 0x8FB823EE }, + { 0x8F1779DA, 0 }, { 0x8E78356D, 0x8E78356D }, + { 0x8DDA5202, 0x8DDA5202 }, { 0x8D3DCB09, 0 }, + { 0x8CA29C04, 0x8CA29C04 }, { 0x8C08C08C, 0x8C08C08C }, + { 0x8B70344A, 0x8B70344A }, { 0x8AD8F2FC, 0 }, + { 0x8A42F870, 0x8A42F870 }, { 0x89AE408A, 0 }, + { 0x891AC73B, 0 }, { 0x88888889, 0 }, + { 0x87F78088, 0 }, { 0x8767AB5F, 0x8767AB5F }, + { 0x86D90545, 0 }, { 0x864B8A7E, 0 }, + { 0x85BF3761, 0x85BF3761 }, { 0x85340853, 0x85340853 }, + { 0x84A9F9C8, 0x84A9F9C8 }, { 0x84210842, 0x84210842 }, + { 0x83993052, 0x83993052 }, { 0x83126E98, 0 }, + { 0x828CBFBF, 0 }, { 0x82082082, 0x82082082 }, + { 0x81848DA9, 0 }, { 0x81020408, 0x81020408 }, + { 0x80808081, 0 }, { 0xFFFFFFFF, 0xFFFFFFFF }, + { 0xFF00FF01, 0 }, { 0xFE03F810, 0 }, + { 0xFD08E551, 0 }, { 0xFC0FC0FD, 0 }, + { 0xFB188566, 0 }, { 0xFA232CF3, 0 }, + { 0xF92FB222, 0 }, { 0xF83E0F84, 0 }, + { 0xF74E3FC3, 0 }, { 0xF6603D99, 0 }, + { 0xF57403D6, 0 }, { 0xF4898D60, 0 }, + { 0xF3A0D52D, 0 }, { 0xF2B9D649, 0 }, + { 0xF1D48BCF, 0 }, { 0xF0F0F0F1, 0 }, + { 0xF00F00F0, 0xF00F00F0 }, { 0xEF2EB720, 0 }, + { 0xEE500EE5, 0xEE500EE5 }, { 0xED7303B6, 0 }, + { 0xEC979119, 0 }, { 0xEBBDB2A6, 0 }, + { 0xEAE56404, 0 }, { 0xEA0EA0EA, 0xEA0EA0EA }, + { 0xE9396520, 0 }, { 0xE865AC7C, 0 }, + { 0xE79372E3, 0 }, { 0xE6C2B449, 0 }, + { 0xE5F36CB0, 0xE5F36CB0 }, { 0xE525982B, 0 }, + { 0xE45932D8, 0 }, { 0xE38E38E4, 0 }, + { 0xE2C4A689, 0 }, { 0xE1FC780F, 0 }, + { 0xE135A9CA, 0 }, { 0xE070381C, 0xE070381C }, + { 0xDFAC1F75, 0 }, { 0xDEE95C4D, 0 }, + { 0xDE27EB2D, 0 }, { 0xDD67C8A6, 0xDD67C8A6 }, + { 0xDCA8F159, 0 }, { 0xDBEB61EF, 0 }, + { 0xDB2F171E, 0 }, { 0xDA740DA8, 0 }, + { 0xD9BA4257, 0 }, { 0xD901B204, 0 }, + { 0xD84A598F, 0 }, { 0xD79435E5, 0xD79435E5 }, + { 0xD6DF43FD, 0 }, { 0xD62B80D7, 0 }, + { 0xD578E97D, 0 }, { 0xD4C77B04, 0 }, + { 0xD417328A, 0 }, { 0xD3680D37, 0 }, + { 0xD2BA083C, 0 }, { 0xD20D20D2, 0xD20D20D2 }, + { 0xD161543E, 0xD161543E }, { 0xD0B69FCC, 0 }, + { 0xD00D00D0, 0xD00D00D0 }, { 0xCF6474A9, 0 }, + { 0xCEBCF8BC, 0 }, { 0xCE168A77, 0xCE168A77 }, + { 0xCD712753, 0 }, { 0xCCCCCCCD, 0 }, + { 0xCC29786D, 0 }, { 0xCB8727C1, 0 }, + { 0xCAE5D85F, 0xCAE5D85F }, { 0xCA4587E7, 0 }, + { 0xC9A633FD, 0 }, { 0xC907DA4F, 0 }, + { 0xC86A7890, 0xC86A7890 }, { 0xC7CE0C7D, 0 }, + { 0xC73293D8, 0 }, { 0xC6980C6A, 0 }, + { 0xC5FE7403, 0xC5FE7403 }, { 0xC565C87C, 0 }, + { 0xC4CE07B0, 0xC4CE07B0 }, { 0xC4372F86, 0 }, + { 0xC3A13DE6, 0xC3A13DE6 }, { 0xC30C30C3, 0xC30C30C3 }, + { 0xC2780614, 0 }, { 0xC1E4BBD6, 0 }, + { 0xC152500C, 0xC152500C }, { 0xC0C0C0C1, 0 }, + { 0xC0300C03, 0xC0300C03 }, { 0xBFA02FE8, 0xBFA02FE8 }, + { 0xBF112A8B, 0 }, { 0xBE82FA0C, 0 }, + { 0xBDF59C92, 0 }, { 0xBD691047, 0xBD691047 }, + { 0xBCDD535E, 0 }, { 0xBC52640C, 0 }, + { 0xBBC8408D, 0 }, { 0xBB3EE722, 0 }, + { 0xBAB65610, 0xBAB65610 }, { 0xBA2E8BA3, 0 }, + { 0xB9A7862A, 0xB9A7862A }, { 0xB92143FA, 0xB92143FA }, + { 0xB89BC36D, 0 }, { 0xB81702E1, 0 }, + { 0xB79300B8, 0 }, { 0xB70FBB5A, 0xB70FBB5A }, + { 0xB68D3134, 0xB68D3134 }, { 0xB60B60B6, 0xB60B60B6 }, + { 0xB58A4855, 0xB58A4855 }, { 0xB509E68B, 0 }, + { 0xB48A39D4, 0xB48A39D4 }, { 0xB40B40B4, 0xB40B40B4 }, + { 0xB38CF9B0, 0xB38CF9B0 }, { 0xB30F6353, 0 }, + { 0xB2927C2A, 0 }, { 0xB21642C9, 0 }, + { 0xB19AB5C5, 0 }, { 0xB11FD3B8, 0xB11FD3B8 }, + { 0xB0A59B42, 0 }, { 0xB02C0B03, 0 }, + { 0xAFB321A1, 0xAFB321A1 }, { 0xAF3ADDC7, 0 }, + { 0xAEC33E20, 0 }, { 0xAE4C415D, 0 }, + { 0xADD5E632, 0xADD5E632 }, { 0xAD602B58, 0xAD602B58 }, + { 0xACEB0F89, 0xACEB0F89 }, { 0xAC769184, 0xAC769184 }, + { 0xAC02B00B, 0 }, { 0xAB8F69E3, 0 }, + { 0xAB1CBDD4, 0 }, { 0xAAAAAAAB, 0 }, + { 0xAA392F36, 0 }, { 0xA9C84A48, 0 }, + { 0xA957FAB5, 0xA957FAB5 }, { 0xA8E83F57, 0xA8E83F57 }, + { 0xA8791709, 0 }, { 0xA80A80A8, 0xA80A80A8 }, + { 0xA79C7B17, 0 }, { 0xA72F053A, 0 }, + { 0xA6C21DF7, 0 }, { 0xA655C439, 0xA655C439 }, + { 0xA5E9F6ED, 0xA5E9F6ED }, { 0xA57EB503, 0 }, + { 0xA513FD6C, 0 }, { 0xA4A9CF1E, 0 }, + { 0xA4402910, 0xA4402910 }, { 0xA3D70A3E, 0 }, + { 0xA36E71A3, 0 }, { 0xA3065E40, 0 }, + { 0xA29ECF16, 0xA29ECF16 }, { 0xA237C32B, 0xA237C32B }, + { 0xA1D13986, 0 }, { 0xA16B312F, 0 }, + { 0xA105A933, 0 }, { 0xA0A0A0A1, 0 }, + { 0xA03C1689, 0 }, { 0x9FD809FE, 0 }, + { 0x9F747A15, 0x9F747A15 }, { 0x9F1165E7, 0x9F1165E7 }, + { 0x9EAECC8D, 0x9EAECC8D }, { 0x9E4CAD24, 0 }, + { 0x9DEB06C9, 0x9DEB06C9 }, { 0x9D89D89E, 0 }, + { 0x9D2921C4, 0 }, { 0x9CC8E161, 0 }, + { 0x9C69169B, 0x9C69169B }, { 0x9C09C09C, 0x9C09C09C }, + { 0x9BAADE8E, 0x9BAADE8E }, { 0x9B4C6F9F, 0 }, + { 0x9AEE72FD, 0 }, { 0x9A90E7D9, 0x9A90E7D9 }, + { 0x9A33CD67, 0x9A33CD67 }, { 0x99D722DB, 0 }, + { 0x997AE76B, 0x997AE76B }, { 0x991F1A51, 0x991F1A51 }, + { 0x98C3BAC7, 0x98C3BAC7 }, { 0x9868C80A, 0 }, + { 0x980E4156, 0x980E4156 }, { 0x97B425ED, 0x97B425ED }, + { 0x975A7510, 0 }, { 0x97012E02, 0x97012E02 }, + { 0x96A8500A, 0 }, { 0x964FDA6C, 0x964FDA6C }, + { 0x95F7CC73, 0 }, { 0x95A02568, 0x95A02568 }, + { 0x9548E498, 0 }, { 0x94F2094F, 0x94F2094F }, + { 0x949B92DE, 0 }, { 0x94458094, 0x94458094 }, + { 0x93EFD1C5, 0x93EFD1C5 }, { 0x939A85C4, 0x939A85C4 }, + { 0x93459BE7, 0 }, { 0x92F11384, 0x92F11384 }, + { 0x929CEBF5, 0 }, { 0x92492492, 0x92492492 }, + { 0x91F5BCB9, 0 }, { 0x91A2B3C5, 0 }, + { 0x91500915, 0x91500915 }, { 0x90FDBC09, 0x90FDBC09 }, + { 0x90ABCC02, 0x90ABCC02 }, { 0x905A3863, 0x905A3863 }, + { 0x90090090, 0x90090090 }, { 0x8FB823EE, 0x8FB823EE }, + { 0x8F67A1E4, 0 }, { 0x8F1779DA, 0 }, + { 0x8EC7AB3A, 0 }, { 0x8E78356D, 0x8E78356D }, + { 0x8E2917E1, 0 }, { 0x8DDA5202, 0x8DDA5202 }, + { 0x8D8BE340, 0 }, { 0x8D3DCB09, 0 }, + { 0x8CF008CF, 0x8CF008CF }, { 0x8CA29C04, 0x8CA29C04 }, + { 0x8C55841D, 0 }, { 0x8C08C08C, 0x8C08C08C }, + { 0x8BBC50C9, 0 }, { 0x8B70344A, 0x8B70344A }, + { 0x8B246A88, 0 }, { 0x8AD8F2FC, 0 }, + { 0x8A8DCD20, 0 }, { 0x8A42F870, 0x8A42F870 }, + { 0x89F8746A, 0 }, { 0x89AE408A, 0 }, + { 0x89645C4F, 0x89645C4F }, { 0x891AC73B, 0 }, + { 0x88D180CD, 0x88D180CD }, { 0x88888889, 0 }, + { 0x883FDDF0, 0x883FDDF0 }, { 0x87F78088, 0 }, + { 0x87AF6FD6, 0 }, { 0x8767AB5F, 0x8767AB5F }, + { 0x872032AC, 0x872032AC }, { 0x86D90545, 0 }, + { 0x869222B2, 0 }, { 0x864B8A7E, 0 }, + { 0x86053C34, 0x86053C34 }, { 0x85BF3761, 0x85BF3761 }, + { 0x85797B91, 0x85797B91 }, { 0x85340853, 0x85340853 }, + { 0x84EEDD36, 0 }, { 0x84A9F9C8, 0x84A9F9C8 }, + { 0x84655D9C, 0 }, { 0x84210842, 0x84210842 }, + { 0x83DCF94E, 0 }, { 0x83993052, 0x83993052 }, + { 0x8355ACE4, 0 }, { 0x83126E98, 0 }, + { 0x82CF7504, 0 }, { 0x828CBFBF, 0 }, + { 0x824A4E61, 0 }, { 0x82082082, 0x82082082 }, + { 0x81C635BC, 0x81C635BC }, { 0x81848DA9, 0 }, + { 0x814327E4, 0 }, { 0x81020408, 0x81020408 }, + { 0x80C121B3, 0 }, { 0x80808081, 0 }, + { 0x80402010, 0x80402010 }, { 0xFFFFFFFF, 0xFFFFFFFF }, + { 0xFF803FE1, 0 }, { 0xFF00FF01, 0 }, + { 0xFE823CA6, 0 }, { 0xFE03F810, 0 }, + { 0xFD863087, 0 }, { 0xFD08E551, 0 }, + { 0xFC8C15B5, 0 }, { 0xFC0FC0FD, 0 }, + { 0xFB93E673, 0 }, { 0xFB188566, 0 }, + { 0xFA9D9D20, 0 }, { 0xFA232CF3, 0 }, + { 0xF9A9342D, 0 }, { 0xF92FB222, 0 }, + { 0xF8B6A622, 0xF8B6A622 }, { 0xF83E0F84, 0 }, + { 0xF7C5ED9D, 0 }, { 0xF74E3FC3, 0 }, + { 0xF6D7054E, 0 }, { 0xF6603D99, 0 }, + { 0xF5E9E7FD, 0 }, { 0xF57403D6, 0 }, + { 0xF4FE9083, 0 }, { 0xF4898D60, 0 }, + { 0xF414F9CE, 0 }, { 0xF3A0D52D, 0 }, + { 0xF32D1EE0, 0 }, { 0xF2B9D649, 0 }, + { 0xF246FACC, 0 }, { 0xF1D48BCF, 0 }, + { 0xF16288B9, 0 }, { 0xF0F0F0F1, 0 }, + { 0xF07FC3E0, 0xF07FC3E0 }, { 0xF00F00F0, 0xF00F00F0 }, + { 0xEF9EA78C, 0 }, { 0xEF2EB720, 0 }, + { 0xEEBF2F19, 0 }, { 0xEE500EE5, 0xEE500EE5 }, + { 0xEDE155F4, 0 }, { 0xED7303B6, 0 }, + { 0xED05179C, 0xED05179C }, { 0xEC979119, 0 }, + { 0xEC2A6FA0, 0xEC2A6FA0 }, { 0xEBBDB2A6, 0 }, + { 0xEB5159A0, 0 }, { 0xEAE56404, 0 }, + { 0xEA79D14A, 0 }, { 0xEA0EA0EA, 0xEA0EA0EA }, + { 0xE9A3D25E, 0xE9A3D25E }, { 0xE9396520, 0 }, + { 0xE8CF58AB, 0 }, { 0xE865AC7C, 0 }, + { 0xE7FC600F, 0 }, { 0xE79372E3, 0 }, + { 0xE72AE476, 0 }, { 0xE6C2B449, 0 }, + { 0xE65AE1DC, 0 }, { 0xE5F36CB0, 0xE5F36CB0 }, + { 0xE58C544A, 0 }, { 0xE525982B, 0 }, + { 0xE4BF37D9, 0 }, { 0xE45932D8, 0 }, + { 0xE3F388AF, 0 }, { 0xE38E38E4, 0 }, + { 0xE32942FF, 0 }, { 0xE2C4A689, 0 }, + { 0xE260630B, 0 }, { 0xE1FC780F, 0 }, + { 0xE198E520, 0 }, { 0xE135A9CA, 0 }, + { 0xE0D2C59A, 0 }, { 0xE070381C, 0xE070381C }, + { 0xE00E00E0, 0xE00E00E0 }, { 0xDFAC1F75, 0 }, + { 0xDF4A9369, 0 }, { 0xDEE95C4D, 0 }, + { 0xDE8879B3, 0 }, { 0xDE27EB2D, 0 }, + { 0xDDC7B04D, 0 }, { 0xDD67C8A6, 0xDD67C8A6 }, + { 0xDD0833CE, 0 }, { 0xDCA8F159, 0 }, + { 0xDC4A00DD, 0 }, { 0xDBEB61EF, 0 }, + { 0xDB8D1428, 0 }, { 0xDB2F171E, 0 }, + { 0xDAD16A6B, 0 }, { 0xDA740DA8, 0 }, + { 0xDA17006D, 0xDA17006D }, { 0xD9BA4257, 0 }, + { 0xD95DD300, 0 }, { 0xD901B204, 0 }, + { 0xD8A5DEFF, 0 }, { 0xD84A598F, 0 }, + { 0xD7EF2152, 0 }, { 0xD79435E5, 0xD79435E5 }, + { 0xD73996E9, 0 }, { 0xD6DF43FD, 0 }, + { 0xD6853CC1, 0 }, { 0xD62B80D7, 0 }, + { 0xD5D20FDF, 0 }, { 0xD578E97D, 0 }, + { 0xD5200D52, 0xD5200D52 }, { 0xD4C77B04, 0 }, + { 0xD46F3235, 0 }, { 0xD417328A, 0 }, + { 0xD3BF7BA9, 0 }, { 0xD3680D37, 0 }, + { 0xD310E6DB, 0 }, { 0xD2BA083C, 0 }, + { 0xD2637101, 0 }, { 0xD20D20D2, 0xD20D20D2 }, + { 0xD1B71759, 0 }, { 0xD161543E, 0xD161543E }, + { 0xD10BD72C, 0 }, { 0xD0B69FCC, 0 }, + { 0xD061ADCA, 0 }, { 0xD00D00D0, 0xD00D00D0 }, + { 0xCFB8988C, 0 }, { 0xCF6474A9, 0 }, + { 0xCF1094D4, 0 }, { 0xCEBCF8BC, 0 }, + { 0xCE69A00D, 0 }, { 0xCE168A77, 0xCE168A77 }, + { 0xCDC3B7A9, 0xCDC3B7A9 }, { 0xCD712753, 0 }, + { 0xCD1ED924, 0 }, { 0xCCCCCCCD, 0 }, + { 0xCC7B0200, 0 }, { 0xCC29786D, 0 }, + { 0xCBD82FC7, 0 }, { 0xCB8727C1, 0 }, + { 0xCB36600D, 0 }, { 0xCAE5D85F, 0xCAE5D85F }, + { 0xCA95906C, 0 }, { 0xCA4587E7, 0 }, + { 0xC9F5BE86, 0 }, { 0xC9A633FD, 0 }, + { 0xC956E803, 0xC956E803 }, { 0xC907DA4F, 0 }, + { 0xC8B90A96, 0 }, { 0xC86A7890, 0xC86A7890 }, + { 0xC81C23F5, 0xC81C23F5 }, { 0xC7CE0C7D, 0 }, + { 0xC78031E0, 0xC78031E0 }, { 0xC73293D8, 0 }, + { 0xC6E5321D, 0 }, { 0xC6980C6A, 0 }, + { 0xC64B2278, 0xC64B2278 }, { 0xC5FE7403, 0xC5FE7403 }, + { 0xC5B200C6, 0 }, { 0xC565C87C, 0 }, + { 0xC519CAE0, 0xC519CAE0 }, { 0xC4CE07B0, 0xC4CE07B0 }, + { 0xC4827EA8, 0xC4827EA8 }, { 0xC4372F86, 0 }, + { 0xC3EC1A06, 0 }, { 0xC3A13DE6, 0xC3A13DE6 }, + { 0xC3569AE6, 0 }, { 0xC30C30C3, 0xC30C30C3 }, + { 0xC2C1FF3E, 0 }, { 0xC2780614, 0 }, + { 0xC22E4507, 0 }, { 0xC1E4BBD6, 0 }, + { 0xC19B6A42, 0 }, { 0xC152500C, 0xC152500C }, + { 0xC1096CF6, 0 }, { 0xC0C0C0C1, 0 }, + { 0xC0784B2F, 0 }, { 0xC0300C03, 0xC0300C03 }, + { 0xBFE80300, 0 }, { 0xBFA02FE8, 0xBFA02FE8 }, + { 0xBF589280, 0 }, { 0xBF112A8B, 0 }, + { 0xBEC9F7CE, 0 }, { 0xBE82FA0C, 0 }, + { 0xBE3C310C, 0 }, { 0xBDF59C92, 0 }, + { 0xBDAF3C64, 0 }, { 0xBD691047, 0xBD691047 }, + { 0xBD231803, 0 }, { 0xBCDD535E, 0 }, + { 0xBC97C21E, 0xBC97C21E }, { 0xBC52640C, 0 }, + { 0xBC0D38EE, 0xBC0D38EE }, { 0xBBC8408D, 0 }, + { 0xBB837AB1, 0 }, { 0xBB3EE722, 0 }, + { 0xBAFA85A9, 0xBAFA85A9 }, { 0xBAB65610, 0xBAB65610 }, + { 0xBA725820, 0xBA725820 }, { 0xBA2E8BA3, 0 }, + { 0xB9EAF063, 0 }, { 0xB9A7862A, 0xB9A7862A }, + { 0xB9644CC4, 0 }, { 0xB92143FA, 0xB92143FA }, + { 0xB8DE6B9A, 0 }, { 0xB89BC36D, 0 }, + { 0xB8594B41, 0 }, { 0xB81702E1, 0 }, + { 0xB7D4EA19, 0xB7D4EA19 }, { 0xB79300B8, 0 }, + { 0xB7514689, 0 }, { 0xB70FBB5A, 0xB70FBB5A }, + { 0xB6CE5EF9, 0xB6CE5EF9 }, { 0xB68D3134, 0xB68D3134 }, + { 0xB64C31D9, 0 }, { 0xB60B60B6, 0xB60B60B6 }, + { 0xB5CABD9B, 0 }, { 0xB58A4855, 0xB58A4855 }, + { 0xB54A00B5, 0xB54A00B5 }, { 0xB509E68B, 0 }, + { 0xB4C9F9A5, 0 }, { 0xB48A39D4, 0xB48A39D4 }, + { 0xB44AA6E9, 0xB44AA6E9 }, { 0xB40B40B4, 0xB40B40B4 }, + { 0xB3CC0706, 0 }, { 0xB38CF9B0, 0xB38CF9B0 }, + { 0xB34E1884, 0 }, { 0xB30F6353, 0 }, + { 0xB2D0D9EF, 0 }, { 0xB2927C2A, 0 }, + { 0xB25449D7, 0 }, { 0xB21642C9, 0 }, + { 0xB1D866D1, 0xB1D866D1 }, { 0xB19AB5C5, 0 }, + { 0xB15D2F76, 0 }, { 0xB11FD3B8, 0xB11FD3B8 }, + { 0xB0E2A260, 0xB0E2A260 }, { 0xB0A59B42, 0 }, + { 0xB068BE31, 0 }, { 0xB02C0B03, 0 }, + { 0xAFEF818C, 0 }, { 0xAFB321A1, 0xAFB321A1 }, + { 0xAF76EB19, 0 }, { 0xAF3ADDC7, 0 }, + { 0xAEFEF982, 0 }, { 0xAEC33E20, 0 }, + { 0xAE87AB76, 0xAE87AB76 }, { 0xAE4C415D, 0 }, + { 0xAE10FFA9, 0 }, { 0xADD5E632, 0xADD5E632 }, + { 0xAD9AF4D0, 0 }, { 0xAD602B58, 0xAD602B58 }, + { 0xAD2589A4, 0 }, { 0xACEB0F89, 0xACEB0F89 }, + { 0xACB0BCE1, 0xACB0BCE1 }, { 0xAC769184, 0xAC769184 }, + { 0xAC3C8D4A, 0 }, { 0xAC02B00B, 0 }, + { 0xABC8F9A0, 0xABC8F9A0 }, { 0xAB8F69E3, 0 }, + { 0xAB5600AC, 0 }, { 0xAB1CBDD4, 0 }, + { 0xAAE3A136, 0 }, { 0xAAAAAAAB, 0 }, + { 0xAA71DA0D, 0 }, { 0xAA392F36, 0 }, + { 0xAA00AA01, 0 }, { 0xA9C84A48, 0 }, + { 0xA9900FE6, 0 }, { 0xA957FAB5, 0xA957FAB5 }, + { 0xA9200A92, 0xA9200A92 }, { 0xA8E83F57, 0xA8E83F57 }, + { 0xA8B098E0, 0xA8B098E0 }, { 0xA8791709, 0 }, + { 0xA841B9AD, 0 }, { 0xA80A80A8, 0xA80A80A8 }, + { 0xA7D36BD8, 0 }, { 0xA79C7B17, 0 }, + { 0xA765AE44, 0 }, { 0xA72F053A, 0 }, + { 0xA6F87FD6, 0xA6F87FD6 }, { 0xA6C21DF7, 0 }, + { 0xA68BDF79, 0 }, { 0xA655C439, 0xA655C439 }, + { 0xA61FCC16, 0xA61FCC16 }, { 0xA5E9F6ED, 0xA5E9F6ED }, + { 0xA5B4449D, 0 }, { 0xA57EB503, 0 }, + { 0xA54947FE, 0 }, { 0xA513FD6C, 0 }, + { 0xA4DED52C, 0xA4DED52C }, { 0xA4A9CF1E, 0 }, + { 0xA474EB1F, 0xA474EB1F }, { 0xA4402910, 0xA4402910 }, + { 0xA40B88D0, 0 }, { 0xA3D70A3E, 0 }, + { 0xA3A2AD39, 0xA3A2AD39 }, { 0xA36E71A3, 0 }, + { 0xA33A575A, 0xA33A575A }, { 0xA3065E40, 0 }, + { 0xA2D28634, 0 }, { 0xA29ECF16, 0xA29ECF16 }, + { 0xA26B38C9, 0 }, { 0xA237C32B, 0xA237C32B }, + { 0xA2046E1F, 0xA2046E1F }, { 0xA1D13986, 0 }, + { 0xA19E2540, 0 }, { 0xA16B312F, 0 }, + { 0xA1385D35, 0 }, { 0xA105A933, 0 }, + { 0xA0D3150C, 0 }, { 0xA0A0A0A1, 0 }, + { 0xA06E4BD4, 0xA06E4BD4 }, { 0xA03C1689, 0 }, + { 0xA00A00A0, 0xA00A00A0 }, { 0x9FD809FE, 0 }, + { 0x9FA63284, 0 }, { 0x9F747A15, 0x9F747A15 }, + { 0x9F42E095, 0x9F42E095 }, { 0x9F1165E7, 0x9F1165E7 }, + { 0x9EE009EE, 0x9EE009EE }, { 0x9EAECC8D, 0x9EAECC8D }, + { 0x9E7DADA9, 0 }, { 0x9E4CAD24, 0 }, + { 0x9E1BCAE3, 0 }, { 0x9DEB06C9, 0x9DEB06C9 }, + { 0x9DBA60BB, 0x9DBA60BB }, { 0x9D89D89E, 0 }, + { 0x9D596E54, 0x9D596E54 }, { 0x9D2921C4, 0 }, + { 0x9CF8F2D1, 0x9CF8F2D1 }, { 0x9CC8E161, 0 }, + { 0x9C98ED58, 0 }, { 0x9C69169B, 0x9C69169B }, + { 0x9C395D10, 0x9C395D10 }, { 0x9C09C09C, 0x9C09C09C }, + { 0x9BDA4124, 0x9BDA4124 }, { 0x9BAADE8E, 0x9BAADE8E }, + { 0x9B7B98C0, 0 }, { 0x9B4C6F9F, 0 }, + { 0x9B1D6311, 0x9B1D6311 }, { 0x9AEE72FD, 0 }, + { 0x9ABF9F48, 0x9ABF9F48 }, { 0x9A90E7D9, 0x9A90E7D9 }, + { 0x9A624C97, 0 }, { 0x9A33CD67, 0x9A33CD67 }, + { 0x9A056A31, 0 }, { 0x99D722DB, 0 }, + { 0x99A8F74C, 0 }, { 0x997AE76B, 0x997AE76B }, + { 0x994CF320, 0x994CF320 }, { 0x991F1A51, 0x991F1A51 }, + { 0x98F15CE7, 0 }, { 0x98C3BAC7, 0x98C3BAC7 }, + { 0x989633DB, 0x989633DB }, { 0x9868C80A, 0 }, + { 0x983B773B, 0 }, { 0x980E4156, 0x980E4156 }, + { 0x97E12644, 0x97E12644 }, { 0x97B425ED, 0x97B425ED }, + { 0x97874039, 0 }, { 0x975A7510, 0 }, + { 0x972DC45B, 0 }, { 0x97012E02, 0x97012E02 }, + { 0x96D4B1EF, 0 }, { 0x96A8500A, 0 }, + { 0x967C083B, 0 }, { 0x964FDA6C, 0x964FDA6C }, + { 0x9623C686, 0x9623C686 }, { 0x95F7CC73, 0 }, + { 0x95CBEC1B, 0 }, { 0x95A02568, 0x95A02568 }, + { 0x95747844, 0 }, { 0x9548E498, 0 }, + { 0x951D6A4E, 0 }, { 0x94F2094F, 0x94F2094F }, + { 0x94C6C187, 0 }, { 0x949B92DE, 0 }, + { 0x94707D3F, 0 }, { 0x94458094, 0x94458094 }, + { 0x941A9CC8, 0x941A9CC8 }, { 0x93EFD1C5, 0x93EFD1C5 }, + { 0x93C51F76, 0 }, { 0x939A85C4, 0x939A85C4 }, + { 0x9370049C, 0 }, { 0x93459BE7, 0 }, + { 0x931B4B91, 0 }, { 0x92F11384, 0x92F11384 }, + { 0x92C6F3AC, 0x92C6F3AC }, { 0x929CEBF5, 0 }, + { 0x9272FC48, 0x9272FC48 }, { 0x92492492, 0x92492492 }, + { 0x921F64BF, 0 }, { 0x91F5BCB9, 0 }, + { 0x91CC2C6C, 0x91CC2C6C }, { 0x91A2B3C5, 0 }, + { 0x917952AF, 0 }, { 0x91500915, 0x91500915 }, + { 0x9126D6E5, 0 }, { 0x90FDBC09, 0x90FDBC09 }, + { 0x90D4B86F, 0 }, { 0x90ABCC02, 0x90ABCC02 }, + { 0x9082F6B0, 0 }, { 0x905A3863, 0x905A3863 }, + { 0x9031910A, 0 }, { 0x90090090, 0x90090090 }, + { 0x8FE086E3, 0 }, { 0x8FB823EE, 0x8FB823EE }, + { 0x8F8FD7A0, 0 }, { 0x8F67A1E4, 0 }, + { 0x8F3F82A8, 0x8F3F82A8 }, { 0x8F1779DA, 0 }, + { 0x8EEF8766, 0 }, { 0x8EC7AB3A, 0 }, + { 0x8E9FE542, 0x8E9FE542 }, { 0x8E78356D, 0x8E78356D }, + { 0x8E509BA8, 0x8E509BA8 }, { 0x8E2917E1, 0 }, + { 0x8E01AA05, 0 }, { 0x8DDA5202, 0x8DDA5202 }, + { 0x8DB30FC6, 0x8DB30FC6 }, { 0x8D8BE340, 0 }, + { 0x8D64CC5C, 0 }, { 0x8D3DCB09, 0 }, + { 0x8D16DF35, 0x8D16DF35 }, { 0x8CF008CF, 0x8CF008CF }, + { 0x8CC947C5, 0 }, { 0x8CA29C04, 0x8CA29C04 }, + { 0x8C7C057D, 0 }, { 0x8C55841D, 0 }, + { 0x8C2F17D2, 0x8C2F17D2 }, { 0x8C08C08C, 0x8C08C08C }, + { 0x8BE27E39, 0x8BE27E39 }, { 0x8BBC50C9, 0 }, + { 0x8B963829, 0x8B963829 }, { 0x8B70344A, 0x8B70344A }, + { 0x8B4A451A, 0 }, { 0x8B246A88, 0 }, + { 0x8AFEA483, 0x8AFEA483 }, { 0x8AD8F2FC, 0 }, + { 0x8AB355E0, 0x8AB355E0 }, { 0x8A8DCD20, 0 }, + { 0x8A6858AB, 0 }, { 0x8A42F870, 0x8A42F870 }, + { 0x8A1DAC60, 0x8A1DAC60 }, { 0x89F8746A, 0 }, + { 0x89D3507D, 0 }, { 0x89AE408A, 0 }, + { 0x89894480, 0 }, { 0x89645C4F, 0x89645C4F }, + { 0x893F87E8, 0x893F87E8 }, { 0x891AC73B, 0 }, + { 0x88F61A37, 0x88F61A37 }, { 0x88D180CD, 0x88D180CD }, + { 0x88ACFAEE, 0 }, { 0x88888889, 0 }, + { 0x8864298F, 0 }, { 0x883FDDF0, 0x883FDDF0 }, + { 0x881BA59E, 0 }, { 0x87F78088, 0 }, + { 0x87D36EA0, 0 }, { 0x87AF6FD6, 0 }, + { 0x878B841B, 0 }, { 0x8767AB5F, 0x8767AB5F }, + { 0x8743E595, 0 }, { 0x872032AC, 0x872032AC }, + { 0x86FC9296, 0x86FC9296 }, { 0x86D90545, 0 }, + { 0x86B58AA8, 0 }, { 0x869222B2, 0 }, + { 0x866ECD53, 0x866ECD53 }, { 0x864B8A7E, 0 }, + { 0x86285A23, 0x86285A23 }, { 0x86053C34, 0x86053C34 }, + { 0x85E230A3, 0x85E230A3 }, { 0x85BF3761, 0x85BF3761 }, + { 0x859C5060, 0x859C5060 }, { 0x85797B91, 0x85797B91 }, + { 0x8556B8E7, 0x8556B8E7 }, { 0x85340853, 0x85340853 }, + { 0x851169C7, 0x851169C7 }, { 0x84EEDD36, 0 }, + { 0x84CC6290, 0 }, { 0x84A9F9C8, 0x84A9F9C8 }, + { 0x8487A2D1, 0 }, { 0x84655D9C, 0 }, + { 0x84432A1B, 0x84432A1B }, { 0x84210842, 0x84210842 }, + { 0x83FEF802, 0x83FEF802 }, { 0x83DCF94E, 0 }, + { 0x83BB0C18, 0 }, { 0x83993052, 0x83993052 }, + { 0x837765F0, 0x837765F0 }, { 0x8355ACE4, 0 }, + { 0x83340520, 0x83340520 }, { 0x83126E98, 0 }, + { 0x82F0E93D, 0x82F0E93D }, { 0x82CF7504, 0 }, + { 0x82AE11DE, 0 }, { 0x828CBFBF, 0 }, + { 0x826B7E99, 0x826B7E99 }, { 0x824A4E61, 0 }, + { 0x82292F08, 0 }, { 0x82082082, 0x82082082 }, + { 0x81E722C2, 0x81E722C2 }, { 0x81C635BC, 0x81C635BC }, + { 0x81A55963, 0 }, { 0x81848DA9, 0 }, + { 0x8163D283, 0 }, { 0x814327E4, 0 }, + { 0x81228DBF, 0 }, { 0x81020408, 0x81020408 }, + { 0x80E18AB3, 0 }, { 0x80C121B3, 0 }, + { 0x80A0C8FB, 0x80A0C8FB }, { 0x80808081, 0 }, + { 0x80604836, 0x80604836 }, { 0x80402010, 0x80402010 }, + { 0x80200802, 0x80200802 }, { 0xFFFFFFFF, 0xFFFFFFFF } +}; diff --git a/third_party/aom/av1/common/odintrin.h b/third_party/aom/av1/common/odintrin.h new file mode 100644 index 0000000000..fe99d80039 --- /dev/null +++ b/third_party/aom/av1/common/odintrin.h @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#ifndef AV1_COMMON_ODINTRIN_H_ +#define AV1_COMMON_ODINTRIN_H_ + +#include +#include +#include + +#include "aom/aom_integer.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/bitops.h" +#include "av1/common/enums.h" + +#ifdef __cplusplus +extern "C" { +#endif + +# if !defined(M_PI) +# define M_PI (3.1415926535897932384626433832795) +# endif + +# if !defined(M_SQRT2) +# define M_SQRT2 (1.41421356237309504880168872420970) +# endif + +# if !defined(M_SQRT1_2) +# define M_SQRT1_2 (0.70710678118654752440084436210485) +# endif + +# if !defined(M_LOG2E) +# define M_LOG2E (1.4426950408889634073599246810019) +# endif + +# if !defined(M_LN2) +# define M_LN2 (0.69314718055994530941723212145818) +# endif + +/*Smallest blocks are 4x4*/ +#define OD_LOG_BSIZE0 (2) +/*There are 5 block sizes total (4x4, 8x8, 16x16, 32x32 and 64x64).*/ +#define OD_NBSIZES (5) + +/*There are 4 transform sizes total in AV1 (4x4, 8x8, 16x16 and 32x32).*/ +#define OD_TXSIZES TX_SIZES +/*The log of the maximum length of the side of a transform.*/ +#define OD_LOG_TXSIZE_MAX (OD_LOG_BSIZE0 + OD_TXSIZES - 1) +/*The maximum length of the side of a transform.*/ +#define OD_TXSIZE_MAX (1 << OD_LOG_TXSIZE_MAX) + +/**The maximum number of color planes allowed in a single frame.*/ +# define OD_NPLANES_MAX (3) + +# define OD_COEFF_SHIFT (4) + +# define OD_DISABLE_CFL (1) +# define OD_DISABLE_FILTER (1) + +#if !defined(NDEBUG) +# define OD_ENABLE_ASSERTIONS (1) +#endif + +# define OD_LOG(a) +# define OD_LOG_PARTIAL(a) + +/*Possible block sizes, note that OD_BLOCK_NXN = log2(N) - 2.*/ +#define OD_BLOCK_4X4 (0) +#define OD_BLOCK_8X8 (1) +#define OD_BLOCK_16X16 (2) +#define OD_BLOCK_32X32 (3) +#define OD_BLOCK_SIZES (OD_BLOCK_32X32 + 1) + +# define OD_LIMIT_BSIZE_MIN (OD_BLOCK_4X4) +# define OD_LIMIT_BSIZE_MAX (OD_BLOCK_32X32) + +typedef int od_coeff; + +#define OD_DIVU_DMAX (1024) + +extern uint32_t OD_DIVU_SMALL_CONSTS[OD_DIVU_DMAX][2]; + +/*Translate unsigned division by small divisors into multiplications.*/ +#define OD_DIVU_SMALL(_x, _d) \ + ((uint32_t)((OD_DIVU_SMALL_CONSTS[(_d)-1][0] * (uint64_t)(_x) + \ + OD_DIVU_SMALL_CONSTS[(_d)-1][1]) >> \ + 32) >> \ + (OD_ILOG_NZ(_d) - 1)) + +#define OD_DIVU(_x, _d) \ + (((_d) < OD_DIVU_DMAX) ? (OD_DIVU_SMALL((_x), (_d))) : ((_x) / (_d))) + +#define OD_MINI AOMMIN +#define OD_MAXI AOMMAX +#define OD_CLAMPI(min, val, max) (OD_MAXI(min, OD_MINI(val, max))) + +#define OD_CLZ0 (1) +#define OD_CLZ(x) (-get_msb(x)) +#define OD_ILOG_NZ(x) (OD_CLZ0 - OD_CLZ(x)) +/*Note that __builtin_clz is not defined when x == 0, according to the gcc + documentation (and that of the x86 BSR instruction that implements it), so + we have to special-case it. + We define a special version of the macro to use when x can be zero.*/ +#define OD_ILOG(x) ((x) ? OD_ILOG_NZ(x) : 0) + +#define OD_LOG2(x) (M_LOG2E*log(x)) +#define OD_EXP2(x) (exp(M_LN2*(x))) + +/*Enable special features for gcc and compatible compilers.*/ +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) +#define OD_GNUC_PREREQ(maj, min, pat) \ + ((__GNUC__ << 16) + (__GNUC_MINOR__ << 8) + __GNUC_PATCHLEVEL__ >= \ + ((maj) << 16) + ((min) << 8) + pat) // NOLINT +#else +#define OD_GNUC_PREREQ(maj, min, pat) (0) +#endif + +#if OD_GNUC_PREREQ(3, 4, 0) +#define OD_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) +#else +#define OD_WARN_UNUSED_RESULT +#endif + +#if OD_GNUC_PREREQ(3, 4, 0) +#define OD_ARG_NONNULL(x) __attribute__((__nonnull__(x))) +#else +#define OD_ARG_NONNULL(x) +#endif + +#if defined(OD_ENABLE_ASSERTIONS) +#if OD_GNUC_PREREQ(2, 5, 0) +__attribute__((noreturn)) +#endif +void od_fatal_impl(const char *_str, const char *_file, int _line); + +#define OD_FATAL(_str) (od_fatal_impl(_str, __FILE__, __LINE__)) + +#define OD_ASSERT(_cond) \ + do { \ + if (!(_cond)) { \ + OD_FATAL("assertion failed: " #_cond); \ + } \ + } while (0) + +#define OD_ASSERT2(_cond, _message) \ + do { \ + if (!(_cond)) { \ + OD_FATAL("assertion failed: " #_cond "\n" _message); \ + } \ + } while (0) + +#define OD_ALWAYS_TRUE(_cond) OD_ASSERT(_cond) + +#else +#define OD_ASSERT(_cond) +#define OD_ASSERT2(_cond, _message) +#define OD_ALWAYS_TRUE(_cond) ((void)(_cond)) +#endif + +/** Copy n elements of memory from src to dst. The 0* term provides + compile-time type checking */ +#if !defined(OVERRIDE_OD_COPY) +#define OD_COPY(dst, src, n) \ + (memcpy((dst), (src), sizeof(*(dst)) * (n) + 0 * ((dst) - (src)))) +#endif + +/** Copy n elements of memory from src to dst, allowing overlapping regions. + The 0* term provides compile-time type checking */ +#if !defined(OVERRIDE_OD_MOVE) +# define OD_MOVE(dst, src, n) \ + (memmove((dst), (src), sizeof(*(dst))*(n) + 0*((dst) - (src)) )) +#endif + +/** Linkage will break without this if using a C++ compiler, and will issue + * warnings without this for a C compiler*/ +#if defined(__cplusplus) +# define OD_EXTERN extern +#else +# define OD_EXTERN +#endif + +/** Set n elements of dst to zero */ +#if !defined(OVERRIDE_OD_CLEAR) +# define OD_CLEAR(dst, n) (memset((dst), 0, sizeof(*(dst))*(n))) +#endif + +/** Silence unused parameter/variable warnings */ +# define OD_UNUSED(expr) (void)(expr) + +#if defined(OD_FLOAT_PVQ) +typedef double od_val16; +typedef double od_val32; +# define OD_QCONST32(x, bits) (x) +# define OD_ROUND16(x) (x) +# define OD_ROUND32(x) (x) +# define OD_SHL(x, shift) (x) +# define OD_SHR(x, shift) (x) +# define OD_SHR_ROUND(x, shift) (x) +# define OD_ABS(x) (fabs(x)) +# define OD_MULT16_16(a, b) ((a)*(b)) +# define OD_MULT16_32_Q16(a, b) ((a)*(b)) +#else +typedef int16_t od_val16; +typedef int32_t od_val32; +/** Compile-time conversion of float constant to 32-bit value */ +# define OD_QCONST32(x, bits) ((od_val32)(.5 + (x)*(((od_val32)1) << (bits)))) +# define OD_ROUND16(x) (int16_t)(floor(.5 + (x))) +# define OD_ROUND32(x) (int32_t)(floor(.5 + (x))) +/*Shift x left by shift*/ +# define OD_SHL(a, shift) ((int32_t)((uint32_t)(a) << (shift))) +/*Shift x right by shift (without rounding)*/ +# define OD_SHR(x, shift) \ + ((int32_t)((x) >> (shift))) +/*Shift x right by shift (with rounding)*/ +# define OD_SHR_ROUND(x, shift) \ + ((int32_t)(((x) + (1 << (shift) >> 1)) >> (shift))) +/*Shift x right by shift (without rounding) or left by -shift if shift + is negative.*/ +# define OD_VSHR(x, shift) \ + (((shift) > 0) ? OD_SHR(x, shift) : OD_SHL(x, -(shift))) +/*Shift x right by shift (with rounding) or left by -shift if shift + is negative.*/ +# define OD_VSHR_ROUND(x, shift) \ + (((shift) > 0) ? OD_SHR_ROUND(x, shift) : OD_SHL(x, -(shift))) +# define OD_ABS(x) (abs(x)) +/* (od_val32)(od_val16) gives TI compiler a hint that it's 16x16->32 multiply */ +/** 16x16 multiplication where the result fits in 32 bits */ +# define OD_MULT16_16(a, b) \ + (((od_val32)(od_val16)(a))*((od_val32)(od_val16)(b))) +/* Multiplies 16-bit a by 32-bit b and keeps bits [16:47]. */ +# define OD_MULT16_32_Q16(a, b) ((int16_t)(a)*(int64_t)(int32_t)(b) >> 16) +/*16x16 multiplication where the result fits in 16 bits, without rounding.*/ +# define OD_MULT16_16_Q15(a, b) \ + (((int16_t)(a)*((int32_t)(int16_t)(b))) >> 15) +/*16x16 multiplication where the result fits in 16 bits, without rounding.*/ +# define OD_MULT16_16_Q16(a, b) \ + ((((int16_t)(a))*((int32_t)(int16_t)(b))) >> 16) +#endif + +/*All of these macros should expect floats as arguments.*/ +/*These two should compile as a single SSE instruction.*/ +# define OD_MINF(a, b) ((a) < (b) ? (a) : (b)) +# define OD_MAXF(a, b) ((a) > (b) ? (a) : (b)) + +# define OD_DIV_R0(x, y) (((x) + OD_FLIPSIGNI((((y) + 1) >> 1) - 1, (x)))/(y)) + +# define OD_SIGNMASK(a) (-((a) < 0)) +# define OD_FLIPSIGNI(a, b) (((a) + OD_SIGNMASK(b)) ^ OD_SIGNMASK(b)) + +# define OD_MULT16_16_Q15(a, b) \ + (((int16_t)(a)*((int32_t)(int16_t)(b))) >> 15) + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_ODINTRIN_H_ diff --git a/third_party/aom/av1/common/onyxc_int.h b/third_party/aom/av1/common/onyxc_int.h new file mode 100644 index 0000000000..7980bde394 --- /dev/null +++ b/third_party/aom/av1/common/onyxc_int.h @@ -0,0 +1,1027 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_ONYXC_INT_H_ +#define AV1_COMMON_ONYXC_INT_H_ + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "aom/internal/aom_codec_internal.h" +#include "aom_util/aom_thread.h" +#if CONFIG_ANS +#include "aom_dsp/ans.h" +#endif +#include "av1/common/alloccommon.h" +#include "av1/common/av1_loopfilter.h" +#include "av1/common/entropy.h" +#include "av1/common/entropymode.h" +#include "av1/common/entropymv.h" +#include "av1/common/frame_buffers.h" +#include "av1/common/mv.h" +#include "av1/common/quant_common.h" +#if CONFIG_LOOP_RESTORATION +#include "av1/common/restoration.h" +#endif // CONFIG_LOOP_RESTORATION +#include "av1/common/tile_common.h" +#include "av1/common/odintrin.h" +#if CONFIG_PVQ +#include "av1/common/pvq.h" +#endif +#if CONFIG_CFL +#include "av1/common/cfl.h" +#endif +#ifdef __cplusplus +extern "C" { +#endif + +#define CDEF_MAX_STRENGTHS 16 + +#define REF_FRAMES_LOG2 3 +#define REF_FRAMES (1 << REF_FRAMES_LOG2) + +// 4 scratch frames for the new frames to support a maximum of 4 cores decoding +// in parallel, 3 for scaled references on the encoder. +// TODO(hkuang): Add ondemand frame buffers instead of hardcoding the number +// of framebuffers. +// TODO(jkoleszar): These 3 extra references could probably come from the +// normal reference pool. +#define FRAME_BUFFERS (REF_FRAMES + 7) + +#if CONFIG_REFERENCE_BUFFER +/* Constant values while waiting for the sequence header */ +#define FRAME_ID_NUMBERS_PRESENT_FLAG 1 +#define FRAME_ID_LENGTH_MINUS7 8 // Allows frame id up to 2^15-1 +#define DELTA_FRAME_ID_LENGTH_MINUS2 12 // Allows frame id deltas up to 2^14-1 +#endif + +#if CONFIG_EXT_REFS +#define FRAME_CONTEXTS_LOG2 3 +#else +#define FRAME_CONTEXTS_LOG2 2 +#endif + +#define FRAME_CONTEXTS (1 << FRAME_CONTEXTS_LOG2) + +#define NUM_PING_PONG_BUFFERS 2 + +typedef enum { + SINGLE_REFERENCE = 0, + COMPOUND_REFERENCE = 1, + REFERENCE_MODE_SELECT = 2, + REFERENCE_MODES = 3, +} REFERENCE_MODE; + +typedef enum { + RESET_FRAME_CONTEXT_NONE = 0, + RESET_FRAME_CONTEXT_CURRENT = 1, + RESET_FRAME_CONTEXT_ALL = 2, +} RESET_FRAME_CONTEXT_MODE; + +typedef enum { + /** + * Update frame context to values resulting from forward probability + * updates signaled in the frame header + */ + REFRESH_FRAME_CONTEXT_FORWARD, + /** + * Update frame context to values resulting from backward probability + * updates based on entropy/counts in the decoded frame + */ + REFRESH_FRAME_CONTEXT_BACKWARD, +} REFRESH_FRAME_CONTEXT_MODE; + +typedef struct { + int_mv mv[2]; +#if CONFIG_REF_MV + int_mv pred_mv[2]; +#endif + MV_REFERENCE_FRAME ref_frame[2]; +} MV_REF; + +typedef struct { + int ref_count; + MV_REF *mvs; + int mi_rows; + int mi_cols; +#if CONFIG_GLOBAL_MOTION + WarpedMotionParams global_motion[TOTAL_REFS_PER_FRAME]; +#endif // CONFIG_GLOBAL_MOTION + aom_codec_frame_buffer_t raw_frame_buffer; + YV12_BUFFER_CONFIG buf; +#if CONFIG_TEMPMV_SIGNALING + uint8_t intra_only; +#endif + // The Following variables will only be used in frame parallel decode. + + // frame_worker_owner indicates which FrameWorker owns this buffer. NULL means + // that no FrameWorker owns, or is decoding, this buffer. + AVxWorker *frame_worker_owner; + + // row and col indicate which position frame has been decoded to in real + // pixel unit. They are reset to -1 when decoding begins and set to INT_MAX + // when the frame is fully decoded. + int row; + int col; +} RefCntBuffer; + +typedef struct BufferPool { +// Protect BufferPool from being accessed by several FrameWorkers at +// the same time during frame parallel decode. +// TODO(hkuang): Try to use atomic variable instead of locking the whole pool. +#if CONFIG_MULTITHREAD + pthread_mutex_t pool_mutex; +#endif + + // Private data associated with the frame buffer callbacks. + void *cb_priv; + + aom_get_frame_buffer_cb_fn_t get_fb_cb; + aom_release_frame_buffer_cb_fn_t release_fb_cb; + + RefCntBuffer frame_bufs[FRAME_BUFFERS]; + + // Frame buffers allocated internally by the codec. + InternalFrameBufferList int_frame_buffers; +} BufferPool; + +typedef struct AV1Common { + struct aom_internal_error_info error; + aom_color_space_t color_space; + int color_range; + int width; + int height; + int render_width; + int render_height; + int last_width; + int last_height; + +#if CONFIG_FRAME_SUPERRES + // The numerator of the superres scale, the denominator is fixed + uint8_t superres_scale_numerator; + int superres_width, superres_height; +#endif // CONFIG_FRAME_SUPERRES + + // TODO(jkoleszar): this implies chroma ss right now, but could vary per + // plane. Revisit as part of the future change to YV12_BUFFER_CONFIG to + // support additional planes. + int subsampling_x; + int subsampling_y; + +#if CONFIG_HIGHBITDEPTH + // Marks if we need to use 16bit frame buffers (1: yes, 0: no). + int use_highbitdepth; +#endif + YV12_BUFFER_CONFIG *frame_to_show; + RefCntBuffer *prev_frame; + + // TODO(hkuang): Combine this with cur_buf in macroblockd. + RefCntBuffer *cur_frame; + + int ref_frame_map[REF_FRAMES]; /* maps fb_idx to reference slot */ + + // Prepare ref_frame_map for the next frame. + // Only used in frame parallel decode. + int next_ref_frame_map[REF_FRAMES]; + + // TODO(jkoleszar): could expand active_ref_idx to 4, with 0 as intra, and + // roll new_fb_idx into it. + + // Each Inter frame can reference INTER_REFS_PER_FRAME buffers + RefBuffer frame_refs[INTER_REFS_PER_FRAME]; + + int new_fb_idx; + + FRAME_TYPE last_frame_type; /* last frame's frame type for motion search.*/ + FRAME_TYPE frame_type; + + int show_frame; + int last_show_frame; + int show_existing_frame; +#if CONFIG_EXT_REFS + // Flag for a frame used as a reference - not written to the bitstream + int is_reference_frame; +#endif // CONFIG_EXT_REFS + + // Flag signaling that the frame is encoded using only INTRA modes. + uint8_t intra_only; + uint8_t last_intra_only; + + int allow_high_precision_mv; + +#if CONFIG_PALETTE + int allow_screen_content_tools; +#endif // CONFIG_PALETTE + + // Flag signaling which frame contexts should be reset to default values. + RESET_FRAME_CONTEXT_MODE reset_frame_context; + + // MBs, mb_rows/cols is in 16-pixel units; mi_rows/cols is in + // MODE_INFO (8-pixel) units. + int MBs; + int mb_rows, mi_rows; + int mb_cols, mi_cols; + int mi_stride; + + /* profile settings */ + TX_MODE tx_mode; + + int base_qindex; + int y_dc_delta_q; + int uv_dc_delta_q; + int uv_ac_delta_q; + int16_t y_dequant[MAX_SEGMENTS][2]; + int16_t uv_dequant[MAX_SEGMENTS][2]; + +#if CONFIG_AOM_QM + // Global quant matrix tables + qm_val_t *giqmatrix[NUM_QM_LEVELS][2][2][TX_SIZES]; + qm_val_t *gqmatrix[NUM_QM_LEVELS][2][2][TX_SIZES]; + + // Local quant matrix tables for each frame + qm_val_t *y_iqmatrix[MAX_SEGMENTS][2][TX_SIZES]; + qm_val_t *uv_iqmatrix[MAX_SEGMENTS][2][TX_SIZES]; + // Encoder + qm_val_t *y_qmatrix[MAX_SEGMENTS][2][TX_SIZES]; + qm_val_t *uv_qmatrix[MAX_SEGMENTS][2][TX_SIZES]; + + int using_qmatrix; + int min_qmlevel; + int max_qmlevel; +#endif +#if CONFIG_NEW_QUANT + dequant_val_type_nuq y_dequant_nuq[MAX_SEGMENTS][QUANT_PROFILES][COEF_BANDS]; + dequant_val_type_nuq uv_dequant_nuq[MAX_SEGMENTS][QUANT_PROFILES][COEF_BANDS]; +#endif + + /* We allocate a MODE_INFO struct for each macroblock, together with + an extra row on top and column on the left to simplify prediction. */ + int mi_alloc_size; + MODE_INFO *mip; /* Base of allocated array */ + MODE_INFO *mi; /* Corresponds to upper left visible macroblock */ + + // TODO(agrange): Move prev_mi into encoder structure. + // prev_mip and prev_mi will only be allocated in encoder. + MODE_INFO *prev_mip; /* MODE_INFO array 'mip' from last decoded frame */ + MODE_INFO *prev_mi; /* 'mi' from last frame (points into prev_mip) */ + + // Separate mi functions between encoder and decoder. + int (*alloc_mi)(struct AV1Common *cm, int mi_size); + void (*free_mi)(struct AV1Common *cm); + void (*setup_mi)(struct AV1Common *cm); + + // Grid of pointers to 8x8 MODE_INFO structs. Any 8x8 not in the visible + // area will be NULL. + MODE_INFO **mi_grid_base; + MODE_INFO **mi_grid_visible; + MODE_INFO **prev_mi_grid_base; + MODE_INFO **prev_mi_grid_visible; + + // Whether to use previous frame's motion vectors for prediction. + int use_prev_frame_mvs; + + // Persistent mb segment id map used in prediction. + int seg_map_idx; + int prev_seg_map_idx; + + uint8_t *seg_map_array[NUM_PING_PONG_BUFFERS]; + uint8_t *last_frame_seg_map; + uint8_t *current_frame_seg_map; + int seg_map_alloc_size; + + InterpFilter interp_filter; + + loop_filter_info_n lf_info; +#if CONFIG_LOOP_RESTORATION + RestorationInfo rst_info[MAX_MB_PLANE]; + RestorationInternal rst_internal; +#endif // CONFIG_LOOP_RESTORATION + + // Flag signaling how frame contexts should be updated at the end of + // a frame decode + REFRESH_FRAME_CONTEXT_MODE refresh_frame_context; + + int ref_frame_sign_bias[TOTAL_REFS_PER_FRAME]; /* Two state 0, 1 */ + + struct loopfilter lf; + struct segmentation seg; + + int frame_parallel_decode; // frame-based threading. + +#if CONFIG_EXT_TX + int reduced_tx_set_used; +#endif // CONFIG_EXT_TX + +// Context probabilities for reference frame prediction +#if CONFIG_EXT_REFS + MV_REFERENCE_FRAME comp_fwd_ref[FWD_REFS]; + MV_REFERENCE_FRAME comp_bwd_ref[BWD_REFS]; +#else + MV_REFERENCE_FRAME comp_fixed_ref; + MV_REFERENCE_FRAME comp_var_ref[COMP_REFS]; +#endif // CONFIG_EXT_REFS + REFERENCE_MODE reference_mode; + + FRAME_CONTEXT *fc; /* this frame entropy */ + FRAME_CONTEXT *frame_contexts; // FRAME_CONTEXTS + unsigned int frame_context_idx; /* Context to use/update */ + FRAME_COUNTS counts; + +#if CONFIG_SUBFRAME_PROB_UPDATE + // The initial probabilities for a frame, before any subframe backward update, + // and after forward update. + av1_coeff_probs_model starting_coef_probs[TX_SIZES][PLANE_TYPES]; + // Number of subframe backward updates already done + uint8_t coef_probs_update_idx; + // Signal if the backward update is subframe or end-of-frame + uint8_t partial_prob_update; + // Frame level flag to turn on/off subframe backward update + uint8_t do_subframe_update; +#endif // CONFIG_SUBFRAME_PROB_UPDATE + + unsigned int current_video_frame; + BITSTREAM_PROFILE profile; + + // AOM_BITS_8 in profile 0 or 1, AOM_BITS_10 or AOM_BITS_12 in profile 2 or 3. + aom_bit_depth_t bit_depth; + aom_bit_depth_t dequant_bit_depth; // bit_depth of current dequantizer + + int error_resilient_mode; + +#if !CONFIG_EXT_TILE + int log2_tile_cols, log2_tile_rows; +#endif // !CONFIG_EXT_TILE + int tile_cols, tile_rows; + int tile_width, tile_height; // In MI units +#if CONFIG_EXT_TILE + unsigned int tile_encoding_mode; +#endif // CONFIG_EXT_TILE + +#if CONFIG_DEPENDENT_HORZTILES + int dependent_horz_tiles; +#if CONFIG_TILE_GROUPS + int tile_group_start_row[MAX_TILE_ROWS][MAX_TILE_COLS]; + int tile_group_start_col[MAX_TILE_ROWS][MAX_TILE_COLS]; +#endif +#endif +#if CONFIG_LOOPFILTERING_ACROSS_TILES + int loop_filter_across_tiles_enabled; +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + + int byte_alignment; + int skip_loop_filter; + + // Private data associated with the frame buffer callbacks. + void *cb_priv; + aom_get_frame_buffer_cb_fn_t get_fb_cb; + aom_release_frame_buffer_cb_fn_t release_fb_cb; + + // Handles memory for the codec. + InternalFrameBufferList int_frame_buffers; + + // External BufferPool passed from outside. + BufferPool *buffer_pool; + + PARTITION_CONTEXT *above_seg_context; + ENTROPY_CONTEXT *above_context[MAX_MB_PLANE]; +#if CONFIG_VAR_TX + TXFM_CONTEXT *above_txfm_context; + TXFM_CONTEXT left_txfm_context[MAX_MIB_SIZE]; +#endif + int above_context_alloc_cols; + + // scratch memory for intraonly/keyframe forward updates from default tables + // - this is intentionally not placed in FRAME_CONTEXT since it's reset upon + // each keyframe and not used afterwards + aom_prob kf_y_prob[INTRA_MODES][INTRA_MODES][INTRA_MODES - 1]; +#if CONFIG_GLOBAL_MOTION + WarpedMotionParams global_motion[TOTAL_REFS_PER_FRAME]; +#endif + + BLOCK_SIZE sb_size; // Size of the superblock used for this frame + int mib_size; // Size of the superblock in units of MI blocks + int mib_size_log2; // Log 2 of above. +#if CONFIG_CDEF + int cdef_dering_damping; + int cdef_clpf_damping; + int nb_cdef_strengths; + int cdef_strengths[CDEF_MAX_STRENGTHS]; + int cdef_uv_strengths[CDEF_MAX_STRENGTHS]; + int cdef_bits; +#endif + +#if CONFIG_DELTA_Q + int delta_q_present_flag; + // Resolution of delta quant + int delta_q_res; +#if CONFIG_EXT_DELTA_Q + int delta_lf_present_flag; + // Resolution of delta lf level + int delta_lf_res; +#endif +#endif +#if CONFIG_TILE_GROUPS + int num_tg; +#endif +#if CONFIG_REFERENCE_BUFFER + int current_frame_id; + int ref_frame_id[REF_FRAMES]; + int valid_for_referencing[REF_FRAMES]; + int refresh_mask; + int invalid_delta_frame_id_minus1; +#endif +#if CONFIG_ANS && ANS_MAX_SYMBOLS + int ans_window_size_log2; +#endif +} AV1_COMMON; + +#if CONFIG_REFERENCE_BUFFER +/* Initial version of sequence header structure */ +typedef struct SequenceHeader { + int frame_id_numbers_present_flag; + int frame_id_length_minus7; + int delta_frame_id_length_minus2; +} SequenceHeader; +#endif + +// TODO(hkuang): Don't need to lock the whole pool after implementing atomic +// frame reference count. +static void lock_buffer_pool(BufferPool *const pool) { +#if CONFIG_MULTITHREAD + pthread_mutex_lock(&pool->pool_mutex); +#else + (void)pool; +#endif +} + +static void unlock_buffer_pool(BufferPool *const pool) { +#if CONFIG_MULTITHREAD + pthread_mutex_unlock(&pool->pool_mutex); +#else + (void)pool; +#endif +} + +static INLINE YV12_BUFFER_CONFIG *get_ref_frame(AV1_COMMON *cm, int index) { + if (index < 0 || index >= REF_FRAMES) return NULL; + if (cm->ref_frame_map[index] < 0) return NULL; + assert(cm->ref_frame_map[index] < FRAME_BUFFERS); + return &cm->buffer_pool->frame_bufs[cm->ref_frame_map[index]].buf; +} + +static INLINE YV12_BUFFER_CONFIG *get_frame_new_buffer( + const AV1_COMMON *const cm) { + return &cm->buffer_pool->frame_bufs[cm->new_fb_idx].buf; +} + +static INLINE int get_free_fb(AV1_COMMON *cm) { + RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; + int i; + + lock_buffer_pool(cm->buffer_pool); + for (i = 0; i < FRAME_BUFFERS; ++i) + if (frame_bufs[i].ref_count == 0) break; + + if (i != FRAME_BUFFERS) { + frame_bufs[i].ref_count = 1; + } else { + // Reset i to be INVALID_IDX to indicate no free buffer found. + i = INVALID_IDX; + } + + unlock_buffer_pool(cm->buffer_pool); + return i; +} + +static INLINE void ref_cnt_fb(RefCntBuffer *bufs, int *idx, int new_idx) { + const int ref_index = *idx; + + if (ref_index >= 0 && bufs[ref_index].ref_count > 0) + bufs[ref_index].ref_count--; + + *idx = new_idx; + + bufs[new_idx].ref_count++; +} + +static INLINE int mi_cols_aligned_to_sb(const AV1_COMMON *cm) { + return ALIGN_POWER_OF_TWO(cm->mi_cols, cm->mib_size_log2); +} + +static INLINE int mi_rows_aligned_to_sb(const AV1_COMMON *cm) { + return ALIGN_POWER_OF_TWO(cm->mi_rows, cm->mib_size_log2); +} + +static INLINE int frame_is_intra_only(const AV1_COMMON *const cm) { + return cm->frame_type == KEY_FRAME || cm->intra_only; +} + +static INLINE void av1_init_macroblockd(AV1_COMMON *cm, MACROBLOCKD *xd, +#if CONFIG_PVQ + tran_low_t *pvq_ref_coeff, +#endif +#if CONFIG_CFL + CFL_CTX *cfl, +#endif + tran_low_t *dqcoeff) { + int i; + for (i = 0; i < MAX_MB_PLANE; ++i) { + xd->plane[i].dqcoeff = dqcoeff; +#if CONFIG_PVQ + xd->plane[i].pvq_ref_coeff = pvq_ref_coeff; +#endif +#if CONFIG_CFL + xd->cfl = cfl; + cfl_init(cfl, cm, xd->plane[AOM_PLANE_U].subsampling_x, + xd->plane[AOM_PLANE_U].subsampling_y); +#endif + xd->above_context[i] = cm->above_context[i]; + if (xd->plane[i].plane_type == PLANE_TYPE_Y) { + memcpy(xd->plane[i].seg_dequant, cm->y_dequant, sizeof(cm->y_dequant)); +#if CONFIG_AOM_QM + memcpy(xd->plane[i].seg_iqmatrix, cm->y_iqmatrix, sizeof(cm->y_iqmatrix)); +#endif + +#if CONFIG_NEW_QUANT + memcpy(xd->plane[i].seg_dequant_nuq, cm->y_dequant_nuq, + sizeof(cm->y_dequant_nuq)); +#endif + } else { + memcpy(xd->plane[i].seg_dequant, cm->uv_dequant, sizeof(cm->uv_dequant)); +#if CONFIG_AOM_QM + memcpy(xd->plane[i].seg_iqmatrix, cm->uv_iqmatrix, + sizeof(cm->uv_iqmatrix)); +#endif +#if CONFIG_NEW_QUANT + memcpy(xd->plane[i].seg_dequant_nuq, cm->uv_dequant_nuq, + sizeof(cm->uv_dequant_nuq)); +#endif + } + xd->fc = cm->fc; + } + xd->above_seg_context = cm->above_seg_context; +#if CONFIG_VAR_TX + xd->above_txfm_context = cm->above_txfm_context; +#endif + xd->mi_stride = cm->mi_stride; + xd->error_info = &cm->error; +} + +static INLINE void set_skip_context(MACROBLOCKD *xd, int mi_row, int mi_col) { + int i; + for (i = 0; i < MAX_MB_PLANE; ++i) { + struct macroblockd_plane *const pd = &xd->plane[i]; +#if CONFIG_CHROMA_SUB8X8 + if (xd->mi[0]->mbmi.sb_type < BLOCK_8X8) { + // Offset the buffer pointer + if (pd->subsampling_y && (mi_row & 0x01)) mi_row -= 1; + if (pd->subsampling_x && (mi_col & 0x01)) mi_col -= 1; + } +#endif + int above_idx = mi_col * 2; + int left_idx = (mi_row * 2) & MAX_MIB_MASK_2; + pd->above_context = &xd->above_context[i][above_idx >> pd->subsampling_x]; + pd->left_context = &xd->left_context[i][left_idx >> pd->subsampling_y]; + } +} + +static INLINE int calc_mi_size(int len) { + // len is in mi units. + return len + MAX_MIB_SIZE; +} + +static INLINE void set_plane_n4(MACROBLOCKD *const xd, int bw, int bh) { + int i; + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].n4_w = (bw << 1) >> xd->plane[i].subsampling_x; + xd->plane[i].n4_h = (bh << 1) >> xd->plane[i].subsampling_y; + + xd->plane[i].width = (bw * MI_SIZE) >> xd->plane[i].subsampling_x; + xd->plane[i].height = (bh * MI_SIZE) >> xd->plane[i].subsampling_y; + +#if !CONFIG_CHROMA_2X2 + xd->plane[i].width = AOMMAX(xd->plane[i].width, 4); + xd->plane[i].height = AOMMAX(xd->plane[i].height, 4); +#endif + } +} + +static INLINE void set_mi_row_col(MACROBLOCKD *xd, const TileInfo *const tile, + int mi_row, int bh, int mi_col, int bw, +#if CONFIG_DEPENDENT_HORZTILES + int dependent_horz_tile_flag, +#endif // CONFIG_DEPENDENT_HORZTILES + int mi_rows, int mi_cols) { + xd->mb_to_top_edge = -((mi_row * MI_SIZE) * 8); + xd->mb_to_bottom_edge = ((mi_rows - bh - mi_row) * MI_SIZE) * 8; + xd->mb_to_left_edge = -((mi_col * MI_SIZE) * 8); + xd->mb_to_right_edge = ((mi_cols - bw - mi_col) * MI_SIZE) * 8; + +#if CONFIG_DEPENDENT_HORZTILES + if (dependent_horz_tile_flag) { +#if CONFIG_TILE_GROUPS + xd->up_available = (mi_row > tile->mi_row_start) || !tile->tg_horz_boundary; +#else + xd->up_available = (mi_row > 0); +#endif // CONFIG_TILE_GROUPS + } else { +#endif // CONFIG_DEPENDENT_HORZTILES + // Are edges available for intra prediction? + xd->up_available = (mi_row > tile->mi_row_start); +#if CONFIG_DEPENDENT_HORZTILES + } +#endif // CONFIG_DEPENDENT_HORZTILES + + xd->left_available = (mi_col > tile->mi_col_start); +#if CONFIG_CHROMA_SUB8X8 + xd->chroma_up_available = xd->up_available; + xd->chroma_left_available = xd->left_available; + if (xd->plane[1].subsampling_x && bw < mi_size_wide[BLOCK_8X8]) + xd->chroma_left_available = (mi_col - 1) > tile->mi_col_start; + if (xd->plane[1].subsampling_y && bh < mi_size_high[BLOCK_8X8]) + xd->chroma_up_available = (mi_row - 1) > tile->mi_row_start; +#endif + if (xd->up_available) { + xd->above_mi = xd->mi[-xd->mi_stride]; + // above_mi may be NULL in encoder's first pass. + xd->above_mbmi = xd->above_mi ? &xd->above_mi->mbmi : NULL; + } else { + xd->above_mi = NULL; + xd->above_mbmi = NULL; + } + + if (xd->left_available) { + xd->left_mi = xd->mi[-1]; + // left_mi may be NULL in encoder's first pass. + xd->left_mbmi = xd->left_mi ? &xd->left_mi->mbmi : NULL; + } else { + xd->left_mi = NULL; + xd->left_mbmi = NULL; + } + + xd->n8_h = bh; + xd->n8_w = bw; +#if CONFIG_REF_MV + xd->is_sec_rect = 0; + if (xd->n8_w < xd->n8_h) + if (mi_col & (xd->n8_h - 1)) xd->is_sec_rect = 1; + + if (xd->n8_w > xd->n8_h) + if (mi_row & (xd->n8_w - 1)) xd->is_sec_rect = 1; +#endif // CONFIG_REF_MV +} + +static INLINE const aom_prob *get_y_mode_probs(const AV1_COMMON *cm, + const MODE_INFO *mi, + const MODE_INFO *above_mi, + const MODE_INFO *left_mi, + int block) { + const PREDICTION_MODE above = av1_above_block_mode(mi, above_mi, block); + const PREDICTION_MODE left = av1_left_block_mode(mi, left_mi, block); + return cm->kf_y_prob[above][left]; +} + +#if CONFIG_EC_MULTISYMBOL +static INLINE aom_cdf_prob *get_y_mode_cdf(FRAME_CONTEXT *tile_ctx, + const MODE_INFO *mi, + const MODE_INFO *above_mi, + const MODE_INFO *left_mi, + int block) { + const PREDICTION_MODE above = av1_above_block_mode(mi, above_mi, block); + const PREDICTION_MODE left = av1_left_block_mode(mi, left_mi, block); + return tile_ctx->kf_y_cdf[above][left]; +} +#endif + +static INLINE void update_partition_context(MACROBLOCKD *xd, int mi_row, + int mi_col, BLOCK_SIZE subsize, + BLOCK_SIZE bsize) { + PARTITION_CONTEXT *const above_ctx = xd->above_seg_context + mi_col; + PARTITION_CONTEXT *const left_ctx = + xd->left_seg_context + (mi_row & MAX_MIB_MASK); + +#if CONFIG_EXT_PARTITION_TYPES + const int bw = mi_size_wide[bsize]; + const int bh = mi_size_high[bsize]; + memset(above_ctx, partition_context_lookup[subsize].above, bw); + memset(left_ctx, partition_context_lookup[subsize].left, bh); +#else + // num_4x4_blocks_wide_lookup[bsize] / 2 + const int bs = mi_size_wide[bsize]; + + // update the partition context at the end notes. set partition bits + // of block sizes larger than the current one to be one, and partition + // bits of smaller block sizes to be zero. + memset(above_ctx, partition_context_lookup[subsize].above, bs); + memset(left_ctx, partition_context_lookup[subsize].left, bs); +#endif // CONFIG_EXT_PARTITION_TYPES +} + +#if CONFIG_CB4X4 +static INLINE int is_chroma_reference(int mi_row, int mi_col, BLOCK_SIZE bsize, + int subsampling_x, int subsampling_y) { +#if CONFIG_CHROMA_2X2 + return 1; +#endif + +#if CONFIG_CHROMA_SUB8X8 + const int bw = mi_size_wide[bsize]; + const int bh = mi_size_high[bsize]; + + int ref_pos = ((mi_row & 0x01) || !(bh & 0x01) || !subsampling_y) && + ((mi_col & 0x01) || !(bw & 0x01) || !subsampling_x); + + return ref_pos; +#else + int ref_pos = !(((mi_row & 0x01) && subsampling_y) || + ((mi_col & 0x01) && subsampling_x)); + + if (bsize >= BLOCK_8X8) ref_pos = 1; + + return ref_pos; +#endif +} + +static INLINE BLOCK_SIZE scale_chroma_bsize(BLOCK_SIZE bsize, int subsampling_x, + int subsampling_y) { + BLOCK_SIZE bs = bsize; + + if (bs < BLOCK_8X8) { + if (subsampling_x == 1 && subsampling_y == 1) + bs = BLOCK_8X8; + else if (subsampling_x == 1) + bs = BLOCK_8X4; + else if (subsampling_y == 1) + bs = BLOCK_4X8; + } + + return bs; +} +#endif + +#if CONFIG_EXT_PARTITION_TYPES +static INLINE void update_ext_partition_context(MACROBLOCKD *xd, int mi_row, + int mi_col, BLOCK_SIZE subsize, + BLOCK_SIZE bsize, + PARTITION_TYPE partition) { + if (bsize >= BLOCK_8X8) { + const int hbs = mi_size_wide[bsize] / 2; + BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT); + switch (partition) { + case PARTITION_SPLIT: + if (bsize != BLOCK_8X8) break; + case PARTITION_NONE: + case PARTITION_HORZ: + case PARTITION_VERT: + update_partition_context(xd, mi_row, mi_col, subsize, bsize); + break; + case PARTITION_HORZ_A: + update_partition_context(xd, mi_row, mi_col, bsize2, subsize); + update_partition_context(xd, mi_row + hbs, mi_col, subsize, subsize); + break; + case PARTITION_HORZ_B: + update_partition_context(xd, mi_row, mi_col, subsize, subsize); + update_partition_context(xd, mi_row + hbs, mi_col, bsize2, subsize); + break; + case PARTITION_VERT_A: + update_partition_context(xd, mi_row, mi_col, bsize2, subsize); + update_partition_context(xd, mi_row, mi_col + hbs, subsize, subsize); + break; + case PARTITION_VERT_B: + update_partition_context(xd, mi_row, mi_col, subsize, subsize); + update_partition_context(xd, mi_row, mi_col + hbs, bsize2, subsize); + break; + default: assert(0 && "Invalid partition type"); + } + } +} +#endif // CONFIG_EXT_PARTITION_TYPES + +static INLINE int partition_plane_context(const MACROBLOCKD *xd, int mi_row, + int mi_col, +#if CONFIG_UNPOISON_PARTITION_CTX + int has_rows, int has_cols, +#endif + BLOCK_SIZE bsize) { +#if CONFIG_UNPOISON_PARTITION_CTX + const PARTITION_CONTEXT *above_ctx = xd->above_seg_context + mi_col; + const PARTITION_CONTEXT *left_ctx = + xd->left_seg_context + (mi_row & MAX_MIB_MASK); + // Minimum partition point is 8x8. Offset the bsl accordingly. + const int bsl = mi_width_log2_lookup[bsize] - mi_width_log2_lookup[BLOCK_8X8]; + int above = (*above_ctx >> bsl) & 1, left = (*left_ctx >> bsl) & 1; + + assert(b_width_log2_lookup[bsize] == b_height_log2_lookup[bsize]); + assert(bsl >= 0); + + if (has_rows && has_cols) + return (left * 2 + above) + bsl * PARTITION_PLOFFSET; + else if (has_rows && !has_cols) + return PARTITION_CONTEXTS_PRIMARY + bsl; + else if (!has_rows && has_cols) + return PARTITION_CONTEXTS_PRIMARY + PARTITION_BLOCK_SIZES + bsl; + else + return PARTITION_CONTEXTS; // Bogus context, forced SPLIT +#else + const PARTITION_CONTEXT *above_ctx = xd->above_seg_context + mi_col; + const PARTITION_CONTEXT *left_ctx = + xd->left_seg_context + (mi_row & MAX_MIB_MASK); + // Minimum partition point is 8x8. Offset the bsl accordingly. + const int bsl = mi_width_log2_lookup[bsize] - mi_width_log2_lookup[BLOCK_8X8]; + int above = (*above_ctx >> bsl) & 1, left = (*left_ctx >> bsl) & 1; + + assert(b_width_log2_lookup[bsize] == b_height_log2_lookup[bsize]); + assert(bsl >= 0); + + return (left * 2 + above) + bsl * PARTITION_PLOFFSET; +#endif +} + +static INLINE int max_block_wide(const MACROBLOCKD *xd, BLOCK_SIZE bsize, + int plane) { + int max_blocks_wide = block_size_wide[bsize]; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + + if (xd->mb_to_right_edge < 0) + max_blocks_wide += xd->mb_to_right_edge >> (3 + pd->subsampling_x); + + // Scale the width in the transform block unit. + return max_blocks_wide >> tx_size_wide_log2[0]; +} + +static INLINE int max_block_high(const MACROBLOCKD *xd, BLOCK_SIZE bsize, + int plane) { + int max_blocks_high = block_size_high[bsize]; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + + if (xd->mb_to_bottom_edge < 0) + max_blocks_high += xd->mb_to_bottom_edge >> (3 + pd->subsampling_y); + + // Scale the width in the transform block unit. + return max_blocks_high >> tx_size_wide_log2[0]; +} + +static INLINE void av1_zero_above_context(AV1_COMMON *const cm, + int mi_col_start, int mi_col_end) { + const int width = mi_col_end - mi_col_start; + const int aligned_width = ALIGN_POWER_OF_TWO(width, cm->mib_size_log2); + + const int offset_y = 2 * mi_col_start; + const int width_y = 2 * aligned_width; + const int offset_uv = offset_y >> cm->subsampling_x; + const int width_uv = width_y >> cm->subsampling_x; + + av1_zero_array(cm->above_context[0] + offset_y, width_y); + av1_zero_array(cm->above_context[1] + offset_uv, width_uv); + av1_zero_array(cm->above_context[2] + offset_uv, width_uv); + + av1_zero_array(cm->above_seg_context + mi_col_start, aligned_width); + +#if CONFIG_VAR_TX + av1_zero_array(cm->above_txfm_context + mi_col_start, aligned_width); +#endif // CONFIG_VAR_TX +} + +static INLINE void av1_zero_left_context(MACROBLOCKD *const xd) { + av1_zero(xd->left_context); + av1_zero(xd->left_seg_context); +#if CONFIG_VAR_TX + av1_zero(xd->left_txfm_context_buffer); +#endif +} + +#if CONFIG_VAR_TX +static INLINE TX_SIZE get_min_tx_size(TX_SIZE tx_size) { + if (tx_size >= TX_SIZES_ALL) assert(0); + return txsize_sqr_map[tx_size]; +} + +static INLINE void set_txfm_ctx(TXFM_CONTEXT *txfm_ctx, uint8_t txs, int len) { + int i; + for (i = 0; i < len; ++i) txfm_ctx[i] = txs; +} + +static INLINE void set_txfm_ctxs(TX_SIZE tx_size, int n8_w, int n8_h, int skip, + const MACROBLOCKD *xd) { + uint8_t bw = tx_size_wide[tx_size]; + uint8_t bh = tx_size_high[tx_size]; + + if (skip) { + bw = n8_w * MI_SIZE; + bh = n8_h * MI_SIZE; + } + + set_txfm_ctx(xd->above_txfm_context, bw, n8_w); + set_txfm_ctx(xd->left_txfm_context, bh, n8_h); +} + +static INLINE void txfm_partition_update(TXFM_CONTEXT *above_ctx, + TXFM_CONTEXT *left_ctx, + TX_SIZE tx_size, TX_SIZE txb_size) { + BLOCK_SIZE bsize = txsize_to_bsize[txb_size]; + int bh = mi_size_high[bsize]; + int bw = mi_size_wide[bsize]; + uint8_t txw = tx_size_wide[tx_size]; + uint8_t txh = tx_size_high[tx_size]; + int i; + for (i = 0; i < bh; ++i) left_ctx[i] = txh; + for (i = 0; i < bw; ++i) above_ctx[i] = txw; +} + +static INLINE int txfm_partition_context(TXFM_CONTEXT *above_ctx, + TXFM_CONTEXT *left_ctx, + BLOCK_SIZE bsize, TX_SIZE tx_size) { + const uint8_t txw = tx_size_wide[tx_size]; + const uint8_t txh = tx_size_high[tx_size]; + const int above = *above_ctx < txw; + const int left = *left_ctx < txh; + TX_SIZE max_tx_size = max_txsize_lookup[bsize]; + int category = TXFM_PARTITION_CONTEXTS - 1; + + // dummy return, not used by others. + if (tx_size <= TX_4X4) return 0; + + switch (AOMMAX(block_size_wide[bsize], block_size_high[bsize])) { +#if CONFIG_EXT_PARTITION + case 128: +#endif + case 64: + case 32: max_tx_size = TX_32X32; break; + case 16: max_tx_size = TX_16X16; break; + case 8: max_tx_size = TX_8X8; break; + default: assert(0); + } + + if (max_tx_size >= TX_8X8) { + category = (tx_size != max_tx_size && max_tx_size > TX_8X8) + + (TX_SIZES - 1 - max_tx_size) * 2; + } + if (category == TXFM_PARTITION_CONTEXTS - 1) return category; + return category * 3 + above + left; +} +#endif + +static INLINE PARTITION_TYPE get_partition(const AV1_COMMON *const cm, + int mi_row, int mi_col, + BLOCK_SIZE bsize) { + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) { + return PARTITION_INVALID; + } else { + const int offset = mi_row * cm->mi_stride + mi_col; + MODE_INFO **mi = cm->mi_grid_visible + offset; + const MB_MODE_INFO *const mbmi = &mi[0]->mbmi; + const int bsl = b_width_log2_lookup[bsize]; + const PARTITION_TYPE partition = partition_lookup[bsl][mbmi->sb_type]; +#if !CONFIG_EXT_PARTITION_TYPES + return partition; +#else + const int hbs = mi_size_wide[bsize] / 2; + + assert(cm->mi_grid_visible[offset] == &cm->mi[offset]); + + if (partition != PARTITION_NONE && bsize > BLOCK_8X8 && + mi_row + hbs < cm->mi_rows && mi_col + hbs < cm->mi_cols) { + const BLOCK_SIZE h = get_subsize(bsize, PARTITION_HORZ_A); + const BLOCK_SIZE v = get_subsize(bsize, PARTITION_VERT_A); + const MB_MODE_INFO *const mbmi_right = &mi[hbs]->mbmi; + const MB_MODE_INFO *const mbmi_below = &mi[hbs * cm->mi_stride]->mbmi; + if (mbmi->sb_type == h) { + return mbmi_below->sb_type == h ? PARTITION_HORZ : PARTITION_HORZ_B; + } else if (mbmi->sb_type == v) { + return mbmi_right->sb_type == v ? PARTITION_VERT : PARTITION_VERT_B; + } else if (mbmi_below->sb_type == h) { + return PARTITION_HORZ_A; + } else if (mbmi_right->sb_type == v) { + return PARTITION_VERT_A; + } else { + return PARTITION_SPLIT; + } + } + + return partition; +#endif // !CONFIG_EXT_PARTITION_TYPES + } +} + +static INLINE void set_sb_size(AV1_COMMON *const cm, BLOCK_SIZE sb_size) { + cm->sb_size = sb_size; + cm->mib_size = mi_size_wide[cm->sb_size]; +#if CONFIG_CB4X4 + cm->mib_size_log2 = b_width_log2_lookup[cm->sb_size]; +#else + cm->mib_size_log2 = mi_width_log2_lookup[cm->sb_size]; +#endif +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_ONYXC_INT_H_ diff --git a/third_party/aom/av1/common/partition.c b/third_party/aom/av1/common/partition.c new file mode 100644 index 0000000000..634a9edd59 --- /dev/null +++ b/third_party/aom/av1/common/partition.c @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "enums.h" +#include "odintrin.h" +#include "partition.h" +#include "zigzag.h" + +OD_EXTERN const index_pair *OD_ZIGZAG4[4] = { + OD_ZIGZAG4_DCT_DCT, + OD_ZIGZAG4_ADST_DCT, + OD_ZIGZAG4_DCT_ADST, + OD_ZIGZAG4_ADST_ADST +}; + +OD_EXTERN const index_pair *OD_ZIGZAG8[4] = { + OD_ZIGZAG8_DCT_DCT, + OD_ZIGZAG8_ADST_DCT, + OD_ZIGZAG8_DCT_ADST, + OD_ZIGZAG8_ADST_ADST +}; + +OD_EXTERN const index_pair *OD_ZIGZAG16[4] = { + OD_ZIGZAG16_DCT_DCT, + OD_ZIGZAG16_ADST_DCT, + OD_ZIGZAG16_DCT_ADST, + OD_ZIGZAG16_ADST_ADST +}; + +OD_EXTERN const index_pair *OD_ZIGZAG32[4] = { + OD_ZIGZAG32_DCT_DCT, + OD_ZIGZAG32_DCT_DCT, + OD_ZIGZAG32_DCT_DCT, + OD_ZIGZAG32_DCT_DCT +}; + +/* The tables below specify how coefficient blocks are translated to + and from PVQ partition coding scan order for 4x4, 8x8 and 16x16 */ + +static const int OD_LAYOUT32_OFFSETS[4] = { 0, 128, 256, 768 }; +const band_layout OD_LAYOUT32 = { + OD_ZIGZAG32, + 32, + 3, + OD_LAYOUT32_OFFSETS +}; + +static const int OD_LAYOUT16_OFFSETS[4] = { 0, 32, 64, 192 }; +const band_layout OD_LAYOUT16 = { + OD_ZIGZAG16, + 16, + 3, + OD_LAYOUT16_OFFSETS +}; + +const int OD_LAYOUT8_OFFSETS[4] = { 0, 8, 16, 48 }; +const band_layout OD_LAYOUT8 = { + OD_ZIGZAG8, + 8, + 3, + OD_LAYOUT8_OFFSETS +}; + +static const int OD_LAYOUT4_OFFSETS[2] = { 0, 15 }; +const band_layout OD_LAYOUT4 = { + OD_ZIGZAG4, + 4, + 1, + OD_LAYOUT4_OFFSETS +}; + +/* First element is the number of bands, followed by the list all the band + boundaries. */ +static const int OD_BAND_OFFSETS4[] = {1, 1, 16}; +static const int OD_BAND_OFFSETS8[] = {4, 1, 16, 24, 32, 64}; +static const int OD_BAND_OFFSETS16[] = {7, 1, 16, 24, 32, 64, 96, 128, 256}; +static const int OD_BAND_OFFSETS32[] = {10, 1, 16, 24, 32, 64, 96, 128, 256, + 384, 512, 1024}; +static const int OD_BAND_OFFSETS64[] = {13, 1, 16, 24, 32, 64, 96, 128, 256, + 384, 512, 1024, 1536, 2048, 4096}; + +const int *const OD_BAND_OFFSETS[OD_TXSIZES + 1] = { + OD_BAND_OFFSETS4, + OD_BAND_OFFSETS8, + OD_BAND_OFFSETS16, + OD_BAND_OFFSETS32, + OD_BAND_OFFSETS64 +}; + +/** Perform a single stage of conversion from a coefficient block in + * raster order into coding scan order + * + * @param [in] layout scan order specification + * @param [out] dst destination vector + * @param [in] src source coefficient block + * @param [int] int source vector row stride + */ +static void od_band_from_raster(const band_layout *layout, tran_low_t *dst, + const tran_low_t *src, int stride, TX_TYPE tx_type) { + int i; + int len; + len = layout->band_offsets[layout->nb_bands]; + for (i = 0; i < len; i++) { + dst[i] = src[layout->dst_table[tx_type][i][1]*stride + layout->dst_table[tx_type][i][0]]; + } +} + +/** Perform a single stage of conversion from a vector in coding scan + order back into a coefficient block in raster order + * + * @param [in] layout scan order specification + * @param [out] dst destination coefficient block + * @param [in] src source vector + * @param [int] stride destination vector row stride + */ +static void od_raster_from_band(const band_layout *layout, tran_low_t *dst, + int stride, TX_TYPE tx_type, const tran_low_t *src) { + int i; + int len; + len = layout->band_offsets[layout->nb_bands]; + for (i = 0; i < len; i++) { + dst[layout->dst_table[tx_type][i][1]*stride + layout->dst_table[tx_type][i][0]] = src[i]; + } +} + +static const band_layout *const OD_LAYOUTS[] = {&OD_LAYOUT4, &OD_LAYOUT8, + &OD_LAYOUT16, &OD_LAYOUT32}; + +/** Converts a coefficient block in raster order into a vector in + * coding scan order with the PVQ partitions laid out one after + * another. This works in stages; the 4x4 conversion is applied to + * the coefficients nearest DC, then the 8x8 applied to the 8x8 block + * nearest DC that was not already coded by 4x4, then 16x16 following + * the same pattern. + * + * @param [out] dst destination vector + * @param [in] n block size (along one side) + * @param [in] ty_type transfrom type + * @param [in] src source coefficient block + * @param [in] stride source vector row stride + */ +void od_raster_to_coding_order(tran_low_t *dst, int n, TX_TYPE ty_type, + const tran_low_t *src, int stride) { + int bs; + /* dst + 1 because DC is not included for 4x4 blocks. */ + od_band_from_raster(OD_LAYOUTS[0], dst + 1, src, stride, ty_type); + for (bs = 1; bs < OD_TXSIZES; bs++) { + int size; + int offset; + /* Length of block size > 4. */ + size = 1 << (OD_LOG_BSIZE0 + bs); + /* Offset is the size of the previous block squared. */ + offset = 1 << 2*(OD_LOG_BSIZE0 - 1 + bs); + if (n >= size) { + /* 3 16x16 bands come after 3 8x8 bands, which come after 2 4x4 bands. */ + od_band_from_raster(OD_LAYOUTS[bs], dst + offset, src, stride, ty_type); + } + } + dst[0] = src[0]; +} + +/** Converts a vector in coding scan order witht he PVQ partitions + * laid out one after another into a coefficient block in raster + * order. This works in stages in the reverse order of raster->scan + * order; the 16x16 conversion is applied to the coefficients that + * don't appear in an 8x8 block, then the 8x8 applied to the 8x8 block + * sans the 4x4 block it contains, then 4x4 is converted sans DC. + * + * @param [out] dst destination coefficient block + * @param [in] stride destination vector row stride + * @param [in] src source vector + * @param [in] n block size (along one side) + */ +void od_coding_order_to_raster(tran_low_t *dst, int stride, TX_TYPE ty_type, + const tran_low_t *src, int n) { + int bs; + /* src + 1 because DC is not included for 4x4 blocks. */ + od_raster_from_band(OD_LAYOUTS[0], dst, stride, ty_type, src + 1); + for (bs = 1; bs < OD_TXSIZES; bs++) { + int size; + int offset; + /* Length of block size > 4 */ + size = 1 << (OD_LOG_BSIZE0 + bs); + /* Offset is the size of the previous block squared. */ + offset = 1 << 2*(OD_LOG_BSIZE0 - 1 + bs); + if (n >= size) { + /* 3 16x16 bands come after 3 8x8 bands, which come after 2 4x4 bands. */ + od_raster_from_band(OD_LAYOUTS[bs], dst, stride, ty_type, src + offset); + } + } + dst[0] = src[0]; +} + +/** Perform a single stage of conversion from a coefficient block in + * raster order into coding scan order + * + * @param [in] layout scan order specification + * @param [out] dst destination vector + * @param [in] src source coefficient block + * @param [int] int source vector row stride + */ +static void od_band_from_raster_16(const band_layout *layout, int16_t *dst, + const int16_t *src, int stride) { + int i; + int len; + len = layout->band_offsets[layout->nb_bands]; + for (i = 0; i < len; i++) { + dst[i] = src[layout->dst_table[DCT_DCT][i][1]*stride + layout->dst_table[DCT_DCT][i][0]]; + } +} + +/** Converts a coefficient block in raster order into a vector in + * coding scan order with the PVQ partitions laid out one after + * another. This works in stages; the 4x4 conversion is applied to + * the coefficients nearest DC, then the 8x8 applied to the 8x8 block + * nearest DC that was not already coded by 4x4, then 16x16 following + * the same pattern. + * + * @param [out] dst destination vector + * @param [in] n block size (along one side) + * @param [in] src source coefficient block + * @param [in] stride source vector row stride + */ +void od_raster_to_coding_order_16(int16_t *dst, int n, const int16_t *src, + int stride) { + int bs; + /* dst + 1 because DC is not included for 4x4 blocks. */ + od_band_from_raster_16(OD_LAYOUTS[0], dst + 1, src, stride); + for (bs = 1; bs < OD_TXSIZES; bs++) { + int size; + int offset; + /* Length of block size > 4. */ + size = 1 << (OD_LOG_BSIZE0 + bs); + /* Offset is the size of the previous block squared. */ + offset = 1 << 2*(OD_LOG_BSIZE0 - 1 + bs); + if (n >= size) { + /* 3 16x16 bands come after 3 8x8 bands, which come after 2 4x4 bands. */ + od_band_from_raster_16(OD_LAYOUTS[bs], dst + offset, src, stride); + } + } + dst[0] = src[0]; +} diff --git a/third_party/aom/av1/common/partition.h b/third_party/aom/av1/common/partition.h new file mode 100644 index 0000000000..bd308f94f9 --- /dev/null +++ b/third_party/aom/av1/common/partition.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#if !defined(_partition_H) +# define _partition_H + +#include "av1/common/enums.h" +#include "odintrin.h" + +typedef unsigned char index_pair[2]; + +typedef struct { + const index_pair **const dst_table; + int size; + int nb_bands; + const int *const band_offsets; +} band_layout; + +extern const int *const OD_BAND_OFFSETS[OD_TXSIZES + 1]; + +void od_raster_to_coding_order(tran_low_t *dst, int n, TX_TYPE ty_type, + const tran_low_t *src, int stride); + +void od_coding_order_to_raster(tran_low_t *dst, int stride, TX_TYPE ty_type, + const tran_low_t *src, int n); + +void od_raster_to_coding_order_16(int16_t *dst, int n, const int16_t *src, + int stride); + +#endif diff --git a/third_party/aom/av1/common/pred_common.c b/third_party/aom/av1/common/pred_common.c new file mode 100644 index 0000000000..905dd3afef --- /dev/null +++ b/third_party/aom/av1/common/pred_common.c @@ -0,0 +1,1408 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/common/common.h" +#include "av1/common/pred_common.h" +#include "av1/common/reconinter.h" +#if CONFIG_EXT_INTRA +#include "av1/common/reconintra.h" +#endif // CONFIG_EXT_INTRA +#include "av1/common/seg_common.h" + +// Returns a context number for the given MB prediction signal +#if CONFIG_DUAL_FILTER +static InterpFilter get_ref_filter_type(const MODE_INFO *mi, + const MACROBLOCKD *xd, int dir, + MV_REFERENCE_FRAME ref_frame) { + InterpFilter ref_type = SWITCHABLE_FILTERS; + const MB_MODE_INFO *ref_mbmi = &mi->mbmi; + int use_subpel[2] = { + has_subpel_mv_component(mi, xd, dir), + has_subpel_mv_component(mi, xd, dir + 2), + }; + + if (ref_mbmi->ref_frame[0] == ref_frame && use_subpel[0]) + ref_type = ref_mbmi->interp_filter[(dir & 0x01)]; + else if (ref_mbmi->ref_frame[1] == ref_frame && use_subpel[1]) + ref_type = ref_mbmi->interp_filter[(dir & 0x01) + 2]; + + return ref_type; +} + +int av1_get_pred_context_switchable_interp(const MACROBLOCKD *xd, int dir) { + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const int ctx_offset = + (mbmi->ref_frame[1] > INTRA_FRAME) * INTER_FILTER_COMP_OFFSET; + MV_REFERENCE_FRAME ref_frame = + (dir < 2) ? mbmi->ref_frame[0] : mbmi->ref_frame[1]; + // Note: + // The mode info data structure has a one element border above and to the + // left of the entries corresponding to real macroblocks. + // The prediction flags in these dummy entries are initialized to 0. + int filter_type_ctx = ctx_offset + (dir & 0x01) * INTER_FILTER_DIR_OFFSET; + int left_type = SWITCHABLE_FILTERS; + int above_type = SWITCHABLE_FILTERS; + + if (xd->left_available) + left_type = get_ref_filter_type(xd->mi[-1], xd, dir, ref_frame); + + if (xd->up_available) + above_type = + get_ref_filter_type(xd->mi[-xd->mi_stride], xd, dir, ref_frame); + + if (left_type == above_type) { + filter_type_ctx += left_type; + } else if (left_type == SWITCHABLE_FILTERS) { + assert(above_type != SWITCHABLE_FILTERS); + filter_type_ctx += above_type; + } else if (above_type == SWITCHABLE_FILTERS) { + assert(left_type != SWITCHABLE_FILTERS); + filter_type_ctx += left_type; + } else { + filter_type_ctx += SWITCHABLE_FILTERS; + } + + return filter_type_ctx; +} +#else +int av1_get_pred_context_switchable_interp(const MACROBLOCKD *xd) { + // Note: + // The mode info data structure has a one element border above and to the + // left of the entries corresponding to real macroblocks. + // The prediction flags in these dummy entries are initialized to 0. + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int left_type = xd->left_available && is_inter_block(left_mbmi) + ? left_mbmi->interp_filter + : SWITCHABLE_FILTERS; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const int above_type = xd->up_available && is_inter_block(above_mbmi) + ? above_mbmi->interp_filter + : SWITCHABLE_FILTERS; + + if (left_type == above_type) { + return left_type; + } else if (left_type == SWITCHABLE_FILTERS) { + assert(above_type != SWITCHABLE_FILTERS); + return above_type; + } else if (above_type == SWITCHABLE_FILTERS) { + assert(left_type != SWITCHABLE_FILTERS); + return left_type; + } else { + return SWITCHABLE_FILTERS; + } +} +#endif + +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP +// Obtain the reference filter type from the above/left neighbor blocks. +static INTRA_FILTER get_ref_intra_filter(const MB_MODE_INFO *ref_mbmi) { + INTRA_FILTER ref_type = INTRA_FILTERS; + + if (ref_mbmi->sb_type >= BLOCK_8X8) { + const PREDICTION_MODE mode = ref_mbmi->mode; + if (is_inter_block(ref_mbmi)) { +#if CONFIG_DUAL_FILTER + switch (ref_mbmi->interp_filter[0]) { +#else + switch (ref_mbmi->interp_filter) { +#endif + case EIGHTTAP_REGULAR: ref_type = INTRA_FILTER_8TAP; break; + case EIGHTTAP_SMOOTH: ref_type = INTRA_FILTER_8TAP_SMOOTH; break; + case MULTITAP_SHARP: ref_type = INTRA_FILTER_8TAP_SHARP; break; + case BILINEAR: ref_type = INTRA_FILTERS; break; + default: break; + } + } else { + if (av1_is_directional_mode(mode, ref_mbmi->sb_type)) { + const int p_angle = + mode_to_angle_map[mode] + ref_mbmi->angle_delta[0] * ANGLE_STEP; + if (av1_is_intra_filter_switchable(p_angle)) { + ref_type = ref_mbmi->intra_filter; + } + } + } + } + return ref_type; +} + +int av1_get_pred_context_intra_interp(const MACROBLOCKD *xd) { + int left_type = INTRA_FILTERS, above_type = INTRA_FILTERS; + + if (xd->left_available) left_type = get_ref_intra_filter(xd->left_mbmi); + + if (xd->up_available) above_type = get_ref_intra_filter(xd->above_mbmi); + + if (left_type == above_type) + return left_type; + else if (left_type == INTRA_FILTERS && above_type != INTRA_FILTERS) + return above_type; + else if (left_type != INTRA_FILTERS && above_type == INTRA_FILTERS) + return left_type; + else + return INTRA_FILTERS; +} +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA + +// The mode info data structure has a one element border above and to the +// left of the entries corresponding to real macroblocks. +// The prediction flags in these dummy entries are initialized to 0. +// 0 - inter/inter, inter/--, --/inter, --/-- +// 1 - intra/inter, inter/intra +// 2 - intra/--, --/intra +// 3 - intra/intra +int av1_get_intra_inter_context(const MACROBLOCKD *xd) { + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int has_above = xd->up_available; + const int has_left = xd->left_available; + + if (has_above && has_left) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + return left_intra && above_intra ? 3 : left_intra || above_intra; + } else if (has_above || has_left) { // one edge available + return 2 * !is_inter_block(has_above ? above_mbmi : left_mbmi); + } else { + return 0; + } +} + +#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF +// The compound/single mode info data structure has one element border above and +// to the left of the entries corresponding to real macroblocks. +// The prediction flags in these dummy entries are initialized to 0. +// 0 - single/single +// 1 - single/--, --/single, --/-- +// 2 - single/comp, comp/single +// 3 - comp/comp, comp/--, --/comp +int av1_get_inter_mode_context(const MACROBLOCKD *xd) { + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int has_above = xd->up_available; + const int has_left = xd->left_available; + + if (has_above && has_left) { // both edges available (0/2/3) + const int above_inter_comp_mode = is_inter_compound_mode(above_mbmi->mode); + const int left_inter_comp_mode = is_inter_compound_mode(left_mbmi->mode); + return (above_inter_comp_mode && left_inter_comp_mode) + ? 3 + : (above_inter_comp_mode || left_inter_comp_mode) * 2; + } else if (has_above || has_left) { // one edge available (1/3) + const MB_MODE_INFO *const edge_mbmi = has_above ? above_mbmi : left_mbmi; + return is_inter_compound_mode(edge_mbmi->mode) ? 3 : 1; + } else { // no edge available (1) + return 1; + } +} +#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF + +#if CONFIG_EXT_REFS +#define CHECK_BACKWARD_REFS(ref_frame) \ + (((ref_frame) >= BWDREF_FRAME) && ((ref_frame) <= ALTREF_FRAME)) +#define IS_BACKWARD_REF_FRAME(ref_frame) CHECK_BACKWARD_REFS(ref_frame) +#else +#define IS_BACKWARD_REF_FRAME(ref_frame) ((ref_frame) == cm->comp_fixed_ref) +#endif // CONFIG_EXT_REFS + +int av1_get_reference_mode_context(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + int ctx; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int has_above = xd->up_available; + const int has_left = xd->left_available; + +#if CONFIG_EXT_REFS + (void)cm; +#endif // CONFIG_EXT_REFS + + // Note: + // The mode info data structure has a one element border above and to the + // left of the entries corresponding to real macroblocks. + // The prediction flags in these dummy entries are initialized to 0. + if (has_above && has_left) { // both edges available + if (!has_second_ref(above_mbmi) && !has_second_ref(left_mbmi)) + // neither edge uses comp pred (0/1) + ctx = IS_BACKWARD_REF_FRAME(above_mbmi->ref_frame[0]) ^ + IS_BACKWARD_REF_FRAME(left_mbmi->ref_frame[0]); + else if (!has_second_ref(above_mbmi)) + // one of two edges uses comp pred (2/3) + ctx = 2 + (IS_BACKWARD_REF_FRAME(above_mbmi->ref_frame[0]) || + !is_inter_block(above_mbmi)); + else if (!has_second_ref(left_mbmi)) + // one of two edges uses comp pred (2/3) + ctx = 2 + (IS_BACKWARD_REF_FRAME(left_mbmi->ref_frame[0]) || + !is_inter_block(left_mbmi)); + else // both edges use comp pred (4) + ctx = 4; + } else if (has_above || has_left) { // one edge available + const MB_MODE_INFO *edge_mbmi = has_above ? above_mbmi : left_mbmi; + + if (!has_second_ref(edge_mbmi)) + // edge does not use comp pred (0/1) + ctx = IS_BACKWARD_REF_FRAME(edge_mbmi->ref_frame[0]); + else + // edge uses comp pred (3) + ctx = 3; + } else { // no edges available (1) + ctx = 1; + } + assert(ctx >= 0 && ctx < COMP_INTER_CONTEXTS); + return ctx; +} + +#if CONFIG_EXT_REFS + +// TODO(zoeliu): Future work will be conducted to optimize the context design +// for the coding of the reference frames. + +#define CHECK_LAST_OR_LAST2(ref_frame) \ + ((ref_frame == LAST_FRAME) || (ref_frame == LAST2_FRAME)) + +#define CHECK_GOLDEN_OR_LAST3(ref_frame) \ + ((ref_frame == GOLDEN_FRAME) || (ref_frame == LAST3_FRAME)) + +// Returns a context number for the given MB prediction signal +// Signal the first reference frame for a compound mode be either +// GOLDEN/LAST3, or LAST/LAST2. +// +// NOTE(zoeliu): The probability of ref_frame[0] is either +// GOLDEN_FRAME or LAST3_FRAME. +#if CONFIG_LOWDELAY_COMPOUND +int av1_get_pred_context_comp_ref_p(UNUSED const AV1_COMMON *cm, + const MACROBLOCKD *xd) { +#else +int av1_get_pred_context_comp_ref_p(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { +#endif + int pred_context; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int above_in_image = xd->up_available; + const int left_in_image = xd->left_available; + +// Note: +// The mode info data structure has a one element border above and to the +// left of the entries correpsonding to real macroblocks. +// The prediction flags in these dummy entries are initialised to 0. +#if CONFIG_LOWDELAY_COMPOUND // No change to bitstream + // Code seems to assume that signbias of cm->comp_bwd_ref[0] is always 1 + const int bwd_ref_sign_idx = 1; +#else + const int bwd_ref_sign_idx = cm->ref_frame_sign_bias[cm->comp_bwd_ref[0]]; +#endif + const int fwd_ref_sign_idx = !bwd_ref_sign_idx; + + if (above_in_image && left_in_image) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + + if (above_intra && left_intra) { // intra/intra (2) + pred_context = 2; + } else if (above_intra || left_intra) { // intra/inter + const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi; + + if (!has_second_ref(edge_mbmi)) // single pred (1/3) + pred_context = + 1 + 2 * (!CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[0])); + else // comp pred (1/3) + pred_context = 1 + + 2 * (!CHECK_GOLDEN_OR_LAST3( + edge_mbmi->ref_frame[fwd_ref_sign_idx])); + } else { // inter/inter + const int l_sg = !has_second_ref(left_mbmi); + const int a_sg = !has_second_ref(above_mbmi); + const MV_REFERENCE_FRAME frfa = + a_sg ? above_mbmi->ref_frame[0] + : above_mbmi->ref_frame[fwd_ref_sign_idx]; + const MV_REFERENCE_FRAME frfl = + l_sg ? left_mbmi->ref_frame[0] + : left_mbmi->ref_frame[fwd_ref_sign_idx]; + + if (frfa == frfl && CHECK_GOLDEN_OR_LAST3(frfa)) { + pred_context = 0; + } else if (l_sg && a_sg) { // single/single + if ((CHECK_BACKWARD_REFS(frfa) && CHECK_LAST_OR_LAST2(frfl)) || + (CHECK_BACKWARD_REFS(frfl) && CHECK_LAST_OR_LAST2(frfa))) { + pred_context = 4; + } else if (CHECK_GOLDEN_OR_LAST3(frfa) || CHECK_GOLDEN_OR_LAST3(frfl)) { + pred_context = 1; + } else { + pred_context = 3; + } + } else if (l_sg || a_sg) { // single/comp + const MV_REFERENCE_FRAME frfc = l_sg ? frfa : frfl; + const MV_REFERENCE_FRAME rfs = a_sg ? frfa : frfl; + + if (CHECK_GOLDEN_OR_LAST3(frfc) && !CHECK_GOLDEN_OR_LAST3(rfs)) + pred_context = 1; + else if (CHECK_GOLDEN_OR_LAST3(rfs) && !CHECK_GOLDEN_OR_LAST3(frfc)) + pred_context = 2; + else + pred_context = 4; + } else { // comp/comp + if ((CHECK_LAST_OR_LAST2(frfa) && CHECK_LAST_OR_LAST2(frfl))) { + pred_context = 4; + } else { + // NOTE(zoeliu): Following assert may be removed once confirmed. + assert(CHECK_GOLDEN_OR_LAST3(frfa) || CHECK_GOLDEN_OR_LAST3(frfl)); + pred_context = 2; + } + } + } + } else if (above_in_image || left_in_image) { // one edge available + const MB_MODE_INFO *edge_mbmi = above_in_image ? above_mbmi : left_mbmi; + + if (!is_inter_block(edge_mbmi)) { + pred_context = 2; + } else { + if (has_second_ref(edge_mbmi)) + pred_context = + 4 * + (!CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[fwd_ref_sign_idx])); + else + pred_context = 3 * (!CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[0])); + } + } else { // no edges available (2) + pred_context = 2; + } + + assert(pred_context >= 0 && pred_context < REF_CONTEXTS); + + return pred_context; +} + +// Returns a context number for the given MB prediction signal +// Signal the first reference frame for a compound mode be LAST, +// conditioning on that it is known either LAST/LAST2. +// +// NOTE(zoeliu): The probability of ref_frame[0] is LAST_FRAME, +// conditioning on it is either LAST_FRAME or LAST2_FRAME. +#if CONFIG_LOWDELAY_COMPOUND +int av1_get_pred_context_comp_ref_p1(UNUSED const AV1_COMMON *cm, + const MACROBLOCKD *xd) { +#else +int av1_get_pred_context_comp_ref_p1(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { +#endif + int pred_context; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int above_in_image = xd->up_available; + const int left_in_image = xd->left_available; + +// Note: +// The mode info data structure has a one element border above and to the +// left of the entries correpsonding to real macroblocks. +// The prediction flags in these dummy entries are initialised to 0. +#if CONFIG_LOWDELAY_COMPOUND // No change to bitstream + // Code seems to assume that signbias of cm->comp_bwd_ref[0] is always 1 + const int bwd_ref_sign_idx = 1; +#else + const int bwd_ref_sign_idx = cm->ref_frame_sign_bias[cm->comp_bwd_ref[0]]; +#endif + const int fwd_ref_sign_idx = !bwd_ref_sign_idx; + + if (above_in_image && left_in_image) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + + if (above_intra && left_intra) { // intra/intra (2) + pred_context = 2; + } else if (above_intra || left_intra) { // intra/inter + const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi; + + if (!has_second_ref(edge_mbmi)) // single pred (1/3) + pred_context = 1 + 2 * (edge_mbmi->ref_frame[0] != LAST_FRAME); + else // comp pred (1/3) + pred_context = + 1 + 2 * (edge_mbmi->ref_frame[fwd_ref_sign_idx] != LAST_FRAME); + } else { // inter/inter + const int l_sg = !has_second_ref(left_mbmi); + const int a_sg = !has_second_ref(above_mbmi); + const MV_REFERENCE_FRAME frfa = + a_sg ? above_mbmi->ref_frame[0] + : above_mbmi->ref_frame[fwd_ref_sign_idx]; + const MV_REFERENCE_FRAME frfl = + l_sg ? left_mbmi->ref_frame[0] + : left_mbmi->ref_frame[fwd_ref_sign_idx]; + + if (frfa == frfl && frfa == LAST_FRAME) + pred_context = 0; + else if (l_sg && a_sg) { // single/single + if (frfa == LAST_FRAME || frfl == LAST_FRAME) + pred_context = 1; + else if (CHECK_GOLDEN_OR_LAST3(frfa) || CHECK_GOLDEN_OR_LAST3(frfl)) + pred_context = 2 + (frfa != frfl); + else if (frfa == frfl || + (CHECK_BACKWARD_REFS(frfa) && CHECK_BACKWARD_REFS(frfl))) + pred_context = 3; + else + pred_context = 4; + } else if (l_sg || a_sg) { // single/comp + const MV_REFERENCE_FRAME frfc = l_sg ? frfa : frfl; + const MV_REFERENCE_FRAME rfs = a_sg ? frfa : frfl; + + if (frfc == LAST_FRAME && rfs != LAST_FRAME) + pred_context = 1; + else if (rfs == LAST_FRAME && frfc != LAST_FRAME) + pred_context = 2; + else + pred_context = + 3 + (frfc == LAST2_FRAME || CHECK_GOLDEN_OR_LAST3(rfs)); + } else { // comp/comp + if (frfa == LAST_FRAME || frfl == LAST_FRAME) + pred_context = 2; + else + pred_context = + 3 + (CHECK_GOLDEN_OR_LAST3(frfa) || CHECK_GOLDEN_OR_LAST3(frfl)); + } + } + } else if (above_in_image || left_in_image) { // one edge available + const MB_MODE_INFO *edge_mbmi = above_in_image ? above_mbmi : left_mbmi; + + if (!is_inter_block(edge_mbmi)) { + pred_context = 2; + } else { + if (has_second_ref(edge_mbmi)) { + pred_context = + 4 * (edge_mbmi->ref_frame[fwd_ref_sign_idx] != LAST_FRAME); + } else { + if (edge_mbmi->ref_frame[0] == LAST_FRAME) + pred_context = 0; + else + pred_context = 2 + CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[0]); + } + } + } else { // no edges available (2) + pred_context = 2; + } + + assert(pred_context >= 0 && pred_context < REF_CONTEXTS); + + return pred_context; +} + +// Returns a context number for the given MB prediction signal +// Signal the first reference frame for a compound mode be GOLDEN, +// conditioning on that it is known either GOLDEN or LAST3. +// +// NOTE(zoeliu): The probability of ref_frame[0] is GOLDEN_FRAME, +// conditioning on it is either GOLDEN or LAST3. +#if CONFIG_LOWDELAY_COMPOUND +int av1_get_pred_context_comp_ref_p2(UNUSED const AV1_COMMON *cm, + const MACROBLOCKD *xd) { +#else +int av1_get_pred_context_comp_ref_p2(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { +#endif + int pred_context; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int above_in_image = xd->up_available; + const int left_in_image = xd->left_available; + +// Note: +// The mode info data structure has a one element border above and to the +// left of the entries correpsonding to real macroblocks. +// The prediction flags in these dummy entries are initialised to 0. +#if CONFIG_LOWDELAY_COMPOUND // No change to bitstream + // Code seems to assume that signbias of cm->comp_bwd_ref[0] is always 1 + const int bwd_ref_sign_idx = 1; +#else + const int bwd_ref_sign_idx = cm->ref_frame_sign_bias[cm->comp_bwd_ref[0]]; +#endif + const int fwd_ref_sign_idx = !bwd_ref_sign_idx; + + if (above_in_image && left_in_image) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + + if (above_intra && left_intra) { // intra/intra (2) + pred_context = 2; + } else if (above_intra || left_intra) { // intra/inter + const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi; + + if (!has_second_ref(edge_mbmi)) // single pred (1/3) + pred_context = 1 + 2 * (edge_mbmi->ref_frame[0] != GOLDEN_FRAME); + else // comp pred (1/3) + pred_context = + 1 + 2 * (edge_mbmi->ref_frame[fwd_ref_sign_idx] != GOLDEN_FRAME); + } else { // inter/inter + const int l_sg = !has_second_ref(left_mbmi); + const int a_sg = !has_second_ref(above_mbmi); + const MV_REFERENCE_FRAME frfa = + a_sg ? above_mbmi->ref_frame[0] + : above_mbmi->ref_frame[fwd_ref_sign_idx]; + const MV_REFERENCE_FRAME frfl = + l_sg ? left_mbmi->ref_frame[0] + : left_mbmi->ref_frame[fwd_ref_sign_idx]; + + if (frfa == frfl && frfa == GOLDEN_FRAME) + pred_context = 0; + else if (l_sg && a_sg) { // single/single + if (frfa == GOLDEN_FRAME || frfl == GOLDEN_FRAME) + pred_context = 1; + else if (CHECK_LAST_OR_LAST2(frfa) || CHECK_LAST_OR_LAST2(frfl)) + pred_context = 2 + (frfa != frfl); + else if (frfa == frfl || + (CHECK_BACKWARD_REFS(frfa) && CHECK_BACKWARD_REFS(frfl))) + pred_context = 3; + else + pred_context = 4; + } else if (l_sg || a_sg) { // single/comp + const MV_REFERENCE_FRAME frfc = l_sg ? frfa : frfl; + const MV_REFERENCE_FRAME rfs = a_sg ? frfa : frfl; + + if (frfc == GOLDEN_FRAME && rfs != GOLDEN_FRAME) + pred_context = 1; + else if (rfs == GOLDEN_FRAME && frfc != GOLDEN_FRAME) + pred_context = 2; + else + pred_context = 3 + (frfc == LAST3_FRAME || CHECK_LAST_OR_LAST2(rfs)); + } else { // comp/comp + if (frfa == GOLDEN_FRAME || frfl == GOLDEN_FRAME) + pred_context = 2; + else + pred_context = + 3 + (CHECK_LAST_OR_LAST2(frfa) || CHECK_LAST_OR_LAST2(frfl)); + } + } + } else if (above_in_image || left_in_image) { // one edge available + const MB_MODE_INFO *edge_mbmi = above_in_image ? above_mbmi : left_mbmi; + + if (!is_inter_block(edge_mbmi)) { + pred_context = 2; + } else { + if (has_second_ref(edge_mbmi)) { + pred_context = + 4 * (edge_mbmi->ref_frame[fwd_ref_sign_idx] != GOLDEN_FRAME); + } else { + if (edge_mbmi->ref_frame[0] == GOLDEN_FRAME) + pred_context = 0; + else + pred_context = 2 + CHECK_LAST_OR_LAST2(edge_mbmi->ref_frame[0]); + } + } + } else { // no edges available (2) + pred_context = 2; + } + + assert(pred_context >= 0 && pred_context < REF_CONTEXTS); + + return pred_context; +} + +// Returns a context number for the given MB prediction signal +#if CONFIG_LOWDELAY_COMPOUND +int av1_get_pred_context_comp_bwdref_p(UNUSED const AV1_COMMON *cm, + const MACROBLOCKD *xd) { +#else +int av1_get_pred_context_comp_bwdref_p(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { +#endif + int pred_context; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int above_in_image = xd->up_available; + const int left_in_image = xd->left_available; + +// Note: +// The mode info data structure has a one element border above and to the +// left of the entries corresponding to real macroblocks. +// The prediction flags in these dummy entries are initialized to 0. +#if CONFIG_LOWDELAY_COMPOUND // No change to bitstream + // Code seems to assume that signbias of cm->comp_bwd_ref[0] is always 1 + const int bwd_ref_sign_idx = 1; +#else + const int bwd_ref_sign_idx = cm->ref_frame_sign_bias[cm->comp_bwd_ref[0]]; +#endif + const int fwd_ref_sign_idx = !bwd_ref_sign_idx; + + if (above_in_image && left_in_image) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + + if (above_intra && left_intra) { // intra/intra (2) + pred_context = 2; + } else if (above_intra || left_intra) { // intra/inter + const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi; + + if (!has_second_ref(edge_mbmi)) // single pred (1/3) + pred_context = 1 + 2 * (edge_mbmi->ref_frame[0] != cm->comp_bwd_ref[1]); + else // comp pred (1/3) + pred_context = + 1 + + 2 * (edge_mbmi->ref_frame[bwd_ref_sign_idx] != cm->comp_bwd_ref[1]); + } else { // inter/inter + const int l_comp = has_second_ref(left_mbmi); + const int a_comp = has_second_ref(above_mbmi); + + const MV_REFERENCE_FRAME l_brf = + l_comp ? left_mbmi->ref_frame[bwd_ref_sign_idx] : NONE_FRAME; + const MV_REFERENCE_FRAME a_brf = + a_comp ? above_mbmi->ref_frame[bwd_ref_sign_idx] : NONE_FRAME; + + const MV_REFERENCE_FRAME l_frf = + !l_comp ? left_mbmi->ref_frame[0] + : left_mbmi->ref_frame[fwd_ref_sign_idx]; + const MV_REFERENCE_FRAME a_frf = + !a_comp ? above_mbmi->ref_frame[0] + : above_mbmi->ref_frame[fwd_ref_sign_idx]; + + if (l_comp && a_comp) { // comp/comp + if (l_brf == a_brf && l_brf == cm->comp_bwd_ref[1]) { + pred_context = 0; + } else if (l_brf == cm->comp_bwd_ref[1] || + a_brf == cm->comp_bwd_ref[1]) { + pred_context = 1; + } else { + // NOTE: Backward ref should be either BWDREF or ALTREF. + assert(l_brf == a_brf && l_brf != cm->comp_bwd_ref[1]); + pred_context = 3; + } + } else if (!l_comp && !a_comp) { // single/single + if (l_frf == a_frf && l_frf == cm->comp_bwd_ref[1]) { + pred_context = 0; + } else if (l_frf == cm->comp_bwd_ref[1] || + a_frf == cm->comp_bwd_ref[1]) { + pred_context = 1; + } else if (l_frf == a_frf) { + pred_context = 3; + } else { + assert(l_frf != a_frf && l_frf != cm->comp_bwd_ref[1] && + a_frf != cm->comp_bwd_ref[1]); + pred_context = 4; + } + } else { // comp/single + assert((l_comp && !a_comp) || (!l_comp && a_comp)); + + if ((l_comp && l_brf == cm->comp_bwd_ref[1] && + a_frf == cm->comp_bwd_ref[1]) || + (a_comp && a_brf == cm->comp_bwd_ref[1] && + l_frf == cm->comp_bwd_ref[1])) { + pred_context = 1; + } else if ((l_comp && l_brf == cm->comp_bwd_ref[1]) || + (a_comp && a_brf == cm->comp_bwd_ref[1]) || + (!l_comp && l_frf == cm->comp_bwd_ref[1]) || + (!a_comp && a_frf == cm->comp_bwd_ref[1])) { + pred_context = 2; + } else { + pred_context = 4; + } + } + } + } else if (above_in_image || left_in_image) { // one edge available + const MB_MODE_INFO *edge_mbmi = above_in_image ? above_mbmi : left_mbmi; + + if (!is_inter_block(edge_mbmi)) { + pred_context = 2; + } else { + if (has_second_ref(edge_mbmi)) { + pred_context = + 4 * (edge_mbmi->ref_frame[bwd_ref_sign_idx] != cm->comp_bwd_ref[1]); + } else { + pred_context = 3 * (edge_mbmi->ref_frame[0] != cm->comp_bwd_ref[1]); + } + } + } else { // no edges available (2) + pred_context = 2; + } + assert(pred_context >= 0 && pred_context < REF_CONTEXTS); + + return pred_context; +} + +#else // CONFIG_EXT_REFS + +// Returns a context number for the given MB prediction signal +int av1_get_pred_context_comp_ref_p(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + int pred_context; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int above_in_image = xd->up_available; + const int left_in_image = xd->left_available; + + // Note: + // The mode info data structure has a one element border above and to the + // left of the entries corresponding to real macroblocks. + // The prediction flags in these dummy entries are initialized to 0. + const int fix_ref_idx = cm->ref_frame_sign_bias[cm->comp_fixed_ref]; + const int var_ref_idx = !fix_ref_idx; + + if (above_in_image && left_in_image) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + + if (above_intra && left_intra) { // intra/intra (2) + pred_context = 2; + } else if (above_intra || left_intra) { // intra/inter + const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi; + + if (!has_second_ref(edge_mbmi)) // single pred (1/3) + pred_context = 1 + 2 * (edge_mbmi->ref_frame[0] != cm->comp_var_ref[1]); + else // comp pred (1/3) + pred_context = + 1 + 2 * (edge_mbmi->ref_frame[var_ref_idx] != cm->comp_var_ref[1]); + } else { // inter/inter + const int l_sg = !has_second_ref(left_mbmi); + const int a_sg = !has_second_ref(above_mbmi); + const MV_REFERENCE_FRAME vrfa = + a_sg ? above_mbmi->ref_frame[0] : above_mbmi->ref_frame[var_ref_idx]; + const MV_REFERENCE_FRAME vrfl = + l_sg ? left_mbmi->ref_frame[0] : left_mbmi->ref_frame[var_ref_idx]; + + if (vrfa == vrfl && cm->comp_var_ref[1] == vrfa) { + pred_context = 0; + } else if (l_sg && a_sg) { // single/single + if ((vrfa == cm->comp_fixed_ref && vrfl == cm->comp_var_ref[0]) || + (vrfl == cm->comp_fixed_ref && vrfa == cm->comp_var_ref[0])) + pred_context = 4; + else if (vrfa == vrfl) + pred_context = 3; + else + pred_context = 1; + } else if (l_sg || a_sg) { // single/comp + const MV_REFERENCE_FRAME vrfc = l_sg ? vrfa : vrfl; + const MV_REFERENCE_FRAME rfs = a_sg ? vrfa : vrfl; + if (vrfc == cm->comp_var_ref[1] && rfs != cm->comp_var_ref[1]) + pred_context = 1; + else if (rfs == cm->comp_var_ref[1] && vrfc != cm->comp_var_ref[1]) + pred_context = 2; + else + pred_context = 4; + } else if (vrfa == vrfl) { // comp/comp + pred_context = 4; + } else { + pred_context = 2; + } + } + } else if (above_in_image || left_in_image) { // one edge available + const MB_MODE_INFO *edge_mbmi = above_in_image ? above_mbmi : left_mbmi; + + if (!is_inter_block(edge_mbmi)) { + pred_context = 2; + } else { + if (has_second_ref(edge_mbmi)) + pred_context = + 4 * (edge_mbmi->ref_frame[var_ref_idx] != cm->comp_var_ref[1]); + else + pred_context = 3 * (edge_mbmi->ref_frame[0] != cm->comp_var_ref[1]); + } + } else { // no edges available (2) + pred_context = 2; + } + assert(pred_context >= 0 && pred_context < REF_CONTEXTS); + + return pred_context; +} + +#endif // CONFIG_EXT_REFS + +#if CONFIG_EXT_REFS + +// For the bit to signal whether the single reference is a ALTREF_FRAME +// or a BWDREF_FRAME. +// +// NOTE(zoeliu): The probability of ref_frame[0] is ALTREF/BWDREF. +int av1_get_pred_context_single_ref_p1(const MACROBLOCKD *xd) { + int pred_context; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int has_above = xd->up_available; + const int has_left = xd->left_available; + + // Note: + // The mode info data structure has a one element border above and to the + // left of the entries correpsonding to real macroblocks. + // The prediction flags in these dummy entries are initialised to 0. + if (has_above && has_left) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + + if (above_intra && left_intra) { // intra/intra + pred_context = 2; + } else if (above_intra || left_intra) { // intra/inter or inter/intra + const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi; + + if (!has_second_ref(edge_mbmi)) // single + pred_context = 4 * (!CHECK_BACKWARD_REFS(edge_mbmi->ref_frame[0])); + else // comp + pred_context = 2; + } else { // inter/inter + const int above_has_second = has_second_ref(above_mbmi); + const int left_has_second = has_second_ref(left_mbmi); + + const MV_REFERENCE_FRAME above0 = above_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME left0 = left_mbmi->ref_frame[0]; + + if (above_has_second && left_has_second) { // comp/comp + pred_context = 2; + } else if (above_has_second || left_has_second) { // single/comp + const MV_REFERENCE_FRAME rfs = !above_has_second ? above0 : left0; + + pred_context = (!CHECK_BACKWARD_REFS(rfs)) ? 4 : 1; + } else { // single/single + pred_context = 2 * (!CHECK_BACKWARD_REFS(above0)) + + 2 * (!CHECK_BACKWARD_REFS(left0)); + } + } + } else if (has_above || has_left) { // one edge available + const MB_MODE_INFO *edge_mbmi = has_above ? above_mbmi : left_mbmi; + if (!is_inter_block(edge_mbmi)) { // intra + pred_context = 2; + } else { // inter + if (!has_second_ref(edge_mbmi)) // single + pred_context = 4 * (!CHECK_BACKWARD_REFS(edge_mbmi->ref_frame[0])); + else // comp + pred_context = 2; + } + } else { // no edges available + pred_context = 2; + } + + assert(pred_context >= 0 && pred_context < REF_CONTEXTS); + return pred_context; +} + +// For the bit to signal whether the single reference is ALTREF_FRAME or +// BWDREF_FRAME, knowing that it shall be either of these 2 choices. +// +// NOTE(zoeliu): The probability of ref_frame[0] is ALTREF_FRAME, conditioning +// on it is either ALTREF_FRAME/BWDREF_FRAME. +int av1_get_pred_context_single_ref_p2(const MACROBLOCKD *xd) { + int pred_context; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int has_above = xd->up_available; + const int has_left = xd->left_available; + + // Note: + // The mode info data structure has a one element border above and to the + // left of the entries correpsonding to real macroblocks. + // The prediction flags in these dummy entries are initialised to 0. + if (has_above && has_left) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + + if (above_intra && left_intra) { // intra/intra + pred_context = 2; + } else if (above_intra || left_intra) { // intra/inter or inter/intra + const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi; + if (!has_second_ref(edge_mbmi)) { // single + if (!CHECK_BACKWARD_REFS(edge_mbmi->ref_frame[0])) + pred_context = 3; + else + pred_context = 4 * (edge_mbmi->ref_frame[0] == BWDREF_FRAME); + } else { // comp + pred_context = 1 + + 2 * (edge_mbmi->ref_frame[0] == BWDREF_FRAME || + edge_mbmi->ref_frame[1] == BWDREF_FRAME); + } + } else { // inter/inter + const int above_has_second = has_second_ref(above_mbmi); + const int left_has_second = has_second_ref(left_mbmi); + const MV_REFERENCE_FRAME above0 = above_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME above1 = above_mbmi->ref_frame[1]; + const MV_REFERENCE_FRAME left0 = left_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME left1 = left_mbmi->ref_frame[1]; + + if (above_has_second && left_has_second) { // comp/comp + if (above0 == left0 && above1 == left1) + pred_context = + 3 * (above0 == BWDREF_FRAME || above1 == BWDREF_FRAME || + left0 == BWDREF_FRAME || left1 == BWDREF_FRAME); + else + pred_context = 2; + } else if (above_has_second || left_has_second) { // single/comp + const MV_REFERENCE_FRAME rfs = !above_has_second ? above0 : left0; + const MV_REFERENCE_FRAME crf1 = above_has_second ? above0 : left0; + const MV_REFERENCE_FRAME crf2 = above_has_second ? above1 : left1; + + if (rfs == BWDREF_FRAME) + pred_context = 3 + (crf1 == BWDREF_FRAME || crf2 == BWDREF_FRAME); + else if (rfs == ALTREF_FRAME) + pred_context = (crf1 == BWDREF_FRAME || crf2 == BWDREF_FRAME); + else + pred_context = 1 + 2 * (crf1 == BWDREF_FRAME || crf2 == BWDREF_FRAME); + } else { // single/single + if (!CHECK_BACKWARD_REFS(above0) && !CHECK_BACKWARD_REFS(left0)) { + pred_context = 2 + (above0 == left0); + } else if (!CHECK_BACKWARD_REFS(above0) || + !CHECK_BACKWARD_REFS(left0)) { + const MV_REFERENCE_FRAME edge0 = + !CHECK_BACKWARD_REFS(above0) ? left0 : above0; + pred_context = 4 * (edge0 == BWDREF_FRAME); + } else { + pred_context = + 2 * (above0 == BWDREF_FRAME) + 2 * (left0 == BWDREF_FRAME); + } + } + } + } else if (has_above || has_left) { // one edge available + const MB_MODE_INFO *edge_mbmi = has_above ? above_mbmi : left_mbmi; + + if (!is_inter_block(edge_mbmi) || + (!CHECK_BACKWARD_REFS(edge_mbmi->ref_frame[0]) && + !has_second_ref(edge_mbmi))) + pred_context = 2; + else if (!has_second_ref(edge_mbmi)) // single + pred_context = 4 * (edge_mbmi->ref_frame[0] == BWDREF_FRAME); + else // comp + pred_context = 3 * (edge_mbmi->ref_frame[0] == BWDREF_FRAME || + edge_mbmi->ref_frame[1] == BWDREF_FRAME); + } else { // no edges available (2) + pred_context = 2; + } + + assert(pred_context >= 0 && pred_context < REF_CONTEXTS); + return pred_context; +} + +// For the bit to signal whether the single reference is LAST3/GOLDEN or +// LAST2/LAST, knowing that it shall be either of these 2 choices. +// +// NOTE(zoeliu): The probability of ref_frame[0] is LAST3/GOLDEN, conditioning +// on it is either LAST3/GOLDEN/LAST2/LAST. +int av1_get_pred_context_single_ref_p3(const MACROBLOCKD *xd) { + int pred_context; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int has_above = xd->up_available; + const int has_left = xd->left_available; + + // Note: + // The mode info data structure has a one element border above and to the + // left of the entries correpsonding to real macroblocks. + // The prediction flags in these dummy entries are initialised to 0. + if (has_above && has_left) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + + if (above_intra && left_intra) { // intra/intra + pred_context = 2; + } else if (above_intra || left_intra) { // intra/inter or inter/intra + const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi; + if (!has_second_ref(edge_mbmi)) { // single + if (CHECK_BACKWARD_REFS(edge_mbmi->ref_frame[0])) + pred_context = 3; + else + pred_context = 4 * CHECK_LAST_OR_LAST2(edge_mbmi->ref_frame[0]); + } else { // comp + pred_context = 1 + + 2 * (CHECK_LAST_OR_LAST2(edge_mbmi->ref_frame[0]) || + CHECK_LAST_OR_LAST2(edge_mbmi->ref_frame[1])); + } + } else { // inter/inter + const int above_has_second = has_second_ref(above_mbmi); + const int left_has_second = has_second_ref(left_mbmi); + const MV_REFERENCE_FRAME above0 = above_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME above1 = above_mbmi->ref_frame[1]; + const MV_REFERENCE_FRAME left0 = left_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME left1 = left_mbmi->ref_frame[1]; + + if (above_has_second && left_has_second) { // comp/comp + if (above0 == left0 && above1 == left1) + pred_context = + 3 * (CHECK_LAST_OR_LAST2(above0) || CHECK_LAST_OR_LAST2(above1) || + CHECK_LAST_OR_LAST2(left0) || CHECK_LAST_OR_LAST2(left1)); + else + pred_context = 2; + } else if (above_has_second || left_has_second) { // single/comp + const MV_REFERENCE_FRAME rfs = !above_has_second ? above0 : left0; + const MV_REFERENCE_FRAME crf1 = above_has_second ? above0 : left0; + const MV_REFERENCE_FRAME crf2 = above_has_second ? above1 : left1; + + if (CHECK_LAST_OR_LAST2(rfs)) + pred_context = + 3 + (CHECK_LAST_OR_LAST2(crf1) || CHECK_LAST_OR_LAST2(crf2)); + else if (CHECK_GOLDEN_OR_LAST3(rfs)) + pred_context = + (CHECK_LAST_OR_LAST2(crf1) || CHECK_LAST_OR_LAST2(crf2)); + else + pred_context = + 1 + 2 * (CHECK_LAST_OR_LAST2(crf1) || CHECK_LAST_OR_LAST2(crf2)); + } else { // single/single + if (CHECK_BACKWARD_REFS(above0) && CHECK_BACKWARD_REFS(left0)) { + pred_context = 2 + (above0 == left0); + } else if (CHECK_BACKWARD_REFS(above0) || CHECK_BACKWARD_REFS(left0)) { + const MV_REFERENCE_FRAME edge0 = + CHECK_BACKWARD_REFS(above0) ? left0 : above0; + pred_context = 4 * CHECK_LAST_OR_LAST2(edge0); + } else { + pred_context = + 2 * CHECK_LAST_OR_LAST2(above0) + 2 * CHECK_LAST_OR_LAST2(left0); + } + } + } + } else if (has_above || has_left) { // one edge available + const MB_MODE_INFO *edge_mbmi = has_above ? above_mbmi : left_mbmi; + + if (!is_inter_block(edge_mbmi) || + (CHECK_BACKWARD_REFS(edge_mbmi->ref_frame[0]) && + !has_second_ref(edge_mbmi))) + pred_context = 2; + else if (!has_second_ref(edge_mbmi)) // single + pred_context = 4 * (CHECK_LAST_OR_LAST2(edge_mbmi->ref_frame[0])); + else // comp + pred_context = 3 * (CHECK_LAST_OR_LAST2(edge_mbmi->ref_frame[0]) || + CHECK_LAST_OR_LAST2(edge_mbmi->ref_frame[1])); + } else { // no edges available (2) + pred_context = 2; + } + + assert(pred_context >= 0 && pred_context < REF_CONTEXTS); + return pred_context; +} + +// For the bit to signal whether the single reference is LAST2_FRAME or +// LAST_FRAME, knowing that it shall be either of these 2 choices. +// +// NOTE(zoeliu): The probability of ref_frame[0] is LAST2_FRAME, conditioning +// on it is either LAST2_FRAME/LAST_FRAME. +int av1_get_pred_context_single_ref_p4(const MACROBLOCKD *xd) { + int pred_context; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int has_above = xd->up_available; + const int has_left = xd->left_available; + + // Note: + // The mode info data structure has a one element border above and to the + // left of the entries correpsonding to real macroblocks. + // The prediction flags in these dummy entries are initialised to 0. + if (has_above && has_left) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + + if (above_intra && left_intra) { // intra/intra + pred_context = 2; + } else if (above_intra || left_intra) { // intra/inter or inter/intra + const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi; + if (!has_second_ref(edge_mbmi)) { // single + if (!CHECK_LAST_OR_LAST2(edge_mbmi->ref_frame[0])) + pred_context = 3; + else + pred_context = 4 * (edge_mbmi->ref_frame[0] == LAST_FRAME); + } else { // comp + pred_context = 1 + + 2 * (edge_mbmi->ref_frame[0] == LAST_FRAME || + edge_mbmi->ref_frame[1] == LAST_FRAME); + } + } else { // inter/inter + const int above_has_second = has_second_ref(above_mbmi); + const int left_has_second = has_second_ref(left_mbmi); + const MV_REFERENCE_FRAME above0 = above_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME above1 = above_mbmi->ref_frame[1]; + const MV_REFERENCE_FRAME left0 = left_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME left1 = left_mbmi->ref_frame[1]; + + if (above_has_second && left_has_second) { // comp/comp + if (above0 == left0 && above1 == left1) + pred_context = 3 * (above0 == LAST_FRAME || above1 == LAST_FRAME || + left0 == LAST_FRAME || left1 == LAST_FRAME); + else + pred_context = 2; + } else if (above_has_second || left_has_second) { // single/comp + const MV_REFERENCE_FRAME rfs = !above_has_second ? above0 : left0; + const MV_REFERENCE_FRAME crf1 = above_has_second ? above0 : left0; + const MV_REFERENCE_FRAME crf2 = above_has_second ? above1 : left1; + + if (rfs == LAST_FRAME) + pred_context = 3 + (crf1 == LAST_FRAME || crf2 == LAST_FRAME); + else if (rfs == LAST2_FRAME) + pred_context = (crf1 == LAST_FRAME || crf2 == LAST_FRAME); + else + pred_context = 1 + 2 * (crf1 == LAST_FRAME || crf2 == LAST_FRAME); + } else { // single/single + if (!CHECK_LAST_OR_LAST2(above0) && !CHECK_LAST_OR_LAST2(left0)) { + pred_context = 2 + (above0 == left0); + } else if (!CHECK_LAST_OR_LAST2(above0) || + !CHECK_LAST_OR_LAST2(left0)) { + const MV_REFERENCE_FRAME edge0 = + !CHECK_LAST_OR_LAST2(above0) ? left0 : above0; + pred_context = 4 * (edge0 == LAST_FRAME); + } else { + pred_context = 2 * (above0 == LAST_FRAME) + 2 * (left0 == LAST_FRAME); + } + } + } + } else if (has_above || has_left) { // one edge available + const MB_MODE_INFO *edge_mbmi = has_above ? above_mbmi : left_mbmi; + + if (!is_inter_block(edge_mbmi) || + (!CHECK_LAST_OR_LAST2(edge_mbmi->ref_frame[0]) && + !has_second_ref(edge_mbmi))) + pred_context = 2; + else if (!has_second_ref(edge_mbmi)) // single + pred_context = 4 * (edge_mbmi->ref_frame[0] == LAST_FRAME); + else // comp + pred_context = 3 * (edge_mbmi->ref_frame[0] == LAST_FRAME || + edge_mbmi->ref_frame[1] == LAST_FRAME); + } else { // no edges available (2) + pred_context = 2; + } + + assert(pred_context >= 0 && pred_context < REF_CONTEXTS); + return pred_context; +} + +// For the bit to signal whether the single reference is GOLDEN_FRAME or +// LAST3_FRAME, knowing that it shall be either of these 2 choices. +// +// NOTE(zoeliu): The probability of ref_frame[0] is GOLDEN_FRAME, conditioning +// on it is either GOLDEN_FRAME/LAST3_FRAME. +int av1_get_pred_context_single_ref_p5(const MACROBLOCKD *xd) { + int pred_context; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int has_above = xd->up_available; + const int has_left = xd->left_available; + + // Note: + // The mode info data structure has a one element border above and to the + // left of the entries correpsonding to real macroblocks. + // The prediction flags in these dummy entries are initialised to 0. + if (has_above && has_left) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + + if (above_intra && left_intra) { // intra/intra + pred_context = 2; + } else if (above_intra || left_intra) { // intra/inter or inter/intra + const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi; + if (!has_second_ref(edge_mbmi)) { // single + if (!CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[0])) + pred_context = 3; + else + pred_context = 4 * (edge_mbmi->ref_frame[0] == LAST3_FRAME); + } else { // comp + pred_context = 1 + + 2 * (edge_mbmi->ref_frame[0] == LAST3_FRAME || + edge_mbmi->ref_frame[1] == LAST3_FRAME); + } + } else { // inter/inter + const int above_has_second = has_second_ref(above_mbmi); + const int left_has_second = has_second_ref(left_mbmi); + const MV_REFERENCE_FRAME above0 = above_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME above1 = above_mbmi->ref_frame[1]; + const MV_REFERENCE_FRAME left0 = left_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME left1 = left_mbmi->ref_frame[1]; + + if (above_has_second && left_has_second) { // comp/comp + if (above0 == left0 && above1 == left1) + pred_context = 3 * (above0 == LAST3_FRAME || above1 == LAST3_FRAME || + left0 == LAST3_FRAME || left1 == LAST3_FRAME); + else + pred_context = 2; + } else if (above_has_second || left_has_second) { // single/comp + const MV_REFERENCE_FRAME rfs = !above_has_second ? above0 : left0; + const MV_REFERENCE_FRAME crf1 = above_has_second ? above0 : left0; + const MV_REFERENCE_FRAME crf2 = above_has_second ? above1 : left1; + + if (rfs == LAST3_FRAME) + pred_context = 3 + (crf1 == LAST3_FRAME || crf2 == LAST3_FRAME); + else if (rfs == GOLDEN_FRAME) + pred_context = (crf1 == LAST3_FRAME || crf2 == LAST3_FRAME); + else + pred_context = 1 + 2 * (crf1 == LAST3_FRAME || crf2 == LAST3_FRAME); + } else { // single/single + if (!CHECK_GOLDEN_OR_LAST3(above0) && !CHECK_GOLDEN_OR_LAST3(left0)) { + pred_context = 2 + (above0 == left0); + } else if (!CHECK_GOLDEN_OR_LAST3(above0) || + !CHECK_GOLDEN_OR_LAST3(left0)) { + const MV_REFERENCE_FRAME edge0 = + !CHECK_GOLDEN_OR_LAST3(above0) ? left0 : above0; + pred_context = 4 * (edge0 == LAST3_FRAME); + } else { + pred_context = + 2 * (above0 == LAST3_FRAME) + 2 * (left0 == LAST3_FRAME); + } + } + } + } else if (has_above || has_left) { // one edge available + const MB_MODE_INFO *edge_mbmi = has_above ? above_mbmi : left_mbmi; + + if (!is_inter_block(edge_mbmi) || + (!CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[0]) && + !has_second_ref(edge_mbmi))) + pred_context = 2; + else if (!has_second_ref(edge_mbmi)) // single + pred_context = 4 * (edge_mbmi->ref_frame[0] == LAST3_FRAME); + else // comp + pred_context = 3 * (edge_mbmi->ref_frame[0] == LAST3_FRAME || + edge_mbmi->ref_frame[1] == LAST3_FRAME); + } else { // no edges available (2) + pred_context = 2; + } + + assert(pred_context >= 0 && pred_context < REF_CONTEXTS); + return pred_context; +} + +#else // CONFIG_EXT_REFS + +int av1_get_pred_context_single_ref_p1(const MACROBLOCKD *xd) { + int pred_context; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int has_above = xd->up_available; + const int has_left = xd->left_available; + // Note: + // The mode info data structure has a one element border above and to the + // left of the entries corresponding to real macroblocks. + // The prediction flags in these dummy entries are initialized to 0. + if (has_above && has_left) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + + if (above_intra && left_intra) { // intra/intra + pred_context = 2; + } else if (above_intra || left_intra) { // intra/inter or inter/intra + const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi; + if (!has_second_ref(edge_mbmi)) + pred_context = 4 * (edge_mbmi->ref_frame[0] == LAST_FRAME); + else + pred_context = 1 + (edge_mbmi->ref_frame[0] == LAST_FRAME || + edge_mbmi->ref_frame[1] == LAST_FRAME); + } else { // inter/inter + const int above_has_second = has_second_ref(above_mbmi); + const int left_has_second = has_second_ref(left_mbmi); + const MV_REFERENCE_FRAME above0 = above_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME above1 = above_mbmi->ref_frame[1]; + const MV_REFERENCE_FRAME left0 = left_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME left1 = left_mbmi->ref_frame[1]; + + if (above_has_second && left_has_second) { + pred_context = 1 + (above0 == LAST_FRAME || above1 == LAST_FRAME || + left0 == LAST_FRAME || left1 == LAST_FRAME); + } else if (above_has_second || left_has_second) { + const MV_REFERENCE_FRAME rfs = !above_has_second ? above0 : left0; + const MV_REFERENCE_FRAME crf1 = above_has_second ? above0 : left0; + const MV_REFERENCE_FRAME crf2 = above_has_second ? above1 : left1; + + if (rfs == LAST_FRAME) + pred_context = 3 + (crf1 == LAST_FRAME || crf2 == LAST_FRAME); + else + pred_context = (crf1 == LAST_FRAME || crf2 == LAST_FRAME); + } else { + pred_context = 2 * (above0 == LAST_FRAME) + 2 * (left0 == LAST_FRAME); + } + } + } else if (has_above || has_left) { // one edge available + const MB_MODE_INFO *edge_mbmi = has_above ? above_mbmi : left_mbmi; + if (!is_inter_block(edge_mbmi)) { // intra + pred_context = 2; + } else { // inter + if (!has_second_ref(edge_mbmi)) + pred_context = 4 * (edge_mbmi->ref_frame[0] == LAST_FRAME); + else + pred_context = 1 + (edge_mbmi->ref_frame[0] == LAST_FRAME || + edge_mbmi->ref_frame[1] == LAST_FRAME); + } + } else { // no edges available + pred_context = 2; + } + + assert(pred_context >= 0 && pred_context < REF_CONTEXTS); + return pred_context; +} + +int av1_get_pred_context_single_ref_p2(const MACROBLOCKD *xd) { + int pred_context; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int has_above = xd->up_available; + const int has_left = xd->left_available; + + // Note: + // The mode info data structure has a one element border above and to the + // left of the entries corresponding to real macroblocks. + // The prediction flags in these dummy entries are initialized to 0. + if (has_above && has_left) { // both edges available + const int above_intra = !is_inter_block(above_mbmi); + const int left_intra = !is_inter_block(left_mbmi); + + if (above_intra && left_intra) { // intra/intra + pred_context = 2; + } else if (above_intra || left_intra) { // intra/inter or inter/intra + const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi; + if (!has_second_ref(edge_mbmi)) { + if (edge_mbmi->ref_frame[0] == LAST_FRAME) + pred_context = 3; + else + pred_context = 4 * (edge_mbmi->ref_frame[0] == GOLDEN_FRAME); + } else { + pred_context = 1 + + 2 * (edge_mbmi->ref_frame[0] == GOLDEN_FRAME || + edge_mbmi->ref_frame[1] == GOLDEN_FRAME); + } + } else { // inter/inter + const int above_has_second = has_second_ref(above_mbmi); + const int left_has_second = has_second_ref(left_mbmi); + const MV_REFERENCE_FRAME above0 = above_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME above1 = above_mbmi->ref_frame[1]; + const MV_REFERENCE_FRAME left0 = left_mbmi->ref_frame[0]; + const MV_REFERENCE_FRAME left1 = left_mbmi->ref_frame[1]; + + if (above_has_second && left_has_second) { + if (above0 == left0 && above1 == left1) + pred_context = + 3 * (above0 == GOLDEN_FRAME || above1 == GOLDEN_FRAME || + left0 == GOLDEN_FRAME || left1 == GOLDEN_FRAME); + else + pred_context = 2; + } else if (above_has_second || left_has_second) { + const MV_REFERENCE_FRAME rfs = !above_has_second ? above0 : left0; + const MV_REFERENCE_FRAME crf1 = above_has_second ? above0 : left0; + const MV_REFERENCE_FRAME crf2 = above_has_second ? above1 : left1; + + if (rfs == GOLDEN_FRAME) + pred_context = 3 + (crf1 == GOLDEN_FRAME || crf2 == GOLDEN_FRAME); + else if (rfs != GOLDEN_FRAME && rfs != LAST_FRAME) + pred_context = crf1 == GOLDEN_FRAME || crf2 == GOLDEN_FRAME; + else + pred_context = 1 + 2 * (crf1 == GOLDEN_FRAME || crf2 == GOLDEN_FRAME); + } else { + if (above0 == LAST_FRAME && left0 == LAST_FRAME) { + pred_context = 3; + } else if (above0 == LAST_FRAME || left0 == LAST_FRAME) { + const MV_REFERENCE_FRAME edge0 = + (above0 == LAST_FRAME) ? left0 : above0; + pred_context = 4 * (edge0 == GOLDEN_FRAME); + } else { + pred_context = + 2 * (above0 == GOLDEN_FRAME) + 2 * (left0 == GOLDEN_FRAME); + } + } + } + } else if (has_above || has_left) { // one edge available + const MB_MODE_INFO *edge_mbmi = has_above ? above_mbmi : left_mbmi; + + if (!is_inter_block(edge_mbmi) || + (edge_mbmi->ref_frame[0] == LAST_FRAME && !has_second_ref(edge_mbmi))) + pred_context = 2; + else if (!has_second_ref(edge_mbmi)) + pred_context = 4 * (edge_mbmi->ref_frame[0] == GOLDEN_FRAME); + else + pred_context = 3 * (edge_mbmi->ref_frame[0] == GOLDEN_FRAME || + edge_mbmi->ref_frame[1] == GOLDEN_FRAME); + } else { // no edges available (2) + pred_context = 2; + } + assert(pred_context >= 0 && pred_context < REF_CONTEXTS); + return pred_context; +} + +#endif // CONFIG_EXT_REFS diff --git a/third_party/aom/av1/common/pred_common.h b/third_party/aom/av1/common/pred_common.h new file mode 100644 index 0000000000..e16ad70f62 --- /dev/null +++ b/third_party/aom/av1/common/pred_common.h @@ -0,0 +1,266 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_PRED_COMMON_H_ +#define AV1_COMMON_PRED_COMMON_H_ + +#include "av1/common/blockd.h" +#include "av1/common/onyxc_int.h" +#include "aom_dsp/aom_dsp_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +static INLINE int get_segment_id(const AV1_COMMON *const cm, + const uint8_t *segment_ids, BLOCK_SIZE bsize, + int mi_row, int mi_col) { + const int mi_offset = mi_row * cm->mi_cols + mi_col; + const int bw = mi_size_wide[bsize]; + const int bh = mi_size_high[bsize]; + const int xmis = AOMMIN(cm->mi_cols - mi_col, bw); + const int ymis = AOMMIN(cm->mi_rows - mi_row, bh); + int x, y, segment_id = MAX_SEGMENTS; + + for (y = 0; y < ymis; ++y) + for (x = 0; x < xmis; ++x) + segment_id = + AOMMIN(segment_id, segment_ids[mi_offset + y * cm->mi_cols + x]); + + assert(segment_id >= 0 && segment_id < MAX_SEGMENTS); + return segment_id; +} + +static INLINE int av1_get_pred_context_seg_id(const MACROBLOCKD *xd) { + const MODE_INFO *const above_mi = xd->above_mi; + const MODE_INFO *const left_mi = xd->left_mi; + const int above_sip = + (above_mi != NULL) ? above_mi->mbmi.seg_id_predicted : 0; + const int left_sip = (left_mi != NULL) ? left_mi->mbmi.seg_id_predicted : 0; + + return above_sip + left_sip; +} + +static INLINE aom_prob av1_get_pred_prob_seg_id( + const struct segmentation_probs *segp, const MACROBLOCKD *xd) { + return segp->pred_probs[av1_get_pred_context_seg_id(xd)]; +} + +static INLINE int av1_get_skip_context(const MACROBLOCKD *xd) { + const MODE_INFO *const above_mi = xd->above_mi; + const MODE_INFO *const left_mi = xd->left_mi; + const int above_skip = (above_mi != NULL) ? above_mi->mbmi.skip : 0; + const int left_skip = (left_mi != NULL) ? left_mi->mbmi.skip : 0; + return above_skip + left_skip; +} + +static INLINE aom_prob av1_get_skip_prob(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + return cm->fc->skip_probs[av1_get_skip_context(xd)]; +} + +#if CONFIG_DUAL_FILTER +int av1_get_pred_context_switchable_interp(const MACROBLOCKD *xd, int dir); +#else +int av1_get_pred_context_switchable_interp(const MACROBLOCKD *xd); +#endif + +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP +int av1_get_pred_context_intra_interp(const MACROBLOCKD *xd); +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA + +int av1_get_intra_inter_context(const MACROBLOCKD *xd); + +static INLINE aom_prob av1_get_intra_inter_prob(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + return cm->fc->intra_inter_prob[av1_get_intra_inter_context(xd)]; +} + +int av1_get_reference_mode_context(const AV1_COMMON *cm, const MACROBLOCKD *xd); + +static INLINE aom_prob av1_get_reference_mode_prob(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + return cm->fc->comp_inter_prob[av1_get_reference_mode_context(cm, xd)]; +} + +int av1_get_pred_context_comp_ref_p(const AV1_COMMON *cm, + const MACROBLOCKD *xd); + +static INLINE aom_prob av1_get_pred_prob_comp_ref_p(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + const int pred_context = av1_get_pred_context_comp_ref_p(cm, xd); + return cm->fc->comp_ref_prob[pred_context][0]; +} + +#if CONFIG_EXT_REFS +int av1_get_pred_context_comp_ref_p1(const AV1_COMMON *cm, + const MACROBLOCKD *xd); + +static INLINE aom_prob av1_get_pred_prob_comp_ref_p1(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + const int pred_context = av1_get_pred_context_comp_ref_p1(cm, xd); + return cm->fc->comp_ref_prob[pred_context][1]; +} + +int av1_get_pred_context_comp_ref_p2(const AV1_COMMON *cm, + const MACROBLOCKD *xd); + +static INLINE aom_prob av1_get_pred_prob_comp_ref_p2(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + const int pred_context = av1_get_pred_context_comp_ref_p2(cm, xd); + return cm->fc->comp_ref_prob[pred_context][2]; +} + +int av1_get_pred_context_comp_bwdref_p(const AV1_COMMON *cm, + const MACROBLOCKD *xd); + +static INLINE aom_prob av1_get_pred_prob_comp_bwdref_p(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + const int pred_context = av1_get_pred_context_comp_bwdref_p(cm, xd); + return cm->fc->comp_bwdref_prob[pred_context][0]; +} +#endif // CONFIG_EXT_REFS + +int av1_get_pred_context_single_ref_p1(const MACROBLOCKD *xd); + +static INLINE aom_prob av1_get_pred_prob_single_ref_p1(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + return cm->fc->single_ref_prob[av1_get_pred_context_single_ref_p1(xd)][0]; +} + +int av1_get_pred_context_single_ref_p2(const MACROBLOCKD *xd); + +static INLINE aom_prob av1_get_pred_prob_single_ref_p2(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + return cm->fc->single_ref_prob[av1_get_pred_context_single_ref_p2(xd)][1]; +} + +#if CONFIG_EXT_REFS +int av1_get_pred_context_single_ref_p3(const MACROBLOCKD *xd); + +static INLINE aom_prob av1_get_pred_prob_single_ref_p3(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + return cm->fc->single_ref_prob[av1_get_pred_context_single_ref_p3(xd)][2]; +} + +int av1_get_pred_context_single_ref_p4(const MACROBLOCKD *xd); + +static INLINE aom_prob av1_get_pred_prob_single_ref_p4(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + return cm->fc->single_ref_prob[av1_get_pred_context_single_ref_p4(xd)][3]; +} + +int av1_get_pred_context_single_ref_p5(const MACROBLOCKD *xd); + +static INLINE aom_prob av1_get_pred_prob_single_ref_p5(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + return cm->fc->single_ref_prob[av1_get_pred_context_single_ref_p5(xd)][4]; +} +#endif // CONFIG_EXT_REFS + +#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF +int av1_get_inter_mode_context(const MACROBLOCKD *xd); + +static INLINE aom_prob av1_get_inter_mode_prob(const AV1_COMMON *cm, + const MACROBLOCKD *xd) { + return cm->fc->comp_inter_mode_prob[av1_get_inter_mode_context(xd)]; +} +#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF + +// Returns a context number for the given MB prediction signal +// The mode info data structure has a one element border above and to the +// left of the entries corresponding to real blocks. +// The prediction flags in these dummy entries are initialized to 0. +static INLINE int get_tx_size_context(const MACROBLOCKD *xd) { + const int max_tx_size = max_txsize_lookup[xd->mi[0]->mbmi.sb_type]; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + const int has_above = xd->up_available; + const int has_left = xd->left_available; + int above_ctx = (has_above && !above_mbmi->skip) + ? (int)txsize_sqr_map[above_mbmi->tx_size] + : max_tx_size; + int left_ctx = (has_left && !left_mbmi->skip) + ? (int)txsize_sqr_map[left_mbmi->tx_size] + : max_tx_size; + + if (!has_left) left_ctx = above_ctx; + + if (!has_above) above_ctx = left_ctx; +#if CONFIG_CB4X4 + // TODO(jingning): Temporary setup. Will rework this after the cb4x4 + // framework is up running. + return (above_ctx + left_ctx) > max_tx_size + 1; +#else + return (above_ctx + left_ctx) > max_tx_size; +#endif +} + +#if CONFIG_VAR_TX +static void update_tx_counts(AV1_COMMON *cm, MACROBLOCKD *xd, + MB_MODE_INFO *mbmi, BLOCK_SIZE plane_bsize, + TX_SIZE tx_size, int blk_row, int blk_col, + TX_SIZE max_tx_size, int ctx) { + const struct macroblockd_plane *const pd = &xd->plane[0]; + const BLOCK_SIZE bsize = txsize_to_bsize[tx_size]; + const int tx_row = blk_row >> (1 - pd->subsampling_y); + const int tx_col = blk_col >> (1 - pd->subsampling_x); + const TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_row][tx_col]; + const int max_blocks_high = max_block_high(xd, plane_bsize, 0); + const int max_blocks_wide = max_block_wide(xd, plane_bsize, 0); + + if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + + if (tx_size == plane_tx_size) { + ++xd->counts->tx_size[max_tx_size - TX_8X8][ctx][tx_size]; + mbmi->tx_size = tx_size; + } else { + int bsl = b_width_log2_lookup[bsize]; + int i; + + assert(bsl > 0); + --bsl; + + for (i = 0; i < 4; ++i) { + const int offsetr = blk_row + ((i >> 1) << bsl); + const int offsetc = blk_col + ((i & 0x01) << bsl); + + if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue; + update_tx_counts(cm, xd, mbmi, plane_bsize, (TX_SIZE)(tx_size - 1), + offsetr, offsetc, max_tx_size, ctx); + } + } +} + +static INLINE void inter_block_tx_count_update(AV1_COMMON *cm, MACROBLOCKD *xd, + MB_MODE_INFO *mbmi, + BLOCK_SIZE plane_bsize, + int ctx) { + const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; + const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0]; + TX_SIZE max_tx_size = max_txsize_lookup[plane_bsize]; + int bh = tx_size_wide_unit[max_tx_size]; + int idx, idy; + + for (idy = 0; idy < mi_height; idy += bh) + for (idx = 0; idx < mi_width; idx += bh) + update_tx_counts(cm, xd, mbmi, plane_bsize, max_tx_size, idy, idx, + max_tx_size, ctx); +} +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_PRED_COMMON_H_ diff --git a/third_party/aom/av1/common/pvq.c b/third_party/aom/av1/common/pvq.c new file mode 100644 index 0000000000..75fe761d74 --- /dev/null +++ b/third_party/aom/av1/common/pvq.c @@ -0,0 +1,1007 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "odintrin.h" +#include "partition.h" +#include "pvq.h" +#include +#include +#include +#include + +/* Imported from encode.c in daala */ +/* These are the PVQ equivalent of quantization matrices, except that + the values are per-band. */ +#define OD_MASKING_DISABLED 0 +#define OD_MASKING_ENABLED 1 + +const unsigned char OD_LUMA_QM_Q4[2][OD_QM_SIZE] = { +/* Flat quantization for PSNR. The DC component isn't 16 because the DC + magnitude compensation is done here for inter (Haar DC doesn't need it). + Masking disabled: */ + { + 16, 16, + 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16 + }, +/* The non-flat AC coefficients compensate for the non-linear scaling caused + by activity masking. The values are currently hand-tuned so that the rate + of each band remains roughly constant when enabling activity masking + on intra. + Masking enabled: */ + { + 16, 16, + 16, 18, 28, 32, + 16, 14, 20, 20, 28, 32, + 16, 11, 14, 14, 17, 17, 22, 28 + } +}; + +const unsigned char OD_CHROMA_QM_Q4[2][OD_QM_SIZE] = { +/* Chroma quantization is different because of the reduced lapping. + FIXME: Use the same matrix as luma for 4:4:4. + Masking disabled: */ + { + 16, 16, + 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16 + }, +/* The AC part is flat for chroma because it has no activity masking. + Masking enabled: */ + { + 16, 16, + 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16 + } +}; + +/* No interpolation, always use od_flat_qm_q4, but use a different scale for + each plane. + FIXME: Add interpolation and properly tune chroma. */ +const od_qm_entry OD_DEFAULT_QMS[2][2][OD_NPLANES_MAX] = { + /* Masking disabled */ + { { { 4, 256, OD_LUMA_QM_Q4[OD_MASKING_DISABLED] }, + { 4, 256, OD_CHROMA_QM_Q4[OD_MASKING_DISABLED] }, + { 4, 256, OD_CHROMA_QM_Q4[OD_MASKING_DISABLED] } }, + { { 0, 0, NULL}, + { 0, 0, NULL}, + { 0, 0, NULL} } }, + /* Masking enabled */ + { { { 4, 256, OD_LUMA_QM_Q4[OD_MASKING_ENABLED] }, + { 4, 256, OD_CHROMA_QM_Q4[OD_MASKING_ENABLED] }, + { 4, 256, OD_CHROMA_QM_Q4[OD_MASKING_ENABLED] } }, + { { 0, 0, NULL}, + { 0, 0, NULL}, + { 0, 0, NULL} } } +}; + +/* Constants for the beta parameter, which controls how activity masking is + used. + beta = 1 / (1 - alpha), so when beta is 1, alpha is 0 and activity + masking is disabled. When beta is 1.5, activity masking is used. Note that + activity masking is neither used for 4x4 blocks nor for chroma. */ +#define OD_BETA(b) OD_QCONST32(b, OD_BETA_SHIFT) +static const od_val16 OD_PVQ_BETA4_LUMA[1] = {OD_BETA(1.)}; +static const od_val16 OD_PVQ_BETA8_LUMA[4] = {OD_BETA(1.), OD_BETA(1.), + OD_BETA(1.), OD_BETA(1.)}; +static const od_val16 OD_PVQ_BETA16_LUMA[7] = {OD_BETA(1.), OD_BETA(1.), + OD_BETA(1.), OD_BETA(1.), OD_BETA(1.), OD_BETA(1.), OD_BETA(1.)}; +static const od_val16 OD_PVQ_BETA32_LUMA[10] = {OD_BETA(1.), OD_BETA(1.), + OD_BETA(1.), OD_BETA(1.), OD_BETA(1.), OD_BETA(1.), OD_BETA(1.), OD_BETA(1.), + OD_BETA(1.), OD_BETA(1.)}; + +static const od_val16 OD_PVQ_BETA4_LUMA_MASKING[1] = {OD_BETA(1.)}; +static const od_val16 OD_PVQ_BETA8_LUMA_MASKING[4] = {OD_BETA(1.5), + OD_BETA(1.5), OD_BETA(1.5), OD_BETA(1.5)}; +static const od_val16 OD_PVQ_BETA16_LUMA_MASKING[7] = {OD_BETA(1.5), + OD_BETA(1.5), OD_BETA(1.5), OD_BETA(1.5), OD_BETA(1.5), OD_BETA(1.5), + OD_BETA(1.5)}; +static const od_val16 OD_PVQ_BETA32_LUMA_MASKING[10] = {OD_BETA(1.5), + OD_BETA(1.5), OD_BETA(1.5), OD_BETA(1.5), OD_BETA(1.5), OD_BETA(1.5), + OD_BETA(1.5), OD_BETA(1.5), OD_BETA(1.5), OD_BETA(1.5)}; + +static const od_val16 OD_PVQ_BETA4_CHROMA[1] = {OD_BETA(1.)}; +static const od_val16 OD_PVQ_BETA8_CHROMA[4] = {OD_BETA(1.), OD_BETA(1.), + OD_BETA(1.), OD_BETA(1.)}; +static const od_val16 OD_PVQ_BETA16_CHROMA[7] = {OD_BETA(1.), OD_BETA(1.), + OD_BETA(1.), OD_BETA(1.), OD_BETA(1.), OD_BETA(1.), OD_BETA(1.)}; +static const od_val16 OD_PVQ_BETA32_CHROMA[10] = {OD_BETA(1.), OD_BETA(1.), + OD_BETA(1.), OD_BETA(1.), OD_BETA(1.), OD_BETA(1.), OD_BETA(1.), OD_BETA(1.), + OD_BETA(1.), OD_BETA(1.)}; + +const od_val16 *const OD_PVQ_BETA[2][OD_NPLANES_MAX][OD_TXSIZES + 1] = { + {{OD_PVQ_BETA4_LUMA, OD_PVQ_BETA8_LUMA, + OD_PVQ_BETA16_LUMA, OD_PVQ_BETA32_LUMA}, + {OD_PVQ_BETA4_CHROMA, OD_PVQ_BETA8_CHROMA, + OD_PVQ_BETA16_CHROMA, OD_PVQ_BETA32_CHROMA}, + {OD_PVQ_BETA4_CHROMA, OD_PVQ_BETA8_CHROMA, + OD_PVQ_BETA16_CHROMA, OD_PVQ_BETA32_CHROMA}}, + {{OD_PVQ_BETA4_LUMA_MASKING, OD_PVQ_BETA8_LUMA_MASKING, + OD_PVQ_BETA16_LUMA_MASKING, OD_PVQ_BETA32_LUMA_MASKING}, + {OD_PVQ_BETA4_CHROMA, OD_PVQ_BETA8_CHROMA, + OD_PVQ_BETA16_CHROMA, OD_PVQ_BETA32_CHROMA}, + {OD_PVQ_BETA4_CHROMA, OD_PVQ_BETA8_CHROMA, + OD_PVQ_BETA16_CHROMA, OD_PVQ_BETA32_CHROMA}} +}; + + +void od_interp_qm(unsigned char *out, int q, const od_qm_entry *entry1, + const od_qm_entry *entry2) { + int i; + if (entry2 == NULL || entry2->qm_q4 == NULL + || q < entry1->interp_q << OD_COEFF_SHIFT) { + /* Use entry1. */ + for (i = 0; i < OD_QM_SIZE; i++) { + out[i] = OD_MINI(255, entry1->qm_q4[i]*entry1->scale_q8 >> 8); + } + } + else if (entry1 == NULL || entry1->qm_q4 == NULL + || q > entry2->interp_q << OD_COEFF_SHIFT) { + /* Use entry2. */ + for (i = 0; i < OD_QM_SIZE; i++) { + out[i] = OD_MINI(255, entry2->qm_q4[i]*entry2->scale_q8 >> 8); + } + } + else { + /* Interpolate between entry1 and entry2. The interpolation is linear + in terms of log(q) vs log(m*scale). Considering that we're ultimately + multiplying the result it makes sense, but we haven't tried other + interpolation methods. */ + double x; + const unsigned char *m1; + const unsigned char *m2; + int q1; + int q2; + m1 = entry1->qm_q4; + m2 = entry2->qm_q4; + q1 = entry1->interp_q << OD_COEFF_SHIFT; + q2 = entry2->interp_q << OD_COEFF_SHIFT; + x = (log(q)-log(q1))/(log(q2)-log(q1)); + for (i = 0; i < OD_QM_SIZE; i++) { + out[i] = OD_MINI(255, (int)floor(.5 + (1./256)*exp( + x*log(m2[i]*entry2->scale_q8) + (1 - x)*log(m1[i]*entry1->scale_q8)))); + } + } +} + +void od_adapt_pvq_ctx_reset(od_pvq_adapt_ctx *state, int is_keyframe) { + od_pvq_codeword_ctx *ctx; + int i; + int pli; + int bs; + ctx = &state->pvq_codeword_ctx; + OD_CDFS_INIT_DYNAMIC(state->pvq_param_model[0].cdf); + OD_CDFS_INIT_DYNAMIC(state->pvq_param_model[1].cdf); + OD_CDFS_INIT_DYNAMIC(state->pvq_param_model[2].cdf); + for (i = 0; i < 2*OD_TXSIZES; i++) { + ctx->pvq_adapt[4*i + OD_ADAPT_K_Q8] = 384; + ctx->pvq_adapt[4*i + OD_ADAPT_SUM_EX_Q8] = 256; + ctx->pvq_adapt[4*i + OD_ADAPT_COUNT_Q8] = 104; + ctx->pvq_adapt[4*i + OD_ADAPT_COUNT_EX_Q8] = 128; + } + OD_CDFS_INIT_DYNAMIC(ctx->pvq_k1_cdf); + for (pli = 0; pli < OD_NPLANES_MAX; pli++) { + for (bs = 0; bs < OD_TXSIZES; bs++) + for (i = 0; i < PVQ_MAX_PARTITIONS; i++) { + state->pvq_exg[pli][bs][i] = 2 << 16; + } + } + for (i = 0; i < OD_TXSIZES*PVQ_MAX_PARTITIONS; i++) { + state->pvq_ext[i] = is_keyframe ? 24576 : 2 << 16; + } + OD_CDFS_INIT_DYNAMIC(state->pvq_gaintheta_cdf); + OD_CDFS_INIT_Q15(state->pvq_skip_dir_cdf); + OD_CDFS_INIT_DYNAMIC(ctx->pvq_split_cdf); +} + +/* QMs are arranged from smallest to largest blocksizes, first for + blocks with decimation=0, followed by blocks with decimation=1.*/ +int od_qm_offset(int bs, int xydec) +{ + return xydec*OD_QM_STRIDE + OD_QM_OFFSET(bs); +} + +#if defined(OD_FLOAT_PVQ) +#define OD_DEFAULT_MAG 1.0 +#else +#define OD_DEFAULT_MAG OD_QM_SCALE +#endif + +/* Initialize the quantization matrix. */ +// Note: When hybrid transform and corresponding scan order is used by PVQ, +// we don't need seperate qm and qm_inv for each transform type, +// because AOM does not do magnitude compensation (i.e. simplay x16 for all coeffs). +void od_init_qm(int16_t *x, int16_t *x_inv, const int *qm) { + int i; + int j; + int16_t y[OD_TXSIZE_MAX*OD_TXSIZE_MAX]; + int16_t y_inv[OD_TXSIZE_MAX*OD_TXSIZE_MAX]; + int16_t *x1; + int16_t *x1_inv; + int off; + int bs; + int xydec; + for (bs = 0; bs < OD_TXSIZES; bs++) { + for (xydec = 0; xydec < 2; xydec++) { + off = od_qm_offset(bs, xydec); + x1 = x + off; + x1_inv = x_inv + off; + for (i = 0; i < 4 << bs; i++) { + for (j = 0; j < 4 << bs; j++) { + /*This will ultimately be clamped to fit in 16 bits.*/ + od_val32 mag; + int16_t ytmp; + mag = OD_DEFAULT_MAG; + if (i != 0 || j != 0) { +#if defined(OD_FLOAT_PVQ) + mag /= 0.0625*qm[(i << 1 >> bs)*8 + (j << 1 >> bs)]; +#else + int qmv; + qmv = qm[(i << 1 >> bs)*8 + (j << 1 >> bs)]; + mag *= 16; + mag = (mag + (qmv >> 1))/qmv; +#endif + OD_ASSERT(mag > 0.0); + } + /*Convert to fit in 16 bits.*/ +#if defined(OD_FLOAT_PVQ) + y[i*(4 << bs) + j] = (int16_t)OD_MINI(OD_QM_SCALE_MAX, + (int32_t)floor(.5 + mag*OD_QM_SCALE)); + y_inv[i*(4 << bs) + j] = (int16_t)floor(.5 + + OD_QM_SCALE*OD_QM_INV_SCALE/(double)y[i*(4 << bs) + j]); +#else + y[i*(4 << bs) + j] = (int16_t)OD_MINI(OD_QM_SCALE_MAX, mag); + ytmp = y[i*(4 << bs) + j]; + y_inv[i*(4 << bs) + j] = (int16_t)((OD_QM_SCALE*OD_QM_INV_SCALE + + (ytmp >> 1))/ytmp); +#endif + } + } + od_raster_to_coding_order_16(x1, 4 << bs, y, 4 << bs); + od_raster_to_coding_order_16(x1_inv, 4 << bs, y_inv, 4 << bs); + } + } +} + +/* Maps each possible size (n) in the split k-tokenizer to a different value. + Possible values of n are: + 2, 3, 4, 7, 8, 14, 15, 16, 31, 32, 63, 64, 127, 128 + Since we don't care about the order (even in the bit-stream) the simplest + ordering (implemented here) is: + 14, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 */ +int od_pvq_size_ctx(int n) { + int logn; + int odd; + logn = OD_ILOG(n - 1); + odd = n & 1; + return 2*logn - 1 - odd - 7*(n == 14); +} + +/* Maps a length n to a context for the (k=1, n<=16) coder, with a special + case when n is the original length (orig_length=1) of the vector (i.e. we + haven't split it yet). For orig_length=0, we use the same mapping as + od_pvq_size_ctx() up to n=16. When orig_length=1, we map lengths + 7, 8, 14, 15 to contexts 8 to 11. */ +int od_pvq_k1_ctx(int n, int orig_length) { + if (orig_length) return 8 + 2*(n > 8) + (n & 1); + else return od_pvq_size_ctx(n); +} + +/* Indexing for the packed quantization matrices. */ +int od_qm_get_index(int bs, int band) { + /* The -band/3 term is due to the fact that we force corresponding horizontal + and vertical bands to have the same quantization. */ + OD_ASSERT(bs >= 0 && bs < OD_TXSIZES); + return bs*(bs + 1) + band - band/3; +} + +#if !defined(OD_FLOAT_PVQ) +/*See celt/mathops.c in Opus and tools/cos_search.c.*/ +static int16_t od_pvq_cos_pi_2(int16_t x) +{ + int16_t x2; + x2 = OD_MULT16_16_Q15(x, x); + return OD_MINI(32767, (1073758164 - x*x + x2*(-7654 + OD_MULT16_16_Q16(x2, + 16573 + OD_MULT16_16_Q16(-2529, x2)))) >> 15); +} +#endif + +/*Approximates cos(x) for -pi < x < pi. + Input is in OD_THETA_SCALE.*/ +od_val16 od_pvq_cos(od_val32 x) { +#if defined(OD_FLOAT_PVQ) + return cos(x); +#else + /*Wrap x around by masking, since cos is periodic.*/ + x = x & 0x0001ffff; + if (x > (1 << 16)) { + x = (1 << 17) - x; + } + if (x & 0x00007fff) { + if (x < (1 << 15)) { + return od_pvq_cos_pi_2((int16_t)x); + } + else { + return -od_pvq_cos_pi_2((int16_t)(65536 - x)); + } + } + else { + if (x & 0x0000ffff) { + return 0; + } + else if (x & 0x0001ffff) { + return -32767; + } + else { + return 32767; + } + } +#endif +} + +/*Approximates sin(x) for 0 <= x < pi. + Input is in OD_THETA_SCALE.*/ +od_val16 od_pvq_sin(od_val32 x) { +#if defined(OD_FLOAT_PVQ) + return sin(x); +#else + return od_pvq_cos(32768 - x); +#endif +} + +#if !defined(OD_FLOAT_PVQ) +/* Computes an upper-bound on the number of bits required to store the L2 norm + of a vector (excluding sign). */ +int od_vector_log_mag(const od_coeff *x, int n) { + int i; + int32_t sum; + sum = 0; + for (i = 0; i < n; i++) { + int16_t tmp; + tmp = x[i] >> 8; + sum += tmp*(int32_t)tmp; + } + /* We add one full bit (instead of rounding OD_ILOG() up) for safety because + the >> 8 above causes the sum to be slightly underestimated. */ + return 8 + 1 + OD_ILOG(n + sum)/2; +} +#endif + +/** Computes Householder reflection that aligns the reference r to the + * dimension in r with the greatest absolute value. The reflection + * vector is returned in r. + * + * @param [in,out] r reference vector to be reflected, reflection + * also returned in r + * @param [in] n number of dimensions in r + * @param [in] gr gain of reference vector + * @param [out] sign sign of reflection + * @return dimension number to which reflection aligns + **/ +int od_compute_householder(od_val16 *r, int n, od_val32 gr, int *sign, + int shift) { + int m; + int i; + int s; + od_val16 maxr; + OD_UNUSED(shift); + /* Pick component with largest magnitude. Not strictly + * necessary, but it helps numerical stability */ + m = 0; + maxr = 0; + for (i = 0; i < n; i++) { + if (OD_ABS(r[i]) > maxr) { + maxr = OD_ABS(r[i]); + m = i; + } + } + s = r[m] > 0 ? 1 : -1; + /* This turns r into a Householder reflection vector that would reflect + * the original r[] to e_m */ + r[m] += OD_SHR_ROUND(gr*s, shift); + *sign = s; + return m; +} + +#if !defined(OD_FLOAT_PVQ) +#define OD_RCP_INSHIFT 15 +#define OD_RCP_OUTSHIFT 14 +static od_val16 od_rcp(od_val16 x) +{ + int i; + od_val16 n; + od_val16 r; + i = OD_ILOG(x) - 1; + /*n is Q15 with range [0,1).*/ + n = OD_VSHR_ROUND(x, i - OD_RCP_INSHIFT) - (1 << OD_RCP_INSHIFT); + /*Start with a linear approximation: + r = 1.8823529411764706-0.9411764705882353*n. + The coefficients and the result are Q14 in the range [15420,30840].*/ + r = 30840 + OD_MULT16_16_Q15(-15420, n); + /*Perform two Newton iterations: + r -= r*((r*n)-1.Q15) + = r*((r*n)+(r-1.Q15)).*/ + r = r - OD_MULT16_16_Q15(r, (OD_MULT16_16_Q15(r, n) + r - 32768)); + /*We subtract an extra 1 in the second iteration to avoid overflow; it also + neatly compensates for truncation error in the rest of the process.*/ + r = r - (1 + OD_MULT16_16_Q15(r, OD_MULT16_16_Q15(r, n) + r - 32768)); + /*r is now the Q15 solution to 2/(n+1), with a maximum relative error + of 7.05346E-5, a (relative) RMSE of 2.14418E-5, and a peak absolute + error of 1.24665/32768.*/ + return OD_VSHR_ROUND(r, i - OD_RCP_OUTSHIFT); +} +#endif + +/** Applies Householder reflection from compute_householder(). The + * reflection is its own inverse. + * + * @param [out] out reflected vector + * @param [in] x vector to be reflected + * @param [in] r reflection + * @param [in] n number of dimensions in x,r + */ +void od_apply_householder(od_val16 *out, const od_val16 *x, const od_val16 *r, + int n) { + int i; + od_val32 proj; + od_val16 proj_1; + od_val32 l2r; +#if !defined(OD_FLOAT_PVQ) + od_val16 proj_norm; + od_val16 l2r_norm; + od_val16 rcp; + int proj_shift; + int l2r_shift; + int outshift; +#endif + /*FIXME: Can we get l2r and/or l2r_shift from an earlier computation?*/ + l2r = 0; + for (i = 0; i < n; i++) { + l2r += OD_MULT16_16(r[i], r[i]); + } + /* Apply Householder reflection */ + proj = 0; + for (i = 0; i < n; i++) { + proj += OD_MULT16_16(r[i], x[i]); + } +#if defined(OD_FLOAT_PVQ) + proj_1 = proj*2./(1e-100 + l2r); + for (i = 0; i < n; i++) { + out[i] = x[i] - r[i]*proj_1; + } +#else + /*l2r_norm is [0.5, 1.0[ in Q15.*/ + l2r_shift = (OD_ILOG(l2r) - 1) - 14; + l2r_norm = OD_VSHR_ROUND(l2r, l2r_shift); + rcp = od_rcp(l2r_norm); + proj_shift = (OD_ILOG(abs(proj)) - 1) - 14; + /*proj_norm is [0.5, 1.0[ in Q15.*/ + proj_norm = OD_VSHR_ROUND(proj, proj_shift); + proj_1 = OD_MULT16_16_Q15(proj_norm, rcp); + /*The proj*2. in the float code becomes -1 in the final outshift. + The sign of l2r_shift is positive since we're taking the reciprocal of + l2r_norm and this is a right shift.*/ + outshift = OD_MINI(30, OD_RCP_OUTSHIFT - proj_shift - 1 + l2r_shift); + if (outshift >= 0) { + for (i = 0; i < n; i++) { + int32_t tmp; + tmp = OD_MULT16_16(r[i], proj_1); + tmp = OD_SHR_ROUND(tmp, outshift); + out[i] = x[i] - tmp; + } + } + else { + /*FIXME: Can we make this case impossible? + Right now, if r[] is all zeros except for 1, 2, or 3 ones, and + if x[] is all zeros except for large values at the same position as the + ones in r[], then we can end up with a shift of -1.*/ + for (i = 0; i < n; i++) { + int32_t tmp; + tmp = OD_MULT16_16(r[i], proj_1); + tmp = OD_SHL(tmp, -outshift); + out[i] = x[i] - tmp; + } + } +#endif +} + +#if !defined(OD_FLOAT_PVQ) +static od_val16 od_beta_rcp(od_val16 beta){ + if (beta == OD_BETA(1.)) + return OD_BETA(1.); + else if (beta == OD_BETA(1.5)) + return OD_BETA(1./1.5); + else { + od_val16 rcp_beta; + /*Shift by 1 less, transposing beta to range [.5, .75] and thus < 32768.*/ + rcp_beta = od_rcp(beta << (OD_RCP_INSHIFT - 1 - OD_BETA_SHIFT)); + return OD_SHR_ROUND(rcp_beta, OD_RCP_OUTSHIFT + 1 - OD_BETA_SHIFT); + } +} + +#define OD_EXP2_INSHIFT 15 +#define OD_EXP2_FRACSHIFT 15 +#define OD_EXP2_OUTSHIFT 15 +static const int32_t OD_EXP2_C[5] = {32768, 22709, 7913, 1704, 443}; +/*Output is [1.0, 2.0) in Q(OD_EXP2_FRACSHIFT). + It does not include the integer offset, which is added in od_exp2 after the + final shift).*/ +static int32_t od_exp2_frac(int32_t x) +{ + return OD_MULT16_16_Q15(x, (OD_EXP2_C[1] + OD_MULT16_16_Q15(x, + (OD_EXP2_C[2] + OD_MULT16_16_Q15(x, (OD_EXP2_C[3] + + OD_MULT16_16_Q15(x, OD_EXP2_C[4]))))))); +} + +/** Base-2 exponential approximation (2^x) with Q15 input and output.*/ +static int32_t od_exp2(int32_t x) +{ + int integer; + int32_t frac; + integer = x >> OD_EXP2_INSHIFT; + if (integer > 14) + return 0x7f000000; + else if (integer < -15) + return 0; + frac = od_exp2_frac(x - OD_SHL(integer, OD_EXP2_INSHIFT)); + return OD_VSHR_ROUND(OD_EXP2_C[0] + frac, -integer) + 1; +} + +#define OD_LOG2_INSHIFT 15 +#define OD_LOG2_OUTSHIFT 15 +#define OD_LOG2_INSCALE_1 (1./(1 << OD_LOG2_INSHIFT)) +#define OD_LOG2_OUTSCALE (1 << OD_LOG2_OUTSHIFT) +static int16_t od_log2(int16_t x) +{ + return x + OD_MULT16_16_Q15(x, (14482 + OD_MULT16_16_Q15(x, (-23234 + + OD_MULT16_16_Q15(x, (13643 + OD_MULT16_16_Q15(x, (-6403 + + OD_MULT16_16_Q15(x, 1515))))))))); +} + +static int32_t od_pow(int32_t x, od_val16 beta) +{ + int16_t t; + int xshift; + int log2_x; + od_val32 logr; + /*FIXME: this conditional is to avoid doing log2(0).*/ + if (x == 0) + return 0; + log2_x = (OD_ILOG(x) - 1); + xshift = log2_x - OD_LOG2_INSHIFT; + /*t should be in range [0.0, 1.0[ in Q(OD_LOG2_INSHIFT).*/ + t = OD_VSHR(x, xshift) - (1 << OD_LOG2_INSHIFT); + /*log2(g/OD_COMPAND_SCALE) = log2(x) - OD_COMPAND_SHIFT in + Q(OD_LOG2_OUTSHIFT).*/ + logr = od_log2(t) + (log2_x - OD_COMPAND_SHIFT)*OD_LOG2_OUTSCALE; + logr = OD_MULT16_32_QBETA(beta, logr); + return od_exp2(logr); +} +#endif + +/** Gain companding: raises gain to the power 1/beta for activity masking. + * + * @param [in] g real (uncompanded) gain + * @param [in] q0 uncompanded quality parameter + * @param [in] beta activity masking beta param (exponent) + * @return g^(1/beta) + */ +static od_val32 od_gain_compand(od_val32 g, int q0, od_val16 beta) { +#if defined(OD_FLOAT_PVQ) + if (beta == 1) return OD_CGAIN_SCALE*g/(double)q0; + else { + return OD_CGAIN_SCALE*OD_COMPAND_SCALE*pow(g*OD_COMPAND_SCALE_1, + 1./beta)/(double)q0; + } +#else + if (beta == OD_BETA(1)) return (OD_CGAIN_SCALE*g + (q0 >> 1))/q0; + else { + int32_t expr; + expr = od_pow(g, od_beta_rcp(beta)); + expr <<= OD_CGAIN_SHIFT + OD_COMPAND_SHIFT - OD_EXP2_OUTSHIFT; + return (expr + (q0 >> 1))/q0; + } +#endif +} + +#if !defined(OD_FLOAT_PVQ) +#define OD_SQRT_INSHIFT 16 +#define OD_SQRT_OUTSHIFT 15 +static int16_t od_rsqrt_norm(int16_t x); + +static int16_t od_sqrt_norm(int32_t x) +{ + OD_ASSERT(x < 65536); + return OD_MINI(OD_SHR_ROUND(x*od_rsqrt_norm(x), OD_SQRT_OUTSHIFT), 32767); +} + +static int16_t od_sqrt(int32_t x, int *sqrt_shift) +{ + int k; + int s; + int32_t t; + if (x == 0) { + *sqrt_shift = 0; + return 0; + } + OD_ASSERT(x < (1 << 30)); + k = ((OD_ILOG(x) - 1) >> 1); + /*t is x in the range [0.25, 1) in QINSHIFT, or x*2^(-s). + Shift by log2(x) - log2(0.25*(1 << INSHIFT)) to ensure 0.25 lower bound.*/ + s = 2*k - (OD_SQRT_INSHIFT - 2); + t = OD_VSHR(x, s); + /*We want to express od_sqrt() in terms of od_sqrt_norm(), which is + defined as (2^OUTSHIFT)*sqrt(t*(2^-INSHIFT)) with t=x*(2^-s). + This simplifies to 2^(OUTSHIFT-(INSHIFT/2)-(s/2))*sqrt(x), so the caller + needs to shift right by OUTSHIFT - INSHIFT/2 - s/2.*/ + *sqrt_shift = OD_SQRT_OUTSHIFT - ((s + OD_SQRT_INSHIFT) >> 1); + return od_sqrt_norm(t); +} +#endif + +/** Gain expanding: raises gain to the power beta for activity masking. + * + * @param [in] cg companded gain + * @param [in] q0 uncompanded quality parameter + * @param [in] beta activity masking beta param (exponent) + * @return g^beta + */ +od_val32 od_gain_expand(od_val32 cg0, int q0, od_val16 beta) { + if (beta == OD_BETA(1)) { + /*The multiply fits into 28 bits because the expanded gain has a range from + 0 to 2^20.*/ + return OD_SHR_ROUND(cg0*q0, OD_CGAIN_SHIFT); + } + else if (beta == OD_BETA(1.5)) { +#if defined(OD_FLOAT_PVQ) + double cg; + cg = cg0*OD_CGAIN_SCALE_1; + cg *= q0*OD_COMPAND_SCALE_1; + return OD_COMPAND_SCALE*cg*sqrt(cg); +#else + int32_t irt; + int64_t tmp; + int sqrt_inshift; + int sqrt_outshift; + /*cg0 is in Q(OD_CGAIN_SHIFT) and we need to divide it by + 2^OD_COMPAND_SHIFT.*/ + irt = od_sqrt(cg0*q0, &sqrt_outshift); + sqrt_inshift = (OD_CGAIN_SHIFT + OD_COMPAND_SHIFT) >> 1; + /*tmp is in Q(OD_CGAIN_SHIFT + OD_COMPAND_SHIFT).*/ + tmp = cg0*q0*(int64_t)irt; + /*Expanded gain must be in Q(OD_COMPAND_SHIFT), thus OD_COMPAND_SHIFT is + not included here.*/ + return OD_MAXI(1, + OD_VSHR_ROUND(tmp, OD_CGAIN_SHIFT + sqrt_outshift + sqrt_inshift)); +#endif + } + else { +#if defined(OD_FLOAT_PVQ) + /*Expanded gain must be in Q(OD_COMPAND_SHIFT), hence the multiply by + OD_COMPAND_SCALE.*/ + double cg; + cg = cg0*OD_CGAIN_SCALE_1; + return OD_COMPAND_SCALE*pow(cg*q0*OD_COMPAND_SCALE_1, beta); +#else + int32_t expr; + int32_t cg; + cg = OD_SHR_ROUND(cg0*q0, OD_CGAIN_SHIFT); + expr = od_pow(cg, beta); + /*Expanded gain must be in Q(OD_COMPAND_SHIFT), hence the subtraction by + OD_COMPAND_SHIFT.*/ + return OD_MAXI(1, OD_SHR_ROUND(expr, OD_EXP2_OUTSHIFT - OD_COMPAND_SHIFT)); +#endif + } +} + +/** Computes the raw and quantized/companded gain of a given input + * vector + * + * @param [in] x vector of input data + * @param [in] n number of elements in vector x + * @param [in] q0 quantizer + * @param [out] g raw gain + * @param [in] beta activity masking beta param + * @param [in] bshift shift to be applied to raw gain + * @return quantized/companded gain + */ +od_val32 od_pvq_compute_gain(const od_val16 *x, int n, int q0, od_val32 *g, + od_val16 beta, int bshift) { + int i; + od_val32 acc; +#if !defined(OD_FLOAT_PVQ) + od_val32 irt; + int sqrt_shift; +#else + OD_UNUSED(bshift); +#endif + acc = 0; + for (i = 0; i < n; i++) { + acc += x[i]*(od_val32)x[i]; + } +#if defined(OD_FLOAT_PVQ) + *g = sqrt(acc); +#else + irt = od_sqrt(acc, &sqrt_shift); + *g = OD_VSHR_ROUND(irt, sqrt_shift - bshift); +#endif + /* Normalize gain by quantization step size and apply companding + (if ACTIVITY != 1). */ + return od_gain_compand(*g, q0, beta); +} + +/** Compute theta quantization range from quantized/companded gain + * + * @param [in] qcg quantized companded gain value + * @param [in] beta activity masking beta param + * @return max theta value + */ +int od_pvq_compute_max_theta(od_val32 qcg, od_val16 beta){ + /* Set angular resolution (in ra) to match the encoded gain */ +#if defined(OD_FLOAT_PVQ) + int ts = (int)floor(.5 + qcg*OD_CGAIN_SCALE_1*M_PI/(2*beta)); +#else + int ts = OD_SHR_ROUND(qcg*OD_MULT16_16_QBETA(OD_QCONST32(M_PI/2, + OD_CGAIN_SHIFT), od_beta_rcp(beta)), OD_CGAIN_SHIFT*2); +#endif + /* Special case for low gains -- will need to be tuned anyway */ + if (qcg < OD_QCONST32(1.4, OD_CGAIN_SHIFT)) ts = 1; + return ts; +} + +/** Decode quantized theta value from coded value + * + * @param [in] t quantized companded gain value + * @param [in] max_theta maximum theta value + * @return decoded theta value + */ +od_val32 od_pvq_compute_theta(int t, int max_theta) { + if (max_theta != 0) { +#if defined(OD_FLOAT_PVQ) + return OD_MINI(t, max_theta - 1)*.5*M_PI/max_theta; +#else + return (OD_MAX_THETA_SCALE*OD_MINI(t, max_theta - 1) + + (max_theta >> 1))/max_theta; +#endif + } + else return 0; +} + +#define OD_SQRT_TBL_SHIFT (10) + +#define OD_ITHETA_SHIFT 15 +/** Compute the number of pulses used for PVQ encoding a vector from + * available metrics (encode and decode side) + * + * @param [in] qcg quantized companded gain value + * @param [in] itheta quantized PVQ error angle theta + * @param [in] noref indicates present or lack of reference + * (prediction) + * @param [in] n number of elements to be coded + * @param [in] beta activity masking beta param + * @return number of pulses to use for coding + */ +int od_pvq_compute_k(od_val32 qcg, int itheta, int noref, int n, + od_val16 beta) { +#if !defined(OD_FLOAT_PVQ) + /*Lookup table for sqrt(n+3/2) and sqrt(n+2/2) in Q10. + Real max values are 32792 and 32784, but clamped to stay within 16 bits. + Update with tools/gen_sqrt_tbl if needed.*/ + static const od_val16 od_sqrt_table[2][13] = { + {0, 0, 0, 0, 2290, 2985, 4222, 0, 8256, 0, 16416, 0, 32767}, + {0, 0, 0, 0, 2401, 3072, 4284, 0, 8287, 0, 16432, 0, 32767}}; +#endif + if (noref) { + if (qcg == 0) return 0; + if (n == 15 && qcg == OD_CGAIN_SCALE && beta > OD_BETA(1.25)) { + return 1; + } + else { +#if defined(OD_FLOAT_PVQ) + return OD_MAXI(1, (int)floor(.5 + (qcg*OD_CGAIN_SCALE_1 - .2)* + sqrt((n + 3)/2)/beta)); +#else + od_val16 rt; + OD_ASSERT(OD_ILOG(n + 1) < 13); + rt = od_sqrt_table[1][OD_ILOG(n + 1)]; + /*FIXME: get rid of 64-bit mul.*/ + return OD_MAXI(1, OD_SHR_ROUND((int64_t)((qcg + - (int64_t)OD_QCONST32(.2, OD_CGAIN_SHIFT))* + OD_MULT16_16_QBETA(od_beta_rcp(beta), rt)), OD_CGAIN_SHIFT + + OD_SQRT_TBL_SHIFT)); +#endif + } + } + else { + if (itheta == 0) return 0; + /* Sets K according to gain and theta, based on the high-rate + PVQ distortion curves (see PVQ document). Low-rate will have to be + perceptually tuned anyway. We subtract 0.2 from the radius as an + approximation for the fact that the coefficients aren't identically + distributed within a band so at low gain the number of dimensions that + are likely to have a pulse is less than n. */ +#if defined(OD_FLOAT_PVQ) + return OD_MAXI(1, (int)floor(.5 + (itheta - .2)*sqrt((n + 2)/2))); +#else + od_val16 rt; + OD_ASSERT(OD_ILOG(n + 1) < 13); + rt = od_sqrt_table[0][OD_ILOG(n + 1)]; + /*FIXME: get rid of 64-bit mul.*/ + return OD_MAXI(1, OD_VSHR_ROUND(((OD_SHL(itheta, OD_ITHETA_SHIFT) + - OD_QCONST32(.2, OD_ITHETA_SHIFT)))*(int64_t)rt, + OD_SQRT_TBL_SHIFT + OD_ITHETA_SHIFT)); +#endif + } +} + +#if !defined(OD_FLOAT_PVQ) +#define OD_RSQRT_INSHIFT 16 +#define OD_RSQRT_OUTSHIFT 14 +/** Reciprocal sqrt approximation where the input is in the range [0.25,1) in + Q16 and the output is in the range (1.0, 2.0] in Q14). + Error is always within +/1 of round(1/sqrt(t))*/ +static int16_t od_rsqrt_norm(int16_t t) +{ + int16_t n; + int32_t r; + int32_t r2; + int32_t ry; + int32_t y; + int32_t ret; + /* Range of n is [-16384,32767] ([-0.5,1) in Q15).*/ + n = t - 32768; + OD_ASSERT(n >= -16384); + /*Get a rough initial guess for the root. + The optimal minimax quadratic approximation (using relative error) is + r = 1.437799046117536+n*(-0.823394375837328+n*0.4096419668459485). + Coefficients here, and the final result r, are Q14.*/ + r = (23565 + OD_MULT16_16_Q15(n, (-13481 + OD_MULT16_16_Q15(n, 6711)))); + /*We want y = t*r*r-1 in Q15, but t is 32-bit Q16 and r is Q14. + We can compute the result from n and r using Q15 multiplies with some + adjustment, carefully done to avoid overflow.*/ + r2 = r*r; + y = (((r2 >> 15)*n + r2) >> 12) - 131077; + ry = r*y; + /*Apply a 2nd-order Householder iteration: r += r*y*(y*0.375-0.5). + This yields the Q14 reciprocal square root of the Q16 t, with a maximum + relative error of 1.04956E-4, a (relative) RMSE of 2.80979E-5, and a peak + absolute error of 2.26591/16384.*/ + ret = r + ((((ry >> 16)*(3*y) >> 3) - ry) >> 18); + OD_ASSERT(ret >= 16384 && ret < 32768); + return (int16_t)ret; +} + +static int16_t od_rsqrt(int32_t x, int *rsqrt_shift) +{ + int k; + int s; + int16_t t; + k = (OD_ILOG(x) - 1) >> 1; + /*t is x in the range [0.25, 1) in QINSHIFT, or x*2^(-s). + Shift by log2(x) - log2(0.25*(1 << INSHIFT)) to ensure 0.25 lower bound.*/ + s = 2*k - (OD_RSQRT_INSHIFT - 2); + t = OD_VSHR(x, s); + /*We want to express od_rsqrt() in terms of od_rsqrt_norm(), which is + defined as (2^OUTSHIFT)/sqrt(t*(2^-INSHIFT)) with t=x*(2^-s). + This simplifies to 2^(OUTSHIFT+(INSHIFT/2)+(s/2))/sqrt(x), so the caller + needs to shift right by OUTSHIFT + INSHIFT/2 + s/2.*/ + *rsqrt_shift = OD_RSQRT_OUTSHIFT + ((s + OD_RSQRT_INSHIFT) >> 1); + return od_rsqrt_norm(t); +} +#endif + +/** Synthesizes one parition of coefficient values from a PVQ-encoded + * vector. This 'partial' version is called by the encode loop where + * the Householder reflection has already been computed and there's no + * need to recompute it. + * + * @param [out] xcoeff output coefficient partition (x in math doc) + * @param [in] ypulse PVQ-encoded values (y in the math doc); in + * the noref case, this vector has n entries, + * in the reference case it contains n-1 entries + * (the m-th entry is not included) + * @param [in] r reference vector (prediction) + * @param [in] n number of elements in this partition + * @param [in] noref indicates presence or lack of prediction + * @param [in] g decoded quantized vector gain + * @param [in] theta decoded theta (prediction error) + * @param [in] m alignment dimension of Householder reflection + * @param [in] s sign of Householder reflection + * @param [in] qm_inv inverse of the QM with magnitude compensation + */ +void od_pvq_synthesis_partial(od_coeff *xcoeff, const od_coeff *ypulse, + const od_val16 *r16, int n, int noref, od_val32 g, od_val32 theta, int m, int s, + const int16_t *qm_inv) { + int i; + int yy; + od_val32 scale; + int nn; +#if !defined(OD_FLOAT_PVQ) + int gshift; + int qshift; +#endif + OD_ASSERT(g != 0); + nn = n-(!noref); /* when noref==0, vector in is sized n-1 */ + yy = 0; + for (i = 0; i < nn; i++) + yy += ypulse[i]*(int32_t)ypulse[i]; +#if !defined(OD_FLOAT_PVQ) + /* Shift required for the magnitude of the pre-qm synthesis to be guaranteed + to fit in 16 bits. In practice, the range will be 8192-16384 after scaling + most of the time. */ + gshift = OD_MAXI(0, OD_ILOG(g) - 14); +#endif + /*scale is g/sqrt(yy) in Q(16-gshift) so that x[]*scale has a norm that fits + in 16 bits.*/ + if (yy == 0) scale = 0; +#if defined(OD_FLOAT_PVQ) + else { + scale = g/sqrt(yy); + } +#else + else { + int rsqrt_shift; + int16_t rsqrt; + /*FIXME: should be < int64_t*/ + int64_t tmp; + rsqrt = od_rsqrt(yy, &rsqrt_shift); + tmp = rsqrt*(int64_t)g; + scale = OD_VSHR_ROUND(tmp, rsqrt_shift + gshift - 16); + } + /* Shift to apply after multiplying by the inverse QM, taking into account + gshift. */ + qshift = OD_QM_INV_SHIFT - gshift; +#endif + if (noref) { + for (i = 0; i < n; i++) { + od_val32 x; + /* This multiply doesn't round, so it introduces some bias. + It would be nice (but not critical) to fix this. */ + x = OD_MULT16_32_Q16(ypulse[i], scale); +#if defined(OD_FLOAT_PVQ) + xcoeff[i] = (od_coeff)floor(.5 + + x*(qm_inv[i]*OD_QM_INV_SCALE_1)); +#else + xcoeff[i] = OD_SHR_ROUND(x*qm_inv[i], qshift); +#endif + } + } + else{ + od_val16 x[MAXN]; + scale = OD_ROUND32(scale*OD_TRIG_SCALE_1*od_pvq_sin(theta)); + /* The following multiply doesn't round, but it's probably OK since + the Householder reflection is likely to undo most of the resulting + bias. */ + for (i = 0; i < m; i++) + x[i] = OD_MULT16_32_Q16(ypulse[i], scale); + x[m] = OD_ROUND16(-s*(OD_SHR_ROUND(g, gshift))*OD_TRIG_SCALE_1* + od_pvq_cos(theta)); + for (i = m; i < nn; i++) + x[i+1] = OD_MULT16_32_Q16(ypulse[i], scale); + od_apply_householder(x, x, r16, n); + for (i = 0; i < n; i++) { +#if defined(OD_FLOAT_PVQ) + xcoeff[i] = (od_coeff)floor(.5 + (x[i]*(qm_inv[i]*OD_QM_INV_SCALE_1))); +#else + xcoeff[i] = OD_SHR_ROUND(x[i]*qm_inv[i], qshift); +#endif + } + } +} diff --git a/third_party/aom/av1/common/pvq.h b/third_party/aom/av1/common/pvq.h new file mode 100644 index 0000000000..17e54d4c5a --- /dev/null +++ b/third_party/aom/av1/common/pvq.h @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#if !defined(_pvq_H) +# define _pvq_H (1) +# include "generic_code.h" +# include "odintrin.h" + +extern const uint16_t EXP_CDF_TABLE[][16]; +extern const uint16_t LAPLACE_OFFSET[]; + +#if CONFIG_DAALA_DIST +#define AV1_PVQ_ENABLE_ACTIVITY_MASKING (1) +#else +#define AV1_PVQ_ENABLE_ACTIVITY_MASKING (0) +#endif + +# define PVQ_MAX_PARTITIONS (1 + 3*(OD_TXSIZES-1)) + +# define OD_NOREF_ADAPT_SPEED (4) +/* Normalized lambda for PVQ quantizer. Since we normalize the gain by q, the + distortion is normalized by q^2 and lambda does not need the q^2 factor. + At high rate, this would be log(2)/6, but we're using a slightly more + aggressive value, closer to: + Li, Xiang, et al. "Laplace distribution based Lagrangian rate distortion + optimization for hybrid video coding." Circuits and Systems for Video + Technology, IEEE Transactions on 19.2 (2009): 193-205. + */ +# define OD_PVQ_LAMBDA (.1146) + +#define OD_PVQ_SKIP_ZERO 1 +#define OD_PVQ_SKIP_COPY 2 + +/* Maximum size for coding a PVQ band. */ +#define OD_MAX_PVQ_SIZE (1024) + +#if defined(OD_FLOAT_PVQ) +#define OD_QM_SHIFT (15) +#else +#define OD_QM_SHIFT (11) +#endif +#define OD_QM_SCALE (1 << OD_QM_SHIFT) +#if defined(OD_FLOAT_PVQ) +#define OD_QM_SCALE_1 (1./OD_QM_SCALE) +#endif +#define OD_QM_SCALE_MAX 32767 +#define OD_QM_INV_SHIFT (12) +#define OD_QM_INV_SCALE (1 << OD_QM_INV_SHIFT) +#if defined(OD_FLOAT_PVQ) +#define OD_QM_INV_SCALE_1 (1./OD_QM_INV_SCALE) +#endif +#define OD_QM_OFFSET(bs) ((((1 << 2*bs) - 1) << 2*OD_LOG_BSIZE0)/3) +#define OD_QM_STRIDE (OD_QM_OFFSET(OD_TXSIZES)) +#define OD_QM_BUFFER_SIZE (2*OD_QM_STRIDE) + +#if !defined(OD_FLOAT_PVQ) +#define OD_THETA_SHIFT (15) +#define OD_THETA_SCALE ((1 << OD_THETA_SHIFT)*2./M_PI) +#define OD_MAX_THETA_SCALE (1 << OD_THETA_SHIFT) +#define OD_TRIG_SCALE (32768) +#define OD_BETA_SHIFT (12) +#define OD_BETA_SCALE_1 (1./(1 << OD_BETA_SHIFT)) +/*Multiplies 16-bit a by 32-bit b and keeps bits [16:64-OD_BETA_SHIFT-1].*/ +#define OD_MULT16_32_QBETA(a, b) \ + ((int16_t)(a)*(int64_t)(int32_t)(b) >> OD_BETA_SHIFT) +# define OD_MULT16_16_QBETA(a, b) \ + ((((int16_t)(a))*((int32_t)(int16_t)(b))) >> OD_BETA_SHIFT) +#define OD_CGAIN_SHIFT (8) +#define OD_CGAIN_SCALE (1 << OD_CGAIN_SHIFT) +#else +#define OD_BETA_SCALE_1 (1.) +#define OD_THETA_SCALE (1) +#define OD_TRIG_SCALE (1) +#define OD_CGAIN_SCALE (1) +#endif +#define OD_THETA_SCALE_1 (1./OD_THETA_SCALE) +#define OD_TRIG_SCALE_1 (1./OD_TRIG_SCALE) +#define OD_CGAIN_SCALE_1 (1./OD_CGAIN_SCALE) +#define OD_CGAIN_SCALE_2 (OD_CGAIN_SCALE_1*OD_CGAIN_SCALE_1) + +/* Largest PVQ partition is half the coefficients of largest block size. */ +#define MAXN (OD_TXSIZE_MAX*OD_TXSIZE_MAX/2) + +#define OD_COMPAND_SHIFT (8 + OD_COEFF_SHIFT) +#define OD_COMPAND_SCALE (1 << OD_COMPAND_SHIFT) +#define OD_COMPAND_SCALE_1 (1./OD_COMPAND_SCALE) + +#define OD_QM_SIZE (OD_TXSIZES*(OD_TXSIZES + 1)) + +#define OD_FLAT_QM 0 +#define OD_HVS_QM 1 + +# define OD_NSB_ADAPT_CTXS (4) + +# define OD_ADAPT_K_Q8 0 +# define OD_ADAPT_SUM_EX_Q8 1 +# define OD_ADAPT_COUNT_Q8 2 +# define OD_ADAPT_COUNT_EX_Q8 3 + +# define OD_ADAPT_NO_VALUE (-2147483647-1) + +typedef enum { + PVQ_SKIP = 0x0, + DC_CODED = 0x1, + AC_CODED = 0x2, + AC_DC_CODED = 0x3, +} PVQ_SKIP_TYPE; + +typedef struct od_pvq_adapt_ctx od_pvq_adapt_ctx; +typedef struct od_pvq_codeword_ctx od_pvq_codeword_ctx; + +struct od_pvq_codeword_ctx { + int pvq_adapt[2*OD_TXSIZES*OD_NSB_ADAPT_CTXS]; + /* CDFs are size 16 despite the fact that we're using less than that. */ + uint16_t pvq_k1_cdf[12][CDF_SIZE(16)]; + uint16_t pvq_split_cdf[22*7][CDF_SIZE(8)]; +}; + +struct od_pvq_adapt_ctx { + od_pvq_codeword_ctx pvq_codeword_ctx; + generic_encoder pvq_param_model[3]; + int pvq_ext[OD_TXSIZES*PVQ_MAX_PARTITIONS]; + int pvq_exg[OD_NPLANES_MAX][OD_TXSIZES][PVQ_MAX_PARTITIONS]; + uint16_t pvq_gaintheta_cdf[2*OD_TXSIZES*PVQ_MAX_PARTITIONS][CDF_SIZE(16)]; + uint16_t pvq_skip_dir_cdf[2*(OD_TXSIZES-1)][CDF_SIZE(7)]; +}; + +typedef struct od_qm_entry { + int interp_q; + int scale_q8; + const unsigned char *qm_q4; +} od_qm_entry; + +extern const od_qm_entry OD_DEFAULT_QMS[2][2][OD_NPLANES_MAX]; + +void od_adapt_pvq_ctx_reset(od_pvq_adapt_ctx *state, int is_keyframe); +int od_pvq_size_ctx(int n); +int od_pvq_k1_ctx(int n, int orig_size); + +od_val16 od_pvq_sin(od_val32 x); +od_val16 od_pvq_cos(od_val32 x); +#if !defined(OD_FLOAT_PVQ) +int od_vector_log_mag(const od_coeff *x, int n); +#endif + +void od_interp_qm(unsigned char *out, int q, const od_qm_entry *entry1, + const od_qm_entry *entry2); + +int od_qm_get_index(int bs, int band); + +extern const od_val16 *const OD_PVQ_BETA[2][OD_NPLANES_MAX][OD_TXSIZES + 1]; + +void od_init_qm(int16_t *x, int16_t *x_inv, const int *qm); +int od_compute_householder(od_val16 *r, int n, od_val32 gr, int *sign, + int shift); +void od_apply_householder(od_val16 *out, const od_val16 *x, const od_val16 *r, + int n); +void od_pvq_synthesis_partial(od_coeff *xcoeff, const od_coeff *ypulse, + const od_val16 *r, int n, + int noref, od_val32 g, + od_val32 theta, int m, int s, + const int16_t *qm_inv); +od_val32 od_gain_expand(od_val32 cg, int q0, od_val16 beta); +od_val32 od_pvq_compute_gain(const od_val16 *x, int n, int q0, od_val32 *g, + od_val16 beta, int bshift); +int od_pvq_compute_max_theta(od_val32 qcg, od_val16 beta); +od_val32 od_pvq_compute_theta(int t, int max_theta); +int od_pvq_compute_k(od_val32 qcg, int itheta, int noref, int n, od_val16 beta); + +int od_vector_is_null(const od_coeff *x, int len); +int od_qm_offset(int bs, int xydec); + +#endif diff --git a/third_party/aom/av1/common/pvq_state.c b/third_party/aom/av1/common/pvq_state.c new file mode 100644 index 0000000000..197b9b3a82 --- /dev/null +++ b/third_party/aom/av1/common/pvq_state.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/common/pvq_state.h" +#include "av1/common/odintrin.h" + +void od_adapt_ctx_reset(od_adapt_ctx *adapt, int is_keyframe) { + int pli; + od_adapt_pvq_ctx_reset(&adapt->pvq, is_keyframe); + OD_CDFS_INIT_Q15(adapt->skip_cdf); + for (pli = 0; pli < OD_NPLANES_MAX; pli++) { + int i; + OD_CDFS_INIT_DYNAMIC(adapt->model_dc[pli].cdf); + for (i = 0; i < OD_TXSIZES; i++) { + int j; + adapt->ex_g[pli][i] = 8; + for (j = 0; j < 3; j++) { + adapt->ex_dc[pli][i][j] = pli > 0 ? 8 : 32768; + } + } + } +} + +void od_init_skipped_coeffs(int16_t *d, int16_t *pred, int is_keyframe, int bo, + int n, int w) { + int i; + int j; + if (is_keyframe) { + for (i = 0; i < n; i++) { + for (j = 0; j < n; j++) { + /* skip DC */ + if (i || j) d[bo + i * w + j] = 0; + } + } + } else { + for (i = 0; i < n; i++) { + for (j = 0; j < n; j++) { + d[bo + i * w + j] = pred[i * n + j]; + } + } + } +} diff --git a/third_party/aom/av1/common/pvq_state.h b/third_party/aom/av1/common/pvq_state.h new file mode 100644 index 0000000000..84d454e700 --- /dev/null +++ b/third_party/aom/av1/common/pvq_state.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#if !defined(_state_H) +# define _state_H (1) + +typedef struct od_state od_state; +typedef struct od_adapt_ctx od_adapt_ctx; + +# include "generic_code.h" +# include "odintrin.h" +# include "pvq.h" + +/*Adaptation speed of scalar Laplace encoding.*/ +# define OD_SCALAR_ADAPT_SPEED (4) + +struct od_adapt_ctx { + /* Support for PVQ encode/decode */ + od_pvq_adapt_ctx pvq; + + generic_encoder model_dc[OD_NPLANES_MAX]; + + int ex_dc[OD_NPLANES_MAX][OD_TXSIZES][3]; + int ex_g[OD_NPLANES_MAX][OD_TXSIZES]; + + /* Joint skip flag for DC and AC */ + uint16_t skip_cdf[OD_TXSIZES*2][CDF_SIZE(4)]; +}; + +struct od_state { + od_adapt_ctx *adapt; + unsigned char pvq_qm_q4[OD_NPLANES_MAX][OD_QM_SIZE]; + /* Quantization matrices and their inverses. */ + int16_t qm[OD_QM_BUFFER_SIZE]; + int16_t qm_inv[OD_QM_BUFFER_SIZE]; +}; + +void od_adapt_ctx_reset(od_adapt_ctx *state, int is_keyframe); +void od_init_skipped_coeffs(int16_t *d, int16_t *pred, int is_keyframe, + int bo, int n, int w); + +#endif diff --git a/third_party/aom/av1/common/quant_common.c b/third_party/aom/av1/common/quant_common.c new file mode 100644 index 0000000000..763465e480 --- /dev/null +++ b/third_party/aom/av1/common/quant_common.c @@ -0,0 +1,11369 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/common/common.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/entropy.h" +#include "av1/common/quant_common.h" +#include "av1/common/seg_common.h" +#include "av1/common/blockd.h" + +#if CONFIG_NEW_QUANT +// Bin widths expressed as a fraction over 128 of the quant stepsize, +// for the quantization bins 0-4. +// So a value x indicates the bin is actually factor x/128 of the +// nominal quantization step. For the zero bin, the width is only +// for one side of zero, so the actual width is twice that. +// +// Functions with nuq correspond to "non uniform quantization" +// TODO(sarahparker, debargha): Optimize these tables + +typedef struct { + uint8_t knots[NUQ_KNOTS]; // offsets + uint8_t doff; // dequantization +} qprofile_type; + +static const qprofile_type nuq[QUANT_PROFILES][COEF_BANDS] = { + { + // lossless + { { 64, 128, 128 }, 0 }, // dc, band 0 + { { 64, 128, 128 }, 0 }, // band 1 + { { 64, 128, 128 }, 0 }, // band 2 + { { 64, 128, 128 }, 0 }, // band 3 + { { 64, 128, 128 }, 0 }, // band 4 + { { 64, 128, 128 }, 0 }, // band 5 + }, + { + { { 64, 128, 128 }, 4 }, // dc, band 0 + { { 64, 128, 128 }, 6 }, // band 1 + { { 64, 128, 128 }, 8 }, // band 2 + { { 64, 128, 128 }, 10 }, // band 3 + { { 72, 128, 128 }, 12 }, // band 4 + { { 80, 128, 128 }, 14 } // band 5 + }, + { + { { 64, 128, 128 }, 6 }, // dc, band 0 + { { 64, 128, 128 }, 8 }, // band 1 + { { 64, 128, 128 }, 10 }, // band 2 + { { 64, 128, 128 }, 12 }, // band 3 + { { 72, 128, 128 }, 14 }, // band 4 + { { 80, 128, 128 }, 16 } // band 5 + }, + { + { { 64, 128, 128 }, 8 }, // dc, band 0 + { { 64, 128, 128 }, 10 }, // band 1 + { { 64, 128, 128 }, 12 }, // band 2 + { { 72, 128, 128 }, 14 }, // band 3 + { { 76, 128, 128 }, 16 }, // band 4 + { { 80, 128, 128 }, 18 } // band 5 + } +}; + +static const uint8_t *get_nuq_knots(int band, int q_profile) { + return nuq[q_profile][band].knots; +} + +static INLINE int16_t quant_to_doff_fixed(int band, int q_profile) { + return nuq[q_profile][band].doff; +} + +// get cumulative bins +static INLINE void get_cuml_bins_nuq(int q, int band, tran_low_t *cuml_bins, + int q_profile) { + const uint8_t *knots = get_nuq_knots(band, q_profile); + int16_t cuml_knots[NUQ_KNOTS]; + int i; + cuml_knots[0] = knots[0]; + for (i = 1; i < NUQ_KNOTS; ++i) cuml_knots[i] = cuml_knots[i - 1] + knots[i]; + for (i = 0; i < NUQ_KNOTS; ++i) + cuml_bins[i] = ROUND_POWER_OF_TWO(cuml_knots[i] * q, 7); +} + +void av1_get_dequant_val_nuq(int q, int band, tran_low_t *dq, + tran_low_t *cuml_bins, int q_profile) { + const uint8_t *knots = get_nuq_knots(band, q_profile); + tran_low_t cuml_bins_[NUQ_KNOTS], *cuml_bins_ptr; + tran_low_t doff; + int i; + cuml_bins_ptr = (cuml_bins ? cuml_bins : cuml_bins_); + get_cuml_bins_nuq(q, band, cuml_bins_ptr, q_profile); + dq[0] = 0; + for (i = 1; i < NUQ_KNOTS; ++i) { + doff = quant_to_doff_fixed(band, q_profile); + doff = ROUND_POWER_OF_TWO(doff * knots[i], 7); + dq[i] = + cuml_bins_ptr[i - 1] + ROUND_POWER_OF_TWO((knots[i] - doff * 2) * q, 8); + } + doff = quant_to_doff_fixed(band, q_profile); + dq[NUQ_KNOTS] = + cuml_bins_ptr[NUQ_KNOTS - 1] + ROUND_POWER_OF_TWO((64 - doff) * q, 7); +} + +tran_low_t av1_dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq) { + if (v <= NUQ_KNOTS) + return dq[v]; + else + return dq[NUQ_KNOTS] + (v - NUQ_KNOTS) * q; +} + +tran_low_t av1_dequant_coeff_nuq(int v, int q, const tran_low_t *dq) { + tran_low_t dqmag = av1_dequant_abscoeff_nuq(abs(v), q, dq); + return (v < 0 ? -dqmag : dqmag); +} +#endif // CONFIG_NEW_QUANT + +static const int16_t dc_qlookup[QINDEX_RANGE] = { + 4, 8, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, + 19, 19, 20, 21, 22, 23, 24, 25, 26, 26, 27, 28, 29, 30, + 31, 32, 32, 33, 34, 35, 36, 37, 38, 38, 39, 40, 41, 42, + 43, 43, 44, 45, 46, 47, 48, 48, 49, 50, 51, 52, 53, 53, + 54, 55, 56, 57, 57, 58, 59, 60, 61, 62, 62, 63, 64, 65, + 66, 66, 67, 68, 69, 70, 70, 71, 72, 73, 74, 74, 75, 76, + 77, 78, 78, 79, 80, 81, 81, 82, 83, 84, 85, 85, 87, 88, + 90, 92, 93, 95, 96, 98, 99, 101, 102, 104, 105, 107, 108, 110, + 111, 113, 114, 116, 117, 118, 120, 121, 123, 125, 127, 129, 131, 134, + 136, 138, 140, 142, 144, 146, 148, 150, 152, 154, 156, 158, 161, 164, + 166, 169, 172, 174, 177, 180, 182, 185, 187, 190, 192, 195, 199, 202, + 205, 208, 211, 214, 217, 220, 223, 226, 230, 233, 237, 240, 243, 247, + 250, 253, 257, 261, 265, 269, 272, 276, 280, 284, 288, 292, 296, 300, + 304, 309, 313, 317, 322, 326, 330, 335, 340, 344, 349, 354, 359, 364, + 369, 374, 379, 384, 389, 395, 400, 406, 411, 417, 423, 429, 435, 441, + 447, 454, 461, 467, 475, 482, 489, 497, 505, 513, 522, 530, 539, 549, + 559, 569, 579, 590, 602, 614, 626, 640, 654, 668, 684, 700, 717, 736, + 755, 775, 796, 819, 843, 869, 896, 925, 955, 988, 1022, 1058, 1098, 1139, + 1184, 1232, 1282, 1336, +}; + +#if CONFIG_HIGHBITDEPTH +static const int16_t dc_qlookup_10[QINDEX_RANGE] = { + 4, 9, 10, 13, 15, 17, 20, 22, 25, 28, 31, 34, 37, + 40, 43, 47, 50, 53, 57, 60, 64, 68, 71, 75, 78, 82, + 86, 90, 93, 97, 101, 105, 109, 113, 116, 120, 124, 128, 132, + 136, 140, 143, 147, 151, 155, 159, 163, 166, 170, 174, 178, 182, + 185, 189, 193, 197, 200, 204, 208, 212, 215, 219, 223, 226, 230, + 233, 237, 241, 244, 248, 251, 255, 259, 262, 266, 269, 273, 276, + 280, 283, 287, 290, 293, 297, 300, 304, 307, 310, 314, 317, 321, + 324, 327, 331, 334, 337, 343, 350, 356, 362, 369, 375, 381, 387, + 394, 400, 406, 412, 418, 424, 430, 436, 442, 448, 454, 460, 466, + 472, 478, 484, 490, 499, 507, 516, 525, 533, 542, 550, 559, 567, + 576, 584, 592, 601, 609, 617, 625, 634, 644, 655, 666, 676, 687, + 698, 708, 718, 729, 739, 749, 759, 770, 782, 795, 807, 819, 831, + 844, 856, 868, 880, 891, 906, 920, 933, 947, 961, 975, 988, 1001, + 1015, 1030, 1045, 1061, 1076, 1090, 1105, 1120, 1137, 1153, 1170, 1186, 1202, + 1218, 1236, 1253, 1271, 1288, 1306, 1323, 1342, 1361, 1379, 1398, 1416, 1436, + 1456, 1476, 1496, 1516, 1537, 1559, 1580, 1601, 1624, 1647, 1670, 1692, 1717, + 1741, 1766, 1791, 1817, 1844, 1871, 1900, 1929, 1958, 1990, 2021, 2054, 2088, + 2123, 2159, 2197, 2236, 2276, 2319, 2363, 2410, 2458, 2508, 2561, 2616, 2675, + 2737, 2802, 2871, 2944, 3020, 3102, 3188, 3280, 3375, 3478, 3586, 3702, 3823, + 3953, 4089, 4236, 4394, 4559, 4737, 4929, 5130, 5347, +}; + +static const int16_t dc_qlookup_12[QINDEX_RANGE] = { + 4, 12, 18, 25, 33, 41, 50, 60, 70, 80, 91, + 103, 115, 127, 140, 153, 166, 180, 194, 208, 222, 237, + 251, 266, 281, 296, 312, 327, 343, 358, 374, 390, 405, + 421, 437, 453, 469, 484, 500, 516, 532, 548, 564, 580, + 596, 611, 627, 643, 659, 674, 690, 706, 721, 737, 752, + 768, 783, 798, 814, 829, 844, 859, 874, 889, 904, 919, + 934, 949, 964, 978, 993, 1008, 1022, 1037, 1051, 1065, 1080, + 1094, 1108, 1122, 1136, 1151, 1165, 1179, 1192, 1206, 1220, 1234, + 1248, 1261, 1275, 1288, 1302, 1315, 1329, 1342, 1368, 1393, 1419, + 1444, 1469, 1494, 1519, 1544, 1569, 1594, 1618, 1643, 1668, 1692, + 1717, 1741, 1765, 1789, 1814, 1838, 1862, 1885, 1909, 1933, 1957, + 1992, 2027, 2061, 2096, 2130, 2165, 2199, 2233, 2267, 2300, 2334, + 2367, 2400, 2434, 2467, 2499, 2532, 2575, 2618, 2661, 2704, 2746, + 2788, 2830, 2872, 2913, 2954, 2995, 3036, 3076, 3127, 3177, 3226, + 3275, 3324, 3373, 3421, 3469, 3517, 3565, 3621, 3677, 3733, 3788, + 3843, 3897, 3951, 4005, 4058, 4119, 4181, 4241, 4301, 4361, 4420, + 4479, 4546, 4612, 4677, 4742, 4807, 4871, 4942, 5013, 5083, 5153, + 5222, 5291, 5367, 5442, 5517, 5591, 5665, 5745, 5825, 5905, 5984, + 6063, 6149, 6234, 6319, 6404, 6495, 6587, 6678, 6769, 6867, 6966, + 7064, 7163, 7269, 7376, 7483, 7599, 7715, 7832, 7958, 8085, 8214, + 8352, 8492, 8635, 8788, 8945, 9104, 9275, 9450, 9639, 9832, 10031, + 10245, 10465, 10702, 10946, 11210, 11482, 11776, 12081, 12409, 12750, 13118, + 13501, 13913, 14343, 14807, 15290, 15812, 16356, 16943, 17575, 18237, 18949, + 19718, 20521, 21387, +}; +#endif + +static const int16_t ac_qlookup[QINDEX_RANGE] = { + 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, + 98, 99, 100, 101, 102, 104, 106, 108, 110, 112, 114, 116, 118, + 120, 122, 124, 126, 128, 130, 132, 134, 136, 138, 140, 142, 144, + 146, 148, 150, 152, 155, 158, 161, 164, 167, 170, 173, 176, 179, + 182, 185, 188, 191, 194, 197, 200, 203, 207, 211, 215, 219, 223, + 227, 231, 235, 239, 243, 247, 251, 255, 260, 265, 270, 275, 280, + 285, 290, 295, 300, 305, 311, 317, 323, 329, 335, 341, 347, 353, + 359, 366, 373, 380, 387, 394, 401, 408, 416, 424, 432, 440, 448, + 456, 465, 474, 483, 492, 501, 510, 520, 530, 540, 550, 560, 571, + 582, 593, 604, 615, 627, 639, 651, 663, 676, 689, 702, 715, 729, + 743, 757, 771, 786, 801, 816, 832, 848, 864, 881, 898, 915, 933, + 951, 969, 988, 1007, 1026, 1046, 1066, 1087, 1108, 1129, 1151, 1173, 1196, + 1219, 1243, 1267, 1292, 1317, 1343, 1369, 1396, 1423, 1451, 1479, 1508, 1537, + 1567, 1597, 1628, 1660, 1692, 1725, 1759, 1793, 1828, +}; + +#if CONFIG_HIGHBITDEPTH +static const int16_t ac_qlookup_10[QINDEX_RANGE] = { + 4, 9, 11, 13, 16, 18, 21, 24, 27, 30, 33, 37, 40, + 44, 48, 51, 55, 59, 63, 67, 71, 75, 79, 83, 88, 92, + 96, 100, 105, 109, 114, 118, 122, 127, 131, 136, 140, 145, 149, + 154, 158, 163, 168, 172, 177, 181, 186, 190, 195, 199, 204, 208, + 213, 217, 222, 226, 231, 235, 240, 244, 249, 253, 258, 262, 267, + 271, 275, 280, 284, 289, 293, 297, 302, 306, 311, 315, 319, 324, + 328, 332, 337, 341, 345, 349, 354, 358, 362, 367, 371, 375, 379, + 384, 388, 392, 396, 401, 409, 417, 425, 433, 441, 449, 458, 466, + 474, 482, 490, 498, 506, 514, 523, 531, 539, 547, 555, 563, 571, + 579, 588, 596, 604, 616, 628, 640, 652, 664, 676, 688, 700, 713, + 725, 737, 749, 761, 773, 785, 797, 809, 825, 841, 857, 873, 889, + 905, 922, 938, 954, 970, 986, 1002, 1018, 1038, 1058, 1078, 1098, 1118, + 1138, 1158, 1178, 1198, 1218, 1242, 1266, 1290, 1314, 1338, 1362, 1386, 1411, + 1435, 1463, 1491, 1519, 1547, 1575, 1603, 1631, 1663, 1695, 1727, 1759, 1791, + 1823, 1859, 1895, 1931, 1967, 2003, 2039, 2079, 2119, 2159, 2199, 2239, 2283, + 2327, 2371, 2415, 2459, 2507, 2555, 2603, 2651, 2703, 2755, 2807, 2859, 2915, + 2971, 3027, 3083, 3143, 3203, 3263, 3327, 3391, 3455, 3523, 3591, 3659, 3731, + 3803, 3876, 3952, 4028, 4104, 4184, 4264, 4348, 4432, 4516, 4604, 4692, 4784, + 4876, 4972, 5068, 5168, 5268, 5372, 5476, 5584, 5692, 5804, 5916, 6032, 6148, + 6268, 6388, 6512, 6640, 6768, 6900, 7036, 7172, 7312, +}; + +static const int16_t ac_qlookup_12[QINDEX_RANGE] = { + 4, 13, 19, 27, 35, 44, 54, 64, 75, 87, 99, + 112, 126, 139, 154, 168, 183, 199, 214, 230, 247, 263, + 280, 297, 314, 331, 349, 366, 384, 402, 420, 438, 456, + 475, 493, 511, 530, 548, 567, 586, 604, 623, 642, 660, + 679, 698, 716, 735, 753, 772, 791, 809, 828, 846, 865, + 884, 902, 920, 939, 957, 976, 994, 1012, 1030, 1049, 1067, + 1085, 1103, 1121, 1139, 1157, 1175, 1193, 1211, 1229, 1246, 1264, + 1282, 1299, 1317, 1335, 1352, 1370, 1387, 1405, 1422, 1440, 1457, + 1474, 1491, 1509, 1526, 1543, 1560, 1577, 1595, 1627, 1660, 1693, + 1725, 1758, 1791, 1824, 1856, 1889, 1922, 1954, 1987, 2020, 2052, + 2085, 2118, 2150, 2183, 2216, 2248, 2281, 2313, 2346, 2378, 2411, + 2459, 2508, 2556, 2605, 2653, 2701, 2750, 2798, 2847, 2895, 2943, + 2992, 3040, 3088, 3137, 3185, 3234, 3298, 3362, 3426, 3491, 3555, + 3619, 3684, 3748, 3812, 3876, 3941, 4005, 4069, 4149, 4230, 4310, + 4390, 4470, 4550, 4631, 4711, 4791, 4871, 4967, 5064, 5160, 5256, + 5352, 5448, 5544, 5641, 5737, 5849, 5961, 6073, 6185, 6297, 6410, + 6522, 6650, 6778, 6906, 7034, 7162, 7290, 7435, 7579, 7723, 7867, + 8011, 8155, 8315, 8475, 8635, 8795, 8956, 9132, 9308, 9484, 9660, + 9836, 10028, 10220, 10412, 10604, 10812, 11020, 11228, 11437, 11661, 11885, + 12109, 12333, 12573, 12813, 13053, 13309, 13565, 13821, 14093, 14365, 14637, + 14925, 15213, 15502, 15806, 16110, 16414, 16734, 17054, 17390, 17726, 18062, + 18414, 18766, 19134, 19502, 19886, 20270, 20670, 21070, 21486, 21902, 22334, + 22766, 23214, 23662, 24126, 24590, 25070, 25551, 26047, 26559, 27071, 27599, + 28143, 28687, 29247, +}; +#endif + +int16_t av1_dc_quant(int qindex, int delta, aom_bit_depth_t bit_depth) { +#if CONFIG_HIGHBITDEPTH + switch (bit_depth) { + case AOM_BITS_8: return dc_qlookup[clamp(qindex + delta, 0, MAXQ)]; + case AOM_BITS_10: return dc_qlookup_10[clamp(qindex + delta, 0, MAXQ)]; + case AOM_BITS_12: return dc_qlookup_12[clamp(qindex + delta, 0, MAXQ)]; + default: + assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12"); + return -1; + } +#else + (void)bit_depth; + return dc_qlookup[clamp(qindex + delta, 0, MAXQ)]; +#endif +} + +int16_t av1_ac_quant(int qindex, int delta, aom_bit_depth_t bit_depth) { +#if CONFIG_HIGHBITDEPTH + switch (bit_depth) { + case AOM_BITS_8: return ac_qlookup[clamp(qindex + delta, 0, MAXQ)]; + case AOM_BITS_10: return ac_qlookup_10[clamp(qindex + delta, 0, MAXQ)]; + case AOM_BITS_12: return ac_qlookup_12[clamp(qindex + delta, 0, MAXQ)]; + default: + assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12"); + return -1; + } +#else + (void)bit_depth; + return ac_qlookup[clamp(qindex + delta, 0, MAXQ)]; +#endif +} + +int16_t av1_qindex_from_ac(int ac, aom_bit_depth_t bit_depth) { + int i; + const int16_t *tab = ac_qlookup; + ac *= 4; +#if CONFIG_HIGHBITDEPTH + switch (bit_depth) { + case AOM_BITS_10: { + tab = ac_qlookup_10; + ac *= 4; + break; + } + case AOM_BITS_12: { + tab = ac_qlookup_12; + ac *= 16; + break; + } + default: + assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12"); + return -1; + } +#endif + (void)bit_depth; + for (i = 0; i < QINDEX_RANGE; i++) { + if (ac <= tab[i]) return i; + } + return QINDEX_RANGE - 1; +} + +int av1_get_qindex(const struct segmentation *seg, int segment_id, + int base_qindex) { + if (segfeature_active(seg, segment_id, SEG_LVL_ALT_Q)) { + const int data = get_segdata(seg, segment_id, SEG_LVL_ALT_Q); + const int seg_qindex = + seg->abs_delta == SEGMENT_ABSDATA ? data : base_qindex + data; + return clamp(seg_qindex, 0, MAXQ); + } else { + return base_qindex; + } +} + +#if CONFIG_AOM_QM +qm_val_t *aom_iqmatrix(AV1_COMMON *cm, int qmlevel, int is_chroma, + int log2sizem2, int is_intra) { + return &cm->giqmatrix[qmlevel][!!is_chroma][!!is_intra][log2sizem2][0]; +} +qm_val_t *aom_qmatrix(AV1_COMMON *cm, int qmlevel, int is_chroma, + int log2sizem2, int is_intra) { + return &cm->gqmatrix[qmlevel][!!is_chroma][!!is_intra][log2sizem2][0]; +} + +static uint16_t iwt_matrix_ref[NUM_QM_LEVELS][2][2] + [4 * 4 + 8 * 8 + 16 * 16 + 32 * 32]; +static uint16_t wt_matrix_ref[NUM_QM_LEVELS][2][2] + [4 * 4 + 8 * 8 + 16 * 16 + 32 * 32]; + +void aom_qm_init(AV1_COMMON *cm) { + int q, c, f, t, size; + int current; + for (q = 0; q < NUM_QM_LEVELS; ++q) { + for (c = 0; c < 2; ++c) { + for (f = 0; f < 2; ++f) { + current = 0; + for (t = 0; t < TX_SIZES; ++t) { + size = 1 << (t + 2); + cm->gqmatrix[q][c][f][t] = &wt_matrix_ref[q][c][f][current]; + cm->giqmatrix[q][c][f][t] = &iwt_matrix_ref[q][c][f][current]; + current += size * size; + } + } + } + } +} + +static uint16_t iwt_matrix_ref[NUM_QM_LEVELS][2][2][4 * 4 + 8 * 8 + 16 * 16 + + 32 * 32] = { + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 71, 124, 214, 71, 112, 165, 241, 124, 165, 254, 331, 214, 241, 331, + 414, + /* Size 8 */ + 64, 47, 51, 69, 97, 132, 173, 218, 47, 54, 52, 62, 81, 109, 142, 181, + 51, 52, 75, 90, 108, 133, 165, 201, 69, 62, 90, 119, 144, 169, 198, 232, + 97, 81, 108, 144, 178, 208, 238, 268, 132, 109, 133, 169, 208, 244, 276, + 305, 173, 142, 165, 198, 238, 276, 309, 338, 218, 181, 201, 232, 268, + 305, 338, 367, + /* Size 16 */ + 64, 54, 47, 49, 51, 59, 69, 81, 97, 111, 132, 150, 173, 193, 218, 218, + 54, 52, 50, 51, 51, 58, 65, 75, 88, 101, 119, 135, 156, 175, 198, 198, + 47, 50, 54, 53, 52, 56, 62, 70, 81, 93, 109, 123, 142, 159, 181, 181, + 49, 51, 53, 57, 61, 67, 73, 82, 93, 104, 120, 134, 153, 170, 191, 191, + 51, 51, 52, 61, 75, 82, 90, 98, 108, 119, 133, 147, 165, 181, 201, 201, + 59, 58, 56, 67, 82, 91, 102, 112, 123, 135, 149, 163, 180, 196, 215, + 215, 69, 65, 62, 73, 90, 102, 119, 130, 144, 155, 169, 182, 198, 214, + 232, 232, 81, 75, 70, 82, 98, 112, 130, 143, 159, 172, 186, 200, 216, + 231, 249, 249, 97, 88, 81, 93, 108, 123, 144, 159, 178, 192, 208, 222, + 238, 252, 268, 268, 111, 101, 93, 104, 119, 135, 155, 172, 192, 207, + 225, 239, 255, 269, 285, 285, 132, 119, 109, 120, 133, 149, 169, 186, + 208, 225, 244, 259, 276, 290, 305, 305, 150, 135, 123, 134, 147, 163, + 182, 200, 222, 239, 259, 274, 291, 305, 321, 321, 173, 156, 142, 153, + 165, 180, 198, 216, 238, 255, 276, 291, 309, 323, 338, 338, 193, 175, + 159, 170, 181, 196, 214, 231, 252, 269, 290, 305, 323, 337, 352, 352, + 218, 198, 181, 191, 201, 215, 232, 249, 268, 285, 305, 321, 338, 352, + 367, 367, 218, 198, 181, 191, 201, 215, 232, 249, 268, 285, 305, 321, + 338, 352, 367, 367, + /* Size 32 */ + 64, 59, 54, 50, 47, 48, 49, 50, 51, 55, 59, 63, 69, 74, 81, 88, 97, 104, + 111, 121, 132, 140, 150, 161, 173, 183, 193, 205, 218, 218, 218, 218, + 59, 56, 53, 51, 49, 49, 50, 51, 51, 54, 58, 62, 67, 72, 78, 84, 92, 99, + 106, 115, 125, 133, 142, 152, 164, 173, 183, 195, 208, 208, 208, 208, + 54, 53, 52, 51, 50, 51, 51, 51, 51, 54, 58, 61, 65, 70, 75, 81, 88, 94, + 101, 110, 119, 127, 135, 145, 156, 165, 175, 186, 198, 198, 198, 198, + 50, 51, 51, 52, 52, 52, 52, 52, 52, 54, 57, 60, 63, 68, 72, 78, 85, 90, + 97, 105, 114, 121, 129, 138, 149, 157, 167, 177, 189, 189, 189, 189, 47, + 49, 50, 52, 54, 54, 53, 52, 52, 54, 56, 59, 62, 66, 70, 75, 81, 87, 93, + 100, 109, 115, 123, 132, 142, 150, 159, 170, 181, 181, 181, 181, 48, 49, + 51, 52, 54, 54, 55, 56, 56, 59, 61, 64, 67, 71, 76, 81, 87, 92, 98, 105, + 114, 121, 128, 137, 147, 155, 164, 174, 186, 186, 186, 186, 49, 50, 51, + 52, 53, 55, 57, 59, 61, 64, 67, 70, 73, 77, 82, 87, 93, 98, 104, 111, + 120, 126, 134, 143, 153, 161, 170, 179, 191, 191, 191, 191, 50, 51, 51, + 52, 52, 56, 59, 63, 68, 71, 74, 77, 81, 85, 89, 94, 100, 105, 111, 118, + 126, 133, 140, 149, 158, 166, 175, 185, 196, 196, 196, 196, 51, 51, 51, + 52, 52, 56, 61, 68, 75, 79, 82, 86, 90, 94, 98, 103, 108, 113, 119, 126, + 133, 140, 147, 155, 165, 172, 181, 191, 201, 201, 201, 201, 55, 54, 54, + 54, 54, 59, 64, 71, 79, 82, 86, 91, 96, 100, 105, 110, 115, 120, 126, + 133, 140, 147, 155, 163, 172, 180, 188, 198, 208, 208, 208, 208, 59, 58, + 58, 57, 56, 61, 67, 74, 82, 86, 91, 96, 102, 107, 112, 117, 123, 129, + 135, 141, 149, 156, 163, 171, 180, 188, 196, 205, 215, 215, 215, 215, + 63, 62, 61, 60, 59, 64, 70, 77, 86, 91, 96, 103, 110, 115, 120, 126, + 133, 138, 144, 151, 158, 165, 172, 180, 189, 196, 204, 213, 223, 223, + 223, 223, 69, 67, 65, 63, 62, 67, 73, 81, 90, 96, 102, 110, 119, 124, + 130, 137, 144, 149, 155, 162, 169, 175, 182, 190, 198, 206, 214, 222, + 232, 232, 232, 232, 74, 72, 70, 68, 66, 71, 77, 85, 94, 100, 107, 115, + 124, 130, 136, 143, 151, 157, 163, 170, 177, 184, 191, 199, 207, 214, + 222, 231, 240, 240, 240, 240, 81, 78, 75, 72, 70, 76, 82, 89, 98, 105, + 112, 120, 130, 136, 143, 151, 159, 165, 172, 179, 186, 193, 200, 208, + 216, 223, 231, 240, 249, 249, 249, 249, 88, 84, 81, 78, 75, 81, 87, 94, + 103, 110, 117, 126, 137, 143, 151, 159, 168, 174, 181, 189, 197, 203, + 211, 218, 226, 234, 241, 249, 258, 258, 258, 258, 97, 92, 88, 85, 81, + 87, 93, 100, 108, 115, 123, 133, 144, 151, 159, 168, 178, 184, 192, 200, + 208, 215, 222, 229, 238, 245, 252, 260, 268, 268, 268, 268, 104, 99, 94, + 90, 87, 92, 98, 105, 113, 120, 129, 138, 149, 157, 165, 174, 184, 191, + 199, 207, 216, 223, 230, 238, 246, 253, 260, 268, 276, 276, 276, 276, + 111, 106, 101, 97, 93, 98, 104, 111, 119, 126, 135, 144, 155, 163, 172, + 181, 192, 199, 207, 215, 225, 232, 239, 247, 255, 262, 269, 277, 285, + 285, 285, 285, 121, 115, 110, 105, 100, 105, 111, 118, 126, 133, 141, + 151, 162, 170, 179, 189, 200, 207, 215, 224, 234, 241, 248, 256, 265, + 272, 279, 287, 295, 295, 295, 295, 132, 125, 119, 114, 109, 114, 120, + 126, 133, 140, 149, 158, 169, 177, 186, 197, 208, 216, 225, 234, 244, + 251, 259, 267, 276, 282, 290, 297, 305, 305, 305, 305, 140, 133, 127, + 121, 115, 121, 126, 133, 140, 147, 156, 165, 175, 184, 193, 203, 215, + 223, 232, 241, 251, 258, 266, 275, 283, 290, 297, 305, 313, 313, 313, + 313, 150, 142, 135, 129, 123, 128, 134, 140, 147, 155, 163, 172, 182, + 191, 200, 211, 222, 230, 239, 248, 259, 266, 274, 283, 291, 298, 305, + 313, 321, 321, 321, 321, 161, 152, 145, 138, 132, 137, 143, 149, 155, + 163, 171, 180, 190, 199, 208, 218, 229, 238, 247, 256, 267, 275, 283, + 291, 300, 307, 314, 322, 329, 329, 329, 329, 173, 164, 156, 149, 142, + 147, 153, 158, 165, 172, 180, 189, 198, 207, 216, 226, 238, 246, 255, + 265, 276, 283, 291, 300, 309, 316, 323, 331, 338, 338, 338, 338, 183, + 173, 165, 157, 150, 155, 161, 166, 172, 180, 188, 196, 206, 214, 223, + 234, 245, 253, 262, 272, 282, 290, 298, 307, 316, 323, 330, 337, 345, + 345, 345, 345, 193, 183, 175, 167, 159, 164, 170, 175, 181, 188, 196, + 204, 214, 222, 231, 241, 252, 260, 269, 279, 290, 297, 305, 314, 323, + 330, 337, 345, 352, 352, 352, 352, 205, 195, 186, 177, 170, 174, 179, + 185, 191, 198, 205, 213, 222, 231, 240, 249, 260, 268, 277, 287, 297, + 305, 313, 322, 331, 337, 345, 352, 360, 360, 360, 360, 218, 208, 198, + 189, 181, 186, 191, 196, 201, 208, 215, 223, 232, 240, 249, 258, 268, + 276, 285, 295, 305, 313, 321, 329, 338, 345, 352, 360, 367, 367, 367, + 367, 218, 208, 198, 189, 181, 186, 191, 196, 201, 208, 215, 223, 232, + 240, 249, 258, 268, 276, 285, 295, 305, 313, 321, 329, 338, 345, 352, + 360, 367, 367, 367, 367, 218, 208, 198, 189, 181, 186, 191, 196, 201, + 208, 215, 223, 232, 240, 249, 258, 268, 276, 285, 295, 305, 313, 321, + 329, 338, 345, 352, 360, 367, 367, 367, 367, 218, 208, 198, 189, 181, + 186, 191, 196, 201, 208, 215, 223, 232, 240, 249, 258, 268, 276, 285, + 295, 305, 313, 321, 329, 338, 345, 352, 360, 367, 367, 367, 367 }, + { /* Intra matrices */ + /* Size 4 */ + 16, 18, 33, 60, 18, 29, 45, 68, 33, 45, 72, 98, 60, 68, 98, 129, + /* Size 8 */ + 20, 14, 16, 21, 31, 43, 58, 75, 14, 17, 16, 19, 25, 35, 46, 61, 16, 16, + 24, 28, 34, 43, 54, 68, 21, 19, 28, 38, 47, 56, 67, 80, 31, 25, 34, 47, + 59, 71, 83, 95, 43, 35, 43, 56, 71, 85, 99, 112, 58, 46, 54, 67, 83, 99, + 113, 127, 75, 61, 68, 80, 95, 112, 127, 141, + /* Size 16 */ + 19, 16, 14, 14, 15, 17, 20, 24, 29, 34, 41, 47, 55, 62, 71, 71, 16, 15, + 15, 15, 15, 17, 19, 22, 26, 31, 36, 42, 49, 55, 64, 64, 14, 15, 16, 16, + 15, 17, 18, 21, 24, 28, 33, 38, 44, 50, 58, 58, 14, 15, 16, 17, 18, 20, + 22, 24, 28, 32, 37, 41, 48, 54, 61, 61, 15, 15, 15, 18, 22, 24, 27, 30, + 33, 36, 41, 46, 52, 58, 65, 65, 17, 17, 17, 20, 24, 27, 31, 34, 38, 42, + 46, 51, 57, 63, 70, 70, 20, 19, 18, 22, 27, 31, 36, 40, 45, 49, 53, 58, + 64, 70, 76, 76, 24, 22, 21, 24, 30, 34, 40, 44, 50, 54, 60, 65, 71, 76, + 83, 83, 29, 26, 24, 28, 33, 38, 45, 50, 56, 61, 67, 73, 79, 84, 91, 91, + 34, 31, 28, 32, 36, 42, 49, 54, 61, 67, 74, 79, 86, 91, 98, 98, 41, 36, + 33, 37, 41, 46, 53, 60, 67, 74, 81, 87, 94, 100, 106, 106, 47, 42, 38, + 41, 46, 51, 58, 65, 73, 79, 87, 93, 100, 106, 113, 113, 55, 49, 44, 48, + 52, 57, 64, 71, 79, 86, 94, 100, 108, 114, 121, 121, 62, 55, 50, 54, 58, + 63, 70, 76, 84, 91, 100, 106, 114, 120, 127, 127, 71, 64, 58, 61, 65, + 70, 76, 83, 91, 98, 106, 113, 121, 127, 134, 134, 71, 64, 58, 61, 65, + 70, 76, 83, 91, 98, 106, 113, 121, 127, 134, 134, + /* Size 32 */ + 18, 17, 15, 14, 13, 14, 14, 14, 15, 16, 17, 18, 20, 22, 23, 26, 28, 30, + 33, 36, 40, 42, 45, 49, 53, 57, 60, 65, 69, 69, 69, 69, 17, 16, 15, 14, + 14, 14, 14, 14, 15, 16, 17, 18, 19, 21, 23, 25, 27, 29, 31, 34, 37, 40, + 43, 46, 50, 53, 57, 61, 66, 66, 66, 66, 15, 15, 15, 15, 14, 14, 14, 15, + 15, 15, 16, 18, 19, 20, 22, 24, 26, 28, 30, 32, 35, 38, 41, 44, 48, 51, + 54, 58, 62, 62, 62, 62, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 16, 17, + 18, 19, 21, 23, 25, 26, 28, 31, 34, 36, 39, 42, 45, 48, 51, 55, 59, 59, + 59, 59, 13, 14, 14, 15, 16, 15, 15, 15, 15, 15, 16, 17, 18, 19, 20, 22, + 24, 25, 27, 29, 32, 34, 37, 40, 43, 46, 49, 52, 56, 56, 56, 56, 14, 14, + 14, 15, 15, 16, 16, 16, 16, 17, 18, 18, 19, 20, 22, 23, 25, 27, 29, 31, + 34, 36, 38, 41, 45, 47, 50, 54, 58, 58, 58, 58, 14, 14, 14, 15, 15, 16, + 16, 17, 18, 18, 19, 20, 21, 22, 24, 25, 27, 29, 31, 33, 36, 38, 40, 43, + 46, 49, 52, 56, 60, 60, 60, 60, 14, 14, 15, 15, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 25, 26, 28, 29, 31, 33, 35, 38, 40, 42, 45, 48, 51, 54, 57, + 61, 61, 61, 61, 15, 15, 15, 15, 15, 16, 18, 19, 22, 23, 24, 25, 26, 27, + 29, 30, 32, 34, 35, 38, 40, 42, 45, 47, 50, 53, 56, 59, 63, 63, 63, 63, + 16, 16, 15, 15, 15, 17, 18, 20, 23, 24, 25, 27, 28, 29, 31, 32, 34, 36, + 38, 40, 42, 45, 47, 50, 53, 56, 59, 62, 66, 66, 66, 66, 17, 17, 16, 16, + 16, 18, 19, 21, 24, 25, 27, 28, 30, 32, 33, 35, 37, 39, 41, 43, 45, 47, + 50, 53, 56, 58, 61, 65, 68, 68, 68, 68, 18, 18, 18, 17, 17, 18, 20, 22, + 25, 27, 28, 30, 33, 34, 36, 38, 40, 42, 44, 46, 48, 51, 53, 56, 59, 62, + 64, 68, 71, 71, 71, 71, 20, 19, 19, 18, 18, 19, 21, 23, 26, 28, 30, 33, + 35, 37, 39, 41, 43, 45, 47, 50, 52, 54, 57, 59, 62, 65, 68, 71, 74, 74, + 74, 74, 22, 21, 20, 19, 19, 20, 22, 25, 27, 29, 32, 34, 37, 39, 41, 43, + 46, 48, 50, 52, 55, 57, 60, 62, 65, 68, 71, 74, 77, 77, 77, 77, 23, 23, + 22, 21, 20, 22, 24, 26, 29, 31, 33, 36, 39, 41, 43, 46, 49, 51, 53, 55, + 58, 60, 63, 66, 69, 71, 74, 77, 81, 81, 81, 81, 26, 25, 24, 23, 22, 23, + 25, 28, 30, 32, 35, 38, 41, 43, 46, 48, 52, 54, 56, 59, 62, 64, 67, 69, + 72, 75, 78, 81, 84, 84, 84, 84, 28, 27, 26, 25, 24, 25, 27, 29, 32, 34, + 37, 40, 43, 46, 49, 52, 55, 57, 60, 63, 66, 68, 71, 74, 77, 79, 82, 85, + 88, 88, 88, 88, 30, 29, 28, 26, 25, 27, 29, 31, 34, 36, 39, 42, 45, 48, + 51, 54, 57, 60, 62, 65, 69, 71, 74, 77, 80, 83, 85, 88, 92, 92, 92, 92, + 33, 31, 30, 28, 27, 29, 31, 33, 35, 38, 41, 44, 47, 50, 53, 56, 60, 62, + 65, 68, 72, 74, 77, 80, 83, 86, 89, 92, 95, 95, 95, 95, 36, 34, 32, 31, + 29, 31, 33, 35, 38, 40, 43, 46, 50, 52, 55, 59, 63, 65, 68, 72, 75, 78, + 81, 84, 87, 90, 93, 96, 99, 99, 99, 99, 40, 37, 35, 34, 32, 34, 36, 38, + 40, 42, 45, 48, 52, 55, 58, 62, 66, 69, 72, 75, 79, 82, 85, 88, 91, 94, + 97, 100, 103, 103, 103, 103, 42, 40, 38, 36, 34, 36, 38, 40, 42, 45, 47, + 51, 54, 57, 60, 64, 68, 71, 74, 78, 82, 85, 88, 91, 94, 97, 100, 103, + 107, 107, 107, 107, 45, 43, 41, 39, 37, 38, 40, 42, 45, 47, 50, 53, 57, + 60, 63, 67, 71, 74, 77, 81, 85, 88, 91, 94, 98, 101, 104, 107, 110, 110, + 110, 110, 49, 46, 44, 42, 40, 41, 43, 45, 47, 50, 53, 56, 59, 62, 66, + 69, 74, 77, 80, 84, 88, 91, 94, 98, 101, 104, 107, 110, 114, 114, 114, + 114, 53, 50, 48, 45, 43, 45, 46, 48, 50, 53, 56, 59, 62, 65, 69, 72, 77, + 80, 83, 87, 91, 94, 98, 101, 105, 108, 111, 114, 118, 118, 118, 118, 57, + 53, 51, 48, 46, 47, 49, 51, 53, 56, 58, 62, 65, 68, 71, 75, 79, 83, 86, + 90, 94, 97, 101, 104, 108, 111, 114, 117, 121, 121, 121, 121, 60, 57, + 54, 51, 49, 50, 52, 54, 56, 59, 61, 64, 68, 71, 74, 78, 82, 85, 89, 93, + 97, 100, 104, 107, 111, 114, 117, 120, 124, 124, 124, 124, 65, 61, 58, + 55, 52, 54, 56, 57, 59, 62, 65, 68, 71, 74, 77, 81, 85, 88, 92, 96, 100, + 103, 107, 110, 114, 117, 120, 124, 127, 127, 127, 127, 69, 66, 62, 59, + 56, 58, 60, 61, 63, 66, 68, 71, 74, 77, 81, 84, 88, 92, 95, 99, 103, + 107, 110, 114, 118, 121, 124, 127, 130, 130, 130, 130, 69, 66, 62, 59, + 56, 58, 60, 61, 63, 66, 68, 71, 74, 77, 81, 84, 88, 92, 95, 99, 103, + 107, 110, 114, 118, 121, 124, 127, 130, 130, 130, 130, 69, 66, 62, 59, + 56, 58, 60, 61, 63, 66, 68, 71, 74, 77, 81, 84, 88, 92, 95, 99, 103, + 107, 110, 114, 118, 121, 124, 127, 130, 130, 130, 130, 69, 66, 62, 59, + 56, 58, 60, 61, 63, 66, 68, 71, 74, 77, 81, 84, 88, 92, 95, 99, 103, + 107, 110, 114, 118, 121, 124, 127, 130, 130, 130, 130 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 106, 117, 154, 106, 131, 141, 167, 117, 141, 191, 225, 154, 167, + 225, 279, + /* Size 8 */ + 64, 51, 98, 104, 113, 128, 148, 172, 51, 76, 100, 89, 92, 103, 118, 136, + 98, 100, 119, 115, 114, 121, 134, 151, 104, 89, 115, 132, 140, 147, 158, + 173, 113, 92, 114, 140, 160, 174, 186, 201, 128, 103, 121, 147, 174, + 195, 213, 229, 148, 118, 134, 158, 186, 213, 236, 256, 172, 136, 151, + 173, 201, 229, 256, 280, + /* Size 16 */ + 64, 57, 51, 67, 98, 101, 104, 108, 113, 120, 128, 137, 148, 159, 172, + 172, 57, 59, 61, 75, 99, 97, 96, 99, 101, 107, 114, 122, 131, 141, 152, + 152, 51, 61, 76, 86, 100, 94, 89, 91, 92, 97, 103, 110, 118, 126, 136, + 136, 67, 75, 86, 96, 109, 104, 100, 101, 102, 106, 111, 118, 125, 134, + 143, 143, 98, 99, 100, 109, 119, 117, 115, 115, 114, 118, 121, 127, 134, + 142, 151, 151, 101, 97, 94, 104, 117, 120, 123, 124, 126, 129, 133, 139, + 145, 153, 161, 161, 104, 96, 89, 100, 115, 123, 132, 136, 140, 144, 147, + 153, 158, 165, 173, 173, 108, 99, 91, 101, 115, 124, 136, 142, 149, 154, + 160, 165, 171, 178, 186, 186, 113, 101, 92, 102, 114, 126, 140, 149, + 160, 167, 174, 180, 186, 193, 201, 201, 120, 107, 97, 106, 118, 129, + 144, 154, 167, 175, 184, 191, 199, 206, 214, 214, 128, 114, 103, 111, + 121, 133, 147, 160, 174, 184, 195, 204, 213, 221, 229, 229, 137, 122, + 110, 118, 127, 139, 153, 165, 180, 191, 204, 213, 224, 233, 242, 242, + 148, 131, 118, 125, 134, 145, 158, 171, 186, 199, 213, 224, 236, 246, + 256, 256, 159, 141, 126, 134, 142, 153, 165, 178, 193, 206, 221, 233, + 246, 256, 267, 267, 172, 152, 136, 143, 151, 161, 173, 186, 201, 214, + 229, 242, 256, 267, 280, 280, 172, 152, 136, 143, 151, 161, 173, 186, + 201, 214, 229, 242, 256, 267, 280, 280, + /* Size 32 */ + 64, 60, 57, 54, 51, 58, 67, 79, 98, 99, 101, 103, 104, 106, 108, 110, + 113, 116, 120, 124, 128, 133, 137, 143, 148, 153, 159, 165, 172, 172, + 172, 172, 60, 59, 58, 56, 55, 62, 71, 82, 98, 99, 99, 100, 100, 102, + 103, 105, 107, 110, 113, 117, 121, 125, 129, 134, 139, 144, 149, 155, + 161, 161, 161, 161, 57, 58, 59, 60, 61, 67, 75, 85, 99, 98, 97, 97, 96, + 97, 99, 100, 101, 104, 107, 110, 114, 118, 122, 126, 131, 136, 141, 146, + 152, 152, 152, 152, 54, 56, 60, 63, 67, 73, 80, 89, 99, 97, 96, 94, 93, + 94, 94, 95, 97, 99, 102, 105, 108, 112, 115, 120, 124, 128, 133, 138, + 144, 144, 144, 144, 51, 55, 61, 67, 76, 81, 86, 92, 100, 97, 94, 92, 89, + 90, 91, 91, 92, 95, 97, 100, 103, 106, 110, 113, 118, 122, 126, 131, + 136, 136, 136, 136, 58, 62, 67, 73, 81, 85, 91, 97, 104, 101, 99, 97, + 94, 95, 96, 96, 97, 99, 101, 104, 107, 110, 113, 117, 121, 125, 130, + 134, 140, 140, 140, 140, 67, 71, 75, 80, 86, 91, 96, 102, 109, 106, 104, + 102, 100, 101, 101, 102, 102, 104, 106, 109, 111, 114, 118, 121, 125, + 129, 134, 138, 143, 143, 143, 143, 79, 82, 85, 89, 92, 97, 102, 108, + 114, 112, 110, 109, 107, 107, 107, 108, 108, 110, 112, 114, 116, 119, + 122, 126, 129, 133, 138, 142, 147, 147, 147, 147, 98, 98, 99, 99, 100, + 104, 109, 114, 119, 118, 117, 116, 115, 115, 115, 114, 114, 116, 118, + 119, 121, 124, 127, 130, 134, 138, 142, 146, 151, 151, 151, 151, 99, 99, + 98, 97, 97, 101, 106, 112, 118, 118, 118, 119, 119, 119, 119, 120, 120, + 122, 123, 125, 127, 130, 133, 136, 139, 143, 147, 151, 156, 156, 156, + 156, 101, 99, 97, 96, 94, 99, 104, 110, 117, 118, 120, 121, 123, 124, + 124, 125, 126, 128, 129, 131, 133, 136, 139, 142, 145, 149, 153, 157, + 161, 161, 161, 161, 103, 100, 97, 94, 92, 97, 102, 109, 116, 119, 121, + 124, 127, 129, 130, 131, 133, 134, 136, 138, 140, 143, 145, 148, 151, + 155, 159, 163, 167, 167, 167, 167, 104, 100, 96, 93, 89, 94, 100, 107, + 115, 119, 123, 127, 132, 134, 136, 138, 140, 142, 144, 146, 147, 150, + 153, 155, 158, 162, 165, 169, 173, 173, 173, 173, 106, 102, 97, 94, 90, + 95, 101, 107, 115, 119, 124, 129, 134, 137, 139, 142, 145, 147, 149, + 151, 153, 156, 159, 162, 164, 168, 172, 175, 179, 179, 179, 179, 108, + 103, 99, 94, 91, 96, 101, 107, 115, 119, 124, 130, 136, 139, 142, 146, + 149, 152, 154, 157, 160, 162, 165, 168, 171, 175, 178, 182, 186, 186, + 186, 186, 110, 105, 100, 95, 91, 96, 102, 108, 114, 120, 125, 131, 138, + 142, 146, 150, 154, 157, 160, 163, 166, 169, 172, 175, 178, 182, 185, + 189, 193, 193, 193, 193, 113, 107, 101, 97, 92, 97, 102, 108, 114, 120, + 126, 133, 140, 145, 149, 154, 160, 163, 167, 170, 174, 177, 180, 183, + 186, 190, 193, 197, 201, 201, 201, 201, 116, 110, 104, 99, 95, 99, 104, + 110, 116, 122, 128, 134, 142, 147, 152, 157, 163, 167, 171, 175, 179, + 182, 185, 189, 192, 196, 199, 203, 207, 207, 207, 207, 120, 113, 107, + 102, 97, 101, 106, 112, 118, 123, 129, 136, 144, 149, 154, 160, 167, + 171, 175, 179, 184, 187, 191, 195, 199, 202, 206, 210, 214, 214, 214, + 214, 124, 117, 110, 105, 100, 104, 109, 114, 119, 125, 131, 138, 146, + 151, 157, 163, 170, 175, 179, 184, 190, 193, 197, 201, 206, 209, 213, + 217, 221, 221, 221, 221, 128, 121, 114, 108, 103, 107, 111, 116, 121, + 127, 133, 140, 147, 153, 160, 166, 174, 179, 184, 190, 195, 200, 204, + 208, 213, 217, 221, 225, 229, 229, 229, 229, 133, 125, 118, 112, 106, + 110, 114, 119, 124, 130, 136, 143, 150, 156, 162, 169, 177, 182, 187, + 193, 200, 204, 209, 213, 218, 222, 227, 231, 235, 235, 235, 235, 137, + 129, 122, 115, 110, 113, 118, 122, 127, 133, 139, 145, 153, 159, 165, + 172, 180, 185, 191, 197, 204, 209, 213, 219, 224, 228, 233, 237, 242, + 242, 242, 242, 143, 134, 126, 120, 113, 117, 121, 126, 130, 136, 142, + 148, 155, 162, 168, 175, 183, 189, 195, 201, 208, 213, 219, 224, 230, + 234, 239, 244, 249, 249, 249, 249, 148, 139, 131, 124, 118, 121, 125, + 129, 134, 139, 145, 151, 158, 164, 171, 178, 186, 192, 199, 206, 213, + 218, 224, 230, 236, 241, 246, 251, 256, 256, 256, 256, 153, 144, 136, + 128, 122, 125, 129, 133, 138, 143, 149, 155, 162, 168, 175, 182, 190, + 196, 202, 209, 217, 222, 228, 234, 241, 246, 251, 256, 262, 262, 262, + 262, 159, 149, 141, 133, 126, 130, 134, 138, 142, 147, 153, 159, 165, + 172, 178, 185, 193, 199, 206, 213, 221, 227, 233, 239, 246, 251, 256, + 262, 267, 267, 267, 267, 165, 155, 146, 138, 131, 134, 138, 142, 146, + 151, 157, 163, 169, 175, 182, 189, 197, 203, 210, 217, 225, 231, 237, + 244, 251, 256, 262, 267, 273, 273, 273, 273, 172, 161, 152, 144, 136, + 140, 143, 147, 151, 156, 161, 167, 173, 179, 186, 193, 201, 207, 214, + 221, 229, 235, 242, 249, 256, 262, 267, 273, 280, 280, 280, 280, 172, + 161, 152, 144, 136, 140, 143, 147, 151, 156, 161, 167, 173, 179, 186, + 193, 201, 207, 214, 221, 229, 235, 242, 249, 256, 262, 267, 273, 280, + 280, 280, 280, 172, 161, 152, 144, 136, 140, 143, 147, 151, 156, 161, + 167, 173, 179, 186, 193, 201, 207, 214, 221, 229, 235, 242, 249, 256, + 262, 267, 273, 280, 280, 280, 280, 172, 161, 152, 144, 136, 140, 143, + 147, 151, 156, 161, 167, 173, 179, 186, 193, 201, 207, 214, 221, 229, + 235, 242, 249, 256, 262, 267, 273, 280, 280, 280, 280 }, + { /* Intra matrices */ + /* Size 4 */ + 23, 40, 44, 59, 40, 50, 54, 64, 44, 54, 74, 89, 59, 64, 89, 114, + /* Size 8 */ + 25, 20, 39, 42, 46, 52, 61, 72, 20, 30, 40, 36, 37, 41, 48, 56, 39, 40, + 49, 47, 46, 49, 55, 62, 42, 36, 47, 54, 58, 61, 66, 73, 46, 37, 46, 58, + 67, 73, 79, 85, 52, 41, 49, 61, 73, 83, 91, 99, 61, 48, 55, 66, 79, 91, + 103, 113, 72, 56, 62, 73, 85, 99, 113, 125, + /* Size 16 */ + 24, 22, 19, 26, 38, 39, 41, 42, 44, 47, 51, 55, 59, 64, 69, 69, 22, 22, + 23, 29, 38, 38, 37, 38, 39, 42, 45, 48, 52, 56, 61, 61, 19, 23, 29, 33, + 39, 37, 34, 35, 36, 38, 40, 43, 46, 50, 54, 54, 26, 29, 33, 37, 42, 41, + 39, 39, 40, 42, 43, 46, 49, 53, 57, 57, 38, 38, 39, 42, 47, 46, 45, 45, + 45, 46, 48, 50, 53, 56, 60, 60, 39, 38, 37, 41, 46, 47, 48, 49, 50, 51, + 53, 55, 58, 61, 65, 65, 41, 37, 34, 39, 45, 48, 52, 54, 56, 57, 59, 61, + 64, 67, 70, 70, 42, 38, 35, 39, 45, 49, 54, 57, 60, 62, 64, 67, 69, 72, + 76, 76, 44, 39, 36, 40, 45, 50, 56, 60, 64, 67, 70, 73, 76, 79, 82, 82, + 47, 42, 38, 42, 46, 51, 57, 62, 67, 71, 75, 78, 82, 85, 89, 89, 51, 45, + 40, 43, 48, 53, 59, 64, 70, 75, 80, 84, 88, 92, 96, 96, 55, 48, 43, 46, + 50, 55, 61, 67, 73, 78, 84, 88, 93, 97, 102, 102, 59, 52, 46, 49, 53, + 58, 64, 69, 76, 82, 88, 93, 99, 104, 109, 109, 64, 56, 50, 53, 56, 61, + 67, 72, 79, 85, 92, 97, 104, 109, 114, 114, 69, 61, 54, 57, 60, 65, 70, + 76, 82, 89, 96, 102, 109, 114, 120, 120, 69, 61, 54, 57, 60, 65, 70, 76, + 82, 89, 96, 102, 109, 114, 120, 120, + /* Size 32 */ + 24, 22, 21, 20, 19, 22, 25, 30, 37, 38, 39, 39, 40, 41, 42, 42, 43, 45, + 46, 48, 50, 51, 54, 56, 58, 60, 63, 65, 68, 68, 68, 68, 22, 22, 22, 21, + 21, 23, 27, 31, 37, 38, 38, 38, 38, 39, 40, 40, 41, 42, 44, 45, 47, 48, + 50, 52, 54, 56, 59, 61, 64, 64, 64, 64, 21, 22, 22, 22, 23, 25, 28, 32, + 38, 37, 37, 37, 37, 37, 38, 38, 39, 40, 41, 42, 44, 45, 47, 49, 51, 53, + 55, 57, 60, 60, 60, 60, 20, 21, 22, 24, 25, 28, 30, 34, 38, 37, 36, 36, + 35, 36, 36, 36, 37, 38, 39, 40, 41, 43, 44, 46, 48, 50, 52, 54, 56, 56, + 56, 56, 19, 21, 23, 25, 29, 30, 33, 35, 38, 37, 36, 35, 34, 34, 34, 35, + 35, 36, 37, 38, 39, 41, 42, 44, 45, 47, 49, 51, 53, 53, 53, 53, 22, 23, + 25, 28, 30, 32, 34, 37, 40, 39, 38, 37, 36, 36, 36, 37, 37, 38, 39, 40, + 41, 42, 44, 45, 47, 49, 50, 52, 54, 54, 54, 54, 25, 27, 28, 30, 33, 34, + 37, 39, 42, 41, 40, 39, 38, 38, 39, 39, 39, 40, 41, 42, 43, 44, 45, 47, + 48, 50, 52, 54, 56, 56, 56, 56, 30, 31, 32, 34, 35, 37, 39, 41, 44, 43, + 42, 42, 41, 41, 41, 41, 41, 42, 43, 44, 45, 46, 47, 49, 50, 52, 54, 55, + 57, 57, 57, 57, 37, 37, 38, 38, 38, 40, 42, 44, 46, 46, 45, 45, 44, 44, + 44, 44, 44, 45, 45, 46, 47, 48, 49, 51, 52, 54, 55, 57, 59, 59, 59, 59, + 38, 38, 37, 37, 37, 39, 41, 43, 46, 46, 46, 46, 46, 46, 46, 46, 46, 47, + 48, 48, 49, 50, 52, 53, 54, 56, 58, 59, 61, 61, 61, 61, 39, 38, 37, 36, + 36, 38, 40, 42, 45, 46, 46, 47, 48, 48, 48, 48, 49, 49, 50, 51, 52, 53, + 54, 55, 57, 58, 60, 62, 64, 64, 64, 64, 39, 38, 37, 36, 35, 37, 39, 42, + 45, 46, 47, 48, 49, 50, 50, 51, 52, 52, 53, 54, 55, 56, 57, 58, 59, 61, + 63, 64, 66, 66, 66, 66, 40, 38, 37, 35, 34, 36, 38, 41, 44, 46, 48, 49, + 51, 52, 53, 54, 55, 55, 56, 57, 58, 59, 60, 61, 62, 64, 65, 67, 69, 69, + 69, 69, 41, 39, 37, 36, 34, 36, 38, 41, 44, 46, 48, 50, 52, 53, 54, 55, + 57, 57, 58, 59, 60, 61, 63, 64, 65, 67, 68, 70, 71, 71, 71, 71, 42, 40, + 38, 36, 34, 36, 39, 41, 44, 46, 48, 50, 53, 54, 56, 57, 59, 60, 61, 62, + 63, 64, 65, 67, 68, 69, 71, 73, 74, 74, 74, 74, 42, 40, 38, 36, 35, 37, + 39, 41, 44, 46, 48, 51, 54, 55, 57, 59, 61, 62, 63, 64, 66, 67, 68, 70, + 71, 73, 74, 76, 78, 78, 78, 78, 43, 41, 39, 37, 35, 37, 39, 41, 44, 46, + 49, 52, 55, 57, 59, 61, 63, 64, 66, 67, 69, 70, 72, 73, 75, 76, 78, 79, + 81, 81, 81, 81, 45, 42, 40, 38, 36, 38, 40, 42, 45, 47, 49, 52, 55, 57, + 60, 62, 64, 66, 68, 69, 71, 73, 74, 76, 77, 79, 80, 82, 84, 84, 84, 84, + 46, 44, 41, 39, 37, 39, 41, 43, 45, 48, 50, 53, 56, 58, 61, 63, 66, 68, + 70, 71, 74, 75, 77, 78, 80, 82, 83, 85, 87, 87, 87, 87, 48, 45, 42, 40, + 38, 40, 42, 44, 46, 48, 51, 54, 57, 59, 62, 64, 67, 69, 71, 74, 76, 78, + 79, 81, 83, 85, 87, 89, 90, 90, 90, 90, 50, 47, 44, 41, 39, 41, 43, 45, + 47, 49, 52, 55, 58, 60, 63, 66, 69, 71, 74, 76, 79, 80, 82, 84, 87, 88, + 90, 92, 94, 94, 94, 94, 51, 48, 45, 43, 41, 42, 44, 46, 48, 50, 53, 56, + 59, 61, 64, 67, 70, 73, 75, 78, 80, 82, 85, 87, 89, 91, 93, 95, 97, 97, + 97, 97, 54, 50, 47, 44, 42, 44, 45, 47, 49, 52, 54, 57, 60, 63, 65, 68, + 72, 74, 77, 79, 82, 85, 87, 89, 92, 94, 96, 98, 100, 100, 100, 100, 56, + 52, 49, 46, 44, 45, 47, 49, 51, 53, 55, 58, 61, 64, 67, 70, 73, 76, 78, + 81, 84, 87, 89, 92, 94, 96, 99, 101, 103, 103, 103, 103, 58, 54, 51, 48, + 45, 47, 48, 50, 52, 54, 57, 59, 62, 65, 68, 71, 75, 77, 80, 83, 87, 89, + 92, 94, 97, 99, 102, 104, 107, 107, 107, 107, 60, 56, 53, 50, 47, 49, + 50, 52, 54, 56, 58, 61, 64, 67, 69, 73, 76, 79, 82, 85, 88, 91, 94, 96, + 99, 102, 104, 107, 109, 109, 109, 109, 63, 59, 55, 52, 49, 50, 52, 54, + 55, 58, 60, 63, 65, 68, 71, 74, 78, 80, 83, 87, 90, 93, 96, 99, 102, + 104, 107, 109, 112, 112, 112, 112, 65, 61, 57, 54, 51, 52, 54, 55, 57, + 59, 62, 64, 67, 70, 73, 76, 79, 82, 85, 89, 92, 95, 98, 101, 104, 107, + 109, 112, 115, 115, 115, 115, 68, 64, 60, 56, 53, 54, 56, 57, 59, 61, + 64, 66, 69, 71, 74, 78, 81, 84, 87, 90, 94, 97, 100, 103, 107, 109, 112, + 115, 118, 118, 118, 118, 68, 64, 60, 56, 53, 54, 56, 57, 59, 61, 64, 66, + 69, 71, 74, 78, 81, 84, 87, 90, 94, 97, 100, 103, 107, 109, 112, 115, + 118, 118, 118, 118, 68, 64, 60, 56, 53, 54, 56, 57, 59, 61, 64, 66, 69, + 71, 74, 78, 81, 84, 87, 90, 94, 97, 100, 103, 107, 109, 112, 115, 118, + 118, 118, 118, 68, 64, 60, 56, 53, 54, 56, 57, 59, 61, 64, 66, 69, 71, + 74, 78, 81, 84, 87, 90, 94, 97, 100, 103, 107, 109, 112, 115, 118, 118, + 118, 118 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 70, 120, 197, 70, 109, 156, 218, 120, 156, 229, 287, 197, 218, 287, + 344, + /* Size 8 */ + 64, 47, 51, 69, 94, 126, 161, 197, 47, 55, 52, 62, 80, 105, 135, 167, + 51, 52, 75, 88, 105, 127, 154, 183, 69, 62, 88, 115, 136, 157, 181, 207, + 94, 80, 105, 136, 165, 189, 212, 234, 126, 105, 127, 157, 189, 216, 240, + 261, 161, 135, 154, 181, 212, 240, 264, 284, 197, 167, 183, 207, 234, + 261, 284, 303, + /* Size 16 */ + 64, 54, 47, 49, 51, 59, 69, 80, 94, 108, 126, 141, 161, 177, 197, 197, + 54, 53, 51, 51, 52, 58, 65, 74, 87, 99, 115, 129, 147, 162, 181, 181, + 47, 51, 55, 53, 52, 57, 62, 70, 80, 91, 105, 118, 135, 149, 167, 167, + 49, 51, 53, 57, 62, 67, 73, 81, 91, 102, 115, 128, 144, 158, 175, 175, + 51, 52, 52, 62, 75, 81, 88, 96, 105, 115, 127, 139, 154, 167, 183, 183, + 59, 58, 57, 67, 81, 89, 100, 108, 118, 128, 140, 152, 166, 179, 195, + 195, 69, 65, 62, 73, 88, 100, 115, 124, 136, 146, 157, 168, 181, 193, + 207, 207, 80, 74, 70, 81, 96, 108, 124, 136, 149, 159, 172, 183, 195, + 207, 220, 220, 94, 87, 80, 91, 105, 118, 136, 149, 165, 176, 189, 200, + 212, 222, 234, 234, 108, 99, 91, 102, 115, 128, 146, 159, 176, 188, 202, + 213, 225, 235, 247, 247, 126, 115, 105, 115, 127, 140, 157, 172, 189, + 202, 216, 228, 240, 250, 261, 261, 141, 129, 118, 128, 139, 152, 168, + 183, 200, 213, 228, 239, 251, 261, 272, 272, 161, 147, 135, 144, 154, + 166, 181, 195, 212, 225, 240, 251, 264, 273, 284, 284, 177, 162, 149, + 158, 167, 179, 193, 207, 222, 235, 250, 261, 273, 283, 293, 293, 197, + 181, 167, 175, 183, 195, 207, 220, 234, 247, 261, 272, 284, 293, 303, + 303, 197, 181, 167, 175, 183, 195, 207, 220, 234, 247, 261, 272, 284, + 293, 303, 303, + /* Size 32 */ + 64, 59, 54, 51, 47, 48, 49, 50, 51, 55, 59, 64, 69, 74, 80, 86, 94, 101, + 108, 116, 126, 133, 141, 150, 161, 169, 177, 186, 197, 197, 197, 197, + 59, 56, 54, 51, 49, 50, 50, 51, 52, 55, 58, 62, 67, 72, 77, 83, 90, 96, + 103, 111, 120, 127, 135, 143, 153, 161, 169, 178, 189, 189, 189, 189, + 54, 54, 53, 52, 51, 51, 51, 52, 52, 55, 58, 61, 65, 69, 74, 80, 87, 92, + 99, 106, 115, 121, 129, 137, 147, 154, 162, 171, 181, 181, 181, 181, 51, + 51, 52, 52, 53, 53, 52, 52, 52, 55, 57, 60, 63, 67, 72, 77, 83, 89, 95, + 102, 110, 116, 123, 131, 141, 148, 155, 164, 174, 174, 174, 174, 47, 49, + 51, 53, 55, 54, 53, 53, 52, 54, 57, 59, 62, 66, 70, 75, 80, 85, 91, 98, + 105, 111, 118, 126, 135, 142, 149, 158, 167, 167, 167, 167, 48, 50, 51, + 53, 54, 55, 55, 56, 57, 59, 61, 64, 67, 71, 75, 80, 85, 90, 96, 102, + 110, 116, 123, 130, 139, 146, 153, 162, 171, 171, 171, 171, 49, 50, 51, + 52, 53, 55, 57, 59, 62, 64, 67, 70, 73, 77, 81, 86, 91, 96, 102, 108, + 115, 121, 128, 135, 144, 150, 158, 166, 175, 175, 175, 175, 50, 51, 52, + 52, 53, 56, 59, 63, 68, 70, 73, 76, 80, 84, 88, 92, 97, 102, 108, 114, + 121, 127, 133, 140, 148, 155, 162, 170, 179, 179, 179, 179, 51, 52, 52, + 52, 52, 57, 62, 68, 75, 78, 81, 84, 88, 92, 96, 100, 105, 109, 115, 120, + 127, 133, 139, 146, 154, 160, 167, 175, 183, 183, 183, 183, 55, 55, 55, + 55, 54, 59, 64, 70, 78, 81, 85, 89, 94, 98, 102, 106, 111, 116, 121, + 127, 133, 139, 145, 152, 160, 166, 173, 181, 189, 189, 189, 189, 59, 58, + 58, 57, 57, 61, 67, 73, 81, 85, 89, 94, 100, 104, 108, 113, 118, 123, + 128, 134, 140, 146, 152, 159, 166, 173, 179, 187, 195, 195, 195, 195, + 64, 62, 61, 60, 59, 64, 70, 76, 84, 89, 94, 100, 107, 111, 116, 121, + 126, 131, 137, 142, 148, 154, 160, 166, 173, 180, 186, 193, 201, 201, + 201, 201, 69, 67, 65, 63, 62, 67, 73, 80, 88, 94, 100, 107, 115, 119, + 124, 130, 136, 141, 146, 151, 157, 163, 168, 175, 181, 187, 193, 200, + 207, 207, 207, 207, 74, 72, 69, 67, 66, 71, 77, 84, 92, 98, 104, 111, + 119, 124, 130, 136, 142, 147, 152, 158, 164, 170, 175, 181, 188, 194, + 200, 206, 213, 213, 213, 213, 80, 77, 74, 72, 70, 75, 81, 88, 96, 102, + 108, 116, 124, 130, 136, 142, 149, 154, 159, 165, 172, 177, 183, 189, + 195, 201, 207, 213, 220, 220, 220, 220, 86, 83, 80, 77, 75, 80, 86, 92, + 100, 106, 113, 121, 130, 136, 142, 149, 156, 162, 167, 173, 180, 185, + 191, 197, 203, 209, 214, 220, 227, 227, 227, 227, 94, 90, 87, 83, 80, + 85, 91, 97, 105, 111, 118, 126, 136, 142, 149, 156, 165, 170, 176, 182, + 189, 194, 200, 205, 212, 217, 222, 228, 234, 234, 234, 234, 101, 96, 92, + 89, 85, 90, 96, 102, 109, 116, 123, 131, 141, 147, 154, 162, 170, 176, + 182, 188, 195, 200, 206, 212, 218, 223, 229, 234, 240, 240, 240, 240, + 108, 103, 99, 95, 91, 96, 102, 108, 115, 121, 128, 137, 146, 152, 159, + 167, 176, 182, 188, 195, 202, 207, 213, 219, 225, 230, 235, 241, 247, + 247, 247, 247, 116, 111, 106, 102, 98, 102, 108, 114, 120, 127, 134, + 142, 151, 158, 165, 173, 182, 188, 195, 201, 209, 214, 220, 226, 232, + 237, 242, 248, 254, 254, 254, 254, 126, 120, 115, 110, 105, 110, 115, + 121, 127, 133, 140, 148, 157, 164, 172, 180, 189, 195, 202, 209, 216, + 222, 228, 234, 240, 245, 250, 255, 261, 261, 261, 261, 133, 127, 121, + 116, 111, 116, 121, 127, 133, 139, 146, 154, 163, 170, 177, 185, 194, + 200, 207, 214, 222, 227, 233, 239, 245, 250, 255, 261, 266, 266, 266, + 266, 141, 135, 129, 123, 118, 123, 128, 133, 139, 145, 152, 160, 168, + 175, 183, 191, 200, 206, 213, 220, 228, 233, 239, 245, 251, 256, 261, + 266, 272, 272, 272, 272, 150, 143, 137, 131, 126, 130, 135, 140, 146, + 152, 159, 166, 175, 181, 189, 197, 205, 212, 219, 226, 234, 239, 245, + 251, 257, 262, 267, 272, 278, 278, 278, 278, 161, 153, 147, 141, 135, + 139, 144, 148, 154, 160, 166, 173, 181, 188, 195, 203, 212, 218, 225, + 232, 240, 245, 251, 257, 264, 268, 273, 278, 284, 284, 284, 284, 169, + 161, 154, 148, 142, 146, 150, 155, 160, 166, 173, 180, 187, 194, 201, + 209, 217, 223, 230, 237, 245, 250, 256, 262, 268, 273, 278, 283, 288, + 288, 288, 288, 177, 169, 162, 155, 149, 153, 158, 162, 167, 173, 179, + 186, 193, 200, 207, 214, 222, 229, 235, 242, 250, 255, 261, 267, 273, + 278, 283, 288, 293, 293, 293, 293, 186, 178, 171, 164, 158, 162, 166, + 170, 175, 181, 187, 193, 200, 206, 213, 220, 228, 234, 241, 248, 255, + 261, 266, 272, 278, 283, 288, 293, 298, 298, 298, 298, 197, 189, 181, + 174, 167, 171, 175, 179, 183, 189, 195, 201, 207, 213, 220, 227, 234, + 240, 247, 254, 261, 266, 272, 278, 284, 288, 293, 298, 303, 303, 303, + 303, 197, 189, 181, 174, 167, 171, 175, 179, 183, 189, 195, 201, 207, + 213, 220, 227, 234, 240, 247, 254, 261, 266, 272, 278, 284, 288, 293, + 298, 303, 303, 303, 303, 197, 189, 181, 174, 167, 171, 175, 179, 183, + 189, 195, 201, 207, 213, 220, 227, 234, 240, 247, 254, 261, 266, 272, + 278, 284, 288, 293, 298, 303, 303, 303, 303, 197, 189, 181, 174, 167, + 171, 175, 179, 183, 189, 195, 201, 207, 213, 220, 227, 234, 240, 247, + 254, 261, 266, 272, 278, 284, 288, 293, 298, 303, 303, 303, 303 }, + { /* Intra matrices */ + /* Size 4 */ + 19, 21, 37, 63, 21, 33, 49, 70, 37, 49, 74, 96, 63, 70, 96, 119, + /* Size 8 */ + 23, 17, 18, 25, 34, 47, 61, 77, 17, 19, 18, 22, 29, 38, 50, 64, 18, 18, + 27, 32, 38, 47, 58, 71, 25, 22, 32, 42, 51, 60, 70, 81, 34, 29, 38, 51, + 63, 73, 83, 94, 47, 38, 47, 60, 73, 85, 96, 106, 61, 50, 58, 70, 83, 96, + 108, 118, 77, 64, 71, 81, 94, 106, 118, 127, + /* Size 16 */ + 22, 18, 16, 17, 17, 20, 23, 27, 33, 38, 45, 51, 58, 65, 73, 73, 18, 18, + 17, 17, 18, 20, 22, 25, 30, 34, 40, 46, 53, 59, 67, 67, 16, 17, 19, 18, + 18, 19, 21, 24, 28, 32, 37, 42, 48, 54, 61, 61, 17, 17, 18, 19, 21, 23, + 25, 28, 31, 35, 40, 45, 52, 57, 64, 64, 17, 18, 18, 21, 26, 28, 31, 33, + 37, 40, 45, 50, 55, 61, 68, 68, 20, 20, 19, 23, 28, 31, 35, 38, 42, 46, + 50, 55, 61, 66, 72, 72, 23, 22, 21, 25, 31, 35, 40, 44, 48, 52, 57, 61, + 67, 72, 78, 78, 27, 25, 24, 28, 33, 38, 44, 48, 54, 58, 63, 67, 73, 78, + 83, 83, 33, 30, 28, 31, 37, 42, 48, 54, 60, 64, 70, 74, 80, 84, 90, 90, + 38, 34, 32, 35, 40, 46, 52, 58, 64, 69, 75, 80, 85, 90, 95, 95, 45, 40, + 37, 40, 45, 50, 57, 63, 70, 75, 82, 87, 92, 97, 102, 102, 51, 46, 42, + 45, 50, 55, 61, 67, 74, 80, 87, 92, 97, 102, 107, 107, 58, 53, 48, 52, + 55, 61, 67, 73, 80, 85, 92, 97, 103, 107, 112, 112, 65, 59, 54, 57, 61, + 66, 72, 78, 84, 90, 97, 102, 107, 112, 117, 117, 73, 67, 61, 64, 68, 72, + 78, 83, 90, 95, 102, 107, 112, 117, 122, 122, 73, 67, 61, 64, 68, 72, + 78, 83, 90, 95, 102, 107, 112, 117, 122, 122, + /* Size 32 */ + 21, 19, 18, 17, 16, 16, 16, 17, 17, 18, 20, 21, 23, 25, 27, 29, 32, 34, + 37, 40, 44, 46, 49, 53, 57, 60, 63, 67, 72, 72, 72, 72, 19, 19, 18, 17, + 16, 16, 17, 17, 17, 18, 19, 21, 22, 24, 26, 28, 31, 33, 35, 38, 41, 44, + 47, 50, 54, 57, 60, 64, 68, 68, 68, 68, 18, 18, 17, 17, 17, 17, 17, 17, + 17, 18, 19, 20, 22, 23, 25, 27, 29, 31, 34, 36, 39, 42, 45, 48, 52, 54, + 57, 61, 65, 65, 65, 65, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 19, 20, + 21, 22, 24, 26, 28, 30, 32, 35, 38, 40, 43, 46, 49, 52, 55, 58, 62, 62, + 62, 62, 16, 16, 17, 17, 18, 18, 18, 17, 17, 18, 19, 20, 21, 22, 23, 25, + 27, 29, 31, 33, 36, 38, 41, 44, 47, 50, 53, 56, 60, 60, 60, 60, 16, 16, + 17, 17, 18, 18, 18, 18, 19, 19, 20, 21, 22, 24, 25, 27, 29, 31, 33, 35, + 38, 40, 42, 45, 49, 51, 54, 57, 61, 61, 61, 61, 16, 17, 17, 17, 18, 18, + 19, 20, 20, 21, 22, 23, 24, 26, 27, 29, 31, 33, 35, 37, 40, 42, 44, 47, + 50, 53, 56, 59, 63, 63, 63, 63, 17, 17, 17, 17, 17, 18, 20, 21, 22, 23, + 24, 26, 27, 28, 30, 31, 33, 35, 37, 39, 42, 44, 46, 49, 52, 55, 58, 61, + 64, 64, 64, 64, 17, 17, 17, 17, 17, 19, 20, 22, 25, 26, 27, 28, 30, 31, + 32, 34, 36, 37, 39, 42, 44, 46, 49, 51, 54, 57, 60, 63, 66, 66, 66, 66, + 18, 18, 18, 18, 18, 19, 21, 23, 26, 27, 29, 30, 32, 33, 35, 36, 38, 40, + 42, 44, 46, 49, 51, 54, 57, 59, 62, 65, 68, 68, 68, 68, 20, 19, 19, 19, + 19, 20, 22, 24, 27, 29, 30, 32, 34, 35, 37, 39, 41, 43, 45, 47, 49, 51, + 54, 56, 59, 62, 64, 67, 71, 71, 71, 71, 21, 21, 20, 20, 20, 21, 23, 26, + 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 57, 59, 62, 64, + 67, 70, 73, 73, 73, 73, 23, 22, 22, 21, 21, 22, 24, 27, 30, 32, 34, 36, + 39, 41, 43, 45, 47, 49, 51, 53, 56, 58, 60, 62, 65, 68, 70, 73, 76, 76, + 76, 76, 25, 24, 23, 22, 22, 24, 26, 28, 31, 33, 35, 38, 41, 43, 45, 47, + 50, 52, 54, 56, 58, 60, 63, 65, 68, 70, 73, 76, 78, 78, 78, 78, 27, 26, + 25, 24, 23, 25, 27, 30, 32, 35, 37, 40, 43, 45, 47, 50, 52, 54, 56, 59, + 61, 63, 66, 68, 71, 73, 76, 78, 81, 81, 81, 81, 29, 28, 27, 26, 25, 27, + 29, 31, 34, 36, 39, 42, 45, 47, 50, 52, 55, 57, 60, 62, 65, 67, 69, 72, + 74, 76, 79, 82, 84, 84, 84, 84, 32, 31, 29, 28, 27, 29, 31, 33, 36, 38, + 41, 44, 47, 50, 52, 55, 58, 61, 63, 66, 68, 70, 73, 75, 78, 80, 82, 85, + 88, 88, 88, 88, 34, 33, 31, 30, 29, 31, 33, 35, 37, 40, 43, 46, 49, 52, + 54, 57, 61, 63, 65, 68, 71, 73, 75, 78, 80, 83, 85, 88, 90, 90, 90, 90, + 37, 35, 34, 32, 31, 33, 35, 37, 39, 42, 45, 48, 51, 54, 56, 60, 63, 65, + 68, 71, 74, 76, 78, 81, 83, 86, 88, 90, 93, 93, 93, 93, 40, 38, 36, 35, + 33, 35, 37, 39, 42, 44, 47, 50, 53, 56, 59, 62, 66, 68, 71, 73, 77, 79, + 81, 84, 87, 89, 91, 93, 96, 96, 96, 96, 44, 41, 39, 38, 36, 38, 40, 42, + 44, 46, 49, 52, 56, 58, 61, 65, 68, 71, 74, 77, 80, 82, 85, 87, 90, 92, + 94, 97, 99, 99, 99, 99, 46, 44, 42, 40, 38, 40, 42, 44, 46, 49, 51, 54, + 58, 60, 63, 67, 70, 73, 76, 79, 82, 84, 87, 90, 92, 95, 97, 99, 102, + 102, 102, 102, 49, 47, 45, 43, 41, 42, 44, 46, 49, 51, 54, 57, 60, 63, + 66, 69, 73, 75, 78, 81, 85, 87, 89, 92, 95, 97, 99, 102, 104, 104, 104, + 104, 53, 50, 48, 46, 44, 45, 47, 49, 51, 54, 56, 59, 62, 65, 68, 72, 75, + 78, 81, 84, 87, 90, 92, 95, 98, 100, 102, 105, 107, 107, 107, 107, 57, + 54, 52, 49, 47, 49, 50, 52, 54, 57, 59, 62, 65, 68, 71, 74, 78, 80, 83, + 87, 90, 92, 95, 98, 101, 103, 105, 107, 110, 110, 110, 110, 60, 57, 54, + 52, 50, 51, 53, 55, 57, 59, 62, 64, 68, 70, 73, 76, 80, 83, 86, 89, 92, + 95, 97, 100, 103, 105, 107, 110, 112, 112, 112, 112, 63, 60, 57, 55, 53, + 54, 56, 58, 60, 62, 64, 67, 70, 73, 76, 79, 82, 85, 88, 91, 94, 97, 99, + 102, 105, 107, 109, 112, 114, 114, 114, 114, 67, 64, 61, 58, 56, 57, 59, + 61, 63, 65, 67, 70, 73, 76, 78, 82, 85, 88, 90, 93, 97, 99, 102, 105, + 107, 110, 112, 114, 117, 117, 117, 117, 72, 68, 65, 62, 60, 61, 63, 64, + 66, 68, 71, 73, 76, 78, 81, 84, 88, 90, 93, 96, 99, 102, 104, 107, 110, + 112, 114, 117, 119, 119, 119, 119, 72, 68, 65, 62, 60, 61, 63, 64, 66, + 68, 71, 73, 76, 78, 81, 84, 88, 90, 93, 96, 99, 102, 104, 107, 110, 112, + 114, 117, 119, 119, 119, 119, 72, 68, 65, 62, 60, 61, 63, 64, 66, 68, + 71, 73, 76, 78, 81, 84, 88, 90, 93, 96, 99, 102, 104, 107, 110, 112, + 114, 117, 119, 119, 119, 119, 72, 68, 65, 62, 60, 61, 63, 64, 66, 68, + 71, 73, 76, 78, 81, 84, 88, 90, 93, 96, 99, 102, 104, 107, 110, 112, + 114, 117, 119, 119, 119, 119 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 104, 114, 146, 104, 126, 136, 158, 114, 136, 178, 207, 146, 158, + 207, 250, + /* Size 8 */ + 64, 51, 96, 102, 109, 123, 141, 161, 51, 75, 98, 88, 91, 100, 114, 131, + 96, 98, 116, 111, 111, 117, 128, 143, 102, 88, 111, 127, 134, 140, 150, + 163, 109, 91, 111, 134, 151, 163, 174, 186, 123, 100, 117, 140, 163, + 181, 196, 209, 141, 114, 128, 150, 174, 196, 214, 230, 161, 131, 143, + 163, 186, 209, 230, 248, + /* Size 16 */ + 64, 57, 51, 67, 96, 99, 102, 106, 109, 116, 123, 132, 141, 151, 161, + 161, 57, 59, 61, 75, 97, 96, 94, 97, 99, 105, 111, 118, 126, 135, 144, + 144, 51, 61, 75, 85, 98, 92, 88, 89, 91, 95, 100, 107, 114, 122, 131, + 131, 67, 75, 85, 94, 106, 102, 98, 99, 100, 104, 108, 114, 121, 128, + 137, 137, 96, 97, 98, 106, 116, 113, 111, 111, 111, 114, 117, 123, 128, + 135, 143, 143, 99, 96, 92, 102, 113, 116, 119, 120, 121, 125, 128, 133, + 138, 145, 152, 152, 102, 94, 88, 98, 111, 119, 127, 131, 134, 137, 140, + 145, 150, 156, 163, 163, 106, 97, 89, 99, 111, 120, 131, 136, 142, 146, + 151, 156, 161, 167, 173, 173, 109, 99, 91, 100, 111, 121, 134, 142, 151, + 157, 163, 168, 174, 179, 186, 186, 116, 105, 95, 104, 114, 125, 137, + 146, 157, 164, 172, 178, 184, 190, 196, 196, 123, 111, 100, 108, 117, + 128, 140, 151, 163, 172, 181, 188, 196, 202, 209, 209, 132, 118, 107, + 114, 123, 133, 145, 156, 168, 178, 188, 196, 205, 211, 219, 219, 141, + 126, 114, 121, 128, 138, 150, 161, 174, 184, 196, 205, 214, 222, 230, + 230, 151, 135, 122, 128, 135, 145, 156, 167, 179, 190, 202, 211, 222, + 230, 238, 238, 161, 144, 131, 137, 143, 152, 163, 173, 186, 196, 209, + 219, 230, 238, 248, 248, 161, 144, 131, 137, 143, 152, 163, 173, 186, + 196, 209, 219, 230, 238, 248, 248, + /* Size 32 */ + 64, 60, 57, 54, 51, 58, 67, 79, 96, 97, 99, 100, 102, 104, 106, 107, + 109, 113, 116, 120, 123, 127, 132, 136, 141, 146, 151, 156, 161, 161, + 161, 161, 60, 59, 58, 57, 56, 62, 70, 81, 96, 97, 97, 98, 98, 99, 101, + 102, 104, 107, 110, 113, 117, 120, 124, 129, 133, 137, 142, 147, 152, + 152, 152, 152, 57, 58, 59, 60, 61, 67, 75, 84, 97, 96, 96, 95, 94, 96, + 97, 98, 99, 102, 105, 107, 111, 114, 118, 122, 126, 130, 135, 139, 144, + 144, 144, 144, 54, 57, 60, 63, 67, 73, 79, 87, 97, 96, 94, 92, 91, 92, + 93, 94, 95, 97, 100, 102, 105, 108, 112, 116, 120, 124, 128, 132, 137, + 137, 137, 137, 51, 56, 61, 67, 75, 80, 85, 91, 98, 95, 92, 90, 88, 89, + 89, 90, 91, 93, 95, 98, 100, 103, 107, 110, 114, 118, 122, 126, 131, + 131, 131, 131, 58, 62, 67, 73, 80, 84, 89, 95, 102, 99, 97, 95, 93, 93, + 94, 94, 95, 97, 99, 102, 104, 107, 110, 114, 117, 121, 125, 129, 134, + 134, 134, 134, 67, 70, 75, 79, 85, 89, 94, 100, 106, 104, 102, 100, 98, + 99, 99, 99, 100, 102, 104, 106, 108, 111, 114, 117, 121, 124, 128, 132, + 137, 137, 137, 137, 79, 81, 84, 87, 91, 95, 100, 105, 110, 109, 107, + 106, 104, 105, 105, 105, 105, 107, 109, 111, 112, 115, 118, 121, 125, + 128, 132, 136, 140, 140, 140, 140, 96, 96, 97, 97, 98, 102, 106, 110, + 116, 114, 113, 112, 111, 111, 111, 111, 111, 112, 114, 116, 117, 120, + 123, 125, 128, 132, 135, 139, 143, 143, 143, 143, 97, 97, 96, 96, 95, + 99, 104, 109, 114, 115, 115, 115, 115, 115, 115, 116, 116, 117, 119, + 121, 122, 125, 128, 130, 133, 137, 140, 144, 148, 148, 148, 148, 99, 97, + 96, 94, 92, 97, 102, 107, 113, 115, 116, 117, 119, 119, 120, 121, 121, + 123, 125, 126, 128, 130, 133, 136, 138, 142, 145, 149, 152, 152, 152, + 152, 100, 98, 95, 92, 90, 95, 100, 106, 112, 115, 117, 120, 123, 124, + 125, 126, 127, 129, 131, 132, 134, 136, 139, 141, 144, 147, 150, 154, + 157, 157, 157, 157, 102, 98, 94, 91, 88, 93, 98, 104, 111, 115, 119, + 123, 127, 129, 131, 132, 134, 136, 137, 139, 140, 143, 145, 147, 150, + 153, 156, 159, 163, 163, 163, 163, 104, 99, 96, 92, 89, 93, 99, 105, + 111, 115, 119, 124, 129, 131, 133, 136, 138, 140, 142, 144, 146, 148, + 150, 153, 155, 158, 161, 164, 168, 168, 168, 168, 106, 101, 97, 93, 89, + 94, 99, 105, 111, 115, 120, 125, 131, 133, 136, 139, 142, 144, 146, 149, + 151, 153, 156, 158, 161, 164, 167, 170, 173, 173, 173, 173, 107, 102, + 98, 94, 90, 94, 99, 105, 111, 116, 121, 126, 132, 136, 139, 143, 147, + 149, 151, 154, 157, 159, 162, 164, 167, 170, 173, 176, 179, 179, 179, + 179, 109, 104, 99, 95, 91, 95, 100, 105, 111, 116, 121, 127, 134, 138, + 142, 147, 151, 154, 157, 160, 163, 166, 168, 171, 174, 177, 179, 182, + 186, 186, 186, 186, 113, 107, 102, 97, 93, 97, 102, 107, 112, 117, 123, + 129, 136, 140, 144, 149, 154, 157, 160, 164, 167, 170, 173, 176, 179, + 182, 185, 188, 191, 191, 191, 191, 116, 110, 105, 100, 95, 99, 104, 109, + 114, 119, 125, 131, 137, 142, 146, 151, 157, 160, 164, 168, 172, 175, + 178, 181, 184, 187, 190, 193, 196, 196, 196, 196, 120, 113, 107, 102, + 98, 102, 106, 111, 116, 121, 126, 132, 139, 144, 149, 154, 160, 164, + 168, 172, 176, 180, 183, 186, 190, 193, 196, 199, 202, 202, 202, 202, + 123, 117, 111, 105, 100, 104, 108, 112, 117, 122, 128, 134, 140, 146, + 151, 157, 163, 167, 172, 176, 181, 185, 188, 192, 196, 199, 202, 205, + 209, 209, 209, 209, 127, 120, 114, 108, 103, 107, 111, 115, 120, 125, + 130, 136, 143, 148, 153, 159, 166, 170, 175, 180, 185, 188, 192, 196, + 200, 203, 207, 210, 214, 214, 214, 214, 132, 124, 118, 112, 107, 110, + 114, 118, 123, 128, 133, 139, 145, 150, 156, 162, 168, 173, 178, 183, + 188, 192, 196, 200, 205, 208, 211, 215, 219, 219, 219, 219, 136, 129, + 122, 116, 110, 114, 117, 121, 125, 130, 136, 141, 147, 153, 158, 164, + 171, 176, 181, 186, 192, 196, 200, 205, 209, 213, 216, 220, 224, 224, + 224, 224, 141, 133, 126, 120, 114, 117, 121, 125, 128, 133, 138, 144, + 150, 155, 161, 167, 174, 179, 184, 190, 196, 200, 205, 209, 214, 218, + 222, 226, 230, 230, 230, 230, 146, 137, 130, 124, 118, 121, 124, 128, + 132, 137, 142, 147, 153, 158, 164, 170, 177, 182, 187, 193, 199, 203, + 208, 213, 218, 222, 226, 230, 234, 234, 234, 234, 151, 142, 135, 128, + 122, 125, 128, 132, 135, 140, 145, 150, 156, 161, 167, 173, 179, 185, + 190, 196, 202, 207, 211, 216, 222, 226, 230, 234, 238, 238, 238, 238, + 156, 147, 139, 132, 126, 129, 132, 136, 139, 144, 149, 154, 159, 164, + 170, 176, 182, 188, 193, 199, 205, 210, 215, 220, 226, 230, 234, 238, + 243, 243, 243, 243, 161, 152, 144, 137, 131, 134, 137, 140, 143, 148, + 152, 157, 163, 168, 173, 179, 186, 191, 196, 202, 209, 214, 219, 224, + 230, 234, 238, 243, 248, 248, 248, 248, 161, 152, 144, 137, 131, 134, + 137, 140, 143, 148, 152, 157, 163, 168, 173, 179, 186, 191, 196, 202, + 209, 214, 219, 224, 230, 234, 238, 243, 248, 248, 248, 248, 161, 152, + 144, 137, 131, 134, 137, 140, 143, 148, 152, 157, 163, 168, 173, 179, + 186, 191, 196, 202, 209, 214, 219, 224, 230, 234, 238, 243, 248, 248, + 248, 248, 161, 152, 144, 137, 131, 134, 137, 140, 143, 148, 152, 157, + 163, 168, 173, 179, 186, 191, 196, 202, 209, 214, 219, 224, 230, 234, + 238, 243, 248, 248, 248, 248 }, + { /* Intra matrices */ + /* Size 4 */ + 25, 42, 46, 60, 42, 51, 55, 65, 46, 55, 75, 88, 60, 65, 88, 109, + /* Size 8 */ + 27, 22, 41, 44, 48, 54, 63, 72, 22, 32, 42, 38, 39, 43, 50, 58, 41, 42, + 50, 49, 48, 51, 57, 64, 44, 38, 49, 56, 59, 62, 67, 73, 48, 39, 48, 59, + 67, 73, 78, 84, 54, 43, 51, 62, 73, 82, 90, 96, 63, 50, 57, 67, 78, 90, + 99, 107, 72, 58, 64, 73, 84, 96, 107, 117, + /* Size 16 */ + 26, 23, 21, 27, 40, 41, 43, 44, 46, 49, 52, 56, 61, 65, 70, 70, 23, 24, + 25, 31, 40, 40, 39, 40, 42, 44, 47, 50, 54, 58, 62, 62, 21, 25, 31, 35, + 41, 39, 37, 37, 38, 40, 42, 45, 48, 52, 56, 56, 27, 31, 35, 39, 45, 43, + 41, 41, 42, 44, 46, 48, 51, 55, 58, 58, 40, 40, 41, 45, 49, 48, 47, 47, + 47, 48, 50, 52, 55, 58, 62, 62, 41, 40, 39, 43, 48, 49, 50, 51, 52, 53, + 54, 57, 59, 62, 66, 66, 43, 39, 37, 41, 47, 50, 54, 56, 57, 59, 60, 62, + 65, 68, 71, 71, 44, 40, 37, 41, 47, 51, 56, 58, 61, 63, 65, 67, 70, 73, + 76, 76, 46, 42, 38, 42, 47, 52, 57, 61, 65, 68, 71, 73, 76, 79, 82, 82, + 49, 44, 40, 44, 48, 53, 59, 63, 68, 71, 75, 78, 81, 84, 87, 87, 52, 47, + 42, 46, 50, 54, 60, 65, 71, 75, 80, 83, 87, 90, 93, 93, 56, 50, 45, 48, + 52, 57, 62, 67, 73, 78, 83, 87, 91, 95, 98, 98, 61, 54, 48, 51, 55, 59, + 65, 70, 76, 81, 87, 91, 96, 100, 104, 104, 65, 58, 52, 55, 58, 62, 68, + 73, 79, 84, 90, 95, 100, 104, 108, 108, 70, 62, 56, 58, 62, 66, 71, 76, + 82, 87, 93, 98, 104, 108, 113, 113, 70, 62, 56, 58, 62, 66, 71, 76, 82, + 87, 93, 98, 104, 108, 113, 113, + /* Size 32 */ + 26, 24, 23, 22, 21, 23, 27, 32, 39, 40, 41, 41, 42, 43, 44, 44, 45, 47, + 48, 50, 52, 53, 55, 57, 60, 62, 64, 66, 69, 69, 69, 69, 24, 24, 23, 23, + 22, 25, 29, 33, 40, 40, 40, 40, 40, 41, 42, 42, 43, 44, 46, 47, 49, 50, + 52, 54, 56, 58, 60, 62, 65, 65, 65, 65, 23, 23, 24, 24, 25, 27, 30, 34, + 40, 40, 39, 39, 39, 39, 40, 40, 41, 42, 43, 45, 46, 47, 49, 51, 53, 55, + 57, 59, 61, 61, 61, 61, 22, 23, 24, 26, 27, 30, 32, 36, 40, 39, 39, 38, + 37, 38, 38, 39, 39, 40, 41, 42, 43, 45, 46, 48, 50, 52, 54, 56, 58, 58, + 58, 58, 21, 22, 25, 27, 31, 33, 35, 37, 40, 39, 38, 37, 36, 36, 37, 37, + 37, 38, 39, 40, 41, 43, 44, 46, 47, 49, 51, 53, 55, 55, 55, 55, 23, 25, + 27, 30, 33, 34, 37, 39, 42, 41, 40, 39, 38, 38, 39, 39, 39, 40, 41, 42, + 43, 44, 46, 47, 49, 50, 52, 54, 56, 56, 56, 56, 27, 29, 30, 32, 35, 37, + 39, 41, 44, 43, 42, 41, 40, 41, 41, 41, 41, 42, 43, 44, 45, 46, 47, 49, + 50, 52, 54, 56, 58, 58, 58, 58, 32, 33, 34, 36, 37, 39, 41, 43, 46, 45, + 44, 44, 43, 43, 43, 43, 43, 44, 45, 46, 47, 48, 49, 51, 52, 54, 55, 57, + 59, 59, 59, 59, 39, 40, 40, 40, 40, 42, 44, 46, 48, 48, 47, 47, 46, 46, + 46, 46, 46, 47, 47, 48, 49, 50, 51, 52, 54, 55, 57, 59, 61, 61, 61, 61, + 40, 40, 40, 39, 39, 41, 43, 45, 48, 48, 48, 48, 48, 48, 48, 48, 48, 49, + 50, 50, 51, 52, 53, 55, 56, 57, 59, 61, 63, 63, 63, 63, 41, 40, 39, 39, + 38, 40, 42, 44, 47, 48, 48, 49, 50, 50, 50, 50, 51, 51, 52, 53, 54, 55, + 56, 57, 58, 60, 61, 63, 65, 65, 65, 65, 41, 40, 39, 38, 37, 39, 41, 44, + 47, 48, 49, 50, 51, 52, 52, 53, 53, 54, 55, 56, 56, 57, 58, 60, 61, 62, + 64, 65, 67, 67, 67, 67, 42, 40, 39, 37, 36, 38, 40, 43, 46, 48, 50, 51, + 53, 54, 55, 56, 56, 57, 58, 59, 59, 60, 61, 62, 64, 65, 66, 68, 69, 69, + 69, 69, 43, 41, 39, 38, 36, 38, 41, 43, 46, 48, 50, 52, 54, 55, 56, 57, + 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 69, 70, 72, 72, 72, 72, 44, 42, + 40, 38, 37, 39, 41, 43, 46, 48, 50, 52, 55, 56, 57, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 69, 70, 71, 73, 74, 74, 74, 74, 44, 42, 40, 39, 37, 39, + 41, 43, 46, 48, 50, 53, 56, 57, 59, 60, 62, 63, 64, 66, 67, 68, 69, 70, + 72, 73, 74, 76, 77, 77, 77, 77, 45, 43, 41, 39, 37, 39, 41, 43, 46, 48, + 51, 53, 56, 58, 60, 62, 64, 65, 67, 68, 70, 71, 72, 73, 75, 76, 77, 79, + 80, 80, 80, 80, 47, 44, 42, 40, 38, 40, 42, 44, 47, 49, 51, 54, 57, 59, + 61, 63, 65, 67, 68, 70, 72, 73, 74, 76, 77, 78, 80, 81, 83, 83, 83, 83, + 48, 46, 43, 41, 39, 41, 43, 45, 47, 50, 52, 55, 58, 60, 62, 64, 67, 68, + 70, 72, 74, 75, 77, 78, 80, 81, 83, 84, 86, 86, 86, 86, 50, 47, 45, 42, + 40, 42, 44, 46, 48, 50, 53, 56, 59, 61, 63, 66, 68, 70, 72, 74, 76, 77, + 79, 81, 82, 84, 85, 87, 89, 89, 89, 89, 52, 49, 46, 43, 41, 43, 45, 47, + 49, 51, 54, 56, 59, 62, 64, 67, 70, 72, 74, 76, 78, 80, 82, 83, 85, 87, + 88, 90, 92, 92, 92, 92, 53, 50, 47, 45, 43, 44, 46, 48, 50, 52, 55, 57, + 60, 63, 65, 68, 71, 73, 75, 77, 80, 82, 83, 85, 87, 89, 91, 92, 94, 94, + 94, 94, 55, 52, 49, 46, 44, 46, 47, 49, 51, 53, 56, 58, 61, 64, 66, 69, + 72, 74, 77, 79, 82, 83, 85, 87, 90, 91, 93, 95, 97, 97, 97, 97, 57, 54, + 51, 48, 46, 47, 49, 51, 52, 55, 57, 60, 62, 65, 67, 70, 73, 76, 78, 81, + 83, 85, 87, 90, 92, 94, 95, 97, 99, 99, 99, 99, 60, 56, 53, 50, 47, 49, + 50, 52, 54, 56, 58, 61, 64, 66, 69, 72, 75, 77, 80, 82, 85, 87, 90, 92, + 94, 96, 98, 100, 102, 102, 102, 102, 62, 58, 55, 52, 49, 50, 52, 54, 55, + 57, 60, 62, 65, 67, 70, 73, 76, 78, 81, 84, 87, 89, 91, 94, 96, 98, 100, + 102, 104, 104, 104, 104, 64, 60, 57, 54, 51, 52, 54, 55, 57, 59, 61, 64, + 66, 69, 71, 74, 77, 80, 83, 85, 88, 91, 93, 95, 98, 100, 102, 104, 107, + 107, 107, 107, 66, 62, 59, 56, 53, 54, 56, 57, 59, 61, 63, 65, 68, 70, + 73, 76, 79, 81, 84, 87, 90, 92, 95, 97, 100, 102, 104, 107, 109, 109, + 109, 109, 69, 65, 61, 58, 55, 56, 58, 59, 61, 63, 65, 67, 69, 72, 74, + 77, 80, 83, 86, 89, 92, 94, 97, 99, 102, 104, 107, 109, 111, 111, 111, + 111, 69, 65, 61, 58, 55, 56, 58, 59, 61, 63, 65, 67, 69, 72, 74, 77, 80, + 83, 86, 89, 92, 94, 97, 99, 102, 104, 107, 109, 111, 111, 111, 111, 69, + 65, 61, 58, 55, 56, 58, 59, 61, 63, 65, 67, 69, 72, 74, 77, 80, 83, 86, + 89, 92, 94, 97, 99, 102, 104, 107, 109, 111, 111, 111, 111, 69, 65, 61, + 58, 55, 56, 58, 59, 61, 63, 65, 67, 69, 72, 74, 77, 80, 83, 86, 89, 92, + 94, 97, 99, 102, 104, 107, 109, 111, 111, 111, 111 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 70, 116, 182, 70, 106, 148, 199, 116, 148, 207, 251, 182, 199, 251, + 292, + /* Size 8 */ + 64, 48, 52, 69, 92, 120, 150, 179, 48, 55, 53, 62, 79, 102, 128, 155, + 52, 53, 74, 87, 101, 121, 144, 168, 69, 62, 87, 110, 129, 147, 166, 186, + 92, 79, 101, 129, 153, 172, 190, 207, 120, 102, 121, 147, 172, 194, 211, + 226, 150, 128, 144, 166, 190, 211, 228, 242, 179, 155, 168, 186, 207, + 226, 242, 255, + /* Size 16 */ + 64, 55, 48, 50, 52, 59, 69, 79, 92, 104, 120, 133, 150, 163, 179, 179, + 55, 53, 51, 52, 52, 58, 65, 74, 85, 96, 110, 123, 138, 151, 166, 166, + 48, 51, 55, 54, 53, 57, 62, 70, 79, 89, 102, 113, 128, 140, 155, 155, + 50, 52, 54, 58, 62, 67, 72, 80, 89, 99, 111, 122, 135, 147, 161, 161, + 52, 52, 53, 62, 74, 80, 87, 93, 101, 110, 121, 131, 144, 155, 168, 168, + 59, 58, 57, 67, 80, 88, 97, 105, 113, 122, 133, 143, 154, 165, 177, 177, + 69, 65, 62, 72, 87, 97, 110, 119, 129, 137, 147, 156, 166, 176, 186, + 186, 79, 74, 70, 80, 93, 105, 119, 128, 140, 149, 158, 167, 177, 186, + 196, 196, 92, 85, 79, 89, 101, 113, 129, 140, 153, 162, 172, 181, 190, + 198, 207, 207, 104, 96, 89, 99, 110, 122, 137, 149, 162, 171, 182, 191, + 200, 208, 216, 216, 120, 110, 102, 111, 121, 133, 147, 158, 172, 182, + 194, 202, 211, 218, 226, 226, 133, 123, 113, 122, 131, 143, 156, 167, + 181, 191, 202, 210, 219, 227, 234, 234, 150, 138, 128, 135, 144, 154, + 166, 177, 190, 200, 211, 219, 228, 235, 242, 242, 163, 151, 140, 147, + 155, 165, 176, 186, 198, 208, 218, 227, 235, 242, 249, 249, 179, 166, + 155, 161, 168, 177, 186, 196, 207, 216, 226, 234, 242, 249, 255, 255, + 179, 166, 155, 161, 168, 177, 186, 196, 207, 216, 226, 234, 242, 249, + 255, 255, + /* Size 32 */ + 64, 59, 55, 51, 48, 49, 50, 51, 52, 55, 59, 64, 69, 73, 79, 85, 92, 98, + 104, 112, 120, 126, 133, 141, 150, 156, 163, 170, 179, 179, 179, 179, + 59, 56, 54, 52, 50, 50, 51, 52, 52, 55, 59, 62, 67, 71, 76, 82, 89, 94, + 100, 107, 115, 121, 128, 135, 144, 150, 156, 164, 172, 172, 172, 172, + 55, 54, 53, 52, 51, 52, 52, 52, 52, 55, 58, 61, 65, 69, 74, 79, 85, 90, + 96, 103, 110, 116, 123, 130, 138, 144, 151, 158, 166, 166, 166, 166, 51, + 52, 52, 53, 53, 53, 53, 53, 53, 55, 58, 60, 63, 67, 72, 77, 82, 87, 93, + 99, 106, 112, 118, 125, 133, 139, 145, 152, 160, 160, 160, 160, 48, 50, + 51, 53, 55, 55, 54, 53, 53, 55, 57, 59, 62, 66, 70, 74, 79, 84, 89, 95, + 102, 107, 113, 120, 128, 134, 140, 147, 155, 155, 155, 155, 49, 50, 52, + 53, 55, 55, 56, 56, 57, 59, 61, 64, 67, 70, 74, 79, 84, 89, 94, 100, + 106, 112, 117, 124, 131, 137, 144, 150, 158, 158, 158, 158, 50, 51, 52, + 53, 54, 56, 58, 60, 62, 64, 67, 69, 72, 76, 80, 84, 89, 94, 99, 104, + 111, 116, 122, 128, 135, 141, 147, 154, 161, 161, 161, 161, 51, 52, 52, + 53, 53, 56, 60, 63, 67, 70, 73, 76, 79, 82, 86, 90, 95, 99, 104, 110, + 116, 121, 126, 133, 139, 145, 151, 157, 164, 164, 164, 164, 52, 52, 52, + 53, 53, 57, 62, 67, 74, 77, 80, 83, 87, 90, 93, 97, 101, 106, 110, 115, + 121, 126, 131, 137, 144, 149, 155, 161, 168, 168, 168, 168, 55, 55, 55, + 55, 55, 59, 64, 70, 77, 80, 84, 87, 92, 95, 99, 103, 107, 111, 116, 121, + 126, 131, 137, 142, 149, 154, 160, 166, 172, 172, 172, 172, 59, 59, 58, + 58, 57, 61, 67, 73, 80, 84, 88, 92, 97, 101, 105, 109, 113, 118, 122, + 127, 133, 137, 143, 148, 154, 159, 165, 170, 177, 177, 177, 177, 64, 62, + 61, 60, 59, 64, 69, 76, 83, 87, 92, 97, 103, 107, 111, 116, 121, 125, + 129, 134, 139, 144, 149, 154, 160, 165, 170, 176, 181, 181, 181, 181, + 69, 67, 65, 63, 62, 67, 72, 79, 87, 92, 97, 103, 110, 114, 119, 124, + 129, 133, 137, 142, 147, 151, 156, 161, 166, 171, 176, 181, 186, 186, + 186, 186, 73, 71, 69, 67, 66, 70, 76, 82, 90, 95, 101, 107, 114, 119, + 123, 129, 134, 138, 143, 147, 152, 157, 161, 166, 172, 176, 181, 186, + 191, 191, 191, 191, 79, 76, 74, 72, 70, 74, 80, 86, 93, 99, 105, 111, + 119, 123, 128, 134, 140, 144, 149, 153, 158, 163, 167, 172, 177, 182, + 186, 191, 196, 196, 196, 196, 85, 82, 79, 77, 74, 79, 84, 90, 97, 103, + 109, 116, 124, 129, 134, 140, 146, 150, 155, 160, 165, 169, 174, 178, + 183, 188, 192, 197, 202, 202, 202, 202, 92, 89, 85, 82, 79, 84, 89, 95, + 101, 107, 113, 121, 129, 134, 140, 146, 153, 157, 162, 167, 172, 176, + 181, 185, 190, 194, 198, 203, 207, 207, 207, 207, 98, 94, 90, 87, 84, + 89, 94, 99, 106, 111, 118, 125, 133, 138, 144, 150, 157, 162, 167, 172, + 177, 181, 186, 190, 195, 199, 203, 207, 212, 212, 212, 212, 104, 100, + 96, 93, 89, 94, 99, 104, 110, 116, 122, 129, 137, 143, 149, 155, 162, + 167, 171, 177, 182, 186, 191, 195, 200, 204, 208, 212, 216, 216, 216, + 216, 112, 107, 103, 99, 95, 100, 104, 110, 115, 121, 127, 134, 142, 147, + 153, 160, 167, 172, 177, 182, 188, 192, 196, 201, 205, 209, 213, 217, + 221, 221, 221, 221, 120, 115, 110, 106, 102, 106, 111, 116, 121, 126, + 133, 139, 147, 152, 158, 165, 172, 177, 182, 188, 194, 198, 202, 206, + 211, 215, 218, 222, 226, 226, 226, 226, 126, 121, 116, 112, 107, 112, + 116, 121, 126, 131, 137, 144, 151, 157, 163, 169, 176, 181, 186, 192, + 198, 202, 206, 211, 215, 219, 222, 226, 230, 230, 230, 230, 133, 128, + 123, 118, 113, 117, 122, 126, 131, 137, 143, 149, 156, 161, 167, 174, + 181, 186, 191, 196, 202, 206, 210, 215, 219, 223, 227, 230, 234, 234, + 234, 234, 141, 135, 130, 125, 120, 124, 128, 133, 137, 142, 148, 154, + 161, 166, 172, 178, 185, 190, 195, 201, 206, 211, 215, 219, 224, 227, + 231, 234, 238, 238, 238, 238, 150, 144, 138, 133, 128, 131, 135, 139, + 144, 149, 154, 160, 166, 172, 177, 183, 190, 195, 200, 205, 211, 215, + 219, 224, 228, 232, 235, 239, 242, 242, 242, 242, 156, 150, 144, 139, + 134, 137, 141, 145, 149, 154, 159, 165, 171, 176, 182, 188, 194, 199, + 204, 209, 215, 219, 223, 227, 232, 235, 238, 242, 246, 246, 246, 246, + 163, 156, 151, 145, 140, 144, 147, 151, 155, 160, 165, 170, 176, 181, + 186, 192, 198, 203, 208, 213, 218, 222, 227, 231, 235, 238, 242, 245, + 249, 249, 249, 249, 170, 164, 158, 152, 147, 150, 154, 157, 161, 166, + 170, 176, 181, 186, 191, 197, 203, 207, 212, 217, 222, 226, 230, 234, + 239, 242, 245, 249, 252, 252, 252, 252, 179, 172, 166, 160, 155, 158, + 161, 164, 168, 172, 177, 181, 186, 191, 196, 202, 207, 212, 216, 221, + 226, 230, 234, 238, 242, 246, 249, 252, 255, 255, 255, 255, 179, 172, + 166, 160, 155, 158, 161, 164, 168, 172, 177, 181, 186, 191, 196, 202, + 207, 212, 216, 221, 226, 230, 234, 238, 242, 246, 249, 252, 255, 255, + 255, 255, 179, 172, 166, 160, 155, 158, 161, 164, 168, 172, 177, 181, + 186, 191, 196, 202, 207, 212, 216, 221, 226, 230, 234, 238, 242, 246, + 249, 252, 255, 255, 255, 255, 179, 172, 166, 160, 155, 158, 161, 164, + 168, 172, 177, 181, 186, 191, 196, 202, 207, 212, 216, 221, 226, 230, + 234, 238, 242, 246, 249, 252, 255, 255, 255, 255 }, + { /* Intra matrices */ + /* Size 4 */ + 21, 23, 40, 65, 23, 36, 52, 72, 40, 52, 75, 93, 65, 72, 93, 111, + /* Size 8 */ + 26, 19, 21, 28, 38, 50, 63, 77, 19, 22, 21, 25, 32, 42, 53, 66, 21, 21, + 30, 35, 42, 50, 61, 72, 28, 25, 35, 46, 54, 62, 71, 81, 38, 32, 42, 54, + 65, 74, 83, 92, 50, 42, 50, 62, 74, 85, 94, 101, 63, 53, 61, 71, 83, 94, + 103, 110, 77, 66, 72, 81, 92, 101, 110, 117, + /* Size 16 */ + 25, 21, 18, 19, 20, 23, 26, 31, 36, 41, 48, 54, 61, 67, 74, 74, 21, 20, + 20, 20, 20, 22, 25, 29, 33, 38, 44, 49, 56, 61, 68, 68, 18, 20, 21, 21, + 20, 22, 24, 27, 31, 35, 40, 45, 51, 57, 63, 63, 19, 20, 21, 22, 24, 26, + 28, 31, 35, 39, 44, 49, 55, 60, 66, 66, 20, 20, 20, 24, 29, 31, 34, 37, + 40, 44, 48, 53, 58, 63, 69, 69, 23, 22, 22, 26, 31, 34, 38, 41, 45, 49, + 53, 58, 63, 68, 73, 73, 26, 25, 24, 28, 34, 38, 44, 47, 52, 55, 60, 64, + 68, 73, 78, 78, 31, 29, 27, 31, 37, 41, 47, 52, 57, 60, 65, 69, 74, 78, + 83, 83, 36, 33, 31, 35, 40, 45, 52, 57, 62, 67, 71, 75, 80, 84, 88, 88, + 41, 38, 35, 39, 44, 49, 55, 60, 67, 71, 76, 80, 84, 88, 92, 92, 48, 44, + 40, 44, 48, 53, 60, 65, 71, 76, 81, 85, 90, 94, 97, 97, 54, 49, 45, 49, + 53, 58, 64, 69, 75, 80, 85, 90, 94, 98, 101, 101, 61, 56, 51, 55, 58, + 63, 68, 74, 80, 84, 90, 94, 99, 102, 106, 106, 67, 61, 57, 60, 63, 68, + 73, 78, 84, 88, 94, 98, 102, 105, 109, 109, 74, 68, 63, 66, 69, 73, 78, + 83, 88, 92, 97, 101, 106, 109, 112, 112, 74, 68, 63, 66, 69, 73, 78, 83, + 88, 92, 97, 101, 106, 109, 112, 112, + /* Size 32 */ + 24, 22, 21, 19, 18, 18, 19, 19, 19, 21, 22, 24, 26, 28, 30, 32, 35, 38, + 40, 43, 47, 50, 53, 56, 60, 63, 66, 69, 73, 73, 73, 73, 22, 21, 20, 19, + 19, 19, 19, 19, 19, 21, 22, 23, 25, 27, 29, 31, 34, 36, 39, 41, 45, 47, + 50, 53, 57, 60, 63, 66, 70, 70, 70, 70, 21, 20, 20, 19, 19, 19, 19, 19, + 20, 21, 22, 23, 25, 26, 28, 30, 33, 35, 37, 40, 43, 45, 48, 51, 55, 57, + 60, 63, 67, 67, 67, 67, 19, 19, 19, 20, 20, 20, 20, 20, 20, 21, 22, 23, + 24, 25, 27, 29, 31, 33, 36, 38, 41, 43, 46, 49, 52, 55, 58, 61, 64, 64, + 64, 64, 18, 19, 19, 20, 21, 20, 20, 20, 20, 21, 21, 22, 23, 25, 26, 28, + 30, 32, 34, 37, 39, 42, 44, 47, 50, 53, 56, 59, 62, 62, 62, 62, 18, 19, + 19, 20, 20, 21, 21, 21, 21, 22, 23, 24, 25, 27, 28, 30, 32, 34, 36, 38, + 41, 43, 46, 49, 52, 54, 57, 60, 63, 63, 63, 63, 19, 19, 19, 20, 20, 21, + 22, 22, 23, 24, 25, 26, 27, 29, 30, 32, 34, 36, 38, 40, 43, 45, 48, 50, + 53, 56, 59, 62, 65, 65, 65, 65, 19, 19, 19, 20, 20, 21, 22, 24, 25, 26, + 27, 29, 30, 31, 33, 35, 36, 38, 40, 43, 45, 47, 50, 52, 55, 58, 60, 63, + 66, 66, 66, 66, 19, 19, 20, 20, 20, 21, 23, 25, 28, 29, 30, 32, 33, 34, + 36, 37, 39, 41, 43, 45, 47, 49, 52, 54, 57, 59, 62, 65, 68, 68, 68, 68, + 21, 21, 21, 21, 21, 22, 24, 26, 29, 30, 32, 33, 35, 37, 38, 40, 42, 43, + 45, 47, 50, 52, 54, 57, 59, 62, 64, 67, 70, 70, 70, 70, 22, 22, 22, 22, + 21, 23, 25, 27, 30, 32, 34, 35, 37, 39, 40, 42, 44, 46, 48, 50, 52, 54, + 57, 59, 62, 64, 66, 69, 72, 72, 72, 72, 24, 23, 23, 23, 22, 24, 26, 29, + 32, 33, 35, 38, 40, 42, 43, 45, 47, 49, 51, 53, 55, 57, 59, 62, 64, 66, + 69, 71, 74, 74, 74, 74, 26, 25, 25, 24, 23, 25, 27, 30, 33, 35, 37, 40, + 43, 45, 46, 48, 51, 52, 54, 56, 58, 60, 62, 65, 67, 69, 71, 74, 76, 76, + 76, 76, 28, 27, 26, 25, 25, 27, 29, 31, 34, 37, 39, 42, 45, 46, 48, 51, + 53, 55, 57, 59, 61, 63, 65, 67, 70, 72, 74, 76, 79, 79, 79, 79, 30, 29, + 28, 27, 26, 28, 30, 33, 36, 38, 40, 43, 46, 48, 51, 53, 55, 57, 59, 61, + 64, 66, 68, 70, 72, 74, 76, 79, 81, 81, 81, 81, 32, 31, 30, 29, 28, 30, + 32, 35, 37, 40, 42, 45, 48, 51, 53, 55, 58, 60, 62, 64, 67, 69, 71, 73, + 75, 77, 79, 81, 83, 83, 83, 83, 35, 34, 33, 31, 30, 32, 34, 36, 39, 42, + 44, 47, 51, 53, 55, 58, 61, 63, 65, 67, 70, 72, 74, 76, 78, 80, 82, 84, + 86, 86, 86, 86, 38, 36, 35, 33, 32, 34, 36, 38, 41, 43, 46, 49, 52, 55, + 57, 60, 63, 65, 67, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 88, 88, 88, + 40, 39, 37, 36, 34, 36, 38, 40, 43, 45, 48, 51, 54, 57, 59, 62, 65, 67, + 70, 72, 74, 76, 78, 81, 83, 85, 86, 88, 91, 91, 91, 91, 43, 41, 40, 38, + 37, 38, 40, 43, 45, 47, 50, 53, 56, 59, 61, 64, 67, 70, 72, 74, 77, 79, + 81, 83, 85, 87, 89, 91, 93, 93, 93, 93, 47, 45, 43, 41, 39, 41, 43, 45, + 47, 50, 52, 55, 58, 61, 64, 67, 70, 72, 74, 77, 80, 82, 84, 86, 88, 90, + 92, 94, 95, 95, 95, 95, 50, 47, 45, 43, 42, 43, 45, 47, 49, 52, 54, 57, + 60, 63, 66, 69, 72, 74, 76, 79, 82, 84, 86, 88, 90, 92, 94, 95, 97, 97, + 97, 97, 53, 50, 48, 46, 44, 46, 48, 50, 52, 54, 57, 59, 62, 65, 68, 71, + 74, 76, 78, 81, 84, 86, 88, 90, 92, 94, 96, 97, 99, 99, 99, 99, 56, 53, + 51, 49, 47, 49, 50, 52, 54, 57, 59, 62, 65, 67, 70, 73, 76, 78, 81, 83, + 86, 88, 90, 92, 94, 96, 98, 100, 101, 101, 101, 101, 60, 57, 55, 52, 50, + 52, 53, 55, 57, 59, 62, 64, 67, 70, 72, 75, 78, 80, 83, 85, 88, 90, 92, + 94, 97, 98, 100, 102, 104, 104, 104, 104, 63, 60, 57, 55, 53, 54, 56, + 58, 59, 62, 64, 66, 69, 72, 74, 77, 80, 82, 85, 87, 90, 92, 94, 96, 98, + 100, 102, 103, 105, 105, 105, 105, 66, 63, 60, 58, 56, 57, 59, 60, 62, + 64, 66, 69, 71, 74, 76, 79, 82, 84, 86, 89, 92, 94, 96, 98, 100, 102, + 103, 105, 107, 107, 107, 107, 69, 66, 63, 61, 59, 60, 62, 63, 65, 67, + 69, 71, 74, 76, 79, 81, 84, 86, 88, 91, 94, 95, 97, 100, 102, 103, 105, + 107, 108, 108, 108, 108, 73, 70, 67, 64, 62, 63, 65, 66, 68, 70, 72, 74, + 76, 79, 81, 83, 86, 88, 91, 93, 95, 97, 99, 101, 104, 105, 107, 108, + 110, 110, 110, 110, 73, 70, 67, 64, 62, 63, 65, 66, 68, 70, 72, 74, 76, + 79, 81, 83, 86, 88, 91, 93, 95, 97, 99, 101, 104, 105, 107, 108, 110, + 110, 110, 110, 73, 70, 67, 64, 62, 63, 65, 66, 68, 70, 72, 74, 76, 79, + 81, 83, 86, 88, 91, 93, 95, 97, 99, 101, 104, 105, 107, 108, 110, 110, + 110, 110, 73, 70, 67, 64, 62, 63, 65, 66, 68, 70, 72, 74, 76, 79, 81, + 83, 86, 88, 91, 93, 95, 97, 99, 101, 104, 105, 107, 108, 110, 110, 110, + 110 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 101, 110, 139, 101, 122, 130, 149, 110, 130, 167, 190, 139, 149, + 190, 225, + /* Size 8 */ + 64, 52, 94, 99, 106, 119, 134, 152, 52, 75, 96, 87, 89, 98, 110, 125, + 94, 96, 112, 108, 108, 113, 123, 136, 99, 87, 108, 122, 128, 134, 142, + 153, 106, 89, 108, 128, 143, 153, 162, 172, 119, 98, 113, 134, 153, 168, + 180, 191, 134, 110, 123, 142, 162, 180, 195, 207, 152, 125, 136, 153, + 172, 191, 207, 221, + /* Size 16 */ + 64, 57, 52, 67, 94, 97, 99, 103, 106, 112, 119, 126, 134, 142, 152, 152, + 57, 59, 61, 74, 95, 94, 93, 95, 97, 102, 107, 114, 121, 129, 137, 137, + 52, 61, 75, 84, 96, 91, 87, 88, 89, 93, 98, 104, 110, 117, 125, 125, 67, + 74, 84, 92, 103, 99, 96, 97, 97, 101, 105, 110, 116, 123, 130, 130, 94, + 95, 96, 103, 112, 110, 108, 108, 108, 110, 113, 118, 123, 129, 136, 136, + 97, 94, 91, 99, 110, 112, 115, 116, 117, 120, 123, 127, 132, 138, 144, + 144, 99, 93, 87, 96, 108, 115, 122, 125, 128, 131, 134, 138, 142, 147, + 153, 153, 103, 95, 88, 97, 108, 116, 125, 130, 135, 139, 143, 147, 151, + 156, 162, 162, 106, 97, 89, 97, 108, 117, 128, 135, 143, 148, 153, 157, + 162, 167, 172, 172, 112, 102, 93, 101, 110, 120, 131, 139, 148, 154, + 160, 165, 171, 176, 181, 181, 119, 107, 98, 105, 113, 123, 134, 143, + 153, 160, 168, 174, 180, 185, 191, 191, 126, 114, 104, 110, 118, 127, + 138, 147, 157, 165, 174, 180, 187, 193, 199, 199, 134, 121, 110, 116, + 123, 132, 142, 151, 162, 171, 180, 187, 195, 201, 207, 207, 142, 129, + 117, 123, 129, 138, 147, 156, 167, 176, 185, 193, 201, 207, 214, 214, + 152, 137, 125, 130, 136, 144, 153, 162, 172, 181, 191, 199, 207, 214, + 221, 221, 152, 137, 125, 130, 136, 144, 153, 162, 172, 181, 191, 199, + 207, 214, 221, 221, + /* Size 32 */ + 64, 60, 57, 54, 52, 58, 67, 78, 94, 95, 97, 98, 99, 101, 103, 104, 106, + 109, 112, 115, 119, 122, 126, 130, 134, 138, 142, 147, 152, 152, 152, + 152, 60, 59, 58, 57, 56, 62, 70, 80, 94, 95, 95, 95, 96, 97, 99, 100, + 101, 104, 107, 110, 113, 116, 120, 123, 127, 131, 135, 139, 144, 144, + 144, 144, 57, 58, 59, 60, 61, 67, 74, 83, 95, 94, 94, 93, 93, 94, 95, + 96, 97, 99, 102, 104, 107, 110, 114, 117, 121, 125, 129, 133, 137, 137, + 137, 137, 54, 57, 60, 63, 67, 72, 79, 86, 95, 94, 92, 91, 89, 90, 91, + 92, 93, 95, 97, 100, 102, 105, 109, 112, 115, 119, 123, 127, 131, 131, + 131, 131, 52, 56, 61, 67, 75, 79, 84, 89, 96, 93, 91, 89, 87, 87, 88, + 88, 89, 91, 93, 96, 98, 101, 104, 107, 110, 114, 117, 121, 125, 125, + 125, 125, 58, 62, 67, 72, 79, 83, 88, 93, 99, 97, 95, 93, 91, 92, 92, + 93, 93, 95, 97, 99, 101, 104, 107, 110, 113, 117, 120, 124, 128, 128, + 128, 128, 67, 70, 74, 79, 84, 88, 92, 97, 103, 101, 99, 98, 96, 96, 97, + 97, 97, 99, 101, 103, 105, 108, 110, 113, 116, 120, 123, 127, 130, 130, + 130, 130, 78, 80, 83, 86, 89, 93, 97, 102, 107, 106, 104, 103, 102, 102, + 102, 102, 102, 104, 106, 107, 109, 112, 114, 117, 120, 123, 126, 130, + 133, 133, 133, 133, 94, 94, 95, 95, 96, 99, 103, 107, 112, 111, 110, + 109, 108, 108, 108, 108, 108, 109, 110, 112, 113, 116, 118, 121, 123, + 126, 129, 133, 136, 136, 136, 136, 95, 95, 94, 94, 93, 97, 101, 106, + 111, 111, 111, 111, 111, 111, 112, 112, 112, 113, 115, 116, 118, 120, + 122, 125, 127, 130, 133, 137, 140, 140, 140, 140, 97, 95, 94, 92, 91, + 95, 99, 104, 110, 111, 112, 113, 115, 115, 116, 116, 117, 118, 120, 121, + 123, 125, 127, 129, 132, 135, 138, 141, 144, 144, 144, 144, 98, 95, 93, + 91, 89, 93, 98, 103, 109, 111, 113, 116, 118, 119, 120, 121, 122, 124, + 125, 127, 128, 130, 132, 134, 137, 139, 142, 145, 148, 148, 148, 148, + 99, 96, 93, 89, 87, 91, 96, 102, 108, 111, 115, 118, 122, 124, 125, 127, + 128, 130, 131, 132, 134, 136, 138, 140, 142, 144, 147, 150, 153, 153, + 153, 153, 101, 97, 94, 90, 87, 92, 96, 102, 108, 111, 115, 119, 124, + 125, 127, 129, 132, 133, 135, 136, 138, 140, 142, 144, 146, 149, 152, + 154, 157, 157, 157, 157, 103, 99, 95, 91, 88, 92, 97, 102, 108, 112, + 116, 120, 125, 127, 130, 133, 135, 137, 139, 141, 143, 145, 147, 149, + 151, 154, 156, 159, 162, 162, 162, 162, 104, 100, 96, 92, 88, 93, 97, + 102, 108, 112, 116, 121, 127, 129, 133, 136, 139, 141, 143, 145, 148, + 150, 152, 154, 156, 159, 161, 164, 167, 167, 167, 167, 106, 101, 97, 93, + 89, 93, 97, 102, 108, 112, 117, 122, 128, 132, 135, 139, 143, 145, 148, + 150, 153, 155, 157, 160, 162, 164, 167, 169, 172, 172, 172, 172, 109, + 104, 99, 95, 91, 95, 99, 104, 109, 113, 118, 124, 130, 133, 137, 141, + 145, 148, 151, 154, 157, 159, 161, 164, 166, 169, 171, 174, 176, 176, + 176, 176, 112, 107, 102, 97, 93, 97, 101, 106, 110, 115, 120, 125, 131, + 135, 139, 143, 148, 151, 154, 157, 160, 163, 165, 168, 171, 173, 176, + 178, 181, 181, 181, 181, 115, 110, 104, 100, 96, 99, 103, 107, 112, 116, + 121, 127, 132, 136, 141, 145, 150, 154, 157, 161, 164, 167, 170, 172, + 175, 178, 180, 183, 186, 186, 186, 186, 119, 113, 107, 102, 98, 101, + 105, 109, 113, 118, 123, 128, 134, 138, 143, 148, 153, 157, 160, 164, + 168, 171, 174, 177, 180, 183, 185, 188, 191, 191, 191, 191, 122, 116, + 110, 105, 101, 104, 108, 112, 116, 120, 125, 130, 136, 140, 145, 150, + 155, 159, 163, 167, 171, 174, 177, 180, 184, 186, 189, 192, 195, 195, + 195, 195, 126, 120, 114, 109, 104, 107, 110, 114, 118, 122, 127, 132, + 138, 142, 147, 152, 157, 161, 165, 170, 174, 177, 180, 184, 187, 190, + 193, 196, 199, 199, 199, 199, 130, 123, 117, 112, 107, 110, 113, 117, + 121, 125, 129, 134, 140, 144, 149, 154, 160, 164, 168, 172, 177, 180, + 184, 187, 191, 194, 197, 200, 203, 203, 203, 203, 134, 127, 121, 115, + 110, 113, 116, 120, 123, 127, 132, 137, 142, 146, 151, 156, 162, 166, + 171, 175, 180, 184, 187, 191, 195, 198, 201, 204, 207, 207, 207, 207, + 138, 131, 125, 119, 114, 117, 120, 123, 126, 130, 135, 139, 144, 149, + 154, 159, 164, 169, 173, 178, 183, 186, 190, 194, 198, 201, 204, 207, + 211, 211, 211, 211, 142, 135, 129, 123, 117, 120, 123, 126, 129, 133, + 138, 142, 147, 152, 156, 161, 167, 171, 176, 180, 185, 189, 193, 197, + 201, 204, 207, 211, 214, 214, 214, 214, 147, 139, 133, 127, 121, 124, + 127, 130, 133, 137, 141, 145, 150, 154, 159, 164, 169, 174, 178, 183, + 188, 192, 196, 200, 204, 207, 211, 214, 217, 217, 217, 217, 152, 144, + 137, 131, 125, 128, 130, 133, 136, 140, 144, 148, 153, 157, 162, 167, + 172, 176, 181, 186, 191, 195, 199, 203, 207, 211, 214, 217, 221, 221, + 221, 221, 152, 144, 137, 131, 125, 128, 130, 133, 136, 140, 144, 148, + 153, 157, 162, 167, 172, 176, 181, 186, 191, 195, 199, 203, 207, 211, + 214, 217, 221, 221, 221, 221, 152, 144, 137, 131, 125, 128, 130, 133, + 136, 140, 144, 148, 153, 157, 162, 167, 172, 176, 181, 186, 191, 195, + 199, 203, 207, 211, 214, 217, 221, 221, 221, 221, 152, 144, 137, 131, + 125, 128, 130, 133, 136, 140, 144, 148, 153, 157, 162, 167, 172, 176, + 181, 186, 191, 195, 199, 203, 207, 211, 214, 217, 221, 221, 221, 221 }, + { /* Intra matrices */ + /* Size 4 */ + 27, 44, 48, 61, 44, 53, 57, 66, 48, 57, 74, 86, 61, 66, 86, 103, + /* Size 8 */ + 29, 23, 43, 46, 49, 56, 63, 72, 23, 34, 44, 40, 41, 45, 51, 59, 43, 44, + 52, 50, 50, 53, 58, 64, 46, 40, 50, 57, 60, 63, 67, 73, 49, 41, 50, 60, + 68, 73, 78, 83, 56, 45, 53, 63, 73, 81, 88, 93, 63, 51, 58, 67, 78, 88, + 96, 102, 72, 59, 64, 73, 83, 93, 102, 110, + /* Size 16 */ + 28, 25, 23, 29, 42, 43, 45, 46, 48, 51, 54, 58, 62, 66, 70, 70, 25, 26, + 27, 33, 42, 42, 42, 43, 44, 46, 49, 52, 55, 59, 63, 63, 23, 27, 33, 37, + 43, 41, 39, 39, 40, 42, 44, 47, 50, 53, 57, 57, 29, 33, 37, 41, 46, 45, + 43, 44, 44, 46, 47, 50, 53, 56, 60, 60, 42, 42, 43, 46, 51, 50, 49, 49, + 49, 50, 51, 54, 56, 59, 63, 63, 43, 42, 41, 45, 50, 51, 52, 53, 53, 55, + 56, 58, 61, 63, 67, 67, 45, 42, 39, 43, 49, 52, 56, 57, 59, 60, 61, 63, + 65, 68, 71, 71, 46, 43, 39, 44, 49, 53, 57, 60, 62, 64, 66, 68, 70, 73, + 75, 75, 48, 44, 40, 44, 49, 53, 59, 62, 66, 69, 71, 73, 76, 78, 81, 81, + 51, 46, 42, 46, 50, 55, 60, 64, 69, 72, 75, 77, 80, 83, 85, 85, 54, 49, + 44, 47, 51, 56, 61, 66, 71, 75, 79, 82, 85, 88, 91, 91, 58, 52, 47, 50, + 54, 58, 63, 68, 73, 77, 82, 85, 89, 92, 95, 95, 62, 55, 50, 53, 56, 61, + 65, 70, 76, 80, 85, 89, 93, 96, 99, 99, 66, 59, 53, 56, 59, 63, 68, 73, + 78, 83, 88, 92, 96, 99, 103, 103, 70, 63, 57, 60, 63, 67, 71, 75, 81, + 85, 91, 95, 99, 103, 107, 107, 70, 63, 57, 60, 63, 67, 71, 75, 81, 85, + 91, 95, 99, 103, 107, 107, + /* Size 32 */ + 28, 26, 25, 23, 22, 25, 29, 34, 41, 42, 43, 43, 44, 45, 46, 46, 47, 49, + 50, 52, 53, 55, 57, 59, 61, 63, 65, 67, 69, 69, 69, 69, 26, 26, 25, 25, + 24, 27, 31, 35, 42, 42, 42, 42, 42, 43, 44, 44, 45, 46, 48, 49, 50, 52, + 54, 55, 57, 59, 61, 63, 66, 66, 66, 66, 25, 25, 26, 26, 26, 29, 32, 37, + 42, 42, 41, 41, 41, 41, 42, 42, 43, 44, 45, 46, 48, 49, 51, 53, 54, 56, + 58, 60, 62, 62, 62, 62, 23, 25, 26, 28, 29, 32, 34, 38, 42, 41, 41, 40, + 39, 40, 40, 41, 41, 42, 43, 44, 45, 47, 48, 50, 52, 53, 55, 57, 59, 59, + 59, 59, 22, 24, 26, 29, 33, 35, 37, 39, 42, 41, 40, 39, 38, 38, 39, 39, + 39, 40, 41, 42, 43, 45, 46, 48, 49, 51, 53, 54, 56, 56, 56, 56, 25, 27, + 29, 32, 35, 37, 39, 41, 44, 43, 42, 41, 40, 40, 41, 41, 41, 42, 43, 44, + 45, 46, 48, 49, 51, 52, 54, 56, 58, 58, 58, 58, 29, 31, 32, 34, 37, 39, + 41, 43, 46, 45, 44, 43, 43, 43, 43, 43, 43, 44, 45, 46, 47, 48, 49, 51, + 52, 54, 55, 57, 59, 59, 59, 59, 34, 35, 37, 38, 39, 41, 43, 45, 48, 47, + 46, 46, 45, 45, 45, 45, 45, 46, 47, 48, 49, 50, 51, 52, 54, 55, 57, 58, + 60, 60, 60, 60, 41, 42, 42, 42, 42, 44, 46, 48, 50, 49, 49, 49, 48, 48, + 48, 48, 48, 49, 49, 50, 51, 52, 53, 54, 55, 57, 58, 60, 62, 62, 62, 62, + 42, 42, 42, 41, 41, 43, 45, 47, 49, 50, 50, 50, 50, 50, 50, 50, 50, 51, + 51, 52, 53, 54, 55, 56, 57, 59, 60, 62, 64, 64, 64, 64, 43, 42, 41, 41, + 40, 42, 44, 46, 49, 50, 50, 51, 51, 52, 52, 52, 52, 53, 54, 54, 55, 56, + 57, 58, 60, 61, 62, 64, 65, 65, 65, 65, 43, 42, 41, 40, 39, 41, 43, 46, + 49, 50, 51, 52, 53, 54, 54, 54, 55, 56, 56, 57, 58, 59, 60, 61, 62, 63, + 65, 66, 68, 68, 68, 68, 44, 42, 41, 39, 38, 40, 43, 45, 48, 50, 51, 53, + 55, 56, 56, 57, 58, 58, 59, 60, 61, 61, 62, 63, 64, 66, 67, 68, 70, 70, + 70, 70, 45, 43, 41, 40, 38, 40, 43, 45, 48, 50, 52, 54, 56, 57, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 71, 72, 72, 72, 72, 46, 44, + 42, 40, 39, 41, 43, 45, 48, 50, 52, 54, 56, 57, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 70, 72, 73, 74, 74, 74, 74, 46, 44, 42, 41, 39, 41, + 43, 45, 48, 50, 52, 54, 57, 58, 60, 61, 63, 64, 65, 66, 67, 68, 70, 71, + 72, 73, 74, 75, 77, 77, 77, 77, 47, 45, 43, 41, 39, 41, 43, 45, 48, 50, + 52, 55, 58, 59, 61, 63, 65, 66, 67, 69, 70, 71, 72, 73, 75, 76, 77, 78, + 79, 79, 79, 79, 49, 46, 44, 42, 40, 42, 44, 46, 49, 51, 53, 56, 58, 60, + 62, 64, 66, 68, 69, 70, 72, 73, 74, 75, 77, 78, 79, 80, 82, 82, 82, 82, + 50, 48, 45, 43, 41, 43, 45, 47, 49, 51, 54, 56, 59, 61, 63, 65, 67, 69, + 70, 72, 74, 75, 76, 78, 79, 80, 81, 83, 84, 84, 84, 84, 52, 49, 46, 44, + 42, 44, 46, 48, 50, 52, 54, 57, 60, 62, 64, 66, 69, 70, 72, 74, 76, 77, + 78, 80, 81, 83, 84, 85, 87, 87, 87, 87, 53, 50, 48, 45, 43, 45, 47, 49, + 51, 53, 55, 58, 61, 63, 65, 67, 70, 72, 74, 76, 78, 79, 81, 82, 84, 85, + 86, 88, 89, 89, 89, 89, 55, 52, 49, 47, 45, 46, 48, 50, 52, 54, 56, 59, + 61, 64, 66, 68, 71, 73, 75, 77, 79, 81, 82, 84, 86, 87, 88, 90, 91, 91, + 91, 91, 57, 54, 51, 48, 46, 48, 49, 51, 53, 55, 57, 60, 62, 65, 67, 70, + 72, 74, 76, 78, 81, 82, 84, 86, 87, 89, 90, 92, 93, 93, 93, 93, 59, 55, + 53, 50, 48, 49, 51, 52, 54, 56, 58, 61, 63, 66, 68, 71, 73, 75, 78, 80, + 82, 84, 86, 87, 89, 91, 92, 94, 96, 96, 96, 96, 61, 57, 54, 52, 49, 51, + 52, 54, 55, 57, 60, 62, 64, 67, 69, 72, 75, 77, 79, 81, 84, 86, 87, 89, + 91, 93, 95, 96, 98, 98, 98, 98, 63, 59, 56, 53, 51, 52, 54, 55, 57, 59, + 61, 63, 66, 68, 70, 73, 76, 78, 80, 83, 85, 87, 89, 91, 93, 95, 96, 98, + 100, 100, 100, 100, 65, 61, 58, 55, 53, 54, 55, 57, 58, 60, 62, 65, 67, + 69, 72, 74, 77, 79, 81, 84, 86, 88, 90, 92, 95, 96, 98, 100, 102, 102, + 102, 102, 67, 63, 60, 57, 54, 56, 57, 58, 60, 62, 64, 66, 68, 71, 73, + 75, 78, 80, 83, 85, 88, 90, 92, 94, 96, 98, 100, 102, 103, 103, 103, + 103, 69, 66, 62, 59, 56, 58, 59, 60, 62, 64, 65, 68, 70, 72, 74, 77, 79, + 82, 84, 87, 89, 91, 93, 96, 98, 100, 102, 103, 105, 105, 105, 105, 69, + 66, 62, 59, 56, 58, 59, 60, 62, 64, 65, 68, 70, 72, 74, 77, 79, 82, 84, + 87, 89, 91, 93, 96, 98, 100, 102, 103, 105, 105, 105, 105, 69, 66, 62, + 59, 56, 58, 59, 60, 62, 64, 65, 68, 70, 72, 74, 77, 79, 82, 84, 87, 89, + 91, 93, 96, 98, 100, 102, 103, 105, 105, 105, 105, 69, 66, 62, 59, 56, + 58, 59, 60, 62, 64, 65, 68, 70, 72, 74, 77, 79, 82, 84, 87, 89, 91, 93, + 96, 98, 100, 102, 103, 105, 105, 105, 105 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 70, 112, 168, 70, 103, 139, 181, 112, 139, 188, 221, 168, 181, 221, + 251, + /* Size 8 */ + 64, 49, 53, 68, 90, 115, 139, 163, 49, 56, 53, 62, 78, 99, 121, 144, 53, + 53, 74, 85, 98, 115, 135, 154, 68, 62, 85, 106, 122, 137, 153, 169, 90, + 78, 98, 122, 142, 158, 172, 185, 115, 99, 115, 137, 158, 174, 188, 199, + 139, 121, 135, 153, 172, 188, 200, 210, 163, 144, 154, 169, 185, 199, + 210, 220, + /* Size 16 */ + 64, 55, 49, 51, 53, 59, 68, 78, 90, 101, 115, 126, 139, 150, 163, 163, + 55, 54, 52, 52, 53, 58, 65, 73, 84, 94, 106, 117, 130, 140, 153, 153, + 49, 52, 56, 54, 53, 57, 62, 69, 78, 87, 99, 109, 121, 132, 144, 144, 51, + 52, 54, 58, 62, 66, 72, 79, 87, 96, 106, 116, 128, 137, 149, 149, 53, + 53, 53, 62, 74, 79, 85, 91, 98, 106, 115, 124, 135, 144, 154, 154, 59, + 58, 57, 66, 79, 86, 94, 101, 109, 117, 125, 134, 143, 152, 161, 161, 68, + 65, 62, 72, 85, 94, 106, 114, 122, 129, 137, 145, 153, 160, 169, 169, + 78, 73, 69, 79, 91, 101, 114, 122, 131, 139, 147, 154, 162, 169, 176, + 176, 90, 84, 78, 87, 98, 109, 122, 131, 142, 149, 158, 164, 172, 178, + 185, 185, 101, 94, 87, 96, 106, 117, 129, 139, 149, 157, 166, 172, 179, + 185, 191, 191, 115, 106, 99, 106, 115, 125, 137, 147, 158, 166, 174, + 181, 188, 193, 199, 199, 126, 117, 109, 116, 124, 134, 145, 154, 164, + 172, 181, 187, 194, 199, 204, 204, 139, 130, 121, 128, 135, 143, 153, + 162, 172, 179, 188, 194, 200, 205, 210, 210, 150, 140, 132, 137, 144, + 152, 160, 169, 178, 185, 193, 199, 205, 210, 215, 215, 163, 153, 144, + 149, 154, 161, 169, 176, 185, 191, 199, 204, 210, 215, 220, 220, 163, + 153, 144, 149, 154, 161, 169, 176, 185, 191, 199, 204, 210, 215, 220, + 220, + /* Size 32 */ + 64, 59, 55, 52, 49, 50, 51, 52, 53, 56, 59, 64, 68, 73, 78, 83, 90, 95, + 101, 107, 115, 120, 126, 132, 139, 145, 150, 156, 163, 163, 163, 163, + 59, 57, 54, 52, 50, 51, 51, 52, 53, 56, 59, 63, 67, 71, 75, 81, 87, 92, + 97, 103, 110, 115, 121, 127, 134, 140, 145, 151, 158, 158, 158, 158, 55, + 54, 54, 53, 52, 52, 52, 53, 53, 55, 58, 62, 65, 69, 73, 78, 84, 88, 94, + 99, 106, 111, 117, 123, 130, 135, 140, 146, 153, 153, 153, 153, 52, 52, + 53, 53, 54, 54, 53, 53, 53, 55, 58, 61, 63, 67, 71, 76, 81, 85, 90, 96, + 102, 107, 113, 119, 125, 130, 136, 142, 148, 148, 148, 148, 49, 50, 52, + 54, 56, 55, 54, 54, 53, 55, 57, 60, 62, 65, 69, 73, 78, 83, 87, 93, 99, + 104, 109, 115, 121, 126, 132, 137, 144, 144, 144, 144, 50, 51, 52, 54, + 55, 56, 56, 57, 57, 59, 62, 64, 67, 70, 74, 78, 82, 87, 91, 97, 102, + 107, 112, 118, 124, 129, 134, 140, 146, 146, 146, 146, 51, 51, 52, 53, + 54, 56, 58, 60, 62, 64, 66, 69, 72, 75, 79, 83, 87, 91, 96, 101, 106, + 111, 116, 122, 128, 132, 137, 143, 149, 149, 149, 149, 52, 52, 53, 53, + 54, 57, 60, 63, 67, 70, 72, 75, 78, 81, 84, 88, 92, 96, 101, 105, 111, + 115, 120, 125, 131, 136, 140, 146, 151, 151, 151, 151, 53, 53, 53, 53, + 53, 57, 62, 67, 74, 76, 79, 82, 85, 88, 91, 95, 98, 102, 106, 111, 115, + 120, 124, 129, 135, 139, 144, 149, 154, 154, 154, 154, 56, 56, 55, 55, + 55, 59, 64, 70, 76, 79, 82, 86, 89, 93, 96, 99, 103, 107, 111, 115, 120, + 124, 129, 134, 139, 143, 148, 152, 158, 158, 158, 158, 59, 59, 58, 58, + 57, 62, 66, 72, 79, 82, 86, 90, 94, 98, 101, 105, 109, 113, 117, 121, + 125, 129, 134, 138, 143, 147, 152, 156, 161, 161, 161, 161, 64, 63, 62, + 61, 60, 64, 69, 75, 82, 86, 90, 95, 100, 103, 107, 111, 115, 119, 122, + 127, 131, 135, 139, 143, 148, 152, 156, 160, 165, 165, 165, 165, 68, 67, + 65, 63, 62, 67, 72, 78, 85, 89, 94, 100, 106, 110, 114, 118, 122, 125, + 129, 133, 137, 141, 145, 149, 153, 157, 160, 165, 169, 169, 169, 169, + 73, 71, 69, 67, 65, 70, 75, 81, 88, 93, 98, 103, 110, 113, 118, 122, + 126, 130, 134, 138, 142, 145, 149, 153, 157, 161, 164, 168, 173, 173, + 173, 173, 78, 75, 73, 71, 69, 74, 79, 84, 91, 96, 101, 107, 114, 118, + 122, 126, 131, 135, 139, 142, 147, 150, 154, 158, 162, 165, 169, 172, + 176, 176, 176, 176, 83, 81, 78, 76, 73, 78, 83, 88, 95, 99, 105, 111, + 118, 122, 126, 131, 136, 140, 144, 148, 152, 155, 159, 163, 166, 170, + 173, 177, 180, 180, 180, 180, 90, 87, 84, 81, 78, 82, 87, 92, 98, 103, + 109, 115, 122, 126, 131, 136, 142, 146, 149, 153, 158, 161, 164, 168, + 172, 175, 178, 181, 185, 185, 185, 185, 95, 92, 88, 85, 83, 87, 91, 96, + 102, 107, 113, 119, 125, 130, 135, 140, 146, 149, 153, 157, 162, 165, + 168, 172, 175, 178, 181, 185, 188, 188, 188, 188, 101, 97, 94, 90, 87, + 91, 96, 101, 106, 111, 117, 122, 129, 134, 139, 144, 149, 153, 157, 161, + 166, 169, 172, 176, 179, 182, 185, 188, 191, 191, 191, 191, 107, 103, + 99, 96, 93, 97, 101, 105, 111, 115, 121, 127, 133, 138, 142, 148, 153, + 157, 161, 165, 170, 173, 176, 180, 183, 186, 189, 192, 195, 195, 195, + 195, 115, 110, 106, 102, 99, 102, 106, 111, 115, 120, 125, 131, 137, + 142, 147, 152, 158, 162, 166, 170, 174, 177, 181, 184, 188, 190, 193, + 196, 199, 199, 199, 199, 120, 115, 111, 107, 104, 107, 111, 115, 120, + 124, 129, 135, 141, 145, 150, 155, 161, 165, 169, 173, 177, 181, 184, + 187, 191, 193, 196, 199, 202, 202, 202, 202, 126, 121, 117, 113, 109, + 112, 116, 120, 124, 129, 134, 139, 145, 149, 154, 159, 164, 168, 172, + 176, 181, 184, 187, 190, 194, 196, 199, 202, 204, 204, 204, 204, 132, + 127, 123, 119, 115, 118, 122, 125, 129, 134, 138, 143, 149, 153, 158, + 163, 168, 172, 176, 180, 184, 187, 190, 194, 197, 199, 202, 205, 207, + 207, 207, 207, 139, 134, 130, 125, 121, 124, 128, 131, 135, 139, 143, + 148, 153, 157, 162, 166, 172, 175, 179, 183, 188, 191, 194, 197, 200, + 203, 205, 208, 210, 210, 210, 210, 145, 140, 135, 130, 126, 129, 132, + 136, 139, 143, 147, 152, 157, 161, 165, 170, 175, 178, 182, 186, 190, + 193, 196, 199, 203, 205, 208, 210, 213, 213, 213, 213, 150, 145, 140, + 136, 132, 134, 137, 140, 144, 148, 152, 156, 160, 164, 169, 173, 178, + 181, 185, 189, 193, 196, 199, 202, 205, 208, 210, 212, 215, 215, 215, + 215, 156, 151, 146, 142, 137, 140, 143, 146, 149, 152, 156, 160, 165, + 168, 172, 177, 181, 185, 188, 192, 196, 199, 202, 205, 208, 210, 212, + 215, 217, 217, 217, 217, 163, 158, 153, 148, 144, 146, 149, 151, 154, + 158, 161, 165, 169, 173, 176, 180, 185, 188, 191, 195, 199, 202, 204, + 207, 210, 213, 215, 217, 220, 220, 220, 220, 163, 158, 153, 148, 144, + 146, 149, 151, 154, 158, 161, 165, 169, 173, 176, 180, 185, 188, 191, + 195, 199, 202, 204, 207, 210, 213, 215, 217, 220, 220, 220, 220, 163, + 158, 153, 148, 144, 146, 149, 151, 154, 158, 161, 165, 169, 173, 176, + 180, 185, 188, 191, 195, 199, 202, 204, 207, 210, 213, 215, 217, 220, + 220, 220, 220, 163, 158, 153, 148, 144, 146, 149, 151, 154, 158, 161, + 165, 169, 173, 176, 180, 185, 188, 191, 195, 199, 202, 204, 207, 210, + 213, 215, 217, 220, 220, 220, 220 }, + { /* Intra matrices */ + /* Size 4 */ + 24, 26, 43, 67, 26, 39, 54, 72, 43, 54, 75, 91, 67, 72, 91, 105, + /* Size 8 */ + 28, 21, 23, 30, 41, 53, 65, 77, 21, 25, 23, 27, 35, 45, 56, 68, 23, 23, + 33, 38, 45, 53, 63, 73, 30, 27, 38, 49, 56, 64, 72, 81, 41, 35, 45, 56, + 67, 75, 82, 89, 53, 45, 53, 64, 75, 84, 91, 97, 65, 56, 63, 72, 82, 91, + 98, 104, 77, 68, 73, 81, 89, 97, 104, 109, + /* Size 16 */ + 27, 24, 21, 21, 22, 25, 29, 34, 39, 44, 51, 56, 63, 68, 75, 75, 24, 23, + 22, 22, 22, 25, 28, 32, 36, 41, 47, 52, 58, 63, 70, 70, 21, 22, 24, 23, + 23, 24, 27, 30, 34, 38, 43, 48, 54, 59, 65, 65, 21, 22, 23, 25, 26, 28, + 31, 34, 38, 42, 47, 52, 57, 62, 68, 68, 22, 22, 23, 26, 32, 34, 37, 40, + 43, 47, 51, 55, 61, 65, 70, 70, 25, 25, 24, 28, 34, 37, 41, 44, 48, 52, + 56, 60, 65, 69, 74, 74, 29, 28, 27, 31, 37, 41, 47, 50, 54, 58, 62, 65, + 70, 74, 78, 78, 34, 32, 30, 34, 40, 44, 50, 54, 59, 62, 67, 70, 74, 78, + 82, 82, 39, 36, 34, 38, 43, 48, 54, 59, 64, 68, 72, 76, 79, 83, 86, 86, + 44, 41, 38, 42, 47, 52, 58, 62, 68, 72, 76, 80, 83, 86, 90, 90, 51, 47, + 43, 47, 51, 56, 62, 67, 72, 76, 81, 84, 88, 91, 94, 94, 56, 52, 48, 52, + 55, 60, 65, 70, 76, 80, 84, 87, 91, 94, 97, 97, 63, 58, 54, 57, 61, 65, + 70, 74, 79, 83, 88, 91, 95, 97, 100, 100, 68, 63, 59, 62, 65, 69, 74, + 78, 83, 86, 91, 94, 97, 100, 103, 103, 75, 70, 65, 68, 70, 74, 78, 82, + 86, 90, 94, 97, 100, 103, 105, 105, 75, 70, 65, 68, 70, 74, 78, 82, 86, + 90, 94, 97, 100, 103, 105, 105, + /* Size 32 */ + 27, 25, 23, 22, 20, 21, 21, 21, 22, 23, 25, 27, 29, 31, 33, 36, 39, 41, + 43, 46, 50, 52, 55, 58, 62, 64, 67, 70, 73, 73, 73, 73, 25, 24, 23, 22, + 21, 21, 21, 22, 22, 23, 25, 26, 28, 30, 32, 34, 37, 39, 42, 45, 48, 50, + 53, 56, 59, 62, 65, 67, 71, 71, 71, 71, 23, 23, 22, 22, 22, 22, 22, 22, + 22, 23, 24, 26, 27, 29, 31, 33, 36, 38, 40, 43, 46, 48, 51, 54, 57, 60, + 62, 65, 68, 68, 68, 68, 22, 22, 22, 22, 22, 22, 22, 22, 22, 23, 24, 25, + 27, 28, 30, 32, 34, 36, 39, 41, 44, 46, 49, 52, 55, 57, 60, 63, 66, 66, + 66, 66, 20, 21, 22, 22, 23, 23, 23, 22, 22, 23, 24, 25, 26, 28, 29, 31, + 33, 35, 37, 40, 43, 45, 47, 50, 53, 55, 58, 61, 64, 64, 64, 64, 21, 21, + 22, 22, 23, 23, 23, 24, 24, 25, 26, 27, 28, 30, 31, 33, 35, 37, 39, 42, + 44, 46, 49, 52, 54, 57, 59, 62, 65, 65, 65, 65, 21, 21, 22, 22, 23, 23, + 24, 25, 26, 27, 28, 29, 30, 32, 33, 35, 37, 39, 41, 43, 46, 48, 51, 53, + 56, 58, 61, 63, 66, 66, 66, 66, 21, 22, 22, 22, 22, 24, 25, 27, 28, 29, + 30, 32, 33, 34, 36, 38, 40, 41, 43, 46, 48, 50, 52, 55, 58, 60, 62, 65, + 68, 68, 68, 68, 22, 22, 22, 22, 22, 24, 26, 28, 31, 32, 33, 35, 36, 38, + 39, 41, 42, 44, 46, 48, 50, 52, 54, 57, 59, 62, 64, 66, 69, 69, 69, 69, + 23, 23, 23, 23, 23, 25, 27, 29, 32, 34, 35, 37, 38, 40, 41, 43, 45, 46, + 48, 50, 52, 54, 57, 59, 61, 64, 66, 68, 71, 71, 71, 71, 25, 25, 24, 24, + 24, 26, 28, 30, 33, 35, 37, 39, 41, 42, 44, 45, 47, 49, 51, 53, 55, 57, + 59, 61, 64, 66, 68, 70, 73, 73, 73, 73, 27, 26, 26, 25, 25, 27, 29, 32, + 35, 37, 39, 41, 43, 45, 46, 48, 50, 52, 54, 56, 58, 60, 61, 64, 66, 68, + 70, 72, 74, 74, 74, 74, 29, 28, 27, 27, 26, 28, 30, 33, 36, 38, 41, 43, + 46, 48, 49, 51, 53, 55, 57, 59, 61, 62, 64, 66, 68, 70, 72, 74, 76, 76, + 76, 76, 31, 30, 29, 28, 28, 30, 32, 34, 38, 40, 42, 45, 48, 49, 51, 53, + 56, 57, 59, 61, 63, 65, 66, 68, 71, 72, 74, 76, 78, 78, 78, 78, 33, 32, + 31, 30, 29, 31, 33, 36, 39, 41, 44, 46, 49, 51, 53, 55, 58, 60, 61, 63, + 65, 67, 69, 71, 73, 75, 76, 78, 80, 80, 80, 80, 36, 34, 33, 32, 31, 33, + 35, 38, 41, 43, 45, 48, 51, 53, 55, 58, 60, 62, 64, 66, 68, 70, 71, 73, + 75, 77, 79, 80, 82, 82, 82, 82, 39, 37, 36, 34, 33, 35, 37, 40, 42, 45, + 47, 50, 53, 56, 58, 60, 63, 65, 67, 69, 71, 72, 74, 76, 78, 79, 81, 83, + 85, 85, 85, 85, 41, 39, 38, 36, 35, 37, 39, 41, 44, 46, 49, 52, 55, 57, + 60, 62, 65, 67, 69, 71, 73, 74, 76, 78, 80, 81, 83, 85, 86, 86, 86, 86, + 43, 42, 40, 39, 37, 39, 41, 43, 46, 48, 51, 54, 57, 59, 61, 64, 67, 69, + 70, 73, 75, 76, 78, 80, 82, 83, 85, 86, 88, 88, 88, 88, 46, 45, 43, 41, + 40, 42, 43, 46, 48, 50, 53, 56, 59, 61, 63, 66, 69, 71, 73, 75, 77, 79, + 80, 82, 84, 85, 87, 88, 90, 90, 90, 90, 50, 48, 46, 44, 43, 44, 46, 48, + 50, 52, 55, 58, 61, 63, 65, 68, 71, 73, 75, 77, 79, 81, 83, 84, 86, 87, + 89, 90, 92, 92, 92, 92, 52, 50, 48, 46, 45, 46, 48, 50, 52, 54, 57, 60, + 62, 65, 67, 70, 72, 74, 76, 79, 81, 82, 84, 86, 88, 89, 90, 92, 93, 93, + 93, 93, 55, 53, 51, 49, 47, 49, 51, 52, 54, 57, 59, 61, 64, 66, 69, 71, + 74, 76, 78, 80, 83, 84, 86, 88, 89, 91, 92, 94, 95, 95, 95, 95, 58, 56, + 54, 52, 50, 52, 53, 55, 57, 59, 61, 64, 66, 68, 71, 73, 76, 78, 80, 82, + 84, 86, 88, 89, 91, 92, 94, 95, 97, 97, 97, 97, 62, 59, 57, 55, 53, 54, + 56, 58, 59, 61, 64, 66, 68, 71, 73, 75, 78, 80, 82, 84, 86, 88, 89, 91, + 93, 94, 95, 97, 98, 98, 98, 98, 64, 62, 60, 57, 55, 57, 58, 60, 62, 64, + 66, 68, 70, 72, 75, 77, 79, 81, 83, 85, 87, 89, 91, 92, 94, 95, 97, 98, + 99, 99, 99, 99, 67, 65, 62, 60, 58, 59, 61, 62, 64, 66, 68, 70, 72, 74, + 76, 79, 81, 83, 85, 87, 89, 90, 92, 94, 95, 97, 98, 99, 101, 101, 101, + 101, 70, 67, 65, 63, 61, 62, 63, 65, 66, 68, 70, 72, 74, 76, 78, 80, 83, + 85, 86, 88, 90, 92, 94, 95, 97, 98, 99, 101, 102, 102, 102, 102, 73, 71, + 68, 66, 64, 65, 66, 68, 69, 71, 73, 74, 76, 78, 80, 82, 85, 86, 88, 90, + 92, 93, 95, 97, 98, 99, 101, 102, 103, 103, 103, 103, 73, 71, 68, 66, + 64, 65, 66, 68, 69, 71, 73, 74, 76, 78, 80, 82, 85, 86, 88, 90, 92, 93, + 95, 97, 98, 99, 101, 102, 103, 103, 103, 103, 73, 71, 68, 66, 64, 65, + 66, 68, 69, 71, 73, 74, 76, 78, 80, 82, 85, 86, 88, 90, 92, 93, 95, 97, + 98, 99, 101, 102, 103, 103, 103, 103, 73, 71, 68, 66, 64, 65, 66, 68, + 69, 71, 73, 74, 76, 78, 80, 82, 85, 86, 88, 90, 92, 93, 95, 97, 98, 99, + 101, 102, 103, 103, 103, 103 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 99, 107, 133, 99, 117, 124, 141, 107, 124, 156, 175, 133, 141, 175, + 203, + /* Size 8 */ + 64, 52, 92, 97, 103, 114, 128, 143, 52, 74, 93, 85, 87, 96, 107, 120, + 92, 93, 108, 105, 104, 109, 118, 129, 97, 85, 105, 117, 122, 127, 134, + 143, 103, 87, 104, 122, 135, 144, 151, 159, 114, 96, 109, 127, 144, 156, + 166, 175, 128, 107, 118, 134, 151, 166, 178, 188, 143, 120, 129, 143, + 159, 175, 188, 198, + /* Size 16 */ + 64, 57, 52, 66, 92, 94, 97, 100, 103, 108, 114, 121, 128, 135, 143, 143, + 57, 59, 61, 74, 93, 92, 91, 93, 95, 99, 104, 110, 116, 123, 130, 130, + 52, 61, 74, 83, 93, 89, 85, 86, 87, 91, 96, 101, 107, 113, 120, 120, 66, + 74, 83, 91, 100, 97, 94, 95, 95, 98, 102, 107, 112, 118, 124, 124, 92, + 93, 93, 100, 108, 106, 105, 104, 104, 107, 109, 114, 118, 123, 129, 129, + 94, 92, 89, 97, 106, 108, 111, 112, 113, 115, 118, 122, 126, 131, 136, + 136, 97, 91, 85, 94, 105, 111, 117, 120, 122, 125, 127, 131, 134, 139, + 143, 143, 100, 93, 86, 95, 104, 112, 120, 124, 129, 132, 135, 139, 142, + 146, 151, 151, 103, 95, 87, 95, 104, 113, 122, 129, 135, 139, 144, 147, + 151, 155, 159, 159, 108, 99, 91, 98, 107, 115, 125, 132, 139, 144, 150, + 154, 158, 162, 167, 167, 114, 104, 96, 102, 109, 118, 127, 135, 144, + 150, 156, 161, 166, 170, 175, 175, 121, 110, 101, 107, 114, 122, 131, + 139, 147, 154, 161, 166, 172, 176, 181, 181, 128, 116, 107, 112, 118, + 126, 134, 142, 151, 158, 166, 172, 178, 183, 188, 188, 135, 123, 113, + 118, 123, 131, 139, 146, 155, 162, 170, 176, 183, 188, 193, 193, 143, + 130, 120, 124, 129, 136, 143, 151, 159, 167, 175, 181, 188, 193, 198, + 198, 143, 130, 120, 124, 129, 136, 143, 151, 159, 167, 175, 181, 188, + 193, 198, 198, + /* Size 32 */ + 64, 61, 57, 55, 52, 58, 66, 77, 92, 93, 94, 96, 97, 98, 100, 101, 103, + 106, 108, 111, 114, 117, 121, 124, 128, 131, 135, 139, 143, 143, 143, + 143, 61, 59, 58, 57, 56, 62, 70, 79, 92, 93, 93, 93, 94, 95, 96, 97, 99, + 101, 104, 106, 109, 112, 115, 118, 122, 125, 129, 132, 136, 136, 136, + 136, 57, 58, 59, 60, 61, 67, 74, 82, 93, 92, 92, 91, 91, 92, 93, 94, 95, + 97, 99, 101, 104, 107, 110, 113, 116, 119, 123, 126, 130, 130, 130, 130, + 55, 57, 60, 63, 67, 72, 78, 85, 93, 92, 90, 89, 88, 89, 89, 90, 91, 93, + 95, 97, 100, 102, 105, 108, 111, 114, 118, 121, 125, 125, 125, 125, 52, + 56, 61, 67, 74, 78, 83, 88, 93, 91, 89, 87, 85, 86, 86, 87, 87, 89, 91, + 93, 96, 98, 101, 104, 107, 110, 113, 116, 120, 120, 120, 120, 58, 62, + 67, 72, 78, 82, 86, 91, 97, 95, 93, 91, 89, 90, 90, 91, 91, 93, 95, 97, + 99, 101, 104, 106, 109, 112, 115, 119, 122, 122, 122, 122, 66, 70, 74, + 78, 83, 86, 91, 95, 100, 99, 97, 95, 94, 94, 95, 95, 95, 97, 98, 100, + 102, 104, 107, 109, 112, 115, 118, 121, 124, 124, 124, 124, 77, 79, 82, + 85, 88, 91, 95, 99, 104, 103, 101, 100, 99, 99, 99, 99, 100, 101, 102, + 104, 106, 108, 110, 113, 115, 118, 121, 124, 127, 127, 127, 127, 92, 92, + 93, 93, 93, 97, 100, 104, 108, 107, 106, 105, 105, 105, 104, 104, 104, + 106, 107, 108, 109, 111, 114, 116, 118, 121, 123, 126, 129, 129, 129, + 129, 93, 93, 92, 92, 91, 95, 99, 103, 107, 107, 107, 107, 108, 108, 108, + 108, 108, 110, 111, 112, 113, 115, 117, 120, 122, 124, 127, 130, 133, + 133, 133, 133, 94, 93, 92, 90, 89, 93, 97, 101, 106, 107, 108, 109, 111, + 111, 112, 112, 113, 114, 115, 116, 118, 120, 122, 124, 126, 128, 131, + 133, 136, 136, 136, 136, 96, 93, 91, 89, 87, 91, 95, 100, 105, 107, 109, + 112, 114, 115, 116, 116, 117, 119, 120, 121, 122, 124, 126, 128, 130, + 132, 135, 137, 140, 140, 140, 140, 97, 94, 91, 88, 85, 89, 94, 99, 105, + 108, 111, 114, 117, 118, 120, 121, 122, 124, 125, 126, 127, 129, 131, + 132, 134, 136, 139, 141, 143, 143, 143, 143, 98, 95, 92, 89, 86, 90, 94, + 99, 105, 108, 111, 115, 118, 120, 122, 124, 125, 127, 128, 130, 131, + 133, 134, 136, 138, 140, 142, 145, 147, 147, 147, 147, 100, 96, 93, 89, + 86, 90, 95, 99, 104, 108, 112, 116, 120, 122, 124, 126, 129, 130, 132, + 133, 135, 137, 139, 140, 142, 144, 146, 149, 151, 151, 151, 151, 101, + 97, 94, 90, 87, 91, 95, 99, 104, 108, 112, 116, 121, 124, 126, 129, 132, + 134, 135, 137, 139, 141, 143, 145, 147, 149, 151, 153, 155, 155, 155, + 155, 103, 99, 95, 91, 87, 91, 95, 100, 104, 108, 113, 117, 122, 125, + 129, 132, 135, 137, 139, 142, 144, 146, 147, 149, 151, 153, 155, 157, + 159, 159, 159, 159, 106, 101, 97, 93, 89, 93, 97, 101, 106, 110, 114, + 119, 124, 127, 130, 134, 137, 140, 142, 144, 147, 149, 151, 153, 155, + 157, 159, 161, 163, 163, 163, 163, 108, 104, 99, 95, 91, 95, 98, 102, + 107, 111, 115, 120, 125, 128, 132, 135, 139, 142, 144, 147, 150, 152, + 154, 156, 158, 160, 162, 164, 167, 167, 167, 167, 111, 106, 101, 97, 93, + 97, 100, 104, 108, 112, 116, 121, 126, 130, 133, 137, 142, 144, 147, + 150, 153, 155, 157, 160, 162, 164, 166, 168, 170, 170, 170, 170, 114, + 109, 104, 100, 96, 99, 102, 106, 109, 113, 118, 122, 127, 131, 135, 139, + 144, 147, 150, 153, 156, 159, 161, 164, 166, 168, 170, 172, 175, 175, + 175, 175, 117, 112, 107, 102, 98, 101, 104, 108, 111, 115, 120, 124, + 129, 133, 137, 141, 146, 149, 152, 155, 159, 161, 164, 166, 169, 171, + 173, 175, 178, 178, 178, 178, 121, 115, 110, 105, 101, 104, 107, 110, + 114, 117, 122, 126, 131, 134, 139, 143, 147, 151, 154, 157, 161, 164, + 166, 169, 172, 174, 176, 178, 181, 181, 181, 181, 124, 118, 113, 108, + 104, 106, 109, 113, 116, 120, 124, 128, 132, 136, 140, 145, 149, 153, + 156, 160, 164, 166, 169, 172, 175, 177, 179, 182, 184, 184, 184, 184, + 128, 122, 116, 111, 107, 109, 112, 115, 118, 122, 126, 130, 134, 138, + 142, 147, 151, 155, 158, 162, 166, 169, 172, 175, 178, 180, 183, 185, + 188, 188, 188, 188, 131, 125, 119, 114, 110, 112, 115, 118, 121, 124, + 128, 132, 136, 140, 144, 149, 153, 157, 160, 164, 168, 171, 174, 177, + 180, 183, 185, 188, 190, 190, 190, 190, 135, 129, 123, 118, 113, 115, + 118, 121, 123, 127, 131, 135, 139, 142, 146, 151, 155, 159, 162, 166, + 170, 173, 176, 179, 183, 185, 188, 190, 193, 193, 193, 193, 139, 132, + 126, 121, 116, 119, 121, 124, 126, 130, 133, 137, 141, 145, 149, 153, + 157, 161, 164, 168, 172, 175, 178, 182, 185, 188, 190, 193, 196, 196, + 196, 196, 143, 136, 130, 125, 120, 122, 124, 127, 129, 133, 136, 140, + 143, 147, 151, 155, 159, 163, 167, 170, 175, 178, 181, 184, 188, 190, + 193, 196, 198, 198, 198, 198, 143, 136, 130, 125, 120, 122, 124, 127, + 129, 133, 136, 140, 143, 147, 151, 155, 159, 163, 167, 170, 175, 178, + 181, 184, 188, 190, 193, 196, 198, 198, 198, 198, 143, 136, 130, 125, + 120, 122, 124, 127, 129, 133, 136, 140, 143, 147, 151, 155, 159, 163, + 167, 170, 175, 178, 181, 184, 188, 190, 193, 196, 198, 198, 198, 198, + 143, 136, 130, 125, 120, 122, 124, 127, 129, 133, 136, 140, 143, 147, + 151, 155, 159, 163, 167, 170, 175, 178, 181, 184, 188, 190, 193, 196, + 198, 198, 198, 198 }, + { /* Intra matrices */ + /* Size 4 */ + 29, 46, 49, 62, 46, 54, 58, 66, 49, 58, 74, 84, 62, 66, 84, 99, + /* Size 8 */ + 31, 25, 45, 48, 51, 57, 64, 72, 25, 36, 46, 42, 43, 47, 53, 60, 45, 46, + 54, 52, 52, 54, 59, 65, 48, 42, 52, 59, 61, 64, 68, 73, 51, 43, 52, 61, + 68, 73, 77, 82, 57, 47, 54, 64, 73, 80, 86, 90, 64, 53, 59, 68, 77, 86, + 92, 98, 72, 60, 65, 73, 82, 90, 98, 104, + /* Size 16 */ + 30, 27, 24, 31, 44, 45, 47, 48, 50, 53, 56, 59, 63, 66, 71, 71, 27, 28, + 29, 35, 44, 44, 43, 44, 45, 48, 50, 53, 57, 60, 64, 64, 24, 29, 35, 39, + 45, 43, 41, 41, 42, 44, 46, 49, 52, 55, 58, 58, 31, 35, 39, 43, 48, 47, + 45, 45, 46, 47, 49, 52, 54, 58, 61, 61, 44, 44, 45, 48, 52, 51, 51, 51, + 50, 52, 53, 55, 58, 60, 63, 63, 45, 44, 43, 47, 51, 53, 54, 54, 55, 56, + 57, 59, 62, 64, 67, 67, 47, 43, 41, 45, 51, 54, 57, 58, 60, 61, 62, 64, + 66, 68, 71, 71, 48, 44, 41, 45, 51, 54, 58, 61, 63, 65, 67, 68, 70, 73, + 75, 75, 50, 45, 42, 46, 50, 55, 60, 63, 67, 69, 71, 73, 75, 77, 80, 80, + 53, 48, 44, 47, 52, 56, 61, 65, 69, 72, 74, 77, 79, 81, 84, 84, 56, 50, + 46, 49, 53, 57, 62, 67, 71, 74, 78, 81, 83, 86, 88, 88, 59, 53, 49, 52, + 55, 59, 64, 68, 73, 77, 81, 83, 87, 89, 92, 92, 63, 57, 52, 54, 58, 62, + 66, 70, 75, 79, 83, 87, 90, 93, 95, 95, 66, 60, 55, 58, 60, 64, 68, 73, + 77, 81, 86, 89, 93, 95, 98, 98, 71, 64, 58, 61, 63, 67, 71, 75, 80, 84, + 88, 92, 95, 98, 102, 102, 71, 64, 58, 61, 63, 67, 71, 75, 80, 84, 88, + 92, 95, 98, 102, 102, + /* Size 32 */ + 30, 28, 27, 25, 24, 27, 31, 36, 43, 44, 45, 45, 46, 47, 48, 48, 49, 50, + 52, 53, 55, 56, 58, 60, 62, 64, 65, 67, 70, 70, 70, 70, 28, 28, 27, 27, + 26, 29, 33, 37, 44, 44, 44, 44, 44, 45, 46, 46, 47, 48, 49, 51, 52, 54, + 55, 57, 59, 60, 62, 64, 66, 66, 66, 66, 27, 27, 28, 28, 28, 31, 35, 39, + 44, 44, 43, 43, 43, 43, 44, 44, 45, 46, 47, 48, 50, 51, 53, 54, 56, 58, + 59, 61, 63, 63, 63, 63, 25, 27, 28, 30, 31, 34, 37, 40, 44, 43, 43, 42, + 41, 42, 42, 43, 43, 44, 45, 46, 47, 49, 50, 52, 53, 55, 57, 58, 60, 60, + 60, 60, 24, 26, 28, 31, 35, 37, 39, 41, 44, 43, 42, 41, 40, 40, 41, 41, + 41, 42, 43, 44, 45, 47, 48, 49, 51, 53, 54, 56, 58, 58, 58, 58, 27, 29, + 31, 34, 37, 39, 41, 43, 46, 45, 44, 43, 42, 42, 43, 43, 43, 44, 45, 46, + 47, 48, 49, 51, 52, 54, 55, 57, 59, 59, 59, 59, 31, 33, 35, 37, 39, 41, + 43, 45, 48, 47, 46, 45, 45, 45, 45, 45, 45, 46, 47, 48, 49, 50, 51, 52, + 54, 55, 57, 58, 60, 60, 60, 60, 36, 37, 39, 40, 41, 43, 45, 47, 50, 49, + 48, 48, 47, 47, 47, 47, 47, 48, 49, 50, 50, 52, 53, 54, 55, 57, 58, 60, + 61, 61, 61, 61, 43, 44, 44, 44, 44, 46, 48, 50, 52, 51, 51, 50, 50, 50, + 50, 50, 50, 50, 51, 52, 52, 53, 54, 56, 57, 58, 60, 61, 63, 63, 63, 63, + 44, 44, 44, 43, 43, 45, 47, 49, 51, 51, 51, 51, 51, 51, 52, 52, 52, 52, + 53, 54, 54, 55, 56, 58, 59, 60, 61, 63, 64, 64, 64, 64, 45, 44, 43, 43, + 42, 44, 46, 48, 51, 51, 52, 52, 53, 53, 53, 54, 54, 55, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 65, 66, 66, 66, 66, 45, 44, 43, 42, 41, 43, 45, 48, + 50, 51, 52, 53, 55, 55, 55, 56, 56, 57, 58, 58, 59, 60, 61, 62, 63, 64, + 65, 67, 68, 68, 68, 68, 46, 44, 43, 41, 40, 42, 45, 47, 50, 51, 53, 55, + 56, 57, 58, 58, 59, 60, 60, 61, 62, 62, 63, 64, 65, 66, 68, 69, 70, 70, + 70, 70, 47, 45, 43, 42, 40, 42, 45, 47, 50, 51, 53, 55, 57, 58, 59, 60, + 61, 61, 62, 63, 64, 64, 65, 66, 67, 68, 70, 71, 72, 72, 72, 72, 48, 46, + 44, 42, 41, 43, 45, 47, 50, 52, 53, 55, 58, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 67, 68, 69, 71, 72, 73, 74, 74, 74, 74, 48, 46, 44, 43, 41, 43, + 45, 47, 50, 52, 54, 56, 58, 60, 61, 62, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 76, 76, 76, 49, 47, 45, 43, 41, 43, 45, 47, 50, 52, + 54, 56, 59, 61, 62, 64, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, + 79, 79, 79, 79, 50, 48, 46, 44, 42, 44, 46, 48, 50, 52, 55, 57, 60, 61, + 63, 65, 67, 68, 69, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 80, 80, 80, + 52, 49, 47, 45, 43, 45, 47, 49, 51, 53, 55, 58, 60, 62, 64, 66, 68, 69, + 71, 72, 73, 75, 76, 77, 78, 79, 80, 81, 82, 82, 82, 82, 53, 51, 48, 46, + 44, 46, 48, 50, 52, 54, 56, 58, 61, 63, 65, 67, 69, 70, 72, 74, 75, 76, + 78, 79, 80, 81, 82, 83, 85, 85, 85, 85, 55, 52, 50, 47, 45, 47, 49, 50, + 52, 54, 57, 59, 62, 64, 66, 68, 70, 72, 73, 75, 77, 78, 80, 81, 82, 83, + 84, 86, 87, 87, 87, 87, 56, 54, 51, 49, 47, 48, 50, 52, 53, 55, 58, 60, + 62, 64, 67, 69, 71, 73, 75, 76, 78, 80, 81, 82, 84, 85, 86, 87, 89, 89, + 89, 89, 58, 55, 53, 50, 48, 49, 51, 53, 54, 56, 59, 61, 63, 65, 67, 70, + 72, 74, 76, 78, 80, 81, 82, 84, 85, 87, 88, 89, 90, 90, 90, 90, 60, 57, + 54, 52, 49, 51, 52, 54, 56, 58, 60, 62, 64, 66, 68, 71, 73, 75, 77, 79, + 81, 82, 84, 85, 87, 88, 90, 91, 92, 92, 92, 92, 62, 59, 56, 53, 51, 52, + 54, 55, 57, 59, 61, 63, 65, 67, 69, 72, 74, 76, 78, 80, 82, 84, 85, 87, + 89, 90, 91, 93, 94, 94, 94, 94, 64, 60, 58, 55, 53, 54, 55, 57, 58, 60, + 62, 64, 66, 68, 71, 73, 75, 77, 79, 81, 83, 85, 87, 88, 90, 91, 93, 94, + 96, 96, 96, 96, 65, 62, 59, 57, 54, 55, 57, 58, 60, 61, 63, 65, 68, 70, + 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 91, 93, 94, 96, 97, 97, 97, 97, + 67, 64, 61, 58, 56, 57, 58, 60, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, + 81, 83, 86, 87, 89, 91, 93, 94, 96, 97, 99, 99, 99, 99, 70, 66, 63, 60, + 58, 59, 60, 61, 63, 64, 66, 68, 70, 72, 74, 76, 79, 80, 82, 85, 87, 89, + 90, 92, 94, 96, 97, 99, 100, 100, 100, 100, 70, 66, 63, 60, 58, 59, 60, + 61, 63, 64, 66, 68, 70, 72, 74, 76, 79, 80, 82, 85, 87, 89, 90, 92, 94, + 96, 97, 99, 100, 100, 100, 100, 70, 66, 63, 60, 58, 59, 60, 61, 63, 64, + 66, 68, 70, 72, 74, 76, 79, 80, 82, 85, 87, 89, 90, 92, 94, 96, 97, 99, + 100, 100, 100, 100, 70, 66, 63, 60, 58, 59, 60, 61, 63, 64, 66, 68, 70, + 72, 74, 76, 79, 80, 82, 85, 87, 89, 90, 92, 94, 96, 97, 99, 100, 100, + 100, 100 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 69, 108, 155, 69, 100, 131, 166, 108, 131, 171, 197, 155, 166, 197, + 218, + /* Size 8 */ + 64, 49, 53, 68, 88, 109, 130, 149, 49, 56, 54, 62, 77, 96, 115, 134, 53, + 54, 73, 83, 95, 110, 126, 142, 68, 62, 83, 102, 116, 128, 141, 154, 88, + 77, 95, 116, 132, 145, 156, 166, 109, 96, 110, 128, 145, 158, 168, 176, + 130, 115, 126, 141, 156, 168, 177, 185, 149, 134, 142, 154, 166, 176, + 185, 191, + /* Size 16 */ + 64, 56, 49, 51, 53, 60, 68, 77, 88, 97, 109, 119, 130, 139, 149, 149, + 56, 54, 52, 53, 53, 59, 65, 73, 82, 91, 102, 111, 122, 131, 141, 141, + 49, 52, 56, 55, 54, 58, 62, 69, 77, 85, 96, 104, 115, 124, 134, 134, 51, + 53, 55, 58, 62, 66, 71, 78, 85, 93, 102, 111, 120, 128, 138, 138, 53, + 53, 54, 62, 73, 78, 83, 89, 95, 102, 110, 117, 126, 134, 142, 142, 60, + 59, 58, 66, 78, 84, 92, 98, 104, 111, 118, 125, 133, 140, 148, 148, 68, + 65, 62, 71, 83, 92, 102, 108, 116, 122, 128, 134, 141, 147, 154, 154, + 77, 73, 69, 78, 89, 98, 108, 115, 123, 129, 136, 142, 148, 153, 159, + 159, 88, 82, 77, 85, 95, 104, 116, 123, 132, 138, 145, 150, 156, 161, + 166, 166, 97, 91, 85, 93, 102, 111, 122, 129, 138, 144, 151, 156, 162, + 166, 171, 171, 109, 102, 96, 102, 110, 118, 128, 136, 145, 151, 158, + 163, 168, 172, 176, 176, 119, 111, 104, 111, 117, 125, 134, 142, 150, + 156, 163, 168, 173, 176, 180, 180, 130, 122, 115, 120, 126, 133, 141, + 148, 156, 162, 168, 173, 177, 181, 185, 185, 139, 131, 124, 128, 134, + 140, 147, 153, 161, 166, 172, 176, 181, 184, 188, 188, 149, 141, 134, + 138, 142, 148, 154, 159, 166, 171, 176, 180, 185, 188, 191, 191, 149, + 141, 134, 138, 142, 148, 154, 159, 166, 171, 176, 180, 185, 188, 191, + 191, + /* Size 32 */ + 64, 60, 56, 52, 49, 50, 51, 52, 53, 56, 60, 64, 68, 72, 77, 82, 88, 92, + 97, 103, 109, 114, 119, 124, 130, 134, 139, 144, 149, 149, 149, 149, 60, + 57, 55, 53, 51, 51, 52, 53, 53, 56, 59, 63, 66, 70, 75, 79, 85, 89, 94, + 100, 106, 110, 115, 120, 126, 130, 135, 140, 145, 145, 145, 145, 56, 55, + 54, 53, 52, 53, 53, 53, 53, 56, 59, 62, 65, 69, 73, 77, 82, 86, 91, 96, + 102, 106, 111, 116, 122, 126, 131, 136, 141, 141, 141, 141, 52, 53, 53, + 54, 54, 54, 54, 54, 54, 56, 58, 61, 64, 67, 71, 75, 80, 84, 88, 93, 99, + 103, 108, 113, 118, 123, 127, 132, 137, 137, 137, 137, 49, 51, 52, 54, + 56, 55, 55, 54, 54, 56, 58, 60, 62, 65, 69, 73, 77, 81, 85, 90, 96, 100, + 104, 109, 115, 119, 124, 128, 134, 134, 134, 134, 50, 51, 53, 54, 55, + 56, 56, 57, 58, 60, 62, 64, 66, 70, 73, 77, 81, 85, 89, 94, 99, 103, + 107, 112, 118, 122, 126, 131, 136, 136, 136, 136, 51, 52, 53, 54, 55, + 56, 58, 60, 62, 64, 66, 69, 71, 74, 78, 81, 85, 89, 93, 97, 102, 106, + 111, 115, 120, 124, 128, 133, 138, 138, 138, 138, 52, 53, 53, 54, 54, + 57, 60, 63, 67, 69, 72, 74, 77, 80, 83, 86, 90, 94, 97, 101, 106, 110, + 114, 118, 123, 127, 131, 135, 140, 140, 140, 140, 53, 53, 53, 54, 54, + 58, 62, 67, 73, 75, 78, 80, 83, 86, 89, 92, 95, 99, 102, 106, 110, 114, + 117, 122, 126, 130, 134, 138, 142, 142, 142, 142, 56, 56, 56, 56, 56, + 60, 64, 69, 75, 78, 81, 84, 87, 90, 93, 96, 100, 103, 106, 110, 114, + 118, 121, 125, 130, 133, 137, 141, 145, 145, 145, 145, 60, 59, 59, 58, + 58, 62, 66, 72, 78, 81, 84, 88, 92, 95, 98, 101, 104, 108, 111, 115, + 118, 122, 125, 129, 133, 136, 140, 144, 148, 148, 148, 148, 64, 63, 62, + 61, 60, 64, 69, 74, 80, 84, 88, 92, 97, 100, 103, 106, 110, 113, 116, + 119, 123, 126, 130, 133, 137, 140, 143, 147, 151, 151, 151, 151, 68, 66, + 65, 64, 62, 66, 71, 77, 83, 87, 92, 97, 102, 105, 108, 112, 116, 119, + 122, 125, 128, 131, 134, 138, 141, 144, 147, 150, 154, 154, 154, 154, + 72, 70, 69, 67, 65, 70, 74, 80, 86, 90, 95, 100, 105, 108, 112, 116, + 119, 122, 125, 129, 132, 135, 138, 141, 144, 147, 150, 153, 156, 156, + 156, 156, 77, 75, 73, 71, 69, 73, 78, 83, 89, 93, 98, 103, 108, 112, + 115, 119, 123, 126, 129, 133, 136, 139, 142, 145, 148, 151, 153, 156, + 159, 159, 159, 159, 82, 79, 77, 75, 73, 77, 81, 86, 92, 96, 101, 106, + 112, 116, 119, 123, 128, 131, 134, 137, 140, 143, 146, 149, 152, 154, + 157, 160, 162, 162, 162, 162, 88, 85, 82, 80, 77, 81, 85, 90, 95, 100, + 104, 110, 116, 119, 123, 128, 132, 135, 138, 141, 145, 147, 150, 153, + 156, 158, 161, 163, 166, 166, 166, 166, 92, 89, 86, 84, 81, 85, 89, 94, + 99, 103, 108, 113, 119, 122, 126, 131, 135, 138, 141, 144, 148, 150, + 153, 156, 159, 161, 163, 166, 168, 168, 168, 168, 97, 94, 91, 88, 85, + 89, 93, 97, 102, 106, 111, 116, 122, 125, 129, 134, 138, 141, 144, 148, + 151, 154, 156, 159, 162, 164, 166, 168, 171, 171, 171, 171, 103, 100, + 96, 93, 90, 94, 97, 101, 106, 110, 115, 119, 125, 129, 133, 137, 141, + 144, 148, 151, 154, 157, 159, 162, 165, 167, 169, 171, 174, 174, 174, + 174, 109, 106, 102, 99, 96, 99, 102, 106, 110, 114, 118, 123, 128, 132, + 136, 140, 145, 148, 151, 154, 158, 160, 163, 165, 168, 170, 172, 174, + 176, 176, 176, 176, 114, 110, 106, 103, 100, 103, 106, 110, 114, 118, + 122, 126, 131, 135, 139, 143, 147, 150, 154, 157, 160, 163, 165, 168, + 170, 172, 174, 176, 178, 178, 178, 178, 119, 115, 111, 108, 104, 107, + 111, 114, 117, 121, 125, 130, 134, 138, 142, 146, 150, 153, 156, 159, + 163, 165, 168, 170, 173, 174, 176, 178, 180, 180, 180, 180, 124, 120, + 116, 113, 109, 112, 115, 118, 122, 125, 129, 133, 138, 141, 145, 149, + 153, 156, 159, 162, 165, 168, 170, 172, 175, 177, 179, 181, 183, 183, + 183, 183, 130, 126, 122, 118, 115, 118, 120, 123, 126, 130, 133, 137, + 141, 144, 148, 152, 156, 159, 162, 165, 168, 170, 173, 175, 177, 179, + 181, 183, 185, 185, 185, 185, 134, 130, 126, 123, 119, 122, 124, 127, + 130, 133, 136, 140, 144, 147, 151, 154, 158, 161, 164, 167, 170, 172, + 174, 177, 179, 181, 183, 185, 186, 186, 186, 186, 139, 135, 131, 127, + 124, 126, 128, 131, 134, 137, 140, 143, 147, 150, 153, 157, 161, 163, + 166, 169, 172, 174, 176, 179, 181, 183, 184, 186, 188, 188, 188, 188, + 144, 140, 136, 132, 128, 131, 133, 135, 138, 141, 144, 147, 150, 153, + 156, 160, 163, 166, 168, 171, 174, 176, 178, 181, 183, 185, 186, 188, + 190, 190, 190, 190, 149, 145, 141, 137, 134, 136, 138, 140, 142, 145, + 148, 151, 154, 156, 159, 162, 166, 168, 171, 174, 176, 178, 180, 183, + 185, 186, 188, 190, 191, 191, 191, 191, 149, 145, 141, 137, 134, 136, + 138, 140, 142, 145, 148, 151, 154, 156, 159, 162, 166, 168, 171, 174, + 176, 178, 180, 183, 185, 186, 188, 190, 191, 191, 191, 191, 149, 145, + 141, 137, 134, 136, 138, 140, 142, 145, 148, 151, 154, 156, 159, 162, + 166, 168, 171, 174, 176, 178, 180, 183, 185, 186, 188, 190, 191, 191, + 191, 191, 149, 145, 141, 137, 134, 136, 138, 140, 142, 145, 148, 151, + 154, 156, 159, 162, 166, 168, 171, 174, 176, 178, 180, 183, 185, 186, + 188, 190, 191, 191, 191, 191 }, + { /* Intra matrices */ + /* Size 4 */ + 26, 29, 46, 68, 29, 42, 57, 73, 46, 57, 75, 88, 68, 73, 88, 99, + /* Size 8 */ + 31, 24, 26, 33, 44, 55, 67, 77, 24, 27, 26, 30, 38, 48, 58, 69, 26, 26, + 36, 41, 47, 55, 64, 73, 33, 30, 41, 51, 59, 65, 73, 80, 44, 38, 47, 59, + 68, 75, 81, 87, 55, 48, 55, 65, 75, 82, 88, 93, 67, 58, 64, 73, 81, 88, + 94, 98, 77, 69, 73, 80, 87, 93, 98, 102, + /* Size 16 */ + 30, 26, 23, 24, 25, 28, 32, 37, 42, 47, 53, 58, 64, 69, 75, 75, 26, 25, + 25, 25, 25, 28, 31, 34, 39, 44, 49, 54, 60, 65, 70, 70, 23, 25, 26, 26, + 25, 27, 29, 33, 37, 41, 46, 51, 56, 61, 66, 66, 24, 25, 26, 27, 29, 31, + 34, 37, 41, 45, 50, 54, 59, 63, 69, 69, 25, 25, 25, 29, 35, 37, 40, 43, + 46, 49, 54, 58, 62, 66, 71, 71, 28, 28, 27, 31, 37, 40, 44, 47, 51, 54, + 58, 62, 66, 70, 74, 74, 32, 31, 29, 34, 40, 44, 50, 53, 57, 60, 63, 67, + 70, 74, 77, 77, 37, 34, 33, 37, 43, 47, 53, 57, 61, 64, 68, 71, 74, 77, + 81, 81, 42, 39, 37, 41, 46, 51, 57, 61, 66, 69, 72, 75, 79, 81, 84, 84, + 47, 44, 41, 45, 49, 54, 60, 64, 69, 72, 76, 79, 82, 84, 87, 87, 53, 49, + 46, 50, 54, 58, 63, 68, 72, 76, 80, 83, 85, 88, 90, 90, 58, 54, 51, 54, + 58, 62, 67, 71, 75, 79, 83, 85, 88, 90, 93, 93, 64, 60, 56, 59, 62, 66, + 70, 74, 79, 82, 85, 88, 91, 93, 95, 95, 69, 65, 61, 63, 66, 70, 74, 77, + 81, 84, 88, 90, 93, 95, 97, 97, 75, 70, 66, 69, 71, 74, 77, 81, 84, 87, + 90, 93, 95, 97, 99, 99, 75, 70, 66, 69, 71, 74, 77, 81, 84, 87, 90, 93, + 95, 97, 99, 99, + /* Size 32 */ + 30, 28, 26, 24, 23, 23, 24, 24, 24, 26, 28, 30, 32, 34, 36, 39, 42, 44, + 46, 49, 52, 55, 57, 60, 63, 66, 68, 71, 73, 73, 73, 73, 28, 26, 25, 24, + 23, 24, 24, 24, 25, 26, 27, 29, 31, 33, 35, 37, 40, 42, 45, 47, 50, 53, + 55, 58, 61, 63, 66, 68, 71, 71, 71, 71, 26, 25, 25, 25, 24, 24, 24, 25, + 25, 26, 27, 29, 30, 32, 34, 36, 39, 41, 43, 46, 49, 51, 53, 56, 59, 61, + 64, 66, 69, 69, 69, 69, 24, 24, 25, 25, 25, 25, 25, 25, 25, 26, 27, 28, + 29, 31, 33, 35, 37, 39, 42, 44, 47, 49, 52, 54, 57, 59, 62, 64, 67, 67, + 67, 67, 23, 23, 24, 25, 26, 26, 25, 25, 25, 26, 27, 28, 29, 30, 32, 34, + 36, 38, 40, 43, 45, 48, 50, 52, 55, 58, 60, 62, 65, 65, 65, 65, 23, 24, + 24, 25, 26, 26, 26, 26, 27, 28, 29, 30, 31, 32, 34, 36, 38, 40, 42, 44, + 47, 49, 51, 54, 57, 59, 61, 64, 66, 66, 66, 66, 24, 24, 24, 25, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 35, 36, 38, 40, 42, 44, 46, 49, 51, 53, 55, + 58, 60, 62, 65, 67, 67, 67, 67, 24, 24, 25, 25, 25, 26, 28, 29, 31, 32, + 33, 35, 36, 37, 39, 41, 43, 44, 46, 48, 51, 53, 55, 57, 60, 62, 64, 66, + 69, 69, 69, 69, 24, 25, 25, 25, 25, 27, 29, 31, 34, 35, 36, 38, 39, 41, + 42, 44, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 70, 70, 70, 70, + 26, 26, 26, 26, 26, 28, 30, 32, 35, 37, 38, 40, 41, 43, 44, 46, 47, 49, + 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 71, 71, 71, 28, 27, 27, 27, + 27, 29, 31, 33, 36, 38, 40, 41, 43, 45, 46, 48, 50, 52, 53, 55, 57, 59, + 61, 63, 65, 67, 69, 71, 73, 73, 73, 73, 30, 29, 29, 28, 28, 30, 32, 35, + 38, 40, 41, 44, 46, 47, 49, 51, 53, 54, 56, 58, 60, 61, 63, 65, 67, 69, + 71, 72, 74, 74, 74, 74, 32, 31, 30, 29, 29, 31, 33, 36, 39, 41, 43, 46, + 49, 50, 52, 54, 56, 57, 59, 61, 62, 64, 66, 67, 69, 71, 72, 74, 76, 76, + 76, 76, 34, 33, 32, 31, 30, 32, 35, 37, 41, 43, 45, 47, 50, 52, 54, 56, + 58, 59, 61, 63, 64, 66, 68, 69, 71, 73, 74, 76, 78, 78, 78, 78, 36, 35, + 34, 33, 32, 34, 36, 39, 42, 44, 46, 49, 52, 54, 56, 58, 60, 61, 63, 65, + 66, 68, 70, 71, 73, 74, 76, 78, 79, 79, 79, 79, 39, 37, 36, 35, 34, 36, + 38, 41, 44, 46, 48, 51, 54, 56, 58, 60, 62, 64, 65, 67, 69, 70, 72, 73, + 75, 76, 78, 79, 81, 81, 81, 81, 42, 40, 39, 37, 36, 38, 40, 43, 45, 47, + 50, 53, 56, 58, 60, 62, 64, 66, 68, 69, 71, 73, 74, 76, 77, 79, 80, 81, + 83, 83, 83, 83, 44, 42, 41, 39, 38, 40, 42, 44, 47, 49, 52, 54, 57, 59, + 61, 64, 66, 68, 69, 71, 73, 74, 76, 77, 79, 80, 81, 83, 84, 84, 84, 84, + 46, 45, 43, 42, 40, 42, 44, 46, 49, 51, 53, 56, 59, 61, 63, 65, 68, 69, + 71, 73, 75, 76, 77, 79, 81, 82, 83, 84, 86, 86, 86, 86, 49, 47, 46, 44, + 43, 44, 46, 48, 51, 53, 55, 58, 61, 63, 65, 67, 69, 71, 73, 75, 77, 78, + 79, 81, 82, 83, 85, 86, 87, 87, 87, 87, 52, 50, 49, 47, 45, 47, 49, 51, + 53, 55, 57, 60, 62, 64, 66, 69, 71, 73, 75, 77, 78, 80, 81, 83, 84, 85, + 86, 88, 89, 89, 89, 89, 55, 53, 51, 49, 48, 49, 51, 53, 55, 57, 59, 61, + 64, 66, 68, 70, 73, 74, 76, 78, 80, 81, 82, 84, 85, 86, 88, 89, 90, 90, + 90, 90, 57, 55, 53, 52, 50, 51, 53, 55, 57, 59, 61, 63, 66, 68, 70, 72, + 74, 76, 77, 79, 81, 82, 84, 85, 87, 88, 89, 90, 91, 91, 91, 91, 60, 58, + 56, 54, 52, 54, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 76, 77, 79, 81, + 83, 84, 85, 87, 88, 89, 90, 91, 92, 92, 92, 92, 63, 61, 59, 57, 55, 57, + 58, 60, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, 81, 82, 84, 85, 87, 88, + 89, 90, 92, 93, 94, 94, 94, 94, 66, 63, 61, 59, 58, 59, 60, 62, 63, 65, + 67, 69, 71, 73, 74, 76, 79, 80, 82, 83, 85, 86, 88, 89, 90, 91, 93, 94, + 95, 95, 95, 95, 68, 66, 64, 62, 60, 61, 62, 64, 65, 67, 69, 71, 72, 74, + 76, 78, 80, 81, 83, 85, 86, 88, 89, 90, 92, 93, 94, 95, 96, 96, 96, 96, + 71, 68, 66, 64, 62, 64, 65, 66, 67, 69, 71, 72, 74, 76, 78, 79, 81, 83, + 84, 86, 88, 89, 90, 91, 93, 94, 95, 96, 97, 97, 97, 97, 73, 71, 69, 67, + 65, 66, 67, 69, 70, 71, 73, 74, 76, 78, 79, 81, 83, 84, 86, 87, 89, 90, + 91, 92, 94, 95, 96, 97, 98, 98, 98, 98, 73, 71, 69, 67, 65, 66, 67, 69, + 70, 71, 73, 74, 76, 78, 79, 81, 83, 84, 86, 87, 89, 90, 91, 92, 94, 95, + 96, 97, 98, 98, 98, 98, 73, 71, 69, 67, 65, 66, 67, 69, 70, 71, 73, 74, + 76, 78, 79, 81, 83, 84, 86, 87, 89, 90, 91, 92, 94, 95, 96, 97, 98, 98, + 98, 98, 73, 71, 69, 67, 65, 66, 67, 69, 70, 71, 73, 74, 76, 78, 79, 81, + 83, 84, 86, 87, 89, 90, 91, 92, 94, 95, 96, 97, 98, 98, 98, 98 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 96, 104, 126, 96, 113, 119, 133, 104, 119, 146, 162, 126, 133, 162, + 183, + /* Size 8 */ + 64, 53, 90, 94, 100, 110, 121, 134, 53, 73, 91, 84, 86, 93, 103, 115, + 90, 91, 104, 101, 101, 105, 113, 123, 94, 84, 101, 112, 117, 121, 127, + 135, 100, 86, 101, 117, 128, 135, 141, 148, 110, 93, 105, 121, 135, 145, + 153, 160, 121, 103, 113, 127, 141, 153, 163, 170, 134, 115, 123, 135, + 148, 160, 170, 179, + /* Size 16 */ + 64, 58, 53, 66, 90, 92, 94, 97, 100, 105, 110, 115, 121, 127, 134, 134, + 58, 59, 61, 73, 90, 90, 89, 90, 92, 96, 101, 106, 112, 117, 123, 123, + 53, 61, 73, 81, 91, 87, 84, 85, 86, 89, 93, 98, 103, 109, 115, 115, 66, + 73, 81, 89, 97, 94, 92, 92, 93, 96, 99, 103, 108, 113, 119, 119, 90, 90, + 91, 97, 104, 103, 101, 101, 101, 103, 105, 109, 113, 118, 123, 123, 92, + 90, 87, 94, 103, 105, 107, 107, 108, 110, 113, 116, 120, 124, 128, 128, + 94, 89, 84, 92, 101, 107, 112, 115, 117, 119, 121, 124, 127, 131, 135, + 135, 97, 90, 85, 92, 101, 107, 115, 118, 122, 125, 128, 131, 134, 137, + 141, 141, 100, 92, 86, 93, 101, 108, 117, 122, 128, 131, 135, 138, 141, + 144, 148, 148, 105, 96, 89, 96, 103, 110, 119, 125, 131, 135, 140, 143, + 147, 150, 154, 154, 110, 101, 93, 99, 105, 113, 121, 128, 135, 140, 145, + 149, 153, 157, 160, 160, 115, 106, 98, 103, 109, 116, 124, 131, 138, + 143, 149, 153, 158, 161, 165, 165, 121, 112, 103, 108, 113, 120, 127, + 134, 141, 147, 153, 158, 163, 166, 170, 170, 127, 117, 109, 113, 118, + 124, 131, 137, 144, 150, 157, 161, 166, 170, 174, 174, 134, 123, 115, + 119, 123, 128, 135, 141, 148, 154, 160, 165, 170, 174, 179, 179, 134, + 123, 115, 119, 123, 128, 135, 141, 148, 154, 160, 165, 170, 174, 179, + 179, + /* Size 32 */ + 64, 61, 58, 55, 53, 59, 66, 76, 90, 91, 92, 93, 94, 96, 97, 98, 100, + 102, 105, 107, 110, 112, 115, 118, 121, 124, 127, 131, 134, 134, 134, + 134, 61, 60, 59, 58, 57, 62, 70, 78, 90, 90, 91, 91, 91, 93, 94, 95, 96, + 98, 100, 103, 105, 108, 110, 113, 116, 119, 122, 125, 129, 129, 129, + 129, 58, 59, 59, 60, 61, 67, 73, 81, 90, 90, 90, 89, 89, 90, 90, 91, 92, + 94, 96, 98, 101, 103, 106, 109, 112, 114, 117, 120, 123, 123, 123, 123, + 55, 58, 60, 63, 67, 72, 77, 83, 91, 90, 88, 87, 86, 87, 87, 88, 89, 91, + 93, 95, 97, 99, 102, 104, 107, 110, 113, 116, 119, 119, 119, 119, 53, + 57, 61, 67, 73, 77, 81, 86, 91, 89, 87, 85, 84, 84, 85, 85, 86, 88, 89, + 91, 93, 95, 98, 100, 103, 106, 109, 111, 115, 115, 115, 115, 59, 62, 67, + 72, 77, 81, 85, 89, 94, 92, 91, 89, 87, 88, 88, 89, 89, 91, 92, 94, 96, + 98, 100, 103, 105, 108, 111, 114, 117, 117, 117, 117, 66, 70, 73, 77, + 81, 85, 89, 93, 97, 96, 94, 93, 92, 92, 92, 92, 93, 94, 96, 97, 99, 101, + 103, 105, 108, 110, 113, 116, 119, 119, 119, 119, 76, 78, 81, 83, 86, + 89, 93, 97, 101, 99, 98, 97, 96, 96, 96, 97, 97, 98, 99, 101, 102, 104, + 106, 108, 110, 113, 115, 118, 121, 121, 121, 121, 90, 90, 90, 91, 91, + 94, 97, 101, 104, 103, 103, 102, 101, 101, 101, 101, 101, 102, 103, 104, + 105, 107, 109, 111, 113, 115, 118, 120, 123, 123, 123, 123, 91, 90, 90, + 90, 89, 92, 96, 99, 103, 104, 104, 104, 104, 104, 104, 104, 105, 106, + 107, 108, 109, 111, 112, 114, 116, 118, 121, 123, 126, 126, 126, 126, + 92, 91, 90, 88, 87, 91, 94, 98, 103, 104, 105, 106, 107, 107, 107, 108, + 108, 109, 110, 112, 113, 114, 116, 118, 120, 122, 124, 126, 128, 128, + 128, 128, 93, 91, 89, 87, 85, 89, 93, 97, 102, 104, 106, 107, 109, 110, + 111, 112, 112, 113, 115, 116, 117, 118, 120, 121, 123, 125, 127, 129, + 131, 131, 131, 131, 94, 91, 89, 86, 84, 87, 92, 96, 101, 104, 107, 109, + 112, 113, 115, 116, 117, 118, 119, 120, 121, 122, 124, 125, 127, 129, + 131, 133, 135, 135, 135, 135, 96, 93, 90, 87, 84, 88, 92, 96, 101, 104, + 107, 110, 113, 115, 116, 118, 119, 121, 122, 123, 124, 126, 127, 129, + 130, 132, 134, 136, 138, 138, 138, 138, 97, 94, 90, 87, 85, 88, 92, 96, + 101, 104, 107, 111, 115, 116, 118, 120, 122, 123, 125, 126, 128, 129, + 131, 132, 134, 135, 137, 139, 141, 141, 141, 141, 98, 95, 91, 88, 85, + 89, 92, 97, 101, 104, 108, 112, 116, 118, 120, 122, 125, 126, 128, 130, + 131, 133, 134, 136, 137, 139, 141, 142, 144, 144, 144, 144, 100, 96, 92, + 89, 86, 89, 93, 97, 101, 105, 108, 112, 117, 119, 122, 125, 128, 130, + 131, 133, 135, 136, 138, 140, 141, 143, 144, 146, 148, 148, 148, 148, + 102, 98, 94, 91, 88, 91, 94, 98, 102, 106, 109, 113, 118, 121, 123, 126, + 130, 131, 133, 135, 137, 139, 141, 142, 144, 146, 147, 149, 151, 151, + 151, 151, 105, 100, 96, 93, 89, 92, 96, 99, 103, 107, 110, 115, 119, + 122, 125, 128, 131, 133, 135, 138, 140, 142, 143, 145, 147, 149, 150, + 152, 154, 154, 154, 154, 107, 103, 98, 95, 91, 94, 97, 101, 104, 108, + 112, 116, 120, 123, 126, 130, 133, 135, 138, 140, 143, 144, 146, 148, + 150, 152, 153, 155, 157, 157, 157, 157, 110, 105, 101, 97, 93, 96, 99, + 102, 105, 109, 113, 117, 121, 124, 128, 131, 135, 137, 140, 143, 145, + 147, 149, 151, 153, 155, 157, 158, 160, 160, 160, 160, 112, 108, 103, + 99, 95, 98, 101, 104, 107, 111, 114, 118, 122, 126, 129, 133, 136, 139, + 142, 144, 147, 149, 151, 153, 156, 157, 159, 161, 162, 162, 162, 162, + 115, 110, 106, 102, 98, 100, 103, 106, 109, 112, 116, 120, 124, 127, + 131, 134, 138, 141, 143, 146, 149, 151, 153, 156, 158, 160, 161, 163, + 165, 165, 165, 165, 118, 113, 109, 104, 100, 103, 105, 108, 111, 114, + 118, 121, 125, 129, 132, 136, 140, 142, 145, 148, 151, 153, 156, 158, + 160, 162, 164, 166, 168, 168, 168, 168, 121, 116, 112, 107, 103, 105, + 108, 110, 113, 116, 120, 123, 127, 130, 134, 137, 141, 144, 147, 150, + 153, 156, 158, 160, 163, 165, 166, 168, 170, 170, 170, 170, 124, 119, + 114, 110, 106, 108, 110, 113, 115, 118, 122, 125, 129, 132, 135, 139, + 143, 146, 149, 152, 155, 157, 160, 162, 165, 166, 168, 170, 172, 172, + 172, 172, 127, 122, 117, 113, 109, 111, 113, 115, 118, 121, 124, 127, + 131, 134, 137, 141, 144, 147, 150, 153, 157, 159, 161, 164, 166, 168, + 170, 172, 174, 174, 174, 174, 131, 125, 120, 116, 111, 114, 116, 118, + 120, 123, 126, 129, 133, 136, 139, 142, 146, 149, 152, 155, 158, 161, + 163, 166, 168, 170, 172, 174, 177, 177, 177, 177, 134, 129, 123, 119, + 115, 117, 119, 121, 123, 126, 128, 131, 135, 138, 141, 144, 148, 151, + 154, 157, 160, 162, 165, 168, 170, 172, 174, 177, 179, 179, 179, 179, + 134, 129, 123, 119, 115, 117, 119, 121, 123, 126, 128, 131, 135, 138, + 141, 144, 148, 151, 154, 157, 160, 162, 165, 168, 170, 172, 174, 177, + 179, 179, 179, 179, 134, 129, 123, 119, 115, 117, 119, 121, 123, 126, + 128, 131, 135, 138, 141, 144, 148, 151, 154, 157, 160, 162, 165, 168, + 170, 172, 174, 177, 179, 179, 179, 179, 134, 129, 123, 119, 115, 117, + 119, 121, 123, 126, 128, 131, 135, 138, 141, 144, 148, 151, 154, 157, + 160, 162, 165, 168, 170, 172, 174, 177, 179, 179, 179, 179 }, + { /* Intra matrices */ + /* Size 4 */ + 31, 47, 51, 63, 47, 56, 59, 67, 51, 59, 74, 82, 63, 67, 82, 95, + /* Size 8 */ + 33, 27, 47, 50, 53, 58, 65, 72, 27, 38, 48, 44, 45, 49, 55, 61, 47, 48, + 55, 54, 53, 56, 60, 66, 50, 44, 54, 60, 62, 65, 68, 73, 53, 45, 53, 62, + 69, 73, 76, 80, 58, 49, 56, 65, 73, 79, 84, 88, 65, 55, 60, 68, 76, 84, + 89, 94, 72, 61, 66, 73, 80, 88, 94, 99, + /* Size 16 */ + 32, 29, 26, 33, 46, 47, 48, 50, 51, 54, 57, 60, 63, 67, 70, 70, 29, 30, + 31, 37, 46, 46, 45, 46, 47, 50, 52, 55, 58, 61, 65, 65, 26, 31, 37, 41, + 47, 45, 43, 43, 44, 46, 48, 50, 53, 56, 60, 60, 33, 37, 41, 45, 50, 48, + 47, 47, 48, 49, 51, 53, 56, 59, 62, 62, 46, 46, 47, 50, 54, 53, 52, 52, + 52, 53, 55, 57, 59, 61, 64, 64, 47, 46, 45, 48, 53, 54, 55, 56, 56, 57, + 59, 60, 62, 65, 67, 67, 48, 45, 43, 47, 52, 55, 58, 60, 61, 62, 63, 65, + 67, 69, 71, 71, 50, 46, 43, 47, 52, 56, 60, 62, 64, 65, 67, 69, 70, 72, + 74, 74, 51, 47, 44, 48, 52, 56, 61, 64, 67, 69, 71, 73, 75, 76, 78, 78, + 54, 50, 46, 49, 53, 57, 62, 65, 69, 71, 74, 76, 78, 80, 82, 82, 57, 52, + 48, 51, 55, 59, 63, 67, 71, 74, 77, 79, 82, 84, 86, 86, 60, 55, 50, 53, + 57, 60, 65, 69, 73, 76, 79, 82, 84, 86, 89, 89, 63, 58, 53, 56, 59, 62, + 67, 70, 75, 78, 82, 84, 87, 89, 92, 92, 67, 61, 56, 59, 61, 65, 69, 72, + 76, 80, 84, 86, 89, 92, 94, 94, 70, 65, 60, 62, 64, 67, 71, 74, 78, 82, + 86, 89, 92, 94, 97, 97, 70, 65, 60, 62, 64, 67, 71, 74, 78, 82, 86, 89, + 92, 94, 97, 97, + /* Size 32 */ + 32, 30, 29, 27, 26, 29, 33, 38, 45, 46, 47, 47, 48, 49, 49, 50, 51, 52, + 53, 55, 56, 58, 59, 61, 63, 64, 66, 68, 70, 70, 70, 70, 30, 30, 29, 29, + 28, 31, 35, 39, 46, 46, 46, 46, 46, 47, 47, 48, 49, 50, 51, 52, 54, 55, + 57, 58, 60, 61, 63, 65, 67, 67, 67, 67, 29, 29, 30, 30, 30, 33, 37, 41, + 46, 46, 45, 45, 45, 45, 46, 46, 47, 48, 49, 50, 51, 53, 54, 56, 57, 59, + 60, 62, 64, 64, 64, 64, 27, 29, 30, 32, 33, 36, 39, 42, 46, 45, 45, 44, + 43, 44, 44, 45, 45, 46, 47, 48, 49, 50, 52, 53, 55, 56, 58, 59, 61, 61, + 61, 61, 26, 28, 30, 33, 37, 39, 41, 43, 46, 45, 44, 43, 42, 42, 43, 43, + 43, 44, 45, 46, 47, 48, 50, 51, 53, 54, 56, 57, 59, 59, 59, 59, 29, 31, + 33, 36, 39, 41, 43, 45, 48, 47, 46, 45, 44, 44, 45, 45, 45, 46, 47, 48, + 49, 50, 51, 52, 54, 55, 57, 58, 60, 60, 60, 60, 33, 35, 37, 39, 41, 43, + 45, 47, 49, 49, 48, 47, 46, 47, 47, 47, 47, 48, 49, 49, 50, 51, 53, 54, + 55, 57, 58, 59, 61, 61, 61, 61, 38, 39, 41, 42, 43, 45, 47, 49, 51, 51, + 50, 49, 49, 49, 49, 49, 49, 50, 51, 51, 52, 53, 54, 55, 57, 58, 59, 61, + 62, 62, 62, 62, 45, 46, 46, 46, 46, 48, 49, 51, 53, 53, 52, 52, 52, 52, + 52, 51, 51, 52, 53, 53, 54, 55, 56, 57, 58, 59, 61, 62, 63, 63, 63, 63, + 46, 46, 46, 45, 45, 47, 49, 51, 53, 53, 53, 53, 53, 53, 53, 53, 53, 54, + 55, 55, 56, 57, 58, 59, 60, 61, 62, 64, 65, 65, 65, 65, 47, 46, 45, 45, + 44, 46, 48, 50, 52, 53, 53, 54, 54, 55, 55, 55, 55, 56, 57, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 67, 67, 67, 67, 47, 46, 45, 44, 43, 45, 47, 49, + 52, 53, 54, 55, 56, 56, 57, 57, 58, 58, 59, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 68, 68, 68, 48, 46, 45, 43, 42, 44, 46, 49, 52, 53, 54, 56, + 58, 58, 59, 59, 60, 61, 61, 62, 62, 63, 64, 65, 66, 67, 68, 69, 70, 70, + 70, 70, 49, 47, 45, 44, 42, 44, 47, 49, 52, 53, 55, 56, 58, 59, 60, 61, + 62, 62, 63, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 72, 72, 72, 49, 47, + 46, 44, 43, 45, 47, 49, 52, 53, 55, 57, 59, 60, 61, 62, 63, 64, 64, 65, + 66, 67, 68, 69, 69, 70, 71, 72, 74, 74, 74, 74, 50, 48, 46, 45, 43, 45, + 47, 49, 51, 53, 55, 57, 59, 61, 62, 63, 65, 65, 66, 67, 68, 69, 70, 71, + 72, 72, 73, 74, 75, 75, 75, 75, 51, 49, 47, 45, 43, 45, 47, 49, 51, 53, + 55, 58, 60, 62, 63, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 76, + 77, 77, 77, 77, 52, 50, 48, 46, 44, 46, 48, 50, 52, 54, 56, 58, 61, 62, + 64, 65, 67, 68, 69, 70, 72, 72, 73, 74, 75, 76, 77, 78, 79, 79, 79, 79, + 53, 51, 49, 47, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 64, 66, 68, 69, + 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 81, 81, 81, 55, 52, 50, 48, + 46, 48, 49, 51, 53, 55, 57, 59, 62, 63, 65, 67, 69, 70, 72, 73, 75, 76, + 77, 78, 79, 80, 81, 82, 83, 83, 83, 83, 56, 54, 51, 49, 47, 49, 50, 52, + 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 73, 75, 76, 77, 78, 79, 81, 82, + 83, 84, 85, 85, 85, 85, 58, 55, 53, 50, 48, 50, 51, 53, 55, 57, 59, 61, + 63, 65, 67, 69, 71, 72, 74, 76, 77, 78, 80, 81, 82, 83, 84, 85, 86, 86, + 86, 86, 59, 57, 54, 52, 50, 51, 53, 54, 56, 58, 60, 62, 64, 66, 68, 70, + 72, 73, 75, 77, 78, 80, 81, 82, 83, 84, 85, 86, 87, 87, 87, 87, 61, 58, + 56, 53, 51, 52, 54, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 74, 76, 78, + 79, 81, 82, 83, 85, 86, 87, 88, 89, 89, 89, 89, 63, 60, 57, 55, 53, 54, + 55, 57, 58, 60, 62, 64, 66, 68, 69, 72, 74, 75, 77, 79, 81, 82, 83, 85, + 86, 87, 88, 89, 91, 91, 91, 91, 64, 61, 59, 56, 54, 55, 57, 58, 59, 61, + 63, 65, 67, 69, 70, 72, 75, 76, 78, 80, 82, 83, 84, 86, 87, 88, 89, 91, + 92, 92, 92, 92, 66, 63, 60, 58, 56, 57, 58, 59, 61, 62, 64, 66, 68, 70, + 71, 73, 76, 77, 79, 81, 83, 84, 85, 87, 88, 89, 91, 92, 93, 93, 93, 93, + 68, 65, 62, 59, 57, 58, 59, 61, 62, 64, 65, 67, 69, 71, 72, 74, 76, 78, + 80, 82, 84, 85, 86, 88, 89, 91, 92, 93, 94, 94, 94, 94, 70, 67, 64, 61, + 59, 60, 61, 62, 63, 65, 67, 68, 70, 72, 74, 75, 77, 79, 81, 83, 85, 86, + 87, 89, 91, 92, 93, 94, 96, 96, 96, 96, 70, 67, 64, 61, 59, 60, 61, 62, + 63, 65, 67, 68, 70, 72, 74, 75, 77, 79, 81, 83, 85, 86, 87, 89, 91, 92, + 93, 94, 96, 96, 96, 96, 70, 67, 64, 61, 59, 60, 61, 62, 63, 65, 67, 68, + 70, 72, 74, 75, 77, 79, 81, 83, 85, 86, 87, 89, 91, 92, 93, 94, 96, 96, + 96, 96, 70, 67, 64, 61, 59, 60, 61, 62, 63, 65, 67, 68, 70, 72, 74, 75, + 77, 79, 81, 83, 85, 86, 87, 89, 91, 92, 93, 94, 96, 96, 96, 96 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 69, 104, 143, 69, 97, 124, 152, 104, 124, 156, 176, 143, 152, 176, + 192, + /* Size 8 */ + 64, 50, 54, 68, 86, 104, 122, 137, 50, 56, 54, 62, 76, 93, 109, 125, 54, + 54, 72, 82, 92, 105, 118, 131, 68, 62, 82, 98, 110, 120, 130, 140, 86, + 76, 92, 110, 123, 133, 142, 150, 104, 93, 105, 120, 133, 144, 151, 158, + 122, 109, 118, 130, 142, 151, 158, 164, 137, 125, 131, 140, 150, 158, + 164, 169, + /* Size 16 */ + 64, 56, 50, 52, 54, 60, 68, 76, 86, 94, 104, 112, 122, 129, 137, 137, + 56, 55, 53, 54, 54, 59, 65, 72, 81, 89, 98, 106, 115, 122, 130, 130, 50, + 53, 56, 55, 54, 58, 62, 69, 76, 84, 93, 100, 109, 116, 125, 125, 52, 54, + 55, 59, 62, 66, 71, 77, 83, 90, 98, 105, 114, 120, 128, 128, 54, 54, 54, + 62, 72, 77, 82, 87, 92, 98, 105, 111, 118, 124, 131, 131, 60, 59, 58, + 66, 77, 82, 89, 94, 100, 106, 112, 118, 124, 130, 136, 136, 68, 65, 62, + 71, 82, 89, 98, 104, 110, 115, 120, 125, 130, 135, 140, 140, 76, 72, 69, + 77, 87, 94, 104, 109, 116, 121, 126, 131, 136, 140, 145, 145, 86, 81, + 76, 83, 92, 100, 110, 116, 123, 128, 133, 138, 142, 146, 150, 150, 94, + 89, 84, 90, 98, 106, 115, 121, 128, 133, 138, 142, 146, 150, 153, 153, + 104, 98, 93, 98, 105, 112, 120, 126, 133, 138, 144, 147, 151, 154, 158, + 158, 112, 106, 100, 105, 111, 118, 125, 131, 138, 142, 147, 151, 155, + 158, 161, 161, 122, 115, 109, 114, 118, 124, 130, 136, 142, 146, 151, + 155, 158, 161, 164, 164, 129, 122, 116, 120, 124, 130, 135, 140, 146, + 150, 154, 158, 161, 164, 166, 166, 137, 130, 125, 128, 131, 136, 140, + 145, 150, 153, 158, 161, 164, 166, 169, 169, 137, 130, 125, 128, 131, + 136, 140, 145, 150, 153, 158, 161, 164, 166, 169, 169, + /* Size 32 */ + 64, 60, 56, 53, 50, 51, 52, 53, 54, 57, 60, 64, 68, 72, 76, 80, 86, 90, + 94, 99, 104, 108, 112, 117, 122, 125, 129, 132, 137, 137, 137, 137, 60, + 58, 55, 53, 52, 52, 53, 53, 54, 57, 59, 63, 66, 70, 74, 78, 83, 87, 91, + 96, 101, 105, 109, 113, 118, 122, 125, 129, 133, 133, 133, 133, 56, 55, + 55, 54, 53, 53, 54, 54, 54, 56, 59, 62, 65, 68, 72, 76, 81, 84, 89, 93, + 98, 102, 106, 110, 115, 118, 122, 126, 130, 130, 130, 130, 53, 53, 54, + 54, 55, 55, 55, 54, 54, 56, 59, 61, 64, 67, 70, 74, 78, 82, 86, 90, 95, + 99, 103, 107, 112, 115, 119, 123, 127, 127, 127, 127, 50, 52, 53, 55, + 56, 56, 55, 55, 54, 56, 58, 60, 62, 65, 69, 72, 76, 80, 84, 88, 93, 96, + 100, 104, 109, 113, 116, 120, 125, 125, 125, 125, 51, 52, 53, 55, 56, + 56, 57, 57, 58, 60, 62, 64, 66, 69, 72, 76, 80, 83, 87, 91, 95, 99, 103, + 107, 111, 115, 118, 122, 126, 126, 126, 126, 52, 53, 54, 55, 55, 57, 59, + 60, 62, 64, 66, 68, 71, 73, 77, 80, 83, 87, 90, 94, 98, 102, 105, 109, + 114, 117, 120, 124, 128, 128, 128, 128, 53, 53, 54, 54, 55, 57, 60, 63, + 67, 69, 71, 73, 76, 78, 81, 84, 88, 91, 94, 98, 101, 105, 108, 112, 116, + 119, 122, 126, 129, 129, 129, 129, 54, 54, 54, 54, 54, 58, 62, 67, 72, + 74, 77, 79, 82, 84, 87, 89, 92, 95, 98, 101, 105, 108, 111, 115, 118, + 121, 124, 128, 131, 131, 131, 131, 57, 57, 56, 56, 56, 60, 64, 69, 74, + 77, 79, 82, 85, 88, 90, 93, 96, 99, 102, 105, 108, 111, 114, 118, 121, + 124, 127, 130, 133, 133, 133, 133, 60, 59, 59, 59, 58, 62, 66, 71, 77, + 79, 82, 86, 89, 92, 94, 97, 100, 103, 106, 109, 112, 115, 118, 121, 124, + 127, 130, 133, 136, 136, 136, 136, 64, 63, 62, 61, 60, 64, 68, 73, 79, + 82, 86, 89, 93, 96, 99, 102, 105, 107, 110, 113, 116, 118, 121, 124, + 127, 130, 132, 135, 138, 138, 138, 138, 68, 66, 65, 64, 62, 66, 71, 76, + 82, 85, 89, 93, 98, 101, 104, 107, 110, 112, 115, 117, 120, 122, 125, + 128, 130, 133, 135, 138, 140, 140, 140, 140, 72, 70, 68, 67, 65, 69, 73, + 78, 84, 88, 92, 96, 101, 104, 106, 110, 113, 115, 118, 120, 123, 125, + 128, 130, 133, 135, 138, 140, 143, 143, 143, 143, 76, 74, 72, 70, 69, + 72, 77, 81, 87, 90, 94, 99, 104, 106, 109, 113, 116, 119, 121, 124, 126, + 129, 131, 133, 136, 138, 140, 142, 145, 145, 145, 145, 80, 78, 76, 74, + 72, 76, 80, 84, 89, 93, 97, 102, 107, 110, 113, 116, 120, 122, 124, 127, + 130, 132, 134, 136, 139, 141, 143, 145, 147, 147, 147, 147, 86, 83, 81, + 78, 76, 80, 83, 88, 92, 96, 100, 105, 110, 113, 116, 120, 123, 126, 128, + 131, 133, 135, 138, 140, 142, 144, 146, 148, 150, 150, 150, 150, 90, 87, + 84, 82, 80, 83, 87, 91, 95, 99, 103, 107, 112, 115, 119, 122, 126, 128, + 131, 133, 136, 138, 140, 142, 144, 146, 148, 150, 151, 151, 151, 151, + 94, 91, 89, 86, 84, 87, 90, 94, 98, 102, 106, 110, 115, 118, 121, 124, + 128, 131, 133, 136, 138, 140, 142, 144, 146, 148, 150, 152, 153, 153, + 153, 153, 99, 96, 93, 90, 88, 91, 94, 98, 101, 105, 109, 113, 117, 120, + 124, 127, 131, 133, 136, 138, 141, 143, 145, 147, 149, 150, 152, 154, + 155, 155, 155, 155, 104, 101, 98, 95, 93, 95, 98, 101, 105, 108, 112, + 116, 120, 123, 126, 130, 133, 136, 138, 141, 144, 145, 147, 149, 151, + 153, 154, 156, 158, 158, 158, 158, 108, 105, 102, 99, 96, 99, 102, 105, + 108, 111, 115, 118, 122, 125, 129, 132, 135, 138, 140, 143, 145, 147, + 149, 151, 153, 154, 156, 158, 159, 159, 159, 159, 112, 109, 106, 103, + 100, 103, 105, 108, 111, 114, 118, 121, 125, 128, 131, 134, 138, 140, + 142, 145, 147, 149, 151, 153, 155, 156, 158, 159, 161, 161, 161, 161, + 117, 113, 110, 107, 104, 107, 109, 112, 115, 118, 121, 124, 128, 130, + 133, 136, 140, 142, 144, 147, 149, 151, 153, 155, 157, 158, 159, 161, + 162, 162, 162, 162, 122, 118, 115, 112, 109, 111, 114, 116, 118, 121, + 124, 127, 130, 133, 136, 139, 142, 144, 146, 149, 151, 153, 155, 157, + 158, 160, 161, 162, 164, 164, 164, 164, 125, 122, 118, 115, 113, 115, + 117, 119, 121, 124, 127, 130, 133, 135, 138, 141, 144, 146, 148, 150, + 153, 154, 156, 158, 160, 161, 162, 164, 165, 165, 165, 165, 129, 125, + 122, 119, 116, 118, 120, 122, 124, 127, 130, 132, 135, 138, 140, 143, + 146, 148, 150, 152, 154, 156, 158, 159, 161, 162, 164, 165, 166, 166, + 166, 166, 132, 129, 126, 123, 120, 122, 124, 126, 128, 130, 133, 135, + 138, 140, 142, 145, 148, 150, 152, 154, 156, 158, 159, 161, 162, 164, + 165, 166, 167, 167, 167, 167, 137, 133, 130, 127, 125, 126, 128, 129, + 131, 133, 136, 138, 140, 143, 145, 147, 150, 151, 153, 155, 158, 159, + 161, 162, 164, 165, 166, 167, 169, 169, 169, 169, 137, 133, 130, 127, + 125, 126, 128, 129, 131, 133, 136, 138, 140, 143, 145, 147, 150, 151, + 153, 155, 158, 159, 161, 162, 164, 165, 166, 167, 169, 169, 169, 169, + 137, 133, 130, 127, 125, 126, 128, 129, 131, 133, 136, 138, 140, 143, + 145, 147, 150, 151, 153, 155, 158, 159, 161, 162, 164, 165, 166, 167, + 169, 169, 169, 169, 137, 133, 130, 127, 125, 126, 128, 129, 131, 133, + 136, 138, 140, 143, 145, 147, 150, 151, 153, 155, 158, 159, 161, 162, + 164, 165, 166, 167, 169, 169, 169, 169 }, + { /* Intra matrices */ + /* Size 4 */ + 29, 32, 48, 68, 32, 45, 58, 73, 48, 58, 75, 86, 68, 73, 86, 94, + /* Size 8 */ + 34, 26, 28, 36, 46, 57, 67, 77, 26, 30, 29, 33, 41, 50, 60, 69, 28, 29, + 39, 44, 50, 57, 65, 73, 36, 33, 44, 53, 60, 66, 73, 79, 46, 41, 50, 60, + 68, 75, 80, 85, 57, 50, 57, 66, 75, 81, 86, 90, 67, 60, 65, 73, 80, 86, + 90, 94, 77, 69, 73, 79, 85, 90, 94, 97, + /* Size 16 */ + 33, 29, 26, 27, 28, 31, 35, 39, 45, 50, 55, 60, 65, 70, 74, 74, 29, 28, + 27, 27, 28, 30, 34, 37, 42, 47, 52, 56, 62, 66, 71, 71, 26, 27, 29, 28, + 28, 30, 32, 35, 40, 44, 49, 53, 58, 62, 67, 67, 27, 27, 28, 30, 32, 34, + 37, 40, 44, 47, 52, 56, 61, 65, 69, 69, 28, 28, 28, 32, 37, 40, 43, 45, + 49, 52, 56, 59, 64, 67, 71, 71, 31, 30, 30, 34, 40, 43, 47, 50, 53, 56, + 60, 63, 67, 70, 74, 74, 35, 34, 32, 37, 43, 47, 52, 55, 59, 61, 65, 67, + 71, 74, 77, 77, 39, 37, 35, 40, 45, 50, 55, 58, 62, 65, 68, 71, 74, 77, + 79, 79, 45, 42, 40, 44, 49, 53, 59, 62, 66, 69, 72, 75, 78, 80, 82, 82, + 50, 47, 44, 47, 52, 56, 61, 65, 69, 72, 75, 78, 80, 82, 85, 85, 55, 52, + 49, 52, 56, 60, 65, 68, 72, 75, 79, 81, 83, 85, 87, 87, 60, 56, 53, 56, + 59, 63, 67, 71, 75, 78, 81, 83, 85, 87, 89, 89, 65, 62, 58, 61, 64, 67, + 71, 74, 78, 80, 83, 85, 88, 89, 91, 91, 70, 66, 62, 65, 67, 70, 74, 77, + 80, 82, 85, 87, 89, 91, 93, 93, 74, 71, 67, 69, 71, 74, 77, 79, 82, 85, + 87, 89, 91, 93, 94, 94, 74, 71, 67, 69, 71, 74, 77, 79, 82, 85, 87, 89, + 91, 93, 94, 94, + /* Size 32 */ + 33, 30, 28, 27, 25, 26, 26, 27, 27, 29, 30, 32, 35, 37, 39, 41, 44, 47, + 49, 52, 55, 57, 59, 62, 65, 67, 69, 71, 73, 73, 73, 73, 30, 29, 28, 27, + 26, 26, 27, 27, 27, 29, 30, 32, 34, 36, 38, 40, 43, 45, 47, 50, 53, 55, + 57, 60, 63, 65, 67, 69, 71, 71, 71, 71, 28, 28, 28, 27, 27, 27, 27, 27, + 27, 29, 30, 31, 33, 35, 37, 39, 42, 44, 46, 48, 51, 53, 56, 58, 61, 63, + 65, 67, 70, 70, 70, 70, 27, 27, 27, 27, 28, 28, 28, 27, 27, 28, 30, 31, + 32, 34, 36, 38, 40, 42, 44, 47, 50, 52, 54, 56, 59, 61, 63, 65, 68, 68, + 68, 68, 25, 26, 27, 28, 29, 28, 28, 28, 27, 28, 29, 30, 32, 33, 35, 37, + 39, 41, 43, 45, 48, 50, 52, 55, 57, 59, 61, 64, 66, 66, 66, 66, 26, 26, + 27, 28, 28, 29, 29, 29, 29, 30, 31, 33, 34, 35, 37, 39, 41, 43, 45, 47, + 50, 52, 54, 56, 59, 61, 63, 65, 67, 67, 67, 67, 26, 27, 27, 28, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 38, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, + 60, 62, 64, 66, 68, 68, 68, 68, 27, 27, 27, 27, 28, 29, 31, 32, 34, 35, + 36, 38, 39, 40, 42, 44, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, + 69, 69, 69, 69, 27, 27, 27, 27, 27, 29, 32, 34, 37, 38, 39, 41, 42, 43, + 45, 46, 48, 49, 51, 53, 55, 57, 59, 61, 63, 64, 66, 68, 70, 70, 70, 70, + 29, 29, 29, 28, 28, 30, 33, 35, 38, 39, 41, 42, 44, 45, 47, 48, 50, 52, + 53, 55, 57, 59, 60, 62, 64, 66, 68, 69, 71, 71, 71, 71, 30, 30, 30, 30, + 29, 31, 34, 36, 39, 41, 42, 44, 46, 48, 49, 51, 52, 54, 55, 57, 59, 61, + 62, 64, 66, 68, 69, 71, 73, 73, 73, 73, 32, 32, 31, 31, 30, 33, 35, 38, + 41, 42, 44, 46, 49, 50, 52, 53, 55, 56, 58, 59, 61, 63, 64, 66, 68, 69, + 71, 72, 74, 74, 74, 74, 35, 34, 33, 32, 32, 34, 36, 39, 42, 44, 46, 49, + 51, 53, 54, 56, 58, 59, 60, 62, 64, 65, 66, 68, 70, 71, 72, 74, 75, 75, + 75, 75, 37, 36, 35, 34, 33, 35, 38, 40, 43, 45, 48, 50, 53, 54, 56, 58, + 59, 61, 62, 64, 65, 67, 68, 70, 71, 73, 74, 75, 77, 77, 77, 77, 39, 38, + 37, 36, 35, 37, 39, 42, 45, 47, 49, 52, 54, 56, 58, 59, 61, 63, 64, 66, + 67, 69, 70, 71, 73, 74, 75, 77, 78, 78, 78, 78, 41, 40, 39, 38, 37, 39, + 41, 44, 46, 48, 51, 53, 56, 58, 59, 61, 63, 65, 66, 68, 69, 71, 72, 73, + 75, 76, 77, 78, 80, 80, 80, 80, 44, 43, 42, 40, 39, 41, 43, 45, 48, 50, + 52, 55, 58, 59, 61, 63, 65, 67, 68, 70, 71, 73, 74, 75, 76, 78, 79, 80, + 81, 81, 81, 81, 47, 45, 44, 42, 41, 43, 45, 47, 49, 52, 54, 56, 59, 61, + 63, 65, 67, 68, 70, 71, 73, 74, 75, 76, 78, 79, 80, 81, 82, 82, 82, 82, + 49, 47, 46, 44, 43, 45, 47, 49, 51, 53, 55, 58, 60, 62, 64, 66, 68, 70, + 71, 73, 74, 75, 77, 78, 79, 80, 81, 82, 83, 83, 83, 83, 52, 50, 48, 47, + 45, 47, 49, 51, 53, 55, 57, 59, 62, 64, 66, 68, 70, 71, 73, 74, 76, 77, + 78, 79, 81, 82, 83, 84, 85, 85, 85, 85, 55, 53, 51, 50, 48, 50, 51, 53, + 55, 57, 59, 61, 64, 65, 67, 69, 71, 73, 74, 76, 77, 79, 80, 81, 82, 83, + 84, 85, 86, 86, 86, 86, 57, 55, 53, 52, 50, 52, 53, 55, 57, 59, 61, 63, + 65, 67, 69, 71, 73, 74, 75, 77, 79, 80, 81, 82, 83, 84, 85, 86, 87, 87, + 87, 87, 59, 57, 56, 54, 52, 54, 55, 57, 59, 60, 62, 64, 66, 68, 70, 72, + 74, 75, 77, 78, 80, 81, 82, 83, 84, 85, 86, 87, 88, 88, 88, 88, 62, 60, + 58, 56, 55, 56, 57, 59, 61, 62, 64, 66, 68, 70, 71, 73, 75, 76, 78, 79, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 89, 89, 89, 65, 63, 61, 59, 57, 59, + 60, 61, 63, 64, 66, 68, 70, 71, 73, 75, 76, 78, 79, 81, 82, 83, 84, 85, + 86, 87, 88, 89, 90, 90, 90, 90, 67, 65, 63, 61, 59, 61, 62, 63, 64, 66, + 68, 69, 71, 73, 74, 76, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 89, 90, + 90, 90, 90, 90, 69, 67, 65, 63, 61, 63, 64, 65, 66, 68, 69, 71, 72, 74, + 75, 77, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 90, 91, 91, 91, 91, + 71, 69, 67, 65, 64, 65, 66, 67, 68, 69, 71, 72, 74, 75, 77, 78, 80, 81, + 82, 84, 85, 86, 87, 88, 89, 90, 90, 91, 92, 92, 92, 92, 73, 71, 70, 68, + 66, 67, 68, 69, 70, 71, 73, 74, 75, 77, 78, 80, 81, 82, 83, 85, 86, 87, + 88, 89, 90, 90, 91, 92, 93, 93, 93, 93, 73, 71, 70, 68, 66, 67, 68, 69, + 70, 71, 73, 74, 75, 77, 78, 80, 81, 82, 83, 85, 86, 87, 88, 89, 90, 90, + 91, 92, 93, 93, 93, 93, 73, 71, 70, 68, 66, 67, 68, 69, 70, 71, 73, 74, + 75, 77, 78, 80, 81, 82, 83, 85, 86, 87, 88, 89, 90, 90, 91, 92, 93, 93, + 93, 93, 73, 71, 70, 68, 66, 67, 68, 69, 70, 71, 73, 74, 75, 77, 78, 80, + 81, 82, 83, 85, 86, 87, 88, 89, 90, 90, 91, 92, 93, 93, 93, 93 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 94, 100, 120, 94, 108, 114, 126, 100, 114, 136, 149, 120, 126, 149, + 166, + /* Size 8 */ + 64, 53, 88, 92, 97, 105, 115, 126, 53, 73, 89, 82, 84, 91, 100, 109, 88, + 89, 100, 98, 98, 102, 108, 116, 92, 82, 98, 107, 111, 115, 120, 126, 97, + 84, 98, 111, 121, 127, 132, 137, 105, 91, 102, 115, 127, 135, 142, 147, + 115, 100, 108, 120, 132, 142, 149, 155, 126, 109, 116, 126, 137, 147, + 155, 162, + /* Size 16 */ + 64, 58, 53, 66, 88, 90, 92, 94, 97, 101, 105, 110, 115, 120, 126, 126, + 58, 60, 61, 72, 88, 87, 87, 88, 90, 94, 97, 102, 107, 112, 117, 117, 53, + 61, 73, 80, 89, 85, 82, 83, 84, 87, 91, 95, 100, 104, 109, 109, 66, 72, + 80, 87, 94, 92, 89, 90, 90, 93, 96, 100, 104, 108, 113, 113, 88, 88, 89, + 94, 100, 99, 98, 98, 98, 100, 102, 105, 108, 112, 116, 116, 90, 87, 85, + 92, 99, 101, 102, 103, 104, 106, 108, 111, 114, 117, 121, 121, 92, 87, + 82, 89, 98, 102, 107, 109, 111, 113, 115, 117, 120, 123, 126, 126, 94, + 88, 83, 90, 98, 103, 109, 113, 116, 118, 121, 123, 126, 128, 132, 132, + 97, 90, 84, 90, 98, 104, 111, 116, 121, 124, 127, 129, 132, 134, 137, + 137, 101, 94, 87, 93, 100, 106, 113, 118, 124, 127, 131, 134, 136, 139, + 142, 142, 105, 97, 91, 96, 102, 108, 115, 121, 127, 131, 135, 138, 142, + 144, 147, 147, 110, 102, 95, 100, 105, 111, 117, 123, 129, 134, 138, + 142, 145, 148, 151, 151, 115, 107, 100, 104, 108, 114, 120, 126, 132, + 136, 142, 145, 149, 152, 155, 155, 120, 112, 104, 108, 112, 117, 123, + 128, 134, 139, 144, 148, 152, 155, 158, 158, 126, 117, 109, 113, 116, + 121, 126, 132, 137, 142, 147, 151, 155, 158, 162, 162, 126, 117, 109, + 113, 116, 121, 126, 132, 137, 142, 147, 151, 155, 158, 162, 162, + /* Size 32 */ + 64, 61, 58, 56, 53, 59, 66, 75, 88, 89, 90, 91, 92, 93, 94, 95, 97, 99, + 101, 103, 105, 108, 110, 113, 115, 118, 120, 123, 126, 126, 126, 126, + 61, 60, 59, 58, 57, 63, 69, 77, 88, 88, 89, 89, 89, 90, 91, 92, 93, 95, + 97, 99, 101, 103, 106, 108, 111, 113, 116, 118, 121, 121, 121, 121, 58, + 59, 60, 61, 61, 67, 72, 80, 88, 88, 87, 87, 87, 88, 88, 89, 90, 92, 94, + 95, 97, 100, 102, 104, 107, 109, 112, 114, 117, 117, 117, 117, 56, 58, + 61, 63, 67, 71, 76, 82, 89, 88, 86, 85, 84, 85, 86, 86, 87, 89, 90, 92, + 94, 96, 98, 101, 103, 105, 108, 110, 113, 113, 113, 113, 53, 57, 61, 67, + 73, 76, 80, 84, 89, 87, 85, 84, 82, 83, 83, 84, 84, 86, 87, 89, 91, 93, + 95, 97, 100, 102, 104, 107, 109, 109, 109, 109, 59, 63, 67, 71, 76, 80, + 83, 87, 92, 90, 88, 87, 86, 86, 86, 87, 87, 89, 90, 92, 93, 95, 97, 99, + 102, 104, 106, 109, 111, 111, 111, 111, 66, 69, 72, 76, 80, 83, 87, 90, + 94, 93, 92, 91, 89, 90, 90, 90, 90, 92, 93, 94, 96, 98, 100, 102, 104, + 106, 108, 110, 113, 113, 113, 113, 75, 77, 80, 82, 84, 87, 90, 94, 97, + 96, 95, 94, 93, 94, 94, 94, 94, 95, 96, 97, 99, 100, 102, 104, 106, 108, + 110, 112, 115, 115, 115, 115, 88, 88, 88, 89, 89, 92, 94, 97, 100, 100, + 99, 99, 98, 98, 98, 98, 98, 99, 100, 101, 102, 103, 105, 106, 108, 110, + 112, 114, 116, 116, 116, 116, 89, 88, 88, 88, 87, 90, 93, 96, 100, 100, + 100, 100, 100, 100, 100, 101, 101, 102, 103, 104, 105, 106, 108, 109, + 111, 113, 115, 117, 119, 119, 119, 119, 90, 89, 87, 86, 85, 88, 92, 95, + 99, 100, 101, 102, 102, 103, 103, 104, 104, 105, 106, 107, 108, 109, + 111, 112, 114, 116, 117, 119, 121, 121, 121, 121, 91, 89, 87, 85, 84, + 87, 91, 94, 99, 100, 102, 103, 105, 106, 106, 107, 108, 109, 109, 110, + 111, 113, 114, 115, 117, 118, 120, 122, 124, 124, 124, 124, 92, 89, 87, + 84, 82, 86, 89, 93, 98, 100, 102, 105, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 119, 120, 122, 123, 125, 126, 126, 126, 126, + 93, 90, 88, 85, 83, 86, 90, 94, 98, 100, 103, 106, 108, 110, 111, 112, + 114, 115, 116, 117, 118, 119, 120, 121, 123, 124, 126, 127, 129, 129, + 129, 129, 94, 91, 88, 86, 83, 86, 90, 94, 98, 100, 103, 106, 109, 111, + 113, 114, 116, 117, 118, 119, 121, 122, 123, 124, 126, 127, 128, 130, + 132, 132, 132, 132, 95, 92, 89, 86, 84, 87, 90, 94, 98, 101, 104, 107, + 110, 112, 114, 116, 118, 119, 121, 122, 124, 125, 126, 127, 129, 130, + 131, 133, 134, 134, 134, 134, 97, 93, 90, 87, 84, 87, 90, 94, 98, 101, + 104, 108, 111, 114, 116, 118, 121, 122, 124, 125, 127, 128, 129, 130, + 132, 133, 134, 136, 137, 137, 137, 137, 99, 95, 92, 89, 86, 89, 92, 95, + 99, 102, 105, 109, 112, 115, 117, 119, 122, 124, 125, 127, 129, 130, + 131, 133, 134, 135, 137, 138, 139, 139, 139, 139, 101, 97, 94, 90, 87, + 90, 93, 96, 100, 103, 106, 109, 113, 116, 118, 121, 124, 125, 127, 129, + 131, 132, 134, 135, 136, 138, 139, 140, 142, 142, 142, 142, 103, 99, 95, + 92, 89, 92, 94, 97, 101, 104, 107, 110, 114, 117, 119, 122, 125, 127, + 129, 131, 133, 134, 136, 137, 139, 140, 142, 143, 144, 144, 144, 144, + 105, 101, 97, 94, 91, 93, 96, 99, 102, 105, 108, 111, 115, 118, 121, + 124, 127, 129, 131, 133, 135, 137, 138, 140, 142, 143, 144, 146, 147, + 147, 147, 147, 108, 103, 100, 96, 93, 95, 98, 100, 103, 106, 109, 113, + 116, 119, 122, 125, 128, 130, 132, 134, 137, 138, 140, 142, 143, 145, + 146, 148, 149, 149, 149, 149, 110, 106, 102, 98, 95, 97, 100, 102, 105, + 108, 111, 114, 117, 120, 123, 126, 129, 131, 134, 136, 138, 140, 142, + 143, 145, 147, 148, 149, 151, 151, 151, 151, 113, 108, 104, 101, 97, 99, + 102, 104, 106, 109, 112, 115, 119, 121, 124, 127, 130, 133, 135, 137, + 140, 142, 143, 145, 147, 149, 150, 151, 153, 153, 153, 153, 115, 111, + 107, 103, 100, 102, 104, 106, 108, 111, 114, 117, 120, 123, 126, 129, + 132, 134, 136, 139, 142, 143, 145, 147, 149, 151, 152, 154, 155, 155, + 155, 155, 118, 113, 109, 105, 102, 104, 106, 108, 110, 113, 116, 118, + 122, 124, 127, 130, 133, 135, 138, 140, 143, 145, 147, 149, 151, 152, + 154, 155, 157, 157, 157, 157, 120, 116, 112, 108, 104, 106, 108, 110, + 112, 115, 117, 120, 123, 126, 128, 131, 134, 137, 139, 142, 144, 146, + 148, 150, 152, 154, 155, 157, 158, 158, 158, 158, 123, 118, 114, 110, + 107, 109, 110, 112, 114, 117, 119, 122, 125, 127, 130, 133, 136, 138, + 140, 143, 146, 148, 149, 151, 154, 155, 157, 158, 160, 160, 160, 160, + 126, 121, 117, 113, 109, 111, 113, 115, 116, 119, 121, 124, 126, 129, + 132, 134, 137, 139, 142, 144, 147, 149, 151, 153, 155, 157, 158, 160, + 162, 162, 162, 162, 126, 121, 117, 113, 109, 111, 113, 115, 116, 119, + 121, 124, 126, 129, 132, 134, 137, 139, 142, 144, 147, 149, 151, 153, + 155, 157, 158, 160, 162, 162, 162, 162, 126, 121, 117, 113, 109, 111, + 113, 115, 116, 119, 121, 124, 126, 129, 132, 134, 137, 139, 142, 144, + 147, 149, 151, 153, 155, 157, 158, 160, 162, 162, 162, 162, 126, 121, + 117, 113, 109, 111, 113, 115, 116, 119, 121, 124, 126, 129, 132, 134, + 137, 139, 142, 144, 147, 149, 151, 153, 155, 157, 158, 160, 162, 162, + 162, 162 }, + { /* Intra matrices */ + /* Size 4 */ + 33, 49, 53, 64, 49, 57, 60, 67, 53, 60, 73, 81, 64, 67, 81, 91, + /* Size 8 */ + 35, 29, 49, 51, 54, 59, 65, 72, 29, 40, 50, 46, 47, 51, 56, 62, 49, 50, + 56, 55, 55, 57, 61, 66, 51, 46, 55, 61, 63, 65, 68, 72, 54, 47, 55, 63, + 69, 72, 76, 79, 59, 51, 57, 65, 72, 78, 82, 85, 65, 56, 61, 68, 76, 82, + 86, 90, 72, 62, 66, 72, 79, 85, 90, 94, + /* Size 16 */ + 34, 31, 28, 36, 48, 49, 50, 52, 53, 55, 58, 61, 64, 67, 70, 70, 31, 32, + 33, 39, 48, 48, 47, 48, 49, 51, 53, 56, 59, 62, 65, 65, 28, 33, 39, 43, + 49, 47, 45, 45, 46, 48, 50, 52, 55, 57, 61, 61, 36, 39, 43, 47, 52, 50, + 49, 49, 49, 51, 53, 55, 57, 60, 63, 63, 48, 48, 49, 52, 55, 55, 54, 54, + 54, 55, 56, 58, 60, 62, 65, 65, 49, 48, 47, 50, 55, 55, 56, 57, 57, 58, + 60, 61, 63, 65, 68, 68, 50, 47, 45, 49, 54, 56, 59, 61, 62, 63, 64, 65, + 67, 69, 71, 71, 52, 48, 45, 49, 54, 57, 61, 62, 64, 66, 67, 69, 70, 72, + 74, 74, 53, 49, 46, 49, 54, 57, 62, 64, 67, 69, 71, 72, 74, 76, 77, 77, + 55, 51, 48, 51, 55, 58, 63, 66, 69, 71, 73, 75, 77, 78, 80, 80, 58, 53, + 50, 53, 56, 60, 64, 67, 71, 73, 76, 78, 80, 82, 83, 83, 61, 56, 52, 55, + 58, 61, 65, 69, 72, 75, 78, 80, 82, 84, 86, 86, 64, 59, 55, 57, 60, 63, + 67, 70, 74, 77, 80, 82, 85, 86, 88, 88, 67, 62, 57, 60, 62, 65, 69, 72, + 76, 78, 82, 84, 86, 88, 90, 90, 70, 65, 61, 63, 65, 68, 71, 74, 77, 80, + 83, 86, 88, 90, 92, 92, 70, 65, 61, 63, 65, 68, 71, 74, 77, 80, 83, 86, + 88, 90, 92, 92, + /* Size 32 */ + 34, 32, 31, 29, 28, 31, 35, 40, 47, 48, 48, 49, 50, 50, 51, 52, 52, 54, + 55, 56, 57, 59, 60, 62, 63, 65, 66, 68, 70, 70, 70, 70, 32, 32, 31, 31, + 30, 33, 37, 42, 47, 48, 48, 48, 48, 49, 49, 50, 50, 52, 53, 54, 55, 56, + 58, 59, 61, 62, 64, 65, 67, 67, 67, 67, 31, 31, 32, 32, 33, 35, 39, 43, + 48, 47, 47, 47, 47, 47, 48, 48, 49, 50, 51, 52, 53, 54, 55, 57, 58, 60, + 61, 63, 64, 64, 64, 64, 29, 31, 32, 34, 35, 38, 41, 44, 48, 47, 47, 46, + 45, 46, 46, 47, 47, 48, 49, 50, 51, 52, 53, 55, 56, 57, 59, 60, 62, 62, + 62, 62, 28, 30, 33, 35, 39, 41, 43, 45, 48, 47, 46, 45, 44, 44, 45, 45, + 45, 46, 47, 48, 49, 50, 51, 53, 54, 55, 57, 58, 60, 60, 60, 60, 31, 33, + 35, 38, 41, 43, 45, 47, 50, 49, 48, 47, 46, 46, 47, 47, 47, 48, 49, 50, + 50, 52, 53, 54, 55, 57, 58, 59, 61, 61, 61, 61, 35, 37, 39, 41, 43, 45, + 47, 49, 51, 50, 50, 49, 48, 48, 49, 49, 49, 50, 50, 51, 52, 53, 54, 55, + 57, 58, 59, 60, 62, 62, 62, 62, 40, 42, 43, 44, 45, 47, 49, 51, 53, 52, + 52, 51, 51, 51, 51, 51, 51, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, + 63, 63, 63, 63, 47, 47, 48, 48, 48, 50, 51, 53, 55, 54, 54, 54, 53, 53, + 53, 53, 53, 54, 54, 55, 55, 56, 57, 58, 59, 60, 61, 63, 64, 64, 64, 64, + 48, 48, 47, 47, 47, 49, 50, 52, 54, 54, 54, 54, 54, 55, 55, 55, 55, 55, + 56, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 65, 65, 65, 48, 48, 47, 47, + 46, 48, 50, 52, 54, 54, 55, 55, 56, 56, 56, 56, 57, 57, 58, 58, 59, 60, + 61, 61, 62, 63, 65, 66, 67, 67, 67, 67, 49, 48, 47, 46, 45, 47, 49, 51, + 54, 54, 55, 56, 57, 58, 58, 58, 59, 59, 60, 60, 61, 62, 63, 63, 64, 65, + 66, 67, 68, 68, 68, 68, 50, 48, 47, 45, 44, 46, 48, 51, 53, 54, 56, 57, + 59, 59, 60, 60, 61, 62, 62, 63, 63, 64, 65, 65, 66, 67, 68, 69, 70, 70, + 70, 70, 50, 49, 47, 46, 44, 46, 48, 51, 53, 55, 56, 58, 59, 60, 61, 62, + 62, 63, 63, 64, 65, 65, 66, 67, 68, 69, 69, 70, 71, 71, 71, 71, 51, 49, + 48, 46, 45, 47, 49, 51, 53, 55, 56, 58, 60, 61, 62, 63, 64, 64, 65, 66, + 66, 67, 68, 69, 69, 70, 71, 72, 73, 73, 73, 73, 52, 50, 48, 47, 45, 47, + 49, 51, 53, 55, 56, 58, 60, 62, 63, 64, 65, 66, 67, 67, 68, 69, 70, 70, + 71, 72, 73, 74, 75, 75, 75, 75, 52, 50, 49, 47, 45, 47, 49, 51, 53, 55, + 57, 59, 61, 62, 64, 65, 66, 67, 68, 69, 70, 71, 72, 72, 73, 74, 75, 76, + 76, 76, 76, 76, 54, 52, 50, 48, 46, 48, 50, 51, 54, 55, 57, 59, 62, 63, + 64, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 75, 76, 77, 78, 78, 78, 78, + 55, 53, 51, 49, 47, 49, 50, 52, 54, 56, 58, 60, 62, 63, 65, 67, 68, 69, + 70, 71, 73, 73, 74, 75, 76, 77, 78, 78, 79, 79, 79, 79, 56, 54, 52, 50, + 48, 50, 51, 53, 55, 56, 58, 60, 63, 64, 66, 67, 69, 70, 71, 73, 74, 75, + 76, 77, 77, 78, 79, 80, 81, 81, 81, 81, 57, 55, 53, 51, 49, 50, 52, 54, + 55, 57, 59, 61, 63, 65, 66, 68, 70, 71, 73, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 82, 82, 82, 82, 59, 56, 54, 52, 50, 52, 53, 55, 56, 58, 60, 62, + 64, 65, 67, 69, 71, 72, 73, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 84, + 84, 84, 60, 58, 55, 53, 51, 53, 54, 56, 57, 59, 61, 63, 65, 66, 68, 70, + 72, 73, 74, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 85, 85, 85, 62, 59, + 57, 55, 53, 54, 55, 57, 58, 60, 61, 63, 65, 67, 69, 70, 72, 74, 75, 77, + 78, 79, 80, 81, 82, 83, 84, 85, 86, 86, 86, 86, 63, 61, 58, 56, 54, 55, + 57, 58, 59, 61, 62, 64, 66, 68, 69, 71, 73, 75, 76, 77, 79, 80, 81, 82, + 84, 85, 86, 86, 87, 87, 87, 87, 65, 62, 60, 57, 55, 57, 58, 59, 60, 62, + 63, 65, 67, 69, 70, 72, 74, 75, 77, 78, 80, 81, 82, 83, 85, 86, 86, 87, + 88, 88, 88, 88, 66, 64, 61, 59, 57, 58, 59, 60, 61, 63, 65, 66, 68, 69, + 71, 73, 75, 76, 78, 79, 81, 82, 83, 84, 86, 86, 87, 88, 89, 89, 89, 89, + 68, 65, 63, 60, 58, 59, 60, 62, 63, 64, 66, 67, 69, 70, 72, 74, 76, 77, + 78, 80, 82, 83, 84, 85, 86, 87, 88, 89, 90, 90, 90, 90, 70, 67, 64, 62, + 60, 61, 62, 63, 64, 65, 67, 68, 70, 71, 73, 75, 76, 78, 79, 81, 82, 84, + 85, 86, 87, 88, 89, 90, 91, 91, 91, 91, 70, 67, 64, 62, 60, 61, 62, 63, + 64, 65, 67, 68, 70, 71, 73, 75, 76, 78, 79, 81, 82, 84, 85, 86, 87, 88, + 89, 90, 91, 91, 91, 91, 70, 67, 64, 62, 60, 61, 62, 63, 64, 65, 67, 68, + 70, 71, 73, 75, 76, 78, 79, 81, 82, 84, 85, 86, 87, 88, 89, 90, 91, 91, + 91, 91, 70, 67, 64, 62, 60, 61, 62, 63, 64, 65, 67, 68, 70, 71, 73, 75, + 76, 78, 79, 81, 82, 84, 85, 86, 87, 88, 89, 90, 91, 91, 91, 91 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 69, 100, 132, 69, 93, 117, 139, 100, 117, 142, 157, 132, 139, 157, + 169, + /* Size 8 */ + 64, 51, 54, 67, 84, 100, 114, 126, 51, 57, 55, 62, 75, 89, 104, 116, 54, + 55, 71, 80, 89, 100, 111, 121, 67, 62, 80, 94, 104, 113, 121, 129, 84, + 75, 89, 104, 115, 123, 130, 136, 100, 89, 100, 113, 123, 131, 137, 142, + 114, 104, 111, 121, 130, 137, 142, 146, 126, 116, 121, 129, 136, 142, + 146, 150, + /* Size 16 */ + 64, 57, 51, 53, 54, 60, 67, 75, 84, 91, 100, 106, 114, 119, 126, 126, + 57, 55, 54, 54, 55, 59, 65, 71, 79, 86, 94, 101, 108, 114, 121, 121, 51, + 54, 57, 56, 55, 58, 62, 68, 75, 82, 89, 96, 104, 109, 116, 116, 53, 54, + 56, 59, 62, 66, 70, 75, 82, 88, 94, 100, 107, 113, 119, 119, 54, 55, 55, + 62, 71, 76, 80, 84, 89, 94, 100, 105, 111, 116, 121, 121, 60, 59, 58, + 66, 76, 81, 87, 91, 96, 101, 106, 111, 116, 120, 125, 125, 67, 65, 62, + 70, 80, 87, 94, 99, 104, 108, 113, 116, 121, 125, 129, 129, 75, 71, 68, + 75, 84, 91, 99, 104, 109, 113, 118, 121, 125, 128, 132, 132, 84, 79, 75, + 82, 89, 96, 104, 109, 115, 119, 123, 126, 130, 133, 136, 136, 91, 86, + 82, 88, 94, 101, 108, 113, 119, 123, 127, 130, 133, 136, 139, 139, 100, + 94, 89, 94, 100, 106, 113, 118, 123, 127, 131, 134, 137, 139, 142, 142, + 106, 101, 96, 100, 105, 111, 116, 121, 126, 130, 134, 137, 140, 142, + 144, 144, 114, 108, 104, 107, 111, 116, 121, 125, 130, 133, 137, 140, + 142, 144, 146, 146, 119, 114, 109, 113, 116, 120, 125, 128, 133, 136, + 139, 142, 144, 146, 148, 148, 126, 121, 116, 119, 121, 125, 129, 132, + 136, 139, 142, 144, 146, 148, 150, 150, 126, 121, 116, 119, 121, 125, + 129, 132, 136, 139, 142, 144, 146, 148, 150, 150, + /* Size 32 */ + 64, 60, 57, 54, 51, 52, 53, 54, 54, 57, 60, 64, 67, 71, 75, 79, 84, 87, + 91, 95, 100, 103, 106, 110, 114, 117, 119, 122, 126, 126, 126, 126, 60, + 58, 56, 54, 52, 53, 53, 54, 55, 57, 60, 63, 66, 69, 73, 77, 81, 85, 88, + 92, 97, 100, 103, 107, 111, 114, 117, 120, 123, 123, 123, 123, 57, 56, + 55, 55, 54, 54, 54, 54, 55, 57, 59, 62, 65, 68, 71, 75, 79, 82, 86, 90, + 94, 97, 101, 104, 108, 111, 114, 117, 121, 121, 121, 121, 54, 54, 55, + 55, 55, 55, 55, 55, 55, 57, 59, 61, 64, 66, 70, 73, 77, 80, 84, 88, 92, + 95, 98, 102, 106, 109, 112, 115, 118, 118, 118, 118, 51, 52, 54, 55, 57, + 56, 56, 56, 55, 57, 58, 60, 62, 65, 68, 71, 75, 78, 82, 85, 89, 93, 96, + 100, 104, 106, 109, 113, 116, 116, 116, 116, 52, 53, 54, 55, 56, 57, 57, + 58, 58, 60, 62, 64, 66, 69, 72, 75, 78, 81, 84, 88, 92, 95, 98, 102, + 105, 108, 111, 114, 117, 117, 117, 117, 53, 53, 54, 55, 56, 57, 59, 61, + 62, 64, 66, 68, 70, 73, 75, 78, 82, 84, 88, 91, 94, 97, 100, 104, 107, + 110, 113, 116, 119, 119, 119, 119, 54, 54, 54, 55, 56, 58, 61, 63, 67, + 68, 70, 72, 75, 77, 80, 82, 85, 88, 91, 94, 97, 100, 103, 106, 109, 112, + 114, 117, 120, 120, 120, 120, 54, 55, 55, 55, 55, 58, 62, 67, 71, 73, + 76, 78, 80, 82, 84, 87, 89, 92, 94, 97, 100, 103, 105, 108, 111, 114, + 116, 119, 121, 121, 121, 121, 57, 57, 57, 57, 57, 60, 64, 68, 73, 76, + 78, 80, 83, 85, 88, 90, 92, 95, 97, 100, 103, 105, 108, 111, 113, 116, + 118, 121, 123, 123, 123, 123, 60, 60, 59, 59, 58, 62, 66, 70, 76, 78, + 81, 84, 87, 89, 91, 93, 96, 98, 101, 103, 106, 108, 111, 113, 116, 118, + 120, 122, 125, 125, 125, 125, 64, 63, 62, 61, 60, 64, 68, 72, 78, 80, + 84, 87, 90, 92, 95, 97, 100, 102, 104, 107, 109, 111, 113, 116, 118, + 120, 122, 124, 127, 127, 127, 127, 67, 66, 65, 64, 62, 66, 70, 75, 80, + 83, 87, 90, 94, 97, 99, 101, 104, 106, 108, 110, 113, 114, 116, 119, + 121, 123, 125, 127, 129, 129, 129, 129, 71, 69, 68, 66, 65, 69, 73, 77, + 82, 85, 89, 92, 97, 99, 101, 104, 107, 109, 111, 113, 115, 117, 119, + 121, 123, 125, 126, 128, 130, 130, 130, 130, 75, 73, 71, 70, 68, 72, 75, + 80, 84, 88, 91, 95, 99, 101, 104, 106, 109, 111, 113, 115, 118, 119, + 121, 123, 125, 127, 128, 130, 132, 132, 132, 132, 79, 77, 75, 73, 71, + 75, 78, 82, 87, 90, 93, 97, 101, 104, 106, 109, 112, 114, 116, 118, 120, + 122, 124, 126, 127, 129, 131, 132, 134, 134, 134, 134, 84, 81, 79, 77, + 75, 78, 82, 85, 89, 92, 96, 100, 104, 107, 109, 112, 115, 117, 119, 121, + 123, 125, 126, 128, 130, 131, 133, 134, 136, 136, 136, 136, 87, 85, 82, + 80, 78, 81, 84, 88, 92, 95, 98, 102, 106, 109, 111, 114, 117, 119, 121, + 123, 125, 127, 128, 130, 132, 133, 134, 136, 137, 137, 137, 137, 91, 88, + 86, 84, 82, 84, 88, 91, 94, 97, 101, 104, 108, 111, 113, 116, 119, 121, + 123, 125, 127, 129, 130, 132, 133, 135, 136, 137, 139, 139, 139, 139, + 95, 92, 90, 88, 85, 88, 91, 94, 97, 100, 103, 107, 110, 113, 115, 118, + 121, 123, 125, 127, 129, 131, 132, 134, 135, 136, 138, 139, 140, 140, + 140, 140, 100, 97, 94, 92, 89, 92, 94, 97, 100, 103, 106, 109, 113, 115, + 118, 120, 123, 125, 127, 129, 131, 133, 134, 135, 137, 138, 139, 141, + 142, 142, 142, 142, 103, 100, 97, 95, 93, 95, 97, 100, 103, 105, 108, + 111, 114, 117, 119, 122, 125, 127, 129, 131, 133, 134, 135, 137, 138, + 139, 141, 142, 143, 143, 143, 143, 106, 103, 101, 98, 96, 98, 100, 103, + 105, 108, 111, 113, 116, 119, 121, 124, 126, 128, 130, 132, 134, 135, + 137, 138, 140, 141, 142, 143, 144, 144, 144, 144, 110, 107, 104, 102, + 100, 102, 104, 106, 108, 111, 113, 116, 119, 121, 123, 126, 128, 130, + 132, 134, 135, 137, 138, 140, 141, 142, 143, 144, 145, 145, 145, 145, + 114, 111, 108, 106, 104, 105, 107, 109, 111, 113, 116, 118, 121, 123, + 125, 127, 130, 132, 133, 135, 137, 138, 140, 141, 142, 143, 144, 145, + 146, 146, 146, 146, 117, 114, 111, 109, 106, 108, 110, 112, 114, 116, + 118, 120, 123, 125, 127, 129, 131, 133, 135, 136, 138, 139, 141, 142, + 143, 144, 145, 146, 147, 147, 147, 147, 119, 117, 114, 112, 109, 111, + 113, 114, 116, 118, 120, 122, 125, 126, 128, 131, 133, 134, 136, 138, + 139, 141, 142, 143, 144, 145, 146, 147, 148, 148, 148, 148, 122, 120, + 117, 115, 113, 114, 116, 117, 119, 121, 122, 124, 127, 128, 130, 132, + 134, 136, 137, 139, 141, 142, 143, 144, 145, 146, 147, 148, 149, 149, + 149, 149, 126, 123, 121, 118, 116, 117, 119, 120, 121, 123, 125, 127, + 129, 130, 132, 134, 136, 137, 139, 140, 142, 143, 144, 145, 146, 147, + 148, 149, 150, 150, 150, 150, 126, 123, 121, 118, 116, 117, 119, 120, + 121, 123, 125, 127, 129, 130, 132, 134, 136, 137, 139, 140, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 150, 150, 150, 126, 123, 121, 118, + 116, 117, 119, 120, 121, 123, 125, 127, 129, 130, 132, 134, 136, 137, + 139, 140, 142, 143, 144, 145, 146, 147, 148, 149, 150, 150, 150, 150, + 126, 123, 121, 118, 116, 117, 119, 120, 121, 123, 125, 127, 129, 130, + 132, 134, 136, 137, 139, 140, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 150, 150, 150 }, + { /* Intra matrices */ + /* Size 4 */ + 32, 34, 51, 69, 34, 47, 60, 73, 51, 60, 75, 83, 69, 73, 83, 90, + /* Size 8 */ + 37, 29, 31, 39, 49, 59, 68, 76, 29, 33, 31, 36, 44, 53, 61, 70, 31, 31, + 41, 47, 52, 59, 66, 73, 39, 36, 47, 56, 62, 67, 73, 78, 49, 44, 52, 62, + 69, 74, 79, 83, 59, 53, 59, 67, 74, 79, 83, 87, 68, 61, 66, 73, 79, 83, + 87, 90, 76, 70, 73, 78, 83, 87, 90, 92, + /* Size 16 */ + 36, 32, 28, 29, 30, 34, 38, 42, 48, 52, 57, 61, 66, 70, 74, 74, 32, 31, + 30, 30, 30, 33, 36, 40, 45, 49, 54, 58, 63, 67, 71, 71, 28, 30, 32, 31, + 31, 33, 35, 38, 42, 46, 51, 55, 60, 64, 68, 68, 29, 30, 31, 33, 35, 37, + 40, 43, 46, 50, 54, 58, 62, 66, 69, 69, 30, 30, 31, 35, 40, 43, 45, 48, + 51, 54, 58, 61, 65, 68, 71, 71, 34, 33, 33, 37, 43, 46, 49, 52, 55, 58, + 61, 64, 68, 70, 73, 73, 38, 36, 35, 40, 45, 49, 54, 57, 60, 63, 65, 68, + 71, 73, 76, 76, 42, 40, 38, 43, 48, 52, 57, 60, 63, 66, 69, 71, 74, 76, + 78, 78, 48, 45, 42, 46, 51, 55, 60, 63, 67, 70, 72, 74, 77, 78, 80, 80, + 52, 49, 46, 50, 54, 58, 63, 66, 70, 72, 75, 77, 79, 81, 82, 82, 57, 54, + 51, 54, 58, 61, 65, 69, 72, 75, 77, 79, 81, 83, 84, 84, 61, 58, 55, 58, + 61, 64, 68, 71, 74, 77, 79, 81, 83, 84, 86, 86, 66, 63, 60, 62, 65, 68, + 71, 74, 77, 79, 81, 83, 85, 86, 87, 87, 70, 67, 64, 66, 68, 70, 73, 76, + 78, 81, 83, 84, 86, 87, 89, 89, 74, 71, 68, 69, 71, 73, 76, 78, 80, 82, + 84, 86, 87, 89, 90, 90, 74, 71, 68, 69, 71, 73, 76, 78, 80, 82, 84, 86, + 87, 89, 90, 90, + /* Size 32 */ + 35, 33, 31, 30, 28, 28, 29, 29, 30, 31, 33, 35, 37, 39, 42, 44, 47, 49, + 51, 54, 57, 59, 61, 63, 65, 67, 69, 71, 73, 73, 73, 73, 33, 32, 31, 30, + 29, 29, 29, 30, 30, 31, 33, 35, 37, 39, 41, 43, 46, 48, 50, 52, 55, 57, + 59, 61, 64, 65, 67, 69, 71, 71, 71, 71, 31, 31, 30, 30, 30, 30, 30, 30, + 30, 31, 33, 34, 36, 38, 40, 42, 44, 46, 48, 51, 53, 55, 57, 60, 62, 64, + 66, 68, 70, 70, 70, 70, 30, 30, 30, 30, 30, 30, 30, 30, 30, 31, 32, 34, + 35, 37, 39, 41, 43, 45, 47, 49, 52, 54, 56, 58, 61, 62, 64, 66, 68, 68, + 68, 68, 28, 29, 30, 30, 31, 31, 31, 31, 30, 31, 32, 33, 34, 36, 38, 40, + 42, 44, 46, 48, 50, 52, 54, 57, 59, 61, 63, 65, 67, 67, 67, 67, 28, 29, + 30, 30, 31, 31, 32, 32, 32, 33, 34, 35, 37, 38, 40, 42, 44, 46, 47, 50, + 52, 54, 56, 58, 60, 62, 64, 66, 68, 68, 68, 68, 29, 29, 30, 30, 31, 32, + 33, 33, 34, 35, 37, 38, 39, 40, 42, 44, 46, 47, 49, 51, 53, 55, 57, 59, + 61, 63, 65, 67, 68, 68, 68, 68, 29, 30, 30, 30, 31, 32, 33, 35, 37, 38, + 39, 40, 42, 43, 45, 46, 48, 50, 51, 53, 55, 57, 59, 60, 62, 64, 66, 67, + 69, 69, 69, 69, 30, 30, 30, 30, 30, 32, 34, 37, 40, 41, 42, 43, 45, 46, + 47, 49, 50, 52, 53, 55, 57, 58, 60, 62, 64, 65, 67, 68, 70, 70, 70, 70, + 31, 31, 31, 31, 31, 33, 35, 38, 41, 42, 44, 45, 47, 48, 49, 51, 52, 54, + 55, 57, 59, 60, 62, 63, 65, 67, 68, 70, 71, 71, 71, 71, 33, 33, 33, 32, + 32, 34, 37, 39, 42, 44, 45, 47, 49, 50, 51, 53, 54, 56, 57, 59, 60, 62, + 63, 65, 67, 68, 69, 71, 72, 72, 72, 72, 35, 35, 34, 34, 33, 35, 38, 40, + 43, 45, 47, 49, 51, 52, 54, 55, 57, 58, 59, 61, 62, 64, 65, 67, 68, 69, + 71, 72, 74, 74, 74, 74, 37, 37, 36, 35, 34, 37, 39, 42, 45, 47, 49, 51, + 53, 55, 56, 58, 59, 61, 62, 63, 65, 66, 67, 68, 70, 71, 72, 73, 75, 75, + 75, 75, 39, 39, 38, 37, 36, 38, 40, 43, 46, 48, 50, 52, 55, 56, 58, 59, + 61, 62, 63, 65, 66, 67, 69, 70, 71, 72, 73, 75, 76, 76, 76, 76, 42, 41, + 40, 39, 38, 40, 42, 45, 47, 49, 51, 54, 56, 58, 59, 61, 63, 64, 65, 66, + 68, 69, 70, 71, 73, 74, 75, 76, 77, 77, 77, 77, 44, 43, 42, 41, 40, 42, + 44, 46, 49, 51, 53, 55, 58, 59, 61, 63, 64, 66, 67, 68, 69, 71, 72, 73, + 74, 75, 76, 77, 78, 78, 78, 78, 47, 46, 44, 43, 42, 44, 46, 48, 50, 52, + 54, 57, 59, 61, 63, 64, 66, 67, 69, 70, 71, 72, 73, 74, 76, 76, 77, 78, + 79, 79, 79, 79, 49, 48, 46, 45, 44, 46, 47, 50, 52, 54, 56, 58, 61, 62, + 64, 66, 67, 69, 70, 71, 73, 73, 75, 76, 77, 78, 78, 79, 80, 80, 80, 80, + 51, 50, 48, 47, 46, 47, 49, 51, 53, 55, 57, 59, 62, 63, 65, 67, 69, 70, + 71, 72, 74, 75, 76, 77, 78, 79, 79, 80, 81, 81, 81, 81, 54, 52, 51, 49, + 48, 50, 51, 53, 55, 57, 59, 61, 63, 65, 66, 68, 70, 71, 72, 74, 75, 76, + 77, 78, 79, 80, 81, 81, 82, 82, 82, 82, 57, 55, 53, 52, 50, 52, 53, 55, + 57, 59, 60, 62, 65, 66, 68, 69, 71, 73, 74, 75, 76, 77, 78, 79, 80, 81, + 82, 82, 83, 83, 83, 83, 59, 57, 55, 54, 52, 54, 55, 57, 58, 60, 62, 64, + 66, 67, 69, 71, 72, 73, 75, 76, 77, 78, 79, 80, 81, 82, 82, 83, 84, 84, + 84, 84, 61, 59, 57, 56, 54, 56, 57, 59, 60, 62, 63, 65, 67, 69, 70, 72, + 73, 75, 76, 77, 78, 79, 80, 81, 82, 83, 83, 84, 85, 85, 85, 85, 63, 61, + 60, 58, 57, 58, 59, 60, 62, 63, 65, 67, 68, 70, 71, 73, 74, 76, 77, 78, + 79, 80, 81, 82, 83, 83, 84, 85, 85, 85, 85, 85, 65, 64, 62, 61, 59, 60, + 61, 62, 64, 65, 67, 68, 70, 71, 73, 74, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 84, 85, 86, 86, 86, 86, 86, 67, 65, 64, 62, 61, 62, 63, 64, 65, 67, + 68, 69, 71, 72, 74, 75, 76, 78, 79, 80, 81, 82, 83, 83, 84, 85, 86, 86, + 87, 87, 87, 87, 69, 67, 66, 64, 63, 64, 65, 66, 67, 68, 69, 71, 72, 73, + 75, 76, 77, 78, 79, 81, 82, 82, 83, 84, 85, 86, 86, 87, 87, 87, 87, 87, + 71, 69, 68, 66, 65, 66, 67, 67, 68, 70, 71, 72, 73, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 86, 87, 87, 88, 88, 88, 88, 73, 71, 70, 68, + 67, 68, 68, 69, 70, 71, 72, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 85, 85, 86, 87, 87, 88, 89, 89, 89, 89, 73, 71, 70, 68, 67, 68, 68, 69, + 70, 71, 72, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 85, 86, 87, + 87, 88, 89, 89, 89, 89, 73, 71, 70, 68, 67, 68, 68, 69, 70, 71, 72, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 85, 86, 87, 87, 88, 89, 89, + 89, 89, 73, 71, 70, 68, 67, 68, 68, 69, 70, 71, 72, 74, 75, 76, 77, 78, + 79, 80, 81, 82, 83, 84, 85, 85, 86, 87, 87, 88, 89, 89, 89, 89 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 91, 97, 113, 91, 104, 108, 119, 97, 108, 127, 138, 113, 119, 138, + 151, + /* Size 8 */ + 64, 54, 86, 89, 93, 101, 109, 118, 54, 72, 87, 81, 82, 88, 96, 104, 86, + 87, 97, 95, 94, 98, 103, 110, 89, 81, 95, 103, 106, 109, 113, 119, 93, + 82, 94, 106, 114, 119, 123, 127, 101, 88, 98, 109, 119, 126, 131, 135, + 109, 96, 103, 113, 123, 131, 137, 142, 118, 104, 110, 119, 127, 135, + 142, 147, + /* Size 16 */ + 64, 58, 54, 66, 86, 87, 89, 91, 93, 97, 101, 105, 109, 114, 118, 118, + 58, 60, 62, 72, 86, 85, 85, 86, 88, 91, 94, 98, 102, 106, 111, 111, 54, + 62, 72, 79, 87, 84, 81, 81, 82, 85, 88, 92, 96, 100, 104, 104, 66, 72, + 79, 85, 91, 89, 87, 87, 88, 90, 93, 96, 100, 103, 107, 107, 86, 86, 87, + 91, 97, 96, 95, 94, 94, 96, 98, 100, 103, 107, 110, 110, 87, 85, 84, 89, + 96, 97, 98, 99, 100, 101, 103, 106, 108, 111, 114, 114, 89, 85, 81, 87, + 95, 98, 103, 104, 106, 108, 109, 111, 113, 116, 119, 119, 91, 86, 81, + 87, 94, 99, 104, 107, 110, 112, 114, 116, 118, 120, 123, 123, 93, 88, + 82, 88, 94, 100, 106, 110, 114, 116, 119, 121, 123, 125, 127, 127, 97, + 91, 85, 90, 96, 101, 108, 112, 116, 119, 122, 124, 127, 129, 131, 131, + 101, 94, 88, 93, 98, 103, 109, 114, 119, 122, 126, 128, 131, 133, 135, + 135, 105, 98, 92, 96, 100, 106, 111, 116, 121, 124, 128, 131, 134, 136, + 138, 138, 109, 102, 96, 100, 103, 108, 113, 118, 123, 127, 131, 134, + 137, 139, 142, 142, 114, 106, 100, 103, 107, 111, 116, 120, 125, 129, + 133, 136, 139, 142, 144, 144, 118, 111, 104, 107, 110, 114, 119, 123, + 127, 131, 135, 138, 142, 144, 147, 147, 118, 111, 104, 107, 110, 114, + 119, 123, 127, 131, 135, 138, 142, 144, 147, 147, + /* Size 32 */ + 64, 61, 58, 56, 54, 59, 66, 75, 86, 86, 87, 88, 89, 90, 91, 92, 93, 95, + 97, 99, 101, 103, 105, 107, 109, 111, 114, 116, 118, 118, 118, 118, 61, + 60, 59, 58, 57, 63, 69, 76, 86, 86, 86, 87, 87, 88, 89, 90, 90, 92, 94, + 96, 97, 99, 101, 103, 106, 108, 110, 112, 114, 114, 114, 114, 58, 59, + 60, 61, 62, 66, 72, 78, 86, 86, 85, 85, 85, 85, 86, 87, 88, 89, 91, 92, + 94, 96, 98, 100, 102, 104, 106, 109, 111, 111, 111, 111, 56, 58, 61, 63, + 66, 71, 75, 80, 86, 85, 84, 84, 83, 83, 84, 84, 85, 86, 88, 89, 91, 93, + 95, 97, 99, 101, 103, 105, 108, 108, 108, 108, 54, 57, 62, 66, 72, 75, + 79, 83, 87, 85, 84, 82, 81, 81, 81, 82, 82, 84, 85, 87, 88, 90, 92, 94, + 96, 98, 100, 102, 104, 104, 104, 104, 59, 63, 66, 71, 75, 78, 82, 85, + 89, 88, 86, 85, 84, 84, 84, 85, 85, 86, 88, 89, 90, 92, 94, 96, 98, 100, + 102, 104, 106, 106, 106, 106, 66, 69, 72, 75, 79, 82, 85, 88, 91, 90, + 89, 88, 87, 87, 87, 88, 88, 89, 90, 91, 93, 94, 96, 98, 100, 101, 103, + 105, 107, 107, 107, 107, 75, 76, 78, 80, 83, 85, 88, 91, 94, 93, 92, 91, + 91, 91, 91, 91, 91, 92, 93, 94, 95, 97, 98, 100, 101, 103, 105, 107, + 109, 109, 109, 109, 86, 86, 86, 86, 87, 89, 91, 94, 97, 96, 96, 95, 95, + 94, 94, 94, 94, 95, 96, 97, 98, 99, 100, 102, 103, 105, 107, 109, 110, + 110, 110, 110, 86, 86, 86, 85, 85, 88, 90, 93, 96, 96, 96, 96, 96, 97, + 97, 97, 97, 98, 99, 99, 100, 102, 103, 104, 106, 107, 109, 111, 112, + 112, 112, 112, 87, 86, 85, 84, 84, 86, 89, 92, 96, 96, 97, 98, 98, 99, + 99, 100, 100, 101, 101, 102, 103, 104, 106, 107, 108, 110, 111, 113, + 114, 114, 114, 114, 88, 87, 85, 84, 82, 85, 88, 91, 95, 96, 98, 99, 101, + 101, 102, 102, 103, 104, 104, 105, 106, 107, 108, 109, 111, 112, 113, + 115, 116, 116, 116, 116, 89, 87, 85, 83, 81, 84, 87, 91, 95, 96, 98, + 101, 103, 104, 104, 105, 106, 107, 108, 108, 109, 110, 111, 112, 113, + 115, 116, 117, 119, 119, 119, 119, 90, 88, 85, 83, 81, 84, 87, 91, 94, + 97, 99, 101, 104, 105, 106, 107, 108, 109, 110, 110, 111, 112, 113, 114, + 116, 117, 118, 119, 121, 121, 121, 121, 91, 89, 86, 84, 81, 84, 87, 91, + 94, 97, 99, 102, 104, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 122, 123, 123, 123, 123, 92, 90, 87, 84, 82, 85, 88, + 91, 94, 97, 100, 102, 105, 107, 108, 110, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 122, 123, 124, 125, 125, 125, 125, 93, 90, 88, 85, 82, + 85, 88, 91, 94, 97, 100, 103, 106, 108, 110, 112, 114, 115, 116, 118, + 119, 120, 121, 122, 123, 124, 125, 126, 127, 127, 127, 127, 95, 92, 89, + 86, 84, 86, 89, 92, 95, 98, 101, 104, 107, 109, 111, 113, 115, 116, 118, + 119, 120, 122, 123, 124, 125, 126, 127, 128, 129, 129, 129, 129, 97, 94, + 91, 88, 85, 88, 90, 93, 96, 99, 101, 104, 108, 110, 112, 114, 116, 118, + 119, 121, 122, 123, 124, 126, 127, 128, 129, 130, 131, 131, 131, 131, + 99, 96, 92, 89, 87, 89, 91, 94, 97, 99, 102, 105, 108, 110, 113, 115, + 118, 119, 121, 122, 124, 125, 126, 128, 129, 130, 131, 132, 133, 133, + 133, 133, 101, 97, 94, 91, 88, 90, 93, 95, 98, 100, 103, 106, 109, 111, + 114, 116, 119, 120, 122, 124, 126, 127, 128, 130, 131, 132, 133, 134, + 135, 135, 135, 135, 103, 99, 96, 93, 90, 92, 94, 97, 99, 102, 104, 107, + 110, 112, 115, 117, 120, 122, 123, 125, 127, 128, 130, 131, 132, 133, + 134, 136, 137, 137, 137, 137, 105, 101, 98, 95, 92, 94, 96, 98, 100, + 103, 106, 108, 111, 113, 116, 118, 121, 123, 124, 126, 128, 130, 131, + 132, 134, 135, 136, 137, 138, 138, 138, 138, 107, 103, 100, 97, 94, 96, + 98, 100, 102, 104, 107, 109, 112, 114, 117, 119, 122, 124, 126, 128, + 130, 131, 132, 134, 135, 136, 138, 139, 140, 140, 140, 140, 109, 106, + 102, 99, 96, 98, 100, 101, 103, 106, 108, 111, 113, 116, 118, 120, 123, + 125, 127, 129, 131, 132, 134, 135, 137, 138, 139, 140, 142, 142, 142, + 142, 111, 108, 104, 101, 98, 100, 101, 103, 105, 107, 110, 112, 115, + 117, 119, 122, 124, 126, 128, 130, 132, 133, 135, 136, 138, 139, 140, + 142, 143, 143, 143, 143, 114, 110, 106, 103, 100, 102, 103, 105, 107, + 109, 111, 113, 116, 118, 120, 123, 125, 127, 129, 131, 133, 134, 136, + 138, 139, 140, 142, 143, 144, 144, 144, 144, 116, 112, 109, 105, 102, + 104, 105, 107, 109, 111, 113, 115, 117, 119, 122, 124, 126, 128, 130, + 132, 134, 136, 137, 139, 140, 142, 143, 144, 145, 145, 145, 145, 118, + 114, 111, 108, 104, 106, 107, 109, 110, 112, 114, 116, 119, 121, 123, + 125, 127, 129, 131, 133, 135, 137, 138, 140, 142, 143, 144, 145, 147, + 147, 147, 147, 118, 114, 111, 108, 104, 106, 107, 109, 110, 112, 114, + 116, 119, 121, 123, 125, 127, 129, 131, 133, 135, 137, 138, 140, 142, + 143, 144, 145, 147, 147, 147, 147, 118, 114, 111, 108, 104, 106, 107, + 109, 110, 112, 114, 116, 119, 121, 123, 125, 127, 129, 131, 133, 135, + 137, 138, 140, 142, 143, 144, 145, 147, 147, 147, 147, 118, 114, 111, + 108, 104, 106, 107, 109, 110, 112, 114, 116, 119, 121, 123, 125, 127, + 129, 131, 133, 135, 137, 138, 140, 142, 143, 144, 145, 147, 147, 147, + 147 }, + { /* Intra matrices */ + /* Size 4 */ + 35, 51, 54, 64, 51, 58, 61, 67, 54, 61, 72, 79, 64, 67, 79, 87, + /* Size 8 */ + 37, 31, 51, 53, 56, 60, 66, 71, 31, 42, 51, 48, 49, 52, 57, 63, 51, 51, + 58, 56, 56, 58, 62, 66, 53, 48, 56, 61, 64, 66, 68, 72, 56, 49, 56, 64, + 69, 72, 75, 77, 60, 52, 58, 66, 72, 76, 80, 83, 66, 57, 62, 68, 75, 80, + 84, 87, 71, 63, 66, 72, 77, 83, 87, 90, + /* Size 16 */ + 37, 33, 31, 38, 50, 51, 52, 53, 55, 57, 59, 62, 64, 67, 70, 70, 33, 34, + 35, 41, 50, 50, 49, 50, 51, 53, 55, 57, 60, 63, 65, 65, 31, 35, 41, 45, + 50, 48, 47, 47, 48, 49, 51, 54, 56, 59, 61, 61, 38, 41, 45, 49, 53, 52, + 51, 51, 51, 53, 54, 56, 58, 61, 63, 63, 50, 50, 50, 53, 57, 56, 55, 55, + 55, 56, 57, 59, 61, 63, 65, 65, 51, 50, 48, 52, 56, 57, 58, 58, 58, 59, + 60, 62, 64, 66, 68, 68, 52, 49, 47, 51, 55, 58, 60, 61, 62, 63, 64, 66, + 67, 69, 70, 70, 53, 50, 47, 51, 55, 58, 61, 63, 65, 66, 67, 69, 70, 71, + 73, 73, 55, 51, 48, 51, 55, 58, 62, 65, 67, 69, 70, 72, 73, 75, 76, 76, + 57, 53, 49, 53, 56, 59, 63, 66, 69, 71, 73, 74, 76, 77, 78, 78, 59, 55, + 51, 54, 57, 60, 64, 67, 70, 73, 75, 77, 78, 80, 81, 81, 62, 57, 54, 56, + 59, 62, 66, 69, 72, 74, 77, 78, 80, 82, 83, 83, 64, 60, 56, 58, 61, 64, + 67, 70, 73, 76, 78, 80, 82, 84, 85, 85, 67, 63, 59, 61, 63, 66, 69, 71, + 75, 77, 80, 82, 84, 85, 87, 87, 70, 65, 61, 63, 65, 68, 70, 73, 76, 78, + 81, 83, 85, 87, 89, 89, 70, 65, 61, 63, 65, 68, 70, 73, 76, 78, 81, 83, + 85, 87, 89, 89, + /* Size 32 */ + 36, 35, 33, 32, 30, 33, 37, 43, 49, 50, 50, 51, 51, 52, 53, 53, 54, 55, + 56, 57, 58, 60, 61, 62, 64, 65, 66, 68, 69, 69, 69, 69, 35, 34, 33, 33, + 32, 35, 39, 44, 49, 49, 50, 50, 50, 50, 51, 52, 52, 53, 54, 55, 56, 58, + 59, 60, 62, 63, 64, 66, 67, 67, 67, 67, 33, 33, 34, 34, 35, 38, 41, 45, + 49, 49, 49, 49, 49, 49, 49, 50, 50, 51, 52, 53, 54, 56, 57, 58, 59, 61, + 62, 63, 65, 65, 65, 65, 32, 33, 34, 36, 38, 40, 43, 46, 50, 49, 48, 48, + 47, 48, 48, 48, 49, 50, 51, 51, 52, 54, 55, 56, 57, 59, 60, 61, 63, 63, + 63, 63, 30, 32, 35, 38, 41, 43, 45, 47, 50, 49, 48, 47, 46, 46, 47, 47, + 47, 48, 49, 50, 51, 52, 53, 54, 55, 57, 58, 59, 61, 61, 61, 61, 33, 35, + 38, 40, 43, 45, 47, 49, 51, 50, 50, 49, 48, 48, 48, 49, 49, 50, 50, 51, + 52, 53, 54, 55, 57, 58, 59, 60, 62, 62, 62, 62, 37, 39, 41, 43, 45, 47, + 49, 51, 53, 52, 51, 51, 50, 50, 50, 50, 51, 51, 52, 53, 54, 54, 56, 57, + 58, 59, 60, 61, 63, 63, 63, 63, 43, 44, 45, 46, 47, 49, 51, 52, 54, 54, + 53, 53, 52, 52, 52, 52, 52, 53, 54, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 63, 63, 63, 49, 49, 49, 50, 50, 51, 53, 54, 56, 56, 55, 55, 55, 55, + 55, 55, 54, 55, 56, 56, 57, 57, 58, 59, 60, 61, 62, 63, 64, 64, 64, 64, + 50, 49, 49, 49, 49, 50, 52, 54, 56, 56, 56, 56, 56, 56, 56, 56, 56, 57, + 57, 58, 58, 59, 60, 61, 62, 63, 64, 65, 66, 66, 66, 66, 50, 50, 49, 48, + 48, 50, 51, 53, 55, 56, 56, 57, 57, 57, 57, 58, 58, 58, 59, 59, 60, 61, + 61, 62, 63, 64, 65, 66, 67, 67, 67, 67, 51, 50, 49, 48, 47, 49, 51, 53, + 55, 56, 57, 57, 58, 59, 59, 59, 60, 60, 61, 61, 62, 62, 63, 64, 65, 65, + 66, 67, 68, 68, 68, 68, 51, 50, 49, 47, 46, 48, 50, 52, 55, 56, 57, 58, + 60, 60, 61, 61, 62, 62, 63, 63, 64, 64, 65, 66, 66, 67, 68, 69, 70, 70, + 70, 70, 52, 50, 49, 48, 46, 48, 50, 52, 55, 56, 57, 59, 60, 61, 62, 62, + 63, 63, 64, 65, 65, 66, 66, 67, 68, 68, 69, 70, 71, 71, 71, 71, 53, 51, + 49, 48, 47, 48, 50, 52, 55, 56, 57, 59, 61, 62, 62, 63, 64, 65, 65, 66, + 67, 67, 68, 69, 69, 70, 71, 71, 72, 72, 72, 72, 53, 52, 50, 48, 47, 49, + 50, 52, 55, 56, 58, 59, 61, 62, 63, 64, 65, 66, 67, 67, 68, 69, 69, 70, + 71, 71, 72, 73, 74, 74, 74, 74, 54, 52, 50, 49, 47, 49, 51, 52, 54, 56, + 58, 60, 62, 63, 64, 65, 67, 67, 68, 69, 70, 70, 71, 72, 72, 73, 74, 74, + 75, 75, 75, 75, 55, 53, 51, 50, 48, 50, 51, 53, 55, 57, 58, 60, 62, 63, + 65, 66, 67, 68, 69, 70, 71, 71, 72, 73, 74, 74, 75, 76, 76, 76, 76, 76, + 56, 54, 52, 51, 49, 50, 52, 54, 56, 57, 59, 61, 63, 64, 65, 67, 68, 69, + 70, 71, 72, 73, 73, 74, 75, 76, 76, 77, 78, 78, 78, 78, 57, 55, 53, 51, + 50, 51, 53, 54, 56, 58, 59, 61, 63, 65, 66, 67, 69, 70, 71, 72, 73, 74, + 75, 75, 76, 77, 78, 78, 79, 79, 79, 79, 58, 56, 54, 52, 51, 52, 54, 55, + 57, 58, 60, 62, 64, 65, 67, 68, 70, 71, 72, 73, 74, 75, 76, 77, 77, 78, + 79, 80, 80, 80, 80, 80, 60, 58, 56, 54, 52, 53, 54, 56, 57, 59, 61, 62, + 64, 66, 67, 69, 70, 71, 73, 74, 75, 76, 77, 78, 78, 79, 80, 81, 81, 81, + 81, 81, 61, 59, 57, 55, 53, 54, 56, 57, 58, 60, 61, 63, 65, 66, 68, 69, + 71, 72, 73, 75, 76, 77, 78, 78, 79, 80, 81, 82, 82, 82, 82, 82, 62, 60, + 58, 56, 54, 55, 57, 58, 59, 61, 62, 64, 66, 67, 69, 70, 72, 73, 74, 75, + 77, 78, 78, 79, 80, 81, 82, 83, 83, 83, 83, 83, 64, 62, 59, 57, 55, 57, + 58, 59, 60, 62, 63, 65, 66, 68, 69, 71, 72, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 83, 84, 84, 84, 84, 84, 65, 63, 61, 59, 57, 58, 59, 60, 61, 63, + 64, 65, 67, 68, 70, 71, 73, 74, 76, 77, 78, 79, 80, 81, 82, 83, 84, 84, + 85, 85, 85, 85, 66, 64, 62, 60, 58, 59, 60, 61, 62, 64, 65, 66, 68, 69, + 71, 72, 74, 75, 76, 78, 79, 80, 81, 82, 83, 84, 84, 85, 86, 86, 86, 86, + 68, 66, 63, 61, 59, 60, 61, 62, 63, 65, 66, 67, 69, 70, 71, 73, 74, 76, + 77, 78, 80, 81, 82, 83, 84, 84, 85, 86, 87, 87, 87, 87, 69, 67, 65, 63, + 61, 62, 63, 63, 64, 66, 67, 68, 70, 71, 72, 74, 75, 76, 78, 79, 80, 81, + 82, 83, 84, 85, 86, 87, 88, 88, 88, 88, 69, 67, 65, 63, 61, 62, 63, 63, + 64, 66, 67, 68, 70, 71, 72, 74, 75, 76, 78, 79, 80, 81, 82, 83, 84, 85, + 86, 87, 88, 88, 88, 88, 69, 67, 65, 63, 61, 62, 63, 63, 64, 66, 67, 68, + 70, 71, 72, 74, 75, 76, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 88, + 88, 88, 69, 67, 65, 63, 61, 62, 63, 63, 64, 66, 67, 68, 70, 71, 72, 74, + 75, 76, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 88, 88, 88 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 68, 96, 122, 68, 90, 110, 128, 96, 110, 130, 142, 122, 128, 142, + 150, + /* Size 8 */ + 64, 52, 55, 67, 81, 95, 107, 116, 52, 58, 56, 63, 74, 86, 98, 108, 55, + 56, 71, 78, 86, 95, 104, 113, 67, 63, 78, 91, 99, 106, 112, 118, 81, 74, + 86, 99, 108, 114, 119, 124, 95, 86, 95, 106, 114, 120, 125, 128, 107, + 98, 104, 112, 119, 125, 129, 132, 116, 108, 113, 118, 124, 128, 132, + 134, + /* Size 16 */ + 64, 57, 52, 53, 55, 61, 67, 74, 81, 88, 95, 100, 107, 111, 116, 116, 57, + 56, 55, 55, 55, 60, 65, 71, 77, 84, 91, 96, 102, 107, 112, 112, 52, 55, + 58, 57, 56, 59, 63, 68, 74, 80, 86, 92, 98, 103, 108, 108, 53, 55, 57, + 59, 62, 66, 70, 74, 80, 85, 91, 96, 101, 106, 110, 110, 55, 55, 56, 62, + 71, 74, 78, 82, 86, 91, 95, 100, 104, 108, 113, 113, 61, 60, 59, 66, 74, + 79, 84, 88, 92, 96, 100, 104, 108, 112, 115, 115, 67, 65, 63, 70, 78, + 84, 91, 94, 99, 102, 106, 109, 112, 115, 118, 118, 74, 71, 68, 74, 82, + 88, 94, 98, 103, 106, 110, 112, 116, 118, 121, 121, 81, 77, 74, 80, 86, + 92, 99, 103, 108, 111, 114, 117, 119, 121, 124, 124, 88, 84, 80, 85, 91, + 96, 102, 106, 111, 114, 117, 119, 122, 124, 126, 126, 95, 91, 86, 91, + 95, 100, 106, 110, 114, 117, 120, 122, 125, 126, 128, 128, 100, 96, 92, + 96, 100, 104, 109, 112, 117, 119, 122, 124, 127, 128, 130, 130, 107, + 102, 98, 101, 104, 108, 112, 116, 119, 122, 125, 127, 129, 130, 132, + 132, 111, 107, 103, 106, 108, 112, 115, 118, 121, 124, 126, 128, 130, + 131, 133, 133, 116, 112, 108, 110, 113, 115, 118, 121, 124, 126, 128, + 130, 132, 133, 134, 134, 116, 112, 108, 110, 113, 115, 118, 121, 124, + 126, 128, 130, 132, 133, 134, 134, + /* Size 32 */ + 64, 60, 57, 55, 52, 53, 53, 54, 55, 58, 61, 64, 67, 70, 74, 77, 81, 84, + 88, 91, 95, 98, 100, 103, 107, 109, 111, 113, 116, 116, 116, 116, 60, + 58, 57, 55, 53, 54, 54, 55, 55, 58, 60, 63, 66, 69, 72, 76, 79, 82, 86, + 89, 93, 95, 98, 101, 104, 107, 109, 111, 114, 114, 114, 114, 57, 57, 56, + 55, 55, 55, 55, 55, 55, 57, 60, 62, 65, 68, 71, 74, 77, 80, 84, 87, 91, + 93, 96, 99, 102, 105, 107, 109, 112, 112, 112, 112, 55, 55, 55, 56, 56, + 56, 56, 56, 56, 57, 59, 61, 64, 66, 69, 72, 76, 79, 82, 85, 88, 91, 94, + 97, 100, 103, 105, 108, 110, 110, 110, 110, 52, 53, 55, 56, 58, 57, 57, + 56, 56, 57, 59, 61, 63, 65, 68, 71, 74, 77, 80, 83, 86, 89, 92, 95, 98, + 101, 103, 106, 108, 108, 108, 108, 53, 54, 55, 56, 57, 58, 58, 58, 59, + 60, 62, 64, 66, 68, 71, 74, 77, 79, 82, 85, 89, 91, 94, 97, 100, 102, + 104, 107, 109, 109, 109, 109, 53, 54, 55, 56, 57, 58, 59, 61, 62, 64, + 66, 68, 70, 72, 74, 77, 80, 82, 85, 88, 91, 93, 96, 98, 101, 103, 106, + 108, 110, 110, 110, 110, 54, 55, 55, 56, 56, 58, 61, 63, 66, 68, 70, 72, + 74, 76, 78, 80, 83, 85, 88, 90, 93, 95, 98, 100, 103, 105, 107, 109, + 112, 112, 112, 112, 55, 55, 55, 56, 56, 59, 62, 66, 71, 73, 74, 76, 78, + 80, 82, 84, 86, 88, 91, 93, 95, 97, 100, 102, 104, 106, 108, 110, 113, + 113, 113, 113, 58, 58, 57, 57, 57, 60, 64, 68, 73, 74, 77, 79, 81, 83, + 85, 87, 89, 91, 93, 95, 98, 100, 102, 104, 106, 108, 110, 112, 114, 114, + 114, 114, 61, 60, 60, 59, 59, 62, 66, 70, 74, 77, 79, 81, 84, 86, 88, + 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 113, 115, 115, + 115, 115, 64, 63, 62, 61, 61, 64, 68, 72, 76, 79, 81, 84, 87, 89, 91, + 93, 95, 97, 99, 101, 103, 105, 106, 108, 110, 112, 113, 115, 117, 117, + 117, 117, 67, 66, 65, 64, 63, 66, 70, 74, 78, 81, 84, 87, 91, 92, 94, + 96, 99, 100, 102, 104, 106, 107, 109, 110, 112, 114, 115, 117, 118, 118, + 118, 118, 70, 69, 68, 66, 65, 68, 72, 76, 80, 83, 86, 89, 92, 94, 96, + 99, 101, 102, 104, 106, 108, 109, 111, 112, 114, 115, 117, 118, 119, + 119, 119, 119, 74, 72, 71, 69, 68, 71, 74, 78, 82, 85, 88, 91, 94, 96, + 98, 101, 103, 105, 106, 108, 110, 111, 112, 114, 116, 117, 118, 119, + 121, 121, 121, 121, 77, 76, 74, 72, 71, 74, 77, 80, 84, 87, 90, 93, 96, + 99, 101, 103, 105, 107, 108, 110, 112, 113, 114, 116, 117, 118, 120, + 121, 122, 122, 122, 122, 81, 79, 77, 76, 74, 77, 80, 83, 86, 89, 92, 95, + 99, 101, 103, 105, 108, 109, 111, 112, 114, 115, 117, 118, 119, 120, + 121, 122, 124, 124, 124, 124, 84, 82, 80, 79, 77, 79, 82, 85, 88, 91, + 94, 97, 100, 102, 105, 107, 109, 111, 112, 114, 115, 117, 118, 119, 120, + 121, 123, 124, 125, 125, 125, 125, 88, 86, 84, 82, 80, 82, 85, 88, 91, + 93, 96, 99, 102, 104, 106, 108, 111, 112, 114, 115, 117, 118, 119, 121, + 122, 123, 124, 125, 126, 126, 126, 126, 91, 89, 87, 85, 83, 85, 88, 90, + 93, 95, 98, 101, 104, 106, 108, 110, 112, 114, 115, 117, 119, 120, 121, + 122, 123, 124, 125, 126, 127, 127, 127, 127, 95, 93, 91, 88, 86, 89, 91, + 93, 95, 98, 100, 103, 106, 108, 110, 112, 114, 115, 117, 119, 120, 121, + 122, 123, 125, 125, 126, 127, 128, 128, 128, 128, 98, 95, 93, 91, 89, + 91, 93, 95, 97, 100, 102, 105, 107, 109, 111, 113, 115, 117, 118, 120, + 121, 122, 123, 124, 126, 126, 127, 128, 129, 129, 129, 129, 100, 98, 96, + 94, 92, 94, 96, 98, 100, 102, 104, 106, 109, 111, 112, 114, 117, 118, + 119, 121, 122, 123, 124, 125, 127, 127, 128, 129, 130, 130, 130, 130, + 103, 101, 99, 97, 95, 97, 98, 100, 102, 104, 106, 108, 110, 112, 114, + 116, 118, 119, 121, 122, 123, 124, 125, 127, 128, 128, 129, 130, 131, + 131, 131, 131, 107, 104, 102, 100, 98, 100, 101, 103, 104, 106, 108, + 110, 112, 114, 116, 117, 119, 120, 122, 123, 125, 126, 127, 128, 129, + 129, 130, 131, 132, 132, 132, 132, 109, 107, 105, 103, 101, 102, 103, + 105, 106, 108, 110, 112, 114, 115, 117, 118, 120, 121, 123, 124, 125, + 126, 127, 128, 129, 130, 131, 131, 132, 132, 132, 132, 111, 109, 107, + 105, 103, 104, 106, 107, 108, 110, 112, 113, 115, 117, 118, 120, 121, + 123, 124, 125, 126, 127, 128, 129, 130, 131, 131, 132, 133, 133, 133, + 133, 113, 111, 109, 108, 106, 107, 108, 109, 110, 112, 113, 115, 117, + 118, 119, 121, 122, 124, 125, 126, 127, 128, 129, 130, 131, 131, 132, + 133, 134, 134, 134, 134, 116, 114, 112, 110, 108, 109, 110, 112, 113, + 114, 115, 117, 118, 119, 121, 122, 124, 125, 126, 127, 128, 129, 130, + 131, 132, 132, 133, 134, 134, 134, 134, 134, 116, 114, 112, 110, 108, + 109, 110, 112, 113, 114, 115, 117, 118, 119, 121, 122, 124, 125, 126, + 127, 128, 129, 130, 131, 132, 132, 133, 134, 134, 134, 134, 134, 116, + 114, 112, 110, 108, 109, 110, 112, 113, 114, 115, 117, 118, 119, 121, + 122, 124, 125, 126, 127, 128, 129, 130, 131, 132, 132, 133, 134, 134, + 134, 134, 134, 116, 114, 112, 110, 108, 109, 110, 112, 113, 114, 115, + 117, 118, 119, 121, 122, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 132, 133, 134, 134, 134, 134, 134 }, + { /* Intra matrices */ + /* Size 4 */ + 35, 37, 53, 69, 37, 50, 61, 72, 53, 61, 74, 81, 69, 72, 81, 86, + /* Size 8 */ + 40, 32, 34, 42, 51, 60, 68, 75, 32, 36, 34, 39, 46, 55, 63, 70, 34, 34, + 44, 49, 54, 61, 67, 73, 42, 39, 49, 57, 63, 68, 72, 76, 51, 46, 54, 63, + 69, 74, 77, 80, 60, 55, 61, 68, 74, 78, 81, 84, 68, 63, 67, 72, 77, 81, + 84, 86, 75, 70, 73, 76, 80, 84, 86, 88, + /* Size 16 */ + 39, 35, 31, 32, 33, 37, 41, 45, 50, 54, 59, 63, 67, 70, 73, 73, 35, 34, + 33, 33, 33, 36, 39, 43, 47, 51, 56, 60, 64, 67, 70, 70, 31, 33, 35, 34, + 34, 36, 38, 41, 45, 49, 53, 57, 61, 64, 68, 68, 32, 33, 34, 36, 38, 40, + 42, 45, 49, 52, 56, 59, 63, 66, 69, 69, 33, 33, 34, 38, 43, 45, 48, 50, + 53, 56, 59, 62, 65, 68, 71, 71, 37, 36, 36, 40, 45, 48, 52, 54, 57, 60, + 62, 65, 68, 70, 73, 73, 41, 39, 38, 42, 48, 52, 56, 59, 61, 64, 66, 68, + 71, 73, 75, 75, 45, 43, 41, 45, 50, 54, 59, 61, 64, 66, 69, 71, 73, 75, + 77, 77, 50, 47, 45, 49, 53, 57, 61, 64, 67, 70, 72, 74, 75, 77, 78, 78, + 54, 51, 49, 52, 56, 60, 64, 66, 70, 72, 74, 76, 77, 79, 80, 80, 59, 56, + 53, 56, 59, 62, 66, 69, 72, 74, 76, 78, 79, 80, 82, 82, 63, 60, 57, 59, + 62, 65, 68, 71, 74, 76, 78, 79, 81, 82, 83, 83, 67, 64, 61, 63, 65, 68, + 71, 73, 75, 77, 79, 81, 82, 83, 84, 84, 70, 67, 64, 66, 68, 70, 73, 75, + 77, 79, 80, 82, 83, 84, 85, 85, 73, 70, 68, 69, 71, 73, 75, 77, 78, 80, + 82, 83, 84, 85, 86, 86, 73, 70, 68, 69, 71, 73, 75, 77, 78, 80, 82, 83, + 84, 85, 86, 86, + /* Size 32 */ + 38, 36, 34, 32, 31, 31, 32, 32, 33, 34, 36, 38, 40, 42, 44, 47, 49, 51, + 53, 56, 58, 60, 62, 64, 66, 67, 69, 71, 72, 72, 72, 72, 36, 35, 34, 33, + 32, 32, 32, 33, 33, 34, 36, 38, 40, 41, 43, 46, 48, 50, 52, 54, 57, 59, + 60, 62, 64, 66, 68, 69, 71, 71, 71, 71, 34, 34, 33, 33, 32, 33, 33, 33, + 33, 34, 36, 37, 39, 41, 42, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, + 66, 68, 70, 70, 70, 70, 32, 33, 33, 33, 33, 33, 33, 33, 33, 34, 35, 37, + 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 63, 65, 67, 68, 68, + 68, 68, 31, 32, 32, 33, 34, 34, 34, 33, 33, 34, 35, 36, 37, 39, 41, 43, + 45, 46, 48, 50, 53, 54, 56, 58, 60, 62, 64, 65, 67, 67, 67, 67, 31, 32, + 33, 33, 34, 34, 35, 35, 35, 36, 37, 38, 40, 41, 43, 44, 46, 48, 50, 52, + 54, 56, 57, 59, 61, 63, 64, 66, 68, 68, 68, 68, 32, 32, 33, 33, 34, 35, + 35, 36, 37, 38, 39, 41, 42, 43, 45, 46, 48, 50, 52, 53, 55, 57, 59, 61, + 62, 64, 65, 67, 69, 69, 69, 69, 32, 33, 33, 33, 33, 35, 36, 38, 40, 41, + 42, 43, 44, 46, 47, 49, 50, 52, 53, 55, 57, 58, 60, 62, 63, 65, 66, 68, + 69, 69, 69, 69, 33, 33, 33, 33, 33, 35, 37, 40, 43, 44, 45, 46, 47, 49, + 50, 51, 53, 54, 55, 57, 58, 60, 61, 63, 65, 66, 67, 69, 70, 70, 70, 70, + 34, 34, 34, 34, 34, 36, 38, 41, 44, 45, 46, 48, 49, 50, 52, 53, 54, 56, + 57, 59, 60, 61, 63, 64, 66, 67, 68, 70, 71, 71, 71, 71, 36, 36, 36, 35, + 35, 37, 39, 42, 45, 46, 48, 49, 51, 52, 54, 55, 56, 58, 59, 60, 62, 63, + 64, 66, 67, 68, 69, 71, 72, 72, 72, 72, 38, 38, 37, 37, 36, 38, 41, 43, + 46, 48, 49, 51, 53, 54, 56, 57, 58, 60, 61, 62, 63, 65, 66, 67, 68, 69, + 71, 72, 73, 73, 73, 73, 40, 40, 39, 38, 37, 40, 42, 44, 47, 49, 51, 53, + 55, 57, 58, 59, 61, 62, 63, 64, 65, 66, 67, 69, 70, 71, 72, 73, 74, 74, + 74, 74, 42, 41, 41, 40, 39, 41, 43, 46, 49, 50, 52, 54, 57, 58, 59, 61, + 62, 63, 64, 65, 67, 68, 69, 70, 71, 72, 73, 74, 75, 75, 75, 75, 44, 43, + 42, 42, 41, 43, 45, 47, 50, 52, 54, 56, 58, 59, 61, 62, 64, 65, 66, 67, + 68, 69, 70, 71, 72, 73, 74, 75, 76, 76, 76, 76, 47, 46, 45, 44, 43, 44, + 46, 49, 51, 53, 55, 57, 59, 61, 62, 64, 65, 66, 67, 68, 69, 70, 71, 72, + 73, 74, 75, 76, 77, 77, 77, 77, 49, 48, 47, 46, 45, 46, 48, 50, 53, 54, + 56, 58, 61, 62, 64, 65, 67, 68, 69, 70, 71, 72, 73, 74, 74, 75, 76, 77, + 78, 78, 78, 78, 51, 50, 49, 48, 46, 48, 50, 52, 54, 56, 58, 60, 62, 63, + 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 75, 76, 77, 78, 78, 78, 78, 78, + 53, 52, 51, 50, 48, 50, 52, 53, 55, 57, 59, 61, 63, 64, 66, 67, 69, 70, + 71, 72, 73, 74, 75, 75, 76, 77, 78, 78, 79, 79, 79, 79, 56, 54, 53, 52, + 50, 52, 53, 55, 57, 59, 60, 62, 64, 65, 67, 68, 70, 71, 72, 73, 74, 75, + 76, 76, 77, 78, 79, 79, 80, 80, 80, 80, 58, 57, 55, 54, 53, 54, 55, 57, + 58, 60, 62, 63, 65, 67, 68, 69, 71, 72, 73, 74, 75, 76, 77, 77, 78, 79, + 79, 80, 81, 81, 81, 81, 60, 59, 57, 56, 54, 56, 57, 58, 60, 61, 63, 65, + 66, 68, 69, 70, 72, 73, 74, 75, 76, 77, 77, 78, 79, 79, 80, 81, 81, 81, + 81, 81, 62, 60, 59, 58, 56, 57, 59, 60, 61, 63, 64, 66, 67, 69, 70, 71, + 73, 74, 75, 76, 77, 77, 78, 79, 80, 80, 81, 81, 82, 82, 82, 82, 64, 62, + 61, 60, 58, 59, 61, 62, 63, 64, 66, 67, 69, 70, 71, 72, 74, 75, 75, 76, + 77, 78, 79, 80, 80, 81, 81, 82, 82, 82, 82, 82, 66, 64, 63, 62, 60, 61, + 62, 63, 65, 66, 67, 68, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 80, + 81, 82, 82, 83, 83, 83, 83, 83, 67, 66, 65, 63, 62, 63, 64, 65, 66, 67, + 68, 69, 71, 72, 73, 74, 75, 76, 77, 78, 79, 79, 80, 81, 82, 82, 83, 83, + 84, 84, 84, 84, 69, 68, 66, 65, 64, 64, 65, 66, 67, 68, 69, 71, 72, 73, + 74, 75, 76, 77, 78, 79, 79, 80, 81, 81, 82, 83, 83, 84, 84, 84, 84, 84, + 71, 69, 68, 67, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, + 78, 79, 80, 81, 81, 82, 83, 83, 84, 84, 84, 84, 84, 84, 72, 71, 70, 68, + 67, 68, 69, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 78, 79, 80, 81, 81, + 82, 82, 83, 84, 84, 84, 85, 85, 85, 85, 72, 71, 70, 68, 67, 68, 69, 69, + 70, 71, 72, 73, 74, 75, 76, 77, 78, 78, 79, 80, 81, 81, 82, 82, 83, 84, + 84, 84, 85, 85, 85, 85, 72, 71, 70, 68, 67, 68, 69, 69, 70, 71, 72, 73, + 74, 75, 76, 77, 78, 78, 79, 80, 81, 81, 82, 82, 83, 84, 84, 84, 85, 85, + 85, 85, 72, 71, 70, 68, 67, 68, 69, 69, 70, 71, 72, 73, 74, 75, 76, 77, + 78, 78, 79, 80, 81, 81, 82, 82, 83, 84, 84, 84, 85, 85, 85, 85 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 88, 93, 107, 88, 99, 103, 112, 93, 103, 119, 127, 107, 112, 127, + 137, + /* Size 8 */ + 64, 54, 83, 87, 90, 97, 104, 111, 54, 71, 84, 79, 81, 86, 92, 100, 83, + 84, 93, 91, 91, 94, 99, 104, 87, 79, 91, 98, 101, 103, 107, 111, 90, 81, + 91, 101, 107, 111, 115, 118, 97, 86, 94, 103, 111, 117, 121, 124, 104, + 92, 99, 107, 115, 121, 126, 129, 111, 100, 104, 111, 118, 124, 129, 133, + /* Size 16 */ + 64, 59, 54, 66, 83, 85, 87, 88, 90, 93, 97, 100, 104, 107, 111, 111, 59, + 60, 62, 71, 84, 83, 83, 84, 85, 88, 91, 94, 98, 101, 105, 105, 54, 62, + 71, 77, 84, 82, 79, 80, 81, 83, 86, 89, 92, 96, 100, 100, 66, 71, 77, + 83, 88, 87, 85, 85, 85, 87, 90, 92, 95, 99, 102, 102, 83, 84, 84, 88, + 93, 92, 91, 91, 91, 92, 94, 96, 99, 101, 104, 104, 85, 83, 82, 87, 92, + 93, 95, 95, 96, 97, 98, 100, 103, 105, 108, 108, 87, 83, 79, 85, 91, 95, + 98, 100, 101, 102, 103, 105, 107, 109, 111, 111, 88, 84, 80, 85, 91, 95, + 100, 102, 104, 106, 107, 109, 111, 113, 115, 115, 90, 85, 81, 85, 91, + 96, 101, 104, 107, 109, 111, 113, 115, 116, 118, 118, 93, 88, 83, 87, + 92, 97, 102, 106, 109, 112, 114, 116, 118, 120, 121, 121, 97, 91, 86, + 90, 94, 98, 103, 107, 111, 114, 117, 119, 121, 123, 124, 124, 100, 94, + 89, 92, 96, 100, 105, 109, 113, 116, 119, 121, 123, 125, 127, 127, 104, + 98, 92, 95, 99, 103, 107, 111, 115, 118, 121, 123, 126, 128, 129, 129, + 107, 101, 96, 99, 101, 105, 109, 113, 116, 120, 123, 125, 128, 129, 131, + 131, 111, 105, 100, 102, 104, 108, 111, 115, 118, 121, 124, 127, 129, + 131, 133, 133, 111, 105, 100, 102, 104, 108, 111, 115, 118, 121, 124, + 127, 129, 131, 133, 133, + /* Size 32 */ + 64, 61, 59, 57, 54, 60, 66, 74, 83, 84, 85, 86, 87, 87, 88, 89, 90, 92, + 93, 95, 97, 98, 100, 102, 104, 105, 107, 109, 111, 111, 111, 111, 61, + 60, 60, 59, 58, 63, 68, 75, 84, 84, 84, 84, 85, 85, 86, 87, 88, 89, 90, + 92, 94, 95, 97, 99, 101, 102, 104, 106, 108, 108, 108, 108, 59, 60, 60, + 61, 62, 66, 71, 77, 84, 84, 83, 83, 83, 83, 84, 84, 85, 86, 88, 89, 91, + 92, 94, 96, 98, 99, 101, 103, 105, 105, 105, 105, 57, 59, 61, 64, 66, + 70, 74, 79, 84, 83, 82, 82, 81, 81, 82, 82, 83, 84, 85, 87, 88, 90, 91, + 93, 95, 97, 98, 100, 102, 102, 102, 102, 54, 58, 62, 66, 71, 74, 77, 81, + 84, 83, 82, 80, 79, 79, 80, 80, 81, 82, 83, 84, 86, 87, 89, 91, 92, 94, + 96, 98, 100, 100, 100, 100, 60, 63, 66, 70, 74, 77, 80, 83, 86, 85, 84, + 83, 82, 82, 82, 83, 83, 84, 85, 86, 88, 89, 91, 92, 94, 95, 97, 99, 101, + 101, 101, 101, 66, 68, 71, 74, 77, 80, 83, 85, 88, 88, 87, 86, 85, 85, + 85, 85, 85, 86, 87, 89, 90, 91, 92, 94, 95, 97, 99, 100, 102, 102, 102, + 102, 74, 75, 77, 79, 81, 83, 85, 88, 91, 90, 89, 89, 88, 88, 88, 88, 88, + 89, 90, 91, 92, 93, 94, 96, 97, 98, 100, 102, 103, 103, 103, 103, 83, + 84, 84, 84, 84, 86, 88, 91, 93, 93, 92, 92, 91, 91, 91, 91, 91, 92, 92, + 93, 94, 95, 96, 97, 99, 100, 101, 103, 104, 104, 104, 104, 84, 84, 84, + 83, 83, 85, 88, 90, 93, 93, 93, 93, 93, 93, 93, 93, 93, 94, 95, 95, 96, + 97, 98, 99, 101, 102, 103, 105, 106, 106, 106, 106, 85, 84, 83, 82, 82, + 84, 87, 89, 92, 93, 93, 94, 95, 95, 95, 95, 96, 96, 97, 98, 98, 99, 100, + 102, 103, 104, 105, 106, 108, 108, 108, 108, 86, 84, 83, 82, 80, 83, 86, + 89, 92, 93, 94, 95, 96, 97, 97, 98, 98, 99, 100, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 109, 109, 109, 87, 85, 83, 81, 79, 82, 85, + 88, 91, 93, 95, 96, 98, 99, 100, 100, 101, 102, 102, 103, 103, 104, 105, + 106, 107, 108, 109, 110, 111, 111, 111, 111, 87, 85, 83, 81, 79, 82, 85, + 88, 91, 93, 95, 97, 99, 100, 101, 102, 102, 103, 104, 105, 105, 106, + 107, 108, 109, 110, 111, 112, 113, 113, 113, 113, 88, 86, 84, 82, 80, + 82, 85, 88, 91, 93, 95, 97, 100, 101, 102, 103, 104, 105, 106, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 115, 115, 115, 89, 87, 84, 82, + 80, 83, 85, 88, 91, 93, 95, 98, 100, 102, 103, 104, 106, 107, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 115, 116, 116, 116, 116, 90, 88, 85, + 83, 81, 83, 85, 88, 91, 93, 96, 98, 101, 102, 104, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 116, 117, 118, 118, 118, 118, 92, 89, + 86, 84, 82, 84, 86, 89, 92, 94, 96, 99, 102, 103, 105, 107, 108, 109, + 111, 112, 113, 114, 114, 115, 116, 117, 118, 119, 120, 120, 120, 120, + 93, 90, 88, 85, 83, 85, 87, 90, 92, 95, 97, 100, 102, 104, 106, 107, + 109, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 120, 121, 121, + 121, 121, 95, 92, 89, 87, 84, 86, 89, 91, 93, 95, 98, 100, 103, 105, + 106, 108, 110, 112, 113, 114, 116, 116, 117, 118, 119, 120, 121, 122, + 123, 123, 123, 123, 97, 94, 91, 88, 86, 88, 90, 92, 94, 96, 98, 101, + 103, 105, 107, 109, 111, 113, 114, 116, 117, 118, 119, 120, 121, 122, + 123, 124, 124, 124, 124, 124, 98, 95, 92, 90, 87, 89, 91, 93, 95, 97, + 99, 102, 104, 106, 108, 110, 112, 114, 115, 116, 118, 119, 120, 121, + 122, 123, 124, 125, 126, 126, 126, 126, 100, 97, 94, 91, 89, 91, 92, 94, + 96, 98, 100, 103, 105, 107, 109, 111, 113, 114, 116, 117, 119, 120, 121, + 122, 123, 124, 125, 126, 127, 127, 127, 127, 102, 99, 96, 93, 91, 92, + 94, 96, 97, 99, 102, 104, 106, 108, 110, 112, 114, 115, 117, 118, 120, + 121, 122, 123, 125, 125, 126, 127, 128, 128, 128, 128, 104, 101, 98, 95, + 92, 94, 95, 97, 99, 101, 103, 105, 107, 109, 111, 113, 115, 116, 118, + 119, 121, 122, 123, 125, 126, 127, 128, 128, 129, 129, 129, 129, 105, + 102, 99, 97, 94, 95, 97, 98, 100, 102, 104, 106, 108, 110, 112, 114, + 116, 117, 119, 120, 122, 123, 124, 125, 127, 128, 128, 129, 130, 130, + 130, 130, 107, 104, 101, 98, 96, 97, 99, 100, 101, 103, 105, 107, 109, + 111, 113, 115, 116, 118, 120, 121, 123, 124, 125, 126, 128, 128, 129, + 130, 131, 131, 131, 131, 109, 106, 103, 100, 98, 99, 100, 102, 103, 105, + 106, 108, 110, 112, 114, 115, 117, 119, 120, 122, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 132, 132, 132, 111, 108, 105, 102, 100, 101, + 102, 103, 104, 106, 108, 109, 111, 113, 115, 116, 118, 120, 121, 123, + 124, 126, 127, 128, 129, 130, 131, 132, 133, 133, 133, 133, 111, 108, + 105, 102, 100, 101, 102, 103, 104, 106, 108, 109, 111, 113, 115, 116, + 118, 120, 121, 123, 124, 126, 127, 128, 129, 130, 131, 132, 133, 133, + 133, 133, 111, 108, 105, 102, 100, 101, 102, 103, 104, 106, 108, 109, + 111, 113, 115, 116, 118, 120, 121, 123, 124, 126, 127, 128, 129, 130, + 131, 132, 133, 133, 133, 133, 111, 108, 105, 102, 100, 101, 102, 103, + 104, 106, 108, 109, 111, 113, 115, 116, 118, 120, 121, 123, 124, 126, + 127, 128, 129, 130, 131, 132, 133, 133, 133, 133 }, + { /* Intra matrices */ + /* Size 4 */ + 37, 52, 55, 64, 52, 59, 62, 67, 55, 62, 72, 77, 64, 67, 77, 84, + /* Size 8 */ + 40, 33, 52, 54, 57, 61, 66, 71, 33, 44, 53, 49, 50, 54, 58, 63, 52, 53, + 59, 57, 57, 59, 63, 66, 54, 49, 57, 62, 64, 66, 68, 71, 57, 50, 57, 64, + 68, 71, 74, 76, 61, 54, 59, 66, 71, 75, 78, 80, 66, 58, 63, 68, 74, 78, + 81, 84, 71, 63, 66, 71, 76, 80, 84, 86, + /* Size 16 */ + 39, 36, 33, 40, 51, 52, 53, 55, 56, 58, 60, 62, 65, 67, 70, 70, 36, 37, + 38, 43, 52, 51, 51, 52, 53, 54, 56, 58, 61, 63, 66, 66, 33, 38, 44, 47, + 52, 50, 49, 49, 50, 51, 53, 55, 57, 60, 62, 62, 40, 43, 47, 51, 55, 53, + 52, 52, 53, 54, 55, 57, 59, 61, 64, 64, 51, 52, 52, 55, 58, 57, 56, 56, + 56, 57, 58, 60, 61, 63, 65, 65, 52, 51, 50, 53, 57, 58, 59, 59, 59, 60, + 61, 63, 64, 66, 68, 68, 53, 51, 49, 52, 56, 59, 61, 62, 63, 64, 65, 66, + 67, 68, 70, 70, 55, 52, 49, 52, 56, 59, 62, 64, 65, 66, 67, 68, 70, 71, + 72, 72, 56, 53, 50, 53, 56, 59, 63, 65, 67, 69, 70, 71, 72, 73, 75, 75, + 58, 54, 51, 54, 57, 60, 64, 66, 69, 70, 72, 73, 74, 76, 77, 77, 60, 56, + 53, 55, 58, 61, 65, 67, 70, 72, 74, 75, 77, 78, 79, 79, 62, 58, 55, 57, + 60, 63, 66, 68, 71, 73, 75, 77, 78, 79, 81, 81, 65, 61, 57, 59, 61, 64, + 67, 70, 72, 74, 77, 78, 80, 81, 82, 82, 67, 63, 60, 61, 63, 66, 68, 71, + 73, 76, 78, 79, 81, 82, 84, 84, 70, 66, 62, 64, 65, 68, 70, 72, 75, 77, + 79, 81, 82, 84, 85, 85, 70, 66, 62, 64, 65, 68, 70, 72, 75, 77, 79, 81, + 82, 84, 85, 85, + /* Size 32 */ + 39, 37, 35, 34, 33, 36, 40, 45, 51, 51, 52, 52, 53, 54, 54, 55, 55, 56, + 57, 58, 59, 61, 62, 63, 64, 65, 67, 68, 69, 69, 69, 69, 37, 36, 36, 35, + 35, 38, 41, 46, 51, 51, 51, 52, 52, 52, 53, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 66, 67, 67, 67, 67, 35, 36, 36, 37, 37, 40, 43, 47, + 51, 51, 51, 51, 50, 51, 51, 52, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 63, 64, 65, 65, 65, 65, 34, 35, 37, 38, 40, 42, 45, 48, 51, 51, 50, 50, + 49, 50, 50, 50, 51, 51, 52, 53, 54, 55, 56, 57, 58, 60, 61, 62, 63, 63, + 63, 63, 33, 35, 37, 40, 43, 45, 47, 49, 52, 51, 50, 49, 48, 48, 49, 49, + 49, 50, 51, 52, 52, 53, 55, 56, 57, 58, 59, 60, 62, 62, 62, 62, 36, 38, + 40, 42, 45, 47, 49, 51, 53, 52, 51, 51, 50, 50, 50, 50, 51, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 62, 62, 62, 40, 41, 43, 45, 47, 49, + 50, 52, 54, 54, 53, 52, 52, 52, 52, 52, 52, 53, 54, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 63, 63, 63, 45, 46, 47, 48, 49, 51, 52, 54, 56, 55, + 55, 54, 54, 54, 54, 54, 54, 55, 55, 56, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 64, 64, 64, 51, 51, 51, 51, 52, 53, 54, 56, 57, 57, 57, 56, 56, 56, + 56, 56, 56, 56, 57, 57, 58, 59, 59, 60, 61, 62, 63, 64, 65, 65, 65, 65, + 51, 51, 51, 51, 51, 52, 54, 55, 57, 57, 57, 57, 57, 57, 57, 57, 57, 58, + 58, 59, 59, 60, 61, 61, 62, 63, 64, 65, 66, 66, 66, 66, 52, 51, 51, 50, + 50, 51, 53, 55, 57, 57, 57, 58, 58, 58, 59, 59, 59, 59, 60, 60, 61, 61, + 62, 63, 64, 64, 65, 66, 67, 67, 67, 67, 52, 52, 51, 50, 49, 51, 52, 54, + 56, 57, 58, 59, 59, 60, 60, 60, 61, 61, 61, 62, 62, 63, 64, 64, 65, 66, + 66, 67, 68, 68, 68, 68, 53, 52, 50, 49, 48, 50, 52, 54, 56, 57, 58, 59, + 61, 61, 61, 62, 62, 63, 63, 64, 64, 65, 65, 66, 66, 67, 68, 69, 69, 69, + 69, 69, 54, 52, 51, 50, 48, 50, 52, 54, 56, 57, 58, 60, 61, 62, 62, 63, + 63, 64, 64, 65, 65, 66, 66, 67, 68, 68, 69, 70, 70, 70, 70, 70, 54, 53, + 51, 50, 49, 50, 52, 54, 56, 57, 59, 60, 61, 62, 63, 64, 64, 65, 66, 66, + 67, 67, 68, 68, 69, 70, 70, 71, 72, 72, 72, 72, 55, 53, 52, 50, 49, 50, + 52, 54, 56, 57, 59, 60, 62, 63, 64, 65, 66, 66, 67, 67, 68, 69, 69, 70, + 70, 71, 71, 72, 73, 73, 73, 73, 55, 54, 52, 51, 49, 51, 52, 54, 56, 57, + 59, 61, 62, 63, 64, 66, 67, 67, 68, 69, 69, 70, 70, 71, 72, 72, 73, 73, + 74, 74, 74, 74, 56, 55, 53, 51, 50, 51, 53, 55, 56, 58, 59, 61, 63, 64, + 65, 66, 67, 68, 69, 70, 70, 71, 71, 72, 73, 73, 74, 74, 75, 75, 75, 75, + 57, 56, 54, 52, 51, 52, 54, 55, 57, 58, 60, 61, 63, 64, 66, 67, 68, 69, + 70, 70, 71, 72, 72, 73, 74, 74, 75, 75, 76, 76, 76, 76, 58, 57, 55, 53, + 52, 53, 54, 56, 57, 59, 60, 62, 64, 65, 66, 67, 69, 70, 70, 71, 72, 73, + 73, 74, 75, 75, 76, 77, 77, 77, 77, 77, 59, 58, 56, 54, 52, 54, 55, 56, + 58, 59, 61, 62, 64, 65, 67, 68, 69, 70, 71, 72, 73, 74, 75, 75, 76, 76, + 77, 78, 78, 78, 78, 78, 61, 59, 57, 55, 53, 55, 56, 57, 59, 60, 61, 63, + 65, 66, 67, 69, 70, 71, 72, 73, 74, 75, 75, 76, 77, 77, 78, 78, 79, 79, + 79, 79, 62, 60, 58, 56, 55, 56, 57, 58, 59, 61, 62, 64, 65, 66, 68, 69, + 70, 71, 72, 73, 75, 75, 76, 77, 77, 78, 79, 79, 80, 80, 80, 80, 63, 61, + 59, 57, 56, 57, 58, 59, 60, 61, 63, 64, 66, 67, 68, 70, 71, 72, 73, 74, + 75, 76, 77, 77, 78, 79, 79, 80, 81, 81, 81, 81, 64, 62, 60, 58, 57, 58, + 59, 60, 61, 62, 64, 65, 66, 68, 69, 70, 72, 73, 74, 75, 76, 77, 77, 78, + 79, 80, 80, 81, 82, 82, 82, 82, 65, 63, 61, 60, 58, 59, 60, 61, 62, 63, + 64, 66, 67, 68, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 80, 81, 82, + 82, 82, 82, 82, 67, 64, 63, 61, 59, 60, 61, 62, 63, 64, 65, 66, 68, 69, + 70, 71, 73, 74, 75, 76, 77, 78, 79, 79, 80, 81, 82, 82, 83, 83, 83, 83, + 68, 66, 64, 62, 60, 61, 62, 63, 64, 65, 66, 67, 69, 70, 71, 72, 73, 74, + 75, 77, 78, 78, 79, 80, 81, 82, 82, 83, 84, 84, 84, 84, 69, 67, 65, 63, + 62, 62, 63, 64, 65, 66, 67, 68, 69, 70, 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 82, 83, 84, 84, 84, 84, 84, 69, 67, 65, 63, 62, 62, 63, 64, + 65, 66, 67, 68, 69, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 82, + 83, 84, 84, 84, 84, 84, 69, 67, 65, 63, 62, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 82, 83, 84, 84, 84, + 84, 84, 69, 67, 65, 63, 62, 62, 63, 64, 65, 66, 67, 68, 69, 70, 72, 73, + 74, 75, 76, 77, 78, 79, 80, 81, 82, 82, 83, 84, 84, 84, 84, 84 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 68, 92, 113, 68, 87, 103, 117, 92, 103, 119, 128, 113, 117, 128, + 134, + /* Size 8 */ + 64, 53, 56, 67, 79, 91, 100, 107, 53, 58, 56, 63, 73, 84, 93, 101, 56, + 56, 70, 77, 83, 91, 98, 105, 67, 63, 77, 87, 94, 99, 104, 109, 79, 73, + 83, 94, 101, 106, 110, 113, 91, 84, 91, 99, 106, 110, 114, 116, 100, 93, + 98, 104, 110, 114, 117, 119, 107, 101, 105, 109, 113, 116, 119, 121, + /* Size 16 */ + 64, 58, 53, 54, 56, 61, 67, 72, 79, 84, 91, 95, 100, 103, 107, 107, 58, + 57, 55, 56, 56, 60, 65, 70, 76, 81, 87, 91, 96, 100, 104, 104, 53, 55, + 58, 57, 56, 59, 63, 67, 73, 78, 84, 88, 93, 97, 101, 101, 54, 56, 57, + 60, 63, 66, 69, 73, 78, 82, 87, 91, 96, 99, 103, 103, 56, 56, 56, 63, + 70, 73, 77, 80, 83, 87, 91, 94, 98, 101, 105, 105, 61, 60, 59, 66, 73, + 77, 81, 85, 88, 91, 95, 98, 101, 104, 107, 107, 67, 65, 63, 69, 77, 81, + 87, 90, 94, 96, 99, 102, 104, 106, 109, 109, 72, 70, 67, 73, 80, 85, 90, + 93, 97, 100, 102, 104, 107, 109, 111, 111, 79, 76, 73, 78, 83, 88, 94, + 97, 101, 103, 106, 108, 110, 111, 113, 113, 84, 81, 78, 82, 87, 91, 96, + 100, 103, 105, 108, 110, 112, 113, 115, 115, 91, 87, 84, 87, 91, 95, 99, + 102, 106, 108, 110, 112, 114, 115, 116, 116, 95, 91, 88, 91, 94, 98, + 102, 104, 108, 110, 112, 114, 115, 116, 118, 118, 100, 96, 93, 96, 98, + 101, 104, 107, 110, 112, 114, 115, 117, 118, 119, 119, 103, 100, 97, 99, + 101, 104, 106, 109, 111, 113, 115, 116, 118, 119, 120, 120, 107, 104, + 101, 103, 105, 107, 109, 111, 113, 115, 116, 118, 119, 120, 121, 121, + 107, 104, 101, 103, 105, 107, 109, 111, 113, 115, 116, 118, 119, 120, + 121, 121, + /* Size 32 */ + 64, 61, 58, 55, 53, 54, 54, 55, 56, 58, 61, 64, 67, 70, 72, 76, 79, 82, + 84, 87, 91, 93, 95, 97, 100, 102, 103, 105, 107, 107, 107, 107, 61, 59, + 57, 56, 54, 55, 55, 56, 56, 58, 60, 63, 66, 68, 71, 74, 78, 80, 83, 86, + 89, 91, 93, 96, 98, 100, 102, 104, 106, 106, 106, 106, 58, 57, 57, 56, + 55, 56, 56, 56, 56, 58, 60, 62, 65, 67, 70, 73, 76, 78, 81, 84, 87, 89, + 91, 94, 96, 98, 100, 102, 104, 104, 104, 104, 55, 56, 56, 56, 57, 57, + 57, 56, 56, 58, 60, 62, 64, 66, 69, 71, 74, 77, 79, 82, 85, 87, 90, 92, + 95, 97, 99, 101, 103, 103, 103, 103, 53, 54, 55, 57, 58, 58, 57, 57, 56, + 58, 59, 61, 63, 65, 67, 70, 73, 75, 78, 81, 84, 86, 88, 91, 93, 95, 97, + 99, 101, 101, 101, 101, 54, 55, 56, 57, 58, 58, 59, 59, 59, 61, 62, 64, + 66, 68, 70, 73, 75, 77, 80, 82, 85, 87, 90, 92, 94, 96, 98, 100, 102, + 102, 102, 102, 54, 55, 56, 57, 57, 59, 60, 61, 63, 64, 66, 67, 69, 71, + 73, 75, 78, 80, 82, 84, 87, 89, 91, 93, 96, 97, 99, 101, 103, 103, 103, + 103, 55, 56, 56, 56, 57, 59, 61, 63, 66, 68, 69, 71, 73, 74, 76, 78, 80, + 82, 84, 87, 89, 91, 93, 95, 97, 99, 100, 102, 104, 104, 104, 104, 56, + 56, 56, 56, 56, 59, 63, 66, 70, 72, 73, 75, 77, 78, 80, 81, 83, 85, 87, + 89, 91, 93, 94, 96, 98, 100, 101, 103, 105, 105, 105, 105, 58, 58, 58, + 58, 58, 61, 64, 68, 72, 73, 75, 77, 79, 80, 82, 84, 86, 87, 89, 91, 93, + 94, 96, 98, 100, 101, 103, 104, 106, 106, 106, 106, 61, 60, 60, 60, 59, + 62, 66, 69, 73, 75, 77, 79, 81, 83, 85, 86, 88, 90, 91, 93, 95, 96, 98, + 99, 101, 102, 104, 105, 107, 107, 107, 107, 64, 63, 62, 62, 61, 64, 67, + 71, 75, 77, 79, 82, 84, 86, 87, 89, 91, 92, 94, 95, 97, 98, 100, 101, + 103, 104, 105, 106, 108, 108, 108, 108, 67, 66, 65, 64, 63, 66, 69, 73, + 77, 79, 81, 84, 87, 88, 90, 92, 94, 95, 96, 98, 99, 100, 102, 103, 104, + 105, 106, 108, 109, 109, 109, 109, 70, 68, 67, 66, 65, 68, 71, 74, 78, + 80, 83, 86, 88, 90, 92, 93, 95, 97, 98, 99, 101, 102, 103, 104, 106, + 107, 108, 109, 110, 110, 110, 110, 72, 71, 70, 69, 67, 70, 73, 76, 80, + 82, 85, 87, 90, 92, 93, 95, 97, 98, 100, 101, 102, 103, 104, 106, 107, + 108, 109, 110, 111, 111, 111, 111, 76, 74, 73, 71, 70, 73, 75, 78, 81, + 84, 86, 89, 92, 93, 95, 97, 99, 100, 101, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 112, 112, 112, 79, 78, 76, 74, 73, 75, 78, 80, 83, + 86, 88, 91, 94, 95, 97, 99, 101, 102, 103, 104, 106, 107, 108, 109, 110, + 110, 111, 112, 113, 113, 113, 113, 82, 80, 78, 77, 75, 77, 80, 82, 85, + 87, 90, 92, 95, 97, 98, 100, 102, 103, 104, 106, 107, 108, 109, 110, + 111, 111, 112, 113, 114, 114, 114, 114, 84, 83, 81, 79, 78, 80, 82, 84, + 87, 89, 91, 94, 96, 98, 100, 101, 103, 104, 105, 107, 108, 109, 110, + 111, 112, 112, 113, 114, 115, 115, 115, 115, 87, 86, 84, 82, 81, 82, 84, + 87, 89, 91, 93, 95, 98, 99, 101, 103, 104, 106, 107, 108, 109, 110, 111, + 112, 113, 113, 114, 115, 116, 116, 116, 116, 91, 89, 87, 85, 84, 85, 87, + 89, 91, 93, 95, 97, 99, 101, 102, 104, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 114, 115, 116, 116, 116, 116, 116, 93, 91, 89, 87, 86, + 87, 89, 91, 93, 94, 96, 98, 100, 102, 103, 105, 107, 108, 109, 110, 111, + 112, 113, 114, 114, 115, 116, 116, 117, 117, 117, 117, 95, 93, 91, 90, + 88, 90, 91, 93, 94, 96, 98, 100, 102, 103, 104, 106, 108, 109, 110, 111, + 112, 113, 114, 114, 115, 116, 116, 117, 118, 118, 118, 118, 97, 96, 94, + 92, 91, 92, 93, 95, 96, 98, 99, 101, 103, 104, 106, 107, 109, 110, 111, + 112, 113, 114, 114, 115, 116, 117, 117, 118, 118, 118, 118, 118, 100, + 98, 96, 95, 93, 94, 96, 97, 98, 100, 101, 103, 104, 106, 107, 108, 110, + 111, 112, 113, 114, 114, 115, 116, 117, 117, 118, 118, 119, 119, 119, + 119, 102, 100, 98, 97, 95, 96, 97, 99, 100, 101, 102, 104, 105, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 117, 118, 118, 119, + 119, 119, 119, 119, 103, 102, 100, 99, 97, 98, 99, 100, 101, 103, 104, + 105, 106, 108, 109, 110, 111, 112, 113, 114, 115, 116, 116, 117, 118, + 118, 119, 119, 120, 120, 120, 120, 105, 104, 102, 101, 99, 100, 101, + 102, 103, 104, 105, 106, 108, 109, 110, 111, 112, 113, 114, 115, 116, + 116, 117, 118, 118, 119, 119, 120, 120, 120, 120, 120, 107, 106, 104, + 103, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 116, 117, 118, 118, 119, 119, 120, 120, 121, 121, 121, + 121, 107, 106, 104, 103, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 116, 117, 118, 118, 119, 119, 120, + 120, 121, 121, 121, 121, 107, 106, 104, 103, 101, 102, 103, 104, 105, + 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 116, 117, 118, + 118, 119, 119, 120, 120, 121, 121, 121, 121, 107, 106, 104, 103, 101, + 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, + 116, 116, 117, 118, 118, 119, 119, 120, 120, 121, 121, 121, 121 }, + { /* Intra matrices */ + /* Size 4 */ + 38, 40, 55, 69, 40, 52, 62, 72, 55, 62, 73, 79, 69, 72, 79, 83, + /* Size 8 */ + 43, 35, 37, 45, 53, 62, 68, 74, 35, 39, 37, 42, 49, 56, 64, 70, 37, 37, + 47, 52, 56, 62, 67, 72, 45, 42, 52, 59, 64, 68, 72, 75, 53, 49, 56, 64, + 69, 73, 76, 78, 62, 56, 62, 68, 73, 76, 79, 81, 68, 64, 67, 72, 76, 79, + 81, 83, 74, 70, 72, 75, 78, 81, 83, 84, + /* Size 16 */ + 42, 38, 34, 35, 36, 40, 44, 48, 52, 56, 60, 63, 67, 70, 72, 72, 38, 37, + 36, 36, 36, 39, 42, 46, 50, 54, 58, 61, 65, 67, 70, 70, 34, 36, 38, 37, + 37, 39, 41, 44, 48, 51, 55, 59, 62, 65, 68, 68, 35, 36, 37, 39, 41, 43, + 45, 48, 51, 54, 58, 61, 64, 67, 69, 69, 36, 36, 37, 41, 46, 48, 50, 53, + 55, 58, 61, 63, 66, 68, 70, 70, 40, 39, 39, 43, 48, 51, 54, 56, 59, 61, + 63, 66, 68, 70, 72, 72, 44, 42, 41, 45, 50, 54, 58, 60, 62, 64, 66, 68, + 70, 72, 74, 74, 48, 46, 44, 48, 53, 56, 60, 62, 65, 67, 69, 70, 72, 74, + 75, 75, 52, 50, 48, 51, 55, 59, 62, 65, 68, 69, 71, 73, 74, 75, 77, 77, + 56, 54, 51, 54, 58, 61, 64, 67, 69, 71, 73, 74, 76, 77, 78, 78, 60, 58, + 55, 58, 61, 63, 66, 69, 71, 73, 75, 76, 77, 78, 79, 79, 63, 61, 59, 61, + 63, 66, 68, 70, 73, 74, 76, 77, 78, 79, 80, 80, 67, 65, 62, 64, 66, 68, + 70, 72, 74, 76, 77, 78, 79, 80, 81, 81, 70, 67, 65, 67, 68, 70, 72, 74, + 75, 77, 78, 79, 80, 81, 82, 82, 72, 70, 68, 69, 70, 72, 74, 75, 77, 78, + 79, 80, 81, 82, 82, 82, 72, 70, 68, 69, 70, 72, 74, 75, 77, 78, 79, 80, + 81, 82, 82, 82, + /* Size 32 */ + 41, 39, 37, 35, 34, 34, 35, 35, 36, 37, 39, 41, 43, 45, 47, 49, 52, 54, + 55, 57, 60, 61, 63, 65, 66, 68, 69, 70, 72, 72, 72, 72, 39, 38, 37, 36, + 35, 35, 35, 36, 36, 37, 39, 41, 43, 44, 46, 48, 51, 52, 54, 56, 58, 60, + 62, 63, 65, 66, 68, 69, 70, 70, 70, 70, 37, 37, 36, 36, 36, 36, 36, 36, + 36, 37, 39, 40, 42, 43, 45, 47, 49, 51, 53, 55, 57, 59, 60, 62, 64, 65, + 67, 68, 69, 69, 69, 69, 35, 36, 36, 36, 36, 36, 36, 36, 36, 37, 38, 40, + 41, 43, 44, 46, 48, 50, 52, 54, 56, 57, 59, 61, 63, 64, 65, 67, 68, 68, + 68, 68, 34, 35, 36, 36, 37, 37, 37, 37, 36, 37, 38, 39, 40, 42, 44, 45, + 47, 49, 51, 53, 55, 56, 58, 60, 62, 63, 64, 66, 67, 67, 67, 67, 34, 35, + 36, 36, 37, 37, 38, 38, 38, 39, 40, 41, 42, 44, 45, 47, 49, 51, 52, 54, + 56, 57, 59, 61, 62, 64, 65, 66, 68, 68, 68, 68, 35, 35, 36, 36, 37, 38, + 38, 39, 40, 41, 42, 44, 45, 46, 48, 49, 51, 52, 54, 55, 57, 59, 60, 62, + 63, 65, 66, 67, 69, 69, 69, 69, 35, 36, 36, 36, 37, 38, 39, 41, 43, 44, + 45, 46, 47, 48, 50, 51, 53, 54, 55, 57, 59, 60, 61, 63, 64, 65, 67, 68, + 69, 69, 69, 69, 36, 36, 36, 36, 36, 38, 40, 43, 45, 46, 48, 49, 50, 51, + 52, 53, 55, 56, 57, 58, 60, 61, 62, 64, 65, 66, 67, 69, 70, 70, 70, 70, + 37, 37, 37, 37, 37, 39, 41, 44, 46, 48, 49, 50, 52, 53, 54, 55, 56, 57, + 59, 60, 61, 62, 64, 65, 66, 67, 68, 69, 71, 71, 71, 71, 39, 39, 39, 38, + 38, 40, 42, 45, 48, 49, 50, 52, 53, 54, 56, 57, 58, 59, 60, 61, 63, 64, + 65, 66, 67, 68, 69, 70, 71, 71, 71, 71, 41, 41, 40, 40, 39, 41, 44, 46, + 49, 50, 52, 53, 55, 56, 57, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 72, 72, 72, 43, 43, 42, 41, 40, 42, 45, 47, 50, 52, 53, 55, + 57, 58, 59, 61, 62, 63, 64, 65, 66, 67, 68, 68, 69, 70, 71, 72, 73, 73, + 73, 73, 45, 44, 43, 43, 42, 44, 46, 48, 51, 53, 54, 56, 58, 59, 61, 62, + 63, 64, 65, 66, 67, 68, 69, 69, 70, 71, 72, 73, 74, 74, 74, 74, 47, 46, + 45, 44, 44, 45, 48, 50, 52, 54, 56, 57, 59, 61, 62, 63, 64, 65, 66, 67, + 68, 69, 70, 71, 71, 72, 73, 74, 74, 74, 74, 74, 49, 48, 47, 46, 45, 47, + 49, 51, 53, 55, 57, 59, 61, 62, 63, 64, 66, 66, 67, 68, 69, 70, 71, 72, + 72, 73, 74, 74, 75, 75, 75, 75, 52, 51, 49, 48, 47, 49, 51, 53, 55, 56, + 58, 60, 62, 63, 64, 66, 67, 68, 69, 70, 71, 71, 72, 73, 73, 74, 75, 75, + 76, 76, 76, 76, 54, 52, 51, 50, 49, 51, 52, 54, 56, 57, 59, 61, 63, 64, + 65, 66, 68, 69, 70, 70, 71, 72, 73, 73, 74, 75, 75, 76, 76, 76, 76, 76, + 55, 54, 53, 52, 51, 52, 54, 55, 57, 59, 60, 62, 64, 65, 66, 67, 69, 70, + 70, 71, 72, 73, 73, 74, 75, 75, 76, 77, 77, 77, 77, 77, 57, 56, 55, 54, + 53, 54, 55, 57, 58, 60, 61, 63, 65, 66, 67, 68, 70, 70, 71, 72, 73, 74, + 74, 75, 76, 76, 77, 77, 78, 78, 78, 78, 60, 58, 57, 56, 55, 56, 57, 59, + 60, 61, 63, 64, 66, 67, 68, 69, 71, 71, 72, 73, 74, 75, 75, 76, 76, 77, + 77, 78, 78, 78, 78, 78, 61, 60, 59, 57, 56, 57, 59, 60, 61, 62, 64, 65, + 67, 68, 69, 70, 71, 72, 73, 74, 75, 75, 76, 76, 77, 77, 78, 78, 79, 79, + 79, 79, 63, 62, 60, 59, 58, 59, 60, 61, 62, 64, 65, 66, 68, 69, 70, 71, + 72, 73, 73, 74, 75, 76, 76, 77, 77, 78, 78, 79, 79, 79, 79, 79, 65, 63, + 62, 61, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 71, 72, 73, 73, 74, 75, + 76, 76, 77, 77, 78, 78, 79, 79, 80, 80, 80, 80, 66, 65, 64, 63, 62, 62, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 76, 77, 77, 78, + 79, 79, 79, 80, 80, 80, 80, 80, 68, 66, 65, 64, 63, 64, 65, 65, 66, 67, + 68, 69, 70, 71, 72, 73, 74, 75, 75, 76, 77, 77, 78, 78, 79, 79, 80, 80, + 81, 81, 81, 81, 69, 68, 67, 65, 64, 65, 66, 67, 67, 68, 69, 70, 71, 72, + 73, 74, 75, 75, 76, 77, 77, 78, 78, 79, 79, 80, 80, 81, 81, 81, 81, 81, + 70, 69, 68, 67, 66, 66, 67, 68, 69, 69, 70, 71, 72, 73, 74, 74, 75, 76, + 77, 77, 78, 78, 79, 79, 80, 80, 81, 81, 81, 81, 81, 81, 72, 70, 69, 68, + 67, 68, 69, 69, 70, 71, 71, 72, 73, 74, 74, 75, 76, 76, 77, 78, 78, 79, + 79, 80, 80, 81, 81, 81, 82, 82, 82, 82, 72, 70, 69, 68, 67, 68, 69, 69, + 70, 71, 71, 72, 73, 74, 74, 75, 76, 76, 77, 78, 78, 79, 79, 80, 80, 81, + 81, 81, 82, 82, 82, 82, 72, 70, 69, 68, 67, 68, 69, 69, 70, 71, 71, 72, + 73, 74, 74, 75, 76, 76, 77, 78, 78, 79, 79, 80, 80, 81, 81, 81, 82, 82, + 82, 82, 72, 70, 69, 68, 67, 68, 69, 69, 70, 71, 71, 72, 73, 74, 74, 75, + 76, 76, 77, 78, 78, 79, 79, 80, 80, 81, 81, 81, 82, 82, 82, 82 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 86, 90, 101, 86, 95, 98, 105, 90, 98, 110, 117, 101, 105, 117, 125, + /* Size 8 */ + 64, 55, 81, 84, 87, 92, 98, 104, 55, 71, 82, 77, 79, 83, 89, 95, 81, 82, + 89, 88, 88, 90, 94, 99, 84, 77, 88, 94, 96, 98, 101, 104, 87, 79, 88, + 96, 101, 104, 107, 110, 92, 83, 90, 98, 104, 109, 112, 115, 98, 89, 94, + 101, 107, 112, 116, 118, 104, 95, 99, 104, 110, 115, 118, 121, + /* Size 16 */ + 64, 59, 55, 66, 81, 82, 84, 85, 87, 90, 92, 95, 98, 101, 104, 104, 59, + 61, 62, 70, 82, 81, 80, 82, 83, 85, 87, 90, 93, 96, 99, 99, 55, 62, 71, + 76, 82, 80, 77, 78, 79, 81, 83, 86, 89, 92, 95, 95, 66, 70, 76, 80, 86, + 84, 82, 83, 83, 85, 86, 89, 91, 94, 97, 97, 81, 82, 82, 86, 89, 89, 88, + 88, 88, 89, 90, 92, 94, 96, 99, 99, 82, 81, 80, 84, 89, 90, 91, 91, 92, + 93, 94, 96, 97, 99, 101, 101, 84, 80, 77, 82, 88, 91, 94, 95, 96, 97, + 98, 99, 101, 102, 104, 104, 85, 82, 78, 83, 88, 91, 95, 97, 98, 100, + 101, 102, 104, 105, 107, 107, 87, 83, 79, 83, 88, 92, 96, 98, 101, 103, + 104, 106, 107, 108, 110, 110, 90, 85, 81, 85, 89, 93, 97, 100, 103, 105, + 107, 108, 109, 111, 112, 112, 92, 87, 83, 86, 90, 94, 98, 101, 104, 107, + 109, 110, 112, 113, 115, 115, 95, 90, 86, 89, 92, 96, 99, 102, 106, 108, + 110, 112, 114, 115, 116, 116, 98, 93, 89, 91, 94, 97, 101, 104, 107, + 109, 112, 114, 116, 117, 118, 118, 101, 96, 92, 94, 96, 99, 102, 105, + 108, 111, 113, 115, 117, 118, 120, 120, 104, 99, 95, 97, 99, 101, 104, + 107, 110, 112, 115, 116, 118, 120, 121, 121, 104, 99, 95, 97, 99, 101, + 104, 107, 110, 112, 115, 116, 118, 120, 121, 121, + /* Size 32 */ + 64, 62, 59, 57, 55, 60, 66, 73, 81, 82, 82, 83, 84, 85, 85, 86, 87, 88, + 90, 91, 92, 94, 95, 97, 98, 100, 101, 102, 104, 104, 104, 104, 62, 61, + 60, 59, 58, 63, 68, 74, 81, 82, 82, 82, 82, 83, 83, 84, 85, 86, 87, 88, + 90, 91, 93, 94, 96, 97, 98, 100, 101, 101, 101, 101, 59, 60, 61, 61, 62, + 66, 70, 76, 82, 81, 81, 81, 80, 81, 82, 82, 83, 84, 85, 86, 87, 89, 90, + 92, 93, 95, 96, 98, 99, 99, 99, 99, 57, 59, 61, 64, 66, 69, 73, 77, 82, + 81, 80, 80, 79, 79, 80, 80, 81, 82, 83, 84, 85, 87, 88, 89, 91, 92, 94, + 95, 97, 97, 97, 97, 55, 58, 62, 66, 71, 73, 76, 79, 82, 81, 80, 78, 77, + 78, 78, 78, 79, 80, 81, 82, 83, 84, 86, 87, 89, 90, 92, 93, 95, 95, 95, + 95, 60, 63, 66, 69, 73, 76, 78, 81, 84, 83, 82, 81, 80, 80, 80, 81, 81, + 82, 83, 84, 85, 86, 87, 89, 90, 91, 93, 94, 96, 96, 96, 96, 66, 68, 70, + 73, 76, 78, 80, 83, 86, 85, 84, 83, 82, 82, 83, 83, 83, 84, 85, 86, 86, + 88, 89, 90, 91, 93, 94, 95, 97, 97, 97, 97, 73, 74, 76, 77, 79, 81, 83, + 85, 87, 87, 86, 86, 85, 85, 85, 85, 85, 86, 87, 87, 88, 89, 90, 92, 93, + 94, 95, 96, 98, 98, 98, 98, 81, 81, 82, 82, 82, 84, 86, 87, 89, 89, 89, + 88, 88, 88, 88, 88, 88, 88, 89, 89, 90, 91, 92, 93, 94, 95, 96, 98, 99, + 99, 99, 99, 82, 82, 81, 81, 81, 83, 85, 87, 89, 89, 89, 89, 89, 89, 89, + 89, 90, 90, 91, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 100, 100, 100, + 82, 82, 81, 80, 80, 82, 84, 86, 89, 89, 90, 90, 91, 91, 91, 91, 92, 92, + 93, 93, 94, 95, 96, 96, 97, 98, 99, 100, 101, 101, 101, 101, 83, 82, 81, + 80, 78, 81, 83, 86, 88, 89, 90, 91, 92, 92, 93, 93, 94, 94, 95, 95, 96, + 97, 97, 98, 99, 100, 101, 102, 103, 103, 103, 103, 84, 82, 80, 79, 77, + 80, 82, 85, 88, 89, 91, 92, 94, 94, 95, 95, 96, 96, 97, 97, 98, 99, 99, + 100, 101, 102, 102, 103, 104, 104, 104, 104, 85, 83, 81, 79, 78, 80, 82, + 85, 88, 89, 91, 92, 94, 95, 96, 96, 97, 98, 98, 99, 99, 100, 101, 102, + 102, 103, 104, 105, 106, 106, 106, 106, 85, 83, 82, 80, 78, 80, 83, 85, + 88, 89, 91, 93, 95, 96, 97, 98, 98, 99, 100, 100, 101, 102, 102, 103, + 104, 105, 105, 106, 107, 107, 107, 107, 86, 84, 82, 80, 78, 81, 83, 85, + 88, 89, 91, 93, 95, 96, 98, 99, 100, 100, 101, 102, 103, 103, 104, 105, + 105, 106, 107, 108, 108, 108, 108, 108, 87, 85, 83, 81, 79, 81, 83, 85, + 88, 90, 92, 94, 96, 97, 98, 100, 101, 102, 103, 104, 104, 105, 106, 106, + 107, 108, 108, 109, 110, 110, 110, 110, 88, 86, 84, 82, 80, 82, 84, 86, + 88, 90, 92, 94, 96, 98, 99, 100, 102, 103, 104, 105, 105, 106, 107, 108, + 108, 109, 110, 110, 111, 111, 111, 111, 90, 87, 85, 83, 81, 83, 85, 87, + 89, 91, 93, 95, 97, 98, 100, 101, 103, 104, 105, 106, 107, 107, 108, + 109, 109, 110, 111, 111, 112, 112, 112, 112, 91, 88, 86, 84, 82, 84, 86, + 87, 89, 91, 93, 95, 97, 99, 100, 102, 104, 105, 106, 107, 108, 108, 109, + 110, 111, 111, 112, 113, 113, 113, 113, 113, 92, 90, 87, 85, 83, 85, 86, + 88, 90, 92, 94, 96, 98, 99, 101, 103, 104, 105, 107, 108, 109, 110, 110, + 111, 112, 113, 113, 114, 115, 115, 115, 115, 94, 91, 89, 87, 84, 86, 88, + 89, 91, 93, 95, 97, 99, 100, 102, 103, 105, 106, 107, 108, 110, 110, + 111, 112, 113, 114, 114, 115, 116, 116, 116, 116, 95, 93, 90, 88, 86, + 87, 89, 90, 92, 94, 96, 97, 99, 101, 102, 104, 106, 107, 108, 109, 110, + 111, 112, 113, 114, 114, 115, 116, 116, 116, 116, 116, 97, 94, 92, 89, + 87, 89, 90, 92, 93, 95, 96, 98, 100, 102, 103, 105, 106, 108, 109, 110, + 111, 112, 113, 114, 115, 115, 116, 117, 117, 117, 117, 117, 98, 96, 93, + 91, 89, 90, 91, 93, 94, 96, 97, 99, 101, 102, 104, 105, 107, 108, 109, + 111, 112, 113, 114, 115, 116, 116, 117, 118, 118, 118, 118, 118, 100, + 97, 95, 92, 90, 91, 93, 94, 95, 97, 98, 100, 102, 103, 105, 106, 108, + 109, 110, 111, 113, 114, 114, 115, 116, 117, 118, 118, 119, 119, 119, + 119, 101, 98, 96, 94, 92, 93, 94, 95, 96, 98, 99, 101, 102, 104, 105, + 107, 108, 110, 111, 112, 113, 114, 115, 116, 117, 118, 118, 119, 120, + 120, 120, 120, 102, 100, 98, 95, 93, 94, 95, 96, 98, 99, 100, 102, 103, + 105, 106, 108, 109, 110, 111, 113, 114, 115, 116, 117, 118, 118, 119, + 120, 121, 121, 121, 121, 104, 101, 99, 97, 95, 96, 97, 98, 99, 100, 101, + 103, 104, 106, 107, 108, 110, 111, 112, 113, 115, 116, 116, 117, 118, + 119, 120, 121, 121, 121, 121, 121, 104, 101, 99, 97, 95, 96, 97, 98, 99, + 100, 101, 103, 104, 106, 107, 108, 110, 111, 112, 113, 115, 116, 116, + 117, 118, 119, 120, 121, 121, 121, 121, 121, 104, 101, 99, 97, 95, 96, + 97, 98, 99, 100, 101, 103, 104, 106, 107, 108, 110, 111, 112, 113, 115, + 116, 116, 117, 118, 119, 120, 121, 121, 121, 121, 121, 104, 101, 99, 97, + 95, 96, 97, 98, 99, 100, 101, 103, 104, 106, 107, 108, 110, 111, 112, + 113, 115, 116, 116, 117, 118, 119, 120, 121, 121, 121, 121, 121 }, + { /* Intra matrices */ + /* Size 4 */ + 40, 54, 57, 65, 54, 60, 62, 67, 57, 62, 71, 75, 65, 67, 75, 81, + /* Size 8 */ + 42, 36, 54, 56, 58, 62, 66, 70, 36, 47, 54, 51, 52, 55, 59, 64, 54, 54, + 60, 59, 58, 60, 63, 66, 56, 51, 59, 63, 64, 66, 68, 70, 58, 52, 58, 64, + 68, 70, 72, 74, 62, 55, 60, 66, 70, 74, 76, 78, 66, 59, 63, 68, 72, 76, + 79, 81, 70, 64, 66, 70, 74, 78, 81, 83, + /* Size 16 */ + 41, 38, 35, 43, 53, 54, 55, 56, 57, 59, 61, 63, 65, 67, 69, 69, 38, 39, + 40, 46, 53, 53, 53, 53, 54, 56, 57, 59, 62, 64, 66, 66, 35, 40, 46, 49, + 54, 52, 51, 51, 51, 53, 54, 56, 58, 60, 63, 63, 43, 46, 49, 53, 56, 55, + 54, 54, 54, 56, 57, 58, 60, 62, 64, 64, 53, 53, 54, 56, 59, 58, 58, 58, + 58, 58, 59, 61, 62, 64, 65, 65, 54, 53, 52, 55, 58, 59, 60, 60, 60, 61, + 62, 63, 64, 66, 67, 67, 55, 53, 51, 54, 58, 60, 62, 63, 63, 64, 65, 66, + 67, 68, 69, 69, 56, 53, 51, 54, 58, 60, 63, 64, 65, 66, 67, 68, 69, 70, + 71, 71, 57, 54, 51, 54, 58, 60, 63, 65, 67, 68, 69, 70, 71, 72, 73, 73, + 59, 56, 53, 56, 58, 61, 64, 66, 68, 70, 71, 72, 73, 74, 75, 75, 61, 57, + 54, 57, 59, 62, 65, 67, 69, 71, 73, 74, 75, 76, 77, 77, 63, 59, 56, 58, + 61, 63, 66, 68, 70, 72, 74, 75, 76, 77, 78, 78, 65, 62, 58, 60, 62, 64, + 67, 69, 71, 73, 75, 76, 78, 79, 80, 80, 67, 64, 60, 62, 64, 66, 68, 70, + 72, 74, 76, 77, 79, 80, 81, 81, 69, 66, 63, 64, 65, 67, 69, 71, 73, 75, + 77, 78, 80, 81, 82, 82, 69, 66, 63, 64, 65, 67, 69, 71, 73, 75, 77, 78, + 80, 81, 82, 82, + /* Size 32 */ + 41, 39, 38, 37, 35, 38, 42, 47, 53, 53, 54, 54, 55, 55, 56, 56, 57, 58, + 59, 59, 60, 61, 62, 63, 65, 65, 66, 68, 69, 69, 69, 69, 39, 39, 38, 38, + 37, 40, 44, 48, 53, 53, 53, 53, 53, 54, 54, 55, 55, 56, 57, 58, 59, 60, + 61, 62, 63, 64, 65, 66, 67, 67, 67, 67, 38, 38, 39, 39, 40, 42, 45, 49, + 53, 53, 53, 52, 52, 53, 53, 53, 54, 55, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 64, 65, 65, 65, 65, 37, 38, 39, 41, 42, 45, 47, 50, 53, 53, 52, 52, + 51, 51, 52, 52, 52, 53, 54, 55, 56, 56, 57, 58, 59, 60, 61, 63, 64, 64, + 64, 64, 35, 37, 40, 42, 46, 47, 49, 51, 53, 52, 52, 51, 50, 50, 51, 51, + 51, 52, 53, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 62, 62, 62, 38, 40, + 42, 45, 47, 49, 51, 52, 54, 54, 53, 52, 52, 52, 52, 52, 52, 53, 54, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 63, 63, 63, 42, 44, 45, 47, 49, 51, + 52, 54, 56, 55, 55, 54, 53, 54, 54, 54, 54, 55, 55, 56, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 64, 64, 64, 47, 48, 49, 50, 51, 52, 54, 55, 57, 57, + 56, 56, 55, 55, 55, 55, 55, 56, 57, 57, 58, 58, 59, 60, 61, 62, 62, 63, + 64, 64, 64, 64, 53, 53, 53, 53, 53, 54, 56, 57, 58, 58, 58, 58, 57, 57, + 57, 57, 57, 58, 58, 58, 59, 60, 60, 61, 62, 62, 63, 64, 65, 65, 65, 65, + 53, 53, 53, 53, 52, 54, 55, 57, 58, 58, 58, 58, 58, 58, 58, 58, 59, 59, + 59, 60, 60, 61, 61, 62, 63, 64, 64, 65, 66, 66, 66, 66, 54, 53, 53, 52, + 52, 53, 55, 56, 58, 58, 59, 59, 59, 59, 60, 60, 60, 60, 61, 61, 61, 62, + 63, 63, 64, 65, 65, 66, 67, 67, 67, 67, 54, 53, 52, 52, 51, 52, 54, 56, + 58, 58, 59, 60, 60, 61, 61, 61, 61, 62, 62, 63, 63, 63, 64, 65, 65, 66, + 66, 67, 68, 68, 68, 68, 55, 53, 52, 51, 50, 52, 53, 55, 57, 58, 59, 60, + 61, 62, 62, 63, 63, 63, 64, 64, 64, 65, 65, 66, 66, 67, 68, 68, 69, 69, + 69, 69, 55, 54, 53, 51, 50, 52, 54, 55, 57, 58, 59, 61, 62, 62, 63, 63, + 64, 64, 65, 65, 65, 66, 66, 67, 67, 68, 69, 69, 70, 70, 70, 70, 56, 54, + 53, 52, 51, 52, 54, 55, 57, 58, 60, 61, 62, 63, 63, 64, 65, 65, 66, 66, + 67, 67, 68, 68, 69, 69, 70, 70, 71, 71, 71, 71, 56, 55, 53, 52, 51, 52, + 54, 55, 57, 58, 60, 61, 63, 63, 64, 65, 66, 66, 67, 67, 68, 68, 69, 69, + 70, 70, 71, 71, 72, 72, 72, 72, 57, 55, 54, 52, 51, 52, 54, 55, 57, 59, + 60, 61, 63, 64, 65, 66, 67, 67, 68, 68, 69, 69, 70, 70, 71, 71, 72, 72, + 73, 73, 73, 73, 58, 56, 55, 53, 52, 53, 55, 56, 58, 59, 60, 62, 63, 64, + 65, 66, 67, 68, 68, 69, 70, 70, 71, 71, 72, 72, 73, 73, 74, 74, 74, 74, + 59, 57, 55, 54, 53, 54, 55, 57, 58, 59, 61, 62, 64, 65, 66, 67, 68, 68, + 69, 70, 70, 71, 71, 72, 73, 73, 73, 74, 74, 74, 74, 74, 59, 58, 56, 55, + 53, 54, 56, 57, 58, 60, 61, 63, 64, 65, 66, 67, 68, 69, 70, 71, 71, 72, + 72, 73, 73, 74, 74, 75, 75, 75, 75, 75, 60, 59, 57, 56, 54, 55, 56, 58, + 59, 60, 61, 63, 64, 65, 67, 68, 69, 70, 70, 71, 72, 73, 73, 74, 74, 75, + 75, 76, 76, 76, 76, 76, 61, 60, 58, 56, 55, 56, 57, 58, 60, 61, 62, 63, + 65, 66, 67, 68, 69, 70, 71, 72, 73, 73, 74, 74, 75, 75, 76, 76, 77, 77, + 77, 77, 62, 61, 59, 57, 56, 57, 58, 59, 60, 61, 63, 64, 65, 66, 68, 69, + 70, 71, 71, 72, 73, 74, 74, 75, 76, 76, 77, 77, 78, 78, 78, 78, 63, 62, + 60, 58, 57, 58, 59, 60, 61, 62, 63, 65, 66, 67, 68, 69, 70, 71, 72, 73, + 74, 74, 75, 76, 76, 77, 77, 78, 78, 78, 78, 78, 65, 63, 61, 59, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 69, 70, 71, 72, 73, 73, 74, 75, 76, 76, + 77, 77, 78, 78, 79, 79, 79, 79, 65, 64, 62, 60, 59, 60, 61, 62, 62, 64, + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 75, 76, 77, 77, 78, 78, 79, + 79, 79, 79, 79, 66, 65, 63, 61, 60, 61, 62, 62, 63, 64, 65, 66, 68, 69, + 70, 71, 72, 73, 73, 74, 75, 76, 77, 77, 78, 78, 79, 79, 80, 80, 80, 80, + 68, 66, 64, 63, 61, 62, 63, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, + 74, 75, 76, 76, 77, 78, 78, 79, 79, 80, 81, 81, 81, 81, 69, 67, 65, 64, + 62, 63, 64, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 74, 75, 76, 77, + 78, 78, 79, 79, 80, 81, 81, 81, 81, 81, 69, 67, 65, 64, 62, 63, 64, 64, + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 74, 75, 76, 77, 78, 78, 79, 79, + 80, 81, 81, 81, 81, 81, 69, 67, 65, 64, 62, 63, 64, 64, 65, 66, 67, 68, + 69, 70, 71, 72, 73, 74, 74, 75, 76, 77, 78, 78, 79, 79, 80, 81, 81, 81, + 81, 81, 69, 67, 65, 64, 62, 63, 64, 64, 65, 66, 67, 68, 69, 70, 71, 72, + 73, 74, 74, 75, 76, 77, 78, 78, 79, 79, 80, 81, 81, 81, 81, 81 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 67, 88, 105, 67, 84, 97, 108, 88, 97, 109, 116, 105, 108, 116, 120, + /* Size 8 */ + 64, 54, 57, 66, 77, 86, 94, 99, 54, 59, 57, 63, 72, 81, 88, 95, 57, 57, + 69, 75, 80, 87, 92, 97, 66, 63, 75, 83, 89, 93, 97, 101, 77, 72, 80, 89, + 94, 98, 101, 104, 86, 81, 87, 93, 98, 102, 104, 106, 94, 88, 92, 97, + 101, 104, 106, 108, 99, 95, 97, 101, 104, 106, 108, 109, + /* Size 16 */ + 64, 59, 54, 55, 57, 61, 66, 71, 77, 81, 86, 90, 94, 96, 99, 99, 59, 57, + 56, 57, 57, 61, 65, 69, 74, 79, 83, 87, 91, 94, 97, 97, 54, 56, 59, 58, + 57, 60, 63, 67, 72, 76, 81, 84, 88, 91, 95, 95, 55, 57, 58, 60, 63, 65, + 68, 72, 76, 79, 83, 87, 90, 93, 96, 96, 57, 57, 57, 63, 69, 72, 75, 78, + 80, 83, 87, 89, 92, 95, 97, 97, 61, 61, 60, 65, 72, 75, 79, 82, 84, 87, + 90, 92, 95, 97, 99, 99, 66, 65, 63, 68, 75, 79, 83, 86, 89, 91, 93, 95, + 97, 99, 101, 101, 71, 69, 67, 72, 78, 82, 86, 89, 91, 93, 95, 97, 99, + 100, 102, 102, 77, 74, 72, 76, 80, 84, 89, 91, 94, 96, 98, 100, 101, + 102, 104, 104, 81, 79, 76, 79, 83, 87, 91, 93, 96, 98, 100, 101, 103, + 104, 105, 105, 86, 83, 81, 83, 87, 90, 93, 95, 98, 100, 102, 103, 104, + 105, 106, 106, 90, 87, 84, 87, 89, 92, 95, 97, 100, 101, 103, 104, 105, + 106, 107, 107, 94, 91, 88, 90, 92, 95, 97, 99, 101, 103, 104, 105, 106, + 107, 108, 108, 96, 94, 91, 93, 95, 97, 99, 100, 102, 104, 105, 106, 107, + 108, 109, 109, 99, 97, 95, 96, 97, 99, 101, 102, 104, 105, 106, 107, + 108, 109, 109, 109, 99, 97, 95, 96, 97, 99, 101, 102, 104, 105, 106, + 107, 108, 109, 109, 109, + /* Size 32 */ + 64, 61, 59, 56, 54, 55, 55, 56, 57, 59, 61, 64, 66, 69, 71, 74, 77, 79, + 81, 84, 86, 88, 90, 92, 94, 95, 96, 98, 99, 99, 99, 99, 61, 60, 58, 57, + 55, 56, 56, 56, 57, 59, 61, 63, 66, 68, 70, 73, 76, 78, 80, 82, 85, 87, + 88, 90, 92, 94, 95, 97, 98, 98, 98, 98, 59, 58, 57, 57, 56, 56, 57, 57, + 57, 59, 61, 63, 65, 67, 69, 72, 74, 76, 79, 81, 83, 85, 87, 89, 91, 92, + 94, 95, 97, 97, 97, 97, 56, 57, 57, 57, 58, 57, 57, 57, 57, 59, 60, 62, + 64, 66, 68, 70, 73, 75, 77, 79, 82, 84, 86, 88, 90, 91, 93, 94, 96, 96, + 96, 96, 54, 55, 56, 58, 59, 58, 58, 58, 57, 59, 60, 61, 63, 65, 67, 69, + 72, 74, 76, 78, 81, 82, 84, 86, 88, 90, 91, 93, 95, 95, 95, 95, 55, 56, + 56, 57, 58, 59, 59, 59, 60, 61, 63, 64, 65, 67, 69, 71, 74, 76, 78, 80, + 82, 84, 86, 87, 89, 91, 92, 94, 95, 95, 95, 95, 55, 56, 57, 57, 58, 59, + 60, 61, 63, 64, 65, 67, 68, 70, 72, 74, 76, 78, 79, 81, 83, 85, 87, 89, + 90, 92, 93, 95, 96, 96, 96, 96, 56, 56, 57, 57, 58, 59, 61, 64, 66, 67, + 68, 70, 71, 73, 75, 76, 78, 80, 81, 83, 85, 86, 88, 90, 91, 93, 94, 95, + 97, 97, 97, 97, 57, 57, 57, 57, 57, 60, 63, 66, 69, 71, 72, 73, 75, 76, + 78, 79, 80, 82, 83, 85, 87, 88, 89, 91, 92, 94, 95, 96, 97, 97, 97, 97, + 59, 59, 59, 59, 59, 61, 64, 67, 71, 72, 74, 75, 77, 78, 79, 81, 82, 84, + 85, 87, 88, 89, 91, 92, 93, 95, 96, 97, 98, 98, 98, 98, 61, 61, 61, 60, + 60, 63, 65, 68, 72, 74, 75, 77, 79, 80, 82, 83, 84, 86, 87, 88, 90, 91, + 92, 93, 95, 96, 97, 98, 99, 99, 99, 99, 64, 63, 63, 62, 61, 64, 67, 70, + 73, 75, 77, 79, 81, 82, 84, 85, 86, 88, 89, 90, 91, 92, 93, 95, 96, 97, + 98, 99, 100, 100, 100, 100, 66, 66, 65, 64, 63, 65, 68, 71, 75, 77, 79, + 81, 83, 85, 86, 87, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, + 101, 101, 101, 101, 69, 68, 67, 66, 65, 67, 70, 73, 76, 78, 80, 82, 85, + 86, 87, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 100, 101, 101, + 101, 101, 71, 70, 69, 68, 67, 69, 72, 75, 78, 79, 82, 84, 86, 87, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 100, 101, 102, 102, 102, + 102, 74, 73, 72, 70, 69, 71, 74, 76, 79, 81, 83, 85, 87, 89, 90, 91, 93, + 94, 95, 96, 97, 98, 98, 99, 100, 101, 101, 102, 103, 103, 103, 103, 77, + 76, 74, 73, 72, 74, 76, 78, 80, 82, 84, 86, 89, 90, 91, 93, 94, 95, 96, + 97, 98, 99, 100, 100, 101, 102, 102, 103, 104, 104, 104, 104, 79, 78, + 76, 75, 74, 76, 78, 80, 82, 84, 86, 88, 90, 91, 92, 94, 95, 96, 97, 98, + 99, 100, 100, 101, 102, 102, 103, 104, 104, 104, 104, 104, 81, 80, 79, + 77, 76, 78, 79, 81, 83, 85, 87, 89, 91, 92, 93, 95, 96, 97, 98, 99, 100, + 101, 101, 102, 103, 103, 104, 104, 105, 105, 105, 105, 84, 82, 81, 79, + 78, 80, 81, 83, 85, 87, 88, 90, 92, 93, 94, 96, 97, 98, 99, 100, 101, + 101, 102, 103, 103, 104, 104, 105, 105, 105, 105, 105, 86, 85, 83, 82, + 81, 82, 83, 85, 87, 88, 90, 91, 93, 94, 95, 97, 98, 99, 100, 101, 102, + 102, 103, 103, 104, 105, 105, 106, 106, 106, 106, 106, 88, 87, 85, 84, + 82, 84, 85, 86, 88, 89, 91, 92, 94, 95, 96, 98, 99, 100, 101, 101, 102, + 103, 103, 104, 105, 105, 106, 106, 107, 107, 107, 107, 90, 88, 87, 86, + 84, 86, 87, 88, 89, 91, 92, 93, 95, 96, 97, 98, 100, 100, 101, 102, 103, + 103, 104, 105, 105, 106, 106, 107, 107, 107, 107, 107, 92, 90, 89, 88, + 86, 87, 89, 90, 91, 92, 93, 95, 96, 97, 98, 99, 100, 101, 102, 103, 103, + 104, 105, 105, 106, 106, 107, 107, 107, 107, 107, 107, 94, 92, 91, 90, + 88, 89, 90, 91, 92, 93, 95, 96, 97, 98, 99, 100, 101, 102, 103, 103, + 104, 105, 105, 106, 106, 107, 107, 108, 108, 108, 108, 108, 95, 94, 92, + 91, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 102, 103, + 104, 105, 105, 106, 106, 107, 107, 107, 108, 108, 108, 108, 108, 96, 95, + 94, 93, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 100, 101, 102, 103, + 104, 104, 105, 106, 106, 107, 107, 107, 108, 108, 109, 109, 109, 109, + 98, 97, 95, 94, 93, 94, 95, 95, 96, 97, 98, 99, 100, 100, 101, 102, 103, + 104, 104, 105, 106, 106, 107, 107, 108, 108, 108, 109, 109, 109, 109, + 109, 99, 98, 97, 96, 95, 95, 96, 97, 97, 98, 99, 100, 101, 101, 102, + 103, 104, 104, 105, 105, 106, 107, 107, 107, 108, 108, 109, 109, 109, + 109, 109, 109, 99, 98, 97, 96, 95, 95, 96, 97, 97, 98, 99, 100, 101, + 101, 102, 103, 104, 104, 105, 105, 106, 107, 107, 107, 108, 108, 109, + 109, 109, 109, 109, 109, 99, 98, 97, 96, 95, 95, 96, 97, 97, 98, 99, + 100, 101, 101, 102, 103, 104, 104, 105, 105, 106, 107, 107, 107, 108, + 108, 109, 109, 109, 109, 109, 109, 99, 98, 97, 96, 95, 95, 96, 97, 97, + 98, 99, 100, 101, 101, 102, 103, 104, 104, 105, 105, 106, 107, 107, 107, + 108, 108, 109, 109, 109, 109, 109, 109 }, + { /* Intra matrices */ + /* Size 4 */ + 41, 43, 57, 69, 43, 54, 63, 71, 57, 63, 72, 77, 69, 71, 77, 80, + /* Size 8 */ + 46, 38, 40, 47, 55, 63, 68, 73, 38, 42, 41, 45, 51, 58, 64, 69, 40, 41, + 50, 54, 58, 63, 67, 71, 47, 45, 54, 60, 64, 68, 71, 74, 55, 51, 58, 64, + 69, 72, 74, 76, 63, 58, 63, 68, 72, 75, 77, 78, 68, 64, 67, 71, 74, 77, + 78, 80, 73, 69, 71, 74, 76, 78, 80, 81, + /* Size 16 */ + 45, 41, 38, 38, 39, 43, 47, 50, 54, 58, 61, 64, 67, 69, 71, 71, 41, 40, + 39, 39, 40, 42, 45, 49, 52, 56, 59, 62, 65, 67, 70, 70, 38, 39, 41, 40, + 40, 42, 44, 47, 50, 54, 57, 60, 63, 65, 68, 68, 38, 39, 40, 42, 44, 46, + 48, 51, 54, 56, 59, 62, 65, 67, 69, 69, 39, 40, 40, 44, 49, 51, 53, 55, + 57, 59, 62, 64, 66, 68, 70, 70, 43, 42, 42, 46, 51, 53, 56, 58, 60, 62, + 64, 66, 68, 69, 71, 71, 47, 45, 44, 48, 53, 56, 59, 61, 63, 65, 67, 68, + 70, 71, 72, 72, 50, 49, 47, 51, 55, 58, 61, 63, 65, 67, 69, 70, 71, 72, + 74, 74, 54, 52, 50, 54, 57, 60, 63, 65, 68, 69, 71, 72, 73, 74, 75, 75, + 58, 56, 54, 56, 59, 62, 65, 67, 69, 70, 72, 73, 74, 75, 76, 76, 61, 59, + 57, 59, 62, 64, 67, 69, 71, 72, 73, 74, 75, 76, 77, 77, 64, 62, 60, 62, + 64, 66, 68, 70, 72, 73, 74, 75, 76, 77, 78, 78, 67, 65, 63, 65, 66, 68, + 70, 71, 73, 74, 75, 76, 77, 78, 78, 78, 69, 67, 65, 67, 68, 69, 71, 72, + 74, 75, 76, 77, 78, 78, 79, 79, 71, 70, 68, 69, 70, 71, 72, 74, 75, 76, + 77, 78, 78, 79, 79, 79, 71, 70, 68, 69, 70, 71, 72, 74, 75, 76, 77, 78, + 78, 79, 79, 79, + /* Size 32 */ + 44, 42, 40, 39, 37, 38, 38, 39, 39, 41, 42, 44, 46, 48, 50, 52, 54, 56, + 57, 59, 61, 62, 64, 65, 66, 68, 69, 70, 71, 71, 71, 71, 42, 41, 40, 39, + 38, 38, 39, 39, 39, 41, 42, 44, 45, 47, 49, 51, 53, 54, 56, 58, 60, 61, + 62, 64, 65, 67, 68, 69, 70, 70, 70, 70, 40, 40, 40, 39, 39, 39, 39, 39, + 39, 41, 42, 43, 45, 46, 48, 50, 52, 53, 55, 57, 59, 60, 61, 63, 64, 66, + 67, 68, 69, 69, 69, 69, 39, 39, 39, 39, 40, 40, 40, 39, 39, 40, 42, 43, + 44, 46, 47, 49, 51, 52, 54, 56, 58, 59, 60, 62, 63, 65, 66, 67, 68, 68, + 68, 68, 37, 38, 39, 40, 41, 40, 40, 40, 39, 40, 41, 42, 44, 45, 47, 48, + 50, 51, 53, 55, 57, 58, 59, 61, 63, 64, 65, 66, 67, 67, 67, 67, 38, 38, + 39, 40, 40, 41, 41, 41, 41, 42, 43, 44, 45, 47, 48, 50, 51, 53, 54, 56, + 58, 59, 60, 62, 63, 64, 65, 67, 68, 68, 68, 68, 38, 39, 39, 40, 40, 41, + 42, 43, 43, 44, 45, 46, 48, 49, 50, 52, 53, 54, 56, 57, 59, 60, 61, 63, + 64, 65, 66, 67, 68, 68, 68, 68, 39, 39, 39, 39, 40, 41, 43, 44, 46, 47, + 48, 49, 50, 51, 52, 53, 55, 56, 57, 59, 60, 61, 62, 63, 65, 66, 67, 68, + 69, 69, 69, 69, 39, 39, 39, 39, 39, 41, 43, 46, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 69, 69, 69, + 41, 41, 41, 40, 40, 42, 44, 47, 49, 50, 51, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 70, 70, 70, 42, 42, 42, 42, + 41, 43, 45, 48, 50, 51, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 70, 71, 71, 71, 71, 44, 44, 43, 43, 42, 44, 46, 49, + 51, 53, 54, 55, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 66, 67, 68, 69, + 70, 70, 71, 71, 71, 71, 46, 45, 45, 44, 44, 45, 48, 50, 52, 54, 55, 57, + 59, 60, 61, 62, 63, 63, 64, 65, 66, 67, 67, 68, 69, 70, 70, 71, 72, 72, + 72, 72, 48, 47, 46, 46, 45, 47, 49, 51, 53, 55, 56, 58, 60, 61, 62, 63, + 64, 64, 65, 66, 67, 68, 68, 69, 70, 70, 71, 72, 72, 72, 72, 72, 50, 49, + 48, 47, 47, 48, 50, 52, 54, 56, 57, 59, 61, 62, 63, 64, 65, 66, 66, 67, + 68, 69, 69, 70, 71, 71, 72, 72, 73, 73, 73, 73, 52, 51, 50, 49, 48, 50, + 52, 53, 55, 57, 58, 60, 62, 63, 64, 65, 66, 67, 67, 68, 69, 69, 70, 71, + 71, 72, 72, 73, 74, 74, 74, 74, 54, 53, 52, 51, 50, 51, 53, 55, 56, 58, + 59, 61, 63, 64, 65, 66, 67, 68, 68, 69, 70, 70, 71, 72, 72, 73, 73, 74, + 74, 74, 74, 74, 56, 54, 53, 52, 51, 53, 54, 56, 58, 59, 60, 62, 63, 64, + 66, 67, 68, 68, 69, 70, 71, 71, 72, 72, 73, 73, 74, 74, 75, 75, 75, 75, + 57, 56, 55, 54, 53, 54, 56, 57, 59, 60, 61, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 71, 72, 72, 73, 73, 74, 74, 75, 75, 75, 75, 75, 59, 58, 57, 56, + 55, 56, 57, 59, 60, 61, 62, 64, 65, 66, 67, 68, 69, 70, 71, 71, 72, 72, + 73, 73, 74, 74, 75, 75, 76, 76, 76, 76, 61, 60, 59, 58, 57, 58, 59, 60, + 61, 62, 63, 65, 66, 67, 68, 69, 70, 71, 71, 72, 73, 73, 74, 74, 75, 75, + 75, 76, 76, 76, 76, 76, 62, 61, 60, 59, 58, 59, 60, 61, 62, 63, 64, 66, + 67, 68, 69, 69, 70, 71, 72, 72, 73, 74, 74, 75, 75, 75, 76, 76, 76, 76, + 76, 76, 64, 62, 61, 60, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, + 71, 72, 72, 73, 74, 74, 75, 75, 75, 76, 76, 76, 77, 77, 77, 77, 65, 64, + 63, 62, 61, 62, 63, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 72, 73, 73, + 74, 75, 75, 75, 76, 76, 77, 77, 77, 77, 77, 77, 66, 65, 64, 63, 63, 63, + 64, 65, 65, 66, 67, 68, 69, 70, 71, 71, 72, 73, 73, 74, 75, 75, 75, 76, + 76, 77, 77, 77, 78, 78, 78, 78, 68, 67, 66, 65, 64, 64, 65, 66, 66, 67, + 68, 69, 70, 70, 71, 72, 73, 73, 74, 74, 75, 75, 76, 76, 77, 77, 77, 78, + 78, 78, 78, 78, 69, 68, 67, 66, 65, 65, 66, 67, 67, 68, 69, 70, 70, 71, + 72, 72, 73, 74, 74, 75, 75, 76, 76, 77, 77, 77, 77, 78, 78, 78, 78, 78, + 70, 69, 68, 67, 66, 67, 67, 68, 68, 69, 70, 70, 71, 72, 72, 73, 74, 74, + 75, 75, 76, 76, 76, 77, 77, 78, 78, 78, 78, 78, 78, 78, 71, 70, 69, 68, + 67, 68, 68, 69, 69, 70, 71, 71, 72, 72, 73, 74, 74, 75, 75, 76, 76, 76, + 77, 77, 78, 78, 78, 78, 79, 79, 79, 79, 71, 70, 69, 68, 67, 68, 68, 69, + 69, 70, 71, 71, 72, 72, 73, 74, 74, 75, 75, 76, 76, 76, 77, 77, 78, 78, + 78, 78, 79, 79, 79, 79, 71, 70, 69, 68, 67, 68, 68, 69, 69, 70, 71, 71, + 72, 72, 73, 74, 74, 75, 75, 76, 76, 76, 77, 77, 78, 78, 78, 78, 79, 79, + 79, 79, 71, 70, 69, 68, 67, 68, 68, 69, 69, 70, 71, 71, 72, 72, 73, 74, + 74, 75, 75, 76, 76, 76, 77, 77, 78, 78, 78, 78, 79, 79, 79, 79 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 83, 86, 96, 83, 90, 93, 98, 86, 93, 103, 108, 96, 98, 108, 114, + /* Size 8 */ + 64, 56, 79, 81, 84, 88, 93, 97, 56, 70, 80, 76, 77, 81, 85, 90, 79, 80, + 86, 84, 84, 86, 90, 93, 81, 76, 84, 89, 91, 93, 95, 98, 84, 77, 84, 91, + 95, 98, 100, 102, 88, 81, 86, 93, 98, 101, 104, 106, 93, 85, 90, 95, + 100, 104, 106, 108, 97, 90, 93, 98, 102, 106, 108, 111, + /* Size 16 */ + 64, 60, 56, 66, 79, 80, 81, 82, 84, 86, 88, 90, 93, 95, 97, 97, 60, 61, + 62, 70, 79, 79, 78, 79, 80, 82, 84, 86, 89, 91, 94, 94, 56, 62, 70, 74, + 80, 78, 76, 76, 77, 79, 81, 83, 85, 88, 90, 90, 66, 70, 74, 78, 83, 81, + 80, 80, 80, 82, 83, 85, 87, 89, 92, 92, 79, 79, 80, 83, 86, 85, 84, 84, + 84, 85, 86, 88, 90, 91, 93, 93, 80, 79, 78, 81, 85, 86, 87, 87, 88, 88, + 89, 91, 92, 94, 95, 95, 81, 78, 76, 80, 84, 87, 89, 90, 91, 92, 93, 94, + 95, 96, 98, 98, 82, 79, 76, 80, 84, 87, 90, 92, 93, 94, 95, 96, 97, 98, + 100, 100, 84, 80, 77, 80, 84, 88, 91, 93, 95, 96, 98, 99, 100, 101, 102, + 102, 86, 82, 79, 82, 85, 88, 92, 94, 96, 98, 99, 101, 102, 103, 104, + 104, 88, 84, 81, 83, 86, 89, 93, 95, 98, 99, 101, 102, 104, 105, 106, + 106, 90, 86, 83, 85, 88, 91, 94, 96, 99, 101, 102, 104, 105, 106, 107, + 107, 93, 89, 85, 87, 90, 92, 95, 97, 100, 102, 104, 105, 106, 107, 108, + 108, 95, 91, 88, 89, 91, 94, 96, 98, 101, 103, 105, 106, 107, 108, 109, + 109, 97, 94, 90, 92, 93, 95, 98, 100, 102, 104, 106, 107, 108, 109, 111, + 111, 97, 94, 90, 92, 93, 95, 98, 100, 102, 104, 106, 107, 108, 109, 111, + 111, + /* Size 32 */ + 64, 62, 60, 58, 56, 60, 66, 72, 79, 79, 80, 81, 81, 82, 82, 83, 84, 85, + 86, 87, 88, 89, 90, 92, 93, 94, 95, 96, 97, 97, 97, 97, 62, 61, 60, 60, + 59, 63, 68, 73, 79, 79, 79, 80, 80, 80, 81, 81, 82, 83, 84, 85, 86, 87, + 88, 90, 91, 92, 93, 94, 95, 95, 95, 95, 60, 60, 61, 62, 62, 66, 70, 74, + 79, 79, 79, 79, 78, 79, 79, 80, 80, 81, 82, 83, 84, 85, 86, 88, 89, 90, + 91, 92, 94, 94, 94, 94, 58, 60, 62, 64, 66, 69, 72, 75, 79, 79, 78, 78, + 77, 77, 78, 78, 78, 79, 80, 81, 82, 83, 85, 86, 87, 88, 89, 91, 92, 92, + 92, 92, 56, 59, 62, 66, 70, 72, 74, 77, 80, 79, 78, 77, 76, 76, 76, 77, + 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 88, 89, 90, 90, 90, 90, 60, 63, + 66, 69, 72, 74, 76, 79, 81, 80, 79, 78, 78, 78, 78, 78, 79, 79, 80, 81, + 82, 83, 84, 85, 86, 87, 89, 90, 91, 91, 91, 91, 66, 68, 70, 72, 74, 76, + 78, 80, 83, 82, 81, 80, 80, 80, 80, 80, 80, 81, 82, 83, 83, 84, 85, 86, + 87, 88, 89, 91, 92, 92, 92, 92, 72, 73, 74, 75, 77, 79, 80, 82, 84, 84, + 83, 83, 82, 82, 82, 82, 82, 83, 83, 84, 85, 86, 87, 87, 88, 89, 90, 91, + 92, 92, 92, 92, 79, 79, 79, 79, 80, 81, 83, 84, 86, 85, 85, 85, 84, 84, + 84, 84, 84, 85, 85, 86, 86, 87, 88, 89, 90, 90, 91, 92, 93, 93, 93, 93, + 79, 79, 79, 79, 79, 80, 82, 84, 85, 85, 85, 85, 86, 86, 86, 86, 86, 86, + 87, 87, 88, 88, 89, 90, 91, 92, 93, 93, 94, 94, 94, 94, 80, 79, 79, 78, + 78, 79, 81, 83, 85, 85, 86, 86, 87, 87, 87, 87, 88, 88, 88, 89, 89, 90, + 91, 91, 92, 93, 94, 95, 95, 95, 95, 95, 81, 80, 79, 78, 77, 78, 80, 83, + 85, 85, 86, 87, 88, 88, 89, 89, 89, 90, 90, 91, 91, 92, 92, 93, 93, 94, + 95, 96, 96, 96, 96, 96, 81, 80, 78, 77, 76, 78, 80, 82, 84, 86, 87, 88, + 89, 90, 90, 91, 91, 91, 92, 92, 93, 93, 94, 94, 95, 96, 96, 97, 98, 98, + 98, 98, 82, 80, 79, 77, 76, 78, 80, 82, 84, 86, 87, 88, 90, 90, 91, 91, + 92, 92, 93, 93, 94, 94, 95, 95, 96, 97, 97, 98, 99, 99, 99, 99, 82, 81, + 79, 78, 76, 78, 80, 82, 84, 86, 87, 89, 90, 91, 92, 92, 93, 94, 94, 95, + 95, 96, 96, 97, 97, 98, 98, 99, 100, 100, 100, 100, 83, 81, 80, 78, 77, + 78, 80, 82, 84, 86, 87, 89, 91, 91, 92, 93, 94, 95, 95, 96, 96, 97, 97, + 98, 99, 99, 100, 100, 101, 101, 101, 101, 84, 82, 80, 78, 77, 79, 80, + 82, 84, 86, 88, 89, 91, 92, 93, 94, 95, 96, 96, 97, 98, 98, 99, 99, 100, + 100, 101, 101, 102, 102, 102, 102, 85, 83, 81, 79, 78, 79, 81, 83, 85, + 86, 88, 90, 91, 92, 94, 95, 96, 96, 97, 98, 99, 99, 100, 100, 101, 101, + 102, 102, 103, 103, 103, 103, 86, 84, 82, 80, 79, 80, 82, 83, 85, 87, + 88, 90, 92, 93, 94, 95, 96, 97, 98, 99, 99, 100, 101, 101, 102, 102, + 103, 103, 104, 104, 104, 104, 87, 85, 83, 81, 80, 81, 83, 84, 86, 87, + 89, 91, 92, 93, 95, 96, 97, 98, 99, 99, 100, 101, 101, 102, 103, 103, + 104, 104, 105, 105, 105, 105, 88, 86, 84, 82, 81, 82, 83, 85, 86, 88, + 89, 91, 93, 94, 95, 96, 98, 99, 99, 100, 101, 102, 102, 103, 104, 104, + 105, 105, 106, 106, 106, 106, 89, 87, 85, 83, 82, 83, 84, 86, 87, 88, + 90, 92, 93, 94, 96, 97, 98, 99, 100, 101, 102, 102, 103, 104, 104, 105, + 105, 106, 106, 106, 106, 106, 90, 88, 86, 85, 83, 84, 85, 87, 88, 89, + 91, 92, 94, 95, 96, 97, 99, 100, 101, 101, 102, 103, 104, 104, 105, 105, + 106, 106, 107, 107, 107, 107, 92, 90, 88, 86, 84, 85, 86, 87, 89, 90, + 91, 93, 94, 95, 97, 98, 99, 100, 101, 102, 103, 104, 104, 105, 106, 106, + 107, 107, 108, 108, 108, 108, 93, 91, 89, 87, 85, 86, 87, 88, 90, 91, + 92, 93, 95, 96, 97, 99, 100, 101, 102, 103, 104, 104, 105, 106, 106, + 107, 107, 108, 108, 108, 108, 108, 94, 92, 90, 88, 86, 87, 88, 89, 90, + 92, 93, 94, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 105, 106, 107, + 107, 108, 108, 109, 109, 109, 109, 95, 93, 91, 89, 88, 89, 89, 90, 91, + 93, 94, 95, 96, 97, 98, 100, 101, 102, 103, 104, 105, 105, 106, 107, + 107, 108, 108, 109, 109, 109, 109, 109, 96, 94, 92, 91, 89, 90, 91, 91, + 92, 93, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 106, 107, + 108, 108, 109, 109, 110, 110, 110, 110, 97, 95, 94, 92, 90, 91, 92, 92, + 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 106, 107, + 108, 108, 109, 109, 110, 111, 111, 111, 111, 97, 95, 94, 92, 90, 91, 92, + 92, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 106, 107, + 108, 108, 109, 109, 110, 111, 111, 111, 111, 97, 95, 94, 92, 90, 91, 92, + 92, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 106, 107, + 108, 108, 109, 109, 110, 111, 111, 111, 111, 97, 95, 94, 92, 90, 91, 92, + 92, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 106, 107, + 108, 108, 109, 109, 110, 111, 111, 111, 111 }, + { /* Intra matrices */ + /* Size 4 */ + 42, 56, 58, 65, 56, 61, 63, 67, 58, 63, 70, 74, 65, 67, 74, 78, + /* Size 8 */ + 45, 39, 55, 57, 59, 62, 66, 69, 39, 49, 56, 53, 54, 57, 60, 64, 55, 56, + 61, 60, 60, 61, 63, 66, 57, 53, 60, 63, 65, 66, 68, 70, 59, 54, 60, 65, + 68, 70, 71, 73, 62, 57, 61, 66, 70, 72, 74, 76, 66, 60, 63, 68, 71, 74, + 76, 78, 69, 64, 66, 70, 73, 76, 78, 80, + /* Size 16 */ + 44, 41, 38, 45, 55, 56, 56, 57, 58, 60, 62, 63, 65, 67, 69, 69, 41, 42, + 43, 48, 55, 55, 54, 55, 56, 57, 59, 60, 62, 64, 66, 66, 38, 43, 48, 52, + 55, 54, 52, 53, 53, 55, 56, 58, 59, 61, 63, 63, 45, 48, 52, 54, 57, 56, + 55, 56, 56, 57, 58, 60, 61, 63, 64, 64, 55, 55, 55, 57, 60, 59, 59, 59, + 59, 59, 60, 61, 63, 64, 66, 66, 56, 55, 54, 56, 59, 60, 61, 61, 61, 62, + 63, 64, 65, 66, 67, 67, 56, 54, 52, 55, 59, 61, 62, 63, 64, 64, 65, 66, + 67, 68, 69, 69, 57, 55, 53, 56, 59, 61, 63, 64, 65, 66, 67, 68, 69, 69, + 70, 70, 58, 56, 53, 56, 59, 61, 64, 65, 67, 68, 69, 70, 70, 71, 72, 72, + 60, 57, 55, 57, 59, 62, 64, 66, 68, 69, 70, 71, 72, 73, 73, 73, 62, 59, + 56, 58, 60, 63, 65, 67, 69, 70, 71, 72, 73, 74, 75, 75, 63, 60, 58, 60, + 61, 64, 66, 68, 70, 71, 72, 73, 74, 75, 76, 76, 65, 62, 59, 61, 63, 65, + 67, 69, 70, 72, 73, 74, 75, 76, 77, 77, 67, 64, 61, 63, 64, 66, 68, 69, + 71, 73, 74, 75, 76, 77, 78, 78, 69, 66, 63, 64, 66, 67, 69, 70, 72, 73, + 75, 76, 77, 78, 79, 79, 69, 66, 63, 64, 66, 67, 69, 70, 72, 73, 75, 76, + 77, 78, 79, 79, + /* Size 32 */ + 44, 42, 41, 39, 38, 41, 45, 49, 54, 55, 55, 56, 56, 57, 57, 58, 58, 59, + 60, 60, 61, 62, 63, 64, 65, 66, 66, 67, 68, 68, 68, 68, 42, 42, 41, 41, + 40, 43, 46, 50, 55, 55, 55, 55, 55, 55, 56, 56, 57, 57, 58, 59, 60, 61, + 61, 62, 63, 64, 65, 66, 67, 67, 67, 67, 41, 41, 42, 42, 42, 45, 48, 51, + 55, 55, 54, 54, 54, 54, 55, 55, 55, 56, 57, 58, 58, 59, 60, 61, 62, 63, + 63, 64, 65, 65, 65, 65, 39, 41, 42, 43, 45, 47, 49, 52, 55, 54, 54, 53, + 53, 53, 54, 54, 54, 55, 55, 56, 57, 58, 59, 59, 60, 61, 62, 63, 64, 64, + 64, 64, 38, 40, 42, 45, 48, 49, 51, 53, 55, 54, 53, 53, 52, 52, 53, 53, + 53, 54, 54, 55, 56, 56, 57, 58, 59, 60, 61, 62, 63, 63, 63, 63, 41, 43, + 45, 47, 49, 51, 53, 54, 56, 55, 55, 54, 54, 54, 54, 54, 54, 55, 55, 56, + 57, 57, 58, 59, 60, 61, 62, 62, 63, 63, 63, 63, 45, 46, 48, 49, 51, 53, + 54, 56, 57, 57, 56, 56, 55, 55, 55, 55, 56, 56, 57, 57, 58, 58, 59, 60, + 61, 61, 62, 63, 64, 64, 64, 64, 49, 50, 51, 52, 53, 54, 56, 57, 58, 58, + 57, 57, 57, 57, 57, 57, 57, 57, 58, 58, 59, 59, 60, 61, 61, 62, 63, 64, + 64, 64, 64, 64, 54, 55, 55, 55, 55, 56, 57, 58, 59, 59, 59, 59, 58, 58, + 58, 58, 58, 59, 59, 59, 60, 60, 61, 62, 62, 63, 64, 64, 65, 65, 65, 65, + 55, 55, 55, 54, 54, 55, 57, 58, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, + 60, 61, 61, 62, 62, 63, 63, 64, 65, 65, 66, 66, 66, 66, 55, 55, 54, 54, + 53, 55, 56, 57, 59, 59, 60, 60, 60, 60, 60, 61, 61, 61, 61, 62, 62, 63, + 63, 64, 64, 65, 65, 66, 67, 67, 67, 67, 56, 55, 54, 53, 53, 54, 56, 57, + 59, 59, 60, 60, 61, 61, 62, 62, 62, 62, 63, 63, 63, 64, 64, 65, 65, 66, + 66, 67, 67, 67, 67, 67, 56, 55, 54, 53, 52, 54, 55, 57, 58, 59, 60, 61, + 62, 62, 63, 63, 63, 64, 64, 64, 65, 65, 65, 66, 66, 67, 67, 68, 68, 68, + 68, 68, 57, 55, 54, 53, 52, 54, 55, 57, 58, 59, 60, 61, 62, 63, 63, 64, + 64, 64, 65, 65, 66, 66, 66, 67, 67, 68, 68, 69, 69, 69, 69, 69, 57, 56, + 55, 54, 53, 54, 55, 57, 58, 59, 60, 62, 63, 63, 64, 64, 65, 65, 66, 66, + 66, 67, 67, 68, 68, 69, 69, 69, 70, 70, 70, 70, 58, 56, 55, 54, 53, 54, + 55, 57, 58, 59, 61, 62, 63, 64, 64, 65, 66, 66, 67, 67, 67, 68, 68, 69, + 69, 69, 70, 70, 71, 71, 71, 71, 58, 57, 55, 54, 53, 54, 56, 57, 58, 60, + 61, 62, 63, 64, 65, 66, 66, 67, 67, 68, 68, 69, 69, 70, 70, 70, 71, 71, + 72, 72, 72, 72, 59, 57, 56, 55, 54, 55, 56, 57, 59, 60, 61, 62, 64, 64, + 65, 66, 67, 67, 68, 68, 69, 69, 70, 70, 71, 71, 71, 72, 72, 72, 72, 72, + 60, 58, 57, 55, 54, 55, 57, 58, 59, 60, 61, 63, 64, 65, 66, 67, 67, 68, + 69, 69, 70, 70, 71, 71, 71, 72, 72, 73, 73, 73, 73, 73, 60, 59, 58, 56, + 55, 56, 57, 58, 59, 61, 62, 63, 64, 65, 66, 67, 68, 68, 69, 70, 70, 71, + 71, 72, 72, 72, 73, 73, 74, 74, 74, 74, 61, 60, 58, 57, 56, 57, 58, 59, + 60, 61, 62, 63, 65, 66, 66, 67, 68, 69, 70, 70, 71, 71, 72, 72, 73, 73, + 74, 74, 74, 74, 74, 74, 62, 61, 59, 58, 56, 57, 58, 59, 60, 62, 63, 64, + 65, 66, 67, 68, 69, 69, 70, 71, 71, 72, 72, 73, 73, 74, 74, 74, 75, 75, + 75, 75, 63, 61, 60, 59, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 71, 71, 72, 72, 73, 73, 74, 74, 75, 75, 75, 75, 75, 75, 64, 62, + 61, 59, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 70, 71, 72, + 72, 73, 73, 74, 74, 75, 75, 76, 76, 76, 76, 76, 65, 63, 62, 60, 59, 60, + 61, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 71, 72, 73, 73, 74, 74, + 75, 75, 76, 76, 76, 76, 76, 76, 66, 64, 63, 61, 60, 61, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 69, 70, 71, 72, 72, 73, 74, 74, 75, 75, 76, 76, 76, + 77, 77, 77, 77, 66, 65, 63, 62, 61, 62, 62, 63, 64, 65, 65, 66, 67, 68, + 69, 70, 71, 71, 72, 73, 74, 74, 75, 75, 76, 76, 76, 77, 77, 77, 77, 77, + 67, 66, 64, 63, 62, 62, 63, 64, 64, 65, 66, 67, 68, 69, 69, 70, 71, 72, + 73, 73, 74, 74, 75, 76, 76, 76, 77, 77, 78, 78, 78, 78, 68, 67, 65, 64, + 63, 63, 64, 64, 65, 66, 67, 67, 68, 69, 70, 71, 72, 72, 73, 74, 74, 75, + 75, 76, 76, 77, 77, 78, 78, 78, 78, 78, 68, 67, 65, 64, 63, 63, 64, 64, + 65, 66, 67, 67, 68, 69, 70, 71, 72, 72, 73, 74, 74, 75, 75, 76, 76, 77, + 77, 78, 78, 78, 78, 78, 68, 67, 65, 64, 63, 63, 64, 64, 65, 66, 67, 67, + 68, 69, 70, 71, 72, 72, 73, 74, 74, 75, 75, 76, 76, 77, 77, 78, 78, 78, + 78, 78, 68, 67, 65, 64, 63, 63, 64, 64, 65, 66, 67, 67, 68, 69, 70, 71, + 72, 72, 73, 74, 74, 75, 75, 76, 76, 77, 77, 78, 78, 78, 78, 78 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 67, 84, 97, 67, 81, 91, 99, 84, 91, 100, 105, 97, 99, 105, 108, + /* Size 8 */ + 64, 55, 58, 66, 75, 82, 88, 92, 55, 59, 58, 63, 70, 78, 84, 89, 58, 58, + 68, 73, 78, 82, 87, 91, 66, 63, 73, 80, 84, 87, 90, 93, 75, 70, 78, 84, + 88, 91, 93, 95, 82, 78, 82, 87, 91, 94, 96, 97, 88, 84, 87, 90, 93, 96, + 97, 98, 92, 89, 91, 93, 95, 97, 98, 99, + /* Size 16 */ + 64, 59, 55, 56, 58, 62, 66, 70, 75, 78, 82, 85, 88, 90, 92, 92, 59, 58, + 57, 58, 58, 61, 65, 68, 73, 76, 80, 83, 86, 88, 90, 90, 55, 57, 59, 59, + 58, 60, 63, 67, 70, 74, 78, 81, 84, 86, 89, 89, 56, 58, 59, 61, 63, 65, + 68, 71, 74, 77, 80, 83, 85, 87, 90, 90, 58, 58, 58, 63, 68, 71, 73, 75, + 78, 80, 82, 85, 87, 89, 91, 91, 62, 61, 60, 65, 71, 73, 76, 78, 81, 83, + 85, 87, 89, 90, 92, 92, 66, 65, 63, 68, 73, 76, 80, 82, 84, 86, 87, 89, + 90, 92, 93, 93, 70, 68, 67, 71, 75, 78, 82, 84, 86, 88, 89, 91, 92, 93, + 94, 94, 75, 73, 70, 74, 78, 81, 84, 86, 88, 90, 91, 92, 93, 94, 95, 95, + 78, 76, 74, 77, 80, 83, 86, 88, 90, 91, 92, 93, 94, 95, 96, 96, 82, 80, + 78, 80, 82, 85, 87, 89, 91, 92, 94, 95, 96, 96, 97, 97, 85, 83, 81, 83, + 85, 87, 89, 91, 92, 93, 95, 96, 96, 97, 98, 98, 88, 86, 84, 85, 87, 89, + 90, 92, 93, 94, 96, 96, 97, 98, 98, 98, 90, 88, 86, 87, 89, 90, 92, 93, + 94, 95, 96, 97, 98, 98, 99, 99, 92, 90, 89, 90, 91, 92, 93, 94, 95, 96, + 97, 98, 98, 99, 99, 99, 92, 90, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, + 98, 99, 99, 99, + /* Size 32 */ + 64, 62, 59, 57, 55, 56, 56, 57, 58, 60, 62, 64, 66, 68, 70, 72, 75, 77, + 78, 80, 82, 84, 85, 86, 88, 89, 90, 91, 92, 92, 92, 92, 62, 60, 59, 58, + 56, 57, 57, 57, 58, 60, 61, 63, 65, 67, 69, 71, 74, 75, 77, 79, 81, 82, + 84, 85, 87, 88, 89, 90, 91, 91, 91, 91, 59, 59, 58, 58, 57, 57, 58, 58, + 58, 59, 61, 63, 65, 66, 68, 70, 73, 74, 76, 78, 80, 81, 83, 84, 86, 87, + 88, 89, 90, 90, 90, 90, 57, 58, 58, 58, 58, 58, 58, 58, 58, 59, 61, 62, + 64, 66, 67, 69, 72, 73, 75, 77, 79, 80, 82, 83, 85, 86, 87, 88, 90, 90, + 90, 90, 55, 56, 57, 58, 59, 59, 59, 58, 58, 59, 60, 62, 63, 65, 67, 68, + 70, 72, 74, 76, 78, 79, 81, 82, 84, 85, 86, 87, 89, 89, 89, 89, 56, 57, + 57, 58, 59, 59, 60, 60, 60, 62, 63, 64, 65, 67, 69, 70, 72, 74, 75, 77, + 79, 80, 82, 83, 85, 86, 87, 88, 89, 89, 89, 89, 56, 57, 58, 58, 59, 60, + 61, 62, 63, 64, 65, 66, 68, 69, 71, 72, 74, 75, 77, 78, 80, 81, 83, 84, + 85, 86, 87, 89, 90, 90, 90, 90, 57, 57, 58, 58, 58, 60, 62, 64, 66, 67, + 68, 69, 70, 72, 73, 74, 76, 77, 78, 80, 81, 82, 84, 85, 86, 87, 88, 89, + 90, 90, 90, 90, 58, 58, 58, 58, 58, 60, 63, 66, 68, 70, 71, 72, 73, 74, + 75, 76, 78, 79, 80, 81, 82, 83, 85, 86, 87, 88, 89, 90, 91, 91, 91, 91, + 60, 60, 59, 59, 59, 62, 64, 67, 70, 71, 72, 73, 75, 76, 77, 78, 79, 80, + 81, 82, 84, 85, 86, 87, 88, 89, 89, 90, 91, 91, 91, 91, 62, 61, 61, 61, + 60, 63, 65, 68, 71, 72, 73, 75, 76, 77, 78, 80, 81, 82, 83, 84, 85, 86, + 87, 88, 89, 89, 90, 91, 92, 92, 92, 92, 64, 63, 63, 62, 62, 64, 66, 69, + 72, 73, 75, 76, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 89, 90, + 91, 92, 92, 92, 92, 92, 66, 65, 65, 64, 63, 65, 68, 70, 73, 75, 76, 78, + 80, 81, 82, 83, 84, 85, 86, 87, 87, 88, 89, 90, 90, 91, 92, 92, 93, 93, + 93, 93, 68, 67, 66, 66, 65, 67, 69, 72, 74, 76, 77, 79, 81, 82, 83, 84, + 85, 86, 87, 87, 88, 89, 90, 90, 91, 92, 92, 93, 94, 94, 94, 94, 70, 69, + 68, 67, 67, 69, 71, 73, 75, 77, 78, 80, 82, 83, 84, 85, 86, 87, 88, 88, + 89, 90, 91, 91, 92, 92, 93, 94, 94, 94, 94, 94, 72, 71, 70, 69, 68, 70, + 72, 74, 76, 78, 80, 81, 83, 84, 85, 86, 87, 88, 89, 89, 90, 91, 91, 92, + 93, 93, 94, 94, 95, 95, 95, 95, 75, 74, 73, 72, 70, 72, 74, 76, 78, 79, + 81, 82, 84, 85, 86, 87, 88, 89, 90, 90, 91, 92, 92, 93, 93, 94, 94, 95, + 95, 95, 95, 95, 77, 75, 74, 73, 72, 74, 75, 77, 79, 80, 82, 83, 85, 86, + 87, 88, 89, 90, 90, 91, 92, 92, 93, 93, 94, 94, 95, 95, 96, 96, 96, 96, + 78, 77, 76, 75, 74, 75, 77, 78, 80, 81, 83, 84, 86, 87, 88, 89, 90, 90, + 91, 92, 92, 93, 93, 94, 94, 95, 95, 96, 96, 96, 96, 96, 80, 79, 78, 77, + 76, 77, 78, 80, 81, 82, 84, 85, 87, 87, 88, 89, 90, 91, 92, 92, 93, 94, + 94, 95, 95, 95, 96, 96, 97, 97, 97, 97, 82, 81, 80, 79, 78, 79, 80, 81, + 82, 84, 85, 86, 87, 88, 89, 90, 91, 92, 92, 93, 94, 94, 95, 95, 96, 96, + 96, 97, 97, 97, 97, 97, 84, 82, 81, 80, 79, 80, 81, 82, 83, 85, 86, 87, + 88, 89, 90, 91, 92, 92, 93, 94, 94, 95, 95, 96, 96, 96, 97, 97, 97, 97, + 97, 97, 85, 84, 83, 82, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 91, + 92, 93, 93, 94, 95, 95, 96, 96, 96, 97, 97, 97, 98, 98, 98, 98, 86, 85, + 84, 83, 82, 83, 84, 85, 86, 87, 88, 89, 90, 90, 91, 92, 93, 93, 94, 95, + 95, 96, 96, 96, 97, 97, 97, 98, 98, 98, 98, 98, 88, 87, 86, 85, 84, 85, + 85, 86, 87, 88, 89, 89, 90, 91, 92, 93, 93, 94, 94, 95, 96, 96, 96, 97, + 97, 97, 98, 98, 98, 98, 98, 98, 89, 88, 87, 86, 85, 86, 86, 87, 88, 89, + 89, 90, 91, 92, 92, 93, 94, 94, 95, 95, 96, 96, 97, 97, 97, 98, 98, 98, + 99, 99, 99, 99, 90, 89, 88, 87, 86, 87, 87, 88, 89, 89, 90, 91, 92, 92, + 93, 94, 94, 95, 95, 96, 96, 97, 97, 97, 98, 98, 98, 99, 99, 99, 99, 99, + 91, 90, 89, 88, 87, 88, 89, 89, 90, 90, 91, 92, 92, 93, 94, 94, 95, 95, + 96, 96, 97, 97, 97, 98, 98, 98, 99, 99, 99, 99, 99, 99, 92, 91, 90, 90, + 89, 89, 90, 90, 91, 91, 92, 92, 93, 94, 94, 95, 95, 96, 96, 97, 97, 97, + 98, 98, 98, 99, 99, 99, 99, 99, 99, 99, 92, 91, 90, 90, 89, 89, 90, 90, + 91, 91, 92, 92, 93, 94, 94, 95, 95, 96, 96, 97, 97, 97, 98, 98, 98, 99, + 99, 99, 99, 99, 99, 99, 92, 91, 90, 90, 89, 89, 90, 90, 91, 91, 92, 92, + 93, 94, 94, 95, 95, 96, 96, 97, 97, 97, 98, 98, 98, 99, 99, 99, 99, 99, + 99, 99, 92, 91, 90, 90, 89, 89, 90, 90, 91, 91, 92, 92, 93, 94, 94, 95, + 95, 96, 96, 97, 97, 97, 98, 98, 98, 99, 99, 99, 99, 99, 99, 99 }, + { /* Intra matrices */ + /* Size 4 */ + 44, 46, 58, 68, 46, 56, 64, 70, 58, 64, 71, 74, 68, 70, 74, 77, + /* Size 8 */ + 49, 42, 44, 50, 57, 63, 68, 71, 42, 45, 44, 48, 54, 60, 65, 69, 44, 44, + 52, 56, 60, 63, 67, 70, 50, 48, 56, 61, 65, 68, 70, 72, 57, 54, 60, 65, + 68, 71, 73, 74, 63, 60, 63, 68, 71, 73, 74, 76, 68, 65, 67, 70, 73, 74, + 76, 77, 71, 69, 70, 72, 74, 76, 77, 78, + /* Size 16 */ + 48, 44, 41, 42, 43, 46, 49, 53, 56, 59, 62, 65, 67, 69, 70, 70, 44, 43, + 43, 43, 43, 45, 48, 51, 55, 57, 61, 63, 65, 67, 69, 69, 41, 43, 44, 44, + 43, 45, 47, 50, 53, 56, 59, 61, 64, 66, 68, 68, 42, 43, 44, 45, 47, 49, + 51, 53, 56, 58, 61, 63, 65, 67, 68, 68, 43, 43, 43, 47, 51, 53, 55, 57, + 59, 61, 63, 64, 66, 68, 69, 69, 46, 45, 45, 49, 53, 55, 58, 59, 61, 63, + 64, 66, 68, 69, 70, 70, 49, 48, 47, 51, 55, 58, 61, 62, 64, 65, 67, 68, + 69, 70, 71, 71, 53, 51, 50, 53, 57, 59, 62, 64, 66, 67, 68, 69, 70, 71, + 72, 72, 56, 55, 53, 56, 59, 61, 64, 66, 67, 69, 70, 71, 72, 72, 73, 73, + 59, 57, 56, 58, 61, 63, 65, 67, 69, 70, 71, 72, 72, 73, 74, 74, 62, 61, + 59, 61, 63, 64, 67, 68, 70, 71, 72, 73, 73, 74, 75, 75, 65, 63, 61, 63, + 64, 66, 68, 69, 71, 72, 73, 73, 74, 75, 75, 75, 67, 65, 64, 65, 66, 68, + 69, 70, 72, 72, 73, 74, 75, 75, 76, 76, 69, 67, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 75, 76, 76, 76, 70, 69, 68, 68, 69, 70, 71, 72, 73, 74, + 75, 75, 76, 76, 76, 76, 70, 69, 68, 68, 69, 70, 71, 72, 73, 74, 75, 75, + 76, 76, 76, 76, + /* Size 32 */ + 47, 46, 44, 42, 41, 41, 42, 42, 43, 44, 46, 47, 49, 51, 52, 54, 56, 57, + 59, 60, 62, 63, 64, 65, 66, 67, 68, 69, 70, 70, 70, 70, 46, 44, 43, 42, + 41, 42, 42, 42, 43, 44, 45, 47, 48, 50, 52, 53, 55, 56, 58, 59, 61, 62, + 63, 64, 66, 66, 67, 68, 69, 69, 69, 69, 44, 43, 43, 43, 42, 42, 42, 43, + 43, 44, 45, 46, 48, 49, 51, 52, 54, 56, 57, 58, 60, 61, 62, 64, 65, 66, + 67, 68, 68, 68, 68, 68, 42, 42, 43, 43, 43, 43, 43, 43, 43, 44, 45, 46, + 47, 49, 50, 52, 53, 55, 56, 58, 59, 60, 61, 63, 64, 65, 66, 67, 68, 68, + 68, 68, 41, 41, 42, 43, 44, 44, 43, 43, 43, 44, 45, 46, 47, 48, 49, 51, + 53, 54, 55, 57, 58, 59, 61, 62, 63, 64, 65, 66, 67, 67, 67, 67, 41, 42, + 42, 43, 44, 44, 44, 44, 45, 46, 46, 47, 48, 50, 51, 52, 54, 55, 56, 58, + 59, 60, 61, 63, 64, 65, 66, 67, 68, 68, 68, 68, 42, 42, 42, 43, 43, 44, + 45, 46, 47, 47, 48, 49, 50, 51, 53, 54, 55, 56, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 68, 68, 68, 42, 42, 43, 43, 43, 44, 46, 47, 49, 50, + 50, 51, 52, 53, 54, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 67, + 68, 68, 68, 68, 43, 43, 43, 43, 43, 45, 47, 49, 51, 52, 53, 54, 55, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 66, 67, 68, 69, 69, 69, 69, + 44, 44, 44, 44, 44, 46, 47, 50, 52, 53, 54, 55, 56, 57, 58, 58, 59, 60, + 61, 62, 63, 64, 65, 65, 66, 67, 68, 68, 69, 69, 69, 69, 46, 45, 45, 45, + 45, 46, 48, 50, 53, 54, 55, 56, 57, 58, 59, 60, 61, 61, 62, 63, 64, 65, + 65, 66, 67, 68, 68, 69, 70, 70, 70, 70, 47, 47, 46, 46, 46, 47, 49, 51, + 54, 55, 56, 57, 59, 59, 60, 61, 62, 63, 63, 64, 65, 66, 66, 67, 68, 68, + 69, 70, 70, 70, 70, 70, 49, 48, 48, 47, 47, 48, 50, 52, 55, 56, 57, 59, + 60, 61, 62, 63, 63, 64, 65, 65, 66, 67, 67, 68, 69, 69, 70, 70, 71, 71, + 71, 71, 51, 50, 49, 49, 48, 50, 51, 53, 55, 57, 58, 59, 61, 62, 63, 63, + 64, 65, 65, 66, 67, 67, 68, 69, 69, 70, 70, 71, 71, 71, 71, 71, 52, 52, + 51, 50, 49, 51, 53, 54, 56, 58, 59, 60, 62, 63, 63, 64, 65, 66, 66, 67, + 68, 68, 69, 69, 70, 70, 71, 71, 72, 72, 72, 72, 54, 53, 52, 52, 51, 52, + 54, 56, 57, 58, 60, 61, 63, 63, 64, 65, 66, 67, 67, 68, 68, 69, 69, 70, + 70, 71, 71, 72, 72, 72, 72, 72, 56, 55, 54, 53, 53, 54, 55, 57, 58, 59, + 61, 62, 63, 64, 65, 66, 67, 67, 68, 69, 69, 70, 70, 71, 71, 71, 72, 72, + 72, 72, 72, 72, 57, 56, 56, 55, 54, 55, 56, 58, 59, 60, 61, 63, 64, 65, + 66, 67, 67, 68, 69, 69, 70, 70, 71, 71, 71, 72, 72, 72, 73, 73, 73, 73, + 59, 58, 57, 56, 55, 56, 58, 59, 60, 61, 62, 63, 65, 65, 66, 67, 68, 69, + 69, 70, 70, 71, 71, 71, 72, 72, 73, 73, 73, 73, 73, 73, 60, 59, 58, 58, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 69, 70, 70, 71, 71, + 72, 72, 72, 73, 73, 73, 74, 74, 74, 74, 62, 61, 60, 59, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 68, 69, 70, 70, 71, 71, 72, 72, 72, 73, 73, + 73, 74, 74, 74, 74, 74, 63, 62, 61, 60, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 67, 68, 69, 70, 70, 71, 71, 72, 72, 72, 73, 73, 73, 74, 74, 74, 74, + 74, 74, 64, 63, 62, 61, 61, 61, 62, 63, 64, 65, 65, 66, 67, 68, 69, 69, + 70, 71, 71, 72, 72, 72, 73, 73, 73, 74, 74, 74, 75, 75, 75, 75, 65, 64, + 64, 63, 62, 63, 63, 64, 65, 65, 66, 67, 68, 69, 69, 70, 71, 71, 71, 72, + 72, 73, 73, 73, 74, 74, 74, 75, 75, 75, 75, 75, 66, 66, 65, 64, 63, 64, + 64, 65, 66, 66, 67, 68, 69, 69, 70, 70, 71, 71, 72, 72, 73, 73, 73, 74, + 74, 74, 75, 75, 75, 75, 75, 75, 67, 66, 66, 65, 64, 65, 65, 66, 66, 67, + 68, 68, 69, 70, 70, 71, 71, 72, 72, 73, 73, 73, 74, 74, 74, 75, 75, 75, + 75, 75, 75, 75, 68, 67, 67, 66, 65, 66, 66, 67, 67, 68, 68, 69, 70, 70, + 71, 71, 72, 72, 73, 73, 73, 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, 75, + 69, 68, 68, 67, 66, 67, 67, 67, 68, 68, 69, 70, 70, 71, 71, 72, 72, 72, + 73, 73, 74, 74, 74, 75, 75, 75, 75, 75, 76, 76, 76, 76, 70, 69, 68, 68, + 67, 68, 68, 68, 69, 69, 70, 70, 71, 71, 72, 72, 72, 73, 73, 74, 74, 74, + 75, 75, 75, 75, 75, 76, 76, 76, 76, 76, 70, 69, 68, 68, 67, 68, 68, 68, + 69, 69, 70, 70, 71, 71, 72, 72, 72, 73, 73, 74, 74, 74, 75, 75, 75, 75, + 75, 76, 76, 76, 76, 76, 70, 69, 68, 68, 67, 68, 68, 68, 69, 69, 70, 70, + 71, 71, 72, 72, 72, 73, 73, 74, 74, 74, 75, 75, 75, 75, 75, 76, 76, 76, + 76, 76, 70, 69, 68, 68, 67, 68, 68, 68, 69, 69, 70, 70, 71, 71, 72, 72, + 72, 73, 73, 74, 74, 74, 75, 75, 75, 75, 75, 76, 76, 76, 76, 76 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 80, 83, 90, 80, 86, 88, 92, 83, 88, 96, 99, 90, 92, 99, 104, + /* Size 8 */ + 64, 57, 77, 78, 80, 84, 88, 91, 57, 69, 77, 74, 75, 78, 82, 86, 77, 77, + 82, 81, 81, 82, 85, 88, 78, 74, 81, 85, 86, 87, 89, 91, 80, 75, 81, 86, + 89, 91, 93, 95, 84, 78, 82, 87, 91, 94, 96, 97, 88, 82, 85, 89, 93, 96, + 98, 99, 91, 86, 88, 91, 95, 97, 99, 101, + /* Size 16 */ + 64, 60, 57, 65, 77, 77, 78, 79, 80, 82, 84, 86, 88, 89, 91, 91, 60, 61, + 62, 69, 77, 76, 76, 77, 78, 79, 81, 83, 85, 86, 88, 88, 57, 62, 69, 73, + 77, 75, 74, 74, 75, 76, 78, 80, 82, 84, 86, 86, 65, 69, 73, 76, 80, 78, + 77, 78, 78, 79, 80, 82, 83, 85, 87, 87, 77, 77, 77, 80, 82, 82, 81, 81, + 81, 82, 82, 84, 85, 87, 88, 88, 77, 76, 75, 78, 82, 82, 83, 83, 83, 84, + 85, 86, 87, 88, 90, 90, 78, 76, 74, 77, 81, 83, 85, 85, 86, 87, 87, 88, + 89, 90, 91, 91, 79, 77, 74, 78, 81, 83, 85, 87, 88, 89, 89, 90, 91, 92, + 93, 93, 80, 78, 75, 78, 81, 83, 86, 88, 89, 90, 91, 92, 93, 94, 95, 95, + 82, 79, 76, 79, 82, 84, 87, 89, 90, 92, 93, 94, 94, 95, 96, 96, 84, 81, + 78, 80, 82, 85, 87, 89, 91, 93, 94, 95, 96, 97, 97, 97, 86, 83, 80, 82, + 84, 86, 88, 90, 92, 94, 95, 96, 97, 98, 98, 98, 88, 85, 82, 83, 85, 87, + 89, 91, 93, 94, 96, 97, 98, 99, 99, 99, 89, 86, 84, 85, 87, 88, 90, 92, + 94, 95, 97, 98, 99, 99, 100, 100, 91, 88, 86, 87, 88, 90, 91, 93, 95, + 96, 97, 98, 99, 100, 101, 101, 91, 88, 86, 87, 88, 90, 91, 93, 95, 96, + 97, 98, 99, 100, 101, 101, + /* Size 32 */ + 64, 62, 60, 59, 57, 61, 65, 70, 77, 77, 77, 78, 78, 79, 79, 80, 80, 81, + 82, 83, 84, 85, 86, 87, 88, 88, 89, 90, 91, 91, 91, 91, 62, 61, 61, 60, + 60, 63, 67, 72, 77, 77, 77, 77, 77, 78, 78, 79, 79, 80, 81, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 90, 90, 90, 60, 61, 61, 62, 62, 65, 69, 73, + 77, 77, 76, 76, 76, 76, 77, 77, 78, 78, 79, 80, 81, 82, 83, 84, 85, 85, + 86, 87, 88, 88, 88, 88, 59, 60, 62, 64, 66, 68, 71, 74, 77, 76, 76, 75, + 75, 75, 76, 76, 76, 77, 78, 79, 79, 80, 81, 82, 83, 84, 85, 86, 87, 87, + 87, 87, 57, 60, 62, 66, 69, 71, 73, 75, 77, 76, 75, 75, 74, 74, 74, 75, + 75, 76, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 86, 86, 86, 61, 63, + 65, 68, 71, 73, 74, 76, 78, 78, 77, 76, 76, 76, 76, 76, 76, 77, 78, 78, + 79, 80, 81, 82, 82, 83, 84, 85, 86, 86, 86, 86, 65, 67, 69, 71, 73, 74, + 76, 78, 80, 79, 78, 78, 77, 77, 78, 78, 78, 78, 79, 80, 80, 81, 82, 82, + 83, 84, 85, 86, 87, 87, 87, 87, 70, 72, 73, 74, 75, 76, 78, 79, 81, 80, + 80, 80, 79, 79, 79, 79, 79, 80, 80, 81, 81, 82, 83, 83, 84, 85, 86, 87, + 87, 87, 87, 87, 77, 77, 77, 77, 77, 78, 80, 81, 82, 82, 82, 81, 81, 81, + 81, 81, 81, 81, 82, 82, 82, 83, 84, 84, 85, 86, 87, 87, 88, 88, 88, 88, + 77, 77, 77, 76, 76, 78, 79, 80, 82, 82, 82, 82, 82, 82, 82, 82, 82, 83, + 83, 83, 84, 84, 85, 85, 86, 87, 87, 88, 89, 89, 89, 89, 77, 77, 76, 76, + 75, 77, 78, 80, 82, 82, 82, 83, 83, 83, 83, 83, 83, 84, 84, 85, 85, 85, + 86, 87, 87, 88, 88, 89, 90, 90, 90, 90, 78, 77, 76, 75, 75, 76, 78, 80, + 81, 82, 83, 83, 84, 84, 84, 85, 85, 85, 86, 86, 86, 87, 87, 88, 88, 89, + 89, 90, 90, 90, 90, 90, 78, 77, 76, 75, 74, 76, 77, 79, 81, 82, 83, 84, + 85, 85, 85, 86, 86, 87, 87, 87, 87, 88, 88, 89, 89, 90, 90, 91, 91, 91, + 91, 91, 79, 78, 76, 75, 74, 76, 77, 79, 81, 82, 83, 84, 85, 86, 86, 87, + 87, 87, 88, 88, 88, 89, 89, 90, 90, 91, 91, 92, 92, 92, 92, 92, 79, 78, + 77, 76, 74, 76, 78, 79, 81, 82, 83, 84, 85, 86, 87, 87, 88, 88, 89, 89, + 89, 90, 90, 91, 91, 92, 92, 92, 93, 93, 93, 93, 80, 79, 77, 76, 75, 76, + 78, 79, 81, 82, 83, 85, 86, 87, 87, 88, 89, 89, 89, 90, 90, 91, 91, 92, + 92, 92, 93, 93, 94, 94, 94, 94, 80, 79, 78, 76, 75, 76, 78, 79, 81, 82, + 83, 85, 86, 87, 88, 89, 89, 90, 90, 91, 91, 92, 92, 93, 93, 93, 94, 94, + 95, 95, 95, 95, 81, 80, 78, 77, 76, 77, 78, 80, 81, 83, 84, 85, 87, 87, + 88, 89, 90, 90, 91, 91, 92, 92, 93, 93, 94, 94, 94, 95, 95, 95, 95, 95, + 82, 81, 79, 78, 76, 78, 79, 80, 82, 83, 84, 86, 87, 88, 89, 89, 90, 91, + 92, 92, 93, 93, 94, 94, 94, 95, 95, 96, 96, 96, 96, 96, 83, 81, 80, 79, + 77, 78, 80, 81, 82, 83, 85, 86, 87, 88, 89, 90, 91, 91, 92, 93, 93, 94, + 94, 95, 95, 95, 96, 96, 97, 97, 97, 97, 84, 82, 81, 79, 78, 79, 80, 81, + 82, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 93, 94, 94, 95, 95, 96, 96, + 97, 97, 97, 97, 97, 97, 85, 83, 82, 80, 79, 80, 81, 82, 83, 84, 85, 87, + 88, 89, 90, 91, 92, 92, 93, 94, 94, 95, 95, 96, 96, 97, 97, 97, 98, 98, + 98, 98, 86, 84, 83, 81, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, + 92, 93, 94, 94, 95, 95, 96, 96, 97, 97, 98, 98, 98, 98, 98, 98, 87, 85, + 84, 82, 81, 82, 82, 83, 84, 85, 87, 88, 89, 90, 91, 92, 93, 93, 94, 95, + 95, 96, 96, 97, 97, 98, 98, 98, 99, 99, 99, 99, 88, 86, 85, 83, 82, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 94, 95, 96, 96, 97, 97, + 98, 98, 99, 99, 99, 99, 99, 99, 88, 87, 85, 84, 83, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, 92, 93, 94, 95, 95, 96, 97, 97, 98, 98, 99, 99, 99, + 100, 100, 100, 100, 89, 88, 86, 85, 84, 84, 85, 86, 87, 87, 88, 89, 90, + 91, 92, 93, 94, 94, 95, 96, 97, 97, 98, 98, 99, 99, 99, 100, 100, 100, + 100, 100, 90, 89, 87, 86, 85, 85, 86, 87, 87, 88, 89, 90, 91, 92, 92, + 93, 94, 95, 96, 96, 97, 97, 98, 98, 99, 99, 100, 100, 101, 101, 101, + 101, 91, 90, 88, 87, 86, 86, 87, 87, 88, 89, 90, 90, 91, 92, 93, 94, 95, + 95, 96, 97, 97, 98, 98, 99, 99, 100, 100, 101, 101, 101, 101, 101, 91, + 90, 88, 87, 86, 86, 87, 87, 88, 89, 90, 90, 91, 92, 93, 94, 95, 95, 96, + 97, 97, 98, 98, 99, 99, 100, 100, 101, 101, 101, 101, 101, 91, 90, 88, + 87, 86, 86, 87, 87, 88, 89, 90, 90, 91, 92, 93, 94, 95, 95, 96, 97, 97, + 98, 98, 99, 99, 100, 100, 101, 101, 101, 101, 101, 91, 90, 88, 87, 86, + 86, 87, 87, 88, 89, 90, 90, 91, 92, 93, 94, 95, 95, 96, 97, 97, 98, 98, + 99, 99, 100, 100, 101, 101, 101, 101, 101 }, + { /* Intra matrices */ + /* Size 4 */ + 45, 57, 59, 65, 57, 62, 63, 67, 59, 63, 69, 72, 65, 67, 72, 76, + /* Size 8 */ + 47, 42, 57, 59, 60, 63, 66, 69, 42, 51, 57, 55, 56, 58, 61, 64, 57, 57, + 61, 61, 61, 62, 64, 66, 59, 55, 61, 64, 65, 66, 67, 69, 60, 56, 61, 65, + 67, 69, 70, 71, 63, 58, 62, 66, 69, 71, 72, 74, 66, 61, 64, 67, 70, 72, + 74, 75, 69, 64, 66, 69, 71, 74, 75, 77, + /* Size 16 */ + 47, 44, 41, 48, 56, 57, 58, 59, 60, 61, 62, 64, 65, 67, 68, 68, 44, 45, + 46, 50, 57, 56, 56, 57, 57, 58, 60, 61, 63, 64, 66, 66, 41, 46, 51, 54, + 57, 56, 54, 55, 55, 56, 57, 59, 60, 62, 63, 63, 48, 50, 54, 56, 59, 58, + 57, 57, 57, 58, 59, 60, 62, 63, 64, 64, 56, 57, 57, 59, 61, 60, 60, 60, + 60, 60, 61, 62, 63, 64, 65, 65, 57, 56, 56, 58, 60, 61, 61, 62, 62, 62, + 63, 64, 65, 66, 67, 67, 58, 56, 54, 57, 60, 61, 63, 63, 64, 65, 65, 66, + 66, 67, 68, 68, 59, 57, 55, 57, 60, 62, 63, 64, 65, 66, 67, 67, 68, 69, + 69, 69, 60, 57, 55, 57, 60, 62, 64, 65, 67, 67, 68, 69, 69, 70, 71, 71, + 61, 58, 56, 58, 60, 62, 65, 66, 67, 68, 69, 70, 71, 71, 72, 72, 62, 60, + 57, 59, 61, 63, 65, 67, 68, 69, 70, 71, 72, 72, 73, 73, 64, 61, 59, 60, + 62, 64, 66, 67, 69, 70, 71, 72, 73, 73, 74, 74, 65, 63, 60, 62, 63, 65, + 66, 68, 69, 71, 72, 73, 73, 74, 75, 75, 67, 64, 62, 63, 64, 66, 67, 69, + 70, 71, 72, 73, 74, 75, 75, 75, 68, 66, 63, 64, 65, 67, 68, 69, 71, 72, + 73, 74, 75, 75, 76, 76, 68, 66, 63, 64, 65, 67, 68, 69, 71, 72, 73, 74, + 75, 75, 76, 76, + /* Size 32 */ + 46, 45, 44, 42, 41, 44, 47, 51, 56, 56, 57, 57, 58, 58, 58, 59, 59, 60, + 61, 61, 62, 63, 63, 64, 65, 65, 66, 67, 68, 68, 68, 68, 45, 45, 44, 44, + 43, 46, 49, 52, 56, 56, 56, 57, 57, 57, 57, 58, 58, 59, 59, 60, 61, 61, + 62, 63, 64, 64, 65, 66, 66, 66, 66, 66, 44, 44, 44, 45, 45, 48, 50, 53, + 56, 56, 56, 56, 56, 56, 56, 57, 57, 58, 58, 59, 59, 60, 61, 62, 62, 63, + 64, 65, 65, 65, 65, 65, 42, 44, 45, 46, 48, 50, 52, 54, 56, 56, 56, 55, + 55, 55, 55, 56, 56, 56, 57, 58, 58, 59, 60, 60, 61, 62, 63, 63, 64, 64, + 64, 64, 41, 43, 45, 48, 50, 52, 53, 55, 57, 56, 55, 55, 54, 54, 54, 55, + 55, 55, 56, 57, 57, 58, 59, 59, 60, 61, 62, 62, 63, 63, 63, 63, 44, 46, + 48, 50, 52, 53, 54, 56, 57, 57, 56, 56, 55, 56, 56, 56, 56, 56, 57, 57, + 58, 59, 59, 60, 61, 61, 62, 63, 64, 64, 64, 64, 47, 49, 50, 52, 53, 54, + 56, 57, 58, 58, 58, 57, 57, 57, 57, 57, 57, 58, 58, 58, 59, 60, 60, 61, + 61, 62, 63, 63, 64, 64, 64, 64, 51, 52, 53, 54, 55, 56, 57, 58, 59, 59, + 59, 58, 58, 58, 58, 58, 58, 59, 59, 59, 60, 60, 61, 62, 62, 63, 63, 64, + 65, 65, 65, 65, 56, 56, 56, 56, 57, 57, 58, 59, 60, 60, 60, 60, 60, 60, + 60, 60, 60, 60, 60, 60, 61, 61, 62, 62, 63, 63, 64, 65, 65, 65, 65, 65, + 56, 56, 56, 56, 56, 57, 58, 59, 60, 60, 60, 60, 60, 60, 60, 60, 61, 61, + 61, 61, 62, 62, 63, 63, 64, 64, 65, 65, 66, 66, 66, 66, 57, 56, 56, 56, + 55, 56, 58, 59, 60, 60, 61, 61, 61, 61, 61, 61, 62, 62, 62, 62, 63, 63, + 64, 64, 64, 65, 65, 66, 66, 66, 66, 66, 57, 57, 56, 55, 55, 56, 57, 58, + 60, 60, 61, 61, 62, 62, 62, 62, 63, 63, 63, 63, 64, 64, 64, 65, 65, 66, + 66, 67, 67, 67, 67, 67, 58, 57, 56, 55, 54, 55, 57, 58, 60, 60, 61, 62, + 63, 63, 63, 63, 64, 64, 64, 64, 65, 65, 65, 66, 66, 66, 67, 67, 68, 68, + 68, 68, 58, 57, 56, 55, 54, 56, 57, 58, 60, 60, 61, 62, 63, 63, 64, 64, + 64, 65, 65, 65, 65, 66, 66, 66, 67, 67, 68, 68, 68, 68, 68, 68, 58, 57, + 56, 55, 54, 56, 57, 58, 60, 60, 61, 62, 63, 64, 64, 64, 65, 65, 66, 66, + 66, 67, 67, 67, 68, 68, 68, 69, 69, 69, 69, 69, 59, 58, 57, 56, 55, 56, + 57, 58, 60, 60, 61, 62, 63, 64, 64, 65, 66, 66, 66, 67, 67, 67, 68, 68, + 68, 69, 69, 69, 70, 70, 70, 70, 59, 58, 57, 56, 55, 56, 57, 58, 60, 61, + 62, 63, 64, 64, 65, 66, 66, 67, 67, 67, 68, 68, 68, 69, 69, 69, 70, 70, + 70, 70, 70, 70, 60, 59, 58, 56, 55, 56, 58, 59, 60, 61, 62, 63, 64, 65, + 65, 66, 67, 67, 67, 68, 68, 69, 69, 69, 70, 70, 70, 71, 71, 71, 71, 71, + 61, 59, 58, 57, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 66, 67, 67, + 68, 68, 69, 69, 69, 70, 70, 70, 71, 71, 71, 71, 71, 71, 61, 60, 59, 58, + 57, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 67, 68, 68, 69, 69, 70, + 70, 70, 71, 71, 71, 72, 72, 72, 72, 72, 62, 61, 59, 58, 57, 58, 59, 60, + 61, 62, 63, 64, 65, 65, 66, 67, 68, 68, 69, 69, 70, 70, 71, 71, 71, 72, + 72, 72, 72, 72, 72, 72, 63, 61, 60, 59, 58, 59, 60, 60, 61, 62, 63, 64, + 65, 66, 67, 67, 68, 69, 69, 70, 70, 71, 71, 71, 72, 72, 72, 73, 73, 73, + 73, 73, 63, 62, 61, 60, 59, 59, 60, 61, 62, 63, 64, 64, 65, 66, 67, 68, + 68, 69, 69, 70, 71, 71, 71, 72, 72, 72, 73, 73, 73, 73, 73, 73, 64, 63, + 62, 60, 59, 60, 61, 62, 62, 63, 64, 65, 66, 66, 67, 68, 69, 69, 70, 70, + 71, 71, 72, 72, 73, 73, 73, 73, 74, 74, 74, 74, 65, 64, 62, 61, 60, 61, + 61, 62, 63, 64, 64, 65, 66, 67, 68, 68, 69, 70, 70, 71, 71, 72, 72, 73, + 73, 73, 74, 74, 74, 74, 74, 74, 65, 64, 63, 62, 61, 61, 62, 63, 63, 64, + 65, 66, 66, 67, 68, 69, 69, 70, 70, 71, 72, 72, 72, 73, 73, 74, 74, 74, + 74, 74, 74, 74, 66, 65, 64, 63, 62, 62, 63, 63, 64, 65, 65, 66, 67, 68, + 68, 69, 70, 70, 71, 71, 72, 72, 73, 73, 74, 74, 74, 74, 75, 75, 75, 75, + 67, 66, 65, 63, 62, 63, 63, 64, 65, 65, 66, 67, 67, 68, 69, 69, 70, 71, + 71, 72, 72, 73, 73, 73, 74, 74, 74, 75, 75, 75, 75, 75, 68, 66, 65, 64, + 63, 64, 64, 65, 65, 66, 66, 67, 68, 68, 69, 70, 70, 71, 71, 72, 72, 73, + 73, 74, 74, 74, 75, 75, 75, 75, 75, 75, 68, 66, 65, 64, 63, 64, 64, 65, + 65, 66, 66, 67, 68, 68, 69, 70, 70, 71, 71, 72, 72, 73, 73, 74, 74, 74, + 75, 75, 75, 75, 75, 75, 68, 66, 65, 64, 63, 64, 64, 65, 65, 66, 66, 67, + 68, 68, 69, 70, 70, 71, 71, 72, 72, 73, 73, 74, 74, 74, 75, 75, 75, 75, + 75, 75, 68, 66, 65, 64, 63, 64, 64, 65, 65, 66, 66, 67, 68, 68, 69, 70, + 70, 71, 71, 72, 72, 73, 73, 74, 74, 74, 75, 75, 75, 75, 75, 75 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 67, 80, 89, 67, 77, 85, 91, 80, 85, 92, 95, 89, 91, 95, 97, + /* Size 8 */ + 64, 57, 59, 66, 73, 78, 82, 85, 57, 60, 59, 63, 69, 75, 80, 83, 59, 59, + 68, 71, 75, 78, 82, 84, 66, 63, 71, 77, 80, 82, 84, 86, 73, 69, 75, 80, + 83, 85, 86, 88, 78, 75, 78, 82, 85, 87, 88, 89, 82, 80, 82, 84, 86, 88, + 89, 90, 85, 83, 84, 86, 88, 89, 90, 91, + /* Size 16 */ + 64, 60, 57, 58, 59, 62, 66, 69, 73, 75, 78, 80, 82, 84, 85, 85, 60, 59, + 58, 59, 59, 62, 64, 68, 71, 74, 77, 79, 81, 83, 84, 84, 57, 58, 60, 60, + 59, 61, 63, 66, 69, 72, 75, 77, 80, 81, 83, 83, 58, 59, 60, 61, 63, 65, + 67, 69, 72, 74, 77, 79, 81, 82, 84, 84, 59, 59, 59, 63, 68, 69, 71, 73, + 75, 77, 78, 80, 82, 83, 84, 84, 62, 62, 61, 65, 69, 72, 74, 75, 77, 79, + 80, 82, 83, 84, 85, 85, 66, 64, 63, 67, 71, 74, 77, 78, 80, 81, 82, 83, + 84, 85, 86, 86, 69, 68, 66, 69, 73, 75, 78, 80, 81, 82, 83, 84, 85, 86, + 87, 87, 73, 71, 69, 72, 75, 77, 80, 81, 83, 84, 85, 86, 86, 87, 88, 88, + 75, 74, 72, 74, 77, 79, 81, 82, 84, 85, 86, 86, 87, 88, 88, 88, 78, 77, + 75, 77, 78, 80, 82, 83, 85, 86, 87, 87, 88, 88, 89, 89, 80, 79, 77, 79, + 80, 82, 83, 84, 86, 86, 87, 88, 89, 89, 89, 89, 82, 81, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 89, 89, 90, 90, 84, 83, 81, 82, 83, 84, 85, 86, + 87, 88, 88, 89, 89, 90, 90, 90, 85, 84, 83, 84, 84, 85, 86, 87, 88, 88, + 89, 89, 90, 90, 91, 91, 85, 84, 83, 84, 84, 85, 86, 87, 88, 88, 89, 89, + 90, 90, 91, 91, + /* Size 32 */ + 64, 62, 60, 58, 57, 57, 58, 58, 59, 60, 62, 64, 66, 67, 69, 71, 73, 74, + 75, 77, 78, 79, 80, 81, 82, 83, 84, 85, 85, 85, 85, 85, 62, 61, 60, 59, + 57, 58, 58, 58, 59, 60, 62, 63, 65, 67, 68, 70, 72, 73, 75, 76, 77, 78, + 79, 81, 82, 82, 83, 84, 85, 85, 85, 85, 60, 60, 59, 59, 58, 58, 59, 59, + 59, 60, 62, 63, 64, 66, 68, 69, 71, 72, 74, 75, 77, 78, 79, 80, 81, 82, + 83, 83, 84, 84, 84, 84, 58, 59, 59, 59, 59, 59, 59, 59, 59, 60, 61, 63, + 64, 65, 67, 68, 70, 71, 73, 74, 76, 77, 78, 79, 80, 81, 82, 83, 84, 84, + 84, 84, 57, 57, 58, 59, 60, 60, 60, 59, 59, 60, 61, 62, 63, 65, 66, 68, + 69, 71, 72, 73, 75, 76, 77, 78, 80, 80, 81, 82, 83, 83, 83, 83, 57, 58, + 58, 59, 60, 60, 60, 61, 61, 62, 63, 64, 65, 66, 68, 69, 71, 72, 73, 74, + 76, 77, 78, 79, 80, 81, 82, 83, 83, 83, 83, 83, 58, 58, 59, 59, 60, 60, + 61, 62, 63, 64, 65, 66, 67, 68, 69, 71, 72, 73, 74, 75, 77, 78, 79, 80, + 81, 81, 82, 83, 84, 84, 84, 84, 58, 58, 59, 59, 59, 61, 62, 64, 65, 66, + 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 78, 78, 79, 80, 81, 82, 83, 83, + 84, 84, 84, 84, 59, 59, 59, 59, 59, 61, 63, 65, 68, 69, 69, 70, 71, 72, + 73, 74, 75, 76, 77, 77, 78, 79, 80, 81, 82, 82, 83, 84, 84, 84, 84, 84, + 60, 60, 60, 60, 60, 62, 64, 66, 69, 69, 70, 71, 73, 73, 74, 75, 76, 77, + 78, 78, 79, 80, 81, 82, 82, 83, 84, 84, 85, 85, 85, 85, 62, 62, 62, 61, + 61, 63, 65, 67, 69, 70, 72, 73, 74, 75, 75, 76, 77, 78, 79, 79, 80, 81, + 82, 82, 83, 84, 84, 85, 85, 85, 85, 85, 64, 63, 63, 63, 62, 64, 66, 68, + 70, 71, 73, 74, 75, 76, 77, 78, 78, 79, 80, 80, 81, 82, 82, 83, 84, 84, + 85, 85, 86, 86, 86, 86, 66, 65, 64, 64, 63, 65, 67, 69, 71, 73, 74, 75, + 77, 77, 78, 79, 80, 80, 81, 81, 82, 83, 83, 84, 84, 85, 85, 86, 86, 86, + 86, 86, 67, 67, 66, 65, 65, 66, 68, 70, 72, 73, 75, 76, 77, 78, 79, 80, + 80, 81, 82, 82, 83, 83, 84, 84, 85, 85, 86, 86, 86, 86, 86, 86, 69, 68, + 68, 67, 66, 68, 69, 71, 73, 74, 75, 77, 78, 79, 80, 80, 81, 82, 82, 83, + 83, 84, 84, 85, 85, 86, 86, 86, 87, 87, 87, 87, 71, 70, 69, 68, 68, 69, + 71, 72, 74, 75, 76, 78, 79, 80, 80, 81, 82, 83, 83, 84, 84, 85, 85, 85, + 86, 86, 87, 87, 87, 87, 87, 87, 73, 72, 71, 70, 69, 71, 72, 73, 75, 76, + 77, 78, 80, 80, 81, 82, 83, 83, 84, 84, 85, 85, 86, 86, 86, 87, 87, 87, + 88, 88, 88, 88, 74, 73, 72, 71, 71, 72, 73, 74, 76, 77, 78, 79, 80, 81, + 82, 83, 83, 84, 84, 85, 85, 86, 86, 86, 87, 87, 87, 88, 88, 88, 88, 88, + 75, 75, 74, 73, 72, 73, 74, 75, 77, 78, 79, 80, 81, 82, 82, 83, 84, 84, + 85, 85, 86, 86, 86, 87, 87, 87, 88, 88, 88, 88, 88, 88, 77, 76, 75, 74, + 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 84, 85, 85, 86, 86, 87, + 87, 87, 88, 88, 88, 88, 89, 89, 89, 89, 78, 77, 77, 76, 75, 76, 77, 78, + 78, 79, 80, 81, 82, 83, 83, 84, 85, 85, 86, 86, 87, 87, 87, 88, 88, 88, + 88, 89, 89, 89, 89, 89, 79, 78, 78, 77, 76, 77, 78, 78, 79, 80, 81, 82, + 83, 83, 84, 85, 85, 86, 86, 87, 87, 87, 88, 88, 88, 88, 89, 89, 89, 89, + 89, 89, 80, 79, 79, 78, 77, 78, 79, 79, 80, 81, 82, 82, 83, 84, 84, 85, + 86, 86, 86, 87, 87, 88, 88, 88, 89, 89, 89, 89, 89, 89, 89, 89, 81, 81, + 80, 79, 78, 79, 80, 80, 81, 82, 82, 83, 84, 84, 85, 85, 86, 86, 87, 87, + 88, 88, 88, 88, 89, 89, 89, 89, 90, 90, 90, 90, 82, 82, 81, 80, 80, 80, + 81, 81, 82, 82, 83, 84, 84, 85, 85, 86, 86, 87, 87, 88, 88, 88, 89, 89, + 89, 89, 89, 90, 90, 90, 90, 90, 83, 82, 82, 81, 80, 81, 81, 82, 82, 83, + 84, 84, 85, 85, 86, 86, 87, 87, 87, 88, 88, 88, 89, 89, 89, 89, 90, 90, + 90, 90, 90, 90, 84, 83, 83, 82, 81, 82, 82, 83, 83, 84, 84, 85, 85, 86, + 86, 87, 87, 87, 88, 88, 88, 89, 89, 89, 89, 90, 90, 90, 90, 90, 90, 90, + 85, 84, 83, 83, 82, 83, 83, 83, 84, 84, 85, 85, 86, 86, 86, 87, 87, 88, + 88, 88, 89, 89, 89, 89, 90, 90, 90, 90, 90, 90, 90, 90, 85, 85, 84, 84, + 83, 83, 84, 84, 84, 85, 85, 86, 86, 86, 87, 87, 88, 88, 88, 89, 89, 89, + 89, 90, 90, 90, 90, 90, 91, 91, 91, 91, 85, 85, 84, 84, 83, 83, 84, 84, + 84, 85, 85, 86, 86, 86, 87, 87, 88, 88, 88, 89, 89, 89, 89, 90, 90, 90, + 90, 90, 91, 91, 91, 91, 85, 85, 84, 84, 83, 83, 84, 84, 84, 85, 85, 86, + 86, 86, 87, 87, 88, 88, 88, 89, 89, 89, 89, 90, 90, 90, 90, 90, 91, 91, + 91, 91, 85, 85, 84, 84, 83, 83, 84, 84, 84, 85, 85, 86, 86, 86, 87, 87, + 88, 88, 88, 89, 89, 89, 89, 90, 90, 90, 90, 90, 91, 91, 91, 91 }, + { /* Intra matrices */ + /* Size 4 */ + 48, 50, 60, 68, 50, 58, 64, 69, 60, 64, 70, 72, 68, 69, 72, 74, + /* Size 8 */ + 52, 45, 47, 53, 59, 64, 68, 70, 45, 48, 47, 51, 56, 61, 65, 68, 47, 47, + 55, 58, 61, 64, 67, 69, 53, 51, 58, 62, 65, 67, 69, 71, 59, 56, 61, 65, + 68, 70, 71, 72, 64, 61, 64, 67, 70, 71, 72, 73, 68, 65, 67, 69, 71, 72, + 73, 74, 70, 68, 69, 71, 72, 73, 74, 75, + /* Size 16 */ + 51, 48, 45, 46, 47, 49, 52, 55, 58, 61, 63, 65, 67, 68, 69, 69, 48, 47, + 46, 46, 47, 49, 51, 54, 57, 59, 62, 63, 65, 67, 68, 68, 45, 46, 48, 47, + 47, 48, 50, 53, 55, 58, 60, 62, 64, 66, 67, 67, 46, 46, 47, 49, 50, 52, + 53, 55, 58, 60, 62, 63, 65, 66, 68, 68, 47, 47, 47, 50, 54, 56, 57, 59, + 60, 62, 63, 65, 66, 67, 68, 68, 49, 49, 48, 52, 56, 57, 59, 61, 62, 63, + 65, 66, 67, 68, 69, 69, 52, 51, 50, 53, 57, 59, 62, 63, 64, 65, 66, 67, + 68, 69, 70, 70, 55, 54, 53, 55, 59, 61, 63, 64, 66, 67, 68, 68, 69, 70, + 71, 71, 58, 57, 55, 58, 60, 62, 64, 66, 67, 68, 69, 69, 70, 71, 71, 71, + 61, 59, 58, 60, 62, 63, 65, 67, 68, 69, 70, 70, 71, 71, 72, 72, 63, 62, + 60, 62, 63, 65, 66, 68, 69, 70, 70, 71, 71, 72, 72, 72, 65, 63, 62, 63, + 65, 66, 67, 68, 69, 70, 71, 71, 72, 72, 73, 73, 67, 65, 64, 65, 66, 67, + 68, 69, 70, 71, 71, 72, 72, 73, 73, 73, 68, 67, 66, 66, 67, 68, 69, 70, + 71, 71, 72, 72, 73, 73, 73, 73, 69, 68, 67, 68, 68, 69, 70, 71, 71, 72, + 72, 73, 73, 73, 74, 74, 69, 68, 67, 68, 68, 69, 70, 71, 71, 72, 72, 73, + 73, 73, 74, 74, + /* Size 32 */ + 51, 49, 47, 46, 45, 45, 45, 46, 46, 48, 49, 50, 52, 53, 55, 56, 58, 59, + 60, 61, 63, 64, 64, 65, 66, 67, 68, 68, 69, 69, 69, 69, 49, 48, 47, 46, + 45, 46, 46, 46, 46, 48, 49, 50, 52, 53, 54, 56, 57, 58, 59, 61, 62, 63, + 64, 65, 66, 66, 67, 68, 68, 68, 68, 68, 47, 47, 47, 46, 46, 46, 46, 46, + 46, 47, 49, 50, 51, 52, 54, 55, 56, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 66, 67, 68, 68, 68, 68, 46, 46, 46, 47, 47, 47, 47, 47, 46, 47, 48, 49, + 50, 52, 53, 54, 56, 57, 58, 59, 61, 61, 62, 63, 64, 65, 66, 67, 67, 67, + 67, 67, 45, 45, 46, 47, 47, 47, 47, 47, 47, 47, 48, 49, 50, 51, 52, 54, + 55, 56, 57, 59, 60, 61, 62, 63, 64, 65, 65, 66, 67, 67, 67, 67, 45, 46, + 46, 47, 47, 47, 48, 48, 48, 49, 50, 51, 51, 53, 54, 55, 56, 57, 58, 59, + 61, 61, 62, 63, 64, 65, 66, 66, 67, 67, 67, 67, 45, 46, 46, 47, 47, 48, + 48, 49, 50, 51, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 65, 66, 67, 67, 67, 67, 67, 46, 46, 46, 47, 47, 48, 49, 50, 52, 52, + 53, 54, 55, 56, 57, 58, 58, 59, 60, 61, 62, 63, 64, 64, 65, 66, 66, 67, + 68, 68, 68, 68, 46, 46, 46, 46, 47, 48, 50, 52, 54, 54, 55, 56, 57, 57, + 58, 59, 60, 60, 61, 62, 63, 64, 64, 65, 66, 66, 67, 67, 68, 68, 68, 68, + 48, 48, 47, 47, 47, 49, 51, 52, 54, 55, 56, 57, 58, 59, 59, 60, 61, 61, + 62, 63, 64, 64, 65, 66, 66, 67, 67, 68, 68, 68, 68, 68, 49, 49, 49, 48, + 48, 50, 51, 53, 55, 56, 57, 58, 59, 60, 60, 61, 62, 62, 63, 64, 64, 65, + 66, 66, 67, 67, 68, 68, 69, 69, 69, 69, 50, 50, 50, 49, 49, 51, 52, 54, + 56, 57, 58, 59, 60, 61, 61, 62, 63, 63, 64, 65, 65, 66, 66, 67, 67, 68, + 68, 69, 69, 69, 69, 69, 52, 52, 51, 50, 50, 51, 53, 55, 57, 58, 59, 60, + 61, 62, 63, 63, 64, 64, 65, 65, 66, 66, 67, 67, 68, 68, 69, 69, 69, 69, + 69, 69, 53, 53, 52, 52, 51, 53, 54, 56, 57, 59, 60, 61, 62, 63, 63, 64, + 65, 65, 66, 66, 67, 67, 67, 68, 68, 69, 69, 69, 70, 70, 70, 70, 55, 54, + 54, 53, 52, 54, 55, 57, 58, 59, 60, 61, 63, 63, 64, 65, 65, 66, 66, 67, + 67, 68, 68, 68, 69, 69, 69, 70, 70, 70, 70, 70, 56, 56, 55, 54, 54, 55, + 56, 58, 59, 60, 61, 62, 63, 64, 65, 65, 66, 66, 67, 67, 68, 68, 68, 69, + 69, 70, 70, 70, 70, 70, 70, 70, 58, 57, 56, 56, 55, 56, 57, 58, 60, 61, + 62, 63, 64, 65, 65, 66, 67, 67, 67, 68, 68, 69, 69, 69, 70, 70, 70, 71, + 71, 71, 71, 71, 59, 58, 58, 57, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, + 66, 66, 67, 67, 68, 68, 69, 69, 69, 70, 70, 70, 71, 71, 71, 71, 71, 71, + 60, 59, 59, 58, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 66, 67, 67, 68, + 68, 69, 69, 69, 70, 70, 70, 71, 71, 71, 71, 71, 71, 71, 61, 61, 60, 59, + 59, 59, 60, 61, 62, 63, 64, 65, 65, 66, 67, 67, 68, 68, 69, 69, 70, 70, + 70, 70, 71, 71, 71, 71, 72, 72, 72, 72, 63, 62, 61, 61, 60, 61, 61, 62, + 63, 64, 64, 65, 66, 67, 67, 68, 68, 69, 69, 70, 70, 70, 70, 71, 71, 71, + 71, 72, 72, 72, 72, 72, 64, 63, 62, 61, 61, 61, 62, 63, 64, 64, 65, 66, + 66, 67, 68, 68, 69, 69, 69, 70, 70, 70, 71, 71, 71, 71, 72, 72, 72, 72, + 72, 72, 64, 64, 63, 62, 62, 62, 63, 64, 64, 65, 66, 66, 67, 67, 68, 68, + 69, 69, 70, 70, 70, 71, 71, 71, 72, 72, 72, 72, 72, 72, 72, 72, 65, 65, + 64, 63, 63, 63, 64, 64, 65, 66, 66, 67, 67, 68, 68, 69, 69, 70, 70, 70, + 71, 71, 71, 72, 72, 72, 72, 72, 72, 72, 72, 72, 66, 66, 65, 64, 64, 64, + 65, 65, 66, 66, 67, 67, 68, 68, 69, 69, 70, 70, 70, 71, 71, 71, 72, 72, + 72, 72, 72, 73, 73, 73, 73, 73, 67, 66, 66, 65, 65, 65, 65, 66, 66, 67, + 67, 68, 68, 69, 69, 70, 70, 70, 71, 71, 71, 71, 72, 72, 72, 72, 73, 73, + 73, 73, 73, 73, 68, 67, 66, 66, 65, 66, 66, 66, 67, 67, 68, 68, 69, 69, + 69, 70, 70, 71, 71, 71, 71, 72, 72, 72, 72, 73, 73, 73, 73, 73, 73, 73, + 68, 68, 67, 67, 66, 66, 67, 67, 67, 68, 68, 69, 69, 69, 70, 70, 71, 71, + 71, 71, 72, 72, 72, 72, 73, 73, 73, 73, 73, 73, 73, 73, 69, 68, 68, 67, + 67, 67, 67, 68, 68, 68, 69, 69, 69, 70, 70, 70, 71, 71, 71, 72, 72, 72, + 72, 72, 73, 73, 73, 73, 73, 73, 73, 73, 69, 68, 68, 67, 67, 67, 67, 68, + 68, 68, 69, 69, 69, 70, 70, 70, 71, 71, 71, 72, 72, 72, 72, 72, 73, 73, + 73, 73, 73, 73, 73, 73, 69, 68, 68, 67, 67, 67, 67, 68, 68, 68, 69, 69, + 69, 70, 70, 70, 71, 71, 71, 72, 72, 72, 72, 72, 73, 73, 73, 73, 73, 73, + 73, 73, 69, 68, 68, 67, 67, 67, 67, 68, 68, 68, 69, 69, 69, 70, 70, 70, + 71, 71, 71, 72, 72, 72, 72, 72, 73, 73, 73, 73, 73, 73, 73, 73 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 77, 79, 85, 77, 81, 83, 86, 79, 83, 89, 91, 85, 86, 91, 94, + /* Size 8 */ + 64, 58, 74, 76, 77, 80, 83, 85, 58, 68, 75, 72, 73, 75, 78, 81, 74, 75, + 78, 78, 78, 79, 81, 83, 76, 72, 78, 80, 82, 83, 84, 85, 77, 73, 78, 82, + 84, 85, 87, 88, 80, 75, 79, 83, 85, 87, 89, 90, 83, 78, 81, 84, 87, 89, + 90, 91, 85, 81, 83, 85, 88, 90, 91, 92, + /* Size 16 */ + 64, 61, 58, 65, 74, 75, 76, 76, 77, 78, 80, 81, 83, 84, 85, 85, 61, 62, + 63, 68, 74, 74, 74, 74, 75, 76, 77, 79, 80, 82, 83, 83, 58, 63, 68, 71, + 75, 73, 72, 72, 73, 74, 75, 77, 78, 80, 81, 81, 65, 68, 71, 74, 76, 76, + 75, 75, 75, 76, 77, 78, 79, 81, 82, 82, 74, 74, 75, 76, 78, 78, 78, 78, + 78, 78, 79, 80, 81, 82, 83, 83, 75, 74, 73, 76, 78, 79, 79, 79, 79, 80, + 81, 81, 82, 83, 84, 84, 76, 74, 72, 75, 78, 79, 80, 81, 82, 82, 83, 83, + 84, 85, 85, 85, 76, 74, 72, 75, 78, 79, 81, 82, 83, 83, 84, 85, 85, 86, + 86, 86, 77, 75, 73, 75, 78, 79, 82, 83, 84, 85, 85, 86, 87, 87, 88, 88, + 78, 76, 74, 76, 78, 80, 82, 83, 85, 85, 86, 87, 88, 88, 89, 89, 80, 77, + 75, 77, 79, 81, 83, 84, 85, 86, 87, 88, 89, 89, 90, 90, 81, 79, 77, 78, + 80, 81, 83, 85, 86, 87, 88, 89, 89, 90, 90, 90, 83, 80, 78, 79, 81, 82, + 84, 85, 87, 88, 89, 89, 90, 91, 91, 91, 84, 82, 80, 81, 82, 83, 85, 86, + 87, 88, 89, 90, 91, 91, 92, 92, 85, 83, 81, 82, 83, 84, 85, 86, 88, 89, + 90, 90, 91, 92, 92, 92, 85, 83, 81, 82, 83, 84, 85, 86, 88, 89, 90, 90, + 91, 92, 92, 92, + /* Size 32 */ + 64, 62, 61, 59, 58, 61, 65, 69, 74, 74, 75, 75, 76, 76, 76, 77, 77, 78, + 78, 79, 80, 80, 81, 82, 83, 83, 84, 85, 85, 85, 85, 85, 62, 62, 61, 61, + 60, 63, 67, 70, 74, 74, 74, 75, 75, 75, 75, 76, 76, 77, 77, 78, 79, 79, + 80, 81, 81, 82, 83, 83, 84, 84, 84, 84, 61, 61, 62, 62, 63, 65, 68, 71, + 74, 74, 74, 74, 74, 74, 74, 75, 75, 76, 76, 77, 77, 78, 79, 80, 80, 81, + 82, 82, 83, 83, 83, 83, 59, 61, 62, 64, 65, 67, 70, 72, 74, 74, 74, 73, + 73, 73, 73, 74, 74, 74, 75, 76, 76, 77, 78, 78, 79, 80, 81, 81, 82, 82, + 82, 82, 58, 60, 63, 65, 68, 70, 71, 73, 75, 74, 73, 73, 72, 72, 72, 73, + 73, 73, 74, 75, 75, 76, 77, 77, 78, 79, 80, 80, 81, 81, 81, 81, 61, 63, + 65, 67, 70, 71, 72, 74, 76, 75, 74, 74, 73, 74, 74, 74, 74, 74, 75, 76, + 76, 77, 77, 78, 79, 79, 80, 81, 81, 81, 81, 81, 65, 67, 68, 70, 71, 72, + 74, 75, 76, 76, 76, 75, 75, 75, 75, 75, 75, 76, 76, 76, 77, 78, 78, 79, + 79, 80, 81, 81, 82, 82, 82, 82, 69, 70, 71, 72, 73, 74, 75, 76, 77, 77, + 77, 76, 76, 76, 76, 76, 76, 77, 77, 77, 78, 78, 79, 79, 80, 81, 81, 82, + 82, 82, 82, 82, 74, 74, 74, 74, 75, 76, 76, 77, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 79, 79, 80, 80, 81, 81, 82, 82, 83, 83, 83, 83, + 74, 74, 74, 74, 74, 75, 76, 77, 78, 78, 78, 78, 78, 78, 78, 78, 78, 79, + 79, 79, 80, 80, 81, 81, 81, 82, 82, 83, 83, 83, 83, 83, 75, 74, 74, 74, + 73, 74, 76, 77, 78, 78, 79, 79, 79, 79, 79, 79, 79, 80, 80, 80, 81, 81, + 81, 82, 82, 83, 83, 84, 84, 84, 84, 84, 75, 75, 74, 73, 73, 74, 75, 76, + 78, 78, 79, 79, 80, 80, 80, 80, 81, 81, 81, 81, 82, 82, 82, 83, 83, 83, + 84, 84, 85, 85, 85, 85, 76, 75, 74, 73, 72, 73, 75, 76, 78, 78, 79, 80, + 80, 81, 81, 81, 82, 82, 82, 82, 83, 83, 83, 83, 84, 84, 85, 85, 85, 85, + 85, 85, 76, 75, 74, 73, 72, 74, 75, 76, 78, 78, 79, 80, 81, 81, 81, 82, + 82, 82, 83, 83, 83, 84, 84, 84, 84, 85, 85, 86, 86, 86, 86, 86, 76, 75, + 74, 73, 72, 74, 75, 76, 78, 78, 79, 80, 81, 81, 82, 82, 83, 83, 83, 84, + 84, 84, 85, 85, 85, 85, 86, 86, 86, 86, 86, 86, 77, 76, 75, 74, 73, 74, + 75, 76, 78, 78, 79, 80, 81, 82, 82, 83, 83, 84, 84, 84, 85, 85, 85, 86, + 86, 86, 86, 87, 87, 87, 87, 87, 77, 76, 75, 74, 73, 74, 75, 76, 78, 78, + 79, 81, 82, 82, 83, 83, 84, 84, 85, 85, 85, 86, 86, 86, 87, 87, 87, 87, + 88, 88, 88, 88, 78, 77, 76, 74, 73, 74, 76, 77, 78, 79, 80, 81, 82, 82, + 83, 84, 84, 85, 85, 85, 86, 86, 86, 87, 87, 87, 88, 88, 88, 88, 88, 88, + 78, 77, 76, 75, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 83, 84, 85, 85, + 85, 86, 86, 87, 87, 87, 88, 88, 88, 88, 89, 89, 89, 89, 79, 78, 77, 76, + 75, 76, 76, 77, 78, 79, 80, 81, 82, 83, 84, 84, 85, 85, 86, 86, 87, 87, + 87, 88, 88, 88, 89, 89, 89, 89, 89, 89, 80, 79, 77, 76, 75, 76, 77, 78, + 79, 80, 81, 82, 83, 83, 84, 85, 85, 86, 86, 87, 87, 88, 88, 88, 89, 89, + 89, 89, 90, 90, 90, 90, 80, 79, 78, 77, 76, 77, 78, 78, 79, 80, 81, 82, + 83, 84, 84, 85, 86, 86, 87, 87, 88, 88, 88, 89, 89, 89, 89, 90, 90, 90, + 90, 90, 81, 80, 79, 78, 77, 77, 78, 79, 80, 81, 81, 82, 83, 84, 85, 85, + 86, 86, 87, 87, 88, 88, 89, 89, 89, 90, 90, 90, 90, 90, 90, 90, 82, 81, + 80, 78, 77, 78, 79, 79, 80, 81, 82, 83, 83, 84, 85, 86, 86, 87, 87, 88, + 88, 89, 89, 89, 90, 90, 90, 90, 91, 91, 91, 91, 83, 81, 80, 79, 78, 79, + 79, 80, 81, 81, 82, 83, 84, 84, 85, 86, 87, 87, 88, 88, 89, 89, 89, 90, + 90, 90, 91, 91, 91, 91, 91, 91, 83, 82, 81, 80, 79, 79, 80, 81, 81, 82, + 83, 83, 84, 85, 85, 86, 87, 87, 88, 88, 89, 89, 90, 90, 90, 91, 91, 91, + 91, 91, 91, 91, 84, 83, 82, 81, 80, 80, 81, 81, 82, 82, 83, 84, 85, 85, + 86, 86, 87, 88, 88, 89, 89, 89, 90, 90, 91, 91, 91, 91, 92, 92, 92, 92, + 85, 83, 82, 81, 80, 81, 81, 82, 82, 83, 84, 84, 85, 86, 86, 87, 87, 88, + 88, 89, 89, 90, 90, 90, 91, 91, 91, 92, 92, 92, 92, 92, 85, 84, 83, 82, + 81, 81, 82, 82, 83, 83, 84, 85, 85, 86, 86, 87, 88, 88, 89, 89, 90, 90, + 90, 91, 91, 91, 92, 92, 92, 92, 92, 92, 85, 84, 83, 82, 81, 81, 82, 82, + 83, 83, 84, 85, 85, 86, 86, 87, 88, 88, 89, 89, 90, 90, 90, 91, 91, 91, + 92, 92, 92, 92, 92, 92, 85, 84, 83, 82, 81, 81, 82, 82, 83, 83, 84, 85, + 85, 86, 86, 87, 88, 88, 89, 89, 90, 90, 90, 91, 91, 91, 92, 92, 92, 92, + 92, 92, 85, 84, 83, 82, 81, 81, 82, 82, 83, 83, 84, 85, 85, 86, 86, 87, + 88, 88, 89, 89, 90, 90, 90, 91, 91, 91, 92, 92, 92, 92, 92, 92 }, + { /* Intra matrices */ + /* Size 4 */ + 48, 59, 60, 65, 59, 62, 64, 66, 60, 64, 68, 70, 65, 66, 70, 73, + /* Size 8 */ + 50, 45, 59, 60, 61, 63, 66, 68, 45, 54, 59, 57, 57, 59, 62, 64, 59, 59, + 62, 61, 61, 62, 64, 66, 60, 57, 61, 64, 65, 66, 67, 68, 61, 57, 61, 65, + 67, 68, 69, 70, 63, 59, 62, 66, 68, 70, 71, 72, 66, 62, 64, 67, 69, 71, + 72, 73, 68, 64, 66, 68, 70, 72, 73, 74, + /* Size 16 */ + 50, 47, 45, 51, 58, 59, 59, 60, 61, 62, 63, 64, 65, 66, 67, 67, 47, 48, + 49, 53, 58, 58, 58, 58, 59, 60, 61, 62, 63, 64, 65, 65, 45, 49, 53, 56, + 58, 57, 56, 57, 57, 58, 59, 60, 61, 63, 64, 64, 51, 53, 56, 58, 60, 59, + 59, 59, 59, 60, 60, 61, 62, 63, 65, 65, 58, 58, 58, 60, 62, 61, 61, 61, + 61, 61, 62, 63, 63, 64, 65, 65, 59, 58, 57, 59, 61, 62, 62, 62, 62, 63, + 63, 64, 65, 66, 66, 66, 59, 58, 56, 59, 61, 62, 63, 64, 64, 65, 65, 66, + 66, 67, 67, 67, 60, 58, 57, 59, 61, 62, 64, 64, 65, 66, 66, 67, 67, 68, + 68, 68, 61, 59, 57, 59, 61, 62, 64, 65, 66, 67, 67, 68, 68, 69, 69, 69, + 62, 60, 58, 60, 61, 63, 65, 66, 67, 68, 68, 69, 69, 70, 70, 70, 63, 61, + 59, 60, 62, 63, 65, 66, 67, 68, 69, 70, 70, 71, 71, 71, 64, 62, 60, 61, + 63, 64, 66, 67, 68, 69, 70, 70, 71, 71, 72, 72, 65, 63, 61, 62, 63, 65, + 66, 67, 68, 69, 70, 71, 71, 72, 72, 72, 66, 64, 63, 63, 64, 66, 67, 68, + 69, 70, 71, 71, 72, 72, 73, 73, 67, 65, 64, 65, 65, 66, 67, 68, 69, 70, + 71, 72, 72, 73, 73, 73, 67, 65, 64, 65, 65, 66, 67, 68, 69, 70, 71, 72, + 72, 73, 73, 73, + /* Size 32 */ + 49, 48, 47, 46, 45, 47, 50, 54, 58, 58, 58, 59, 59, 59, 60, 60, 60, 61, + 61, 62, 62, 63, 64, 64, 65, 65, 66, 66, 67, 67, 67, 67, 48, 48, 47, 47, + 46, 49, 52, 55, 58, 58, 58, 58, 58, 58, 59, 59, 59, 60, 60, 61, 61, 62, + 63, 63, 64, 64, 65, 65, 66, 66, 66, 66, 47, 47, 48, 48, 48, 50, 53, 55, + 58, 58, 58, 58, 57, 58, 58, 58, 58, 59, 59, 60, 61, 61, 62, 62, 63, 63, + 64, 65, 65, 65, 65, 65, 46, 47, 48, 49, 51, 52, 54, 56, 58, 58, 57, 57, + 57, 57, 57, 57, 58, 58, 59, 59, 60, 60, 61, 61, 62, 63, 63, 64, 64, 64, + 64, 64, 45, 46, 48, 51, 53, 54, 55, 57, 58, 58, 57, 57, 56, 56, 56, 57, + 57, 57, 58, 58, 59, 59, 60, 60, 61, 62, 62, 63, 63, 63, 63, 63, 47, 49, + 50, 52, 54, 55, 56, 58, 59, 58, 58, 58, 57, 57, 57, 57, 58, 58, 58, 59, + 59, 60, 60, 61, 62, 62, 63, 63, 64, 64, 64, 64, 50, 52, 53, 54, 55, 56, + 57, 59, 60, 59, 59, 59, 58, 58, 58, 58, 59, 59, 59, 60, 60, 61, 61, 62, + 62, 63, 63, 64, 64, 64, 64, 64, 54, 55, 55, 56, 57, 58, 59, 60, 60, 60, + 60, 60, 59, 59, 59, 60, 60, 60, 60, 60, 61, 61, 62, 62, 63, 63, 64, 64, + 65, 65, 65, 65, 58, 58, 58, 58, 58, 59, 60, 60, 61, 61, 61, 61, 61, 61, + 61, 61, 61, 61, 61, 61, 62, 62, 62, 63, 63, 64, 64, 65, 65, 65, 65, 65, + 58, 58, 58, 58, 58, 58, 59, 60, 61, 61, 61, 61, 61, 61, 61, 61, 61, 62, + 62, 62, 62, 63, 63, 63, 64, 64, 65, 65, 66, 66, 66, 66, 58, 58, 58, 57, + 57, 58, 59, 60, 61, 61, 61, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, + 64, 64, 64, 65, 65, 66, 66, 66, 66, 66, 59, 58, 58, 57, 57, 58, 59, 60, + 61, 61, 62, 62, 62, 63, 63, 63, 63, 63, 63, 64, 64, 64, 65, 65, 65, 65, + 66, 66, 67, 67, 67, 67, 59, 58, 57, 57, 56, 57, 58, 59, 61, 61, 62, 62, + 63, 63, 63, 64, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, + 67, 67, 59, 58, 58, 57, 56, 57, 58, 59, 61, 61, 62, 63, 63, 64, 64, 64, + 64, 65, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 68, 68, 68, 68, 60, 59, + 58, 57, 56, 57, 58, 59, 61, 61, 62, 63, 63, 64, 64, 65, 65, 65, 65, 66, + 66, 66, 66, 67, 67, 67, 67, 68, 68, 68, 68, 68, 60, 59, 58, 57, 57, 57, + 58, 60, 61, 61, 62, 63, 64, 64, 65, 65, 65, 66, 66, 66, 67, 67, 67, 67, + 68, 68, 68, 68, 69, 69, 69, 69, 60, 59, 58, 58, 57, 58, 59, 60, 61, 61, + 62, 63, 64, 64, 65, 65, 66, 66, 67, 67, 67, 67, 68, 68, 68, 68, 69, 69, + 69, 69, 69, 69, 61, 60, 59, 58, 57, 58, 59, 60, 61, 62, 62, 63, 64, 65, + 65, 66, 66, 67, 67, 67, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, + 61, 60, 59, 59, 58, 58, 59, 60, 61, 62, 63, 63, 64, 65, 65, 66, 67, 67, + 67, 68, 68, 68, 68, 69, 69, 69, 69, 70, 70, 70, 70, 70, 62, 61, 60, 59, + 58, 59, 60, 60, 61, 62, 63, 64, 65, 65, 66, 66, 67, 67, 68, 68, 68, 69, + 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 62, 61, 61, 60, 59, 59, 60, 61, + 62, 62, 63, 64, 65, 65, 66, 67, 67, 68, 68, 68, 69, 69, 69, 70, 70, 70, + 70, 70, 71, 71, 71, 71, 63, 62, 61, 60, 59, 60, 61, 61, 62, 63, 63, 64, + 65, 66, 66, 67, 67, 68, 68, 69, 69, 69, 70, 70, 70, 70, 71, 71, 71, 71, + 71, 71, 64, 63, 62, 61, 60, 60, 61, 62, 62, 63, 64, 65, 65, 66, 66, 67, + 68, 68, 68, 69, 69, 70, 70, 70, 70, 71, 71, 71, 71, 71, 71, 71, 64, 63, + 62, 61, 60, 61, 62, 62, 63, 63, 64, 65, 66, 66, 67, 67, 68, 68, 69, 69, + 70, 70, 70, 70, 71, 71, 71, 71, 72, 72, 72, 72, 65, 64, 63, 62, 61, 62, + 62, 63, 63, 64, 64, 65, 66, 66, 67, 68, 68, 69, 69, 69, 70, 70, 70, 71, + 71, 71, 71, 72, 72, 72, 72, 72, 65, 64, 63, 63, 62, 62, 63, 63, 64, 64, + 65, 65, 66, 67, 67, 68, 68, 69, 69, 70, 70, 70, 71, 71, 71, 71, 72, 72, + 72, 72, 72, 72, 66, 65, 64, 63, 62, 63, 63, 64, 64, 65, 65, 66, 66, 67, + 67, 68, 69, 69, 69, 70, 70, 71, 71, 71, 71, 72, 72, 72, 72, 72, 72, 72, + 66, 65, 65, 64, 63, 63, 64, 64, 65, 65, 66, 66, 67, 67, 68, 68, 69, 69, + 70, 70, 70, 71, 71, 71, 72, 72, 72, 72, 73, 73, 73, 73, 67, 66, 65, 64, + 63, 64, 64, 65, 65, 66, 66, 67, 67, 68, 68, 69, 69, 69, 70, 70, 71, 71, + 71, 72, 72, 72, 72, 73, 73, 73, 73, 73, 67, 66, 65, 64, 63, 64, 64, 65, + 65, 66, 66, 67, 67, 68, 68, 69, 69, 69, 70, 70, 71, 71, 71, 72, 72, 72, + 72, 73, 73, 73, 73, 73, 67, 66, 65, 64, 63, 64, 64, 65, 65, 66, 66, 67, + 67, 68, 68, 69, 69, 69, 70, 70, 71, 71, 71, 72, 72, 72, 72, 73, 73, 73, + 73, 73, 67, 66, 65, 64, 63, 64, 64, 65, 65, 66, 66, 67, 67, 68, 68, 69, + 69, 69, 70, 70, 71, 71, 71, 72, 72, 72, 72, 73, 73, 73, 73, 73 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 66, 76, 82, 66, 74, 79, 83, 76, 79, 84, 86, 82, 83, 86, 87, + /* Size 8 */ + 64, 58, 60, 65, 71, 75, 77, 79, 58, 61, 60, 63, 68, 72, 75, 78, 60, 60, + 67, 69, 72, 75, 77, 79, 65, 63, 69, 73, 75, 77, 79, 80, 71, 68, 72, 75, + 78, 79, 80, 81, 75, 72, 75, 77, 79, 80, 81, 82, 77, 75, 77, 79, 80, 81, + 82, 82, 79, 78, 79, 80, 81, 82, 82, 83, + /* Size 16 */ + 64, 61, 58, 59, 60, 62, 65, 68, 71, 72, 75, 76, 77, 78, 79, 79, 61, 60, + 60, 60, 60, 62, 64, 67, 69, 71, 73, 75, 76, 77, 79, 79, 58, 60, 61, 61, + 60, 62, 63, 66, 68, 70, 72, 74, 75, 77, 78, 78, 59, 60, 61, 62, 63, 65, + 66, 68, 70, 72, 73, 75, 76, 77, 78, 78, 60, 60, 60, 63, 67, 68, 69, 71, + 72, 73, 75, 76, 77, 78, 79, 79, 62, 62, 62, 65, 68, 70, 71, 73, 74, 75, + 76, 77, 78, 79, 79, 79, 65, 64, 63, 66, 69, 71, 73, 74, 75, 76, 77, 78, + 79, 79, 80, 80, 68, 67, 66, 68, 71, 73, 74, 75, 77, 77, 78, 79, 79, 80, + 80, 80, 71, 69, 68, 70, 72, 74, 75, 77, 78, 78, 79, 80, 80, 80, 81, 81, + 72, 71, 70, 72, 73, 75, 76, 77, 78, 79, 80, 80, 81, 81, 81, 81, 75, 73, + 72, 73, 75, 76, 77, 78, 79, 80, 80, 81, 81, 81, 82, 82, 76, 75, 74, 75, + 76, 77, 78, 79, 80, 80, 81, 81, 81, 82, 82, 82, 77, 76, 75, 76, 77, 78, + 79, 79, 80, 81, 81, 81, 82, 82, 82, 82, 78, 77, 77, 77, 78, 79, 79, 80, + 80, 81, 81, 82, 82, 82, 83, 83, 79, 79, 78, 78, 79, 79, 80, 80, 81, 81, + 82, 82, 82, 83, 83, 83, 79, 79, 78, 78, 79, 79, 80, 80, 81, 81, 82, 82, + 82, 83, 83, 83, + /* Size 32 */ + 64, 62, 61, 60, 58, 59, 59, 59, 60, 61, 62, 64, 65, 67, 68, 69, 71, 71, + 72, 73, 75, 75, 76, 77, 77, 78, 78, 79, 79, 79, 79, 79, 62, 61, 61, 60, + 59, 59, 59, 60, 60, 61, 62, 64, 65, 66, 67, 69, 70, 71, 72, 73, 74, 75, + 75, 76, 77, 77, 78, 78, 79, 79, 79, 79, 61, 61, 60, 60, 60, 60, 60, 60, + 60, 61, 62, 63, 64, 65, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 76, 77, + 77, 78, 79, 79, 79, 79, 60, 60, 60, 60, 60, 60, 60, 60, 60, 61, 62, 63, + 64, 65, 66, 67, 69, 70, 71, 72, 73, 73, 74, 75, 76, 76, 77, 78, 78, 78, + 78, 78, 58, 59, 60, 60, 61, 61, 61, 60, 60, 61, 62, 63, 63, 64, 66, 67, + 68, 69, 70, 71, 72, 73, 74, 75, 75, 76, 77, 77, 78, 78, 78, 78, 59, 59, + 60, 60, 61, 61, 61, 61, 62, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, + 73, 73, 74, 75, 76, 76, 77, 77, 78, 78, 78, 78, 59, 59, 60, 60, 61, 61, + 62, 63, 63, 64, 65, 66, 66, 67, 68, 69, 70, 71, 72, 72, 73, 74, 75, 75, + 76, 77, 77, 78, 78, 78, 78, 78, 59, 60, 60, 60, 60, 61, 63, 64, 65, 66, + 66, 67, 68, 69, 69, 70, 71, 72, 72, 73, 74, 75, 75, 76, 77, 77, 77, 78, + 78, 78, 78, 78, 60, 60, 60, 60, 60, 62, 63, 65, 67, 67, 68, 69, 69, 70, + 71, 71, 72, 73, 73, 74, 75, 75, 76, 76, 77, 77, 78, 78, 79, 79, 79, 79, + 61, 61, 61, 61, 61, 62, 64, 66, 67, 68, 69, 70, 70, 71, 72, 72, 73, 73, + 74, 75, 75, 76, 76, 77, 77, 78, 78, 79, 79, 79, 79, 79, 62, 62, 62, 62, + 62, 63, 65, 66, 68, 69, 70, 71, 71, 72, 73, 73, 74, 74, 75, 75, 76, 76, + 77, 77, 78, 78, 79, 79, 79, 79, 79, 79, 64, 64, 63, 63, 63, 64, 66, 67, + 69, 70, 71, 71, 72, 73, 73, 74, 75, 75, 76, 76, 76, 77, 77, 78, 78, 79, + 79, 79, 80, 80, 80, 80, 65, 65, 64, 64, 63, 65, 66, 68, 69, 70, 71, 72, + 73, 74, 74, 75, 75, 76, 76, 77, 77, 78, 78, 78, 79, 79, 79, 80, 80, 80, + 80, 80, 67, 66, 65, 65, 64, 66, 67, 69, 70, 71, 72, 73, 74, 74, 75, 75, + 76, 76, 77, 77, 78, 78, 78, 79, 79, 79, 80, 80, 80, 80, 80, 80, 68, 67, + 67, 66, 66, 67, 68, 69, 71, 72, 73, 73, 74, 75, 75, 76, 77, 77, 77, 78, + 78, 78, 79, 79, 79, 80, 80, 80, 80, 80, 80, 80, 69, 69, 68, 67, 67, 68, + 69, 70, 71, 72, 73, 74, 75, 75, 76, 77, 77, 77, 78, 78, 79, 79, 79, 79, + 80, 80, 80, 80, 81, 81, 81, 81, 71, 70, 69, 69, 68, 69, 70, 71, 72, 73, + 74, 75, 75, 76, 77, 77, 78, 78, 78, 79, 79, 79, 80, 80, 80, 80, 80, 81, + 81, 81, 81, 81, 71, 71, 70, 70, 69, 70, 71, 72, 73, 73, 74, 75, 76, 76, + 77, 77, 78, 78, 79, 79, 79, 80, 80, 80, 80, 80, 81, 81, 81, 81, 81, 81, + 72, 72, 71, 71, 70, 71, 72, 72, 73, 74, 75, 76, 76, 77, 77, 78, 78, 79, + 79, 79, 80, 80, 80, 80, 81, 81, 81, 81, 81, 81, 81, 81, 73, 73, 72, 72, + 71, 72, 72, 73, 74, 75, 75, 76, 77, 77, 78, 78, 79, 79, 79, 80, 80, 80, + 80, 81, 81, 81, 81, 81, 82, 82, 82, 82, 75, 74, 73, 73, 72, 73, 73, 74, + 75, 75, 76, 76, 77, 78, 78, 79, 79, 79, 80, 80, 80, 80, 81, 81, 81, 81, + 81, 82, 82, 82, 82, 82, 75, 75, 74, 73, 73, 73, 74, 75, 75, 76, 76, 77, + 78, 78, 78, 79, 79, 80, 80, 80, 80, 81, 81, 81, 81, 81, 82, 82, 82, 82, + 82, 82, 76, 75, 75, 74, 74, 74, 75, 75, 76, 76, 77, 77, 78, 78, 79, 79, + 80, 80, 80, 80, 81, 81, 81, 81, 81, 82, 82, 82, 82, 82, 82, 82, 77, 76, + 76, 75, 75, 75, 75, 76, 76, 77, 77, 78, 78, 79, 79, 79, 80, 80, 80, 81, + 81, 81, 81, 81, 82, 82, 82, 82, 82, 82, 82, 82, 77, 77, 76, 76, 75, 76, + 76, 77, 77, 77, 78, 78, 79, 79, 79, 80, 80, 80, 81, 81, 81, 81, 81, 82, + 82, 82, 82, 82, 82, 82, 82, 82, 78, 77, 77, 76, 76, 76, 77, 77, 77, 78, + 78, 79, 79, 79, 80, 80, 80, 80, 81, 81, 81, 81, 82, 82, 82, 82, 82, 82, + 82, 82, 82, 82, 78, 78, 77, 77, 77, 77, 77, 77, 78, 78, 79, 79, 79, 80, + 80, 80, 80, 81, 81, 81, 81, 82, 82, 82, 82, 82, 82, 82, 83, 83, 83, 83, + 79, 78, 78, 78, 77, 77, 78, 78, 78, 79, 79, 79, 80, 80, 80, 80, 81, 81, + 81, 81, 82, 82, 82, 82, 82, 82, 82, 83, 83, 83, 83, 83, 79, 79, 79, 78, + 78, 78, 78, 78, 79, 79, 79, 80, 80, 80, 80, 81, 81, 81, 81, 82, 82, 82, + 82, 82, 82, 82, 83, 83, 83, 83, 83, 83, 79, 79, 79, 78, 78, 78, 78, 78, + 79, 79, 79, 80, 80, 80, 80, 81, 81, 81, 81, 82, 82, 82, 82, 82, 82, 82, + 83, 83, 83, 83, 83, 83, 79, 79, 79, 78, 78, 78, 78, 78, 79, 79, 79, 80, + 80, 80, 80, 81, 81, 81, 81, 82, 82, 82, 82, 82, 82, 82, 83, 83, 83, 83, + 83, 83, 79, 79, 79, 78, 78, 78, 78, 78, 79, 79, 79, 80, 80, 80, 80, 81, + 81, 81, 81, 82, 82, 82, 82, 82, 82, 82, 83, 83, 83, 83, 83, 83 }, + { /* Intra matrices */ + /* Size 4 */ + 51, 53, 61, 67, 53, 60, 65, 68, 61, 65, 68, 70, 67, 68, 70, 71, + /* Size 8 */ + 55, 49, 51, 56, 61, 64, 67, 69, 49, 52, 51, 54, 58, 62, 65, 67, 51, 51, + 57, 60, 62, 64, 66, 68, 56, 54, 60, 63, 65, 67, 68, 69, 61, 58, 62, 65, + 67, 68, 69, 70, 64, 62, 64, 67, 68, 70, 70, 71, 67, 65, 66, 68, 69, 70, + 71, 71, 69, 67, 68, 69, 70, 71, 71, 72, + /* Size 16 */ + 54, 51, 49, 50, 50, 53, 55, 58, 60, 62, 64, 65, 66, 67, 68, 68, 51, 51, + 50, 50, 51, 52, 54, 57, 59, 61, 63, 64, 65, 66, 67, 67, 49, 50, 51, 51, + 51, 52, 54, 56, 58, 60, 61, 63, 64, 65, 67, 67, 50, 50, 51, 52, 53, 55, + 56, 58, 59, 61, 63, 64, 65, 66, 67, 67, 50, 51, 51, 53, 57, 58, 59, 60, + 61, 63, 64, 65, 66, 67, 67, 67, 53, 52, 52, 55, 58, 59, 61, 62, 63, 64, + 65, 66, 67, 67, 68, 68, 55, 54, 54, 56, 59, 61, 63, 63, 64, 65, 66, 67, + 67, 68, 69, 69, 58, 57, 56, 58, 60, 62, 63, 64, 65, 66, 67, 67, 68, 68, + 69, 69, 60, 59, 58, 59, 61, 63, 64, 65, 66, 67, 68, 68, 69, 69, 69, 69, + 62, 61, 60, 61, 63, 64, 65, 66, 67, 68, 68, 69, 69, 69, 70, 70, 64, 63, + 61, 63, 64, 65, 66, 67, 68, 68, 69, 69, 70, 70, 70, 70, 65, 64, 63, 64, + 65, 66, 67, 67, 68, 69, 69, 70, 70, 70, 70, 70, 66, 65, 64, 65, 66, 67, + 67, 68, 69, 69, 70, 70, 70, 71, 71, 71, 67, 66, 65, 66, 67, 67, 68, 68, + 69, 69, 70, 70, 71, 71, 71, 71, 68, 67, 67, 67, 67, 68, 69, 69, 69, 70, + 70, 70, 71, 71, 71, 71, 68, 67, 67, 67, 67, 68, 69, 69, 69, 70, 70, 70, + 71, 71, 71, 71, + /* Size 32 */ + 54, 52, 51, 50, 49, 49, 49, 50, 50, 51, 53, 54, 55, 56, 57, 58, 60, 61, + 61, 62, 63, 64, 65, 65, 66, 66, 67, 67, 68, 68, 68, 68, 52, 52, 51, 50, + 49, 50, 50, 50, 50, 51, 52, 53, 55, 56, 57, 58, 59, 60, 61, 62, 63, 63, + 64, 65, 65, 66, 66, 67, 67, 67, 67, 67, 51, 51, 51, 50, 50, 50, 50, 50, + 50, 51, 52, 53, 54, 55, 56, 57, 59, 59, 60, 61, 62, 63, 64, 64, 65, 65, + 66, 67, 67, 67, 67, 67, 50, 50, 50, 50, 51, 51, 50, 50, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 62, 63, 64, 65, 65, 66, 66, 67, 67, + 67, 67, 49, 49, 50, 51, 51, 51, 51, 51, 50, 51, 52, 53, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 63, 64, 65, 65, 66, 66, 66, 66, 66, 49, 50, + 50, 51, 51, 51, 51, 52, 52, 52, 53, 54, 55, 55, 56, 57, 58, 59, 60, 61, + 62, 62, 63, 64, 64, 65, 65, 66, 67, 67, 67, 67, 49, 50, 50, 50, 51, 51, + 52, 53, 53, 54, 55, 55, 56, 57, 58, 58, 59, 60, 61, 61, 62, 63, 63, 64, + 65, 65, 66, 66, 67, 67, 67, 67, 50, 50, 50, 50, 51, 52, 53, 54, 55, 55, + 56, 57, 57, 58, 59, 59, 60, 61, 61, 62, 63, 63, 64, 65, 65, 66, 66, 66, + 67, 67, 67, 67, 50, 50, 50, 50, 50, 52, 53, 55, 56, 57, 58, 58, 59, 59, + 60, 60, 61, 62, 62, 63, 63, 64, 64, 65, 65, 66, 66, 67, 67, 67, 67, 67, + 51, 51, 51, 51, 51, 52, 54, 55, 57, 58, 58, 59, 60, 60, 61, 61, 62, 62, + 63, 63, 64, 64, 65, 65, 66, 66, 67, 67, 67, 67, 67, 67, 53, 52, 52, 52, + 52, 53, 55, 56, 58, 58, 59, 60, 60, 61, 62, 62, 63, 63, 64, 64, 65, 65, + 65, 66, 66, 67, 67, 67, 68, 68, 68, 68, 54, 53, 53, 53, 53, 54, 55, 57, + 58, 59, 60, 61, 61, 62, 62, 63, 63, 64, 64, 65, 65, 65, 66, 66, 67, 67, + 67, 68, 68, 68, 68, 68, 55, 55, 54, 54, 53, 55, 56, 57, 59, 60, 60, 61, + 62, 63, 63, 64, 64, 65, 65, 65, 66, 66, 66, 67, 67, 67, 68, 68, 68, 68, + 68, 68, 56, 56, 55, 55, 54, 55, 57, 58, 59, 60, 61, 62, 63, 63, 64, 64, + 65, 65, 65, 66, 66, 66, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 57, 57, + 56, 56, 55, 56, 58, 59, 60, 61, 62, 62, 63, 64, 64, 65, 65, 65, 66, 66, + 67, 67, 67, 67, 68, 68, 68, 68, 69, 69, 69, 69, 58, 58, 57, 57, 56, 57, + 58, 59, 60, 61, 62, 63, 64, 64, 65, 65, 66, 66, 66, 67, 67, 67, 67, 68, + 68, 68, 68, 69, 69, 69, 69, 69, 60, 59, 59, 58, 57, 58, 59, 60, 61, 62, + 63, 63, 64, 65, 65, 66, 66, 66, 67, 67, 67, 68, 68, 68, 68, 69, 69, 69, + 69, 69, 69, 69, 61, 60, 59, 59, 58, 59, 60, 61, 62, 62, 63, 64, 65, 65, + 65, 66, 66, 67, 67, 67, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, + 61, 61, 60, 60, 59, 60, 61, 61, 62, 63, 64, 64, 65, 65, 66, 66, 67, 67, + 67, 68, 68, 68, 68, 69, 69, 69, 69, 69, 70, 70, 70, 70, 62, 62, 61, 61, + 60, 61, 61, 62, 63, 63, 64, 65, 65, 66, 66, 67, 67, 67, 68, 68, 68, 68, + 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 63, 63, 62, 62, 61, 62, 62, 63, + 63, 64, 65, 65, 66, 66, 67, 67, 67, 68, 68, 68, 69, 69, 69, 69, 69, 69, + 70, 70, 70, 70, 70, 70, 64, 63, 63, 62, 62, 62, 63, 63, 64, 64, 65, 65, + 66, 66, 67, 67, 68, 68, 68, 68, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, + 70, 70, 65, 64, 64, 63, 63, 63, 63, 64, 64, 65, 65, 66, 66, 67, 67, 67, + 68, 68, 68, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 70, 65, 65, + 64, 64, 63, 64, 64, 65, 65, 65, 66, 66, 67, 67, 67, 68, 68, 68, 69, 69, + 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 70, 70, 66, 65, 65, 65, 64, 64, + 65, 65, 65, 66, 66, 67, 67, 67, 68, 68, 68, 69, 69, 69, 69, 69, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 66, 66, 65, 65, 65, 65, 65, 66, 66, 66, + 67, 67, 67, 68, 68, 68, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, + 71, 71, 71, 71, 67, 66, 66, 66, 65, 65, 66, 66, 66, 67, 67, 67, 68, 68, + 68, 68, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 71, 71, 71, 71, 71, + 67, 67, 67, 66, 66, 66, 66, 66, 67, 67, 67, 68, 68, 68, 68, 69, 69, 69, + 69, 70, 70, 70, 70, 70, 70, 70, 71, 71, 71, 71, 71, 71, 68, 67, 67, 67, + 66, 67, 67, 67, 67, 67, 68, 68, 68, 68, 69, 69, 69, 69, 70, 70, 70, 70, + 70, 70, 70, 71, 71, 71, 71, 71, 71, 71, 68, 67, 67, 67, 66, 67, 67, 67, + 67, 67, 68, 68, 68, 68, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 71, + 71, 71, 71, 71, 71, 71, 68, 67, 67, 67, 66, 67, 67, 67, 67, 67, 68, 68, + 68, 68, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 71, 71, 71, 71, 71, + 71, 71, 68, 67, 67, 67, 66, 67, 67, 67, 67, 67, 68, 68, 68, 68, 69, 69, + 69, 69, 70, 70, 70, 70, 70, 70, 70, 71, 71, 71, 71, 71, 71, 71 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 74, 75, 79, 74, 77, 78, 80, 75, 78, 82, 84, 79, 80, 84, 86, + /* Size 8 */ + 64, 59, 72, 73, 74, 76, 78, 80, 59, 67, 72, 70, 71, 72, 75, 77, 72, 72, + 75, 74, 74, 75, 76, 78, 73, 70, 74, 76, 77, 78, 79, 80, 74, 71, 74, 77, + 79, 80, 80, 81, 76, 72, 75, 78, 80, 81, 82, 82, 78, 75, 76, 79, 80, 82, + 83, 83, 80, 77, 78, 80, 81, 82, 83, 84, + /* Size 16 */ + 64, 62, 59, 65, 72, 72, 73, 73, 74, 75, 76, 77, 78, 79, 80, 80, 62, 62, + 63, 67, 72, 72, 71, 72, 72, 73, 74, 75, 76, 77, 78, 78, 59, 63, 67, 70, + 72, 71, 70, 70, 71, 72, 72, 74, 75, 76, 77, 77, 65, 67, 70, 71, 73, 73, + 72, 72, 72, 73, 74, 75, 75, 76, 77, 77, 72, 72, 72, 73, 75, 74, 74, 74, + 74, 75, 75, 76, 76, 77, 78, 78, 72, 72, 71, 73, 74, 75, 75, 75, 76, 76, + 76, 77, 77, 78, 79, 79, 73, 71, 70, 72, 74, 75, 76, 77, 77, 77, 78, 78, + 79, 79, 80, 80, 73, 72, 70, 72, 74, 75, 77, 77, 78, 78, 79, 79, 79, 80, + 80, 80, 74, 72, 71, 72, 74, 76, 77, 78, 79, 79, 80, 80, 80, 81, 81, 81, + 75, 73, 72, 73, 75, 76, 77, 78, 79, 80, 80, 81, 81, 81, 82, 82, 76, 74, + 72, 74, 75, 76, 78, 79, 80, 80, 81, 81, 82, 82, 82, 82, 77, 75, 74, 75, + 76, 77, 78, 79, 80, 81, 81, 82, 82, 83, 83, 83, 78, 76, 75, 75, 76, 77, + 79, 79, 80, 81, 82, 82, 83, 83, 83, 83, 79, 77, 76, 76, 77, 78, 79, 80, + 81, 81, 82, 83, 83, 83, 84, 84, 80, 78, 77, 77, 78, 79, 80, 80, 81, 82, + 82, 83, 83, 84, 84, 84, 80, 78, 77, 77, 78, 79, 80, 80, 81, 82, 82, 83, + 83, 84, 84, 84, + /* Size 32 */ + 64, 63, 62, 60, 59, 62, 65, 68, 72, 72, 72, 72, 73, 73, 73, 74, 74, 74, + 75, 75, 76, 76, 77, 77, 78, 78, 79, 79, 80, 80, 80, 80, 63, 62, 62, 61, + 61, 63, 66, 69, 72, 72, 72, 72, 72, 72, 73, 73, 73, 74, 74, 74, 75, 75, + 76, 76, 77, 77, 78, 78, 79, 79, 79, 79, 62, 62, 62, 63, 63, 65, 67, 69, + 72, 72, 72, 72, 71, 72, 72, 72, 72, 73, 73, 74, 74, 75, 75, 76, 76, 77, + 77, 78, 78, 78, 78, 78, 60, 61, 63, 64, 65, 67, 68, 70, 72, 72, 71, 71, + 71, 71, 71, 71, 71, 72, 72, 73, 73, 74, 74, 75, 75, 76, 76, 77, 77, 77, + 77, 77, 59, 61, 63, 65, 67, 68, 70, 71, 72, 72, 71, 71, 70, 70, 70, 71, + 71, 71, 72, 72, 72, 73, 74, 74, 75, 75, 76, 76, 77, 77, 77, 77, 62, 63, + 65, 67, 68, 69, 70, 72, 73, 72, 72, 72, 71, 71, 71, 71, 72, 72, 72, 73, + 73, 74, 74, 75, 75, 75, 76, 76, 77, 77, 77, 77, 65, 66, 67, 68, 70, 70, + 71, 72, 73, 73, 73, 72, 72, 72, 72, 72, 72, 73, 73, 73, 74, 74, 75, 75, + 75, 76, 76, 77, 77, 77, 77, 77, 68, 69, 69, 70, 71, 72, 72, 73, 74, 74, + 74, 73, 73, 73, 73, 73, 73, 74, 74, 74, 74, 75, 75, 76, 76, 76, 77, 77, + 78, 78, 78, 78, 72, 72, 72, 72, 72, 73, 73, 74, 75, 75, 74, 74, 74, 74, + 74, 74, 74, 74, 75, 75, 75, 75, 76, 76, 76, 77, 77, 78, 78, 78, 78, 78, + 72, 72, 72, 72, 72, 72, 73, 74, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, + 75, 75, 76, 76, 76, 77, 77, 77, 78, 78, 78, 78, 78, 78, 72, 72, 72, 71, + 71, 72, 73, 74, 74, 75, 75, 75, 75, 75, 75, 75, 76, 76, 76, 76, 76, 77, + 77, 77, 77, 78, 78, 78, 79, 79, 79, 79, 72, 72, 72, 71, 71, 72, 72, 73, + 74, 75, 75, 75, 76, 76, 76, 76, 76, 76, 77, 77, 77, 77, 77, 78, 78, 78, + 79, 79, 79, 79, 79, 79, 73, 72, 71, 71, 70, 71, 72, 73, 74, 75, 75, 76, + 76, 76, 77, 77, 77, 77, 77, 78, 78, 78, 78, 78, 79, 79, 79, 79, 80, 80, + 80, 80, 73, 72, 72, 71, 70, 71, 72, 73, 74, 75, 75, 76, 76, 77, 77, 77, + 77, 78, 78, 78, 78, 78, 79, 79, 79, 79, 80, 80, 80, 80, 80, 80, 73, 73, + 72, 71, 70, 71, 72, 73, 74, 75, 75, 76, 77, 77, 77, 78, 78, 78, 78, 78, + 79, 79, 79, 79, 79, 80, 80, 80, 80, 80, 80, 80, 74, 73, 72, 71, 71, 71, + 72, 73, 74, 75, 75, 76, 77, 77, 78, 78, 78, 78, 79, 79, 79, 79, 80, 80, + 80, 80, 80, 81, 81, 81, 81, 81, 74, 73, 72, 71, 71, 72, 72, 73, 74, 75, + 76, 76, 77, 77, 78, 78, 79, 79, 79, 79, 80, 80, 80, 80, 80, 81, 81, 81, + 81, 81, 81, 81, 74, 74, 73, 72, 71, 72, 73, 74, 74, 75, 76, 76, 77, 78, + 78, 78, 79, 79, 79, 80, 80, 80, 80, 81, 81, 81, 81, 81, 82, 82, 82, 82, + 75, 74, 73, 72, 72, 72, 73, 74, 75, 75, 76, 77, 77, 78, 78, 79, 79, 79, + 80, 80, 80, 80, 81, 81, 81, 81, 81, 82, 82, 82, 82, 82, 75, 74, 74, 73, + 72, 73, 73, 74, 75, 75, 76, 77, 78, 78, 78, 79, 79, 80, 80, 80, 81, 81, + 81, 81, 81, 82, 82, 82, 82, 82, 82, 82, 76, 75, 74, 73, 72, 73, 74, 74, + 75, 76, 76, 77, 78, 78, 79, 79, 80, 80, 80, 81, 81, 81, 81, 82, 82, 82, + 82, 82, 82, 82, 82, 82, 76, 75, 75, 74, 73, 74, 74, 75, 75, 76, 77, 77, + 78, 78, 79, 79, 80, 80, 80, 81, 81, 81, 82, 82, 82, 82, 82, 83, 83, 83, + 83, 83, 77, 76, 75, 74, 74, 74, 75, 75, 76, 76, 77, 77, 78, 79, 79, 80, + 80, 80, 81, 81, 81, 82, 82, 82, 82, 82, 83, 83, 83, 83, 83, 83, 77, 76, + 76, 75, 74, 75, 75, 76, 76, 77, 77, 78, 78, 79, 79, 80, 80, 81, 81, 81, + 82, 82, 82, 82, 83, 83, 83, 83, 83, 83, 83, 83, 78, 77, 76, 75, 75, 75, + 75, 76, 76, 77, 77, 78, 79, 79, 79, 80, 80, 81, 81, 81, 82, 82, 82, 83, + 83, 83, 83, 83, 83, 83, 83, 83, 78, 77, 77, 76, 75, 75, 76, 76, 77, 77, + 78, 78, 79, 79, 80, 80, 81, 81, 81, 82, 82, 82, 82, 83, 83, 83, 83, 83, + 84, 84, 84, 84, 79, 78, 77, 76, 76, 76, 76, 77, 77, 78, 78, 79, 79, 80, + 80, 80, 81, 81, 81, 82, 82, 82, 83, 83, 83, 83, 83, 84, 84, 84, 84, 84, + 79, 78, 78, 77, 76, 76, 77, 77, 78, 78, 78, 79, 79, 80, 80, 81, 81, 81, + 82, 82, 82, 83, 83, 83, 83, 83, 84, 84, 84, 84, 84, 84, 80, 79, 78, 77, + 77, 77, 77, 78, 78, 78, 79, 79, 80, 80, 80, 81, 81, 82, 82, 82, 82, 83, + 83, 83, 83, 84, 84, 84, 84, 84, 84, 84, 80, 79, 78, 77, 77, 77, 77, 78, + 78, 78, 79, 79, 80, 80, 80, 81, 81, 82, 82, 82, 82, 83, 83, 83, 83, 84, + 84, 84, 84, 84, 84, 84, 80, 79, 78, 77, 77, 77, 77, 78, 78, 78, 79, 79, + 80, 80, 80, 81, 81, 82, 82, 82, 82, 83, 83, 83, 83, 84, 84, 84, 84, 84, + 84, 84, 80, 79, 78, 77, 77, 77, 77, 78, 78, 78, 79, 79, 80, 80, 80, 81, + 81, 82, 82, 82, 82, 83, 83, 83, 83, 84, 84, 84, 84, 84, 84, 84 }, + { /* Intra matrices */ + /* Size 4 */ + 52, 60, 61, 65, 60, 63, 64, 66, 61, 64, 67, 69, 65, 66, 69, 71, + /* Size 8 */ + 53, 49, 60, 61, 62, 64, 65, 67, 49, 56, 60, 59, 59, 61, 63, 64, 60, 60, + 63, 62, 62, 63, 64, 66, 61, 59, 62, 64, 65, 65, 66, 67, 62, 59, 62, 65, + 66, 67, 68, 69, 64, 61, 63, 65, 67, 68, 69, 70, 65, 63, 64, 66, 68, 69, + 70, 71, 67, 64, 66, 67, 69, 70, 71, 71, + /* Size 16 */ + 53, 51, 49, 54, 60, 60, 61, 61, 62, 62, 63, 64, 65, 66, 67, 67, 51, 51, + 52, 56, 60, 60, 59, 60, 60, 61, 62, 63, 64, 64, 65, 65, 49, 52, 56, 58, + 60, 59, 58, 58, 59, 59, 60, 61, 62, 63, 64, 64, 54, 56, 58, 59, 61, 61, + 60, 60, 60, 61, 61, 62, 63, 64, 65, 65, 60, 60, 60, 61, 62, 62, 62, 62, + 62, 62, 63, 63, 64, 64, 65, 65, 60, 60, 59, 61, 62, 62, 63, 63, 63, 63, + 64, 64, 65, 65, 66, 66, 61, 59, 58, 60, 62, 63, 64, 64, 64, 65, 65, 65, + 66, 66, 67, 67, 61, 60, 58, 60, 62, 63, 64, 64, 65, 65, 66, 66, 67, 67, + 67, 67, 62, 60, 59, 60, 62, 63, 64, 65, 66, 66, 67, 67, 67, 68, 68, 68, + 62, 61, 59, 61, 62, 63, 65, 65, 66, 67, 67, 68, 68, 68, 69, 69, 63, 62, + 60, 61, 63, 64, 65, 66, 67, 67, 68, 68, 69, 69, 69, 69, 64, 63, 61, 62, + 63, 64, 65, 66, 67, 68, 68, 69, 69, 69, 70, 70, 65, 64, 62, 63, 64, 65, + 66, 67, 67, 68, 69, 69, 69, 70, 70, 70, 66, 64, 63, 64, 64, 65, 66, 67, + 68, 68, 69, 69, 70, 70, 70, 70, 67, 65, 64, 65, 65, 66, 67, 67, 68, 69, + 69, 70, 70, 70, 71, 71, 67, 65, 64, 65, 65, 66, 67, 67, 68, 69, 69, 70, + 70, 70, 71, 71, + /* Size 32 */ + 53, 52, 51, 50, 49, 51, 53, 56, 59, 60, 60, 60, 60, 61, 61, 61, 61, 62, + 62, 63, 63, 63, 64, 64, 65, 65, 66, 66, 66, 66, 66, 66, 52, 51, 51, 50, + 50, 52, 54, 57, 59, 60, 60, 60, 60, 60, 60, 60, 61, 61, 61, 62, 62, 63, + 63, 64, 64, 64, 65, 65, 66, 66, 66, 66, 51, 51, 51, 51, 52, 54, 55, 57, + 60, 59, 59, 59, 59, 59, 60, 60, 60, 60, 61, 61, 62, 62, 62, 63, 63, 64, + 64, 65, 65, 65, 65, 65, 50, 50, 51, 52, 54, 55, 56, 58, 60, 59, 59, 59, + 59, 59, 59, 59, 59, 60, 60, 60, 61, 61, 62, 62, 63, 63, 63, 64, 64, 64, + 64, 64, 49, 50, 52, 54, 55, 56, 58, 59, 60, 59, 59, 58, 58, 58, 58, 58, + 59, 59, 59, 60, 60, 61, 61, 61, 62, 62, 63, 63, 64, 64, 64, 64, 51, 52, + 54, 55, 56, 57, 58, 59, 60, 60, 60, 59, 59, 59, 59, 59, 59, 60, 60, 60, + 61, 61, 61, 62, 62, 63, 63, 64, 64, 64, 64, 64, 53, 54, 55, 56, 58, 58, + 59, 60, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 61, 61, 61, 62, 62, 62, + 63, 63, 64, 64, 64, 64, 64, 64, 56, 57, 57, 58, 59, 59, 60, 61, 61, 61, + 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 62, 62, 62, 63, 63, 63, 64, 64, + 65, 65, 65, 65, 59, 59, 60, 60, 60, 60, 61, 61, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 64, 64, 64, 65, 65, 65, 65, 65, + 60, 60, 59, 59, 59, 60, 61, 61, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, + 63, 63, 63, 63, 63, 64, 64, 64, 65, 65, 65, 65, 65, 65, 60, 60, 59, 59, + 59, 60, 60, 61, 62, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, 64, + 64, 64, 64, 65, 65, 65, 66, 66, 66, 66, 60, 60, 59, 59, 58, 59, 60, 61, + 62, 62, 62, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, + 65, 66, 66, 66, 66, 66, 60, 60, 59, 59, 58, 59, 60, 61, 62, 62, 62, 63, + 63, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, + 66, 66, 61, 60, 59, 59, 58, 59, 60, 61, 62, 62, 63, 63, 64, 64, 64, 64, + 64, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 61, 60, + 60, 59, 58, 59, 60, 61, 62, 62, 63, 63, 64, 64, 64, 65, 65, 65, 65, 65, + 66, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 61, 60, 60, 59, 58, 59, + 60, 61, 62, 62, 63, 63, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, 66, 67, + 67, 67, 67, 67, 67, 67, 67, 67, 61, 61, 60, 59, 59, 59, 60, 61, 62, 62, + 63, 63, 64, 64, 65, 65, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 68, + 68, 68, 68, 68, 62, 61, 60, 60, 59, 60, 60, 61, 62, 62, 63, 64, 64, 65, + 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, + 62, 61, 61, 60, 59, 60, 61, 61, 62, 63, 63, 64, 64, 65, 65, 66, 66, 66, + 66, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 63, 62, 61, 60, + 60, 60, 61, 61, 62, 63, 63, 64, 65, 65, 65, 66, 66, 66, 67, 67, 67, 67, + 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 63, 62, 62, 61, 60, 61, 61, 62, + 62, 63, 63, 64, 65, 65, 66, 66, 66, 67, 67, 67, 68, 68, 68, 68, 68, 68, + 69, 69, 69, 69, 69, 69, 63, 63, 62, 61, 61, 61, 62, 62, 63, 63, 64, 64, + 65, 65, 66, 66, 67, 67, 67, 67, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, + 69, 69, 64, 63, 62, 62, 61, 61, 62, 62, 63, 63, 64, 65, 65, 65, 66, 66, + 67, 67, 67, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 64, 64, + 63, 62, 61, 62, 62, 63, 63, 64, 64, 65, 65, 66, 66, 67, 67, 67, 68, 68, + 68, 68, 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, 65, 64, 63, 63, 62, 62, + 63, 63, 64, 64, 64, 65, 65, 66, 66, 67, 67, 67, 68, 68, 68, 69, 69, 69, + 69, 69, 70, 70, 70, 70, 70, 70, 65, 64, 64, 63, 62, 63, 63, 63, 64, 64, + 65, 65, 66, 66, 66, 67, 67, 68, 68, 68, 68, 69, 69, 69, 69, 70, 70, 70, + 70, 70, 70, 70, 66, 65, 64, 63, 63, 63, 64, 64, 64, 65, 65, 65, 66, 66, + 67, 67, 67, 68, 68, 68, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 70, + 66, 65, 65, 64, 63, 64, 64, 64, 65, 65, 65, 66, 66, 67, 67, 67, 68, 68, + 68, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 70, 66, 66, 65, 64, + 64, 64, 64, 65, 65, 65, 66, 66, 66, 67, 67, 67, 68, 68, 68, 69, 69, 69, + 69, 70, 70, 70, 70, 70, 70, 70, 70, 70, 66, 66, 65, 64, 64, 64, 64, 65, + 65, 65, 66, 66, 66, 67, 67, 67, 68, 68, 68, 69, 69, 69, 69, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 66, 66, 65, 64, 64, 64, 64, 65, 65, 65, 66, 66, + 66, 67, 67, 67, 68, 68, 68, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 66, 66, 65, 64, 64, 64, 64, 65, 65, 65, 66, 66, 66, 67, 67, 67, + 68, 68, 68, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 70, 70 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 65, 72, 76, 65, 71, 74, 76, 72, 74, 77, 78, 76, 76, 78, 79, + /* Size 8 */ + 64, 60, 61, 65, 68, 71, 73, 74, 60, 62, 61, 64, 67, 69, 71, 73, 61, 61, + 66, 68, 69, 71, 72, 73, 65, 64, 68, 70, 71, 72, 73, 74, 68, 67, 69, 71, + 73, 74, 74, 75, 71, 69, 71, 72, 74, 74, 75, 75, 73, 71, 72, 73, 74, 75, + 75, 76, 74, 73, 73, 74, 75, 75, 76, 76, + /* Size 16 */ + 64, 62, 60, 60, 61, 63, 65, 67, 68, 70, 71, 72, 73, 73, 74, 74, 62, 61, + 61, 61, 61, 63, 64, 66, 68, 69, 70, 71, 72, 73, 73, 73, 60, 61, 62, 62, + 61, 62, 64, 65, 67, 68, 69, 70, 71, 72, 73, 73, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 70, 71, 72, 73, 73, 73, 61, 61, 61, 64, 66, 67, 68, 68, + 69, 70, 71, 72, 72, 73, 73, 73, 63, 63, 62, 65, 67, 68, 69, 70, 70, 71, + 72, 72, 73, 73, 74, 74, 65, 64, 64, 66, 68, 69, 70, 71, 71, 72, 72, 73, + 73, 74, 74, 74, 67, 66, 65, 67, 68, 70, 71, 71, 72, 73, 73, 73, 74, 74, + 74, 74, 68, 68, 67, 68, 69, 70, 71, 72, 73, 73, 74, 74, 74, 74, 75, 75, + 70, 69, 68, 69, 70, 71, 72, 73, 73, 74, 74, 74, 75, 75, 75, 75, 71, 70, + 69, 70, 71, 72, 72, 73, 74, 74, 74, 75, 75, 75, 75, 75, 72, 71, 70, 71, + 72, 72, 73, 73, 74, 74, 75, 75, 75, 75, 75, 75, 73, 72, 71, 72, 72, 73, + 73, 74, 74, 75, 75, 75, 75, 75, 76, 76, 73, 73, 72, 73, 73, 73, 74, 74, + 74, 75, 75, 75, 75, 76, 76, 76, 74, 73, 73, 73, 73, 74, 74, 74, 75, 75, + 75, 75, 76, 76, 76, 76, 74, 73, 73, 73, 73, 74, 74, 74, 75, 75, 75, 75, + 76, 76, 76, 76, + /* Size 32 */ + 64, 63, 62, 61, 60, 60, 60, 61, 61, 62, 63, 64, 65, 66, 67, 67, 68, 69, + 70, 70, 71, 71, 72, 72, 73, 73, 73, 74, 74, 74, 74, 74, 63, 62, 62, 61, + 60, 61, 61, 61, 61, 62, 63, 64, 65, 65, 66, 67, 68, 69, 69, 70, 70, 71, + 71, 72, 72, 73, 73, 73, 74, 74, 74, 74, 62, 62, 61, 61, 61, 61, 61, 61, + 61, 62, 63, 63, 64, 65, 66, 67, 68, 68, 69, 69, 70, 71, 71, 72, 72, 72, + 73, 73, 73, 73, 73, 73, 61, 61, 61, 61, 61, 61, 61, 61, 61, 62, 63, 63, + 64, 65, 65, 66, 67, 68, 68, 69, 70, 70, 71, 71, 72, 72, 72, 73, 73, 73, + 73, 73, 60, 60, 61, 61, 62, 62, 62, 61, 61, 62, 62, 63, 64, 64, 65, 66, + 67, 67, 68, 69, 69, 70, 70, 71, 71, 72, 72, 73, 73, 73, 73, 73, 60, 61, + 61, 61, 62, 62, 62, 62, 62, 63, 63, 64, 65, 65, 66, 67, 67, 68, 69, 69, + 70, 70, 71, 71, 72, 72, 72, 73, 73, 73, 73, 73, 60, 61, 61, 61, 62, 62, + 63, 63, 64, 64, 65, 65, 66, 66, 67, 67, 68, 69, 69, 70, 70, 71, 71, 71, + 72, 72, 73, 73, 73, 73, 73, 73, 61, 61, 61, 61, 61, 62, 63, 64, 65, 65, + 66, 66, 67, 67, 68, 68, 69, 69, 70, 70, 71, 71, 71, 72, 72, 72, 73, 73, + 73, 73, 73, 73, 61, 61, 61, 61, 61, 62, 64, 65, 66, 66, 67, 67, 68, 68, + 68, 69, 69, 70, 70, 71, 71, 71, 72, 72, 72, 73, 73, 73, 73, 73, 73, 73, + 62, 62, 62, 62, 62, 63, 64, 65, 66, 67, 67, 68, 68, 69, 69, 69, 70, 70, + 71, 71, 71, 72, 72, 72, 73, 73, 73, 73, 74, 74, 74, 74, 63, 63, 63, 63, + 62, 63, 65, 66, 67, 67, 68, 68, 69, 69, 70, 70, 70, 71, 71, 71, 72, 72, + 72, 73, 73, 73, 73, 74, 74, 74, 74, 74, 64, 64, 63, 63, 63, 64, 65, 66, + 67, 68, 68, 69, 69, 70, 70, 71, 71, 71, 72, 72, 72, 72, 73, 73, 73, 73, + 74, 74, 74, 74, 74, 74, 65, 65, 64, 64, 64, 65, 66, 67, 68, 68, 69, 69, + 70, 70, 71, 71, 71, 72, 72, 72, 72, 73, 73, 73, 73, 74, 74, 74, 74, 74, + 74, 74, 66, 65, 65, 65, 64, 65, 66, 67, 68, 69, 69, 70, 70, 71, 71, 71, + 72, 72, 72, 73, 73, 73, 73, 73, 74, 74, 74, 74, 74, 74, 74, 74, 67, 66, + 66, 65, 65, 66, 67, 68, 68, 69, 70, 70, 71, 71, 71, 72, 72, 72, 73, 73, + 73, 73, 73, 74, 74, 74, 74, 74, 74, 74, 74, 74, 67, 67, 67, 66, 66, 67, + 67, 68, 69, 69, 70, 71, 71, 71, 72, 72, 72, 73, 73, 73, 73, 74, 74, 74, + 74, 74, 74, 74, 75, 75, 75, 75, 68, 68, 68, 67, 67, 67, 68, 69, 69, 70, + 70, 71, 71, 72, 72, 72, 73, 73, 73, 73, 74, 74, 74, 74, 74, 74, 74, 75, + 75, 75, 75, 75, 69, 69, 68, 68, 67, 68, 69, 69, 70, 70, 71, 71, 72, 72, + 72, 73, 73, 73, 73, 74, 74, 74, 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, + 70, 69, 69, 68, 68, 69, 69, 70, 70, 71, 71, 72, 72, 72, 73, 73, 73, 73, + 74, 74, 74, 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, 75, 70, 70, 69, 69, + 69, 69, 70, 70, 71, 71, 71, 72, 72, 73, 73, 73, 73, 74, 74, 74, 74, 74, + 74, 75, 75, 75, 75, 75, 75, 75, 75, 75, 71, 70, 70, 70, 69, 70, 70, 71, + 71, 71, 72, 72, 72, 73, 73, 73, 74, 74, 74, 74, 74, 74, 75, 75, 75, 75, + 75, 75, 75, 75, 75, 75, 71, 71, 71, 70, 70, 70, 71, 71, 71, 72, 72, 72, + 73, 73, 73, 74, 74, 74, 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, 75, 75, + 75, 75, 72, 71, 71, 71, 70, 71, 71, 71, 72, 72, 72, 73, 73, 73, 73, 74, + 74, 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 72, 72, + 72, 71, 71, 71, 71, 72, 72, 72, 73, 73, 73, 73, 74, 74, 74, 74, 74, 75, + 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 73, 72, 72, 72, 71, 72, + 72, 72, 72, 73, 73, 73, 73, 74, 74, 74, 74, 74, 75, 75, 75, 75, 75, 75, + 75, 75, 75, 75, 76, 76, 76, 76, 73, 73, 72, 72, 72, 72, 72, 72, 73, 73, + 73, 73, 74, 74, 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 76, + 76, 76, 76, 76, 73, 73, 73, 72, 72, 72, 73, 73, 73, 73, 73, 74, 74, 74, + 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, 75, 75, 76, 76, 76, 76, 76, 76, + 74, 73, 73, 73, 73, 73, 73, 73, 73, 73, 74, 74, 74, 74, 74, 74, 75, 75, + 75, 75, 75, 75, 75, 75, 75, 76, 76, 76, 76, 76, 76, 76, 74, 74, 73, 73, + 73, 73, 73, 73, 73, 74, 74, 74, 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, + 75, 75, 76, 76, 76, 76, 76, 76, 76, 76, 74, 74, 73, 73, 73, 73, 73, 73, + 73, 74, 74, 74, 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, 75, 75, 76, 76, + 76, 76, 76, 76, 76, 76, 74, 74, 73, 73, 73, 73, 73, 73, 73, 74, 74, 74, + 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, 75, 75, 76, 76, 76, 76, 76, 76, + 76, 76, 74, 74, 73, 73, 73, 73, 73, 73, 73, 74, 74, 74, 74, 74, 74, 75, + 75, 75, 75, 75, 75, 75, 75, 75, 76, 76, 76, 76, 76, 76, 76, 76 }, + { /* Intra matrices */ + /* Size 4 */ + 55, 57, 62, 66, 57, 61, 65, 67, 62, 65, 67, 68, 66, 67, 68, 69, + /* Size 8 */ + 58, 54, 55, 59, 62, 64, 66, 67, 54, 56, 55, 57, 60, 63, 65, 66, 55, 55, + 60, 61, 63, 64, 66, 67, 59, 57, 61, 64, 65, 66, 67, 68, 62, 60, 63, 65, + 66, 67, 68, 68, 64, 63, 64, 66, 67, 68, 68, 69, 66, 65, 66, 67, 68, 68, + 69, 69, 67, 66, 67, 68, 68, 69, 69, 69, + /* Size 16 */ + 57, 55, 53, 54, 55, 56, 58, 60, 62, 63, 64, 65, 66, 66, 67, 67, 55, 55, + 54, 55, 55, 56, 58, 59, 61, 62, 63, 64, 65, 66, 66, 66, 53, 54, 55, 55, + 55, 56, 57, 58, 60, 61, 63, 63, 64, 65, 66, 66, 54, 55, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 66, 66, 55, 55, 55, 57, 59, 60, 61, 62, + 62, 63, 64, 65, 65, 66, 66, 66, 56, 56, 56, 58, 60, 61, 62, 63, 63, 64, + 65, 65, 66, 66, 67, 67, 58, 58, 57, 59, 61, 62, 63, 64, 65, 65, 66, 66, + 66, 67, 67, 67, 60, 59, 58, 60, 62, 63, 64, 65, 65, 66, 66, 66, 67, 67, + 67, 67, 62, 61, 60, 61, 62, 63, 65, 65, 66, 66, 67, 67, 67, 67, 68, 68, + 63, 62, 61, 62, 63, 64, 65, 66, 66, 67, 67, 67, 67, 68, 68, 68, 64, 63, + 63, 63, 64, 65, 66, 66, 67, 67, 67, 68, 68, 68, 68, 68, 65, 64, 63, 64, + 65, 65, 66, 66, 67, 67, 68, 68, 68, 68, 68, 68, 66, 65, 64, 65, 65, 66, + 66, 67, 67, 67, 68, 68, 68, 68, 68, 68, 66, 66, 65, 66, 66, 66, 67, 67, + 67, 68, 68, 68, 68, 68, 69, 69, 67, 66, 66, 66, 66, 67, 67, 67, 68, 68, + 68, 68, 68, 69, 69, 69, 67, 66, 66, 66, 66, 67, 67, 67, 68, 68, 68, 68, + 68, 69, 69, 69, + /* Size 32 */ + 57, 56, 55, 54, 53, 54, 54, 54, 54, 55, 56, 57, 58, 59, 60, 60, 61, 62, + 63, 63, 64, 64, 65, 65, 65, 66, 66, 66, 67, 67, 67, 67, 56, 56, 55, 54, + 54, 54, 54, 54, 54, 55, 56, 57, 58, 58, 59, 60, 61, 62, 62, 63, 63, 64, + 64, 65, 65, 65, 66, 66, 66, 66, 66, 66, 55, 55, 55, 54, 54, 54, 54, 54, + 55, 55, 56, 57, 57, 58, 59, 60, 61, 61, 62, 62, 63, 63, 64, 64, 65, 65, + 65, 66, 66, 66, 66, 66, 54, 54, 54, 55, 55, 55, 55, 55, 55, 55, 56, 56, + 57, 58, 59, 59, 60, 61, 61, 62, 63, 63, 64, 64, 65, 65, 65, 66, 66, 66, + 66, 66, 53, 54, 54, 55, 55, 55, 55, 55, 55, 55, 56, 56, 57, 57, 58, 59, + 60, 60, 61, 62, 62, 63, 63, 64, 64, 65, 65, 65, 66, 66, 66, 66, 54, 54, + 54, 55, 55, 55, 55, 56, 56, 56, 57, 57, 58, 58, 59, 60, 60, 61, 61, 62, + 63, 63, 64, 64, 64, 65, 65, 65, 66, 66, 66, 66, 54, 54, 54, 55, 55, 55, + 56, 56, 57, 57, 58, 58, 59, 59, 60, 60, 61, 61, 62, 63, 63, 63, 64, 64, + 65, 65, 65, 66, 66, 66, 66, 66, 54, 54, 54, 55, 55, 56, 56, 57, 58, 58, + 59, 59, 60, 60, 61, 61, 62, 62, 62, 63, 63, 64, 64, 65, 65, 65, 65, 66, + 66, 66, 66, 66, 54, 54, 55, 55, 55, 56, 57, 58, 59, 59, 60, 60, 61, 61, + 61, 62, 62, 63, 63, 63, 64, 64, 64, 65, 65, 65, 66, 66, 66, 66, 66, 66, + 55, 55, 55, 55, 55, 56, 57, 58, 59, 60, 60, 61, 61, 62, 62, 62, 63, 63, + 63, 64, 64, 64, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 56, 56, 56, 56, + 56, 57, 58, 59, 60, 60, 61, 61, 62, 62, 63, 63, 63, 64, 64, 64, 65, 65, + 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 57, 57, 57, 56, 56, 57, 58, 59, + 60, 61, 61, 62, 62, 63, 63, 63, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, + 66, 66, 67, 67, 67, 67, 58, 58, 57, 57, 57, 58, 59, 60, 61, 61, 62, 62, + 63, 63, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 67, 67, 67, + 67, 67, 59, 58, 58, 58, 57, 58, 59, 60, 61, 62, 62, 63, 63, 64, 64, 64, + 65, 65, 65, 65, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 60, 59, + 59, 59, 58, 59, 60, 61, 61, 62, 63, 63, 64, 64, 64, 65, 65, 65, 65, 66, + 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 67, 60, 60, 60, 59, 59, 60, + 60, 61, 62, 62, 63, 63, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, 66, 67, + 67, 67, 67, 67, 67, 67, 67, 67, 61, 61, 61, 60, 60, 60, 61, 62, 62, 63, + 63, 64, 64, 65, 65, 65, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, + 67, 67, 67, 67, 62, 62, 61, 61, 60, 61, 61, 62, 63, 63, 64, 64, 65, 65, + 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, + 63, 62, 62, 61, 61, 61, 62, 62, 63, 63, 64, 64, 65, 65, 65, 66, 66, 66, + 66, 67, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 63, 63, 62, 62, + 62, 62, 63, 63, 63, 64, 64, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, + 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 64, 63, 63, 63, 62, 63, 63, 63, + 64, 64, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 68, 68, + 68, 68, 68, 68, 68, 68, 64, 64, 63, 63, 63, 63, 63, 64, 64, 64, 65, 65, + 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 65, 64, 64, 64, 63, 64, 64, 64, 64, 65, 65, 65, 66, 66, 66, 66, + 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 65, 65, + 64, 64, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, + 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 65, 65, 65, 65, 64, 64, + 65, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, + 68, 68, 68, 68, 68, 68, 68, 68, 66, 65, 65, 65, 65, 65, 65, 65, 65, 66, + 66, 66, 66, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 68, 68, 66, 66, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 67, + 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, + 66, 66, 66, 66, 65, 65, 66, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, + 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 67, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, + 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 67, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 68, 68, 68, 68, 67, 66, 66, 66, 66, 66, 66, 66, 66, 66, 67, 67, + 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 67, 66, 66, 66, 66, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, + 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 71, 72, 74, 71, 73, 73, 75, 72, 73, 76, 77, 74, 75, 77, 78, + /* Size 8 */ + 64, 61, 69, 70, 71, 72, 73, 74, 61, 66, 69, 68, 69, 70, 71, 72, 69, 69, + 71, 71, 71, 71, 72, 73, 70, 68, 71, 72, 73, 73, 74, 74, 71, 69, 71, 73, + 74, 74, 75, 75, 72, 70, 71, 73, 74, 75, 75, 76, 73, 71, 72, 74, 75, 75, + 76, 76, 74, 72, 73, 74, 75, 76, 76, 77, + /* Size 16 */ + 64, 62, 61, 65, 69, 70, 70, 70, 71, 71, 72, 72, 73, 74, 74, 74, 62, 63, + 63, 66, 69, 69, 69, 69, 70, 70, 71, 71, 72, 73, 73, 73, 61, 63, 66, 68, + 69, 69, 68, 68, 69, 69, 70, 70, 71, 72, 72, 72, 65, 66, 68, 69, 70, 70, + 69, 70, 70, 70, 70, 71, 72, 72, 73, 73, 69, 69, 69, 70, 71, 71, 71, 71, + 71, 71, 71, 72, 72, 73, 73, 73, 70, 69, 69, 70, 71, 71, 71, 72, 72, 72, + 72, 72, 73, 73, 74, 74, 70, 69, 68, 69, 71, 71, 72, 72, 73, 73, 73, 73, + 74, 74, 74, 74, 70, 69, 68, 70, 71, 72, 72, 73, 73, 73, 74, 74, 74, 74, + 75, 75, 71, 70, 69, 70, 71, 72, 73, 73, 74, 74, 74, 74, 75, 75, 75, 75, + 71, 70, 69, 70, 71, 72, 73, 73, 74, 74, 75, 75, 75, 75, 76, 76, 72, 71, + 70, 70, 71, 72, 73, 74, 74, 75, 75, 75, 75, 76, 76, 76, 72, 71, 70, 71, + 72, 72, 73, 74, 74, 75, 75, 75, 76, 76, 76, 76, 73, 72, 71, 72, 72, 73, + 74, 74, 75, 75, 75, 76, 76, 76, 76, 76, 74, 73, 72, 72, 73, 73, 74, 74, + 75, 75, 76, 76, 76, 76, 77, 77, 74, 73, 72, 73, 73, 74, 74, 75, 75, 76, + 76, 76, 76, 77, 77, 77, 74, 73, 72, 73, 73, 74, 74, 75, 75, 76, 76, 76, + 76, 77, 77, 77, + /* Size 32 */ + 64, 63, 62, 61, 61, 63, 65, 67, 69, 69, 70, 70, 70, 70, 70, 70, 71, 71, + 71, 72, 72, 72, 72, 73, 73, 73, 74, 74, 74, 74, 74, 74, 63, 63, 62, 62, + 62, 64, 65, 67, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 71, 71, 71, 72, + 72, 72, 73, 73, 73, 73, 74, 74, 74, 74, 62, 62, 63, 63, 63, 65, 66, 68, + 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, 71, 71, 71, 72, 72, 72, + 73, 73, 73, 73, 73, 73, 61, 62, 63, 64, 65, 66, 67, 68, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 70, 70, 70, 71, 71, 71, 72, 72, 72, 72, 73, 73, + 73, 73, 61, 62, 63, 65, 66, 67, 68, 69, 69, 69, 69, 68, 68, 68, 68, 68, + 69, 69, 69, 69, 70, 70, 70, 71, 71, 71, 72, 72, 72, 72, 72, 72, 63, 64, + 65, 66, 67, 68, 68, 69, 70, 70, 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, + 70, 70, 71, 71, 71, 72, 72, 72, 73, 73, 73, 73, 65, 65, 66, 67, 68, 68, + 69, 70, 70, 70, 70, 70, 69, 70, 70, 70, 70, 70, 70, 70, 70, 71, 71, 71, + 72, 72, 72, 72, 73, 73, 73, 73, 67, 67, 68, 68, 69, 69, 70, 70, 71, 71, + 70, 70, 70, 70, 70, 70, 70, 70, 71, 71, 71, 71, 71, 72, 72, 72, 72, 73, + 73, 73, 73, 73, 69, 69, 69, 69, 69, 70, 70, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 72, 72, 72, 72, 72, 73, 73, 73, 73, 73, 73, + 69, 69, 69, 69, 69, 70, 70, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 72, 72, 72, 72, 72, 73, 73, 73, 73, 73, 73, 73, 73, 70, 69, 69, 69, + 69, 69, 70, 70, 71, 71, 71, 71, 71, 72, 72, 72, 72, 72, 72, 72, 72, 72, + 72, 73, 73, 73, 73, 73, 74, 74, 74, 74, 70, 69, 69, 69, 68, 69, 70, 70, + 71, 71, 71, 72, 72, 72, 72, 72, 72, 72, 72, 72, 73, 73, 73, 73, 73, 73, + 74, 74, 74, 74, 74, 74, 70, 69, 69, 69, 68, 69, 69, 70, 71, 71, 71, 72, + 72, 72, 72, 72, 73, 73, 73, 73, 73, 73, 73, 73, 74, 74, 74, 74, 74, 74, + 74, 74, 70, 70, 69, 69, 68, 69, 70, 70, 71, 71, 72, 72, 72, 72, 73, 73, + 73, 73, 73, 73, 73, 73, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 70, 70, + 69, 69, 68, 69, 70, 70, 71, 71, 72, 72, 72, 73, 73, 73, 73, 73, 73, 73, + 74, 74, 74, 74, 74, 74, 74, 75, 75, 75, 75, 75, 70, 70, 69, 69, 68, 69, + 70, 70, 71, 71, 72, 72, 72, 73, 73, 73, 73, 73, 74, 74, 74, 74, 74, 74, + 74, 74, 75, 75, 75, 75, 75, 75, 71, 70, 70, 69, 69, 69, 70, 70, 71, 71, + 72, 72, 73, 73, 73, 73, 74, 74, 74, 74, 74, 74, 74, 75, 75, 75, 75, 75, + 75, 75, 75, 75, 71, 70, 70, 69, 69, 69, 70, 70, 71, 71, 72, 72, 73, 73, + 73, 73, 74, 74, 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, + 71, 71, 70, 70, 69, 70, 70, 71, 71, 71, 72, 72, 73, 73, 73, 74, 74, 74, + 74, 74, 75, 75, 75, 75, 75, 75, 75, 75, 76, 76, 76, 76, 72, 71, 70, 70, + 69, 70, 70, 71, 71, 72, 72, 72, 73, 73, 73, 74, 74, 74, 74, 75, 75, 75, + 75, 75, 75, 75, 75, 76, 76, 76, 76, 76, 72, 71, 71, 70, 70, 70, 70, 71, + 71, 72, 72, 73, 73, 73, 74, 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, 76, + 76, 76, 76, 76, 76, 76, 72, 72, 71, 71, 70, 70, 71, 71, 72, 72, 72, 73, + 73, 73, 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, 76, 76, 76, 76, 76, 76, + 76, 76, 72, 72, 71, 71, 70, 71, 71, 71, 72, 72, 72, 73, 73, 74, 74, 74, + 74, 75, 75, 75, 75, 75, 75, 76, 76, 76, 76, 76, 76, 76, 76, 76, 73, 72, + 72, 71, 71, 71, 71, 72, 72, 72, 73, 73, 73, 74, 74, 74, 75, 75, 75, 75, + 75, 75, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 73, 73, 72, 72, 71, 71, + 72, 72, 72, 73, 73, 73, 74, 74, 74, 74, 75, 75, 75, 75, 75, 76, 76, 76, + 76, 76, 76, 76, 76, 76, 76, 76, 73, 73, 72, 72, 71, 72, 72, 72, 72, 73, + 73, 73, 74, 74, 74, 74, 75, 75, 75, 75, 76, 76, 76, 76, 76, 76, 76, 76, + 77, 77, 77, 77, 74, 73, 73, 72, 72, 72, 72, 72, 73, 73, 73, 74, 74, 74, + 74, 75, 75, 75, 75, 75, 76, 76, 76, 76, 76, 76, 76, 77, 77, 77, 77, 77, + 74, 73, 73, 72, 72, 72, 72, 73, 73, 73, 73, 74, 74, 74, 75, 75, 75, 75, + 75, 76, 76, 76, 76, 76, 76, 76, 77, 77, 77, 77, 77, 77, 74, 74, 73, 73, + 72, 73, 73, 73, 73, 73, 74, 74, 74, 74, 75, 75, 75, 75, 76, 76, 76, 76, + 76, 76, 76, 77, 77, 77, 77, 77, 77, 77, 74, 74, 73, 73, 72, 73, 73, 73, + 73, 73, 74, 74, 74, 74, 75, 75, 75, 75, 76, 76, 76, 76, 76, 76, 76, 77, + 77, 77, 77, 77, 77, 77, 74, 74, 73, 73, 72, 73, 73, 73, 73, 73, 74, 74, + 74, 74, 75, 75, 75, 75, 76, 76, 76, 76, 76, 76, 76, 77, 77, 77, 77, 77, + 77, 77, 74, 74, 73, 73, 72, 73, 73, 73, 73, 73, 74, 74, 74, 74, 75, 75, + 75, 75, 76, 76, 76, 76, 76, 76, 76, 77, 77, 77, 77, 77, 77, 77 }, + { /* Intra matrices */ + /* Size 4 */ + 55, 61, 62, 65, 61, 63, 64, 65, 62, 64, 66, 67, 65, 65, 67, 68, + /* Size 8 */ + 57, 53, 61, 62, 63, 64, 65, 66, 53, 59, 62, 60, 61, 62, 63, 64, 61, 62, + 63, 63, 63, 63, 64, 65, 62, 60, 63, 64, 65, 65, 66, 66, 63, 61, 63, 65, + 66, 66, 67, 67, 64, 62, 63, 65, 66, 67, 67, 68, 65, 63, 64, 66, 67, 67, + 68, 68, 66, 64, 65, 66, 67, 68, 68, 69, + /* Size 16 */ + 56, 55, 53, 57, 61, 61, 62, 62, 62, 63, 64, 64, 65, 65, 66, 66, 55, 55, + 56, 58, 61, 61, 61, 61, 61, 62, 63, 63, 64, 64, 65, 65, 53, 56, 58, 60, + 61, 61, 60, 60, 61, 61, 62, 62, 63, 63, 64, 64, 57, 58, 60, 61, 62, 62, + 61, 61, 62, 62, 62, 63, 63, 64, 64, 64, 61, 61, 61, 62, 63, 63, 63, 63, + 63, 63, 63, 64, 64, 64, 65, 65, 61, 61, 61, 62, 63, 63, 63, 63, 63, 64, + 64, 64, 65, 65, 65, 65, 62, 61, 60, 61, 63, 63, 64, 64, 64, 64, 65, 65, + 65, 65, 66, 66, 62, 61, 60, 61, 63, 63, 64, 64, 65, 65, 65, 65, 66, 66, + 66, 66, 62, 61, 61, 62, 63, 63, 64, 65, 65, 66, 66, 66, 66, 66, 67, 67, + 63, 62, 61, 62, 63, 64, 64, 65, 66, 66, 66, 66, 67, 67, 67, 67, 64, 63, + 62, 62, 63, 64, 65, 65, 66, 66, 67, 67, 67, 67, 67, 67, 64, 63, 62, 63, + 64, 64, 65, 65, 66, 66, 67, 67, 67, 68, 68, 68, 65, 64, 63, 63, 64, 65, + 65, 66, 66, 67, 67, 67, 68, 68, 68, 68, 65, 64, 63, 64, 64, 65, 65, 66, + 66, 67, 67, 68, 68, 68, 68, 68, 66, 65, 64, 64, 65, 65, 66, 66, 67, 67, + 67, 68, 68, 68, 68, 68, 66, 65, 64, 64, 65, 65, 66, 66, 67, 67, 67, 68, + 68, 68, 68, 68, + /* Size 32 */ + 56, 55, 55, 54, 53, 55, 57, 59, 61, 61, 61, 61, 62, 62, 62, 62, 62, 63, + 63, 63, 63, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, 55, 55, 55, 54, + 54, 56, 57, 59, 61, 61, 61, 61, 61, 61, 62, 62, 62, 62, 62, 63, 63, 63, + 63, 64, 64, 64, 65, 65, 65, 65, 65, 65, 55, 55, 55, 55, 55, 57, 58, 60, + 61, 61, 61, 61, 61, 61, 61, 61, 61, 62, 62, 62, 62, 63, 63, 63, 64, 64, + 64, 64, 65, 65, 65, 65, 54, 54, 55, 56, 57, 58, 59, 60, 61, 61, 61, 61, + 60, 61, 61, 61, 61, 61, 61, 62, 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, + 64, 64, 53, 54, 55, 57, 58, 59, 60, 60, 61, 61, 61, 60, 60, 60, 60, 60, + 60, 61, 61, 61, 61, 62, 62, 62, 63, 63, 63, 64, 64, 64, 64, 64, 55, 56, + 57, 58, 59, 60, 60, 61, 62, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 62, + 62, 62, 62, 63, 63, 63, 64, 64, 64, 64, 64, 64, 57, 57, 58, 59, 60, 60, + 61, 61, 62, 62, 62, 61, 61, 61, 61, 61, 61, 62, 62, 62, 62, 62, 63, 63, + 63, 64, 64, 64, 64, 64, 64, 64, 59, 59, 60, 60, 60, 61, 61, 62, 62, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, + 64, 64, 64, 64, 61, 61, 61, 61, 61, 62, 62, 62, 63, 63, 63, 63, 62, 62, + 62, 62, 62, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, + 61, 61, 61, 61, 61, 61, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 61, 61, 61, 61, + 61, 61, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, + 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 61, 61, 61, 61, 60, 61, 61, 62, + 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 62, 61, 61, 60, 60, 61, 61, 62, 62, 63, 63, 63, + 64, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, + 66, 66, 62, 61, 61, 61, 60, 61, 61, 62, 62, 63, 63, 63, 64, 64, 64, 64, + 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 62, 62, + 61, 61, 60, 61, 61, 62, 62, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, + 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 62, 62, 61, 61, 60, 61, + 61, 62, 62, 63, 63, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 66, 66, + 66, 66, 66, 66, 66, 66, 66, 66, 62, 62, 61, 61, 60, 61, 61, 62, 62, 63, + 63, 64, 64, 64, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, + 67, 67, 67, 67, 63, 62, 62, 61, 61, 61, 62, 62, 63, 63, 63, 64, 64, 64, + 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, + 63, 62, 62, 61, 61, 61, 62, 62, 63, 63, 64, 64, 64, 65, 65, 65, 65, 66, + 66, 66, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 63, 63, 62, 62, + 61, 62, 62, 62, 63, 63, 64, 64, 64, 65, 65, 65, 66, 66, 66, 66, 66, 66, + 66, 67, 67, 67, 67, 67, 67, 67, 67, 67, 63, 63, 62, 62, 61, 62, 62, 63, + 63, 63, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, + 67, 67, 67, 67, 67, 67, 64, 63, 63, 62, 62, 62, 62, 63, 63, 64, 64, 64, + 65, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, + 67, 67, 64, 63, 63, 63, 62, 62, 63, 63, 63, 64, 64, 64, 65, 65, 65, 66, + 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 64, 64, + 63, 63, 62, 63, 63, 63, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, 67, + 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 65, 64, 64, 63, 63, 63, + 63, 64, 64, 64, 64, 65, 65, 65, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, + 67, 68, 68, 68, 68, 68, 68, 68, 65, 64, 64, 63, 63, 63, 64, 64, 64, 64, + 65, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, + 68, 68, 68, 68, 65, 65, 64, 64, 63, 64, 64, 64, 64, 65, 65, 65, 65, 66, + 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, + 65, 65, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, 67, + 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 66, 65, 65, 64, + 64, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, + 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 66, 65, 65, 64, 64, 64, 64, 64, + 65, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, + 68, 68, 68, 68, 68, 68, 66, 65, 65, 64, 64, 64, 64, 64, 65, 65, 65, 65, + 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 66, 65, 65, 64, 64, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, + 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 65, 68, 70, 65, 67, 69, 70, 68, 69, 70, 71, 70, 70, 71, 71, + /* Size 8 */ + 64, 62, 62, 64, 66, 67, 68, 69, 62, 63, 63, 64, 65, 67, 68, 68, 62, 63, + 65, 66, 67, 67, 68, 69, 64, 64, 66, 67, 68, 68, 69, 69, 66, 65, 67, 68, + 68, 69, 69, 69, 67, 67, 67, 68, 69, 69, 69, 69, 68, 68, 68, 69, 69, 69, + 69, 69, 69, 68, 69, 69, 69, 69, 69, 70, + /* Size 16 */ + 64, 63, 62, 62, 62, 63, 64, 65, 66, 67, 67, 68, 68, 68, 69, 69, 63, 63, + 62, 62, 63, 63, 64, 65, 66, 66, 67, 67, 68, 68, 69, 69, 62, 62, 63, 63, + 63, 63, 64, 65, 65, 66, 67, 67, 68, 68, 68, 68, 62, 62, 63, 63, 64, 64, + 65, 65, 66, 67, 67, 67, 68, 68, 68, 68, 62, 63, 63, 64, 65, 65, 66, 66, + 67, 67, 67, 68, 68, 68, 69, 69, 63, 63, 63, 64, 65, 66, 66, 67, 67, 67, + 68, 68, 68, 68, 69, 69, 64, 64, 64, 65, 66, 66, 67, 67, 68, 68, 68, 68, + 69, 69, 69, 69, 65, 65, 65, 65, 66, 67, 67, 68, 68, 68, 68, 69, 69, 69, + 69, 69, 66, 66, 65, 66, 67, 67, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, + 67, 66, 66, 67, 67, 67, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 67, 67, + 67, 67, 67, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 68, 67, 67, 67, + 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 68, 68, 68, 68, 68, 68, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 68, 68, 68, 68, 68, 68, 69, 69, + 69, 69, 69, 69, 69, 69, 70, 70, 69, 69, 68, 68, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 70, 70, 70, 69, 69, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 70, 70, 70, + /* Size 32 */ + 64, 63, 63, 62, 62, 62, 62, 62, 62, 63, 63, 64, 64, 65, 65, 66, 66, 66, + 67, 67, 67, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 63, 63, 63, 62, + 62, 62, 62, 62, 62, 63, 63, 64, 64, 65, 65, 66, 66, 66, 67, 67, 67, 67, + 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 63, 63, 63, 62, 62, 62, 62, 62, + 63, 63, 63, 64, 64, 65, 65, 65, 66, 66, 66, 67, 67, 67, 67, 68, 68, 68, + 68, 68, 69, 69, 69, 69, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, 64, + 64, 64, 65, 65, 66, 66, 66, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 64, 65, 65, + 65, 66, 66, 66, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 62, 62, + 62, 63, 63, 63, 63, 63, 63, 63, 64, 64, 64, 65, 65, 65, 66, 66, 66, 67, + 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 62, 62, 62, 63, 63, 63, + 63, 63, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, 67, 67, 67, 67, 67, 68, + 68, 68, 68, 68, 68, 68, 68, 68, 62, 62, 62, 63, 63, 63, 63, 64, 64, 65, + 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, + 68, 68, 68, 68, 62, 62, 63, 63, 63, 63, 64, 64, 65, 65, 65, 66, 66, 66, + 66, 66, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, + 63, 63, 63, 63, 63, 63, 64, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, + 67, 67, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 63, 63, 63, 63, + 63, 64, 64, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 68, 68, 68, + 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 64, 64, 64, 64, 63, 64, 65, 65, + 66, 66, 66, 66, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, + 69, 69, 69, 69, 69, 69, 64, 64, 64, 64, 64, 64, 65, 65, 66, 66, 66, 67, + 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, + 69, 69, 65, 65, 65, 64, 64, 65, 65, 66, 66, 66, 67, 67, 67, 67, 67, 68, + 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 65, 65, + 65, 65, 65, 65, 65, 66, 66, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, + 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 66, 66, 65, 65, 65, 65, + 66, 66, 66, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 66, 66, 66, 66, 65, 66, 66, 66, 67, 67, + 67, 67, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 69, 66, 66, 66, 66, 66, 66, 66, 67, 67, 67, 67, 68, 68, 68, + 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 67, 67, 66, 66, 66, 66, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 67, 67, 67, 67, + 66, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 67, 67, 67, 67, 67, 67, 67, 67, + 67, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 68, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, + 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 68, 68, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 69, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 68, 68, + 68, 68, 67, 68, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 68, 68, 68, 68, 68, 68, + 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 70, 70, 70, 70, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, + 69, 68, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 69, 69, 69, 68, + 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 69, 69, 69, 68, 68, 68, 68, 68, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 70, + 70, 70, 70, 70, 70, 70, 69, 69, 69, 68, 68, 68, 68, 68, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, + 70, 70, 69, 69, 69, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70 }, + { /* Intra matrices */ + /* Size 4 */ + 59, 60, 63, 65, 60, 63, 64, 65, 63, 64, 66, 66, 65, 65, 66, 66, + /* Size 8 */ + 61, 59, 59, 61, 63, 64, 65, 66, 59, 60, 59, 61, 62, 64, 65, 65, 59, 59, + 62, 63, 64, 64, 65, 65, 61, 61, 63, 64, 65, 65, 65, 66, 63, 62, 64, 65, + 65, 66, 66, 66, 64, 64, 64, 65, 66, 66, 66, 66, 65, 65, 65, 65, 66, 66, + 66, 66, 66, 65, 65, 66, 66, 66, 66, 67, + /* Size 16 */ + 61, 60, 58, 59, 59, 60, 61, 62, 63, 63, 64, 64, 65, 65, 65, 65, 60, 59, + 59, 59, 59, 60, 61, 62, 62, 63, 64, 64, 65, 65, 65, 65, 58, 59, 60, 59, + 59, 60, 60, 61, 62, 63, 63, 64, 64, 65, 65, 65, 59, 59, 59, 60, 60, 61, + 61, 62, 63, 63, 64, 64, 65, 65, 65, 65, 59, 59, 59, 60, 62, 62, 63, 63, + 63, 64, 64, 64, 65, 65, 65, 65, 60, 60, 60, 61, 62, 63, 63, 63, 64, 64, + 64, 65, 65, 65, 65, 65, 61, 61, 60, 61, 63, 63, 64, 64, 64, 65, 65, 65, + 65, 65, 66, 66, 62, 62, 61, 62, 63, 63, 64, 64, 65, 65, 65, 65, 65, 66, + 66, 66, 63, 62, 62, 63, 63, 64, 64, 65, 65, 65, 65, 65, 66, 66, 66, 66, + 63, 63, 63, 63, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 64, 64, + 63, 64, 64, 64, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 64, 64, 64, 64, + 64, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 65, 65, 64, 65, 65, 65, + 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 66, + 66, 66, 66, 66, 66, 66, 66, 66, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 66, + /* Size 32 */ + 61, 60, 59, 59, 58, 59, 59, 59, 59, 60, 60, 61, 61, 61, 62, 62, 63, 63, + 63, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 60, 60, 59, 59, + 59, 59, 59, 59, 59, 59, 60, 60, 61, 61, 62, 62, 63, 63, 63, 63, 64, 64, + 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 59, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 60, 60, 61, 61, 62, 62, 62, 63, 63, 63, 64, 64, 64, 64, 65, 65, + 65, 65, 65, 65, 65, 65, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, + 61, 61, 61, 62, 62, 62, 63, 63, 63, 64, 64, 64, 64, 65, 65, 65, 65, 65, + 65, 65, 58, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 60, 61, 61, 62, + 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 59, 59, + 59, 59, 59, 59, 60, 60, 60, 60, 60, 61, 61, 61, 62, 62, 62, 63, 63, 63, + 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 59, 59, 59, 59, 59, 60, + 60, 60, 60, 61, 61, 61, 61, 62, 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, + 64, 65, 65, 65, 65, 65, 65, 65, 59, 59, 59, 59, 59, 60, 60, 60, 61, 61, + 61, 62, 62, 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, + 65, 65, 65, 65, 59, 59, 59, 59, 59, 60, 60, 61, 62, 62, 62, 62, 62, 63, + 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, + 60, 59, 59, 59, 59, 60, 61, 61, 62, 62, 62, 62, 63, 63, 63, 63, 63, 64, + 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 60, 60, 60, 60, + 60, 60, 61, 61, 62, 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 64, + 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 61, 60, 60, 60, 60, 61, 61, 62, + 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 61, 61, 61, 61, 60, 61, 61, 62, 62, 63, 63, 63, + 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 61, 61, 61, 61, 61, 61, 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, + 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 62, 62, + 62, 61, 61, 62, 62, 62, 63, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 62, 62, 62, 62, 62, 62, + 62, 63, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 66, 66, 66, 66, 66, 66, 63, 63, 62, 62, 62, 62, 63, 63, 63, 63, + 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, + 66, 66, 66, 66, 63, 63, 63, 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, 65, + 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, + 63, 63, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 64, 63, 63, 63, + 63, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, + 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 64, 64, 64, 63, 63, 63, 64, 64, + 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 64, 64, 64, 64, 63, 64, 64, 64, 64, 64, 64, 65, + 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, + 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 64, + 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 65, 65, 64, 64, 64, + 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 66, 66, 65, 65, 65, 65, 64, 64, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, + 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 67, 68, 69, 67, 68, 69, 69, 68, 69, 70, 70, 69, 69, 70, 71, + /* Size 8 */ + 64, 62, 67, 67, 67, 68, 68, 69, 62, 65, 67, 66, 66, 67, 68, 68, 67, 67, + 68, 67, 67, 68, 68, 69, 67, 66, 67, 68, 68, 68, 69, 69, 67, 66, 67, 68, + 69, 69, 69, 69, 68, 67, 68, 68, 69, 69, 70, 70, 68, 68, 68, 69, 69, 70, + 70, 70, 69, 68, 69, 69, 69, 70, 70, 70, + /* Size 16 */ + 64, 63, 62, 64, 67, 67, 67, 67, 67, 68, 68, 68, 68, 69, 69, 69, 63, 63, + 64, 65, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 69, 69, 62, 64, 65, 66, + 67, 66, 66, 66, 66, 67, 67, 67, 68, 68, 68, 68, 64, 65, 66, 67, 67, 67, + 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 67, 67, 67, 67, 68, 67, 67, 67, + 67, 68, 68, 68, 68, 68, 69, 69, 67, 67, 66, 67, 67, 68, 68, 68, 68, 68, + 68, 68, 68, 69, 69, 69, 67, 67, 66, 67, 67, 68, 68, 68, 68, 68, 68, 69, + 69, 69, 69, 69, 67, 67, 66, 67, 67, 68, 68, 68, 68, 69, 69, 69, 69, 69, + 69, 69, 67, 67, 66, 67, 67, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, + 68, 67, 67, 67, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 70, 70, 68, 67, + 67, 67, 68, 68, 68, 69, 69, 69, 69, 69, 70, 70, 70, 70, 68, 68, 67, 68, + 68, 68, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 68, 68, 68, 68, 68, 68, + 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 69, 68, 68, 68, 68, 69, 69, 69, + 69, 69, 70, 70, 70, 70, 70, 70, 69, 69, 68, 68, 69, 69, 69, 69, 69, 70, + 70, 70, 70, 70, 70, 70, 69, 69, 68, 68, 69, 69, 69, 69, 69, 70, 70, 70, + 70, 70, 70, 70, + /* Size 32 */ + 64, 64, 63, 63, 62, 63, 64, 65, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, + 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 64, 63, 63, 63, + 63, 64, 65, 66, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 68, 68, + 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 63, 63, 63, 63, 64, 64, 65, 66, + 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, + 68, 68, 69, 69, 69, 69, 63, 63, 63, 64, 64, 65, 66, 66, 67, 67, 67, 66, + 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 62, 63, 64, 64, 65, 66, 66, 66, 67, 67, 66, 66, 66, 66, 66, 66, + 66, 66, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 63, 64, + 64, 65, 66, 66, 66, 67, 67, 67, 67, 67, 66, 66, 67, 67, 67, 67, 67, 67, + 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 64, 65, 65, 66, 66, 66, + 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 68, 68, + 68, 68, 68, 68, 68, 68, 68, 68, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, + 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 68, 68, 67, 67, 67, 67, 67, 67, 67, 67, 68, 68, 67, 67, 67, 67, + 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, + 67, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 67, 67, 67, 67, + 66, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 67, 67, 67, 66, 66, 67, 67, 67, + 67, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 69, 69, + 69, 69, 69, 69, 69, 69, 67, 67, 67, 66, 66, 66, 67, 67, 67, 68, 68, 68, + 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 67, 67, 67, 66, 66, 66, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 67, 67, + 67, 66, 66, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 67, 67, 67, 67, 66, 67, + 67, 67, 67, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 67, 67, 67, 67, 66, 67, 67, 67, 67, 68, + 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 69, 67, 67, 67, 67, 66, 67, 67, 67, 67, 68, 68, 68, 68, 68, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 68, 67, 67, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 68, 67, 67, 67, + 67, 67, 67, 67, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 68, 68, 67, 67, 67, 67, 67, 67, + 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 68, 68, 68, 67, 67, 67, 67, 68, 68, 68, 68, 68, + 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 68, 68, 68, 67, 67, 67, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68, 68, + 68, 68, 67, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68, 68, 68, 68, 68, 68, + 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 69, 68, 68, 68, 68, 68, 68, 68, 68, 68, + 68, 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 69, 68, 68, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 69, 69, 68, 68, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 69, 69, 69, 68, + 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 69, 69, 69, 68, 68, 68, 68, 68, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 70, 70, 70, 70, 69, 69, 69, 68, 68, 68, 68, 68, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 69, 69, 69, 68, 68, 68, 68, 68, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70 }, + { /* Intra matrices */ + /* Size 4 */ + 59, 63, 63, 64, 63, 64, 64, 65, 63, 64, 65, 66, 64, 65, 66, 66, + /* Size 8 */ + 60, 58, 63, 63, 63, 64, 65, 65, 58, 61, 63, 62, 62, 63, 64, 64, 63, 63, + 64, 64, 64, 64, 64, 65, 63, 62, 64, 64, 64, 65, 65, 65, 63, 62, 64, 64, + 65, 65, 65, 66, 64, 63, 64, 65, 65, 65, 66, 66, 65, 64, 64, 65, 65, 66, + 66, 66, 65, 64, 65, 65, 66, 66, 66, 66, + /* Size 16 */ + 60, 59, 58, 60, 63, 63, 63, 63, 63, 64, 64, 64, 64, 65, 65, 65, 59, 59, + 60, 61, 63, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 58, 60, 61, 62, + 63, 62, 62, 62, 62, 63, 63, 63, 63, 64, 64, 64, 60, 61, 62, 62, 63, 63, + 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 63, 63, 63, 63, 64, 63, 63, 63, + 63, 63, 64, 64, 64, 64, 64, 64, 63, 63, 62, 63, 63, 64, 64, 64, 64, 64, + 64, 64, 64, 65, 65, 65, 63, 63, 62, 63, 63, 64, 64, 64, 64, 64, 64, 65, + 65, 65, 65, 65, 63, 63, 62, 63, 63, 64, 64, 64, 64, 65, 65, 65, 65, 65, + 65, 65, 63, 63, 62, 63, 63, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, + 64, 63, 63, 63, 63, 64, 64, 65, 65, 65, 65, 65, 65, 65, 66, 66, 64, 63, + 63, 63, 64, 64, 64, 65, 65, 65, 65, 65, 66, 66, 66, 66, 64, 64, 63, 63, + 64, 64, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 64, 64, 63, 64, 64, 64, + 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 65, 64, 64, 64, 64, 65, 65, 65, + 65, 65, 66, 66, 66, 66, 66, 66, 65, 64, 64, 64, 64, 65, 65, 65, 65, 66, + 66, 66, 66, 66, 66, 66, 65, 64, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, + 66, 66, 66, 66, + /* Size 32 */ + 60, 59, 59, 59, 58, 59, 60, 61, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, + 63, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 59, 59, 59, 59, + 59, 60, 61, 62, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 64, + 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 59, 59, 59, 59, 60, 60, 61, 62, + 63, 63, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 59, 59, 59, 60, 60, 61, 61, 62, 63, 62, 62, 62, + 62, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, + 64, 64, 58, 59, 60, 60, 61, 61, 62, 62, 63, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 64, 59, 60, + 60, 61, 61, 62, 62, 62, 63, 63, 63, 62, 62, 62, 62, 62, 62, 63, 63, 63, + 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 64, 64, 60, 61, 61, 61, 62, 62, + 62, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 61, 62, 62, 62, 62, 62, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 63, 63, 63, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 63, 63, 62, 62, + 62, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 63, 63, 62, 62, 62, 62, 63, 63, + 63, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 65, 65, 65, 65, 65, 65, 63, 63, 62, 62, 62, 62, 63, 63, 63, 63, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, + 65, 65, 63, 63, 62, 62, 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 63, 63, + 63, 62, 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 65, + 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 63, 63, 63, 62, 62, 62, + 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, 63, 63, 63, 62, 62, 62, 63, 63, 63, 63, + 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 63, 63, 63, 63, 62, 63, 63, 63, 63, 64, 64, 64, 64, 64, + 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 63, 63, 63, 63, 62, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 64, 63, 63, 63, + 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 64, 64, 63, 63, 63, 63, 63, 63, + 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 66, 66, 66, 66, 66, 66, 64, 64, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, + 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, + 66, 66, 64, 64, 64, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 64, 64, + 64, 63, 63, 63, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 64, 64, 64, 64, 63, 64, + 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 66, 66, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 66, 65, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, + 65, 64, 64, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 65, 64, 64, + 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 65, 64, 64, 64, 64, 64, 64, + 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 65, 65, 64, 64, 64, 64, 64, 64, 64, 64, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 65, 65, 64, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 8 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 16 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, + /* Size 32 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 }, + { /* Intra matrices */ + /* Size 4 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 8 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 16 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, + /* Size 32 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 8 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 16 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, + /* Size 32 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 }, + { /* Intra matrices */ + /* Size 4 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 8 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 16 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, + /* Size 32 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 } } } +}; + +static uint16_t wt_matrix_ref[NUM_QM_LEVELS][2][2][4 * 4 + 8 * 8 + 16 * 16 + + 32 * 32] = { + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 58, 33, 19, 58, 37, 25, 17, 33, 25, 16, 12, 19, 17, 12, 10, + /* Size 8 */ + 64, 87, 80, 59, 42, 31, 24, 19, 87, 75, 79, 66, 50, 38, 29, 23, 80, 79, + 54, 46, 38, 31, 25, 20, 59, 66, 46, 34, 29, 24, 21, 18, 42, 50, 38, 29, + 23, 20, 17, 15, 31, 38, 31, 24, 20, 17, 15, 13, 24, 29, 25, 21, 17, 15, + 13, 12, 19, 23, 20, 18, 15, 13, 12, 11, + /* Size 16 */ + 64, 76, 87, 84, 80, 70, 59, 51, 42, 37, 31, 27, 24, 21, 19, 19, 76, 79, + 81, 81, 80, 71, 63, 55, 46, 40, 34, 30, 26, 23, 21, 21, 87, 81, 75, 77, + 79, 73, 66, 58, 50, 44, 38, 33, 29, 26, 23, 23, 84, 81, 77, 72, 67, 61, + 56, 50, 44, 39, 34, 31, 27, 24, 21, 21, 80, 80, 79, 67, 54, 50, 46, 42, + 38, 34, 31, 28, 25, 23, 20, 20, 70, 71, 73, 61, 50, 45, 40, 37, 33, 30, + 28, 25, 23, 21, 19, 19, 59, 63, 66, 56, 46, 40, 34, 31, 29, 26, 24, 22, + 21, 19, 18, 18, 51, 55, 58, 50, 42, 37, 31, 29, 26, 24, 22, 20, 19, 18, + 16, 16, 42, 46, 50, 44, 38, 33, 29, 26, 23, 21, 20, 18, 17, 16, 15, 15, + 37, 40, 44, 39, 34, 30, 26, 24, 21, 20, 18, 17, 16, 15, 14, 14, 31, 34, + 38, 34, 31, 28, 24, 22, 20, 18, 17, 16, 15, 14, 13, 13, 27, 30, 33, 31, + 28, 25, 22, 20, 18, 17, 16, 15, 14, 13, 13, 13, 24, 26, 29, 27, 25, 23, + 21, 19, 17, 16, 15, 14, 13, 13, 12, 12, 21, 23, 26, 24, 23, 21, 19, 18, + 16, 15, 14, 13, 13, 12, 12, 12, 19, 21, 23, 21, 20, 19, 18, 16, 15, 14, + 13, 13, 12, 12, 11, 11, 19, 21, 23, 21, 20, 19, 18, 16, 15, 14, 13, 13, + 12, 12, 11, 11, + /* Size 32 */ + 64, 70, 76, 82, 87, 86, 84, 82, 80, 75, 70, 65, 59, 55, 51, 47, 42, 40, + 37, 34, 31, 29, 27, 26, 24, 22, 21, 20, 19, 19, 19, 19, 70, 74, 77, 81, + 84, 83, 82, 81, 80, 75, 71, 66, 61, 57, 53, 49, 44, 41, 39, 36, 33, 31, + 29, 27, 25, 24, 22, 21, 20, 20, 20, 20, 76, 77, 79, 80, 81, 81, 81, 80, + 80, 75, 71, 67, 63, 59, 55, 51, 46, 43, 40, 37, 34, 32, 30, 28, 26, 25, + 23, 22, 21, 21, 21, 21, 82, 81, 80, 79, 78, 79, 79, 79, 79, 76, 72, 68, + 65, 61, 56, 52, 48, 45, 42, 39, 36, 34, 32, 30, 27, 26, 25, 23, 22, 22, + 22, 22, 87, 84, 81, 78, 75, 76, 77, 78, 79, 76, 73, 70, 66, 62, 58, 54, + 50, 47, 44, 41, 38, 36, 33, 31, 29, 27, 26, 24, 23, 23, 23, 23, 86, 83, + 81, 79, 76, 75, 75, 74, 73, 70, 67, 64, 61, 58, 54, 51, 47, 44, 42, 39, + 36, 34, 32, 30, 28, 26, 25, 23, 22, 22, 22, 22, 84, 82, 81, 79, 77, 75, + 72, 69, 67, 64, 61, 59, 56, 53, 50, 47, 44, 42, 39, 37, 34, 32, 31, 29, + 27, 25, 24, 23, 21, 21, 21, 21, 82, 81, 80, 79, 78, 74, 69, 65, 61, 58, + 56, 53, 51, 48, 46, 44, 41, 39, 37, 35, 33, 31, 29, 28, 26, 25, 23, 22, + 21, 21, 21, 21, 80, 80, 80, 79, 79, 73, 67, 61, 54, 52, 50, 48, 46, 44, + 42, 40, 38, 36, 34, 33, 31, 29, 28, 26, 25, 24, 23, 22, 20, 20, 20, 20, + 75, 75, 75, 76, 76, 70, 64, 58, 52, 50, 47, 45, 43, 41, 39, 37, 36, 34, + 32, 31, 29, 28, 27, 25, 24, 23, 22, 21, 20, 20, 20, 20, 70, 71, 71, 72, + 73, 67, 61, 56, 50, 47, 45, 42, 40, 38, 37, 35, 33, 32, 30, 29, 28, 26, + 25, 24, 23, 22, 21, 20, 19, 19, 19, 19, 65, 66, 67, 68, 70, 64, 59, 53, + 48, 45, 42, 40, 37, 36, 34, 32, 31, 30, 28, 27, 26, 25, 24, 23, 22, 21, + 20, 19, 18, 18, 18, 18, 59, 61, 63, 65, 66, 61, 56, 51, 46, 43, 40, 37, + 34, 33, 31, 30, 29, 27, 26, 25, 24, 23, 22, 22, 21, 20, 19, 18, 18, 18, + 18, 18, 55, 57, 59, 61, 62, 58, 53, 48, 44, 41, 38, 36, 33, 31, 30, 29, + 27, 26, 25, 24, 23, 22, 21, 21, 20, 19, 18, 18, 17, 17, 17, 17, 51, 53, + 55, 56, 58, 54, 50, 46, 42, 39, 37, 34, 31, 30, 29, 27, 26, 25, 24, 23, + 22, 21, 20, 20, 19, 18, 18, 17, 16, 16, 16, 16, 47, 49, 51, 52, 54, 51, + 47, 44, 40, 37, 35, 32, 30, 29, 27, 26, 24, 24, 23, 22, 21, 20, 19, 19, + 18, 18, 17, 16, 16, 16, 16, 16, 42, 44, 46, 48, 50, 47, 44, 41, 38, 36, + 33, 31, 29, 27, 26, 24, 23, 22, 21, 21, 20, 19, 18, 18, 17, 17, 16, 16, + 15, 15, 15, 15, 40, 41, 43, 45, 47, 44, 42, 39, 36, 34, 32, 30, 27, 26, + 25, 24, 22, 21, 21, 20, 19, 18, 18, 17, 17, 16, 16, 15, 15, 15, 15, 15, + 37, 39, 40, 42, 44, 42, 39, 37, 34, 32, 30, 28, 26, 25, 24, 23, 21, 21, + 20, 19, 18, 18, 17, 17, 16, 16, 15, 15, 14, 14, 14, 14, 34, 36, 37, 39, + 41, 39, 37, 35, 33, 31, 29, 27, 25, 24, 23, 22, 21, 20, 19, 18, 18, 17, + 16, 16, 15, 15, 15, 14, 14, 14, 14, 14, 31, 33, 34, 36, 38, 36, 34, 33, + 31, 29, 28, 26, 24, 23, 22, 21, 20, 19, 18, 18, 17, 16, 16, 15, 15, 15, + 14, 14, 13, 13, 13, 13, 29, 31, 32, 34, 36, 34, 32, 31, 29, 28, 26, 25, + 23, 22, 21, 20, 19, 18, 18, 17, 16, 16, 15, 15, 14, 14, 14, 13, 13, 13, + 13, 13, 27, 29, 30, 32, 33, 32, 31, 29, 28, 27, 25, 24, 22, 21, 20, 19, + 18, 18, 17, 16, 16, 15, 15, 14, 14, 14, 13, 13, 13, 13, 13, 13, 26, 27, + 28, 30, 31, 30, 29, 28, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 17, 16, + 15, 15, 14, 14, 14, 13, 13, 13, 12, 12, 12, 12, 24, 25, 26, 27, 29, 28, + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 17, 16, 15, 15, 14, 14, 14, + 13, 13, 13, 12, 12, 12, 12, 12, 22, 24, 25, 26, 27, 26, 25, 25, 24, 23, + 22, 21, 20, 19, 18, 18, 17, 16, 16, 15, 15, 14, 14, 13, 13, 13, 12, 12, + 12, 12, 12, 12, 21, 22, 23, 25, 26, 25, 24, 23, 23, 22, 21, 20, 19, 18, + 18, 17, 16, 16, 15, 15, 14, 14, 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, + 20, 21, 22, 23, 24, 23, 23, 22, 22, 21, 20, 19, 18, 18, 17, 16, 16, 15, + 15, 14, 14, 13, 13, 13, 12, 12, 12, 12, 11, 11, 11, 11, 19, 20, 21, 22, + 23, 22, 21, 21, 20, 20, 19, 18, 18, 17, 16, 16, 15, 15, 14, 14, 13, 13, + 13, 12, 12, 12, 12, 11, 11, 11, 11, 11, 19, 20, 21, 22, 23, 22, 21, 21, + 20, 20, 19, 18, 18, 17, 16, 16, 15, 15, 14, 14, 13, 13, 13, 12, 12, 12, + 12, 11, 11, 11, 11, 11, 19, 20, 21, 22, 23, 22, 21, 21, 20, 20, 19, 18, + 18, 17, 16, 16, 15, 15, 14, 14, 13, 13, 13, 12, 12, 12, 12, 11, 11, 11, + 11, 11, 19, 20, 21, 22, 23, 22, 21, 21, 20, 20, 19, 18, 18, 17, 16, 16, + 15, 15, 14, 14, 13, 13, 13, 12, 12, 12, 12, 11, 11, 11, 11, 11 }, + { /* Intra matrices */ + /* Size 4 */ + 249, 225, 124, 69, 225, 139, 91, 60, 124, 91, 57, 42, 69, 60, 42, 32, + /* Size 8 */ + 206, 285, 261, 191, 134, 96, 71, 55, 285, 245, 257, 214, 161, 118, 88, + 68, 261, 257, 174, 145, 119, 95, 75, 60, 191, 214, 145, 107, 87, 73, 61, + 51, 134, 161, 119, 87, 69, 58, 50, 43, 96, 118, 95, 73, 58, 48, 42, 37, + 71, 88, 75, 61, 50, 42, 36, 32, 55, 68, 60, 51, 43, 37, 32, 29, + /* Size 16 */ + 217, 259, 300, 287, 275, 237, 200, 171, 141, 121, 101, 88, 75, 66, 57, + 57, 259, 269, 279, 275, 272, 243, 213, 184, 155, 134, 113, 98, 84, 74, + 64, 64, 300, 279, 257, 264, 270, 248, 226, 197, 169, 147, 124, 109, 93, + 82, 71, 71, 287, 275, 264, 245, 227, 208, 189, 168, 147, 130, 112, 99, + 86, 77, 67, 67, 275, 272, 270, 227, 183, 168, 152, 139, 125, 113, 100, + 90, 79, 71, 63, 63, 237, 243, 248, 208, 168, 150, 132, 120, 109, 98, 88, + 80, 72, 65, 58, 58, 200, 213, 226, 189, 152, 132, 113, 102, 92, 84, 77, + 70, 64, 59, 54, 54, 171, 184, 197, 168, 139, 120, 102, 92, 82, 75, 69, + 63, 58, 54, 49, 49, 141, 155, 169, 147, 125, 109, 92, 82, 73, 67, 61, + 56, 52, 49, 45, 45, 121, 134, 147, 130, 113, 98, 84, 75, 67, 61, 56, 52, + 48, 45, 42, 42, 101, 113, 124, 112, 100, 88, 77, 69, 61, 56, 50, 47, 44, + 41, 39, 39, 88, 98, 109, 99, 90, 80, 70, 63, 56, 52, 47, 44, 41, 39, 36, + 36, 75, 84, 93, 86, 79, 72, 64, 58, 52, 48, 44, 41, 38, 36, 34, 34, 66, + 74, 82, 77, 71, 65, 59, 54, 49, 45, 41, 39, 36, 34, 32, 32, 57, 64, 71, + 67, 63, 58, 54, 49, 45, 42, 39, 36, 34, 32, 31, 31, 57, 64, 71, 67, 63, + 58, 54, 49, 45, 42, 39, 36, 34, 32, 31, 31, + /* Size 32 */ + 223, 244, 265, 287, 308, 301, 295, 288, 282, 263, 244, 225, 206, 190, + 175, 160, 145, 134, 124, 114, 104, 97, 90, 83, 77, 72, 68, 63, 59, 59, + 59, 59, 244, 257, 271, 284, 297, 293, 289, 285, 281, 264, 246, 229, 212, + 197, 182, 167, 152, 141, 131, 120, 110, 103, 95, 88, 81, 77, 72, 67, 62, + 62, 62, 62, 265, 271, 276, 281, 286, 284, 283, 281, 280, 264, 249, 234, + 219, 204, 189, 174, 159, 148, 137, 127, 116, 108, 101, 93, 86, 81, 76, + 71, 66, 66, 66, 66, 287, 284, 281, 278, 275, 276, 277, 278, 278, 265, + 252, 238, 225, 210, 196, 181, 166, 155, 144, 133, 122, 114, 106, 98, 91, + 85, 80, 75, 69, 69, 69, 69, 308, 297, 286, 275, 264, 267, 271, 274, 277, + 266, 254, 243, 231, 217, 203, 188, 174, 162, 151, 139, 128, 120, 111, + 103, 95, 90, 84, 78, 73, 73, 73, 73, 301, 293, 284, 276, 267, 264, 261, + 258, 255, 244, 234, 223, 213, 200, 187, 175, 162, 152, 142, 132, 121, + 114, 107, 99, 92, 87, 81, 76, 71, 71, 71, 71, 295, 289, 283, 277, 271, + 261, 252, 242, 233, 223, 213, 203, 194, 183, 172, 162, 151, 142, 133, + 124, 115, 108, 102, 95, 88, 83, 79, 74, 69, 69, 69, 69, 288, 285, 281, + 278, 274, 258, 242, 226, 210, 201, 193, 184, 175, 166, 157, 149, 140, + 132, 124, 117, 109, 103, 97, 91, 85, 80, 76, 71, 67, 67, 67, 67, 282, + 281, 280, 278, 277, 255, 233, 210, 188, 180, 172, 164, 156, 149, 142, + 135, 129, 122, 116, 109, 103, 97, 92, 87, 81, 77, 73, 69, 65, 65, 65, + 65, 263, 264, 264, 265, 266, 244, 223, 201, 180, 171, 163, 154, 146, + 139, 133, 126, 120, 114, 108, 103, 97, 92, 87, 82, 77, 74, 70, 66, 62, + 62, 62, 62, 244, 246, 249, 252, 254, 234, 213, 193, 172, 163, 154, 145, + 136, 130, 124, 118, 111, 106, 101, 96, 91, 86, 82, 78, 73, 70, 67, 63, + 60, 60, 60, 60, 225, 229, 234, 238, 243, 223, 203, 184, 164, 154, 145, + 135, 126, 120, 114, 109, 103, 98, 94, 89, 85, 81, 77, 73, 70, 67, 64, + 61, 57, 57, 57, 57, 206, 212, 219, 225, 231, 213, 194, 175, 156, 146, + 136, 126, 116, 110, 105, 100, 94, 90, 87, 83, 79, 76, 72, 69, 66, 63, + 60, 58, 55, 55, 55, 55, 190, 197, 204, 210, 217, 200, 183, 166, 149, + 139, 130, 120, 110, 105, 100, 95, 89, 86, 82, 78, 75, 72, 69, 66, 63, + 60, 58, 55, 53, 53, 53, 53, 175, 182, 189, 196, 203, 187, 172, 157, 142, + 133, 124, 114, 105, 100, 95, 90, 84, 81, 77, 74, 71, 68, 65, 62, 60, 57, + 55, 53, 51, 51, 51, 51, 160, 167, 174, 181, 188, 175, 162, 149, 135, + 126, 118, 109, 100, 95, 90, 84, 79, 76, 73, 70, 66, 64, 61, 59, 57, 55, + 53, 51, 49, 49, 49, 49, 145, 152, 159, 166, 174, 162, 151, 140, 129, + 120, 111, 103, 94, 89, 84, 79, 74, 71, 68, 65, 62, 60, 58, 56, 53, 52, + 50, 48, 46, 46, 46, 46, 134, 141, 148, 155, 162, 152, 142, 132, 122, + 114, 106, 98, 90, 86, 81, 76, 71, 68, 66, 63, 60, 58, 55, 53, 51, 50, + 48, 46, 45, 45, 45, 45, 124, 131, 137, 144, 151, 142, 133, 124, 116, + 108, 101, 94, 87, 82, 77, 73, 68, 66, 63, 60, 57, 55, 53, 51, 49, 48, + 46, 45, 43, 43, 43, 43, 114, 120, 127, 133, 139, 132, 124, 117, 109, + 103, 96, 89, 83, 78, 74, 70, 65, 63, 60, 57, 54, 53, 51, 49, 47, 46, 44, + 43, 41, 41, 41, 41, 104, 110, 116, 122, 128, 121, 115, 109, 103, 97, 91, + 85, 79, 75, 71, 66, 62, 60, 57, 54, 52, 50, 48, 47, 45, 44, 42, 41, 40, + 40, 40, 40, 97, 103, 108, 114, 120, 114, 108, 103, 97, 92, 86, 81, 76, + 72, 68, 64, 60, 58, 55, 53, 50, 48, 47, 45, 43, 42, 41, 40, 38, 38, 38, + 38, 90, 95, 101, 106, 111, 107, 102, 97, 92, 87, 82, 77, 72, 69, 65, 61, + 58, 55, 53, 51, 48, 47, 45, 44, 42, 41, 40, 38, 37, 37, 37, 37, 83, 88, + 93, 98, 103, 99, 95, 91, 87, 82, 78, 73, 69, 66, 62, 59, 56, 53, 51, 49, + 47, 45, 44, 42, 40, 39, 38, 37, 36, 36, 36, 36, 77, 81, 86, 91, 95, 92, + 88, 85, 81, 77, 73, 70, 66, 63, 60, 57, 53, 51, 49, 47, 45, 43, 42, 40, + 39, 38, 37, 36, 35, 35, 35, 35, 72, 77, 81, 85, 90, 87, 83, 80, 77, 74, + 70, 67, 63, 60, 57, 55, 52, 50, 48, 46, 44, 42, 41, 39, 38, 37, 36, 35, + 34, 34, 34, 34, 68, 72, 76, 80, 84, 81, 79, 76, 73, 70, 67, 64, 60, 58, + 55, 53, 50, 48, 46, 44, 42, 41, 40, 38, 37, 36, 35, 34, 33, 33, 33, 33, + 63, 67, 71, 75, 78, 76, 74, 71, 69, 66, 63, 61, 58, 55, 53, 51, 48, 46, + 45, 43, 41, 40, 38, 37, 36, 35, 34, 33, 32, 32, 32, 32, 59, 62, 66, 69, + 73, 71, 69, 67, 65, 62, 60, 57, 55, 53, 51, 49, 46, 45, 43, 41, 40, 38, + 37, 36, 35, 34, 33, 32, 31, 31, 31, 31, 59, 62, 66, 69, 73, 71, 69, 67, + 65, 62, 60, 57, 55, 53, 51, 49, 46, 45, 43, 41, 40, 38, 37, 36, 35, 34, + 33, 32, 31, 31, 31, 31, 59, 62, 66, 69, 73, 71, 69, 67, 65, 62, 60, 57, + 55, 53, 51, 49, 46, 45, 43, 41, 40, 38, 37, 36, 35, 34, 33, 32, 31, 31, + 31, 31, 59, 62, 66, 69, 73, 71, 69, 67, 65, 62, 60, 57, 55, 53, 51, 49, + 46, 45, 43, 41, 40, 38, 37, 36, 35, 34, 33, 32, 31, 31, 31, 31 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 39, 35, 27, 39, 31, 29, 25, 35, 29, 21, 18, 27, 25, 18, 15, + /* Size 8 */ + 64, 81, 42, 39, 36, 32, 28, 24, 81, 54, 41, 46, 44, 40, 35, 30, 42, 41, + 34, 36, 36, 34, 31, 27, 39, 46, 36, 31, 29, 28, 26, 24, 36, 44, 36, 29, + 26, 24, 22, 20, 32, 40, 34, 28, 24, 21, 19, 18, 28, 35, 31, 26, 22, 19, + 17, 16, 24, 30, 27, 24, 20, 18, 16, 15, + /* Size 16 */ + 64, 72, 81, 61, 42, 41, 39, 38, 36, 34, 32, 30, 28, 26, 24, 24, 72, 70, + 67, 54, 42, 42, 43, 42, 40, 38, 36, 34, 31, 29, 27, 27, 81, 67, 54, 48, + 41, 44, 46, 45, 44, 42, 40, 37, 35, 32, 30, 30, 61, 54, 48, 43, 38, 39, + 41, 40, 40, 39, 37, 35, 33, 31, 29, 29, 42, 42, 41, 38, 34, 35, 36, 36, + 36, 35, 34, 32, 31, 29, 27, 27, 41, 42, 44, 39, 35, 34, 33, 33, 33, 32, + 31, 30, 28, 27, 25, 25, 39, 43, 46, 41, 36, 33, 31, 30, 29, 29, 28, 27, + 26, 25, 24, 24, 38, 42, 45, 40, 36, 33, 30, 29, 27, 27, 26, 25, 24, 23, + 22, 22, 36, 40, 44, 40, 36, 33, 29, 27, 26, 25, 24, 23, 22, 21, 20, 20, + 34, 38, 42, 39, 35, 32, 29, 27, 25, 23, 22, 21, 21, 20, 19, 19, 32, 36, + 40, 37, 34, 31, 28, 26, 24, 22, 21, 20, 19, 19, 18, 18, 30, 34, 37, 35, + 32, 30, 27, 25, 23, 21, 20, 19, 18, 18, 17, 17, 28, 31, 35, 33, 31, 28, + 26, 24, 22, 21, 19, 18, 17, 17, 16, 16, 26, 29, 32, 31, 29, 27, 25, 23, + 21, 20, 19, 18, 17, 16, 15, 15, 24, 27, 30, 29, 27, 25, 24, 22, 20, 19, + 18, 17, 16, 15, 15, 15, 24, 27, 30, 29, 27, 25, 24, 22, 20, 19, 18, 17, + 16, 15, 15, 15, + /* Size 32 */ + 64, 68, 72, 76, 81, 71, 61, 52, 42, 41, 41, 40, 39, 39, 38, 37, 36, 35, + 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 24, 24, 24, 68, 70, 71, 73, + 74, 66, 58, 50, 42, 42, 41, 41, 41, 40, 40, 39, 38, 37, 36, 35, 34, 33, + 32, 31, 29, 28, 27, 26, 25, 25, 25, 25, 72, 71, 70, 69, 67, 61, 54, 48, + 42, 42, 42, 42, 43, 42, 42, 41, 40, 39, 38, 37, 36, 35, 34, 32, 31, 30, + 29, 28, 27, 27, 27, 27, 76, 73, 69, 65, 61, 56, 51, 46, 41, 42, 43, 44, + 44, 44, 43, 43, 42, 41, 40, 39, 38, 37, 36, 34, 33, 32, 31, 30, 29, 29, + 29, 29, 81, 74, 67, 61, 54, 51, 48, 44, 41, 42, 44, 45, 46, 46, 45, 45, + 44, 43, 42, 41, 40, 39, 37, 36, 35, 34, 32, 31, 30, 30, 30, 30, 71, 66, + 61, 56, 51, 48, 45, 42, 39, 40, 41, 42, 43, 43, 43, 43, 42, 41, 40, 39, + 38, 37, 36, 35, 34, 33, 32, 30, 29, 29, 29, 29, 61, 58, 54, 51, 48, 45, + 43, 40, 38, 38, 39, 40, 41, 41, 40, 40, 40, 39, 39, 38, 37, 36, 35, 34, + 33, 32, 31, 30, 29, 29, 29, 29, 52, 50, 48, 46, 44, 42, 40, 38, 36, 37, + 37, 38, 38, 38, 38, 38, 38, 37, 37, 36, 35, 34, 33, 33, 32, 31, 30, 29, + 28, 28, 28, 28, 42, 42, 42, 41, 41, 39, 38, 36, 34, 35, 35, 35, 36, 36, + 36, 36, 36, 35, 35, 34, 34, 33, 32, 31, 31, 30, 29, 28, 27, 27, 27, 27, + 41, 42, 42, 42, 42, 40, 38, 37, 35, 35, 35, 35, 35, 34, 34, 34, 34, 34, + 33, 33, 32, 32, 31, 30, 29, 29, 28, 27, 26, 26, 26, 26, 41, 41, 42, 43, + 44, 41, 39, 37, 35, 35, 34, 34, 33, 33, 33, 33, 33, 32, 32, 31, 31, 30, + 30, 29, 28, 28, 27, 26, 25, 25, 25, 25, 40, 41, 42, 44, 45, 42, 40, 38, + 35, 35, 34, 33, 32, 32, 32, 31, 31, 30, 30, 30, 29, 29, 28, 28, 27, 26, + 26, 25, 25, 25, 25, 25, 39, 41, 43, 44, 46, 43, 41, 38, 36, 35, 33, 32, + 31, 31, 30, 30, 29, 29, 29, 28, 28, 27, 27, 26, 26, 25, 25, 24, 24, 24, + 24, 24, 39, 40, 42, 44, 46, 43, 41, 38, 36, 34, 33, 32, 31, 30, 29, 29, + 28, 28, 28, 27, 27, 26, 26, 25, 25, 24, 24, 23, 23, 23, 23, 23, 38, 40, + 42, 43, 45, 43, 40, 38, 36, 34, 33, 32, 30, 29, 29, 28, 27, 27, 27, 26, + 26, 25, 25, 24, 24, 23, 23, 23, 22, 22, 22, 22, 37, 39, 41, 43, 45, 43, + 40, 38, 36, 34, 33, 31, 30, 29, 28, 27, 27, 26, 26, 25, 25, 24, 24, 23, + 23, 23, 22, 22, 21, 21, 21, 21, 36, 38, 40, 42, 44, 42, 40, 38, 36, 34, + 33, 31, 29, 28, 27, 27, 26, 25, 25, 24, 24, 23, 23, 22, 22, 22, 21, 21, + 20, 20, 20, 20, 35, 37, 39, 41, 43, 41, 39, 37, 35, 34, 32, 30, 29, 28, + 27, 26, 25, 25, 24, 23, 23, 23, 22, 22, 21, 21, 21, 20, 20, 20, 20, 20, + 34, 36, 38, 40, 42, 40, 39, 37, 35, 33, 32, 30, 29, 28, 27, 26, 25, 24, + 23, 23, 22, 22, 21, 21, 21, 20, 20, 20, 19, 19, 19, 19, 33, 35, 37, 39, + 41, 39, 38, 36, 34, 33, 31, 30, 28, 27, 26, 25, 24, 23, 23, 22, 22, 21, + 21, 20, 20, 20, 19, 19, 19, 19, 19, 19, 32, 34, 36, 38, 40, 38, 37, 35, + 34, 32, 31, 29, 28, 27, 26, 25, 24, 23, 22, 22, 21, 21, 20, 20, 19, 19, + 19, 18, 18, 18, 18, 18, 31, 33, 35, 37, 39, 37, 36, 34, 33, 32, 30, 29, + 27, 26, 25, 24, 23, 23, 22, 21, 21, 20, 20, 19, 19, 18, 18, 18, 17, 17, + 17, 17, 30, 32, 34, 36, 37, 36, 35, 33, 32, 31, 30, 28, 27, 26, 25, 24, + 23, 22, 21, 21, 20, 20, 19, 19, 18, 18, 18, 17, 17, 17, 17, 17, 29, 31, + 32, 34, 36, 35, 34, 33, 31, 30, 29, 28, 26, 25, 24, 23, 22, 22, 21, 20, + 20, 19, 19, 18, 18, 17, 17, 17, 16, 16, 16, 16, 28, 29, 31, 33, 35, 34, + 33, 32, 31, 29, 28, 27, 26, 25, 24, 23, 22, 21, 21, 20, 19, 19, 18, 18, + 17, 17, 17, 16, 16, 16, 16, 16, 27, 28, 30, 32, 34, 33, 32, 31, 30, 29, + 28, 26, 25, 24, 23, 23, 22, 21, 20, 20, 19, 18, 18, 17, 17, 17, 16, 16, + 16, 16, 16, 16, 26, 27, 29, 31, 32, 32, 31, 30, 29, 28, 27, 26, 25, 24, + 23, 22, 21, 21, 20, 19, 19, 18, 18, 17, 17, 16, 16, 16, 15, 15, 15, 15, + 25, 26, 28, 30, 31, 30, 30, 29, 28, 27, 26, 25, 24, 23, 23, 22, 21, 20, + 20, 19, 18, 18, 17, 17, 16, 16, 16, 15, 15, 15, 15, 15, 24, 25, 27, 29, + 30, 29, 29, 28, 27, 26, 25, 25, 24, 23, 22, 21, 20, 20, 19, 19, 18, 17, + 17, 16, 16, 16, 15, 15, 15, 15, 15, 15, 24, 25, 27, 29, 30, 29, 29, 28, + 27, 26, 25, 25, 24, 23, 22, 21, 20, 20, 19, 19, 18, 17, 17, 16, 16, 16, + 15, 15, 15, 15, 15, 15, 24, 25, 27, 29, 30, 29, 29, 28, 27, 26, 25, 25, + 24, 23, 22, 21, 20, 20, 19, 19, 18, 17, 17, 16, 16, 16, 15, 15, 15, 15, + 15, 15, 24, 25, 27, 29, 30, 29, 29, 28, 27, 26, 25, 25, 24, 23, 22, 21, + 20, 20, 19, 19, 18, 17, 17, 16, 16, 16, 15, 15, 15, 15, 15, 15 }, + { /* Intra matrices */ + /* Size 4 */ + 175, 103, 93, 70, 103, 83, 76, 64, 93, 76, 55, 46, 70, 64, 46, 36, + /* Size 8 */ + 162, 205, 104, 97, 90, 78, 67, 57, 205, 136, 102, 115, 111, 99, 86, 73, + 104, 102, 84, 88, 88, 83, 75, 66, 97, 115, 88, 75, 71, 67, 62, 56, 90, + 111, 88, 71, 62, 56, 52, 48, 78, 99, 83, 67, 56, 49, 45, 41, 67, 86, 75, + 62, 52, 45, 40, 36, 57, 73, 66, 56, 48, 41, 36, 33, + /* Size 16 */ + 168, 190, 213, 160, 108, 104, 101, 97, 93, 87, 81, 75, 69, 64, 59, 59, + 190, 184, 177, 142, 107, 108, 110, 107, 104, 98, 92, 85, 79, 73, 67, 67, + 213, 177, 141, 123, 106, 112, 119, 117, 115, 109, 103, 96, 89, 82, 76, + 76, 160, 142, 123, 110, 96, 101, 105, 104, 103, 99, 94, 89, 83, 77, 72, + 72, 108, 107, 106, 96, 87, 89, 91, 91, 91, 89, 86, 82, 77, 73, 68, 68, + 104, 108, 112, 101, 89, 87, 85, 84, 82, 80, 78, 74, 71, 67, 63, 63, 101, + 110, 119, 105, 91, 85, 78, 76, 74, 72, 70, 67, 64, 61, 58, 58, 97, 107, + 117, 104, 91, 84, 76, 72, 69, 66, 64, 62, 59, 57, 54, 54, 93, 104, 115, + 103, 91, 82, 74, 69, 64, 61, 58, 56, 54, 52, 50, 50, 87, 98, 109, 99, + 89, 80, 72, 66, 61, 58, 55, 52, 50, 48, 46, 46, 81, 92, 103, 94, 86, 78, + 70, 64, 58, 55, 51, 49, 46, 45, 43, 43, 75, 85, 96, 89, 82, 74, 67, 62, + 56, 52, 49, 46, 44, 42, 40, 40, 69, 79, 89, 83, 77, 71, 64, 59, 54, 50, + 46, 44, 41, 40, 38, 38, 64, 73, 82, 77, 73, 67, 61, 57, 52, 48, 45, 42, + 40, 38, 36, 36, 59, 67, 76, 72, 68, 63, 58, 54, 50, 46, 43, 40, 38, 36, + 34, 34, 59, 67, 76, 72, 68, 63, 58, 54, 50, 46, 43, 40, 38, 36, 34, 34, + /* Size 32 */ + 171, 182, 194, 205, 217, 190, 163, 137, 110, 108, 106, 104, 103, 101, + 99, 97, 95, 92, 89, 86, 83, 80, 77, 74, 71, 68, 65, 63, 60, 60, 60, 60, + 182, 186, 190, 194, 198, 176, 154, 132, 109, 109, 108, 108, 107, 105, + 104, 102, 100, 97, 94, 91, 88, 85, 82, 79, 76, 73, 70, 67, 64, 64, 64, + 64, 194, 190, 187, 184, 180, 162, 144, 127, 109, 110, 110, 111, 112, + 110, 109, 107, 106, 103, 100, 97, 94, 90, 87, 84, 80, 78, 75, 72, 69, + 69, 69, 69, 205, 194, 184, 173, 162, 148, 135, 122, 108, 110, 112, 114, + 116, 115, 114, 113, 111, 108, 105, 102, 99, 96, 92, 89, 85, 82, 79, 76, + 73, 73, 73, 73, 217, 198, 180, 162, 144, 135, 126, 117, 108, 111, 114, + 118, 121, 120, 119, 118, 117, 114, 111, 108, 104, 101, 97, 94, 90, 87, + 84, 80, 77, 77, 77, 77, 190, 176, 162, 148, 135, 127, 119, 111, 103, + 106, 108, 111, 114, 113, 112, 112, 111, 108, 106, 103, 100, 97, 94, 91, + 87, 84, 81, 78, 75, 75, 75, 75, 163, 154, 144, 135, 126, 119, 112, 105, + 98, 100, 103, 105, 107, 106, 106, 105, 105, 103, 101, 98, 96, 93, 90, + 87, 85, 82, 79, 76, 73, 73, 73, 73, 137, 132, 127, 122, 117, 111, 105, + 99, 94, 95, 97, 98, 100, 100, 99, 99, 99, 97, 95, 94, 92, 89, 87, 84, + 82, 79, 76, 74, 71, 71, 71, 71, 110, 109, 109, 108, 108, 103, 98, 94, + 89, 90, 91, 92, 93, 93, 93, 93, 93, 92, 90, 89, 87, 85, 83, 81, 79, 76, + 74, 72, 69, 69, 69, 69, 108, 109, 110, 110, 111, 106, 100, 95, 90, 90, + 90, 90, 89, 89, 89, 89, 89, 87, 86, 85, 83, 81, 79, 77, 75, 73, 71, 69, + 67, 67, 67, 67, 106, 108, 110, 112, 114, 108, 103, 97, 91, 90, 89, 87, + 86, 86, 85, 85, 84, 83, 82, 80, 79, 77, 76, 74, 72, 70, 68, 66, 64, 64, + 64, 64, 104, 108, 111, 114, 118, 111, 105, 98, 92, 90, 87, 85, 83, 82, + 81, 80, 79, 78, 77, 76, 75, 74, 72, 70, 69, 67, 65, 64, 62, 62, 62, 62, + 103, 107, 112, 116, 121, 114, 107, 100, 93, 89, 86, 83, 80, 78, 77, 76, + 75, 74, 73, 72, 71, 70, 68, 67, 66, 64, 63, 61, 60, 60, 60, 60, 101, + 105, 110, 115, 120, 113, 106, 100, 93, 89, 86, 82, 78, 77, 75, 74, 72, + 71, 70, 69, 68, 67, 65, 64, 63, 62, 60, 59, 57, 57, 57, 57, 99, 104, + 109, 114, 119, 112, 106, 99, 93, 89, 85, 81, 77, 75, 74, 72, 70, 69, 68, + 66, 65, 64, 63, 61, 60, 59, 58, 56, 55, 55, 55, 55, 97, 102, 107, 113, + 118, 112, 105, 99, 93, 89, 85, 80, 76, 74, 72, 70, 67, 66, 65, 64, 62, + 61, 60, 59, 58, 56, 55, 54, 53, 53, 53, 53, 95, 100, 106, 111, 117, 111, + 105, 99, 93, 89, 84, 79, 75, 72, 70, 67, 65, 64, 62, 61, 59, 58, 57, 56, + 55, 54, 53, 52, 51, 51, 51, 51, 92, 97, 103, 108, 114, 108, 103, 97, 92, + 87, 83, 78, 74, 71, 69, 66, 64, 62, 61, 59, 57, 56, 55, 54, 53, 52, 51, + 50, 49, 49, 49, 49, 89, 94, 100, 105, 111, 106, 101, 95, 90, 86, 82, 77, + 73, 70, 68, 65, 62, 61, 59, 57, 56, 55, 53, 52, 51, 50, 49, 48, 47, 47, + 47, 47, 86, 91, 97, 102, 108, 103, 98, 94, 89, 85, 80, 76, 72, 69, 66, + 64, 61, 59, 57, 56, 54, 53, 52, 50, 49, 48, 47, 46, 45, 45, 45, 45, 83, + 88, 94, 99, 104, 100, 96, 92, 87, 83, 79, 75, 71, 68, 65, 62, 59, 57, + 56, 54, 52, 51, 50, 48, 47, 46, 45, 44, 44, 44, 44, 44, 80, 85, 90, 96, + 101, 97, 93, 89, 85, 81, 77, 74, 70, 67, 64, 61, 58, 56, 55, 53, 51, 50, + 48, 47, 46, 45, 44, 43, 42, 42, 42, 42, 77, 82, 87, 92, 97, 94, 90, 87, + 83, 79, 76, 72, 68, 65, 63, 60, 57, 55, 53, 52, 50, 48, 47, 46, 45, 44, + 43, 42, 41, 41, 41, 41, 74, 79, 84, 89, 94, 91, 87, 84, 81, 77, 74, 70, + 67, 64, 61, 59, 56, 54, 52, 50, 48, 47, 46, 45, 43, 42, 42, 41, 40, 40, + 40, 40, 71, 76, 80, 85, 90, 87, 85, 82, 79, 75, 72, 69, 66, 63, 60, 58, + 55, 53, 51, 49, 47, 46, 45, 43, 42, 41, 40, 39, 38, 38, 38, 38, 68, 73, + 78, 82, 87, 84, 82, 79, 76, 73, 70, 67, 64, 62, 59, 56, 54, 52, 50, 48, + 46, 45, 44, 42, 41, 40, 39, 38, 37, 37, 37, 37, 65, 70, 75, 79, 84, 81, + 79, 76, 74, 71, 68, 65, 63, 60, 58, 55, 53, 51, 49, 47, 45, 44, 43, 42, + 40, 39, 38, 37, 37, 37, 37, 37, 63, 67, 72, 76, 80, 78, 76, 74, 72, 69, + 66, 64, 61, 59, 56, 54, 52, 50, 48, 46, 44, 43, 42, 41, 39, 38, 37, 37, + 36, 36, 36, 36, 60, 64, 69, 73, 77, 75, 73, 71, 69, 67, 64, 62, 60, 57, + 55, 53, 51, 49, 47, 45, 44, 42, 41, 40, 38, 37, 37, 36, 35, 35, 35, 35, + 60, 64, 69, 73, 77, 75, 73, 71, 69, 67, 64, 62, 60, 57, 55, 53, 51, 49, + 47, 45, 44, 42, 41, 40, 38, 37, 37, 36, 35, 35, 35, 35, 60, 64, 69, 73, + 77, 75, 73, 71, 69, 67, 64, 62, 60, 57, 55, 53, 51, 49, 47, 45, 44, 42, + 41, 40, 38, 37, 37, 36, 35, 35, 35, 35, 60, 64, 69, 73, 77, 75, 73, 71, + 69, 67, 64, 62, 60, 57, 55, 53, 51, 49, 47, 45, 44, 42, 41, 40, 38, 37, + 37, 36, 35, 35, 35, 35 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 58, 34, 21, 58, 38, 26, 19, 34, 26, 18, 14, 21, 19, 14, 12, + /* Size 8 */ + 64, 86, 80, 59, 43, 33, 25, 21, 86, 75, 78, 66, 51, 39, 30, 24, 80, 78, + 55, 46, 39, 32, 27, 22, 59, 66, 46, 36, 30, 26, 23, 20, 43, 51, 39, 30, + 25, 22, 19, 17, 33, 39, 32, 26, 22, 19, 17, 16, 25, 30, 27, 23, 19, 17, + 16, 14, 21, 24, 22, 20, 17, 16, 14, 14, + /* Size 16 */ + 64, 75, 86, 83, 80, 70, 59, 51, 43, 38, 33, 29, 25, 23, 21, 21, 75, 78, + 81, 80, 79, 71, 63, 55, 47, 41, 36, 32, 28, 25, 23, 23, 86, 81, 75, 77, + 78, 72, 66, 59, 51, 45, 39, 35, 30, 27, 24, 24, 83, 80, 77, 72, 67, 61, + 56, 51, 45, 40, 36, 32, 29, 26, 23, 23, 80, 79, 78, 67, 55, 51, 46, 43, + 39, 36, 32, 29, 27, 24, 22, 22, 70, 71, 72, 61, 51, 46, 41, 38, 35, 32, + 29, 27, 25, 23, 21, 21, 59, 63, 66, 56, 46, 41, 36, 33, 30, 28, 26, 24, + 23, 21, 20, 20, 51, 55, 59, 51, 43, 38, 33, 30, 28, 26, 24, 22, 21, 20, + 19, 19, 43, 47, 51, 45, 39, 35, 30, 28, 25, 23, 22, 21, 19, 18, 17, 17, + 38, 41, 45, 40, 36, 32, 28, 26, 23, 22, 20, 19, 18, 17, 17, 17, 33, 36, + 39, 36, 32, 29, 26, 24, 22, 20, 19, 18, 17, 16, 16, 16, 29, 32, 35, 32, + 29, 27, 24, 22, 21, 19, 18, 17, 16, 16, 15, 15, 25, 28, 30, 29, 27, 25, + 23, 21, 19, 18, 17, 16, 16, 15, 14, 14, 23, 25, 27, 26, 24, 23, 21, 20, + 18, 17, 16, 16, 15, 14, 14, 14, 21, 23, 24, 23, 22, 21, 20, 19, 17, 17, + 16, 15, 14, 14, 14, 14, 21, 23, 24, 23, 22, 21, 20, 19, 17, 17, 16, 15, + 14, 14, 14, 14, + /* Size 32 */ + 64, 70, 75, 81, 86, 85, 83, 81, 80, 75, 70, 64, 59, 55, 51, 47, 43, 41, + 38, 35, 33, 31, 29, 27, 25, 24, 23, 22, 21, 21, 21, 21, 70, 73, 77, 80, + 84, 82, 81, 80, 79, 75, 70, 66, 61, 57, 53, 49, 45, 43, 40, 37, 34, 32, + 30, 29, 27, 25, 24, 23, 22, 22, 22, 22, 75, 77, 78, 79, 81, 80, 80, 79, + 79, 75, 71, 67, 63, 59, 55, 51, 47, 44, 41, 39, 36, 34, 32, 30, 28, 27, + 25, 24, 23, 23, 23, 23, 81, 80, 79, 78, 78, 78, 78, 78, 79, 75, 72, 68, + 65, 61, 57, 53, 49, 46, 43, 40, 37, 35, 33, 31, 29, 28, 26, 25, 24, 24, + 24, 24, 86, 84, 81, 78, 75, 76, 77, 77, 78, 75, 72, 69, 66, 62, 59, 55, + 51, 48, 45, 42, 39, 37, 35, 33, 30, 29, 27, 26, 24, 24, 24, 24, 85, 82, + 80, 78, 76, 75, 74, 73, 72, 70, 67, 64, 61, 58, 55, 51, 48, 45, 43, 40, + 37, 35, 33, 31, 29, 28, 27, 25, 24, 24, 24, 24, 83, 81, 80, 78, 77, 74, + 72, 69, 67, 64, 61, 59, 56, 54, 51, 48, 45, 43, 40, 38, 36, 34, 32, 30, + 29, 27, 26, 25, 23, 23, 23, 23, 81, 80, 79, 78, 77, 73, 69, 65, 61, 58, + 56, 54, 51, 49, 47, 44, 42, 40, 38, 36, 34, 32, 31, 29, 28, 26, 25, 24, + 23, 23, 23, 23, 80, 79, 79, 79, 78, 72, 67, 61, 55, 53, 51, 48, 46, 45, + 43, 41, 39, 37, 36, 34, 32, 31, 29, 28, 27, 26, 24, 23, 22, 22, 22, 22, + 75, 75, 75, 75, 75, 70, 64, 58, 53, 50, 48, 46, 44, 42, 40, 39, 37, 35, + 34, 32, 31, 29, 28, 27, 26, 25, 24, 23, 22, 22, 22, 22, 70, 70, 71, 72, + 72, 67, 61, 56, 51, 48, 46, 43, 41, 39, 38, 36, 35, 33, 32, 31, 29, 28, + 27, 26, 25, 24, 23, 22, 21, 21, 21, 21, 64, 66, 67, 68, 69, 64, 59, 54, + 48, 46, 43, 41, 38, 37, 35, 34, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, + 22, 21, 20, 20, 20, 20, 59, 61, 63, 65, 66, 61, 56, 51, 46, 44, 41, 38, + 36, 34, 33, 32, 30, 29, 28, 27, 26, 25, 24, 23, 23, 22, 21, 20, 20, 20, + 20, 20, 55, 57, 59, 61, 62, 58, 54, 49, 45, 42, 39, 37, 34, 33, 32, 30, + 29, 28, 27, 26, 25, 24, 23, 23, 22, 21, 20, 20, 19, 19, 19, 19, 51, 53, + 55, 57, 59, 55, 51, 47, 43, 40, 38, 35, 33, 32, 30, 29, 28, 27, 26, 25, + 24, 23, 22, 22, 21, 20, 20, 19, 19, 19, 19, 19, 47, 49, 51, 53, 55, 51, + 48, 44, 41, 39, 36, 34, 32, 30, 29, 28, 26, 25, 24, 24, 23, 22, 21, 21, + 20, 20, 19, 19, 18, 18, 18, 18, 43, 45, 47, 49, 51, 48, 45, 42, 39, 37, + 35, 32, 30, 29, 28, 26, 25, 24, 23, 22, 22, 21, 21, 20, 19, 19, 18, 18, + 17, 17, 17, 17, 41, 43, 44, 46, 48, 45, 43, 40, 37, 35, 33, 31, 29, 28, + 27, 25, 24, 23, 23, 22, 21, 20, 20, 19, 19, 18, 18, 17, 17, 17, 17, 17, + 38, 40, 41, 43, 45, 43, 40, 38, 36, 34, 32, 30, 28, 27, 26, 24, 23, 23, + 22, 21, 20, 20, 19, 19, 18, 18, 17, 17, 17, 17, 17, 17, 35, 37, 39, 40, + 42, 40, 38, 36, 34, 32, 31, 29, 27, 26, 25, 24, 22, 22, 21, 20, 20, 19, + 19, 18, 18, 17, 17, 17, 16, 16, 16, 16, 33, 34, 36, 37, 39, 37, 36, 34, + 32, 31, 29, 28, 26, 25, 24, 23, 22, 21, 20, 20, 19, 18, 18, 18, 17, 17, + 16, 16, 16, 16, 16, 16, 31, 32, 34, 35, 37, 35, 34, 32, 31, 29, 28, 27, + 25, 24, 23, 22, 21, 20, 20, 19, 18, 18, 18, 17, 17, 16, 16, 16, 15, 15, + 15, 15, 29, 30, 32, 33, 35, 33, 32, 31, 29, 28, 27, 26, 24, 23, 22, 21, + 21, 20, 19, 19, 18, 18, 17, 17, 16, 16, 16, 15, 15, 15, 15, 15, 27, 29, + 30, 31, 33, 31, 30, 29, 28, 27, 26, 25, 23, 23, 22, 21, 20, 19, 19, 18, + 18, 17, 17, 16, 16, 16, 15, 15, 15, 15, 15, 15, 25, 27, 28, 29, 30, 29, + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 19, 18, 18, 17, 17, 16, 16, + 16, 15, 15, 15, 14, 14, 14, 14, 24, 25, 27, 28, 29, 28, 27, 26, 26, 25, + 24, 23, 22, 21, 20, 20, 19, 18, 18, 17, 17, 16, 16, 16, 15, 15, 15, 14, + 14, 14, 14, 14, 23, 24, 25, 26, 27, 27, 26, 25, 24, 24, 23, 22, 21, 20, + 20, 19, 18, 18, 17, 17, 16, 16, 16, 15, 15, 15, 14, 14, 14, 14, 14, 14, + 22, 23, 24, 25, 26, 25, 25, 24, 23, 23, 22, 21, 20, 20, 19, 19, 18, 17, + 17, 17, 16, 16, 15, 15, 15, 14, 14, 14, 14, 14, 14, 14, 21, 22, 23, 24, + 24, 24, 23, 23, 22, 22, 21, 20, 20, 19, 19, 18, 17, 17, 17, 16, 16, 15, + 15, 15, 14, 14, 14, 14, 14, 14, 14, 14, 21, 22, 23, 24, 24, 24, 23, 23, + 22, 22, 21, 20, 20, 19, 19, 18, 17, 17, 17, 16, 16, 15, 15, 15, 14, 14, + 14, 14, 14, 14, 14, 14, 21, 22, 23, 24, 24, 24, 23, 23, 22, 22, 21, 20, + 20, 19, 19, 18, 17, 17, 17, 16, 16, 15, 15, 15, 14, 14, 14, 14, 14, 14, + 14, 14, 21, 22, 23, 24, 24, 24, 23, 23, 22, 22, 21, 20, 20, 19, 19, 18, + 17, 17, 17, 16, 16, 15, 15, 15, 14, 14, 14, 14, 14, 14, 14, 14 }, + { /* Intra matrices */ + /* Size 4 */ + 217, 196, 112, 65, 196, 124, 84, 58, 112, 84, 55, 43, 65, 58, 43, 34, + /* Size 8 */ + 180, 246, 226, 167, 120, 88, 67, 53, 246, 212, 222, 187, 142, 107, 81, + 64, 226, 222, 153, 128, 107, 87, 71, 58, 167, 187, 128, 97, 81, 69, 59, + 50, 120, 142, 107, 81, 65, 56, 49, 44, 88, 107, 87, 69, 56, 48, 43, 39, + 67, 81, 71, 59, 49, 43, 38, 35, 53, 64, 58, 50, 44, 39, 35, 32, + /* Size 16 */ + 188, 223, 257, 246, 236, 205, 174, 150, 125, 108, 92, 81, 70, 63, 56, + 56, 223, 231, 239, 237, 234, 209, 185, 161, 137, 119, 102, 90, 78, 70, + 62, 62, 257, 239, 221, 227, 232, 214, 195, 172, 148, 130, 111, 98, 85, + 76, 67, 67, 246, 237, 227, 211, 196, 180, 165, 148, 130, 116, 101, 90, + 80, 72, 64, 64, 236, 234, 232, 196, 160, 147, 134, 123, 112, 102, 91, + 83, 74, 67, 61, 61, 205, 209, 214, 180, 147, 133, 118, 108, 98, 90, 82, + 75, 68, 62, 57, 57, 174, 185, 195, 165, 134, 118, 102, 93, 84, 78, 72, + 67, 61, 57, 53, 53, 150, 161, 172, 148, 123, 108, 93, 85, 76, 71, 65, + 61, 56, 53, 49, 49, 125, 137, 148, 130, 112, 98, 84, 76, 68, 64, 59, 55, + 51, 49, 46, 46, 108, 119, 130, 116, 102, 90, 78, 71, 64, 59, 54, 51, 48, + 46, 43, 43, 92, 102, 111, 101, 91, 82, 72, 65, 59, 54, 50, 47, 45, 42, + 40, 40, 81, 90, 98, 90, 83, 75, 67, 61, 55, 51, 47, 45, 42, 40, 38, 38, + 70, 78, 85, 80, 74, 68, 61, 56, 51, 48, 45, 42, 40, 38, 36, 36, 63, 70, + 76, 72, 67, 62, 57, 53, 49, 46, 42, 40, 38, 37, 35, 35, 56, 62, 67, 64, + 61, 57, 53, 49, 46, 43, 40, 38, 36, 35, 34, 34, 56, 62, 67, 64, 61, 57, + 53, 49, 46, 43, 40, 38, 36, 35, 34, 34, + /* Size 32 */ + 193, 210, 228, 245, 263, 258, 252, 247, 241, 226, 210, 194, 178, 166, + 153, 141, 128, 120, 111, 103, 94, 89, 83, 77, 72, 68, 65, 61, 57, 57, + 57, 57, 210, 221, 232, 243, 254, 250, 247, 244, 240, 226, 212, 198, 184, + 171, 159, 146, 134, 125, 117, 108, 99, 93, 87, 82, 76, 72, 68, 64, 60, + 60, 60, 60, 228, 232, 236, 240, 245, 243, 242, 241, 239, 227, 214, 202, + 189, 177, 165, 152, 140, 131, 122, 113, 104, 98, 92, 86, 80, 75, 71, 67, + 63, 63, 63, 63, 245, 243, 240, 238, 236, 236, 237, 238, 238, 227, 216, + 205, 194, 182, 170, 158, 146, 137, 127, 118, 109, 103, 96, 90, 83, 79, + 75, 70, 66, 66, 66, 66, 263, 254, 245, 236, 227, 229, 232, 235, 238, + 228, 219, 209, 200, 188, 176, 164, 152, 142, 133, 123, 114, 107, 101, + 94, 87, 83, 78, 73, 69, 69, 69, 69, 258, 250, 243, 236, 229, 227, 224, + 222, 219, 210, 202, 193, 184, 174, 163, 153, 143, 134, 126, 117, 109, + 103, 97, 90, 84, 80, 76, 71, 67, 67, 67, 67, 252, 247, 242, 237, 232, + 224, 216, 208, 201, 193, 185, 177, 169, 160, 151, 142, 133, 126, 118, + 111, 104, 98, 93, 87, 81, 77, 73, 69, 65, 65, 65, 65, 247, 244, 241, + 238, 235, 222, 208, 195, 182, 175, 168, 160, 153, 146, 139, 131, 124, + 118, 111, 105, 98, 93, 88, 83, 78, 75, 71, 67, 64, 64, 64, 64, 241, 240, + 239, 238, 238, 219, 201, 182, 164, 157, 151, 144, 137, 132, 126, 120, + 115, 109, 104, 99, 93, 89, 84, 80, 76, 72, 69, 65, 62, 62, 62, 62, 226, + 226, 227, 227, 228, 210, 193, 175, 157, 150, 143, 136, 129, 124, 118, + 113, 108, 103, 98, 93, 88, 84, 80, 76, 72, 69, 66, 63, 60, 60, 60, 60, + 210, 212, 214, 216, 219, 202, 185, 168, 151, 143, 136, 128, 121, 116, + 111, 106, 101, 96, 92, 88, 83, 80, 76, 73, 69, 66, 64, 61, 58, 58, 58, + 58, 194, 198, 202, 205, 209, 193, 177, 160, 144, 136, 128, 120, 112, + 108, 103, 98, 94, 90, 86, 82, 79, 75, 72, 69, 66, 64, 61, 59, 56, 56, + 56, 56, 178, 184, 189, 194, 200, 184, 169, 153, 137, 129, 121, 112, 104, + 100, 95, 91, 86, 83, 80, 77, 74, 71, 68, 66, 63, 61, 58, 56, 54, 54, 54, + 54, 166, 171, 177, 182, 188, 174, 160, 146, 132, 124, 116, 108, 100, 95, + 91, 87, 82, 79, 76, 73, 70, 68, 65, 63, 60, 58, 56, 54, 52, 52, 52, 52, + 153, 159, 165, 170, 176, 163, 151, 139, 126, 118, 111, 103, 95, 91, 87, + 82, 78, 75, 73, 70, 67, 65, 62, 60, 58, 56, 54, 52, 50, 50, 50, 50, 141, + 146, 152, 158, 164, 153, 142, 131, 120, 113, 106, 98, 91, 87, 82, 78, + 74, 71, 69, 66, 63, 61, 59, 57, 55, 54, 52, 50, 49, 49, 49, 49, 128, + 134, 140, 146, 152, 143, 133, 124, 115, 108, 101, 94, 86, 82, 78, 74, + 70, 68, 65, 63, 60, 58, 56, 54, 53, 51, 50, 48, 47, 47, 47, 47, 120, + 125, 131, 137, 142, 134, 126, 118, 109, 103, 96, 90, 83, 79, 75, 71, 68, + 65, 63, 60, 58, 56, 54, 53, 51, 50, 48, 47, 45, 45, 45, 45, 111, 117, + 122, 127, 133, 126, 118, 111, 104, 98, 92, 86, 80, 76, 73, 69, 65, 63, + 60, 58, 56, 54, 52, 51, 49, 48, 47, 45, 44, 44, 44, 44, 103, 108, 113, + 118, 123, 117, 111, 105, 99, 93, 88, 82, 77, 73, 70, 66, 63, 60, 58, 56, + 53, 52, 50, 49, 47, 46, 45, 44, 43, 43, 43, 43, 94, 99, 104, 109, 114, + 109, 104, 98, 93, 88, 83, 79, 74, 70, 67, 63, 60, 58, 56, 53, 51, 50, + 48, 47, 46, 44, 43, 42, 41, 41, 41, 41, 89, 93, 98, 103, 107, 103, 98, + 93, 89, 84, 80, 75, 71, 68, 65, 61, 58, 56, 54, 52, 50, 49, 47, 46, 44, + 43, 42, 41, 40, 40, 40, 40, 83, 87, 92, 96, 101, 97, 93, 88, 84, 80, 76, + 72, 68, 65, 62, 59, 56, 54, 52, 50, 48, 47, 46, 44, 43, 42, 41, 40, 39, + 39, 39, 39, 77, 82, 86, 90, 94, 90, 87, 83, 80, 76, 73, 69, 66, 63, 60, + 57, 54, 53, 51, 49, 47, 46, 44, 43, 42, 41, 40, 39, 38, 38, 38, 38, 72, + 76, 80, 83, 87, 84, 81, 78, 76, 72, 69, 66, 63, 60, 58, 55, 53, 51, 49, + 47, 46, 44, 43, 42, 41, 40, 39, 38, 37, 37, 37, 37, 68, 72, 75, 79, 83, + 80, 77, 75, 72, 69, 66, 64, 61, 58, 56, 54, 51, 50, 48, 46, 44, 43, 42, + 41, 40, 39, 38, 37, 37, 37, 37, 37, 65, 68, 71, 75, 78, 76, 73, 71, 69, + 66, 64, 61, 58, 56, 54, 52, 50, 48, 47, 45, 43, 42, 41, 40, 39, 38, 37, + 37, 36, 36, 36, 36, 61, 64, 67, 70, 73, 71, 69, 67, 65, 63, 61, 59, 56, + 54, 52, 50, 48, 47, 45, 44, 42, 41, 40, 39, 38, 37, 37, 36, 35, 35, 35, + 35, 57, 60, 63, 66, 69, 67, 65, 64, 62, 60, 58, 56, 54, 52, 50, 49, 47, + 45, 44, 43, 41, 40, 39, 38, 37, 37, 36, 35, 34, 34, 34, 34, 57, 60, 63, + 66, 69, 67, 65, 64, 62, 60, 58, 56, 54, 52, 50, 49, 47, 45, 44, 43, 41, + 40, 39, 38, 37, 37, 36, 35, 34, 34, 34, 34, 57, 60, 63, 66, 69, 67, 65, + 64, 62, 60, 58, 56, 54, 52, 50, 49, 47, 45, 44, 43, 41, 40, 39, 38, 37, + 37, 36, 35, 34, 34, 34, 34, 57, 60, 63, 66, 69, 67, 65, 64, 62, 60, 58, + 56, 54, 52, 50, 49, 47, 45, 44, 43, 41, 40, 39, 38, 37, 37, 36, 35, 34, + 34, 34, 34 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 39, 36, 28, 39, 32, 30, 26, 36, 30, 23, 20, 28, 26, 20, 16, + /* Size 8 */ + 64, 80, 43, 40, 37, 33, 29, 25, 80, 55, 42, 47, 45, 41, 36, 31, 43, 42, + 35, 37, 37, 35, 32, 29, 40, 47, 37, 32, 31, 29, 27, 25, 37, 45, 37, 31, + 27, 25, 24, 22, 33, 41, 35, 29, 25, 23, 21, 20, 29, 36, 32, 27, 24, 21, + 19, 18, 25, 31, 29, 25, 22, 20, 18, 17, + /* Size 16 */ + 64, 72, 80, 61, 43, 41, 40, 39, 37, 35, 33, 31, 29, 27, 25, 25, 72, 70, + 67, 55, 42, 43, 43, 42, 41, 39, 37, 35, 32, 30, 28, 28, 80, 67, 55, 48, + 42, 44, 47, 46, 45, 43, 41, 38, 36, 34, 31, 31, 61, 55, 48, 43, 39, 40, + 42, 41, 41, 39, 38, 36, 34, 32, 30, 30, 43, 42, 42, 39, 35, 36, 37, 37, + 37, 36, 35, 33, 32, 30, 29, 29, 41, 43, 44, 40, 36, 35, 34, 34, 34, 33, + 32, 31, 30, 28, 27, 27, 40, 43, 47, 42, 37, 34, 32, 31, 31, 30, 29, 28, + 27, 26, 25, 25, 39, 42, 46, 41, 37, 34, 31, 30, 29, 28, 27, 26, 25, 25, + 24, 24, 37, 41, 45, 41, 37, 34, 31, 29, 27, 26, 25, 24, 24, 23, 22, 22, + 35, 39, 43, 39, 36, 33, 30, 28, 26, 25, 24, 23, 22, 22, 21, 21, 33, 37, + 41, 38, 35, 32, 29, 27, 25, 24, 23, 22, 21, 20, 20, 20, 31, 35, 38, 36, + 33, 31, 28, 26, 24, 23, 22, 21, 20, 19, 19, 19, 29, 32, 36, 34, 32, 30, + 27, 25, 24, 22, 21, 20, 19, 18, 18, 18, 27, 30, 34, 32, 30, 28, 26, 25, + 23, 22, 20, 19, 18, 18, 17, 17, 25, 28, 31, 30, 29, 27, 25, 24, 22, 21, + 20, 19, 18, 17, 17, 17, 25, 28, 31, 30, 29, 27, 25, 24, 22, 21, 20, 19, + 18, 17, 17, 17, + /* Size 32 */ + 64, 68, 72, 76, 80, 71, 61, 52, 43, 42, 41, 41, 40, 39, 39, 38, 37, 36, + 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 25, 25, 25, 68, 69, 71, 72, + 74, 66, 58, 50, 43, 42, 42, 42, 42, 41, 41, 40, 39, 38, 37, 36, 35, 34, + 33, 32, 31, 30, 29, 28, 27, 27, 27, 27, 72, 71, 70, 68, 67, 61, 55, 49, + 42, 43, 43, 43, 43, 43, 42, 42, 41, 40, 39, 38, 37, 36, 35, 34, 32, 31, + 30, 29, 28, 28, 28, 28, 76, 72, 68, 65, 61, 56, 52, 47, 42, 43, 44, 44, + 45, 45, 44, 44, 43, 42, 41, 40, 39, 38, 37, 35, 34, 33, 32, 31, 30, 30, + 30, 30, 80, 74, 67, 61, 55, 51, 48, 45, 42, 43, 44, 45, 47, 46, 46, 46, + 45, 44, 43, 42, 41, 40, 38, 37, 36, 35, 34, 33, 31, 31, 31, 31, 71, 66, + 61, 56, 51, 49, 46, 43, 40, 41, 42, 43, 44, 44, 44, 43, 43, 42, 41, 40, + 39, 38, 37, 36, 35, 34, 33, 32, 31, 31, 31, 31, 61, 58, 55, 52, 48, 46, + 43, 41, 39, 39, 40, 41, 42, 42, 41, 41, 41, 40, 39, 39, 38, 37, 36, 35, + 34, 33, 32, 31, 30, 30, 30, 30, 52, 50, 49, 47, 45, 43, 41, 39, 37, 38, + 38, 39, 39, 39, 39, 39, 39, 38, 38, 37, 36, 36, 35, 34, 33, 32, 31, 30, + 29, 29, 29, 29, 43, 43, 42, 42, 42, 40, 39, 37, 35, 36, 36, 36, 37, 37, + 37, 37, 37, 36, 36, 35, 35, 34, 33, 33, 32, 31, 30, 29, 29, 29, 29, 29, + 42, 42, 43, 43, 43, 41, 39, 38, 36, 36, 36, 36, 36, 36, 35, 35, 35, 35, + 34, 34, 33, 33, 32, 31, 31, 30, 29, 28, 28, 28, 28, 28, 41, 42, 43, 44, + 44, 42, 40, 38, 36, 36, 35, 35, 34, 34, 34, 34, 34, 33, 33, 32, 32, 31, + 31, 30, 30, 29, 28, 28, 27, 27, 27, 27, 41, 42, 43, 44, 45, 43, 41, 39, + 36, 36, 35, 34, 33, 33, 33, 32, 32, 32, 31, 31, 31, 30, 30, 29, 28, 28, + 27, 27, 26, 26, 26, 26, 40, 42, 43, 45, 47, 44, 42, 39, 37, 36, 34, 33, + 32, 32, 31, 31, 31, 30, 30, 30, 29, 29, 28, 28, 27, 27, 26, 26, 25, 25, + 25, 25, 39, 41, 43, 45, 46, 44, 42, 39, 37, 36, 34, 33, 32, 31, 31, 30, + 30, 29, 29, 29, 28, 28, 27, 27, 26, 26, 25, 25, 24, 24, 24, 24, 39, 41, + 42, 44, 46, 44, 41, 39, 37, 35, 34, 33, 31, 31, 30, 29, 29, 28, 28, 28, + 27, 27, 26, 26, 25, 25, 25, 24, 24, 24, 24, 24, 38, 40, 42, 44, 46, 43, + 41, 39, 37, 35, 34, 32, 31, 30, 29, 29, 28, 27, 27, 27, 26, 26, 25, 25, + 25, 24, 24, 23, 23, 23, 23, 23, 37, 39, 41, 43, 45, 43, 41, 39, 37, 35, + 34, 32, 31, 30, 29, 28, 27, 27, 26, 26, 25, 25, 24, 24, 24, 23, 23, 22, + 22, 22, 22, 22, 36, 38, 40, 42, 44, 42, 40, 38, 36, 35, 33, 32, 30, 29, + 28, 27, 27, 26, 26, 25, 24, 24, 24, 23, 23, 23, 22, 22, 21, 21, 21, 21, + 35, 37, 39, 41, 43, 41, 39, 38, 36, 34, 33, 31, 30, 29, 28, 27, 26, 26, + 25, 24, 24, 23, 23, 23, 22, 22, 22, 21, 21, 21, 21, 21, 34, 36, 38, 40, + 42, 40, 39, 37, 35, 34, 32, 31, 30, 29, 28, 27, 26, 25, 24, 24, 23, 23, + 22, 22, 22, 21, 21, 21, 20, 20, 20, 20, 33, 35, 37, 39, 41, 39, 38, 36, + 35, 33, 32, 31, 29, 28, 27, 26, 25, 24, 24, 23, 23, 22, 22, 21, 21, 21, + 20, 20, 20, 20, 20, 20, 32, 34, 36, 38, 40, 38, 37, 36, 34, 33, 31, 30, + 29, 28, 27, 26, 25, 24, 23, 23, 22, 22, 21, 21, 20, 20, 20, 19, 19, 19, + 19, 19, 31, 33, 35, 37, 38, 37, 36, 35, 33, 32, 31, 30, 28, 27, 26, 25, + 24, 24, 23, 22, 22, 21, 21, 20, 20, 20, 19, 19, 19, 19, 19, 19, 30, 32, + 34, 35, 37, 36, 35, 34, 33, 31, 30, 29, 28, 27, 26, 25, 24, 23, 23, 22, + 21, 21, 20, 20, 20, 19, 19, 19, 18, 18, 18, 18, 29, 31, 32, 34, 36, 35, + 34, 33, 32, 31, 30, 28, 27, 26, 25, 25, 24, 23, 22, 22, 21, 20, 20, 20, + 19, 19, 18, 18, 18, 18, 18, 18, 28, 30, 31, 33, 35, 34, 33, 32, 31, 30, + 29, 28, 27, 26, 25, 24, 23, 23, 22, 21, 21, 20, 20, 19, 19, 18, 18, 18, + 18, 18, 18, 18, 27, 29, 30, 32, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, + 25, 24, 23, 22, 22, 21, 20, 20, 19, 19, 18, 18, 18, 18, 17, 17, 17, 17, + 26, 28, 29, 31, 33, 32, 31, 30, 29, 28, 28, 27, 26, 25, 24, 23, 22, 22, + 21, 21, 20, 19, 19, 19, 18, 18, 18, 17, 17, 17, 17, 17, 25, 27, 28, 30, + 31, 31, 30, 29, 29, 28, 27, 26, 25, 24, 24, 23, 22, 21, 21, 20, 20, 19, + 19, 18, 18, 18, 17, 17, 17, 17, 17, 17, 25, 27, 28, 30, 31, 31, 30, 29, + 29, 28, 27, 26, 25, 24, 24, 23, 22, 21, 21, 20, 20, 19, 19, 18, 18, 18, + 17, 17, 17, 17, 17, 17, 25, 27, 28, 30, 31, 31, 30, 29, 29, 28, 27, 26, + 25, 24, 24, 23, 22, 21, 21, 20, 20, 19, 19, 18, 18, 18, 17, 17, 17, 17, + 17, 17, 25, 27, 28, 30, 31, 31, 30, 29, 29, 28, 27, 26, 25, 24, 24, 23, + 22, 21, 21, 20, 20, 19, 19, 18, 18, 18, 17, 17, 17, 17, 17, 17 }, + { /* Intra matrices */ + /* Size 4 */ + 162, 98, 89, 68, 98, 80, 74, 63, 89, 74, 55, 47, 68, 63, 47, 38, + /* Size 8 */ + 151, 190, 99, 93, 86, 76, 66, 57, 190, 128, 97, 108, 105, 94, 82, 71, + 99, 97, 81, 84, 85, 80, 72, 64, 93, 108, 84, 73, 69, 66, 61, 56, 86, + 105, 85, 69, 61, 56, 52, 49, 76, 94, 80, 66, 56, 50, 46, 43, 66, 82, 72, + 61, 52, 46, 41, 38, 57, 71, 64, 56, 49, 43, 38, 35, + /* Size 16 */ + 156, 176, 196, 149, 102, 99, 96, 92, 89, 83, 78, 73, 68, 63, 58, 58, + 176, 170, 164, 133, 101, 103, 104, 101, 99, 93, 88, 82, 76, 71, 66, 66, + 196, 164, 132, 116, 100, 106, 112, 110, 108, 103, 97, 91, 85, 79, 73, + 73, 149, 133, 116, 104, 92, 96, 99, 99, 98, 94, 90, 85, 80, 75, 70, 70, + 102, 101, 100, 92, 84, 85, 87, 87, 87, 85, 82, 79, 75, 71, 67, 67, 99, + 103, 106, 96, 85, 83, 81, 80, 79, 77, 75, 72, 69, 66, 62, 62, 96, 104, + 112, 99, 87, 81, 76, 74, 71, 70, 68, 66, 63, 61, 58, 58, 92, 101, 110, + 99, 87, 80, 74, 70, 67, 65, 63, 61, 59, 56, 54, 54, 89, 99, 108, 98, 87, + 79, 71, 67, 63, 60, 58, 56, 54, 52, 50, 50, 83, 93, 103, 94, 85, 77, 70, + 65, 60, 57, 55, 53, 51, 49, 47, 47, 78, 88, 97, 90, 82, 75, 68, 63, 58, + 55, 51, 49, 47, 46, 44, 44, 73, 82, 91, 85, 79, 72, 66, 61, 56, 53, 49, + 47, 45, 43, 42, 42, 68, 76, 85, 80, 75, 69, 63, 59, 54, 51, 47, 45, 43, + 41, 39, 39, 63, 71, 79, 75, 71, 66, 61, 56, 52, 49, 46, 43, 41, 39, 38, + 38, 58, 66, 73, 70, 67, 62, 58, 54, 50, 47, 44, 42, 39, 38, 36, 36, 58, + 66, 73, 70, 67, 62, 58, 54, 50, 47, 44, 42, 39, 38, 36, 36, + /* Size 32 */ + 158, 169, 179, 189, 199, 175, 152, 128, 104, 102, 101, 99, 97, 96, 94, + 92, 90, 88, 85, 82, 79, 77, 74, 71, 69, 66, 64, 62, 59, 59, 59, 59, 169, + 172, 176, 179, 183, 163, 143, 123, 103, 103, 102, 102, 101, 100, 98, 97, + 95, 93, 90, 87, 84, 82, 79, 76, 73, 71, 68, 66, 63, 63, 63, 63, 179, + 176, 173, 170, 167, 151, 135, 119, 103, 104, 104, 105, 106, 104, 103, + 102, 100, 97, 95, 92, 89, 86, 83, 81, 78, 75, 72, 70, 67, 67, 67, 67, + 189, 179, 170, 160, 150, 138, 126, 114, 102, 104, 106, 108, 110, 109, + 107, 106, 105, 102, 100, 97, 94, 91, 88, 85, 82, 79, 76, 74, 71, 71, 71, + 71, 199, 183, 167, 150, 134, 126, 118, 110, 102, 105, 108, 111, 114, + 113, 112, 111, 110, 107, 105, 102, 99, 96, 93, 90, 86, 84, 81, 78, 75, + 75, 75, 75, 175, 163, 151, 138, 126, 119, 112, 105, 98, 100, 103, 105, + 107, 107, 106, 106, 105, 102, 100, 98, 95, 92, 90, 87, 84, 81, 78, 76, + 73, 73, 73, 73, 152, 143, 135, 126, 118, 112, 106, 100, 94, 95, 97, 99, + 101, 101, 100, 100, 100, 98, 96, 94, 91, 89, 86, 84, 81, 79, 76, 74, 71, + 71, 71, 71, 128, 123, 119, 114, 110, 105, 100, 95, 89, 91, 92, 94, 95, + 95, 95, 94, 94, 93, 91, 89, 88, 85, 83, 81, 79, 76, 74, 72, 69, 69, 69, + 69, 104, 103, 103, 102, 102, 98, 94, 89, 85, 86, 87, 88, 89, 89, 89, 89, + 89, 88, 86, 85, 84, 82, 80, 78, 76, 74, 72, 70, 68, 68, 68, 68, 102, + 103, 104, 104, 105, 100, 95, 91, 86, 86, 86, 86, 86, 85, 85, 85, 85, 84, + 83, 81, 80, 78, 77, 75, 73, 71, 69, 67, 66, 66, 66, 66, 101, 102, 104, + 106, 108, 103, 97, 92, 87, 86, 85, 84, 83, 82, 82, 81, 81, 80, 79, 78, + 77, 75, 73, 72, 70, 69, 67, 65, 63, 63, 63, 63, 99, 102, 105, 108, 111, + 105, 99, 94, 88, 86, 84, 82, 80, 79, 78, 78, 77, 76, 75, 74, 73, 71, 70, + 69, 67, 66, 64, 63, 61, 61, 61, 61, 97, 101, 106, 110, 114, 107, 101, + 95, 89, 86, 83, 80, 77, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, 66, 64, + 63, 62, 60, 59, 59, 59, 59, 96, 100, 104, 109, 113, 107, 101, 95, 89, + 85, 82, 79, 76, 74, 73, 72, 70, 69, 68, 68, 67, 65, 64, 63, 62, 61, 60, + 58, 57, 57, 57, 57, 94, 98, 103, 107, 112, 106, 100, 95, 89, 85, 82, 78, + 75, 73, 72, 70, 68, 67, 66, 65, 64, 63, 62, 61, 60, 58, 57, 56, 55, 55, + 55, 55, 92, 97, 102, 106, 111, 106, 100, 94, 89, 85, 81, 78, 74, 72, 70, + 68, 66, 65, 64, 63, 61, 60, 59, 58, 57, 56, 55, 54, 53, 53, 53, 53, 90, + 95, 100, 105, 110, 105, 100, 94, 89, 85, 81, 77, 73, 70, 68, 66, 64, 63, + 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 51, 51, 51, 88, 93, 97, 102, + 107, 102, 98, 93, 88, 84, 80, 76, 72, 69, 67, 65, 63, 61, 60, 59, 57, + 56, 55, 54, 53, 52, 51, 50, 49, 49, 49, 49, 85, 90, 95, 100, 105, 100, + 96, 91, 86, 83, 79, 75, 71, 68, 66, 64, 61, 60, 58, 57, 56, 55, 53, 52, + 51, 51, 50, 49, 48, 48, 48, 48, 82, 87, 92, 97, 102, 98, 94, 89, 85, 81, + 78, 74, 70, 68, 65, 63, 60, 59, 57, 55, 54, 53, 52, 51, 50, 49, 48, 47, + 46, 46, 46, 46, 79, 84, 89, 94, 99, 95, 91, 88, 84, 80, 77, 73, 69, 67, + 64, 61, 59, 57, 56, 54, 52, 51, 50, 49, 48, 47, 46, 46, 45, 45, 45, 45, + 77, 82, 86, 91, 96, 92, 89, 85, 82, 78, 75, 71, 68, 65, 63, 60, 58, 56, + 55, 53, 51, 50, 49, 48, 47, 46, 45, 44, 44, 44, 44, 44, 74, 79, 83, 88, + 93, 90, 86, 83, 80, 77, 73, 70, 67, 64, 62, 59, 57, 55, 53, 52, 50, 49, + 48, 47, 46, 45, 44, 43, 42, 42, 42, 42, 71, 76, 81, 85, 90, 87, 84, 81, + 78, 75, 72, 69, 66, 63, 61, 58, 56, 54, 52, 51, 49, 48, 47, 46, 45, 44, + 43, 42, 41, 41, 41, 41, 69, 73, 78, 82, 86, 84, 81, 79, 76, 73, 70, 67, + 64, 62, 60, 57, 55, 53, 51, 50, 48, 47, 46, 45, 43, 43, 42, 41, 40, 40, + 40, 40, 66, 71, 75, 79, 84, 81, 79, 76, 74, 71, 69, 66, 63, 61, 58, 56, + 54, 52, 51, 49, 47, 46, 45, 44, 43, 42, 41, 40, 39, 39, 39, 39, 64, 68, + 72, 76, 81, 78, 76, 74, 72, 69, 67, 64, 62, 60, 57, 55, 53, 51, 50, 48, + 46, 45, 44, 43, 42, 41, 40, 39, 38, 38, 38, 38, 62, 66, 70, 74, 78, 76, + 74, 72, 70, 67, 65, 63, 60, 58, 56, 54, 52, 50, 49, 47, 46, 44, 43, 42, + 41, 40, 39, 38, 38, 38, 38, 38, 59, 63, 67, 71, 75, 73, 71, 69, 68, 66, + 63, 61, 59, 57, 55, 53, 51, 49, 48, 46, 45, 44, 42, 41, 40, 39, 38, 38, + 37, 37, 37, 37, 59, 63, 67, 71, 75, 73, 71, 69, 68, 66, 63, 61, 59, 57, + 55, 53, 51, 49, 48, 46, 45, 44, 42, 41, 40, 39, 38, 38, 37, 37, 37, 37, + 59, 63, 67, 71, 75, 73, 71, 69, 68, 66, 63, 61, 59, 57, 55, 53, 51, 49, + 48, 46, 45, 44, 42, 41, 40, 39, 38, 38, 37, 37, 37, 37, 59, 63, 67, 71, + 75, 73, 71, 69, 68, 66, 63, 61, 59, 57, 55, 53, 51, 49, 48, 46, 45, 44, + 42, 41, 40, 39, 38, 38, 37, 37, 37, 37 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 58, 35, 23, 58, 39, 28, 21, 35, 28, 20, 16, 23, 21, 16, 14, + /* Size 8 */ + 64, 85, 79, 60, 44, 34, 27, 23, 85, 74, 78, 66, 52, 40, 32, 26, 79, 78, + 55, 47, 40, 34, 29, 24, 60, 66, 47, 37, 32, 28, 25, 22, 44, 52, 40, 32, + 27, 24, 22, 20, 34, 40, 34, 28, 24, 21, 19, 18, 27, 32, 29, 25, 22, 19, + 18, 17, 23, 26, 24, 22, 20, 18, 17, 16, + /* Size 16 */ + 64, 75, 85, 82, 79, 69, 60, 52, 44, 39, 34, 31, 27, 25, 23, 23, 75, 77, + 80, 79, 78, 71, 63, 55, 48, 43, 37, 33, 30, 27, 25, 25, 85, 80, 74, 76, + 78, 72, 66, 59, 52, 46, 40, 36, 32, 29, 26, 26, 82, 79, 76, 71, 66, 62, + 57, 51, 46, 42, 37, 34, 30, 28, 25, 25, 79, 78, 78, 66, 55, 51, 47, 44, + 40, 37, 34, 31, 29, 26, 24, 24, 69, 71, 72, 62, 51, 47, 42, 39, 36, 33, + 31, 29, 27, 25, 23, 23, 60, 63, 66, 57, 47, 42, 37, 34, 32, 30, 28, 26, + 25, 23, 22, 22, 52, 55, 59, 51, 44, 39, 34, 32, 29, 28, 26, 24, 23, 22, + 21, 21, 44, 48, 52, 46, 40, 36, 32, 29, 27, 25, 24, 23, 22, 21, 20, 20, + 39, 43, 46, 42, 37, 33, 30, 28, 25, 24, 22, 21, 20, 20, 19, 19, 34, 37, + 40, 37, 34, 31, 28, 26, 24, 22, 21, 20, 19, 19, 18, 18, 31, 33, 36, 34, + 31, 29, 26, 24, 23, 21, 20, 19, 19, 18, 17, 17, 27, 30, 32, 30, 29, 27, + 25, 23, 22, 20, 19, 19, 18, 17, 17, 17, 25, 27, 29, 28, 26, 25, 23, 22, + 21, 20, 19, 18, 17, 17, 16, 16, 23, 25, 26, 25, 24, 23, 22, 21, 20, 19, + 18, 17, 17, 16, 16, 16, 23, 25, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, + 17, 16, 16, 16, + /* Size 32 */ + 64, 69, 75, 80, 85, 84, 82, 80, 79, 74, 69, 64, 60, 56, 52, 48, 44, 42, + 39, 37, 34, 32, 31, 29, 27, 26, 25, 24, 23, 23, 23, 23, 69, 73, 76, 79, + 83, 82, 81, 80, 78, 74, 70, 66, 61, 58, 54, 50, 46, 44, 41, 38, 36, 34, + 32, 30, 29, 27, 26, 25, 24, 24, 24, 24, 75, 76, 77, 79, 80, 79, 79, 79, + 78, 74, 71, 67, 63, 59, 55, 52, 48, 45, 43, 40, 37, 35, 33, 32, 30, 28, + 27, 26, 25, 25, 25, 25, 80, 79, 79, 78, 77, 77, 77, 78, 78, 75, 71, 68, + 65, 61, 57, 54, 50, 47, 44, 41, 39, 37, 35, 33, 31, 30, 28, 27, 26, 26, + 26, 26, 85, 83, 80, 77, 74, 75, 76, 77, 78, 75, 72, 69, 66, 63, 59, 55, + 52, 49, 46, 43, 40, 38, 36, 34, 32, 31, 29, 28, 26, 26, 26, 26, 84, 82, + 79, 77, 75, 74, 74, 73, 72, 69, 67, 64, 61, 58, 55, 52, 49, 46, 44, 41, + 39, 37, 35, 33, 31, 30, 29, 27, 26, 26, 26, 26, 82, 81, 79, 77, 76, 74, + 71, 69, 66, 64, 62, 59, 57, 54, 51, 49, 46, 44, 42, 39, 37, 35, 34, 32, + 30, 29, 28, 27, 25, 25, 25, 25, 80, 80, 79, 78, 77, 73, 69, 65, 61, 59, + 56, 54, 52, 50, 48, 45, 43, 41, 39, 37, 35, 34, 32, 31, 29, 28, 27, 26, + 25, 25, 25, 25, 79, 78, 78, 78, 78, 72, 66, 61, 55, 53, 51, 49, 47, 46, + 44, 42, 40, 39, 37, 36, 34, 33, 31, 30, 29, 27, 26, 25, 24, 24, 24, 24, + 74, 74, 74, 75, 75, 69, 64, 59, 53, 51, 49, 47, 45, 43, 41, 40, 38, 37, + 35, 34, 32, 31, 30, 29, 28, 27, 26, 25, 24, 24, 24, 24, 69, 70, 71, 71, + 72, 67, 62, 56, 51, 49, 47, 44, 42, 41, 39, 38, 36, 35, 33, 32, 31, 30, + 29, 28, 27, 26, 25, 24, 23, 23, 23, 23, 64, 66, 67, 68, 69, 64, 59, 54, + 49, 47, 44, 42, 40, 38, 37, 35, 34, 33, 32, 31, 29, 28, 28, 27, 26, 25, + 24, 23, 23, 23, 23, 23, 60, 61, 63, 65, 66, 61, 57, 52, 47, 45, 42, 40, + 37, 36, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 25, 24, 23, 23, 22, 22, + 22, 22, 56, 58, 59, 61, 63, 58, 54, 50, 46, 43, 41, 38, 36, 34, 33, 32, + 31, 30, 29, 28, 27, 26, 25, 25, 24, 23, 23, 22, 21, 21, 21, 21, 52, 54, + 55, 57, 59, 55, 51, 48, 44, 41, 39, 37, 34, 33, 32, 31, 29, 28, 28, 27, + 26, 25, 24, 24, 23, 23, 22, 21, 21, 21, 21, 21, 48, 50, 52, 54, 55, 52, + 49, 45, 42, 40, 38, 35, 33, 32, 31, 29, 28, 27, 26, 26, 25, 24, 24, 23, + 22, 22, 21, 21, 20, 20, 20, 20, 44, 46, 48, 50, 52, 49, 46, 43, 40, 38, + 36, 34, 32, 31, 29, 28, 27, 26, 25, 25, 24, 23, 23, 22, 22, 21, 21, 20, + 20, 20, 20, 20, 42, 44, 45, 47, 49, 46, 44, 41, 39, 37, 35, 33, 31, 30, + 28, 27, 26, 25, 25, 24, 23, 23, 22, 22, 21, 21, 20, 20, 19, 19, 19, 19, + 39, 41, 43, 44, 46, 44, 42, 39, 37, 35, 33, 32, 30, 29, 28, 26, 25, 25, + 24, 23, 22, 22, 21, 21, 20, 20, 20, 19, 19, 19, 19, 19, 37, 38, 40, 41, + 43, 41, 39, 37, 36, 34, 32, 31, 29, 28, 27, 26, 25, 24, 23, 22, 22, 21, + 21, 20, 20, 20, 19, 19, 19, 19, 19, 19, 34, 36, 37, 39, 40, 39, 37, 35, + 34, 32, 31, 29, 28, 27, 26, 25, 24, 23, 22, 22, 21, 21, 20, 20, 19, 19, + 19, 18, 18, 18, 18, 18, 32, 34, 35, 37, 38, 37, 35, 34, 33, 31, 30, 28, + 27, 26, 25, 24, 23, 23, 22, 21, 21, 20, 20, 19, 19, 19, 18, 18, 18, 18, + 18, 18, 31, 32, 33, 35, 36, 35, 34, 32, 31, 30, 29, 28, 26, 25, 24, 24, + 23, 22, 21, 21, 20, 20, 19, 19, 19, 18, 18, 18, 17, 17, 17, 17, 29, 30, + 32, 33, 34, 33, 32, 31, 30, 29, 28, 27, 25, 25, 24, 23, 22, 22, 21, 20, + 20, 19, 19, 19, 18, 18, 18, 17, 17, 17, 17, 17, 27, 29, 30, 31, 32, 31, + 30, 29, 29, 28, 27, 26, 25, 24, 23, 22, 22, 21, 20, 20, 19, 19, 19, 18, + 18, 18, 17, 17, 17, 17, 17, 17, 26, 27, 28, 30, 31, 30, 29, 28, 27, 27, + 26, 25, 24, 23, 23, 22, 21, 21, 20, 20, 19, 19, 18, 18, 18, 17, 17, 17, + 17, 17, 17, 17, 25, 26, 27, 28, 29, 29, 28, 27, 26, 26, 25, 24, 23, 23, + 22, 21, 21, 20, 20, 19, 19, 18, 18, 18, 17, 17, 17, 17, 16, 16, 16, 16, + 24, 25, 26, 27, 28, 27, 27, 26, 25, 25, 24, 23, 23, 22, 21, 21, 20, 20, + 19, 19, 18, 18, 18, 17, 17, 17, 17, 16, 16, 16, 16, 16, 23, 24, 25, 26, + 26, 26, 25, 25, 24, 24, 23, 23, 22, 21, 21, 20, 20, 19, 19, 19, 18, 18, + 17, 17, 17, 17, 16, 16, 16, 16, 16, 16, 23, 24, 25, 26, 26, 26, 25, 25, + 24, 24, 23, 23, 22, 21, 21, 20, 20, 19, 19, 19, 18, 18, 17, 17, 17, 17, + 16, 16, 16, 16, 16, 16, 23, 24, 25, 26, 26, 26, 25, 25, 24, 24, 23, 23, + 22, 21, 21, 20, 20, 19, 19, 19, 18, 18, 17, 17, 17, 17, 16, 16, 16, 16, + 16, 16, 23, 24, 25, 26, 26, 26, 25, 25, 24, 24, 23, 23, 22, 21, 21, 20, + 20, 19, 19, 19, 18, 18, 17, 17, 17, 17, 16, 16, 16, 16, 16, 16 }, + { /* Intra matrices */ + /* Size 4 */ + 192, 174, 103, 63, 174, 113, 79, 57, 103, 79, 55, 44, 63, 57, 44, 37, + /* Size 8 */ + 160, 216, 199, 149, 109, 82, 65, 53, 216, 187, 195, 166, 128, 98, 77, + 62, 199, 195, 137, 116, 98, 81, 67, 57, 149, 166, 116, 90, 76, 66, 57, + 50, 109, 128, 98, 76, 63, 55, 49, 45, 82, 98, 81, 66, 55, 48, 44, 40, + 65, 77, 67, 57, 49, 44, 40, 37, 53, 62, 57, 50, 45, 40, 37, 35, + /* Size 16 */ + 167, 195, 224, 216, 207, 181, 155, 134, 113, 99, 85, 76, 67, 61, 55, 55, + 195, 202, 209, 207, 205, 184, 164, 143, 123, 108, 94, 84, 74, 67, 60, + 60, 224, 209, 194, 199, 203, 188, 172, 153, 133, 117, 102, 91, 80, 72, + 65, 65, 216, 207, 199, 186, 173, 160, 147, 132, 118, 106, 93, 84, 75, + 68, 62, 62, 207, 205, 203, 173, 143, 132, 121, 112, 102, 94, 85, 78, 70, + 65, 59, 59, 181, 184, 188, 160, 132, 120, 107, 99, 91, 84, 77, 71, 65, + 60, 56, 56, 155, 164, 172, 147, 121, 107, 94, 86, 79, 74, 69, 64, 60, + 56, 53, 53, 134, 143, 153, 132, 112, 99, 86, 79, 72, 68, 63, 59, 56, 53, + 50, 50, 113, 123, 133, 118, 102, 91, 79, 72, 66, 62, 57, 54, 51, 49, 47, + 47, 99, 108, 117, 106, 94, 84, 74, 68, 62, 58, 54, 51, 48, 46, 44, 44, + 85, 94, 102, 93, 85, 77, 69, 63, 57, 54, 50, 48, 46, 44, 42, 42, 76, 84, + 91, 84, 78, 71, 64, 59, 54, 51, 48, 46, 44, 42, 40, 40, 67, 74, 80, 75, + 70, 65, 60, 56, 51, 48, 46, 44, 42, 40, 39, 39, 61, 67, 72, 68, 65, 60, + 56, 53, 49, 46, 44, 42, 40, 39, 38, 38, 55, 60, 65, 62, 59, 56, 53, 50, + 47, 44, 42, 40, 39, 38, 36, 36, 55, 60, 65, 62, 59, 56, 53, 50, 47, 44, + 42, 40, 39, 38, 36, 36, + /* Size 32 */ + 170, 185, 200, 214, 229, 225, 220, 215, 211, 198, 185, 171, 158, 147, + 137, 126, 116, 109, 101, 94, 87, 83, 78, 73, 69, 66, 62, 59, 56, 56, 56, + 56, 185, 194, 203, 212, 221, 219, 216, 213, 210, 198, 186, 174, 163, + 152, 142, 131, 121, 113, 106, 99, 91, 87, 82, 77, 72, 69, 65, 62, 59, + 59, 59, 59, 200, 203, 207, 210, 214, 213, 212, 210, 209, 199, 188, 178, + 167, 157, 146, 136, 126, 118, 111, 103, 96, 90, 85, 80, 75, 72, 68, 65, + 61, 61, 61, 61, 214, 212, 210, 208, 206, 207, 207, 208, 209, 199, 190, + 181, 172, 161, 151, 141, 131, 123, 115, 108, 100, 94, 89, 84, 78, 75, + 71, 67, 64, 64, 64, 64, 229, 221, 214, 206, 199, 201, 203, 205, 208, + 200, 192, 184, 176, 166, 156, 146, 136, 128, 120, 112, 104, 98, 93, 87, + 81, 78, 74, 70, 66, 66, 66, 66, 225, 219, 213, 207, 201, 199, 197, 194, + 192, 185, 178, 170, 163, 154, 145, 137, 128, 121, 114, 107, 100, 95, 89, + 84, 79, 75, 72, 68, 65, 65, 65, 65, 220, 216, 212, 207, 203, 197, 190, + 183, 177, 170, 163, 157, 150, 142, 135, 128, 120, 114, 108, 102, 95, 91, + 86, 81, 77, 73, 70, 67, 63, 63, 63, 63, 215, 213, 210, 208, 205, 194, + 183, 172, 161, 155, 149, 143, 137, 131, 125, 118, 112, 107, 102, 96, 91, + 87, 83, 78, 74, 71, 68, 65, 62, 62, 62, 62, 211, 210, 209, 209, 208, + 192, 177, 161, 146, 140, 135, 129, 124, 119, 114, 109, 105, 100, 96, 91, + 87, 83, 79, 75, 72, 69, 66, 63, 60, 60, 60, 60, 198, 198, 199, 199, 200, + 185, 170, 155, 140, 134, 128, 123, 117, 112, 108, 103, 99, 95, 91, 87, + 82, 79, 76, 72, 69, 66, 64, 61, 59, 59, 59, 59, 185, 186, 188, 190, 192, + 178, 163, 149, 135, 128, 122, 116, 110, 105, 101, 97, 93, 89, 86, 82, + 78, 75, 72, 69, 66, 64, 62, 59, 57, 57, 57, 57, 171, 174, 178, 181, 184, + 170, 157, 143, 129, 123, 116, 109, 103, 99, 95, 91, 87, 84, 81, 77, 74, + 72, 69, 66, 64, 62, 60, 57, 55, 55, 55, 55, 158, 163, 167, 172, 176, + 163, 150, 137, 124, 117, 110, 103, 96, 92, 88, 85, 81, 78, 75, 73, 70, + 68, 66, 63, 61, 59, 57, 55, 54, 54, 54, 54, 147, 152, 157, 161, 166, + 154, 142, 131, 119, 112, 105, 99, 92, 88, 85, 81, 77, 75, 72, 70, 67, + 65, 63, 61, 59, 57, 55, 54, 52, 52, 52, 52, 137, 142, 146, 151, 156, + 145, 135, 125, 114, 108, 101, 95, 88, 85, 81, 78, 74, 72, 69, 67, 64, + 62, 61, 59, 57, 55, 54, 52, 51, 51, 51, 51, 126, 131, 136, 141, 146, + 137, 128, 118, 109, 103, 97, 91, 85, 81, 78, 74, 71, 68, 66, 64, 61, 60, + 58, 56, 55, 53, 52, 50, 49, 49, 49, 49, 116, 121, 126, 131, 136, 128, + 120, 112, 105, 99, 93, 87, 81, 77, 74, 71, 67, 65, 63, 61, 59, 57, 56, + 54, 52, 51, 50, 49, 48, 48, 48, 48, 109, 113, 118, 123, 128, 121, 114, + 107, 100, 95, 89, 84, 78, 75, 72, 68, 65, 63, 61, 59, 57, 55, 54, 52, + 51, 50, 49, 48, 46, 46, 46, 46, 101, 106, 111, 115, 120, 114, 108, 102, + 96, 91, 86, 81, 75, 72, 69, 66, 63, 61, 59, 57, 55, 54, 52, 51, 49, 48, + 47, 46, 45, 45, 45, 45, 94, 99, 103, 108, 112, 107, 102, 96, 91, 87, 82, + 77, 73, 70, 67, 64, 61, 59, 57, 55, 53, 52, 51, 49, 48, 47, 46, 45, 44, + 44, 44, 44, 87, 91, 96, 100, 104, 100, 95, 91, 87, 82, 78, 74, 70, 67, + 64, 61, 59, 57, 55, 53, 51, 50, 49, 48, 47, 46, 45, 44, 43, 43, 43, 43, + 83, 87, 90, 94, 98, 95, 91, 87, 83, 79, 75, 72, 68, 65, 62, 60, 57, 55, + 54, 52, 50, 49, 48, 47, 45, 45, 44, 43, 42, 42, 42, 42, 78, 82, 85, 89, + 93, 89, 86, 83, 79, 76, 72, 69, 66, 63, 61, 58, 56, 54, 52, 51, 49, 48, + 47, 46, 44, 44, 43, 42, 41, 41, 41, 41, 73, 77, 80, 84, 87, 84, 81, 78, + 75, 72, 69, 66, 63, 61, 59, 56, 54, 52, 51, 49, 48, 47, 46, 45, 43, 43, + 42, 41, 40, 40, 40, 40, 69, 72, 75, 78, 81, 79, 77, 74, 72, 69, 66, 64, + 61, 59, 57, 55, 52, 51, 49, 48, 47, 45, 44, 43, 42, 42, 41, 40, 40, 40, + 40, 40, 66, 69, 72, 75, 78, 75, 73, 71, 69, 66, 64, 62, 59, 57, 55, 53, + 51, 50, 48, 47, 46, 45, 44, 43, 42, 41, 40, 40, 39, 39, 39, 39, 62, 65, + 68, 71, 74, 72, 70, 68, 66, 64, 62, 60, 57, 55, 54, 52, 50, 49, 47, 46, + 45, 44, 43, 42, 41, 40, 40, 39, 38, 38, 38, 38, 59, 62, 65, 67, 70, 68, + 67, 65, 63, 61, 59, 57, 55, 54, 52, 50, 49, 48, 46, 45, 44, 43, 42, 41, + 40, 40, 39, 38, 38, 38, 38, 38, 56, 59, 61, 64, 66, 65, 63, 62, 60, 59, + 57, 55, 54, 52, 51, 49, 48, 46, 45, 44, 43, 42, 41, 40, 40, 39, 38, 38, + 37, 37, 37, 37, 56, 59, 61, 64, 66, 65, 63, 62, 60, 59, 57, 55, 54, 52, + 51, 49, 48, 46, 45, 44, 43, 42, 41, 40, 40, 39, 38, 38, 37, 37, 37, 37, + 56, 59, 61, 64, 66, 65, 63, 62, 60, 59, 57, 55, 54, 52, 51, 49, 48, 46, + 45, 44, 43, 42, 41, 40, 40, 39, 38, 38, 37, 37, 37, 37, 56, 59, 61, 64, + 66, 65, 63, 62, 60, 59, 57, 55, 54, 52, 51, 49, 48, 46, 45, 44, 43, 42, + 41, 40, 40, 39, 38, 38, 37, 37, 37, 37 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 40, 37, 29, 40, 34, 32, 27, 37, 32, 25, 22, 29, 27, 22, 18, + /* Size 8 */ + 64, 79, 44, 41, 39, 35, 31, 27, 79, 55, 43, 47, 46, 42, 37, 33, 44, 43, + 37, 38, 38, 36, 33, 30, 41, 47, 38, 34, 32, 31, 29, 27, 39, 46, 38, 32, + 29, 27, 25, 24, 35, 42, 36, 31, 27, 24, 23, 21, 31, 37, 33, 29, 25, 23, + 21, 20, 27, 33, 30, 27, 24, 21, 20, 19, + /* Size 16 */ + 64, 72, 79, 62, 44, 42, 41, 40, 39, 37, 35, 33, 31, 29, 27, 27, 72, 69, + 67, 55, 43, 44, 44, 43, 42, 40, 38, 36, 34, 32, 30, 30, 79, 67, 55, 49, + 43, 45, 47, 47, 46, 44, 42, 39, 37, 35, 33, 33, 62, 55, 49, 44, 40, 41, + 43, 42, 42, 41, 39, 37, 35, 33, 31, 31, 44, 43, 43, 40, 37, 37, 38, 38, + 38, 37, 36, 35, 33, 32, 30, 30, 42, 44, 45, 41, 37, 37, 36, 35, 35, 34, + 33, 32, 31, 30, 28, 28, 41, 44, 47, 43, 38, 36, 34, 33, 32, 31, 31, 30, + 29, 28, 27, 27, 40, 43, 47, 42, 38, 35, 33, 32, 30, 29, 29, 28, 27, 26, + 25, 25, 39, 42, 46, 42, 38, 35, 32, 30, 29, 28, 27, 26, 25, 25, 24, 24, + 37, 40, 44, 41, 37, 34, 31, 29, 28, 27, 26, 25, 24, 23, 23, 23, 35, 38, + 42, 39, 36, 33, 31, 29, 27, 26, 24, 24, 23, 22, 21, 21, 33, 36, 39, 37, + 35, 32, 30, 28, 26, 25, 24, 23, 22, 21, 21, 21, 31, 34, 37, 35, 33, 31, + 29, 27, 25, 24, 23, 22, 21, 20, 20, 20, 29, 32, 35, 33, 32, 30, 28, 26, + 25, 23, 22, 21, 20, 20, 19, 19, 27, 30, 33, 31, 30, 28, 27, 25, 24, 23, + 21, 21, 20, 19, 19, 19, 27, 30, 33, 31, 30, 28, 27, 25, 24, 23, 21, 21, + 20, 19, 19, 19, + /* Size 32 */ + 64, 68, 72, 75, 79, 70, 62, 53, 44, 43, 42, 42, 41, 41, 40, 39, 39, 38, + 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 27, 27, 27, 68, 69, 71, 72, + 73, 66, 58, 51, 43, 43, 43, 43, 43, 42, 42, 41, 40, 39, 38, 37, 36, 35, + 34, 33, 32, 31, 30, 29, 28, 28, 28, 28, 72, 71, 69, 68, 67, 61, 55, 49, + 43, 44, 44, 44, 44, 44, 43, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, + 32, 31, 30, 30, 30, 30, 75, 72, 68, 65, 61, 57, 52, 48, 43, 44, 44, 45, + 46, 45, 45, 45, 44, 43, 42, 41, 40, 39, 38, 37, 35, 34, 33, 32, 31, 31, + 31, 31, 79, 73, 67, 61, 55, 52, 49, 46, 43, 44, 45, 46, 47, 47, 47, 46, + 46, 45, 44, 43, 42, 41, 39, 38, 37, 36, 35, 34, 33, 33, 33, 33, 70, 66, + 61, 57, 52, 49, 47, 44, 41, 42, 43, 44, 45, 45, 44, 44, 44, 43, 42, 41, + 40, 39, 38, 37, 36, 35, 34, 33, 32, 32, 32, 32, 62, 58, 55, 52, 49, 47, + 44, 42, 40, 40, 41, 42, 43, 42, 42, 42, 42, 41, 41, 40, 39, 38, 37, 36, + 35, 34, 33, 32, 31, 31, 31, 31, 53, 51, 49, 48, 46, 44, 42, 40, 38, 39, + 39, 40, 40, 40, 40, 40, 40, 39, 39, 38, 38, 37, 36, 35, 34, 33, 32, 32, + 31, 31, 31, 31, 44, 43, 43, 43, 43, 41, 40, 38, 37, 37, 37, 38, 38, 38, + 38, 38, 38, 38, 37, 37, 36, 35, 35, 34, 33, 32, 32, 31, 30, 30, 30, 30, + 43, 43, 44, 44, 44, 42, 40, 39, 37, 37, 37, 37, 37, 37, 37, 37, 37, 36, + 36, 35, 35, 34, 33, 33, 32, 31, 31, 30, 29, 29, 29, 29, 42, 43, 44, 44, + 45, 43, 41, 39, 37, 37, 37, 36, 36, 36, 35, 35, 35, 35, 34, 34, 33, 33, + 32, 32, 31, 30, 30, 29, 28, 28, 28, 28, 42, 43, 44, 45, 46, 44, 42, 40, + 38, 37, 36, 35, 35, 34, 34, 34, 33, 33, 33, 32, 32, 31, 31, 30, 30, 29, + 29, 28, 28, 28, 28, 28, 41, 43, 44, 46, 47, 45, 43, 40, 38, 37, 36, 35, + 34, 33, 33, 32, 32, 32, 31, 31, 31, 30, 30, 29, 29, 28, 28, 27, 27, 27, + 27, 27, 41, 42, 44, 45, 47, 45, 42, 40, 38, 37, 36, 34, 33, 33, 32, 32, + 31, 31, 30, 30, 30, 29, 29, 28, 28, 27, 27, 27, 26, 26, 26, 26, 40, 42, + 43, 45, 47, 44, 42, 40, 38, 37, 35, 34, 33, 32, 32, 31, 30, 30, 29, 29, + 29, 28, 28, 27, 27, 27, 26, 26, 25, 25, 25, 25, 39, 41, 43, 45, 46, 44, + 42, 40, 38, 37, 35, 34, 32, 32, 31, 30, 29, 29, 29, 28, 28, 27, 27, 27, + 26, 26, 25, 25, 25, 25, 25, 25, 39, 40, 42, 44, 46, 44, 42, 40, 38, 37, + 35, 33, 32, 31, 30, 29, 29, 28, 28, 27, 27, 26, 26, 26, 25, 25, 25, 24, + 24, 24, 24, 24, 38, 39, 41, 43, 45, 43, 41, 39, 38, 36, 35, 33, 32, 31, + 30, 29, 28, 28, 27, 27, 26, 26, 25, 25, 25, 24, 24, 24, 23, 23, 23, 23, + 37, 38, 40, 42, 44, 42, 41, 39, 37, 36, 34, 33, 31, 30, 29, 29, 28, 27, + 27, 26, 26, 25, 25, 24, 24, 24, 23, 23, 23, 23, 23, 23, 36, 37, 39, 41, + 43, 41, 40, 38, 37, 35, 34, 32, 31, 30, 29, 28, 27, 27, 26, 26, 25, 25, + 24, 24, 23, 23, 23, 22, 22, 22, 22, 22, 35, 36, 38, 40, 42, 40, 39, 38, + 36, 35, 33, 32, 31, 30, 29, 28, 27, 26, 26, 25, 24, 24, 24, 23, 23, 22, + 22, 22, 21, 21, 21, 21, 34, 35, 37, 39, 41, 39, 38, 37, 35, 34, 33, 31, + 30, 29, 28, 27, 26, 26, 25, 25, 24, 24, 23, 23, 22, 22, 22, 21, 21, 21, + 21, 21, 33, 34, 36, 38, 39, 38, 37, 36, 35, 33, 32, 31, 30, 29, 28, 27, + 26, 25, 25, 24, 24, 23, 23, 22, 22, 22, 21, 21, 21, 21, 21, 21, 32, 33, + 35, 37, 38, 37, 36, 35, 34, 33, 32, 30, 29, 28, 27, 27, 26, 25, 24, 24, + 23, 23, 22, 22, 21, 21, 21, 21, 20, 20, 20, 20, 31, 32, 34, 35, 37, 36, + 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 25, 24, 23, 23, 22, 22, 21, + 21, 21, 20, 20, 20, 20, 20, 20, 30, 31, 33, 34, 36, 35, 34, 33, 32, 31, + 30, 29, 28, 27, 27, 26, 25, 24, 24, 23, 22, 22, 22, 21, 21, 20, 20, 20, + 19, 19, 19, 19, 29, 30, 32, 33, 35, 34, 33, 32, 32, 31, 30, 29, 28, 27, + 26, 25, 25, 24, 23, 23, 22, 22, 21, 21, 20, 20, 20, 19, 19, 19, 19, 19, + 28, 29, 31, 32, 34, 33, 32, 32, 31, 30, 29, 28, 27, 27, 26, 25, 24, 24, + 23, 22, 22, 21, 21, 21, 20, 20, 19, 19, 19, 19, 19, 19, 27, 28, 30, 31, + 33, 32, 31, 31, 30, 29, 28, 28, 27, 26, 25, 25, 24, 23, 23, 22, 21, 21, + 21, 20, 20, 19, 19, 19, 19, 19, 19, 19, 27, 28, 30, 31, 33, 32, 31, 31, + 30, 29, 28, 28, 27, 26, 25, 25, 24, 23, 23, 22, 21, 21, 21, 20, 20, 19, + 19, 19, 19, 19, 19, 19, 27, 28, 30, 31, 33, 32, 31, 31, 30, 29, 28, 28, + 27, 26, 25, 25, 24, 23, 23, 22, 21, 21, 21, 20, 20, 19, 19, 19, 19, 19, + 19, 19, 27, 28, 30, 31, 33, 32, 31, 31, 30, 29, 28, 28, 27, 26, 25, 25, + 24, 23, 23, 22, 21, 21, 21, 20, 20, 19, 19, 19, 19, 19, 19, 19 }, + { /* Intra matrices */ + /* Size 4 */ + 152, 94, 86, 67, 94, 77, 72, 62, 86, 72, 55, 48, 67, 62, 48, 40, + /* Size 8 */ + 141, 176, 95, 89, 83, 74, 65, 57, 176, 120, 93, 103, 100, 90, 80, 70, + 95, 93, 79, 81, 82, 77, 71, 64, 89, 103, 81, 71, 68, 65, 61, 56, 83, + 100, 82, 68, 60, 56, 53, 49, 74, 90, 77, 65, 56, 50, 47, 44, 65, 80, 71, + 61, 53, 47, 43, 40, 57, 70, 64, 56, 49, 44, 40, 37, + /* Size 16 */ + 145, 163, 181, 139, 97, 94, 91, 88, 85, 81, 76, 71, 66, 62, 58, 58, 163, + 158, 152, 124, 96, 98, 99, 96, 94, 89, 84, 79, 74, 70, 65, 65, 181, 152, + 124, 110, 95, 101, 106, 104, 103, 98, 93, 87, 82, 77, 72, 72, 139, 124, + 110, 99, 88, 91, 95, 94, 93, 90, 86, 82, 77, 73, 69, 69, 97, 96, 95, 88, + 81, 82, 84, 84, 84, 82, 80, 76, 73, 69, 65, 65, 94, 98, 101, 91, 82, 80, + 79, 78, 77, 75, 73, 70, 68, 65, 62, 62, 91, 99, 106, 95, 84, 79, 74, 72, + 70, 68, 67, 65, 63, 60, 58, 58, 88, 96, 104, 94, 84, 78, 72, 69, 66, 64, + 62, 60, 58, 56, 54, 54, 85, 94, 103, 93, 84, 77, 70, 66, 62, 60, 58, 56, + 54, 52, 51, 51, 81, 89, 98, 90, 82, 75, 68, 64, 60, 57, 55, 53, 51, 50, + 48, 48, 76, 84, 93, 86, 80, 73, 67, 62, 58, 55, 52, 50, 48, 47, 45, 45, + 71, 79, 87, 82, 76, 70, 65, 60, 56, 53, 50, 48, 46, 45, 43, 43, 66, 74, + 82, 77, 73, 68, 63, 58, 54, 51, 48, 46, 44, 43, 41, 41, 62, 70, 77, 73, + 69, 65, 60, 56, 52, 50, 47, 45, 43, 41, 40, 40, 58, 65, 72, 69, 65, 62, + 58, 54, 51, 48, 45, 43, 41, 40, 38, 38, 58, 65, 72, 69, 65, 62, 58, 54, + 51, 48, 45, 43, 41, 40, 38, 38, + /* Size 32 */ + 147, 156, 165, 175, 184, 163, 141, 120, 99, 97, 96, 94, 93, 91, 90, 88, + 87, 84, 82, 79, 77, 75, 72, 70, 67, 65, 63, 61, 59, 59, 59, 59, 156, + 160, 163, 166, 169, 151, 134, 116, 98, 98, 97, 97, 97, 95, 94, 92, 91, + 89, 86, 84, 81, 79, 76, 74, 71, 69, 67, 65, 62, 62, 62, 62, 165, 163, + 160, 157, 155, 140, 126, 112, 98, 98, 99, 100, 100, 99, 98, 97, 95, 93, + 91, 88, 86, 83, 80, 78, 75, 73, 71, 68, 66, 66, 66, 66, 175, 166, 157, + 149, 140, 129, 119, 108, 97, 99, 101, 102, 104, 103, 102, 101, 100, 97, + 95, 93, 90, 87, 85, 82, 79, 77, 74, 72, 69, 69, 69, 69, 184, 169, 155, + 140, 126, 118, 111, 104, 97, 100, 102, 105, 108, 107, 106, 105, 104, + 102, 99, 97, 94, 92, 89, 86, 83, 81, 78, 75, 73, 73, 73, 73, 163, 151, + 140, 129, 118, 112, 106, 99, 93, 95, 98, 100, 102, 101, 101, 100, 100, + 97, 95, 93, 91, 88, 86, 83, 81, 78, 76, 74, 71, 71, 71, 71, 141, 134, + 126, 119, 111, 106, 100, 95, 89, 91, 93, 95, 96, 96, 96, 95, 95, 93, 91, + 89, 88, 85, 83, 81, 79, 76, 74, 72, 70, 70, 70, 70, 120, 116, 112, 108, + 104, 99, 95, 90, 86, 87, 88, 89, 91, 91, 90, 90, 90, 89, 87, 86, 84, 82, + 80, 78, 76, 74, 72, 70, 68, 68, 68, 68, 99, 98, 98, 97, 97, 93, 89, 86, + 82, 83, 84, 84, 85, 85, 85, 85, 85, 84, 83, 82, 81, 79, 77, 76, 74, 72, + 70, 68, 66, 66, 66, 66, 97, 98, 98, 99, 100, 95, 91, 87, 83, 83, 83, 83, + 82, 82, 82, 82, 82, 81, 80, 79, 78, 76, 74, 73, 71, 70, 68, 66, 64, 64, + 64, 64, 96, 97, 99, 101, 102, 98, 93, 88, 84, 83, 82, 81, 80, 79, 79, + 79, 78, 77, 76, 75, 74, 73, 71, 70, 69, 67, 66, 64, 63, 63, 63, 63, 94, + 97, 100, 102, 105, 100, 95, 89, 84, 83, 81, 79, 77, 77, 76, 75, 75, 74, + 73, 72, 71, 70, 69, 67, 66, 65, 63, 62, 61, 61, 61, 61, 93, 97, 100, + 104, 108, 102, 96, 91, 85, 82, 80, 77, 75, 74, 73, 72, 71, 70, 69, 68, + 68, 67, 66, 65, 64, 62, 61, 60, 59, 59, 59, 59, 91, 95, 99, 103, 107, + 101, 96, 91, 85, 82, 79, 77, 74, 72, 71, 70, 69, 68, 67, 66, 65, 64, 63, + 62, 61, 60, 59, 58, 57, 57, 57, 57, 90, 94, 98, 102, 106, 101, 96, 90, + 85, 82, 79, 76, 73, 71, 70, 68, 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, + 57, 56, 55, 55, 55, 55, 88, 92, 97, 101, 105, 100, 95, 90, 85, 82, 79, + 75, 72, 70, 68, 67, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, + 53, 53, 53, 87, 91, 95, 100, 104, 100, 95, 90, 85, 82, 78, 75, 71, 69, + 67, 65, 63, 62, 61, 60, 58, 58, 57, 56, 55, 54, 53, 52, 52, 52, 52, 52, + 84, 89, 93, 97, 102, 97, 93, 89, 84, 81, 77, 74, 70, 68, 66, 64, 62, 61, + 59, 58, 57, 56, 55, 54, 53, 53, 52, 51, 50, 50, 50, 50, 82, 86, 91, 95, + 99, 95, 91, 87, 83, 80, 76, 73, 69, 67, 65, 63, 61, 59, 58, 57, 56, 55, + 54, 53, 52, 51, 50, 50, 49, 49, 49, 49, 79, 84, 88, 93, 97, 93, 89, 86, + 82, 79, 75, 72, 68, 66, 64, 62, 60, 58, 57, 56, 54, 53, 52, 51, 50, 50, + 49, 48, 47, 47, 47, 47, 77, 81, 86, 90, 94, 91, 88, 84, 81, 78, 74, 71, + 68, 65, 63, 61, 58, 57, 56, 54, 53, 52, 51, 50, 49, 48, 47, 47, 46, 46, + 46, 46, 75, 79, 83, 87, 92, 88, 85, 82, 79, 76, 73, 70, 67, 64, 62, 60, + 58, 56, 55, 53, 52, 51, 50, 49, 48, 47, 46, 46, 45, 45, 45, 45, 72, 76, + 80, 85, 89, 86, 83, 80, 77, 74, 71, 69, 66, 63, 61, 59, 57, 55, 54, 52, + 51, 50, 49, 48, 47, 46, 45, 45, 44, 44, 44, 44, 70, 74, 78, 82, 86, 83, + 81, 78, 76, 73, 70, 67, 65, 62, 60, 58, 56, 54, 53, 51, 50, 49, 48, 47, + 46, 45, 44, 44, 43, 43, 43, 43, 67, 71, 75, 79, 83, 81, 79, 76, 74, 71, + 69, 66, 64, 61, 59, 57, 55, 53, 52, 50, 49, 48, 47, 46, 45, 44, 43, 43, + 42, 42, 42, 42, 65, 69, 73, 77, 81, 78, 76, 74, 72, 70, 67, 65, 62, 60, + 58, 56, 54, 53, 51, 50, 48, 47, 46, 45, 44, 43, 43, 42, 41, 41, 41, 41, + 63, 67, 71, 74, 78, 76, 74, 72, 70, 68, 66, 63, 61, 59, 57, 55, 53, 52, + 50, 49, 47, 46, 45, 44, 43, 43, 42, 41, 40, 40, 40, 40, 61, 65, 68, 72, + 75, 74, 72, 70, 68, 66, 64, 62, 60, 58, 56, 54, 52, 51, 50, 48, 47, 46, + 45, 44, 43, 42, 41, 40, 40, 40, 40, 40, 59, 62, 66, 69, 73, 71, 70, 68, + 66, 64, 63, 61, 59, 57, 55, 53, 52, 50, 49, 47, 46, 45, 44, 43, 42, 41, + 40, 40, 39, 39, 39, 39, 59, 62, 66, 69, 73, 71, 70, 68, 66, 64, 63, 61, + 59, 57, 55, 53, 52, 50, 49, 47, 46, 45, 44, 43, 42, 41, 40, 40, 39, 39, + 39, 39, 59, 62, 66, 69, 73, 71, 70, 68, 66, 64, 63, 61, 59, 57, 55, 53, + 52, 50, 49, 47, 46, 45, 44, 43, 42, 41, 40, 40, 39, 39, 39, 39, 59, 62, + 66, 69, 73, 71, 70, 68, 66, 64, 63, 61, 59, 57, 55, 53, 52, 50, 49, 47, + 46, 45, 44, 43, 42, 41, 40, 40, 39, 39, 39, 39 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 59, 37, 24, 59, 40, 29, 23, 37, 29, 22, 19, 24, 23, 19, 16, + /* Size 8 */ + 64, 84, 78, 60, 45, 36, 29, 25, 84, 74, 77, 66, 52, 41, 34, 28, 78, 77, + 56, 48, 42, 36, 30, 27, 60, 66, 48, 39, 34, 30, 27, 24, 45, 52, 42, 34, + 29, 26, 24, 22, 36, 41, 36, 30, 26, 23, 22, 21, 29, 34, 30, 27, 24, 22, + 20, 19, 25, 28, 27, 24, 22, 21, 19, 19, + /* Size 16 */ + 64, 74, 84, 81, 78, 69, 60, 53, 45, 41, 36, 33, 29, 27, 25, 25, 74, 77, + 79, 78, 77, 70, 63, 56, 49, 44, 39, 35, 32, 29, 27, 27, 84, 79, 74, 75, + 77, 71, 66, 59, 52, 47, 41, 38, 34, 31, 28, 28, 81, 78, 75, 71, 66, 62, + 57, 52, 47, 43, 38, 35, 32, 30, 28, 28, 78, 77, 77, 66, 56, 52, 48, 45, + 42, 39, 36, 33, 30, 29, 27, 27, 69, 70, 71, 62, 52, 48, 43, 40, 38, 35, + 33, 31, 29, 27, 25, 25, 60, 63, 66, 57, 48, 43, 39, 36, 34, 32, 30, 28, + 27, 26, 24, 24, 53, 56, 59, 52, 45, 40, 36, 34, 31, 30, 28, 27, 25, 24, + 23, 23, 45, 49, 52, 47, 42, 38, 34, 31, 29, 27, 26, 25, 24, 23, 22, 22, + 41, 44, 47, 43, 39, 35, 32, 30, 27, 26, 25, 24, 23, 22, 21, 21, 36, 39, + 41, 38, 36, 33, 30, 28, 26, 25, 23, 23, 22, 21, 21, 21, 33, 35, 38, 35, + 33, 31, 28, 27, 25, 24, 23, 22, 21, 21, 20, 20, 29, 32, 34, 32, 30, 29, + 27, 25, 24, 23, 22, 21, 20, 20, 19, 19, 27, 29, 31, 30, 29, 27, 26, 24, + 23, 22, 21, 21, 20, 20, 19, 19, 25, 27, 28, 28, 27, 25, 24, 23, 22, 21, + 21, 20, 19, 19, 19, 19, 25, 27, 28, 28, 27, 25, 24, 23, 22, 21, 21, 20, + 19, 19, 19, 19, + /* Size 32 */ + 64, 69, 74, 79, 84, 83, 81, 80, 78, 73, 69, 64, 60, 56, 53, 49, 45, 43, + 41, 38, 36, 34, 33, 31, 29, 28, 27, 26, 25, 25, 25, 25, 69, 72, 75, 78, + 82, 81, 80, 79, 78, 74, 70, 66, 61, 58, 54, 51, 47, 45, 42, 40, 37, 35, + 34, 32, 30, 29, 28, 27, 26, 26, 26, 26, 74, 75, 77, 78, 79, 79, 78, 78, + 77, 74, 70, 67, 63, 59, 56, 52, 49, 46, 44, 41, 39, 37, 35, 33, 32, 30, + 29, 28, 27, 27, 27, 27, 79, 78, 78, 77, 76, 77, 77, 77, 77, 74, 71, 68, + 65, 61, 58, 54, 51, 48, 45, 43, 40, 38, 36, 35, 33, 31, 30, 29, 28, 28, + 28, 28, 84, 82, 79, 76, 74, 75, 75, 76, 77, 74, 71, 69, 66, 63, 59, 56, + 52, 50, 47, 44, 41, 40, 38, 36, 34, 32, 31, 30, 28, 28, 28, 28, 83, 81, + 79, 77, 75, 74, 73, 72, 72, 69, 67, 64, 62, 59, 56, 53, 50, 47, 45, 42, + 40, 38, 36, 35, 33, 32, 30, 29, 28, 28, 28, 28, 81, 80, 78, 77, 75, 73, + 71, 69, 66, 64, 62, 59, 57, 55, 52, 50, 47, 45, 43, 41, 38, 37, 35, 34, + 32, 31, 30, 29, 28, 28, 28, 28, 80, 79, 78, 77, 76, 72, 69, 65, 61, 59, + 57, 55, 53, 51, 48, 46, 44, 43, 41, 39, 37, 36, 34, 33, 31, 30, 29, 28, + 27, 27, 27, 27, 78, 78, 77, 77, 77, 72, 66, 61, 56, 54, 52, 50, 48, 47, + 45, 43, 42, 40, 39, 37, 36, 34, 33, 32, 30, 29, 29, 28, 27, 27, 27, 27, + 73, 74, 74, 74, 74, 69, 64, 59, 54, 52, 50, 48, 46, 44, 43, 41, 40, 38, + 37, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 26, 26, 26, 69, 70, 70, 71, + 71, 67, 62, 57, 52, 50, 48, 46, 43, 42, 40, 39, 38, 36, 35, 34, 33, 32, + 31, 30, 29, 28, 27, 26, 25, 25, 25, 25, 64, 66, 67, 68, 69, 64, 59, 55, + 50, 48, 46, 43, 41, 40, 38, 37, 36, 35, 33, 32, 31, 30, 30, 29, 28, 27, + 26, 26, 25, 25, 25, 25, 60, 61, 63, 65, 66, 62, 57, 53, 48, 46, 43, 41, + 39, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 28, 27, 26, 26, 25, 24, 24, + 24, 24, 56, 58, 59, 61, 63, 59, 55, 51, 47, 44, 42, 40, 37, 36, 35, 34, + 32, 32, 31, 30, 29, 28, 27, 27, 26, 25, 25, 24, 24, 24, 24, 24, 53, 54, + 56, 58, 59, 56, 52, 48, 45, 43, 40, 38, 36, 35, 34, 32, 31, 30, 30, 29, + 28, 27, 27, 26, 25, 25, 24, 24, 23, 23, 23, 23, 49, 51, 52, 54, 56, 53, + 50, 46, 43, 41, 39, 37, 35, 34, 32, 31, 30, 29, 28, 28, 27, 26, 26, 25, + 25, 24, 24, 23, 23, 23, 23, 23, 45, 47, 49, 51, 52, 50, 47, 44, 42, 40, + 38, 36, 34, 32, 31, 30, 29, 28, 27, 27, 26, 25, 25, 24, 24, 23, 23, 23, + 22, 22, 22, 22, 43, 45, 46, 48, 50, 47, 45, 43, 40, 38, 36, 35, 33, 32, + 30, 29, 28, 27, 27, 26, 25, 25, 24, 24, 23, 23, 23, 22, 22, 22, 22, 22, + 41, 42, 44, 45, 47, 45, 43, 41, 39, 37, 35, 33, 32, 31, 30, 28, 27, 27, + 26, 25, 25, 24, 24, 23, 23, 22, 22, 22, 21, 21, 21, 21, 38, 40, 41, 43, + 44, 42, 41, 39, 37, 35, 34, 32, 31, 30, 29, 28, 27, 26, 25, 25, 24, 24, + 23, 23, 22, 22, 22, 21, 21, 21, 21, 21, 36, 37, 39, 40, 41, 40, 38, 37, + 36, 34, 33, 31, 30, 29, 28, 27, 26, 25, 25, 24, 23, 23, 23, 22, 22, 22, + 21, 21, 21, 21, 21, 21, 34, 35, 37, 38, 40, 38, 37, 36, 34, 33, 32, 30, + 29, 28, 27, 26, 25, 25, 24, 24, 23, 23, 22, 22, 21, 21, 21, 21, 20, 20, + 20, 20, 33, 34, 35, 36, 38, 36, 35, 34, 33, 32, 31, 30, 28, 27, 27, 26, + 25, 24, 24, 23, 23, 22, 22, 22, 21, 21, 21, 20, 20, 20, 20, 20, 31, 32, + 33, 35, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 24, 23, 23, + 22, 22, 22, 21, 21, 21, 20, 20, 20, 20, 20, 20, 29, 30, 32, 33, 34, 33, + 32, 31, 30, 30, 29, 28, 27, 26, 25, 25, 24, 23, 23, 22, 22, 21, 21, 21, + 20, 20, 20, 20, 19, 19, 19, 19, 28, 29, 30, 31, 32, 32, 31, 30, 29, 29, + 28, 27, 26, 25, 25, 24, 23, 23, 22, 22, 22, 21, 21, 21, 20, 20, 20, 19, + 19, 19, 19, 19, 27, 28, 29, 30, 31, 30, 30, 29, 29, 28, 27, 26, 26, 25, + 24, 24, 23, 23, 22, 22, 21, 21, 21, 20, 20, 20, 20, 19, 19, 19, 19, 19, + 26, 27, 28, 29, 30, 29, 29, 28, 28, 27, 26, 26, 25, 24, 24, 23, 23, 22, + 22, 21, 21, 21, 20, 20, 20, 19, 19, 19, 19, 19, 19, 19, 25, 26, 27, 28, + 28, 28, 28, 27, 27, 26, 25, 25, 24, 24, 23, 23, 22, 22, 21, 21, 21, 20, + 20, 20, 19, 19, 19, 19, 19, 19, 19, 19, 25, 26, 27, 28, 28, 28, 28, 27, + 27, 26, 25, 25, 24, 24, 23, 23, 22, 22, 21, 21, 21, 20, 20, 20, 19, 19, + 19, 19, 19, 19, 19, 19, 25, 26, 27, 28, 28, 28, 28, 27, 27, 26, 25, 25, + 24, 24, 23, 23, 22, 22, 21, 21, 21, 20, 20, 20, 19, 19, 19, 19, 19, 19, + 19, 19, 25, 26, 27, 28, 28, 28, 28, 27, 27, 26, 25, 25, 24, 24, 23, 23, + 22, 22, 21, 21, 21, 20, 20, 20, 19, 19, 19, 19, 19, 19, 19, 19 }, + { /* Intra matrices */ + /* Size 4 */ + 171, 157, 95, 62, 157, 104, 75, 57, 95, 75, 54, 45, 62, 57, 45, 39, + /* Size 8 */ + 144, 192, 177, 135, 101, 78, 63, 53, 192, 167, 174, 149, 117, 91, 73, + 61, 177, 174, 125, 107, 92, 77, 65, 56, 135, 149, 107, 84, 73, 64, 57, + 51, 101, 117, 92, 73, 62, 55, 50, 46, 78, 91, 77, 64, 55, 49, 45, 42, + 63, 73, 65, 57, 50, 45, 42, 39, 53, 61, 56, 51, 46, 42, 39, 38, + /* Size 16 */ + 149, 174, 199, 191, 184, 162, 140, 122, 104, 92, 81, 73, 65, 60, 55, 55, + 174, 180, 186, 184, 182, 165, 147, 130, 113, 100, 88, 79, 70, 65, 59, + 59, 199, 186, 173, 177, 181, 168, 154, 138, 121, 108, 95, 85, 76, 69, + 63, 63, 191, 184, 177, 166, 155, 144, 133, 120, 108, 98, 87, 80, 72, 66, + 61, 61, 184, 182, 181, 155, 129, 120, 111, 103, 95, 88, 80, 74, 68, 63, + 58, 58, 162, 165, 168, 144, 120, 110, 99, 92, 85, 79, 73, 68, 63, 59, + 55, 55, 140, 147, 154, 133, 111, 99, 88, 81, 75, 71, 66, 63, 59, 56, 53, + 53, 122, 130, 138, 120, 103, 92, 81, 75, 70, 66, 62, 58, 55, 53, 50, 50, + 104, 113, 121, 108, 95, 85, 75, 70, 64, 60, 57, 54, 52, 50, 48, 48, 92, + 100, 108, 98, 88, 79, 71, 66, 60, 57, 54, 51, 49, 47, 46, 46, 81, 88, + 95, 87, 80, 73, 66, 62, 57, 54, 51, 49, 47, 45, 44, 44, 73, 79, 85, 80, + 74, 68, 63, 58, 54, 51, 49, 47, 45, 44, 42, 42, 65, 70, 76, 72, 68, 63, + 59, 55, 52, 49, 47, 45, 43, 42, 41, 41, 60, 65, 69, 66, 63, 59, 56, 53, + 50, 47, 45, 44, 42, 41, 40, 40, 55, 59, 63, 61, 58, 55, 53, 50, 48, 46, + 44, 42, 41, 40, 39, 39, 55, 59, 63, 61, 58, 55, 53, 50, 48, 46, 44, 42, + 41, 40, 39, 39, + /* Size 32 */ + 152, 165, 177, 190, 202, 198, 195, 191, 187, 176, 165, 153, 142, 133, + 124, 115, 106, 100, 94, 88, 82, 78, 74, 70, 66, 64, 61, 58, 56, 56, 56, + 56, 165, 173, 180, 188, 196, 193, 191, 189, 186, 176, 166, 156, 146, + 137, 128, 119, 111, 104, 98, 92, 86, 82, 77, 73, 69, 66, 63, 61, 58, 58, + 58, 58, 177, 180, 183, 186, 189, 188, 187, 187, 186, 177, 168, 159, 150, + 141, 132, 124, 115, 108, 102, 96, 89, 85, 80, 76, 72, 69, 66, 63, 60, + 60, 60, 60, 190, 188, 186, 185, 183, 183, 184, 184, 185, 177, 169, 161, + 154, 145, 136, 128, 119, 112, 106, 99, 93, 88, 84, 79, 74, 71, 68, 65, + 62, 62, 62, 62, 202, 196, 189, 183, 176, 178, 180, 182, 184, 177, 171, + 164, 157, 149, 140, 132, 123, 117, 110, 103, 96, 92, 87, 82, 77, 74, 71, + 67, 64, 64, 64, 64, 198, 193, 188, 183, 178, 177, 175, 173, 171, 165, + 159, 152, 146, 139, 131, 124, 117, 111, 105, 99, 93, 88, 84, 80, 75, 72, + 69, 66, 63, 63, 63, 63, 195, 191, 187, 184, 180, 175, 169, 164, 158, + 152, 147, 141, 135, 129, 123, 116, 110, 105, 99, 94, 89, 85, 81, 77, 73, + 70, 67, 65, 62, 62, 62, 62, 191, 189, 187, 184, 182, 173, 164, 154, 145, + 140, 134, 129, 124, 119, 114, 109, 103, 99, 94, 90, 85, 82, 78, 75, 71, + 68, 66, 63, 61, 61, 61, 61, 187, 186, 186, 185, 184, 171, 158, 145, 132, + 127, 122, 118, 113, 109, 105, 101, 97, 93, 89, 85, 82, 78, 75, 72, 69, + 67, 64, 62, 59, 59, 59, 59, 176, 176, 177, 177, 177, 165, 152, 140, 127, + 122, 117, 112, 107, 103, 99, 96, 92, 88, 85, 81, 78, 75, 72, 70, 67, 64, + 62, 60, 58, 58, 58, 58, 165, 166, 168, 169, 171, 159, 147, 134, 122, + 117, 112, 106, 101, 97, 94, 90, 87, 84, 81, 78, 75, 72, 69, 67, 64, 62, + 60, 58, 56, 56, 56, 56, 153, 156, 159, 161, 164, 152, 141, 129, 118, + 112, 106, 101, 95, 92, 88, 85, 82, 79, 76, 74, 71, 69, 67, 64, 62, 60, + 59, 57, 55, 55, 55, 55, 142, 146, 150, 154, 157, 146, 135, 124, 113, + 107, 101, 95, 89, 86, 83, 80, 77, 74, 72, 70, 68, 66, 64, 62, 60, 58, + 57, 55, 54, 54, 54, 54, 133, 137, 141, 145, 149, 139, 129, 119, 109, + 103, 97, 92, 86, 83, 80, 77, 74, 72, 69, 67, 65, 63, 62, 60, 58, 57, 55, + 54, 52, 52, 52, 52, 124, 128, 132, 136, 140, 131, 123, 114, 105, 99, 94, + 88, 83, 80, 77, 74, 71, 69, 67, 65, 63, 61, 59, 58, 56, 55, 54, 52, 51, + 51, 51, 51, 115, 119, 124, 128, 132, 124, 116, 109, 101, 96, 90, 85, 80, + 77, 74, 71, 68, 66, 64, 62, 60, 59, 57, 56, 54, 53, 52, 51, 50, 50, 50, + 50, 106, 111, 115, 119, 123, 117, 110, 103, 97, 92, 87, 82, 77, 74, 71, + 68, 65, 63, 61, 60, 58, 57, 55, 54, 53, 52, 51, 50, 48, 48, 48, 48, 100, + 104, 108, 112, 117, 111, 105, 99, 93, 88, 84, 79, 74, 72, 69, 66, 63, + 61, 60, 58, 56, 55, 54, 53, 51, 50, 49, 48, 47, 47, 47, 47, 94, 98, 102, + 106, 110, 105, 99, 94, 89, 85, 81, 76, 72, 69, 67, 64, 61, 60, 58, 56, + 55, 54, 52, 51, 50, 49, 48, 47, 46, 46, 46, 46, 88, 92, 96, 99, 103, 99, + 94, 90, 85, 81, 78, 74, 70, 67, 65, 62, 60, 58, 56, 55, 53, 52, 51, 50, + 49, 48, 47, 46, 46, 46, 46, 46, 82, 86, 89, 93, 96, 93, 89, 85, 82, 78, + 75, 71, 68, 65, 63, 60, 58, 56, 55, 53, 52, 51, 50, 49, 48, 47, 46, 45, + 45, 45, 45, 45, 78, 82, 85, 88, 92, 88, 85, 82, 78, 75, 72, 69, 66, 63, + 61, 59, 57, 55, 54, 52, 51, 50, 49, 48, 47, 46, 45, 45, 44, 44, 44, 44, + 74, 77, 80, 84, 87, 84, 81, 78, 75, 72, 69, 67, 64, 62, 59, 57, 55, 54, + 52, 51, 50, 49, 48, 47, 46, 45, 44, 44, 43, 43, 43, 43, 70, 73, 76, 79, + 82, 80, 77, 75, 72, 70, 67, 64, 62, 60, 58, 56, 54, 53, 51, 50, 49, 48, + 47, 46, 45, 44, 44, 43, 42, 42, 42, 42, 66, 69, 72, 74, 77, 75, 73, 71, + 69, 67, 64, 62, 60, 58, 56, 54, 53, 51, 50, 49, 48, 47, 46, 45, 44, 44, + 43, 42, 42, 42, 42, 42, 64, 66, 69, 71, 74, 72, 70, 68, 67, 64, 62, 60, + 58, 57, 55, 53, 52, 50, 49, 48, 47, 46, 45, 44, 44, 43, 42, 42, 41, 41, + 41, 41, 61, 63, 66, 68, 71, 69, 67, 66, 64, 62, 60, 59, 57, 55, 54, 52, + 51, 49, 48, 47, 46, 45, 44, 44, 43, 42, 42, 41, 41, 41, 41, 41, 58, 61, + 63, 65, 67, 66, 65, 63, 62, 60, 58, 57, 55, 54, 52, 51, 50, 48, 47, 46, + 45, 45, 44, 43, 42, 42, 41, 41, 40, 40, 40, 40, 56, 58, 60, 62, 64, 63, + 62, 61, 59, 58, 56, 55, 54, 52, 51, 50, 48, 47, 46, 46, 45, 44, 43, 42, + 42, 41, 41, 40, 40, 40, 40, 40, 56, 58, 60, 62, 64, 63, 62, 61, 59, 58, + 56, 55, 54, 52, 51, 50, 48, 47, 46, 46, 45, 44, 43, 42, 42, 41, 41, 40, + 40, 40, 40, 40, 56, 58, 60, 62, 64, 63, 62, 61, 59, 58, 56, 55, 54, 52, + 51, 50, 48, 47, 46, 46, 45, 44, 43, 42, 42, 41, 41, 40, 40, 40, 40, 40, + 56, 58, 60, 62, 64, 63, 62, 61, 59, 58, 56, 55, 54, 52, 51, 50, 48, 47, + 46, 46, 45, 44, 43, 42, 42, 41, 41, 40, 40, 40, 40, 40 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 41, 38, 31, 41, 35, 33, 29, 38, 33, 26, 23, 31, 29, 23, 20, + /* Size 8 */ + 64, 79, 45, 42, 40, 36, 32, 29, 79, 55, 44, 48, 47, 43, 38, 34, 45, 44, + 38, 39, 39, 37, 35, 32, 42, 48, 39, 35, 33, 32, 31, 29, 40, 47, 39, 33, + 30, 28, 27, 26, 36, 43, 37, 32, 28, 26, 25, 23, 32, 38, 35, 31, 27, 25, + 23, 22, 29, 34, 32, 29, 26, 23, 22, 21, + /* Size 16 */ + 64, 71, 79, 62, 45, 43, 42, 41, 40, 38, 36, 34, 32, 30, 29, 29, 71, 69, + 67, 56, 44, 45, 45, 44, 43, 41, 39, 37, 35, 33, 31, 31, 79, 67, 55, 50, + 44, 46, 48, 47, 47, 45, 43, 41, 38, 36, 34, 34, 62, 56, 50, 45, 41, 42, + 44, 43, 43, 42, 40, 38, 37, 35, 33, 33, 45, 44, 44, 41, 38, 39, 39, 39, + 39, 38, 37, 36, 35, 33, 32, 32, 43, 45, 46, 42, 39, 38, 37, 37, 36, 36, + 35, 34, 33, 31, 30, 30, 42, 45, 48, 44, 39, 37, 35, 34, 33, 33, 32, 31, + 31, 30, 29, 29, 41, 44, 47, 43, 39, 37, 34, 33, 32, 31, 30, 30, 29, 28, + 27, 27, 40, 43, 47, 43, 39, 36, 33, 32, 30, 29, 28, 28, 27, 26, 26, 26, + 38, 41, 45, 42, 38, 36, 33, 31, 29, 28, 27, 27, 26, 25, 25, 25, 36, 39, + 43, 40, 37, 35, 32, 30, 28, 27, 26, 25, 25, 24, 23, 23, 34, 37, 41, 38, + 36, 34, 31, 30, 28, 27, 25, 25, 24, 23, 23, 23, 32, 35, 38, 37, 35, 33, + 31, 29, 27, 26, 25, 24, 23, 22, 22, 22, 30, 33, 36, 35, 33, 31, 30, 28, + 26, 25, 24, 23, 22, 22, 21, 21, 29, 31, 34, 33, 32, 30, 29, 27, 26, 25, + 23, 23, 22, 21, 21, 21, 29, 31, 34, 33, 32, 30, 29, 27, 26, 25, 23, 23, + 22, 21, 21, 21, + /* Size 32 */ + 64, 68, 71, 75, 79, 70, 62, 53, 45, 44, 43, 43, 42, 42, 41, 40, 40, 39, + 38, 37, 36, 35, 34, 33, 32, 31, 30, 30, 29, 29, 29, 29, 68, 69, 70, 72, + 73, 66, 59, 52, 44, 44, 44, 44, 44, 43, 43, 42, 42, 41, 40, 39, 38, 37, + 36, 35, 34, 33, 32, 31, 30, 30, 30, 30, 71, 70, 69, 68, 67, 61, 56, 50, + 44, 44, 45, 45, 45, 45, 44, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, + 33, 32, 31, 31, 31, 31, 75, 72, 68, 65, 61, 57, 53, 48, 44, 45, 45, 46, + 47, 46, 46, 45, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 33, + 33, 33, 79, 73, 67, 61, 55, 52, 50, 47, 44, 45, 46, 47, 48, 48, 47, 47, + 47, 46, 45, 44, 43, 42, 41, 40, 38, 37, 36, 35, 34, 34, 34, 34, 70, 66, + 61, 57, 52, 50, 47, 45, 42, 43, 44, 45, 46, 46, 45, 45, 45, 44, 43, 42, + 42, 41, 39, 38, 37, 36, 36, 35, 34, 34, 34, 34, 62, 59, 56, 53, 50, 47, + 45, 43, 41, 42, 42, 43, 44, 43, 43, 43, 43, 42, 42, 41, 40, 39, 38, 37, + 37, 36, 35, 34, 33, 33, 33, 33, 53, 52, 50, 48, 47, 45, 43, 41, 39, 40, + 40, 41, 41, 41, 41, 41, 41, 41, 40, 39, 39, 38, 37, 36, 36, 35, 34, 33, + 32, 32, 32, 32, 45, 44, 44, 44, 44, 42, 41, 39, 38, 38, 39, 39, 39, 39, + 39, 39, 39, 39, 38, 38, 37, 37, 36, 35, 35, 34, 33, 32, 32, 32, 32, 32, + 44, 44, 44, 45, 45, 43, 42, 40, 38, 38, 38, 38, 38, 38, 38, 38, 38, 37, + 37, 37, 36, 36, 35, 34, 34, 33, 32, 32, 31, 31, 31, 31, 43, 44, 45, 45, + 46, 44, 42, 40, 39, 38, 38, 37, 37, 37, 37, 37, 36, 36, 36, 35, 35, 34, + 34, 33, 33, 32, 31, 31, 30, 30, 30, 30, 43, 44, 45, 46, 47, 45, 43, 41, + 39, 38, 37, 37, 36, 36, 35, 35, 35, 35, 34, 34, 34, 33, 33, 32, 32, 31, + 30, 30, 29, 29, 29, 29, 42, 44, 45, 47, 48, 46, 44, 41, 39, 38, 37, 36, + 35, 35, 34, 34, 33, 33, 33, 33, 32, 32, 31, 31, 31, 30, 30, 29, 29, 29, + 29, 29, 42, 43, 45, 46, 48, 46, 43, 41, 39, 38, 37, 36, 35, 34, 34, 33, + 33, 32, 32, 32, 31, 31, 30, 30, 30, 29, 29, 28, 28, 28, 28, 28, 41, 43, + 44, 46, 47, 45, 43, 41, 39, 38, 37, 35, 34, 34, 33, 32, 32, 31, 31, 31, + 30, 30, 30, 29, 29, 28, 28, 28, 27, 27, 27, 27, 40, 42, 44, 45, 47, 45, + 43, 41, 39, 38, 37, 35, 34, 33, 32, 32, 31, 31, 30, 30, 29, 29, 29, 28, + 28, 28, 27, 27, 26, 26, 26, 26, 40, 42, 43, 45, 47, 45, 43, 41, 39, 38, + 36, 35, 33, 33, 32, 31, 30, 30, 29, 29, 28, 28, 28, 27, 27, 27, 26, 26, + 26, 26, 26, 26, 39, 41, 42, 44, 46, 44, 42, 41, 39, 37, 36, 35, 33, 32, + 31, 31, 30, 29, 29, 28, 28, 28, 27, 27, 26, 26, 26, 25, 25, 25, 25, 25, + 38, 40, 41, 43, 45, 43, 42, 40, 38, 37, 36, 34, 33, 32, 31, 30, 29, 29, + 28, 28, 27, 27, 27, 26, 26, 26, 25, 25, 25, 25, 25, 25, 37, 39, 40, 42, + 44, 42, 41, 39, 38, 37, 35, 34, 33, 32, 31, 30, 29, 28, 28, 27, 27, 26, + 26, 26, 25, 25, 25, 24, 24, 24, 24, 24, 36, 38, 39, 41, 43, 42, 40, 39, + 37, 36, 35, 34, 32, 31, 30, 29, 28, 28, 27, 27, 26, 26, 25, 25, 25, 24, + 24, 24, 23, 23, 23, 23, 35, 37, 38, 40, 42, 41, 39, 38, 37, 36, 34, 33, + 32, 31, 30, 29, 28, 28, 27, 26, 26, 25, 25, 25, 24, 24, 24, 23, 23, 23, + 23, 23, 34, 36, 37, 39, 41, 39, 38, 37, 36, 35, 34, 33, 31, 30, 30, 29, + 28, 27, 27, 26, 25, 25, 25, 24, 24, 24, 23, 23, 23, 23, 23, 23, 33, 35, + 36, 38, 40, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 27, 26, 26, + 25, 25, 24, 24, 23, 23, 23, 23, 22, 22, 22, 22, 32, 34, 35, 37, 38, 37, + 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 26, 25, 25, 24, 24, 23, + 23, 23, 22, 22, 22, 22, 22, 22, 31, 33, 34, 36, 37, 36, 36, 35, 34, 33, + 32, 31, 30, 29, 28, 28, 27, 26, 26, 25, 24, 24, 24, 23, 23, 22, 22, 22, + 22, 22, 22, 22, 30, 32, 33, 35, 36, 36, 35, 34, 33, 32, 31, 30, 30, 29, + 28, 27, 26, 26, 25, 25, 24, 24, 23, 23, 22, 22, 22, 22, 21, 21, 21, 21, + 30, 31, 32, 34, 35, 35, 34, 33, 32, 32, 31, 30, 29, 28, 28, 27, 26, 25, + 25, 24, 24, 23, 23, 23, 22, 22, 22, 21, 21, 21, 21, 21, 29, 30, 31, 33, + 34, 34, 33, 32, 32, 31, 30, 29, 29, 28, 27, 26, 26, 25, 25, 24, 23, 23, + 23, 22, 22, 22, 21, 21, 21, 21, 21, 21, 29, 30, 31, 33, 34, 34, 33, 32, + 32, 31, 30, 29, 29, 28, 27, 26, 26, 25, 25, 24, 23, 23, 23, 22, 22, 22, + 21, 21, 21, 21, 21, 21, 29, 30, 31, 33, 34, 34, 33, 32, 32, 31, 30, 29, + 29, 28, 27, 26, 26, 25, 25, 24, 23, 23, 23, 22, 22, 22, 21, 21, 21, 21, + 21, 21, 29, 30, 31, 33, 34, 34, 33, 32, 32, 31, 30, 29, 29, 28, 27, 26, + 26, 25, 25, 24, 23, 23, 23, 22, 22, 22, 21, 21, 21, 21, 21, 21 }, + { /* Intra matrices */ + /* Size 4 */ + 142, 90, 83, 66, 90, 75, 71, 62, 83, 71, 55, 49, 66, 62, 49, 41, + /* Size 8 */ + 132, 163, 91, 85, 80, 72, 64, 57, 163, 113, 89, 98, 95, 87, 77, 68, 91, + 89, 76, 79, 79, 75, 69, 63, 85, 98, 79, 70, 67, 64, 60, 56, 80, 95, 79, + 67, 60, 56, 53, 50, 72, 87, 75, 64, 56, 51, 48, 45, 64, 77, 69, 60, 53, + 48, 44, 42, 57, 68, 63, 56, 50, 45, 42, 39, + /* Size 16 */ + 136, 152, 168, 130, 93, 90, 88, 85, 82, 78, 74, 70, 65, 62, 58, 58, 152, + 147, 142, 117, 92, 93, 94, 92, 90, 86, 81, 77, 72, 68, 64, 64, 168, 142, + 116, 104, 91, 96, 101, 99, 98, 93, 89, 84, 79, 75, 70, 70, 130, 117, + 104, 94, 85, 88, 91, 90, 90, 86, 83, 79, 75, 71, 67, 67, 93, 92, 91, 85, + 78, 80, 81, 81, 81, 79, 77, 74, 71, 68, 65, 65, 90, 93, 96, 88, 80, 78, + 76, 76, 75, 73, 71, 69, 67, 64, 61, 61, 88, 94, 101, 91, 81, 76, 72, 70, + 68, 67, 66, 64, 62, 60, 58, 58, 85, 92, 99, 90, 81, 76, 70, 68, 65, 63, + 62, 60, 58, 56, 55, 55, 82, 90, 98, 90, 81, 75, 68, 65, 61, 60, 58, 56, + 54, 53, 51, 51, 78, 86, 93, 86, 79, 73, 67, 63, 60, 57, 55, 53, 52, 50, + 49, 49, 74, 81, 89, 83, 77, 71, 66, 62, 58, 55, 52, 51, 49, 48, 47, 47, + 70, 77, 84, 79, 74, 69, 64, 60, 56, 53, 51, 49, 47, 46, 45, 45, 65, 72, + 79, 75, 71, 67, 62, 58, 54, 52, 49, 47, 46, 44, 43, 43, 62, 68, 75, 71, + 68, 64, 60, 56, 53, 50, 48, 46, 44, 43, 42, 42, 58, 64, 70, 67, 65, 61, + 58, 55, 51, 49, 47, 45, 43, 42, 40, 40, 58, 64, 70, 67, 65, 61, 58, 55, + 51, 49, 47, 45, 43, 42, 40, 40, + /* Size 32 */ + 137, 146, 154, 162, 170, 151, 132, 113, 94, 93, 92, 90, 89, 88, 86, 85, + 83, 81, 79, 77, 75, 73, 71, 68, 66, 64, 63, 61, 59, 59, 59, 59, 146, + 148, 151, 154, 157, 141, 125, 110, 94, 93, 93, 93, 92, 91, 90, 89, 87, + 85, 83, 81, 79, 76, 74, 72, 70, 68, 66, 64, 62, 62, 62, 62, 154, 151, + 149, 146, 144, 131, 119, 106, 93, 94, 94, 95, 96, 94, 93, 92, 91, 89, + 87, 85, 83, 80, 78, 76, 73, 71, 69, 67, 65, 65, 65, 65, 162, 154, 146, + 139, 131, 122, 112, 103, 93, 94, 96, 97, 99, 98, 97, 96, 95, 93, 91, 89, + 86, 84, 82, 79, 77, 75, 72, 70, 68, 68, 68, 68, 170, 157, 144, 131, 118, + 112, 105, 99, 93, 95, 97, 100, 102, 101, 101, 100, 99, 97, 95, 93, 90, + 88, 85, 83, 80, 78, 76, 73, 71, 71, 71, 71, 151, 141, 131, 122, 112, + 106, 100, 95, 89, 91, 93, 95, 97, 97, 96, 95, 95, 93, 91, 89, 87, 85, + 83, 81, 78, 76, 74, 72, 70, 70, 70, 70, 132, 125, 119, 112, 105, 100, + 96, 91, 86, 87, 89, 91, 92, 92, 91, 91, 91, 89, 88, 86, 84, 82, 80, 78, + 76, 74, 72, 70, 68, 68, 68, 68, 113, 110, 106, 103, 99, 95, 91, 87, 83, + 84, 85, 86, 87, 87, 87, 87, 87, 85, 84, 83, 81, 80, 78, 76, 74, 72, 70, + 69, 67, 67, 67, 67, 94, 94, 93, 93, 93, 89, 86, 83, 79, 80, 81, 81, 82, + 82, 82, 82, 82, 81, 80, 79, 78, 77, 75, 74, 72, 70, 69, 67, 65, 65, 65, + 65, 93, 93, 94, 94, 95, 91, 87, 84, 80, 80, 80, 80, 80, 80, 79, 79, 79, + 78, 77, 76, 75, 74, 73, 71, 70, 68, 67, 65, 64, 64, 64, 64, 92, 93, 94, + 96, 97, 93, 89, 85, 81, 80, 79, 78, 77, 77, 77, 76, 76, 75, 74, 73, 72, + 71, 70, 69, 67, 66, 65, 63, 62, 62, 62, 62, 90, 93, 95, 97, 100, 95, 91, + 86, 81, 80, 78, 77, 75, 74, 74, 73, 73, 72, 71, 70, 69, 68, 67, 66, 65, + 64, 63, 61, 60, 60, 60, 60, 89, 92, 96, 99, 102, 97, 92, 87, 82, 80, 77, + 75, 73, 72, 71, 70, 69, 69, 68, 67, 67, 66, 65, 64, 63, 62, 61, 60, 59, + 59, 59, 59, 88, 91, 94, 98, 101, 97, 92, 87, 82, 80, 77, 74, 72, 71, 70, + 69, 68, 67, 66, 65, 64, 64, 63, 62, 61, 60, 59, 58, 57, 57, 57, 57, 86, + 90, 93, 97, 101, 96, 91, 87, 82, 79, 77, 74, 71, 70, 68, 67, 66, 65, 64, + 63, 62, 62, 61, 60, 59, 58, 57, 56, 55, 55, 55, 55, 85, 89, 92, 96, 100, + 95, 91, 87, 82, 79, 76, 73, 70, 69, 67, 66, 64, 63, 62, 61, 60, 60, 59, + 58, 57, 56, 55, 55, 54, 54, 54, 54, 83, 87, 91, 95, 99, 95, 91, 87, 82, + 79, 76, 73, 69, 68, 66, 64, 62, 61, 60, 59, 58, 58, 57, 56, 55, 54, 54, + 53, 52, 52, 52, 52, 81, 85, 89, 93, 97, 93, 89, 85, 81, 78, 75, 72, 69, + 67, 65, 63, 61, 60, 59, 58, 57, 56, 55, 55, 54, 53, 52, 52, 51, 51, 51, + 51, 79, 83, 87, 91, 95, 91, 88, 84, 80, 77, 74, 71, 68, 66, 64, 62, 60, + 59, 58, 57, 56, 55, 54, 53, 53, 52, 51, 50, 50, 50, 50, 50, 77, 81, 85, + 89, 93, 89, 86, 83, 79, 76, 73, 70, 67, 65, 63, 61, 59, 58, 57, 56, 54, + 54, 53, 52, 51, 50, 50, 49, 48, 48, 48, 48, 75, 79, 83, 86, 90, 87, 84, + 81, 78, 75, 72, 69, 67, 64, 62, 60, 58, 57, 56, 54, 53, 52, 52, 51, 50, + 49, 48, 48, 47, 47, 47, 47, 73, 76, 80, 84, 88, 85, 82, 80, 77, 74, 71, + 68, 66, 64, 62, 60, 58, 56, 55, 54, 52, 52, 51, 50, 49, 48, 48, 47, 46, + 46, 46, 46, 71, 74, 78, 82, 85, 83, 80, 78, 75, 73, 70, 67, 65, 63, 61, + 59, 57, 55, 54, 53, 52, 51, 50, 49, 48, 47, 47, 46, 45, 45, 45, 45, 68, + 72, 76, 79, 83, 81, 78, 76, 74, 71, 69, 66, 64, 62, 60, 58, 56, 55, 53, + 52, 51, 50, 49, 48, 47, 46, 46, 45, 44, 44, 44, 44, 66, 70, 73, 77, 80, + 78, 76, 74, 72, 70, 67, 65, 63, 61, 59, 57, 55, 54, 53, 51, 50, 49, 48, + 47, 46, 45, 45, 44, 44, 44, 44, 44, 64, 68, 71, 75, 78, 76, 74, 72, 70, + 68, 66, 64, 62, 60, 58, 56, 54, 53, 52, 50, 49, 48, 47, 46, 45, 45, 44, + 44, 43, 43, 43, 43, 63, 66, 69, 72, 76, 74, 72, 70, 69, 67, 65, 63, 61, + 59, 57, 55, 54, 52, 51, 50, 48, 48, 47, 46, 45, 44, 44, 43, 42, 42, 42, + 42, 61, 64, 67, 70, 73, 72, 70, 69, 67, 65, 63, 61, 60, 58, 56, 55, 53, + 52, 50, 49, 48, 47, 46, 45, 44, 44, 43, 42, 42, 42, 42, 42, 59, 62, 65, + 68, 71, 70, 68, 67, 65, 64, 62, 60, 59, 57, 55, 54, 52, 51, 50, 48, 47, + 46, 45, 44, 44, 43, 42, 42, 41, 41, 41, 41, 59, 62, 65, 68, 71, 70, 68, + 67, 65, 64, 62, 60, 59, 57, 55, 54, 52, 51, 50, 48, 47, 46, 45, 44, 44, + 43, 42, 42, 41, 41, 41, 41, 59, 62, 65, 68, 71, 70, 68, 67, 65, 64, 62, + 60, 59, 57, 55, 54, 52, 51, 50, 48, 47, 46, 45, 44, 44, 43, 42, 42, 41, + 41, 41, 41, 59, 62, 65, 68, 71, 70, 68, 67, 65, 64, 62, 60, 59, 57, 55, + 54, 52, 51, 50, 48, 47, 46, 45, 44, 44, 43, 42, 42, 41, 41, 41, + 41 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 59, 38, 26, 59, 41, 31, 25, 38, 31, 24, 21, 26, 25, 21, 19, + /* Size 8 */ + 64, 83, 77, 60, 47, 37, 31, 28, 83, 73, 76, 66, 53, 43, 36, 31, 77, 76, + 56, 49, 43, 37, 32, 29, 60, 66, 49, 40, 35, 32, 29, 27, 47, 53, 43, 35, + 31, 28, 26, 25, 37, 43, 37, 32, 28, 26, 24, 23, 31, 36, 32, 29, 26, 24, + 23, 22, 28, 31, 29, 27, 25, 23, 22, 21, + /* Size 16 */ + 64, 73, 83, 80, 77, 69, 60, 53, 47, 42, 37, 34, 31, 30, 28, 28, 73, 76, + 78, 77, 77, 70, 63, 56, 50, 45, 40, 37, 34, 31, 29, 29, 83, 78, 73, 75, + 76, 71, 66, 59, 53, 48, 43, 39, 36, 33, 31, 31, 80, 77, 75, 70, 66, 62, + 58, 53, 48, 44, 40, 37, 34, 32, 30, 30, 77, 77, 76, 66, 56, 53, 49, 46, + 43, 40, 37, 35, 32, 31, 29, 29, 69, 70, 71, 62, 53, 49, 45, 42, 39, 37, + 35, 33, 31, 29, 28, 28, 60, 63, 66, 58, 49, 45, 40, 38, 35, 34, 32, 31, + 29, 28, 27, 27, 53, 56, 59, 53, 46, 42, 38, 35, 33, 32, 30, 29, 28, 27, + 26, 26, 47, 50, 53, 48, 43, 39, 35, 33, 31, 30, 28, 27, 26, 26, 25, 25, + 42, 45, 48, 44, 40, 37, 34, 32, 30, 28, 27, 26, 25, 25, 24, 24, 37, 40, + 43, 40, 37, 35, 32, 30, 28, 27, 26, 25, 24, 24, 23, 23, 34, 37, 39, 37, + 35, 33, 31, 29, 27, 26, 25, 24, 24, 23, 23, 23, 31, 34, 36, 34, 32, 31, + 29, 28, 26, 25, 24, 24, 23, 23, 22, 22, 30, 31, 33, 32, 31, 29, 28, 27, + 26, 25, 24, 23, 23, 22, 22, 22, 28, 29, 31, 30, 29, 28, 27, 26, 25, 24, + 23, 23, 22, 22, 21, 21, 28, 29, 31, 30, 29, 28, 27, 26, 25, 24, 23, 23, + 22, 22, 21, 21, + /* Size 32 */ + 64, 69, 73, 78, 83, 81, 80, 79, 77, 73, 69, 64, 60, 57, 53, 50, 47, 44, + 42, 40, 37, 36, 34, 33, 31, 30, 30, 29, 28, 28, 28, 28, 69, 72, 75, 78, + 80, 80, 79, 78, 77, 73, 69, 65, 62, 58, 55, 52, 48, 46, 44, 41, 39, 37, + 36, 34, 33, 31, 30, 29, 28, 28, 28, 28, 73, 75, 76, 77, 78, 78, 77, 77, + 77, 73, 70, 66, 63, 60, 56, 53, 50, 47, 45, 43, 40, 38, 37, 35, 34, 32, + 31, 30, 29, 29, 29, 29, 78, 78, 77, 76, 76, 76, 76, 76, 76, 73, 70, 67, + 64, 61, 58, 55, 51, 49, 46, 44, 41, 40, 38, 36, 35, 33, 32, 31, 30, 30, + 30, 30, 83, 80, 78, 76, 73, 74, 75, 75, 76, 74, 71, 68, 66, 63, 59, 56, + 53, 50, 48, 45, 43, 41, 39, 37, 36, 34, 33, 32, 31, 31, 31, 31, 81, 80, + 78, 76, 74, 73, 72, 72, 71, 69, 66, 64, 62, 59, 56, 53, 51, 48, 46, 44, + 41, 40, 38, 36, 35, 34, 33, 31, 30, 30, 30, 30, 80, 79, 77, 76, 75, 72, + 70, 68, 66, 64, 62, 60, 58, 55, 53, 50, 48, 46, 44, 42, 40, 39, 37, 36, + 34, 33, 32, 31, 30, 30, 30, 30, 79, 78, 77, 76, 75, 72, 68, 65, 61, 59, + 57, 55, 53, 51, 49, 47, 46, 44, 42, 40, 39, 37, 36, 35, 33, 32, 31, 30, + 29, 29, 29, 29, 77, 77, 77, 76, 76, 71, 66, 61, 56, 54, 53, 51, 49, 48, + 46, 45, 43, 42, 40, 39, 37, 36, 35, 34, 32, 32, 31, 30, 29, 29, 29, 29, + 73, 73, 73, 73, 74, 69, 64, 59, 54, 53, 51, 49, 47, 45, 44, 43, 41, 40, + 39, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 28, 28, 28, 69, 69, 70, 70, + 71, 66, 62, 57, 53, 51, 49, 47, 45, 43, 42, 41, 39, 38, 37, 36, 35, 34, + 33, 32, 31, 30, 29, 29, 28, 28, 28, 28, 64, 65, 66, 67, 68, 64, 60, 55, + 51, 49, 47, 45, 42, 41, 40, 39, 37, 36, 35, 34, 33, 32, 32, 31, 30, 29, + 29, 28, 27, 27, 27, 27, 60, 62, 63, 64, 66, 62, 58, 53, 49, 47, 45, 42, + 40, 39, 38, 37, 35, 35, 34, 33, 32, 31, 31, 30, 29, 28, 28, 27, 27, 27, + 27, 27, 57, 58, 60, 61, 63, 59, 55, 51, 48, 45, 43, 41, 39, 38, 37, 35, + 34, 33, 33, 32, 31, 30, 30, 29, 28, 28, 27, 27, 26, 26, 26, 26, 53, 55, + 56, 58, 59, 56, 53, 49, 46, 44, 42, 40, 38, 37, 35, 34, 33, 32, 32, 31, + 30, 30, 29, 28, 28, 27, 27, 26, 26, 26, 26, 26, 50, 52, 53, 55, 56, 53, + 50, 47, 45, 43, 41, 39, 37, 35, 34, 33, 32, 31, 31, 30, 29, 29, 28, 28, + 27, 27, 26, 26, 25, 25, 25, 25, 47, 48, 50, 51, 53, 51, 48, 46, 43, 41, + 39, 37, 35, 34, 33, 32, 31, 30, 30, 29, 28, 28, 27, 27, 26, 26, 26, 25, + 25, 25, 25, 25, 44, 46, 47, 49, 50, 48, 46, 44, 42, 40, 38, 36, 35, 33, + 32, 31, 30, 30, 29, 28, 28, 27, 27, 26, 26, 25, 25, 25, 24, 24, 24, 24, + 42, 44, 45, 46, 48, 46, 44, 42, 40, 39, 37, 35, 34, 33, 32, 31, 30, 29, + 28, 28, 27, 27, 26, 26, 25, 25, 25, 24, 24, 24, 24, 24, 40, 41, 43, 44, + 45, 44, 42, 40, 39, 37, 36, 34, 33, 32, 31, 30, 29, 28, 28, 27, 27, 26, + 26, 25, 25, 25, 24, 24, 24, 24, 24, 24, 37, 39, 40, 41, 43, 41, 40, 39, + 37, 36, 35, 33, 32, 31, 30, 29, 28, 28, 27, 27, 26, 26, 25, 25, 24, 24, + 24, 24, 23, 23, 23, 23, 36, 37, 38, 40, 41, 40, 39, 37, 36, 35, 34, 32, + 31, 30, 30, 29, 28, 27, 27, 26, 26, 25, 25, 24, 24, 24, 24, 23, 23, 23, + 23, 23, 34, 36, 37, 38, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, + 27, 27, 26, 26, 25, 25, 24, 24, 24, 23, 23, 23, 23, 23, 23, 23, 33, 34, + 35, 36, 37, 36, 36, 35, 34, 33, 32, 31, 30, 29, 28, 28, 27, 26, 26, 25, + 25, 24, 24, 24, 23, 23, 23, 23, 22, 22, 22, 22, 31, 33, 34, 35, 36, 35, + 34, 33, 32, 32, 31, 30, 29, 28, 28, 27, 26, 26, 25, 25, 24, 24, 24, 23, + 23, 23, 23, 22, 22, 22, 22, 22, 30, 31, 32, 33, 34, 34, 33, 32, 32, 31, + 30, 29, 28, 28, 27, 27, 26, 25, 25, 25, 24, 24, 23, 23, 23, 23, 22, 22, + 22, 22, 22, 22, 30, 30, 31, 32, 33, 33, 32, 31, 31, 30, 29, 29, 28, 27, + 27, 26, 26, 25, 25, 24, 24, 24, 23, 23, 23, 22, 22, 22, 22, 22, 22, 22, + 29, 29, 30, 31, 32, 31, 31, 30, 30, 29, 29, 28, 27, 27, 26, 26, 25, 25, + 24, 24, 24, 23, 23, 23, 22, 22, 22, 22, 22, 22, 22, 22, 28, 28, 29, 30, + 31, 30, 30, 29, 29, 28, 28, 27, 27, 26, 26, 25, 25, 24, 24, 24, 23, 23, + 23, 22, 22, 22, 22, 22, 21, 21, 21, 21, 28, 28, 29, 30, 31, 30, 30, 29, + 29, 28, 28, 27, 27, 26, 26, 25, 25, 24, 24, 24, 23, 23, 23, 22, 22, 22, + 22, 22, 21, 21, 21, 21, 28, 28, 29, 30, 31, 30, 30, 29, 29, 28, 28, 27, + 27, 26, 26, 25, 25, 24, 24, 24, 23, 23, 23, 22, 22, 22, 22, 22, 21, 21, + 21, 21, 28, 28, 29, 30, 31, 30, 30, 29, 29, 28, 28, 27, 27, 26, 26, 25, + 25, 24, 24, 24, 23, 23, 23, 22, 22, 22, 22, 22, 21, 21, 21, 21 }, + { /* Intra matrices */ + /* Size 4 */ + 155, 142, 90, 61, 142, 97, 72, 56, 90, 72, 54, 46, 61, 56, 46, 41, + /* Size 8 */ + 131, 172, 159, 123, 94, 74, 62, 53, 172, 151, 157, 135, 108, 86, 70, 60, + 159, 157, 115, 99, 86, 74, 64, 56, 123, 135, 99, 80, 70, 63, 56, 51, 94, + 108, 86, 70, 61, 55, 50, 47, 74, 86, 74, 63, 55, 50, 46, 44, 62, 70, 64, + 56, 50, 46, 44, 42, 53, 60, 56, 51, 47, 44, 42, 40, + /* Size 16 */ + 136, 157, 177, 171, 165, 146, 127, 112, 97, 87, 77, 70, 64, 59, 55, 55, + 157, 162, 167, 165, 163, 148, 133, 119, 104, 93, 83, 75, 68, 63, 58, 58, + 177, 167, 156, 159, 162, 151, 140, 126, 111, 100, 89, 81, 73, 67, 62, + 62, 171, 165, 159, 150, 140, 131, 121, 111, 100, 91, 83, 76, 69, 65, 60, + 60, 165, 163, 162, 140, 118, 110, 103, 96, 89, 83, 76, 71, 66, 62, 58, + 58, 146, 148, 151, 131, 110, 102, 93, 87, 81, 76, 71, 66, 62, 59, 55, + 55, 127, 133, 140, 121, 103, 93, 83, 78, 72, 68, 65, 61, 58, 56, 53, 53, + 112, 119, 126, 111, 96, 87, 78, 72, 67, 64, 61, 58, 55, 53, 51, 51, 97, + 104, 111, 100, 89, 81, 72, 67, 63, 60, 57, 54, 52, 50, 49, 49, 87, 93, + 100, 91, 83, 76, 68, 64, 60, 57, 54, 52, 50, 49, 47, 47, 77, 83, 89, 83, + 76, 71, 65, 61, 57, 54, 51, 50, 48, 47, 45, 45, 70, 75, 81, 76, 71, 66, + 61, 58, 54, 52, 50, 48, 46, 45, 44, 44, 64, 68, 73, 69, 66, 62, 58, 55, + 52, 50, 48, 46, 45, 44, 43, 43, 59, 63, 67, 65, 62, 59, 56, 53, 50, 49, + 47, 45, 44, 43, 42, 42, 55, 58, 62, 60, 58, 55, 53, 51, 49, 47, 45, 44, + 43, 42, 41, 41, 55, 58, 62, 60, 58, 55, 53, 51, 49, 47, 45, 44, 43, 42, + 41, 41, + /* Size 32 */ + 138, 148, 159, 170, 180, 177, 174, 171, 167, 158, 148, 139, 129, 122, + 114, 106, 99, 94, 88, 83, 78, 75, 71, 68, 65, 62, 60, 58, 56, 56, 56, + 56, 148, 155, 162, 168, 175, 173, 171, 169, 167, 158, 150, 141, 132, + 125, 117, 110, 102, 97, 92, 86, 81, 78, 74, 71, 67, 65, 62, 60, 58, 58, + 58, 58, 159, 162, 164, 167, 169, 169, 168, 167, 166, 159, 151, 143, 136, + 128, 121, 113, 106, 100, 95, 90, 84, 80, 77, 73, 69, 67, 64, 62, 59, 59, + 59, 59, 170, 168, 167, 165, 164, 164, 165, 165, 166, 159, 152, 146, 139, + 132, 124, 117, 110, 104, 98, 93, 87, 83, 79, 76, 72, 69, 66, 64, 61, 61, + 61, 61, 180, 175, 169, 164, 158, 160, 162, 163, 165, 159, 154, 148, 142, + 135, 128, 120, 113, 107, 102, 96, 90, 86, 82, 78, 74, 71, 68, 66, 63, + 63, 63, 63, 177, 173, 169, 164, 160, 159, 157, 155, 154, 149, 143, 138, + 133, 126, 120, 114, 108, 102, 97, 92, 87, 83, 80, 76, 72, 70, 67, 64, + 62, 62, 62, 62, 174, 171, 168, 165, 162, 157, 152, 147, 143, 138, 133, + 128, 123, 118, 113, 107, 102, 97, 93, 88, 84, 81, 77, 74, 70, 68, 66, + 63, 61, 61, 61, 61, 171, 169, 167, 165, 163, 155, 147, 139, 132, 127, + 123, 118, 114, 109, 105, 101, 96, 92, 89, 85, 81, 78, 75, 72, 69, 66, + 64, 62, 60, 60, 60, 60, 167, 167, 166, 166, 165, 154, 143, 132, 120, + 116, 112, 108, 104, 101, 98, 94, 91, 87, 84, 81, 78, 75, 72, 70, 67, 65, + 63, 61, 59, 59, 59, 59, 158, 158, 159, 159, 159, 149, 138, 127, 116, + 112, 108, 104, 99, 96, 93, 90, 86, 83, 81, 78, 75, 72, 70, 67, 65, 63, + 61, 59, 57, 57, 57, 57, 148, 150, 151, 152, 154, 143, 133, 123, 112, + 108, 103, 99, 94, 91, 88, 85, 82, 79, 77, 74, 72, 70, 67, 65, 63, 61, + 60, 58, 56, 56, 56, 56, 139, 141, 143, 146, 148, 138, 128, 118, 108, + 104, 99, 94, 89, 86, 83, 81, 78, 76, 73, 71, 69, 67, 65, 63, 61, 60, 58, + 57, 55, 55, 55, 55, 129, 132, 136, 139, 142, 133, 123, 114, 104, 99, 94, + 89, 84, 81, 79, 76, 74, 72, 70, 68, 66, 64, 62, 61, 59, 58, 57, 55, 54, + 54, 54, 54, 122, 125, 128, 132, 135, 126, 118, 109, 101, 96, 91, 86, 81, + 79, 76, 74, 71, 69, 67, 66, 64, 62, 61, 59, 58, 56, 55, 54, 53, 53, 53, + 53, 114, 117, 121, 124, 128, 120, 113, 105, 98, 93, 88, 83, 79, 76, 74, + 71, 69, 67, 65, 63, 62, 60, 59, 57, 56, 55, 54, 53, 52, 52, 52, 52, 106, + 110, 113, 117, 120, 114, 107, 101, 94, 90, 85, 81, 76, 74, 71, 69, 66, + 64, 63, 61, 60, 58, 57, 56, 55, 54, 53, 52, 51, 51, 51, 51, 99, 102, + 106, 110, 113, 108, 102, 96, 91, 86, 82, 78, 74, 71, 69, 66, 64, 62, 61, + 59, 57, 56, 55, 54, 53, 52, 51, 50, 49, 49, 49, 49, 94, 97, 100, 104, + 107, 102, 97, 92, 87, 83, 79, 76, 72, 69, 67, 64, 62, 61, 59, 58, 56, + 55, 54, 53, 52, 51, 50, 49, 49, 49, 49, 49, 88, 92, 95, 98, 102, 97, 93, + 89, 84, 81, 77, 73, 70, 67, 65, 63, 61, 59, 58, 56, 55, 54, 53, 52, 51, + 50, 49, 49, 48, 48, 48, 48, 83, 86, 90, 93, 96, 92, 88, 85, 81, 78, 74, + 71, 68, 66, 63, 61, 59, 58, 56, 55, 54, 53, 52, 51, 50, 49, 48, 48, 47, + 47, 47, 47, 78, 81, 84, 87, 90, 87, 84, 81, 78, 75, 72, 69, 66, 64, 62, + 60, 57, 56, 55, 54, 52, 51, 50, 50, 49, 48, 47, 47, 46, 46, 46, 46, 75, + 78, 80, 83, 86, 83, 81, 78, 75, 72, 70, 67, 64, 62, 60, 58, 56, 55, 54, + 53, 51, 51, 50, 49, 48, 47, 47, 46, 46, 46, 46, 46, 71, 74, 77, 79, 82, + 80, 77, 75, 72, 70, 67, 65, 62, 61, 59, 57, 55, 54, 53, 52, 50, 50, 49, + 48, 47, 47, 46, 45, 45, 45, 45, 45, 68, 71, 73, 76, 78, 76, 74, 72, 70, + 67, 65, 63, 61, 59, 57, 56, 54, 53, 52, 51, 50, 49, 48, 47, 47, 46, 45, + 45, 44, 44, 44, 44, 65, 67, 69, 72, 74, 72, 70, 69, 67, 65, 63, 61, 59, + 58, 56, 55, 53, 52, 51, 50, 49, 48, 47, 47, 46, 45, 45, 44, 44, 44, 44, + 44, 62, 65, 67, 69, 71, 70, 68, 66, 65, 63, 61, 60, 58, 56, 55, 54, 52, + 51, 50, 49, 48, 47, 47, 46, 45, 45, 44, 44, 43, 43, 43, 43, 60, 62, 64, + 66, 68, 67, 66, 64, 63, 61, 60, 58, 57, 55, 54, 53, 51, 50, 49, 48, 47, + 47, 46, 45, 45, 44, 44, 43, 43, 43, 43, 43, 58, 60, 62, 64, 66, 64, 63, + 62, 61, 59, 58, 57, 55, 54, 53, 52, 50, 49, 49, 48, 47, 46, 45, 45, 44, + 44, 43, 43, 42, 42, 42, 42, 56, 58, 59, 61, 63, 62, 61, 60, 59, 57, 56, + 55, 54, 53, 52, 51, 49, 49, 48, 47, 46, 46, 45, 44, 44, 43, 43, 42, 42, + 42, 42, 42, 56, 58, 59, 61, 63, 62, 61, 60, 59, 57, 56, 55, 54, 53, 52, + 51, 49, 49, 48, 47, 46, 46, 45, 44, 44, 43, 43, 42, 42, 42, 42, 42, 56, + 58, 59, 61, 63, 62, 61, 60, 59, 57, 56, 55, 54, 53, 52, 51, 49, 49, 48, + 47, 46, 46, 45, 44, 44, 43, 43, 42, 42, 42, 42, 42, 56, 58, 59, 61, 63, + 62, 61, 60, 59, 57, 56, 55, 54, 53, 52, 51, 49, 49, 48, 47, 46, 46, 45, + 44, 44, 43, 43, 42, 42, 42, 42, 42 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 42, 40, 32, 42, 36, 34, 31, 40, 34, 28, 25, 32, 31, 25, 22, + /* Size 8 */ + 64, 78, 46, 43, 41, 37, 34, 31, 78, 56, 45, 49, 48, 44, 40, 36, 46, 45, + 39, 40, 41, 39, 36, 33, 43, 49, 40, 36, 35, 34, 32, 30, 41, 48, 41, 35, + 32, 30, 29, 28, 37, 44, 39, 34, 30, 28, 27, 26, 34, 40, 36, 32, 29, 27, + 25, 24, 31, 36, 33, 30, 28, 26, 24, 23, + /* Size 16 */ + 64, 71, 78, 62, 46, 45, 43, 42, 41, 39, 37, 36, 34, 32, 31, 31, 71, 69, + 67, 56, 45, 46, 46, 45, 44, 43, 41, 39, 37, 35, 33, 33, 78, 67, 56, 50, + 45, 47, 49, 48, 48, 46, 44, 42, 40, 38, 36, 36, 62, 56, 50, 46, 42, 43, + 45, 44, 44, 43, 41, 40, 38, 36, 35, 35, 46, 45, 45, 42, 39, 40, 40, 40, + 41, 40, 39, 38, 36, 35, 33, 33, 45, 46, 47, 43, 40, 39, 38, 38, 38, 37, + 36, 35, 34, 33, 32, 32, 43, 46, 49, 45, 40, 38, 36, 36, 35, 34, 34, 33, + 32, 31, 30, 30, 42, 45, 48, 44, 40, 38, 36, 35, 34, 33, 32, 31, 31, 30, + 29, 29, 41, 44, 48, 44, 41, 38, 35, 34, 32, 31, 30, 30, 29, 28, 28, 28, + 39, 43, 46, 43, 40, 37, 34, 33, 31, 30, 29, 29, 28, 27, 27, 27, 37, 41, + 44, 41, 39, 36, 34, 32, 30, 29, 28, 27, 27, 26, 26, 26, 36, 39, 42, 40, + 38, 35, 33, 31, 30, 29, 27, 27, 26, 25, 25, 25, 34, 37, 40, 38, 36, 34, + 32, 31, 29, 28, 27, 26, 25, 25, 24, 24, 32, 35, 38, 36, 35, 33, 31, 30, + 28, 27, 26, 25, 25, 24, 23, 23, 31, 33, 36, 35, 33, 32, 30, 29, 28, 27, + 26, 25, 24, 23, 23, 23, 31, 33, 36, 35, 33, 32, 30, 29, 28, 27, 26, 25, + 24, 23, 23, 23, + /* Size 32 */ + 64, 67, 71, 74, 78, 70, 62, 54, 46, 45, 45, 44, 43, 43, 42, 42, 41, 40, + 39, 38, 37, 36, 36, 35, 34, 33, 32, 31, 31, 31, 31, 31, 67, 69, 70, 71, + 72, 66, 59, 52, 45, 45, 45, 45, 45, 44, 44, 43, 43, 42, 41, 40, 39, 38, + 37, 36, 35, 34, 34, 33, 32, 32, 32, 32, 71, 70, 69, 68, 67, 61, 56, 51, + 45, 46, 46, 46, 46, 46, 45, 45, 44, 43, 43, 42, 41, 40, 39, 38, 37, 36, + 35, 34, 33, 33, 33, 33, 74, 71, 68, 65, 61, 57, 53, 49, 45, 46, 46, 47, + 48, 47, 47, 46, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 34, + 34, 34, 78, 72, 67, 61, 56, 53, 50, 48, 45, 46, 47, 48, 49, 49, 48, 48, + 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 36, 36, 36, 70, 66, + 61, 57, 53, 51, 48, 46, 44, 44, 45, 46, 47, 47, 46, 46, 46, 45, 44, 44, + 43, 42, 41, 40, 39, 38, 37, 36, 35, 35, 35, 35, 62, 59, 56, 53, 50, 48, + 46, 44, 42, 43, 43, 44, 45, 45, 44, 44, 44, 43, 43, 42, 41, 41, 40, 39, + 38, 37, 36, 35, 35, 35, 35, 35, 54, 52, 51, 49, 48, 46, 44, 42, 41, 41, + 42, 42, 43, 43, 42, 42, 42, 42, 41, 41, 40, 39, 39, 38, 37, 36, 36, 35, + 34, 34, 34, 34, 46, 45, 45, 45, 45, 44, 42, 41, 39, 40, 40, 40, 40, 40, + 40, 41, 41, 40, 40, 39, 39, 38, 38, 37, 36, 35, 35, 34, 33, 33, 33, 33, + 45, 45, 46, 46, 46, 44, 43, 41, 40, 40, 40, 39, 39, 39, 39, 39, 39, 39, + 38, 38, 38, 37, 36, 36, 35, 35, 34, 33, 33, 33, 33, 33, 45, 45, 46, 46, + 47, 45, 43, 42, 40, 40, 39, 39, 38, 38, 38, 38, 38, 37, 37, 37, 36, 36, + 35, 35, 34, 34, 33, 32, 32, 32, 32, 32, 44, 45, 46, 47, 48, 46, 44, 42, + 40, 39, 39, 38, 37, 37, 37, 37, 36, 36, 36, 35, 35, 35, 34, 34, 33, 33, + 32, 32, 31, 31, 31, 31, 43, 45, 46, 48, 49, 47, 45, 43, 40, 39, 38, 37, + 36, 36, 36, 35, 35, 35, 34, 34, 34, 33, 33, 33, 32, 32, 31, 31, 30, 30, + 30, 30, 43, 44, 46, 47, 49, 47, 45, 43, 40, 39, 38, 37, 36, 36, 35, 35, + 34, 34, 34, 33, 33, 33, 32, 32, 31, 31, 31, 30, 30, 30, 30, 30, 42, 44, + 45, 47, 48, 46, 44, 42, 40, 39, 38, 37, 36, 35, 35, 34, 34, 33, 33, 32, + 32, 32, 31, 31, 31, 30, 30, 29, 29, 29, 29, 29, 42, 43, 45, 46, 48, 46, + 44, 42, 41, 39, 38, 37, 35, 35, 34, 33, 33, 32, 32, 32, 31, 31, 31, 30, + 30, 29, 29, 29, 28, 28, 28, 28, 41, 43, 44, 46, 48, 46, 44, 42, 41, 39, + 38, 36, 35, 34, 34, 33, 32, 32, 31, 31, 30, 30, 30, 29, 29, 29, 28, 28, + 28, 28, 28, 28, 40, 42, 43, 45, 47, 45, 43, 42, 40, 39, 37, 36, 35, 34, + 33, 32, 32, 31, 31, 30, 30, 29, 29, 29, 28, 28, 28, 28, 27, 27, 27, 27, + 39, 41, 43, 44, 46, 44, 43, 41, 40, 38, 37, 36, 34, 34, 33, 32, 31, 31, + 30, 30, 29, 29, 29, 28, 28, 28, 27, 27, 27, 27, 27, 27, 38, 40, 42, 43, + 45, 44, 42, 41, 39, 38, 37, 35, 34, 33, 32, 32, 31, 30, 30, 29, 29, 28, + 28, 28, 27, 27, 27, 26, 26, 26, 26, 26, 37, 39, 41, 42, 44, 43, 41, 40, + 39, 38, 36, 35, 34, 33, 32, 31, 30, 30, 29, 29, 28, 28, 27, 27, 27, 26, + 26, 26, 26, 26, 26, 26, 36, 38, 40, 41, 43, 42, 41, 39, 38, 37, 36, 35, + 33, 33, 32, 31, 30, 29, 29, 28, 28, 27, 27, 27, 26, 26, 26, 25, 25, 25, + 25, 25, 36, 37, 39, 40, 42, 41, 40, 39, 38, 36, 35, 34, 33, 32, 31, 31, + 30, 29, 29, 28, 27, 27, 27, 26, 26, 26, 25, 25, 25, 25, 25, 25, 35, 36, + 38, 39, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 29, 28, 28, + 27, 27, 26, 26, 26, 25, 25, 25, 24, 24, 24, 24, 34, 35, 37, 38, 40, 39, + 38, 37, 36, 35, 34, 33, 32, 31, 31, 30, 29, 28, 28, 27, 27, 26, 26, 26, + 25, 25, 25, 24, 24, 24, 24, 24, 33, 34, 36, 37, 39, 38, 37, 36, 35, 35, + 34, 33, 32, 31, 30, 29, 29, 28, 28, 27, 26, 26, 26, 25, 25, 25, 24, 24, + 24, 24, 24, 24, 32, 34, 35, 36, 38, 37, 36, 36, 35, 34, 33, 32, 31, 31, + 30, 29, 28, 28, 27, 27, 26, 26, 25, 25, 25, 24, 24, 24, 23, 23, 23, 23, + 31, 33, 34, 35, 37, 36, 35, 35, 34, 33, 32, 32, 31, 30, 29, 29, 28, 28, + 27, 26, 26, 25, 25, 25, 24, 24, 24, 23, 23, 23, 23, 23, 31, 32, 33, 34, + 36, 35, 35, 34, 33, 33, 32, 31, 30, 30, 29, 28, 28, 27, 27, 26, 26, 25, + 25, 24, 24, 24, 23, 23, 23, 23, 23, 23, 31, 32, 33, 34, 36, 35, 35, 34, + 33, 33, 32, 31, 30, 30, 29, 28, 28, 27, 27, 26, 26, 25, 25, 24, 24, 24, + 23, 23, 23, 23, 23, 23, 31, 32, 33, 34, 36, 35, 35, 34, 33, 33, 32, 31, + 30, 30, 29, 28, 28, 27, 27, 26, 26, 25, 25, 24, 24, 24, 23, 23, 23, 23, + 23, 23, 31, 32, 33, 34, 36, 35, 35, 34, 33, 33, 32, 31, 30, 30, 29, 28, + 28, 27, 27, 26, 26, 25, 25, 24, 24, 24, 23, 23, 23, 23, 23, 23 }, + { /* Intra matrices */ + /* Size 4 */ + 133, 87, 80, 65, 87, 73, 69, 61, 80, 69, 56, 50, 65, 61, 50, 43, + /* Size 8 */ + 124, 152, 87, 83, 78, 70, 63, 57, 152, 107, 86, 94, 91, 84, 75, 67, 87, + 86, 74, 77, 77, 73, 68, 62, 83, 94, 77, 69, 66, 63, 60, 56, 78, 91, 77, + 66, 60, 56, 54, 51, 70, 84, 73, 63, 56, 52, 49, 47, 63, 75, 68, 60, 54, + 49, 46, 44, 57, 67, 62, 56, 51, 47, 44, 41, + /* Size 16 */ + 127, 141, 155, 122, 89, 87, 84, 82, 80, 76, 72, 68, 65, 61, 58, 58, 141, + 137, 133, 111, 88, 89, 90, 88, 87, 83, 79, 75, 71, 67, 63, 63, 155, 133, + 110, 99, 88, 92, 96, 95, 93, 90, 86, 81, 77, 73, 69, 69, 122, 111, 99, + 90, 82, 85, 87, 87, 86, 83, 80, 77, 73, 70, 66, 66, 89, 88, 88, 82, 76, + 77, 78, 79, 79, 77, 75, 72, 70, 67, 64, 64, 87, 89, 92, 85, 77, 76, 74, + 74, 73, 71, 70, 68, 66, 63, 61, 61, 84, 90, 96, 87, 78, 74, 70, 69, 67, + 66, 65, 63, 62, 60, 58, 58, 82, 88, 95, 87, 79, 74, 69, 67, 64, 63, 61, + 60, 58, 57, 55, 55, 80, 87, 93, 86, 79, 73, 67, 64, 61, 59, 58, 56, 55, + 54, 52, 52, 76, 83, 90, 83, 77, 71, 66, 63, 59, 57, 55, 54, 53, 51, 50, + 50, 72, 79, 86, 80, 75, 70, 65, 61, 58, 55, 53, 52, 50, 49, 48, 48, 68, + 75, 81, 77, 72, 68, 63, 60, 56, 54, 52, 50, 49, 47, 46, 46, 65, 71, 77, + 73, 70, 66, 62, 58, 55, 53, 50, 49, 47, 46, 45, 45, 61, 67, 73, 70, 67, + 63, 60, 57, 54, 51, 49, 47, 46, 45, 43, 43, 58, 63, 69, 66, 64, 61, 58, + 55, 52, 50, 48, 46, 45, 43, 42, 42, 58, 63, 69, 66, 64, 61, 58, 55, 52, + 50, 48, 46, 45, 43, 42, 42, + /* Size 32 */ + 129, 136, 143, 150, 157, 141, 124, 107, 90, 89, 88, 87, 86, 84, 83, 82, + 81, 79, 77, 75, 73, 71, 69, 67, 65, 64, 62, 60, 59, 59, 59, 59, 136, + 138, 141, 143, 146, 132, 118, 104, 90, 90, 89, 89, 88, 87, 86, 85, 84, + 82, 80, 78, 76, 74, 72, 71, 69, 67, 65, 63, 62, 62, 62, 62, 143, 141, + 139, 137, 134, 123, 112, 101, 89, 90, 90, 91, 91, 90, 89, 89, 88, 86, + 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64, 64, 64, 64, 150, 143, 137, + 130, 123, 114, 106, 98, 89, 90, 92, 93, 94, 93, 93, 92, 91, 89, 87, 85, + 83, 81, 79, 77, 75, 73, 71, 69, 67, 67, 67, 67, 157, 146, 134, 123, 111, + 106, 100, 94, 89, 91, 93, 95, 97, 97, 96, 95, 95, 93, 91, 89, 87, 85, + 82, 80, 78, 76, 74, 72, 70, 70, 70, 70, 141, 132, 123, 114, 106, 101, + 96, 91, 86, 88, 89, 91, 93, 92, 92, 91, 91, 89, 87, 86, 84, 82, 80, 78, + 76, 74, 72, 70, 68, 68, 68, 68, 124, 118, 112, 106, 100, 96, 91, 87, 83, + 84, 86, 87, 88, 88, 88, 87, 87, 86, 84, 83, 81, 80, 78, 76, 74, 72, 71, + 69, 67, 67, 67, 67, 107, 104, 101, 98, 94, 91, 87, 84, 80, 81, 82, 83, + 84, 84, 84, 83, 83, 82, 81, 80, 79, 77, 76, 74, 72, 71, 69, 68, 66, 66, + 66, 66, 90, 90, 89, 89, 89, 86, 83, 80, 77, 78, 78, 79, 79, 79, 80, 80, + 80, 79, 78, 77, 76, 75, 73, 72, 71, 69, 68, 66, 65, 65, 65, 65, 89, 90, + 90, 90, 91, 88, 84, 81, 78, 78, 77, 77, 77, 77, 77, 77, 77, 76, 75, 74, + 73, 72, 71, 70, 69, 67, 66, 64, 63, 63, 63, 63, 88, 89, 90, 92, 93, 89, + 86, 82, 78, 77, 77, 76, 75, 75, 75, 74, 74, 73, 72, 72, 71, 70, 69, 68, + 66, 65, 64, 63, 62, 62, 62, 62, 87, 89, 91, 93, 95, 91, 87, 83, 79, 77, + 76, 75, 73, 73, 72, 72, 71, 70, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, + 60, 60, 60, 60, 86, 88, 91, 94, 97, 93, 88, 84, 79, 77, 75, 73, 71, 70, + 70, 69, 68, 68, 67, 66, 66, 65, 64, 63, 62, 61, 60, 59, 59, 59, 59, 59, + 84, 87, 90, 93, 97, 92, 88, 84, 79, 77, 75, 73, 70, 69, 68, 68, 67, 66, + 65, 65, 64, 63, 62, 61, 61, 60, 59, 58, 57, 57, 57, 57, 83, 86, 89, 93, + 96, 92, 88, 84, 80, 77, 75, 72, 70, 68, 67, 66, 65, 64, 64, 63, 62, 61, + 60, 60, 59, 58, 57, 57, 56, 56, 56, 56, 82, 85, 89, 92, 95, 91, 87, 83, + 80, 77, 74, 72, 69, 68, 66, 65, 63, 63, 62, 61, 60, 59, 59, 58, 57, 57, + 56, 55, 54, 54, 54, 54, 81, 84, 88, 91, 95, 91, 87, 83, 80, 77, 74, 71, + 68, 67, 65, 63, 62, 61, 60, 59, 58, 58, 57, 56, 56, 55, 54, 54, 53, 53, + 53, 53, 79, 82, 86, 89, 93, 89, 86, 82, 79, 76, 73, 70, 68, 66, 64, 63, + 61, 60, 59, 58, 57, 57, 56, 55, 54, 54, 53, 52, 52, 52, 52, 52, 77, 80, + 84, 87, 91, 87, 84, 81, 78, 75, 72, 70, 67, 65, 64, 62, 60, 59, 58, 57, + 56, 55, 55, 54, 53, 53, 52, 51, 51, 51, 51, 51, 75, 78, 82, 85, 89, 86, + 83, 80, 77, 74, 72, 69, 66, 65, 63, 61, 59, 58, 57, 56, 55, 54, 53, 53, + 52, 51, 51, 50, 50, 50, 50, 50, 73, 76, 80, 83, 87, 84, 81, 79, 76, 73, + 71, 68, 66, 64, 62, 60, 58, 57, 56, 55, 54, 53, 52, 52, 51, 50, 50, 49, + 48, 48, 48, 48, 71, 74, 78, 81, 85, 82, 80, 77, 75, 72, 70, 67, 65, 63, + 61, 59, 58, 57, 55, 54, 53, 52, 52, 51, 50, 49, 49, 48, 48, 48, 48, 48, + 69, 72, 76, 79, 82, 80, 78, 76, 73, 71, 69, 66, 64, 62, 60, 59, 57, 56, + 55, 53, 52, 52, 51, 50, 49, 49, 48, 47, 47, 47, 47, 47, 67, 71, 74, 77, + 80, 78, 76, 74, 72, 70, 68, 65, 63, 61, 60, 58, 56, 55, 54, 53, 52, 51, + 50, 49, 48, 48, 47, 47, 46, 46, 46, 46, 65, 69, 72, 75, 78, 76, 74, 72, + 71, 69, 66, 64, 62, 61, 59, 57, 56, 54, 53, 52, 51, 50, 49, 48, 48, 47, + 46, 46, 45, 45, 45, 45, 64, 67, 70, 73, 76, 74, 72, 71, 69, 67, 65, 63, + 61, 60, 58, 57, 55, 54, 53, 51, 50, 49, 49, 48, 47, 46, 46, 45, 45, 45, + 45, 45, 62, 65, 68, 71, 74, 72, 71, 69, 68, 66, 64, 62, 60, 59, 57, 56, + 54, 53, 52, 51, 50, 49, 48, 47, 46, 46, 45, 45, 44, 44, 44, 44, 60, 63, + 66, 69, 72, 70, 69, 68, 66, 64, 63, 61, 59, 58, 57, 55, 54, 52, 51, 50, + 49, 48, 47, 47, 46, 45, 45, 44, 43, 43, 43, 43, 59, 62, 64, 67, 70, 68, + 67, 66, 65, 63, 62, 60, 59, 57, 56, 54, 53, 52, 51, 50, 48, 48, 47, 46, + 45, 45, 44, 43, 43, 43, 43, 43, 59, 62, 64, 67, 70, 68, 67, 66, 65, 63, + 62, 60, 59, 57, 56, 54, 53, 52, 51, 50, 48, 48, 47, 46, 45, 45, 44, 43, + 43, 43, 43, 43, 59, 62, 64, 67, 70, 68, 67, 66, 65, 63, 62, 60, 59, 57, + 56, 54, 53, 52, 51, 50, 48, 48, 47, 46, 45, 45, 44, 43, 43, 43, 43, 43, + 59, 62, 64, 67, 70, 68, 67, 66, 65, 63, 62, 60, 59, 57, 56, 54, 53, 52, + 51, 50, 48, 48, 47, 46, 45, 45, 44, 43, 43, 43, 43, 43 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 59, 40, 29, 59, 42, 33, 27, 40, 33, 26, 23, 29, 27, 23, 21, + /* Size 8 */ + 64, 82, 76, 60, 48, 39, 34, 30, 82, 73, 75, 66, 54, 44, 38, 33, 76, 75, + 57, 50, 44, 39, 35, 31, 60, 66, 50, 42, 37, 34, 31, 29, 48, 54, 44, 37, + 33, 31, 29, 27, 39, 44, 39, 34, 31, 29, 27, 26, 34, 38, 35, 31, 29, 27, + 26, 25, 30, 33, 31, 29, 27, 26, 25, 24, + /* Size 16 */ + 64, 73, 82, 79, 76, 68, 60, 54, 48, 44, 39, 36, 34, 32, 30, 30, 73, 75, + 77, 76, 76, 69, 63, 57, 51, 46, 42, 39, 36, 34, 31, 31, 82, 77, 73, 74, + 75, 71, 66, 60, 54, 49, 44, 41, 38, 35, 33, 33, 79, 76, 74, 70, 66, 62, + 58, 54, 49, 45, 42, 39, 36, 34, 32, 32, 76, 76, 75, 66, 57, 53, 50, 47, + 44, 42, 39, 37, 35, 33, 31, 31, 68, 69, 71, 62, 53, 50, 46, 43, 41, 39, + 37, 35, 33, 32, 30, 30, 60, 63, 66, 58, 50, 46, 42, 40, 37, 36, 34, 33, + 31, 30, 29, 29, 54, 57, 60, 54, 47, 43, 40, 37, 35, 34, 32, 31, 30, 29, + 28, 28, 48, 51, 54, 49, 44, 41, 37, 35, 33, 32, 31, 30, 29, 28, 27, 27, + 44, 46, 49, 45, 42, 39, 36, 34, 32, 31, 30, 29, 28, 27, 27, 27, 39, 42, + 44, 42, 39, 37, 34, 32, 31, 30, 29, 28, 27, 27, 26, 26, 36, 39, 41, 39, + 37, 35, 33, 31, 30, 29, 28, 27, 26, 26, 25, 25, 34, 36, 38, 36, 35, 33, + 31, 30, 29, 28, 27, 26, 26, 25, 25, 25, 32, 34, 35, 34, 33, 32, 30, 29, + 28, 27, 27, 26, 25, 25, 25, 25, 30, 31, 33, 32, 31, 30, 29, 28, 27, 27, + 26, 25, 25, 25, 24, 24, 30, 31, 33, 32, 31, 30, 29, 28, 27, 27, 26, 25, + 25, 25, 24, 24, + /* Size 32 */ + 64, 68, 73, 77, 82, 80, 79, 78, 76, 72, 68, 64, 60, 57, 54, 51, 48, 46, + 44, 41, 39, 38, 36, 35, 34, 33, 32, 31, 30, 30, 30, 30, 68, 71, 74, 77, + 79, 79, 78, 77, 76, 72, 69, 65, 62, 59, 56, 52, 49, 47, 45, 43, 41, 39, + 38, 36, 35, 34, 33, 32, 31, 31, 31, 31, 73, 74, 75, 76, 77, 77, 76, 76, + 76, 73, 69, 66, 63, 60, 57, 54, 51, 49, 46, 44, 42, 40, 39, 37, 36, 35, + 34, 32, 31, 31, 31, 31, 77, 77, 76, 75, 75, 75, 75, 75, 76, 73, 70, 67, + 64, 61, 58, 55, 52, 50, 48, 45, 43, 41, 40, 38, 37, 35, 34, 33, 32, 32, + 32, 32, 82, 79, 77, 75, 73, 73, 74, 75, 75, 73, 71, 68, 66, 63, 60, 57, + 54, 51, 49, 47, 44, 43, 41, 39, 38, 36, 35, 34, 33, 33, 33, 33, 80, 79, + 77, 75, 73, 73, 72, 71, 71, 68, 66, 64, 62, 59, 57, 54, 51, 49, 47, 45, + 43, 41, 40, 38, 37, 36, 35, 34, 32, 32, 32, 32, 79, 78, 76, 75, 74, 72, + 70, 68, 66, 64, 62, 60, 58, 56, 54, 51, 49, 47, 45, 44, 42, 40, 39, 37, + 36, 35, 34, 33, 32, 32, 32, 32, 78, 77, 76, 75, 75, 71, 68, 65, 61, 60, + 58, 56, 54, 52, 50, 49, 47, 45, 44, 42, 40, 39, 38, 37, 35, 34, 33, 33, + 32, 32, 32, 32, 76, 76, 76, 76, 75, 71, 66, 61, 57, 55, 53, 52, 50, 49, + 47, 46, 44, 43, 42, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 31, 31, 31, + 72, 72, 73, 73, 73, 68, 64, 60, 55, 53, 52, 50, 48, 47, 45, 44, 43, 41, + 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 31, 31, 31, 31, 68, 69, 69, 70, + 71, 66, 62, 58, 53, 52, 50, 48, 46, 45, 43, 42, 41, 40, 39, 38, 37, 36, + 35, 34, 33, 32, 32, 31, 30, 30, 30, 30, 64, 65, 66, 67, 68, 64, 60, 56, + 52, 50, 48, 46, 44, 43, 41, 40, 39, 38, 37, 36, 35, 35, 34, 33, 32, 32, + 31, 30, 30, 30, 30, 30, 60, 62, 63, 64, 66, 62, 58, 54, 50, 48, 46, 44, + 42, 41, 40, 38, 37, 37, 36, 35, 34, 33, 33, 32, 31, 31, 30, 30, 29, 29, + 29, 29, 57, 59, 60, 61, 63, 59, 56, 52, 49, 47, 45, 43, 41, 40, 38, 37, + 36, 36, 35, 34, 33, 33, 32, 31, 31, 30, 30, 29, 29, 29, 29, 29, 54, 56, + 57, 58, 60, 57, 54, 50, 47, 45, 43, 41, 40, 38, 37, 36, 35, 35, 34, 33, + 32, 32, 31, 31, 30, 30, 29, 29, 28, 28, 28, 28, 51, 52, 54, 55, 57, 54, + 51, 49, 46, 44, 42, 40, 38, 37, 36, 35, 34, 34, 33, 32, 32, 31, 31, 30, + 30, 29, 29, 28, 28, 28, 28, 28, 48, 49, 51, 52, 54, 51, 49, 47, 44, 43, + 41, 39, 37, 36, 35, 34, 33, 33, 32, 31, 31, 30, 30, 29, 29, 28, 28, 28, + 27, 27, 27, 27, 46, 47, 49, 50, 51, 49, 47, 45, 43, 41, 40, 38, 37, 36, + 35, 34, 33, 32, 31, 31, 30, 30, 29, 29, 28, 28, 28, 27, 27, 27, 27, 27, + 44, 45, 46, 48, 49, 47, 45, 44, 42, 40, 39, 37, 36, 35, 34, 33, 32, 31, + 31, 30, 30, 29, 29, 28, 28, 28, 27, 27, 27, 27, 27, 27, 41, 43, 44, 45, + 47, 45, 44, 42, 40, 39, 38, 36, 35, 34, 33, 32, 31, 31, 30, 30, 29, 29, + 28, 28, 28, 27, 27, 27, 26, 26, 26, 26, 39, 41, 42, 43, 44, 43, 42, 40, + 39, 38, 37, 35, 34, 33, 32, 32, 31, 30, 30, 29, 29, 28, 28, 27, 27, 27, + 27, 26, 26, 26, 26, 26, 38, 39, 40, 41, 43, 41, 40, 39, 38, 37, 36, 35, + 33, 33, 32, 31, 30, 30, 29, 29, 28, 28, 27, 27, 27, 27, 26, 26, 26, 26, + 26, 26, 36, 38, 39, 40, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 31, + 30, 29, 29, 28, 28, 27, 27, 27, 26, 26, 26, 26, 25, 25, 25, 25, 35, 36, + 37, 38, 39, 38, 37, 37, 36, 35, 34, 33, 32, 31, 31, 30, 29, 29, 28, 28, + 27, 27, 27, 26, 26, 26, 26, 25, 25, 25, 25, 25, 34, 35, 36, 37, 38, 37, + 36, 35, 35, 34, 33, 32, 31, 31, 30, 30, 29, 28, 28, 28, 27, 27, 26, 26, + 26, 26, 25, 25, 25, 25, 25, 25, 33, 34, 35, 35, 36, 36, 35, 34, 34, 33, + 32, 32, 31, 30, 30, 29, 28, 28, 28, 27, 27, 27, 26, 26, 26, 25, 25, 25, + 25, 25, 25, 25, 32, 33, 34, 34, 35, 35, 34, 33, 33, 32, 32, 31, 30, 30, + 29, 29, 28, 28, 27, 27, 27, 26, 26, 26, 25, 25, 25, 25, 25, 25, 25, 25, + 31, 32, 32, 33, 34, 34, 33, 33, 32, 31, 31, 30, 30, 29, 29, 28, 28, 27, + 27, 27, 26, 26, 26, 25, 25, 25, 25, 25, 24, 24, 24, 24, 30, 31, 31, 32, + 33, 32, 32, 32, 31, 31, 30, 30, 29, 29, 28, 28, 27, 27, 27, 26, 26, 26, + 25, 25, 25, 25, 25, 24, 24, 24, 24, 24, 30, 31, 31, 32, 33, 32, 32, 32, + 31, 31, 30, 30, 29, 29, 28, 28, 27, 27, 27, 26, 26, 26, 25, 25, 25, 25, + 25, 24, 24, 24, 24, 24, 30, 31, 31, 32, 33, 32, 32, 32, 31, 31, 30, 30, + 29, 29, 28, 28, 27, 27, 27, 26, 26, 26, 25, 25, 25, 25, 25, 24, 24, 24, + 24, 24, 30, 31, 31, 32, 33, 32, 32, 32, 31, 31, 30, 30, 29, 29, 28, 28, + 27, 27, 27, 26, 26, 26, 25, 25, 25, 25, 25, 24, 24, 24, 24, 24 }, + { /* Intra matrices */ + /* Size 4 */ + 141, 130, 85, 60, 130, 91, 70, 56, 85, 70, 55, 48, 60, 56, 48, 43, + /* Size 8 */ + 120, 155, 144, 113, 88, 72, 61, 53, 155, 137, 143, 124, 100, 82, 68, 59, + 144, 143, 106, 93, 82, 71, 63, 56, 113, 124, 93, 77, 68, 62, 56, 52, 88, + 100, 82, 68, 60, 55, 51, 48, 72, 82, 71, 62, 55, 51, 48, 46, 61, 68, 63, + 56, 51, 48, 45, 44, 53, 59, 56, 52, 48, 46, 44, 42, + /* Size 16 */ + 124, 142, 160, 154, 149, 133, 117, 104, 91, 82, 74, 68, 63, 59, 55, 55, + 142, 146, 150, 149, 148, 135, 122, 110, 97, 88, 79, 73, 66, 62, 58, 58, + 160, 150, 141, 144, 147, 137, 128, 115, 103, 94, 84, 77, 70, 66, 61, 61, + 154, 149, 144, 136, 128, 120, 112, 103, 94, 86, 79, 73, 67, 63, 59, 59, + 149, 148, 147, 128, 109, 103, 96, 90, 84, 79, 73, 69, 64, 61, 58, 58, + 133, 135, 137, 120, 103, 95, 87, 82, 77, 73, 68, 65, 61, 58, 56, 56, + 117, 122, 128, 112, 96, 87, 79, 74, 70, 67, 63, 61, 58, 56, 53, 53, 104, + 110, 115, 103, 90, 82, 74, 70, 66, 63, 60, 58, 55, 53, 52, 52, 91, 97, + 103, 94, 84, 77, 70, 66, 62, 59, 57, 55, 53, 51, 50, 50, 82, 88, 94, 86, + 79, 73, 67, 63, 59, 57, 54, 53, 51, 50, 48, 48, 74, 79, 84, 79, 73, 68, + 63, 60, 57, 54, 52, 51, 49, 48, 47, 47, 68, 73, 77, 73, 69, 65, 61, 58, + 55, 53, 51, 49, 48, 47, 46, 46, 63, 66, 70, 67, 64, 61, 58, 55, 53, 51, + 49, 48, 47, 46, 45, 45, 59, 62, 66, 63, 61, 58, 56, 53, 51, 50, 48, 47, + 46, 45, 44, 44, 55, 58, 61, 59, 58, 56, 53, 52, 50, 48, 47, 46, 45, 44, + 44, 44, 55, 58, 61, 59, 58, 56, 53, 52, 50, 48, 47, 46, 45, 44, 44, 44, + /* Size 32 */ + 126, 135, 144, 153, 162, 159, 157, 154, 151, 143, 135, 127, 118, 112, + 105, 99, 92, 88, 84, 79, 75, 72, 69, 66, 63, 62, 60, 58, 56, 56, 56, 56, + 135, 141, 146, 152, 157, 156, 154, 152, 151, 143, 136, 129, 121, 115, + 108, 102, 96, 91, 87, 82, 78, 75, 71, 68, 65, 63, 61, 59, 57, 57, 57, + 57, 144, 146, 148, 151, 153, 152, 151, 151, 150, 144, 137, 130, 124, + 118, 111, 105, 99, 94, 89, 85, 80, 77, 74, 71, 67, 65, 63, 61, 59, 59, + 59, 59, 153, 152, 151, 149, 148, 148, 149, 149, 150, 144, 138, 132, 127, + 120, 114, 108, 102, 97, 92, 87, 83, 79, 76, 73, 69, 67, 65, 63, 60, 60, + 60, 60, 162, 157, 153, 148, 143, 145, 146, 148, 149, 144, 139, 134, 130, + 123, 117, 111, 105, 100, 95, 90, 85, 82, 78, 75, 71, 69, 67, 64, 62, 62, + 62, 62, 159, 156, 152, 148, 145, 143, 142, 141, 139, 135, 130, 126, 121, + 116, 111, 105, 100, 96, 91, 87, 83, 79, 76, 73, 70, 68, 65, 63, 61, 61, + 61, 61, 157, 154, 151, 149, 146, 142, 138, 134, 130, 126, 122, 118, 113, + 109, 104, 100, 95, 91, 88, 84, 80, 77, 74, 71, 68, 66, 64, 62, 60, 60, + 60, 60, 154, 152, 151, 149, 148, 141, 134, 127, 120, 117, 113, 109, 105, + 102, 98, 94, 90, 87, 84, 81, 77, 75, 72, 69, 67, 65, 63, 61, 59, 59, 59, + 59, 151, 151, 150, 150, 149, 139, 130, 120, 111, 108, 104, 101, 97, 94, + 91, 89, 86, 83, 80, 77, 75, 72, 70, 68, 65, 64, 62, 60, 58, 58, 58, 58, + 143, 143, 144, 144, 144, 135, 126, 117, 108, 104, 100, 97, 93, 90, 87, + 85, 82, 79, 77, 74, 72, 70, 68, 66, 64, 62, 61, 59, 57, 57, 57, 57, 135, + 136, 137, 138, 139, 130, 122, 113, 104, 100, 96, 93, 89, 86, 84, 81, 78, + 76, 74, 72, 69, 68, 66, 64, 62, 61, 59, 58, 56, 56, 56, 56, 127, 129, + 130, 132, 134, 126, 118, 109, 101, 97, 93, 88, 84, 82, 80, 77, 75, 73, + 71, 69, 67, 65, 64, 62, 60, 59, 58, 57, 55, 55, 55, 55, 118, 121, 124, + 127, 130, 121, 113, 105, 97, 93, 89, 84, 80, 78, 76, 73, 71, 69, 68, 66, + 64, 63, 62, 60, 59, 58, 57, 55, 54, 54, 54, 54, 112, 115, 118, 120, 123, + 116, 109, 102, 94, 90, 86, 82, 78, 76, 73, 71, 69, 67, 66, 64, 63, 61, + 60, 59, 58, 56, 55, 54, 53, 53, 53, 53, 105, 108, 111, 114, 117, 111, + 104, 98, 91, 87, 84, 80, 76, 73, 71, 69, 67, 65, 64, 62, 61, 60, 59, 57, + 56, 55, 54, 53, 52, 52, 52, 52, 99, 102, 105, 108, 111, 105, 100, 94, + 89, 85, 81, 77, 73, 71, 69, 67, 65, 63, 62, 60, 59, 58, 57, 56, 55, 54, + 53, 52, 51, 51, 51, 51, 92, 96, 99, 102, 105, 100, 95, 90, 86, 82, 78, + 75, 71, 69, 67, 65, 63, 61, 60, 59, 57, 56, 55, 55, 54, 53, 52, 51, 51, + 51, 51, 51, 88, 91, 94, 97, 100, 96, 91, 87, 83, 79, 76, 73, 69, 67, 65, + 63, 61, 60, 59, 57, 56, 55, 54, 54, 53, 52, 51, 51, 50, 50, 50, 50, 84, + 87, 89, 92, 95, 91, 88, 84, 80, 77, 74, 71, 68, 66, 64, 62, 60, 59, 58, + 56, 55, 54, 53, 53, 52, 51, 50, 50, 49, 49, 49, 49, 79, 82, 85, 87, 90, + 87, 84, 81, 77, 74, 72, 69, 66, 64, 62, 60, 59, 57, 56, 55, 54, 53, 52, + 52, 51, 50, 50, 49, 48, 48, 48, 48, 75, 78, 80, 83, 85, 83, 80, 77, 75, + 72, 69, 67, 64, 63, 61, 59, 57, 56, 55, 54, 53, 52, 51, 51, 50, 49, 49, + 48, 48, 48, 48, 48, 72, 75, 77, 79, 82, 79, 77, 75, 72, 70, 68, 65, 63, + 61, 60, 58, 56, 55, 54, 53, 52, 51, 51, 50, 49, 49, 48, 48, 47, 47, 47, + 47, 69, 71, 74, 76, 78, 76, 74, 72, 70, 68, 66, 64, 62, 60, 59, 57, 55, + 54, 53, 52, 51, 51, 50, 49, 49, 48, 48, 47, 47, 47, 47, 47, 66, 68, 71, + 73, 75, 73, 71, 69, 68, 66, 64, 62, 60, 59, 57, 56, 55, 54, 53, 52, 51, + 50, 49, 49, 48, 48, 47, 47, 46, 46, 46, 46, 63, 65, 67, 69, 71, 70, 68, + 67, 65, 64, 62, 60, 59, 58, 56, 55, 54, 53, 52, 51, 50, 49, 49, 48, 47, + 47, 47, 46, 46, 46, 46, 46, 62, 63, 65, 67, 69, 68, 66, 65, 64, 62, 61, + 59, 58, 56, 55, 54, 53, 52, 51, 50, 49, 49, 48, 48, 47, 47, 46, 46, 45, + 45, 45, 45, 60, 61, 63, 65, 67, 65, 64, 63, 62, 61, 59, 58, 57, 55, 54, + 53, 52, 51, 50, 50, 49, 48, 48, 47, 47, 46, 46, 45, 45, 45, 45, 45, 58, + 59, 61, 63, 64, 63, 62, 61, 60, 59, 58, 57, 55, 54, 53, 52, 51, 51, 50, + 49, 48, 48, 47, 47, 46, 46, 45, 45, 45, 45, 45, 45, 56, 57, 59, 60, 62, + 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 51, 50, 49, 48, 48, 47, 47, + 46, 46, 45, 45, 45, 44, 44, 44, 44, 56, 57, 59, 60, 62, 61, 60, 59, 58, + 57, 56, 55, 54, 53, 52, 51, 51, 50, 49, 48, 48, 47, 47, 46, 46, 45, 45, + 45, 44, 44, 44, 44, 56, 57, 59, 60, 62, 61, 60, 59, 58, 57, 56, 55, 54, + 53, 52, 51, 51, 50, 49, 48, 48, 47, 47, 46, 46, 45, 45, 45, 44, 44, 44, + 44, 56, 57, 59, 60, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 51, + 50, 49, 48, 48, 47, 47, 46, 46, 45, 45, 45, 44, 44, 44, 44 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 44, 41, 34, 44, 38, 36, 33, 41, 36, 30, 27, 34, 33, 27, 25, + /* Size 8 */ + 64, 77, 47, 45, 42, 39, 36, 33, 77, 56, 46, 50, 49, 45, 41, 37, 47, 46, + 41, 42, 42, 40, 38, 35, 45, 50, 42, 38, 37, 36, 34, 32, 42, 49, 42, 37, + 34, 32, 31, 30, 39, 45, 40, 36, 32, 30, 29, 28, 36, 41, 38, 34, 31, 29, + 27, 26, 33, 37, 35, 32, 30, 28, 26, 25, + /* Size 16 */ + 64, 71, 77, 62, 47, 46, 45, 43, 42, 41, 39, 37, 36, 34, 33, 33, 71, 69, + 67, 57, 46, 47, 47, 46, 46, 44, 42, 40, 38, 37, 35, 35, 77, 67, 56, 51, + 46, 48, 50, 49, 49, 47, 45, 43, 41, 39, 37, 37, 62, 57, 51, 47, 43, 45, + 46, 46, 45, 44, 43, 41, 40, 38, 36, 36, 47, 46, 46, 43, 41, 41, 42, 42, + 42, 41, 40, 39, 38, 37, 35, 35, 46, 47, 48, 45, 41, 41, 40, 40, 39, 39, + 38, 37, 36, 35, 34, 34, 45, 47, 50, 46, 42, 40, 38, 37, 37, 36, 36, 35, + 34, 33, 32, 32, 43, 46, 49, 46, 42, 40, 37, 36, 35, 35, 34, 33, 33, 32, + 31, 31, 42, 46, 49, 45, 42, 39, 37, 35, 34, 33, 32, 32, 31, 30, 30, 30, + 41, 44, 47, 44, 41, 39, 36, 35, 33, 32, 31, 31, 30, 29, 29, 29, 39, 42, + 45, 43, 40, 38, 36, 34, 32, 31, 30, 30, 29, 28, 28, 28, 37, 40, 43, 41, + 39, 37, 35, 33, 32, 31, 30, 29, 28, 28, 27, 27, 36, 38, 41, 40, 38, 36, + 34, 33, 31, 30, 29, 28, 27, 27, 26, 26, 34, 37, 39, 38, 37, 35, 33, 32, + 30, 29, 28, 28, 27, 26, 26, 26, 33, 35, 37, 36, 35, 34, 32, 31, 30, 29, + 28, 27, 26, 26, 25, 25, 33, 35, 37, 36, 35, 34, 32, 31, 30, 29, 28, 27, + 26, 26, 25, 25, + /* Size 32 */ + 64, 67, 71, 74, 77, 69, 62, 54, 47, 46, 46, 45, 45, 44, 43, 43, 42, 42, + 41, 40, 39, 38, 37, 36, 36, 35, 34, 33, 33, 33, 33, 33, 67, 68, 70, 71, + 72, 66, 59, 53, 47, 46, 46, 46, 46, 45, 45, 44, 44, 43, 42, 41, 40, 40, + 39, 38, 37, 36, 35, 35, 34, 34, 34, 34, 71, 70, 69, 68, 67, 62, 57, 51, + 46, 47, 47, 47, 47, 47, 46, 46, 46, 45, 44, 43, 42, 41, 40, 39, 38, 38, + 37, 36, 35, 35, 35, 35, 74, 71, 68, 65, 61, 58, 54, 50, 46, 47, 47, 48, + 49, 48, 48, 47, 47, 46, 45, 44, 44, 43, 42, 41, 40, 39, 38, 37, 36, 36, + 36, 36, 77, 72, 67, 61, 56, 54, 51, 49, 46, 47, 48, 49, 50, 50, 49, 49, + 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 37, 37, 37, 69, 66, + 62, 58, 54, 51, 49, 47, 45, 46, 46, 47, 48, 48, 47, 47, 47, 46, 45, 45, + 44, 43, 42, 41, 40, 39, 39, 38, 37, 37, 37, 37, 62, 59, 57, 54, 51, 49, + 47, 45, 43, 44, 45, 45, 46, 46, 46, 45, 45, 45, 44, 43, 43, 42, 41, 40, + 40, 39, 38, 37, 36, 36, 36, 36, 54, 53, 51, 50, 49, 47, 45, 44, 42, 43, + 43, 43, 44, 44, 44, 44, 44, 43, 43, 42, 42, 41, 40, 39, 39, 38, 37, 36, + 36, 36, 36, 36, 47, 47, 46, 46, 46, 45, 43, 42, 41, 41, 41, 42, 42, 42, + 42, 42, 42, 42, 41, 41, 40, 40, 39, 38, 38, 37, 37, 36, 35, 35, 35, 35, + 46, 46, 47, 47, 47, 46, 44, 43, 41, 41, 41, 41, 41, 41, 41, 41, 41, 40, + 40, 40, 39, 39, 38, 37, 37, 36, 36, 35, 34, 34, 34, 34, 46, 46, 47, 47, + 48, 46, 45, 43, 41, 41, 41, 40, 40, 40, 40, 40, 39, 39, 39, 38, 38, 37, + 37, 36, 36, 35, 35, 34, 34, 34, 34, 34, 45, 46, 47, 48, 49, 47, 45, 43, + 42, 41, 40, 40, 39, 39, 39, 38, 38, 38, 37, 37, 37, 36, 36, 36, 35, 35, + 34, 34, 33, 33, 33, 33, 45, 46, 47, 49, 50, 48, 46, 44, 42, 41, 40, 39, + 38, 38, 37, 37, 37, 36, 36, 36, 36, 35, 35, 35, 34, 34, 33, 33, 32, 32, + 32, 32, 44, 45, 47, 48, 50, 48, 46, 44, 42, 41, 40, 39, 38, 37, 37, 36, + 36, 36, 35, 35, 35, 34, 34, 34, 33, 33, 33, 32, 32, 32, 32, 32, 43, 45, + 46, 48, 49, 47, 46, 44, 42, 41, 40, 39, 37, 37, 36, 36, 35, 35, 35, 34, + 34, 34, 33, 33, 33, 32, 32, 32, 31, 31, 31, 31, 43, 44, 46, 47, 49, 47, + 45, 44, 42, 41, 40, 38, 37, 36, 36, 35, 35, 34, 34, 34, 33, 33, 33, 32, + 32, 32, 31, 31, 31, 31, 31, 31, 42, 44, 46, 47, 49, 47, 45, 44, 42, 41, + 39, 38, 37, 36, 35, 35, 34, 34, 33, 33, 32, 32, 32, 31, 31, 31, 30, 30, + 30, 30, 30, 30, 42, 43, 45, 46, 48, 46, 45, 43, 42, 40, 39, 38, 36, 36, + 35, 34, 34, 33, 33, 32, 32, 32, 31, 31, 31, 30, 30, 30, 29, 29, 29, 29, + 41, 42, 44, 45, 47, 45, 44, 43, 41, 40, 39, 37, 36, 35, 35, 34, 33, 33, + 32, 32, 31, 31, 31, 30, 30, 30, 29, 29, 29, 29, 29, 29, 40, 41, 43, 44, + 46, 45, 43, 42, 41, 40, 38, 37, 36, 35, 34, 34, 33, 32, 32, 31, 31, 30, + 30, 30, 29, 29, 29, 29, 28, 28, 28, 28, 39, 40, 42, 44, 45, 44, 43, 42, + 40, 39, 38, 37, 36, 35, 34, 33, 32, 32, 31, 31, 30, 30, 30, 29, 29, 29, + 28, 28, 28, 28, 28, 28, 38, 40, 41, 43, 44, 43, 42, 41, 40, 39, 37, 36, + 35, 34, 34, 33, 32, 32, 31, 30, 30, 30, 29, 29, 29, 28, 28, 28, 28, 28, + 28, 28, 37, 39, 40, 42, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 33, + 32, 31, 31, 30, 30, 29, 29, 29, 28, 28, 28, 27, 27, 27, 27, 27, 36, 38, + 39, 41, 42, 41, 40, 39, 38, 37, 36, 36, 35, 34, 33, 32, 31, 31, 30, 30, + 29, 29, 29, 28, 28, 28, 27, 27, 27, 27, 27, 27, 36, 37, 38, 40, 41, 40, + 40, 39, 38, 37, 36, 35, 34, 33, 33, 32, 31, 31, 30, 29, 29, 29, 28, 28, + 27, 27, 27, 27, 26, 26, 26, 26, 35, 36, 38, 39, 40, 39, 39, 38, 37, 36, + 35, 35, 34, 33, 32, 32, 31, 30, 30, 29, 29, 28, 28, 28, 27, 27, 27, 26, + 26, 26, 26, 26, 34, 35, 37, 38, 39, 39, 38, 37, 37, 36, 35, 34, 33, 33, + 32, 31, 30, 30, 29, 29, 28, 28, 28, 27, 27, 27, 26, 26, 26, 26, 26, 26, + 33, 35, 36, 37, 38, 38, 37, 36, 36, 35, 34, 34, 33, 32, 32, 31, 30, 30, + 29, 29, 28, 28, 27, 27, 27, 26, 26, 26, 26, 26, 26, 26, 33, 34, 35, 36, + 37, 37, 36, 36, 35, 34, 34, 33, 32, 32, 31, 31, 30, 29, 29, 28, 28, 28, + 27, 27, 26, 26, 26, 26, 25, 25, 25, 25, 33, 34, 35, 36, 37, 37, 36, 36, + 35, 34, 34, 33, 32, 32, 31, 31, 30, 29, 29, 28, 28, 28, 27, 27, 26, 26, + 26, 26, 25, 25, 25, 25, 33, 34, 35, 36, 37, 37, 36, 36, 35, 34, 34, 33, + 32, 32, 31, 31, 30, 29, 29, 28, 28, 28, 27, 27, 26, 26, 26, 26, 25, 25, + 25, 25, 33, 34, 35, 36, 37, 37, 36, 36, 35, 34, 34, 33, 32, 32, 31, 31, + 30, 29, 29, 28, 28, 28, 27, 27, 26, 26, 26, 26, 25, 25, 25, 25 }, + { /* Intra matrices */ + /* Size 4 */ + 125, 84, 78, 64, 84, 72, 68, 61, 78, 68, 56, 51, 64, 61, 51, 45, + /* Size 8 */ + 117, 141, 84, 80, 76, 69, 63, 57, 141, 102, 83, 90, 88, 81, 73, 66, 84, + 83, 73, 75, 75, 72, 67, 62, 80, 90, 75, 68, 65, 63, 60, 57, 76, 88, 75, + 65, 60, 57, 54, 52, 69, 81, 72, 63, 57, 53, 50, 48, 63, 73, 67, 60, 54, + 50, 47, 45, 57, 66, 62, 57, 52, 48, 45, 43, + /* Size 16 */ + 119, 132, 144, 115, 86, 84, 82, 79, 77, 74, 71, 67, 64, 61, 58, 58, 132, + 128, 124, 105, 85, 86, 87, 85, 83, 80, 77, 73, 69, 66, 63, 63, 144, 124, + 104, 94, 84, 88, 92, 91, 89, 86, 83, 79, 75, 71, 68, 68, 115, 105, 94, + 87, 79, 82, 84, 83, 83, 80, 78, 75, 72, 69, 65, 65, 86, 85, 84, 79, 74, + 75, 76, 76, 76, 75, 73, 71, 68, 66, 63, 63, 84, 86, 88, 82, 75, 74, 73, + 72, 71, 70, 69, 67, 65, 63, 61, 61, 82, 87, 92, 84, 76, 73, 69, 68, 66, + 65, 64, 63, 61, 60, 58, 58, 79, 85, 91, 83, 76, 72, 68, 66, 64, 62, 61, + 60, 58, 57, 56, 56, 77, 83, 89, 83, 76, 71, 66, 64, 61, 59, 58, 57, 55, + 54, 53, 53, 74, 80, 86, 80, 75, 70, 65, 62, 59, 58, 56, 55, 53, 52, 51, + 51, 71, 77, 83, 78, 73, 69, 64, 61, 58, 56, 54, 53, 51, 50, 49, 49, 67, + 73, 79, 75, 71, 67, 63, 60, 57, 55, 53, 51, 50, 49, 48, 48, 64, 69, 75, + 72, 68, 65, 61, 58, 55, 53, 51, 50, 48, 47, 46, 46, 61, 66, 71, 69, 66, + 63, 60, 57, 54, 52, 50, 49, 47, 46, 45, 45, 58, 63, 68, 65, 63, 61, 58, + 56, 53, 51, 49, 48, 46, 45, 44, 44, 58, 63, 68, 65, 63, 61, 58, 56, 53, + 51, 49, 48, 46, 45, 44, 44, + /* Size 32 */ + 120, 127, 133, 140, 146, 131, 116, 101, 87, 86, 85, 84, 82, 81, 80, 79, + 78, 76, 75, 73, 71, 70, 68, 66, 65, 63, 62, 60, 59, 59, 59, 59, 127, + 129, 131, 134, 136, 123, 111, 99, 86, 86, 86, 85, 85, 84, 83, 82, 81, + 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, 63, 61, 61, 61, 61, 133, 131, + 129, 127, 126, 116, 106, 96, 86, 86, 87, 87, 88, 87, 86, 85, 84, 83, 81, + 79, 77, 76, 74, 72, 70, 69, 67, 65, 64, 64, 64, 64, 140, 134, 127, 121, + 115, 108, 101, 93, 86, 87, 88, 89, 90, 89, 89, 88, 87, 86, 84, 82, 81, + 79, 77, 75, 73, 71, 70, 68, 66, 66, 66, 66, 146, 136, 126, 115, 105, + 100, 95, 90, 85, 87, 89, 91, 93, 92, 92, 91, 90, 89, 87, 85, 84, 82, 80, + 78, 76, 74, 72, 70, 68, 68, 68, 68, 131, 123, 116, 108, 100, 96, 92, 87, + 83, 84, 86, 87, 89, 88, 88, 88, 87, 86, 84, 83, 81, 79, 78, 76, 74, 72, + 71, 69, 67, 67, 67, 67, 116, 111, 106, 101, 95, 92, 88, 84, 80, 81, 83, + 84, 85, 85, 84, 84, 84, 83, 81, 80, 79, 77, 76, 74, 72, 71, 69, 68, 66, + 66, 66, 66, 101, 99, 96, 93, 90, 87, 84, 81, 78, 78, 79, 80, 81, 81, 81, + 81, 81, 80, 79, 78, 76, 75, 74, 72, 71, 69, 68, 67, 65, 65, 65, 65, 87, + 86, 86, 86, 85, 83, 80, 78, 75, 75, 76, 77, 77, 77, 77, 77, 77, 76, 76, + 75, 74, 73, 72, 70, 69, 68, 67, 65, 64, 64, 64, 64, 86, 86, 86, 87, 87, + 84, 81, 78, 75, 75, 75, 75, 75, 75, 75, 75, 75, 74, 73, 73, 72, 71, 70, + 69, 67, 66, 65, 64, 63, 63, 63, 63, 85, 86, 87, 88, 89, 86, 83, 79, 76, + 75, 75, 74, 73, 73, 73, 73, 72, 72, 71, 70, 70, 69, 68, 67, 66, 65, 63, + 62, 61, 61, 61, 61, 84, 85, 87, 89, 91, 87, 84, 80, 77, 75, 74, 73, 72, + 71, 71, 70, 70, 69, 68, 68, 67, 66, 66, 65, 64, 63, 62, 61, 60, 60, 60, + 60, 82, 85, 88, 90, 93, 89, 85, 81, 77, 75, 73, 72, 70, 69, 68, 68, 67, + 67, 66, 65, 65, 64, 63, 63, 62, 61, 60, 59, 59, 59, 59, 59, 81, 84, 87, + 89, 92, 88, 85, 81, 77, 75, 73, 71, 69, 68, 67, 67, 66, 65, 65, 64, 63, + 63, 62, 61, 61, 60, 59, 58, 57, 57, 57, 57, 80, 83, 86, 89, 92, 88, 84, + 81, 77, 75, 73, 71, 68, 67, 66, 65, 64, 64, 63, 62, 62, 61, 60, 60, 59, + 58, 58, 57, 56, 56, 56, 56, 79, 82, 85, 88, 91, 88, 84, 81, 77, 75, 73, + 70, 68, 67, 65, 64, 63, 62, 62, 61, 60, 59, 59, 58, 58, 57, 56, 56, 55, + 55, 55, 55, 78, 81, 84, 87, 90, 87, 84, 81, 77, 75, 72, 70, 67, 66, 64, + 63, 62, 61, 60, 59, 58, 58, 57, 57, 56, 55, 55, 54, 54, 54, 54, 54, 76, + 80, 83, 86, 89, 86, 83, 80, 76, 74, 72, 69, 67, 65, 64, 62, 61, 60, 59, + 58, 57, 57, 56, 56, 55, 54, 54, 53, 53, 53, 53, 53, 75, 78, 81, 84, 87, + 84, 81, 79, 76, 73, 71, 68, 66, 65, 63, 62, 60, 59, 58, 57, 56, 56, 55, + 55, 54, 53, 53, 52, 52, 52, 52, 52, 73, 76, 79, 82, 85, 83, 80, 78, 75, + 73, 70, 68, 65, 64, 62, 61, 59, 58, 57, 56, 55, 55, 54, 54, 53, 52, 52, + 51, 51, 51, 51, 51, 71, 74, 77, 81, 84, 81, 79, 76, 74, 72, 70, 67, 65, + 63, 62, 60, 58, 57, 56, 55, 54, 54, 53, 52, 52, 51, 51, 50, 50, 50, 50, + 50, 70, 73, 76, 79, 82, 79, 77, 75, 73, 71, 69, 66, 64, 63, 61, 59, 58, + 57, 56, 55, 54, 53, 52, 52, 51, 51, 50, 50, 49, 49, 49, 49, 68, 71, 74, + 77, 80, 78, 76, 74, 72, 70, 68, 66, 63, 62, 60, 59, 57, 56, 55, 54, 53, + 52, 52, 51, 50, 50, 49, 49, 48, 48, 48, 48, 66, 69, 72, 75, 78, 76, 74, + 72, 70, 69, 67, 65, 63, 61, 60, 58, 57, 56, 55, 54, 52, 52, 51, 50, 50, + 49, 49, 48, 48, 48, 48, 48, 65, 67, 70, 73, 76, 74, 72, 71, 69, 67, 66, + 64, 62, 61, 59, 58, 56, 55, 54, 53, 52, 51, 50, 50, 49, 48, 48, 47, 47, + 47, 47, 47, 63, 66, 69, 71, 74, 72, 71, 69, 68, 66, 65, 63, 61, 60, 58, + 57, 55, 54, 53, 52, 51, 51, 50, 49, 48, 48, 47, 47, 46, 46, 46, 46, 62, + 64, 67, 70, 72, 71, 69, 68, 67, 65, 63, 62, 60, 59, 58, 56, 55, 54, 53, + 52, 51, 50, 49, 49, 48, 47, 47, 46, 46, 46, 46, 46, 60, 63, 65, 68, 70, + 69, 68, 67, 65, 64, 62, 61, 59, 58, 57, 56, 54, 53, 52, 51, 50, 50, 49, + 48, 47, 47, 46, 46, 45, 45, 45, 45, 59, 61, 64, 66, 68, 67, 66, 65, 64, + 63, 61, 60, 59, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 48, 47, 46, 46, + 45, 45, 45, 45, 45, 59, 61, 64, 66, 68, 67, 66, 65, 64, 63, 61, 60, 59, + 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 48, 47, 46, 46, 45, 45, 45, 45, + 45, 59, 61, 64, 66, 68, 67, 66, 65, 64, 63, 61, 60, 59, 57, 56, 55, 54, + 53, 52, 51, 50, 49, 48, 48, 47, 46, 46, 45, 45, 45, 45, 45, 59, 61, 64, + 66, 68, 67, 66, 65, 64, 63, 61, 60, 59, 57, 56, 55, 54, 53, 52, 51, 50, + 49, 48, 48, 47, 46, 46, 45, 45, 45, 45, 45 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 60, 41, 31, 60, 44, 35, 29, 41, 35, 29, 26, 31, 29, 26, 24, + /* Size 8 */ + 64, 80, 75, 61, 49, 41, 36, 33, 80, 72, 74, 66, 55, 46, 40, 35, 75, 74, + 57, 51, 46, 41, 37, 34, 61, 66, 51, 43, 39, 36, 34, 32, 49, 55, 46, 39, + 36, 33, 32, 30, 41, 46, 41, 36, 33, 31, 30, 29, 36, 40, 37, 34, 32, 30, + 29, 28, 33, 35, 34, 32, 30, 29, 28, 27, + /* Size 16 */ + 64, 72, 80, 78, 75, 68, 61, 55, 49, 45, 41, 39, 36, 34, 33, 33, 72, 74, + 76, 75, 75, 69, 63, 57, 52, 48, 43, 41, 38, 36, 34, 34, 80, 76, 72, 73, + 74, 70, 66, 60, 55, 50, 46, 43, 40, 37, 35, 35, 78, 75, 73, 69, 66, 62, + 58, 54, 50, 47, 43, 41, 38, 36, 34, 34, 75, 75, 74, 66, 57, 54, 51, 49, + 46, 43, 41, 39, 37, 35, 34, 34, 68, 69, 70, 62, 54, 51, 47, 45, 43, 41, + 39, 37, 35, 34, 33, 33, 61, 63, 66, 58, 51, 47, 43, 41, 39, 38, 36, 35, + 34, 33, 32, 32, 55, 57, 60, 54, 49, 45, 41, 39, 37, 36, 35, 34, 33, 32, + 31, 31, 49, 52, 55, 50, 46, 43, 39, 37, 36, 34, 33, 32, 32, 31, 30, 30, + 45, 48, 50, 47, 43, 41, 38, 36, 34, 33, 32, 31, 31, 30, 30, 30, 41, 43, + 46, 43, 41, 39, 36, 35, 33, 32, 31, 31, 30, 29, 29, 29, 39, 41, 43, 41, + 39, 37, 35, 34, 32, 31, 31, 30, 29, 29, 28, 28, 36, 38, 40, 38, 37, 35, + 34, 33, 32, 31, 30, 29, 29, 28, 28, 28, 34, 36, 37, 36, 35, 34, 33, 32, + 31, 30, 29, 29, 28, 28, 28, 28, 33, 34, 35, 34, 34, 33, 32, 31, 30, 30, + 29, 28, 28, 28, 27, 27, 33, 34, 35, 34, 34, 33, 32, 31, 30, 30, 29, 28, + 28, 28, 27, 27, + /* Size 32 */ + 64, 68, 72, 76, 80, 79, 78, 77, 75, 72, 68, 64, 61, 58, 55, 52, 49, 47, + 45, 43, 41, 40, 39, 37, 36, 35, 34, 33, 33, 33, 33, 33, 68, 71, 73, 76, + 78, 77, 77, 76, 75, 72, 69, 65, 62, 59, 56, 53, 50, 48, 46, 44, 42, 41, + 40, 38, 37, 36, 35, 34, 33, 33, 33, 33, 72, 73, 74, 75, 76, 76, 75, 75, + 75, 72, 69, 66, 63, 60, 57, 55, 52, 50, 48, 46, 43, 42, 41, 39, 38, 37, + 36, 35, 34, 34, 34, 34, 76, 76, 75, 75, 74, 74, 74, 74, 75, 72, 70, 67, + 64, 62, 59, 56, 53, 51, 49, 47, 45, 43, 42, 40, 39, 38, 37, 36, 35, 35, + 35, 35, 80, 78, 76, 74, 72, 72, 73, 74, 74, 72, 70, 68, 66, 63, 60, 57, + 55, 52, 50, 48, 46, 44, 43, 41, 40, 38, 37, 36, 35, 35, 35, 35, 79, 77, + 76, 74, 72, 72, 71, 71, 70, 68, 66, 64, 62, 60, 57, 55, 52, 50, 48, 47, + 45, 43, 42, 40, 39, 38, 37, 36, 35, 35, 35, 35, 78, 77, 75, 74, 73, 71, + 69, 68, 66, 64, 62, 60, 58, 56, 54, 52, 50, 49, 47, 45, 43, 42, 41, 39, + 38, 37, 36, 35, 34, 34, 34, 34, 77, 76, 75, 74, 74, 71, 68, 65, 62, 60, + 58, 56, 55, 53, 51, 50, 48, 47, 45, 44, 42, 41, 40, 39, 38, 37, 36, 35, + 34, 34, 34, 34, 75, 75, 75, 75, 74, 70, 66, 62, 57, 56, 54, 53, 51, 50, + 49, 47, 46, 45, 43, 42, 41, 40, 39, 38, 37, 36, 35, 35, 34, 34, 34, 34, + 72, 72, 72, 72, 72, 68, 64, 60, 56, 54, 53, 51, 49, 48, 47, 46, 44, 43, + 42, 41, 40, 39, 38, 37, 36, 35, 35, 34, 33, 33, 33, 33, 68, 69, 69, 70, + 70, 66, 62, 58, 54, 53, 51, 49, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, + 37, 36, 35, 35, 34, 33, 33, 33, 33, 33, 64, 65, 66, 67, 68, 64, 60, 56, + 53, 51, 49, 47, 45, 44, 43, 42, 41, 40, 39, 38, 38, 37, 36, 35, 35, 34, + 33, 33, 32, 32, 32, 32, 61, 62, 63, 64, 66, 62, 58, 55, 51, 49, 47, 45, + 43, 42, 41, 40, 39, 39, 38, 37, 36, 36, 35, 35, 34, 33, 33, 32, 32, 32, + 32, 32, 58, 59, 60, 62, 63, 60, 56, 53, 50, 48, 46, 44, 42, 41, 40, 39, + 38, 38, 37, 36, 36, 35, 34, 34, 33, 33, 32, 32, 31, 31, 31, 31, 55, 56, + 57, 59, 60, 57, 54, 51, 49, 47, 45, 43, 41, 40, 39, 38, 37, 37, 36, 35, + 35, 34, 34, 33, 33, 32, 32, 31, 31, 31, 31, 31, 52, 53, 55, 56, 57, 55, + 52, 50, 47, 46, 44, 42, 40, 39, 38, 37, 37, 36, 35, 35, 34, 34, 33, 33, + 32, 32, 31, 31, 31, 31, 31, 31, 49, 50, 52, 53, 55, 52, 50, 48, 46, 44, + 43, 41, 39, 38, 37, 37, 36, 35, 34, 34, 33, 33, 32, 32, 32, 31, 31, 31, + 30, 30, 30, 30, 47, 48, 50, 51, 52, 50, 49, 47, 45, 43, 42, 40, 39, 38, + 37, 36, 35, 34, 34, 33, 33, 32, 32, 32, 31, 31, 31, 30, 30, 30, 30, 30, + 45, 46, 48, 49, 50, 48, 47, 45, 43, 42, 41, 39, 38, 37, 36, 35, 34, 34, + 33, 33, 32, 32, 31, 31, 31, 30, 30, 30, 30, 30, 30, 30, 43, 44, 46, 47, + 48, 47, 45, 44, 42, 41, 40, 38, 37, 36, 35, 35, 34, 33, 33, 32, 32, 31, + 31, 31, 30, 30, 30, 29, 29, 29, 29, 29, 41, 42, 43, 45, 46, 45, 43, 42, + 41, 40, 39, 38, 36, 36, 35, 34, 33, 33, 32, 32, 31, 31, 31, 30, 30, 30, + 29, 29, 29, 29, 29, 29, 40, 41, 42, 43, 44, 43, 42, 41, 40, 39, 38, 37, + 36, 35, 34, 34, 33, 32, 32, 31, 31, 31, 30, 30, 30, 29, 29, 29, 29, 29, + 29, 29, 39, 40, 41, 42, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 34, 33, + 32, 32, 31, 31, 31, 30, 30, 30, 29, 29, 29, 29, 28, 28, 28, 28, 37, 38, + 39, 40, 41, 40, 39, 39, 38, 37, 36, 35, 35, 34, 33, 33, 32, 32, 31, 31, + 30, 30, 30, 29, 29, 29, 29, 28, 28, 28, 28, 28, 36, 37, 38, 39, 40, 39, + 38, 38, 37, 36, 35, 35, 34, 33, 33, 32, 32, 31, 31, 30, 30, 30, 29, 29, + 29, 29, 28, 28, 28, 28, 28, 28, 35, 36, 37, 38, 38, 38, 37, 37, 36, 35, + 35, 34, 33, 33, 32, 32, 31, 31, 30, 30, 30, 29, 29, 29, 29, 28, 28, 28, + 28, 28, 28, 28, 34, 35, 36, 37, 37, 37, 36, 36, 35, 35, 34, 33, 33, 32, + 32, 31, 31, 31, 30, 30, 29, 29, 29, 29, 28, 28, 28, 28, 28, 28, 28, 28, + 33, 34, 35, 36, 36, 36, 35, 35, 35, 34, 33, 33, 32, 32, 31, 31, 31, 30, + 30, 29, 29, 29, 29, 28, 28, 28, 28, 28, 27, 27, 27, 27, 33, 33, 34, 35, + 35, 35, 34, 34, 34, 33, 33, 32, 32, 31, 31, 31, 30, 30, 30, 29, 29, 29, + 28, 28, 28, 28, 28, 27, 27, 27, 27, 27, 33, 33, 34, 35, 35, 35, 34, 34, + 34, 33, 33, 32, 32, 31, 31, 31, 30, 30, 30, 29, 29, 29, 28, 28, 28, 28, + 28, 27, 27, 27, 27, 27, 33, 33, 34, 35, 35, 35, 34, 34, 34, 33, 33, 32, + 32, 31, 31, 31, 30, 30, 30, 29, 29, 29, 28, 28, 28, 28, 28, 27, 27, 27, + 27, 27, 33, 33, 34, 35, 35, 35, 34, 34, 34, 33, 33, 32, 32, 31, 31, 31, + 30, 30, 30, 29, 29, 29, 28, 28, 28, 28, 28, 27, 27, 27, 27, 27 }, + { /* Intra matrices */ + /* Size 4 */ + 129, 119, 81, 60, 119, 86, 68, 56, 81, 68, 55, 49, 60, 56, 49, 45, + /* Size 8 */ + 111, 141, 132, 105, 84, 70, 60, 54, 141, 125, 130, 114, 94, 78, 67, 59, + 132, 130, 99, 88, 78, 69, 62, 56, 105, 114, 88, 74, 66, 61, 56, 53, 84, + 94, 78, 66, 59, 55, 52, 50, 70, 78, 69, 61, 55, 52, 49, 47, 60, 67, 62, + 56, 52, 49, 47, 46, 54, 59, 56, 53, 50, 47, 46, 44, + /* Size 16 */ + 114, 129, 144, 140, 135, 122, 108, 97, 86, 79, 71, 67, 62, 59, 55, 55, + 129, 133, 137, 135, 134, 123, 113, 102, 91, 84, 76, 70, 65, 62, 58, 58, + 144, 137, 129, 131, 134, 125, 117, 107, 96, 88, 80, 74, 68, 64, 60, 60, + 140, 135, 131, 124, 118, 111, 104, 96, 88, 82, 76, 71, 66, 62, 59, 59, + 135, 134, 134, 118, 102, 96, 90, 85, 80, 76, 71, 67, 63, 60, 58, 58, + 122, 123, 125, 111, 96, 89, 83, 79, 74, 71, 67, 64, 61, 58, 56, 56, 108, + 113, 117, 104, 90, 83, 76, 72, 68, 65, 63, 60, 58, 56, 54, 54, 97, 102, + 107, 96, 85, 79, 72, 68, 65, 62, 60, 58, 56, 54, 53, 53, 86, 91, 96, 88, + 80, 74, 68, 65, 61, 59, 57, 55, 54, 52, 51, 51, 79, 84, 88, 82, 76, 71, + 65, 62, 59, 57, 55, 53, 52, 51, 50, 50, 71, 76, 80, 76, 71, 67, 63, 60, + 57, 55, 53, 52, 50, 50, 49, 49, 67, 70, 74, 71, 67, 64, 60, 58, 55, 53, + 52, 51, 49, 49, 48, 48, 62, 65, 68, 66, 63, 61, 58, 56, 54, 52, 50, 49, + 48, 48, 47, 47, 59, 62, 64, 62, 60, 58, 56, 54, 52, 51, 50, 49, 48, 47, + 46, 46, 55, 58, 60, 59, 58, 56, 54, 53, 51, 50, 49, 48, 47, 46, 46, 46, + 55, 58, 60, 59, 58, 56, 54, 53, 51, 50, 49, 48, 47, 46, 46, 46, + /* Size 32 */ + 116, 123, 131, 139, 146, 144, 142, 139, 137, 130, 123, 116, 109, 104, + 98, 93, 87, 84, 80, 76, 72, 70, 68, 65, 63, 61, 59, 58, 56, 56, 56, 56, + 123, 128, 133, 138, 142, 141, 140, 138, 137, 130, 124, 118, 112, 106, + 101, 95, 90, 86, 82, 78, 75, 72, 69, 67, 64, 63, 61, 59, 57, 57, 57, 57, + 131, 133, 135, 137, 138, 138, 137, 137, 136, 131, 125, 120, 114, 109, + 103, 98, 93, 89, 85, 81, 77, 74, 71, 69, 66, 64, 62, 61, 59, 59, 59, 59, + 139, 138, 137, 136, 134, 135, 135, 135, 136, 131, 126, 121, 116, 111, + 106, 100, 95, 91, 87, 83, 79, 76, 73, 71, 68, 66, 64, 62, 60, 60, 60, + 60, 146, 142, 138, 134, 130, 132, 133, 134, 135, 131, 127, 123, 119, + 113, 108, 103, 98, 94, 89, 85, 81, 78, 75, 72, 69, 67, 65, 63, 61, 61, + 61, 61, 144, 141, 138, 135, 132, 131, 129, 128, 127, 123, 120, 116, 112, + 107, 103, 98, 94, 90, 86, 83, 79, 76, 73, 71, 68, 66, 64, 62, 61, 61, + 61, 61, 142, 140, 137, 135, 133, 129, 126, 123, 119, 116, 112, 109, 105, + 101, 97, 93, 90, 86, 83, 80, 77, 74, 72, 69, 67, 65, 63, 62, 60, 60, 60, + 60, 139, 138, 137, 135, 134, 128, 123, 117, 111, 108, 105, 101, 98, 95, + 92, 89, 86, 83, 80, 77, 74, 72, 70, 68, 66, 64, 62, 61, 59, 59, 59, 59, + 137, 137, 136, 136, 135, 127, 119, 111, 103, 100, 97, 94, 91, 89, 86, + 84, 81, 79, 77, 74, 72, 70, 68, 66, 64, 63, 61, 60, 58, 58, 58, 58, 130, + 130, 131, 131, 131, 123, 116, 108, 100, 97, 94, 91, 88, 85, 83, 81, 78, + 76, 74, 72, 70, 68, 66, 65, 63, 62, 60, 59, 57, 57, 57, 57, 123, 124, + 125, 126, 127, 120, 112, 105, 97, 94, 91, 87, 84, 82, 80, 77, 75, 73, + 71, 70, 68, 66, 65, 63, 61, 60, 59, 58, 57, 57, 57, 57, 116, 118, 120, + 121, 123, 116, 109, 101, 94, 91, 87, 84, 80, 78, 76, 74, 72, 71, 69, 67, + 66, 64, 63, 61, 60, 59, 58, 57, 56, 56, 56, 56, 109, 112, 114, 116, 119, + 112, 105, 98, 91, 88, 84, 80, 77, 75, 73, 71, 69, 68, 66, 65, 63, 62, + 61, 60, 59, 58, 57, 56, 55, 55, 55, 55, 104, 106, 109, 111, 113, 107, + 101, 95, 89, 85, 82, 78, 75, 73, 71, 69, 67, 66, 65, 63, 62, 61, 60, 59, + 58, 57, 56, 55, 54, 54, 54, 54, 98, 101, 103, 106, 108, 103, 97, 92, 86, + 83, 80, 76, 73, 71, 69, 67, 65, 64, 63, 62, 60, 59, 58, 57, 56, 56, 55, + 54, 53, 53, 53, 53, 93, 95, 98, 100, 103, 98, 93, 89, 84, 81, 77, 74, + 71, 69, 67, 65, 64, 62, 61, 60, 59, 58, 57, 56, 55, 55, 54, 53, 52, 52, + 52, 52, 87, 90, 93, 95, 98, 94, 90, 86, 81, 78, 75, 72, 69, 67, 65, 64, + 62, 61, 60, 59, 57, 57, 56, 55, 54, 54, 53, 52, 52, 52, 52, 52, 84, 86, + 89, 91, 94, 90, 86, 83, 79, 76, 73, 71, 68, 66, 64, 62, 61, 60, 59, 58, + 56, 56, 55, 54, 53, 53, 52, 52, 51, 51, 51, 51, 80, 82, 85, 87, 89, 86, + 83, 80, 77, 74, 71, 69, 66, 65, 63, 61, 60, 59, 58, 57, 56, 55, 54, 53, + 53, 52, 52, 51, 50, 50, 50, 50, 76, 78, 81, 83, 85, 83, 80, 77, 74, 72, + 70, 67, 65, 63, 62, 60, 59, 58, 57, 56, 55, 54, 53, 53, 52, 51, 51, 50, + 50, 50, 50, 50, 72, 75, 77, 79, 81, 79, 77, 74, 72, 70, 68, 66, 63, 62, + 60, 59, 57, 56, 56, 55, 54, 53, 52, 52, 51, 51, 50, 50, 49, 49, 49, 49, + 70, 72, 74, 76, 78, 76, 74, 72, 70, 68, 66, 64, 62, 61, 59, 58, 57, 56, + 55, 54, 53, 52, 52, 51, 51, 50, 50, 49, 49, 49, 49, 49, 68, 69, 71, 73, + 75, 73, 72, 70, 68, 66, 65, 63, 61, 60, 58, 57, 56, 55, 54, 53, 52, 52, + 51, 51, 50, 50, 49, 49, 48, 48, 48, 48, 65, 67, 69, 71, 72, 71, 69, 68, + 66, 65, 63, 61, 60, 59, 57, 56, 55, 54, 53, 53, 52, 51, 51, 50, 50, 49, + 49, 48, 48, 48, 48, 48, 63, 64, 66, 68, 69, 68, 67, 66, 64, 63, 61, 60, + 59, 58, 56, 55, 54, 53, 53, 52, 51, 51, 50, 50, 49, 49, 48, 48, 47, 47, + 47, 47, 61, 63, 64, 66, 67, 66, 65, 64, 63, 62, 60, 59, 58, 57, 56, 55, + 54, 53, 52, 51, 51, 50, 50, 49, 49, 48, 48, 48, 47, 47, 47, 47, 59, 61, + 62, 64, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 52, 51, + 50, 50, 49, 49, 48, 48, 48, 47, 47, 47, 47, 47, 58, 59, 61, 62, 63, 62, + 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 52, 51, 50, 50, 49, 49, 48, + 48, 48, 47, 47, 47, 47, 47, 47, 56, 57, 59, 60, 61, 61, 60, 59, 58, 57, + 57, 56, 55, 54, 53, 52, 52, 51, 50, 50, 49, 49, 48, 48, 47, 47, 47, 47, + 46, 46, 46, 46, 56, 57, 59, 60, 61, 61, 60, 59, 58, 57, 57, 56, 55, 54, + 53, 52, 52, 51, 50, 50, 49, 49, 48, 48, 47, 47, 47, 47, 46, 46, 46, 46, + 56, 57, 59, 60, 61, 61, 60, 59, 58, 57, 57, 56, 55, 54, 53, 52, 52, 51, + 50, 50, 49, 49, 48, 48, 47, 47, 47, 47, 46, 46, 46, 46, 56, 57, 59, 60, + 61, 61, 60, 59, 58, 57, 57, 56, 55, 54, 53, 52, 52, 51, 50, 50, 49, 49, + 48, 48, 47, 47, 47, 47, 46, 46, 46, 46 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 45, 42, 36, 45, 40, 38, 35, 42, 38, 32, 30, 36, 35, 30, 27, + /* Size 8 */ + 64, 76, 48, 46, 44, 41, 37, 35, 76, 57, 47, 51, 50, 46, 43, 39, 48, 47, + 42, 43, 43, 42, 40, 37, 46, 51, 43, 40, 39, 38, 36, 35, 44, 50, 43, 39, + 36, 34, 33, 32, 41, 46, 42, 38, 34, 33, 31, 30, 37, 43, 40, 36, 33, 31, + 30, 29, 35, 39, 37, 35, 32, 30, 29, 28, + /* Size 16 */ + 64, 70, 76, 62, 48, 47, 46, 45, 44, 42, 41, 39, 37, 36, 35, 35, 70, 68, + 66, 57, 48, 48, 48, 48, 47, 45, 44, 42, 40, 39, 37, 37, 76, 66, 57, 52, + 47, 49, 51, 50, 50, 48, 46, 45, 43, 41, 39, 39, 62, 57, 52, 48, 45, 46, + 47, 47, 47, 45, 44, 43, 41, 40, 38, 38, 48, 48, 47, 45, 42, 43, 43, 43, + 43, 43, 42, 41, 40, 38, 37, 37, 47, 48, 49, 46, 43, 42, 42, 41, 41, 40, + 40, 39, 38, 37, 36, 36, 46, 48, 51, 47, 43, 42, 40, 39, 39, 38, 38, 37, + 36, 35, 35, 35, 45, 48, 50, 47, 43, 41, 39, 38, 37, 37, 36, 35, 35, 34, + 33, 33, 44, 47, 50, 47, 43, 41, 39, 37, 36, 35, 34, 34, 33, 33, 32, 32, + 42, 45, 48, 45, 43, 40, 38, 37, 35, 34, 34, 33, 32, 32, 31, 31, 41, 44, + 46, 44, 42, 40, 38, 36, 34, 34, 33, 32, 31, 31, 30, 30, 39, 42, 45, 43, + 41, 39, 37, 35, 34, 33, 32, 31, 31, 30, 30, 30, 37, 40, 43, 41, 40, 38, + 36, 35, 33, 32, 31, 31, 30, 29, 29, 29, 36, 39, 41, 40, 38, 37, 35, 34, + 33, 32, 31, 30, 29, 29, 28, 28, 35, 37, 39, 38, 37, 36, 35, 33, 32, 31, + 30, 30, 29, 28, 28, 28, 35, 37, 39, 38, 37, 36, 35, 33, 32, 31, 30, 30, + 29, 28, 28, 28, + /* Size 32 */ + 64, 67, 70, 73, 76, 69, 62, 55, 48, 47, 47, 46, 46, 45, 45, 44, 44, 43, + 42, 41, 41, 40, 39, 38, 37, 37, 36, 35, 35, 35, 35, 35, 67, 68, 69, 70, + 71, 65, 60, 54, 48, 48, 47, 47, 47, 47, 46, 46, 45, 44, 44, 43, 42, 41, + 40, 40, 39, 38, 37, 37, 36, 36, 36, 36, 70, 69, 68, 67, 66, 62, 57, 52, + 48, 48, 48, 48, 48, 48, 48, 47, 47, 46, 45, 44, 44, 43, 42, 41, 40, 39, + 39, 38, 37, 37, 37, 37, 73, 70, 67, 65, 62, 58, 55, 51, 47, 48, 49, 49, + 50, 49, 49, 49, 48, 47, 47, 46, 45, 44, 43, 42, 41, 41, 40, 39, 38, 38, + 38, 38, 76, 71, 66, 62, 57, 54, 52, 50, 47, 48, 49, 50, 51, 51, 50, 50, + 50, 49, 48, 47, 46, 46, 45, 44, 43, 42, 41, 40, 39, 39, 39, 39, 69, 65, + 62, 58, 54, 52, 50, 48, 46, 47, 47, 48, 49, 49, 49, 48, 48, 47, 47, 46, + 45, 44, 44, 43, 42, 41, 40, 39, 39, 39, 39, 39, 62, 60, 57, 55, 52, 50, + 48, 47, 45, 45, 46, 46, 47, 47, 47, 47, 47, 46, 45, 45, 44, 43, 43, 42, + 41, 40, 40, 39, 38, 38, 38, 38, 55, 54, 52, 51, 50, 48, 47, 45, 44, 44, + 44, 45, 45, 45, 45, 45, 45, 45, 44, 44, 43, 42, 42, 41, 40, 40, 39, 38, + 38, 38, 38, 38, 48, 48, 48, 47, 47, 46, 45, 44, 42, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 42, 42, 41, 41, 40, 40, 39, 38, 38, 37, 37, 37, 37, + 47, 48, 48, 48, 48, 47, 45, 44, 43, 43, 43, 42, 42, 42, 42, 42, 42, 42, + 42, 41, 41, 40, 40, 39, 39, 38, 38, 37, 36, 36, 36, 36, 47, 47, 48, 49, + 49, 47, 46, 44, 43, 43, 42, 42, 42, 41, 41, 41, 41, 41, 40, 40, 40, 39, + 39, 38, 38, 37, 37, 36, 36, 36, 36, 36, 46, 47, 48, 49, 50, 48, 46, 45, + 43, 42, 42, 41, 41, 40, 40, 40, 40, 40, 39, 39, 39, 38, 38, 37, 37, 37, + 36, 36, 35, 35, 35, 35, 46, 47, 48, 50, 51, 49, 47, 45, 43, 42, 42, 41, + 40, 40, 39, 39, 39, 38, 38, 38, 38, 37, 37, 37, 36, 36, 35, 35, 35, 35, + 35, 35, 45, 47, 48, 49, 51, 49, 47, 45, 43, 42, 41, 40, 40, 39, 39, 38, + 38, 38, 37, 37, 37, 36, 36, 36, 35, 35, 35, 34, 34, 34, 34, 34, 45, 46, + 48, 49, 50, 49, 47, 45, 43, 42, 41, 40, 39, 39, 38, 38, 37, 37, 37, 36, + 36, 36, 35, 35, 35, 34, 34, 34, 33, 33, 33, 33, 44, 46, 47, 49, 50, 48, + 47, 45, 43, 42, 41, 40, 39, 38, 38, 37, 37, 36, 36, 36, 35, 35, 35, 34, + 34, 34, 33, 33, 33, 33, 33, 33, 44, 45, 47, 48, 50, 48, 47, 45, 43, 42, + 41, 40, 39, 38, 37, 37, 36, 36, 35, 35, 34, 34, 34, 34, 33, 33, 33, 32, + 32, 32, 32, 32, 43, 44, 46, 47, 49, 47, 46, 45, 43, 42, 41, 40, 38, 38, + 37, 36, 36, 35, 35, 34, 34, 34, 33, 33, 33, 33, 32, 32, 32, 32, 32, 32, + 42, 44, 45, 47, 48, 47, 45, 44, 43, 42, 40, 39, 38, 37, 37, 36, 35, 35, + 34, 34, 34, 33, 33, 33, 32, 32, 32, 32, 31, 31, 31, 31, 41, 43, 44, 46, + 47, 46, 45, 44, 42, 41, 40, 39, 38, 37, 36, 36, 35, 34, 34, 33, 33, 33, + 32, 32, 32, 32, 31, 31, 31, 31, 31, 31, 41, 42, 44, 45, 46, 45, 44, 43, + 42, 41, 40, 39, 38, 37, 36, 35, 34, 34, 34, 33, 33, 32, 32, 32, 31, 31, + 31, 31, 30, 30, 30, 30, 40, 41, 43, 44, 46, 44, 43, 42, 41, 40, 39, 38, + 37, 36, 36, 35, 34, 34, 33, 33, 32, 32, 32, 31, 31, 31, 30, 30, 30, 30, + 30, 30, 39, 40, 42, 43, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 35, + 34, 33, 33, 32, 32, 32, 31, 31, 31, 30, 30, 30, 30, 30, 30, 30, 38, 40, + 41, 42, 44, 43, 42, 41, 40, 39, 38, 37, 37, 36, 35, 34, 34, 33, 33, 32, + 32, 31, 31, 31, 30, 30, 30, 30, 29, 29, 29, 29, 37, 39, 40, 41, 43, 42, + 41, 40, 40, 39, 38, 37, 36, 35, 35, 34, 33, 33, 32, 32, 31, 31, 31, 30, + 30, 30, 29, 29, 29, 29, 29, 29, 37, 38, 39, 41, 42, 41, 40, 40, 39, 38, + 37, 37, 36, 35, 34, 34, 33, 33, 32, 32, 31, 31, 30, 30, 30, 29, 29, 29, + 29, 29, 29, 29, 36, 37, 39, 40, 41, 40, 40, 39, 38, 38, 37, 36, 35, 35, + 34, 33, 33, 32, 32, 31, 31, 30, 30, 30, 29, 29, 29, 29, 28, 28, 28, 28, + 35, 37, 38, 39, 40, 39, 39, 38, 38, 37, 36, 36, 35, 34, 34, 33, 32, 32, + 32, 31, 31, 30, 30, 30, 29, 29, 29, 28, 28, 28, 28, 28, 35, 36, 37, 38, + 39, 39, 38, 38, 37, 36, 36, 35, 35, 34, 33, 33, 32, 32, 31, 31, 30, 30, + 30, 29, 29, 29, 28, 28, 28, 28, 28, 28, 35, 36, 37, 38, 39, 39, 38, 38, + 37, 36, 36, 35, 35, 34, 33, 33, 32, 32, 31, 31, 30, 30, 30, 29, 29, 29, + 28, 28, 28, 28, 28, 28, 35, 36, 37, 38, 39, 39, 38, 38, 37, 36, 36, 35, + 35, 34, 33, 33, 32, 32, 31, 31, 30, 30, 30, 29, 29, 29, 28, 28, 28, 28, + 28, 28, 35, 36, 37, 38, 39, 39, 38, 38, 37, 36, 36, 35, 35, 34, 33, 33, + 32, 32, 31, 31, 30, 30, 30, 29, 29, 29, 28, 28, 28, 28, 28, 28 }, + { /* Intra matrices */ + /* Size 4 */ + 117, 81, 76, 64, 81, 71, 67, 61, 76, 67, 57, 52, 64, 61, 52, 47, + /* Size 8 */ + 110, 131, 81, 77, 74, 68, 62, 57, 131, 97, 80, 86, 84, 78, 72, 65, 81, + 80, 71, 73, 73, 70, 66, 62, 77, 86, 73, 67, 64, 63, 60, 57, 74, 84, 73, + 64, 60, 57, 55, 53, 68, 78, 70, 63, 57, 54, 51, 50, 62, 72, 66, 60, 55, + 51, 49, 47, 57, 65, 62, 57, 53, 50, 47, 45, + /* Size 16 */ + 112, 123, 134, 108, 83, 81, 79, 77, 75, 72, 69, 66, 64, 61, 59, 59, 123, + 120, 116, 99, 82, 83, 83, 82, 81, 78, 75, 71, 68, 65, 63, 63, 134, 116, + 99, 90, 81, 85, 88, 87, 86, 83, 80, 77, 73, 70, 67, 67, 108, 99, 90, 84, + 77, 79, 81, 81, 80, 78, 76, 73, 70, 68, 65, 65, 83, 82, 81, 77, 72, 73, + 74, 74, 74, 73, 72, 70, 67, 65, 63, 63, 81, 83, 85, 79, 73, 72, 71, 71, + 70, 69, 68, 66, 64, 62, 61, 61, 79, 83, 88, 81, 74, 71, 68, 67, 66, 65, + 64, 62, 61, 60, 58, 58, 77, 82, 87, 81, 74, 71, 67, 65, 63, 62, 61, 60, + 59, 57, 56, 56, 75, 81, 86, 80, 74, 70, 66, 63, 61, 59, 58, 57, 56, 55, + 54, 54, 72, 78, 83, 78, 73, 69, 65, 62, 59, 58, 56, 55, 54, 53, 52, 52, + 69, 75, 80, 76, 72, 68, 64, 61, 58, 56, 55, 54, 52, 51, 51, 51, 66, 71, + 77, 73, 70, 66, 62, 60, 57, 55, 54, 52, 51, 50, 49, 49, 64, 68, 73, 70, + 67, 64, 61, 59, 56, 54, 52, 51, 50, 49, 48, 48, 61, 65, 70, 68, 65, 62, + 60, 57, 55, 53, 51, 50, 49, 48, 47, 47, 59, 63, 67, 65, 63, 61, 58, 56, + 54, 52, 51, 49, 48, 47, 46, 46, 59, 63, 67, 65, 63, 61, 58, 56, 54, 52, + 51, 49, 48, 47, 46, 46, + /* Size 32 */ + 113, 119, 124, 130, 135, 122, 109, 96, 83, 82, 82, 81, 80, 79, 78, 77, + 76, 74, 73, 71, 70, 69, 67, 66, 64, 63, 62, 60, 59, 59, 59, 59, 119, + 121, 123, 124, 126, 116, 105, 94, 83, 83, 83, 82, 82, 81, 80, 79, 79, + 77, 76, 74, 73, 71, 70, 68, 67, 65, 64, 63, 61, 61, 61, 61, 124, 123, + 121, 119, 118, 109, 100, 91, 83, 83, 84, 84, 84, 84, 83, 82, 81, 80, 78, + 77, 75, 74, 72, 71, 69, 68, 66, 65, 63, 63, 63, 63, 130, 124, 119, 114, + 109, 102, 96, 89, 83, 84, 84, 85, 86, 86, 85, 85, 84, 83, 81, 80, 78, + 76, 75, 73, 71, 70, 68, 67, 65, 65, 65, 65, 135, 126, 118, 109, 100, 95, + 91, 87, 82, 84, 85, 87, 89, 88, 88, 87, 87, 85, 84, 82, 81, 79, 77, 76, + 74, 72, 71, 69, 67, 67, 67, 67, 122, 116, 109, 102, 95, 92, 88, 84, 80, + 81, 83, 84, 85, 85, 85, 84, 84, 83, 81, 80, 79, 77, 76, 74, 72, 71, 69, + 68, 66, 66, 66, 66, 109, 105, 100, 96, 91, 88, 84, 81, 78, 79, 80, 81, + 82, 82, 81, 81, 81, 80, 79, 78, 77, 75, 74, 72, 71, 70, 68, 67, 65, 65, + 65, 65, 96, 94, 91, 89, 87, 84, 81, 78, 75, 76, 77, 78, 78, 78, 78, 78, + 78, 77, 76, 75, 74, 73, 72, 71, 70, 68, 67, 66, 65, 65, 65, 65, 83, 83, + 83, 83, 82, 80, 78, 75, 73, 74, 74, 75, 75, 75, 75, 75, 75, 74, 74, 73, + 72, 71, 70, 69, 68, 67, 66, 65, 64, 64, 64, 64, 82, 83, 83, 84, 84, 81, + 79, 76, 74, 74, 73, 73, 73, 73, 73, 73, 73, 72, 72, 71, 70, 69, 68, 68, + 67, 66, 64, 63, 62, 62, 62, 62, 82, 83, 84, 84, 85, 83, 80, 77, 74, 73, + 73, 72, 72, 72, 71, 71, 71, 70, 70, 69, 68, 68, 67, 66, 65, 64, 63, 62, + 61, 61, 61, 61, 81, 82, 84, 85, 87, 84, 81, 78, 75, 73, 72, 71, 70, 70, + 69, 69, 69, 68, 67, 67, 66, 66, 65, 64, 63, 63, 62, 61, 60, 60, 60, 60, + 80, 82, 84, 86, 89, 85, 82, 78, 75, 73, 72, 70, 69, 68, 67, 67, 66, 66, + 65, 65, 64, 64, 63, 62, 62, 61, 60, 60, 59, 59, 59, 59, 79, 81, 84, 86, + 88, 85, 82, 78, 75, 73, 72, 70, 68, 67, 67, 66, 65, 65, 64, 63, 63, 62, + 62, 61, 60, 60, 59, 58, 58, 58, 58, 58, 78, 80, 83, 85, 88, 85, 81, 78, + 75, 73, 71, 69, 67, 67, 66, 65, 64, 63, 63, 62, 62, 61, 60, 60, 59, 59, + 58, 57, 57, 57, 57, 57, 77, 79, 82, 85, 87, 84, 81, 78, 75, 73, 71, 69, + 67, 66, 65, 64, 63, 62, 61, 61, 60, 60, 59, 58, 58, 57, 57, 56, 56, 56, + 56, 56, 76, 79, 81, 84, 87, 84, 81, 78, 75, 73, 71, 69, 66, 65, 64, 63, + 61, 61, 60, 59, 59, 58, 58, 57, 57, 56, 56, 55, 54, 54, 54, 54, 74, 77, + 80, 83, 85, 83, 80, 77, 74, 72, 70, 68, 66, 65, 63, 62, 61, 60, 59, 59, + 58, 57, 57, 56, 56, 55, 55, 54, 54, 54, 54, 54, 73, 76, 78, 81, 84, 81, + 79, 76, 74, 72, 70, 67, 65, 64, 63, 61, 60, 59, 59, 58, 57, 56, 56, 55, + 55, 54, 54, 53, 53, 53, 53, 53, 71, 74, 77, 80, 82, 80, 78, 75, 73, 71, + 69, 67, 65, 63, 62, 61, 59, 59, 58, 57, 56, 56, 55, 54, 54, 53, 53, 52, + 52, 52, 52, 52, 70, 73, 75, 78, 81, 79, 77, 74, 72, 70, 68, 66, 64, 63, + 62, 60, 59, 58, 57, 56, 55, 55, 54, 53, 53, 52, 52, 51, 51, 51, 51, 51, + 69, 71, 74, 76, 79, 77, 75, 73, 71, 69, 68, 66, 64, 62, 61, 60, 58, 57, + 56, 56, 55, 54, 53, 53, 52, 52, 51, 51, 50, 50, 50, 50, 67, 70, 72, 75, + 77, 76, 74, 72, 70, 68, 67, 65, 63, 62, 60, 59, 58, 57, 56, 55, 54, 53, + 53, 52, 52, 51, 51, 50, 50, 50, 50, 50, 66, 68, 71, 73, 76, 74, 72, 71, + 69, 68, 66, 64, 62, 61, 60, 58, 57, 56, 55, 54, 53, 53, 52, 52, 51, 51, + 50, 50, 49, 49, 49, 49, 64, 67, 69, 71, 74, 72, 71, 70, 68, 67, 65, 63, + 62, 60, 59, 58, 57, 56, 55, 54, 53, 52, 52, 51, 50, 50, 49, 49, 49, 49, + 49, 49, 63, 65, 68, 70, 72, 71, 70, 68, 67, 66, 64, 63, 61, 60, 59, 57, + 56, 55, 54, 53, 52, 52, 51, 51, 50, 49, 49, 49, 48, 48, 48, 48, 62, 64, + 66, 68, 71, 69, 68, 67, 66, 64, 63, 62, 60, 59, 58, 57, 56, 55, 54, 53, + 52, 51, 51, 50, 49, 49, 49, 48, 48, 48, 48, 48, 60, 63, 65, 67, 69, 68, + 67, 66, 65, 63, 62, 61, 60, 58, 57, 56, 55, 54, 53, 52, 51, 51, 50, 50, + 49, 49, 48, 48, 47, 47, 47, 47, 59, 61, 63, 65, 67, 66, 65, 65, 64, 62, + 61, 60, 59, 58, 57, 56, 54, 54, 53, 52, 51, 50, 50, 49, 49, 48, 48, 47, + 47, 47, 47, 47, 59, 61, 63, 65, 67, 66, 65, 65, 64, 62, 61, 60, 59, 58, + 57, 56, 54, 54, 53, 52, 51, 50, 50, 49, 49, 48, 48, 47, 47, 47, 47, 47, + 59, 61, 63, 65, 67, 66, 65, 65, 64, 62, 61, 60, 59, 58, 57, 56, 54, 54, + 53, 52, 51, 50, 50, 49, 49, 48, 48, 47, 47, 47, 47, 47, 59, 61, 63, 65, + 67, 66, 65, 65, 64, 62, 61, 60, 59, 58, 57, 56, 54, 54, 53, 52, 51, 50, + 50, 49, 49, 48, 48, 47, 47, 47, 47, 47 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 60, 43, 33, 60, 45, 37, 32, 43, 37, 31, 29, 33, 32, 29, 27, + /* Size 8 */ + 64, 79, 74, 61, 50, 43, 38, 35, 79, 71, 73, 66, 55, 47, 42, 38, 74, 73, + 58, 52, 48, 43, 39, 36, 61, 66, 52, 45, 42, 39, 37, 35, 50, 55, 48, 42, + 38, 36, 34, 33, 43, 47, 43, 39, 36, 34, 33, 32, 38, 42, 39, 37, 34, 33, + 32, 31, 35, 38, 36, 35, 33, 32, 31, 31, + /* Size 16 */ + 64, 71, 79, 77, 74, 68, 61, 56, 50, 47, 43, 41, 38, 37, 35, 35, 71, 73, + 75, 74, 74, 69, 63, 58, 53, 49, 45, 43, 40, 38, 37, 37, 79, 75, 71, 72, + 73, 70, 66, 60, 55, 51, 47, 45, 42, 40, 38, 38, 77, 74, 72, 69, 66, 62, + 59, 55, 51, 48, 45, 43, 40, 39, 37, 37, 74, 74, 73, 66, 58, 55, 52, 50, + 48, 45, 43, 41, 39, 38, 36, 36, 68, 69, 70, 62, 55, 52, 49, 47, 45, 43, + 41, 39, 38, 37, 36, 36, 61, 63, 66, 59, 52, 49, 45, 43, 42, 40, 39, 38, + 37, 36, 35, 35, 56, 58, 60, 55, 50, 47, 43, 42, 40, 39, 37, 36, 35, 35, + 34, 34, 50, 53, 55, 51, 48, 45, 42, 40, 38, 37, 36, 35, 34, 34, 33, 33, + 47, 49, 51, 48, 45, 43, 40, 39, 37, 36, 35, 34, 34, 33, 33, 33, 43, 45, + 47, 45, 43, 41, 39, 37, 36, 35, 34, 33, 33, 32, 32, 32, 41, 43, 45, 43, + 41, 39, 38, 36, 35, 34, 33, 33, 32, 32, 32, 32, 38, 40, 42, 40, 39, 38, + 37, 35, 34, 34, 33, 32, 32, 31, 31, 31, 37, 38, 40, 39, 38, 37, 36, 35, + 34, 33, 32, 32, 31, 31, 31, 31, 35, 37, 38, 37, 36, 36, 35, 34, 33, 33, + 32, 32, 31, 31, 31, 31, 35, 37, 38, 37, 36, 36, 35, 34, 33, 33, 32, 32, + 31, 31, 31, 31, + /* Size 32 */ + 64, 68, 71, 75, 79, 78, 77, 75, 74, 71, 68, 64, 61, 58, 56, 53, 50, 49, + 47, 45, 43, 42, 41, 40, 38, 38, 37, 36, 35, 35, 35, 35, 68, 70, 72, 75, + 77, 76, 76, 75, 74, 71, 68, 65, 62, 59, 57, 54, 52, 50, 48, 46, 44, 43, + 42, 40, 39, 38, 38, 37, 36, 36, 36, 36, 71, 72, 73, 74, 75, 75, 74, 74, + 74, 71, 69, 66, 63, 61, 58, 55, 53, 51, 49, 47, 45, 44, 43, 41, 40, 39, + 38, 37, 37, 37, 37, 37, 75, 75, 74, 74, 73, 73, 73, 74, 74, 71, 69, 67, + 64, 62, 59, 57, 54, 52, 50, 48, 46, 45, 44, 42, 41, 40, 39, 38, 37, 37, + 37, 37, 79, 77, 75, 73, 71, 72, 72, 73, 73, 72, 70, 68, 66, 63, 60, 58, + 55, 53, 51, 49, 47, 46, 45, 43, 42, 41, 40, 39, 38, 38, 38, 38, 78, 76, + 75, 73, 72, 71, 71, 70, 70, 68, 66, 64, 62, 60, 58, 56, 53, 52, 50, 48, + 46, 45, 44, 42, 41, 40, 39, 38, 37, 37, 37, 37, 77, 76, 74, 73, 72, 71, + 69, 67, 66, 64, 62, 61, 59, 57, 55, 53, 51, 50, 48, 47, 45, 44, 43, 42, + 40, 40, 39, 38, 37, 37, 37, 37, 75, 75, 74, 74, 73, 70, 67, 65, 62, 60, + 59, 57, 56, 54, 53, 51, 49, 48, 47, 45, 44, 43, 42, 41, 40, 39, 38, 37, + 37, 37, 37, 37, 74, 74, 74, 74, 73, 70, 66, 62, 58, 56, 55, 54, 52, 51, + 50, 49, 48, 46, 45, 44, 43, 42, 41, 40, 39, 39, 38, 37, 36, 36, 36, 36, + 71, 71, 71, 71, 72, 68, 64, 60, 56, 55, 54, 52, 51, 49, 48, 47, 46, 45, + 44, 43, 42, 41, 40, 39, 39, 38, 37, 37, 36, 36, 36, 36, 68, 68, 69, 69, + 70, 66, 62, 59, 55, 54, 52, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, + 39, 39, 38, 37, 37, 36, 36, 36, 36, 36, 64, 65, 66, 67, 68, 64, 61, 57, + 54, 52, 50, 49, 47, 46, 45, 44, 43, 42, 41, 41, 40, 39, 39, 38, 37, 37, + 36, 36, 35, 35, 35, 35, 61, 62, 63, 64, 66, 62, 59, 56, 52, 51, 49, 47, + 45, 44, 43, 42, 42, 41, 40, 39, 39, 38, 38, 37, 37, 36, 36, 35, 35, 35, + 35, 35, 58, 59, 61, 62, 63, 60, 57, 54, 51, 49, 48, 46, 44, 43, 42, 42, + 41, 40, 39, 39, 38, 38, 37, 37, 36, 36, 35, 35, 34, 34, 34, 34, 56, 57, + 58, 59, 60, 58, 55, 53, 50, 48, 47, 45, 43, 42, 42, 41, 40, 39, 39, 38, + 37, 37, 36, 36, 35, 35, 35, 34, 34, 34, 34, 34, 53, 54, 55, 57, 58, 56, + 53, 51, 49, 47, 46, 44, 42, 42, 41, 40, 39, 38, 38, 37, 37, 36, 36, 35, + 35, 35, 34, 34, 34, 34, 34, 34, 50, 52, 53, 54, 55, 53, 51, 49, 48, 46, + 45, 43, 42, 41, 40, 39, 38, 38, 37, 36, 36, 36, 35, 35, 34, 34, 34, 33, + 33, 33, 33, 33, 49, 50, 51, 52, 53, 52, 50, 48, 46, 45, 44, 42, 41, 40, + 39, 38, 38, 37, 36, 36, 35, 35, 35, 34, 34, 34, 33, 33, 33, 33, 33, 33, + 47, 48, 49, 50, 51, 50, 48, 47, 45, 44, 43, 41, 40, 39, 39, 38, 37, 36, + 36, 36, 35, 35, 34, 34, 34, 33, 33, 33, 33, 33, 33, 33, 45, 46, 47, 48, + 49, 48, 47, 45, 44, 43, 42, 41, 39, 39, 38, 37, 36, 36, 36, 35, 35, 34, + 34, 34, 33, 33, 33, 33, 32, 32, 32, 32, 43, 44, 45, 46, 47, 46, 45, 44, + 43, 42, 41, 40, 39, 38, 37, 37, 36, 35, 35, 35, 34, 34, 33, 33, 33, 33, + 32, 32, 32, 32, 32, 32, 42, 43, 44, 45, 46, 45, 44, 43, 42, 41, 40, 39, + 38, 38, 37, 36, 36, 35, 35, 34, 34, 33, 33, 33, 33, 32, 32, 32, 32, 32, + 32, 32, 41, 42, 43, 44, 45, 44, 43, 42, 41, 40, 39, 39, 38, 37, 36, 36, + 35, 35, 34, 34, 33, 33, 33, 33, 32, 32, 32, 32, 32, 32, 32, 32, 40, 40, + 41, 42, 43, 42, 42, 41, 40, 39, 39, 38, 37, 37, 36, 35, 35, 34, 34, 34, + 33, 33, 33, 32, 32, 32, 32, 32, 31, 31, 31, 31, 38, 39, 40, 41, 42, 41, + 40, 40, 39, 39, 38, 37, 37, 36, 35, 35, 34, 34, 34, 33, 33, 33, 32, 32, + 32, 32, 31, 31, 31, 31, 31, 31, 38, 38, 39, 40, 41, 40, 40, 39, 39, 38, + 37, 37, 36, 36, 35, 35, 34, 34, 33, 33, 33, 32, 32, 32, 32, 31, 31, 31, + 31, 31, 31, 31, 37, 38, 38, 39, 40, 39, 39, 38, 38, 37, 37, 36, 36, 35, + 35, 34, 34, 33, 33, 33, 32, 32, 32, 32, 31, 31, 31, 31, 31, 31, 31, 31, + 36, 37, 37, 38, 39, 38, 38, 37, 37, 37, 36, 36, 35, 35, 34, 34, 33, 33, + 33, 33, 32, 32, 32, 32, 31, 31, 31, 31, 31, 31, 31, 31, 35, 36, 37, 37, + 38, 37, 37, 37, 36, 36, 36, 35, 35, 34, 34, 34, 33, 33, 33, 32, 32, 32, + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 35, 36, 37, 37, 38, 37, 37, 37, + 36, 36, 36, 35, 35, 34, 34, 34, 33, 33, 33, 32, 32, 32, 32, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 35, 36, 37, 37, 38, 37, 37, 37, 36, 36, 36, 35, + 35, 34, 34, 34, 33, 33, 33, 32, 32, 32, 32, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 35, 36, 37, 37, 38, 37, 37, 37, 36, 36, 36, 35, 35, 34, 34, 34, + 33, 33, 33, 32, 32, 32, 32, 31, 31, 31, 31, 31, 31, 31, 31, 31 }, + { /* Intra matrices */ + /* Size 4 */ + 118, 110, 77, 59, 110, 82, 67, 57, 77, 67, 56, 51, 59, 57, 51, 47, + /* Size 8 */ + 103, 128, 121, 98, 80, 68, 60, 55, 128, 115, 119, 106, 89, 75, 65, 59, + 121, 119, 93, 83, 75, 68, 61, 56, 98, 106, 83, 71, 65, 61, 57, 54, 80, + 89, 75, 65, 59, 56, 53, 51, 68, 75, 68, 61, 56, 53, 51, 49, 60, 65, 61, + 57, 53, 51, 49, 48, 55, 59, 56, 54, 51, 49, 48, 47, + /* Size 16 */ + 106, 118, 131, 127, 123, 112, 100, 91, 82, 76, 69, 65, 61, 59, 56, 56, + 118, 122, 125, 124, 123, 113, 104, 95, 86, 80, 73, 69, 64, 61, 58, 58, + 131, 125, 118, 120, 122, 115, 108, 99, 91, 84, 77, 72, 67, 64, 60, 60, + 127, 124, 120, 114, 108, 103, 97, 90, 84, 78, 73, 69, 65, 62, 59, 59, + 123, 123, 122, 108, 95, 90, 85, 81, 77, 73, 69, 66, 63, 60, 58, 58, 112, + 113, 115, 103, 90, 85, 79, 76, 72, 69, 66, 63, 60, 58, 56, 56, 100, 104, + 108, 97, 85, 79, 73, 70, 67, 64, 62, 60, 58, 56, 55, 55, 91, 95, 99, 90, + 81, 76, 70, 67, 64, 62, 60, 58, 56, 55, 54, 54, 82, 86, 91, 84, 77, 72, + 67, 64, 61, 59, 57, 56, 54, 53, 52, 52, 76, 80, 84, 78, 73, 69, 64, 62, + 59, 57, 55, 54, 53, 52, 51, 51, 69, 73, 77, 73, 69, 66, 62, 60, 57, 55, + 54, 53, 52, 51, 50, 50, 65, 69, 72, 69, 66, 63, 60, 58, 56, 54, 53, 52, + 51, 50, 49, 49, 61, 64, 67, 65, 63, 60, 58, 56, 54, 53, 52, 51, 50, 49, + 49, 49, 59, 61, 64, 62, 60, 58, 56, 55, 53, 52, 51, 50, 49, 49, 48, 48, + 56, 58, 60, 59, 58, 56, 55, 54, 52, 51, 50, 49, 49, 48, 48, 48, 56, 58, + 60, 59, 58, 56, 55, 54, 52, 51, 50, 49, 49, 48, 48, 48, + /* Size 32 */ + 107, 113, 120, 126, 133, 131, 129, 127, 125, 119, 113, 107, 102, 97, 92, + 88, 83, 80, 77, 73, 70, 68, 66, 64, 62, 61, 59, 58, 57, 57, 57, 57, 113, + 117, 121, 125, 129, 128, 127, 126, 125, 119, 114, 109, 104, 99, 94, 90, + 85, 82, 79, 75, 72, 70, 68, 66, 64, 62, 61, 59, 58, 58, 58, 58, 120, + 121, 123, 125, 126, 126, 125, 125, 124, 119, 115, 110, 105, 101, 96, 92, + 87, 84, 81, 77, 74, 72, 69, 67, 65, 63, 62, 60, 59, 59, 59, 59, 126, + 125, 125, 124, 123, 123, 123, 124, 124, 120, 116, 112, 107, 103, 98, 94, + 90, 86, 83, 79, 76, 73, 71, 69, 66, 65, 63, 61, 60, 60, 60, 60, 133, + 129, 126, 123, 119, 120, 121, 122, 123, 120, 116, 113, 109, 105, 101, + 96, 92, 88, 85, 81, 78, 75, 73, 70, 68, 66, 64, 63, 61, 61, 61, 61, 131, + 128, 126, 123, 120, 119, 118, 118, 117, 113, 110, 107, 104, 100, 96, 92, + 88, 85, 82, 79, 76, 74, 71, 69, 67, 65, 64, 62, 60, 60, 60, 60, 129, + 127, 125, 123, 121, 118, 116, 113, 110, 107, 104, 101, 98, 95, 91, 88, + 85, 82, 79, 77, 74, 72, 70, 68, 66, 64, 63, 61, 60, 60, 60, 60, 127, + 126, 125, 124, 122, 118, 113, 108, 103, 100, 98, 95, 92, 89, 87, 84, 81, + 79, 77, 74, 72, 70, 68, 66, 65, 63, 62, 60, 59, 59, 59, 59, 125, 125, + 124, 124, 123, 117, 110, 103, 96, 94, 91, 89, 86, 84, 82, 80, 78, 76, + 74, 72, 70, 68, 67, 65, 63, 62, 61, 60, 58, 58, 58, 58, 119, 119, 119, + 120, 120, 113, 107, 100, 94, 91, 88, 86, 83, 81, 79, 77, 75, 74, 72, 70, + 68, 67, 65, 64, 62, 61, 60, 59, 58, 58, 58, 58, 113, 114, 115, 116, 116, + 110, 104, 98, 91, 88, 86, 83, 80, 78, 76, 75, 73, 71, 70, 68, 66, 65, + 64, 62, 61, 60, 59, 58, 57, 57, 57, 57, 107, 109, 110, 112, 113, 107, + 101, 95, 89, 86, 83, 80, 77, 75, 74, 72, 70, 69, 67, 66, 65, 63, 62, 61, + 60, 59, 58, 57, 56, 56, 56, 56, 102, 104, 105, 107, 109, 104, 98, 92, + 86, 83, 80, 77, 74, 72, 71, 69, 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, + 57, 56, 55, 55, 55, 55, 97, 99, 101, 103, 105, 100, 95, 89, 84, 81, 78, + 75, 72, 71, 69, 68, 66, 65, 64, 63, 61, 61, 60, 59, 58, 57, 56, 56, 55, + 55, 55, 55, 92, 94, 96, 98, 101, 96, 91, 87, 82, 79, 76, 74, 71, 69, 68, + 66, 64, 63, 62, 61, 60, 59, 59, 58, 57, 56, 56, 55, 54, 54, 54, 54, 88, + 90, 92, 94, 96, 92, 88, 84, 80, 77, 75, 72, 69, 68, 66, 64, 63, 62, 61, + 60, 59, 58, 57, 57, 56, 55, 55, 54, 53, 53, 53, 53, 83, 85, 87, 90, 92, + 88, 85, 81, 78, 75, 73, 70, 67, 66, 64, 63, 61, 60, 60, 59, 58, 57, 56, + 56, 55, 54, 54, 53, 53, 53, 53, 53, 80, 82, 84, 86, 88, 85, 82, 79, 76, + 74, 71, 69, 66, 65, 63, 62, 60, 60, 59, 58, 57, 56, 56, 55, 54, 54, 53, + 53, 52, 52, 52, 52, 77, 79, 81, 83, 85, 82, 79, 77, 74, 72, 70, 67, 65, + 64, 62, 61, 60, 59, 58, 57, 56, 55, 55, 54, 54, 53, 53, 52, 52, 52, 52, + 52, 73, 75, 77, 79, 81, 79, 77, 74, 72, 70, 68, 66, 64, 63, 61, 60, 59, + 58, 57, 56, 55, 55, 54, 54, 53, 53, 52, 52, 51, 51, 51, 51, 70, 72, 74, + 76, 78, 76, 74, 72, 70, 68, 66, 65, 63, 61, 60, 59, 58, 57, 56, 55, 54, + 54, 53, 53, 52, 52, 52, 51, 51, 51, 51, 51, 68, 70, 72, 73, 75, 74, 72, + 70, 68, 67, 65, 63, 62, 61, 59, 58, 57, 56, 55, 55, 54, 53, 53, 52, 52, + 52, 51, 51, 50, 50, 50, 50, 66, 68, 69, 71, 73, 71, 70, 68, 67, 65, 64, + 62, 61, 60, 59, 57, 56, 56, 55, 54, 53, 53, 52, 52, 51, 51, 51, 50, 50, + 50, 50, 50, 64, 66, 67, 69, 70, 69, 68, 66, 65, 64, 62, 61, 60, 59, 58, + 57, 56, 55, 54, 54, 53, 52, 52, 51, 51, 51, 50, 50, 50, 50, 50, 50, 62, + 64, 65, 66, 68, 67, 66, 65, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 54, + 53, 52, 52, 51, 51, 51, 50, 50, 50, 49, 49, 49, 49, 61, 62, 63, 65, 66, + 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 54, 53, 53, 52, 52, 51, + 51, 50, 50, 50, 49, 49, 49, 49, 49, 59, 61, 62, 63, 64, 64, 63, 62, 61, + 60, 59, 58, 57, 56, 56, 55, 54, 53, 53, 52, 52, 51, 51, 50, 50, 50, 49, + 49, 49, 49, 49, 49, 58, 59, 60, 61, 63, 62, 61, 60, 60, 59, 58, 57, 56, + 56, 55, 54, 53, 53, 52, 52, 51, 51, 50, 50, 50, 49, 49, 49, 49, 49, 49, + 49, 57, 58, 59, 60, 61, 60, 60, 59, 58, 58, 57, 56, 55, 55, 54, 53, 53, + 52, 52, 51, 51, 50, 50, 50, 49, 49, 49, 49, 48, 48, 48, 48, 57, 58, 59, + 60, 61, 60, 60, 59, 58, 58, 57, 56, 55, 55, 54, 53, 53, 52, 52, 51, 51, + 50, 50, 50, 49, 49, 49, 49, 48, 48, 48, 48, 57, 58, 59, 60, 61, 60, 60, + 59, 58, 58, 57, 56, 55, 55, 54, 53, 53, 52, 52, 51, 51, 50, 50, 50, 49, + 49, 49, 49, 48, 48, 48, 48, 57, 58, 59, 60, 61, 60, 60, 59, 58, 58, 57, + 56, 55, 55, 54, 53, 53, 52, 52, 51, 51, 50, 50, 50, 49, 49, 49, 49, 48, + 48, 48, 48 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 46, 44, 38, 46, 41, 40, 37, 44, 40, 35, 32, 38, 37, 32, 30, + /* Size 8 */ + 64, 75, 49, 47, 45, 42, 40, 37, 75, 57, 49, 52, 51, 48, 44, 41, 49, 49, + 44, 45, 45, 44, 42, 39, 47, 52, 45, 42, 41, 40, 38, 37, 45, 51, 45, 41, + 38, 37, 36, 35, 42, 48, 44, 40, 37, 35, 34, 33, 40, 44, 42, 38, 36, 34, + 33, 32, 37, 41, 39, 37, 35, 33, 32, 31, + /* Size 16 */ + 64, 70, 75, 62, 49, 48, 47, 46, 45, 44, 42, 41, 40, 38, 37, 37, 70, 68, + 66, 58, 49, 49, 50, 49, 48, 47, 45, 44, 42, 40, 39, 39, 75, 66, 57, 53, + 49, 50, 52, 51, 51, 49, 48, 46, 44, 43, 41, 41, 62, 58, 53, 50, 46, 47, + 48, 48, 48, 47, 46, 44, 43, 42, 40, 40, 49, 49, 49, 46, 44, 44, 45, 45, + 45, 44, 44, 43, 42, 40, 39, 39, 48, 49, 50, 47, 44, 44, 43, 43, 43, 42, + 42, 41, 40, 39, 38, 38, 47, 50, 52, 48, 45, 43, 42, 41, 41, 40, 40, 39, + 38, 38, 37, 37, 46, 49, 51, 48, 45, 43, 41, 40, 39, 39, 38, 38, 37, 36, + 36, 36, 45, 48, 51, 48, 45, 43, 41, 39, 38, 37, 37, 36, 36, 35, 35, 35, + 44, 47, 49, 47, 44, 42, 40, 39, 37, 37, 36, 35, 35, 34, 34, 34, 42, 45, + 48, 46, 44, 42, 40, 38, 37, 36, 35, 34, 34, 33, 33, 33, 41, 44, 46, 44, + 43, 41, 39, 38, 36, 35, 34, 34, 33, 33, 32, 32, 40, 42, 44, 43, 42, 40, + 38, 37, 36, 35, 34, 33, 33, 32, 32, 32, 38, 40, 43, 42, 40, 39, 38, 36, + 35, 34, 33, 33, 32, 32, 31, 31, 37, 39, 41, 40, 39, 38, 37, 36, 35, 34, + 33, 32, 32, 31, 31, 31, 37, 39, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, + 32, 31, 31, 31, + /* Size 32 */ + 64, 67, 70, 72, 75, 69, 62, 56, 49, 49, 48, 48, 47, 47, 46, 46, 45, 45, + 44, 43, 42, 42, 41, 40, 40, 39, 38, 38, 37, 37, 37, 37, 67, 68, 69, 70, + 71, 65, 60, 54, 49, 49, 49, 49, 48, 48, 48, 47, 47, 46, 45, 45, 44, 43, + 42, 41, 41, 40, 39, 39, 38, 38, 38, 38, 70, 69, 68, 67, 66, 62, 58, 53, + 49, 49, 49, 49, 50, 49, 49, 48, 48, 47, 47, 46, 45, 44, 44, 43, 42, 41, + 40, 40, 39, 39, 39, 39, 72, 70, 67, 64, 62, 59, 55, 52, 49, 49, 50, 50, + 51, 50, 50, 50, 49, 49, 48, 47, 46, 46, 45, 44, 43, 42, 42, 41, 40, 40, + 40, 40, 75, 71, 66, 62, 57, 55, 53, 51, 49, 49, 50, 51, 52, 52, 51, 51, + 51, 50, 49, 49, 48, 47, 46, 45, 44, 44, 43, 42, 41, 41, 41, 41, 69, 65, + 62, 59, 55, 53, 51, 49, 47, 48, 49, 49, 50, 50, 50, 50, 49, 49, 48, 47, + 47, 46, 45, 44, 44, 43, 42, 41, 41, 41, 41, 41, 62, 60, 58, 55, 53, 51, + 50, 48, 46, 47, 47, 48, 48, 48, 48, 48, 48, 47, 47, 46, 46, 45, 44, 44, + 43, 42, 42, 41, 40, 40, 40, 40, 56, 54, 53, 52, 51, 49, 48, 47, 45, 46, + 46, 46, 47, 47, 47, 47, 46, 46, 46, 45, 45, 44, 43, 43, 42, 42, 41, 40, + 40, 40, 40, 40, 49, 49, 49, 49, 49, 47, 46, 45, 44, 44, 44, 45, 45, 45, + 45, 45, 45, 45, 44, 44, 44, 43, 43, 42, 42, 41, 40, 40, 39, 39, 39, 39, + 49, 49, 49, 49, 49, 48, 47, 46, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 43, 43, 43, 42, 42, 41, 41, 40, 40, 39, 39, 39, 39, 39, 48, 49, 49, 50, + 50, 49, 47, 46, 44, 44, 44, 44, 43, 43, 43, 43, 43, 43, 42, 42, 42, 41, + 41, 40, 40, 39, 39, 38, 38, 38, 38, 38, 48, 49, 49, 50, 51, 49, 48, 46, + 45, 44, 44, 43, 43, 42, 42, 42, 42, 41, 41, 41, 41, 40, 40, 39, 39, 39, + 38, 38, 37, 37, 37, 37, 47, 48, 50, 51, 52, 50, 48, 47, 45, 44, 43, 43, + 42, 41, 41, 41, 41, 40, 40, 40, 40, 39, 39, 39, 38, 38, 38, 37, 37, 37, + 37, 37, 47, 48, 49, 50, 52, 50, 48, 47, 45, 44, 43, 42, 41, 41, 41, 40, + 40, 40, 39, 39, 39, 39, 38, 38, 38, 37, 37, 37, 36, 36, 36, 36, 46, 48, + 49, 50, 51, 50, 48, 47, 45, 44, 43, 42, 41, 41, 40, 40, 39, 39, 39, 38, + 38, 38, 38, 37, 37, 37, 36, 36, 36, 36, 36, 36, 46, 47, 48, 50, 51, 50, + 48, 47, 45, 44, 43, 42, 41, 40, 40, 39, 39, 38, 38, 38, 37, 37, 37, 37, + 36, 36, 36, 35, 35, 35, 35, 35, 45, 47, 48, 49, 51, 49, 48, 46, 45, 44, + 43, 42, 41, 40, 39, 39, 38, 38, 37, 37, 37, 37, 36, 36, 36, 35, 35, 35, + 35, 35, 35, 35, 45, 46, 47, 49, 50, 49, 47, 46, 45, 44, 43, 41, 40, 40, + 39, 38, 38, 37, 37, 37, 36, 36, 36, 36, 35, 35, 35, 34, 34, 34, 34, 34, + 44, 45, 47, 48, 49, 48, 47, 46, 44, 43, 42, 41, 40, 39, 39, 38, 37, 37, + 37, 36, 36, 36, 35, 35, 35, 35, 34, 34, 34, 34, 34, 34, 43, 45, 46, 47, + 49, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 38, 37, 37, 36, 36, 35, 35, + 35, 35, 34, 34, 34, 34, 33, 33, 33, 33, 42, 44, 45, 46, 48, 47, 46, 45, + 44, 43, 42, 41, 40, 39, 38, 37, 37, 36, 36, 35, 35, 35, 34, 34, 34, 34, + 33, 33, 33, 33, 33, 33, 42, 43, 44, 46, 47, 46, 45, 44, 43, 42, 41, 40, + 39, 39, 38, 37, 37, 36, 36, 35, 35, 34, 34, 34, 34, 33, 33, 33, 33, 33, + 33, 33, 41, 42, 44, 45, 46, 45, 44, 43, 43, 42, 41, 40, 39, 38, 38, 37, + 36, 36, 35, 35, 34, 34, 34, 34, 33, 33, 33, 33, 32, 32, 32, 32, 40, 41, + 43, 44, 45, 44, 44, 43, 42, 41, 40, 39, 39, 38, 37, 37, 36, 36, 35, 35, + 34, 34, 34, 33, 33, 33, 32, 32, 32, 32, 32, 32, 40, 41, 42, 43, 44, 44, + 43, 42, 42, 41, 40, 39, 38, 38, 37, 36, 36, 35, 35, 34, 34, 34, 33, 33, + 33, 32, 32, 32, 32, 32, 32, 32, 39, 40, 41, 42, 44, 43, 42, 42, 41, 40, + 39, 39, 38, 37, 37, 36, 35, 35, 35, 34, 34, 33, 33, 33, 32, 32, 32, 32, + 31, 31, 31, 31, 38, 39, 40, 42, 43, 42, 42, 41, 40, 40, 39, 38, 38, 37, + 36, 36, 35, 35, 34, 34, 33, 33, 33, 32, 32, 32, 32, 31, 31, 31, 31, 31, + 38, 39, 40, 41, 42, 41, 41, 40, 40, 39, 38, 38, 37, 37, 36, 35, 35, 34, + 34, 34, 33, 33, 33, 32, 32, 32, 31, 31, 31, 31, 31, 31, 37, 38, 39, 40, + 41, 41, 40, 40, 39, 39, 38, 37, 37, 36, 36, 35, 35, 34, 34, 33, 33, 33, + 32, 32, 32, 31, 31, 31, 31, 31, 31, 31, 37, 38, 39, 40, 41, 41, 40, 40, + 39, 39, 38, 37, 37, 36, 36, 35, 35, 34, 34, 33, 33, 33, 32, 32, 32, 31, + 31, 31, 31, 31, 31, 31, 37, 38, 39, 40, 41, 41, 40, 40, 39, 39, 38, 37, + 37, 36, 36, 35, 35, 34, 34, 33, 33, 33, 32, 32, 32, 31, 31, 31, 31, 31, + 31, 31, 37, 38, 39, 40, 41, 41, 40, 40, 39, 39, 38, 37, 37, 36, 36, 35, + 35, 34, 34, 33, 33, 33, 32, 32, 32, 31, 31, 31, 31, 31, 31, 31 }, + { /* Intra matrices */ + /* Size 4 */ + 110, 78, 74, 64, 78, 69, 66, 61, 74, 66, 57, 53, 64, 61, 53, 49, + /* Size 8 */ + 103, 122, 78, 75, 72, 67, 62, 58, 122, 92, 77, 83, 81, 76, 70, 65, 78, + 77, 70, 71, 71, 69, 66, 62, 75, 83, 71, 66, 64, 62, 60, 58, 72, 81, 71, + 64, 60, 58, 56, 54, 67, 76, 69, 62, 58, 55, 53, 51, 62, 70, 66, 60, 56, + 53, 50, 49, 58, 65, 62, 58, 54, 51, 49, 47, + /* Size 16 */ + 105, 115, 124, 102, 80, 78, 77, 75, 73, 71, 68, 66, 63, 61, 59, 59, 115, + 112, 109, 94, 79, 80, 80, 79, 78, 75, 73, 70, 67, 65, 62, 62, 124, 109, + 94, 86, 79, 82, 84, 83, 83, 80, 77, 74, 72, 69, 66, 66, 102, 94, 86, 81, + 75, 77, 78, 78, 78, 76, 74, 71, 69, 67, 64, 64, 80, 79, 79, 75, 71, 72, + 73, 73, 73, 71, 70, 68, 67, 65, 63, 63, 78, 80, 82, 77, 72, 71, 70, 69, + 69, 68, 67, 65, 64, 62, 61, 61, 77, 80, 84, 78, 73, 70, 67, 66, 65, 64, + 63, 62, 61, 60, 59, 59, 75, 79, 83, 78, 73, 69, 66, 64, 63, 62, 61, 60, + 59, 58, 57, 57, 73, 78, 83, 78, 73, 69, 65, 63, 61, 60, 59, 58, 57, 56, + 55, 55, 71, 75, 80, 76, 71, 68, 64, 62, 60, 58, 57, 56, 55, 54, 53, 53, + 68, 73, 77, 74, 70, 67, 63, 61, 59, 57, 55, 54, 53, 53, 52, 52, 66, 70, + 74, 71, 68, 65, 62, 60, 58, 56, 54, 53, 52, 52, 51, 51, 63, 67, 72, 69, + 67, 64, 61, 59, 57, 55, 53, 52, 51, 51, 50, 50, 61, 65, 69, 67, 65, 62, + 60, 58, 56, 54, 53, 52, 51, 50, 49, 49, 59, 62, 66, 64, 63, 61, 59, 57, + 55, 53, 52, 51, 50, 49, 48, 48, 59, 62, 66, 64, 63, 61, 59, 57, 55, 53, + 52, 51, 50, 49, 48, 48, + /* Size 32 */ + 106, 111, 116, 121, 126, 114, 103, 92, 80, 80, 79, 78, 77, 76, 76, 75, + 74, 73, 71, 70, 69, 68, 66, 65, 64, 63, 62, 60, 59, 59, 59, 59, 111, + 113, 114, 116, 118, 108, 99, 90, 80, 80, 80, 79, 79, 79, 78, 77, 76, 75, + 74, 72, 71, 70, 69, 67, 66, 65, 64, 62, 61, 61, 61, 61, 116, 114, 113, + 112, 110, 103, 95, 87, 80, 80, 81, 81, 81, 81, 80, 79, 79, 77, 76, 75, + 73, 72, 71, 69, 68, 67, 65, 64, 63, 63, 63, 63, 121, 116, 112, 107, 102, + 97, 91, 85, 80, 81, 81, 82, 83, 83, 82, 82, 81, 80, 78, 77, 76, 74, 73, + 71, 70, 69, 67, 66, 65, 65, 65, 65, 126, 118, 110, 102, 95, 91, 87, 83, + 79, 81, 82, 84, 85, 85, 84, 84, 83, 82, 81, 79, 78, 77, 75, 74, 72, 71, + 69, 68, 67, 67, 67, 67, 114, 108, 103, 97, 91, 87, 84, 81, 77, 79, 80, + 81, 82, 82, 81, 81, 81, 80, 79, 77, 76, 75, 74, 72, 71, 70, 68, 67, 66, + 66, 66, 66, 103, 99, 95, 91, 87, 84, 81, 78, 75, 76, 77, 78, 79, 79, 79, + 79, 78, 77, 76, 75, 75, 73, 72, 71, 70, 68, 67, 66, 65, 65, 65, 65, 92, + 90, 87, 85, 83, 81, 78, 76, 74, 74, 75, 75, 76, 76, 76, 76, 76, 75, 74, + 73, 73, 72, 71, 70, 68, 67, 66, 65, 64, 64, 64, 64, 80, 80, 80, 80, 79, + 77, 75, 74, 72, 72, 72, 73, 73, 73, 73, 73, 73, 73, 72, 72, 71, 70, 69, + 68, 67, 66, 65, 64, 63, 63, 63, 63, 80, 80, 80, 81, 81, 79, 76, 74, 72, + 72, 72, 72, 72, 72, 72, 71, 71, 71, 70, 70, 69, 68, 68, 67, 66, 65, 64, + 63, 62, 62, 62, 62, 79, 80, 81, 81, 82, 80, 77, 75, 72, 72, 71, 71, 70, + 70, 70, 70, 69, 69, 68, 68, 67, 67, 66, 65, 64, 64, 63, 62, 61, 61, 61, + 61, 78, 79, 81, 82, 84, 81, 78, 75, 73, 72, 71, 70, 69, 69, 68, 68, 68, + 67, 67, 66, 66, 65, 64, 64, 63, 62, 62, 61, 60, 60, 60, 60, 77, 79, 81, + 83, 85, 82, 79, 76, 73, 72, 70, 69, 68, 67, 67, 66, 66, 65, 65, 64, 64, + 63, 63, 62, 62, 61, 60, 60, 59, 59, 59, 59, 76, 79, 81, 83, 85, 82, 79, + 76, 73, 72, 70, 69, 67, 66, 66, 65, 65, 64, 64, 63, 63, 62, 62, 61, 61, + 60, 59, 59, 58, 58, 58, 58, 76, 78, 80, 82, 84, 81, 79, 76, 73, 72, 70, + 68, 67, 66, 65, 64, 64, 63, 62, 62, 61, 61, 60, 60, 59, 59, 58, 58, 57, + 57, 57, 57, 75, 77, 79, 82, 84, 81, 79, 76, 73, 71, 70, 68, 66, 65, 64, + 63, 62, 62, 61, 61, 60, 60, 59, 59, 58, 58, 57, 57, 56, 56, 56, 56, 74, + 76, 79, 81, 83, 81, 78, 76, 73, 71, 69, 68, 66, 65, 64, 62, 61, 61, 60, + 60, 59, 59, 58, 58, 57, 57, 56, 56, 55, 55, 55, 55, 73, 75, 77, 80, 82, + 80, 77, 75, 73, 71, 69, 67, 65, 64, 63, 62, 61, 60, 60, 59, 58, 58, 57, + 57, 56, 56, 55, 55, 55, 55, 55, 55, 71, 74, 76, 78, 81, 79, 76, 74, 72, + 70, 68, 67, 65, 64, 62, 61, 60, 60, 59, 58, 58, 57, 57, 56, 56, 55, 55, + 54, 54, 54, 54, 54, 70, 72, 75, 77, 79, 77, 75, 73, 72, 70, 68, 66, 64, + 63, 62, 61, 60, 59, 58, 57, 57, 56, 56, 55, 55, 54, 54, 54, 53, 53, 53, + 53, 69, 71, 73, 76, 78, 76, 75, 73, 71, 69, 67, 66, 64, 63, 61, 60, 59, + 58, 58, 57, 56, 55, 55, 54, 54, 54, 53, 53, 52, 52, 52, 52, 68, 70, 72, + 74, 77, 75, 73, 72, 70, 68, 67, 65, 63, 62, 61, 60, 59, 58, 57, 56, 55, + 55, 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, 66, 69, 71, 73, 75, 74, 72, + 71, 69, 68, 66, 64, 63, 62, 60, 59, 58, 57, 57, 56, 55, 54, 54, 53, 53, + 52, 52, 52, 51, 51, 51, 51, 65, 67, 69, 71, 74, 72, 71, 70, 68, 67, 65, + 64, 62, 61, 60, 59, 58, 57, 56, 55, 54, 54, 53, 53, 52, 52, 52, 51, 51, + 51, 51, 51, 64, 66, 68, 70, 72, 71, 70, 68, 67, 66, 64, 63, 62, 61, 59, + 58, 57, 56, 56, 55, 54, 53, 53, 52, 52, 51, 51, 51, 50, 50, 50, 50, 63, + 65, 67, 69, 71, 70, 68, 67, 66, 65, 64, 62, 61, 60, 59, 58, 57, 56, 55, + 54, 54, 53, 52, 52, 51, 51, 51, 50, 50, 50, 50, 50, 62, 64, 65, 67, 69, + 68, 67, 66, 65, 64, 63, 62, 60, 59, 58, 57, 56, 55, 55, 54, 53, 53, 52, + 52, 51, 51, 50, 50, 49, 49, 49, 49, 60, 62, 64, 66, 68, 67, 66, 65, 64, + 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 54, 53, 52, 52, 51, 51, 50, 50, + 49, 49, 49, 49, 49, 59, 61, 63, 65, 67, 66, 65, 64, 63, 62, 61, 60, 59, + 58, 57, 56, 55, 55, 54, 53, 52, 52, 51, 51, 50, 50, 49, 49, 49, 49, 49, + 49, 59, 61, 63, 65, 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, + 55, 54, 53, 52, 52, 51, 51, 50, 50, 49, 49, 49, 49, 49, 49, 59, 61, 63, + 65, 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 55, 54, 53, 52, + 52, 51, 51, 50, 50, 49, 49, 49, 49, 49, 49, 59, 61, 63, 65, 67, 66, 65, + 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 55, 54, 53, 52, 52, 51, 51, 50, + 50, 49, 49, 49, 49, 49, 49 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 60, 45, 36, 60, 47, 40, 35, 45, 40, 34, 32, 36, 35, 32, 31, + /* Size 8 */ + 64, 77, 73, 61, 52, 45, 41, 38, 77, 70, 73, 65, 56, 49, 44, 40, 73, 73, + 58, 53, 49, 45, 42, 39, 61, 65, 53, 47, 44, 41, 39, 38, 52, 56, 49, 44, + 41, 39, 37, 36, 45, 49, 45, 41, 39, 37, 36, 35, 41, 44, 42, 39, 37, 36, + 35, 34, 38, 40, 39, 38, 36, 35, 34, 34, + /* Size 16 */ + 64, 71, 77, 75, 73, 67, 61, 56, 52, 48, 45, 43, 41, 40, 38, 38, 71, 72, + 74, 73, 73, 68, 63, 59, 54, 51, 47, 45, 42, 41, 39, 39, 77, 74, 70, 71, + 73, 69, 65, 61, 56, 53, 49, 46, 44, 42, 40, 40, 75, 73, 71, 69, 66, 62, + 59, 56, 53, 50, 47, 45, 43, 41, 40, 40, 73, 73, 73, 66, 58, 56, 53, 51, + 49, 47, 45, 43, 42, 40, 39, 39, 67, 68, 69, 62, 56, 53, 50, 48, 46, 45, + 43, 42, 41, 39, 38, 38, 61, 63, 65, 59, 53, 50, 47, 45, 44, 43, 41, 40, + 39, 38, 38, 38, 56, 59, 61, 56, 51, 48, 45, 44, 42, 41, 40, 39, 38, 38, + 37, 37, 52, 54, 56, 53, 49, 46, 44, 42, 41, 40, 39, 38, 37, 37, 36, 36, + 48, 51, 53, 50, 47, 45, 43, 41, 40, 39, 38, 37, 37, 36, 36, 36, 45, 47, + 49, 47, 45, 43, 41, 40, 39, 38, 37, 37, 36, 36, 35, 35, 43, 45, 46, 45, + 43, 42, 40, 39, 38, 37, 37, 36, 36, 35, 35, 35, 41, 42, 44, 43, 42, 41, + 39, 38, 37, 37, 36, 36, 35, 35, 34, 34, 40, 41, 42, 41, 40, 39, 38, 38, + 37, 36, 36, 35, 35, 34, 34, 34, 38, 39, 40, 40, 39, 38, 38, 37, 36, 36, + 35, 35, 34, 34, 34, 34, 38, 39, 40, 40, 39, 38, 38, 37, 36, 36, 35, 35, + 34, 34, 34, 34, + /* Size 32 */ + 64, 67, 71, 74, 77, 76, 75, 74, 73, 70, 67, 64, 61, 59, 56, 54, 52, 50, + 48, 47, 45, 44, 43, 42, 41, 40, 40, 39, 38, 38, 38, 38, 67, 69, 71, 74, + 76, 75, 74, 74, 73, 70, 68, 65, 62, 60, 58, 55, 53, 51, 50, 48, 46, 45, + 44, 43, 42, 41, 40, 40, 39, 39, 39, 39, 71, 71, 72, 73, 74, 74, 73, 73, + 73, 71, 68, 66, 63, 61, 59, 56, 54, 52, 51, 49, 47, 46, 45, 44, 42, 42, + 41, 40, 39, 39, 39, 39, 74, 74, 73, 73, 72, 72, 72, 73, 73, 71, 69, 66, + 64, 62, 60, 57, 55, 53, 52, 50, 48, 47, 46, 44, 43, 42, 42, 41, 40, 40, + 40, 40, 77, 76, 74, 72, 70, 71, 71, 72, 73, 71, 69, 67, 65, 63, 61, 59, + 56, 54, 53, 51, 49, 48, 46, 45, 44, 43, 42, 41, 40, 40, 40, 40, 76, 75, + 74, 72, 71, 70, 70, 70, 69, 67, 66, 64, 62, 60, 58, 56, 54, 53, 51, 50, + 48, 47, 46, 45, 43, 43, 42, 41, 40, 40, 40, 40, 75, 74, 73, 72, 71, 70, + 69, 67, 66, 64, 62, 61, 59, 58, 56, 54, 53, 51, 50, 48, 47, 46, 45, 44, + 43, 42, 41, 41, 40, 40, 40, 40, 74, 74, 73, 73, 72, 70, 67, 65, 62, 61, + 59, 58, 56, 55, 54, 52, 51, 50, 49, 47, 46, 45, 44, 43, 42, 42, 41, 40, + 39, 39, 39, 39, 73, 73, 73, 73, 73, 69, 66, 62, 58, 57, 56, 55, 53, 52, + 51, 50, 49, 48, 47, 46, 45, 44, 43, 43, 42, 41, 40, 40, 39, 39, 39, 39, + 70, 70, 71, 71, 71, 67, 64, 61, 57, 56, 55, 53, 52, 51, 50, 49, 48, 47, + 46, 45, 44, 43, 43, 42, 41, 41, 40, 39, 39, 39, 39, 39, 67, 68, 68, 69, + 69, 66, 62, 59, 56, 55, 53, 52, 50, 49, 48, 47, 46, 46, 45, 44, 43, 43, + 42, 41, 41, 40, 39, 39, 38, 38, 38, 38, 64, 65, 66, 66, 67, 64, 61, 58, + 55, 53, 52, 50, 49, 48, 47, 46, 45, 44, 44, 43, 42, 42, 41, 41, 40, 39, + 39, 38, 38, 38, 38, 38, 61, 62, 63, 64, 65, 62, 59, 56, 53, 52, 50, 49, + 47, 46, 45, 45, 44, 43, 43, 42, 41, 41, 40, 40, 39, 39, 38, 38, 38, 38, + 38, 38, 59, 60, 61, 62, 63, 60, 58, 55, 52, 51, 49, 48, 46, 45, 45, 44, + 43, 42, 42, 41, 41, 40, 40, 39, 39, 38, 38, 38, 37, 37, 37, 37, 56, 58, + 59, 60, 61, 58, 56, 54, 51, 50, 48, 47, 45, 45, 44, 43, 42, 42, 41, 41, + 40, 40, 39, 39, 38, 38, 38, 37, 37, 37, 37, 37, 54, 55, 56, 57, 59, 56, + 54, 52, 50, 49, 47, 46, 45, 44, 43, 42, 41, 41, 40, 40, 39, 39, 39, 38, + 38, 38, 37, 37, 37, 37, 37, 37, 52, 53, 54, 55, 56, 54, 53, 51, 49, 48, + 46, 45, 44, 43, 42, 41, 41, 40, 40, 39, 39, 38, 38, 38, 37, 37, 37, 37, + 36, 36, 36, 36, 50, 51, 52, 53, 54, 53, 51, 50, 48, 47, 46, 44, 43, 42, + 42, 41, 40, 40, 39, 39, 38, 38, 38, 37, 37, 37, 37, 36, 36, 36, 36, 36, + 48, 50, 51, 52, 53, 51, 50, 49, 47, 46, 45, 44, 43, 42, 41, 40, 40, 39, + 39, 38, 38, 38, 37, 37, 37, 36, 36, 36, 36, 36, 36, 36, 47, 48, 49, 50, + 51, 50, 48, 47, 46, 45, 44, 43, 42, 41, 41, 40, 39, 39, 38, 38, 38, 37, + 37, 37, 36, 36, 36, 36, 35, 35, 35, 35, 45, 46, 47, 48, 49, 48, 47, 46, + 45, 44, 43, 42, 41, 41, 40, 39, 39, 38, 38, 38, 37, 37, 37, 36, 36, 36, + 36, 35, 35, 35, 35, 35, 44, 45, 46, 47, 48, 47, 46, 45, 44, 43, 43, 42, + 41, 40, 40, 39, 38, 38, 38, 37, 37, 37, 36, 36, 36, 36, 35, 35, 35, 35, + 35, 35, 43, 44, 45, 46, 46, 46, 45, 44, 43, 43, 42, 41, 40, 40, 39, 39, + 38, 38, 37, 37, 37, 36, 36, 36, 36, 35, 35, 35, 35, 35, 35, 35, 42, 43, + 44, 44, 45, 45, 44, 43, 43, 42, 41, 41, 40, 39, 39, 38, 38, 37, 37, 37, + 36, 36, 36, 36, 35, 35, 35, 35, 35, 35, 35, 35, 41, 42, 42, 43, 44, 43, + 43, 42, 42, 41, 41, 40, 39, 39, 38, 38, 37, 37, 37, 36, 36, 36, 36, 35, + 35, 35, 35, 35, 34, 34, 34, 34, 40, 41, 42, 42, 43, 43, 42, 42, 41, 41, + 40, 39, 39, 38, 38, 38, 37, 37, 36, 36, 36, 36, 35, 35, 35, 35, 35, 34, + 34, 34, 34, 34, 40, 40, 41, 42, 42, 42, 41, 41, 40, 40, 39, 39, 38, 38, + 38, 37, 37, 37, 36, 36, 36, 35, 35, 35, 35, 35, 34, 34, 34, 34, 34, 34, + 39, 40, 40, 41, 41, 41, 41, 40, 40, 39, 39, 38, 38, 38, 37, 37, 37, 36, + 36, 36, 35, 35, 35, 35, 35, 34, 34, 34, 34, 34, 34, 34, 38, 39, 39, 40, + 40, 40, 40, 39, 39, 39, 38, 38, 38, 37, 37, 37, 36, 36, 36, 35, 35, 35, + 35, 35, 34, 34, 34, 34, 34, 34, 34, 34, 38, 39, 39, 40, 40, 40, 40, 39, + 39, 39, 38, 38, 38, 37, 37, 37, 36, 36, 36, 35, 35, 35, 35, 35, 34, 34, + 34, 34, 34, 34, 34, 34, 38, 39, 39, 40, 40, 40, 40, 39, 39, 39, 38, 38, + 38, 37, 37, 37, 36, 36, 36, 35, 35, 35, 35, 35, 34, 34, 34, 34, 34, 34, + 34, 34, 38, 39, 39, 40, 40, 40, 40, 39, 39, 39, 38, 38, 38, 37, 37, 37, + 36, 36, 36, 35, 35, 35, 35, 35, 34, 34, 34, 34, 34, 34, 34, 34 }, + { /* Intra matrices */ + /* Size 4 */ + 109, 102, 75, 59, 102, 79, 66, 57, 75, 66, 56, 52, 59, 57, 52, 49, + /* Size 8 */ + 96, 117, 111, 92, 77, 67, 60, 55, 117, 106, 110, 98, 84, 73, 64, 59, + 111, 110, 87, 80, 73, 66, 61, 57, 92, 98, 80, 69, 64, 60, 57, 55, 77, + 84, 73, 64, 59, 56, 54, 52, 67, 73, 66, 60, 56, 54, 52, 51, 60, 64, 61, + 57, 54, 52, 51, 49, 55, 59, 57, 55, 52, 51, 49, 49, + /* Size 16 */ + 98, 109, 120, 116, 113, 103, 94, 86, 78, 73, 68, 65, 61, 59, 57, 57, + 109, 111, 114, 113, 112, 105, 97, 89, 82, 76, 71, 67, 63, 61, 58, 58, + 120, 114, 108, 110, 112, 106, 100, 93, 86, 80, 74, 70, 66, 63, 60, 60, + 116, 113, 110, 105, 101, 96, 91, 85, 80, 75, 71, 67, 64, 62, 59, 59, + 113, 112, 112, 101, 89, 85, 81, 78, 74, 71, 68, 65, 62, 60, 58, 58, 103, + 105, 106, 96, 85, 81, 76, 73, 70, 67, 65, 62, 60, 59, 57, 57, 94, 97, + 100, 91, 81, 76, 71, 68, 66, 64, 62, 60, 58, 57, 56, 56, 86, 89, 93, 85, + 78, 73, 68, 66, 63, 61, 60, 58, 57, 56, 55, 55, 78, 82, 86, 80, 74, 70, + 66, 63, 61, 59, 57, 56, 55, 54, 53, 53, 73, 76, 80, 75, 71, 67, 64, 61, + 59, 58, 56, 55, 54, 53, 53, 53, 68, 71, 74, 71, 68, 65, 62, 60, 57, 56, + 55, 54, 53, 52, 52, 52, 65, 67, 70, 67, 65, 62, 60, 58, 56, 55, 54, 53, + 52, 52, 51, 51, 61, 63, 66, 64, 62, 60, 58, 57, 55, 54, 53, 52, 52, 51, + 51, 51, 59, 61, 63, 62, 60, 59, 57, 56, 54, 53, 52, 52, 51, 51, 50, 50, + 57, 58, 60, 59, 58, 57, 56, 55, 53, 53, 52, 51, 51, 50, 50, 50, 57, 58, + 60, 59, 58, 57, 56, 55, 53, 53, 52, 51, 51, 50, 50, 50, + /* Size 32 */ + 99, 105, 110, 115, 121, 119, 118, 116, 114, 109, 104, 100, 95, 91, 87, + 83, 79, 76, 74, 71, 69, 67, 65, 63, 62, 61, 59, 58, 57, 57, 57, 57, 105, + 108, 111, 115, 118, 117, 116, 115, 114, 110, 105, 101, 96, 93, 89, 85, + 81, 78, 76, 73, 70, 68, 67, 65, 63, 62, 61, 59, 58, 58, 58, 58, 110, + 111, 113, 114, 115, 115, 114, 114, 114, 110, 106, 102, 98, 94, 90, 87, + 83, 80, 77, 74, 72, 70, 68, 66, 64, 63, 62, 60, 59, 59, 59, 59, 115, + 115, 114, 113, 112, 113, 113, 113, 113, 110, 106, 103, 100, 96, 92, 88, + 85, 82, 79, 76, 73, 71, 69, 67, 65, 64, 63, 61, 60, 60, 60, 60, 121, + 118, 115, 112, 110, 110, 111, 112, 113, 110, 107, 104, 101, 98, 94, 90, + 87, 84, 81, 78, 75, 73, 71, 69, 67, 65, 64, 62, 61, 61, 61, 61, 119, + 117, 115, 113, 110, 110, 109, 108, 107, 105, 102, 99, 96, 93, 90, 87, + 84, 81, 78, 76, 73, 71, 69, 68, 66, 64, 63, 62, 60, 60, 60, 60, 118, + 116, 114, 113, 111, 109, 106, 104, 102, 99, 97, 94, 92, 89, 86, 83, 81, + 78, 76, 74, 72, 70, 68, 66, 65, 63, 62, 61, 60, 60, 60, 60, 116, 115, + 114, 113, 112, 108, 104, 100, 96, 94, 91, 89, 87, 85, 82, 80, 78, 76, + 74, 72, 70, 68, 67, 65, 64, 63, 62, 60, 59, 59, 59, 59, 114, 114, 114, + 113, 113, 107, 102, 96, 90, 88, 86, 84, 82, 80, 79, 77, 75, 73, 72, 70, + 68, 67, 66, 64, 63, 62, 61, 60, 59, 59, 59, 59, 109, 110, 110, 110, 110, + 105, 99, 94, 88, 86, 84, 82, 79, 78, 76, 74, 73, 71, 70, 68, 67, 66, 64, + 63, 62, 61, 60, 59, 58, 58, 58, 58, 104, 105, 106, 106, 107, 102, 97, + 91, 86, 84, 81, 79, 77, 75, 74, 72, 71, 69, 68, 67, 65, 64, 63, 62, 61, + 60, 59, 58, 57, 57, 57, 57, 100, 101, 102, 103, 104, 99, 94, 89, 84, 82, + 79, 77, 74, 73, 71, 70, 68, 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, 58, + 57, 57, 57, 57, 95, 96, 98, 100, 101, 96, 92, 87, 82, 79, 77, 74, 72, + 70, 69, 68, 66, 65, 64, 63, 62, 61, 61, 60, 59, 58, 58, 57, 56, 56, 56, + 56, 91, 93, 94, 96, 98, 93, 89, 85, 80, 78, 75, 73, 70, 69, 68, 66, 65, + 64, 63, 62, 61, 60, 60, 59, 58, 58, 57, 56, 56, 56, 56, 56, 87, 89, 90, + 92, 94, 90, 86, 82, 79, 76, 74, 71, 69, 68, 66, 65, 64, 63, 62, 61, 60, + 59, 59, 58, 57, 57, 56, 56, 55, 55, 55, 55, 83, 85, 87, 88, 90, 87, 83, + 80, 77, 74, 72, 70, 68, 66, 65, 64, 62, 62, 61, 60, 59, 59, 58, 57, 57, + 56, 56, 55, 55, 55, 55, 55, 79, 81, 83, 85, 87, 84, 81, 78, 75, 73, 71, + 68, 66, 65, 64, 62, 61, 60, 60, 59, 58, 58, 57, 56, 56, 55, 55, 54, 54, + 54, 54, 54, 76, 78, 80, 82, 84, 81, 78, 76, 73, 71, 69, 67, 65, 64, 63, + 62, 60, 60, 59, 58, 57, 57, 56, 56, 55, 55, 54, 54, 54, 54, 54, 54, 74, + 76, 77, 79, 81, 78, 76, 74, 72, 70, 68, 66, 64, 63, 62, 61, 60, 59, 58, + 57, 57, 56, 56, 55, 55, 54, 54, 54, 53, 53, 53, 53, 71, 73, 74, 76, 78, + 76, 74, 72, 70, 68, 67, 65, 63, 62, 61, 60, 59, 58, 57, 57, 56, 56, 55, + 55, 54, 54, 53, 53, 53, 53, 53, 53, 69, 70, 72, 73, 75, 73, 72, 70, 68, + 67, 65, 64, 62, 61, 60, 59, 58, 57, 57, 56, 55, 55, 55, 54, 54, 53, 53, + 53, 52, 52, 52, 52, 67, 68, 70, 71, 73, 71, 70, 68, 67, 66, 64, 63, 61, + 60, 59, 59, 58, 57, 56, 56, 55, 55, 54, 54, 53, 53, 53, 52, 52, 52, 52, + 52, 65, 67, 68, 69, 71, 69, 68, 67, 66, 64, 63, 62, 61, 60, 59, 58, 57, + 56, 56, 55, 55, 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, 52, 63, 65, 66, + 67, 69, 68, 66, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 56, 55, 55, 54, + 54, 53, 53, 52, 52, 52, 52, 51, 51, 51, 51, 62, 63, 64, 65, 67, 66, 65, + 64, 63, 62, 61, 60, 59, 58, 57, 57, 56, 55, 55, 54, 54, 53, 53, 52, 52, + 52, 52, 51, 51, 51, 51, 51, 61, 62, 63, 64, 65, 64, 63, 63, 62, 61, 60, + 59, 58, 58, 57, 56, 55, 55, 54, 54, 53, 53, 53, 52, 52, 52, 51, 51, 51, + 51, 51, 51, 59, 61, 62, 63, 64, 63, 62, 62, 61, 60, 59, 58, 58, 57, 56, + 56, 55, 54, 54, 53, 53, 53, 52, 52, 52, 51, 51, 51, 51, 51, 51, 51, 58, + 59, 60, 61, 62, 62, 61, 60, 60, 59, 58, 58, 57, 56, 56, 55, 54, 54, 54, + 53, 53, 52, 52, 52, 51, 51, 51, 51, 50, 50, 50, 50, 57, 58, 59, 60, 61, + 60, 60, 59, 59, 58, 57, 57, 56, 56, 55, 55, 54, 54, 53, 53, 52, 52, 52, + 51, 51, 51, 51, 50, 50, 50, 50, 50, 57, 58, 59, 60, 61, 60, 60, 59, 59, + 58, 57, 57, 56, 56, 55, 55, 54, 54, 53, 53, 52, 52, 52, 51, 51, 51, 51, + 50, 50, 50, 50, 50, 57, 58, 59, 60, 61, 60, 60, 59, 59, 58, 57, 57, 56, + 56, 55, 55, 54, 54, 53, 53, 52, 52, 52, 51, 51, 51, 51, 50, 50, 50, 50, + 50, 57, 58, 59, 60, 61, 60, 60, 59, 59, 58, 57, 57, 56, 56, 55, 55, 54, + 54, 53, 53, 52, 52, 52, 51, 51, 51, 51, 50, 50, 50, 50, 50 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 48, 46, 40, 48, 43, 42, 39, 46, 42, 37, 35, 40, 39, 35, 33, + /* Size 8 */ + 64, 74, 50, 49, 47, 44, 42, 39, 74, 58, 50, 53, 52, 49, 46, 43, 50, 50, + 46, 47, 47, 45, 44, 41, 49, 53, 47, 44, 43, 42, 41, 39, 47, 52, 47, 43, + 40, 39, 38, 37, 44, 49, 45, 42, 39, 38, 37, 36, 42, 46, 44, 41, 38, 37, + 35, 35, 39, 43, 41, 39, 37, 36, 35, 34, + /* Size 16 */ + 64, 69, 74, 62, 50, 50, 49, 48, 47, 46, 44, 43, 42, 41, 39, 39, 69, 68, + 66, 58, 50, 51, 51, 50, 50, 48, 47, 45, 44, 43, 41, 41, 74, 66, 58, 54, + 50, 51, 53, 52, 52, 51, 49, 48, 46, 45, 43, 43, 62, 58, 54, 51, 48, 49, + 50, 50, 49, 48, 47, 46, 45, 44, 42, 42, 50, 50, 50, 48, 46, 46, 47, 47, + 47, 46, 45, 45, 44, 43, 41, 41, 50, 51, 51, 49, 46, 46, 45, 45, 45, 44, + 44, 43, 42, 41, 40, 40, 49, 51, 53, 50, 47, 45, 44, 43, 43, 42, 42, 41, + 41, 40, 39, 39, 48, 50, 52, 50, 47, 45, 43, 42, 42, 41, 41, 40, 39, 39, + 38, 38, 47, 50, 52, 49, 47, 45, 43, 42, 40, 40, 39, 39, 38, 38, 37, 37, + 46, 48, 51, 48, 46, 44, 42, 41, 40, 39, 38, 38, 37, 37, 37, 37, 44, 47, + 49, 47, 45, 44, 42, 41, 39, 38, 38, 37, 37, 36, 36, 36, 43, 45, 48, 46, + 45, 43, 41, 40, 39, 38, 37, 37, 36, 36, 35, 35, 42, 44, 46, 45, 44, 42, + 41, 39, 38, 37, 37, 36, 35, 35, 35, 35, 41, 43, 45, 44, 43, 41, 40, 39, + 38, 37, 36, 36, 35, 35, 34, 34, 39, 41, 43, 42, 41, 40, 39, 38, 37, 37, + 36, 35, 35, 34, 34, 34, 39, 41, 43, 42, 41, 40, 39, 38, 37, 37, 36, 35, + 35, 34, 34, 34, + /* Size 32 */ + 64, 67, 69, 72, 74, 68, 62, 56, 50, 50, 50, 49, 49, 48, 48, 48, 47, 46, + 46, 45, 44, 44, 43, 42, 42, 41, 41, 40, 39, 39, 39, 39, 67, 67, 68, 69, + 70, 65, 60, 55, 50, 50, 50, 50, 50, 49, 49, 49, 48, 48, 47, 46, 46, 45, + 44, 44, 43, 42, 42, 41, 40, 40, 40, 40, 69, 68, 68, 67, 66, 62, 58, 54, + 50, 50, 51, 51, 51, 51, 50, 50, 50, 49, 48, 48, 47, 46, 45, 45, 44, 43, + 43, 42, 41, 41, 41, 41, 72, 69, 67, 64, 62, 59, 56, 53, 50, 51, 51, 51, + 52, 52, 51, 51, 51, 50, 49, 49, 48, 47, 47, 46, 45, 44, 44, 43, 42, 42, + 42, 42, 74, 70, 66, 62, 58, 56, 54, 52, 50, 51, 51, 52, 53, 53, 52, 52, + 52, 51, 51, 50, 49, 48, 48, 47, 46, 45, 45, 44, 43, 43, 43, 43, 68, 65, + 62, 59, 56, 54, 52, 51, 49, 50, 50, 51, 51, 51, 51, 51, 51, 50, 50, 49, + 48, 48, 47, 46, 45, 45, 44, 43, 43, 43, 43, 43, 62, 60, 58, 56, 54, 52, + 51, 49, 48, 48, 49, 49, 50, 50, 50, 49, 49, 49, 48, 48, 47, 47, 46, 45, + 45, 44, 44, 43, 42, 42, 42, 42, 56, 55, 54, 53, 52, 51, 49, 48, 47, 47, + 48, 48, 48, 48, 48, 48, 48, 48, 47, 47, 46, 46, 45, 45, 44, 44, 43, 42, + 42, 42, 42, 42, 50, 50, 50, 50, 50, 49, 48, 47, 46, 46, 46, 46, 47, 47, + 47, 47, 47, 46, 46, 46, 45, 45, 45, 44, 44, 43, 43, 42, 41, 41, 41, 41, + 50, 50, 50, 51, 51, 50, 48, 47, 46, 46, 46, 46, 46, 46, 46, 46, 46, 45, + 45, 45, 45, 44, 44, 43, 43, 42, 42, 41, 41, 41, 41, 41, 50, 50, 51, 51, + 51, 50, 49, 48, 46, 46, 46, 45, 45, 45, 45, 45, 45, 44, 44, 44, 44, 43, + 43, 42, 42, 42, 41, 41, 40, 40, 40, 40, 49, 50, 51, 51, 52, 51, 49, 48, + 46, 46, 45, 45, 44, 44, 44, 44, 44, 43, 43, 43, 43, 42, 42, 42, 41, 41, + 41, 40, 40, 40, 40, 40, 49, 50, 51, 52, 53, 51, 50, 48, 47, 46, 45, 44, + 44, 43, 43, 43, 43, 42, 42, 42, 42, 42, 41, 41, 41, 40, 40, 40, 39, 39, + 39, 39, 48, 49, 51, 52, 53, 51, 50, 48, 47, 46, 45, 44, 43, 43, 43, 42, + 42, 42, 42, 41, 41, 41, 41, 40, 40, 40, 39, 39, 39, 39, 39, 39, 48, 49, + 50, 51, 52, 51, 50, 48, 47, 46, 45, 44, 43, 43, 42, 42, 42, 41, 41, 41, + 41, 40, 40, 40, 39, 39, 39, 39, 38, 38, 38, 38, 48, 49, 50, 51, 52, 51, + 49, 48, 47, 46, 45, 44, 43, 42, 42, 42, 41, 41, 40, 40, 40, 40, 39, 39, + 39, 39, 38, 38, 38, 38, 38, 38, 47, 48, 50, 51, 52, 51, 49, 48, 47, 46, + 45, 44, 43, 42, 42, 41, 40, 40, 40, 40, 39, 39, 39, 39, 38, 38, 38, 38, + 37, 37, 37, 37, 46, 48, 49, 50, 51, 50, 49, 48, 46, 45, 44, 43, 42, 42, + 41, 41, 40, 40, 40, 39, 39, 39, 38, 38, 38, 38, 37, 37, 37, 37, 37, 37, + 46, 47, 48, 49, 51, 50, 48, 47, 46, 45, 44, 43, 42, 42, 41, 40, 40, 40, + 39, 39, 38, 38, 38, 38, 37, 37, 37, 37, 37, 37, 37, 37, 45, 46, 48, 49, + 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 41, 40, 40, 39, 39, 38, 38, 38, + 38, 37, 37, 37, 37, 36, 36, 36, 36, 36, 44, 46, 47, 48, 49, 48, 47, 46, + 45, 45, 44, 43, 42, 41, 41, 40, 39, 39, 38, 38, 38, 37, 37, 37, 37, 36, + 36, 36, 36, 36, 36, 36, 44, 45, 46, 47, 48, 48, 47, 46, 45, 44, 43, 42, + 42, 41, 40, 40, 39, 39, 38, 38, 37, 37, 37, 37, 36, 36, 36, 36, 35, 35, + 35, 35, 43, 44, 45, 47, 48, 47, 46, 45, 45, 44, 43, 42, 41, 41, 40, 39, + 39, 38, 38, 38, 37, 37, 37, 36, 36, 36, 36, 35, 35, 35, 35, 35, 42, 44, + 45, 46, 47, 46, 45, 45, 44, 43, 42, 42, 41, 40, 40, 39, 39, 38, 38, 37, + 37, 37, 36, 36, 36, 36, 35, 35, 35, 35, 35, 35, 42, 43, 44, 45, 46, 45, + 45, 44, 44, 43, 42, 41, 41, 40, 39, 39, 38, 38, 37, 37, 37, 36, 36, 36, + 35, 35, 35, 35, 35, 35, 35, 35, 41, 42, 43, 44, 45, 45, 44, 44, 43, 42, + 42, 41, 40, 40, 39, 39, 38, 38, 37, 37, 36, 36, 36, 36, 35, 35, 35, 35, + 34, 34, 34, 34, 41, 42, 43, 44, 45, 44, 44, 43, 43, 42, 41, 41, 40, 39, + 39, 38, 38, 37, 37, 37, 36, 36, 36, 35, 35, 35, 35, 34, 34, 34, 34, 34, + 40, 41, 42, 43, 44, 43, 43, 42, 42, 41, 41, 40, 40, 39, 39, 38, 38, 37, + 37, 36, 36, 36, 35, 35, 35, 35, 34, 34, 34, 34, 34, 34, 39, 40, 41, 42, + 43, 43, 42, 42, 41, 41, 40, 40, 39, 39, 38, 38, 37, 37, 37, 36, 36, 35, + 35, 35, 35, 34, 34, 34, 34, 34, 34, 34, 39, 40, 41, 42, 43, 43, 42, 42, + 41, 41, 40, 40, 39, 39, 38, 38, 37, 37, 37, 36, 36, 35, 35, 35, 35, 34, + 34, 34, 34, 34, 34, 34, 39, 40, 41, 42, 43, 43, 42, 42, 41, 41, 40, 40, + 39, 39, 38, 38, 37, 37, 37, 36, 36, 35, 35, 35, 35, 34, 34, 34, 34, 34, + 34, 34, 39, 40, 41, 42, 43, 43, 42, 42, 41, 41, 40, 40, 39, 39, 38, 38, + 37, 37, 37, 36, 36, 35, 35, 35, 35, 34, 34, 34, 34, 34, 34, 34 }, + { /* Intra matrices */ + /* Size 4 */ + 103, 76, 72, 63, 76, 68, 66, 61, 72, 66, 58, 54, 63, 61, 54, 51, + /* Size 8 */ + 98, 114, 76, 73, 71, 66, 62, 58, 114, 88, 75, 80, 78, 74, 69, 64, 76, + 75, 69, 70, 70, 68, 65, 62, 73, 80, 70, 65, 64, 62, 60, 58, 71, 78, 70, + 64, 60, 58, 57, 55, 66, 74, 68, 62, 58, 56, 54, 53, 62, 69, 65, 60, 57, + 54, 52, 51, 58, 64, 62, 58, 55, 53, 51, 49, + /* Size 16 */ + 99, 107, 115, 96, 77, 76, 74, 73, 72, 69, 67, 65, 63, 61, 59, 59, 107, + 105, 102, 90, 77, 77, 78, 77, 76, 73, 71, 69, 67, 64, 62, 62, 115, 102, + 89, 83, 76, 79, 81, 80, 80, 77, 75, 73, 70, 68, 65, 65, 96, 90, 83, 78, + 73, 74, 76, 76, 75, 74, 72, 70, 68, 66, 64, 64, 77, 77, 76, 73, 70, 70, + 71, 71, 71, 70, 69, 68, 66, 64, 63, 63, 76, 77, 79, 74, 70, 69, 69, 68, + 68, 67, 66, 65, 64, 62, 61, 61, 74, 78, 81, 76, 71, 69, 66, 65, 65, 64, + 63, 62, 61, 60, 59, 59, 73, 77, 80, 76, 71, 68, 65, 64, 63, 62, 61, 60, + 59, 58, 57, 57, 72, 76, 80, 75, 71, 68, 65, 63, 61, 60, 59, 58, 57, 57, + 56, 56, 69, 73, 77, 74, 70, 67, 64, 62, 60, 59, 58, 57, 56, 55, 55, 55, + 67, 71, 75, 72, 69, 66, 63, 61, 59, 58, 56, 56, 55, 54, 53, 53, 65, 69, + 73, 70, 68, 65, 62, 60, 58, 57, 56, 55, 54, 53, 52, 52, 63, 67, 70, 68, + 66, 64, 61, 59, 57, 56, 55, 54, 53, 52, 51, 51, 61, 64, 68, 66, 64, 62, + 60, 58, 57, 55, 54, 53, 52, 51, 51, 51, 59, 62, 65, 64, 63, 61, 59, 57, + 56, 55, 53, 52, 51, 51, 50, 50, 59, 62, 65, 64, 63, 61, 59, 57, 56, 55, + 53, 52, 51, 51, 50, 50, + /* Size 32 */ + 100, 104, 108, 112, 116, 107, 97, 87, 78, 77, 76, 76, 75, 74, 74, 73, + 72, 71, 70, 69, 68, 67, 66, 65, 63, 63, 62, 61, 60, 60, 60, 60, 104, + 105, 107, 108, 110, 102, 94, 86, 78, 77, 77, 77, 77, 76, 75, 75, 74, 73, + 72, 71, 70, 69, 68, 66, 65, 64, 63, 62, 61, 61, 61, 61, 108, 107, 106, + 104, 103, 97, 90, 84, 77, 78, 78, 78, 78, 78, 77, 77, 76, 75, 74, 73, + 72, 71, 69, 68, 67, 66, 65, 64, 63, 63, 63, 63, 112, 108, 104, 100, 96, + 92, 87, 82, 77, 78, 79, 79, 80, 80, 79, 79, 78, 77, 76, 75, 74, 73, 71, + 70, 69, 68, 67, 65, 64, 64, 64, 64, 116, 110, 103, 96, 90, 87, 83, 80, + 77, 78, 79, 81, 82, 81, 81, 81, 80, 79, 78, 77, 76, 74, 73, 72, 71, 69, + 68, 67, 66, 66, 66, 66, 107, 102, 97, 92, 87, 84, 81, 78, 75, 76, 77, + 78, 79, 79, 79, 78, 78, 77, 76, 75, 74, 73, 72, 71, 70, 68, 67, 66, 65, + 65, 65, 65, 97, 94, 90, 87, 83, 81, 78, 76, 74, 74, 75, 76, 77, 76, 76, + 76, 76, 75, 74, 73, 73, 72, 71, 70, 69, 68, 67, 65, 64, 64, 64, 64, 87, + 86, 84, 82, 80, 78, 76, 74, 72, 72, 73, 73, 74, 74, 74, 74, 74, 73, 72, + 72, 71, 70, 69, 68, 67, 67, 66, 65, 64, 64, 64, 64, 78, 78, 77, 77, 77, + 75, 74, 72, 70, 70, 71, 71, 72, 72, 72, 72, 72, 71, 71, 70, 70, 69, 68, + 67, 66, 66, 65, 64, 63, 63, 63, 63, 77, 77, 78, 78, 78, 76, 74, 72, 70, + 70, 70, 70, 70, 70, 70, 70, 70, 70, 69, 69, 68, 67, 67, 66, 65, 64, 64, + 63, 62, 62, 62, 62, 76, 77, 78, 79, 79, 77, 75, 73, 71, 70, 70, 70, 69, + 69, 69, 69, 68, 68, 67, 67, 67, 66, 65, 65, 64, 63, 63, 62, 61, 61, 61, + 61, 76, 77, 78, 79, 81, 78, 76, 73, 71, 70, 70, 69, 68, 68, 67, 67, 67, + 66, 66, 66, 65, 65, 64, 63, 63, 62, 62, 61, 60, 60, 60, 60, 75, 77, 78, + 80, 82, 79, 77, 74, 72, 70, 69, 68, 67, 66, 66, 65, 65, 65, 64, 64, 64, + 63, 63, 62, 62, 61, 61, 60, 60, 60, 60, 60, 74, 76, 78, 80, 81, 79, 76, + 74, 72, 70, 69, 68, 66, 66, 65, 65, 64, 64, 63, 63, 63, 62, 62, 61, 61, + 60, 60, 59, 59, 59, 59, 59, 74, 75, 77, 79, 81, 79, 76, 74, 72, 70, 69, + 67, 66, 65, 65, 64, 63, 63, 62, 62, 62, 61, 61, 60, 60, 59, 59, 58, 58, + 58, 58, 58, 73, 75, 77, 79, 81, 78, 76, 74, 72, 70, 69, 67, 65, 65, 64, + 63, 62, 62, 61, 61, 60, 60, 60, 59, 59, 58, 58, 58, 57, 57, 57, 57, 72, + 74, 76, 78, 80, 78, 76, 74, 72, 70, 68, 67, 65, 64, 63, 62, 61, 61, 60, + 60, 59, 59, 59, 58, 58, 57, 57, 57, 56, 56, 56, 56, 71, 73, 75, 77, 79, + 77, 75, 73, 71, 70, 68, 66, 65, 64, 63, 62, 61, 60, 60, 59, 59, 58, 58, + 58, 57, 57, 56, 56, 56, 56, 56, 56, 70, 72, 74, 76, 78, 76, 74, 72, 71, + 69, 67, 66, 64, 63, 62, 61, 60, 60, 59, 59, 58, 58, 57, 57, 56, 56, 56, + 55, 55, 55, 55, 55, 69, 71, 73, 75, 77, 75, 73, 72, 70, 69, 67, 66, 64, + 63, 62, 61, 60, 59, 59, 58, 57, 57, 57, 56, 56, 55, 55, 55, 54, 54, 54, + 54, 68, 70, 72, 74, 76, 74, 73, 71, 70, 68, 67, 65, 64, 63, 62, 60, 59, + 59, 58, 57, 57, 56, 56, 56, 55, 55, 54, 54, 54, 54, 54, 54, 67, 69, 71, + 73, 74, 73, 72, 70, 69, 67, 66, 65, 63, 62, 61, 60, 59, 58, 58, 57, 56, + 56, 56, 55, 55, 54, 54, 54, 53, 53, 53, 53, 66, 68, 69, 71, 73, 72, 71, + 69, 68, 67, 65, 64, 63, 62, 61, 60, 59, 58, 57, 57, 56, 56, 55, 55, 54, + 54, 53, 53, 53, 53, 53, 53, 65, 66, 68, 70, 72, 71, 70, 68, 67, 66, 65, + 63, 62, 61, 60, 59, 58, 58, 57, 56, 56, 55, 55, 54, 54, 53, 53, 53, 52, + 52, 52, 52, 63, 65, 67, 69, 71, 70, 69, 67, 66, 65, 64, 63, 62, 61, 60, + 59, 58, 57, 56, 56, 55, 55, 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, 63, + 64, 66, 68, 69, 68, 68, 67, 66, 64, 63, 62, 61, 60, 59, 58, 57, 57, 56, + 55, 55, 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, 52, 62, 63, 65, 67, 68, + 67, 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 56, 55, 54, 54, 53, + 53, 53, 52, 52, 52, 51, 51, 51, 51, 61, 62, 64, 65, 67, 66, 65, 65, 64, + 63, 62, 61, 60, 59, 58, 58, 57, 56, 55, 55, 54, 54, 53, 53, 52, 52, 52, + 51, 51, 51, 51, 51, 60, 61, 63, 64, 66, 65, 64, 64, 63, 62, 61, 60, 60, + 59, 58, 57, 56, 56, 55, 54, 54, 53, 53, 52, 52, 52, 51, 51, 51, 51, 51, + 51, 60, 61, 63, 64, 66, 65, 64, 64, 63, 62, 61, 60, 60, 59, 58, 57, 56, + 56, 55, 54, 54, 53, 53, 52, 52, 52, 51, 51, 51, 51, 51, 51, 60, 61, 63, + 64, 66, 65, 64, 64, 63, 62, 61, 60, 60, 59, 58, 57, 56, 56, 55, 54, 54, + 53, 53, 52, 52, 52, 51, 51, 51, 51, 51, 51, 60, 61, 63, 64, 66, 65, 64, + 64, 63, 62, 61, 60, 60, 59, 58, 57, 56, 56, 55, 54, 54, 53, 53, 52, 52, + 52, 51, 51, 51, 51, 51, 51 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 61, 47, 39, 61, 49, 42, 38, 47, 42, 38, 35, 39, 38, 35, 34, + /* Size 8 */ + 64, 76, 72, 62, 53, 47, 44, 41, 76, 70, 72, 65, 57, 51, 46, 43, 72, 72, + 59, 55, 51, 47, 44, 42, 62, 65, 55, 49, 46, 44, 42, 41, 53, 57, 51, 46, + 43, 42, 41, 40, 47, 51, 47, 44, 42, 40, 39, 39, 44, 46, 44, 42, 41, 39, + 39, 38, 41, 43, 42, 41, 40, 39, 38, 37, + /* Size 16 */ + 64, 70, 76, 74, 72, 67, 62, 57, 53, 50, 47, 46, 44, 43, 41, 41, 70, 71, + 73, 72, 72, 68, 63, 59, 55, 52, 49, 47, 45, 44, 42, 42, 76, 73, 70, 71, + 72, 68, 65, 61, 57, 54, 51, 49, 46, 45, 43, 43, 74, 72, 71, 68, 65, 63, + 60, 57, 54, 52, 49, 47, 45, 44, 43, 43, 72, 72, 72, 65, 59, 57, 55, 53, + 51, 49, 47, 46, 44, 43, 42, 42, 67, 68, 68, 63, 57, 54, 52, 50, 49, 47, + 46, 44, 43, 42, 41, 41, 62, 63, 65, 60, 55, 52, 49, 48, 46, 45, 44, 43, + 42, 41, 41, 41, 57, 59, 61, 57, 53, 50, 48, 46, 45, 44, 43, 42, 41, 41, + 40, 40, 53, 55, 57, 54, 51, 49, 46, 45, 43, 43, 42, 41, 41, 40, 40, 40, + 50, 52, 54, 52, 49, 47, 45, 44, 43, 42, 41, 40, 40, 40, 39, 39, 47, 49, + 51, 49, 47, 46, 44, 43, 42, 41, 40, 40, 39, 39, 39, 39, 46, 47, 49, 47, + 46, 44, 43, 42, 41, 40, 40, 39, 39, 39, 38, 38, 44, 45, 46, 45, 44, 43, + 42, 41, 41, 40, 39, 39, 39, 38, 38, 38, 43, 44, 45, 44, 43, 42, 41, 41, + 40, 40, 39, 39, 38, 38, 38, 38, 41, 42, 43, 43, 42, 41, 41, 40, 40, 39, + 39, 38, 38, 38, 37, 37, 41, 42, 43, 43, 42, 41, 41, 40, 40, 39, 39, 38, + 38, 38, 37, 37, + /* Size 32 */ + 64, 67, 70, 73, 76, 75, 74, 73, 72, 70, 67, 64, 62, 60, 57, 55, 53, 52, + 50, 49, 47, 47, 46, 45, 44, 43, 43, 42, 41, 41, 41, 41, 67, 69, 71, 72, + 74, 74, 73, 73, 72, 70, 67, 65, 63, 60, 58, 56, 54, 53, 51, 50, 48, 47, + 46, 45, 44, 44, 43, 42, 42, 42, 42, 42, 70, 71, 71, 72, 73, 73, 72, 72, + 72, 70, 68, 66, 63, 61, 59, 57, 55, 54, 52, 51, 49, 48, 47, 46, 45, 44, + 44, 43, 42, 42, 42, 42, 73, 72, 72, 72, 71, 71, 71, 72, 72, 70, 68, 66, + 64, 62, 60, 58, 56, 55, 53, 52, 50, 49, 48, 47, 46, 45, 44, 43, 43, 43, + 43, 43, 76, 74, 73, 71, 70, 70, 71, 71, 72, 70, 68, 67, 65, 63, 61, 59, + 57, 56, 54, 52, 51, 50, 49, 47, 46, 46, 45, 44, 43, 43, 43, 43, 75, 74, + 73, 71, 70, 70, 69, 69, 68, 67, 66, 64, 63, 61, 59, 57, 56, 54, 53, 51, + 50, 49, 48, 47, 46, 45, 44, 44, 43, 43, 43, 43, 74, 73, 72, 71, 71, 69, + 68, 67, 65, 64, 63, 61, 60, 58, 57, 56, 54, 53, 52, 50, 49, 48, 47, 46, + 45, 45, 44, 43, 43, 43, 43, 43, 73, 73, 72, 72, 71, 69, 67, 64, 62, 61, + 60, 59, 57, 56, 55, 54, 52, 51, 50, 49, 48, 47, 47, 46, 45, 44, 44, 43, + 42, 42, 42, 42, 72, 72, 72, 72, 72, 68, 65, 62, 59, 58, 57, 56, 55, 54, + 53, 52, 51, 50, 49, 48, 47, 47, 46, 45, 44, 44, 43, 43, 42, 42, 42, 42, + 70, 70, 70, 70, 70, 67, 64, 61, 58, 57, 56, 55, 53, 52, 52, 51, 50, 49, + 48, 47, 47, 46, 45, 45, 44, 43, 43, 42, 42, 42, 42, 42, 67, 67, 68, 68, + 68, 66, 63, 60, 57, 56, 54, 53, 52, 51, 50, 49, 49, 48, 47, 46, 46, 45, + 44, 44, 43, 43, 42, 42, 41, 41, 41, 41, 64, 65, 66, 66, 67, 64, 61, 59, + 56, 55, 53, 52, 51, 50, 49, 48, 47, 47, 46, 45, 45, 44, 44, 43, 43, 42, + 42, 42, 41, 41, 41, 41, 62, 63, 63, 64, 65, 63, 60, 57, 55, 53, 52, 51, + 49, 48, 48, 47, 46, 46, 45, 45, 44, 44, 43, 43, 42, 42, 41, 41, 41, 41, + 41, 41, 60, 60, 61, 62, 63, 61, 58, 56, 54, 52, 51, 50, 48, 48, 47, 46, + 46, 45, 44, 44, 43, 43, 43, 42, 42, 41, 41, 41, 40, 40, 40, 40, 57, 58, + 59, 60, 61, 59, 57, 55, 53, 52, 50, 49, 48, 47, 46, 46, 45, 44, 44, 43, + 43, 43, 42, 42, 41, 41, 41, 40, 40, 40, 40, 40, 55, 56, 57, 58, 59, 57, + 56, 54, 52, 51, 49, 48, 47, 46, 46, 45, 44, 44, 43, 43, 42, 42, 42, 41, + 41, 41, 40, 40, 40, 40, 40, 40, 53, 54, 55, 56, 57, 56, 54, 52, 51, 50, + 49, 47, 46, 46, 45, 44, 43, 43, 43, 42, 42, 41, 41, 41, 41, 40, 40, 40, + 40, 40, 40, 40, 52, 53, 54, 55, 56, 54, 53, 51, 50, 49, 48, 47, 46, 45, + 44, 44, 43, 43, 42, 42, 41, 41, 41, 41, 40, 40, 40, 40, 39, 39, 39, 39, + 50, 51, 52, 53, 54, 53, 52, 50, 49, 48, 47, 46, 45, 44, 44, 43, 43, 42, + 42, 41, 41, 41, 40, 40, 40, 40, 40, 39, 39, 39, 39, 39, 49, 50, 51, 52, + 52, 51, 50, 49, 48, 47, 46, 45, 45, 44, 43, 43, 42, 42, 41, 41, 41, 40, + 40, 40, 40, 39, 39, 39, 39, 39, 39, 39, 47, 48, 49, 50, 51, 50, 49, 48, + 47, 47, 46, 45, 44, 43, 43, 42, 42, 41, 41, 41, 40, 40, 40, 40, 39, 39, + 39, 39, 39, 39, 39, 39, 47, 47, 48, 49, 50, 49, 48, 47, 47, 46, 45, 44, + 44, 43, 43, 42, 41, 41, 41, 40, 40, 40, 40, 39, 39, 39, 39, 39, 38, 38, + 38, 38, 46, 46, 47, 48, 49, 48, 47, 47, 46, 45, 44, 44, 43, 43, 42, 42, + 41, 41, 40, 40, 40, 40, 39, 39, 39, 39, 39, 38, 38, 38, 38, 38, 45, 45, + 46, 47, 47, 47, 46, 46, 45, 45, 44, 43, 43, 42, 42, 41, 41, 41, 40, 40, + 40, 39, 39, 39, 39, 39, 38, 38, 38, 38, 38, 38, 44, 44, 45, 46, 46, 46, + 45, 45, 44, 44, 43, 43, 42, 42, 41, 41, 41, 40, 40, 40, 39, 39, 39, 39, + 39, 38, 38, 38, 38, 38, 38, 38, 43, 44, 44, 45, 46, 45, 45, 44, 44, 43, + 43, 42, 42, 41, 41, 41, 40, 40, 40, 39, 39, 39, 39, 39, 38, 38, 38, 38, + 38, 38, 38, 38, 43, 43, 44, 44, 45, 44, 44, 44, 43, 43, 42, 42, 41, 41, + 41, 40, 40, 40, 40, 39, 39, 39, 39, 38, 38, 38, 38, 38, 38, 38, 38, 38, + 42, 42, 43, 43, 44, 44, 43, 43, 43, 42, 42, 42, 41, 41, 40, 40, 40, 40, + 39, 39, 39, 39, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 41, 42, 42, 43, + 43, 43, 43, 42, 42, 42, 41, 41, 41, 40, 40, 40, 40, 39, 39, 39, 39, 38, + 38, 38, 38, 38, 38, 38, 37, 37, 37, 37, 41, 42, 42, 43, 43, 43, 43, 42, + 42, 42, 41, 41, 41, 40, 40, 40, 40, 39, 39, 39, 39, 38, 38, 38, 38, 38, + 38, 38, 37, 37, 37, 37, 41, 42, 42, 43, 43, 43, 43, 42, 42, 42, 41, 41, + 41, 40, 40, 40, 40, 39, 39, 39, 39, 38, 38, 38, 38, 38, 38, 38, 37, 37, + 37, 37, 41, 42, 42, 43, 43, 43, 43, 42, 42, 42, 41, 41, 41, 40, 40, 40, + 40, 39, 39, 39, 39, 38, 38, 38, 38, 38, 38, 38, 37, 37, 37, 37 }, + { /* Intra matrices */ + /* Size 4 */ + 100, 95, 72, 60, 95, 76, 65, 58, 72, 65, 57, 54, 60, 58, 54, 51, + /* Size 8 */ + 90, 107, 102, 86, 74, 65, 60, 56, 107, 98, 101, 92, 80, 70, 64, 59, 102, + 101, 83, 76, 71, 65, 61, 58, 86, 92, 76, 68, 64, 60, 58, 56, 74, 80, 71, + 64, 60, 57, 55, 54, 65, 70, 65, 60, 57, 55, 53, 52, 60, 64, 61, 58, 55, + 53, 52, 51, 56, 59, 58, 56, 54, 52, 51, 51, + /* Size 16 */ + 92, 100, 109, 107, 104, 96, 88, 82, 75, 71, 67, 64, 61, 59, 57, 57, 100, + 102, 105, 104, 103, 97, 91, 84, 78, 74, 69, 66, 63, 61, 59, 59, 109, + 105, 100, 101, 103, 98, 93, 87, 81, 76, 72, 68, 65, 63, 60, 60, 107, + 104, 101, 97, 94, 89, 85, 81, 77, 73, 69, 66, 63, 61, 59, 59, 104, 103, + 103, 94, 84, 81, 78, 75, 72, 69, 66, 64, 62, 60, 59, 59, 96, 97, 98, 89, + 81, 77, 73, 71, 68, 66, 64, 62, 60, 59, 58, 58, 88, 91, 93, 85, 78, 73, + 69, 67, 65, 63, 62, 60, 59, 58, 57, 57, 82, 84, 87, 81, 75, 71, 67, 65, + 63, 61, 60, 59, 58, 57, 56, 56, 75, 78, 81, 77, 72, 68, 65, 63, 61, 59, + 58, 57, 56, 55, 55, 55, 71, 74, 76, 73, 69, 66, 63, 61, 59, 58, 57, 56, + 55, 55, 54, 54, 67, 69, 72, 69, 66, 64, 62, 60, 58, 57, 56, 55, 54, 54, + 53, 53, 64, 66, 68, 66, 64, 62, 60, 59, 57, 56, 55, 54, 54, 53, 53, 53, + 61, 63, 65, 63, 62, 60, 59, 58, 56, 55, 54, 54, 53, 53, 52, 52, 59, 61, + 63, 61, 60, 59, 58, 57, 55, 55, 54, 53, 53, 52, 52, 52, 57, 59, 60, 59, + 59, 58, 57, 56, 55, 54, 53, 53, 52, 52, 52, 52, 57, 59, 60, 59, 59, 58, + 57, 56, 55, 54, 53, 53, 52, 52, 52, 52, + /* Size 32 */ + 92, 97, 101, 106, 110, 109, 107, 106, 105, 101, 97, 93, 89, 86, 82, 79, + 76, 74, 72, 69, 67, 66, 64, 63, 62, 61, 60, 59, 58, 58, 58, 58, 97, 100, + 102, 105, 108, 107, 106, 105, 104, 101, 97, 94, 90, 87, 84, 81, 77, 75, + 73, 71, 69, 67, 66, 64, 63, 62, 61, 60, 59, 59, 59, 59, 101, 102, 103, + 105, 106, 105, 105, 105, 104, 101, 98, 95, 91, 88, 85, 82, 79, 77, 74, + 72, 70, 68, 67, 65, 64, 63, 61, 60, 59, 59, 59, 59, 106, 105, 105, 104, + 103, 103, 104, 104, 104, 101, 98, 96, 93, 90, 87, 84, 80, 78, 76, 73, + 71, 69, 68, 66, 65, 63, 62, 61, 60, 60, 60, 60, 110, 108, 106, 103, 101, + 102, 102, 103, 104, 101, 99, 97, 94, 91, 88, 85, 82, 80, 77, 75, 72, 71, + 69, 67, 66, 64, 63, 62, 61, 61, 61, 61, 109, 107, 105, 103, 102, 101, + 100, 100, 99, 97, 95, 92, 90, 88, 85, 82, 80, 77, 75, 73, 71, 69, 68, + 66, 65, 64, 63, 61, 60, 60, 60, 60, 107, 106, 105, 104, 102, 100, 98, + 96, 94, 92, 90, 88, 86, 84, 82, 79, 77, 75, 73, 72, 70, 68, 67, 65, 64, + 63, 62, 61, 60, 60, 60, 60, 106, 105, 105, 104, 103, 100, 96, 93, 90, + 88, 86, 84, 82, 80, 79, 77, 75, 73, 72, 70, 68, 67, 66, 65, 63, 62, 61, + 60, 60, 60, 60, 60, 105, 104, 104, 104, 104, 99, 94, 90, 85, 83, 82, 80, + 78, 77, 75, 74, 73, 71, 70, 68, 67, 66, 65, 64, 63, 62, 61, 60, 59, 59, + 59, 59, 101, 101, 101, 101, 101, 97, 92, 88, 83, 82, 80, 78, 76, 75, 73, + 72, 71, 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, 59, 59, 59, 59, 59, 97, + 97, 98, 98, 99, 95, 90, 86, 82, 80, 78, 76, 74, 73, 71, 70, 69, 68, 67, + 66, 65, 64, 63, 62, 61, 60, 60, 59, 58, 58, 58, 58, 93, 94, 95, 96, 97, + 92, 88, 84, 80, 78, 76, 74, 72, 71, 70, 68, 67, 66, 65, 64, 63, 63, 62, + 61, 60, 59, 59, 58, 58, 58, 58, 58, 89, 90, 91, 93, 94, 90, 86, 82, 78, + 76, 74, 72, 70, 69, 68, 66, 65, 65, 64, 63, 62, 61, 61, 60, 59, 59, 58, + 58, 57, 57, 57, 57, 86, 87, 88, 90, 91, 88, 84, 80, 77, 75, 73, 71, 69, + 68, 66, 65, 64, 64, 63, 62, 61, 61, 60, 59, 59, 58, 58, 57, 57, 57, 57, + 57, 82, 84, 85, 87, 88, 85, 82, 79, 75, 73, 71, 70, 68, 66, 65, 64, 63, + 63, 62, 61, 60, 60, 59, 59, 58, 58, 57, 57, 56, 56, 56, 56, 79, 81, 82, + 84, 85, 82, 79, 77, 74, 72, 70, 68, 66, 65, 64, 63, 62, 62, 61, 60, 59, + 59, 58, 58, 57, 57, 57, 56, 56, 56, 56, 56, 76, 77, 79, 80, 82, 80, 77, + 75, 73, 71, 69, 67, 65, 64, 63, 62, 61, 61, 60, 59, 59, 58, 58, 57, 57, + 56, 56, 56, 55, 55, 55, 55, 74, 75, 77, 78, 80, 77, 75, 73, 71, 69, 68, + 66, 65, 64, 63, 62, 61, 60, 59, 59, 58, 58, 57, 57, 56, 56, 56, 55, 55, + 55, 55, 55, 72, 73, 74, 76, 77, 75, 73, 72, 70, 68, 67, 65, 64, 63, 62, + 61, 60, 59, 59, 58, 57, 57, 57, 56, 56, 55, 55, 55, 55, 55, 55, 55, 69, + 71, 72, 73, 75, 73, 72, 70, 68, 67, 66, 64, 63, 62, 61, 60, 59, 59, 58, + 58, 57, 57, 56, 56, 55, 55, 55, 54, 54, 54, 54, 54, 67, 69, 70, 71, 72, + 71, 70, 68, 67, 66, 65, 63, 62, 61, 60, 59, 59, 58, 57, 57, 56, 56, 56, + 55, 55, 55, 54, 54, 54, 54, 54, 54, 66, 67, 68, 69, 71, 69, 68, 67, 66, + 65, 64, 63, 61, 61, 60, 59, 58, 58, 57, 57, 56, 56, 55, 55, 55, 54, 54, + 54, 54, 54, 54, 54, 64, 66, 67, 68, 69, 68, 67, 66, 65, 64, 63, 62, 61, + 60, 59, 58, 58, 57, 57, 56, 56, 55, 55, 55, 54, 54, 54, 54, 53, 53, 53, + 53, 63, 64, 65, 66, 67, 66, 65, 65, 64, 63, 62, 61, 60, 59, 59, 58, 57, + 57, 56, 56, 55, 55, 55, 54, 54, 54, 54, 53, 53, 53, 53, 53, 62, 63, 64, + 65, 66, 65, 64, 63, 63, 62, 61, 60, 59, 59, 58, 57, 57, 56, 56, 55, 55, + 55, 54, 54, 54, 53, 53, 53, 53, 53, 53, 53, 61, 62, 63, 63, 64, 64, 63, + 62, 62, 61, 60, 59, 59, 58, 58, 57, 56, 56, 55, 55, 55, 54, 54, 54, 53, + 53, 53, 53, 53, 53, 53, 53, 60, 61, 61, 62, 63, 63, 62, 61, 61, 60, 60, + 59, 58, 58, 57, 57, 56, 56, 55, 55, 54, 54, 54, 54, 53, 53, 53, 53, 52, + 52, 52, 52, 59, 60, 60, 61, 62, 61, 61, 60, 60, 59, 59, 58, 58, 57, 57, + 56, 56, 55, 55, 54, 54, 54, 54, 53, 53, 53, 53, 52, 52, 52, 52, 52, 58, + 59, 59, 60, 61, 60, 60, 60, 59, 59, 58, 58, 57, 57, 56, 56, 55, 55, 55, + 54, 54, 54, 53, 53, 53, 53, 52, 52, 52, 52, 52, 52, 58, 59, 59, 60, 61, + 60, 60, 60, 59, 59, 58, 58, 57, 57, 56, 56, 55, 55, 55, 54, 54, 54, 53, + 53, 53, 53, 52, 52, 52, 52, 52, 52, 58, 59, 59, 60, 61, 60, 60, 60, 59, + 59, 58, 58, 57, 57, 56, 56, 55, 55, 55, 54, 54, 54, 53, 53, 53, 53, 52, + 52, 52, 52, 52, 52, 58, 59, 59, 60, 61, 60, 60, 60, 59, 59, 58, 58, 57, + 57, 56, 56, 55, 55, 55, 54, 54, 54, 53, 53, 53, 53, 52, 52, 52, 52, 52, + 52 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 50, 48, 43, 50, 45, 44, 42, 48, 44, 40, 38, 43, 42, 38, 36, + /* Size 8 */ + 64, 73, 52, 50, 49, 47, 44, 42, 73, 59, 51, 54, 53, 51, 48, 45, 52, 51, + 48, 49, 49, 47, 46, 44, 50, 54, 49, 46, 45, 44, 43, 42, 49, 53, 49, 45, + 43, 42, 41, 40, 47, 51, 47, 44, 42, 40, 40, 39, 44, 48, 46, 43, 41, 40, + 39, 38, 42, 45, 44, 42, 40, 39, 38, 37, + /* Size 16 */ + 64, 69, 73, 63, 52, 51, 50, 50, 49, 48, 47, 45, 44, 43, 42, 42, 69, 67, + 66, 59, 52, 52, 52, 52, 51, 50, 49, 47, 46, 45, 44, 44, 73, 66, 59, 55, + 51, 53, 54, 54, 53, 52, 51, 49, 48, 47, 45, 45, 63, 59, 55, 52, 50, 50, + 51, 51, 51, 50, 49, 48, 47, 46, 45, 45, 52, 52, 51, 50, 48, 48, 49, 49, + 49, 48, 47, 47, 46, 45, 44, 44, 51, 52, 53, 50, 48, 48, 47, 47, 47, 46, + 46, 45, 44, 44, 43, 43, 50, 52, 54, 51, 49, 47, 46, 45, 45, 45, 44, 44, + 43, 43, 42, 42, 50, 52, 54, 51, 49, 47, 45, 45, 44, 44, 43, 43, 42, 42, + 41, 41, 49, 51, 53, 51, 49, 47, 45, 44, 43, 42, 42, 41, 41, 41, 40, 40, + 48, 50, 52, 50, 48, 46, 45, 44, 42, 42, 41, 41, 40, 40, 39, 39, 47, 49, + 51, 49, 47, 46, 44, 43, 42, 41, 40, 40, 40, 39, 39, 39, 45, 47, 49, 48, + 47, 45, 44, 43, 41, 41, 40, 40, 39, 39, 38, 38, 44, 46, 48, 47, 46, 44, + 43, 42, 41, 40, 40, 39, 39, 38, 38, 38, 43, 45, 47, 46, 45, 44, 43, 42, + 41, 40, 39, 39, 38, 38, 37, 37, 42, 44, 45, 45, 44, 43, 42, 41, 40, 39, + 39, 38, 38, 37, 37, 37, 42, 44, 45, 45, 44, 43, 42, 41, 40, 39, 39, 38, + 38, 37, 37, 37, + /* Size 32 */ + 64, 66, 69, 71, 73, 68, 63, 57, 52, 52, 51, 51, 50, 50, 50, 49, 49, 48, + 48, 47, 47, 46, 45, 45, 44, 44, 43, 43, 42, 42, 42, 42, 66, 67, 68, 69, + 69, 65, 61, 56, 52, 52, 52, 52, 51, 51, 51, 50, 50, 49, 49, 48, 48, 47, + 46, 46, 45, 45, 44, 43, 43, 43, 43, 43, 69, 68, 67, 67, 66, 62, 59, 55, + 52, 52, 52, 52, 52, 52, 52, 51, 51, 51, 50, 49, 49, 48, 47, 47, 46, 46, + 45, 44, 44, 44, 44, 44, 71, 69, 67, 64, 62, 60, 57, 54, 52, 52, 52, 53, + 53, 53, 53, 52, 52, 52, 51, 50, 50, 49, 48, 48, 47, 46, 46, 45, 45, 45, + 45, 45, 73, 69, 66, 62, 59, 57, 55, 53, 51, 52, 53, 53, 54, 54, 54, 54, + 53, 53, 52, 51, 51, 50, 49, 49, 48, 47, 47, 46, 45, 45, 45, 45, 68, 65, + 62, 60, 57, 55, 54, 52, 51, 51, 52, 52, 53, 53, 52, 52, 52, 52, 51, 51, + 50, 49, 49, 48, 47, 47, 46, 46, 45, 45, 45, 45, 63, 61, 59, 57, 55, 54, + 52, 51, 50, 50, 50, 51, 51, 51, 51, 51, 51, 51, 50, 50, 49, 49, 48, 47, + 47, 46, 46, 45, 45, 45, 45, 45, 57, 56, 55, 54, 53, 52, 51, 50, 49, 49, + 49, 50, 50, 50, 50, 50, 50, 49, 49, 49, 48, 48, 47, 47, 46, 46, 45, 45, + 44, 44, 44, 44, 52, 52, 52, 52, 51, 51, 50, 49, 48, 48, 48, 48, 49, 49, + 49, 49, 49, 48, 48, 48, 47, 47, 47, 46, 46, 45, 45, 44, 44, 44, 44, 44, + 52, 52, 52, 52, 52, 51, 50, 49, 48, 48, 48, 48, 48, 48, 48, 48, 48, 47, + 47, 47, 47, 46, 46, 46, 45, 45, 44, 44, 43, 43, 43, 43, 51, 52, 52, 52, + 53, 52, 50, 49, 48, 48, 48, 47, 47, 47, 47, 47, 47, 47, 46, 46, 46, 46, + 45, 45, 44, 44, 44, 43, 43, 43, 43, 43, 51, 52, 52, 53, 53, 52, 51, 50, + 48, 48, 47, 47, 47, 46, 46, 46, 46, 46, 45, 45, 45, 45, 44, 44, 44, 43, + 43, 43, 42, 42, 42, 42, 50, 51, 52, 53, 54, 53, 51, 50, 49, 48, 47, 47, + 46, 46, 45, 45, 45, 45, 45, 44, 44, 44, 44, 43, 43, 43, 43, 42, 42, 42, + 42, 42, 50, 51, 52, 53, 54, 53, 51, 50, 49, 48, 47, 46, 46, 45, 45, 45, + 45, 44, 44, 44, 44, 43, 43, 43, 43, 42, 42, 42, 42, 42, 42, 42, 50, 51, + 52, 53, 54, 52, 51, 50, 49, 48, 47, 46, 45, 45, 45, 44, 44, 44, 44, 43, + 43, 43, 43, 42, 42, 42, 42, 41, 41, 41, 41, 41, 49, 50, 51, 52, 54, 52, + 51, 50, 49, 48, 47, 46, 45, 45, 44, 44, 44, 43, 43, 43, 42, 42, 42, 42, + 42, 41, 41, 41, 41, 41, 41, 41, 49, 50, 51, 52, 53, 52, 51, 50, 49, 48, + 47, 46, 45, 45, 44, 44, 43, 43, 42, 42, 42, 42, 41, 41, 41, 41, 41, 40, + 40, 40, 40, 40, 48, 49, 51, 52, 53, 52, 51, 49, 48, 47, 47, 46, 45, 44, + 44, 43, 43, 42, 42, 42, 42, 41, 41, 41, 41, 40, 40, 40, 40, 40, 40, 40, + 48, 49, 50, 51, 52, 51, 50, 49, 48, 47, 46, 45, 45, 44, 44, 43, 42, 42, + 42, 42, 41, 41, 41, 41, 40, 40, 40, 40, 39, 39, 39, 39, 47, 48, 49, 50, + 51, 51, 50, 49, 48, 47, 46, 45, 44, 44, 43, 43, 42, 42, 42, 41, 41, 41, + 40, 40, 40, 40, 40, 39, 39, 39, 39, 39, 47, 48, 49, 50, 51, 50, 49, 48, + 47, 47, 46, 45, 44, 44, 43, 42, 42, 42, 41, 41, 40, 40, 40, 40, 40, 39, + 39, 39, 39, 39, 39, 39, 46, 47, 48, 49, 50, 49, 49, 48, 47, 46, 46, 45, + 44, 43, 43, 42, 42, 41, 41, 41, 40, 40, 40, 40, 39, 39, 39, 39, 39, 39, + 39, 39, 45, 46, 47, 48, 49, 49, 48, 47, 47, 46, 45, 44, 44, 43, 43, 42, + 41, 41, 41, 40, 40, 40, 40, 39, 39, 39, 39, 38, 38, 38, 38, 38, 45, 46, + 47, 48, 49, 48, 47, 47, 46, 46, 45, 44, 43, 43, 42, 42, 41, 41, 41, 40, + 40, 40, 39, 39, 39, 39, 38, 38, 38, 38, 38, 38, 44, 45, 46, 47, 48, 47, + 47, 46, 46, 45, 44, 44, 43, 43, 42, 42, 41, 41, 40, 40, 40, 39, 39, 39, + 39, 38, 38, 38, 38, 38, 38, 38, 44, 45, 46, 46, 47, 47, 46, 46, 45, 45, + 44, 43, 43, 42, 42, 41, 41, 40, 40, 40, 39, 39, 39, 39, 38, 38, 38, 38, + 38, 38, 38, 38, 43, 44, 45, 46, 47, 46, 46, 45, 45, 44, 44, 43, 43, 42, + 42, 41, 41, 40, 40, 40, 39, 39, 39, 38, 38, 38, 38, 38, 37, 37, 37, 37, + 43, 43, 44, 45, 46, 46, 45, 45, 44, 44, 43, 43, 42, 42, 41, 41, 40, 40, + 40, 39, 39, 39, 38, 38, 38, 38, 38, 37, 37, 37, 37, 37, 42, 43, 44, 45, + 45, 45, 45, 44, 44, 43, 43, 42, 42, 42, 41, 41, 40, 40, 39, 39, 39, 39, + 38, 38, 38, 38, 37, 37, 37, 37, 37, 37, 42, 43, 44, 45, 45, 45, 45, 44, + 44, 43, 43, 42, 42, 42, 41, 41, 40, 40, 39, 39, 39, 39, 38, 38, 38, 38, + 37, 37, 37, 37, 37, 37, 42, 43, 44, 45, 45, 45, 45, 44, 44, 43, 43, 42, + 42, 42, 41, 41, 40, 40, 39, 39, 39, 39, 38, 38, 38, 38, 37, 37, 37, 37, + 37, 37, 42, 43, 44, 45, 45, 45, 45, 44, 44, 43, 43, 42, 42, 42, 41, 41, + 40, 40, 39, 39, 39, 39, 38, 38, 38, 38, 37, 37, 37, 37, 37, 37 }, + { /* Intra matrices */ + /* Size 4 */ + 97, 74, 71, 63, 74, 67, 65, 61, 71, 65, 58, 56, 63, 61, 56, 52, + /* Size 8 */ + 92, 106, 74, 72, 69, 66, 62, 59, 106, 84, 73, 77, 76, 72, 68, 64, 74, + 73, 68, 69, 69, 67, 65, 62, 72, 77, 69, 65, 63, 62, 61, 59, 69, 76, 69, + 63, 60, 59, 57, 56, 66, 72, 67, 62, 59, 57, 55, 54, 62, 68, 65, 61, 57, + 55, 54, 53, 59, 64, 62, 59, 56, 54, 53, 51, + /* Size 16 */ + 93, 100, 107, 91, 75, 74, 73, 71, 70, 68, 67, 65, 63, 61, 60, 60, 100, + 98, 96, 85, 74, 75, 75, 74, 74, 72, 70, 68, 66, 64, 62, 62, 107, 96, 85, + 79, 74, 76, 78, 77, 77, 75, 73, 71, 69, 67, 65, 65, 91, 85, 79, 75, 71, + 73, 74, 74, 73, 72, 71, 69, 67, 65, 64, 64, 75, 74, 74, 71, 68, 69, 70, + 70, 70, 69, 68, 67, 65, 64, 63, 63, 74, 75, 76, 73, 69, 68, 68, 67, 67, + 66, 65, 64, 63, 62, 61, 61, 73, 75, 78, 74, 70, 68, 66, 65, 64, 64, 63, + 62, 61, 60, 60, 60, 71, 74, 77, 74, 70, 67, 65, 64, 63, 62, 61, 61, 60, + 59, 58, 58, 70, 74, 77, 73, 70, 67, 64, 63, 61, 60, 59, 59, 58, 58, 57, + 57, 68, 72, 75, 72, 69, 66, 64, 62, 60, 59, 58, 58, 57, 56, 56, 56, 67, + 70, 73, 71, 68, 65, 63, 61, 59, 58, 57, 57, 56, 55, 55, 55, 65, 68, 71, + 69, 67, 64, 62, 61, 59, 58, 57, 56, 55, 55, 54, 54, 63, 66, 69, 67, 65, + 63, 61, 60, 58, 57, 56, 55, 54, 54, 53, 53, 61, 64, 67, 65, 64, 62, 60, + 59, 58, 56, 55, 55, 54, 53, 53, 53, 60, 62, 65, 64, 63, 61, 60, 58, 57, + 56, 55, 54, 53, 53, 52, 52, 60, 62, 65, 64, 63, 61, 60, 58, 57, 56, 55, + 54, 53, 53, 52, 52, + /* Size 32 */ + 94, 97, 101, 104, 108, 100, 91, 83, 75, 75, 74, 74, 73, 72, 72, 71, 71, + 70, 69, 68, 67, 66, 65, 64, 63, 63, 62, 61, 60, 60, 60, 60, 97, 98, 100, + 101, 102, 95, 89, 82, 75, 75, 75, 75, 74, 74, 73, 73, 72, 71, 70, 70, + 69, 68, 67, 66, 65, 64, 63, 62, 61, 61, 61, 61, 101, 100, 99, 98, 97, + 91, 86, 80, 75, 75, 75, 76, 76, 75, 75, 74, 74, 73, 72, 71, 70, 69, 68, + 67, 66, 65, 65, 64, 63, 63, 63, 63, 104, 101, 98, 94, 91, 87, 83, 79, + 75, 75, 76, 77, 77, 77, 76, 76, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, + 66, 65, 64, 64, 64, 64, 108, 102, 97, 91, 85, 83, 80, 77, 75, 76, 77, + 78, 79, 78, 78, 78, 77, 76, 75, 75, 74, 73, 71, 70, 69, 68, 67, 66, 65, + 65, 65, 65, 100, 95, 91, 87, 83, 80, 78, 76, 73, 74, 75, 76, 76, 76, 76, + 76, 76, 75, 74, 73, 72, 71, 70, 69, 68, 68, 67, 66, 65, 65, 65, 65, 91, + 89, 86, 83, 80, 78, 76, 74, 72, 72, 73, 74, 74, 74, 74, 74, 74, 73, 72, + 72, 71, 70, 69, 68, 68, 67, 66, 65, 64, 64, 64, 64, 83, 82, 80, 79, 77, + 76, 74, 72, 70, 71, 71, 72, 72, 72, 72, 72, 72, 71, 71, 70, 70, 69, 68, + 67, 67, 66, 65, 64, 64, 64, 64, 64, 75, 75, 75, 75, 75, 73, 72, 70, 69, + 69, 69, 70, 70, 70, 70, 70, 70, 70, 69, 69, 68, 68, 67, 66, 66, 65, 64, + 64, 63, 63, 63, 63, 75, 75, 75, 75, 76, 74, 72, 71, 69, 69, 69, 69, 69, + 69, 69, 69, 69, 68, 68, 68, 67, 67, 66, 65, 65, 64, 63, 63, 62, 62, 62, + 62, 74, 75, 75, 76, 77, 75, 73, 71, 69, 69, 69, 68, 68, 68, 68, 68, 67, + 67, 67, 66, 66, 65, 65, 64, 64, 63, 63, 62, 61, 61, 61, 61, 74, 75, 76, + 77, 78, 76, 74, 72, 70, 69, 68, 68, 67, 67, 67, 66, 66, 66, 65, 65, 65, + 64, 64, 63, 63, 62, 62, 61, 61, 61, 61, 61, 73, 74, 76, 77, 79, 76, 74, + 72, 70, 69, 68, 67, 66, 66, 65, 65, 65, 64, 64, 64, 63, 63, 63, 62, 62, + 61, 61, 60, 60, 60, 60, 60, 72, 74, 75, 77, 78, 76, 74, 72, 70, 69, 68, + 67, 66, 65, 65, 64, 64, 64, 63, 63, 63, 62, 62, 61, 61, 61, 60, 60, 59, + 59, 59, 59, 72, 73, 75, 76, 78, 76, 74, 72, 70, 69, 68, 67, 65, 65, 64, + 64, 63, 63, 62, 62, 62, 61, 61, 61, 60, 60, 59, 59, 59, 59, 59, 59, 71, + 73, 74, 76, 78, 76, 74, 72, 70, 69, 68, 66, 65, 64, 64, 63, 62, 62, 62, + 61, 61, 60, 60, 60, 59, 59, 59, 58, 58, 58, 58, 58, 71, 72, 74, 76, 77, + 76, 74, 72, 70, 69, 67, 66, 65, 64, 63, 62, 62, 61, 61, 60, 60, 60, 59, + 59, 59, 58, 58, 58, 57, 57, 57, 57, 70, 71, 73, 75, 76, 75, 73, 71, 70, + 68, 67, 66, 64, 64, 63, 62, 61, 61, 60, 60, 59, 59, 59, 58, 58, 58, 57, + 57, 57, 57, 57, 57, 69, 70, 72, 74, 75, 74, 72, 71, 69, 68, 67, 65, 64, + 63, 62, 62, 61, 60, 60, 59, 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 56, + 56, 68, 70, 71, 73, 75, 73, 72, 70, 69, 68, 66, 65, 64, 63, 62, 61, 60, + 60, 59, 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 56, 56, 56, 67, 69, 70, + 72, 74, 72, 71, 70, 68, 67, 66, 65, 63, 63, 62, 61, 60, 59, 59, 58, 58, + 57, 57, 57, 56, 56, 56, 55, 55, 55, 55, 55, 66, 68, 69, 71, 73, 71, 70, + 69, 68, 67, 65, 64, 63, 62, 61, 60, 60, 59, 58, 58, 57, 57, 57, 56, 56, + 56, 55, 55, 55, 55, 55, 55, 65, 67, 68, 70, 71, 70, 69, 68, 67, 66, 65, + 64, 63, 62, 61, 60, 59, 59, 58, 58, 57, 57, 56, 56, 55, 55, 55, 55, 54, + 54, 54, 54, 64, 66, 67, 69, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, 61, + 60, 59, 58, 58, 57, 57, 56, 56, 55, 55, 55, 55, 54, 54, 54, 54, 54, 63, + 65, 66, 68, 69, 68, 68, 67, 66, 65, 64, 63, 62, 61, 60, 59, 59, 58, 57, + 57, 56, 56, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 63, 64, 65, 67, 68, + 68, 67, 66, 65, 64, 63, 62, 61, 61, 60, 59, 58, 58, 57, 57, 56, 56, 55, + 55, 54, 54, 54, 54, 53, 53, 53, 53, 62, 63, 65, 66, 67, 67, 66, 65, 64, + 63, 63, 62, 61, 60, 59, 59, 58, 57, 57, 56, 56, 55, 55, 55, 54, 54, 54, + 53, 53, 53, 53, 53, 61, 62, 64, 65, 66, 66, 65, 64, 64, 63, 62, 61, 60, + 60, 59, 58, 58, 57, 56, 56, 55, 55, 55, 54, 54, 54, 53, 53, 53, 53, 53, + 53, 60, 61, 63, 64, 65, 65, 64, 64, 63, 62, 61, 61, 60, 59, 59, 58, 57, + 57, 56, 56, 55, 55, 54, 54, 54, 53, 53, 53, 52, 52, 52, 52, 60, 61, 63, + 64, 65, 65, 64, 64, 63, 62, 61, 61, 60, 59, 59, 58, 57, 57, 56, 56, 55, + 55, 54, 54, 54, 53, 53, 53, 52, 52, 52, 52, 60, 61, 63, 64, 65, 65, 64, + 64, 63, 62, 61, 61, 60, 59, 59, 58, 57, 57, 56, 56, 55, 55, 54, 54, 54, + 53, 53, 53, 52, 52, 52, 52, 60, 61, 63, 64, 65, 65, 64, 64, 63, 62, 61, + 61, 60, 59, 59, 58, 57, 57, 56, 56, 55, 55, 54, 54, 54, 53, 53, 53, 52, + 52, 52, 52 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 61, 49, 42, 61, 51, 45, 41, 49, 45, 41, 39, 42, 41, 39, 38, + /* Size 8 */ + 64, 74, 71, 62, 55, 50, 47, 45, 74, 69, 70, 65, 58, 53, 49, 46, 71, 70, + 60, 56, 53, 50, 47, 45, 62, 65, 56, 51, 49, 47, 45, 44, 55, 58, 53, 49, + 46, 45, 44, 43, 50, 53, 50, 47, 45, 44, 43, 42, 47, 49, 47, 45, 44, 43, + 42, 42, 45, 46, 45, 44, 43, 42, 42, 41, + /* Size 16 */ + 64, 69, 74, 73, 71, 66, 62, 58, 55, 52, 50, 48, 47, 46, 45, 45, 69, 70, + 71, 71, 71, 67, 63, 60, 56, 54, 51, 49, 48, 47, 45, 45, 74, 71, 69, 70, + 70, 68, 65, 62, 58, 55, 53, 51, 49, 48, 46, 46, 73, 71, 70, 67, 65, 63, + 61, 58, 55, 53, 51, 50, 48, 47, 46, 46, 71, 71, 70, 65, 60, 58, 56, 54, + 53, 51, 50, 48, 47, 46, 45, 45, 66, 67, 68, 63, 58, 56, 54, 52, 51, 50, + 48, 47, 46, 45, 45, 45, 62, 63, 65, 61, 56, 54, 51, 50, 49, 48, 47, 46, + 45, 45, 44, 44, 58, 60, 62, 58, 54, 52, 50, 49, 48, 47, 46, 45, 45, 44, + 44, 44, 55, 56, 58, 55, 53, 51, 49, 48, 46, 46, 45, 44, 44, 43, 43, 43, + 52, 54, 55, 53, 51, 50, 48, 47, 46, 45, 44, 44, 43, 43, 43, 43, 50, 51, + 53, 51, 50, 48, 47, 46, 45, 44, 44, 43, 43, 43, 42, 42, 48, 49, 51, 50, + 48, 47, 46, 45, 44, 44, 43, 43, 42, 42, 42, 42, 47, 48, 49, 48, 47, 46, + 45, 45, 44, 43, 43, 42, 42, 42, 42, 42, 46, 47, 48, 47, 46, 45, 45, 44, + 43, 43, 43, 42, 42, 42, 41, 41, 45, 45, 46, 46, 45, 45, 44, 44, 43, 43, + 42, 42, 42, 41, 41, 41, 45, 45, 46, 46, 45, 45, 44, 44, 43, 43, 42, 42, + 42, 41, 41, 41, + /* Size 32 */ + 64, 67, 69, 72, 74, 73, 73, 72, 71, 69, 66, 64, 62, 60, 58, 57, 55, 53, + 52, 51, 50, 49, 48, 47, 47, 46, 46, 45, 45, 45, 45, 45, 67, 68, 70, 71, + 73, 72, 72, 71, 71, 69, 67, 65, 63, 61, 59, 57, 56, 54, 53, 52, 51, 50, + 49, 48, 47, 47, 46, 45, 45, 45, 45, 45, 69, 70, 70, 71, 71, 71, 71, 71, + 71, 69, 67, 65, 63, 62, 60, 58, 56, 55, 54, 53, 51, 50, 49, 49, 48, 47, + 47, 46, 45, 45, 45, 45, 72, 71, 71, 71, 70, 70, 70, 70, 71, 69, 67, 66, + 64, 63, 61, 59, 57, 56, 55, 53, 52, 51, 50, 49, 48, 48, 47, 46, 46, 46, + 46, 46, 74, 73, 71, 70, 69, 69, 70, 70, 70, 69, 68, 66, 65, 63, 62, 60, + 58, 57, 55, 54, 53, 52, 51, 50, 49, 48, 48, 47, 46, 46, 46, 46, 73, 72, + 71, 70, 69, 69, 69, 68, 68, 67, 65, 64, 63, 61, 60, 58, 57, 56, 54, 53, + 52, 51, 50, 49, 48, 48, 47, 47, 46, 46, 46, 46, 73, 72, 71, 70, 70, 69, + 67, 66, 65, 64, 63, 62, 61, 59, 58, 57, 55, 54, 53, 52, 51, 50, 50, 49, + 48, 47, 47, 46, 46, 46, 46, 46, 72, 71, 71, 70, 70, 68, 66, 64, 62, 61, + 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 50, 49, 48, 48, 47, 47, 46, + 45, 45, 45, 45, 71, 71, 71, 71, 70, 68, 65, 62, 60, 59, 58, 57, 56, 55, + 54, 54, 53, 52, 51, 50, 50, 49, 48, 48, 47, 47, 46, 46, 45, 45, 45, 45, + 69, 69, 69, 69, 69, 67, 64, 61, 59, 58, 57, 56, 55, 54, 53, 53, 52, 51, + 50, 50, 49, 48, 48, 47, 47, 46, 46, 45, 45, 45, 45, 45, 66, 67, 67, 67, + 68, 65, 63, 60, 58, 57, 56, 55, 54, 53, 52, 51, 51, 50, 50, 49, 48, 48, + 47, 47, 46, 46, 45, 45, 45, 45, 45, 45, 64, 65, 65, 66, 66, 64, 62, 59, + 57, 56, 55, 54, 52, 52, 51, 50, 50, 49, 49, 48, 48, 47, 47, 46, 46, 45, + 45, 45, 44, 44, 44, 44, 62, 63, 63, 64, 65, 63, 61, 58, 56, 55, 54, 52, + 51, 51, 50, 49, 49, 48, 48, 47, 47, 46, 46, 46, 45, 45, 45, 44, 44, 44, + 44, 44, 60, 61, 62, 63, 63, 61, 59, 57, 55, 54, 53, 52, 51, 50, 49, 49, + 48, 48, 47, 47, 46, 46, 46, 45, 45, 45, 44, 44, 44, 44, 44, 44, 58, 59, + 60, 61, 62, 60, 58, 56, 54, 53, 52, 51, 50, 49, 49, 48, 48, 47, 47, 46, + 46, 46, 45, 45, 45, 44, 44, 44, 44, 44, 44, 44, 57, 57, 58, 59, 60, 58, + 57, 55, 54, 53, 51, 50, 49, 49, 48, 48, 47, 47, 46, 46, 45, 45, 45, 45, + 44, 44, 44, 44, 43, 43, 43, 43, 55, 56, 56, 57, 58, 57, 55, 54, 53, 52, + 51, 50, 49, 48, 48, 47, 46, 46, 46, 45, 45, 45, 44, 44, 44, 44, 43, 43, + 43, 43, 43, 43, 53, 54, 55, 56, 57, 56, 54, 53, 52, 51, 50, 49, 48, 48, + 47, 47, 46, 46, 45, 45, 45, 44, 44, 44, 44, 43, 43, 43, 43, 43, 43, 43, + 52, 53, 54, 55, 55, 54, 53, 52, 51, 50, 50, 49, 48, 47, 47, 46, 46, 45, + 45, 45, 44, 44, 44, 44, 43, 43, 43, 43, 43, 43, 43, 43, 51, 52, 53, 53, + 54, 53, 52, 51, 50, 50, 49, 48, 47, 47, 46, 46, 45, 45, 45, 44, 44, 44, + 44, 43, 43, 43, 43, 43, 42, 42, 42, 42, 50, 51, 51, 52, 53, 52, 51, 50, + 50, 49, 48, 48, 47, 46, 46, 45, 45, 45, 44, 44, 44, 43, 43, 43, 43, 43, + 43, 42, 42, 42, 42, 42, 49, 50, 50, 51, 52, 51, 50, 50, 49, 48, 48, 47, + 46, 46, 46, 45, 45, 44, 44, 44, 43, 43, 43, 43, 43, 43, 42, 42, 42, 42, + 42, 42, 48, 49, 49, 50, 51, 50, 50, 49, 48, 48, 47, 47, 46, 46, 45, 45, + 44, 44, 44, 44, 43, 43, 43, 43, 42, 42, 42, 42, 42, 42, 42, 42, 47, 48, + 49, 49, 50, 49, 49, 48, 48, 47, 47, 46, 46, 45, 45, 45, 44, 44, 44, 43, + 43, 43, 43, 43, 42, 42, 42, 42, 42, 42, 42, 42, 47, 47, 48, 48, 49, 48, + 48, 48, 47, 47, 46, 46, 45, 45, 45, 44, 44, 44, 43, 43, 43, 43, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 46, 47, 47, 48, 48, 48, 47, 47, 47, 46, + 46, 45, 45, 45, 44, 44, 44, 43, 43, 43, 43, 43, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 46, 46, 47, 47, 48, 47, 47, 47, 46, 46, 45, 45, 45, 44, + 44, 44, 43, 43, 43, 43, 43, 42, 42, 42, 42, 42, 42, 42, 41, 41, 41, 41, + 45, 45, 46, 46, 47, 47, 46, 46, 46, 45, 45, 45, 44, 44, 44, 44, 43, 43, + 43, 43, 42, 42, 42, 42, 42, 42, 42, 41, 41, 41, 41, 41, 45, 45, 45, 46, + 46, 46, 46, 45, 45, 45, 45, 44, 44, 44, 44, 43, 43, 43, 43, 42, 42, 42, + 42, 42, 42, 42, 41, 41, 41, 41, 41, 41, 45, 45, 45, 46, 46, 46, 46, 45, + 45, 45, 45, 44, 44, 44, 44, 43, 43, 43, 43, 42, 42, 42, 42, 42, 42, 42, + 41, 41, 41, 41, 41, 41, 45, 45, 45, 46, 46, 46, 46, 45, 45, 45, 45, 44, + 44, 44, 44, 43, 43, 43, 43, 42, 42, 42, 42, 42, 42, 42, 41, 41, 41, 41, + 41, 41, 45, 45, 45, 46, 46, 46, 46, 45, 45, 45, 45, 44, 44, 44, 44, 43, + 43, 43, 43, 42, 42, 42, 42, 42, 42, 42, 41, 41, 41, 41, 41, 41 }, + { /* Intra matrices */ + /* Size 4 */ + 93, 88, 70, 60, 88, 73, 64, 58, 70, 64, 58, 55, 60, 58, 55, 53, + /* Size 8 */ + 84, 98, 94, 82, 71, 65, 60, 57, 98, 91, 93, 86, 76, 69, 63, 60, 94, 93, + 79, 73, 69, 65, 61, 58, 82, 86, 73, 67, 63, 61, 58, 57, 71, 76, 69, 63, + 60, 58, 56, 55, 65, 69, 65, 61, 58, 56, 55, 54, 60, 63, 61, 58, 56, 55, + 54, 53, 57, 60, 58, 57, 55, 54, 53, 53, + /* Size 16 */ + 86, 93, 100, 98, 96, 89, 83, 78, 73, 69, 66, 63, 61, 60, 58, 58, 93, 94, + 96, 96, 95, 90, 85, 80, 75, 71, 68, 65, 63, 61, 59, 59, 100, 96, 93, 94, + 95, 91, 87, 82, 77, 74, 70, 67, 64, 62, 61, 61, 98, 96, 94, 90, 87, 84, + 81, 77, 74, 71, 68, 65, 63, 61, 60, 60, 96, 95, 95, 87, 80, 77, 74, 72, + 70, 68, 66, 64, 62, 61, 59, 59, 89, 90, 91, 84, 77, 74, 71, 69, 67, 65, + 64, 62, 61, 59, 58, 58, 83, 85, 87, 81, 74, 71, 68, 66, 64, 63, 62, 60, + 59, 58, 58, 58, 78, 80, 82, 77, 72, 69, 66, 64, 62, 61, 60, 59, 58, 58, + 57, 57, 73, 75, 77, 74, 70, 67, 64, 62, 61, 60, 59, 58, 57, 57, 56, 56, + 69, 71, 74, 71, 68, 65, 63, 61, 60, 59, 58, 57, 57, 56, 56, 56, 66, 68, + 70, 68, 66, 64, 62, 60, 59, 58, 57, 56, 56, 55, 55, 55, 63, 65, 67, 65, + 64, 62, 60, 59, 58, 57, 56, 56, 55, 55, 55, 55, 61, 63, 64, 63, 62, 61, + 59, 58, 57, 57, 56, 55, 55, 55, 54, 54, 60, 61, 62, 61, 61, 59, 58, 58, + 57, 56, 55, 55, 55, 54, 54, 54, 58, 59, 61, 60, 59, 58, 58, 57, 56, 56, + 55, 55, 54, 54, 54, 54, 58, 59, 61, 60, 59, 58, 58, 57, 56, 56, 55, 55, + 54, 54, 54, 54, + /* Size 32 */ + 86, 90, 93, 97, 101, 100, 98, 97, 96, 93, 90, 87, 83, 81, 78, 76, 73, + 71, 70, 68, 66, 65, 64, 63, 62, 61, 60, 59, 59, 59, 59, 59, 90, 92, 94, + 97, 99, 98, 97, 97, 96, 93, 90, 87, 84, 82, 79, 77, 74, 73, 71, 69, 67, + 66, 65, 64, 62, 62, 61, 60, 59, 59, 59, 59, 93, 94, 95, 96, 97, 97, 96, + 96, 96, 93, 91, 88, 86, 83, 81, 78, 76, 74, 72, 70, 68, 67, 66, 64, 63, + 62, 62, 61, 60, 60, 60, 60, 97, 97, 96, 96, 95, 95, 95, 96, 96, 93, 91, + 89, 87, 84, 82, 79, 77, 75, 73, 71, 69, 68, 67, 65, 64, 63, 62, 61, 60, + 60, 60, 60, 101, 99, 97, 95, 93, 94, 94, 95, 95, 94, 92, 90, 88, 85, 83, + 80, 78, 76, 74, 72, 70, 69, 68, 66, 65, 64, 63, 62, 61, 61, 61, 61, 100, + 98, 97, 95, 94, 93, 93, 92, 92, 90, 88, 86, 85, 82, 80, 78, 76, 74, 73, + 71, 69, 68, 67, 65, 64, 63, 62, 62, 61, 61, 61, 61, 98, 97, 96, 95, 94, + 93, 91, 90, 88, 86, 85, 83, 81, 80, 78, 76, 74, 73, 71, 70, 68, 67, 66, + 65, 64, 63, 62, 61, 60, 60, 60, 60, 97, 97, 96, 96, 95, 92, 90, 87, 84, + 83, 81, 80, 78, 77, 75, 74, 72, 71, 70, 68, 67, 66, 65, 64, 63, 62, 61, + 61, 60, 60, 60, 60, 96, 96, 96, 96, 95, 92, 88, 84, 80, 79, 78, 76, 75, + 74, 73, 72, 70, 69, 68, 67, 66, 65, 64, 63, 62, 62, 61, 60, 60, 60, 60, + 60, 93, 93, 93, 93, 94, 90, 86, 83, 79, 78, 76, 75, 73, 72, 71, 70, 69, + 68, 67, 66, 65, 64, 63, 63, 62, 61, 60, 60, 59, 59, 59, 59, 90, 90, 91, + 91, 92, 88, 85, 81, 78, 76, 75, 73, 72, 71, 70, 69, 67, 67, 66, 65, 64, + 63, 63, 62, 61, 61, 60, 59, 59, 59, 59, 59, 87, 87, 88, 89, 90, 86, 83, + 80, 76, 75, 73, 72, 70, 69, 68, 67, 66, 65, 65, 64, 63, 62, 62, 61, 60, + 60, 59, 59, 58, 58, 58, 58, 83, 84, 86, 87, 88, 85, 81, 78, 75, 73, 72, + 70, 68, 67, 66, 66, 65, 64, 63, 63, 62, 61, 61, 60, 60, 59, 59, 58, 58, + 58, 58, 58, 81, 82, 83, 84, 85, 82, 80, 77, 74, 72, 71, 69, 67, 66, 66, + 65, 64, 63, 63, 62, 61, 61, 60, 60, 59, 59, 58, 58, 58, 58, 58, 58, 78, + 79, 81, 82, 83, 80, 78, 75, 73, 71, 70, 68, 66, 66, 65, 64, 63, 62, 62, + 61, 61, 60, 60, 59, 59, 58, 58, 58, 57, 57, 57, 57, 76, 77, 78, 79, 80, + 78, 76, 74, 72, 70, 69, 67, 66, 65, 64, 63, 62, 62, 61, 60, 60, 59, 59, + 59, 58, 58, 58, 57, 57, 57, 57, 57, 73, 74, 76, 77, 78, 76, 74, 72, 70, + 69, 67, 66, 65, 64, 63, 62, 61, 61, 60, 60, 59, 59, 58, 58, 58, 57, 57, + 57, 57, 57, 57, 57, 71, 73, 74, 75, 76, 74, 73, 71, 69, 68, 67, 65, 64, + 63, 62, 62, 61, 60, 60, 59, 59, 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, + 56, 70, 71, 72, 73, 74, 73, 71, 70, 68, 67, 66, 65, 63, 63, 62, 61, 60, + 60, 59, 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 56, 56, 56, 68, 69, 70, + 71, 72, 71, 70, 68, 67, 66, 65, 64, 63, 62, 61, 60, 60, 59, 59, 58, 58, + 58, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, 66, 67, 68, 69, 70, 69, 68, + 67, 66, 65, 64, 63, 62, 61, 61, 60, 59, 59, 58, 58, 57, 57, 57, 57, 56, + 56, 56, 56, 55, 55, 55, 55, 65, 66, 67, 68, 69, 68, 67, 66, 65, 64, 63, + 62, 61, 61, 60, 59, 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 56, 55, 55, + 55, 55, 55, 64, 65, 66, 67, 68, 67, 66, 65, 64, 63, 63, 62, 61, 60, 60, + 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 63, + 64, 64, 65, 66, 65, 65, 64, 63, 63, 62, 61, 60, 60, 59, 59, 58, 58, 57, + 57, 57, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 62, 62, 63, 64, 65, + 64, 64, 63, 62, 62, 61, 60, 60, 59, 59, 58, 58, 57, 57, 57, 56, 56, 56, + 56, 55, 55, 55, 55, 55, 55, 55, 55, 61, 62, 62, 63, 64, 63, 63, 62, 62, + 61, 61, 60, 59, 59, 58, 58, 57, 57, 57, 56, 56, 56, 56, 55, 55, 55, 55, + 55, 54, 54, 54, 54, 60, 61, 62, 62, 63, 62, 62, 61, 61, 60, 60, 59, 59, + 58, 58, 58, 57, 57, 56, 56, 56, 56, 55, 55, 55, 55, 55, 54, 54, 54, 54, + 54, 59, 60, 61, 61, 62, 62, 61, 61, 60, 60, 59, 59, 58, 58, 58, 57, 57, + 57, 56, 56, 56, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 59, 59, 60, + 60, 61, 61, 60, 60, 60, 59, 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 55, + 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 59, 59, 60, 60, 61, 61, 60, + 60, 60, 59, 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 55, 55, 55, 55, 55, + 54, 54, 54, 54, 54, 54, 54, 59, 59, 60, 60, 61, 61, 60, 60, 60, 59, 59, + 58, 58, 58, 57, 57, 57, 56, 56, 56, 55, 55, 55, 55, 55, 54, 54, 54, 54, + 54, 54, 54, 59, 59, 60, 60, 61, 61, 60, 60, 60, 59, 59, 58, 58, 58, 57, + 57, 57, 56, 56, 56, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 51, 50, 45, 51, 48, 47, 44, 50, 47, 43, 41, 45, 44, 41, 40, + /* Size 8 */ + 64, 72, 54, 52, 51, 49, 47, 45, 72, 59, 53, 55, 55, 53, 50, 48, 54, 53, + 50, 51, 51, 50, 48, 47, 52, 55, 51, 48, 48, 47, 46, 45, 51, 55, 51, 48, + 46, 45, 44, 43, 49, 53, 50, 47, 45, 44, 43, 42, 47, 50, 48, 46, 44, 43, + 42, 41, 45, 48, 47, 45, 43, 42, 41, 41, + /* Size 16 */ + 64, 68, 72, 63, 54, 53, 52, 52, 51, 50, 49, 48, 47, 46, 45, 45, 68, 67, + 66, 59, 53, 54, 54, 53, 53, 52, 51, 50, 48, 47, 46, 46, 72, 66, 59, 56, + 53, 54, 55, 55, 55, 54, 53, 51, 50, 49, 48, 48, 63, 59, 56, 54, 52, 52, + 53, 53, 53, 52, 51, 50, 49, 48, 47, 47, 54, 53, 53, 52, 50, 50, 51, 51, + 51, 50, 50, 49, 48, 47, 47, 47, 53, 54, 54, 52, 50, 50, 49, 49, 49, 49, + 48, 48, 47, 46, 46, 46, 52, 54, 55, 53, 51, 49, 48, 48, 48, 47, 47, 46, + 46, 45, 45, 45, 52, 53, 55, 53, 51, 49, 48, 47, 47, 46, 46, 45, 45, 45, + 44, 44, 51, 53, 55, 53, 51, 49, 48, 47, 46, 45, 45, 44, 44, 44, 43, 43, + 50, 52, 54, 52, 50, 49, 47, 46, 45, 45, 44, 44, 43, 43, 43, 43, 49, 51, + 53, 51, 50, 48, 47, 46, 45, 44, 44, 43, 43, 42, 42, 42, 48, 50, 51, 50, + 49, 48, 46, 45, 44, 44, 43, 43, 42, 42, 42, 42, 47, 48, 50, 49, 48, 47, + 46, 45, 44, 43, 43, 42, 42, 42, 41, 41, 46, 47, 49, 48, 47, 46, 45, 45, + 44, 43, 42, 42, 42, 41, 41, 41, 45, 46, 48, 47, 47, 46, 45, 44, 43, 43, + 42, 42, 41, 41, 41, 41, 45, 46, 48, 47, 47, 46, 45, 44, 43, 43, 42, 42, + 41, 41, 41, 41, + /* Size 32 */ + 64, 66, 68, 70, 72, 67, 63, 58, 54, 53, 53, 53, 52, 52, 52, 51, 51, 50, + 50, 49, 49, 48, 48, 47, 47, 46, 46, 45, 45, 45, 45, 45, 66, 67, 67, 68, + 69, 65, 61, 57, 53, 53, 53, 53, 53, 53, 52, 52, 52, 51, 51, 50, 50, 49, + 49, 48, 48, 47, 47, 46, 46, 46, 46, 46, 68, 67, 67, 66, 66, 63, 59, 56, + 53, 53, 54, 54, 54, 54, 53, 53, 53, 52, 52, 51, 51, 50, 50, 49, 48, 48, + 47, 47, 46, 46, 46, 46, 70, 68, 66, 64, 62, 60, 58, 56, 53, 54, 54, 54, + 55, 54, 54, 54, 54, 53, 53, 52, 52, 51, 50, 50, 49, 49, 48, 48, 47, 47, + 47, 47, 72, 69, 66, 62, 59, 58, 56, 55, 53, 54, 54, 55, 55, 55, 55, 55, + 55, 54, 54, 53, 53, 52, 51, 51, 50, 50, 49, 48, 48, 48, 48, 48, 67, 65, + 63, 60, 58, 56, 55, 54, 52, 53, 53, 54, 54, 54, 54, 54, 54, 53, 53, 52, + 52, 51, 51, 50, 50, 49, 49, 48, 48, 48, 48, 48, 63, 61, 59, 58, 56, 55, + 54, 53, 52, 52, 52, 53, 53, 53, 53, 53, 53, 52, 52, 52, 51, 51, 50, 50, + 49, 49, 48, 48, 47, 47, 47, 47, 58, 57, 56, 56, 55, 54, 53, 52, 51, 51, + 51, 52, 52, 52, 52, 52, 52, 51, 51, 51, 50, 50, 50, 49, 49, 48, 48, 47, + 47, 47, 47, 47, 54, 53, 53, 53, 53, 52, 52, 51, 50, 50, 50, 50, 51, 51, + 51, 51, 51, 50, 50, 50, 50, 49, 49, 49, 48, 48, 47, 47, 47, 47, 47, 47, + 53, 53, 53, 54, 54, 53, 52, 51, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, + 49, 49, 49, 49, 48, 48, 48, 47, 47, 46, 46, 46, 46, 46, 53, 53, 54, 54, + 54, 53, 52, 51, 50, 50, 50, 50, 49, 49, 49, 49, 49, 49, 49, 48, 48, 48, + 48, 47, 47, 47, 46, 46, 46, 46, 46, 46, 53, 53, 54, 54, 55, 54, 53, 52, + 50, 50, 50, 49, 49, 49, 49, 48, 48, 48, 48, 48, 48, 47, 47, 47, 46, 46, + 46, 46, 45, 45, 45, 45, 52, 53, 54, 55, 55, 54, 53, 52, 51, 50, 49, 49, + 48, 48, 48, 48, 48, 47, 47, 47, 47, 47, 46, 46, 46, 46, 45, 45, 45, 45, + 45, 45, 52, 53, 54, 54, 55, 54, 53, 52, 51, 50, 49, 49, 48, 48, 48, 47, + 47, 47, 47, 47, 46, 46, 46, 46, 45, 45, 45, 45, 44, 44, 44, 44, 52, 52, + 53, 54, 55, 54, 53, 52, 51, 50, 49, 49, 48, 48, 47, 47, 47, 46, 46, 46, + 46, 46, 45, 45, 45, 45, 45, 44, 44, 44, 44, 44, 51, 52, 53, 54, 55, 54, + 53, 52, 51, 50, 49, 48, 48, 47, 47, 47, 46, 46, 46, 46, 45, 45, 45, 45, + 45, 44, 44, 44, 44, 44, 44, 44, 51, 52, 53, 54, 55, 54, 53, 52, 51, 50, + 49, 48, 48, 47, 47, 46, 46, 46, 45, 45, 45, 45, 44, 44, 44, 44, 44, 44, + 43, 43, 43, 43, 50, 51, 52, 53, 54, 53, 52, 51, 50, 50, 49, 48, 47, 47, + 46, 46, 46, 45, 45, 45, 45, 44, 44, 44, 44, 44, 43, 43, 43, 43, 43, 43, + 50, 51, 52, 53, 54, 53, 52, 51, 50, 49, 49, 48, 47, 47, 46, 46, 45, 45, + 45, 44, 44, 44, 44, 44, 43, 43, 43, 43, 43, 43, 43, 43, 49, 50, 51, 52, + 53, 52, 52, 51, 50, 49, 48, 48, 47, 47, 46, 46, 45, 45, 44, 44, 44, 44, + 43, 43, 43, 43, 43, 43, 42, 42, 42, 42, 49, 50, 51, 52, 53, 52, 51, 50, + 50, 49, 48, 48, 47, 46, 46, 45, 45, 45, 44, 44, 44, 43, 43, 43, 43, 43, + 42, 42, 42, 42, 42, 42, 48, 49, 50, 51, 52, 51, 51, 50, 49, 49, 48, 47, + 47, 46, 46, 45, 45, 44, 44, 44, 43, 43, 43, 43, 43, 42, 42, 42, 42, 42, + 42, 42, 48, 49, 50, 50, 51, 51, 50, 50, 49, 48, 48, 47, 46, 46, 45, 45, + 44, 44, 44, 43, 43, 43, 43, 43, 42, 42, 42, 42, 42, 42, 42, 42, 47, 48, + 49, 50, 51, 50, 50, 49, 49, 48, 47, 47, 46, 46, 45, 45, 44, 44, 44, 43, + 43, 43, 43, 42, 42, 42, 42, 42, 41, 41, 41, 41, 47, 48, 48, 49, 50, 50, + 49, 49, 48, 48, 47, 46, 46, 45, 45, 45, 44, 44, 43, 43, 43, 43, 42, 42, + 42, 42, 42, 41, 41, 41, 41, 41, 46, 47, 48, 49, 50, 49, 49, 48, 48, 47, + 47, 46, 46, 45, 45, 44, 44, 44, 43, 43, 43, 42, 42, 42, 42, 42, 41, 41, + 41, 41, 41, 41, 46, 47, 47, 48, 49, 49, 48, 48, 47, 47, 46, 46, 45, 45, + 45, 44, 44, 43, 43, 43, 42, 42, 42, 42, 42, 41, 41, 41, 41, 41, 41, 41, + 45, 46, 47, 48, 48, 48, 48, 47, 47, 46, 46, 46, 45, 45, 44, 44, 44, 43, + 43, 43, 42, 42, 42, 42, 41, 41, 41, 41, 41, 41, 41, 41, 45, 46, 46, 47, + 48, 48, 47, 47, 47, 46, 46, 45, 45, 44, 44, 44, 43, 43, 43, 42, 42, 42, + 42, 41, 41, 41, 41, 41, 41, 41, 41, 41, 45, 46, 46, 47, 48, 48, 47, 47, + 47, 46, 46, 45, 45, 44, 44, 44, 43, 43, 43, 42, 42, 42, 42, 41, 41, 41, + 41, 41, 41, 41, 41, 41, 45, 46, 46, 47, 48, 48, 47, 47, 47, 46, 46, 45, + 45, 44, 44, 44, 43, 43, 43, 42, 42, 42, 42, 41, 41, 41, 41, 41, 41, 41, + 41, 41, 45, 46, 46, 47, 48, 48, 47, 47, 47, 46, 46, 45, 45, 44, 44, 44, + 43, 43, 43, 42, 42, 42, 42, 41, 41, 41, 41, 41, 41, 41, 41, 41 }, + { /* Intra matrices */ + /* Size 4 */ + 90, 72, 69, 63, 72, 66, 65, 62, 69, 65, 59, 57, 63, 62, 57, 54, + /* Size 8 */ + 87, 98, 72, 70, 68, 65, 62, 60, 98, 80, 71, 74, 74, 70, 67, 64, 72, 71, + 67, 68, 68, 66, 64, 62, 70, 74, 68, 64, 63, 62, 61, 59, 68, 74, 68, 63, + 61, 59, 58, 57, 65, 70, 66, 62, 59, 58, 57, 56, 62, 67, 64, 61, 58, 57, + 55, 54, 60, 64, 62, 59, 57, 56, 54, 53, + /* Size 16 */ + 88, 93, 99, 86, 73, 72, 71, 70, 69, 67, 66, 64, 63, 62, 60, 60, 93, 92, + 90, 81, 72, 73, 73, 72, 72, 70, 69, 67, 65, 64, 62, 62, 99, 90, 81, 76, + 72, 74, 75, 75, 74, 73, 71, 69, 68, 66, 65, 65, 86, 81, 76, 73, 70, 71, + 72, 72, 71, 70, 69, 68, 66, 65, 64, 64, 73, 72, 72, 70, 67, 68, 68, 68, + 68, 68, 67, 66, 65, 64, 63, 63, 72, 73, 74, 71, 68, 67, 67, 66, 66, 66, + 65, 64, 63, 62, 61, 61, 71, 73, 75, 72, 68, 67, 65, 65, 64, 63, 63, 62, + 62, 61, 60, 60, 70, 72, 75, 72, 68, 66, 65, 64, 63, 62, 62, 61, 60, 60, + 59, 59, 69, 72, 74, 71, 68, 66, 64, 63, 61, 61, 60, 60, 59, 58, 58, 58, + 67, 70, 73, 70, 68, 66, 63, 62, 61, 60, 59, 59, 58, 58, 57, 57, 66, 69, + 71, 69, 67, 65, 63, 62, 60, 59, 58, 58, 57, 57, 56, 56, 64, 67, 69, 68, + 66, 64, 62, 61, 60, 59, 58, 57, 56, 56, 56, 56, 63, 65, 68, 66, 65, 63, + 62, 60, 59, 58, 57, 56, 56, 55, 55, 55, 62, 64, 66, 65, 64, 62, 61, 60, + 58, 58, 57, 56, 55, 55, 54, 54, 60, 62, 65, 64, 63, 61, 60, 59, 58, 57, + 56, 56, 55, 54, 54, 54, 60, 62, 65, 64, 63, 61, 60, 59, 58, 57, 56, 56, + 55, 54, 54, 54, + /* Size 32 */ + 88, 91, 94, 97, 100, 93, 86, 80, 73, 73, 72, 72, 71, 71, 70, 70, 69, 68, + 68, 67, 66, 65, 65, 64, 63, 63, 62, 61, 61, 61, 61, 61, 91, 92, 93, 94, + 95, 89, 84, 78, 73, 73, 73, 72, 72, 72, 71, 71, 71, 70, 69, 68, 68, 67, + 66, 65, 64, 64, 63, 62, 62, 62, 62, 62, 94, 93, 92, 91, 90, 86, 82, 77, + 73, 73, 73, 73, 73, 73, 73, 72, 72, 71, 70, 70, 69, 68, 67, 66, 66, 65, + 64, 63, 63, 63, 63, 63, 97, 94, 91, 89, 86, 83, 79, 76, 73, 73, 74, 74, + 75, 74, 74, 74, 73, 73, 72, 71, 70, 69, 69, 68, 67, 66, 65, 65, 64, 64, + 64, 64, 100, 95, 90, 86, 81, 79, 77, 75, 72, 73, 74, 75, 76, 75, 75, 75, + 75, 74, 73, 72, 72, 71, 70, 69, 68, 67, 66, 66, 65, 65, 65, 65, 93, 89, + 86, 83, 79, 77, 75, 73, 71, 72, 73, 73, 74, 74, 74, 73, 73, 73, 72, 71, + 71, 70, 69, 68, 67, 67, 66, 65, 64, 64, 64, 64, 86, 84, 82, 79, 77, 75, + 73, 72, 70, 71, 71, 72, 72, 72, 72, 72, 72, 71, 71, 70, 70, 69, 68, 67, + 67, 66, 65, 65, 64, 64, 64, 64, 80, 78, 77, 76, 75, 73, 72, 70, 69, 69, + 70, 70, 70, 70, 70, 70, 70, 70, 69, 69, 68, 68, 67, 67, 66, 65, 65, 64, + 63, 63, 63, 63, 73, 73, 73, 73, 72, 71, 70, 69, 68, 68, 68, 68, 69, 69, + 69, 69, 69, 68, 68, 68, 67, 67, 66, 66, 65, 65, 64, 63, 63, 63, 63, 63, + 73, 73, 73, 73, 73, 72, 71, 69, 68, 68, 68, 68, 68, 68, 68, 68, 68, 67, + 67, 67, 66, 66, 65, 65, 64, 64, 63, 63, 62, 62, 62, 62, 72, 73, 73, 74, + 74, 73, 71, 70, 68, 68, 68, 67, 67, 67, 67, 67, 67, 66, 66, 66, 65, 65, + 64, 64, 64, 63, 63, 62, 62, 62, 62, 62, 72, 72, 73, 74, 75, 73, 72, 70, + 68, 68, 67, 67, 66, 66, 66, 66, 65, 65, 65, 65, 64, 64, 64, 63, 63, 62, + 62, 62, 61, 61, 61, 61, 71, 72, 73, 75, 76, 74, 72, 70, 69, 68, 67, 66, + 65, 65, 65, 65, 64, 64, 64, 64, 63, 63, 63, 62, 62, 62, 61, 61, 60, 60, + 60, 60, 71, 72, 73, 74, 75, 74, 72, 70, 69, 68, 67, 66, 65, 65, 64, 64, + 64, 63, 63, 63, 63, 62, 62, 62, 61, 61, 61, 60, 60, 60, 60, 60, 70, 71, + 73, 74, 75, 74, 72, 70, 69, 68, 67, 66, 65, 64, 64, 64, 63, 63, 62, 62, + 62, 62, 61, 61, 61, 60, 60, 60, 59, 59, 59, 59, 70, 71, 72, 74, 75, 73, + 72, 70, 69, 68, 67, 66, 65, 64, 64, 63, 62, 62, 62, 61, 61, 61, 61, 60, + 60, 60, 59, 59, 59, 59, 59, 59, 69, 71, 72, 73, 75, 73, 72, 70, 69, 68, + 67, 65, 64, 64, 63, 62, 62, 61, 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, + 58, 58, 58, 58, 68, 70, 71, 73, 74, 73, 71, 70, 68, 67, 66, 65, 64, 63, + 63, 62, 61, 61, 61, 60, 60, 60, 59, 59, 59, 59, 58, 58, 58, 58, 58, 58, + 68, 69, 70, 72, 73, 72, 71, 69, 68, 67, 66, 65, 64, 63, 62, 62, 61, 61, + 60, 60, 60, 59, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, 67, 68, 70, 71, + 72, 71, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, 61, 60, 60, 60, 59, 59, + 58, 58, 58, 58, 57, 57, 57, 57, 57, 57, 66, 68, 69, 70, 72, 71, 70, 68, + 67, 66, 65, 64, 63, 63, 62, 61, 60, 60, 60, 59, 59, 58, 58, 58, 57, 57, + 57, 57, 57, 57, 57, 57, 65, 67, 68, 69, 71, 70, 69, 68, 67, 66, 65, 64, + 63, 62, 62, 61, 60, 60, 59, 59, 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, + 56, 56, 65, 66, 67, 69, 70, 69, 68, 67, 66, 65, 64, 64, 63, 62, 61, 61, + 60, 59, 59, 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, 64, 65, + 66, 68, 69, 68, 67, 67, 66, 65, 64, 63, 62, 62, 61, 60, 60, 59, 59, 58, + 58, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, 56, 63, 64, 66, 67, 68, 67, + 67, 66, 65, 64, 64, 63, 62, 61, 61, 60, 59, 59, 58, 58, 57, 57, 57, 56, + 56, 56, 56, 55, 55, 55, 55, 55, 63, 64, 65, 66, 67, 67, 66, 65, 65, 64, + 63, 62, 62, 61, 60, 60, 59, 59, 58, 58, 57, 57, 57, 56, 56, 56, 55, 55, + 55, 55, 55, 55, 62, 63, 64, 65, 66, 66, 65, 65, 64, 63, 63, 62, 61, 61, + 60, 59, 59, 58, 58, 57, 57, 57, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, + 61, 62, 63, 65, 66, 65, 65, 64, 63, 63, 62, 62, 61, 60, 60, 59, 59, 58, + 58, 57, 57, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 61, 62, 63, 64, + 65, 64, 64, 63, 63, 62, 62, 61, 60, 60, 59, 59, 58, 58, 57, 57, 57, 56, + 56, 56, 55, 55, 55, 55, 54, 54, 54, 54, 61, 62, 63, 64, 65, 64, 64, 63, + 63, 62, 62, 61, 60, 60, 59, 59, 58, 58, 57, 57, 57, 56, 56, 56, 55, 55, + 55, 55, 54, 54, 54, 54, 61, 62, 63, 64, 65, 64, 64, 63, 63, 62, 62, 61, + 60, 60, 59, 59, 58, 58, 57, 57, 57, 56, 56, 56, 55, 55, 55, 55, 54, 54, + 54, 54, 61, 62, 63, 64, 65, 64, 64, 63, 63, 62, 62, 61, 60, 60, 59, 59, + 58, 58, 57, 57, 57, 56, 56, 56, 55, 55, 55, 55, 54, 54, 54, 54 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 62, 51, 46, 62, 53, 48, 45, 51, 48, 45, 43, 46, 45, 43, 42, + /* Size 8 */ + 64, 72, 70, 62, 56, 52, 50, 48, 72, 68, 69, 65, 59, 55, 52, 49, 70, 69, + 61, 57, 55, 52, 50, 49, 62, 65, 57, 53, 51, 50, 49, 48, 56, 59, 55, 51, + 49, 48, 47, 47, 52, 55, 52, 50, 48, 47, 47, 46, 50, 52, 50, 49, 47, 47, + 46, 46, 48, 49, 49, 48, 47, 46, 46, 45, + /* Size 16 */ + 64, 68, 72, 71, 70, 66, 62, 59, 56, 54, 52, 51, 50, 49, 48, 48, 68, 69, + 70, 70, 70, 67, 64, 61, 58, 56, 54, 52, 51, 50, 49, 49, 72, 70, 68, 69, + 69, 67, 65, 62, 59, 57, 55, 53, 52, 50, 49, 49, 71, 70, 69, 67, 65, 63, + 61, 59, 57, 55, 53, 52, 51, 50, 49, 49, 70, 70, 69, 65, 61, 59, 57, 56, + 55, 53, 52, 51, 50, 49, 49, 49, 66, 67, 67, 63, 59, 57, 55, 54, 53, 52, + 51, 50, 49, 49, 48, 48, 62, 64, 65, 61, 57, 55, 53, 52, 51, 51, 50, 49, + 49, 48, 48, 48, 59, 61, 62, 59, 56, 54, 52, 51, 50, 50, 49, 49, 48, 48, + 47, 47, 56, 58, 59, 57, 55, 53, 51, 50, 49, 49, 48, 48, 47, 47, 47, 47, + 54, 56, 57, 55, 53, 52, 51, 50, 49, 48, 48, 47, 47, 47, 46, 46, 52, 54, + 55, 53, 52, 51, 50, 49, 48, 48, 47, 47, 47, 46, 46, 46, 51, 52, 53, 52, + 51, 50, 49, 49, 48, 47, 47, 47, 46, 46, 46, 46, 50, 51, 52, 51, 50, 49, + 49, 48, 47, 47, 47, 46, 46, 46, 46, 46, 49, 50, 50, 50, 49, 49, 48, 48, + 47, 47, 46, 46, 46, 46, 45, 45, 48, 49, 49, 49, 49, 48, 48, 47, 47, 46, + 46, 46, 46, 45, 45, 45, 48, 49, 49, 49, 49, 48, 48, 47, 47, 46, 46, 46, + 46, 45, 45, 45, + /* Size 32 */ + 64, 66, 68, 70, 72, 72, 71, 70, 70, 68, 66, 64, 62, 61, 59, 58, 56, 55, + 54, 53, 52, 52, 51, 50, 50, 49, 49, 48, 48, 48, 48, 48, 66, 67, 69, 70, + 71, 71, 70, 70, 70, 68, 66, 65, 63, 61, 60, 59, 57, 56, 55, 54, 53, 52, + 52, 51, 50, 50, 49, 49, 48, 48, 48, 48, 68, 69, 69, 70, 70, 70, 70, 70, + 70, 68, 67, 65, 64, 62, 61, 59, 58, 57, 56, 55, 54, 53, 52, 51, 51, 50, + 50, 49, 49, 49, 49, 49, 70, 70, 70, 69, 69, 69, 69, 69, 69, 68, 67, 66, + 64, 63, 61, 60, 58, 57, 56, 55, 54, 53, 53, 52, 51, 51, 50, 49, 49, 49, + 49, 49, 72, 71, 70, 69, 68, 68, 69, 69, 69, 68, 67, 66, 65, 63, 62, 61, + 59, 58, 57, 56, 55, 54, 53, 52, 52, 51, 50, 50, 49, 49, 49, 49, 72, 71, + 70, 69, 68, 68, 68, 67, 67, 66, 65, 64, 63, 62, 61, 59, 58, 57, 56, 55, + 54, 53, 53, 52, 51, 51, 50, 50, 49, 49, 49, 49, 71, 70, 70, 69, 69, 68, + 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 53, 52, 51, + 51, 50, 50, 49, 49, 49, 49, 49, 70, 70, 70, 69, 69, 67, 66, 64, 63, 62, + 61, 60, 59, 58, 58, 57, 56, 55, 54, 54, 53, 52, 52, 51, 50, 50, 50, 49, + 49, 49, 49, 49, 70, 70, 70, 69, 69, 67, 65, 63, 61, 60, 59, 58, 57, 57, + 56, 55, 55, 54, 53, 53, 52, 52, 51, 51, 50, 50, 49, 49, 49, 49, 49, 49, + 68, 68, 68, 68, 68, 66, 64, 62, 60, 59, 58, 57, 56, 56, 55, 55, 54, 53, + 53, 52, 52, 51, 51, 50, 50, 49, 49, 49, 48, 48, 48, 48, 66, 66, 67, 67, + 67, 65, 63, 61, 59, 58, 57, 56, 55, 55, 54, 54, 53, 53, 52, 52, 51, 51, + 50, 50, 49, 49, 49, 48, 48, 48, 48, 48, 64, 65, 65, 66, 66, 64, 62, 60, + 58, 57, 56, 55, 54, 54, 53, 53, 52, 52, 51, 51, 50, 50, 50, 49, 49, 49, + 48, 48, 48, 48, 48, 48, 62, 63, 64, 64, 65, 63, 61, 59, 57, 56, 55, 54, + 53, 53, 52, 52, 51, 51, 51, 50, 50, 50, 49, 49, 49, 48, 48, 48, 48, 48, + 48, 48, 61, 61, 62, 63, 63, 62, 60, 58, 57, 56, 55, 54, 53, 52, 52, 51, + 51, 51, 50, 50, 49, 49, 49, 49, 48, 48, 48, 48, 47, 47, 47, 47, 59, 60, + 61, 61, 62, 61, 59, 58, 56, 55, 54, 53, 52, 52, 51, 51, 50, 50, 50, 49, + 49, 49, 49, 48, 48, 48, 48, 47, 47, 47, 47, 47, 58, 59, 59, 60, 61, 59, + 58, 57, 55, 55, 54, 53, 52, 51, 51, 50, 50, 50, 49, 49, 49, 48, 48, 48, + 48, 48, 47, 47, 47, 47, 47, 47, 56, 57, 58, 58, 59, 58, 57, 56, 55, 54, + 53, 52, 51, 51, 50, 50, 49, 49, 49, 49, 48, 48, 48, 48, 47, 47, 47, 47, + 47, 47, 47, 47, 55, 56, 57, 57, 58, 57, 56, 55, 54, 53, 53, 52, 51, 51, + 50, 50, 49, 49, 49, 48, 48, 48, 48, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 54, 55, 56, 56, 57, 56, 55, 54, 53, 53, 52, 51, 51, 50, 50, 49, 49, 49, + 48, 48, 48, 48, 47, 47, 47, 47, 47, 47, 46, 46, 46, 46, 53, 54, 55, 55, + 56, 55, 54, 54, 53, 52, 52, 51, 50, 50, 49, 49, 49, 48, 48, 48, 48, 47, + 47, 47, 47, 47, 47, 46, 46, 46, 46, 46, 52, 53, 54, 54, 55, 54, 53, 53, + 52, 52, 51, 50, 50, 49, 49, 49, 48, 48, 48, 48, 47, 47, 47, 47, 47, 46, + 46, 46, 46, 46, 46, 46, 52, 52, 53, 53, 54, 53, 53, 52, 52, 51, 51, 50, + 50, 49, 49, 48, 48, 48, 48, 47, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, + 46, 46, 51, 52, 52, 53, 53, 53, 52, 52, 51, 51, 50, 50, 49, 49, 49, 48, + 48, 48, 47, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 46, 46, 46, 50, 51, + 51, 52, 52, 52, 51, 51, 51, 50, 50, 49, 49, 49, 48, 48, 48, 47, 47, 47, + 47, 47, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 50, 50, 51, 51, 52, 51, + 51, 50, 50, 50, 49, 49, 49, 48, 48, 48, 47, 47, 47, 47, 47, 46, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 46, 49, 50, 50, 51, 51, 51, 50, 50, 50, 49, + 49, 49, 48, 48, 48, 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 46, 46, + 46, 46, 46, 46, 49, 49, 50, 50, 50, 50, 50, 50, 49, 49, 49, 48, 48, 48, + 48, 47, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 46, 46, 45, 45, 45, 45, + 48, 49, 49, 49, 50, 50, 49, 49, 49, 49, 48, 48, 48, 48, 47, 47, 47, 47, + 47, 46, 46, 46, 46, 46, 46, 46, 46, 45, 45, 45, 45, 45, 48, 48, 49, 49, + 49, 49, 49, 49, 49, 48, 48, 48, 48, 47, 47, 47, 47, 47, 46, 46, 46, 46, + 46, 46, 46, 46, 45, 45, 45, 45, 45, 45, 48, 48, 49, 49, 49, 49, 49, 49, + 49, 48, 48, 48, 48, 47, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 46, 46, + 45, 45, 45, 45, 45, 45, 48, 48, 49, 49, 49, 49, 49, 49, 49, 48, 48, 48, + 48, 47, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 46, 46, 45, 45, 45, 45, + 45, 45, 48, 48, 49, 49, 49, 49, 49, 49, 49, 48, 48, 48, 48, 47, 47, 47, + 47, 47, 46, 46, 46, 46, 46, 46, 46, 46, 45, 45, 45, 45, 45, 45 }, + { /* Intra matrices */ + /* Size 4 */ + 86, 83, 68, 60, 83, 70, 64, 59, 68, 64, 59, 57, 60, 59, 57, 55, + /* Size 8 */ + 79, 90, 87, 77, 69, 64, 61, 58, 90, 85, 86, 80, 73, 67, 63, 60, 87, 86, + 75, 71, 67, 64, 61, 59, 77, 80, 71, 66, 63, 61, 59, 58, 69, 73, 67, 63, + 60, 59, 58, 57, 64, 67, 64, 61, 59, 57, 57, 56, 61, 63, 61, 59, 58, 57, + 56, 55, 58, 60, 59, 58, 57, 56, 55, 55, + /* Size 16 */ + 80, 86, 91, 90, 88, 83, 78, 74, 70, 68, 65, 63, 61, 60, 59, 59, 86, 87, + 89, 88, 88, 84, 80, 76, 72, 69, 66, 65, 63, 61, 60, 60, 91, 89, 86, 87, + 87, 84, 82, 78, 74, 71, 68, 66, 64, 62, 61, 61, 90, 88, 87, 84, 82, 79, + 77, 74, 71, 69, 66, 65, 63, 62, 60, 60, 88, 88, 87, 82, 76, 74, 72, 70, + 68, 66, 65, 63, 62, 61, 60, 60, 83, 84, 84, 79, 74, 71, 69, 68, 66, 65, + 63, 62, 61, 60, 59, 59, 78, 80, 82, 77, 72, 69, 66, 65, 64, 63, 62, 61, + 60, 59, 59, 59, 74, 76, 78, 74, 70, 68, 65, 64, 62, 62, 61, 60, 59, 59, + 58, 58, 70, 72, 74, 71, 68, 66, 64, 62, 61, 60, 60, 59, 58, 58, 57, 57, + 68, 69, 71, 69, 66, 65, 63, 62, 60, 60, 59, 58, 58, 57, 57, 57, 65, 66, + 68, 66, 65, 63, 62, 61, 60, 59, 58, 58, 57, 57, 57, 57, 63, 65, 66, 65, + 63, 62, 61, 60, 59, 58, 58, 57, 57, 57, 56, 56, 61, 63, 64, 63, 62, 61, + 60, 59, 58, 58, 57, 57, 57, 56, 56, 56, 60, 61, 62, 62, 61, 60, 59, 59, + 58, 57, 57, 57, 56, 56, 56, 56, 59, 60, 61, 60, 60, 59, 59, 58, 57, 57, + 57, 56, 56, 56, 56, 56, 59, 60, 61, 60, 60, 59, 59, 58, 57, 57, 57, 56, + 56, 56, 56, 56, + /* Size 32 */ + 81, 84, 86, 89, 92, 91, 90, 89, 89, 86, 84, 81, 79, 77, 75, 73, 71, 69, + 68, 67, 65, 64, 64, 63, 62, 61, 61, 60, 59, 59, 59, 59, 84, 85, 87, 89, + 91, 90, 89, 89, 88, 86, 84, 82, 79, 78, 76, 74, 72, 70, 69, 67, 66, 65, + 64, 63, 62, 62, 61, 61, 60, 60, 60, 60, 86, 87, 88, 88, 89, 89, 89, 88, + 88, 86, 84, 82, 80, 78, 76, 75, 73, 71, 70, 68, 67, 66, 65, 64, 63, 62, + 62, 61, 60, 60, 60, 60, 89, 89, 88, 88, 88, 88, 88, 88, 88, 86, 85, 83, + 81, 79, 77, 75, 74, 72, 71, 69, 68, 67, 66, 65, 64, 63, 62, 62, 61, 61, + 61, 61, 92, 91, 89, 88, 86, 87, 87, 88, 88, 86, 85, 83, 82, 80, 78, 76, + 74, 73, 71, 70, 68, 67, 66, 65, 64, 64, 63, 62, 61, 61, 61, 61, 91, 90, + 89, 88, 87, 86, 86, 85, 85, 84, 82, 81, 80, 78, 76, 75, 73, 72, 70, 69, + 68, 67, 66, 65, 64, 63, 62, 62, 61, 61, 61, 61, 90, 89, 89, 88, 87, 86, + 85, 83, 82, 81, 80, 78, 77, 76, 74, 73, 72, 70, 69, 68, 67, 66, 65, 64, + 63, 63, 62, 61, 61, 61, 61, 61, 89, 89, 88, 88, 88, 85, 83, 81, 79, 78, + 77, 76, 75, 73, 72, 71, 70, 69, 68, 67, 66, 65, 64, 64, 63, 62, 62, 61, + 61, 61, 61, 61, 89, 88, 88, 88, 88, 85, 82, 79, 76, 75, 74, 73, 72, 71, + 70, 69, 69, 68, 67, 66, 65, 64, 64, 63, 62, 62, 61, 61, 60, 60, 60, 60, + 86, 86, 86, 86, 86, 84, 81, 78, 75, 74, 73, 72, 71, 70, 69, 68, 67, 67, + 66, 65, 64, 64, 63, 63, 62, 61, 61, 60, 60, 60, 60, 60, 84, 84, 84, 85, + 85, 82, 80, 77, 74, 73, 72, 71, 70, 69, 68, 67, 66, 66, 65, 64, 64, 63, + 63, 62, 61, 61, 61, 60, 60, 60, 60, 60, 81, 82, 82, 83, 83, 81, 78, 76, + 73, 72, 71, 69, 68, 67, 67, 66, 65, 65, 64, 63, 63, 62, 62, 61, 61, 60, + 60, 60, 59, 59, 59, 59, 79, 79, 80, 81, 82, 80, 77, 75, 72, 71, 70, 68, + 67, 66, 65, 65, 64, 64, 63, 63, 62, 62, 61, 61, 60, 60, 60, 59, 59, 59, + 59, 59, 77, 78, 78, 79, 80, 78, 76, 73, 71, 70, 69, 67, 66, 66, 65, 64, + 63, 63, 63, 62, 62, 61, 61, 60, 60, 60, 59, 59, 59, 59, 59, 59, 75, 76, + 76, 77, 78, 76, 74, 72, 70, 69, 68, 67, 65, 65, 64, 63, 63, 62, 62, 61, + 61, 61, 60, 60, 60, 59, 59, 59, 58, 58, 58, 58, 73, 74, 75, 75, 76, 75, + 73, 71, 69, 68, 67, 66, 65, 64, 63, 63, 62, 62, 61, 61, 60, 60, 60, 60, + 59, 59, 59, 58, 58, 58, 58, 58, 71, 72, 73, 74, 74, 73, 72, 70, 69, 67, + 66, 65, 64, 63, 63, 62, 62, 61, 61, 60, 60, 60, 59, 59, 59, 59, 58, 58, + 58, 58, 58, 58, 69, 70, 71, 72, 73, 72, 70, 69, 68, 67, 66, 65, 64, 63, + 62, 62, 61, 61, 60, 60, 60, 59, 59, 59, 58, 58, 58, 58, 58, 58, 58, 58, + 68, 69, 70, 71, 71, 70, 69, 68, 67, 66, 65, 64, 63, 63, 62, 61, 61, 60, + 60, 60, 59, 59, 59, 58, 58, 58, 58, 58, 57, 57, 57, 57, 67, 67, 68, 69, + 70, 69, 68, 67, 66, 65, 64, 63, 63, 62, 61, 61, 60, 60, 60, 59, 59, 59, + 58, 58, 58, 58, 58, 57, 57, 57, 57, 57, 65, 66, 67, 68, 68, 68, 67, 66, + 65, 64, 64, 63, 62, 62, 61, 60, 60, 60, 59, 59, 59, 58, 58, 58, 58, 57, + 57, 57, 57, 57, 57, 57, 64, 65, 66, 67, 67, 67, 66, 65, 64, 64, 63, 62, + 62, 61, 61, 60, 60, 59, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, 57, 57, + 57, 57, 64, 64, 65, 66, 66, 66, 65, 64, 64, 63, 63, 62, 61, 61, 60, 60, + 59, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, 57, 57, 57, 57, 57, 63, 63, + 64, 65, 65, 65, 64, 64, 63, 63, 62, 61, 61, 60, 60, 60, 59, 59, 58, 58, + 58, 58, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 62, 62, 63, 64, 64, 64, + 63, 63, 62, 62, 61, 61, 60, 60, 60, 59, 59, 58, 58, 58, 58, 57, 57, 57, + 57, 57, 57, 56, 56, 56, 56, 56, 61, 62, 62, 63, 64, 63, 63, 62, 62, 61, + 61, 60, 60, 60, 59, 59, 59, 58, 58, 58, 57, 57, 57, 57, 57, 57, 56, 56, + 56, 56, 56, 56, 61, 61, 62, 62, 63, 62, 62, 62, 61, 61, 61, 60, 60, 59, + 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, + 60, 61, 61, 62, 62, 62, 61, 61, 61, 60, 60, 60, 59, 59, 59, 58, 58, 58, + 58, 57, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, 56, 59, 60, 60, 61, + 61, 61, 61, 61, 60, 60, 60, 59, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, + 57, 57, 56, 56, 56, 56, 56, 56, 56, 56, 59, 60, 60, 61, 61, 61, 61, 61, + 60, 60, 60, 59, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, 57, 57, 56, 56, + 56, 56, 56, 56, 56, 56, 59, 60, 60, 61, 61, 61, 61, 61, 60, 60, 60, 59, + 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, + 56, 56, 59, 60, 60, 61, 61, 61, 61, 61, 60, 60, 60, 59, 59, 59, 58, 58, + 58, 58, 57, 57, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, 56 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 53, 52, 48, 53, 50, 49, 48, 52, 49, 46, 45, 48, 48, 45, 43, + /* Size 8 */ + 64, 71, 55, 54, 53, 51, 50, 48, 71, 60, 55, 57, 56, 54, 52, 51, 55, 55, + 52, 53, 53, 52, 51, 49, 54, 57, 53, 51, 50, 50, 49, 48, 53, 56, 53, 50, + 49, 48, 47, 47, 51, 54, 52, 50, 48, 47, 46, 46, 50, 52, 51, 49, 47, 46, + 46, 45, 48, 51, 49, 48, 47, 46, 45, 44, + /* Size 16 */ + 64, 67, 71, 63, 55, 55, 54, 54, 53, 52, 51, 50, 50, 49, 48, 48, 67, 66, + 65, 60, 55, 55, 56, 55, 55, 54, 53, 52, 51, 50, 49, 49, 71, 65, 60, 58, + 55, 56, 57, 57, 56, 55, 54, 53, 52, 51, 51, 51, 63, 60, 58, 56, 54, 54, + 55, 55, 55, 54, 53, 52, 52, 51, 50, 50, 55, 55, 55, 54, 52, 53, 53, 53, + 53, 52, 52, 51, 51, 50, 49, 49, 55, 55, 56, 54, 53, 52, 52, 52, 52, 51, + 51, 50, 50, 49, 49, 49, 54, 56, 57, 55, 53, 52, 51, 51, 50, 50, 50, 49, + 49, 48, 48, 48, 54, 55, 57, 55, 53, 52, 51, 50, 50, 49, 49, 48, 48, 48, + 47, 47, 53, 55, 56, 55, 53, 52, 50, 50, 49, 48, 48, 48, 47, 47, 47, 47, + 52, 54, 55, 54, 52, 51, 50, 49, 48, 48, 47, 47, 47, 47, 46, 46, 51, 53, + 54, 53, 52, 51, 50, 49, 48, 47, 47, 47, 46, 46, 46, 46, 50, 52, 53, 52, + 51, 50, 49, 48, 48, 47, 47, 46, 46, 46, 45, 45, 50, 51, 52, 52, 51, 50, + 49, 48, 47, 47, 46, 46, 46, 45, 45, 45, 49, 50, 51, 51, 50, 49, 48, 48, + 47, 47, 46, 46, 45, 45, 45, 45, 48, 49, 51, 50, 49, 49, 48, 47, 47, 46, + 46, 45, 45, 45, 44, 44, 48, 49, 51, 50, 49, 49, 48, 47, 47, 46, 46, 45, + 45, 45, 44, 44, + /* Size 32 */ + 64, 66, 67, 69, 71, 67, 63, 59, 55, 55, 55, 54, 54, 54, 54, 53, 53, 53, + 52, 52, 51, 51, 50, 50, 50, 49, 49, 48, 48, 48, 48, 48, 66, 66, 67, 67, + 68, 65, 62, 58, 55, 55, 55, 55, 55, 55, 54, 54, 54, 53, 53, 53, 52, 52, + 51, 51, 50, 50, 50, 49, 49, 49, 49, 49, 67, 67, 66, 66, 65, 63, 60, 58, + 55, 55, 55, 55, 56, 55, 55, 55, 55, 54, 54, 53, 53, 52, 52, 51, 51, 51, + 50, 50, 49, 49, 49, 49, 69, 67, 66, 64, 63, 61, 59, 57, 55, 55, 56, 56, + 56, 56, 56, 56, 55, 55, 55, 54, 54, 53, 53, 52, 52, 51, 51, 50, 50, 50, + 50, 50, 71, 68, 65, 63, 60, 59, 58, 56, 55, 55, 56, 56, 57, 57, 57, 56, + 56, 56, 55, 55, 54, 54, 53, 53, 52, 52, 51, 51, 51, 51, 51, 51, 67, 65, + 63, 61, 59, 58, 57, 55, 54, 55, 55, 55, 56, 56, 56, 56, 55, 55, 55, 54, + 54, 53, 53, 52, 52, 52, 51, 51, 50, 50, 50, 50, 63, 62, 60, 59, 58, 57, + 56, 55, 54, 54, 54, 54, 55, 55, 55, 55, 55, 54, 54, 54, 53, 53, 52, 52, + 52, 51, 51, 50, 50, 50, 50, 50, 59, 58, 58, 57, 56, 55, 55, 54, 53, 53, + 53, 54, 54, 54, 54, 54, 54, 53, 53, 53, 53, 52, 52, 52, 51, 51, 50, 50, + 50, 50, 50, 50, 55, 55, 55, 55, 55, 54, 54, 53, 52, 52, 53, 53, 53, 53, + 53, 53, 53, 53, 52, 52, 52, 52, 51, 51, 51, 50, 50, 50, 49, 49, 49, 49, + 55, 55, 55, 55, 55, 55, 54, 53, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 51, 51, 51, 51, 50, 50, 50, 49, 49, 49, 49, 49, 55, 55, 55, 56, + 56, 55, 54, 53, 53, 52, 52, 52, 52, 52, 52, 52, 52, 51, 51, 51, 51, 51, + 50, 50, 50, 50, 49, 49, 49, 49, 49, 49, 54, 55, 55, 56, 56, 55, 54, 54, + 53, 52, 52, 52, 51, 51, 51, 51, 51, 51, 51, 50, 50, 50, 50, 50, 49, 49, + 49, 49, 48, 48, 48, 48, 54, 55, 56, 56, 57, 56, 55, 54, 53, 52, 52, 51, + 51, 51, 51, 50, 50, 50, 50, 50, 50, 49, 49, 49, 49, 49, 48, 48, 48, 48, + 48, 48, 54, 55, 55, 56, 57, 56, 55, 54, 53, 52, 52, 51, 51, 51, 50, 50, + 50, 50, 50, 49, 49, 49, 49, 49, 49, 48, 48, 48, 48, 48, 48, 48, 54, 54, + 55, 56, 57, 56, 55, 54, 53, 52, 52, 51, 51, 50, 50, 50, 50, 49, 49, 49, + 49, 49, 48, 48, 48, 48, 48, 48, 47, 47, 47, 47, 53, 54, 55, 56, 56, 56, + 55, 54, 53, 52, 52, 51, 50, 50, 50, 49, 49, 49, 49, 49, 48, 48, 48, 48, + 48, 48, 47, 47, 47, 47, 47, 47, 53, 54, 55, 55, 56, 55, 55, 54, 53, 52, + 52, 51, 50, 50, 50, 49, 49, 49, 48, 48, 48, 48, 48, 48, 47, 47, 47, 47, + 47, 47, 47, 47, 53, 53, 54, 55, 56, 55, 54, 53, 53, 52, 51, 51, 50, 50, + 49, 49, 49, 48, 48, 48, 48, 48, 47, 47, 47, 47, 47, 47, 46, 46, 46, 46, + 52, 53, 54, 55, 55, 55, 54, 53, 52, 52, 51, 51, 50, 50, 49, 49, 48, 48, + 48, 48, 47, 47, 47, 47, 47, 47, 47, 46, 46, 46, 46, 46, 52, 53, 53, 54, + 55, 54, 54, 53, 52, 52, 51, 50, 50, 49, 49, 49, 48, 48, 48, 47, 47, 47, + 47, 47, 47, 46, 46, 46, 46, 46, 46, 46, 51, 52, 53, 54, 54, 54, 53, 53, + 52, 51, 51, 50, 50, 49, 49, 48, 48, 48, 47, 47, 47, 47, 47, 46, 46, 46, + 46, 46, 46, 46, 46, 46, 51, 52, 52, 53, 54, 53, 53, 52, 52, 51, 51, 50, + 49, 49, 49, 48, 48, 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 46, 46, + 46, 46, 50, 51, 52, 53, 53, 53, 52, 52, 51, 51, 50, 50, 49, 49, 48, 48, + 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 45, 45, 45, 45, 45, 50, 51, + 51, 52, 53, 52, 52, 52, 51, 51, 50, 50, 49, 49, 48, 48, 48, 47, 47, 47, + 46, 46, 46, 46, 46, 46, 45, 45, 45, 45, 45, 45, 50, 50, 51, 52, 52, 52, + 52, 51, 51, 50, 50, 49, 49, 49, 48, 48, 47, 47, 47, 47, 46, 46, 46, 46, + 46, 45, 45, 45, 45, 45, 45, 45, 49, 50, 51, 51, 52, 52, 51, 51, 50, 50, + 50, 49, 49, 48, 48, 48, 47, 47, 47, 46, 46, 46, 46, 46, 45, 45, 45, 45, + 45, 45, 45, 45, 49, 50, 50, 51, 51, 51, 51, 50, 50, 50, 49, 49, 48, 48, + 48, 47, 47, 47, 47, 46, 46, 46, 46, 45, 45, 45, 45, 45, 45, 45, 45, 45, + 48, 49, 50, 50, 51, 51, 50, 50, 50, 49, 49, 49, 48, 48, 48, 47, 47, 47, + 46, 46, 46, 46, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 48, 49, 49, 50, + 51, 50, 50, 50, 49, 49, 49, 48, 48, 48, 47, 47, 47, 46, 46, 46, 46, 46, + 45, 45, 45, 45, 45, 45, 44, 44, 44, 44, 48, 49, 49, 50, 51, 50, 50, 50, + 49, 49, 49, 48, 48, 48, 47, 47, 47, 46, 46, 46, 46, 46, 45, 45, 45, 45, + 45, 45, 44, 44, 44, 44, 48, 49, 49, 50, 51, 50, 50, 50, 49, 49, 49, 48, + 48, 48, 47, 47, 47, 46, 46, 46, 46, 46, 45, 45, 45, 45, 45, 45, 44, 44, + 44, 44, 48, 49, 49, 50, 51, 50, 50, 50, 49, 49, 49, 48, 48, 48, 47, 47, + 47, 46, 46, 46, 46, 46, 45, 45, 45, 45, 45, 45, 44, 44, 44, 44 }, + { /* Intra matrices */ + /* Size 4 */ + 85, 70, 68, 63, 70, 66, 64, 62, 68, 64, 60, 58, 63, 62, 58, 56, + /* Size 8 */ + 82, 91, 70, 69, 67, 65, 62, 60, 91, 76, 70, 72, 71, 69, 66, 64, 70, 70, + 66, 67, 67, 66, 64, 62, 69, 72, 67, 64, 63, 62, 61, 60, 67, 71, 67, 63, + 61, 60, 59, 59, 65, 69, 66, 62, 60, 59, 58, 57, 62, 66, 64, 61, 59, 58, + 57, 56, 60, 64, 62, 60, 59, 57, 56, 55, + /* Size 16 */ + 82, 87, 91, 81, 71, 70, 69, 68, 68, 66, 65, 64, 63, 62, 61, 61, 87, 86, + 84, 77, 70, 71, 71, 70, 70, 69, 67, 66, 65, 64, 63, 63, 91, 84, 77, 74, + 70, 71, 73, 72, 72, 71, 70, 68, 67, 65, 64, 64, 81, 77, 74, 71, 68, 69, + 70, 70, 70, 69, 68, 67, 66, 65, 63, 63, 71, 70, 70, 68, 67, 67, 67, 67, + 67, 67, 66, 65, 65, 64, 63, 63, 70, 71, 71, 69, 67, 66, 66, 66, 66, 65, + 65, 64, 63, 62, 62, 62, 69, 71, 73, 70, 67, 66, 65, 64, 64, 63, 63, 62, + 62, 61, 61, 61, 68, 70, 72, 70, 67, 66, 64, 64, 63, 62, 62, 61, 61, 60, + 60, 60, 68, 70, 72, 70, 67, 66, 64, 63, 62, 61, 61, 60, 60, 59, 59, 59, + 66, 69, 71, 69, 67, 65, 63, 62, 61, 61, 60, 60, 59, 59, 58, 58, 65, 67, + 70, 68, 66, 65, 63, 62, 61, 60, 59, 59, 58, 58, 58, 58, 64, 66, 68, 67, + 65, 64, 62, 61, 60, 60, 59, 58, 58, 58, 57, 57, 63, 65, 67, 66, 65, 63, + 62, 61, 60, 59, 58, 58, 57, 57, 57, 57, 62, 64, 65, 65, 64, 62, 61, 60, + 59, 59, 58, 58, 57, 57, 56, 56, 61, 63, 64, 63, 63, 62, 61, 60, 59, 58, + 58, 57, 57, 56, 56, 56, 61, 63, 64, 63, 63, 62, 61, 60, 59, 58, 58, 57, + 57, 56, 56, 56, + /* Size 32 */ + 83, 85, 87, 90, 92, 87, 81, 76, 71, 71, 70, 70, 69, 69, 69, 68, 68, 67, + 67, 66, 66, 65, 64, 64, 63, 63, 62, 62, 61, 61, 61, 61, 85, 86, 87, 87, + 88, 84, 79, 75, 71, 71, 71, 70, 70, 70, 70, 69, 69, 68, 68, 67, 67, 66, + 65, 65, 64, 64, 63, 63, 62, 62, 62, 62, 87, 87, 86, 85, 85, 81, 78, 74, + 71, 71, 71, 71, 71, 71, 71, 70, 70, 69, 69, 68, 68, 67, 66, 66, 65, 65, + 64, 63, 63, 63, 63, 63, 90, 87, 85, 83, 81, 78, 76, 73, 71, 71, 71, 72, + 72, 72, 72, 71, 71, 71, 70, 69, 69, 68, 67, 67, 66, 66, 65, 64, 64, 64, + 64, 64, 92, 88, 85, 81, 77, 76, 74, 72, 70, 71, 72, 72, 73, 73, 73, 72, + 72, 72, 71, 70, 70, 69, 68, 68, 67, 66, 66, 65, 65, 65, 65, 65, 87, 84, + 81, 78, 76, 74, 73, 71, 70, 70, 71, 71, 72, 72, 71, 71, 71, 71, 70, 70, + 69, 68, 68, 67, 67, 66, 65, 65, 64, 64, 64, 64, 81, 79, 78, 76, 74, 73, + 71, 70, 69, 69, 69, 70, 70, 70, 70, 70, 70, 70, 69, 69, 68, 68, 67, 66, + 66, 65, 65, 64, 64, 64, 64, 64, 76, 75, 74, 73, 72, 71, 70, 69, 68, 68, + 68, 69, 69, 69, 69, 69, 69, 68, 68, 68, 67, 67, 66, 66, 65, 65, 64, 64, + 63, 63, 63, 63, 71, 71, 71, 71, 70, 70, 69, 68, 67, 67, 67, 67, 68, 68, + 68, 68, 68, 67, 67, 67, 67, 66, 66, 65, 65, 64, 64, 63, 63, 63, 63, 63, + 71, 71, 71, 71, 71, 70, 69, 68, 67, 67, 67, 67, 67, 67, 67, 67, 67, 66, + 66, 66, 66, 65, 65, 65, 64, 64, 63, 63, 62, 62, 62, 62, 70, 71, 71, 71, + 72, 71, 69, 68, 67, 67, 67, 66, 66, 66, 66, 66, 66, 66, 65, 65, 65, 65, + 64, 64, 64, 63, 63, 62, 62, 62, 62, 62, 70, 70, 71, 72, 72, 71, 70, 69, + 67, 67, 66, 66, 66, 65, 65, 65, 65, 65, 65, 64, 64, 64, 63, 63, 63, 63, + 62, 62, 62, 62, 62, 62, 69, 70, 71, 72, 73, 72, 70, 69, 68, 67, 66, 66, + 65, 65, 65, 64, 64, 64, 64, 63, 63, 63, 63, 63, 62, 62, 62, 61, 61, 61, + 61, 61, 69, 70, 71, 72, 73, 72, 70, 69, 68, 67, 66, 65, 65, 64, 64, 64, + 64, 63, 63, 63, 63, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 69, 70, + 71, 72, 73, 71, 70, 69, 68, 67, 66, 65, 65, 64, 64, 63, 63, 63, 63, 62, + 62, 62, 62, 61, 61, 61, 61, 60, 60, 60, 60, 60, 68, 69, 70, 71, 72, 71, + 70, 69, 68, 67, 66, 65, 64, 64, 63, 63, 63, 62, 62, 62, 62, 61, 61, 61, + 61, 60, 60, 60, 60, 60, 60, 60, 68, 69, 70, 71, 72, 71, 70, 69, 68, 67, + 66, 65, 64, 64, 63, 63, 62, 62, 62, 61, 61, 61, 61, 60, 60, 60, 60, 60, + 59, 59, 59, 59, 67, 68, 69, 71, 72, 71, 70, 68, 67, 66, 66, 65, 64, 63, + 63, 62, 62, 62, 61, 61, 61, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, + 67, 68, 69, 70, 71, 70, 69, 68, 67, 66, 65, 65, 64, 63, 63, 62, 62, 61, + 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 66, 67, 68, 69, + 70, 70, 69, 68, 67, 66, 65, 64, 63, 63, 62, 62, 61, 61, 61, 60, 60, 60, + 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 66, 67, 68, 69, 70, 69, 68, 67, + 67, 66, 65, 64, 63, 63, 62, 62, 61, 61, 60, 60, 60, 59, 59, 59, 59, 58, + 58, 58, 58, 58, 58, 58, 65, 66, 67, 68, 69, 68, 68, 67, 66, 65, 65, 64, + 63, 62, 62, 61, 61, 60, 60, 60, 59, 59, 59, 59, 58, 58, 58, 58, 58, 58, + 58, 58, 64, 65, 66, 67, 68, 68, 67, 66, 66, 65, 64, 63, 63, 62, 62, 61, + 61, 60, 60, 59, 59, 59, 59, 58, 58, 58, 58, 58, 57, 57, 57, 57, 64, 65, + 66, 67, 68, 67, 66, 66, 65, 65, 64, 63, 63, 62, 61, 61, 60, 60, 60, 59, + 59, 59, 58, 58, 58, 58, 58, 57, 57, 57, 57, 57, 63, 64, 65, 66, 67, 67, + 66, 65, 65, 64, 64, 63, 62, 62, 61, 61, 60, 60, 59, 59, 59, 58, 58, 58, + 58, 57, 57, 57, 57, 57, 57, 57, 63, 64, 65, 66, 66, 66, 65, 65, 64, 64, + 63, 63, 62, 61, 61, 60, 60, 60, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, + 57, 57, 57, 57, 62, 63, 64, 65, 66, 65, 65, 64, 64, 63, 63, 62, 62, 61, + 61, 60, 60, 59, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, 57, 57, 57, 57, + 62, 63, 63, 64, 65, 65, 64, 64, 63, 63, 62, 62, 61, 61, 60, 60, 60, 59, + 59, 58, 58, 58, 58, 57, 57, 57, 57, 57, 56, 56, 56, 56, 61, 62, 63, 64, + 65, 64, 64, 63, 63, 62, 62, 62, 61, 61, 60, 60, 59, 59, 59, 58, 58, 58, + 57, 57, 57, 57, 57, 56, 56, 56, 56, 56, 61, 62, 63, 64, 65, 64, 64, 63, + 63, 62, 62, 62, 61, 61, 60, 60, 59, 59, 59, 58, 58, 58, 57, 57, 57, 57, + 57, 56, 56, 56, 56, 56, 61, 62, 63, 64, 65, 64, 64, 63, 63, 62, 62, 62, + 61, 61, 60, 60, 59, 59, 59, 58, 58, 58, 57, 57, 57, 57, 57, 56, 56, 56, + 56, 56, 61, 62, 63, 64, 65, 64, 64, 63, 63, 62, 62, 62, 61, 61, 60, 60, + 59, 59, 59, 58, 58, 58, 57, 57, 57, 57, 57, 56, 56, 56, 56, 56 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 62, 54, 50, 62, 55, 52, 49, 54, 52, 49, 48, 50, 49, 48, 47, + /* Size 8 */ + 64, 70, 68, 63, 58, 55, 53, 52, 70, 67, 68, 65, 60, 57, 54, 53, 68, 68, + 61, 59, 57, 55, 53, 52, 63, 65, 59, 56, 54, 53, 52, 51, 58, 60, 57, 54, + 53, 52, 51, 51, 55, 57, 55, 53, 52, 51, 51, 50, 53, 54, 53, 52, 51, 51, + 50, 50, 52, 53, 52, 51, 51, 50, 50, 50, + /* Size 16 */ + 64, 67, 70, 69, 68, 66, 63, 60, 58, 57, 55, 54, 53, 52, 52, 52, 67, 68, + 69, 69, 68, 66, 64, 61, 59, 58, 56, 55, 54, 53, 52, 52, 70, 69, 67, 68, + 68, 66, 65, 62, 60, 59, 57, 56, 54, 53, 53, 53, 69, 69, 68, 66, 65, 63, + 62, 60, 59, 57, 56, 55, 54, 53, 52, 52, 68, 68, 68, 65, 61, 60, 59, 58, + 57, 56, 55, 54, 53, 53, 52, 52, 66, 66, 66, 63, 60, 59, 57, 56, 56, 55, + 54, 53, 53, 52, 52, 52, 63, 64, 65, 62, 59, 57, 56, 55, 54, 54, 53, 53, + 52, 52, 51, 51, 60, 61, 62, 60, 58, 56, 55, 54, 54, 53, 52, 52, 52, 51, + 51, 51, 58, 59, 60, 59, 57, 56, 54, 54, 53, 52, 52, 52, 51, 51, 51, 51, + 57, 58, 59, 57, 56, 55, 54, 53, 52, 52, 51, 51, 51, 51, 50, 50, 55, 56, + 57, 56, 55, 54, 53, 52, 52, 51, 51, 51, 51, 50, 50, 50, 54, 55, 56, 55, + 54, 53, 53, 52, 52, 51, 51, 51, 50, 50, 50, 50, 53, 54, 54, 54, 53, 53, + 52, 52, 51, 51, 51, 50, 50, 50, 50, 50, 52, 53, 53, 53, 53, 52, 52, 51, + 51, 51, 50, 50, 50, 50, 50, 50, 52, 52, 53, 52, 52, 52, 51, 51, 51, 50, + 50, 50, 50, 50, 50, 50, 52, 52, 53, 52, 52, 52, 51, 51, 51, 50, 50, 50, + 50, 50, 50, 50, + /* Size 32 */ + 64, 66, 67, 69, 70, 70, 69, 69, 68, 67, 66, 64, 63, 62, 60, 59, 58, 57, + 57, 56, 55, 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, 52, 66, 67, 68, 69, + 70, 69, 69, 69, 68, 67, 66, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 55, + 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, 67, 68, 68, 68, 69, 69, 69, 68, + 68, 67, 66, 65, 64, 63, 61, 60, 59, 58, 58, 57, 56, 55, 55, 54, 54, 53, + 53, 52, 52, 52, 52, 52, 69, 69, 68, 68, 68, 68, 68, 68, 68, 67, 66, 65, + 64, 63, 62, 61, 60, 59, 58, 57, 56, 56, 55, 55, 54, 54, 53, 53, 52, 52, + 52, 52, 70, 70, 69, 68, 67, 67, 68, 68, 68, 67, 66, 66, 65, 64, 62, 61, + 60, 59, 59, 58, 57, 56, 56, 55, 54, 54, 53, 53, 53, 53, 53, 53, 70, 69, + 69, 68, 67, 67, 67, 67, 66, 66, 65, 64, 63, 62, 61, 60, 59, 59, 58, 57, + 56, 56, 55, 55, 54, 54, 53, 53, 52, 52, 52, 52, 69, 69, 69, 68, 68, 67, + 66, 65, 65, 64, 63, 63, 62, 61, 60, 59, 59, 58, 57, 57, 56, 55, 55, 54, + 54, 53, 53, 53, 52, 52, 52, 52, 69, 69, 68, 68, 68, 67, 65, 64, 63, 62, + 62, 61, 60, 60, 59, 58, 58, 57, 57, 56, 55, 55, 54, 54, 54, 53, 53, 53, + 52, 52, 52, 52, 68, 68, 68, 68, 68, 66, 65, 63, 61, 61, 60, 60, 59, 58, + 58, 57, 57, 56, 56, 55, 55, 54, 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, + 67, 67, 67, 67, 67, 66, 64, 62, 61, 60, 59, 59, 58, 58, 57, 57, 56, 56, + 55, 55, 54, 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, 52, 66, 66, 66, 66, + 66, 65, 63, 62, 60, 59, 59, 58, 57, 57, 56, 56, 56, 55, 55, 54, 54, 54, + 53, 53, 53, 52, 52, 52, 52, 52, 52, 52, 64, 64, 65, 65, 66, 64, 63, 61, + 60, 59, 58, 57, 57, 56, 56, 55, 55, 55, 54, 54, 54, 53, 53, 53, 52, 52, + 52, 52, 51, 51, 51, 51, 63, 63, 64, 64, 65, 63, 62, 60, 59, 58, 57, 57, + 56, 55, 55, 55, 54, 54, 54, 53, 53, 53, 53, 52, 52, 52, 52, 51, 51, 51, + 51, 51, 62, 62, 63, 63, 64, 62, 61, 60, 58, 58, 57, 56, 55, 55, 55, 54, + 54, 54, 53, 53, 53, 53, 52, 52, 52, 52, 52, 51, 51, 51, 51, 51, 60, 61, + 61, 62, 62, 61, 60, 59, 58, 57, 56, 56, 55, 55, 54, 54, 54, 53, 53, 53, + 52, 52, 52, 52, 52, 51, 51, 51, 51, 51, 51, 51, 59, 60, 60, 61, 61, 60, + 59, 58, 57, 57, 56, 55, 55, 54, 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, + 51, 51, 51, 51, 51, 51, 51, 51, 58, 59, 59, 60, 60, 59, 59, 58, 57, 56, + 56, 55, 54, 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, 51, 51, 51, 51, 51, + 51, 51, 51, 51, 57, 58, 58, 59, 59, 59, 58, 57, 56, 56, 55, 55, 54, 54, + 53, 53, 53, 52, 52, 52, 52, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 57, 57, 58, 58, 59, 58, 57, 57, 56, 55, 55, 54, 54, 53, 53, 53, 52, 52, + 52, 52, 51, 51, 51, 51, 51, 51, 51, 50, 50, 50, 50, 50, 56, 56, 57, 57, + 58, 57, 57, 56, 55, 55, 54, 54, 53, 53, 53, 52, 52, 52, 52, 51, 51, 51, + 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, 55, 55, 56, 56, 57, 56, 56, 55, + 55, 54, 54, 54, 53, 53, 52, 52, 52, 52, 51, 51, 51, 51, 51, 51, 51, 50, + 50, 50, 50, 50, 50, 50, 54, 55, 55, 56, 56, 56, 55, 55, 54, 54, 54, 53, + 53, 53, 52, 52, 52, 51, 51, 51, 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, + 50, 50, 54, 54, 55, 55, 56, 55, 55, 54, 54, 54, 53, 53, 53, 52, 52, 52, + 52, 51, 51, 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, 50, 50, 50, 53, 54, + 54, 55, 55, 55, 54, 54, 54, 53, 53, 53, 52, 52, 52, 52, 51, 51, 51, 51, + 51, 51, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 53, 53, 54, 54, 54, 54, + 54, 54, 53, 53, 53, 52, 52, 52, 52, 51, 51, 51, 51, 51, 51, 50, 50, 50, + 50, 50, 50, 50, 50, 50, 50, 50, 53, 53, 53, 54, 54, 54, 53, 53, 53, 53, + 52, 52, 52, 52, 51, 51, 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, 50, 50, + 50, 50, 50, 50, 52, 53, 53, 53, 53, 53, 53, 53, 53, 52, 52, 52, 52, 52, + 51, 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, + 52, 52, 52, 53, 53, 53, 53, 53, 52, 52, 52, 52, 51, 51, 51, 51, 51, 51, + 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 52, 52, 52, 52, + 53, 52, 52, 52, 52, 52, 52, 51, 51, 51, 51, 51, 51, 51, 50, 50, 50, 50, + 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 52, 52, 52, 52, 53, 52, 52, 52, + 52, 52, 52, 51, 51, 51, 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, 50, 50, + 50, 50, 50, 50, 50, 50, 52, 52, 52, 52, 53, 52, 52, 52, 52, 52, 52, 51, + 51, 51, 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, + 50, 50, 52, 52, 52, 52, 53, 52, 52, 52, 52, 52, 52, 51, 51, 51, 51, 51, + 51, 51, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50 }, + { /* Intra matrices */ + /* Size 4 */ + 80, 77, 67, 61, 77, 68, 63, 60, 67, 63, 60, 58, 61, 60, 58, 57, + /* Size 8 */ + 75, 83, 80, 73, 68, 64, 61, 60, 83, 79, 80, 76, 70, 66, 63, 61, 80, 80, + 72, 69, 66, 64, 62, 60, 73, 76, 69, 65, 63, 61, 60, 59, 68, 70, 66, 63, + 61, 60, 59, 58, 64, 66, 64, 61, 60, 59, 58, 58, 61, 63, 62, 60, 59, 58, + 58, 57, 60, 61, 60, 59, 58, 58, 57, 57, + /* Size 16 */ + 76, 80, 84, 82, 81, 78, 74, 71, 68, 66, 64, 63, 62, 61, 60, 60, 80, 81, + 82, 81, 81, 78, 75, 72, 70, 68, 66, 64, 63, 62, 61, 61, 84, 82, 80, 80, + 81, 79, 76, 74, 71, 69, 67, 65, 64, 63, 61, 61, 82, 81, 80, 78, 77, 75, + 73, 71, 69, 67, 65, 64, 63, 62, 61, 61, 81, 81, 81, 77, 72, 71, 69, 68, + 67, 66, 64, 63, 62, 61, 61, 61, 78, 78, 79, 75, 71, 69, 67, 66, 65, 64, + 63, 62, 62, 61, 60, 60, 74, 75, 76, 73, 69, 67, 66, 65, 64, 63, 62, 61, + 61, 60, 60, 60, 71, 72, 74, 71, 68, 66, 65, 64, 63, 62, 61, 61, 60, 60, + 59, 59, 68, 70, 71, 69, 67, 65, 64, 63, 62, 61, 60, 60, 60, 59, 59, 59, + 66, 68, 69, 67, 66, 64, 63, 62, 61, 61, 60, 60, 59, 59, 59, 59, 64, 66, + 67, 65, 64, 63, 62, 61, 60, 60, 59, 59, 59, 59, 58, 58, 63, 64, 65, 64, + 63, 62, 61, 61, 60, 60, 59, 59, 59, 58, 58, 58, 62, 63, 64, 63, 62, 62, + 61, 60, 60, 59, 59, 59, 58, 58, 58, 58, 61, 62, 63, 62, 61, 61, 60, 60, + 59, 59, 59, 58, 58, 58, 58, 58, 60, 61, 61, 61, 61, 60, 60, 59, 59, 59, + 58, 58, 58, 58, 58, 58, 60, 61, 61, 61, 61, 60, 60, 59, 59, 59, 58, 58, + 58, 58, 58, 58, + /* Size 32 */ + 76, 78, 80, 82, 84, 83, 83, 82, 82, 80, 78, 76, 74, 73, 71, 70, 69, 68, + 67, 66, 65, 64, 63, 63, 62, 62, 61, 61, 60, 60, 60, 60, 78, 79, 81, 82, + 83, 83, 82, 82, 82, 80, 78, 77, 75, 74, 72, 71, 69, 68, 67, 66, 65, 65, + 64, 63, 63, 62, 62, 61, 61, 61, 61, 61, 80, 81, 81, 82, 82, 82, 82, 82, + 81, 80, 79, 77, 76, 74, 73, 71, 70, 69, 68, 67, 66, 65, 64, 64, 63, 63, + 62, 62, 61, 61, 61, 61, 82, 82, 82, 81, 81, 81, 81, 81, 81, 80, 79, 77, + 76, 75, 73, 72, 71, 70, 69, 67, 66, 66, 65, 64, 63, 63, 62, 62, 61, 61, + 61, 61, 84, 83, 82, 81, 80, 80, 81, 81, 81, 80, 79, 78, 77, 75, 74, 73, + 71, 70, 69, 68, 67, 66, 65, 65, 64, 63, 63, 62, 62, 62, 62, 62, 83, 83, + 82, 81, 80, 80, 80, 79, 79, 78, 77, 76, 75, 74, 73, 71, 70, 69, 68, 67, + 66, 66, 65, 64, 64, 63, 63, 62, 62, 62, 62, 62, 83, 82, 82, 81, 81, 80, + 79, 78, 77, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, 67, 66, 65, 65, 64, + 63, 63, 62, 62, 61, 61, 61, 61, 82, 82, 82, 81, 81, 79, 78, 76, 75, 74, + 73, 72, 71, 71, 70, 69, 68, 67, 67, 66, 65, 65, 64, 63, 63, 62, 62, 62, + 61, 61, 61, 61, 82, 82, 81, 81, 81, 79, 77, 75, 73, 72, 71, 70, 70, 69, + 68, 68, 67, 66, 66, 65, 65, 64, 64, 63, 63, 62, 62, 61, 61, 61, 61, 61, + 80, 80, 80, 80, 80, 78, 76, 74, 72, 71, 70, 70, 69, 68, 67, 67, 66, 66, + 65, 65, 64, 64, 63, 63, 62, 62, 61, 61, 61, 61, 61, 61, 78, 78, 79, 79, + 79, 77, 75, 73, 71, 70, 69, 69, 68, 67, 67, 66, 65, 65, 64, 64, 63, 63, + 63, 62, 62, 62, 61, 61, 61, 61, 61, 61, 76, 77, 77, 77, 78, 76, 74, 72, + 70, 70, 69, 68, 67, 66, 66, 65, 65, 64, 64, 63, 63, 63, 62, 62, 61, 61, + 61, 61, 60, 60, 60, 60, 74, 75, 76, 76, 77, 75, 73, 71, 70, 69, 68, 67, + 66, 65, 65, 64, 64, 63, 63, 63, 62, 62, 62, 61, 61, 61, 61, 60, 60, 60, + 60, 60, 73, 74, 74, 75, 75, 74, 72, 71, 69, 68, 67, 66, 65, 65, 64, 64, + 63, 63, 63, 62, 62, 62, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 71, 72, + 73, 73, 74, 73, 71, 70, 68, 67, 67, 66, 65, 64, 64, 63, 63, 63, 62, 62, + 62, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 70, 71, 71, 72, 73, 71, + 70, 69, 68, 67, 66, 65, 64, 64, 63, 63, 62, 62, 62, 61, 61, 61, 61, 60, + 60, 60, 60, 60, 59, 59, 59, 59, 69, 69, 70, 71, 71, 70, 69, 68, 67, 66, + 65, 65, 64, 63, 63, 62, 62, 62, 61, 61, 61, 61, 60, 60, 60, 60, 60, 59, + 59, 59, 59, 59, 68, 68, 69, 70, 70, 69, 68, 67, 66, 66, 65, 64, 63, 63, + 63, 62, 62, 61, 61, 61, 61, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, + 67, 67, 68, 69, 69, 68, 67, 67, 66, 65, 64, 64, 63, 63, 62, 62, 61, 61, + 61, 61, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 66, 66, 67, 67, + 68, 67, 67, 66, 65, 65, 64, 63, 63, 62, 62, 61, 61, 61, 61, 60, 60, 60, + 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 65, 65, 66, 66, 67, 66, 66, 65, + 65, 64, 63, 63, 62, 62, 62, 61, 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 64, 65, 65, 66, 66, 66, 65, 65, 64, 64, 63, 63, + 62, 62, 61, 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 58, 58, + 58, 58, 63, 64, 64, 65, 65, 65, 65, 64, 64, 63, 63, 62, 62, 61, 61, 61, + 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 63, 63, + 64, 64, 65, 64, 64, 63, 63, 63, 62, 62, 61, 61, 61, 60, 60, 60, 60, 59, + 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 58, 62, 63, 63, 63, 64, 64, + 63, 63, 63, 62, 62, 61, 61, 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, 59, + 59, 58, 58, 58, 58, 58, 58, 58, 62, 62, 63, 63, 63, 63, 63, 62, 62, 62, + 62, 61, 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, + 58, 58, 58, 58, 61, 62, 62, 62, 63, 63, 62, 62, 62, 61, 61, 61, 61, 60, + 60, 60, 60, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 58, 58, 58, 58, + 61, 61, 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, 60, 60, 60, 60, 59, 59, + 59, 59, 59, 59, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 60, 61, 61, 61, + 62, 62, 61, 61, 61, 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 58, + 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 60, 61, 61, 61, 62, 62, 61, 61, + 61, 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, + 58, 58, 58, 58, 58, 58, 60, 61, 61, 61, 62, 62, 61, 61, 61, 61, 61, 60, + 60, 60, 60, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 58, 58, 58, 58, + 58, 58, 60, 61, 61, 61, 62, 62, 61, 61, 61, 61, 61, 60, 60, 60, 60, 59, + 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 56, 54, 52, 56, 53, 52, 51, 54, 52, 50, 49, 52, 51, 49, 48, + /* Size 8 */ + 64, 69, 57, 56, 55, 54, 53, 52, 69, 61, 57, 58, 58, 57, 55, 53, 57, 57, + 55, 55, 55, 55, 54, 53, 56, 58, 55, 54, 53, 53, 52, 51, 55, 58, 55, 53, + 52, 51, 51, 50, 54, 57, 55, 53, 51, 51, 50, 50, 53, 55, 54, 52, 51, 50, + 49, 49, 52, 53, 53, 51, 50, 50, 49, 49, + /* Size 16 */ + 64, 67, 69, 63, 57, 57, 56, 56, 55, 55, 54, 53, 53, 52, 52, 52, 67, 66, + 65, 61, 57, 57, 57, 57, 57, 56, 55, 55, 54, 53, 52, 52, 69, 65, 61, 59, + 57, 58, 58, 58, 58, 57, 57, 56, 55, 54, 53, 53, 63, 61, 59, 57, 56, 56, + 57, 57, 57, 56, 56, 55, 54, 54, 53, 53, 57, 57, 57, 56, 55, 55, 55, 55, + 55, 55, 55, 54, 54, 53, 53, 53, 57, 57, 58, 56, 55, 55, 54, 54, 54, 54, + 54, 53, 53, 52, 52, 52, 56, 57, 58, 57, 55, 54, 54, 53, 53, 53, 53, 52, + 52, 52, 51, 51, 56, 57, 58, 57, 55, 54, 53, 53, 53, 52, 52, 52, 52, 51, + 51, 51, 55, 57, 58, 57, 55, 54, 53, 53, 52, 52, 51, 51, 51, 51, 50, 50, + 55, 56, 57, 56, 55, 54, 53, 52, 52, 51, 51, 51, 51, 50, 50, 50, 54, 55, + 57, 56, 55, 54, 53, 52, 51, 51, 51, 50, 50, 50, 50, 50, 53, 55, 56, 55, + 54, 53, 52, 52, 51, 51, 50, 50, 50, 50, 49, 49, 53, 54, 55, 54, 54, 53, + 52, 52, 51, 51, 50, 50, 49, 49, 49, 49, 52, 53, 54, 54, 53, 52, 52, 51, + 51, 50, 50, 50, 49, 49, 49, 49, 52, 52, 53, 53, 53, 52, 51, 51, 50, 50, + 50, 49, 49, 49, 49, 49, 52, 52, 53, 53, 53, 52, 51, 51, 50, 50, 50, 49, + 49, 49, 49, 49, + /* Size 32 */ + 64, 65, 67, 68, 69, 66, 63, 60, 57, 57, 57, 57, 56, 56, 56, 56, 55, 55, + 55, 54, 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, 52, 52, 65, 66, 66, 67, + 67, 65, 62, 60, 57, 57, 57, 57, 57, 57, 56, 56, 56, 56, 55, 55, 55, 54, + 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, 67, 66, 66, 65, 65, 63, 61, 59, + 57, 57, 57, 57, 57, 57, 57, 57, 57, 56, 56, 56, 55, 55, 55, 54, 54, 53, + 53, 53, 52, 52, 52, 52, 68, 67, 65, 64, 63, 61, 60, 58, 57, 57, 57, 58, + 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, 55, 55, 54, 54, 54, 53, 53, 53, + 53, 53, 69, 67, 65, 63, 61, 60, 59, 58, 57, 57, 58, 58, 58, 58, 58, 58, + 58, 58, 57, 57, 57, 56, 56, 55, 55, 55, 54, 54, 53, 53, 53, 53, 66, 65, + 63, 61, 60, 59, 58, 57, 56, 57, 57, 57, 58, 58, 57, 57, 57, 57, 57, 56, + 56, 56, 55, 55, 55, 54, 54, 54, 53, 53, 53, 53, 63, 62, 61, 60, 59, 58, + 57, 57, 56, 56, 56, 57, 57, 57, 57, 57, 57, 56, 56, 56, 56, 55, 55, 55, + 54, 54, 54, 53, 53, 53, 53, 53, 60, 60, 59, 58, 58, 57, 57, 56, 55, 55, + 56, 56, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 54, 54, 54, 53, 53, + 53, 53, 53, 53, 57, 57, 57, 57, 57, 56, 56, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, 53, 53, 53, 53, 53, 53, 53, + 57, 57, 57, 57, 57, 57, 56, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 54, 54, 54, 54, 54, 53, 53, 53, 53, 53, 52, 52, 52, 52, 57, 57, 57, 57, + 58, 57, 56, 56, 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 54, 53, + 53, 53, 53, 53, 52, 52, 52, 52, 52, 52, 57, 57, 57, 58, 58, 57, 57, 56, + 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 53, 53, 53, 53, 53, 53, 53, 52, + 52, 52, 52, 52, 52, 52, 56, 57, 57, 58, 58, 58, 57, 56, 55, 55, 54, 54, + 54, 54, 53, 53, 53, 53, 53, 53, 53, 53, 52, 52, 52, 52, 52, 52, 51, 51, + 51, 51, 56, 57, 57, 58, 58, 58, 57, 56, 55, 55, 54, 54, 54, 53, 53, 53, + 53, 53, 53, 53, 52, 52, 52, 52, 52, 52, 52, 51, 51, 51, 51, 51, 56, 56, + 57, 58, 58, 57, 57, 56, 55, 55, 54, 54, 53, 53, 53, 53, 53, 52, 52, 52, + 52, 52, 52, 52, 52, 51, 51, 51, 51, 51, 51, 51, 56, 56, 57, 57, 58, 57, + 57, 56, 55, 55, 54, 54, 53, 53, 53, 53, 52, 52, 52, 52, 52, 52, 51, 51, + 51, 51, 51, 51, 51, 51, 51, 51, 55, 56, 57, 57, 58, 57, 57, 56, 55, 55, + 54, 54, 53, 53, 53, 52, 52, 52, 52, 52, 51, 51, 51, 51, 51, 51, 51, 51, + 50, 50, 50, 50, 55, 56, 56, 57, 58, 57, 56, 56, 55, 55, 54, 54, 53, 53, + 52, 52, 52, 52, 52, 51, 51, 51, 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, + 55, 55, 56, 57, 57, 57, 56, 55, 55, 54, 54, 53, 53, 53, 52, 52, 52, 52, + 51, 51, 51, 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, 50, 54, 55, 56, 56, + 57, 56, 56, 55, 55, 54, 54, 53, 53, 53, 52, 52, 52, 51, 51, 51, 51, 51, + 51, 50, 50, 50, 50, 50, 50, 50, 50, 50, 54, 55, 55, 56, 57, 56, 56, 55, + 55, 54, 54, 53, 53, 52, 52, 52, 51, 51, 51, 51, 51, 50, 50, 50, 50, 50, + 50, 50, 50, 50, 50, 50, 54, 54, 55, 56, 56, 56, 55, 55, 54, 54, 53, 53, + 53, 52, 52, 52, 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, + 50, 50, 53, 54, 55, 55, 56, 55, 55, 55, 54, 54, 53, 53, 52, 52, 52, 51, + 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, 50, 49, 49, 49, 49, 49, 53, 54, + 54, 55, 55, 55, 55, 54, 54, 53, 53, 53, 52, 52, 52, 51, 51, 51, 51, 50, + 50, 50, 50, 50, 50, 50, 49, 49, 49, 49, 49, 49, 53, 53, 54, 54, 55, 55, + 54, 54, 54, 53, 53, 53, 52, 52, 52, 51, 51, 51, 51, 50, 50, 50, 50, 50, + 49, 49, 49, 49, 49, 49, 49, 49, 52, 53, 53, 54, 55, 54, 54, 54, 53, 53, + 53, 52, 52, 52, 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, 49, 49, 49, 49, + 49, 49, 49, 49, 52, 53, 53, 54, 54, 54, 54, 53, 53, 53, 52, 52, 52, 52, + 51, 51, 51, 50, 50, 50, 50, 50, 50, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 52, 52, 53, 53, 54, 54, 53, 53, 53, 53, 52, 52, 52, 51, 51, 51, 51, 50, + 50, 50, 50, 50, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 52, 52, 52, 53, + 53, 53, 53, 53, 53, 52, 52, 52, 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 52, 52, 52, 53, 53, 53, 53, 53, + 53, 52, 52, 52, 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 52, 52, 52, 53, 53, 53, 53, 53, 53, 52, 52, 52, + 51, 51, 51, 51, 50, 50, 50, 50, 50, 50, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 52, 52, 52, 53, 53, 53, 53, 53, 53, 52, 52, 52, 51, 51, 51, 51, + 50, 50, 50, 50, 50, 50, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49 }, + { /* Intra matrices */ + /* Size 4 */ + 79, 68, 67, 63, 68, 65, 64, 62, 67, 64, 61, 60, 63, 62, 60, 58, + /* Size 8 */ + 77, 84, 68, 67, 66, 64, 63, 61, 84, 73, 68, 70, 69, 67, 65, 64, 68, 68, + 65, 66, 66, 65, 64, 62, 67, 70, 66, 64, 63, 63, 62, 61, 66, 69, 66, 63, + 62, 61, 60, 60, 64, 67, 65, 63, 61, 60, 59, 59, 63, 65, 64, 62, 60, 59, + 59, 58, 61, 64, 62, 61, 60, 59, 58, 58, + /* Size 16 */ + 77, 81, 84, 76, 69, 68, 68, 67, 67, 66, 65, 64, 63, 62, 62, 62, 81, 80, + 79, 74, 69, 69, 69, 69, 68, 67, 66, 65, 64, 64, 63, 63, 84, 79, 74, 71, + 68, 69, 70, 70, 70, 69, 68, 67, 66, 65, 64, 64, 76, 74, 71, 69, 67, 68, + 68, 68, 68, 67, 67, 66, 65, 64, 63, 63, 69, 69, 68, 67, 66, 66, 66, 66, + 66, 66, 66, 65, 64, 64, 63, 63, 68, 69, 69, 68, 66, 66, 65, 65, 65, 65, + 64, 64, 63, 63, 62, 62, 68, 69, 70, 68, 66, 65, 64, 64, 64, 63, 63, 63, + 62, 62, 61, 61, 67, 69, 70, 68, 66, 65, 64, 64, 63, 63, 62, 62, 62, 61, + 61, 61, 67, 68, 70, 68, 66, 65, 64, 63, 62, 62, 61, 61, 61, 61, 60, 60, + 66, 67, 69, 67, 66, 65, 63, 63, 62, 61, 61, 61, 60, 60, 60, 60, 65, 66, + 68, 67, 66, 64, 63, 62, 61, 61, 60, 60, 60, 59, 59, 59, 64, 65, 67, 66, + 65, 64, 63, 62, 61, 61, 60, 60, 59, 59, 59, 59, 63, 64, 66, 65, 64, 63, + 62, 62, 61, 60, 60, 59, 59, 59, 58, 58, 62, 64, 65, 64, 64, 63, 62, 61, + 61, 60, 59, 59, 59, 58, 58, 58, 62, 63, 64, 63, 63, 62, 61, 61, 60, 60, + 59, 59, 58, 58, 58, 58, 62, 63, 64, 63, 63, 62, 61, 61, 60, 60, 59, 59, + 58, 58, 58, 58, + /* Size 32 */ + 78, 79, 81, 83, 84, 81, 77, 73, 69, 69, 68, 68, 68, 68, 67, 67, 67, 66, + 66, 65, 65, 65, 64, 64, 63, 63, 63, 62, 62, 62, 62, 62, 79, 80, 81, 81, + 82, 79, 75, 72, 69, 69, 69, 69, 69, 68, 68, 68, 68, 67, 67, 66, 66, 65, + 65, 64, 64, 64, 63, 63, 62, 62, 62, 62, 81, 81, 80, 80, 79, 77, 74, 71, + 69, 69, 69, 69, 69, 69, 69, 69, 68, 68, 67, 67, 67, 66, 66, 65, 65, 64, + 64, 63, 63, 63, 63, 63, 83, 81, 80, 78, 76, 75, 73, 71, 69, 69, 69, 70, + 70, 70, 70, 69, 69, 69, 68, 68, 67, 67, 66, 66, 65, 65, 65, 64, 64, 64, + 64, 64, 84, 82, 79, 76, 74, 73, 71, 70, 69, 69, 70, 70, 71, 70, 70, 70, + 70, 70, 69, 69, 68, 68, 67, 67, 66, 66, 65, 65, 64, 64, 64, 64, 81, 79, + 77, 75, 73, 71, 70, 69, 68, 68, 69, 69, 70, 69, 69, 69, 69, 69, 68, 68, + 68, 67, 67, 66, 66, 65, 65, 64, 64, 64, 64, 64, 77, 75, 74, 73, 71, 70, + 69, 68, 67, 68, 68, 68, 69, 68, 68, 68, 68, 68, 68, 67, 67, 67, 66, 66, + 65, 65, 64, 64, 64, 64, 64, 64, 73, 72, 71, 71, 70, 69, 68, 67, 67, 67, + 67, 67, 68, 67, 67, 67, 67, 67, 67, 67, 66, 66, 66, 65, 65, 65, 64, 64, + 63, 63, 63, 63, 69, 69, 69, 69, 69, 68, 67, 67, 66, 66, 66, 66, 66, 67, + 67, 67, 67, 66, 66, 66, 66, 65, 65, 65, 64, 64, 64, 63, 63, 63, 63, 63, + 69, 69, 69, 69, 69, 68, 68, 67, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, + 66, 65, 65, 65, 65, 64, 64, 64, 63, 63, 63, 63, 63, 63, 68, 69, 69, 69, + 70, 69, 68, 67, 66, 66, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 64, + 64, 64, 64, 63, 63, 63, 62, 62, 62, 62, 68, 69, 69, 70, 70, 69, 68, 67, + 66, 66, 66, 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, 64, 63, 63, 63, 63, + 63, 62, 62, 62, 62, 62, 68, 69, 69, 70, 71, 70, 69, 68, 66, 66, 66, 65, + 65, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, + 62, 62, 68, 68, 69, 70, 70, 69, 68, 67, 67, 66, 65, 65, 64, 64, 64, 64, + 64, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, 61, 67, 68, + 69, 70, 70, 69, 68, 67, 67, 66, 65, 65, 64, 64, 64, 63, 63, 63, 63, 63, + 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 67, 68, 69, 69, 70, 69, + 68, 67, 67, 66, 65, 65, 64, 64, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, + 61, 61, 61, 61, 61, 61, 61, 61, 67, 68, 68, 69, 70, 69, 68, 67, 67, 66, + 65, 65, 64, 64, 63, 63, 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, + 60, 60, 60, 60, 66, 67, 68, 69, 70, 69, 68, 67, 66, 66, 65, 64, 64, 63, + 63, 63, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, + 66, 67, 67, 68, 69, 68, 68, 67, 66, 66, 65, 64, 64, 63, 63, 62, 62, 62, + 62, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 65, 66, 67, 68, + 69, 68, 67, 67, 66, 65, 65, 64, 63, 63, 63, 62, 62, 62, 61, 61, 61, 61, + 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 65, 66, 67, 67, 68, 68, 67, 66, + 66, 65, 65, 64, 63, 63, 62, 62, 62, 61, 61, 61, 61, 60, 60, 60, 60, 60, + 60, 60, 59, 59, 59, 59, 65, 65, 66, 67, 68, 67, 67, 66, 65, 65, 64, 64, + 63, 63, 62, 62, 62, 61, 61, 61, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, + 59, 59, 64, 65, 66, 66, 67, 67, 66, 66, 65, 65, 64, 63, 63, 63, 62, 62, + 61, 61, 61, 61, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 64, 64, + 65, 66, 67, 66, 66, 65, 65, 64, 64, 63, 63, 62, 62, 62, 61, 61, 61, 60, + 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 63, 64, 65, 65, 66, 66, + 65, 65, 64, 64, 64, 63, 63, 62, 62, 61, 61, 61, 60, 60, 60, 60, 60, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 63, 64, 64, 65, 66, 65, 65, 65, 64, 64, + 63, 63, 62, 62, 62, 61, 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 63, 63, 64, 65, 65, 65, 64, 64, 64, 63, 63, 63, 62, 62, + 61, 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, + 62, 63, 63, 64, 65, 64, 64, 64, 63, 63, 63, 62, 62, 62, 61, 61, 61, 60, + 60, 60, 60, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 62, 62, 63, 64, + 64, 64, 64, 63, 63, 63, 62, 62, 62, 61, 61, 61, 60, 60, 60, 60, 59, 59, + 59, 59, 59, 59, 58, 58, 58, 58, 58, 58, 62, 62, 63, 64, 64, 64, 64, 63, + 63, 63, 62, 62, 62, 61, 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, + 58, 58, 58, 58, 58, 58, 62, 62, 63, 64, 64, 64, 64, 63, 63, 63, 62, 62, + 62, 61, 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, + 58, 58, 62, 62, 63, 64, 64, 64, 64, 63, 63, 63, 62, 62, 62, 61, 61, 61, + 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 58 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 63, 57, 54, 63, 58, 55, 54, 57, 55, 53, 53, 54, 54, 53, 52, + /* Size 8 */ + 64, 68, 67, 63, 60, 58, 56, 55, 68, 66, 67, 64, 61, 59, 57, 56, 67, 67, + 62, 61, 59, 58, 57, 56, 63, 64, 61, 58, 57, 56, 56, 55, 60, 61, 59, 57, + 56, 56, 55, 55, 58, 59, 58, 56, 56, 55, 55, 54, 56, 57, 57, 56, 55, 55, + 54, 54, 55, 56, 56, 55, 55, 54, 54, 54, + /* Size 16 */ + 64, 66, 68, 68, 67, 65, 63, 62, 60, 59, 58, 57, 56, 56, 55, 55, 66, 67, + 67, 67, 67, 65, 64, 62, 61, 60, 58, 58, 57, 56, 56, 56, 68, 67, 66, 66, + 67, 66, 64, 63, 61, 60, 59, 58, 57, 57, 56, 56, 68, 67, 66, 65, 65, 63, + 62, 61, 60, 59, 58, 58, 57, 56, 56, 56, 67, 67, 67, 65, 62, 61, 61, 60, + 59, 58, 58, 57, 57, 56, 56, 56, 65, 65, 66, 63, 61, 60, 59, 59, 58, 58, + 57, 57, 56, 56, 56, 56, 63, 64, 64, 62, 61, 59, 58, 58, 57, 57, 56, 56, + 56, 56, 55, 55, 62, 62, 63, 61, 60, 59, 58, 57, 57, 56, 56, 56, 55, 55, + 55, 55, 60, 61, 61, 60, 59, 58, 57, 57, 56, 56, 56, 55, 55, 55, 55, 55, + 59, 60, 60, 59, 58, 58, 57, 56, 56, 56, 55, 55, 55, 55, 55, 55, 58, 58, + 59, 58, 58, 57, 56, 56, 56, 55, 55, 55, 55, 55, 54, 54, 57, 58, 58, 58, + 57, 57, 56, 56, 55, 55, 55, 55, 55, 54, 54, 54, 56, 57, 57, 57, 57, 56, + 56, 55, 55, 55, 55, 55, 54, 54, 54, 54, 56, 56, 57, 56, 56, 56, 56, 55, + 55, 55, 55, 54, 54, 54, 54, 54, 55, 56, 56, 56, 56, 56, 55, 55, 55, 55, + 54, 54, 54, 54, 54, 54, 55, 56, 56, 56, 56, 56, 55, 55, 55, 55, 54, 54, + 54, 54, 54, 54, + /* Size 32 */ + 64, 65, 66, 67, 68, 68, 68, 67, 67, 66, 65, 64, 63, 62, 62, 61, 60, 59, + 59, 58, 58, 57, 57, 57, 56, 56, 56, 56, 55, 55, 55, 55, 65, 66, 66, 67, + 68, 68, 67, 67, 67, 66, 65, 64, 63, 63, 62, 61, 60, 60, 59, 59, 58, 58, + 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, 66, 66, 67, 67, 67, 67, 67, 67, + 67, 66, 65, 65, 64, 63, 62, 61, 61, 60, 60, 59, 58, 58, 58, 57, 57, 57, + 56, 56, 56, 56, 56, 56, 67, 67, 67, 67, 67, 67, 67, 67, 67, 66, 66, 65, + 64, 63, 63, 62, 61, 60, 60, 59, 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, + 56, 56, 68, 68, 67, 67, 66, 66, 66, 67, 67, 66, 66, 65, 64, 64, 63, 62, + 61, 61, 60, 60, 59, 59, 58, 58, 57, 57, 57, 56, 56, 56, 56, 56, 68, 68, + 67, 67, 66, 66, 66, 66, 66, 65, 65, 64, 63, 63, 62, 62, 61, 60, 60, 59, + 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 56, 56, 68, 67, 67, 67, 66, 66, + 65, 65, 65, 64, 63, 63, 62, 62, 61, 61, 60, 60, 59, 59, 58, 58, 58, 57, + 57, 57, 56, 56, 56, 56, 56, 56, 67, 67, 67, 67, 67, 66, 65, 64, 63, 63, + 62, 62, 62, 61, 61, 60, 60, 59, 59, 58, 58, 58, 57, 57, 57, 57, 56, 56, + 56, 56, 56, 56, 67, 67, 67, 67, 67, 66, 65, 63, 62, 62, 61, 61, 61, 60, + 60, 59, 59, 59, 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, + 66, 66, 66, 66, 66, 65, 64, 63, 62, 61, 61, 60, 60, 60, 59, 59, 59, 58, + 58, 58, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, 56, 65, 65, 65, 66, + 66, 65, 63, 62, 61, 61, 60, 60, 59, 59, 59, 59, 58, 58, 58, 57, 57, 57, + 57, 56, 56, 56, 56, 56, 56, 56, 56, 56, 64, 64, 65, 65, 65, 64, 63, 62, + 61, 60, 60, 59, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, 56, + 56, 56, 55, 55, 55, 55, 63, 63, 64, 64, 64, 63, 62, 62, 61, 60, 59, 59, + 58, 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, 55, 55, 55, + 55, 55, 62, 63, 63, 63, 64, 63, 62, 61, 60, 60, 59, 59, 58, 58, 58, 57, + 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 62, 62, + 62, 63, 63, 62, 61, 61, 60, 59, 59, 58, 58, 58, 57, 57, 57, 57, 56, 56, + 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 61, 61, 61, 62, 62, 62, + 61, 60, 59, 59, 59, 58, 58, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 60, 60, 61, 61, 61, 61, 60, 60, 59, 59, + 58, 58, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 59, 60, 60, 60, 61, 60, 60, 59, 59, 58, 58, 58, 57, 57, + 57, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 59, 59, 60, 60, 60, 60, 59, 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 56, + 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 58, 59, 59, 59, + 60, 59, 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 56, 56, 56, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 58, 58, 58, 59, 59, 59, 58, 58, + 58, 57, 57, 57, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 54, 54, 54, 54, 57, 58, 58, 58, 59, 58, 58, 58, 57, 57, 57, 57, + 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 54, 54, 54, + 54, 54, 57, 57, 58, 58, 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, 56, 56, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 57, 57, + 57, 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 56, 57, 57, 57, 57, 57, + 57, 57, 57, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 55, 54, + 54, 54, 54, 54, 54, 54, 54, 54, 56, 56, 57, 57, 57, 57, 57, 57, 56, 56, + 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, + 54, 54, 54, 54, 56, 56, 56, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 55, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 55, 54, 54, + 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 55, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, + 54, 54, 54, 54, 54, 54, 55, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, + 54, 54, 55, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54 }, + { /* Intra matrices */ + /* Size 4 */ + 74, 72, 66, 62, 72, 67, 63, 61, 66, 63, 61, 60, 62, 61, 60, 59, + /* Size 8 */ + 71, 76, 75, 70, 66, 64, 62, 61, 76, 73, 74, 71, 68, 65, 63, 62, 75, 74, + 69, 67, 65, 64, 62, 61, 70, 71, 67, 64, 63, 62, 61, 61, 66, 68, 65, 63, + 62, 61, 61, 60, 64, 65, 64, 62, 61, 60, 60, 60, 62, 63, 62, 61, 61, 60, + 60, 59, 61, 62, 61, 61, 60, 60, 59, 59, + /* Size 16 */ + 71, 74, 77, 76, 75, 73, 70, 68, 67, 65, 64, 63, 62, 62, 61, 61, 74, 75, + 75, 75, 75, 73, 71, 69, 67, 66, 65, 64, 63, 62, 62, 62, 77, 75, 74, 74, + 75, 73, 72, 70, 68, 67, 66, 65, 64, 63, 62, 62, 76, 75, 74, 73, 72, 71, + 70, 68, 67, 66, 65, 64, 63, 63, 62, 62, 75, 75, 75, 72, 69, 68, 67, 66, + 66, 65, 64, 63, 63, 62, 62, 62, 73, 73, 73, 71, 68, 67, 66, 65, 65, 64, + 63, 63, 62, 62, 61, 61, 70, 71, 72, 70, 67, 66, 65, 64, 63, 63, 63, 62, + 62, 61, 61, 61, 68, 69, 70, 68, 66, 65, 64, 64, 63, 62, 62, 62, 61, 61, + 61, 61, 67, 67, 68, 67, 66, 65, 63, 63, 62, 62, 62, 61, 61, 61, 61, 61, + 65, 66, 67, 66, 65, 64, 63, 62, 62, 62, 61, 61, 61, 61, 60, 60, 64, 65, + 66, 65, 64, 63, 63, 62, 62, 61, 61, 61, 60, 60, 60, 60, 63, 64, 65, 64, + 63, 63, 62, 62, 61, 61, 61, 60, 60, 60, 60, 60, 62, 63, 64, 63, 63, 62, + 62, 61, 61, 61, 60, 60, 60, 60, 60, 60, 62, 62, 63, 63, 62, 62, 61, 61, + 61, 61, 60, 60, 60, 60, 60, 60, 61, 62, 62, 62, 62, 61, 61, 61, 61, 60, + 60, 60, 60, 60, 60, 60, 61, 62, 62, 62, 62, 61, 61, 61, 61, 60, 60, 60, + 60, 60, 60, 60, + /* Size 32 */ + 72, 73, 74, 76, 77, 76, 76, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, 66, + 66, 65, 64, 64, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 73, 74, 75, 75, + 76, 76, 76, 75, 75, 74, 73, 72, 71, 70, 69, 68, 67, 67, 66, 65, 65, 64, + 64, 63, 63, 63, 62, 62, 62, 62, 62, 62, 74, 75, 75, 75, 76, 75, 75, 75, + 75, 74, 73, 72, 71, 70, 70, 69, 68, 67, 66, 66, 65, 65, 64, 64, 63, 63, + 63, 62, 62, 62, 62, 62, 76, 75, 75, 75, 75, 75, 75, 75, 75, 74, 73, 73, + 72, 71, 70, 69, 68, 67, 67, 66, 65, 65, 64, 64, 63, 63, 63, 62, 62, 62, + 62, 62, 77, 76, 76, 75, 74, 74, 75, 75, 75, 74, 74, 73, 72, 71, 70, 69, + 69, 68, 67, 66, 66, 65, 65, 64, 64, 63, 63, 63, 62, 62, 62, 62, 76, 76, + 75, 75, 74, 74, 74, 74, 74, 73, 72, 72, 71, 70, 69, 69, 68, 67, 67, 66, + 65, 65, 64, 64, 64, 63, 63, 63, 62, 62, 62, 62, 76, 76, 75, 75, 75, 74, + 73, 73, 72, 72, 71, 70, 70, 69, 69, 68, 67, 67, 66, 66, 65, 65, 64, 64, + 63, 63, 63, 62, 62, 62, 62, 62, 76, 75, 75, 75, 75, 74, 73, 72, 71, 70, + 70, 69, 69, 68, 68, 67, 66, 66, 66, 65, 65, 64, 64, 63, 63, 63, 63, 62, + 62, 62, 62, 62, 75, 75, 75, 75, 75, 74, 72, 71, 69, 69, 68, 68, 67, 67, + 67, 66, 66, 65, 65, 65, 64, 64, 64, 63, 63, 63, 62, 62, 62, 62, 62, 62, + 74, 74, 74, 74, 74, 73, 72, 70, 69, 68, 68, 67, 67, 66, 66, 66, 65, 65, + 65, 64, 64, 64, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 73, 73, 73, 73, + 74, 72, 71, 70, 68, 68, 67, 67, 66, 66, 65, 65, 65, 64, 64, 64, 63, 63, + 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 72, 72, 72, 73, 73, 72, 70, 69, + 68, 67, 67, 66, 66, 65, 65, 65, 64, 64, 64, 63, 63, 63, 63, 62, 62, 62, + 62, 62, 61, 61, 61, 61, 71, 71, 71, 72, 72, 71, 70, 69, 67, 67, 66, 66, + 65, 65, 64, 64, 64, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, + 61, 61, 70, 70, 70, 71, 71, 70, 69, 68, 67, 66, 66, 65, 65, 64, 64, 64, + 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 69, 69, + 70, 70, 70, 69, 69, 68, 67, 66, 65, 65, 64, 64, 64, 63, 63, 63, 63, 62, + 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 68, 68, 69, 69, 69, 69, + 68, 67, 66, 66, 65, 65, 64, 64, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, + 61, 61, 61, 61, 61, 61, 61, 61, 67, 67, 68, 68, 69, 68, 67, 66, 66, 65, + 65, 64, 64, 63, 63, 63, 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, + 61, 61, 61, 61, 66, 67, 67, 67, 68, 67, 67, 66, 65, 65, 64, 64, 63, 63, + 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, + 66, 66, 66, 67, 67, 67, 66, 66, 65, 65, 64, 64, 63, 63, 63, 62, 62, 62, + 62, 62, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 65, 65, 66, 66, + 66, 66, 66, 65, 65, 64, 64, 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, + 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 64, 65, 65, 65, 66, 65, 65, 65, + 64, 64, 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 61, + 60, 60, 60, 60, 60, 60, 64, 64, 65, 65, 65, 65, 65, 64, 64, 64, 63, 63, + 63, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, + 60, 60, 63, 64, 64, 64, 65, 64, 64, 64, 64, 63, 63, 63, 62, 62, 62, 62, + 61, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 63, 63, + 64, 64, 64, 64, 64, 63, 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, 61, + 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 63, 63, 63, 63, 64, 64, + 63, 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 60, 60, + 60, 60, 60, 60, 60, 60, 60, 60, 62, 63, 63, 63, 63, 63, 63, 63, 63, 62, + 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, + 60, 60, 60, 60, 62, 62, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 61, + 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, + 62, 62, 62, 62, 63, 63, 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, + 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 62, 62, 62, 62, + 62, 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, + 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 62, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, + 60, 60, 60, 60, 60, 60, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 61, + 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, + 60, 60, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, + 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 58, 57, 55, 58, 56, 56, 55, 57, 56, 54, 53, 55, 55, 53, 53, + /* Size 8 */ + 64, 68, 59, 59, 58, 57, 56, 55, 68, 62, 59, 60, 60, 59, 58, 57, 59, 59, + 58, 58, 58, 57, 57, 56, 59, 60, 58, 57, 56, 56, 56, 55, 58, 60, 58, 56, + 56, 55, 55, 55, 57, 59, 57, 56, 55, 55, 54, 54, 56, 58, 57, 56, 55, 54, + 54, 54, 55, 57, 56, 55, 55, 54, 54, 53, + /* Size 16 */ + 64, 66, 68, 63, 59, 59, 59, 58, 58, 58, 57, 57, 56, 56, 55, 55, 66, 65, + 65, 62, 59, 59, 59, 59, 59, 58, 58, 57, 57, 56, 56, 56, 68, 65, 62, 60, + 59, 60, 60, 60, 60, 59, 59, 58, 58, 57, 57, 57, 63, 62, 60, 59, 58, 59, + 59, 59, 59, 58, 58, 58, 57, 57, 56, 56, 59, 59, 59, 58, 58, 58, 58, 58, + 58, 58, 57, 57, 57, 56, 56, 56, 59, 59, 60, 59, 58, 58, 57, 57, 57, 57, + 57, 57, 56, 56, 56, 56, 59, 59, 60, 59, 58, 57, 57, 57, 56, 56, 56, 56, + 56, 55, 55, 55, 58, 59, 60, 59, 58, 57, 57, 56, 56, 56, 56, 55, 55, 55, + 55, 55, 58, 59, 60, 59, 58, 57, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, + 58, 58, 59, 58, 58, 57, 56, 56, 55, 55, 55, 55, 55, 54, 54, 54, 57, 58, + 59, 58, 57, 57, 56, 56, 55, 55, 55, 54, 54, 54, 54, 54, 57, 57, 58, 58, + 57, 57, 56, 55, 55, 55, 54, 54, 54, 54, 54, 54, 56, 57, 58, 57, 57, 56, + 56, 55, 55, 55, 54, 54, 54, 54, 54, 54, 56, 56, 57, 57, 56, 56, 55, 55, + 55, 54, 54, 54, 54, 54, 53, 53, 55, 56, 57, 56, 56, 56, 55, 55, 55, 54, + 54, 54, 54, 53, 53, 53, 55, 56, 57, 56, 56, 56, 55, 55, 55, 54, 54, 54, + 54, 53, 53, 53, + /* Size 32 */ + 64, 65, 66, 67, 68, 66, 63, 61, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, + 58, 57, 57, 57, 57, 56, 56, 56, 56, 55, 55, 55, 55, 55, 65, 65, 66, 66, + 66, 64, 63, 61, 59, 59, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 57, 57, + 57, 57, 56, 56, 56, 56, 56, 56, 56, 56, 66, 66, 65, 65, 65, 63, 62, 61, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, + 56, 56, 56, 56, 56, 56, 67, 66, 65, 64, 63, 62, 61, 60, 59, 59, 59, 60, + 60, 60, 60, 59, 59, 59, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, 56, 56, + 56, 56, 68, 66, 65, 63, 62, 61, 60, 60, 59, 59, 60, 60, 60, 60, 60, 60, + 60, 60, 59, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, 57, 57, 57, 66, 64, + 63, 62, 61, 61, 60, 59, 59, 59, 59, 59, 60, 59, 59, 59, 59, 59, 59, 59, + 58, 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, 56, 63, 63, 62, 61, 60, 60, + 59, 59, 58, 58, 59, 59, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 57, + 57, 57, 57, 57, 56, 56, 56, 56, 61, 61, 61, 60, 60, 59, 59, 58, 58, 58, + 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 57, 57, 57, 57, 57, 56, + 56, 56, 56, 56, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 58, 58, 58, + 58, 58, 58, 58, 58, 58, 57, 57, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, + 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 57, + 57, 57, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, 56, 59, 59, 59, 59, + 60, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, + 57, 56, 56, 56, 56, 56, 56, 56, 56, 56, 59, 59, 59, 60, 60, 59, 59, 58, + 58, 58, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, + 56, 56, 55, 55, 55, 55, 59, 59, 59, 60, 60, 60, 59, 58, 58, 58, 57, 57, + 57, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, + 55, 55, 58, 59, 59, 60, 60, 59, 59, 58, 58, 58, 57, 57, 57, 57, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 58, 59, + 59, 60, 60, 59, 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 58, 59, 59, 59, 60, 59, + 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 58, 58, 59, 59, 60, 59, 59, 58, 58, 58, + 57, 57, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 58, 58, 59, 59, 60, 59, 59, 58, 58, 57, 57, 57, 56, 56, + 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, + 58, 58, 58, 59, 59, 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 56, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 57, 58, 58, 59, + 59, 59, 58, 58, 58, 57, 57, 57, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, + 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 57, 57, 58, 58, 59, 58, 58, 58, + 57, 57, 57, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, + 54, 54, 54, 54, 54, 54, 57, 57, 58, 58, 58, 58, 58, 58, 57, 57, 57, 56, + 56, 56, 56, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 54, + 54, 54, 57, 57, 57, 58, 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, 55, 55, + 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 56, 57, + 57, 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, + 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 56, 56, 57, 57, 58, 57, + 57, 57, 57, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, + 54, 54, 54, 54, 54, 54, 54, 54, 56, 56, 57, 57, 57, 57, 57, 57, 57, 56, + 56, 56, 56, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, + 53, 53, 53, 53, 56, 56, 56, 57, 57, 57, 57, 57, 56, 56, 56, 56, 55, 55, + 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 54, 53, 53, 53, 53, 53, + 55, 56, 56, 57, 57, 57, 57, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 54, + 54, 54, 54, 54, 54, 54, 54, 54, 53, 53, 53, 53, 53, 53, 55, 56, 56, 56, + 57, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, + 54, 54, 54, 53, 53, 53, 53, 53, 53, 53, 55, 56, 56, 56, 57, 56, 56, 56, + 56, 56, 56, 55, 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 53, + 53, 53, 53, 53, 53, 53, 55, 56, 56, 56, 57, 56, 56, 56, 56, 56, 56, 55, + 55, 55, 55, 55, 55, 54, 54, 54, 54, 54, 54, 54, 54, 53, 53, 53, 53, 53, + 53, 53, 55, 56, 56, 56, 57, 56, 56, 56, 56, 56, 56, 55, 55, 55, 55, 55, + 55, 54, 54, 54, 54, 54, 54, 54, 54, 53, 53, 53, 53, 53, 53, 53 }, + { /* Intra matrices */ + /* Size 4 */ + 74, 67, 66, 63, 67, 65, 64, 63, 66, 64, 62, 61, 63, 63, 61, 60, + /* Size 8 */ + 72, 77, 67, 66, 65, 64, 63, 62, 77, 70, 66, 68, 67, 66, 65, 64, 67, 66, + 65, 65, 65, 65, 64, 63, 66, 68, 65, 64, 63, 63, 63, 62, 65, 67, 65, 63, + 62, 62, 62, 61, 64, 66, 65, 63, 62, 61, 61, 60, 63, 65, 64, 63, 62, 61, + 60, 60, 62, 64, 63, 62, 61, 60, 60, 60, + /* Size 16 */ + 73, 75, 77, 72, 67, 67, 66, 66, 66, 65, 64, 64, 63, 63, 62, 62, 75, 74, + 74, 70, 67, 67, 67, 67, 67, 66, 65, 65, 64, 64, 63, 63, 77, 74, 70, 69, + 67, 67, 68, 68, 68, 67, 67, 66, 65, 65, 64, 64, 72, 70, 69, 67, 66, 66, + 67, 67, 67, 66, 66, 65, 65, 64, 64, 64, 67, 67, 67, 66, 65, 65, 65, 65, + 65, 65, 65, 64, 64, 64, 63, 63, 67, 67, 67, 66, 65, 65, 65, 65, 65, 64, + 64, 64, 63, 63, 63, 63, 66, 67, 68, 67, 65, 65, 64, 64, 64, 64, 63, 63, + 63, 63, 62, 62, 66, 67, 68, 67, 65, 65, 64, 64, 63, 63, 63, 63, 62, 62, + 62, 62, 66, 67, 68, 67, 65, 65, 64, 63, 63, 62, 62, 62, 62, 62, 61, 61, + 65, 66, 67, 66, 65, 64, 64, 63, 62, 62, 62, 62, 61, 61, 61, 61, 64, 65, + 67, 66, 65, 64, 63, 63, 62, 62, 62, 61, 61, 61, 61, 61, 64, 65, 66, 65, + 64, 64, 63, 63, 62, 62, 61, 61, 61, 61, 60, 60, 63, 64, 65, 65, 64, 63, + 63, 62, 62, 61, 61, 61, 61, 60, 60, 60, 63, 64, 65, 64, 64, 63, 63, 62, + 62, 61, 61, 61, 60, 60, 60, 60, 62, 63, 64, 64, 63, 63, 62, 62, 61, 61, + 61, 60, 60, 60, 60, 60, 62, 63, 64, 64, 63, 63, 62, 62, 61, 61, 61, 60, + 60, 60, 60, 60, + /* Size 32 */ + 73, 74, 75, 76, 77, 75, 72, 70, 67, 67, 67, 67, 66, 66, 66, 66, 66, 65, + 65, 65, 65, 64, 64, 64, 63, 63, 63, 63, 62, 62, 62, 62, 74, 74, 75, 75, + 76, 73, 71, 69, 67, 67, 67, 67, 67, 67, 67, 66, 66, 66, 66, 65, 65, 65, + 65, 64, 64, 64, 63, 63, 63, 63, 63, 63, 75, 75, 74, 74, 74, 72, 70, 69, + 67, 67, 67, 67, 67, 67, 67, 67, 67, 66, 66, 66, 66, 65, 65, 65, 64, 64, + 64, 64, 63, 63, 63, 63, 76, 75, 74, 73, 72, 71, 70, 68, 67, 67, 67, 68, + 68, 68, 68, 67, 67, 67, 67, 66, 66, 66, 65, 65, 65, 65, 64, 64, 64, 64, + 64, 64, 77, 76, 74, 72, 70, 70, 69, 68, 67, 67, 68, 68, 68, 68, 68, 68, + 68, 68, 67, 67, 67, 66, 66, 66, 65, 65, 65, 64, 64, 64, 64, 64, 75, 73, + 72, 71, 70, 69, 68, 67, 67, 67, 67, 67, 68, 67, 67, 67, 67, 67, 67, 67, + 66, 66, 66, 65, 65, 65, 64, 64, 64, 64, 64, 64, 72, 71, 70, 70, 69, 68, + 67, 67, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 66, 66, 66, 66, 65, 65, + 65, 65, 64, 64, 64, 64, 64, 64, 70, 69, 69, 68, 68, 67, 67, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 65, 65, 65, 64, 64, 64, 64, + 64, 64, 64, 64, 67, 67, 67, 67, 67, 67, 66, 66, 65, 65, 65, 65, 66, 66, + 66, 66, 66, 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, 64, 63, 63, 63, 63, + 67, 67, 67, 67, 67, 67, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 67, 67, 67, 67, + 68, 67, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, + 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 67, 67, 67, 68, 68, 67, 67, 66, + 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 66, 67, 67, 68, 68, 68, 67, 66, 66, 65, 65, 65, + 64, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, + 62, 62, 66, 67, 67, 68, 68, 67, 67, 66, 66, 65, 65, 65, 64, 64, 64, 64, + 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 66, 67, + 67, 68, 68, 67, 67, 66, 66, 65, 65, 64, 64, 64, 64, 64, 63, 63, 63, 63, + 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 66, 66, 67, 67, 68, 67, + 67, 66, 66, 65, 65, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 62, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 66, 66, 67, 67, 68, 67, 67, 66, 66, 65, + 65, 64, 64, 64, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 62, 65, 66, 66, 67, 68, 67, 67, 66, 65, 65, 65, 64, 64, 64, + 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, + 65, 66, 66, 67, 67, 67, 66, 66, 65, 65, 64, 64, 64, 63, 63, 63, 63, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 65, 65, 66, 66, + 67, 67, 66, 66, 65, 65, 64, 64, 64, 63, 63, 63, 63, 62, 62, 62, 62, 62, + 62, 62, 61, 61, 61, 61, 61, 61, 61, 61, 65, 65, 66, 66, 67, 66, 66, 65, + 65, 65, 64, 64, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, + 61, 61, 61, 61, 61, 61, 64, 65, 65, 66, 66, 66, 66, 65, 65, 64, 64, 64, + 63, 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 61, 61, + 61, 61, 64, 65, 65, 65, 66, 66, 65, 65, 65, 64, 64, 64, 63, 63, 63, 62, + 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 64, 64, + 65, 65, 66, 65, 65, 65, 64, 64, 64, 63, 63, 63, 63, 62, 62, 62, 62, 62, + 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 63, 64, 64, 65, 65, 65, + 65, 64, 64, 64, 64, 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, + 61, 61, 61, 60, 60, 60, 60, 60, 63, 64, 64, 65, 65, 65, 65, 64, 64, 64, + 63, 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 60, 60, + 60, 60, 60, 60, 63, 63, 64, 64, 65, 64, 64, 64, 64, 64, 63, 63, 63, 62, + 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, + 63, 63, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 62, 62, 62, 62, 61, + 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 62, 63, 63, 64, + 64, 64, 64, 64, 63, 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, + 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 62, 63, 63, 64, 64, 64, 64, 64, + 63, 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 60, 60, + 60, 60, 60, 60, 60, 60, 62, 63, 63, 64, 64, 64, 64, 64, 63, 63, 63, 63, + 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, + 60, 60, 62, 63, 63, 64, 64, 64, 64, 64, 63, 63, 63, 63, 62, 62, 62, 62, + 62, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 63, 60, 59, 63, 61, 59, 59, 60, 59, 58, 58, 59, 59, 58, 58, + /* Size 8 */ + 64, 66, 66, 64, 62, 61, 60, 60, 66, 65, 65, 64, 63, 61, 61, 60, 66, 65, + 63, 62, 61, 61, 60, 60, 64, 64, 62, 61, 61, 60, 60, 59, 62, 63, 61, 61, + 60, 60, 59, 59, 61, 61, 61, 60, 60, 59, 59, 59, 60, 61, 60, 60, 59, 59, + 59, 59, 60, 60, 60, 59, 59, 59, 59, 59, + /* Size 16 */ + 64, 65, 66, 66, 66, 65, 64, 63, 62, 61, 61, 60, 60, 60, 60, 60, 65, 65, + 66, 66, 66, 65, 64, 63, 62, 62, 61, 61, 60, 60, 60, 60, 66, 66, 65, 65, + 65, 65, 64, 63, 63, 62, 61, 61, 61, 60, 60, 60, 66, 66, 65, 65, 64, 64, + 63, 63, 62, 62, 61, 61, 60, 60, 60, 60, 66, 66, 65, 64, 63, 63, 62, 62, + 61, 61, 61, 60, 60, 60, 60, 60, 65, 65, 65, 64, 63, 62, 62, 61, 61, 61, + 60, 60, 60, 60, 60, 60, 64, 64, 64, 63, 62, 62, 61, 61, 61, 60, 60, 60, + 60, 60, 59, 59, 63, 63, 63, 63, 62, 61, 61, 61, 60, 60, 60, 60, 60, 59, + 59, 59, 62, 62, 63, 62, 61, 61, 61, 60, 60, 60, 60, 60, 59, 59, 59, 59, + 61, 62, 62, 62, 61, 61, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 61, 61, + 61, 61, 61, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 60, 61, 61, 61, + 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 60, 60, 61, 60, 60, 60, + 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 60, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, + /* Size 32 */ + 64, 65, 65, 66, 66, 66, 66, 66, 66, 65, 65, 64, 64, 63, 63, 62, 62, 62, + 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 65, 65, 65, 66, + 66, 66, 66, 66, 66, 65, 65, 64, 64, 63, 63, 62, 62, 62, 62, 61, 61, 61, + 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 65, 65, 65, 66, 66, 66, 66, 66, + 66, 65, 65, 64, 64, 63, 63, 63, 62, 62, 62, 61, 61, 61, 61, 61, 60, 60, + 60, 60, 60, 60, 60, 60, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 64, + 64, 64, 63, 63, 62, 62, 62, 62, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, + 60, 60, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 64, 64, 63, 63, + 63, 62, 62, 62, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 66, 66, + 66, 65, 65, 65, 65, 65, 65, 65, 64, 64, 64, 63, 63, 63, 62, 62, 62, 62, + 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 66, 66, 66, 65, 65, 65, + 65, 65, 64, 64, 64, 63, 63, 63, 63, 62, 62, 62, 62, 61, 61, 61, 61, 61, + 60, 60, 60, 60, 60, 60, 60, 60, 66, 66, 66, 65, 65, 65, 65, 64, 64, 63, + 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, + 60, 60, 60, 60, 66, 66, 66, 65, 65, 65, 64, 64, 63, 63, 63, 62, 62, 62, + 62, 62, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, + 65, 65, 65, 65, 65, 65, 64, 63, 63, 63, 62, 62, 62, 62, 62, 61, 61, 61, + 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 65, 65, 65, 65, + 65, 64, 64, 63, 63, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 60, 60, + 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 64, 64, 64, 64, 65, 64, 63, 63, + 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, + 60, 60, 60, 60, 60, 60, 64, 64, 64, 64, 64, 64, 63, 63, 62, 62, 62, 61, + 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, + 59, 59, 63, 63, 63, 64, 64, 63, 63, 62, 62, 62, 61, 61, 61, 61, 61, 61, + 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 63, 63, + 63, 63, 63, 63, 63, 62, 62, 62, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, + 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 62, 62, 63, 63, 63, 63, + 62, 62, 62, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, + 60, 59, 59, 59, 59, 59, 59, 59, 62, 62, 62, 62, 63, 62, 62, 62, 61, 61, + 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, + 59, 59, 59, 59, 62, 62, 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, 60, 60, + 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 61, 62, 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, + 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 61, 61, 61, 62, + 62, 62, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 61, 61, 61, 61, 61, 61, 61, 61, + 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 61, 61, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, + 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 60, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, + 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, + 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 61, 60, + 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, + 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, + 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 60, 60, + 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59 }, + { /* Intra matrices */ + /* Size 4 */ + 69, 68, 65, 63, 68, 65, 64, 63, 65, 64, 62, 62, 63, 63, 62, 62, + /* Size 8 */ + 67, 70, 69, 67, 65, 64, 63, 62, 70, 69, 69, 68, 66, 64, 63, 63, 69, 69, + 66, 65, 64, 64, 63, 63, 67, 68, 65, 64, 63, 63, 63, 62, 65, 66, 64, 63, + 63, 62, 62, 62, 64, 64, 64, 63, 62, 62, 62, 62, 63, 63, 63, 63, 62, 62, + 62, 62, 62, 63, 63, 62, 62, 62, 62, 62, + /* Size 16 */ + 68, 69, 70, 70, 69, 68, 67, 66, 65, 65, 64, 64, 63, 63, 63, 63, 69, 69, + 69, 69, 69, 68, 67, 66, 66, 65, 64, 64, 63, 63, 63, 63, 70, 69, 69, 69, + 69, 68, 68, 67, 66, 65, 65, 64, 64, 63, 63, 63, 70, 69, 69, 68, 68, 67, + 67, 66, 65, 65, 64, 64, 63, 63, 63, 63, 69, 69, 69, 68, 66, 66, 65, 65, + 65, 64, 64, 64, 63, 63, 63, 63, 68, 68, 68, 67, 66, 65, 65, 65, 64, 64, + 64, 63, 63, 63, 63, 63, 67, 67, 68, 67, 65, 65, 64, 64, 64, 63, 63, 63, + 63, 63, 62, 62, 66, 66, 67, 66, 65, 65, 64, 64, 63, 63, 63, 63, 63, 62, + 62, 62, 65, 66, 66, 65, 65, 64, 64, 63, 63, 63, 63, 63, 62, 62, 62, 62, + 65, 65, 65, 65, 64, 64, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 64, 64, + 65, 64, 64, 64, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 64, 64, 64, 64, + 64, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 63, 63, 64, 63, 63, 63, + 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, + 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 62, + /* Size 32 */ + 68, 68, 69, 70, 70, 70, 70, 70, 69, 69, 68, 68, 67, 67, 66, 66, 65, 65, + 65, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 68, 69, 69, 69, + 70, 70, 70, 69, 69, 69, 68, 68, 67, 67, 66, 66, 65, 65, 65, 65, 64, 64, + 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 69, 69, 69, 69, 70, 69, 69, 69, + 69, 69, 68, 68, 67, 67, 67, 66, 66, 65, 65, 65, 64, 64, 64, 64, 63, 63, + 63, 63, 63, 63, 63, 63, 70, 69, 69, 69, 69, 69, 69, 69, 69, 69, 68, 68, + 68, 67, 67, 66, 66, 66, 65, 65, 65, 64, 64, 64, 64, 63, 63, 63, 63, 63, + 63, 63, 70, 70, 70, 69, 69, 69, 69, 69, 69, 69, 69, 68, 68, 67, 67, 67, + 66, 66, 65, 65, 65, 65, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 70, 70, + 69, 69, 69, 69, 69, 69, 69, 68, 68, 68, 67, 67, 67, 66, 66, 65, 65, 65, + 65, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 70, 70, 69, 69, 69, 69, + 68, 68, 68, 68, 67, 67, 67, 66, 66, 66, 65, 65, 65, 65, 64, 64, 64, 64, + 64, 63, 63, 63, 63, 63, 63, 63, 70, 69, 69, 69, 69, 69, 68, 68, 67, 67, + 67, 66, 66, 66, 66, 65, 65, 65, 65, 64, 64, 64, 64, 64, 63, 63, 63, 63, + 63, 63, 63, 63, 69, 69, 69, 69, 69, 69, 68, 67, 67, 66, 66, 66, 66, 65, + 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, + 69, 69, 69, 69, 69, 68, 68, 67, 66, 66, 66, 66, 65, 65, 65, 65, 65, 64, + 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 68, 68, 68, 68, + 69, 68, 67, 67, 66, 66, 66, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 64, + 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 68, 68, 68, 68, 68, 68, 67, 66, + 66, 66, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 67, 67, 67, 68, 68, 67, 67, 66, 66, 65, 65, 65, + 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 67, 67, 67, 67, 67, 67, 66, 66, 65, 65, 65, 65, 64, 64, 64, 64, + 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 66, 66, + 67, 67, 67, 67, 66, 66, 65, 65, 65, 64, 64, 64, 64, 64, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 66, 66, 66, 66, 67, 66, + 66, 65, 65, 65, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 62, 62, 62, 62, 62, 62, 65, 65, 66, 66, 66, 66, 65, 65, 65, 65, + 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, + 62, 62, 62, 62, 65, 65, 65, 66, 66, 65, 65, 65, 65, 64, 64, 64, 64, 63, + 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, + 65, 65, 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 64, 65, 65, 65, + 65, 65, 65, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 64, 64, 64, 65, 65, 65, 64, 64, + 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 62, 62, 62, 64, 64, 64, 64, 65, 64, 64, 64, 64, 64, 64, 63, + 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 63, 64, + 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 64, 64, 64, + 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, 64, 64, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, + 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, + 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 61, 60, 59, 61, 60, 60, 59, 60, 60, 59, 58, 59, 59, 58, 58, + /* Size 8 */ + 64, 66, 61, 61, 61, 60, 60, 59, 66, 63, 61, 62, 62, 61, 61, 60, 61, 61, + 61, 61, 61, 61, 60, 60, 61, 62, 61, 60, 60, 60, 60, 59, 61, 62, 61, 60, + 60, 59, 59, 59, 60, 61, 61, 60, 59, 59, 59, 59, 60, 61, 60, 60, 59, 59, + 59, 59, 59, 60, 60, 59, 59, 59, 59, 58, + /* Size 16 */ + 64, 65, 66, 64, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 59, 59, 65, 65, + 64, 63, 61, 61, 62, 61, 61, 61, 61, 61, 60, 60, 60, 60, 66, 64, 63, 62, + 61, 62, 62, 62, 62, 62, 61, 61, 61, 60, 60, 60, 64, 63, 62, 62, 61, 61, + 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 61, 61, 61, 61, 61, 61, 61, 61, + 61, 61, 61, 60, 60, 60, 60, 60, 61, 61, 62, 61, 61, 61, 60, 60, 60, 60, + 60, 60, 60, 60, 60, 60, 61, 62, 62, 61, 61, 60, 60, 60, 60, 60, 60, 60, + 60, 60, 59, 59, 61, 61, 62, 61, 61, 60, 60, 60, 60, 60, 60, 60, 59, 59, + 59, 59, 61, 61, 62, 61, 61, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, + 61, 61, 62, 61, 61, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 60, 61, + 61, 61, 61, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 60, 61, 61, 61, + 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 61, 60, 60, 60, + 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 60, 59, + 59, 59, 59, 59, 59, 59, 58, 58, 59, 60, 60, 60, 60, 60, 59, 59, 59, 59, + 59, 59, 59, 58, 58, 58, 59, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, + 59, 58, 58, 58, + /* Size 32 */ + 64, 64, 65, 65, 66, 65, 64, 63, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, + 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 64, 65, 65, 65, + 65, 64, 63, 62, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 60, + 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 65, 65, 65, 65, 64, 64, 63, 62, + 61, 61, 61, 62, 62, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, + 60, 60, 60, 60, 60, 60, 65, 65, 65, 64, 64, 63, 63, 62, 61, 61, 62, 62, + 62, 62, 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, + 60, 60, 66, 65, 64, 64, 63, 63, 62, 62, 61, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 65, 64, + 64, 63, 63, 62, 62, 62, 61, 61, 61, 62, 62, 62, 62, 62, 62, 61, 61, 61, + 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 64, 63, 63, 63, 62, 62, + 62, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, + 60, 60, 60, 60, 60, 60, 60, 60, 63, 62, 62, 62, 62, 62, 61, 61, 61, 61, + 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, + 60, 60, 60, 60, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, + 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, + 61, 61, 61, 61, 62, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, + 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 61, 61, 61, 62, + 62, 61, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, + 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 61, 61, 62, 62, 62, 62, 61, 61, + 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, + 60, 60, 59, 59, 59, 59, 61, 61, 62, 62, 62, 62, 61, 61, 61, 61, 60, 60, + 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, + 59, 59, 61, 61, 61, 62, 62, 62, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, + 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 61, 61, + 61, 62, 62, 62, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, + 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 61, 61, 61, 62, 62, 62, + 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 61, 61, 61, 62, 62, 62, 61, 61, 61, 61, + 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 61, 61, 61, 61, 62, 61, 61, 61, 61, 61, 60, 60, 60, 60, + 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 61, 61, 61, 61, 62, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 61, 61, 61, + 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 61, 61, 61, 61, 61, 61, 61, + 61, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 60, 60, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, + 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 60, 60, 61, 61, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, + 60, 61, 61, 61, 61, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 61, 61, + 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 61, 60, 60, 60, 60, 60, + 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 58, 58, 58, 58, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, + 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 58, 59, 60, 60, 60, + 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 58, 58, 58, 58, 58, 58, 58, 59, 60, 60, 60, 60, 60, 60, 60, + 60, 60, 60, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 58, + 58, 58, 58, 58, 58, 58, 59, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, + 58, 58, 59, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, 59, 59, 59, 58, 58, 58, 58, 58, 58, 58 }, + { /* Intra matrices */ + /* Size 4 */ + 69, 65, 65, 64, 65, 64, 64, 63, 65, 64, 63, 62, 64, 63, 62, 62, + /* Size 8 */ + 68, 70, 65, 65, 65, 64, 63, 63, 70, 67, 65, 66, 66, 65, 64, 64, 65, 65, + 64, 64, 65, 64, 64, 63, 65, 66, 64, 64, 64, 63, 63, 63, 65, 66, 65, 64, + 63, 63, 63, 63, 64, 65, 64, 63, 63, 63, 62, 62, 63, 64, 64, 63, 63, 62, + 62, 62, 63, 64, 63, 63, 63, 62, 62, 62, + /* Size 16 */ + 68, 69, 70, 68, 65, 65, 65, 65, 65, 64, 64, 64, 64, 63, 63, 63, 69, 69, + 69, 67, 65, 65, 66, 65, 65, 65, 65, 64, 64, 64, 64, 64, 70, 69, 67, 66, + 65, 66, 66, 66, 66, 65, 65, 65, 65, 64, 64, 64, 68, 67, 66, 66, 65, 65, + 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, 65, 65, 65, 65, 64, 65, 65, 65, + 65, 65, 64, 64, 64, 64, 64, 64, 65, 65, 66, 65, 65, 64, 64, 64, 64, 64, + 64, 64, 64, 63, 63, 63, 65, 66, 66, 65, 65, 64, 64, 64, 64, 64, 64, 63, + 63, 63, 63, 63, 65, 65, 66, 65, 65, 64, 64, 64, 64, 63, 63, 63, 63, 63, + 63, 63, 65, 65, 66, 65, 65, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, + 64, 65, 65, 65, 65, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 65, + 65, 65, 64, 64, 64, 63, 63, 63, 63, 63, 63, 62, 62, 62, 64, 64, 65, 65, + 64, 64, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 64, 64, 65, 64, 64, 64, + 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 63, 64, 64, 64, 64, 63, 63, 63, + 63, 63, 62, 62, 62, 62, 62, 62, 63, 64, 64, 64, 64, 63, 63, 63, 63, 63, + 62, 62, 62, 62, 62, 62, 63, 64, 64, 64, 64, 63, 63, 63, 63, 63, 62, 62, + 62, 62, 62, 62, + /* Size 32 */ + 68, 69, 69, 70, 71, 69, 68, 67, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 69, 69, 69, 69, + 70, 69, 68, 67, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 64, 64, + 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 69, 69, 69, 69, 69, 68, 67, 66, + 65, 66, 66, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 70, 69, 69, 68, 68, 67, 67, 66, 65, 66, 66, 66, + 66, 66, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, + 64, 64, 71, 70, 69, 68, 67, 67, 66, 66, 65, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 64, 69, 69, + 68, 67, 67, 66, 66, 66, 65, 65, 65, 66, 66, 66, 66, 66, 66, 65, 65, 65, + 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 64, 64, 68, 68, 67, 67, 66, 66, + 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 67, 67, 66, 66, 66, 66, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 65, 65, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 65, 65, 66, 66, + 66, 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 65, 65, 66, 66, 66, 66, 65, 65, + 65, 65, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 63, 63, 63, 63, 63, 63, 65, 65, 66, 66, 66, 66, 65, 65, 65, 65, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, + 63, 63, 65, 65, 66, 66, 66, 66, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 65, 65, + 65, 66, 66, 66, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 64, 64, 64, 63, + 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 65, 65, 65, 66, 66, 66, + 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 63, 63, 65, 65, 65, 66, 66, 66, 65, 65, 65, 65, + 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63, 65, 65, 65, 65, 66, 65, 65, 65, 65, 64, 64, 64, 64, 64, + 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, + 65, 65, 65, 65, 66, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 65, 65, 65, + 65, 65, 65, 65, 65, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 64, 64, 65, 65, 65, 65, 65, 65, + 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, + 63, 62, 62, 62, 62, 62, 64, 64, 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, + 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, + 62, 62, 64, 64, 64, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 64, 64, + 64, 65, 65, 65, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 64, 64, 64, 64, 65, 64, + 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 62, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, + 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 64, 64, + 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 64, 64, 64, 64, 64, 64, + 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 62, 62, 62, 62, 63, 63, 64, 64, 64, 64, 64, 64, 64, 64, 63, 63, + 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, + 62, 62, 63, 63, 64, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62 } } }, + { { /* Luma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 8 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 16 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, + /* Size 32 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 }, + { /* Intra matrices */ + /* Size 4 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 8 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 16 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, + /* Size 32 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 } }, + { /* Chroma matrices */ + { /* Inter matrices */ + /* Size 4 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 8 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 16 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, + /* Size 32 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 }, + { /* Intra matrices */ + /* Size 4 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 8 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + /* Size 16 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, + /* Size 32 */ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 } } } +}; + +#endif + +#if CONFIG_PVQ || CONFIG_DAALA_DIST +/* Quantization matrices for 8x8. For other block sizes, we currently just do + resampling. */ +/* Flat quantization, i.e. optimize for PSNR. */ +const int OD_QM8_Q4_FLAT[] = { 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16 }; +#if 0 +/* M1: MPEG2 matrix for inter (which has a dead zone). */ +const int OD_QM8_Q4[] = { + 16, 17, 18, 19, 20, 21, 22, 23, + 17, 18, 19, 20, 21, 22, 23, 24, + 18, 19, 20, 21, 22, 23, 24, 25, + 19, 20, 21, 22, 23, 24, 26, 27, + 20, 21, 22, 23, 25, 26, 27, 28, + 21, 22, 23, 24, 26, 27, 28, 30, + 22, 23, 24, 26, 27, 28, 30, 31, + 23, 24, 25, 27, 28, 30, 31, 33}; +#endif +#if 0 +/* M2: MPEG2 matrix for intra (no dead zone). */ +const int OD_QM8_Q4[] = { + 16, 16, 19, 22, 22, 26, 26, 27, + 16, 16, 22, 22, 26, 27, 27, 29, + 19, 22, 26, 26, 27, 29, 29, 35, + 22, 24, 27, 27, 29, 32, 34, 38, + 26, 27, 29, 29, 32, 35, 38, 46, + 27, 29, 34, 34, 35, 40, 46, 56, + 29, 34, 34, 37, 40, 48, 56, 69, + 34, 37, 38, 40, 48, 58, 69, 83 +}; +#endif +#if 0 +/* M3: Taken from dump_psnrhvs. */ +const int OD_QM8_Q4[] = { + 16, 16, 17, 20, 24, 29, 36, 42, + 16, 17, 17, 19, 22, 26, 31, 37, + 17, 17, 21, 23, 26, 30, 34, 40, + 20, 19, 23, 28, 31, 35, 39, 45, + 24, 22, 26, 31, 36, 41, 46, 51, + 29, 26, 30, 35, 41, 47, 52, 58, + 36, 31, 34, 39, 46, 52, 59, 66, + 42, 37, 40, 45, 51, 58, 66, 73 +}; +#endif +#if 1 +/* M4: a compromise equal to .5*(M3 + .5*(M2+transpose(M2))) */ +const int OD_QM8_Q4_HVS[] = { 16, 16, 18, 21, 24, 28, 32, 36, 16, 17, 20, + 21, 24, 27, 31, 35, 18, 20, 24, 25, 27, 31, + 33, 38, 21, 21, 25, 28, 30, 34, 37, 42, 24, + 24, 27, 30, 34, 38, 43, 49, 28, 27, 31, 34, + 38, 44, 50, 58, 32, 31, 33, 37, 43, 50, 58, + 68, 36, 35, 38, 42, 49, 58, 68, 78 }; +#endif +#endif diff --git a/third_party/aom/av1/common/quant_common.h b/third_party/aom/av1/common/quant_common.h new file mode 100644 index 0000000000..3f442427de --- /dev/null +++ b/third_party/aom/av1/common/quant_common.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_QUANT_COMMON_H_ +#define AV1_COMMON_QUANT_COMMON_H_ + +#include "aom/aom_codec.h" +#include "av1/common/seg_common.h" +#include "av1/common/enums.h" +#include "av1/common/entropy.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MINQ 0 +#define MAXQ 255 +#define QINDEX_RANGE (MAXQ - MINQ + 1) +#define QINDEX_BITS 8 +#if CONFIG_AOM_QM +// Total number of QM sets stored +#define QM_LEVEL_BITS 4 +#define NUM_QM_LEVELS (1 << QM_LEVEL_BITS) +/* Offset into the list of QMs. Actual number of levels used is + (NUM_QM_LEVELS-AOM_QM_OFFSET) + Lower value of AOM_QM_OFFSET implies more heavily weighted matrices.*/ +#define DEFAULT_QM_FIRST (NUM_QM_LEVELS / 2) +#define DEFAULT_QM_LAST (NUM_QM_LEVELS - 1) +#endif + +struct AV1Common; + +int16_t av1_dc_quant(int qindex, int delta, aom_bit_depth_t bit_depth); +int16_t av1_ac_quant(int qindex, int delta, aom_bit_depth_t bit_depth); +int16_t av1_qindex_from_ac(int ac, aom_bit_depth_t bit_depth); + +int av1_get_qindex(const struct segmentation *seg, int segment_id, + int base_qindex); +#if CONFIG_AOM_QM +// Reduce the large number of quantizers to a smaller number of levels for which +// different matrices may be defined +static INLINE int aom_get_qmlevel(int qindex, int first, int last) { + int qmlevel = (qindex * (last + 1 - first) + QINDEX_RANGE / 2) / QINDEX_RANGE; + qmlevel = AOMMIN(qmlevel + first, NUM_QM_LEVELS - 1); + return qmlevel; +} +void aom_qm_init(struct AV1Common *cm); +qm_val_t *aom_iqmatrix(struct AV1Common *cm, int qindex, int comp, + int log2sizem2, int is_intra); +qm_val_t *aom_qmatrix(struct AV1Common *cm, int qindex, int comp, + int log2sizem2, int is_intra); +#endif + +#if CONFIG_NEW_QUANT + +#define QUANT_PROFILES 4 +#define QUANT_RANGES 2 +#define NUQ_KNOTS 3 + +typedef tran_low_t dequant_val_type_nuq[NUQ_KNOTS + 1]; +typedef tran_low_t cuml_bins_type_nuq[NUQ_KNOTS]; +void av1_get_dequant_val_nuq(int q, int band, tran_low_t *dq, + tran_low_t *cuml_bins, int dq_off_index); +tran_low_t av1_dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq); +tran_low_t av1_dequant_coeff_nuq(int v, int q, const tran_low_t *dq); + +static INLINE int qindex_to_qrange(int qindex) { + return (qindex < 140 ? 1 : 0); +} + +static INLINE int get_dq_profile_from_ctx(int qindex, int q_ctx, int is_inter, + PLANE_TYPE plane_type) { + // intra/inter, Y/UV, ctx, qrange + static const int + def_dq_profile_lookup[REF_TYPES][PLANE_TYPES][COEFF_CONTEXTS0] + [QUANT_RANGES] = { + { + // intra + { { 2, 1 }, { 2, 1 }, { 2, 1 } }, // Y + { { 3, 1 }, { 3, 1 }, { 3, 1 } }, // UV + }, + { + // inter + { { 3, 1 }, { 2, 1 }, { 2, 1 } }, // Y + { { 3, 1 }, { 3, 1 }, { 3, 1 } }, // UV + }, + }; + if (!qindex) return 0; // lossless + return def_dq_profile_lookup[is_inter][plane_type][q_ctx] + [qindex_to_qrange(qindex)]; +} +#endif // CONFIG_NEW_QUANT + +#if CONFIG_PVQ || CONFIG_DAALA_DIST +extern const int OD_QM8_Q4_FLAT[]; +extern const int OD_QM8_Q4_HVS[]; +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_QUANT_COMMON_H_ diff --git a/third_party/aom/av1/common/reconinter.c b/third_party/aom/av1/common/reconinter.c new file mode 100644 index 0000000000..ed7065757b --- /dev/null +++ b/third_party/aom/av1/common/reconinter.c @@ -0,0 +1,3083 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_scale_rtcd.h" +#include "./aom_dsp_rtcd.h" +#include "./aom_config.h" + +#include "aom/aom_integer.h" +#include "aom_dsp/blend.h" + +#include "av1/common/blockd.h" +#include "av1/common/reconinter.h" +#include "av1/common/reconintra.h" +#if CONFIG_MOTION_VAR +#include "av1/common/onyxc_int.h" +#endif // CONFIG_MOTION_VAR + +#if CONFIG_EXT_INTER + +#define NSMOOTHERS 1 + +// [smoother][negative][direction] +DECLARE_ALIGNED(16, static uint8_t, + wedge_mask_obl[NSMOOTHERS][2][WEDGE_DIRECTIONS] + [MASK_MASTER_SIZE * MASK_MASTER_SIZE]); + +DECLARE_ALIGNED(16, static uint8_t, + wedge_signflip_lookup[BLOCK_SIZES][MAX_WEDGE_TYPES]); + +// 3 * MAX_WEDGE_SQUARE is an easy to compute and fairly tight upper bound +// on the sum of all mask sizes up to an including MAX_WEDGE_SQUARE. +DECLARE_ALIGNED(16, static uint8_t, + wedge_mask_buf[2 * MAX_WEDGE_TYPES * 3 * MAX_WEDGE_SQUARE]); + +static wedge_masks_type wedge_masks[BLOCK_SIZES][2]; + +// Some unused wedge codebooks left temporarily to facilitate experiments. +// To be removed when settled. +/* +static wedge_code_type wedge_codebook_8_hgtw[8] = { + { WEDGE_OBLIQUE27, 4, 4 }, { WEDGE_OBLIQUE63, 4, 4 }, + { WEDGE_OBLIQUE117, 4, 4 }, { WEDGE_OBLIQUE153, 4, 4 }, + { WEDGE_OBLIQUE27, 4, 2 }, { WEDGE_OBLIQUE27, 4, 6 }, + { WEDGE_OBLIQUE153, 4, 2 }, { WEDGE_OBLIQUE153, 4, 6 }, +}; + +static wedge_code_type wedge_codebook_8_hltw[8] = { + { WEDGE_OBLIQUE27, 4, 4 }, { WEDGE_OBLIQUE63, 4, 4 }, + { WEDGE_OBLIQUE117, 4, 4 }, { WEDGE_OBLIQUE153, 4, 4 }, + { WEDGE_OBLIQUE63, 2, 4 }, { WEDGE_OBLIQUE63, 6, 4 }, + { WEDGE_OBLIQUE117, 2, 4 }, { WEDGE_OBLIQUE117, 6, 4 }, +}; + +static wedge_code_type wedge_codebook_8_heqw[8] = { + { WEDGE_OBLIQUE27, 4, 4 }, { WEDGE_OBLIQUE63, 4, 4 }, + { WEDGE_OBLIQUE117, 4, 4 }, { WEDGE_OBLIQUE153, 4, 4 }, + { WEDGE_HORIZONTAL, 4, 2 }, { WEDGE_HORIZONTAL, 4, 6 }, + { WEDGE_VERTICAL, 2, 4 }, { WEDGE_VERTICAL, 6, 4 }, +}; + +static const wedge_code_type wedge_codebook_32_hgtw[32] = { + { WEDGE_OBLIQUE27, 4, 4 }, { WEDGE_OBLIQUE63, 4, 4 }, + { WEDGE_OBLIQUE117, 4, 4 }, { WEDGE_OBLIQUE153, 4, 4 }, + { WEDGE_HORIZONTAL, 4, 2 }, { WEDGE_HORIZONTAL, 4, 4 }, + { WEDGE_HORIZONTAL, 4, 6 }, { WEDGE_VERTICAL, 4, 4 }, + { WEDGE_OBLIQUE27, 4, 1 }, { WEDGE_OBLIQUE27, 4, 2 }, + { WEDGE_OBLIQUE27, 4, 3 }, { WEDGE_OBLIQUE27, 4, 5 }, + { WEDGE_OBLIQUE27, 4, 6 }, { WEDGE_OBLIQUE27, 4, 7 }, + { WEDGE_OBLIQUE153, 4, 1 }, { WEDGE_OBLIQUE153, 4, 2 }, + { WEDGE_OBLIQUE153, 4, 3 }, { WEDGE_OBLIQUE153, 4, 5 }, + { WEDGE_OBLIQUE153, 4, 6 }, { WEDGE_OBLIQUE153, 4, 7 }, + { WEDGE_OBLIQUE63, 1, 4 }, { WEDGE_OBLIQUE63, 2, 4 }, + { WEDGE_OBLIQUE63, 3, 4 }, { WEDGE_OBLIQUE63, 5, 4 }, + { WEDGE_OBLIQUE63, 6, 4 }, { WEDGE_OBLIQUE63, 7, 4 }, + { WEDGE_OBLIQUE117, 1, 4 }, { WEDGE_OBLIQUE117, 2, 4 }, + { WEDGE_OBLIQUE117, 3, 4 }, { WEDGE_OBLIQUE117, 5, 4 }, + { WEDGE_OBLIQUE117, 6, 4 }, { WEDGE_OBLIQUE117, 7, 4 }, +}; + +static const wedge_code_type wedge_codebook_32_hltw[32] = { + { WEDGE_OBLIQUE27, 4, 4 }, { WEDGE_OBLIQUE63, 4, 4 }, + { WEDGE_OBLIQUE117, 4, 4 }, { WEDGE_OBLIQUE153, 4, 4 }, + { WEDGE_VERTICAL, 2, 4 }, { WEDGE_VERTICAL, 4, 4 }, + { WEDGE_VERTICAL, 6, 4 }, { WEDGE_HORIZONTAL, 4, 4 }, + { WEDGE_OBLIQUE27, 4, 1 }, { WEDGE_OBLIQUE27, 4, 2 }, + { WEDGE_OBLIQUE27, 4, 3 }, { WEDGE_OBLIQUE27, 4, 5 }, + { WEDGE_OBLIQUE27, 4, 6 }, { WEDGE_OBLIQUE27, 4, 7 }, + { WEDGE_OBLIQUE153, 4, 1 }, { WEDGE_OBLIQUE153, 4, 2 }, + { WEDGE_OBLIQUE153, 4, 3 }, { WEDGE_OBLIQUE153, 4, 5 }, + { WEDGE_OBLIQUE153, 4, 6 }, { WEDGE_OBLIQUE153, 4, 7 }, + { WEDGE_OBLIQUE63, 1, 4 }, { WEDGE_OBLIQUE63, 2, 4 }, + { WEDGE_OBLIQUE63, 3, 4 }, { WEDGE_OBLIQUE63, 5, 4 }, + { WEDGE_OBLIQUE63, 6, 4 }, { WEDGE_OBLIQUE63, 7, 4 }, + { WEDGE_OBLIQUE117, 1, 4 }, { WEDGE_OBLIQUE117, 2, 4 }, + { WEDGE_OBLIQUE117, 3, 4 }, { WEDGE_OBLIQUE117, 5, 4 }, + { WEDGE_OBLIQUE117, 6, 4 }, { WEDGE_OBLIQUE117, 7, 4 }, +}; + +static const wedge_code_type wedge_codebook_32_heqw[32] = { + { WEDGE_OBLIQUE27, 4, 4 }, { WEDGE_OBLIQUE63, 4, 4 }, + { WEDGE_OBLIQUE117, 4, 4 }, { WEDGE_OBLIQUE153, 4, 4 }, + { WEDGE_HORIZONTAL, 4, 2 }, { WEDGE_HORIZONTAL, 4, 6 }, + { WEDGE_VERTICAL, 2, 4 }, { WEDGE_VERTICAL, 6, 4 }, + { WEDGE_OBLIQUE27, 4, 1 }, { WEDGE_OBLIQUE27, 4, 2 }, + { WEDGE_OBLIQUE27, 4, 3 }, { WEDGE_OBLIQUE27, 4, 5 }, + { WEDGE_OBLIQUE27, 4, 6 }, { WEDGE_OBLIQUE27, 4, 7 }, + { WEDGE_OBLIQUE153, 4, 1 }, { WEDGE_OBLIQUE153, 4, 2 }, + { WEDGE_OBLIQUE153, 4, 3 }, { WEDGE_OBLIQUE153, 4, 5 }, + { WEDGE_OBLIQUE153, 4, 6 }, { WEDGE_OBLIQUE153, 4, 7 }, + { WEDGE_OBLIQUE63, 1, 4 }, { WEDGE_OBLIQUE63, 2, 4 }, + { WEDGE_OBLIQUE63, 3, 4 }, { WEDGE_OBLIQUE63, 5, 4 }, + { WEDGE_OBLIQUE63, 6, 4 }, { WEDGE_OBLIQUE63, 7, 4 }, + { WEDGE_OBLIQUE117, 1, 4 }, { WEDGE_OBLIQUE117, 2, 4 }, + { WEDGE_OBLIQUE117, 3, 4 }, { WEDGE_OBLIQUE117, 5, 4 }, + { WEDGE_OBLIQUE117, 6, 4 }, { WEDGE_OBLIQUE117, 7, 4 }, +}; +*/ + +static const wedge_code_type wedge_codebook_16_hgtw[16] = { + { WEDGE_OBLIQUE27, 4, 4 }, { WEDGE_OBLIQUE63, 4, 4 }, + { WEDGE_OBLIQUE117, 4, 4 }, { WEDGE_OBLIQUE153, 4, 4 }, + { WEDGE_HORIZONTAL, 4, 2 }, { WEDGE_HORIZONTAL, 4, 4 }, + { WEDGE_HORIZONTAL, 4, 6 }, { WEDGE_VERTICAL, 4, 4 }, + { WEDGE_OBLIQUE27, 4, 2 }, { WEDGE_OBLIQUE27, 4, 6 }, + { WEDGE_OBLIQUE153, 4, 2 }, { WEDGE_OBLIQUE153, 4, 6 }, + { WEDGE_OBLIQUE63, 2, 4 }, { WEDGE_OBLIQUE63, 6, 4 }, + { WEDGE_OBLIQUE117, 2, 4 }, { WEDGE_OBLIQUE117, 6, 4 }, +}; + +static const wedge_code_type wedge_codebook_16_hltw[16] = { + { WEDGE_OBLIQUE27, 4, 4 }, { WEDGE_OBLIQUE63, 4, 4 }, + { WEDGE_OBLIQUE117, 4, 4 }, { WEDGE_OBLIQUE153, 4, 4 }, + { WEDGE_VERTICAL, 2, 4 }, { WEDGE_VERTICAL, 4, 4 }, + { WEDGE_VERTICAL, 6, 4 }, { WEDGE_HORIZONTAL, 4, 4 }, + { WEDGE_OBLIQUE27, 4, 2 }, { WEDGE_OBLIQUE27, 4, 6 }, + { WEDGE_OBLIQUE153, 4, 2 }, { WEDGE_OBLIQUE153, 4, 6 }, + { WEDGE_OBLIQUE63, 2, 4 }, { WEDGE_OBLIQUE63, 6, 4 }, + { WEDGE_OBLIQUE117, 2, 4 }, { WEDGE_OBLIQUE117, 6, 4 }, +}; + +static const wedge_code_type wedge_codebook_16_heqw[16] = { + { WEDGE_OBLIQUE27, 4, 4 }, { WEDGE_OBLIQUE63, 4, 4 }, + { WEDGE_OBLIQUE117, 4, 4 }, { WEDGE_OBLIQUE153, 4, 4 }, + { WEDGE_HORIZONTAL, 4, 2 }, { WEDGE_HORIZONTAL, 4, 6 }, + { WEDGE_VERTICAL, 2, 4 }, { WEDGE_VERTICAL, 6, 4 }, + { WEDGE_OBLIQUE27, 4, 2 }, { WEDGE_OBLIQUE27, 4, 6 }, + { WEDGE_OBLIQUE153, 4, 2 }, { WEDGE_OBLIQUE153, 4, 6 }, + { WEDGE_OBLIQUE63, 2, 4 }, { WEDGE_OBLIQUE63, 6, 4 }, + { WEDGE_OBLIQUE117, 2, 4 }, { WEDGE_OBLIQUE117, 6, 4 }, +}; + +const wedge_params_type wedge_params_lookup[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + { 0, NULL, NULL, 0, NULL }, + { 0, NULL, NULL, 0, NULL }, + { 0, NULL, NULL, 0, NULL }, +#endif // CONFIG_CB4X4 + { 0, NULL, NULL, 0, NULL }, + { 0, NULL, NULL, 0, NULL }, + { 0, NULL, NULL, 0, NULL }, +#if CONFIG_WEDGE + { 4, wedge_codebook_16_heqw, wedge_signflip_lookup[BLOCK_8X8], 0, + wedge_masks[BLOCK_8X8] }, + { 4, wedge_codebook_16_hgtw, wedge_signflip_lookup[BLOCK_8X16], 0, + wedge_masks[BLOCK_8X16] }, + { 4, wedge_codebook_16_hltw, wedge_signflip_lookup[BLOCK_16X8], 0, + wedge_masks[BLOCK_16X8] }, + { 4, wedge_codebook_16_heqw, wedge_signflip_lookup[BLOCK_16X16], 0, + wedge_masks[BLOCK_16X16] }, + { 4, wedge_codebook_16_hgtw, wedge_signflip_lookup[BLOCK_16X32], 0, + wedge_masks[BLOCK_16X32] }, + { 4, wedge_codebook_16_hltw, wedge_signflip_lookup[BLOCK_32X16], 0, + wedge_masks[BLOCK_32X16] }, + { 4, wedge_codebook_16_heqw, wedge_signflip_lookup[BLOCK_32X32], 0, + wedge_masks[BLOCK_32X32] }, + { 0, wedge_codebook_16_hgtw, wedge_signflip_lookup[BLOCK_32X64], 0, + wedge_masks[BLOCK_32X64] }, + { 0, wedge_codebook_16_hltw, wedge_signflip_lookup[BLOCK_64X32], 0, + wedge_masks[BLOCK_64X32] }, + { 0, wedge_codebook_16_heqw, wedge_signflip_lookup[BLOCK_64X64], 0, + wedge_masks[BLOCK_64X64] }, +#else + { 0, wedge_codebook_16_heqw, wedge_signflip_lookup[BLOCK_8X8], 0, + wedge_masks[BLOCK_8X8] }, + { 0, wedge_codebook_16_hgtw, wedge_signflip_lookup[BLOCK_8X16], 0, + wedge_masks[BLOCK_8X16] }, + { 0, wedge_codebook_16_hltw, wedge_signflip_lookup[BLOCK_16X8], 0, + wedge_masks[BLOCK_16X8] }, + { 0, wedge_codebook_16_heqw, wedge_signflip_lookup[BLOCK_16X16], 0, + wedge_masks[BLOCK_16X16] }, + { 0, wedge_codebook_16_hgtw, wedge_signflip_lookup[BLOCK_16X32], 0, + wedge_masks[BLOCK_16X32] }, + { 0, wedge_codebook_16_hltw, wedge_signflip_lookup[BLOCK_32X16], 0, + wedge_masks[BLOCK_32X16] }, + { 0, wedge_codebook_16_heqw, wedge_signflip_lookup[BLOCK_32X32], 0, + wedge_masks[BLOCK_32X32] }, + { 0, wedge_codebook_16_hgtw, wedge_signflip_lookup[BLOCK_32X64], 0, + wedge_masks[BLOCK_32X64] }, + { 0, wedge_codebook_16_hltw, wedge_signflip_lookup[BLOCK_64X32], 0, + wedge_masks[BLOCK_64X32] }, + { 0, wedge_codebook_16_heqw, wedge_signflip_lookup[BLOCK_64X64], 0, + wedge_masks[BLOCK_64X64] }, +#endif // CONFIG_WEDGE +#if CONFIG_EXT_PARTITION + { 0, NULL, NULL, 0, NULL }, + { 0, NULL, NULL, 0, NULL }, + { 0, NULL, NULL, 0, NULL }, +#endif // CONFIG_EXT_PARTITION +}; + +static const uint8_t *get_wedge_mask_inplace(int wedge_index, int neg, + BLOCK_SIZE sb_type) { + const uint8_t *master; + const int bh = block_size_high[sb_type]; + const int bw = block_size_wide[sb_type]; + const wedge_code_type *a = + wedge_params_lookup[sb_type].codebook + wedge_index; + const int smoother = wedge_params_lookup[sb_type].smoother; + int woff, hoff; + const uint8_t wsignflip = wedge_params_lookup[sb_type].signflip[wedge_index]; + + assert(wedge_index >= 0 && + wedge_index < (1 << get_wedge_bits_lookup(sb_type))); + woff = (a->x_offset * bw) >> 3; + hoff = (a->y_offset * bh) >> 3; + master = wedge_mask_obl[smoother][neg ^ wsignflip][a->direction] + + MASK_MASTER_STRIDE * (MASK_MASTER_SIZE / 2 - hoff) + + MASK_MASTER_SIZE / 2 - woff; + return master; +} + +const uint8_t *av1_get_soft_mask(int wedge_index, int wedge_sign, + BLOCK_SIZE sb_type, int offset_x, + int offset_y) { + const uint8_t *mask = + get_wedge_mask_inplace(wedge_index, wedge_sign, sb_type); + if (mask) mask -= (offset_x + offset_y * MASK_MASTER_STRIDE); + return mask; +} + +#if CONFIG_COMPOUND_SEGMENT +static uint8_t *invert_mask(uint8_t *mask_inv_buffer, const uint8_t *const mask, + int h, int w, int stride) { + int i, j; + + for (i = 0; i < h; ++i) + for (j = 0; j < w; ++j) { + mask_inv_buffer[i * stride + j] = + AOM_BLEND_A64_MAX_ALPHA - mask[i * stride + j]; + } + return mask_inv_buffer; +} +#endif // CONFIG_COMPOUND_SEGMENT + +const uint8_t *av1_get_compound_type_mask_inverse( + const INTERINTER_COMPOUND_DATA *const comp_data, +#if CONFIG_COMPOUND_SEGMENT + uint8_t *mask_buffer, int h, int w, int stride, +#endif + BLOCK_SIZE sb_type) { + assert(is_masked_compound_type(comp_data->interinter_compound_type)); + (void)sb_type; + switch (comp_data->interinter_compound_type) { +#if CONFIG_WEDGE + case COMPOUND_WEDGE: + return av1_get_contiguous_soft_mask(comp_data->wedge_index, + !comp_data->wedge_sign, sb_type); +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + case COMPOUND_SEG: + return invert_mask(mask_buffer, comp_data->seg_mask, h, w, stride); +#endif // CONFIG_COMPOUND_SEGMENT + default: assert(0); return NULL; + } +} + +const uint8_t *av1_get_compound_type_mask( + const INTERINTER_COMPOUND_DATA *const comp_data, BLOCK_SIZE sb_type) { + assert(is_masked_compound_type(comp_data->interinter_compound_type)); + (void)sb_type; + switch (comp_data->interinter_compound_type) { +#if CONFIG_WEDGE + case COMPOUND_WEDGE: + return av1_get_contiguous_soft_mask(comp_data->wedge_index, + comp_data->wedge_sign, sb_type); +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + case COMPOUND_SEG: return comp_data->seg_mask; +#endif // CONFIG_COMPOUND_SEGMENT + default: assert(0); return NULL; + } +} + +#if CONFIG_COMPOUND_SEGMENT +#if COMPOUND_SEGMENT_TYPE == 0 +static void uniform_mask(uint8_t *mask, int which_inverse, BLOCK_SIZE sb_type, + int h, int w, int mask_val) { + int i, j; + int block_stride = block_size_wide[sb_type]; + for (i = 0; i < h; ++i) + for (j = 0; j < w; ++j) { + mask[i * block_stride + j] = + which_inverse ? AOM_BLEND_A64_MAX_ALPHA - mask_val : mask_val; + } +} + +void build_compound_seg_mask(uint8_t *mask, SEG_MASK_TYPE mask_type, + const uint8_t *src0, int src0_stride, + const uint8_t *src1, int src1_stride, + BLOCK_SIZE sb_type, int h, int w) { + (void)src0; + (void)src1; + (void)src0_stride; + (void)src1_stride; + switch (mask_type) { + case UNIFORM_45: uniform_mask(mask, 0, sb_type, h, w, 45); break; + case UNIFORM_45_INV: uniform_mask(mask, 1, sb_type, h, w, 45); break; + default: assert(0); + } +} + +#if CONFIG_HIGHBITDEPTH +void build_compound_seg_mask_highbd(uint8_t *mask, SEG_MASK_TYPE mask_type, + const uint8_t *src0, int src0_stride, + const uint8_t *src1, int src1_stride, + BLOCK_SIZE sb_type, int h, int w, int bd) { + (void)src0; + (void)src1; + (void)src0_stride; + (void)src1_stride; + (void)bd; + switch (mask_type) { + case UNIFORM_45: uniform_mask(mask, 0, sb_type, h, w, 45); break; + case UNIFORM_45_INV: uniform_mask(mask, 1, sb_type, h, w, 45); break; + default: assert(0); + } +} +#endif // CONFIG_HIGHBITDEPTH + +#elif COMPOUND_SEGMENT_TYPE == 1 +#define DIFF_FACTOR 16 +static void diffwtd_mask(uint8_t *mask, int which_inverse, int mask_base, + const uint8_t *src0, int src0_stride, + const uint8_t *src1, int src1_stride, + BLOCK_SIZE sb_type, int h, int w) { + int i, j, m, diff; + int block_stride = block_size_wide[sb_type]; + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + diff = + abs((int)src0[i * src0_stride + j] - (int)src1[i * src1_stride + j]); + m = clamp(mask_base + (diff / DIFF_FACTOR), 0, AOM_BLEND_A64_MAX_ALPHA); + mask[i * block_stride + j] = + which_inverse ? AOM_BLEND_A64_MAX_ALPHA - m : m; + } + } +} + +void build_compound_seg_mask(uint8_t *mask, SEG_MASK_TYPE mask_type, + const uint8_t *src0, int src0_stride, + const uint8_t *src1, int src1_stride, + BLOCK_SIZE sb_type, int h, int w) { + switch (mask_type) { + case DIFFWTD_42: + diffwtd_mask(mask, 0, 42, src0, src0_stride, src1, src1_stride, sb_type, + h, w); + break; + case DIFFWTD_42_INV: + diffwtd_mask(mask, 1, 42, src0, src0_stride, src1, src1_stride, sb_type, + h, w); + break; + default: assert(0); + } +} + +#if CONFIG_HIGHBITDEPTH +static void diffwtd_mask_highbd(uint8_t *mask, int which_inverse, int mask_base, + const uint16_t *src0, int src0_stride, + const uint16_t *src1, int src1_stride, + BLOCK_SIZE sb_type, int h, int w, int bd) { + int i, j, m, diff; + int block_stride = block_size_wide[sb_type]; + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + diff = abs((int)src0[i * src0_stride + j] - + (int)src1[i * src1_stride + j]) >> + (bd - 8); + m = clamp(mask_base + (diff / DIFF_FACTOR), 0, AOM_BLEND_A64_MAX_ALPHA); + mask[i * block_stride + j] = + which_inverse ? AOM_BLEND_A64_MAX_ALPHA - m : m; + } + } +} + +void build_compound_seg_mask_highbd(uint8_t *mask, SEG_MASK_TYPE mask_type, + const uint8_t *src0, int src0_stride, + const uint8_t *src1, int src1_stride, + BLOCK_SIZE sb_type, int h, int w, int bd) { + switch (mask_type) { + case DIFFWTD_42: + diffwtd_mask_highbd(mask, 0, 42, CONVERT_TO_SHORTPTR(src0), src0_stride, + CONVERT_TO_SHORTPTR(src1), src1_stride, sb_type, h, w, + bd); + break; + case DIFFWTD_42_INV: + diffwtd_mask_highbd(mask, 1, 42, CONVERT_TO_SHORTPTR(src0), src0_stride, + CONVERT_TO_SHORTPTR(src1), src1_stride, sb_type, h, w, + bd); + break; + default: assert(0); + } +} +#endif // CONFIG_HIGHBITDEPTH +#endif // COMPOUND_SEGMENT_TYPE +#endif // CONFIG_COMPOUND_SEGMENT + +#if MASK_MASTER_SIZE == 64 +static const uint8_t wedge_master_oblique_odd[NSMOOTHERS][MASK_MASTER_SIZE] = { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 6, 18, + 37, 53, 60, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + } +}; +static const uint8_t wedge_master_oblique_even[NSMOOTHERS][MASK_MASTER_SIZE] = { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 11, 27, + 46, 58, 62, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + } +}; +static const uint8_t wedge_master_vertical[NSMOOTHERS][MASK_MASTER_SIZE] = { { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 21, + 43, 57, 62, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, +} }; + +static void shift_copy(const uint8_t *src, uint8_t *dst, int shift, int width) { + if (shift >= 0) { + memcpy(dst + shift, src, width - shift); + memset(dst, src[0], shift); + } else { + shift = -shift; + memcpy(dst, src + shift, width - shift); + memset(dst + width - shift, src[width - 1], shift); + } +} +#else +static const double smoother_param[NSMOOTHERS] = { 2.83 }; +#endif // MASK_MASTER_SIZE == 64 + +static void init_wedge_master_masks() { + int i, j, s; + const int w = MASK_MASTER_SIZE; + const int h = MASK_MASTER_SIZE; + const int stride = MASK_MASTER_STRIDE; + for (s = 0; s < NSMOOTHERS; s++) { +#if MASK_MASTER_SIZE == 64 + // Generate prototype by shifting the masters + int shift = h / 4; + for (i = 0; i < h; i += 2) { + shift_copy(wedge_master_oblique_even[s], + &wedge_mask_obl[s][1][WEDGE_OBLIQUE63][i * stride], shift, + MASK_MASTER_SIZE); + shift--; + shift_copy(wedge_master_oblique_odd[s], + &wedge_mask_obl[s][1][WEDGE_OBLIQUE63][(i + 1) * stride], + shift, MASK_MASTER_SIZE); + memcpy(&wedge_mask_obl[s][1][WEDGE_VERTICAL][i * stride], + wedge_master_vertical[s], + MASK_MASTER_SIZE * sizeof(wedge_master_vertical[s][0])); + memcpy(&wedge_mask_obl[s][1][WEDGE_VERTICAL][(i + 1) * stride], + wedge_master_vertical[s], + MASK_MASTER_SIZE * sizeof(wedge_master_vertical[s][0])); + } +#else + const int a[2] = { 2, 1 }; + const double asqrt = sqrt(a[0] * a[0] + a[1] * a[1]); + for (i = 0; i < h; i++) { + for (j = 0; j < w; ++j) { + int x = (2 * j + 1 - w); + int y = (2 * i + 1 - h); + double d = (a[0] * x + a[1] * y) / asqrt; + const int msk = (int)rint((1.0 + tanh(d / smoother_param[s])) * 32); + wedge_mask_obl[s][1][WEDGE_OBLIQUE63][i * stride + j] = msk; + const int mskx = (int)rint((1.0 + tanh(x / smoother_param[s])) * 32); + wedge_mask_obl[s][1][WEDGE_VERTICAL][i * stride + j] = mskx; + } + } +#endif // MASK_MASTER_SIZE == 64 + for (i = 0; i < h; ++i) { + for (j = 0; j < w; ++j) { + const int msk = wedge_mask_obl[s][1][WEDGE_OBLIQUE63][i * stride + j]; + wedge_mask_obl[s][1][WEDGE_OBLIQUE27][j * stride + i] = msk; + wedge_mask_obl[s][1][WEDGE_OBLIQUE117][i * stride + w - 1 - j] = + wedge_mask_obl[s][1][WEDGE_OBLIQUE153][(w - 1 - j) * stride + i] = + (1 << WEDGE_WEIGHT_BITS) - msk; + wedge_mask_obl[s][0][WEDGE_OBLIQUE63][i * stride + j] = + wedge_mask_obl[s][0][WEDGE_OBLIQUE27][j * stride + i] = + (1 << WEDGE_WEIGHT_BITS) - msk; + wedge_mask_obl[s][0][WEDGE_OBLIQUE117][i * stride + w - 1 - j] = + wedge_mask_obl[s][0][WEDGE_OBLIQUE153][(w - 1 - j) * stride + i] = + msk; + const int mskx = wedge_mask_obl[s][1][WEDGE_VERTICAL][i * stride + j]; + wedge_mask_obl[s][1][WEDGE_HORIZONTAL][j * stride + i] = mskx; + wedge_mask_obl[s][0][WEDGE_VERTICAL][i * stride + j] = + wedge_mask_obl[s][0][WEDGE_HORIZONTAL][j * stride + i] = + (1 << WEDGE_WEIGHT_BITS) - mskx; + } + } + } +} + +// If the signs for the wedges for various blocksizes are +// inconsistent flip the sign flag. Do it only once for every +// wedge codebook. +static void init_wedge_signs() { + BLOCK_SIZE sb_type; + memset(wedge_signflip_lookup, 0, sizeof(wedge_signflip_lookup)); + for (sb_type = BLOCK_4X4; sb_type < BLOCK_SIZES; ++sb_type) { + const int bw = block_size_wide[sb_type]; + const int bh = block_size_high[sb_type]; + const wedge_params_type wedge_params = wedge_params_lookup[sb_type]; + const int wbits = wedge_params.bits; + const int wtypes = 1 << wbits; + int i, w; + if (wbits == 0) continue; + for (w = 0; w < wtypes; ++w) { + const uint8_t *mask = get_wedge_mask_inplace(w, 0, sb_type); + int sum = 0; + for (i = 0; i < bw; ++i) sum += mask[i]; + for (i = 0; i < bh; ++i) sum += mask[i * MASK_MASTER_STRIDE]; + sum = (sum + (bw + bh) / 2) / (bw + bh); + wedge_params.signflip[w] = (sum < 32); + } + } +} + +static void init_wedge_masks() { + uint8_t *dst = wedge_mask_buf; + BLOCK_SIZE bsize; + memset(wedge_masks, 0, sizeof(wedge_masks)); + for (bsize = BLOCK_4X4; bsize < BLOCK_SIZES; ++bsize) { + const uint8_t *mask; + const int bw = block_size_wide[bsize]; + const int bh = block_size_high[bsize]; + const wedge_params_type *wedge_params = &wedge_params_lookup[bsize]; + const int wbits = wedge_params->bits; + const int wtypes = 1 << wbits; + int w; + if (wbits == 0) continue; + for (w = 0; w < wtypes; ++w) { + mask = get_wedge_mask_inplace(w, 0, bsize); + aom_convolve_copy(mask, MASK_MASTER_STRIDE, dst, bw, NULL, 0, NULL, 0, bw, + bh); + wedge_params->masks[0][w] = dst; + dst += bw * bh; + + mask = get_wedge_mask_inplace(w, 1, bsize); + aom_convolve_copy(mask, MASK_MASTER_STRIDE, dst, bw, NULL, 0, NULL, 0, bw, + bh); + wedge_params->masks[1][w] = dst; + dst += bw * bh; + } + assert(sizeof(wedge_mask_buf) >= (size_t)(dst - wedge_mask_buf)); + } +} + +// Equation of line: f(x, y) = a[0]*(x - a[2]*w/8) + a[1]*(y - a[3]*h/8) = 0 +void av1_init_wedge_masks() { + init_wedge_master_masks(); + init_wedge_signs(); + init_wedge_masks(); +} + +#if CONFIG_SUPERTX +static void build_masked_compound_wedge_extend( + uint8_t *dst, int dst_stride, const uint8_t *src0, int src0_stride, + const uint8_t *src1, int src1_stride, + const INTERINTER_COMPOUND_DATA *const comp_data, BLOCK_SIZE sb_type, + int wedge_offset_x, int wedge_offset_y, int h, int w) { + const int subh = (2 << b_height_log2_lookup[sb_type]) == h; + const int subw = (2 << b_width_log2_lookup[sb_type]) == w; + const uint8_t *mask; + size_t mask_stride; + switch (comp_data->interinter_compound_type) { + case COMPOUND_WEDGE: + mask = av1_get_soft_mask(comp_data->wedge_index, comp_data->wedge_sign, + sb_type, wedge_offset_x, wedge_offset_y); + mask_stride = MASK_MASTER_STRIDE; + break; +#if CONFIG_COMPOUND_SEGMENT + case COMPOUND_SEG: + mask = comp_data->seg_mask; + mask_stride = block_size_wide[sb_type]; + break; +#endif + default: assert(0); return; + } + aom_blend_a64_mask(dst, dst_stride, src0, src0_stride, src1, src1_stride, + mask, (int)mask_stride, h, w, subh, subw); +} + +#if CONFIG_HIGHBITDEPTH +static void build_masked_compound_wedge_extend_highbd( + uint8_t *dst_8, int dst_stride, const uint8_t *src0_8, int src0_stride, + const uint8_t *src1_8, int src1_stride, + const INTERINTER_COMPOUND_DATA *const comp_data, BLOCK_SIZE sb_type, + int wedge_offset_x, int wedge_offset_y, int h, int w, int bd) { + const int subh = (2 << b_height_log2_lookup[sb_type]) == h; + const int subw = (2 << b_width_log2_lookup[sb_type]) == w; + const uint8_t *mask; + size_t mask_stride; + switch (comp_data->interinter_compound_type) { + case COMPOUND_WEDGE: + mask = av1_get_soft_mask(comp_data->wedge_index, comp_data->wedge_sign, + sb_type, wedge_offset_x, wedge_offset_y); + mask_stride = MASK_MASTER_STRIDE; + break; +#if CONFIG_COMPOUND_SEGMENT + case COMPOUND_SEG: + mask = comp_data->seg_mask; + mask_stride = block_size_wide[sb_type]; + break; +#endif + default: assert(0); return; + } + aom_highbd_blend_a64_mask(dst_8, dst_stride, src0_8, src0_stride, src1_8, + src1_stride, mask, (int)mask_stride, h, w, subh, + subw, bd); +} +#endif // CONFIG_HIGHBITDEPTH +#else +static void build_masked_compound( + uint8_t *dst, int dst_stride, const uint8_t *src0, int src0_stride, + const uint8_t *src1, int src1_stride, + const INTERINTER_COMPOUND_DATA *const comp_data, BLOCK_SIZE sb_type, int h, + int w) { + // Derive subsampling from h and w passed in. May be refactored to + // pass in subsampling factors directly. + const int subh = (2 << b_height_log2_lookup[sb_type]) == h; + const int subw = (2 << b_width_log2_lookup[sb_type]) == w; + const uint8_t *mask = av1_get_compound_type_mask(comp_data, sb_type); + aom_blend_a64_mask(dst, dst_stride, src0, src0_stride, src1, src1_stride, + mask, block_size_wide[sb_type], h, w, subh, subw); +} + +#if CONFIG_HIGHBITDEPTH +static void build_masked_compound_highbd( + uint8_t *dst_8, int dst_stride, const uint8_t *src0_8, int src0_stride, + const uint8_t *src1_8, int src1_stride, + const INTERINTER_COMPOUND_DATA *const comp_data, BLOCK_SIZE sb_type, int h, + int w, int bd) { + // Derive subsampling from h and w passed in. May be refactored to + // pass in subsampling factors directly. + const int subh = (2 << b_height_log2_lookup[sb_type]) == h; + const int subw = (2 << b_width_log2_lookup[sb_type]) == w; + const uint8_t *mask = av1_get_compound_type_mask(comp_data, sb_type); + // const uint8_t *mask = + // av1_get_contiguous_soft_mask(wedge_index, wedge_sign, sb_type); + aom_highbd_blend_a64_mask(dst_8, dst_stride, src0_8, src0_stride, src1_8, + src1_stride, mask, block_size_wide[sb_type], h, w, + subh, subw, bd); +} +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_SUPERTX + +void av1_make_masked_inter_predictor(const uint8_t *pre, int pre_stride, + uint8_t *dst, int dst_stride, + const int subpel_x, const int subpel_y, + const struct scale_factors *sf, int w, + int h, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif + int xs, int ys, +#if CONFIG_SUPERTX + int wedge_offset_x, int wedge_offset_y, +#endif // CONFIG_SUPERTX + int plane, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + const WarpTypesAllowed *warp_types, + int p_col, int p_row, int ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + MACROBLOCKD *xd) { + MODE_INFO *mi = xd->mi[0]; + const INTERINTER_COMPOUND_DATA comp_data = { +#if CONFIG_WEDGE + mi->mbmi.wedge_index, + mi->mbmi.wedge_sign, +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + mi->mbmi.mask_type, + xd->seg_mask, +#endif // CONFIG_COMPOUND_SEGMENT + mi->mbmi.interinter_compound_type + }; +// The prediction filter types used here should be those for +// the second reference block. +#if CONFIG_DUAL_FILTER + InterpFilter tmp_ipf[4] = { + interp_filter[2], interp_filter[3], interp_filter[2], interp_filter[3], + }; +#else + InterpFilter tmp_ipf = interp_filter; +#endif // CONFIG_DUAL_FILTER + ConvolveParams conv_params = get_conv_params(0, plane); + +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint8_t, tmp_dst_[2 * MAX_SB_SQUARE]); + uint8_t *tmp_dst = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + ? CONVERT_TO_BYTEPTR(tmp_dst_) + : tmp_dst_; + av1_make_inter_predictor(pre, pre_stride, tmp_dst, MAX_SB_SIZE, subpel_x, + subpel_y, sf, w, h, &conv_params, tmp_ipf, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + warp_types, p_col, p_row, plane, ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#if CONFIG_MOTION_VAR + 0, 0, +#endif + xs, ys, xd); +#if CONFIG_COMPOUND_SEGMENT + if (!plane && comp_data.interinter_compound_type == COMPOUND_SEG) { + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + build_compound_seg_mask_highbd(comp_data.seg_mask, comp_data.mask_type, + dst, dst_stride, tmp_dst, MAX_SB_SIZE, + mi->mbmi.sb_type, h, w, xd->bd); + else + build_compound_seg_mask(comp_data.seg_mask, comp_data.mask_type, dst, + dst_stride, tmp_dst, MAX_SB_SIZE, + mi->mbmi.sb_type, h, w); + } +#endif // CONFIG_COMPOUND_SEGMENT + +#if CONFIG_SUPERTX + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + build_masked_compound_wedge_extend_highbd( + dst, dst_stride, dst, dst_stride, tmp_dst, MAX_SB_SIZE, &comp_data, + mi->mbmi.sb_type, wedge_offset_x, wedge_offset_y, h, w, xd->bd); + else + build_masked_compound_wedge_extend( + dst, dst_stride, dst, dst_stride, tmp_dst, MAX_SB_SIZE, &comp_data, + mi->mbmi.sb_type, wedge_offset_x, wedge_offset_y, h, w); +#else + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + build_masked_compound_highbd(dst, dst_stride, dst, dst_stride, tmp_dst, + MAX_SB_SIZE, &comp_data, mi->mbmi.sb_type, h, + w, xd->bd); + else + build_masked_compound(dst, dst_stride, dst, dst_stride, tmp_dst, + MAX_SB_SIZE, &comp_data, mi->mbmi.sb_type, h, w); +#endif // CONFIG_SUPERTX + +#else // CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint8_t, tmp_dst[MAX_SB_SQUARE]); + av1_make_inter_predictor(pre, pre_stride, tmp_dst, MAX_SB_SIZE, subpel_x, + subpel_y, sf, w, h, &conv_params, tmp_ipf, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + warp_types, p_col, p_row, plane, ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#if CONFIG_MOTION_VAR + 0, 0, +#endif + xs, ys, xd); +#if CONFIG_COMPOUND_SEGMENT + if (!plane && comp_data.interinter_compound_type == COMPOUND_SEG) + build_compound_seg_mask(comp_data.seg_mask, comp_data.mask_type, dst, + dst_stride, tmp_dst, MAX_SB_SIZE, mi->mbmi.sb_type, + h, w); +#endif // CONFIG_COMPOUND_SEGMENT +#if CONFIG_SUPERTX + build_masked_compound_wedge_extend(dst, dst_stride, dst, dst_stride, tmp_dst, + MAX_SB_SIZE, &comp_data, mi->mbmi.sb_type, + wedge_offset_x, wedge_offset_y, h, w); +#else + build_masked_compound(dst, dst_stride, dst, dst_stride, tmp_dst, MAX_SB_SIZE, + &comp_data, mi->mbmi.sb_type, h, w); +#endif // CONFIG_SUPERTX +#endif // CONFIG_HIGHBITDEPTH +#if CONFIG_COMPOUND_SEGMENT + (void)plane; +#endif // CONFIG_COMPOUND_SEGMENT +} +#endif // CONFIG_EXT_INTER + +// TODO(sarahparker) av1_highbd_build_inter_predictor and +// av1_build_inter_predictor should be combined with +// av1_make_inter_predictor +#if CONFIG_HIGHBITDEPTH +void av1_highbd_build_inter_predictor( + const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, + const MV *src_mv, const struct scale_factors *sf, int w, int h, int ref, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + const WarpTypesAllowed *warp_types, int p_col, int p_row, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + int plane, enum mv_precision precision, int x, int y, + const MACROBLOCKD *xd) { + const int is_q4 = precision == MV_PRECISION_Q4; + const MV mv_q4 = { is_q4 ? src_mv->row : src_mv->row * 2, + is_q4 ? src_mv->col : src_mv->col * 2 }; + MV32 mv = av1_scale_mv(&mv_q4, x, y, sf); + const int subpel_x = mv.col & SUBPEL_MASK; + const int subpel_y = mv.row & SUBPEL_MASK; + ConvolveParams conv_params = get_conv_params(ref, plane); + + src += (mv.row >> SUBPEL_BITS) * src_stride + (mv.col >> SUBPEL_BITS); + + av1_make_inter_predictor(src, src_stride, dst, dst_stride, subpel_x, subpel_y, + sf, w, h, &conv_params, interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + warp_types, p_col, p_row, plane, ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#if CONFIG_MOTION_VAR + 0, 0, +#endif + sf->x_step_q4, sf->y_step_q4, xd); +} +#endif // CONFIG_HIGHBITDEPTH + +void av1_build_inter_predictor(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, const MV *src_mv, + const struct scale_factors *sf, int w, int h, + ConvolveParams *conv_params, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + const WarpTypesAllowed *warp_types, int p_col, + int p_row, int plane, int ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + enum mv_precision precision, int x, int y, + const MACROBLOCKD *xd) { + const int is_q4 = precision == MV_PRECISION_Q4; + const MV mv_q4 = { is_q4 ? src_mv->row : src_mv->row * 2, + is_q4 ? src_mv->col : src_mv->col * 2 }; + MV32 mv = av1_scale_mv(&mv_q4, x, y, sf); + const int subpel_x = mv.col & SUBPEL_MASK; + const int subpel_y = mv.row & SUBPEL_MASK; + + src += (mv.row >> SUBPEL_BITS) * src_stride + (mv.col >> SUBPEL_BITS); + + av1_make_inter_predictor(src, src_stride, dst, dst_stride, subpel_x, subpel_y, + sf, w, h, conv_params, interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + warp_types, p_col, p_row, plane, ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#if CONFIG_MOTION_VAR + 0, 0, +#endif + sf->x_step_q4, sf->y_step_q4, xd); +} + +typedef struct SubpelParams { + int xs; + int ys; + int subpel_x; + int subpel_y; +} SubpelParams; + +void build_inter_predictors(MACROBLOCKD *xd, int plane, +#if CONFIG_MOTION_VAR + int mi_col_offset, int mi_row_offset, +#endif // CONFIG_MOTION_VAR + int block, int bw, int bh, int x, int y, int w, + int h, +#if CONFIG_SUPERTX && CONFIG_EXT_INTER + int wedge_offset_x, int wedge_offset_y, +#endif // CONFIG_SUPERTX && CONFIG_EXT_INTER + int mi_x, int mi_y) { + struct macroblockd_plane *const pd = &xd->plane[plane]; +#if CONFIG_MOTION_VAR + const MODE_INFO *mi = xd->mi[mi_col_offset + xd->mi_stride * mi_row_offset]; +#if !CONFIG_CB4X4 || CONFIG_SUB8X8_MC + const int build_for_obmc = !(mi_col_offset == 0 && mi_row_offset == 0); +#endif // !CONFIG_CB4X4 || CONFIG_SUB8X8_MC +#else + const MODE_INFO *mi = xd->mi[0]; +#endif // CONFIG_MOTION_VAR + const int is_compound = has_second_ref(&mi->mbmi); + int ref; +#if CONFIG_INTRABC + const int is_intrabc = is_intrabc_block(&mi->mbmi); + struct scale_factors sf_identity; +#if CONFIG_HIGHBITDEPTH + av1_setup_scale_factors_for_frame( + &sf_identity, 64, 64, 64, 64, + xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH); +#else + av1_setup_scale_factors_for_frame(&sf_identity, 64, 64, 64, 64); +#endif // CONFIG_HIGHBITDEPTH + assert(IMPLIES(is_intrabc, !is_compound)); +#endif // CONFIG_INTRABC +#if CONFIG_GLOBAL_MOTION + int is_global[2]; + for (ref = 0; ref < 1 + is_compound; ++ref) { + WarpedMotionParams *const wm = &xd->global_motion[mi->mbmi.ref_frame[ref]]; + is_global[ref] = is_global_mv_block(mi, block, wm->wmtype); + } +#endif // CONFIG_GLOBAL_MOTION + +#if CONFIG_CB4X4 + (void)block; +#endif + +#if CONFIG_SUB8X8_MC +#if CONFIG_MOTION_VAR + if (mi->mbmi.sb_type < BLOCK_8X8 && plane > 0 && !build_for_obmc) { +#else + if (mi->mbmi.sb_type < BLOCK_8X8 && plane > 0) { +#endif // CONFIG_MOTION_VAR + // block size in log2 + const int b4_wl = b_width_log2_lookup[mi->mbmi.sb_type]; + const int b4_hl = b_height_log2_lookup[mi->mbmi.sb_type]; + const int b8_sl = b_width_log2_lookup[BLOCK_8X8]; + + // block size + const int b4_w = 1 << b4_wl; + const int b4_h = 1 << b4_hl; + const int b8_s = 1 << b8_sl; + int idx, idy; + + const int x_base = x; + const int y_base = y; + + // processing unit size + const int x_step = w >> (b8_sl - b4_wl); + const int y_step = h >> (b8_sl - b4_hl); + + for (idy = 0; idy < b8_s; idy += b4_h) { + for (idx = 0; idx < b8_s; idx += b4_w) { + const int chr_idx = (idy * 2) + idx; + for (ref = 0; ref < 1 + is_compound; ++ref) { + struct buf_2d *const dst_buf = &pd->dst; +#if CONFIG_INTRABC + const struct scale_factors *const sf = + is_intrabc ? &sf_identity : &xd->block_refs[ref]->sf; + struct buf_2d *const pre_buf = is_intrabc ? dst_buf : &pd->pre[ref]; +#else + const struct scale_factors *const sf = &xd->block_refs[ref]->sf; + struct buf_2d *const pre_buf = &pd->pre[ref]; +#endif // CONFIG_INTRABC + uint8_t *dst = dst_buf->buf; + const MV mv = mi->bmi[chr_idx].as_mv[ref].as_mv; + const MV mv_q4 = clamp_mv_to_umv_border_sb( + xd, &mv, bw, bh, pd->subsampling_x, pd->subsampling_y); + uint8_t *pre; + MV32 scaled_mv; + int xs, ys, subpel_x, subpel_y; + const int is_scaled = av1_is_scaled(sf); + ConvolveParams conv_params = get_conv_params(ref, plane); +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + WarpTypesAllowed warp_types; +#if CONFIG_GLOBAL_MOTION + warp_types.global_warp_allowed = is_global[ref]; +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_WARPED_MOTION + warp_types.local_warp_allowed = mi->mbmi.motion_mode == WARPED_CAUSAL; +#endif // CONFIG_WARPED_MOTION +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + + x = x_base + idx * x_step; + y = y_base + idy * y_step; + + dst += dst_buf->stride * y + x; + + if (is_scaled) { + pre = + pre_buf->buf + scaled_buffer_offset(x, y, pre_buf->stride, sf); + scaled_mv = av1_scale_mv(&mv_q4, mi_x + x, mi_y + y, sf); + xs = sf->x_step_q4; + ys = sf->y_step_q4; + } else { + pre = pre_buf->buf + y * pre_buf->stride + x; + scaled_mv.row = mv_q4.row; + scaled_mv.col = mv_q4.col; + xs = ys = 16; + } + + subpel_x = scaled_mv.col & SUBPEL_MASK; + subpel_y = scaled_mv.row & SUBPEL_MASK; + pre += (scaled_mv.row >> SUBPEL_BITS) * pre_buf->stride + + (scaled_mv.col >> SUBPEL_BITS); + +#if CONFIG_EXT_INTER + if (ref && is_masked_compound_type(mi->mbmi.interinter_compound_type)) + av1_make_masked_inter_predictor( + pre, pre_buf->stride, dst, dst_buf->stride, subpel_x, subpel_y, + sf, w, h, mi->mbmi.interp_filter, xs, ys, +#if CONFIG_SUPERTX + wedge_offset_x, wedge_offset_y, +#endif // CONFIG_SUPERTX + plane, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, (mi_x >> pd->subsampling_x) + x, + (mi_y >> pd->subsampling_y) + y, ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + xd); + else +#endif // CONFIG_EXT_INTER + av1_make_inter_predictor( + pre, pre_buf->stride, dst, dst_buf->stride, subpel_x, subpel_y, + sf, x_step, y_step, &conv_params, mi->mbmi.interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, (mi_x >> pd->subsampling_x) + x, + (mi_y >> pd->subsampling_y) + y, plane, ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#if CONFIG_MOTION_VAR + mi_col_offset, mi_row_offset, +#endif + xs, ys, xd); + } + } + } + return; + } +#endif + + { + struct buf_2d *const dst_buf = &pd->dst; + uint8_t *const dst = dst_buf->buf + dst_buf->stride * y + x; + uint8_t *pre[2]; + MV32 scaled_mv[2]; + SubpelParams subpel_params[2]; +#if CONFIG_CONVOLVE_ROUND + DECLARE_ALIGNED(16, int32_t, tmp_dst[MAX_SB_SIZE * MAX_SB_SIZE]); + av1_zero(tmp_dst); +#endif // CONFIG_CONVOLVE_ROUND + + for (ref = 0; ref < 1 + is_compound; ++ref) { +#if CONFIG_INTRABC + const struct scale_factors *const sf = + is_intrabc ? &sf_identity : &xd->block_refs[ref]->sf; + struct buf_2d *const pre_buf = is_intrabc ? dst_buf : &pd->pre[ref]; +#else + const struct scale_factors *const sf = &xd->block_refs[ref]->sf; + struct buf_2d *const pre_buf = &pd->pre[ref]; +#endif // CONFIG_INTRABC +#if CONFIG_CB4X4 + const MV mv = mi->mbmi.mv[ref].as_mv; +#else + const MV mv = +#if CONFIG_MOTION_VAR + (mi->mbmi.sb_type < BLOCK_8X8 && !build_for_obmc) + ? +#else + mi->mbmi.sb_type < BLOCK_8X8 ? +#endif + average_split_mvs(pd, mi, ref, block) + : mi->mbmi.mv[ref].as_mv; +#endif + + // TODO(jkoleszar): This clamping is done in the incorrect place for the + // scaling case. It needs to be done on the scaled MV, not the pre-scaling + // MV. Note however that it performs the subsampling aware scaling so + // that the result is always q4. + // mv_precision precision is MV_PRECISION_Q4. + const MV mv_q4 = clamp_mv_to_umv_border_sb( + xd, &mv, bw, bh, pd->subsampling_x, pd->subsampling_y); + + const int is_scaled = av1_is_scaled(sf); + if (is_scaled) { + pre[ref] = + pre_buf->buf + scaled_buffer_offset(x, y, pre_buf->stride, sf); + scaled_mv[ref] = av1_scale_mv(&mv_q4, mi_x + x, mi_y + y, sf); + subpel_params[ref].xs = sf->x_step_q4; + subpel_params[ref].ys = sf->y_step_q4; + } else { + pre[ref] = pre_buf->buf + (y * pre_buf->stride + x); + scaled_mv[ref].row = mv_q4.row; + scaled_mv[ref].col = mv_q4.col; + subpel_params[ref].xs = 16; + subpel_params[ref].ys = 16; + } + + subpel_params[ref].subpel_x = scaled_mv[ref].col & SUBPEL_MASK; + subpel_params[ref].subpel_y = scaled_mv[ref].row & SUBPEL_MASK; + pre[ref] += (scaled_mv[ref].row >> SUBPEL_BITS) * pre_buf->stride + + (scaled_mv[ref].col >> SUBPEL_BITS); + } + +#if CONFIG_CONVOLVE_ROUND + ConvolveParams conv_params = + get_conv_params_no_round(ref, plane, tmp_dst, MAX_SB_SIZE); +#else + ConvolveParams conv_params = get_conv_params(ref, plane); +#endif // CONFIG_CONVOLVE_ROUND + for (ref = 0; ref < 1 + is_compound; ++ref) { +#if CONFIG_INTRABC + const struct scale_factors *const sf = + is_intrabc ? &sf_identity : &xd->block_refs[ref]->sf; + struct buf_2d *const pre_buf = is_intrabc ? dst_buf : &pd->pre[ref]; +#else + const struct scale_factors *const sf = &xd->block_refs[ref]->sf; + struct buf_2d *const pre_buf = &pd->pre[ref]; +#endif // CONFIG_INTRABC +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + WarpTypesAllowed warp_types; +#if CONFIG_GLOBAL_MOTION + warp_types.global_warp_allowed = is_global[ref]; +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_WARPED_MOTION + warp_types.local_warp_allowed = mi->mbmi.motion_mode == WARPED_CAUSAL; +#endif // CONFIG_WARPED_MOTION +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + conv_params.ref = ref; +#if CONFIG_EXT_INTER + if (ref && is_masked_compound_type(mi->mbmi.interinter_compound_type)) + av1_make_masked_inter_predictor( + pre[ref], pre_buf->stride, dst, dst_buf->stride, + subpel_params[ref].subpel_x, subpel_params[ref].subpel_y, sf, w, h, + mi->mbmi.interp_filter, subpel_params[ref].xs, + subpel_params[ref].ys, +#if CONFIG_SUPERTX + wedge_offset_x, wedge_offset_y, +#endif // CONFIG_SUPERTX + plane, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, (mi_x >> pd->subsampling_x) + x, + (mi_y >> pd->subsampling_y) + y, ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + xd); + else +#endif // CONFIG_EXT_INTER + av1_make_inter_predictor( + pre[ref], pre_buf->stride, dst, dst_buf->stride, + subpel_params[ref].subpel_x, subpel_params[ref].subpel_y, sf, w, h, + &conv_params, mi->mbmi.interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, (mi_x >> pd->subsampling_x) + x, + (mi_y >> pd->subsampling_y) + y, plane, ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#if CONFIG_MOTION_VAR + mi_col_offset, mi_row_offset, +#endif + subpel_params[ref].xs, subpel_params[ref].ys, xd); + } + +#if CONFIG_CONVOLVE_ROUND +// TODO(angiebird): This part needs optimization +#if CONFIG_HIGHBITDEPTH + if (!(xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)) +#endif // CONFIG_HIGHBITDEPTH + av1_convolve_rounding(tmp_dst, MAX_SB_SIZE, dst, dst_buf->stride, w, h, + FILTER_BITS * 2 + is_compound - + conv_params.round_0 - conv_params.round_1); +#endif // CONFIG_CONVOLVE_ROUND + } +} + +void av1_build_inter_predictor_sub8x8(MACROBLOCKD *xd, int plane, int i, int ir, + int ic, int mi_row, int mi_col) { + struct macroblockd_plane *const pd = &xd->plane[plane]; + MODE_INFO *const mi = xd->mi[0]; + const BLOCK_SIZE plane_bsize = get_plane_block_size(mi->mbmi.sb_type, pd); + const int width = block_size_wide[plane_bsize]; + const int height = block_size_high[plane_bsize]; + uint8_t *const dst = &pd->dst.buf[(ir * pd->dst.stride + ic) << 2]; + int ref; + const int is_compound = has_second_ref(&mi->mbmi); +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + WarpTypesAllowed warp_types; + const int p_col = ((mi_col * MI_SIZE) >> pd->subsampling_x) + 4 * ic; + const int p_row = ((mi_row * MI_SIZE) >> pd->subsampling_y) + 4 * ir; +#if CONFIG_GLOBAL_MOTION + int is_global[2]; + for (ref = 0; ref < 1 + is_compound; ++ref) { + WarpedMotionParams *const wm = &xd->global_motion[mi->mbmi.ref_frame[ref]]; + is_global[ref] = is_global_mv_block(mi, i, wm->wmtype); + } +#endif // CONFIG_GLOBAL_MOTION +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + + for (ref = 0; ref < 1 + is_compound; ++ref) { + ConvolveParams conv_params = get_conv_params(ref, plane); + const uint8_t *pre = + &pd->pre[ref].buf[(ir * pd->pre[ref].stride + ic) << 2]; +#if CONFIG_GLOBAL_MOTION + warp_types.global_warp_allowed = is_global[ref]; +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_WARPED_MOTION + warp_types.local_warp_allowed = mi->mbmi.motion_mode == WARPED_CAUSAL; +#endif // CONFIG_WARPED_MOTION + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + av1_highbd_build_inter_predictor( + pre, pd->pre[ref].stride, dst, pd->dst.stride, + &mi->bmi[i].as_mv[ref].as_mv, &xd->block_refs[ref]->sf, width, height, + ref, mi->mbmi.interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, p_col, p_row, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + plane, MV_PRECISION_Q3, mi_col * MI_SIZE + 4 * ic, + mi_row * MI_SIZE + 4 * ir, xd); + else +#endif // CONFIG_HIGHBITDEPTH + av1_build_inter_predictor(pre, pd->pre[ref].stride, dst, pd->dst.stride, + &mi->bmi[i].as_mv[ref].as_mv, + &xd->block_refs[ref]->sf, width, height, + &conv_params, mi->mbmi.interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, p_col, p_row, plane, ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + MV_PRECISION_Q3, mi_col * MI_SIZE + 4 * ic, + mi_row * MI_SIZE + 4 * ir, xd); + } +} + +static void build_inter_predictors_for_planes(MACROBLOCKD *xd, BLOCK_SIZE bsize, + int mi_row, int mi_col, + int plane_from, int plane_to) { + int plane; + const int mi_x = mi_col * MI_SIZE; + const int mi_y = mi_row * MI_SIZE; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + for (plane = plane_from; plane <= plane_to; ++plane) { + const struct macroblockd_plane *pd = &xd->plane[plane]; + const int bw = pd->width; + const int bh = pd->height; + +#if CONFIG_CB4X4 + if (!is_chroma_reference(mi_row, mi_col, bsize, pd->subsampling_x, + pd->subsampling_y)) + continue; +#endif + + if (xd->mi[0]->mbmi.sb_type < BLOCK_8X8 && !unify_bsize) { + const PARTITION_TYPE bp = bsize - xd->mi[0]->mbmi.sb_type; + const int have_vsplit = bp != PARTITION_HORZ; + const int have_hsplit = bp != PARTITION_VERT; + const int num_4x4_w = 2 >> ((!have_vsplit) | pd->subsampling_x); + const int num_4x4_h = 2 >> ((!have_hsplit) | pd->subsampling_y); + const int pw = 8 >> (have_vsplit | pd->subsampling_x); + const int ph = 8 >> (have_hsplit | pd->subsampling_y); + int x, y; + assert(bp != PARTITION_NONE && bp < PARTITION_TYPES); + assert(bsize == BLOCK_8X8); + assert(pw * num_4x4_w == bw && ph * num_4x4_h == bh); + for (y = 0; y < num_4x4_h; ++y) + for (x = 0; x < num_4x4_w; ++x) + build_inter_predictors(xd, plane, +#if CONFIG_MOTION_VAR + 0, 0, +#endif // CONFIG_MOTION_VAR + y * 2 + x, bw, bh, 4 * x, 4 * y, pw, ph, +#if CONFIG_SUPERTX && CONFIG_EXT_INTER + 0, 0, +#endif // CONFIG_SUPERTX && CONFIG_EXT_INTER + mi_x, mi_y); + } else { + build_inter_predictors(xd, plane, +#if CONFIG_MOTION_VAR + 0, 0, +#endif // CONFIG_MOTION_VAR + 0, bw, bh, 0, 0, bw, bh, +#if CONFIG_SUPERTX && CONFIG_EXT_INTER + 0, 0, +#endif // CONFIG_SUPERTX && CONFIG_EXT_INTER + mi_x, mi_y); + } + } +} + +void av1_build_inter_predictors_sby(MACROBLOCKD *xd, int mi_row, int mi_col, + BUFFER_SET *ctx, BLOCK_SIZE bsize) { + build_inter_predictors_for_planes(xd, bsize, mi_row, mi_col, 0, 0); +#if CONFIG_EXT_INTER + if (is_interintra_pred(&xd->mi[0]->mbmi)) { + BUFFER_SET default_ctx = { { xd->plane[0].dst.buf, NULL, NULL }, + { xd->plane[0].dst.stride, 0, 0 } }; + if (!ctx) ctx = &default_ctx; + av1_build_interintra_predictors_sby(xd, xd->plane[0].dst.buf, + xd->plane[0].dst.stride, ctx, bsize); + } +#else + (void)ctx; +#endif // CONFIG_EXT_INTER +} + +void av1_build_inter_predictors_sbuv(MACROBLOCKD *xd, int mi_row, int mi_col, + BUFFER_SET *ctx, BLOCK_SIZE bsize) { + build_inter_predictors_for_planes(xd, bsize, mi_row, mi_col, 1, + MAX_MB_PLANE - 1); +#if CONFIG_EXT_INTER + if (is_interintra_pred(&xd->mi[0]->mbmi)) { + BUFFER_SET default_ctx = { + { NULL, xd->plane[1].dst.buf, xd->plane[2].dst.buf }, + { 0, xd->plane[1].dst.stride, xd->plane[2].dst.stride } + }; + if (!ctx) ctx = &default_ctx; + av1_build_interintra_predictors_sbuv( + xd, xd->plane[1].dst.buf, xd->plane[2].dst.buf, xd->plane[1].dst.stride, + xd->plane[2].dst.stride, ctx, bsize); + } +#else + (void)ctx; +#endif // CONFIG_EXT_INTER +} + +// TODO(afergs): Check if ctx can be made constant +void av1_build_inter_predictors_sb(MACROBLOCKD *xd, int mi_row, int mi_col, + BUFFER_SET *ctx, BLOCK_SIZE bsize) { + build_inter_predictors_for_planes(xd, bsize, mi_row, mi_col, 0, + MAX_MB_PLANE - 1); +#if CONFIG_EXT_INTER + if (is_interintra_pred(&xd->mi[0]->mbmi)) { + BUFFER_SET default_ctx = { + { xd->plane[0].dst.buf, xd->plane[1].dst.buf, xd->plane[2].dst.buf }, + { xd->plane[0].dst.stride, xd->plane[1].dst.stride, + xd->plane[2].dst.stride } + }; + if (!ctx) ctx = &default_ctx; + av1_build_interintra_predictors( + xd, xd->plane[0].dst.buf, xd->plane[1].dst.buf, xd->plane[2].dst.buf, + xd->plane[0].dst.stride, xd->plane[1].dst.stride, + xd->plane[2].dst.stride, ctx, bsize); + } +#else + (void)ctx; +#endif // CONFIG_EXT_INTER +} + +void av1_setup_dst_planes(struct macroblockd_plane planes[MAX_MB_PLANE], + BLOCK_SIZE bsize, const YV12_BUFFER_CONFIG *src, + int mi_row, int mi_col) { + uint8_t *const buffers[MAX_MB_PLANE] = { src->y_buffer, src->u_buffer, + src->v_buffer }; + const int widths[MAX_MB_PLANE] = { src->y_crop_width, src->uv_crop_width, + src->uv_crop_width }; + const int heights[MAX_MB_PLANE] = { src->y_crop_height, src->uv_crop_height, + src->uv_crop_height }; + const int strides[MAX_MB_PLANE] = { src->y_stride, src->uv_stride, + src->uv_stride }; + int i; + + for (i = 0; i < MAX_MB_PLANE; ++i) { + struct macroblockd_plane *const pd = &planes[i]; + setup_pred_plane(&pd->dst, bsize, buffers[i], widths[i], heights[i], + strides[i], mi_row, mi_col, NULL, pd->subsampling_x, + pd->subsampling_y); + } +} + +void av1_setup_pre_planes(MACROBLOCKD *xd, int idx, + const YV12_BUFFER_CONFIG *src, int mi_row, int mi_col, + const struct scale_factors *sf) { + if (src != NULL) { + int i; + uint8_t *const buffers[MAX_MB_PLANE] = { src->y_buffer, src->u_buffer, + src->v_buffer }; + const int widths[MAX_MB_PLANE] = { src->y_crop_width, src->uv_crop_width, + src->uv_crop_width }; + const int heights[MAX_MB_PLANE] = { src->y_crop_height, src->uv_crop_height, + src->uv_crop_height }; + const int strides[MAX_MB_PLANE] = { src->y_stride, src->uv_stride, + src->uv_stride }; + for (i = 0; i < MAX_MB_PLANE; ++i) { + struct macroblockd_plane *const pd = &xd->plane[i]; + setup_pred_plane(&pd->pre[idx], xd->mi[0]->mbmi.sb_type, buffers[i], + widths[i], heights[i], strides[i], mi_row, mi_col, sf, + pd->subsampling_x, pd->subsampling_y); + } + } +} + +#if CONFIG_SUPERTX +#if CONFIG_CB4X4 +static const uint8_t mask_4[4] = { 64, 52, 12, 0 }; +static const uint8_t mask_4_uv[4] = { 64, 52, 12, 0 }; +#endif // CONFIG_CB4X4 +static const uint8_t mask_8[8] = { 64, 64, 62, 52, 12, 2, 0, 0 }; + +static const uint8_t mask_16[16] = { 63, 62, 60, 58, 55, 50, 43, 36, + 28, 21, 14, 9, 6, 4, 2, 1 }; + +static const uint8_t mask_32[32] = { 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 63, + 61, 57, 52, 45, 36, 28, 19, 12, 7, 3, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + +static const uint8_t mask_8_uv[8] = { 64, 64, 62, 52, 12, 2, 0, 0 }; + +static const uint8_t mask_16_uv[16] = { 64, 64, 64, 64, 61, 53, 45, 36, + 28, 19, 11, 3, 0, 0, 0, 0 }; + +static const uint8_t mask_32_uv[32] = { 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 60, 54, 46, 36, + 28, 18, 10, 4, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 }; + +static const uint8_t *get_supertx_mask(int length, int plane) { + switch (length) { +#if CONFIG_CB4X4 + case 4: return plane ? mask_4_uv : mask_4; +#endif // CONFIG_CB4X4 + case 8: return plane ? mask_8_uv : mask_8; + case 16: return plane ? mask_16_uv : mask_16; + case 32: return plane ? mask_32_uv : mask_32; + default: assert(0); + } + return NULL; +} + +void av1_build_masked_inter_predictor_complex( + MACROBLOCKD *xd, uint8_t *dst, int dst_stride, const uint8_t *pre, + int pre_stride, int mi_row, int mi_col, int mi_row_ori, int mi_col_ori, + BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, PARTITION_TYPE partition, + int plane) { + const struct macroblockd_plane *pd = &xd->plane[plane]; + const int ssx = pd->subsampling_x; + const int ssy = pd->subsampling_y; + const int top_w = block_size_wide[top_bsize] >> ssx; + const int top_h = block_size_high[top_bsize] >> ssy; + const int w = block_size_wide[bsize] >> ssx; + const int h = block_size_high[bsize] >> ssy; + const int w_offset = ((mi_col - mi_col_ori) * MI_SIZE) >> ssx; + const int h_offset = ((mi_row - mi_row_ori) * MI_SIZE) >> ssy; + + int w_remain, h_remain; + +#if CONFIG_HIGHBITDEPTH + const int is_hdb = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? 1 : 0; +#endif // CONFIG_HIGHBITDEPTH + + assert(bsize <= BLOCK_32X32); + assert(IMPLIES(plane == 0, ssx == 0)); + assert(IMPLIES(plane == 0, ssy == 0)); + + switch (partition) { + case PARTITION_HORZ: { + const uint8_t *const mask = get_supertx_mask(h, ssy); + + w_remain = top_w; + h_remain = top_h - h_offset - h; + dst += h_offset * dst_stride; + pre += h_offset * pre_stride; + +#if CONFIG_HIGHBITDEPTH + if (is_hdb) + aom_highbd_blend_a64_vmask(dst, dst_stride, dst, dst_stride, pre, + pre_stride, mask, h, top_w, xd->bd); + else +#endif // CONFIG_HIGHBITDEPTH + aom_blend_a64_vmask(dst, dst_stride, dst, dst_stride, pre, pre_stride, + mask, h, top_w); + + dst += h * dst_stride; + pre += h * pre_stride; + break; + } + case PARTITION_VERT: { + const uint8_t *const mask = get_supertx_mask(w, ssx); + + w_remain = top_w - w_offset - w; + h_remain = top_h; + dst += w_offset; + pre += w_offset; + +#if CONFIG_HIGHBITDEPTH + if (is_hdb) + aom_highbd_blend_a64_hmask(dst, dst_stride, dst, dst_stride, pre, + pre_stride, mask, top_h, w, xd->bd); + else +#endif // CONFIG_HIGHBITDEPTH + aom_blend_a64_hmask(dst, dst_stride, dst, dst_stride, pre, pre_stride, + mask, top_h, w); + + dst += w; + pre += w; + break; + } + default: { + assert(0); + return; + } + } + + if (w_remain == 0 || h_remain == 0) { + return; + } + +#if CONFIG_HIGHBITDEPTH + if (is_hdb) { + dst = (uint8_t *)CONVERT_TO_SHORTPTR(dst); + pre = (const uint8_t *)CONVERT_TO_SHORTPTR(pre); + dst_stride *= 2; + pre_stride *= 2; + w_remain *= 2; + } +#endif // CONFIG_HIGHBITDEPTH + + do { + memcpy(dst, pre, w_remain * sizeof(uint8_t)); + dst += dst_stride; + pre += pre_stride; + } while (--h_remain); +} + +void av1_build_inter_predictors_sb_sub8x8_extend(MACROBLOCKD *xd, +#if CONFIG_EXT_INTER + int mi_row_ori, int mi_col_ori, +#endif // CONFIG_EXT_INTER + int mi_row, int mi_col, + BLOCK_SIZE bsize, int block) { + // Prediction function used in supertx: + // Use the mv at current block (which is less than 8x8) + // to get prediction of a block located at (mi_row, mi_col) at size of bsize + // bsize can be larger than 8x8. + // block (0-3): the sub8x8 location of current block + int plane; + const int mi_x = mi_col * MI_SIZE; + const int mi_y = mi_row * MI_SIZE; +#if CONFIG_EXT_INTER + const int wedge_offset_x = (mi_col_ori - mi_col) * MI_SIZE; + const int wedge_offset_y = (mi_row_ori - mi_row) * MI_SIZE; +#endif // CONFIG_EXT_INTER + + // For sub8x8 uv: + // Skip uv prediction in supertx except the first block (block = 0) + int max_plane = block ? 1 : MAX_MB_PLANE; + + for (plane = 0; plane < max_plane; plane++) { + const BLOCK_SIZE plane_bsize = + get_plane_block_size(bsize, &xd->plane[plane]); + const int num_4x4_w = num_4x4_blocks_wide_lookup[plane_bsize]; + const int num_4x4_h = num_4x4_blocks_high_lookup[plane_bsize]; + const int bw = 4 * num_4x4_w; + const int bh = 4 * num_4x4_h; + + build_inter_predictors(xd, plane, +#if CONFIG_MOTION_VAR + 0, 0, +#endif // CONFIG_MOTION_VAR + block, bw, bh, 0, 0, bw, bh, +#if CONFIG_EXT_INTER + wedge_offset_x, wedge_offset_y, +#endif // CONFIG_EXT_INTER + mi_x, mi_y); + } +#if CONFIG_EXT_INTER + if (is_interintra_pred(&xd->mi[0]->mbmi)) { + BUFFER_SET ctx = { { xd->plane[0].dst.buf, xd->plane[1].dst.buf, + xd->plane[2].dst.buf }, + { xd->plane[0].dst.stride, xd->plane[1].dst.stride, + xd->plane[2].dst.stride } }; + av1_build_interintra_predictors( + xd, xd->plane[0].dst.buf, xd->plane[1].dst.buf, xd->plane[2].dst.buf, + xd->plane[0].dst.stride, xd->plane[1].dst.stride, + xd->plane[2].dst.stride, &ctx, bsize); + } +#endif // CONFIG_EXT_INTER +} + +void av1_build_inter_predictors_sb_extend(MACROBLOCKD *xd, +#if CONFIG_EXT_INTER + int mi_row_ori, int mi_col_ori, +#endif // CONFIG_EXT_INTER + int mi_row, int mi_col, + BLOCK_SIZE bsize) { + int plane; + const int mi_x = mi_col * MI_SIZE; + const int mi_y = mi_row * MI_SIZE; +#if CONFIG_EXT_INTER + const int wedge_offset_x = (mi_col_ori - mi_col) * MI_SIZE; + const int wedge_offset_y = (mi_row_ori - mi_row) * MI_SIZE; +#endif // CONFIG_EXT_INTER + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + const BLOCK_SIZE plane_bsize = + get_plane_block_size(bsize, &xd->plane[plane]); + const int bw = block_size_wide[plane_bsize]; + const int bh = block_size_high[plane_bsize]; + + build_inter_predictors(xd, plane, +#if CONFIG_MOTION_VAR + 0, 0, +#endif // CONFIG_MOTION_VAR + 0, bw, bh, 0, 0, bw, bh, +#if CONFIG_EXT_INTER + wedge_offset_x, wedge_offset_y, +#endif // CONFIG_EXT_INTER + mi_x, mi_y); + } +} +#endif // CONFIG_SUPERTX + +#if CONFIG_MOTION_VAR +// obmc_mask_N[overlap_position] +static const uint8_t obmc_mask_1[1] = { 64 }; + +static const uint8_t obmc_mask_2[2] = { 45, 64 }; + +static const uint8_t obmc_mask_4[4] = { 39, 50, 59, 64 }; + +static const uint8_t obmc_mask_8[8] = { 36, 42, 48, 53, 57, 61, 64, 64 }; + +static const uint8_t obmc_mask_16[16] = { 34, 37, 40, 43, 46, 49, 52, 54, + 56, 58, 60, 61, 64, 64, 64, 64 }; + +static const uint8_t obmc_mask_32[32] = { 33, 35, 36, 38, 40, 41, 43, 44, + 45, 47, 48, 50, 51, 52, 53, 55, + 56, 57, 58, 59, 60, 60, 61, 62, + 64, 64, 64, 64, 64, 64, 64, 64 }; + +#if CONFIG_EXT_PARTITION +static const uint8_t obmc_mask_64[64] = { + 33, 34, 35, 35, 36, 37, 38, 39, 40, 40, 41, 42, 43, 44, 44, 44, + 45, 46, 47, 47, 48, 49, 50, 51, 51, 51, 52, 52, 53, 54, 55, 56, + 56, 56, 57, 57, 58, 58, 59, 60, 60, 60, 60, 60, 61, 62, 62, 62, + 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, +}; +#endif // CONFIG_EXT_PARTITION + +const uint8_t *av1_get_obmc_mask(int length) { + switch (length) { + case 1: return obmc_mask_1; + case 2: return obmc_mask_2; + case 4: return obmc_mask_4; + case 8: return obmc_mask_8; + case 16: return obmc_mask_16; + case 32: return obmc_mask_32; +#if CONFIG_EXT_PARTITION + case 64: return obmc_mask_64; +#endif // CONFIG_EXT_PARTITION + default: assert(0); return NULL; + } +} + +#if CONFIG_NCOBMC +// obmc_mask_flipN[overlap_position] +static const uint8_t obmc_mask_flip1[1] = { 55 }; + +static const uint8_t obmc_mask_flip2[2] = { 62, 45 }; + +static const uint8_t obmc_mask_flip4[4] = { 64, 59, 50, 39 }; + +static const uint8_t obmc_mask_flip8[8] = { 64, 63, 61, 57, 53, 48, 42, 36 }; + +static const uint8_t obmc_mask_flip16[16] = { 64, 64, 64, 63, 61, 60, 58, 56, + 54, 52, 49, 46, 43, 40, 37, 34 }; + +static const uint8_t obmc_mask_flip32[32] = { 64, 64, 64, 64, 64, 63, 63, 62, + 62, 61, 60, 60, 59, 58, 57, 56, + 55, 53, 52, 51, 50, 48, 47, 45, + 44, 43, 41, 40, 38, 36, 35, 33 }; + +#if CONFIG_EXT_PARTITION +static const uint8_t obmc_mask_flip64[64] = { + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 62, 62, + 62, 62, 62, 61, 60, 60, 60, 60, 60, 59, 58, 58, 57, 57, 56, 56, + 56, 55, 54, 53, 52, 52, 51, 51, 51, 50, 49, 48, 47, 47, 46, 45, + 44, 44, 44, 43, 42, 41, 40, 40, 39, 38, 37, 36, 35, 35, 34, 33, +}; +#endif // CONFIG_EXT_PARTITION + +const uint8_t *av1_get_obmc_mask_flipped(int length) { + switch (length) { + case 1: return obmc_mask_flip1; + case 2: return obmc_mask_flip2; + case 4: return obmc_mask_flip4; + case 8: return obmc_mask_flip8; + case 16: return obmc_mask_flip16; + case 32: return obmc_mask_flip32; +#if CONFIG_EXT_PARTITION + case 64: return obmc_mask_flip64; +#endif // CONFIG_EXT_PARTITION + default: assert(0); return NULL; + } +} +#endif // CONFIG_NCOBMC + +void av1_count_overlappable_neighbors(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col) { + int i, mi_step; + + xd->mi[0]->mbmi.overlappable_neighbors[0] = 0; + xd->mi[0]->mbmi.overlappable_neighbors[1] = 0; + + if (xd->up_available) { + const int ilimit = AOMMIN(xd->n8_w, cm->mi_cols - mi_col); + for (i = 0; i < ilimit; i += mi_step) { + int mi_row_offset = -1; + int mi_col_offset = i; + MODE_INFO *above_mi = + xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]; + MB_MODE_INFO *above_mbmi = &above_mi->mbmi; + + mi_step = AOMMIN(xd->n8_w, mi_size_wide[above_mbmi->sb_type]); + + if (is_neighbor_overlappable(above_mbmi)) + xd->mi[0]->mbmi.overlappable_neighbors[0]++; + } + } + + if (xd->left_available) { + const int ilimit = AOMMIN(xd->n8_h, cm->mi_rows - mi_row); + for (i = 0; i < ilimit; i += mi_step) { + int mi_row_offset = i; + int mi_col_offset = -1; + MODE_INFO *left_mi = + xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]; + MB_MODE_INFO *left_mbmi = &left_mi->mbmi; + + mi_step = AOMMIN(xd->n8_h, mi_size_high[left_mbmi->sb_type]); + + if (is_neighbor_overlappable(left_mbmi)) + xd->mi[0]->mbmi.overlappable_neighbors[1]++; + } + } +} + +// HW does not support < 4x4 prediction. To limit the bandwidth requirement, for +// small blocks, only blend with neighbors from one side. If block-size of +// current plane is 4x4 or 8x4, the above neighbor (dir = 0) will be skipped. If +// it is 4x8, the left neighbor (dir = 1) will be skipped. +#define DISABLE_CHROMA_U8X8_OBMC 0 // 0: one-sided obmc; 1: disable + +int skip_u4x4_pred_in_obmc(BLOCK_SIZE bsize, const struct macroblockd_plane *pd, + int dir) { + assert(is_motion_variation_allowed_bsize(bsize)); + + BLOCK_SIZE bsize_plane = + ss_size_lookup[bsize][pd->subsampling_x][pd->subsampling_y]; +#if CONFIG_CB4X4 + if (bsize_plane < BLOCK_4X4) return 1; +#endif + switch (bsize_plane) { +#if DISABLE_CHROMA_U8X8_OBMC + case BLOCK_4X4: + case BLOCK_8X4: + case BLOCK_4X8: return 1; break; +#else + case BLOCK_4X4: + case BLOCK_8X4: + case BLOCK_4X8: return dir == 1; break; +#endif + default: return 0; + } +} + +// This function combines motion compensated predictions that is generated by +// top/left neighboring blocks' inter predictors with the regular inter +// prediction. We assume the original prediction (bmc) is stored in +// xd->plane[].dst.buf +void av1_build_obmc_inter_prediction(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col, + uint8_t *above[MAX_MB_PLANE], + int above_stride[MAX_MB_PLANE], + uint8_t *left[MAX_MB_PLANE], + int left_stride[MAX_MB_PLANE]) { + const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + int plane, i; +#if CONFIG_HIGHBITDEPTH + const int is_hbd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? 1 : 0; +#endif // CONFIG_HIGHBITDEPTH + + // handle above row + if (xd->up_available) { + const int overlap = num_4x4_blocks_high_lookup[bsize] * 2; + const int miw = AOMMIN(xd->n8_w, cm->mi_cols - mi_col); + const int mi_row_offset = -1; + const int neighbor_limit = max_neighbor_obmc[b_width_log2_lookup[bsize]]; + int neighbor_count = 0; + + assert(miw > 0); + + i = 0; + do { // for each mi in the above row + const int mi_col_offset = i; + const MB_MODE_INFO *const above_mbmi = + &xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]->mbmi; + const BLOCK_SIZE a_bsize = above_mbmi->sb_type; + const int mi_step = AOMMIN(xd->n8_w, mi_size_wide[a_bsize]); + + if (is_neighbor_overlappable(above_mbmi)) { + neighbor_count++; + if (neighbor_count > neighbor_limit) break; + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + const struct macroblockd_plane *pd = &xd->plane[plane]; + const int bw = (mi_step * MI_SIZE) >> pd->subsampling_x; + const int bh = overlap >> pd->subsampling_y; + + if (skip_u4x4_pred_in_obmc(bsize, pd, 0)) continue; + + const int dst_stride = pd->dst.stride; + uint8_t *const dst = &pd->dst.buf[(i * MI_SIZE) >> pd->subsampling_x]; + const int tmp_stride = above_stride[plane]; + const uint8_t *const tmp = + &above[plane][(i * MI_SIZE) >> pd->subsampling_x]; + const uint8_t *const mask = av1_get_obmc_mask(bh); + +#if CONFIG_HIGHBITDEPTH + if (is_hbd) + aom_highbd_blend_a64_vmask(dst, dst_stride, dst, dst_stride, tmp, + tmp_stride, mask, bh, bw, xd->bd); + else +#endif // CONFIG_HIGHBITDEPTH + aom_blend_a64_vmask(dst, dst_stride, dst, dst_stride, tmp, + tmp_stride, mask, bh, bw); + } + } + i += mi_step; + } while (i < miw); + } + + // handle left column + if (xd->left_available) { + const int overlap = num_4x4_blocks_wide_lookup[bsize] * 2; + const int mih = AOMMIN(xd->n8_h, cm->mi_rows - mi_row); + const int mi_col_offset = -1; + const int neighbor_limit = max_neighbor_obmc[b_height_log2_lookup[bsize]]; + int neighbor_count = 0; + + assert(mih > 0); + + i = 0; + do { // for each mi in the left column + const int mi_row_offset = i; + const MB_MODE_INFO *const left_mbmi = + &xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]->mbmi; + const BLOCK_SIZE l_bsize = left_mbmi->sb_type; + const int mi_step = AOMMIN(xd->n8_h, mi_size_high[l_bsize]); + + if (is_neighbor_overlappable(left_mbmi)) { + neighbor_count++; + if (neighbor_count > neighbor_limit) break; + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + const struct macroblockd_plane *pd = &xd->plane[plane]; + const int bw = overlap >> pd->subsampling_x; + const int bh = (mi_step * MI_SIZE) >> pd->subsampling_y; + + if (skip_u4x4_pred_in_obmc(bsize, pd, 1)) continue; + + const int dst_stride = pd->dst.stride; + uint8_t *const dst = + &pd->dst.buf[(i * MI_SIZE * dst_stride) >> pd->subsampling_y]; + const int tmp_stride = left_stride[plane]; + const uint8_t *const tmp = + &left[plane][(i * MI_SIZE * tmp_stride) >> pd->subsampling_y]; + const uint8_t *const mask = av1_get_obmc_mask(bw); + +#if CONFIG_HIGHBITDEPTH + if (is_hbd) + aom_highbd_blend_a64_hmask(dst, dst_stride, dst, dst_stride, tmp, + tmp_stride, mask, bh, bw, xd->bd); + else +#endif // CONFIG_HIGHBITDEPTH + aom_blend_a64_hmask(dst, dst_stride, dst, dst_stride, tmp, + tmp_stride, mask, bh, bw); + } + } + i += mi_step; + } while (i < mih); + } +} + +void modify_neighbor_predictor_for_obmc(MB_MODE_INFO *mbmi) { +#if CONFIG_EXT_INTER + if (is_interintra_pred(mbmi)) { + mbmi->ref_frame[1] = NONE_FRAME; + } else if (has_second_ref(mbmi) && + is_masked_compound_type(mbmi->interinter_compound_type)) { + mbmi->interinter_compound_type = COMPOUND_AVERAGE; + mbmi->ref_frame[1] = NONE_FRAME; + } +#endif // CONFIG_EXT_INTER + if (has_second_ref(mbmi)) mbmi->ref_frame[1] = NONE_FRAME; + return; +} + +void av1_build_prediction_by_above_preds(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col, + uint8_t *tmp_buf[MAX_MB_PLANE], + int tmp_width[MAX_MB_PLANE], + int tmp_height[MAX_MB_PLANE], + int tmp_stride[MAX_MB_PLANE]) { + const TileInfo *const tile = &xd->tile; + BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + int i, j, mi_step, ref; + const int ilimit = AOMMIN(xd->n8_w, cm->mi_cols - mi_col); + int mb_to_right_edge_base = xd->mb_to_right_edge; + const int neighbor_limit = max_neighbor_obmc[b_width_log2_lookup[bsize]]; + int neighbor_count = 0; + + if (mi_row <= tile->mi_row_start) return; + + xd->mb_to_bottom_edge += xd->n8_h * 32; + for (i = 0; i < ilimit; i += mi_step) { + int mi_row_offset = -1; + int mi_col_offset = i; + int mi_x, mi_y, bw, bh; + MODE_INFO *above_mi = xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]; + MB_MODE_INFO *above_mbmi = &above_mi->mbmi; + const BLOCK_SIZE a_bsize = above_mbmi->sb_type; + MB_MODE_INFO backup_mbmi; + + mi_step = AOMMIN(xd->n8_w, mi_size_wide[a_bsize]); + + if (!is_neighbor_overlappable(above_mbmi)) continue; + + neighbor_count++; + if (neighbor_count > neighbor_limit) break; + + backup_mbmi = *above_mbmi; + modify_neighbor_predictor_for_obmc(above_mbmi); + + for (j = 0; j < MAX_MB_PLANE; ++j) { + struct macroblockd_plane *const pd = &xd->plane[j]; + setup_pred_plane(&pd->dst, AOMMAX(a_bsize, BLOCK_8X8), tmp_buf[j], + tmp_width[j], tmp_height[j], tmp_stride[j], 0, i, NULL, + pd->subsampling_x, pd->subsampling_y); + } + for (ref = 0; ref < 1 + has_second_ref(above_mbmi); ++ref) { + const MV_REFERENCE_FRAME frame = above_mbmi->ref_frame[ref]; + const RefBuffer *const ref_buf = &cm->frame_refs[frame - LAST_FRAME]; + + xd->block_refs[ref] = ref_buf; + if ((!av1_is_valid_scale(&ref_buf->sf))) + aom_internal_error(xd->error_info, AOM_CODEC_UNSUP_BITSTREAM, + "Reference frame has invalid dimensions"); + av1_setup_pre_planes(xd, ref, ref_buf->buf, mi_row, mi_col + i, + &ref_buf->sf); + } + + xd->mb_to_left_edge = -(((mi_col + i) * MI_SIZE) * 8); + xd->mb_to_right_edge = + mb_to_right_edge_base + (xd->n8_w - i - mi_step) * 64; + mi_x = (mi_col + i) << MI_SIZE_LOG2; + mi_y = mi_row << MI_SIZE_LOG2; + + for (j = 0; j < MAX_MB_PLANE; ++j) { + const struct macroblockd_plane *pd = &xd->plane[j]; + bw = (mi_step * MI_SIZE) >> pd->subsampling_x; + bh = AOMMAX((num_4x4_blocks_high_lookup[bsize] * 2) >> pd->subsampling_y, + 4); + + if (skip_u4x4_pred_in_obmc(bsize, pd, 0)) continue; + build_inter_predictors(xd, j, mi_col_offset, mi_row_offset, 0, bw, bh, 0, + 0, bw, bh, +#if CONFIG_SUPERTX && CONFIG_EXT_INTER + 0, 0, +#endif // CONFIG_SUPERTX && CONFIG_EXT_INTER + mi_x, mi_y); + } + *above_mbmi = backup_mbmi; + } + xd->mb_to_left_edge = -((mi_col * MI_SIZE) * 8); + xd->mb_to_right_edge = mb_to_right_edge_base; + xd->mb_to_bottom_edge -= xd->n8_h * 32; +} + +void av1_build_prediction_by_left_preds(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col, + uint8_t *tmp_buf[MAX_MB_PLANE], + int tmp_width[MAX_MB_PLANE], + int tmp_height[MAX_MB_PLANE], + int tmp_stride[MAX_MB_PLANE]) { + const TileInfo *const tile = &xd->tile; + BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + int i, j, mi_step, ref; + const int ilimit = AOMMIN(xd->n8_h, cm->mi_rows - mi_row); + int mb_to_bottom_edge_base = xd->mb_to_bottom_edge; + const int neighbor_limit = max_neighbor_obmc[b_height_log2_lookup[bsize]]; + int neighbor_count = 0; + + if (mi_col == 0 || (mi_col - 1 < tile->mi_col_start)) return; + + xd->mb_to_right_edge += xd->n8_w * 32; + for (i = 0; i < ilimit; i += mi_step) { + int mi_row_offset = i; + int mi_col_offset = -1; + int mi_x, mi_y, bw, bh; + MODE_INFO *left_mi = xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]; + MB_MODE_INFO *left_mbmi = &left_mi->mbmi; + const BLOCK_SIZE l_bsize = left_mbmi->sb_type; + MB_MODE_INFO backup_mbmi; + + mi_step = AOMMIN(xd->n8_h, mi_size_high[l_bsize]); + + if (!is_neighbor_overlappable(left_mbmi)) continue; + + neighbor_count++; + if (neighbor_count > neighbor_limit) break; + + backup_mbmi = *left_mbmi; + modify_neighbor_predictor_for_obmc(left_mbmi); + + for (j = 0; j < MAX_MB_PLANE; ++j) { + struct macroblockd_plane *const pd = &xd->plane[j]; + setup_pred_plane(&pd->dst, AOMMAX(l_bsize, BLOCK_8X8), tmp_buf[j], + tmp_width[j], tmp_height[j], tmp_stride[j], i, 0, NULL, + pd->subsampling_x, pd->subsampling_y); + } + for (ref = 0; ref < 1 + has_second_ref(left_mbmi); ++ref) { + const MV_REFERENCE_FRAME frame = left_mbmi->ref_frame[ref]; + const RefBuffer *const ref_buf = &cm->frame_refs[frame - LAST_FRAME]; + + xd->block_refs[ref] = ref_buf; + if ((!av1_is_valid_scale(&ref_buf->sf))) + aom_internal_error(xd->error_info, AOM_CODEC_UNSUP_BITSTREAM, + "Reference frame has invalid dimensions"); + av1_setup_pre_planes(xd, ref, ref_buf->buf, mi_row + i, mi_col, + &ref_buf->sf); + } + + xd->mb_to_top_edge = -(((mi_row + i) * MI_SIZE) * 8); + xd->mb_to_bottom_edge = + mb_to_bottom_edge_base + (xd->n8_h - i - mi_step) * 64; + mi_x = mi_col << MI_SIZE_LOG2; + mi_y = (mi_row + i) << MI_SIZE_LOG2; + + for (j = 0; j < MAX_MB_PLANE; ++j) { + const struct macroblockd_plane *pd = &xd->plane[j]; + bw = AOMMAX((num_4x4_blocks_wide_lookup[bsize] * 2) >> pd->subsampling_x, + 4); + bh = (mi_step << MI_SIZE_LOG2) >> pd->subsampling_y; + + if (skip_u4x4_pred_in_obmc(bsize, pd, 1)) continue; + build_inter_predictors(xd, j, mi_col_offset, mi_row_offset, 0, bw, bh, 0, + 0, bw, bh, +#if CONFIG_SUPERTX && CONFIG_EXT_INTER + 0, 0, +#endif // CONFIG_SUPERTX && CONFIG_EXT_INTER + mi_x, mi_y); + } + *left_mbmi = backup_mbmi; + } + xd->mb_to_top_edge = -((mi_row * MI_SIZE) * 8); + xd->mb_to_bottom_edge = mb_to_bottom_edge_base; + xd->mb_to_right_edge -= xd->n8_w * 32; +} + +void av1_build_obmc_inter_predictors_sb(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col) { +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint8_t, tmp_buf1[2 * MAX_MB_PLANE * MAX_SB_SQUARE]); + DECLARE_ALIGNED(16, uint8_t, tmp_buf2[2 * MAX_MB_PLANE * MAX_SB_SQUARE]); +#else + DECLARE_ALIGNED(16, uint8_t, tmp_buf1[MAX_MB_PLANE * MAX_SB_SQUARE]); + DECLARE_ALIGNED(16, uint8_t, tmp_buf2[MAX_MB_PLANE * MAX_SB_SQUARE]); +#endif // CONFIG_HIGHBITDEPTH + uint8_t *dst_buf1[MAX_MB_PLANE], *dst_buf2[MAX_MB_PLANE]; + int dst_stride1[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_stride2[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_width1[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_width2[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_height1[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_height2[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + int len = sizeof(uint16_t); + dst_buf1[0] = CONVERT_TO_BYTEPTR(tmp_buf1); + dst_buf1[1] = CONVERT_TO_BYTEPTR(tmp_buf1 + MAX_SB_SQUARE * len); + dst_buf1[2] = CONVERT_TO_BYTEPTR(tmp_buf1 + MAX_SB_SQUARE * 2 * len); + dst_buf2[0] = CONVERT_TO_BYTEPTR(tmp_buf2); + dst_buf2[1] = CONVERT_TO_BYTEPTR(tmp_buf2 + MAX_SB_SQUARE * len); + dst_buf2[2] = CONVERT_TO_BYTEPTR(tmp_buf2 + MAX_SB_SQUARE * 2 * len); + } else { +#endif // CONFIG_HIGHBITDEPTH + dst_buf1[0] = tmp_buf1; + dst_buf1[1] = tmp_buf1 + MAX_SB_SQUARE; + dst_buf1[2] = tmp_buf1 + MAX_SB_SQUARE * 2; + dst_buf2[0] = tmp_buf2; + dst_buf2[1] = tmp_buf2 + MAX_SB_SQUARE; + dst_buf2[2] = tmp_buf2 + MAX_SB_SQUARE * 2; +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + av1_build_prediction_by_above_preds(cm, xd, mi_row, mi_col, dst_buf1, + dst_width1, dst_height1, dst_stride1); + av1_build_prediction_by_left_preds(cm, xd, mi_row, mi_col, dst_buf2, + dst_width2, dst_height2, dst_stride2); + av1_setup_dst_planes(xd->plane, xd->mi[0]->mbmi.sb_type, + get_frame_new_buffer(cm), mi_row, mi_col); + av1_build_obmc_inter_prediction(cm, xd, mi_row, mi_col, dst_buf1, dst_stride1, + dst_buf2, dst_stride2); +} + +#if CONFIG_NCOBMC +void av1_build_prediction_by_bottom_preds(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col, + uint8_t *tmp_buf[MAX_MB_PLANE], + int tmp_width[MAX_MB_PLANE], + int tmp_height[MAX_MB_PLANE], + int tmp_stride[MAX_MB_PLANE]) { + const TileInfo *const tile = &xd->tile; + BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + int i, j, mi_step, ref; + const int ilimit = AOMMIN(xd->n8_w, cm->mi_cols - mi_col); + int mb_to_right_edge_base = xd->mb_to_right_edge; + + if (mi_row + xd->n8_h >= tile->mi_row_end || + (mi_row + xd->n8_h) % MI_SIZE == 0 || (mi_row + xd->n8_h) >= cm->mi_rows) + return; + assert(bsize >= BLOCK_8X8); + + xd->mb_to_top_edge -= xd->n8_h * 32; + for (i = 0; i < ilimit; i += mi_step) { + int mi_row_offset = xd->n8_h; + int mi_col_offset = i; + int mi_x, mi_y, bw, bh; + MODE_INFO *mi = xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]; + MB_MODE_INFO *mbmi = &mi->mbmi; +#if CONFIG_EXT_INTER + MB_MODE_INFO backup_mbmi; +#endif // CONFIG_EXT_INTER + + mi_step = AOMMIN(xd->n8_w, mi_size_wide[mbmi->sb_type]); + + if (!is_neighbor_overlappable(mbmi)) continue; + +#if CONFIG_EXT_INTER + backup_mbmi = *mbmi; + modify_neighbor_predictor_for_obmc(mbmi); +#endif // CONFIG_EXT_INTER + + for (j = 0; j < MAX_MB_PLANE; ++j) { + struct macroblockd_plane *const pd = &xd->plane[j]; + setup_pred_plane(&pd->dst, AOMMAX(mbmi->sb_type, BLOCK_8X8), tmp_buf[j], + tmp_width[j], tmp_height[j], tmp_stride[j], + (xd->n8_h >> 1), i, NULL, pd->subsampling_x, + pd->subsampling_y); + } + for (ref = 0; ref < 1 + has_second_ref(mbmi); ++ref) { + const MV_REFERENCE_FRAME frame = mbmi->ref_frame[ref]; + const RefBuffer *const ref_buf = &cm->frame_refs[frame - LAST_FRAME]; + + xd->block_refs[ref] = ref_buf; + if ((!av1_is_valid_scale(&ref_buf->sf))) + aom_internal_error(xd->error_info, AOM_CODEC_UNSUP_BITSTREAM, + "Reference frame has invalid dimensions"); + av1_setup_pre_planes(xd, ref, ref_buf->buf, mi_row + (xd->n8_h >> 1), + mi_col + i, &ref_buf->sf); + } + + xd->mb_to_left_edge = -(((mi_col + i) * MI_SIZE) * 8); + xd->mb_to_right_edge = + mb_to_right_edge_base + (xd->n8_w - i - mi_step) * 64; + mi_x = (mi_col + i) << MI_SIZE_LOG2; + mi_y = (mi_row << MI_SIZE_LOG2) + xd->n8_h * 4; + + for (j = 0; j < MAX_MB_PLANE; ++j) { + const struct macroblockd_plane *pd = &xd->plane[j]; + bw = (mi_step << MI_SIZE_LOG2) >> pd->subsampling_x; + bh = (num_4x4_blocks_high_lookup[bsize] << 1) >> pd->subsampling_y; + + if (mbmi->sb_type < BLOCK_8X8 && !CONFIG_CB4X4) { + const PARTITION_TYPE bp = BLOCK_8X8 - mbmi->sb_type; + const int have_vsplit = bp != PARTITION_HORZ; + const int have_hsplit = bp != PARTITION_VERT; + const int num_4x4_w = 2 >> (!have_vsplit); + const int num_4x4_h = 2 >> (!have_hsplit); + const int pw = 8 >> (have_vsplit + pd->subsampling_x); + int x, y; + + for (y = 0; y < num_4x4_h; ++y) + for (x = 0; x < num_4x4_w; ++x) { + if ((bp == PARTITION_HORZ || bp == PARTITION_SPLIT) && y != 0) + continue; + + build_inter_predictors( + xd, j, mi_col_offset, mi_row_offset, y * 2 + x, bw, bh, + (4 * x) >> pd->subsampling_x, + xd->n8_h == 1 ? (4 >> pd->subsampling_y) : 0, pw, bh, +#if CONFIG_SUPERTX && CONFIG_EXT_INTER + 0, 0, +#endif // CONFIG_SUPERTX && CONFIG_EXT_INTER + mi_x, mi_y); + } + } else { + build_inter_predictors(xd, j, mi_col_offset, mi_row_offset, 0, bw, bh, + 0, xd->n8_h == 1 ? (4 >> pd->subsampling_y) : 0, + bw, bh, +#if CONFIG_SUPERTX && CONFIG_EXT_INTER + 0, 0, +#endif // CONFIG_SUPERTX && CONFIG_EXT_INTER + mi_x, mi_y); + } + } +#if CONFIG_EXT_INTER + *mbmi = backup_mbmi; +#endif // CONFIG_EXT_INTER + } + xd->mb_to_left_edge = -((mi_col * MI_SIZE) * 8); + xd->mb_to_right_edge = mb_to_right_edge_base; + xd->mb_to_top_edge += xd->n8_h * 32; +} + +void av1_build_prediction_by_right_preds(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col, + uint8_t *tmp_buf[MAX_MB_PLANE], + int tmp_width[MAX_MB_PLANE], + int tmp_height[MAX_MB_PLANE], + const int tmp_stride[MAX_MB_PLANE]) { + const TileInfo *const tile = &xd->tile; + BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + int i, j, mi_step, ref; + const int ilimit = AOMMIN(xd->n8_h, cm->mi_rows - mi_row); + int mb_to_bottom_edge_base = xd->mb_to_bottom_edge; + + if (mi_col + xd->n8_w >= tile->mi_col_end || + (mi_col + xd->n8_w) % MI_SIZE == 0 || (mi_col + xd->n8_w) >= cm->mi_cols) + return; + + xd->mb_to_left_edge -= xd->n8_w * 32; + for (i = 0; i < ilimit; i += mi_step) { + int mi_row_offset = i; + int mi_col_offset = xd->n8_w; + int mi_x, mi_y, bw, bh; + MODE_INFO *mi = xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]; + MB_MODE_INFO *mbmi = &mi->mbmi; +#if CONFIG_EXT_INTER + MB_MODE_INFO backup_mbmi; +#endif // CONFIG_EXT_INTER + + mi_step = AOMMIN(xd->n8_h, mi_size_high[mbmi->sb_type]); + + if (!is_neighbor_overlappable(mbmi)) continue; + +#if CONFIG_EXT_INTER + backup_mbmi = *mbmi; + modify_neighbor_predictor_for_obmc(mbmi); +#endif // CONFIG_EXT_INTER + + for (j = 0; j < MAX_MB_PLANE; ++j) { + struct macroblockd_plane *const pd = &xd->plane[j]; + setup_pred_plane(&pd->dst, AOMMAX(mbmi->sb_type, BLOCK_8X8), tmp_buf[j], + tmp_width[j], tmp_height[j], tmp_stride[j], i, + xd->n8_w >> 1, NULL, pd->subsampling_x, + pd->subsampling_y); + } + for (ref = 0; ref < 1 + has_second_ref(mbmi); ++ref) { + const MV_REFERENCE_FRAME frame = mbmi->ref_frame[ref]; + const RefBuffer *const ref_buf = &cm->frame_refs[frame - LAST_FRAME]; + + xd->block_refs[ref] = ref_buf; + if ((!av1_is_valid_scale(&ref_buf->sf))) + aom_internal_error(xd->error_info, AOM_CODEC_UNSUP_BITSTREAM, + "Reference frame has invalid dimensions"); + av1_setup_pre_planes(xd, ref, ref_buf->buf, mi_row + i, + mi_col + (xd->n8_w >> 1), &ref_buf->sf); + } + + xd->mb_to_top_edge = -(((mi_row + i) * MI_SIZE) * 8); + xd->mb_to_bottom_edge = + mb_to_bottom_edge_base + (xd->n8_h - i - mi_step) * 64; + mi_x = (mi_col << MI_SIZE_LOG2) + xd->n8_w * 4; + mi_y = (mi_row + i) << MI_SIZE_LOG2; + + for (j = 0; j < MAX_MB_PLANE; ++j) { + const struct macroblockd_plane *pd = &xd->plane[j]; + bw = (num_4x4_blocks_wide_lookup[bsize] << 1) >> pd->subsampling_x; + bh = (mi_step << MI_SIZE_LOG2) >> pd->subsampling_y; + + if (mbmi->sb_type < BLOCK_8X8 && !CONFIG_CB4X4) { + const PARTITION_TYPE bp = BLOCK_8X8 - mbmi->sb_type; + const int have_vsplit = bp != PARTITION_HORZ; + const int have_hsplit = bp != PARTITION_VERT; + const int num_4x4_w = 2 >> (!have_vsplit); + const int num_4x4_h = 2 >> (!have_hsplit); + const int ph = 8 >> (have_hsplit + pd->subsampling_y); + int x, y; + + for (y = 0; y < num_4x4_h; ++y) + for (x = 0; x < num_4x4_w; ++x) { + if ((bp == PARTITION_VERT || bp == PARTITION_SPLIT) && x != 0) + continue; + + build_inter_predictors(xd, j, mi_col_offset, mi_row_offset, + y * 2 + x, bw, bh, + xd->n8_w == 1 ? 4 >> pd->subsampling_x : 0, + (4 * y) >> pd->subsampling_y, bw, ph, +#if CONFIG_SUPERTX && CONFIG_EXT_INTER + 0, 0, +#endif // CONFIG_SUPERTX && CONFIG_EXT_INTER + mi_x, mi_y); + } + } else { + build_inter_predictors(xd, j, mi_col_offset, mi_row_offset, 0, bw, bh, + xd->n8_w == 1 ? 4 >> pd->subsampling_x : 0, 0, + bw, bh, +#if CONFIG_SUPERTX && CONFIG_EXT_INTER + 0, 0, +#endif // CONFIG_SUPERTX && CONFIG_EXT_INTER + mi_x, mi_y); + } + } +#if CONFIG_EXT_INTER + *mbmi = backup_mbmi; +#endif // CONFIG_EXT_INTER + } + xd->mb_to_top_edge = -((mi_row * MI_SIZE) * 8); + xd->mb_to_bottom_edge = mb_to_bottom_edge_base; + xd->mb_to_left_edge += xd->n8_w * 32; +} + +// This function combines motion compensated predictions that is generated by +// bottom/right neighboring blocks' inter predictors with prediction in dst +// buffer. +void av1_merge_dst_bottom_right_preds(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col, + uint8_t *bottom[MAX_MB_PLANE], + const int bottom_stride[MAX_MB_PLANE], + uint8_t *right[MAX_MB_PLANE], + const int right_stride[MAX_MB_PLANE]) { + const TileInfo *const tile = &xd->tile; + BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + int plane, i, mi_step; + const int bottom_available = mi_row + xd->n8_h < tile->mi_row_end && + (mi_row + xd->n8_h) % MI_SIZE != 0 && + (mi_row + xd->n8_h) < cm->mi_rows; +#if CONFIG_HIGHBITDEPTH + int is_hbd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? 1 : 0; +#endif // CONFIG_HIGHBITDEPTH + + // handle bottom row + for (i = 0; bottom_available && i < AOMMIN(xd->n8_w, cm->mi_cols - mi_col); + i += mi_step) { + int mi_row_offset = xd->n8_h; + int mi_col_offset = i; + MODE_INFO *mi = xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]; + MB_MODE_INFO *mbmi = &mi->mbmi; + int overlap; + + mi_step = AOMMIN(xd->n8_w, mi_size_wide[mbmi->sb_type]); + + if (!is_neighbor_overlappable(mbmi)) continue; + + overlap = num_4x4_blocks_high_lookup[bsize] << 1; + + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + const struct macroblockd_plane *pd = &xd->plane[plane]; + const int bw = (mi_step * MI_SIZE) >> pd->subsampling_x; + const int bh = overlap >> pd->subsampling_y; + const int dst_stride = pd->dst.stride; + uint8_t *dst = + &pd->dst.buf[((i * MI_SIZE) >> pd->subsampling_x) + + (((xd->n8_h * MI_SIZE - overlap) * dst_stride) >> + pd->subsampling_y)]; + const int tmp_stride = bottom_stride[plane]; + const uint8_t *const tmp = + &bottom[plane][((i * MI_SIZE) >> pd->subsampling_x) + + (((xd->n8_h * MI_SIZE - overlap) * tmp_stride) >> + pd->subsampling_y)]; + const uint8_t *const mask = av1_get_obmc_mask_flipped(bh); + +#if CONFIG_HIGHBITDEPTH + if (is_hbd) + aom_highbd_blend_a64_vmask(dst, dst_stride, dst, dst_stride, tmp, + tmp_stride, mask, bh, bw, xd->bd); + else +#endif // CONFIG_HIGHBITDEPTH + aom_blend_a64_vmask(dst, dst_stride, dst, dst_stride, tmp, tmp_stride, + mask, bh, bw); + } + } // each mi in the bottom row + + // handle right column + if (mi_col + xd->n8_w >= tile->mi_col_end || + (mi_col + xd->n8_w) % MI_SIZE == 0 || (mi_col + xd->n8_w) >= cm->mi_cols) + return; + + for (i = 0; i < AOMMIN(xd->n8_h, cm->mi_rows - mi_row); i += mi_step) { + int mi_row_offset = i; + int mi_col_offset = xd->n8_w; + int overlap; + MODE_INFO *mi = xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]; + MB_MODE_INFO *mbmi = &mi->mbmi; + + mi_step = AOMMIN(xd->n8_h, mi_size_high[mbmi->sb_type]); + + if (!is_neighbor_overlappable(mbmi)) continue; + + overlap = num_4x4_blocks_wide_lookup[bsize] << 1; + + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + const struct macroblockd_plane *pd = &xd->plane[plane]; + const int bw = overlap >> pd->subsampling_x; + const int bh = (mi_step * MI_SIZE) >> pd->subsampling_y; + const int dst_stride = pd->dst.stride; + uint8_t *dst = + &pd->dst.buf[((i * MI_SIZE * dst_stride) >> pd->subsampling_y) + + ((xd->n8_w * MI_SIZE - overlap) >> pd->subsampling_x)]; + const int tmp_stride = right_stride[plane]; + const uint8_t *const tmp = + &right[plane][((i * MI_SIZE * tmp_stride) >> pd->subsampling_y) + + ((xd->n8_w * MI_SIZE - overlap) >> pd->subsampling_x)]; + const uint8_t *const mask = av1_get_obmc_mask_flipped(bw); + +#if CONFIG_HIGHBITDEPTH + if (is_hbd) + aom_highbd_blend_a64_hmask(dst, dst_stride, dst, dst_stride, tmp, + tmp_stride, mask, bh, bw, xd->bd); + else +#endif // CONFIG_HIGHBITDEPTH + aom_blend_a64_hmask(dst, dst_stride, dst, dst_stride, tmp, tmp_stride, + mask, bh, bw); + } + } // each mi in the right column +} + +// This function generates 4 sided obmc. (1) Prediction blocks generated by +// bottom and right motion vectors are calculated. (2) Combine them with the +// original prediction block (which should be pre-stored in xd->plane[].dst.buf +// before calling this function). The results is updated in xd->plane[].dst.buf +// (3) Call causal obmc prediction function, which will generate left and above +// preds, and then merge them and xd->plane[].dst.buf. +void av1_build_ncobmc_inter_predictors_sb(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col) { +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint8_t, tmp_buf1[2 * MAX_MB_PLANE * MAX_SB_SQUARE]); + DECLARE_ALIGNED(16, uint8_t, tmp_buf2[2 * MAX_MB_PLANE * MAX_SB_SQUARE]); +#else + DECLARE_ALIGNED(16, uint8_t, tmp_buf1[MAX_MB_PLANE * MAX_SB_SQUARE]); + DECLARE_ALIGNED(16, uint8_t, tmp_buf2[MAX_MB_PLANE * MAX_SB_SQUARE]); +#endif // CONFIG_HIGHBITDEPTH + uint8_t *dst_buf1[MAX_MB_PLANE], *dst_buf2[MAX_MB_PLANE]; + int dst_stride1[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_stride2[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_width1[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_width2[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_height1[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_height2[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + int len = sizeof(uint16_t); + dst_buf1[0] = CONVERT_TO_BYTEPTR(tmp_buf1); + dst_buf1[1] = CONVERT_TO_BYTEPTR(tmp_buf1 + MAX_SB_SQUARE * len); + dst_buf1[2] = CONVERT_TO_BYTEPTR(tmp_buf1 + MAX_SB_SQUARE * 2 * len); + dst_buf2[0] = CONVERT_TO_BYTEPTR(tmp_buf2); + dst_buf2[1] = CONVERT_TO_BYTEPTR(tmp_buf2 + MAX_SB_SQUARE * len); + dst_buf2[2] = CONVERT_TO_BYTEPTR(tmp_buf2 + MAX_SB_SQUARE * 2 * len); + } else { +#endif // CONFIG_HIGHBITDEPTH + dst_buf1[0] = tmp_buf1; + dst_buf1[1] = tmp_buf1 + MAX_SB_SQUARE; + dst_buf1[2] = tmp_buf1 + MAX_SB_SQUARE * 2; + dst_buf2[0] = tmp_buf2; + dst_buf2[1] = tmp_buf2 + MAX_SB_SQUARE; + dst_buf2[2] = tmp_buf2 + MAX_SB_SQUARE * 2; +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + + const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + av1_build_prediction_by_bottom_preds(cm, xd, mi_row, mi_col, dst_buf1, + dst_width1, dst_height1, dst_stride1); + av1_build_prediction_by_right_preds(cm, xd, mi_row, mi_col, dst_buf2, + dst_width2, dst_height2, dst_stride2); + av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, + mi_col); + av1_merge_dst_bottom_right_preds(cm, xd, mi_row, mi_col, dst_buf1, + dst_stride1, dst_buf2, dst_stride2); + av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, + mi_col); + av1_build_obmc_inter_predictors_sb(cm, xd, mi_row, mi_col); + av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, + mi_col); +} +#endif // CONFIG_NCOBMC +#endif // CONFIG_MOTION_VAR + +#if CONFIG_EXT_INTER +/* clang-format off */ +#if CONFIG_EXT_PARTITION +static const int ii_weights1d[MAX_SB_SIZE] = { + 26, 25, 24, 24, 23, 23, 22, 22, 21, 21, 20, 20, 19, 19, 18, 18, 17, 17, 17, + 16, 16, 16, 15, 15, 15, 15, 14, 14, 14, 13, 13, 13, 13, 13, 12, 12, 12, 12, + 12, 11, 11, 11, 11, 11, 11, 10, 10, 10, 10, 10, 10, 10, 10, 10, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 +}; +static int ii_size_scales[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 32, 32, 32, +#endif + 32, 16, 16, 16, 8, 8, 8, 4, + 4, 4, 2, 2, 2, 1, 1, 1, +}; +#else +static const int ii_weights1d[MAX_SB_SIZE] = { + 26, 25, 24, 24, 23, 23, 22, 22, 21, 21, 20, 20, 19, 19, 18, 18, + 17, 17, 17, 16, 16, 16, 15, 15, 15, 15, 14, 14, 14, 13, 13, 13, + 13, 13, 12, 12, 12, 12, 12, 11, 11, 11, 11, 11, 11, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 +}; +static int ii_size_scales[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 16, 16, 16, +#endif + 16, 8, 8, 8, 4, 4, 4, + 2, 2, 2, 1, 1, 1, +}; +/* clang-format on */ +#endif // CONFIG_EXT_PARTITION + +static void combine_interintra(INTERINTRA_MODE mode, int use_wedge_interintra, + int wedge_index, int wedge_sign, + BLOCK_SIZE bsize, BLOCK_SIZE plane_bsize, + uint8_t *comppred, int compstride, + const uint8_t *interpred, int interstride, + const uint8_t *intrapred, int intrastride) { + const int bw = block_size_wide[plane_bsize]; + const int bh = block_size_high[plane_bsize]; + const int size_scale = ii_size_scales[plane_bsize]; + int i, j; + + if (use_wedge_interintra) { + if (is_interintra_wedge_used(bsize)) { + const uint8_t *mask = + av1_get_contiguous_soft_mask(wedge_index, wedge_sign, bsize); + const int subw = 2 * num_4x4_blocks_wide_lookup[bsize] == bw; + const int subh = 2 * num_4x4_blocks_high_lookup[bsize] == bh; + aom_blend_a64_mask(comppred, compstride, intrapred, intrastride, + interpred, interstride, mask, block_size_wide[bsize], + bh, bw, subh, subw); + } + return; + } + + switch (mode) { + case II_V_PRED: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + int scale = ii_weights1d[i * size_scale]; + comppred[i * compstride + j] = + AOM_BLEND_A64(scale, intrapred[i * intrastride + j], + interpred[i * interstride + j]); + } + } + break; + + case II_H_PRED: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + int scale = ii_weights1d[j * size_scale]; + comppred[i * compstride + j] = + AOM_BLEND_A64(scale, intrapred[i * intrastride + j], + interpred[i * interstride + j]); + } + } + break; + + case II_D63_PRED: + case II_D117_PRED: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + int scale = (ii_weights1d[i * size_scale] * 3 + + ii_weights1d[j * size_scale]) >> + 2; + comppred[i * compstride + j] = + AOM_BLEND_A64(scale, intrapred[i * intrastride + j], + interpred[i * interstride + j]); + } + } + break; + + case II_D207_PRED: + case II_D153_PRED: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + int scale = (ii_weights1d[j * size_scale] * 3 + + ii_weights1d[i * size_scale]) >> + 2; + comppred[i * compstride + j] = + AOM_BLEND_A64(scale, intrapred[i * intrastride + j], + interpred[i * interstride + j]); + } + } + break; + + case II_D135_PRED: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + int scale = ii_weights1d[(i < j ? i : j) * size_scale]; + comppred[i * compstride + j] = + AOM_BLEND_A64(scale, intrapred[i * intrastride + j], + interpred[i * interstride + j]); + } + } + break; + + case II_D45_PRED: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + int scale = + (ii_weights1d[i * size_scale] + ii_weights1d[j * size_scale]) >> + 1; + comppred[i * compstride + j] = + AOM_BLEND_A64(scale, intrapred[i * intrastride + j], + interpred[i * interstride + j]); + } + } + break; + + case II_TM_PRED: + case II_DC_PRED: + default: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + comppred[i * compstride + j] = AOM_BLEND_AVG( + intrapred[i * intrastride + j], interpred[i * interstride + j]); + } + } + break; + } +} + +#if CONFIG_HIGHBITDEPTH +static void combine_interintra_highbd( + INTERINTRA_MODE mode, int use_wedge_interintra, int wedge_index, + int wedge_sign, BLOCK_SIZE bsize, BLOCK_SIZE plane_bsize, + uint8_t *comppred8, int compstride, const uint8_t *interpred8, + int interstride, const uint8_t *intrapred8, int intrastride, int bd) { + const int bw = block_size_wide[plane_bsize]; + const int bh = block_size_high[plane_bsize]; + const int size_scale = ii_size_scales[plane_bsize]; + int i, j; + + uint16_t *comppred = CONVERT_TO_SHORTPTR(comppred8); + const uint16_t *interpred = CONVERT_TO_SHORTPTR(interpred8); + const uint16_t *intrapred = CONVERT_TO_SHORTPTR(intrapred8); + + if (use_wedge_interintra) { + if (is_interintra_wedge_used(bsize)) { + const uint8_t *mask = + av1_get_contiguous_soft_mask(wedge_index, wedge_sign, bsize); + const int subh = 2 * num_4x4_blocks_high_lookup[bsize] == bh; + const int subw = 2 * num_4x4_blocks_wide_lookup[bsize] == bw; + aom_highbd_blend_a64_mask(comppred8, compstride, intrapred8, intrastride, + interpred8, interstride, mask, bw, bh, bw, subh, + subw, bd); + } + return; + } + + switch (mode) { + case II_V_PRED: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + int scale = ii_weights1d[i * size_scale]; + comppred[i * compstride + j] = + AOM_BLEND_A64(scale, intrapred[i * intrastride + j], + interpred[i * interstride + j]); + } + } + break; + + case II_H_PRED: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + int scale = ii_weights1d[j * size_scale]; + comppred[i * compstride + j] = + AOM_BLEND_A64(scale, intrapred[i * intrastride + j], + interpred[i * interstride + j]); + } + } + break; + + case II_D63_PRED: + case II_D117_PRED: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + int scale = (ii_weights1d[i * size_scale] * 3 + + ii_weights1d[j * size_scale]) >> + 2; + comppred[i * compstride + j] = + AOM_BLEND_A64(scale, intrapred[i * intrastride + j], + interpred[i * interstride + j]); + } + } + break; + + case II_D207_PRED: + case II_D153_PRED: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + int scale = (ii_weights1d[j * size_scale] * 3 + + ii_weights1d[i * size_scale]) >> + 2; + comppred[i * compstride + j] = + AOM_BLEND_A64(scale, intrapred[i * intrastride + j], + interpred[i * interstride + j]); + } + } + break; + + case II_D135_PRED: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + int scale = ii_weights1d[(i < j ? i : j) * size_scale]; + comppred[i * compstride + j] = + AOM_BLEND_A64(scale, intrapred[i * intrastride + j], + interpred[i * interstride + j]); + } + } + break; + + case II_D45_PRED: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + int scale = + (ii_weights1d[i * size_scale] + ii_weights1d[j * size_scale]) >> + 1; + comppred[i * compstride + j] = + AOM_BLEND_A64(scale, intrapred[i * intrastride + j], + interpred[i * interstride + j]); + } + } + break; + + case II_TM_PRED: + case II_DC_PRED: + default: + for (i = 0; i < bh; ++i) { + for (j = 0; j < bw; ++j) { + comppred[i * compstride + j] = AOM_BLEND_AVG( + interpred[i * interstride + j], intrapred[i * intrastride + j]); + } + } + break; + } +} +#endif // CONFIG_HIGHBITDEPTH + +void av1_build_intra_predictors_for_interintra(MACROBLOCKD *xd, + BLOCK_SIZE bsize, int plane, + BUFFER_SET *ctx, uint8_t *dst, + int dst_stride) { + struct macroblockd_plane *const pd = &xd->plane[plane]; + BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, &xd->plane[plane]); + PREDICTION_MODE mode = + interintra_to_intra_mode[xd->mi[0]->mbmi.interintra_mode]; + + av1_predict_intra_block(xd, pd->width, pd->height, plane_bsize, mode, + ctx->plane[plane], ctx->stride[plane], dst, + dst_stride, 0, 0, plane); +} + +void av1_combine_interintra(MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane, + const uint8_t *inter_pred, int inter_stride, + const uint8_t *intra_pred, int intra_stride) { + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, &xd->plane[plane]); +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + combine_interintra_highbd( + xd->mi[0]->mbmi.interintra_mode, xd->mi[0]->mbmi.use_wedge_interintra, + xd->mi[0]->mbmi.interintra_wedge_index, + xd->mi[0]->mbmi.interintra_wedge_sign, bsize, plane_bsize, + xd->plane[plane].dst.buf, xd->plane[plane].dst.stride, inter_pred, + inter_stride, intra_pred, intra_stride, xd->bd); + return; + } +#endif // CONFIG_HIGHBITDEPTH + combine_interintra(xd->mi[0]->mbmi.interintra_mode, + xd->mi[0]->mbmi.use_wedge_interintra, + xd->mi[0]->mbmi.interintra_wedge_index, + xd->mi[0]->mbmi.interintra_wedge_sign, bsize, plane_bsize, + xd->plane[plane].dst.buf, xd->plane[plane].dst.stride, + inter_pred, inter_stride, intra_pred, intra_stride); +} + +void av1_build_interintra_predictors_sby(MACROBLOCKD *xd, uint8_t *ypred, + int ystride, BUFFER_SET *ctx, + BLOCK_SIZE bsize) { +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + DECLARE_ALIGNED(16, uint16_t, intrapredictor[MAX_SB_SQUARE]); + av1_build_intra_predictors_for_interintra( + xd, bsize, 0, ctx, CONVERT_TO_BYTEPTR(intrapredictor), MAX_SB_SIZE); + av1_combine_interintra(xd, bsize, 0, ypred, ystride, + CONVERT_TO_BYTEPTR(intrapredictor), MAX_SB_SIZE); + return; + } +#endif // CONFIG_HIGHBITDEPTH + { + DECLARE_ALIGNED(16, uint8_t, intrapredictor[MAX_SB_SQUARE]); + av1_build_intra_predictors_for_interintra(xd, bsize, 0, ctx, intrapredictor, + MAX_SB_SIZE); + av1_combine_interintra(xd, bsize, 0, ypred, ystride, intrapredictor, + MAX_SB_SIZE); + } +} + +void av1_build_interintra_predictors_sbc(MACROBLOCKD *xd, uint8_t *upred, + int ustride, BUFFER_SET *ctx, + int plane, BLOCK_SIZE bsize) { +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + DECLARE_ALIGNED(16, uint16_t, uintrapredictor[MAX_SB_SQUARE]); + av1_build_intra_predictors_for_interintra( + xd, bsize, plane, ctx, CONVERT_TO_BYTEPTR(uintrapredictor), + MAX_SB_SIZE); + av1_combine_interintra(xd, bsize, plane, upred, ustride, + CONVERT_TO_BYTEPTR(uintrapredictor), MAX_SB_SIZE); + return; + } +#endif // CONFIG_HIGHBITDEPTH + { + DECLARE_ALIGNED(16, uint8_t, uintrapredictor[MAX_SB_SQUARE]); + av1_build_intra_predictors_for_interintra(xd, bsize, plane, ctx, + uintrapredictor, MAX_SB_SIZE); + av1_combine_interintra(xd, bsize, plane, upred, ustride, uintrapredictor, + MAX_SB_SIZE); + } +} + +void av1_build_interintra_predictors_sbuv(MACROBLOCKD *xd, uint8_t *upred, + uint8_t *vpred, int ustride, + int vstride, BUFFER_SET *ctx, + BLOCK_SIZE bsize) { + av1_build_interintra_predictors_sbc(xd, upred, ustride, ctx, 1, bsize); + av1_build_interintra_predictors_sbc(xd, vpred, vstride, ctx, 2, bsize); +} + +void av1_build_interintra_predictors(MACROBLOCKD *xd, uint8_t *ypred, + uint8_t *upred, uint8_t *vpred, + int ystride, int ustride, int vstride, + BUFFER_SET *ctx, BLOCK_SIZE bsize) { + av1_build_interintra_predictors_sby(xd, ypred, ystride, ctx, bsize); + av1_build_interintra_predictors_sbuv(xd, upred, vpred, ustride, vstride, ctx, + bsize); +} + +// Builds the inter-predictor for the single ref case +// for use in the encoder to search the wedges efficiently. +static void build_inter_predictors_single_buf(MACROBLOCKD *xd, int plane, + int block, int bw, int bh, int x, + int y, int w, int h, int mi_x, + int mi_y, int ref, + uint8_t *const ext_dst, + int ext_dst_stride) { + struct macroblockd_plane *const pd = &xd->plane[plane]; + const MODE_INFO *mi = xd->mi[0]; + + const struct scale_factors *const sf = &xd->block_refs[ref]->sf; + struct buf_2d *const pre_buf = &pd->pre[ref]; +#if CONFIG_HIGHBITDEPTH + uint8_t *const dst = + (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH ? CONVERT_TO_BYTEPTR(ext_dst) + : ext_dst) + + ext_dst_stride * y + x; +#else + uint8_t *const dst = ext_dst + ext_dst_stride * y + x; +#endif + const MV mv = mi->mbmi.sb_type < BLOCK_8X8 + ? average_split_mvs(pd, mi, ref, block) + : mi->mbmi.mv[ref].as_mv; + + // TODO(jkoleszar): This clamping is done in the incorrect place for the + // scaling case. It needs to be done on the scaled MV, not the pre-scaling + // MV. Note however that it performs the subsampling aware scaling so + // that the result is always q4. + // mv_precision precision is MV_PRECISION_Q4. + const MV mv_q4 = clamp_mv_to_umv_border_sb(xd, &mv, bw, bh, pd->subsampling_x, + pd->subsampling_y); + + uint8_t *pre; + MV32 scaled_mv; + int xs, ys, subpel_x, subpel_y; + const int is_scaled = av1_is_scaled(sf); + ConvolveParams conv_params = get_conv_params(0, plane); +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + WarpTypesAllowed warp_types; +#if CONFIG_GLOBAL_MOTION + WarpedMotionParams *const wm = &xd->global_motion[mi->mbmi.ref_frame[ref]]; + warp_types.global_warp_allowed = is_global_mv_block(mi, block, wm->wmtype); +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_WARPED_MOTION + warp_types.local_warp_allowed = mi->mbmi.motion_mode == WARPED_CAUSAL; +#endif // CONFIG_WARPED_MOTION +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + + if (is_scaled) { + pre = pre_buf->buf + scaled_buffer_offset(x, y, pre_buf->stride, sf); + scaled_mv = av1_scale_mv(&mv_q4, mi_x + x, mi_y + y, sf); + xs = sf->x_step_q4; + ys = sf->y_step_q4; + } else { + pre = pre_buf->buf + (y * pre_buf->stride + x); + scaled_mv.row = mv_q4.row; + scaled_mv.col = mv_q4.col; + xs = ys = 16; + } + + subpel_x = scaled_mv.col & SUBPEL_MASK; + subpel_y = scaled_mv.row & SUBPEL_MASK; + pre += (scaled_mv.row >> SUBPEL_BITS) * pre_buf->stride + + (scaled_mv.col >> SUBPEL_BITS); + + av1_make_inter_predictor(pre, pre_buf->stride, dst, ext_dst_stride, subpel_x, + subpel_y, sf, w, h, &conv_params, + mi->mbmi.interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, (mi_x >> pd->subsampling_x) + x, + (mi_y >> pd->subsampling_y) + y, plane, ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#if CONFIG_MOTION_VAR + 0, 0, +#endif + xs, ys, xd); +} + +void av1_build_inter_predictors_for_planes_single_buf( + MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane_from, int plane_to, int mi_row, + int mi_col, int ref, uint8_t *ext_dst[3], int ext_dst_stride[3]) { + int plane; + const int mi_x = mi_col * MI_SIZE; + const int mi_y = mi_row * MI_SIZE; + for (plane = plane_from; plane <= plane_to; ++plane) { + const BLOCK_SIZE plane_bsize = + get_plane_block_size(bsize, &xd->plane[plane]); + const int num_4x4_w = num_4x4_blocks_wide_lookup[plane_bsize]; + const int num_4x4_h = num_4x4_blocks_high_lookup[plane_bsize]; + const int bw = block_size_wide[plane_bsize]; + const int bh = block_size_high[plane_bsize]; + + if (xd->mi[0]->mbmi.sb_type < BLOCK_8X8 && !CONFIG_CB4X4) { + int x, y; + assert(bsize == BLOCK_8X8); + for (y = 0; y < num_4x4_h; ++y) + for (x = 0; x < num_4x4_w; ++x) + build_inter_predictors_single_buf( + xd, plane, y * 2 + x, bw, bh, 4 * x, 4 * y, 4, 4, mi_x, mi_y, ref, + ext_dst[plane], ext_dst_stride[plane]); + } else { + build_inter_predictors_single_buf(xd, plane, 0, bw, bh, 0, 0, bw, bh, + mi_x, mi_y, ref, ext_dst[plane], + ext_dst_stride[plane]); + } + } +} + +static void build_wedge_inter_predictor_from_buf( + MACROBLOCKD *xd, int plane, int x, int y, int w, int h, +#if CONFIG_SUPERTX + int wedge_offset_x, int wedge_offset_y, +#endif // CONFIG_SUPERTX + uint8_t *ext_dst0, int ext_dst_stride0, uint8_t *ext_dst1, + int ext_dst_stride1) { + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const int is_compound = has_second_ref(mbmi); + MACROBLOCKD_PLANE *const pd = &xd->plane[plane]; + struct buf_2d *const dst_buf = &pd->dst; + uint8_t *const dst = dst_buf->buf + dst_buf->stride * y + x; + const INTERINTER_COMPOUND_DATA comp_data = { +#if CONFIG_WEDGE + mbmi->wedge_index, + mbmi->wedge_sign, +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + mbmi->mask_type, + xd->seg_mask, +#endif // CONFIG_COMPOUND_SEGMENT + mbmi->interinter_compound_type + }; + + if (is_compound && is_masked_compound_type(mbmi->interinter_compound_type)) { +#if CONFIG_COMPOUND_SEGMENT + if (!plane && comp_data.interinter_compound_type == COMPOUND_SEG) { +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + build_compound_seg_mask_highbd( + comp_data.seg_mask, comp_data.mask_type, + CONVERT_TO_BYTEPTR(ext_dst0), ext_dst_stride0, + CONVERT_TO_BYTEPTR(ext_dst1), ext_dst_stride1, mbmi->sb_type, h, w, + xd->bd); + else +#endif // CONFIG_HIGHBITDEPTH + build_compound_seg_mask(comp_data.seg_mask, comp_data.mask_type, + ext_dst0, ext_dst_stride0, ext_dst1, + ext_dst_stride1, mbmi->sb_type, h, w); + } +#endif // CONFIG_COMPOUND_SEGMENT + +#if CONFIG_SUPERTX +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + build_masked_compound_wedge_extend_highbd( + dst, dst_buf->stride, CONVERT_TO_BYTEPTR(ext_dst0), ext_dst_stride0, + CONVERT_TO_BYTEPTR(ext_dst1), ext_dst_stride1, &comp_data, + mbmi->sb_type, wedge_offset_x, wedge_offset_y, h, w, xd->bd); + else +#endif // CONFIG_HIGHBITDEPTH + build_masked_compound_wedge_extend( + dst, dst_buf->stride, ext_dst0, ext_dst_stride0, ext_dst1, + ext_dst_stride1, &comp_data, mbmi->sb_type, wedge_offset_x, + wedge_offset_y, h, w); +#else +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + build_masked_compound_highbd( + dst, dst_buf->stride, CONVERT_TO_BYTEPTR(ext_dst0), ext_dst_stride0, + CONVERT_TO_BYTEPTR(ext_dst1), ext_dst_stride1, &comp_data, + mbmi->sb_type, h, w, xd->bd); + else +#endif // CONFIG_HIGHBITDEPTH + build_masked_compound(dst, dst_buf->stride, ext_dst0, ext_dst_stride0, + ext_dst1, ext_dst_stride1, &comp_data, + mbmi->sb_type, h, w); +#endif // CONFIG_SUPERTX + } else { +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + aom_highbd_convolve_copy(CONVERT_TO_BYTEPTR(ext_dst0), ext_dst_stride0, + dst, dst_buf->stride, NULL, 0, NULL, 0, w, h, + xd->bd); + else +#endif // CONFIG_HIGHBITDEPTH + aom_convolve_copy(ext_dst0, ext_dst_stride0, dst, dst_buf->stride, NULL, + 0, NULL, 0, w, h); + } +} + +void av1_build_wedge_inter_predictor_from_buf( + MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane_from, int plane_to, +#if CONFIG_SUPERTX + int wedge_offset_x, int wedge_offset_y, +#endif // CONFIG_SUPERTX + uint8_t *ext_dst0[3], int ext_dst_stride0[3], uint8_t *ext_dst1[3], + int ext_dst_stride1[3]) { + int plane; + for (plane = plane_from; plane <= plane_to; ++plane) { + const BLOCK_SIZE plane_bsize = + get_plane_block_size(bsize, &xd->plane[plane]); + const int num_4x4_w = num_4x4_blocks_wide_lookup[plane_bsize]; + const int num_4x4_h = num_4x4_blocks_high_lookup[plane_bsize]; + + if (xd->mi[0]->mbmi.sb_type < BLOCK_8X8 && !CONFIG_CB4X4) { + int x, y; + assert(bsize == BLOCK_8X8); + for (y = 0; y < num_4x4_h; ++y) + for (x = 0; x < num_4x4_w; ++x) + build_wedge_inter_predictor_from_buf( + xd, plane, 4 * x, 4 * y, 4, 4, +#if CONFIG_SUPERTX + wedge_offset_x, wedge_offset_y, +#endif // CONFIG_SUPERTX + ext_dst0[plane], ext_dst_stride0[plane], ext_dst1[plane], + ext_dst_stride1[plane]); + } else { + const int bw = block_size_wide[plane_bsize]; + const int bh = block_size_high[plane_bsize]; + build_wedge_inter_predictor_from_buf( + xd, plane, 0, 0, bw, bh, +#if CONFIG_SUPERTX + wedge_offset_x, wedge_offset_y, +#endif // CONFIG_SUPERTX + ext_dst0[plane], ext_dst_stride0[plane], ext_dst1[plane], + ext_dst_stride1[plane]); + } + } +} +#endif // CONFIG_EXT_INTER diff --git a/third_party/aom/av1/common/reconinter.h b/third_party/aom/av1/common/reconinter.h new file mode 100644 index 0000000000..10933a751c --- /dev/null +++ b/third_party/aom/av1/common/reconinter.h @@ -0,0 +1,828 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_RECONINTER_H_ +#define AV1_COMMON_RECONINTER_H_ + +#include "av1/common/filter.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/convolve.h" +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#include "av1/common/warped_motion.h" +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#include "aom/aom_integer.h" + +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION +#define WARP_WM_NEIGHBORS_WITH_OBMC 0 +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + +#if CONFIG_MOTION_VAR && CONFIG_GLOBAL_MOTION +#define WARP_GM_NEIGHBORS_WITH_OBMC 0 +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + +#ifdef __cplusplus +extern "C" { +#endif + +static INLINE void inter_predictor(const uint8_t *src, int src_stride, + uint8_t *dst, int dst_stride, + const int subpel_x, const int subpel_y, + const struct scale_factors *sf, int w, int h, + ConvolveParams *conv_params, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif + int xs, int ys) { +#if CONFIG_DUAL_FILTER + InterpFilter filter_x = av1_get_plane_interp_filter( + interp_filter[1 + 2 * conv_params->ref], conv_params->plane); + InterpFilter filter_y = av1_get_plane_interp_filter( + interp_filter[0 + 2 * conv_params->ref], conv_params->plane); + InterpFilterParams interp_filter_params_x = + av1_get_interp_filter_params(filter_x); + InterpFilterParams interp_filter_params_y = + av1_get_interp_filter_params(filter_y); +#else + InterpFilterParams interp_filter_params = + av1_get_interp_filter_params(interp_filter); +#endif + + assert(sf); +#if CONFIG_DUAL_FILTER + if (interp_filter_params_x.taps == SUBPEL_TAPS && + interp_filter_params_y.taps == SUBPEL_TAPS && w > 2 && h > 2 && + conv_params->round == CONVOLVE_OPT_ROUND && xs == 16 && ys == 16) { + const int16_t *kernel_x = + av1_get_interp_filter_subpel_kernel(interp_filter_params_x, subpel_x); + const int16_t *kernel_y = + av1_get_interp_filter_subpel_kernel(interp_filter_params_y, subpel_y); +#else + if (interp_filter_params.taps == SUBPEL_TAPS && w > 2 && h > 2 && + conv_params->round == CONVOLVE_OPT_ROUND && xs == 16 && ys == 16) { + const int16_t *kernel_x = + av1_get_interp_filter_subpel_kernel(interp_filter_params, subpel_x); + const int16_t *kernel_y = + av1_get_interp_filter_subpel_kernel(interp_filter_params, subpel_y); +#endif + sf->predict[subpel_x != 0][subpel_y != 0][conv_params->ref]( + src, src_stride, dst, dst_stride, kernel_x, xs, kernel_y, ys, w, h); + } else { +// ref_idx > 0 means this is the second reference frame +// first reference frame's prediction result is already in dst +// therefore we need to average the first and second results +#if CONFIG_CONVOLVE_ROUND + if (conv_params->round == CONVOLVE_OPT_NO_ROUND && xs == 16 && ys == 16) + av1_convolve_2d_facade(src, src_stride, dst, dst_stride, w, h, +#if CONFIG_DUAL_FILTER + interp_filter, +#else + &interp_filter, +#endif + subpel_x, xs, subpel_y, ys, conv_params); + else +#endif + { + if (xs == 16 && ys == 16) { + av1_convolve(src, src_stride, dst, dst_stride, w, h, interp_filter, + subpel_x, xs, subpel_y, ys, conv_params); + } else { + // If xs == 16 || ys == 16 scaling is happening and the SSE2 + // instructions don't support scaling; use the C versions to be safe. + av1_convolve_c(src, src_stride, dst, dst_stride, w, h, interp_filter, + subpel_x, xs, subpel_y, ys, conv_params); + } + } + } +} + +#if CONFIG_HIGHBITDEPTH +static INLINE void highbd_inter_predictor(const uint8_t *src, int src_stride, + uint8_t *dst, int dst_stride, + const int subpel_x, + const int subpel_y, + const struct scale_factors *sf, int w, + int h, int ref, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif + int xs, int ys, int bd) { +#if CONFIG_DUAL_FILTER + InterpFilterParams interp_filter_params_x = + av1_get_interp_filter_params(interp_filter[1 + 2 * ref]); + InterpFilterParams interp_filter_params_y = + av1_get_interp_filter_params(interp_filter[0 + 2 * ref]); +#else + InterpFilterParams interp_filter_params = + av1_get_interp_filter_params(interp_filter); +#endif + +#if CONFIG_DUAL_FILTER + if (interp_filter_params_x.taps == SUBPEL_TAPS && + interp_filter_params_y.taps == SUBPEL_TAPS && w > 2 && h > 2) { + const int16_t *kernel_x = + av1_get_interp_filter_subpel_kernel(interp_filter_params_x, subpel_x); + const int16_t *kernel_y = + av1_get_interp_filter_subpel_kernel(interp_filter_params_y, subpel_y); +#else + if (interp_filter_params.taps == SUBPEL_TAPS && w > 2 && h > 2) { + const int16_t *kernel_x = + av1_get_interp_filter_subpel_kernel(interp_filter_params, subpel_x); + const int16_t *kernel_y = + av1_get_interp_filter_subpel_kernel(interp_filter_params, subpel_y); +#endif // CONFIG_DUAL_FILTER + sf->highbd_predict[subpel_x != 0][subpel_y != 0][ref]( + src, src_stride, dst, dst_stride, kernel_x, xs, kernel_y, ys, w, h, bd); + } else { + // ref > 0 means this is the second reference frame + // first reference frame's prediction result is already in dst + // therefore we need to average the first and second results + int avg = ref > 0; + av1_highbd_convolve(src, src_stride, dst, dst_stride, w, h, interp_filter, + subpel_x, xs, subpel_y, ys, avg, bd); + } +} +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_EXT_INTER +// Set to (1 << 5) if the 32-ary codebooks are used for any bock size +#define MAX_WEDGE_TYPES (1 << 4) + +#define MAX_WEDGE_SIZE_LOG2 5 // 32x32 +#define MAX_WEDGE_SIZE (1 << MAX_WEDGE_SIZE_LOG2) +#define MAX_WEDGE_SQUARE (MAX_WEDGE_SIZE * MAX_WEDGE_SIZE) + +#define WEDGE_WEIGHT_BITS 6 + +#define WEDGE_NONE -1 + +// Angles are with respect to horizontal anti-clockwise +typedef enum { + WEDGE_HORIZONTAL = 0, + WEDGE_VERTICAL = 1, + WEDGE_OBLIQUE27 = 2, + WEDGE_OBLIQUE63 = 3, + WEDGE_OBLIQUE117 = 4, + WEDGE_OBLIQUE153 = 5, + WEDGE_DIRECTIONS +} WedgeDirectionType; + +// 3-tuple: {direction, x_offset, y_offset} +typedef struct { + WedgeDirectionType direction; + int x_offset; + int y_offset; +} wedge_code_type; + +typedef uint8_t *wedge_masks_type[MAX_WEDGE_TYPES]; + +typedef struct { + int bits; + const wedge_code_type *codebook; + uint8_t *signflip; + int smoother; + wedge_masks_type *masks; +} wedge_params_type; + +extern const wedge_params_type wedge_params_lookup[BLOCK_SIZES]; + +static INLINE int is_interinter_compound_used(COMPOUND_TYPE type, + BLOCK_SIZE sb_type) { + (void)sb_type; + switch (type) { + case COMPOUND_AVERAGE: return 1; +#if CONFIG_WEDGE + case COMPOUND_WEDGE: return wedge_params_lookup[sb_type].bits > 0; +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + case COMPOUND_SEG: return sb_type >= BLOCK_8X8; +#endif // CONFIG_COMPOUND_SEGMENT + default: assert(0); return 0; + } +} + +static INLINE int is_any_masked_compound_used(BLOCK_SIZE sb_type) { + COMPOUND_TYPE comp_type; + for (comp_type = 0; comp_type < COMPOUND_TYPES; comp_type++) { + if (is_masked_compound_type(comp_type) && + is_interinter_compound_used(comp_type, sb_type)) + return 1; + } + return 0; +} + +static INLINE int get_wedge_bits_lookup(BLOCK_SIZE sb_type) { + return wedge_params_lookup[sb_type].bits; +} + +static INLINE int get_interinter_wedge_bits(BLOCK_SIZE sb_type) { + const int wbits = wedge_params_lookup[sb_type].bits; + return (wbits > 0) ? wbits + 1 : 0; +} + +static INLINE int is_interintra_wedge_used(BLOCK_SIZE sb_type) { + (void)sb_type; + return wedge_params_lookup[sb_type].bits > 0; +} + +static INLINE int get_interintra_wedge_bits(BLOCK_SIZE sb_type) { + return wedge_params_lookup[sb_type].bits; +} + +#if CONFIG_COMPOUND_SEGMENT +void build_compound_seg_mask(uint8_t *mask, SEG_MASK_TYPE mask_type, + const uint8_t *src0, int src0_stride, + const uint8_t *src1, int src1_stride, + BLOCK_SIZE sb_type, int h, int w); +#if CONFIG_HIGHBITDEPTH +void build_compound_seg_mask_highbd(uint8_t *mask, SEG_MASK_TYPE mask_type, + const uint8_t *src0, int src0_stride, + const uint8_t *src1, int src1_stride, + BLOCK_SIZE sb_type, int h, int w, int bd); +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_COMPOUND_SEGMENT +#endif // CONFIG_EXT_INTER + +void build_inter_predictors(MACROBLOCKD *xd, int plane, +#if CONFIG_MOTION_VAR + int mi_col_offset, int mi_row_offset, +#endif // CONFIG_MOTION_VAR + int block, int bw, int bh, int x, int y, int w, + int h, +#if CONFIG_SUPERTX && CONFIG_EXT_INTER + int wedge_offset_x, int wedge_offset_y, +#endif // CONFIG_SUPERTX && CONFIG_EXT_INTER + int mi_x, int mi_y); + +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +// This function will determine whether or not to create a warped +// prediction and return the appropriate motion model depending +// on the configuration. Behavior will change with different +// combinations of GLOBAL_MOTION, WARPED_MOTION and MOTION_VAR. +static INLINE int allow_warp(const MODE_INFO *const mi, + const WarpTypesAllowed *const warp_types, +#if CONFIG_GLOBAL_MOTION + const WarpedMotionParams *const gm_params, +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_MOTION_VAR + int mi_col_offset, int mi_row_offset, +#endif // CONFIG_MOTION_VAR + WarpedMotionParams *final_warp_params) { + const MB_MODE_INFO *const mbmi = &mi->mbmi; + set_default_warp_params(final_warp_params); + +// Only global motion configured +#if CONFIG_GLOBAL_MOTION && !CONFIG_WARPED_MOTION && !CONFIG_MOTION_VAR + (void)mbmi; + if (warp_types->global_warp_allowed) { + memcpy(final_warp_params, gm_params, sizeof(*final_warp_params)); + return 1; + } +#endif // CONFIG_GLOBAL_MOTION && !CONFIG_WARPED_MOTION && !CONFIG_MOTION_VAR + +// Only warped motion configured +#if CONFIG_WARPED_MOTION && !CONFIG_GLOBAL_MOTION && !CONFIG_MOTION_VAR + if (warp_types->local_warp_allowed) { + memcpy(final_warp_params, &mbmi->wm_params[0], sizeof(*final_warp_params)); + return 1; + } +#endif // CONFIG_WARPED_MOTION && !CONFIG_GLOBAL_MOTION && !CONFIG_MOTION_VAR + +// Warped and global motion configured +#if CONFIG_GLOBAL_MOTION && CONFIG_WARPED_MOTION && !CONFIG_MOTION_VAR + // When both are enabled, warped will take priority. The global parameters + // will only be used to compute projection samples to find the warped model. + // Note that, if SEPARATE_GLOBAL_MOTION is enabled and a block chooses + // global, it will not be possible to select WARPED_CAUSAL. + if (warp_types->local_warp_allowed) { + memcpy(final_warp_params, &mbmi->wm_params[0], sizeof(*final_warp_params)); + return 1; + } else if (warp_types->global_warp_allowed) { + memcpy(final_warp_params, gm_params, sizeof(*final_warp_params)); + return 1; + } +#endif // CONFIG_GLOBAL_MOTION && CONFIG_WARPED_MOTION && !CONFIG_MOTION_VAR + +// Motion var and global motion configured +#if CONFIG_GLOBAL_MOTION && CONFIG_MOTION_VAR && !CONFIG_WARPED_MOTION + // We warp if either case is true: + // 1.) We are predicting a block which uses global motion + // 2.) We are predicting a neighboring block of a block using OBMC, + // the neighboring block uses global motion, and we have enabled + // WARP_GM_NEIGHBORS_WITH_OBMC + const int build_for_obmc = !(mi_col_offset == 0 && mi_row_offset == 0); + (void)mbmi; + if (warp_types->global_warp_allowed && + (WARP_GM_NEIGHBORS_WITH_OBMC || !build_for_obmc)) { + memcpy(final_warp_params, gm_params, sizeof(*final_warp_params)); + return 1; + } +#endif // CONFIG_GLOBAL_MOTION && CONFIG_MOTION_VAR && !CONFIG_WARPED_MOTION + +// Motion var and warped motion configured +#if CONFIG_WARPED_MOTION && CONFIG_MOTION_VAR && !CONFIG_GLOBAL_MOTION + // We warp if either case is true: + // 1.) We are predicting a block with motion mode WARPED_CAUSAL + // 2.) We are predicting a neighboring block of a block using OBMC, + // the neighboring block has mode WARPED_CAUSAL, and we have enabled + // WARP_WM_NEIGHBORS_WITH_OBMC + const int build_for_obmc = !(mi_col_offset == 0 && mi_row_offset == 0); + if (warp_types->local_warp_allowed) { + if ((build_for_obmc && WARP_WM_NEIGHBORS_WITH_OBMC) || (!build_for_obmc)) { + memcpy(final_warp_params, &mbmi->wm_params[0], + sizeof(*final_warp_params)); + return 1; + } + } +#endif // CONFIG_WARPED_MOTION && CONFIG_MOTION_VAR && !CONFIG_GLOBAL_MOTION + +// Motion var, warped motion and global motion all configured +#if CONFIG_WARPED_MOTION && CONFIG_MOTION_VAR && CONFIG_GLOBAL_MOTION + const int build_for_obmc = !(mi_col_offset == 0 && mi_row_offset == 0); + if (warp_types->local_warp_allowed) { + if ((build_for_obmc && WARP_WM_NEIGHBORS_WITH_OBMC) || (!build_for_obmc)) { + memcpy(final_warp_params, &mbmi->wm_params[0], + sizeof(*final_warp_params)); + return 1; + } + } else if (warp_types->global_warp_allowed && + (WARP_GM_NEIGHBORS_WITH_OBMC || !build_for_obmc)) { + memcpy(final_warp_params, gm_params, sizeof(*final_warp_params)); + return 1; + } +#endif // CONFIG_WARPED_MOTION && CONFIG_MOTION_VAR && CONFIG_GLOBAL_MOTION + + return 0; +} +#endif // CONFIG_GLOBAL_MOTION ||CONFIG_WARPED_MOTION + +static INLINE void av1_make_inter_predictor( + const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, + const int subpel_x, const int subpel_y, const struct scale_factors *sf, + int w, int h, ConvolveParams *conv_params, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + const WarpTypesAllowed *warp_types, int p_col, int p_row, int plane, + int ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#if CONFIG_MOTION_VAR + int mi_col_offset, int mi_row_offset, +#endif + int xs, int ys, const MACROBLOCKD *xd) { + (void)xd; + +#if CONFIG_MOTION_VAR + const MODE_INFO *mi = xd->mi[mi_col_offset + xd->mi_stride * mi_row_offset]; +#else + const MODE_INFO *mi = xd->mi[0]; + (void)mi; +#endif // CONFIG_MOTION_VAR + +// Make sure the selected motion mode is valid for this configuration +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + assert_motion_mode_valid(mi->mbmi.motion_mode, +#if CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + 0, xd->global_motion, +#endif // CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + mi); +#endif // CONFIG MOTION_VAR || CONFIG_WARPED_MOTION + +#if CONFIG_WARPED_MOTION || CONFIG_GLOBAL_MOTION + WarpedMotionParams final_warp_params; + const int do_warp = allow_warp(mi, warp_types, +#if CONFIG_GLOBAL_MOTION + &xd->global_motion[mi->mbmi.ref_frame[ref]], +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_MOTION_VAR + mi_col_offset, mi_row_offset, +#endif // CONFIG_MOTION_VAR + &final_warp_params); + if (do_warp) { + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const struct buf_2d *const pre_buf = &pd->pre[ref]; + av1_warp_plane(&final_warp_params, +#if CONFIG_HIGHBITDEPTH + xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH, xd->bd, +#endif // CONFIG_HIGHBITDEPTH + pre_buf->buf0, pre_buf->width, pre_buf->height, + pre_buf->stride, dst, p_col, p_row, w, h, dst_stride, + pd->subsampling_x, pd->subsampling_y, xs, ys, ref); + return; + } +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + highbd_inter_predictor(src, src_stride, dst, dst_stride, subpel_x, subpel_y, + sf, w, h, conv_params->ref, interp_filter, xs, ys, + xd->bd); + return; + } +#endif // CONFIG_HIGHBITDEPTH + inter_predictor(src, src_stride, dst, dst_stride, subpel_x, subpel_y, sf, w, + h, conv_params, interp_filter, xs, ys); +} + +#if CONFIG_EXT_INTER +void av1_make_masked_inter_predictor(const uint8_t *pre, int pre_stride, + uint8_t *dst, int dst_stride, + const int subpel_x, const int subpel_y, + const struct scale_factors *sf, int w, + int h, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif + int xs, int ys, +#if CONFIG_SUPERTX + int wedge_offset_x, int wedge_offset_y, +#endif // CONFIG_SUPERTX + int plane, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + const WarpTypesAllowed *warp_types, + int p_col, int p_row, int ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + MACROBLOCKD *xd); +#endif // CONFIG_EXT_INTER + +static INLINE int round_mv_comp_q4(int value) { + return (value < 0 ? value - 2 : value + 2) / 4; +} + +static MV mi_mv_pred_q4(const MODE_INFO *mi, int idx) { + MV res = { + round_mv_comp_q4( + mi->bmi[0].as_mv[idx].as_mv.row + mi->bmi[1].as_mv[idx].as_mv.row + + mi->bmi[2].as_mv[idx].as_mv.row + mi->bmi[3].as_mv[idx].as_mv.row), + round_mv_comp_q4( + mi->bmi[0].as_mv[idx].as_mv.col + mi->bmi[1].as_mv[idx].as_mv.col + + mi->bmi[2].as_mv[idx].as_mv.col + mi->bmi[3].as_mv[idx].as_mv.col) + }; + return res; +} + +static INLINE int round_mv_comp_q2(int value) { + return (value < 0 ? value - 1 : value + 1) / 2; +} + +static MV mi_mv_pred_q2(const MODE_INFO *mi, int idx, int block0, int block1) { + MV res = { round_mv_comp_q2(mi->bmi[block0].as_mv[idx].as_mv.row + + mi->bmi[block1].as_mv[idx].as_mv.row), + round_mv_comp_q2(mi->bmi[block0].as_mv[idx].as_mv.col + + mi->bmi[block1].as_mv[idx].as_mv.col) }; + return res; +} + +// TODO(jkoleszar): yet another mv clamping function :-( +static INLINE MV clamp_mv_to_umv_border_sb(const MACROBLOCKD *xd, + const MV *src_mv, int bw, int bh, + int ss_x, int ss_y) { + // If the MV points so far into the UMV border that no visible pixels + // are used for reconstruction, the subpel part of the MV can be + // discarded and the MV limited to 16 pixels with equivalent results. + const int spel_left = (AOM_INTERP_EXTEND + bw) << SUBPEL_BITS; + const int spel_right = spel_left - SUBPEL_SHIFTS; + const int spel_top = (AOM_INTERP_EXTEND + bh) << SUBPEL_BITS; + const int spel_bottom = spel_top - SUBPEL_SHIFTS; + MV clamped_mv = { src_mv->row * (1 << (1 - ss_y)), + src_mv->col * (1 << (1 - ss_x)) }; + assert(ss_x <= 1); + assert(ss_y <= 1); + + clamp_mv(&clamped_mv, xd->mb_to_left_edge * (1 << (1 - ss_x)) - spel_left, + xd->mb_to_right_edge * (1 << (1 - ss_x)) + spel_right, + xd->mb_to_top_edge * (1 << (1 - ss_y)) - spel_top, + xd->mb_to_bottom_edge * (1 << (1 - ss_y)) + spel_bottom); + + return clamped_mv; +} + +static INLINE MV average_split_mvs(const struct macroblockd_plane *pd, + const MODE_INFO *mi, int ref, int block) { + const int ss_idx = ((pd->subsampling_x > 0) << 1) | (pd->subsampling_y > 0); + MV res = { 0, 0 }; + switch (ss_idx) { + case 0: res = mi->bmi[block].as_mv[ref].as_mv; break; + case 1: res = mi_mv_pred_q2(mi, ref, block, block + 2); break; + case 2: res = mi_mv_pred_q2(mi, ref, block, block + 1); break; + case 3: res = mi_mv_pred_q4(mi, ref); break; + default: assert(ss_idx <= 3 && ss_idx >= 0); + } + return res; +} + +void av1_build_inter_predictor_sub8x8(MACROBLOCKD *xd, int plane, int i, int ir, + int ic, int mi_row, int mi_col); + +void av1_build_inter_predictors_sby(MACROBLOCKD *xd, int mi_row, int mi_col, + BUFFER_SET *ctx, BLOCK_SIZE bsize); + +void av1_build_inter_predictors_sbuv(MACROBLOCKD *xd, int mi_row, int mi_col, + BUFFER_SET *ctx, BLOCK_SIZE bsize); + +void av1_build_inter_predictors_sb(MACROBLOCKD *xd, int mi_row, int mi_col, + BUFFER_SET *ctx, BLOCK_SIZE bsize); + +#if CONFIG_SUPERTX +void av1_build_inter_predictors_sb_sub8x8_extend(MACROBLOCKD *xd, +#if CONFIG_EXT_INTER + int mi_row_ori, int mi_col_ori, +#endif // CONFIG_EXT_INTER + int mi_row, int mi_col, + BLOCK_SIZE bsize, int block); + +void av1_build_inter_predictors_sb_extend(MACROBLOCKD *xd, +#if CONFIG_EXT_INTER + int mi_row_ori, int mi_col_ori, +#endif // CONFIG_EXT_INTER + int mi_row, int mi_col, + BLOCK_SIZE bsize); +struct macroblockd_plane; +void av1_build_masked_inter_predictor_complex( + MACROBLOCKD *xd, uint8_t *dst, int dst_stride, const uint8_t *pre, + int pre_stride, int mi_row, int mi_col, int mi_row_ori, int mi_col_ori, + BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, PARTITION_TYPE partition, + int plane); +#endif // CONFIG_SUPERTX + +void av1_build_inter_predictor(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, const MV *src_mv, + const struct scale_factors *sf, int w, int h, + ConvolveParams *conv_params, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + const WarpTypesAllowed *warp_types, int p_col, + int p_row, int plane, int ref, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + enum mv_precision precision, int x, int y, + const MACROBLOCKD *xd); + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_build_inter_predictor( + const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, + const MV *mv_q3, const struct scale_factors *sf, int w, int h, int do_avg, +#if CONFIG_DUAL_FILTER + const InterpFilter *interp_filter, +#else + const InterpFilter interp_filter, +#endif +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + const WarpTypesAllowed *warp_types, int p_col, int p_row, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + int plane, enum mv_precision precision, int x, int y, + const MACROBLOCKD *xd); +#endif + +static INLINE int scaled_buffer_offset(int x_offset, int y_offset, int stride, + const struct scale_factors *sf) { + const int x = sf ? sf->scale_value_x(x_offset, sf) : x_offset; + const int y = sf ? sf->scale_value_y(y_offset, sf) : y_offset; + return y * stride + x; +} + +static INLINE void setup_pred_plane(struct buf_2d *dst, BLOCK_SIZE bsize, + uint8_t *src, int width, int height, + int stride, int mi_row, int mi_col, + const struct scale_factors *scale, + int subsampling_x, int subsampling_y) { +#if CONFIG_CHROMA_SUB8X8 + if (bsize < BLOCK_8X8) { + // Offset the buffer pointer + if (subsampling_y && (mi_row & 0x01)) mi_row -= 1; + if (subsampling_x && (mi_col & 0x01)) mi_col -= 1; + } +#else + (void)bsize; +#endif + + const int x = (MI_SIZE * mi_col) >> subsampling_x; + const int y = (MI_SIZE * mi_row) >> subsampling_y; + dst->buf = src + scaled_buffer_offset(x, y, stride, scale); + dst->buf0 = src; + dst->width = width; + dst->height = height; + dst->stride = stride; +} + +void av1_setup_dst_planes(struct macroblockd_plane planes[MAX_MB_PLANE], + BLOCK_SIZE bsize, const YV12_BUFFER_CONFIG *src, + int mi_row, int mi_col); + +void av1_setup_pre_planes(MACROBLOCKD *xd, int idx, + const YV12_BUFFER_CONFIG *src, int mi_row, int mi_col, + const struct scale_factors *sf); + +// Detect if the block have sub-pixel level motion vectors +// per component. +#define CHECK_SUBPEL 0 +static INLINE int has_subpel_mv_component(const MODE_INFO *const mi, + const MACROBLOCKD *const xd, + int dir) { +#if CHECK_SUBPEL + const MB_MODE_INFO *const mbmi = &mi->mbmi; + const BLOCK_SIZE bsize = mbmi->sb_type; + int plane; + int ref = (dir >> 1); +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + + if (bsize >= BLOCK_8X8 || unify_bsize) { + if (dir & 0x01) { + if (mbmi->mv[ref].as_mv.col & SUBPEL_MASK) return 1; + } else { + if (mbmi->mv[ref].as_mv.row & SUBPEL_MASK) return 1; + } + } else { + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + const PARTITION_TYPE bp = BLOCK_8X8 - bsize; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const int have_vsplit = bp != PARTITION_HORZ; + const int have_hsplit = bp != PARTITION_VERT; + const int num_4x4_w = 2 >> ((!have_vsplit) | pd->subsampling_x); + const int num_4x4_h = 2 >> ((!have_hsplit) | pd->subsampling_y); + + int x, y; + for (y = 0; y < num_4x4_h; ++y) { + for (x = 0; x < num_4x4_w; ++x) { + const MV mv = average_split_mvs(pd, mi, ref, y * 2 + x); + if (dir & 0x01) { + if (mv.col & SUBPEL_MASK) return 1; + } else { + if (mv.row & SUBPEL_MASK) return 1; + } + } + } + } + } + + return 0; +#else + (void)mi; + (void)xd; + (void)dir; + return 1; +#endif +} + +static INLINE void set_default_interp_filters( + MB_MODE_INFO *const mbmi, InterpFilter frame_interp_filter) { +#if CONFIG_DUAL_FILTER + int dir; + for (dir = 0; dir < 4; ++dir) + mbmi->interp_filter[dir] = frame_interp_filter == SWITCHABLE + ? EIGHTTAP_REGULAR + : frame_interp_filter; +#else + mbmi->interp_filter = frame_interp_filter == SWITCHABLE ? EIGHTTAP_REGULAR + : frame_interp_filter; +#endif // CONFIG_DUAL_FILTER +} + +static INLINE int av1_is_interp_needed(const MACROBLOCKD *const xd) { + (void)xd; +#if CONFIG_WARPED_MOTION + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + if (mbmi->motion_mode == WARPED_CAUSAL) return 0; +#endif // CONFIG_WARPED_MOTION +#if CONFIG_GLOBAL_MOTION + if (is_nontrans_global_motion(xd)) return 0; +#endif // CONFIG_GLOBAL_MOTION + return 1; +} + +static INLINE int av1_is_interp_search_needed(const MACROBLOCKD *const xd) { + MODE_INFO *const mi = xd->mi[0]; + const int is_compound = has_second_ref(&mi->mbmi); + int ref; + for (ref = 0; ref < 1 + is_compound; ++ref) { + int row_col; + for (row_col = 0; row_col < 2; ++row_col) { + const int dir = (ref << 1) + row_col; + if (has_subpel_mv_component(mi, xd, dir)) { + return 1; + } + } + } + return 0; +} + +#if CONFIG_MOTION_VAR +const uint8_t *av1_get_obmc_mask(int length); +void av1_count_overlappable_neighbors(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col); +void av1_build_obmc_inter_prediction(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col, + uint8_t *above[MAX_MB_PLANE], + int above_stride[MAX_MB_PLANE], + uint8_t *left[MAX_MB_PLANE], + int left_stride[MAX_MB_PLANE]); +void av1_build_prediction_by_above_preds(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col, + uint8_t *tmp_buf[MAX_MB_PLANE], + int tmp_width[MAX_MB_PLANE], + int tmp_height[MAX_MB_PLANE], + int tmp_stride[MAX_MB_PLANE]); +void av1_build_prediction_by_left_preds(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col, + uint8_t *tmp_buf[MAX_MB_PLANE], + int tmp_width[MAX_MB_PLANE], + int tmp_height[MAX_MB_PLANE], + int tmp_stride[MAX_MB_PLANE]); +void av1_build_obmc_inter_predictors_sb(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col); +#if CONFIG_NCOBMC +void av1_build_ncobmc_inter_predictors_sb(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col); +#endif +#endif // CONFIG_MOTION_VAR + +#if CONFIG_EXT_INTER +#define MASK_MASTER_SIZE ((MAX_WEDGE_SIZE) << 1) +#define MASK_MASTER_STRIDE (MASK_MASTER_SIZE) + +void av1_init_wedge_masks(); + +static INLINE const uint8_t *av1_get_contiguous_soft_mask(int wedge_index, + int wedge_sign, + BLOCK_SIZE sb_type) { + return wedge_params_lookup[sb_type].masks[wedge_sign][wedge_index]; +} + +const uint8_t *av1_get_soft_mask(int wedge_index, int wedge_sign, + BLOCK_SIZE sb_type, int wedge_offset_x, + int wedge_offset_y); + +const uint8_t *av1_get_compound_type_mask_inverse( + const INTERINTER_COMPOUND_DATA *const comp_data, +#if CONFIG_COMPOUND_SEGMENT + uint8_t *mask_buffer, int h, int w, int stride, +#endif + BLOCK_SIZE sb_type); + +const uint8_t *av1_get_compound_type_mask( + const INTERINTER_COMPOUND_DATA *const comp_data, BLOCK_SIZE sb_type); + +void av1_build_interintra_predictors(MACROBLOCKD *xd, uint8_t *ypred, + uint8_t *upred, uint8_t *vpred, + int ystride, int ustride, int vstride, + BUFFER_SET *ctx, BLOCK_SIZE bsize); +void av1_build_interintra_predictors_sby(MACROBLOCKD *xd, uint8_t *ypred, + int ystride, BUFFER_SET *ctx, + BLOCK_SIZE bsize); +void av1_build_interintra_predictors_sbc(MACROBLOCKD *xd, uint8_t *upred, + int ustride, BUFFER_SET *ctx, + int plane, BLOCK_SIZE bsize); +void av1_build_interintra_predictors_sbuv(MACROBLOCKD *xd, uint8_t *upred, + uint8_t *vpred, int ustride, + int vstride, BUFFER_SET *ctx, + BLOCK_SIZE bsize); + +void av1_build_intra_predictors_for_interintra(MACROBLOCKD *xd, + BLOCK_SIZE bsize, int plane, + BUFFER_SET *ctx, + uint8_t *intra_pred, + int intra_stride); +void av1_combine_interintra(MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane, + const uint8_t *inter_pred, int inter_stride, + const uint8_t *intra_pred, int intra_stride); + +// Encoder only +void av1_build_inter_predictors_for_planes_single_buf( + MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane_from, int plane_to, int mi_row, + int mi_col, int ref, uint8_t *ext_dst[3], int ext_dst_stride[3]); +void av1_build_wedge_inter_predictor_from_buf( + MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane_from, int plane_to, +#if CONFIG_SUPERTX + int wedge_offset_x, int wedge_offset_y, +#endif // CONFIG_SUPERTX + uint8_t *ext_dst0[3], int ext_dst_stride0[3], uint8_t *ext_dst1[3], + int ext_dst_stride1[3]); +#endif // CONFIG_EXT_INTER + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_RECONINTER_H_ diff --git a/third_party/aom/av1/common/reconintra.c b/third_party/aom/av1/common/reconintra.c new file mode 100644 index 0000000000..6e0ff52ce6 --- /dev/null +++ b/third_party/aom/av1/common/reconintra.c @@ -0,0 +1,2467 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./av1_rtcd.h" +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom_ports/system_state.h" + +#if CONFIG_HIGHBITDEPTH +#include "aom_dsp/aom_dsp_common.h" +#endif // CONFIG_HIGHBITDEPTH +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" +#include "aom_ports/aom_once.h" +#include "av1/common/reconintra.h" +#include "av1/common/onyxc_int.h" +#if CONFIG_CFL +#include "av1/common/cfl.h" +#endif + +enum { + NEED_LEFT = 1 << 1, + NEED_ABOVE = 1 << 2, + NEED_ABOVERIGHT = 1 << 3, + NEED_ABOVELEFT = 1 << 4, + NEED_BOTTOMLEFT = 1 << 5, +}; + +static const uint8_t extend_modes[INTRA_MODES] = { + NEED_ABOVE | NEED_LEFT, // DC + NEED_ABOVE, // V + NEED_LEFT, // H + NEED_ABOVE | NEED_ABOVERIGHT, // D45 + NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT, // D135 + NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT, // D117 + NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT, // D153 + NEED_LEFT | NEED_BOTTOMLEFT, // D207 + NEED_ABOVE | NEED_ABOVERIGHT, // D63 +#if CONFIG_ALT_INTRA + NEED_LEFT | NEED_ABOVE, // SMOOTH +#endif // CONFIG_ALT_INTRA + NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT, // TM +}; + +static const uint16_t orders_128x128[1] = { 0 }; +static const uint16_t orders_128x64[2] = { 0, 1 }; +static const uint16_t orders_64x128[2] = { 0, 1 }; +static const uint16_t orders_64x64[4] = { + 0, 1, 2, 3, +}; +static const uint16_t orders_64x32[8] = { + 0, 2, 1, 3, 4, 6, 5, 7, +}; +static const uint16_t orders_32x64[8] = { + 0, 1, 2, 3, 4, 5, 6, 7, +}; +static const uint16_t orders_32x32[16] = { + 0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15, +}; +static const uint16_t orders_32x16[32] = { + 0, 2, 8, 10, 1, 3, 9, 11, 4, 6, 12, 14, 5, 7, 13, 15, + 16, 18, 24, 26, 17, 19, 25, 27, 20, 22, 28, 30, 21, 23, 29, 31, +}; +static const uint16_t orders_16x32[32] = { + 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15, + 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31, +}; +static const uint16_t orders_16x16[64] = { + 0, 1, 4, 5, 16, 17, 20, 21, 2, 3, 6, 7, 18, 19, 22, 23, + 8, 9, 12, 13, 24, 25, 28, 29, 10, 11, 14, 15, 26, 27, 30, 31, + 32, 33, 36, 37, 48, 49, 52, 53, 34, 35, 38, 39, 50, 51, 54, 55, + 40, 41, 44, 45, 56, 57, 60, 61, 42, 43, 46, 47, 58, 59, 62, 63, +}; + +#if CONFIG_CB4X4 || CONFIG_EXT_PARTITION +static const uint16_t orders_16x8[128] = { + 0, 2, 8, 10, 32, 34, 40, 42, 1, 3, 9, 11, 33, 35, 41, 43, + 4, 6, 12, 14, 36, 38, 44, 46, 5, 7, 13, 15, 37, 39, 45, 47, + 16, 18, 24, 26, 48, 50, 56, 58, 17, 19, 25, 27, 49, 51, 57, 59, + 20, 22, 28, 30, 52, 54, 60, 62, 21, 23, 29, 31, 53, 55, 61, 63, + 64, 66, 72, 74, 96, 98, 104, 106, 65, 67, 73, 75, 97, 99, 105, 107, + 68, 70, 76, 78, 100, 102, 108, 110, 69, 71, 77, 79, 101, 103, 109, 111, + 80, 82, 88, 90, 112, 114, 120, 122, 81, 83, 89, 91, 113, 115, 121, 123, + 84, 86, 92, 94, 116, 118, 124, 126, 85, 87, 93, 95, 117, 119, 125, 127, +}; +static const uint16_t orders_8x16[128] = { + 0, 1, 2, 3, 8, 9, 10, 11, 32, 33, 34, 35, 40, 41, 42, 43, + 4, 5, 6, 7, 12, 13, 14, 15, 36, 37, 38, 39, 44, 45, 46, 47, + 16, 17, 18, 19, 24, 25, 26, 27, 48, 49, 50, 51, 56, 57, 58, 59, + 20, 21, 22, 23, 28, 29, 30, 31, 52, 53, 54, 55, 60, 61, 62, 63, + 64, 65, 66, 67, 72, 73, 74, 75, 96, 97, 98, 99, 104, 105, 106, 107, + 68, 69, 70, 71, 76, 77, 78, 79, 100, 101, 102, 103, 108, 109, 110, 111, + 80, 81, 82, 83, 88, 89, 90, 91, 112, 113, 114, 115, 120, 121, 122, 123, + 84, 85, 86, 87, 92, 93, 94, 95, 116, 117, 118, 119, 124, 125, 126, 127, +}; +static const uint16_t orders_8x8[256] = { + 0, 1, 4, 5, 16, 17, 20, 21, 64, 65, 68, 69, 80, 81, 84, + 85, 2, 3, 6, 7, 18, 19, 22, 23, 66, 67, 70, 71, 82, 83, + 86, 87, 8, 9, 12, 13, 24, 25, 28, 29, 72, 73, 76, 77, 88, + 89, 92, 93, 10, 11, 14, 15, 26, 27, 30, 31, 74, 75, 78, 79, + 90, 91, 94, 95, 32, 33, 36, 37, 48, 49, 52, 53, 96, 97, 100, + 101, 112, 113, 116, 117, 34, 35, 38, 39, 50, 51, 54, 55, 98, 99, + 102, 103, 114, 115, 118, 119, 40, 41, 44, 45, 56, 57, 60, 61, 104, + 105, 108, 109, 120, 121, 124, 125, 42, 43, 46, 47, 58, 59, 62, 63, + 106, 107, 110, 111, 122, 123, 126, 127, 128, 129, 132, 133, 144, 145, 148, + 149, 192, 193, 196, 197, 208, 209, 212, 213, 130, 131, 134, 135, 146, 147, + 150, 151, 194, 195, 198, 199, 210, 211, 214, 215, 136, 137, 140, 141, 152, + 153, 156, 157, 200, 201, 204, 205, 216, 217, 220, 221, 138, 139, 142, 143, + 154, 155, 158, 159, 202, 203, 206, 207, 218, 219, 222, 223, 160, 161, 164, + 165, 176, 177, 180, 181, 224, 225, 228, 229, 240, 241, 244, 245, 162, 163, + 166, 167, 178, 179, 182, 183, 226, 227, 230, 231, 242, 243, 246, 247, 168, + 169, 172, 173, 184, 185, 188, 189, 232, 233, 236, 237, 248, 249, 252, 253, + 170, 171, 174, 175, 186, 187, 190, 191, 234, 235, 238, 239, 250, 251, 254, + 255, +}; + +#if CONFIG_CB4X4 && CONFIG_EXT_PARTITION +static const uint16_t orders_4x8[512] = { + 0, 1, 2, 3, 8, 9, 10, 11, 32, 33, 34, 35, 40, 41, 42, + 43, 128, 129, 130, 131, 136, 137, 138, 139, 160, 161, 162, 163, 168, 169, + 170, 171, 4, 5, 6, 7, 12, 13, 14, 15, 36, 37, 38, 39, 44, + 45, 46, 47, 132, 133, 134, 135, 140, 141, 142, 143, 164, 165, 166, 167, + 172, 173, 174, 175, 16, 17, 18, 19, 24, 25, 26, 27, 48, 49, 50, + 51, 56, 57, 58, 59, 144, 145, 146, 147, 152, 153, 154, 155, 176, 177, + 178, 179, 184, 185, 186, 187, 20, 21, 22, 23, 28, 29, 30, 31, 52, + 53, 54, 55, 60, 61, 62, 63, 148, 149, 150, 151, 156, 157, 158, 159, + 180, 181, 182, 183, 188, 189, 190, 191, 64, 65, 66, 67, 72, 73, 74, + 75, 96, 97, 98, 99, 104, 105, 106, 107, 192, 193, 194, 195, 200, 201, + 202, 203, 224, 225, 226, 227, 232, 233, 234, 235, 68, 69, 70, 71, 76, + 77, 78, 79, 100, 101, 102, 103, 108, 109, 110, 111, 196, 197, 198, 199, + 204, 205, 206, 207, 228, 229, 230, 231, 236, 237, 238, 239, 80, 81, 82, + 83, 88, 89, 90, 91, 112, 113, 114, 115, 120, 121, 122, 123, 208, 209, + 210, 211, 216, 217, 218, 219, 240, 241, 242, 243, 248, 249, 250, 251, 84, + 85, 86, 87, 92, 93, 94, 95, 116, 117, 118, 119, 124, 125, 126, 127, + 212, 213, 214, 215, 220, 221, 222, 223, 244, 245, 246, 247, 252, 253, 254, + 255, 256, 257, 258, 259, 264, 265, 266, 267, 288, 289, 290, 291, 296, 297, + 298, 299, 384, 385, 386, 387, 392, 393, 394, 395, 416, 417, 418, 419, 424, + 425, 426, 427, 260, 261, 262, 263, 268, 269, 270, 271, 292, 293, 294, 295, + 300, 301, 302, 303, 388, 389, 390, 391, 396, 397, 398, 399, 420, 421, 422, + 423, 428, 429, 430, 431, 272, 273, 274, 275, 280, 281, 282, 283, 304, 305, + 306, 307, 312, 313, 314, 315, 400, 401, 402, 403, 408, 409, 410, 411, 432, + 433, 434, 435, 440, 441, 442, 443, 276, 277, 278, 279, 284, 285, 286, 287, + 308, 309, 310, 311, 316, 317, 318, 319, 404, 405, 406, 407, 412, 413, 414, + 415, 436, 437, 438, 439, 444, 445, 446, 447, 320, 321, 322, 323, 328, 329, + 330, 331, 352, 353, 354, 355, 360, 361, 362, 363, 448, 449, 450, 451, 456, + 457, 458, 459, 480, 481, 482, 483, 488, 489, 490, 491, 324, 325, 326, 327, + 332, 333, 334, 335, 356, 357, 358, 359, 364, 365, 366, 367, 452, 453, 454, + 455, 460, 461, 462, 463, 484, 485, 486, 487, 492, 493, 494, 495, 336, 337, + 338, 339, 344, 345, 346, 347, 368, 369, 370, 371, 376, 377, 378, 379, 464, + 465, 466, 467, 472, 473, 474, 475, 496, 497, 498, 499, 504, 505, 506, 507, + 340, 341, 342, 343, 348, 349, 350, 351, 372, 373, 374, 375, 380, 381, 382, + 383, 468, 469, 470, 471, 476, 477, 478, 479, 500, 501, 502, 503, 508, 509, + 510, 511, +}; + +static const uint16_t orders_8x4[512] = { + 0, 2, 8, 10, 32, 34, 40, 42, 128, 130, 136, 138, 160, 162, 168, + 170, 1, 3, 9, 11, 33, 35, 41, 43, 129, 131, 137, 139, 161, 163, + 169, 171, 4, 6, 12, 14, 36, 38, 44, 46, 132, 134, 140, 142, 164, + 166, 172, 174, 5, 7, 13, 15, 37, 39, 45, 47, 133, 135, 141, 143, + 165, 167, 173, 175, 16, 18, 24, 26, 48, 50, 56, 58, 144, 146, 152, + 154, 176, 178, 184, 186, 17, 19, 25, 27, 49, 51, 57, 59, 145, 147, + 153, 155, 177, 179, 185, 187, 20, 22, 28, 30, 52, 54, 60, 62, 148, + 150, 156, 158, 180, 182, 188, 190, 21, 23, 29, 31, 53, 55, 61, 63, + 149, 151, 157, 159, 181, 183, 189, 191, 64, 66, 72, 74, 96, 98, 104, + 106, 192, 194, 200, 202, 224, 226, 232, 234, 65, 67, 73, 75, 97, 99, + 105, 107, 193, 195, 201, 203, 225, 227, 233, 235, 68, 70, 76, 78, 100, + 102, 108, 110, 196, 198, 204, 206, 228, 230, 236, 238, 69, 71, 77, 79, + 101, 103, 109, 111, 197, 199, 205, 207, 229, 231, 237, 239, 80, 82, 88, + 90, 112, 114, 120, 122, 208, 210, 216, 218, 240, 242, 248, 250, 81, 83, + 89, 91, 113, 115, 121, 123, 209, 211, 217, 219, 241, 243, 249, 251, 84, + 86, 92, 94, 116, 118, 124, 126, 212, 214, 220, 222, 244, 246, 252, 254, + 85, 87, 93, 95, 117, 119, 125, 127, 213, 215, 221, 223, 245, 247, 253, + 255, 256, 258, 264, 266, 288, 290, 296, 298, 384, 386, 392, 394, 416, 418, + 424, 426, 257, 259, 265, 267, 289, 291, 297, 299, 385, 387, 393, 395, 417, + 419, 425, 427, 260, 262, 268, 270, 292, 294, 300, 302, 388, 390, 396, 398, + 420, 422, 428, 430, 261, 263, 269, 271, 293, 295, 301, 303, 389, 391, 397, + 399, 421, 423, 429, 431, 272, 274, 280, 282, 304, 306, 312, 314, 400, 402, + 408, 410, 432, 434, 440, 442, 273, 275, 281, 283, 305, 307, 313, 315, 401, + 403, 409, 411, 433, 435, 441, 443, 276, 278, 284, 286, 308, 310, 316, 318, + 404, 406, 412, 414, 436, 438, 444, 446, 277, 279, 285, 287, 309, 311, 317, + 319, 405, 407, 413, 415, 437, 439, 445, 447, 320, 322, 328, 330, 352, 354, + 360, 362, 448, 450, 456, 458, 480, 482, 488, 490, 321, 323, 329, 331, 353, + 355, 361, 363, 449, 451, 457, 459, 481, 483, 489, 491, 324, 326, 332, 334, + 356, 358, 364, 366, 452, 454, 460, 462, 484, 486, 492, 494, 325, 327, 333, + 335, 357, 359, 365, 367, 453, 455, 461, 463, 485, 487, 493, 495, 336, 338, + 344, 346, 368, 370, 376, 378, 464, 466, 472, 474, 496, 498, 504, 506, 337, + 339, 345, 347, 369, 371, 377, 379, 465, 467, 473, 475, 497, 499, 505, 507, + 340, 342, 348, 350, 372, 374, 380, 382, 468, 470, 476, 478, 500, 502, 508, + 510, 341, 343, 349, 351, 373, 375, 381, 383, 469, 471, 477, 479, 501, 503, + 509, 511, +}; + +static const uint16_t orders_4x4[1024] = { + 0, 1, 4, 5, 16, 17, 20, 21, 64, 65, 68, 69, 80, + 81, 84, 85, 256, 257, 260, 261, 272, 273, 276, 277, 320, 321, + 324, 325, 336, 337, 340, 341, 2, 3, 6, 7, 18, 19, 22, + 23, 66, 67, 70, 71, 82, 83, 86, 87, 258, 259, 262, 263, + 274, 275, 278, 279, 322, 323, 326, 327, 338, 339, 342, 343, 8, + 9, 12, 13, 24, 25, 28, 29, 72, 73, 76, 77, 88, 89, + 92, 93, 264, 265, 268, 269, 280, 281, 284, 285, 328, 329, 332, + 333, 344, 345, 348, 349, 10, 11, 14, 15, 26, 27, 30, 31, + 74, 75, 78, 79, 90, 91, 94, 95, 266, 267, 270, 271, 282, + 283, 286, 287, 330, 331, 334, 335, 346, 347, 350, 351, 32, 33, + 36, 37, 48, 49, 52, 53, 96, 97, 100, 101, 112, 113, 116, + 117, 288, 289, 292, 293, 304, 305, 308, 309, 352, 353, 356, 357, + 368, 369, 372, 373, 34, 35, 38, 39, 50, 51, 54, 55, 98, + 99, 102, 103, 114, 115, 118, 119, 290, 291, 294, 295, 306, 307, + 310, 311, 354, 355, 358, 359, 370, 371, 374, 375, 40, 41, 44, + 45, 56, 57, 60, 61, 104, 105, 108, 109, 120, 121, 124, 125, + 296, 297, 300, 301, 312, 313, 316, 317, 360, 361, 364, 365, 376, + 377, 380, 381, 42, 43, 46, 47, 58, 59, 62, 63, 106, 107, + 110, 111, 122, 123, 126, 127, 298, 299, 302, 303, 314, 315, 318, + 319, 362, 363, 366, 367, 378, 379, 382, 383, 128, 129, 132, 133, + 144, 145, 148, 149, 192, 193, 196, 197, 208, 209, 212, 213, 384, + 385, 388, 389, 400, 401, 404, 405, 448, 449, 452, 453, 464, 465, + 468, 469, 130, 131, 134, 135, 146, 147, 150, 151, 194, 195, 198, + 199, 210, 211, 214, 215, 386, 387, 390, 391, 402, 403, 406, 407, + 450, 451, 454, 455, 466, 467, 470, 471, 136, 137, 140, 141, 152, + 153, 156, 157, 200, 201, 204, 205, 216, 217, 220, 221, 392, 393, + 396, 397, 408, 409, 412, 413, 456, 457, 460, 461, 472, 473, 476, + 477, 138, 139, 142, 143, 154, 155, 158, 159, 202, 203, 206, 207, + 218, 219, 222, 223, 394, 395, 398, 399, 410, 411, 414, 415, 458, + 459, 462, 463, 474, 475, 478, 479, 160, 161, 164, 165, 176, 177, + 180, 181, 224, 225, 228, 229, 240, 241, 244, 245, 416, 417, 420, + 421, 432, 433, 436, 437, 480, 481, 484, 485, 496, 497, 500, 501, + 162, 163, 166, 167, 178, 179, 182, 183, 226, 227, 230, 231, 242, + 243, 246, 247, 418, 419, 422, 423, 434, 435, 438, 439, 482, 483, + 486, 487, 498, 499, 502, 503, 168, 169, 172, 173, 184, 185, 188, + 189, 232, 233, 236, 237, 248, 249, 252, 253, 424, 425, 428, 429, + 440, 441, 444, 445, 488, 489, 492, 493, 504, 505, 508, 509, 170, + 171, 174, 175, 186, 187, 190, 191, 234, 235, 238, 239, 250, 251, + 254, 255, 426, 427, 430, 431, 442, 443, 446, 447, 490, 491, 494, + 495, 506, 507, 510, 511, 512, 513, 516, 517, 528, 529, 532, 533, + 576, 577, 580, 581, 592, 593, 596, 597, 768, 769, 772, 773, 784, + 785, 788, 789, 832, 833, 836, 837, 848, 849, 852, 853, 514, 515, + 518, 519, 530, 531, 534, 535, 578, 579, 582, 583, 594, 595, 598, + 599, 770, 771, 774, 775, 786, 787, 790, 791, 834, 835, 838, 839, + 850, 851, 854, 855, 520, 521, 524, 525, 536, 537, 540, 541, 584, + 585, 588, 589, 600, 601, 604, 605, 776, 777, 780, 781, 792, 793, + 796, 797, 840, 841, 844, 845, 856, 857, 860, 861, 522, 523, 526, + 527, 538, 539, 542, 543, 586, 587, 590, 591, 602, 603, 606, 607, + 778, 779, 782, 783, 794, 795, 798, 799, 842, 843, 846, 847, 858, + 859, 862, 863, 544, 545, 548, 549, 560, 561, 564, 565, 608, 609, + 612, 613, 624, 625, 628, 629, 800, 801, 804, 805, 816, 817, 820, + 821, 864, 865, 868, 869, 880, 881, 884, 885, 546, 547, 550, 551, + 562, 563, 566, 567, 610, 611, 614, 615, 626, 627, 630, 631, 802, + 803, 806, 807, 818, 819, 822, 823, 866, 867, 870, 871, 882, 883, + 886, 887, 552, 553, 556, 557, 568, 569, 572, 573, 616, 617, 620, + 621, 632, 633, 636, 637, 808, 809, 812, 813, 824, 825, 828, 829, + 872, 873, 876, 877, 888, 889, 892, 893, 554, 555, 558, 559, 570, + 571, 574, 575, 618, 619, 622, 623, 634, 635, 638, 639, 810, 811, + 814, 815, 826, 827, 830, 831, 874, 875, 878, 879, 890, 891, 894, + 895, 640, 641, 644, 645, 656, 657, 660, 661, 704, 705, 708, 709, + 720, 721, 724, 725, 896, 897, 900, 901, 912, 913, 916, 917, 960, + 961, 964, 965, 976, 977, 980, 981, 642, 643, 646, 647, 658, 659, + 662, 663, 706, 707, 710, 711, 722, 723, 726, 727, 898, 899, 902, + 903, 914, 915, 918, 919, 962, 963, 966, 967, 978, 979, 982, 983, + 648, 649, 652, 653, 664, 665, 668, 669, 712, 713, 716, 717, 728, + 729, 732, 733, 904, 905, 908, 909, 920, 921, 924, 925, 968, 969, + 972, 973, 984, 985, 988, 989, 650, 651, 654, 655, 666, 667, 670, + 671, 714, 715, 718, 719, 730, 731, 734, 735, 906, 907, 910, 911, + 922, 923, 926, 927, 970, 971, 974, 975, 986, 987, 990, 991, 672, + 673, 676, 677, 688, 689, 692, 693, 736, 737, 740, 741, 752, 753, + 756, 757, 928, 929, 932, 933, 944, 945, 948, 949, 992, 993, 996, + 997, 1008, 1009, 1012, 1013, 674, 675, 678, 679, 690, 691, 694, 695, + 738, 739, 742, 743, 754, 755, 758, 759, 930, 931, 934, 935, 946, + 947, 950, 951, 994, 995, 998, 999, 1010, 1011, 1014, 1015, 680, 681, + 684, 685, 696, 697, 700, 701, 744, 745, 748, 749, 760, 761, 764, + 765, 936, 937, 940, 941, 952, 953, 956, 957, 1000, 1001, 1004, 1005, + 1016, 1017, 1020, 1021, 682, 683, 686, 687, 698, 699, 702, 703, 746, + 747, 750, 751, 762, 763, 766, 767, 938, 939, 942, 943, 954, 955, + 958, 959, 1002, 1003, 1006, 1007, 1018, 1019, 1022, 1023, +}; +#endif +#endif // CONFIG_CB4X4 || CONFIG_EXT_PARTITION + +#if CONFIG_EXT_PARTITION +/* clang-format off */ +static const uint16_t *const orders[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2 + orders_4x4, orders_4x4, orders_4x4, + // 4X4 + orders_4x4, + // 4X8, 8X4, 8X8 + orders_4x8, orders_8x4, orders_8x8, +#else + // 4X4 + orders_8x8, + // 4X8, 8X4, 8X8 + orders_8x8, orders_8x8, orders_8x8, +#endif + // 8X16, 16X8, 16X16 + orders_8x16, orders_16x8, orders_16x16, + // 16X32, 32X16, 32X32 + orders_16x32, orders_32x16, orders_32x32, + // 32X64, 64X32, 64X64 + orders_32x64, orders_64x32, orders_64x64, + // 64x128, 128x64, 128x128 + orders_64x128, orders_128x64, orders_128x128 +}; +/* clang-format on */ +#else +/* clang-format off */ +static const uint16_t *const orders[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2 + orders_8x8, orders_8x8, orders_8x8, + // 4X4 + orders_8x8, + // 4X8, 8X4, 8X8 + orders_8x16, orders_16x8, orders_16x16, +#else + // 4X4 + orders_16x16, + // 4X8, 8X4, 8X8 + orders_16x16, orders_16x16, orders_16x16, +#endif + // 8X16, 16X8, 16X16 + orders_16x32, orders_32x16, orders_32x32, + // 16X32, 32X16, 32X32 + orders_32x64, orders_64x32, orders_64x64, + // 32X64, 64X32, 64X64 + orders_64x128, orders_128x64, orders_128x128 +}; +/* clang-format on */ +#endif // CONFIG_EXT_PARTITION + +#if CONFIG_EXT_PARTITION_TYPES +static const uint16_t orders_verta_64x64[4] = { + 0, 2, 1, 2, +}; +static const uint16_t orders_verta_32x32[16] = { + 0, 2, 4, 6, 1, 2, 5, 6, 8, 10, 12, 14, 9, 10, 13, 14, +}; +static const uint16_t orders_verta_16x16[64] = { + 0, 2, 4, 6, 16, 18, 20, 22, 1, 2, 5, 6, 17, 18, 21, 22, + 8, 10, 12, 14, 24, 26, 28, 30, 9, 10, 13, 14, 25, 26, 29, 30, + 32, 34, 36, 38, 48, 50, 52, 54, 33, 34, 37, 38, 49, 50, 53, 54, + 40, 42, 44, 46, 56, 58, 60, 62, 41, 42, 45, 46, 57, 58, 61, 62, +}; +#if CONFIG_EXT_PARTITION || CONFIG_CB4X4 +static const uint16_t orders_verta_8x8[256] = { + 0, 2, 4, 6, 16, 18, 20, 22, 64, 66, 68, 70, 80, 82, 84, + 86, 1, 2, 5, 6, 17, 18, 21, 22, 65, 66, 69, 70, 81, 82, + 85, 86, 8, 10, 12, 14, 24, 26, 28, 30, 72, 74, 76, 78, 88, + 90, 92, 94, 9, 10, 13, 14, 25, 26, 29, 30, 73, 74, 77, 78, + 89, 90, 93, 94, 32, 34, 36, 38, 48, 50, 52, 54, 96, 98, 100, + 102, 112, 114, 116, 118, 33, 34, 37, 38, 49, 50, 53, 54, 97, 98, + 101, 102, 113, 114, 117, 118, 40, 42, 44, 46, 56, 58, 60, 62, 104, + 106, 108, 110, 120, 122, 124, 126, 41, 42, 45, 46, 57, 58, 61, 62, + 105, 106, 109, 110, 121, 122, 125, 126, 128, 130, 132, 134, 144, 146, 148, + 150, 192, 194, 196, 198, 208, 210, 212, 214, 129, 130, 133, 134, 145, 146, + 149, 150, 193, 194, 197, 198, 209, 210, 213, 214, 136, 138, 140, 142, 152, + 154, 156, 158, 200, 202, 204, 206, 216, 218, 220, 222, 137, 138, 141, 142, + 153, 154, 157, 158, 201, 202, 205, 206, 217, 218, 221, 222, 160, 162, 164, + 166, 176, 178, 180, 182, 224, 226, 228, 230, 240, 242, 244, 246, 161, 162, + 165, 166, 177, 178, 181, 182, 225, 226, 229, 230, 241, 242, 245, 246, 168, + 170, 172, 174, 184, 186, 188, 190, 232, 234, 236, 238, 248, 250, 252, 254, + 169, 170, 173, 174, 185, 186, 189, 190, 233, 234, 237, 238, 249, 250, 253, + 254, +}; +#endif // CONFIG_EXT_PARTITION || CONFIG_CB4X4 + +#if CONFIG_EXT_PARTITION +/* clang-format off */ +static const uint16_t *const orders_verta[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2 + orders_4x4, orders_4x4, orders_4x4, +#endif + // 4X4 + orders_verta_8x8, + // 4X8, 8X4, 8X8 + orders_verta_8x8, orders_verta_8x8, orders_verta_8x8, + // 8X16, 16X8, 16X16 + orders_8x16, orders_16x8, orders_verta_16x16, + // 16X32, 32X16, 32X32 + orders_16x32, orders_32x16, orders_verta_32x32, + // 32X64, 64X32, 64X64 + orders_32x64, orders_64x32, orders_verta_64x64, + // 64x128, 128x64, 128x128 + orders_64x128, orders_128x64, orders_128x128 +}; +/* clang-format on */ +#else +/* clang-format off */ +static const uint16_t *const orders_verta[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + // 2X2, 2X4, 4X2 + orders_verta_8x8, orders_verta_8x8, orders_verta_8x8, + // 4X4 + orders_verta_8x8, + // 4X8, 8X4, 8X8 + orders_verta_8x8, orders_verta_8x8, orders_verta_16x16, +#else + // 4X4 + orders_verta_16x16, + // 4X8, 8X4, 8X8 + orders_verta_16x16, orders_verta_16x16, orders_verta_16x16, +#endif + // 8X16, 16X8, 16X16 + orders_16x32, orders_32x16, orders_verta_32x32, + // 16X32, 32X16, 32X32 + orders_32x64, orders_64x32, orders_verta_64x64, + // 32X64, 64X32, 64X64 + orders_64x128, orders_128x64, orders_128x128 +}; +/* clang-format on */ +#endif // CONFIG_EXT_PARTITION +#endif // CONFIG_EXT_PARTITION_TYPES + +static int has_top_right(BLOCK_SIZE bsize, int mi_row, int mi_col, + int top_available, int right_available, +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_TYPE partition, +#endif + TX_SIZE txsz, int row_off, int col_off, int ss_x) { + if (!top_available || !right_available) return 0; + +#if !CONFIG_CB4X4 + // TODO(bshacklett, huisu): Currently the RD loop traverses 4X8 blocks in + // inverted N order while in the bitstream the subblocks are stored in Z + // order. This discrepancy makes this function incorrect when considering 4X8 + // blocks in the RD loop, so we disable the extended right edge for these + // blocks. The correct solution is to change the bitstream to store these + // blocks in inverted N order, and then update this function appropriately. + if (bsize == BLOCK_4X8 && row_off == 1) return 0; +#endif + + const int bw_unit = block_size_wide[bsize] >> tx_size_wide_log2[0]; + const int plane_bw_unit = AOMMAX(bw_unit >> ss_x, 1); + const int top_right_count_unit = tx_size_wide_unit[txsz]; + + // Special handling for block sizes 4x8 and 4x4. + if (ss_x == 0 && bw_unit < 2 && col_off == 0) return 1; + + if (row_off > 0) { // Just need to check if enough pixels on the right. + return col_off + top_right_count_unit < plane_bw_unit; + } else { + // All top-right pixels are in the block above, which is already available. + if (col_off + top_right_count_unit < plane_bw_unit) return 1; + + const int bw_in_mi_log2 = mi_width_log2_lookup[bsize]; + const int bh_in_mi_log2 = mi_height_log2_lookup[bsize]; + const int blk_row_in_sb = (mi_row & MAX_MIB_MASK) >> bh_in_mi_log2; + const int blk_col_in_sb = (mi_col & MAX_MIB_MASK) >> bw_in_mi_log2; + + // Top row of superblock: so top-right pixels are in the top and/or + // top-right superblocks, both of which are already available. + if (blk_row_in_sb == 0) return 1; + + // Rightmost column of superblock (and not the top row): so top-right pixels + // fall in the right superblock, which is not available yet. + if (((blk_col_in_sb + 1) << bw_in_mi_log2) >= MAX_MIB_SIZE) return 0; + + // General case (neither top row nor rightmost column): check if the + // top-right block is coded before the current block. + const uint16_t *const order = +#if CONFIG_EXT_PARTITION_TYPES + (partition == PARTITION_VERT_A) ? orders_verta[bsize] : +#endif // CONFIG_EXT_PARTITION_TYPES + orders[bsize]; + const int this_blk_index = + ((blk_row_in_sb + 0) << (MAX_MIB_SIZE_LOG2 - bw_in_mi_log2)) + + blk_col_in_sb + 0; + const uint16_t this_blk_order = order[this_blk_index]; + const int tr_blk_index = + ((blk_row_in_sb - 1) << (MAX_MIB_SIZE_LOG2 - bw_in_mi_log2)) + + blk_col_in_sb + 1; + const uint16_t tr_blk_order = order[tr_blk_index]; + return tr_blk_order < this_blk_order; + } +} + +static int has_bottom_left(BLOCK_SIZE bsize, int mi_row, int mi_col, + int bottom_available, int left_available, + TX_SIZE txsz, int row_off, int col_off, int ss_y) { + if (!bottom_available || !left_available) return 0; + + if (col_off > 0) { + // Bottom-left pixels are in the bottom-left block, which is not available. + return 0; + } else { + const int bh_unit = block_size_high[bsize] >> tx_size_high_log2[0]; + const int plane_bh_unit = AOMMAX(bh_unit >> ss_y, 1); + const int bottom_left_count_unit = tx_size_high_unit[txsz]; + +#if !CONFIG_CB4X4 + // Special handling for block sizes 8x4 and 4x4. + if (ss_y == 0 && bh_unit < 2 && row_off == 0) return 1; +#endif + + // All bottom-left pixels are in the left block, which is already available. + if (row_off + bottom_left_count_unit < plane_bh_unit) return 1; + + const int bw_in_mi_log2 = mi_width_log2_lookup[bsize]; + const int bh_in_mi_log2 = mi_height_log2_lookup[bsize]; + const int blk_row_in_sb = (mi_row & MAX_MIB_MASK) >> bh_in_mi_log2; + const int blk_col_in_sb = (mi_col & MAX_MIB_MASK) >> bw_in_mi_log2; + + // Leftmost column of superblock: so bottom-left pixels maybe in the left + // and/or bottom-left superblocks. But only the left superblock is + // available, so check if all required pixels fall in that superblock. + if (blk_col_in_sb == 0) { + const int blk_start_row_off = blk_row_in_sb << (bh_in_mi_log2 + !ss_y); + const int row_off_in_sb = blk_start_row_off + row_off; + const int sb_height_unit = MAX_MIB_SIZE << !ss_y; + return row_off_in_sb + bottom_left_count_unit < sb_height_unit; + } + + // Bottom row of superblock (and not the leftmost column): so bottom-left + // pixels fall in the bottom superblock, which is not available yet. + if (((blk_row_in_sb + 1) << bh_in_mi_log2) >= MAX_MIB_SIZE) return 0; + + // General case (neither leftmost column nor bottom row): check if the + // bottom-left block is coded before the current block. + const uint16_t *const order = orders[bsize]; + const int this_blk_index = + ((blk_row_in_sb + 0) << (MAX_MIB_SIZE_LOG2 - bw_in_mi_log2)) + + blk_col_in_sb + 0; + const uint16_t this_blk_order = order[this_blk_index]; + const int bl_blk_index = + ((blk_row_in_sb + 1) << (MAX_MIB_SIZE_LOG2 - bw_in_mi_log2)) + + blk_col_in_sb - 1; + const uint16_t bl_blk_order = order[bl_blk_index]; + return bl_blk_order < this_blk_order; + } +} + +typedef void (*intra_pred_fn)(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left); + +static intra_pred_fn pred[INTRA_MODES][TX_SIZES]; +static intra_pred_fn dc_pred[2][2][TX_SIZES]; + +#if CONFIG_HIGHBITDEPTH +typedef void (*intra_high_pred_fn)(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, const uint16_t *left, + int bd); +static intra_high_pred_fn pred_high[INTRA_MODES][TX_SIZES]; +static intra_high_pred_fn dc_pred_high[2][2][TX_SIZES]; +#endif // CONFIG_HIGHBITDEPTH + +static void av1_init_intra_predictors_internal(void) { +#if CONFIG_TX64X64 +#define INIT_NO_4X4(p, type) \ + p[TX_8X8] = aom_##type##_predictor_8x8; \ + p[TX_16X16] = aom_##type##_predictor_16x16; \ + p[TX_32X32] = aom_##type##_predictor_32x32; \ + p[TX_64X64] = aom_##type##_predictor_64x64 +#else +#define INIT_NO_4X4(p, type) \ + p[TX_8X8] = aom_##type##_predictor_8x8; \ + p[TX_16X16] = aom_##type##_predictor_16x16; \ + p[TX_32X32] = aom_##type##_predictor_32x32 +#endif // CONFIG_TX64X64 + +#if CONFIG_CB4X4 +#define INIT_ALL_SIZES(p, type) \ + p[TX_2X2] = aom_##type##_predictor_2x2; \ + p[TX_4X4] = aom_##type##_predictor_4x4; \ + INIT_NO_4X4(p, type) +#else +#define INIT_ALL_SIZES(p, type) \ + p[TX_4X4] = aom_##type##_predictor_4x4; \ + INIT_NO_4X4(p, type) +#endif + + INIT_ALL_SIZES(pred[V_PRED], v); + INIT_ALL_SIZES(pred[H_PRED], h); + INIT_ALL_SIZES(pred[D207_PRED], d207e); + INIT_ALL_SIZES(pred[D45_PRED], d45e); + INIT_ALL_SIZES(pred[D63_PRED], d63e); + INIT_ALL_SIZES(pred[D117_PRED], d117); + INIT_ALL_SIZES(pred[D135_PRED], d135); + INIT_ALL_SIZES(pred[D153_PRED], d153); + +#if CONFIG_ALT_INTRA + INIT_ALL_SIZES(pred[TM_PRED], paeth); + INIT_ALL_SIZES(pred[SMOOTH_PRED], smooth); +#else + INIT_ALL_SIZES(pred[TM_PRED], tm); +#endif // CONFIG_ALT_INTRA + + INIT_ALL_SIZES(dc_pred[0][0], dc_128); + INIT_ALL_SIZES(dc_pred[0][1], dc_top); + INIT_ALL_SIZES(dc_pred[1][0], dc_left); + INIT_ALL_SIZES(dc_pred[1][1], dc); + +#if CONFIG_HIGHBITDEPTH + INIT_ALL_SIZES(pred_high[V_PRED], highbd_v); + INIT_ALL_SIZES(pred_high[H_PRED], highbd_h); + INIT_ALL_SIZES(pred_high[D207_PRED], highbd_d207e); + INIT_ALL_SIZES(pred_high[D45_PRED], highbd_d45e); + INIT_ALL_SIZES(pred_high[D63_PRED], highbd_d63e); + INIT_ALL_SIZES(pred_high[D117_PRED], highbd_d117); + INIT_ALL_SIZES(pred_high[D135_PRED], highbd_d135); + INIT_ALL_SIZES(pred_high[D153_PRED], highbd_d153); + +#if CONFIG_ALT_INTRA + INIT_ALL_SIZES(pred_high[TM_PRED], highbd_paeth); + INIT_ALL_SIZES(pred_high[SMOOTH_PRED], highbd_smooth); +#else + INIT_ALL_SIZES(pred_high[TM_PRED], highbd_tm); +#endif // CONFIG_ALT_INTRA + + INIT_ALL_SIZES(dc_pred_high[0][0], highbd_dc_128); + INIT_ALL_SIZES(dc_pred_high[0][1], highbd_dc_top); + INIT_ALL_SIZES(dc_pred_high[1][0], highbd_dc_left); + INIT_ALL_SIZES(dc_pred_high[1][1], highbd_dc); +#endif // CONFIG_HIGHBITDEPTH + +#undef intra_pred_allsizes +} + +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP +static int intra_subpel_interp(int base, int shift, const uint8_t *ref, + int ref_start_idx, int ref_end_idx, + INTRA_FILTER filter_type) { + int val, k, idx, filter_idx = 0; + const int16_t *filter = NULL; + + if (filter_type == INTRA_FILTER_LINEAR) { + val = ref[base] * (256 - shift) + ref[base + 1] * shift; + val = ROUND_POWER_OF_TWO(val, 8); + } else { + filter_idx = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS); + filter = av1_intra_filter_kernels[filter_type][filter_idx]; + + if (filter_idx < (1 << SUBPEL_BITS)) { + val = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) { + idx = base + 1 - (SUBPEL_TAPS / 2) + k; + idx = AOMMAX(AOMMIN(idx, ref_end_idx), ref_start_idx); + val += ref[idx] * filter[k]; + } + val = ROUND_POWER_OF_TWO(val, FILTER_BITS); + } else { + val = ref[base + 1]; + } + } + + return val; +} +#endif // CONFIG_INTRA_INTERP + +// Directional prediction, zone 1: 0 < angle < 90 +static void dr_prediction_z1(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left, +#if CONFIG_INTRA_INTERP + INTRA_FILTER filter_type, +#endif // CONFIG_INTRA_INTERP + int dx, int dy) { + int r, c, x, base, shift, val; + + (void)left; + (void)dy; + assert(dy == 1); + assert(dx > 0); + +#if CONFIG_INTRA_INTERP + if (filter_type != INTRA_FILTER_LINEAR) { + const int pad_size = SUBPEL_TAPS >> 1; + int len; + DECLARE_ALIGNED(16, uint8_t, buf[SUBPEL_SHIFTS][MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint8_t, src[MAX_SB_SIZE + SUBPEL_TAPS]); + uint8_t flags[SUBPEL_SHIFTS]; + + memset(flags, 0, SUBPEL_SHIFTS * sizeof(flags[0])); + memset(src, above[0], pad_size * sizeof(above[0])); + memcpy(src + pad_size, above, 2 * bs * sizeof(above[0])); + memset(src + pad_size + 2 * bs, above[2 * bs - 1], + pad_size * sizeof(above[0])); + flags[0] = 1; + x = dx; + for (r = 0; r < bs; ++r, dst += stride, x += dx) { + base = x >> 8; + shift = x & 0xFF; + shift = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS); + if (shift == SUBPEL_SHIFTS) { + base += 1; + shift = 0; + } + len = AOMMIN(bs, 2 * bs - 1 - base); + if (len <= 0) { + int i; + for (i = r; i < bs; ++i) { + memset(dst, above[2 * bs - 1], bs * sizeof(dst[0])); + dst += stride; + } + return; + } + + if (len <= (bs >> 1) && !flags[shift]) { + base = x >> 8; + shift = x & 0xFF; + for (c = 0; c < len; ++c) { + val = intra_subpel_interp(base, shift, above, 0, 2 * bs - 1, + filter_type); + dst[c] = clip_pixel(val); + ++base; + } + } else { + if (!flags[shift]) { + const int16_t *filter = av1_intra_filter_kernels[filter_type][shift]; + aom_convolve8_horiz(src + pad_size, 2 * bs, buf[shift], 2 * bs, + filter, 16, NULL, 16, 2 * bs, + 2 * bs < 16 ? 2 : 1); + flags[shift] = 1; + } + memcpy(dst, shift == 0 ? src + pad_size + base : &buf[shift][base], + len * sizeof(dst[0])); + } + + if (len < bs) + memset(dst + len, above[2 * bs - 1], (bs - len) * sizeof(dst[0])); + } + return; + } +#endif // CONFIG_INTRA_INTERP + + x = dx; + for (r = 0; r < bs; ++r, dst += stride, x += dx) { + base = x >> 8; + shift = x & 0xFF; + + if (base >= 2 * bs - 1) { + int i; + for (i = r; i < bs; ++i) { + memset(dst, above[2 * bs - 1], bs * sizeof(dst[0])); + dst += stride; + } + return; + } + + for (c = 0; c < bs; ++c, ++base) { + if (base < 2 * bs - 1) { + val = above[base] * (256 - shift) + above[base + 1] * shift; + val = ROUND_POWER_OF_TWO(val, 8); + dst[c] = clip_pixel(val); + } else { + dst[c] = above[2 * bs - 1]; + } + } + } +} + +// Directional prediction, zone 2: 90 < angle < 180 +static void dr_prediction_z2(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left, +#if CONFIG_INTRA_INTERP + INTRA_FILTER filter_type, +#endif // CONFIG_INTRA_INTERP + int dx, int dy) { + int r, c, x, y, shift1, shift2, val, base1, base2; + + assert(dx > 0); + assert(dy > 0); + + x = -dx; + for (r = 0; r < bs; ++r, x -= dx, dst += stride) { + base1 = x >> 8; + y = (r << 8) - dy; + for (c = 0; c < bs; ++c, ++base1, y -= dy) { + if (base1 >= -1) { + shift1 = x & 0xFF; +#if CONFIG_INTRA_INTERP + val = + intra_subpel_interp(base1, shift1, above, -1, bs - 1, filter_type); +#else + val = above[base1] * (256 - shift1) + above[base1 + 1] * shift1; + val = ROUND_POWER_OF_TWO(val, 8); +#endif // CONFIG_INTRA_INTERP + } else { + base2 = y >> 8; + shift2 = y & 0xFF; +#if CONFIG_INTRA_INTERP + val = intra_subpel_interp(base2, shift2, left, -1, bs - 1, filter_type); +#else + val = left[base2] * (256 - shift2) + left[base2 + 1] * shift2; + val = ROUND_POWER_OF_TWO(val, 8); +#endif // CONFIG_INTRA_INTERP + } + dst[c] = clip_pixel(val); + } + } +} + +// Directional prediction, zone 3: 180 < angle < 270 +static void dr_prediction_z3(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left, +#if CONFIG_INTRA_INTERP + INTRA_FILTER filter_type, +#endif // CONFIG_INTRA_INTERP + int dx, int dy) { + int r, c, y, base, shift, val; + + (void)above; + (void)dx; + + assert(dx == 1); + assert(dy > 0); + +#if CONFIG_INTRA_INTERP + if (filter_type != INTRA_FILTER_LINEAR) { + const int pad_size = SUBPEL_TAPS >> 1; + int len, i; + DECLARE_ALIGNED(16, uint8_t, buf[MAX_SB_SIZE][4 * SUBPEL_SHIFTS]); + DECLARE_ALIGNED(16, uint8_t, src[(MAX_SB_SIZE + SUBPEL_TAPS) * 4]); + uint8_t flags[SUBPEL_SHIFTS]; + + memset(flags, 0, SUBPEL_SHIFTS * sizeof(flags[0])); + for (i = 0; i < pad_size; ++i) src[4 * i] = left[0]; + for (i = 0; i < 2 * bs; ++i) src[4 * (i + pad_size)] = left[i]; + for (i = 0; i < pad_size; ++i) + src[4 * (i + 2 * bs + pad_size)] = left[2 * bs - 1]; + flags[0] = 1; + y = dy; + for (c = 0; c < bs; ++c, y += dy) { + base = y >> 8; + shift = y & 0xFF; + shift = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS); + if (shift == SUBPEL_SHIFTS) { + base += 1; + shift = 0; + } + len = AOMMIN(bs, 2 * bs - 1 - base); + + if (len <= 0) { + for (r = 0; r < bs; ++r) { + dst[r * stride + c] = left[2 * bs - 1]; + } + continue; + } + + if (len <= (bs >> 1) && !flags[shift]) { + base = y >> 8; + shift = y & 0xFF; + for (r = 0; r < len; ++r) { + val = intra_subpel_interp(base, shift, left, 0, 2 * bs - 1, + filter_type); + dst[r * stride + c] = clip_pixel(val); + ++base; + } + } else { + if (!flags[shift]) { + const int16_t *filter = av1_intra_filter_kernels[filter_type][shift]; + aom_convolve8_vert(src + 4 * pad_size, 4, buf[0] + 4 * shift, + 4 * SUBPEL_SHIFTS, NULL, 16, filter, 16, + 2 * bs < 16 ? 4 : 4, 2 * bs); + flags[shift] = 1; + } + + if (shift == 0) { + for (r = 0; r < len; ++r) { + dst[r * stride + c] = left[r + base]; + } + } else { + for (r = 0; r < len; ++r) { + dst[r * stride + c] = buf[r + base][4 * shift]; + } + } + } + + if (len < bs) { + for (r = len; r < bs; ++r) { + dst[r * stride + c] = left[2 * bs - 1]; + } + } + } + return; + } +#endif // CONFIG_INTRA_INTERP + + y = dy; + for (c = 0; c < bs; ++c, y += dy) { + base = y >> 8; + shift = y & 0xFF; + + for (r = 0; r < bs; ++r, ++base) { + if (base < 2 * bs - 1) { + val = left[base] * (256 - shift) + left[base + 1] * shift; + val = ROUND_POWER_OF_TWO(val, 8); + dst[r * stride + c] = clip_pixel(val); + } else { + for (; r < bs; ++r) dst[r * stride + c] = left[2 * bs - 1]; + break; + } + } + } +} + +// Get the shift (up-scaled by 256) in X w.r.t a unit change in Y. +// If angle > 0 && angle < 90, dx = -((int)(256 / t)); +// If angle > 90 && angle < 180, dx = (int)(256 / t); +// If angle > 180 && angle < 270, dx = 1; +static INLINE int get_dx(int angle) { + if (angle > 0 && angle < 90) { + return dr_intra_derivative[angle]; + } else if (angle > 90 && angle < 180) { + return dr_intra_derivative[180 - angle]; + } else { + // In this case, we are not really going to use dx. We may return any value. + return 1; + } +} + +// Get the shift (up-scaled by 256) in Y w.r.t a unit change in X. +// If angle > 0 && angle < 90, dy = 1; +// If angle > 90 && angle < 180, dy = (int)(256 * t); +// If angle > 180 && angle < 270, dy = -((int)(256 * t)); +static INLINE int get_dy(int angle) { + if (angle > 90 && angle < 180) { + return dr_intra_derivative[angle - 90]; + } else if (angle > 180 && angle < 270) { + return dr_intra_derivative[270 - angle]; + } else { + // In this case, we are not really going to use dy. We may return any value. + return 1; + } +} + +static void dr_predictor(uint8_t *dst, ptrdiff_t stride, TX_SIZE tx_size, + const uint8_t *above, const uint8_t *left, +#if CONFIG_INTRA_INTERP + INTRA_FILTER filter_type, +#endif // CONFIG_INTRA_INTERP + int angle) { + const int dx = get_dx(angle); + const int dy = get_dy(angle); + const int bs = tx_size_wide[tx_size]; + assert(angle > 0 && angle < 270); + + if (angle > 0 && angle < 90) { + dr_prediction_z1(dst, stride, bs, above, left, +#if CONFIG_INTRA_INTERP + filter_type, +#endif // CONFIG_INTRA_INTERP + dx, dy); + } else if (angle > 90 && angle < 180) { + dr_prediction_z2(dst, stride, bs, above, left, +#if CONFIG_INTRA_INTERP + filter_type, +#endif // CONFIG_INTRA_INTERP + dx, dy); + } else if (angle > 180 && angle < 270) { + dr_prediction_z3(dst, stride, bs, above, left, +#if CONFIG_INTRA_INTERP + filter_type, +#endif // CONFIG_INTRA_INTERP + dx, dy); + } else if (angle == 90) { + pred[V_PRED][tx_size](dst, stride, above, left); + } else if (angle == 180) { + pred[H_PRED][tx_size](dst, stride, above, left); + } +} + +#if CONFIG_HIGHBITDEPTH +#if CONFIG_INTRA_INTERP +static int highbd_intra_subpel_interp(int base, int shift, const uint16_t *ref, + int ref_start_idx, int ref_end_idx, + INTRA_FILTER filter_type) { + int val, k, idx, filter_idx = 0; + const int16_t *filter = NULL; + + if (filter_type == INTRA_FILTER_LINEAR) { + val = ref[base] * (256 - shift) + ref[base + 1] * shift; + val = ROUND_POWER_OF_TWO(val, 8); + } else { + filter_idx = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS); + filter = av1_intra_filter_kernels[filter_type][filter_idx]; + + if (filter_idx < (1 << SUBPEL_BITS)) { + val = 0; + for (k = 0; k < SUBPEL_TAPS; ++k) { + idx = base + 1 - (SUBPEL_TAPS / 2) + k; + idx = AOMMAX(AOMMIN(idx, ref_end_idx), ref_start_idx); + val += ref[idx] * filter[k]; + } + val = ROUND_POWER_OF_TWO(val, FILTER_BITS); + } else { + val = ref[base + 1]; + } + } + + return val; +} +#endif // CONFIG_INTRA_INTERP + +// Directional prediction, zone 1: 0 < angle < 90 +static void highbd_dr_prediction_z1(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, const uint16_t *left, +#if CONFIG_INTRA_INTERP + INTRA_FILTER filter_type, +#endif // CONFIG_INTRA_INTERP + int dx, int dy, int bd) { + int r, c, x, base, shift, val; + + (void)left; + (void)dy; + assert(dy == 1); + assert(dx > 0); + + x = dx; + for (r = 0; r < bs; ++r, dst += stride, x += dx) { + base = x >> 8; + shift = x & 0xFF; + + if (base >= 2 * bs - 1) { + int i; + for (i = r; i < bs; ++i) { + aom_memset16(dst, above[2 * bs - 1], bs); + dst += stride; + } + return; + } + + for (c = 0; c < bs; ++c, ++base) { + if (base < 2 * bs - 1) { +#if CONFIG_INTRA_INTERP + val = highbd_intra_subpel_interp(base, shift, above, 0, 2 * bs - 1, + filter_type); +#else + val = above[base] * (256 - shift) + above[base + 1] * shift; + val = ROUND_POWER_OF_TWO(val, 8); +#endif // CONFIG_INTRA_INTERP + dst[c] = clip_pixel_highbd(val, bd); + } else { + dst[c] = above[2 * bs - 1]; + } + } + } +} + +// Directional prediction, zone 2: 90 < angle < 180 +static void highbd_dr_prediction_z2(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, const uint16_t *left, +#if CONFIG_INTRA_INTERP + INTRA_FILTER filter_type, +#endif // CONFIG_INTRA_INTERP + int dx, int dy, int bd) { + int r, c, x, y, shift, val, base; + + assert(dx > 0); + assert(dy > 0); + + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) { + y = r + 1; + x = (c << 8) - y * dx; + base = x >> 8; + if (base >= -1) { + shift = x & 0xFF; +#if CONFIG_INTRA_INTERP + val = highbd_intra_subpel_interp(base, shift, above, -1, bs - 1, + filter_type); +#else + val = above[base] * (256 - shift) + above[base + 1] * shift; + val = ROUND_POWER_OF_TWO(val, 8); +#endif // CONFIG_INTRA_INTERP + } else { + x = c + 1; + y = (r << 8) - x * dy; + base = y >> 8; + shift = y & 0xFF; +#if CONFIG_INTRA_INTERP + val = highbd_intra_subpel_interp(base, shift, left, -1, bs - 1, + filter_type); +#else + val = left[base] * (256 - shift) + left[base + 1] * shift; + val = ROUND_POWER_OF_TWO(val, 8); +#endif // CONFIG_INTRA_INTERP + } + dst[c] = clip_pixel_highbd(val, bd); + } + dst += stride; + } +} + +// Directional prediction, zone 3: 180 < angle < 270 +static void highbd_dr_prediction_z3(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, const uint16_t *left, +#if CONFIG_INTRA_INTERP + INTRA_FILTER filter_type, +#endif // CONFIG_INTRA_INTERP + int dx, int dy, int bd) { + int r, c, y, base, shift, val; + + (void)above; + (void)dx; + assert(dx == 1); + assert(dy > 0); + + y = dy; + for (c = 0; c < bs; ++c, y += dy) { + base = y >> 8; + shift = y & 0xFF; + + for (r = 0; r < bs; ++r, ++base) { + if (base < 2 * bs - 1) { +#if CONFIG_INTRA_INTERP + val = highbd_intra_subpel_interp(base, shift, left, 0, 2 * bs - 1, + filter_type); +#else + val = left[base] * (256 - shift) + left[base + 1] * shift; + val = ROUND_POWER_OF_TWO(val, 8); +#endif // CONFIG_INTRA_INTERP + dst[r * stride + c] = clip_pixel_highbd(val, bd); + } else { + for (; r < bs; ++r) dst[r * stride + c] = left[2 * bs - 1]; + break; + } + } + } +} + +static INLINE void highbd_v_predictor(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { + int r; + (void)left; + (void)bd; + for (r = 0; r < bs; r++) { + memcpy(dst, above, bs * sizeof(uint16_t)); + dst += stride; + } +} + +static INLINE void highbd_h_predictor(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { + int r; + (void)above; + (void)bd; + for (r = 0; r < bs; r++) { + aom_memset16(dst, left[r], bs); + dst += stride; + } +} + +static void highbd_dr_predictor(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, const uint16_t *left, +#if CONFIG_INTRA_INTERP + INTRA_FILTER filter, +#endif // CONFIG_INTRA_INTERP + int angle, int bd) { + const int dx = get_dx(angle); + const int dy = get_dy(angle); + assert(angle > 0 && angle < 270); + + if (angle > 0 && angle < 90) { + highbd_dr_prediction_z1(dst, stride, bs, above, left, +#if CONFIG_INTRA_INTERP + filter, +#endif // CONFIG_INTRA_INTERP + dx, dy, bd); + } else if (angle > 90 && angle < 180) { + highbd_dr_prediction_z2(dst, stride, bs, above, left, +#if CONFIG_INTRA_INTERP + filter, +#endif // CONFIG_INTRA_INTERP + dx, dy, bd); + } else if (angle > 180 && angle < 270) { + highbd_dr_prediction_z3(dst, stride, bs, above, left, +#if CONFIG_INTRA_INTERP + filter, +#endif // CONFIG_INTRA_INTERP + dx, dy, bd); + } else if (angle == 90) { + highbd_v_predictor(dst, stride, bs, above, left, bd); + } else if (angle == 180) { + highbd_h_predictor(dst, stride, bs, above, left, bd); + } +} +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_EXT_INTRA + +#if CONFIG_FILTER_INTRA +#if USE_3TAP_INTRA_FILTER +int av1_filter_intra_taps_3[TX_SIZES][INTRA_MODES][3] = { +#if CONFIG_CB4X4 + { + { 697, 836, -509 }, + { 993, 513, -482 }, + { 381, 984, -341 }, + { 642, 1169, -787 }, + { 590, 553, -119 }, + { 762, 385, -123 }, + { 358, 687, -21 }, + { 411, 1083, -470 }, + { 912, 814, -702 }, + { 883, 902, 761 }, + }, +#endif + { + { 697, 836, -509 }, + { 993, 513, -482 }, + { 381, 984, -341 }, + { 642, 1169, -787 }, + { 590, 553, -119 }, + { 762, 385, -123 }, + { 358, 687, -21 }, + { 411, 1083, -470 }, + { 912, 814, -702 }, + { 883, 902, 761 }, + }, + { + { 659, 816, -451 }, + { 980, 625, -581 }, + { 558, 962, -496 }, + { 681, 888, -545 }, + { 591, 613, 180 }, + { 778, 399, -153 }, + { 495, 641, -112 }, + { 671, 937, -584 }, + { 745, 940, -661 }, + { 839, 911, -726 }, + }, + { + { 539, 927, -442 }, + { 1003, 714, -693 }, + { 349, 1271, -596 }, + { 820, 764, -560 }, + { 524, 816, -316 }, + { 780, 681, -437 }, + { 586, 795, -357 }, + { 551, 1135, -663 }, + { 593, 1061, -630 }, + { 974, 970, -920 }, + }, + { + { 595, 919, -490 }, + { 945, 668, -579 }, + { 495, 962, -433 }, + { 385, 1551, -912 }, + { 455, 554, 15 }, + { 852, 478, -306 }, + { 177, 760, -87 }, + { -65, 1611, -522 }, + { 815, 894, -685 }, + { 846, 1010, -832 }, + }, +#if CONFIG_TX64X64 + { + { 595, 919, -490 }, + { 945, 668, -579 }, + { 495, 962, -433 }, + { 385, 1551, -912 }, + { 455, 554, 15 }, + { 852, 478, -306 }, + { 177, 760, -87 }, + { -65, 1611, -522 }, + { 815, 894, -685 }, + { 846, 1010, -832 }, + }, +#endif // CONFIG_TX64X64 +}; +#else +int av1_filter_intra_taps_4[TX_SIZES][INTRA_MODES][4] = { +#if CONFIG_CB4X4 + { + { 735, 881, -537, -54 }, + { 1005, 519, -488, -11 }, + { 383, 990, -343, -6 }, + { 442, 805, -542, 319 }, + { 658, 616, -133, -116 }, + { 875, 442, -141, -151 }, + { 386, 741, -23, -80 }, + { 390, 1027, -446, 51 }, + { 679, 606, -523, 262 }, + { 903, 922, -778, -23 }, + }, +#endif + { + { 735, 881, -537, -54 }, + { 1005, 519, -488, -11 }, + { 383, 990, -343, -6 }, + { 442, 805, -542, 319 }, + { 658, 616, -133, -116 }, + { 875, 442, -141, -151 }, + { 386, 741, -23, -80 }, + { 390, 1027, -446, 51 }, + { 679, 606, -523, 262 }, + { 903, 922, -778, -23 }, + }, + { + { 648, 803, -444, 16 }, + { 972, 620, -576, 7 }, + { 561, 967, -499, -5 }, + { 585, 762, -468, 144 }, + { 596, 619, -182, -9 }, + { 895, 459, -176, -153 }, + { 557, 722, -126, -129 }, + { 601, 839, -523, 105 }, + { 562, 709, -499, 251 }, + { 803, 872, -695, 43 }, + }, + { + { 423, 728, -347, 111 }, + { 963, 685, -665, 23 }, + { 281, 1024, -480, 216 }, + { 640, 596, -437, 78 }, + { 429, 669, -259, 99 }, + { 740, 646, -415, 23 }, + { 568, 771, -346, 40 }, + { 404, 833, -486, 209 }, + { 398, 712, -423, 307 }, + { 939, 935, -887, 17 }, + }, + { + { 477, 737, -393, 150 }, + { 881, 630, -546, 67 }, + { 506, 984, -443, -20 }, + { 114, 459, -270, 528 }, + { 433, 528, 14, 3 }, + { 837, 470, -301, -30 }, + { 181, 777, 89, -107 }, + { -29, 716, -232, 259 }, + { 589, 646, -495, 255 }, + { 740, 884, -728, 77 }, + }, +#if CONFIG_TX64X64 + { + { 477, 737, -393, 150 }, + { 881, 630, -546, 67 }, + { 506, 984, -443, -20 }, + { 114, 459, -270, 528 }, + { 433, 528, 14, 3 }, + { 837, 470, -301, -30 }, + { 181, 777, 89, -107 }, + { -29, 716, -232, 259 }, + { 589, 646, -495, 255 }, + { 740, 884, -728, 77 }, + }, +#endif // CONFIG_TX64X64 +}; +#endif + +static INLINE TX_SIZE get_txsize_from_blocklen(int bs) { + switch (bs) { + case 4: return TX_4X4; + case 8: return TX_8X8; + case 16: return TX_16X16; + case 32: return TX_32X32; +#if CONFIG_TX64X64 + case 64: return TX_64X64; +#endif // CONFIG_TX64X64 + default: assert(0); return TX_INVALID; + } +} + +#if USE_3TAP_INTRA_FILTER +static void filter_intra_predictors_3tap(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, + const uint8_t *left, int mode) { + int k, r, c; + int mean, ipred; +#if CONFIG_TX64X64 + int buffer[65][65]; +#else + int buffer[33][33]; +#endif // CONFIG_TX64X64 + const TX_SIZE tx_size = get_txsize_from_blocklen(bs); + const int c0 = av1_filter_intra_taps_3[tx_size][mode][0]; + const int c1 = av1_filter_intra_taps_3[tx_size][mode][1]; + const int c2 = av1_filter_intra_taps_3[tx_size][mode][2]; + + k = 0; + mean = 0; + while (k < bs) { + mean = mean + (int)left[k]; + mean = mean + (int)above[k]; + k++; + } + mean = (mean + bs) / (2 * bs); + + for (r = 0; r < bs; ++r) buffer[r + 1][0] = (int)left[r] - mean; + + for (c = 0; c < bs + 1; ++c) buffer[0][c] = (int)above[c - 1] - mean; + + for (r = 1; r < bs + 1; ++r) + for (c = 1; c < bs + 1; ++c) { + ipred = c0 * buffer[r - 1][c] + c1 * buffer[r][c - 1] + + c2 * buffer[r - 1][c - 1]; + buffer[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_PREC_BITS); + } + + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) { + ipred = buffer[r + 1][c + 1] + mean; + dst[c] = clip_pixel(ipred); + } + dst += stride; + } +} +#else +static void filter_intra_predictors_4tap(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, + const uint8_t *left, int mode) { + int k, r, c; + int mean, ipred; +#if CONFIG_TX64X64 + int buffer[65][129]; +#else + int buffer[33][65]; +#endif // CONFIG_TX64X64 + const TX_SIZE tx_size = get_txsize_from_blocklen(bs); + const int c0 = av1_filter_intra_taps_4[tx_size][mode][0]; + const int c1 = av1_filter_intra_taps_4[tx_size][mode][1]; + const int c2 = av1_filter_intra_taps_4[tx_size][mode][2]; + const int c3 = av1_filter_intra_taps_4[tx_size][mode][3]; + + k = 0; + mean = 0; + while (k < bs) { + mean = mean + (int)left[k]; + mean = mean + (int)above[k]; + k++; + } + mean = (mean + bs) / (2 * bs); + + for (r = 0; r < bs; ++r) buffer[r + 1][0] = (int)left[r] - mean; + + for (c = 0; c < 2 * bs + 1; ++c) buffer[0][c] = (int)above[c - 1] - mean; + + for (r = 1; r < bs + 1; ++r) + for (c = 1; c < 2 * bs + 1 - r; ++c) { + ipred = c0 * buffer[r - 1][c] + c1 * buffer[r][c - 1] + + c2 * buffer[r - 1][c - 1] + c3 * buffer[r - 1][c + 1]; + buffer[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_PREC_BITS); + } + + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) { + ipred = buffer[r + 1][c + 1] + mean; + dst[c] = clip_pixel(ipred); + } + dst += stride; + } +} +#endif + +void av1_dc_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { +#if USE_3TAP_INTRA_FILTER + filter_intra_predictors_3tap(dst, stride, bs, above, left, DC_PRED); +#else + filter_intra_predictors_4tap(dst, stride, bs, above, left, DC_PRED); +#endif +} + +void av1_v_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { +#if USE_3TAP_INTRA_FILTER + filter_intra_predictors_3tap(dst, stride, bs, above, left, V_PRED); +#else + filter_intra_predictors_4tap(dst, stride, bs, above, left, V_PRED); +#endif +} + +void av1_h_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { +#if USE_3TAP_INTRA_FILTER + filter_intra_predictors_3tap(dst, stride, bs, above, left, H_PRED); +#else + filter_intra_predictors_4tap(dst, stride, bs, above, left, H_PRED); +#endif +} + +void av1_d45_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { +#if USE_3TAP_INTRA_FILTER + filter_intra_predictors_3tap(dst, stride, bs, above, left, D45_PRED); +#else + filter_intra_predictors_4tap(dst, stride, bs, above, left, D45_PRED); +#endif +} + +void av1_d135_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { +#if USE_3TAP_INTRA_FILTER + filter_intra_predictors_3tap(dst, stride, bs, above, left, D135_PRED); +#else + filter_intra_predictors_4tap(dst, stride, bs, above, left, D135_PRED); +#endif +} + +void av1_d117_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { +#if USE_3TAP_INTRA_FILTER + filter_intra_predictors_3tap(dst, stride, bs, above, left, D117_PRED); +#else + filter_intra_predictors_4tap(dst, stride, bs, above, left, D117_PRED); +#endif +} + +void av1_d153_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { +#if USE_3TAP_INTRA_FILTER + filter_intra_predictors_3tap(dst, stride, bs, above, left, D153_PRED); +#else + filter_intra_predictors_4tap(dst, stride, bs, above, left, D153_PRED); +#endif +} + +void av1_d207_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { +#if USE_3TAP_INTRA_FILTER + filter_intra_predictors_3tap(dst, stride, bs, above, left, D207_PRED); +#else + filter_intra_predictors_4tap(dst, stride, bs, above, left, D207_PRED); +#endif +} + +void av1_d63_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { +#if USE_3TAP_INTRA_FILTER + filter_intra_predictors_3tap(dst, stride, bs, above, left, D63_PRED); +#else + filter_intra_predictors_4tap(dst, stride, bs, above, left, D63_PRED); +#endif +} + +void av1_tm_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { +#if USE_3TAP_INTRA_FILTER + filter_intra_predictors_3tap(dst, stride, bs, above, left, TM_PRED); +#else + filter_intra_predictors_4tap(dst, stride, bs, above, left, TM_PRED); +#endif +} + +static void filter_intra_predictors(FILTER_INTRA_MODE mode, uint8_t *dst, + ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + switch (mode) { + case FILTER_DC_PRED: + av1_dc_filter_predictor(dst, stride, bs, above, left); + break; + case FILTER_V_PRED: + av1_v_filter_predictor(dst, stride, bs, above, left); + break; + case FILTER_H_PRED: + av1_h_filter_predictor(dst, stride, bs, above, left); + break; + case FILTER_D45_PRED: + av1_d45_filter_predictor(dst, stride, bs, above, left); + break; + case FILTER_D135_PRED: + av1_d135_filter_predictor(dst, stride, bs, above, left); + break; + case FILTER_D117_PRED: + av1_d117_filter_predictor(dst, stride, bs, above, left); + break; + case FILTER_D153_PRED: + av1_d153_filter_predictor(dst, stride, bs, above, left); + break; + case FILTER_D207_PRED: + av1_d207_filter_predictor(dst, stride, bs, above, left); + break; + case FILTER_D63_PRED: + av1_d63_filter_predictor(dst, stride, bs, above, left); + break; + case FILTER_TM_PRED: + av1_tm_filter_predictor(dst, stride, bs, above, left); + break; + default: assert(0); + } +} +#if CONFIG_HIGHBITDEPTH +#if USE_3TAP_INTRA_FILTER +static void highbd_filter_intra_predictors_3tap(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int mode, + int bd) { + int k, r, c; + int mean, ipred; +#if CONFIG_TX64X64 + int preds[65][65]; +#else + int preds[33][33]; +#endif // CONFIG_TX64X64 + const TX_SIZE tx_size = get_txsize_from_blocklen(bs); + const int c0 = av1_filter_intra_taps_3[tx_size][mode][0]; + const int c1 = av1_filter_intra_taps_3[tx_size][mode][1]; + const int c2 = av1_filter_intra_taps_3[tx_size][mode][2]; + + k = 0; + mean = 0; + while (k < bs) { + mean = mean + (int)left[k]; + mean = mean + (int)above[k]; + k++; + } + mean = (mean + bs) / (2 * bs); + + for (r = 0; r < bs; ++r) preds[r + 1][0] = (int)left[r] - mean; + + for (c = 0; c < bs + 1; ++c) preds[0][c] = (int)above[c - 1] - mean; + + for (r = 1; r < bs + 1; ++r) + for (c = 1; c < bs + 1; ++c) { + ipred = c0 * preds[r - 1][c] + c1 * preds[r][c - 1] + + c2 * preds[r - 1][c - 1]; + preds[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_PREC_BITS); + } + + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) { + ipred = preds[r + 1][c + 1] + mean; + dst[c] = clip_pixel_highbd(ipred, bd); + } + dst += stride; + } +} +#else +static void highbd_filter_intra_predictors_4tap(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int mode, + int bd) { + int k, r, c; + int mean, ipred; +#if CONFIG_TX64X64 + int preds[65][129]; +#else + int preds[33][65]; +#endif // CONFIG_TX64X64 + const TX_SIZE tx_size = get_txsize_from_blocklen(bs); + const int c0 = av1_filter_intra_taps_4[tx_size][mode][0]; + const int c1 = av1_filter_intra_taps_4[tx_size][mode][1]; + const int c2 = av1_filter_intra_taps_4[tx_size][mode][2]; + const int c3 = av1_filter_intra_taps_4[tx_size][mode][3]; + + k = 0; + mean = 0; + while (k < bs) { + mean = mean + (int)left[k]; + mean = mean + (int)above[k]; + k++; + } + mean = (mean + bs) / (2 * bs); + + for (r = 0; r < bs; ++r) preds[r + 1][0] = (int)left[r] - mean; + + for (c = 0; c < 2 * bs + 1; ++c) preds[0][c] = (int)above[c - 1] - mean; + + for (r = 1; r < bs + 1; ++r) + for (c = 1; c < 2 * bs + 1 - r; ++c) { + ipred = c0 * preds[r - 1][c] + c1 * preds[r][c - 1] + + c2 * preds[r - 1][c - 1] + c3 * preds[r - 1][c + 1]; + preds[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_PREC_BITS); + } + + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) { + ipred = preds[r + 1][c + 1] + mean; + dst[c] = clip_pixel_highbd(ipred, bd); + } + dst += stride; + } +} +#endif + +void av1_highbd_dc_filter_predictor_c(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { +#if USE_3TAP_INTRA_FILTER + highbd_filter_intra_predictors_3tap(dst, stride, bs, above, left, DC_PRED, + bd); +#else + highbd_filter_intra_predictors_4tap(dst, stride, bs, above, left, DC_PRED, + bd); +#endif +} + +void av1_highbd_v_filter_predictor_c(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { +#if USE_3TAP_INTRA_FILTER + highbd_filter_intra_predictors_3tap(dst, stride, bs, above, left, V_PRED, bd); +#else + highbd_filter_intra_predictors_4tap(dst, stride, bs, above, left, V_PRED, bd); +#endif +} + +void av1_highbd_h_filter_predictor_c(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { +#if USE_3TAP_INTRA_FILTER + highbd_filter_intra_predictors_3tap(dst, stride, bs, above, left, H_PRED, bd); +#else + highbd_filter_intra_predictors_4tap(dst, stride, bs, above, left, H_PRED, bd); +#endif +} + +void av1_highbd_d45_filter_predictor_c(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { +#if USE_3TAP_INTRA_FILTER + highbd_filter_intra_predictors_3tap(dst, stride, bs, above, left, D45_PRED, + bd); +#else + highbd_filter_intra_predictors_4tap(dst, stride, bs, above, left, D45_PRED, + bd); +#endif +} + +void av1_highbd_d135_filter_predictor_c(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { +#if USE_3TAP_INTRA_FILTER + highbd_filter_intra_predictors_3tap(dst, stride, bs, above, left, D135_PRED, + bd); +#else + highbd_filter_intra_predictors_4tap(dst, stride, bs, above, left, D135_PRED, + bd); +#endif +} + +void av1_highbd_d117_filter_predictor_c(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { +#if USE_3TAP_INTRA_FILTER + highbd_filter_intra_predictors_3tap(dst, stride, bs, above, left, D117_PRED, + bd); +#else + highbd_filter_intra_predictors_4tap(dst, stride, bs, above, left, D117_PRED, + bd); +#endif +} + +void av1_highbd_d153_filter_predictor_c(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { +#if USE_3TAP_INTRA_FILTER + highbd_filter_intra_predictors_3tap(dst, stride, bs, above, left, D153_PRED, + bd); +#else + highbd_filter_intra_predictors_4tap(dst, stride, bs, above, left, D153_PRED, + bd); +#endif +} + +void av1_highbd_d207_filter_predictor_c(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { +#if USE_3TAP_INTRA_FILTER + highbd_filter_intra_predictors_3tap(dst, stride, bs, above, left, D207_PRED, + bd); +#else + highbd_filter_intra_predictors_4tap(dst, stride, bs, above, left, D207_PRED, + bd); +#endif +} + +void av1_highbd_d63_filter_predictor_c(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { +#if USE_3TAP_INTRA_FILTER + highbd_filter_intra_predictors_3tap(dst, stride, bs, above, left, D63_PRED, + bd); +#else + highbd_filter_intra_predictors_4tap(dst, stride, bs, above, left, D63_PRED, + bd); +#endif +} + +void av1_highbd_tm_filter_predictor_c(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, + const uint16_t *left, int bd) { +#if USE_3TAP_INTRA_FILTER + highbd_filter_intra_predictors_3tap(dst, stride, bs, above, left, TM_PRED, + bd); +#else + highbd_filter_intra_predictors_4tap(dst, stride, bs, above, left, TM_PRED, + bd); +#endif +} + +static void highbd_filter_intra_predictors(FILTER_INTRA_MODE mode, + uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + switch (mode) { + case FILTER_DC_PRED: + av1_highbd_dc_filter_predictor(dst, stride, bs, above, left, bd); + break; + case FILTER_V_PRED: + av1_highbd_v_filter_predictor(dst, stride, bs, above, left, bd); + break; + case FILTER_H_PRED: + av1_highbd_h_filter_predictor(dst, stride, bs, above, left, bd); + break; + case FILTER_D45_PRED: + av1_highbd_d45_filter_predictor(dst, stride, bs, above, left, bd); + break; + case FILTER_D135_PRED: + av1_highbd_d135_filter_predictor(dst, stride, bs, above, left, bd); + break; + case FILTER_D117_PRED: + av1_highbd_d117_filter_predictor(dst, stride, bs, above, left, bd); + break; + case FILTER_D153_PRED: + av1_highbd_d153_filter_predictor(dst, stride, bs, above, left, bd); + break; + case FILTER_D207_PRED: + av1_highbd_d207_filter_predictor(dst, stride, bs, above, left, bd); + break; + case FILTER_D63_PRED: + av1_highbd_d63_filter_predictor(dst, stride, bs, above, left, bd); + break; + case FILTER_TM_PRED: + av1_highbd_tm_filter_predictor(dst, stride, bs, above, left, bd); + break; + default: assert(0); + } +} +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_HIGHBITDEPTH +static void build_intra_predictors_high( + const MACROBLOCKD *xd, const uint8_t *ref8, int ref_stride, uint8_t *dst8, + int dst_stride, PREDICTION_MODE mode, TX_SIZE tx_size, int n_top_px, + int n_topright_px, int n_left_px, int n_bottomleft_px, int plane) { + int i; + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + DECLARE_ALIGNED(16, uint16_t, left_data[MAX_TX_SIZE * 2 + 16]); + DECLARE_ALIGNED(16, uint16_t, above_data[MAX_TX_SIZE * 2 + 16]); + uint16_t *above_row = above_data + 16; + uint16_t *left_col = left_data + 16; + const uint16_t *const_above_row = above_row; + const int bs = tx_size_wide[tx_size]; + int need_left = extend_modes[mode] & NEED_LEFT; + int need_above = extend_modes[mode] & NEED_ABOVE; + int need_above_left = extend_modes[mode] & NEED_ABOVELEFT; + const uint16_t *above_ref = ref - ref_stride; +#if CONFIG_EXT_INTRA + int p_angle = 0; + const int is_dr_mode = av1_is_directional_mode(mode, xd->mi[0]->mbmi.sb_type); +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + const FILTER_INTRA_MODE_INFO *filter_intra_mode_info = + &xd->mi[0]->mbmi.filter_intra_mode_info; + const FILTER_INTRA_MODE filter_intra_mode = + filter_intra_mode_info->filter_intra_mode[plane != 0]; +#endif // CONFIG_FILTER_INTRA + int base = 128 << (xd->bd - 8); + assert(tx_size_wide[tx_size] == tx_size_high[tx_size]); + + // base-1 base-1 base-1 .. base-1 base-1 base-1 base-1 base-1 base-1 + // base+1 A B .. Y Z + // base+1 C D .. W X + // base+1 E F .. U V + // base+1 G H .. S T T T T T + aom_memset16(left_data, base + 1, sizeof(left_data) / sizeof(*left_data)); + +#if CONFIG_EXT_INTRA + if (is_dr_mode) { + p_angle = mode_to_angle_map[mode] + + xd->mi[0]->mbmi.angle_delta[plane != 0] * ANGLE_STEP; + if (p_angle <= 90) + need_above = 1, need_left = 0, need_above_left = 1; + else if (p_angle < 180) + need_above = 1, need_left = 1, need_above_left = 1; + else + need_above = 0, need_left = 1, need_above_left = 1; + } +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + if (filter_intra_mode_info->use_filter_intra_mode[plane != 0]) + need_left = need_above = need_above_left = 1; +#endif // CONFIG_FILTER_INTRA + + (void)plane; + assert(n_top_px >= 0); + assert(n_topright_px >= 0); + assert(n_left_px >= 0); + assert(n_bottomleft_px >= 0); + + if ((!need_above && n_left_px == 0) || (!need_left && n_top_px == 0)) { + const int val = need_left ? base + 1 : base - 1; + for (i = 0; i < bs; ++i) { + aom_memset16(dst, val, bs); + dst += dst_stride; + } + return; + } + + // NEED_LEFT + if (need_left) { +#if CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA + int need_bottom = !!(extend_modes[mode] & NEED_BOTTOMLEFT); +#if CONFIG_FILTER_INTRA + if (filter_intra_mode_info->use_filter_intra_mode[plane != 0]) + need_bottom = 0; +#endif // CONFIG_FILTER_INTRA +#if CONFIG_EXT_INTRA + if (is_dr_mode) need_bottom = p_angle > 180; +#endif // CONFIG_EXT_INTRA +#else + const int need_bottom = !!(extend_modes[mode] & NEED_BOTTOMLEFT); +#endif // CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA + i = 0; + if (n_left_px > 0) { + for (; i < n_left_px; i++) left_col[i] = ref[i * ref_stride - 1]; + if (need_bottom && n_bottomleft_px > 0) { + assert(i == bs); + for (; i < bs + n_bottomleft_px; i++) + left_col[i] = ref[i * ref_stride - 1]; + } + if (i < (bs << need_bottom)) + aom_memset16(&left_col[i], left_col[i - 1], (bs << need_bottom) - i); + } else { + aom_memset16(left_col, base + 1, bs << need_bottom); + } + } + + // NEED_ABOVE + if (need_above) { +#if CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA + int need_right = !!(extend_modes[mode] & NEED_ABOVERIGHT); +#if CONFIG_FILTER_INTRA + if (filter_intra_mode_info->use_filter_intra_mode[plane != 0]) + need_right = 1; +#endif // CONFIG_FILTER_INTRA +#if CONFIG_EXT_INTRA + if (is_dr_mode) need_right = p_angle < 90; +#endif // CONFIG_EXT_INTRA +#else + const int need_right = !!(extend_modes[mode] & NEED_ABOVERIGHT); +#endif // CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA + if (n_top_px > 0) { + memcpy(above_row, above_ref, n_top_px * sizeof(above_ref[0])); + i = n_top_px; + if (need_right && n_topright_px > 0) { + assert(n_top_px == bs); + memcpy(above_row + bs, above_ref + bs, + n_topright_px * sizeof(above_ref[0])); + i += n_topright_px; + } + if (i < (bs << need_right)) + aom_memset16(&above_row[i], above_row[i - 1], (bs << need_right) - i); + } else { + aom_memset16(above_row, base - 1, bs << need_right); + } + } + + if (need_above_left) { + above_row[-1] = + n_top_px > 0 ? (n_left_px > 0 ? above_ref[-1] : base + 1) : base - 1; + left_col[-1] = above_row[-1]; + } + +#if CONFIG_FILTER_INTRA + if (filter_intra_mode_info->use_filter_intra_mode[plane != 0]) { + highbd_filter_intra_predictors(filter_intra_mode, dst, dst_stride, bs, + const_above_row, left_col, xd->bd); + return; + } +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_EXT_INTRA + if (is_dr_mode) { +#if CONFIG_INTRA_INTERP + INTRA_FILTER filter = INTRA_FILTER_LINEAR; + if (plane == 0 && av1_is_intra_filter_switchable(p_angle)) + filter = xd->mi[0]->mbmi.intra_filter; +#endif // CONFIG_INTRA_INTERP + highbd_dr_predictor(dst, dst_stride, bs, const_above_row, left_col, +#if CONFIG_INTRA_INTERP + filter, +#endif // CONFIG_INTRA_INTERP + p_angle, xd->bd); + return; + } +#endif // CONFIG_EXT_INTRA + + // predict + if (mode == DC_PRED) { + dc_pred_high[n_left_px > 0][n_top_px > 0][tx_size]( + dst, dst_stride, const_above_row, left_col, xd->bd); + } else { + pred_high[mode][tx_size](dst, dst_stride, const_above_row, left_col, + xd->bd); + } +} +#endif // CONFIG_HIGHBITDEPTH + +static void build_intra_predictors(const MACROBLOCKD *xd, const uint8_t *ref, + int ref_stride, uint8_t *dst, int dst_stride, + PREDICTION_MODE mode, TX_SIZE tx_size, + int n_top_px, int n_topright_px, + int n_left_px, int n_bottomleft_px, + int plane) { + int i; + const uint8_t *above_ref = ref - ref_stride; + DECLARE_ALIGNED(16, uint8_t, left_data[MAX_TX_SIZE * 2 + 16]); + DECLARE_ALIGNED(16, uint8_t, above_data[MAX_TX_SIZE * 2 + 16]); + uint8_t *above_row = above_data + 16; + uint8_t *left_col = left_data + 16; + const uint8_t *const_above_row = above_row; + const int bs = tx_size_wide[tx_size]; + int need_left = extend_modes[mode] & NEED_LEFT; + int need_above = extend_modes[mode] & NEED_ABOVE; + int need_above_left = extend_modes[mode] & NEED_ABOVELEFT; +#if CONFIG_EXT_INTRA + int p_angle = 0; + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const int is_dr_mode = av1_is_directional_mode(mode, mbmi->sb_type); +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + const FILTER_INTRA_MODE_INFO *filter_intra_mode_info = + &xd->mi[0]->mbmi.filter_intra_mode_info; + const FILTER_INTRA_MODE filter_intra_mode = + filter_intra_mode_info->filter_intra_mode[plane != 0]; +#endif // CONFIG_FILTER_INTRA + assert(tx_size_wide[tx_size] == tx_size_high[tx_size]); + + // 127 127 127 .. 127 127 127 127 127 127 + // 129 A B .. Y Z + // 129 C D .. W X + // 129 E F .. U V + // 129 G H .. S T T T T T + // .. + memset(left_data, 129, sizeof(left_data)); + +#if CONFIG_EXT_INTRA + if (is_dr_mode) { + p_angle = mode_to_angle_map[mode] + + xd->mi[0]->mbmi.angle_delta[plane != 0] * ANGLE_STEP; + if (p_angle <= 90) + need_above = 1, need_left = 0, need_above_left = 1; + else if (p_angle < 180) + need_above = 1, need_left = 1, need_above_left = 1; + else + need_above = 0, need_left = 1, need_above_left = 1; + } +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + if (filter_intra_mode_info->use_filter_intra_mode[plane != 0]) + need_left = need_above = need_above_left = 1; +#endif // CONFIG_FILTER_INTRA + + (void)xd; + (void)plane; + assert(n_top_px >= 0); + assert(n_topright_px >= 0); + assert(n_left_px >= 0); + assert(n_bottomleft_px >= 0); + + if ((!need_above && n_left_px == 0) || (!need_left && n_top_px == 0)) { + const int val = need_left ? 129 : 127; + for (i = 0; i < bs; ++i) { + memset(dst, val, bs); + dst += dst_stride; + } + return; + } + + // NEED_LEFT + if (need_left) { +#if CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA + int need_bottom = !!(extend_modes[mode] & NEED_BOTTOMLEFT); +#if CONFIG_FILTER_INTRA + if (filter_intra_mode_info->use_filter_intra_mode[plane != 0]) + need_bottom = 0; +#endif // CONFIG_FILTER_INTRA +#if CONFIG_EXT_INTRA + if (is_dr_mode) need_bottom = p_angle > 180; +#endif // CONFIG_EXT_INTRA +#else + const int need_bottom = !!(extend_modes[mode] & NEED_BOTTOMLEFT); +#endif // CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA + i = 0; + if (n_left_px > 0) { + for (; i < n_left_px; i++) left_col[i] = ref[i * ref_stride - 1]; + if (need_bottom && n_bottomleft_px > 0) { + assert(i == bs); + for (; i < bs + n_bottomleft_px; i++) + left_col[i] = ref[i * ref_stride - 1]; + } + if (i < (bs << need_bottom)) + memset(&left_col[i], left_col[i - 1], (bs << need_bottom) - i); + } else { + memset(left_col, 129, bs << need_bottom); + } + } + + // NEED_ABOVE + if (need_above) { +#if CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA + int need_right = !!(extend_modes[mode] & NEED_ABOVERIGHT); +#if CONFIG_FILTER_INTRA + if (filter_intra_mode_info->use_filter_intra_mode[plane != 0]) + need_right = 1; +#endif // CONFIG_FILTER_INTRA +#if CONFIG_EXT_INTRA + if (is_dr_mode) need_right = p_angle < 90; +#endif // CONFIG_EXT_INTRA +#else + const int need_right = !!(extend_modes[mode] & NEED_ABOVERIGHT); +#endif // CONFIG_EXT_INTRA || CONFIG_FITLER_INTRA + if (n_top_px > 0) { + memcpy(above_row, above_ref, n_top_px); + i = n_top_px; + if (need_right && n_topright_px > 0) { + assert(n_top_px == bs); + memcpy(above_row + bs, above_ref + bs, n_topright_px); + i += n_topright_px; + } + if (i < (bs << need_right)) + memset(&above_row[i], above_row[i - 1], (bs << need_right) - i); + } else { + memset(above_row, 127, bs << need_right); + } + } + + if (need_above_left) { + above_row[-1] = n_top_px > 0 ? (n_left_px > 0 ? above_ref[-1] : 129) : 127; + left_col[-1] = above_row[-1]; + } + +#if CONFIG_FILTER_INTRA + if (filter_intra_mode_info->use_filter_intra_mode[plane != 0]) { + filter_intra_predictors(filter_intra_mode, dst, dst_stride, bs, + const_above_row, left_col); + return; + } +#endif // CONFIG_FILTER_INTRA +#if CONFIG_EXT_INTRA + if (is_dr_mode) { +#if CONFIG_INTRA_INTERP + INTRA_FILTER filter = INTRA_FILTER_LINEAR; + if (plane == 0 && av1_is_intra_filter_switchable(p_angle)) + filter = xd->mi[0]->mbmi.intra_filter; +#endif // CONFIG_INTRA_INTERP + dr_predictor(dst, dst_stride, tx_size, const_above_row, left_col, +#if CONFIG_INTRA_INTERP + filter, +#endif // CONFIG_INTRA_INTERP + p_angle); + return; + } +#endif // CONFIG_EXT_INTRA + + // predict + if (mode == DC_PRED) { +#if CONFIG_CFL + // CFL predict its own DC_PRED for Chromatic planes + if (plane == AOM_PLANE_Y) { +#endif + dc_pred[n_left_px > 0][n_top_px > 0][tx_size](dst, dst_stride, + const_above_row, left_col); +#if CONFIG_CFL + } +#endif + + } else { + pred[mode][tx_size](dst, dst_stride, const_above_row, left_col); + } +} + +static void predict_square_intra_block(const MACROBLOCKD *xd, int wpx, int hpx, + TX_SIZE tx_size, PREDICTION_MODE mode, + const uint8_t *ref, int ref_stride, + uint8_t *dst, int dst_stride, + int col_off, int row_off, int plane) { + BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const int txw = tx_size_wide_unit[tx_size]; +#if CONFIG_CB4X4 && CONFIG_CHROMA_SUB8X8 + const int have_top = row_off || (pd->subsampling_y ? xd->chroma_up_available + : xd->up_available); + const int have_left = + col_off || + (pd->subsampling_x ? xd->chroma_left_available : xd->left_available); +#else + const int have_top = row_off || xd->up_available; + const int have_left = col_off || xd->left_available; +#endif + const int x = col_off << tx_size_wide_log2[0]; + const int y = row_off << tx_size_high_log2[0]; + const int mi_row = -xd->mb_to_top_edge >> (3 + MI_SIZE_LOG2); + const int mi_col = -xd->mb_to_left_edge >> (3 + MI_SIZE_LOG2); + const int txwpx = tx_size_wide[tx_size]; + const int txhpx = tx_size_high[tx_size]; +#if CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + const int xr_chr_offset = (pd->subsampling_x && bsize < BLOCK_8X8) ? 2 : 0; + const int yd_chr_offset = (pd->subsampling_y && bsize < BLOCK_8X8) ? 2 : 0; +#else + const int xr_chr_offset = 0; + const int yd_chr_offset = 0; +#endif + + // Distance between the right edge of this prediction block to + // the frame right edge + const int xr = (xd->mb_to_right_edge >> (3 + pd->subsampling_x)) + + (wpx - x - txwpx) - xr_chr_offset; + // Distance between the bottom edge of this prediction block to + // the frame bottom edge + const int yd = (xd->mb_to_bottom_edge >> (3 + pd->subsampling_y)) + + (hpx - y - txhpx) - yd_chr_offset; + const int right_available = + (mi_col + ((col_off + txw) >> (1 - pd->subsampling_x))) < + xd->tile.mi_col_end; + const int bottom_available = (yd > 0); +#if CONFIG_EXT_PARTITION_TYPES + const PARTITION_TYPE partition = xd->mi[0]->mbmi.partition; +#endif + +#if CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + // force 4x4 chroma component block size. + bsize = scale_chroma_bsize(bsize, pd->subsampling_x, pd->subsampling_y); +#endif + + const int have_top_right = + has_top_right(bsize, mi_row, mi_col, have_top, right_available, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + tx_size, row_off, col_off, pd->subsampling_x); + const int have_bottom_left = + has_bottom_left(bsize, mi_row, mi_col, bottom_available, have_left, + tx_size, row_off, col_off, pd->subsampling_y); + assert(txwpx == txhpx); + +#if CONFIG_PALETTE + if (xd->mi[0]->mbmi.palette_mode_info.palette_size[plane != 0] > 0) { + const int bs = tx_size_wide[tx_size]; + const int stride = wpx; + int r, c; + const uint8_t *const map = xd->plane[plane != 0].color_index_map; +#if CONFIG_HIGHBITDEPTH + uint16_t *palette = xd->mi[0]->mbmi.palette_mode_info.palette_colors + + plane * PALETTE_MAX_SIZE; +#else + uint8_t *palette = xd->mi[0]->mbmi.palette_mode_info.palette_colors + + plane * PALETTE_MAX_SIZE; +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst); + for (r = 0; r < bs; ++r) + for (c = 0; c < bs; ++c) + dst16[r * dst_stride + c] = palette[map[(r + y) * stride + c + x]]; + } else { + for (r = 0; r < bs; ++r) + for (c = 0; c < bs; ++c) + dst[r * dst_stride + c] = + (uint8_t)(palette[map[(r + y) * stride + c + x]]); + } +#else + for (r = 0; r < bs; ++r) + for (c = 0; c < bs; ++c) + dst[r * dst_stride + c] = palette[map[(r + y) * stride + c + x]]; +#endif // CONFIG_HIGHBITDEPTH + return; + } +#endif // CONFIG_PALETTE + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + build_intra_predictors_high( + xd, ref, ref_stride, dst, dst_stride, mode, tx_size, + have_top ? AOMMIN(txwpx, xr + txwpx) : 0, + have_top_right ? AOMMIN(txwpx, xr) : 0, + have_left ? AOMMIN(txhpx, yd + txhpx) : 0, + have_bottom_left ? AOMMIN(txhpx, yd) : 0, plane); + return; + } +#endif + build_intra_predictors(xd, ref, ref_stride, dst, dst_stride, mode, tx_size, + have_top ? AOMMIN(txwpx, xr + txwpx) : 0, + have_top_right ? AOMMIN(txwpx, xr) : 0, + have_left ? AOMMIN(txhpx, yd + txhpx) : 0, + have_bottom_left ? AOMMIN(txhpx, yd) : 0, plane); +} + +void av1_predict_intra_block_facade(MACROBLOCKD *xd, int plane, int block_idx, + int blk_col, int blk_row, TX_SIZE tx_size) { + struct macroblockd_plane *const pd = &xd->plane[plane]; + const int dst_stride = pd->dst.stride; + uint8_t *dst = + &pd->dst.buf[(blk_row * dst_stride + blk_col) << tx_size_wide_log2[0]]; + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const int block_raster_idx = + av1_block_index_to_raster_order(tx_size, block_idx); + const PREDICTION_MODE mode = + (plane == 0) ? get_y_mode(xd->mi[0], block_raster_idx) : mbmi->uv_mode; + av1_predict_intra_block(xd, pd->width, pd->height, txsize_to_bsize[tx_size], + mode, dst, dst_stride, dst, dst_stride, blk_col, + blk_row, plane); +#if CONFIG_CFL + if (plane != AOM_PLANE_Y && mbmi->uv_mode == DC_PRED) { + if (plane == AOM_PLANE_U && blk_col == 0 && blk_row == 0) { + // Compute the block-level DC_PRED for both chromatic planes prior to + // processing the first chromatic plane in order to compute alpha_cb and + // alpha_cr. Note: This is not required on the decoder side because alpha + // is signaled. + cfl_dc_pred(xd, get_plane_block_size(block_idx, pd), tx_size); + } + cfl_predict_block(xd->cfl, dst, pd->dst.stride, blk_row, blk_col, tx_size, + xd->cfl->dc_pred[plane - 1]); + } +#endif +} + +void av1_predict_intra_block(const MACROBLOCKD *xd, int wpx, int hpx, + BLOCK_SIZE bsize, PREDICTION_MODE mode, + const uint8_t *ref, int ref_stride, uint8_t *dst, + int dst_stride, int col_off, int row_off, + int plane) { + const int block_width = block_size_wide[bsize]; + const int block_height = block_size_high[bsize]; + TX_SIZE tx_size = max_txsize_lookup[bsize]; + assert(tx_size < TX_SIZES); + if (block_width == block_height) { + predict_square_intra_block(xd, wpx, hpx, tx_size, mode, ref, ref_stride, + dst, dst_stride, col_off, row_off, plane); + } else { +#if (CONFIG_RECT_TX && (CONFIG_VAR_TX || CONFIG_EXT_TX)) || (CONFIG_EXT_INTER) +#if CONFIG_HIGHBITDEPTH + uint16_t tmp16[MAX_SB_SIZE]; +#endif + uint8_t tmp[MAX_SB_SIZE]; + assert((block_width == wpx && block_height == hpx) || + (block_width == (wpx >> 1) && block_height == hpx) || + (block_width == wpx && block_height == (hpx >> 1))); + + if (block_width < block_height) { + assert(block_height == (block_width << 1)); + // Predict the top square sub-block. + predict_square_intra_block(xd, wpx, hpx, tx_size, mode, ref, ref_stride, + dst, dst_stride, col_off, row_off, plane); + { + const int half_block_height = block_height >> 1; + const int half_block_height_unit = + half_block_height >> tx_size_wide_log2[0]; + // Cast away const to modify 'ref' temporarily; will be restored later. + uint8_t *src_2 = (uint8_t *)ref + half_block_height * ref_stride; + uint8_t *dst_2 = dst + half_block_height * dst_stride; + const int row_off_2 = row_off + half_block_height_unit; + // Save the last row of top square sub-block as 'above' row for bottom + // square sub-block. + if (src_2 != dst_2 || ref_stride != dst_stride) { +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + uint16_t *src_2_16 = CONVERT_TO_SHORTPTR(src_2); + uint16_t *dst_2_16 = CONVERT_TO_SHORTPTR(dst_2); + memcpy(tmp16, src_2_16 - ref_stride, + block_width * sizeof(*src_2_16)); + memcpy(src_2_16 - ref_stride, dst_2_16 - dst_stride, + block_width * sizeof(*src_2_16)); + } else { +#endif // CONFIG_HIGHBITDEPTH + memcpy(tmp, src_2 - ref_stride, block_width * sizeof(*src_2)); + memcpy(src_2 - ref_stride, dst_2 - dst_stride, + block_width * sizeof(*src_2)); +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + } + // Predict the bottom square sub-block. + predict_square_intra_block(xd, wpx, hpx, tx_size, mode, src_2, + ref_stride, dst_2, dst_stride, col_off, + row_off_2, plane); + // Restore the last row of top square sub-block. + if (src_2 != dst_2 || ref_stride != dst_stride) { +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + uint16_t *src_2_16 = CONVERT_TO_SHORTPTR(src_2); + memcpy(src_2_16 - ref_stride, tmp16, + block_width * sizeof(*src_2_16)); + } else { +#endif // CONFIG_HIGHBITDEPTH + memcpy(src_2 - ref_stride, tmp, block_width * sizeof(*src_2)); +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + } + } + } else { // block_width > block_height + assert(block_width == (block_height << 1)); + // Predict the left square sub-block + predict_square_intra_block(xd, wpx, hpx, tx_size, mode, ref, ref_stride, + dst, dst_stride, col_off, row_off, plane); + { + int i; + const int half_block_width = block_width >> 1; + const int half_block_width_unit = + half_block_width >> tx_size_wide_log2[0]; + // Cast away const to modify 'ref' temporarily; will be restored later. + uint8_t *src_2 = (uint8_t *)ref + half_block_width; + uint8_t *dst_2 = dst + half_block_width; + const int col_off_2 = col_off + half_block_width_unit; + // Save the last column of left square sub-block as 'left' column for + // right square sub-block. + const int save_src = src_2 != dst_2 || ref_stride != dst_stride; + if (save_src) { +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + uint16_t *src_2_16 = CONVERT_TO_SHORTPTR(src_2); + uint16_t *dst_2_16 = CONVERT_TO_SHORTPTR(dst_2); + for (i = 0; i < block_height; ++i) { + tmp16[i] = src_2_16[i * ref_stride - 1]; + src_2_16[i * ref_stride - 1] = dst_2_16[i * dst_stride - 1]; + } + } else { +#endif // CONFIG_HIGHBITDEPTH + for (i = 0; i < block_height; ++i) { + tmp[i] = src_2[i * ref_stride - 1]; + src_2[i * ref_stride - 1] = dst_2[i * dst_stride - 1]; + } +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + } + // Predict the right square sub-block. + predict_square_intra_block(xd, wpx, hpx, tx_size, mode, src_2, + ref_stride, dst_2, dst_stride, col_off_2, + row_off, plane); + // Restore the last column of left square sub-block. + if (save_src) { +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + uint16_t *src_2_16 = CONVERT_TO_SHORTPTR(src_2); + for (i = 0; i < block_height; ++i) { + src_2_16[i * ref_stride - 1] = tmp16[i]; + } + } else { +#endif // CONFIG_HIGHBITDEPTH + for (i = 0; i < block_height; ++i) { + src_2[i * ref_stride - 1] = tmp[i]; + } +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + } + } + } +#else + assert(0); +#endif // (CONFIG_RECT_TX && (CONFIG_VAR_TX || CONFIG_EXT_TX)) || + // (CONFIG_EXT_INTER) + } +} + +void av1_init_intra_predictors(void) { + once(av1_init_intra_predictors_internal); +} diff --git a/third_party/aom/av1/common/reconintra.h b/third_party/aom/av1/common/reconintra.h new file mode 100644 index 0000000000..7ee0c495e0 --- /dev/null +++ b/third_party/aom/av1/common/reconintra.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_RECONINTRA_H_ +#define AV1_COMMON_RECONINTRA_H_ + +#include "aom/aom_integer.h" +#include "av1/common/blockd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_init_intra_predictors(void); +void av1_predict_intra_block_facade(MACROBLOCKD *xd, int plane, int block_idx, + int blk_col, int blk_row, TX_SIZE tx_size); +void av1_predict_intra_block(const MACROBLOCKD *xd, int bw, int bh, + BLOCK_SIZE bsize, PREDICTION_MODE mode, + const uint8_t *ref, int ref_stride, uint8_t *dst, + int dst_stride, int aoff, int loff, int plane); + +#if CONFIG_EXT_INTER +// Mapping of interintra to intra mode for use in the intra component +static const PREDICTION_MODE interintra_to_intra_mode[INTERINTRA_MODES] = { + DC_PRED, V_PRED, H_PRED, D45_PRED, D135_PRED, + D117_PRED, D153_PRED, D207_PRED, D63_PRED, TM_PRED +}; + +// Mapping of intra mode to the interintra mode +static const INTERINTRA_MODE intra_to_interintra_mode[INTRA_MODES] = { + II_DC_PRED, II_V_PRED, II_H_PRED, II_D45_PRED, II_D135_PRED, + II_D117_PRED, II_D153_PRED, II_D207_PRED, II_D63_PRED, +#if CONFIG_ALT_INTRA + II_DC_PRED, // Note: Filler value, as there's no II_SMOOTH_PRED. +#endif // CONFIG_ALT_INTRA + II_TM_PRED +}; +#endif // CONFIG_EXT_INTER +#ifdef __cplusplus +} // extern "C" +#endif + +#if CONFIG_FILTER_INTRA +#define FILTER_INTRA_PREC_BITS 10 +extern int av1_filter_intra_taps_4[TX_SIZES][INTRA_MODES][4]; +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_EXT_INTRA +static INLINE int av1_is_directional_mode(PREDICTION_MODE mode, + BLOCK_SIZE bsize) { + return mode != DC_PRED && mode != TM_PRED && +#if CONFIG_ALT_INTRA + mode != SMOOTH_PRED && +#endif // CONFIG_ALT_INTRA + bsize >= BLOCK_8X8; +} +#endif // CONFIG_EXT_INTRA + +#endif // AV1_COMMON_RECONINTRA_H_ diff --git a/third_party/aom/av1/common/resize.c b/third_party/aom/av1/common/resize.c new file mode 100644 index 0000000000..8c0d3aa09b --- /dev/null +++ b/third_party/aom/av1/common/resize.c @@ -0,0 +1,821 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include +#include +#include + +#include "./aom_config.h" +#if CONFIG_HIGHBITDEPTH +#include "aom_dsp/aom_dsp_common.h" +#endif // CONFIG_HIGHBITDEPTH +#include "aom_ports/mem.h" +#include "av1/common/common.h" +#include "av1/common/resize.h" + +#define FILTER_BITS 7 + +#define INTERP_TAPS 8 +#define SUBPEL_BITS 5 +#define SUBPEL_MASK ((1 << SUBPEL_BITS) - 1) +#define INTERP_PRECISION_BITS 32 + +typedef int16_t interp_kernel[INTERP_TAPS]; + +// Filters for interpolation (0.5-band) - note this also filters integer pels. +static const interp_kernel filteredinterp_filters500[(1 << SUBPEL_BITS)] = { + { -3, 0, 35, 64, 35, 0, -3, 0 }, { -3, -1, 34, 64, 36, 1, -3, 0 }, + { -3, -1, 32, 64, 38, 1, -3, 0 }, { -2, -2, 31, 63, 39, 2, -3, 0 }, + { -2, -2, 29, 63, 41, 2, -3, 0 }, { -2, -2, 28, 63, 42, 3, -4, 0 }, + { -2, -3, 27, 63, 43, 4, -4, 0 }, { -2, -3, 25, 62, 45, 5, -4, 0 }, + { -2, -3, 24, 62, 46, 5, -4, 0 }, { -2, -3, 23, 61, 47, 6, -4, 0 }, + { -2, -3, 21, 60, 49, 7, -4, 0 }, { -1, -4, 20, 60, 50, 8, -4, -1 }, + { -1, -4, 19, 59, 51, 9, -4, -1 }, { -1, -4, 17, 58, 52, 10, -4, 0 }, + { -1, -4, 16, 57, 53, 12, -4, -1 }, { -1, -4, 15, 56, 54, 13, -4, -1 }, + { -1, -4, 14, 55, 55, 14, -4, -1 }, { -1, -4, 13, 54, 56, 15, -4, -1 }, + { -1, -4, 12, 53, 57, 16, -4, -1 }, { 0, -4, 10, 52, 58, 17, -4, -1 }, + { -1, -4, 9, 51, 59, 19, -4, -1 }, { -1, -4, 8, 50, 60, 20, -4, -1 }, + { 0, -4, 7, 49, 60, 21, -3, -2 }, { 0, -4, 6, 47, 61, 23, -3, -2 }, + { 0, -4, 5, 46, 62, 24, -3, -2 }, { 0, -4, 5, 45, 62, 25, -3, -2 }, + { 0, -4, 4, 43, 63, 27, -3, -2 }, { 0, -4, 3, 42, 63, 28, -2, -2 }, + { 0, -3, 2, 41, 63, 29, -2, -2 }, { 0, -3, 2, 39, 63, 31, -2, -2 }, + { 0, -3, 1, 38, 64, 32, -1, -3 }, { 0, -3, 1, 36, 64, 34, -1, -3 } +}; + +// Filters for interpolation (0.625-band) - note this also filters integer pels. +static const interp_kernel filteredinterp_filters625[(1 << SUBPEL_BITS)] = { + { -1, -8, 33, 80, 33, -8, -1, 0 }, { -1, -8, 30, 80, 35, -8, -1, 1 }, + { -1, -8, 28, 80, 37, -7, -2, 1 }, { 0, -8, 26, 79, 39, -7, -2, 1 }, + { 0, -8, 24, 79, 41, -7, -2, 1 }, { 0, -8, 22, 78, 43, -6, -2, 1 }, + { 0, -8, 20, 78, 45, -5, -3, 1 }, { 0, -8, 18, 77, 48, -5, -3, 1 }, + { 0, -8, 16, 76, 50, -4, -3, 1 }, { 0, -8, 15, 75, 52, -3, -4, 1 }, + { 0, -7, 13, 74, 54, -3, -4, 1 }, { 0, -7, 11, 73, 56, -2, -4, 1 }, + { 0, -7, 10, 71, 58, -1, -4, 1 }, { 1, -7, 8, 70, 60, 0, -5, 1 }, + { 1, -6, 6, 68, 62, 1, -5, 1 }, { 1, -6, 5, 67, 63, 2, -5, 1 }, + { 1, -6, 4, 65, 65, 4, -6, 1 }, { 1, -5, 2, 63, 67, 5, -6, 1 }, + { 1, -5, 1, 62, 68, 6, -6, 1 }, { 1, -5, 0, 60, 70, 8, -7, 1 }, + { 1, -4, -1, 58, 71, 10, -7, 0 }, { 1, -4, -2, 56, 73, 11, -7, 0 }, + { 1, -4, -3, 54, 74, 13, -7, 0 }, { 1, -4, -3, 52, 75, 15, -8, 0 }, + { 1, -3, -4, 50, 76, 16, -8, 0 }, { 1, -3, -5, 48, 77, 18, -8, 0 }, + { 1, -3, -5, 45, 78, 20, -8, 0 }, { 1, -2, -6, 43, 78, 22, -8, 0 }, + { 1, -2, -7, 41, 79, 24, -8, 0 }, { 1, -2, -7, 39, 79, 26, -8, 0 }, + { 1, -2, -7, 37, 80, 28, -8, -1 }, { 1, -1, -8, 35, 80, 30, -8, -1 }, +}; + +// Filters for interpolation (0.75-band) - note this also filters integer pels. +static const interp_kernel filteredinterp_filters750[(1 << SUBPEL_BITS)] = { + { 2, -11, 25, 96, 25, -11, 2, 0 }, { 2, -11, 22, 96, 28, -11, 2, 0 }, + { 2, -10, 19, 95, 31, -11, 2, 0 }, { 2, -10, 17, 95, 34, -12, 2, 0 }, + { 2, -9, 14, 94, 37, -12, 2, 0 }, { 2, -8, 12, 93, 40, -12, 1, 0 }, + { 2, -8, 9, 92, 43, -12, 1, 1 }, { 2, -7, 7, 91, 46, -12, 1, 0 }, + { 2, -7, 5, 90, 49, -12, 1, 0 }, { 2, -6, 3, 88, 52, -12, 0, 1 }, + { 2, -5, 1, 86, 55, -12, 0, 1 }, { 2, -5, -1, 84, 58, -11, 0, 1 }, + { 2, -4, -2, 82, 61, -11, -1, 1 }, { 2, -4, -4, 80, 64, -10, -1, 1 }, + { 1, -3, -5, 77, 67, -9, -1, 1 }, { 1, -3, -6, 75, 70, -8, -2, 1 }, + { 1, -2, -7, 72, 72, -7, -2, 1 }, { 1, -2, -8, 70, 75, -6, -3, 1 }, + { 1, -1, -9, 67, 77, -5, -3, 1 }, { 1, -1, -10, 64, 80, -4, -4, 2 }, + { 1, -1, -11, 61, 82, -2, -4, 2 }, { 1, 0, -11, 58, 84, -1, -5, 2 }, + { 1, 0, -12, 55, 86, 1, -5, 2 }, { 1, 0, -12, 52, 88, 3, -6, 2 }, + { 0, 1, -12, 49, 90, 5, -7, 2 }, { 0, 1, -12, 46, 91, 7, -7, 2 }, + { 1, 1, -12, 43, 92, 9, -8, 2 }, { 0, 1, -12, 40, 93, 12, -8, 2 }, + { 0, 2, -12, 37, 94, 14, -9, 2 }, { 0, 2, -12, 34, 95, 17, -10, 2 }, + { 0, 2, -11, 31, 95, 19, -10, 2 }, { 0, 2, -11, 28, 96, 22, -11, 2 } +}; + +// Filters for interpolation (0.875-band) - note this also filters integer pels. +static const interp_kernel filteredinterp_filters875[(1 << SUBPEL_BITS)] = { + { 3, -8, 13, 112, 13, -8, 3, 0 }, { 3, -7, 10, 112, 17, -9, 3, -1 }, + { 2, -6, 7, 111, 21, -9, 3, -1 }, { 2, -5, 4, 111, 24, -10, 3, -1 }, + { 2, -4, 1, 110, 28, -11, 3, -1 }, { 1, -3, -1, 108, 32, -12, 4, -1 }, + { 1, -2, -3, 106, 36, -13, 4, -1 }, { 1, -1, -6, 105, 40, -14, 4, -1 }, + { 1, -1, -7, 102, 44, -14, 4, -1 }, { 1, 0, -9, 100, 48, -15, 4, -1 }, + { 1, 1, -11, 97, 53, -16, 4, -1 }, { 0, 1, -12, 95, 57, -16, 4, -1 }, + { 0, 2, -13, 91, 61, -16, 4, -1 }, { 0, 2, -14, 88, 65, -16, 4, -1 }, + { 0, 3, -15, 84, 69, -17, 4, 0 }, { 0, 3, -16, 81, 73, -16, 3, 0 }, + { 0, 3, -16, 77, 77, -16, 3, 0 }, { 0, 3, -16, 73, 81, -16, 3, 0 }, + { 0, 4, -17, 69, 84, -15, 3, 0 }, { -1, 4, -16, 65, 88, -14, 2, 0 }, + { -1, 4, -16, 61, 91, -13, 2, 0 }, { -1, 4, -16, 57, 95, -12, 1, 0 }, + { -1, 4, -16, 53, 97, -11, 1, 1 }, { -1, 4, -15, 48, 100, -9, 0, 1 }, + { -1, 4, -14, 44, 102, -7, -1, 1 }, { -1, 4, -14, 40, 105, -6, -1, 1 }, + { -1, 4, -13, 36, 106, -3, -2, 1 }, { -1, 4, -12, 32, 108, -1, -3, 1 }, + { -1, 3, -11, 28, 110, 1, -4, 2 }, { -1, 3, -10, 24, 111, 4, -5, 2 }, + { -1, 3, -9, 21, 111, 7, -6, 2 }, { -1, 3, -9, 17, 112, 10, -7, 3 } +}; + +// Filters for interpolation (full-band) - no filtering for integer pixels +static const interp_kernel filteredinterp_filters1000[(1 << SUBPEL_BITS)] = { + { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, 1, -3, 128, 3, -1, 0, 0 }, + { -1, 2, -6, 127, 7, -2, 1, 0 }, { -1, 3, -9, 126, 12, -4, 1, 0 }, + { -1, 4, -12, 125, 16, -5, 1, 0 }, { -1, 4, -14, 123, 20, -6, 2, 0 }, + { -1, 5, -15, 120, 25, -8, 2, 0 }, { -1, 5, -17, 118, 30, -9, 3, -1 }, + { -1, 6, -18, 114, 35, -10, 3, -1 }, { -1, 6, -19, 111, 41, -12, 3, -1 }, + { -1, 6, -20, 107, 46, -13, 4, -1 }, { -1, 6, -21, 103, 52, -14, 4, -1 }, + { -1, 6, -21, 99, 57, -16, 5, -1 }, { -1, 6, -21, 94, 63, -17, 5, -1 }, + { -1, 6, -20, 89, 68, -18, 5, -1 }, { -1, 6, -20, 84, 73, -19, 6, -1 }, + { -1, 6, -20, 79, 79, -20, 6, -1 }, { -1, 6, -19, 73, 84, -20, 6, -1 }, + { -1, 5, -18, 68, 89, -20, 6, -1 }, { -1, 5, -17, 63, 94, -21, 6, -1 }, + { -1, 5, -16, 57, 99, -21, 6, -1 }, { -1, 4, -14, 52, 103, -21, 6, -1 }, + { -1, 4, -13, 46, 107, -20, 6, -1 }, { -1, 3, -12, 41, 111, -19, 6, -1 }, + { -1, 3, -10, 35, 114, -18, 6, -1 }, { -1, 3, -9, 30, 118, -17, 5, -1 }, + { 0, 2, -8, 25, 120, -15, 5, -1 }, { 0, 2, -6, 20, 123, -14, 4, -1 }, + { 0, 1, -5, 16, 125, -12, 4, -1 }, { 0, 1, -4, 12, 126, -9, 3, -1 }, + { 0, 1, -2, 7, 127, -6, 2, -1 }, { 0, 0, -1, 3, 128, -3, 1, 0 } +}; + +// Filters for factor of 2 downsampling. +static const int16_t av1_down2_symeven_half_filter[] = { 56, 12, -3, -1 }; +static const int16_t av1_down2_symodd_half_filter[] = { 64, 35, 0, -3 }; + +static const interp_kernel *choose_interp_filter(int inlength, int outlength) { + int outlength16 = outlength * 16; + if (outlength16 >= inlength * 16) + return filteredinterp_filters1000; + else if (outlength16 >= inlength * 13) + return filteredinterp_filters875; + else if (outlength16 >= inlength * 11) + return filteredinterp_filters750; + else if (outlength16 >= inlength * 9) + return filteredinterp_filters625; + else + return filteredinterp_filters500; +} + +static void interpolate(const uint8_t *const input, int inlength, + uint8_t *output, int outlength) { + const int64_t delta = + (((uint64_t)inlength << 32) + outlength / 2) / outlength; + const int64_t offset = + inlength > outlength + ? (((int64_t)(inlength - outlength) << 31) + outlength / 2) / + outlength + : -(((int64_t)(outlength - inlength) << 31) + outlength / 2) / + outlength; + uint8_t *optr = output; + int x, x1, x2, sum, k, int_pel, sub_pel; + int64_t y; + + const interp_kernel *interp_filters = + choose_interp_filter(inlength, outlength); + + x = 0; + y = offset; + while ((y >> INTERP_PRECISION_BITS) < (INTERP_TAPS / 2 - 1)) { + x++; + y += delta; + } + x1 = x; + x = outlength - 1; + y = delta * x + offset; + while ((y >> INTERP_PRECISION_BITS) + (int64_t)(INTERP_TAPS / 2) >= + inlength) { + x--; + y -= delta; + } + x2 = x; + if (x1 > x2) { + for (x = 0, y = offset; x < outlength; ++x, y += delta) { + const int16_t *filter; + int_pel = y >> INTERP_PRECISION_BITS; + sub_pel = (y >> (INTERP_PRECISION_BITS - SUBPEL_BITS)) & SUBPEL_MASK; + filter = interp_filters[sub_pel]; + sum = 0; + for (k = 0; k < INTERP_TAPS; ++k) { + const int pk = int_pel - INTERP_TAPS / 2 + 1 + k; + sum += filter[k] * + input[(pk < 0 ? 0 : (pk >= inlength ? inlength - 1 : pk))]; + } + *optr++ = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); + } + } else { + // Initial part. + for (x = 0, y = offset; x < x1; ++x, y += delta) { + const int16_t *filter; + int_pel = y >> INTERP_PRECISION_BITS; + sub_pel = (y >> (INTERP_PRECISION_BITS - SUBPEL_BITS)) & SUBPEL_MASK; + filter = interp_filters[sub_pel]; + sum = 0; + for (k = 0; k < INTERP_TAPS; ++k) + sum += filter[k] * input[(int_pel - INTERP_TAPS / 2 + 1 + k < 0 + ? 0 + : int_pel - INTERP_TAPS / 2 + 1 + k)]; + *optr++ = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); + } + // Middle part. + for (; x <= x2; ++x, y += delta) { + const int16_t *filter; + int_pel = y >> INTERP_PRECISION_BITS; + sub_pel = (y >> (INTERP_PRECISION_BITS - SUBPEL_BITS)) & SUBPEL_MASK; + filter = interp_filters[sub_pel]; + sum = 0; + for (k = 0; k < INTERP_TAPS; ++k) + sum += filter[k] * input[int_pel - INTERP_TAPS / 2 + 1 + k]; + *optr++ = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); + } + // End part. + for (; x < outlength; ++x, y += delta) { + const int16_t *filter; + int_pel = y >> INTERP_PRECISION_BITS; + sub_pel = (y >> (INTERP_PRECISION_BITS - SUBPEL_BITS)) & SUBPEL_MASK; + filter = interp_filters[sub_pel]; + sum = 0; + for (k = 0; k < INTERP_TAPS; ++k) + sum += filter[k] * input[(int_pel - INTERP_TAPS / 2 + 1 + k >= inlength + ? inlength - 1 + : int_pel - INTERP_TAPS / 2 + 1 + k)]; + *optr++ = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); + } + } +} + +static void down2_symeven(const uint8_t *const input, int length, + uint8_t *output) { + // Actual filter len = 2 * filter_len_half. + const int16_t *filter = av1_down2_symeven_half_filter; + const int filter_len_half = sizeof(av1_down2_symeven_half_filter) / 2; + int i, j; + uint8_t *optr = output; + int l1 = filter_len_half; + int l2 = (length - filter_len_half); + l1 += (l1 & 1); + l2 += (l2 & 1); + if (l1 > l2) { + // Short input length. + for (i = 0; i < length; i += 2) { + int sum = (1 << (FILTER_BITS - 1)); + for (j = 0; j < filter_len_half; ++j) { + sum += (input[(i - j < 0 ? 0 : i - j)] + + input[(i + 1 + j >= length ? length - 1 : i + 1 + j)]) * + filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel(sum); + } + } else { + // Initial part. + for (i = 0; i < l1; i += 2) { + int sum = (1 << (FILTER_BITS - 1)); + for (j = 0; j < filter_len_half; ++j) { + sum += (input[(i - j < 0 ? 0 : i - j)] + input[i + 1 + j]) * filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel(sum); + } + // Middle part. + for (; i < l2; i += 2) { + int sum = (1 << (FILTER_BITS - 1)); + for (j = 0; j < filter_len_half; ++j) { + sum += (input[i - j] + input[i + 1 + j]) * filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel(sum); + } + // End part. + for (; i < length; i += 2) { + int sum = (1 << (FILTER_BITS - 1)); + for (j = 0; j < filter_len_half; ++j) { + sum += (input[i - j] + + input[(i + 1 + j >= length ? length - 1 : i + 1 + j)]) * + filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel(sum); + } + } +} + +static void down2_symodd(const uint8_t *const input, int length, + uint8_t *output) { + // Actual filter len = 2 * filter_len_half - 1. + const int16_t *filter = av1_down2_symodd_half_filter; + const int filter_len_half = sizeof(av1_down2_symodd_half_filter) / 2; + int i, j; + uint8_t *optr = output; + int l1 = filter_len_half - 1; + int l2 = (length - filter_len_half + 1); + l1 += (l1 & 1); + l2 += (l2 & 1); + if (l1 > l2) { + // Short input length. + for (i = 0; i < length; i += 2) { + int sum = (1 << (FILTER_BITS - 1)) + input[i] * filter[0]; + for (j = 1; j < filter_len_half; ++j) { + sum += (input[(i - j < 0 ? 0 : i - j)] + + input[(i + j >= length ? length - 1 : i + j)]) * + filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel(sum); + } + } else { + // Initial part. + for (i = 0; i < l1; i += 2) { + int sum = (1 << (FILTER_BITS - 1)) + input[i] * filter[0]; + for (j = 1; j < filter_len_half; ++j) { + sum += (input[(i - j < 0 ? 0 : i - j)] + input[i + j]) * filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel(sum); + } + // Middle part. + for (; i < l2; i += 2) { + int sum = (1 << (FILTER_BITS - 1)) + input[i] * filter[0]; + for (j = 1; j < filter_len_half; ++j) { + sum += (input[i - j] + input[i + j]) * filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel(sum); + } + // End part. + for (; i < length; i += 2) { + int sum = (1 << (FILTER_BITS - 1)) + input[i] * filter[0]; + for (j = 1; j < filter_len_half; ++j) { + sum += (input[i - j] + input[(i + j >= length ? length - 1 : i + j)]) * + filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel(sum); + } + } +} + +static int get_down2_length(int length, int steps) { + int s; + for (s = 0; s < steps; ++s) length = (length + 1) >> 1; + return length; +} + +static int get_down2_steps(int in_length, int out_length) { + int steps = 0; + int proj_in_length; + while ((proj_in_length = get_down2_length(in_length, 1)) >= out_length) { + ++steps; + in_length = proj_in_length; + } + return steps; +} + +static void resize_multistep(const uint8_t *const input, int length, + uint8_t *output, int olength, uint8_t *otmp) { + int steps; + if (length == olength) { + memcpy(output, input, sizeof(output[0]) * length); + return; + } + steps = get_down2_steps(length, olength); + + if (steps > 0) { + int s; + uint8_t *out = NULL; + uint8_t *otmp2; + int filteredlength = length; + + assert(otmp != NULL); + otmp2 = otmp + get_down2_length(length, 1); + for (s = 0; s < steps; ++s) { + const int proj_filteredlength = get_down2_length(filteredlength, 1); + const uint8_t *const in = (s == 0 ? input : out); + if (s == steps - 1 && proj_filteredlength == olength) + out = output; + else + out = (s & 1 ? otmp2 : otmp); + if (filteredlength & 1) + down2_symodd(in, filteredlength, out); + else + down2_symeven(in, filteredlength, out); + filteredlength = proj_filteredlength; + } + if (filteredlength != olength) { + interpolate(out, filteredlength, output, olength); + } + } else { + interpolate(input, length, output, olength); + } +} + +static void fill_col_to_arr(uint8_t *img, int stride, int len, uint8_t *arr) { + int i; + uint8_t *iptr = img; + uint8_t *aptr = arr; + for (i = 0; i < len; ++i, iptr += stride) { + *aptr++ = *iptr; + } +} + +static void fill_arr_to_col(uint8_t *img, int stride, int len, uint8_t *arr) { + int i; + uint8_t *iptr = img; + uint8_t *aptr = arr; + for (i = 0; i < len; ++i, iptr += stride) { + *iptr = *aptr++; + } +} + +void av1_resize_plane(const uint8_t *const input, int height, int width, + int in_stride, uint8_t *output, int height2, int width2, + int out_stride) { + int i; + uint8_t *intbuf = (uint8_t *)malloc(sizeof(uint8_t) * width2 * height); + uint8_t *tmpbuf = + (uint8_t *)malloc(sizeof(uint8_t) * (width < height ? height : width)); + uint8_t *arrbuf = (uint8_t *)malloc(sizeof(uint8_t) * height); + uint8_t *arrbuf2 = (uint8_t *)malloc(sizeof(uint8_t) * height2); + if (intbuf == NULL || tmpbuf == NULL || arrbuf == NULL || arrbuf2 == NULL) + goto Error; + assert(width > 0); + assert(height > 0); + assert(width2 > 0); + assert(height2 > 0); + for (i = 0; i < height; ++i) + resize_multistep(input + in_stride * i, width, intbuf + width2 * i, width2, + tmpbuf); + for (i = 0; i < width2; ++i) { + fill_col_to_arr(intbuf + i, width2, height, arrbuf); + resize_multistep(arrbuf, height, arrbuf2, height2, tmpbuf); + fill_arr_to_col(output + i, out_stride, height2, arrbuf2); + } + +Error: + free(intbuf); + free(tmpbuf); + free(arrbuf); + free(arrbuf2); +} + +#if CONFIG_HIGHBITDEPTH +static void highbd_interpolate(const uint16_t *const input, int inlength, + uint16_t *output, int outlength, int bd) { + const int64_t delta = + (((uint64_t)inlength << 32) + outlength / 2) / outlength; + const int64_t offset = + inlength > outlength + ? (((int64_t)(inlength - outlength) << 31) + outlength / 2) / + outlength + : -(((int64_t)(outlength - inlength) << 31) + outlength / 2) / + outlength; + uint16_t *optr = output; + int x, x1, x2, sum, k, int_pel, sub_pel; + int64_t y; + + const interp_kernel *interp_filters = + choose_interp_filter(inlength, outlength); + + x = 0; + y = offset; + while ((y >> INTERP_PRECISION_BITS) < (INTERP_TAPS / 2 - 1)) { + x++; + y += delta; + } + x1 = x; + x = outlength - 1; + y = delta * x + offset; + while ((y >> INTERP_PRECISION_BITS) + (int64_t)(INTERP_TAPS / 2) >= + inlength) { + x--; + y -= delta; + } + x2 = x; + if (x1 > x2) { + for (x = 0, y = offset; x < outlength; ++x, y += delta) { + const int16_t *filter; + int_pel = y >> INTERP_PRECISION_BITS; + sub_pel = (y >> (INTERP_PRECISION_BITS - SUBPEL_BITS)) & SUBPEL_MASK; + filter = interp_filters[sub_pel]; + sum = 0; + for (k = 0; k < INTERP_TAPS; ++k) { + const int pk = int_pel - INTERP_TAPS / 2 + 1 + k; + sum += filter[k] * + input[(pk < 0 ? 0 : (pk >= inlength ? inlength - 1 : pk))]; + } + *optr++ = clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd); + } + } else { + // Initial part. + for (x = 0, y = offset; x < x1; ++x, y += delta) { + const int16_t *filter; + int_pel = y >> INTERP_PRECISION_BITS; + sub_pel = (y >> (INTERP_PRECISION_BITS - SUBPEL_BITS)) & SUBPEL_MASK; + filter = interp_filters[sub_pel]; + sum = 0; + for (k = 0; k < INTERP_TAPS; ++k) + sum += filter[k] * input[(int_pel - INTERP_TAPS / 2 + 1 + k < 0 + ? 0 + : int_pel - INTERP_TAPS / 2 + 1 + k)]; + *optr++ = clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd); + } + // Middle part. + for (; x <= x2; ++x, y += delta) { + const int16_t *filter; + int_pel = y >> INTERP_PRECISION_BITS; + sub_pel = (y >> (INTERP_PRECISION_BITS - SUBPEL_BITS)) & SUBPEL_MASK; + filter = interp_filters[sub_pel]; + sum = 0; + for (k = 0; k < INTERP_TAPS; ++k) + sum += filter[k] * input[int_pel - INTERP_TAPS / 2 + 1 + k]; + *optr++ = clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd); + } + // End part. + for (; x < outlength; ++x, y += delta) { + const int16_t *filter; + int_pel = y >> INTERP_PRECISION_BITS; + sub_pel = (y >> (INTERP_PRECISION_BITS - SUBPEL_BITS)) & SUBPEL_MASK; + filter = interp_filters[sub_pel]; + sum = 0; + for (k = 0; k < INTERP_TAPS; ++k) + sum += filter[k] * input[(int_pel - INTERP_TAPS / 2 + 1 + k >= inlength + ? inlength - 1 + : int_pel - INTERP_TAPS / 2 + 1 + k)]; + *optr++ = clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd); + } + } +} + +static void highbd_down2_symeven(const uint16_t *const input, int length, + uint16_t *output, int bd) { + // Actual filter len = 2 * filter_len_half. + static const int16_t *filter = av1_down2_symeven_half_filter; + const int filter_len_half = sizeof(av1_down2_symeven_half_filter) / 2; + int i, j; + uint16_t *optr = output; + int l1 = filter_len_half; + int l2 = (length - filter_len_half); + l1 += (l1 & 1); + l2 += (l2 & 1); + if (l1 > l2) { + // Short input length. + for (i = 0; i < length; i += 2) { + int sum = (1 << (FILTER_BITS - 1)); + for (j = 0; j < filter_len_half; ++j) { + sum += (input[(i - j < 0 ? 0 : i - j)] + + input[(i + 1 + j >= length ? length - 1 : i + 1 + j)]) * + filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel_highbd(sum, bd); + } + } else { + // Initial part. + for (i = 0; i < l1; i += 2) { + int sum = (1 << (FILTER_BITS - 1)); + for (j = 0; j < filter_len_half; ++j) { + sum += (input[(i - j < 0 ? 0 : i - j)] + input[i + 1 + j]) * filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel_highbd(sum, bd); + } + // Middle part. + for (; i < l2; i += 2) { + int sum = (1 << (FILTER_BITS - 1)); + for (j = 0; j < filter_len_half; ++j) { + sum += (input[i - j] + input[i + 1 + j]) * filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel_highbd(sum, bd); + } + // End part. + for (; i < length; i += 2) { + int sum = (1 << (FILTER_BITS - 1)); + for (j = 0; j < filter_len_half; ++j) { + sum += (input[i - j] + + input[(i + 1 + j >= length ? length - 1 : i + 1 + j)]) * + filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel_highbd(sum, bd); + } + } +} + +static void highbd_down2_symodd(const uint16_t *const input, int length, + uint16_t *output, int bd) { + // Actual filter len = 2 * filter_len_half - 1. + static const int16_t *filter = av1_down2_symodd_half_filter; + const int filter_len_half = sizeof(av1_down2_symodd_half_filter) / 2; + int i, j; + uint16_t *optr = output; + int l1 = filter_len_half - 1; + int l2 = (length - filter_len_half + 1); + l1 += (l1 & 1); + l2 += (l2 & 1); + if (l1 > l2) { + // Short input length. + for (i = 0; i < length; i += 2) { + int sum = (1 << (FILTER_BITS - 1)) + input[i] * filter[0]; + for (j = 1; j < filter_len_half; ++j) { + sum += (input[(i - j < 0 ? 0 : i - j)] + + input[(i + j >= length ? length - 1 : i + j)]) * + filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel_highbd(sum, bd); + } + } else { + // Initial part. + for (i = 0; i < l1; i += 2) { + int sum = (1 << (FILTER_BITS - 1)) + input[i] * filter[0]; + for (j = 1; j < filter_len_half; ++j) { + sum += (input[(i - j < 0 ? 0 : i - j)] + input[i + j]) * filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel_highbd(sum, bd); + } + // Middle part. + for (; i < l2; i += 2) { + int sum = (1 << (FILTER_BITS - 1)) + input[i] * filter[0]; + for (j = 1; j < filter_len_half; ++j) { + sum += (input[i - j] + input[i + j]) * filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel_highbd(sum, bd); + } + // End part. + for (; i < length; i += 2) { + int sum = (1 << (FILTER_BITS - 1)) + input[i] * filter[0]; + for (j = 1; j < filter_len_half; ++j) { + sum += (input[i - j] + input[(i + j >= length ? length - 1 : i + j)]) * + filter[j]; + } + sum >>= FILTER_BITS; + *optr++ = clip_pixel_highbd(sum, bd); + } + } +} + +static void highbd_resize_multistep(const uint16_t *const input, int length, + uint16_t *output, int olength, + uint16_t *otmp, int bd) { + int steps; + if (length == olength) { + memcpy(output, input, sizeof(output[0]) * length); + return; + } + steps = get_down2_steps(length, olength); + + if (steps > 0) { + int s; + uint16_t *out = NULL; + uint16_t *otmp2; + int filteredlength = length; + + assert(otmp != NULL); + otmp2 = otmp + get_down2_length(length, 1); + for (s = 0; s < steps; ++s) { + const int proj_filteredlength = get_down2_length(filteredlength, 1); + const uint16_t *const in = (s == 0 ? input : out); + if (s == steps - 1 && proj_filteredlength == olength) + out = output; + else + out = (s & 1 ? otmp2 : otmp); + if (filteredlength & 1) + highbd_down2_symodd(in, filteredlength, out, bd); + else + highbd_down2_symeven(in, filteredlength, out, bd); + filteredlength = proj_filteredlength; + } + if (filteredlength != olength) { + highbd_interpolate(out, filteredlength, output, olength, bd); + } + } else { + highbd_interpolate(input, length, output, olength, bd); + } +} + +static void highbd_fill_col_to_arr(uint16_t *img, int stride, int len, + uint16_t *arr) { + int i; + uint16_t *iptr = img; + uint16_t *aptr = arr; + for (i = 0; i < len; ++i, iptr += stride) { + *aptr++ = *iptr; + } +} + +static void highbd_fill_arr_to_col(uint16_t *img, int stride, int len, + uint16_t *arr) { + int i; + uint16_t *iptr = img; + uint16_t *aptr = arr; + for (i = 0; i < len; ++i, iptr += stride) { + *iptr = *aptr++; + } +} + +void av1_highbd_resize_plane(const uint8_t *const input, int height, int width, + int in_stride, uint8_t *output, int height2, + int width2, int out_stride, int bd) { + int i; + uint16_t *intbuf = (uint16_t *)malloc(sizeof(uint16_t) * width2 * height); + uint16_t *tmpbuf = + (uint16_t *)malloc(sizeof(uint16_t) * (width < height ? height : width)); + uint16_t *arrbuf = (uint16_t *)malloc(sizeof(uint16_t) * height); + uint16_t *arrbuf2 = (uint16_t *)malloc(sizeof(uint16_t) * height2); + if (intbuf == NULL || tmpbuf == NULL || arrbuf == NULL || arrbuf2 == NULL) + goto Error; + for (i = 0; i < height; ++i) { + highbd_resize_multistep(CONVERT_TO_SHORTPTR(input + in_stride * i), width, + intbuf + width2 * i, width2, tmpbuf, bd); + } + for (i = 0; i < width2; ++i) { + highbd_fill_col_to_arr(intbuf + i, width2, height, arrbuf); + highbd_resize_multistep(arrbuf, height, arrbuf2, height2, tmpbuf, bd); + highbd_fill_arr_to_col(CONVERT_TO_SHORTPTR(output + i), out_stride, height2, + arrbuf2); + } + +Error: + free(intbuf); + free(tmpbuf); + free(arrbuf); + free(arrbuf2); +} +#endif // CONFIG_HIGHBITDEPTH + +void av1_resize_frame420(const uint8_t *const y, int y_stride, + const uint8_t *const u, const uint8_t *const v, + int uv_stride, int height, int width, uint8_t *oy, + int oy_stride, uint8_t *ou, uint8_t *ov, + int ouv_stride, int oheight, int owidth) { + av1_resize_plane(y, height, width, y_stride, oy, oheight, owidth, oy_stride); + av1_resize_plane(u, height / 2, width / 2, uv_stride, ou, oheight / 2, + owidth / 2, ouv_stride); + av1_resize_plane(v, height / 2, width / 2, uv_stride, ov, oheight / 2, + owidth / 2, ouv_stride); +} + +void av1_resize_frame422(const uint8_t *const y, int y_stride, + const uint8_t *const u, const uint8_t *const v, + int uv_stride, int height, int width, uint8_t *oy, + int oy_stride, uint8_t *ou, uint8_t *ov, + int ouv_stride, int oheight, int owidth) { + av1_resize_plane(y, height, width, y_stride, oy, oheight, owidth, oy_stride); + av1_resize_plane(u, height, width / 2, uv_stride, ou, oheight, owidth / 2, + ouv_stride); + av1_resize_plane(v, height, width / 2, uv_stride, ov, oheight, owidth / 2, + ouv_stride); +} + +void av1_resize_frame444(const uint8_t *const y, int y_stride, + const uint8_t *const u, const uint8_t *const v, + int uv_stride, int height, int width, uint8_t *oy, + int oy_stride, uint8_t *ou, uint8_t *ov, + int ouv_stride, int oheight, int owidth) { + av1_resize_plane(y, height, width, y_stride, oy, oheight, owidth, oy_stride); + av1_resize_plane(u, height, width, uv_stride, ou, oheight, owidth, + ouv_stride); + av1_resize_plane(v, height, width, uv_stride, ov, oheight, owidth, + ouv_stride); +} + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_resize_frame420(const uint8_t *const y, int y_stride, + const uint8_t *const u, const uint8_t *const v, + int uv_stride, int height, int width, + uint8_t *oy, int oy_stride, uint8_t *ou, + uint8_t *ov, int ouv_stride, int oheight, + int owidth, int bd) { + av1_highbd_resize_plane(y, height, width, y_stride, oy, oheight, owidth, + oy_stride, bd); + av1_highbd_resize_plane(u, height / 2, width / 2, uv_stride, ou, oheight / 2, + owidth / 2, ouv_stride, bd); + av1_highbd_resize_plane(v, height / 2, width / 2, uv_stride, ov, oheight / 2, + owidth / 2, ouv_stride, bd); +} + +void av1_highbd_resize_frame422(const uint8_t *const y, int y_stride, + const uint8_t *const u, const uint8_t *const v, + int uv_stride, int height, int width, + uint8_t *oy, int oy_stride, uint8_t *ou, + uint8_t *ov, int ouv_stride, int oheight, + int owidth, int bd) { + av1_highbd_resize_plane(y, height, width, y_stride, oy, oheight, owidth, + oy_stride, bd); + av1_highbd_resize_plane(u, height, width / 2, uv_stride, ou, oheight, + owidth / 2, ouv_stride, bd); + av1_highbd_resize_plane(v, height, width / 2, uv_stride, ov, oheight, + owidth / 2, ouv_stride, bd); +} + +void av1_highbd_resize_frame444(const uint8_t *const y, int y_stride, + const uint8_t *const u, const uint8_t *const v, + int uv_stride, int height, int width, + uint8_t *oy, int oy_stride, uint8_t *ou, + uint8_t *ov, int ouv_stride, int oheight, + int owidth, int bd) { + av1_highbd_resize_plane(y, height, width, y_stride, oy, oheight, owidth, + oy_stride, bd); + av1_highbd_resize_plane(u, height, width, uv_stride, ou, oheight, owidth, + ouv_stride, bd); + av1_highbd_resize_plane(v, height, width, uv_stride, ov, oheight, owidth, + ouv_stride, bd); +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/av1/common/resize.h b/third_party/aom/av1/common/resize.h new file mode 100644 index 0000000000..959cda969a --- /dev/null +++ b/third_party/aom/av1/common/resize.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_RESIZE_H_ +#define AV1_ENCODER_RESIZE_H_ + +#include +#include "aom/aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_resize_plane(const uint8_t *const input, int height, int width, + int in_stride, uint8_t *output, int height2, int width2, + int out_stride); +void av1_resize_frame420(const uint8_t *const y, int y_stride, + const uint8_t *const u, const uint8_t *const v, + int uv_stride, int height, int width, uint8_t *oy, + int oy_stride, uint8_t *ou, uint8_t *ov, + int ouv_stride, int oheight, int owidth); +void av1_resize_frame422(const uint8_t *const y, int y_stride, + const uint8_t *const u, const uint8_t *const v, + int uv_stride, int height, int width, uint8_t *oy, + int oy_stride, uint8_t *ou, uint8_t *ov, + int ouv_stride, int oheight, int owidth); +void av1_resize_frame444(const uint8_t *const y, int y_stride, + const uint8_t *const u, const uint8_t *const v, + int uv_stride, int height, int width, uint8_t *oy, + int oy_stride, uint8_t *ou, uint8_t *ov, + int ouv_stride, int oheight, int owidth); + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_resize_plane(const uint8_t *const input, int height, int width, + int in_stride, uint8_t *output, int height2, + int width2, int out_stride, int bd); +void av1_highbd_resize_frame420(const uint8_t *const y, int y_stride, + const uint8_t *const u, const uint8_t *const v, + int uv_stride, int height, int width, + uint8_t *oy, int oy_stride, uint8_t *ou, + uint8_t *ov, int ouv_stride, int oheight, + int owidth, int bd); +void av1_highbd_resize_frame422(const uint8_t *const y, int y_stride, + const uint8_t *const u, const uint8_t *const v, + int uv_stride, int height, int width, + uint8_t *oy, int oy_stride, uint8_t *ou, + uint8_t *ov, int ouv_stride, int oheight, + int owidth, int bd); +void av1_highbd_resize_frame444(const uint8_t *const y, int y_stride, + const uint8_t *const u, const uint8_t *const v, + int uv_stride, int height, int width, + uint8_t *oy, int oy_stride, uint8_t *ou, + uint8_t *ov, int ouv_stride, int oheight, + int owidth, int bd); +#endif // CONFIG_HIGHBITDEPTH + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_RESIZE_H_ diff --git a/third_party/aom/av1/common/restoration.c b/third_party/aom/av1/common/restoration.c new file mode 100644 index 0000000000..b7ed9f98bc --- /dev/null +++ b/third_party/aom/av1/common/restoration.c @@ -0,0 +1,1401 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + * + */ + +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "./aom_scale_rtcd.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/restoration.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +const sgr_params_type sgr_params[SGRPROJ_PARAMS] = { +#if USE_HIGHPASS_IN_SGRPROJ + // corner, edge, r2, eps2 + { -1, 2, 1, 1 }, { -1, 2, 1, 2 }, { -1, 2, 1, 3 }, { -1, 2, 1, 4 }, + { -1, 2, 1, 5 }, { -2, 3, 1, 2 }, { -2, 3, 1, 3 }, { -2, 3, 1, 4 }, + { -2, 3, 1, 5 }, { -2, 3, 1, 6 }, { -3, 4, 1, 3 }, { -3, 4, 1, 4 }, + { -3, 4, 1, 5 }, { -3, 4, 1, 6 }, { -3, 4, 1, 7 }, { -3, 4, 1, 8 } +#else + // r1, eps1, r2, eps2 + { 2, 12, 1, 4 }, { 2, 15, 1, 6 }, { 2, 18, 1, 8 }, { 2, 20, 1, 9 }, + { 2, 22, 1, 10 }, { 2, 25, 1, 11 }, { 2, 35, 1, 12 }, { 2, 45, 1, 13 }, + { 2, 55, 1, 14 }, { 2, 65, 1, 15 }, { 2, 75, 1, 16 }, { 3, 30, 1, 10 }, + { 3, 50, 1, 12 }, { 3, 50, 2, 25 }, { 3, 60, 2, 35 }, { 3, 70, 2, 45 }, +#endif +}; + +typedef void (*restore_func_type)(uint8_t *data8, int width, int height, + int stride, RestorationInternal *rst, + uint8_t *dst8, int dst_stride); +#if CONFIG_HIGHBITDEPTH +typedef void (*restore_func_highbd_type)(uint8_t *data8, int width, int height, + int stride, RestorationInternal *rst, + int bit_depth, uint8_t *dst8, + int dst_stride); +#endif // CONFIG_HIGHBITDEPTH + +int av1_alloc_restoration_struct(AV1_COMMON *cm, RestorationInfo *rst_info, + int width, int height) { + const int ntiles = av1_get_rest_ntiles( + width, height, rst_info->restoration_tilesize, NULL, NULL, NULL, NULL); + aom_free(rst_info->restoration_type); + CHECK_MEM_ERROR(cm, rst_info->restoration_type, + (RestorationType *)aom_malloc( + sizeof(*rst_info->restoration_type) * ntiles)); + aom_free(rst_info->wiener_info); + CHECK_MEM_ERROR( + cm, rst_info->wiener_info, + (WienerInfo *)aom_memalign(16, sizeof(*rst_info->wiener_info) * ntiles)); + memset(rst_info->wiener_info, 0, sizeof(*rst_info->wiener_info) * ntiles); + aom_free(rst_info->sgrproj_info); + CHECK_MEM_ERROR( + cm, rst_info->sgrproj_info, + (SgrprojInfo *)aom_malloc(sizeof(*rst_info->sgrproj_info) * ntiles)); + return ntiles; +} + +void av1_free_restoration_struct(RestorationInfo *rst_info) { + aom_free(rst_info->restoration_type); + rst_info->restoration_type = NULL; + aom_free(rst_info->wiener_info); + rst_info->wiener_info = NULL; + aom_free(rst_info->sgrproj_info); + rst_info->sgrproj_info = NULL; +} + +#define MAX_RADIUS 3 // Only 1, 2, 3 allowed +#define MAX_EPS 80 // Max value of eps +#define MAX_NELEM ((2 * MAX_RADIUS + 1) * (2 * MAX_RADIUS + 1)) +#define SGRPROJ_MTABLE_BITS 20 +#define SGRPROJ_RECIP_BITS 12 + +// TODO(debargha): This table can be substantially reduced since only a few +// values are actually used. +int sgrproj_mtable[MAX_EPS][MAX_NELEM]; + +static void GenSgrprojVtable() { + int e, n; + for (e = 1; e <= MAX_EPS; ++e) + for (n = 1; n <= MAX_NELEM; ++n) { + const int n2e = n * n * e; + sgrproj_mtable[e - 1][n - 1] = + (((1 << SGRPROJ_MTABLE_BITS) + n2e / 2) / n2e); + } +} + +void av1_loop_restoration_precal() { GenSgrprojVtable(); } + +static void loop_restoration_init(RestorationInternal *rst, int kf) { + rst->keyframe = kf; +} + +void extend_frame(uint8_t *data, int width, int height, int stride) { + uint8_t *data_p; + int i; + for (i = 0; i < height; ++i) { + data_p = data + i * stride; + memset(data_p - WIENER_HALFWIN, data_p[0], WIENER_HALFWIN); + memset(data_p + width, data_p[width - 1], WIENER_HALFWIN); + } + data_p = data - WIENER_HALFWIN; + for (i = -WIENER_HALFWIN; i < 0; ++i) { + memcpy(data_p + i * stride, data_p, width + 2 * WIENER_HALFWIN); + } + for (i = height; i < height + WIENER_HALFWIN; ++i) { + memcpy(data_p + i * stride, data_p + (height - 1) * stride, + width + 2 * WIENER_HALFWIN); + } +} + +static void loop_copy_tile(uint8_t *data, int tile_idx, int subtile_idx, + int subtile_bits, int width, int height, int stride, + RestorationInternal *rst, uint8_t *dst, + int dst_stride) { + const int tile_width = rst->tile_width; + const int tile_height = rst->tile_height; + int i; + int h_start, h_end, v_start, v_end; + av1_get_rest_tile_limits(tile_idx, subtile_idx, subtile_bits, rst->nhtiles, + rst->nvtiles, tile_width, tile_height, width, height, + 0, 0, &h_start, &h_end, &v_start, &v_end); + for (i = v_start; i < v_end; ++i) + memcpy(dst + i * dst_stride + h_start, data + i * stride + h_start, + h_end - h_start); +} + +static void loop_wiener_filter_tile(uint8_t *data, int tile_idx, int width, + int height, int stride, + RestorationInternal *rst, uint8_t *dst, + int dst_stride) { + const int tile_width = rst->tile_width; + const int tile_height = rst->tile_height; + int i, j; + int h_start, h_end, v_start, v_end; + if (rst->rsi->restoration_type[tile_idx] == RESTORE_NONE) { + loop_copy_tile(data, tile_idx, 0, 0, width, height, stride, rst, dst, + dst_stride); + return; + } + av1_get_rest_tile_limits(tile_idx, 0, 0, rst->nhtiles, rst->nvtiles, + tile_width, tile_height, width, height, 0, 0, + &h_start, &h_end, &v_start, &v_end); + // Convolve the whole tile (done in blocks here to match the requirements + // of the vectorized convolve functions, but the result is equivalent) + for (i = v_start; i < v_end; i += MAX_SB_SIZE) + for (j = h_start; j < h_end; j += MAX_SB_SIZE) { + int w = AOMMIN(MAX_SB_SIZE, (h_end - j + 15) & ~15); + int h = AOMMIN(MAX_SB_SIZE, (v_end - i + 15) & ~15); + const uint8_t *data_p = data + i * stride + j; + uint8_t *dst_p = dst + i * dst_stride + j; + aom_convolve8_add_src(data_p, stride, dst_p, dst_stride, + rst->rsi->wiener_info[tile_idx].hfilter, 16, + rst->rsi->wiener_info[tile_idx].vfilter, 16, w, h); + } +} + +static void loop_wiener_filter(uint8_t *data, int width, int height, int stride, + RestorationInternal *rst, uint8_t *dst, + int dst_stride) { + int tile_idx; + extend_frame(data, width, height, stride); + for (tile_idx = 0; tile_idx < rst->ntiles; ++tile_idx) { + loop_wiener_filter_tile(data, tile_idx, width, height, stride, rst, dst, + dst_stride); + } +} + +/* Calculate windowed sums (if sqr=0) or sums of squares (if sqr=1) + over the input. The window is of size (2r + 1)x(2r + 1), and we + specialize to r = 1, 2, 3. A default function is used for r > 3. + + Each loop follows the same format: We keep a window's worth of input + in individual variables and select data out of that as appropriate. +*/ +static void boxsum1(int32_t *src, int width, int height, int src_stride, + int sqr, int32_t *dst, int dst_stride) { + int i, j, a, b, c; + + // Vertical sum over 3-pixel regions, from src into dst. + if (!sqr) { + for (j = 0; j < width; ++j) { + a = src[j]; + b = src[src_stride + j]; + c = src[2 * src_stride + j]; + + dst[j] = a + b; + for (i = 1; i < height - 2; ++i) { + // Loop invariant: At the start of each iteration, + // a = src[(i - 1) * src_stride + j] + // b = src[(i ) * src_stride + j] + // c = src[(i + 1) * src_stride + j] + dst[i * dst_stride + j] = a + b + c; + a = b; + b = c; + c = src[(i + 2) * src_stride + j]; + } + dst[i * dst_stride + j] = a + b + c; + dst[(i + 1) * dst_stride + j] = b + c; + } + } else { + for (j = 0; j < width; ++j) { + a = src[j] * src[j]; + b = src[src_stride + j] * src[src_stride + j]; + c = src[2 * src_stride + j] * src[2 * src_stride + j]; + + dst[j] = a + b; + for (i = 1; i < height - 2; ++i) { + dst[i * dst_stride + j] = a + b + c; + a = b; + b = c; + c = src[(i + 2) * src_stride + j] * src[(i + 2) * src_stride + j]; + } + dst[i * dst_stride + j] = a + b + c; + dst[(i + 1) * dst_stride + j] = b + c; + } + } + + // Horizontal sum over 3-pixel regions of dst + for (i = 0; i < height; ++i) { + a = dst[i * dst_stride]; + b = dst[i * dst_stride + 1]; + c = dst[i * dst_stride + 2]; + + dst[i * dst_stride] = a + b; + for (j = 1; j < width - 2; ++j) { + // Loop invariant: At the start of each iteration, + // a = src[i * src_stride + (j - 1)] + // b = src[i * src_stride + (j )] + // c = src[i * src_stride + (j + 1)] + dst[i * dst_stride + j] = a + b + c; + a = b; + b = c; + c = dst[i * dst_stride + (j + 2)]; + } + dst[i * dst_stride + j] = a + b + c; + dst[i * dst_stride + (j + 1)] = b + c; + } +} + +static void boxsum2(int32_t *src, int width, int height, int src_stride, + int sqr, int32_t *dst, int dst_stride) { + int i, j, a, b, c, d, e; + + // Vertical sum over 5-pixel regions, from src into dst. + if (!sqr) { + for (j = 0; j < width; ++j) { + a = src[j]; + b = src[src_stride + j]; + c = src[2 * src_stride + j]; + d = src[3 * src_stride + j]; + e = src[4 * src_stride + j]; + + dst[j] = a + b + c; + dst[dst_stride + j] = a + b + c + d; + for (i = 2; i < height - 3; ++i) { + // Loop invariant: At the start of each iteration, + // a = src[(i - 2) * src_stride + j] + // b = src[(i - 1) * src_stride + j] + // c = src[(i ) * src_stride + j] + // d = src[(i + 1) * src_stride + j] + // e = src[(i + 2) * src_stride + j] + dst[i * dst_stride + j] = a + b + c + d + e; + a = b; + b = c; + c = d; + d = e; + e = src[(i + 3) * src_stride + j]; + } + dst[i * dst_stride + j] = a + b + c + d + e; + dst[(i + 1) * dst_stride + j] = b + c + d + e; + dst[(i + 2) * dst_stride + j] = c + d + e; + } + } else { + for (j = 0; j < width; ++j) { + a = src[j] * src[j]; + b = src[src_stride + j] * src[src_stride + j]; + c = src[2 * src_stride + j] * src[2 * src_stride + j]; + d = src[3 * src_stride + j] * src[3 * src_stride + j]; + e = src[4 * src_stride + j] * src[4 * src_stride + j]; + + dst[j] = a + b + c; + dst[dst_stride + j] = a + b + c + d; + for (i = 2; i < height - 3; ++i) { + dst[i * dst_stride + j] = a + b + c + d + e; + a = b; + b = c; + c = d; + d = e; + e = src[(i + 3) * src_stride + j] * src[(i + 3) * src_stride + j]; + } + dst[i * dst_stride + j] = a + b + c + d + e; + dst[(i + 1) * dst_stride + j] = b + c + d + e; + dst[(i + 2) * dst_stride + j] = c + d + e; + } + } + + // Horizontal sum over 5-pixel regions of dst + for (i = 0; i < height; ++i) { + a = dst[i * dst_stride]; + b = dst[i * dst_stride + 1]; + c = dst[i * dst_stride + 2]; + d = dst[i * dst_stride + 3]; + e = dst[i * dst_stride + 4]; + + dst[i * dst_stride] = a + b + c; + dst[i * dst_stride + 1] = a + b + c + d; + for (j = 2; j < width - 3; ++j) { + // Loop invariant: At the start of each iteration, + // a = src[i * src_stride + (j - 2)] + // b = src[i * src_stride + (j - 1)] + // c = src[i * src_stride + (j )] + // d = src[i * src_stride + (j + 1)] + // e = src[i * src_stride + (j + 2)] + dst[i * dst_stride + j] = a + b + c + d + e; + a = b; + b = c; + c = d; + d = e; + e = dst[i * dst_stride + (j + 3)]; + } + dst[i * dst_stride + j] = a + b + c + d + e; + dst[i * dst_stride + (j + 1)] = b + c + d + e; + dst[i * dst_stride + (j + 2)] = c + d + e; + } +} + +static void boxsum3(int32_t *src, int width, int height, int src_stride, + int sqr, int32_t *dst, int dst_stride) { + int i, j, a, b, c, d, e, f, g; + + // Vertical sum over 7-pixel regions, from src into dst. + if (!sqr) { + for (j = 0; j < width; ++j) { + a = src[j]; + b = src[1 * src_stride + j]; + c = src[2 * src_stride + j]; + d = src[3 * src_stride + j]; + e = src[4 * src_stride + j]; + f = src[5 * src_stride + j]; + g = src[6 * src_stride + j]; + + dst[j] = a + b + c + d; + dst[dst_stride + j] = a + b + c + d + e; + dst[2 * dst_stride + j] = a + b + c + d + e + f; + for (i = 3; i < height - 4; ++i) { + dst[i * dst_stride + j] = a + b + c + d + e + f + g; + a = b; + b = c; + c = d; + d = e; + e = f; + f = g; + g = src[(i + 4) * src_stride + j]; + } + dst[i * dst_stride + j] = a + b + c + d + e + f + g; + dst[(i + 1) * dst_stride + j] = b + c + d + e + f + g; + dst[(i + 2) * dst_stride + j] = c + d + e + f + g; + dst[(i + 3) * dst_stride + j] = d + e + f + g; + } + } else { + for (j = 0; j < width; ++j) { + a = src[j] * src[j]; + b = src[1 * src_stride + j] * src[1 * src_stride + j]; + c = src[2 * src_stride + j] * src[2 * src_stride + j]; + d = src[3 * src_stride + j] * src[3 * src_stride + j]; + e = src[4 * src_stride + j] * src[4 * src_stride + j]; + f = src[5 * src_stride + j] * src[5 * src_stride + j]; + g = src[6 * src_stride + j] * src[6 * src_stride + j]; + + dst[j] = a + b + c + d; + dst[dst_stride + j] = a + b + c + d + e; + dst[2 * dst_stride + j] = a + b + c + d + e + f; + for (i = 3; i < height - 4; ++i) { + dst[i * dst_stride + j] = a + b + c + d + e + f + g; + a = b; + b = c; + c = d; + d = e; + e = f; + f = g; + g = src[(i + 4) * src_stride + j] * src[(i + 4) * src_stride + j]; + } + dst[i * dst_stride + j] = a + b + c + d + e + f + g; + dst[(i + 1) * dst_stride + j] = b + c + d + e + f + g; + dst[(i + 2) * dst_stride + j] = c + d + e + f + g; + dst[(i + 3) * dst_stride + j] = d + e + f + g; + } + } + + // Horizontal sum over 7-pixel regions of dst + for (i = 0; i < height; ++i) { + a = dst[i * dst_stride]; + b = dst[i * dst_stride + 1]; + c = dst[i * dst_stride + 2]; + d = dst[i * dst_stride + 3]; + e = dst[i * dst_stride + 4]; + f = dst[i * dst_stride + 5]; + g = dst[i * dst_stride + 6]; + + dst[i * dst_stride] = a + b + c + d; + dst[i * dst_stride + 1] = a + b + c + d + e; + dst[i * dst_stride + 2] = a + b + c + d + e + f; + for (j = 3; j < width - 4; ++j) { + dst[i * dst_stride + j] = a + b + c + d + e + f + g; + a = b; + b = c; + c = d; + d = e; + e = f; + f = g; + g = dst[i * dst_stride + (j + 4)]; + } + dst[i * dst_stride + j] = a + b + c + d + e + f + g; + dst[i * dst_stride + (j + 1)] = b + c + d + e + f + g; + dst[i * dst_stride + (j + 2)] = c + d + e + f + g; + dst[i * dst_stride + (j + 3)] = d + e + f + g; + } +} + +// Generic version for any r. To be removed after experiments are done. +static void boxsumr(int32_t *src, int width, int height, int src_stride, int r, + int sqr, int32_t *dst, int dst_stride) { + int32_t *tmp = aom_malloc(width * height * sizeof(*tmp)); + int tmp_stride = width; + int i, j; + if (sqr) { + for (j = 0; j < width; ++j) tmp[j] = src[j] * src[j]; + for (j = 0; j < width; ++j) + for (i = 1; i < height; ++i) + tmp[i * tmp_stride + j] = + tmp[(i - 1) * tmp_stride + j] + + src[i * src_stride + j] * src[i * src_stride + j]; + } else { + memcpy(tmp, src, sizeof(*tmp) * width); + for (j = 0; j < width; ++j) + for (i = 1; i < height; ++i) + tmp[i * tmp_stride + j] = + tmp[(i - 1) * tmp_stride + j] + src[i * src_stride + j]; + } + for (i = 0; i <= r; ++i) + memcpy(&dst[i * dst_stride], &tmp[(i + r) * tmp_stride], + sizeof(*tmp) * width); + for (i = r + 1; i < height - r; ++i) + for (j = 0; j < width; ++j) + dst[i * dst_stride + j] = + tmp[(i + r) * tmp_stride + j] - tmp[(i - r - 1) * tmp_stride + j]; + for (i = height - r; i < height; ++i) + for (j = 0; j < width; ++j) + dst[i * dst_stride + j] = tmp[(height - 1) * tmp_stride + j] - + tmp[(i - r - 1) * tmp_stride + j]; + + for (i = 0; i < height; ++i) tmp[i * tmp_stride] = dst[i * dst_stride]; + for (i = 0; i < height; ++i) + for (j = 1; j < width; ++j) + tmp[i * tmp_stride + j] = + tmp[i * tmp_stride + j - 1] + dst[i * src_stride + j]; + + for (j = 0; j <= r; ++j) + for (i = 0; i < height; ++i) + dst[i * dst_stride + j] = tmp[i * tmp_stride + j + r]; + for (j = r + 1; j < width - r; ++j) + for (i = 0; i < height; ++i) + dst[i * dst_stride + j] = + tmp[i * tmp_stride + j + r] - tmp[i * tmp_stride + j - r - 1]; + for (j = width - r; j < width; ++j) + for (i = 0; i < height; ++i) + dst[i * dst_stride + j] = + tmp[i * tmp_stride + width - 1] - tmp[i * tmp_stride + j - r - 1]; + aom_free(tmp); +} + +static void boxsum(int32_t *src, int width, int height, int src_stride, int r, + int sqr, int32_t *dst, int dst_stride) { + if (r == 1) + boxsum1(src, width, height, src_stride, sqr, dst, dst_stride); + else if (r == 2) + boxsum2(src, width, height, src_stride, sqr, dst, dst_stride); + else if (r == 3) + boxsum3(src, width, height, src_stride, sqr, dst, dst_stride); + else + boxsumr(src, width, height, src_stride, r, sqr, dst, dst_stride); +} + +static void boxnum(int width, int height, int r, int8_t *num, int num_stride) { + int i, j; + for (i = 0; i <= r; ++i) { + for (j = 0; j <= r; ++j) { + num[i * num_stride + j] = (r + 1 + i) * (r + 1 + j); + num[i * num_stride + (width - 1 - j)] = num[i * num_stride + j]; + num[(height - 1 - i) * num_stride + j] = num[i * num_stride + j]; + num[(height - 1 - i) * num_stride + (width - 1 - j)] = + num[i * num_stride + j]; + } + } + for (j = 0; j <= r; ++j) { + const int val = (2 * r + 1) * (r + 1 + j); + for (i = r + 1; i < height - r; ++i) { + num[i * num_stride + j] = val; + num[i * num_stride + (width - 1 - j)] = val; + } + } + for (i = 0; i <= r; ++i) { + const int val = (2 * r + 1) * (r + 1 + i); + for (j = r + 1; j < width - r; ++j) { + num[i * num_stride + j] = val; + num[(height - 1 - i) * num_stride + j] = val; + } + } + for (i = r + 1; i < height - r; ++i) { + for (j = r + 1; j < width - r; ++j) { + num[i * num_stride + j] = (2 * r + 1) * (2 * r + 1); + } + } +} + +void decode_xq(int *xqd, int *xq) { + xq[0] = xqd[0]; + xq[1] = (1 << SGRPROJ_PRJ_BITS) - xq[0] - xqd[1]; +} + +const int32_t x_by_xplus1[256] = { + 0, 128, 171, 192, 205, 213, 219, 224, 228, 230, 233, 235, 236, 238, 239, + 240, 241, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247, 247, 247, 247, + 248, 248, 248, 248, 249, 249, 249, 249, 249, 250, 250, 250, 250, 250, 250, + 250, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 252, 252, 252, 252, + 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 253, 253, + 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, + 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 254, 254, 254, + 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, + 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, + 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, + 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, + 254, 254, 254, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 256, +}; + +const int32_t one_by_x[MAX_NELEM] = { + 4096, 2048, 1365, 1024, 819, 683, 585, 512, 455, 410, 372, 341, 315, + 293, 273, 256, 241, 228, 216, 205, 195, 186, 178, 171, 164, 158, + 152, 146, 141, 137, 132, 128, 124, 120, 117, 114, 111, 108, 105, + 102, 100, 98, 95, 93, 91, 89, 87, 85, 84 +}; + +static void av1_selfguided_restoration_internal(int32_t *dgd, int width, + int height, int stride, + int bit_depth, int r, int eps, + int32_t *tmpbuf) { + int32_t *A = tmpbuf; + int32_t *B = A + SGRPROJ_OUTBUF_SIZE; + int8_t num[RESTORATION_TILEPELS_MAX]; + int i, j; + // Adjusting the stride of A and B here appears to avoid bad cache effects, + // leading to a significant speed improvement. + // We also align the stride to a multiple of 16 bytes, for consistency + // with the SIMD version of this function. + int buf_stride = ((width + 3) & ~3) + 16; + + // Don't filter tiles with dimensions < 5 on any axis + if ((width < 5) || (height < 5)) return; + + boxsum(dgd, width, height, stride, r, 0, B, buf_stride); + boxsum(dgd, width, height, stride, r, 1, A, buf_stride); + boxnum(width, height, r, num, width); + assert(r <= 3); + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const int k = i * buf_stride + j; + const int n = num[i * width + j]; + + // a < 2^16 * n < 2^22 regardless of bit depth + uint32_t a = ROUND_POWER_OF_TWO(A[k], 2 * (bit_depth - 8)); + // b < 2^8 * n < 2^14 regardless of bit depth + uint32_t b = ROUND_POWER_OF_TWO(B[k], bit_depth - 8); + + // Each term in calculating p = a * n - b * b is < 2^16 * n^2 < 2^28, + // and p itself satisfies p < 2^14 * n^2 < 2^26. + // Note: Sometimes, in high bit depth, we can end up with a*n < b*b. + // This is an artefact of rounding, and can only happen if all pixels + // are (almost) identical, so in this case we saturate to p=0. + uint32_t p = (a * n < b * b) ? 0 : a * n - b * b; + uint32_t s = sgrproj_mtable[eps - 1][n - 1]; + + // p * s < (2^14 * n^2) * round(2^20 / n^2 eps) < 2^34 / eps < 2^32 + // as long as eps >= 4. So p * s fits into a uint32_t, and z < 2^12 + // (this holds even after accounting for the rounding in s) + const uint32_t z = ROUND_POWER_OF_TWO(p * s, SGRPROJ_MTABLE_BITS); + + A[k] = x_by_xplus1[AOMMIN(z, 255)]; // < 2^8 + + // SGRPROJ_SGR - A[k] < 2^8, B[k] < 2^(bit_depth) * n, + // one_by_x[n - 1] = round(2^12 / n) + // => the product here is < 2^(20 + bit_depth) <= 2^32, + // and B[k] is set to a value < 2^(8 + bit depth) + B[k] = (int32_t)ROUND_POWER_OF_TWO((uint32_t)(SGRPROJ_SGR - A[k]) * + (uint32_t)B[k] * + (uint32_t)one_by_x[n - 1], + SGRPROJ_RECIP_BITS); + } + } + i = 0; + j = 0; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int nb = 3; + const int32_t a = + 3 * A[k] + 2 * A[k + 1] + 2 * A[k + buf_stride] + A[k + buf_stride + 1]; + const int32_t b = + 3 * B[k] + 2 * B[k + 1] + 2 * B[k + buf_stride] + B[k + buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dgd[l] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + i = 0; + j = width - 1; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int nb = 3; + const int32_t a = + 3 * A[k] + 2 * A[k - 1] + 2 * A[k + buf_stride] + A[k + buf_stride - 1]; + const int32_t b = + 3 * B[k] + 2 * B[k - 1] + 2 * B[k + buf_stride] + B[k + buf_stride - 1]; + const int32_t v = a * dgd[l] + b; + dgd[l] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + i = height - 1; + j = 0; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int nb = 3; + const int32_t a = + 3 * A[k] + 2 * A[k + 1] + 2 * A[k - buf_stride] + A[k - buf_stride + 1]; + const int32_t b = + 3 * B[k] + 2 * B[k + 1] + 2 * B[k - buf_stride] + B[k - buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dgd[l] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + i = height - 1; + j = width - 1; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int nb = 3; + const int32_t a = + 3 * A[k] + 2 * A[k - 1] + 2 * A[k - buf_stride] + A[k - buf_stride - 1]; + const int32_t b = + 3 * B[k] + 2 * B[k - 1] + 2 * B[k - buf_stride] + B[k - buf_stride - 1]; + const int32_t v = a * dgd[l] + b; + dgd[l] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + i = 0; + for (j = 1; j < width - 1; ++j) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int nb = 3; + const int32_t a = A[k] + 2 * (A[k - 1] + A[k + 1]) + A[k + buf_stride] + + A[k + buf_stride - 1] + A[k + buf_stride + 1]; + const int32_t b = B[k] + 2 * (B[k - 1] + B[k + 1]) + B[k + buf_stride] + + B[k + buf_stride - 1] + B[k + buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dgd[l] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + i = height - 1; + for (j = 1; j < width - 1; ++j) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int nb = 3; + const int32_t a = A[k] + 2 * (A[k - 1] + A[k + 1]) + A[k - buf_stride] + + A[k - buf_stride - 1] + A[k - buf_stride + 1]; + const int32_t b = B[k] + 2 * (B[k - 1] + B[k + 1]) + B[k - buf_stride] + + B[k - buf_stride - 1] + B[k - buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dgd[l] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + j = 0; + for (i = 1; i < height - 1; ++i) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int nb = 3; + const int32_t a = A[k] + 2 * (A[k - buf_stride] + A[k + buf_stride]) + + A[k + 1] + A[k - buf_stride + 1] + A[k + buf_stride + 1]; + const int32_t b = B[k] + 2 * (B[k - buf_stride] + B[k + buf_stride]) + + B[k + 1] + B[k - buf_stride + 1] + B[k + buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dgd[l] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + j = width - 1; + for (i = 1; i < height - 1; ++i) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int nb = 3; + const int32_t a = A[k] + 2 * (A[k - buf_stride] + A[k + buf_stride]) + + A[k - 1] + A[k - buf_stride - 1] + A[k + buf_stride - 1]; + const int32_t b = B[k] + 2 * (B[k - buf_stride] + B[k + buf_stride]) + + B[k - 1] + B[k - buf_stride - 1] + B[k + buf_stride - 1]; + const int32_t v = a * dgd[l] + b; + dgd[l] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + for (i = 1; i < height - 1; ++i) { + for (j = 1; j < width - 1; ++j) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int nb = 5; + const int32_t a = + (A[k] + A[k - 1] + A[k + 1] + A[k - buf_stride] + A[k + buf_stride]) * + 4 + + (A[k - 1 - buf_stride] + A[k - 1 + buf_stride] + + A[k + 1 - buf_stride] + A[k + 1 + buf_stride]) * + 3; + const int32_t b = + (B[k] + B[k - 1] + B[k + 1] + B[k - buf_stride] + B[k + buf_stride]) * + 4 + + (B[k - 1 - buf_stride] + B[k - 1 + buf_stride] + + B[k + 1 - buf_stride] + B[k + 1 + buf_stride]) * + 3; + const int32_t v = a * dgd[l] + b; + dgd[l] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + } +} + +void av1_selfguided_restoration_c(uint8_t *dgd, int width, int height, + int stride, int32_t *dst, int dst_stride, + int r, int eps, int32_t *tmpbuf) { + int i, j; + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + dst[i * dst_stride + j] = dgd[i * stride + j]; + } + } + av1_selfguided_restoration_internal(dst, width, height, dst_stride, 8, r, eps, + tmpbuf); +} + +void av1_highpass_filter_c(uint8_t *dgd, int width, int height, int stride, + int32_t *dst, int dst_stride, int corner, int edge) { + int i, j; + const int center = (1 << SGRPROJ_RST_BITS) - 4 * (corner + edge); + + i = 0; + j = 0; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k + 1] + dgd[k + stride] + dgd[k] * 2) + + corner * (dgd[k + stride + 1] + dgd[k + 1] + dgd[k + stride] + dgd[k]); + } + i = 0; + j = width - 1; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k - 1] + dgd[k + stride] + dgd[k] * 2) + + corner * (dgd[k + stride - 1] + dgd[k - 1] + dgd[k + stride] + dgd[k]); + } + i = height - 1; + j = 0; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k + 1] + dgd[k - stride] + dgd[k] * 2) + + corner * (dgd[k - stride + 1] + dgd[k + 1] + dgd[k - stride] + dgd[k]); + } + i = height - 1; + j = width - 1; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k - 1] + dgd[k - stride] + dgd[k] * 2) + + corner * (dgd[k - stride - 1] + dgd[k - 1] + dgd[k - stride] + dgd[k]); + } + i = 0; + for (j = 1; j < width - 1; ++j) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = center * dgd[k] + + edge * (dgd[k - 1] + dgd[k + stride] + dgd[k + 1] + dgd[k]) + + corner * (dgd[k + stride - 1] + dgd[k + stride + 1] + dgd[k - 1] + + dgd[k + 1]); + } + i = height - 1; + for (j = 1; j < width - 1; ++j) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = center * dgd[k] + + edge * (dgd[k - 1] + dgd[k - stride] + dgd[k + 1] + dgd[k]) + + corner * (dgd[k - stride - 1] + dgd[k - stride + 1] + dgd[k - 1] + + dgd[k + 1]); + } + j = 0; + for (i = 1; i < height - 1; ++i) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = center * dgd[k] + + edge * (dgd[k - stride] + dgd[k + 1] + dgd[k + stride] + dgd[k]) + + corner * (dgd[k + stride + 1] + dgd[k - stride + 1] + + dgd[k - stride] + dgd[k + stride]); + } + j = width - 1; + for (i = 1; i < height - 1; ++i) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = center * dgd[k] + + edge * (dgd[k - stride] + dgd[k - 1] + dgd[k + stride] + dgd[k]) + + corner * (dgd[k + stride - 1] + dgd[k - stride - 1] + + dgd[k - stride] + dgd[k + stride]); + } + for (i = 1; i < height - 1; ++i) { + for (j = 1; j < width - 1; ++j) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + + edge * (dgd[k - stride] + dgd[k - 1] + dgd[k + stride] + dgd[k + 1]) + + corner * (dgd[k + stride - 1] + dgd[k - stride - 1] + + dgd[k - stride + 1] + dgd[k + stride + 1]); + } + } +} + +void apply_selfguided_restoration_c(uint8_t *dat, int width, int height, + int stride, int eps, int *xqd, uint8_t *dst, + int dst_stride, int32_t *tmpbuf) { + int xq[2]; + int32_t *flt1 = tmpbuf; + int32_t *flt2 = flt1 + RESTORATION_TILEPELS_MAX; + int32_t *tmpbuf2 = flt2 + RESTORATION_TILEPELS_MAX; + int i, j; + assert(width * height <= RESTORATION_TILEPELS_MAX); +#if USE_HIGHPASS_IN_SGRPROJ + av1_highpass_filter_c(dat, width, height, stride, flt1, width, + sgr_params[eps].corner, sgr_params[eps].edge); +#else + av1_selfguided_restoration_c(dat, width, height, stride, flt1, width, + sgr_params[eps].r1, sgr_params[eps].e1, tmpbuf2); +#endif // USE_HIGHPASS_IN_SGRPROJ + av1_selfguided_restoration_c(dat, width, height, stride, flt2, width, + sgr_params[eps].r2, sgr_params[eps].e2, tmpbuf2); + decode_xq(xqd, xq); + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const int k = i * width + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int32_t u = ((int32_t)dat[l] << SGRPROJ_RST_BITS); + const int32_t f1 = (int32_t)flt1[k] - u; + const int32_t f2 = (int32_t)flt2[k] - u; + const int32_t v = xq[0] * f1 + xq[1] * f2 + (u << SGRPROJ_PRJ_BITS); + const int16_t w = + (int16_t)ROUND_POWER_OF_TWO(v, SGRPROJ_PRJ_BITS + SGRPROJ_RST_BITS); + dst[m] = clip_pixel(w); + } + } +} + +static void loop_sgrproj_filter_tile(uint8_t *data, int tile_idx, int width, + int height, int stride, + RestorationInternal *rst, uint8_t *dst, + int dst_stride) { + const int tile_width = rst->tile_width; + const int tile_height = rst->tile_height; + int h_start, h_end, v_start, v_end; + uint8_t *data_p, *dst_p; + + if (rst->rsi->restoration_type[tile_idx] == RESTORE_NONE) { + loop_copy_tile(data, tile_idx, 0, 0, width, height, stride, rst, dst, + dst_stride); + return; + } + av1_get_rest_tile_limits(tile_idx, 0, 0, rst->nhtiles, rst->nvtiles, + tile_width, tile_height, width, height, 0, 0, + &h_start, &h_end, &v_start, &v_end); + data_p = data + h_start + v_start * stride; + dst_p = dst + h_start + v_start * dst_stride; + apply_selfguided_restoration(data_p, h_end - h_start, v_end - v_start, stride, + rst->rsi->sgrproj_info[tile_idx].ep, + rst->rsi->sgrproj_info[tile_idx].xqd, dst_p, + dst_stride, rst->tmpbuf); +} + +static void loop_sgrproj_filter(uint8_t *data, int width, int height, + int stride, RestorationInternal *rst, + uint8_t *dst, int dst_stride) { + int tile_idx; + for (tile_idx = 0; tile_idx < rst->ntiles; ++tile_idx) { + loop_sgrproj_filter_tile(data, tile_idx, width, height, stride, rst, dst, + dst_stride); + } +} + +static void loop_switchable_filter(uint8_t *data, int width, int height, + int stride, RestorationInternal *rst, + uint8_t *dst, int dst_stride) { + int tile_idx; + extend_frame(data, width, height, stride); + for (tile_idx = 0; tile_idx < rst->ntiles; ++tile_idx) { + if (rst->rsi->restoration_type[tile_idx] == RESTORE_NONE) { + loop_copy_tile(data, tile_idx, 0, 0, width, height, stride, rst, dst, + dst_stride); + } else if (rst->rsi->restoration_type[tile_idx] == RESTORE_WIENER) { + loop_wiener_filter_tile(data, tile_idx, width, height, stride, rst, dst, + dst_stride); + } else if (rst->rsi->restoration_type[tile_idx] == RESTORE_SGRPROJ) { + loop_sgrproj_filter_tile(data, tile_idx, width, height, stride, rst, dst, + dst_stride); + } + } +} + +#if CONFIG_HIGHBITDEPTH +void extend_frame_highbd(uint16_t *data, int width, int height, int stride) { + uint16_t *data_p; + int i, j; + for (i = 0; i < height; ++i) { + data_p = data + i * stride; + for (j = -WIENER_HALFWIN; j < 0; ++j) data_p[j] = data_p[0]; + for (j = width; j < width + WIENER_HALFWIN; ++j) + data_p[j] = data_p[width - 1]; + } + data_p = data - WIENER_HALFWIN; + for (i = -WIENER_HALFWIN; i < 0; ++i) { + memcpy(data_p + i * stride, data_p, + (width + 2 * WIENER_HALFWIN) * sizeof(uint16_t)); + } + for (i = height; i < height + WIENER_HALFWIN; ++i) { + memcpy(data_p + i * stride, data_p + (height - 1) * stride, + (width + 2 * WIENER_HALFWIN) * sizeof(uint16_t)); + } +} + +static void loop_copy_tile_highbd(uint16_t *data, int tile_idx, int subtile_idx, + int subtile_bits, int width, int height, + int stride, RestorationInternal *rst, + uint16_t *dst, int dst_stride) { + const int tile_width = rst->tile_width; + const int tile_height = rst->tile_height; + int i; + int h_start, h_end, v_start, v_end; + av1_get_rest_tile_limits(tile_idx, subtile_idx, subtile_bits, rst->nhtiles, + rst->nvtiles, tile_width, tile_height, width, height, + 0, 0, &h_start, &h_end, &v_start, &v_end); + for (i = v_start; i < v_end; ++i) + memcpy(dst + i * dst_stride + h_start, data + i * stride + h_start, + (h_end - h_start) * sizeof(*dst)); +} + +static void loop_wiener_filter_tile_highbd(uint16_t *data, int tile_idx, + int width, int height, int stride, + RestorationInternal *rst, + int bit_depth, uint16_t *dst, + int dst_stride) { + const int tile_width = rst->tile_width; + const int tile_height = rst->tile_height; + int h_start, h_end, v_start, v_end; + int i, j; + + if (rst->rsi->restoration_type[tile_idx] == RESTORE_NONE) { + loop_copy_tile_highbd(data, tile_idx, 0, 0, width, height, stride, rst, dst, + dst_stride); + return; + } + av1_get_rest_tile_limits(tile_idx, 0, 0, rst->nhtiles, rst->nvtiles, + tile_width, tile_height, width, height, 0, 0, + &h_start, &h_end, &v_start, &v_end); + // Convolve the whole tile (done in blocks here to match the requirements + // of the vectorized convolve functions, but the result is equivalent) + for (i = v_start; i < v_end; i += MAX_SB_SIZE) + for (j = h_start; j < h_end; j += MAX_SB_SIZE) { + int w = AOMMIN(MAX_SB_SIZE, (h_end - j + 15) & ~15); + int h = AOMMIN(MAX_SB_SIZE, (v_end - i + 15) & ~15); + const uint16_t *data_p = data + i * stride + j; + uint16_t *dst_p = dst + i * dst_stride + j; + aom_highbd_convolve8_add_src( + CONVERT_TO_BYTEPTR(data_p), stride, CONVERT_TO_BYTEPTR(dst_p), + dst_stride, rst->rsi->wiener_info[tile_idx].hfilter, 16, + rst->rsi->wiener_info[tile_idx].vfilter, 16, w, h, bit_depth); + } +} + +static void loop_wiener_filter_highbd(uint8_t *data8, int width, int height, + int stride, RestorationInternal *rst, + int bit_depth, uint8_t *dst8, + int dst_stride) { + uint16_t *data = CONVERT_TO_SHORTPTR(data8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + int tile_idx; + extend_frame_highbd(data, width, height, stride); + for (tile_idx = 0; tile_idx < rst->ntiles; ++tile_idx) { + loop_wiener_filter_tile_highbd(data, tile_idx, width, height, stride, rst, + bit_depth, dst, dst_stride); + } +} + +void av1_selfguided_restoration_highbd_c(uint16_t *dgd, int width, int height, + int stride, int32_t *dst, + int dst_stride, int bit_depth, int r, + int eps, int32_t *tmpbuf) { + int i, j; + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + dst[i * dst_stride + j] = dgd[i * stride + j]; + } + } + av1_selfguided_restoration_internal(dst, width, height, dst_stride, bit_depth, + r, eps, tmpbuf); +} + +void av1_highpass_filter_highbd_c(uint16_t *dgd, int width, int height, + int stride, int32_t *dst, int dst_stride, + int corner, int edge) { + int i, j; + const int center = (1 << SGRPROJ_RST_BITS) - 4 * (corner + edge); + + i = 0; + j = 0; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k + 1] + dgd[k + stride] + dgd[k] * 2) + + corner * (dgd[k + stride + 1] + dgd[k + 1] + dgd[k + stride] + dgd[k]); + } + i = 0; + j = width - 1; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k - 1] + dgd[k + stride] + dgd[k] * 2) + + corner * (dgd[k + stride - 1] + dgd[k - 1] + dgd[k + stride] + dgd[k]); + } + i = height - 1; + j = 0; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k + 1] + dgd[k - stride] + dgd[k] * 2) + + corner * (dgd[k - stride + 1] + dgd[k + 1] + dgd[k - stride] + dgd[k]); + } + i = height - 1; + j = width - 1; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k - 1] + dgd[k - stride] + dgd[k] * 2) + + corner * (dgd[k - stride - 1] + dgd[k - 1] + dgd[k - stride] + dgd[k]); + } + i = 0; + for (j = 1; j < width - 1; ++j) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = center * dgd[k] + + edge * (dgd[k - 1] + dgd[k + stride] + dgd[k + 1] + dgd[k]) + + corner * (dgd[k + stride - 1] + dgd[k + stride + 1] + dgd[k - 1] + + dgd[k + 1]); + } + i = height - 1; + for (j = 1; j < width - 1; ++j) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = center * dgd[k] + + edge * (dgd[k - 1] + dgd[k - stride] + dgd[k + 1] + dgd[k]) + + corner * (dgd[k - stride - 1] + dgd[k - stride + 1] + dgd[k - 1] + + dgd[k + 1]); + } + j = 0; + for (i = 1; i < height - 1; ++i) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = center * dgd[k] + + edge * (dgd[k - stride] + dgd[k + 1] + dgd[k + stride] + dgd[k]) + + corner * (dgd[k + stride + 1] + dgd[k - stride + 1] + + dgd[k - stride] + dgd[k + stride]); + } + j = width - 1; + for (i = 1; i < height - 1; ++i) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = center * dgd[k] + + edge * (dgd[k - stride] + dgd[k - 1] + dgd[k + stride] + dgd[k]) + + corner * (dgd[k + stride - 1] + dgd[k - stride - 1] + + dgd[k - stride] + dgd[k + stride]); + } + for (i = 1; i < height - 1; ++i) { + for (j = 1; j < width - 1; ++j) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + + edge * (dgd[k - stride] + dgd[k - 1] + dgd[k + stride] + dgd[k + 1]) + + corner * (dgd[k + stride - 1] + dgd[k - stride - 1] + + dgd[k - stride + 1] + dgd[k + stride + 1]); + } + } +} + +void apply_selfguided_restoration_highbd_c(uint16_t *dat, int width, int height, + int stride, int bit_depth, int eps, + int *xqd, uint16_t *dst, + int dst_stride, int32_t *tmpbuf) { + int xq[2]; + int32_t *flt1 = tmpbuf; + int32_t *flt2 = flt1 + RESTORATION_TILEPELS_MAX; + int32_t *tmpbuf2 = flt2 + RESTORATION_TILEPELS_MAX; + int i, j; + assert(width * height <= RESTORATION_TILEPELS_MAX); +#if USE_HIGHPASS_IN_SGRPROJ + av1_highpass_filter_highbd_c(dat, width, height, stride, flt1, width, + sgr_params[eps].corner, sgr_params[eps].edge); +#else + av1_selfguided_restoration_highbd_c(dat, width, height, stride, flt1, width, + bit_depth, sgr_params[eps].r1, + sgr_params[eps].e1, tmpbuf2); +#endif // USE_HIGHPASS_IN_SGRPROJ + av1_selfguided_restoration_highbd_c(dat, width, height, stride, flt2, width, + bit_depth, sgr_params[eps].r2, + sgr_params[eps].e2, tmpbuf2); + decode_xq(xqd, xq); + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const int k = i * width + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int32_t u = ((int32_t)dat[l] << SGRPROJ_RST_BITS); + const int32_t f1 = (int32_t)flt1[k] - u; + const int32_t f2 = (int32_t)flt2[k] - u; + const int32_t v = xq[0] * f1 + xq[1] * f2 + (u << SGRPROJ_PRJ_BITS); + const int16_t w = + (int16_t)ROUND_POWER_OF_TWO(v, SGRPROJ_PRJ_BITS + SGRPROJ_RST_BITS); + dst[m] = (uint16_t)clip_pixel_highbd(w, bit_depth); + } + } +} + +static void loop_sgrproj_filter_tile_highbd(uint16_t *data, int tile_idx, + int width, int height, int stride, + RestorationInternal *rst, + int bit_depth, uint16_t *dst, + int dst_stride) { + const int tile_width = rst->tile_width; + const int tile_height = rst->tile_height; + int h_start, h_end, v_start, v_end; + uint16_t *data_p, *dst_p; + + if (rst->rsi->restoration_type[tile_idx] == RESTORE_NONE) { + loop_copy_tile_highbd(data, tile_idx, 0, 0, width, height, stride, rst, dst, + dst_stride); + return; + } + av1_get_rest_tile_limits(tile_idx, 0, 0, rst->nhtiles, rst->nvtiles, + tile_width, tile_height, width, height, 0, 0, + &h_start, &h_end, &v_start, &v_end); + data_p = data + h_start + v_start * stride; + dst_p = dst + h_start + v_start * dst_stride; + apply_selfguided_restoration_highbd( + data_p, h_end - h_start, v_end - v_start, stride, bit_depth, + rst->rsi->sgrproj_info[tile_idx].ep, rst->rsi->sgrproj_info[tile_idx].xqd, + dst_p, dst_stride, rst->tmpbuf); +} + +static void loop_sgrproj_filter_highbd(uint8_t *data8, int width, int height, + int stride, RestorationInternal *rst, + int bit_depth, uint8_t *dst8, + int dst_stride) { + int tile_idx; + uint16_t *data = CONVERT_TO_SHORTPTR(data8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + for (tile_idx = 0; tile_idx < rst->ntiles; ++tile_idx) { + loop_sgrproj_filter_tile_highbd(data, tile_idx, width, height, stride, rst, + bit_depth, dst, dst_stride); + } +} + +static void loop_switchable_filter_highbd(uint8_t *data8, int width, int height, + int stride, RestorationInternal *rst, + int bit_depth, uint8_t *dst8, + int dst_stride) { + uint16_t *data = CONVERT_TO_SHORTPTR(data8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + int tile_idx; + extend_frame_highbd(data, width, height, stride); + for (tile_idx = 0; tile_idx < rst->ntiles; ++tile_idx) { + if (rst->rsi->restoration_type[tile_idx] == RESTORE_NONE) { + loop_copy_tile_highbd(data, tile_idx, 0, 0, width, height, stride, rst, + dst, dst_stride); + } else if (rst->rsi->restoration_type[tile_idx] == RESTORE_WIENER) { + loop_wiener_filter_tile_highbd(data, tile_idx, width, height, stride, rst, + bit_depth, dst, dst_stride); + } else if (rst->rsi->restoration_type[tile_idx] == RESTORE_SGRPROJ) { + loop_sgrproj_filter_tile_highbd(data, tile_idx, width, height, stride, + rst, bit_depth, dst, dst_stride); + } + } +} +#endif // CONFIG_HIGHBITDEPTH + +static void loop_restoration_rows(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, + int start_mi_row, int end_mi_row, + int components_pattern, RestorationInfo *rsi, + YV12_BUFFER_CONFIG *dst) { + const int ywidth = frame->y_crop_width; + const int ystride = frame->y_stride; + const int uvwidth = frame->uv_crop_width; + const int uvstride = frame->uv_stride; + const int ystart = start_mi_row << MI_SIZE_LOG2; + const int uvstart = ystart >> cm->subsampling_y; + int yend = end_mi_row << MI_SIZE_LOG2; + int uvend = yend >> cm->subsampling_y; + restore_func_type restore_funcs[RESTORE_TYPES] = { + NULL, loop_wiener_filter, loop_sgrproj_filter, loop_switchable_filter + }; +#if CONFIG_HIGHBITDEPTH + restore_func_highbd_type restore_funcs_highbd[RESTORE_TYPES] = { + NULL, loop_wiener_filter_highbd, loop_sgrproj_filter_highbd, + loop_switchable_filter_highbd + }; +#endif // CONFIG_HIGHBITDEPTH + restore_func_type restore_func; +#if CONFIG_HIGHBITDEPTH + restore_func_highbd_type restore_func_highbd; +#endif // CONFIG_HIGHBITDEPTH + YV12_BUFFER_CONFIG dst_; + + yend = AOMMIN(yend, cm->height); + uvend = AOMMIN(uvend, cm->subsampling_y ? (cm->height + 1) >> 1 : cm->height); + + if (components_pattern == (1 << AOM_PLANE_Y)) { + // Only y + if (rsi[0].frame_restoration_type == RESTORE_NONE) { + if (dst) aom_yv12_copy_y(frame, dst); + return; + } + } else if (components_pattern == (1 << AOM_PLANE_U)) { + // Only U + if (rsi[1].frame_restoration_type == RESTORE_NONE) { + if (dst) aom_yv12_copy_u(frame, dst); + return; + } + } else if (components_pattern == (1 << AOM_PLANE_V)) { + // Only V + if (rsi[2].frame_restoration_type == RESTORE_NONE) { + if (dst) aom_yv12_copy_v(frame, dst); + return; + } + } else if (components_pattern == + ((1 << AOM_PLANE_Y) | (1 << AOM_PLANE_U) | (1 << AOM_PLANE_V))) { + // All components + if (rsi[0].frame_restoration_type == RESTORE_NONE && + rsi[1].frame_restoration_type == RESTORE_NONE && + rsi[2].frame_restoration_type == RESTORE_NONE) { + if (dst) aom_yv12_copy_frame(frame, dst); + return; + } + } + + if (!dst) { + dst = &dst_; + memset(dst, 0, sizeof(YV12_BUFFER_CONFIG)); + if (aom_realloc_frame_buffer( + dst, cm->width, cm->height, cm->subsampling_x, cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + AOM_BORDER_IN_PIXELS, cm->byte_alignment, NULL, NULL, NULL) < 0) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate restoration dst buffer"); + } + + if ((components_pattern >> AOM_PLANE_Y) & 1) { + if (rsi[0].frame_restoration_type != RESTORE_NONE) { + cm->rst_internal.ntiles = av1_get_rest_ntiles( + cm->width, cm->height, cm->rst_info[AOM_PLANE_Y].restoration_tilesize, + &cm->rst_internal.tile_width, &cm->rst_internal.tile_height, + &cm->rst_internal.nhtiles, &cm->rst_internal.nvtiles); + cm->rst_internal.rsi = &rsi[0]; + restore_func = + restore_funcs[cm->rst_internal.rsi->frame_restoration_type]; +#if CONFIG_HIGHBITDEPTH + restore_func_highbd = + restore_funcs_highbd[cm->rst_internal.rsi->frame_restoration_type]; + if (cm->use_highbitdepth) + restore_func_highbd( + frame->y_buffer + ystart * ystride, ywidth, yend - ystart, ystride, + &cm->rst_internal, cm->bit_depth, + dst->y_buffer + ystart * dst->y_stride, dst->y_stride); + else +#endif // CONFIG_HIGHBITDEPTH + restore_func(frame->y_buffer + ystart * ystride, ywidth, yend - ystart, + ystride, &cm->rst_internal, + dst->y_buffer + ystart * dst->y_stride, dst->y_stride); + } else { + aom_yv12_copy_y(frame, dst); + } + } + + if ((components_pattern >> AOM_PLANE_U) & 1) { + if (rsi[AOM_PLANE_U].frame_restoration_type != RESTORE_NONE) { + cm->rst_internal.ntiles = av1_get_rest_ntiles( + ROUND_POWER_OF_TWO(cm->width, cm->subsampling_x), + ROUND_POWER_OF_TWO(cm->height, cm->subsampling_y), + cm->rst_info[AOM_PLANE_U].restoration_tilesize, + &cm->rst_internal.tile_width, &cm->rst_internal.tile_height, + &cm->rst_internal.nhtiles, &cm->rst_internal.nvtiles); + cm->rst_internal.rsi = &rsi[AOM_PLANE_U]; + restore_func = + restore_funcs[cm->rst_internal.rsi->frame_restoration_type]; +#if CONFIG_HIGHBITDEPTH + restore_func_highbd = + restore_funcs_highbd[cm->rst_internal.rsi->frame_restoration_type]; + if (cm->use_highbitdepth) + restore_func_highbd( + frame->u_buffer + uvstart * uvstride, uvwidth, uvend - uvstart, + uvstride, &cm->rst_internal, cm->bit_depth, + dst->u_buffer + uvstart * dst->uv_stride, dst->uv_stride); + else +#endif // CONFIG_HIGHBITDEPTH + restore_func(frame->u_buffer + uvstart * uvstride, uvwidth, + uvend - uvstart, uvstride, &cm->rst_internal, + dst->u_buffer + uvstart * dst->uv_stride, dst->uv_stride); + } else { + aom_yv12_copy_u(frame, dst); + } + } + + if ((components_pattern >> AOM_PLANE_V) & 1) { + if (rsi[AOM_PLANE_V].frame_restoration_type != RESTORE_NONE) { + cm->rst_internal.ntiles = av1_get_rest_ntiles( + ROUND_POWER_OF_TWO(cm->width, cm->subsampling_x), + ROUND_POWER_OF_TWO(cm->height, cm->subsampling_y), + cm->rst_info[AOM_PLANE_V].restoration_tilesize, + &cm->rst_internal.tile_width, &cm->rst_internal.tile_height, + &cm->rst_internal.nhtiles, &cm->rst_internal.nvtiles); + cm->rst_internal.rsi = &rsi[AOM_PLANE_V]; + restore_func = + restore_funcs[cm->rst_internal.rsi->frame_restoration_type]; +#if CONFIG_HIGHBITDEPTH + restore_func_highbd = + restore_funcs_highbd[cm->rst_internal.rsi->frame_restoration_type]; + if (cm->use_highbitdepth) + restore_func_highbd( + frame->v_buffer + uvstart * uvstride, uvwidth, uvend - uvstart, + uvstride, &cm->rst_internal, cm->bit_depth, + dst->v_buffer + uvstart * dst->uv_stride, dst->uv_stride); + else +#endif // CONFIG_HIGHBITDEPTH + restore_func(frame->v_buffer + uvstart * uvstride, uvwidth, + uvend - uvstart, uvstride, &cm->rst_internal, + dst->v_buffer + uvstart * dst->uv_stride, dst->uv_stride); + } else { + aom_yv12_copy_v(frame, dst); + } + } + + if (dst == &dst_) { + if ((components_pattern >> AOM_PLANE_Y) & 1) aom_yv12_copy_y(dst, frame); + if ((components_pattern >> AOM_PLANE_U) & 1) aom_yv12_copy_u(dst, frame); + if ((components_pattern >> AOM_PLANE_V) & 1) aom_yv12_copy_v(dst, frame); + aom_free_frame_buffer(dst); + } +} + +void av1_loop_restoration_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, + RestorationInfo *rsi, int components_pattern, + int partial_frame, YV12_BUFFER_CONFIG *dst) { + int start_mi_row, end_mi_row, mi_rows_to_filter; + start_mi_row = 0; + mi_rows_to_filter = cm->mi_rows; + if (partial_frame && cm->mi_rows > 8) { + start_mi_row = cm->mi_rows >> 1; + start_mi_row &= 0xfffffff8; + mi_rows_to_filter = AOMMAX(cm->mi_rows / 8, 8); + } + end_mi_row = start_mi_row + mi_rows_to_filter; + loop_restoration_init(&cm->rst_internal, cm->frame_type == KEY_FRAME); + loop_restoration_rows(frame, cm, start_mi_row, end_mi_row, components_pattern, + rsi, dst); +} diff --git a/third_party/aom/av1/common/restoration.h b/third_party/aom/av1/common/restoration.h new file mode 100644 index 0000000000..866f78b792 --- /dev/null +++ b/third_party/aom/av1/common/restoration.h @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_RESTORATION_H_ +#define AV1_COMMON_RESTORATION_H_ + +#include "aom_ports/mem.h" +#include "./aom_config.h" + +#include "av1/common/blockd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define CLIP(x, lo, hi) ((x) < (lo) ? (lo) : (x) > (hi) ? (hi) : (x)) +#define RINT(x) ((x) < 0 ? (int)((x)-0.5) : (int)((x) + 0.5)) + +#define RESTORATION_TILESIZE_MAX 256 +#define RESTORATION_TILEPELS_MAX \ + (RESTORATION_TILESIZE_MAX * RESTORATION_TILESIZE_MAX * 9 / 4) + +// 4 32-bit buffers needed for the filter: +// 2 for the restored versions of the frame and +// 2 for each restoration operation +#define SGRPROJ_OUTBUF_SIZE \ + ((RESTORATION_TILESIZE_MAX * 3 / 2) * (RESTORATION_TILESIZE_MAX * 3 / 2 + 16)) +#define SGRPROJ_TMPBUF_SIZE \ + (RESTORATION_TILEPELS_MAX * 2 * sizeof(int32_t) + \ + SGRPROJ_OUTBUF_SIZE * 2 * sizeof(int32_t)) +#define SGRPROJ_EXTBUF_SIZE (0) +#define SGRPROJ_PARAMS_BITS 4 +#define SGRPROJ_PARAMS (1 << SGRPROJ_PARAMS_BITS) +#define USE_HIGHPASS_IN_SGRPROJ 0 + +// Precision bits for projection +#define SGRPROJ_PRJ_BITS 7 +// Restoration precision bits generated higher than source before projection +#define SGRPROJ_RST_BITS 4 +// Internal precision bits for core selfguided_restoration +#define SGRPROJ_SGR_BITS 8 +#define SGRPROJ_SGR (1 << SGRPROJ_SGR_BITS) + +#if USE_HIGHPASS_IN_SGRPROJ +#define SGRPROJ_PRJ_MIN0 (-(1 << SGRPROJ_PRJ_BITS) / 8) +#define SGRPROJ_PRJ_MAX0 (SGRPROJ_PRJ_MIN0 + (1 << SGRPROJ_PRJ_BITS) - 1) +#define SGRPROJ_PRJ_MIN1 (-(1 << SGRPROJ_PRJ_BITS) / 2) +#define SGRPROJ_PRJ_MAX1 (SGRPROJ_PRJ_MIN1 + (1 << SGRPROJ_PRJ_BITS) - 1) +#else +#define SGRPROJ_PRJ_MIN0 (-(1 << SGRPROJ_PRJ_BITS) * 3 / 4) +#define SGRPROJ_PRJ_MAX0 (SGRPROJ_PRJ_MIN0 + (1 << SGRPROJ_PRJ_BITS) - 1) +#define SGRPROJ_PRJ_MIN1 (-(1 << SGRPROJ_PRJ_BITS) / 4) +#define SGRPROJ_PRJ_MAX1 (SGRPROJ_PRJ_MIN1 + (1 << SGRPROJ_PRJ_BITS) - 1) +#endif // USE_HIGHPASS_IN_SGRPROJ + +#define SGRPROJ_PRJ_SUBEXP_K 4 + +#define SGRPROJ_BITS (SGRPROJ_PRJ_BITS * 2 + SGRPROJ_PARAMS_BITS) + +#define MAX_RADIUS 3 // Only 1, 2, 3 allowed +#define MAX_EPS 80 // Max value of eps +#define MAX_NELEM ((2 * MAX_RADIUS + 1) * (2 * MAX_RADIUS + 1)) +#define SGRPROJ_MTABLE_BITS 20 +#define SGRPROJ_RECIP_BITS 12 + +#define WIENER_HALFWIN 3 +#define WIENER_HALFWIN1 (WIENER_HALFWIN + 1) +#define WIENER_WIN (2 * WIENER_HALFWIN + 1) +#define WIENER_WIN2 ((WIENER_WIN) * (WIENER_WIN)) +#define WIENER_TMPBUF_SIZE (0) +#define WIENER_EXTBUF_SIZE (0) + +#define WIENER_FILT_PREC_BITS 7 +#define WIENER_FILT_STEP (1 << WIENER_FILT_PREC_BITS) + +// Central values for the taps +#define WIENER_FILT_TAP0_MIDV (3) +#define WIENER_FILT_TAP1_MIDV (-7) +#define WIENER_FILT_TAP2_MIDV (15) + +#define WIENER_FILT_TAP0_BITS 4 +#define WIENER_FILT_TAP1_BITS 5 +#define WIENER_FILT_TAP2_BITS 6 + +#define WIENER_FILT_BITS \ + ((WIENER_FILT_TAP0_BITS + WIENER_FILT_TAP1_BITS + WIENER_FILT_TAP2_BITS) * 2) + +#define WIENER_FILT_TAP0_MINV \ + (WIENER_FILT_TAP0_MIDV - (1 << WIENER_FILT_TAP0_BITS) / 2) +#define WIENER_FILT_TAP1_MINV \ + (WIENER_FILT_TAP1_MIDV - (1 << WIENER_FILT_TAP1_BITS) / 2) +#define WIENER_FILT_TAP2_MINV \ + (WIENER_FILT_TAP2_MIDV - (1 << WIENER_FILT_TAP2_BITS) / 2) + +#define WIENER_FILT_TAP0_MAXV \ + (WIENER_FILT_TAP0_MIDV - 1 + (1 << WIENER_FILT_TAP0_BITS) / 2) +#define WIENER_FILT_TAP1_MAXV \ + (WIENER_FILT_TAP1_MIDV - 1 + (1 << WIENER_FILT_TAP1_BITS) / 2) +#define WIENER_FILT_TAP2_MAXV \ + (WIENER_FILT_TAP2_MIDV - 1 + (1 << WIENER_FILT_TAP2_BITS) / 2) + +#define WIENER_FILT_TAP0_SUBEXP_K 1 +#define WIENER_FILT_TAP1_SUBEXP_K 2 +#define WIENER_FILT_TAP2_SUBEXP_K 3 + +// Max of SGRPROJ_TMPBUF_SIZE, DOMAINTXFMRF_TMPBUF_SIZE, WIENER_TMPBUF_SIZE +#define RESTORATION_TMPBUF_SIZE (SGRPROJ_TMPBUF_SIZE) + +// Max of SGRPROJ_EXTBUF_SIZE, WIENER_EXTBUF_SIZE +#define RESTORATION_EXTBUF_SIZE (WIENER_EXTBUF_SIZE) + +// Check the assumptions of the existing code +#if SUBPEL_TAPS != WIENER_WIN + 1 +#error "Wiener filter currently only works if SUBPEL_TAPS == WIENER_WIN + 1" +#endif +#if WIENER_FILT_PREC_BITS != 7 +#error "Wiener filter currently only works if WIENER_FILT_PREC_BITS == 7" +#endif +typedef struct { + DECLARE_ALIGNED(16, InterpKernel, vfilter); + DECLARE_ALIGNED(16, InterpKernel, hfilter); +} WienerInfo; + +typedef struct { +#if USE_HIGHPASS_IN_SGRPROJ + int corner; + int edge; +#else + int r1; + int e1; +#endif // USE_HIGHPASS_IN_SGRPROJ + int r2; + int e2; +} sgr_params_type; + +typedef struct { + int ep; + int xqd[2]; +} SgrprojInfo; + +typedef struct { + int restoration_tilesize; + RestorationType frame_restoration_type; + RestorationType *restoration_type; + // Wiener filter + WienerInfo *wiener_info; + // Selfguided proj filter + SgrprojInfo *sgrproj_info; +} RestorationInfo; + +typedef struct { + RestorationInfo *rsi; + int keyframe; + int ntiles; + int tile_width, tile_height; + int nhtiles, nvtiles; + int32_t *tmpbuf; +} RestorationInternal; + +static INLINE void set_default_sgrproj(SgrprojInfo *sgrproj_info) { + sgrproj_info->xqd[0] = (SGRPROJ_PRJ_MIN0 + SGRPROJ_PRJ_MAX0) / 2; + sgrproj_info->xqd[1] = (SGRPROJ_PRJ_MIN1 + SGRPROJ_PRJ_MAX1) / 2; +} + +static INLINE void set_default_wiener(WienerInfo *wiener_info) { + wiener_info->vfilter[0] = wiener_info->hfilter[0] = WIENER_FILT_TAP0_MIDV; + wiener_info->vfilter[1] = wiener_info->hfilter[1] = WIENER_FILT_TAP1_MIDV; + wiener_info->vfilter[2] = wiener_info->hfilter[2] = WIENER_FILT_TAP2_MIDV; + wiener_info->vfilter[WIENER_HALFWIN] = wiener_info->hfilter[WIENER_HALFWIN] = + -2 * + (WIENER_FILT_TAP2_MIDV + WIENER_FILT_TAP1_MIDV + WIENER_FILT_TAP0_MIDV); + wiener_info->vfilter[4] = wiener_info->hfilter[4] = WIENER_FILT_TAP2_MIDV; + wiener_info->vfilter[5] = wiener_info->hfilter[5] = WIENER_FILT_TAP1_MIDV; + wiener_info->vfilter[6] = wiener_info->hfilter[6] = WIENER_FILT_TAP0_MIDV; +} + +static INLINE int av1_get_rest_ntiles(int width, int height, int tilesize, + int *tile_width, int *tile_height, + int *nhtiles, int *nvtiles) { + int nhtiles_, nvtiles_; + int tile_width_, tile_height_; + tile_width_ = (tilesize < 0) ? width : AOMMIN(tilesize, width); + tile_height_ = (tilesize < 0) ? height : AOMMIN(tilesize, height); + nhtiles_ = (width + (tile_width_ >> 1)) / tile_width_; + nvtiles_ = (height + (tile_height_ >> 1)) / tile_height_; + if (tile_width) *tile_width = tile_width_; + if (tile_height) *tile_height = tile_height_; + if (nhtiles) *nhtiles = nhtiles_; + if (nvtiles) *nvtiles = nvtiles_; + return (nhtiles_ * nvtiles_); +} + +static INLINE void av1_get_rest_tile_limits( + int tile_idx, int subtile_idx, int subtile_bits, int nhtiles, int nvtiles, + int tile_width, int tile_height, int im_width, int im_height, int clamp_h, + int clamp_v, int *h_start, int *h_end, int *v_start, int *v_end) { + const int htile_idx = tile_idx % nhtiles; + const int vtile_idx = tile_idx / nhtiles; + *h_start = htile_idx * tile_width; + *v_start = vtile_idx * tile_height; + *h_end = (htile_idx < nhtiles - 1) ? *h_start + tile_width : im_width; + *v_end = (vtile_idx < nvtiles - 1) ? *v_start + tile_height : im_height; + if (subtile_bits) { + const int num_subtiles_1d = (1 << subtile_bits); + const int subtile_width = (*h_end - *h_start) >> subtile_bits; + const int subtile_height = (*v_end - *v_start) >> subtile_bits; + const int subtile_idx_h = subtile_idx & (num_subtiles_1d - 1); + const int subtile_idx_v = subtile_idx >> subtile_bits; + *h_start += subtile_idx_h * subtile_width; + *v_start += subtile_idx_v * subtile_height; + *h_end = subtile_idx_h == num_subtiles_1d - 1 ? *h_end + : *h_start + subtile_width; + *v_end = subtile_idx_v == num_subtiles_1d - 1 ? *v_end + : *v_start + subtile_height; + } + if (clamp_h) { + *h_start = AOMMAX(*h_start, clamp_h); + *h_end = AOMMIN(*h_end, im_width - clamp_h); + } + if (clamp_v) { + *v_start = AOMMAX(*v_start, clamp_v); + *v_end = AOMMIN(*v_end, im_height - clamp_v); + } +} + +extern const sgr_params_type sgr_params[SGRPROJ_PARAMS]; +extern int sgrproj_mtable[MAX_EPS][MAX_NELEM]; +extern const int32_t x_by_xplus1[256]; +extern const int32_t one_by_x[MAX_NELEM]; + +int av1_alloc_restoration_struct(struct AV1Common *cm, + RestorationInfo *rst_info, int width, + int height); +void av1_free_restoration_struct(RestorationInfo *rst_info); + +void extend_frame(uint8_t *data, int width, int height, int stride); +#if CONFIG_HIGHBITDEPTH +void extend_frame_highbd(uint16_t *data, int width, int height, int stride); +#endif // CONFIG_HIGHBITDEPTH +void decode_xq(int *xqd, int *xq); +void av1_loop_restoration_frame(YV12_BUFFER_CONFIG *frame, struct AV1Common *cm, + RestorationInfo *rsi, int components_pattern, + int partial_frame, YV12_BUFFER_CONFIG *dst); +void av1_loop_restoration_precal(); +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_RESTORATION_H_ diff --git a/third_party/aom/av1/common/scale.c b/third_party/aom/av1/common/scale.c new file mode 100644 index 0000000000..76beaa2bdb --- /dev/null +++ b/third_party/aom/av1/common/scale.c @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_dsp_rtcd.h" +#include "av1/common/filter.h" +#include "av1/common/scale.h" +#include "aom_dsp/aom_filter.h" + +static INLINE int scaled_x(int val, const struct scale_factors *sf) { + return (int)((int64_t)val * sf->x_scale_fp >> REF_SCALE_SHIFT); +} + +static INLINE int scaled_y(int val, const struct scale_factors *sf) { + return (int)((int64_t)val * sf->y_scale_fp >> REF_SCALE_SHIFT); +} + +static int unscaled_value(int val, const struct scale_factors *sf) { + (void)sf; + return val; +} + +static int get_fixed_point_scale_factor(int other_size, int this_size) { + // Calculate scaling factor once for each reference frame + // and use fixed point scaling factors in decoding and encoding routines. + // Hardware implementations can calculate scale factor in device driver + // and use multiplication and shifting on hardware instead of division. + return (other_size << REF_SCALE_SHIFT) / this_size; +} + +MV32 av1_scale_mv(const MV *mv, int x, int y, const struct scale_factors *sf) { + const int x_off_q4 = scaled_x(x << SUBPEL_BITS, sf) & SUBPEL_MASK; + const int y_off_q4 = scaled_y(y << SUBPEL_BITS, sf) & SUBPEL_MASK; + const MV32 res = { scaled_y(mv->row, sf) + y_off_q4, + scaled_x(mv->col, sf) + x_off_q4 }; + return res; +} + +#if CONFIG_HIGHBITDEPTH +void av1_setup_scale_factors_for_frame(struct scale_factors *sf, int other_w, + int other_h, int this_w, int this_h, + int use_highbd) { +#else +void av1_setup_scale_factors_for_frame(struct scale_factors *sf, int other_w, + int other_h, int this_w, int this_h) { +#endif + if (!valid_ref_frame_size(other_w, other_h, this_w, this_h)) { + sf->x_scale_fp = REF_INVALID_SCALE; + sf->y_scale_fp = REF_INVALID_SCALE; + return; + } + + sf->x_scale_fp = get_fixed_point_scale_factor(other_w, this_w); + sf->y_scale_fp = get_fixed_point_scale_factor(other_h, this_h); + sf->x_step_q4 = scaled_x(16, sf); + sf->y_step_q4 = scaled_y(16, sf); + + if (av1_is_scaled(sf)) { + sf->scale_value_x = scaled_x; + sf->scale_value_y = scaled_y; + } else { + sf->scale_value_x = unscaled_value; + sf->scale_value_y = unscaled_value; + } + + // TODO(agrange): Investigate the best choice of functions to use here + // for EIGHTTAP_SMOOTH. Since it is not interpolating, need to choose what + // to do at full-pel offsets. The current selection, where the filter is + // applied in one direction only, and not at all for 0,0, seems to give the + // best quality, but it may be worth trying an additional mode that does + // do the filtering on full-pel. + if (sf->x_step_q4 == 16) { + if (sf->y_step_q4 == 16) { + // No scaling in either direction. + sf->predict[0][0][0] = aom_convolve_copy; + sf->predict[0][0][1] = aom_convolve_avg; + sf->predict[0][1][0] = aom_convolve8_vert; + sf->predict[0][1][1] = aom_convolve8_avg_vert; + sf->predict[1][0][0] = aom_convolve8_horiz; + sf->predict[1][0][1] = aom_convolve8_avg_horiz; + } else { + // No scaling in x direction. Must always scale in the y direction. + sf->predict[0][0][0] = aom_convolve8_vert; + sf->predict[0][0][1] = aom_convolve8_avg_vert; + sf->predict[0][1][0] = aom_convolve8_vert; + sf->predict[0][1][1] = aom_convolve8_avg_vert; + sf->predict[1][0][0] = aom_convolve8; + sf->predict[1][0][1] = aom_convolve8_avg; + } + } else { + if (sf->y_step_q4 == 16) { + // No scaling in the y direction. Must always scale in the x direction. + sf->predict[0][0][0] = aom_convolve8_horiz; + sf->predict[0][0][1] = aom_convolve8_avg_horiz; + sf->predict[0][1][0] = aom_convolve8; + sf->predict[0][1][1] = aom_convolve8_avg; + sf->predict[1][0][0] = aom_convolve8_horiz; + sf->predict[1][0][1] = aom_convolve8_avg_horiz; + } else { + // Must always scale in both directions. + sf->predict[0][0][0] = aom_convolve8; + sf->predict[0][0][1] = aom_convolve8_avg; + sf->predict[0][1][0] = aom_convolve8; + sf->predict[0][1][1] = aom_convolve8_avg; + sf->predict[1][0][0] = aom_convolve8; + sf->predict[1][0][1] = aom_convolve8_avg; + } + } + // 2D subpel motion always gets filtered in both directions + sf->predict[1][1][0] = aom_convolve8; + sf->predict[1][1][1] = aom_convolve8_avg; + +#if CONFIG_HIGHBITDEPTH + if (use_highbd) { + if (sf->x_step_q4 == 16) { + if (sf->y_step_q4 == 16) { + // No scaling in either direction. + sf->highbd_predict[0][0][0] = aom_highbd_convolve_copy; + sf->highbd_predict[0][0][1] = aom_highbd_convolve_avg; + sf->highbd_predict[0][1][0] = aom_highbd_convolve8_vert; + sf->highbd_predict[0][1][1] = aom_highbd_convolve8_avg_vert; + sf->highbd_predict[1][0][0] = aom_highbd_convolve8_horiz; + sf->highbd_predict[1][0][1] = aom_highbd_convolve8_avg_horiz; + } else { + // No scaling in x direction. Must always scale in the y direction. + sf->highbd_predict[0][0][0] = aom_highbd_convolve8_vert; + sf->highbd_predict[0][0][1] = aom_highbd_convolve8_avg_vert; + sf->highbd_predict[0][1][0] = aom_highbd_convolve8_vert; + sf->highbd_predict[0][1][1] = aom_highbd_convolve8_avg_vert; + sf->highbd_predict[1][0][0] = aom_highbd_convolve8; + sf->highbd_predict[1][0][1] = aom_highbd_convolve8_avg; + } + } else { + if (sf->y_step_q4 == 16) { + // No scaling in the y direction. Must always scale in the x direction. + sf->highbd_predict[0][0][0] = aom_highbd_convolve8_horiz; + sf->highbd_predict[0][0][1] = aom_highbd_convolve8_avg_horiz; + sf->highbd_predict[0][1][0] = aom_highbd_convolve8; + sf->highbd_predict[0][1][1] = aom_highbd_convolve8_avg; + sf->highbd_predict[1][0][0] = aom_highbd_convolve8_horiz; + sf->highbd_predict[1][0][1] = aom_highbd_convolve8_avg_horiz; + } else { + // Must always scale in both directions. + sf->highbd_predict[0][0][0] = aom_highbd_convolve8; + sf->highbd_predict[0][0][1] = aom_highbd_convolve8_avg; + sf->highbd_predict[0][1][0] = aom_highbd_convolve8; + sf->highbd_predict[0][1][1] = aom_highbd_convolve8_avg; + sf->highbd_predict[1][0][0] = aom_highbd_convolve8; + sf->highbd_predict[1][0][1] = aom_highbd_convolve8_avg; + } + } + // 2D subpel motion always gets filtered in both directions. + sf->highbd_predict[1][1][0] = aom_highbd_convolve8; + sf->highbd_predict[1][1][1] = aom_highbd_convolve8_avg; + } +#endif // CONFIG_HIGHBITDEPTH +} diff --git a/third_party/aom/av1/common/scale.h b/third_party/aom/av1/common/scale.h new file mode 100644 index 0000000000..ea81efab0e --- /dev/null +++ b/third_party/aom/av1/common/scale.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_SCALE_H_ +#define AV1_COMMON_SCALE_H_ + +#include "av1/common/mv.h" +#include "aom_dsp/aom_convolve.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define REF_SCALE_SHIFT 14 +#define REF_NO_SCALE (1 << REF_SCALE_SHIFT) +#define REF_INVALID_SCALE -1 + +struct scale_factors { + int x_scale_fp; // horizontal fixed point scale factor + int y_scale_fp; // vertical fixed point scale factor + int x_step_q4; + int y_step_q4; + + int (*scale_value_x)(int val, const struct scale_factors *sf); + int (*scale_value_y)(int val, const struct scale_factors *sf); + + convolve_fn_t predict[2][2][2]; // horiz, vert, avg +#if CONFIG_HIGHBITDEPTH + highbd_convolve_fn_t highbd_predict[2][2][2]; // horiz, vert, avg +#endif // CONFIG_HIGHBITDEPTH +}; + +MV32 av1_scale_mv(const MV *mv, int x, int y, const struct scale_factors *sf); + +#if CONFIG_HIGHBITDEPTH +void av1_setup_scale_factors_for_frame(struct scale_factors *sf, int other_w, + int other_h, int this_w, int this_h, + int use_high); +#else +void av1_setup_scale_factors_for_frame(struct scale_factors *sf, int other_w, + int other_h, int this_w, int this_h); +#endif // CONFIG_HIGHBITDEPTH + +static INLINE int av1_is_valid_scale(const struct scale_factors *sf) { + return sf->x_scale_fp != REF_INVALID_SCALE && + sf->y_scale_fp != REF_INVALID_SCALE; +} + +static INLINE int av1_is_scaled(const struct scale_factors *sf) { + return av1_is_valid_scale(sf) && + (sf->x_scale_fp != REF_NO_SCALE || sf->y_scale_fp != REF_NO_SCALE); +} + +static INLINE int valid_ref_frame_size(int ref_width, int ref_height, + int this_width, int this_height) { + return 2 * this_width >= ref_width && 2 * this_height >= ref_height && + this_width <= 16 * ref_width && this_height <= 16 * ref_height; +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_SCALE_H_ diff --git a/third_party/aom/av1/common/scan.c b/third_party/aom/av1/common/scan.c new file mode 100644 index 0000000000..9ad6c0b2f6 --- /dev/null +++ b/third_party/aom/av1/common/scan.c @@ -0,0 +1,6853 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/common_data.h" +#include "av1/common/scan.h" + +#if CONFIG_CB4X4 +DECLARE_ALIGNED(16, static const int16_t, default_scan_2x2[4]) = { + 0, 1, 2, 3, +}; +#endif + +DECLARE_ALIGNED(16, static const int16_t, default_scan_4x4[16]) = { + 0, 4, 1, 5, 8, 2, 12, 9, 3, 6, 13, 10, 7, 14, 11, 15, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_4x4[16]) = { + 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15, +}; + +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_4x4[16]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, col_scan_4x4[16]) = { + 0, 4, 8, 1, 12, 5, 9, 2, 13, 6, 10, 3, 7, 14, 11, 15, +}; + +DECLARE_ALIGNED(16, static const int16_t, row_scan_4x4[16]) = { + 0, 1, 4, 2, 5, 3, 6, 8, 9, 7, 12, 10, 13, 11, 14, 15, +}; + +DECLARE_ALIGNED(16, static const int16_t, default_scan_4x8[32]) = { + 0, 1, 4, 5, 2, 8, 6, 9, 10, 3, 12, 7, 13, 11, 14, 16, + 17, 15, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, +}; + +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_4x8[32]) = { + 0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29, + 2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31, +}; + +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_4x8[32]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, +}; + +DECLARE_ALIGNED(16, static const int16_t, default_scan_8x4[32]) = { + 0, 1, 8, 9, 2, 16, 10, 17, 18, 3, 24, 11, 25, 19, 26, 4, + 12, 27, 20, 5, 28, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31, +}; + +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_8x4[32]) = { + 0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27, + 4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31, +}; + +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_8x4[32]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, +}; + +DECLARE_ALIGNED(16, static const int16_t, default_scan_4x16[64]) = { + 0, 1, 4, 2, 5, 8, 3, 6, 9, 12, 7, 10, 13, 16, 11, 14, + 17, 20, 15, 18, 21, 24, 19, 22, 25, 28, 23, 26, 29, 32, 27, 30, + 33, 36, 31, 34, 37, 40, 35, 38, 41, 44, 39, 42, 45, 48, 43, 46, + 49, 52, 47, 50, 53, 56, 51, 54, 57, 60, 55, 58, 61, 59, 62, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, default_scan_16x4[64]) = { + 0, 1, 16, 2, 17, 32, 3, 18, 33, 48, 4, 19, 34, 49, 5, 20, + 35, 50, 6, 21, 36, 51, 7, 22, 37, 52, 8, 23, 38, 53, 9, 24, + 39, 54, 10, 25, 40, 55, 11, 26, 41, 56, 12, 27, 42, 57, 13, 28, + 43, 58, 14, 29, 44, 59, 15, 30, 45, 60, 31, 46, 61, 47, 62, 63, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_4x16[64]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_16x4[64]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_4x16[64]) = { + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, + 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, + 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62, + 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_16x4[64]) = { + 0, 16, 32, 48, 1, 17, 33, 49, 2, 18, 34, 50, 3, 19, 35, 51, + 4, 20, 36, 52, 5, 21, 37, 53, 6, 22, 38, 54, 7, 23, 39, 55, + 8, 24, 40, 56, 9, 25, 41, 57, 10, 26, 42, 58, 11, 27, 43, 59, + 12, 28, 44, 60, 13, 29, 45, 61, 14, 30, 46, 62, 15, 31, 47, 63, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, default_scan_8x32[256]) = { + 0, 1, 8, 2, 9, 16, 3, 10, 17, 24, 4, 11, 18, 25, 32, + 5, 12, 19, 26, 33, 40, 6, 13, 20, 27, 34, 41, 48, 7, 14, + 21, 28, 35, 42, 49, 56, 15, 22, 29, 36, 43, 50, 57, 64, 23, + 30, 37, 44, 51, 58, 65, 72, 31, 38, 45, 52, 59, 66, 73, 80, + 39, 46, 53, 60, 67, 74, 81, 88, 47, 54, 61, 68, 75, 82, 89, + 96, 55, 62, 69, 76, 83, 90, 97, 104, 63, 70, 77, 84, 91, 98, + 105, 112, 71, 78, 85, 92, 99, 106, 113, 120, 79, 86, 93, 100, 107, + 114, 121, 128, 87, 94, 101, 108, 115, 122, 129, 136, 95, 102, 109, 116, + 123, 130, 137, 144, 103, 110, 117, 124, 131, 138, 145, 152, 111, 118, 125, + 132, 139, 146, 153, 160, 119, 126, 133, 140, 147, 154, 161, 168, 127, 134, + 141, 148, 155, 162, 169, 176, 135, 142, 149, 156, 163, 170, 177, 184, 143, + 150, 157, 164, 171, 178, 185, 192, 151, 158, 165, 172, 179, 186, 193, 200, + 159, 166, 173, 180, 187, 194, 201, 208, 167, 174, 181, 188, 195, 202, 209, + 216, 175, 182, 189, 196, 203, 210, 217, 224, 183, 190, 197, 204, 211, 218, + 225, 232, 191, 198, 205, 212, 219, 226, 233, 240, 199, 206, 213, 220, 227, + 234, 241, 248, 207, 214, 221, 228, 235, 242, 249, 215, 222, 229, 236, 243, + 250, 223, 230, 237, 244, 251, 231, 238, 245, 252, 239, 246, 253, 247, 254, + 255, +}; + +DECLARE_ALIGNED(16, static const int16_t, default_scan_32x8[256]) = { + 0, 1, 32, 2, 33, 64, 3, 34, 65, 96, 4, 35, 66, 97, 128, + 5, 36, 67, 98, 129, 160, 6, 37, 68, 99, 130, 161, 192, 7, 38, + 69, 100, 131, 162, 193, 224, 8, 39, 70, 101, 132, 163, 194, 225, 9, + 40, 71, 102, 133, 164, 195, 226, 10, 41, 72, 103, 134, 165, 196, 227, + 11, 42, 73, 104, 135, 166, 197, 228, 12, 43, 74, 105, 136, 167, 198, + 229, 13, 44, 75, 106, 137, 168, 199, 230, 14, 45, 76, 107, 138, 169, + 200, 231, 15, 46, 77, 108, 139, 170, 201, 232, 16, 47, 78, 109, 140, + 171, 202, 233, 17, 48, 79, 110, 141, 172, 203, 234, 18, 49, 80, 111, + 142, 173, 204, 235, 19, 50, 81, 112, 143, 174, 205, 236, 20, 51, 82, + 113, 144, 175, 206, 237, 21, 52, 83, 114, 145, 176, 207, 238, 22, 53, + 84, 115, 146, 177, 208, 239, 23, 54, 85, 116, 147, 178, 209, 240, 24, + 55, 86, 117, 148, 179, 210, 241, 25, 56, 87, 118, 149, 180, 211, 242, + 26, 57, 88, 119, 150, 181, 212, 243, 27, 58, 89, 120, 151, 182, 213, + 244, 28, 59, 90, 121, 152, 183, 214, 245, 29, 60, 91, 122, 153, 184, + 215, 246, 30, 61, 92, 123, 154, 185, 216, 247, 31, 62, 93, 124, 155, + 186, 217, 248, 63, 94, 125, 156, 187, 218, 249, 95, 126, 157, 188, 219, + 250, 127, 158, 189, 220, 251, 159, 190, 221, 252, 191, 222, 253, 223, 254, + 255, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_8x32[256]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, +}; + +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_32x8[256]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, +}; + +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_8x32[256]) = { + 0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, + 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, + 240, 248, 1, 9, 17, 25, 33, 41, 49, 57, 65, 73, 81, 89, 97, + 105, 113, 121, 129, 137, 145, 153, 161, 169, 177, 185, 193, 201, 209, 217, + 225, 233, 241, 249, 2, 10, 18, 26, 34, 42, 50, 58, 66, 74, 82, + 90, 98, 106, 114, 122, 130, 138, 146, 154, 162, 170, 178, 186, 194, 202, + 210, 218, 226, 234, 242, 250, 3, 11, 19, 27, 35, 43, 51, 59, 67, + 75, 83, 91, 99, 107, 115, 123, 131, 139, 147, 155, 163, 171, 179, 187, + 195, 203, 211, 219, 227, 235, 243, 251, 4, 12, 20, 28, 36, 44, 52, + 60, 68, 76, 84, 92, 100, 108, 116, 124, 132, 140, 148, 156, 164, 172, + 180, 188, 196, 204, 212, 220, 228, 236, 244, 252, 5, 13, 21, 29, 37, + 45, 53, 61, 69, 77, 85, 93, 101, 109, 117, 125, 133, 141, 149, 157, + 165, 173, 181, 189, 197, 205, 213, 221, 229, 237, 245, 253, 6, 14, 22, + 30, 38, 46, 54, 62, 70, 78, 86, 94, 102, 110, 118, 126, 134, 142, + 150, 158, 166, 174, 182, 190, 198, 206, 214, 222, 230, 238, 246, 254, 7, + 15, 23, 31, 39, 47, 55, 63, 71, 79, 87, 95, 103, 111, 119, 127, + 135, 143, 151, 159, 167, 175, 183, 191, 199, 207, 215, 223, 231, 239, 247, + 255, +}; + +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_32x8[256]) = { + 0, 32, 64, 96, 128, 160, 192, 224, 1, 33, 65, 97, 129, 161, 193, 225, + 2, 34, 66, 98, 130, 162, 194, 226, 3, 35, 67, 99, 131, 163, 195, 227, + 4, 36, 68, 100, 132, 164, 196, 228, 5, 37, 69, 101, 133, 165, 197, 229, + 6, 38, 70, 102, 134, 166, 198, 230, 7, 39, 71, 103, 135, 167, 199, 231, + 8, 40, 72, 104, 136, 168, 200, 232, 9, 41, 73, 105, 137, 169, 201, 233, + 10, 42, 74, 106, 138, 170, 202, 234, 11, 43, 75, 107, 139, 171, 203, 235, + 12, 44, 76, 108, 140, 172, 204, 236, 13, 45, 77, 109, 141, 173, 205, 237, + 14, 46, 78, 110, 142, 174, 206, 238, 15, 47, 79, 111, 143, 175, 207, 239, + 16, 48, 80, 112, 144, 176, 208, 240, 17, 49, 81, 113, 145, 177, 209, 241, + 18, 50, 82, 114, 146, 178, 210, 242, 19, 51, 83, 115, 147, 179, 211, 243, + 20, 52, 84, 116, 148, 180, 212, 244, 21, 53, 85, 117, 149, 181, 213, 245, + 22, 54, 86, 118, 150, 182, 214, 246, 23, 55, 87, 119, 151, 183, 215, 247, + 24, 56, 88, 120, 152, 184, 216, 248, 25, 57, 89, 121, 153, 185, 217, 249, + 26, 58, 90, 122, 154, 186, 218, 250, 27, 59, 91, 123, 155, 187, 219, 251, + 28, 60, 92, 124, 156, 188, 220, 252, 29, 61, 93, 125, 157, 189, 221, 253, + 30, 62, 94, 126, 158, 190, 222, 254, 31, 63, 95, 127, 159, 191, 223, 255, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, default_scan_8x8[64]) = { + 0, 8, 1, 16, 9, 2, 17, 24, 10, 3, 18, 25, 32, 11, 4, 26, + 33, 19, 40, 12, 34, 27, 5, 41, 20, 48, 13, 35, 42, 28, 21, 6, + 49, 56, 36, 43, 29, 7, 14, 50, 57, 44, 22, 37, 15, 51, 58, 30, + 45, 23, 52, 59, 38, 31, 60, 53, 46, 39, 61, 54, 47, 62, 55, 63, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_8x8[64]) = { + 0, 8, 16, 24, 32, 40, 48, 56, 1, 9, 17, 25, 33, 41, 49, 57, + 2, 10, 18, 26, 34, 42, 50, 58, 3, 11, 19, 27, 35, 43, 51, 59, + 4, 12, 20, 28, 36, 44, 52, 60, 5, 13, 21, 29, 37, 45, 53, 61, + 6, 14, 22, 30, 38, 46, 54, 62, 7, 15, 23, 31, 39, 47, 55, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_8x8[64]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, col_scan_8x8[64]) = { + 0, 8, 16, 1, 24, 9, 32, 17, 2, 40, 25, 10, 33, 18, 48, 3, + 26, 41, 11, 56, 19, 34, 4, 49, 27, 42, 12, 35, 20, 57, 50, 28, + 5, 43, 13, 36, 58, 51, 21, 44, 6, 29, 59, 37, 14, 52, 22, 7, + 45, 60, 30, 15, 38, 53, 23, 46, 31, 61, 39, 54, 47, 62, 55, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, row_scan_8x8[64]) = { + 0, 1, 2, 8, 9, 3, 16, 10, 4, 17, 11, 24, 5, 18, 25, 12, + 19, 26, 32, 6, 13, 20, 33, 27, 7, 34, 40, 21, 28, 41, 14, 35, + 48, 42, 29, 36, 49, 22, 43, 15, 56, 37, 50, 44, 30, 57, 23, 51, + 58, 45, 38, 52, 31, 59, 53, 46, 60, 39, 61, 47, 54, 55, 62, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, default_scan_8x16[128]) = { + 0, 1, 8, 2, 9, 16, 3, 10, 17, 24, 4, 11, 18, 25, 32, + 5, 12, 19, 26, 33, 40, 6, 13, 20, 27, 34, 41, 48, 7, 14, + 21, 28, 35, 42, 49, 56, 15, 22, 29, 36, 43, 50, 57, 64, 23, + 30, 37, 44, 51, 58, 65, 72, 31, 38, 45, 52, 59, 66, 73, 80, + 39, 46, 53, 60, 67, 74, 81, 88, 47, 54, 61, 68, 75, 82, 89, + 96, 55, 62, 69, 76, 83, 90, 97, 104, 63, 70, 77, 84, 91, 98, + 105, 112, 71, 78, 85, 92, 99, 106, 113, 120, 79, 86, 93, 100, 107, + 114, 121, 87, 94, 101, 108, 115, 122, 95, 102, 109, 116, 123, 103, 110, + 117, 124, 111, 118, 125, 119, 126, 127, +}; + +DECLARE_ALIGNED(16, static const int16_t, default_scan_16x8[128]) = { + 0, 1, 16, 2, 17, 32, 3, 18, 33, 48, 4, 19, 34, 49, 64, 5, + 20, 35, 50, 65, 80, 6, 21, 36, 51, 66, 81, 96, 7, 22, 37, 52, + 67, 82, 97, 112, 8, 23, 38, 53, 68, 83, 98, 113, 9, 24, 39, 54, + 69, 84, 99, 114, 10, 25, 40, 55, 70, 85, 100, 115, 11, 26, 41, 56, + 71, 86, 101, 116, 12, 27, 42, 57, 72, 87, 102, 117, 13, 28, 43, 58, + 73, 88, 103, 118, 14, 29, 44, 59, 74, 89, 104, 119, 15, 30, 45, 60, + 75, 90, 105, 120, 31, 46, 61, 76, 91, 106, 121, 47, 62, 77, 92, 107, + 122, 63, 78, 93, 108, 123, 79, 94, 109, 124, 95, 110, 125, 111, 126, 127, +}; + +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_8x16[128]) = { + 0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, + 1, 9, 17, 25, 33, 41, 49, 57, 65, 73, 81, 89, 97, 105, 113, 121, + 2, 10, 18, 26, 34, 42, 50, 58, 66, 74, 82, 90, 98, 106, 114, 122, + 3, 11, 19, 27, 35, 43, 51, 59, 67, 75, 83, 91, 99, 107, 115, 123, + 4, 12, 20, 28, 36, 44, 52, 60, 68, 76, 84, 92, 100, 108, 116, 124, + 5, 13, 21, 29, 37, 45, 53, 61, 69, 77, 85, 93, 101, 109, 117, 125, + 6, 14, 22, 30, 38, 46, 54, 62, 70, 78, 86, 94, 102, 110, 118, 126, + 7, 15, 23, 31, 39, 47, 55, 63, 71, 79, 87, 95, 103, 111, 119, 127, +}; + +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_16x8[128]) = { + 0, 16, 32, 48, 64, 80, 96, 112, 1, 17, 33, 49, 65, 81, 97, 113, + 2, 18, 34, 50, 66, 82, 98, 114, 3, 19, 35, 51, 67, 83, 99, 115, + 4, 20, 36, 52, 68, 84, 100, 116, 5, 21, 37, 53, 69, 85, 101, 117, + 6, 22, 38, 54, 70, 86, 102, 118, 7, 23, 39, 55, 71, 87, 103, 119, + 8, 24, 40, 56, 72, 88, 104, 120, 9, 25, 41, 57, 73, 89, 105, 121, + 10, 26, 42, 58, 74, 90, 106, 122, 11, 27, 43, 59, 75, 91, 107, 123, + 12, 28, 44, 60, 76, 92, 108, 124, 13, 29, 45, 61, 77, 93, 109, 125, + 14, 30, 46, 62, 78, 94, 110, 126, 15, 31, 47, 63, 79, 95, 111, 127, +}; + +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_8x16[128]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, +}; + +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_16x8[128]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, +}; + +DECLARE_ALIGNED(16, static const int16_t, default_scan_16x32[512]) = { + 0, 1, 16, 2, 17, 32, 3, 18, 33, 48, 4, 19, 34, 49, 64, + 5, 20, 35, 50, 65, 80, 6, 21, 36, 51, 66, 81, 96, 7, 22, + 37, 52, 67, 82, 97, 112, 8, 23, 38, 53, 68, 83, 98, 113, 128, + 9, 24, 39, 54, 69, 84, 99, 114, 129, 144, 10, 25, 40, 55, 70, + 85, 100, 115, 130, 145, 160, 11, 26, 41, 56, 71, 86, 101, 116, 131, + 146, 161, 176, 12, 27, 42, 57, 72, 87, 102, 117, 132, 147, 162, 177, + 192, 13, 28, 43, 58, 73, 88, 103, 118, 133, 148, 163, 178, 193, 208, + 14, 29, 44, 59, 74, 89, 104, 119, 134, 149, 164, 179, 194, 209, 224, + 15, 30, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, + 240, 31, 46, 61, 76, 91, 106, 121, 136, 151, 166, 181, 196, 211, 226, + 241, 256, 47, 62, 77, 92, 107, 122, 137, 152, 167, 182, 197, 212, 227, + 242, 257, 272, 63, 78, 93, 108, 123, 138, 153, 168, 183, 198, 213, 228, + 243, 258, 273, 288, 79, 94, 109, 124, 139, 154, 169, 184, 199, 214, 229, + 244, 259, 274, 289, 304, 95, 110, 125, 140, 155, 170, 185, 200, 215, 230, + 245, 260, 275, 290, 305, 320, 111, 126, 141, 156, 171, 186, 201, 216, 231, + 246, 261, 276, 291, 306, 321, 336, 127, 142, 157, 172, 187, 202, 217, 232, + 247, 262, 277, 292, 307, 322, 337, 352, 143, 158, 173, 188, 203, 218, 233, + 248, 263, 278, 293, 308, 323, 338, 353, 368, 159, 174, 189, 204, 219, 234, + 249, 264, 279, 294, 309, 324, 339, 354, 369, 384, 175, 190, 205, 220, 235, + 250, 265, 280, 295, 310, 325, 340, 355, 370, 385, 400, 191, 206, 221, 236, + 251, 266, 281, 296, 311, 326, 341, 356, 371, 386, 401, 416, 207, 222, 237, + 252, 267, 282, 297, 312, 327, 342, 357, 372, 387, 402, 417, 432, 223, 238, + 253, 268, 283, 298, 313, 328, 343, 358, 373, 388, 403, 418, 433, 448, 239, + 254, 269, 284, 299, 314, 329, 344, 359, 374, 389, 404, 419, 434, 449, 464, + 255, 270, 285, 300, 315, 330, 345, 360, 375, 390, 405, 420, 435, 450, 465, + 480, 271, 286, 301, 316, 331, 346, 361, 376, 391, 406, 421, 436, 451, 466, + 481, 496, 287, 302, 317, 332, 347, 362, 377, 392, 407, 422, 437, 452, 467, + 482, 497, 303, 318, 333, 348, 363, 378, 393, 408, 423, 438, 453, 468, 483, + 498, 319, 334, 349, 364, 379, 394, 409, 424, 439, 454, 469, 484, 499, 335, + 350, 365, 380, 395, 410, 425, 440, 455, 470, 485, 500, 351, 366, 381, 396, + 411, 426, 441, 456, 471, 486, 501, 367, 382, 397, 412, 427, 442, 457, 472, + 487, 502, 383, 398, 413, 428, 443, 458, 473, 488, 503, 399, 414, 429, 444, + 459, 474, 489, 504, 415, 430, 445, 460, 475, 490, 505, 431, 446, 461, 476, + 491, 506, 447, 462, 477, 492, 507, 463, 478, 493, 508, 479, 494, 509, 495, + 510, 511, +}; + +DECLARE_ALIGNED(16, static const int16_t, default_scan_32x16[512]) = { + 0, 1, 32, 2, 33, 64, 3, 34, 65, 96, 4, 35, 66, 97, 128, + 5, 36, 67, 98, 129, 160, 6, 37, 68, 99, 130, 161, 192, 7, 38, + 69, 100, 131, 162, 193, 224, 8, 39, 70, 101, 132, 163, 194, 225, 256, + 9, 40, 71, 102, 133, 164, 195, 226, 257, 288, 10, 41, 72, 103, 134, + 165, 196, 227, 258, 289, 320, 11, 42, 73, 104, 135, 166, 197, 228, 259, + 290, 321, 352, 12, 43, 74, 105, 136, 167, 198, 229, 260, 291, 322, 353, + 384, 13, 44, 75, 106, 137, 168, 199, 230, 261, 292, 323, 354, 385, 416, + 14, 45, 76, 107, 138, 169, 200, 231, 262, 293, 324, 355, 386, 417, 448, + 15, 46, 77, 108, 139, 170, 201, 232, 263, 294, 325, 356, 387, 418, 449, + 480, 16, 47, 78, 109, 140, 171, 202, 233, 264, 295, 326, 357, 388, 419, + 450, 481, 17, 48, 79, 110, 141, 172, 203, 234, 265, 296, 327, 358, 389, + 420, 451, 482, 18, 49, 80, 111, 142, 173, 204, 235, 266, 297, 328, 359, + 390, 421, 452, 483, 19, 50, 81, 112, 143, 174, 205, 236, 267, 298, 329, + 360, 391, 422, 453, 484, 20, 51, 82, 113, 144, 175, 206, 237, 268, 299, + 330, 361, 392, 423, 454, 485, 21, 52, 83, 114, 145, 176, 207, 238, 269, + 300, 331, 362, 393, 424, 455, 486, 22, 53, 84, 115, 146, 177, 208, 239, + 270, 301, 332, 363, 394, 425, 456, 487, 23, 54, 85, 116, 147, 178, 209, + 240, 271, 302, 333, 364, 395, 426, 457, 488, 24, 55, 86, 117, 148, 179, + 210, 241, 272, 303, 334, 365, 396, 427, 458, 489, 25, 56, 87, 118, 149, + 180, 211, 242, 273, 304, 335, 366, 397, 428, 459, 490, 26, 57, 88, 119, + 150, 181, 212, 243, 274, 305, 336, 367, 398, 429, 460, 491, 27, 58, 89, + 120, 151, 182, 213, 244, 275, 306, 337, 368, 399, 430, 461, 492, 28, 59, + 90, 121, 152, 183, 214, 245, 276, 307, 338, 369, 400, 431, 462, 493, 29, + 60, 91, 122, 153, 184, 215, 246, 277, 308, 339, 370, 401, 432, 463, 494, + 30, 61, 92, 123, 154, 185, 216, 247, 278, 309, 340, 371, 402, 433, 464, + 495, 31, 62, 93, 124, 155, 186, 217, 248, 279, 310, 341, 372, 403, 434, + 465, 496, 63, 94, 125, 156, 187, 218, 249, 280, 311, 342, 373, 404, 435, + 466, 497, 95, 126, 157, 188, 219, 250, 281, 312, 343, 374, 405, 436, 467, + 498, 127, 158, 189, 220, 251, 282, 313, 344, 375, 406, 437, 468, 499, 159, + 190, 221, 252, 283, 314, 345, 376, 407, 438, 469, 500, 191, 222, 253, 284, + 315, 346, 377, 408, 439, 470, 501, 223, 254, 285, 316, 347, 378, 409, 440, + 471, 502, 255, 286, 317, 348, 379, 410, 441, 472, 503, 287, 318, 349, 380, + 411, 442, 473, 504, 319, 350, 381, 412, 443, 474, 505, 351, 382, 413, 444, + 475, 506, 383, 414, 445, 476, 507, 415, 446, 477, 508, 447, 478, 509, 479, + 510, 511, +}; + +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_16x32[512]) = { + 0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, + 240, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, + 480, 496, 1, 17, 33, 49, 65, 81, 97, 113, 129, 145, 161, 177, 193, + 209, 225, 241, 257, 273, 289, 305, 321, 337, 353, 369, 385, 401, 417, 433, + 449, 465, 481, 497, 2, 18, 34, 50, 66, 82, 98, 114, 130, 146, 162, + 178, 194, 210, 226, 242, 258, 274, 290, 306, 322, 338, 354, 370, 386, 402, + 418, 434, 450, 466, 482, 498, 3, 19, 35, 51, 67, 83, 99, 115, 131, + 147, 163, 179, 195, 211, 227, 243, 259, 275, 291, 307, 323, 339, 355, 371, + 387, 403, 419, 435, 451, 467, 483, 499, 4, 20, 36, 52, 68, 84, 100, + 116, 132, 148, 164, 180, 196, 212, 228, 244, 260, 276, 292, 308, 324, 340, + 356, 372, 388, 404, 420, 436, 452, 468, 484, 500, 5, 21, 37, 53, 69, + 85, 101, 117, 133, 149, 165, 181, 197, 213, 229, 245, 261, 277, 293, 309, + 325, 341, 357, 373, 389, 405, 421, 437, 453, 469, 485, 501, 6, 22, 38, + 54, 70, 86, 102, 118, 134, 150, 166, 182, 198, 214, 230, 246, 262, 278, + 294, 310, 326, 342, 358, 374, 390, 406, 422, 438, 454, 470, 486, 502, 7, + 23, 39, 55, 71, 87, 103, 119, 135, 151, 167, 183, 199, 215, 231, 247, + 263, 279, 295, 311, 327, 343, 359, 375, 391, 407, 423, 439, 455, 471, 487, + 503, 8, 24, 40, 56, 72, 88, 104, 120, 136, 152, 168, 184, 200, 216, + 232, 248, 264, 280, 296, 312, 328, 344, 360, 376, 392, 408, 424, 440, 456, + 472, 488, 504, 9, 25, 41, 57, 73, 89, 105, 121, 137, 153, 169, 185, + 201, 217, 233, 249, 265, 281, 297, 313, 329, 345, 361, 377, 393, 409, 425, + 441, 457, 473, 489, 505, 10, 26, 42, 58, 74, 90, 106, 122, 138, 154, + 170, 186, 202, 218, 234, 250, 266, 282, 298, 314, 330, 346, 362, 378, 394, + 410, 426, 442, 458, 474, 490, 506, 11, 27, 43, 59, 75, 91, 107, 123, + 139, 155, 171, 187, 203, 219, 235, 251, 267, 283, 299, 315, 331, 347, 363, + 379, 395, 411, 427, 443, 459, 475, 491, 507, 12, 28, 44, 60, 76, 92, + 108, 124, 140, 156, 172, 188, 204, 220, 236, 252, 268, 284, 300, 316, 332, + 348, 364, 380, 396, 412, 428, 444, 460, 476, 492, 508, 13, 29, 45, 61, + 77, 93, 109, 125, 141, 157, 173, 189, 205, 221, 237, 253, 269, 285, 301, + 317, 333, 349, 365, 381, 397, 413, 429, 445, 461, 477, 493, 509, 14, 30, + 46, 62, 78, 94, 110, 126, 142, 158, 174, 190, 206, 222, 238, 254, 270, + 286, 302, 318, 334, 350, 366, 382, 398, 414, 430, 446, 462, 478, 494, 510, + 15, 31, 47, 63, 79, 95, 111, 127, 143, 159, 175, 191, 207, 223, 239, + 255, 271, 287, 303, 319, 335, 351, 367, 383, 399, 415, 431, 447, 463, 479, + 495, 511, +}; + +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_32x16[512]) = { + 0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448, 480, + 1, 33, 65, 97, 129, 161, 193, 225, 257, 289, 321, 353, 385, 417, 449, 481, + 2, 34, 66, 98, 130, 162, 194, 226, 258, 290, 322, 354, 386, 418, 450, 482, + 3, 35, 67, 99, 131, 163, 195, 227, 259, 291, 323, 355, 387, 419, 451, 483, + 4, 36, 68, 100, 132, 164, 196, 228, 260, 292, 324, 356, 388, 420, 452, 484, + 5, 37, 69, 101, 133, 165, 197, 229, 261, 293, 325, 357, 389, 421, 453, 485, + 6, 38, 70, 102, 134, 166, 198, 230, 262, 294, 326, 358, 390, 422, 454, 486, + 7, 39, 71, 103, 135, 167, 199, 231, 263, 295, 327, 359, 391, 423, 455, 487, + 8, 40, 72, 104, 136, 168, 200, 232, 264, 296, 328, 360, 392, 424, 456, 488, + 9, 41, 73, 105, 137, 169, 201, 233, 265, 297, 329, 361, 393, 425, 457, 489, + 10, 42, 74, 106, 138, 170, 202, 234, 266, 298, 330, 362, 394, 426, 458, 490, + 11, 43, 75, 107, 139, 171, 203, 235, 267, 299, 331, 363, 395, 427, 459, 491, + 12, 44, 76, 108, 140, 172, 204, 236, 268, 300, 332, 364, 396, 428, 460, 492, + 13, 45, 77, 109, 141, 173, 205, 237, 269, 301, 333, 365, 397, 429, 461, 493, + 14, 46, 78, 110, 142, 174, 206, 238, 270, 302, 334, 366, 398, 430, 462, 494, + 15, 47, 79, 111, 143, 175, 207, 239, 271, 303, 335, 367, 399, 431, 463, 495, + 16, 48, 80, 112, 144, 176, 208, 240, 272, 304, 336, 368, 400, 432, 464, 496, + 17, 49, 81, 113, 145, 177, 209, 241, 273, 305, 337, 369, 401, 433, 465, 497, + 18, 50, 82, 114, 146, 178, 210, 242, 274, 306, 338, 370, 402, 434, 466, 498, + 19, 51, 83, 115, 147, 179, 211, 243, 275, 307, 339, 371, 403, 435, 467, 499, + 20, 52, 84, 116, 148, 180, 212, 244, 276, 308, 340, 372, 404, 436, 468, 500, + 21, 53, 85, 117, 149, 181, 213, 245, 277, 309, 341, 373, 405, 437, 469, 501, + 22, 54, 86, 118, 150, 182, 214, 246, 278, 310, 342, 374, 406, 438, 470, 502, + 23, 55, 87, 119, 151, 183, 215, 247, 279, 311, 343, 375, 407, 439, 471, 503, + 24, 56, 88, 120, 152, 184, 216, 248, 280, 312, 344, 376, 408, 440, 472, 504, + 25, 57, 89, 121, 153, 185, 217, 249, 281, 313, 345, 377, 409, 441, 473, 505, + 26, 58, 90, 122, 154, 186, 218, 250, 282, 314, 346, 378, 410, 442, 474, 506, + 27, 59, 91, 123, 155, 187, 219, 251, 283, 315, 347, 379, 411, 443, 475, 507, + 28, 60, 92, 124, 156, 188, 220, 252, 284, 316, 348, 380, 412, 444, 476, 508, + 29, 61, 93, 125, 157, 189, 221, 253, 285, 317, 349, 381, 413, 445, 477, 509, + 30, 62, 94, 126, 158, 190, 222, 254, 286, 318, 350, 382, 414, 446, 478, 510, + 31, 63, 95, 127, 159, 191, 223, 255, 287, 319, 351, 383, 415, 447, 479, 511, +}; + +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_16x32[512]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, + 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, + 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, + 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, + 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, + 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, + 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, + 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, + 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, + 510, 511, +}; + +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_32x16[512]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, + 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, + 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, + 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, + 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, + 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, + 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, + 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, + 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, + 510, 511, +}; + +DECLARE_ALIGNED(16, static const int16_t, default_scan_16x16[256]) = { + 0, 16, 1, 32, 17, 2, 48, 33, 18, 3, 64, 34, 49, 19, 65, + 80, 50, 4, 35, 66, 20, 81, 96, 51, 5, 36, 82, 97, 67, 112, + 21, 52, 98, 37, 83, 113, 6, 68, 128, 53, 22, 99, 114, 84, 7, + 129, 38, 69, 100, 115, 144, 130, 85, 54, 23, 8, 145, 39, 70, 116, + 101, 131, 160, 146, 55, 86, 24, 71, 132, 117, 161, 40, 9, 102, 147, + 176, 162, 87, 56, 25, 133, 118, 177, 148, 72, 103, 41, 163, 10, 192, + 178, 88, 57, 134, 149, 119, 26, 164, 73, 104, 193, 42, 179, 208, 11, + 135, 89, 165, 120, 150, 58, 194, 180, 27, 74, 209, 105, 151, 136, 43, + 90, 224, 166, 195, 181, 121, 210, 59, 12, 152, 106, 167, 196, 75, 137, + 225, 211, 240, 182, 122, 91, 28, 197, 13, 226, 168, 183, 153, 44, 212, + 138, 107, 241, 60, 29, 123, 198, 184, 227, 169, 242, 76, 213, 154, 45, + 92, 14, 199, 139, 61, 228, 214, 170, 185, 243, 108, 77, 155, 30, 15, + 200, 229, 124, 215, 244, 93, 46, 186, 171, 201, 109, 140, 230, 62, 216, + 245, 31, 125, 78, 156, 231, 47, 187, 202, 217, 94, 246, 141, 63, 232, + 172, 110, 247, 157, 79, 218, 203, 126, 233, 188, 248, 95, 173, 142, 219, + 111, 249, 234, 158, 127, 189, 204, 250, 235, 143, 174, 220, 205, 159, 251, + 190, 221, 175, 236, 237, 191, 206, 252, 222, 253, 207, 238, 223, 254, 239, + 255, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_16x16[256]) = { + 0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, + 1, 17, 33, 49, 65, 81, 97, 113, 129, 145, 161, 177, 193, 209, 225, 241, + 2, 18, 34, 50, 66, 82, 98, 114, 130, 146, 162, 178, 194, 210, 226, 242, + 3, 19, 35, 51, 67, 83, 99, 115, 131, 147, 163, 179, 195, 211, 227, 243, + 4, 20, 36, 52, 68, 84, 100, 116, 132, 148, 164, 180, 196, 212, 228, 244, + 5, 21, 37, 53, 69, 85, 101, 117, 133, 149, 165, 181, 197, 213, 229, 245, + 6, 22, 38, 54, 70, 86, 102, 118, 134, 150, 166, 182, 198, 214, 230, 246, + 7, 23, 39, 55, 71, 87, 103, 119, 135, 151, 167, 183, 199, 215, 231, 247, + 8, 24, 40, 56, 72, 88, 104, 120, 136, 152, 168, 184, 200, 216, 232, 248, + 9, 25, 41, 57, 73, 89, 105, 121, 137, 153, 169, 185, 201, 217, 233, 249, + 10, 26, 42, 58, 74, 90, 106, 122, 138, 154, 170, 186, 202, 218, 234, 250, + 11, 27, 43, 59, 75, 91, 107, 123, 139, 155, 171, 187, 203, 219, 235, 251, + 12, 28, 44, 60, 76, 92, 108, 124, 140, 156, 172, 188, 204, 220, 236, 252, + 13, 29, 45, 61, 77, 93, 109, 125, 141, 157, 173, 189, 205, 221, 237, 253, + 14, 30, 46, 62, 78, 94, 110, 126, 142, 158, 174, 190, 206, 222, 238, 254, + 15, 31, 47, 63, 79, 95, 111, 127, 143, 159, 175, 191, 207, 223, 239, 255, +}; + +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_16x16[256]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, col_scan_16x16[256]) = { + 0, 16, 32, 48, 1, 64, 17, 80, 33, 96, 49, 2, 65, 112, 18, + 81, 34, 128, 50, 97, 3, 66, 144, 19, 113, 35, 82, 160, 98, 51, + 129, 4, 67, 176, 20, 114, 145, 83, 36, 99, 130, 52, 192, 5, 161, + 68, 115, 21, 146, 84, 208, 177, 37, 131, 100, 53, 162, 224, 69, 6, + 116, 193, 147, 85, 22, 240, 132, 38, 178, 101, 163, 54, 209, 117, 70, + 7, 148, 194, 86, 179, 225, 23, 133, 39, 164, 8, 102, 210, 241, 55, + 195, 118, 149, 71, 180, 24, 87, 226, 134, 165, 211, 40, 103, 56, 72, + 150, 196, 242, 119, 9, 181, 227, 88, 166, 25, 135, 41, 104, 212, 57, + 151, 197, 120, 73, 243, 182, 136, 167, 213, 89, 10, 228, 105, 152, 198, + 26, 42, 121, 183, 244, 168, 58, 137, 229, 74, 214, 90, 153, 199, 184, + 11, 106, 245, 27, 122, 230, 169, 43, 215, 59, 200, 138, 185, 246, 75, + 12, 91, 154, 216, 231, 107, 28, 44, 201, 123, 170, 60, 247, 232, 76, + 139, 13, 92, 217, 186, 248, 155, 108, 29, 124, 45, 202, 233, 171, 61, + 14, 77, 140, 15, 249, 93, 30, 187, 156, 218, 46, 109, 125, 62, 172, + 78, 203, 31, 141, 234, 94, 47, 188, 63, 157, 110, 250, 219, 79, 126, + 204, 173, 142, 95, 189, 111, 235, 158, 220, 251, 127, 174, 143, 205, 236, + 159, 190, 221, 252, 175, 206, 237, 191, 253, 222, 238, 207, 254, 223, 239, + 255, +}; + +DECLARE_ALIGNED(16, static const int16_t, row_scan_16x16[256]) = { + 0, 1, 2, 16, 3, 17, 4, 18, 32, 5, 33, 19, 6, 34, 48, + 20, 49, 7, 35, 21, 50, 64, 8, 36, 65, 22, 51, 37, 80, 9, + 66, 52, 23, 38, 81, 67, 10, 53, 24, 82, 68, 96, 39, 11, 54, + 83, 97, 69, 25, 98, 84, 40, 112, 55, 12, 70, 99, 113, 85, 26, + 41, 56, 114, 100, 13, 71, 128, 86, 27, 115, 101, 129, 42, 57, 72, + 116, 14, 87, 130, 102, 144, 73, 131, 117, 28, 58, 15, 88, 43, 145, + 103, 132, 146, 118, 74, 160, 89, 133, 104, 29, 59, 147, 119, 44, 161, + 148, 90, 105, 134, 162, 120, 176, 75, 135, 149, 30, 60, 163, 177, 45, + 121, 91, 106, 164, 178, 150, 192, 136, 165, 179, 31, 151, 193, 76, 122, + 61, 137, 194, 107, 152, 180, 208, 46, 166, 167, 195, 92, 181, 138, 209, + 123, 153, 224, 196, 77, 168, 210, 182, 240, 108, 197, 62, 154, 225, 183, + 169, 211, 47, 139, 93, 184, 226, 212, 241, 198, 170, 124, 155, 199, 78, + 213, 185, 109, 227, 200, 63, 228, 242, 140, 214, 171, 186, 156, 229, 243, + 125, 94, 201, 244, 215, 216, 230, 141, 187, 202, 79, 172, 110, 157, 245, + 217, 231, 95, 246, 232, 126, 203, 247, 233, 173, 218, 142, 111, 158, 188, + 248, 127, 234, 219, 249, 189, 204, 143, 174, 159, 250, 235, 205, 220, 175, + 190, 251, 221, 191, 206, 236, 207, 237, 252, 222, 253, 223, 238, 239, 254, + 255, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, mcol_scan_32x32[1024]) = { + 0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, + 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800, 832, 864, + 896, 928, 960, 992, 1, 33, 65, 97, 129, 161, 193, 225, 257, 289, + 321, 353, 385, 417, 449, 481, 513, 545, 577, 609, 641, 673, 705, 737, + 769, 801, 833, 865, 897, 929, 961, 993, 2, 34, 66, 98, 130, 162, + 194, 226, 258, 290, 322, 354, 386, 418, 450, 482, 514, 546, 578, 610, + 642, 674, 706, 738, 770, 802, 834, 866, 898, 930, 962, 994, 3, 35, + 67, 99, 131, 163, 195, 227, 259, 291, 323, 355, 387, 419, 451, 483, + 515, 547, 579, 611, 643, 675, 707, 739, 771, 803, 835, 867, 899, 931, + 963, 995, 4, 36, 68, 100, 132, 164, 196, 228, 260, 292, 324, 356, + 388, 420, 452, 484, 516, 548, 580, 612, 644, 676, 708, 740, 772, 804, + 836, 868, 900, 932, 964, 996, 5, 37, 69, 101, 133, 165, 197, 229, + 261, 293, 325, 357, 389, 421, 453, 485, 517, 549, 581, 613, 645, 677, + 709, 741, 773, 805, 837, 869, 901, 933, 965, 997, 6, 38, 70, 102, + 134, 166, 198, 230, 262, 294, 326, 358, 390, 422, 454, 486, 518, 550, + 582, 614, 646, 678, 710, 742, 774, 806, 838, 870, 902, 934, 966, 998, + 7, 39, 71, 103, 135, 167, 199, 231, 263, 295, 327, 359, 391, 423, + 455, 487, 519, 551, 583, 615, 647, 679, 711, 743, 775, 807, 839, 871, + 903, 935, 967, 999, 8, 40, 72, 104, 136, 168, 200, 232, 264, 296, + 328, 360, 392, 424, 456, 488, 520, 552, 584, 616, 648, 680, 712, 744, + 776, 808, 840, 872, 904, 936, 968, 1000, 9, 41, 73, 105, 137, 169, + 201, 233, 265, 297, 329, 361, 393, 425, 457, 489, 521, 553, 585, 617, + 649, 681, 713, 745, 777, 809, 841, 873, 905, 937, 969, 1001, 10, 42, + 74, 106, 138, 170, 202, 234, 266, 298, 330, 362, 394, 426, 458, 490, + 522, 554, 586, 618, 650, 682, 714, 746, 778, 810, 842, 874, 906, 938, + 970, 1002, 11, 43, 75, 107, 139, 171, 203, 235, 267, 299, 331, 363, + 395, 427, 459, 491, 523, 555, 587, 619, 651, 683, 715, 747, 779, 811, + 843, 875, 907, 939, 971, 1003, 12, 44, 76, 108, 140, 172, 204, 236, + 268, 300, 332, 364, 396, 428, 460, 492, 524, 556, 588, 620, 652, 684, + 716, 748, 780, 812, 844, 876, 908, 940, 972, 1004, 13, 45, 77, 109, + 141, 173, 205, 237, 269, 301, 333, 365, 397, 429, 461, 493, 525, 557, + 589, 621, 653, 685, 717, 749, 781, 813, 845, 877, 909, 941, 973, 1005, + 14, 46, 78, 110, 142, 174, 206, 238, 270, 302, 334, 366, 398, 430, + 462, 494, 526, 558, 590, 622, 654, 686, 718, 750, 782, 814, 846, 878, + 910, 942, 974, 1006, 15, 47, 79, 111, 143, 175, 207, 239, 271, 303, + 335, 367, 399, 431, 463, 495, 527, 559, 591, 623, 655, 687, 719, 751, + 783, 815, 847, 879, 911, 943, 975, 1007, 16, 48, 80, 112, 144, 176, + 208, 240, 272, 304, 336, 368, 400, 432, 464, 496, 528, 560, 592, 624, + 656, 688, 720, 752, 784, 816, 848, 880, 912, 944, 976, 1008, 17, 49, + 81, 113, 145, 177, 209, 241, 273, 305, 337, 369, 401, 433, 465, 497, + 529, 561, 593, 625, 657, 689, 721, 753, 785, 817, 849, 881, 913, 945, + 977, 1009, 18, 50, 82, 114, 146, 178, 210, 242, 274, 306, 338, 370, + 402, 434, 466, 498, 530, 562, 594, 626, 658, 690, 722, 754, 786, 818, + 850, 882, 914, 946, 978, 1010, 19, 51, 83, 115, 147, 179, 211, 243, + 275, 307, 339, 371, 403, 435, 467, 499, 531, 563, 595, 627, 659, 691, + 723, 755, 787, 819, 851, 883, 915, 947, 979, 1011, 20, 52, 84, 116, + 148, 180, 212, 244, 276, 308, 340, 372, 404, 436, 468, 500, 532, 564, + 596, 628, 660, 692, 724, 756, 788, 820, 852, 884, 916, 948, 980, 1012, + 21, 53, 85, 117, 149, 181, 213, 245, 277, 309, 341, 373, 405, 437, + 469, 501, 533, 565, 597, 629, 661, 693, 725, 757, 789, 821, 853, 885, + 917, 949, 981, 1013, 22, 54, 86, 118, 150, 182, 214, 246, 278, 310, + 342, 374, 406, 438, 470, 502, 534, 566, 598, 630, 662, 694, 726, 758, + 790, 822, 854, 886, 918, 950, 982, 1014, 23, 55, 87, 119, 151, 183, + 215, 247, 279, 311, 343, 375, 407, 439, 471, 503, 535, 567, 599, 631, + 663, 695, 727, 759, 791, 823, 855, 887, 919, 951, 983, 1015, 24, 56, + 88, 120, 152, 184, 216, 248, 280, 312, 344, 376, 408, 440, 472, 504, + 536, 568, 600, 632, 664, 696, 728, 760, 792, 824, 856, 888, 920, 952, + 984, 1016, 25, 57, 89, 121, 153, 185, 217, 249, 281, 313, 345, 377, + 409, 441, 473, 505, 537, 569, 601, 633, 665, 697, 729, 761, 793, 825, + 857, 889, 921, 953, 985, 1017, 26, 58, 90, 122, 154, 186, 218, 250, + 282, 314, 346, 378, 410, 442, 474, 506, 538, 570, 602, 634, 666, 698, + 730, 762, 794, 826, 858, 890, 922, 954, 986, 1018, 27, 59, 91, 123, + 155, 187, 219, 251, 283, 315, 347, 379, 411, 443, 475, 507, 539, 571, + 603, 635, 667, 699, 731, 763, 795, 827, 859, 891, 923, 955, 987, 1019, + 28, 60, 92, 124, 156, 188, 220, 252, 284, 316, 348, 380, 412, 444, + 476, 508, 540, 572, 604, 636, 668, 700, 732, 764, 796, 828, 860, 892, + 924, 956, 988, 1020, 29, 61, 93, 125, 157, 189, 221, 253, 285, 317, + 349, 381, 413, 445, 477, 509, 541, 573, 605, 637, 669, 701, 733, 765, + 797, 829, 861, 893, 925, 957, 989, 1021, 30, 62, 94, 126, 158, 190, + 222, 254, 286, 318, 350, 382, 414, 446, 478, 510, 542, 574, 606, 638, + 670, 702, 734, 766, 798, 830, 862, 894, 926, 958, 990, 1022, 31, 63, + 95, 127, 159, 191, 223, 255, 287, 319, 351, 383, 415, 447, 479, 511, + 543, 575, 607, 639, 671, 703, 735, 767, 799, 831, 863, 895, 927, 959, + 991, 1023, +}; + +DECLARE_ALIGNED(16, static const int16_t, mrow_scan_32x32[1024]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, + 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, + 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, + 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, + 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, + 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, + 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, + 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, + 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, + 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, + 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, + 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, + 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, + 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, + 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, + 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, + 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, + 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, + 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, + 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, + 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, + 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, + 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, + 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, + 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, + 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, + 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, + 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, + 624, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, + 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, + 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, + 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, + 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, 688, + 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, + 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, + 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, + 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, + 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, + 754, 755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, + 767, 768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, + 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, + 793, 794, 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, 805, + 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, + 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, + 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, + 845, 846, 847, 848, 849, 850, 851, 852, 853, 854, 855, 856, 857, + 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, + 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, + 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, + 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, + 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, + 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, + 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, + 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, + 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, + 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, + 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000, + 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, + 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, default_scan_32x32[1024]) = { + 0, 32, 1, 64, 33, 2, 96, 65, 34, 128, 3, 97, 66, + 160, 129, 35, 98, 4, 67, 130, 161, 192, 36, 99, 224, 5, + 162, 193, 68, 131, 37, 100, 225, 194, 256, 163, 69, 132, 6, + 226, 257, 288, 195, 101, 164, 38, 258, 7, 227, 289, 133, 320, + 70, 196, 165, 290, 259, 228, 39, 321, 102, 352, 8, 197, 71, + 134, 322, 291, 260, 353, 384, 229, 166, 103, 40, 354, 323, 292, + 135, 385, 198, 261, 72, 9, 416, 167, 386, 355, 230, 324, 104, + 293, 41, 417, 199, 136, 262, 387, 448, 325, 356, 10, 73, 418, + 231, 168, 449, 294, 388, 105, 419, 263, 42, 200, 357, 450, 137, + 480, 74, 326, 232, 11, 389, 169, 295, 420, 106, 451, 481, 358, + 264, 327, 201, 43, 138, 512, 482, 390, 296, 233, 170, 421, 75, + 452, 359, 12, 513, 265, 483, 328, 107, 202, 514, 544, 422, 391, + 453, 139, 44, 234, 484, 297, 360, 171, 76, 515, 545, 266, 329, + 454, 13, 423, 203, 108, 546, 485, 576, 298, 235, 140, 361, 330, + 172, 547, 45, 455, 267, 577, 486, 77, 204, 362, 608, 14, 299, + 578, 109, 236, 487, 609, 331, 141, 579, 46, 15, 173, 610, 363, + 78, 205, 16, 110, 237, 611, 142, 47, 174, 79, 206, 17, 111, + 238, 48, 143, 80, 175, 112, 207, 49, 18, 239, 81, 113, 19, + 50, 82, 114, 51, 83, 115, 640, 516, 392, 268, 144, 20, 672, + 641, 548, 517, 424, 393, 300, 269, 176, 145, 52, 21, 704, 673, + 642, 580, 549, 518, 456, 425, 394, 332, 301, 270, 208, 177, 146, + 84, 53, 22, 736, 705, 674, 643, 612, 581, 550, 519, 488, 457, + 426, 395, 364, 333, 302, 271, 240, 209, 178, 147, 116, 85, 54, + 23, 737, 706, 675, 613, 582, 551, 489, 458, 427, 365, 334, 303, + 241, 210, 179, 117, 86, 55, 738, 707, 614, 583, 490, 459, 366, + 335, 242, 211, 118, 87, 739, 615, 491, 367, 243, 119, 768, 644, + 520, 396, 272, 148, 24, 800, 769, 676, 645, 552, 521, 428, 397, + 304, 273, 180, 149, 56, 25, 832, 801, 770, 708, 677, 646, 584, + 553, 522, 460, 429, 398, 336, 305, 274, 212, 181, 150, 88, 57, + 26, 864, 833, 802, 771, 740, 709, 678, 647, 616, 585, 554, 523, + 492, 461, 430, 399, 368, 337, 306, 275, 244, 213, 182, 151, 120, + 89, 58, 27, 865, 834, 803, 741, 710, 679, 617, 586, 555, 493, + 462, 431, 369, 338, 307, 245, 214, 183, 121, 90, 59, 866, 835, + 742, 711, 618, 587, 494, 463, 370, 339, 246, 215, 122, 91, 867, + 743, 619, 495, 371, 247, 123, 896, 772, 648, 524, 400, 276, 152, + 28, 928, 897, 804, 773, 680, 649, 556, 525, 432, 401, 308, 277, + 184, 153, 60, 29, 960, 929, 898, 836, 805, 774, 712, 681, 650, + 588, 557, 526, 464, 433, 402, 340, 309, 278, 216, 185, 154, 92, + 61, 30, 992, 961, 930, 899, 868, 837, 806, 775, 744, 713, 682, + 651, 620, 589, 558, 527, 496, 465, 434, 403, 372, 341, 310, 279, + 248, 217, 186, 155, 124, 93, 62, 31, 993, 962, 931, 869, 838, + 807, 745, 714, 683, 621, 590, 559, 497, 466, 435, 373, 342, 311, + 249, 218, 187, 125, 94, 63, 994, 963, 870, 839, 746, 715, 622, + 591, 498, 467, 374, 343, 250, 219, 126, 95, 995, 871, 747, 623, + 499, 375, 251, 127, 900, 776, 652, 528, 404, 280, 156, 932, 901, + 808, 777, 684, 653, 560, 529, 436, 405, 312, 281, 188, 157, 964, + 933, 902, 840, 809, 778, 716, 685, 654, 592, 561, 530, 468, 437, + 406, 344, 313, 282, 220, 189, 158, 996, 965, 934, 903, 872, 841, + 810, 779, 748, 717, 686, 655, 624, 593, 562, 531, 500, 469, 438, + 407, 376, 345, 314, 283, 252, 221, 190, 159, 997, 966, 935, 873, + 842, 811, 749, 718, 687, 625, 594, 563, 501, 470, 439, 377, 346, + 315, 253, 222, 191, 998, 967, 874, 843, 750, 719, 626, 595, 502, + 471, 378, 347, 254, 223, 999, 875, 751, 627, 503, 379, 255, 904, + 780, 656, 532, 408, 284, 936, 905, 812, 781, 688, 657, 564, 533, + 440, 409, 316, 285, 968, 937, 906, 844, 813, 782, 720, 689, 658, + 596, 565, 534, 472, 441, 410, 348, 317, 286, 1000, 969, 938, 907, + 876, 845, 814, 783, 752, 721, 690, 659, 628, 597, 566, 535, 504, + 473, 442, 411, 380, 349, 318, 287, 1001, 970, 939, 877, 846, 815, + 753, 722, 691, 629, 598, 567, 505, 474, 443, 381, 350, 319, 1002, + 971, 878, 847, 754, 723, 630, 599, 506, 475, 382, 351, 1003, 879, + 755, 631, 507, 383, 908, 784, 660, 536, 412, 940, 909, 816, 785, + 692, 661, 568, 537, 444, 413, 972, 941, 910, 848, 817, 786, 724, + 693, 662, 600, 569, 538, 476, 445, 414, 1004, 973, 942, 911, 880, + 849, 818, 787, 756, 725, 694, 663, 632, 601, 570, 539, 508, 477, + 446, 415, 1005, 974, 943, 881, 850, 819, 757, 726, 695, 633, 602, + 571, 509, 478, 447, 1006, 975, 882, 851, 758, 727, 634, 603, 510, + 479, 1007, 883, 759, 635, 511, 912, 788, 664, 540, 944, 913, 820, + 789, 696, 665, 572, 541, 976, 945, 914, 852, 821, 790, 728, 697, + 666, 604, 573, 542, 1008, 977, 946, 915, 884, 853, 822, 791, 760, + 729, 698, 667, 636, 605, 574, 543, 1009, 978, 947, 885, 854, 823, + 761, 730, 699, 637, 606, 575, 1010, 979, 886, 855, 762, 731, 638, + 607, 1011, 887, 763, 639, 916, 792, 668, 948, 917, 824, 793, 700, + 669, 980, 949, 918, 856, 825, 794, 732, 701, 670, 1012, 981, 950, + 919, 888, 857, 826, 795, 764, 733, 702, 671, 1013, 982, 951, 889, + 858, 827, 765, 734, 703, 1014, 983, 890, 859, 766, 735, 1015, 891, + 767, 920, 796, 952, 921, 828, 797, 984, 953, 922, 860, 829, 798, + 1016, 985, 954, 923, 892, 861, 830, 799, 1017, 986, 955, 893, 862, + 831, 1018, 987, 894, 863, 1019, 895, 924, 956, 925, 988, 957, 926, + 1020, 989, 958, 927, 1021, 990, 959, 1022, 991, 1023, +}; + +// Scan over two rectangular vertical partitions one after the other +DECLARE_ALIGNED(16, static const int16_t, v2_scan_32x32[1024]) = { + 0, 1, 32, 33, 2, 64, 34, 65, 66, 3, 96, 35, 97, + 67, 98, 4, 128, 36, 129, 99, 68, 130, 5, 100, 131, 160, + 37, 161, 69, 162, 132, 101, 163, 6, 192, 38, 193, 70, 194, + 133, 164, 102, 195, 7, 224, 39, 165, 225, 134, 196, 71, 226, + 103, 227, 166, 197, 8, 256, 40, 135, 228, 257, 72, 258, 198, + 104, 259, 167, 229, 136, 260, 9, 288, 41, 289, 73, 199, 230, + 290, 168, 261, 105, 291, 137, 292, 231, 10, 200, 262, 320, 42, + 321, 74, 322, 169, 293, 106, 323, 232, 263, 138, 324, 201, 294, + 11, 352, 43, 353, 75, 170, 325, 354, 264, 107, 233, 295, 355, + 202, 326, 139, 356, 12, 384, 44, 265, 296, 385, 171, 357, 76, + 386, 234, 327, 108, 387, 203, 358, 140, 388, 297, 266, 328, 13, + 172, 389, 416, 45, 235, 359, 417, 77, 418, 109, 419, 204, 390, + 298, 329, 141, 267, 360, 420, 236, 391, 173, 421, 14, 448, 46, + 449, 78, 330, 450, 299, 361, 110, 205, 422, 451, 268, 392, 142, + 452, 237, 423, 174, 331, 362, 453, 15, 300, 393, 480, 47, 481, + 79, 482, 206, 454, 269, 424, 111, 483, 143, 484, 363, 332, 394, + 238, 455, 175, 301, 425, 485, 512, 513, 270, 456, 514, 207, 486, + 364, 395, 515, 333, 426, 516, 239, 487, 302, 457, 517, 396, 271, + 488, 544, 365, 427, 545, 518, 546, 334, 458, 547, 519, 548, 303, + 489, 397, 428, 549, 366, 459, 520, 576, 335, 490, 550, 577, 578, + 579, 521, 429, 551, 398, 460, 580, 367, 491, 581, 552, 522, 582, + 608, 609, 430, 461, 610, 399, 492, 553, 611, 583, 523, 612, 613, + 584, 554, 462, 431, 493, 614, 524, 640, 641, 642, 585, 643, 555, + 615, 644, 463, 494, 586, 525, 616, 645, 556, 646, 672, 617, 673, + 587, 674, 647, 495, 675, 526, 676, 557, 618, 648, 677, 588, 678, + 527, 649, 619, 704, 558, 705, 706, 679, 589, 707, 650, 708, 620, + 680, 709, 559, 590, 710, 651, 681, 736, 621, 737, 711, 738, 739, + 682, 652, 740, 712, 591, 741, 622, 683, 713, 742, 653, 768, 769, + 743, 770, 714, 684, 771, 623, 772, 744, 654, 773, 715, 685, 745, + 774, 655, 775, 800, 801, 716, 746, 802, 803, 686, 776, 804, 747, + 805, 717, 777, 806, 687, 748, 807, 778, 832, 833, 718, 834, 835, + 808, 836, 779, 749, 837, 809, 719, 838, 780, 750, 810, 839, 864, + 865, 866, 867, 840, 781, 868, 811, 751, 869, 841, 870, 812, 782, + 842, 871, 896, 897, 898, 872, 899, 813, 843, 900, 783, 901, 873, + 844, 902, 814, 874, 903, 928, 929, 845, 930, 904, 815, 875, 931, + 932, 905, 933, 846, 876, 934, 906, 935, 877, 960, 847, 961, 962, + 907, 936, 963, 964, 937, 878, 965, 908, 966, 938, 967, 909, 879, + 992, 939, 993, 968, 994, 995, 996, 910, 969, 940, 997, 998, 970, + 911, 941, 999, 971, 1000, 942, 1001, 972, 1002, 943, 973, 1003, 974, + 1004, 975, 1005, 1006, 1007, 16, 48, 80, 112, 144, 176, 17, 49, + 208, 81, 113, 145, 240, 177, 272, 18, 50, 209, 82, 114, 304, + 241, 146, 178, 273, 336, 210, 19, 51, 83, 115, 305, 242, 147, + 368, 179, 274, 337, 211, 20, 400, 52, 84, 306, 116, 243, 369, + 148, 338, 180, 275, 432, 401, 212, 21, 53, 307, 85, 370, 244, + 117, 464, 149, 433, 339, 276, 181, 402, 213, 308, 496, 371, 22, + 54, 465, 86, 245, 118, 434, 150, 340, 277, 403, 182, 528, 497, + 214, 466, 372, 309, 23, 55, 435, 87, 246, 119, 341, 404, 151, + 529, 560, 278, 498, 183, 467, 373, 215, 310, 436, 24, 56, 247, + 561, 88, 530, 592, 342, 120, 405, 499, 152, 279, 468, 184, 374, + 311, 437, 216, 562, 593, 531, 624, 25, 248, 500, 57, 406, 89, + 343, 121, 469, 280, 153, 594, 185, 375, 563, 625, 438, 532, 656, + 312, 217, 501, 407, 249, 26, 344, 58, 90, 470, 122, 595, 626, + 281, 564, 657, 154, 376, 533, 688, 439, 186, 313, 502, 218, 408, + 627, 596, 658, 250, 345, 471, 27, 59, 565, 689, 91, 123, 282, + 534, 720, 155, 440, 377, 187, 503, 314, 628, 659, 219, 597, 690, + 409, 472, 566, 721, 346, 251, 28, 60, 535, 752, 92, 124, 283, + 441, 378, 156, 660, 504, 629, 691, 598, 722, 188, 315, 567, 753, + 220, 410, 473, 347, 536, 784, 252, 29, 661, 692, 61, 93, 442, + 630, 723, 284, 125, 379, 505, 599, 754, 157, 316, 568, 785, 189, + 474, 411, 221, 537, 816, 693, 348, 662, 724, 253, 631, 755, 443, + 30, 600, 786, 62, 506, 94, 285, 380, 126, 569, 817, 158, 317, + 190, 475, 694, 725, 412, 663, 756, 538, 848, 222, 632, 787, 349, + 254, 601, 818, 444, 507, 31, 63, 381, 286, 95, 570, 849, 726, + 127, 695, 757, 664, 788, 159, 476, 318, 413, 539, 880, 191, 633, + 819, 223, 350, 602, 850, 508, 255, 445, 727, 758, 696, 789, 571, + 881, 382, 287, 665, 820, 477, 634, 851, 540, 912, 319, 414, 603, + 882, 759, 728, 790, 351, 509, 697, 821, 446, 572, 913, 666, 852, + 383, 635, 883, 478, 541, 944, 415, 760, 791, 604, 914, 729, 822, + 698, 853, 510, 667, 884, 447, 573, 945, 636, 915, 792, 761, 823, + 542, 976, 479, 730, 854, 605, 946, 699, 885, 668, 916, 511, 574, + 977, 793, 824, 637, 947, 762, 855, 731, 886, 543, 1008, 606, 978, + 700, 917, 669, 948, 575, 825, 1009, 794, 856, 763, 887, 638, 979, + 732, 918, 701, 949, 607, 1010, 670, 980, 826, 857, 795, 888, 764, + 919, 639, 1011, 733, 950, 702, 981, 858, 827, 889, 796, 920, 671, + 1012, 765, 951, 734, 982, 703, 1013, 859, 890, 828, 921, 797, 952, + 766, 983, 735, 1014, 891, 860, 922, 829, 953, 798, 984, 767, 1015, + 892, 923, 861, 954, 830, 985, 799, 1016, 924, 893, 955, 862, 986, + 831, 1017, 925, 956, 894, 987, 863, 1018, 957, 926, 988, 895, 1019, + 958, 989, 927, 1020, 990, 959, 1021, 991, 1022, 1023, +}; + +// Scan over two rectangular horizontal partitions one after the other +DECLARE_ALIGNED(16, static const int16_t, h2_scan_32x32[1024]) = { + 0, 1, 32, 33, 2, 64, 34, 65, 66, 3, 96, 35, 97, + 67, 98, 4, 128, 36, 129, 99, 68, 130, 5, 100, 131, 160, + 37, 161, 69, 162, 132, 101, 163, 6, 192, 38, 193, 70, 194, + 133, 164, 102, 195, 7, 224, 39, 165, 225, 134, 196, 71, 226, + 103, 227, 166, 197, 8, 256, 40, 135, 228, 257, 72, 258, 198, + 104, 259, 167, 229, 136, 260, 9, 288, 41, 289, 73, 199, 230, + 290, 168, 261, 105, 291, 137, 292, 231, 10, 200, 262, 320, 42, + 321, 74, 322, 169, 293, 106, 323, 232, 263, 138, 324, 201, 294, + 11, 352, 43, 353, 75, 170, 325, 354, 264, 107, 233, 295, 355, + 202, 326, 139, 356, 12, 384, 44, 265, 296, 385, 171, 357, 76, + 386, 234, 327, 108, 387, 203, 358, 140, 388, 297, 266, 328, 13, + 172, 389, 416, 45, 235, 359, 417, 77, 418, 109, 419, 204, 390, + 298, 329, 141, 267, 360, 420, 236, 391, 173, 421, 14, 448, 46, + 449, 78, 330, 450, 299, 361, 110, 205, 422, 451, 268, 392, 142, + 452, 237, 423, 174, 331, 362, 453, 15, 300, 393, 480, 47, 481, + 79, 482, 206, 454, 269, 424, 111, 483, 143, 484, 363, 332, 394, + 238, 455, 175, 301, 425, 485, 16, 48, 80, 270, 456, 207, 486, + 112, 364, 395, 333, 426, 144, 239, 487, 302, 457, 176, 396, 17, + 271, 488, 49, 365, 427, 208, 81, 334, 458, 113, 145, 240, 303, + 489, 397, 428, 177, 366, 459, 272, 18, 50, 209, 335, 490, 82, + 114, 304, 241, 429, 146, 398, 460, 367, 491, 178, 273, 336, 210, + 19, 51, 83, 430, 461, 399, 492, 115, 305, 242, 147, 368, 179, + 274, 337, 462, 431, 493, 211, 20, 400, 52, 84, 306, 116, 243, + 369, 148, 463, 494, 338, 180, 275, 432, 401, 212, 21, 53, 307, + 85, 370, 244, 117, 495, 464, 149, 433, 339, 276, 181, 402, 213, + 308, 496, 371, 22, 54, 465, 86, 245, 118, 434, 150, 340, 277, + 403, 182, 497, 214, 466, 372, 309, 23, 55, 435, 87, 246, 119, + 341, 404, 151, 278, 498, 183, 467, 373, 215, 310, 436, 24, 56, + 247, 88, 342, 120, 405, 499, 152, 279, 468, 184, 374, 311, 437, + 216, 25, 248, 500, 57, 406, 89, 343, 121, 469, 280, 153, 185, + 375, 438, 312, 217, 501, 407, 249, 26, 344, 58, 90, 470, 122, + 281, 154, 376, 439, 186, 313, 502, 218, 408, 250, 345, 471, 27, + 59, 91, 123, 282, 155, 440, 377, 187, 503, 314, 219, 409, 472, + 346, 251, 28, 60, 92, 124, 283, 441, 378, 156, 504, 188, 315, + 220, 410, 473, 347, 252, 29, 61, 93, 442, 284, 125, 379, 505, + 157, 316, 189, 474, 411, 221, 348, 253, 443, 30, 62, 506, 94, + 285, 380, 126, 158, 317, 190, 475, 412, 222, 349, 254, 444, 507, + 31, 63, 381, 286, 95, 127, 159, 476, 318, 413, 191, 223, 350, + 508, 255, 445, 382, 287, 477, 319, 414, 351, 509, 446, 383, 478, + 415, 510, 447, 479, 511, 512, 513, 514, 515, 516, 517, 544, 545, + 518, 546, 547, 519, 548, 549, 520, 576, 550, 577, 578, 579, 521, + 551, 580, 581, 552, 522, 582, 608, 609, 610, 553, 611, 583, 523, + 612, 613, 584, 554, 614, 524, 640, 641, 642, 585, 643, 555, 615, + 644, 586, 525, 616, 645, 556, 646, 672, 617, 673, 587, 674, 647, + 675, 526, 676, 557, 618, 648, 677, 588, 678, 527, 649, 619, 704, + 558, 705, 706, 679, 589, 707, 650, 708, 620, 680, 709, 528, 559, + 590, 710, 651, 681, 736, 621, 737, 711, 738, 739, 682, 652, 529, + 560, 740, 712, 591, 741, 622, 683, 713, 742, 653, 768, 769, 561, + 743, 530, 592, 770, 714, 684, 771, 623, 772, 744, 654, 773, 715, + 685, 745, 774, 562, 593, 531, 624, 655, 775, 800, 801, 716, 746, + 802, 803, 686, 776, 804, 594, 563, 625, 747, 805, 717, 532, 656, + 777, 806, 687, 748, 807, 778, 832, 833, 718, 834, 595, 626, 835, + 564, 657, 808, 836, 533, 688, 779, 749, 837, 809, 719, 838, 780, + 627, 596, 658, 750, 810, 839, 864, 565, 689, 865, 866, 867, 534, + 720, 840, 781, 868, 811, 751, 869, 841, 628, 659, 597, 690, 870, + 812, 782, 566, 721, 842, 871, 896, 535, 752, 897, 898, 872, 899, + 813, 843, 660, 900, 783, 629, 691, 598, 722, 901, 873, 567, 753, + 844, 902, 814, 874, 536, 784, 903, 661, 692, 928, 929, 630, 723, + 845, 930, 904, 815, 875, 931, 599, 754, 932, 568, 785, 905, 933, + 846, 876, 934, 537, 816, 693, 662, 724, 906, 631, 755, 935, 877, + 600, 786, 960, 847, 961, 962, 907, 936, 963, 569, 817, 964, 937, + 694, 725, 878, 965, 908, 663, 756, 538, 848, 966, 632, 787, 938, + 601, 818, 967, 909, 879, 992, 939, 993, 968, 570, 849, 994, 726, + 695, 757, 995, 664, 788, 996, 910, 969, 539, 880, 940, 633, 819, + 997, 998, 602, 850, 970, 911, 941, 999, 727, 758, 696, 789, 571, + 881, 971, 665, 820, 1000, 634, 851, 942, 540, 912, 1001, 972, 603, + 882, 759, 728, 790, 1002, 697, 821, 943, 973, 572, 913, 666, 852, + 1003, 635, 883, 974, 541, 944, 760, 791, 1004, 604, 914, 729, 822, + 698, 853, 975, 667, 884, 573, 945, 1005, 636, 915, 792, 761, 823, + 542, 976, 1006, 730, 854, 605, 946, 699, 885, 668, 916, 1007, 574, + 977, 793, 824, 637, 947, 762, 855, 731, 886, 543, 1008, 606, 978, + 700, 917, 669, 948, 575, 825, 1009, 794, 856, 763, 887, 638, 979, + 732, 918, 701, 949, 607, 1010, 670, 980, 826, 857, 795, 888, 764, + 919, 639, 1011, 733, 950, 702, 981, 858, 827, 889, 796, 920, 671, + 1012, 765, 951, 734, 982, 703, 1013, 859, 890, 828, 921, 797, 952, + 766, 983, 735, 1014, 891, 860, 922, 829, 953, 798, 984, 767, 1015, + 892, 923, 861, 954, 830, 985, 799, 1016, 924, 893, 955, 862, 986, + 831, 1017, 925, 956, 894, 987, 863, 1018, 957, 926, 988, 895, 1019, + 958, 989, 927, 1020, 990, 959, 1021, 991, 1022, 1023, +}; + +// Scan where the top left quarter is scanned first +DECLARE_ALIGNED(16, static const int16_t, qtr_scan_32x32[1024]) = { + 0, 1, 32, 33, 2, 64, 34, 65, 66, 3, 96, 35, 97, + 67, 98, 4, 128, 36, 129, 99, 68, 130, 5, 100, 131, 160, + 37, 161, 69, 162, 132, 101, 163, 6, 192, 38, 193, 70, 194, + 133, 164, 102, 195, 7, 224, 39, 165, 225, 134, 196, 71, 226, + 103, 227, 166, 197, 8, 256, 40, 135, 228, 257, 72, 258, 198, + 104, 259, 167, 229, 136, 260, 9, 288, 41, 289, 73, 199, 230, + 290, 168, 261, 105, 291, 137, 292, 231, 10, 200, 262, 320, 42, + 321, 74, 322, 169, 293, 106, 323, 232, 263, 138, 324, 201, 294, + 11, 352, 43, 353, 75, 170, 325, 354, 264, 107, 233, 295, 355, + 202, 326, 139, 356, 12, 384, 44, 265, 296, 385, 171, 357, 76, + 386, 234, 327, 108, 387, 203, 358, 140, 388, 297, 266, 328, 13, + 172, 389, 416, 45, 235, 359, 417, 77, 418, 109, 419, 204, 390, + 298, 329, 141, 267, 360, 420, 236, 391, 173, 421, 14, 448, 46, + 449, 78, 330, 450, 299, 361, 110, 205, 422, 451, 268, 392, 142, + 452, 237, 423, 174, 331, 362, 453, 15, 300, 393, 480, 47, 481, + 79, 482, 206, 454, 269, 424, 111, 483, 143, 484, 363, 332, 394, + 238, 455, 175, 301, 425, 485, 270, 456, 207, 486, 364, 395, 333, + 426, 239, 487, 302, 457, 396, 271, 488, 365, 427, 334, 458, 303, + 489, 397, 428, 366, 459, 335, 490, 429, 398, 460, 367, 491, 430, + 461, 399, 492, 462, 431, 493, 463, 494, 495, 16, 512, 48, 513, + 80, 514, 112, 515, 144, 516, 176, 517, 17, 544, 49, 545, 208, + 518, 81, 546, 113, 547, 145, 240, 519, 548, 177, 549, 272, 520, + 18, 576, 50, 209, 550, 577, 82, 578, 114, 579, 304, 521, 241, + 551, 146, 580, 178, 581, 273, 552, 336, 522, 210, 582, 19, 608, + 51, 609, 83, 610, 115, 305, 553, 611, 242, 583, 147, 368, 523, + 612, 179, 613, 274, 584, 337, 554, 211, 614, 20, 400, 524, 640, + 52, 641, 84, 642, 306, 585, 116, 643, 243, 369, 555, 615, 148, + 644, 338, 586, 180, 275, 432, 525, 616, 645, 401, 556, 212, 646, + 21, 672, 53, 307, 617, 673, 85, 370, 587, 674, 244, 647, 117, + 675, 464, 526, 149, 676, 433, 557, 339, 618, 276, 648, 181, 677, + 402, 588, 213, 678, 308, 496, 527, 649, 371, 619, 22, 704, 54, + 465, 558, 705, 86, 706, 245, 679, 118, 434, 589, 707, 150, 340, + 650, 708, 277, 403, 620, 680, 182, 709, 528, 497, 559, 214, 466, + 590, 710, 372, 651, 309, 681, 23, 736, 55, 435, 621, 737, 87, + 246, 711, 738, 119, 739, 341, 682, 404, 652, 151, 529, 560, 740, + 278, 712, 498, 591, 183, 741, 467, 622, 373, 683, 215, 310, 713, + 742, 436, 653, 24, 768, 56, 769, 247, 561, 743, 88, 530, 592, + 770, 342, 714, 120, 405, 684, 771, 499, 623, 152, 772, 279, 744, + 468, 654, 184, 773, 374, 715, 311, 437, 685, 745, 216, 774, 562, + 593, 531, 624, 25, 248, 500, 655, 775, 800, 57, 801, 406, 716, + 89, 343, 746, 802, 121, 803, 469, 686, 280, 776, 153, 804, 594, + 185, 375, 563, 625, 747, 805, 438, 717, 532, 656, 312, 777, 217, + 806, 501, 687, 407, 748, 249, 807, 26, 344, 778, 832, 58, 833, + 90, 470, 718, 834, 122, 595, 626, 835, 281, 564, 657, 808, 154, + 836, 376, 533, 688, 779, 439, 749, 186, 837, 313, 809, 502, 719, + 218, 838, 408, 780, 627, 596, 658, 250, 345, 471, 750, 810, 839, + 27, 864, 59, 565, 689, 865, 91, 866, 123, 867, 282, 534, 720, + 840, 155, 440, 781, 868, 377, 811, 187, 503, 751, 869, 314, 841, + 628, 659, 219, 597, 690, 870, 409, 812, 472, 782, 566, 721, 346, + 842, 251, 871, 28, 896, 60, 535, 752, 897, 92, 898, 124, 283, + 872, 899, 441, 813, 378, 843, 156, 660, 900, 504, 783, 629, 691, + 598, 722, 188, 901, 315, 873, 567, 753, 220, 410, 844, 902, 473, + 814, 347, 874, 536, 784, 252, 903, 29, 661, 692, 928, 61, 929, + 93, 442, 630, 723, 845, 930, 284, 904, 125, 379, 505, 815, 875, + 931, 599, 754, 157, 932, 316, 568, 785, 905, 189, 933, 474, 846, + 411, 876, 221, 934, 537, 816, 693, 348, 662, 724, 906, 253, 631, + 755, 935, 443, 877, 30, 600, 786, 960, 62, 506, 847, 961, 94, + 962, 285, 380, 907, 936, 126, 963, 569, 817, 158, 964, 317, 937, + 190, 475, 694, 725, 878, 965, 412, 908, 663, 756, 538, 848, 222, + 966, 632, 787, 349, 938, 254, 601, 818, 967, 444, 909, 507, 879, + 31, 992, 63, 381, 939, 993, 286, 968, 95, 570, 849, 994, 726, + 127, 695, 757, 995, 664, 788, 159, 996, 476, 910, 318, 969, 413, + 539, 880, 940, 191, 633, 819, 997, 223, 998, 350, 602, 850, 970, + 508, 911, 255, 445, 941, 999, 727, 758, 696, 789, 571, 881, 382, + 971, 287, 665, 820, 1000, 477, 634, 851, 942, 540, 912, 319, 1001, + 414, 972, 603, 882, 759, 728, 790, 351, 1002, 509, 697, 821, 943, + 446, 973, 572, 913, 666, 852, 383, 1003, 635, 883, 478, 974, 541, + 944, 415, 760, 791, 1004, 604, 914, 729, 822, 698, 853, 510, 975, + 667, 884, 447, 573, 945, 1005, 636, 915, 792, 761, 823, 542, 976, + 479, 1006, 730, 854, 605, 946, 699, 885, 668, 916, 511, 1007, 574, + 977, 793, 824, 637, 947, 762, 855, 731, 886, 543, 1008, 606, 978, + 700, 917, 669, 948, 575, 825, 1009, 794, 856, 763, 887, 638, 979, + 732, 918, 701, 949, 607, 1010, 670, 980, 826, 857, 795, 888, 764, + 919, 639, 1011, 733, 950, 702, 981, 858, 827, 889, 796, 920, 671, + 1012, 765, 951, 734, 982, 703, 1013, 859, 890, 828, 921, 797, 952, + 766, 983, 735, 1014, 891, 860, 922, 829, 953, 798, 984, 767, 1015, + 892, 923, 861, 954, 830, 985, 799, 1016, 924, 893, 955, 862, 986, + 831, 1017, 925, 956, 894, 987, 863, 1018, 957, 926, 988, 895, 1019, + 958, 989, 927, 1020, 990, 959, 1021, 991, 1022, 1023, +}; + +#if CONFIG_TX64X64 +DECLARE_ALIGNED(16, static const int16_t, default_scan_64x64[4096]) = { + 0, 1, 64, 65, 2, 128, 66, 129, 130, 3, 192, 67, 193, + 131, 194, 4, 256, 68, 257, 195, 132, 258, 5, 196, 259, 320, + 69, 321, 133, 322, 260, 197, 323, 6, 384, 70, 385, 134, 386, + 261, 324, 198, 387, 7, 448, 71, 325, 449, 262, 388, 135, 450, + 199, 451, 326, 389, 8, 512, 72, 263, 452, 513, 136, 514, 390, + 200, 515, 327, 453, 264, 516, 9, 576, 73, 577, 137, 391, 454, + 578, 328, 517, 201, 579, 265, 580, 455, 10, 392, 518, 640, 74, + 641, 138, 642, 329, 581, 202, 643, 456, 519, 266, 644, 393, 582, + 11, 704, 75, 705, 139, 330, 645, 706, 520, 203, 457, 583, 707, + 394, 646, 267, 708, 12, 768, 76, 521, 584, 769, 331, 709, 140, + 770, 458, 647, 204, 771, 395, 710, 268, 772, 585, 522, 648, 13, + 332, 773, 832, 77, 459, 711, 833, 141, 834, 205, 835, 396, 774, + 586, 649, 269, 523, 712, 836, 460, 775, 333, 837, 14, 896, 78, + 897, 142, 650, 898, 587, 713, 206, 397, 838, 899, 524, 776, 270, + 900, 461, 839, 334, 651, 714, 901, 15, 588, 777, 960, 79, 961, + 143, 962, 398, 902, 525, 840, 207, 963, 271, 964, 715, 652, 778, + 462, 903, 335, 589, 841, 965, 16, 1024, 80, 1025, 144, 526, 904, + 1026, 399, 966, 208, 716, 779, 1027, 653, 842, 272, 1028, 463, 967, + 590, 905, 336, 1029, 780, 17, 527, 968, 1088, 81, 717, 843, 1089, + 400, 1030, 145, 1090, 654, 906, 209, 1091, 273, 464, 1031, 1092, 591, + 969, 781, 844, 337, 1093, 718, 907, 528, 1032, 18, 1152, 82, 401, + 655, 970, 1094, 1153, 146, 1154, 210, 1155, 592, 1033, 465, 845, 1095, + 274, 782, 908, 1156, 719, 971, 338, 1157, 529, 1096, 656, 1034, 402, + 1158, 19, 1216, 83, 1217, 147, 846, 909, 1218, 783, 972, 211, 593, + 1097, 1219, 466, 1159, 275, 720, 1035, 1220, 339, 1221, 530, 1160, 657, + 1098, 910, 847, 973, 403, 1222, 20, 784, 1036, 1280, 84, 1281, 148, + 1282, 594, 1161, 212, 1283, 467, 721, 1099, 1223, 276, 1284, 911, 974, + 658, 1162, 340, 531, 848, 1037, 1224, 1285, 785, 1100, 404, 1286, 21, + 1344, 85, 595, 1225, 1345, 149, 722, 1163, 1346, 468, 1287, 213, 975, + 1347, 912, 1038, 277, 1348, 849, 1101, 659, 1226, 532, 1288, 341, 1349, + 786, 1164, 405, 1350, 596, 976, 1039, 1289, 723, 1227, 22, 1408, 86, + 913, 1102, 1409, 150, 1410, 469, 1351, 214, 850, 1165, 1411, 278, 660, + 1290, 1412, 533, 787, 1228, 1352, 342, 1413, 1040, 977, 1103, 406, 914, + 1166, 1414, 724, 1291, 597, 1353, 23, 1472, 87, 851, 1229, 1473, 151, + 470, 1415, 1474, 215, 1475, 661, 1354, 788, 1292, 279, 1041, 1104, 1476, + 534, 1416, 978, 1167, 343, 1477, 915, 1230, 725, 1355, 407, 598, 1417, + 1478, 852, 1293, 24, 1536, 88, 1537, 471, 1105, 1479, 152, 1042, 1168, + 1538, 662, 1418, 216, 789, 1356, 1539, 979, 1231, 280, 1540, 535, 1480, + 916, 1294, 344, 1541, 726, 1419, 599, 853, 1357, 1481, 408, 1542, 1106, + 1169, 1043, 1232, 25, 472, 980, 1295, 1543, 1600, 89, 1601, 790, 1420, + 153, 663, 1482, 1602, 217, 1603, 917, 1358, 536, 1544, 281, 1604, 1170, + 345, 727, 1107, 1233, 1483, 1605, 854, 1421, 1044, 1296, 600, 1545, 409, + 1606, 981, 1359, 791, 1484, 473, 1607, 26, 664, 1546, 1664, 90, 1665, + 154, 918, 1422, 1666, 218, 1171, 1234, 1667, 537, 1108, 1297, 1608, 282, + 1668, 728, 1045, 1360, 1547, 855, 1485, 346, 1669, 601, 1609, 982, 1423, + 410, 1670, 792, 1548, 1235, 1172, 1298, 474, 665, 919, 1486, 1610, 1671, + 27, 1728, 91, 1109, 1361, 1729, 155, 1730, 219, 1731, 538, 1046, 1424, + 1672, 283, 856, 1549, 1732, 729, 1611, 347, 983, 1487, 1733, 602, 1673, + 1236, 1299, 411, 1173, 1362, 1734, 793, 1612, 920, 1550, 1110, 1425, 666, + 1674, 475, 1735, 28, 1792, 92, 1047, 1488, 1793, 156, 1794, 220, 539, + 1736, 1795, 857, 1613, 730, 1675, 284, 1300, 1796, 984, 1551, 1237, 1363, + 1174, 1426, 348, 1797, 603, 1737, 1111, 1489, 412, 794, 1676, 1798, 921, + 1614, 667, 1738, 1048, 1552, 476, 1799, 29, 1301, 1364, 1856, 93, 1857, + 157, 858, 1238, 1427, 1677, 1858, 540, 1800, 221, 731, 985, 1615, 1739, + 1859, 1175, 1490, 285, 1860, 604, 1112, 1553, 1801, 349, 1861, 922, 1678, + 795, 1740, 413, 1862, 1049, 1616, 1365, 668, 1302, 1428, 1802, 477, 1239, + 1491, 1863, 859, 1741, 30, 1176, 1554, 1920, 94, 986, 1679, 1921, 158, + 1922, 541, 732, 1803, 1864, 222, 1923, 1113, 1617, 286, 1924, 605, 1865, + 350, 923, 1366, 1429, 1742, 1925, 796, 1804, 1303, 1492, 1050, 1680, 414, + 1926, 1240, 1555, 669, 1866, 478, 1177, 1618, 1927, 860, 1805, 987, 1743, + 31, 1984, 95, 733, 1867, 1985, 542, 1928, 159, 1114, 1681, 1986, 1430, + 223, 1367, 1493, 1987, 1304, 1556, 287, 1988, 924, 1806, 606, 1929, 797, + 1051, 1744, 1868, 351, 1241, 1619, 1989, 415, 1990, 670, 1178, 1682, 1930, + 988, 1807, 479, 861, 1869, 1991, 1431, 1494, 1368, 1557, 1115, 1745, 734, + 1931, 32, 2048, 96, 543, 1305, 1620, 1992, 2049, 160, 2050, 224, 2051, + 925, 1242, 1683, 1870, 288, 1052, 1808, 2052, 607, 1993, 798, 1932, 352, + 2053, 1179, 1746, 1495, 416, 1432, 1558, 2054, 671, 1994, 989, 1369, 1621, + 1871, 862, 1933, 480, 1116, 1809, 2055, 1306, 1684, 735, 1995, 544, 2056, + 33, 2112, 97, 1243, 1747, 2113, 161, 2114, 926, 1934, 1053, 1872, 225, + 2115, 289, 608, 799, 1496, 1559, 1996, 2057, 2116, 1180, 1810, 1433, 1622, + 353, 2117, 1370, 1685, 672, 2058, 417, 990, 1935, 2118, 1307, 1748, 863, + 1117, 1873, 1997, 481, 2119, 736, 1244, 1811, 2059, 1560, 545, 2120, 1497, + 1623, 34, 1054, 1936, 2176, 98, 927, 1998, 2177, 162, 1434, 1686, 2178, + 226, 1181, 1874, 2179, 800, 2060, 609, 1371, 1749, 2121, 290, 2180, 354, + 2181, 1308, 1812, 991, 1999, 673, 1118, 1937, 2122, 418, 2182, 864, 2061, + 1561, 1624, 1245, 1875, 482, 1498, 1687, 2183, 737, 2123, 1435, 1750, 1055, + 2000, 546, 928, 2062, 2184, 1182, 1938, 35, 1372, 1813, 2240, 99, 2241, + 163, 2242, 801, 2124, 227, 2243, 610, 2185, 291, 1309, 1876, 2244, 992, + 2063, 355, 1119, 1625, 2001, 2245, 1562, 1688, 674, 2186, 865, 1499, 1751, + 2125, 419, 1246, 1939, 2246, 1436, 1814, 483, 2247, 738, 2187, 1056, 2064, + 1373, 1877, 929, 1183, 2002, 2126, 547, 2248, 36, 2304, 100, 2305, 164, + 802, 1310, 1940, 2188, 2306, 1626, 1689, 228, 1563, 1752, 2307, 611, 2249, + 292, 2308, 1120, 1500, 1815, 2065, 993, 2127, 356, 2309, 1247, 2003, 675, + 866, 1437, 1878, 2189, 2250, 420, 2310, 1374, 1941, 484, 1057, 2128, 2311, + 739, 2251, 1184, 2066, 930, 1690, 2190, 1627, 1753, 548, 1564, 1816, 2312, + 1311, 2004, 37, 803, 2252, 2368, 101, 1501, 1879, 2369, 165, 2370, 612, + 2313, 229, 1121, 2129, 2371, 994, 2191, 1438, 1942, 293, 1248, 2067, 2372, + 357, 867, 2253, 2373, 676, 2314, 1375, 2005, 421, 1691, 1754, 2374, 1628, + 1817, 1058, 2192, 1185, 2130, 740, 1565, 1880, 2315, 485, 2375, 931, 2254, + 1312, 2068, 1502, 1943, 549, 2376, 804, 2316, 38, 2432, 102, 1122, 1439, + 2006, 2193, 2433, 166, 2434, 613, 995, 1249, 2131, 2255, 2377, 230, 2435, + 1755, 294, 1692, 1818, 2436, 868, 1376, 2069, 2317, 1629, 1881, 358, 677, + 2378, 2437, 1566, 1944, 422, 1186, 2194, 2438, 1059, 2256, 1313, 2132, 741, + 1503, 2007, 2379, 932, 2318, 486, 2439, 550, 1440, 2070, 2440, 805, 1756, + 1819, 2380, 1123, 2257, 1250, 1693, 1882, 2195, 39, 996, 2319, 2496, 103, + 2497, 167, 614, 1630, 1945, 2441, 2498, 231, 1377, 2133, 2499, 295, 1567, + 2008, 2500, 869, 2381, 678, 2442, 359, 2501, 1187, 2258, 1060, 2320, 1504, + 2071, 1314, 2196, 423, 2502, 742, 933, 2382, 2443, 1820, 487, 1757, 1883, + 2503, 1441, 2134, 1694, 1946, 551, 1124, 2321, 2504, 1251, 1631, 2009, 2259, + 806, 2444, 997, 2383, 1378, 2197, 40, 1568, 2072, 2560, 104, 2561, 615, + 2505, 168, 2562, 232, 2563, 870, 2445, 296, 2564, 1505, 2135, 1188, 2322, + 679, 2506, 360, 1061, 1315, 1821, 1884, 2260, 2384, 2565, 1758, 1947, 424, + 2566, 1695, 2010, 934, 1442, 2198, 2446, 743, 2507, 488, 1632, 2073, 2567, + 1252, 2323, 1125, 2385, 552, 2568, 807, 1569, 2136, 2508, 1379, 2261, 998, + 2447, 41, 616, 2569, 2624, 105, 1885, 2625, 1822, 1948, 169, 1506, 2199, + 2626, 233, 871, 1759, 2011, 2509, 2627, 1189, 2386, 1316, 2324, 297, 2628, + 680, 1062, 1696, 2074, 2448, 2570, 361, 2629, 1443, 2262, 1633, 2137, 425, + 935, 2510, 2630, 744, 2571, 489, 1253, 2387, 2631, 1570, 2200, 1126, 2449, + 1380, 2325, 1886, 1949, 808, 2572, 553, 1823, 2012, 2632, 999, 2511, 1760, + 2075, 1507, 2263, 617, 2633, 42, 2688, 106, 1697, 2138, 2689, 170, 1190, + 2450, 2690, 872, 1317, 2388, 2573, 234, 2691, 1063, 2512, 298, 1444, 2326, + 2692, 681, 1634, 2201, 2634, 362, 2693, 936, 2574, 426, 1950, 2694, 1571, + 2264, 745, 1887, 2013, 2635, 1254, 2451, 1824, 2076, 1127, 1381, 2389, 2513, + 490, 2695, 1761, 2139, 809, 1000, 1508, 2327, 2575, 2636, 554, 2696, 1698, + 2202, 1318, 2452, 618, 1191, 2514, 2697, 43, 2752, 107, 873, 1635, 2265, + 2637, 2753, 171, 1445, 2390, 2754, 1064, 2576, 235, 2755, 1951, 2014, 682, + 2698, 299, 1888, 2077, 2756, 1572, 2328, 1825, 2140, 363, 2757, 937, 2638, + 1255, 2515, 427, 746, 1382, 1762, 2203, 2453, 2699, 2758, 1128, 2577, 491, + 1509, 2391, 2759, 1699, 2266, 1001, 2639, 810, 2700, 555, 2760, 1319, 1636, + 2329, 2516, 2015, 1192, 1952, 2078, 2578, 1446, 2454, 619, 1889, 2141, 2761, + 874, 2701, 44, 2816, 108, 1065, 2640, 2817, 172, 1826, 2204, 2818, 236, + 1573, 2392, 2819, 683, 2762, 300, 2820, 1763, 2267, 938, 2702, 364, 1256, + 2579, 2821, 1383, 2517, 747, 1129, 2641, 2763, 428, 1700, 2330, 2822, 1510, + 2455, 492, 2016, 2079, 2823, 1002, 1953, 2142, 2703, 811, 2764, 1637, 2393, + 1890, 2205, 556, 1320, 2580, 2824, 1193, 1447, 2518, 2642, 1827, 2268, 620, + 2825, 875, 2765, 1066, 1574, 2456, 2704, 45, 1764, 2331, 2880, 109, 2881, + 173, 2882, 237, 2883, 684, 2826, 301, 1384, 2581, 2884, 1257, 2643, 939, + 1701, 2394, 2766, 2080, 365, 1511, 2017, 2143, 2519, 2885, 1130, 2705, 1954, + 2206, 748, 2827, 429, 2886, 1891, 2269, 1638, 2457, 493, 1003, 2767, 2887, + 812, 1828, 2332, 2828, 1321, 2644, 1448, 2582, 1194, 2706, 557, 2888, 1575, + 2520, 1765, 2395, 876, 1067, 2768, 2829, 621, 2889, 2081, 2144, 46, 2944, + 110, 2018, 2207, 2945, 174, 1702, 2458, 2946, 1385, 2645, 238, 685, 1258, + 1955, 2270, 2707, 2890, 2947, 1512, 2583, 302, 940, 2830, 2948, 1892, 2333, + 1131, 2769, 366, 2949, 749, 1639, 2521, 2891, 430, 2950, 1829, 2396, 1004, + 2831, 1322, 2708, 494, 1449, 2646, 2951, 813, 2892, 1195, 1766, 2459, 2770, + 1576, 2584, 2145, 558, 2082, 2208, 2952, 2019, 2271, 1068, 2832, 877, 2893, + 1956, 2334, 622, 1703, 2522, 2953, 1386, 2709, 47, 3008, 111, 1259, 1513, + 1893, 2397, 2647, 2771, 3009, 175, 3010, 686, 2954, 239, 3011, 941, 2894, + 303, 1132, 1640, 2585, 2833, 3012, 1830, 2460, 367, 3013, 750, 2955, 431, + 2146, 2209, 3014, 1450, 2710, 1323, 2083, 2272, 2772, 1005, 1767, 2523, 2895, + 1577, 2020, 2335, 2648, 495, 3015, 814, 1196, 2834, 2956, 1957, 2398, 559, + 3016, 1704, 2586, 1069, 2896, 878, 1894, 2461, 2957, 623, 1387, 2773, 3017, + 1514, 2711, 1260, 2835, 48, 3072, 112, 1831, 2524, 3073, 1641, 2649, 176, + 3074, 687, 3018, 942, 2210, 2958, 240, 3075, 1133, 2147, 2273, 2897, 304, + 2084, 2336, 3076, 368, 1768, 2587, 3077, 751, 2021, 2399, 3019, 1451, 2774, + 1324, 2836, 432, 1578, 2712, 3078, 1006, 2959, 1958, 2462, 1197, 2898, 496, + 815, 3020, 3079, 1705, 2650, 1895, 2525, 560, 3080, 1070, 2960, 1388, 2837, + 879, 1515, 2775, 3021, 2211, 2274, 1832, 2588, 624, 2148, 2337, 3081, 1261, + 2899, 1642, 2713, 2085, 2400, 49, 3136, 113, 3137, 688, 3082, 177, 943, + 1134, 2022, 2463, 2961, 3022, 3138, 241, 1769, 2651, 3139, 305, 3140, 1452, + 2838, 1959, 2526, 752, 1325, 1579, 2776, 2900, 3083, 369, 3141, 1007, 3023, + 433, 3142, 1198, 1706, 2714, 2962, 1896, 2589, 816, 3084, 497, 2275, 3143, + 2212, 2338, 2149, 2401, 561, 1071, 1516, 1833, 2652, 2839, 3024, 3144, 1389, + 2901, 2086, 2464, 880, 3085, 1643, 2777, 1262, 2963, 625, 2023, 2527, 3145, + 1770, 2715, 1135, 3025, 50, 944, 1960, 2590, 3086, 3200, 114, 689, 3146, + 3201, 178, 3202, 242, 1453, 2902, 3203, 1580, 2840, 306, 1326, 2964, 3204, + 2276, 2339, 753, 1897, 2653, 3147, 370, 1707, 2213, 2402, 2778, 3205, 1008, + 3087, 1199, 2150, 2465, 3026, 434, 3206, 817, 2087, 2528, 3148, 1834, 2716, + 498, 3207, 1517, 2903, 1390, 2965, 1072, 3088, 1644, 2024, 2591, 2841, 562, + 3208, 881, 1263, 3027, 3149, 1771, 2779, 626, 1961, 2654, 3209, 2340, 1136, + 3089, 2277, 2403, 945, 3150, 690, 1454, 2214, 2466, 2966, 3210, 51, 1581, + 2904, 3264, 115, 3265, 179, 1898, 2717, 3266, 1327, 3028, 243, 2151, 2529, + 3267, 1708, 2842, 307, 3268, 754, 3211, 2088, 2592, 371, 1009, 3151, 3269, + 1200, 3090, 1835, 2780, 435, 3270, 2025, 2655, 818, 3212, 1518, 2967, 499, + 1391, 1645, 2905, 3029, 3271, 1073, 3152, 1962, 2718, 563, 1264, 1772, 2341, + 2404, 2843, 3091, 3272, 882, 2278, 2467, 3213, 2215, 2530, 627, 3273, 2152, + 2593, 1137, 1899, 2781, 3153, 1582, 2968, 1455, 3030, 946, 3214, 691, 1709, + 2906, 3274, 52, 1328, 3092, 3328, 116, 2089, 2656, 3329, 180, 3330, 244, + 3331, 308, 1836, 2844, 3332, 755, 3275, 1010, 1201, 2026, 2719, 3154, 3215, + 372, 3333, 1519, 2405, 3031, 436, 2342, 2468, 3334, 1646, 2969, 819, 1392, + 3093, 3276, 2279, 2531, 1963, 2782, 500, 3335, 1773, 2907, 1074, 2216, 2594, + 3216, 1265, 3155, 564, 3336, 883, 2153, 2657, 3277, 1900, 2845, 628, 1583, + 3032, 3337, 1456, 2090, 2720, 3094, 1138, 3217, 1710, 2970, 947, 3278, 1329, + 3156, 692, 3338, 53, 1837, 2908, 3392, 117, 2027, 2783, 3393, 181, 2406, + 2469, 3394, 2343, 2532, 245, 3395, 1202, 3218, 309, 756, 2280, 2595, 3339, + 3396, 1011, 3279, 1520, 3095, 373, 1647, 3033, 3397, 1964, 2846, 2217, 2658, + 1393, 3157, 437, 1774, 2971, 3398, 820, 3340, 2154, 2721, 1075, 3280, 501, + 3399, 1266, 3219, 1901, 2909, 565, 884, 2091, 2784, 3341, 3400, 1584, 3096, + 1457, 1711, 3034, 3158, 2470, 629, 1139, 2407, 2533, 3281, 3401, 2344, 2596, + 2028, 2847, 948, 1330, 1838, 2972, 3220, 3342, 2281, 2659, 693, 3402, 54, + 3456, 118, 3457, 182, 2218, 2722, 3458, 246, 1203, 1965, 2910, 3282, 3459, + 1012, 1648, 3097, 3343, 757, 1521, 3159, 3403, 310, 3460, 1775, 2155, 2785, + 3035, 374, 1394, 3221, 3461, 438, 3462, 821, 3404, 1902, 2973, 1076, 2092, + 2848, 3344, 1267, 3283, 502, 2471, 2534, 3463, 2408, 2597, 1585, 2345, 2660, + 3160, 885, 3405, 566, 1712, 3098, 3464, 1458, 3222, 2029, 2911, 2282, 2723, + 1140, 1839, 3036, 3345, 630, 3465, 1331, 3284, 949, 2219, 2786, 3406, 694, + 1966, 2974, 3466, 55, 2156, 2849, 3520, 119, 1649, 3161, 3521, 1204, 3346, + 183, 1522, 3223, 3522, 1776, 3099, 247, 1013, 3407, 3523, 758, 3467, 311, + 3524, 1395, 2535, 3285, 2472, 2598, 2093, 2912, 375, 1903, 2409, 2661, 3037, + 3525, 822, 2346, 2724, 3468, 439, 3526, 1077, 1268, 3347, 3408, 503, 2283, + 2787, 3527, 1586, 3224, 1713, 2030, 2975, 3162, 886, 1459, 3286, 3469, 1840, + 3100, 567, 3528, 2220, 2850, 1141, 3409, 1332, 3348, 631, 3529, 1967, 3038, + 950, 3470, 2157, 2913, 2536, 2599, 695, 1650, 2473, 2662, 3225, 3530, 1523, + 1777, 3163, 3287, 1205, 2410, 2725, 3410, 56, 3584, 120, 3585, 184, 2094, + 2976, 3586, 1014, 3471, 248, 1396, 1904, 2347, 2788, 3101, 3349, 3587, 759, + 3531, 312, 3588, 376, 2284, 2851, 3589, 823, 3532, 1269, 2031, 3039, 3411, + 440, 1078, 3472, 3590, 1714, 3226, 1587, 3288, 2221, 2914, 504, 1841, 3164, + 3591, 1460, 3350, 887, 3533, 568, 2600, 3592, 2537, 2663, 1968, 3102, 1142, + 2158, 2977, 3473, 2474, 2726, 1333, 3412, 632, 3593, 2411, 2789, 951, 3534, + 1651, 3289, 1778, 3227, 2348, 2852, 1524, 2095, 3040, 3351, 696, 3594, 1206, + 3474, 1905, 3165, 57, 3648, 121, 1015, 1397, 2285, 2915, 3413, 3535, 3649, + 185, 3650, 760, 3595, 249, 3651, 313, 2032, 3103, 3652, 2222, 2978, 377, + 3653, 1270, 1715, 3290, 3475, 824, 1588, 3352, 3596, 1079, 2601, 2664, 3536, + 1842, 3228, 441, 2538, 2727, 3654, 1461, 2475, 2790, 3414, 505, 2159, 3041, + 3655, 1969, 3166, 888, 2412, 2853, 3597, 569, 3656, 1143, 3537, 1334, 3476, + 2349, 2916, 2096, 3104, 1652, 3353, 633, 1779, 3291, 3657, 952, 3598, 1525, + 3415, 1906, 2286, 2979, 3229, 697, 1207, 3538, 3658, 1398, 3477, 1016, 3599, + 2033, 2665, 3167, 58, 2602, 2728, 3712, 122, 2223, 3042, 3713, 186, 3714, + 761, 2539, 2791, 3659, 250, 3715, 314, 1716, 2476, 2854, 3354, 3716, 1589, + 1843, 3292, 3416, 1271, 3539, 378, 3717, 1080, 3600, 825, 2160, 3105, 3660, + 2413, 2917, 442, 1462, 1970, 3230, 3478, 3718, 2350, 2980, 506, 3719, 889, + 3661, 1144, 1335, 2097, 3168, 3540, 3601, 570, 3720, 1780, 3355, 1653, 2287, + 3043, 3417, 1907, 3293, 634, 953, 1526, 2666, 2729, 3479, 3662, 3721, 2603, + 2792, 2540, 2855, 1208, 2224, 3106, 3602, 2034, 3231, 698, 3722, 1399, 3541, + 2477, 2918, 1017, 3663, 59, 3776, 123, 3777, 187, 762, 1717, 2414, 2981, + 3418, 3723, 3778, 1844, 3356, 251, 2161, 3169, 3779, 1590, 3480, 315, 1272, + 3603, 3780, 1971, 3294, 1081, 2351, 3044, 3664, 379, 3781, 826, 3724, 1463, + 3542, 443, 3782, 2098, 3232, 2730, 2288, 3107, 507, 2667, 2793, 3783, 890, + 3725, 1336, 2604, 2856, 3604, 1145, 1781, 3419, 3665, 1654, 3481, 571, 1908, + 3357, 3784, 2541, 2919, 1527, 3543, 2225, 3170, 954, 2478, 2982, 3726, 635, + 2035, 3295, 3785, 1209, 3666, 1400, 3605, 2415, 3045, 699, 3786, 1018, 2162, + 3233, 3727, 1718, 3482, 1845, 3420, 60, 2352, 3108, 3840, 124, 1591, 3544, + 3841, 763, 3787, 188, 1972, 3358, 3842, 252, 3843, 1273, 3667, 2731, 2794, + 316, 3844, 2668, 2857, 1082, 1464, 3606, 3728, 380, 827, 2099, 2605, 2920, + 3296, 3788, 3845, 2289, 3171, 444, 3846, 2542, 2983, 1782, 3483, 508, 1337, + 3668, 3847, 891, 1655, 1909, 3421, 3545, 3789, 1146, 2479, 3046, 3729, 2226, + 3234, 572, 3848, 1528, 2036, 3359, 3607, 2416, 3109, 955, 3790, 636, 3849, + 1210, 3730, 1401, 2163, 3297, 3669, 2353, 3172, 2795, 700, 1846, 2732, 2858, + 3484, 3850, 1719, 3546, 1019, 2669, 2921, 3791, 1973, 3422, 1592, 3608, 2606, + 2984, 61, 764, 3851, 3904, 125, 3905, 189, 1274, 2290, 3235, 3731, 3906, + 2100, 3360, 253, 2543, 3047, 3907, 1465, 3670, 317, 1083, 3792, 3908, 828, + 3852, 381, 3909, 2480, 3110, 1783, 3547, 445, 1910, 2227, 3298, 3485, 3910, + 1656, 3609, 1338, 3732, 892, 3853, 509, 1147, 2037, 2417, 3173, 3423, 3793, + 3911, 1529, 3671, 573, 2796, 2859, 3912, 2733, 2922, 2164, 3361, 956, 2354, + 3236, 3854, 2670, 2985, 637, 3913, 1211, 1402, 3733, 3794, 1847, 2607, 3048, + 3548, 1720, 3610, 1974, 3486, 701, 3914, 1020, 1593, 2544, 3111, 3672, 3855, + 2291, 3299, 2101, 3424, 765, 1275, 3795, 3915, 62, 3968, 126, 2481, 3174, + 3969, 190, 1466, 3734, 3970, 254, 3971, 1084, 3856, 318, 2228, 3362, 3972, + 829, 1784, 3611, 3916, 1911, 3549, 382, 2418, 3237, 3973, 2860, 1657, 2797, + 2923, 3673, 2038, 3487, 446, 2734, 2986, 3974, 1339, 3796, 1148, 3857, 893, + 2671, 3049, 3917, 510, 1530, 3735, 3975, 2355, 3300, 2165, 3425, 2608, 3112, + 574, 3976, 957, 3918, 1848, 3612, 1403, 2545, 3175, 3797, 1212, 3858, 638, + 1721, 1975, 3550, 3674, 3977, 2292, 3363, 1594, 2102, 3488, 3736, 702, 2482, + 3238, 3978, 1021, 3919, 1276, 2861, 2924, 3859, 766, 1467, 2229, 2798, 2987, + 3426, 3798, 3979, 63, 4032, 127, 2419, 3301, 4033, 191, 2735, 3050, 4034, + 1085, 1912, 3613, 3920, 255, 1785, 3675, 4035, 319, 2672, 3113, 4036, 2039, + 3551, 830, 3980, 1658, 3737, 383, 4037, 1340, 2356, 3364, 3860, 2609, 3176, + 447, 2166, 3489, 4038, 1149, 1531, 3799, 3921, 894, 3981, 511, 4039, 2546, + 3239, 575, 1849, 3676, 4040, 2293, 3427, 1976, 3614, 958, 1722, 3738, 3982, + 1404, 3861, 1213, 2483, 3302, 3922, 2103, 3552, 639, 2925, 4041, 2862, 2988, + 1595, 3800, 2799, 3051, 2736, 3114, 703, 1022, 3983, 4042, 2230, 3490, 2420, + 3365, 1277, 2673, 3177, 3923, 1468, 3862, 767, 1913, 3677, 4043, 1786, 3739, + 2040, 3615, 1086, 2610, 3240, 3984, 2357, 3428, 1659, 3801, 831, 4044, 2167, + 3553, 1341, 3924, 2547, 3303, 1532, 3863, 1150, 3985, 895, 4045, 2294, 2926, + 2989, 3491, 2863, 3052, 1850, 2484, 3366, 3740, 1977, 3678, 2800, 3115, 1723, + 3802, 2104, 3616, 1405, 3925, 959, 2737, 3178, 4046, 1214, 3986, 1596, 3864, + 2421, 3429, 2231, 2674, 3241, 3554, 1023, 4047, 2611, 3304, 1278, 1469, 1914, + 3741, 3926, 3987, 1787, 2041, 3679, 3803, 2358, 3492, 1087, 1660, 2168, 2548, + 3367, 3617, 3865, 4048, 2990, 2927, 3053, 2864, 3116, 1342, 3988, 1533, 2295, + 2801, 3179, 3555, 3927, 2485, 3430, 1151, 4049, 1978, 2738, 3242, 3742, 1851, + 3804, 2105, 3680, 1724, 3866, 2675, 3305, 1406, 2422, 3493, 3989, 2232, 3618, + 1215, 4050, 1597, 3928, 2612, 3368, 2359, 3556, 1915, 3805, 2042, 2991, 3054, + 3743, 1470, 3990, 1788, 2928, 3117, 3867, 1279, 2549, 3431, 4051, 2865, 3180, + 2169, 3681, 1661, 3929, 2802, 3243, 2486, 3494, 2296, 3619, 2739, 3306, 1343, + 4052, 1534, 3991, 1979, 3806, 1852, 3868, 2676, 3369, 2106, 3744, 2423, 3557, + 1725, 3930, 2233, 3682, 2613, 3432, 1407, 4053, 3055, 1598, 2992, 3118, 3992, + 2929, 3181, 2360, 3620, 2866, 3244, 2550, 3495, 1916, 3869, 2043, 3807, 1789, + 2803, 3307, 3931, 1471, 2170, 3745, 4054, 2740, 3370, 1662, 2487, 3558, 3993, + 2297, 3683, 2677, 3433, 1535, 4055, 1980, 3870, 1853, 2107, 2424, 3621, 3808, + 3932, 3056, 3119, 2614, 3496, 2993, 3182, 1726, 2234, 3746, 3994, 2930, 3245, + 2867, 3308, 1599, 2361, 3684, 4056, 2551, 3559, 2804, 3371, 2044, 3871, 1917, + 3933, 2171, 3809, 1790, 2741, 3434, 3995, 2488, 3622, 2298, 3747, 1663, 4057, + 2678, 3497, 3120, 3057, 3183, 2994, 3246, 2425, 3685, 1981, 3934, 2108, 3872, + 2615, 3560, 2931, 3309, 1854, 3996, 2235, 3810, 2868, 3372, 1727, 4058, 2552, + 3623, 2805, 3435, 2362, 3748, 2742, 3498, 2045, 3935, 1918, 3997, 2172, 3873, + 2489, 3686, 1791, 4059, 3121, 3184, 2299, 2679, 3561, 3811, 3058, 3247, 2995, + 3310, 2932, 3373, 2426, 3749, 2616, 3624, 1982, 3998, 2109, 2869, 3436, 3936, + 1855, 4060, 2236, 3874, 2806, 3499, 2553, 3687, 2363, 3812, 2743, 3562, 3185, + 3122, 3248, 2046, 3999, 2490, 3750, 1919, 2173, 3059, 3311, 3937, 4061, 2680, + 3625, 2996, 3374, 2300, 3875, 2933, 3437, 2617, 3688, 2427, 3813, 2870, 3500, + 2110, 4000, 1983, 4062, 2807, 3563, 2237, 3938, 2554, 3751, 2364, 3876, 2744, + 3626, 3186, 3249, 3123, 3312, 3060, 3375, 2491, 2997, 3438, 3814, 2047, 2681, + 3689, 4063, 2174, 4001, 2934, 3501, 2301, 3939, 2871, 3564, 2618, 3752, 2428, + 3877, 2808, 3627, 2111, 4064, 2238, 3250, 4002, 2555, 3187, 3313, 3815, 3124, + 3376, 2745, 3690, 2365, 3940, 3061, 3439, 2998, 3502, 2492, 3878, 2682, 3753, + 2935, 3565, 2175, 4065, 2302, 4003, 2872, 3628, 2619, 3816, 2429, 3941, 2809, + 3691, 3251, 3314, 3188, 3377, 3125, 3440, 2556, 3879, 2239, 3062, 3503, 4066, + 2746, 3754, 2366, 4004, 2999, 3566, 2936, 3629, 2683, 3817, 2493, 3942, 2873, + 3692, 2303, 4067, 2620, 3880, 3315, 3252, 3378, 3189, 3441, 2430, 2810, 3755, + 4005, 3126, 3504, 3063, 3567, 2557, 3943, 2747, 3818, 3000, 3630, 2367, 4068, + 2937, 3693, 2684, 3881, 2494, 4006, 2874, 3756, 3316, 3379, 3253, 3442, 3190, + 3505, 2621, 3944, 3127, 3568, 2811, 3819, 2431, 4069, 3064, 3631, 2748, 3882, + 2558, 3001, 3694, 4007, 2938, 3757, 2685, 3945, 3380, 3317, 3443, 2495, 4070, + 3254, 3506, 2875, 3820, 3191, 3569, 3128, 3632, 2622, 4008, 2812, 3883, 3065, + 3695, 3002, 3758, 2749, 3946, 2559, 4071, 2939, 3821, 3381, 3444, 3318, 3507, + 2686, 3255, 3570, 4009, 2876, 3884, 3192, 3633, 3129, 3696, 2623, 4072, 2813, + 3947, 3066, 3759, 3003, 3822, 2750, 4010, 3445, 3382, 3508, 2940, 3885, 3319, + 3571, 3256, 3634, 2687, 3193, 3697, 4073, 2877, 3948, 3130, 3760, 3067, 3823, + 2814, 4011, 3004, 3886, 3446, 3509, 3383, 3572, 2751, 4074, 3320, 3635, 2941, + 3949, 3257, 3698, 3194, 3761, 2878, 4012, 3131, 3824, 3068, 3887, 2815, 4075, + 3510, 3447, 3573, 3005, 3950, 3384, 3636, 3321, 3699, 3258, 3762, 2942, 4013, + 3195, 3825, 3132, 3888, 2879, 4076, 3069, 3951, 3511, 3574, 3448, 3637, 3385, + 3700, 3006, 4014, 3322, 3763, 3259, 3826, 2943, 4077, 3196, 3889, 3133, 3952, + 3575, 3512, 3638, 3070, 4015, 3449, 3701, 3386, 3764, 3323, 3827, 3007, 4078, + 3260, 3890, 3197, 3953, 3134, 4016, 3576, 3639, 3513, 3702, 3450, 3765, 3071, + 4079, 3387, 3828, 3324, 3891, 3261, 3954, 3198, 4017, 3640, 3135, 4080, 3577, + 3703, 3514, 3766, 3451, 3829, 3388, 3892, 3325, 3955, 3262, 4018, 3199, 4081, + 3641, 3704, 3578, 3767, 3515, 3830, 3452, 3893, 3389, 3956, 3326, 4019, 3263, + 4082, 3705, 3642, 3768, 3579, 3831, 3516, 3894, 3453, 3957, 3390, 4020, 3327, + 4083, 3706, 3769, 3643, 3832, 3580, 3895, 3517, 3958, 3454, 4021, 3391, 4084, + 3770, 3707, 3833, 3644, 3896, 3581, 3959, 3518, 4022, 3455, 4085, 3771, 3834, + 3708, 3897, 3645, 3960, 3582, 4023, 3519, 4086, 3835, 3772, 3898, 3709, 3961, + 3646, 4024, 3583, 4087, 3836, 3899, 3773, 3962, 3710, 4025, 3647, 4088, 3900, + 3837, 3963, 3774, 4026, 3711, 4089, 3901, 3964, 3838, 4027, 3775, 4090, 3965, + 3902, 4028, 3839, 4091, 3966, 4029, 3903, 4092, 4030, 3967, 4093, 4031, 4094, + 4095, +}; +#endif // CONFIG_TX64X64 + +#if CONFIG_CB4X4 +DECLARE_ALIGNED(16, static const int16_t, + default_scan_2x2_neighbors[5 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 1, 1, 2, 0, 0, +}; +#endif + +// Neighborhood 2-tuples for various scans and blocksizes, +// in {top, left} order for each position in corresponding scan order. +DECLARE_ALIGNED(16, static const int16_t, + default_scan_4x4_neighbors[17 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 4, 0, 1, 4, 4, 5, 5, 1, 8, 8, 5, 8, 2, + 2, 2, 5, 9, 12, 6, 9, 3, 6, 10, 13, 7, 10, 11, 14, 0, 0, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_4x4_neighbors[17 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 4, 4, 8, 8, 0, 0, 1, 4, 5, 8, 9, 12, 1, + 1, 2, 5, 6, 9, 10, 13, 2, 2, 3, 6, 7, 10, 11, 14, 0, 0, +}; + +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_4x4_neighbors[17 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 0, 0, 1, 4, 2, 5, 3, 6, 4, + 4, 5, 8, 6, 9, 7, 10, 8, 8, 9, 12, 10, 13, 11, 14, 0, 0, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, + col_scan_4x4_neighbors[17 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 4, 4, 4, 0, 8, 8, 1, 4, 5, 8, 5, 1, 9, + 12, 2, 5, 6, 9, 6, 2, 3, 6, 10, 13, 7, 10, 11, 14, 0, 0, +}; + +DECLARE_ALIGNED(16, static const int16_t, + row_scan_4x4_neighbors[17 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 1, 1, 1, 1, 4, 2, 2, 2, 5, 4, 5, 5, + 8, 3, 6, 8, 9, 6, 9, 9, 12, 7, 10, 10, 13, 11, 14, 0, 0, +}; + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_4x8_neighbors[33 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 4, 1, 1, 4, 4, 2, 5, 5, 8, 6, + 9, 2, 2, 8, 8, 3, 6, 9, 12, 7, 10, 10, 13, 12, 12, 13, 16, + 11, 14, 14, 17, 15, 18, 16, 16, 17, 20, 18, 21, 19, 22, 20, 20, 21, + 24, 22, 25, 23, 26, 24, 24, 25, 28, 26, 29, 27, 30, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_4x8_neighbors[33 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 4, 4, 8, 8, 12, 12, 16, 16, 20, 20, 24, 24, 0, + 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 1, 1, + 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 2, 2, 3, + 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_4x8_neighbors[33 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 0, 0, 1, 4, 2, 5, 3, 6, 4, + 4, 5, 8, 6, 9, 7, 10, 8, 8, 9, 12, 10, 13, 11, 14, 12, 12, + 13, 16, 14, 17, 15, 18, 16, 16, 17, 20, 18, 21, 19, 22, 20, 20, 21, + 24, 22, 25, 23, 26, 24, 24, 25, 28, 26, 29, 27, 30, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_8x4_neighbors[33 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 8, 1, 1, 8, 8, 2, 9, 9, 16, 10, + 17, 2, 2, 16, 16, 3, 10, 17, 24, 11, 18, 18, 25, 3, 3, 4, 11, + 19, 26, 12, 19, 4, 4, 20, 27, 5, 12, 13, 20, 21, 28, 5, 5, 6, + 13, 14, 21, 22, 29, 6, 6, 7, 14, 15, 22, 23, 30, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_8x4_neighbors[33 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 8, 8, 16, 16, 0, 0, 1, 8, 9, 16, 17, 24, 1, + 1, 2, 9, 10, 17, 18, 25, 2, 2, 3, 10, 11, 18, 19, 26, 3, 3, + 4, 11, 12, 19, 20, 27, 4, 4, 5, 12, 13, 20, 21, 28, 5, 5, 6, + 13, 14, 21, 22, 29, 6, 6, 7, 14, 15, 22, 23, 30, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_8x4_neighbors[33 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 0, + 0, 1, 8, 2, 9, 3, 10, 4, 11, 5, 12, 6, 13, 7, 14, 8, 8, + 9, 16, 10, 17, 11, 18, 12, 19, 13, 20, 14, 21, 15, 22, 16, 16, 17, + 24, 18, 25, 19, 26, 20, 27, 21, 28, 22, 29, 23, 30, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_4x16_neighbors[65 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 1, 1, 4, 4, 4, 2, 2, 2, 5, 5, 8, 8, + 8, 3, 6, 6, 9, 9, 12, 12, 12, 7, 10, 10, 13, 13, 16, 16, 16, 11, 14, + 14, 17, 17, 20, 20, 20, 15, 18, 18, 21, 21, 24, 24, 24, 19, 22, 22, 25, 25, + 28, 28, 28, 23, 26, 26, 29, 29, 32, 32, 32, 27, 30, 30, 33, 33, 36, 36, 36, + 31, 34, 34, 37, 37, 40, 40, 40, 35, 38, 38, 41, 41, 44, 44, 44, 39, 42, 42, + 45, 45, 48, 48, 48, 43, 46, 46, 49, 49, 52, 52, 52, 47, 50, 50, 53, 53, 56, + 56, 56, 51, 54, 54, 57, 57, 60, 55, 58, 58, 61, 59, 62, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_16x4_neighbors[65 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 1, 1, 16, 16, 16, 2, 2, 2, 17, 17, 32, 32, + 32, 3, 3, 3, 18, 18, 33, 33, 48, 4, 4, 4, 19, 19, 34, 34, 49, 5, 5, + 5, 20, 20, 35, 35, 50, 6, 6, 6, 21, 21, 36, 36, 51, 7, 7, 7, 22, 22, + 37, 37, 52, 8, 8, 8, 23, 23, 38, 38, 53, 9, 9, 9, 24, 24, 39, 39, 54, + 10, 10, 10, 25, 25, 40, 40, 55, 11, 11, 11, 26, 26, 41, 41, 56, 12, 12, 12, + 27, 27, 42, 42, 57, 13, 13, 13, 28, 28, 43, 43, 58, 14, 14, 14, 29, 29, 44, + 44, 59, 15, 30, 30, 45, 45, 60, 31, 46, 46, 61, 47, 62, 0, 0 +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_4x16_neighbors[65 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 1, 1, 16, 16, 16, 2, 2, 2, 17, 17, 32, 32, + 32, 3, 3, 3, 18, 18, 33, 33, 48, 4, 4, 4, 19, 19, 34, 34, 49, 5, 5, + 5, 20, 20, 35, 35, 50, 6, 6, 6, 21, 21, 36, 36, 51, 7, 7, 7, 22, 22, + 37, 37, 52, 8, 8, 8, 23, 23, 38, 38, 53, 9, 9, 9, 24, 24, 39, 39, 54, + 10, 10, 10, 25, 25, 40, 40, 55, 11, 11, 11, 26, 26, 41, 41, 56, 12, 12, 12, + 27, 27, 42, 42, 57, 13, 13, 13, 28, 28, 43, 43, 58, 14, 14, 14, 29, 29, 44, + 44, 59, 15, 30, 30, 45, 45, 60, 31, 46, 46, 61, 47, 62, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_16x4_neighbors[65 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 0, 0, 1, 16, 2, 17, + 3, 18, 4, 19, 5, 20, 6, 21, 7, 22, 8, 23, 9, 24, 10, 25, 11, 26, 12, + 27, 13, 28, 14, 29, 15, 30, 16, 16, 17, 32, 18, 33, 19, 34, 20, 35, 21, 36, + 22, 37, 23, 38, 24, 39, 25, 40, 26, 41, 27, 42, 28, 43, 29, 44, 30, 45, 31, + 46, 32, 32, 33, 48, 34, 49, 35, 50, 36, 51, 37, 52, 38, 53, 39, 54, 40, 55, + 41, 56, 42, 57, 43, 58, 44, 59, 45, 60, 46, 61, 47, 62, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_4x16_neighbors[65 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 4, 4, 8, 8, 12, 12, 16, 16, 20, 20, 24, 24, 28, 28, 32, + 32, 36, 36, 40, 40, 44, 44, 48, 48, 52, 52, 56, 56, 0, 0, 1, 4, 5, 8, + 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29, 32, 33, 36, 37, 40, 41, 44, 45, + 48, 49, 52, 53, 56, 57, 60, 1, 1, 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, + 22, 25, 26, 29, 30, 33, 34, 37, 38, 41, 42, 45, 46, 49, 50, 53, 54, 57, 58, + 61, 2, 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31, 34, + 35, 38, 39, 42, 43, 46, 47, 50, 51, 54, 55, 58, 59, 62, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_16x4_neighbors[65 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 16, 16, 32, 32, 0, 0, 1, 16, 17, 32, 33, 48, 1, 1, 2, + 17, 18, 33, 34, 49, 2, 2, 3, 18, 19, 34, 35, 50, 3, 3, 4, 19, 20, 35, + 36, 51, 4, 4, 5, 20, 21, 36, 37, 52, 5, 5, 6, 21, 22, 37, 38, 53, 6, + 6, 7, 22, 23, 38, 39, 54, 7, 7, 8, 23, 24, 39, 40, 55, 8, 8, 9, 24, + 25, 40, 41, 56, 9, 9, 10, 25, 26, 41, 42, 57, 10, 10, 11, 26, 27, 42, 43, + 58, 11, 11, 12, 27, 28, 43, 44, 59, 12, 12, 13, 28, 29, 44, 45, 60, 13, 13, + 14, 29, 30, 45, 46, 61, 14, 14, 15, 30, 31, 46, 47, 62, 0, 0 +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_8x32_neighbors[257 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 1, 1, 8, 8, 8, 2, 2, 2, + 9, 9, 16, 16, 16, 3, 3, 3, 10, 10, 17, 17, 24, 24, 24, + 4, 4, 4, 11, 11, 18, 18, 25, 25, 32, 32, 32, 5, 5, 5, + 12, 12, 19, 19, 26, 26, 33, 33, 40, 40, 40, 6, 6, 6, 13, + 13, 20, 20, 27, 27, 34, 34, 41, 41, 48, 48, 48, 7, 14, 14, + 21, 21, 28, 28, 35, 35, 42, 42, 49, 49, 56, 56, 56, 15, 22, + 22, 29, 29, 36, 36, 43, 43, 50, 50, 57, 57, 64, 64, 64, 23, + 30, 30, 37, 37, 44, 44, 51, 51, 58, 58, 65, 65, 72, 72, 72, + 31, 38, 38, 45, 45, 52, 52, 59, 59, 66, 66, 73, 73, 80, 80, + 80, 39, 46, 46, 53, 53, 60, 60, 67, 67, 74, 74, 81, 81, 88, + 88, 88, 47, 54, 54, 61, 61, 68, 68, 75, 75, 82, 82, 89, 89, + 96, 96, 96, 55, 62, 62, 69, 69, 76, 76, 83, 83, 90, 90, 97, + 97, 104, 104, 104, 63, 70, 70, 77, 77, 84, 84, 91, 91, 98, 98, + 105, 105, 112, 112, 112, 71, 78, 78, 85, 85, 92, 92, 99, 99, 106, + 106, 113, 113, 120, 120, 120, 79, 86, 86, 93, 93, 100, 100, 107, 107, + 114, 114, 121, 121, 128, 128, 128, 87, 94, 94, 101, 101, 108, 108, 115, + 115, 122, 122, 129, 129, 136, 136, 136, 95, 102, 102, 109, 109, 116, 116, + 123, 123, 130, 130, 137, 137, 144, 144, 144, 103, 110, 110, 117, 117, 124, + 124, 131, 131, 138, 138, 145, 145, 152, 152, 152, 111, 118, 118, 125, 125, + 132, 132, 139, 139, 146, 146, 153, 153, 160, 160, 160, 119, 126, 126, 133, + 133, 140, 140, 147, 147, 154, 154, 161, 161, 168, 168, 168, 127, 134, 134, + 141, 141, 148, 148, 155, 155, 162, 162, 169, 169, 176, 176, 176, 135, 142, + 142, 149, 149, 156, 156, 163, 163, 170, 170, 177, 177, 184, 184, 184, 143, + 150, 150, 157, 157, 164, 164, 171, 171, 178, 178, 185, 185, 192, 192, 192, + 151, 158, 158, 165, 165, 172, 172, 179, 179, 186, 186, 193, 193, 200, 200, + 200, 159, 166, 166, 173, 173, 180, 180, 187, 187, 194, 194, 201, 201, 208, + 208, 208, 167, 174, 174, 181, 181, 188, 188, 195, 195, 202, 202, 209, 209, + 216, 216, 216, 175, 182, 182, 189, 189, 196, 196, 203, 203, 210, 210, 217, + 217, 224, 224, 224, 183, 190, 190, 197, 197, 204, 204, 211, 211, 218, 218, + 225, 225, 232, 232, 232, 191, 198, 198, 205, 205, 212, 212, 219, 219, 226, + 226, 233, 233, 240, 240, 240, 199, 206, 206, 213, 213, 220, 220, 227, 227, + 234, 234, 241, 241, 248, 207, 214, 214, 221, 221, 228, 228, 235, 235, 242, + 242, 249, 215, 222, 222, 229, 229, 236, 236, 243, 243, 250, 223, 230, 230, + 237, 237, 244, 244, 251, 231, 238, 238, 245, 245, 252, 239, 246, 246, 253, + 247, 254, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_32x8_neighbors[257 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 1, 1, 32, 32, 32, 2, 2, 2, + 33, 33, 64, 64, 64, 3, 3, 3, 34, 34, 65, 65, 96, 96, 96, + 4, 4, 4, 35, 35, 66, 66, 97, 97, 128, 128, 128, 5, 5, 5, + 36, 36, 67, 67, 98, 98, 129, 129, 160, 160, 160, 6, 6, 6, 37, + 37, 68, 68, 99, 99, 130, 130, 161, 161, 192, 192, 192, 7, 7, 7, + 38, 38, 69, 69, 100, 100, 131, 131, 162, 162, 193, 193, 224, 8, 8, + 8, 39, 39, 70, 70, 101, 101, 132, 132, 163, 163, 194, 194, 225, 9, + 9, 9, 40, 40, 71, 71, 102, 102, 133, 133, 164, 164, 195, 195, 226, + 10, 10, 10, 41, 41, 72, 72, 103, 103, 134, 134, 165, 165, 196, 196, + 227, 11, 11, 11, 42, 42, 73, 73, 104, 104, 135, 135, 166, 166, 197, + 197, 228, 12, 12, 12, 43, 43, 74, 74, 105, 105, 136, 136, 167, 167, + 198, 198, 229, 13, 13, 13, 44, 44, 75, 75, 106, 106, 137, 137, 168, + 168, 199, 199, 230, 14, 14, 14, 45, 45, 76, 76, 107, 107, 138, 138, + 169, 169, 200, 200, 231, 15, 15, 15, 46, 46, 77, 77, 108, 108, 139, + 139, 170, 170, 201, 201, 232, 16, 16, 16, 47, 47, 78, 78, 109, 109, + 140, 140, 171, 171, 202, 202, 233, 17, 17, 17, 48, 48, 79, 79, 110, + 110, 141, 141, 172, 172, 203, 203, 234, 18, 18, 18, 49, 49, 80, 80, + 111, 111, 142, 142, 173, 173, 204, 204, 235, 19, 19, 19, 50, 50, 81, + 81, 112, 112, 143, 143, 174, 174, 205, 205, 236, 20, 20, 20, 51, 51, + 82, 82, 113, 113, 144, 144, 175, 175, 206, 206, 237, 21, 21, 21, 52, + 52, 83, 83, 114, 114, 145, 145, 176, 176, 207, 207, 238, 22, 22, 22, + 53, 53, 84, 84, 115, 115, 146, 146, 177, 177, 208, 208, 239, 23, 23, + 23, 54, 54, 85, 85, 116, 116, 147, 147, 178, 178, 209, 209, 240, 24, + 24, 24, 55, 55, 86, 86, 117, 117, 148, 148, 179, 179, 210, 210, 241, + 25, 25, 25, 56, 56, 87, 87, 118, 118, 149, 149, 180, 180, 211, 211, + 242, 26, 26, 26, 57, 57, 88, 88, 119, 119, 150, 150, 181, 181, 212, + 212, 243, 27, 27, 27, 58, 58, 89, 89, 120, 120, 151, 151, 182, 182, + 213, 213, 244, 28, 28, 28, 59, 59, 90, 90, 121, 121, 152, 152, 183, + 183, 214, 214, 245, 29, 29, 29, 60, 60, 91, 91, 122, 122, 153, 153, + 184, 184, 215, 215, 246, 30, 30, 30, 61, 61, 92, 92, 123, 123, 154, + 154, 185, 185, 216, 216, 247, 31, 62, 62, 93, 93, 124, 124, 155, 155, + 186, 186, 217, 217, 248, 63, 94, 94, 125, 125, 156, 156, 187, 187, 218, + 218, 249, 95, 126, 126, 157, 157, 188, 188, 219, 219, 250, 127, 158, 158, + 189, 189, 220, 220, 251, 159, 190, 190, 221, 221, 252, 191, 222, 222, 253, + 223, 254, 0, 0 +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_8x32_neighbors[257 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, + 6, 0, 0, 1, 8, 2, 9, 3, 10, 4, 11, 5, 12, 6, 13, + 7, 14, 8, 8, 9, 16, 10, 17, 11, 18, 12, 19, 13, 20, 14, + 21, 15, 22, 16, 16, 17, 24, 18, 25, 19, 26, 20, 27, 21, 28, + 22, 29, 23, 30, 24, 24, 25, 32, 26, 33, 27, 34, 28, 35, 29, + 36, 30, 37, 31, 38, 32, 32, 33, 40, 34, 41, 35, 42, 36, 43, + 37, 44, 38, 45, 39, 46, 40, 40, 41, 48, 42, 49, 43, 50, 44, + 51, 45, 52, 46, 53, 47, 54, 48, 48, 49, 56, 50, 57, 51, 58, + 52, 59, 53, 60, 54, 61, 55, 62, 56, 56, 57, 64, 58, 65, 59, + 66, 60, 67, 61, 68, 62, 69, 63, 70, 64, 64, 65, 72, 66, 73, + 67, 74, 68, 75, 69, 76, 70, 77, 71, 78, 72, 72, 73, 80, 74, + 81, 75, 82, 76, 83, 77, 84, 78, 85, 79, 86, 80, 80, 81, 88, + 82, 89, 83, 90, 84, 91, 85, 92, 86, 93, 87, 94, 88, 88, 89, + 96, 90, 97, 91, 98, 92, 99, 93, 100, 94, 101, 95, 102, 96, 96, + 97, 104, 98, 105, 99, 106, 100, 107, 101, 108, 102, 109, 103, 110, 104, + 104, 105, 112, 106, 113, 107, 114, 108, 115, 109, 116, 110, 117, 111, 118, + 112, 112, 113, 120, 114, 121, 115, 122, 116, 123, 117, 124, 118, 125, 119, + 126, 120, 120, 121, 128, 122, 129, 123, 130, 124, 131, 125, 132, 126, 133, + 127, 134, 128, 128, 129, 136, 130, 137, 131, 138, 132, 139, 133, 140, 134, + 141, 135, 142, 136, 136, 137, 144, 138, 145, 139, 146, 140, 147, 141, 148, + 142, 149, 143, 150, 144, 144, 145, 152, 146, 153, 147, 154, 148, 155, 149, + 156, 150, 157, 151, 158, 152, 152, 153, 160, 154, 161, 155, 162, 156, 163, + 157, 164, 158, 165, 159, 166, 160, 160, 161, 168, 162, 169, 163, 170, 164, + 171, 165, 172, 166, 173, 167, 174, 168, 168, 169, 176, 170, 177, 171, 178, + 172, 179, 173, 180, 174, 181, 175, 182, 176, 176, 177, 184, 178, 185, 179, + 186, 180, 187, 181, 188, 182, 189, 183, 190, 184, 184, 185, 192, 186, 193, + 187, 194, 188, 195, 189, 196, 190, 197, 191, 198, 192, 192, 193, 200, 194, + 201, 195, 202, 196, 203, 197, 204, 198, 205, 199, 206, 200, 200, 201, 208, + 202, 209, 203, 210, 204, 211, 205, 212, 206, 213, 207, 214, 208, 208, 209, + 216, 210, 217, 211, 218, 212, 219, 213, 220, 214, 221, 215, 222, 216, 216, + 217, 224, 218, 225, 219, 226, 220, 227, 221, 228, 222, 229, 223, 230, 224, + 224, 225, 232, 226, 233, 227, 234, 228, 235, 229, 236, 230, 237, 231, 238, + 232, 232, 233, 240, 234, 241, 235, 242, 236, 243, 237, 244, 238, 245, 239, + 246, 240, 240, 241, 248, 242, 249, 243, 250, 244, 251, 245, 252, 246, 253, + 247, 254, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_32x8_neighbors[257 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, + 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, + 21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, + 29, 29, 30, 30, 0, 0, 1, 32, 2, 33, 3, 34, 4, 35, 5, + 36, 6, 37, 7, 38, 8, 39, 9, 40, 10, 41, 11, 42, 12, 43, + 13, 44, 14, 45, 15, 46, 16, 47, 17, 48, 18, 49, 19, 50, 20, + 51, 21, 52, 22, 53, 23, 54, 24, 55, 25, 56, 26, 57, 27, 58, + 28, 59, 29, 60, 30, 61, 31, 62, 32, 32, 33, 64, 34, 65, 35, + 66, 36, 67, 37, 68, 38, 69, 39, 70, 40, 71, 41, 72, 42, 73, + 43, 74, 44, 75, 45, 76, 46, 77, 47, 78, 48, 79, 49, 80, 50, + 81, 51, 82, 52, 83, 53, 84, 54, 85, 55, 86, 56, 87, 57, 88, + 58, 89, 59, 90, 60, 91, 61, 92, 62, 93, 63, 94, 64, 64, 65, + 96, 66, 97, 67, 98, 68, 99, 69, 100, 70, 101, 71, 102, 72, 103, + 73, 104, 74, 105, 75, 106, 76, 107, 77, 108, 78, 109, 79, 110, 80, + 111, 81, 112, 82, 113, 83, 114, 84, 115, 85, 116, 86, 117, 87, 118, + 88, 119, 89, 120, 90, 121, 91, 122, 92, 123, 93, 124, 94, 125, 95, + 126, 96, 96, 97, 128, 98, 129, 99, 130, 100, 131, 101, 132, 102, 133, + 103, 134, 104, 135, 105, 136, 106, 137, 107, 138, 108, 139, 109, 140, 110, + 141, 111, 142, 112, 143, 113, 144, 114, 145, 115, 146, 116, 147, 117, 148, + 118, 149, 119, 150, 120, 151, 121, 152, 122, 153, 123, 154, 124, 155, 125, + 156, 126, 157, 127, 158, 128, 128, 129, 160, 130, 161, 131, 162, 132, 163, + 133, 164, 134, 165, 135, 166, 136, 167, 137, 168, 138, 169, 139, 170, 140, + 171, 141, 172, 142, 173, 143, 174, 144, 175, 145, 176, 146, 177, 147, 178, + 148, 179, 149, 180, 150, 181, 151, 182, 152, 183, 153, 184, 154, 185, 155, + 186, 156, 187, 157, 188, 158, 189, 159, 190, 160, 160, 161, 192, 162, 193, + 163, 194, 164, 195, 165, 196, 166, 197, 167, 198, 168, 199, 169, 200, 170, + 201, 171, 202, 172, 203, 173, 204, 174, 205, 175, 206, 176, 207, 177, 208, + 178, 209, 179, 210, 180, 211, 181, 212, 182, 213, 183, 214, 184, 215, 185, + 216, 186, 217, 187, 218, 188, 219, 189, 220, 190, 221, 191, 222, 192, 192, + 193, 224, 194, 225, 195, 226, 196, 227, 197, 228, 198, 229, 199, 230, 200, + 231, 201, 232, 202, 233, 203, 234, 204, 235, 205, 236, 206, 237, 207, 238, + 208, 239, 209, 240, 210, 241, 211, 242, 212, 243, 213, 244, 214, 245, 215, + 246, 216, 247, 217, 248, 218, 249, 219, 250, 220, 251, 221, 252, 222, 253, + 223, 254, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_8x32_neighbors[257 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 8, 8, 16, 16, 24, 24, 32, 32, 40, 40, 48, + 48, 56, 56, 64, 64, 72, 72, 80, 80, 88, 88, 96, 96, 104, 104, + 112, 112, 120, 120, 128, 128, 136, 136, 144, 144, 152, 152, 160, 160, 168, + 168, 176, 176, 184, 184, 192, 192, 200, 200, 208, 208, 216, 216, 224, 224, + 232, 232, 240, 240, 0, 0, 1, 8, 9, 16, 17, 24, 25, 32, 33, + 40, 41, 48, 49, 56, 57, 64, 65, 72, 73, 80, 81, 88, 89, 96, + 97, 104, 105, 112, 113, 120, 121, 128, 129, 136, 137, 144, 145, 152, 153, + 160, 161, 168, 169, 176, 177, 184, 185, 192, 193, 200, 201, 208, 209, 216, + 217, 224, 225, 232, 233, 240, 241, 248, 1, 1, 2, 9, 10, 17, 18, + 25, 26, 33, 34, 41, 42, 49, 50, 57, 58, 65, 66, 73, 74, 81, + 82, 89, 90, 97, 98, 105, 106, 113, 114, 121, 122, 129, 130, 137, 138, + 145, 146, 153, 154, 161, 162, 169, 170, 177, 178, 185, 186, 193, 194, 201, + 202, 209, 210, 217, 218, 225, 226, 233, 234, 241, 242, 249, 2, 2, 3, + 10, 11, 18, 19, 26, 27, 34, 35, 42, 43, 50, 51, 58, 59, 66, + 67, 74, 75, 82, 83, 90, 91, 98, 99, 106, 107, 114, 115, 122, 123, + 130, 131, 138, 139, 146, 147, 154, 155, 162, 163, 170, 171, 178, 179, 186, + 187, 194, 195, 202, 203, 210, 211, 218, 219, 226, 227, 234, 235, 242, 243, + 250, 3, 3, 4, 11, 12, 19, 20, 27, 28, 35, 36, 43, 44, 51, + 52, 59, 60, 67, 68, 75, 76, 83, 84, 91, 92, 99, 100, 107, 108, + 115, 116, 123, 124, 131, 132, 139, 140, 147, 148, 155, 156, 163, 164, 171, + 172, 179, 180, 187, 188, 195, 196, 203, 204, 211, 212, 219, 220, 227, 228, + 235, 236, 243, 244, 251, 4, 4, 5, 12, 13, 20, 21, 28, 29, 36, + 37, 44, 45, 52, 53, 60, 61, 68, 69, 76, 77, 84, 85, 92, 93, + 100, 101, 108, 109, 116, 117, 124, 125, 132, 133, 140, 141, 148, 149, 156, + 157, 164, 165, 172, 173, 180, 181, 188, 189, 196, 197, 204, 205, 212, 213, + 220, 221, 228, 229, 236, 237, 244, 245, 252, 5, 5, 6, 13, 14, 21, + 22, 29, 30, 37, 38, 45, 46, 53, 54, 61, 62, 69, 70, 77, 78, + 85, 86, 93, 94, 101, 102, 109, 110, 117, 118, 125, 126, 133, 134, 141, + 142, 149, 150, 157, 158, 165, 166, 173, 174, 181, 182, 189, 190, 197, 198, + 205, 206, 213, 214, 221, 222, 229, 230, 237, 238, 245, 246, 253, 6, 6, + 7, 14, 15, 22, 23, 30, 31, 38, 39, 46, 47, 54, 55, 62, 63, + 70, 71, 78, 79, 86, 87, 94, 95, 102, 103, 110, 111, 118, 119, 126, + 127, 134, 135, 142, 143, 150, 151, 158, 159, 166, 167, 174, 175, 182, 183, + 190, 191, 198, 199, 206, 207, 214, 215, 222, 223, 230, 231, 238, 239, 246, + 247, 254, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_32x8_neighbors[257 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 32, 32, 64, 64, 96, 96, 128, 128, 160, 160, 192, 192, + 0, 0, 1, 32, 33, 64, 65, 96, 97, 128, 129, 160, 161, 192, 193, 224, + 1, 1, 2, 33, 34, 65, 66, 97, 98, 129, 130, 161, 162, 193, 194, 225, + 2, 2, 3, 34, 35, 66, 67, 98, 99, 130, 131, 162, 163, 194, 195, 226, + 3, 3, 4, 35, 36, 67, 68, 99, 100, 131, 132, 163, 164, 195, 196, 227, + 4, 4, 5, 36, 37, 68, 69, 100, 101, 132, 133, 164, 165, 196, 197, 228, + 5, 5, 6, 37, 38, 69, 70, 101, 102, 133, 134, 165, 166, 197, 198, 229, + 6, 6, 7, 38, 39, 70, 71, 102, 103, 134, 135, 166, 167, 198, 199, 230, + 7, 7, 8, 39, 40, 71, 72, 103, 104, 135, 136, 167, 168, 199, 200, 231, + 8, 8, 9, 40, 41, 72, 73, 104, 105, 136, 137, 168, 169, 200, 201, 232, + 9, 9, 10, 41, 42, 73, 74, 105, 106, 137, 138, 169, 170, 201, 202, 233, + 10, 10, 11, 42, 43, 74, 75, 106, 107, 138, 139, 170, 171, 202, 203, 234, + 11, 11, 12, 43, 44, 75, 76, 107, 108, 139, 140, 171, 172, 203, 204, 235, + 12, 12, 13, 44, 45, 76, 77, 108, 109, 140, 141, 172, 173, 204, 205, 236, + 13, 13, 14, 45, 46, 77, 78, 109, 110, 141, 142, 173, 174, 205, 206, 237, + 14, 14, 15, 46, 47, 78, 79, 110, 111, 142, 143, 174, 175, 206, 207, 238, + 15, 15, 16, 47, 48, 79, 80, 111, 112, 143, 144, 175, 176, 207, 208, 239, + 16, 16, 17, 48, 49, 80, 81, 112, 113, 144, 145, 176, 177, 208, 209, 240, + 17, 17, 18, 49, 50, 81, 82, 113, 114, 145, 146, 177, 178, 209, 210, 241, + 18, 18, 19, 50, 51, 82, 83, 114, 115, 146, 147, 178, 179, 210, 211, 242, + 19, 19, 20, 51, 52, 83, 84, 115, 116, 147, 148, 179, 180, 211, 212, 243, + 20, 20, 21, 52, 53, 84, 85, 116, 117, 148, 149, 180, 181, 212, 213, 244, + 21, 21, 22, 53, 54, 85, 86, 117, 118, 149, 150, 181, 182, 213, 214, 245, + 22, 22, 23, 54, 55, 86, 87, 118, 119, 150, 151, 182, 183, 214, 215, 246, + 23, 23, 24, 55, 56, 87, 88, 119, 120, 151, 152, 183, 184, 215, 216, 247, + 24, 24, 25, 56, 57, 88, 89, 120, 121, 152, 153, 184, 185, 216, 217, 248, + 25, 25, 26, 57, 58, 89, 90, 121, 122, 153, 154, 185, 186, 217, 218, 249, + 26, 26, 27, 58, 59, 90, 91, 122, 123, 154, 155, 186, 187, 218, 219, 250, + 27, 27, 28, 59, 60, 91, 92, 123, 124, 155, 156, 187, 188, 219, 220, 251, + 28, 28, 29, 60, 61, 92, 93, 124, 125, 156, 157, 188, 189, 220, 221, 252, + 29, 29, 30, 61, 62, 93, 94, 125, 126, 157, 158, 189, 190, 221, 222, 253, + 30, 30, 31, 62, 63, 94, 95, 126, 127, 158, 159, 190, 191, 222, 223, 254, + 0, 0 +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, + col_scan_8x8_neighbors[65 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 8, 8, 8, 0, 16, 16, 1, 8, 24, 24, 9, 16, 9, 1, 32, + 32, 17, 24, 2, 9, 25, 32, 10, 17, 40, 40, 10, 2, 18, 25, 33, 40, 3, 10, + 48, 48, 11, 18, 26, 33, 11, 3, 41, 48, 19, 26, 34, 41, 4, 11, 27, 34, 12, + 19, 49, 56, 42, 49, 20, 27, 12, 4, 35, 42, 5, 12, 28, 35, 50, 57, 43, 50, + 13, 20, 36, 43, 13, 5, 21, 28, 51, 58, 29, 36, 6, 13, 44, 51, 14, 21, 14, + 6, 37, 44, 52, 59, 22, 29, 7, 14, 30, 37, 45, 52, 15, 22, 38, 45, 23, 30, + 53, 60, 31, 38, 46, 53, 39, 46, 54, 61, 47, 54, 55, 62, 0, 0, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_8x8_neighbors[65 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 8, 8, 16, 16, 24, 24, 32, 32, 40, 40, 48, 48, 0, 0, 1, + 8, 9, 16, 17, 24, 25, 32, 33, 40, 41, 48, 49, 56, 1, 1, 2, 9, 10, 17, + 18, 25, 26, 33, 34, 41, 42, 49, 50, 57, 2, 2, 3, 10, 11, 18, 19, 26, 27, + 34, 35, 42, 43, 50, 51, 58, 3, 3, 4, 11, 12, 19, 20, 27, 28, 35, 36, 43, + 44, 51, 52, 59, 4, 4, 5, 12, 13, 20, 21, 28, 29, 36, 37, 44, 45, 52, 53, + 60, 5, 5, 6, 13, 14, 21, 22, 29, 30, 37, 38, 45, 46, 53, 54, 61, 6, 6, + 7, 14, 15, 22, 23, 30, 31, 38, 39, 46, 47, 54, 55, 62, 0, 0, +}; + +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_8x8_neighbors[65 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 0, 0, 1, + 8, 2, 9, 3, 10, 4, 11, 5, 12, 6, 13, 7, 14, 8, 8, 9, 16, 10, 17, + 11, 18, 12, 19, 13, 20, 14, 21, 15, 22, 16, 16, 17, 24, 18, 25, 19, 26, 20, + 27, 21, 28, 22, 29, 23, 30, 24, 24, 25, 32, 26, 33, 27, 34, 28, 35, 29, 36, + 30, 37, 31, 38, 32, 32, 33, 40, 34, 41, 35, 42, 36, 43, 37, 44, 38, 45, 39, + 46, 40, 40, 41, 48, 42, 49, 43, 50, 44, 51, 45, 52, 46, 53, 47, 54, 48, 48, + 49, 56, 50, 57, 51, 58, 52, 59, 53, 60, 54, 61, 55, 62, 0, 0, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, + row_scan_8x8_neighbors[65 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 0, 1, 1, 8, 2, 2, 8, 9, 2, 9, 3, 3, 9, + 16, 3, 10, 16, 17, 4, 4, 10, 17, 17, 24, 4, 11, 11, 18, 18, 25, 24, 25, + 5, 5, 5, 12, 12, 19, 25, 32, 19, 26, 6, 6, 26, 33, 32, 33, 13, 20, 20, + 27, 33, 40, 6, 13, 27, 34, 40, 41, 34, 41, 21, 28, 28, 35, 41, 48, 14, 21, + 35, 42, 7, 14, 48, 49, 29, 36, 42, 49, 36, 43, 22, 29, 49, 56, 15, 22, 43, + 50, 50, 57, 37, 44, 30, 37, 44, 51, 23, 30, 51, 58, 45, 52, 38, 45, 52, 59, + 31, 38, 53, 60, 39, 46, 46, 53, 47, 54, 54, 61, 55, 62, 0, 0, +}; + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_8x8_neighbors[65 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 8, 0, 8, 8, 1, 8, 9, 1, 9, 16, 16, 17, 2, 9, 10, + 2, 10, 17, 17, 24, 24, 25, 3, 10, 11, 3, 18, 25, 25, 32, 11, 18, 32, 33, + 4, 11, 26, 33, 19, 26, 12, 4, 33, 40, 12, 19, 40, 41, 5, 12, 27, 34, 34, + 41, 20, 27, 13, 20, 13, 5, 41, 48, 48, 49, 28, 35, 35, 42, 21, 28, 6, 6, + 6, 13, 42, 49, 49, 56, 36, 43, 14, 21, 29, 36, 7, 14, 43, 50, 50, 57, 22, + 29, 37, 44, 15, 22, 44, 51, 51, 58, 30, 37, 23, 30, 52, 59, 45, 52, 38, 45, + 31, 38, 53, 60, 46, 53, 39, 46, 54, 61, 47, 54, 55, 62, 0, 0, +}; + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_8x16_neighbors[129 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 1, 1, 8, 8, 8, 2, 2, 2, + 9, 9, 16, 16, 16, 3, 3, 3, 10, 10, 17, 17, 24, 24, 24, + 4, 4, 4, 11, 11, 18, 18, 25, 25, 32, 32, 32, 5, 5, 5, + 12, 12, 19, 19, 26, 26, 33, 33, 40, 40, 40, 6, 6, 6, 13, + 13, 20, 20, 27, 27, 34, 34, 41, 41, 48, 48, 48, 7, 14, 14, + 21, 21, 28, 28, 35, 35, 42, 42, 49, 49, 56, 56, 56, 15, 22, + 22, 29, 29, 36, 36, 43, 43, 50, 50, 57, 57, 64, 64, 64, 23, + 30, 30, 37, 37, 44, 44, 51, 51, 58, 58, 65, 65, 72, 72, 72, + 31, 38, 38, 45, 45, 52, 52, 59, 59, 66, 66, 73, 73, 80, 80, + 80, 39, 46, 46, 53, 53, 60, 60, 67, 67, 74, 74, 81, 81, 88, + 88, 88, 47, 54, 54, 61, 61, 68, 68, 75, 75, 82, 82, 89, 89, + 96, 96, 96, 55, 62, 62, 69, 69, 76, 76, 83, 83, 90, 90, 97, + 97, 104, 104, 104, 63, 70, 70, 77, 77, 84, 84, 91, 91, 98, 98, + 105, 105, 112, 112, 112, 71, 78, 78, 85, 85, 92, 92, 99, 99, 106, + 106, 113, 113, 120, 79, 86, 86, 93, 93, 100, 100, 107, 107, 114, 114, + 121, 87, 94, 94, 101, 101, 108, 108, 115, 115, 122, 95, 102, 102, 109, + 109, 116, 116, 123, 103, 110, 110, 117, 117, 124, 111, 118, 118, 125, 119, + 126, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_16x8_neighbors[129 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 1, 1, 16, 16, 16, 2, 2, 2, + 17, 17, 32, 32, 32, 3, 3, 3, 18, 18, 33, 33, 48, 48, 48, + 4, 4, 4, 19, 19, 34, 34, 49, 49, 64, 64, 64, 5, 5, 5, + 20, 20, 35, 35, 50, 50, 65, 65, 80, 80, 80, 6, 6, 6, 21, + 21, 36, 36, 51, 51, 66, 66, 81, 81, 96, 96, 96, 7, 7, 7, + 22, 22, 37, 37, 52, 52, 67, 67, 82, 82, 97, 97, 112, 8, 8, + 8, 23, 23, 38, 38, 53, 53, 68, 68, 83, 83, 98, 98, 113, 9, + 9, 9, 24, 24, 39, 39, 54, 54, 69, 69, 84, 84, 99, 99, 114, + 10, 10, 10, 25, 25, 40, 40, 55, 55, 70, 70, 85, 85, 100, 100, + 115, 11, 11, 11, 26, 26, 41, 41, 56, 56, 71, 71, 86, 86, 101, + 101, 116, 12, 12, 12, 27, 27, 42, 42, 57, 57, 72, 72, 87, 87, + 102, 102, 117, 13, 13, 13, 28, 28, 43, 43, 58, 58, 73, 73, 88, + 88, 103, 103, 118, 14, 14, 14, 29, 29, 44, 44, 59, 59, 74, 74, + 89, 89, 104, 104, 119, 15, 30, 30, 45, 45, 60, 60, 75, 75, 90, + 90, 105, 105, 120, 31, 46, 46, 61, 61, 76, 76, 91, 91, 106, 106, + 121, 47, 62, 62, 77, 77, 92, 92, 107, 107, 122, 63, 78, 78, 93, + 93, 108, 108, 123, 79, 94, 94, 109, 109, 124, 95, 110, 110, 125, 111, + 126, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_8x16_neighbors[129 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 8, 8, 16, 16, 24, 24, 32, 32, 40, 40, 48, 48, + 56, 56, 64, 64, 72, 72, 80, 80, 88, 88, 96, 96, 104, 104, 112, 112, + 0, 0, 1, 8, 9, 16, 17, 24, 25, 32, 33, 40, 41, 48, 49, 56, + 57, 64, 65, 72, 73, 80, 81, 88, 89, 96, 97, 104, 105, 112, 113, 120, + 1, 1, 2, 9, 10, 17, 18, 25, 26, 33, 34, 41, 42, 49, 50, 57, + 58, 65, 66, 73, 74, 81, 82, 89, 90, 97, 98, 105, 106, 113, 114, 121, + 2, 2, 3, 10, 11, 18, 19, 26, 27, 34, 35, 42, 43, 50, 51, 58, + 59, 66, 67, 74, 75, 82, 83, 90, 91, 98, 99, 106, 107, 114, 115, 122, + 3, 3, 4, 11, 12, 19, 20, 27, 28, 35, 36, 43, 44, 51, 52, 59, + 60, 67, 68, 75, 76, 83, 84, 91, 92, 99, 100, 107, 108, 115, 116, 123, + 4, 4, 5, 12, 13, 20, 21, 28, 29, 36, 37, 44, 45, 52, 53, 60, + 61, 68, 69, 76, 77, 84, 85, 92, 93, 100, 101, 108, 109, 116, 117, 124, + 5, 5, 6, 13, 14, 21, 22, 29, 30, 37, 38, 45, 46, 53, 54, 61, + 62, 69, 70, 77, 78, 85, 86, 93, 94, 101, 102, 109, 110, 117, 118, 125, + 6, 6, 7, 14, 15, 22, 23, 30, 31, 38, 39, 46, 47, 54, 55, 62, + 63, 70, 71, 78, 79, 86, 87, 94, 95, 102, 103, 110, 111, 118, 119, 126, + 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_16x8_neighbors[129 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 16, 16, 32, 32, 48, 48, 64, 64, 80, 80, 96, 96, + 0, 0, 1, 16, 17, 32, 33, 48, 49, 64, 65, 80, 81, 96, 97, 112, + 1, 1, 2, 17, 18, 33, 34, 49, 50, 65, 66, 81, 82, 97, 98, 113, + 2, 2, 3, 18, 19, 34, 35, 50, 51, 66, 67, 82, 83, 98, 99, 114, + 3, 3, 4, 19, 20, 35, 36, 51, 52, 67, 68, 83, 84, 99, 100, 115, + 4, 4, 5, 20, 21, 36, 37, 52, 53, 68, 69, 84, 85, 100, 101, 116, + 5, 5, 6, 21, 22, 37, 38, 53, 54, 69, 70, 85, 86, 101, 102, 117, + 6, 6, 7, 22, 23, 38, 39, 54, 55, 70, 71, 86, 87, 102, 103, 118, + 7, 7, 8, 23, 24, 39, 40, 55, 56, 71, 72, 87, 88, 103, 104, 119, + 8, 8, 9, 24, 25, 40, 41, 56, 57, 72, 73, 88, 89, 104, 105, 120, + 9, 9, 10, 25, 26, 41, 42, 57, 58, 73, 74, 89, 90, 105, 106, 121, + 10, 10, 11, 26, 27, 42, 43, 58, 59, 74, 75, 90, 91, 106, 107, 122, + 11, 11, 12, 27, 28, 43, 44, 59, 60, 75, 76, 91, 92, 107, 108, 123, + 12, 12, 13, 28, 29, 44, 45, 60, 61, 76, 77, 92, 93, 108, 109, 124, + 13, 13, 14, 29, 30, 45, 46, 61, 62, 77, 78, 93, 94, 109, 110, 125, + 14, 14, 15, 30, 31, 46, 47, 62, 63, 78, 79, 94, 95, 110, 111, 126, + 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_8x16_neighbors[129 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, + 6, 0, 0, 1, 8, 2, 9, 3, 10, 4, 11, 5, 12, 6, 13, + 7, 14, 8, 8, 9, 16, 10, 17, 11, 18, 12, 19, 13, 20, 14, + 21, 15, 22, 16, 16, 17, 24, 18, 25, 19, 26, 20, 27, 21, 28, + 22, 29, 23, 30, 24, 24, 25, 32, 26, 33, 27, 34, 28, 35, 29, + 36, 30, 37, 31, 38, 32, 32, 33, 40, 34, 41, 35, 42, 36, 43, + 37, 44, 38, 45, 39, 46, 40, 40, 41, 48, 42, 49, 43, 50, 44, + 51, 45, 52, 46, 53, 47, 54, 48, 48, 49, 56, 50, 57, 51, 58, + 52, 59, 53, 60, 54, 61, 55, 62, 56, 56, 57, 64, 58, 65, 59, + 66, 60, 67, 61, 68, 62, 69, 63, 70, 64, 64, 65, 72, 66, 73, + 67, 74, 68, 75, 69, 76, 70, 77, 71, 78, 72, 72, 73, 80, 74, + 81, 75, 82, 76, 83, 77, 84, 78, 85, 79, 86, 80, 80, 81, 88, + 82, 89, 83, 90, 84, 91, 85, 92, 86, 93, 87, 94, 88, 88, 89, + 96, 90, 97, 91, 98, 92, 99, 93, 100, 94, 101, 95, 102, 96, 96, + 97, 104, 98, 105, 99, 106, 100, 107, 101, 108, 102, 109, 103, 110, 104, + 104, 105, 112, 106, 113, 107, 114, 108, 115, 109, 116, 110, 117, 111, 118, + 112, 112, 113, 120, 114, 121, 115, 122, 116, 123, 117, 124, 118, 125, 119, + 126, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_16x8_neighbors[129 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, + 14, 14, 0, 0, 1, 16, 2, 17, 3, 18, 4, 19, 5, 20, 6, + 21, 7, 22, 8, 23, 9, 24, 10, 25, 11, 26, 12, 27, 13, 28, + 14, 29, 15, 30, 16, 16, 17, 32, 18, 33, 19, 34, 20, 35, 21, + 36, 22, 37, 23, 38, 24, 39, 25, 40, 26, 41, 27, 42, 28, 43, + 29, 44, 30, 45, 31, 46, 32, 32, 33, 48, 34, 49, 35, 50, 36, + 51, 37, 52, 38, 53, 39, 54, 40, 55, 41, 56, 42, 57, 43, 58, + 44, 59, 45, 60, 46, 61, 47, 62, 48, 48, 49, 64, 50, 65, 51, + 66, 52, 67, 53, 68, 54, 69, 55, 70, 56, 71, 57, 72, 58, 73, + 59, 74, 60, 75, 61, 76, 62, 77, 63, 78, 64, 64, 65, 80, 66, + 81, 67, 82, 68, 83, 69, 84, 70, 85, 71, 86, 72, 87, 73, 88, + 74, 89, 75, 90, 76, 91, 77, 92, 78, 93, 79, 94, 80, 80, 81, + 96, 82, 97, 83, 98, 84, 99, 85, 100, 86, 101, 87, 102, 88, 103, + 89, 104, 90, 105, 91, 106, 92, 107, 93, 108, 94, 109, 95, 110, 96, + 96, 97, 112, 98, 113, 99, 114, 100, 115, 101, 116, 102, 117, 103, 118, + 104, 119, 105, 120, 106, 121, 107, 122, 108, 123, 109, 124, 110, 125, 111, + 126, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_16x32_neighbors[513 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 1, 1, 16, 16, 16, 2, 2, 2, + 17, 17, 32, 32, 32, 3, 3, 3, 18, 18, 33, 33, 48, 48, 48, + 4, 4, 4, 19, 19, 34, 34, 49, 49, 64, 64, 64, 5, 5, 5, + 20, 20, 35, 35, 50, 50, 65, 65, 80, 80, 80, 6, 6, 6, 21, + 21, 36, 36, 51, 51, 66, 66, 81, 81, 96, 96, 96, 7, 7, 7, + 22, 22, 37, 37, 52, 52, 67, 67, 82, 82, 97, 97, 112, 112, 112, + 8, 8, 8, 23, 23, 38, 38, 53, 53, 68, 68, 83, 83, 98, 98, + 113, 113, 128, 128, 128, 9, 9, 9, 24, 24, 39, 39, 54, 54, 69, + 69, 84, 84, 99, 99, 114, 114, 129, 129, 144, 144, 144, 10, 10, 10, + 25, 25, 40, 40, 55, 55, 70, 70, 85, 85, 100, 100, 115, 115, 130, + 130, 145, 145, 160, 160, 160, 11, 11, 11, 26, 26, 41, 41, 56, 56, + 71, 71, 86, 86, 101, 101, 116, 116, 131, 131, 146, 146, 161, 161, 176, + 176, 176, 12, 12, 12, 27, 27, 42, 42, 57, 57, 72, 72, 87, 87, + 102, 102, 117, 117, 132, 132, 147, 147, 162, 162, 177, 177, 192, 192, 192, + 13, 13, 13, 28, 28, 43, 43, 58, 58, 73, 73, 88, 88, 103, 103, + 118, 118, 133, 133, 148, 148, 163, 163, 178, 178, 193, 193, 208, 208, 208, + 14, 14, 14, 29, 29, 44, 44, 59, 59, 74, 74, 89, 89, 104, 104, + 119, 119, 134, 134, 149, 149, 164, 164, 179, 179, 194, 194, 209, 209, 224, + 224, 224, 15, 30, 30, 45, 45, 60, 60, 75, 75, 90, 90, 105, 105, + 120, 120, 135, 135, 150, 150, 165, 165, 180, 180, 195, 195, 210, 210, 225, + 225, 240, 240, 240, 31, 46, 46, 61, 61, 76, 76, 91, 91, 106, 106, + 121, 121, 136, 136, 151, 151, 166, 166, 181, 181, 196, 196, 211, 211, 226, + 226, 241, 241, 256, 256, 256, 47, 62, 62, 77, 77, 92, 92, 107, 107, + 122, 122, 137, 137, 152, 152, 167, 167, 182, 182, 197, 197, 212, 212, 227, + 227, 242, 242, 257, 257, 272, 272, 272, 63, 78, 78, 93, 93, 108, 108, + 123, 123, 138, 138, 153, 153, 168, 168, 183, 183, 198, 198, 213, 213, 228, + 228, 243, 243, 258, 258, 273, 273, 288, 288, 288, 79, 94, 94, 109, 109, + 124, 124, 139, 139, 154, 154, 169, 169, 184, 184, 199, 199, 214, 214, 229, + 229, 244, 244, 259, 259, 274, 274, 289, 289, 304, 304, 304, 95, 110, 110, + 125, 125, 140, 140, 155, 155, 170, 170, 185, 185, 200, 200, 215, 215, 230, + 230, 245, 245, 260, 260, 275, 275, 290, 290, 305, 305, 320, 320, 320, 111, + 126, 126, 141, 141, 156, 156, 171, 171, 186, 186, 201, 201, 216, 216, 231, + 231, 246, 246, 261, 261, 276, 276, 291, 291, 306, 306, 321, 321, 336, 336, + 336, 127, 142, 142, 157, 157, 172, 172, 187, 187, 202, 202, 217, 217, 232, + 232, 247, 247, 262, 262, 277, 277, 292, 292, 307, 307, 322, 322, 337, 337, + 352, 352, 352, 143, 158, 158, 173, 173, 188, 188, 203, 203, 218, 218, 233, + 233, 248, 248, 263, 263, 278, 278, 293, 293, 308, 308, 323, 323, 338, 338, + 353, 353, 368, 368, 368, 159, 174, 174, 189, 189, 204, 204, 219, 219, 234, + 234, 249, 249, 264, 264, 279, 279, 294, 294, 309, 309, 324, 324, 339, 339, + 354, 354, 369, 369, 384, 384, 384, 175, 190, 190, 205, 205, 220, 220, 235, + 235, 250, 250, 265, 265, 280, 280, 295, 295, 310, 310, 325, 325, 340, 340, + 355, 355, 370, 370, 385, 385, 400, 400, 400, 191, 206, 206, 221, 221, 236, + 236, 251, 251, 266, 266, 281, 281, 296, 296, 311, 311, 326, 326, 341, 341, + 356, 356, 371, 371, 386, 386, 401, 401, 416, 416, 416, 207, 222, 222, 237, + 237, 252, 252, 267, 267, 282, 282, 297, 297, 312, 312, 327, 327, 342, 342, + 357, 357, 372, 372, 387, 387, 402, 402, 417, 417, 432, 432, 432, 223, 238, + 238, 253, 253, 268, 268, 283, 283, 298, 298, 313, 313, 328, 328, 343, 343, + 358, 358, 373, 373, 388, 388, 403, 403, 418, 418, 433, 433, 448, 448, 448, + 239, 254, 254, 269, 269, 284, 284, 299, 299, 314, 314, 329, 329, 344, 344, + 359, 359, 374, 374, 389, 389, 404, 404, 419, 419, 434, 434, 449, 449, 464, + 464, 464, 255, 270, 270, 285, 285, 300, 300, 315, 315, 330, 330, 345, 345, + 360, 360, 375, 375, 390, 390, 405, 405, 420, 420, 435, 435, 450, 450, 465, + 465, 480, 480, 480, 271, 286, 286, 301, 301, 316, 316, 331, 331, 346, 346, + 361, 361, 376, 376, 391, 391, 406, 406, 421, 421, 436, 436, 451, 451, 466, + 466, 481, 481, 496, 287, 302, 302, 317, 317, 332, 332, 347, 347, 362, 362, + 377, 377, 392, 392, 407, 407, 422, 422, 437, 437, 452, 452, 467, 467, 482, + 482, 497, 303, 318, 318, 333, 333, 348, 348, 363, 363, 378, 378, 393, 393, + 408, 408, 423, 423, 438, 438, 453, 453, 468, 468, 483, 483, 498, 319, 334, + 334, 349, 349, 364, 364, 379, 379, 394, 394, 409, 409, 424, 424, 439, 439, + 454, 454, 469, 469, 484, 484, 499, 335, 350, 350, 365, 365, 380, 380, 395, + 395, 410, 410, 425, 425, 440, 440, 455, 455, 470, 470, 485, 485, 500, 351, + 366, 366, 381, 381, 396, 396, 411, 411, 426, 426, 441, 441, 456, 456, 471, + 471, 486, 486, 501, 367, 382, 382, 397, 397, 412, 412, 427, 427, 442, 442, + 457, 457, 472, 472, 487, 487, 502, 383, 398, 398, 413, 413, 428, 428, 443, + 443, 458, 458, 473, 473, 488, 488, 503, 399, 414, 414, 429, 429, 444, 444, + 459, 459, 474, 474, 489, 489, 504, 415, 430, 430, 445, 445, 460, 460, 475, + 475, 490, 490, 505, 431, 446, 446, 461, 461, 476, 476, 491, 491, 506, 447, + 462, 462, 477, 477, 492, 492, 507, 463, 478, 478, 493, 493, 508, 479, 494, + 494, 509, 495, 510, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_32x16_neighbors[513 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 1, 1, 32, 32, 32, 2, 2, 2, + 33, 33, 64, 64, 64, 3, 3, 3, 34, 34, 65, 65, 96, 96, 96, + 4, 4, 4, 35, 35, 66, 66, 97, 97, 128, 128, 128, 5, 5, 5, + 36, 36, 67, 67, 98, 98, 129, 129, 160, 160, 160, 6, 6, 6, 37, + 37, 68, 68, 99, 99, 130, 130, 161, 161, 192, 192, 192, 7, 7, 7, + 38, 38, 69, 69, 100, 100, 131, 131, 162, 162, 193, 193, 224, 224, 224, + 8, 8, 8, 39, 39, 70, 70, 101, 101, 132, 132, 163, 163, 194, 194, + 225, 225, 256, 256, 256, 9, 9, 9, 40, 40, 71, 71, 102, 102, 133, + 133, 164, 164, 195, 195, 226, 226, 257, 257, 288, 288, 288, 10, 10, 10, + 41, 41, 72, 72, 103, 103, 134, 134, 165, 165, 196, 196, 227, 227, 258, + 258, 289, 289, 320, 320, 320, 11, 11, 11, 42, 42, 73, 73, 104, 104, + 135, 135, 166, 166, 197, 197, 228, 228, 259, 259, 290, 290, 321, 321, 352, + 352, 352, 12, 12, 12, 43, 43, 74, 74, 105, 105, 136, 136, 167, 167, + 198, 198, 229, 229, 260, 260, 291, 291, 322, 322, 353, 353, 384, 384, 384, + 13, 13, 13, 44, 44, 75, 75, 106, 106, 137, 137, 168, 168, 199, 199, + 230, 230, 261, 261, 292, 292, 323, 323, 354, 354, 385, 385, 416, 416, 416, + 14, 14, 14, 45, 45, 76, 76, 107, 107, 138, 138, 169, 169, 200, 200, + 231, 231, 262, 262, 293, 293, 324, 324, 355, 355, 386, 386, 417, 417, 448, + 448, 448, 15, 15, 15, 46, 46, 77, 77, 108, 108, 139, 139, 170, 170, + 201, 201, 232, 232, 263, 263, 294, 294, 325, 325, 356, 356, 387, 387, 418, + 418, 449, 449, 480, 16, 16, 16, 47, 47, 78, 78, 109, 109, 140, 140, + 171, 171, 202, 202, 233, 233, 264, 264, 295, 295, 326, 326, 357, 357, 388, + 388, 419, 419, 450, 450, 481, 17, 17, 17, 48, 48, 79, 79, 110, 110, + 141, 141, 172, 172, 203, 203, 234, 234, 265, 265, 296, 296, 327, 327, 358, + 358, 389, 389, 420, 420, 451, 451, 482, 18, 18, 18, 49, 49, 80, 80, + 111, 111, 142, 142, 173, 173, 204, 204, 235, 235, 266, 266, 297, 297, 328, + 328, 359, 359, 390, 390, 421, 421, 452, 452, 483, 19, 19, 19, 50, 50, + 81, 81, 112, 112, 143, 143, 174, 174, 205, 205, 236, 236, 267, 267, 298, + 298, 329, 329, 360, 360, 391, 391, 422, 422, 453, 453, 484, 20, 20, 20, + 51, 51, 82, 82, 113, 113, 144, 144, 175, 175, 206, 206, 237, 237, 268, + 268, 299, 299, 330, 330, 361, 361, 392, 392, 423, 423, 454, 454, 485, 21, + 21, 21, 52, 52, 83, 83, 114, 114, 145, 145, 176, 176, 207, 207, 238, + 238, 269, 269, 300, 300, 331, 331, 362, 362, 393, 393, 424, 424, 455, 455, + 486, 22, 22, 22, 53, 53, 84, 84, 115, 115, 146, 146, 177, 177, 208, + 208, 239, 239, 270, 270, 301, 301, 332, 332, 363, 363, 394, 394, 425, 425, + 456, 456, 487, 23, 23, 23, 54, 54, 85, 85, 116, 116, 147, 147, 178, + 178, 209, 209, 240, 240, 271, 271, 302, 302, 333, 333, 364, 364, 395, 395, + 426, 426, 457, 457, 488, 24, 24, 24, 55, 55, 86, 86, 117, 117, 148, + 148, 179, 179, 210, 210, 241, 241, 272, 272, 303, 303, 334, 334, 365, 365, + 396, 396, 427, 427, 458, 458, 489, 25, 25, 25, 56, 56, 87, 87, 118, + 118, 149, 149, 180, 180, 211, 211, 242, 242, 273, 273, 304, 304, 335, 335, + 366, 366, 397, 397, 428, 428, 459, 459, 490, 26, 26, 26, 57, 57, 88, + 88, 119, 119, 150, 150, 181, 181, 212, 212, 243, 243, 274, 274, 305, 305, + 336, 336, 367, 367, 398, 398, 429, 429, 460, 460, 491, 27, 27, 27, 58, + 58, 89, 89, 120, 120, 151, 151, 182, 182, 213, 213, 244, 244, 275, 275, + 306, 306, 337, 337, 368, 368, 399, 399, 430, 430, 461, 461, 492, 28, 28, + 28, 59, 59, 90, 90, 121, 121, 152, 152, 183, 183, 214, 214, 245, 245, + 276, 276, 307, 307, 338, 338, 369, 369, 400, 400, 431, 431, 462, 462, 493, + 29, 29, 29, 60, 60, 91, 91, 122, 122, 153, 153, 184, 184, 215, 215, + 246, 246, 277, 277, 308, 308, 339, 339, 370, 370, 401, 401, 432, 432, 463, + 463, 494, 30, 30, 30, 61, 61, 92, 92, 123, 123, 154, 154, 185, 185, + 216, 216, 247, 247, 278, 278, 309, 309, 340, 340, 371, 371, 402, 402, 433, + 433, 464, 464, 495, 31, 62, 62, 93, 93, 124, 124, 155, 155, 186, 186, + 217, 217, 248, 248, 279, 279, 310, 310, 341, 341, 372, 372, 403, 403, 434, + 434, 465, 465, 496, 63, 94, 94, 125, 125, 156, 156, 187, 187, 218, 218, + 249, 249, 280, 280, 311, 311, 342, 342, 373, 373, 404, 404, 435, 435, 466, + 466, 497, 95, 126, 126, 157, 157, 188, 188, 219, 219, 250, 250, 281, 281, + 312, 312, 343, 343, 374, 374, 405, 405, 436, 436, 467, 467, 498, 127, 158, + 158, 189, 189, 220, 220, 251, 251, 282, 282, 313, 313, 344, 344, 375, 375, + 406, 406, 437, 437, 468, 468, 499, 159, 190, 190, 221, 221, 252, 252, 283, + 283, 314, 314, 345, 345, 376, 376, 407, 407, 438, 438, 469, 469, 500, 191, + 222, 222, 253, 253, 284, 284, 315, 315, 346, 346, 377, 377, 408, 408, 439, + 439, 470, 470, 501, 223, 254, 254, 285, 285, 316, 316, 347, 347, 378, 378, + 409, 409, 440, 440, 471, 471, 502, 255, 286, 286, 317, 317, 348, 348, 379, + 379, 410, 410, 441, 441, 472, 472, 503, 287, 318, 318, 349, 349, 380, 380, + 411, 411, 442, 442, 473, 473, 504, 319, 350, 350, 381, 381, 412, 412, 443, + 443, 474, 474, 505, 351, 382, 382, 413, 413, 444, 444, 475, 475, 506, 383, + 414, 414, 445, 445, 476, 476, 507, 415, 446, 446, 477, 477, 508, 447, 478, + 478, 509, 479, 510, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_16x32_neighbors[513 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 16, 16, 32, 32, 48, 48, 64, 64, 80, 80, 96, + 96, 112, 112, 128, 128, 144, 144, 160, 160, 176, 176, 192, 192, 208, 208, + 224, 224, 240, 240, 256, 256, 272, 272, 288, 288, 304, 304, 320, 320, 336, + 336, 352, 352, 368, 368, 384, 384, 400, 400, 416, 416, 432, 432, 448, 448, + 464, 464, 480, 480, 0, 0, 1, 16, 17, 32, 33, 48, 49, 64, 65, + 80, 81, 96, 97, 112, 113, 128, 129, 144, 145, 160, 161, 176, 177, 192, + 193, 208, 209, 224, 225, 240, 241, 256, 257, 272, 273, 288, 289, 304, 305, + 320, 321, 336, 337, 352, 353, 368, 369, 384, 385, 400, 401, 416, 417, 432, + 433, 448, 449, 464, 465, 480, 481, 496, 1, 1, 2, 17, 18, 33, 34, + 49, 50, 65, 66, 81, 82, 97, 98, 113, 114, 129, 130, 145, 146, 161, + 162, 177, 178, 193, 194, 209, 210, 225, 226, 241, 242, 257, 258, 273, 274, + 289, 290, 305, 306, 321, 322, 337, 338, 353, 354, 369, 370, 385, 386, 401, + 402, 417, 418, 433, 434, 449, 450, 465, 466, 481, 482, 497, 2, 2, 3, + 18, 19, 34, 35, 50, 51, 66, 67, 82, 83, 98, 99, 114, 115, 130, + 131, 146, 147, 162, 163, 178, 179, 194, 195, 210, 211, 226, 227, 242, 243, + 258, 259, 274, 275, 290, 291, 306, 307, 322, 323, 338, 339, 354, 355, 370, + 371, 386, 387, 402, 403, 418, 419, 434, 435, 450, 451, 466, 467, 482, 483, + 498, 3, 3, 4, 19, 20, 35, 36, 51, 52, 67, 68, 83, 84, 99, + 100, 115, 116, 131, 132, 147, 148, 163, 164, 179, 180, 195, 196, 211, 212, + 227, 228, 243, 244, 259, 260, 275, 276, 291, 292, 307, 308, 323, 324, 339, + 340, 355, 356, 371, 372, 387, 388, 403, 404, 419, 420, 435, 436, 451, 452, + 467, 468, 483, 484, 499, 4, 4, 5, 20, 21, 36, 37, 52, 53, 68, + 69, 84, 85, 100, 101, 116, 117, 132, 133, 148, 149, 164, 165, 180, 181, + 196, 197, 212, 213, 228, 229, 244, 245, 260, 261, 276, 277, 292, 293, 308, + 309, 324, 325, 340, 341, 356, 357, 372, 373, 388, 389, 404, 405, 420, 421, + 436, 437, 452, 453, 468, 469, 484, 485, 500, 5, 5, 6, 21, 22, 37, + 38, 53, 54, 69, 70, 85, 86, 101, 102, 117, 118, 133, 134, 149, 150, + 165, 166, 181, 182, 197, 198, 213, 214, 229, 230, 245, 246, 261, 262, 277, + 278, 293, 294, 309, 310, 325, 326, 341, 342, 357, 358, 373, 374, 389, 390, + 405, 406, 421, 422, 437, 438, 453, 454, 469, 470, 485, 486, 501, 6, 6, + 7, 22, 23, 38, 39, 54, 55, 70, 71, 86, 87, 102, 103, 118, 119, + 134, 135, 150, 151, 166, 167, 182, 183, 198, 199, 214, 215, 230, 231, 246, + 247, 262, 263, 278, 279, 294, 295, 310, 311, 326, 327, 342, 343, 358, 359, + 374, 375, 390, 391, 406, 407, 422, 423, 438, 439, 454, 455, 470, 471, 486, + 487, 502, 7, 7, 8, 23, 24, 39, 40, 55, 56, 71, 72, 87, 88, + 103, 104, 119, 120, 135, 136, 151, 152, 167, 168, 183, 184, 199, 200, 215, + 216, 231, 232, 247, 248, 263, 264, 279, 280, 295, 296, 311, 312, 327, 328, + 343, 344, 359, 360, 375, 376, 391, 392, 407, 408, 423, 424, 439, 440, 455, + 456, 471, 472, 487, 488, 503, 8, 8, 9, 24, 25, 40, 41, 56, 57, + 72, 73, 88, 89, 104, 105, 120, 121, 136, 137, 152, 153, 168, 169, 184, + 185, 200, 201, 216, 217, 232, 233, 248, 249, 264, 265, 280, 281, 296, 297, + 312, 313, 328, 329, 344, 345, 360, 361, 376, 377, 392, 393, 408, 409, 424, + 425, 440, 441, 456, 457, 472, 473, 488, 489, 504, 9, 9, 10, 25, 26, + 41, 42, 57, 58, 73, 74, 89, 90, 105, 106, 121, 122, 137, 138, 153, + 154, 169, 170, 185, 186, 201, 202, 217, 218, 233, 234, 249, 250, 265, 266, + 281, 282, 297, 298, 313, 314, 329, 330, 345, 346, 361, 362, 377, 378, 393, + 394, 409, 410, 425, 426, 441, 442, 457, 458, 473, 474, 489, 490, 505, 10, + 10, 11, 26, 27, 42, 43, 58, 59, 74, 75, 90, 91, 106, 107, 122, + 123, 138, 139, 154, 155, 170, 171, 186, 187, 202, 203, 218, 219, 234, 235, + 250, 251, 266, 267, 282, 283, 298, 299, 314, 315, 330, 331, 346, 347, 362, + 363, 378, 379, 394, 395, 410, 411, 426, 427, 442, 443, 458, 459, 474, 475, + 490, 491, 506, 11, 11, 12, 27, 28, 43, 44, 59, 60, 75, 76, 91, + 92, 107, 108, 123, 124, 139, 140, 155, 156, 171, 172, 187, 188, 203, 204, + 219, 220, 235, 236, 251, 252, 267, 268, 283, 284, 299, 300, 315, 316, 331, + 332, 347, 348, 363, 364, 379, 380, 395, 396, 411, 412, 427, 428, 443, 444, + 459, 460, 475, 476, 491, 492, 507, 12, 12, 13, 28, 29, 44, 45, 60, + 61, 76, 77, 92, 93, 108, 109, 124, 125, 140, 141, 156, 157, 172, 173, + 188, 189, 204, 205, 220, 221, 236, 237, 252, 253, 268, 269, 284, 285, 300, + 301, 316, 317, 332, 333, 348, 349, 364, 365, 380, 381, 396, 397, 412, 413, + 428, 429, 444, 445, 460, 461, 476, 477, 492, 493, 508, 13, 13, 14, 29, + 30, 45, 46, 61, 62, 77, 78, 93, 94, 109, 110, 125, 126, 141, 142, + 157, 158, 173, 174, 189, 190, 205, 206, 221, 222, 237, 238, 253, 254, 269, + 270, 285, 286, 301, 302, 317, 318, 333, 334, 349, 350, 365, 366, 381, 382, + 397, 398, 413, 414, 429, 430, 445, 446, 461, 462, 477, 478, 493, 494, 509, + 14, 14, 15, 30, 31, 46, 47, 62, 63, 78, 79, 94, 95, 110, 111, + 126, 127, 142, 143, 158, 159, 174, 175, 190, 191, 206, 207, 222, 223, 238, + 239, 254, 255, 270, 271, 286, 287, 302, 303, 318, 319, 334, 335, 350, 351, + 366, 367, 382, 383, 398, 399, 414, 415, 430, 431, 446, 447, 462, 463, 478, + 479, 494, 495, 510, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_32x16_neighbors[513 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 32, 32, 64, 64, 96, 96, 128, 128, 160, 160, 192, + 192, 224, 224, 256, 256, 288, 288, 320, 320, 352, 352, 384, 384, 416, 416, + 448, 448, 0, 0, 1, 32, 33, 64, 65, 96, 97, 128, 129, 160, 161, + 192, 193, 224, 225, 256, 257, 288, 289, 320, 321, 352, 353, 384, 385, 416, + 417, 448, 449, 480, 1, 1, 2, 33, 34, 65, 66, 97, 98, 129, 130, + 161, 162, 193, 194, 225, 226, 257, 258, 289, 290, 321, 322, 353, 354, 385, + 386, 417, 418, 449, 450, 481, 2, 2, 3, 34, 35, 66, 67, 98, 99, + 130, 131, 162, 163, 194, 195, 226, 227, 258, 259, 290, 291, 322, 323, 354, + 355, 386, 387, 418, 419, 450, 451, 482, 3, 3, 4, 35, 36, 67, 68, + 99, 100, 131, 132, 163, 164, 195, 196, 227, 228, 259, 260, 291, 292, 323, + 324, 355, 356, 387, 388, 419, 420, 451, 452, 483, 4, 4, 5, 36, 37, + 68, 69, 100, 101, 132, 133, 164, 165, 196, 197, 228, 229, 260, 261, 292, + 293, 324, 325, 356, 357, 388, 389, 420, 421, 452, 453, 484, 5, 5, 6, + 37, 38, 69, 70, 101, 102, 133, 134, 165, 166, 197, 198, 229, 230, 261, + 262, 293, 294, 325, 326, 357, 358, 389, 390, 421, 422, 453, 454, 485, 6, + 6, 7, 38, 39, 70, 71, 102, 103, 134, 135, 166, 167, 198, 199, 230, + 231, 262, 263, 294, 295, 326, 327, 358, 359, 390, 391, 422, 423, 454, 455, + 486, 7, 7, 8, 39, 40, 71, 72, 103, 104, 135, 136, 167, 168, 199, + 200, 231, 232, 263, 264, 295, 296, 327, 328, 359, 360, 391, 392, 423, 424, + 455, 456, 487, 8, 8, 9, 40, 41, 72, 73, 104, 105, 136, 137, 168, + 169, 200, 201, 232, 233, 264, 265, 296, 297, 328, 329, 360, 361, 392, 393, + 424, 425, 456, 457, 488, 9, 9, 10, 41, 42, 73, 74, 105, 106, 137, + 138, 169, 170, 201, 202, 233, 234, 265, 266, 297, 298, 329, 330, 361, 362, + 393, 394, 425, 426, 457, 458, 489, 10, 10, 11, 42, 43, 74, 75, 106, + 107, 138, 139, 170, 171, 202, 203, 234, 235, 266, 267, 298, 299, 330, 331, + 362, 363, 394, 395, 426, 427, 458, 459, 490, 11, 11, 12, 43, 44, 75, + 76, 107, 108, 139, 140, 171, 172, 203, 204, 235, 236, 267, 268, 299, 300, + 331, 332, 363, 364, 395, 396, 427, 428, 459, 460, 491, 12, 12, 13, 44, + 45, 76, 77, 108, 109, 140, 141, 172, 173, 204, 205, 236, 237, 268, 269, + 300, 301, 332, 333, 364, 365, 396, 397, 428, 429, 460, 461, 492, 13, 13, + 14, 45, 46, 77, 78, 109, 110, 141, 142, 173, 174, 205, 206, 237, 238, + 269, 270, 301, 302, 333, 334, 365, 366, 397, 398, 429, 430, 461, 462, 493, + 14, 14, 15, 46, 47, 78, 79, 110, 111, 142, 143, 174, 175, 206, 207, + 238, 239, 270, 271, 302, 303, 334, 335, 366, 367, 398, 399, 430, 431, 462, + 463, 494, 15, 15, 16, 47, 48, 79, 80, 111, 112, 143, 144, 175, 176, + 207, 208, 239, 240, 271, 272, 303, 304, 335, 336, 367, 368, 399, 400, 431, + 432, 463, 464, 495, 16, 16, 17, 48, 49, 80, 81, 112, 113, 144, 145, + 176, 177, 208, 209, 240, 241, 272, 273, 304, 305, 336, 337, 368, 369, 400, + 401, 432, 433, 464, 465, 496, 17, 17, 18, 49, 50, 81, 82, 113, 114, + 145, 146, 177, 178, 209, 210, 241, 242, 273, 274, 305, 306, 337, 338, 369, + 370, 401, 402, 433, 434, 465, 466, 497, 18, 18, 19, 50, 51, 82, 83, + 114, 115, 146, 147, 178, 179, 210, 211, 242, 243, 274, 275, 306, 307, 338, + 339, 370, 371, 402, 403, 434, 435, 466, 467, 498, 19, 19, 20, 51, 52, + 83, 84, 115, 116, 147, 148, 179, 180, 211, 212, 243, 244, 275, 276, 307, + 308, 339, 340, 371, 372, 403, 404, 435, 436, 467, 468, 499, 20, 20, 21, + 52, 53, 84, 85, 116, 117, 148, 149, 180, 181, 212, 213, 244, 245, 276, + 277, 308, 309, 340, 341, 372, 373, 404, 405, 436, 437, 468, 469, 500, 21, + 21, 22, 53, 54, 85, 86, 117, 118, 149, 150, 181, 182, 213, 214, 245, + 246, 277, 278, 309, 310, 341, 342, 373, 374, 405, 406, 437, 438, 469, 470, + 501, 22, 22, 23, 54, 55, 86, 87, 118, 119, 150, 151, 182, 183, 214, + 215, 246, 247, 278, 279, 310, 311, 342, 343, 374, 375, 406, 407, 438, 439, + 470, 471, 502, 23, 23, 24, 55, 56, 87, 88, 119, 120, 151, 152, 183, + 184, 215, 216, 247, 248, 279, 280, 311, 312, 343, 344, 375, 376, 407, 408, + 439, 440, 471, 472, 503, 24, 24, 25, 56, 57, 88, 89, 120, 121, 152, + 153, 184, 185, 216, 217, 248, 249, 280, 281, 312, 313, 344, 345, 376, 377, + 408, 409, 440, 441, 472, 473, 504, 25, 25, 26, 57, 58, 89, 90, 121, + 122, 153, 154, 185, 186, 217, 218, 249, 250, 281, 282, 313, 314, 345, 346, + 377, 378, 409, 410, 441, 442, 473, 474, 505, 26, 26, 27, 58, 59, 90, + 91, 122, 123, 154, 155, 186, 187, 218, 219, 250, 251, 282, 283, 314, 315, + 346, 347, 378, 379, 410, 411, 442, 443, 474, 475, 506, 27, 27, 28, 59, + 60, 91, 92, 123, 124, 155, 156, 187, 188, 219, 220, 251, 252, 283, 284, + 315, 316, 347, 348, 379, 380, 411, 412, 443, 444, 475, 476, 507, 28, 28, + 29, 60, 61, 92, 93, 124, 125, 156, 157, 188, 189, 220, 221, 252, 253, + 284, 285, 316, 317, 348, 349, 380, 381, 412, 413, 444, 445, 476, 477, 508, + 29, 29, 30, 61, 62, 93, 94, 125, 126, 157, 158, 189, 190, 221, 222, + 253, 254, 285, 286, 317, 318, 349, 350, 381, 382, 413, 414, 445, 446, 477, + 478, 509, 30, 30, 31, 62, 63, 94, 95, 126, 127, 158, 159, 190, 191, + 222, 223, 254, 255, 286, 287, 318, 319, 350, 351, 382, 383, 414, 415, 446, + 447, 478, 479, 510, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_16x32_neighbors[513 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, + 14, 14, 0, 0, 1, 16, 2, 17, 3, 18, 4, 19, 5, 20, 6, + 21, 7, 22, 8, 23, 9, 24, 10, 25, 11, 26, 12, 27, 13, 28, + 14, 29, 15, 30, 16, 16, 17, 32, 18, 33, 19, 34, 20, 35, 21, + 36, 22, 37, 23, 38, 24, 39, 25, 40, 26, 41, 27, 42, 28, 43, + 29, 44, 30, 45, 31, 46, 32, 32, 33, 48, 34, 49, 35, 50, 36, + 51, 37, 52, 38, 53, 39, 54, 40, 55, 41, 56, 42, 57, 43, 58, + 44, 59, 45, 60, 46, 61, 47, 62, 48, 48, 49, 64, 50, 65, 51, + 66, 52, 67, 53, 68, 54, 69, 55, 70, 56, 71, 57, 72, 58, 73, + 59, 74, 60, 75, 61, 76, 62, 77, 63, 78, 64, 64, 65, 80, 66, + 81, 67, 82, 68, 83, 69, 84, 70, 85, 71, 86, 72, 87, 73, 88, + 74, 89, 75, 90, 76, 91, 77, 92, 78, 93, 79, 94, 80, 80, 81, + 96, 82, 97, 83, 98, 84, 99, 85, 100, 86, 101, 87, 102, 88, 103, + 89, 104, 90, 105, 91, 106, 92, 107, 93, 108, 94, 109, 95, 110, 96, + 96, 97, 112, 98, 113, 99, 114, 100, 115, 101, 116, 102, 117, 103, 118, + 104, 119, 105, 120, 106, 121, 107, 122, 108, 123, 109, 124, 110, 125, 111, + 126, 112, 112, 113, 128, 114, 129, 115, 130, 116, 131, 117, 132, 118, 133, + 119, 134, 120, 135, 121, 136, 122, 137, 123, 138, 124, 139, 125, 140, 126, + 141, 127, 142, 128, 128, 129, 144, 130, 145, 131, 146, 132, 147, 133, 148, + 134, 149, 135, 150, 136, 151, 137, 152, 138, 153, 139, 154, 140, 155, 141, + 156, 142, 157, 143, 158, 144, 144, 145, 160, 146, 161, 147, 162, 148, 163, + 149, 164, 150, 165, 151, 166, 152, 167, 153, 168, 154, 169, 155, 170, 156, + 171, 157, 172, 158, 173, 159, 174, 160, 160, 161, 176, 162, 177, 163, 178, + 164, 179, 165, 180, 166, 181, 167, 182, 168, 183, 169, 184, 170, 185, 171, + 186, 172, 187, 173, 188, 174, 189, 175, 190, 176, 176, 177, 192, 178, 193, + 179, 194, 180, 195, 181, 196, 182, 197, 183, 198, 184, 199, 185, 200, 186, + 201, 187, 202, 188, 203, 189, 204, 190, 205, 191, 206, 192, 192, 193, 208, + 194, 209, 195, 210, 196, 211, 197, 212, 198, 213, 199, 214, 200, 215, 201, + 216, 202, 217, 203, 218, 204, 219, 205, 220, 206, 221, 207, 222, 208, 208, + 209, 224, 210, 225, 211, 226, 212, 227, 213, 228, 214, 229, 215, 230, 216, + 231, 217, 232, 218, 233, 219, 234, 220, 235, 221, 236, 222, 237, 223, 238, + 224, 224, 225, 240, 226, 241, 227, 242, 228, 243, 229, 244, 230, 245, 231, + 246, 232, 247, 233, 248, 234, 249, 235, 250, 236, 251, 237, 252, 238, 253, + 239, 254, 240, 240, 241, 256, 242, 257, 243, 258, 244, 259, 245, 260, 246, + 261, 247, 262, 248, 263, 249, 264, 250, 265, 251, 266, 252, 267, 253, 268, + 254, 269, 255, 270, 256, 256, 257, 272, 258, 273, 259, 274, 260, 275, 261, + 276, 262, 277, 263, 278, 264, 279, 265, 280, 266, 281, 267, 282, 268, 283, + 269, 284, 270, 285, 271, 286, 272, 272, 273, 288, 274, 289, 275, 290, 276, + 291, 277, 292, 278, 293, 279, 294, 280, 295, 281, 296, 282, 297, 283, 298, + 284, 299, 285, 300, 286, 301, 287, 302, 288, 288, 289, 304, 290, 305, 291, + 306, 292, 307, 293, 308, 294, 309, 295, 310, 296, 311, 297, 312, 298, 313, + 299, 314, 300, 315, 301, 316, 302, 317, 303, 318, 304, 304, 305, 320, 306, + 321, 307, 322, 308, 323, 309, 324, 310, 325, 311, 326, 312, 327, 313, 328, + 314, 329, 315, 330, 316, 331, 317, 332, 318, 333, 319, 334, 320, 320, 321, + 336, 322, 337, 323, 338, 324, 339, 325, 340, 326, 341, 327, 342, 328, 343, + 329, 344, 330, 345, 331, 346, 332, 347, 333, 348, 334, 349, 335, 350, 336, + 336, 337, 352, 338, 353, 339, 354, 340, 355, 341, 356, 342, 357, 343, 358, + 344, 359, 345, 360, 346, 361, 347, 362, 348, 363, 349, 364, 350, 365, 351, + 366, 352, 352, 353, 368, 354, 369, 355, 370, 356, 371, 357, 372, 358, 373, + 359, 374, 360, 375, 361, 376, 362, 377, 363, 378, 364, 379, 365, 380, 366, + 381, 367, 382, 368, 368, 369, 384, 370, 385, 371, 386, 372, 387, 373, 388, + 374, 389, 375, 390, 376, 391, 377, 392, 378, 393, 379, 394, 380, 395, 381, + 396, 382, 397, 383, 398, 384, 384, 385, 400, 386, 401, 387, 402, 388, 403, + 389, 404, 390, 405, 391, 406, 392, 407, 393, 408, 394, 409, 395, 410, 396, + 411, 397, 412, 398, 413, 399, 414, 400, 400, 401, 416, 402, 417, 403, 418, + 404, 419, 405, 420, 406, 421, 407, 422, 408, 423, 409, 424, 410, 425, 411, + 426, 412, 427, 413, 428, 414, 429, 415, 430, 416, 416, 417, 432, 418, 433, + 419, 434, 420, 435, 421, 436, 422, 437, 423, 438, 424, 439, 425, 440, 426, + 441, 427, 442, 428, 443, 429, 444, 430, 445, 431, 446, 432, 432, 433, 448, + 434, 449, 435, 450, 436, 451, 437, 452, 438, 453, 439, 454, 440, 455, 441, + 456, 442, 457, 443, 458, 444, 459, 445, 460, 446, 461, 447, 462, 448, 448, + 449, 464, 450, 465, 451, 466, 452, 467, 453, 468, 454, 469, 455, 470, 456, + 471, 457, 472, 458, 473, 459, 474, 460, 475, 461, 476, 462, 477, 463, 478, + 464, 464, 465, 480, 466, 481, 467, 482, 468, 483, 469, 484, 470, 485, 471, + 486, 472, 487, 473, 488, 474, 489, 475, 490, 476, 491, 477, 492, 478, 493, + 479, 494, 480, 480, 481, 496, 482, 497, 483, 498, 484, 499, 485, 500, 486, + 501, 487, 502, 488, 503, 489, 504, 490, 505, 491, 506, 492, 507, 493, 508, + 494, 509, 495, 510, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_32x16_neighbors[513 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, + 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, + 21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, + 29, 29, 30, 30, 0, 0, 1, 32, 2, 33, 3, 34, 4, 35, 5, + 36, 6, 37, 7, 38, 8, 39, 9, 40, 10, 41, 11, 42, 12, 43, + 13, 44, 14, 45, 15, 46, 16, 47, 17, 48, 18, 49, 19, 50, 20, + 51, 21, 52, 22, 53, 23, 54, 24, 55, 25, 56, 26, 57, 27, 58, + 28, 59, 29, 60, 30, 61, 31, 62, 32, 32, 33, 64, 34, 65, 35, + 66, 36, 67, 37, 68, 38, 69, 39, 70, 40, 71, 41, 72, 42, 73, + 43, 74, 44, 75, 45, 76, 46, 77, 47, 78, 48, 79, 49, 80, 50, + 81, 51, 82, 52, 83, 53, 84, 54, 85, 55, 86, 56, 87, 57, 88, + 58, 89, 59, 90, 60, 91, 61, 92, 62, 93, 63, 94, 64, 64, 65, + 96, 66, 97, 67, 98, 68, 99, 69, 100, 70, 101, 71, 102, 72, 103, + 73, 104, 74, 105, 75, 106, 76, 107, 77, 108, 78, 109, 79, 110, 80, + 111, 81, 112, 82, 113, 83, 114, 84, 115, 85, 116, 86, 117, 87, 118, + 88, 119, 89, 120, 90, 121, 91, 122, 92, 123, 93, 124, 94, 125, 95, + 126, 96, 96, 97, 128, 98, 129, 99, 130, 100, 131, 101, 132, 102, 133, + 103, 134, 104, 135, 105, 136, 106, 137, 107, 138, 108, 139, 109, 140, 110, + 141, 111, 142, 112, 143, 113, 144, 114, 145, 115, 146, 116, 147, 117, 148, + 118, 149, 119, 150, 120, 151, 121, 152, 122, 153, 123, 154, 124, 155, 125, + 156, 126, 157, 127, 158, 128, 128, 129, 160, 130, 161, 131, 162, 132, 163, + 133, 164, 134, 165, 135, 166, 136, 167, 137, 168, 138, 169, 139, 170, 140, + 171, 141, 172, 142, 173, 143, 174, 144, 175, 145, 176, 146, 177, 147, 178, + 148, 179, 149, 180, 150, 181, 151, 182, 152, 183, 153, 184, 154, 185, 155, + 186, 156, 187, 157, 188, 158, 189, 159, 190, 160, 160, 161, 192, 162, 193, + 163, 194, 164, 195, 165, 196, 166, 197, 167, 198, 168, 199, 169, 200, 170, + 201, 171, 202, 172, 203, 173, 204, 174, 205, 175, 206, 176, 207, 177, 208, + 178, 209, 179, 210, 180, 211, 181, 212, 182, 213, 183, 214, 184, 215, 185, + 216, 186, 217, 187, 218, 188, 219, 189, 220, 190, 221, 191, 222, 192, 192, + 193, 224, 194, 225, 195, 226, 196, 227, 197, 228, 198, 229, 199, 230, 200, + 231, 201, 232, 202, 233, 203, 234, 204, 235, 205, 236, 206, 237, 207, 238, + 208, 239, 209, 240, 210, 241, 211, 242, 212, 243, 213, 244, 214, 245, 215, + 246, 216, 247, 217, 248, 218, 249, 219, 250, 220, 251, 221, 252, 222, 253, + 223, 254, 224, 224, 225, 256, 226, 257, 227, 258, 228, 259, 229, 260, 230, + 261, 231, 262, 232, 263, 233, 264, 234, 265, 235, 266, 236, 267, 237, 268, + 238, 269, 239, 270, 240, 271, 241, 272, 242, 273, 243, 274, 244, 275, 245, + 276, 246, 277, 247, 278, 248, 279, 249, 280, 250, 281, 251, 282, 252, 283, + 253, 284, 254, 285, 255, 286, 256, 256, 257, 288, 258, 289, 259, 290, 260, + 291, 261, 292, 262, 293, 263, 294, 264, 295, 265, 296, 266, 297, 267, 298, + 268, 299, 269, 300, 270, 301, 271, 302, 272, 303, 273, 304, 274, 305, 275, + 306, 276, 307, 277, 308, 278, 309, 279, 310, 280, 311, 281, 312, 282, 313, + 283, 314, 284, 315, 285, 316, 286, 317, 287, 318, 288, 288, 289, 320, 290, + 321, 291, 322, 292, 323, 293, 324, 294, 325, 295, 326, 296, 327, 297, 328, + 298, 329, 299, 330, 300, 331, 301, 332, 302, 333, 303, 334, 304, 335, 305, + 336, 306, 337, 307, 338, 308, 339, 309, 340, 310, 341, 311, 342, 312, 343, + 313, 344, 314, 345, 315, 346, 316, 347, 317, 348, 318, 349, 319, 350, 320, + 320, 321, 352, 322, 353, 323, 354, 324, 355, 325, 356, 326, 357, 327, 358, + 328, 359, 329, 360, 330, 361, 331, 362, 332, 363, 333, 364, 334, 365, 335, + 366, 336, 367, 337, 368, 338, 369, 339, 370, 340, 371, 341, 372, 342, 373, + 343, 374, 344, 375, 345, 376, 346, 377, 347, 378, 348, 379, 349, 380, 350, + 381, 351, 382, 352, 352, 353, 384, 354, 385, 355, 386, 356, 387, 357, 388, + 358, 389, 359, 390, 360, 391, 361, 392, 362, 393, 363, 394, 364, 395, 365, + 396, 366, 397, 367, 398, 368, 399, 369, 400, 370, 401, 371, 402, 372, 403, + 373, 404, 374, 405, 375, 406, 376, 407, 377, 408, 378, 409, 379, 410, 380, + 411, 381, 412, 382, 413, 383, 414, 384, 384, 385, 416, 386, 417, 387, 418, + 388, 419, 389, 420, 390, 421, 391, 422, 392, 423, 393, 424, 394, 425, 395, + 426, 396, 427, 397, 428, 398, 429, 399, 430, 400, 431, 401, 432, 402, 433, + 403, 434, 404, 435, 405, 436, 406, 437, 407, 438, 408, 439, 409, 440, 410, + 441, 411, 442, 412, 443, 413, 444, 414, 445, 415, 446, 416, 416, 417, 448, + 418, 449, 419, 450, 420, 451, 421, 452, 422, 453, 423, 454, 424, 455, 425, + 456, 426, 457, 427, 458, 428, 459, 429, 460, 430, 461, 431, 462, 432, 463, + 433, 464, 434, 465, 435, 466, 436, 467, 437, 468, 438, 469, 439, 470, 440, + 471, 441, 472, 442, 473, 443, 474, 444, 475, 445, 476, 446, 477, 447, 478, + 448, 448, 449, 480, 450, 481, 451, 482, 452, 483, 453, 484, 454, 485, 455, + 486, 456, 487, 457, 488, 458, 489, 459, 490, 460, 491, 461, 492, 462, 493, + 463, 494, 464, 495, 465, 496, 466, 497, 467, 498, 468, 499, 469, 500, 470, + 501, 471, 502, 472, 503, 473, 504, 474, 505, 475, 506, 476, 507, 477, 508, + 478, 509, 479, 510, 0, 0 +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_16x16_neighbors[257 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 16, 16, 32, 32, 48, 48, 64, 64, 80, 80, 96, + 96, 112, 112, 128, 128, 144, 144, 160, 160, 176, 176, 192, 192, 208, 208, + 224, 224, 0, 0, 1, 16, 17, 32, 33, 48, 49, 64, 65, 80, 81, + 96, 97, 112, 113, 128, 129, 144, 145, 160, 161, 176, 177, 192, 193, 208, + 209, 224, 225, 240, 1, 1, 2, 17, 18, 33, 34, 49, 50, 65, 66, + 81, 82, 97, 98, 113, 114, 129, 130, 145, 146, 161, 162, 177, 178, 193, + 194, 209, 210, 225, 226, 241, 2, 2, 3, 18, 19, 34, 35, 50, 51, + 66, 67, 82, 83, 98, 99, 114, 115, 130, 131, 146, 147, 162, 163, 178, + 179, 194, 195, 210, 211, 226, 227, 242, 3, 3, 4, 19, 20, 35, 36, + 51, 52, 67, 68, 83, 84, 99, 100, 115, 116, 131, 132, 147, 148, 163, + 164, 179, 180, 195, 196, 211, 212, 227, 228, 243, 4, 4, 5, 20, 21, + 36, 37, 52, 53, 68, 69, 84, 85, 100, 101, 116, 117, 132, 133, 148, + 149, 164, 165, 180, 181, 196, 197, 212, 213, 228, 229, 244, 5, 5, 6, + 21, 22, 37, 38, 53, 54, 69, 70, 85, 86, 101, 102, 117, 118, 133, + 134, 149, 150, 165, 166, 181, 182, 197, 198, 213, 214, 229, 230, 245, 6, + 6, 7, 22, 23, 38, 39, 54, 55, 70, 71, 86, 87, 102, 103, 118, + 119, 134, 135, 150, 151, 166, 167, 182, 183, 198, 199, 214, 215, 230, 231, + 246, 7, 7, 8, 23, 24, 39, 40, 55, 56, 71, 72, 87, 88, 103, + 104, 119, 120, 135, 136, 151, 152, 167, 168, 183, 184, 199, 200, 215, 216, + 231, 232, 247, 8, 8, 9, 24, 25, 40, 41, 56, 57, 72, 73, 88, + 89, 104, 105, 120, 121, 136, 137, 152, 153, 168, 169, 184, 185, 200, 201, + 216, 217, 232, 233, 248, 9, 9, 10, 25, 26, 41, 42, 57, 58, 73, + 74, 89, 90, 105, 106, 121, 122, 137, 138, 153, 154, 169, 170, 185, 186, + 201, 202, 217, 218, 233, 234, 249, 10, 10, 11, 26, 27, 42, 43, 58, + 59, 74, 75, 90, 91, 106, 107, 122, 123, 138, 139, 154, 155, 170, 171, + 186, 187, 202, 203, 218, 219, 234, 235, 250, 11, 11, 12, 27, 28, 43, + 44, 59, 60, 75, 76, 91, 92, 107, 108, 123, 124, 139, 140, 155, 156, + 171, 172, 187, 188, 203, 204, 219, 220, 235, 236, 251, 12, 12, 13, 28, + 29, 44, 45, 60, 61, 76, 77, 92, 93, 108, 109, 124, 125, 140, 141, + 156, 157, 172, 173, 188, 189, 204, 205, 220, 221, 236, 237, 252, 13, 13, + 14, 29, 30, 45, 46, 61, 62, 77, 78, 93, 94, 109, 110, 125, 126, + 141, 142, 157, 158, 173, 174, 189, 190, 205, 206, 221, 222, 237, 238, 253, + 14, 14, 15, 30, 31, 46, 47, 62, 63, 78, 79, 94, 95, 110, 111, + 126, 127, 142, 143, 158, 159, 174, 175, 190, 191, 206, 207, 222, 223, 238, + 239, 254, 0, 0, +}; + +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_16x16_neighbors[257 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, + 14, 14, 0, 0, 1, 16, 2, 17, 3, 18, 4, 19, 5, 20, 6, + 21, 7, 22, 8, 23, 9, 24, 10, 25, 11, 26, 12, 27, 13, 28, + 14, 29, 15, 30, 16, 16, 17, 32, 18, 33, 19, 34, 20, 35, 21, + 36, 22, 37, 23, 38, 24, 39, 25, 40, 26, 41, 27, 42, 28, 43, + 29, 44, 30, 45, 31, 46, 32, 32, 33, 48, 34, 49, 35, 50, 36, + 51, 37, 52, 38, 53, 39, 54, 40, 55, 41, 56, 42, 57, 43, 58, + 44, 59, 45, 60, 46, 61, 47, 62, 48, 48, 49, 64, 50, 65, 51, + 66, 52, 67, 53, 68, 54, 69, 55, 70, 56, 71, 57, 72, 58, 73, + 59, 74, 60, 75, 61, 76, 62, 77, 63, 78, 64, 64, 65, 80, 66, + 81, 67, 82, 68, 83, 69, 84, 70, 85, 71, 86, 72, 87, 73, 88, + 74, 89, 75, 90, 76, 91, 77, 92, 78, 93, 79, 94, 80, 80, 81, + 96, 82, 97, 83, 98, 84, 99, 85, 100, 86, 101, 87, 102, 88, 103, + 89, 104, 90, 105, 91, 106, 92, 107, 93, 108, 94, 109, 95, 110, 96, + 96, 97, 112, 98, 113, 99, 114, 100, 115, 101, 116, 102, 117, 103, 118, + 104, 119, 105, 120, 106, 121, 107, 122, 108, 123, 109, 124, 110, 125, 111, + 126, 112, 112, 113, 128, 114, 129, 115, 130, 116, 131, 117, 132, 118, 133, + 119, 134, 120, 135, 121, 136, 122, 137, 123, 138, 124, 139, 125, 140, 126, + 141, 127, 142, 128, 128, 129, 144, 130, 145, 131, 146, 132, 147, 133, 148, + 134, 149, 135, 150, 136, 151, 137, 152, 138, 153, 139, 154, 140, 155, 141, + 156, 142, 157, 143, 158, 144, 144, 145, 160, 146, 161, 147, 162, 148, 163, + 149, 164, 150, 165, 151, 166, 152, 167, 153, 168, 154, 169, 155, 170, 156, + 171, 157, 172, 158, 173, 159, 174, 160, 160, 161, 176, 162, 177, 163, 178, + 164, 179, 165, 180, 166, 181, 167, 182, 168, 183, 169, 184, 170, 185, 171, + 186, 172, 187, 173, 188, 174, 189, 175, 190, 176, 176, 177, 192, 178, 193, + 179, 194, 180, 195, 181, 196, 182, 197, 183, 198, 184, 199, 185, 200, 186, + 201, 187, 202, 188, 203, 189, 204, 190, 205, 191, 206, 192, 192, 193, 208, + 194, 209, 195, 210, 196, 211, 197, 212, 198, 213, 199, 214, 200, 215, 201, + 216, 202, 217, 203, 218, 204, 219, 205, 220, 206, 221, 207, 222, 208, 208, + 209, 224, 210, 225, 211, 226, 212, 227, 213, 228, 214, 229, 215, 230, 216, + 231, 217, 232, 218, 233, 219, 234, 220, 235, 221, 236, 222, 237, 223, 238, + 224, 224, 225, 240, 226, 241, 227, 242, 228, 243, 229, 244, 230, 245, 231, + 246, 232, 247, 233, 248, 234, 249, 235, 250, 236, 251, 237, 252, 238, 253, + 239, 254, 0, 0, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, + col_scan_16x16_neighbors[257 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 16, 16, 32, 32, 16, 0, 48, 48, 1, 16, 64, + 64, 17, 32, 80, 80, 33, 48, 17, 1, 49, 64, 96, 96, 2, 17, + 65, 80, 18, 33, 112, 112, 34, 49, 81, 96, 18, 2, 50, 65, 128, + 128, 3, 18, 97, 112, 19, 34, 66, 81, 144, 144, 82, 97, 35, 50, + 113, 128, 19, 3, 51, 66, 160, 160, 4, 19, 98, 113, 129, 144, 67, + 82, 20, 35, 83, 98, 114, 129, 36, 51, 176, 176, 20, 4, 145, 160, + 52, 67, 99, 114, 5, 20, 130, 145, 68, 83, 192, 192, 161, 176, 21, + 36, 115, 130, 84, 99, 37, 52, 146, 161, 208, 208, 53, 68, 21, 5, + 100, 115, 177, 192, 131, 146, 69, 84, 6, 21, 224, 224, 116, 131, 22, + 37, 162, 177, 85, 100, 147, 162, 38, 53, 193, 208, 101, 116, 54, 69, + 22, 6, 132, 147, 178, 193, 70, 85, 163, 178, 209, 224, 7, 22, 117, + 132, 23, 38, 148, 163, 23, 7, 86, 101, 194, 209, 225, 240, 39, 54, + 179, 194, 102, 117, 133, 148, 55, 70, 164, 179, 8, 23, 71, 86, 210, + 225, 118, 133, 149, 164, 195, 210, 24, 39, 87, 102, 40, 55, 56, 71, + 134, 149, 180, 195, 226, 241, 103, 118, 24, 8, 165, 180, 211, 226, 72, + 87, 150, 165, 9, 24, 119, 134, 25, 40, 88, 103, 196, 211, 41, 56, + 135, 150, 181, 196, 104, 119, 57, 72, 227, 242, 166, 181, 120, 135, 151, + 166, 197, 212, 73, 88, 25, 9, 212, 227, 89, 104, 136, 151, 182, 197, + 10, 25, 26, 41, 105, 120, 167, 182, 228, 243, 152, 167, 42, 57, 121, + 136, 213, 228, 58, 73, 198, 213, 74, 89, 137, 152, 183, 198, 168, 183, + 26, 10, 90, 105, 229, 244, 11, 26, 106, 121, 214, 229, 153, 168, 27, + 42, 199, 214, 43, 58, 184, 199, 122, 137, 169, 184, 230, 245, 59, 74, + 27, 11, 75, 90, 138, 153, 200, 215, 215, 230, 91, 106, 12, 27, 28, + 43, 185, 200, 107, 122, 154, 169, 44, 59, 231, 246, 216, 231, 60, 75, + 123, 138, 28, 12, 76, 91, 201, 216, 170, 185, 232, 247, 139, 154, 92, + 107, 13, 28, 108, 123, 29, 44, 186, 201, 217, 232, 155, 170, 45, 60, + 29, 13, 61, 76, 124, 139, 14, 14, 233, 248, 77, 92, 14, 29, 171, + 186, 140, 155, 202, 217, 30, 45, 93, 108, 109, 124, 46, 61, 156, 171, + 62, 77, 187, 202, 15, 30, 125, 140, 218, 233, 78, 93, 31, 46, 172, + 187, 47, 62, 141, 156, 94, 109, 234, 249, 203, 218, 63, 78, 110, 125, + 188, 203, 157, 172, 126, 141, 79, 94, 173, 188, 95, 110, 219, 234, 142, + 157, 204, 219, 235, 250, 111, 126, 158, 173, 127, 142, 189, 204, 220, 235, + 143, 158, 174, 189, 205, 220, 236, 251, 159, 174, 190, 205, 221, 236, 175, + 190, 237, 252, 206, 221, 222, 237, 191, 206, 238, 253, 207, 222, 223, 238, + 239, 254, 0, 0, +}; + +DECLARE_ALIGNED(16, static const int16_t, + row_scan_16x16_neighbors[257 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 0, 1, 2, 2, 1, 16, 3, 3, 2, + 17, 16, 17, 4, 4, 17, 32, 3, 18, 5, 5, 18, 33, 32, 33, + 4, 19, 33, 48, 6, 6, 19, 34, 5, 20, 34, 49, 48, 49, 7, + 7, 20, 35, 49, 64, 6, 21, 35, 50, 21, 36, 64, 65, 8, 8, + 50, 65, 36, 51, 7, 22, 22, 37, 65, 80, 51, 66, 9, 9, 37, + 52, 8, 23, 66, 81, 52, 67, 80, 81, 23, 38, 10, 10, 38, 53, + 67, 82, 81, 96, 53, 68, 9, 24, 82, 97, 68, 83, 24, 39, 96, + 97, 39, 54, 11, 11, 54, 69, 83, 98, 97, 112, 69, 84, 10, 25, + 25, 40, 40, 55, 98, 113, 84, 99, 12, 12, 55, 70, 112, 113, 70, + 85, 11, 26, 99, 114, 85, 100, 113, 128, 26, 41, 41, 56, 56, 71, + 100, 115, 13, 13, 71, 86, 114, 129, 86, 101, 128, 129, 57, 72, 115, + 130, 101, 116, 12, 27, 42, 57, 14, 14, 72, 87, 27, 42, 129, 144, + 87, 102, 116, 131, 130, 145, 102, 117, 58, 73, 144, 145, 73, 88, 117, + 132, 88, 103, 13, 28, 43, 58, 131, 146, 103, 118, 28, 43, 145, 160, + 132, 147, 74, 89, 89, 104, 118, 133, 146, 161, 104, 119, 160, 161, 59, + 74, 119, 134, 133, 148, 14, 29, 44, 59, 147, 162, 161, 176, 29, 44, + 105, 120, 75, 90, 90, 105, 148, 163, 162, 177, 134, 149, 176, 177, 120, + 135, 149, 164, 163, 178, 15, 30, 135, 150, 177, 192, 60, 75, 106, 121, + 45, 60, 121, 136, 178, 193, 91, 106, 136, 151, 164, 179, 192, 193, 30, + 45, 150, 165, 151, 166, 179, 194, 76, 91, 165, 180, 122, 137, 193, 208, + 107, 122, 137, 152, 208, 209, 180, 195, 61, 76, 152, 167, 194, 209, 166, + 181, 224, 224, 92, 107, 181, 196, 46, 61, 138, 153, 209, 224, 167, 182, + 153, 168, 195, 210, 31, 46, 123, 138, 77, 92, 168, 183, 210, 225, 196, + 211, 225, 240, 182, 197, 154, 169, 108, 123, 139, 154, 183, 198, 62, 77, + 197, 212, 169, 184, 93, 108, 211, 226, 184, 199, 47, 62, 212, 227, 226, + 241, 124, 139, 198, 213, 155, 170, 170, 185, 140, 155, 213, 228, 227, 242, + 109, 124, 78, 93, 185, 200, 228, 243, 199, 214, 200, 215, 214, 229, 125, + 140, 171, 186, 186, 201, 63, 78, 156, 171, 94, 109, 141, 156, 229, 244, + 201, 216, 215, 230, 79, 94, 230, 245, 216, 231, 110, 125, 187, 202, 231, + 246, 217, 232, 157, 172, 202, 217, 126, 141, 95, 110, 142, 157, 172, 187, + 232, 247, 111, 126, 218, 233, 203, 218, 233, 248, 173, 188, 188, 203, 127, + 142, 158, 173, 143, 158, 234, 249, 219, 234, 189, 204, 204, 219, 159, 174, + 174, 189, 235, 250, 205, 220, 175, 190, 190, 205, 220, 235, 191, 206, 221, + 236, 236, 251, 206, 221, 237, 252, 207, 222, 222, 237, 223, 238, 238, 253, + 239, 254, 0, 0, +}; + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_16x16_neighbors[257 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 16, 0, 16, 16, 1, 16, 17, 1, 32, 32, 17, + 32, 2, 17, 18, 2, 48, 48, 18, 33, 33, 48, 3, 18, 49, 64, + 64, 65, 34, 49, 19, 3, 19, 34, 50, 65, 4, 19, 65, 80, 80, + 81, 35, 50, 20, 4, 20, 35, 66, 81, 81, 96, 51, 66, 96, 97, + 5, 20, 36, 51, 82, 97, 21, 36, 67, 82, 97, 112, 21, 5, 52, + 67, 112, 113, 37, 52, 6, 21, 83, 98, 98, 113, 68, 83, 22, 6, + 113, 128, 22, 37, 53, 68, 84, 99, 99, 114, 128, 129, 114, 129, 69, + 84, 38, 53, 7, 22, 23, 7, 129, 144, 23, 38, 54, 69, 100, 115, + 85, 100, 115, 130, 144, 145, 130, 145, 39, 54, 70, 85, 8, 23, 55, + 70, 116, 131, 101, 116, 145, 160, 24, 39, 24, 8, 86, 101, 131, 146, + 160, 161, 146, 161, 71, 86, 40, 55, 9, 24, 117, 132, 102, 117, 161, + 176, 132, 147, 56, 71, 87, 102, 25, 40, 147, 162, 25, 9, 176, 177, + 162, 177, 72, 87, 41, 56, 118, 133, 133, 148, 103, 118, 10, 25, 148, + 163, 57, 72, 88, 103, 177, 192, 26, 41, 163, 178, 192, 193, 26, 10, + 119, 134, 73, 88, 149, 164, 104, 119, 134, 149, 42, 57, 178, 193, 164, + 179, 11, 26, 58, 73, 193, 208, 89, 104, 135, 150, 120, 135, 27, 42, + 74, 89, 208, 209, 150, 165, 179, 194, 165, 180, 105, 120, 194, 209, 43, + 58, 27, 11, 136, 151, 90, 105, 151, 166, 180, 195, 59, 74, 121, 136, + 209, 224, 195, 210, 224, 225, 166, 181, 106, 121, 75, 90, 12, 27, 181, + 196, 28, 12, 210, 225, 152, 167, 167, 182, 137, 152, 28, 43, 196, 211, + 122, 137, 91, 106, 225, 240, 44, 59, 13, 28, 107, 122, 182, 197, 168, + 183, 211, 226, 153, 168, 226, 241, 60, 75, 197, 212, 138, 153, 29, 44, + 76, 91, 29, 13, 183, 198, 123, 138, 45, 60, 212, 227, 198, 213, 154, + 169, 169, 184, 227, 242, 92, 107, 61, 76, 139, 154, 14, 29, 30, 14, + 184, 199, 213, 228, 108, 123, 199, 214, 228, 243, 77, 92, 30, 45, 170, + 185, 155, 170, 185, 200, 93, 108, 124, 139, 214, 229, 46, 61, 200, 215, + 229, 244, 15, 30, 109, 124, 62, 77, 140, 155, 215, 230, 31, 46, 171, + 186, 186, 201, 201, 216, 78, 93, 230, 245, 125, 140, 47, 62, 216, 231, + 156, 171, 94, 109, 231, 246, 141, 156, 63, 78, 202, 217, 187, 202, 110, + 125, 217, 232, 172, 187, 232, 247, 79, 94, 157, 172, 126, 141, 203, 218, + 95, 110, 233, 248, 218, 233, 142, 157, 111, 126, 173, 188, 188, 203, 234, + 249, 219, 234, 127, 142, 158, 173, 204, 219, 189, 204, 143, 158, 235, 250, + 174, 189, 205, 220, 159, 174, 220, 235, 221, 236, 175, 190, 190, 205, 236, + 251, 206, 221, 237, 252, 191, 206, 222, 237, 207, 222, 238, 253, 223, 238, + 239, 254, 0, 0, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, + mcol_scan_32x32_neighbors[1025 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 32, 32, 64, 64, 96, 96, 128, 128, 160, 160, + 192, 192, 224, 224, 256, 256, 288, 288, 320, 320, 352, 352, 384, 384, + 416, 416, 448, 448, 480, 480, 512, 512, 544, 544, 576, 576, 608, 608, + 640, 640, 672, 672, 704, 704, 736, 736, 768, 768, 800, 800, 832, 832, + 864, 864, 896, 896, 928, 928, 960, 960, 0, 0, 1, 32, 33, 64, + 65, 96, 97, 128, 129, 160, 161, 192, 193, 224, 225, 256, 257, 288, + 289, 320, 321, 352, 353, 384, 385, 416, 417, 448, 449, 480, 481, 512, + 513, 544, 545, 576, 577, 608, 609, 640, 641, 672, 673, 704, 705, 736, + 737, 768, 769, 800, 801, 832, 833, 864, 865, 896, 897, 928, 929, 960, + 961, 992, 1, 1, 2, 33, 34, 65, 66, 97, 98, 129, 130, 161, + 162, 193, 194, 225, 226, 257, 258, 289, 290, 321, 322, 353, 354, 385, + 386, 417, 418, 449, 450, 481, 482, 513, 514, 545, 546, 577, 578, 609, + 610, 641, 642, 673, 674, 705, 706, 737, 738, 769, 770, 801, 802, 833, + 834, 865, 866, 897, 898, 929, 930, 961, 962, 993, 2, 2, 3, 34, + 35, 66, 67, 98, 99, 130, 131, 162, 163, 194, 195, 226, 227, 258, + 259, 290, 291, 322, 323, 354, 355, 386, 387, 418, 419, 450, 451, 482, + 483, 514, 515, 546, 547, 578, 579, 610, 611, 642, 643, 674, 675, 706, + 707, 738, 739, 770, 771, 802, 803, 834, 835, 866, 867, 898, 899, 930, + 931, 962, 963, 994, 3, 3, 4, 35, 36, 67, 68, 99, 100, 131, + 132, 163, 164, 195, 196, 227, 228, 259, 260, 291, 292, 323, 324, 355, + 356, 387, 388, 419, 420, 451, 452, 483, 484, 515, 516, 547, 548, 579, + 580, 611, 612, 643, 644, 675, 676, 707, 708, 739, 740, 771, 772, 803, + 804, 835, 836, 867, 868, 899, 900, 931, 932, 963, 964, 995, 4, 4, + 5, 36, 37, 68, 69, 100, 101, 132, 133, 164, 165, 196, 197, 228, + 229, 260, 261, 292, 293, 324, 325, 356, 357, 388, 389, 420, 421, 452, + 453, 484, 485, 516, 517, 548, 549, 580, 581, 612, 613, 644, 645, 676, + 677, 708, 709, 740, 741, 772, 773, 804, 805, 836, 837, 868, 869, 900, + 901, 932, 933, 964, 965, 996, 5, 5, 6, 37, 38, 69, 70, 101, + 102, 133, 134, 165, 166, 197, 198, 229, 230, 261, 262, 293, 294, 325, + 326, 357, 358, 389, 390, 421, 422, 453, 454, 485, 486, 517, 518, 549, + 550, 581, 582, 613, 614, 645, 646, 677, 678, 709, 710, 741, 742, 773, + 774, 805, 806, 837, 838, 869, 870, 901, 902, 933, 934, 965, 966, 997, + 6, 6, 7, 38, 39, 70, 71, 102, 103, 134, 135, 166, 167, 198, + 199, 230, 231, 262, 263, 294, 295, 326, 327, 358, 359, 390, 391, 422, + 423, 454, 455, 486, 487, 518, 519, 550, 551, 582, 583, 614, 615, 646, + 647, 678, 679, 710, 711, 742, 743, 774, 775, 806, 807, 838, 839, 870, + 871, 902, 903, 934, 935, 966, 967, 998, 7, 7, 8, 39, 40, 71, + 72, 103, 104, 135, 136, 167, 168, 199, 200, 231, 232, 263, 264, 295, + 296, 327, 328, 359, 360, 391, 392, 423, 424, 455, 456, 487, 488, 519, + 520, 551, 552, 583, 584, 615, 616, 647, 648, 679, 680, 711, 712, 743, + 744, 775, 776, 807, 808, 839, 840, 871, 872, 903, 904, 935, 936, 967, + 968, 999, 8, 8, 9, 40, 41, 72, 73, 104, 105, 136, 137, 168, + 169, 200, 201, 232, 233, 264, 265, 296, 297, 328, 329, 360, 361, 392, + 393, 424, 425, 456, 457, 488, 489, 520, 521, 552, 553, 584, 585, 616, + 617, 648, 649, 680, 681, 712, 713, 744, 745, 776, 777, 808, 809, 840, + 841, 872, 873, 904, 905, 936, 937, 968, 969, 1000, 9, 9, 10, 41, + 42, 73, 74, 105, 106, 137, 138, 169, 170, 201, 202, 233, 234, 265, + 266, 297, 298, 329, 330, 361, 362, 393, 394, 425, 426, 457, 458, 489, + 490, 521, 522, 553, 554, 585, 586, 617, 618, 649, 650, 681, 682, 713, + 714, 745, 746, 777, 778, 809, 810, 841, 842, 873, 874, 905, 906, 937, + 938, 969, 970, 1001, 10, 10, 11, 42, 43, 74, 75, 106, 107, 138, + 139, 170, 171, 202, 203, 234, 235, 266, 267, 298, 299, 330, 331, 362, + 363, 394, 395, 426, 427, 458, 459, 490, 491, 522, 523, 554, 555, 586, + 587, 618, 619, 650, 651, 682, 683, 714, 715, 746, 747, 778, 779, 810, + 811, 842, 843, 874, 875, 906, 907, 938, 939, 970, 971, 1002, 11, 11, + 12, 43, 44, 75, 76, 107, 108, 139, 140, 171, 172, 203, 204, 235, + 236, 267, 268, 299, 300, 331, 332, 363, 364, 395, 396, 427, 428, 459, + 460, 491, 492, 523, 524, 555, 556, 587, 588, 619, 620, 651, 652, 683, + 684, 715, 716, 747, 748, 779, 780, 811, 812, 843, 844, 875, 876, 907, + 908, 939, 940, 971, 972, 1003, 12, 12, 13, 44, 45, 76, 77, 108, + 109, 140, 141, 172, 173, 204, 205, 236, 237, 268, 269, 300, 301, 332, + 333, 364, 365, 396, 397, 428, 429, 460, 461, 492, 493, 524, 525, 556, + 557, 588, 589, 620, 621, 652, 653, 684, 685, 716, 717, 748, 749, 780, + 781, 812, 813, 844, 845, 876, 877, 908, 909, 940, 941, 972, 973, 1004, + 13, 13, 14, 45, 46, 77, 78, 109, 110, 141, 142, 173, 174, 205, + 206, 237, 238, 269, 270, 301, 302, 333, 334, 365, 366, 397, 398, 429, + 430, 461, 462, 493, 494, 525, 526, 557, 558, 589, 590, 621, 622, 653, + 654, 685, 686, 717, 718, 749, 750, 781, 782, 813, 814, 845, 846, 877, + 878, 909, 910, 941, 942, 973, 974, 1005, 14, 14, 15, 46, 47, 78, + 79, 110, 111, 142, 143, 174, 175, 206, 207, 238, 239, 270, 271, 302, + 303, 334, 335, 366, 367, 398, 399, 430, 431, 462, 463, 494, 495, 526, + 527, 558, 559, 590, 591, 622, 623, 654, 655, 686, 687, 718, 719, 750, + 751, 782, 783, 814, 815, 846, 847, 878, 879, 910, 911, 942, 943, 974, + 975, 1006, 15, 15, 16, 47, 48, 79, 80, 111, 112, 143, 144, 175, + 176, 207, 208, 239, 240, 271, 272, 303, 304, 335, 336, 367, 368, 399, + 400, 431, 432, 463, 464, 495, 496, 527, 528, 559, 560, 591, 592, 623, + 624, 655, 656, 687, 688, 719, 720, 751, 752, 783, 784, 815, 816, 847, + 848, 879, 880, 911, 912, 943, 944, 975, 976, 1007, 16, 16, 17, 48, + 49, 80, 81, 112, 113, 144, 145, 176, 177, 208, 209, 240, 241, 272, + 273, 304, 305, 336, 337, 368, 369, 400, 401, 432, 433, 464, 465, 496, + 497, 528, 529, 560, 561, 592, 593, 624, 625, 656, 657, 688, 689, 720, + 721, 752, 753, 784, 785, 816, 817, 848, 849, 880, 881, 912, 913, 944, + 945, 976, 977, 1008, 17, 17, 18, 49, 50, 81, 82, 113, 114, 145, + 146, 177, 178, 209, 210, 241, 242, 273, 274, 305, 306, 337, 338, 369, + 370, 401, 402, 433, 434, 465, 466, 497, 498, 529, 530, 561, 562, 593, + 594, 625, 626, 657, 658, 689, 690, 721, 722, 753, 754, 785, 786, 817, + 818, 849, 850, 881, 882, 913, 914, 945, 946, 977, 978, 1009, 18, 18, + 19, 50, 51, 82, 83, 114, 115, 146, 147, 178, 179, 210, 211, 242, + 243, 274, 275, 306, 307, 338, 339, 370, 371, 402, 403, 434, 435, 466, + 467, 498, 499, 530, 531, 562, 563, 594, 595, 626, 627, 658, 659, 690, + 691, 722, 723, 754, 755, 786, 787, 818, 819, 850, 851, 882, 883, 914, + 915, 946, 947, 978, 979, 1010, 19, 19, 20, 51, 52, 83, 84, 115, + 116, 147, 148, 179, 180, 211, 212, 243, 244, 275, 276, 307, 308, 339, + 340, 371, 372, 403, 404, 435, 436, 467, 468, 499, 500, 531, 532, 563, + 564, 595, 596, 627, 628, 659, 660, 691, 692, 723, 724, 755, 756, 787, + 788, 819, 820, 851, 852, 883, 884, 915, 916, 947, 948, 979, 980, 1011, + 20, 20, 21, 52, 53, 84, 85, 116, 117, 148, 149, 180, 181, 212, + 213, 244, 245, 276, 277, 308, 309, 340, 341, 372, 373, 404, 405, 436, + 437, 468, 469, 500, 501, 532, 533, 564, 565, 596, 597, 628, 629, 660, + 661, 692, 693, 724, 725, 756, 757, 788, 789, 820, 821, 852, 853, 884, + 885, 916, 917, 948, 949, 980, 981, 1012, 21, 21, 22, 53, 54, 85, + 86, 117, 118, 149, 150, 181, 182, 213, 214, 245, 246, 277, 278, 309, + 310, 341, 342, 373, 374, 405, 406, 437, 438, 469, 470, 501, 502, 533, + 534, 565, 566, 597, 598, 629, 630, 661, 662, 693, 694, 725, 726, 757, + 758, 789, 790, 821, 822, 853, 854, 885, 886, 917, 918, 949, 950, 981, + 982, 1013, 22, 22, 23, 54, 55, 86, 87, 118, 119, 150, 151, 182, + 183, 214, 215, 246, 247, 278, 279, 310, 311, 342, 343, 374, 375, 406, + 407, 438, 439, 470, 471, 502, 503, 534, 535, 566, 567, 598, 599, 630, + 631, 662, 663, 694, 695, 726, 727, 758, 759, 790, 791, 822, 823, 854, + 855, 886, 887, 918, 919, 950, 951, 982, 983, 1014, 23, 23, 24, 55, + 56, 87, 88, 119, 120, 151, 152, 183, 184, 215, 216, 247, 248, 279, + 280, 311, 312, 343, 344, 375, 376, 407, 408, 439, 440, 471, 472, 503, + 504, 535, 536, 567, 568, 599, 600, 631, 632, 663, 664, 695, 696, 727, + 728, 759, 760, 791, 792, 823, 824, 855, 856, 887, 888, 919, 920, 951, + 952, 983, 984, 1015, 24, 24, 25, 56, 57, 88, 89, 120, 121, 152, + 153, 184, 185, 216, 217, 248, 249, 280, 281, 312, 313, 344, 345, 376, + 377, 408, 409, 440, 441, 472, 473, 504, 505, 536, 537, 568, 569, 600, + 601, 632, 633, 664, 665, 696, 697, 728, 729, 760, 761, 792, 793, 824, + 825, 856, 857, 888, 889, 920, 921, 952, 953, 984, 985, 1016, 25, 25, + 26, 57, 58, 89, 90, 121, 122, 153, 154, 185, 186, 217, 218, 249, + 250, 281, 282, 313, 314, 345, 346, 377, 378, 409, 410, 441, 442, 473, + 474, 505, 506, 537, 538, 569, 570, 601, 602, 633, 634, 665, 666, 697, + 698, 729, 730, 761, 762, 793, 794, 825, 826, 857, 858, 889, 890, 921, + 922, 953, 954, 985, 986, 1017, 26, 26, 27, 58, 59, 90, 91, 122, + 123, 154, 155, 186, 187, 218, 219, 250, 251, 282, 283, 314, 315, 346, + 347, 378, 379, 410, 411, 442, 443, 474, 475, 506, 507, 538, 539, 570, + 571, 602, 603, 634, 635, 666, 667, 698, 699, 730, 731, 762, 763, 794, + 795, 826, 827, 858, 859, 890, 891, 922, 923, 954, 955, 986, 987, 1018, + 27, 27, 28, 59, 60, 91, 92, 123, 124, 155, 156, 187, 188, 219, + 220, 251, 252, 283, 284, 315, 316, 347, 348, 379, 380, 411, 412, 443, + 444, 475, 476, 507, 508, 539, 540, 571, 572, 603, 604, 635, 636, 667, + 668, 699, 700, 731, 732, 763, 764, 795, 796, 827, 828, 859, 860, 891, + 892, 923, 924, 955, 956, 987, 988, 1019, 28, 28, 29, 60, 61, 92, + 93, 124, 125, 156, 157, 188, 189, 220, 221, 252, 253, 284, 285, 316, + 317, 348, 349, 380, 381, 412, 413, 444, 445, 476, 477, 508, 509, 540, + 541, 572, 573, 604, 605, 636, 637, 668, 669, 700, 701, 732, 733, 764, + 765, 796, 797, 828, 829, 860, 861, 892, 893, 924, 925, 956, 957, 988, + 989, 1020, 29, 29, 30, 61, 62, 93, 94, 125, 126, 157, 158, 189, + 190, 221, 222, 253, 254, 285, 286, 317, 318, 349, 350, 381, 382, 413, + 414, 445, 446, 477, 478, 509, 510, 541, 542, 573, 574, 605, 606, 637, + 638, 669, 670, 701, 702, 733, 734, 765, 766, 797, 798, 829, 830, 861, + 862, 893, 894, 925, 926, 957, 958, 989, 990, 1021, 30, 30, 31, 62, + 63, 94, 95, 126, 127, 158, 159, 190, 191, 222, 223, 254, 255, 286, + 287, 318, 319, 350, 351, 382, 383, 414, 415, 446, 447, 478, 479, 510, + 511, 542, 543, 574, 575, 606, 607, 638, 639, 670, 671, 702, 703, 734, + 735, 766, 767, 798, 799, 830, 831, 862, 863, 894, 895, 926, 927, 958, + 959, 990, 991, 1022, 0, 0, +}; + +DECLARE_ALIGNED(16, static const int16_t, + mrow_scan_32x32_neighbors[1025 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, + 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, + 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 21, 21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, + 27, 27, 28, 28, 29, 29, 30, 30, 0, 0, 1, 32, 2, 33, + 3, 34, 4, 35, 5, 36, 6, 37, 7, 38, 8, 39, 9, 40, + 10, 41, 11, 42, 12, 43, 13, 44, 14, 45, 15, 46, 16, 47, + 17, 48, 18, 49, 19, 50, 20, 51, 21, 52, 22, 53, 23, 54, + 24, 55, 25, 56, 26, 57, 27, 58, 28, 59, 29, 60, 30, 61, + 31, 62, 32, 32, 33, 64, 34, 65, 35, 66, 36, 67, 37, 68, + 38, 69, 39, 70, 40, 71, 41, 72, 42, 73, 43, 74, 44, 75, + 45, 76, 46, 77, 47, 78, 48, 79, 49, 80, 50, 81, 51, 82, + 52, 83, 53, 84, 54, 85, 55, 86, 56, 87, 57, 88, 58, 89, + 59, 90, 60, 91, 61, 92, 62, 93, 63, 94, 64, 64, 65, 96, + 66, 97, 67, 98, 68, 99, 69, 100, 70, 101, 71, 102, 72, 103, + 73, 104, 74, 105, 75, 106, 76, 107, 77, 108, 78, 109, 79, 110, + 80, 111, 81, 112, 82, 113, 83, 114, 84, 115, 85, 116, 86, 117, + 87, 118, 88, 119, 89, 120, 90, 121, 91, 122, 92, 123, 93, 124, + 94, 125, 95, 126, 96, 96, 97, 128, 98, 129, 99, 130, 100, 131, + 101, 132, 102, 133, 103, 134, 104, 135, 105, 136, 106, 137, 107, 138, + 108, 139, 109, 140, 110, 141, 111, 142, 112, 143, 113, 144, 114, 145, + 115, 146, 116, 147, 117, 148, 118, 149, 119, 150, 120, 151, 121, 152, + 122, 153, 123, 154, 124, 155, 125, 156, 126, 157, 127, 158, 128, 128, + 129, 160, 130, 161, 131, 162, 132, 163, 133, 164, 134, 165, 135, 166, + 136, 167, 137, 168, 138, 169, 139, 170, 140, 171, 141, 172, 142, 173, + 143, 174, 144, 175, 145, 176, 146, 177, 147, 178, 148, 179, 149, 180, + 150, 181, 151, 182, 152, 183, 153, 184, 154, 185, 155, 186, 156, 187, + 157, 188, 158, 189, 159, 190, 160, 160, 161, 192, 162, 193, 163, 194, + 164, 195, 165, 196, 166, 197, 167, 198, 168, 199, 169, 200, 170, 201, + 171, 202, 172, 203, 173, 204, 174, 205, 175, 206, 176, 207, 177, 208, + 178, 209, 179, 210, 180, 211, 181, 212, 182, 213, 183, 214, 184, 215, + 185, 216, 186, 217, 187, 218, 188, 219, 189, 220, 190, 221, 191, 222, + 192, 192, 193, 224, 194, 225, 195, 226, 196, 227, 197, 228, 198, 229, + 199, 230, 200, 231, 201, 232, 202, 233, 203, 234, 204, 235, 205, 236, + 206, 237, 207, 238, 208, 239, 209, 240, 210, 241, 211, 242, 212, 243, + 213, 244, 214, 245, 215, 246, 216, 247, 217, 248, 218, 249, 219, 250, + 220, 251, 221, 252, 222, 253, 223, 254, 224, 224, 225, 256, 226, 257, + 227, 258, 228, 259, 229, 260, 230, 261, 231, 262, 232, 263, 233, 264, + 234, 265, 235, 266, 236, 267, 237, 268, 238, 269, 239, 270, 240, 271, + 241, 272, 242, 273, 243, 274, 244, 275, 245, 276, 246, 277, 247, 278, + 248, 279, 249, 280, 250, 281, 251, 282, 252, 283, 253, 284, 254, 285, + 255, 286, 256, 256, 257, 288, 258, 289, 259, 290, 260, 291, 261, 292, + 262, 293, 263, 294, 264, 295, 265, 296, 266, 297, 267, 298, 268, 299, + 269, 300, 270, 301, 271, 302, 272, 303, 273, 304, 274, 305, 275, 306, + 276, 307, 277, 308, 278, 309, 279, 310, 280, 311, 281, 312, 282, 313, + 283, 314, 284, 315, 285, 316, 286, 317, 287, 318, 288, 288, 289, 320, + 290, 321, 291, 322, 292, 323, 293, 324, 294, 325, 295, 326, 296, 327, + 297, 328, 298, 329, 299, 330, 300, 331, 301, 332, 302, 333, 303, 334, + 304, 335, 305, 336, 306, 337, 307, 338, 308, 339, 309, 340, 310, 341, + 311, 342, 312, 343, 313, 344, 314, 345, 315, 346, 316, 347, 317, 348, + 318, 349, 319, 350, 320, 320, 321, 352, 322, 353, 323, 354, 324, 355, + 325, 356, 326, 357, 327, 358, 328, 359, 329, 360, 330, 361, 331, 362, + 332, 363, 333, 364, 334, 365, 335, 366, 336, 367, 337, 368, 338, 369, + 339, 370, 340, 371, 341, 372, 342, 373, 343, 374, 344, 375, 345, 376, + 346, 377, 347, 378, 348, 379, 349, 380, 350, 381, 351, 382, 352, 352, + 353, 384, 354, 385, 355, 386, 356, 387, 357, 388, 358, 389, 359, 390, + 360, 391, 361, 392, 362, 393, 363, 394, 364, 395, 365, 396, 366, 397, + 367, 398, 368, 399, 369, 400, 370, 401, 371, 402, 372, 403, 373, 404, + 374, 405, 375, 406, 376, 407, 377, 408, 378, 409, 379, 410, 380, 411, + 381, 412, 382, 413, 383, 414, 384, 384, 385, 416, 386, 417, 387, 418, + 388, 419, 389, 420, 390, 421, 391, 422, 392, 423, 393, 424, 394, 425, + 395, 426, 396, 427, 397, 428, 398, 429, 399, 430, 400, 431, 401, 432, + 402, 433, 403, 434, 404, 435, 405, 436, 406, 437, 407, 438, 408, 439, + 409, 440, 410, 441, 411, 442, 412, 443, 413, 444, 414, 445, 415, 446, + 416, 416, 417, 448, 418, 449, 419, 450, 420, 451, 421, 452, 422, 453, + 423, 454, 424, 455, 425, 456, 426, 457, 427, 458, 428, 459, 429, 460, + 430, 461, 431, 462, 432, 463, 433, 464, 434, 465, 435, 466, 436, 467, + 437, 468, 438, 469, 439, 470, 440, 471, 441, 472, 442, 473, 443, 474, + 444, 475, 445, 476, 446, 477, 447, 478, 448, 448, 449, 480, 450, 481, + 451, 482, 452, 483, 453, 484, 454, 485, 455, 486, 456, 487, 457, 488, + 458, 489, 459, 490, 460, 491, 461, 492, 462, 493, 463, 494, 464, 495, + 465, 496, 466, 497, 467, 498, 468, 499, 469, 500, 470, 501, 471, 502, + 472, 503, 473, 504, 474, 505, 475, 506, 476, 507, 477, 508, 478, 509, + 479, 510, 480, 480, 481, 512, 482, 513, 483, 514, 484, 515, 485, 516, + 486, 517, 487, 518, 488, 519, 489, 520, 490, 521, 491, 522, 492, 523, + 493, 524, 494, 525, 495, 526, 496, 527, 497, 528, 498, 529, 499, 530, + 500, 531, 501, 532, 502, 533, 503, 534, 504, 535, 505, 536, 506, 537, + 507, 538, 508, 539, 509, 540, 510, 541, 511, 542, 512, 512, 513, 544, + 514, 545, 515, 546, 516, 547, 517, 548, 518, 549, 519, 550, 520, 551, + 521, 552, 522, 553, 523, 554, 524, 555, 525, 556, 526, 557, 527, 558, + 528, 559, 529, 560, 530, 561, 531, 562, 532, 563, 533, 564, 534, 565, + 535, 566, 536, 567, 537, 568, 538, 569, 539, 570, 540, 571, 541, 572, + 542, 573, 543, 574, 544, 544, 545, 576, 546, 577, 547, 578, 548, 579, + 549, 580, 550, 581, 551, 582, 552, 583, 553, 584, 554, 585, 555, 586, + 556, 587, 557, 588, 558, 589, 559, 590, 560, 591, 561, 592, 562, 593, + 563, 594, 564, 595, 565, 596, 566, 597, 567, 598, 568, 599, 569, 600, + 570, 601, 571, 602, 572, 603, 573, 604, 574, 605, 575, 606, 576, 576, + 577, 608, 578, 609, 579, 610, 580, 611, 581, 612, 582, 613, 583, 614, + 584, 615, 585, 616, 586, 617, 587, 618, 588, 619, 589, 620, 590, 621, + 591, 622, 592, 623, 593, 624, 594, 625, 595, 626, 596, 627, 597, 628, + 598, 629, 599, 630, 600, 631, 601, 632, 602, 633, 603, 634, 604, 635, + 605, 636, 606, 637, 607, 638, 608, 608, 609, 640, 610, 641, 611, 642, + 612, 643, 613, 644, 614, 645, 615, 646, 616, 647, 617, 648, 618, 649, + 619, 650, 620, 651, 621, 652, 622, 653, 623, 654, 624, 655, 625, 656, + 626, 657, 627, 658, 628, 659, 629, 660, 630, 661, 631, 662, 632, 663, + 633, 664, 634, 665, 635, 666, 636, 667, 637, 668, 638, 669, 639, 670, + 640, 640, 641, 672, 642, 673, 643, 674, 644, 675, 645, 676, 646, 677, + 647, 678, 648, 679, 649, 680, 650, 681, 651, 682, 652, 683, 653, 684, + 654, 685, 655, 686, 656, 687, 657, 688, 658, 689, 659, 690, 660, 691, + 661, 692, 662, 693, 663, 694, 664, 695, 665, 696, 666, 697, 667, 698, + 668, 699, 669, 700, 670, 701, 671, 702, 672, 672, 673, 704, 674, 705, + 675, 706, 676, 707, 677, 708, 678, 709, 679, 710, 680, 711, 681, 712, + 682, 713, 683, 714, 684, 715, 685, 716, 686, 717, 687, 718, 688, 719, + 689, 720, 690, 721, 691, 722, 692, 723, 693, 724, 694, 725, 695, 726, + 696, 727, 697, 728, 698, 729, 699, 730, 700, 731, 701, 732, 702, 733, + 703, 734, 704, 704, 705, 736, 706, 737, 707, 738, 708, 739, 709, 740, + 710, 741, 711, 742, 712, 743, 713, 744, 714, 745, 715, 746, 716, 747, + 717, 748, 718, 749, 719, 750, 720, 751, 721, 752, 722, 753, 723, 754, + 724, 755, 725, 756, 726, 757, 727, 758, 728, 759, 729, 760, 730, 761, + 731, 762, 732, 763, 733, 764, 734, 765, 735, 766, 736, 736, 737, 768, + 738, 769, 739, 770, 740, 771, 741, 772, 742, 773, 743, 774, 744, 775, + 745, 776, 746, 777, 747, 778, 748, 779, 749, 780, 750, 781, 751, 782, + 752, 783, 753, 784, 754, 785, 755, 786, 756, 787, 757, 788, 758, 789, + 759, 790, 760, 791, 761, 792, 762, 793, 763, 794, 764, 795, 765, 796, + 766, 797, 767, 798, 768, 768, 769, 800, 770, 801, 771, 802, 772, 803, + 773, 804, 774, 805, 775, 806, 776, 807, 777, 808, 778, 809, 779, 810, + 780, 811, 781, 812, 782, 813, 783, 814, 784, 815, 785, 816, 786, 817, + 787, 818, 788, 819, 789, 820, 790, 821, 791, 822, 792, 823, 793, 824, + 794, 825, 795, 826, 796, 827, 797, 828, 798, 829, 799, 830, 800, 800, + 801, 832, 802, 833, 803, 834, 804, 835, 805, 836, 806, 837, 807, 838, + 808, 839, 809, 840, 810, 841, 811, 842, 812, 843, 813, 844, 814, 845, + 815, 846, 816, 847, 817, 848, 818, 849, 819, 850, 820, 851, 821, 852, + 822, 853, 823, 854, 824, 855, 825, 856, 826, 857, 827, 858, 828, 859, + 829, 860, 830, 861, 831, 862, 832, 832, 833, 864, 834, 865, 835, 866, + 836, 867, 837, 868, 838, 869, 839, 870, 840, 871, 841, 872, 842, 873, + 843, 874, 844, 875, 845, 876, 846, 877, 847, 878, 848, 879, 849, 880, + 850, 881, 851, 882, 852, 883, 853, 884, 854, 885, 855, 886, 856, 887, + 857, 888, 858, 889, 859, 890, 860, 891, 861, 892, 862, 893, 863, 894, + 864, 864, 865, 896, 866, 897, 867, 898, 868, 899, 869, 900, 870, 901, + 871, 902, 872, 903, 873, 904, 874, 905, 875, 906, 876, 907, 877, 908, + 878, 909, 879, 910, 880, 911, 881, 912, 882, 913, 883, 914, 884, 915, + 885, 916, 886, 917, 887, 918, 888, 919, 889, 920, 890, 921, 891, 922, + 892, 923, 893, 924, 894, 925, 895, 926, 896, 896, 897, 928, 898, 929, + 899, 930, 900, 931, 901, 932, 902, 933, 903, 934, 904, 935, 905, 936, + 906, 937, 907, 938, 908, 939, 909, 940, 910, 941, 911, 942, 912, 943, + 913, 944, 914, 945, 915, 946, 916, 947, 917, 948, 918, 949, 919, 950, + 920, 951, 921, 952, 922, 953, 923, 954, 924, 955, 925, 956, 926, 957, + 927, 958, 928, 928, 929, 960, 930, 961, 931, 962, 932, 963, 933, 964, + 934, 965, 935, 966, 936, 967, 937, 968, 938, 969, 939, 970, 940, 971, + 941, 972, 942, 973, 943, 974, 944, 975, 945, 976, 946, 977, 947, 978, + 948, 979, 949, 980, 950, 981, 951, 982, 952, 983, 953, 984, 954, 985, + 955, 986, 956, 987, 957, 988, 958, 989, 959, 990, 960, 960, 961, 992, + 962, 993, 963, 994, 964, 995, 965, 996, 966, 997, 967, 998, 968, 999, + 969, 1000, 970, 1001, 971, 1002, 972, 1003, 973, 1004, 974, 1005, 975, 1006, + 976, 1007, 977, 1008, 978, 1009, 979, 1010, 980, 1011, 981, 1012, 982, 1013, + 983, 1014, 984, 1015, 985, 1016, 986, 1017, 987, 1018, 988, 1019, 989, 1020, + 990, 1021, 991, 1022, 0, 0, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, + default_scan_32x32_neighbors[1025 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 32, 0, 32, 32, 1, 32, 33, 1, 64, 64, + 33, 64, 2, 33, 96, 96, 34, 2, 65, 96, 34, 65, 128, 128, + 97, 128, 3, 34, 66, 97, 35, 3, 35, 66, 98, 129, 129, 160, + 160, 161, 4, 35, 67, 98, 192, 192, 36, 4, 130, 161, 161, 192, + 36, 67, 99, 130, 5, 36, 68, 99, 193, 224, 162, 193, 224, 225, + 131, 162, 37, 68, 100, 131, 37, 5, 194, 225, 225, 256, 256, 257, + 163, 194, 69, 100, 132, 163, 6, 37, 226, 257, 38, 6, 195, 226, + 257, 288, 101, 132, 288, 289, 38, 69, 164, 195, 133, 164, 258, 289, + 227, 258, 196, 227, 7, 38, 289, 320, 70, 101, 320, 321, 39, 7, + 165, 196, 39, 70, 102, 133, 290, 321, 259, 290, 228, 259, 321, 352, + 352, 353, 197, 228, 134, 165, 71, 102, 8, 39, 322, 353, 291, 322, + 260, 291, 103, 134, 353, 384, 166, 197, 229, 260, 40, 71, 40, 8, + 384, 385, 135, 166, 354, 385, 323, 354, 198, 229, 292, 323, 72, 103, + 261, 292, 9, 40, 385, 416, 167, 198, 104, 135, 230, 261, 355, 386, + 416, 417, 293, 324, 324, 355, 41, 9, 41, 72, 386, 417, 199, 230, + 136, 167, 417, 448, 262, 293, 356, 387, 73, 104, 387, 418, 231, 262, + 10, 41, 168, 199, 325, 356, 418, 449, 105, 136, 448, 449, 42, 73, + 294, 325, 200, 231, 42, 10, 357, 388, 137, 168, 263, 294, 388, 419, + 74, 105, 419, 450, 449, 480, 326, 357, 232, 263, 295, 326, 169, 200, + 11, 42, 106, 137, 480, 481, 450, 481, 358, 389, 264, 295, 201, 232, + 138, 169, 389, 420, 43, 74, 420, 451, 327, 358, 43, 11, 481, 512, + 233, 264, 451, 482, 296, 327, 75, 106, 170, 201, 482, 513, 512, 513, + 390, 421, 359, 390, 421, 452, 107, 138, 12, 43, 202, 233, 452, 483, + 265, 296, 328, 359, 139, 170, 44, 75, 483, 514, 513, 544, 234, 265, + 297, 328, 422, 453, 44, 12, 391, 422, 171, 202, 76, 107, 514, 545, + 453, 484, 544, 545, 266, 297, 203, 234, 108, 139, 329, 360, 298, 329, + 140, 171, 515, 546, 13, 44, 423, 454, 235, 266, 545, 576, 454, 485, + 45, 76, 172, 203, 330, 361, 576, 577, 45, 13, 267, 298, 546, 577, + 77, 108, 204, 235, 455, 486, 577, 608, 299, 330, 109, 140, 547, 578, + 14, 45, 46, 14, 141, 172, 578, 609, 331, 362, 46, 77, 173, 204, + 15, 15, 78, 109, 205, 236, 579, 610, 110, 141, 15, 46, 142, 173, + 47, 78, 174, 205, 16, 16, 79, 110, 206, 237, 16, 47, 111, 142, + 48, 79, 143, 174, 80, 111, 175, 206, 17, 48, 49, 17, 207, 238, + 49, 80, 81, 112, 18, 18, 18, 49, 50, 81, 82, 113, 19, 50, + 51, 82, 83, 114, 608, 609, 484, 515, 360, 391, 236, 267, 112, 143, + 51, 19, 640, 640, 609, 640, 516, 547, 485, 516, 392, 423, 361, 392, + 268, 299, 237, 268, 144, 175, 113, 144, 20, 51, 52, 20, 672, 672, + 641, 672, 610, 641, 548, 579, 517, 548, 486, 517, 424, 455, 393, 424, + 362, 393, 300, 331, 269, 300, 238, 269, 176, 207, 145, 176, 114, 145, + 52, 83, 21, 52, 53, 21, 704, 704, 673, 704, 642, 673, 611, 642, + 580, 611, 549, 580, 518, 549, 487, 518, 456, 487, 425, 456, 394, 425, + 363, 394, 332, 363, 301, 332, 270, 301, 239, 270, 208, 239, 177, 208, + 146, 177, 115, 146, 84, 115, 53, 84, 22, 53, 54, 22, 705, 736, + 674, 705, 643, 674, 581, 612, 550, 581, 519, 550, 457, 488, 426, 457, + 395, 426, 333, 364, 302, 333, 271, 302, 209, 240, 178, 209, 147, 178, + 85, 116, 54, 85, 23, 54, 706, 737, 675, 706, 582, 613, 551, 582, + 458, 489, 427, 458, 334, 365, 303, 334, 210, 241, 179, 210, 86, 117, + 55, 86, 707, 738, 583, 614, 459, 490, 335, 366, 211, 242, 87, 118, + 736, 737, 612, 643, 488, 519, 364, 395, 240, 271, 116, 147, 55, 23, + 768, 768, 737, 768, 644, 675, 613, 644, 520, 551, 489, 520, 396, 427, + 365, 396, 272, 303, 241, 272, 148, 179, 117, 148, 24, 55, 56, 24, + 800, 800, 769, 800, 738, 769, 676, 707, 645, 676, 614, 645, 552, 583, + 521, 552, 490, 521, 428, 459, 397, 428, 366, 397, 304, 335, 273, 304, + 242, 273, 180, 211, 149, 180, 118, 149, 56, 87, 25, 56, 57, 25, + 832, 832, 801, 832, 770, 801, 739, 770, 708, 739, 677, 708, 646, 677, + 615, 646, 584, 615, 553, 584, 522, 553, 491, 522, 460, 491, 429, 460, + 398, 429, 367, 398, 336, 367, 305, 336, 274, 305, 243, 274, 212, 243, + 181, 212, 150, 181, 119, 150, 88, 119, 57, 88, 26, 57, 58, 26, + 833, 864, 802, 833, 771, 802, 709, 740, 678, 709, 647, 678, 585, 616, + 554, 585, 523, 554, 461, 492, 430, 461, 399, 430, 337, 368, 306, 337, + 275, 306, 213, 244, 182, 213, 151, 182, 89, 120, 58, 89, 27, 58, + 834, 865, 803, 834, 710, 741, 679, 710, 586, 617, 555, 586, 462, 493, + 431, 462, 338, 369, 307, 338, 214, 245, 183, 214, 90, 121, 59, 90, + 835, 866, 711, 742, 587, 618, 463, 494, 339, 370, 215, 246, 91, 122, + 864, 865, 740, 771, 616, 647, 492, 523, 368, 399, 244, 275, 120, 151, + 59, 27, 896, 896, 865, 896, 772, 803, 741, 772, 648, 679, 617, 648, + 524, 555, 493, 524, 400, 431, 369, 400, 276, 307, 245, 276, 152, 183, + 121, 152, 28, 59, 60, 28, 928, 928, 897, 928, 866, 897, 804, 835, + 773, 804, 742, 773, 680, 711, 649, 680, 618, 649, 556, 587, 525, 556, + 494, 525, 432, 463, 401, 432, 370, 401, 308, 339, 277, 308, 246, 277, + 184, 215, 153, 184, 122, 153, 60, 91, 29, 60, 61, 29, 960, 960, + 929, 960, 898, 929, 867, 898, 836, 867, 805, 836, 774, 805, 743, 774, + 712, 743, 681, 712, 650, 681, 619, 650, 588, 619, 557, 588, 526, 557, + 495, 526, 464, 495, 433, 464, 402, 433, 371, 402, 340, 371, 309, 340, + 278, 309, 247, 278, 216, 247, 185, 216, 154, 185, 123, 154, 92, 123, + 61, 92, 30, 61, 62, 30, 961, 992, 930, 961, 899, 930, 837, 868, + 806, 837, 775, 806, 713, 744, 682, 713, 651, 682, 589, 620, 558, 589, + 527, 558, 465, 496, 434, 465, 403, 434, 341, 372, 310, 341, 279, 310, + 217, 248, 186, 217, 155, 186, 93, 124, 62, 93, 31, 62, 962, 993, + 931, 962, 838, 869, 807, 838, 714, 745, 683, 714, 590, 621, 559, 590, + 466, 497, 435, 466, 342, 373, 311, 342, 218, 249, 187, 218, 94, 125, + 63, 94, 963, 994, 839, 870, 715, 746, 591, 622, 467, 498, 343, 374, + 219, 250, 95, 126, 868, 899, 744, 775, 620, 651, 496, 527, 372, 403, + 248, 279, 124, 155, 900, 931, 869, 900, 776, 807, 745, 776, 652, 683, + 621, 652, 528, 559, 497, 528, 404, 435, 373, 404, 280, 311, 249, 280, + 156, 187, 125, 156, 932, 963, 901, 932, 870, 901, 808, 839, 777, 808, + 746, 777, 684, 715, 653, 684, 622, 653, 560, 591, 529, 560, 498, 529, + 436, 467, 405, 436, 374, 405, 312, 343, 281, 312, 250, 281, 188, 219, + 157, 188, 126, 157, 964, 995, 933, 964, 902, 933, 871, 902, 840, 871, + 809, 840, 778, 809, 747, 778, 716, 747, 685, 716, 654, 685, 623, 654, + 592, 623, 561, 592, 530, 561, 499, 530, 468, 499, 437, 468, 406, 437, + 375, 406, 344, 375, 313, 344, 282, 313, 251, 282, 220, 251, 189, 220, + 158, 189, 127, 158, 965, 996, 934, 965, 903, 934, 841, 872, 810, 841, + 779, 810, 717, 748, 686, 717, 655, 686, 593, 624, 562, 593, 531, 562, + 469, 500, 438, 469, 407, 438, 345, 376, 314, 345, 283, 314, 221, 252, + 190, 221, 159, 190, 966, 997, 935, 966, 842, 873, 811, 842, 718, 749, + 687, 718, 594, 625, 563, 594, 470, 501, 439, 470, 346, 377, 315, 346, + 222, 253, 191, 222, 967, 998, 843, 874, 719, 750, 595, 626, 471, 502, + 347, 378, 223, 254, 872, 903, 748, 779, 624, 655, 500, 531, 376, 407, + 252, 283, 904, 935, 873, 904, 780, 811, 749, 780, 656, 687, 625, 656, + 532, 563, 501, 532, 408, 439, 377, 408, 284, 315, 253, 284, 936, 967, + 905, 936, 874, 905, 812, 843, 781, 812, 750, 781, 688, 719, 657, 688, + 626, 657, 564, 595, 533, 564, 502, 533, 440, 471, 409, 440, 378, 409, + 316, 347, 285, 316, 254, 285, 968, 999, 937, 968, 906, 937, 875, 906, + 844, 875, 813, 844, 782, 813, 751, 782, 720, 751, 689, 720, 658, 689, + 627, 658, 596, 627, 565, 596, 534, 565, 503, 534, 472, 503, 441, 472, + 410, 441, 379, 410, 348, 379, 317, 348, 286, 317, 255, 286, 969, 1000, + 938, 969, 907, 938, 845, 876, 814, 845, 783, 814, 721, 752, 690, 721, + 659, 690, 597, 628, 566, 597, 535, 566, 473, 504, 442, 473, 411, 442, + 349, 380, 318, 349, 287, 318, 970, 1001, 939, 970, 846, 877, 815, 846, + 722, 753, 691, 722, 598, 629, 567, 598, 474, 505, 443, 474, 350, 381, + 319, 350, 971, 1002, 847, 878, 723, 754, 599, 630, 475, 506, 351, 382, + 876, 907, 752, 783, 628, 659, 504, 535, 380, 411, 908, 939, 877, 908, + 784, 815, 753, 784, 660, 691, 629, 660, 536, 567, 505, 536, 412, 443, + 381, 412, 940, 971, 909, 940, 878, 909, 816, 847, 785, 816, 754, 785, + 692, 723, 661, 692, 630, 661, 568, 599, 537, 568, 506, 537, 444, 475, + 413, 444, 382, 413, 972, 1003, 941, 972, 910, 941, 879, 910, 848, 879, + 817, 848, 786, 817, 755, 786, 724, 755, 693, 724, 662, 693, 631, 662, + 600, 631, 569, 600, 538, 569, 507, 538, 476, 507, 445, 476, 414, 445, + 383, 414, 973, 1004, 942, 973, 911, 942, 849, 880, 818, 849, 787, 818, + 725, 756, 694, 725, 663, 694, 601, 632, 570, 601, 539, 570, 477, 508, + 446, 477, 415, 446, 974, 1005, 943, 974, 850, 881, 819, 850, 726, 757, + 695, 726, 602, 633, 571, 602, 478, 509, 447, 478, 975, 1006, 851, 882, + 727, 758, 603, 634, 479, 510, 880, 911, 756, 787, 632, 663, 508, 539, + 912, 943, 881, 912, 788, 819, 757, 788, 664, 695, 633, 664, 540, 571, + 509, 540, 944, 975, 913, 944, 882, 913, 820, 851, 789, 820, 758, 789, + 696, 727, 665, 696, 634, 665, 572, 603, 541, 572, 510, 541, 976, 1007, + 945, 976, 914, 945, 883, 914, 852, 883, 821, 852, 790, 821, 759, 790, + 728, 759, 697, 728, 666, 697, 635, 666, 604, 635, 573, 604, 542, 573, + 511, 542, 977, 1008, 946, 977, 915, 946, 853, 884, 822, 853, 791, 822, + 729, 760, 698, 729, 667, 698, 605, 636, 574, 605, 543, 574, 978, 1009, + 947, 978, 854, 885, 823, 854, 730, 761, 699, 730, 606, 637, 575, 606, + 979, 1010, 855, 886, 731, 762, 607, 638, 884, 915, 760, 791, 636, 667, + 916, 947, 885, 916, 792, 823, 761, 792, 668, 699, 637, 668, 948, 979, + 917, 948, 886, 917, 824, 855, 793, 824, 762, 793, 700, 731, 669, 700, + 638, 669, 980, 1011, 949, 980, 918, 949, 887, 918, 856, 887, 825, 856, + 794, 825, 763, 794, 732, 763, 701, 732, 670, 701, 639, 670, 981, 1012, + 950, 981, 919, 950, 857, 888, 826, 857, 795, 826, 733, 764, 702, 733, + 671, 702, 982, 1013, 951, 982, 858, 889, 827, 858, 734, 765, 703, 734, + 983, 1014, 859, 890, 735, 766, 888, 919, 764, 795, 920, 951, 889, 920, + 796, 827, 765, 796, 952, 983, 921, 952, 890, 921, 828, 859, 797, 828, + 766, 797, 984, 1015, 953, 984, 922, 953, 891, 922, 860, 891, 829, 860, + 798, 829, 767, 798, 985, 1016, 954, 985, 923, 954, 861, 892, 830, 861, + 799, 830, 986, 1017, 955, 986, 862, 893, 831, 862, 987, 1018, 863, 894, + 892, 923, 924, 955, 893, 924, 956, 987, 925, 956, 894, 925, 988, 1019, + 957, 988, 926, 957, 895, 926, 989, 1020, 958, 989, 927, 958, 990, 1021, + 959, 990, 991, 1022, 0, 0, +}; + +DECLARE_ALIGNED(16, static const int16_t, + v2_scan_32x32_neighbors[1025 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 32, 1, 1, 32, 32, 2, 33, + 33, 64, 34, 65, 2, 2, 64, 64, 3, 34, 65, 96, 35, 66, + 66, 97, 3, 3, 96, 96, 4, 35, 97, 128, 67, 98, 36, 67, + 98, 129, 4, 4, 68, 99, 99, 130, 128, 128, 5, 36, 129, 160, + 37, 68, 130, 161, 100, 131, 69, 100, 131, 162, 5, 5, 160, 160, + 6, 37, 161, 192, 38, 69, 162, 193, 101, 132, 132, 163, 70, 101, + 163, 194, 6, 6, 192, 192, 7, 38, 133, 164, 193, 224, 102, 133, + 164, 195, 39, 70, 194, 225, 71, 102, 195, 226, 134, 165, 165, 196, + 7, 7, 224, 224, 8, 39, 103, 134, 196, 227, 225, 256, 40, 71, + 226, 257, 166, 197, 72, 103, 227, 258, 135, 166, 197, 228, 104, 135, + 228, 259, 8, 8, 256, 256, 9, 40, 257, 288, 41, 72, 167, 198, + 198, 229, 258, 289, 136, 167, 229, 260, 73, 104, 259, 290, 105, 136, + 260, 291, 199, 230, 9, 9, 168, 199, 230, 261, 288, 288, 10, 41, + 289, 320, 42, 73, 290, 321, 137, 168, 261, 292, 74, 105, 291, 322, + 200, 231, 231, 262, 106, 137, 292, 323, 169, 200, 262, 293, 10, 10, + 320, 320, 11, 42, 321, 352, 43, 74, 138, 169, 293, 324, 322, 353, + 232, 263, 75, 106, 201, 232, 263, 294, 323, 354, 170, 201, 294, 325, + 107, 138, 324, 355, 11, 11, 352, 352, 12, 43, 233, 264, 264, 295, + 353, 384, 139, 170, 325, 356, 44, 75, 354, 385, 202, 233, 295, 326, + 76, 107, 355, 386, 171, 202, 326, 357, 108, 139, 356, 387, 265, 296, + 234, 265, 296, 327, 12, 12, 140, 171, 357, 388, 384, 384, 13, 44, + 203, 234, 327, 358, 385, 416, 45, 76, 386, 417, 77, 108, 387, 418, + 172, 203, 358, 389, 266, 297, 297, 328, 109, 140, 235, 266, 328, 359, + 388, 419, 204, 235, 359, 390, 141, 172, 389, 420, 13, 13, 416, 416, + 14, 45, 417, 448, 46, 77, 298, 329, 418, 449, 267, 298, 329, 360, + 78, 109, 173, 204, 390, 421, 419, 450, 236, 267, 360, 391, 110, 141, + 420, 451, 205, 236, 391, 422, 142, 173, 299, 330, 330, 361, 421, 452, + 14, 14, 268, 299, 361, 392, 448, 448, 15, 46, 449, 480, 47, 78, + 450, 481, 174, 205, 422, 453, 237, 268, 392, 423, 79, 110, 451, 482, + 111, 142, 452, 483, 331, 362, 300, 331, 362, 393, 206, 237, 423, 454, + 143, 174, 269, 300, 393, 424, 453, 484, 480, 480, 481, 512, 238, 269, + 424, 455, 482, 513, 175, 206, 454, 485, 332, 363, 363, 394, 483, 514, + 301, 332, 394, 425, 484, 515, 207, 238, 455, 486, 270, 301, 425, 456, + 485, 516, 364, 395, 239, 270, 456, 487, 512, 512, 333, 364, 395, 426, + 513, 544, 486, 517, 514, 545, 302, 333, 426, 457, 515, 546, 487, 518, + 516, 547, 271, 302, 457, 488, 365, 396, 396, 427, 517, 548, 334, 365, + 427, 458, 488, 519, 544, 544, 303, 334, 458, 489, 518, 549, 545, 576, + 546, 577, 547, 578, 489, 520, 397, 428, 519, 550, 366, 397, 428, 459, + 548, 579, 335, 366, 459, 490, 549, 580, 520, 551, 490, 521, 550, 581, + 576, 576, 577, 608, 398, 429, 429, 460, 578, 609, 367, 398, 460, 491, + 521, 552, 579, 610, 551, 582, 491, 522, 580, 611, 581, 612, 552, 583, + 522, 553, 430, 461, 399, 430, 461, 492, 582, 613, 492, 523, 608, 608, + 609, 640, 610, 641, 553, 584, 611, 642, 523, 554, 583, 614, 612, 643, + 431, 462, 462, 493, 554, 585, 493, 524, 584, 615, 613, 644, 524, 555, + 614, 645, 640, 640, 585, 616, 641, 672, 555, 586, 642, 673, 615, 646, + 463, 494, 643, 674, 494, 525, 644, 675, 525, 556, 586, 617, 616, 647, + 645, 676, 556, 587, 646, 677, 495, 526, 617, 648, 587, 618, 672, 672, + 526, 557, 673, 704, 674, 705, 647, 678, 557, 588, 675, 706, 618, 649, + 676, 707, 588, 619, 648, 679, 677, 708, 527, 558, 558, 589, 678, 709, + 619, 650, 649, 680, 704, 704, 589, 620, 705, 736, 679, 710, 706, 737, + 707, 738, 650, 681, 620, 651, 708, 739, 680, 711, 559, 590, 709, 740, + 590, 621, 651, 682, 681, 712, 710, 741, 621, 652, 736, 736, 737, 768, + 711, 742, 738, 769, 682, 713, 652, 683, 739, 770, 591, 622, 740, 771, + 712, 743, 622, 653, 741, 772, 683, 714, 653, 684, 713, 744, 742, 773, + 623, 654, 743, 774, 768, 768, 769, 800, 684, 715, 714, 745, 770, 801, + 771, 802, 654, 685, 744, 775, 772, 803, 715, 746, 773, 804, 685, 716, + 745, 776, 774, 805, 655, 686, 716, 747, 775, 806, 746, 777, 800, 800, + 801, 832, 686, 717, 802, 833, 803, 834, 776, 807, 804, 835, 747, 778, + 717, 748, 805, 836, 777, 808, 687, 718, 806, 837, 748, 779, 718, 749, + 778, 809, 807, 838, 832, 832, 833, 864, 834, 865, 835, 866, 808, 839, + 749, 780, 836, 867, 779, 810, 719, 750, 837, 868, 809, 840, 838, 869, + 780, 811, 750, 781, 810, 841, 839, 870, 864, 864, 865, 896, 866, 897, + 840, 871, 867, 898, 781, 812, 811, 842, 868, 899, 751, 782, 869, 900, + 841, 872, 812, 843, 870, 901, 782, 813, 842, 873, 871, 902, 896, 896, + 897, 928, 813, 844, 898, 929, 872, 903, 783, 814, 843, 874, 899, 930, + 900, 931, 873, 904, 901, 932, 814, 845, 844, 875, 902, 933, 874, 905, + 903, 934, 845, 876, 928, 928, 815, 846, 929, 960, 930, 961, 875, 906, + 904, 935, 931, 962, 932, 963, 905, 936, 846, 877, 933, 964, 876, 907, + 934, 965, 906, 937, 935, 966, 877, 908, 847, 878, 960, 960, 907, 938, + 961, 992, 936, 967, 962, 993, 963, 994, 964, 995, 878, 909, 937, 968, + 908, 939, 965, 996, 966, 997, 938, 969, 879, 910, 909, 940, 967, 998, + 939, 970, 968, 999, 910, 941, 969, 1000, 940, 971, 970, 1001, 911, 942, + 941, 972, 971, 1002, 942, 973, 972, 1003, 943, 974, 973, 1004, 974, 1005, + 975, 1006, 15, 15, 16, 47, 48, 79, 80, 111, 112, 143, 144, 175, + 16, 16, 17, 48, 176, 207, 49, 80, 81, 112, 113, 144, 208, 239, + 145, 176, 240, 271, 17, 17, 18, 49, 177, 208, 50, 81, 82, 113, + 272, 303, 209, 240, 114, 145, 146, 177, 241, 272, 304, 335, 178, 209, + 18, 18, 19, 50, 51, 82, 83, 114, 273, 304, 210, 241, 115, 146, + 336, 367, 147, 178, 242, 273, 305, 336, 179, 210, 19, 19, 368, 399, + 20, 51, 52, 83, 274, 305, 84, 115, 211, 242, 337, 368, 116, 147, + 306, 337, 148, 179, 243, 274, 400, 431, 369, 400, 180, 211, 20, 20, + 21, 52, 275, 306, 53, 84, 338, 369, 212, 243, 85, 116, 432, 463, + 117, 148, 401, 432, 307, 338, 244, 275, 149, 180, 370, 401, 181, 212, + 276, 307, 464, 495, 339, 370, 21, 21, 22, 53, 433, 464, 54, 85, + 213, 244, 86, 117, 402, 433, 118, 149, 308, 339, 245, 276, 371, 402, + 150, 181, 496, 527, 465, 496, 182, 213, 434, 465, 340, 371, 277, 308, + 22, 22, 23, 54, 403, 434, 55, 86, 214, 245, 87, 118, 309, 340, + 372, 403, 119, 150, 497, 528, 528, 559, 246, 277, 466, 497, 151, 182, + 435, 466, 341, 372, 183, 214, 278, 309, 404, 435, 23, 23, 24, 55, + 215, 246, 529, 560, 56, 87, 498, 529, 560, 591, 310, 341, 88, 119, + 373, 404, 467, 498, 120, 151, 247, 278, 436, 467, 152, 183, 342, 373, + 279, 310, 405, 436, 184, 215, 530, 561, 561, 592, 499, 530, 592, 623, + 24, 24, 216, 247, 468, 499, 25, 56, 374, 405, 57, 88, 311, 342, + 89, 120, 437, 468, 248, 279, 121, 152, 562, 593, 153, 184, 343, 374, + 531, 562, 593, 624, 406, 437, 500, 531, 624, 655, 280, 311, 185, 216, + 469, 500, 375, 406, 217, 248, 25, 25, 312, 343, 26, 57, 58, 89, + 438, 469, 90, 121, 563, 594, 594, 625, 249, 280, 532, 563, 625, 656, + 122, 153, 344, 375, 501, 532, 656, 687, 407, 438, 154, 185, 281, 312, + 470, 501, 186, 217, 376, 407, 595, 626, 564, 595, 626, 657, 218, 249, + 313, 344, 439, 470, 26, 26, 27, 58, 533, 564, 657, 688, 59, 90, + 91, 122, 250, 281, 502, 533, 688, 719, 123, 154, 408, 439, 345, 376, + 155, 186, 471, 502, 282, 313, 596, 627, 627, 658, 187, 218, 565, 596, + 658, 689, 377, 408, 440, 471, 534, 565, 689, 720, 314, 345, 219, 250, + 27, 27, 28, 59, 503, 534, 720, 751, 60, 91, 92, 123, 251, 282, + 409, 440, 346, 377, 124, 155, 628, 659, 472, 503, 597, 628, 659, 690, + 566, 597, 690, 721, 156, 187, 283, 314, 535, 566, 721, 752, 188, 219, + 378, 409, 441, 472, 315, 346, 504, 535, 752, 783, 220, 251, 28, 28, + 629, 660, 660, 691, 29, 60, 61, 92, 410, 441, 598, 629, 691, 722, + 252, 283, 93, 124, 347, 378, 473, 504, 567, 598, 722, 753, 125, 156, + 284, 315, 536, 567, 753, 784, 157, 188, 442, 473, 379, 410, 189, 220, + 505, 536, 784, 815, 661, 692, 316, 347, 630, 661, 692, 723, 221, 252, + 599, 630, 723, 754, 411, 442, 29, 29, 568, 599, 754, 785, 30, 61, + 474, 505, 62, 93, 253, 284, 348, 379, 94, 125, 537, 568, 785, 816, + 126, 157, 285, 316, 158, 189, 443, 474, 662, 693, 693, 724, 380, 411, + 631, 662, 724, 755, 506, 537, 816, 847, 190, 221, 600, 631, 755, 786, + 317, 348, 222, 253, 569, 600, 786, 817, 412, 443, 475, 506, 30, 30, + 31, 62, 349, 380, 254, 285, 63, 94, 538, 569, 817, 848, 694, 725, + 95, 126, 663, 694, 725, 756, 632, 663, 756, 787, 127, 158, 444, 475, + 286, 317, 381, 412, 507, 538, 848, 879, 159, 190, 601, 632, 787, 818, + 191, 222, 318, 349, 570, 601, 818, 849, 476, 507, 223, 254, 413, 444, + 695, 726, 726, 757, 664, 695, 757, 788, 539, 570, 849, 880, 350, 381, + 255, 286, 633, 664, 788, 819, 445, 476, 602, 633, 819, 850, 508, 539, + 880, 911, 287, 318, 382, 413, 571, 602, 850, 881, 727, 758, 696, 727, + 758, 789, 319, 350, 477, 508, 665, 696, 789, 820, 414, 445, 540, 571, + 881, 912, 634, 665, 820, 851, 351, 382, 603, 634, 851, 882, 446, 477, + 509, 540, 912, 943, 383, 414, 728, 759, 759, 790, 572, 603, 882, 913, + 697, 728, 790, 821, 666, 697, 821, 852, 478, 509, 635, 666, 852, 883, + 415, 446, 541, 572, 913, 944, 604, 635, 883, 914, 760, 791, 729, 760, + 791, 822, 510, 541, 944, 975, 447, 478, 698, 729, 822, 853, 573, 604, + 914, 945, 667, 698, 853, 884, 636, 667, 884, 915, 479, 510, 542, 573, + 945, 976, 761, 792, 792, 823, 605, 636, 915, 946, 730, 761, 823, 854, + 699, 730, 854, 885, 511, 542, 976, 1007, 574, 605, 946, 977, 668, 699, + 885, 916, 637, 668, 916, 947, 543, 574, 793, 824, 977, 1008, 762, 793, + 824, 855, 731, 762, 855, 886, 606, 637, 947, 978, 700, 731, 886, 917, + 669, 700, 917, 948, 575, 606, 978, 1009, 638, 669, 948, 979, 794, 825, + 825, 856, 763, 794, 856, 887, 732, 763, 887, 918, 607, 638, 979, 1010, + 701, 732, 918, 949, 670, 701, 949, 980, 826, 857, 795, 826, 857, 888, + 764, 795, 888, 919, 639, 670, 980, 1011, 733, 764, 919, 950, 702, 733, + 950, 981, 671, 702, 981, 1012, 827, 858, 858, 889, 796, 827, 889, 920, + 765, 796, 920, 951, 734, 765, 951, 982, 703, 734, 982, 1013, 859, 890, + 828, 859, 890, 921, 797, 828, 921, 952, 766, 797, 952, 983, 735, 766, + 983, 1014, 860, 891, 891, 922, 829, 860, 922, 953, 798, 829, 953, 984, + 767, 798, 984, 1015, 892, 923, 861, 892, 923, 954, 830, 861, 954, 985, + 799, 830, 985, 1016, 893, 924, 924, 955, 862, 893, 955, 986, 831, 862, + 986, 1017, 925, 956, 894, 925, 956, 987, 863, 894, 987, 1018, 926, 957, + 957, 988, 895, 926, 988, 1019, 958, 989, 927, 958, 989, 1020, 959, 990, + 990, 1021, 991, 1022, 0, 0, +}; + +DECLARE_ALIGNED(16, static const int16_t, + h2_scan_32x32_neighbors[1025 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 32, 1, 1, 32, 32, 2, 33, + 33, 64, 34, 65, 2, 2, 64, 64, 3, 34, 65, 96, 35, 66, + 66, 97, 3, 3, 96, 96, 4, 35, 97, 128, 67, 98, 36, 67, + 98, 129, 4, 4, 68, 99, 99, 130, 128, 128, 5, 36, 129, 160, + 37, 68, 130, 161, 100, 131, 69, 100, 131, 162, 5, 5, 160, 160, + 6, 37, 161, 192, 38, 69, 162, 193, 101, 132, 132, 163, 70, 101, + 163, 194, 6, 6, 192, 192, 7, 38, 133, 164, 193, 224, 102, 133, + 164, 195, 39, 70, 194, 225, 71, 102, 195, 226, 134, 165, 165, 196, + 7, 7, 224, 224, 8, 39, 103, 134, 196, 227, 225, 256, 40, 71, + 226, 257, 166, 197, 72, 103, 227, 258, 135, 166, 197, 228, 104, 135, + 228, 259, 8, 8, 256, 256, 9, 40, 257, 288, 41, 72, 167, 198, + 198, 229, 258, 289, 136, 167, 229, 260, 73, 104, 259, 290, 105, 136, + 260, 291, 199, 230, 9, 9, 168, 199, 230, 261, 288, 288, 10, 41, + 289, 320, 42, 73, 290, 321, 137, 168, 261, 292, 74, 105, 291, 322, + 200, 231, 231, 262, 106, 137, 292, 323, 169, 200, 262, 293, 10, 10, + 320, 320, 11, 42, 321, 352, 43, 74, 138, 169, 293, 324, 322, 353, + 232, 263, 75, 106, 201, 232, 263, 294, 323, 354, 170, 201, 294, 325, + 107, 138, 324, 355, 11, 11, 352, 352, 12, 43, 233, 264, 264, 295, + 353, 384, 139, 170, 325, 356, 44, 75, 354, 385, 202, 233, 295, 326, + 76, 107, 355, 386, 171, 202, 326, 357, 108, 139, 356, 387, 265, 296, + 234, 265, 296, 327, 12, 12, 140, 171, 357, 388, 384, 384, 13, 44, + 203, 234, 327, 358, 385, 416, 45, 76, 386, 417, 77, 108, 387, 418, + 172, 203, 358, 389, 266, 297, 297, 328, 109, 140, 235, 266, 328, 359, + 388, 419, 204, 235, 359, 390, 141, 172, 389, 420, 13, 13, 416, 416, + 14, 45, 417, 448, 46, 77, 298, 329, 418, 449, 267, 298, 329, 360, + 78, 109, 173, 204, 390, 421, 419, 450, 236, 267, 360, 391, 110, 141, + 420, 451, 205, 236, 391, 422, 142, 173, 299, 330, 330, 361, 421, 452, + 14, 14, 268, 299, 361, 392, 448, 448, 15, 46, 449, 480, 47, 78, + 450, 481, 174, 205, 422, 453, 237, 268, 392, 423, 79, 110, 451, 482, + 111, 142, 452, 483, 331, 362, 300, 331, 362, 393, 206, 237, 423, 454, + 143, 174, 269, 300, 393, 424, 453, 484, 15, 15, 16, 47, 48, 79, + 238, 269, 424, 455, 175, 206, 454, 485, 80, 111, 332, 363, 363, 394, + 301, 332, 394, 425, 112, 143, 207, 238, 455, 486, 270, 301, 425, 456, + 144, 175, 364, 395, 16, 16, 239, 270, 456, 487, 17, 48, 333, 364, + 395, 426, 176, 207, 49, 80, 302, 333, 426, 457, 81, 112, 113, 144, + 208, 239, 271, 302, 457, 488, 365, 396, 396, 427, 145, 176, 334, 365, + 427, 458, 240, 271, 17, 17, 18, 49, 177, 208, 303, 334, 458, 489, + 50, 81, 82, 113, 272, 303, 209, 240, 397, 428, 114, 145, 366, 397, + 428, 459, 335, 366, 459, 490, 146, 177, 241, 272, 304, 335, 178, 209, + 18, 18, 19, 50, 51, 82, 398, 429, 429, 460, 367, 398, 460, 491, + 83, 114, 273, 304, 210, 241, 115, 146, 336, 367, 147, 178, 242, 273, + 305, 336, 430, 461, 399, 430, 461, 492, 179, 210, 19, 19, 368, 399, + 20, 51, 52, 83, 274, 305, 84, 115, 211, 242, 337, 368, 116, 147, + 431, 462, 462, 493, 306, 337, 148, 179, 243, 274, 400, 431, 369, 400, + 180, 211, 20, 20, 21, 52, 275, 306, 53, 84, 338, 369, 212, 243, + 85, 116, 463, 494, 432, 463, 117, 148, 401, 432, 307, 338, 244, 275, + 149, 180, 370, 401, 181, 212, 276, 307, 464, 495, 339, 370, 21, 21, + 22, 53, 433, 464, 54, 85, 213, 244, 86, 117, 402, 433, 118, 149, + 308, 339, 245, 276, 371, 402, 150, 181, 465, 496, 182, 213, 434, 465, + 340, 371, 277, 308, 22, 22, 23, 54, 403, 434, 55, 86, 214, 245, + 87, 118, 309, 340, 372, 403, 119, 150, 246, 277, 466, 497, 151, 182, + 435, 466, 341, 372, 183, 214, 278, 309, 404, 435, 23, 23, 24, 55, + 215, 246, 56, 87, 310, 341, 88, 119, 373, 404, 467, 498, 120, 151, + 247, 278, 436, 467, 152, 183, 342, 373, 279, 310, 405, 436, 184, 215, + 24, 24, 216, 247, 468, 499, 25, 56, 374, 405, 57, 88, 311, 342, + 89, 120, 437, 468, 248, 279, 121, 152, 153, 184, 343, 374, 406, 437, + 280, 311, 185, 216, 469, 500, 375, 406, 217, 248, 25, 25, 312, 343, + 26, 57, 58, 89, 438, 469, 90, 121, 249, 280, 122, 153, 344, 375, + 407, 438, 154, 185, 281, 312, 470, 501, 186, 217, 376, 407, 218, 249, + 313, 344, 439, 470, 26, 26, 27, 58, 59, 90, 91, 122, 250, 281, + 123, 154, 408, 439, 345, 376, 155, 186, 471, 502, 282, 313, 187, 218, + 377, 408, 440, 471, 314, 345, 219, 250, 27, 27, 28, 59, 60, 91, + 92, 123, 251, 282, 409, 440, 346, 377, 124, 155, 472, 503, 156, 187, + 283, 314, 188, 219, 378, 409, 441, 472, 315, 346, 220, 251, 28, 28, + 29, 60, 61, 92, 410, 441, 252, 283, 93, 124, 347, 378, 473, 504, + 125, 156, 284, 315, 157, 188, 442, 473, 379, 410, 189, 220, 316, 347, + 221, 252, 411, 442, 29, 29, 30, 61, 474, 505, 62, 93, 253, 284, + 348, 379, 94, 125, 126, 157, 285, 316, 158, 189, 443, 474, 380, 411, + 190, 221, 317, 348, 222, 253, 412, 443, 475, 506, 30, 30, 31, 62, + 349, 380, 254, 285, 63, 94, 95, 126, 127, 158, 444, 475, 286, 317, + 381, 412, 159, 190, 191, 222, 318, 349, 476, 507, 223, 254, 413, 444, + 350, 381, 255, 286, 445, 476, 287, 318, 382, 413, 319, 350, 477, 508, + 414, 445, 351, 382, 446, 477, 383, 414, 478, 509, 415, 446, 447, 478, + 479, 510, 480, 480, 481, 512, 482, 513, 483, 514, 484, 515, 485, 516, + 512, 512, 513, 544, 486, 517, 514, 545, 515, 546, 487, 518, 516, 547, + 517, 548, 488, 519, 544, 544, 518, 549, 545, 576, 546, 577, 547, 578, + 489, 520, 519, 550, 548, 579, 549, 580, 520, 551, 490, 521, 550, 581, + 576, 576, 577, 608, 578, 609, 521, 552, 579, 610, 551, 582, 491, 522, + 580, 611, 581, 612, 552, 583, 522, 553, 582, 613, 492, 523, 608, 608, + 609, 640, 610, 641, 553, 584, 611, 642, 523, 554, 583, 614, 612, 643, + 554, 585, 493, 524, 584, 615, 613, 644, 524, 555, 614, 645, 640, 640, + 585, 616, 641, 672, 555, 586, 642, 673, 615, 646, 643, 674, 494, 525, + 644, 675, 525, 556, 586, 617, 616, 647, 645, 676, 556, 587, 646, 677, + 495, 526, 617, 648, 587, 618, 672, 672, 526, 557, 673, 704, 674, 705, + 647, 678, 557, 588, 675, 706, 618, 649, 676, 707, 588, 619, 648, 679, + 677, 708, 496, 527, 527, 558, 558, 589, 678, 709, 619, 650, 649, 680, + 704, 704, 589, 620, 705, 736, 679, 710, 706, 737, 707, 738, 650, 681, + 620, 651, 497, 528, 528, 559, 708, 739, 680, 711, 559, 590, 709, 740, + 590, 621, 651, 682, 681, 712, 710, 741, 621, 652, 736, 736, 737, 768, + 529, 560, 711, 742, 498, 529, 560, 591, 738, 769, 682, 713, 652, 683, + 739, 770, 591, 622, 740, 771, 712, 743, 622, 653, 741, 772, 683, 714, + 653, 684, 713, 744, 742, 773, 530, 561, 561, 592, 499, 530, 592, 623, + 623, 654, 743, 774, 768, 768, 769, 800, 684, 715, 714, 745, 770, 801, + 771, 802, 654, 685, 744, 775, 772, 803, 562, 593, 531, 562, 593, 624, + 715, 746, 773, 804, 685, 716, 500, 531, 624, 655, 745, 776, 774, 805, + 655, 686, 716, 747, 775, 806, 746, 777, 800, 800, 801, 832, 686, 717, + 802, 833, 563, 594, 594, 625, 803, 834, 532, 563, 625, 656, 776, 807, + 804, 835, 501, 532, 656, 687, 747, 778, 717, 748, 805, 836, 777, 808, + 687, 718, 806, 837, 748, 779, 595, 626, 564, 595, 626, 657, 718, 749, + 778, 809, 807, 838, 832, 832, 533, 564, 657, 688, 833, 864, 834, 865, + 835, 866, 502, 533, 688, 719, 808, 839, 749, 780, 836, 867, 779, 810, + 719, 750, 837, 868, 809, 840, 596, 627, 627, 658, 565, 596, 658, 689, + 838, 869, 780, 811, 750, 781, 534, 565, 689, 720, 810, 841, 839, 870, + 864, 864, 503, 534, 720, 751, 865, 896, 866, 897, 840, 871, 867, 898, + 781, 812, 811, 842, 628, 659, 868, 899, 751, 782, 597, 628, 659, 690, + 566, 597, 690, 721, 869, 900, 841, 872, 535, 566, 721, 752, 812, 843, + 870, 901, 782, 813, 842, 873, 504, 535, 752, 783, 871, 902, 629, 660, + 660, 691, 896, 896, 897, 928, 598, 629, 691, 722, 813, 844, 898, 929, + 872, 903, 783, 814, 843, 874, 899, 930, 567, 598, 722, 753, 900, 931, + 536, 567, 753, 784, 873, 904, 901, 932, 814, 845, 844, 875, 902, 933, + 505, 536, 784, 815, 661, 692, 630, 661, 692, 723, 874, 905, 599, 630, + 723, 754, 903, 934, 845, 876, 568, 599, 754, 785, 928, 928, 815, 846, + 929, 960, 930, 961, 875, 906, 904, 935, 931, 962, 537, 568, 785, 816, + 932, 963, 905, 936, 662, 693, 693, 724, 846, 877, 933, 964, 876, 907, + 631, 662, 724, 755, 506, 537, 816, 847, 934, 965, 600, 631, 755, 786, + 906, 937, 569, 600, 786, 817, 935, 966, 877, 908, 847, 878, 960, 960, + 907, 938, 961, 992, 936, 967, 538, 569, 817, 848, 962, 993, 694, 725, + 663, 694, 725, 756, 963, 994, 632, 663, 756, 787, 964, 995, 878, 909, + 937, 968, 507, 538, 848, 879, 908, 939, 601, 632, 787, 818, 965, 996, + 966, 997, 570, 601, 818, 849, 938, 969, 879, 910, 909, 940, 967, 998, + 695, 726, 726, 757, 664, 695, 757, 788, 539, 570, 849, 880, 939, 970, + 633, 664, 788, 819, 968, 999, 602, 633, 819, 850, 910, 941, 508, 539, + 880, 911, 969, 1000, 940, 971, 571, 602, 850, 881, 727, 758, 696, 727, + 758, 789, 970, 1001, 665, 696, 789, 820, 911, 942, 941, 972, 540, 571, + 881, 912, 634, 665, 820, 851, 971, 1002, 603, 634, 851, 882, 942, 973, + 509, 540, 912, 943, 728, 759, 759, 790, 972, 1003, 572, 603, 882, 913, + 697, 728, 790, 821, 666, 697, 821, 852, 943, 974, 635, 666, 852, 883, + 541, 572, 913, 944, 973, 1004, 604, 635, 883, 914, 760, 791, 729, 760, + 791, 822, 510, 541, 944, 975, 974, 1005, 698, 729, 822, 853, 573, 604, + 914, 945, 667, 698, 853, 884, 636, 667, 884, 915, 975, 1006, 542, 573, + 945, 976, 761, 792, 792, 823, 605, 636, 915, 946, 730, 761, 823, 854, + 699, 730, 854, 885, 511, 542, 976, 1007, 574, 605, 946, 977, 668, 699, + 885, 916, 637, 668, 916, 947, 543, 574, 793, 824, 977, 1008, 762, 793, + 824, 855, 731, 762, 855, 886, 606, 637, 947, 978, 700, 731, 886, 917, + 669, 700, 917, 948, 575, 606, 978, 1009, 638, 669, 948, 979, 794, 825, + 825, 856, 763, 794, 856, 887, 732, 763, 887, 918, 607, 638, 979, 1010, + 701, 732, 918, 949, 670, 701, 949, 980, 826, 857, 795, 826, 857, 888, + 764, 795, 888, 919, 639, 670, 980, 1011, 733, 764, 919, 950, 702, 733, + 950, 981, 671, 702, 981, 1012, 827, 858, 858, 889, 796, 827, 889, 920, + 765, 796, 920, 951, 734, 765, 951, 982, 703, 734, 982, 1013, 859, 890, + 828, 859, 890, 921, 797, 828, 921, 952, 766, 797, 952, 983, 735, 766, + 983, 1014, 860, 891, 891, 922, 829, 860, 922, 953, 798, 829, 953, 984, + 767, 798, 984, 1015, 892, 923, 861, 892, 923, 954, 830, 861, 954, 985, + 799, 830, 985, 1016, 893, 924, 924, 955, 862, 893, 955, 986, 831, 862, + 986, 1017, 925, 956, 894, 925, 956, 987, 863, 894, 987, 1018, 926, 957, + 957, 988, 895, 926, 988, 1019, 958, 989, 927, 958, 989, 1020, 959, 990, + 990, 1021, 991, 1022, 0, 0 +}; + +DECLARE_ALIGNED(16, static const int16_t, + qtr_scan_32x32_neighbors[1025 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 32, 1, 1, 32, 32, 2, 33, + 33, 64, 34, 65, 2, 2, 64, 64, 3, 34, 65, 96, 35, 66, + 66, 97, 3, 3, 96, 96, 4, 35, 97, 128, 67, 98, 36, 67, + 98, 129, 4, 4, 68, 99, 99, 130, 128, 128, 5, 36, 129, 160, + 37, 68, 130, 161, 100, 131, 69, 100, 131, 162, 5, 5, 160, 160, + 6, 37, 161, 192, 38, 69, 162, 193, 101, 132, 132, 163, 70, 101, + 163, 194, 6, 6, 192, 192, 7, 38, 133, 164, 193, 224, 102, 133, + 164, 195, 39, 70, 194, 225, 71, 102, 195, 226, 134, 165, 165, 196, + 7, 7, 224, 224, 8, 39, 103, 134, 196, 227, 225, 256, 40, 71, + 226, 257, 166, 197, 72, 103, 227, 258, 135, 166, 197, 228, 104, 135, + 228, 259, 8, 8, 256, 256, 9, 40, 257, 288, 41, 72, 167, 198, + 198, 229, 258, 289, 136, 167, 229, 260, 73, 104, 259, 290, 105, 136, + 260, 291, 199, 230, 9, 9, 168, 199, 230, 261, 288, 288, 10, 41, + 289, 320, 42, 73, 290, 321, 137, 168, 261, 292, 74, 105, 291, 322, + 200, 231, 231, 262, 106, 137, 292, 323, 169, 200, 262, 293, 10, 10, + 320, 320, 11, 42, 321, 352, 43, 74, 138, 169, 293, 324, 322, 353, + 232, 263, 75, 106, 201, 232, 263, 294, 323, 354, 170, 201, 294, 325, + 107, 138, 324, 355, 11, 11, 352, 352, 12, 43, 233, 264, 264, 295, + 353, 384, 139, 170, 325, 356, 44, 75, 354, 385, 202, 233, 295, 326, + 76, 107, 355, 386, 171, 202, 326, 357, 108, 139, 356, 387, 265, 296, + 234, 265, 296, 327, 12, 12, 140, 171, 357, 388, 384, 384, 13, 44, + 203, 234, 327, 358, 385, 416, 45, 76, 386, 417, 77, 108, 387, 418, + 172, 203, 358, 389, 266, 297, 297, 328, 109, 140, 235, 266, 328, 359, + 388, 419, 204, 235, 359, 390, 141, 172, 389, 420, 13, 13, 416, 416, + 14, 45, 417, 448, 46, 77, 298, 329, 418, 449, 267, 298, 329, 360, + 78, 109, 173, 204, 390, 421, 419, 450, 236, 267, 360, 391, 110, 141, + 420, 451, 205, 236, 391, 422, 142, 173, 299, 330, 330, 361, 421, 452, + 14, 14, 268, 299, 361, 392, 448, 448, 15, 46, 449, 480, 47, 78, + 450, 481, 174, 205, 422, 453, 237, 268, 392, 423, 79, 110, 451, 482, + 111, 142, 452, 483, 331, 362, 300, 331, 362, 393, 206, 237, 423, 454, + 143, 174, 269, 300, 393, 424, 453, 484, 238, 269, 424, 455, 175, 206, + 454, 485, 332, 363, 363, 394, 301, 332, 394, 425, 207, 238, 455, 486, + 270, 301, 425, 456, 364, 395, 239, 270, 456, 487, 333, 364, 395, 426, + 302, 333, 426, 457, 271, 302, 457, 488, 365, 396, 396, 427, 334, 365, + 427, 458, 303, 334, 458, 489, 397, 428, 366, 397, 428, 459, 335, 366, + 459, 490, 398, 429, 429, 460, 367, 398, 460, 491, 430, 461, 399, 430, + 461, 492, 431, 462, 462, 493, 463, 494, 15, 15, 480, 480, 16, 47, + 481, 512, 48, 79, 482, 513, 80, 111, 483, 514, 112, 143, 484, 515, + 144, 175, 485, 516, 16, 16, 512, 512, 17, 48, 513, 544, 176, 207, + 486, 517, 49, 80, 514, 545, 81, 112, 515, 546, 113, 144, 208, 239, + 487, 518, 516, 547, 145, 176, 517, 548, 240, 271, 488, 519, 17, 17, + 544, 544, 18, 49, 177, 208, 518, 549, 545, 576, 50, 81, 546, 577, + 82, 113, 547, 578, 272, 303, 489, 520, 209, 240, 519, 550, 114, 145, + 548, 579, 146, 177, 549, 580, 241, 272, 520, 551, 304, 335, 490, 521, + 178, 209, 550, 581, 18, 18, 576, 576, 19, 50, 577, 608, 51, 82, + 578, 609, 83, 114, 273, 304, 521, 552, 579, 610, 210, 241, 551, 582, + 115, 146, 336, 367, 491, 522, 580, 611, 147, 178, 581, 612, 242, 273, + 552, 583, 305, 336, 522, 553, 179, 210, 582, 613, 19, 19, 368, 399, + 492, 523, 608, 608, 20, 51, 609, 640, 52, 83, 610, 641, 274, 305, + 553, 584, 84, 115, 611, 642, 211, 242, 337, 368, 523, 554, 583, 614, + 116, 147, 612, 643, 306, 337, 554, 585, 148, 179, 243, 274, 400, 431, + 493, 524, 584, 615, 613, 644, 369, 400, 524, 555, 180, 211, 614, 645, + 20, 20, 640, 640, 21, 52, 275, 306, 585, 616, 641, 672, 53, 84, + 338, 369, 555, 586, 642, 673, 212, 243, 615, 646, 85, 116, 643, 674, + 432, 463, 494, 525, 117, 148, 644, 675, 401, 432, 525, 556, 307, 338, + 586, 617, 244, 275, 616, 647, 149, 180, 645, 676, 370, 401, 556, 587, + 181, 212, 646, 677, 276, 307, 464, 495, 495, 526, 617, 648, 339, 370, + 587, 618, 21, 21, 672, 672, 22, 53, 433, 464, 526, 557, 673, 704, + 54, 85, 674, 705, 213, 244, 647, 678, 86, 117, 402, 433, 557, 588, + 675, 706, 118, 149, 308, 339, 618, 649, 676, 707, 245, 276, 371, 402, + 588, 619, 648, 679, 150, 181, 677, 708, 496, 527, 465, 496, 527, 558, + 182, 213, 434, 465, 558, 589, 678, 709, 340, 371, 619, 650, 277, 308, + 649, 680, 22, 22, 704, 704, 23, 54, 403, 434, 589, 620, 705, 736, + 55, 86, 214, 245, 679, 710, 706, 737, 87, 118, 707, 738, 309, 340, + 650, 681, 372, 403, 620, 651, 119, 150, 497, 528, 528, 559, 708, 739, + 246, 277, 680, 711, 466, 497, 559, 590, 151, 182, 709, 740, 435, 466, + 590, 621, 341, 372, 651, 682, 183, 214, 278, 309, 681, 712, 710, 741, + 404, 435, 621, 652, 23, 23, 736, 736, 24, 55, 737, 768, 215, 246, + 529, 560, 711, 742, 56, 87, 498, 529, 560, 591, 738, 769, 310, 341, + 682, 713, 88, 119, 373, 404, 652, 683, 739, 770, 467, 498, 591, 622, + 120, 151, 740, 771, 247, 278, 712, 743, 436, 467, 622, 653, 152, 183, + 741, 772, 342, 373, 683, 714, 279, 310, 405, 436, 653, 684, 713, 744, + 184, 215, 742, 773, 530, 561, 561, 592, 499, 530, 592, 623, 24, 24, + 216, 247, 468, 499, 623, 654, 743, 774, 768, 768, 25, 56, 769, 800, + 374, 405, 684, 715, 57, 88, 311, 342, 714, 745, 770, 801, 89, 120, + 771, 802, 437, 468, 654, 685, 248, 279, 744, 775, 121, 152, 772, 803, + 562, 593, 153, 184, 343, 374, 531, 562, 593, 624, 715, 746, 773, 804, + 406, 437, 685, 716, 500, 531, 624, 655, 280, 311, 745, 776, 185, 216, + 774, 805, 469, 500, 655, 686, 375, 406, 716, 747, 217, 248, 775, 806, + 25, 25, 312, 343, 746, 777, 800, 800, 26, 57, 801, 832, 58, 89, + 438, 469, 686, 717, 802, 833, 90, 121, 563, 594, 594, 625, 803, 834, + 249, 280, 532, 563, 625, 656, 776, 807, 122, 153, 804, 835, 344, 375, + 501, 532, 656, 687, 747, 778, 407, 438, 717, 748, 154, 185, 805, 836, + 281, 312, 777, 808, 470, 501, 687, 718, 186, 217, 806, 837, 376, 407, + 748, 779, 595, 626, 564, 595, 626, 657, 218, 249, 313, 344, 439, 470, + 718, 749, 778, 809, 807, 838, 26, 26, 832, 832, 27, 58, 533, 564, + 657, 688, 833, 864, 59, 90, 834, 865, 91, 122, 835, 866, 250, 281, + 502, 533, 688, 719, 808, 839, 123, 154, 408, 439, 749, 780, 836, 867, + 345, 376, 779, 810, 155, 186, 471, 502, 719, 750, 837, 868, 282, 313, + 809, 840, 596, 627, 627, 658, 187, 218, 565, 596, 658, 689, 838, 869, + 377, 408, 780, 811, 440, 471, 750, 781, 534, 565, 689, 720, 314, 345, + 810, 841, 219, 250, 839, 870, 27, 27, 864, 864, 28, 59, 503, 534, + 720, 751, 865, 896, 60, 91, 866, 897, 92, 123, 251, 282, 840, 871, + 867, 898, 409, 440, 781, 812, 346, 377, 811, 842, 124, 155, 628, 659, + 868, 899, 472, 503, 751, 782, 597, 628, 659, 690, 566, 597, 690, 721, + 156, 187, 869, 900, 283, 314, 841, 872, 535, 566, 721, 752, 188, 219, + 378, 409, 812, 843, 870, 901, 441, 472, 782, 813, 315, 346, 842, 873, + 504, 535, 752, 783, 220, 251, 871, 902, 28, 28, 629, 660, 660, 691, + 896, 896, 29, 60, 897, 928, 61, 92, 410, 441, 598, 629, 691, 722, + 813, 844, 898, 929, 252, 283, 872, 903, 93, 124, 347, 378, 473, 504, + 783, 814, 843, 874, 899, 930, 567, 598, 722, 753, 125, 156, 900, 931, + 284, 315, 536, 567, 753, 784, 873, 904, 157, 188, 901, 932, 442, 473, + 814, 845, 379, 410, 844, 875, 189, 220, 902, 933, 505, 536, 784, 815, + 661, 692, 316, 347, 630, 661, 692, 723, 874, 905, 221, 252, 599, 630, + 723, 754, 903, 934, 411, 442, 845, 876, 29, 29, 568, 599, 754, 785, + 928, 928, 30, 61, 474, 505, 815, 846, 929, 960, 62, 93, 930, 961, + 253, 284, 348, 379, 875, 906, 904, 935, 94, 125, 931, 962, 537, 568, + 785, 816, 126, 157, 932, 963, 285, 316, 905, 936, 158, 189, 443, 474, + 662, 693, 693, 724, 846, 877, 933, 964, 380, 411, 876, 907, 631, 662, + 724, 755, 506, 537, 816, 847, 190, 221, 934, 965, 600, 631, 755, 786, + 317, 348, 906, 937, 222, 253, 569, 600, 786, 817, 935, 966, 412, 443, + 877, 908, 475, 506, 847, 878, 30, 30, 960, 960, 31, 62, 349, 380, + 907, 938, 961, 992, 254, 285, 936, 967, 63, 94, 538, 569, 817, 848, + 962, 993, 694, 725, 95, 126, 663, 694, 725, 756, 963, 994, 632, 663, + 756, 787, 127, 158, 964, 995, 444, 475, 878, 909, 286, 317, 937, 968, + 381, 412, 507, 538, 848, 879, 908, 939, 159, 190, 601, 632, 787, 818, + 965, 996, 191, 222, 966, 997, 318, 349, 570, 601, 818, 849, 938, 969, + 476, 507, 879, 910, 223, 254, 413, 444, 909, 940, 967, 998, 695, 726, + 726, 757, 664, 695, 757, 788, 539, 570, 849, 880, 350, 381, 939, 970, + 255, 286, 633, 664, 788, 819, 968, 999, 445, 476, 602, 633, 819, 850, + 910, 941, 508, 539, 880, 911, 287, 318, 969, 1000, 382, 413, 940, 971, + 571, 602, 850, 881, 727, 758, 696, 727, 758, 789, 319, 350, 970, 1001, + 477, 508, 665, 696, 789, 820, 911, 942, 414, 445, 941, 972, 540, 571, + 881, 912, 634, 665, 820, 851, 351, 382, 971, 1002, 603, 634, 851, 882, + 446, 477, 942, 973, 509, 540, 912, 943, 383, 414, 728, 759, 759, 790, + 972, 1003, 572, 603, 882, 913, 697, 728, 790, 821, 666, 697, 821, 852, + 478, 509, 943, 974, 635, 666, 852, 883, 415, 446, 541, 572, 913, 944, + 973, 1004, 604, 635, 883, 914, 760, 791, 729, 760, 791, 822, 510, 541, + 944, 975, 447, 478, 974, 1005, 698, 729, 822, 853, 573, 604, 914, 945, + 667, 698, 853, 884, 636, 667, 884, 915, 479, 510, 975, 1006, 542, 573, + 945, 976, 761, 792, 792, 823, 605, 636, 915, 946, 730, 761, 823, 854, + 699, 730, 854, 885, 511, 542, 976, 1007, 574, 605, 946, 977, 668, 699, + 885, 916, 637, 668, 916, 947, 543, 574, 793, 824, 977, 1008, 762, 793, + 824, 855, 731, 762, 855, 886, 606, 637, 947, 978, 700, 731, 886, 917, + 669, 700, 917, 948, 575, 606, 978, 1009, 638, 669, 948, 979, 794, 825, + 825, 856, 763, 794, 856, 887, 732, 763, 887, 918, 607, 638, 979, 1010, + 701, 732, 918, 949, 670, 701, 949, 980, 826, 857, 795, 826, 857, 888, + 764, 795, 888, 919, 639, 670, 980, 1011, 733, 764, 919, 950, 702, 733, + 950, 981, 671, 702, 981, 1012, 827, 858, 858, 889, 796, 827, 889, 920, + 765, 796, 920, 951, 734, 765, 951, 982, 703, 734, 982, 1013, 859, 890, + 828, 859, 890, 921, 797, 828, 921, 952, 766, 797, 952, 983, 735, 766, + 983, 1014, 860, 891, 891, 922, 829, 860, 922, 953, 798, 829, 953, 984, + 767, 798, 984, 1015, 892, 923, 861, 892, 923, 954, 830, 861, 954, 985, + 799, 830, 985, 1016, 893, 924, 924, 955, 862, 893, 955, 986, 831, 862, + 986, 1017, 925, 956, 894, 925, 956, 987, 863, 894, 987, 1018, 926, 957, + 957, 988, 895, 926, 988, 1019, 958, 989, 927, 958, 989, 1020, 959, 990, + 990, 1021, 991, 1022, 0, 0 +}; + +#if CONFIG_TX64X64 +DECLARE_ALIGNED(16, static const int16_t, + default_scan_64x64_neighbors[4097 * MAX_NEIGHBORS]) = { + 0, 0, 0, 0, 0, 0, 1, 64, 1, 1, 64, 64, 2, + 65, 65, 128, 66, 129, 2, 2, 128, 128, 3, 66, 129, 192, + 67, 130, 130, 193, 3, 3, 192, 192, 4, 67, 193, 256, 131, + 194, 68, 131, 194, 257, 4, 4, 132, 195, 195, 258, 256, 256, + 5, 68, 257, 320, 69, 132, 258, 321, 196, 259, 133, 196, 259, + 322, 5, 5, 320, 320, 6, 69, 321, 384, 70, 133, 322, 385, + 197, 260, 260, 323, 134, 197, 323, 386, 6, 6, 384, 384, 7, + 70, 261, 324, 385, 448, 198, 261, 324, 387, 71, 134, 386, 449, + 135, 198, 387, 450, 262, 325, 325, 388, 7, 7, 448, 448, 8, + 71, 199, 262, 388, 451, 449, 512, 72, 135, 450, 513, 326, 389, + 136, 199, 451, 514, 263, 326, 389, 452, 200, 263, 452, 515, 8, + 8, 512, 512, 9, 72, 513, 576, 73, 136, 327, 390, 390, 453, + 514, 577, 264, 327, 453, 516, 137, 200, 515, 578, 201, 264, 516, + 579, 391, 454, 9, 9, 328, 391, 454, 517, 576, 576, 10, 73, + 577, 640, 74, 137, 578, 641, 265, 328, 517, 580, 138, 201, 579, + 642, 392, 455, 455, 518, 202, 265, 580, 643, 329, 392, 518, 581, + 10, 10, 640, 640, 11, 74, 641, 704, 75, 138, 266, 329, 581, + 644, 642, 705, 456, 519, 139, 202, 393, 456, 519, 582, 643, 706, + 330, 393, 582, 645, 203, 266, 644, 707, 11, 11, 704, 704, 12, + 75, 457, 520, 520, 583, 705, 768, 267, 330, 645, 708, 76, 139, + 706, 769, 394, 457, 583, 646, 140, 203, 707, 770, 331, 394, 646, + 709, 204, 267, 708, 771, 521, 584, 458, 521, 584, 647, 12, 12, + 268, 331, 709, 772, 768, 768, 13, 76, 395, 458, 647, 710, 769, + 832, 77, 140, 770, 833, 141, 204, 771, 834, 332, 395, 710, 773, + 522, 585, 585, 648, 205, 268, 459, 522, 648, 711, 772, 835, 396, + 459, 711, 774, 269, 332, 773, 836, 13, 13, 832, 832, 14, 77, + 833, 896, 78, 141, 586, 649, 834, 897, 523, 586, 649, 712, 142, + 205, 333, 396, 774, 837, 835, 898, 460, 523, 712, 775, 206, 269, + 836, 899, 397, 460, 775, 838, 270, 333, 587, 650, 650, 713, 837, + 900, 14, 14, 524, 587, 713, 776, 896, 896, 15, 78, 897, 960, + 79, 142, 898, 961, 334, 397, 838, 901, 461, 524, 776, 839, 143, + 206, 899, 962, 207, 270, 900, 963, 651, 714, 588, 651, 714, 777, + 398, 461, 839, 902, 271, 334, 525, 588, 777, 840, 901, 964, 15, + 15, 960, 960, 16, 79, 961, 1024, 80, 143, 462, 525, 840, 903, + 962, 1025, 335, 398, 902, 965, 144, 207, 652, 715, 715, 778, 963, + 1026, 589, 652, 778, 841, 208, 271, 964, 1027, 399, 462, 903, 966, + 526, 589, 841, 904, 272, 335, 965, 1028, 716, 779, 16, 16, 463, + 526, 904, 967, 1024, 1024, 17, 80, 653, 716, 779, 842, 1025, 1088, + 336, 399, 966, 1029, 81, 144, 1026, 1089, 590, 653, 842, 905, 145, + 208, 1027, 1090, 209, 272, 400, 463, 967, 1030, 1028, 1091, 527, 590, + 905, 968, 717, 780, 780, 843, 273, 336, 1029, 1092, 654, 717, 843, + 906, 464, 527, 968, 1031, 17, 17, 1088, 1088, 18, 81, 337, 400, + 591, 654, 906, 969, 1030, 1093, 1089, 1152, 82, 145, 1090, 1153, 146, + 209, 1091, 1154, 528, 591, 969, 1032, 401, 464, 781, 844, 1031, 1094, + 210, 273, 718, 781, 844, 907, 1092, 1155, 655, 718, 907, 970, 274, + 337, 1093, 1156, 465, 528, 1032, 1095, 592, 655, 970, 1033, 338, 401, + 1094, 1157, 18, 18, 1152, 1152, 19, 82, 1153, 1216, 83, 146, 782, + 845, 845, 908, 1154, 1217, 719, 782, 908, 971, 147, 210, 529, 592, + 1033, 1096, 1155, 1218, 402, 465, 1095, 1158, 211, 274, 656, 719, 971, + 1034, 1156, 1219, 275, 338, 1157, 1220, 466, 529, 1096, 1159, 593, 656, + 1034, 1097, 846, 909, 783, 846, 909, 972, 339, 402, 1158, 1221, 19, + 19, 720, 783, 972, 1035, 1216, 1216, 20, 83, 1217, 1280, 84, 147, + 1218, 1281, 530, 593, 1097, 1160, 148, 211, 1219, 1282, 403, 466, 657, + 720, 1035, 1098, 1159, 1222, 212, 275, 1220, 1283, 847, 910, 910, 973, + 594, 657, 1098, 1161, 276, 339, 467, 530, 784, 847, 973, 1036, 1160, + 1223, 1221, 1284, 721, 784, 1036, 1099, 340, 403, 1222, 1285, 20, 20, + 1280, 1280, 21, 84, 531, 594, 1161, 1224, 1281, 1344, 85, 148, 658, + 721, 1099, 1162, 1282, 1345, 404, 467, 1223, 1286, 149, 212, 911, 974, + 1283, 1346, 848, 911, 974, 1037, 213, 276, 1284, 1347, 785, 848, 1037, + 1100, 595, 658, 1162, 1225, 468, 531, 1224, 1287, 277, 340, 1285, 1348, + 722, 785, 1100, 1163, 341, 404, 1286, 1349, 532, 595, 912, 975, 975, + 1038, 1225, 1288, 659, 722, 1163, 1226, 21, 21, 1344, 1344, 22, 85, + 849, 912, 1038, 1101, 1345, 1408, 86, 149, 1346, 1409, 405, 468, 1287, + 1350, 150, 213, 786, 849, 1101, 1164, 1347, 1410, 214, 277, 596, 659, + 1226, 1289, 1348, 1411, 469, 532, 723, 786, 1164, 1227, 1288, 1351, 278, + 341, 1349, 1412, 976, 1039, 913, 976, 1039, 1102, 342, 405, 850, 913, + 1102, 1165, 1350, 1413, 660, 723, 1227, 1290, 533, 596, 1289, 1352, 22, + 22, 1408, 1408, 23, 86, 787, 850, 1165, 1228, 1409, 1472, 87, 150, + 406, 469, 1351, 1414, 1410, 1473, 151, 214, 1411, 1474, 597, 660, 1290, + 1353, 724, 787, 1228, 1291, 215, 278, 977, 1040, 1040, 1103, 1412, 1475, + 470, 533, 1352, 1415, 914, 977, 1103, 1166, 279, 342, 1413, 1476, 851, + 914, 1166, 1229, 661, 724, 1291, 1354, 343, 406, 534, 597, 1353, 1416, + 1414, 1477, 788, 851, 1229, 1292, 23, 23, 1472, 1472, 24, 87, 1473, + 1536, 407, 470, 1041, 1104, 1415, 1478, 88, 151, 978, 1041, 1104, 1167, + 1474, 1537, 598, 661, 1354, 1417, 152, 215, 725, 788, 1292, 1355, 1475, + 1538, 915, 978, 1167, 1230, 216, 279, 1476, 1539, 471, 534, 1416, 1479, + 852, 915, 1230, 1293, 280, 343, 1477, 1540, 662, 725, 1355, 1418, 535, + 598, 789, 852, 1293, 1356, 1417, 1480, 344, 407, 1478, 1541, 1042, 1105, + 1105, 1168, 979, 1042, 1168, 1231, 24, 24, 408, 471, 916, 979, 1231, + 1294, 1479, 1542, 1536, 1536, 25, 88, 1537, 1600, 726, 789, 1356, 1419, + 89, 152, 599, 662, 1418, 1481, 1538, 1601, 153, 216, 1539, 1602, 853, + 916, 1294, 1357, 472, 535, 1480, 1543, 217, 280, 1540, 1603, 1106, 1169, + 281, 344, 663, 726, 1043, 1106, 1169, 1232, 1419, 1482, 1541, 1604, 790, + 853, 1357, 1420, 980, 1043, 1232, 1295, 536, 599, 1481, 1544, 345, 408, + 1542, 1605, 917, 980, 1295, 1358, 727, 790, 1420, 1483, 409, 472, 1543, + 1606, 25, 25, 600, 663, 1482, 1545, 1600, 1600, 26, 89, 1601, 1664, + 90, 153, 854, 917, 1358, 1421, 1602, 1665, 154, 217, 1107, 1170, 1170, + 1233, 1603, 1666, 473, 536, 1044, 1107, 1233, 1296, 1544, 1607, 218, 281, + 1604, 1667, 664, 727, 981, 1044, 1296, 1359, 1483, 1546, 791, 854, 1421, + 1484, 282, 345, 1605, 1668, 537, 600, 1545, 1608, 918, 981, 1359, 1422, + 346, 409, 1606, 1669, 728, 791, 1484, 1547, 1171, 1234, 1108, 1171, 1234, + 1297, 410, 473, 601, 664, 855, 918, 1422, 1485, 1546, 1609, 1607, 1670, + 26, 26, 1664, 1664, 27, 90, 1045, 1108, 1297, 1360, 1665, 1728, 91, + 154, 1666, 1729, 155, 218, 1667, 1730, 474, 537, 982, 1045, 1360, 1423, + 1608, 1671, 219, 282, 792, 855, 1485, 1548, 1668, 1731, 665, 728, 1547, + 1610, 283, 346, 919, 982, 1423, 1486, 1669, 1732, 538, 601, 1609, 1672, + 1172, 1235, 1235, 1298, 347, 410, 1109, 1172, 1298, 1361, 1670, 1733, 729, + 792, 1548, 1611, 856, 919, 1486, 1549, 1046, 1109, 1361, 1424, 602, 665, + 1610, 1673, 411, 474, 1671, 1734, 27, 27, 1728, 1728, 28, 91, 983, + 1046, 1424, 1487, 1729, 1792, 92, 155, 1730, 1793, 156, 219, 475, 538, + 1672, 1735, 1731, 1794, 793, 856, 1549, 1612, 666, 729, 1611, 1674, 220, + 283, 1236, 1299, 1732, 1795, 920, 983, 1487, 1550, 1173, 1236, 1299, 1362, + 1110, 1173, 1362, 1425, 284, 347, 1733, 1796, 539, 602, 1673, 1736, 1047, + 1110, 1425, 1488, 348, 411, 730, 793, 1612, 1675, 1734, 1797, 857, 920, + 1550, 1613, 603, 666, 1674, 1737, 984, 1047, 1488, 1551, 412, 475, 1735, + 1798, 28, 28, 1237, 1300, 1300, 1363, 1792, 1792, 29, 92, 1793, 1856, + 93, 156, 794, 857, 1174, 1237, 1363, 1426, 1613, 1676, 1794, 1857, 476, + 539, 1736, 1799, 157, 220, 667, 730, 921, 984, 1551, 1614, 1675, 1738, + 1795, 1858, 1111, 1174, 1426, 1489, 221, 284, 1796, 1859, 540, 603, 1048, + 1111, 1489, 1552, 1737, 1800, 285, 348, 1797, 1860, 858, 921, 1614, 1677, + 731, 794, 1676, 1739, 349, 412, 1798, 1861, 985, 1048, 1552, 1615, 1301, + 1364, 604, 667, 1238, 1301, 1364, 1427, 1738, 1801, 413, 476, 1175, 1238, + 1427, 1490, 1799, 1862, 795, 858, 1677, 1740, 29, 29, 1112, 1175, 1490, + 1553, 1856, 1856, 30, 93, 922, 985, 1615, 1678, 1857, 1920, 94, 157, + 1858, 1921, 477, 540, 668, 731, 1739, 1802, 1800, 1863, 158, 221, 1859, + 1922, 1049, 1112, 1553, 1616, 222, 285, 1860, 1923, 541, 604, 1801, 1864, + 286, 349, 859, 922, 1302, 1365, 1365, 1428, 1678, 1741, 1861, 1924, 732, + 795, 1740, 1803, 1239, 1302, 1428, 1491, 986, 1049, 1616, 1679, 350, 413, + 1862, 1925, 1176, 1239, 1491, 1554, 605, 668, 1802, 1865, 414, 477, 1113, + 1176, 1554, 1617, 1863, 1926, 796, 859, 1741, 1804, 923, 986, 1679, 1742, + 30, 30, 1920, 1920, 31, 94, 669, 732, 1803, 1866, 1921, 1984, 478, + 541, 1864, 1927, 95, 158, 1050, 1113, 1617, 1680, 1922, 1985, 1366, 1429, + 159, 222, 1303, 1366, 1429, 1492, 1923, 1986, 1240, 1303, 1492, 1555, 223, + 286, 1924, 1987, 860, 923, 1742, 1805, 542, 605, 1865, 1928, 733, 796, + 987, 1050, 1680, 1743, 1804, 1867, 287, 350, 1177, 1240, 1555, 1618, 1925, + 1988, 351, 414, 1926, 1989, 606, 669, 1114, 1177, 1618, 1681, 1866, 1929, + 924, 987, 1743, 1806, 415, 478, 797, 860, 1805, 1868, 1927, 1990, 1367, + 1430, 1430, 1493, 1304, 1367, 1493, 1556, 1051, 1114, 1681, 1744, 670, 733, + 1867, 1930, 31, 31, 1984, 1984, 32, 95, 479, 542, 1241, 1304, 1556, + 1619, 1928, 1991, 1985, 2048, 96, 159, 1986, 2049, 160, 223, 1987, 2050, + 861, 924, 1178, 1241, 1619, 1682, 1806, 1869, 224, 287, 988, 1051, 1744, + 1807, 1988, 2051, 543, 606, 1929, 1992, 734, 797, 1868, 1931, 288, 351, + 1989, 2052, 1115, 1178, 1682, 1745, 1431, 1494, 352, 415, 1368, 1431, 1494, + 1557, 1990, 2053, 607, 670, 1930, 1993, 925, 988, 1305, 1368, 1557, 1620, + 1807, 1870, 798, 861, 1869, 1932, 416, 479, 1052, 1115, 1745, 1808, 1991, + 2054, 1242, 1305, 1620, 1683, 671, 734, 1931, 1994, 480, 543, 1992, 2055, + 32, 32, 2048, 2048, 33, 96, 1179, 1242, 1683, 1746, 2049, 2112, 97, + 160, 2050, 2113, 862, 925, 1870, 1933, 989, 1052, 1808, 1871, 161, 224, + 2051, 2114, 225, 288, 544, 607, 735, 798, 1432, 1495, 1495, 1558, 1932, + 1995, 1993, 2056, 2052, 2115, 1116, 1179, 1746, 1809, 1369, 1432, 1558, 1621, + 289, 352, 2053, 2116, 1306, 1369, 1621, 1684, 608, 671, 1994, 2057, 353, + 416, 926, 989, 1871, 1934, 2054, 2117, 1243, 1306, 1684, 1747, 799, 862, + 1053, 1116, 1809, 1872, 1933, 1996, 417, 480, 2055, 2118, 672, 735, 1180, + 1243, 1747, 1810, 1995, 2058, 1496, 1559, 481, 544, 2056, 2119, 1433, 1496, + 1559, 1622, 33, 33, 990, 1053, 1872, 1935, 2112, 2112, 34, 97, 863, + 926, 1934, 1997, 2113, 2176, 98, 161, 1370, 1433, 1622, 1685, 2114, 2177, + 162, 225, 1117, 1180, 1810, 1873, 2115, 2178, 736, 799, 1996, 2059, 545, + 608, 1307, 1370, 1685, 1748, 2057, 2120, 226, 289, 2116, 2179, 290, 353, + 2117, 2180, 1244, 1307, 1748, 1811, 927, 990, 1935, 1998, 609, 672, 1054, + 1117, 1873, 1936, 2058, 2121, 354, 417, 2118, 2181, 800, 863, 1997, 2060, + 1497, 1560, 1560, 1623, 1181, 1244, 1811, 1874, 418, 481, 1434, 1497, 1623, + 1686, 2119, 2182, 673, 736, 2059, 2122, 1371, 1434, 1686, 1749, 991, 1054, + 1936, 1999, 482, 545, 864, 927, 1998, 2061, 2120, 2183, 1118, 1181, 1874, + 1937, 34, 34, 1308, 1371, 1749, 1812, 2176, 2176, 35, 98, 2177, 2240, + 99, 162, 2178, 2241, 737, 800, 2060, 2123, 163, 226, 2179, 2242, 546, + 609, 2121, 2184, 227, 290, 1245, 1308, 1812, 1875, 2180, 2243, 928, 991, + 1999, 2062, 291, 354, 1055, 1118, 1561, 1624, 1937, 2000, 2181, 2244, 1498, + 1561, 1624, 1687, 610, 673, 2122, 2185, 801, 864, 1435, 1498, 1687, 1750, + 2061, 2124, 355, 418, 1182, 1245, 1875, 1938, 2182, 2245, 1372, 1435, 1750, + 1813, 419, 482, 2183, 2246, 674, 737, 2123, 2186, 992, 1055, 2000, 2063, + 1309, 1372, 1813, 1876, 865, 928, 1119, 1182, 1938, 2001, 2062, 2125, 483, + 546, 2184, 2247, 35, 35, 2240, 2240, 36, 99, 2241, 2304, 100, 163, + 738, 801, 1246, 1309, 1876, 1939, 2124, 2187, 2242, 2305, 1562, 1625, 1625, + 1688, 164, 227, 1499, 1562, 1688, 1751, 2243, 2306, 547, 610, 2185, 2248, + 228, 291, 2244, 2307, 1056, 1119, 1436, 1499, 1751, 1814, 2001, 2064, 929, + 992, 2063, 2126, 292, 355, 2245, 2308, 1183, 1246, 1939, 2002, 611, 674, + 802, 865, 1373, 1436, 1814, 1877, 2125, 2188, 2186, 2249, 356, 419, 2246, + 2309, 1310, 1373, 1877, 1940, 420, 483, 993, 1056, 2064, 2127, 2247, 2310, + 675, 738, 2187, 2250, 1120, 1183, 2002, 2065, 866, 929, 1626, 1689, 2126, + 2189, 1563, 1626, 1689, 1752, 484, 547, 1500, 1563, 1752, 1815, 2248, 2311, + 1247, 1310, 1940, 2003, 36, 36, 739, 802, 2188, 2251, 2304, 2304, 37, + 100, 1437, 1500, 1815, 1878, 2305, 2368, 101, 164, 2306, 2369, 548, 611, + 2249, 2312, 165, 228, 1057, 1120, 2065, 2128, 2307, 2370, 930, 993, 2127, + 2190, 1374, 1437, 1878, 1941, 229, 292, 1184, 1247, 2003, 2066, 2308, 2371, + 293, 356, 803, 866, 2189, 2252, 2309, 2372, 612, 675, 2250, 2313, 1311, + 1374, 1941, 2004, 357, 420, 1627, 1690, 1690, 1753, 2310, 2373, 1564, 1627, + 1753, 1816, 994, 1057, 2128, 2191, 1121, 1184, 2066, 2129, 676, 739, 1501, + 1564, 1816, 1879, 2251, 2314, 421, 484, 2311, 2374, 867, 930, 2190, 2253, + 1248, 1311, 2004, 2067, 1438, 1501, 1879, 1942, 485, 548, 2312, 2375, 740, + 803, 2252, 2315, 37, 37, 2368, 2368, 38, 101, 1058, 1121, 1375, 1438, + 1942, 2005, 2129, 2192, 2369, 2432, 102, 165, 2370, 2433, 549, 612, 931, + 994, 1185, 1248, 2067, 2130, 2191, 2254, 2313, 2376, 166, 229, 2371, 2434, + 1691, 1754, 230, 293, 1628, 1691, 1754, 1817, 2372, 2435, 804, 867, 1312, + 1375, 2005, 2068, 2253, 2316, 1565, 1628, 1817, 1880, 294, 357, 613, 676, + 2314, 2377, 2373, 2436, 1502, 1565, 1880, 1943, 358, 421, 1122, 1185, 2130, + 2193, 2374, 2437, 995, 1058, 2192, 2255, 1249, 1312, 2068, 2131, 677, 740, + 1439, 1502, 1943, 2006, 2315, 2378, 868, 931, 2254, 2317, 422, 485, 2375, + 2438, 486, 549, 1376, 1439, 2006, 2069, 2376, 2439, 741, 804, 1692, 1755, + 1755, 1818, 2316, 2379, 1059, 1122, 2193, 2256, 1186, 1249, 1629, 1692, 1818, + 1881, 2131, 2194, 38, 38, 932, 995, 2255, 2318, 2432, 2432, 39, 102, + 2433, 2496, 103, 166, 550, 613, 1566, 1629, 1881, 1944, 2377, 2440, 2434, + 2497, 167, 230, 1313, 1376, 2069, 2132, 2435, 2498, 231, 294, 1503, 1566, + 1944, 2007, 2436, 2499, 805, 868, 2317, 2380, 614, 677, 2378, 2441, 295, + 358, 2437, 2500, 1123, 1186, 2194, 2257, 996, 1059, 2256, 2319, 1440, 1503, + 2007, 2070, 1250, 1313, 2132, 2195, 359, 422, 2438, 2501, 678, 741, 869, + 932, 2318, 2381, 2379, 2442, 1756, 1819, 423, 486, 1693, 1756, 1819, 1882, + 2439, 2502, 1377, 1440, 2070, 2133, 1630, 1693, 1882, 1945, 487, 550, 1060, + 1123, 2257, 2320, 2440, 2503, 1187, 1250, 1567, 1630, 1945, 2008, 2195, 2258, + 742, 805, 2380, 2443, 933, 996, 2319, 2382, 1314, 1377, 2133, 2196, 39, + 39, 1504, 1567, 2008, 2071, 2496, 2496, 40, 103, 2497, 2560, 551, 614, + 2441, 2504, 104, 167, 2498, 2561, 168, 231, 2499, 2562, 806, 869, 2381, + 2444, 232, 295, 2500, 2563, 1441, 1504, 2071, 2134, 1124, 1187, 2258, 2321, + 615, 678, 2442, 2505, 296, 359, 997, 1060, 1251, 1314, 1757, 1820, 1820, + 1883, 2196, 2259, 2320, 2383, 2501, 2564, 1694, 1757, 1883, 1946, 360, 423, + 2502, 2565, 1631, 1694, 1946, 2009, 870, 933, 1378, 1441, 2134, 2197, 2382, + 2445, 679, 742, 2443, 2506, 424, 487, 1568, 1631, 2009, 2072, 2503, 2566, + 1188, 1251, 2259, 2322, 1061, 1124, 2321, 2384, 488, 551, 2504, 2567, 743, + 806, 1505, 1568, 2072, 2135, 2444, 2507, 1315, 1378, 2197, 2260, 934, 997, + 2383, 2446, 40, 40, 552, 615, 2505, 2568, 2560, 2560, 41, 104, 1821, + 1884, 2561, 2624, 1758, 1821, 1884, 1947, 105, 168, 1442, 1505, 2135, 2198, + 2562, 2625, 169, 232, 807, 870, 1695, 1758, 1947, 2010, 2445, 2508, 2563, + 2626, 1125, 1188, 2322, 2385, 1252, 1315, 2260, 2323, 233, 296, 2564, 2627, + 616, 679, 998, 1061, 1632, 1695, 2010, 2073, 2384, 2447, 2506, 2569, 297, + 360, 2565, 2628, 1379, 1442, 2198, 2261, 1569, 1632, 2073, 2136, 361, 424, + 871, 934, 2446, 2509, 2566, 2629, 680, 743, 2507, 2570, 425, 488, 1189, + 1252, 2323, 2386, 2567, 2630, 1506, 1569, 2136, 2199, 1062, 1125, 2385, 2448, + 1316, 1379, 2261, 2324, 1822, 1885, 1885, 1948, 744, 807, 2508, 2571, 489, + 552, 1759, 1822, 1948, 2011, 2568, 2631, 935, 998, 2447, 2510, 1696, 1759, + 2011, 2074, 1443, 1506, 2199, 2262, 553, 616, 2569, 2632, 41, 41, 2624, + 2624, 42, 105, 1633, 1696, 2074, 2137, 2625, 2688, 106, 169, 1126, 1189, + 2386, 2449, 2626, 2689, 808, 871, 1253, 1316, 2324, 2387, 2509, 2572, 170, + 233, 2627, 2690, 999, 1062, 2448, 2511, 234, 297, 1380, 1443, 2262, 2325, + 2628, 2691, 617, 680, 1570, 1633, 2137, 2200, 2570, 2633, 298, 361, 2629, + 2692, 872, 935, 2510, 2573, 362, 425, 1886, 1949, 2630, 2693, 1507, 1570, + 2200, 2263, 681, 744, 1823, 1886, 1949, 2012, 2571, 2634, 1190, 1253, 2387, + 2450, 1760, 1823, 2012, 2075, 1063, 1126, 1317, 1380, 2325, 2388, 2449, 2512, + 426, 489, 2631, 2694, 1697, 1760, 2075, 2138, 745, 808, 936, 999, 1444, + 1507, 2263, 2326, 2511, 2574, 2572, 2635, 490, 553, 2632, 2695, 1634, 1697, + 2138, 2201, 1254, 1317, 2388, 2451, 554, 617, 1127, 1190, 2450, 2513, 2633, + 2696, 42, 42, 2688, 2688, 43, 106, 809, 872, 1571, 1634, 2201, 2264, + 2573, 2636, 2689, 2752, 107, 170, 1381, 1444, 2326, 2389, 2690, 2753, 1000, + 1063, 2512, 2575, 171, 234, 2691, 2754, 1887, 1950, 1950, 2013, 618, 681, + 2634, 2697, 235, 298, 1824, 1887, 2013, 2076, 2692, 2755, 1508, 1571, 2264, + 2327, 1761, 1824, 2076, 2139, 299, 362, 2693, 2756, 873, 936, 2574, 2637, + 1191, 1254, 2451, 2514, 363, 426, 682, 745, 1318, 1381, 1698, 1761, 2139, + 2202, 2389, 2452, 2635, 2698, 2694, 2757, 1064, 1127, 2513, 2576, 427, 490, + 1445, 1508, 2327, 2390, 2695, 2758, 1635, 1698, 2202, 2265, 937, 1000, 2575, + 2638, 746, 809, 2636, 2699, 491, 554, 2696, 2759, 1255, 1318, 1572, 1635, + 2265, 2328, 2452, 2515, 1951, 2014, 1128, 1191, 1888, 1951, 2014, 2077, 2514, + 2577, 1382, 1445, 2390, 2453, 555, 618, 1825, 1888, 2077, 2140, 2697, 2760, + 810, 873, 2637, 2700, 43, 43, 2752, 2752, 44, 107, 1001, 1064, 2576, + 2639, 2753, 2816, 108, 171, 1762, 1825, 2140, 2203, 2754, 2817, 172, 235, + 1509, 1572, 2328, 2391, 2755, 2818, 619, 682, 2698, 2761, 236, 299, 2756, + 2819, 1699, 1762, 2203, 2266, 874, 937, 2638, 2701, 300, 363, 1192, 1255, + 2515, 2578, 2757, 2820, 1319, 1382, 2453, 2516, 683, 746, 1065, 1128, 2577, + 2640, 2699, 2762, 364, 427, 1636, 1699, 2266, 2329, 2758, 2821, 1446, 1509, + 2391, 2454, 428, 491, 1952, 2015, 2015, 2078, 2759, 2822, 938, 1001, 1889, + 1952, 2078, 2141, 2639, 2702, 747, 810, 2700, 2763, 1573, 1636, 2329, 2392, + 1826, 1889, 2141, 2204, 492, 555, 1256, 1319, 2516, 2579, 2760, 2823, 1129, + 1192, 1383, 1446, 2454, 2517, 2578, 2641, 1763, 1826, 2204, 2267, 556, 619, + 2761, 2824, 811, 874, 2701, 2764, 1002, 1065, 1510, 1573, 2392, 2455, 2640, + 2703, 44, 44, 1700, 1763, 2267, 2330, 2816, 2816, 45, 108, 2817, 2880, + 109, 172, 2818, 2881, 173, 236, 2819, 2882, 620, 683, 2762, 2825, 237, + 300, 1320, 1383, 2517, 2580, 2820, 2883, 1193, 1256, 2579, 2642, 875, 938, + 1637, 1700, 2330, 2393, 2702, 2765, 2016, 2079, 301, 364, 1447, 1510, 1953, + 2016, 2079, 2142, 2455, 2518, 2821, 2884, 1066, 1129, 2641, 2704, 1890, 1953, + 2142, 2205, 684, 747, 2763, 2826, 365, 428, 2822, 2885, 1827, 1890, 2205, + 2268, 1574, 1637, 2393, 2456, 429, 492, 939, 1002, 2703, 2766, 2823, 2886, + 748, 811, 1764, 1827, 2268, 2331, 2764, 2827, 1257, 1320, 2580, 2643, 1384, + 1447, 2518, 2581, 1130, 1193, 2642, 2705, 493, 556, 2824, 2887, 1511, 1574, + 2456, 2519, 1701, 1764, 2331, 2394, 812, 875, 1003, 1066, 2704, 2767, 2765, + 2828, 557, 620, 2825, 2888, 2017, 2080, 2080, 2143, 45, 45, 2880, 2880, + 46, 109, 1954, 2017, 2143, 2206, 2881, 2944, 110, 173, 1638, 1701, 2394, + 2457, 2882, 2945, 1321, 1384, 2581, 2644, 174, 237, 621, 684, 1194, 1257, + 1891, 1954, 2206, 2269, 2643, 2706, 2826, 2889, 2883, 2946, 1448, 1511, 2519, + 2582, 238, 301, 876, 939, 2766, 2829, 2884, 2947, 1828, 1891, 2269, 2332, + 1067, 1130, 2705, 2768, 302, 365, 2885, 2948, 685, 748, 1575, 1638, 2457, + 2520, 2827, 2890, 366, 429, 2886, 2949, 1765, 1828, 2332, 2395, 940, 1003, + 2767, 2830, 1258, 1321, 2644, 2707, 430, 493, 1385, 1448, 2582, 2645, 2887, + 2950, 749, 812, 2828, 2891, 1131, 1194, 1702, 1765, 2395, 2458, 2706, 2769, + 1512, 1575, 2520, 2583, 2081, 2144, 494, 557, 2018, 2081, 2144, 2207, 2888, + 2951, 1955, 2018, 2207, 2270, 1004, 1067, 2768, 2831, 813, 876, 2829, 2892, + 1892, 1955, 2270, 2333, 558, 621, 1639, 1702, 2458, 2521, 2889, 2952, 1322, + 1385, 2645, 2708, 46, 46, 2944, 2944, 47, 110, 1195, 1258, 1449, 1512, + 1829, 1892, 2333, 2396, 2583, 2646, 2707, 2770, 2945, 3008, 111, 174, 2946, + 3009, 622, 685, 2890, 2953, 175, 238, 2947, 3010, 877, 940, 2830, 2893, + 239, 302, 1068, 1131, 1576, 1639, 2521, 2584, 2769, 2832, 2948, 3011, 1766, + 1829, 2396, 2459, 303, 366, 2949, 3012, 686, 749, 2891, 2954, 367, 430, + 2082, 2145, 2145, 2208, 2950, 3013, 1386, 1449, 2646, 2709, 1259, 1322, 2019, + 2082, 2208, 2271, 2708, 2771, 941, 1004, 1703, 1766, 2459, 2522, 2831, 2894, + 1513, 1576, 1956, 2019, 2271, 2334, 2584, 2647, 431, 494, 2951, 3014, 750, + 813, 1132, 1195, 2770, 2833, 2892, 2955, 1893, 1956, 2334, 2397, 495, 558, + 2952, 3015, 1640, 1703, 2522, 2585, 1005, 1068, 2832, 2895, 814, 877, 1830, + 1893, 2397, 2460, 2893, 2956, 559, 622, 1323, 1386, 2709, 2772, 2953, 3016, + 1450, 1513, 2647, 2710, 1196, 1259, 2771, 2834, 47, 47, 3008, 3008, 48, + 111, 1767, 1830, 2460, 2523, 3009, 3072, 1577, 1640, 2585, 2648, 112, 175, + 3010, 3073, 623, 686, 2954, 3017, 878, 941, 2146, 2209, 2894, 2957, 176, + 239, 3011, 3074, 1069, 1132, 2083, 2146, 2209, 2272, 2833, 2896, 240, 303, + 2020, 2083, 2272, 2335, 3012, 3075, 304, 367, 1704, 1767, 2523, 2586, 3013, + 3076, 687, 750, 1957, 2020, 2335, 2398, 2955, 3018, 1387, 1450, 2710, 2773, + 1260, 1323, 2772, 2835, 368, 431, 1514, 1577, 2648, 2711, 3014, 3077, 942, + 1005, 2895, 2958, 1894, 1957, 2398, 2461, 1133, 1196, 2834, 2897, 432, 495, + 751, 814, 2956, 3019, 3015, 3078, 1641, 1704, 2586, 2649, 1831, 1894, 2461, + 2524, 496, 559, 3016, 3079, 1006, 1069, 2896, 2959, 1324, 1387, 2773, 2836, + 815, 878, 1451, 1514, 2711, 2774, 2957, 3020, 2147, 2210, 2210, 2273, 1768, + 1831, 2524, 2587, 560, 623, 2084, 2147, 2273, 2336, 3017, 3080, 1197, 1260, + 2835, 2898, 1578, 1641, 2649, 2712, 2021, 2084, 2336, 2399, 48, 48, 3072, + 3072, 49, 112, 3073, 3136, 624, 687, 3018, 3081, 113, 176, 879, 942, + 1070, 1133, 1958, 2021, 2399, 2462, 2897, 2960, 2958, 3021, 3074, 3137, 177, + 240, 1705, 1768, 2587, 2650, 3075, 3138, 241, 304, 3076, 3139, 1388, 1451, + 2774, 2837, 1895, 1958, 2462, 2525, 688, 751, 1261, 1324, 1515, 1578, 2712, + 2775, 2836, 2899, 3019, 3082, 305, 368, 3077, 3140, 943, 1006, 2959, 3022, + 369, 432, 3078, 3141, 1134, 1197, 1642, 1705, 2650, 2713, 2898, 2961, 1832, + 1895, 2525, 2588, 752, 815, 3020, 3083, 433, 496, 2211, 2274, 3079, 3142, + 2148, 2211, 2274, 2337, 2085, 2148, 2337, 2400, 497, 560, 1007, 1070, 1452, + 1515, 1769, 1832, 2588, 2651, 2775, 2838, 2960, 3023, 3080, 3143, 1325, 1388, + 2837, 2900, 2022, 2085, 2400, 2463, 816, 879, 3021, 3084, 1579, 1642, 2713, + 2776, 1198, 1261, 2899, 2962, 561, 624, 1959, 2022, 2463, 2526, 3081, 3144, + 1706, 1769, 2651, 2714, 1071, 1134, 2961, 3024, 49, 49, 880, 943, 1896, + 1959, 2526, 2589, 3022, 3085, 3136, 3136, 50, 113, 625, 688, 3082, 3145, + 3137, 3200, 114, 177, 3138, 3201, 178, 241, 1389, 1452, 2838, 2901, 3139, + 3202, 1516, 1579, 2776, 2839, 242, 305, 1262, 1325, 2900, 2963, 3140, 3203, + 2212, 2275, 2275, 2338, 689, 752, 1833, 1896, 2589, 2652, 3083, 3146, 306, + 369, 1643, 1706, 2149, 2212, 2338, 2401, 2714, 2777, 3141, 3204, 944, 1007, + 3023, 3086, 1135, 1198, 2086, 2149, 2401, 2464, 2962, 3025, 370, 433, 3142, + 3205, 753, 816, 2023, 2086, 2464, 2527, 3084, 3147, 1770, 1833, 2652, 2715, + 434, 497, 3143, 3206, 1453, 1516, 2839, 2902, 1326, 1389, 2901, 2964, 1008, + 1071, 3024, 3087, 1580, 1643, 1960, 2023, 2527, 2590, 2777, 2840, 498, 561, + 3144, 3207, 817, 880, 1199, 1262, 2963, 3026, 3085, 3148, 1707, 1770, 2715, + 2778, 562, 625, 1897, 1960, 2590, 2653, 3145, 3208, 2276, 2339, 1072, 1135, + 3025, 3088, 2213, 2276, 2339, 2402, 881, 944, 3086, 3149, 626, 689, 1390, + 1453, 2150, 2213, 2402, 2465, 2902, 2965, 3146, 3209, 50, 50, 1517, 1580, + 2840, 2903, 3200, 3200, 51, 114, 3201, 3264, 115, 178, 1834, 1897, 2653, + 2716, 3202, 3265, 1263, 1326, 2964, 3027, 179, 242, 2087, 2150, 2465, 2528, + 3203, 3266, 1644, 1707, 2778, 2841, 243, 306, 3204, 3267, 690, 753, 3147, + 3210, 2024, 2087, 2528, 2591, 307, 370, 945, 1008, 3087, 3150, 3205, 3268, + 1136, 1199, 3026, 3089, 1771, 1834, 2716, 2779, 371, 434, 3206, 3269, 1961, + 2024, 2591, 2654, 754, 817, 3148, 3211, 1454, 1517, 2903, 2966, 435, 498, + 1327, 1390, 1581, 1644, 2841, 2904, 2965, 3028, 3207, 3270, 1009, 1072, 3088, + 3151, 1898, 1961, 2654, 2717, 499, 562, 1200, 1263, 1708, 1771, 2277, 2340, + 2340, 2403, 2779, 2842, 3027, 3090, 3208, 3271, 818, 881, 2214, 2277, 2403, + 2466, 3149, 3212, 2151, 2214, 2466, 2529, 563, 626, 3209, 3272, 2088, 2151, + 2529, 2592, 1073, 1136, 1835, 1898, 2717, 2780, 3089, 3152, 1518, 1581, 2904, + 2967, 1391, 1454, 2966, 3029, 882, 945, 3150, 3213, 627, 690, 1645, 1708, + 2842, 2905, 3210, 3273, 51, 51, 1264, 1327, 3028, 3091, 3264, 3264, 52, + 115, 2025, 2088, 2592, 2655, 3265, 3328, 116, 179, 3266, 3329, 180, 243, + 3267, 3330, 244, 307, 1772, 1835, 2780, 2843, 3268, 3331, 691, 754, 3211, + 3274, 946, 1009, 1137, 1200, 1962, 2025, 2655, 2718, 3090, 3153, 3151, 3214, + 308, 371, 3269, 3332, 1455, 1518, 2341, 2404, 2967, 3030, 372, 435, 2278, + 2341, 2404, 2467, 3270, 3333, 1582, 1645, 2905, 2968, 755, 818, 1328, 1391, + 3029, 3092, 3212, 3275, 2215, 2278, 2467, 2530, 1899, 1962, 2718, 2781, 436, + 499, 3271, 3334, 1709, 1772, 2843, 2906, 1010, 1073, 2152, 2215, 2530, 2593, + 3152, 3215, 1201, 1264, 3091, 3154, 500, 563, 3272, 3335, 819, 882, 2089, + 2152, 2593, 2656, 3213, 3276, 1836, 1899, 2781, 2844, 564, 627, 1519, 1582, + 2968, 3031, 3273, 3336, 1392, 1455, 2026, 2089, 2656, 2719, 3030, 3093, 1074, + 1137, 3153, 3216, 1646, 1709, 2906, 2969, 883, 946, 3214, 3277, 1265, 1328, + 3092, 3155, 628, 691, 3274, 3337, 52, 52, 1773, 1836, 2844, 2907, 3328, + 3328, 53, 116, 1963, 2026, 2719, 2782, 3329, 3392, 117, 180, 2342, 2405, + 2405, 2468, 3330, 3393, 2279, 2342, 2468, 2531, 181, 244, 3331, 3394, 1138, + 1201, 3154, 3217, 245, 308, 692, 755, 2216, 2279, 2531, 2594, 3275, 3338, + 3332, 3395, 947, 1010, 3215, 3278, 1456, 1519, 3031, 3094, 309, 372, 1583, + 1646, 2969, 3032, 3333, 3396, 1900, 1963, 2782, 2845, 2153, 2216, 2594, 2657, + 1329, 1392, 3093, 3156, 373, 436, 1710, 1773, 2907, 2970, 3334, 3397, 756, + 819, 3276, 3339, 2090, 2153, 2657, 2720, 1011, 1074, 3216, 3279, 437, 500, + 3335, 3398, 1202, 1265, 3155, 3218, 1837, 1900, 2845, 2908, 501, 564, 820, + 883, 2027, 2090, 2720, 2783, 3277, 3340, 3336, 3399, 1520, 1583, 3032, 3095, + 1393, 1456, 1647, 1710, 2970, 3033, 3094, 3157, 2406, 2469, 565, 628, 1075, + 1138, 2343, 2406, 2469, 2532, 3217, 3280, 3337, 3400, 2280, 2343, 2532, 2595, + 1964, 2027, 2783, 2846, 884, 947, 1266, 1329, 1774, 1837, 2908, 2971, 3156, + 3219, 3278, 3341, 2217, 2280, 2595, 2658, 629, 692, 3338, 3401, 53, 53, + 3392, 3392, 54, 117, 3393, 3456, 118, 181, 2154, 2217, 2658, 2721, 3394, + 3457, 182, 245, 1139, 1202, 1901, 1964, 2846, 2909, 3218, 3281, 3395, 3458, + 948, 1011, 1584, 1647, 3033, 3096, 3279, 3342, 693, 756, 1457, 1520, 3095, + 3158, 3339, 3402, 246, 309, 3396, 3459, 1711, 1774, 2091, 2154, 2721, 2784, + 2971, 3034, 310, 373, 1330, 1393, 3157, 3220, 3397, 3460, 374, 437, 3398, + 3461, 757, 820, 3340, 3403, 1838, 1901, 2909, 2972, 1012, 1075, 2028, 2091, + 2784, 2847, 3280, 3343, 1203, 1266, 3219, 3282, 438, 501, 2407, 2470, 2470, + 2533, 3399, 3462, 2344, 2407, 2533, 2596, 1521, 1584, 2281, 2344, 2596, 2659, + 3096, 3159, 821, 884, 3341, 3404, 502, 565, 1648, 1711, 3034, 3097, 3400, + 3463, 1394, 1457, 3158, 3221, 1965, 2028, 2847, 2910, 2218, 2281, 2659, 2722, + 1076, 1139, 1775, 1838, 2972, 3035, 3281, 3344, 566, 629, 3401, 3464, 1267, + 1330, 3220, 3283, 885, 948, 2155, 2218, 2722, 2785, 3342, 3405, 630, 693, + 1902, 1965, 2910, 2973, 3402, 3465, 54, 54, 2092, 2155, 2785, 2848, 3456, + 3456, 55, 118, 1585, 1648, 3097, 3160, 3457, 3520, 1140, 1203, 3282, 3345, + 119, 182, 1458, 1521, 3159, 3222, 3458, 3521, 1712, 1775, 3035, 3098, 183, + 246, 949, 1012, 3343, 3406, 3459, 3522, 694, 757, 3403, 3466, 247, 310, + 3460, 3523, 1331, 1394, 2471, 2534, 3221, 3284, 2408, 2471, 2534, 2597, 2029, + 2092, 2848, 2911, 311, 374, 1839, 1902, 2345, 2408, 2597, 2660, 2973, 3036, + 3461, 3524, 758, 821, 2282, 2345, 2660, 2723, 3404, 3467, 375, 438, 3462, + 3525, 1013, 1076, 1204, 1267, 3283, 3346, 3344, 3407, 439, 502, 2219, 2282, + 2723, 2786, 3463, 3526, 1522, 1585, 3160, 3223, 1649, 1712, 1966, 2029, 2911, + 2974, 3098, 3161, 822, 885, 1395, 1458, 3222, 3285, 3405, 3468, 1776, 1839, + 3036, 3099, 503, 566, 3464, 3527, 2156, 2219, 2786, 2849, 1077, 1140, 3345, + 3408, 1268, 1331, 3284, 3347, 567, 630, 3465, 3528, 1903, 1966, 2974, 3037, + 886, 949, 3406, 3469, 2093, 2156, 2849, 2912, 2472, 2535, 2535, 2598, 631, + 694, 1586, 1649, 2409, 2472, 2598, 2661, 3161, 3224, 3466, 3529, 1459, 1522, + 1713, 1776, 3099, 3162, 3223, 3286, 1141, 1204, 2346, 2409, 2661, 2724, 3346, + 3409, 55, 55, 3520, 3520, 56, 119, 3521, 3584, 120, 183, 2030, 2093, + 2912, 2975, 3522, 3585, 950, 1013, 3407, 3470, 184, 247, 1332, 1395, 1840, + 1903, 2283, 2346, 2724, 2787, 3037, 3100, 3285, 3348, 3523, 3586, 695, 758, + 3467, 3530, 248, 311, 3524, 3587, 312, 375, 2220, 2283, 2787, 2850, 3525, + 3588, 759, 822, 3468, 3531, 1205, 1268, 1967, 2030, 2975, 3038, 3347, 3410, + 376, 439, 1014, 1077, 3408, 3471, 3526, 3589, 1650, 1713, 3162, 3225, 1523, + 1586, 3224, 3287, 2157, 2220, 2850, 2913, 440, 503, 1777, 1840, 3100, 3163, + 3527, 3590, 1396, 1459, 3286, 3349, 823, 886, 3469, 3532, 504, 567, 2536, + 2599, 3528, 3591, 2473, 2536, 2599, 2662, 1904, 1967, 3038, 3101, 1078, 1141, + 2094, 2157, 2913, 2976, 3409, 3472, 2410, 2473, 2662, 2725, 1269, 1332, 3348, + 3411, 568, 631, 3529, 3592, 2347, 2410, 2725, 2788, 887, 950, 3470, 3533, + 1587, 1650, 3225, 3288, 1714, 1777, 3163, 3226, 2284, 2347, 2788, 2851, 1460, + 1523, 2031, 2094, 2976, 3039, 3287, 3350, 632, 695, 3530, 3593, 1142, 1205, + 3410, 3473, 1841, 1904, 3101, 3164, 56, 56, 3584, 3584, 57, 120, 951, + 1014, 1333, 1396, 2221, 2284, 2851, 2914, 3349, 3412, 3471, 3534, 3585, 3648, + 121, 184, 3586, 3649, 696, 759, 3531, 3594, 185, 248, 3587, 3650, 249, + 312, 1968, 2031, 3039, 3102, 3588, 3651, 2158, 2221, 2914, 2977, 313, 376, + 3589, 3652, 1206, 1269, 1651, 1714, 3226, 3289, 3411, 3474, 760, 823, 1524, + 1587, 3288, 3351, 3532, 3595, 1015, 1078, 2537, 2600, 2600, 2663, 3472, 3535, + 1778, 1841, 3164, 3227, 377, 440, 2474, 2537, 2663, 2726, 3590, 3653, 1397, + 1460, 2411, 2474, 2726, 2789, 3350, 3413, 441, 504, 2095, 2158, 2977, 3040, + 3591, 3654, 1905, 1968, 3102, 3165, 824, 887, 2348, 2411, 2789, 2852, 3533, + 3596, 505, 568, 3592, 3655, 1079, 1142, 3473, 3536, 1270, 1333, 3412, 3475, + 2285, 2348, 2852, 2915, 2032, 2095, 3040, 3103, 1588, 1651, 3289, 3352, 569, + 632, 1715, 1778, 3227, 3290, 3593, 3656, 888, 951, 3534, 3597, 1461, 1524, + 3351, 3414, 1842, 1905, 2222, 2285, 2915, 2978, 3165, 3228, 633, 696, 1143, + 1206, 3474, 3537, 3594, 3657, 1334, 1397, 3413, 3476, 952, 1015, 3535, 3598, + 1969, 2032, 2601, 2664, 3103, 3166, 57, 57, 2538, 2601, 2664, 2727, 3648, + 3648, 58, 121, 2159, 2222, 2978, 3041, 3649, 3712, 122, 185, 3650, 3713, + 697, 760, 2475, 2538, 2727, 2790, 3595, 3658, 186, 249, 3651, 3714, 250, + 313, 1652, 1715, 2412, 2475, 2790, 2853, 3290, 3353, 3652, 3715, 1525, 1588, + 1779, 1842, 3228, 3291, 3352, 3415, 1207, 1270, 3475, 3538, 314, 377, 3653, + 3716, 1016, 1079, 3536, 3599, 761, 824, 2096, 2159, 3041, 3104, 3596, 3659, + 2349, 2412, 2853, 2916, 378, 441, 1398, 1461, 1906, 1969, 3166, 3229, 3414, + 3477, 3654, 3717, 2286, 2349, 2916, 2979, 442, 505, 3655, 3718, 825, 888, + 3597, 3660, 1080, 1143, 1271, 1334, 2033, 2096, 3104, 3167, 3476, 3539, 3537, + 3600, 506, 569, 3656, 3719, 1716, 1779, 3291, 3354, 1589, 1652, 2223, 2286, + 2979, 3042, 3353, 3416, 1843, 1906, 3229, 3292, 570, 633, 889, 952, 1462, + 1525, 2602, 2665, 2665, 2728, 3415, 3478, 3598, 3661, 3657, 3720, 2539, 2602, + 2728, 2791, 2476, 2539, 2791, 2854, 1144, 1207, 2160, 2223, 3042, 3105, 3538, + 3601, 1970, 2033, 3167, 3230, 634, 697, 3658, 3721, 1335, 1398, 3477, 3540, + 2413, 2476, 2854, 2917, 953, 1016, 3599, 3662, 58, 58, 3712, 3712, 59, + 122, 3713, 3776, 123, 186, 698, 761, 1653, 1716, 2350, 2413, 2917, 2980, + 3354, 3417, 3659, 3722, 3714, 3777, 1780, 1843, 3292, 3355, 187, 250, 2097, + 2160, 3105, 3168, 3715, 3778, 1526, 1589, 3416, 3479, 251, 314, 1208, 1271, + 3539, 3602, 3716, 3779, 1907, 1970, 3230, 3293, 1017, 1080, 2287, 2350, 2980, + 3043, 3600, 3663, 315, 378, 3717, 3780, 762, 825, 3660, 3723, 1399, 1462, + 3478, 3541, 379, 442, 3718, 3781, 2034, 2097, 3168, 3231, 2666, 2729, 2224, + 2287, 3043, 3106, 443, 506, 2603, 2666, 2729, 2792, 3719, 3782, 826, 889, + 3661, 3724, 1272, 1335, 2540, 2603, 2792, 2855, 3540, 3603, 1081, 1144, 1717, + 1780, 3355, 3418, 3601, 3664, 1590, 1653, 3417, 3480, 507, 570, 1844, 1907, + 3293, 3356, 3720, 3783, 2477, 2540, 2855, 2918, 1463, 1526, 3479, 3542, 2161, + 2224, 3106, 3169, 890, 953, 2414, 2477, 2918, 2981, 3662, 3725, 571, 634, + 1971, 2034, 3231, 3294, 3721, 3784, 1145, 1208, 3602, 3665, 1336, 1399, 3541, + 3604, 2351, 2414, 2981, 3044, 635, 698, 3722, 3785, 954, 1017, 2098, 2161, + 3169, 3232, 3663, 3726, 1654, 1717, 3418, 3481, 1781, 1844, 3356, 3419, 59, + 59, 2288, 2351, 3044, 3107, 3776, 3776, 60, 123, 1527, 1590, 3480, 3543, + 3777, 3840, 699, 762, 3723, 3786, 124, 187, 1908, 1971, 3294, 3357, 3778, + 3841, 188, 251, 3779, 3842, 1209, 1272, 3603, 3666, 2667, 2730, 2730, 2793, + 252, 315, 3780, 3843, 2604, 2667, 2793, 2856, 1018, 1081, 1400, 1463, 3542, + 3605, 3664, 3727, 316, 379, 763, 826, 2035, 2098, 2541, 2604, 2856, 2919, + 3232, 3295, 3724, 3787, 3781, 3844, 2225, 2288, 3107, 3170, 380, 443, 3782, + 3845, 2478, 2541, 2919, 2982, 1718, 1781, 3419, 3482, 444, 507, 1273, 1336, + 3604, 3667, 3783, 3846, 827, 890, 1591, 1654, 1845, 1908, 3357, 3420, 3481, + 3544, 3725, 3788, 1082, 1145, 2415, 2478, 2982, 3045, 3665, 3728, 2162, 2225, + 3170, 3233, 508, 571, 3784, 3847, 1464, 1527, 1972, 2035, 3295, 3358, 3543, + 3606, 2352, 2415, 3045, 3108, 891, 954, 3726, 3789, 572, 635, 3785, 3848, + 1146, 1209, 3666, 3729, 1337, 1400, 2099, 2162, 3233, 3296, 3605, 3668, 2289, + 2352, 3108, 3171, 2731, 2794, 636, 699, 1782, 1845, 2668, 2731, 2794, 2857, + 3420, 3483, 3786, 3849, 1655, 1718, 3482, 3545, 955, 1018, 2605, 2668, 2857, + 2920, 3727, 3790, 1909, 1972, 3358, 3421, 1528, 1591, 3544, 3607, 2542, 2605, + 2920, 2983, 60, 60, 700, 763, 3787, 3850, 3840, 3840, 61, 124, 3841, + 3904, 125, 188, 1210, 1273, 2226, 2289, 3171, 3234, 3667, 3730, 3842, 3905, + 2036, 2099, 3296, 3359, 189, 252, 2479, 2542, 2983, 3046, 3843, 3906, 1401, + 1464, 3606, 3669, 253, 316, 1019, 1082, 3728, 3791, 3844, 3907, 764, 827, + 3788, 3851, 317, 380, 3845, 3908, 2416, 2479, 3046, 3109, 1719, 1782, 3483, + 3546, 381, 444, 1846, 1909, 2163, 2226, 3234, 3297, 3421, 3484, 3846, 3909, + 1592, 1655, 3545, 3608, 1274, 1337, 3668, 3731, 828, 891, 3789, 3852, 445, + 508, 1083, 1146, 1973, 2036, 2353, 2416, 3109, 3172, 3359, 3422, 3729, 3792, + 3847, 3910, 1465, 1528, 3607, 3670, 509, 572, 2732, 2795, 2795, 2858, 3848, + 3911, 2669, 2732, 2858, 2921, 2100, 2163, 3297, 3360, 892, 955, 2290, 2353, + 3172, 3235, 3790, 3853, 2606, 2669, 2921, 2984, 573, 636, 3849, 3912, 1147, + 1210, 1338, 1401, 3669, 3732, 3730, 3793, 1783, 1846, 2543, 2606, 2984, 3047, + 3484, 3547, 1656, 1719, 3546, 3609, 1910, 1973, 3422, 3485, 637, 700, 3850, + 3913, 956, 1019, 1529, 1592, 2480, 2543, 3047, 3110, 3608, 3671, 3791, 3854, + 2227, 2290, 3235, 3298, 2037, 2100, 3360, 3423, 701, 764, 1211, 1274, 3731, + 3794, 3851, 3914, 61, 61, 3904, 3904, 62, 125, 2417, 2480, 3110, 3173, + 3905, 3968, 126, 189, 1402, 1465, 3670, 3733, 3906, 3969, 190, 253, 3907, + 3970, 1020, 1083, 3792, 3855, 254, 317, 2164, 2227, 3298, 3361, 3908, 3971, + 765, 828, 1720, 1783, 3547, 3610, 3852, 3915, 1847, 1910, 3485, 3548, 318, + 381, 2354, 2417, 3173, 3236, 3909, 3972, 2796, 2859, 1593, 1656, 2733, 2796, + 2859, 2922, 3609, 3672, 1974, 2037, 3423, 3486, 382, 445, 2670, 2733, 2922, + 2985, 3910, 3973, 1275, 1338, 3732, 3795, 1084, 1147, 3793, 3856, 829, 892, + 2607, 2670, 2985, 3048, 3853, 3916, 446, 509, 1466, 1529, 3671, 3734, 3911, + 3974, 2291, 2354, 3236, 3299, 2101, 2164, 3361, 3424, 2544, 2607, 3048, 3111, + 510, 573, 3912, 3975, 893, 956, 3854, 3917, 1784, 1847, 3548, 3611, 1339, + 1402, 2481, 2544, 3111, 3174, 3733, 3796, 1148, 1211, 3794, 3857, 574, 637, + 1657, 1720, 1911, 1974, 3486, 3549, 3610, 3673, 3913, 3976, 2228, 2291, 3299, + 3362, 1530, 1593, 2038, 2101, 3424, 3487, 3672, 3735, 638, 701, 2418, 2481, + 3174, 3237, 3914, 3977, 957, 1020, 3855, 3918, 1212, 1275, 2797, 2860, 2860, + 2923, 3795, 3858, 702, 765, 1403, 1466, 2165, 2228, 2734, 2797, 2923, 2986, + 3362, 3425, 3734, 3797, 3915, 3978, 62, 62, 3968, 3968, 63, 126, 2355, + 2418, 3237, 3300, 3969, 4032, 127, 190, 2671, 2734, 2986, 3049, 3970, 4033, + 1021, 1084, 1848, 1911, 3549, 3612, 3856, 3919, 191, 254, 1721, 1784, 3611, + 3674, 3971, 4034, 255, 318, 2608, 2671, 3049, 3112, 3972, 4035, 1975, 2038, + 3487, 3550, 766, 829, 3916, 3979, 1594, 1657, 3673, 3736, 319, 382, 3973, + 4036, 1276, 1339, 2292, 2355, 3300, 3363, 3796, 3859, 2545, 2608, 3112, 3175, + 383, 446, 2102, 2165, 3425, 3488, 3974, 4037, 1085, 1148, 1467, 1530, 3735, + 3798, 3857, 3920, 830, 893, 3917, 3980, 447, 510, 3975, 4038, 2482, 2545, + 3175, 3238, 511, 574, 1785, 1848, 3612, 3675, 3976, 4039, 2229, 2292, 3363, + 3426, 1912, 1975, 3550, 3613, 894, 957, 1658, 1721, 3674, 3737, 3918, 3981, + 1340, 1403, 3797, 3860, 1149, 1212, 2419, 2482, 3238, 3301, 3858, 3921, 2039, + 2102, 3488, 3551, 575, 638, 2861, 2924, 3977, 4040, 2798, 2861, 2924, 2987, + 1531, 1594, 3736, 3799, 2735, 2798, 2987, 3050, 2672, 2735, 3050, 3113, 639, + 702, 958, 1021, 3919, 3982, 3978, 4041, 2166, 2229, 3426, 3489, 2356, 2419, + 3301, 3364, 1213, 1276, 2609, 2672, 3113, 3176, 3859, 3922, 1404, 1467, 3798, + 3861, 703, 766, 1849, 1912, 3613, 3676, 3979, 4042, 1722, 1785, 3675, 3738, + 1976, 2039, 3551, 3614, 1022, 1085, 2546, 2609, 3176, 3239, 3920, 3983, 2293, + 2356, 3364, 3427, 1595, 1658, 3737, 3800, 767, 830, 3980, 4043, 2103, 2166, + 3489, 3552, 1277, 1340, 3860, 3923, 2483, 2546, 3239, 3302, 1468, 1531, 3799, + 3862, 1086, 1149, 3921, 3984, 831, 894, 3981, 4044, 2230, 2293, 2862, 2925, + 2925, 2988, 3427, 3490, 2799, 2862, 2988, 3051, 1786, 1849, 2420, 2483, 3302, + 3365, 3676, 3739, 1913, 1976, 3614, 3677, 2736, 2799, 3051, 3114, 1659, 1722, + 3738, 3801, 2040, 2103, 3552, 3615, 1341, 1404, 3861, 3924, 895, 958, 2673, + 2736, 3114, 3177, 3982, 4045, 1150, 1213, 3922, 3985, 1532, 1595, 3800, 3863, + 2357, 2420, 3365, 3428, 2167, 2230, 2610, 2673, 3177, 3240, 3490, 3553, 959, + 1022, 3983, 4046, 2547, 2610, 3240, 3303, 1214, 1277, 1405, 1468, 1850, 1913, + 3677, 3740, 3862, 3925, 3923, 3986, 1723, 1786, 1977, 2040, 3615, 3678, 3739, + 3802, 2294, 2357, 3428, 3491, 1023, 1086, 1596, 1659, 2104, 2167, 2484, 2547, + 3303, 3366, 3553, 3616, 3801, 3864, 3984, 4047, 2926, 2989, 2863, 2926, 2989, + 3052, 2800, 2863, 3052, 3115, 1278, 1341, 3924, 3987, 1469, 1532, 2231, 2294, + 2737, 2800, 3115, 3178, 3491, 3554, 3863, 3926, 2421, 2484, 3366, 3429, 1087, + 1150, 3985, 4048, 1914, 1977, 2674, 2737, 3178, 3241, 3678, 3741, 1787, 1850, + 3740, 3803, 2041, 2104, 3616, 3679, 1660, 1723, 3802, 3865, 2611, 2674, 3241, + 3304, 1342, 1405, 2358, 2421, 3429, 3492, 3925, 3988, 2168, 2231, 3554, 3617, + 1151, 1214, 3986, 4049, 1533, 1596, 3864, 3927, 2548, 2611, 3304, 3367, 2295, + 2358, 3492, 3555, 1851, 1914, 3741, 3804, 1978, 2041, 2927, 2990, 2990, 3053, + 3679, 3742, 1406, 1469, 3926, 3989, 1724, 1787, 2864, 2927, 3053, 3116, 3803, + 3866, 1215, 1278, 2485, 2548, 3367, 3430, 3987, 4050, 2801, 2864, 3116, 3179, + 2105, 2168, 3617, 3680, 1597, 1660, 3865, 3928, 2738, 2801, 3179, 3242, 2422, + 2485, 3430, 3493, 2232, 2295, 3555, 3618, 2675, 2738, 3242, 3305, 1279, 1342, + 3988, 4051, 1470, 1533, 3927, 3990, 1915, 1978, 3742, 3805, 1788, 1851, 3804, + 3867, 2612, 2675, 3305, 3368, 2042, 2105, 3680, 3743, 2359, 2422, 3493, 3556, + 1661, 1724, 3866, 3929, 2169, 2232, 3618, 3681, 2549, 2612, 3368, 3431, 1343, + 1406, 3989, 4052, 2991, 3054, 1534, 1597, 2928, 2991, 3054, 3117, 3928, 3991, + 2865, 2928, 3117, 3180, 2296, 2359, 3556, 3619, 2802, 2865, 3180, 3243, 2486, + 2549, 3431, 3494, 1852, 1915, 3805, 3868, 1979, 2042, 3743, 3806, 1725, 1788, + 2739, 2802, 3243, 3306, 3867, 3930, 1407, 1470, 2106, 2169, 3681, 3744, 3990, + 4053, 2676, 2739, 3306, 3369, 1598, 1661, 2423, 2486, 3494, 3557, 3929, 3992, + 2233, 2296, 3619, 3682, 2613, 2676, 3369, 3432, 1471, 1534, 3991, 4054, 1916, + 1979, 3806, 3869, 1789, 1852, 2043, 2106, 2360, 2423, 3557, 3620, 3744, 3807, + 3868, 3931, 2992, 3055, 3055, 3118, 2550, 2613, 3432, 3495, 2929, 2992, 3118, + 3181, 1662, 1725, 2170, 2233, 3682, 3745, 3930, 3993, 2866, 2929, 3181, 3244, + 2803, 2866, 3244, 3307, 1535, 1598, 2297, 2360, 3620, 3683, 3992, 4055, 2487, + 2550, 3495, 3558, 2740, 2803, 3307, 3370, 1980, 2043, 3807, 3870, 1853, 1916, + 3869, 3932, 2107, 2170, 3745, 3808, 1726, 1789, 2677, 2740, 3370, 3433, 3931, + 3994, 2424, 2487, 3558, 3621, 2234, 2297, 3683, 3746, 1599, 1662, 3993, 4056, + 2614, 2677, 3433, 3496, 3056, 3119, 2993, 3056, 3119, 3182, 2930, 2993, 3182, + 3245, 2361, 2424, 3621, 3684, 1917, 1980, 3870, 3933, 2044, 2107, 3808, 3871, + 2551, 2614, 3496, 3559, 2867, 2930, 3245, 3308, 1790, 1853, 3932, 3995, 2171, + 2234, 3746, 3809, 2804, 2867, 3308, 3371, 1663, 1726, 3994, 4057, 2488, 2551, + 3559, 3622, 2741, 2804, 3371, 3434, 2298, 2361, 3684, 3747, 2678, 2741, 3434, + 3497, 1981, 2044, 3871, 3934, 1854, 1917, 3933, 3996, 2108, 2171, 3809, 3872, + 2425, 2488, 3622, 3685, 1727, 1790, 3995, 4058, 3057, 3120, 3120, 3183, 2235, + 2298, 2615, 2678, 3497, 3560, 3747, 3810, 2994, 3057, 3183, 3246, 2931, 2994, + 3246, 3309, 2868, 2931, 3309, 3372, 2362, 2425, 3685, 3748, 2552, 2615, 3560, + 3623, 1918, 1981, 3934, 3997, 2045, 2108, 2805, 2868, 3372, 3435, 3872, 3935, + 1791, 1854, 3996, 4059, 2172, 2235, 3810, 3873, 2742, 2805, 3435, 3498, 2489, + 2552, 3623, 3686, 2299, 2362, 3748, 3811, 2679, 2742, 3498, 3561, 3121, 3184, + 3058, 3121, 3184, 3247, 1982, 2045, 3935, 3998, 2426, 2489, 3686, 3749, 1855, + 1918, 2109, 2172, 2995, 3058, 3247, 3310, 3873, 3936, 3997, 4060, 2616, 2679, + 3561, 3624, 2932, 2995, 3310, 3373, 2236, 2299, 3811, 3874, 2869, 2932, 3373, + 3436, 2553, 2616, 3624, 3687, 2363, 2426, 3749, 3812, 2806, 2869, 3436, 3499, + 2046, 2109, 3936, 3999, 1919, 1982, 3998, 4061, 2743, 2806, 3499, 3562, 2173, + 2236, 3874, 3937, 2490, 2553, 3687, 3750, 2300, 2363, 3812, 3875, 2680, 2743, + 3562, 3625, 3122, 3185, 3185, 3248, 3059, 3122, 3248, 3311, 2996, 3059, 3311, + 3374, 2427, 2490, 2933, 2996, 3374, 3437, 3750, 3813, 1983, 2046, 2617, 2680, + 3625, 3688, 3999, 4062, 2110, 2173, 3937, 4000, 2870, 2933, 3437, 3500, 2237, + 2300, 3875, 3938, 2807, 2870, 3500, 3563, 2554, 2617, 3688, 3751, 2364, 2427, + 3813, 3876, 2744, 2807, 3563, 3626, 2047, 2110, 4000, 4063, 2174, 2237, 3186, + 3249, 3938, 4001, 2491, 2554, 3123, 3186, 3249, 3312, 3751, 3814, 3060, 3123, + 3312, 3375, 2681, 2744, 3626, 3689, 2301, 2364, 3876, 3939, 2997, 3060, 3375, + 3438, 2934, 2997, 3438, 3501, 2428, 2491, 3814, 3877, 2618, 2681, 3689, 3752, + 2871, 2934, 3501, 3564, 2111, 2174, 4001, 4064, 2238, 2301, 3939, 4002, 2808, + 2871, 3564, 3627, 2555, 2618, 3752, 3815, 2365, 2428, 3877, 3940, 2745, 2808, + 3627, 3690, 3187, 3250, 3250, 3313, 3124, 3187, 3313, 3376, 3061, 3124, 3376, + 3439, 2492, 2555, 3815, 3878, 2175, 2238, 2998, 3061, 3439, 3502, 4002, 4065, + 2682, 2745, 3690, 3753, 2302, 2365, 3940, 4003, 2935, 2998, 3502, 3565, 2872, + 2935, 3565, 3628, 2619, 2682, 3753, 3816, 2429, 2492, 3878, 3941, 2809, 2872, + 3628, 3691, 2239, 2302, 4003, 4066, 2556, 2619, 3816, 3879, 3251, 3314, 3188, + 3251, 3314, 3377, 3125, 3188, 3377, 3440, 2366, 2429, 2746, 2809, 3691, 3754, + 3941, 4004, 3062, 3125, 3440, 3503, 2999, 3062, 3503, 3566, 2493, 2556, 3879, + 3942, 2683, 2746, 3754, 3817, 2936, 2999, 3566, 3629, 2303, 2366, 4004, 4067, + 2873, 2936, 3629, 3692, 2620, 2683, 3817, 3880, 2430, 2493, 3942, 4005, 2810, + 2873, 3692, 3755, 3252, 3315, 3315, 3378, 3189, 3252, 3378, 3441, 3126, 3189, + 3441, 3504, 2557, 2620, 3880, 3943, 3063, 3126, 3504, 3567, 2747, 2810, 3755, + 3818, 2367, 2430, 4005, 4068, 3000, 3063, 3567, 3630, 2684, 2747, 3818, 3881, + 2494, 2557, 2937, 3000, 3630, 3693, 3943, 4006, 2874, 2937, 3693, 3756, 2621, + 2684, 3881, 3944, 3316, 3379, 3253, 3316, 3379, 3442, 2431, 2494, 4006, 4069, + 3190, 3253, 3442, 3505, 2811, 2874, 3756, 3819, 3127, 3190, 3505, 3568, 3064, + 3127, 3568, 3631, 2558, 2621, 3944, 4007, 2748, 2811, 3819, 3882, 3001, 3064, + 3631, 3694, 2938, 3001, 3694, 3757, 2685, 2748, 3882, 3945, 2495, 2558, 4007, + 4070, 2875, 2938, 3757, 3820, 3317, 3380, 3380, 3443, 3254, 3317, 3443, 3506, + 2622, 2685, 3191, 3254, 3506, 3569, 3945, 4008, 2812, 2875, 3820, 3883, 3128, + 3191, 3569, 3632, 3065, 3128, 3632, 3695, 2559, 2622, 4008, 4071, 2749, 2812, + 3883, 3946, 3002, 3065, 3695, 3758, 2939, 3002, 3758, 3821, 2686, 2749, 3946, + 4009, 3381, 3444, 3318, 3381, 3444, 3507, 2876, 2939, 3821, 3884, 3255, 3318, + 3507, 3570, 3192, 3255, 3570, 3633, 2623, 2686, 3129, 3192, 3633, 3696, 4009, + 4072, 2813, 2876, 3884, 3947, 3066, 3129, 3696, 3759, 3003, 3066, 3759, 3822, + 2750, 2813, 3947, 4010, 2940, 3003, 3822, 3885, 3382, 3445, 3445, 3508, 3319, + 3382, 3508, 3571, 2687, 2750, 4010, 4073, 3256, 3319, 3571, 3634, 2877, 2940, + 3885, 3948, 3193, 3256, 3634, 3697, 3130, 3193, 3697, 3760, 2814, 2877, 3948, + 4011, 3067, 3130, 3760, 3823, 3004, 3067, 3823, 3886, 2751, 2814, 4011, 4074, + 3446, 3509, 3383, 3446, 3509, 3572, 2941, 3004, 3886, 3949, 3320, 3383, 3572, + 3635, 3257, 3320, 3635, 3698, 3194, 3257, 3698, 3761, 2878, 2941, 3949, 4012, + 3131, 3194, 3761, 3824, 3068, 3131, 3824, 3887, 2815, 2878, 4012, 4075, 3005, + 3068, 3887, 3950, 3447, 3510, 3510, 3573, 3384, 3447, 3573, 3636, 3321, 3384, + 3636, 3699, 2942, 3005, 3950, 4013, 3258, 3321, 3699, 3762, 3195, 3258, 3762, + 3825, 2879, 2942, 4013, 4076, 3132, 3195, 3825, 3888, 3069, 3132, 3888, 3951, + 3511, 3574, 3448, 3511, 3574, 3637, 3006, 3069, 3951, 4014, 3385, 3448, 3637, + 3700, 3322, 3385, 3700, 3763, 3259, 3322, 3763, 3826, 2943, 3006, 4014, 4077, + 3196, 3259, 3826, 3889, 3133, 3196, 3889, 3952, 3070, 3133, 3952, 4015, 3512, + 3575, 3575, 3638, 3449, 3512, 3638, 3701, 3386, 3449, 3701, 3764, 3007, 3070, + 4015, 4078, 3323, 3386, 3764, 3827, 3260, 3323, 3827, 3890, 3197, 3260, 3890, + 3953, 3134, 3197, 3953, 4016, 3576, 3639, 3071, 3134, 4016, 4079, 3513, 3576, + 3639, 3702, 3450, 3513, 3702, 3765, 3387, 3450, 3765, 3828, 3324, 3387, 3828, + 3891, 3261, 3324, 3891, 3954, 3198, 3261, 3954, 4017, 3135, 3198, 4017, 4080, + 3577, 3640, 3640, 3703, 3514, 3577, 3703, 3766, 3451, 3514, 3766, 3829, 3388, + 3451, 3829, 3892, 3325, 3388, 3892, 3955, 3262, 3325, 3955, 4018, 3199, 3262, + 4018, 4081, 3641, 3704, 3578, 3641, 3704, 3767, 3515, 3578, 3767, 3830, 3452, + 3515, 3830, 3893, 3389, 3452, 3893, 3956, 3326, 3389, 3956, 4019, 3263, 3326, + 4019, 4082, 3642, 3705, 3705, 3768, 3579, 3642, 3768, 3831, 3516, 3579, 3831, + 3894, 3453, 3516, 3894, 3957, 3390, 3453, 3957, 4020, 3327, 3390, 4020, 4083, + 3706, 3769, 3643, 3706, 3769, 3832, 3580, 3643, 3832, 3895, 3517, 3580, 3895, + 3958, 3454, 3517, 3958, 4021, 3391, 3454, 4021, 4084, 3707, 3770, 3770, 3833, + 3644, 3707, 3833, 3896, 3581, 3644, 3896, 3959, 3518, 3581, 3959, 4022, 3455, + 3518, 4022, 4085, 3771, 3834, 3708, 3771, 3834, 3897, 3645, 3708, 3897, 3960, + 3582, 3645, 3960, 4023, 3519, 3582, 4023, 4086, 3772, 3835, 3835, 3898, 3709, + 3772, 3898, 3961, 3646, 3709, 3961, 4024, 3583, 3646, 4024, 4087, 3836, 3899, + 3773, 3836, 3899, 3962, 3710, 3773, 3962, 4025, 3647, 3710, 4025, 4088, 3837, + 3900, 3900, 3963, 3774, 3837, 3963, 4026, 3711, 3774, 4026, 4089, 3901, 3964, + 3838, 3901, 3964, 4027, 3775, 3838, 4027, 4090, 3902, 3965, 3965, 4028, 3839, + 3902, 4028, 4091, 3966, 4029, 3903, 3966, 4029, 4092, 3967, 4030, 4030, 4093, + 4031, 4094, 0, 0, +}; +#endif // CONFIG_TX64X64 + +#if CONFIG_CB4X4 +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_2x2[4]) = { 0, 1, 2, + 3 }; +#endif + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_4x4[16]) = { + 0, 2, 5, 8, 1, 3, 9, 12, 4, 7, 11, 14, 6, 10, 13, 15, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_4x4[16]) = { + 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_4x4[16]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, av1_col_iscan_4x4[16]) = { + 0, 3, 7, 11, 1, 5, 9, 12, 2, 6, 10, 14, 4, 8, 13, 15, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_row_iscan_4x4[16]) = { + 0, 1, 3, 5, 2, 4, 6, 9, 7, 8, 11, 13, 10, 12, 14, 15, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_4x8[32]) = { + 0, 1, 4, 9, 2, 3, 6, 11, 5, 7, 8, 13, 10, 12, 14, 17, + 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_4x8[32]) = { + 0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27, + 4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_4x8[32]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_8x4[32]) = { + 0, 1, 4, 9, 15, 19, 24, 28, 2, 3, 6, 11, 16, 21, 25, 29, + 5, 7, 8, 13, 18, 22, 26, 30, 10, 12, 14, 17, 20, 23, 27, 31, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_8x4[32]) = { + 0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29, + 2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_8x4[32]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_4x16[64]) = { + 0, 1, 3, 6, 2, 4, 7, 10, 5, 8, 11, 14, 9, 12, 15, 18, + 13, 16, 19, 22, 17, 20, 23, 26, 21, 24, 27, 30, 25, 28, 31, 34, + 29, 32, 35, 38, 33, 36, 39, 42, 37, 40, 43, 46, 41, 44, 47, 50, + 45, 48, 51, 54, 49, 52, 55, 58, 53, 56, 59, 61, 57, 60, 62, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_16x4[64]) = { + 0, 1, 3, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, + 2, 4, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 58, + 5, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 59, 61, + 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 60, 62, 63, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_4x16[64]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_16x4[64]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_4x16[64]) = { + 0, 16, 32, 48, 1, 17, 33, 49, 2, 18, 34, 50, 3, 19, 35, 51, + 4, 20, 36, 52, 5, 21, 37, 53, 6, 22, 38, 54, 7, 23, 39, 55, + 8, 24, 40, 56, 9, 25, 41, 57, 10, 26, 42, 58, 11, 27, 43, 59, + 12, 28, 44, 60, 13, 29, 45, 61, 14, 30, 46, 62, 15, 31, 47, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_16x4[64]) = { + 0, 16, 32, 48, 1, 17, 33, 49, 2, 18, 34, 50, 3, 19, 35, 51, + 4, 20, 36, 52, 5, 21, 37, 53, 6, 22, 38, 54, 7, 23, 39, 55, + 8, 24, 40, 56, 9, 25, 41, 57, 10, 26, 42, 58, 11, 27, 43, 59, + 12, 28, 44, 60, 13, 29, 45, 61, 14, 30, 46, 62, 15, 31, 47, 63, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_8x32[256]) = { + 0, 1, 3, 6, 10, 15, 21, 28, 2, 4, 7, 11, 16, 22, 29, + 36, 5, 8, 12, 17, 23, 30, 37, 44, 9, 13, 18, 24, 31, 38, + 45, 52, 14, 19, 25, 32, 39, 46, 53, 60, 20, 26, 33, 40, 47, + 54, 61, 68, 27, 34, 41, 48, 55, 62, 69, 76, 35, 42, 49, 56, + 63, 70, 77, 84, 43, 50, 57, 64, 71, 78, 85, 92, 51, 58, 65, + 72, 79, 86, 93, 100, 59, 66, 73, 80, 87, 94, 101, 108, 67, 74, + 81, 88, 95, 102, 109, 116, 75, 82, 89, 96, 103, 110, 117, 124, 83, + 90, 97, 104, 111, 118, 125, 132, 91, 98, 105, 112, 119, 126, 133, 140, + 99, 106, 113, 120, 127, 134, 141, 148, 107, 114, 121, 128, 135, 142, 149, + 156, 115, 122, 129, 136, 143, 150, 157, 164, 123, 130, 137, 144, 151, 158, + 165, 172, 131, 138, 145, 152, 159, 166, 173, 180, 139, 146, 153, 160, 167, + 174, 181, 188, 147, 154, 161, 168, 175, 182, 189, 196, 155, 162, 169, 176, + 183, 190, 197, 204, 163, 170, 177, 184, 191, 198, 205, 212, 171, 178, 185, + 192, 199, 206, 213, 220, 179, 186, 193, 200, 207, 214, 221, 228, 187, 194, + 201, 208, 215, 222, 229, 235, 195, 202, 209, 216, 223, 230, 236, 241, 203, + 210, 217, 224, 231, 237, 242, 246, 211, 218, 225, 232, 238, 243, 247, 250, + 219, 226, 233, 239, 244, 248, 251, 253, 227, 234, 240, 245, 249, 252, 254, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_32x8[256]) = { + 0, 1, 3, 6, 10, 15, 21, 28, 36, 44, 52, 60, 68, 76, 84, + 92, 100, 108, 116, 124, 132, 140, 148, 156, 164, 172, 180, 188, 196, 204, + 212, 220, 2, 4, 7, 11, 16, 22, 29, 37, 45, 53, 61, 69, 77, + 85, 93, 101, 109, 117, 125, 133, 141, 149, 157, 165, 173, 181, 189, 197, + 205, 213, 221, 228, 5, 8, 12, 17, 23, 30, 38, 46, 54, 62, 70, + 78, 86, 94, 102, 110, 118, 126, 134, 142, 150, 158, 166, 174, 182, 190, + 198, 206, 214, 222, 229, 235, 9, 13, 18, 24, 31, 39, 47, 55, 63, + 71, 79, 87, 95, 103, 111, 119, 127, 135, 143, 151, 159, 167, 175, 183, + 191, 199, 207, 215, 223, 230, 236, 241, 14, 19, 25, 32, 40, 48, 56, + 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, + 184, 192, 200, 208, 216, 224, 231, 237, 242, 246, 20, 26, 33, 41, 49, + 57, 65, 73, 81, 89, 97, 105, 113, 121, 129, 137, 145, 153, 161, 169, + 177, 185, 193, 201, 209, 217, 225, 232, 238, 243, 247, 250, 27, 34, 42, + 50, 58, 66, 74, 82, 90, 98, 106, 114, 122, 130, 138, 146, 154, 162, + 170, 178, 186, 194, 202, 210, 218, 226, 233, 239, 244, 248, 251, 253, 35, + 43, 51, 59, 67, 75, 83, 91, 99, 107, 115, 123, 131, 139, 147, 155, + 163, 171, 179, 187, 195, 203, 211, 219, 227, 234, 240, 245, 249, 252, 254, + 255, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_8x32[256]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_32x8[256]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_8x32[256]) = { + 0, 32, 64, 96, 128, 160, 192, 224, 1, 33, 65, 97, 129, 161, 193, 225, + 2, 34, 66, 98, 130, 162, 194, 226, 3, 35, 67, 99, 131, 163, 195, 227, + 4, 36, 68, 100, 132, 164, 196, 228, 5, 37, 69, 101, 133, 165, 197, 229, + 6, 38, 70, 102, 134, 166, 198, 230, 7, 39, 71, 103, 135, 167, 199, 231, + 8, 40, 72, 104, 136, 168, 200, 232, 9, 41, 73, 105, 137, 169, 201, 233, + 10, 42, 74, 106, 138, 170, 202, 234, 11, 43, 75, 107, 139, 171, 203, 235, + 12, 44, 76, 108, 140, 172, 204, 236, 13, 45, 77, 109, 141, 173, 205, 237, + 14, 46, 78, 110, 142, 174, 206, 238, 15, 47, 79, 111, 143, 175, 207, 239, + 16, 48, 80, 112, 144, 176, 208, 240, 17, 49, 81, 113, 145, 177, 209, 241, + 18, 50, 82, 114, 146, 178, 210, 242, 19, 51, 83, 115, 147, 179, 211, 243, + 20, 52, 84, 116, 148, 180, 212, 244, 21, 53, 85, 117, 149, 181, 213, 245, + 22, 54, 86, 118, 150, 182, 214, 246, 23, 55, 87, 119, 151, 183, 215, 247, + 24, 56, 88, 120, 152, 184, 216, 248, 25, 57, 89, 121, 153, 185, 217, 249, + 26, 58, 90, 122, 154, 186, 218, 250, 27, 59, 91, 123, 155, 187, 219, 251, + 28, 60, 92, 124, 156, 188, 220, 252, 29, 61, 93, 125, 157, 189, 221, 253, + 30, 62, 94, 126, 158, 190, 222, 254, 31, 63, 95, 127, 159, 191, 223, 255, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_32x8[256]) = { + 0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, + 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, + 240, 248, 1, 9, 17, 25, 33, 41, 49, 57, 65, 73, 81, 89, 97, + 105, 113, 121, 129, 137, 145, 153, 161, 169, 177, 185, 193, 201, 209, 217, + 225, 233, 241, 249, 2, 10, 18, 26, 34, 42, 50, 58, 66, 74, 82, + 90, 98, 106, 114, 122, 130, 138, 146, 154, 162, 170, 178, 186, 194, 202, + 210, 218, 226, 234, 242, 250, 3, 11, 19, 27, 35, 43, 51, 59, 67, + 75, 83, 91, 99, 107, 115, 123, 131, 139, 147, 155, 163, 171, 179, 187, + 195, 203, 211, 219, 227, 235, 243, 251, 4, 12, 20, 28, 36, 44, 52, + 60, 68, 76, 84, 92, 100, 108, 116, 124, 132, 140, 148, 156, 164, 172, + 180, 188, 196, 204, 212, 220, 228, 236, 244, 252, 5, 13, 21, 29, 37, + 45, 53, 61, 69, 77, 85, 93, 101, 109, 117, 125, 133, 141, 149, 157, + 165, 173, 181, 189, 197, 205, 213, 221, 229, 237, 245, 253, 6, 14, 22, + 30, 38, 46, 54, 62, 70, 78, 86, 94, 102, 110, 118, 126, 134, 142, + 150, 158, 166, 174, 182, 190, 198, 206, 214, 222, 230, 238, 246, 254, 7, + 15, 23, 31, 39, 47, 55, 63, 71, 79, 87, 95, 103, 111, 119, 127, + 135, 143, 151, 159, 167, 175, 183, 191, 199, 207, 215, 223, 231, 239, 247, + 255, +}; +#endif // CONFIG_EXT_TX + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_8x8[64]) = { + 0, 8, 16, 24, 32, 40, 48, 56, 1, 9, 17, 25, 33, 41, 49, 57, + 2, 10, 18, 26, 34, 42, 50, 58, 3, 11, 19, 27, 35, 43, 51, 59, + 4, 12, 20, 28, 36, 44, 52, 60, 5, 13, 21, 29, 37, 45, 53, 61, + 6, 14, 22, 30, 38, 46, 54, 62, 7, 15, 23, 31, 39, 47, 55, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_8x8[64]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, av1_col_iscan_8x8[64]) = { + 0, 3, 8, 15, 22, 32, 40, 47, 1, 5, 11, 18, 26, 34, 44, 51, + 2, 7, 13, 20, 28, 38, 46, 54, 4, 10, 16, 24, 31, 41, 50, 56, + 6, 12, 21, 27, 35, 43, 52, 58, 9, 17, 25, 33, 39, 48, 55, 60, + 14, 23, 30, 37, 45, 53, 59, 62, 19, 29, 36, 42, 49, 57, 61, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_row_iscan_8x8[64]) = { + 0, 1, 2, 5, 8, 12, 19, 24, 3, 4, 7, 10, 15, 20, 30, 39, + 6, 9, 13, 16, 21, 27, 37, 46, 11, 14, 17, 23, 28, 34, 44, 52, + 18, 22, 25, 31, 35, 41, 50, 57, 26, 29, 33, 38, 43, 49, 55, 59, + 32, 36, 42, 47, 51, 54, 60, 61, 40, 45, 48, 53, 56, 58, 62, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_8x8[64]) = { + 0, 2, 5, 9, 14, 22, 31, 37, 1, 4, 8, 13, 19, 26, 38, 44, + 3, 6, 10, 17, 24, 30, 42, 49, 7, 11, 15, 21, 29, 36, 47, 53, + 12, 16, 20, 27, 34, 43, 52, 57, 18, 23, 28, 35, 41, 48, 56, 60, + 25, 32, 39, 45, 50, 55, 59, 62, 33, 40, 46, 51, 54, 58, 61, 63, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_8x16[128]) = { + 0, 1, 3, 6, 10, 15, 21, 28, 2, 4, 7, 11, 16, 22, 29, 36, + 5, 8, 12, 17, 23, 30, 37, 44, 9, 13, 18, 24, 31, 38, 45, 52, + 14, 19, 25, 32, 39, 46, 53, 60, 20, 26, 33, 40, 47, 54, 61, 68, + 27, 34, 41, 48, 55, 62, 69, 76, 35, 42, 49, 56, 63, 70, 77, 84, + 43, 50, 57, 64, 71, 78, 85, 92, 51, 58, 65, 72, 79, 86, 93, 100, + 59, 66, 73, 80, 87, 94, 101, 107, 67, 74, 81, 88, 95, 102, 108, 113, + 75, 82, 89, 96, 103, 109, 114, 118, 83, 90, 97, 104, 110, 115, 119, 122, + 91, 98, 105, 111, 116, 120, 123, 125, 99, 106, 112, 117, 121, 124, 126, 127, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_16x8[128]) = { + 0, 1, 3, 6, 10, 15, 21, 28, 36, 44, 52, 60, 68, 76, 84, 92, + 2, 4, 7, 11, 16, 22, 29, 37, 45, 53, 61, 69, 77, 85, 93, 100, + 5, 8, 12, 17, 23, 30, 38, 46, 54, 62, 70, 78, 86, 94, 101, 107, + 9, 13, 18, 24, 31, 39, 47, 55, 63, 71, 79, 87, 95, 102, 108, 113, + 14, 19, 25, 32, 40, 48, 56, 64, 72, 80, 88, 96, 103, 109, 114, 118, + 20, 26, 33, 41, 49, 57, 65, 73, 81, 89, 97, 104, 110, 115, 119, 122, + 27, 34, 42, 50, 58, 66, 74, 82, 90, 98, 105, 111, 116, 120, 123, 125, + 35, 43, 51, 59, 67, 75, 83, 91, 99, 106, 112, 117, 121, 124, 126, 127, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_8x16[128]) = { + 0, 16, 32, 48, 64, 80, 96, 112, 1, 17, 33, 49, 65, 81, 97, 113, + 2, 18, 34, 50, 66, 82, 98, 114, 3, 19, 35, 51, 67, 83, 99, 115, + 4, 20, 36, 52, 68, 84, 100, 116, 5, 21, 37, 53, 69, 85, 101, 117, + 6, 22, 38, 54, 70, 86, 102, 118, 7, 23, 39, 55, 71, 87, 103, 119, + 8, 24, 40, 56, 72, 88, 104, 120, 9, 25, 41, 57, 73, 89, 105, 121, + 10, 26, 42, 58, 74, 90, 106, 122, 11, 27, 43, 59, 75, 91, 107, 123, + 12, 28, 44, 60, 76, 92, 108, 124, 13, 29, 45, 61, 77, 93, 109, 125, + 14, 30, 46, 62, 78, 94, 110, 126, 15, 31, 47, 63, 79, 95, 111, 127, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_16x8[128]) = { + 0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, + 1, 9, 17, 25, 33, 41, 49, 57, 65, 73, 81, 89, 97, 105, 113, 121, + 2, 10, 18, 26, 34, 42, 50, 58, 66, 74, 82, 90, 98, 106, 114, 122, + 3, 11, 19, 27, 35, 43, 51, 59, 67, 75, 83, 91, 99, 107, 115, 123, + 4, 12, 20, 28, 36, 44, 52, 60, 68, 76, 84, 92, 100, 108, 116, 124, + 5, 13, 21, 29, 37, 45, 53, 61, 69, 77, 85, 93, 101, 109, 117, 125, + 6, 14, 22, 30, 38, 46, 54, 62, 70, 78, 86, 94, 102, 110, 118, 126, + 7, 15, 23, 31, 39, 47, 55, 63, 71, 79, 87, 95, 103, 111, 119, 127, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_8x16[128]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_16x8[128]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_16x32[512]) = { + 0, 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 66, 78, 91, 105, + 120, 2, 4, 7, 11, 16, 22, 29, 37, 46, 56, 67, 79, 92, 106, + 121, 136, 5, 8, 12, 17, 23, 30, 38, 47, 57, 68, 80, 93, 107, + 122, 137, 152, 9, 13, 18, 24, 31, 39, 48, 58, 69, 81, 94, 108, + 123, 138, 153, 168, 14, 19, 25, 32, 40, 49, 59, 70, 82, 95, 109, + 124, 139, 154, 169, 184, 20, 26, 33, 41, 50, 60, 71, 83, 96, 110, + 125, 140, 155, 170, 185, 200, 27, 34, 42, 51, 61, 72, 84, 97, 111, + 126, 141, 156, 171, 186, 201, 216, 35, 43, 52, 62, 73, 85, 98, 112, + 127, 142, 157, 172, 187, 202, 217, 232, 44, 53, 63, 74, 86, 99, 113, + 128, 143, 158, 173, 188, 203, 218, 233, 248, 54, 64, 75, 87, 100, 114, + 129, 144, 159, 174, 189, 204, 219, 234, 249, 264, 65, 76, 88, 101, 115, + 130, 145, 160, 175, 190, 205, 220, 235, 250, 265, 280, 77, 89, 102, 116, + 131, 146, 161, 176, 191, 206, 221, 236, 251, 266, 281, 296, 90, 103, 117, + 132, 147, 162, 177, 192, 207, 222, 237, 252, 267, 282, 297, 312, 104, 118, + 133, 148, 163, 178, 193, 208, 223, 238, 253, 268, 283, 298, 313, 328, 119, + 134, 149, 164, 179, 194, 209, 224, 239, 254, 269, 284, 299, 314, 329, 344, + 135, 150, 165, 180, 195, 210, 225, 240, 255, 270, 285, 300, 315, 330, 345, + 360, 151, 166, 181, 196, 211, 226, 241, 256, 271, 286, 301, 316, 331, 346, + 361, 376, 167, 182, 197, 212, 227, 242, 257, 272, 287, 302, 317, 332, 347, + 362, 377, 392, 183, 198, 213, 228, 243, 258, 273, 288, 303, 318, 333, 348, + 363, 378, 393, 407, 199, 214, 229, 244, 259, 274, 289, 304, 319, 334, 349, + 364, 379, 394, 408, 421, 215, 230, 245, 260, 275, 290, 305, 320, 335, 350, + 365, 380, 395, 409, 422, 434, 231, 246, 261, 276, 291, 306, 321, 336, 351, + 366, 381, 396, 410, 423, 435, 446, 247, 262, 277, 292, 307, 322, 337, 352, + 367, 382, 397, 411, 424, 436, 447, 457, 263, 278, 293, 308, 323, 338, 353, + 368, 383, 398, 412, 425, 437, 448, 458, 467, 279, 294, 309, 324, 339, 354, + 369, 384, 399, 413, 426, 438, 449, 459, 468, 476, 295, 310, 325, 340, 355, + 370, 385, 400, 414, 427, 439, 450, 460, 469, 477, 484, 311, 326, 341, 356, + 371, 386, 401, 415, 428, 440, 451, 461, 470, 478, 485, 491, 327, 342, 357, + 372, 387, 402, 416, 429, 441, 452, 462, 471, 479, 486, 492, 497, 343, 358, + 373, 388, 403, 417, 430, 442, 453, 463, 472, 480, 487, 493, 498, 502, 359, + 374, 389, 404, 418, 431, 443, 454, 464, 473, 481, 488, 494, 499, 503, 506, + 375, 390, 405, 419, 432, 444, 455, 465, 474, 482, 489, 495, 500, 504, 507, + 509, 391, 406, 420, 433, 445, 456, 466, 475, 483, 490, 496, 501, 505, 508, + 510, 511, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_32x16[512]) = { + 0, 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 66, 78, 91, 105, + 120, 136, 152, 168, 184, 200, 216, 232, 248, 264, 280, 296, 312, 328, 344, + 360, 376, 2, 4, 7, 11, 16, 22, 29, 37, 46, 56, 67, 79, 92, + 106, 121, 137, 153, 169, 185, 201, 217, 233, 249, 265, 281, 297, 313, 329, + 345, 361, 377, 392, 5, 8, 12, 17, 23, 30, 38, 47, 57, 68, 80, + 93, 107, 122, 138, 154, 170, 186, 202, 218, 234, 250, 266, 282, 298, 314, + 330, 346, 362, 378, 393, 407, 9, 13, 18, 24, 31, 39, 48, 58, 69, + 81, 94, 108, 123, 139, 155, 171, 187, 203, 219, 235, 251, 267, 283, 299, + 315, 331, 347, 363, 379, 394, 408, 421, 14, 19, 25, 32, 40, 49, 59, + 70, 82, 95, 109, 124, 140, 156, 172, 188, 204, 220, 236, 252, 268, 284, + 300, 316, 332, 348, 364, 380, 395, 409, 422, 434, 20, 26, 33, 41, 50, + 60, 71, 83, 96, 110, 125, 141, 157, 173, 189, 205, 221, 237, 253, 269, + 285, 301, 317, 333, 349, 365, 381, 396, 410, 423, 435, 446, 27, 34, 42, + 51, 61, 72, 84, 97, 111, 126, 142, 158, 174, 190, 206, 222, 238, 254, + 270, 286, 302, 318, 334, 350, 366, 382, 397, 411, 424, 436, 447, 457, 35, + 43, 52, 62, 73, 85, 98, 112, 127, 143, 159, 175, 191, 207, 223, 239, + 255, 271, 287, 303, 319, 335, 351, 367, 383, 398, 412, 425, 437, 448, 458, + 467, 44, 53, 63, 74, 86, 99, 113, 128, 144, 160, 176, 192, 208, 224, + 240, 256, 272, 288, 304, 320, 336, 352, 368, 384, 399, 413, 426, 438, 449, + 459, 468, 476, 54, 64, 75, 87, 100, 114, 129, 145, 161, 177, 193, 209, + 225, 241, 257, 273, 289, 305, 321, 337, 353, 369, 385, 400, 414, 427, 439, + 450, 460, 469, 477, 484, 65, 76, 88, 101, 115, 130, 146, 162, 178, 194, + 210, 226, 242, 258, 274, 290, 306, 322, 338, 354, 370, 386, 401, 415, 428, + 440, 451, 461, 470, 478, 485, 491, 77, 89, 102, 116, 131, 147, 163, 179, + 195, 211, 227, 243, 259, 275, 291, 307, 323, 339, 355, 371, 387, 402, 416, + 429, 441, 452, 462, 471, 479, 486, 492, 497, 90, 103, 117, 132, 148, 164, + 180, 196, 212, 228, 244, 260, 276, 292, 308, 324, 340, 356, 372, 388, 403, + 417, 430, 442, 453, 463, 472, 480, 487, 493, 498, 502, 104, 118, 133, 149, + 165, 181, 197, 213, 229, 245, 261, 277, 293, 309, 325, 341, 357, 373, 389, + 404, 418, 431, 443, 454, 464, 473, 481, 488, 494, 499, 503, 506, 119, 134, + 150, 166, 182, 198, 214, 230, 246, 262, 278, 294, 310, 326, 342, 358, 374, + 390, 405, 419, 432, 444, 455, 465, 474, 482, 489, 495, 500, 504, 507, 509, + 135, 151, 167, 183, 199, 215, 231, 247, 263, 279, 295, 311, 327, 343, 359, + 375, 391, 406, 420, 433, 445, 456, 466, 475, 483, 490, 496, 501, 505, 508, + 510, 511, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_16x32[512]) = { + 0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448, 480, + 1, 33, 65, 97, 129, 161, 193, 225, 257, 289, 321, 353, 385, 417, 449, 481, + 2, 34, 66, 98, 130, 162, 194, 226, 258, 290, 322, 354, 386, 418, 450, 482, + 3, 35, 67, 99, 131, 163, 195, 227, 259, 291, 323, 355, 387, 419, 451, 483, + 4, 36, 68, 100, 132, 164, 196, 228, 260, 292, 324, 356, 388, 420, 452, 484, + 5, 37, 69, 101, 133, 165, 197, 229, 261, 293, 325, 357, 389, 421, 453, 485, + 6, 38, 70, 102, 134, 166, 198, 230, 262, 294, 326, 358, 390, 422, 454, 486, + 7, 39, 71, 103, 135, 167, 199, 231, 263, 295, 327, 359, 391, 423, 455, 487, + 8, 40, 72, 104, 136, 168, 200, 232, 264, 296, 328, 360, 392, 424, 456, 488, + 9, 41, 73, 105, 137, 169, 201, 233, 265, 297, 329, 361, 393, 425, 457, 489, + 10, 42, 74, 106, 138, 170, 202, 234, 266, 298, 330, 362, 394, 426, 458, 490, + 11, 43, 75, 107, 139, 171, 203, 235, 267, 299, 331, 363, 395, 427, 459, 491, + 12, 44, 76, 108, 140, 172, 204, 236, 268, 300, 332, 364, 396, 428, 460, 492, + 13, 45, 77, 109, 141, 173, 205, 237, 269, 301, 333, 365, 397, 429, 461, 493, + 14, 46, 78, 110, 142, 174, 206, 238, 270, 302, 334, 366, 398, 430, 462, 494, + 15, 47, 79, 111, 143, 175, 207, 239, 271, 303, 335, 367, 399, 431, 463, 495, + 16, 48, 80, 112, 144, 176, 208, 240, 272, 304, 336, 368, 400, 432, 464, 496, + 17, 49, 81, 113, 145, 177, 209, 241, 273, 305, 337, 369, 401, 433, 465, 497, + 18, 50, 82, 114, 146, 178, 210, 242, 274, 306, 338, 370, 402, 434, 466, 498, + 19, 51, 83, 115, 147, 179, 211, 243, 275, 307, 339, 371, 403, 435, 467, 499, + 20, 52, 84, 116, 148, 180, 212, 244, 276, 308, 340, 372, 404, 436, 468, 500, + 21, 53, 85, 117, 149, 181, 213, 245, 277, 309, 341, 373, 405, 437, 469, 501, + 22, 54, 86, 118, 150, 182, 214, 246, 278, 310, 342, 374, 406, 438, 470, 502, + 23, 55, 87, 119, 151, 183, 215, 247, 279, 311, 343, 375, 407, 439, 471, 503, + 24, 56, 88, 120, 152, 184, 216, 248, 280, 312, 344, 376, 408, 440, 472, 504, + 25, 57, 89, 121, 153, 185, 217, 249, 281, 313, 345, 377, 409, 441, 473, 505, + 26, 58, 90, 122, 154, 186, 218, 250, 282, 314, 346, 378, 410, 442, 474, 506, + 27, 59, 91, 123, 155, 187, 219, 251, 283, 315, 347, 379, 411, 443, 475, 507, + 28, 60, 92, 124, 156, 188, 220, 252, 284, 316, 348, 380, 412, 444, 476, 508, + 29, 61, 93, 125, 157, 189, 221, 253, 285, 317, 349, 381, 413, 445, 477, 509, + 30, 62, 94, 126, 158, 190, 222, 254, 286, 318, 350, 382, 414, 446, 478, 510, + 31, 63, 95, 127, 159, 191, 223, 255, 287, 319, 351, 383, 415, 447, 479, 511, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_32x16[512]) = { + 0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, + 240, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, + 480, 496, 1, 17, 33, 49, 65, 81, 97, 113, 129, 145, 161, 177, 193, + 209, 225, 241, 257, 273, 289, 305, 321, 337, 353, 369, 385, 401, 417, 433, + 449, 465, 481, 497, 2, 18, 34, 50, 66, 82, 98, 114, 130, 146, 162, + 178, 194, 210, 226, 242, 258, 274, 290, 306, 322, 338, 354, 370, 386, 402, + 418, 434, 450, 466, 482, 498, 3, 19, 35, 51, 67, 83, 99, 115, 131, + 147, 163, 179, 195, 211, 227, 243, 259, 275, 291, 307, 323, 339, 355, 371, + 387, 403, 419, 435, 451, 467, 483, 499, 4, 20, 36, 52, 68, 84, 100, + 116, 132, 148, 164, 180, 196, 212, 228, 244, 260, 276, 292, 308, 324, 340, + 356, 372, 388, 404, 420, 436, 452, 468, 484, 500, 5, 21, 37, 53, 69, + 85, 101, 117, 133, 149, 165, 181, 197, 213, 229, 245, 261, 277, 293, 309, + 325, 341, 357, 373, 389, 405, 421, 437, 453, 469, 485, 501, 6, 22, 38, + 54, 70, 86, 102, 118, 134, 150, 166, 182, 198, 214, 230, 246, 262, 278, + 294, 310, 326, 342, 358, 374, 390, 406, 422, 438, 454, 470, 486, 502, 7, + 23, 39, 55, 71, 87, 103, 119, 135, 151, 167, 183, 199, 215, 231, 247, + 263, 279, 295, 311, 327, 343, 359, 375, 391, 407, 423, 439, 455, 471, 487, + 503, 8, 24, 40, 56, 72, 88, 104, 120, 136, 152, 168, 184, 200, 216, + 232, 248, 264, 280, 296, 312, 328, 344, 360, 376, 392, 408, 424, 440, 456, + 472, 488, 504, 9, 25, 41, 57, 73, 89, 105, 121, 137, 153, 169, 185, + 201, 217, 233, 249, 265, 281, 297, 313, 329, 345, 361, 377, 393, 409, 425, + 441, 457, 473, 489, 505, 10, 26, 42, 58, 74, 90, 106, 122, 138, 154, + 170, 186, 202, 218, 234, 250, 266, 282, 298, 314, 330, 346, 362, 378, 394, + 410, 426, 442, 458, 474, 490, 506, 11, 27, 43, 59, 75, 91, 107, 123, + 139, 155, 171, 187, 203, 219, 235, 251, 267, 283, 299, 315, 331, 347, 363, + 379, 395, 411, 427, 443, 459, 475, 491, 507, 12, 28, 44, 60, 76, 92, + 108, 124, 140, 156, 172, 188, 204, 220, 236, 252, 268, 284, 300, 316, 332, + 348, 364, 380, 396, 412, 428, 444, 460, 476, 492, 508, 13, 29, 45, 61, + 77, 93, 109, 125, 141, 157, 173, 189, 205, 221, 237, 253, 269, 285, 301, + 317, 333, 349, 365, 381, 397, 413, 429, 445, 461, 477, 493, 509, 14, 30, + 46, 62, 78, 94, 110, 126, 142, 158, 174, 190, 206, 222, 238, 254, 270, + 286, 302, 318, 334, 350, 366, 382, 398, 414, 430, 446, 462, 478, 494, 510, + 15, 31, 47, 63, 79, 95, 111, 127, 143, 159, 175, 191, 207, 223, 239, + 255, 271, 287, 303, 319, 335, 351, 367, 383, 399, 415, 431, 447, 463, 479, + 495, 511, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_16x32[512]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, + 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, + 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, + 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, + 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, + 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, + 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, + 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, + 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, + 510, 511, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_32x16[512]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, + 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, + 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, + 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, + 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, + 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, + 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, + 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, + 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, + 510, 511, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_16x16[256]) = { + 0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, + 1, 17, 33, 49, 65, 81, 97, 113, 129, 145, 161, 177, 193, 209, 225, 241, + 2, 18, 34, 50, 66, 82, 98, 114, 130, 146, 162, 178, 194, 210, 226, 242, + 3, 19, 35, 51, 67, 83, 99, 115, 131, 147, 163, 179, 195, 211, 227, 243, + 4, 20, 36, 52, 68, 84, 100, 116, 132, 148, 164, 180, 196, 212, 228, 244, + 5, 21, 37, 53, 69, 85, 101, 117, 133, 149, 165, 181, 197, 213, 229, 245, + 6, 22, 38, 54, 70, 86, 102, 118, 134, 150, 166, 182, 198, 214, 230, 246, + 7, 23, 39, 55, 71, 87, 103, 119, 135, 151, 167, 183, 199, 215, 231, 247, + 8, 24, 40, 56, 72, 88, 104, 120, 136, 152, 168, 184, 200, 216, 232, 248, + 9, 25, 41, 57, 73, 89, 105, 121, 137, 153, 169, 185, 201, 217, 233, 249, + 10, 26, 42, 58, 74, 90, 106, 122, 138, 154, 170, 186, 202, 218, 234, 250, + 11, 27, 43, 59, 75, 91, 107, 123, 139, 155, 171, 187, 203, 219, 235, 251, + 12, 28, 44, 60, 76, 92, 108, 124, 140, 156, 172, 188, 204, 220, 236, 252, + 13, 29, 45, 61, 77, 93, 109, 125, 141, 157, 173, 189, 205, 221, 237, 253, + 14, 30, 46, 62, 78, 94, 110, 126, 142, 158, 174, 190, 206, 222, 238, 254, + 15, 31, 47, 63, 79, 95, 111, 127, 143, 159, 175, 191, 207, 223, 239, 255, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_16x16[256]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, av1_col_iscan_16x16[256]) = { + 0, 4, 11, 20, 31, 43, 59, 75, 85, 109, 130, 150, 165, 181, 195, 198, + 1, 6, 14, 23, 34, 47, 64, 81, 95, 114, 135, 153, 171, 188, 201, 212, + 2, 8, 16, 25, 38, 52, 67, 83, 101, 116, 136, 157, 172, 190, 205, 216, + 3, 10, 18, 29, 41, 55, 71, 89, 103, 119, 141, 159, 176, 194, 208, 218, + 5, 12, 21, 32, 45, 58, 74, 93, 104, 123, 144, 164, 179, 196, 210, 223, + 7, 15, 26, 37, 49, 63, 78, 96, 112, 129, 146, 166, 182, 200, 215, 228, + 9, 19, 28, 39, 54, 69, 86, 102, 117, 132, 151, 170, 187, 206, 220, 230, + 13, 24, 35, 46, 60, 73, 91, 108, 122, 137, 154, 174, 189, 207, 224, 235, + 17, 30, 40, 53, 66, 82, 98, 115, 126, 142, 161, 180, 197, 213, 227, 237, + 22, 36, 48, 62, 76, 92, 105, 120, 133, 147, 167, 186, 203, 219, 232, 240, + 27, 44, 56, 70, 84, 99, 113, 127, 140, 156, 175, 193, 209, 226, 236, 244, + 33, 51, 68, 79, 94, 110, 125, 138, 149, 162, 184, 202, 217, 229, 241, 247, + 42, 61, 77, 90, 106, 121, 134, 148, 160, 173, 191, 211, 225, 238, 245, 251, + 50, 72, 87, 100, 118, 128, 145, 158, 168, 183, 204, 222, 233, 242, 249, 253, + 57, 80, 97, 111, 131, 143, 155, 169, 178, 192, 214, 231, 239, 246, 250, 254, + 65, 88, 107, 124, 139, 152, 163, 177, 185, 199, 221, 234, 243, 248, 252, 255, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_row_iscan_16x16[256]) = { + 0, 1, 2, 4, 6, 9, 12, 17, 22, 29, 36, 43, 54, 64, 76, + 86, 3, 5, 7, 11, 15, 19, 25, 32, 38, 48, 59, 68, 84, 99, + 115, 130, 8, 10, 13, 18, 23, 27, 33, 42, 51, 60, 72, 88, 103, + 119, 142, 167, 14, 16, 20, 26, 31, 37, 44, 53, 61, 73, 85, 100, + 116, 135, 161, 185, 21, 24, 30, 35, 40, 47, 55, 65, 74, 81, 94, + 112, 133, 154, 179, 205, 28, 34, 39, 45, 50, 58, 67, 77, 87, 96, + 106, 121, 146, 169, 196, 212, 41, 46, 49, 56, 63, 70, 79, 90, 98, + 107, 122, 138, 159, 182, 207, 222, 52, 57, 62, 69, 75, 83, 93, 102, + 110, 120, 134, 150, 176, 195, 215, 226, 66, 71, 78, 82, 91, 97, 108, + 113, 127, 136, 148, 168, 188, 202, 221, 232, 80, 89, 92, 101, 105, 114, + 125, 131, 139, 151, 162, 177, 192, 208, 223, 234, 95, 104, 109, 117, 123, + 128, 143, 144, 155, 165, 175, 190, 206, 219, 233, 239, 111, 118, 124, 129, + 140, 147, 157, 164, 170, 181, 191, 203, 224, 230, 240, 243, 126, 132, 137, + 145, 153, 160, 174, 178, 184, 197, 204, 216, 231, 237, 244, 246, 141, 149, + 156, 166, 172, 180, 189, 199, 200, 210, 220, 228, 238, 242, 249, 251, 152, + 163, 171, 183, 186, 193, 201, 211, 214, 218, 227, 236, 245, 247, 252, 253, + 158, 173, 187, 194, 198, 209, 213, 217, 225, 229, 235, 241, 248, 250, 254, + 255, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_16x16[256]) = { + 0, 2, 5, 9, 17, 24, 36, 44, 55, 72, 88, 104, 128, 143, 166, + 179, 1, 4, 8, 13, 20, 30, 40, 54, 66, 79, 96, 113, 141, 154, + 178, 196, 3, 7, 11, 18, 25, 33, 46, 57, 71, 86, 101, 119, 148, + 164, 186, 201, 6, 12, 16, 23, 31, 39, 53, 64, 78, 92, 110, 127, + 153, 169, 193, 208, 10, 14, 19, 28, 37, 47, 58, 67, 84, 98, 114, + 133, 161, 176, 198, 214, 15, 21, 26, 34, 43, 52, 65, 77, 91, 106, + 120, 140, 165, 185, 205, 221, 22, 27, 32, 41, 48, 60, 73, 85, 99, + 116, 130, 151, 175, 190, 211, 225, 29, 35, 42, 49, 59, 69, 81, 95, + 108, 125, 139, 155, 182, 197, 217, 229, 38, 45, 51, 61, 68, 80, 93, + 105, 118, 134, 150, 168, 191, 207, 223, 234, 50, 56, 63, 74, 83, 94, + 109, 117, 129, 147, 163, 177, 199, 213, 228, 238, 62, 70, 76, 87, 97, + 107, 122, 131, 145, 159, 172, 188, 210, 222, 235, 242, 75, 82, 90, 102, + 112, 124, 138, 146, 157, 173, 187, 202, 219, 230, 240, 245, 89, 100, 111, + 123, 132, 142, 156, 167, 180, 189, 203, 216, 231, 237, 246, 250, 103, 115, + 126, 136, 149, 162, 171, 183, 194, 204, 215, 224, 236, 241, 248, 252, 121, + 135, 144, 158, 170, 181, 192, 200, 209, 218, 227, 233, 243, 244, 251, 254, + 137, 152, 160, 174, 184, 195, 206, 212, 220, 226, 232, 239, 247, 249, 253, + 255, +}; + +#if CONFIG_EXT_TX +DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_32x32[1024]) = { + 0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, + 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800, 832, 864, + 896, 928, 960, 992, 1, 33, 65, 97, 129, 161, 193, 225, 257, 289, + 321, 353, 385, 417, 449, 481, 513, 545, 577, 609, 641, 673, 705, 737, + 769, 801, 833, 865, 897, 929, 961, 993, 2, 34, 66, 98, 130, 162, + 194, 226, 258, 290, 322, 354, 386, 418, 450, 482, 514, 546, 578, 610, + 642, 674, 706, 738, 770, 802, 834, 866, 898, 930, 962, 994, 3, 35, + 67, 99, 131, 163, 195, 227, 259, 291, 323, 355, 387, 419, 451, 483, + 515, 547, 579, 611, 643, 675, 707, 739, 771, 803, 835, 867, 899, 931, + 963, 995, 4, 36, 68, 100, 132, 164, 196, 228, 260, 292, 324, 356, + 388, 420, 452, 484, 516, 548, 580, 612, 644, 676, 708, 740, 772, 804, + 836, 868, 900, 932, 964, 996, 5, 37, 69, 101, 133, 165, 197, 229, + 261, 293, 325, 357, 389, 421, 453, 485, 517, 549, 581, 613, 645, 677, + 709, 741, 773, 805, 837, 869, 901, 933, 965, 997, 6, 38, 70, 102, + 134, 166, 198, 230, 262, 294, 326, 358, 390, 422, 454, 486, 518, 550, + 582, 614, 646, 678, 710, 742, 774, 806, 838, 870, 902, 934, 966, 998, + 7, 39, 71, 103, 135, 167, 199, 231, 263, 295, 327, 359, 391, 423, + 455, 487, 519, 551, 583, 615, 647, 679, 711, 743, 775, 807, 839, 871, + 903, 935, 967, 999, 8, 40, 72, 104, 136, 168, 200, 232, 264, 296, + 328, 360, 392, 424, 456, 488, 520, 552, 584, 616, 648, 680, 712, 744, + 776, 808, 840, 872, 904, 936, 968, 1000, 9, 41, 73, 105, 137, 169, + 201, 233, 265, 297, 329, 361, 393, 425, 457, 489, 521, 553, 585, 617, + 649, 681, 713, 745, 777, 809, 841, 873, 905, 937, 969, 1001, 10, 42, + 74, 106, 138, 170, 202, 234, 266, 298, 330, 362, 394, 426, 458, 490, + 522, 554, 586, 618, 650, 682, 714, 746, 778, 810, 842, 874, 906, 938, + 970, 1002, 11, 43, 75, 107, 139, 171, 203, 235, 267, 299, 331, 363, + 395, 427, 459, 491, 523, 555, 587, 619, 651, 683, 715, 747, 779, 811, + 843, 875, 907, 939, 971, 1003, 12, 44, 76, 108, 140, 172, 204, 236, + 268, 300, 332, 364, 396, 428, 460, 492, 524, 556, 588, 620, 652, 684, + 716, 748, 780, 812, 844, 876, 908, 940, 972, 1004, 13, 45, 77, 109, + 141, 173, 205, 237, 269, 301, 333, 365, 397, 429, 461, 493, 525, 557, + 589, 621, 653, 685, 717, 749, 781, 813, 845, 877, 909, 941, 973, 1005, + 14, 46, 78, 110, 142, 174, 206, 238, 270, 302, 334, 366, 398, 430, + 462, 494, 526, 558, 590, 622, 654, 686, 718, 750, 782, 814, 846, 878, + 910, 942, 974, 1006, 15, 47, 79, 111, 143, 175, 207, 239, 271, 303, + 335, 367, 399, 431, 463, 495, 527, 559, 591, 623, 655, 687, 719, 751, + 783, 815, 847, 879, 911, 943, 975, 1007, 16, 48, 80, 112, 144, 176, + 208, 240, 272, 304, 336, 368, 400, 432, 464, 496, 528, 560, 592, 624, + 656, 688, 720, 752, 784, 816, 848, 880, 912, 944, 976, 1008, 17, 49, + 81, 113, 145, 177, 209, 241, 273, 305, 337, 369, 401, 433, 465, 497, + 529, 561, 593, 625, 657, 689, 721, 753, 785, 817, 849, 881, 913, 945, + 977, 1009, 18, 50, 82, 114, 146, 178, 210, 242, 274, 306, 338, 370, + 402, 434, 466, 498, 530, 562, 594, 626, 658, 690, 722, 754, 786, 818, + 850, 882, 914, 946, 978, 1010, 19, 51, 83, 115, 147, 179, 211, 243, + 275, 307, 339, 371, 403, 435, 467, 499, 531, 563, 595, 627, 659, 691, + 723, 755, 787, 819, 851, 883, 915, 947, 979, 1011, 20, 52, 84, 116, + 148, 180, 212, 244, 276, 308, 340, 372, 404, 436, 468, 500, 532, 564, + 596, 628, 660, 692, 724, 756, 788, 820, 852, 884, 916, 948, 980, 1012, + 21, 53, 85, 117, 149, 181, 213, 245, 277, 309, 341, 373, 405, 437, + 469, 501, 533, 565, 597, 629, 661, 693, 725, 757, 789, 821, 853, 885, + 917, 949, 981, 1013, 22, 54, 86, 118, 150, 182, 214, 246, 278, 310, + 342, 374, 406, 438, 470, 502, 534, 566, 598, 630, 662, 694, 726, 758, + 790, 822, 854, 886, 918, 950, 982, 1014, 23, 55, 87, 119, 151, 183, + 215, 247, 279, 311, 343, 375, 407, 439, 471, 503, 535, 567, 599, 631, + 663, 695, 727, 759, 791, 823, 855, 887, 919, 951, 983, 1015, 24, 56, + 88, 120, 152, 184, 216, 248, 280, 312, 344, 376, 408, 440, 472, 504, + 536, 568, 600, 632, 664, 696, 728, 760, 792, 824, 856, 888, 920, 952, + 984, 1016, 25, 57, 89, 121, 153, 185, 217, 249, 281, 313, 345, 377, + 409, 441, 473, 505, 537, 569, 601, 633, 665, 697, 729, 761, 793, 825, + 857, 889, 921, 953, 985, 1017, 26, 58, 90, 122, 154, 186, 218, 250, + 282, 314, 346, 378, 410, 442, 474, 506, 538, 570, 602, 634, 666, 698, + 730, 762, 794, 826, 858, 890, 922, 954, 986, 1018, 27, 59, 91, 123, + 155, 187, 219, 251, 283, 315, 347, 379, 411, 443, 475, 507, 539, 571, + 603, 635, 667, 699, 731, 763, 795, 827, 859, 891, 923, 955, 987, 1019, + 28, 60, 92, 124, 156, 188, 220, 252, 284, 316, 348, 380, 412, 444, + 476, 508, 540, 572, 604, 636, 668, 700, 732, 764, 796, 828, 860, 892, + 924, 956, 988, 1020, 29, 61, 93, 125, 157, 189, 221, 253, 285, 317, + 349, 381, 413, 445, 477, 509, 541, 573, 605, 637, 669, 701, 733, 765, + 797, 829, 861, 893, 925, 957, 989, 1021, 30, 62, 94, 126, 158, 190, + 222, 254, 286, 318, 350, 382, 414, 446, 478, 510, 542, 574, 606, 638, + 670, 702, 734, 766, 798, 830, 862, 894, 926, 958, 990, 1022, 31, 63, + 95, 127, 159, 191, 223, 255, 287, 319, 351, 383, 415, 447, 479, 511, + 543, 575, 607, 639, 671, 703, 735, 767, 799, 831, 863, 895, 927, 959, + 991, 1023, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_mrow_iscan_32x32[1024]) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, + 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, + 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, + 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, + 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, + 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, + 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, + 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, + 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, + 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, + 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, + 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, + 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, + 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, + 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, + 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, + 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, + 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, + 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, + 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, + 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, + 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, + 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, + 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, + 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, + 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, + 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, + 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, + 624, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, + 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, + 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, + 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, + 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, 688, + 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, + 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, + 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, + 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, + 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, + 754, 755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, + 767, 768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, + 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, + 793, 794, 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, 805, + 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, + 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, + 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, + 845, 846, 847, 848, 849, 850, 851, 852, 853, 854, 855, 856, 857, + 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, + 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, + 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, + 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, + 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, + 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, + 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, + 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, + 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, + 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, + 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000, + 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, + 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, +}; +#endif // CONFIG_EXT_TX + +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_32x32[1024]) = { + 0, 2, 5, 10, 17, 25, 38, 47, 62, 83, 101, 121, 145, + 170, 193, 204, 210, 219, 229, 233, 245, 257, 275, 299, 342, 356, + 377, 405, 455, 471, 495, 527, 1, 4, 8, 15, 22, 30, 45, + 58, 74, 92, 112, 133, 158, 184, 203, 215, 222, 228, 234, 237, + 256, 274, 298, 317, 355, 376, 404, 426, 470, 494, 526, 551, 3, + 7, 12, 18, 28, 36, 52, 64, 82, 102, 118, 142, 164, 189, + 208, 217, 224, 231, 235, 238, 273, 297, 316, 329, 375, 403, 425, + 440, 493, 525, 550, 567, 6, 11, 16, 23, 31, 43, 60, 73, + 90, 109, 126, 150, 173, 196, 211, 220, 226, 232, 236, 239, 296, + 315, 328, 335, 402, 424, 439, 447, 524, 549, 566, 575, 9, 14, + 19, 29, 37, 50, 65, 78, 95, 116, 134, 157, 179, 201, 214, + 223, 244, 255, 272, 295, 341, 354, 374, 401, 454, 469, 492, 523, + 582, 596, 617, 645, 13, 20, 26, 35, 44, 54, 72, 85, 105, + 123, 140, 163, 182, 205, 216, 225, 254, 271, 294, 314, 353, 373, + 400, 423, 468, 491, 522, 548, 595, 616, 644, 666, 21, 27, 33, + 42, 53, 63, 80, 94, 113, 132, 151, 172, 190, 209, 218, 227, + 270, 293, 313, 327, 372, 399, 422, 438, 490, 521, 547, 565, 615, + 643, 665, 680, 24, 32, 39, 48, 57, 71, 88, 104, 120, 139, + 159, 178, 197, 212, 221, 230, 292, 312, 326, 334, 398, 421, 437, + 446, 520, 546, 564, 574, 642, 664, 679, 687, 34, 40, 46, 56, + 68, 81, 96, 111, 130, 147, 167, 186, 243, 253, 269, 291, 340, + 352, 371, 397, 453, 467, 489, 519, 581, 594, 614, 641, 693, 705, + 723, 747, 41, 49, 55, 67, 77, 91, 107, 124, 138, 161, 177, + 194, 252, 268, 290, 311, 351, 370, 396, 420, 466, 488, 518, 545, + 593, 613, 640, 663, 704, 722, 746, 765, 51, 59, 66, 76, 89, + 99, 119, 131, 149, 168, 181, 200, 267, 289, 310, 325, 369, 395, + 419, 436, 487, 517, 544, 563, 612, 639, 662, 678, 721, 745, 764, + 777, 61, 69, 75, 87, 100, 114, 129, 144, 162, 180, 191, 207, + 288, 309, 324, 333, 394, 418, 435, 445, 516, 543, 562, 573, 638, + 661, 677, 686, 744, 763, 776, 783, 70, 79, 86, 97, 108, 122, + 137, 155, 242, 251, 266, 287, 339, 350, 368, 393, 452, 465, 486, + 515, 580, 592, 611, 637, 692, 703, 720, 743, 788, 798, 813, 833, + 84, 93, 103, 110, 125, 141, 154, 171, 250, 265, 286, 308, 349, + 367, 392, 417, 464, 485, 514, 542, 591, 610, 636, 660, 702, 719, + 742, 762, 797, 812, 832, 848, 98, 106, 115, 127, 143, 156, 169, + 185, 264, 285, 307, 323, 366, 391, 416, 434, 484, 513, 541, 561, + 609, 635, 659, 676, 718, 741, 761, 775, 811, 831, 847, 858, 117, + 128, 136, 148, 160, 175, 188, 198, 284, 306, 322, 332, 390, 415, + 433, 444, 512, 540, 560, 572, 634, 658, 675, 685, 740, 760, 774, + 782, 830, 846, 857, 863, 135, 146, 152, 165, 241, 249, 263, 283, + 338, 348, 365, 389, 451, 463, 483, 511, 579, 590, 608, 633, 691, + 701, 717, 739, 787, 796, 810, 829, 867, 875, 887, 903, 153, 166, + 174, 183, 248, 262, 282, 305, 347, 364, 388, 414, 462, 482, 510, + 539, 589, 607, 632, 657, 700, 716, 738, 759, 795, 809, 828, 845, + 874, 886, 902, 915, 176, 187, 195, 202, 261, 281, 304, 321, 363, + 387, 413, 432, 481, 509, 538, 559, 606, 631, 656, 674, 715, 737, + 758, 773, 808, 827, 844, 856, 885, 901, 914, 923, 192, 199, 206, + 213, 280, 303, 320, 331, 386, 412, 431, 443, 508, 537, 558, 571, + 630, 655, 673, 684, 736, 757, 772, 781, 826, 843, 855, 862, 900, + 913, 922, 927, 240, 247, 260, 279, 337, 346, 362, 385, 450, 461, + 480, 507, 578, 588, 605, 629, 690, 699, 714, 735, 786, 794, 807, + 825, 866, 873, 884, 899, 930, 936, 945, 957, 246, 259, 278, 302, + 345, 361, 384, 411, 460, 479, 506, 536, 587, 604, 628, 654, 698, + 713, 734, 756, 793, 806, 824, 842, 872, 883, 898, 912, 935, 944, + 956, 966, 258, 277, 301, 319, 360, 383, 410, 430, 478, 505, 535, + 557, 603, 627, 653, 672, 712, 733, 755, 771, 805, 823, 841, 854, + 882, 897, 911, 921, 943, 955, 965, 972, 276, 300, 318, 330, 382, + 409, 429, 442, 504, 534, 556, 570, 626, 652, 671, 683, 732, 754, + 770, 780, 822, 840, 853, 861, 896, 910, 920, 926, 954, 964, 971, + 975, 336, 344, 359, 381, 449, 459, 477, 503, 577, 586, 602, 625, + 689, 697, 711, 731, 785, 792, 804, 821, 865, 871, 881, 895, 929, + 934, 942, 953, 977, 981, 987, 995, 343, 358, 380, 408, 458, 476, + 502, 533, 585, 601, 624, 651, 696, 710, 730, 753, 791, 803, 820, + 839, 870, 880, 894, 909, 933, 941, 952, 963, 980, 986, 994, 1001, + 357, 379, 407, 428, 475, 501, 532, 555, 600, 623, 650, 670, 709, + 729, 752, 769, 802, 819, 838, 852, 879, 893, 908, 919, 940, 951, + 962, 970, 985, 993, 1000, 1005, 378, 406, 427, 441, 500, 531, 554, + 569, 622, 649, 669, 682, 728, 751, 768, 779, 818, 837, 851, 860, + 892, 907, 918, 925, 950, 961, 969, 974, 992, 999, 1004, 1007, 448, + 457, 474, 499, 576, 584, 599, 621, 688, 695, 708, 727, 784, 790, + 801, 817, 864, 869, 878, 891, 928, 932, 939, 949, 976, 979, 984, + 991, 1008, 1010, 1013, 1017, 456, 473, 498, 530, 583, 598, 620, 648, + 694, 707, 726, 750, 789, 800, 816, 836, 868, 877, 890, 906, 931, + 938, 948, 960, 978, 983, 990, 998, 1009, 1012, 1016, 1020, 472, 497, + 529, 553, 597, 619, 647, 668, 706, 725, 749, 767, 799, 815, 835, + 850, 876, 889, 905, 917, 937, 947, 959, 968, 982, 989, 997, 1003, + 1011, 1015, 1019, 1022, 496, 528, 552, 568, 618, 646, 667, 681, 724, + 748, 766, 778, 814, 834, 849, 859, 888, 904, 916, 924, 946, 958, + 967, 973, 988, 996, 1002, 1006, 1014, 1018, 1021, 1023, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_v2_iscan_32x32[1024]) = { + 0, 1, 4, 9, 15, 22, 33, 43, 56, 71, 86, 104, 121, + 142, 166, 189, 512, 518, 527, 539, 551, 566, 584, 602, 621, 644, + 668, 695, 721, 748, 780, 811, 2, 3, 6, 11, 17, 26, 35, + 45, 58, 73, 90, 106, 123, 146, 168, 193, 513, 519, 528, 540, + 553, 567, 585, 603, 622, 647, 670, 696, 722, 751, 783, 812, 5, + 7, 8, 13, 20, 28, 37, 50, 62, 75, 92, 108, 129, 150, + 170, 195, 514, 521, 530, 541, 554, 569, 587, 605, 625, 649, 671, + 699, 725, 752, 785, 815, 10, 12, 14, 19, 23, 31, 41, 52, + 65, 81, 96, 113, 133, 152, 175, 201, 515, 522, 531, 542, 556, + 572, 589, 607, 629, 651, 673, 700, 726, 757, 788, 819, 16, 18, + 21, 24, 30, 39, 48, 59, 69, 83, 100, 119, 137, 158, 181, + 203, 516, 523, 534, 545, 559, 574, 591, 610, 632, 654, 679, 704, + 730, 762, 791, 824, 25, 27, 29, 32, 40, 46, 54, 67, 79, + 94, 109, 127, 143, 164, 185, 210, 517, 525, 535, 547, 561, 578, + 595, 615, 635, 656, 684, 707, 737, 766, 793, 830, 34, 36, 38, + 42, 49, 55, 64, 76, 87, 102, 117, 135, 154, 176, 197, 219, + 520, 529, 538, 550, 565, 580, 598, 618, 639, 664, 687, 712, 741, + 769, 802, 833, 44, 47, 51, 53, 60, 68, 77, 85, 98, 114, + 131, 147, 162, 183, 208, 227, 524, 533, 544, 557, 571, 588, 606, + 623, 645, 667, 692, 720, 747, 776, 806, 838, 57, 61, 63, 66, + 70, 80, 88, 99, 112, 124, 140, 159, 179, 199, 216, 233, 526, + 536, 548, 562, 577, 593, 613, 633, 653, 676, 701, 727, 756, 786, + 814, 847, 72, 74, 78, 82, 84, 95, 103, 115, 125, 139, 156, + 173, 190, 211, 229, 246, 532, 543, 555, 568, 581, 601, 619, 637, + 663, 685, 709, 738, 763, 792, 826, 855, 89, 91, 93, 97, 101, + 110, 118, 132, 141, 157, 171, 186, 206, 224, 241, 255, 537, 549, + 560, 576, 592, 608, 628, 650, 669, 693, 719, 744, 773, 805, 834, + 862, 105, 107, 111, 116, 120, 128, 136, 148, 160, 174, 187, 205, + 221, 236, 251, 267, 546, 558, 570, 583, 600, 617, 636, 657, 680, + 706, 729, 758, 787, 813, 846, 871, 122, 126, 130, 134, 138, 144, + 155, 163, 180, 191, 207, 222, 232, 248, 264, 278, 552, 564, 579, + 594, 609, 630, 648, 666, 688, 715, 742, 768, 797, 827, 856, 877, + 145, 149, 151, 153, 161, 165, 177, 184, 200, 212, 225, 237, 249, + 262, 275, 289, 563, 575, 590, 604, 620, 638, 660, 683, 705, 728, + 753, 779, 809, 839, 866, 889, 167, 169, 172, 178, 182, 188, 198, + 209, 217, 230, 242, 252, 265, 276, 288, 301, 573, 586, 599, 616, + 634, 652, 672, 694, 716, 743, 767, 794, 825, 850, 874, 899, 192, + 194, 196, 202, 204, 213, 220, 228, 234, 247, 256, 268, 279, 290, + 302, 315, 582, 597, 614, 631, 646, 665, 686, 708, 732, 759, 784, + 810, 837, 863, 886, 908, 214, 215, 218, 223, 226, 231, 239, 244, + 253, 261, 271, 283, 292, 304, 317, 325, 596, 611, 626, 642, 661, + 681, 702, 723, 745, 770, 800, 828, 853, 875, 897, 919, 235, 238, + 240, 243, 245, 250, 257, 263, 270, 280, 287, 298, 307, 319, 329, + 340, 612, 624, 640, 658, 677, 697, 717, 739, 764, 789, 816, 844, + 867, 890, 909, 927, 254, 258, 259, 260, 266, 269, 272, 282, 286, + 296, 303, 312, 323, 333, 341, 355, 627, 641, 655, 674, 690, 713, + 735, 760, 781, 807, 835, 857, 880, 902, 921, 940, 273, 274, 277, + 281, 284, 285, 291, 299, 305, 310, 320, 327, 337, 346, 357, 369, + 643, 659, 675, 689, 710, 733, 754, 777, 803, 831, 851, 872, 892, + 913, 934, 950, 293, 294, 295, 297, 300, 306, 308, 314, 321, 326, + 335, 343, 352, 361, 372, 378, 662, 678, 691, 711, 731, 749, 774, + 798, 822, 848, 869, 887, 906, 925, 942, 961, 309, 311, 313, 316, + 318, 322, 324, 332, 338, 344, 351, 358, 367, 375, 386, 394, 682, + 698, 714, 734, 750, 772, 795, 820, 842, 864, 884, 904, 923, 938, + 954, 967, 328, 330, 331, 334, 336, 339, 342, 348, 354, 359, 366, + 374, 382, 391, 400, 409, 703, 718, 736, 755, 775, 796, 818, 840, + 860, 882, 900, 917, 936, 952, 965, 977, 345, 347, 349, 350, 353, + 356, 360, 364, 371, 376, 383, 389, 395, 406, 412, 423, 724, 740, + 761, 778, 799, 821, 841, 859, 878, 895, 915, 932, 948, 963, 975, + 986, 362, 363, 365, 368, 370, 373, 377, 379, 387, 392, 397, 405, + 411, 420, 428, 439, 746, 765, 782, 804, 823, 843, 861, 879, 894, + 911, 930, 946, 959, 973, 984, 994, 380, 381, 384, 385, 388, 390, + 393, 396, 403, 408, 413, 422, 427, 436, 444, 452, 771, 790, 808, + 832, 849, 865, 883, 896, 912, 928, 944, 957, 971, 982, 992, 1001, + 398, 399, 401, 402, 404, 407, 410, 414, 419, 425, 429, 437, 442, + 449, 458, 465, 801, 817, 836, 852, 870, 885, 901, 916, 931, 945, + 956, 969, 980, 990, 999, 1007, 415, 416, 417, 418, 421, 424, 426, + 430, 434, 441, 445, 453, 459, 463, 473, 480, 829, 845, 858, 873, + 888, 905, 918, 933, 947, 958, 970, 979, 988, 997, 1005, 1012, 431, + 432, 433, 435, 438, 440, 443, 446, 451, 456, 461, 468, 475, 479, + 488, 494, 854, 868, 881, 893, 907, 924, 937, 949, 960, 972, 981, + 989, 996, 1003, 1010, 1016, 447, 448, 450, 454, 455, 457, 460, 462, + 469, 472, 477, 482, 490, 495, 499, 503, 876, 891, 903, 914, 926, + 939, 953, 964, 974, 983, 991, 998, 1004, 1009, 1014, 1019, 464, 466, + 467, 470, 471, 474, 476, 478, 484, 489, 493, 497, 501, 504, 506, + 508, 898, 910, 922, 935, 943, 955, 966, 976, 985, 993, 1000, 1006, + 1011, 1015, 1018, 1021, 481, 483, 485, 486, 487, 491, 492, 496, 498, + 500, 502, 505, 507, 509, 510, 511, 920, 929, 941, 951, 962, 968, + 978, 987, 995, 1002, 1008, 1013, 1017, 1020, 1022, 1023, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_h2_iscan_32x32[1024]) = { + 0, 1, 4, 9, 15, 22, 33, 43, 56, 71, 86, 104, 121, + 142, 166, 189, 214, 233, 254, 273, 292, 309, 328, 345, 362, 378, + 397, 415, 431, 447, 464, 481, 2, 3, 6, 11, 17, 26, 35, + 45, 58, 73, 90, 106, 123, 146, 168, 193, 215, 236, 255, 274, + 294, 310, 329, 346, 363, 381, 399, 416, 432, 448, 465, 482, 5, + 7, 8, 13, 20, 28, 37, 50, 62, 75, 92, 108, 129, 150, + 170, 195, 216, 240, 259, 275, 295, 312, 331, 348, 365, 383, 400, + 417, 433, 449, 467, 485, 10, 12, 14, 19, 23, 31, 41, 52, + 65, 81, 96, 113, 133, 152, 175, 201, 221, 243, 260, 280, 297, + 315, 333, 350, 367, 385, 402, 418, 434, 452, 470, 486, 16, 18, + 21, 24, 30, 39, 48, 59, 69, 83, 100, 119, 137, 158, 181, + 203, 226, 244, 264, 283, 300, 318, 335, 353, 370, 388, 404, 420, + 438, 455, 471, 487, 25, 27, 29, 32, 40, 46, 54, 67, 79, + 94, 109, 127, 143, 164, 185, 210, 231, 250, 269, 285, 304, 322, + 339, 356, 373, 389, 407, 423, 440, 457, 473, 491, 34, 36, 38, + 42, 49, 55, 64, 76, 87, 102, 117, 135, 154, 176, 197, 219, + 239, 256, 272, 291, 308, 324, 341, 359, 377, 393, 410, 426, 442, + 460, 476, 492, 44, 47, 51, 53, 60, 68, 77, 85, 98, 114, + 131, 147, 162, 183, 208, 227, 245, 262, 282, 298, 314, 332, 349, + 364, 379, 396, 412, 430, 446, 462, 478, 495, 57, 61, 63, 66, + 70, 80, 88, 99, 112, 124, 140, 159, 179, 199, 217, 234, 253, + 270, 286, 305, 321, 337, 354, 371, 387, 403, 419, 435, 451, 468, + 484, 498, 72, 74, 78, 82, 84, 95, 103, 115, 125, 139, 156, + 173, 190, 211, 229, 246, 261, 281, 296, 311, 325, 344, 360, 375, + 392, 408, 425, 441, 456, 472, 489, 500, 89, 91, 93, 97, 101, + 110, 118, 132, 141, 157, 171, 186, 206, 224, 241, 257, 271, 287, + 303, 320, 336, 351, 366, 384, 398, 413, 429, 445, 461, 477, 493, + 502, 105, 107, 111, 116, 120, 128, 136, 148, 160, 174, 187, 205, + 222, 237, 251, 267, 284, 299, 313, 327, 343, 358, 374, 390, 405, + 422, 437, 453, 469, 483, 497, 505, 122, 126, 130, 134, 138, 144, + 155, 163, 180, 191, 207, 223, 232, 248, 265, 278, 293, 307, 323, + 338, 352, 368, 382, 395, 411, 427, 443, 459, 475, 490, 501, 507, + 145, 149, 151, 153, 161, 165, 177, 184, 200, 212, 225, 238, 249, + 263, 276, 289, 306, 319, 334, 347, 361, 376, 391, 406, 421, 436, + 450, 463, 479, 496, 504, 509, 167, 169, 172, 178, 182, 188, 198, + 209, 218, 230, 242, 252, 266, 277, 288, 301, 317, 330, 342, 357, + 372, 386, 401, 414, 428, 444, 458, 474, 488, 499, 506, 510, 192, + 194, 196, 202, 204, 213, 220, 228, 235, 247, 258, 268, 279, 290, + 302, 316, 326, 340, 355, 369, 380, 394, 409, 424, 439, 454, 466, + 480, 494, 503, 508, 511, 512, 513, 514, 515, 516, 517, 520, 523, + 526, 532, 537, 545, 551, 561, 573, 581, 596, 610, 625, 642, 661, + 680, 701, 722, 745, 770, 800, 827, 853, 875, 897, 919, 518, 519, + 521, 522, 524, 525, 528, 533, 536, 542, 549, 557, 564, 575, 585, + 597, 611, 623, 640, 656, 676, 696, 717, 739, 763, 789, 815, 844, + 867, 889, 909, 927, 527, 529, 530, 531, 534, 535, 538, 544, 548, + 555, 560, 569, 579, 589, 598, 614, 626, 641, 655, 673, 690, 712, + 735, 760, 780, 806, 834, 857, 880, 902, 921, 940, 539, 540, 541, + 543, 546, 547, 550, 558, 562, 567, 576, 583, 593, 603, 616, 631, + 643, 657, 674, 689, 710, 733, 752, 776, 803, 830, 850, 872, 892, + 913, 934, 950, 552, 553, 554, 556, 559, 563, 565, 571, 577, 582, + 591, 600, 609, 620, 634, 644, 662, 677, 691, 711, 730, 748, 773, + 798, 822, 847, 869, 887, 906, 925, 942, 961, 566, 568, 570, 572, + 574, 578, 580, 588, 594, 601, 608, 617, 629, 637, 652, 665, 681, + 697, 713, 734, 749, 772, 793, 819, 842, 863, 884, 904, 923, 938, + 954, 967, 584, 586, 587, 590, 592, 595, 599, 605, 613, 618, 628, + 636, 648, 660, 671, 686, 702, 718, 736, 753, 774, 794, 818, 840, + 860, 882, 900, 917, 936, 952, 965, 977, 602, 604, 606, 607, 612, + 615, 619, 624, 633, 638, 649, 658, 666, 683, 692, 707, 723, 740, + 761, 777, 799, 820, 841, 859, 877, 895, 915, 932, 948, 963, 975, + 986, 621, 622, 627, 630, 632, 635, 639, 645, 653, 663, 668, 682, + 688, 704, 716, 732, 746, 764, 781, 804, 823, 843, 861, 878, 894, + 911, 930, 946, 959, 973, 984, 994, 646, 647, 650, 651, 654, 659, + 664, 667, 678, 685, 693, 706, 715, 728, 743, 757, 771, 790, 807, + 831, 848, 864, 883, 896, 912, 928, 944, 957, 971, 982, 992, 1001, + 669, 670, 672, 675, 679, 684, 687, 694, 703, 709, 719, 729, 741, + 754, 767, 783, 801, 816, 835, 851, 870, 885, 901, 916, 931, 945, + 956, 969, 980, 990, 999, 1007, 695, 698, 699, 700, 705, 708, 714, + 720, 726, 738, 744, 758, 768, 779, 795, 810, 828, 845, 858, 873, + 888, 905, 918, 933, 947, 958, 970, 979, 988, 997, 1005, 1012, 721, + 724, 725, 727, 731, 737, 742, 747, 756, 765, 775, 786, 797, 809, + 825, 837, 854, 868, 881, 893, 907, 924, 937, 949, 960, 972, 981, + 989, 996, 1003, 1010, 1016, 750, 751, 755, 759, 762, 766, 769, 778, + 787, 792, 805, 812, 829, 838, 852, 865, 876, 890, 903, 914, 926, + 939, 953, 964, 974, 983, 991, 998, 1004, 1009, 1014, 1019, 782, 784, + 785, 788, 791, 796, 802, 808, 814, 826, 836, 846, 856, 866, 874, + 886, 898, 910, 922, 935, 943, 955, 966, 976, 985, 993, 1000, 1006, + 1011, 1015, 1018, 1021, 811, 813, 817, 821, 824, 832, 833, 839, 849, + 855, 862, 871, 879, 891, 899, 908, 920, 929, 941, 951, 962, 968, + 978, 987, 995, 1002, 1008, 1013, 1017, 1020, 1022, 1023, +}; + +DECLARE_ALIGNED(16, static const int16_t, av1_qtr_iscan_32x32[1024]) = { + 0, 1, 4, 9, 15, 22, 33, 43, 56, 71, 86, 104, 121, + 142, 166, 189, 256, 268, 286, 310, 334, 364, 400, 435, 471, 510, + 553, 598, 640, 683, 732, 780, 2, 3, 6, 11, 17, 26, 35, + 45, 58, 73, 90, 106, 123, 146, 168, 193, 258, 270, 288, 312, + 338, 366, 402, 437, 473, 516, 557, 600, 642, 687, 736, 782, 5, + 7, 8, 13, 20, 28, 37, 50, 62, 75, 92, 108, 129, 150, + 170, 195, 260, 274, 292, 314, 340, 370, 406, 441, 478, 520, 559, + 604, 646, 689, 740, 788, 10, 12, 14, 19, 23, 31, 41, 52, + 65, 81, 96, 113, 133, 152, 175, 201, 262, 276, 294, 316, 344, + 376, 410, 445, 484, 524, 563, 606, 648, 697, 746, 793, 16, 18, + 21, 24, 30, 39, 48, 59, 69, 83, 100, 119, 137, 158, 181, + 203, 264, 278, 300, 322, 350, 380, 414, 451, 490, 530, 571, 612, + 656, 705, 750, 799, 25, 27, 29, 32, 40, 46, 54, 67, 79, + 94, 109, 127, 143, 164, 185, 210, 266, 282, 302, 326, 354, 388, + 422, 459, 496, 533, 579, 618, 665, 711, 754, 809, 34, 36, 38, + 42, 49, 55, 64, 76, 87, 102, 117, 135, 154, 176, 197, 216, + 272, 289, 308, 332, 362, 392, 427, 465, 504, 545, 585, 626, 671, + 717, 766, 813, 44, 47, 51, 53, 60, 68, 77, 85, 98, 114, + 131, 147, 162, 183, 208, 222, 279, 298, 320, 346, 374, 408, 442, + 475, 511, 551, 592, 638, 681, 726, 772, 821, 57, 61, 63, 66, + 70, 80, 88, 99, 112, 124, 140, 159, 179, 199, 214, 227, 284, + 304, 328, 355, 386, 418, 455, 492, 528, 567, 608, 649, 695, 742, + 786, 833, 72, 74, 78, 82, 84, 95, 103, 115, 125, 139, 156, + 173, 190, 211, 224, 233, 296, 317, 342, 367, 394, 433, 466, 500, + 543, 581, 622, 667, 707, 752, 803, 843, 89, 91, 93, 97, 101, + 110, 118, 132, 141, 157, 171, 186, 206, 220, 231, 239, 306, 330, + 352, 384, 415, 447, 482, 521, 554, 593, 636, 677, 722, 770, 815, + 852, 105, 107, 111, 116, 120, 128, 136, 148, 160, 174, 187, 205, + 218, 229, 237, 244, 323, 347, 371, 398, 431, 463, 498, 534, 573, + 616, 654, 698, 743, 783, 831, 864, 122, 126, 130, 134, 138, 144, + 155, 163, 180, 191, 207, 219, 226, 235, 242, 248, 335, 360, 390, + 419, 449, 485, 518, 549, 587, 630, 672, 715, 760, 805, 845, 872, + 145, 149, 151, 153, 161, 165, 177, 184, 200, 212, 221, 230, 236, + 241, 246, 251, 356, 382, 411, 438, 469, 501, 539, 577, 613, 652, + 690, 730, 776, 822, 858, 886, 167, 169, 172, 178, 182, 188, 198, + 209, 215, 225, 232, 238, 243, 247, 250, 253, 378, 403, 428, 461, + 494, 526, 560, 594, 632, 675, 713, 755, 801, 837, 868, 897, 192, + 194, 196, 202, 204, 213, 217, 223, 228, 234, 240, 245, 249, 252, + 254, 255, 395, 425, 457, 488, 512, 547, 583, 619, 659, 699, 737, + 778, 819, 854, 882, 907, 257, 259, 261, 263, 265, 267, 273, 280, + 285, 297, 307, 324, 336, 357, 379, 396, 424, 452, 479, 508, 541, + 574, 609, 643, 679, 719, 764, 806, 841, 870, 895, 919, 269, 271, + 275, 277, 281, 283, 290, 299, 305, 318, 331, 348, 361, 383, 404, + 426, 453, 476, 506, 535, 568, 601, 634, 669, 708, 748, 789, 829, + 860, 887, 909, 927, 287, 291, 293, 295, 301, 303, 309, 321, 329, + 343, 353, 372, 391, 412, 429, 458, 480, 507, 532, 564, 590, 627, + 663, 703, 733, 773, 816, 847, 876, 901, 921, 940, 311, 313, 315, + 319, 325, 327, 333, 349, 358, 368, 385, 399, 420, 439, 462, 489, + 509, 536, 565, 589, 624, 661, 691, 727, 768, 810, 838, 866, 890, + 913, 934, 950, 337, 339, 341, 345, 351, 359, 363, 375, 387, 397, + 416, 432, 450, 470, 495, 513, 542, 569, 591, 625, 657, 684, 723, + 762, 797, 834, 862, 884, 905, 925, 942, 961, 365, 369, 373, 377, + 381, 389, 393, 409, 421, 434, 448, 464, 486, 502, 527, 548, 575, + 602, 628, 662, 685, 721, 756, 794, 827, 855, 880, 903, 923, 938, + 954, 967, 401, 405, 407, 413, 417, 423, 430, 443, 456, 467, 483, + 499, 519, 540, 561, 584, 610, 635, 664, 692, 724, 757, 792, 825, + 850, 878, 899, 917, 936, 952, 965, 977, 436, 440, 444, 446, 454, + 460, 468, 477, 493, 503, 522, 537, 550, 578, 595, 620, 644, 670, + 704, 728, 763, 795, 826, 849, 873, 893, 915, 932, 948, 963, 975, + 986, 472, 474, 481, 487, 491, 497, 505, 514, 529, 544, 555, 576, + 588, 614, 633, 660, 680, 709, 734, 769, 798, 828, 851, 874, 892, + 911, 930, 946, 959, 973, 984, 994, 515, 517, 523, 525, 531, 538, + 546, 552, 570, 582, 596, 617, 631, 653, 676, 700, 720, 749, 774, + 811, 835, 856, 879, 894, 912, 928, 944, 957, 971, 982, 992, 1001, + 556, 558, 562, 566, 572, 580, 586, 597, 611, 623, 637, 655, 673, + 693, 714, 738, 765, 790, 817, 839, 863, 881, 900, 916, 931, 945, + 956, 969, 980, 990, 999, 1007, 599, 603, 605, 607, 615, 621, 629, + 639, 650, 668, 678, 701, 716, 731, 758, 779, 807, 830, 848, 867, + 885, 904, 918, 933, 947, 958, 970, 979, 988, 997, 1005, 1012, 641, + 645, 647, 651, 658, 666, 674, 682, 696, 710, 725, 744, 761, 777, + 802, 820, 842, 861, 877, 891, 906, 924, 937, 949, 960, 972, 981, + 989, 996, 1003, 1010, 1016, 686, 688, 694, 702, 706, 712, 718, 729, + 745, 753, 771, 784, 808, 823, 840, 857, 871, 888, 902, 914, 926, + 939, 953, 964, 974, 983, 991, 998, 1004, 1009, 1014, 1019, 735, 739, + 741, 747, 751, 759, 767, 775, 787, 804, 818, 832, 846, 859, 869, + 883, 896, 910, 922, 935, 943, 955, 966, 976, 985, 993, 1000, 1006, + 1011, 1015, 1018, 1021, 781, 785, 791, 796, 800, 812, 814, 824, 836, + 844, 853, 865, 875, 889, 898, 908, 920, 929, 941, 951, 962, 968, + 978, 987, 995, 1002, 1008, 1013, 1017, 1020, 1022, 1023, +}; + +#if CONFIG_TX64X64 +DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_64x64[4096]) = { + 0, 1, 4, 9, 15, 22, 33, 43, 56, 71, 86, 104, 121, + 142, 166, 189, 214, 239, 269, 300, 331, 363, 400, 435, 471, 510, + 553, 598, 640, 683, 732, 780, 833, 884, 937, 995, 1048, 1107, 1165, + 1230, 1293, 1353, 1422, 1489, 1562, 1632, 1701, 1776, 1850, 1929, 2006, 2091, + 2173, 2252, 2339, 2421, 2516, 2603, 2694, 2786, 2879, 2978, 3076, 3175, 2, + 3, 6, 11, 17, 26, 35, 45, 58, 73, 90, 106, 123, 146, + 168, 193, 216, 243, 271, 302, 335, 365, 402, 437, 473, 516, 557, + 600, 642, 687, 736, 782, 835, 886, 941, 999, 1050, 1111, 1167, 1234, + 1297, 1357, 1424, 1491, 1564, 1636, 1703, 1778, 1852, 1931, 2012, 2095, 2177, + 2256, 2341, 2425, 2518, 2605, 2698, 2788, 2883, 2982, 3078, 3177, 5, 7, + 8, 13, 20, 28, 37, 50, 62, 75, 92, 108, 129, 150, 170, + 195, 218, 249, 277, 304, 337, 369, 406, 441, 478, 520, 559, 604, + 646, 689, 740, 788, 841, 890, 945, 1001, 1052, 1115, 1173, 1236, 1301, + 1362, 1428, 1497, 1568, 1638, 1707, 1786, 1858, 1935, 2016, 2097, 2181, 2260, + 2343, 2431, 2520, 2613, 2702, 2790, 2889, 2984, 3082, 3181, 10, 12, 14, + 19, 23, 31, 41, 52, 65, 81, 96, 113, 133, 152, 175, 201, + 224, 253, 279, 310, 341, 375, 410, 445, 484, 524, 563, 606, 648, + 697, 746, 793, 843, 896, 949, 1005, 1060, 1119, 1181, 1242, 1303, 1366, + 1436, 1503, 1572, 1640, 1713, 1790, 1865, 1943, 2018, 2103, 2183, 2266, 2347, + 2437, 2526, 2617, 2708, 2800, 2893, 2992, 3086, 3189, 16, 18, 21, 24, + 30, 39, 48, 59, 69, 83, 100, 119, 137, 158, 181, 203, 230, + 255, 286, 316, 347, 380, 414, 451, 490, 530, 571, 612, 656, 705, + 750, 799, 849, 898, 959, 1009, 1066, 1127, 1184, 1246, 1307, 1376, 1440, + 1509, 1578, 1644, 1723, 1794, 1871, 1947, 2024, 2109, 2185, 2270, 2361, 2443, + 2536, 2619, 2710, 2806, 2899, 2998, 3090, 3193, 25, 27, 29, 32, 40, + 46, 54, 67, 79, 94, 109, 127, 143, 164, 185, 210, 236, 263, + 292, 320, 353, 388, 422, 459, 496, 533, 579, 618, 665, 711, 754, + 809, 857, 910, 961, 1015, 1074, 1131, 1194, 1254, 1315, 1384, 1448, 1517, + 1584, 1655, 1731, 1802, 1875, 1959, 2034, 2115, 2197, 2280, 2367, 2452, 2538, + 2625, 2722, 2816, 2907, 3004, 3100, 3203, 34, 36, 38, 42, 49, 55, + 64, 76, 87, 102, 117, 135, 154, 176, 197, 222, 247, 272, 298, + 329, 361, 392, 427, 465, 504, 545, 585, 626, 671, 717, 766, 813, + 862, 916, 971, 1028, 1084, 1139, 1200, 1264, 1325, 1390, 1452, 1523, 1594, + 1667, 1737, 1806, 1887, 1963, 2046, 2123, 2202, 2290, 2371, 2462, 2548, 2641, + 2732, 2822, 2917, 3010, 3111, 3211, 44, 47, 51, 53, 60, 68, 77, + 85, 98, 114, 131, 147, 162, 183, 208, 232, 256, 283, 314, 343, + 373, 408, 442, 475, 511, 551, 592, 638, 681, 726, 772, 821, 874, + 926, 979, 1034, 1088, 1153, 1214, 1271, 1335, 1396, 1469, 1533, 1600, 1673, + 1745, 1824, 1897, 1973, 2054, 2131, 2216, 2300, 2383, 2468, 2558, 2649, 2740, + 2829, 2923, 3022, 3123, 3221, 57, 61, 63, 66, 70, 80, 88, 99, + 112, 124, 140, 159, 179, 199, 219, 240, 267, 294, 322, 354, 386, + 418, 455, 492, 528, 567, 608, 649, 695, 742, 786, 836, 882, 933, + 989, 1046, 1101, 1161, 1216, 1279, 1343, 1410, 1479, 1543, 1614, 1687, 1758, + 1832, 1905, 1980, 2066, 2141, 2226, 2306, 2395, 2484, 2566, 2659, 2750, 2845, + 2939, 3032, 3133, 3225, 72, 74, 78, 82, 84, 95, 103, 115, 125, + 139, 156, 173, 190, 211, 234, 259, 281, 311, 339, 366, 394, 433, + 466, 500, 543, 581, 622, 667, 707, 752, 803, 853, 899, 955, 1007, + 1064, 1117, 1175, 1237, 1299, 1354, 1420, 1485, 1556, 1624, 1697, 1770, 1842, + 1919, 1998, 2074, 2155, 2234, 2319, 2409, 2492, 2581, 2671, 2760, 2859, 2949, + 3046, 3145, 3245, 89, 91, 93, 97, 101, 110, 118, 132, 141, 157, + 171, 186, 206, 228, 251, 273, 296, 324, 351, 384, 415, 447, 482, + 521, 554, 593, 636, 677, 722, 770, 815, 866, 914, 967, 1022, 1078, + 1135, 1195, 1252, 1313, 1378, 1444, 1507, 1576, 1642, 1714, 1788, 1860, 1933, + 2013, 2085, 2169, 2250, 2337, 2417, 2502, 2597, 2683, 2778, 2869, 2960, 3060, + 3157, 3256, 105, 107, 111, 116, 120, 128, 136, 148, 160, 174, 187, + 205, 225, 244, 265, 290, 317, 344, 370, 398, 431, 463, 498, 534, + 573, 616, 654, 698, 743, 783, 831, 880, 928, 983, 1036, 1092, 1149, + 1208, 1266, 1333, 1394, 1457, 1524, 1590, 1665, 1733, 1804, 1879, 1953, 2030, + 2111, 2189, 2271, 2357, 2441, 2534, 2615, 2704, 2791, 2887, 2979, 3072, 3167, + 3270, 122, 126, 130, 134, 138, 144, 155, 163, 180, 191, 207, 226, + 238, 261, 287, 308, 332, 359, 390, 419, 449, 485, 518, 549, 587, + 630, 672, 715, 760, 805, 855, 900, 953, 1003, 1053, 1108, 1163, 1220, + 1287, 1345, 1408, 1473, 1541, 1608, 1677, 1749, 1826, 1898, 1971, 2048, 2127, + 2208, 2294, 2373, 2458, 2542, 2631, 2726, 2818, 2908, 3002, 3094, 3199, 3286, + 145, 149, 151, 153, 161, 165, 177, 184, 200, 212, 229, 245, 262, + 284, 305, 327, 355, 382, 411, 438, 469, 501, 539, 577, 613, 652, + 690, 730, 776, 822, 872, 922, 973, 1024, 1079, 1132, 1188, 1250, 1305, + 1367, 1432, 1492, 1560, 1626, 1693, 1766, 1838, 1911, 1992, 2068, 2149, 2228, + 2307, 2393, 2478, 2564, 2655, 2742, 2833, 2927, 3020, 3119, 3219, 3298, 167, + 169, 172, 178, 182, 188, 198, 209, 220, 235, 252, 266, 288, 306, + 326, 349, 378, 403, 428, 461, 494, 526, 560, 594, 632, 675, 713, + 755, 801, 845, 892, 942, 990, 1042, 1096, 1155, 1212, 1267, 1329, 1391, + 1450, 1519, 1582, 1650, 1724, 1792, 1862, 1936, 2007, 2083, 2167, 2246, 2329, + 2413, 2496, 2585, 2675, 2761, 2855, 2947, 3040, 3135, 3233, 3320, 192, 194, + 196, 202, 204, 213, 223, 233, 241, 260, 274, 291, 309, 328, 350, + 376, 395, 425, 457, 488, 512, 547, 583, 619, 659, 699, 737, 778, + 819, 868, 917, 965, 1013, 1072, 1123, 1176, 1231, 1289, 1351, 1414, 1474, + 1539, 1604, 1674, 1741, 1816, 1891, 1961, 2040, 2116, 2191, 2276, 2353, 2438, + 2524, 2606, 2689, 2784, 2871, 2968, 3062, 3161, 3257, 3334, 215, 217, 221, + 227, 231, 237, 248, 257, 268, 282, 297, 318, 333, 356, 379, 396, + 424, 452, 479, 508, 541, 574, 609, 643, 679, 719, 764, 806, 850, + 894, 938, 987, 1038, 1089, 1145, 1204, 1258, 1316, 1379, 1438, 1501, 1565, + 1628, 1694, 1764, 1836, 1907, 1981, 2060, 2137, 2220, 2298, 2377, 2464, 2549, + 2635, 2724, 2812, 2903, 2999, 3088, 3185, 3278, 3350, 242, 246, 250, 254, + 258, 264, 275, 285, 295, 312, 325, 345, 360, 383, 404, 426, 453, + 476, 506, 535, 568, 601, 634, 669, 708, 748, 789, 829, 875, 923, + 968, 1016, 1068, 1120, 1168, 1224, 1280, 1341, 1402, 1465, 1531, 1591, 1661, + 1729, 1795, 1867, 1937, 2004, 2079, 2159, 2242, 2320, 2405, 2488, 2573, 2661, + 2744, 2839, 2933, 3023, 3117, 3215, 3296, 3373, 270, 276, 278, 280, 289, + 293, 299, 315, 323, 340, 352, 371, 391, 412, 429, 458, 480, 507, + 532, 564, 590, 627, 663, 703, 733, 773, 816, 859, 906, 950, 993, + 1043, 1094, 1147, 1201, 1256, 1311, 1372, 1429, 1486, 1550, 1618, 1685, 1751, + 1827, 1895, 1965, 2042, 2119, 2192, 2268, 2348, 2429, 2512, 2599, 2684, 2772, + 2863, 2951, 3048, 3143, 3239, 3324, 3393, 301, 303, 307, 313, 319, 321, + 330, 346, 357, 367, 385, 399, 420, 439, 462, 489, 509, 536, 565, + 589, 624, 661, 691, 727, 768, 810, 846, 887, 929, 977, 1029, 1076, + 1128, 1177, 1226, 1283, 1339, 1397, 1461, 1521, 1585, 1648, 1715, 1779, 1848, + 1923, 1996, 2069, 2142, 2224, 2302, 2381, 2465, 2544, 2627, 2720, 2807, 2895, + 2985, 3073, 3163, 3264, 3338, 3413, 334, 336, 338, 342, 348, 358, 362, + 374, 387, 397, 416, 432, 450, 470, 495, 513, 542, 569, 591, 625, + 657, 684, 723, 762, 797, 837, 878, 920, 963, 1010, 1054, 1105, 1157, + 1206, 1262, 1317, 1374, 1433, 1483, 1545, 1615, 1681, 1743, 1812, 1885, 1954, + 2025, 2101, 2174, 2248, 2330, 2411, 2490, 2579, 2663, 2745, 2835, 2924, 3018, + 3115, 3205, 3290, 3363, 3431, 364, 368, 372, 377, 381, 389, 393, 409, + 421, 434, 448, 464, 486, 502, 527, 548, 575, 602, 628, 662, 685, + 721, 756, 794, 827, 869, 912, 956, 996, 1040, 1086, 1137, 1189, 1243, + 1291, 1349, 1404, 1466, 1525, 1588, 1645, 1711, 1774, 1843, 1909, 1988, 2058, + 2132, 2209, 2288, 2368, 2445, 2527, 2607, 2687, 2780, 2865, 2953, 3049, 3139, + 3237, 3318, 3387, 3451, 401, 405, 407, 413, 417, 423, 430, 443, 456, + 467, 483, 499, 519, 540, 561, 584, 610, 635, 664, 692, 724, 757, + 792, 825, 863, 908, 946, 985, 1032, 1080, 1125, 1169, 1217, 1275, 1330, + 1386, 1441, 1498, 1554, 1619, 1683, 1746, 1810, 1883, 1949, 2019, 2086, 2165, + 2238, 2314, 2399, 2479, 2562, 2645, 2733, 2820, 2904, 2996, 3083, 3168, 3268, + 3339, 3407, 3474, 436, 440, 444, 446, 454, 460, 468, 477, 493, 503, + 522, 537, 550, 578, 595, 620, 644, 670, 704, 728, 763, 795, 826, + 861, 901, 935, 980, 1025, 1069, 1112, 1159, 1209, 1260, 1309, 1363, 1418, + 1475, 1534, 1598, 1656, 1721, 1780, 1846, 1912, 1982, 2056, 2129, 2199, 2278, + 2358, 2432, 2508, 2593, 2677, 2762, 2851, 2941, 3030, 3124, 3216, 3294, 3365, + 3433, 3488, 472, 474, 481, 487, 491, 497, 505, 514, 529, 544, 555, + 576, 588, 614, 633, 660, 680, 709, 734, 769, 798, 828, 864, 902, + 932, 975, 1020, 1061, 1102, 1150, 1198, 1247, 1294, 1346, 1400, 1455, 1513, + 1573, 1629, 1689, 1755, 1820, 1888, 1955, 2022, 2092, 2163, 2235, 2312, 2389, + 2472, 2554, 2632, 2716, 2804, 2884, 2974, 3063, 3153, 3250, 3326, 3395, 3454, + 3512, 515, 517, 523, 525, 531, 538, 546, 552, 570, 582, 596, 617, + 631, 653, 676, 700, 720, 749, 774, 811, 838, 870, 909, 936, 976, + 1017, 1058, 1099, 1143, 1192, 1238, 1284, 1336, 1388, 1445, 1493, 1546, 1610, + 1671, 1734, 1796, 1856, 1925, 1994, 2062, 2133, 2206, 2281, 2354, 2426, 2503, + 2587, 2669, 2754, 2843, 2928, 3016, 3105, 3201, 3284, 3351, 3421, 3480, 3534, + 556, 558, 562, 566, 572, 580, 586, 597, 611, 623, 637, 655, 673, + 693, 714, 738, 765, 790, 817, 847, 879, 913, 947, 981, 1021, 1059, + 1097, 1140, 1185, 1227, 1277, 1327, 1380, 1425, 1481, 1537, 1595, 1651, 1708, + 1771, 1834, 1901, 1966, 2035, 2107, 2170, 2244, 2315, 2396, 2474, 2552, 2628, + 2711, 2792, 2875, 2966, 3056, 3146, 3234, 3314, 3383, 3445, 3504, 3559, 599, + 603, 605, 607, 615, 621, 629, 639, 650, 668, 678, 701, 716, 731, + 758, 779, 807, 830, 860, 888, 921, 957, 986, 1026, 1062, 1100, 1141, + 1183, 1221, 1272, 1323, 1368, 1416, 1471, 1526, 1580, 1633, 1691, 1752, 1817, + 1876, 1944, 2002, 2072, 2143, 2218, 2291, 2363, 2435, 2509, 2589, 2672, 2752, + 2840, 2921, 3008, 3095, 3190, 3274, 3344, 3409, 3470, 3526, 3577, 641, 645, + 647, 651, 658, 666, 674, 682, 696, 710, 725, 744, 761, 777, 802, + 820, 851, 876, 907, 930, 964, 997, 1033, 1070, 1103, 1144, 1186, 1222, + 1270, 1318, 1360, 1411, 1463, 1515, 1569, 1622, 1678, 1739, 1800, 1853, 1917, + 1983, 2052, 2121, 2186, 2253, 2331, 2406, 2482, 2559, 2639, 2717, 2798, 2877, + 2961, 3052, 3137, 3226, 3306, 3379, 3437, 3492, 3553, 3601, 686, 688, 694, + 702, 706, 712, 718, 729, 745, 753, 771, 784, 808, 823, 848, 871, + 895, 924, 951, 978, 1011, 1041, 1081, 1113, 1151, 1193, 1228, 1273, 1319, + 1358, 1406, 1458, 1510, 1557, 1612, 1669, 1727, 1781, 1839, 1903, 1969, 2031, + 2098, 2160, 2232, 2304, 2375, 2453, 2528, 2601, 2679, 2758, 2846, 2929, 3011, + 3098, 3186, 3271, 3340, 3401, 3466, 3522, 3571, 3620, 735, 739, 741, 747, + 751, 759, 767, 775, 787, 804, 818, 832, 856, 873, 893, 918, 939, + 969, 994, 1030, 1055, 1087, 1126, 1160, 1199, 1239, 1278, 1324, 1361, 1407, + 1453, 1505, 1551, 1605, 1663, 1716, 1768, 1830, 1893, 1951, 2008, 2075, 2139, + 2214, 2284, 2349, 2418, 2494, 2571, 2653, 2734, 2810, 2890, 2972, 3058, 3147, + 3231, 3310, 3375, 3435, 3490, 3545, 3595, 3642, 781, 785, 791, 796, 800, + 812, 814, 824, 839, 854, 867, 881, 903, 925, 943, 966, 988, 1018, + 1044, 1077, 1106, 1138, 1170, 1210, 1248, 1285, 1328, 1369, 1412, 1459, 1506, + 1549, 1601, 1657, 1704, 1762, 1821, 1880, 1938, 1999, 2063, 2125, 2193, 2257, + 2327, 2401, 2475, 2545, 2620, 2691, 2776, 2860, 2942, 3024, 3109, 3197, 3276, + 3345, 3403, 3468, 3520, 3569, 3616, 3664, 834, 840, 842, 844, 852, 858, + 865, 877, 883, 904, 915, 931, 954, 974, 991, 1014, 1039, 1071, 1095, + 1129, 1158, 1190, 1218, 1261, 1295, 1337, 1381, 1417, 1464, 1511, 1552, 1602, + 1654, 1699, 1759, 1813, 1872, 1927, 1990, 2049, 2113, 2178, 2239, 2308, 2378, + 2450, 2521, 2594, 2667, 2746, 2824, 2909, 2990, 3070, 3154, 3243, 3316, 3381, + 3441, 3493, 3547, 3597, 3640, 3682, 885, 889, 891, 897, 905, 911, 919, + 927, 934, 958, 970, 984, 1004, 1027, 1045, 1073, 1090, 1121, 1148, 1178, + 1207, 1244, 1276, 1310, 1347, 1389, 1426, 1472, 1516, 1558, 1606, 1658, 1700, + 1757, 1807, 1868, 1920, 1978, 2043, 2104, 2157, 2229, 2296, 2364, 2422, 2498, + 2574, 2650, 2727, 2801, 2872, 2954, 3038, 3129, 3212, 3288, 3352, 3419, 3475, + 3524, 3573, 3621, 3668, 3707, 940, 944, 948, 952, 960, 962, 972, 982, + 992, 1008, 1023, 1037, 1056, 1082, 1098, 1124, 1146, 1171, 1202, 1229, 1263, + 1292, 1331, 1364, 1401, 1446, 1482, 1527, 1570, 1613, 1664, 1705, 1760, 1808, + 1863, 1915, 1976, 2036, 2087, 2153, 2221, 2286, 2344, 2414, 2486, 2556, 2623, + 2699, 2773, 2853, 2937, 3012, 3091, 3169, 3260, 3330, 3391, 3447, 3505, 3555, + 3603, 3646, 3684, 3727, 998, 1000, 1002, 1006, 1012, 1019, 1031, 1035, 1047, + 1065, 1083, 1093, 1109, 1133, 1156, 1179, 1205, 1225, 1257, 1286, 1320, 1350, + 1387, 1419, 1456, 1494, 1538, 1581, 1623, 1670, 1717, 1763, 1814, 1869, 1916, + 1974, 2028, 2081, 2150, 2212, 2272, 2335, 2403, 2469, 2539, 2608, 2680, 2755, + 2827, 2915, 2986, 3068, 3151, 3229, 3300, 3366, 3427, 3484, 3532, 3581, 3630, + 3672, 3709, 3745, 1049, 1051, 1057, 1063, 1067, 1075, 1085, 1091, 1104, 1118, + 1136, 1152, 1164, 1191, 1213, 1232, 1259, 1281, 1312, 1340, 1375, 1405, 1442, + 1476, 1514, 1547, 1596, 1634, 1679, 1728, 1769, 1822, 1873, 1921, 1977, 2029, + 2078, 2144, 2203, 2264, 2325, 2390, 2459, 2529, 2591, 2665, 2738, 2813, 2880, + 2957, 3041, 3127, 3206, 3282, 3348, 3399, 3460, 3513, 3565, 3609, 3650, 3695, + 3733, 3768, 1110, 1114, 1116, 1122, 1130, 1134, 1142, 1154, 1162, 1180, 1196, + 1211, 1223, 1251, 1268, 1290, 1321, 1342, 1373, 1398, 1434, 1467, 1499, 1535, + 1574, 1611, 1652, 1692, 1740, 1782, 1831, 1881, 1928, 1979, 2037, 2082, 2145, + 2200, 2261, 2321, 2387, 2454, 2513, 2583, 2656, 2730, 2793, 2867, 2945, 3025, + 3101, 3178, 3262, 3328, 3388, 3443, 3494, 3543, 3591, 3636, 3678, 3715, 3754, + 3790, 1166, 1172, 1174, 1182, 1187, 1197, 1203, 1215, 1219, 1240, 1253, 1269, + 1288, 1306, 1332, 1352, 1382, 1403, 1430, 1462, 1484, 1528, 1555, 1599, 1630, + 1672, 1709, 1753, 1801, 1840, 1894, 1939, 1991, 2044, 2088, 2151, 2204, 2262, + 2318, 2384, 2448, 2504, 2577, 2646, 2712, 2782, 2856, 2934, 3006, 3079, 3158, + 3240, 3307, 3371, 3425, 3481, 3530, 3575, 3618, 3660, 3701, 3741, 3774, 3807, + 1233, 1235, 1241, 1245, 1249, 1255, 1265, 1274, 1282, 1300, 1314, 1334, 1348, + 1370, 1392, 1415, 1439, 1468, 1487, 1522, 1548, 1589, 1620, 1659, 1690, 1735, + 1772, 1818, 1854, 1904, 1952, 2000, 2050, 2105, 2154, 2213, 2265, 2322, 2385, + 2446, 2500, 2569, 2642, 2705, 2770, 2849, 2919, 2993, 3064, 3140, 3223, 3292, + 3353, 3414, 3464, 3516, 3561, 3607, 3648, 3687, 3725, 3762, 3796, 3827, 1296, + 1298, 1302, 1304, 1308, 1322, 1326, 1338, 1344, 1355, 1383, 1395, 1409, 1435, + 1451, 1477, 1502, 1532, 1553, 1586, 1616, 1646, 1684, 1722, 1756, 1797, 1835, + 1877, 1918, 1970, 2009, 2064, 2114, 2158, 2222, 2273, 2326, 2388, 2449, 2501, + 2567, 2636, 2695, 2768, 2836, 2910, 2976, 3053, 3131, 3209, 3279, 3336, 3397, + 3449, 3500, 3549, 3593, 3634, 3676, 3713, 3747, 3784, 3817, 3845, 1356, 1359, + 1365, 1371, 1377, 1385, 1393, 1399, 1413, 1421, 1447, 1460, 1478, 1495, 1520, + 1540, 1566, 1592, 1621, 1649, 1682, 1712, 1747, 1783, 1823, 1857, 1902, 1945, + 1984, 2032, 2076, 2126, 2179, 2230, 2287, 2336, 2391, 2455, 2505, 2570, 2637, + 2692, 2763, 2830, 2901, 2969, 3044, 3120, 3194, 3265, 3331, 3385, 3439, 3486, + 3536, 3582, 3626, 3665, 3703, 3739, 3772, 3802, 3835, 3864, 1423, 1427, 1431, + 1437, 1443, 1449, 1454, 1470, 1480, 1488, 1508, 1529, 1542, 1561, 1583, 1607, + 1631, 1662, 1686, 1718, 1744, 1775, 1811, 1847, 1889, 1926, 1967, 2003, 2053, + 2099, 2140, 2194, 2240, 2297, 2345, 2404, 2460, 2514, 2578, 2643, 2696, 2764, + 2826, 2897, 2962, 3036, 3112, 3182, 3254, 3321, 3376, 3429, 3478, 3527, 3567, + 3611, 3652, 3693, 3731, 3764, 3794, 3825, 3853, 3882, 1490, 1496, 1500, 1504, + 1512, 1518, 1530, 1536, 1544, 1559, 1577, 1593, 1609, 1627, 1653, 1675, 1695, + 1730, 1754, 1784, 1815, 1844, 1884, 1913, 1956, 1995, 2038, 2073, 2122, 2161, + 2215, 2258, 2309, 2365, 2415, 2470, 2530, 2584, 2647, 2706, 2769, 2831, 2898, + 2959, 3033, 3106, 3170, 3252, 3312, 3367, 3423, 3471, 3518, 3563, 3605, 3644, + 3680, 3717, 3755, 3788, 3819, 3847, 3874, 3898, 1563, 1567, 1571, 1575, 1579, + 1587, 1597, 1603, 1617, 1625, 1643, 1666, 1680, 1696, 1725, 1742, 1765, 1798, + 1828, 1849, 1886, 1910, 1950, 1985, 2023, 2065, 2108, 2146, 2187, 2233, 2285, + 2328, 2379, 2423, 2487, 2540, 2592, 2657, 2713, 2771, 2837, 2902, 2963, 3034, + 3104, 3164, 3248, 3304, 3361, 3417, 3462, 3510, 3557, 3598, 3638, 3674, 3711, + 3743, 3776, 3811, 3839, 3868, 3892, 3917, 1635, 1637, 1639, 1641, 1647, 1660, + 1668, 1676, 1688, 1698, 1719, 1736, 1750, 1767, 1793, 1819, 1837, 1870, 1896, + 1924, 1957, 1989, 2020, 2057, 2093, 2134, 2171, 2219, 2254, 2305, 2350, 2402, + 2451, 2499, 2557, 2609, 2666, 2731, 2783, 2850, 2911, 2970, 3037, 3107, 3165, + 3246, 3301, 3359, 3410, 3458, 3508, 3551, 3589, 3632, 3670, 3705, 3737, 3770, + 3800, 3829, 3858, 3886, 3911, 3933, 1702, 1706, 1710, 1720, 1726, 1732, 1738, + 1748, 1761, 1773, 1789, 1805, 1829, 1841, 1864, 1892, 1908, 1940, 1968, 1997, + 2026, 2059, 2089, 2130, 2164, 2207, 2245, 2292, 2332, 2376, 2419, 2476, 2522, + 2575, 2624, 2681, 2739, 2794, 2857, 2920, 2977, 3045, 3113, 3171, 3249, 3302, + 3358, 3404, 3455, 3502, 3541, 3587, 3628, 3661, 3699, 3735, 3766, 3797, 3823, + 3851, 3876, 3903, 3927, 3950, 1777, 1785, 1787, 1791, 1799, 1803, 1809, 1825, + 1833, 1845, 1861, 1882, 1899, 1914, 1941, 1962, 1986, 2005, 2045, 2070, 2102, + 2135, 2166, 2201, 2236, 2282, 2316, 2366, 2407, 2456, 2495, 2546, 2595, 2651, + 2700, 2756, 2814, 2868, 2935, 2994, 3054, 3121, 3183, 3253, 3305, 3360, 3405, + 3453, 3498, 3539, 3585, 3622, 3658, 3697, 3728, 3760, 3792, 3821, 3849, 3872, + 3896, 3919, 3942, 3964, 1851, 1855, 1859, 1866, 1874, 1878, 1890, 1900, 1906, + 1922, 1934, 1958, 1972, 1993, 2010, 2041, 2061, 2080, 2120, 2147, 2175, 2210, + 2241, 2279, 2313, 2355, 2397, 2436, 2483, 2531, 2572, 2621, 2668, 2728, 2774, + 2828, 2881, 2946, 3007, 3065, 3132, 3195, 3255, 3313, 3362, 3411, 3456, 3499, + 3538, 3579, 3614, 3656, 3691, 3723, 3758, 3786, 3815, 3843, 3870, 3894, 3915, + 3937, 3956, 3975, 1930, 1932, 1942, 1946, 1948, 1960, 1964, 1975, 1987, 2001, + 2014, 2033, 2051, 2071, 2084, 2117, 2138, 2162, 2195, 2225, 2249, 2289, 2317, + 2359, 2392, 2427, 2477, 2510, 2560, 2602, 2654, 2693, 2747, 2802, 2854, 2916, + 2958, 3026, 3080, 3141, 3210, 3266, 3322, 3368, 3418, 3459, 3503, 3540, 3580, + 3613, 3654, 3688, 3721, 3752, 3782, 3813, 3841, 3865, 3890, 3913, 3935, 3954, + 3972, 3989, 2011, 2015, 2017, 2021, 2027, 2039, 2047, 2055, 2067, 2077, 2090, + 2112, 2128, 2152, 2168, 2196, 2223, 2243, 2269, 2303, 2333, 2369, 2400, 2433, + 2473, 2506, 2553, 2590, 2640, 2682, 2735, 2777, 2825, 2873, 2938, 2987, 3042, + 3102, 3159, 3224, 3280, 3332, 3377, 3424, 3463, 3509, 3542, 3586, 3615, 3655, + 3685, 3719, 3750, 3780, 3809, 3836, 3862, 3888, 3909, 3931, 3952, 3970, 3987, + 4003, 2094, 2096, 2100, 2106, 2110, 2118, 2124, 2136, 2148, 2156, 2172, 2190, + 2211, 2231, 2247, 2277, 2299, 2323, 2351, 2382, 2412, 2447, 2480, 2511, 2555, + 2588, 2629, 2673, 2718, 2759, 2811, 2861, 2912, 2955, 3013, 3069, 3128, 3179, + 3241, 3293, 3337, 3386, 3430, 3472, 3511, 3552, 3588, 3623, 3657, 3689, 3720, + 3749, 3778, 3805, 3833, 3860, 3884, 3907, 3929, 3948, 3968, 3985, 4001, 4016, + 2176, 2180, 2182, 2184, 2188, 2198, 2205, 2217, 2227, 2237, 2251, 2274, 2295, + 2310, 2334, 2356, 2380, 2408, 2430, 2466, 2491, 2532, 2563, 2596, 2633, 2670, + 2714, 2753, 2799, 2847, 2891, 2943, 2991, 3039, 3092, 3152, 3207, 3263, 3308, + 3354, 3398, 3440, 3479, 3519, 3558, 3590, 3629, 3659, 3692, 3722, 3751, 3779, + 3804, 3831, 3856, 3880, 3905, 3925, 3946, 3966, 3983, 3999, 4014, 4028, 2255, + 2259, 2263, 2267, 2275, 2283, 2293, 2301, 2311, 2324, 2338, 2360, 2374, 2394, + 2416, 2439, 2467, 2489, 2515, 2547, 2580, 2610, 2648, 2678, 2719, 2757, 2795, + 2841, 2878, 2930, 2973, 3027, 3071, 3130, 3172, 3230, 3283, 3329, 3372, 3415, + 3450, 3487, 3528, 3564, 3599, 3633, 3662, 3698, 3724, 3753, 3781, 3806, 3832, + 3855, 3878, 3901, 3923, 3944, 3962, 3981, 3997, 4012, 4026, 4039, 2340, 2342, + 2346, 2352, 2362, 2370, 2372, 2386, 2398, 2410, 2420, 2442, 2461, 2481, 2497, + 2525, 2550, 2576, 2600, 2630, 2664, 2688, 2736, 2765, 2805, 2844, 2876, 2922, + 2964, 3014, 3059, 3110, 3155, 3213, 3261, 3303, 3349, 3389, 3426, 3465, 3501, + 3537, 3568, 3606, 3639, 3671, 3700, 3729, 3759, 3783, 3810, 3834, 3857, 3879, + 3900, 3921, 3940, 3960, 3979, 3995, 4010, 4024, 4037, 4049, 2424, 2428, 2434, + 2440, 2444, 2457, 2463, 2471, 2485, 2493, 2507, 2535, 2543, 2565, 2586, 2611, + 2638, 2662, 2685, 2721, 2748, 2781, 2821, 2852, 2885, 2931, 2967, 3009, 3055, + 3099, 3148, 3198, 3244, 3289, 3333, 3369, 3400, 3444, 3482, 3517, 3550, 3583, + 3612, 3645, 3675, 3706, 3736, 3761, 3787, 3814, 3837, 3861, 3881, 3902, 3922, + 3939, 3958, 3977, 3993, 4008, 4022, 4035, 4047, 4058, 2517, 2519, 2523, 2533, + 2537, 2541, 2551, 2561, 2568, 2582, 2598, 2616, 2634, 2658, 2676, 2690, 2725, + 2749, 2775, 2808, 2838, 2866, 2905, 2944, 2975, 3017, 3057, 3096, 3138, 3187, + 3232, 3277, 3317, 3355, 3392, 3428, 3461, 3495, 3531, 3562, 3594, 3627, 3653, + 3681, 3712, 3738, 3767, 3793, 3816, 3842, 3863, 3885, 3906, 3924, 3941, 3959, + 3974, 3991, 4006, 4020, 4033, 4045, 4056, 4066, 2604, 2612, 2614, 2618, 2622, + 2626, 2644, 2652, 2660, 2674, 2686, 2707, 2729, 2743, 2766, 2785, 2815, 2842, + 2864, 2896, 2925, 2956, 2997, 3031, 3066, 3108, 3149, 3191, 3227, 3272, 3311, + 3346, 3382, 3420, 3448, 3485, 3514, 3544, 3576, 3608, 3635, 3666, 3694, 3718, + 3744, 3771, 3798, 3822, 3844, 3866, 3889, 3908, 3926, 3945, 3961, 3978, 3992, + 4005, 4018, 4031, 4043, 4054, 4064, 4073, 2697, 2701, 2703, 2709, 2715, 2723, + 2737, 2741, 2751, 2767, 2779, 2796, 2819, 2834, 2858, 2874, 2906, 2936, 2952, + 2988, 3019, 3050, 3084, 3125, 3156, 3202, 3235, 3275, 3309, 3341, 3378, 3406, + 3442, 3476, 3506, 3533, 3566, 3592, 3619, 3649, 3677, 3704, 3732, 3756, 3777, + 3801, 3824, 3850, 3871, 3891, 3910, 3930, 3947, 3963, 3980, 3994, 4007, 4019, + 4030, 4041, 4052, 4062, 4071, 4079, 2787, 2789, 2797, 2803, 2809, 2817, 2823, + 2832, 2848, 2862, 2870, 2888, 2913, 2932, 2948, 2971, 3000, 3028, 3051, 3074, + 3116, 3142, 3173, 3217, 3251, 3285, 3315, 3347, 3380, 3402, 3436, 3469, 3496, + 3525, 3556, 3584, 3610, 3637, 3663, 3690, 3714, 3740, 3765, 3789, 3812, 3830, + 3852, 3873, 3895, 3914, 3932, 3949, 3967, 3982, 3996, 4009, 4021, 4032, 4042, + 4051, 4060, 4069, 4077, 4084, 2882, 2886, 2892, 2894, 2900, 2914, 2918, 2926, + 2940, 2950, 2965, 2980, 3003, 3021, 3043, 3067, 3089, 3118, 3144, 3166, 3208, + 3238, 3269, 3295, 3327, 3356, 3384, 3412, 3438, 3467, 3491, 3521, 3548, 3574, + 3604, 3631, 3651, 3679, 3702, 3726, 3748, 3773, 3795, 3820, 3840, 3859, 3877, + 3897, 3916, 3936, 3953, 3969, 3984, 3998, 4011, 4023, 4034, 4044, 4053, 4061, + 4068, 4075, 4082, 4088, 2981, 2983, 2989, 2995, 3001, 3005, 3015, 3029, 3035, + 3047, 3061, 3075, 3097, 3122, 3136, 3162, 3188, 3218, 3242, 3267, 3291, 3319, + 3342, 3370, 3396, 3422, 3446, 3473, 3497, 3523, 3546, 3570, 3600, 3624, 3647, + 3673, 3696, 3716, 3742, 3763, 3785, 3803, 3826, 3848, 3869, 3887, 3904, 3920, + 3938, 3955, 3971, 3986, 4000, 4013, 4025, 4036, 4046, 4055, 4063, 4070, 4076, + 4081, 4086, 4091, 3077, 3081, 3085, 3087, 3093, 3103, 3114, 3126, 3134, 3150, + 3160, 3174, 3200, 3220, 3236, 3258, 3281, 3297, 3325, 3343, 3364, 3390, 3408, + 3434, 3457, 3483, 3507, 3529, 3554, 3572, 3596, 3617, 3641, 3669, 3686, 3710, + 3734, 3757, 3775, 3799, 3818, 3838, 3854, 3875, 3893, 3912, 3928, 3943, 3957, + 3973, 3988, 4002, 4015, 4027, 4038, 4048, 4057, 4065, 4072, 4078, 4083, 4087, + 4090, 4093, 3176, 3180, 3184, 3192, 3196, 3204, 3214, 3222, 3228, 3247, 3259, + 3273, 3287, 3299, 3323, 3335, 3357, 3374, 3394, 3416, 3432, 3452, 3477, 3489, + 3515, 3535, 3560, 3578, 3602, 3625, 3643, 3667, 3683, 3708, 3730, 3746, 3769, + 3791, 3808, 3828, 3846, 3867, 3883, 3899, 3918, 3934, 3951, 3965, 3976, 3990, + 4004, 4017, 4029, 4040, 4050, 4059, 4067, 4074, 4080, 4085, 4089, 4092, 4094, + 4095, +}; +#endif // CONFIG_TX64X64 + +const SCAN_ORDER av1_default_scan_orders[TX_SIZES] = { +#if CONFIG_CB4X4 + { default_scan_2x2, av1_default_iscan_2x2, default_scan_2x2_neighbors }, +#endif + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, default_scan_16x16_neighbors }, + { default_scan_32x32, av1_default_iscan_32x32, default_scan_32x32_neighbors }, +#if CONFIG_TX64X64 + { default_scan_64x64, av1_default_iscan_64x64, default_scan_64x64_neighbors }, +#endif // CONFIG_TX64X64 +}; + +const SCAN_ORDER av1_intra_scan_orders[TX_SIZES_ALL][TX_TYPES] = { +#if CONFIG_CB4X4 + { + // TX_2X2 + { default_scan_2x2, av1_default_iscan_2x2, default_scan_2x2_neighbors }, + { default_scan_2x2, av1_default_iscan_2x2, default_scan_2x2_neighbors }, + { default_scan_2x2, av1_default_iscan_2x2, default_scan_2x2_neighbors }, + { default_scan_2x2, av1_default_iscan_2x2, default_scan_2x2_neighbors }, +#if CONFIG_EXT_TX + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors }, + { row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors }, + { col_scan_4x4, av1_col_iscan_4x4, col_scan_4x4_neighbors }, + { row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors }, + { col_scan_4x4, av1_col_iscan_4x4, col_scan_4x4_neighbors }, + { row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors }, + { col_scan_4x4, av1_col_iscan_4x4, col_scan_4x4_neighbors }, +#endif // CONFIG_EXT_TX + }, +#endif + { + // TX_4X4 + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors }, + { col_scan_4x4, av1_col_iscan_4x4, col_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, +#if CONFIG_EXT_TX + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors }, + { row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors }, + { col_scan_4x4, av1_col_iscan_4x4, col_scan_4x4_neighbors }, + { row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors }, + { col_scan_4x4, av1_col_iscan_4x4, col_scan_4x4_neighbors }, + { row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors }, + { col_scan_4x4, av1_col_iscan_4x4, col_scan_4x4_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_8X8 + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { row_scan_8x8, av1_row_iscan_8x8, row_scan_8x8_neighbors }, + { col_scan_8x8, av1_col_iscan_8x8, col_scan_8x8_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, +#if CONFIG_EXT_TX + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { mrow_scan_8x8, av1_mrow_iscan_8x8, mrow_scan_8x8_neighbors }, + { row_scan_8x8, av1_row_iscan_8x8, row_scan_8x8_neighbors }, + { col_scan_8x8, av1_col_iscan_8x8, col_scan_8x8_neighbors }, + { row_scan_8x8, av1_row_iscan_8x8, row_scan_8x8_neighbors }, + { col_scan_8x8, av1_col_iscan_8x8, col_scan_8x8_neighbors }, + { row_scan_8x8, av1_row_iscan_8x8, row_scan_8x8_neighbors }, + { col_scan_8x8, av1_col_iscan_8x8, col_scan_8x8_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_16X16 + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { row_scan_16x16, av1_row_iscan_16x16, row_scan_16x16_neighbors }, + { col_scan_16x16, av1_col_iscan_16x16, col_scan_16x16_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, +#if CONFIG_EXT_TX + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { mrow_scan_16x16, av1_mrow_iscan_16x16, mrow_scan_16x16_neighbors }, + { row_scan_16x16, av1_row_iscan_16x16, row_scan_16x16_neighbors }, + { col_scan_16x16, av1_col_iscan_16x16, col_scan_16x16_neighbors }, + { row_scan_16x16, av1_row_iscan_16x16, row_scan_16x16_neighbors }, + { col_scan_16x16, av1_col_iscan_16x16, col_scan_16x16_neighbors }, + { row_scan_16x16, av1_row_iscan_16x16, row_scan_16x16_neighbors }, + { col_scan_16x16, av1_col_iscan_16x16, col_scan_16x16_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_32X32 + { default_scan_32x32, av1_default_iscan_32x32, + default_scan_32x32_neighbors }, + { h2_scan_32x32, av1_h2_iscan_32x32, h2_scan_32x32_neighbors }, + { v2_scan_32x32, av1_v2_iscan_32x32, v2_scan_32x32_neighbors }, + { qtr_scan_32x32, av1_qtr_iscan_32x32, qtr_scan_32x32_neighbors }, +#if CONFIG_EXT_TX + { h2_scan_32x32, av1_h2_iscan_32x32, h2_scan_32x32_neighbors }, + { v2_scan_32x32, av1_v2_iscan_32x32, v2_scan_32x32_neighbors }, + { qtr_scan_32x32, av1_qtr_iscan_32x32, qtr_scan_32x32_neighbors }, + { qtr_scan_32x32, av1_qtr_iscan_32x32, qtr_scan_32x32_neighbors }, + { qtr_scan_32x32, av1_qtr_iscan_32x32, qtr_scan_32x32_neighbors }, + { mrow_scan_32x32, av1_mrow_iscan_32x32, mrow_scan_32x32_neighbors }, + { mrow_scan_32x32, av1_mrow_iscan_32x32, mrow_scan_32x32_neighbors }, + { mcol_scan_32x32, av1_mcol_iscan_32x32, mcol_scan_32x32_neighbors }, + { mrow_scan_32x32, av1_mrow_iscan_32x32, mrow_scan_32x32_neighbors }, + { mcol_scan_32x32, av1_mcol_iscan_32x32, mcol_scan_32x32_neighbors }, + { mrow_scan_32x32, av1_mrow_iscan_32x32, mrow_scan_32x32_neighbors }, + { mcol_scan_32x32, av1_mcol_iscan_32x32, mcol_scan_32x32_neighbors }, +#endif // CONFIG_EXT_TX + }, +#if CONFIG_TX64X64 + { + // TX_64X64 + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, +#if CONFIG_EXT_TX + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, +#endif // CONFIG_EXT_TX + }, +#endif // CONFIG_TX64X64 + { + // TX_4X8 + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { mrow_scan_4x8, av1_mrow_iscan_4x8, mrow_scan_4x8_neighbors }, + { mcol_scan_4x8, av1_mcol_iscan_4x8, mcol_scan_4x8_neighbors }, + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, +#if CONFIG_EXT_TX + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { mrow_scan_4x8, av1_mrow_iscan_4x8, mrow_scan_4x8_neighbors }, + { mrow_scan_4x8, av1_mrow_iscan_4x8, mrow_scan_4x8_neighbors }, + { mcol_scan_4x8, av1_mcol_iscan_4x8, mcol_scan_4x8_neighbors }, + { mrow_scan_4x8, av1_mrow_iscan_4x8, mrow_scan_4x8_neighbors }, + { mcol_scan_4x8, av1_mcol_iscan_4x8, mcol_scan_4x8_neighbors }, + { mrow_scan_4x8, av1_mrow_iscan_4x8, mrow_scan_4x8_neighbors }, + { mcol_scan_4x8, av1_mcol_iscan_4x8, mcol_scan_4x8_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_8X4 + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { mrow_scan_8x4, av1_mrow_iscan_8x4, mrow_scan_8x4_neighbors }, + { mcol_scan_8x4, av1_mcol_iscan_8x4, mcol_scan_8x4_neighbors }, + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, +#if CONFIG_EXT_TX + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { mrow_scan_8x4, av1_mrow_iscan_8x4, mrow_scan_8x4_neighbors }, + { mrow_scan_8x4, av1_mrow_iscan_8x4, mrow_scan_8x4_neighbors }, + { mcol_scan_8x4, av1_mcol_iscan_8x4, mcol_scan_8x4_neighbors }, + { mrow_scan_8x4, av1_mrow_iscan_8x4, mrow_scan_8x4_neighbors }, + { mcol_scan_8x4, av1_mcol_iscan_8x4, mcol_scan_8x4_neighbors }, + { mrow_scan_8x4, av1_mrow_iscan_8x4, mrow_scan_8x4_neighbors }, + { mcol_scan_8x4, av1_mcol_iscan_8x4, mcol_scan_8x4_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_8X16 + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { mrow_scan_8x16, av1_mrow_iscan_8x16, mrow_scan_8x16_neighbors }, + { mcol_scan_8x16, av1_mcol_iscan_8x16, mcol_scan_8x16_neighbors }, + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, +#if CONFIG_EXT_TX + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { mrow_scan_8x16, av1_mrow_iscan_8x16, mrow_scan_8x16_neighbors }, + { mrow_scan_8x16, av1_mrow_iscan_8x16, mrow_scan_8x16_neighbors }, + { mcol_scan_8x16, av1_mcol_iscan_8x16, mcol_scan_8x16_neighbors }, + { mrow_scan_8x16, av1_mrow_iscan_8x16, mrow_scan_8x16_neighbors }, + { mcol_scan_8x16, av1_mcol_iscan_8x16, mcol_scan_8x16_neighbors }, + { mrow_scan_8x16, av1_mrow_iscan_8x16, mrow_scan_8x16_neighbors }, + { mcol_scan_8x16, av1_mcol_iscan_8x16, mcol_scan_8x16_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_16X8 + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { mrow_scan_16x8, av1_mrow_iscan_16x8, mrow_scan_16x8_neighbors }, + { mcol_scan_16x8, av1_mcol_iscan_16x8, mcol_scan_16x8_neighbors }, + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, +#if CONFIG_EXT_TX + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { mrow_scan_16x8, av1_mrow_iscan_16x8, mrow_scan_16x8_neighbors }, + { mrow_scan_16x8, av1_mrow_iscan_16x8, mrow_scan_16x8_neighbors }, + { mcol_scan_16x8, av1_mcol_iscan_16x8, mcol_scan_16x8_neighbors }, + { mrow_scan_16x8, av1_mrow_iscan_16x8, mrow_scan_16x8_neighbors }, + { mcol_scan_16x8, av1_mcol_iscan_16x8, mcol_scan_16x8_neighbors }, + { mrow_scan_16x8, av1_mrow_iscan_16x8, mrow_scan_16x8_neighbors }, + { mcol_scan_16x8, av1_mcol_iscan_16x8, mcol_scan_16x8_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_16X32 + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { mrow_scan_16x32, av1_mrow_iscan_16x32, mrow_scan_16x32_neighbors }, + { mcol_scan_16x32, av1_mcol_iscan_16x32, mcol_scan_16x32_neighbors }, + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, +#if CONFIG_EXT_TX + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { mrow_scan_16x32, av1_mrow_iscan_16x32, mrow_scan_16x32_neighbors }, + { mrow_scan_16x32, av1_mrow_iscan_16x32, mrow_scan_16x32_neighbors }, + { mcol_scan_16x32, av1_mcol_iscan_16x32, mcol_scan_16x32_neighbors }, + { mrow_scan_16x32, av1_mrow_iscan_16x32, mrow_scan_16x32_neighbors }, + { mcol_scan_16x32, av1_mcol_iscan_16x32, mcol_scan_16x32_neighbors }, + { mrow_scan_16x32, av1_mrow_iscan_16x32, mrow_scan_16x32_neighbors }, + { mcol_scan_16x32, av1_mcol_iscan_16x32, mcol_scan_16x32_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_32X16 + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { mrow_scan_32x16, av1_mrow_iscan_32x16, mrow_scan_32x16_neighbors }, + { mcol_scan_32x16, av1_mcol_iscan_32x16, mcol_scan_32x16_neighbors }, + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, +#if CONFIG_EXT_TX + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { mrow_scan_32x16, av1_mrow_iscan_32x16, mrow_scan_32x16_neighbors }, + { mrow_scan_32x16, av1_mrow_iscan_32x16, mrow_scan_32x16_neighbors }, + { mcol_scan_32x16, av1_mcol_iscan_32x16, mcol_scan_32x16_neighbors }, + { mrow_scan_32x16, av1_mrow_iscan_32x16, mrow_scan_32x16_neighbors }, + { mcol_scan_32x16, av1_mcol_iscan_32x16, mcol_scan_32x16_neighbors }, + { mrow_scan_32x16, av1_mrow_iscan_32x16, mrow_scan_32x16_neighbors }, + { mcol_scan_32x16, av1_mcol_iscan_32x16, mcol_scan_32x16_neighbors }, +#endif // CONFIG_EXT_TX + }, +}; + +const SCAN_ORDER av1_inter_scan_orders[TX_SIZES_ALL][TX_TYPES] = { +#if CONFIG_CB4X4 + { + // TX_2X2 + { default_scan_2x2, av1_default_iscan_2x2, default_scan_2x2_neighbors }, + { default_scan_2x2, av1_default_iscan_2x2, default_scan_2x2_neighbors }, + { default_scan_2x2, av1_default_iscan_2x2, default_scan_2x2_neighbors }, + { default_scan_2x2, av1_default_iscan_2x2, default_scan_2x2_neighbors }, +#if CONFIG_EXT_TX + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors }, + { mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors }, + { mcol_scan_4x4, av1_mcol_iscan_4x4, mcol_scan_4x4_neighbors }, + { mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors }, + { mcol_scan_4x4, av1_mcol_iscan_4x4, mcol_scan_4x4_neighbors }, + { mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors }, + { mcol_scan_4x4, av1_mcol_iscan_4x4, mcol_scan_4x4_neighbors }, +#endif // CONFIG_EXT_TX + }, +#endif + { + // TX_4X4 + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, +#if CONFIG_EXT_TX + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors }, + { mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors }, + { mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors }, + { mcol_scan_4x4, av1_mcol_iscan_4x4, mcol_scan_4x4_neighbors }, + { mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors }, + { mcol_scan_4x4, av1_mcol_iscan_4x4, mcol_scan_4x4_neighbors }, + { mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors }, + { mcol_scan_4x4, av1_mcol_iscan_4x4, mcol_scan_4x4_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_8X8 + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, +#if CONFIG_EXT_TX + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors }, + { mrow_scan_8x8, av1_mrow_iscan_8x8, mrow_scan_8x8_neighbors }, + { mrow_scan_8x8, av1_mrow_iscan_8x8, mrow_scan_8x8_neighbors }, + { mcol_scan_8x8, av1_mcol_iscan_8x8, mcol_scan_8x8_neighbors }, + { mrow_scan_8x8, av1_mrow_iscan_8x8, mrow_scan_8x8_neighbors }, + { mcol_scan_8x8, av1_mcol_iscan_8x8, mcol_scan_8x8_neighbors }, + { mrow_scan_8x8, av1_mrow_iscan_8x8, mrow_scan_8x8_neighbors }, + { mcol_scan_8x8, av1_mcol_iscan_8x8, mcol_scan_8x8_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_16X16 + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, +#if CONFIG_EXT_TX + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { default_scan_16x16, av1_default_iscan_16x16, + default_scan_16x16_neighbors }, + { mrow_scan_16x16, av1_mrow_iscan_16x16, mrow_scan_16x16_neighbors }, + { mrow_scan_16x16, av1_mrow_iscan_16x16, mrow_scan_16x16_neighbors }, + { mcol_scan_16x16, av1_mcol_iscan_16x16, mcol_scan_16x16_neighbors }, + { mrow_scan_16x16, av1_mrow_iscan_16x16, mrow_scan_16x16_neighbors }, + { mcol_scan_16x16, av1_mcol_iscan_16x16, mcol_scan_16x16_neighbors }, + { mrow_scan_16x16, av1_mrow_iscan_16x16, mrow_scan_16x16_neighbors }, + { mcol_scan_16x16, av1_mcol_iscan_16x16, mcol_scan_16x16_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_32X32 + { default_scan_32x32, av1_default_iscan_32x32, + default_scan_32x32_neighbors }, + { h2_scan_32x32, av1_h2_iscan_32x32, h2_scan_32x32_neighbors }, + { v2_scan_32x32, av1_v2_iscan_32x32, v2_scan_32x32_neighbors }, + { qtr_scan_32x32, av1_qtr_iscan_32x32, qtr_scan_32x32_neighbors }, +#if CONFIG_EXT_TX + { h2_scan_32x32, av1_h2_iscan_32x32, h2_scan_32x32_neighbors }, + { v2_scan_32x32, av1_v2_iscan_32x32, v2_scan_32x32_neighbors }, + { qtr_scan_32x32, av1_qtr_iscan_32x32, qtr_scan_32x32_neighbors }, + { qtr_scan_32x32, av1_qtr_iscan_32x32, qtr_scan_32x32_neighbors }, + { qtr_scan_32x32, av1_qtr_iscan_32x32, qtr_scan_32x32_neighbors }, + { mrow_scan_32x32, av1_mrow_iscan_32x32, mrow_scan_32x32_neighbors }, + { mrow_scan_32x32, av1_mrow_iscan_32x32, mrow_scan_32x32_neighbors }, + { mcol_scan_32x32, av1_mcol_iscan_32x32, mcol_scan_32x32_neighbors }, + { mrow_scan_32x32, av1_mrow_iscan_32x32, mrow_scan_32x32_neighbors }, + { mcol_scan_32x32, av1_mcol_iscan_32x32, mcol_scan_32x32_neighbors }, + { mrow_scan_32x32, av1_mrow_iscan_32x32, mrow_scan_32x32_neighbors }, + { mcol_scan_32x32, av1_mcol_iscan_32x32, mcol_scan_32x32_neighbors }, +#endif // CONFIG_EXT_TX + }, +#if CONFIG_TX64X64 + { + // TX_64X64 + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, +#if CONFIG_EXT_TX + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, + { default_scan_64x64, av1_default_iscan_64x64, + default_scan_64x64_neighbors }, +#endif // CONFIG_EXT_TX + }, +#endif // CONFIG_TX64X64 + { + // TX_4X8 + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, +#if CONFIG_EXT_TX + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { default_scan_4x8, av1_default_iscan_4x8, default_scan_4x8_neighbors }, + { mrow_scan_4x8, av1_mrow_iscan_4x8, mrow_scan_4x8_neighbors }, + { mrow_scan_4x8, av1_mrow_iscan_4x8, mrow_scan_4x8_neighbors }, + { mcol_scan_4x8, av1_mcol_iscan_4x8, mcol_scan_4x8_neighbors }, + { mrow_scan_4x8, av1_mrow_iscan_4x8, mrow_scan_4x8_neighbors }, + { mcol_scan_4x8, av1_mcol_iscan_4x8, mcol_scan_4x8_neighbors }, + { mrow_scan_4x8, av1_mrow_iscan_4x8, mrow_scan_4x8_neighbors }, + { mcol_scan_4x8, av1_mcol_iscan_4x8, mcol_scan_4x8_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_8X4 + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, +#if CONFIG_EXT_TX + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { default_scan_8x4, av1_default_iscan_8x4, default_scan_8x4_neighbors }, + { mrow_scan_8x4, av1_mrow_iscan_8x4, mrow_scan_8x4_neighbors }, + { mrow_scan_8x4, av1_mrow_iscan_8x4, mrow_scan_8x4_neighbors }, + { mcol_scan_8x4, av1_mcol_iscan_8x4, mcol_scan_8x4_neighbors }, + { mrow_scan_8x4, av1_mrow_iscan_8x4, mrow_scan_8x4_neighbors }, + { mcol_scan_8x4, av1_mcol_iscan_8x4, mcol_scan_8x4_neighbors }, + { mrow_scan_8x4, av1_mrow_iscan_8x4, mrow_scan_8x4_neighbors }, + { mcol_scan_8x4, av1_mcol_iscan_8x4, mcol_scan_8x4_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_8X16 + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, +#if CONFIG_EXT_TX + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { default_scan_8x16, av1_default_iscan_8x16, + default_scan_8x16_neighbors }, + { mrow_scan_8x16, av1_mrow_iscan_8x16, mrow_scan_8x16_neighbors }, + { mrow_scan_8x16, av1_mrow_iscan_8x16, mrow_scan_8x16_neighbors }, + { mcol_scan_8x16, av1_mcol_iscan_8x16, mcol_scan_8x16_neighbors }, + { mrow_scan_8x16, av1_mrow_iscan_8x16, mrow_scan_8x16_neighbors }, + { mcol_scan_8x16, av1_mcol_iscan_8x16, mcol_scan_8x16_neighbors }, + { mrow_scan_8x16, av1_mrow_iscan_8x16, mrow_scan_8x16_neighbors }, + { mcol_scan_8x16, av1_mcol_iscan_8x16, mcol_scan_8x16_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_16X8 + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, +#if CONFIG_EXT_TX + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { default_scan_16x8, av1_default_iscan_16x8, + default_scan_16x8_neighbors }, + { mrow_scan_16x8, av1_mrow_iscan_16x8, mrow_scan_16x8_neighbors }, + { mrow_scan_16x8, av1_mrow_iscan_16x8, mrow_scan_16x8_neighbors }, + { mcol_scan_16x8, av1_mcol_iscan_16x8, mcol_scan_16x8_neighbors }, + { mrow_scan_16x8, av1_mrow_iscan_16x8, mrow_scan_16x8_neighbors }, + { mcol_scan_16x8, av1_mcol_iscan_16x8, mcol_scan_16x8_neighbors }, + { mrow_scan_16x8, av1_mrow_iscan_16x8, mrow_scan_16x8_neighbors }, + { mcol_scan_16x8, av1_mcol_iscan_16x8, mcol_scan_16x8_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_16X32 + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, +#if CONFIG_EXT_TX + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { default_scan_16x32, av1_default_iscan_16x32, + default_scan_16x32_neighbors }, + { mrow_scan_16x32, av1_mrow_iscan_16x32, mrow_scan_16x32_neighbors }, + { mrow_scan_16x32, av1_mrow_iscan_16x32, mrow_scan_16x32_neighbors }, + { mcol_scan_16x32, av1_mcol_iscan_16x32, mcol_scan_16x32_neighbors }, + { mrow_scan_16x32, av1_mrow_iscan_16x32, mrow_scan_16x32_neighbors }, + { mcol_scan_16x32, av1_mcol_iscan_16x32, mcol_scan_16x32_neighbors }, + { mrow_scan_16x32, av1_mrow_iscan_16x32, mrow_scan_16x32_neighbors }, + { mcol_scan_16x32, av1_mcol_iscan_16x32, mcol_scan_16x32_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_32X16 + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, +#if CONFIG_EXT_TX + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { default_scan_32x16, av1_default_iscan_32x16, + default_scan_32x16_neighbors }, + { mrow_scan_32x16, av1_mrow_iscan_32x16, mrow_scan_32x16_neighbors }, + { mrow_scan_32x16, av1_mrow_iscan_32x16, mrow_scan_32x16_neighbors }, + { mcol_scan_32x16, av1_mcol_iscan_32x16, mcol_scan_32x16_neighbors }, + { mrow_scan_32x16, av1_mrow_iscan_32x16, mrow_scan_32x16_neighbors }, + { mcol_scan_32x16, av1_mcol_iscan_32x16, mcol_scan_32x16_neighbors }, + { mrow_scan_32x16, av1_mrow_iscan_32x16, mrow_scan_32x16_neighbors }, + { mcol_scan_32x16, av1_mcol_iscan_32x16, mcol_scan_32x16_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_4X16 + { default_scan_4x16, av1_default_iscan_4x16, + default_scan_4x16_neighbors }, + { default_scan_4x16, av1_default_iscan_4x16, + default_scan_4x16_neighbors }, + { default_scan_4x16, av1_default_iscan_4x16, + default_scan_4x16_neighbors }, + { default_scan_4x16, av1_default_iscan_4x16, + default_scan_4x16_neighbors }, +#if CONFIG_EXT_TX + { default_scan_4x16, av1_default_iscan_4x16, + default_scan_4x16_neighbors }, + { default_scan_4x16, av1_default_iscan_4x16, + default_scan_4x16_neighbors }, + { default_scan_4x16, av1_default_iscan_4x16, + default_scan_4x16_neighbors }, + { default_scan_4x16, av1_default_iscan_4x16, + default_scan_4x16_neighbors }, + { default_scan_4x16, av1_default_iscan_4x16, + default_scan_4x16_neighbors }, + { mrow_scan_4x16, av1_mrow_iscan_4x16, mrow_scan_4x16_neighbors }, + { mrow_scan_4x16, av1_mrow_iscan_4x16, mrow_scan_4x16_neighbors }, + { mcol_scan_4x16, av1_mcol_iscan_4x16, mcol_scan_4x16_neighbors }, + { mrow_scan_4x16, av1_mrow_iscan_4x16, mrow_scan_4x16_neighbors }, + { mcol_scan_4x16, av1_mcol_iscan_4x16, mcol_scan_4x16_neighbors }, + { mrow_scan_4x16, av1_mrow_iscan_4x16, mrow_scan_4x16_neighbors }, + { mcol_scan_4x16, av1_mcol_iscan_4x16, mcol_scan_4x16_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_16X4 + { default_scan_16x4, av1_default_iscan_16x4, + default_scan_16x4_neighbors }, + { default_scan_16x4, av1_default_iscan_16x4, + default_scan_16x4_neighbors }, + { default_scan_16x4, av1_default_iscan_16x4, + default_scan_16x4_neighbors }, + { default_scan_16x4, av1_default_iscan_16x4, + default_scan_16x4_neighbors }, +#if CONFIG_EXT_TX + { default_scan_16x4, av1_default_iscan_16x4, + default_scan_16x4_neighbors }, + { default_scan_16x4, av1_default_iscan_16x4, + default_scan_16x4_neighbors }, + { default_scan_16x4, av1_default_iscan_16x4, + default_scan_16x4_neighbors }, + { default_scan_16x4, av1_default_iscan_16x4, + default_scan_16x4_neighbors }, + { default_scan_16x4, av1_default_iscan_16x4, + default_scan_16x4_neighbors }, + { mrow_scan_16x4, av1_mrow_iscan_16x4, mrow_scan_16x4_neighbors }, + { mrow_scan_16x4, av1_mrow_iscan_16x4, mrow_scan_16x4_neighbors }, + { mcol_scan_16x4, av1_mcol_iscan_16x4, mcol_scan_16x4_neighbors }, + { mrow_scan_16x4, av1_mrow_iscan_16x4, mrow_scan_16x4_neighbors }, + { mcol_scan_16x4, av1_mcol_iscan_16x4, mcol_scan_16x4_neighbors }, + { mrow_scan_16x4, av1_mrow_iscan_16x4, mrow_scan_16x4_neighbors }, + { mcol_scan_16x4, av1_mcol_iscan_16x4, mcol_scan_16x4_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_8X32 + { default_scan_8x32, av1_default_iscan_8x32, + default_scan_8x32_neighbors }, + { default_scan_8x32, av1_default_iscan_8x32, + default_scan_8x32_neighbors }, + { default_scan_8x32, av1_default_iscan_8x32, + default_scan_8x32_neighbors }, + { default_scan_8x32, av1_default_iscan_8x32, + default_scan_8x32_neighbors }, +#if CONFIG_EXT_TX + { default_scan_8x32, av1_default_iscan_8x32, + default_scan_8x32_neighbors }, + { default_scan_8x32, av1_default_iscan_8x32, + default_scan_8x32_neighbors }, + { default_scan_8x32, av1_default_iscan_8x32, + default_scan_8x32_neighbors }, + { default_scan_8x32, av1_default_iscan_8x32, + default_scan_8x32_neighbors }, + { default_scan_8x32, av1_default_iscan_8x32, + default_scan_8x32_neighbors }, + { mrow_scan_8x32, av1_mrow_iscan_8x32, mrow_scan_8x32_neighbors }, + { mrow_scan_8x32, av1_mrow_iscan_8x32, mrow_scan_8x32_neighbors }, + { mcol_scan_8x32, av1_mcol_iscan_8x32, mcol_scan_8x32_neighbors }, + { mrow_scan_8x32, av1_mrow_iscan_8x32, mrow_scan_8x32_neighbors }, + { mcol_scan_8x32, av1_mcol_iscan_8x32, mcol_scan_8x32_neighbors }, + { mrow_scan_8x32, av1_mrow_iscan_8x32, mrow_scan_8x32_neighbors }, + { mcol_scan_8x32, av1_mcol_iscan_8x32, mcol_scan_8x32_neighbors }, +#endif // CONFIG_EXT_TX + }, + { + // TX_32X8 + { default_scan_32x8, av1_default_iscan_32x8, + default_scan_32x8_neighbors }, + { default_scan_32x8, av1_default_iscan_32x8, + default_scan_32x8_neighbors }, + { default_scan_32x8, av1_default_iscan_32x8, + default_scan_32x8_neighbors }, + { default_scan_32x8, av1_default_iscan_32x8, + default_scan_32x8_neighbors }, +#if CONFIG_EXT_TX + { default_scan_32x8, av1_default_iscan_32x8, + default_scan_32x8_neighbors }, + { default_scan_32x8, av1_default_iscan_32x8, + default_scan_32x8_neighbors }, + { default_scan_32x8, av1_default_iscan_32x8, + default_scan_32x8_neighbors }, + { default_scan_32x8, av1_default_iscan_32x8, + default_scan_32x8_neighbors }, + { default_scan_32x8, av1_default_iscan_32x8, + default_scan_32x8_neighbors }, + { mrow_scan_32x8, av1_mrow_iscan_32x8, mrow_scan_32x8_neighbors }, + { mrow_scan_32x8, av1_mrow_iscan_32x8, mrow_scan_32x8_neighbors }, + { mcol_scan_32x8, av1_mcol_iscan_32x8, mcol_scan_32x8_neighbors }, + { mrow_scan_32x8, av1_mrow_iscan_32x8, mrow_scan_32x8_neighbors }, + { mcol_scan_32x8, av1_mcol_iscan_32x8, mcol_scan_32x8_neighbors }, + { mrow_scan_32x8, av1_mrow_iscan_32x8, mrow_scan_32x8_neighbors }, + { mcol_scan_32x8, av1_mcol_iscan_32x8, mcol_scan_32x8_neighbors }, +#endif // CONFIG_EXT_TX + }, +}; + +#if CONFIG_ADAPT_SCAN +// TX_32X32 will has 1024 coefficients whose indexes can be represented in 10 +// bits +#define COEFF_IDX_BITS (10 + CONFIG_TX64X64) +#define COEFF_IDX_SIZE (1 << COEFF_IDX_BITS) +#define COEFF_IDX_MASK (COEFF_IDX_SIZE - 1) + +static uint32_t *get_non_zero_prob(FRAME_CONTEXT *fc, TX_SIZE tx_size, + TX_TYPE tx_type) { + switch (tx_size) { +#if CONFIG_CB4X4 + case TX_2X2: return fc->non_zero_prob_2x2[tx_type]; +#endif + case TX_4X4: return fc->non_zero_prob_4X4[tx_type]; + case TX_8X8: return fc->non_zero_prob_8X8[tx_type]; + case TX_16X16: return fc->non_zero_prob_16X16[tx_type]; + case TX_32X32: return fc->non_zero_prob_32X32[tx_type]; +#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + case TX_4X8: return fc->non_zero_prob_4X8[tx_type]; + case TX_8X4: return fc->non_zero_prob_8X4[tx_type]; + case TX_8X16: return fc->non_zero_prob_8X16[tx_type]; + case TX_16X8: return fc->non_zero_prob_16X8[tx_type]; + case TX_16X32: return fc->non_zero_prob_16X32[tx_type]; + case TX_32X16: return fc->non_zero_prob_32X16[tx_type]; +#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + default: assert(0); return NULL; + } +} + +static int16_t *get_adapt_scan(FRAME_CONTEXT *fc, TX_SIZE tx_size, + TX_TYPE tx_type) { + switch (tx_size) { +#if CONFIG_CB4X4 + case TX_2X2: return fc->scan_2x2[tx_type]; +#endif + case TX_4X4: return fc->scan_4X4[tx_type]; + case TX_8X8: return fc->scan_8X8[tx_type]; + case TX_16X16: return fc->scan_16X16[tx_type]; + case TX_32X32: return fc->scan_32X32[tx_type]; +#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + case TX_4X8: return fc->scan_4X8[tx_type]; + case TX_8X4: return fc->scan_8X4[tx_type]; + case TX_8X16: return fc->scan_8X16[tx_type]; + case TX_16X8: return fc->scan_16X8[tx_type]; + case TX_16X32: return fc->scan_16X32[tx_type]; + case TX_32X16: return fc->scan_32X16[tx_type]; +#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + default: assert(0); return NULL; + } +} + +static int16_t *get_adapt_iscan(FRAME_CONTEXT *fc, TX_SIZE tx_size, + TX_TYPE tx_type) { + switch (tx_size) { +#if CONFIG_CB4X4 + case TX_2X2: return fc->iscan_2x2[tx_type]; +#endif + case TX_4X4: return fc->iscan_4X4[tx_type]; + case TX_8X8: return fc->iscan_8X8[tx_type]; + case TX_16X16: return fc->iscan_16X16[tx_type]; + case TX_32X32: return fc->iscan_32X32[tx_type]; +#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + case TX_4X8: return fc->iscan_4X8[tx_type]; + case TX_8X4: return fc->iscan_8X4[tx_type]; + case TX_8X16: return fc->iscan_8X16[tx_type]; + case TX_16X8: return fc->iscan_16X8[tx_type]; + case TX_16X32: return fc->iscan_16X32[tx_type]; + case TX_32X16: return fc->iscan_32X16[tx_type]; +#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + default: assert(0); return NULL; + } +} + +static int16_t *get_adapt_nb(FRAME_CONTEXT *fc, TX_SIZE tx_size, + TX_TYPE tx_type) { + switch (tx_size) { +#if CONFIG_CB4X4 + case TX_2X2: return fc->nb_2x2[tx_type]; +#endif + case TX_4X4: return fc->nb_4X4[tx_type]; + case TX_8X8: return fc->nb_8X8[tx_type]; + case TX_16X16: return fc->nb_16X16[tx_type]; + case TX_32X32: return fc->nb_32X32[tx_type]; +#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + case TX_4X8: return fc->nb_4X8[tx_type]; + case TX_8X4: return fc->nb_8X4[tx_type]; + case TX_8X16: return fc->nb_8X16[tx_type]; + case TX_16X8: return fc->nb_16X8[tx_type]; + case TX_16X32: return fc->nb_16X32[tx_type]; + case TX_32X16: return fc->nb_32X16[tx_type]; +#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + default: assert(0); return NULL; + } +} + +static uint32_t *get_non_zero_counts(FRAME_COUNTS *counts, TX_SIZE tx_size, + TX_TYPE tx_type) { + switch (tx_size) { +#if CONFIG_CB4X4 + case TX_2X2: return counts->non_zero_count_2x2[tx_type]; +#endif + case TX_4X4: return counts->non_zero_count_4X4[tx_type]; + case TX_8X8: return counts->non_zero_count_8X8[tx_type]; + case TX_16X16: return counts->non_zero_count_16X16[tx_type]; + case TX_32X32: return counts->non_zero_count_32X32[tx_type]; +#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + case TX_4X8: return counts->non_zero_count_4x8[tx_type]; + case TX_8X4: return counts->non_zero_count_8x4[tx_type]; + case TX_8X16: return counts->non_zero_count_8x16[tx_type]; + case TX_16X8: return counts->non_zero_count_16x8[tx_type]; + case TX_16X32: return counts->non_zero_count_16x32[tx_type]; + case TX_32X16: return counts->non_zero_count_32x16[tx_type]; +#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + default: assert(0); return NULL; + } +} + +static INLINE int clamp_64(int64_t value, int low, int high) { + return value < low ? low : (value > high ? high : (int)value); +} + +static void update_scan_prob(AV1_COMMON *cm, TX_SIZE tx_size, TX_TYPE tx_type, + int rate_16) { + FRAME_CONTEXT *pre_fc = &cm->frame_contexts[cm->frame_context_idx]; + uint32_t *prev_non_zero_prob = get_non_zero_prob(pre_fc, tx_size, tx_type); + uint32_t *non_zero_prob = get_non_zero_prob(cm->fc, tx_size, tx_type); + uint32_t *non_zero_count = get_non_zero_counts(&cm->counts, tx_size, tx_type); + const int tx2d_size = tx_size_2d[tx_size]; + unsigned int block_num = cm->counts.txb_count[tx_size][tx_type]; + int i; + for (i = 0; i < tx2d_size; i++) { + int64_t curr_prob = + block_num == 0 ? 0 : (non_zero_count[i] << 16) / block_num; + int64_t prev_prob = prev_non_zero_prob[i]; + int64_t pred_prob = + (curr_prob * rate_16 + prev_prob * ((1 << 16) - rate_16)) >> 16; + // TODO(angiebird): reduce the bit usage of probabilities and remove + // clamp_64() + non_zero_prob[i] = clamp_64(pred_prob, 0, UINT16_MAX); + } +} + +static void update_scan_count(int16_t *scan, int max_scan, + const tran_low_t *dqcoeffs, + uint32_t *non_zero_count) { + int i; + for (i = 0; i < max_scan; ++i) { + int coeff_idx = scan[i]; + non_zero_count[coeff_idx] += (dqcoeffs[coeff_idx] != 0); + } +} + +void av1_update_scan_count_facade(AV1_COMMON *cm, FRAME_COUNTS *counts, + TX_SIZE tx_size, TX_TYPE tx_type, + const tran_low_t *dqcoeffs, int max_scan) { + int16_t *scan = get_adapt_scan(cm->fc, tx_size, tx_type); + uint32_t *non_zero_count = get_non_zero_counts(counts, tx_size, tx_type); + update_scan_count(scan, max_scan, dqcoeffs, non_zero_count); + ++counts->txb_count[tx_size][tx_type]; +} + +static int cmp_prob(const void *a, const void *b) { + return *(const uint32_t *)b > *(const uint32_t *)a ? 1 : -1; +} + +void av1_augment_prob(TX_SIZE tx_size, TX_TYPE tx_type, uint32_t *prob) { + // TODO(angiebird): check if we need is_inter here + const SCAN_ORDER *sc = get_default_scan(tx_size, tx_type, 0); + const int tx1d_wide = tx_size_wide[tx_size]; + const int tx1d_high = tx_size_high[tx_size]; + int r, c; + for (r = 0; r < tx1d_high; r++) { + for (c = 0; c < tx1d_wide; c++) { + const int idx = r * tx1d_wide + c; + const uint32_t mask_16 = ((1 << 16) - 1); + const uint32_t tie_breaker = ~((uint32_t)sc->iscan[idx]); + // prob[idx]: 16 bits dummy: 6 bits scan_idx: 10 bits + prob[idx] = (prob[idx] << 16) | (mask_16 & tie_breaker); + } + } +} + +// topological sort +static void dfs_scan(int tx1d_size, int *scan_idx, int coeff_idx, int16_t *scan, + int16_t *iscan) { + const int r = coeff_idx / tx1d_size; + const int c = coeff_idx % tx1d_size; + + if (iscan[coeff_idx] != -1) return; + + if (r > 0) dfs_scan(tx1d_size, scan_idx, coeff_idx - tx1d_size, scan, iscan); + + if (c > 0) dfs_scan(tx1d_size, scan_idx, coeff_idx - 1, scan, iscan); + + scan[*scan_idx] = coeff_idx; + iscan[coeff_idx] = *scan_idx; + ++(*scan_idx); +} + +void av1_update_neighbors(int tx_size, const int16_t *scan, + const int16_t *iscan, int16_t *neighbors) { + const int tx1d_wide = tx_size_wide[tx_size]; + const int tx1d_high = tx_size_high[tx_size]; + const int tx2d_size = tx_size_2d[tx_size]; + int scan_idx; + for (scan_idx = 0; scan_idx < tx2d_size; ++scan_idx) { + const int coeff_idx = scan[scan_idx]; + const int r = coeff_idx / tx1d_wide; + const int c = coeff_idx % tx1d_wide; + const int nb_offset_r[5] = { -1, 0, -1, -1, 1 }; + const int nb_offset_c[5] = { 0, -1, -1, 1, -1 }; + const int nb_num = 5; + int nb_count = 0; + int nb_idx; + + for (nb_idx = 0; nb_idx < nb_num; ++nb_idx) { + if (nb_count < 2) { + int nb_r = r + nb_offset_r[nb_idx]; + int nb_c = c + nb_offset_c[nb_idx]; + int nb_coeff_idx = nb_r * tx1d_wide + nb_c; + int valid_pos = + nb_r >= 0 && nb_r < tx1d_high && nb_c >= 0 && nb_c < tx1d_wide; + if (valid_pos && iscan[nb_coeff_idx] < scan_idx) { + neighbors[scan_idx * MAX_NEIGHBORS + nb_count] = nb_coeff_idx; + ++nb_count; + } + } else { + break; + } + } + + if (nb_count == 1) { + neighbors[scan_idx * MAX_NEIGHBORS + 1] = + neighbors[scan_idx * MAX_NEIGHBORS + 0]; + } else if (nb_count == 0) { + neighbors[scan_idx * MAX_NEIGHBORS + 0] = scan[0]; + neighbors[scan_idx * MAX_NEIGHBORS + 1] = scan[0]; + } + } + neighbors[tx2d_size * MAX_NEIGHBORS + 0] = scan[0]; + neighbors[tx2d_size * MAX_NEIGHBORS + 1] = scan[0]; +} + +void av1_update_sort_order(TX_SIZE tx_size, TX_TYPE tx_type, + const uint32_t *non_zero_prob, int16_t *sort_order) { + const SCAN_ORDER *sc = get_default_scan(tx_size, tx_type, 0); + uint32_t temp[COEFF_IDX_SIZE]; + const int tx2d_size = tx_size_2d[tx_size]; + int sort_idx; + assert(tx2d_size <= COEFF_IDX_SIZE); + memcpy(temp, non_zero_prob, tx2d_size * sizeof(*non_zero_prob)); + av1_augment_prob(tx_size, tx_type, temp); + qsort(temp, tx2d_size, sizeof(*temp), cmp_prob); + for (sort_idx = 0; sort_idx < tx2d_size; ++sort_idx) { + const int default_scan_idx = + (temp[sort_idx] & COEFF_IDX_MASK) ^ COEFF_IDX_MASK; + const int coeff_idx = sc->scan[default_scan_idx]; + sort_order[sort_idx] = coeff_idx; + } +} + +void av1_update_scan_order(TX_SIZE tx_size, int16_t *sort_order, int16_t *scan, + int16_t *iscan) { + int coeff_idx; + int scan_idx; + int sort_idx; + const int tx1d_size = tx_size_wide[tx_size]; + const int tx2d_size = tx_size_2d[tx_size]; + + for (coeff_idx = 0; coeff_idx < tx2d_size; ++coeff_idx) { + iscan[coeff_idx] = -1; + } + + scan_idx = 0; + for (sort_idx = 0; sort_idx < tx2d_size; ++sort_idx) { + coeff_idx = sort_order[sort_idx]; + dfs_scan(tx1d_size, &scan_idx, coeff_idx, scan, iscan); + } +} + +static void update_scan_order_facade(AV1_COMMON *cm, TX_SIZE tx_size, + TX_TYPE tx_type) { + int16_t sort_order[COEFF_IDX_SIZE]; + uint32_t *non_zero_prob = get_non_zero_prob(cm->fc, tx_size, tx_type); + int16_t *scan = get_adapt_scan(cm->fc, tx_size, tx_type); + int16_t *iscan = get_adapt_iscan(cm->fc, tx_size, tx_type); + int16_t *nb = get_adapt_nb(cm->fc, tx_size, tx_type); + assert(tx_size_2d[tx_size] <= COEFF_IDX_SIZE); + av1_update_sort_order(tx_size, tx_type, non_zero_prob, sort_order); + av1_update_scan_order(tx_size, sort_order, scan, iscan); + av1_update_neighbors(tx_size, scan, iscan, nb); +} + +static void update_eob_threshold(AV1_COMMON *cm, TX_SIZE tx_size, + TX_TYPE tx_type) { + int i, row, col, row_limit, col_limit, cal_idx = 0; + const int tx_width = tx_size_wide[tx_size]; + const int tx_height = tx_size_high[tx_size]; + + row_limit = tx_width >> 1; + col_limit = tx_height >> 1; + + if (tx_width >= 8 && tx_height >= 8) { + SCAN_ORDER *sc = &cm->fc->sc[tx_size][tx_type]; + int16_t *threshold = &cm->fc->eob_threshold[tx_size][tx_type][0]; + const int tx2d_size = tx_size_2d[tx_size]; + + while (cal_idx < EOB_THRESHOLD_NUM) { + for (i = 0; i < tx2d_size; ++i) { + row = sc->scan[i] / tx_height; + col = sc->scan[i] % tx_width; + if (row >= row_limit || col >= col_limit) break; + } + row_limit >>= 1; + col_limit >>= 1; + threshold[cal_idx] = i; + cal_idx++; + } + } +} + +void av1_init_scan_order(AV1_COMMON *cm) { + TX_SIZE tx_size; + TX_TYPE tx_type; + for (tx_size = 0; tx_size < TX_SIZES_ALL; ++tx_size) { +#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + if (tx_size > TX_32X16) continue; +#else + if (tx_size >= TX_SIZES) continue; +#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) { + uint32_t *non_zero_prob = get_non_zero_prob(cm->fc, tx_size, tx_type); + const int tx2d_size = tx_size_2d[tx_size]; + int i; + SCAN_ORDER *sc = &cm->fc->sc[tx_size][tx_type]; + for (i = 0; i < tx2d_size; ++i) { + non_zero_prob[i] = (1 << 16) / 2; // init non_zero_prob to 0.5 + } + update_scan_order_facade(cm, tx_size, tx_type); + sc->scan = get_adapt_scan(cm->fc, tx_size, tx_type); + sc->iscan = get_adapt_iscan(cm->fc, tx_size, tx_type); + sc->neighbors = get_adapt_nb(cm->fc, tx_size, tx_type); + update_eob_threshold(cm, tx_size, tx_type); + } + } +} + +void av1_adapt_scan_order(AV1_COMMON *cm) { + TX_SIZE tx_size; + for (tx_size = 0; tx_size < TX_SIZES_ALL; ++tx_size) { +#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + if (tx_size > TX_32X16) continue; +#else + if (tx_size >= TX_SIZES) continue; +#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) + TX_TYPE tx_type; + for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) { + update_scan_prob(cm, tx_size, tx_type, ADAPT_SCAN_UPDATE_RATE_16); + update_scan_order_facade(cm, tx_size, tx_type); + update_eob_threshold(cm, tx_size, tx_type); + } + } +} + +void av1_deliver_eob_threshold(const AV1_COMMON *cm, MACROBLOCKD *xd) { + xd->eob_threshold_md = (const EobThresholdMD *)cm->fc->eob_threshold; +} +#endif // CONFIG_ADAPT_SCAN diff --git a/third_party/aom/av1/common/scan.h b/third_party/aom/av1/common/scan.h new file mode 100644 index 0000000000..ecef113686 --- /dev/null +++ b/third_party/aom/av1/common/scan.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_SCAN_H_ +#define AV1_COMMON_SCAN_H_ + +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" + +#include "av1/common/enums.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/blockd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MAX_NEIGHBORS 2 + +extern const SCAN_ORDER av1_default_scan_orders[TX_SIZES]; +extern const SCAN_ORDER av1_intra_scan_orders[TX_SIZES_ALL][TX_TYPES]; +extern const SCAN_ORDER av1_inter_scan_orders[TX_SIZES_ALL][TX_TYPES]; + +#if CONFIG_ADAPT_SCAN +void av1_update_scan_count_facade(AV1_COMMON *cm, FRAME_COUNTS *counts, + TX_SIZE tx_size, TX_TYPE tx_type, + const tran_low_t *dqcoeffs, int max_scan); + +// embed r + c and coeff_idx info with nonzero probabilities. When sorting the +// nonzero probabilities, if there is a tie, the coefficient with smaller r + c +// will be scanned first +void av1_augment_prob(TX_SIZE tx_size, TX_TYPE tx_type, uint32_t *prob); + +// apply quick sort on nonzero probabilities to obtain a sort order +void av1_update_sort_order(TX_SIZE tx_size, TX_TYPE tx_type, + const uint32_t *non_zero_prob, int16_t *sort_order); + +// apply topological sort on the nonzero probabilities sorting order to +// guarantee each to-be-scanned coefficient's upper and left coefficient will be +// scanned before the to-be-scanned coefficient. +void av1_update_scan_order(TX_SIZE tx_size, int16_t *sort_order, int16_t *scan, + int16_t *iscan); + +// For each coeff_idx in scan[], update its above and left neighbors in +// neighbors[] accordingly. +void av1_update_neighbors(int tx_size, const int16_t *scan, + const int16_t *iscan, int16_t *neighbors); +void av1_init_scan_order(AV1_COMMON *cm); +void av1_adapt_scan_order(AV1_COMMON *cm); +#endif +void av1_deliver_eob_threshold(const AV1_COMMON *cm, MACROBLOCKD *xd); + +static INLINE int get_coef_context(const int16_t *neighbors, + const uint8_t *token_cache, int c) { + return (1 + token_cache[neighbors[MAX_NEIGHBORS * c + 0]] + + token_cache[neighbors[MAX_NEIGHBORS * c + 1]]) >> + 1; +} + +static INLINE const SCAN_ORDER *get_default_scan(TX_SIZE tx_size, + TX_TYPE tx_type, + int is_inter) { +#if CONFIG_EXT_TX || CONFIG_VAR_TX + return is_inter ? &av1_inter_scan_orders[tx_size][tx_type] + : &av1_intra_scan_orders[tx_size][tx_type]; +#else + (void)is_inter; + return &av1_intra_scan_orders[tx_size][tx_type]; +#endif // CONFIG_EXT_TX +} + +static INLINE const SCAN_ORDER *get_scan(const AV1_COMMON *cm, TX_SIZE tx_size, + TX_TYPE tx_type, int is_inter) { +#if CONFIG_ADAPT_SCAN + (void)is_inter; + return &cm->fc->sc[tx_size][tx_type]; +#else // CONFIG_ADAPT_SCAN + (void)cm; + return get_default_scan(tx_size, tx_type, is_inter); +#endif // CONFIG_ADAPT_SCAN +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_SCAN_H_ diff --git a/third_party/aom/av1/common/seg_common.c b/third_party/aom/av1/common/seg_common.c new file mode 100644 index 0000000000..21a8536297 --- /dev/null +++ b/third_party/aom/av1/common/seg_common.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/av1_loopfilter.h" +#include "av1/common/blockd.h" +#include "av1/common/seg_common.h" +#include "av1/common/quant_common.h" + +static const int seg_feature_data_signed[SEG_LVL_MAX] = { 1, 1, 0, 0 }; + +static const int seg_feature_data_max[SEG_LVL_MAX] = { MAXQ, MAX_LOOP_FILTER, 3, + 0 }; + +// These functions provide access to new segment level features. +// Eventually these function may be "optimized out" but for the moment, +// the coding mechanism is still subject to change so these provide a +// convenient single point of change. + +void av1_clearall_segfeatures(struct segmentation *seg) { + av1_zero(seg->feature_data); + av1_zero(seg->feature_mask); +} + +void av1_enable_segfeature(struct segmentation *seg, int segment_id, + SEG_LVL_FEATURES feature_id) { + seg->feature_mask[segment_id] |= 1 << feature_id; +} + +int av1_seg_feature_data_max(SEG_LVL_FEATURES feature_id) { + return seg_feature_data_max[feature_id]; +} + +int av1_is_segfeature_signed(SEG_LVL_FEATURES feature_id) { + return seg_feature_data_signed[feature_id]; +} + +void av1_set_segdata(struct segmentation *seg, int segment_id, + SEG_LVL_FEATURES feature_id, int seg_data) { + assert(seg_data <= seg_feature_data_max[feature_id]); + if (seg_data < 0) { + assert(seg_feature_data_signed[feature_id]); + assert(-seg_data <= seg_feature_data_max[feature_id]); + } + + seg->feature_data[segment_id][feature_id] = seg_data; +} + +const aom_tree_index av1_segment_tree[TREE_SIZE(MAX_SEGMENTS)] = { + 2, 4, 6, 8, 10, 12, 0, -1, -2, -3, -4, -5, -6, -7 +}; + +// TBD? Functions to read and write segment data with range / validity checking diff --git a/third_party/aom/av1/common/seg_common.h b/third_party/aom/av1/common/seg_common.h new file mode 100644 index 0000000000..03ed38e790 --- /dev/null +++ b/third_party/aom/av1/common/seg_common.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_SEG_COMMON_H_ +#define AV1_COMMON_SEG_COMMON_H_ + +#include "aom_dsp/prob.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define SEGMENT_DELTADATA 0 +#define SEGMENT_ABSDATA 1 + +#define MAX_SEGMENTS 8 +#define SEG_TREE_PROBS (MAX_SEGMENTS - 1) + +#define PREDICTION_PROBS 3 + +// Segment level features. +typedef enum { + SEG_LVL_ALT_Q = 0, // Use alternate Quantizer .... + SEG_LVL_ALT_LF = 1, // Use alternate loop filter value... + SEG_LVL_REF_FRAME = 2, // Optional Segment reference frame + SEG_LVL_SKIP = 3, // Optional Segment (0,0) + skip mode + SEG_LVL_MAX = 4 // Number of features supported +} SEG_LVL_FEATURES; + +struct segmentation { + uint8_t enabled; + uint8_t update_map; + uint8_t update_data; + uint8_t abs_delta; + uint8_t temporal_update; + + int16_t feature_data[MAX_SEGMENTS][SEG_LVL_MAX]; + unsigned int feature_mask[MAX_SEGMENTS]; +}; + +struct segmentation_probs { + aom_prob tree_probs[SEG_TREE_PROBS]; +#if CONFIG_EC_MULTISYMBOL + aom_cdf_prob tree_cdf[CDF_SIZE(MAX_SEGMENTS)]; +#endif + aom_prob pred_probs[PREDICTION_PROBS]; +}; + +static INLINE int segfeature_active(const struct segmentation *seg, + int segment_id, + SEG_LVL_FEATURES feature_id) { + return seg->enabled && (seg->feature_mask[segment_id] & (1 << feature_id)); +} + +void av1_clearall_segfeatures(struct segmentation *seg); + +void av1_enable_segfeature(struct segmentation *seg, int segment_id, + SEG_LVL_FEATURES feature_id); + +int av1_seg_feature_data_max(SEG_LVL_FEATURES feature_id); + +int av1_is_segfeature_signed(SEG_LVL_FEATURES feature_id); + +void av1_set_segdata(struct segmentation *seg, int segment_id, + SEG_LVL_FEATURES feature_id, int seg_data); + +static INLINE int get_segdata(const struct segmentation *seg, int segment_id, + SEG_LVL_FEATURES feature_id) { + return seg->feature_data[segment_id][feature_id]; +} + +extern const aom_tree_index av1_segment_tree[TREE_SIZE(MAX_SEGMENTS)]; + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_SEG_COMMON_H_ diff --git a/third_party/aom/av1/common/thread_common.c b/third_party/aom/av1/common/thread_common.c new file mode 100644 index 0000000000..ca8b1b3bd3 --- /dev/null +++ b/third_party/aom/av1/common/thread_common.c @@ -0,0 +1,529 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "av1/common/entropymode.h" +#include "av1/common/thread_common.h" +#include "av1/common/reconinter.h" + +#if CONFIG_MULTITHREAD +static INLINE void mutex_lock(pthread_mutex_t *const mutex) { + const int kMaxTryLocks = 4000; + int locked = 0; + int i; + + for (i = 0; i < kMaxTryLocks; ++i) { + if (!pthread_mutex_trylock(mutex)) { + locked = 1; + break; + } + } + + if (!locked) pthread_mutex_lock(mutex); +} +#endif // CONFIG_MULTITHREAD + +static INLINE void sync_read(AV1LfSync *const lf_sync, int r, int c) { +#if CONFIG_MULTITHREAD + const int nsync = lf_sync->sync_range; + + if (r && !(c & (nsync - 1))) { + pthread_mutex_t *const mutex = &lf_sync->mutex_[r - 1]; + mutex_lock(mutex); + + while (c > lf_sync->cur_sb_col[r - 1] - nsync) { + pthread_cond_wait(&lf_sync->cond_[r - 1], mutex); + } + pthread_mutex_unlock(mutex); + } +#else + (void)lf_sync; + (void)r; + (void)c; +#endif // CONFIG_MULTITHREAD +} + +static INLINE void sync_write(AV1LfSync *const lf_sync, int r, int c, + const int sb_cols) { +#if CONFIG_MULTITHREAD + const int nsync = lf_sync->sync_range; + int cur; + // Only signal when there are enough filtered SB for next row to run. + int sig = 1; + + if (c < sb_cols - 1) { + cur = c; + if (c % nsync) sig = 0; + } else { + cur = sb_cols + nsync; + } + + if (sig) { + mutex_lock(&lf_sync->mutex_[r]); + + lf_sync->cur_sb_col[r] = cur; + + pthread_cond_signal(&lf_sync->cond_[r]); + pthread_mutex_unlock(&lf_sync->mutex_[r]); + } +#else + (void)lf_sync; + (void)r; + (void)c; + (void)sb_cols; +#endif // CONFIG_MULTITHREAD +} + +#if !CONFIG_EXT_PARTITION_TYPES +static INLINE enum lf_path get_loop_filter_path( + int y_only, struct macroblockd_plane planes[MAX_MB_PLANE]) { + if (y_only) + return LF_PATH_444; + else if (planes[1].subsampling_y == 1 && planes[1].subsampling_x == 1) + return LF_PATH_420; + else if (planes[1].subsampling_y == 0 && planes[1].subsampling_x == 0) + return LF_PATH_444; + else + return LF_PATH_SLOW; +} + +static INLINE void loop_filter_block_plane_ver( + AV1_COMMON *cm, struct macroblockd_plane planes[MAX_MB_PLANE], int plane, + MODE_INFO **mi, int mi_row, int mi_col, enum lf_path path, + LOOP_FILTER_MASK *lfm) { + if (plane == 0) { + av1_filter_block_plane_ss00_ver(cm, &planes[0], mi_row, lfm); + } else { + switch (path) { + case LF_PATH_420: + av1_filter_block_plane_ss11_ver(cm, &planes[plane], mi_row, lfm); + break; + case LF_PATH_444: + av1_filter_block_plane_ss00_ver(cm, &planes[plane], mi_row, lfm); + break; + case LF_PATH_SLOW: + av1_filter_block_plane_non420_ver(cm, &planes[plane], mi, mi_row, + mi_col); + break; + } + } +} + +static INLINE void loop_filter_block_plane_hor( + AV1_COMMON *cm, struct macroblockd_plane planes[MAX_MB_PLANE], int plane, + MODE_INFO **mi, int mi_row, int mi_col, enum lf_path path, + LOOP_FILTER_MASK *lfm) { + if (plane == 0) { + av1_filter_block_plane_ss00_hor(cm, &planes[0], mi_row, lfm); + } else { + switch (path) { + case LF_PATH_420: + av1_filter_block_plane_ss11_hor(cm, &planes[plane], mi_row, lfm); + break; + case LF_PATH_444: + av1_filter_block_plane_ss00_hor(cm, &planes[plane], mi_row, lfm); + break; + case LF_PATH_SLOW: + av1_filter_block_plane_non420_hor(cm, &planes[plane], mi, mi_row, + mi_col); + break; + } + } +} +#endif +// Row-based multi-threaded loopfilter hook +#if CONFIG_PARALLEL_DEBLOCKING +static int loop_filter_ver_row_worker(AV1LfSync *const lf_sync, + LFWorkerData *const lf_data) { + const int num_planes = lf_data->y_only ? 1 : MAX_MB_PLANE; + int mi_row, mi_col; +#if !CONFIG_EXT_PARTITION_TYPES + enum lf_path path = get_loop_filter_path(lf_data->y_only, lf_data->planes); +#endif + for (mi_row = lf_data->start; mi_row < lf_data->stop; + mi_row += lf_sync->num_workers * lf_data->cm->mib_size) { + MODE_INFO **const mi = + lf_data->cm->mi_grid_visible + mi_row * lf_data->cm->mi_stride; + + for (mi_col = 0; mi_col < lf_data->cm->mi_cols; + mi_col += lf_data->cm->mib_size) { + LOOP_FILTER_MASK lfm; + int plane; + + av1_setup_dst_planes(lf_data->planes, lf_data->cm->sb_size, + lf_data->frame_buffer, mi_row, mi_col); + av1_setup_mask(lf_data->cm, mi_row, mi_col, mi + mi_col, + lf_data->cm->mi_stride, &lfm); + +#if CONFIG_EXT_PARTITION_TYPES + for (plane = 0; plane < num_planes; ++plane) + av1_filter_block_plane_non420_ver(lf_data->cm, &lf_data->planes[plane], + mi + mi_col, mi_row, mi_col); +#else + + for (plane = 0; plane < num_planes; ++plane) + loop_filter_block_plane_ver(lf_data->cm, lf_data->planes, plane, + mi + mi_col, mi_row, mi_col, path, &lfm); +#endif + } + } + return 1; +} + +static int loop_filter_hor_row_worker(AV1LfSync *const lf_sync, + LFWorkerData *const lf_data) { + const int num_planes = lf_data->y_only ? 1 : MAX_MB_PLANE; + const int sb_cols = + mi_cols_aligned_to_sb(lf_data->cm) >> lf_data->cm->mib_size_log2; + int mi_row, mi_col; +#if !CONFIG_EXT_PARTITION_TYPES + enum lf_path path = get_loop_filter_path(lf_data->y_only, lf_data->planes); +#endif + + for (mi_row = lf_data->start; mi_row < lf_data->stop; + mi_row += lf_sync->num_workers * lf_data->cm->mib_size) { + MODE_INFO **const mi = + lf_data->cm->mi_grid_visible + mi_row * lf_data->cm->mi_stride; + + for (mi_col = 0; mi_col < lf_data->cm->mi_cols; + mi_col += lf_data->cm->mib_size) { + const int r = mi_row >> lf_data->cm->mib_size_log2; + const int c = mi_col >> lf_data->cm->mib_size_log2; + LOOP_FILTER_MASK lfm; + int plane; + + // TODO(wenhao.zhang@intel.com): For better parallelization, reorder + // the outer loop to column-based and remove the synchronizations here. + sync_read(lf_sync, r, c); + + av1_setup_dst_planes(lf_data->planes, lf_data->cm->sb_size, + lf_data->frame_buffer, mi_row, mi_col); + av1_setup_mask(lf_data->cm, mi_row, mi_col, mi + mi_col, + lf_data->cm->mi_stride, &lfm); +#if CONFIG_EXT_PARTITION_TYPES + for (plane = 0; plane < num_planes; ++plane) + av1_filter_block_plane_non420_hor(lf_data->cm, &lf_data->planes[plane], + mi + mi_col, mi_row, mi_col); +#else + for (plane = 0; plane < num_planes; ++plane) + loop_filter_block_plane_hor(lf_data->cm, lf_data->planes, plane, + mi + mi_col, mi_row, mi_col, path, &lfm); +#endif + sync_write(lf_sync, r, c, sb_cols); + } + } + return 1; +} +#else // CONFIG_PARALLEL_DEBLOCKING +static int loop_filter_row_worker(AV1LfSync *const lf_sync, + LFWorkerData *const lf_data) { + const int num_planes = lf_data->y_only ? 1 : MAX_MB_PLANE; + const int sb_cols = + mi_cols_aligned_to_sb(lf_data->cm) >> lf_data->cm->mib_size_log2; + int mi_row, mi_col; +#if !CONFIG_EXT_PARTITION_TYPES + enum lf_path path = get_loop_filter_path(lf_data->y_only, lf_data->planes); +#endif // !CONFIG_EXT_PARTITION_TYPES + +#if CONFIG_EXT_PARTITION + printf( + "STOPPING: This code has not been modified to work with the " + "extended coding unit size experiment"); + exit(EXIT_FAILURE); +#endif // CONFIG_EXT_PARTITION + + for (mi_row = lf_data->start; mi_row < lf_data->stop; + mi_row += lf_sync->num_workers * lf_data->cm->mib_size) { + MODE_INFO **const mi = + lf_data->cm->mi_grid_visible + mi_row * lf_data->cm->mi_stride; + + for (mi_col = 0; mi_col < lf_data->cm->mi_cols; + mi_col += lf_data->cm->mib_size) { + const int r = mi_row >> lf_data->cm->mib_size_log2; + const int c = mi_col >> lf_data->cm->mib_size_log2; +#if !CONFIG_EXT_PARTITION_TYPES + LOOP_FILTER_MASK lfm; +#endif + int plane; + + sync_read(lf_sync, r, c); + + av1_setup_dst_planes(lf_data->planes, lf_data->cm->sb_size, + lf_data->frame_buffer, mi_row, mi_col); +#if CONFIG_EXT_PARTITION_TYPES + for (plane = 0; plane < num_planes; ++plane) { + av1_filter_block_plane_non420_ver(lf_data->cm, &lf_data->planes[plane], + mi + mi_col, mi_row, mi_col); + av1_filter_block_plane_non420_hor(lf_data->cm, &lf_data->planes[plane], + mi + mi_col, mi_row, mi_col); + } +#else + av1_setup_mask(lf_data->cm, mi_row, mi_col, mi + mi_col, + lf_data->cm->mi_stride, &lfm); + + for (plane = 0; plane < num_planes; ++plane) { + loop_filter_block_plane_ver(lf_data->cm, lf_data->planes, plane, + mi + mi_col, mi_row, mi_col, path, &lfm); + loop_filter_block_plane_hor(lf_data->cm, lf_data->planes, plane, + mi + mi_col, mi_row, mi_col, path, &lfm); + } +#endif // CONFIG_EXT_PARTITION_TYPES + sync_write(lf_sync, r, c, sb_cols); + } + } + return 1; +} +#endif // CONFIG_PARALLEL_DEBLOCKING + +static void loop_filter_rows_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, + struct macroblockd_plane planes[MAX_MB_PLANE], + int start, int stop, int y_only, + AVxWorker *workers, int nworkers, + AV1LfSync *lf_sync) { + const AVxWorkerInterface *const winterface = aom_get_worker_interface(); + // Number of superblock rows and cols + const int sb_rows = mi_rows_aligned_to_sb(cm) >> cm->mib_size_log2; + // Decoder may allocate more threads than number of tiles based on user's + // input. + const int tile_cols = cm->tile_cols; + const int num_workers = AOMMIN(nworkers, tile_cols); + int i; + +#if CONFIG_EXT_PARTITION + printf( + "STOPPING: This code has not been modified to work with the " + "extended coding unit size experiment"); + exit(EXIT_FAILURE); +#endif // CONFIG_EXT_PARTITION + + if (!lf_sync->sync_range || sb_rows != lf_sync->rows || + num_workers > lf_sync->num_workers) { + av1_loop_filter_dealloc(lf_sync); + av1_loop_filter_alloc(lf_sync, cm, sb_rows, cm->width, num_workers); + } + +// Set up loopfilter thread data. +// The decoder is capping num_workers because it has been observed that using +// more threads on the loopfilter than there are cores will hurt performance +// on Android. This is because the system will only schedule the tile decode +// workers on cores equal to the number of tile columns. Then if the decoder +// tries to use more threads for the loopfilter, it will hurt performance +// because of contention. If the multithreading code changes in the future +// then the number of workers used by the loopfilter should be revisited. + +#if CONFIG_PARALLEL_DEBLOCKING + // Initialize cur_sb_col to -1 for all SB rows. + memset(lf_sync->cur_sb_col, -1, sizeof(*lf_sync->cur_sb_col) * sb_rows); + + // Filter all the vertical edges in the whole frame + for (i = 0; i < num_workers; ++i) { + AVxWorker *const worker = &workers[i]; + LFWorkerData *const lf_data = &lf_sync->lfdata[i]; + + worker->hook = (AVxWorkerHook)loop_filter_ver_row_worker; + worker->data1 = lf_sync; + worker->data2 = lf_data; + + // Loopfilter data + av1_loop_filter_data_reset(lf_data, frame, cm, planes); + lf_data->start = start + i * cm->mib_size; + lf_data->stop = stop; + lf_data->y_only = y_only; + + // Start loopfiltering + if (i == num_workers - 1) { + winterface->execute(worker); + } else { + winterface->launch(worker); + } + } + + // Wait till all rows are finished + for (i = 0; i < num_workers; ++i) { + winterface->sync(&workers[i]); + } + + memset(lf_sync->cur_sb_col, -1, sizeof(*lf_sync->cur_sb_col) * sb_rows); + // Filter all the horizontal edges in the whole frame + for (i = 0; i < num_workers; ++i) { + AVxWorker *const worker = &workers[i]; + LFWorkerData *const lf_data = &lf_sync->lfdata[i]; + + worker->hook = (AVxWorkerHook)loop_filter_hor_row_worker; + worker->data1 = lf_sync; + worker->data2 = lf_data; + + // Loopfilter data + av1_loop_filter_data_reset(lf_data, frame, cm, planes); + lf_data->start = start + i * cm->mib_size; + lf_data->stop = stop; + lf_data->y_only = y_only; + + // Start loopfiltering + if (i == num_workers - 1) { + winterface->execute(worker); + } else { + winterface->launch(worker); + } + } + + // Wait till all rows are finished + for (i = 0; i < num_workers; ++i) { + winterface->sync(&workers[i]); + } +#else // CONFIG_PARALLEL_DEBLOCKING + // Initialize cur_sb_col to -1 for all SB rows. + memset(lf_sync->cur_sb_col, -1, sizeof(*lf_sync->cur_sb_col) * sb_rows); + + for (i = 0; i < num_workers; ++i) { + AVxWorker *const worker = &workers[i]; + LFWorkerData *const lf_data = &lf_sync->lfdata[i]; + + worker->hook = (AVxWorkerHook)loop_filter_row_worker; + worker->data1 = lf_sync; + worker->data2 = lf_data; + + // Loopfilter data + av1_loop_filter_data_reset(lf_data, frame, cm, planes); + lf_data->start = start + i * cm->mib_size; + lf_data->stop = stop; + lf_data->y_only = y_only; + + // Start loopfiltering + if (i == num_workers - 1) { + winterface->execute(worker); + } else { + winterface->launch(worker); + } + } + + // Wait till all rows are finished + for (i = 0; i < num_workers; ++i) { + winterface->sync(&workers[i]); + } +#endif // CONFIG_PARALLEL_DEBLOCKING +} + +void av1_loop_filter_frame_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, + struct macroblockd_plane planes[MAX_MB_PLANE], + int frame_filter_level, int y_only, + int partial_frame, AVxWorker *workers, + int num_workers, AV1LfSync *lf_sync) { + int start_mi_row, end_mi_row, mi_rows_to_filter; + + if (!frame_filter_level) return; + + start_mi_row = 0; + mi_rows_to_filter = cm->mi_rows; + if (partial_frame && cm->mi_rows > 8) { + start_mi_row = cm->mi_rows >> 1; + start_mi_row &= 0xfffffff8; + mi_rows_to_filter = AOMMAX(cm->mi_rows / 8, 8); + } + end_mi_row = start_mi_row + mi_rows_to_filter; + av1_loop_filter_frame_init(cm, frame_filter_level); + + loop_filter_rows_mt(frame, cm, planes, start_mi_row, end_mi_row, y_only, + workers, num_workers, lf_sync); +} + +// Set up nsync by width. +static INLINE int get_sync_range(int width) { + // nsync numbers are picked by testing. For example, for 4k + // video, using 4 gives best performance. + if (width < 640) + return 1; + else if (width <= 1280) + return 2; + else if (width <= 4096) + return 4; + else + return 8; +} + +// Allocate memory for lf row synchronization +void av1_loop_filter_alloc(AV1LfSync *lf_sync, AV1_COMMON *cm, int rows, + int width, int num_workers) { + lf_sync->rows = rows; +#if CONFIG_MULTITHREAD + { + int i; + + CHECK_MEM_ERROR(cm, lf_sync->mutex_, + aom_malloc(sizeof(*lf_sync->mutex_) * rows)); + if (lf_sync->mutex_) { + for (i = 0; i < rows; ++i) { + pthread_mutex_init(&lf_sync->mutex_[i], NULL); + } + } + + CHECK_MEM_ERROR(cm, lf_sync->cond_, + aom_malloc(sizeof(*lf_sync->cond_) * rows)); + if (lf_sync->cond_) { + for (i = 0; i < rows; ++i) { + pthread_cond_init(&lf_sync->cond_[i], NULL); + } + } + } +#endif // CONFIG_MULTITHREAD + + CHECK_MEM_ERROR(cm, lf_sync->lfdata, + aom_malloc(num_workers * sizeof(*lf_sync->lfdata))); + lf_sync->num_workers = num_workers; + + CHECK_MEM_ERROR(cm, lf_sync->cur_sb_col, + aom_malloc(sizeof(*lf_sync->cur_sb_col) * rows)); + + // Set up nsync. + lf_sync->sync_range = get_sync_range(width); +} + +// Deallocate lf synchronization related mutex and data +void av1_loop_filter_dealloc(AV1LfSync *lf_sync) { + if (lf_sync != NULL) { +#if CONFIG_MULTITHREAD + int i; + + if (lf_sync->mutex_ != NULL) { + for (i = 0; i < lf_sync->rows; ++i) { + pthread_mutex_destroy(&lf_sync->mutex_[i]); + } + aom_free(lf_sync->mutex_); + } + if (lf_sync->cond_ != NULL) { + for (i = 0; i < lf_sync->rows; ++i) { + pthread_cond_destroy(&lf_sync->cond_[i]); + } + aom_free(lf_sync->cond_); + } +#endif // CONFIG_MULTITHREAD + aom_free(lf_sync->lfdata); + aom_free(lf_sync->cur_sb_col); + // clear the structure as the source of this call may be a resize in which + // case this call will be followed by an _alloc() which may fail. + av1_zero(*lf_sync); + } +} + +// Accumulate frame counts. FRAME_COUNTS consist solely of 'unsigned int' +// members, so we treat it as an array, and sum over the whole length. +void av1_accumulate_frame_counts(FRAME_COUNTS *acc_counts, + FRAME_COUNTS *counts) { + unsigned int *const acc = (unsigned int *)acc_counts; + const unsigned int *const cnt = (unsigned int *)counts; + + const unsigned int n_counts = sizeof(FRAME_COUNTS) / sizeof(unsigned int); + unsigned int i; + + for (i = 0; i < n_counts; i++) acc[i] += cnt[i]; +} diff --git a/third_party/aom/av1/common/thread_common.h b/third_party/aom/av1/common/thread_common.h new file mode 100644 index 0000000000..7b57ae8f34 --- /dev/null +++ b/third_party/aom/av1/common/thread_common.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_LOOPFILTER_THREAD_H_ +#define AV1_COMMON_LOOPFILTER_THREAD_H_ +#include "./aom_config.h" +#include "av1/common/av1_loopfilter.h" +#include "aom_util/aom_thread.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct AV1Common; +struct FRAME_COUNTS; + +// Loopfilter row synchronization +typedef struct AV1LfSyncData { +#if CONFIG_MULTITHREAD + pthread_mutex_t *mutex_; + pthread_cond_t *cond_; +#endif + // Allocate memory to store the loop-filtered superblock index in each row. + int *cur_sb_col; + // The optimal sync_range for different resolution and platform should be + // determined by testing. Currently, it is chosen to be a power-of-2 number. + int sync_range; + int rows; + + // Row-based parallel loopfilter data + LFWorkerData *lfdata; + int num_workers; +} AV1LfSync; + +// Allocate memory for loopfilter row synchronization. +void av1_loop_filter_alloc(AV1LfSync *lf_sync, struct AV1Common *cm, int rows, + int width, int num_workers); + +// Deallocate loopfilter synchronization related mutex and data. +void av1_loop_filter_dealloc(AV1LfSync *lf_sync); + +// Multi-threaded loopfilter that uses the tile threads. +void av1_loop_filter_frame_mt(YV12_BUFFER_CONFIG *frame, struct AV1Common *cm, + struct macroblockd_plane planes[MAX_MB_PLANE], + int frame_filter_level, int y_only, + int partial_frame, AVxWorker *workers, + int num_workers, AV1LfSync *lf_sync); + +void av1_accumulate_frame_counts(struct FRAME_COUNTS *acc_counts, + struct FRAME_COUNTS *counts); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_LOOPFILTER_THREAD_H_ diff --git a/third_party/aom/av1/common/tile_common.c b/third_party/aom/av1/common/tile_common.c new file mode 100644 index 0000000000..b8008ac2e0 --- /dev/null +++ b/third_party/aom/av1/common/tile_common.c @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/common/tile_common.h" +#include "av1/common/onyxc_int.h" +#include "aom_dsp/aom_dsp_common.h" + +void av1_tile_set_row(TileInfo *tile, const AV1_COMMON *cm, int row) { + tile->mi_row_start = row * cm->tile_height; + tile->mi_row_end = AOMMIN(tile->mi_row_start + cm->tile_height, cm->mi_rows); +} + +void av1_tile_set_col(TileInfo *tile, const AV1_COMMON *cm, int col) { + tile->mi_col_start = col * cm->tile_width; + tile->mi_col_end = AOMMIN(tile->mi_col_start + cm->tile_width, cm->mi_cols); +} + +#if CONFIG_DEPENDENT_HORZTILES && CONFIG_TILE_GROUPS +void av1_tile_set_tg_boundary(TileInfo *tile, const AV1_COMMON *const cm, + int row, int col) { + if (row < cm->tile_rows - 1) { + tile->tg_horz_boundary = + col >= cm->tile_group_start_col[row][col] + ? (row == cm->tile_group_start_row[row][col] ? 1 : 0) + : (row == cm->tile_group_start_row[row + 1][col] ? 1 : 0); + } else { + assert(col >= cm->tile_group_start_col[row][col]); + tile->tg_horz_boundary = + (row == cm->tile_group_start_row[row][col] ? 1 : 0); + } +} +#endif +void av1_tile_init(TileInfo *tile, const AV1_COMMON *cm, int row, int col) { + av1_tile_set_row(tile, cm, row); + av1_tile_set_col(tile, cm, col); +#if CONFIG_DEPENDENT_HORZTILES && CONFIG_TILE_GROUPS + av1_tile_set_tg_boundary(tile, cm, row, col); +#endif +} + +#if !CONFIG_EXT_TILE + +#if CONFIG_EXT_PARTITION +#define MIN_TILE_WIDTH_MAX_SB 2 +#define MAX_TILE_WIDTH_MAX_SB 32 +#else +#define MIN_TILE_WIDTH_MAX_SB 4 +#define MAX_TILE_WIDTH_MAX_SB 64 +#endif // CONFIG_EXT_PARTITION + +static int get_min_log2_tile_cols(int max_sb_cols) { + int min_log2 = 0; + while ((MAX_TILE_WIDTH_MAX_SB << min_log2) < max_sb_cols) ++min_log2; + return min_log2; +} + +static int get_max_log2_tile_cols(int max_sb_cols) { + int max_log2 = 1; + while ((max_sb_cols >> max_log2) >= MIN_TILE_WIDTH_MAX_SB) ++max_log2; + return max_log2 - 1; +} + +void av1_get_tile_n_bits(int mi_cols, int *min_log2_tile_cols, + int *max_log2_tile_cols) { + const int max_sb_cols = + ALIGN_POWER_OF_TWO(mi_cols, MAX_MIB_SIZE_LOG2) >> MAX_MIB_SIZE_LOG2; + *min_log2_tile_cols = get_min_log2_tile_cols(max_sb_cols); + *max_log2_tile_cols = get_max_log2_tile_cols(max_sb_cols); + assert(*min_log2_tile_cols <= *max_log2_tile_cols); +} +#endif // !CONFIG_EXT_TILE + +void av1_update_boundary_info(const struct AV1Common *cm, + const TileInfo *const tile_info, int mi_row, + int mi_col) { + int row, col; + for (row = mi_row; row < (mi_row + cm->mib_size); row++) + for (col = mi_col; col < (mi_col + cm->mib_size); col++) { + MODE_INFO *const mi = cm->mi + row * cm->mi_stride + col; + mi->mbmi.boundary_info = 0; + if (cm->tile_cols * cm->tile_rows > 1) { +#if CONFIG_DEPENDENT_HORZTILES + if (row == tile_info->mi_row_start && + (!cm->dependent_horz_tiles || tile_info->tg_horz_boundary)) +#if CONFIG_TILE_GROUPS +#else + if (row == tile_info->mi_row_start && !cm->dependent_horz_tiles) +#endif // CONFIG_TILE_GROUPS +#else + if (row == tile_info->mi_row_start) +#endif // CONFIG_DEPENDENT_HORZTILES + mi->mbmi.boundary_info |= TILE_ABOVE_BOUNDARY; + if (col == tile_info->mi_col_start) + mi->mbmi.boundary_info |= TILE_LEFT_BOUNDARY; + if ((row + 1) >= tile_info->mi_row_end) + mi->mbmi.boundary_info |= TILE_BOTTOM_BOUNDARY; + if ((col + 1) >= tile_info->mi_col_end) + mi->mbmi.boundary_info |= TILE_RIGHT_BOUNDARY; + } + // Frame boundary is treated as tile boundary + if (row == 0) + mi->mbmi.boundary_info |= FRAME_ABOVE_BOUNDARY | TILE_ABOVE_BOUNDARY; + if (col == 0) + mi->mbmi.boundary_info |= FRAME_LEFT_BOUNDARY | TILE_LEFT_BOUNDARY; + if ((row + 1) >= cm->mi_rows) + mi->mbmi.boundary_info |= FRAME_BOTTOM_BOUNDARY | TILE_BOTTOM_BOUNDARY; + if ((col + 1) >= cm->mi_cols) + mi->mbmi.boundary_info |= FRAME_RIGHT_BOUNDARY | TILE_RIGHT_BOUNDARY; + } +} + +#if CONFIG_LOOPFILTERING_ACROSS_TILES +int av1_disable_loopfilter_on_tile_boundary(const struct AV1Common *cm) { + return (!cm->loop_filter_across_tiles_enabled && + (cm->tile_cols * cm->tile_rows > 1)); +} +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES diff --git a/third_party/aom/av1/common/tile_common.h b/third_party/aom/av1/common/tile_common.h new file mode 100644 index 0000000000..617dda2028 --- /dev/null +++ b/third_party/aom/av1/common/tile_common.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_TILE_COMMON_H_ +#define AV1_COMMON_TILE_COMMON_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./aom_config.h" + +struct AV1Common; + +#if CONFIG_TILE_GROUPS +#define DEFAULT_MAX_NUM_TG 1 +#endif + +typedef struct TileInfo { + int mi_row_start, mi_row_end; + int mi_col_start, mi_col_end; + int tg_horz_boundary; +} TileInfo; + +// initializes 'tile->mi_(row|col)_(start|end)' for (row, col) based on +// 'cm->log2_tile_(rows|cols)' & 'cm->mi_(rows|cols)' +void av1_tile_init(TileInfo *tile, const struct AV1Common *cm, int row, + int col); + +void av1_tile_set_row(TileInfo *tile, const struct AV1Common *cm, int row); +void av1_tile_set_col(TileInfo *tile, const struct AV1Common *cm, int col); +#if CONFIG_DEPENDENT_HORZTILES && CONFIG_TILE_GROUPS +void av1_tile_set_tg_boundary(TileInfo *tile, const struct AV1Common *cm, + int row, int col); +#endif +void av1_get_tile_n_bits(int mi_cols, int *min_log2_tile_cols, + int *max_log2_tile_cols); + +void av1_update_boundary_info(const struct AV1Common *cm, + const TileInfo *const tile_info, int mi_row, + int mi_col); + +#if CONFIG_LOOPFILTERING_ACROSS_TILES +int av1_disable_loopfilter_on_tile_boundary(const struct AV1Common *cm); +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_COMMON_TILE_COMMON_H_ diff --git a/third_party/aom/av1/common/txb_common.c b/third_party/aom/av1/common/txb_common.c new file mode 100644 index 0000000000..08a685b597 --- /dev/null +++ b/third_party/aom/av1/common/txb_common.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include "aom/aom_integer.h" +#include "av1/common/onyxc_int.h" + +const int16_t av1_coeff_band_4x4[16] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 }; + +const int16_t av1_coeff_band_8x8[64] = { + 0, 1, 2, 2, 3, 3, 4, 4, 5, 6, 2, 2, 3, 3, 4, 4, + 7, 7, 8, 8, 9, 9, 10, 10, 7, 7, 8, 8, 9, 9, 10, 10, + 11, 11, 12, 12, 13, 13, 14, 14, 11, 11, 12, 12, 13, 13, 14, 14, + 15, 15, 16, 16, 17, 17, 18, 18, 15, 15, 16, 16, 17, 17, 18, 18, +}; + +const int16_t av1_coeff_band_16x16[256] = { + 0, 1, 4, 4, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 2, 3, 4, + 4, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 5, 5, 6, 6, 7, 7, + 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 5, 5, 6, 6, 7, 7, 7, 7, 8, + 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, + 13, 13, 13, 13, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 13, + 13, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 13, 13, 10, 10, + 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, + 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17, 14, 14, 14, 14, 15, 15, 15, 15, + 16, 16, 16, 16, 17, 17, 17, 17, 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, + 16, 17, 17, 17, 17, 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, + 17, 17, 18, 18, 18, 18, 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 18, + 18, 18, 18, 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 18, 18, 18, 18, + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 18, 18, 18, 18, 19, 19, 19, + 19, 20, 20, 20, 20, 21, 21, 21, 21, +}; + +const int16_t av1_coeff_band_32x32[1024] = { + 0, 1, 4, 4, 7, 7, 7, 7, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, + 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 2, 3, 4, 4, 7, 7, + 7, 7, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, 12, + 12, 12, 12, 12, 12, 12, 12, 5, 5, 6, 6, 7, 7, 7, 7, 10, 10, 10, 10, + 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, + 12, 5, 5, 6, 6, 7, 7, 7, 7, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, + 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 8, 8, 8, 8, 9, + 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, + 12, 12, 12, 12, 12, 12, 12, 12, 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, + 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, + 12, 12, 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 11, + 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 8, 8, 8, 8, + 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, + 11, 12, 12, 12, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, + 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 16, 16, 16, 16, 16, + 16, 16, 16, 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, + 15, 15, 15, 15, 15, 15, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 13, 13, 13, + 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, + 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 13, 13, 13, 13, 13, 13, 13, 13, 14, + 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 16, 16, 16, 16, + 16, 16, 16, 16, 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, + 14, 15, 15, 15, 15, 15, 15, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 13, 13, + 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, + 15, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 13, 13, 13, 13, 13, 13, 13, 13, + 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 16, 16, 16, + 16, 16, 16, 16, 16, 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, + 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 17, + 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, + 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 17, 17, 17, 17, 17, 17, 17, + 17, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, + 20, 20, 20, 20, 20, 20, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, + 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, + 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, + 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 17, 17, 17, 17, 17, 17, + 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 20, + 20, 20, 20, 20, 20, 20, 20, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, + 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, + 20, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, + 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 17, 17, 17, 17, 17, + 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, + 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 22, 22, 22, + 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, + 24, 24, 21, 21, 21, 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, + 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 21, 21, 21, 21, + 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, + 23, 24, 24, 24, 24, 24, 24, 24, 24, 21, 21, 21, 21, 21, 21, 21, 21, 22, 22, + 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, + 24, 24, 24, 21, 21, 21, 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 21, 21, 21, + 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, + 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 21, 21, 21, 21, 21, 21, 21, 21, 22, + 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, + 24, 24, 24, 24, 21, 21, 21, 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, + 22, 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, +}; + +void av1_adapt_txb_probs(AV1_COMMON *cm, unsigned int count_sat, + unsigned int update_factor) { + FRAME_CONTEXT *fc = cm->fc; + const FRAME_CONTEXT *pre_fc = &cm->frame_contexts[cm->frame_context_idx]; + const FRAME_COUNTS *counts = &cm->counts; + TX_SIZE tx_size; + int plane, ctx, level; + + // Update probability models for transform block skip flag + for (tx_size = 0; tx_size < TX_SIZES; ++tx_size) + for (ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx) + fc->txb_skip[tx_size][ctx] = mode_mv_merge_probs( + pre_fc->txb_skip[tx_size][ctx], counts->txb_skip[tx_size][ctx]); + + for (plane = 0; plane < PLANE_TYPES; ++plane) + for (ctx = 0; ctx < DC_SIGN_CONTEXTS; ++ctx) + fc->dc_sign[plane][ctx] = mode_mv_merge_probs( + pre_fc->dc_sign[plane][ctx], counts->dc_sign[plane][ctx]); + // Update probability models for non-zero coefficient map and eob flag. + for (level = 0; level < NUM_BASE_LEVELS; ++level) + for (tx_size = 0; tx_size < TX_SIZES; ++tx_size) + for (plane = 0; plane < PLANE_TYPES; ++plane) + for (ctx = 0; ctx < COEFF_BASE_CONTEXTS; ++ctx) + fc->coeff_base[tx_size][plane][level][ctx] = + merge_probs(pre_fc->coeff_base[tx_size][plane][level][ctx], + counts->coeff_base[tx_size][plane][level][ctx], + count_sat, update_factor); + + for (tx_size = 0; tx_size < TX_SIZES; ++tx_size) { + for (plane = 0; plane < PLANE_TYPES; ++plane) { + for (ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx) { + fc->nz_map[tx_size][plane][ctx] = merge_probs( + pre_fc->nz_map[tx_size][plane][ctx], + counts->nz_map[tx_size][plane][ctx], count_sat, update_factor); + } + + for (ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx) { + fc->eob_flag[tx_size][plane][ctx] = merge_probs( + pre_fc->eob_flag[tx_size][plane][ctx], + counts->eob_flag[tx_size][plane][ctx], count_sat, update_factor); + } + } + } + + for (tx_size = 0; tx_size < TX_SIZES; ++tx_size) { + for (plane = 0; plane < PLANE_TYPES; ++plane) + for (ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx) + fc->coeff_lps[tx_size][plane][ctx] = merge_probs( + pre_fc->coeff_lps[tx_size][plane][ctx], + counts->coeff_lps[tx_size][plane][ctx], count_sat, update_factor); + } +} diff --git a/third_party/aom/av1/common/txb_common.h b/third_party/aom/av1/common/txb_common.h new file mode 100644 index 0000000000..cdd9ca26eb --- /dev/null +++ b/third_party/aom/av1/common/txb_common.h @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_TXB_COMMON_H_ +#define AV1_COMMON_TXB_COMMON_H_ +extern const int16_t av1_coeff_band_4x4[16]; + +extern const int16_t av1_coeff_band_8x8[64]; + +extern const int16_t av1_coeff_band_16x16[256]; + +extern const int16_t av1_coeff_band_32x32[1024]; + +typedef struct txb_ctx { + int txb_skip_ctx; + int dc_sign_ctx; +} TXB_CTX; + +#define BASE_CONTEXT_POSITION_NUM 12 +static int base_ref_offset[BASE_CONTEXT_POSITION_NUM][2] = { + /* clang-format off*/ + { -2, 0 }, { -1, -1 }, { -1, 0 }, { -1, 1 }, { 0, -2 }, { 0, -1 }, { 0, 1 }, + { 0, 2 }, { 1, -1 }, { 1, 0 }, { 1, 1 }, { 2, 0 } + /* clang-format on*/ +}; + +static INLINE int get_base_ctx(const tran_low_t *tcoeffs, + int c, // raster order + const int bwl, const int level) { + const int row = c >> bwl; + const int col = c - (row << bwl); + const int stride = 1 << bwl; + const int level_minus_1 = level - 1; + int ctx = 0; + int mag = 0; + int idx; + int ctx_idx = -1; + tran_low_t abs_coeff; + + ctx = 0; + for (idx = 0; idx < BASE_CONTEXT_POSITION_NUM; ++idx) { + int ref_row = row + base_ref_offset[idx][0]; + int ref_col = col + base_ref_offset[idx][1]; + int pos = (ref_row << bwl) + ref_col; + + if (ref_row < 0 || ref_col < 0 || ref_row >= stride || ref_col >= stride) + continue; + + abs_coeff = abs(tcoeffs[pos]); + ctx += abs_coeff > level_minus_1; + + if (base_ref_offset[idx][0] >= 0 && base_ref_offset[idx][1] >= 0) + mag |= abs_coeff > level; + } + ctx = (ctx + 1) >> 1; + if (row == 0 && col == 0) { + ctx_idx = (ctx << 1) + mag; + assert(ctx_idx < 8); + } else if (row == 0) { + ctx_idx = 8 + (ctx << 1) + mag; + assert(ctx_idx < 18); + } else if (col == 0) { + ctx_idx = 8 + 10 + (ctx << 1) + mag; + assert(ctx_idx < 28); + } else { + ctx_idx = 8 + 10 + 10 + (ctx << 1) + mag; + assert(ctx_idx < COEFF_BASE_CONTEXTS); + } + return ctx_idx; +} + +#define BR_CONTEXT_POSITION_NUM 8 // Base range coefficient context +static int br_ref_offset[BR_CONTEXT_POSITION_NUM][2] = { + /* clang-format off*/ + { -1, -1 }, { -1, 0 }, { -1, 1 }, { 0, -1 }, + { 0, 1 }, { 1, -1 }, { 1, 0 }, { 1, 1 }, + /* clang-format on*/ +}; + +static int br_level_map[9] = { + 0, 0, 1, 1, 2, 2, 3, 3, 3, +}; + +static INLINE int get_level_ctx(const tran_low_t *tcoeffs, + const int c, // raster order + const int bwl) { + const int row = c >> bwl; + const int col = c - (row << bwl); + const int stride = 1 << bwl; + const int level_minus_1 = NUM_BASE_LEVELS; + int ctx = 0; + int idx; + tran_low_t abs_coeff; + int mag = 0, offset = 0; + + for (idx = 0; idx < BR_CONTEXT_POSITION_NUM; ++idx) { + int ref_row = row + br_ref_offset[idx][0]; + int ref_col = col + br_ref_offset[idx][1]; + int pos = (ref_row << bwl) + ref_col; + + if (ref_row < 0 || ref_col < 0 || ref_row >= stride || ref_col >= stride) + continue; + + abs_coeff = abs(tcoeffs[pos]); + ctx += abs_coeff > level_minus_1; + + if (br_ref_offset[idx][0] >= 0 && br_ref_offset[idx][1] >= 0) + mag = AOMMAX(mag, abs_coeff); + } + + if (mag <= 1) + offset = 0; + else if (mag <= 3) + offset = 1; + else if (mag <= 6) + offset = 2; + else + offset = 3; + + ctx = br_level_map[ctx]; + + ctx += offset * BR_TMP_OFFSET; + + // DC: 0 - 1 + if (row == 0 && col == 0) return ctx; + + // Top row: 2 - 4 + if (row == 0) return 2 + ctx; + + // Left column: 5 - 7 + if (col == 0) return 5 + ctx; + + // others: 8 - 11 + return 8 + ctx; +} + +static int sig_ref_offset[11][2] = { + { -2, -1 }, { -2, 0 }, { -2, 1 }, { -1, -2 }, { -1, -1 }, { -1, 0 }, + { -1, 1 }, { 0, -2 }, { 0, -1 }, { 1, -2 }, { 1, -1 }, +}; + +static INLINE int get_nz_map_ctx(const tran_low_t *tcoeffs, + const uint8_t *txb_mask, + const int c, // raster order + const int bwl) { + const int row = c >> bwl; + const int col = c - (row << bwl); + int ctx = 0; + int idx; + int stride = 1 << bwl; + + if (row == 0 && col == 0) return 0; + + if (row == 0 && col == 1) return 1 + (tcoeffs[0] != 0); + + if (row == 1 && col == 0) return 3 + (tcoeffs[0] != 0); + + if (row == 1 && col == 1) { + int pos; + ctx = (tcoeffs[0] != 0); + + if (txb_mask[1]) ctx += (tcoeffs[1] != 0); + pos = 1 << bwl; + if (txb_mask[pos]) ctx += (tcoeffs[pos] != 0); + + ctx = (ctx + 1) >> 1; + + assert(5 + ctx <= 7); + + return 5 + ctx; + } + + for (idx = 0; idx < 11; ++idx) { + int ref_row = row + sig_ref_offset[idx][0]; + int ref_col = col + sig_ref_offset[idx][1]; + int pos; + + if (ref_row < 0 || ref_col < 0 || ref_row >= stride || ref_col >= stride) + continue; + + pos = (ref_row << bwl) + ref_col; + + if (txb_mask[pos]) ctx += (tcoeffs[pos] != 0); + } + + if (row == 0) { + ctx = (ctx + 1) >> 1; + + assert(ctx < 3); + return 8 + ctx; + } + + if (col == 0) { + ctx = (ctx + 1) >> 1; + + assert(ctx < 3); + return 11 + ctx; + } + + ctx >>= 1; + + assert(14 + ctx < 20); + + return 14 + ctx; +} + +static INLINE int get_eob_ctx(const tran_low_t *tcoeffs, + const int c, // raster order + const int bwl) { + (void)tcoeffs; + if (bwl == 2) return av1_coeff_band_4x4[c]; + if (bwl == 3) return av1_coeff_band_8x8[c]; + if (bwl == 4) return av1_coeff_band_16x16[c]; + if (bwl == 5) return av1_coeff_band_32x32[c]; + + assert(0); + return 0; +} + +static INLINE void set_dc_sign(int *cul_level, tran_low_t v) { + if (v < 0) + *cul_level |= 1 << COEFF_CONTEXT_BITS; + else if (v > 0) + *cul_level += 2 << COEFF_CONTEXT_BITS; +} + +static INLINE int get_dc_sign_ctx(int dc_sign) { + int dc_sign_ctx = 0; + if (dc_sign < 0) + dc_sign_ctx = 1; + else if (dc_sign > 0) + dc_sign_ctx = 2; + + return dc_sign_ctx; +} + +static INLINE void get_txb_ctx(BLOCK_SIZE plane_bsize, TX_SIZE tx_size, + int plane, const ENTROPY_CONTEXT *a, + const ENTROPY_CONTEXT *l, TXB_CTX *txb_ctx) { + const int tx_size_in_blocks = 1 << tx_size; + int ctx_offset = (plane == 0) ? 0 : 7; + int k; + + if (plane_bsize > txsize_to_bsize[tx_size]) ctx_offset += 3; + + int dc_sign = 0; + for (k = 0; k < tx_size_in_blocks; ++k) { + int sign = ((uint8_t)a[k]) >> COEFF_CONTEXT_BITS; + if (sign == 1) + --dc_sign; + else if (sign == 2) + ++dc_sign; + else if (sign != 0) + assert(0); + + sign = ((uint8_t)l[k]) >> COEFF_CONTEXT_BITS; + if (sign == 1) + --dc_sign; + else if (sign == 2) + ++dc_sign; + else if (sign != 0) + assert(0); + } + txb_ctx->dc_sign_ctx = get_dc_sign_ctx(dc_sign); + + if (plane == 0) { + int top = 0; + int left = 0; + for (k = 0; k < tx_size_in_blocks; ++k) { + top = AOMMAX(top, ((uint8_t)a[k] & COEFF_CONTEXT_MASK)); + left = AOMMAX(left, ((uint8_t)l[k] & COEFF_CONTEXT_MASK)); + } + top = AOMMIN(top, 255); + left = AOMMIN(left, 255); + + if (plane_bsize == txsize_to_bsize[tx_size]) + txb_ctx->txb_skip_ctx = 0; + else if (top == 0 && left == 0) + txb_ctx->txb_skip_ctx = 1; + else if (top == 0 || left == 0) + txb_ctx->txb_skip_ctx = 2 + (AOMMAX(top, left) > 3); + else if (AOMMAX(top, left) <= 3) + txb_ctx->txb_skip_ctx = 4; + else if (AOMMIN(top, left) <= 3) + txb_ctx->txb_skip_ctx = 5; + else + txb_ctx->txb_skip_ctx = 6; + } else { + int ctx_base = get_entropy_context(tx_size, a, l); + txb_ctx->txb_skip_ctx = ctx_offset + ctx_base; + } +} + +void av1_adapt_txb_probs(AV1_COMMON *cm, unsigned int count_sat, + unsigned int update_factor); +#endif // AV1_COMMON_TXB_COMMON_H_ diff --git a/third_party/aom/av1/common/warped_motion.c b/third_party/aom/av1/common/warped_motion.c new file mode 100644 index 0000000000..9d13dc705a --- /dev/null +++ b/third_party/aom/av1/common/warped_motion.c @@ -0,0 +1,1773 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include +#include + +#include "./av1_rtcd.h" +#include "av1/common/warped_motion.h" + +/* clang-format off */ +static const int error_measure_lut[512] = { + // pow 0.7 + 16384, 16339, 16294, 16249, 16204, 16158, 16113, 16068, + 16022, 15977, 15932, 15886, 15840, 15795, 15749, 15703, + 15657, 15612, 15566, 15520, 15474, 15427, 15381, 15335, + 15289, 15242, 15196, 15149, 15103, 15056, 15010, 14963, + 14916, 14869, 14822, 14775, 14728, 14681, 14634, 14587, + 14539, 14492, 14445, 14397, 14350, 14302, 14254, 14206, + 14159, 14111, 14063, 14015, 13967, 13918, 13870, 13822, + 13773, 13725, 13676, 13628, 13579, 13530, 13481, 13432, + 13383, 13334, 13285, 13236, 13187, 13137, 13088, 13038, + 12988, 12939, 12889, 12839, 12789, 12739, 12689, 12639, + 12588, 12538, 12487, 12437, 12386, 12335, 12285, 12234, + 12183, 12132, 12080, 12029, 11978, 11926, 11875, 11823, + 11771, 11719, 11667, 11615, 11563, 11511, 11458, 11406, + 11353, 11301, 11248, 11195, 11142, 11089, 11036, 10982, + 10929, 10875, 10822, 10768, 10714, 10660, 10606, 10552, + 10497, 10443, 10388, 10333, 10279, 10224, 10168, 10113, + 10058, 10002, 9947, 9891, 9835, 9779, 9723, 9666, + 9610, 9553, 9497, 9440, 9383, 9326, 9268, 9211, + 9153, 9095, 9037, 8979, 8921, 8862, 8804, 8745, + 8686, 8627, 8568, 8508, 8449, 8389, 8329, 8269, + 8208, 8148, 8087, 8026, 7965, 7903, 7842, 7780, + 7718, 7656, 7593, 7531, 7468, 7405, 7341, 7278, + 7214, 7150, 7086, 7021, 6956, 6891, 6826, 6760, + 6695, 6628, 6562, 6495, 6428, 6361, 6293, 6225, + 6157, 6089, 6020, 5950, 5881, 5811, 5741, 5670, + 5599, 5527, 5456, 5383, 5311, 5237, 5164, 5090, + 5015, 4941, 4865, 4789, 4713, 4636, 4558, 4480, + 4401, 4322, 4242, 4162, 4080, 3998, 3916, 3832, + 3748, 3663, 3577, 3490, 3402, 3314, 3224, 3133, + 3041, 2948, 2854, 2758, 2661, 2562, 2461, 2359, + 2255, 2148, 2040, 1929, 1815, 1698, 1577, 1452, + 1323, 1187, 1045, 894, 731, 550, 339, 0, + 339, 550, 731, 894, 1045, 1187, 1323, 1452, + 1577, 1698, 1815, 1929, 2040, 2148, 2255, 2359, + 2461, 2562, 2661, 2758, 2854, 2948, 3041, 3133, + 3224, 3314, 3402, 3490, 3577, 3663, 3748, 3832, + 3916, 3998, 4080, 4162, 4242, 4322, 4401, 4480, + 4558, 4636, 4713, 4789, 4865, 4941, 5015, 5090, + 5164, 5237, 5311, 5383, 5456, 5527, 5599, 5670, + 5741, 5811, 5881, 5950, 6020, 6089, 6157, 6225, + 6293, 6361, 6428, 6495, 6562, 6628, 6695, 6760, + 6826, 6891, 6956, 7021, 7086, 7150, 7214, 7278, + 7341, 7405, 7468, 7531, 7593, 7656, 7718, 7780, + 7842, 7903, 7965, 8026, 8087, 8148, 8208, 8269, + 8329, 8389, 8449, 8508, 8568, 8627, 8686, 8745, + 8804, 8862, 8921, 8979, 9037, 9095, 9153, 9211, + 9268, 9326, 9383, 9440, 9497, 9553, 9610, 9666, + 9723, 9779, 9835, 9891, 9947, 10002, 10058, 10113, + 10168, 10224, 10279, 10333, 10388, 10443, 10497, 10552, + 10606, 10660, 10714, 10768, 10822, 10875, 10929, 10982, + 11036, 11089, 11142, 11195, 11248, 11301, 11353, 11406, + 11458, 11511, 11563, 11615, 11667, 11719, 11771, 11823, + 11875, 11926, 11978, 12029, 12080, 12132, 12183, 12234, + 12285, 12335, 12386, 12437, 12487, 12538, 12588, 12639, + 12689, 12739, 12789, 12839, 12889, 12939, 12988, 13038, + 13088, 13137, 13187, 13236, 13285, 13334, 13383, 13432, + 13481, 13530, 13579, 13628, 13676, 13725, 13773, 13822, + 13870, 13918, 13967, 14015, 14063, 14111, 14159, 14206, + 14254, 14302, 14350, 14397, 14445, 14492, 14539, 14587, + 14634, 14681, 14728, 14775, 14822, 14869, 14916, 14963, + 15010, 15056, 15103, 15149, 15196, 15242, 15289, 15335, + 15381, 15427, 15474, 15520, 15566, 15612, 15657, 15703, + 15749, 15795, 15840, 15886, 15932, 15977, 16022, 16068, + 16113, 16158, 16204, 16249, 16294, 16339, 16384, 16384, +}; +/* clang-format on */ + +static ProjectPointsFunc get_project_points_type(TransformationType type) { + switch (type) { + case HOMOGRAPHY: return project_points_homography; + case AFFINE: return project_points_affine; + case ROTZOOM: return project_points_rotzoom; + case TRANSLATION: return project_points_translation; + default: assert(0); return NULL; + } +} + +void project_points_translation(int32_t *mat, int *points, int *proj, + const int n, const int stride_points, + const int stride_proj, const int subsampling_x, + const int subsampling_y) { + int i; + for (i = 0; i < n; ++i) { + const int x = *(points++), y = *(points++); + if (subsampling_x) + *(proj++) = ROUND_POWER_OF_TWO_SIGNED( + ((x * (1 << (WARPEDMODEL_PREC_BITS + 1))) + mat[0]), + WARPEDDIFF_PREC_BITS + 1); + else + *(proj++) = ROUND_POWER_OF_TWO_SIGNED( + ((x * (1 << WARPEDMODEL_PREC_BITS)) + mat[0]), WARPEDDIFF_PREC_BITS); + if (subsampling_y) + *(proj++) = ROUND_POWER_OF_TWO_SIGNED( + ((y * (1 << (WARPEDMODEL_PREC_BITS + 1))) + mat[1]), + WARPEDDIFF_PREC_BITS + 1); + else + *(proj++) = ROUND_POWER_OF_TWO_SIGNED( + ((y * (1 << WARPEDMODEL_PREC_BITS))) + mat[1], WARPEDDIFF_PREC_BITS); + points += stride_points - 2; + proj += stride_proj - 2; + } +} + +void project_points_rotzoom(int32_t *mat, int *points, int *proj, const int n, + const int stride_points, const int stride_proj, + const int subsampling_x, const int subsampling_y) { + int i; + for (i = 0; i < n; ++i) { + const int x = *(points++), y = *(points++); + if (subsampling_x) + *(proj++) = ROUND_POWER_OF_TWO_SIGNED( + mat[2] * 2 * x + mat[3] * 2 * y + mat[0] + + (mat[2] + mat[3] - (1 << WARPEDMODEL_PREC_BITS)) / 2, + WARPEDDIFF_PREC_BITS + 1); + else + *(proj++) = ROUND_POWER_OF_TWO_SIGNED(mat[2] * x + mat[3] * y + mat[0], + WARPEDDIFF_PREC_BITS); + if (subsampling_y) + *(proj++) = ROUND_POWER_OF_TWO_SIGNED( + -mat[3] * 2 * x + mat[2] * 2 * y + mat[1] + + (-mat[3] + mat[2] - (1 << WARPEDMODEL_PREC_BITS)) / 2, + WARPEDDIFF_PREC_BITS + 1); + else + *(proj++) = ROUND_POWER_OF_TWO_SIGNED(-mat[3] * x + mat[2] * y + mat[1], + WARPEDDIFF_PREC_BITS); + points += stride_points - 2; + proj += stride_proj - 2; + } +} + +void project_points_affine(int32_t *mat, int *points, int *proj, const int n, + const int stride_points, const int stride_proj, + const int subsampling_x, const int subsampling_y) { + int i; + for (i = 0; i < n; ++i) { + const int x = *(points++), y = *(points++); + if (subsampling_x) + *(proj++) = ROUND_POWER_OF_TWO_SIGNED( + mat[2] * 2 * x + mat[3] * 2 * y + mat[0] + + (mat[2] + mat[3] - (1 << WARPEDMODEL_PREC_BITS)) / 2, + WARPEDDIFF_PREC_BITS + 1); + else + *(proj++) = ROUND_POWER_OF_TWO_SIGNED(mat[2] * x + mat[3] * y + mat[0], + WARPEDDIFF_PREC_BITS); + if (subsampling_y) + *(proj++) = ROUND_POWER_OF_TWO_SIGNED( + mat[4] * 2 * x + mat[5] * 2 * y + mat[1] + + (mat[4] + mat[5] - (1 << WARPEDMODEL_PREC_BITS)) / 2, + WARPEDDIFF_PREC_BITS + 1); + else + *(proj++) = ROUND_POWER_OF_TWO_SIGNED(mat[4] * x + mat[5] * y + mat[1], + WARPEDDIFF_PREC_BITS); + points += stride_points - 2; + proj += stride_proj - 2; + } +} + +void project_points_hortrapezoid(int32_t *mat, int *points, int *proj, + const int n, const int stride_points, + const int stride_proj, const int subsampling_x, + const int subsampling_y) { + int i; + int64_t x, y, Z; + int64_t xp, yp; + for (i = 0; i < n; ++i) { + x = *(points++), y = *(points++); + x = (subsampling_x ? 4 * x + 1 : 2 * x); + y = (subsampling_y ? 4 * y + 1 : 2 * y); + + Z = (mat[7] * y + (1 << (WARPEDMODEL_ROW3HOMO_PREC_BITS + 1))); + xp = (mat[2] * x + mat[3] * y + 2 * mat[0]) * + (1 << (WARPEDPIXEL_PREC_BITS + WARPEDMODEL_ROW3HOMO_PREC_BITS - + WARPEDMODEL_PREC_BITS)); + yp = (mat[5] * y + 2 * mat[1]) * + (1 << (WARPEDPIXEL_PREC_BITS + WARPEDMODEL_ROW3HOMO_PREC_BITS - + WARPEDMODEL_PREC_BITS)); + + xp = xp > 0 ? (xp + Z / 2) / Z : (xp - Z / 2) / Z; + yp = yp > 0 ? (yp + Z / 2) / Z : (yp - Z / 2) / Z; + + if (subsampling_x) xp = (xp - (1 << (WARPEDPIXEL_PREC_BITS - 1))) / 2; + if (subsampling_y) yp = (yp - (1 << (WARPEDPIXEL_PREC_BITS - 1))) / 2; + *(proj++) = (int)xp; + *(proj++) = (int)yp; + + points += stride_points - 2; + proj += stride_proj - 2; + } +} + +void project_points_vertrapezoid(int32_t *mat, int *points, int *proj, + const int n, const int stride_points, + const int stride_proj, const int subsampling_x, + const int subsampling_y) { + int i; + int64_t x, y, Z; + int64_t xp, yp; + for (i = 0; i < n; ++i) { + x = *(points++), y = *(points++); + x = (subsampling_x ? 4 * x + 1 : 2 * x); + y = (subsampling_y ? 4 * y + 1 : 2 * y); + + Z = (mat[6] * x + (1 << (WARPEDMODEL_ROW3HOMO_PREC_BITS + 1))); + xp = (mat[2] * x + 2 * mat[0]) * + (1 << (WARPEDPIXEL_PREC_BITS + WARPEDMODEL_ROW3HOMO_PREC_BITS - + WARPEDMODEL_PREC_BITS)); + yp = (mat[4] * x + mat[5] * y + 2 * mat[1]) * + (1 << (WARPEDPIXEL_PREC_BITS + WARPEDMODEL_ROW3HOMO_PREC_BITS - + WARPEDMODEL_PREC_BITS)); + + xp = xp > 0 ? (xp + Z / 2) / Z : (xp - Z / 2) / Z; + yp = yp > 0 ? (yp + Z / 2) / Z : (yp - Z / 2) / Z; + + if (subsampling_x) xp = (xp - (1 << (WARPEDPIXEL_PREC_BITS - 1))) / 2; + if (subsampling_y) yp = (yp - (1 << (WARPEDPIXEL_PREC_BITS - 1))) / 2; + *(proj++) = (int)xp; + *(proj++) = (int)yp; + + points += stride_points - 2; + proj += stride_proj - 2; + } +} + +void project_points_homography(int32_t *mat, int *points, int *proj, + const int n, const int stride_points, + const int stride_proj, const int subsampling_x, + const int subsampling_y) { + int i; + int64_t x, y, Z; + int64_t xp, yp; + for (i = 0; i < n; ++i) { + x = *(points++), y = *(points++); + x = (subsampling_x ? 4 * x + 1 : 2 * x); + y = (subsampling_y ? 4 * y + 1 : 2 * y); + + Z = (mat[6] * x + mat[7] * y + (1 << (WARPEDMODEL_ROW3HOMO_PREC_BITS + 1))); + xp = (mat[2] * x + mat[3] * y + 2 * mat[0]) * + (1 << (WARPEDPIXEL_PREC_BITS + WARPEDMODEL_ROW3HOMO_PREC_BITS - + WARPEDMODEL_PREC_BITS)); + yp = (mat[4] * x + mat[5] * y + 2 * mat[1]) * + (1 << (WARPEDPIXEL_PREC_BITS + WARPEDMODEL_ROW3HOMO_PREC_BITS - + WARPEDMODEL_PREC_BITS)); + + xp = xp > 0 ? (xp + Z / 2) / Z : (xp - Z / 2) / Z; + yp = yp > 0 ? (yp + Z / 2) / Z : (yp - Z / 2) / Z; + + if (subsampling_x) xp = (xp - (1 << (WARPEDPIXEL_PREC_BITS - 1))) / 2; + if (subsampling_y) yp = (yp - (1 << (WARPEDPIXEL_PREC_BITS - 1))) / 2; + *(proj++) = (int)xp; + *(proj++) = (int)yp; + + points += stride_points - 2; + proj += stride_proj - 2; + } +} + +// 'points' are at original scale, output 'proj's are scaled up by +// 1 << WARPEDPIXEL_PREC_BITS +void project_points(WarpedMotionParams *wm_params, int *points, int *proj, + const int n, const int stride_points, const int stride_proj, + const int subsampling_x, const int subsampling_y) { + switch (wm_params->wmtype) { + case AFFINE: + project_points_affine(wm_params->wmmat, points, proj, n, stride_points, + stride_proj, subsampling_x, subsampling_y); + break; + case ROTZOOM: + project_points_rotzoom(wm_params->wmmat, points, proj, n, stride_points, + stride_proj, subsampling_x, subsampling_y); + break; + case HOMOGRAPHY: + project_points_homography(wm_params->wmmat, points, proj, n, + stride_points, stride_proj, subsampling_x, + subsampling_y); + break; + default: assert(0 && "Invalid warped motion type!"); return; + } +} + +static const int16_t + filter_ntap[WARPEDPIXEL_PREC_SHIFTS][WARPEDPIXEL_FILTER_TAPS] = { +#if WARPEDPIXEL_PREC_BITS == 6 + { 0, 0, 128, 0, 0, 0 }, { 0, -1, 128, 2, -1, 0 }, + { 1, -3, 127, 4, -1, 0 }, { 1, -4, 126, 6, -2, 1 }, + { 1, -5, 126, 8, -3, 1 }, { 1, -6, 125, 11, -4, 1 }, + { 1, -7, 124, 13, -4, 1 }, { 2, -8, 123, 15, -5, 1 }, + { 2, -9, 122, 18, -6, 1 }, { 2, -10, 121, 20, -6, 1 }, + { 2, -11, 120, 22, -7, 2 }, { 2, -12, 119, 25, -8, 2 }, + { 3, -13, 117, 27, -8, 2 }, { 3, -13, 116, 29, -9, 2 }, + { 3, -14, 114, 32, -10, 3 }, { 3, -15, 113, 35, -10, 2 }, + { 3, -15, 111, 37, -11, 3 }, { 3, -16, 109, 40, -11, 3 }, + { 3, -16, 108, 42, -12, 3 }, { 4, -17, 106, 45, -13, 3 }, + { 4, -17, 104, 47, -13, 3 }, { 4, -17, 102, 50, -14, 3 }, + { 4, -17, 100, 52, -14, 3 }, { 4, -18, 98, 55, -15, 4 }, + { 4, -18, 96, 58, -15, 3 }, { 4, -18, 94, 60, -16, 4 }, + { 4, -18, 91, 63, -16, 4 }, { 4, -18, 89, 65, -16, 4 }, + { 4, -18, 87, 68, -17, 4 }, { 4, -18, 85, 70, -17, 4 }, + { 4, -18, 82, 73, -17, 4 }, { 4, -18, 80, 75, -17, 4 }, + { 4, -18, 78, 78, -18, 4 }, { 4, -17, 75, 80, -18, 4 }, + { 4, -17, 73, 82, -18, 4 }, { 4, -17, 70, 85, -18, 4 }, + { 4, -17, 68, 87, -18, 4 }, { 4, -16, 65, 89, -18, 4 }, + { 4, -16, 63, 91, -18, 4 }, { 4, -16, 60, 94, -18, 4 }, + { 3, -15, 58, 96, -18, 4 }, { 4, -15, 55, 98, -18, 4 }, + { 3, -14, 52, 100, -17, 4 }, { 3, -14, 50, 102, -17, 4 }, + { 3, -13, 47, 104, -17, 4 }, { 3, -13, 45, 106, -17, 4 }, + { 3, -12, 42, 108, -16, 3 }, { 3, -11, 40, 109, -16, 3 }, + { 3, -11, 37, 111, -15, 3 }, { 2, -10, 35, 113, -15, 3 }, + { 3, -10, 32, 114, -14, 3 }, { 2, -9, 29, 116, -13, 3 }, + { 2, -8, 27, 117, -13, 3 }, { 2, -8, 25, 119, -12, 2 }, + { 2, -7, 22, 120, -11, 2 }, { 1, -6, 20, 121, -10, 2 }, + { 1, -6, 18, 122, -9, 2 }, { 1, -5, 15, 123, -8, 2 }, + { 1, -4, 13, 124, -7, 1 }, { 1, -4, 11, 125, -6, 1 }, + { 1, -3, 8, 126, -5, 1 }, { 1, -2, 6, 126, -4, 1 }, + { 0, -1, 4, 127, -3, 1 }, { 0, -1, 2, 128, -1, 0 }, +#elif WARPEDPIXEL_PREC_BITS == 5 + { 0, 0, 128, 0, 0, 0 }, { 1, -3, 127, 4, -1, 0 }, + { 1, -5, 126, 8, -3, 1 }, { 1, -7, 124, 13, -4, 1 }, + { 2, -9, 122, 18, -6, 1 }, { 2, -11, 120, 22, -7, 2 }, + { 3, -13, 117, 27, -8, 2 }, { 3, -14, 114, 32, -10, 3 }, + { 3, -15, 111, 37, -11, 3 }, { 3, -16, 108, 42, -12, 3 }, + { 4, -17, 104, 47, -13, 3 }, { 4, -17, 100, 52, -14, 3 }, + { 4, -18, 96, 58, -15, 3 }, { 4, -18, 91, 63, -16, 4 }, + { 4, -18, 87, 68, -17, 4 }, { 4, -18, 82, 73, -17, 4 }, + { 4, -18, 78, 78, -18, 4 }, { 4, -17, 73, 82, -18, 4 }, + { 4, -17, 68, 87, -18, 4 }, { 4, -16, 63, 91, -18, 4 }, + { 3, -15, 58, 96, -18, 4 }, { 3, -14, 52, 100, -17, 4 }, + { 3, -13, 47, 104, -17, 4 }, { 3, -12, 42, 108, -16, 3 }, + { 3, -11, 37, 111, -15, 3 }, { 3, -10, 32, 114, -14, 3 }, + { 2, -8, 27, 117, -13, 3 }, { 2, -7, 22, 120, -11, 2 }, + { 1, -6, 18, 122, -9, 2 }, { 1, -4, 13, 124, -7, 1 }, + { 1, -3, 8, 126, -5, 1 }, { 0, -1, 4, 127, -3, 1 }, +#endif // WARPEDPIXEL_PREC_BITS == 6 + }; + +static int32_t do_ntap_filter(int32_t *p, int x) { + int i; + int32_t sum = 0; + for (i = 0; i < WARPEDPIXEL_FILTER_TAPS; ++i) { + sum += p[i - WARPEDPIXEL_FILTER_TAPS / 2 + 1] * filter_ntap[x][i]; + } + return sum; +} + +static int32_t do_cubic_filter(int32_t *p, int x) { + if (x == 0) { + return p[0] * (1 << WARPEDPIXEL_FILTER_BITS); + } else if (x == (1 << WARPEDPIXEL_PREC_BITS)) { + return p[1] * (1 << WARPEDPIXEL_FILTER_BITS); + } else { + const int64_t v1 = (int64_t)x * x * x * (3 * (p[0] - p[1]) + p[2] - p[-1]); + const int64_t v2 = + (int64_t)x * x * (2 * p[-1] - 5 * p[0] + 4 * p[1] - p[2]); + const int64_t v3 = x * (p[1] - p[-1]); + const int64_t v4 = 2 * p[0]; + return (int32_t)ROUND_POWER_OF_TWO_SIGNED( + (v4 * (1 << (3 * WARPEDPIXEL_PREC_BITS))) + + (v3 * (1 << (2 * WARPEDPIXEL_PREC_BITS))) + + (v2 * (1 << WARPEDPIXEL_PREC_BITS)) + v1, + 3 * WARPEDPIXEL_PREC_BITS + 1 - WARPEDPIXEL_FILTER_BITS); + } +} + +static INLINE void get_subcolumn(int taps, uint8_t *ref, int32_t *col, + int stride, int x, int y_start) { + int i; + for (i = 0; i < taps; ++i) { + col[i] = ref[(i + y_start) * stride + x]; + } +} + +static uint8_t bi_ntap_filter(uint8_t *ref, int x, int y, int stride) { + int32_t val, arr[WARPEDPIXEL_FILTER_TAPS]; + int k; + int i = (int)x >> WARPEDPIXEL_PREC_BITS; + int j = (int)y >> WARPEDPIXEL_PREC_BITS; + for (k = 0; k < WARPEDPIXEL_FILTER_TAPS; ++k) { + int32_t arr_temp[WARPEDPIXEL_FILTER_TAPS]; + get_subcolumn(WARPEDPIXEL_FILTER_TAPS, ref, arr_temp, stride, + i + k + 1 - WARPEDPIXEL_FILTER_TAPS / 2, + j + 1 - WARPEDPIXEL_FILTER_TAPS / 2); + arr[k] = do_ntap_filter(arr_temp + WARPEDPIXEL_FILTER_TAPS / 2 - 1, + y - (j * (1 << WARPEDPIXEL_PREC_BITS))); + } + val = do_ntap_filter(arr + WARPEDPIXEL_FILTER_TAPS / 2 - 1, + x - (i * (1 << WARPEDPIXEL_PREC_BITS))); + val = ROUND_POWER_OF_TWO_SIGNED(val, WARPEDPIXEL_FILTER_BITS * 2); + return (uint8_t)clip_pixel(val); +} + +static uint8_t bi_cubic_filter(uint8_t *ref, int x, int y, int stride) { + int32_t val, arr[4]; + int k; + int i = (int)x >> WARPEDPIXEL_PREC_BITS; + int j = (int)y >> WARPEDPIXEL_PREC_BITS; + for (k = 0; k < 4; ++k) { + int32_t arr_temp[4]; + get_subcolumn(4, ref, arr_temp, stride, i + k - 1, j - 1); + arr[k] = + do_cubic_filter(arr_temp + 1, y - (j * (1 << WARPEDPIXEL_PREC_BITS))); + } + val = do_cubic_filter(arr + 1, x - (i * (1 << WARPEDPIXEL_PREC_BITS))); + val = ROUND_POWER_OF_TWO_SIGNED(val, WARPEDPIXEL_FILTER_BITS * 2); + return (uint8_t)clip_pixel(val); +} + +static uint8_t bi_linear_filter(uint8_t *ref, int x, int y, int stride) { + const int ix = x >> WARPEDPIXEL_PREC_BITS; + const int iy = y >> WARPEDPIXEL_PREC_BITS; + const int sx = x - (ix * (1 << WARPEDPIXEL_PREC_BITS)); + const int sy = y - (iy * (1 << WARPEDPIXEL_PREC_BITS)); + int32_t val; + val = ROUND_POWER_OF_TWO_SIGNED( + ref[iy * stride + ix] * (WARPEDPIXEL_PREC_SHIFTS - sy) * + (WARPEDPIXEL_PREC_SHIFTS - sx) + + ref[iy * stride + ix + 1] * (WARPEDPIXEL_PREC_SHIFTS - sy) * sx + + ref[(iy + 1) * stride + ix] * sy * (WARPEDPIXEL_PREC_SHIFTS - sx) + + ref[(iy + 1) * stride + ix + 1] * sy * sx, + WARPEDPIXEL_PREC_BITS * 2); + return (uint8_t)clip_pixel(val); +} + +static uint8_t warp_interpolate(uint8_t *ref, int x, int y, int width, + int height, int stride) { + int ix = x >> WARPEDPIXEL_PREC_BITS; + int iy = y >> WARPEDPIXEL_PREC_BITS; + int sx = x - (ix * (1 << WARPEDPIXEL_PREC_BITS)); + int sy = y - (iy * (1 << WARPEDPIXEL_PREC_BITS)); + int32_t v; + + if (ix < 0 && iy < 0) + return ref[0]; + else if (ix < 0 && iy >= height - 1) + return ref[(height - 1) * stride]; + else if (ix >= width - 1 && iy < 0) + return ref[width - 1]; + else if (ix >= width - 1 && iy >= height - 1) + return ref[(height - 1) * stride + (width - 1)]; + else if (ix < 0) { + v = ROUND_POWER_OF_TWO_SIGNED( + ref[iy * stride] * (WARPEDPIXEL_PREC_SHIFTS - sy) + + ref[(iy + 1) * stride] * sy, + WARPEDPIXEL_PREC_BITS); + return clip_pixel(v); + } else if (iy < 0) { + v = ROUND_POWER_OF_TWO_SIGNED( + ref[ix] * (WARPEDPIXEL_PREC_SHIFTS - sx) + ref[ix + 1] * sx, + WARPEDPIXEL_PREC_BITS); + return clip_pixel(v); + } else if (ix >= width - 1) { + v = ROUND_POWER_OF_TWO_SIGNED( + ref[iy * stride + width - 1] * (WARPEDPIXEL_PREC_SHIFTS - sy) + + ref[(iy + 1) * stride + width - 1] * sy, + WARPEDPIXEL_PREC_BITS); + return clip_pixel(v); + } else if (iy >= height - 1) { + v = ROUND_POWER_OF_TWO_SIGNED( + ref[(height - 1) * stride + ix] * (WARPEDPIXEL_PREC_SHIFTS - sx) + + ref[(height - 1) * stride + ix + 1] * sx, + WARPEDPIXEL_PREC_BITS); + return clip_pixel(v); + } else if (ix >= WARPEDPIXEL_FILTER_TAPS / 2 - 1 && + iy >= WARPEDPIXEL_FILTER_TAPS / 2 - 1 && + ix < width - WARPEDPIXEL_FILTER_TAPS / 2 && + iy < height - WARPEDPIXEL_FILTER_TAPS / 2) { + return bi_ntap_filter(ref, x, y, stride); + } else if (ix >= 1 && iy >= 1 && ix < width - 2 && iy < height - 2) { + return bi_cubic_filter(ref, x, y, stride); + } else { + return bi_linear_filter(ref, x, y, stride); + } +} + +// For warping, we really use a 6-tap filter, but we do blocks of 8 pixels +// at a time. The zoom/rotation/shear in the model are applied to the +// "fractional" position of each pixel, which therefore varies within +// [-1, 2) * WARPEDPIXEL_PREC_SHIFTS. +// We need an extra 2 taps to fit this in, for a total of 8 taps. +/* clang-format off */ +const int16_t warped_filter[WARPEDPIXEL_PREC_SHIFTS * 3 + 1][8] = { +#if WARPEDPIXEL_PREC_BITS == 6 + // [-1, 0) + { 0, 0, 127, 1, 0, 0, 0, 0 }, { 0, - 1, 127, 2, 0, 0, 0, 0 }, + { 1, - 3, 127, 4, - 1, 0, 0, 0 }, { 1, - 4, 126, 6, - 2, 1, 0, 0 }, + { 1, - 5, 126, 8, - 3, 1, 0, 0 }, { 1, - 6, 125, 11, - 4, 1, 0, 0 }, + { 1, - 7, 124, 13, - 4, 1, 0, 0 }, { 2, - 8, 123, 15, - 5, 1, 0, 0 }, + { 2, - 9, 122, 18, - 6, 1, 0, 0 }, { 2, -10, 121, 20, - 6, 1, 0, 0 }, + { 2, -11, 120, 22, - 7, 2, 0, 0 }, { 2, -12, 119, 25, - 8, 2, 0, 0 }, + { 3, -13, 117, 27, - 8, 2, 0, 0 }, { 3, -13, 116, 29, - 9, 2, 0, 0 }, + { 3, -14, 114, 32, -10, 3, 0, 0 }, { 3, -15, 113, 35, -10, 2, 0, 0 }, + { 3, -15, 111, 37, -11, 3, 0, 0 }, { 3, -16, 109, 40, -11, 3, 0, 0 }, + { 3, -16, 108, 42, -12, 3, 0, 0 }, { 4, -17, 106, 45, -13, 3, 0, 0 }, + { 4, -17, 104, 47, -13, 3, 0, 0 }, { 4, -17, 102, 50, -14, 3, 0, 0 }, + { 4, -17, 100, 52, -14, 3, 0, 0 }, { 4, -18, 98, 55, -15, 4, 0, 0 }, + { 4, -18, 96, 58, -15, 3, 0, 0 }, { 4, -18, 94, 60, -16, 4, 0, 0 }, + { 4, -18, 91, 63, -16, 4, 0, 0 }, { 4, -18, 89, 65, -16, 4, 0, 0 }, + { 4, -18, 87, 68, -17, 4, 0, 0 }, { 4, -18, 85, 70, -17, 4, 0, 0 }, + { 4, -18, 82, 73, -17, 4, 0, 0 }, { 4, -18, 80, 75, -17, 4, 0, 0 }, + { 4, -18, 78, 78, -18, 4, 0, 0 }, { 4, -17, 75, 80, -18, 4, 0, 0 }, + { 4, -17, 73, 82, -18, 4, 0, 0 }, { 4, -17, 70, 85, -18, 4, 0, 0 }, + { 4, -17, 68, 87, -18, 4, 0, 0 }, { 4, -16, 65, 89, -18, 4, 0, 0 }, + { 4, -16, 63, 91, -18, 4, 0, 0 }, { 4, -16, 60, 94, -18, 4, 0, 0 }, + { 3, -15, 58, 96, -18, 4, 0, 0 }, { 4, -15, 55, 98, -18, 4, 0, 0 }, + { 3, -14, 52, 100, -17, 4, 0, 0 }, { 3, -14, 50, 102, -17, 4, 0, 0 }, + { 3, -13, 47, 104, -17, 4, 0, 0 }, { 3, -13, 45, 106, -17, 4, 0, 0 }, + { 3, -12, 42, 108, -16, 3, 0, 0 }, { 3, -11, 40, 109, -16, 3, 0, 0 }, + { 3, -11, 37, 111, -15, 3, 0, 0 }, { 2, -10, 35, 113, -15, 3, 0, 0 }, + { 3, -10, 32, 114, -14, 3, 0, 0 }, { 2, - 9, 29, 116, -13, 3, 0, 0 }, + { 2, - 8, 27, 117, -13, 3, 0, 0 }, { 2, - 8, 25, 119, -12, 2, 0, 0 }, + { 2, - 7, 22, 120, -11, 2, 0, 0 }, { 1, - 6, 20, 121, -10, 2, 0, 0 }, + { 1, - 6, 18, 122, - 9, 2, 0, 0 }, { 1, - 5, 15, 123, - 8, 2, 0, 0 }, + { 1, - 4, 13, 124, - 7, 1, 0, 0 }, { 1, - 4, 11, 125, - 6, 1, 0, 0 }, + { 1, - 3, 8, 126, - 5, 1, 0, 0 }, { 1, - 2, 6, 126, - 4, 1, 0, 0 }, + { 0, - 1, 4, 127, - 3, 1, 0, 0 }, { 0, 0, 2, 127, - 1, 0, 0, 0 }, + + // [0, 1) + { 0, 0, 0, 127, 1, 0, 0, 0}, { 0, 0, -1, 127, 2, 0, 0, 0}, + { 0, 1, -3, 127, 4, -2, 1, 0}, { 0, 1, -5, 127, 6, -2, 1, 0}, + { 0, 2, -6, 126, 8, -3, 1, 0}, {-1, 2, -7, 126, 11, -4, 2, -1}, + {-1, 3, -8, 125, 13, -5, 2, -1}, {-1, 3, -10, 124, 16, -6, 3, -1}, + {-1, 4, -11, 123, 18, -7, 3, -1}, {-1, 4, -12, 122, 20, -7, 3, -1}, + {-1, 4, -13, 121, 23, -8, 3, -1}, {-2, 5, -14, 120, 25, -9, 4, -1}, + {-1, 5, -15, 119, 27, -10, 4, -1}, {-1, 5, -16, 118, 30, -11, 4, -1}, + {-2, 6, -17, 116, 33, -12, 5, -1}, {-2, 6, -17, 114, 35, -12, 5, -1}, + {-2, 6, -18, 113, 38, -13, 5, -1}, {-2, 7, -19, 111, 41, -14, 6, -2}, + {-2, 7, -19, 110, 43, -15, 6, -2}, {-2, 7, -20, 108, 46, -15, 6, -2}, + {-2, 7, -20, 106, 49, -16, 6, -2}, {-2, 7, -21, 104, 51, -16, 7, -2}, + {-2, 7, -21, 102, 54, -17, 7, -2}, {-2, 8, -21, 100, 56, -18, 7, -2}, + {-2, 8, -22, 98, 59, -18, 7, -2}, {-2, 8, -22, 96, 62, -19, 7, -2}, + {-2, 8, -22, 94, 64, -19, 7, -2}, {-2, 8, -22, 91, 67, -20, 8, -2}, + {-2, 8, -22, 89, 69, -20, 8, -2}, {-2, 8, -22, 87, 72, -21, 8, -2}, + {-2, 8, -21, 84, 74, -21, 8, -2}, {-2, 8, -22, 82, 77, -21, 8, -2}, + {-2, 8, -21, 79, 79, -21, 8, -2}, {-2, 8, -21, 77, 82, -22, 8, -2}, + {-2, 8, -21, 74, 84, -21, 8, -2}, {-2, 8, -21, 72, 87, -22, 8, -2}, + {-2, 8, -20, 69, 89, -22, 8, -2}, {-2, 8, -20, 67, 91, -22, 8, -2}, + {-2, 7, -19, 64, 94, -22, 8, -2}, {-2, 7, -19, 62, 96, -22, 8, -2}, + {-2, 7, -18, 59, 98, -22, 8, -2}, {-2, 7, -18, 56, 100, -21, 8, -2}, + {-2, 7, -17, 54, 102, -21, 7, -2}, {-2, 7, -16, 51, 104, -21, 7, -2}, + {-2, 6, -16, 49, 106, -20, 7, -2}, {-2, 6, -15, 46, 108, -20, 7, -2}, + {-2, 6, -15, 43, 110, -19, 7, -2}, {-2, 6, -14, 41, 111, -19, 7, -2}, + {-1, 5, -13, 38, 113, -18, 6, -2}, {-1, 5, -12, 35, 114, -17, 6, -2}, + {-1, 5, -12, 33, 116, -17, 6, -2}, {-1, 4, -11, 30, 118, -16, 5, -1}, + {-1, 4, -10, 27, 119, -15, 5, -1}, {-1, 4, -9, 25, 120, -14, 5, -2}, + {-1, 3, -8, 23, 121, -13, 4, -1}, {-1, 3, -7, 20, 122, -12, 4, -1}, + {-1, 3, -7, 18, 123, -11, 4, -1}, {-1, 3, -6, 16, 124, -10, 3, -1}, + {-1, 2, -5, 13, 125, -8, 3, -1}, {-1, 2, -4, 11, 126, -7, 2, -1}, + { 0, 1, -3, 8, 126, -6, 2, 0}, { 0, 1, -2, 6, 127, -5, 1, 0}, + { 0, 1, -2, 4, 127, -3, 1, 0}, { 0, 0, 0, 2, 127, -1, 0, 0}, + + // [1, 2) + { 0, 0, 0, 1, 127, 0, 0, 0 }, { 0, 0, 0, - 1, 127, 2, 0, 0 }, + { 0, 0, 1, - 3, 127, 4, - 1, 0 }, { 0, 0, 1, - 4, 126, 6, - 2, 1 }, + { 0, 0, 1, - 5, 126, 8, - 3, 1 }, { 0, 0, 1, - 6, 125, 11, - 4, 1 }, + { 0, 0, 1, - 7, 124, 13, - 4, 1 }, { 0, 0, 2, - 8, 123, 15, - 5, 1 }, + { 0, 0, 2, - 9, 122, 18, - 6, 1 }, { 0, 0, 2, -10, 121, 20, - 6, 1 }, + { 0, 0, 2, -11, 120, 22, - 7, 2 }, { 0, 0, 2, -12, 119, 25, - 8, 2 }, + { 0, 0, 3, -13, 117, 27, - 8, 2 }, { 0, 0, 3, -13, 116, 29, - 9, 2 }, + { 0, 0, 3, -14, 114, 32, -10, 3 }, { 0, 0, 3, -15, 113, 35, -10, 2 }, + { 0, 0, 3, -15, 111, 37, -11, 3 }, { 0, 0, 3, -16, 109, 40, -11, 3 }, + { 0, 0, 3, -16, 108, 42, -12, 3 }, { 0, 0, 4, -17, 106, 45, -13, 3 }, + { 0, 0, 4, -17, 104, 47, -13, 3 }, { 0, 0, 4, -17, 102, 50, -14, 3 }, + { 0, 0, 4, -17, 100, 52, -14, 3 }, { 0, 0, 4, -18, 98, 55, -15, 4 }, + { 0, 0, 4, -18, 96, 58, -15, 3 }, { 0, 0, 4, -18, 94, 60, -16, 4 }, + { 0, 0, 4, -18, 91, 63, -16, 4 }, { 0, 0, 4, -18, 89, 65, -16, 4 }, + { 0, 0, 4, -18, 87, 68, -17, 4 }, { 0, 0, 4, -18, 85, 70, -17, 4 }, + { 0, 0, 4, -18, 82, 73, -17, 4 }, { 0, 0, 4, -18, 80, 75, -17, 4 }, + { 0, 0, 4, -18, 78, 78, -18, 4 }, { 0, 0, 4, -17, 75, 80, -18, 4 }, + { 0, 0, 4, -17, 73, 82, -18, 4 }, { 0, 0, 4, -17, 70, 85, -18, 4 }, + { 0, 0, 4, -17, 68, 87, -18, 4 }, { 0, 0, 4, -16, 65, 89, -18, 4 }, + { 0, 0, 4, -16, 63, 91, -18, 4 }, { 0, 0, 4, -16, 60, 94, -18, 4 }, + { 0, 0, 3, -15, 58, 96, -18, 4 }, { 0, 0, 4, -15, 55, 98, -18, 4 }, + { 0, 0, 3, -14, 52, 100, -17, 4 }, { 0, 0, 3, -14, 50, 102, -17, 4 }, + { 0, 0, 3, -13, 47, 104, -17, 4 }, { 0, 0, 3, -13, 45, 106, -17, 4 }, + { 0, 0, 3, -12, 42, 108, -16, 3 }, { 0, 0, 3, -11, 40, 109, -16, 3 }, + { 0, 0, 3, -11, 37, 111, -15, 3 }, { 0, 0, 2, -10, 35, 113, -15, 3 }, + { 0, 0, 3, -10, 32, 114, -14, 3 }, { 0, 0, 2, - 9, 29, 116, -13, 3 }, + { 0, 0, 2, - 8, 27, 117, -13, 3 }, { 0, 0, 2, - 8, 25, 119, -12, 2 }, + { 0, 0, 2, - 7, 22, 120, -11, 2 }, { 0, 0, 1, - 6, 20, 121, -10, 2 }, + { 0, 0, 1, - 6, 18, 122, - 9, 2 }, { 0, 0, 1, - 5, 15, 123, - 8, 2 }, + { 0, 0, 1, - 4, 13, 124, - 7, 1 }, { 0, 0, 1, - 4, 11, 125, - 6, 1 }, + { 0, 0, 1, - 3, 8, 126, - 5, 1 }, { 0, 0, 1, - 2, 6, 126, - 4, 1 }, + { 0, 0, 0, - 1, 4, 127, - 3, 1 }, { 0, 0, 0, 0, 2, 127, - 1, 0 }, + +#elif WARPEDPIXEL_PREC_BITS == 5 + // [-1, 0) + {0, 0, 127, 1, 0, 0, 0, 0}, {1, -3, 127, 4, -1, 0, 0, 0}, + {1, -5, 126, 8, -3, 1, 0, 0}, {1, -7, 124, 13, -4, 1, 0, 0}, + {2, -9, 122, 18, -6, 1, 0, 0}, {2, -11, 120, 22, -7, 2, 0, 0}, + {3, -13, 117, 27, -8, 2, 0, 0}, {3, -14, 114, 32, -10, 3, 0, 0}, + {3, -15, 111, 37, -11, 3, 0, 0}, {3, -16, 108, 42, -12, 3, 0, 0}, + {4, -17, 104, 47, -13, 3, 0, 0}, {4, -17, 100, 52, -14, 3, 0, 0}, + {4, -18, 96, 58, -15, 3, 0, 0}, {4, -18, 91, 63, -16, 4, 0, 0}, + {4, -18, 87, 68, -17, 4, 0, 0}, {4, -18, 82, 73, -17, 4, 0, 0}, + {4, -18, 78, 78, -18, 4, 0, 0}, {4, -17, 73, 82, -18, 4, 0, 0}, + {4, -17, 68, 87, -18, 4, 0, 0}, {4, -16, 63, 91, -18, 4, 0, 0}, + {3, -15, 58, 96, -18, 4, 0, 0}, {3, -14, 52, 100, -17, 4, 0, 0}, + {3, -13, 47, 104, -17, 4, 0, 0}, {3, -12, 42, 108, -16, 3, 0, 0}, + {3, -11, 37, 111, -15, 3, 0, 0}, {3, -10, 32, 114, -14, 3, 0, 0}, + {2, -8, 27, 117, -13, 3, 0, 0}, {2, -7, 22, 120, -11, 2, 0, 0}, + {1, -6, 18, 122, -9, 2, 0, 0}, {1, -4, 13, 124, -7, 1, 0, 0}, + {1, -3, 8, 126, -5, 1, 0, 0}, {0, -1, 4, 127, -3, 1, 0, 0}, + // [0, 1) + { 0, 0, 0, 127, 1, 0, 0, 0}, { 0, 1, -3, 127, 4, -2, 1, 0}, + { 0, 2, -6, 126, 8, -3, 1, 0}, {-1, 3, -8, 125, 13, -5, 2, -1}, + {-1, 4, -11, 123, 18, -7, 3, -1}, {-1, 4, -13, 121, 23, -8, 3, -1}, + {-1, 5, -15, 119, 27, -10, 4, -1}, {-2, 6, -17, 116, 33, -12, 5, -1}, + {-2, 6, -18, 113, 38, -13, 5, -1}, {-2, 7, -19, 110, 43, -15, 6, -2}, + {-2, 7, -20, 106, 49, -16, 6, -2}, {-2, 7, -21, 102, 54, -17, 7, -2}, + {-2, 8, -22, 98, 59, -18, 7, -2}, {-2, 8, -22, 94, 64, -19, 7, -2}, + {-2, 8, -22, 89, 69, -20, 8, -2}, {-2, 8, -21, 84, 74, -21, 8, -2}, + {-2, 8, -21, 79, 79, -21, 8, -2}, {-2, 8, -21, 74, 84, -21, 8, -2}, + {-2, 8, -20, 69, 89, -22, 8, -2}, {-2, 7, -19, 64, 94, -22, 8, -2}, + {-2, 7, -18, 59, 98, -22, 8, -2}, {-2, 7, -17, 54, 102, -21, 7, -2}, + {-2, 6, -16, 49, 106, -20, 7, -2}, {-2, 6, -15, 43, 110, -19, 7, -2}, + {-1, 5, -13, 38, 113, -18, 6, -2}, {-1, 5, -12, 33, 116, -17, 6, -2}, + {-1, 4, -10, 27, 119, -15, 5, -1}, {-1, 3, -8, 23, 121, -13, 4, -1}, + {-1, 3, -7, 18, 123, -11, 4, -1}, {-1, 2, -5, 13, 125, -8, 3, -1}, + { 0, 1, -3, 8, 126, -6, 2, 0}, { 0, 1, -2, 4, 127, -3, 1, 0}, + // [1, 2) + {0, 0, 0, 1, 127, 0, 0, 0}, {0, 0, 1, -3, 127, 4, -1, 0}, + {0, 0, 1, -5, 126, 8, -3, 1}, {0, 0, 1, -7, 124, 13, -4, 1}, + {0, 0, 2, -9, 122, 18, -6, 1}, {0, 0, 2, -11, 120, 22, -7, 2}, + {0, 0, 3, -13, 117, 27, -8, 2}, {0, 0, 3, -14, 114, 32, -10, 3}, + {0, 0, 3, -15, 111, 37, -11, 3}, {0, 0, 3, -16, 108, 42, -12, 3}, + {0, 0, 4, -17, 104, 47, -13, 3}, {0, 0, 4, -17, 100, 52, -14, 3}, + {0, 0, 4, -18, 96, 58, -15, 3}, {0, 0, 4, -18, 91, 63, -16, 4}, + {0, 0, 4, -18, 87, 68, -17, 4}, {0, 0, 4, -18, 82, 73, -17, 4}, + {0, 0, 4, -18, 78, 78, -18, 4}, {0, 0, 4, -17, 73, 82, -18, 4}, + {0, 0, 4, -17, 68, 87, -18, 4}, {0, 0, 4, -16, 63, 91, -18, 4}, + {0, 0, 3, -15, 58, 96, -18, 4}, {0, 0, 3, -14, 52, 100, -17, 4}, + {0, 0, 3, -13, 47, 104, -17, 4}, {0, 0, 3, -12, 42, 108, -16, 3}, + {0, 0, 3, -11, 37, 111, -15, 3}, {0, 0, 3, -10, 32, 114, -14, 3}, + {0, 0, 2, -8, 27, 117, -13, 3}, {0, 0, 2, -7, 22, 120, -11, 2}, + {0, 0, 1, -6, 18, 122, -9, 2}, {0, 0, 1, -4, 13, 124, -7, 1}, + {0, 0, 1, -3, 8, 126, -5, 1}, {0, 0, 0, -1, 4, 127, -3, 1}, + +#endif // WARPEDPIXEL_PREC_BITS == 6 + + // dummy + { 0, 0, 0, 0, 1, 127, 0, 0 }, +}; + +/* clang-format on */ + +#define DIV_LUT_PREC_BITS 14 +#define DIV_LUT_BITS 8 +#define DIV_LUT_NUM (1 << DIV_LUT_BITS) + +static const uint16_t div_lut[DIV_LUT_NUM + 1] = { + 16384, 16320, 16257, 16194, 16132, 16070, 16009, 15948, 15888, 15828, 15768, + 15709, 15650, 15592, 15534, 15477, 15420, 15364, 15308, 15252, 15197, 15142, + 15087, 15033, 14980, 14926, 14873, 14821, 14769, 14717, 14665, 14614, 14564, + 14513, 14463, 14413, 14364, 14315, 14266, 14218, 14170, 14122, 14075, 14028, + 13981, 13935, 13888, 13843, 13797, 13752, 13707, 13662, 13618, 13574, 13530, + 13487, 13443, 13400, 13358, 13315, 13273, 13231, 13190, 13148, 13107, 13066, + 13026, 12985, 12945, 12906, 12866, 12827, 12788, 12749, 12710, 12672, 12633, + 12596, 12558, 12520, 12483, 12446, 12409, 12373, 12336, 12300, 12264, 12228, + 12193, 12157, 12122, 12087, 12053, 12018, 11984, 11950, 11916, 11882, 11848, + 11815, 11782, 11749, 11716, 11683, 11651, 11619, 11586, 11555, 11523, 11491, + 11460, 11429, 11398, 11367, 11336, 11305, 11275, 11245, 11215, 11185, 11155, + 11125, 11096, 11067, 11038, 11009, 10980, 10951, 10923, 10894, 10866, 10838, + 10810, 10782, 10755, 10727, 10700, 10673, 10645, 10618, 10592, 10565, 10538, + 10512, 10486, 10460, 10434, 10408, 10382, 10356, 10331, 10305, 10280, 10255, + 10230, 10205, 10180, 10156, 10131, 10107, 10082, 10058, 10034, 10010, 9986, + 9963, 9939, 9916, 9892, 9869, 9846, 9823, 9800, 9777, 9754, 9732, + 9709, 9687, 9664, 9642, 9620, 9598, 9576, 9554, 9533, 9511, 9489, + 9468, 9447, 9425, 9404, 9383, 9362, 9341, 9321, 9300, 9279, 9259, + 9239, 9218, 9198, 9178, 9158, 9138, 9118, 9098, 9079, 9059, 9039, + 9020, 9001, 8981, 8962, 8943, 8924, 8905, 8886, 8867, 8849, 8830, + 8812, 8793, 8775, 8756, 8738, 8720, 8702, 8684, 8666, 8648, 8630, + 8613, 8595, 8577, 8560, 8542, 8525, 8508, 8490, 8473, 8456, 8439, + 8422, 8405, 8389, 8372, 8355, 8339, 8322, 8306, 8289, 8273, 8257, + 8240, 8224, 8208, 8192, +}; + +static INLINE int16_t saturate_int16(int32_t v) { + if (v > 32767) + return 32767; + else if (v < -32768) + return -32768; + return v; +} + +#if CONFIG_WARPED_MOTION +// Decomposes a divisor D such that 1/D = y/2^shift, where y is returned +// at precision of DIV_LUT_PREC_BITS along with the shift. +static int16_t resolve_divisor_64(uint64_t D, int16_t *shift) { + int64_t e, f; + *shift = (int16_t)((D >> 32) ? get_msb((unsigned int)(D >> 32)) + 32 + : get_msb((unsigned int)D)); + // e is obtained from D after resetting the most significant 1 bit. + e = D - ((uint64_t)1 << *shift); + // Get the most significant DIV_LUT_BITS (8) bits of e into f + if (*shift > DIV_LUT_BITS) + f = ROUND_POWER_OF_TWO_64(e, *shift - DIV_LUT_BITS); + else + f = e << (DIV_LUT_BITS - *shift); + assert(f <= DIV_LUT_NUM); + *shift += DIV_LUT_PREC_BITS; + // Use f as lookup into the precomputed table of multipliers + return div_lut[f]; +} +#endif // CONFIG_WARPED_MOTION + +static int16_t resolve_divisor_32(uint32_t D, int16_t *shift) { + int32_t e, f; + *shift = get_msb(D); + // e is obtained from D after resetting the most significant 1 bit. + e = D - ((uint32_t)1 << *shift); + // Get the most significant DIV_LUT_BITS (8) bits of e into f + if (*shift > DIV_LUT_BITS) + f = ROUND_POWER_OF_TWO(e, *shift - DIV_LUT_BITS); + else + f = e << (DIV_LUT_BITS - *shift); + assert(f <= DIV_LUT_NUM); + *shift += DIV_LUT_PREC_BITS; + // Use f as lookup into the precomputed table of multipliers + return div_lut[f]; +} + +static int is_affine_valid(WarpedMotionParams *wm) { + const int32_t *mat = wm->wmmat; + return (mat[2] > 0); +} + +static int is_affine_shear_allowed(int16_t alpha, int16_t beta, int16_t gamma, + int16_t delta) { + if ((4 * abs(alpha) + 7 * abs(beta) >= (1 << WARPEDMODEL_PREC_BITS)) || + (4 * abs(gamma) + 4 * abs(delta) >= (1 << WARPEDMODEL_PREC_BITS))) + return 0; + else + return 1; +} + +// Returns 1 on success or 0 on an invalid affine set +int get_shear_params(WarpedMotionParams *wm) { + const int32_t *mat = wm->wmmat; + if (!is_affine_valid(wm)) return 0; + wm->alpha = + clamp(mat[2] - (1 << WARPEDMODEL_PREC_BITS), INT16_MIN, INT16_MAX); + wm->beta = clamp(mat[3], INT16_MIN, INT16_MAX); + int16_t shift; + int16_t y = resolve_divisor_32(abs(mat[2]), &shift) * (mat[2] < 0 ? -1 : 1); + int64_t v; + v = ((int64_t)mat[4] * (1 << WARPEDMODEL_PREC_BITS)) * y; + wm->gamma = + clamp((int)ROUND_POWER_OF_TWO_SIGNED_64(v, shift), INT16_MIN, INT16_MAX); + v = ((int64_t)mat[3] * mat[4]) * y; + wm->delta = clamp(mat[5] - (int)ROUND_POWER_OF_TWO_SIGNED_64(v, shift) - + (1 << WARPEDMODEL_PREC_BITS), + INT16_MIN, INT16_MAX); + if (!is_affine_shear_allowed(wm->alpha, wm->beta, wm->gamma, wm->delta)) + return 0; + return 1; +} + +#if CONFIG_HIGHBITDEPTH +static INLINE void highbd_get_subcolumn(int taps, uint16_t *ref, int32_t *col, + int stride, int x, int y_start) { + int i; + for (i = 0; i < taps; ++i) { + col[i] = ref[(i + y_start) * stride + x]; + } +} + +static uint16_t highbd_bi_ntap_filter(uint16_t *ref, int x, int y, int stride, + int bd) { + int32_t val, arr[WARPEDPIXEL_FILTER_TAPS]; + int k; + int i = (int)x >> WARPEDPIXEL_PREC_BITS; + int j = (int)y >> WARPEDPIXEL_PREC_BITS; + for (k = 0; k < WARPEDPIXEL_FILTER_TAPS; ++k) { + int32_t arr_temp[WARPEDPIXEL_FILTER_TAPS]; + highbd_get_subcolumn(WARPEDPIXEL_FILTER_TAPS, ref, arr_temp, stride, + i + k + 1 - WARPEDPIXEL_FILTER_TAPS / 2, + j + 1 - WARPEDPIXEL_FILTER_TAPS / 2); + arr[k] = do_ntap_filter(arr_temp + WARPEDPIXEL_FILTER_TAPS / 2 - 1, + y - (j * (1 << WARPEDPIXEL_PREC_BITS))); + } + val = do_ntap_filter(arr + WARPEDPIXEL_FILTER_TAPS / 2 - 1, + x - (i * (1 << WARPEDPIXEL_PREC_BITS))); + val = ROUND_POWER_OF_TWO_SIGNED(val, WARPEDPIXEL_FILTER_BITS * 2); + return (uint16_t)clip_pixel_highbd(val, bd); +} + +static uint16_t highbd_bi_cubic_filter(uint16_t *ref, int x, int y, int stride, + int bd) { + int32_t val, arr[4]; + int k; + int i = (int)x >> WARPEDPIXEL_PREC_BITS; + int j = (int)y >> WARPEDPIXEL_PREC_BITS; + for (k = 0; k < 4; ++k) { + int32_t arr_temp[4]; + highbd_get_subcolumn(4, ref, arr_temp, stride, i + k - 1, j - 1); + arr[k] = + do_cubic_filter(arr_temp + 1, y - (j * (1 << WARPEDPIXEL_PREC_BITS))); + } + val = do_cubic_filter(arr + 1, x - (i * (1 << WARPEDPIXEL_PREC_BITS))); + val = ROUND_POWER_OF_TWO_SIGNED(val, WARPEDPIXEL_FILTER_BITS * 2); + return (uint16_t)clip_pixel_highbd(val, bd); +} + +static uint16_t highbd_bi_linear_filter(uint16_t *ref, int x, int y, int stride, + int bd) { + const int ix = x >> WARPEDPIXEL_PREC_BITS; + const int iy = y >> WARPEDPIXEL_PREC_BITS; + const int sx = x - (ix * (1 << WARPEDPIXEL_PREC_BITS)); + const int sy = y - (iy * (1 << WARPEDPIXEL_PREC_BITS)); + int32_t val; + val = ROUND_POWER_OF_TWO_SIGNED( + ref[iy * stride + ix] * (WARPEDPIXEL_PREC_SHIFTS - sy) * + (WARPEDPIXEL_PREC_SHIFTS - sx) + + ref[iy * stride + ix + 1] * (WARPEDPIXEL_PREC_SHIFTS - sy) * sx + + ref[(iy + 1) * stride + ix] * sy * (WARPEDPIXEL_PREC_SHIFTS - sx) + + ref[(iy + 1) * stride + ix + 1] * sy * sx, + WARPEDPIXEL_PREC_BITS * 2); + return (uint16_t)clip_pixel_highbd(val, bd); +} + +static uint16_t highbd_warp_interpolate(uint16_t *ref, int x, int y, int width, + int height, int stride, int bd) { + int ix = x >> WARPEDPIXEL_PREC_BITS; + int iy = y >> WARPEDPIXEL_PREC_BITS; + int sx = x - (ix * (1 << WARPEDPIXEL_PREC_BITS)); + int sy = y - (iy * (1 << WARPEDPIXEL_PREC_BITS)); + int32_t v; + + if (ix < 0 && iy < 0) + return ref[0]; + else if (ix < 0 && iy > height - 1) + return ref[(height - 1) * stride]; + else if (ix > width - 1 && iy < 0) + return ref[width - 1]; + else if (ix > width - 1 && iy > height - 1) + return ref[(height - 1) * stride + (width - 1)]; + else if (ix < 0) { + v = ROUND_POWER_OF_TWO_SIGNED( + ref[iy * stride] * (WARPEDPIXEL_PREC_SHIFTS - sy) + + ref[(iy + 1) * stride] * sy, + WARPEDPIXEL_PREC_BITS); + return clip_pixel_highbd(v, bd); + } else if (iy < 0) { + v = ROUND_POWER_OF_TWO_SIGNED( + ref[ix] * (WARPEDPIXEL_PREC_SHIFTS - sx) + ref[ix + 1] * sx, + WARPEDPIXEL_PREC_BITS); + return clip_pixel_highbd(v, bd); + } else if (ix > width - 1) { + v = ROUND_POWER_OF_TWO_SIGNED( + ref[iy * stride + width - 1] * (WARPEDPIXEL_PREC_SHIFTS - sy) + + ref[(iy + 1) * stride + width - 1] * sy, + WARPEDPIXEL_PREC_BITS); + return clip_pixel_highbd(v, bd); + } else if (iy > height - 1) { + v = ROUND_POWER_OF_TWO_SIGNED( + ref[(height - 1) * stride + ix] * (WARPEDPIXEL_PREC_SHIFTS - sx) + + ref[(height - 1) * stride + ix + 1] * sx, + WARPEDPIXEL_PREC_BITS); + return clip_pixel_highbd(v, bd); + } else if (ix >= WARPEDPIXEL_FILTER_TAPS / 2 - 1 && + iy >= WARPEDPIXEL_FILTER_TAPS / 2 - 1 && + ix < width - WARPEDPIXEL_FILTER_TAPS / 2 && + iy < height - WARPEDPIXEL_FILTER_TAPS / 2) { + return highbd_bi_ntap_filter(ref, x, y, stride, bd); + } else if (ix >= 1 && iy >= 1 && ix < width - 2 && iy < height - 2) { + return highbd_bi_cubic_filter(ref, x, y, stride, bd); + } else { + return highbd_bi_linear_filter(ref, x, y, stride, bd); + } +} + +static INLINE int highbd_error_measure(int err, int bd) { + const int b = bd - 8; + const int bmask = (1 << b) - 1; + const int v = (1 << b); + int e1, e2; + err = abs(err); + e1 = err >> b; + e2 = err & bmask; + return error_measure_lut[255 + e1] * (v - e2) + + error_measure_lut[256 + e1] * e2; +} + +static void highbd_warp_plane_old(WarpedMotionParams *wm, uint8_t *ref8, + int width, int height, int stride, + uint8_t *pred8, int p_col, int p_row, + int p_width, int p_height, int p_stride, + int subsampling_x, int subsampling_y, + int x_scale, int y_scale, int bd, + int ref_frm) { + int i, j; + ProjectPointsFunc projectpoints = get_project_points_type(wm->wmtype); + uint16_t *pred = CONVERT_TO_SHORTPTR(pred8); + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + if (projectpoints == NULL) return; + for (i = p_row; i < p_row + p_height; ++i) { + for (j = p_col; j < p_col + p_width; ++j) { + int in[2], out[2]; + in[0] = j; + in[1] = i; + projectpoints(wm->wmmat, in, out, 1, 2, 2, subsampling_x, subsampling_y); + out[0] = ROUND_POWER_OF_TWO_SIGNED(out[0] * x_scale, 4); + out[1] = ROUND_POWER_OF_TWO_SIGNED(out[1] * y_scale, 4); + if (ref_frm) + pred[(j - p_col) + (i - p_row) * p_stride] = ROUND_POWER_OF_TWO( + pred[(j - p_col) + (i - p_row) * p_stride] + + highbd_warp_interpolate(ref, out[0], out[1], width, height, + stride, bd), + 1); + else + pred[(j - p_col) + (i - p_row) * p_stride] = highbd_warp_interpolate( + ref, out[0], out[1], width, height, stride, bd); + } + } +} + +// Note: For an explanation of the warp algorithm, see the comment +// above warp_plane() +// +// Note also: The "worst case" in terms of modulus of the data stored into 'tmp' +// (ie, the result of 'sum' in the horizontal filter) occurs when: +// coeffs = { -2, 8, -22, 87, 72, -21, 8, -2}, and +// ref = { 0, 255, 0, 255, 255, 0, 255, 0} +// Before rounding, this gives sum = 716625. After rounding, +// HORSHEAR_REDUCE_PREC_BITS = 4 => sum = 44789 > 2^15 +// HORSHEAR_REDUCE_PREC_BITS = 5 => sum = 22395 < 2^15 +// +// So, as long as HORSHEAR_REDUCE_PREC_BITS >= 5, we can safely use a 16-bit +// intermediate array. +void av1_highbd_warp_affine_c(int32_t *mat, uint16_t *ref, int width, + int height, int stride, uint16_t *pred, int p_col, + int p_row, int p_width, int p_height, + int p_stride, int subsampling_x, + int subsampling_y, int bd, int ref_frm, + int16_t alpha, int16_t beta, int16_t gamma, + int16_t delta) { +#if HORSHEAR_REDUCE_PREC_BITS >= 5 + int16_t tmp[15 * 8]; +#else + int32_t tmp[15 * 8]; +#endif + int i, j, k, l, m; + + /* Note: For this code to work, the left/right frame borders need to be + extended by at least 13 pixels each. By the time we get here, other + code will have set up this border, but we allow an explicit check + for debugging purposes. + */ + /*for (i = 0; i < height; ++i) { + for (j = 0; j < 13; ++j) { + assert(ref[i * stride - 13 + j] == ref[i * stride]); + assert(ref[i * stride + width + j] == ref[i * stride + (width - 1)]); + } + }*/ + + for (i = p_row; i < p_row + p_height; i += 8) { + for (j = p_col; j < p_col + p_width; j += 8) { + int32_t x4, y4, ix4, sx4, iy4, sy4; + if (subsampling_x) + x4 = ROUND_POWER_OF_TWO_SIGNED( + mat[2] * 2 * (j + 4) + mat[3] * 2 * (i + 4) + mat[0] + + (mat[2] + mat[3] - (1 << WARPEDMODEL_PREC_BITS)) / 2, + 1); + else + x4 = mat[2] * (j + 4) + mat[3] * (i + 4) + mat[0]; + + if (subsampling_y) + y4 = ROUND_POWER_OF_TWO_SIGNED( + mat[4] * 2 * (j + 4) + mat[5] * 2 * (i + 4) + mat[1] + + (mat[4] + mat[5] - (1 << WARPEDMODEL_PREC_BITS)) / 2, + 1); + else + y4 = mat[4] * (j + 4) + mat[5] * (i + 4) + mat[1]; + + ix4 = x4 >> WARPEDMODEL_PREC_BITS; + sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + iy4 = y4 >> WARPEDMODEL_PREC_BITS; + sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + + // Horizontal filter + for (k = -7; k < 8; ++k) { + int iy = iy4 + k; + if (iy < 0) + iy = 0; + else if (iy > height - 1) + iy = height - 1; + + if (ix4 <= -7) { + for (l = 0; l < 8; ++l) { + tmp[(k + 7) * 8 + l] = + ref[iy * stride] * + (1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS)); + } + } else if (ix4 >= width + 6) { + for (l = 0; l < 8; ++l) { + tmp[(k + 7) * 8 + l] = + ref[iy * stride + (width - 1)] * + (1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS)); + } + } else { + int sx = sx4 + alpha * (-4) + beta * k; + + for (l = -4; l < 4; ++l) { + int ix = ix4 + l - 3; + const int offs = ROUND_POWER_OF_TWO(sx, WARPEDDIFF_PREC_BITS) + + WARPEDPIXEL_PREC_SHIFTS; + const int16_t *coeffs = warped_filter[offs]; + int32_t sum = 0; + // assert(offs >= 0 && offs <= WARPEDPIXEL_PREC_SHIFTS * 3); + for (m = 0; m < 8; ++m) { + sum += ref[iy * stride + ix + m] * coeffs[m]; + } + sum = ROUND_POWER_OF_TWO(sum, HORSHEAR_REDUCE_PREC_BITS); +#if HORSHEAR_REDUCE_PREC_BITS >= 5 + tmp[(k + 7) * 8 + (l + 4)] = saturate_int16(sum); +#else + tmp[(k + 7) * 8 + (l + 4)] = sum; +#endif + sx += alpha; + } + } + } + + // Vertical filter + for (k = -4; k < AOMMIN(4, p_row + p_height - i - 4); ++k) { + int sy = sy4 + gamma * (-4) + delta * k; + for (l = -4; l < 4; ++l) { + uint16_t *p = + &pred[(i - p_row + k + 4) * p_stride + (j - p_col + l + 4)]; + const int offs = ROUND_POWER_OF_TWO(sy, WARPEDDIFF_PREC_BITS) + + WARPEDPIXEL_PREC_SHIFTS; + const int16_t *coeffs = warped_filter[offs]; + int32_t sum = 0; + // assert(offs >= 0 && offs <= WARPEDPIXEL_PREC_SHIFTS * 3); + for (m = 0; m < 8; ++m) { + sum += tmp[(k + m + 4) * 8 + (l + 4)] * coeffs[m]; + } + sum = clip_pixel_highbd( + ROUND_POWER_OF_TWO(sum, VERSHEAR_REDUCE_PREC_BITS), bd); + if (ref_frm) + *p = ROUND_POWER_OF_TWO(*p + sum, 1); + else + *p = sum; + sy += gamma; + } + } + } + } +} + +static void highbd_warp_plane(WarpedMotionParams *wm, uint8_t *ref8, int width, + int height, int stride, uint8_t *pred8, int p_col, + int p_row, int p_width, int p_height, + int p_stride, int subsampling_x, + int subsampling_y, int x_scale, int y_scale, + int bd, int ref_frm) { + if (wm->wmtype == ROTZOOM) { + wm->wmmat[5] = wm->wmmat[2]; + wm->wmmat[4] = -wm->wmmat[3]; + } + if ((wm->wmtype == ROTZOOM || wm->wmtype == AFFINE) && x_scale == 16 && + y_scale == 16) { + int32_t *mat = wm->wmmat; + const int16_t alpha = wm->alpha; + const int16_t beta = wm->beta; + const int16_t gamma = wm->gamma; + const int16_t delta = wm->delta; + + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + uint16_t *pred = CONVERT_TO_SHORTPTR(pred8); + av1_highbd_warp_affine(mat, ref, width, height, stride, pred, p_col, p_row, + p_width, p_height, p_stride, subsampling_x, + subsampling_y, bd, ref_frm, alpha, beta, gamma, + delta); + } else { + highbd_warp_plane_old(wm, ref8, width, height, stride, pred8, p_col, p_row, + p_width, p_height, p_stride, subsampling_x, + subsampling_y, x_scale, y_scale, bd, ref_frm); + } +} + +static double highbd_warp_erroradv(WarpedMotionParams *wm, uint8_t *ref8, + int width, int height, int stride, + uint8_t *dst8, int p_col, int p_row, + int p_width, int p_height, int p_stride, + int subsampling_x, int subsampling_y, + int x_scale, int y_scale, int bd) { + int gm_err = 0, no_gm_err = 0; + int64_t gm_sumerr = 0, no_gm_sumerr = 0; + int i, j; + uint16_t *tmp = aom_malloc(p_width * p_height * sizeof(*tmp)); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + highbd_warp_plane(wm, ref8, width, height, stride, CONVERT_TO_BYTEPTR(tmp), + p_col, p_row, p_width, p_height, p_width, subsampling_x, + subsampling_y, x_scale, y_scale, bd, 0); + for (i = 0; i < p_height; ++i) { + for (j = 0; j < p_width; ++j) { + gm_err = dst[j + i * p_stride] - tmp[j + i * p_width]; + no_gm_err = + dst[j + i * p_stride] - ref[(j + p_col) + (i + p_row) * stride]; + gm_sumerr += highbd_error_measure(gm_err, bd); + no_gm_sumerr += highbd_error_measure(no_gm_err, bd); + } + } + aom_free(tmp); + return (double)gm_sumerr / no_gm_sumerr; +} +#endif // CONFIG_HIGHBITDEPTH + +static INLINE int error_measure(int err) { + return error_measure_lut[255 + err]; +} + +static void warp_plane_old(WarpedMotionParams *wm, uint8_t *ref, int width, + int height, int stride, uint8_t *pred, int p_col, + int p_row, int p_width, int p_height, int p_stride, + int subsampling_x, int subsampling_y, int x_scale, + int y_scale, int ref_frm) { + int i, j; + ProjectPointsFunc projectpoints = get_project_points_type(wm->wmtype); + if (projectpoints == NULL) return; + for (i = p_row; i < p_row + p_height; ++i) { + for (j = p_col; j < p_col + p_width; ++j) { + int in[2], out[2]; + in[0] = j; + in[1] = i; + projectpoints(wm->wmmat, in, out, 1, 2, 2, subsampling_x, subsampling_y); + out[0] = ROUND_POWER_OF_TWO_SIGNED(out[0] * x_scale, 4); + out[1] = ROUND_POWER_OF_TWO_SIGNED(out[1] * y_scale, 4); + if (ref_frm) + pred[(j - p_col) + (i - p_row) * p_stride] = ROUND_POWER_OF_TWO( + pred[(j - p_col) + (i - p_row) * p_stride] + + warp_interpolate(ref, out[0], out[1], width, height, stride), + 1); + else + pred[(j - p_col) + (i - p_row) * p_stride] = + warp_interpolate(ref, out[0], out[1], width, height, stride); + } + } +} + +/* The warp filter for ROTZOOM and AFFINE models works as follows: + * Split the input into 8x8 blocks + * For each block, project the point (4, 4) within the block, to get the + overall block position. Split into integer and fractional coordinates, + maintaining full WARPEDMODEL precision + * Filter horizontally: Generate 15 rows of 8 pixels each. Each pixel gets a + variable horizontal offset. This means that, while the rows of the + intermediate buffer align with the rows of the *reference* image, the + columns align with the columns of the *destination* image. + * Filter vertically: Generate the output block (up to 8x8 pixels, but if the + destination is too small we crop the output at this stage). Each pixel has + a variable vertical offset, so that the resulting rows are aligned with + the rows of the destination image. + + To accomplish these alignments, we factor the warp matrix as a + product of two shear / asymmetric zoom matrices: + / a b \ = / 1 0 \ * / 1+alpha beta \ + \ c d / \ gamma 1+delta / \ 0 1 / + where a, b, c, d are wmmat[2], wmmat[3], wmmat[4], wmmat[5] respectively. + The second shear (with alpha and beta) is applied by the horizontal filter, + then the first shear (with gamma and delta) is applied by the vertical + filter. + + The only limitation is that, to fit this in a fixed 8-tap filter size, + the fractional pixel offsets must be at most +-1. Since the horizontal filter + generates 15 rows of 8 columns, and the initial point we project is at (4, 4) + within the block, the parameters must satisfy + 4 * |alpha| + 7 * |beta| <= 1 and 4 * |gamma| + 7 * |delta| <= 1 + for this filter to be applicable. + + Note: warp_affine() assumes that the caller has done all of the relevant + checks, ie. that we have a ROTZOOM or AFFINE model, that wm[4] and wm[5] + are set appropriately (if using a ROTZOOM model), and that alpha, beta, + gamma, delta are all in range. + + TODO(david.barker): Maybe support scaled references? +*/ +void av1_warp_affine_c(int32_t *mat, uint8_t *ref, int width, int height, + int stride, uint8_t *pred, int p_col, int p_row, + int p_width, int p_height, int p_stride, + int subsampling_x, int subsampling_y, int ref_frm, + int16_t alpha, int16_t beta, int16_t gamma, + int16_t delta) { + int16_t tmp[15 * 8]; + int i, j, k, l, m; + + /* Note: For this code to work, the left/right frame borders need to be + extended by at least 13 pixels each. By the time we get here, other + code will have set up this border, but we allow an explicit check + for debugging purposes. + */ + /*for (i = 0; i < height; ++i) { + for (j = 0; j < 13; ++j) { + assert(ref[i * stride - 13 + j] == ref[i * stride]); + assert(ref[i * stride + width + j] == ref[i * stride + (width - 1)]); + } + }*/ + + for (i = p_row; i < p_row + p_height; i += 8) { + for (j = p_col; j < p_col + p_width; j += 8) { + int32_t x4, y4, ix4, sx4, iy4, sy4; + if (subsampling_x) + x4 = ROUND_POWER_OF_TWO_SIGNED( + mat[2] * 2 * (j + 4) + mat[3] * 2 * (i + 4) + mat[0] + + (mat[2] + mat[3] - (1 << WARPEDMODEL_PREC_BITS)) / 2, + 1); + else + x4 = mat[2] * (j + 4) + mat[3] * (i + 4) + mat[0]; + + if (subsampling_y) + y4 = ROUND_POWER_OF_TWO_SIGNED( + mat[4] * 2 * (j + 4) + mat[5] * 2 * (i + 4) + mat[1] + + (mat[4] + mat[5] - (1 << WARPEDMODEL_PREC_BITS)) / 2, + 1); + else + y4 = mat[4] * (j + 4) + mat[5] * (i + 4) + mat[1]; + + ix4 = x4 >> WARPEDMODEL_PREC_BITS; + sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + iy4 = y4 >> WARPEDMODEL_PREC_BITS; + sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + + // Horizontal filter + for (k = -7; k < 8; ++k) { + int iy = iy4 + k; + if (iy < 0) + iy = 0; + else if (iy > height - 1) + iy = height - 1; + + if (ix4 <= -7) { + // In this case, the rightmost pixel sampled is in column + // ix4 + 3 + 7 - 3 = ix4 + 7 <= 0, ie. the entire block + // will sample only from the leftmost column + // (once border extension is taken into account) + for (l = 0; l < 8; ++l) { + tmp[(k + 7) * 8 + l] = + ref[iy * stride] * + (1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS)); + } + } else if (ix4 >= width + 6) { + // In this case, the leftmost pixel sampled is in column + // ix4 - 4 + 0 - 3 = ix4 - 7 >= width - 1, ie. the entire block + // will sample only from the rightmost column + // (once border extension is taken into account) + for (l = 0; l < 8; ++l) { + tmp[(k + 7) * 8 + l] = + ref[iy * stride + (width - 1)] * + (1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS)); + } + } else { + // If we get here, then + // the leftmost pixel sampled is + // ix4 - 4 + 0 - 3 = ix4 - 7 >= -13 + // and the rightmost pixel sampled is at most + // ix4 + 3 + 7 - 3 = ix4 + 7 <= width + 12 + // So, assuming that border extension has been done, we + // don't need to explicitly clamp values. + int sx = sx4 + alpha * (-4) + beta * k; + + for (l = -4; l < 4; ++l) { + int ix = ix4 + l - 3; + // At this point, sx = sx4 + alpha * l + beta * k + const int offs = ROUND_POWER_OF_TWO(sx, WARPEDDIFF_PREC_BITS) + + WARPEDPIXEL_PREC_SHIFTS; + const int16_t *coeffs = warped_filter[offs]; + int32_t sum = 0; + // assert(offs >= 0 && offs <= WARPEDPIXEL_PREC_SHIFTS * 3); + for (m = 0; m < 8; ++m) { + sum += ref[iy * stride + ix + m] * coeffs[m]; + } + sum = ROUND_POWER_OF_TWO(sum, HORSHEAR_REDUCE_PREC_BITS); + tmp[(k + 7) * 8 + (l + 4)] = saturate_int16(sum); + sx += alpha; + } + } + } + + // Vertical filter + for (k = -4; k < AOMMIN(4, p_row + p_height - i - 4); ++k) { + int sy = sy4 + gamma * (-4) + delta * k; + for (l = -4; l < AOMMIN(4, p_col + p_width - j - 4); ++l) { + uint8_t *p = + &pred[(i - p_row + k + 4) * p_stride + (j - p_col + l + 4)]; + // At this point, sy = sy4 + gamma * l + delta * k + const int offs = ROUND_POWER_OF_TWO(sy, WARPEDDIFF_PREC_BITS) + + WARPEDPIXEL_PREC_SHIFTS; + const int16_t *coeffs = warped_filter[offs]; + int32_t sum = 0; + // assert(offs >= 0 && offs <= WARPEDPIXEL_PREC_SHIFTS * 3); + for (m = 0; m < 8; ++m) { + sum += tmp[(k + m + 4) * 8 + (l + 4)] * coeffs[m]; + } + sum = clip_pixel(ROUND_POWER_OF_TWO(sum, VERSHEAR_REDUCE_PREC_BITS)); + if (ref_frm) + *p = ROUND_POWER_OF_TWO(*p + sum, 1); + else + *p = sum; + sy += gamma; + } + } + } + } +} + +static void warp_plane(WarpedMotionParams *wm, uint8_t *ref, int width, + int height, int stride, uint8_t *pred, int p_col, + int p_row, int p_width, int p_height, int p_stride, + int subsampling_x, int subsampling_y, int x_scale, + int y_scale, int ref_frm) { + if (wm->wmtype == ROTZOOM) { + wm->wmmat[5] = wm->wmmat[2]; + wm->wmmat[4] = -wm->wmmat[3]; + } + if ((wm->wmtype == ROTZOOM || wm->wmtype == AFFINE) && x_scale == 16 && + y_scale == 16) { + int32_t *mat = wm->wmmat; + const int16_t alpha = wm->alpha; + const int16_t beta = wm->beta; + const int16_t gamma = wm->gamma; + const int16_t delta = wm->delta; + + av1_warp_affine(mat, ref, width, height, stride, pred, p_col, p_row, + p_width, p_height, p_stride, subsampling_x, subsampling_y, + ref_frm, alpha, beta, gamma, delta); + } else { + warp_plane_old(wm, ref, width, height, stride, pred, p_col, p_row, p_width, + p_height, p_stride, subsampling_x, subsampling_y, x_scale, + y_scale, ref_frm); + } +} + +static double warp_erroradv(WarpedMotionParams *wm, uint8_t *ref, int width, + int height, int stride, uint8_t *dst, int p_col, + int p_row, int p_width, int p_height, int p_stride, + int subsampling_x, int subsampling_y, int x_scale, + int y_scale) { + int gm_err = 0, no_gm_err = 0; + int gm_sumerr = 0, no_gm_sumerr = 0; + int i, j; + uint8_t *tmp = aom_malloc(p_width * p_height); + warp_plane(wm, ref, width, height, stride, tmp, p_col, p_row, p_width, + p_height, p_width, subsampling_x, subsampling_y, x_scale, y_scale, + 0); + + for (i = 0; i < p_height; ++i) { + for (j = 0; j < p_width; ++j) { + gm_err = dst[j + i * p_stride] - tmp[j + i * p_width]; + no_gm_err = + dst[j + i * p_stride] - ref[(j + p_col) + (i + p_row) * stride]; + gm_sumerr += error_measure(gm_err); + no_gm_sumerr += error_measure(no_gm_err); + } + } + + aom_free(tmp); + return (double)gm_sumerr / no_gm_sumerr; +} + +double av1_warp_erroradv(WarpedMotionParams *wm, +#if CONFIG_HIGHBITDEPTH + int use_hbd, int bd, +#endif // CONFIG_HIGHBITDEPTH + uint8_t *ref, int width, int height, int stride, + uint8_t *dst, int p_col, int p_row, int p_width, + int p_height, int p_stride, int subsampling_x, + int subsampling_y, int x_scale, int y_scale) { + if (wm->wmtype <= AFFINE) + if (!get_shear_params(wm)) return 1; +#if CONFIG_HIGHBITDEPTH + if (use_hbd) + return highbd_warp_erroradv( + wm, ref, width, height, stride, dst, p_col, p_row, p_width, p_height, + p_stride, subsampling_x, subsampling_y, x_scale, y_scale, bd); +#endif // CONFIG_HIGHBITDEPTH + return warp_erroradv(wm, ref, width, height, stride, dst, p_col, p_row, + p_width, p_height, p_stride, subsampling_x, + subsampling_y, x_scale, y_scale); +} + +void av1_warp_plane(WarpedMotionParams *wm, +#if CONFIG_HIGHBITDEPTH + int use_hbd, int bd, +#endif // CONFIG_HIGHBITDEPTH + uint8_t *ref, int width, int height, int stride, + uint8_t *pred, int p_col, int p_row, int p_width, + int p_height, int p_stride, int subsampling_x, + int subsampling_y, int x_scale, int y_scale, int ref_frm) { +#if CONFIG_HIGHBITDEPTH + if (use_hbd) + highbd_warp_plane(wm, ref, width, height, stride, pred, p_col, p_row, + p_width, p_height, p_stride, subsampling_x, subsampling_y, + x_scale, y_scale, bd, ref_frm); + else +#endif // CONFIG_HIGHBITDEPTH + warp_plane(wm, ref, width, height, stride, pred, p_col, p_row, p_width, + p_height, p_stride, subsampling_x, subsampling_y, x_scale, + y_scale, ref_frm); +} + +#if CONFIG_WARPED_MOTION +#define LEAST_SQUARES_ORDER 2 + +#define LS_MV_MAX 256 // max mv in 1/8-pel +#define LS_STEP 2 + +// Assuming LS_MV_MAX is < MAX_SB_SIZE * 8, +// the precision needed is: +// (MAX_SB_SIZE_LOG2 + 3) [for sx * sx magnitude] + +// (MAX_SB_SIZE_LOG2 + 4) [for sx * dx magnitude] + +// 1 [for sign] + +// LEAST_SQUARES_SAMPLES_MAX_BITS +// [for adding up to LEAST_SQUARES_SAMPLES_MAX samples] +// The value is 23 +#define LS_MAT_RANGE_BITS \ + ((MAX_SB_SIZE_LOG2 + 4) * 2 + LEAST_SQUARES_SAMPLES_MAX_BITS) + +// Bit-depth reduction from the full-range +#define LS_MAT_DOWN_BITS 2 + +// bits range of A, Bx and By after downshifting +#define LS_MAT_BITS (LS_MAT_RANGE_BITS - LS_MAT_DOWN_BITS) +#define LS_MAT_MIN (-(1 << (LS_MAT_BITS - 1))) +#define LS_MAT_MAX ((1 << (LS_MAT_BITS - 1)) - 1) + +#define LS_SUM(a) ((a)*4 + LS_STEP * 2) +#define LS_SQUARE(a) \ + (((a) * (a)*4 + (a)*4 * LS_STEP + LS_STEP * LS_STEP * 2) >> 2) +#define LS_PRODUCT1(a, b) \ + (((a) * (b)*4 + ((a) + (b)) * 2 * LS_STEP + LS_STEP * LS_STEP) >> 2) +#define LS_PRODUCT2(a, b) \ + (((a) * (b)*4 + ((a) + (b)) * 2 * LS_STEP + LS_STEP * LS_STEP * 2) >> 2) + +#if LEAST_SQUARES_ORDER == 2 +static int find_affine_int(int np, int *pts1, int *pts2, BLOCK_SIZE bsize, + int mvy, int mvx, WarpedMotionParams *wm, int mi_row, + int mi_col) { + int32_t A[2][2] = { { 0, 0 }, { 0, 0 } }; + int32_t Bx[2] = { 0, 0 }; + int32_t By[2] = { 0, 0 }; + int i, n = 0; + + const int bw = block_size_wide[bsize]; + const int bh = block_size_high[bsize]; + const int suy = (mi_row * MI_SIZE + AOMMAX(bh, MI_SIZE) / 2 - 1) * 8; + const int sux = (mi_col * MI_SIZE + AOMMAX(bw, MI_SIZE) / 2 - 1) * 8; + const int duy = suy + mvy; + const int dux = sux + mvx; + + // Assume the center pixel of the block has exactly the same motion vector + // as transmitted for the block. First shift the origin of the source + // points to the block center, and the origin of the destination points to + // the block center added to the motion vector transmitted. + // Let (xi, yi) denote the source points and (xi', yi') denote destination + // points after origin shfifting, for i = 0, 1, 2, .... n-1. + // Then if P = [x0, y0, + // x1, y1 + // x2, y1, + // .... + // ] + // q = [x0', x1', x2', ... ]' + // r = [y0', y1', y2', ... ]' + // the least squares problems that need to be solved are: + // [h1, h2]' = inv(P'P)P'q and + // [h3, h4]' = inv(P'P)P'r + // where the affine transformation is given by: + // x' = h1.x + h2.y + // y' = h3.x + h4.y + // + // The loop below computes: A = P'P, Bx = P'q, By = P'r + // We need to just compute inv(A).Bx and inv(A).By for the solutions. + int sx, sy, dx, dy; + // Contribution from neighbor block + for (i = 0; i < np && n < LEAST_SQUARES_SAMPLES_MAX; i++) { + dx = pts2[i * 2] - dux; + dy = pts2[i * 2 + 1] - duy; + sx = pts1[i * 2] - sux; + sy = pts1[i * 2 + 1] - suy; + if (abs(sx - dx) < LS_MV_MAX && abs(sy - dy) < LS_MV_MAX) { + A[0][0] += LS_SQUARE(sx); + A[0][1] += LS_PRODUCT1(sx, sy); + A[1][1] += LS_SQUARE(sy); + Bx[0] += LS_PRODUCT2(sx, dx); + Bx[1] += LS_PRODUCT1(sy, dx); + By[0] += LS_PRODUCT1(sx, dy); + By[1] += LS_PRODUCT2(sy, dy); + n++; + } + } + int downshift; + if (n >= 4) + downshift = LS_MAT_DOWN_BITS; + else if (n >= 2) + downshift = LS_MAT_DOWN_BITS - 1; + else + downshift = LS_MAT_DOWN_BITS - 2; + + // Reduce precision by downshift bits + A[0][0] = clamp(ROUND_POWER_OF_TWO_SIGNED(A[0][0], downshift), LS_MAT_MIN, + LS_MAT_MAX); + A[0][1] = clamp(ROUND_POWER_OF_TWO_SIGNED(A[0][1], downshift), LS_MAT_MIN, + LS_MAT_MAX); + A[1][1] = clamp(ROUND_POWER_OF_TWO_SIGNED(A[1][1], downshift), LS_MAT_MIN, + LS_MAT_MAX); + Bx[0] = clamp(ROUND_POWER_OF_TWO_SIGNED(Bx[0], downshift), LS_MAT_MIN, + LS_MAT_MAX); + Bx[1] = clamp(ROUND_POWER_OF_TWO_SIGNED(Bx[1], downshift), LS_MAT_MIN, + LS_MAT_MAX); + By[0] = clamp(ROUND_POWER_OF_TWO_SIGNED(By[0], downshift), LS_MAT_MIN, + LS_MAT_MAX); + By[1] = clamp(ROUND_POWER_OF_TWO_SIGNED(By[1], downshift), LS_MAT_MIN, + LS_MAT_MAX); + + int64_t Px[2], Py[2], Det; + int16_t iDet, shift; + + // These divided by the Det, are the least squares solutions + Px[0] = (int64_t)A[1][1] * Bx[0] - (int64_t)A[0][1] * Bx[1]; + Px[1] = -(int64_t)A[0][1] * Bx[0] + (int64_t)A[0][0] * Bx[1]; + Py[0] = (int64_t)A[1][1] * By[0] - (int64_t)A[0][1] * By[1]; + Py[1] = -(int64_t)A[0][1] * By[0] + (int64_t)A[0][0] * By[1]; + + // Compute Determinant of A + Det = (int64_t)A[0][0] * A[1][1] - (int64_t)A[0][1] * A[0][1]; + if (Det == 0) return 1; + iDet = resolve_divisor_64(llabs(Det), &shift) * (Det < 0 ? -1 : 1); + shift -= WARPEDMODEL_PREC_BITS; + if (shift < 0) { + iDet <<= (-shift); + shift = 0; + } + + int64_t v; + v = Px[0] * (int64_t)iDet; + wm->wmmat[2] = (int32_t)(ROUND_POWER_OF_TWO_SIGNED_64(v, shift)); + v = Px[1] * (int64_t)iDet; + wm->wmmat[3] = (int32_t)(ROUND_POWER_OF_TWO_SIGNED_64(v, shift)); + v = ((int64_t)dux * (1 << WARPEDMODEL_PREC_BITS)) - + (int64_t)sux * wm->wmmat[2] - (int64_t)suy * wm->wmmat[3]; + wm->wmmat[0] = (int32_t)(ROUND_POWER_OF_TWO_SIGNED(v, 3)); + + v = Py[0] * (int64_t)iDet; + wm->wmmat[4] = (int32_t)(ROUND_POWER_OF_TWO_SIGNED_64(v, shift)); + v = Py[1] * (int64_t)iDet; + wm->wmmat[5] = (int32_t)(ROUND_POWER_OF_TWO_SIGNED_64(v, shift)); + v = ((int64_t)duy * (1 << WARPEDMODEL_PREC_BITS)) - + (int64_t)sux * wm->wmmat[4] - (int64_t)suy * wm->wmmat[5]; + wm->wmmat[1] = (int32_t)(ROUND_POWER_OF_TWO_SIGNED(v, 3)); + + wm->wmmat[6] = wm->wmmat[7] = 0; + + // Clamp values + wm->wmmat[0] = clamp(wm->wmmat[0], -WARPEDMODEL_TRANS_CLAMP, + WARPEDMODEL_TRANS_CLAMP - 1); + wm->wmmat[1] = clamp(wm->wmmat[1], -WARPEDMODEL_TRANS_CLAMP, + WARPEDMODEL_TRANS_CLAMP - 1); + wm->wmmat[2] = clamp(wm->wmmat[2], -WARPEDMODEL_DIAGAFFINE_CLAMP, + WARPEDMODEL_DIAGAFFINE_CLAMP - 1); + wm->wmmat[5] = clamp(wm->wmmat[5], -WARPEDMODEL_DIAGAFFINE_CLAMP, + WARPEDMODEL_DIAGAFFINE_CLAMP - 1); + wm->wmmat[3] = clamp(wm->wmmat[3], -WARPEDMODEL_NONDIAGAFFINE_CLAMP, + WARPEDMODEL_NONDIAGAFFINE_CLAMP - 1); + wm->wmmat[4] = clamp(wm->wmmat[4], -WARPEDMODEL_NONDIAGAFFINE_CLAMP, + WARPEDMODEL_NONDIAGAFFINE_CLAMP - 1); + return 0; +} + +#else + +static int find_affine_int(int np, int *pts1, int *pts2, BLOCK_SIZE bsize, + int mvy, int mvx, WarpedMotionParams *wm, int mi_row, + int mi_col) { + int32_t A[3][3] = { { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 } }; + int32_t Bx[3] = { 0, 0, 0 }; + int32_t By[3] = { 0, 0, 0 }; + int i, n = 0, off; + + int64_t C00, C01, C02, C11, C12, C22; + int64_t Px[3], Py[3]; + int64_t Det, v; + const int bw = block_size_wide[bsize]; + const int bh = block_size_high[bsize]; + const int cy_offset = AOMMAX(bh, MI_SIZE) / 2 - 1; + const int cx_offset = AOMMAX(bw, MI_SIZE) / 2 - 1; + + // Offsets to make the values in the arrays smaller + const int ux = mi_col * MI_SIZE * 8, uy = mi_row * MI_SIZE * 8; + // Let source points (xi, yi) map to destimation points (xi', yi'), + // for i = 0, 1, 2, .... n-1 + // Then if P = [x0, y0, 1, + // x1, y1, 1 + // x2, y2, 1, + // .... + // ] + // q = [x0', x1', x2', ... ]' + // r = [y0', y1', y2', ... ]' + // the least squares problems that need to be solved are: + // [h1, h2, dx]' = inv(P'P)P'q and + // [h3, h4, dy]' = inv(P'P)P'r + // where the affine transformation is given by: + // x' = h1.x + h2.y + dx + // y' = h3.x + h4.y + dy + // + // The loop below computes: A = P'P, Bx = P'q, By = P'r + // We need to just compute inv(A).Bx and inv(A).By for the solutions. + // + int sx, sy, dx, dy; + // Contribution from sample in current block + sx = cx_offset * 8; + sy = cy_offset * 8; + dx = sx + mvx; + dy = sy + mvy; + if (abs(sx - dx) < LS_MV_MAX && abs(sy - dy) < LS_MV_MAX) { + A[0][0] += LS_SQUARE(sx); + A[0][1] += LS_PRODUCT1(sx, sy); + A[0][2] += LS_SUM(sx); + A[1][1] += LS_SQUARE(sy); + A[1][2] += LS_SUM(sy); + A[2][2] += 4; + Bx[0] += LS_PRODUCT2(sx, dx); + Bx[1] += LS_PRODUCT1(sy, dx); + Bx[2] += LS_SUM(dx); + By[0] += LS_PRODUCT1(sx, dy); + By[1] += LS_PRODUCT2(sy, dy); + By[2] += LS_SUM(dy); + n++; + } + // Contribution from neighbor block + for (i = 0; i < np && n < LEAST_SQUARES_SAMPLES_MAX; i++) { + dx = pts2[i * 2] - ux; + dy = pts2[i * 2 + 1] - uy; + sx = pts1[i * 2] - ux; + sy = pts1[i * 2 + 1] - uy; + if (abs(sx - dx) < LS_MV_MAX && abs(sy - dy) < LS_MV_MAX) { + A[0][0] += LS_SQUARE(sx); + A[0][1] += LS_PRODUCT1(sx, sy); + A[0][2] += LS_SUM(sx); + A[1][1] += LS_SQUARE(sy); + A[1][2] += LS_SUM(sy); + A[2][2] += 4; + Bx[0] += LS_PRODUCT2(sx, dx); + Bx[1] += LS_PRODUCT1(sy, dx); + Bx[2] += LS_SUM(dx); + By[0] += LS_PRODUCT1(sx, dy); + By[1] += LS_PRODUCT2(sy, dy); + By[2] += LS_SUM(dy); + n++; + } + } + // Compute Cofactors of A + C00 = (int64_t)A[1][1] * A[2][2] - (int64_t)A[1][2] * A[1][2]; + C01 = (int64_t)A[1][2] * A[0][2] - (int64_t)A[0][1] * A[2][2]; + C02 = (int64_t)A[0][1] * A[1][2] - (int64_t)A[0][2] * A[1][1]; + C11 = (int64_t)A[0][0] * A[2][2] - (int64_t)A[0][2] * A[0][2]; + C12 = (int64_t)A[0][1] * A[0][2] - (int64_t)A[0][0] * A[1][2]; + C22 = (int64_t)A[0][0] * A[1][1] - (int64_t)A[0][1] * A[0][1]; + + // Scale by 1/64 + C00 = ROUND_POWER_OF_TWO_SIGNED(C00, 6); + C01 = ROUND_POWER_OF_TWO_SIGNED(C01, 6); + C02 = ROUND_POWER_OF_TWO_SIGNED(C02, 6); + C11 = ROUND_POWER_OF_TWO_SIGNED(C11, 6); + C12 = ROUND_POWER_OF_TWO_SIGNED(C12, 6); + C22 = ROUND_POWER_OF_TWO_SIGNED(C22, 6); + + // Compute Determinant of A + Det = C00 * A[0][0] + C01 * A[0][1] + C02 * A[0][2]; + if (Det == 0) return 1; + + // These divided by the Det, are the least squares solutions + Px[0] = C00 * Bx[0] + C01 * Bx[1] + C02 * Bx[2]; + Px[1] = C01 * Bx[0] + C11 * Bx[1] + C12 * Bx[2]; + Px[2] = C02 * Bx[0] + C12 * Bx[1] + C22 * Bx[2]; + Py[0] = C00 * By[0] + C01 * By[1] + C02 * By[2]; + Py[1] = C01 * By[0] + C11 * By[1] + C12 * By[2]; + Py[2] = C02 * By[0] + C12 * By[1] + C22 * By[2]; + + int16_t shift; + int64_t iDet; + iDet = resolve_divisor_64(llabs(Det), &shift) * (Det < 0 ? -1 : 1); + shift -= WARPEDMODEL_PREC_BITS; + if (shift < 0) { + iDet <<= (-shift); + shift = 0; + } + + v = Px[0] * iDet; + wm->wmmat[2] = ROUND_POWER_OF_TWO_SIGNED_64(v, shift); + v = Px[1] * iDet; + wm->wmmat[3] = ROUND_POWER_OF_TWO_SIGNED_64(v, shift); + v = Px[2] * iDet; + wm->wmmat[0] = ROUND_POWER_OF_TWO_SIGNED_64(v, shift + 3); + // Adjust x displacement for the offset + off = (ux << WARPEDMODEL_PREC_BITS) - ux * wm->wmmat[2] - uy * wm->wmmat[3]; + wm->wmmat[0] += ROUND_POWER_OF_TWO_SIGNED(off, 3); + + v = Py[0] * iDet; + wm->wmmat[4] = ROUND_POWER_OF_TWO_SIGNED_64(v, shift); + v = Py[1] * iDet; + wm->wmmat[5] = ROUND_POWER_OF_TWO_SIGNED_64(v, shift); + v = Py[2] * iDet; + wm->wmmat[1] = ROUND_POWER_OF_TWO_SIGNED_64(v, shift + 3); + // Adjust y displacement for the offset + off = (uy << WARPEDMODEL_PREC_BITS) - ux * wm->wmmat[4] - uy * wm->wmmat[5]; + wm->wmmat[1] += ROUND_POWER_OF_TWO_SIGNED(off, 3); + wm->wmmat[6] = wm->wmmat[7] = 0; + + // Clamp values + wm->wmmat[0] = clamp(wm->wmmat[0], -WARPEDMODEL_TRANS_CLAMP, + WARPEDMODEL_TRANS_CLAMP - 1); + wm->wmmat[1] = clamp(wm->wmmat[1], -WARPEDMODEL_TRANS_CLAMP, + WARPEDMODEL_TRANS_CLAMP - 1); + wm->wmmat[2] = clamp(wm->wmmat[2], -WARPEDMODEL_DIAGAFFINE_CLAMP, + WARPEDMODEL_DIAGAFFINE_CLAMP - 1); + wm->wmmat[5] = clamp(wm->wmmat[5], -WARPEDMODEL_DIAGAFFINE_CLAMP, + WARPEDMODEL_DIAGAFFINE_CLAMP - 1); + wm->wmmat[3] = clamp(wm->wmmat[3], -WARPEDMODEL_NONDIAGAFFINE_CLAMP, + WARPEDMODEL_NONDIAGAFFINE_CLAMP - 1); + wm->wmmat[4] = clamp(wm->wmmat[4], -WARPEDMODEL_NONDIAGAFFINE_CLAMP, + WARPEDMODEL_NONDIAGAFFINE_CLAMP - 1); + + return 0; +} +#endif // LEAST_SQUARES_ORDER == 2 + +int find_projection(int np, int *pts1, int *pts2, BLOCK_SIZE bsize, int mvy, + int mvx, WarpedMotionParams *wm_params, int mi_row, + int mi_col) { + int result = 1; + switch (wm_params->wmtype) { + case AFFINE: + result = find_affine_int(np, pts1, pts2, bsize, mvy, mvx, wm_params, + mi_row, mi_col); + break; + default: assert(0 && "Invalid warped motion type!"); return 1; + } + if (result == 0) { + if (wm_params->wmtype == ROTZOOM) { + wm_params->wmmat[5] = wm_params->wmmat[2]; + wm_params->wmmat[4] = -wm_params->wmmat[3]; + } + if (wm_params->wmtype == AFFINE || wm_params->wmtype == ROTZOOM) { + // check compatibility with the fast warp filter + if (!get_shear_params(wm_params)) return 1; + } + } + + return result; +} +#endif // CONFIG_WARPED_MOTION diff --git a/third_party/aom/av1/common/warped_motion.h b/third_party/aom/av1/common/warped_motion.h new file mode 100644 index 0000000000..dfd8dae34b --- /dev/null +++ b/third_party/aom/av1/common/warped_motion.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_COMMON_WARPED_MOTION_H_ +#define AV1_COMMON_WARPED_MOTION_H_ + +#include +#include +#include +#include +#include + +#include "./aom_config.h" +#include "aom_ports/mem.h" +#include "aom_dsp/aom_dsp_common.h" +#include "av1/common/mv.h" + +#define MAX_PARAMDIM 9 +#if CONFIG_WARPED_MOTION +#define SAMPLES_ARRAY_SIZE ((2 * MAX_MIB_SIZE + 2) * 2) + +#define LEAST_SQUARES_SAMPLES_MAX_BITS 3 +#define LEAST_SQUARES_SAMPLES_MAX (1 << LEAST_SQUARES_SAMPLES_MAX_BITS) + +#define DEFAULT_WMTYPE AFFINE +#endif // CONFIG_WARPED_MOTION + +extern const int16_t warped_filter[WARPEDPIXEL_PREC_SHIFTS * 3 + 1][8]; + +typedef void (*ProjectPointsFunc)(int32_t *mat, int *points, int *proj, + const int n, const int stride_points, + const int stride_proj, + const int subsampling_x, + const int subsampling_y); + +void project_points_translation(int32_t *mat, int *points, int *proj, + const int n, const int stride_points, + const int stride_proj, const int subsampling_x, + const int subsampling_y); + +void project_points_rotzoom(int32_t *mat, int *points, int *proj, const int n, + const int stride_points, const int stride_proj, + const int subsampling_x, const int subsampling_y); + +void project_points_affine(int32_t *mat, int *points, int *proj, const int n, + const int stride_points, const int stride_proj, + const int subsampling_x, const int subsampling_y); + +void project_points_hortrapezoid(int32_t *mat, int *points, int *proj, + const int n, const int stride_points, + const int stride_proj, const int subsampling_x, + const int subsampling_y); +void project_points_vertrapezoid(int32_t *mat, int *points, int *proj, + const int n, const int stride_points, + const int stride_proj, const int subsampling_x, + const int subsampling_y); +void project_points_homography(int32_t *mat, int *points, int *proj, + const int n, const int stride_points, + const int stride_proj, const int subsampling_x, + const int subsampling_y); + +void project_points(WarpedMotionParams *wm_params, int *points, int *proj, + const int n, const int stride_points, const int stride_proj, + const int subsampling_x, const int subsampling_y); + +double av1_warp_erroradv(WarpedMotionParams *wm, +#if CONFIG_HIGHBITDEPTH + int use_hbd, int bd, +#endif // CONFIG_HIGHBITDEPTH + uint8_t *ref, int width, int height, int stride, + uint8_t *dst, int p_col, int p_row, int p_width, + int p_height, int p_stride, int subsampling_x, + int subsampling_y, int x_scale, int y_scale); + +void av1_warp_plane(WarpedMotionParams *wm, +#if CONFIG_HIGHBITDEPTH + int use_hbd, int bd, +#endif // CONFIG_HIGHBITDEPTH + uint8_t *ref, int width, int height, int stride, + uint8_t *pred, int p_col, int p_row, int p_width, + int p_height, int p_stride, int subsampling_x, + int subsampling_y, int x_scale, int y_scale, int ref_frm); + +int find_projection(int np, int *pts1, int *pts2, BLOCK_SIZE bsize, int mvy, + int mvx, WarpedMotionParams *wm_params, int mi_row, + int mi_col); + +int get_shear_params(WarpedMotionParams *wm); +#endif // AV1_COMMON_WARPED_MOTION_H_ diff --git a/third_party/aom/av1/common/x86/av1_convolve_ssse3.c b/third_party/aom/av1/common/x86/av1_convolve_ssse3.c new file mode 100644 index 0000000000..91102bbaf5 --- /dev/null +++ b/third_party/aom/av1/common/x86/av1_convolve_ssse3.c @@ -0,0 +1,1029 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "av1/common/filter.h" + +#define WIDTH_BOUND (16) +#define HEIGHT_BOUND (16) + +#if CONFIG_DUAL_FILTER +DECLARE_ALIGNED(16, static int8_t, + sub_pel_filters_12sharp_signal_dir[15][2][16]); + +DECLARE_ALIGNED(16, static int8_t, + sub_pel_filters_12sharp_ver_signal_dir[15][6][16]); +#endif // CONFIG_DUAL_FILTER + +#if USE_TEMPORALFILTER_12TAP +DECLARE_ALIGNED(16, static int8_t, + sub_pel_filters_temporalfilter_12_signal_dir[15][2][16]); + +DECLARE_ALIGNED(16, static int8_t, + sub_pel_filters_temporalfilter_12_ver_signal_dir[15][6][16]); +#endif + +typedef int8_t (*SubpelFilterCoeffs)[16]; + +static INLINE SubpelFilterCoeffs +get_subpel_filter_signal_dir(const InterpFilterParams p, int index) { +#if CONFIG_DUAL_FILTER + if (p.interp_filter == MULTITAP_SHARP) { + return &sub_pel_filters_12sharp_signal_dir[index][0]; + } +#endif +#if USE_TEMPORALFILTER_12TAP + if (p.interp_filter == TEMPORALFILTER_12TAP) { + return &sub_pel_filters_temporalfilter_12_signal_dir[index][0]; + } +#endif + (void)p; + (void)index; + return NULL; +} + +static INLINE SubpelFilterCoeffs +get_subpel_filter_ver_signal_dir(const InterpFilterParams p, int index) { +#if CONFIG_DUAL_FILTER + if (p.interp_filter == MULTITAP_SHARP) { + return &sub_pel_filters_12sharp_ver_signal_dir[index][0]; + } +#endif +#if USE_TEMPORALFILTER_12TAP + if (p.interp_filter == TEMPORALFILTER_12TAP) { + return &sub_pel_filters_temporalfilter_12_ver_signal_dir[index][0]; + } +#endif + (void)p; + (void)index; + return NULL; +} + +static INLINE void transpose_4x8(const __m128i *in, __m128i *out) { + __m128i t0, t1; + + t0 = _mm_unpacklo_epi16(in[0], in[1]); + t1 = _mm_unpacklo_epi16(in[2], in[3]); + + out[0] = _mm_unpacklo_epi32(t0, t1); + out[1] = _mm_srli_si128(out[0], 8); + out[2] = _mm_unpackhi_epi32(t0, t1); + out[3] = _mm_srli_si128(out[2], 8); + + t0 = _mm_unpackhi_epi16(in[0], in[1]); + t1 = _mm_unpackhi_epi16(in[2], in[3]); + + out[4] = _mm_unpacklo_epi32(t0, t1); + out[5] = _mm_srli_si128(out[4], 8); + // Note: We ignore out[6] and out[7] because + // they're zero vectors. +} + +typedef void (*store_pixel_t)(const __m128i *x, uint8_t *dst); + +static INLINE __m128i accumulate_store(const __m128i *x, uint8_t *src) { + const __m128i zero = _mm_setzero_si128(); + const __m128i one = _mm_set1_epi16(1); + __m128i y = _mm_loadl_epi64((__m128i const *)src); + y = _mm_unpacklo_epi8(y, zero); + y = _mm_add_epi16(*x, y); + y = _mm_add_epi16(y, one); + y = _mm_srai_epi16(y, 1); + y = _mm_packus_epi16(y, y); + return y; +} + +static INLINE void store_2_pixel_only(const __m128i *x, uint8_t *dst) { + uint32_t temp; + __m128i u = _mm_packus_epi16(*x, *x); + temp = _mm_cvtsi128_si32(u); + *(uint16_t *)dst = (uint16_t)temp; +} + +static INLINE void accumulate_store_2_pixel(const __m128i *x, uint8_t *dst) { + uint32_t temp; + __m128i y = accumulate_store(x, dst); + temp = _mm_cvtsi128_si32(y); + *(uint16_t *)dst = (uint16_t)temp; +} + +static store_pixel_t store2pixelTab[2] = { store_2_pixel_only, + accumulate_store_2_pixel }; + +static INLINE void store_4_pixel_only(const __m128i *x, uint8_t *dst) { + __m128i u = _mm_packus_epi16(*x, *x); + *(int *)dst = _mm_cvtsi128_si32(u); +} + +static INLINE void accumulate_store_4_pixel(const __m128i *x, uint8_t *dst) { + __m128i y = accumulate_store(x, dst); + *(int *)dst = _mm_cvtsi128_si32(y); +} + +static store_pixel_t store4pixelTab[2] = { store_4_pixel_only, + accumulate_store_4_pixel }; + +static void horiz_w4_ssse3(const uint8_t *src, const __m128i *f, int tapsNum, + store_pixel_t store_func, uint8_t *dst) { + __m128i sumPairRow[4]; + __m128i sumPairCol[8]; + __m128i pixel; + const __m128i k_256 = _mm_set1_epi16(1 << 8); + const __m128i zero = _mm_setzero_si128(); + + if (10 == tapsNum) { + src -= 1; + } + + pixel = _mm_loadu_si128((__m128i const *)src); + sumPairRow[0] = _mm_maddubs_epi16(pixel, f[0]); + sumPairRow[2] = _mm_maddubs_epi16(pixel, f[1]); + sumPairRow[2] = _mm_srli_si128(sumPairRow[2], 2); + + pixel = _mm_loadu_si128((__m128i const *)(src + 1)); + sumPairRow[1] = _mm_maddubs_epi16(pixel, f[0]); + sumPairRow[3] = _mm_maddubs_epi16(pixel, f[1]); + sumPairRow[3] = _mm_srli_si128(sumPairRow[3], 2); + + transpose_4x8(sumPairRow, sumPairCol); + + sumPairRow[0] = _mm_adds_epi16(sumPairCol[0], sumPairCol[1]); + sumPairRow[1] = _mm_adds_epi16(sumPairCol[4], sumPairCol[5]); + + sumPairRow[2] = _mm_min_epi16(sumPairCol[2], sumPairCol[3]); + sumPairRow[3] = _mm_max_epi16(sumPairCol[2], sumPairCol[3]); + + sumPairRow[0] = _mm_adds_epi16(sumPairRow[0], sumPairRow[1]); + sumPairRow[0] = _mm_adds_epi16(sumPairRow[0], sumPairRow[2]); + sumPairRow[0] = _mm_adds_epi16(sumPairRow[0], sumPairRow[3]); + + sumPairRow[1] = _mm_mulhrs_epi16(sumPairRow[0], k_256); + sumPairRow[1] = _mm_packus_epi16(sumPairRow[1], sumPairRow[1]); + sumPairRow[1] = _mm_unpacklo_epi8(sumPairRow[1], zero); + + store_func(&sumPairRow[1], dst); +} + +static void horiz_w8_ssse3(const uint8_t *src, const __m128i *f, int tapsNum, + store_pixel_t store, uint8_t *buf) { + horiz_w4_ssse3(src, f, tapsNum, store, buf); + src += 4; + buf += 4; + horiz_w4_ssse3(src, f, tapsNum, store, buf); +} + +static void horiz_w16_ssse3(const uint8_t *src, const __m128i *f, int tapsNum, + store_pixel_t store, uint8_t *buf) { + horiz_w8_ssse3(src, f, tapsNum, store, buf); + src += 8; + buf += 8; + horiz_w8_ssse3(src, f, tapsNum, store, buf); +} + +static void horiz_w32_ssse3(const uint8_t *src, const __m128i *f, int tapsNum, + store_pixel_t store, uint8_t *buf) { + horiz_w16_ssse3(src, f, tapsNum, store, buf); + src += 16; + buf += 16; + horiz_w16_ssse3(src, f, tapsNum, store, buf); +} + +static void horiz_w64_ssse3(const uint8_t *src, const __m128i *f, int tapsNum, + store_pixel_t store, uint8_t *buf) { + horiz_w32_ssse3(src, f, tapsNum, store, buf); + src += 32; + buf += 32; + horiz_w32_ssse3(src, f, tapsNum, store, buf); +} + +static void horiz_w128_ssse3(const uint8_t *src, const __m128i *f, int tapsNum, + store_pixel_t store, uint8_t *buf) { + horiz_w64_ssse3(src, f, tapsNum, store, buf); + src += 64; + buf += 64; + horiz_w64_ssse3(src, f, tapsNum, store, buf); +} + +static void (*horizTab[6])(const uint8_t *, const __m128i *, int, store_pixel_t, + uint8_t *) = { + horiz_w4_ssse3, horiz_w8_ssse3, horiz_w16_ssse3, + horiz_w32_ssse3, horiz_w64_ssse3, horiz_w128_ssse3, +}; + +static void filter_horiz_ssse3(const uint8_t *src, __m128i *f, int tapsNum, + int width, store_pixel_t store, uint8_t *dst) { + switch (width) { + // Note: + // For width=2 and 4, store function must be different + case 2: + case 4: horizTab[0](src, f, tapsNum, store, dst); break; + case 8: horizTab[1](src, f, tapsNum, store, dst); break; + case 16: horizTab[2](src, f, tapsNum, store, dst); break; + case 32: horizTab[3](src, f, tapsNum, store, dst); break; + case 64: horizTab[4](src, f, tapsNum, store, dst); break; + case 128: horizTab[5](src, f, tapsNum, store, dst); break; + default: assert(0); + } +} + +// Vertical 8-pixel parallel +typedef void (*transpose_to_dst_t)(const uint16_t *src, int src_stride, + uint8_t *dst, int dst_stride); + +static INLINE void transpose8x8_direct_to_dst(const uint16_t *src, + int src_stride, uint8_t *dst, + int dst_stride) { + const __m128i k_256 = _mm_set1_epi16(1 << 8); + __m128i v0, v1, v2, v3; + + __m128i u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride)); + __m128i u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride)); + __m128i u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride)); + __m128i u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride)); + __m128i u4 = _mm_loadu_si128((__m128i const *)(src + 4 * src_stride)); + __m128i u5 = _mm_loadu_si128((__m128i const *)(src + 5 * src_stride)); + __m128i u6 = _mm_loadu_si128((__m128i const *)(src + 6 * src_stride)); + __m128i u7 = _mm_loadu_si128((__m128i const *)(src + 7 * src_stride)); + + u0 = _mm_mulhrs_epi16(u0, k_256); + u1 = _mm_mulhrs_epi16(u1, k_256); + u2 = _mm_mulhrs_epi16(u2, k_256); + u3 = _mm_mulhrs_epi16(u3, k_256); + u4 = _mm_mulhrs_epi16(u4, k_256); + u5 = _mm_mulhrs_epi16(u5, k_256); + u6 = _mm_mulhrs_epi16(u6, k_256); + u7 = _mm_mulhrs_epi16(u7, k_256); + + v0 = _mm_packus_epi16(u0, u1); + v1 = _mm_packus_epi16(u2, u3); + v2 = _mm_packus_epi16(u4, u5); + v3 = _mm_packus_epi16(u6, u7); + + u0 = _mm_unpacklo_epi8(v0, v1); + u1 = _mm_unpackhi_epi8(v0, v1); + u2 = _mm_unpacklo_epi8(v2, v3); + u3 = _mm_unpackhi_epi8(v2, v3); + + u4 = _mm_unpacklo_epi8(u0, u1); + u5 = _mm_unpacklo_epi8(u2, u3); + u6 = _mm_unpackhi_epi8(u0, u1); + u7 = _mm_unpackhi_epi8(u2, u3); + + u0 = _mm_unpacklo_epi32(u4, u5); + u1 = _mm_unpackhi_epi32(u4, u5); + u2 = _mm_unpacklo_epi32(u6, u7); + u3 = _mm_unpackhi_epi32(u6, u7); + + u4 = _mm_srli_si128(u0, 8); + u5 = _mm_srli_si128(u1, 8); + u6 = _mm_srli_si128(u2, 8); + u7 = _mm_srli_si128(u3, 8); + + _mm_storel_epi64((__m128i *)dst, u0); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 1), u4); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 2), u1); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 3), u5); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 4), u2); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 5), u6); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 6), u3); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 7), u7); +} + +static INLINE void transpose8x8_accumu_to_dst(const uint16_t *src, + int src_stride, uint8_t *dst, + int dst_stride) { + const __m128i k_256 = _mm_set1_epi16(1 << 8); + const __m128i zero = _mm_setzero_si128(); + const __m128i one = _mm_set1_epi16(1); + __m128i v0, v1, v2, v3, v4, v5, v6, v7; + + __m128i u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride)); + __m128i u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride)); + __m128i u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride)); + __m128i u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride)); + __m128i u4 = _mm_loadu_si128((__m128i const *)(src + 4 * src_stride)); + __m128i u5 = _mm_loadu_si128((__m128i const *)(src + 5 * src_stride)); + __m128i u6 = _mm_loadu_si128((__m128i const *)(src + 6 * src_stride)); + __m128i u7 = _mm_loadu_si128((__m128i const *)(src + 7 * src_stride)); + + u0 = _mm_mulhrs_epi16(u0, k_256); + u1 = _mm_mulhrs_epi16(u1, k_256); + u2 = _mm_mulhrs_epi16(u2, k_256); + u3 = _mm_mulhrs_epi16(u3, k_256); + u4 = _mm_mulhrs_epi16(u4, k_256); + u5 = _mm_mulhrs_epi16(u5, k_256); + u6 = _mm_mulhrs_epi16(u6, k_256); + u7 = _mm_mulhrs_epi16(u7, k_256); + + v0 = _mm_packus_epi16(u0, u1); + v1 = _mm_packus_epi16(u2, u3); + v2 = _mm_packus_epi16(u4, u5); + v3 = _mm_packus_epi16(u6, u7); + + u0 = _mm_unpacklo_epi8(v0, v1); + u1 = _mm_unpackhi_epi8(v0, v1); + u2 = _mm_unpacklo_epi8(v2, v3); + u3 = _mm_unpackhi_epi8(v2, v3); + + u4 = _mm_unpacklo_epi8(u0, u1); + u5 = _mm_unpacklo_epi8(u2, u3); + u6 = _mm_unpackhi_epi8(u0, u1); + u7 = _mm_unpackhi_epi8(u2, u3); + + u0 = _mm_unpacklo_epi32(u4, u5); + u1 = _mm_unpackhi_epi32(u4, u5); + u2 = _mm_unpacklo_epi32(u6, u7); + u3 = _mm_unpackhi_epi32(u6, u7); + + u4 = _mm_srli_si128(u0, 8); + u5 = _mm_srli_si128(u1, 8); + u6 = _mm_srli_si128(u2, 8); + u7 = _mm_srli_si128(u3, 8); + + v0 = _mm_loadl_epi64((__m128i const *)(dst + 0 * dst_stride)); + v1 = _mm_loadl_epi64((__m128i const *)(dst + 1 * dst_stride)); + v2 = _mm_loadl_epi64((__m128i const *)(dst + 2 * dst_stride)); + v3 = _mm_loadl_epi64((__m128i const *)(dst + 3 * dst_stride)); + v4 = _mm_loadl_epi64((__m128i const *)(dst + 4 * dst_stride)); + v5 = _mm_loadl_epi64((__m128i const *)(dst + 5 * dst_stride)); + v6 = _mm_loadl_epi64((__m128i const *)(dst + 6 * dst_stride)); + v7 = _mm_loadl_epi64((__m128i const *)(dst + 7 * dst_stride)); + + u0 = _mm_unpacklo_epi8(u0, zero); + u1 = _mm_unpacklo_epi8(u1, zero); + u2 = _mm_unpacklo_epi8(u2, zero); + u3 = _mm_unpacklo_epi8(u3, zero); + u4 = _mm_unpacklo_epi8(u4, zero); + u5 = _mm_unpacklo_epi8(u5, zero); + u6 = _mm_unpacklo_epi8(u6, zero); + u7 = _mm_unpacklo_epi8(u7, zero); + + v0 = _mm_unpacklo_epi8(v0, zero); + v1 = _mm_unpacklo_epi8(v1, zero); + v2 = _mm_unpacklo_epi8(v2, zero); + v3 = _mm_unpacklo_epi8(v3, zero); + v4 = _mm_unpacklo_epi8(v4, zero); + v5 = _mm_unpacklo_epi8(v5, zero); + v6 = _mm_unpacklo_epi8(v6, zero); + v7 = _mm_unpacklo_epi8(v7, zero); + + v0 = _mm_adds_epi16(u0, v0); + v1 = _mm_adds_epi16(u4, v1); + v2 = _mm_adds_epi16(u1, v2); + v3 = _mm_adds_epi16(u5, v3); + v4 = _mm_adds_epi16(u2, v4); + v5 = _mm_adds_epi16(u6, v5); + v6 = _mm_adds_epi16(u3, v6); + v7 = _mm_adds_epi16(u7, v7); + + v0 = _mm_adds_epi16(v0, one); + v1 = _mm_adds_epi16(v1, one); + v2 = _mm_adds_epi16(v2, one); + v3 = _mm_adds_epi16(v3, one); + v4 = _mm_adds_epi16(v4, one); + v5 = _mm_adds_epi16(v5, one); + v6 = _mm_adds_epi16(v6, one); + v7 = _mm_adds_epi16(v7, one); + + v0 = _mm_srai_epi16(v0, 1); + v1 = _mm_srai_epi16(v1, 1); + v2 = _mm_srai_epi16(v2, 1); + v3 = _mm_srai_epi16(v3, 1); + v4 = _mm_srai_epi16(v4, 1); + v5 = _mm_srai_epi16(v5, 1); + v6 = _mm_srai_epi16(v6, 1); + v7 = _mm_srai_epi16(v7, 1); + + u0 = _mm_packus_epi16(v0, v1); + u1 = _mm_packus_epi16(v2, v3); + u2 = _mm_packus_epi16(v4, v5); + u3 = _mm_packus_epi16(v6, v7); + + u4 = _mm_srli_si128(u0, 8); + u5 = _mm_srli_si128(u1, 8); + u6 = _mm_srli_si128(u2, 8); + u7 = _mm_srli_si128(u3, 8); + + _mm_storel_epi64((__m128i *)dst, u0); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 1), u4); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 2), u1); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 3), u5); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 4), u2); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 5), u6); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 6), u3); + _mm_storel_epi64((__m128i *)(dst + dst_stride * 7), u7); +} + +static transpose_to_dst_t trans8x8Tab[2] = { transpose8x8_direct_to_dst, + transpose8x8_accumu_to_dst }; + +static INLINE void transpose_8x16(const __m128i *in, __m128i *out) { + __m128i t0, t1, t2, t3, u0, u1; + + t0 = _mm_unpacklo_epi16(in[0], in[1]); + t1 = _mm_unpacklo_epi16(in[2], in[3]); + t2 = _mm_unpacklo_epi16(in[4], in[5]); + t3 = _mm_unpacklo_epi16(in[6], in[7]); + + u0 = _mm_unpacklo_epi32(t0, t1); + u1 = _mm_unpacklo_epi32(t2, t3); + + out[0] = _mm_unpacklo_epi64(u0, u1); + out[1] = _mm_unpackhi_epi64(u0, u1); + + u0 = _mm_unpackhi_epi32(t0, t1); + u1 = _mm_unpackhi_epi32(t2, t3); + + out[2] = _mm_unpacklo_epi64(u0, u1); + out[3] = _mm_unpackhi_epi64(u0, u1); + + t0 = _mm_unpackhi_epi16(in[0], in[1]); + t1 = _mm_unpackhi_epi16(in[2], in[3]); + t2 = _mm_unpackhi_epi16(in[4], in[5]); + t3 = _mm_unpackhi_epi16(in[6], in[7]); + + u0 = _mm_unpacklo_epi32(t0, t1); + u1 = _mm_unpacklo_epi32(t2, t3); + + out[4] = _mm_unpacklo_epi64(u0, u1); + out[5] = _mm_unpackhi_epi64(u0, u1); + + // Ignore out[6] and out[7] + // they're zero vectors. +} + +static void filter_horiz_v8p_ssse3(const uint8_t *src_ptr, ptrdiff_t src_pitch, + __m128i *f, int tapsNum, uint16_t *buf) { + __m128i s[8], t[6]; + __m128i min_x2x3, max_x2x3; + __m128i temp; + + if (tapsNum == 10) { + src_ptr -= 1; + } + s[0] = _mm_loadu_si128((const __m128i *)src_ptr); + s[1] = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch)); + s[2] = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2)); + s[3] = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3)); + s[4] = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4)); + s[5] = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5)); + s[6] = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6)); + s[7] = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 7)); + + // TRANSPOSE... + // Vecotor represents column pixel pairs instead of a row + transpose_8x16(s, t); + + // multiply 2 adjacent elements with the filter and add the result + s[0] = _mm_maddubs_epi16(t[0], f[0]); + s[1] = _mm_maddubs_epi16(t[1], f[1]); + s[2] = _mm_maddubs_epi16(t[2], f[2]); + s[3] = _mm_maddubs_epi16(t[3], f[3]); + s[4] = _mm_maddubs_epi16(t[4], f[4]); + s[5] = _mm_maddubs_epi16(t[5], f[5]); + + // add and saturate the results together + min_x2x3 = _mm_min_epi16(s[2], s[3]); + max_x2x3 = _mm_max_epi16(s[2], s[3]); + temp = _mm_adds_epi16(s[0], s[1]); + temp = _mm_adds_epi16(temp, s[5]); + temp = _mm_adds_epi16(temp, s[4]); + + temp = _mm_adds_epi16(temp, min_x2x3); + temp = _mm_adds_epi16(temp, max_x2x3); + + _mm_storeu_si128((__m128i *)buf, temp); +} + +// Vertical 4-pixel parallel +static INLINE void transpose4x4_direct_to_dst(const uint16_t *src, + int src_stride, uint8_t *dst, + int dst_stride) { + const __m128i k_256 = _mm_set1_epi16(1 << 8); + __m128i v0, v1, v2, v3; + + // TODO(luoyi): two loads, 8 elements per load (two bytes per element) + __m128i u0 = _mm_loadl_epi64((__m128i const *)(src + 0 * src_stride)); + __m128i u1 = _mm_loadl_epi64((__m128i const *)(src + 1 * src_stride)); + __m128i u2 = _mm_loadl_epi64((__m128i const *)(src + 2 * src_stride)); + __m128i u3 = _mm_loadl_epi64((__m128i const *)(src + 3 * src_stride)); + + v0 = _mm_unpacklo_epi16(u0, u1); + v1 = _mm_unpacklo_epi16(u2, u3); + + v2 = _mm_unpacklo_epi32(v0, v1); + v3 = _mm_unpackhi_epi32(v0, v1); + + u0 = _mm_mulhrs_epi16(v2, k_256); + u1 = _mm_mulhrs_epi16(v3, k_256); + + u0 = _mm_packus_epi16(u0, u1); + u1 = _mm_srli_si128(u0, 4); + u2 = _mm_srli_si128(u0, 8); + u3 = _mm_srli_si128(u0, 12); + + *(int *)(dst) = _mm_cvtsi128_si32(u0); + *(int *)(dst + dst_stride) = _mm_cvtsi128_si32(u1); + *(int *)(dst + dst_stride * 2) = _mm_cvtsi128_si32(u2); + *(int *)(dst + dst_stride * 3) = _mm_cvtsi128_si32(u3); +} + +static INLINE void transpose4x4_accumu_to_dst(const uint16_t *src, + int src_stride, uint8_t *dst, + int dst_stride) { + const __m128i k_256 = _mm_set1_epi16(1 << 8); + const __m128i zero = _mm_setzero_si128(); + const __m128i one = _mm_set1_epi16(1); + + __m128i v0, v1, v2, v3; + + __m128i u0 = _mm_loadl_epi64((__m128i const *)(src)); + __m128i u1 = _mm_loadl_epi64((__m128i const *)(src + src_stride)); + __m128i u2 = _mm_loadl_epi64((__m128i const *)(src + 2 * src_stride)); + __m128i u3 = _mm_loadl_epi64((__m128i const *)(src + 3 * src_stride)); + + v0 = _mm_unpacklo_epi16(u0, u1); + v1 = _mm_unpacklo_epi16(u2, u3); + + v2 = _mm_unpacklo_epi32(v0, v1); + v3 = _mm_unpackhi_epi32(v0, v1); + + u0 = _mm_mulhrs_epi16(v2, k_256); + u1 = _mm_mulhrs_epi16(v3, k_256); + + u2 = _mm_packus_epi16(u0, u1); + u0 = _mm_unpacklo_epi8(u2, zero); + u1 = _mm_unpackhi_epi8(u2, zero); + + // load pixel values + v0 = _mm_loadl_epi64((__m128i const *)(dst)); + v1 = _mm_loadl_epi64((__m128i const *)(dst + dst_stride)); + v2 = _mm_loadl_epi64((__m128i const *)(dst + 2 * dst_stride)); + v3 = _mm_loadl_epi64((__m128i const *)(dst + 3 * dst_stride)); + + v0 = _mm_unpacklo_epi8(v0, zero); + v1 = _mm_unpacklo_epi8(v1, zero); + v2 = _mm_unpacklo_epi8(v2, zero); + v3 = _mm_unpacklo_epi8(v3, zero); + + v0 = _mm_unpacklo_epi64(v0, v1); + v1 = _mm_unpacklo_epi64(v2, v3); + + u0 = _mm_adds_epi16(u0, v0); + u1 = _mm_adds_epi16(u1, v1); + + u0 = _mm_adds_epi16(u0, one); + u1 = _mm_adds_epi16(u1, one); + + u0 = _mm_srai_epi16(u0, 1); + u1 = _mm_srai_epi16(u1, 1); + + // saturation and pack to pixels + u0 = _mm_packus_epi16(u0, u1); + u1 = _mm_srli_si128(u0, 4); + u2 = _mm_srli_si128(u0, 8); + u3 = _mm_srli_si128(u0, 12); + + *(int *)(dst) = _mm_cvtsi128_si32(u0); + *(int *)(dst + dst_stride) = _mm_cvtsi128_si32(u1); + *(int *)(dst + dst_stride * 2) = _mm_cvtsi128_si32(u2); + *(int *)(dst + dst_stride * 3) = _mm_cvtsi128_si32(u3); +} + +static transpose_to_dst_t trans4x4Tab[2] = { transpose4x4_direct_to_dst, + transpose4x4_accumu_to_dst }; + +static void filter_horiz_v4p_ssse3(const uint8_t *src_ptr, ptrdiff_t src_pitch, + __m128i *f, int tapsNum, uint16_t *buf) { + __m128i A, B, C, D; + __m128i tr0_0, tr0_1, s1s0, s3s2, s5s4, s7s6, s9s8, sbsa; + __m128i x0, x1, x2, x3, x4, x5; + __m128i min_x2x3, max_x2x3, temp; + + if (tapsNum == 10) { + src_ptr -= 1; + } + A = _mm_loadu_si128((const __m128i *)src_ptr); + B = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch)); + C = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2)); + D = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3)); + + // TRANSPOSE... + // Vecotor represents column pixel pairs instead of a row + // 00 01 10 11 02 03 12 13 04 05 14 15 06 07 16 17 + tr0_0 = _mm_unpacklo_epi16(A, B); + // 20 21 30 31 22 23 32 33 24 25 34 35 26 27 36 37 + tr0_1 = _mm_unpacklo_epi16(C, D); + // 00 01 10 11 20 21 30 31 02 03 12 13 22 23 32 33 + s1s0 = _mm_unpacklo_epi32(tr0_0, tr0_1); + // 04 05 14 15 24 25 34 35 06 07 16 17 26 27 36 37 + s5s4 = _mm_unpackhi_epi32(tr0_0, tr0_1); + // 02 03 12 13 22 23 32 33 + s3s2 = _mm_srli_si128(s1s0, 8); + // 06 07 16 17 26 27 36 37 + s7s6 = _mm_srli_si128(s5s4, 8); + + tr0_0 = _mm_unpackhi_epi16(A, B); + tr0_1 = _mm_unpackhi_epi16(C, D); + s9s8 = _mm_unpacklo_epi32(tr0_0, tr0_1); + sbsa = _mm_srli_si128(s9s8, 8); + + // multiply 2 adjacent elements with the filter and add the result + x0 = _mm_maddubs_epi16(s1s0, f[0]); + x1 = _mm_maddubs_epi16(s3s2, f[1]); + x2 = _mm_maddubs_epi16(s5s4, f[2]); + x3 = _mm_maddubs_epi16(s7s6, f[3]); + x4 = _mm_maddubs_epi16(s9s8, f[4]); + x5 = _mm_maddubs_epi16(sbsa, f[5]); + // add and saturate the results together + min_x2x3 = _mm_min_epi16(x2, x3); + max_x2x3 = _mm_max_epi16(x2, x3); + temp = _mm_adds_epi16(x0, x1); + temp = _mm_adds_epi16(temp, x5); + temp = _mm_adds_epi16(temp, x4); + + temp = _mm_adds_epi16(temp, min_x2x3); + temp = _mm_adds_epi16(temp, max_x2x3); + _mm_storel_epi64((__m128i *)buf, temp); +} + +// Note: +// This function assumes: +// (1) 10/12-taps filters +// (2) x_step_q4 = 16 then filter is fixed at the call + +void av1_convolve_horiz_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, + const InterpFilterParams filter_params, + const int subpel_x_q4, int x_step_q4, + ConvolveParams *conv_params) { + DECLARE_ALIGNED(16, uint16_t, temp[8 * 8]); + __m128i verf[6]; + __m128i horf[2]; + SubpelFilterCoeffs hCoeffs, vCoeffs; + const uint8_t *src_ptr; + store_pixel_t store2p = store2pixelTab[conv_params->ref]; + store_pixel_t store4p = store4pixelTab[conv_params->ref]; + transpose_to_dst_t transpose_4x4 = trans4x4Tab[conv_params->ref]; + transpose_to_dst_t transpose_8x8 = trans8x8Tab[conv_params->ref]; + + const int tapsNum = filter_params.taps; + int block_height, block_residu; + int i, col, count; + (void)x_step_q4; + + if (0 == subpel_x_q4 || 16 != x_step_q4) { + av1_convolve_horiz_c(src, src_stride, dst, dst_stride, w, h, filter_params, + subpel_x_q4, x_step_q4, conv_params); + return; + } + + hCoeffs = get_subpel_filter_signal_dir(filter_params, subpel_x_q4 - 1); + vCoeffs = get_subpel_filter_ver_signal_dir(filter_params, subpel_x_q4 - 1); + + if (!hCoeffs || !vCoeffs) { + av1_convolve_horiz_c(src, src_stride, dst, dst_stride, w, h, filter_params, + subpel_x_q4, x_step_q4, conv_params); + return; + } + + verf[0] = *((const __m128i *)(vCoeffs)); + verf[1] = *((const __m128i *)(vCoeffs + 1)); + verf[2] = *((const __m128i *)(vCoeffs + 2)); + verf[3] = *((const __m128i *)(vCoeffs + 3)); + verf[4] = *((const __m128i *)(vCoeffs + 4)); + verf[5] = *((const __m128i *)(vCoeffs + 5)); + + horf[0] = *((const __m128i *)(hCoeffs)); + horf[1] = *((const __m128i *)(hCoeffs + 1)); + + count = 0; + + // here tapsNum is filter size + src -= (tapsNum >> 1) - 1; + src_ptr = src; + if (w > WIDTH_BOUND && h > HEIGHT_BOUND) { + // 8-pixels parallel + block_height = h >> 3; + block_residu = h & 7; + + do { + for (col = 0; col < w; col += 8) { + for (i = 0; i < 8; ++i) { + filter_horiz_v8p_ssse3(src_ptr, src_stride, verf, tapsNum, + temp + (i * 8)); + src_ptr += 1; + } + transpose_8x8(temp, 8, dst + col, dst_stride); + } + count++; + src_ptr = src + count * src_stride * 8; + dst += dst_stride * 8; + } while (count < block_height); + + for (i = 0; i < block_residu; ++i) { + filter_horiz_ssse3(src_ptr, horf, tapsNum, w, store4p, dst); + src_ptr += src_stride; + dst += dst_stride; + } + } else { + if (w > 2) { + // 4-pixels parallel + block_height = h >> 2; + block_residu = h & 3; + + do { + for (col = 0; col < w; col += 4) { + for (i = 0; i < 4; ++i) { + filter_horiz_v4p_ssse3(src_ptr, src_stride, verf, tapsNum, + temp + (i * 4)); + src_ptr += 1; + } + transpose_4x4(temp, 4, dst + col, dst_stride); + } + count++; + src_ptr = src + count * src_stride * 4; + dst += dst_stride * 4; + } while (count < block_height); + + for (i = 0; i < block_residu; ++i) { + filter_horiz_ssse3(src_ptr, horf, tapsNum, w, store4p, dst); + src_ptr += src_stride; + dst += dst_stride; + } + } else { + for (i = 0; i < h; i++) { + filter_horiz_ssse3(src_ptr, horf, tapsNum, w, store2p, dst); + src_ptr += src_stride; + dst += dst_stride; + } + } + } +} + +// Vertical convolution filtering +static INLINE void store_8_pixel_only(const __m128i *x, uint8_t *dst) { + __m128i u = _mm_packus_epi16(*x, *x); + _mm_storel_epi64((__m128i *)dst, u); +} + +static INLINE void accumulate_store_8_pixel(const __m128i *x, uint8_t *dst) { + __m128i y = accumulate_store(x, dst); + _mm_storel_epi64((__m128i *)dst, y); +} + +static store_pixel_t store8pixelTab[2] = { store_8_pixel_only, + accumulate_store_8_pixel }; + +static __m128i filter_vert_ssse3(const uint8_t *src, int src_stride, + int tapsNum, __m128i *f) { + __m128i s[12]; + const __m128i k_256 = _mm_set1_epi16(1 << 8); + const __m128i zero = _mm_setzero_si128(); + __m128i min_x2x3, max_x2x3, sum; + int i = 0; + int r = 0; + + if (10 == tapsNum) { + i += 1; + s[0] = zero; + } + while (i < 12) { + s[i] = _mm_loadu_si128((__m128i const *)(src + r * src_stride)); + i += 1; + r += 1; + } + + s[0] = _mm_unpacklo_epi8(s[0], s[1]); + s[2] = _mm_unpacklo_epi8(s[2], s[3]); + s[4] = _mm_unpacklo_epi8(s[4], s[5]); + s[6] = _mm_unpacklo_epi8(s[6], s[7]); + s[8] = _mm_unpacklo_epi8(s[8], s[9]); + s[10] = _mm_unpacklo_epi8(s[10], s[11]); + + s[0] = _mm_maddubs_epi16(s[0], f[0]); + s[2] = _mm_maddubs_epi16(s[2], f[1]); + s[4] = _mm_maddubs_epi16(s[4], f[2]); + s[6] = _mm_maddubs_epi16(s[6], f[3]); + s[8] = _mm_maddubs_epi16(s[8], f[4]); + s[10] = _mm_maddubs_epi16(s[10], f[5]); + + min_x2x3 = _mm_min_epi16(s[4], s[6]); + max_x2x3 = _mm_max_epi16(s[4], s[6]); + sum = _mm_adds_epi16(s[0], s[2]); + sum = _mm_adds_epi16(sum, s[10]); + sum = _mm_adds_epi16(sum, s[8]); + + sum = _mm_adds_epi16(sum, min_x2x3); + sum = _mm_adds_epi16(sum, max_x2x3); + + sum = _mm_mulhrs_epi16(sum, k_256); + sum = _mm_packus_epi16(sum, sum); + sum = _mm_unpacklo_epi8(sum, zero); + return sum; +} + +static void filter_vert_horiz_parallel_ssse3(const uint8_t *src, int src_stride, + __m128i *f, int tapsNum, + store_pixel_t store_func, + uint8_t *dst) { + __m128i sum = filter_vert_ssse3(src, src_stride, tapsNum, f); + store_func(&sum, dst); +} + +static void filter_vert_compute_small(const uint8_t *src, int src_stride, + __m128i *f, int tapsNum, + store_pixel_t store_func, int h, + uint8_t *dst, int dst_stride) { + int rowIndex = 0; + do { + filter_vert_horiz_parallel_ssse3(src, src_stride, f, tapsNum, store_func, + dst); + rowIndex++; + src += src_stride; + dst += dst_stride; + } while (rowIndex < h); +} + +static void filter_vert_compute_large(const uint8_t *src, int src_stride, + __m128i *f, int tapsNum, + store_pixel_t store_func, int w, int h, + uint8_t *dst, int dst_stride) { + int col; + int rowIndex = 0; + const uint8_t *src_ptr = src; + uint8_t *dst_ptr = dst; + + do { + for (col = 0; col < w; col += 8) { + filter_vert_horiz_parallel_ssse3(src_ptr, src_stride, f, tapsNum, + store_func, dst_ptr); + src_ptr += 8; + dst_ptr += 8; + } + rowIndex++; + src_ptr = src + rowIndex * src_stride; + dst_ptr = dst + rowIndex * dst_stride; + } while (rowIndex < h); +} + +void av1_convolve_vert_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, + const InterpFilterParams filter_params, + const int subpel_y_q4, int y_step_q4, + ConvolveParams *conv_params) { + __m128i verf[6]; + SubpelFilterCoeffs vCoeffs; + const uint8_t *src_ptr; + uint8_t *dst_ptr = dst; + store_pixel_t store2p = store2pixelTab[conv_params->ref]; + store_pixel_t store4p = store4pixelTab[conv_params->ref]; + store_pixel_t store8p = store8pixelTab[conv_params->ref]; + const int tapsNum = filter_params.taps; + + if (0 == subpel_y_q4 || 16 != y_step_q4) { + av1_convolve_vert_c(src, src_stride, dst, dst_stride, w, h, filter_params, + subpel_y_q4, y_step_q4, conv_params); + return; + } + + vCoeffs = get_subpel_filter_ver_signal_dir(filter_params, subpel_y_q4 - 1); + + if (!vCoeffs) { + av1_convolve_vert_c(src, src_stride, dst, dst_stride, w, h, filter_params, + subpel_y_q4, y_step_q4, conv_params); + return; + } + + verf[0] = *((const __m128i *)(vCoeffs)); + verf[1] = *((const __m128i *)(vCoeffs + 1)); + verf[2] = *((const __m128i *)(vCoeffs + 2)); + verf[3] = *((const __m128i *)(vCoeffs + 3)); + verf[4] = *((const __m128i *)(vCoeffs + 4)); + verf[5] = *((const __m128i *)(vCoeffs + 5)); + + src -= src_stride * ((tapsNum >> 1) - 1); + src_ptr = src; + + if (w > 4) { + filter_vert_compute_large(src_ptr, src_stride, verf, tapsNum, store8p, w, h, + dst_ptr, dst_stride); + } else if (4 == w) { + filter_vert_compute_small(src_ptr, src_stride, verf, tapsNum, store4p, h, + dst_ptr, dst_stride); + } else if (2 == w) { + filter_vert_compute_small(src_ptr, src_stride, verf, tapsNum, store2p, h, + dst_ptr, dst_stride); + } else { + assert(0); + } +} + +static void init_simd_horiz_filter(const int16_t *filter_ptr, int taps, + int8_t (*simd_horiz_filter)[2][16]) { + int shift; + int offset = (12 - taps) / 2; + const int16_t *filter_row; + for (shift = 1; shift < SUBPEL_SHIFTS; ++shift) { + int i; + filter_row = filter_ptr + shift * taps; + for (i = 0; i < offset; ++i) simd_horiz_filter[shift - 1][0][i] = 0; + + for (i = 0; i < offset + 2; ++i) simd_horiz_filter[shift - 1][1][i] = 0; + + for (i = 0; i < taps; ++i) { + simd_horiz_filter[shift - 1][0][i + offset] = (int8_t)filter_row[i]; + simd_horiz_filter[shift - 1][1][i + offset + 2] = (int8_t)filter_row[i]; + } + + for (i = offset + taps; i < 16; ++i) simd_horiz_filter[shift - 1][0][i] = 0; + + for (i = offset + 2 + taps; i < 16; ++i) + simd_horiz_filter[shift - 1][1][i] = 0; + } +} + +static void init_simd_vert_filter(const int16_t *filter_ptr, int taps, + int8_t (*simd_vert_filter)[6][16]) { + int shift; + int offset = (12 - taps) / 2; + const int16_t *filter_row; + for (shift = 1; shift < SUBPEL_SHIFTS; ++shift) { + int i; + filter_row = filter_ptr + shift * taps; + for (i = 0; i < 6; ++i) { + int j; + for (j = 0; j < 16; ++j) { + int c = i * 2 + (j % 2) - offset; + if (c >= 0 && c < taps) + simd_vert_filter[shift - 1][i][j] = (int8_t)filter_row[c]; + else + simd_vert_filter[shift - 1][i][j] = 0; + } + } + } +} + +typedef struct SimdFilter { + InterpFilter interp_filter; + int8_t (*simd_horiz_filter)[2][16]; + int8_t (*simd_vert_filter)[6][16]; +} SimdFilter; + +#if CONFIG_DUAL_FILTER +#define MULTITAP_FILTER_NUM 1 +SimdFilter simd_filters[MULTITAP_FILTER_NUM] = { + { MULTITAP_SHARP, &sub_pel_filters_12sharp_signal_dir[0], + &sub_pel_filters_12sharp_ver_signal_dir[0] }, +}; +#endif + +#if USE_TEMPORALFILTER_12TAP +SimdFilter temporal_simd_filter = { + TEMPORALFILTER_12TAP, &sub_pel_filters_temporalfilter_12_signal_dir[0], + &sub_pel_filters_temporalfilter_12_ver_signal_dir[0] +}; +#endif + +void av1_lowbd_convolve_init_ssse3(void) { +#if USE_TEMPORALFILTER_12TAP + { + InterpFilterParams filter_params = + av1_get_interp_filter_params(temporal_simd_filter.interp_filter); + int taps = filter_params.taps; + const int16_t *filter_ptr = filter_params.filter_ptr; + init_simd_horiz_filter(filter_ptr, taps, + temporal_simd_filter.simd_horiz_filter); + init_simd_vert_filter(filter_ptr, taps, + temporal_simd_filter.simd_vert_filter); + } +#endif +#if CONFIG_DUAL_FILTER + { + int i; + for (i = 0; i < MULTITAP_FILTER_NUM; ++i) { + InterpFilter interp_filter = simd_filters[i].interp_filter; + InterpFilterParams filter_params = + av1_get_interp_filter_params(interp_filter); + int taps = filter_params.taps; + const int16_t *filter_ptr = filter_params.filter_ptr; + init_simd_horiz_filter(filter_ptr, taps, + simd_filters[i].simd_horiz_filter); + init_simd_vert_filter(filter_ptr, taps, simd_filters[i].simd_vert_filter); + } + } +#endif + return; +} diff --git a/third_party/aom/av1/common/x86/av1_fwd_txfm1d_sse4.c b/third_party/aom/av1/common/x86/av1_fwd_txfm1d_sse4.c new file mode 100644 index 0000000000..d04b667f19 --- /dev/null +++ b/third_party/aom/av1/common/x86/av1_fwd_txfm1d_sse4.c @@ -0,0 +1,839 @@ +#include "av1/common/x86/av1_txfm1d_sse4.h" + +void av1_fdct32_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int txfm_size = 32; + const int num_per_128 = 4; + const int32_t *cospi; + __m128i buf0[32]; + __m128i buf1[32]; + int col_num = txfm_size / num_per_128; + int bit; + int col; + (void)stage_range; + for (col = 0; col < col_num; col++) { + // stage 0; + int32_t stage_idx = 0; + int j; + for (j = 0; j < 32; ++j) { + buf0[j] = input[j * col_num + col]; + } + + // stage 1 + stage_idx++; + buf1[0] = _mm_add_epi32(buf0[0], buf0[31]); + buf1[31] = _mm_sub_epi32(buf0[0], buf0[31]); + buf1[1] = _mm_add_epi32(buf0[1], buf0[30]); + buf1[30] = _mm_sub_epi32(buf0[1], buf0[30]); + buf1[2] = _mm_add_epi32(buf0[2], buf0[29]); + buf1[29] = _mm_sub_epi32(buf0[2], buf0[29]); + buf1[3] = _mm_add_epi32(buf0[3], buf0[28]); + buf1[28] = _mm_sub_epi32(buf0[3], buf0[28]); + buf1[4] = _mm_add_epi32(buf0[4], buf0[27]); + buf1[27] = _mm_sub_epi32(buf0[4], buf0[27]); + buf1[5] = _mm_add_epi32(buf0[5], buf0[26]); + buf1[26] = _mm_sub_epi32(buf0[5], buf0[26]); + buf1[6] = _mm_add_epi32(buf0[6], buf0[25]); + buf1[25] = _mm_sub_epi32(buf0[6], buf0[25]); + buf1[7] = _mm_add_epi32(buf0[7], buf0[24]); + buf1[24] = _mm_sub_epi32(buf0[7], buf0[24]); + buf1[8] = _mm_add_epi32(buf0[8], buf0[23]); + buf1[23] = _mm_sub_epi32(buf0[8], buf0[23]); + buf1[9] = _mm_add_epi32(buf0[9], buf0[22]); + buf1[22] = _mm_sub_epi32(buf0[9], buf0[22]); + buf1[10] = _mm_add_epi32(buf0[10], buf0[21]); + buf1[21] = _mm_sub_epi32(buf0[10], buf0[21]); + buf1[11] = _mm_add_epi32(buf0[11], buf0[20]); + buf1[20] = _mm_sub_epi32(buf0[11], buf0[20]); + buf1[12] = _mm_add_epi32(buf0[12], buf0[19]); + buf1[19] = _mm_sub_epi32(buf0[12], buf0[19]); + buf1[13] = _mm_add_epi32(buf0[13], buf0[18]); + buf1[18] = _mm_sub_epi32(buf0[13], buf0[18]); + buf1[14] = _mm_add_epi32(buf0[14], buf0[17]); + buf1[17] = _mm_sub_epi32(buf0[14], buf0[17]); + buf1[15] = _mm_add_epi32(buf0[15], buf0[16]); + buf1[16] = _mm_sub_epi32(buf0[15], buf0[16]); + + // stage 2 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + buf0[0] = _mm_add_epi32(buf1[0], buf1[15]); + buf0[15] = _mm_sub_epi32(buf1[0], buf1[15]); + buf0[1] = _mm_add_epi32(buf1[1], buf1[14]); + buf0[14] = _mm_sub_epi32(buf1[1], buf1[14]); + buf0[2] = _mm_add_epi32(buf1[2], buf1[13]); + buf0[13] = _mm_sub_epi32(buf1[2], buf1[13]); + buf0[3] = _mm_add_epi32(buf1[3], buf1[12]); + buf0[12] = _mm_sub_epi32(buf1[3], buf1[12]); + buf0[4] = _mm_add_epi32(buf1[4], buf1[11]); + buf0[11] = _mm_sub_epi32(buf1[4], buf1[11]); + buf0[5] = _mm_add_epi32(buf1[5], buf1[10]); + buf0[10] = _mm_sub_epi32(buf1[5], buf1[10]); + buf0[6] = _mm_add_epi32(buf1[6], buf1[9]); + buf0[9] = _mm_sub_epi32(buf1[6], buf1[9]); + buf0[7] = _mm_add_epi32(buf1[7], buf1[8]); + buf0[8] = _mm_sub_epi32(buf1[7], buf1[8]); + buf0[16] = buf1[16]; + buf0[17] = buf1[17]; + buf0[18] = buf1[18]; + buf0[19] = buf1[19]; + btf_32_sse4_1_type0(-cospi[32], cospi[32], buf1[20], buf1[27], buf0[20], + buf0[27], bit); + btf_32_sse4_1_type0(-cospi[32], cospi[32], buf1[21], buf1[26], buf0[21], + buf0[26], bit); + btf_32_sse4_1_type0(-cospi[32], cospi[32], buf1[22], buf1[25], buf0[22], + buf0[25], bit); + btf_32_sse4_1_type0(-cospi[32], cospi[32], buf1[23], buf1[24], buf0[23], + buf0[24], bit); + buf0[28] = buf1[28]; + buf0[29] = buf1[29]; + buf0[30] = buf1[30]; + buf0[31] = buf1[31]; + + // stage 3 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + buf1[0] = _mm_add_epi32(buf0[0], buf0[7]); + buf1[7] = _mm_sub_epi32(buf0[0], buf0[7]); + buf1[1] = _mm_add_epi32(buf0[1], buf0[6]); + buf1[6] = _mm_sub_epi32(buf0[1], buf0[6]); + buf1[2] = _mm_add_epi32(buf0[2], buf0[5]); + buf1[5] = _mm_sub_epi32(buf0[2], buf0[5]); + buf1[3] = _mm_add_epi32(buf0[3], buf0[4]); + buf1[4] = _mm_sub_epi32(buf0[3], buf0[4]); + buf1[8] = buf0[8]; + buf1[9] = buf0[9]; + btf_32_sse4_1_type0(-cospi[32], cospi[32], buf0[10], buf0[13], buf1[10], + buf1[13], bit); + btf_32_sse4_1_type0(-cospi[32], cospi[32], buf0[11], buf0[12], buf1[11], + buf1[12], bit); + buf1[14] = buf0[14]; + buf1[15] = buf0[15]; + buf1[16] = _mm_add_epi32(buf0[16], buf0[23]); + buf1[23] = _mm_sub_epi32(buf0[16], buf0[23]); + buf1[17] = _mm_add_epi32(buf0[17], buf0[22]); + buf1[22] = _mm_sub_epi32(buf0[17], buf0[22]); + buf1[18] = _mm_add_epi32(buf0[18], buf0[21]); + buf1[21] = _mm_sub_epi32(buf0[18], buf0[21]); + buf1[19] = _mm_add_epi32(buf0[19], buf0[20]); + buf1[20] = _mm_sub_epi32(buf0[19], buf0[20]); + buf1[24] = _mm_sub_epi32(buf0[31], buf0[24]); + buf1[31] = _mm_add_epi32(buf0[31], buf0[24]); + buf1[25] = _mm_sub_epi32(buf0[30], buf0[25]); + buf1[30] = _mm_add_epi32(buf0[30], buf0[25]); + buf1[26] = _mm_sub_epi32(buf0[29], buf0[26]); + buf1[29] = _mm_add_epi32(buf0[29], buf0[26]); + buf1[27] = _mm_sub_epi32(buf0[28], buf0[27]); + buf1[28] = _mm_add_epi32(buf0[28], buf0[27]); + + // stage 4 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + buf0[0] = _mm_add_epi32(buf1[0], buf1[3]); + buf0[3] = _mm_sub_epi32(buf1[0], buf1[3]); + buf0[1] = _mm_add_epi32(buf1[1], buf1[2]); + buf0[2] = _mm_sub_epi32(buf1[1], buf1[2]); + buf0[4] = buf1[4]; + btf_32_sse4_1_type0(-cospi[32], cospi[32], buf1[5], buf1[6], buf0[5], + buf0[6], bit); + buf0[7] = buf1[7]; + buf0[8] = _mm_add_epi32(buf1[8], buf1[11]); + buf0[11] = _mm_sub_epi32(buf1[8], buf1[11]); + buf0[9] = _mm_add_epi32(buf1[9], buf1[10]); + buf0[10] = _mm_sub_epi32(buf1[9], buf1[10]); + buf0[12] = _mm_sub_epi32(buf1[15], buf1[12]); + buf0[15] = _mm_add_epi32(buf1[15], buf1[12]); + buf0[13] = _mm_sub_epi32(buf1[14], buf1[13]); + buf0[14] = _mm_add_epi32(buf1[14], buf1[13]); + buf0[16] = buf1[16]; + buf0[17] = buf1[17]; + btf_32_sse4_1_type0(-cospi[16], cospi[48], buf1[18], buf1[29], buf0[18], + buf0[29], bit); + btf_32_sse4_1_type0(-cospi[16], cospi[48], buf1[19], buf1[28], buf0[19], + buf0[28], bit); + btf_32_sse4_1_type0(-cospi[48], -cospi[16], buf1[20], buf1[27], buf0[20], + buf0[27], bit); + btf_32_sse4_1_type0(-cospi[48], -cospi[16], buf1[21], buf1[26], buf0[21], + buf0[26], bit); + buf0[22] = buf1[22]; + buf0[23] = buf1[23]; + buf0[24] = buf1[24]; + buf0[25] = buf1[25]; + buf0[30] = buf1[30]; + buf0[31] = buf1[31]; + + // stage 5 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + btf_32_sse4_1_type0(cospi[32], cospi[32], buf0[0], buf0[1], buf1[0], + buf1[1], bit); + btf_32_sse4_1_type1(cospi[48], cospi[16], buf0[2], buf0[3], buf1[2], + buf1[3], bit); + buf1[4] = _mm_add_epi32(buf0[4], buf0[5]); + buf1[5] = _mm_sub_epi32(buf0[4], buf0[5]); + buf1[6] = _mm_sub_epi32(buf0[7], buf0[6]); + buf1[7] = _mm_add_epi32(buf0[7], buf0[6]); + buf1[8] = buf0[8]; + btf_32_sse4_1_type0(-cospi[16], cospi[48], buf0[9], buf0[14], buf1[9], + buf1[14], bit); + btf_32_sse4_1_type0(-cospi[48], -cospi[16], buf0[10], buf0[13], buf1[10], + buf1[13], bit); + buf1[11] = buf0[11]; + buf1[12] = buf0[12]; + buf1[15] = buf0[15]; + buf1[16] = _mm_add_epi32(buf0[16], buf0[19]); + buf1[19] = _mm_sub_epi32(buf0[16], buf0[19]); + buf1[17] = _mm_add_epi32(buf0[17], buf0[18]); + buf1[18] = _mm_sub_epi32(buf0[17], buf0[18]); + buf1[20] = _mm_sub_epi32(buf0[23], buf0[20]); + buf1[23] = _mm_add_epi32(buf0[23], buf0[20]); + buf1[21] = _mm_sub_epi32(buf0[22], buf0[21]); + buf1[22] = _mm_add_epi32(buf0[22], buf0[21]); + buf1[24] = _mm_add_epi32(buf0[24], buf0[27]); + buf1[27] = _mm_sub_epi32(buf0[24], buf0[27]); + buf1[25] = _mm_add_epi32(buf0[25], buf0[26]); + buf1[26] = _mm_sub_epi32(buf0[25], buf0[26]); + buf1[28] = _mm_sub_epi32(buf0[31], buf0[28]); + buf1[31] = _mm_add_epi32(buf0[31], buf0[28]); + buf1[29] = _mm_sub_epi32(buf0[30], buf0[29]); + buf1[30] = _mm_add_epi32(buf0[30], buf0[29]); + + // stage 6 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + buf0[0] = buf1[0]; + buf0[1] = buf1[1]; + buf0[2] = buf1[2]; + buf0[3] = buf1[3]; + btf_32_sse4_1_type1(cospi[56], cospi[8], buf1[4], buf1[7], buf0[4], buf0[7], + bit); + btf_32_sse4_1_type1(cospi[24], cospi[40], buf1[5], buf1[6], buf0[5], + buf0[6], bit); + buf0[8] = _mm_add_epi32(buf1[8], buf1[9]); + buf0[9] = _mm_sub_epi32(buf1[8], buf1[9]); + buf0[10] = _mm_sub_epi32(buf1[11], buf1[10]); + buf0[11] = _mm_add_epi32(buf1[11], buf1[10]); + buf0[12] = _mm_add_epi32(buf1[12], buf1[13]); + buf0[13] = _mm_sub_epi32(buf1[12], buf1[13]); + buf0[14] = _mm_sub_epi32(buf1[15], buf1[14]); + buf0[15] = _mm_add_epi32(buf1[15], buf1[14]); + buf0[16] = buf1[16]; + btf_32_sse4_1_type0(-cospi[8], cospi[56], buf1[17], buf1[30], buf0[17], + buf0[30], bit); + btf_32_sse4_1_type0(-cospi[56], -cospi[8], buf1[18], buf1[29], buf0[18], + buf0[29], bit); + buf0[19] = buf1[19]; + buf0[20] = buf1[20]; + btf_32_sse4_1_type0(-cospi[40], cospi[24], buf1[21], buf1[26], buf0[21], + buf0[26], bit); + btf_32_sse4_1_type0(-cospi[24], -cospi[40], buf1[22], buf1[25], buf0[22], + buf0[25], bit); + buf0[23] = buf1[23]; + buf0[24] = buf1[24]; + buf0[27] = buf1[27]; + buf0[28] = buf1[28]; + buf0[31] = buf1[31]; + + // stage 7 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + buf1[0] = buf0[0]; + buf1[1] = buf0[1]; + buf1[2] = buf0[2]; + buf1[3] = buf0[3]; + buf1[4] = buf0[4]; + buf1[5] = buf0[5]; + buf1[6] = buf0[6]; + buf1[7] = buf0[7]; + btf_32_sse4_1_type1(cospi[60], cospi[4], buf0[8], buf0[15], buf1[8], + buf1[15], bit); + btf_32_sse4_1_type1(cospi[28], cospi[36], buf0[9], buf0[14], buf1[9], + buf1[14], bit); + btf_32_sse4_1_type1(cospi[44], cospi[20], buf0[10], buf0[13], buf1[10], + buf1[13], bit); + btf_32_sse4_1_type1(cospi[12], cospi[52], buf0[11], buf0[12], buf1[11], + buf1[12], bit); + buf1[16] = _mm_add_epi32(buf0[16], buf0[17]); + buf1[17] = _mm_sub_epi32(buf0[16], buf0[17]); + buf1[18] = _mm_sub_epi32(buf0[19], buf0[18]); + buf1[19] = _mm_add_epi32(buf0[19], buf0[18]); + buf1[20] = _mm_add_epi32(buf0[20], buf0[21]); + buf1[21] = _mm_sub_epi32(buf0[20], buf0[21]); + buf1[22] = _mm_sub_epi32(buf0[23], buf0[22]); + buf1[23] = _mm_add_epi32(buf0[23], buf0[22]); + buf1[24] = _mm_add_epi32(buf0[24], buf0[25]); + buf1[25] = _mm_sub_epi32(buf0[24], buf0[25]); + buf1[26] = _mm_sub_epi32(buf0[27], buf0[26]); + buf1[27] = _mm_add_epi32(buf0[27], buf0[26]); + buf1[28] = _mm_add_epi32(buf0[28], buf0[29]); + buf1[29] = _mm_sub_epi32(buf0[28], buf0[29]); + buf1[30] = _mm_sub_epi32(buf0[31], buf0[30]); + buf1[31] = _mm_add_epi32(buf0[31], buf0[30]); + + // stage 8 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + buf0[0] = buf1[0]; + buf0[1] = buf1[1]; + buf0[2] = buf1[2]; + buf0[3] = buf1[3]; + buf0[4] = buf1[4]; + buf0[5] = buf1[5]; + buf0[6] = buf1[6]; + buf0[7] = buf1[7]; + buf0[8] = buf1[8]; + buf0[9] = buf1[9]; + buf0[10] = buf1[10]; + buf0[11] = buf1[11]; + buf0[12] = buf1[12]; + buf0[13] = buf1[13]; + buf0[14] = buf1[14]; + buf0[15] = buf1[15]; + btf_32_sse4_1_type1(cospi[62], cospi[2], buf1[16], buf1[31], buf0[16], + buf0[31], bit); + btf_32_sse4_1_type1(cospi[30], cospi[34], buf1[17], buf1[30], buf0[17], + buf0[30], bit); + btf_32_sse4_1_type1(cospi[46], cospi[18], buf1[18], buf1[29], buf0[18], + buf0[29], bit); + btf_32_sse4_1_type1(cospi[14], cospi[50], buf1[19], buf1[28], buf0[19], + buf0[28], bit); + btf_32_sse4_1_type1(cospi[54], cospi[10], buf1[20], buf1[27], buf0[20], + buf0[27], bit); + btf_32_sse4_1_type1(cospi[22], cospi[42], buf1[21], buf1[26], buf0[21], + buf0[26], bit); + btf_32_sse4_1_type1(cospi[38], cospi[26], buf1[22], buf1[25], buf0[22], + buf0[25], bit); + btf_32_sse4_1_type1(cospi[6], cospi[58], buf1[23], buf1[24], buf0[23], + buf0[24], bit); + + // stage 9 + stage_idx++; + buf1[0] = buf0[0]; + buf1[1] = buf0[16]; + buf1[2] = buf0[8]; + buf1[3] = buf0[24]; + buf1[4] = buf0[4]; + buf1[5] = buf0[20]; + buf1[6] = buf0[12]; + buf1[7] = buf0[28]; + buf1[8] = buf0[2]; + buf1[9] = buf0[18]; + buf1[10] = buf0[10]; + buf1[11] = buf0[26]; + buf1[12] = buf0[6]; + buf1[13] = buf0[22]; + buf1[14] = buf0[14]; + buf1[15] = buf0[30]; + buf1[16] = buf0[1]; + buf1[17] = buf0[17]; + buf1[18] = buf0[9]; + buf1[19] = buf0[25]; + buf1[20] = buf0[5]; + buf1[21] = buf0[21]; + buf1[22] = buf0[13]; + buf1[23] = buf0[29]; + buf1[24] = buf0[3]; + buf1[25] = buf0[19]; + buf1[26] = buf0[11]; + buf1[27] = buf0[27]; + buf1[28] = buf0[7]; + buf1[29] = buf0[23]; + buf1[30] = buf0[15]; + buf1[31] = buf0[31]; + + for (j = 0; j < 32; ++j) { + output[j * col_num + col] = buf1[j]; + } + } +} + +void av1_fadst4_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int txfm_size = 4; + const int num_per_128 = 4; + const int32_t *cospi; + __m128i buf0[4]; + __m128i buf1[4]; + int col_num = txfm_size / num_per_128; + int bit; + int col; + (void)stage_range; + for (col = 0; col < col_num; col++) { + // stage 0; + int32_t stage_idx = 0; + int j; + for (j = 0; j < 4; ++j) { + buf0[j] = input[j * col_num + col]; + } + + // stage 1 + stage_idx++; + buf1[0] = buf0[3]; + buf1[1] = buf0[0]; + buf1[2] = buf0[1]; + buf1[3] = buf0[2]; + + // stage 2 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + btf_32_sse4_1_type0(cospi[8], cospi[56], buf1[0], buf1[1], buf0[0], buf0[1], + bit); + btf_32_sse4_1_type0(cospi[40], cospi[24], buf1[2], buf1[3], buf0[2], + buf0[3], bit); + + // stage 3 + stage_idx++; + buf1[0] = _mm_add_epi32(buf0[0], buf0[2]); + buf1[2] = _mm_sub_epi32(buf0[0], buf0[2]); + buf1[1] = _mm_add_epi32(buf0[1], buf0[3]); + buf1[3] = _mm_sub_epi32(buf0[1], buf0[3]); + + // stage 4 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + buf0[0] = buf1[0]; + buf0[1] = buf1[1]; + btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[2], buf1[3], buf0[2], + buf0[3], bit); + + // stage 5 + stage_idx++; + buf1[0] = buf0[0]; + buf1[1] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[2]); + buf1[2] = buf0[3]; + buf1[3] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[1]); + + for (j = 0; j < 4; ++j) { + output[j * col_num + col] = buf1[j]; + } + } +} + +void av1_fadst32_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range) { + const int txfm_size = 32; + const int num_per_128 = 4; + const int32_t *cospi; + __m128i buf0[32]; + __m128i buf1[32]; + int col_num = txfm_size / num_per_128; + int bit; + int col; + (void)stage_range; + for (col = 0; col < col_num; col++) { + // stage 0; + int32_t stage_idx = 0; + int j; + for (j = 0; j < 32; ++j) { + buf0[j] = input[j * col_num + col]; + } + + // stage 1 + stage_idx++; + buf1[0] = buf0[31]; + buf1[1] = buf0[0]; + buf1[2] = buf0[29]; + buf1[3] = buf0[2]; + buf1[4] = buf0[27]; + buf1[5] = buf0[4]; + buf1[6] = buf0[25]; + buf1[7] = buf0[6]; + buf1[8] = buf0[23]; + buf1[9] = buf0[8]; + buf1[10] = buf0[21]; + buf1[11] = buf0[10]; + buf1[12] = buf0[19]; + buf1[13] = buf0[12]; + buf1[14] = buf0[17]; + buf1[15] = buf0[14]; + buf1[16] = buf0[15]; + buf1[17] = buf0[16]; + buf1[18] = buf0[13]; + buf1[19] = buf0[18]; + buf1[20] = buf0[11]; + buf1[21] = buf0[20]; + buf1[22] = buf0[9]; + buf1[23] = buf0[22]; + buf1[24] = buf0[7]; + buf1[25] = buf0[24]; + buf1[26] = buf0[5]; + buf1[27] = buf0[26]; + buf1[28] = buf0[3]; + buf1[29] = buf0[28]; + buf1[30] = buf0[1]; + buf1[31] = buf0[30]; + + // stage 2 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + btf_32_sse4_1_type0(cospi[1], cospi[63], buf1[0], buf1[1], buf0[0], buf0[1], + bit); + btf_32_sse4_1_type0(cospi[5], cospi[59], buf1[2], buf1[3], buf0[2], buf0[3], + bit); + btf_32_sse4_1_type0(cospi[9], cospi[55], buf1[4], buf1[5], buf0[4], buf0[5], + bit); + btf_32_sse4_1_type0(cospi[13], cospi[51], buf1[6], buf1[7], buf0[6], + buf0[7], bit); + btf_32_sse4_1_type0(cospi[17], cospi[47], buf1[8], buf1[9], buf0[8], + buf0[9], bit); + btf_32_sse4_1_type0(cospi[21], cospi[43], buf1[10], buf1[11], buf0[10], + buf0[11], bit); + btf_32_sse4_1_type0(cospi[25], cospi[39], buf1[12], buf1[13], buf0[12], + buf0[13], bit); + btf_32_sse4_1_type0(cospi[29], cospi[35], buf1[14], buf1[15], buf0[14], + buf0[15], bit); + btf_32_sse4_1_type0(cospi[33], cospi[31], buf1[16], buf1[17], buf0[16], + buf0[17], bit); + btf_32_sse4_1_type0(cospi[37], cospi[27], buf1[18], buf1[19], buf0[18], + buf0[19], bit); + btf_32_sse4_1_type0(cospi[41], cospi[23], buf1[20], buf1[21], buf0[20], + buf0[21], bit); + btf_32_sse4_1_type0(cospi[45], cospi[19], buf1[22], buf1[23], buf0[22], + buf0[23], bit); + btf_32_sse4_1_type0(cospi[49], cospi[15], buf1[24], buf1[25], buf0[24], + buf0[25], bit); + btf_32_sse4_1_type0(cospi[53], cospi[11], buf1[26], buf1[27], buf0[26], + buf0[27], bit); + btf_32_sse4_1_type0(cospi[57], cospi[7], buf1[28], buf1[29], buf0[28], + buf0[29], bit); + btf_32_sse4_1_type0(cospi[61], cospi[3], buf1[30], buf1[31], buf0[30], + buf0[31], bit); + + // stage 3 + stage_idx++; + buf1[0] = _mm_add_epi32(buf0[0], buf0[16]); + buf1[16] = _mm_sub_epi32(buf0[0], buf0[16]); + buf1[1] = _mm_add_epi32(buf0[1], buf0[17]); + buf1[17] = _mm_sub_epi32(buf0[1], buf0[17]); + buf1[2] = _mm_add_epi32(buf0[2], buf0[18]); + buf1[18] = _mm_sub_epi32(buf0[2], buf0[18]); + buf1[3] = _mm_add_epi32(buf0[3], buf0[19]); + buf1[19] = _mm_sub_epi32(buf0[3], buf0[19]); + buf1[4] = _mm_add_epi32(buf0[4], buf0[20]); + buf1[20] = _mm_sub_epi32(buf0[4], buf0[20]); + buf1[5] = _mm_add_epi32(buf0[5], buf0[21]); + buf1[21] = _mm_sub_epi32(buf0[5], buf0[21]); + buf1[6] = _mm_add_epi32(buf0[6], buf0[22]); + buf1[22] = _mm_sub_epi32(buf0[6], buf0[22]); + buf1[7] = _mm_add_epi32(buf0[7], buf0[23]); + buf1[23] = _mm_sub_epi32(buf0[7], buf0[23]); + buf1[8] = _mm_add_epi32(buf0[8], buf0[24]); + buf1[24] = _mm_sub_epi32(buf0[8], buf0[24]); + buf1[9] = _mm_add_epi32(buf0[9], buf0[25]); + buf1[25] = _mm_sub_epi32(buf0[9], buf0[25]); + buf1[10] = _mm_add_epi32(buf0[10], buf0[26]); + buf1[26] = _mm_sub_epi32(buf0[10], buf0[26]); + buf1[11] = _mm_add_epi32(buf0[11], buf0[27]); + buf1[27] = _mm_sub_epi32(buf0[11], buf0[27]); + buf1[12] = _mm_add_epi32(buf0[12], buf0[28]); + buf1[28] = _mm_sub_epi32(buf0[12], buf0[28]); + buf1[13] = _mm_add_epi32(buf0[13], buf0[29]); + buf1[29] = _mm_sub_epi32(buf0[13], buf0[29]); + buf1[14] = _mm_add_epi32(buf0[14], buf0[30]); + buf1[30] = _mm_sub_epi32(buf0[14], buf0[30]); + buf1[15] = _mm_add_epi32(buf0[15], buf0[31]); + buf1[31] = _mm_sub_epi32(buf0[15], buf0[31]); + + // stage 4 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + buf0[0] = buf1[0]; + buf0[1] = buf1[1]; + buf0[2] = buf1[2]; + buf0[3] = buf1[3]; + buf0[4] = buf1[4]; + buf0[5] = buf1[5]; + buf0[6] = buf1[6]; + buf0[7] = buf1[7]; + buf0[8] = buf1[8]; + buf0[9] = buf1[9]; + buf0[10] = buf1[10]; + buf0[11] = buf1[11]; + buf0[12] = buf1[12]; + buf0[13] = buf1[13]; + buf0[14] = buf1[14]; + buf0[15] = buf1[15]; + btf_32_sse4_1_type0(cospi[4], cospi[60], buf1[16], buf1[17], buf0[16], + buf0[17], bit); + btf_32_sse4_1_type0(cospi[20], cospi[44], buf1[18], buf1[19], buf0[18], + buf0[19], bit); + btf_32_sse4_1_type0(cospi[36], cospi[28], buf1[20], buf1[21], buf0[20], + buf0[21], bit); + btf_32_sse4_1_type0(cospi[52], cospi[12], buf1[22], buf1[23], buf0[22], + buf0[23], bit); + btf_32_sse4_1_type0(-cospi[60], cospi[4], buf1[24], buf1[25], buf0[24], + buf0[25], bit); + btf_32_sse4_1_type0(-cospi[44], cospi[20], buf1[26], buf1[27], buf0[26], + buf0[27], bit); + btf_32_sse4_1_type0(-cospi[28], cospi[36], buf1[28], buf1[29], buf0[28], + buf0[29], bit); + btf_32_sse4_1_type0(-cospi[12], cospi[52], buf1[30], buf1[31], buf0[30], + buf0[31], bit); + + // stage 5 + stage_idx++; + buf1[0] = _mm_add_epi32(buf0[0], buf0[8]); + buf1[8] = _mm_sub_epi32(buf0[0], buf0[8]); + buf1[1] = _mm_add_epi32(buf0[1], buf0[9]); + buf1[9] = _mm_sub_epi32(buf0[1], buf0[9]); + buf1[2] = _mm_add_epi32(buf0[2], buf0[10]); + buf1[10] = _mm_sub_epi32(buf0[2], buf0[10]); + buf1[3] = _mm_add_epi32(buf0[3], buf0[11]); + buf1[11] = _mm_sub_epi32(buf0[3], buf0[11]); + buf1[4] = _mm_add_epi32(buf0[4], buf0[12]); + buf1[12] = _mm_sub_epi32(buf0[4], buf0[12]); + buf1[5] = _mm_add_epi32(buf0[5], buf0[13]); + buf1[13] = _mm_sub_epi32(buf0[5], buf0[13]); + buf1[6] = _mm_add_epi32(buf0[6], buf0[14]); + buf1[14] = _mm_sub_epi32(buf0[6], buf0[14]); + buf1[7] = _mm_add_epi32(buf0[7], buf0[15]); + buf1[15] = _mm_sub_epi32(buf0[7], buf0[15]); + buf1[16] = _mm_add_epi32(buf0[16], buf0[24]); + buf1[24] = _mm_sub_epi32(buf0[16], buf0[24]); + buf1[17] = _mm_add_epi32(buf0[17], buf0[25]); + buf1[25] = _mm_sub_epi32(buf0[17], buf0[25]); + buf1[18] = _mm_add_epi32(buf0[18], buf0[26]); + buf1[26] = _mm_sub_epi32(buf0[18], buf0[26]); + buf1[19] = _mm_add_epi32(buf0[19], buf0[27]); + buf1[27] = _mm_sub_epi32(buf0[19], buf0[27]); + buf1[20] = _mm_add_epi32(buf0[20], buf0[28]); + buf1[28] = _mm_sub_epi32(buf0[20], buf0[28]); + buf1[21] = _mm_add_epi32(buf0[21], buf0[29]); + buf1[29] = _mm_sub_epi32(buf0[21], buf0[29]); + buf1[22] = _mm_add_epi32(buf0[22], buf0[30]); + buf1[30] = _mm_sub_epi32(buf0[22], buf0[30]); + buf1[23] = _mm_add_epi32(buf0[23], buf0[31]); + buf1[31] = _mm_sub_epi32(buf0[23], buf0[31]); + + // stage 6 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + buf0[0] = buf1[0]; + buf0[1] = buf1[1]; + buf0[2] = buf1[2]; + buf0[3] = buf1[3]; + buf0[4] = buf1[4]; + buf0[5] = buf1[5]; + buf0[6] = buf1[6]; + buf0[7] = buf1[7]; + btf_32_sse4_1_type0(cospi[8], cospi[56], buf1[8], buf1[9], buf0[8], buf0[9], + bit); + btf_32_sse4_1_type0(cospi[40], cospi[24], buf1[10], buf1[11], buf0[10], + buf0[11], bit); + btf_32_sse4_1_type0(-cospi[56], cospi[8], buf1[12], buf1[13], buf0[12], + buf0[13], bit); + btf_32_sse4_1_type0(-cospi[24], cospi[40], buf1[14], buf1[15], buf0[14], + buf0[15], bit); + buf0[16] = buf1[16]; + buf0[17] = buf1[17]; + buf0[18] = buf1[18]; + buf0[19] = buf1[19]; + buf0[20] = buf1[20]; + buf0[21] = buf1[21]; + buf0[22] = buf1[22]; + buf0[23] = buf1[23]; + btf_32_sse4_1_type0(cospi[8], cospi[56], buf1[24], buf1[25], buf0[24], + buf0[25], bit); + btf_32_sse4_1_type0(cospi[40], cospi[24], buf1[26], buf1[27], buf0[26], + buf0[27], bit); + btf_32_sse4_1_type0(-cospi[56], cospi[8], buf1[28], buf1[29], buf0[28], + buf0[29], bit); + btf_32_sse4_1_type0(-cospi[24], cospi[40], buf1[30], buf1[31], buf0[30], + buf0[31], bit); + + // stage 7 + stage_idx++; + buf1[0] = _mm_add_epi32(buf0[0], buf0[4]); + buf1[4] = _mm_sub_epi32(buf0[0], buf0[4]); + buf1[1] = _mm_add_epi32(buf0[1], buf0[5]); + buf1[5] = _mm_sub_epi32(buf0[1], buf0[5]); + buf1[2] = _mm_add_epi32(buf0[2], buf0[6]); + buf1[6] = _mm_sub_epi32(buf0[2], buf0[6]); + buf1[3] = _mm_add_epi32(buf0[3], buf0[7]); + buf1[7] = _mm_sub_epi32(buf0[3], buf0[7]); + buf1[8] = _mm_add_epi32(buf0[8], buf0[12]); + buf1[12] = _mm_sub_epi32(buf0[8], buf0[12]); + buf1[9] = _mm_add_epi32(buf0[9], buf0[13]); + buf1[13] = _mm_sub_epi32(buf0[9], buf0[13]); + buf1[10] = _mm_add_epi32(buf0[10], buf0[14]); + buf1[14] = _mm_sub_epi32(buf0[10], buf0[14]); + buf1[11] = _mm_add_epi32(buf0[11], buf0[15]); + buf1[15] = _mm_sub_epi32(buf0[11], buf0[15]); + buf1[16] = _mm_add_epi32(buf0[16], buf0[20]); + buf1[20] = _mm_sub_epi32(buf0[16], buf0[20]); + buf1[17] = _mm_add_epi32(buf0[17], buf0[21]); + buf1[21] = _mm_sub_epi32(buf0[17], buf0[21]); + buf1[18] = _mm_add_epi32(buf0[18], buf0[22]); + buf1[22] = _mm_sub_epi32(buf0[18], buf0[22]); + buf1[19] = _mm_add_epi32(buf0[19], buf0[23]); + buf1[23] = _mm_sub_epi32(buf0[19], buf0[23]); + buf1[24] = _mm_add_epi32(buf0[24], buf0[28]); + buf1[28] = _mm_sub_epi32(buf0[24], buf0[28]); + buf1[25] = _mm_add_epi32(buf0[25], buf0[29]); + buf1[29] = _mm_sub_epi32(buf0[25], buf0[29]); + buf1[26] = _mm_add_epi32(buf0[26], buf0[30]); + buf1[30] = _mm_sub_epi32(buf0[26], buf0[30]); + buf1[27] = _mm_add_epi32(buf0[27], buf0[31]); + buf1[31] = _mm_sub_epi32(buf0[27], buf0[31]); + + // stage 8 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + buf0[0] = buf1[0]; + buf0[1] = buf1[1]; + buf0[2] = buf1[2]; + buf0[3] = buf1[3]; + btf_32_sse4_1_type0(cospi[16], cospi[48], buf1[4], buf1[5], buf0[4], + buf0[5], bit); + btf_32_sse4_1_type0(-cospi[48], cospi[16], buf1[6], buf1[7], buf0[6], + buf0[7], bit); + buf0[8] = buf1[8]; + buf0[9] = buf1[9]; + buf0[10] = buf1[10]; + buf0[11] = buf1[11]; + btf_32_sse4_1_type0(cospi[16], cospi[48], buf1[12], buf1[13], buf0[12], + buf0[13], bit); + btf_32_sse4_1_type0(-cospi[48], cospi[16], buf1[14], buf1[15], buf0[14], + buf0[15], bit); + buf0[16] = buf1[16]; + buf0[17] = buf1[17]; + buf0[18] = buf1[18]; + buf0[19] = buf1[19]; + btf_32_sse4_1_type0(cospi[16], cospi[48], buf1[20], buf1[21], buf0[20], + buf0[21], bit); + btf_32_sse4_1_type0(-cospi[48], cospi[16], buf1[22], buf1[23], buf0[22], + buf0[23], bit); + buf0[24] = buf1[24]; + buf0[25] = buf1[25]; + buf0[26] = buf1[26]; + buf0[27] = buf1[27]; + btf_32_sse4_1_type0(cospi[16], cospi[48], buf1[28], buf1[29], buf0[28], + buf0[29], bit); + btf_32_sse4_1_type0(-cospi[48], cospi[16], buf1[30], buf1[31], buf0[30], + buf0[31], bit); + + // stage 9 + stage_idx++; + buf1[0] = _mm_add_epi32(buf0[0], buf0[2]); + buf1[2] = _mm_sub_epi32(buf0[0], buf0[2]); + buf1[1] = _mm_add_epi32(buf0[1], buf0[3]); + buf1[3] = _mm_sub_epi32(buf0[1], buf0[3]); + buf1[4] = _mm_add_epi32(buf0[4], buf0[6]); + buf1[6] = _mm_sub_epi32(buf0[4], buf0[6]); + buf1[5] = _mm_add_epi32(buf0[5], buf0[7]); + buf1[7] = _mm_sub_epi32(buf0[5], buf0[7]); + buf1[8] = _mm_add_epi32(buf0[8], buf0[10]); + buf1[10] = _mm_sub_epi32(buf0[8], buf0[10]); + buf1[9] = _mm_add_epi32(buf0[9], buf0[11]); + buf1[11] = _mm_sub_epi32(buf0[9], buf0[11]); + buf1[12] = _mm_add_epi32(buf0[12], buf0[14]); + buf1[14] = _mm_sub_epi32(buf0[12], buf0[14]); + buf1[13] = _mm_add_epi32(buf0[13], buf0[15]); + buf1[15] = _mm_sub_epi32(buf0[13], buf0[15]); + buf1[16] = _mm_add_epi32(buf0[16], buf0[18]); + buf1[18] = _mm_sub_epi32(buf0[16], buf0[18]); + buf1[17] = _mm_add_epi32(buf0[17], buf0[19]); + buf1[19] = _mm_sub_epi32(buf0[17], buf0[19]); + buf1[20] = _mm_add_epi32(buf0[20], buf0[22]); + buf1[22] = _mm_sub_epi32(buf0[20], buf0[22]); + buf1[21] = _mm_add_epi32(buf0[21], buf0[23]); + buf1[23] = _mm_sub_epi32(buf0[21], buf0[23]); + buf1[24] = _mm_add_epi32(buf0[24], buf0[26]); + buf1[26] = _mm_sub_epi32(buf0[24], buf0[26]); + buf1[25] = _mm_add_epi32(buf0[25], buf0[27]); + buf1[27] = _mm_sub_epi32(buf0[25], buf0[27]); + buf1[28] = _mm_add_epi32(buf0[28], buf0[30]); + buf1[30] = _mm_sub_epi32(buf0[28], buf0[30]); + buf1[29] = _mm_add_epi32(buf0[29], buf0[31]); + buf1[31] = _mm_sub_epi32(buf0[29], buf0[31]); + + // stage 10 + stage_idx++; + bit = cos_bit[stage_idx]; + cospi = cospi_arr[bit - cos_bit_min]; + buf0[0] = buf1[0]; + buf0[1] = buf1[1]; + btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[2], buf1[3], buf0[2], + buf0[3], bit); + buf0[4] = buf1[4]; + buf0[5] = buf1[5]; + btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[6], buf1[7], buf0[6], + buf0[7], bit); + buf0[8] = buf1[8]; + buf0[9] = buf1[9]; + btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[10], buf1[11], buf0[10], + buf0[11], bit); + buf0[12] = buf1[12]; + buf0[13] = buf1[13]; + btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[14], buf1[15], buf0[14], + buf0[15], bit); + buf0[16] = buf1[16]; + buf0[17] = buf1[17]; + btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[18], buf1[19], buf0[18], + buf0[19], bit); + buf0[20] = buf1[20]; + buf0[21] = buf1[21]; + btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[22], buf1[23], buf0[22], + buf0[23], bit); + buf0[24] = buf1[24]; + buf0[25] = buf1[25]; + btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[26], buf1[27], buf0[26], + buf0[27], bit); + buf0[28] = buf1[28]; + buf0[29] = buf1[29]; + btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[30], buf1[31], buf0[30], + buf0[31], bit); + + // stage 11 + stage_idx++; + buf1[0] = buf0[0]; + buf1[1] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[16]); + buf1[2] = buf0[24]; + buf1[3] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[8]); + buf1[4] = buf0[12]; + buf1[5] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[28]); + buf1[6] = buf0[20]; + buf1[7] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[4]); + buf1[8] = buf0[6]; + buf1[9] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[22]); + buf1[10] = buf0[30]; + buf1[11] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[14]); + buf1[12] = buf0[10]; + buf1[13] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[26]); + buf1[14] = buf0[18]; + buf1[15] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[2]); + buf1[16] = buf0[3]; + buf1[17] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[19]); + buf1[18] = buf0[27]; + buf1[19] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[11]); + buf1[20] = buf0[15]; + buf1[21] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[31]); + buf1[22] = buf0[23]; + buf1[23] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[7]); + buf1[24] = buf0[5]; + buf1[25] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[21]); + buf1[26] = buf0[29]; + buf1[27] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[13]); + buf1[28] = buf0[9]; + buf1[29] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[25]); + buf1[30] = buf0[17]; + buf1[31] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[1]); + + for (j = 0; j < 32; ++j) { + output[j * col_num + col] = buf1[j]; + } + } +} diff --git a/third_party/aom/av1/common/x86/av1_fwd_txfm2d_sse4.c b/third_party/aom/av1/common/x86/av1_fwd_txfm2d_sse4.c new file mode 100644 index 0000000000..78c261374a --- /dev/null +++ b/third_party/aom/av1/common/x86/av1_fwd_txfm2d_sse4.c @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./av1_rtcd.h" +#include "av1/common/enums.h" +#include "av1/common/av1_txfm.h" +#include "av1/common/x86/av1_txfm1d_sse4.h" + +static INLINE void int16_array_with_stride_to_int32_array_without_stride( + const int16_t *input, int stride, int32_t *output, int txfm1d_size) { + int r, c; + for (r = 0; r < txfm1d_size; r++) { + for (c = 0; c < txfm1d_size; c++) { + output[r * txfm1d_size + c] = (int32_t)input[r * stride + c]; + } + } +} + +typedef void (*TxfmFuncSSE2)(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); + +static INLINE TxfmFuncSSE2 fwd_txfm_type_to_func(TXFM_TYPE txfm_type) { + switch (txfm_type) { + case TXFM_TYPE_DCT32: return av1_fdct32_new_sse4_1; break; + case TXFM_TYPE_ADST32: return av1_fadst32_new_sse4_1; break; + default: assert(0); + } + return NULL; +} + +static INLINE void fwd_txfm2d_sse4_1(const int16_t *input, int32_t *output, + const int stride, const TXFM_2D_CFG *cfg, + int32_t *txfm_buf) { + const int txfm_size = cfg->txfm_size; + const int8_t *shift = cfg->shift; + const int8_t *stage_range_col = cfg->stage_range_col; + const int8_t *stage_range_row = cfg->stage_range_row; + const int8_t *cos_bit_col = cfg->cos_bit_col; + const int8_t *cos_bit_row = cfg->cos_bit_row; + const TxfmFuncSSE2 txfm_func_col = fwd_txfm_type_to_func(cfg->txfm_type_col); + const TxfmFuncSSE2 txfm_func_row = fwd_txfm_type_to_func(cfg->txfm_type_row); + + __m128i *buf_128 = (__m128i *)txfm_buf; + __m128i *out_128 = (__m128i *)output; + int num_per_128 = 4; + int txfm2d_size_128 = txfm_size * txfm_size / num_per_128; + + int16_array_with_stride_to_int32_array_without_stride(input, stride, txfm_buf, + txfm_size); + round_shift_array_32_sse4_1(buf_128, out_128, txfm2d_size_128, -shift[0]); + txfm_func_col(out_128, buf_128, cos_bit_col, stage_range_col); + round_shift_array_32_sse4_1(buf_128, out_128, txfm2d_size_128, -shift[1]); + transpose_32(txfm_size, out_128, buf_128); + txfm_func_row(buf_128, out_128, cos_bit_row, stage_range_row); + round_shift_array_32_sse4_1(out_128, buf_128, txfm2d_size_128, -shift[2]); + transpose_32(txfm_size, buf_128, out_128); +} + +void av1_fwd_txfm2d_32x32_sse4_1(const int16_t *input, int32_t *output, + int stride, int tx_type, int bd) { + DECLARE_ALIGNED(16, int32_t, txfm_buf[1024]); + TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_32X32); + (void)bd; + fwd_txfm2d_sse4_1(input, output, stride, cfg.cfg, txfm_buf); +} + +void av1_fwd_txfm2d_64x64_sse4_1(const int16_t *input, int32_t *output, + int stride, int tx_type, int bd) { + DECLARE_ALIGNED(16, int32_t, txfm_buf[4096]); + TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_64x64_cfg(tx_type); + (void)bd; + fwd_txfm2d_sse4_1(input, output, stride, cfg.cfg, txfm_buf); +} diff --git a/third_party/aom/av1/common/x86/av1_highbd_convolve_sse4.c b/third_party/aom/av1/common/x86/av1_highbd_convolve_sse4.c new file mode 100644 index 0000000000..cf6249bdc9 --- /dev/null +++ b/third_party/aom/av1/common/x86/av1_highbd_convolve_sse4.c @@ -0,0 +1,533 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./av1_rtcd.h" +#include "av1/common/filter.h" + +#if CONFIG_DUAL_FILTER +DECLARE_ALIGNED(16, static int16_t, subpel_filters_sharp[15][6][8]); +#endif + +#if USE_TEMPORALFILTER_12TAP +DECLARE_ALIGNED(16, static int16_t, subpel_temporalfilter[15][6][8]); +#endif + +typedef int16_t (*HbdSubpelFilterCoeffs)[8]; + +typedef void (*TransposeSave)(int width, int pixelsNum, uint32_t *src, + int src_stride, uint16_t *dst, int dst_stride, + int bd); + +static INLINE HbdSubpelFilterCoeffs +hbd_get_subpel_filter_ver_signal_dir(const InterpFilterParams p, int index) { +#if CONFIG_DUAL_FILTER + if (p.interp_filter == MULTITAP_SHARP) { + return &subpel_filters_sharp[index][0]; + } +#endif +#if USE_TEMPORALFILTER_12TAP + if (p.interp_filter == TEMPORALFILTER_12TAP) { + return &subpel_temporalfilter[index][0]; + } +#endif + (void)p; + (void)index; + return NULL; +} + +static void init_simd_filter(const int16_t *filter_ptr, int taps, + int16_t (*simd_filter)[6][8]) { + int shift; + int offset = (12 - taps) / 2; + for (shift = 1; shift < SUBPEL_SHIFTS; ++shift) { + const int16_t *filter_row = filter_ptr + shift * taps; + int i, j; + for (i = 0; i < 12; ++i) { + for (j = 0; j < 4; ++j) { + int r = i / 2; + int c = j * 2 + (i % 2); + if (i - offset >= 0 && i - offset < taps) + simd_filter[shift - 1][r][c] = filter_row[i - offset]; + else + simd_filter[shift - 1][r][c] = 0; + } + } + } +} + +void av1_highbd_convolve_init_sse4_1(void) { +#if USE_TEMPORALFILTER_12TAP + { + InterpFilterParams filter_params = + av1_get_interp_filter_params(TEMPORALFILTER_12TAP); + int taps = filter_params.taps; + const int16_t *filter_ptr = filter_params.filter_ptr; + init_simd_filter(filter_ptr, taps, subpel_temporalfilter); + } +#endif +#if CONFIG_DUAL_FILTER + { + InterpFilterParams filter_params = + av1_get_interp_filter_params(MULTITAP_SHARP); + int taps = filter_params.taps; + const int16_t *filter_ptr = filter_params.filter_ptr; + init_simd_filter(filter_ptr, taps, subpel_filters_sharp); + } +#endif +} + +// pixelsNum 0: write all 4 pixels +// 1/2/3: residual pixels 1/2/3 +static void writePixel(__m128i *u, int width, int pixelsNum, uint16_t *dst, + int dst_stride) { + if (2 == width) { + if (0 == pixelsNum) { + *(int *)dst = _mm_cvtsi128_si32(u[0]); + *(int *)(dst + dst_stride) = _mm_cvtsi128_si32(u[1]); + *(int *)(dst + 2 * dst_stride) = _mm_cvtsi128_si32(u[2]); + *(int *)(dst + 3 * dst_stride) = _mm_cvtsi128_si32(u[3]); + } else if (1 == pixelsNum) { + *(int *)dst = _mm_cvtsi128_si32(u[0]); + } else if (2 == pixelsNum) { + *(int *)dst = _mm_cvtsi128_si32(u[0]); + *(int *)(dst + dst_stride) = _mm_cvtsi128_si32(u[1]); + } else if (3 == pixelsNum) { + *(int *)dst = _mm_cvtsi128_si32(u[0]); + *(int *)(dst + dst_stride) = _mm_cvtsi128_si32(u[1]); + *(int *)(dst + 2 * dst_stride) = _mm_cvtsi128_si32(u[2]); + } + } else { + if (0 == pixelsNum) { + _mm_storel_epi64((__m128i *)dst, u[0]); + _mm_storel_epi64((__m128i *)(dst + dst_stride), u[1]); + _mm_storel_epi64((__m128i *)(dst + 2 * dst_stride), u[2]); + _mm_storel_epi64((__m128i *)(dst + 3 * dst_stride), u[3]); + } else if (1 == pixelsNum) { + _mm_storel_epi64((__m128i *)dst, u[0]); + } else if (2 == pixelsNum) { + _mm_storel_epi64((__m128i *)dst, u[0]); + _mm_storel_epi64((__m128i *)(dst + dst_stride), u[1]); + } else if (3 == pixelsNum) { + _mm_storel_epi64((__m128i *)dst, u[0]); + _mm_storel_epi64((__m128i *)(dst + dst_stride), u[1]); + _mm_storel_epi64((__m128i *)(dst + 2 * dst_stride), u[2]); + } + } +} + +// 16-bit pixels clip with bd (10/12) +static void highbd_clip(__m128i *p, int numVecs, int bd) { + const __m128i zero = _mm_setzero_si128(); + const __m128i one = _mm_set1_epi16(1); + const __m128i max = _mm_sub_epi16(_mm_slli_epi16(one, bd), one); + __m128i clamped, mask; + int i; + + for (i = 0; i < numVecs; i++) { + mask = _mm_cmpgt_epi16(p[i], max); + clamped = _mm_andnot_si128(mask, p[i]); + mask = _mm_and_si128(mask, max); + clamped = _mm_or_si128(mask, clamped); + mask = _mm_cmpgt_epi16(clamped, zero); + p[i] = _mm_and_si128(clamped, mask); + } +} + +static void transClipPixel(uint32_t *src, int src_stride, __m128i *u, int bd) { + __m128i v0, v1; + __m128i rnd = _mm_set1_epi32(1 << (FILTER_BITS - 1)); + + u[0] = _mm_loadu_si128((__m128i const *)src); + u[1] = _mm_loadu_si128((__m128i const *)(src + src_stride)); + u[2] = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride)); + u[3] = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride)); + + u[0] = _mm_add_epi32(u[0], rnd); + u[1] = _mm_add_epi32(u[1], rnd); + u[2] = _mm_add_epi32(u[2], rnd); + u[3] = _mm_add_epi32(u[3], rnd); + + u[0] = _mm_srai_epi32(u[0], FILTER_BITS); + u[1] = _mm_srai_epi32(u[1], FILTER_BITS); + u[2] = _mm_srai_epi32(u[2], FILTER_BITS); + u[3] = _mm_srai_epi32(u[3], FILTER_BITS); + + u[0] = _mm_packus_epi32(u[0], u[1]); + u[1] = _mm_packus_epi32(u[2], u[3]); + + highbd_clip(u, 2, bd); + + v0 = _mm_unpacklo_epi16(u[0], u[1]); + v1 = _mm_unpackhi_epi16(u[0], u[1]); + + u[0] = _mm_unpacklo_epi16(v0, v1); + u[2] = _mm_unpackhi_epi16(v0, v1); + + u[1] = _mm_srli_si128(u[0], 8); + u[3] = _mm_srli_si128(u[2], 8); +} + +// pixelsNum = 0 : all 4 rows of pixels will be saved. +// pixelsNum = 1/2/3 : residual 1/2/4 rows of pixels will be saved. +void trans_save_4x4(int width, int pixelsNum, uint32_t *src, int src_stride, + uint16_t *dst, int dst_stride, int bd) { + __m128i u[4]; + transClipPixel(src, src_stride, u, bd); + writePixel(u, width, pixelsNum, dst, dst_stride); +} + +void trans_accum_save_4x4(int width, int pixelsNum, uint32_t *src, + int src_stride, uint16_t *dst, int dst_stride, + int bd) { + __m128i u[4], v[4]; + const __m128i ones = _mm_set1_epi16(1); + + transClipPixel(src, src_stride, u, bd); + + v[0] = _mm_loadl_epi64((__m128i const *)dst); + v[1] = _mm_loadl_epi64((__m128i const *)(dst + dst_stride)); + v[2] = _mm_loadl_epi64((__m128i const *)(dst + 2 * dst_stride)); + v[3] = _mm_loadl_epi64((__m128i const *)(dst + 3 * dst_stride)); + + u[0] = _mm_add_epi16(u[0], v[0]); + u[1] = _mm_add_epi16(u[1], v[1]); + u[2] = _mm_add_epi16(u[2], v[2]); + u[3] = _mm_add_epi16(u[3], v[3]); + + u[0] = _mm_add_epi16(u[0], ones); + u[1] = _mm_add_epi16(u[1], ones); + u[2] = _mm_add_epi16(u[2], ones); + u[3] = _mm_add_epi16(u[3], ones); + + u[0] = _mm_srai_epi16(u[0], 1); + u[1] = _mm_srai_epi16(u[1], 1); + u[2] = _mm_srai_epi16(u[2], 1); + u[3] = _mm_srai_epi16(u[3], 1); + + writePixel(u, width, pixelsNum, dst, dst_stride); +} + +static TransposeSave transSaveTab[2] = { trans_save_4x4, trans_accum_save_4x4 }; + +static INLINE void transpose_pair(__m128i *in, __m128i *out) { + __m128i x0, x1; + + x0 = _mm_unpacklo_epi32(in[0], in[1]); + x1 = _mm_unpacklo_epi32(in[2], in[3]); + + out[0] = _mm_unpacklo_epi64(x0, x1); + out[1] = _mm_unpackhi_epi64(x0, x1); + + x0 = _mm_unpackhi_epi32(in[0], in[1]); + x1 = _mm_unpackhi_epi32(in[2], in[3]); + + out[2] = _mm_unpacklo_epi64(x0, x1); + out[3] = _mm_unpackhi_epi64(x0, x1); + + x0 = _mm_unpacklo_epi32(in[4], in[5]); + x1 = _mm_unpacklo_epi32(in[6], in[7]); + + out[4] = _mm_unpacklo_epi64(x0, x1); + out[5] = _mm_unpackhi_epi64(x0, x1); +} + +static void highbd_filter_horiz(const uint16_t *src, int src_stride, __m128i *f, + int tapsNum, uint32_t *buf) { + __m128i u[8], v[6]; + + if (tapsNum == 10) { + src -= 1; + } + + u[0] = _mm_loadu_si128((__m128i const *)src); + u[1] = _mm_loadu_si128((__m128i const *)(src + src_stride)); + u[2] = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride)); + u[3] = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride)); + + u[4] = _mm_loadu_si128((__m128i const *)(src + 8)); + u[5] = _mm_loadu_si128((__m128i const *)(src + src_stride + 8)); + u[6] = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride + 8)); + u[7] = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride + 8)); + + transpose_pair(u, v); + + u[0] = _mm_madd_epi16(v[0], f[0]); + u[1] = _mm_madd_epi16(v[1], f[1]); + u[2] = _mm_madd_epi16(v[2], f[2]); + u[3] = _mm_madd_epi16(v[3], f[3]); + u[4] = _mm_madd_epi16(v[4], f[4]); + u[5] = _mm_madd_epi16(v[5], f[5]); + + u[6] = _mm_min_epi32(u[2], u[3]); + u[7] = _mm_max_epi32(u[2], u[3]); + + u[0] = _mm_add_epi32(u[0], u[1]); + u[0] = _mm_add_epi32(u[0], u[5]); + u[0] = _mm_add_epi32(u[0], u[4]); + u[0] = _mm_add_epi32(u[0], u[6]); + u[0] = _mm_add_epi32(u[0], u[7]); + + _mm_storeu_si128((__m128i *)buf, u[0]); +} + +void av1_highbd_convolve_horiz_sse4_1(const uint16_t *src, int src_stride, + uint16_t *dst, int dst_stride, int w, + int h, + const InterpFilterParams filter_params, + const int subpel_x_q4, int x_step_q4, + int avg, int bd) { + DECLARE_ALIGNED(16, uint32_t, temp[4 * 4]); + __m128i verf[6]; + HbdSubpelFilterCoeffs vCoeffs; + const uint16_t *srcPtr; + const int tapsNum = filter_params.taps; + int i, col, count, blkResidu, blkHeight; + TransposeSave transSave = transSaveTab[avg]; + (void)x_step_q4; + + if (0 == subpel_x_q4 || 16 != x_step_q4) { + av1_highbd_convolve_horiz_c(src, src_stride, dst, dst_stride, w, h, + filter_params, subpel_x_q4, x_step_q4, avg, bd); + return; + } + + vCoeffs = + hbd_get_subpel_filter_ver_signal_dir(filter_params, subpel_x_q4 - 1); + if (!vCoeffs) { + av1_highbd_convolve_horiz_c(src, src_stride, dst, dst_stride, w, h, + filter_params, subpel_x_q4, x_step_q4, avg, bd); + return; + } + + verf[0] = *((const __m128i *)(vCoeffs)); + verf[1] = *((const __m128i *)(vCoeffs + 1)); + verf[2] = *((const __m128i *)(vCoeffs + 2)); + verf[3] = *((const __m128i *)(vCoeffs + 3)); + verf[4] = *((const __m128i *)(vCoeffs + 4)); + verf[5] = *((const __m128i *)(vCoeffs + 5)); + + src -= (tapsNum >> 1) - 1; + srcPtr = src; + + count = 0; + blkHeight = h >> 2; + blkResidu = h & 3; + + while (blkHeight != 0) { + for (col = 0; col < w; col += 4) { + for (i = 0; i < 4; ++i) { + highbd_filter_horiz(srcPtr, src_stride, verf, tapsNum, temp + (i * 4)); + srcPtr += 1; + } + transSave(w, 0, temp, 4, dst + col, dst_stride, bd); + } + count++; + srcPtr = src + count * src_stride * 4; + dst += dst_stride * 4; + blkHeight--; + } + + if (blkResidu == 0) return; + + for (col = 0; col < w; col += 4) { + for (i = 0; i < 4; ++i) { + highbd_filter_horiz(srcPtr, src_stride, verf, tapsNum, temp + (i * 4)); + srcPtr += 1; + } + transSave(w, blkResidu, temp, 4, dst + col, dst_stride, bd); + } +} + +// Vertical convolutional filter + +typedef void (*WritePixels)(__m128i *u, int bd, uint16_t *dst); + +static void highbdRndingPacks(__m128i *u) { + __m128i rnd = _mm_set1_epi32(1 << (FILTER_BITS - 1)); + u[0] = _mm_add_epi32(u[0], rnd); + u[0] = _mm_srai_epi32(u[0], FILTER_BITS); + u[0] = _mm_packus_epi32(u[0], u[0]); +} + +static void write2pixelsOnly(__m128i *u, int bd, uint16_t *dst) { + highbdRndingPacks(u); + highbd_clip(u, 1, bd); + *(uint32_t *)dst = _mm_cvtsi128_si32(u[0]); +} + +static void write2pixelsAccum(__m128i *u, int bd, uint16_t *dst) { + __m128i v = _mm_loadl_epi64((__m128i const *)dst); + const __m128i ones = _mm_set1_epi16(1); + + highbdRndingPacks(u); + highbd_clip(u, 1, bd); + + v = _mm_add_epi16(v, u[0]); + v = _mm_add_epi16(v, ones); + v = _mm_srai_epi16(v, 1); + *(uint32_t *)dst = _mm_cvtsi128_si32(v); +} + +WritePixels write2pixelsTab[2] = { write2pixelsOnly, write2pixelsAccum }; + +static void write4pixelsOnly(__m128i *u, int bd, uint16_t *dst) { + highbdRndingPacks(u); + highbd_clip(u, 1, bd); + _mm_storel_epi64((__m128i *)dst, u[0]); +} + +static void write4pixelsAccum(__m128i *u, int bd, uint16_t *dst) { + __m128i v = _mm_loadl_epi64((__m128i const *)dst); + const __m128i ones = _mm_set1_epi16(1); + + highbdRndingPacks(u); + highbd_clip(u, 1, bd); + + v = _mm_add_epi16(v, u[0]); + v = _mm_add_epi16(v, ones); + v = _mm_srai_epi16(v, 1); + _mm_storel_epi64((__m128i *)dst, v); +} + +WritePixels write4pixelsTab[2] = { write4pixelsOnly, write4pixelsAccum }; + +static void filter_vert_horiz_parallel(const uint16_t *src, int src_stride, + const __m128i *f, int taps, + uint16_t *dst, WritePixels saveFunc, + int bd) { + __m128i s[12]; + __m128i zero = _mm_setzero_si128(); + int i = 0; + int r = 0; + + // TODO(luoyi) treat s[12] as a circular buffer in width = 2 case + if (10 == taps) { + i += 1; + s[0] = zero; + } + while (i < 12) { + s[i] = _mm_loadu_si128((__m128i const *)(src + r * src_stride)); + i += 1; + r += 1; + } + + s[0] = _mm_unpacklo_epi16(s[0], s[1]); + s[2] = _mm_unpacklo_epi16(s[2], s[3]); + s[4] = _mm_unpacklo_epi16(s[4], s[5]); + s[6] = _mm_unpacklo_epi16(s[6], s[7]); + s[8] = _mm_unpacklo_epi16(s[8], s[9]); + s[10] = _mm_unpacklo_epi16(s[10], s[11]); + + s[0] = _mm_madd_epi16(s[0], f[0]); + s[2] = _mm_madd_epi16(s[2], f[1]); + s[4] = _mm_madd_epi16(s[4], f[2]); + s[6] = _mm_madd_epi16(s[6], f[3]); + s[8] = _mm_madd_epi16(s[8], f[4]); + s[10] = _mm_madd_epi16(s[10], f[5]); + + s[1] = _mm_min_epi32(s[4], s[6]); + s[3] = _mm_max_epi32(s[4], s[6]); + + s[0] = _mm_add_epi32(s[0], s[2]); + s[0] = _mm_add_epi32(s[0], s[10]); + s[0] = _mm_add_epi32(s[0], s[8]); + s[0] = _mm_add_epi32(s[0], s[1]); + s[0] = _mm_add_epi32(s[0], s[3]); + + saveFunc(s, bd, dst); +} + +static void highbd_filter_vert_compute_large(const uint16_t *src, + int src_stride, const __m128i *f, + int taps, int w, int h, + uint16_t *dst, int dst_stride, + int avg, int bd) { + int col; + int rowIndex = 0; + const uint16_t *src_ptr = src; + uint16_t *dst_ptr = dst; + const int step = 4; + WritePixels write4pixels = write4pixelsTab[avg]; + + do { + for (col = 0; col < w; col += step) { + filter_vert_horiz_parallel(src_ptr, src_stride, f, taps, dst_ptr, + write4pixels, bd); + src_ptr += step; + dst_ptr += step; + } + rowIndex++; + src_ptr = src + rowIndex * src_stride; + dst_ptr = dst + rowIndex * dst_stride; + } while (rowIndex < h); +} + +static void highbd_filter_vert_compute_small(const uint16_t *src, + int src_stride, const __m128i *f, + int taps, int w, int h, + uint16_t *dst, int dst_stride, + int avg, int bd) { + int rowIndex = 0; + WritePixels write2pixels = write2pixelsTab[avg]; + (void)w; + + do { + filter_vert_horiz_parallel(src, src_stride, f, taps, dst, write2pixels, bd); + rowIndex++; + src += src_stride; + dst += dst_stride; + } while (rowIndex < h); +} + +void av1_highbd_convolve_vert_sse4_1(const uint16_t *src, int src_stride, + uint16_t *dst, int dst_stride, int w, + int h, + const InterpFilterParams filter_params, + const int subpel_y_q4, int y_step_q4, + int avg, int bd) { + __m128i verf[6]; + HbdSubpelFilterCoeffs vCoeffs; + const int tapsNum = filter_params.taps; + + if (0 == subpel_y_q4 || 16 != y_step_q4) { + av1_highbd_convolve_vert_c(src, src_stride, dst, dst_stride, w, h, + filter_params, subpel_y_q4, y_step_q4, avg, bd); + return; + } + + vCoeffs = + hbd_get_subpel_filter_ver_signal_dir(filter_params, subpel_y_q4 - 1); + if (!vCoeffs) { + av1_highbd_convolve_vert_c(src, src_stride, dst, dst_stride, w, h, + filter_params, subpel_y_q4, y_step_q4, avg, bd); + return; + } + + verf[0] = *((const __m128i *)(vCoeffs)); + verf[1] = *((const __m128i *)(vCoeffs + 1)); + verf[2] = *((const __m128i *)(vCoeffs + 2)); + verf[3] = *((const __m128i *)(vCoeffs + 3)); + verf[4] = *((const __m128i *)(vCoeffs + 4)); + verf[5] = *((const __m128i *)(vCoeffs + 5)); + + src -= src_stride * ((tapsNum >> 1) - 1); + + if (w > 2) { + highbd_filter_vert_compute_large(src, src_stride, verf, tapsNum, w, h, dst, + dst_stride, avg, bd); + } else { + highbd_filter_vert_compute_small(src, src_stride, verf, tapsNum, w, h, dst, + dst_stride, avg, bd); + } +} diff --git a/third_party/aom/av1/common/x86/av1_txfm1d_sse4.h b/third_party/aom/av1/common/x86/av1_txfm1d_sse4.h new file mode 100644 index 0000000000..af7afb7ee5 --- /dev/null +++ b/third_party/aom/av1/common/x86/av1_txfm1d_sse4.h @@ -0,0 +1,144 @@ +#ifndef AV1_TXMF1D_SSE2_H_ +#define AV1_TXMF1D_SSE2_H_ + +#include +#include "av1/common/av1_txfm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_fdct4_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_fdct8_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_fdct16_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_fdct32_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_fdct64_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); + +void av1_fadst4_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_fadst8_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_fadst16_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_fadst32_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); + +void av1_idct4_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_idct8_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_idct16_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_idct32_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_idct64_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); + +void av1_iadst4_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_iadst8_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_iadst16_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); +void av1_iadst32_new_sse4_1(const __m128i *input, __m128i *output, + const int8_t *cos_bit, const int8_t *stage_range); + +static INLINE void transpose_32_4x4(int stride, const __m128i *input, + __m128i *output) { + __m128i temp0 = _mm_unpacklo_epi32(input[0 * stride], input[2 * stride]); + __m128i temp1 = _mm_unpackhi_epi32(input[0 * stride], input[2 * stride]); + __m128i temp2 = _mm_unpacklo_epi32(input[1 * stride], input[3 * stride]); + __m128i temp3 = _mm_unpackhi_epi32(input[1 * stride], input[3 * stride]); + + output[0 * stride] = _mm_unpacklo_epi32(temp0, temp2); + output[1 * stride] = _mm_unpackhi_epi32(temp0, temp2); + output[2 * stride] = _mm_unpacklo_epi32(temp1, temp3); + output[3 * stride] = _mm_unpackhi_epi32(temp1, temp3); +} + +// the entire input block can be represent by a grid of 4x4 blocks +// each 4x4 blocks can be represent by 4 vertical __m128i +// we first transpose each 4x4 block internally +// than transpose the grid +static INLINE void transpose_32(int txfm_size, const __m128i *input, + __m128i *output) { + const int num_per_128 = 4; + const int row_size = txfm_size; + const int col_size = txfm_size / num_per_128; + int r, c; + + // transpose each 4x4 block internally + for (r = 0; r < row_size; r += 4) { + for (c = 0; c < col_size; c++) { + transpose_32_4x4(col_size, &input[r * col_size + c], + &output[c * 4 * col_size + r / 4]); + } + } +} + +static INLINE __m128i round_shift_32_sse4_1(__m128i vec, int bit) { + __m128i tmp, round; + round = _mm_set1_epi32(1 << (bit - 1)); + tmp = _mm_add_epi32(vec, round); + return _mm_srai_epi32(tmp, bit); +} + +static INLINE void round_shift_array_32_sse4_1(__m128i *input, __m128i *output, + const int size, const int bit) { + if (bit > 0) { + int i; + for (i = 0; i < size; i++) { + output[i] = round_shift_32_sse4_1(input[i], bit); + } + } else { + int i; + for (i = 0; i < size; i++) { + output[i] = _mm_slli_epi32(input[i], -bit); + } + } +} + +// out0 = in0*w0 + in1*w1 +// out1 = -in1*w0 + in0*w1 +#define btf_32_sse4_1_type0(w0, w1, in0, in1, out0, out1, bit) \ + do { \ + __m128i ww0, ww1, in0_w0, in1_w1, in0_w1, in1_w0; \ + ww0 = _mm_set1_epi32(w0); \ + ww1 = _mm_set1_epi32(w1); \ + in0_w0 = _mm_mullo_epi32(in0, ww0); \ + in1_w1 = _mm_mullo_epi32(in1, ww1); \ + out0 = _mm_add_epi32(in0_w0, in1_w1); \ + out0 = round_shift_32_sse4_1(out0, bit); \ + in0_w1 = _mm_mullo_epi32(in0, ww1); \ + in1_w0 = _mm_mullo_epi32(in1, ww0); \ + out1 = _mm_sub_epi32(in0_w1, in1_w0); \ + out1 = round_shift_32_sse4_1(out1, bit); \ + } while (0) + +// out0 = in0*w0 + in1*w1 +// out1 = in1*w0 - in0*w1 +#define btf_32_sse4_1_type1(w0, w1, in0, in1, out0, out1, bit) \ + do { \ + __m128i ww0, ww1, in0_w0, in1_w1, in0_w1, in1_w0; \ + ww0 = _mm_set1_epi32(w0); \ + ww1 = _mm_set1_epi32(w1); \ + in0_w0 = _mm_mullo_epi32(in0, ww0); \ + in1_w1 = _mm_mullo_epi32(in1, ww1); \ + out0 = _mm_add_epi32(in0_w0, in1_w1); \ + out0 = round_shift_32_sse4_1(out0, bit); \ + in0_w1 = _mm_mullo_epi32(in0, ww1); \ + in1_w0 = _mm_mullo_epi32(in1, ww0); \ + out1 = _mm_sub_epi32(in1_w0, in0_w1); \ + out1 = round_shift_32_sse4_1(out1, bit); \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif // AV1_TXMF1D_SSE2_H_ diff --git a/third_party/aom/av1/common/x86/filterintra_sse4.c b/third_party/aom/av1/common/x86/filterintra_sse4.c new file mode 100644 index 0000000000..4f77da4463 --- /dev/null +++ b/third_party/aom/av1/common/x86/filterintra_sse4.c @@ -0,0 +1,898 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./av1_rtcd.h" +#include "aom_ports/mem.h" +#include "av1/common/enums.h" +#include "av1/common/reconintra.h" + +#if USE_3TAP_INTRA_FILTER +void filterintra_sse4_3tap_dummy_func(void); +void filterintra_sse4_3tap_dummy_func(void) {} +#else + +static INLINE void AddPixelsSmall(const uint8_t *above, const uint8_t *left, + __m128i *sum) { + const __m128i a = _mm_loadu_si128((const __m128i *)above); + const __m128i l = _mm_loadu_si128((const __m128i *)left); + const __m128i zero = _mm_setzero_si128(); + + __m128i u0 = _mm_unpacklo_epi8(a, zero); + __m128i u1 = _mm_unpacklo_epi8(l, zero); + + sum[0] = _mm_add_epi16(u0, u1); +} + +static INLINE int GetMeanValue4x4(const uint8_t *above, const uint8_t *left, + __m128i *params) { + const __m128i zero = _mm_setzero_si128(); + __m128i sum_vector, u; + uint16_t sum_value; + + AddPixelsSmall(above, left, &sum_vector); + + sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 2 values + u = _mm_srli_si128(sum_vector, 2); + sum_vector = _mm_add_epi16(sum_vector, u); + + sum_value = _mm_extract_epi16(sum_vector, 0); + sum_value += 4; + sum_value >>= 3; + *params = _mm_set1_epi32(sum_value); + return sum_value; +} + +static INLINE int GetMeanValue8x8(const uint8_t *above, const uint8_t *left, + __m128i *params) { + const __m128i zero = _mm_setzero_si128(); + __m128i sum_vector, u; + uint16_t sum_value; + + AddPixelsSmall(above, left, &sum_vector); + + sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 4 values + sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 2 values + + u = _mm_srli_si128(sum_vector, 2); + sum_vector = _mm_add_epi16(sum_vector, u); + + sum_value = _mm_extract_epi16(sum_vector, 0); + sum_value += 8; + sum_value >>= 4; + *params = _mm_set1_epi32(sum_value); + return sum_value; +} + +static INLINE void AddPixelsLarge(const uint8_t *above, const uint8_t *left, + __m128i *sum) { + const __m128i a = _mm_loadu_si128((const __m128i *)above); + const __m128i l = _mm_loadu_si128((const __m128i *)left); + const __m128i zero = _mm_setzero_si128(); + + __m128i u0 = _mm_unpacklo_epi8(a, zero); + __m128i u1 = _mm_unpacklo_epi8(l, zero); + + sum[0] = _mm_add_epi16(u0, u1); + + u0 = _mm_unpackhi_epi8(a, zero); + u1 = _mm_unpackhi_epi8(l, zero); + + sum[0] = _mm_add_epi16(sum[0], u0); + sum[0] = _mm_add_epi16(sum[0], u1); +} + +static INLINE int GetMeanValue16x16(const uint8_t *above, const uint8_t *left, + __m128i *params) { + const __m128i zero = _mm_setzero_si128(); + __m128i sum_vector, u; + uint16_t sum_value; + + AddPixelsLarge(above, left, &sum_vector); + + sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 4 values + sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 2 values + + u = _mm_srli_si128(sum_vector, 2); + sum_vector = _mm_add_epi16(sum_vector, u); + + sum_value = _mm_extract_epi16(sum_vector, 0); + sum_value += 16; + sum_value >>= 5; + *params = _mm_set1_epi32(sum_value); + return sum_value; +} + +static INLINE int GetMeanValue32x32(const uint8_t *above, const uint8_t *left, + __m128i *params) { + const __m128i zero = _mm_setzero_si128(); + __m128i sum_vector[2], u; + uint16_t sum_value; + + AddPixelsLarge(above, left, &sum_vector[0]); + AddPixelsLarge(above + 16, left + 16, &sum_vector[1]); + + sum_vector[0] = _mm_add_epi16(sum_vector[0], sum_vector[1]); + sum_vector[0] = _mm_hadd_epi16(sum_vector[0], zero); // still has 4 values + sum_vector[0] = _mm_hadd_epi16(sum_vector[0], zero); // still has 2 values + + u = _mm_srli_si128(sum_vector[0], 2); + sum_vector[0] = _mm_add_epi16(sum_vector[0], u); + + sum_value = _mm_extract_epi16(sum_vector[0], 0); + sum_value += 32; + sum_value >>= 6; + *params = _mm_set1_epi32(sum_value); + return sum_value; +} + +// Note: +// params[4] : mean value, 4 int32_t repetition +// +static INLINE int CalcRefPixelsMeanValue(const uint8_t *above, + const uint8_t *left, int bs, + __m128i *params) { + int meanValue = 0; + switch (bs) { + case 4: meanValue = GetMeanValue4x4(above, left, params); break; + case 8: meanValue = GetMeanValue8x8(above, left, params); break; + case 16: meanValue = GetMeanValue16x16(above, left, params); break; + case 32: meanValue = GetMeanValue32x32(above, left, params); break; + default: assert(0); + } + return meanValue; +} + +// Note: +// params[0-3] : 4-tap filter coefficients (int32_t per coefficient) +// +static INLINE void GetIntraFilterParams(int bs, int mode, __m128i *params) { + const TX_SIZE tx_size = + (bs == 32) ? TX_32X32 + : ((bs == 16) ? TX_16X16 : ((bs == 8) ? TX_8X8 : (TX_4X4))); + // c0 + params[0] = _mm_set_epi32(av1_filter_intra_taps_4[tx_size][mode][0], + av1_filter_intra_taps_4[tx_size][mode][0], + av1_filter_intra_taps_4[tx_size][mode][0], + av1_filter_intra_taps_4[tx_size][mode][0]); + // c1 + params[1] = _mm_set_epi32(av1_filter_intra_taps_4[tx_size][mode][1], + av1_filter_intra_taps_4[tx_size][mode][1], + av1_filter_intra_taps_4[tx_size][mode][1], + av1_filter_intra_taps_4[tx_size][mode][1]); + // c2 + params[2] = _mm_set_epi32(av1_filter_intra_taps_4[tx_size][mode][2], + av1_filter_intra_taps_4[tx_size][mode][2], + av1_filter_intra_taps_4[tx_size][mode][2], + av1_filter_intra_taps_4[tx_size][mode][2]); + // c3 + params[3] = _mm_set_epi32(av1_filter_intra_taps_4[tx_size][mode][3], + av1_filter_intra_taps_4[tx_size][mode][3], + av1_filter_intra_taps_4[tx_size][mode][3], + av1_filter_intra_taps_4[tx_size][mode][3]); +} + +static const int maxBlkSize = 32; + +static INLINE void SavePred4x4(int *pred, const __m128i *mean, uint8_t *dst, + ptrdiff_t stride) { + const int predStride = (maxBlkSize << 1) + 1; + __m128i p0 = _mm_loadu_si128((const __m128i *)pred); + __m128i p1 = _mm_loadu_si128((const __m128i *)(pred + predStride)); + __m128i p2 = _mm_loadu_si128((const __m128i *)(pred + 2 * predStride)); + __m128i p3 = _mm_loadu_si128((const __m128i *)(pred + 3 * predStride)); + + p0 = _mm_add_epi32(p0, mean[0]); + p1 = _mm_add_epi32(p1, mean[0]); + p2 = _mm_add_epi32(p2, mean[0]); + p3 = _mm_add_epi32(p3, mean[0]); + + p0 = _mm_packus_epi32(p0, p1); + p1 = _mm_packus_epi32(p2, p3); + p0 = _mm_packus_epi16(p0, p1); + + *((int *)dst) = _mm_cvtsi128_si32(p0); + p0 = _mm_srli_si128(p0, 4); + *((int *)(dst + stride)) = _mm_cvtsi128_si32(p0); + p0 = _mm_srli_si128(p0, 4); + *((int *)(dst + 2 * stride)) = _mm_cvtsi128_si32(p0); + p0 = _mm_srli_si128(p0, 4); + *((int *)(dst + 3 * stride)) = _mm_cvtsi128_si32(p0); +} + +static void SavePred8x8(int *pred, const __m128i *mean, uint8_t *dst, + ptrdiff_t stride) { + const int predStride = (maxBlkSize << 1) + 1; + __m128i p0, p1, p2, p3; + int r = 0; + + while (r < 8) { + p0 = _mm_loadu_si128((const __m128i *)(pred + r * predStride)); + p1 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 4)); + r += 1; + p2 = _mm_loadu_si128((const __m128i *)(pred + r * predStride)); + p3 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 4)); + + p0 = _mm_add_epi32(p0, mean[0]); + p1 = _mm_add_epi32(p1, mean[0]); + p2 = _mm_add_epi32(p2, mean[0]); + p3 = _mm_add_epi32(p3, mean[0]); + + p0 = _mm_packus_epi32(p0, p1); + p1 = _mm_packus_epi32(p2, p3); + p0 = _mm_packus_epi16(p0, p1); + + _mm_storel_epi64((__m128i *)dst, p0); + dst += stride; + p0 = _mm_srli_si128(p0, 8); + _mm_storel_epi64((__m128i *)dst, p0); + dst += stride; + r += 1; + } +} + +static void SavePred16x16(int *pred, const __m128i *mean, uint8_t *dst, + ptrdiff_t stride) { + const int predStride = (maxBlkSize << 1) + 1; + __m128i p0, p1, p2, p3; + int r = 0; + + while (r < 16) { + p0 = _mm_loadu_si128((const __m128i *)(pred + r * predStride)); + p1 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 4)); + p2 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 8)); + p3 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 12)); + + p0 = _mm_add_epi32(p0, mean[0]); + p1 = _mm_add_epi32(p1, mean[0]); + p2 = _mm_add_epi32(p2, mean[0]); + p3 = _mm_add_epi32(p3, mean[0]); + + p0 = _mm_packus_epi32(p0, p1); + p1 = _mm_packus_epi32(p2, p3); + p0 = _mm_packus_epi16(p0, p1); + + _mm_storel_epi64((__m128i *)dst, p0); + p0 = _mm_srli_si128(p0, 8); + _mm_storel_epi64((__m128i *)(dst + 8), p0); + dst += stride; + r += 1; + } +} + +static void SavePred32x32(int *pred, const __m128i *mean, uint8_t *dst, + ptrdiff_t stride) { + const int predStride = (maxBlkSize << 1) + 1; + __m128i p0, p1, p2, p3, p4, p5, p6, p7; + int r = 0; + + while (r < 32) { + p0 = _mm_loadu_si128((const __m128i *)(pred + r * predStride)); + p1 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 4)); + p2 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 8)); + p3 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 12)); + + p4 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 16)); + p5 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 20)); + p6 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 24)); + p7 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 28)); + + p0 = _mm_add_epi32(p0, mean[0]); + p1 = _mm_add_epi32(p1, mean[0]); + p2 = _mm_add_epi32(p2, mean[0]); + p3 = _mm_add_epi32(p3, mean[0]); + + p4 = _mm_add_epi32(p4, mean[0]); + p5 = _mm_add_epi32(p5, mean[0]); + p6 = _mm_add_epi32(p6, mean[0]); + p7 = _mm_add_epi32(p7, mean[0]); + + p0 = _mm_packus_epi32(p0, p1); + p1 = _mm_packus_epi32(p2, p3); + p0 = _mm_packus_epi16(p0, p1); + + p4 = _mm_packus_epi32(p4, p5); + p5 = _mm_packus_epi32(p6, p7); + p4 = _mm_packus_epi16(p4, p5); + + _mm_storel_epi64((__m128i *)dst, p0); + p0 = _mm_srli_si128(p0, 8); + _mm_storel_epi64((__m128i *)(dst + 8), p0); + + _mm_storel_epi64((__m128i *)(dst + 16), p4); + p4 = _mm_srli_si128(p4, 8); + _mm_storel_epi64((__m128i *)(dst + 24), p4); + + dst += stride; + r += 1; + } +} + +static void SavePrediction(int *pred, const __m128i *mean, int bs, uint8_t *dst, + ptrdiff_t stride) { + switch (bs) { + case 4: SavePred4x4(pred, mean, dst, stride); break; + case 8: SavePred8x8(pred, mean, dst, stride); break; + case 16: SavePred16x16(pred, mean, dst, stride); break; + case 32: SavePred32x32(pred, mean, dst, stride); break; + default: assert(0); + } +} + +typedef void (*ProducePixelsFunc)(__m128i *p, const __m128i *prm, int *pred, + const int predStride); + +static void ProduceFourPixels(__m128i *p, const __m128i *prm, int *pred, + const int predStride) { + __m128i u0, u1, u2; + int c0 = _mm_extract_epi32(prm[1], 0); + int x = *(pred + predStride); + int sum; + + u0 = _mm_mullo_epi32(p[0], prm[2]); + u1 = _mm_mullo_epi32(p[1], prm[0]); + u2 = _mm_mullo_epi32(p[2], prm[3]); + + u0 = _mm_add_epi32(u0, u1); + u0 = _mm_add_epi32(u0, u2); + + sum = _mm_extract_epi32(u0, 0); + sum += c0 * x; + x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS); + *(pred + predStride + 1) = x; + + sum = _mm_extract_epi32(u0, 1); + sum += c0 * x; + x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS); + *(pred + predStride + 2) = x; + + sum = _mm_extract_epi32(u0, 2); + sum += c0 * x; + x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS); + *(pred + predStride + 3) = x; + + sum = _mm_extract_epi32(u0, 3); + sum += c0 * x; + x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS); + *(pred + predStride + 4) = x; +} + +static void ProduceThreePixels(__m128i *p, const __m128i *prm, int *pred, + const int predStride) { + __m128i u0, u1, u2; + int c0 = _mm_extract_epi32(prm[1], 0); + int x = *(pred + predStride); + int sum; + + u0 = _mm_mullo_epi32(p[0], prm[2]); + u1 = _mm_mullo_epi32(p[1], prm[0]); + u2 = _mm_mullo_epi32(p[2], prm[3]); + + u0 = _mm_add_epi32(u0, u1); + u0 = _mm_add_epi32(u0, u2); + + sum = _mm_extract_epi32(u0, 0); + sum += c0 * x; + x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS); + *(pred + predStride + 1) = x; + + sum = _mm_extract_epi32(u0, 1); + sum += c0 * x; + x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS); + *(pred + predStride + 2) = x; + + sum = _mm_extract_epi32(u0, 2); + sum += c0 * x; + x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS); + *(pred + predStride + 3) = x; +} + +static void ProduceTwoPixels(__m128i *p, const __m128i *prm, int *pred, + const int predStride) { + __m128i u0, u1, u2; + int c0 = _mm_extract_epi32(prm[1], 0); + int x = *(pred + predStride); + int sum; + + u0 = _mm_mullo_epi32(p[0], prm[2]); + u1 = _mm_mullo_epi32(p[1], prm[0]); + u2 = _mm_mullo_epi32(p[2], prm[3]); + + u0 = _mm_add_epi32(u0, u1); + u0 = _mm_add_epi32(u0, u2); + + sum = _mm_extract_epi32(u0, 0); + sum += c0 * x; + x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS); + *(pred + predStride + 1) = x; + + sum = _mm_extract_epi32(u0, 1); + sum += c0 * x; + x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS); + *(pred + predStride + 2) = x; +} + +static void ProduceOnePixels(__m128i *p, const __m128i *prm, int *pred, + const int predStride) { + __m128i u0, u1, u2; + int c0 = _mm_extract_epi32(prm[1], 0); + int x = *(pred + predStride); + int sum; + + u0 = _mm_mullo_epi32(p[0], prm[2]); + u1 = _mm_mullo_epi32(p[1], prm[0]); + u2 = _mm_mullo_epi32(p[2], prm[3]); + + u0 = _mm_add_epi32(u0, u1); + u0 = _mm_add_epi32(u0, u2); + + sum = _mm_extract_epi32(u0, 0); + sum += c0 * x; + x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS); + *(pred + predStride + 1) = x; +} + +static ProducePixelsFunc prodPixelsFuncTab[4] = { + ProduceOnePixels, ProduceTwoPixels, ProduceThreePixels, ProduceFourPixels +}; + +static void ProducePixels(int *pred, const __m128i *prm, int remain) { + __m128i p[3]; + const int predStride = (maxBlkSize << 1) + 1; + int index; + + p[0] = _mm_loadu_si128((const __m128i *)pred); + p[1] = _mm_loadu_si128((const __m128i *)(pred + 1)); + p[2] = _mm_loadu_si128((const __m128i *)(pred + 2)); + + if (remain <= 2) { + return; + } + if (remain > 5) { + index = 3; + } else { + index = remain - 3; + } + prodPixelsFuncTab[index](p, prm, pred, predStride); +} + +// Note: +// At column index c, the remaining pixels are R = 2 * bs + 1 - r - c +// the number of pixels to produce is R - 2 = 2 * bs - r - c - 1 +static void GeneratePrediction(const uint8_t *above, const uint8_t *left, + const int bs, const __m128i *prm, int meanValue, + uint8_t *dst, ptrdiff_t stride) { + int pred[33][65]; + int r, c, colBound; + int remainings; + + for (r = 0; r < bs; ++r) { + pred[r + 1][0] = (int)left[r] - meanValue; + } + + above -= 1; + for (c = 0; c < 2 * bs + 1; ++c) { + pred[0][c] = (int)above[c] - meanValue; + } + + r = 0; + c = 0; + while (r < bs) { + colBound = (bs << 1) - r; + for (c = 0; c < colBound; c += 4) { + remainings = colBound - c + 1; + ProducePixels(&pred[r][c], prm, remainings); + } + r += 1; + } + + SavePrediction(&pred[1][1], &prm[4], bs, dst, stride); +} + +static void FilterPrediction(const uint8_t *above, const uint8_t *left, int bs, + __m128i *prm, uint8_t *dst, ptrdiff_t stride) { + int meanValue = 0; + meanValue = CalcRefPixelsMeanValue(above, left, bs, &prm[4]); + GeneratePrediction(above, left, bs, prm, meanValue, dst, stride); +} + +void av1_dc_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + __m128i prm[5]; + GetIntraFilterParams(bs, DC_PRED, &prm[0]); + FilterPrediction(above, left, bs, prm, dst, stride); +} + +void av1_v_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + __m128i prm[5]; + GetIntraFilterParams(bs, V_PRED, &prm[0]); + FilterPrediction(above, left, bs, prm, dst, stride); +} + +void av1_h_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + __m128i prm[5]; + GetIntraFilterParams(bs, H_PRED, &prm[0]); + FilterPrediction(above, left, bs, prm, dst, stride); +} + +void av1_d45_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, + const uint8_t *left) { + __m128i prm[5]; + GetIntraFilterParams(bs, D45_PRED, &prm[0]); + FilterPrediction(above, left, bs, prm, dst, stride); +} + +void av1_d135_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, + const uint8_t *left) { + __m128i prm[5]; + GetIntraFilterParams(bs, D135_PRED, &prm[0]); + FilterPrediction(above, left, bs, prm, dst, stride); +} + +void av1_d117_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, + const uint8_t *left) { + __m128i prm[5]; + GetIntraFilterParams(bs, D117_PRED, &prm[0]); + FilterPrediction(above, left, bs, prm, dst, stride); +} + +void av1_d153_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, + const uint8_t *left) { + __m128i prm[5]; + GetIntraFilterParams(bs, D153_PRED, &prm[0]); + FilterPrediction(above, left, bs, prm, dst, stride); +} + +void av1_d207_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, + const uint8_t *left) { + __m128i prm[5]; + GetIntraFilterParams(bs, D207_PRED, &prm[0]); + FilterPrediction(above, left, bs, prm, dst, stride); +} + +void av1_d63_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, + const uint8_t *left) { + __m128i prm[5]; + GetIntraFilterParams(bs, D63_PRED, &prm[0]); + FilterPrediction(above, left, bs, prm, dst, stride); +} + +void av1_tm_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left) { + __m128i prm[5]; + GetIntraFilterParams(bs, TM_PRED, &prm[0]); + FilterPrediction(above, left, bs, prm, dst, stride); +} + +// ============== High Bit Depth ============== +#if CONFIG_HIGHBITDEPTH +static INLINE int HighbdGetMeanValue4x4(const uint16_t *above, + const uint16_t *left, const int bd, + __m128i *params) { + const __m128i a = _mm_loadu_si128((const __m128i *)above); + const __m128i l = _mm_loadu_si128((const __m128i *)left); + const __m128i zero = _mm_setzero_si128(); + __m128i sum_vector, u; + uint16_t sum_value; + (void)bd; + + sum_vector = _mm_add_epi16(a, l); + + sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 2 values + u = _mm_srli_si128(sum_vector, 2); + sum_vector = _mm_add_epi16(sum_vector, u); + + sum_value = _mm_extract_epi16(sum_vector, 0); + sum_value += 4; + sum_value >>= 3; + *params = _mm_set1_epi32(sum_value); + return sum_value; +} + +static INLINE int HighbdGetMeanValue8x8(const uint16_t *above, + const uint16_t *left, const int bd, + __m128i *params) { + const __m128i a = _mm_loadu_si128((const __m128i *)above); + const __m128i l = _mm_loadu_si128((const __m128i *)left); + const __m128i zero = _mm_setzero_si128(); + __m128i sum_vector, u; + uint16_t sum_value; + (void)bd; + + sum_vector = _mm_add_epi16(a, l); + + sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 4 values + sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 2 values + + u = _mm_srli_si128(sum_vector, 2); + sum_vector = _mm_add_epi16(sum_vector, u); + + sum_value = _mm_extract_epi16(sum_vector, 0); + sum_value += 8; + sum_value >>= 4; + *params = _mm_set1_epi32(sum_value); + return sum_value; +} + +// Note: +// Process 16 pixels above and left, 10-bit depth +// Add to the last 8 pixels sum +static INLINE void AddPixels10bit(const uint16_t *above, const uint16_t *left, + __m128i *sum) { + __m128i a = _mm_loadu_si128((const __m128i *)above); + __m128i l = _mm_loadu_si128((const __m128i *)left); + sum[0] = _mm_add_epi16(a, l); + a = _mm_loadu_si128((const __m128i *)(above + 8)); + l = _mm_loadu_si128((const __m128i *)(left + 8)); + sum[0] = _mm_add_epi16(sum[0], a); + sum[0] = _mm_add_epi16(sum[0], l); +} + +// Note: +// Process 16 pixels above and left, 12-bit depth +// Add to the last 8 pixels sum +static INLINE void AddPixels12bit(const uint16_t *above, const uint16_t *left, + __m128i *sum) { + __m128i a = _mm_loadu_si128((const __m128i *)above); + __m128i l = _mm_loadu_si128((const __m128i *)left); + const __m128i zero = _mm_setzero_si128(); + __m128i v0, v1; + + v0 = _mm_unpacklo_epi16(a, zero); + v1 = _mm_unpacklo_epi16(l, zero); + sum[0] = _mm_add_epi32(v0, v1); + + v0 = _mm_unpackhi_epi16(a, zero); + v1 = _mm_unpackhi_epi16(l, zero); + sum[0] = _mm_add_epi32(sum[0], v0); + sum[0] = _mm_add_epi32(sum[0], v1); + + a = _mm_loadu_si128((const __m128i *)(above + 8)); + l = _mm_loadu_si128((const __m128i *)(left + 8)); + + v0 = _mm_unpacklo_epi16(a, zero); + v1 = _mm_unpacklo_epi16(l, zero); + sum[0] = _mm_add_epi32(sum[0], v0); + sum[0] = _mm_add_epi32(sum[0], v1); + + v0 = _mm_unpackhi_epi16(a, zero); + v1 = _mm_unpackhi_epi16(l, zero); + sum[0] = _mm_add_epi32(sum[0], v0); + sum[0] = _mm_add_epi32(sum[0], v1); +} + +static INLINE int HighbdGetMeanValue16x16(const uint16_t *above, + const uint16_t *left, const int bd, + __m128i *params) { + const __m128i zero = _mm_setzero_si128(); + __m128i sum_vector, u; + uint32_t sum_value = 0; + + if (10 == bd) { + AddPixels10bit(above, left, &sum_vector); + sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 4 values + sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 2 values + + u = _mm_srli_si128(sum_vector, 2); + sum_vector = _mm_add_epi16(sum_vector, u); + sum_value = _mm_extract_epi16(sum_vector, 0); + } else if (12 == bd) { + AddPixels12bit(above, left, &sum_vector); + + sum_vector = _mm_hadd_epi32(sum_vector, zero); + u = _mm_srli_si128(sum_vector, 4); + sum_vector = _mm_add_epi32(u, sum_vector); + sum_value = _mm_extract_epi32(sum_vector, 0); + } + + sum_value += 16; + sum_value >>= 5; + *params = _mm_set1_epi32(sum_value); + return sum_value; +} + +static INLINE int HighbdGetMeanValue32x32(const uint16_t *above, + const uint16_t *left, const int bd, + __m128i *params) { + const __m128i zero = _mm_setzero_si128(); + __m128i sum_vector[2], u; + uint32_t sum_value = 0; + + if (10 == bd) { + AddPixels10bit(above, left, &sum_vector[0]); + AddPixels10bit(above + 16, left + 16, &sum_vector[1]); + + sum_vector[0] = _mm_add_epi16(sum_vector[0], sum_vector[1]); + sum_vector[0] = _mm_hadd_epi16(sum_vector[0], zero); // still has 4 values + sum_vector[0] = _mm_hadd_epi16(sum_vector[0], zero); // still has 2 values + + u = _mm_srli_si128(sum_vector[0], 2); + sum_vector[0] = _mm_add_epi16(sum_vector[0], u); + sum_value = _mm_extract_epi16(sum_vector[0], 0); + } else if (12 == bd) { + AddPixels12bit(above, left, &sum_vector[0]); + AddPixels12bit(above + 16, left + 16, &sum_vector[1]); + + sum_vector[0] = _mm_add_epi32(sum_vector[0], sum_vector[1]); + sum_vector[0] = _mm_hadd_epi32(sum_vector[0], zero); + u = _mm_srli_si128(sum_vector[0], 4); + sum_vector[0] = _mm_add_epi32(u, sum_vector[0]); + sum_value = _mm_extract_epi32(sum_vector[0], 0); + } + + sum_value += 32; + sum_value >>= 6; + *params = _mm_set1_epi32(sum_value); + return sum_value; +} + +// Note: +// params[4] : mean value, 4 int32_t repetition +// +static INLINE int HighbdCalcRefPixelsMeanValue(const uint16_t *above, + const uint16_t *left, int bs, + const int bd, __m128i *params) { + int meanValue = 0; + switch (bs) { + case 4: meanValue = HighbdGetMeanValue4x4(above, left, bd, params); break; + case 8: meanValue = HighbdGetMeanValue8x8(above, left, bd, params); break; + case 16: + meanValue = HighbdGetMeanValue16x16(above, left, bd, params); + break; + case 32: + meanValue = HighbdGetMeanValue32x32(above, left, bd, params); + break; + default: assert(0); + } + return meanValue; +} + +// Note: +// At column index c, the remaining pixels are R = 2 * bs + 1 - r - c +// the number of pixels to produce is R - 2 = 2 * bs - r - c - 1 +static void HighbdGeneratePrediction(const uint16_t *above, + const uint16_t *left, const int bs, + const int bd, const __m128i *prm, + int meanValue, uint16_t *dst, + ptrdiff_t stride) { + int pred[33][65]; + int r, c, colBound; + int remainings; + int ipred; + + for (r = 0; r < bs; ++r) { + pred[r + 1][0] = (int)left[r] - meanValue; + } + + above -= 1; + for (c = 0; c < 2 * bs + 1; ++c) { + pred[0][c] = (int)above[c] - meanValue; + } + + r = 0; + c = 0; + while (r < bs) { + colBound = (bs << 1) - r; + for (c = 0; c < colBound; c += 4) { + remainings = colBound - c + 1; + ProducePixels(&pred[r][c], prm, remainings); + } + r += 1; + } + + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) { + ipred = pred[r + 1][c + 1] + meanValue; + dst[c] = clip_pixel_highbd(ipred, bd); + } + dst += stride; + } +} + +static void HighbdFilterPrediction(const uint16_t *above, const uint16_t *left, + int bs, const int bd, __m128i *prm, + uint16_t *dst, ptrdiff_t stride) { + int meanValue = 0; + meanValue = HighbdCalcRefPixelsMeanValue(above, left, bs, bd, &prm[4]); + HighbdGeneratePrediction(above, left, bs, bd, prm, meanValue, dst, stride); +} + +void av1_highbd_dc_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + __m128i prm[5]; + GetIntraFilterParams(bs, DC_PRED, &prm[0]); + HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride); +} + +void av1_highbd_v_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + __m128i prm[5]; + GetIntraFilterParams(bs, V_PRED, &prm[0]); + HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride); +} + +void av1_highbd_h_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + __m128i prm[5]; + GetIntraFilterParams(bs, H_PRED, &prm[0]); + HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride); +} + +void av1_highbd_d45_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + __m128i prm[5]; + GetIntraFilterParams(bs, D45_PRED, &prm[0]); + HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride); +} + +void av1_highbd_d135_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + __m128i prm[5]; + GetIntraFilterParams(bs, D135_PRED, &prm[0]); + HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride); +} + +void av1_highbd_d117_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + __m128i prm[5]; + GetIntraFilterParams(bs, D117_PRED, &prm[0]); + HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride); +} + +void av1_highbd_d153_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + __m128i prm[5]; + GetIntraFilterParams(bs, D153_PRED, &prm[0]); + HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride); +} + +void av1_highbd_d207_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + __m128i prm[5]; + GetIntraFilterParams(bs, D207_PRED, &prm[0]); + HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride); +} + +void av1_highbd_d63_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + __m128i prm[5]; + GetIntraFilterParams(bs, D63_PRED, &prm[0]); + HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride); +} + +void av1_highbd_tm_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride, + int bs, const uint16_t *above, + const uint16_t *left, int bd) { + __m128i prm[5]; + GetIntraFilterParams(bs, TM_PRED, &prm[0]); + HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride); +} +#endif // CONFIG_HIGHBITDEPTH + +#endif // USE_3TAP_INTRA_FILTER diff --git a/third_party/aom/av1/common/x86/highbd_inv_txfm_avx2.c b/third_party/aom/av1/common/x86/highbd_inv_txfm_avx2.c new file mode 100644 index 0000000000..d10f1ccc21 --- /dev/null +++ b/third_party/aom/av1/common/x86/highbd_inv_txfm_avx2.c @@ -0,0 +1,557 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include +#include + +#include "./av1_rtcd.h" +#include "./aom_config.h" +#include "av1/common/av1_inv_txfm2d_cfg.h" + +// Note: +// Total 32x4 registers to represent 32x32 block coefficients. +// For high bit depth, each coefficient is 4-byte. +// Each __m256i register holds 8 coefficients. +// So each "row" we needs 4 register. Totally 32 rows +// Register layout: +// v0, v1, v2, v3, +// v4, v5, v6, v7, +// ... ... +// v124, v125, v126, v127 + +static void transpose_32x32_8x8(const __m256i *in, __m256i *out) { + __m256i u0, u1, u2, u3, u4, u5, u6, u7; + __m256i x0, x1; + + u0 = _mm256_unpacklo_epi32(in[0], in[4]); + u1 = _mm256_unpackhi_epi32(in[0], in[4]); + + u2 = _mm256_unpacklo_epi32(in[8], in[12]); + u3 = _mm256_unpackhi_epi32(in[8], in[12]); + + u4 = _mm256_unpacklo_epi32(in[16], in[20]); + u5 = _mm256_unpackhi_epi32(in[16], in[20]); + + u6 = _mm256_unpacklo_epi32(in[24], in[28]); + u7 = _mm256_unpackhi_epi32(in[24], in[28]); + + x0 = _mm256_unpacklo_epi64(u0, u2); + x1 = _mm256_unpacklo_epi64(u4, u6); + out[0] = _mm256_permute2f128_si256(x0, x1, 0x20); + out[16] = _mm256_permute2f128_si256(x0, x1, 0x31); + + x0 = _mm256_unpackhi_epi64(u0, u2); + x1 = _mm256_unpackhi_epi64(u4, u6); + out[4] = _mm256_permute2f128_si256(x0, x1, 0x20); + out[20] = _mm256_permute2f128_si256(x0, x1, 0x31); + + x0 = _mm256_unpacklo_epi64(u1, u3); + x1 = _mm256_unpacklo_epi64(u5, u7); + out[8] = _mm256_permute2f128_si256(x0, x1, 0x20); + out[24] = _mm256_permute2f128_si256(x0, x1, 0x31); + + x0 = _mm256_unpackhi_epi64(u1, u3); + x1 = _mm256_unpackhi_epi64(u5, u7); + out[12] = _mm256_permute2f128_si256(x0, x1, 0x20); + out[28] = _mm256_permute2f128_si256(x0, x1, 0x31); +} + +static void transpose_32x32_16x16(const __m256i *in, __m256i *out) { + transpose_32x32_8x8(&in[0], &out[0]); + transpose_32x32_8x8(&in[1], &out[32]); + transpose_32x32_8x8(&in[32], &out[1]); + transpose_32x32_8x8(&in[33], &out[33]); +} + +static void transpose_32x32(const __m256i *in, __m256i *out) { + transpose_32x32_16x16(&in[0], &out[0]); + transpose_32x32_16x16(&in[2], &out[64]); + transpose_32x32_16x16(&in[64], &out[2]); + transpose_32x32_16x16(&in[66], &out[66]); +} + +static void load_buffer_32x32(const int32_t *coeff, __m256i *in) { + int i; + for (i = 0; i < 128; ++i) { + in[i] = _mm256_loadu_si256((const __m256i *)coeff); + coeff += 8; + } +} + +static void round_shift_32x32(__m256i *in, int shift) { + __m256i rnding = _mm256_set1_epi32(1 << (shift - 1)); + int i = 0; + + while (i < 128) { + in[i] = _mm256_add_epi32(in[i], rnding); + in[i] = _mm256_srai_epi32(in[i], shift); + i++; + } +} + +static __m256i highbd_clamp_epi32(__m256i x, int bd) { + const __m256i zero = _mm256_setzero_si256(); + const __m256i one = _mm256_set1_epi16(1); + const __m256i max = _mm256_sub_epi16(_mm256_slli_epi16(one, bd), one); + __m256i clamped, mask; + + mask = _mm256_cmpgt_epi16(x, max); + clamped = _mm256_andnot_si256(mask, x); + mask = _mm256_and_si256(mask, max); + clamped = _mm256_or_si256(mask, clamped); + mask = _mm256_cmpgt_epi16(clamped, zero); + clamped = _mm256_and_si256(clamped, mask); + + return clamped; +} + +static void write_buffer_32x32(__m256i *in, uint16_t *output, int stride, + int fliplr, int flipud, int shift, int bd) { + __m256i u0, u1, x0, x1, x2, x3, v0, v1, v2, v3; + const __m256i zero = _mm256_setzero_si256(); + int i = 0; + (void)fliplr; + (void)flipud; + + round_shift_32x32(in, shift); + + while (i < 128) { + u0 = _mm256_loadu_si256((const __m256i *)output); + u1 = _mm256_loadu_si256((const __m256i *)(output + 16)); + + x0 = _mm256_unpacklo_epi16(u0, zero); + x1 = _mm256_unpackhi_epi16(u0, zero); + x2 = _mm256_unpacklo_epi16(u1, zero); + x3 = _mm256_unpackhi_epi16(u1, zero); + + v0 = _mm256_permute2f128_si256(in[i], in[i + 1], 0x20); + v1 = _mm256_permute2f128_si256(in[i], in[i + 1], 0x31); + v2 = _mm256_permute2f128_si256(in[i + 2], in[i + 3], 0x20); + v3 = _mm256_permute2f128_si256(in[i + 2], in[i + 3], 0x31); + + v0 = _mm256_add_epi32(v0, x0); + v1 = _mm256_add_epi32(v1, x1); + v2 = _mm256_add_epi32(v2, x2); + v3 = _mm256_add_epi32(v3, x3); + + v0 = _mm256_packus_epi32(v0, v1); + v2 = _mm256_packus_epi32(v2, v3); + + v0 = highbd_clamp_epi32(v0, bd); + v2 = highbd_clamp_epi32(v2, bd); + + _mm256_storeu_si256((__m256i *)output, v0); + _mm256_storeu_si256((__m256i *)(output + 16), v2); + output += stride; + i += 4; + } +} + +static INLINE __m256i half_btf_avx2(__m256i w0, __m256i n0, __m256i w1, + __m256i n1, __m256i rounding, int bit) { + __m256i x, y; + + x = _mm256_mullo_epi32(w0, n0); + y = _mm256_mullo_epi32(w1, n1); + x = _mm256_add_epi32(x, y); + x = _mm256_add_epi32(x, rounding); + x = _mm256_srai_epi32(x, bit); + return x; +} + +static void idct32_avx2(__m256i *in, __m256i *out, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m256i cospi62 = _mm256_set1_epi32(cospi[62]); + const __m256i cospi30 = _mm256_set1_epi32(cospi[30]); + const __m256i cospi46 = _mm256_set1_epi32(cospi[46]); + const __m256i cospi14 = _mm256_set1_epi32(cospi[14]); + const __m256i cospi54 = _mm256_set1_epi32(cospi[54]); + const __m256i cospi22 = _mm256_set1_epi32(cospi[22]); + const __m256i cospi38 = _mm256_set1_epi32(cospi[38]); + const __m256i cospi6 = _mm256_set1_epi32(cospi[6]); + const __m256i cospi58 = _mm256_set1_epi32(cospi[58]); + const __m256i cospi26 = _mm256_set1_epi32(cospi[26]); + const __m256i cospi42 = _mm256_set1_epi32(cospi[42]); + const __m256i cospi10 = _mm256_set1_epi32(cospi[10]); + const __m256i cospi50 = _mm256_set1_epi32(cospi[50]); + const __m256i cospi18 = _mm256_set1_epi32(cospi[18]); + const __m256i cospi34 = _mm256_set1_epi32(cospi[34]); + const __m256i cospi2 = _mm256_set1_epi32(cospi[2]); + const __m256i cospim58 = _mm256_set1_epi32(-cospi[58]); + const __m256i cospim26 = _mm256_set1_epi32(-cospi[26]); + const __m256i cospim42 = _mm256_set1_epi32(-cospi[42]); + const __m256i cospim10 = _mm256_set1_epi32(-cospi[10]); + const __m256i cospim50 = _mm256_set1_epi32(-cospi[50]); + const __m256i cospim18 = _mm256_set1_epi32(-cospi[18]); + const __m256i cospim34 = _mm256_set1_epi32(-cospi[34]); + const __m256i cospim2 = _mm256_set1_epi32(-cospi[2]); + const __m256i cospi60 = _mm256_set1_epi32(cospi[60]); + const __m256i cospi28 = _mm256_set1_epi32(cospi[28]); + const __m256i cospi44 = _mm256_set1_epi32(cospi[44]); + const __m256i cospi12 = _mm256_set1_epi32(cospi[12]); + const __m256i cospi52 = _mm256_set1_epi32(cospi[52]); + const __m256i cospi20 = _mm256_set1_epi32(cospi[20]); + const __m256i cospi36 = _mm256_set1_epi32(cospi[36]); + const __m256i cospi4 = _mm256_set1_epi32(cospi[4]); + const __m256i cospim52 = _mm256_set1_epi32(-cospi[52]); + const __m256i cospim20 = _mm256_set1_epi32(-cospi[20]); + const __m256i cospim36 = _mm256_set1_epi32(-cospi[36]); + const __m256i cospim4 = _mm256_set1_epi32(-cospi[4]); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospi24 = _mm256_set1_epi32(cospi[24]); + const __m256i cospi40 = _mm256_set1_epi32(cospi[40]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospim40 = _mm256_set1_epi32(-cospi[40]); + const __m256i cospim8 = _mm256_set1_epi32(-cospi[8]); + const __m256i cospim56 = _mm256_set1_epi32(-cospi[56]); + const __m256i cospim24 = _mm256_set1_epi32(-cospi[24]); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i cospim32 = _mm256_set1_epi32(-cospi[32]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospim16 = _mm256_set1_epi32(-cospi[16]); + const __m256i rounding = _mm256_set1_epi32(1 << (bit - 1)); + __m256i bf1[32], bf0[32]; + int col; + + for (col = 0; col < 4; ++col) { + // stage 0 + // stage 1 + bf1[0] = in[0 * 4 + col]; + bf1[1] = in[16 * 4 + col]; + bf1[2] = in[8 * 4 + col]; + bf1[3] = in[24 * 4 + col]; + bf1[4] = in[4 * 4 + col]; + bf1[5] = in[20 * 4 + col]; + bf1[6] = in[12 * 4 + col]; + bf1[7] = in[28 * 4 + col]; + bf1[8] = in[2 * 4 + col]; + bf1[9] = in[18 * 4 + col]; + bf1[10] = in[10 * 4 + col]; + bf1[11] = in[26 * 4 + col]; + bf1[12] = in[6 * 4 + col]; + bf1[13] = in[22 * 4 + col]; + bf1[14] = in[14 * 4 + col]; + bf1[15] = in[30 * 4 + col]; + bf1[16] = in[1 * 4 + col]; + bf1[17] = in[17 * 4 + col]; + bf1[18] = in[9 * 4 + col]; + bf1[19] = in[25 * 4 + col]; + bf1[20] = in[5 * 4 + col]; + bf1[21] = in[21 * 4 + col]; + bf1[22] = in[13 * 4 + col]; + bf1[23] = in[29 * 4 + col]; + bf1[24] = in[3 * 4 + col]; + bf1[25] = in[19 * 4 + col]; + bf1[26] = in[11 * 4 + col]; + bf1[27] = in[27 * 4 + col]; + bf1[28] = in[7 * 4 + col]; + bf1[29] = in[23 * 4 + col]; + bf1[30] = in[15 * 4 + col]; + bf1[31] = in[31 * 4 + col]; + + // stage 2 + bf0[0] = bf1[0]; + bf0[1] = bf1[1]; + bf0[2] = bf1[2]; + bf0[3] = bf1[3]; + bf0[4] = bf1[4]; + bf0[5] = bf1[5]; + bf0[6] = bf1[6]; + bf0[7] = bf1[7]; + bf0[8] = bf1[8]; + bf0[9] = bf1[9]; + bf0[10] = bf1[10]; + bf0[11] = bf1[11]; + bf0[12] = bf1[12]; + bf0[13] = bf1[13]; + bf0[14] = bf1[14]; + bf0[15] = bf1[15]; + bf0[16] = half_btf_avx2(cospi62, bf1[16], cospim2, bf1[31], rounding, bit); + bf0[17] = half_btf_avx2(cospi30, bf1[17], cospim34, bf1[30], rounding, bit); + bf0[18] = half_btf_avx2(cospi46, bf1[18], cospim18, bf1[29], rounding, bit); + bf0[19] = half_btf_avx2(cospi14, bf1[19], cospim50, bf1[28], rounding, bit); + bf0[20] = half_btf_avx2(cospi54, bf1[20], cospim10, bf1[27], rounding, bit); + bf0[21] = half_btf_avx2(cospi22, bf1[21], cospim42, bf1[26], rounding, bit); + bf0[22] = half_btf_avx2(cospi38, bf1[22], cospim26, bf1[25], rounding, bit); + bf0[23] = half_btf_avx2(cospi6, bf1[23], cospim58, bf1[24], rounding, bit); + bf0[24] = half_btf_avx2(cospi58, bf1[23], cospi6, bf1[24], rounding, bit); + bf0[25] = half_btf_avx2(cospi26, bf1[22], cospi38, bf1[25], rounding, bit); + bf0[26] = half_btf_avx2(cospi42, bf1[21], cospi22, bf1[26], rounding, bit); + bf0[27] = half_btf_avx2(cospi10, bf1[20], cospi54, bf1[27], rounding, bit); + bf0[28] = half_btf_avx2(cospi50, bf1[19], cospi14, bf1[28], rounding, bit); + bf0[29] = half_btf_avx2(cospi18, bf1[18], cospi46, bf1[29], rounding, bit); + bf0[30] = half_btf_avx2(cospi34, bf1[17], cospi30, bf1[30], rounding, bit); + bf0[31] = half_btf_avx2(cospi2, bf1[16], cospi62, bf1[31], rounding, bit); + + // stage 3 + bf1[0] = bf0[0]; + bf1[1] = bf0[1]; + bf1[2] = bf0[2]; + bf1[3] = bf0[3]; + bf1[4] = bf0[4]; + bf1[5] = bf0[5]; + bf1[6] = bf0[6]; + bf1[7] = bf0[7]; + bf1[8] = half_btf_avx2(cospi60, bf0[8], cospim4, bf0[15], rounding, bit); + bf1[9] = half_btf_avx2(cospi28, bf0[9], cospim36, bf0[14], rounding, bit); + bf1[10] = half_btf_avx2(cospi44, bf0[10], cospim20, bf0[13], rounding, bit); + bf1[11] = half_btf_avx2(cospi12, bf0[11], cospim52, bf0[12], rounding, bit); + bf1[12] = half_btf_avx2(cospi52, bf0[11], cospi12, bf0[12], rounding, bit); + bf1[13] = half_btf_avx2(cospi20, bf0[10], cospi44, bf0[13], rounding, bit); + bf1[14] = half_btf_avx2(cospi36, bf0[9], cospi28, bf0[14], rounding, bit); + bf1[15] = half_btf_avx2(cospi4, bf0[8], cospi60, bf0[15], rounding, bit); + bf1[16] = _mm256_add_epi32(bf0[16], bf0[17]); + bf1[17] = _mm256_sub_epi32(bf0[16], bf0[17]); + bf1[18] = _mm256_sub_epi32(bf0[19], bf0[18]); + bf1[19] = _mm256_add_epi32(bf0[18], bf0[19]); + bf1[20] = _mm256_add_epi32(bf0[20], bf0[21]); + bf1[21] = _mm256_sub_epi32(bf0[20], bf0[21]); + bf1[22] = _mm256_sub_epi32(bf0[23], bf0[22]); + bf1[23] = _mm256_add_epi32(bf0[22], bf0[23]); + bf1[24] = _mm256_add_epi32(bf0[24], bf0[25]); + bf1[25] = _mm256_sub_epi32(bf0[24], bf0[25]); + bf1[26] = _mm256_sub_epi32(bf0[27], bf0[26]); + bf1[27] = _mm256_add_epi32(bf0[26], bf0[27]); + bf1[28] = _mm256_add_epi32(bf0[28], bf0[29]); + bf1[29] = _mm256_sub_epi32(bf0[28], bf0[29]); + bf1[30] = _mm256_sub_epi32(bf0[31], bf0[30]); + bf1[31] = _mm256_add_epi32(bf0[30], bf0[31]); + + // stage 4 + bf0[0] = bf1[0]; + bf0[1] = bf1[1]; + bf0[2] = bf1[2]; + bf0[3] = bf1[3]; + bf0[4] = half_btf_avx2(cospi56, bf1[4], cospim8, bf1[7], rounding, bit); + bf0[5] = half_btf_avx2(cospi24, bf1[5], cospim40, bf1[6], rounding, bit); + bf0[6] = half_btf_avx2(cospi40, bf1[5], cospi24, bf1[6], rounding, bit); + bf0[7] = half_btf_avx2(cospi8, bf1[4], cospi56, bf1[7], rounding, bit); + bf0[8] = _mm256_add_epi32(bf1[8], bf1[9]); + bf0[9] = _mm256_sub_epi32(bf1[8], bf1[9]); + bf0[10] = _mm256_sub_epi32(bf1[11], bf1[10]); + bf0[11] = _mm256_add_epi32(bf1[10], bf1[11]); + bf0[12] = _mm256_add_epi32(bf1[12], bf1[13]); + bf0[13] = _mm256_sub_epi32(bf1[12], bf1[13]); + bf0[14] = _mm256_sub_epi32(bf1[15], bf1[14]); + bf0[15] = _mm256_add_epi32(bf1[14], bf1[15]); + bf0[16] = bf1[16]; + bf0[17] = half_btf_avx2(cospim8, bf1[17], cospi56, bf1[30], rounding, bit); + bf0[18] = half_btf_avx2(cospim56, bf1[18], cospim8, bf1[29], rounding, bit); + bf0[19] = bf1[19]; + bf0[20] = bf1[20]; + bf0[21] = half_btf_avx2(cospim40, bf1[21], cospi24, bf1[26], rounding, bit); + bf0[22] = + half_btf_avx2(cospim24, bf1[22], cospim40, bf1[25], rounding, bit); + bf0[23] = bf1[23]; + bf0[24] = bf1[24]; + bf0[25] = half_btf_avx2(cospim40, bf1[22], cospi24, bf1[25], rounding, bit); + bf0[26] = half_btf_avx2(cospi24, bf1[21], cospi40, bf1[26], rounding, bit); + bf0[27] = bf1[27]; + bf0[28] = bf1[28]; + bf0[29] = half_btf_avx2(cospim8, bf1[18], cospi56, bf1[29], rounding, bit); + bf0[30] = half_btf_avx2(cospi56, bf1[17], cospi8, bf1[30], rounding, bit); + bf0[31] = bf1[31]; + + // stage 5 + bf1[0] = half_btf_avx2(cospi32, bf0[0], cospi32, bf0[1], rounding, bit); + bf1[1] = half_btf_avx2(cospi32, bf0[0], cospim32, bf0[1], rounding, bit); + bf1[2] = half_btf_avx2(cospi48, bf0[2], cospim16, bf0[3], rounding, bit); + bf1[3] = half_btf_avx2(cospi16, bf0[2], cospi48, bf0[3], rounding, bit); + bf1[4] = _mm256_add_epi32(bf0[4], bf0[5]); + bf1[5] = _mm256_sub_epi32(bf0[4], bf0[5]); + bf1[6] = _mm256_sub_epi32(bf0[7], bf0[6]); + bf1[7] = _mm256_add_epi32(bf0[6], bf0[7]); + bf1[8] = bf0[8]; + bf1[9] = half_btf_avx2(cospim16, bf0[9], cospi48, bf0[14], rounding, bit); + bf1[10] = + half_btf_avx2(cospim48, bf0[10], cospim16, bf0[13], rounding, bit); + bf1[11] = bf0[11]; + bf1[12] = bf0[12]; + bf1[13] = half_btf_avx2(cospim16, bf0[10], cospi48, bf0[13], rounding, bit); + bf1[14] = half_btf_avx2(cospi48, bf0[9], cospi16, bf0[14], rounding, bit); + bf1[15] = bf0[15]; + bf1[16] = _mm256_add_epi32(bf0[16], bf0[19]); + bf1[17] = _mm256_add_epi32(bf0[17], bf0[18]); + bf1[18] = _mm256_sub_epi32(bf0[17], bf0[18]); + bf1[19] = _mm256_sub_epi32(bf0[16], bf0[19]); + bf1[20] = _mm256_sub_epi32(bf0[23], bf0[20]); + bf1[21] = _mm256_sub_epi32(bf0[22], bf0[21]); + bf1[22] = _mm256_add_epi32(bf0[21], bf0[22]); + bf1[23] = _mm256_add_epi32(bf0[20], bf0[23]); + bf1[24] = _mm256_add_epi32(bf0[24], bf0[27]); + bf1[25] = _mm256_add_epi32(bf0[25], bf0[26]); + bf1[26] = _mm256_sub_epi32(bf0[25], bf0[26]); + bf1[27] = _mm256_sub_epi32(bf0[24], bf0[27]); + bf1[28] = _mm256_sub_epi32(bf0[31], bf0[28]); + bf1[29] = _mm256_sub_epi32(bf0[30], bf0[29]); + bf1[30] = _mm256_add_epi32(bf0[29], bf0[30]); + bf1[31] = _mm256_add_epi32(bf0[28], bf0[31]); + + // stage 6 + bf0[0] = _mm256_add_epi32(bf1[0], bf1[3]); + bf0[1] = _mm256_add_epi32(bf1[1], bf1[2]); + bf0[2] = _mm256_sub_epi32(bf1[1], bf1[2]); + bf0[3] = _mm256_sub_epi32(bf1[0], bf1[3]); + bf0[4] = bf1[4]; + bf0[5] = half_btf_avx2(cospim32, bf1[5], cospi32, bf1[6], rounding, bit); + bf0[6] = half_btf_avx2(cospi32, bf1[5], cospi32, bf1[6], rounding, bit); + bf0[7] = bf1[7]; + bf0[8] = _mm256_add_epi32(bf1[8], bf1[11]); + bf0[9] = _mm256_add_epi32(bf1[9], bf1[10]); + bf0[10] = _mm256_sub_epi32(bf1[9], bf1[10]); + bf0[11] = _mm256_sub_epi32(bf1[8], bf1[11]); + bf0[12] = _mm256_sub_epi32(bf1[15], bf1[12]); + bf0[13] = _mm256_sub_epi32(bf1[14], bf1[13]); + bf0[14] = _mm256_add_epi32(bf1[13], bf1[14]); + bf0[15] = _mm256_add_epi32(bf1[12], bf1[15]); + bf0[16] = bf1[16]; + bf0[17] = bf1[17]; + bf0[18] = half_btf_avx2(cospim16, bf1[18], cospi48, bf1[29], rounding, bit); + bf0[19] = half_btf_avx2(cospim16, bf1[19], cospi48, bf1[28], rounding, bit); + bf0[20] = + half_btf_avx2(cospim48, bf1[20], cospim16, bf1[27], rounding, bit); + bf0[21] = + half_btf_avx2(cospim48, bf1[21], cospim16, bf1[26], rounding, bit); + bf0[22] = bf1[22]; + bf0[23] = bf1[23]; + bf0[24] = bf1[24]; + bf0[25] = bf1[25]; + bf0[26] = half_btf_avx2(cospim16, bf1[21], cospi48, bf1[26], rounding, bit); + bf0[27] = half_btf_avx2(cospim16, bf1[20], cospi48, bf1[27], rounding, bit); + bf0[28] = half_btf_avx2(cospi48, bf1[19], cospi16, bf1[28], rounding, bit); + bf0[29] = half_btf_avx2(cospi48, bf1[18], cospi16, bf1[29], rounding, bit); + bf0[30] = bf1[30]; + bf0[31] = bf1[31]; + + // stage 7 + bf1[0] = _mm256_add_epi32(bf0[0], bf0[7]); + bf1[1] = _mm256_add_epi32(bf0[1], bf0[6]); + bf1[2] = _mm256_add_epi32(bf0[2], bf0[5]); + bf1[3] = _mm256_add_epi32(bf0[3], bf0[4]); + bf1[4] = _mm256_sub_epi32(bf0[3], bf0[4]); + bf1[5] = _mm256_sub_epi32(bf0[2], bf0[5]); + bf1[6] = _mm256_sub_epi32(bf0[1], bf0[6]); + bf1[7] = _mm256_sub_epi32(bf0[0], bf0[7]); + bf1[8] = bf0[8]; + bf1[9] = bf0[9]; + bf1[10] = half_btf_avx2(cospim32, bf0[10], cospi32, bf0[13], rounding, bit); + bf1[11] = half_btf_avx2(cospim32, bf0[11], cospi32, bf0[12], rounding, bit); + bf1[12] = half_btf_avx2(cospi32, bf0[11], cospi32, bf0[12], rounding, bit); + bf1[13] = half_btf_avx2(cospi32, bf0[10], cospi32, bf0[13], rounding, bit); + bf1[14] = bf0[14]; + bf1[15] = bf0[15]; + bf1[16] = _mm256_add_epi32(bf0[16], bf0[23]); + bf1[17] = _mm256_add_epi32(bf0[17], bf0[22]); + bf1[18] = _mm256_add_epi32(bf0[18], bf0[21]); + bf1[19] = _mm256_add_epi32(bf0[19], bf0[20]); + bf1[20] = _mm256_sub_epi32(bf0[19], bf0[20]); + bf1[21] = _mm256_sub_epi32(bf0[18], bf0[21]); + bf1[22] = _mm256_sub_epi32(bf0[17], bf0[22]); + bf1[23] = _mm256_sub_epi32(bf0[16], bf0[23]); + bf1[24] = _mm256_sub_epi32(bf0[31], bf0[24]); + bf1[25] = _mm256_sub_epi32(bf0[30], bf0[25]); + bf1[26] = _mm256_sub_epi32(bf0[29], bf0[26]); + bf1[27] = _mm256_sub_epi32(bf0[28], bf0[27]); + bf1[28] = _mm256_add_epi32(bf0[27], bf0[28]); + bf1[29] = _mm256_add_epi32(bf0[26], bf0[29]); + bf1[30] = _mm256_add_epi32(bf0[25], bf0[30]); + bf1[31] = _mm256_add_epi32(bf0[24], bf0[31]); + + // stage 8 + bf0[0] = _mm256_add_epi32(bf1[0], bf1[15]); + bf0[1] = _mm256_add_epi32(bf1[1], bf1[14]); + bf0[2] = _mm256_add_epi32(bf1[2], bf1[13]); + bf0[3] = _mm256_add_epi32(bf1[3], bf1[12]); + bf0[4] = _mm256_add_epi32(bf1[4], bf1[11]); + bf0[5] = _mm256_add_epi32(bf1[5], bf1[10]); + bf0[6] = _mm256_add_epi32(bf1[6], bf1[9]); + bf0[7] = _mm256_add_epi32(bf1[7], bf1[8]); + bf0[8] = _mm256_sub_epi32(bf1[7], bf1[8]); + bf0[9] = _mm256_sub_epi32(bf1[6], bf1[9]); + bf0[10] = _mm256_sub_epi32(bf1[5], bf1[10]); + bf0[11] = _mm256_sub_epi32(bf1[4], bf1[11]); + bf0[12] = _mm256_sub_epi32(bf1[3], bf1[12]); + bf0[13] = _mm256_sub_epi32(bf1[2], bf1[13]); + bf0[14] = _mm256_sub_epi32(bf1[1], bf1[14]); + bf0[15] = _mm256_sub_epi32(bf1[0], bf1[15]); + bf0[16] = bf1[16]; + bf0[17] = bf1[17]; + bf0[18] = bf1[18]; + bf0[19] = bf1[19]; + bf0[20] = half_btf_avx2(cospim32, bf1[20], cospi32, bf1[27], rounding, bit); + bf0[21] = half_btf_avx2(cospim32, bf1[21], cospi32, bf1[26], rounding, bit); + bf0[22] = half_btf_avx2(cospim32, bf1[22], cospi32, bf1[25], rounding, bit); + bf0[23] = half_btf_avx2(cospim32, bf1[23], cospi32, bf1[24], rounding, bit); + bf0[24] = half_btf_avx2(cospi32, bf1[23], cospi32, bf1[24], rounding, bit); + bf0[25] = half_btf_avx2(cospi32, bf1[22], cospi32, bf1[25], rounding, bit); + bf0[26] = half_btf_avx2(cospi32, bf1[21], cospi32, bf1[26], rounding, bit); + bf0[27] = half_btf_avx2(cospi32, bf1[20], cospi32, bf1[27], rounding, bit); + bf0[28] = bf1[28]; + bf0[29] = bf1[29]; + bf0[30] = bf1[30]; + bf0[31] = bf1[31]; + + // stage 9 + out[0 * 4 + col] = _mm256_add_epi32(bf0[0], bf0[31]); + out[1 * 4 + col] = _mm256_add_epi32(bf0[1], bf0[30]); + out[2 * 4 + col] = _mm256_add_epi32(bf0[2], bf0[29]); + out[3 * 4 + col] = _mm256_add_epi32(bf0[3], bf0[28]); + out[4 * 4 + col] = _mm256_add_epi32(bf0[4], bf0[27]); + out[5 * 4 + col] = _mm256_add_epi32(bf0[5], bf0[26]); + out[6 * 4 + col] = _mm256_add_epi32(bf0[6], bf0[25]); + out[7 * 4 + col] = _mm256_add_epi32(bf0[7], bf0[24]); + out[8 * 4 + col] = _mm256_add_epi32(bf0[8], bf0[23]); + out[9 * 4 + col] = _mm256_add_epi32(bf0[9], bf0[22]); + out[10 * 4 + col] = _mm256_add_epi32(bf0[10], bf0[21]); + out[11 * 4 + col] = _mm256_add_epi32(bf0[11], bf0[20]); + out[12 * 4 + col] = _mm256_add_epi32(bf0[12], bf0[19]); + out[13 * 4 + col] = _mm256_add_epi32(bf0[13], bf0[18]); + out[14 * 4 + col] = _mm256_add_epi32(bf0[14], bf0[17]); + out[15 * 4 + col] = _mm256_add_epi32(bf0[15], bf0[16]); + out[16 * 4 + col] = _mm256_sub_epi32(bf0[15], bf0[16]); + out[17 * 4 + col] = _mm256_sub_epi32(bf0[14], bf0[17]); + out[18 * 4 + col] = _mm256_sub_epi32(bf0[13], bf0[18]); + out[19 * 4 + col] = _mm256_sub_epi32(bf0[12], bf0[19]); + out[20 * 4 + col] = _mm256_sub_epi32(bf0[11], bf0[20]); + out[21 * 4 + col] = _mm256_sub_epi32(bf0[10], bf0[21]); + out[22 * 4 + col] = _mm256_sub_epi32(bf0[9], bf0[22]); + out[23 * 4 + col] = _mm256_sub_epi32(bf0[8], bf0[23]); + out[24 * 4 + col] = _mm256_sub_epi32(bf0[7], bf0[24]); + out[25 * 4 + col] = _mm256_sub_epi32(bf0[6], bf0[25]); + out[26 * 4 + col] = _mm256_sub_epi32(bf0[5], bf0[26]); + out[27 * 4 + col] = _mm256_sub_epi32(bf0[4], bf0[27]); + out[28 * 4 + col] = _mm256_sub_epi32(bf0[3], bf0[28]); + out[29 * 4 + col] = _mm256_sub_epi32(bf0[2], bf0[29]); + out[30 * 4 + col] = _mm256_sub_epi32(bf0[1], bf0[30]); + out[31 * 4 + col] = _mm256_sub_epi32(bf0[0], bf0[31]); + } +} + +void av1_inv_txfm2d_add_32x32_avx2(const int32_t *coeff, uint16_t *output, + int stride, int tx_type, int bd) { + __m256i in[128], out[128]; + const TXFM_2D_CFG *cfg = NULL; + + switch (tx_type) { + case DCT_DCT: + cfg = &inv_txfm_2d_cfg_dct_dct_32; + load_buffer_32x32(coeff, in); + transpose_32x32(in, out); + idct32_avx2(out, in, cfg->cos_bit_row[2]); + round_shift_32x32(in, -cfg->shift[0]); + transpose_32x32(in, out); + idct32_avx2(out, in, cfg->cos_bit_col[2]); + write_buffer_32x32(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; + default: assert(0); + } +} diff --git a/third_party/aom/av1/common/x86/highbd_inv_txfm_sse4.c b/third_party/aom/av1/common/x86/highbd_inv_txfm_sse4.c new file mode 100644 index 0000000000..24b2760b94 --- /dev/null +++ b/third_party/aom/av1/common/x86/highbd_inv_txfm_sse4.c @@ -0,0 +1,1398 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include +#include /* SSE4.1 */ + +#include "./av1_rtcd.h" +#include "./aom_config.h" +#include "av1/common/av1_inv_txfm2d_cfg.h" +#include "av1/common/x86/highbd_txfm_utility_sse4.h" + +static INLINE void load_buffer_4x4(const int32_t *coeff, __m128i *in) { + in[0] = _mm_load_si128((const __m128i *)(coeff + 0)); + in[1] = _mm_load_si128((const __m128i *)(coeff + 4)); + in[2] = _mm_load_si128((const __m128i *)(coeff + 8)); + in[3] = _mm_load_si128((const __m128i *)(coeff + 12)); +} + +static void idct4x4_sse4_1(__m128i *in, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m128i cospi32 = _mm_set1_epi32(cospi[32]); + const __m128i cospi48 = _mm_set1_epi32(cospi[48]); + const __m128i cospi16 = _mm_set1_epi32(cospi[16]); + const __m128i cospim16 = _mm_set1_epi32(-cospi[16]); + const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); + __m128i u0, u1, u2, u3; + __m128i v0, v1, v2, v3, x, y; + + v0 = _mm_unpacklo_epi32(in[0], in[1]); + v1 = _mm_unpackhi_epi32(in[0], in[1]); + v2 = _mm_unpacklo_epi32(in[2], in[3]); + v3 = _mm_unpackhi_epi32(in[2], in[3]); + + u0 = _mm_unpacklo_epi64(v0, v2); + u1 = _mm_unpackhi_epi64(v0, v2); + u2 = _mm_unpacklo_epi64(v1, v3); + u3 = _mm_unpackhi_epi64(v1, v3); + + x = _mm_mullo_epi32(u0, cospi32); + y = _mm_mullo_epi32(u2, cospi32); + v0 = _mm_add_epi32(x, y); + v0 = _mm_add_epi32(v0, rnding); + v0 = _mm_srai_epi32(v0, bit); + + v1 = _mm_sub_epi32(x, y); + v1 = _mm_add_epi32(v1, rnding); + v1 = _mm_srai_epi32(v1, bit); + + x = _mm_mullo_epi32(u1, cospi48); + y = _mm_mullo_epi32(u3, cospim16); + v2 = _mm_add_epi32(x, y); + v2 = _mm_add_epi32(v2, rnding); + v2 = _mm_srai_epi32(v2, bit); + + x = _mm_mullo_epi32(u1, cospi16); + y = _mm_mullo_epi32(u3, cospi48); + v3 = _mm_add_epi32(x, y); + v3 = _mm_add_epi32(v3, rnding); + v3 = _mm_srai_epi32(v3, bit); + + in[0] = _mm_add_epi32(v0, v3); + in[1] = _mm_add_epi32(v1, v2); + in[2] = _mm_sub_epi32(v1, v2); + in[3] = _mm_sub_epi32(v0, v3); +} + +static void iadst4x4_sse4_1(__m128i *in, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m128i cospi32 = _mm_set1_epi32(cospi[32]); + const __m128i cospi8 = _mm_set1_epi32(cospi[8]); + const __m128i cospim8 = _mm_set1_epi32(-cospi[8]); + const __m128i cospi40 = _mm_set1_epi32(cospi[40]); + const __m128i cospim40 = _mm_set1_epi32(-cospi[40]); + const __m128i cospi24 = _mm_set1_epi32(cospi[24]); + const __m128i cospi56 = _mm_set1_epi32(cospi[56]); + const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); + const __m128i zero = _mm_setzero_si128(); + __m128i u0, u1, u2, u3; + __m128i v0, v1, v2, v3, x, y; + + v0 = _mm_unpacklo_epi32(in[0], in[1]); + v1 = _mm_unpackhi_epi32(in[0], in[1]); + v2 = _mm_unpacklo_epi32(in[2], in[3]); + v3 = _mm_unpackhi_epi32(in[2], in[3]); + + u0 = _mm_unpacklo_epi64(v0, v2); + u1 = _mm_unpackhi_epi64(v0, v2); + u2 = _mm_unpacklo_epi64(v1, v3); + u3 = _mm_unpackhi_epi64(v1, v3); + + // stage 0 + // stage 1 + u1 = _mm_sub_epi32(zero, u1); + u3 = _mm_sub_epi32(zero, u3); + + // stage 2 + v0 = u0; + v1 = u3; + x = _mm_mullo_epi32(u1, cospi32); + y = _mm_mullo_epi32(u2, cospi32); + v2 = _mm_add_epi32(x, y); + v2 = _mm_add_epi32(v2, rnding); + v2 = _mm_srai_epi32(v2, bit); + + v3 = _mm_sub_epi32(x, y); + v3 = _mm_add_epi32(v3, rnding); + v3 = _mm_srai_epi32(v3, bit); + + // stage 3 + u0 = _mm_add_epi32(v0, v2); + u1 = _mm_add_epi32(v1, v3); + u2 = _mm_sub_epi32(v0, v2); + u3 = _mm_sub_epi32(v1, v3); + + // stage 4 + x = _mm_mullo_epi32(u0, cospi8); + y = _mm_mullo_epi32(u1, cospi56); + in[3] = _mm_add_epi32(x, y); + in[3] = _mm_add_epi32(in[3], rnding); + in[3] = _mm_srai_epi32(in[3], bit); + + x = _mm_mullo_epi32(u0, cospi56); + y = _mm_mullo_epi32(u1, cospim8); + in[0] = _mm_add_epi32(x, y); + in[0] = _mm_add_epi32(in[0], rnding); + in[0] = _mm_srai_epi32(in[0], bit); + + x = _mm_mullo_epi32(u2, cospi40); + y = _mm_mullo_epi32(u3, cospi24); + in[1] = _mm_add_epi32(x, y); + in[1] = _mm_add_epi32(in[1], rnding); + in[1] = _mm_srai_epi32(in[1], bit); + + x = _mm_mullo_epi32(u2, cospi24); + y = _mm_mullo_epi32(u3, cospim40); + in[2] = _mm_add_epi32(x, y); + in[2] = _mm_add_epi32(in[2], rnding); + in[2] = _mm_srai_epi32(in[2], bit); +} + +static INLINE void round_shift_4x4(__m128i *in, int shift) { + __m128i rnding = _mm_set1_epi32(1 << (shift - 1)); + + in[0] = _mm_add_epi32(in[0], rnding); + in[1] = _mm_add_epi32(in[1], rnding); + in[2] = _mm_add_epi32(in[2], rnding); + in[3] = _mm_add_epi32(in[3], rnding); + + in[0] = _mm_srai_epi32(in[0], shift); + in[1] = _mm_srai_epi32(in[1], shift); + in[2] = _mm_srai_epi32(in[2], shift); + in[3] = _mm_srai_epi32(in[3], shift); +} + +static INLINE __m128i highbd_clamp_epi16(__m128i u, int bd) { + const __m128i zero = _mm_setzero_si128(); + const __m128i one = _mm_set1_epi16(1); + const __m128i max = _mm_sub_epi16(_mm_slli_epi16(one, bd), one); + __m128i clamped, mask; + + mask = _mm_cmpgt_epi16(u, max); + clamped = _mm_andnot_si128(mask, u); + mask = _mm_and_si128(mask, max); + clamped = _mm_or_si128(mask, clamped); + mask = _mm_cmpgt_epi16(clamped, zero); + clamped = _mm_and_si128(clamped, mask); + + return clamped; +} + +static void write_buffer_4x4(__m128i *in, uint16_t *output, int stride, + int fliplr, int flipud, int shift, int bd) { + const __m128i zero = _mm_setzero_si128(); + __m128i u0, u1, u2, u3; + __m128i v0, v1, v2, v3; + + round_shift_4x4(in, shift); + + v0 = _mm_loadl_epi64((__m128i const *)(output + 0 * stride)); + v1 = _mm_loadl_epi64((__m128i const *)(output + 1 * stride)); + v2 = _mm_loadl_epi64((__m128i const *)(output + 2 * stride)); + v3 = _mm_loadl_epi64((__m128i const *)(output + 3 * stride)); + + v0 = _mm_unpacklo_epi16(v0, zero); + v1 = _mm_unpacklo_epi16(v1, zero); + v2 = _mm_unpacklo_epi16(v2, zero); + v3 = _mm_unpacklo_epi16(v3, zero); + + if (fliplr) { + in[0] = _mm_shuffle_epi32(in[0], 0x1B); + in[1] = _mm_shuffle_epi32(in[1], 0x1B); + in[2] = _mm_shuffle_epi32(in[2], 0x1B); + in[3] = _mm_shuffle_epi32(in[3], 0x1B); + } + + if (flipud) { + u0 = _mm_add_epi32(in[3], v0); + u1 = _mm_add_epi32(in[2], v1); + u2 = _mm_add_epi32(in[1], v2); + u3 = _mm_add_epi32(in[0], v3); + } else { + u0 = _mm_add_epi32(in[0], v0); + u1 = _mm_add_epi32(in[1], v1); + u2 = _mm_add_epi32(in[2], v2); + u3 = _mm_add_epi32(in[3], v3); + } + + v0 = _mm_packus_epi32(u0, u1); + v2 = _mm_packus_epi32(u2, u3); + + u0 = highbd_clamp_epi16(v0, bd); + u2 = highbd_clamp_epi16(v2, bd); + + v0 = _mm_unpacklo_epi64(u0, u0); + v1 = _mm_unpackhi_epi64(u0, u0); + v2 = _mm_unpacklo_epi64(u2, u2); + v3 = _mm_unpackhi_epi64(u2, u2); + + _mm_storel_epi64((__m128i *)(output + 0 * stride), v0); + _mm_storel_epi64((__m128i *)(output + 1 * stride), v1); + _mm_storel_epi64((__m128i *)(output + 2 * stride), v2); + _mm_storel_epi64((__m128i *)(output + 3 * stride), v3); +} + +void av1_inv_txfm2d_add_4x4_sse4_1(const int32_t *coeff, uint16_t *output, + int stride, int tx_type, int bd) { + __m128i in[4]; + const TXFM_2D_CFG *cfg = NULL; + + switch (tx_type) { + case DCT_DCT: + cfg = &inv_txfm_2d_cfg_dct_dct_4; + load_buffer_4x4(coeff, in); + idct4x4_sse4_1(in, cfg->cos_bit_row[2]); + idct4x4_sse4_1(in, cfg->cos_bit_col[2]); + write_buffer_4x4(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; + case ADST_DCT: + cfg = &inv_txfm_2d_cfg_adst_dct_4; + load_buffer_4x4(coeff, in); + idct4x4_sse4_1(in, cfg->cos_bit_row[2]); + iadst4x4_sse4_1(in, cfg->cos_bit_col[2]); + write_buffer_4x4(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; + case DCT_ADST: + cfg = &inv_txfm_2d_cfg_dct_adst_4; + load_buffer_4x4(coeff, in); + iadst4x4_sse4_1(in, cfg->cos_bit_row[2]); + idct4x4_sse4_1(in, cfg->cos_bit_col[2]); + write_buffer_4x4(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; + case ADST_ADST: + cfg = &inv_txfm_2d_cfg_adst_adst_4; + load_buffer_4x4(coeff, in); + iadst4x4_sse4_1(in, cfg->cos_bit_row[2]); + iadst4x4_sse4_1(in, cfg->cos_bit_col[2]); + write_buffer_4x4(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + cfg = &inv_txfm_2d_cfg_adst_dct_4; + load_buffer_4x4(coeff, in); + idct4x4_sse4_1(in, cfg->cos_bit_row[2]); + iadst4x4_sse4_1(in, cfg->cos_bit_col[2]); + write_buffer_4x4(in, output, stride, 0, 1, -cfg->shift[1], bd); + break; + case DCT_FLIPADST: + cfg = &inv_txfm_2d_cfg_dct_adst_4; + load_buffer_4x4(coeff, in); + iadst4x4_sse4_1(in, cfg->cos_bit_row[2]); + idct4x4_sse4_1(in, cfg->cos_bit_col[2]); + write_buffer_4x4(in, output, stride, 1, 0, -cfg->shift[1], bd); + break; + case FLIPADST_FLIPADST: + cfg = &inv_txfm_2d_cfg_adst_adst_4; + load_buffer_4x4(coeff, in); + iadst4x4_sse4_1(in, cfg->cos_bit_row[2]); + iadst4x4_sse4_1(in, cfg->cos_bit_col[2]); + write_buffer_4x4(in, output, stride, 1, 1, -cfg->shift[1], bd); + break; + case ADST_FLIPADST: + cfg = &inv_txfm_2d_cfg_adst_adst_4; + load_buffer_4x4(coeff, in); + iadst4x4_sse4_1(in, cfg->cos_bit_row[2]); + iadst4x4_sse4_1(in, cfg->cos_bit_col[2]); + write_buffer_4x4(in, output, stride, 1, 0, -cfg->shift[1], bd); + break; + case FLIPADST_ADST: + cfg = &inv_txfm_2d_cfg_adst_adst_4; + load_buffer_4x4(coeff, in); + iadst4x4_sse4_1(in, cfg->cos_bit_row[2]); + iadst4x4_sse4_1(in, cfg->cos_bit_col[2]); + write_buffer_4x4(in, output, stride, 0, 1, -cfg->shift[1], bd); + break; +#endif // CONFIG_EXT_TX + default: assert(0); + } +} + +// 8x8 +static void load_buffer_8x8(const int32_t *coeff, __m128i *in) { + in[0] = _mm_load_si128((const __m128i *)(coeff + 0)); + in[1] = _mm_load_si128((const __m128i *)(coeff + 4)); + in[2] = _mm_load_si128((const __m128i *)(coeff + 8)); + in[3] = _mm_load_si128((const __m128i *)(coeff + 12)); + in[4] = _mm_load_si128((const __m128i *)(coeff + 16)); + in[5] = _mm_load_si128((const __m128i *)(coeff + 20)); + in[6] = _mm_load_si128((const __m128i *)(coeff + 24)); + in[7] = _mm_load_si128((const __m128i *)(coeff + 28)); + in[8] = _mm_load_si128((const __m128i *)(coeff + 32)); + in[9] = _mm_load_si128((const __m128i *)(coeff + 36)); + in[10] = _mm_load_si128((const __m128i *)(coeff + 40)); + in[11] = _mm_load_si128((const __m128i *)(coeff + 44)); + in[12] = _mm_load_si128((const __m128i *)(coeff + 48)); + in[13] = _mm_load_si128((const __m128i *)(coeff + 52)); + in[14] = _mm_load_si128((const __m128i *)(coeff + 56)); + in[15] = _mm_load_si128((const __m128i *)(coeff + 60)); +} + +static void idct8x8_sse4_1(__m128i *in, __m128i *out, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m128i cospi56 = _mm_set1_epi32(cospi[56]); + const __m128i cospim8 = _mm_set1_epi32(-cospi[8]); + const __m128i cospi24 = _mm_set1_epi32(cospi[24]); + const __m128i cospim40 = _mm_set1_epi32(-cospi[40]); + const __m128i cospi40 = _mm_set1_epi32(cospi[40]); + const __m128i cospi8 = _mm_set1_epi32(cospi[8]); + const __m128i cospi32 = _mm_set1_epi32(cospi[32]); + const __m128i cospi48 = _mm_set1_epi32(cospi[48]); + const __m128i cospim16 = _mm_set1_epi32(-cospi[16]); + const __m128i cospi16 = _mm_set1_epi32(cospi[16]); + const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); + __m128i u0, u1, u2, u3, u4, u5, u6, u7; + __m128i v0, v1, v2, v3, v4, v5, v6, v7; + __m128i x, y; + int col; + + // Note: + // Even column: 0, 2, ..., 14 + // Odd column: 1, 3, ..., 15 + // one even column plus one odd column constructs one row (8 coeffs) + // total we have 8 rows (8x8). + for (col = 0; col < 2; ++col) { + // stage 0 + // stage 1 + // stage 2 + u0 = in[0 * 2 + col]; + u1 = in[4 * 2 + col]; + u2 = in[2 * 2 + col]; + u3 = in[6 * 2 + col]; + + x = _mm_mullo_epi32(in[1 * 2 + col], cospi56); + y = _mm_mullo_epi32(in[7 * 2 + col], cospim8); + u4 = _mm_add_epi32(x, y); + u4 = _mm_add_epi32(u4, rnding); + u4 = _mm_srai_epi32(u4, bit); + + x = _mm_mullo_epi32(in[1 * 2 + col], cospi8); + y = _mm_mullo_epi32(in[7 * 2 + col], cospi56); + u7 = _mm_add_epi32(x, y); + u7 = _mm_add_epi32(u7, rnding); + u7 = _mm_srai_epi32(u7, bit); + + x = _mm_mullo_epi32(in[5 * 2 + col], cospi24); + y = _mm_mullo_epi32(in[3 * 2 + col], cospim40); + u5 = _mm_add_epi32(x, y); + u5 = _mm_add_epi32(u5, rnding); + u5 = _mm_srai_epi32(u5, bit); + + x = _mm_mullo_epi32(in[5 * 2 + col], cospi40); + y = _mm_mullo_epi32(in[3 * 2 + col], cospi24); + u6 = _mm_add_epi32(x, y); + u6 = _mm_add_epi32(u6, rnding); + u6 = _mm_srai_epi32(u6, bit); + + // stage 3 + x = _mm_mullo_epi32(u0, cospi32); + y = _mm_mullo_epi32(u1, cospi32); + v0 = _mm_add_epi32(x, y); + v0 = _mm_add_epi32(v0, rnding); + v0 = _mm_srai_epi32(v0, bit); + + v1 = _mm_sub_epi32(x, y); + v1 = _mm_add_epi32(v1, rnding); + v1 = _mm_srai_epi32(v1, bit); + + x = _mm_mullo_epi32(u2, cospi48); + y = _mm_mullo_epi32(u3, cospim16); + v2 = _mm_add_epi32(x, y); + v2 = _mm_add_epi32(v2, rnding); + v2 = _mm_srai_epi32(v2, bit); + + x = _mm_mullo_epi32(u2, cospi16); + y = _mm_mullo_epi32(u3, cospi48); + v3 = _mm_add_epi32(x, y); + v3 = _mm_add_epi32(v3, rnding); + v3 = _mm_srai_epi32(v3, bit); + + v4 = _mm_add_epi32(u4, u5); + v5 = _mm_sub_epi32(u4, u5); + v6 = _mm_sub_epi32(u7, u6); + v7 = _mm_add_epi32(u6, u7); + + // stage 4 + u0 = _mm_add_epi32(v0, v3); + u1 = _mm_add_epi32(v1, v2); + u2 = _mm_sub_epi32(v1, v2); + u3 = _mm_sub_epi32(v0, v3); + u4 = v4; + u7 = v7; + + x = _mm_mullo_epi32(v5, cospi32); + y = _mm_mullo_epi32(v6, cospi32); + u6 = _mm_add_epi32(y, x); + u6 = _mm_add_epi32(u6, rnding); + u6 = _mm_srai_epi32(u6, bit); + + u5 = _mm_sub_epi32(y, x); + u5 = _mm_add_epi32(u5, rnding); + u5 = _mm_srai_epi32(u5, bit); + + // stage 5 + out[0 * 2 + col] = _mm_add_epi32(u0, u7); + out[1 * 2 + col] = _mm_add_epi32(u1, u6); + out[2 * 2 + col] = _mm_add_epi32(u2, u5); + out[3 * 2 + col] = _mm_add_epi32(u3, u4); + out[4 * 2 + col] = _mm_sub_epi32(u3, u4); + out[5 * 2 + col] = _mm_sub_epi32(u2, u5); + out[6 * 2 + col] = _mm_sub_epi32(u1, u6); + out[7 * 2 + col] = _mm_sub_epi32(u0, u7); + } +} + +static void iadst8x8_sse4_1(__m128i *in, __m128i *out, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m128i cospi32 = _mm_set1_epi32(cospi[32]); + const __m128i cospi16 = _mm_set1_epi32(cospi[16]); + const __m128i cospim16 = _mm_set1_epi32(-cospi[16]); + const __m128i cospi48 = _mm_set1_epi32(cospi[48]); + const __m128i cospim48 = _mm_set1_epi32(-cospi[48]); + const __m128i cospi4 = _mm_set1_epi32(cospi[4]); + const __m128i cospim4 = _mm_set1_epi32(-cospi[4]); + const __m128i cospi60 = _mm_set1_epi32(cospi[60]); + const __m128i cospi20 = _mm_set1_epi32(cospi[20]); + const __m128i cospim20 = _mm_set1_epi32(-cospi[20]); + const __m128i cospi44 = _mm_set1_epi32(cospi[44]); + const __m128i cospi28 = _mm_set1_epi32(cospi[28]); + const __m128i cospi36 = _mm_set1_epi32(cospi[36]); + const __m128i cospim36 = _mm_set1_epi32(-cospi[36]); + const __m128i cospi52 = _mm_set1_epi32(cospi[52]); + const __m128i cospim52 = _mm_set1_epi32(-cospi[52]); + const __m128i cospi12 = _mm_set1_epi32(cospi[12]); + const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); + const __m128i zero = _mm_setzero_si128(); + __m128i u0, u1, u2, u3, u4, u5, u6, u7; + __m128i v0, v1, v2, v3, v4, v5, v6, v7; + __m128i x, y; + int col; + + // Note: + // Even column: 0, 2, ..., 14 + // Odd column: 1, 3, ..., 15 + // one even column plus one odd column constructs one row (8 coeffs) + // total we have 8 rows (8x8). + for (col = 0; col < 2; ++col) { + // stage 0 + // stage 1 + u0 = in[2 * 0 + col]; + u1 = _mm_sub_epi32(zero, in[2 * 7 + col]); + u2 = _mm_sub_epi32(zero, in[2 * 3 + col]); + u3 = in[2 * 4 + col]; + u4 = _mm_sub_epi32(zero, in[2 * 1 + col]); + u5 = in[2 * 6 + col]; + u6 = in[2 * 2 + col]; + u7 = _mm_sub_epi32(zero, in[2 * 5 + col]); + + // stage 2 + v0 = u0; + v1 = u1; + + x = _mm_mullo_epi32(u2, cospi32); + y = _mm_mullo_epi32(u3, cospi32); + v2 = _mm_add_epi32(x, y); + v2 = _mm_add_epi32(v2, rnding); + v2 = _mm_srai_epi32(v2, bit); + + v3 = _mm_sub_epi32(x, y); + v3 = _mm_add_epi32(v3, rnding); + v3 = _mm_srai_epi32(v3, bit); + + v4 = u4; + v5 = u5; + + x = _mm_mullo_epi32(u6, cospi32); + y = _mm_mullo_epi32(u7, cospi32); + v6 = _mm_add_epi32(x, y); + v6 = _mm_add_epi32(v6, rnding); + v6 = _mm_srai_epi32(v6, bit); + + v7 = _mm_sub_epi32(x, y); + v7 = _mm_add_epi32(v7, rnding); + v7 = _mm_srai_epi32(v7, bit); + + // stage 3 + u0 = _mm_add_epi32(v0, v2); + u1 = _mm_add_epi32(v1, v3); + u2 = _mm_sub_epi32(v0, v2); + u3 = _mm_sub_epi32(v1, v3); + u4 = _mm_add_epi32(v4, v6); + u5 = _mm_add_epi32(v5, v7); + u6 = _mm_sub_epi32(v4, v6); + u7 = _mm_sub_epi32(v5, v7); + + // stage 4 + v0 = u0; + v1 = u1; + v2 = u2; + v3 = u3; + + x = _mm_mullo_epi32(u4, cospi16); + y = _mm_mullo_epi32(u5, cospi48); + v4 = _mm_add_epi32(x, y); + v4 = _mm_add_epi32(v4, rnding); + v4 = _mm_srai_epi32(v4, bit); + + x = _mm_mullo_epi32(u4, cospi48); + y = _mm_mullo_epi32(u5, cospim16); + v5 = _mm_add_epi32(x, y); + v5 = _mm_add_epi32(v5, rnding); + v5 = _mm_srai_epi32(v5, bit); + + x = _mm_mullo_epi32(u6, cospim48); + y = _mm_mullo_epi32(u7, cospi16); + v6 = _mm_add_epi32(x, y); + v6 = _mm_add_epi32(v6, rnding); + v6 = _mm_srai_epi32(v6, bit); + + x = _mm_mullo_epi32(u6, cospi16); + y = _mm_mullo_epi32(u7, cospi48); + v7 = _mm_add_epi32(x, y); + v7 = _mm_add_epi32(v7, rnding); + v7 = _mm_srai_epi32(v7, bit); + + // stage 5 + u0 = _mm_add_epi32(v0, v4); + u1 = _mm_add_epi32(v1, v5); + u2 = _mm_add_epi32(v2, v6); + u3 = _mm_add_epi32(v3, v7); + u4 = _mm_sub_epi32(v0, v4); + u5 = _mm_sub_epi32(v1, v5); + u6 = _mm_sub_epi32(v2, v6); + u7 = _mm_sub_epi32(v3, v7); + + // stage 6 + x = _mm_mullo_epi32(u0, cospi4); + y = _mm_mullo_epi32(u1, cospi60); + v0 = _mm_add_epi32(x, y); + v0 = _mm_add_epi32(v0, rnding); + v0 = _mm_srai_epi32(v0, bit); + + x = _mm_mullo_epi32(u0, cospi60); + y = _mm_mullo_epi32(u1, cospim4); + v1 = _mm_add_epi32(x, y); + v1 = _mm_add_epi32(v1, rnding); + v1 = _mm_srai_epi32(v1, bit); + + x = _mm_mullo_epi32(u2, cospi20); + y = _mm_mullo_epi32(u3, cospi44); + v2 = _mm_add_epi32(x, y); + v2 = _mm_add_epi32(v2, rnding); + v2 = _mm_srai_epi32(v2, bit); + + x = _mm_mullo_epi32(u2, cospi44); + y = _mm_mullo_epi32(u3, cospim20); + v3 = _mm_add_epi32(x, y); + v3 = _mm_add_epi32(v3, rnding); + v3 = _mm_srai_epi32(v3, bit); + + x = _mm_mullo_epi32(u4, cospi36); + y = _mm_mullo_epi32(u5, cospi28); + v4 = _mm_add_epi32(x, y); + v4 = _mm_add_epi32(v4, rnding); + v4 = _mm_srai_epi32(v4, bit); + + x = _mm_mullo_epi32(u4, cospi28); + y = _mm_mullo_epi32(u5, cospim36); + v5 = _mm_add_epi32(x, y); + v5 = _mm_add_epi32(v5, rnding); + v5 = _mm_srai_epi32(v5, bit); + + x = _mm_mullo_epi32(u6, cospi52); + y = _mm_mullo_epi32(u7, cospi12); + v6 = _mm_add_epi32(x, y); + v6 = _mm_add_epi32(v6, rnding); + v6 = _mm_srai_epi32(v6, bit); + + x = _mm_mullo_epi32(u6, cospi12); + y = _mm_mullo_epi32(u7, cospim52); + v7 = _mm_add_epi32(x, y); + v7 = _mm_add_epi32(v7, rnding); + v7 = _mm_srai_epi32(v7, bit); + + // stage 7 + out[2 * 0 + col] = v1; + out[2 * 1 + col] = v6; + out[2 * 2 + col] = v3; + out[2 * 3 + col] = v4; + out[2 * 4 + col] = v5; + out[2 * 5 + col] = v2; + out[2 * 6 + col] = v7; + out[2 * 7 + col] = v0; + } +} + +static void round_shift_8x8(__m128i *in, int shift) { + round_shift_4x4(&in[0], shift); + round_shift_4x4(&in[4], shift); + round_shift_4x4(&in[8], shift); + round_shift_4x4(&in[12], shift); +} + +static __m128i get_recon_8x8(const __m128i pred, __m128i res_lo, __m128i res_hi, + int fliplr, int bd) { + __m128i x0, x1; + const __m128i zero = _mm_setzero_si128(); + + x0 = _mm_unpacklo_epi16(pred, zero); + x1 = _mm_unpackhi_epi16(pred, zero); + + if (fliplr) { + res_lo = _mm_shuffle_epi32(res_lo, 0x1B); + res_hi = _mm_shuffle_epi32(res_hi, 0x1B); + x0 = _mm_add_epi32(res_hi, x0); + x1 = _mm_add_epi32(res_lo, x1); + + } else { + x0 = _mm_add_epi32(res_lo, x0); + x1 = _mm_add_epi32(res_hi, x1); + } + + x0 = _mm_packus_epi32(x0, x1); + return highbd_clamp_epi16(x0, bd); +} + +static void write_buffer_8x8(__m128i *in, uint16_t *output, int stride, + int fliplr, int flipud, int shift, int bd) { + __m128i u0, u1, u2, u3, u4, u5, u6, u7; + __m128i v0, v1, v2, v3, v4, v5, v6, v7; + + round_shift_8x8(in, shift); + + v0 = _mm_load_si128((__m128i const *)(output + 0 * stride)); + v1 = _mm_load_si128((__m128i const *)(output + 1 * stride)); + v2 = _mm_load_si128((__m128i const *)(output + 2 * stride)); + v3 = _mm_load_si128((__m128i const *)(output + 3 * stride)); + v4 = _mm_load_si128((__m128i const *)(output + 4 * stride)); + v5 = _mm_load_si128((__m128i const *)(output + 5 * stride)); + v6 = _mm_load_si128((__m128i const *)(output + 6 * stride)); + v7 = _mm_load_si128((__m128i const *)(output + 7 * stride)); + + if (flipud) { + u0 = get_recon_8x8(v0, in[14], in[15], fliplr, bd); + u1 = get_recon_8x8(v1, in[12], in[13], fliplr, bd); + u2 = get_recon_8x8(v2, in[10], in[11], fliplr, bd); + u3 = get_recon_8x8(v3, in[8], in[9], fliplr, bd); + u4 = get_recon_8x8(v4, in[6], in[7], fliplr, bd); + u5 = get_recon_8x8(v5, in[4], in[5], fliplr, bd); + u6 = get_recon_8x8(v6, in[2], in[3], fliplr, bd); + u7 = get_recon_8x8(v7, in[0], in[1], fliplr, bd); + } else { + u0 = get_recon_8x8(v0, in[0], in[1], fliplr, bd); + u1 = get_recon_8x8(v1, in[2], in[3], fliplr, bd); + u2 = get_recon_8x8(v2, in[4], in[5], fliplr, bd); + u3 = get_recon_8x8(v3, in[6], in[7], fliplr, bd); + u4 = get_recon_8x8(v4, in[8], in[9], fliplr, bd); + u5 = get_recon_8x8(v5, in[10], in[11], fliplr, bd); + u6 = get_recon_8x8(v6, in[12], in[13], fliplr, bd); + u7 = get_recon_8x8(v7, in[14], in[15], fliplr, bd); + } + + _mm_store_si128((__m128i *)(output + 0 * stride), u0); + _mm_store_si128((__m128i *)(output + 1 * stride), u1); + _mm_store_si128((__m128i *)(output + 2 * stride), u2); + _mm_store_si128((__m128i *)(output + 3 * stride), u3); + _mm_store_si128((__m128i *)(output + 4 * stride), u4); + _mm_store_si128((__m128i *)(output + 5 * stride), u5); + _mm_store_si128((__m128i *)(output + 6 * stride), u6); + _mm_store_si128((__m128i *)(output + 7 * stride), u7); +} + +void av1_inv_txfm2d_add_8x8_sse4_1(const int32_t *coeff, uint16_t *output, + int stride, int tx_type, int bd) { + __m128i in[16], out[16]; + const TXFM_2D_CFG *cfg = NULL; + + switch (tx_type) { + case DCT_DCT: + cfg = &inv_txfm_2d_cfg_dct_dct_8; + load_buffer_8x8(coeff, in); + transpose_8x8(in, out); + idct8x8_sse4_1(out, in, cfg->cos_bit_row[2]); + transpose_8x8(in, out); + idct8x8_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_8x8(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; + case DCT_ADST: + cfg = &inv_txfm_2d_cfg_dct_adst_8; + load_buffer_8x8(coeff, in); + transpose_8x8(in, out); + iadst8x8_sse4_1(out, in, cfg->cos_bit_row[2]); + transpose_8x8(in, out); + idct8x8_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_8x8(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; + case ADST_DCT: + cfg = &inv_txfm_2d_cfg_adst_dct_8; + load_buffer_8x8(coeff, in); + transpose_8x8(in, out); + idct8x8_sse4_1(out, in, cfg->cos_bit_row[2]); + transpose_8x8(in, out); + iadst8x8_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_8x8(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; + case ADST_ADST: + cfg = &inv_txfm_2d_cfg_adst_adst_8; + load_buffer_8x8(coeff, in); + transpose_8x8(in, out); + iadst8x8_sse4_1(out, in, cfg->cos_bit_row[2]); + transpose_8x8(in, out); + iadst8x8_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_8x8(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + cfg = &inv_txfm_2d_cfg_adst_dct_8; + load_buffer_8x8(coeff, in); + transpose_8x8(in, out); + idct8x8_sse4_1(out, in, cfg->cos_bit_row[2]); + transpose_8x8(in, out); + iadst8x8_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_8x8(in, output, stride, 0, 1, -cfg->shift[1], bd); + break; + case DCT_FLIPADST: + cfg = &inv_txfm_2d_cfg_dct_adst_8; + load_buffer_8x8(coeff, in); + transpose_8x8(in, out); + iadst8x8_sse4_1(out, in, cfg->cos_bit_row[2]); + transpose_8x8(in, out); + idct8x8_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_8x8(in, output, stride, 1, 0, -cfg->shift[1], bd); + break; + case ADST_FLIPADST: + cfg = &inv_txfm_2d_cfg_adst_adst_8; + load_buffer_8x8(coeff, in); + transpose_8x8(in, out); + iadst8x8_sse4_1(out, in, cfg->cos_bit_row[2]); + transpose_8x8(in, out); + iadst8x8_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_8x8(in, output, stride, 1, 0, -cfg->shift[1], bd); + break; + case FLIPADST_FLIPADST: + cfg = &inv_txfm_2d_cfg_adst_adst_8; + load_buffer_8x8(coeff, in); + transpose_8x8(in, out); + iadst8x8_sse4_1(out, in, cfg->cos_bit_row[2]); + transpose_8x8(in, out); + iadst8x8_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_8x8(in, output, stride, 1, 1, -cfg->shift[1], bd); + break; + case FLIPADST_ADST: + cfg = &inv_txfm_2d_cfg_adst_adst_8; + load_buffer_8x8(coeff, in); + transpose_8x8(in, out); + iadst8x8_sse4_1(out, in, cfg->cos_bit_row[2]); + transpose_8x8(in, out); + iadst8x8_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_8x8(in, output, stride, 0, 1, -cfg->shift[1], bd); + break; +#endif // CONFIG_EXT_TX + default: assert(0); + } +} + +// 16x16 +static void load_buffer_16x16(const int32_t *coeff, __m128i *in) { + int i; + for (i = 0; i < 64; ++i) { + in[i] = _mm_load_si128((const __m128i *)(coeff + (i << 2))); + } +} + +static void assign_8x8_input_from_16x16(const __m128i *in, __m128i *in8x8, + int col) { + int i; + for (i = 0; i < 16; i += 2) { + in8x8[i] = in[col]; + in8x8[i + 1] = in[col + 1]; + col += 4; + } +} + +static void swap_addr(uint16_t **output1, uint16_t **output2) { + uint16_t *tmp; + tmp = *output1; + *output1 = *output2; + *output2 = tmp; +} + +static void write_buffer_16x16(__m128i *in, uint16_t *output, int stride, + int fliplr, int flipud, int shift, int bd) { + __m128i in8x8[16]; + uint16_t *leftUp = &output[0]; + uint16_t *rightUp = &output[8]; + uint16_t *leftDown = &output[8 * stride]; + uint16_t *rightDown = &output[8 * stride + 8]; + + if (fliplr) { + swap_addr(&leftUp, &rightUp); + swap_addr(&leftDown, &rightDown); + } + + if (flipud) { + swap_addr(&leftUp, &leftDown); + swap_addr(&rightUp, &rightDown); + } + + // Left-up quarter + assign_8x8_input_from_16x16(in, in8x8, 0); + write_buffer_8x8(in8x8, leftUp, stride, fliplr, flipud, shift, bd); + + // Right-up quarter + assign_8x8_input_from_16x16(in, in8x8, 2); + write_buffer_8x8(in8x8, rightUp, stride, fliplr, flipud, shift, bd); + + // Left-down quarter + assign_8x8_input_from_16x16(in, in8x8, 32); + write_buffer_8x8(in8x8, leftDown, stride, fliplr, flipud, shift, bd); + + // Right-down quarter + assign_8x8_input_from_16x16(in, in8x8, 34); + write_buffer_8x8(in8x8, rightDown, stride, fliplr, flipud, shift, bd); +} + +static void idct16x16_sse4_1(__m128i *in, __m128i *out, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m128i cospi60 = _mm_set1_epi32(cospi[60]); + const __m128i cospim4 = _mm_set1_epi32(-cospi[4]); + const __m128i cospi28 = _mm_set1_epi32(cospi[28]); + const __m128i cospim36 = _mm_set1_epi32(-cospi[36]); + const __m128i cospi44 = _mm_set1_epi32(cospi[44]); + const __m128i cospi20 = _mm_set1_epi32(cospi[20]); + const __m128i cospim20 = _mm_set1_epi32(-cospi[20]); + const __m128i cospi12 = _mm_set1_epi32(cospi[12]); + const __m128i cospim52 = _mm_set1_epi32(-cospi[52]); + const __m128i cospi52 = _mm_set1_epi32(cospi[52]); + const __m128i cospi36 = _mm_set1_epi32(cospi[36]); + const __m128i cospi4 = _mm_set1_epi32(cospi[4]); + const __m128i cospi56 = _mm_set1_epi32(cospi[56]); + const __m128i cospim8 = _mm_set1_epi32(-cospi[8]); + const __m128i cospi24 = _mm_set1_epi32(cospi[24]); + const __m128i cospim40 = _mm_set1_epi32(-cospi[40]); + const __m128i cospi40 = _mm_set1_epi32(cospi[40]); + const __m128i cospi8 = _mm_set1_epi32(cospi[8]); + const __m128i cospi32 = _mm_set1_epi32(cospi[32]); + const __m128i cospi48 = _mm_set1_epi32(cospi[48]); + const __m128i cospi16 = _mm_set1_epi32(cospi[16]); + const __m128i cospim16 = _mm_set1_epi32(-cospi[16]); + const __m128i cospim48 = _mm_set1_epi32(-cospi[48]); + const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); + __m128i u[16], v[16], x, y; + int col; + + for (col = 0; col < 4; ++col) { + // stage 0 + // stage 1 + u[0] = in[0 * 4 + col]; + u[1] = in[8 * 4 + col]; + u[2] = in[4 * 4 + col]; + u[3] = in[12 * 4 + col]; + u[4] = in[2 * 4 + col]; + u[5] = in[10 * 4 + col]; + u[6] = in[6 * 4 + col]; + u[7] = in[14 * 4 + col]; + u[8] = in[1 * 4 + col]; + u[9] = in[9 * 4 + col]; + u[10] = in[5 * 4 + col]; + u[11] = in[13 * 4 + col]; + u[12] = in[3 * 4 + col]; + u[13] = in[11 * 4 + col]; + u[14] = in[7 * 4 + col]; + u[15] = in[15 * 4 + col]; + + // stage 2 + v[0] = u[0]; + v[1] = u[1]; + v[2] = u[2]; + v[3] = u[3]; + v[4] = u[4]; + v[5] = u[5]; + v[6] = u[6]; + v[7] = u[7]; + + v[8] = half_btf_sse4_1(cospi60, u[8], cospim4, u[15], rnding, bit); + v[9] = half_btf_sse4_1(cospi28, u[9], cospim36, u[14], rnding, bit); + v[10] = half_btf_sse4_1(cospi44, u[10], cospim20, u[13], rnding, bit); + v[11] = half_btf_sse4_1(cospi12, u[11], cospim52, u[12], rnding, bit); + v[12] = half_btf_sse4_1(cospi52, u[11], cospi12, u[12], rnding, bit); + v[13] = half_btf_sse4_1(cospi20, u[10], cospi44, u[13], rnding, bit); + v[14] = half_btf_sse4_1(cospi36, u[9], cospi28, u[14], rnding, bit); + v[15] = half_btf_sse4_1(cospi4, u[8], cospi60, u[15], rnding, bit); + + // stage 3 + u[0] = v[0]; + u[1] = v[1]; + u[2] = v[2]; + u[3] = v[3]; + u[4] = half_btf_sse4_1(cospi56, v[4], cospim8, v[7], rnding, bit); + u[5] = half_btf_sse4_1(cospi24, v[5], cospim40, v[6], rnding, bit); + u[6] = half_btf_sse4_1(cospi40, v[5], cospi24, v[6], rnding, bit); + u[7] = half_btf_sse4_1(cospi8, v[4], cospi56, v[7], rnding, bit); + u[8] = _mm_add_epi32(v[8], v[9]); + u[9] = _mm_sub_epi32(v[8], v[9]); + u[10] = _mm_sub_epi32(v[11], v[10]); + u[11] = _mm_add_epi32(v[10], v[11]); + u[12] = _mm_add_epi32(v[12], v[13]); + u[13] = _mm_sub_epi32(v[12], v[13]); + u[14] = _mm_sub_epi32(v[15], v[14]); + u[15] = _mm_add_epi32(v[14], v[15]); + + // stage 4 + x = _mm_mullo_epi32(u[0], cospi32); + y = _mm_mullo_epi32(u[1], cospi32); + v[0] = _mm_add_epi32(x, y); + v[0] = _mm_add_epi32(v[0], rnding); + v[0] = _mm_srai_epi32(v[0], bit); + + v[1] = _mm_sub_epi32(x, y); + v[1] = _mm_add_epi32(v[1], rnding); + v[1] = _mm_srai_epi32(v[1], bit); + + v[2] = half_btf_sse4_1(cospi48, u[2], cospim16, u[3], rnding, bit); + v[3] = half_btf_sse4_1(cospi16, u[2], cospi48, u[3], rnding, bit); + v[4] = _mm_add_epi32(u[4], u[5]); + v[5] = _mm_sub_epi32(u[4], u[5]); + v[6] = _mm_sub_epi32(u[7], u[6]); + v[7] = _mm_add_epi32(u[6], u[7]); + v[8] = u[8]; + v[9] = half_btf_sse4_1(cospim16, u[9], cospi48, u[14], rnding, bit); + v[10] = half_btf_sse4_1(cospim48, u[10], cospim16, u[13], rnding, bit); + v[11] = u[11]; + v[12] = u[12]; + v[13] = half_btf_sse4_1(cospim16, u[10], cospi48, u[13], rnding, bit); + v[14] = half_btf_sse4_1(cospi48, u[9], cospi16, u[14], rnding, bit); + v[15] = u[15]; + + // stage 5 + u[0] = _mm_add_epi32(v[0], v[3]); + u[1] = _mm_add_epi32(v[1], v[2]); + u[2] = _mm_sub_epi32(v[1], v[2]); + u[3] = _mm_sub_epi32(v[0], v[3]); + u[4] = v[4]; + + x = _mm_mullo_epi32(v[5], cospi32); + y = _mm_mullo_epi32(v[6], cospi32); + u[5] = _mm_sub_epi32(y, x); + u[5] = _mm_add_epi32(u[5], rnding); + u[5] = _mm_srai_epi32(u[5], bit); + + u[6] = _mm_add_epi32(y, x); + u[6] = _mm_add_epi32(u[6], rnding); + u[6] = _mm_srai_epi32(u[6], bit); + + u[7] = v[7]; + u[8] = _mm_add_epi32(v[8], v[11]); + u[9] = _mm_add_epi32(v[9], v[10]); + u[10] = _mm_sub_epi32(v[9], v[10]); + u[11] = _mm_sub_epi32(v[8], v[11]); + u[12] = _mm_sub_epi32(v[15], v[12]); + u[13] = _mm_sub_epi32(v[14], v[13]); + u[14] = _mm_add_epi32(v[13], v[14]); + u[15] = _mm_add_epi32(v[12], v[15]); + + // stage 6 + v[0] = _mm_add_epi32(u[0], u[7]); + v[1] = _mm_add_epi32(u[1], u[6]); + v[2] = _mm_add_epi32(u[2], u[5]); + v[3] = _mm_add_epi32(u[3], u[4]); + v[4] = _mm_sub_epi32(u[3], u[4]); + v[5] = _mm_sub_epi32(u[2], u[5]); + v[6] = _mm_sub_epi32(u[1], u[6]); + v[7] = _mm_sub_epi32(u[0], u[7]); + v[8] = u[8]; + v[9] = u[9]; + + x = _mm_mullo_epi32(u[10], cospi32); + y = _mm_mullo_epi32(u[13], cospi32); + v[10] = _mm_sub_epi32(y, x); + v[10] = _mm_add_epi32(v[10], rnding); + v[10] = _mm_srai_epi32(v[10], bit); + + v[13] = _mm_add_epi32(x, y); + v[13] = _mm_add_epi32(v[13], rnding); + v[13] = _mm_srai_epi32(v[13], bit); + + x = _mm_mullo_epi32(u[11], cospi32); + y = _mm_mullo_epi32(u[12], cospi32); + v[11] = _mm_sub_epi32(y, x); + v[11] = _mm_add_epi32(v[11], rnding); + v[11] = _mm_srai_epi32(v[11], bit); + + v[12] = _mm_add_epi32(x, y); + v[12] = _mm_add_epi32(v[12], rnding); + v[12] = _mm_srai_epi32(v[12], bit); + + v[14] = u[14]; + v[15] = u[15]; + + // stage 7 + out[0 * 4 + col] = _mm_add_epi32(v[0], v[15]); + out[1 * 4 + col] = _mm_add_epi32(v[1], v[14]); + out[2 * 4 + col] = _mm_add_epi32(v[2], v[13]); + out[3 * 4 + col] = _mm_add_epi32(v[3], v[12]); + out[4 * 4 + col] = _mm_add_epi32(v[4], v[11]); + out[5 * 4 + col] = _mm_add_epi32(v[5], v[10]); + out[6 * 4 + col] = _mm_add_epi32(v[6], v[9]); + out[7 * 4 + col] = _mm_add_epi32(v[7], v[8]); + out[8 * 4 + col] = _mm_sub_epi32(v[7], v[8]); + out[9 * 4 + col] = _mm_sub_epi32(v[6], v[9]); + out[10 * 4 + col] = _mm_sub_epi32(v[5], v[10]); + out[11 * 4 + col] = _mm_sub_epi32(v[4], v[11]); + out[12 * 4 + col] = _mm_sub_epi32(v[3], v[12]); + out[13 * 4 + col] = _mm_sub_epi32(v[2], v[13]); + out[14 * 4 + col] = _mm_sub_epi32(v[1], v[14]); + out[15 * 4 + col] = _mm_sub_epi32(v[0], v[15]); + } +} + +static void iadst16x16_sse4_1(__m128i *in, __m128i *out, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m128i cospi32 = _mm_set1_epi32(cospi[32]); + const __m128i cospi48 = _mm_set1_epi32(cospi[48]); + const __m128i cospi16 = _mm_set1_epi32(cospi[16]); + const __m128i cospim16 = _mm_set1_epi32(-cospi[16]); + const __m128i cospim48 = _mm_set1_epi32(-cospi[48]); + const __m128i cospi8 = _mm_set1_epi32(cospi[8]); + const __m128i cospi56 = _mm_set1_epi32(cospi[56]); + const __m128i cospim56 = _mm_set1_epi32(-cospi[56]); + const __m128i cospim8 = _mm_set1_epi32(-cospi[8]); + const __m128i cospi24 = _mm_set1_epi32(cospi[24]); + const __m128i cospim24 = _mm_set1_epi32(-cospi[24]); + const __m128i cospim40 = _mm_set1_epi32(-cospi[40]); + const __m128i cospi40 = _mm_set1_epi32(cospi[40]); + const __m128i cospi2 = _mm_set1_epi32(cospi[2]); + const __m128i cospi62 = _mm_set1_epi32(cospi[62]); + const __m128i cospim2 = _mm_set1_epi32(-cospi[2]); + const __m128i cospi10 = _mm_set1_epi32(cospi[10]); + const __m128i cospi54 = _mm_set1_epi32(cospi[54]); + const __m128i cospim10 = _mm_set1_epi32(-cospi[10]); + const __m128i cospi18 = _mm_set1_epi32(cospi[18]); + const __m128i cospi46 = _mm_set1_epi32(cospi[46]); + const __m128i cospim18 = _mm_set1_epi32(-cospi[18]); + const __m128i cospi26 = _mm_set1_epi32(cospi[26]); + const __m128i cospi38 = _mm_set1_epi32(cospi[38]); + const __m128i cospim26 = _mm_set1_epi32(-cospi[26]); + const __m128i cospi34 = _mm_set1_epi32(cospi[34]); + const __m128i cospi30 = _mm_set1_epi32(cospi[30]); + const __m128i cospim34 = _mm_set1_epi32(-cospi[34]); + const __m128i cospi42 = _mm_set1_epi32(cospi[42]); + const __m128i cospi22 = _mm_set1_epi32(cospi[22]); + const __m128i cospim42 = _mm_set1_epi32(-cospi[42]); + const __m128i cospi50 = _mm_set1_epi32(cospi[50]); + const __m128i cospi14 = _mm_set1_epi32(cospi[14]); + const __m128i cospim50 = _mm_set1_epi32(-cospi[50]); + const __m128i cospi58 = _mm_set1_epi32(cospi[58]); + const __m128i cospi6 = _mm_set1_epi32(cospi[6]); + const __m128i cospim58 = _mm_set1_epi32(-cospi[58]); + const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); + const __m128i zero = _mm_setzero_si128(); + + __m128i u[16], v[16], x, y; + int col; + + for (col = 0; col < 4; ++col) { + // stage 0 + // stage 1 + u[0] = in[0 * 4 + col]; + u[1] = _mm_sub_epi32(zero, in[15 * 4 + col]); + u[2] = _mm_sub_epi32(zero, in[7 * 4 + col]); + u[3] = in[8 * 4 + col]; + u[4] = _mm_sub_epi32(zero, in[3 * 4 + col]); + u[5] = in[12 * 4 + col]; + u[6] = in[4 * 4 + col]; + u[7] = _mm_sub_epi32(zero, in[11 * 4 + col]); + u[8] = _mm_sub_epi32(zero, in[1 * 4 + col]); + u[9] = in[14 * 4 + col]; + u[10] = in[6 * 4 + col]; + u[11] = _mm_sub_epi32(zero, in[9 * 4 + col]); + u[12] = in[2 * 4 + col]; + u[13] = _mm_sub_epi32(zero, in[13 * 4 + col]); + u[14] = _mm_sub_epi32(zero, in[5 * 4 + col]); + u[15] = in[10 * 4 + col]; + + // stage 2 + v[0] = u[0]; + v[1] = u[1]; + + x = _mm_mullo_epi32(u[2], cospi32); + y = _mm_mullo_epi32(u[3], cospi32); + v[2] = _mm_add_epi32(x, y); + v[2] = _mm_add_epi32(v[2], rnding); + v[2] = _mm_srai_epi32(v[2], bit); + + v[3] = _mm_sub_epi32(x, y); + v[3] = _mm_add_epi32(v[3], rnding); + v[3] = _mm_srai_epi32(v[3], bit); + + v[4] = u[4]; + v[5] = u[5]; + + x = _mm_mullo_epi32(u[6], cospi32); + y = _mm_mullo_epi32(u[7], cospi32); + v[6] = _mm_add_epi32(x, y); + v[6] = _mm_add_epi32(v[6], rnding); + v[6] = _mm_srai_epi32(v[6], bit); + + v[7] = _mm_sub_epi32(x, y); + v[7] = _mm_add_epi32(v[7], rnding); + v[7] = _mm_srai_epi32(v[7], bit); + + v[8] = u[8]; + v[9] = u[9]; + + x = _mm_mullo_epi32(u[10], cospi32); + y = _mm_mullo_epi32(u[11], cospi32); + v[10] = _mm_add_epi32(x, y); + v[10] = _mm_add_epi32(v[10], rnding); + v[10] = _mm_srai_epi32(v[10], bit); + + v[11] = _mm_sub_epi32(x, y); + v[11] = _mm_add_epi32(v[11], rnding); + v[11] = _mm_srai_epi32(v[11], bit); + + v[12] = u[12]; + v[13] = u[13]; + + x = _mm_mullo_epi32(u[14], cospi32); + y = _mm_mullo_epi32(u[15], cospi32); + v[14] = _mm_add_epi32(x, y); + v[14] = _mm_add_epi32(v[14], rnding); + v[14] = _mm_srai_epi32(v[14], bit); + + v[15] = _mm_sub_epi32(x, y); + v[15] = _mm_add_epi32(v[15], rnding); + v[15] = _mm_srai_epi32(v[15], bit); + + // stage 3 + u[0] = _mm_add_epi32(v[0], v[2]); + u[1] = _mm_add_epi32(v[1], v[3]); + u[2] = _mm_sub_epi32(v[0], v[2]); + u[3] = _mm_sub_epi32(v[1], v[3]); + u[4] = _mm_add_epi32(v[4], v[6]); + u[5] = _mm_add_epi32(v[5], v[7]); + u[6] = _mm_sub_epi32(v[4], v[6]); + u[7] = _mm_sub_epi32(v[5], v[7]); + u[8] = _mm_add_epi32(v[8], v[10]); + u[9] = _mm_add_epi32(v[9], v[11]); + u[10] = _mm_sub_epi32(v[8], v[10]); + u[11] = _mm_sub_epi32(v[9], v[11]); + u[12] = _mm_add_epi32(v[12], v[14]); + u[13] = _mm_add_epi32(v[13], v[15]); + u[14] = _mm_sub_epi32(v[12], v[14]); + u[15] = _mm_sub_epi32(v[13], v[15]); + + // stage 4 + v[0] = u[0]; + v[1] = u[1]; + v[2] = u[2]; + v[3] = u[3]; + v[4] = half_btf_sse4_1(cospi16, u[4], cospi48, u[5], rnding, bit); + v[5] = half_btf_sse4_1(cospi48, u[4], cospim16, u[5], rnding, bit); + v[6] = half_btf_sse4_1(cospim48, u[6], cospi16, u[7], rnding, bit); + v[7] = half_btf_sse4_1(cospi16, u[6], cospi48, u[7], rnding, bit); + v[8] = u[8]; + v[9] = u[9]; + v[10] = u[10]; + v[11] = u[11]; + v[12] = half_btf_sse4_1(cospi16, u[12], cospi48, u[13], rnding, bit); + v[13] = half_btf_sse4_1(cospi48, u[12], cospim16, u[13], rnding, bit); + v[14] = half_btf_sse4_1(cospim48, u[14], cospi16, u[15], rnding, bit); + v[15] = half_btf_sse4_1(cospi16, u[14], cospi48, u[15], rnding, bit); + + // stage 5 + u[0] = _mm_add_epi32(v[0], v[4]); + u[1] = _mm_add_epi32(v[1], v[5]); + u[2] = _mm_add_epi32(v[2], v[6]); + u[3] = _mm_add_epi32(v[3], v[7]); + u[4] = _mm_sub_epi32(v[0], v[4]); + u[5] = _mm_sub_epi32(v[1], v[5]); + u[6] = _mm_sub_epi32(v[2], v[6]); + u[7] = _mm_sub_epi32(v[3], v[7]); + u[8] = _mm_add_epi32(v[8], v[12]); + u[9] = _mm_add_epi32(v[9], v[13]); + u[10] = _mm_add_epi32(v[10], v[14]); + u[11] = _mm_add_epi32(v[11], v[15]); + u[12] = _mm_sub_epi32(v[8], v[12]); + u[13] = _mm_sub_epi32(v[9], v[13]); + u[14] = _mm_sub_epi32(v[10], v[14]); + u[15] = _mm_sub_epi32(v[11], v[15]); + + // stage 6 + v[0] = u[0]; + v[1] = u[1]; + v[2] = u[2]; + v[3] = u[3]; + v[4] = u[4]; + v[5] = u[5]; + v[6] = u[6]; + v[7] = u[7]; + v[8] = half_btf_sse4_1(cospi8, u[8], cospi56, u[9], rnding, bit); + v[9] = half_btf_sse4_1(cospi56, u[8], cospim8, u[9], rnding, bit); + v[10] = half_btf_sse4_1(cospi40, u[10], cospi24, u[11], rnding, bit); + v[11] = half_btf_sse4_1(cospi24, u[10], cospim40, u[11], rnding, bit); + v[12] = half_btf_sse4_1(cospim56, u[12], cospi8, u[13], rnding, bit); + v[13] = half_btf_sse4_1(cospi8, u[12], cospi56, u[13], rnding, bit); + v[14] = half_btf_sse4_1(cospim24, u[14], cospi40, u[15], rnding, bit); + v[15] = half_btf_sse4_1(cospi40, u[14], cospi24, u[15], rnding, bit); + + // stage 7 + u[0] = _mm_add_epi32(v[0], v[8]); + u[1] = _mm_add_epi32(v[1], v[9]); + u[2] = _mm_add_epi32(v[2], v[10]); + u[3] = _mm_add_epi32(v[3], v[11]); + u[4] = _mm_add_epi32(v[4], v[12]); + u[5] = _mm_add_epi32(v[5], v[13]); + u[6] = _mm_add_epi32(v[6], v[14]); + u[7] = _mm_add_epi32(v[7], v[15]); + u[8] = _mm_sub_epi32(v[0], v[8]); + u[9] = _mm_sub_epi32(v[1], v[9]); + u[10] = _mm_sub_epi32(v[2], v[10]); + u[11] = _mm_sub_epi32(v[3], v[11]); + u[12] = _mm_sub_epi32(v[4], v[12]); + u[13] = _mm_sub_epi32(v[5], v[13]); + u[14] = _mm_sub_epi32(v[6], v[14]); + u[15] = _mm_sub_epi32(v[7], v[15]); + + // stage 8 + v[0] = half_btf_sse4_1(cospi2, u[0], cospi62, u[1], rnding, bit); + v[1] = half_btf_sse4_1(cospi62, u[0], cospim2, u[1], rnding, bit); + v[2] = half_btf_sse4_1(cospi10, u[2], cospi54, u[3], rnding, bit); + v[3] = half_btf_sse4_1(cospi54, u[2], cospim10, u[3], rnding, bit); + v[4] = half_btf_sse4_1(cospi18, u[4], cospi46, u[5], rnding, bit); + v[5] = half_btf_sse4_1(cospi46, u[4], cospim18, u[5], rnding, bit); + v[6] = half_btf_sse4_1(cospi26, u[6], cospi38, u[7], rnding, bit); + v[7] = half_btf_sse4_1(cospi38, u[6], cospim26, u[7], rnding, bit); + v[8] = half_btf_sse4_1(cospi34, u[8], cospi30, u[9], rnding, bit); + v[9] = half_btf_sse4_1(cospi30, u[8], cospim34, u[9], rnding, bit); + v[10] = half_btf_sse4_1(cospi42, u[10], cospi22, u[11], rnding, bit); + v[11] = half_btf_sse4_1(cospi22, u[10], cospim42, u[11], rnding, bit); + v[12] = half_btf_sse4_1(cospi50, u[12], cospi14, u[13], rnding, bit); + v[13] = half_btf_sse4_1(cospi14, u[12], cospim50, u[13], rnding, bit); + v[14] = half_btf_sse4_1(cospi58, u[14], cospi6, u[15], rnding, bit); + v[15] = half_btf_sse4_1(cospi6, u[14], cospim58, u[15], rnding, bit); + + // stage 9 + out[0 * 4 + col] = v[1]; + out[1 * 4 + col] = v[14]; + out[2 * 4 + col] = v[3]; + out[3 * 4 + col] = v[12]; + out[4 * 4 + col] = v[5]; + out[5 * 4 + col] = v[10]; + out[6 * 4 + col] = v[7]; + out[7 * 4 + col] = v[8]; + out[8 * 4 + col] = v[9]; + out[9 * 4 + col] = v[6]; + out[10 * 4 + col] = v[11]; + out[11 * 4 + col] = v[4]; + out[12 * 4 + col] = v[13]; + out[13 * 4 + col] = v[2]; + out[14 * 4 + col] = v[15]; + out[15 * 4 + col] = v[0]; + } +} + +static void round_shift_16x16(__m128i *in, int shift) { + round_shift_8x8(&in[0], shift); + round_shift_8x8(&in[16], shift); + round_shift_8x8(&in[32], shift); + round_shift_8x8(&in[48], shift); +} + +void av1_inv_txfm2d_add_16x16_sse4_1(const int32_t *coeff, uint16_t *output, + int stride, int tx_type, int bd) { + __m128i in[64], out[64]; + const TXFM_2D_CFG *cfg = NULL; + + switch (tx_type) { + case DCT_DCT: + cfg = &inv_txfm_2d_cfg_dct_dct_16; + load_buffer_16x16(coeff, in); + transpose_16x16(in, out); + idct16x16_sse4_1(out, in, cfg->cos_bit_row[2]); + round_shift_16x16(in, -cfg->shift[0]); + transpose_16x16(in, out); + idct16x16_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_16x16(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; + case DCT_ADST: + cfg = &inv_txfm_2d_cfg_dct_adst_16; + load_buffer_16x16(coeff, in); + transpose_16x16(in, out); + iadst16x16_sse4_1(out, in, cfg->cos_bit_row[2]); + round_shift_16x16(in, -cfg->shift[0]); + transpose_16x16(in, out); + idct16x16_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_16x16(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; + case ADST_DCT: + cfg = &inv_txfm_2d_cfg_adst_dct_16; + load_buffer_16x16(coeff, in); + transpose_16x16(in, out); + idct16x16_sse4_1(out, in, cfg->cos_bit_row[2]); + round_shift_16x16(in, -cfg->shift[0]); + transpose_16x16(in, out); + iadst16x16_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_16x16(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; + case ADST_ADST: + cfg = &inv_txfm_2d_cfg_adst_adst_16; + load_buffer_16x16(coeff, in); + transpose_16x16(in, out); + iadst16x16_sse4_1(out, in, cfg->cos_bit_row[2]); + round_shift_16x16(in, -cfg->shift[0]); + transpose_16x16(in, out); + iadst16x16_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_16x16(in, output, stride, 0, 0, -cfg->shift[1], bd); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + cfg = &inv_txfm_2d_cfg_adst_dct_16; + load_buffer_16x16(coeff, in); + transpose_16x16(in, out); + idct16x16_sse4_1(out, in, cfg->cos_bit_row[2]); + round_shift_16x16(in, -cfg->shift[0]); + transpose_16x16(in, out); + iadst16x16_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_16x16(in, output, stride, 0, 1, -cfg->shift[1], bd); + break; + case DCT_FLIPADST: + cfg = &inv_txfm_2d_cfg_dct_adst_16; + load_buffer_16x16(coeff, in); + transpose_16x16(in, out); + iadst16x16_sse4_1(out, in, cfg->cos_bit_row[2]); + round_shift_16x16(in, -cfg->shift[0]); + transpose_16x16(in, out); + idct16x16_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_16x16(in, output, stride, 1, 0, -cfg->shift[1], bd); + break; + case ADST_FLIPADST: + cfg = &inv_txfm_2d_cfg_adst_adst_16; + load_buffer_16x16(coeff, in); + transpose_16x16(in, out); + iadst16x16_sse4_1(out, in, cfg->cos_bit_row[2]); + round_shift_16x16(in, -cfg->shift[0]); + transpose_16x16(in, out); + iadst16x16_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_16x16(in, output, stride, 1, 0, -cfg->shift[1], bd); + break; + case FLIPADST_FLIPADST: + cfg = &inv_txfm_2d_cfg_adst_adst_16; + load_buffer_16x16(coeff, in); + transpose_16x16(in, out); + iadst16x16_sse4_1(out, in, cfg->cos_bit_row[2]); + round_shift_16x16(in, -cfg->shift[0]); + transpose_16x16(in, out); + iadst16x16_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_16x16(in, output, stride, 1, 1, -cfg->shift[1], bd); + break; + case FLIPADST_ADST: + cfg = &inv_txfm_2d_cfg_adst_adst_16; + load_buffer_16x16(coeff, in); + transpose_16x16(in, out); + iadst16x16_sse4_1(out, in, cfg->cos_bit_row[2]); + round_shift_16x16(in, -cfg->shift[0]); + transpose_16x16(in, out); + iadst16x16_sse4_1(out, in, cfg->cos_bit_col[2]); + write_buffer_16x16(in, output, stride, 0, 1, -cfg->shift[1], bd); + break; +#endif + default: assert(0); + } +} diff --git a/third_party/aom/av1/common/x86/highbd_txfm_utility_sse4.h b/third_party/aom/av1/common/x86/highbd_txfm_utility_sse4.h new file mode 100644 index 0000000000..bc96defe3b --- /dev/null +++ b/third_party/aom/av1/common/x86/highbd_txfm_utility_sse4.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef _HIGHBD_TXFM_UTILITY_SSE4_H +#define _HIGHBD_TXFM_UTILITY_SSE4_H + +#include /* SSE4.1 */ + +#define TRANSPOSE_4X4(x0, x1, x2, x3, y0, y1, y2, y3) \ + do { \ + __m128i u0, u1, u2, u3; \ + u0 = _mm_unpacklo_epi32(x0, x1); \ + u1 = _mm_unpackhi_epi32(x0, x1); \ + u2 = _mm_unpacklo_epi32(x2, x3); \ + u3 = _mm_unpackhi_epi32(x2, x3); \ + y0 = _mm_unpacklo_epi64(u0, u2); \ + y1 = _mm_unpackhi_epi64(u0, u2); \ + y2 = _mm_unpacklo_epi64(u1, u3); \ + y3 = _mm_unpackhi_epi64(u1, u3); \ + } while (0) + +static INLINE void transpose_8x8(const __m128i *in, __m128i *out) { + TRANSPOSE_4X4(in[0], in[2], in[4], in[6], out[0], out[2], out[4], out[6]); + TRANSPOSE_4X4(in[1], in[3], in[5], in[7], out[8], out[10], out[12], out[14]); + TRANSPOSE_4X4(in[8], in[10], in[12], in[14], out[1], out[3], out[5], out[7]); + TRANSPOSE_4X4(in[9], in[11], in[13], in[15], out[9], out[11], out[13], + out[15]); +} + +static INLINE void transpose_16x16(const __m128i *in, __m128i *out) { + // Upper left 8x8 + TRANSPOSE_4X4(in[0], in[4], in[8], in[12], out[0], out[4], out[8], out[12]); + TRANSPOSE_4X4(in[1], in[5], in[9], in[13], out[16], out[20], out[24], + out[28]); + TRANSPOSE_4X4(in[16], in[20], in[24], in[28], out[1], out[5], out[9], + out[13]); + TRANSPOSE_4X4(in[17], in[21], in[25], in[29], out[17], out[21], out[25], + out[29]); + + // Upper right 8x8 + TRANSPOSE_4X4(in[2], in[6], in[10], in[14], out[32], out[36], out[40], + out[44]); + TRANSPOSE_4X4(in[3], in[7], in[11], in[15], out[48], out[52], out[56], + out[60]); + TRANSPOSE_4X4(in[18], in[22], in[26], in[30], out[33], out[37], out[41], + out[45]); + TRANSPOSE_4X4(in[19], in[23], in[27], in[31], out[49], out[53], out[57], + out[61]); + + // Lower left 8x8 + TRANSPOSE_4X4(in[32], in[36], in[40], in[44], out[2], out[6], out[10], + out[14]); + TRANSPOSE_4X4(in[33], in[37], in[41], in[45], out[18], out[22], out[26], + out[30]); + TRANSPOSE_4X4(in[48], in[52], in[56], in[60], out[3], out[7], out[11], + out[15]); + TRANSPOSE_4X4(in[49], in[53], in[57], in[61], out[19], out[23], out[27], + out[31]); + // Lower right 8x8 + TRANSPOSE_4X4(in[34], in[38], in[42], in[46], out[34], out[38], out[42], + out[46]); + TRANSPOSE_4X4(in[35], in[39], in[43], in[47], out[50], out[54], out[58], + out[62]); + TRANSPOSE_4X4(in[50], in[54], in[58], in[62], out[35], out[39], out[43], + out[47]); + TRANSPOSE_4X4(in[51], in[55], in[59], in[63], out[51], out[55], out[59], + out[63]); +} + +// Note: +// rounding = 1 << (bit - 1) +static INLINE __m128i half_btf_sse4_1(__m128i w0, __m128i n0, __m128i w1, + __m128i n1, __m128i rounding, int bit) { + __m128i x, y; + + x = _mm_mullo_epi32(w0, n0); + y = _mm_mullo_epi32(w1, n1); + x = _mm_add_epi32(x, y); + x = _mm_add_epi32(x, rounding); + x = _mm_srai_epi32(x, bit); + return x; +} + +#endif // _HIGHBD_TXFM_UTILITY_SSE4_H diff --git a/third_party/aom/av1/common/x86/highbd_warp_plane_ssse3.c b/third_party/aom/av1/common/x86/highbd_warp_plane_ssse3.c new file mode 100644 index 0000000000..c25db88b70 --- /dev/null +++ b/third_party/aom/av1/common/x86/highbd_warp_plane_ssse3.c @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./av1_rtcd.h" +#include "av1/common/warped_motion.h" + +static const __m128i *const filter = (const __m128i *const)warped_filter; + +/* SSE2 version of the rotzoom/affine warp filter */ +void av1_highbd_warp_affine_ssse3(int32_t *mat, uint16_t *ref, int width, + int height, int stride, uint16_t *pred, + int p_col, int p_row, int p_width, + int p_height, int p_stride, int subsampling_x, + int subsampling_y, int bd, int ref_frm, + int16_t alpha, int16_t beta, int16_t gamma, + int16_t delta) { +#if HORSHEAR_REDUCE_PREC_BITS >= 5 + __m128i tmp[15]; +#else +#error "HORSHEAR_REDUCE_PREC_BITS < 5 not currently supported by SSSE3 filter" +#endif + int i, j, k; + + /* Note: For this code to work, the left/right frame borders need to be + extended by at least 13 pixels each. By the time we get here, other + code will have set up this border, but we allow an explicit check + for debugging purposes. + */ + /*for (i = 0; i < height; ++i) { + for (j = 0; j < 13; ++j) { + assert(ref[i * stride - 13 + j] == ref[i * stride]); + assert(ref[i * stride + width + j] == ref[i * stride + (width - 1)]); + } + }*/ + + for (i = 0; i < p_height; i += 8) { + for (j = 0; j < p_width; j += 8) { + // (x, y) coordinates of the center of this block in the destination + // image + int32_t dst_x = p_col + j + 4; + int32_t dst_y = p_row + i + 4; + + int32_t x4, y4, ix4, sx4, iy4, sy4; + if (subsampling_x) + x4 = ROUND_POWER_OF_TWO_SIGNED( + mat[2] * 2 * dst_x + mat[3] * 2 * dst_y + mat[0] + + (mat[2] + mat[3] - (1 << WARPEDMODEL_PREC_BITS)) / 2, + 1); + else + x4 = mat[2] * dst_x + mat[3] * dst_y + mat[0]; + + if (subsampling_y) + y4 = ROUND_POWER_OF_TWO_SIGNED( + mat[4] * 2 * dst_x + mat[5] * 2 * dst_y + mat[1] + + (mat[4] + mat[5] - (1 << WARPEDMODEL_PREC_BITS)) / 2, + 1); + else + y4 = mat[4] * dst_x + mat[5] * dst_y + mat[1]; + + ix4 = x4 >> WARPEDMODEL_PREC_BITS; + sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + iy4 = y4 >> WARPEDMODEL_PREC_BITS; + sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + + // Horizontal filter + for (k = -7; k < AOMMIN(8, p_height - i); ++k) { + int iy = iy4 + k; + if (iy < 0) + iy = 0; + else if (iy > height - 1) + iy = height - 1; + + // If the block is aligned such that, after clamping, every sample + // would be taken from the leftmost/rightmost column, then we can + // skip the expensive horizontal filter. + if (ix4 <= -7) { + tmp[k + 7] = _mm_set1_epi16( + ref[iy * stride] * + (1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS))); + } else if (ix4 >= width + 6) { + tmp[k + 7] = _mm_set1_epi16( + ref[iy * stride + (width - 1)] * + (1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS))); + } else { + int sx = sx4 + alpha * (-4) + beta * k + + // Include rounding and offset here + (1 << (WARPEDDIFF_PREC_BITS - 1)) + + (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); + + // Load source pixels + __m128i src = + _mm_loadu_si128((__m128i *)(ref + iy * stride + ix4 - 7)); + __m128i src2 = + _mm_loadu_si128((__m128i *)(ref + iy * stride + ix4 + 1)); + + // Filter even-index pixels + __m128i tmp_0 = filter[(sx + 0 * alpha) >> WARPEDDIFF_PREC_BITS]; + __m128i tmp_2 = filter[(sx + 2 * alpha) >> WARPEDDIFF_PREC_BITS]; + __m128i tmp_4 = filter[(sx + 4 * alpha) >> WARPEDDIFF_PREC_BITS]; + __m128i tmp_6 = filter[(sx + 6 * alpha) >> WARPEDDIFF_PREC_BITS]; + + // coeffs 0 1 0 1 2 3 2 3 for pixels 0, 2 + __m128i tmp_8 = _mm_unpacklo_epi32(tmp_0, tmp_2); + // coeffs 0 1 0 1 2 3 2 3 for pixels 4, 6 + __m128i tmp_10 = _mm_unpacklo_epi32(tmp_4, tmp_6); + // coeffs 4 5 4 5 6 7 6 7 for pixels 0, 2 + __m128i tmp_12 = _mm_unpackhi_epi32(tmp_0, tmp_2); + // coeffs 4 5 4 5 6 7 6 7 for pixels 4, 6 + __m128i tmp_14 = _mm_unpackhi_epi32(tmp_4, tmp_6); + + // coeffs 0 1 0 1 0 1 0 1 for pixels 0, 2, 4, 6 + __m128i coeff_0 = _mm_unpacklo_epi64(tmp_8, tmp_10); + // coeffs 2 3 2 3 2 3 2 3 for pixels 0, 2, 4, 6 + __m128i coeff_2 = _mm_unpackhi_epi64(tmp_8, tmp_10); + // coeffs 4 5 4 5 4 5 4 5 for pixels 0, 2, 4, 6 + __m128i coeff_4 = _mm_unpacklo_epi64(tmp_12, tmp_14); + // coeffs 6 7 6 7 6 7 6 7 for pixels 0, 2, 4, 6 + __m128i coeff_6 = _mm_unpackhi_epi64(tmp_12, tmp_14); + + __m128i round_const = + _mm_set1_epi32((1 << HORSHEAR_REDUCE_PREC_BITS) >> 1); + + // Calculate filtered results + __m128i res_0 = _mm_madd_epi16(src, coeff_0); + __m128i res_2 = + _mm_madd_epi16(_mm_alignr_epi8(src2, src, 4), coeff_2); + __m128i res_4 = + _mm_madd_epi16(_mm_alignr_epi8(src2, src, 8), coeff_4); + __m128i res_6 = + _mm_madd_epi16(_mm_alignr_epi8(src2, src, 12), coeff_6); + + __m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_4), + _mm_add_epi32(res_2, res_6)); + res_even = _mm_srai_epi32(_mm_add_epi32(res_even, round_const), + HORSHEAR_REDUCE_PREC_BITS); + + // Filter odd-index pixels + __m128i tmp_1 = filter[(sx + 1 * alpha) >> WARPEDDIFF_PREC_BITS]; + __m128i tmp_3 = filter[(sx + 3 * alpha) >> WARPEDDIFF_PREC_BITS]; + __m128i tmp_5 = filter[(sx + 5 * alpha) >> WARPEDDIFF_PREC_BITS]; + __m128i tmp_7 = filter[(sx + 7 * alpha) >> WARPEDDIFF_PREC_BITS]; + + __m128i tmp_9 = _mm_unpacklo_epi32(tmp_1, tmp_3); + __m128i tmp_11 = _mm_unpacklo_epi32(tmp_5, tmp_7); + __m128i tmp_13 = _mm_unpackhi_epi32(tmp_1, tmp_3); + __m128i tmp_15 = _mm_unpackhi_epi32(tmp_5, tmp_7); + + __m128i coeff_1 = _mm_unpacklo_epi64(tmp_9, tmp_11); + __m128i coeff_3 = _mm_unpackhi_epi64(tmp_9, tmp_11); + __m128i coeff_5 = _mm_unpacklo_epi64(tmp_13, tmp_15); + __m128i coeff_7 = _mm_unpackhi_epi64(tmp_13, tmp_15); + + __m128i res_1 = + _mm_madd_epi16(_mm_alignr_epi8(src2, src, 2), coeff_1); + __m128i res_3 = + _mm_madd_epi16(_mm_alignr_epi8(src2, src, 6), coeff_3); + __m128i res_5 = + _mm_madd_epi16(_mm_alignr_epi8(src2, src, 10), coeff_5); + __m128i res_7 = + _mm_madd_epi16(_mm_alignr_epi8(src2, src, 14), coeff_7); + + __m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_5), + _mm_add_epi32(res_3, res_7)); + res_odd = _mm_srai_epi32(_mm_add_epi32(res_odd, round_const), + HORSHEAR_REDUCE_PREC_BITS); + + // Combine results into one register. + // We store the columns in the order 0, 2, 4, 6, 1, 3, 5, 7 + // as this order helps with the vertical filter. + tmp[k + 7] = _mm_packs_epi32(res_even, res_odd); + } + } + + // Vertical filter + for (k = -4; k < AOMMIN(4, p_height - i - 4); ++k) { + int sy = sy4 + gamma * (-4) + delta * k + + (1 << (WARPEDDIFF_PREC_BITS - 1)) + + (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); + + // Load from tmp and rearrange pairs of consecutive rows into the + // column order 0 0 2 2 4 4 6 6; 1 1 3 3 5 5 7 7 + __m128i *src = tmp + (k + 4); + __m128i src_0 = _mm_unpacklo_epi16(src[0], src[1]); + __m128i src_2 = _mm_unpacklo_epi16(src[2], src[3]); + __m128i src_4 = _mm_unpacklo_epi16(src[4], src[5]); + __m128i src_6 = _mm_unpacklo_epi16(src[6], src[7]); + + // Filter even-index pixels + __m128i tmp_0 = filter[(sy + 0 * gamma) >> WARPEDDIFF_PREC_BITS]; + __m128i tmp_2 = filter[(sy + 2 * gamma) >> WARPEDDIFF_PREC_BITS]; + __m128i tmp_4 = filter[(sy + 4 * gamma) >> WARPEDDIFF_PREC_BITS]; + __m128i tmp_6 = filter[(sy + 6 * gamma) >> WARPEDDIFF_PREC_BITS]; + + __m128i tmp_8 = _mm_unpacklo_epi32(tmp_0, tmp_2); + __m128i tmp_10 = _mm_unpacklo_epi32(tmp_4, tmp_6); + __m128i tmp_12 = _mm_unpackhi_epi32(tmp_0, tmp_2); + __m128i tmp_14 = _mm_unpackhi_epi32(tmp_4, tmp_6); + + __m128i coeff_0 = _mm_unpacklo_epi64(tmp_8, tmp_10); + __m128i coeff_2 = _mm_unpackhi_epi64(tmp_8, tmp_10); + __m128i coeff_4 = _mm_unpacklo_epi64(tmp_12, tmp_14); + __m128i coeff_6 = _mm_unpackhi_epi64(tmp_12, tmp_14); + + __m128i res_0 = _mm_madd_epi16(src_0, coeff_0); + __m128i res_2 = _mm_madd_epi16(src_2, coeff_2); + __m128i res_4 = _mm_madd_epi16(src_4, coeff_4); + __m128i res_6 = _mm_madd_epi16(src_6, coeff_6); + + __m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_2), + _mm_add_epi32(res_4, res_6)); + + // Filter odd-index pixels + __m128i src_1 = _mm_unpackhi_epi16(src[0], src[1]); + __m128i src_3 = _mm_unpackhi_epi16(src[2], src[3]); + __m128i src_5 = _mm_unpackhi_epi16(src[4], src[5]); + __m128i src_7 = _mm_unpackhi_epi16(src[6], src[7]); + + __m128i tmp_1 = filter[(sy + 1 * gamma) >> WARPEDDIFF_PREC_BITS]; + __m128i tmp_3 = filter[(sy + 3 * gamma) >> WARPEDDIFF_PREC_BITS]; + __m128i tmp_5 = filter[(sy + 5 * gamma) >> WARPEDDIFF_PREC_BITS]; + __m128i tmp_7 = filter[(sy + 7 * gamma) >> WARPEDDIFF_PREC_BITS]; + + __m128i tmp_9 = _mm_unpacklo_epi32(tmp_1, tmp_3); + __m128i tmp_11 = _mm_unpacklo_epi32(tmp_5, tmp_7); + __m128i tmp_13 = _mm_unpackhi_epi32(tmp_1, tmp_3); + __m128i tmp_15 = _mm_unpackhi_epi32(tmp_5, tmp_7); + + __m128i coeff_1 = _mm_unpacklo_epi64(tmp_9, tmp_11); + __m128i coeff_3 = _mm_unpackhi_epi64(tmp_9, tmp_11); + __m128i coeff_5 = _mm_unpacklo_epi64(tmp_13, tmp_15); + __m128i coeff_7 = _mm_unpackhi_epi64(tmp_13, tmp_15); + + __m128i res_1 = _mm_madd_epi16(src_1, coeff_1); + __m128i res_3 = _mm_madd_epi16(src_3, coeff_3); + __m128i res_5 = _mm_madd_epi16(src_5, coeff_5); + __m128i res_7 = _mm_madd_epi16(src_7, coeff_7); + + __m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_3), + _mm_add_epi32(res_5, res_7)); + + // Rearrange pixels back into the order 0 ... 7 + __m128i res_lo = _mm_unpacklo_epi32(res_even, res_odd); + __m128i res_hi = _mm_unpackhi_epi32(res_even, res_odd); + + // Round and pack into 8 bits + __m128i round_const = + _mm_set1_epi32((1 << VERSHEAR_REDUCE_PREC_BITS) >> 1); + + __m128i res_lo_round = _mm_srai_epi32( + _mm_add_epi32(res_lo, round_const), VERSHEAR_REDUCE_PREC_BITS); + __m128i res_hi_round = _mm_srai_epi32( + _mm_add_epi32(res_hi, round_const), VERSHEAR_REDUCE_PREC_BITS); + + __m128i res_16bit = _mm_packs_epi32(res_lo_round, res_hi_round); + // Clamp res_16bit to the range [0, 2^bd - 1] + __m128i max_val = _mm_set1_epi16((1 << bd) - 1); + __m128i zero = _mm_setzero_si128(); + res_16bit = _mm_max_epi16(_mm_min_epi16(res_16bit, max_val), zero); + + // Store, blending with 'pred' if needed + __m128i *p = (__m128i *)&pred[(i + k + 4) * p_stride + j]; + + // Note: If we're outputting a 4x4 block, we need to be very careful + // to only output 4 pixels at this point, to avoid encode/decode + // mismatches when encoding with multiple threads. + if (p_width == 4) { + if (ref_frm) res_16bit = _mm_avg_epu16(res_16bit, _mm_loadl_epi64(p)); + _mm_storel_epi64(p, res_16bit); + } else { + if (ref_frm) res_16bit = _mm_avg_epu16(res_16bit, _mm_loadu_si128(p)); + _mm_storeu_si128(p, res_16bit); + } + } + } + } +} diff --git a/third_party/aom/av1/common/x86/hybrid_inv_txfm_avx2.c b/third_party/aom/av1/common/x86/hybrid_inv_txfm_avx2.c new file mode 100644 index 0000000000..efc8d1e240 --- /dev/null +++ b/third_party/aom/av1/common/x86/hybrid_inv_txfm_avx2.c @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // avx2 + +#include "./aom_config.h" +#include "./av1_rtcd.h" + +#include "aom_dsp/x86/txfm_common_avx2.h" + +static INLINE void load_coeff(const tran_low_t *coeff, __m256i *in) { +#if CONFIG_HIGHBITDEPTH + *in = _mm256_setr_epi16( + (int16_t)coeff[0], (int16_t)coeff[1], (int16_t)coeff[2], + (int16_t)coeff[3], (int16_t)coeff[4], (int16_t)coeff[5], + (int16_t)coeff[6], (int16_t)coeff[7], (int16_t)coeff[8], + (int16_t)coeff[9], (int16_t)coeff[10], (int16_t)coeff[11], + (int16_t)coeff[12], (int16_t)coeff[13], (int16_t)coeff[14], + (int16_t)coeff[15]); +#else + *in = _mm256_loadu_si256((const __m256i *)coeff); +#endif +} + +static void load_buffer_16x16(const tran_low_t *coeff, __m256i *in) { + int i = 0; + while (i < 16) { + load_coeff(coeff + (i << 4), &in[i]); + i += 1; + } +} + +static void recon_and_store(const __m256i *res, uint8_t *output) { + const __m128i zero = _mm_setzero_si128(); + __m128i x = _mm_loadu_si128((__m128i const *)output); + __m128i p0 = _mm_unpacklo_epi8(x, zero); + __m128i p1 = _mm_unpackhi_epi8(x, zero); + + p0 = _mm_add_epi16(p0, _mm256_castsi256_si128(*res)); + p1 = _mm_add_epi16(p1, _mm256_extractf128_si256(*res, 1)); + x = _mm_packus_epi16(p0, p1); + _mm_storeu_si128((__m128i *)output, x); +} + +#define IDCT_ROUNDING_POS (6) + +static void write_buffer_16x16(__m256i *in, const int stride, uint8_t *output) { + const __m256i rounding = _mm256_set1_epi16(1 << (IDCT_ROUNDING_POS - 1)); + int i = 0; + + while (i < 16) { + in[i] = _mm256_add_epi16(in[i], rounding); + in[i] = _mm256_srai_epi16(in[i], IDCT_ROUNDING_POS); + recon_and_store(&in[i], output + i * stride); + i += 1; + } +} + +static INLINE void unpack_butter_fly(const __m256i *a0, const __m256i *a1, + const __m256i *c0, const __m256i *c1, + __m256i *b0, __m256i *b1) { + __m256i x0, x1; + x0 = _mm256_unpacklo_epi16(*a0, *a1); + x1 = _mm256_unpackhi_epi16(*a0, *a1); + *b0 = butter_fly(x0, x1, *c0); + *b1 = butter_fly(x0, x1, *c1); +} + +static void idct16_avx2(__m256i *in) { + const __m256i cospi_p30_m02 = pair256_set_epi16(cospi_30_64, -cospi_2_64); + const __m256i cospi_p02_p30 = pair256_set_epi16(cospi_2_64, cospi_30_64); + const __m256i cospi_p14_m18 = pair256_set_epi16(cospi_14_64, -cospi_18_64); + const __m256i cospi_p18_p14 = pair256_set_epi16(cospi_18_64, cospi_14_64); + const __m256i cospi_p22_m10 = pair256_set_epi16(cospi_22_64, -cospi_10_64); + const __m256i cospi_p10_p22 = pair256_set_epi16(cospi_10_64, cospi_22_64); + const __m256i cospi_p06_m26 = pair256_set_epi16(cospi_6_64, -cospi_26_64); + const __m256i cospi_p26_p06 = pair256_set_epi16(cospi_26_64, cospi_6_64); + const __m256i cospi_p28_m04 = pair256_set_epi16(cospi_28_64, -cospi_4_64); + const __m256i cospi_p04_p28 = pair256_set_epi16(cospi_4_64, cospi_28_64); + const __m256i cospi_p12_m20 = pair256_set_epi16(cospi_12_64, -cospi_20_64); + const __m256i cospi_p20_p12 = pair256_set_epi16(cospi_20_64, cospi_12_64); + const __m256i cospi_p16_p16 = _mm256_set1_epi16((int16_t)cospi_16_64); + const __m256i cospi_p16_m16 = pair256_set_epi16(cospi_16_64, -cospi_16_64); + const __m256i cospi_p24_m08 = pair256_set_epi16(cospi_24_64, -cospi_8_64); + const __m256i cospi_p08_p24 = pair256_set_epi16(cospi_8_64, cospi_24_64); + const __m256i cospi_m08_p24 = pair256_set_epi16(-cospi_8_64, cospi_24_64); + const __m256i cospi_p24_p08 = pair256_set_epi16(cospi_24_64, cospi_8_64); + const __m256i cospi_m24_m08 = pair256_set_epi16(-cospi_24_64, -cospi_8_64); + __m256i u0, u1, u2, u3, u4, u5, u6, u7; + __m256i v0, v1, v2, v3, v4, v5, v6, v7; + __m256i t0, t1, t2, t3, t4, t5, t6, t7; + + // stage 1, (0-7) + u0 = in[0]; + u1 = in[8]; + u2 = in[4]; + u3 = in[12]; + u4 = in[2]; + u5 = in[10]; + u6 = in[6]; + u7 = in[14]; + + // stage 2, (0-7) + // stage 3, (0-7) + t0 = u0; + t1 = u1; + t2 = u2; + t3 = u3; + unpack_butter_fly(&u4, &u7, &cospi_p28_m04, &cospi_p04_p28, &t4, &t7); + unpack_butter_fly(&u5, &u6, &cospi_p12_m20, &cospi_p20_p12, &t5, &t6); + + // stage 4, (0-7) + unpack_butter_fly(&t0, &t1, &cospi_p16_p16, &cospi_p16_m16, &u0, &u1); + unpack_butter_fly(&t2, &t3, &cospi_p24_m08, &cospi_p08_p24, &u2, &u3); + u4 = _mm256_add_epi16(t4, t5); + u5 = _mm256_sub_epi16(t4, t5); + u6 = _mm256_sub_epi16(t7, t6); + u7 = _mm256_add_epi16(t7, t6); + + // stage 5, (0-7) + t0 = _mm256_add_epi16(u0, u3); + t1 = _mm256_add_epi16(u1, u2); + t2 = _mm256_sub_epi16(u1, u2); + t3 = _mm256_sub_epi16(u0, u3); + t4 = u4; + t7 = u7; + unpack_butter_fly(&u6, &u5, &cospi_p16_m16, &cospi_p16_p16, &t5, &t6); + + // stage 6, (0-7) + u0 = _mm256_add_epi16(t0, t7); + u1 = _mm256_add_epi16(t1, t6); + u2 = _mm256_add_epi16(t2, t5); + u3 = _mm256_add_epi16(t3, t4); + u4 = _mm256_sub_epi16(t3, t4); + u5 = _mm256_sub_epi16(t2, t5); + u6 = _mm256_sub_epi16(t1, t6); + u7 = _mm256_sub_epi16(t0, t7); + + // stage 1, (8-15) + v0 = in[1]; + v1 = in[9]; + v2 = in[5]; + v3 = in[13]; + v4 = in[3]; + v5 = in[11]; + v6 = in[7]; + v7 = in[15]; + + // stage 2, (8-15) + unpack_butter_fly(&v0, &v7, &cospi_p30_m02, &cospi_p02_p30, &t0, &t7); + unpack_butter_fly(&v1, &v6, &cospi_p14_m18, &cospi_p18_p14, &t1, &t6); + unpack_butter_fly(&v2, &v5, &cospi_p22_m10, &cospi_p10_p22, &t2, &t5); + unpack_butter_fly(&v3, &v4, &cospi_p06_m26, &cospi_p26_p06, &t3, &t4); + + // stage 3, (8-15) + v0 = _mm256_add_epi16(t0, t1); + v1 = _mm256_sub_epi16(t0, t1); + v2 = _mm256_sub_epi16(t3, t2); + v3 = _mm256_add_epi16(t2, t3); + v4 = _mm256_add_epi16(t4, t5); + v5 = _mm256_sub_epi16(t4, t5); + v6 = _mm256_sub_epi16(t7, t6); + v7 = _mm256_add_epi16(t6, t7); + + // stage 4, (8-15) + t0 = v0; + t7 = v7; + t3 = v3; + t4 = v4; + unpack_butter_fly(&v1, &v6, &cospi_m08_p24, &cospi_p24_p08, &t1, &t6); + unpack_butter_fly(&v2, &v5, &cospi_m24_m08, &cospi_m08_p24, &t2, &t5); + + // stage 5, (8-15) + v0 = _mm256_add_epi16(t0, t3); + v1 = _mm256_add_epi16(t1, t2); + v2 = _mm256_sub_epi16(t1, t2); + v3 = _mm256_sub_epi16(t0, t3); + v4 = _mm256_sub_epi16(t7, t4); + v5 = _mm256_sub_epi16(t6, t5); + v6 = _mm256_add_epi16(t6, t5); + v7 = _mm256_add_epi16(t7, t4); + + // stage 6, (8-15) + t0 = v0; + t1 = v1; + t6 = v6; + t7 = v7; + unpack_butter_fly(&v5, &v2, &cospi_p16_m16, &cospi_p16_p16, &t2, &t5); + unpack_butter_fly(&v4, &v3, &cospi_p16_m16, &cospi_p16_p16, &t3, &t4); + + // stage 7 + in[0] = _mm256_add_epi16(u0, t7); + in[1] = _mm256_add_epi16(u1, t6); + in[2] = _mm256_add_epi16(u2, t5); + in[3] = _mm256_add_epi16(u3, t4); + in[4] = _mm256_add_epi16(u4, t3); + in[5] = _mm256_add_epi16(u5, t2); + in[6] = _mm256_add_epi16(u6, t1); + in[7] = _mm256_add_epi16(u7, t0); + in[8] = _mm256_sub_epi16(u7, t0); + in[9] = _mm256_sub_epi16(u6, t1); + in[10] = _mm256_sub_epi16(u5, t2); + in[11] = _mm256_sub_epi16(u4, t3); + in[12] = _mm256_sub_epi16(u3, t4); + in[13] = _mm256_sub_epi16(u2, t5); + in[14] = _mm256_sub_epi16(u1, t6); + in[15] = _mm256_sub_epi16(u0, t7); +} + +static void idct16(__m256i *in) { + mm256_transpose_16x16(in); + idct16_avx2(in); +} + +static INLINE void butterfly_32b(const __m256i *a0, const __m256i *a1, + const __m256i *c0, const __m256i *c1, + __m256i *b) { + __m256i x0, x1; + x0 = _mm256_unpacklo_epi16(*a0, *a1); + x1 = _mm256_unpackhi_epi16(*a0, *a1); + b[0] = _mm256_madd_epi16(x0, *c0); + b[1] = _mm256_madd_epi16(x1, *c0); + b[2] = _mm256_madd_epi16(x0, *c1); + b[3] = _mm256_madd_epi16(x1, *c1); +} + +static INLINE void group_rounding(__m256i *a, int num) { + const __m256i dct_rounding = _mm256_set1_epi32(DCT_CONST_ROUNDING); + int i; + for (i = 0; i < num; ++i) { + a[i] = _mm256_add_epi32(a[i], dct_rounding); + a[i] = _mm256_srai_epi32(a[i], DCT_CONST_BITS); + } +} + +static INLINE void add_rnd(const __m256i *a, const __m256i *b, __m256i *out) { + __m256i x[4]; + x[0] = _mm256_add_epi32(a[0], b[0]); + x[1] = _mm256_add_epi32(a[1], b[1]); + x[2] = _mm256_add_epi32(a[2], b[2]); + x[3] = _mm256_add_epi32(a[3], b[3]); + + group_rounding(x, 4); + + out[0] = _mm256_packs_epi32(x[0], x[1]); + out[1] = _mm256_packs_epi32(x[2], x[3]); +} + +static INLINE void sub_rnd(const __m256i *a, const __m256i *b, __m256i *out) { + __m256i x[4]; + x[0] = _mm256_sub_epi32(a[0], b[0]); + x[1] = _mm256_sub_epi32(a[1], b[1]); + x[2] = _mm256_sub_epi32(a[2], b[2]); + x[3] = _mm256_sub_epi32(a[3], b[3]); + + group_rounding(x, 4); + + out[0] = _mm256_packs_epi32(x[0], x[1]); + out[1] = _mm256_packs_epi32(x[2], x[3]); +} + +static INLINE void butterfly_rnd(__m256i *a, __m256i *out) { + group_rounding(a, 4); + out[0] = _mm256_packs_epi32(a[0], a[1]); + out[1] = _mm256_packs_epi32(a[2], a[3]); +} + +static void iadst16_avx2(__m256i *in) { + const __m256i cospi_p01_p31 = pair256_set_epi16(cospi_1_64, cospi_31_64); + const __m256i cospi_p31_m01 = pair256_set_epi16(cospi_31_64, -cospi_1_64); + const __m256i cospi_p05_p27 = pair256_set_epi16(cospi_5_64, cospi_27_64); + const __m256i cospi_p27_m05 = pair256_set_epi16(cospi_27_64, -cospi_5_64); + const __m256i cospi_p09_p23 = pair256_set_epi16(cospi_9_64, cospi_23_64); + const __m256i cospi_p23_m09 = pair256_set_epi16(cospi_23_64, -cospi_9_64); + const __m256i cospi_p13_p19 = pair256_set_epi16(cospi_13_64, cospi_19_64); + const __m256i cospi_p19_m13 = pair256_set_epi16(cospi_19_64, -cospi_13_64); + const __m256i cospi_p17_p15 = pair256_set_epi16(cospi_17_64, cospi_15_64); + const __m256i cospi_p15_m17 = pair256_set_epi16(cospi_15_64, -cospi_17_64); + const __m256i cospi_p21_p11 = pair256_set_epi16(cospi_21_64, cospi_11_64); + const __m256i cospi_p11_m21 = pair256_set_epi16(cospi_11_64, -cospi_21_64); + const __m256i cospi_p25_p07 = pair256_set_epi16(cospi_25_64, cospi_7_64); + const __m256i cospi_p07_m25 = pair256_set_epi16(cospi_7_64, -cospi_25_64); + const __m256i cospi_p29_p03 = pair256_set_epi16(cospi_29_64, cospi_3_64); + const __m256i cospi_p03_m29 = pair256_set_epi16(cospi_3_64, -cospi_29_64); + const __m256i cospi_p04_p28 = pair256_set_epi16(cospi_4_64, cospi_28_64); + const __m256i cospi_p28_m04 = pair256_set_epi16(cospi_28_64, -cospi_4_64); + const __m256i cospi_p20_p12 = pair256_set_epi16(cospi_20_64, cospi_12_64); + const __m256i cospi_p12_m20 = pair256_set_epi16(cospi_12_64, -cospi_20_64); + const __m256i cospi_m28_p04 = pair256_set_epi16(-cospi_28_64, cospi_4_64); + const __m256i cospi_m12_p20 = pair256_set_epi16(-cospi_12_64, cospi_20_64); + const __m256i cospi_p08_p24 = pair256_set_epi16(cospi_8_64, cospi_24_64); + const __m256i cospi_p24_m08 = pair256_set_epi16(cospi_24_64, -cospi_8_64); + const __m256i cospi_m24_p08 = pair256_set_epi16(-cospi_24_64, cospi_8_64); + const __m256i cospi_m16_m16 = _mm256_set1_epi16((int16_t)-cospi_16_64); + const __m256i cospi_p16_p16 = _mm256_set1_epi16((int16_t)cospi_16_64); + const __m256i cospi_p16_m16 = pair256_set_epi16(cospi_16_64, -cospi_16_64); + const __m256i cospi_m16_p16 = pair256_set_epi16(-cospi_16_64, cospi_16_64); + const __m256i zero = _mm256_setzero_si256(); + __m256i x[16], s[16]; + __m256i u[4], v[4]; + + // stage 1 + butterfly_32b(&in[15], &in[0], &cospi_p01_p31, &cospi_p31_m01, u); + butterfly_32b(&in[7], &in[8], &cospi_p17_p15, &cospi_p15_m17, v); + add_rnd(u, v, &x[0]); + sub_rnd(u, v, &x[8]); + + butterfly_32b(&in[13], &in[2], &cospi_p05_p27, &cospi_p27_m05, u); + butterfly_32b(&in[5], &in[10], &cospi_p21_p11, &cospi_p11_m21, v); + add_rnd(u, v, &x[2]); + sub_rnd(u, v, &x[10]); + + butterfly_32b(&in[11], &in[4], &cospi_p09_p23, &cospi_p23_m09, u); + butterfly_32b(&in[3], &in[12], &cospi_p25_p07, &cospi_p07_m25, v); + add_rnd(u, v, &x[4]); + sub_rnd(u, v, &x[12]); + + butterfly_32b(&in[9], &in[6], &cospi_p13_p19, &cospi_p19_m13, u); + butterfly_32b(&in[1], &in[14], &cospi_p29_p03, &cospi_p03_m29, v); + add_rnd(u, v, &x[6]); + sub_rnd(u, v, &x[14]); + + // stage 2 + s[0] = _mm256_add_epi16(x[0], x[4]); + s[1] = _mm256_add_epi16(x[1], x[5]); + s[2] = _mm256_add_epi16(x[2], x[6]); + s[3] = _mm256_add_epi16(x[3], x[7]); + s[4] = _mm256_sub_epi16(x[0], x[4]); + s[5] = _mm256_sub_epi16(x[1], x[5]); + s[6] = _mm256_sub_epi16(x[2], x[6]); + s[7] = _mm256_sub_epi16(x[3], x[7]); + butterfly_32b(&x[8], &x[9], &cospi_p04_p28, &cospi_p28_m04, u); + butterfly_32b(&x[12], &x[13], &cospi_m28_p04, &cospi_p04_p28, v); + add_rnd(u, v, &s[8]); + sub_rnd(u, v, &s[12]); + + butterfly_32b(&x[10], &x[11], &cospi_p20_p12, &cospi_p12_m20, u); + butterfly_32b(&x[14], &x[15], &cospi_m12_p20, &cospi_p20_p12, v); + add_rnd(u, v, &s[10]); + sub_rnd(u, v, &s[14]); + + // stage 3 + x[0] = _mm256_add_epi16(s[0], s[2]); + x[1] = _mm256_add_epi16(s[1], s[3]); + x[2] = _mm256_sub_epi16(s[0], s[2]); + x[3] = _mm256_sub_epi16(s[1], s[3]); + + x[8] = _mm256_add_epi16(s[8], s[10]); + x[9] = _mm256_add_epi16(s[9], s[11]); + x[10] = _mm256_sub_epi16(s[8], s[10]); + x[11] = _mm256_sub_epi16(s[9], s[11]); + + butterfly_32b(&s[4], &s[5], &cospi_p08_p24, &cospi_p24_m08, u); + butterfly_32b(&s[6], &s[7], &cospi_m24_p08, &cospi_p08_p24, v); + add_rnd(u, v, &x[4]); + sub_rnd(u, v, &x[6]); + + butterfly_32b(&s[12], &s[13], &cospi_p08_p24, &cospi_p24_m08, u); + butterfly_32b(&s[14], &s[15], &cospi_m24_p08, &cospi_p08_p24, v); + add_rnd(u, v, &x[12]); + sub_rnd(u, v, &x[14]); + + // stage 4 + butterfly_32b(&x[2], &x[3], &cospi_m16_m16, &cospi_p16_m16, u); + butterfly_32b(&x[6], &x[7], &cospi_p16_p16, &cospi_m16_p16, v); + butterfly_rnd(u, &x[2]); + butterfly_rnd(v, &x[6]); + + butterfly_32b(&x[10], &x[11], &cospi_p16_p16, &cospi_m16_p16, u); + butterfly_32b(&x[14], &x[15], &cospi_m16_m16, &cospi_p16_m16, v); + butterfly_rnd(u, &x[10]); + butterfly_rnd(v, &x[14]); + + in[0] = x[0]; + in[1] = _mm256_sub_epi16(zero, x[8]); + in[2] = x[12]; + in[3] = _mm256_sub_epi16(zero, x[4]); + in[4] = x[6]; + in[5] = x[14]; + in[6] = x[10]; + in[7] = x[2]; + in[8] = x[3]; + in[9] = x[11]; + in[10] = x[15]; + in[11] = x[7]; + in[12] = x[5]; + in[13] = _mm256_sub_epi16(zero, x[13]); + in[14] = x[9]; + in[15] = _mm256_sub_epi16(zero, x[1]); +} + +static void iadst16(__m256i *in) { + mm256_transpose_16x16(in); + iadst16_avx2(in); +} + +#if CONFIG_EXT_TX +static void flip_row(__m256i *in, int rows) { + int i; + for (i = 0; i < rows; ++i) { + mm256_reverse_epi16(&in[i]); + } +} + +static void flip_col(uint8_t **dest, int *stride, int rows) { + *dest = *dest + (rows - 1) * (*stride); + *stride = -*stride; +} + +static void iidtx16(__m256i *in) { + mm256_transpose_16x16(in); + txfm_scaling16_avx2(Sqrt2, in); +} +#endif + +void av1_iht16x16_256_add_avx2(const tran_low_t *input, uint8_t *dest, + int stride, int tx_type) { + __m256i in[16]; + + load_buffer_16x16(input, in); + switch (tx_type) { + case DCT_DCT: + idct16(in); + idct16(in); + break; + case ADST_DCT: + idct16(in); + iadst16(in); + break; + case DCT_ADST: + iadst16(in); + idct16(in); + break; + case ADST_ADST: + iadst16(in); + iadst16(in); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + idct16(in); + iadst16(in); + flip_col(&dest, &stride, 16); + break; + case DCT_FLIPADST: + iadst16(in); + idct16(in); + flip_row(in, 16); + break; + case FLIPADST_FLIPADST: + iadst16(in); + iadst16(in); + flip_row(in, 16); + flip_col(&dest, &stride, 16); + break; + case ADST_FLIPADST: + iadst16(in); + iadst16(in); + flip_row(in, 16); + break; + case FLIPADST_ADST: + iadst16(in); + iadst16(in); + flip_col(&dest, &stride, 16); + break; + case IDTX: + iidtx16(in); + iidtx16(in); + break; + case V_DCT: + iidtx16(in); + idct16(in); + break; + case H_DCT: + idct16(in); + iidtx16(in); + break; + case V_ADST: + iidtx16(in); + iadst16(in); + break; + case H_ADST: + iadst16(in); + iidtx16(in); + break; + case V_FLIPADST: + iidtx16(in); + iadst16(in); + flip_col(&dest, &stride, 16); + break; + case H_FLIPADST: + iadst16(in); + iidtx16(in); + flip_row(in, 16); + break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } + write_buffer_16x16(in, stride, dest); +} diff --git a/third_party/aom/av1/common/x86/idct_intrin_sse2.c b/third_party/aom/av1/common/x86/idct_intrin_sse2.c new file mode 100644 index 0000000000..522e8988c3 --- /dev/null +++ b/third_party/aom/av1/common/x86/idct_intrin_sse2.c @@ -0,0 +1,1402 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./av1_rtcd.h" +#include "aom_dsp/x86/inv_txfm_sse2.h" +#include "aom_dsp/x86/synonyms.h" +#include "aom_dsp/x86/txfm_common_sse2.h" +#include "aom_ports/mem.h" +#include "av1/common/enums.h" + +#if CONFIG_EXT_TX +static INLINE void fliplr_4x4(__m128i in[2]) { + in[0] = _mm_shufflelo_epi16(in[0], 0x1b); + in[0] = _mm_shufflehi_epi16(in[0], 0x1b); + in[1] = _mm_shufflelo_epi16(in[1], 0x1b); + in[1] = _mm_shufflehi_epi16(in[1], 0x1b); +} + +static INLINE void fliplr_8x8(__m128i in[8]) { + in[0] = mm_reverse_epi16(in[0]); + in[1] = mm_reverse_epi16(in[1]); + in[2] = mm_reverse_epi16(in[2]); + in[3] = mm_reverse_epi16(in[3]); + + in[4] = mm_reverse_epi16(in[4]); + in[5] = mm_reverse_epi16(in[5]); + in[6] = mm_reverse_epi16(in[6]); + in[7] = mm_reverse_epi16(in[7]); +} + +static INLINE void fliplr_16x8(__m128i in[16]) { + fliplr_8x8(&in[0]); + fliplr_8x8(&in[8]); +} + +#define FLIPLR_16x16(in0, in1) \ + do { \ + __m128i *tmp; \ + fliplr_16x8(in0); \ + fliplr_16x8(in1); \ + tmp = (in0); \ + (in0) = (in1); \ + (in1) = tmp; \ + } while (0) + +#define FLIPUD_PTR(dest, stride, size) \ + do { \ + (dest) = (dest) + ((size)-1) * (stride); \ + (stride) = -(stride); \ + } while (0) +#endif + +void av1_iht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + __m128i in[2]; + const __m128i zero = _mm_setzero_si128(); + const __m128i eight = _mm_set1_epi16(8); + + in[0] = load_input_data(input); + in[1] = load_input_data(input + 8); + + switch (tx_type) { + case DCT_DCT: + aom_idct4_sse2(in); + aom_idct4_sse2(in); + break; + case ADST_DCT: + aom_idct4_sse2(in); + aom_iadst4_sse2(in); + break; + case DCT_ADST: + aom_iadst4_sse2(in); + aom_idct4_sse2(in); + break; + case ADST_ADST: + aom_iadst4_sse2(in); + aom_iadst4_sse2(in); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + aom_idct4_sse2(in); + aom_iadst4_sse2(in); + FLIPUD_PTR(dest, stride, 4); + break; + case DCT_FLIPADST: + aom_iadst4_sse2(in); + aom_idct4_sse2(in); + fliplr_4x4(in); + break; + case FLIPADST_FLIPADST: + aom_iadst4_sse2(in); + aom_iadst4_sse2(in); + FLIPUD_PTR(dest, stride, 4); + fliplr_4x4(in); + break; + case ADST_FLIPADST: + aom_iadst4_sse2(in); + aom_iadst4_sse2(in); + fliplr_4x4(in); + break; + case FLIPADST_ADST: + aom_iadst4_sse2(in); + aom_iadst4_sse2(in); + FLIPUD_PTR(dest, stride, 4); + break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } + + // Final round and shift + in[0] = _mm_add_epi16(in[0], eight); + in[1] = _mm_add_epi16(in[1], eight); + + in[0] = _mm_srai_epi16(in[0], 4); + in[1] = _mm_srai_epi16(in[1], 4); + + // Reconstruction and Store + { + __m128i d0 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 0)); + __m128i d1 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 1)); + __m128i d2 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 2)); + __m128i d3 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 3)); + d0 = _mm_unpacklo_epi32(d0, d1); + d2 = _mm_unpacklo_epi32(d2, d3); + d0 = _mm_unpacklo_epi8(d0, zero); + d2 = _mm_unpacklo_epi8(d2, zero); + d0 = _mm_add_epi16(d0, in[0]); + d2 = _mm_add_epi16(d2, in[1]); + d0 = _mm_packus_epi16(d0, d2); + // store result[0] + *(int *)dest = _mm_cvtsi128_si32(d0); + // store result[1] + d0 = _mm_srli_si128(d0, 4); + *(int *)(dest + stride) = _mm_cvtsi128_si32(d0); + // store result[2] + d0 = _mm_srli_si128(d0, 4); + *(int *)(dest + stride * 2) = _mm_cvtsi128_si32(d0); + // store result[3] + d0 = _mm_srli_si128(d0, 4); + *(int *)(dest + stride * 3) = _mm_cvtsi128_si32(d0); + } +} + +void av1_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + __m128i in[8]; + const __m128i zero = _mm_setzero_si128(); + const __m128i final_rounding = _mm_set1_epi16(1 << 4); + + // load input data + in[0] = load_input_data(input); + in[1] = load_input_data(input + 8 * 1); + in[2] = load_input_data(input + 8 * 2); + in[3] = load_input_data(input + 8 * 3); + in[4] = load_input_data(input + 8 * 4); + in[5] = load_input_data(input + 8 * 5); + in[6] = load_input_data(input + 8 * 6); + in[7] = load_input_data(input + 8 * 7); + + switch (tx_type) { + case DCT_DCT: + aom_idct8_sse2(in); + aom_idct8_sse2(in); + break; + case ADST_DCT: + aom_idct8_sse2(in); + aom_iadst8_sse2(in); + break; + case DCT_ADST: + aom_iadst8_sse2(in); + aom_idct8_sse2(in); + break; + case ADST_ADST: + aom_iadst8_sse2(in); + aom_iadst8_sse2(in); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + aom_idct8_sse2(in); + aom_iadst8_sse2(in); + FLIPUD_PTR(dest, stride, 8); + break; + case DCT_FLIPADST: + aom_iadst8_sse2(in); + aom_idct8_sse2(in); + fliplr_8x8(in); + break; + case FLIPADST_FLIPADST: + aom_iadst8_sse2(in); + aom_iadst8_sse2(in); + FLIPUD_PTR(dest, stride, 8); + fliplr_8x8(in); + break; + case ADST_FLIPADST: + aom_iadst8_sse2(in); + aom_iadst8_sse2(in); + fliplr_8x8(in); + break; + case FLIPADST_ADST: + aom_iadst8_sse2(in); + aom_iadst8_sse2(in); + FLIPUD_PTR(dest, stride, 8); + break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } + + // Final rounding and shift + in[0] = _mm_adds_epi16(in[0], final_rounding); + in[1] = _mm_adds_epi16(in[1], final_rounding); + in[2] = _mm_adds_epi16(in[2], final_rounding); + in[3] = _mm_adds_epi16(in[3], final_rounding); + in[4] = _mm_adds_epi16(in[4], final_rounding); + in[5] = _mm_adds_epi16(in[5], final_rounding); + in[6] = _mm_adds_epi16(in[6], final_rounding); + in[7] = _mm_adds_epi16(in[7], final_rounding); + + in[0] = _mm_srai_epi16(in[0], 5); + in[1] = _mm_srai_epi16(in[1], 5); + in[2] = _mm_srai_epi16(in[2], 5); + in[3] = _mm_srai_epi16(in[3], 5); + in[4] = _mm_srai_epi16(in[4], 5); + in[5] = _mm_srai_epi16(in[5], 5); + in[6] = _mm_srai_epi16(in[6], 5); + in[7] = _mm_srai_epi16(in[7], 5); + + RECON_AND_STORE(dest + 0 * stride, in[0]); + RECON_AND_STORE(dest + 1 * stride, in[1]); + RECON_AND_STORE(dest + 2 * stride, in[2]); + RECON_AND_STORE(dest + 3 * stride, in[3]); + RECON_AND_STORE(dest + 4 * stride, in[4]); + RECON_AND_STORE(dest + 5 * stride, in[5]); + RECON_AND_STORE(dest + 6 * stride, in[6]); + RECON_AND_STORE(dest + 7 * stride, in[7]); +} + +#if CONFIG_EXT_TX +static void iidtx16_sse2(__m128i *in0, __m128i *in1) { + array_transpose_16x16(in0, in1); + idtx16_8col(in0); + idtx16_8col(in1); +} +#endif // CONFIG_EXT_TX + +void av1_iht16x16_256_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride, int tx_type) { + __m128i in[32]; + __m128i *in0 = &in[0]; + __m128i *in1 = &in[16]; + + load_buffer_8x16(input, in0); + input += 8; + load_buffer_8x16(input, in1); + + switch (tx_type) { + case DCT_DCT: + aom_idct16_sse2(in0, in1); + aom_idct16_sse2(in0, in1); + break; + case ADST_DCT: + aom_idct16_sse2(in0, in1); + aom_iadst16_sse2(in0, in1); + break; + case DCT_ADST: + aom_iadst16_sse2(in0, in1); + aom_idct16_sse2(in0, in1); + break; + case ADST_ADST: + aom_iadst16_sse2(in0, in1); + aom_iadst16_sse2(in0, in1); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + aom_idct16_sse2(in0, in1); + aom_iadst16_sse2(in0, in1); + FLIPUD_PTR(dest, stride, 16); + break; + case DCT_FLIPADST: + aom_iadst16_sse2(in0, in1); + aom_idct16_sse2(in0, in1); + FLIPLR_16x16(in0, in1); + break; + case FLIPADST_FLIPADST: + aom_iadst16_sse2(in0, in1); + aom_iadst16_sse2(in0, in1); + FLIPUD_PTR(dest, stride, 16); + FLIPLR_16x16(in0, in1); + break; + case ADST_FLIPADST: + aom_iadst16_sse2(in0, in1); + aom_iadst16_sse2(in0, in1); + FLIPLR_16x16(in0, in1); + break; + case FLIPADST_ADST: + aom_iadst16_sse2(in0, in1); + aom_iadst16_sse2(in0, in1); + FLIPUD_PTR(dest, stride, 16); + break; + case IDTX: + iidtx16_sse2(in0, in1); + iidtx16_sse2(in0, in1); + break; + case V_DCT: + iidtx16_sse2(in0, in1); + aom_idct16_sse2(in0, in1); + break; + case H_DCT: + aom_idct16_sse2(in0, in1); + iidtx16_sse2(in0, in1); + break; + case V_ADST: + iidtx16_sse2(in0, in1); + aom_iadst16_sse2(in0, in1); + break; + case H_ADST: + aom_iadst16_sse2(in0, in1); + iidtx16_sse2(in0, in1); + break; + case V_FLIPADST: + iidtx16_sse2(in0, in1); + aom_iadst16_sse2(in0, in1); + FLIPUD_PTR(dest, stride, 16); + break; + case H_FLIPADST: + aom_iadst16_sse2(in0, in1); + iidtx16_sse2(in0, in1); + FLIPLR_16x16(in0, in1); + break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } + + write_buffer_8x16(dest, in0, stride); + dest += 8; + write_buffer_8x16(dest, in1, stride); +} + +#if CONFIG_EXT_TX +static void iidtx8_sse2(__m128i *in) { + in[0] = _mm_slli_epi16(in[0], 1); + in[1] = _mm_slli_epi16(in[1], 1); + in[2] = _mm_slli_epi16(in[2], 1); + in[3] = _mm_slli_epi16(in[3], 1); + in[4] = _mm_slli_epi16(in[4], 1); + in[5] = _mm_slli_epi16(in[5], 1); + in[6] = _mm_slli_epi16(in[6], 1); + in[7] = _mm_slli_epi16(in[7], 1); +} + +static INLINE void iidtx4_sse2(__m128i *in) { + const __m128i v_scale_w = _mm_set1_epi16(Sqrt2); + + const __m128i v_p0l_w = _mm_mullo_epi16(in[0], v_scale_w); + const __m128i v_p0h_w = _mm_mulhi_epi16(in[0], v_scale_w); + const __m128i v_p1l_w = _mm_mullo_epi16(in[1], v_scale_w); + const __m128i v_p1h_w = _mm_mulhi_epi16(in[1], v_scale_w); + + const __m128i v_p0a_d = _mm_unpacklo_epi16(v_p0l_w, v_p0h_w); + const __m128i v_p0b_d = _mm_unpackhi_epi16(v_p0l_w, v_p0h_w); + const __m128i v_p1a_d = _mm_unpacklo_epi16(v_p1l_w, v_p1h_w); + const __m128i v_p1b_d = _mm_unpackhi_epi16(v_p1l_w, v_p1h_w); + + in[0] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p0a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p0b_d, DCT_CONST_BITS)); + in[1] = _mm_packs_epi32(xx_roundn_epi32_unsigned(v_p1a_d, DCT_CONST_BITS), + xx_roundn_epi32_unsigned(v_p1b_d, DCT_CONST_BITS)); +} + +// load 8x8 array +static INLINE void flip_buffer_lr_8x8(__m128i *in) { + in[0] = mm_reverse_epi16(in[0]); + in[1] = mm_reverse_epi16(in[1]); + in[2] = mm_reverse_epi16(in[2]); + in[3] = mm_reverse_epi16(in[3]); + in[4] = mm_reverse_epi16(in[4]); + in[5] = mm_reverse_epi16(in[5]); + in[6] = mm_reverse_epi16(in[6]); + in[7] = mm_reverse_epi16(in[7]); +} +#endif // CONFIG_EXT_TX + +void av1_iht8x16_128_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride, int tx_type) { + __m128i in[16]; + + in[0] = load_input_data(input + 0 * 8); + in[1] = load_input_data(input + 1 * 8); + in[2] = load_input_data(input + 2 * 8); + in[3] = load_input_data(input + 3 * 8); + in[4] = load_input_data(input + 4 * 8); + in[5] = load_input_data(input + 5 * 8); + in[6] = load_input_data(input + 6 * 8); + in[7] = load_input_data(input + 7 * 8); + + in[8] = load_input_data(input + 8 * 8); + in[9] = load_input_data(input + 9 * 8); + in[10] = load_input_data(input + 10 * 8); + in[11] = load_input_data(input + 11 * 8); + in[12] = load_input_data(input + 12 * 8); + in[13] = load_input_data(input + 13 * 8); + in[14] = load_input_data(input + 14 * 8); + in[15] = load_input_data(input + 15 * 8); + + // Row transform + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case H_DCT: +#endif + aom_idct8_sse2(in); + array_transpose_8x8(in, in); + aom_idct8_sse2(in + 8); + array_transpose_8x8(in + 8, in + 8); + break; + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case H_ADST: + case H_FLIPADST: +#endif + aom_iadst8_sse2(in); + array_transpose_8x8(in, in); + aom_iadst8_sse2(in + 8); + array_transpose_8x8(in + 8, in + 8); + break; +#if CONFIG_EXT_TX + case V_FLIPADST: + case V_ADST: + case V_DCT: + case IDTX: + iidtx8_sse2(in); + iidtx8_sse2(in + 8); + break; +#endif + default: assert(0); break; + } + scale_sqrt2_8x8(in); + scale_sqrt2_8x8(in + 8); + + // Column transform + switch (tx_type) { + case DCT_DCT: + case DCT_ADST: +#if CONFIG_EXT_TX + case DCT_FLIPADST: + case V_DCT: +#endif + idct16_8col(in); + break; + case ADST_DCT: + case ADST_ADST: +#if CONFIG_EXT_TX + case FLIPADST_ADST: + case ADST_FLIPADST: + case FLIPADST_FLIPADST: + case FLIPADST_DCT: + case V_ADST: + case V_FLIPADST: +#endif + iadst16_8col(in); + break; +#if CONFIG_EXT_TX + case H_DCT: + case H_ADST: + case H_FLIPADST: + case IDTX: idtx16_8col(in); break; +#endif + default: assert(0); break; + } + + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: +#if CONFIG_EXT_TX + case H_DCT: +#endif + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case H_ADST: + case V_ADST: + case V_DCT: + case IDTX: +#endif + write_buffer_8x16(dest, in, stride); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case FLIPADST_ADST: + case V_FLIPADST: write_buffer_8x16(dest + stride * 15, in, -stride); break; + case DCT_FLIPADST: + case ADST_FLIPADST: + case H_FLIPADST: + flip_buffer_lr_8x8(in); + flip_buffer_lr_8x8(in + 8); + write_buffer_8x16(dest, in, stride); + break; + case FLIPADST_FLIPADST: + flip_buffer_lr_8x8(in); + flip_buffer_lr_8x8(in + 8); + write_buffer_8x16(dest + stride * 15, in, -stride); + break; +#endif + default: assert(0); break; + } +} + +static INLINE void write_buffer_8x8_round6(uint8_t *dest, __m128i *in, + int stride) { + const __m128i final_rounding = _mm_set1_epi16(1 << 5); + const __m128i zero = _mm_setzero_si128(); + // Final rounding and shift + in[0] = _mm_adds_epi16(in[0], final_rounding); + in[1] = _mm_adds_epi16(in[1], final_rounding); + in[2] = _mm_adds_epi16(in[2], final_rounding); + in[3] = _mm_adds_epi16(in[3], final_rounding); + in[4] = _mm_adds_epi16(in[4], final_rounding); + in[5] = _mm_adds_epi16(in[5], final_rounding); + in[6] = _mm_adds_epi16(in[6], final_rounding); + in[7] = _mm_adds_epi16(in[7], final_rounding); + + in[0] = _mm_srai_epi16(in[0], 6); + in[1] = _mm_srai_epi16(in[1], 6); + in[2] = _mm_srai_epi16(in[2], 6); + in[3] = _mm_srai_epi16(in[3], 6); + in[4] = _mm_srai_epi16(in[4], 6); + in[5] = _mm_srai_epi16(in[5], 6); + in[6] = _mm_srai_epi16(in[6], 6); + in[7] = _mm_srai_epi16(in[7], 6); + + RECON_AND_STORE(dest + 0 * stride, in[0]); + RECON_AND_STORE(dest + 1 * stride, in[1]); + RECON_AND_STORE(dest + 2 * stride, in[2]); + RECON_AND_STORE(dest + 3 * stride, in[3]); + RECON_AND_STORE(dest + 4 * stride, in[4]); + RECON_AND_STORE(dest + 5 * stride, in[5]); + RECON_AND_STORE(dest + 6 * stride, in[6]); + RECON_AND_STORE(dest + 7 * stride, in[7]); +} + +void av1_iht16x8_128_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride, int tx_type) { + __m128i in[16]; + + // Transpose 16x8 input into in[] + in[0] = load_input_data(input + 0 * 16); + in[1] = load_input_data(input + 1 * 16); + in[2] = load_input_data(input + 2 * 16); + in[3] = load_input_data(input + 3 * 16); + in[4] = load_input_data(input + 4 * 16); + in[5] = load_input_data(input + 5 * 16); + in[6] = load_input_data(input + 6 * 16); + in[7] = load_input_data(input + 7 * 16); + array_transpose_8x8(in, in); + + in[8] = load_input_data(input + 8 + 0 * 16); + in[9] = load_input_data(input + 8 + 1 * 16); + in[10] = load_input_data(input + 8 + 2 * 16); + in[11] = load_input_data(input + 8 + 3 * 16); + in[12] = load_input_data(input + 8 + 4 * 16); + in[13] = load_input_data(input + 8 + 5 * 16); + in[14] = load_input_data(input + 8 + 6 * 16); + in[15] = load_input_data(input + 8 + 7 * 16); + array_transpose_8x8(in + 8, in + 8); + + // Row transform + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case H_DCT: +#endif + idct16_8col(in); + break; + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case H_ADST: + case H_FLIPADST: +#endif + iadst16_8col(in); + break; +#if CONFIG_EXT_TX + case V_FLIPADST: + case V_ADST: + case V_DCT: + case IDTX: idtx16_8col(in); break; +#endif + default: assert(0); break; + } + + // Scale + scale_sqrt2_8x8(in); + scale_sqrt2_8x8(in + 8); + + // Column transform + switch (tx_type) { + case DCT_DCT: + case DCT_ADST: +#if CONFIG_EXT_TX + case DCT_FLIPADST: + case V_DCT: +#endif + aom_idct8_sse2(in); + aom_idct8_sse2(in + 8); + break; + case ADST_DCT: + case ADST_ADST: +#if CONFIG_EXT_TX + case FLIPADST_ADST: + case ADST_FLIPADST: + case FLIPADST_FLIPADST: + case FLIPADST_DCT: + case V_ADST: + case V_FLIPADST: +#endif + aom_iadst8_sse2(in); + aom_iadst8_sse2(in + 8); + break; +#if CONFIG_EXT_TX + case H_DCT: + case H_ADST: + case H_FLIPADST: + case IDTX: + array_transpose_8x8(in, in); + array_transpose_8x8(in + 8, in + 8); + iidtx8_sse2(in); + iidtx8_sse2(in + 8); + break; +#endif + default: assert(0); break; + } + + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case H_DCT: + case H_ADST: + case V_ADST: + case V_DCT: + case IDTX: +#endif + write_buffer_8x8_round6(dest, in, stride); + write_buffer_8x8_round6(dest + 8, in + 8, stride); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case FLIPADST_ADST: + case V_FLIPADST: + write_buffer_8x8_round6(dest + stride * 7, in, -stride); + write_buffer_8x8_round6(dest + stride * 7 + 8, in + 8, -stride); + break; + case DCT_FLIPADST: + case ADST_FLIPADST: + case H_FLIPADST: + flip_buffer_lr_8x8(in); + flip_buffer_lr_8x8(in + 8); + write_buffer_8x8_round6(dest, in + 8, stride); + write_buffer_8x8_round6(dest + 8, in, stride); + break; + case FLIPADST_FLIPADST: + flip_buffer_lr_8x8(in); + flip_buffer_lr_8x8(in + 8); + write_buffer_8x8_round6(dest + stride * 7, in + 8, -stride); + write_buffer_8x8_round6(dest + stride * 7 + 8, in, -stride); + break; +#endif + default: assert(0); break; + } +} + +static INLINE void write_buffer_8x4_round5(uint8_t *dest, __m128i *in, + int stride) { + const __m128i final_rounding = _mm_set1_epi16(1 << 4); + const __m128i zero = _mm_setzero_si128(); + // Final rounding and shift + in[0] = _mm_adds_epi16(in[0], final_rounding); + in[1] = _mm_adds_epi16(in[1], final_rounding); + in[2] = _mm_adds_epi16(in[2], final_rounding); + in[3] = _mm_adds_epi16(in[3], final_rounding); + + in[0] = _mm_srai_epi16(in[0], 5); + in[1] = _mm_srai_epi16(in[1], 5); + in[2] = _mm_srai_epi16(in[2], 5); + in[3] = _mm_srai_epi16(in[3], 5); + + RECON_AND_STORE(dest + 0 * stride, in[0]); + RECON_AND_STORE(dest + 1 * stride, in[1]); + RECON_AND_STORE(dest + 2 * stride, in[2]); + RECON_AND_STORE(dest + 3 * stride, in[3]); +} + +void av1_iht8x4_32_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + __m128i in[8]; + + in[0] = load_input_data(input + 0 * 8); + in[1] = load_input_data(input + 1 * 8); + in[2] = load_input_data(input + 2 * 8); + in[3] = load_input_data(input + 3 * 8); + + // Row transform + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case H_DCT: +#endif + aom_idct8_sse2(in); + break; + case DCT_ADST: + case ADST_ADST: aom_iadst8_sse2(in); break; +#if CONFIG_EXT_TX + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case H_ADST: + case H_FLIPADST: aom_iadst8_sse2(in); break; + case V_FLIPADST: + case V_ADST: + case V_DCT: + case IDTX: iidtx8_sse2(in); array_transpose_8x8(in, in); +#endif + break; + default: assert(0); break; + } + + scale_sqrt2_8x8(in); + + // Repack data. We pack into the bottom half of 'in' + // so that the next repacking stage can pack into the + // top half without overwriting anything + in[7] = _mm_unpacklo_epi64(in[6], in[7]); + in[6] = _mm_unpacklo_epi64(in[4], in[5]); + in[5] = _mm_unpacklo_epi64(in[2], in[3]); + in[4] = _mm_unpacklo_epi64(in[0], in[1]); + + // Column transform + switch (tx_type) { + case DCT_DCT: + case DCT_ADST: +#if CONFIG_EXT_TX + case DCT_FLIPADST: + case V_DCT: +#endif + aom_idct4_sse2(in + 4); + aom_idct4_sse2(in + 6); + break; + case ADST_DCT: + case ADST_ADST: +#if CONFIG_EXT_TX + case FLIPADST_ADST: + case ADST_FLIPADST: + case FLIPADST_FLIPADST: + case FLIPADST_DCT: + case V_ADST: + case V_FLIPADST: +#endif + aom_iadst4_sse2(in + 4); + aom_iadst4_sse2(in + 6); + break; +#if CONFIG_EXT_TX + case H_DCT: + case H_ADST: + case H_FLIPADST: + case IDTX: + iidtx4_sse2(in + 4); + array_transpose_4x4(in + 4); + iidtx4_sse2(in + 6); + array_transpose_4x4(in + 6); + break; +#endif + default: assert(0); break; + } + + // Repack data + in[0] = _mm_unpacklo_epi64(in[4], in[6]); + in[1] = _mm_unpackhi_epi64(in[4], in[6]); + in[2] = _mm_unpacklo_epi64(in[5], in[7]); + in[3] = _mm_unpackhi_epi64(in[5], in[7]); + + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case H_DCT: + case H_ADST: + case V_ADST: + case V_DCT: + case IDTX: break; + case FLIPADST_DCT: + case FLIPADST_ADST: + case V_FLIPADST: FLIPUD_PTR(dest, stride, 4); break; + case DCT_FLIPADST: + case ADST_FLIPADST: + case H_FLIPADST: + in[0] = mm_reverse_epi16(in[0]); + in[1] = mm_reverse_epi16(in[1]); + in[2] = mm_reverse_epi16(in[2]); + in[3] = mm_reverse_epi16(in[3]); + break; + case FLIPADST_FLIPADST: + in[0] = mm_reverse_epi16(in[0]); + in[1] = mm_reverse_epi16(in[1]); + in[2] = mm_reverse_epi16(in[2]); + in[3] = mm_reverse_epi16(in[3]); + FLIPUD_PTR(dest, stride, 4); +#endif + break; + default: assert(0); break; + } + write_buffer_8x4_round5(dest, in, stride); +} + +static INLINE void write_buffer_4x8_round5(uint8_t *dest, __m128i *in, + int stride) { + const __m128i final_rounding = _mm_set1_epi16(1 << 4); + const __m128i zero = _mm_setzero_si128(); + // Final rounding and shift + in[0] = _mm_adds_epi16(in[0], final_rounding); + in[1] = _mm_adds_epi16(in[1], final_rounding); + in[2] = _mm_adds_epi16(in[2], final_rounding); + in[3] = _mm_adds_epi16(in[3], final_rounding); + + in[0] = _mm_srai_epi16(in[0], 5); + in[1] = _mm_srai_epi16(in[1], 5); + in[2] = _mm_srai_epi16(in[2], 5); + in[3] = _mm_srai_epi16(in[3], 5); + + // Reconstruction and Store + { + __m128i d0 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 0)); + __m128i d1 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 1)); + __m128i d2 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 2)); + __m128i d3 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 3)); + __m128i d4 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 4)); + __m128i d5 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 5)); + __m128i d6 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 6)); + __m128i d7 = _mm_cvtsi32_si128(*(const int *)(dest + stride * 7)); + + d0 = _mm_unpacklo_epi32(d0, d1); + d2 = _mm_unpacklo_epi32(d2, d3); + d4 = _mm_unpacklo_epi32(d4, d5); + d6 = _mm_unpacklo_epi32(d6, d7); + d0 = _mm_unpacklo_epi8(d0, zero); + d2 = _mm_unpacklo_epi8(d2, zero); + d4 = _mm_unpacklo_epi8(d4, zero); + d6 = _mm_unpacklo_epi8(d6, zero); + d0 = _mm_add_epi16(d0, in[0]); + d2 = _mm_add_epi16(d2, in[1]); + d4 = _mm_add_epi16(d4, in[2]); + d6 = _mm_add_epi16(d6, in[3]); + + d0 = _mm_packus_epi16(d0, d2); + *(int *)dest = _mm_cvtsi128_si32(d0); + d0 = _mm_srli_si128(d0, 4); + *(int *)(dest + stride) = _mm_cvtsi128_si32(d0); + d0 = _mm_srli_si128(d0, 4); + *(int *)(dest + stride * 2) = _mm_cvtsi128_si32(d0); + d0 = _mm_srli_si128(d0, 4); + *(int *)(dest + stride * 3) = _mm_cvtsi128_si32(d0); + d0 = _mm_packus_epi16(d4, d6); + *(int *)(dest + stride * 4) = _mm_cvtsi128_si32(d0); + d0 = _mm_srli_si128(d0, 4); + *(int *)(dest + stride * 5) = _mm_cvtsi128_si32(d0); + d0 = _mm_srli_si128(d0, 4); + *(int *)(dest + stride * 6) = _mm_cvtsi128_si32(d0); + d0 = _mm_srli_si128(d0, 4); + *(int *)(dest + stride * 7) = _mm_cvtsi128_si32(d0); + } +} + +void av1_iht4x8_32_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, + int tx_type) { + __m128i in[8]; + + // Load rows, packed two per element of 'in'. + // We pack into the bottom half of 'in' so that the + // later repacking stage can pack into the + // top half without overwriting anything + in[4] = load_input_data(input + 0 * 8); + in[5] = load_input_data(input + 1 * 8); + in[6] = load_input_data(input + 2 * 8); + in[7] = load_input_data(input + 3 * 8); + + // Row transform + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case H_DCT: +#endif + aom_idct4_sse2(in + 4); + aom_idct4_sse2(in + 6); + break; + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case H_ADST: + case H_FLIPADST: +#endif + aom_iadst4_sse2(in + 4); + aom_iadst4_sse2(in + 6); + break; +#if CONFIG_EXT_TX + case V_FLIPADST: + case V_ADST: + case V_DCT: + case IDTX: + iidtx4_sse2(in + 4); + array_transpose_4x4(in + 4); + iidtx4_sse2(in + 6); + array_transpose_4x4(in + 6); + break; +#endif + default: assert(0); break; + } + + scale_sqrt2_8x4(in + 4); + + // Repack data + in[0] = _mm_unpacklo_epi64(in[4], in[6]); + in[1] = _mm_unpackhi_epi64(in[4], in[6]); + in[2] = _mm_unpacklo_epi64(in[5], in[7]); + in[3] = _mm_unpackhi_epi64(in[5], in[7]); + + // Column transform + switch (tx_type) { + case DCT_DCT: + case DCT_ADST: +#if CONFIG_EXT_TX + case DCT_FLIPADST: + case V_DCT: +#endif + aom_idct8_sse2(in); + break; + case ADST_DCT: + case ADST_ADST: +#if CONFIG_EXT_TX + case FLIPADST_ADST: + case ADST_FLIPADST: + case FLIPADST_FLIPADST: + case FLIPADST_DCT: + case V_ADST: + case V_FLIPADST: +#endif + aom_iadst8_sse2(in); + break; +#if CONFIG_EXT_TX + case H_DCT: + case H_ADST: + case H_FLIPADST: + case IDTX: + iidtx8_sse2(in); + array_transpose_8x8(in, in); + break; +#endif + default: assert(0); break; + } + + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case H_DCT: + case H_ADST: + case V_ADST: + case V_DCT: + case IDTX: +#endif + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case FLIPADST_ADST: + case V_FLIPADST: FLIPUD_PTR(dest, stride, 8); break; + case DCT_FLIPADST: + case ADST_FLIPADST: + case H_FLIPADST: + in[0] = _mm_shufflelo_epi16(in[0], 0x1b); + in[1] = _mm_shufflelo_epi16(in[1], 0x1b); + in[2] = _mm_shufflelo_epi16(in[2], 0x1b); + in[3] = _mm_shufflelo_epi16(in[3], 0x1b); + in[4] = _mm_shufflelo_epi16(in[4], 0x1b); + in[5] = _mm_shufflelo_epi16(in[5], 0x1b); + in[6] = _mm_shufflelo_epi16(in[6], 0x1b); + in[7] = _mm_shufflelo_epi16(in[7], 0x1b); + break; + case FLIPADST_FLIPADST: + in[0] = _mm_shufflelo_epi16(in[0], 0x1b); + in[1] = _mm_shufflelo_epi16(in[1], 0x1b); + in[2] = _mm_shufflelo_epi16(in[2], 0x1b); + in[3] = _mm_shufflelo_epi16(in[3], 0x1b); + in[4] = _mm_shufflelo_epi16(in[4], 0x1b); + in[5] = _mm_shufflelo_epi16(in[5], 0x1b); + in[6] = _mm_shufflelo_epi16(in[6], 0x1b); + in[7] = _mm_shufflelo_epi16(in[7], 0x1b); + FLIPUD_PTR(dest, stride, 8); + break; +#endif + default: assert(0); break; + } + in[0] = _mm_unpacklo_epi64(in[0], in[1]); + in[1] = _mm_unpacklo_epi64(in[2], in[3]); + in[2] = _mm_unpacklo_epi64(in[4], in[5]); + in[3] = _mm_unpacklo_epi64(in[6], in[7]); + write_buffer_4x8_round5(dest, in, stride); +} + +// Note: The 16-column 32-element transforms take input in the form of four +// 8x16 blocks (each stored as a __m128i[16]), which are the four quadrants +// of the overall 16x32 input buffer. +static INLINE void idct32_16col(__m128i *tl, __m128i *tr, __m128i *bl, + __m128i *br) { + array_transpose_16x16(tl, tr); + array_transpose_16x16(bl, br); + idct32_8col(tl, bl); + idct32_8col(tr, br); +} + +static INLINE void ihalfright32_16col(__m128i *tl, __m128i *tr, __m128i *bl, + __m128i *br) { + __m128i tmpl[16], tmpr[16]; + int i; + + // Copy the top half of the input to temporary storage + for (i = 0; i < 16; ++i) { + tmpl[i] = tl[i]; + tmpr[i] = tr[i]; + } + + // Generate the top half of the output + for (i = 0; i < 16; ++i) { + tl[i] = _mm_slli_epi16(bl[i], 2); + tr[i] = _mm_slli_epi16(br[i], 2); + } + array_transpose_16x16(tl, tr); + + // Copy the temporary storage back to the bottom half of the input + for (i = 0; i < 16; ++i) { + bl[i] = tmpl[i]; + br[i] = tmpr[i]; + } + + // Generate the bottom half of the output + scale_sqrt2_8x16(bl); + scale_sqrt2_8x16(br); + aom_idct16_sse2(bl, br); // Includes a transposition +} + +#if CONFIG_EXT_TX +static INLINE void iidtx32_16col(__m128i *tl, __m128i *tr, __m128i *bl, + __m128i *br) { + int i; + array_transpose_16x16(tl, tr); + array_transpose_16x16(bl, br); + for (i = 0; i < 16; ++i) { + tl[i] = _mm_slli_epi16(tl[i], 2); + tr[i] = _mm_slli_epi16(tr[i], 2); + bl[i] = _mm_slli_epi16(bl[i], 2); + br[i] = _mm_slli_epi16(br[i], 2); + } +} +#endif // CONFIG_EXT_TX + +static INLINE void write_buffer_16x32_round6(uint8_t *dest, __m128i *intl, + __m128i *intr, __m128i *inbl, + __m128i *inbr, int stride) { + const __m128i zero = _mm_setzero_si128(); + const __m128i final_rounding = _mm_set1_epi16(1 << 5); + int i; + + for (i = 0; i < 16; ++i) { + intl[i] = _mm_adds_epi16(intl[i], final_rounding); + intr[i] = _mm_adds_epi16(intr[i], final_rounding); + inbl[i] = _mm_adds_epi16(inbl[i], final_rounding); + inbr[i] = _mm_adds_epi16(inbr[i], final_rounding); + intl[i] = _mm_srai_epi16(intl[i], 6); + intr[i] = _mm_srai_epi16(intr[i], 6); + inbl[i] = _mm_srai_epi16(inbl[i], 6); + inbr[i] = _mm_srai_epi16(inbr[i], 6); + RECON_AND_STORE(dest + i * stride + 0, intl[i]); + RECON_AND_STORE(dest + i * stride + 8, intr[i]); + RECON_AND_STORE(dest + (i + 16) * stride + 0, inbl[i]); + RECON_AND_STORE(dest + (i + 16) * stride + 8, inbr[i]); + } +} + +void av1_iht16x32_512_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride, int tx_type) { + __m128i intl[16], intr[16], inbl[16], inbr[16]; + + int i; + for (i = 0; i < 16; ++i) { + intl[i] = load_input_data(input + i * 16 + 0); + intr[i] = load_input_data(input + i * 16 + 8); + inbl[i] = load_input_data(input + (i + 16) * 16 + 0); + inbr[i] = load_input_data(input + (i + 16) * 16 + 8); + } + + // Row transform + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case H_DCT: +#endif + aom_idct16_sse2(intl, intr); + aom_idct16_sse2(inbl, inbr); + break; + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case H_ADST: + case H_FLIPADST: +#endif + aom_iadst16_sse2(intl, intr); + aom_iadst16_sse2(inbl, inbr); + break; +#if CONFIG_EXT_TX + case V_FLIPADST: + case V_ADST: + case V_DCT: + case IDTX: + iidtx16_sse2(intl, intr); + iidtx16_sse2(inbl, inbr); + break; +#endif + default: assert(0); break; + } + + scale_sqrt2_8x16(intl); + scale_sqrt2_8x16(intr); + scale_sqrt2_8x16(inbl); + scale_sqrt2_8x16(inbr); + + // Column transform + switch (tx_type) { + case DCT_DCT: + case DCT_ADST: +#if CONFIG_EXT_TX + case DCT_FLIPADST: + case V_DCT: +#endif + idct32_16col(intl, intr, inbl, inbr); + break; + case ADST_DCT: + case ADST_ADST: +#if CONFIG_EXT_TX + case FLIPADST_ADST: + case ADST_FLIPADST: + case FLIPADST_FLIPADST: + case FLIPADST_DCT: + case V_ADST: + case V_FLIPADST: +#endif + ihalfright32_16col(intl, intr, inbl, inbr); + break; +#if CONFIG_EXT_TX + case H_DCT: + case H_ADST: + case H_FLIPADST: + case IDTX: iidtx32_16col(intl, intr, inbl, inbr); break; +#endif + default: assert(0); break; + } + + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case H_DCT: + case H_ADST: + case V_ADST: + case V_DCT: + case IDTX: +#endif + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case FLIPADST_ADST: + case V_FLIPADST: FLIPUD_PTR(dest, stride, 32); break; + case DCT_FLIPADST: + case ADST_FLIPADST: + case H_FLIPADST: + for (i = 0; i < 16; ++i) { + __m128i tmp = intl[i]; + intl[i] = mm_reverse_epi16(intr[i]); + intr[i] = mm_reverse_epi16(tmp); + tmp = inbl[i]; + inbl[i] = mm_reverse_epi16(inbr[i]); + inbr[i] = mm_reverse_epi16(tmp); + } + break; + case FLIPADST_FLIPADST: + for (i = 0; i < 16; ++i) { + __m128i tmp = intl[i]; + intl[i] = mm_reverse_epi16(intr[i]); + intr[i] = mm_reverse_epi16(tmp); + tmp = inbl[i]; + inbl[i] = mm_reverse_epi16(inbr[i]); + inbr[i] = mm_reverse_epi16(tmp); + } + FLIPUD_PTR(dest, stride, 32); + break; +#endif + default: assert(0); break; + } + write_buffer_16x32_round6(dest, intl, intr, inbl, inbr, stride); +} + +static INLINE void write_buffer_32x16_round6(uint8_t *dest, __m128i *in0, + __m128i *in1, __m128i *in2, + __m128i *in3, int stride) { + const __m128i zero = _mm_setzero_si128(); + const __m128i final_rounding = _mm_set1_epi16(1 << 5); + int i; + + for (i = 0; i < 16; ++i) { + in0[i] = _mm_adds_epi16(in0[i], final_rounding); + in1[i] = _mm_adds_epi16(in1[i], final_rounding); + in2[i] = _mm_adds_epi16(in2[i], final_rounding); + in3[i] = _mm_adds_epi16(in3[i], final_rounding); + in0[i] = _mm_srai_epi16(in0[i], 6); + in1[i] = _mm_srai_epi16(in1[i], 6); + in2[i] = _mm_srai_epi16(in2[i], 6); + in3[i] = _mm_srai_epi16(in3[i], 6); + RECON_AND_STORE(dest + i * stride + 0, in0[i]); + RECON_AND_STORE(dest + i * stride + 8, in1[i]); + RECON_AND_STORE(dest + i * stride + 16, in2[i]); + RECON_AND_STORE(dest + i * stride + 24, in3[i]); + } +} + +void av1_iht32x16_512_add_sse2(const tran_low_t *input, uint8_t *dest, + int stride, int tx_type) { + __m128i in0[16], in1[16], in2[16], in3[16]; + int i; + + for (i = 0; i < 16; ++i) { + in0[i] = load_input_data(input + i * 32 + 0); + in1[i] = load_input_data(input + i * 32 + 8); + in2[i] = load_input_data(input + i * 32 + 16); + in3[i] = load_input_data(input + i * 32 + 24); + } + + // Row transform + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case H_DCT: +#endif + idct32_16col(in0, in1, in2, in3); + break; + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case H_ADST: + case H_FLIPADST: +#endif + ihalfright32_16col(in0, in1, in2, in3); + break; +#if CONFIG_EXT_TX + case V_FLIPADST: + case V_ADST: + case V_DCT: + case IDTX: iidtx32_16col(in0, in1, in2, in3); break; +#endif + default: assert(0); break; + } + + scale_sqrt2_8x16(in0); + scale_sqrt2_8x16(in1); + scale_sqrt2_8x16(in2); + scale_sqrt2_8x16(in3); + + // Column transform + switch (tx_type) { + case DCT_DCT: + case DCT_ADST: +#if CONFIG_EXT_TX + case DCT_FLIPADST: + case V_DCT: +#endif + aom_idct16_sse2(in0, in1); + aom_idct16_sse2(in2, in3); + break; + case ADST_DCT: + case ADST_ADST: +#if CONFIG_EXT_TX + case FLIPADST_ADST: + case ADST_FLIPADST: + case FLIPADST_FLIPADST: + case FLIPADST_DCT: + case V_ADST: + case V_FLIPADST: +#endif + aom_iadst16_sse2(in0, in1); + aom_iadst16_sse2(in2, in3); + break; +#if CONFIG_EXT_TX + case H_DCT: + case H_ADST: + case H_FLIPADST: + case IDTX: + iidtx16_sse2(in0, in1); + iidtx16_sse2(in2, in3); + break; +#endif + default: assert(0); break; + } + + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: +#if CONFIG_EXT_TX + case H_DCT: + case H_ADST: + case V_ADST: + case V_DCT: + case IDTX: +#endif + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case FLIPADST_ADST: + case V_FLIPADST: FLIPUD_PTR(dest, stride, 16); break; + case DCT_FLIPADST: + case ADST_FLIPADST: + case H_FLIPADST: + for (i = 0; i < 16; ++i) { + __m128i tmp1 = in0[i]; + __m128i tmp2 = in1[i]; + in0[i] = mm_reverse_epi16(in3[i]); + in1[i] = mm_reverse_epi16(in2[i]); + in2[i] = mm_reverse_epi16(tmp2); + in3[i] = mm_reverse_epi16(tmp1); + } + break; + case FLIPADST_FLIPADST: + for (i = 0; i < 16; ++i) { + __m128i tmp1 = in0[i]; + __m128i tmp2 = in1[i]; + in0[i] = mm_reverse_epi16(in3[i]); + in1[i] = mm_reverse_epi16(in2[i]); + in2[i] = mm_reverse_epi16(tmp2); + in3[i] = mm_reverse_epi16(tmp1); + } + FLIPUD_PTR(dest, stride, 16); + break; +#endif + default: assert(0); break; + } + write_buffer_32x16_round6(dest, in0, in1, in2, in3, stride); +} diff --git a/third_party/aom/av1/common/x86/pvq_sse4.c b/third_party/aom/av1/common/x86/pvq_sse4.c new file mode 100644 index 0000000000..b3ed9efdf4 --- /dev/null +++ b/third_party/aom/av1/common/x86/pvq_sse4.c @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include + +#include "./av1_rtcd.h" +#include "av1/common/x86/pvq_sse4.h" +#include "../odintrin.h" +#include "av1/common/pvq.h" + +#define EPSILON 1e-15f + +static __m128 horizontal_sum_ps(__m128 x) { + x = _mm_add_ps(x, _mm_shuffle_ps(x, x, _MM_SHUFFLE(1, 0, 3, 2))); + x = _mm_add_ps(x, _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1))); + return x; +} + +static __m128i horizontal_sum_epi32(__m128i x) { + x = _mm_add_epi32(x, _mm_shuffle_epi32(x, _MM_SHUFFLE(1, 0, 3, 2))); + x = _mm_add_epi32(x, _mm_shuffle_epi32(x, _MM_SHUFFLE(2, 3, 0, 1))); + return x; +} + +static INLINE float rsqrtf(float x) { + float y; + _mm_store_ss(&y, _mm_rsqrt_ss(_mm_load_ss(&x))); + return y; +} + +/** Find the codepoint on the given PSphere closest to the desired + * vector. This is a float-precision PVQ search just to make sure + * our tests aren't limited by numerical accuracy. It's close to the + * pvq_search_rdo_double_c implementation, but is not bit accurate and + * it performs slightly worse on PSNR. One reason is that this code runs + * more RDO iterations than the C code. It also uses single precision + * floating point math, whereas the C version uses double precision. + * + * @param [in] xcoeff input vector to quantize (x in the math doc) + * @param [in] n number of dimensions + * @param [in] k number of pulses + * @param [out] ypulse optimal codevector found (y in the math doc) + * @param [in] g2 multiplier for the distortion (typically squared + * gain units) + * @param [in] pvq_norm_lambda enc->pvq_norm_lambda for quantized RDO + * @param [in] prev_k number of pulses already in ypulse that we should + * reuse for the search (or 0 for a new search) + * @return cosine distance between x and y (between 0 and 1) + */ +double pvq_search_rdo_double_sse4_1(const od_val16 *xcoeff, int n, int k, + int *ypulse, double g2, + double pvq_norm_lambda, int prev_k) { + int i, j; + int reuse_pulses = prev_k > 0 && prev_k <= k; + /* TODO - This blows our 8kB stack space budget and should be fixed when + converting PVQ to fixed point. */ + float xx = 0, xy = 0, yy = 0; + float x[MAXN + 3]; + float y[MAXN + 3]; + float sign_y[MAXN + 3]; + for (i = 0; i < n; i++) { + float tmp = (float)xcoeff[i]; + xx += tmp * tmp; + x[i] = xcoeff[i]; + } + + x[n] = x[n + 1] = x[n + 2] = 0; + ypulse[n] = ypulse[n + 1] = ypulse[n + 2] = 0; + + __m128 sums = _mm_setzero_ps(); + for (i = 0; i < n; i += 4) { + __m128 x4 = _mm_loadu_ps(&x[i]); + __m128 s4 = _mm_cmplt_ps(x4, _mm_setzero_ps()); + /* Save the sign, we'll put it back later. */ + _mm_storeu_ps(&sign_y[i], s4); + /* Get rid of the sign. */ + x4 = _mm_andnot_ps(_mm_set_ps1(-0.f), x4); + sums = _mm_add_ps(sums, x4); + if (!reuse_pulses) { + /* Clear y and ypulse in case we don't do the projection. */ + _mm_storeu_ps(&y[i], _mm_setzero_ps()); + _mm_storeu_si128((__m128i *)&ypulse[i], _mm_setzero_si128()); + } + _mm_storeu_ps(&x[i], x4); + } + sums = horizontal_sum_ps(sums); + int pulses_left = k; + { + __m128i pulses_sum; + __m128 yy4, xy4; + xy4 = yy4 = _mm_setzero_ps(); + pulses_sum = _mm_setzero_si128(); + if (reuse_pulses) { + /* We reuse pulses from a previous search so we don't have to search them + again. */ + for (j = 0; j < n; j += 4) { + __m128 x4, y4; + __m128i iy4; + iy4 = _mm_abs_epi32(_mm_loadu_si128((__m128i *)&ypulse[j])); + pulses_sum = _mm_add_epi32(pulses_sum, iy4); + _mm_storeu_si128((__m128i *)&ypulse[j], iy4); + y4 = _mm_cvtepi32_ps(iy4); + x4 = _mm_loadu_ps(&x[j]); + xy4 = _mm_add_ps(xy4, _mm_mul_ps(x4, y4)); + yy4 = _mm_add_ps(yy4, _mm_mul_ps(y4, y4)); + /* Double the y[] vector so we don't have to do it in the search loop. + */ + _mm_storeu_ps(&y[j], _mm_add_ps(y4, y4)); + } + pulses_left -= _mm_cvtsi128_si32(horizontal_sum_epi32(pulses_sum)); + xy4 = horizontal_sum_ps(xy4); + xy = _mm_cvtss_f32(xy4); + yy4 = horizontal_sum_ps(yy4); + yy = _mm_cvtss_f32(yy4); + } else if (k > (n >> 1)) { + /* Do a pre-search by projecting on the pyramid. */ + __m128 rcp4; + float sum = _mm_cvtss_f32(sums); + /* If x is too small, just replace it with a pulse at 0. This prevents + infinities and NaNs from causing too many pulses to be allocated. Here, + 64 is an + approximation of infinity. */ + if (sum <= EPSILON) { + x[0] = 1.f; + for (i = 1; i < n; i++) { + x[i] = 0; + } + sums = _mm_set_ps1(1.f); + } + /* Using k + e with e < 1 guarantees we cannot get more than k pulses. */ + rcp4 = _mm_mul_ps(_mm_set_ps1((float)k + .8f), _mm_rcp_ps(sums)); + xy4 = yy4 = _mm_setzero_ps(); + pulses_sum = _mm_setzero_si128(); + for (j = 0; j < n; j += 4) { + __m128 rx4, x4, y4; + __m128i iy4; + x4 = _mm_loadu_ps(&x[j]); + rx4 = _mm_mul_ps(x4, rcp4); + iy4 = _mm_cvttps_epi32(rx4); + pulses_sum = _mm_add_epi32(pulses_sum, iy4); + _mm_storeu_si128((__m128i *)&ypulse[j], iy4); + y4 = _mm_cvtepi32_ps(iy4); + xy4 = _mm_add_ps(xy4, _mm_mul_ps(x4, y4)); + yy4 = _mm_add_ps(yy4, _mm_mul_ps(y4, y4)); + /* Double the y[] vector so we don't have to do it in the search loop. + */ + _mm_storeu_ps(&y[j], _mm_add_ps(y4, y4)); + } + pulses_left -= _mm_cvtsi128_si32(horizontal_sum_epi32(pulses_sum)); + xy = _mm_cvtss_f32(horizontal_sum_ps(xy4)); + yy = _mm_cvtss_f32(horizontal_sum_ps(yy4)); + } + x[n] = x[n + 1] = x[n + 2] = -100; + y[n] = y[n + 1] = y[n + 2] = 100; + } + + /* This should never happen. */ + OD_ASSERT(pulses_left <= n + 3); + + float lambda_delta_rate[MAXN + 3]; + if (pulses_left) { + /* Hoist lambda to avoid the multiply in the loop. */ + float lambda = + 0.5f * sqrtf(xx) * (float)pvq_norm_lambda / (FLT_MIN + (float)g2); + float delta_rate = 3.f / n; + __m128 count = _mm_set_ps(3, 2, 1, 0); + for (i = 0; i < n; i += 4) { + _mm_storeu_ps(&lambda_delta_rate[i], + _mm_mul_ps(count, _mm_set_ps1(lambda * delta_rate))); + count = _mm_add_ps(count, _mm_set_ps(4, 4, 4, 4)); + } + } + lambda_delta_rate[n] = lambda_delta_rate[n + 1] = lambda_delta_rate[n + 2] = + 1e30f; + + for (i = 0; i < pulses_left; i++) { + int best_id = 0; + __m128 xy4, yy4; + __m128 max, max2; + __m128i count; + __m128i pos; + + /* The squared magnitude term gets added anyway, so we might as well + add it outside the loop. */ + yy = yy + 1; + xy4 = _mm_load1_ps(&xy); + yy4 = _mm_load1_ps(&yy); + max = _mm_setzero_ps(); + pos = _mm_setzero_si128(); + count = _mm_set_epi32(3, 2, 1, 0); + for (j = 0; j < n; j += 4) { + __m128 x4, y4, r4; + x4 = _mm_loadu_ps(&x[j]); + y4 = _mm_loadu_ps(&y[j]); + x4 = _mm_add_ps(x4, xy4); + y4 = _mm_add_ps(y4, yy4); + y4 = _mm_rsqrt_ps(y4); + r4 = _mm_mul_ps(x4, y4); + /* Subtract lambda. */ + r4 = _mm_sub_ps(r4, _mm_loadu_ps(&lambda_delta_rate[j])); + /* Update the index of the max. */ + pos = _mm_max_epi16( + pos, _mm_and_si128(count, _mm_castps_si128(_mm_cmpgt_ps(r4, max)))); + /* Update the max. */ + max = _mm_max_ps(max, r4); + /* Update the indices (+4) */ + count = _mm_add_epi32(count, _mm_set_epi32(4, 4, 4, 4)); + } + /* Horizontal max. */ + max2 = _mm_max_ps(max, _mm_shuffle_ps(max, max, _MM_SHUFFLE(1, 0, 3, 2))); + max2 = + _mm_max_ps(max2, _mm_shuffle_ps(max2, max2, _MM_SHUFFLE(2, 3, 0, 1))); + /* Now that max2 contains the max at all positions, look at which value(s) + of the + partial max is equal to the global max. */ + pos = _mm_and_si128(pos, _mm_castps_si128(_mm_cmpeq_ps(max, max2))); + pos = _mm_max_epi16(pos, _mm_unpackhi_epi64(pos, pos)); + pos = _mm_max_epi16(pos, _mm_shufflelo_epi16(pos, _MM_SHUFFLE(1, 0, 3, 2))); + best_id = _mm_cvtsi128_si32(pos); + OD_ASSERT(best_id < n); + /* Updating the sums of the new pulse(s) */ + xy = xy + x[best_id]; + /* We're multiplying y[j] by two so we don't have to do it here. */ + yy = yy + y[best_id]; + /* Only now that we've made the final choice, update y/ypulse. */ + /* Multiplying y[j] by 2 so we don't have to do it everywhere else. */ + y[best_id] += 2; + ypulse[best_id]++; + } + + /* Put the original sign back. */ + for (i = 0; i < n; i += 4) { + __m128i y4; + __m128i s4; + y4 = _mm_loadu_si128((__m128i *)&ypulse[i]); + s4 = _mm_castps_si128(_mm_loadu_ps(&sign_y[i])); + y4 = _mm_xor_si128(_mm_add_epi32(y4, s4), s4); + _mm_storeu_si128((__m128i *)&ypulse[i], y4); + } + return xy * rsqrtf(xx * yy + FLT_MIN); +} diff --git a/third_party/aom/av1/common/x86/pvq_sse4.h b/third_party/aom/av1/common/x86/pvq_sse4.h new file mode 100644 index 0000000000..3c4ce85435 --- /dev/null +++ b/third_party/aom/av1/common/x86/pvq_sse4.h @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_COMMON_PVQ_X86_SSE4_H_ +#define AOM_COMMON_PVQ_X86_SSE4_H_ +#endif // AOM_COMMON_PVQ_X86_SSE4_H_ diff --git a/third_party/aom/av1/common/x86/selfguided_sse4.c b/third_party/aom/av1/common/x86/selfguided_sse4.c new file mode 100644 index 0000000000..260faa8c97 --- /dev/null +++ b/third_party/aom/av1/common/x86/selfguided_sse4.c @@ -0,0 +1,1805 @@ +#include + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "av1/common/restoration.h" + +/* Calculate four consecutive entries of the intermediate A and B arrays + (corresponding to the first loop in the C version of + av1_selfguided_restoration) +*/ +static void calc_block(__m128i sum, __m128i sum_sq, __m128i n, + __m128i one_over_n, __m128i s, int bit_depth, int idx, + int32_t *A, int32_t *B) { + __m128i a, b, p; +#if CONFIG_HIGHBITDEPTH + if (bit_depth > 8) { + __m128i rounding_a = _mm_set1_epi32((1 << (2 * (bit_depth - 8))) >> 1); + __m128i rounding_b = _mm_set1_epi32((1 << (bit_depth - 8)) >> 1); + __m128i shift_a = _mm_set_epi64x(0, 2 * (bit_depth - 8)); + __m128i shift_b = _mm_set_epi64x(0, bit_depth - 8); + a = _mm_srl_epi32(_mm_add_epi32(sum_sq, rounding_a), shift_a); + b = _mm_srl_epi32(_mm_add_epi32(sum, rounding_b), shift_b); + a = _mm_mullo_epi32(a, n); + b = _mm_mullo_epi32(b, b); + p = _mm_sub_epi32(_mm_max_epi32(a, b), b); + } else { +#endif + (void)bit_depth; + a = _mm_mullo_epi32(sum_sq, n); + b = _mm_mullo_epi32(sum, sum); + p = _mm_sub_epi32(a, b); +#if CONFIG_HIGHBITDEPTH + } +#endif + + __m128i rounding_z = _mm_set1_epi32((1 << SGRPROJ_MTABLE_BITS) >> 1); + __m128i z = _mm_srli_epi32(_mm_add_epi32(_mm_mullo_epi32(p, s), rounding_z), + SGRPROJ_MTABLE_BITS); + z = _mm_min_epi32(z, _mm_set1_epi32(255)); + + // 'Gather' type instructions are not available pre-AVX2, so synthesize a + // gather using scalar loads. + __m128i a_res = _mm_set_epi32(x_by_xplus1[_mm_extract_epi32(z, 3)], + x_by_xplus1[_mm_extract_epi32(z, 2)], + x_by_xplus1[_mm_extract_epi32(z, 1)], + x_by_xplus1[_mm_extract_epi32(z, 0)]); + + _mm_storeu_si128((__m128i *)&A[idx], a_res); + + __m128i rounding_res = _mm_set1_epi32((1 << SGRPROJ_RECIP_BITS) >> 1); + __m128i a_complement = _mm_sub_epi32(_mm_set1_epi32(SGRPROJ_SGR), a_res); + __m128i b_int = + _mm_mullo_epi32(a_complement, _mm_mullo_epi32(sum, one_over_n)); + __m128i b_res = + _mm_srli_epi32(_mm_add_epi32(b_int, rounding_res), SGRPROJ_RECIP_BITS); + + _mm_storeu_si128((__m128i *)&B[idx], b_res); +} + +static void selfguided_restoration_1_v(uint8_t *src, int width, int height, + int src_stride, int32_t *A, int32_t *B, + int buf_stride) { + int i, j; + + // Vertical sum + // When the width is not a multiple of 4, we know that 'stride' is rounded up + // to a multiple of 4. So it is safe for this loop to calculate extra columns + // at the right-hand edge of the frame. + int width_extend = (width + 3) & ~3; + for (j = 0; j < width_extend; j += 4) { + __m128i a, b, x, y, x2, y2; + __m128i sum, sum_sq, tmp; + + a = _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i *)&src[j])); + b = _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i *)&src[src_stride + j])); + + sum = _mm_cvtepi16_epi32(_mm_add_epi16(a, b)); + tmp = _mm_unpacklo_epi16(a, b); + sum_sq = _mm_madd_epi16(tmp, tmp); + + _mm_store_si128((__m128i *)&B[j], sum); + _mm_store_si128((__m128i *)&A[j], sum_sq); + + x = _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i *)&src[2 * src_stride + j])); + sum = _mm_add_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_add_epi32(sum_sq, x2); + + for (i = 1; i < height - 2; ++i) { + _mm_store_si128((__m128i *)&B[i * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[i * buf_stride + j], sum_sq); + + x = _mm_cvtepu8_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 1) * src_stride + j])); + y = _mm_cvtepu8_epi32( + _mm_loadl_epi64((__m128i *)&src[(i + 2) * src_stride + j])); + + sum = _mm_add_epi32(sum, _mm_sub_epi32(y, x)); + + x2 = _mm_mullo_epi32(x, x); + y2 = _mm_mullo_epi32(y, y); + + sum_sq = _mm_add_epi32(sum_sq, _mm_sub_epi32(y2, x2)); + } + _mm_store_si128((__m128i *)&B[i * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[i * buf_stride + j], sum_sq); + + x = _mm_cvtepu8_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 1) * src_stride + j])); + sum = _mm_sub_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_sub_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[(i + 1) * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[(i + 1) * buf_stride + j], sum_sq); + } +} + +static void selfguided_restoration_1_h(int32_t *A, int32_t *B, int width, + int height, int buf_stride, int eps, + int bit_depth) { + int i, j; + + // Horizontal sum + int width_extend = (width + 3) & ~3; + for (i = 0; i < height; ++i) { + int h = AOMMIN(2, height - i) + AOMMIN(1, i); + + __m128i a1 = _mm_loadu_si128((__m128i *)&A[i * buf_stride]); + __m128i b1 = _mm_loadu_si128((__m128i *)&B[i * buf_stride]); + __m128i a2 = _mm_loadu_si128((__m128i *)&A[i * buf_stride + 4]); + __m128i b2 = _mm_loadu_si128((__m128i *)&B[i * buf_stride + 4]); + + // Note: The _mm_slli_si128 call sets up a register containing + // {0, A[i * buf_stride], ..., A[i * buf_stride + 2]}, + // so that the first element of 'sum' (which should only add two values + // together) ends up calculated correctly. + __m128i sum_ = _mm_add_epi32(_mm_slli_si128(b1, 4), + _mm_add_epi32(b1, _mm_alignr_epi8(b2, b1, 4))); + __m128i sum_sq_ = _mm_add_epi32( + _mm_slli_si128(a1, 4), _mm_add_epi32(a1, _mm_alignr_epi8(a2, a1, 4))); + __m128i n = _mm_set_epi32(3 * h, 3 * h, 3 * h, 2 * h); + __m128i one_over_n = + _mm_set_epi32(one_by_x[3 * h - 1], one_by_x[3 * h - 1], + one_by_x[3 * h - 1], one_by_x[2 * h - 1]); + __m128i s = _mm_set_epi32( + sgrproj_mtable[eps - 1][3 * h - 1], sgrproj_mtable[eps - 1][3 * h - 1], + sgrproj_mtable[eps - 1][3 * h - 1], sgrproj_mtable[eps - 1][2 * h - 1]); + calc_block(sum_, sum_sq_, n, one_over_n, s, bit_depth, i * buf_stride, A, + B); + + n = _mm_set1_epi32(3 * h); + one_over_n = _mm_set1_epi32(one_by_x[3 * h - 1]); + s = _mm_set1_epi32(sgrproj_mtable[eps - 1][3 * h - 1]); + + // Re-align a1 and b1 so that they start at index i * buf_stride + 3 + a2 = _mm_alignr_epi8(a2, a1, 12); + b2 = _mm_alignr_epi8(b2, b1, 12); + + // Note: When the width is not a multiple of 4, this loop may end up + // writing to the last 4 columns of the frame, potentially with incorrect + // values (especially for r=2 and r=3). + // This is fine, since we fix up those values in the block after this + // loop, and in exchange we never have more than four values to + // write / fix up after this loop finishes. + for (j = 4; j < width_extend - 4; j += 4) { + a1 = a2; + b1 = b2; + a2 = _mm_loadu_si128((__m128i *)&A[i * buf_stride + j + 3]); + b2 = _mm_loadu_si128((__m128i *)&B[i * buf_stride + j + 3]); + /* Loop invariant: At this point, + a1 = original A[i * buf_stride + j - 1 : i * buf_stride + j + 3] + a2 = original A[i * buf_stride + j + 3 : i * buf_stride + j + 7] + and similar for b1,b2 and B + */ + sum_ = _mm_add_epi32(b1, _mm_add_epi32(_mm_alignr_epi8(b2, b1, 4), + _mm_alignr_epi8(b2, b1, 8))); + sum_sq_ = _mm_add_epi32(a1, _mm_add_epi32(_mm_alignr_epi8(a2, a1, 4), + _mm_alignr_epi8(a2, a1, 8))); + calc_block(sum_, sum_sq_, n, one_over_n, s, bit_depth, i * buf_stride + j, + A, B); + } + __m128i a3 = _mm_loadu_si128((__m128i *)&A[i * buf_stride + j + 3]); + __m128i b3 = _mm_loadu_si128((__m128i *)&B[i * buf_stride + j + 3]); + + j = width - 4; + switch (width % 4) { + case 0: + a1 = a2; + b1 = b2; + a2 = a3; + b2 = b3; + break; + case 1: + a1 = _mm_alignr_epi8(a2, a1, 4); + b1 = _mm_alignr_epi8(b2, b1, 4); + a2 = _mm_alignr_epi8(a3, a2, 4); + b2 = _mm_alignr_epi8(b3, b2, 4); + break; + case 2: + a1 = _mm_alignr_epi8(a2, a1, 8); + b1 = _mm_alignr_epi8(b2, b1, 8); + a2 = _mm_alignr_epi8(a3, a2, 8); + b2 = _mm_alignr_epi8(b3, b2, 8); + break; + case 3: + a1 = _mm_alignr_epi8(a2, a1, 12); + b1 = _mm_alignr_epi8(b2, b1, 12); + a2 = _mm_alignr_epi8(a3, a2, 12); + b2 = _mm_alignr_epi8(b3, b2, 12); + break; + } + + // Zero out the data loaded from "off the edge" of the array + __m128i zero = _mm_setzero_si128(); + a2 = _mm_blend_epi16(a2, zero, 0xfc); + b2 = _mm_blend_epi16(b2, zero, 0xfc); + + sum_ = _mm_add_epi32(b1, _mm_add_epi32(_mm_alignr_epi8(b2, b1, 4), + _mm_alignr_epi8(b2, b1, 8))); + sum_sq_ = _mm_add_epi32(a1, _mm_add_epi32(_mm_alignr_epi8(a2, a1, 4), + _mm_alignr_epi8(a2, a1, 8))); + n = _mm_set_epi32(2 * h, 3 * h, 3 * h, 3 * h); + one_over_n = _mm_set_epi32(one_by_x[2 * h - 1], one_by_x[3 * h - 1], + one_by_x[3 * h - 1], one_by_x[3 * h - 1]); + s = _mm_set_epi32( + sgrproj_mtable[eps - 1][2 * h - 1], sgrproj_mtable[eps - 1][3 * h - 1], + sgrproj_mtable[eps - 1][3 * h - 1], sgrproj_mtable[eps - 1][3 * h - 1]); + calc_block(sum_, sum_sq_, n, one_over_n, s, bit_depth, i * buf_stride + j, + A, B); + } +} + +static void selfguided_restoration_2_v(uint8_t *src, int width, int height, + int src_stride, int32_t *A, int32_t *B, + int buf_stride) { + int i, j; + + // Vertical sum + int width_extend = (width + 3) & ~3; + for (j = 0; j < width_extend; j += 4) { + __m128i a, b, c, c2, x, y, x2, y2; + __m128i sum, sum_sq, tmp; + + a = _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i *)&src[j])); + b = _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i *)&src[src_stride + j])); + c = _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i *)&src[2 * src_stride + j])); + + sum = _mm_cvtepi16_epi32(_mm_add_epi16(_mm_add_epi16(a, b), c)); + // Important: Since c may be up to 2^8, the result on squaring may + // be up to 2^16. So we need to zero-extend, not sign-extend. + c2 = _mm_cvtepu16_epi32(_mm_mullo_epi16(c, c)); + tmp = _mm_unpacklo_epi16(a, b); + sum_sq = _mm_add_epi32(_mm_madd_epi16(tmp, tmp), c2); + + _mm_store_si128((__m128i *)&B[j], sum); + _mm_store_si128((__m128i *)&A[j], sum_sq); + + x = _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i *)&src[3 * src_stride + j])); + sum = _mm_add_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_add_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[buf_stride + j], sum_sq); + + x = _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i *)&src[4 * src_stride + j])); + sum = _mm_add_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_add_epi32(sum_sq, x2); + + for (i = 2; i < height - 3; ++i) { + _mm_store_si128((__m128i *)&B[i * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[i * buf_stride + j], sum_sq); + + x = _mm_cvtepu8_epi32( + _mm_cvtsi32_si128(*((int *)&src[(i - 2) * src_stride + j]))); + y = _mm_cvtepu8_epi32( + _mm_cvtsi32_si128(*((int *)&src[(i + 3) * src_stride + j]))); + + sum = _mm_add_epi32(sum, _mm_sub_epi32(y, x)); + + x2 = _mm_mullo_epi32(x, x); + y2 = _mm_mullo_epi32(y, y); + + sum_sq = _mm_add_epi32(sum_sq, _mm_sub_epi32(y2, x2)); + } + _mm_store_si128((__m128i *)&B[i * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[i * buf_stride + j], sum_sq); + + x = _mm_cvtepu8_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 2) * src_stride + j])); + sum = _mm_sub_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_sub_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[(i + 1) * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[(i + 1) * buf_stride + j], sum_sq); + + x = _mm_cvtepu8_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 1) * src_stride + j])); + sum = _mm_sub_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_sub_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[(i + 2) * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[(i + 2) * buf_stride + j], sum_sq); + } +} + +static void selfguided_restoration_2_h(int32_t *A, int32_t *B, int width, + int height, int buf_stride, int eps, + int bit_depth) { + int i, j; + + // Horizontal sum + int width_extend = (width + 3) & ~3; + for (i = 0; i < height; ++i) { + int h = AOMMIN(3, height - i) + AOMMIN(2, i); + + __m128i a1 = _mm_loadu_si128((__m128i *)&A[i * buf_stride]); + __m128i b1 = _mm_loadu_si128((__m128i *)&B[i * buf_stride]); + __m128i a2 = _mm_loadu_si128((__m128i *)&A[i * buf_stride + 4]); + __m128i b2 = _mm_loadu_si128((__m128i *)&B[i * buf_stride + 4]); + + __m128i sum_ = _mm_add_epi32( + _mm_add_epi32( + _mm_add_epi32(_mm_slli_si128(b1, 8), _mm_slli_si128(b1, 4)), + _mm_add_epi32(b1, _mm_alignr_epi8(b2, b1, 4))), + _mm_alignr_epi8(b2, b1, 8)); + __m128i sum_sq_ = _mm_add_epi32( + _mm_add_epi32( + _mm_add_epi32(_mm_slli_si128(a1, 8), _mm_slli_si128(a1, 4)), + _mm_add_epi32(a1, _mm_alignr_epi8(a2, a1, 4))), + _mm_alignr_epi8(a2, a1, 8)); + + __m128i n = _mm_set_epi32(5 * h, 5 * h, 4 * h, 3 * h); + __m128i one_over_n = + _mm_set_epi32(one_by_x[5 * h - 1], one_by_x[5 * h - 1], + one_by_x[4 * h - 1], one_by_x[3 * h - 1]); + __m128i s = _mm_set_epi32( + sgrproj_mtable[eps - 1][5 * h - 1], sgrproj_mtable[eps - 1][5 * h - 1], + sgrproj_mtable[eps - 1][4 * h - 1], sgrproj_mtable[eps - 1][3 * h - 1]); + calc_block(sum_, sum_sq_, n, one_over_n, s, bit_depth, i * buf_stride, A, + B); + + // Re-align a1 and b1 so that they start at index i * buf_stride + 2 + a2 = _mm_alignr_epi8(a2, a1, 8); + b2 = _mm_alignr_epi8(b2, b1, 8); + + n = _mm_set1_epi32(5 * h); + one_over_n = _mm_set1_epi32(one_by_x[5 * h - 1]); + s = _mm_set1_epi32(sgrproj_mtable[eps - 1][5 * h - 1]); + + for (j = 4; j < width_extend - 4; j += 4) { + a1 = a2; + a2 = _mm_loadu_si128((__m128i *)&A[i * buf_stride + j + 2]); + b1 = b2; + b2 = _mm_loadu_si128((__m128i *)&B[i * buf_stride + j + 2]); + /* Loop invariant: At this point, + a1 = original A[i * buf_stride + j - 2 : i * buf_stride + j + 2] + a2 = original A[i * buf_stride + j + 2 : i * buf_stride + j + 6] + and similar for b1,b2 and B + */ + sum_ = _mm_add_epi32( + _mm_add_epi32(b1, _mm_add_epi32(_mm_alignr_epi8(b2, b1, 4), + _mm_alignr_epi8(b2, b1, 8))), + _mm_add_epi32(_mm_alignr_epi8(b2, b1, 12), b2)); + sum_sq_ = _mm_add_epi32( + _mm_add_epi32(a1, _mm_add_epi32(_mm_alignr_epi8(a2, a1, 4), + _mm_alignr_epi8(a2, a1, 8))), + _mm_add_epi32(_mm_alignr_epi8(a2, a1, 12), a2)); + + calc_block(sum_, sum_sq_, n, one_over_n, s, bit_depth, i * buf_stride + j, + A, B); + } + // If the width is not a multiple of 4, we need to reset j to width - 4 + // and adjust a1, a2, b1, b2 so that the loop invariant above is maintained + __m128i a3 = _mm_loadu_si128((__m128i *)&A[i * buf_stride + j + 2]); + __m128i b3 = _mm_loadu_si128((__m128i *)&B[i * buf_stride + j + 2]); + + j = width - 4; + switch (width % 4) { + case 0: + a1 = a2; + b1 = b2; + a2 = a3; + b2 = b3; + break; + case 1: + a1 = _mm_alignr_epi8(a2, a1, 4); + b1 = _mm_alignr_epi8(b2, b1, 4); + a2 = _mm_alignr_epi8(a3, a2, 4); + b2 = _mm_alignr_epi8(b3, b2, 4); + break; + case 2: + a1 = _mm_alignr_epi8(a2, a1, 8); + b1 = _mm_alignr_epi8(b2, b1, 8); + a2 = _mm_alignr_epi8(a3, a2, 8); + b2 = _mm_alignr_epi8(b3, b2, 8); + break; + case 3: + a1 = _mm_alignr_epi8(a2, a1, 12); + b1 = _mm_alignr_epi8(b2, b1, 12); + a2 = _mm_alignr_epi8(a3, a2, 12); + b2 = _mm_alignr_epi8(b3, b2, 12); + break; + } + + // Zero out the data loaded from "off the edge" of the array + __m128i zero = _mm_setzero_si128(); + a2 = _mm_blend_epi16(a2, zero, 0xf0); + b2 = _mm_blend_epi16(b2, zero, 0xf0); + + sum_ = _mm_add_epi32( + _mm_add_epi32(b1, _mm_add_epi32(_mm_alignr_epi8(b2, b1, 4), + _mm_alignr_epi8(b2, b1, 8))), + _mm_add_epi32(_mm_alignr_epi8(b2, b1, 12), b2)); + sum_sq_ = _mm_add_epi32( + _mm_add_epi32(a1, _mm_add_epi32(_mm_alignr_epi8(a2, a1, 4), + _mm_alignr_epi8(a2, a1, 8))), + _mm_add_epi32(_mm_alignr_epi8(a2, a1, 12), a2)); + + n = _mm_set_epi32(3 * h, 4 * h, 5 * h, 5 * h); + one_over_n = _mm_set_epi32(one_by_x[3 * h - 1], one_by_x[4 * h - 1], + one_by_x[5 * h - 1], one_by_x[5 * h - 1]); + s = _mm_set_epi32( + sgrproj_mtable[eps - 1][3 * h - 1], sgrproj_mtable[eps - 1][4 * h - 1], + sgrproj_mtable[eps - 1][5 * h - 1], sgrproj_mtable[eps - 1][5 * h - 1]); + calc_block(sum_, sum_sq_, n, one_over_n, s, bit_depth, i * buf_stride + j, + A, B); + } +} + +static void selfguided_restoration_3_v(uint8_t *src, int width, int height, + int src_stride, int32_t *A, int32_t *B, + int buf_stride) { + int i, j; + + // Vertical sum over 7-pixel regions, 4 columns at a time + int width_extend = (width + 3) & ~3; + for (j = 0; j < width_extend; j += 4) { + __m128i a, b, c, d, x, y, x2, y2; + __m128i sum, sum_sq, tmp, tmp2; + + a = _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i *)&src[j])); + b = _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i *)&src[src_stride + j])); + c = _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i *)&src[2 * src_stride + j])); + d = _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i *)&src[3 * src_stride + j])); + + sum = _mm_cvtepi16_epi32( + _mm_add_epi16(_mm_add_epi16(a, b), _mm_add_epi16(c, d))); + tmp = _mm_unpacklo_epi16(a, b); + tmp2 = _mm_unpacklo_epi16(c, d); + sum_sq = + _mm_add_epi32(_mm_madd_epi16(tmp, tmp), _mm_madd_epi16(tmp2, tmp2)); + + _mm_store_si128((__m128i *)&B[j], sum); + _mm_store_si128((__m128i *)&A[j], sum_sq); + + x = _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i *)&src[4 * src_stride + j])); + sum = _mm_add_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_add_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[buf_stride + j], sum_sq); + + x = _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i *)&src[5 * src_stride + j])); + sum = _mm_add_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_add_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[2 * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[2 * buf_stride + j], sum_sq); + + x = _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i *)&src[6 * src_stride + j])); + sum = _mm_add_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_add_epi32(sum_sq, x2); + + for (i = 3; i < height - 4; ++i) { + _mm_store_si128((__m128i *)&B[i * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[i * buf_stride + j], sum_sq); + + x = _mm_cvtepu8_epi32( + _mm_cvtsi32_si128(*((int *)&src[(i - 3) * src_stride + j]))); + y = _mm_cvtepu8_epi32( + _mm_cvtsi32_si128(*((int *)&src[(i + 4) * src_stride + j]))); + + sum = _mm_add_epi32(sum, _mm_sub_epi32(y, x)); + + x2 = _mm_mullo_epi32(x, x); + y2 = _mm_mullo_epi32(y, y); + + sum_sq = _mm_add_epi32(sum_sq, _mm_sub_epi32(y2, x2)); + } + _mm_store_si128((__m128i *)&B[i * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[i * buf_stride + j], sum_sq); + + x = _mm_cvtepu8_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 3) * src_stride + j])); + sum = _mm_sub_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_sub_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[(i + 1) * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[(i + 1) * buf_stride + j], sum_sq); + + x = _mm_cvtepu8_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 2) * src_stride + j])); + sum = _mm_sub_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_sub_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[(i + 2) * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[(i + 2) * buf_stride + j], sum_sq); + + x = _mm_cvtepu8_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 1) * src_stride + j])); + sum = _mm_sub_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_sub_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[(i + 3) * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[(i + 3) * buf_stride + j], sum_sq); + } +} + +static void selfguided_restoration_3_h(int32_t *A, int32_t *B, int width, + int height, int buf_stride, int eps, + int bit_depth) { + int i, j; + // Horizontal sum over 7-pixel regions of dst + int width_extend = (width + 3) & ~3; + for (i = 0; i < height; ++i) { + int h = AOMMIN(4, height - i) + AOMMIN(3, i); + + __m128i a1 = _mm_loadu_si128((__m128i *)&A[i * buf_stride]); + __m128i b1 = _mm_loadu_si128((__m128i *)&B[i * buf_stride]); + __m128i a2 = _mm_loadu_si128((__m128i *)&A[i * buf_stride + 4]); + __m128i b2 = _mm_loadu_si128((__m128i *)&B[i * buf_stride + 4]); + + __m128i sum_ = _mm_add_epi32( + _mm_add_epi32( + _mm_add_epi32(_mm_slli_si128(b1, 12), _mm_slli_si128(b1, 8)), + _mm_add_epi32(_mm_slli_si128(b1, 4), b1)), + _mm_add_epi32(_mm_add_epi32(_mm_alignr_epi8(b2, b1, 4), + _mm_alignr_epi8(b2, b1, 8)), + _mm_alignr_epi8(b2, b1, 12))); + __m128i sum_sq_ = _mm_add_epi32( + _mm_add_epi32( + _mm_add_epi32(_mm_slli_si128(a1, 12), _mm_slli_si128(a1, 8)), + _mm_add_epi32(_mm_slli_si128(a1, 4), a1)), + _mm_add_epi32(_mm_add_epi32(_mm_alignr_epi8(a2, a1, 4), + _mm_alignr_epi8(a2, a1, 8)), + _mm_alignr_epi8(a2, a1, 12))); + + __m128i n = _mm_set_epi32(7 * h, 6 * h, 5 * h, 4 * h); + __m128i one_over_n = + _mm_set_epi32(one_by_x[7 * h - 1], one_by_x[6 * h - 1], + one_by_x[5 * h - 1], one_by_x[4 * h - 1]); + __m128i s = _mm_set_epi32( + sgrproj_mtable[eps - 1][7 * h - 1], sgrproj_mtable[eps - 1][6 * h - 1], + sgrproj_mtable[eps - 1][5 * h - 1], sgrproj_mtable[eps - 1][4 * h - 1]); + calc_block(sum_, sum_sq_, n, one_over_n, s, bit_depth, i * buf_stride, A, + B); + + // Re-align a1 and b1 so that they start at index i * buf_stride + 1 + a2 = _mm_alignr_epi8(a2, a1, 4); + b2 = _mm_alignr_epi8(b2, b1, 4); + + n = _mm_set1_epi32(7 * h); + one_over_n = _mm_set1_epi32(one_by_x[7 * h - 1]); + s = _mm_set1_epi32(sgrproj_mtable[eps - 1][7 * h - 1]); + + for (j = 4; j < width_extend - 4; j += 4) { + a1 = a2; + a2 = _mm_loadu_si128((__m128i *)&A[i * buf_stride + j + 1]); + b1 = b2; + b2 = _mm_loadu_si128((__m128i *)&B[i * buf_stride + j + 1]); + __m128i a3 = _mm_loadu_si128((__m128i *)&A[i * buf_stride + j + 5]); + __m128i b3 = _mm_loadu_si128((__m128i *)&B[i * buf_stride + j + 5]); + /* Loop invariant: At this point, + a1 = original A[i * buf_stride + j - 3 : i * buf_stride + j + 1] + a2 = original A[i * buf_stride + j + 1 : i * buf_stride + j + 5] + a3 = original A[i * buf_stride + j + 5 : i * buf_stride + j + 9] + and similar for b1,b2,b3 and B + */ + sum_ = _mm_add_epi32( + _mm_add_epi32(_mm_add_epi32(b1, _mm_alignr_epi8(b2, b1, 4)), + _mm_add_epi32(_mm_alignr_epi8(b2, b1, 8), + _mm_alignr_epi8(b2, b1, 12))), + _mm_add_epi32(_mm_add_epi32(b2, _mm_alignr_epi8(b3, b2, 4)), + _mm_alignr_epi8(b3, b2, 8))); + sum_sq_ = _mm_add_epi32( + _mm_add_epi32(_mm_add_epi32(a1, _mm_alignr_epi8(a2, a1, 4)), + _mm_add_epi32(_mm_alignr_epi8(a2, a1, 8), + _mm_alignr_epi8(a2, a1, 12))), + _mm_add_epi32(_mm_add_epi32(a2, _mm_alignr_epi8(a3, a2, 4)), + _mm_alignr_epi8(a3, a2, 8))); + + calc_block(sum_, sum_sq_, n, one_over_n, s, bit_depth, i * buf_stride + j, + A, B); + } + __m128i a3 = _mm_loadu_si128((__m128i *)&A[i * buf_stride + j + 1]); + __m128i b3 = _mm_loadu_si128((__m128i *)&B[i * buf_stride + j + 1]); + + j = width - 4; + switch (width % 4) { + case 0: + a1 = a2; + b1 = b2; + a2 = a3; + b2 = b3; + break; + case 1: + a1 = _mm_alignr_epi8(a2, a1, 4); + b1 = _mm_alignr_epi8(b2, b1, 4); + a2 = _mm_alignr_epi8(a3, a2, 4); + b2 = _mm_alignr_epi8(b3, b2, 4); + break; + case 2: + a1 = _mm_alignr_epi8(a2, a1, 8); + b1 = _mm_alignr_epi8(b2, b1, 8); + a2 = _mm_alignr_epi8(a3, a2, 8); + b2 = _mm_alignr_epi8(b3, b2, 8); + break; + case 3: + a1 = _mm_alignr_epi8(a2, a1, 12); + b1 = _mm_alignr_epi8(b2, b1, 12); + a2 = _mm_alignr_epi8(a3, a2, 12); + b2 = _mm_alignr_epi8(b3, b2, 12); + break; + } + + // Zero out the data loaded from "off the edge" of the array + __m128i zero = _mm_setzero_si128(); + a2 = _mm_blend_epi16(a2, zero, 0xc0); + b2 = _mm_blend_epi16(b2, zero, 0xc0); + + sum_ = _mm_add_epi32( + _mm_add_epi32(_mm_add_epi32(b1, _mm_alignr_epi8(b2, b1, 4)), + _mm_add_epi32(_mm_alignr_epi8(b2, b1, 8), + _mm_alignr_epi8(b2, b1, 12))), + _mm_add_epi32(_mm_add_epi32(b2, _mm_alignr_epi8(zero, b2, 4)), + _mm_alignr_epi8(zero, b2, 8))); + sum_sq_ = _mm_add_epi32( + _mm_add_epi32(_mm_add_epi32(a1, _mm_alignr_epi8(a2, a1, 4)), + _mm_add_epi32(_mm_alignr_epi8(a2, a1, 8), + _mm_alignr_epi8(a2, a1, 12))), + _mm_add_epi32(_mm_add_epi32(a2, _mm_alignr_epi8(zero, a2, 4)), + _mm_alignr_epi8(zero, a2, 8))); + + n = _mm_set_epi32(4 * h, 5 * h, 6 * h, 7 * h); + one_over_n = _mm_set_epi32(one_by_x[4 * h - 1], one_by_x[5 * h - 1], + one_by_x[6 * h - 1], one_by_x[7 * h - 1]); + s = _mm_set_epi32( + sgrproj_mtable[eps - 1][4 * h - 1], sgrproj_mtable[eps - 1][5 * h - 1], + sgrproj_mtable[eps - 1][6 * h - 1], sgrproj_mtable[eps - 1][7 * h - 1]); + calc_block(sum_, sum_sq_, n, one_over_n, s, bit_depth, i * buf_stride + j, + A, B); + } +} + +void av1_selfguided_restoration_sse4_1(uint8_t *dgd, int width, int height, + int stride, int32_t *dst, int dst_stride, + int r, int eps, int32_t *tmpbuf) { + int32_t *A = tmpbuf; + int32_t *B = A + SGRPROJ_OUTBUF_SIZE; + int i, j; + // Adjusting the stride of A and B here appears to avoid bad cache effects, + // leading to a significant speed improvement. + // We also align the stride to a multiple of 16 bytes for efficiency. + int buf_stride = ((width + 3) & ~3) + 16; + + // Don't filter tiles with dimensions < 5 on any axis + if ((width < 5) || (height < 5)) return; + + if (r == 1) { + selfguided_restoration_1_v(dgd, width, height, stride, A, B, buf_stride); + selfguided_restoration_1_h(A, B, width, height, buf_stride, eps, 8); + } else if (r == 2) { + selfguided_restoration_2_v(dgd, width, height, stride, A, B, buf_stride); + selfguided_restoration_2_h(A, B, width, height, buf_stride, eps, 8); + } else if (r == 3) { + selfguided_restoration_3_v(dgd, width, height, stride, A, B, buf_stride); + selfguided_restoration_3_h(A, B, width, height, buf_stride, eps, 8); + } else { + assert(0); + } + + { + i = 0; + j = 0; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = 3 * A[k] + 2 * A[k + 1] + 2 * A[k + buf_stride] + + A[k + buf_stride + 1]; + const int32_t b = 3 * B[k] + 2 * B[k + 1] + 2 * B[k + buf_stride] + + B[k + buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + for (j = 1; j < width - 1; ++j) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = A[k] + 2 * (A[k - 1] + A[k + 1]) + A[k + buf_stride] + + A[k + buf_stride - 1] + A[k + buf_stride + 1]; + const int32_t b = B[k] + 2 * (B[k - 1] + B[k + 1]) + B[k + buf_stride] + + B[k + buf_stride - 1] + B[k + buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + j = width - 1; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = 3 * A[k] + 2 * A[k - 1] + 2 * A[k + buf_stride] + + A[k + buf_stride - 1]; + const int32_t b = 3 * B[k] + 2 * B[k - 1] + 2 * B[k + buf_stride] + + B[k + buf_stride - 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + } + for (i = 1; i < height - 1; ++i) { + j = 0; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = A[k] + 2 * (A[k - buf_stride] + A[k + buf_stride]) + + A[k + 1] + A[k - buf_stride + 1] + + A[k + buf_stride + 1]; + const int32_t b = B[k] + 2 * (B[k - buf_stride] + B[k + buf_stride]) + + B[k + 1] + B[k - buf_stride + 1] + + B[k + buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + + // Vectorize the innermost loop + for (j = 1; j < width - 1; j += 4) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 5; + + __m128i tmp0 = _mm_loadu_si128((__m128i *)&A[k - 1 - buf_stride]); + __m128i tmp1 = _mm_loadu_si128((__m128i *)&A[k + 3 - buf_stride]); + __m128i tmp2 = _mm_loadu_si128((__m128i *)&A[k - 1]); + __m128i tmp3 = _mm_loadu_si128((__m128i *)&A[k + 3]); + __m128i tmp4 = _mm_loadu_si128((__m128i *)&A[k - 1 + buf_stride]); + __m128i tmp5 = _mm_loadu_si128((__m128i *)&A[k + 3 + buf_stride]); + + __m128i a0 = _mm_add_epi32( + _mm_add_epi32(_mm_add_epi32(_mm_alignr_epi8(tmp3, tmp2, 4), tmp2), + _mm_add_epi32(_mm_alignr_epi8(tmp3, tmp2, 8), + _mm_alignr_epi8(tmp5, tmp4, 4))), + _mm_alignr_epi8(tmp1, tmp0, 4)); + __m128i a1 = _mm_add_epi32(_mm_add_epi32(tmp0, tmp4), + _mm_add_epi32(_mm_alignr_epi8(tmp1, tmp0, 8), + _mm_alignr_epi8(tmp5, tmp4, 8))); + __m128i a = _mm_sub_epi32(_mm_slli_epi32(_mm_add_epi32(a0, a1), 2), a1); + + __m128i tmp6 = _mm_loadu_si128((__m128i *)&B[k - 1 - buf_stride]); + __m128i tmp7 = _mm_loadu_si128((__m128i *)&B[k + 3 - buf_stride]); + __m128i tmp8 = _mm_loadu_si128((__m128i *)&B[k - 1]); + __m128i tmp9 = _mm_loadu_si128((__m128i *)&B[k + 3]); + __m128i tmp10 = _mm_loadu_si128((__m128i *)&B[k - 1 + buf_stride]); + __m128i tmp11 = _mm_loadu_si128((__m128i *)&B[k + 3 + buf_stride]); + + __m128i b0 = _mm_add_epi32( + _mm_add_epi32(_mm_add_epi32(_mm_alignr_epi8(tmp9, tmp8, 4), tmp8), + _mm_add_epi32(_mm_alignr_epi8(tmp9, tmp8, 8), + _mm_alignr_epi8(tmp11, tmp10, 4))), + _mm_alignr_epi8(tmp7, tmp6, 4)); + __m128i b1 = + _mm_add_epi32(_mm_add_epi32(tmp6, tmp10), + _mm_add_epi32(_mm_alignr_epi8(tmp7, tmp6, 8), + _mm_alignr_epi8(tmp11, tmp10, 8))); + __m128i b = _mm_sub_epi32(_mm_slli_epi32(_mm_add_epi32(b0, b1), 2), b1); + + __m128i src = _mm_cvtepu8_epi32(_mm_loadu_si128((__m128i *)&dgd[l])); + + __m128i rounding = _mm_set1_epi32( + (1 << (SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS)) >> 1); + __m128i v = _mm_add_epi32(_mm_mullo_epi32(a, src), b); + __m128i w = _mm_srai_epi32(_mm_add_epi32(v, rounding), + SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + _mm_storeu_si128((__m128i *)&dst[m], w); + } + + // Deal with any extra pixels at the right-hand edge of the frame + // (typically have 2 such pixels, but may have anywhere between 0 and 3) + for (; j < width - 1; ++j) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 5; + const int32_t a = + (A[k] + A[k - 1] + A[k + 1] + A[k - buf_stride] + A[k + buf_stride]) * + 4 + + (A[k - 1 - buf_stride] + A[k - 1 + buf_stride] + + A[k + 1 - buf_stride] + A[k + 1 + buf_stride]) * + 3; + const int32_t b = + (B[k] + B[k - 1] + B[k + 1] + B[k - buf_stride] + B[k + buf_stride]) * + 4 + + (B[k - 1 - buf_stride] + B[k - 1 + buf_stride] + + B[k + 1 - buf_stride] + B[k + 1 + buf_stride]) * + 3; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + + j = width - 1; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = A[k] + 2 * (A[k - buf_stride] + A[k + buf_stride]) + + A[k - 1] + A[k - buf_stride - 1] + + A[k + buf_stride - 1]; + const int32_t b = B[k] + 2 * (B[k - buf_stride] + B[k + buf_stride]) + + B[k - 1] + B[k - buf_stride - 1] + + B[k + buf_stride - 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + } + + { + i = height - 1; + j = 0; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = 3 * A[k] + 2 * A[k + 1] + 2 * A[k - buf_stride] + + A[k - buf_stride + 1]; + const int32_t b = 3 * B[k] + 2 * B[k + 1] + 2 * B[k - buf_stride] + + B[k - buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + for (j = 1; j < width - 1; ++j) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = A[k] + 2 * (A[k - 1] + A[k + 1]) + A[k - buf_stride] + + A[k - buf_stride - 1] + A[k - buf_stride + 1]; + const int32_t b = B[k] + 2 * (B[k - 1] + B[k + 1]) + B[k - buf_stride] + + B[k - buf_stride - 1] + B[k - buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + j = width - 1; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = 3 * A[k] + 2 * A[k - 1] + 2 * A[k - buf_stride] + + A[k - buf_stride - 1]; + const int32_t b = 3 * B[k] + 2 * B[k - 1] + 2 * B[k - buf_stride] + + B[k - buf_stride - 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + } +} + +void av1_highpass_filter_sse4_1(uint8_t *dgd, int width, int height, int stride, + int32_t *dst, int dst_stride, int corner, + int edge) { + int i, j; + const int center = (1 << SGRPROJ_RST_BITS) - 4 * (corner + edge); + + { + i = 0; + j = 0; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k + 1] + dgd[k + stride] + dgd[k] * 2) + + corner * + (dgd[k + stride + 1] + dgd[k + 1] + dgd[k + stride] + dgd[k]); + } + for (j = 1; j < width - 1; ++j) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = center * dgd[k] + + edge * (dgd[k - 1] + dgd[k + stride] + dgd[k + 1] + dgd[k]) + + corner * (dgd[k + stride - 1] + dgd[k + stride + 1] + + dgd[k - 1] + dgd[k + 1]); + } + j = width - 1; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k - 1] + dgd[k + stride] + dgd[k] * 2) + + corner * + (dgd[k + stride - 1] + dgd[k - 1] + dgd[k + stride] + dgd[k]); + } + } + { + i = height - 1; + j = 0; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k + 1] + dgd[k - stride] + dgd[k] * 2) + + corner * + (dgd[k - stride + 1] + dgd[k + 1] + dgd[k - stride] + dgd[k]); + } + for (j = 1; j < width - 1; ++j) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = center * dgd[k] + + edge * (dgd[k - 1] + dgd[k - stride] + dgd[k + 1] + dgd[k]) + + corner * (dgd[k - stride - 1] + dgd[k - stride + 1] + + dgd[k - 1] + dgd[k + 1]); + } + j = width - 1; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k - 1] + dgd[k - stride] + dgd[k] * 2) + + corner * + (dgd[k - stride - 1] + dgd[k - 1] + dgd[k - stride] + dgd[k]); + } + } + __m128i center_ = _mm_set1_epi16(center); + __m128i edge_ = _mm_set1_epi16(edge); + __m128i corner_ = _mm_set1_epi16(corner); + for (i = 1; i < height - 1; ++i) { + j = 0; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + + edge * (dgd[k - stride] + dgd[k + 1] + dgd[k + stride] + dgd[k]) + + corner * (dgd[k + stride + 1] + dgd[k - stride + 1] + + dgd[k - stride] + dgd[k + stride]); + } + // Process in units of 8 pixels at a time. + for (j = 1; j < width - 8; j += 8) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + + __m128i a = _mm_loadu_si128((__m128i *)&dgd[k - stride - 1]); + __m128i b = _mm_loadu_si128((__m128i *)&dgd[k - 1]); + __m128i c = _mm_loadu_si128((__m128i *)&dgd[k + stride - 1]); + + __m128i tl = _mm_cvtepu8_epi16(a); + __m128i tr = _mm_cvtepu8_epi16(_mm_srli_si128(a, 8)); + __m128i cl = _mm_cvtepu8_epi16(b); + __m128i cr = _mm_cvtepu8_epi16(_mm_srli_si128(b, 8)); + __m128i bl = _mm_cvtepu8_epi16(c); + __m128i br = _mm_cvtepu8_epi16(_mm_srli_si128(c, 8)); + + __m128i x = _mm_alignr_epi8(cr, cl, 2); + __m128i y = _mm_add_epi16(_mm_add_epi16(_mm_alignr_epi8(tr, tl, 2), cl), + _mm_add_epi16(_mm_alignr_epi8(br, bl, 2), + _mm_alignr_epi8(cr, cl, 4))); + __m128i z = _mm_add_epi16(_mm_add_epi16(tl, bl), + _mm_add_epi16(_mm_alignr_epi8(tr, tl, 4), + _mm_alignr_epi8(br, bl, 4))); + + __m128i res = _mm_add_epi16(_mm_mullo_epi16(x, center_), + _mm_add_epi16(_mm_mullo_epi16(y, edge_), + _mm_mullo_epi16(z, corner_))); + + _mm_storeu_si128((__m128i *)&dst[l], _mm_cvtepi16_epi32(res)); + _mm_storeu_si128((__m128i *)&dst[l + 4], + _mm_cvtepi16_epi32(_mm_srli_si128(res, 8))); + } + // If there are enough pixels left in this row, do another batch of 4 + // pixels. + for (; j < width - 4; j += 4) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + + __m128i a = _mm_loadl_epi64((__m128i *)&dgd[k - stride - 1]); + __m128i b = _mm_loadl_epi64((__m128i *)&dgd[k - 1]); + __m128i c = _mm_loadl_epi64((__m128i *)&dgd[k + stride - 1]); + + __m128i tl = _mm_cvtepu8_epi16(a); + __m128i cl = _mm_cvtepu8_epi16(b); + __m128i bl = _mm_cvtepu8_epi16(c); + + __m128i x = _mm_srli_si128(cl, 2); + __m128i y = _mm_add_epi16( + _mm_add_epi16(_mm_srli_si128(tl, 2), cl), + _mm_add_epi16(_mm_srli_si128(bl, 2), _mm_srli_si128(cl, 4))); + __m128i z = _mm_add_epi16( + _mm_add_epi16(tl, bl), + _mm_add_epi16(_mm_srli_si128(tl, 4), _mm_srli_si128(bl, 4))); + + __m128i res = _mm_add_epi16(_mm_mullo_epi16(x, center_), + _mm_add_epi16(_mm_mullo_epi16(y, edge_), + _mm_mullo_epi16(z, corner_))); + + _mm_storeu_si128((__m128i *)&dst[l], _mm_cvtepi16_epi32(res)); + } + // Handle any leftover pixels + for (; j < width - 1; ++j) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + + edge * (dgd[k - stride] + dgd[k - 1] + dgd[k + stride] + dgd[k + 1]) + + corner * (dgd[k + stride - 1] + dgd[k - stride - 1] + + dgd[k - stride + 1] + dgd[k + stride + 1]); + } + j = width - 1; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + + edge * (dgd[k - stride] + dgd[k - 1] + dgd[k + stride] + dgd[k]) + + corner * (dgd[k + stride - 1] + dgd[k - stride - 1] + + dgd[k - stride] + dgd[k + stride]); + } + } +} + +void apply_selfguided_restoration_sse4_1(uint8_t *dat, int width, int height, + int stride, int eps, int *xqd, + uint8_t *dst, int dst_stride, + int32_t *tmpbuf) { + int xq[2]; + int32_t *flt1 = tmpbuf; + int32_t *flt2 = flt1 + RESTORATION_TILEPELS_MAX; + int32_t *tmpbuf2 = flt2 + RESTORATION_TILEPELS_MAX; + int i, j; + assert(width * height <= RESTORATION_TILEPELS_MAX); +#if USE_HIGHPASS_IN_SGRPROJ + av1_highpass_filter_sse4_1(dat, width, height, stride, flt1, width, + sgr_params[eps].corner, sgr_params[eps].edge); +#else + av1_selfguided_restoration_sse4_1(dat, width, height, stride, flt1, width, + sgr_params[eps].r1, sgr_params[eps].e1, + tmpbuf2); +#endif // USE_HIGHPASS_IN_SGRPROJ + av1_selfguided_restoration_sse4_1(dat, width, height, stride, flt2, width, + sgr_params[eps].r2, sgr_params[eps].e2, + tmpbuf2); + decode_xq(xqd, xq); + + __m128i xq0 = _mm_set1_epi32(xq[0]); + __m128i xq1 = _mm_set1_epi32(xq[1]); + for (i = 0; i < height; ++i) { + // Calculate output in batches of 8 pixels + for (j = 0; j < width; j += 8) { + const int k = i * width + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + __m128i src = + _mm_slli_epi16(_mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i *)&dat[l])), + SGRPROJ_RST_BITS); + + const __m128i u_0 = _mm_cvtepu16_epi32(src); + const __m128i u_1 = _mm_cvtepu16_epi32(_mm_srli_si128(src, 8)); + + const __m128i f1_0 = + _mm_sub_epi32(_mm_loadu_si128((__m128i *)&flt1[k]), u_0); + const __m128i f2_0 = + _mm_sub_epi32(_mm_loadu_si128((__m128i *)&flt2[k]), u_0); + const __m128i f1_1 = + _mm_sub_epi32(_mm_loadu_si128((__m128i *)&flt1[k + 4]), u_1); + const __m128i f2_1 = + _mm_sub_epi32(_mm_loadu_si128((__m128i *)&flt2[k + 4]), u_1); + + const __m128i v_0 = _mm_add_epi32( + _mm_add_epi32(_mm_mullo_epi32(xq0, f1_0), _mm_mullo_epi32(xq1, f2_0)), + _mm_slli_epi32(u_0, SGRPROJ_PRJ_BITS)); + const __m128i v_1 = _mm_add_epi32( + _mm_add_epi32(_mm_mullo_epi32(xq0, f1_1), _mm_mullo_epi32(xq1, f2_1)), + _mm_slli_epi32(u_1, SGRPROJ_PRJ_BITS)); + + const __m128i rounding = + _mm_set1_epi32((1 << (SGRPROJ_PRJ_BITS + SGRPROJ_RST_BITS)) >> 1); + const __m128i w_0 = _mm_srai_epi32(_mm_add_epi32(v_0, rounding), + SGRPROJ_PRJ_BITS + SGRPROJ_RST_BITS); + const __m128i w_1 = _mm_srai_epi32(_mm_add_epi32(v_1, rounding), + SGRPROJ_PRJ_BITS + SGRPROJ_RST_BITS); + + const __m128i tmp = _mm_packs_epi32(w_0, w_1); + const __m128i res = _mm_packus_epi16(tmp, tmp /* "don't care" value */); + _mm_storel_epi64((__m128i *)&dst[m], res); + } + // Process leftover pixels + for (; j < width; ++j) { + const int k = i * width + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int32_t u = ((int32_t)dat[l] << SGRPROJ_RST_BITS); + const int32_t f1 = (int32_t)flt1[k] - u; + const int32_t f2 = (int32_t)flt2[k] - u; + const int32_t v = xq[0] * f1 + xq[1] * f2 + (u << SGRPROJ_PRJ_BITS); + const int16_t w = + (int16_t)ROUND_POWER_OF_TWO(v, SGRPROJ_PRJ_BITS + SGRPROJ_RST_BITS); + dst[m] = (uint16_t)clip_pixel(w); + } + } +} + +#if CONFIG_HIGHBITDEPTH +// Only the vertical sums need to be adjusted for highbitdepth + +static void highbd_selfguided_restoration_1_v(uint16_t *src, int width, + int height, int src_stride, + int32_t *A, int32_t *B, + int buf_stride) { + int i, j; + + int width_extend = (width + 3) & ~3; + for (j = 0; j < width_extend; j += 4) { + __m128i a, b, x, y, x2, y2; + __m128i sum, sum_sq, tmp; + + a = _mm_loadl_epi64((__m128i *)&src[j]); + b = _mm_loadl_epi64((__m128i *)&src[src_stride + j]); + + sum = _mm_cvtepi16_epi32(_mm_add_epi16(a, b)); + tmp = _mm_unpacklo_epi16(a, b); + sum_sq = _mm_madd_epi16(tmp, tmp); + + _mm_store_si128((__m128i *)&B[j], sum); + _mm_store_si128((__m128i *)&A[j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[2 * src_stride + j])); + sum = _mm_add_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_add_epi32(sum_sq, x2); + + for (i = 1; i < height - 2; ++i) { + _mm_store_si128((__m128i *)&B[i * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[i * buf_stride + j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 1) * src_stride + j])); + y = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[(i + 2) * src_stride + j])); + + sum = _mm_add_epi32(sum, _mm_sub_epi32(y, x)); + + x2 = _mm_mullo_epi32(x, x); + y2 = _mm_mullo_epi32(y, y); + + sum_sq = _mm_add_epi32(sum_sq, _mm_sub_epi32(y2, x2)); + } + _mm_store_si128((__m128i *)&B[i * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[i * buf_stride + j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 1) * src_stride + j])); + sum = _mm_sub_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_sub_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[(i + 1) * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[(i + 1) * buf_stride + j], sum_sq); + } +} + +static void highbd_selfguided_restoration_2_v(uint16_t *src, int width, + int height, int src_stride, + int32_t *A, int32_t *B, + int buf_stride) { + int i, j; + + int width_extend = (width + 3) & ~3; + for (j = 0; j < width_extend; j += 4) { + __m128i a, b, c, c2, x, y, x2, y2; + __m128i sum, sum_sq, tmp; + + a = _mm_loadl_epi64((__m128i *)&src[j]); + b = _mm_loadl_epi64((__m128i *)&src[src_stride + j]); + c = _mm_loadl_epi64((__m128i *)&src[2 * src_stride + j]); + + sum = _mm_cvtepi16_epi32(_mm_add_epi16(_mm_add_epi16(a, b), c)); + // Important: We need to widen *before* squaring here, since + // c^2 may be up to 2^24. + c = _mm_cvtepu16_epi32(c); + c2 = _mm_mullo_epi32(c, c); + tmp = _mm_unpacklo_epi16(a, b); + sum_sq = _mm_add_epi32(_mm_madd_epi16(tmp, tmp), c2); + + _mm_store_si128((__m128i *)&B[j], sum); + _mm_store_si128((__m128i *)&A[j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[3 * src_stride + j])); + sum = _mm_add_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_add_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[buf_stride + j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[4 * src_stride + j])); + sum = _mm_add_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_add_epi32(sum_sq, x2); + + for (i = 2; i < height - 3; ++i) { + _mm_store_si128((__m128i *)&B[i * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[i * buf_stride + j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 2) * src_stride + j])); + y = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[(i + 3) * src_stride + j])); + + sum = _mm_add_epi32(sum, _mm_sub_epi32(y, x)); + + x2 = _mm_mullo_epi32(x, x); + y2 = _mm_mullo_epi32(y, y); + + sum_sq = _mm_add_epi32(sum_sq, _mm_sub_epi32(y2, x2)); + } + _mm_store_si128((__m128i *)&B[i * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[i * buf_stride + j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 2) * src_stride + j])); + sum = _mm_sub_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_sub_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[(i + 1) * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[(i + 1) * buf_stride + j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 1) * src_stride + j])); + sum = _mm_sub_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_sub_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[(i + 2) * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[(i + 2) * buf_stride + j], sum_sq); + } +} + +static void highbd_selfguided_restoration_3_v(uint16_t *src, int width, + int height, int src_stride, + int32_t *A, int32_t *B, + int buf_stride) { + int i, j; + + int width_extend = (width + 3) & ~3; + for (j = 0; j < width_extend; j += 4) { + __m128i a, b, c, d, x, y, x2, y2; + __m128i sum, sum_sq, tmp, tmp2; + + a = _mm_loadl_epi64((__m128i *)&src[j]); + b = _mm_loadl_epi64((__m128i *)&src[src_stride + j]); + c = _mm_loadl_epi64((__m128i *)&src[2 * src_stride + j]); + d = _mm_loadl_epi64((__m128i *)&src[3 * src_stride + j]); + + sum = _mm_cvtepi16_epi32( + _mm_add_epi16(_mm_add_epi16(a, b), _mm_add_epi16(c, d))); + tmp = _mm_unpacklo_epi16(a, b); + tmp2 = _mm_unpacklo_epi16(c, d); + sum_sq = + _mm_add_epi32(_mm_madd_epi16(tmp, tmp), _mm_madd_epi16(tmp2, tmp2)); + + _mm_store_si128((__m128i *)&B[j], sum); + _mm_store_si128((__m128i *)&A[j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[4 * src_stride + j])); + sum = _mm_add_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_add_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[buf_stride + j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[5 * src_stride + j])); + sum = _mm_add_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_add_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[2 * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[2 * buf_stride + j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[6 * src_stride + j])); + sum = _mm_add_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_add_epi32(sum_sq, x2); + + for (i = 3; i < height - 4; ++i) { + _mm_store_si128((__m128i *)&B[i * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[i * buf_stride + j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 3) * src_stride + j])); + y = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[(i + 4) * src_stride + j])); + + sum = _mm_add_epi32(sum, _mm_sub_epi32(y, x)); + + x2 = _mm_mullo_epi32(x, x); + y2 = _mm_mullo_epi32(y, y); + + sum_sq = _mm_add_epi32(sum_sq, _mm_sub_epi32(y2, x2)); + } + _mm_store_si128((__m128i *)&B[i * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[i * buf_stride + j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 3) * src_stride + j])); + sum = _mm_sub_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_sub_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[(i + 1) * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[(i + 1) * buf_stride + j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 2) * src_stride + j])); + sum = _mm_sub_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_sub_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[(i + 2) * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[(i + 2) * buf_stride + j], sum_sq); + + x = _mm_cvtepu16_epi32( + _mm_loadl_epi64((__m128i *)&src[(i - 1) * src_stride + j])); + sum = _mm_sub_epi32(sum, x); + x2 = _mm_mullo_epi32(x, x); + sum_sq = _mm_sub_epi32(sum_sq, x2); + + _mm_store_si128((__m128i *)&B[(i + 3) * buf_stride + j], sum); + _mm_store_si128((__m128i *)&A[(i + 3) * buf_stride + j], sum_sq); + } +} + +void av1_selfguided_restoration_highbd_sse4_1(uint16_t *dgd, int width, + int height, int stride, + int32_t *dst, int dst_stride, + int bit_depth, int r, int eps, + int32_t *tmpbuf) { + int32_t *A = tmpbuf; + int32_t *B = A + SGRPROJ_OUTBUF_SIZE; + int i, j; + // Adjusting the stride of A and B here appears to avoid bad cache effects, + // leading to a significant speed improvement. + // We also align the stride to a multiple of 16 bytes for efficiency. + int buf_stride = ((width + 3) & ~3) + 16; + + // Don't filter tiles with dimensions < 5 on any axis + if ((width < 5) || (height < 5)) return; + + if (r == 1) { + highbd_selfguided_restoration_1_v(dgd, width, height, stride, A, B, + buf_stride); + selfguided_restoration_1_h(A, B, width, height, buf_stride, eps, bit_depth); + } else if (r == 2) { + highbd_selfguided_restoration_2_v(dgd, width, height, stride, A, B, + buf_stride); + selfguided_restoration_2_h(A, B, width, height, buf_stride, eps, bit_depth); + } else if (r == 3) { + highbd_selfguided_restoration_3_v(dgd, width, height, stride, A, B, + buf_stride); + selfguided_restoration_3_h(A, B, width, height, buf_stride, eps, bit_depth); + } else { + assert(0); + } + + { + i = 0; + j = 0; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = 3 * A[k] + 2 * A[k + 1] + 2 * A[k + buf_stride] + + A[k + buf_stride + 1]; + const int32_t b = 3 * B[k] + 2 * B[k + 1] + 2 * B[k + buf_stride] + + B[k + buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + for (j = 1; j < width - 1; ++j) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = A[k] + 2 * (A[k - 1] + A[k + 1]) + A[k + buf_stride] + + A[k + buf_stride - 1] + A[k + buf_stride + 1]; + const int32_t b = B[k] + 2 * (B[k - 1] + B[k + 1]) + B[k + buf_stride] + + B[k + buf_stride - 1] + B[k + buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + j = width - 1; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = 3 * A[k] + 2 * A[k - 1] + 2 * A[k + buf_stride] + + A[k + buf_stride - 1]; + const int32_t b = 3 * B[k] + 2 * B[k - 1] + 2 * B[k + buf_stride] + + B[k + buf_stride - 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + } + for (i = 1; i < height - 1; ++i) { + j = 0; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = A[k] + 2 * (A[k - buf_stride] + A[k + buf_stride]) + + A[k + 1] + A[k - buf_stride + 1] + + A[k + buf_stride + 1]; + const int32_t b = B[k] + 2 * (B[k - buf_stride] + B[k + buf_stride]) + + B[k + 1] + B[k - buf_stride + 1] + + B[k + buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + + // Vectorize the innermost loop + for (j = 1; j < width - 1; j += 4) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 5; + + __m128i tmp0 = _mm_loadu_si128((__m128i *)&A[k - 1 - buf_stride]); + __m128i tmp1 = _mm_loadu_si128((__m128i *)&A[k + 3 - buf_stride]); + __m128i tmp2 = _mm_loadu_si128((__m128i *)&A[k - 1]); + __m128i tmp3 = _mm_loadu_si128((__m128i *)&A[k + 3]); + __m128i tmp4 = _mm_loadu_si128((__m128i *)&A[k - 1 + buf_stride]); + __m128i tmp5 = _mm_loadu_si128((__m128i *)&A[k + 3 + buf_stride]); + + __m128i a0 = _mm_add_epi32( + _mm_add_epi32(_mm_add_epi32(_mm_alignr_epi8(tmp3, tmp2, 4), tmp2), + _mm_add_epi32(_mm_alignr_epi8(tmp3, tmp2, 8), + _mm_alignr_epi8(tmp5, tmp4, 4))), + _mm_alignr_epi8(tmp1, tmp0, 4)); + __m128i a1 = _mm_add_epi32(_mm_add_epi32(tmp0, tmp4), + _mm_add_epi32(_mm_alignr_epi8(tmp1, tmp0, 8), + _mm_alignr_epi8(tmp5, tmp4, 8))); + __m128i a = _mm_sub_epi32(_mm_slli_epi32(_mm_add_epi32(a0, a1), 2), a1); + + __m128i tmp6 = _mm_loadu_si128((__m128i *)&B[k - 1 - buf_stride]); + __m128i tmp7 = _mm_loadu_si128((__m128i *)&B[k + 3 - buf_stride]); + __m128i tmp8 = _mm_loadu_si128((__m128i *)&B[k - 1]); + __m128i tmp9 = _mm_loadu_si128((__m128i *)&B[k + 3]); + __m128i tmp10 = _mm_loadu_si128((__m128i *)&B[k - 1 + buf_stride]); + __m128i tmp11 = _mm_loadu_si128((__m128i *)&B[k + 3 + buf_stride]); + + __m128i b0 = _mm_add_epi32( + _mm_add_epi32(_mm_add_epi32(_mm_alignr_epi8(tmp9, tmp8, 4), tmp8), + _mm_add_epi32(_mm_alignr_epi8(tmp9, tmp8, 8), + _mm_alignr_epi8(tmp11, tmp10, 4))), + _mm_alignr_epi8(tmp7, tmp6, 4)); + __m128i b1 = + _mm_add_epi32(_mm_add_epi32(tmp6, tmp10), + _mm_add_epi32(_mm_alignr_epi8(tmp7, tmp6, 8), + _mm_alignr_epi8(tmp11, tmp10, 8))); + __m128i b = _mm_sub_epi32(_mm_slli_epi32(_mm_add_epi32(b0, b1), 2), b1); + + __m128i src = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)&dgd[l])); + + __m128i rounding = _mm_set1_epi32( + (1 << (SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS)) >> 1); + __m128i v = _mm_add_epi32(_mm_mullo_epi32(a, src), b); + __m128i w = _mm_srai_epi32(_mm_add_epi32(v, rounding), + SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + _mm_storeu_si128((__m128i *)&dst[m], w); + } + + // Deal with any extra pixels at the right-hand edge of the frame + // (typically have 2 such pixels, but may have anywhere between 0 and 3) + for (; j < width - 1; ++j) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 5; + const int32_t a = + (A[k] + A[k - 1] + A[k + 1] + A[k - buf_stride] + A[k + buf_stride]) * + 4 + + (A[k - 1 - buf_stride] + A[k - 1 + buf_stride] + + A[k + 1 - buf_stride] + A[k + 1 + buf_stride]) * + 3; + const int32_t b = + (B[k] + B[k - 1] + B[k + 1] + B[k - buf_stride] + B[k + buf_stride]) * + 4 + + (B[k - 1 - buf_stride] + B[k - 1 + buf_stride] + + B[k + 1 - buf_stride] + B[k + 1 + buf_stride]) * + 3; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + + j = width - 1; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = A[k] + 2 * (A[k - buf_stride] + A[k + buf_stride]) + + A[k - 1] + A[k - buf_stride - 1] + + A[k + buf_stride - 1]; + const int32_t b = B[k] + 2 * (B[k - buf_stride] + B[k + buf_stride]) + + B[k - 1] + B[k - buf_stride - 1] + + B[k + buf_stride - 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + } + + { + i = height - 1; + j = 0; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = 3 * A[k] + 2 * A[k + 1] + 2 * A[k - buf_stride] + + A[k - buf_stride + 1]; + const int32_t b = 3 * B[k] + 2 * B[k + 1] + 2 * B[k - buf_stride] + + B[k - buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + for (j = 1; j < width - 1; ++j) { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = A[k] + 2 * (A[k - 1] + A[k + 1]) + A[k - buf_stride] + + A[k - buf_stride - 1] + A[k - buf_stride + 1]; + const int32_t b = B[k] + 2 * (B[k - 1] + B[k + 1]) + B[k - buf_stride] + + B[k - buf_stride - 1] + B[k - buf_stride + 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + j = width - 1; + { + const int k = i * buf_stride + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int nb = 3; + const int32_t a = 3 * A[k] + 2 * A[k - 1] + 2 * A[k - buf_stride] + + A[k - buf_stride - 1]; + const int32_t b = 3 * B[k] + 2 * B[k - 1] + 2 * B[k - buf_stride] + + B[k - buf_stride - 1]; + const int32_t v = a * dgd[l] + b; + dst[m] = ROUND_POWER_OF_TWO(v, SGRPROJ_SGR_BITS + nb - SGRPROJ_RST_BITS); + } + } +} + +void av1_highpass_filter_highbd_sse4_1(uint16_t *dgd, int width, int height, + int stride, int32_t *dst, int dst_stride, + int corner, int edge) { + int i, j; + const int center = (1 << SGRPROJ_RST_BITS) - 4 * (corner + edge); + + { + i = 0; + j = 0; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k + 1] + dgd[k + stride] + dgd[k] * 2) + + corner * + (dgd[k + stride + 1] + dgd[k + 1] + dgd[k + stride] + dgd[k]); + } + for (j = 1; j < width - 1; ++j) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = center * dgd[k] + + edge * (dgd[k - 1] + dgd[k + stride] + dgd[k + 1] + dgd[k]) + + corner * (dgd[k + stride - 1] + dgd[k + stride + 1] + + dgd[k - 1] + dgd[k + 1]); + } + j = width - 1; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k - 1] + dgd[k + stride] + dgd[k] * 2) + + corner * + (dgd[k + stride - 1] + dgd[k - 1] + dgd[k + stride] + dgd[k]); + } + } + __m128i center_ = _mm_set1_epi32(center); + __m128i edge_ = _mm_set1_epi32(edge); + __m128i corner_ = _mm_set1_epi32(corner); + for (i = 1; i < height - 1; ++i) { + j = 0; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + + edge * (dgd[k - stride] + dgd[k + 1] + dgd[k + stride] + dgd[k]) + + corner * (dgd[k + stride + 1] + dgd[k - stride + 1] + + dgd[k - stride] + dgd[k + stride]); + } + // Process 4 pixels at a time + for (j = 1; j < width - 4; j += 4) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + + __m128i a = _mm_loadu_si128((__m128i *)&dgd[k - stride - 1]); + __m128i b = _mm_loadu_si128((__m128i *)&dgd[k - 1]); + __m128i c = _mm_loadu_si128((__m128i *)&dgd[k + stride - 1]); + + __m128i tl = _mm_cvtepu16_epi32(a); + __m128i tr = _mm_cvtepu16_epi32(_mm_srli_si128(a, 8)); + __m128i cl = _mm_cvtepu16_epi32(b); + __m128i cr = _mm_cvtepu16_epi32(_mm_srli_si128(b, 8)); + __m128i bl = _mm_cvtepu16_epi32(c); + __m128i br = _mm_cvtepu16_epi32(_mm_srli_si128(c, 8)); + + __m128i x = _mm_alignr_epi8(cr, cl, 4); + __m128i y = _mm_add_epi32(_mm_add_epi32(_mm_alignr_epi8(tr, tl, 4), cl), + _mm_add_epi32(_mm_alignr_epi8(br, bl, 4), + _mm_alignr_epi8(cr, cl, 8))); + __m128i z = _mm_add_epi32(_mm_add_epi32(tl, bl), + _mm_add_epi32(_mm_alignr_epi8(tr, tl, 8), + _mm_alignr_epi8(br, bl, 8))); + + __m128i res = _mm_add_epi32(_mm_mullo_epi32(x, center_), + _mm_add_epi32(_mm_mullo_epi32(y, edge_), + _mm_mullo_epi32(z, corner_))); + + _mm_storeu_si128((__m128i *)&dst[l], res); + } + // Handle any leftover pixels + for (; j < width - 1; ++j) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + + edge * (dgd[k - stride] + dgd[k - 1] + dgd[k + stride] + dgd[k + 1]) + + corner * (dgd[k + stride - 1] + dgd[k - stride - 1] + + dgd[k - stride + 1] + dgd[k + stride + 1]); + } + j = width - 1; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + + edge * (dgd[k - stride] + dgd[k - 1] + dgd[k + stride] + dgd[k]) + + corner * (dgd[k + stride - 1] + dgd[k - stride - 1] + + dgd[k - stride] + dgd[k + stride]); + } + } + { + i = height - 1; + j = 0; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k + 1] + dgd[k - stride] + dgd[k] * 2) + + corner * + (dgd[k - stride + 1] + dgd[k + 1] + dgd[k - stride] + dgd[k]); + } + for (j = 1; j < width - 1; ++j) { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = center * dgd[k] + + edge * (dgd[k - 1] + dgd[k - stride] + dgd[k + 1] + dgd[k]) + + corner * (dgd[k - stride - 1] + dgd[k - stride + 1] + + dgd[k - 1] + dgd[k + 1]); + } + j = width - 1; + { + const int k = i * stride + j; + const int l = i * dst_stride + j; + dst[l] = + center * dgd[k] + edge * (dgd[k - 1] + dgd[k - stride] + dgd[k] * 2) + + corner * + (dgd[k - stride - 1] + dgd[k - 1] + dgd[k - stride] + dgd[k]); + } + } +} + +void apply_selfguided_restoration_highbd_sse4_1( + uint16_t *dat, int width, int height, int stride, int bit_depth, int eps, + int *xqd, uint16_t *dst, int dst_stride, int32_t *tmpbuf) { + int xq[2]; + int32_t *flt1 = tmpbuf; + int32_t *flt2 = flt1 + RESTORATION_TILEPELS_MAX; + int32_t *tmpbuf2 = flt2 + RESTORATION_TILEPELS_MAX; + int i, j; + assert(width * height <= RESTORATION_TILEPELS_MAX); +#if USE_HIGHPASS_IN_SGRPROJ + av1_highpass_filter_highbd_sse4_1(dat, width, height, stride, flt1, width, + sgr_params[eps].corner, + sgr_params[eps].edge); +#else + av1_selfguided_restoration_highbd_sse4_1(dat, width, height, stride, flt1, + width, bit_depth, sgr_params[eps].r1, + sgr_params[eps].e1, tmpbuf2); +#endif // USE_HIGHPASS_IN_SGRPROJ + av1_selfguided_restoration_highbd_sse4_1(dat, width, height, stride, flt2, + width, bit_depth, sgr_params[eps].r2, + sgr_params[eps].e2, tmpbuf2); + decode_xq(xqd, xq); + + __m128i xq0 = _mm_set1_epi32(xq[0]); + __m128i xq1 = _mm_set1_epi32(xq[1]); + for (i = 0; i < height; ++i) { + // Calculate output in batches of 8 pixels + for (j = 0; j < width; j += 8) { + const int k = i * width + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + __m128i src = + _mm_slli_epi16(_mm_load_si128((__m128i *)&dat[l]), SGRPROJ_RST_BITS); + + const __m128i u_0 = _mm_cvtepu16_epi32(src); + const __m128i u_1 = _mm_cvtepu16_epi32(_mm_srli_si128(src, 8)); + + const __m128i f1_0 = + _mm_sub_epi32(_mm_loadu_si128((__m128i *)&flt1[k]), u_0); + const __m128i f2_0 = + _mm_sub_epi32(_mm_loadu_si128((__m128i *)&flt2[k]), u_0); + const __m128i f1_1 = + _mm_sub_epi32(_mm_loadu_si128((__m128i *)&flt1[k + 4]), u_1); + const __m128i f2_1 = + _mm_sub_epi32(_mm_loadu_si128((__m128i *)&flt2[k + 4]), u_1); + + const __m128i v_0 = _mm_add_epi32( + _mm_add_epi32(_mm_mullo_epi32(xq0, f1_0), _mm_mullo_epi32(xq1, f2_0)), + _mm_slli_epi32(u_0, SGRPROJ_PRJ_BITS)); + const __m128i v_1 = _mm_add_epi32( + _mm_add_epi32(_mm_mullo_epi32(xq0, f1_1), _mm_mullo_epi32(xq1, f2_1)), + _mm_slli_epi32(u_1, SGRPROJ_PRJ_BITS)); + + const __m128i rounding = + _mm_set1_epi32((1 << (SGRPROJ_PRJ_BITS + SGRPROJ_RST_BITS)) >> 1); + const __m128i w_0 = _mm_srai_epi32(_mm_add_epi32(v_0, rounding), + SGRPROJ_PRJ_BITS + SGRPROJ_RST_BITS); + const __m128i w_1 = _mm_srai_epi32(_mm_add_epi32(v_1, rounding), + SGRPROJ_PRJ_BITS + SGRPROJ_RST_BITS); + + // Pack into 16 bits and clamp to [0, 2^bit_depth) + const __m128i tmp = _mm_packus_epi32(w_0, w_1); + const __m128i max = _mm_set1_epi16((1 << bit_depth) - 1); + const __m128i res = _mm_min_epi16(tmp, max); + + _mm_store_si128((__m128i *)&dst[m], res); + } + // Process leftover pixels + for (; j < width; ++j) { + const int k = i * width + j; + const int l = i * stride + j; + const int m = i * dst_stride + j; + const int32_t u = ((int32_t)dat[l] << SGRPROJ_RST_BITS); + const int32_t f1 = (int32_t)flt1[k] - u; + const int32_t f2 = (int32_t)flt2[k] - u; + const int32_t v = xq[0] * f1 + xq[1] * f2 + (u << SGRPROJ_PRJ_BITS); + const int16_t w = + (int16_t)ROUND_POWER_OF_TWO(v, SGRPROJ_PRJ_BITS + SGRPROJ_RST_BITS); + dst[m] = (uint16_t)clip_pixel_highbd(w, bit_depth); + } + } +} + +#endif diff --git a/third_party/aom/av1/common/x86/warp_plane_sse2.c b/third_party/aom/av1/common/x86/warp_plane_sse2.c new file mode 100644 index 0000000000..925e4650d9 --- /dev/null +++ b/third_party/aom/av1/common/x86/warp_plane_sse2.c @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./av1_rtcd.h" +#include "av1/common/warped_motion.h" + +static const __m128i *const filter = (const __m128i *const)warped_filter; + +/* SSE2 version of the rotzoom/affine warp filter */ +void av1_warp_affine_sse2(int32_t *mat, uint8_t *ref, int width, int height, + int stride, uint8_t *pred, int p_col, int p_row, + int p_width, int p_height, int p_stride, + int subsampling_x, int subsampling_y, int ref_frm, + int16_t alpha, int16_t beta, int16_t gamma, + int16_t delta) { + __m128i tmp[15]; + int i, j, k; + + /* Note: For this code to work, the left/right frame borders need to be + extended by at least 13 pixels each. By the time we get here, other + code will have set up this border, but we allow an explicit check + for debugging purposes. + */ + /*for (i = 0; i < height; ++i) { + for (j = 0; j < 13; ++j) { + assert(ref[i * stride - 13 + j] == ref[i * stride]); + assert(ref[i * stride + width + j] == ref[i * stride + (width - 1)]); + } + }*/ + + for (i = 0; i < p_height; i += 8) { + for (j = 0; j < p_width; j += 8) { + // (x, y) coordinates of the center of this block in the destination + // image + int32_t dst_x = p_col + j + 4; + int32_t dst_y = p_row + i + 4; + + int32_t x4, y4, ix4, sx4, iy4, sy4; + if (subsampling_x) + x4 = ROUND_POWER_OF_TWO_SIGNED( + mat[2] * 2 * dst_x + mat[3] * 2 * dst_y + mat[0] + + (mat[2] + mat[3] - (1 << WARPEDMODEL_PREC_BITS)) / 2, + 1); + else + x4 = mat[2] * dst_x + mat[3] * dst_y + mat[0]; + + if (subsampling_y) + y4 = ROUND_POWER_OF_TWO_SIGNED( + mat[4] * 2 * dst_x + mat[5] * 2 * dst_y + mat[1] + + (mat[4] + mat[5] - (1 << WARPEDMODEL_PREC_BITS)) / 2, + 1); + else + y4 = mat[4] * dst_x + mat[5] * dst_y + mat[1]; + + ix4 = x4 >> WARPEDMODEL_PREC_BITS; + sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + iy4 = y4 >> WARPEDMODEL_PREC_BITS; + sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + + // Horizontal filter + for (k = -7; k < AOMMIN(8, p_height - i); ++k) { + int iy = iy4 + k; + if (iy < 0) + iy = 0; + else if (iy > height - 1) + iy = height - 1; + + // If the block is aligned such that, after clamping, every sample + // would be taken from the leftmost/rightmost column, then we can + // skip the expensive horizontal filter. + if (ix4 <= -7) { + tmp[k + 7] = _mm_set1_epi16( + ref[iy * stride] * + (1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS))); + } else if (ix4 >= width + 6) { + tmp[k + 7] = _mm_set1_epi16( + ref[iy * stride + (width - 1)] * + (1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS))); + } else { + int sx = sx4 + alpha * (-4) + beta * k + + // Include rounding and offset here + (1 << (WARPEDDIFF_PREC_BITS - 1)) + + (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); + + // Load source pixels + __m128i zero = _mm_setzero_si128(); + __m128i src = + _mm_loadu_si128((__m128i *)(ref + iy * stride + ix4 - 7)); + + // Filter even-index pixels + __m128i tmp_0 = _mm_loadu_si128( + (__m128i *)(filter + ((sx + 0 * alpha) >> WARPEDDIFF_PREC_BITS))); + __m128i tmp_2 = _mm_loadu_si128( + (__m128i *)(filter + ((sx + 2 * alpha) >> WARPEDDIFF_PREC_BITS))); + __m128i tmp_4 = _mm_loadu_si128( + (__m128i *)(filter + ((sx + 4 * alpha) >> WARPEDDIFF_PREC_BITS))); + __m128i tmp_6 = _mm_loadu_si128( + (__m128i *)(filter + ((sx + 6 * alpha) >> WARPEDDIFF_PREC_BITS))); + + // coeffs 0 1 0 1 2 3 2 3 for pixels 0, 2 + __m128i tmp_8 = _mm_unpacklo_epi32(tmp_0, tmp_2); + // coeffs 0 1 0 1 2 3 2 3 for pixels 4, 6 + __m128i tmp_10 = _mm_unpacklo_epi32(tmp_4, tmp_6); + // coeffs 4 5 4 5 6 7 6 7 for pixels 0, 2 + __m128i tmp_12 = _mm_unpackhi_epi32(tmp_0, tmp_2); + // coeffs 4 5 4 5 6 7 6 7 for pixels 4, 6 + __m128i tmp_14 = _mm_unpackhi_epi32(tmp_4, tmp_6); + + // coeffs 0 1 0 1 0 1 0 1 for pixels 0, 2, 4, 6 + __m128i coeff_0 = _mm_unpacklo_epi64(tmp_8, tmp_10); + // coeffs 2 3 2 3 2 3 2 3 for pixels 0, 2, 4, 6 + __m128i coeff_2 = _mm_unpackhi_epi64(tmp_8, tmp_10); + // coeffs 4 5 4 5 4 5 4 5 for pixels 0, 2, 4, 6 + __m128i coeff_4 = _mm_unpacklo_epi64(tmp_12, tmp_14); + // coeffs 6 7 6 7 6 7 6 7 for pixels 0, 2, 4, 6 + __m128i coeff_6 = _mm_unpackhi_epi64(tmp_12, tmp_14); + + __m128i round_const = + _mm_set1_epi32((1 << HORSHEAR_REDUCE_PREC_BITS) >> 1); + + // Calculate filtered results + __m128i src_0 = _mm_unpacklo_epi8(src, zero); + __m128i res_0 = _mm_madd_epi16(src_0, coeff_0); + __m128i src_2 = _mm_unpacklo_epi8(_mm_srli_si128(src, 2), zero); + __m128i res_2 = _mm_madd_epi16(src_2, coeff_2); + __m128i src_4 = _mm_unpacklo_epi8(_mm_srli_si128(src, 4), zero); + __m128i res_4 = _mm_madd_epi16(src_4, coeff_4); + __m128i src_6 = _mm_unpacklo_epi8(_mm_srli_si128(src, 6), zero); + __m128i res_6 = _mm_madd_epi16(src_6, coeff_6); + + __m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_4), + _mm_add_epi32(res_2, res_6)); + res_even = _mm_srai_epi32(_mm_add_epi32(res_even, round_const), + HORSHEAR_REDUCE_PREC_BITS); + + // Filter odd-index pixels + __m128i tmp_1 = _mm_loadu_si128( + (__m128i *)(filter + ((sx + 1 * alpha) >> WARPEDDIFF_PREC_BITS))); + __m128i tmp_3 = _mm_loadu_si128( + (__m128i *)(filter + ((sx + 3 * alpha) >> WARPEDDIFF_PREC_BITS))); + __m128i tmp_5 = _mm_loadu_si128( + (__m128i *)(filter + ((sx + 5 * alpha) >> WARPEDDIFF_PREC_BITS))); + __m128i tmp_7 = _mm_loadu_si128( + (__m128i *)(filter + ((sx + 7 * alpha) >> WARPEDDIFF_PREC_BITS))); + + __m128i tmp_9 = _mm_unpacklo_epi32(tmp_1, tmp_3); + __m128i tmp_11 = _mm_unpacklo_epi32(tmp_5, tmp_7); + __m128i tmp_13 = _mm_unpackhi_epi32(tmp_1, tmp_3); + __m128i tmp_15 = _mm_unpackhi_epi32(tmp_5, tmp_7); + + __m128i coeff_1 = _mm_unpacklo_epi64(tmp_9, tmp_11); + __m128i coeff_3 = _mm_unpackhi_epi64(tmp_9, tmp_11); + __m128i coeff_5 = _mm_unpacklo_epi64(tmp_13, tmp_15); + __m128i coeff_7 = _mm_unpackhi_epi64(tmp_13, tmp_15); + + __m128i src_1 = _mm_unpacklo_epi8(_mm_srli_si128(src, 1), zero); + __m128i res_1 = _mm_madd_epi16(src_1, coeff_1); + __m128i src_3 = _mm_unpacklo_epi8(_mm_srli_si128(src, 3), zero); + __m128i res_3 = _mm_madd_epi16(src_3, coeff_3); + __m128i src_5 = _mm_unpacklo_epi8(_mm_srli_si128(src, 5), zero); + __m128i res_5 = _mm_madd_epi16(src_5, coeff_5); + __m128i src_7 = _mm_unpacklo_epi8(_mm_srli_si128(src, 7), zero); + __m128i res_7 = _mm_madd_epi16(src_7, coeff_7); + + __m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_5), + _mm_add_epi32(res_3, res_7)); + res_odd = _mm_srai_epi32(_mm_add_epi32(res_odd, round_const), + HORSHEAR_REDUCE_PREC_BITS); + + // Combine results into one register. + // We store the columns in the order 0, 2, 4, 6, 1, 3, 5, 7 + // as this order helps with the vertical filter. + tmp[k + 7] = _mm_packs_epi32(res_even, res_odd); + } + } + + // Vertical filter + for (k = -4; k < AOMMIN(4, p_height - i - 4); ++k) { + int sy = sy4 + gamma * (-4) + delta * k + + (1 << (WARPEDDIFF_PREC_BITS - 1)) + + (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); + + // Load from tmp and rearrange pairs of consecutive rows into the + // column order 0 0 2 2 4 4 6 6; 1 1 3 3 5 5 7 7 + __m128i *src = tmp + (k + 4); + __m128i src_0 = _mm_unpacklo_epi16(src[0], src[1]); + __m128i src_2 = _mm_unpacklo_epi16(src[2], src[3]); + __m128i src_4 = _mm_unpacklo_epi16(src[4], src[5]); + __m128i src_6 = _mm_unpacklo_epi16(src[6], src[7]); + + // Filter even-index pixels + __m128i tmp_0 = _mm_loadu_si128( + (__m128i *)(filter + ((sy + 0 * gamma) >> WARPEDDIFF_PREC_BITS))); + __m128i tmp_2 = _mm_loadu_si128( + (__m128i *)(filter + ((sy + 2 * gamma) >> WARPEDDIFF_PREC_BITS))); + __m128i tmp_4 = _mm_loadu_si128( + (__m128i *)(filter + ((sy + 4 * gamma) >> WARPEDDIFF_PREC_BITS))); + __m128i tmp_6 = _mm_loadu_si128( + (__m128i *)(filter + ((sy + 6 * gamma) >> WARPEDDIFF_PREC_BITS))); + + __m128i tmp_8 = _mm_unpacklo_epi32(tmp_0, tmp_2); + __m128i tmp_10 = _mm_unpacklo_epi32(tmp_4, tmp_6); + __m128i tmp_12 = _mm_unpackhi_epi32(tmp_0, tmp_2); + __m128i tmp_14 = _mm_unpackhi_epi32(tmp_4, tmp_6); + + __m128i coeff_0 = _mm_unpacklo_epi64(tmp_8, tmp_10); + __m128i coeff_2 = _mm_unpackhi_epi64(tmp_8, tmp_10); + __m128i coeff_4 = _mm_unpacklo_epi64(tmp_12, tmp_14); + __m128i coeff_6 = _mm_unpackhi_epi64(tmp_12, tmp_14); + + __m128i res_0 = _mm_madd_epi16(src_0, coeff_0); + __m128i res_2 = _mm_madd_epi16(src_2, coeff_2); + __m128i res_4 = _mm_madd_epi16(src_4, coeff_4); + __m128i res_6 = _mm_madd_epi16(src_6, coeff_6); + + __m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_2), + _mm_add_epi32(res_4, res_6)); + + // Filter odd-index pixels + __m128i src_1 = _mm_unpackhi_epi16(src[0], src[1]); + __m128i src_3 = _mm_unpackhi_epi16(src[2], src[3]); + __m128i src_5 = _mm_unpackhi_epi16(src[4], src[5]); + __m128i src_7 = _mm_unpackhi_epi16(src[6], src[7]); + + __m128i tmp_1 = _mm_loadu_si128( + (__m128i *)(filter + ((sy + 1 * gamma) >> WARPEDDIFF_PREC_BITS))); + __m128i tmp_3 = _mm_loadu_si128( + (__m128i *)(filter + ((sy + 3 * gamma) >> WARPEDDIFF_PREC_BITS))); + __m128i tmp_5 = _mm_loadu_si128( + (__m128i *)(filter + ((sy + 5 * gamma) >> WARPEDDIFF_PREC_BITS))); + __m128i tmp_7 = _mm_loadu_si128( + (__m128i *)(filter + ((sy + 7 * gamma) >> WARPEDDIFF_PREC_BITS))); + + __m128i tmp_9 = _mm_unpacklo_epi32(tmp_1, tmp_3); + __m128i tmp_11 = _mm_unpacklo_epi32(tmp_5, tmp_7); + __m128i tmp_13 = _mm_unpackhi_epi32(tmp_1, tmp_3); + __m128i tmp_15 = _mm_unpackhi_epi32(tmp_5, tmp_7); + + __m128i coeff_1 = _mm_unpacklo_epi64(tmp_9, tmp_11); + __m128i coeff_3 = _mm_unpackhi_epi64(tmp_9, tmp_11); + __m128i coeff_5 = _mm_unpacklo_epi64(tmp_13, tmp_15); + __m128i coeff_7 = _mm_unpackhi_epi64(tmp_13, tmp_15); + + __m128i res_1 = _mm_madd_epi16(src_1, coeff_1); + __m128i res_3 = _mm_madd_epi16(src_3, coeff_3); + __m128i res_5 = _mm_madd_epi16(src_5, coeff_5); + __m128i res_7 = _mm_madd_epi16(src_7, coeff_7); + + __m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_3), + _mm_add_epi32(res_5, res_7)); + + // Rearrange pixels back into the order 0 ... 7 + __m128i res_lo = _mm_unpacklo_epi32(res_even, res_odd); + __m128i res_hi = _mm_unpackhi_epi32(res_even, res_odd); + + // Round and pack into 8 bits + __m128i round_const = + _mm_set1_epi32((1 << VERSHEAR_REDUCE_PREC_BITS) >> 1); + + __m128i res_lo_round = _mm_srai_epi32( + _mm_add_epi32(res_lo, round_const), VERSHEAR_REDUCE_PREC_BITS); + __m128i res_hi_round = _mm_srai_epi32( + _mm_add_epi32(res_hi, round_const), VERSHEAR_REDUCE_PREC_BITS); + + __m128i res_16bit = _mm_packs_epi32(res_lo_round, res_hi_round); + __m128i res_8bit = _mm_packus_epi16(res_16bit, res_16bit); + + // Store, blending with 'pred' if needed + __m128i *p = (__m128i *)&pred[(i + k + 4) * p_stride + j]; + + // Note: If we're outputting a 4x4 block, we need to be very careful + // to only output 4 pixels at this point, to avoid encode/decode + // mismatches when encoding with multiple threads. + if (p_width == 4) { + if (ref_frm) { + const __m128i orig = _mm_cvtsi32_si128(*(uint32_t *)p); + res_8bit = _mm_avg_epu8(res_8bit, orig); + } + *(uint32_t *)p = _mm_cvtsi128_si32(res_8bit); + } else { + if (ref_frm) res_8bit = _mm_avg_epu8(res_8bit, _mm_loadl_epi64(p)); + _mm_storel_epi64(p, res_8bit); + } + } + } + } +} diff --git a/third_party/aom/av1/common/zigzag.h b/third_party/aom/av1/common/zigzag.h new file mode 100644 index 0000000000..c58b18b571 --- /dev/null +++ b/third_party/aom/av1/common/zigzag.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#if !defined(_zigzag_H) +# define _zigzag_H (1) + +extern const unsigned char OD_ZIGZAG4_DCT_DCT[15][2]; +extern const unsigned char OD_ZIGZAG4_ADST_DCT[15][2]; +extern const unsigned char OD_ZIGZAG4_DCT_ADST[15][2]; +#define OD_ZIGZAG4_ADST_ADST OD_ZIGZAG4_DCT_DCT + +extern const unsigned char OD_ZIGZAG8_DCT_DCT[48][2]; +extern const unsigned char OD_ZIGZAG8_ADST_DCT[48][2]; +extern const unsigned char OD_ZIGZAG8_DCT_ADST[48][2]; +#define OD_ZIGZAG8_ADST_ADST OD_ZIGZAG8_DCT_DCT + +extern const unsigned char OD_ZIGZAG16_DCT_DCT[192][2]; +extern const unsigned char OD_ZIGZAG16_ADST_DCT[192][2]; +extern const unsigned char OD_ZIGZAG16_DCT_ADST[192][2]; +#define OD_ZIGZAG16_ADST_ADST OD_ZIGZAG16_DCT_DCT + +extern const unsigned char OD_ZIGZAG32_DCT_DCT[768][2]; +#endif diff --git a/third_party/aom/av1/common/zigzag16.c b/third_party/aom/av1/common/zigzag16.c new file mode 100644 index 0000000000..6df6e3855f --- /dev/null +++ b/third_party/aom/av1/common/zigzag16.c @@ -0,0 +1,157 @@ +/* This file is generated by gen_zigzag16.m */ + +/* clang-format off */ + +#include "odintrin.h" +OD_EXTERN const unsigned char OD_ZIGZAG16_DCT_DCT[192][2] = { + {8, 0}, {8, 1}, {8, 2}, {9, 0}, + {8, 3}, {9, 1}, {9, 2}, {10, 0}, + {9, 3}, {10, 1}, {10, 2}, {11, 0}, + {10, 3}, {11, 1}, {11, 2}, {11, 3}, + {12, 0}, {12, 1}, {13, 0}, {12, 2}, + {12, 3}, {13, 1}, {13, 2}, {14, 0}, + {13, 3}, {14, 1}, {15, 0}, {14, 2}, + {14, 3}, {15, 1}, {15, 2}, {15, 3}, + {0, 8}, {1, 8}, {0, 9}, {2, 8}, + {1, 9}, {3, 8}, {0, 10}, {2, 9}, + {1, 10}, {3, 9}, {0, 11}, {2, 10}, + {1, 11}, {3, 10}, {0, 12}, {2, 11}, + {1, 12}, {3, 11}, {0, 13}, {2, 12}, + {1, 13}, {0, 14}, {3, 12}, {2, 13}, + {1, 14}, {3, 13}, {0, 15}, {2, 14}, + {1, 15}, {3, 14}, {2, 15}, {3, 15}, + {4, 8}, {5, 8}, {4, 9}, {8, 4}, + {8, 5}, {6, 8}, {5, 9}, {4, 10}, + {9, 4}, {8, 6}, {7, 8}, {9, 5}, + {5, 10}, {8, 7}, {6, 9}, {4, 11}, + {10, 4}, {9, 6}, {7, 9}, {8, 8}, + {10, 5}, {6, 10}, {5, 11}, {9, 7}, + {8, 9}, {10, 6}, {7, 10}, {4, 12}, + {11, 4}, {9, 8}, {6, 11}, {10, 7}, + {11, 5}, {5, 12}, {8, 10}, {7, 11}, + {9, 9}, {4, 13}, {10, 8}, {11, 6}, + {11, 7}, {6, 12}, {8, 11}, {9, 10}, + {12, 4}, {5, 13}, {10, 9}, {12, 5}, + {7, 12}, {11, 8}, {4, 14}, {6, 13}, + {10, 10}, {9, 11}, {12, 6}, {13, 4}, + {11, 9}, {8, 12}, {5, 14}, {12, 7}, + {7, 13}, {4, 15}, {13, 5}, {10, 11}, + {11, 10}, {9, 12}, {13, 6}, {12, 8}, + {6, 14}, {8, 13}, {5, 15}, {13, 7}, + {14, 4}, {12, 9}, {7, 14}, {11, 11}, + {10, 12}, {9, 13}, {14, 5}, {6, 15}, + {13, 8}, {8, 14}, {12, 10}, {14, 6}, + {7, 15}, {13, 9}, {15, 4}, {10, 13}, + {11, 12}, {14, 7}, {9, 14}, {12, 11}, + {8, 15}, {15, 5}, {13, 10}, {14, 8}, + {11, 13}, {15, 6}, {9, 15}, {10, 14}, + {14, 9}, {15, 7}, {13, 11}, {12, 12}, + {10, 15}, {11, 14}, {15, 8}, {14, 10}, + {12, 13}, {13, 12}, {15, 9}, {11, 15}, + {14, 11}, {13, 13}, {15, 10}, {12, 14}, + {13, 14}, {15, 11}, {14, 12}, {12, 15}, + {14, 13}, {13, 15}, {15, 12}, {14, 14}, + {15, 13}, {14, 15}, {15, 14}, {15, 15} + }; + +OD_EXTERN const unsigned char OD_ZIGZAG16_ADST_DCT[192][2] = { + {8, 0}, {9, 0}, {10, 0}, {8, 1}, + {11, 0}, {9, 1}, {8, 2}, {12, 0}, + {10, 1}, {9, 2}, {8, 3}, {13, 0}, + {11, 1}, {10, 2}, {9, 3}, {14, 0}, + {12, 1}, {10, 3}, {15, 0}, {11, 2}, + {13, 1}, {11, 3}, {12, 2}, {14, 1}, + {12, 3}, {13, 2}, {15, 1}, {13, 3}, + {14, 2}, {14, 3}, {15, 2}, {15, 3}, + {0, 8}, {1, 8}, {2, 8}, {0, 9}, + {3, 8}, {1, 9}, {2, 9}, {0, 10}, + {3, 9}, {1, 10}, {2, 10}, {0, 11}, + {3, 10}, {1, 11}, {2, 11}, {0, 12}, + {3, 11}, {1, 12}, {2, 12}, {0, 13}, + {3, 12}, {1, 13}, {0, 14}, {2, 13}, + {0, 15}, {1, 14}, {3, 13}, {2, 14}, + {1, 15}, {3, 14}, {2, 15}, {3, 15}, + {8, 4}, {9, 4}, {8, 5}, {4, 8}, + {10, 4}, {9, 5}, {5, 8}, {8, 6}, + {4, 9}, {10, 5}, {9, 6}, {6, 8}, + {8, 7}, {11, 4}, {7, 8}, {5, 9}, + {9, 7}, {11, 5}, {10, 6}, {4, 10}, + {6, 9}, {8, 8}, {5, 10}, {7, 9}, + {12, 4}, {10, 7}, {9, 8}, {11, 6}, + {8, 9}, {4, 11}, {6, 10}, {7, 10}, + {12, 5}, {5, 11}, {10, 8}, {11, 7}, + {9, 9}, {4, 12}, {13, 4}, {8, 10}, + {6, 11}, {12, 6}, {5, 12}, {10, 9}, + {7, 11}, {9, 10}, {11, 8}, {13, 5}, + {8, 11}, {4, 13}, {6, 12}, {10, 10}, + {12, 7}, {11, 9}, {7, 12}, {14, 4}, + {5, 13}, {9, 11}, {13, 6}, {8, 12}, + {4, 14}, {12, 8}, {6, 13}, {11, 10}, + {10, 11}, {12, 9}, {5, 14}, {13, 7}, + {14, 5}, {9, 12}, {4, 15}, {7, 13}, + {8, 13}, {6, 14}, {13, 8}, {11, 11}, + {10, 12}, {15, 4}, {12, 10}, {14, 6}, + {13, 9}, {5, 15}, {9, 13}, {7, 14}, + {15, 5}, {6, 15}, {8, 14}, {14, 7}, + {11, 12}, {7, 15}, {9, 14}, {13, 10}, + {10, 13}, {14, 8}, {15, 6}, {14, 9}, + {12, 11}, {8, 15}, {15, 7}, {10, 14}, + {11, 13}, {9, 15}, {13, 11}, {12, 12}, + {15, 8}, {14, 10}, {15, 9}, {10, 15}, + {11, 14}, {13, 12}, {12, 13}, {15, 10}, + {14, 11}, {11, 15}, {13, 13}, {15, 11}, + {14, 12}, {12, 14}, {15, 12}, {13, 14}, + {12, 15}, {14, 13}, {13, 15}, {15, 13}, + {14, 14}, {15, 14}, {14, 15}, {15, 15} + }; + +OD_EXTERN const unsigned char OD_ZIGZAG16_DCT_ADST[192][2] = { + {8, 0}, {8, 1}, {8, 2}, {8, 3}, + {9, 0}, {9, 1}, {9, 2}, {9, 3}, + {10, 0}, {10, 1}, {10, 2}, {10, 3}, + {11, 0}, {11, 1}, {11, 2}, {11, 3}, + {12, 0}, {12, 1}, {12, 2}, {12, 3}, + {13, 0}, {13, 1}, {13, 2}, {13, 3}, + {14, 0}, {15, 0}, {14, 1}, {14, 2}, + {14, 3}, {15, 1}, {15, 2}, {15, 3}, + {0, 8}, {0, 9}, {0, 10}, {1, 8}, + {0, 11}, {1, 9}, {2, 8}, {0, 12}, + {1, 10}, {2, 9}, {0, 13}, {1, 11}, + {3, 8}, {2, 10}, {0, 14}, {1, 12}, + {3, 9}, {0, 15}, {2, 11}, {3, 10}, + {1, 13}, {2, 12}, {3, 11}, {1, 14}, + {2, 13}, {1, 15}, {3, 12}, {2, 14}, + {3, 13}, {2, 15}, {3, 14}, {3, 15}, + {4, 8}, {4, 9}, {5, 8}, {4, 10}, + {5, 9}, {4, 11}, {6, 8}, {5, 10}, + {8, 4}, {6, 9}, {4, 12}, {5, 11}, + {8, 5}, {6, 10}, {7, 8}, {8, 6}, + {4, 13}, {7, 9}, {5, 12}, {8, 7}, + {9, 4}, {6, 11}, {8, 8}, {7, 10}, + {5, 13}, {9, 5}, {4, 14}, {9, 6}, + {8, 9}, {6, 12}, {9, 7}, {7, 11}, + {4, 15}, {8, 10}, {9, 8}, {5, 14}, + {10, 4}, {6, 13}, {10, 5}, {9, 9}, + {7, 12}, {8, 11}, {10, 6}, {5, 15}, + {10, 7}, {6, 14}, {9, 10}, {7, 13}, + {8, 12}, {10, 8}, {9, 11}, {6, 15}, + {11, 4}, {11, 5}, {10, 9}, {8, 13}, + {7, 14}, {11, 6}, {9, 12}, {11, 7}, + {10, 10}, {7, 15}, {8, 14}, {12, 4}, + {11, 8}, {12, 5}, {9, 13}, {10, 11}, + {8, 15}, {11, 9}, {12, 6}, {12, 7}, + {10, 12}, {9, 14}, {11, 10}, {13, 4}, + {12, 8}, {9, 15}, {13, 5}, {11, 11}, + {12, 9}, {10, 13}, {13, 6}, {13, 7}, + {12, 10}, {14, 4}, {11, 12}, {13, 8}, + {10, 14}, {14, 5}, {12, 11}, {13, 9}, + {14, 6}, {10, 15}, {11, 13}, {15, 4}, + {14, 7}, {12, 12}, {13, 10}, {14, 8}, + {15, 5}, {13, 11}, {15, 6}, {11, 14}, + {14, 9}, {12, 13}, {11, 15}, {15, 7}, + {14, 10}, {15, 8}, {13, 12}, {12, 14}, + {15, 9}, {14, 11}, {13, 13}, {12, 15}, + {15, 10}, {14, 12}, {13, 14}, {15, 11}, + {13, 15}, {14, 13}, {14, 14}, {15, 12}, + {14, 15}, {15, 13}, {15, 14}, {15, 15} + }; diff --git a/third_party/aom/av1/common/zigzag32.c b/third_party/aom/av1/common/zigzag32.c new file mode 100644 index 0000000000..cb3b9bc632 --- /dev/null +++ b/third_party/aom/av1/common/zigzag32.c @@ -0,0 +1,199 @@ +/* This file is generated by gen_zigzag32.m */ + +/* clang-format off */ + +#include "odintrin.h" +OD_EXTERN const unsigned char OD_ZIGZAG32_DCT_DCT[768][2] = { + { 16, 0 }, { 17, 0 }, { 18, 0 }, { 19, 0 }, + { 16, 1 }, { 17, 1 }, { 20, 0 }, { 16, 2 }, + { 18, 1 }, { 21, 0 }, { 17, 2 }, { 16, 3 }, + { 19, 1 }, { 22, 0 }, { 18, 2 }, { 17, 3 }, + { 20, 1 }, { 16, 4 }, { 23, 0 }, { 19, 2 }, + { 24, 0 }, { 16, 5 }, { 21, 1 }, { 17, 4 }, + { 18, 3 }, { 20, 2 }, { 17, 5 }, { 16, 6 }, + { 19, 3 }, { 18, 4 }, { 25, 0 }, { 22, 1 }, + { 16, 7 }, { 21, 2 }, { 17, 6 }, { 20, 3 }, + { 26, 0 }, { 18, 5 }, { 19, 4 }, { 17, 7 }, + { 23, 1 }, { 22, 2 }, { 18, 6 }, { 27, 0 }, + { 19, 5 }, { 24, 1 }, { 21, 3 }, { 28, 0 }, + { 20, 4 }, { 18, 7 }, { 19, 6 }, { 23, 2 }, + { 29, 0 }, { 25, 1 }, { 21, 4 }, { 30, 0 }, + { 20, 5 }, { 22, 3 }, { 31, 0 }, { 19, 7 }, + { 24, 2 }, { 26, 1 }, { 20, 6 }, { 21, 5 }, + { 22, 4 }, { 23, 3 }, { 27, 1 }, { 25, 2 }, + { 20, 7 }, { 28, 1 }, { 24, 3 }, { 21, 6 }, + { 22, 5 }, { 23, 4 }, { 26, 2 }, { 21, 7 }, + { 29, 1 }, { 25, 3 }, { 30, 1 }, { 27, 2 }, + { 22, 6 }, { 23, 5 }, { 31, 1 }, { 24, 4 }, + { 26, 3 }, { 28, 2 }, { 22, 7 }, { 23, 6 }, + { 25, 4 }, { 24, 5 }, { 29, 2 }, { 30, 2 }, + { 27, 3 }, { 23, 7 }, { 31, 2 }, { 24, 6 }, + { 26, 4 }, { 25, 5 }, { 28, 3 }, { 24, 7 }, + { 27, 4 }, { 29, 3 }, { 25, 6 }, { 26, 5 }, + { 30, 3 }, { 31, 3 }, { 28, 4 }, { 27, 5 }, + { 25, 7 }, { 29, 4 }, { 26, 6 }, { 28, 5 }, + { 30, 4 }, { 26, 7 }, { 27, 6 }, { 31, 4 }, + { 29, 5 }, { 27, 7 }, { 30, 5 }, { 28, 6 }, + { 31, 5 }, { 29, 6 }, { 28, 7 }, { 30, 6 }, + { 31, 6 }, { 29, 7 }, { 30, 7 }, { 31, 7 }, + { 0, 16 }, { 0, 17 }, { 1, 16 }, { 0, 18 }, + { 1, 17 }, { 0, 19 }, { 2, 16 }, { 1, 18 }, + { 0, 20 }, { 2, 17 }, { 3, 16 }, { 1, 19 }, + { 2, 18 }, { 0, 21 }, { 3, 17 }, { 4, 16 }, + { 1, 20 }, { 2, 19 }, { 0, 22 }, { 3, 18 }, + { 4, 17 }, { 5, 16 }, { 0, 23 }, { 3, 19 }, + { 2, 20 }, { 1, 21 }, { 4, 18 }, { 6, 16 }, + { 5, 17 }, { 3, 20 }, { 2, 21 }, { 1, 22 }, + { 0, 24 }, { 0, 25 }, { 4, 19 }, { 7, 16 }, + { 6, 17 }, { 5, 18 }, { 0, 26 }, { 3, 21 }, + { 2, 22 }, { 1, 23 }, { 4, 20 }, { 5, 19 }, + { 6, 18 }, { 1, 24 }, { 7, 17 }, { 0, 27 }, + { 2, 23 }, { 3, 22 }, { 4, 21 }, { 1, 25 }, + { 5, 20 }, { 7, 18 }, { 0, 28 }, { 6, 19 }, + { 2, 24 }, { 1, 26 }, { 0, 29 }, { 4, 22 }, + { 3, 23 }, { 2, 25 }, { 5, 21 }, { 0, 31 }, + { 7, 19 }, { 6, 20 }, { 0, 30 }, { 1, 27 }, + { 3, 24 }, { 2, 26 }, { 4, 23 }, { 5, 22 }, + { 7, 20 }, { 1, 28 }, { 6, 21 }, { 3, 25 }, + { 2, 27 }, { 1, 29 }, { 4, 24 }, { 2, 28 }, + { 1, 30 }, { 7, 21 }, { 5, 23 }, { 3, 26 }, + { 6, 22 }, { 1, 31 }, { 4, 25 }, { 7, 22 }, + { 3, 27 }, { 2, 29 }, { 2, 30 }, { 5, 24 }, + { 2, 31 }, { 6, 23 }, { 4, 26 }, { 3, 28 }, + { 5, 25 }, { 3, 29 }, { 6, 24 }, { 7, 23 }, + { 3, 30 }, { 4, 27 }, { 3, 31 }, { 5, 26 }, + { 6, 25 }, { 4, 28 }, { 7, 24 }, { 4, 29 }, + { 5, 27 }, { 4, 30 }, { 4, 31 }, { 6, 26 }, + { 5, 28 }, { 7, 25 }, { 6, 27 }, { 5, 29 }, + { 7, 26 }, { 5, 30 }, { 5, 31 }, { 6, 28 }, + { 7, 27 }, { 6, 29 }, { 6, 30 }, { 7, 28 }, + { 6, 31 }, { 7, 29 }, { 7, 30 }, { 7, 31 }, + { 8, 16 }, { 9, 16 }, { 8, 17 }, { 10, 16 }, + { 9, 17 }, { 16, 8 }, { 8, 18 }, { 16, 9 }, + { 10, 17 }, { 11, 16 }, { 17, 8 }, { 9, 18 }, + { 8, 19 }, { 16, 10 }, { 11, 17 }, { 12, 16 }, + { 10, 18 }, { 17, 9 }, { 9, 19 }, { 16, 11 }, + { 8, 20 }, { 18, 8 }, { 17, 10 }, { 10, 19 }, + { 12, 17 }, { 11, 18 }, { 9, 20 }, { 16, 12 }, + { 18, 9 }, { 8, 21 }, { 13, 16 }, { 17, 11 }, + { 19, 8 }, { 18, 10 }, { 13, 17 }, { 16, 13 }, + { 11, 19 }, { 12, 18 }, { 10, 20 }, { 17, 12 }, + { 9, 21 }, { 19, 9 }, { 8, 22 }, { 14, 16 }, + { 18, 11 }, { 11, 20 }, { 10, 21 }, { 20, 8 }, + { 13, 18 }, { 16, 14 }, { 12, 19 }, { 17, 13 }, + { 19, 10 }, { 14, 17 }, { 9, 22 }, { 18, 12 }, + { 8, 23 }, { 17, 14 }, { 20, 9 }, { 15, 16 }, + { 16, 15 }, { 13, 19 }, { 10, 22 }, { 19, 11 }, + { 11, 21 }, { 14, 18 }, { 12, 20 }, { 18, 13 }, + { 20, 10 }, { 21, 8 }, { 15, 17 }, { 9, 23 }, + { 19, 12 }, { 11, 22 }, { 8, 24 }, { 21, 9 }, + { 17, 15 }, { 16, 16 }, { 14, 19 }, { 18, 14 }, + { 12, 21 }, { 13, 20 }, { 20, 11 }, { 10, 23 }, + { 19, 13 }, { 15, 18 }, { 16, 17 }, { 21, 10 }, + { 22, 8 }, { 9, 24 }, { 8, 25 }, { 20, 12 }, + { 15, 19 }, { 11, 23 }, { 17, 16 }, { 18, 15 }, + { 14, 20 }, { 12, 22 }, { 10, 24 }, { 22, 9 }, + { 21, 11 }, { 19, 14 }, { 13, 21 }, { 16, 18 }, + { 9, 25 }, { 17, 17 }, { 8, 26 }, { 20, 13 }, + { 23, 8 }, { 12, 23 }, { 13, 22 }, { 22, 10 }, + { 19, 15 }, { 15, 20 }, { 16, 19 }, { 21, 12 }, + { 11, 24 }, { 14, 21 }, { 8, 27 }, { 18, 16 }, + { 10, 25 }, { 9, 26 }, { 22, 11 }, { 20, 14 }, + { 23, 9 }, { 18, 17 }, { 17, 18 }, { 17, 19 }, + { 19, 16 }, { 21, 13 }, { 10, 26 }, { 12, 24 }, + { 23, 10 }, { 24, 8 }, { 8, 28 }, { 16, 20 }, + { 9, 27 }, { 15, 21 }, { 22, 12 }, { 14, 22 }, + { 13, 23 }, { 20, 15 }, { 11, 25 }, { 24, 9 }, + { 18, 18 }, { 19, 17 }, { 23, 11 }, { 10, 27 }, + { 8, 29 }, { 12, 25 }, { 9, 28 }, { 8, 30 }, + { 21, 14 }, { 13, 24 }, { 11, 26 }, { 25, 8 }, + { 24, 10 }, { 20, 16 }, { 19, 18 }, { 14, 23 }, + { 22, 13 }, { 8, 31 }, { 17, 20 }, { 9, 29 }, + { 23, 12 }, { 15, 22 }, { 25, 9 }, { 11, 27 }, + { 10, 28 }, { 20, 17 }, { 21, 15 }, { 18, 19 }, + { 16, 21 }, { 24, 11 }, { 9, 30 }, { 12, 26 }, + { 10, 29 }, { 22, 14 }, { 14, 24 }, { 9, 31 }, + { 26, 8 }, { 13, 25 }, { 25, 10 }, { 18, 20 }, + { 19, 19 }, { 11, 28 }, { 15, 23 }, { 20, 18 }, + { 10, 30 }, { 12, 27 }, { 17, 21 }, { 23, 13 }, + { 24, 12 }, { 21, 16 }, { 16, 22 }, { 26, 9 }, + { 27, 8 }, { 13, 26 }, { 22, 15 }, { 10, 31 }, + { 14, 25 }, { 12, 28 }, { 25, 11 }, { 21, 17 }, + { 26, 10 }, { 20, 19 }, { 11, 29 }, { 15, 24 }, + { 23, 14 }, { 27, 9 }, { 11, 30 }, { 13, 27 }, + { 19, 20 }, { 24, 13 }, { 28, 8 }, { 11, 31 }, + { 22, 16 }, { 17, 22 }, { 16, 23 }, { 25, 12 }, + { 18, 21 }, { 12, 29 }, { 21, 18 }, { 28, 9 }, + { 27, 10 }, { 26, 11 }, { 29, 8 }, { 14, 26 }, + { 15, 25 }, { 13, 28 }, { 12, 30 }, { 23, 15 }, + { 30, 8 }, { 16, 24 }, { 13, 29 }, { 25, 13 }, + { 24, 14 }, { 20, 20 }, { 31, 8 }, { 12, 31 }, + { 14, 27 }, { 28, 10 }, { 26, 12 }, { 22, 17 }, + { 21, 19 }, { 17, 23 }, { 18, 22 }, { 29, 9 }, + { 27, 11 }, { 19, 21 }, { 27, 12 }, { 30, 9 }, + { 31, 9 }, { 13, 30 }, { 24, 15 }, { 23, 16 }, + { 15, 26 }, { 14, 28 }, { 29, 10 }, { 28, 11 }, + { 26, 13 }, { 17, 24 }, { 13, 31 }, { 25, 14 }, + { 22, 18 }, { 16, 25 }, { 30, 10 }, { 14, 29 }, + { 15, 27 }, { 19, 22 }, { 21, 20 }, { 20, 21 }, + { 27, 13 }, { 29, 11 }, { 18, 23 }, { 23, 17 }, + { 16, 26 }, { 31, 10 }, { 24, 16 }, { 14, 30 }, + { 22, 19 }, { 14, 31 }, { 28, 12 }, { 26, 14 }, + { 30, 11 }, { 15, 28 }, { 25, 15 }, { 17, 25 }, + { 23, 18 }, { 18, 24 }, { 15, 30 }, { 29, 12 }, + { 31, 11 }, { 16, 27 }, { 24, 17 }, { 28, 13 }, + { 19, 23 }, { 15, 29 }, { 25, 16 }, { 17, 26 }, + { 27, 14 }, { 22, 20 }, { 15, 31 }, { 20, 22 }, + { 21, 21 }, { 16, 28 }, { 17, 27 }, { 30, 12 }, + { 26, 15 }, { 19, 24 }, { 18, 25 }, { 23, 19 }, + { 29, 13 }, { 31, 12 }, { 24, 18 }, { 26, 16 }, + { 25, 17 }, { 16, 29 }, { 28, 14 }, { 20, 23 }, + { 18, 26 }, { 21, 22 }, { 19, 25 }, { 22, 21 }, + { 27, 15 }, { 17, 28 }, { 16, 30 }, { 26, 17 }, + { 23, 20 }, { 16, 31 }, { 25, 18 }, { 27, 16 }, + { 20, 24 }, { 24, 19 }, { 31, 13 }, { 30, 13 }, + { 29, 14 }, { 18, 27 }, { 28, 15 }, { 17, 29 }, + { 19, 26 }, { 17, 30 }, { 21, 23 }, { 22, 22 }, + { 30, 14 }, { 20, 25 }, { 23, 21 }, { 17, 31 }, + { 18, 28 }, { 25, 19 }, { 24, 20 }, { 28, 16 }, + { 31, 14 }, { 26, 18 }, { 19, 27 }, { 29, 15 }, + { 27, 17 }, { 30, 15 }, { 21, 24 }, { 22, 23 }, + { 26, 19 }, { 23, 22 }, { 28, 17 }, { 29, 16 }, + { 18, 30 }, { 24, 21 }, { 25, 20 }, { 18, 31 }, + { 18, 29 }, { 20, 26 }, { 19, 28 }, { 27, 18 }, + { 31, 15 }, { 20, 27 }, { 30, 16 }, { 19, 29 }, + { 29, 17 }, { 31, 16 }, { 27, 19 }, { 21, 25 }, + { 28, 18 }, { 26, 20 }, { 22, 24 }, { 25, 21 }, + { 19, 30 }, { 24, 22 }, { 30, 17 }, { 21, 26 }, + { 23, 23 }, { 19, 31 }, { 20, 28 }, { 31, 17 }, + { 28, 19 }, { 27, 20 }, { 21, 27 }, { 29, 18 }, + { 30, 18 }, { 25, 22 }, { 26, 21 }, { 20, 29 }, + { 22, 25 }, { 24, 23 }, { 29, 19 }, { 23, 24 }, + { 20, 31 }, { 20, 30 }, { 28, 20 }, { 21, 28 }, + { 22, 26 }, { 31, 18 }, { 27, 21 }, { 30, 19 }, + { 22, 27 }, { 29, 20 }, { 23, 25 }, { 24, 24 }, + { 26, 22 }, { 21, 29 }, { 25, 23 }, { 31, 19 }, + { 21, 30 }, { 23, 26 }, { 28, 21 }, { 21, 31 }, + { 22, 28 }, { 30, 20 }, { 25, 24 }, { 27, 22 }, + { 29, 21 }, { 26, 23 }, { 24, 25 }, { 31, 20 }, + { 23, 27 }, { 22, 29 }, { 30, 21 }, { 28, 22 }, + { 24, 26 }, { 25, 25 }, { 27, 23 }, { 22, 30 }, + { 23, 28 }, { 22, 31 }, { 26, 24 }, { 31, 21 }, + { 24, 27 }, { 29, 22 }, { 27, 24 }, { 30, 22 }, + { 25, 26 }, { 28, 23 }, { 23, 30 }, { 23, 29 }, + { 24, 28 }, { 25, 27 }, { 31, 22 }, { 23, 31 }, + { 26, 25 }, { 28, 24 }, { 29, 23 }, { 24, 29 }, + { 24, 30 }, { 27, 25 }, { 25, 28 }, { 26, 26 }, + { 30, 23 }, { 26, 27 }, { 31, 23 }, { 28, 25 }, + { 27, 26 }, { 25, 29 }, { 24, 31 }, { 29, 24 }, + { 30, 24 }, { 27, 27 }, { 29, 25 }, { 26, 28 }, + { 31, 24 }, { 25, 30 }, { 25, 31 }, { 28, 26 }, + { 27, 28 }, { 26, 29 }, { 30, 25 }, { 29, 26 }, + { 28, 27 }, { 26, 30 }, { 31, 25 }, { 27, 29 }, + { 26, 31 }, { 30, 26 }, { 28, 28 }, { 31, 26 }, + { 29, 27 }, { 27, 30 }, { 28, 29 }, { 27, 31 }, + { 30, 27 }, { 31, 27 }, { 28, 30 }, { 29, 28 }, + { 30, 28 }, { 29, 29 }, { 30, 29 }, { 31, 28 }, + { 28, 31 }, { 29, 30 }, { 29, 31 }, { 31, 29 }, + { 30, 30 }, { 30, 31 }, { 31, 30 }, { 31, 31 } +}; diff --git a/third_party/aom/av1/common/zigzag4.c b/third_party/aom/av1/common/zigzag4.c new file mode 100644 index 0000000000..1fb5a320b7 --- /dev/null +++ b/third_party/aom/av1/common/zigzag4.c @@ -0,0 +1,22 @@ +/* This file is generated by gen_zigzag4.m */ + +/* clang-format off */ + +#include "odintrin.h" +OD_EXTERN const unsigned char OD_ZIGZAG4_DCT_DCT[15][2] = { + {0, 1}, {1, 0}, {1, 1}, {0, 2}, + {2, 0}, {0, 3}, {1, 2}, {3, 0}, + {2, 1}, {1, 3}, {2, 2}, {3, 1}, + {2, 3}, {3, 2}, {3, 3} }; + +OD_EXTERN const unsigned char OD_ZIGZAG4_ADST_DCT[15][2] = { + {1, 0}, {0, 1}, {2, 0}, {1, 1}, + {3, 0}, {2, 1}, {0, 2}, {1, 2}, + {3, 1}, {0, 3}, {2, 2}, {1, 3}, + {3, 2}, {2, 3}, {3, 3} }; + +OD_EXTERN const unsigned char OD_ZIGZAG4_DCT_ADST[15][2] = { + {0, 1}, {0, 2}, {1, 0}, {0, 3}, + {1, 1}, {1, 2}, {2, 0}, {1, 3}, + {2, 1}, {2, 2}, {3, 0}, {3, 1}, + {2, 3}, {3, 2}, {3, 3} }; diff --git a/third_party/aom/av1/common/zigzag8.c b/third_party/aom/av1/common/zigzag8.c new file mode 100644 index 0000000000..3f11e0c035 --- /dev/null +++ b/third_party/aom/av1/common/zigzag8.c @@ -0,0 +1,50 @@ +/* This file is generated by gen_zigzag8.m */ + +/* clang-format off */ + +#include "odintrin.h" + +OD_EXTERN const unsigned char OD_ZIGZAG8_DCT_DCT[48][2] = { + {4, 0}, {4, 1}, {5, 0}, {5, 1}, + {6, 0}, {7, 0}, {6, 1}, {7, 1}, + {0, 4}, {1, 4}, {0, 5}, {1, 5}, + {0, 6}, {1, 6}, {0, 7}, {1, 7}, + {2, 4}, {4, 2}, {3, 4}, {2, 5}, + {4, 3}, {5, 2}, {4, 4}, {3, 5}, + {5, 3}, {2, 6}, {4, 5}, {6, 2}, + {5, 4}, {3, 6}, {2, 7}, {6, 3}, + {5, 5}, {7, 2}, {4, 6}, {3, 7}, + {6, 4}, {7, 3}, {4, 7}, {5, 6}, + {6, 5}, {7, 4}, {5, 7}, {6, 6}, + {7, 5}, {6, 7}, {7, 6}, {7, 7} + }; + +OD_EXTERN const unsigned char OD_ZIGZAG8_ADST_DCT[48][2] = { + {4, 0}, {5, 0}, {4, 1}, {6, 0}, + {5, 1}, {7, 0}, {6, 1}, {7, 1}, + {0, 4}, {1, 4}, {0, 5}, {1, 5}, + {0, 6}, {1, 6}, {0, 7}, {1, 7}, + {4, 2}, {2, 4}, {5, 2}, {4, 3}, + {3, 4}, {2, 5}, {5, 3}, {4, 4}, + {6, 2}, {3, 5}, {5, 4}, {2, 6}, + {4, 5}, {6, 3}, {7, 2}, {3, 6}, + {2, 7}, {5, 5}, {6, 4}, {4, 6}, + {7, 3}, {3, 7}, {5, 6}, {6, 5}, + {4, 7}, {7, 4}, {5, 7}, {7, 5}, + {6, 6}, {7, 6}, {6, 7}, {7, 7} + }; + +OD_EXTERN const unsigned char OD_ZIGZAG8_DCT_ADST[48][2] = { + {4, 0}, {4, 1}, {5, 0}, {5, 1}, + {6, 0}, {6, 1}, {7, 0}, {7, 1}, + {0, 4}, {0, 5}, {1, 4}, {0, 6}, + {1, 5}, {0, 7}, {1, 6}, {1, 7}, + {2, 4}, {2, 5}, {3, 4}, {4, 2}, + {2, 6}, {4, 3}, {3, 5}, {4, 4}, + {2, 7}, {3, 6}, {5, 2}, {4, 5}, + {5, 3}, {3, 7}, {5, 4}, {4, 6}, + {6, 2}, {5, 5}, {4, 7}, {6, 3}, + {6, 4}, {5, 6}, {7, 2}, {6, 5}, + {7, 3}, {5, 7}, {7, 4}, {6, 6}, + {7, 5}, {6, 7}, {7, 6}, {7, 7} + }; diff --git a/third_party/aom/av1/decoder/accounting.c b/third_party/aom/av1/decoder/accounting.c new file mode 100644 index 0000000000..ba243c9e1f --- /dev/null +++ b/third_party/aom/av1/decoder/accounting.c @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include + +#include "aom/aom_integer.h" +#include "./accounting.h" + +static int aom_accounting_hash(const char *str) { + uint32_t val; + const unsigned char *ustr; + val = 0; + ustr = (const unsigned char *)str; + /* This is about the worst hash one can design, but it should be good enough + here. */ + while (*ustr) val += *ustr++; + return val % AOM_ACCOUNTING_HASH_SIZE; +} + +/* Dictionary lookup based on an open-addressing hash table. */ +int aom_accounting_dictionary_lookup(Accounting *accounting, const char *str) { + int hash; + int len; + AccountingDictionary *dictionary; + dictionary = &accounting->syms.dictionary; + hash = aom_accounting_hash(str); + while (accounting->hash_dictionary[hash] != -1) { + if (strcmp(dictionary->strs[accounting->hash_dictionary[hash]], str) == 0) { + return accounting->hash_dictionary[hash]; + } + hash++; + if (hash == AOM_ACCOUNTING_HASH_SIZE) hash = 0; + } + /* No match found. */ + assert(dictionary->num_strs + 1 < MAX_SYMBOL_TYPES); + accounting->hash_dictionary[hash] = dictionary->num_strs; + len = strlen(str); + dictionary->strs[dictionary->num_strs] = malloc(len + 1); + snprintf(dictionary->strs[dictionary->num_strs], len + 1, "%s", str); + dictionary->num_strs++; + return dictionary->num_strs - 1; +} + +void aom_accounting_init(Accounting *accounting) { + int i; + accounting->num_syms_allocated = 1000; + accounting->syms.syms = + malloc(sizeof(AccountingSymbol) * accounting->num_syms_allocated); + accounting->syms.dictionary.num_strs = 0; + assert(AOM_ACCOUNTING_HASH_SIZE > 2 * MAX_SYMBOL_TYPES); + for (i = 0; i < AOM_ACCOUNTING_HASH_SIZE; i++) + accounting->hash_dictionary[i] = -1; + aom_accounting_reset(accounting); +} + +void aom_accounting_reset(Accounting *accounting) { + accounting->syms.num_syms = 0; + accounting->syms.num_binary_syms = 0; + accounting->syms.num_multi_syms = 0; + accounting->context.x = -1; + accounting->context.y = -1; + accounting->last_tell_frac = 0; +} + +void aom_accounting_clear(Accounting *accounting) { + int i; + AccountingDictionary *dictionary; + free(accounting->syms.syms); + dictionary = &accounting->syms.dictionary; + for (i = 0; i < dictionary->num_strs; i++) { + free(dictionary->strs[i]); + } +} + +void aom_accounting_set_context(Accounting *accounting, int16_t x, int16_t y) { + accounting->context.x = x; + accounting->context.y = y; +} + +void aom_accounting_record(Accounting *accounting, const char *str, + uint32_t bits) { + AccountingSymbol sym; + // Reuse previous symbol if it has the same context and symbol id. + if (accounting->syms.num_syms) { + AccountingSymbol *last_sym; + last_sym = &accounting->syms.syms[accounting->syms.num_syms - 1]; + if (memcmp(&last_sym->context, &accounting->context, + sizeof(AccountingSymbolContext)) == 0) { + uint32_t id; + id = aom_accounting_dictionary_lookup(accounting, str); + if (id == last_sym->id) { + last_sym->bits += bits; + last_sym->samples++; + return; + } + } + } + sym.context = accounting->context; + sym.samples = 1; + sym.bits = bits; + sym.id = aom_accounting_dictionary_lookup(accounting, str); + assert(sym.id <= 255); + if (accounting->syms.num_syms == accounting->num_syms_allocated) { + accounting->num_syms_allocated *= 2; + accounting->syms.syms = + realloc(accounting->syms.syms, + sizeof(AccountingSymbol) * accounting->num_syms_allocated); + assert(accounting->syms.syms != NULL); + } + accounting->syms.syms[accounting->syms.num_syms++] = sym; +} + +void aom_accounting_dump(Accounting *accounting) { + int i; + AccountingSymbol *sym; + printf("\n----- Number of recorded syntax elements = %d -----\n", + accounting->syms.num_syms); + printf("----- Total number of symbol calls = %d (%d binary) -----\n", + accounting->syms.num_multi_syms + accounting->syms.num_binary_syms, + accounting->syms.num_binary_syms); + for (i = 0; i < accounting->syms.num_syms; i++) { + sym = &accounting->syms.syms[i]; + printf("%s x: %d, y: %d bits: %f samples: %d\n", + accounting->syms.dictionary.strs[sym->id], sym->context.x, + sym->context.y, (float)sym->bits / 8.0, sym->samples); + } +} diff --git a/third_party/aom/av1/decoder/accounting.h b/third_party/aom/av1/decoder/accounting.h new file mode 100644 index 0000000000..889865b2e7 --- /dev/null +++ b/third_party/aom/av1/decoder/accounting.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_ACCOUNTING_H_ +#define AOM_ACCOUNTING_H_ +#include + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +#define AOM_ACCOUNTING_HASH_SIZE (1021) + +/* Max number of entries for symbol types in the dictionary (increase as + necessary). */ +#define MAX_SYMBOL_TYPES (256) + +/*The resolution of fractional-precision bit usage measurements, i.e., + 3 => 1/8th bits.*/ +#define AOM_ACCT_BITRES (3) + +typedef struct { + int16_t x; + int16_t y; +} AccountingSymbolContext; + +typedef struct { + AccountingSymbolContext context; + uint32_t id; + /** Number of bits in units of 1/8 bit. */ + uint32_t bits; + uint32_t samples; +} AccountingSymbol; + +/** Dictionary for translating strings into id. */ +typedef struct { + char *(strs[MAX_SYMBOL_TYPES]); + int num_strs; +} AccountingDictionary; + +typedef struct { + /** All recorded symbols decoded. */ + AccountingSymbol *syms; + /** Number of syntax actually recorded. */ + int num_syms; + /** Raw symbol decoding calls for non-binary values. */ + int num_multi_syms; + /** Raw binary symbol decoding calls. */ + int num_binary_syms; + /** Dictionary for translating strings into id. */ + AccountingDictionary dictionary; +} AccountingSymbols; + +typedef struct Accounting Accounting; + +struct Accounting { + AccountingSymbols syms; + /** Size allocated for symbols (not all may be used). */ + int num_syms_allocated; + int16_t hash_dictionary[AOM_ACCOUNTING_HASH_SIZE]; + AccountingSymbolContext context; + uint32_t last_tell_frac; +}; + +void aom_accounting_init(Accounting *accounting); +void aom_accounting_reset(Accounting *accounting); +void aom_accounting_clear(Accounting *accounting); +void aom_accounting_set_context(Accounting *accounting, int16_t x, int16_t y); +int aom_accounting_dictionary_lookup(Accounting *accounting, const char *str); +void aom_accounting_record(Accounting *accounting, const char *str, + uint32_t bits); +void aom_accounting_dump(Accounting *accounting); +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus +#endif // AOM_ACCOUNTING_H_ diff --git a/third_party/aom/av1/decoder/decint.h b/third_party/aom/av1/decoder/decint.h new file mode 100644 index 0000000000..e887ad5e0d --- /dev/null +++ b/third_party/aom/av1/decoder/decint.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#if !defined(_decint_H) +# define _decint_H (1) +# include "av1/common/pvq_state.h" +# include "aom_dsp/bitreader.h" +# include "aom_dsp/entdec.h" + +typedef struct daala_dec_ctx daala_dec_ctx; + +typedef struct daala_dec_ctx od_dec_ctx; + + +struct daala_dec_ctx { + /* Stores context-adaptive CDFs for PVQ. */ + od_state state; + /* AOM entropy decoder. */ + aom_reader *r; + int use_activity_masking; + /* Mode of quantization matrice : FLAT (0) or HVS (1) */ + int qm; +}; + +#endif diff --git a/third_party/aom/av1/decoder/decodeframe.c b/third_party/aom/av1/decoder/decodeframe.c new file mode 100644 index 0000000000..289d386705 --- /dev/null +++ b/third_party/aom/av1/decoder/decodeframe.c @@ -0,0 +1,5159 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include // qsort() + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "./aom_scale_rtcd.h" +#include "./av1_rtcd.h" + +#include "aom/aom_codec.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/bitreader.h" +#include "aom_dsp/bitreader_buffer.h" +#include "aom_dsp/binary_codes_reader.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" +#include "aom_ports/mem_ops.h" +#include "aom_scale/aom_scale.h" +#include "aom_util/aom_thread.h" + +#if CONFIG_BITSTREAM_DEBUG +#include "aom_util/debug_util.h" +#endif // CONFIG_BITSTREAM_DEBUG + +#include "av1/common/alloccommon.h" +#if CONFIG_CDEF +#include "av1/common/cdef.h" +#include "av1/common/clpf.h" +#endif +#if CONFIG_INSPECTION +#include "av1/decoder/inspection.h" +#endif +#include "av1/common/common.h" +#include "av1/common/entropy.h" +#include "av1/common/entropymode.h" +#include "av1/common/entropymv.h" +#include "av1/common/idct.h" +#include "av1/common/pred_common.h" +#include "av1/common/quant_common.h" +#include "av1/common/reconinter.h" +#include "av1/common/reconintra.h" +#include "av1/common/seg_common.h" +#include "av1/common/thread_common.h" +#include "av1/common/tile_common.h" + +#include "av1/decoder/decodeframe.h" +#include "av1/decoder/decodemv.h" +#include "av1/decoder/decoder.h" +#if CONFIG_LV_MAP +#include "av1/decoder/decodetxb.h" +#endif +#include "av1/decoder/detokenize.h" +#include "av1/decoder/dsubexp.h" + +#if CONFIG_WARPED_MOTION || CONFIG_GLOBAL_MOTION +#include "av1/common/warped_motion.h" +#endif // CONFIG_WARPED_MOTION || CONFIG_GLOBAL_MOTION + +#define MAX_AV1_HEADER_SIZE 80 +#define ACCT_STR __func__ + +#if CONFIG_PVQ +#include "av1/common/partition.h" +#include "av1/common/pvq.h" +#include "av1/common/scan.h" +#include "av1/decoder/decint.h" +#include "av1/decoder/pvq_decoder.h" +#include "av1/encoder/encodemb.h" +#include "av1/encoder/hybrid_fwd_txfm.h" +#endif + +#if CONFIG_CFL +#include "av1/common/cfl.h" +#endif + +static struct aom_read_bit_buffer *init_read_bit_buffer( + AV1Decoder *pbi, struct aom_read_bit_buffer *rb, const uint8_t *data, + const uint8_t *data_end, uint8_t clear_data[MAX_AV1_HEADER_SIZE]); +static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data, + size_t partition_size); +static size_t read_uncompressed_header(AV1Decoder *pbi, + struct aom_read_bit_buffer *rb); + +static int is_compound_reference_allowed(const AV1_COMMON *cm) { +#if CONFIG_LOWDELAY_COMPOUND // Normative in decoder + return !frame_is_intra_only(cm); +#else + int i; + if (frame_is_intra_only(cm)) return 0; + for (i = 1; i < INTER_REFS_PER_FRAME; ++i) + if (cm->ref_frame_sign_bias[i + 1] != cm->ref_frame_sign_bias[1]) return 1; + + return 0; +#endif +} + +static void setup_compound_reference_mode(AV1_COMMON *cm) { +#if CONFIG_EXT_REFS + cm->comp_fwd_ref[0] = LAST_FRAME; + cm->comp_fwd_ref[1] = LAST2_FRAME; + cm->comp_fwd_ref[2] = LAST3_FRAME; + cm->comp_fwd_ref[3] = GOLDEN_FRAME; + + cm->comp_bwd_ref[0] = BWDREF_FRAME; + cm->comp_bwd_ref[1] = ALTREF_FRAME; +#else + if (cm->ref_frame_sign_bias[LAST_FRAME] == + cm->ref_frame_sign_bias[GOLDEN_FRAME]) { + cm->comp_fixed_ref = ALTREF_FRAME; + cm->comp_var_ref[0] = LAST_FRAME; + cm->comp_var_ref[1] = GOLDEN_FRAME; + } else if (cm->ref_frame_sign_bias[LAST_FRAME] == + cm->ref_frame_sign_bias[ALTREF_FRAME]) { + cm->comp_fixed_ref = GOLDEN_FRAME; + cm->comp_var_ref[0] = LAST_FRAME; + cm->comp_var_ref[1] = ALTREF_FRAME; + } else { + cm->comp_fixed_ref = LAST_FRAME; + cm->comp_var_ref[0] = GOLDEN_FRAME; + cm->comp_var_ref[1] = ALTREF_FRAME; + } +#endif // CONFIG_EXT_REFS +} + +static int read_is_valid(const uint8_t *start, size_t len, const uint8_t *end) { + return len != 0 && len <= (size_t)(end - start); +} + +static int decode_unsigned_max(struct aom_read_bit_buffer *rb, int max) { + const int data = aom_rb_read_literal(rb, get_unsigned_bits(max)); + return data > max ? max : data; +} + +static TX_MODE read_tx_mode(AV1_COMMON *cm, MACROBLOCKD *xd, + struct aom_read_bit_buffer *rb) { + int i, all_lossless = 1; +#if CONFIG_TX64X64 + TX_MODE tx_mode; +#endif + + if (cm->seg.enabled) { + for (i = 0; i < MAX_SEGMENTS; ++i) { + if (!xd->lossless[i]) { + all_lossless = 0; + break; + } + } + } else { + all_lossless = xd->lossless[0]; + } + + if (all_lossless) return ONLY_4X4; +#if CONFIG_TX64X64 + tx_mode = aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2); + if (tx_mode == ALLOW_32X32) tx_mode += aom_rb_read_bit(rb); + return tx_mode; +#else + return aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2); +#endif // CONFIG_TX64X64 +} + +#if !CONFIG_EC_ADAPT +static void read_tx_size_probs(FRAME_CONTEXT *fc, aom_reader *r) { + int i, j, k; + for (i = 0; i < MAX_TX_DEPTH; ++i) + for (j = 0; j < TX_SIZE_CONTEXTS; ++j) + for (k = 0; k < i + 1; ++k) + av1_diff_update_prob(r, &fc->tx_size_probs[i][j][k], ACCT_STR); +} +#endif + +#if !CONFIG_EC_ADAPT +static void read_switchable_interp_probs(FRAME_CONTEXT *fc, aom_reader *r) { + int i, j; + for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) { + for (i = 0; i < SWITCHABLE_FILTERS - 1; ++i) + av1_diff_update_prob(r, &fc->switchable_interp_prob[j][i], ACCT_STR); + } +} +#endif + +static void read_inter_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) { +#if CONFIG_REF_MV + int i; + for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i) + av1_diff_update_prob(r, &fc->newmv_prob[i], ACCT_STR); + for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i) + av1_diff_update_prob(r, &fc->zeromv_prob[i], ACCT_STR); + for (i = 0; i < REFMV_MODE_CONTEXTS; ++i) + av1_diff_update_prob(r, &fc->refmv_prob[i], ACCT_STR); + for (i = 0; i < DRL_MODE_CONTEXTS; ++i) + av1_diff_update_prob(r, &fc->drl_prob[i], ACCT_STR); +#else +#if !CONFIG_EC_ADAPT + int i, j; + for (i = 0; i < INTER_MODE_CONTEXTS; ++i) { + for (j = 0; j < INTER_MODES - 1; ++j) + av1_diff_update_prob(r, &fc->inter_mode_probs[i][j], ACCT_STR); + } +#else + (void)fc; + (void)r; +#endif +#endif +} + +#if CONFIG_EXT_INTER +static void read_inter_compound_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) { + int i, j; + if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) { + for (j = 0; j < INTER_MODE_CONTEXTS; ++j) { + for (i = 0; i < INTER_COMPOUND_MODES - 1; ++i) { + av1_diff_update_prob(r, &fc->inter_compound_mode_probs[j][i], ACCT_STR); + } + } + } +} +#endif // CONFIG_EXT_INTER +#if !CONFIG_EC_ADAPT +#if !CONFIG_EXT_TX +static void read_ext_tx_probs(FRAME_CONTEXT *fc, aom_reader *r) { + int i, j, k; + if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) { + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + for (j = 0; j < TX_TYPES; ++j) { + for (k = 0; k < TX_TYPES - 1; ++k) + av1_diff_update_prob(r, &fc->intra_ext_tx_prob[i][j][k], ACCT_STR); + } + } + } + if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) { + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + for (k = 0; k < TX_TYPES - 1; ++k) + av1_diff_update_prob(r, &fc->inter_ext_tx_prob[i][k], ACCT_STR); + } + } +} +#endif +#endif + +static REFERENCE_MODE read_frame_reference_mode( + const AV1_COMMON *cm, struct aom_read_bit_buffer *rb) { + if (is_compound_reference_allowed(cm)) { +#if CONFIG_REF_ADAPT + return aom_rb_read_bit(rb) ? REFERENCE_MODE_SELECT : SINGLE_REFERENCE; +#else + return aom_rb_read_bit(rb) + ? REFERENCE_MODE_SELECT + : (aom_rb_read_bit(rb) ? COMPOUND_REFERENCE : SINGLE_REFERENCE); +#endif // CONFIG_REF_ADAPT + } else { + return SINGLE_REFERENCE; + } +} + +static void read_frame_reference_mode_probs(AV1_COMMON *cm, aom_reader *r) { + FRAME_CONTEXT *const fc = cm->fc; + int i, j; + + if (cm->reference_mode == REFERENCE_MODE_SELECT) + for (i = 0; i < COMP_INTER_CONTEXTS; ++i) + av1_diff_update_prob(r, &fc->comp_inter_prob[i], ACCT_STR); + + if (cm->reference_mode != COMPOUND_REFERENCE) { + for (i = 0; i < REF_CONTEXTS; ++i) { + for (j = 0; j < (SINGLE_REFS - 1); ++j) { + av1_diff_update_prob(r, &fc->single_ref_prob[i][j], ACCT_STR); + } + } + } + + if (cm->reference_mode != SINGLE_REFERENCE) { + for (i = 0; i < REF_CONTEXTS; ++i) { +#if CONFIG_EXT_REFS + for (j = 0; j < (FWD_REFS - 1); ++j) + av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR); + for (j = 0; j < (BWD_REFS - 1); ++j) + av1_diff_update_prob(r, &fc->comp_bwdref_prob[i][j], ACCT_STR); +#else + for (j = 0; j < (COMP_REFS - 1); ++j) + av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR); +#endif // CONFIG_EXT_REFS + } + } +} + +static void update_mv_probs(aom_prob *p, int n, aom_reader *r) { + int i; + for (i = 0; i < n; ++i) av1_diff_update_prob(r, &p[i], ACCT_STR); +} + +static void read_mv_probs(nmv_context *ctx, int allow_hp, aom_reader *r) { + int i; + +#if !CONFIG_EC_ADAPT + int j; + update_mv_probs(ctx->joints, MV_JOINTS - 1, r); + + for (i = 0; i < 2; ++i) { + nmv_component *const comp_ctx = &ctx->comps[i]; + update_mv_probs(&comp_ctx->sign, 1, r); + update_mv_probs(comp_ctx->classes, MV_CLASSES - 1, r); + update_mv_probs(comp_ctx->class0, CLASS0_SIZE - 1, r); + update_mv_probs(comp_ctx->bits, MV_OFFSET_BITS, r); + } + for (i = 0; i < 2; ++i) { + nmv_component *const comp_ctx = &ctx->comps[i]; + for (j = 0; j < CLASS0_SIZE; ++j) { + update_mv_probs(comp_ctx->class0_fp[j], MV_FP_SIZE - 1, r); + } + update_mv_probs(comp_ctx->fp, MV_FP_SIZE - 1, r); + } +#endif // !CONFIG_EC_ADAPT + + if (allow_hp) { + for (i = 0; i < 2; ++i) { + nmv_component *const comp_ctx = &ctx->comps[i]; + update_mv_probs(&comp_ctx->class0_hp, 1, r); + update_mv_probs(&comp_ctx->hp, 1, r); + } + } +} + +static void inverse_transform_block(MACROBLOCKD *xd, int plane, + const TX_TYPE tx_type, + const TX_SIZE tx_size, uint8_t *dst, + int stride, int16_t scan_line, int eob) { + struct macroblockd_plane *const pd = &xd->plane[plane]; + tran_low_t *const dqcoeff = pd->dqcoeff; + av1_inverse_transform_block(xd, dqcoeff, tx_type, tx_size, dst, stride, eob); + memset(dqcoeff, 0, (scan_line + 1) * sizeof(dqcoeff[0])); +} + +#if CONFIG_PVQ +static int av1_pvq_decode_helper(MACROBLOCKD *xd, tran_low_t *ref_coeff, + tran_low_t *dqcoeff, int16_t *quant, int pli, + int bs, TX_TYPE tx_type, int xdec, + PVQ_SKIP_TYPE ac_dc_coded) { + unsigned int flags; // used for daala's stream analyzer. + int off; + const int is_keyframe = 0; + const int has_dc_skip = 1; + int coeff_shift = 3 - av1_get_tx_scale(bs); + int hbd_downshift = 0; + int rounding_mask; + // DC quantizer for PVQ + int pvq_dc_quant; + int lossless = (quant[0] == 0); + const int blk_size = tx_size_wide[bs]; + int eob = 0; + int i; + od_dec_ctx *dec = &xd->daala_dec; + int use_activity_masking = dec->use_activity_masking; + DECLARE_ALIGNED(16, tran_low_t, dqcoeff_pvq[OD_TXSIZE_MAX * OD_TXSIZE_MAX]); + DECLARE_ALIGNED(16, tran_low_t, ref_coeff_pvq[OD_TXSIZE_MAX * OD_TXSIZE_MAX]); + + od_coeff ref_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX]; + od_coeff out_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX]; + +#if CONFIG_HIGHBITDEPTH + hbd_downshift = xd->bd - 8; +#endif // CONFIG_HIGHBITDEPTH + + od_raster_to_coding_order(ref_coeff_pvq, blk_size, tx_type, ref_coeff, + blk_size); + + assert(OD_COEFF_SHIFT >= 4); + if (lossless) + pvq_dc_quant = 1; + else { + if (use_activity_masking) + pvq_dc_quant = OD_MAXI( + 1, (quant[0] << (OD_COEFF_SHIFT - 3) >> hbd_downshift) * + dec->state.pvq_qm_q4[pli][od_qm_get_index(bs, 0)] >> + 4); + else + pvq_dc_quant = + OD_MAXI(1, quant[0] << (OD_COEFF_SHIFT - 3) >> hbd_downshift); + } + + off = od_qm_offset(bs, xdec); + + // copy int16 inputs to int32 + for (i = 0; i < blk_size * blk_size; i++) { + ref_int32[i] = + AOM_SIGNED_SHL(ref_coeff_pvq[i], OD_COEFF_SHIFT - coeff_shift) >> + hbd_downshift; + } + + od_pvq_decode(dec, ref_int32, out_int32, + OD_MAXI(1, quant[1] << (OD_COEFF_SHIFT - 3) >> hbd_downshift), + pli, bs, OD_PVQ_BETA[use_activity_masking][pli][bs], + is_keyframe, &flags, ac_dc_coded, dec->state.qm + off, + dec->state.qm_inv + off); + + if (!has_dc_skip || out_int32[0]) { + out_int32[0] = + has_dc_skip + generic_decode(dec->r, &dec->state.adapt->model_dc[pli], + &dec->state.adapt->ex_dc[pli][bs][0], 2, + "dc:mag"); + if (out_int32[0]) out_int32[0] *= aom_read_bit(dec->r, "dc:sign") ? -1 : 1; + } + out_int32[0] = out_int32[0] * pvq_dc_quant + ref_int32[0]; + + // copy int32 result back to int16 + assert(OD_COEFF_SHIFT > coeff_shift); + rounding_mask = (1 << (OD_COEFF_SHIFT - coeff_shift - 1)) - 1; + for (i = 0; i < blk_size * blk_size; i++) { + out_int32[i] = AOM_SIGNED_SHL(out_int32[i], hbd_downshift); + dqcoeff_pvq[i] = (out_int32[i] + (out_int32[i] < 0) + rounding_mask) >> + (OD_COEFF_SHIFT - coeff_shift); + } + + od_coding_order_to_raster(dqcoeff, blk_size, tx_type, dqcoeff_pvq, blk_size); + + eob = blk_size * blk_size; + + return eob; +} + +static PVQ_SKIP_TYPE read_pvq_skip(AV1_COMMON *cm, MACROBLOCKD *const xd, + int plane, TX_SIZE tx_size) { + // decode ac/dc coded flag. bit0: DC coded, bit1 : AC coded + // NOTE : we don't use 5 symbols for luma here in aom codebase, + // since block partition is taken care of by aom. + // So, only AC/DC skip info is coded + const int ac_dc_coded = aom_read_symbol( + xd->daala_dec.r, + xd->daala_dec.state.adapt->skip_cdf[2 * tx_size + (plane != 0)], 4, + "skip"); + if (ac_dc_coded < 0 || ac_dc_coded > 3) { + aom_internal_error(&cm->error, AOM_CODEC_INVALID_PARAM, + "Invalid PVQ Skip Type"); + } + return ac_dc_coded; +} + +static int av1_pvq_decode_helper2(AV1_COMMON *cm, MACROBLOCKD *const xd, + MB_MODE_INFO *const mbmi, int plane, int row, + int col, TX_SIZE tx_size, TX_TYPE tx_type) { + struct macroblockd_plane *const pd = &xd->plane[plane]; + // transform block size in pixels + int tx_blk_size = tx_size_wide[tx_size]; + int i, j; + tran_low_t *pvq_ref_coeff = pd->pvq_ref_coeff; + const int diff_stride = tx_blk_size; + int16_t *pred = pd->pred; + tran_low_t *const dqcoeff = pd->dqcoeff; + uint8_t *dst; + int eob; + const PVQ_SKIP_TYPE ac_dc_coded = read_pvq_skip(cm, xd, plane, tx_size); + + eob = 0; + dst = &pd->dst.buf[4 * row * pd->dst.stride + 4 * col]; + + if (ac_dc_coded) { + int xdec = pd->subsampling_x; + int seg_id = mbmi->segment_id; + int16_t *quant; + FWD_TXFM_PARAM fwd_txfm_param; + // ToDo(yaowu): correct this with optimal number from decoding process. + const int max_scan_line = tx_size_2d[tx_size]; +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + for (j = 0; j < tx_blk_size; j++) + for (i = 0; i < tx_blk_size; i++) + pred[diff_stride * j + i] = + CONVERT_TO_SHORTPTR(dst)[pd->dst.stride * j + i]; + } else { +#endif + for (j = 0; j < tx_blk_size; j++) + for (i = 0; i < tx_blk_size; i++) + pred[diff_stride * j + i] = dst[pd->dst.stride * j + i]; +#if CONFIG_HIGHBITDEPTH + } +#endif + + fwd_txfm_param.tx_type = tx_type; + fwd_txfm_param.tx_size = tx_size; + fwd_txfm_param.lossless = xd->lossless[seg_id]; + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + fwd_txfm_param.bd = xd->bd; + av1_highbd_fwd_txfm(pred, pvq_ref_coeff, diff_stride, &fwd_txfm_param); + } else { +#endif // CONFIG_HIGHBITDEPTH + av1_fwd_txfm(pred, pvq_ref_coeff, diff_stride, &fwd_txfm_param); +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + + quant = &pd->seg_dequant[seg_id][0]; // aom's quantizer + + eob = av1_pvq_decode_helper(xd, pvq_ref_coeff, dqcoeff, quant, plane, + tx_size, tx_type, xdec, ac_dc_coded); + + inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride, + max_scan_line, eob); + } + + return eob; +} +#endif + +static int get_block_idx(const MACROBLOCKD *xd, int plane, int row, int col) { + const int bsize = xd->mi[0]->mbmi.sb_type; + const struct macroblockd_plane *pd = &xd->plane[plane]; +#if CONFIG_CB4X4 +#if CONFIG_CHROMA_2X2 + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); +#else + const BLOCK_SIZE plane_bsize = + AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd)); +#endif // CONFIG_CHROMA_2X2 +#else + const BLOCK_SIZE plane_bsize = + get_plane_block_size(AOMMAX(BLOCK_8X8, bsize), pd); +#endif + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + const TX_SIZE tx_size = get_tx_size(plane, xd); + const uint8_t txh_unit = tx_size_high_unit[tx_size]; + return row * max_blocks_wide + col * txh_unit; +} + +static void predict_and_reconstruct_intra_block( + AV1_COMMON *cm, MACROBLOCKD *const xd, aom_reader *const r, + MB_MODE_INFO *const mbmi, int plane, int row, int col, TX_SIZE tx_size) { + PLANE_TYPE plane_type = get_plane_type(plane); + const int block_idx = get_block_idx(xd, plane, row, col); +#if CONFIG_PVQ + (void)r; +#endif + av1_predict_intra_block_facade(xd, plane, block_idx, col, row, tx_size); + + if (!mbmi->skip) { +#if !CONFIG_PVQ + struct macroblockd_plane *const pd = &xd->plane[plane]; +#if CONFIG_LV_MAP + int16_t max_scan_line = 0; + int eob; + av1_read_coeffs_txb_facade(cm, xd, r, row, col, block_idx, plane, + pd->dqcoeff, &max_scan_line, &eob); + // tx_type will be read out in av1_read_coeffs_txb_facade + TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size); +#else // CONFIG_LV_MAP + TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size); + const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 0); + int16_t max_scan_line = 0; + const int eob = + av1_decode_block_tokens(cm, xd, plane, scan_order, col, row, tx_size, + tx_type, &max_scan_line, r, mbmi->segment_id); +#endif // CONFIG_LV_MAP + if (eob) { + uint8_t *dst = + &pd->dst.buf[(row * pd->dst.stride + col) << tx_size_wide_log2[0]]; + inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride, + max_scan_line, eob); + } +#else + TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size); + av1_pvq_decode_helper2(cm, xd, mbmi, plane, row, col, tx_size, tx_type); +#endif + } +#if CONFIG_CFL + if (plane == AOM_PLANE_Y) { + struct macroblockd_plane *const pd = &xd->plane[plane]; + uint8_t *dst = + &pd->dst.buf[(row * pd->dst.stride + col) << tx_size_wide_log2[0]]; + cfl_store(xd->cfl, dst, pd->dst.stride, row, col, tx_size); + } +#endif +} + +#if CONFIG_VAR_TX && !CONFIG_COEF_INTERLEAVE +static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd, + aom_reader *r, MB_MODE_INFO *const mbmi, + int plane, BLOCK_SIZE plane_bsize, + int blk_row, int blk_col, TX_SIZE tx_size, + int *eob_total) { + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const BLOCK_SIZE bsize = txsize_to_bsize[tx_size]; + const int tx_row = blk_row >> (1 - pd->subsampling_y); + const int tx_col = blk_col >> (1 - pd->subsampling_x); + const TX_SIZE plane_tx_size = + plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0] + : mbmi->inter_tx_size[tx_row][tx_col]; + // Scale to match transform block unit. + const int max_blocks_high = max_block_high(xd, plane_bsize, plane); + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + + if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + + if (tx_size == plane_tx_size) { + PLANE_TYPE plane_type = get_plane_type(plane); + int block_idx = get_block_idx(xd, plane, blk_row, blk_col); +#if CONFIG_LV_MAP + (void)segment_id; + int16_t max_scan_line = 0; + int eob; + av1_read_coeffs_txb_facade(cm, xd, r, row, col, block_idx, plane, + pd->dqcoeff, &max_scan_line, &eob); + // tx_type will be read out in av1_read_coeffs_txb_facade + TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, plane_tx_size); +#else // CONFIG_LV_MAP + TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, plane_tx_size); + const SCAN_ORDER *sc = get_scan(cm, plane_tx_size, tx_type, 1); + int16_t max_scan_line = 0; + const int eob = av1_decode_block_tokens( + cm, xd, plane, sc, blk_col, blk_row, plane_tx_size, tx_type, + &max_scan_line, r, mbmi->segment_id); +#endif // CONFIG_LV_MAP + inverse_transform_block(xd, plane, tx_type, plane_tx_size, + &pd->dst.buf[(blk_row * pd->dst.stride + blk_col) + << tx_size_wide_log2[0]], + pd->dst.stride, max_scan_line, eob); + *eob_total += eob; + } else { + const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; + const int bsl = tx_size_wide_unit[sub_txs]; + int i; + + assert(bsl > 0); + + for (i = 0; i < 4; ++i) { + const int offsetr = blk_row + (i >> 1) * bsl; + const int offsetc = blk_col + (i & 0x01) * bsl; + + if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue; + + decode_reconstruct_tx(cm, xd, r, mbmi, plane, plane_bsize, offsetr, + offsetc, sub_txs, eob_total); + } + } +} +#endif // CONFIG_VAR_TX + +#if !CONFIG_VAR_TX || CONFIG_SUPERTX || CONFIG_COEF_INTERLEAVE || \ + (!CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX) +static int reconstruct_inter_block(AV1_COMMON *cm, MACROBLOCKD *const xd, + aom_reader *const r, int segment_id, + int plane, int row, int col, + TX_SIZE tx_size) { + PLANE_TYPE plane_type = get_plane_type(plane); + int block_idx = get_block_idx(xd, plane, row, col); +#if CONFIG_PVQ + int eob; + (void)r; + (void)segment_id; +#else + struct macroblockd_plane *const pd = &xd->plane[plane]; +#endif + +#if !CONFIG_PVQ +#if CONFIG_LV_MAP + (void)segment_id; + int16_t max_scan_line = 0; + int eob; + av1_read_coeffs_txb_facade(cm, xd, r, row, col, block_idx, plane, pd->dqcoeff, + &max_scan_line, &eob); + // tx_type will be read out in av1_read_coeffs_txb_facade + TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size); +#else // CONFIG_LV_MAP + int16_t max_scan_line = 0; + TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size); + const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 1); + const int eob = + av1_decode_block_tokens(cm, xd, plane, scan_order, col, row, tx_size, + tx_type, &max_scan_line, r, segment_id); +#endif // CONFIG_LV_MAP + uint8_t *dst = + &pd->dst.buf[(row * pd->dst.stride + col) << tx_size_wide_log2[0]]; + if (eob) + inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride, + max_scan_line, eob); +#else + TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size); + eob = av1_pvq_decode_helper2(cm, xd, &xd->mi[0]->mbmi, plane, row, col, + tx_size, tx_type); +#endif + return eob; +} +#endif // !CONFIG_VAR_TX || CONFIG_SUPER_TX + +static void set_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd, + BLOCK_SIZE bsize, int mi_row, int mi_col, int bw, + int bh, int x_mis, int y_mis) { + const int offset = mi_row * cm->mi_stride + mi_col; + int x, y; + const TileInfo *const tile = &xd->tile; + + xd->mi = cm->mi_grid_visible + offset; + xd->mi[0] = &cm->mi[offset]; + // TODO(slavarnway): Generate sb_type based on bwl and bhl, instead of + // passing bsize from decode_partition(). + xd->mi[0]->mbmi.sb_type = bsize; +#if CONFIG_RD_DEBUG + xd->mi[0]->mbmi.mi_row = mi_row; + xd->mi[0]->mbmi.mi_col = mi_col; +#endif + for (y = 0; y < y_mis; ++y) + for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0]; + + set_plane_n4(xd, bw, bh); + set_skip_context(xd, mi_row, mi_col); + +#if CONFIG_VAR_TX + xd->max_tx_size = max_txsize_lookup[bsize]; +#endif + + // Distance of Mb to the various image edges. These are specified to 8th pel + // as they are always compared to values that are in 1/8th pel units + set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + + av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, + mi_col); +} + +#if CONFIG_SUPERTX +static MB_MODE_INFO *set_offsets_extend(AV1_COMMON *const cm, + MACROBLOCKD *const xd, + const TileInfo *const tile, + BLOCK_SIZE bsize_pred, int mi_row_pred, + int mi_col_pred, int mi_row_ori, + int mi_col_ori) { + // Used in supertx + // (mi_row_ori, mi_col_ori): location for mv + // (mi_row_pred, mi_col_pred, bsize_pred): region to predict + const int bw = mi_size_wide[bsize_pred]; + const int bh = mi_size_high[bsize_pred]; + const int offset = mi_row_ori * cm->mi_stride + mi_col_ori; + xd->mi = cm->mi_grid_visible + offset; + xd->mi[0] = cm->mi + offset; + set_mi_row_col(xd, tile, mi_row_pred, bh, mi_col_pred, bw, +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + + xd->up_available = (mi_row_ori > tile->mi_row_start); + xd->left_available = (mi_col_ori > tile->mi_col_start); + + set_plane_n4(xd, bw, bh); + + return &xd->mi[0]->mbmi; +} + +#if CONFIG_SUPERTX +static MB_MODE_INFO *set_mb_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd, + BLOCK_SIZE bsize, int mi_row, int mi_col, + int bw, int bh, int x_mis, int y_mis) { + const int offset = mi_row * cm->mi_stride + mi_col; + const TileInfo *const tile = &xd->tile; + int x, y; + + xd->mi = cm->mi_grid_visible + offset; + xd->mi[0] = cm->mi + offset; + xd->mi[0]->mbmi.sb_type = bsize; + for (y = 0; y < y_mis; ++y) + for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0]; + + set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + return &xd->mi[0]->mbmi; +} +#endif + +static void set_offsets_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd, + const TileInfo *const tile, BLOCK_SIZE bsize, + int mi_row, int mi_col) { + const int bw = mi_size_wide[bsize]; + const int bh = mi_size_high[bsize]; + const int offset = mi_row * cm->mi_stride + mi_col; + + xd->mi = cm->mi_grid_visible + offset; + xd->mi[0] = cm->mi + offset; + + set_plane_n4(xd, bw, bh); + + set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + + av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, + mi_col); +} + +static void set_param_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd, + BLOCK_SIZE bsize, int mi_row, int mi_col, + int txfm, int skip) { + const int bw = mi_size_wide[bsize]; + const int bh = mi_size_high[bsize]; + const int x_mis = AOMMIN(bw, cm->mi_cols - mi_col); + const int y_mis = AOMMIN(bh, cm->mi_rows - mi_row); + const int offset = mi_row * cm->mi_stride + mi_col; + int x, y; + + xd->mi = cm->mi_grid_visible + offset; + xd->mi[0] = cm->mi + offset; + + for (y = 0; y < y_mis; ++y) + for (x = 0; x < x_mis; ++x) { + xd->mi[y * cm->mi_stride + x]->mbmi.skip = skip; + xd->mi[y * cm->mi_stride + x]->mbmi.tx_type = txfm; + } +#if CONFIG_VAR_TX + xd->above_txfm_context = cm->above_txfm_context + mi_col; + xd->left_txfm_context = + xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK); + set_txfm_ctxs(xd->mi[0]->mbmi.tx_size, bw, bh, skip, xd); +#endif +} + +static void set_ref(AV1_COMMON *const cm, MACROBLOCKD *const xd, int idx, + int mi_row, int mi_col) { + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + RefBuffer *ref_buffer = &cm->frame_refs[mbmi->ref_frame[idx] - LAST_FRAME]; + xd->block_refs[idx] = ref_buffer; + if (!av1_is_valid_scale(&ref_buffer->sf)) + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "Invalid scale factors"); + av1_setup_pre_planes(xd, idx, ref_buffer->buf, mi_row, mi_col, + &ref_buffer->sf); + aom_merge_corrupted_flag(&xd->corrupted, ref_buffer->buf->corrupted); +} + +static void dec_predict_b_extend( + AV1Decoder *const pbi, MACROBLOCKD *const xd, const TileInfo *const tile, + int block, int mi_row_ori, int mi_col_ori, int mi_row_pred, int mi_col_pred, + int mi_row_top, int mi_col_top, uint8_t *dst_buf[3], int dst_stride[3], + BLOCK_SIZE bsize_top, BLOCK_SIZE bsize_pred, int b_sub8x8, int bextend) { + // Used in supertx + // (mi_row_ori, mi_col_ori): location for mv + // (mi_row_pred, mi_col_pred, bsize_pred): region to predict + // (mi_row_top, mi_col_top, bsize_top): region of the top partition size + // block: sub location of sub8x8 blocks + // b_sub8x8: 1: ori is sub8x8; 0: ori is not sub8x8 + // bextend: 1: region to predict is an extension of ori; 0: not + int r = (mi_row_pred - mi_row_top) * MI_SIZE; + int c = (mi_col_pred - mi_col_top) * MI_SIZE; + const int mi_width_top = mi_size_wide[bsize_top]; + const int mi_height_top = mi_size_high[bsize_top]; + MB_MODE_INFO *mbmi; + AV1_COMMON *const cm = &pbi->common; + + if (mi_row_pred < mi_row_top || mi_col_pred < mi_col_top || + mi_row_pred >= mi_row_top + mi_height_top || + mi_col_pred >= mi_col_top + mi_width_top || mi_row_pred >= cm->mi_rows || + mi_col_pred >= cm->mi_cols) + return; + + mbmi = set_offsets_extend(cm, xd, tile, bsize_pred, mi_row_pred, mi_col_pred, + mi_row_ori, mi_col_ori); + set_ref(cm, xd, 0, mi_row_pred, mi_col_pred); + if (has_second_ref(&xd->mi[0]->mbmi)) + set_ref(cm, xd, 1, mi_row_pred, mi_col_pred); + + if (!bextend) mbmi->tx_size = max_txsize_lookup[bsize_top]; + + xd->plane[0].dst.stride = dst_stride[0]; + xd->plane[1].dst.stride = dst_stride[1]; + xd->plane[2].dst.stride = dst_stride[2]; + xd->plane[0].dst.buf = dst_buf[0] + + (r >> xd->plane[0].subsampling_y) * dst_stride[0] + + (c >> xd->plane[0].subsampling_x); + xd->plane[1].dst.buf = dst_buf[1] + + (r >> xd->plane[1].subsampling_y) * dst_stride[1] + + (c >> xd->plane[1].subsampling_x); + xd->plane[2].dst.buf = dst_buf[2] + + (r >> xd->plane[2].subsampling_y) * dst_stride[2] + + (c >> xd->plane[2].subsampling_x); + + if (!b_sub8x8) + av1_build_inter_predictors_sb_extend(xd, +#if CONFIG_EXT_INTER + mi_row_ori, mi_col_ori, +#endif // CONFIG_EXT_INTER + mi_row_pred, mi_col_pred, bsize_pred); + else + av1_build_inter_predictors_sb_sub8x8_extend(xd, +#if CONFIG_EXT_INTER + mi_row_ori, mi_col_ori, +#endif // CONFIG_EXT_INTER + mi_row_pred, mi_col_pred, + bsize_pred, block); +} + +static void dec_extend_dir(AV1Decoder *const pbi, MACROBLOCKD *const xd, + const TileInfo *const tile, int block, + BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row, + int mi_col, int mi_row_top, int mi_col_top, + uint8_t *dst_buf[3], int dst_stride[3], int dir) { + // dir: 0-lower, 1-upper, 2-left, 3-right + // 4-lowerleft, 5-upperleft, 6-lowerright, 7-upperright + const int mi_width = mi_size_wide[bsize]; + const int mi_height = mi_size_high[bsize]; + int xss = xd->plane[1].subsampling_x; + int yss = xd->plane[1].subsampling_y; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + int b_sub8x8 = (bsize < BLOCK_8X8) && !unify_bsize ? 1 : 0; + BLOCK_SIZE extend_bsize; + int mi_row_pred, mi_col_pred; + + int wide_unit, high_unit; + int i, j; + int ext_offset = 0; + + if (dir == 0 || dir == 1) { + extend_bsize = + (mi_width == mi_size_wide[BLOCK_8X8] || bsize < BLOCK_8X8 || xss < yss) + ? BLOCK_8X8 + : BLOCK_16X8; +#if CONFIG_CB4X4 + if (bsize < BLOCK_8X8) { + extend_bsize = BLOCK_4X4; + ext_offset = mi_size_wide[BLOCK_8X8]; + } +#endif + + wide_unit = mi_size_wide[extend_bsize]; + high_unit = mi_size_high[extend_bsize]; + + mi_row_pred = mi_row + ((dir == 0) ? mi_height : -(mi_height + ext_offset)); + mi_col_pred = mi_col; + + for (j = 0; j < mi_height + ext_offset; j += high_unit) + for (i = 0; i < mi_width + ext_offset; i += wide_unit) + dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, + mi_row_pred + j, mi_col_pred + i, mi_row_top, + mi_col_top, dst_buf, dst_stride, top_bsize, + extend_bsize, b_sub8x8, 1); + } else if (dir == 2 || dir == 3) { + extend_bsize = + (mi_height == mi_size_high[BLOCK_8X8] || bsize < BLOCK_8X8 || yss < xss) + ? BLOCK_8X8 + : BLOCK_8X16; +#if CONFIG_CB4X4 + if (bsize < BLOCK_8X8) { + extend_bsize = BLOCK_4X4; + ext_offset = mi_size_wide[BLOCK_8X8]; + } +#endif + + wide_unit = mi_size_wide[extend_bsize]; + high_unit = mi_size_high[extend_bsize]; + + mi_row_pred = mi_row; + mi_col_pred = mi_col + ((dir == 3) ? mi_width : -(mi_width + ext_offset)); + + for (j = 0; j < mi_height + ext_offset; j += high_unit) + for (i = 0; i < mi_width + ext_offset; i += wide_unit) + dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, + mi_row_pred + j, mi_col_pred + i, mi_row_top, + mi_col_top, dst_buf, dst_stride, top_bsize, + extend_bsize, b_sub8x8, 1); + } else { + extend_bsize = BLOCK_8X8; +#if CONFIG_CB4X4 + if (bsize < BLOCK_8X8) { + extend_bsize = BLOCK_4X4; + ext_offset = mi_size_wide[BLOCK_8X8]; + } +#endif + wide_unit = mi_size_wide[extend_bsize]; + high_unit = mi_size_high[extend_bsize]; + + mi_row_pred = mi_row + ((dir == 4 || dir == 6) ? mi_height + : -(mi_height + ext_offset)); + mi_col_pred = + mi_col + ((dir == 6 || dir == 7) ? mi_width : -(mi_width + ext_offset)); + + for (j = 0; j < mi_height + ext_offset; j += high_unit) + for (i = 0; i < mi_width + ext_offset; i += wide_unit) + dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, + mi_row_pred + j, mi_col_pred + i, mi_row_top, + mi_col_top, dst_buf, dst_stride, top_bsize, + extend_bsize, b_sub8x8, 1); + } +} + +static void dec_extend_all(AV1Decoder *const pbi, MACROBLOCKD *const xd, + const TileInfo *const tile, int block, + BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row, + int mi_col, int mi_row_top, int mi_col_top, + uint8_t *dst_buf[3], int dst_stride[3]) { + for (int i = 0; i < 8; ++i) { + dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, i); + } +} + +static void dec_predict_sb_complex(AV1Decoder *const pbi, MACROBLOCKD *const xd, + const TileInfo *const tile, int mi_row, + int mi_col, int mi_row_top, int mi_col_top, + BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, + uint8_t *dst_buf[3], int dst_stride[3]) { + const AV1_COMMON *const cm = &pbi->common; + const int hbs = mi_size_wide[bsize] / 2; + const PARTITION_TYPE partition = get_partition(cm, mi_row, mi_col, bsize); + const BLOCK_SIZE subsize = get_subsize(bsize, partition); +#if CONFIG_EXT_PARTITION_TYPES + const BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT); +#endif + int i; + const int mi_offset = mi_row * cm->mi_stride + mi_col; + uint8_t *dst_buf1[3], *dst_buf2[3], *dst_buf3[3]; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + + DECLARE_ALIGNED(16, uint8_t, tmp_buf1[MAX_MB_PLANE * MAX_TX_SQUARE * 2]); + DECLARE_ALIGNED(16, uint8_t, tmp_buf2[MAX_MB_PLANE * MAX_TX_SQUARE * 2]); + DECLARE_ALIGNED(16, uint8_t, tmp_buf3[MAX_MB_PLANE * MAX_TX_SQUARE * 2]); + int dst_stride1[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE }; + int dst_stride2[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE }; + int dst_stride3[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE }; + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + int len = sizeof(uint16_t); + dst_buf1[0] = CONVERT_TO_BYTEPTR(tmp_buf1); + dst_buf1[1] = CONVERT_TO_BYTEPTR(tmp_buf1 + MAX_TX_SQUARE * len); + dst_buf1[2] = CONVERT_TO_BYTEPTR(tmp_buf1 + 2 * MAX_TX_SQUARE * len); + dst_buf2[0] = CONVERT_TO_BYTEPTR(tmp_buf2); + dst_buf2[1] = CONVERT_TO_BYTEPTR(tmp_buf2 + MAX_TX_SQUARE * len); + dst_buf2[2] = CONVERT_TO_BYTEPTR(tmp_buf2 + 2 * MAX_TX_SQUARE * len); + dst_buf3[0] = CONVERT_TO_BYTEPTR(tmp_buf3); + dst_buf3[1] = CONVERT_TO_BYTEPTR(tmp_buf3 + MAX_TX_SQUARE * len); + dst_buf3[2] = CONVERT_TO_BYTEPTR(tmp_buf3 + 2 * MAX_TX_SQUARE * len); + } else { +#endif + dst_buf1[0] = tmp_buf1; + dst_buf1[1] = tmp_buf1 + MAX_TX_SQUARE; + dst_buf1[2] = tmp_buf1 + 2 * MAX_TX_SQUARE; + dst_buf2[0] = tmp_buf2; + dst_buf2[1] = tmp_buf2 + MAX_TX_SQUARE; + dst_buf2[2] = tmp_buf2 + 2 * MAX_TX_SQUARE; + dst_buf3[0] = tmp_buf3; + dst_buf3[1] = tmp_buf3 + MAX_TX_SQUARE; + dst_buf3[2] = tmp_buf3 + 2 * MAX_TX_SQUARE; +#if CONFIG_HIGHBITDEPTH + } +#endif + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + xd->mi = cm->mi_grid_visible + mi_offset; + xd->mi[0] = cm->mi + mi_offset; + + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + } + + switch (partition) { + case PARTITION_NONE: + assert(bsize < top_bsize); + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, + top_bsize, bsize, 0, 0); + dec_extend_all(pbi, xd, tile, 0, bsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride); + break; + case PARTITION_HORZ: + if (bsize == BLOCK_8X8 && !unify_bsize) { + // For sub8x8, predict in 8x8 unit + // First half + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, + top_bsize, BLOCK_8X8, 1, 0); + if (bsize < top_bsize) + dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride); + + // Second half + dec_predict_b_extend(pbi, xd, tile, 2, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf1, dst_stride1, + top_bsize, BLOCK_8X8, 1, 1); + if (bsize < top_bsize) + dec_extend_all(pbi, xd, tile, 2, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf1, dst_stride1); + + // weighted average to smooth the boundary + xd->plane[0].dst.buf = dst_buf[0]; + xd->plane[0].dst.stride = dst_stride[0]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[0], dst_stride[0], dst_buf1[0], dst_stride1[0], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_HORZ, + 0); + } else { + // First half + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, + top_bsize, subsize, 0, 0); + if (bsize < top_bsize) + dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride); + else + dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, 0); + + if (mi_row + hbs < cm->mi_rows) { + // Second half + dec_predict_b_extend(pbi, xd, tile, 0, mi_row + hbs, mi_col, + mi_row + hbs, mi_col, mi_row_top, mi_col_top, + dst_buf1, dst_stride1, top_bsize, subsize, 0, 0); + if (bsize < top_bsize) + dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dst_buf1, + dst_stride1); + else + dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dst_buf1, + dst_stride1, 1); + + // weighted average to smooth the boundary + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_HORZ, i); + } + } + } + break; + case PARTITION_VERT: + if (bsize == BLOCK_8X8 && !unify_bsize) { + // First half + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, + top_bsize, BLOCK_8X8, 1, 0); + if (bsize < top_bsize) + dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride); + + // Second half + dec_predict_b_extend(pbi, xd, tile, 1, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf1, dst_stride1, + top_bsize, BLOCK_8X8, 1, 1); + if (bsize < top_bsize) + dec_extend_all(pbi, xd, tile, 1, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf1, dst_stride1); + + // Smooth + xd->plane[0].dst.buf = dst_buf[0]; + xd->plane[0].dst.stride = dst_stride[0]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[0], dst_stride[0], dst_buf1[0], dst_stride1[0], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_VERT, + 0); + } else { + // First half + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, + top_bsize, subsize, 0, 0); + if (bsize < top_bsize) + dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride); + else + dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, 3); + + // Second half + if (mi_col + hbs < cm->mi_cols) { + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col + hbs, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf1, + dst_stride1, top_bsize, subsize, 0, 0); + if (bsize < top_bsize) + dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf1, + dst_stride1); + else + dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf1, + dst_stride1, 2); + + // Smooth + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_VERT, i); + } + } + } + break; + case PARTITION_SPLIT: + if (bsize == BLOCK_8X8 && !unify_bsize) { + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, + top_bsize, BLOCK_8X8, 1, 0); + dec_predict_b_extend(pbi, xd, tile, 1, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf1, dst_stride1, + top_bsize, BLOCK_8X8, 1, 1); + dec_predict_b_extend(pbi, xd, tile, 2, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf2, dst_stride2, + top_bsize, BLOCK_8X8, 1, 1); + dec_predict_b_extend(pbi, xd, tile, 3, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf3, dst_stride3, + top_bsize, BLOCK_8X8, 1, 1); + if (bsize < top_bsize) { + dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride); + dec_extend_all(pbi, xd, tile, 1, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf1, dst_stride1); + dec_extend_all(pbi, xd, tile, 2, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf2, dst_stride2); + dec_extend_all(pbi, xd, tile, 3, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf3, dst_stride3); + } + } else { + dec_predict_sb_complex(pbi, xd, tile, mi_row, mi_col, mi_row_top, + mi_col_top, subsize, top_bsize, dst_buf, + dst_stride); + if (mi_row < cm->mi_rows && mi_col + hbs < cm->mi_cols) + dec_predict_sb_complex(pbi, xd, tile, mi_row, mi_col + hbs, + mi_row_top, mi_col_top, subsize, top_bsize, + dst_buf1, dst_stride1); + if (mi_row + hbs < cm->mi_rows && mi_col < cm->mi_cols) + dec_predict_sb_complex(pbi, xd, tile, mi_row + hbs, mi_col, + mi_row_top, mi_col_top, subsize, top_bsize, + dst_buf2, dst_stride2); + if (mi_row + hbs < cm->mi_rows && mi_col + hbs < cm->mi_cols) + dec_predict_sb_complex(pbi, xd, tile, mi_row + hbs, mi_col + hbs, + mi_row_top, mi_col_top, subsize, top_bsize, + dst_buf3, dst_stride3); + } + for (i = 0; i < MAX_MB_PLANE; i++) { +#if !CONFIG_CB4X4 + if (bsize == BLOCK_8X8 && i != 0) + continue; // Skip <4x4 chroma smoothing +#endif + if (mi_row < cm->mi_rows && mi_col + hbs < cm->mi_cols) { + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_VERT, i); + if (mi_row + hbs < cm->mi_rows) { + av1_build_masked_inter_predictor_complex( + xd, dst_buf2[i], dst_stride2[i], dst_buf3[i], dst_stride3[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_VERT, i); + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf2[i], dst_stride2[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_HORZ, i); + } + } else if (mi_row + hbs < cm->mi_rows && mi_col < cm->mi_cols) { + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf2[i], dst_stride2[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_HORZ, i); + } + } + break; +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_HORZ_A: + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, + top_bsize, bsize2, 0, 0); + dec_extend_all(pbi, xd, tile, 0, bsize2, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride); + + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col + hbs, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf1, + dst_stride1, top_bsize, bsize2, 0, 0); + dec_extend_all(pbi, xd, tile, 0, bsize2, top_bsize, mi_row, mi_col + hbs, + mi_row_top, mi_col_top, dst_buf1, dst_stride1); + + dec_predict_b_extend(pbi, xd, tile, 0, mi_row + hbs, mi_col, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dst_buf2, + dst_stride2, top_bsize, subsize, 0, 0); + if (bsize < top_bsize) + dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dst_buf2, dst_stride2); + else + dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dst_buf2, dst_stride2, + 1); + + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_VERT, + i); + } + for (i = 0; i < MAX_MB_PLANE; i++) { + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf2[i], dst_stride2[i], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_HORZ, + i); + } + break; + case PARTITION_VERT_A: + + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, + top_bsize, bsize2, 0, 0); + dec_extend_all(pbi, xd, tile, 0, bsize2, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride); + + dec_predict_b_extend(pbi, xd, tile, 0, mi_row + hbs, mi_col, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dst_buf1, + dst_stride1, top_bsize, bsize2, 0, 0); + dec_extend_all(pbi, xd, tile, 0, bsize2, top_bsize, mi_row + hbs, mi_col, + mi_row_top, mi_col_top, dst_buf1, dst_stride1); + + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col + hbs, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf2, + dst_stride2, top_bsize, subsize, 0, 0); + if (bsize < top_bsize) + dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf2, + dst_stride2); + else + dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf2, + dst_stride2, 2); + + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_HORZ, + i); + } + for (i = 0; i < MAX_MB_PLANE; i++) { + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf2[i], dst_stride2[i], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_VERT, + i); + } + break; + case PARTITION_HORZ_B: + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, + top_bsize, subsize, 0, 0); + if (bsize < top_bsize) + dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride); + else + dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, 0); + + dec_predict_b_extend(pbi, xd, tile, 0, mi_row + hbs, mi_col, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dst_buf1, + dst_stride1, top_bsize, bsize2, 0, 0); + dec_extend_all(pbi, xd, tile, 0, bsize2, top_bsize, mi_row + hbs, mi_col, + mi_row_top, mi_col_top, dst_buf1, dst_stride1); + + dec_predict_b_extend(pbi, xd, tile, 0, mi_row + hbs, mi_col + hbs, + mi_row + hbs, mi_col + hbs, mi_row_top, mi_col_top, + dst_buf2, dst_stride2, top_bsize, bsize2, 0, 0); + dec_extend_all(pbi, xd, tile, 0, bsize2, top_bsize, mi_row + hbs, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf2, + dst_stride2); + + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf1[i]; + xd->plane[i].dst.stride = dst_stride1[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf1[i], dst_stride1[i], dst_buf2[i], dst_stride2[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_VERT, i); + } + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_HORZ, + i); + } + break; + case PARTITION_VERT_B: + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, + top_bsize, subsize, 0, 0); + if (bsize < top_bsize) + dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride); + else + dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, 3); + + dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col + hbs, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf1, + dst_stride1, top_bsize, bsize2, 0, 0); + dec_extend_all(pbi, xd, tile, 0, bsize2, top_bsize, mi_row, mi_col + hbs, + mi_row_top, mi_col_top, dst_buf1, dst_stride1); + + dec_predict_b_extend(pbi, xd, tile, 0, mi_row + hbs, mi_col + hbs, + mi_row + hbs, mi_col + hbs, mi_row_top, mi_col_top, + dst_buf2, dst_stride2, top_bsize, bsize2, 0, 0); + dec_extend_all(pbi, xd, tile, 0, bsize2, top_bsize, mi_row + hbs, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf2, + dst_stride2); + + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf1[i]; + xd->plane[i].dst.stride = dst_stride1[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf1[i], dst_stride1[i], dst_buf2[i], dst_stride2[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_HORZ, i); + } + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_VERT, + i); + } + break; +#endif // CONFIG_EXT_PARTITION_TYPES + default: assert(0); + } +} + +static void set_segment_id_supertx(const AV1_COMMON *const cm, int mi_row, + int mi_col, BLOCK_SIZE bsize) { + const struct segmentation *seg = &cm->seg; + const int miw = AOMMIN(mi_size_wide[bsize], cm->mi_cols - mi_col); + const int mih = AOMMIN(mi_size_high[bsize], cm->mi_rows - mi_row); + const int mi_offset = mi_row * cm->mi_stride + mi_col; + MODE_INFO **const mip = cm->mi_grid_visible + mi_offset; + int r, c; + int seg_id_supertx = MAX_SEGMENTS; + + if (!seg->enabled) { + seg_id_supertx = 0; + } else { + // Find the minimum segment_id + for (r = 0; r < mih; r++) + for (c = 0; c < miw; c++) + seg_id_supertx = + AOMMIN(mip[r * cm->mi_stride + c]->mbmi.segment_id, seg_id_supertx); + assert(0 <= seg_id_supertx && seg_id_supertx < MAX_SEGMENTS); + } + + // Assign the the segment_id back to segment_id_supertx + for (r = 0; r < mih; r++) + for (c = 0; c < miw; c++) + mip[r * cm->mi_stride + c]->mbmi.segment_id_supertx = seg_id_supertx; +} +#endif // CONFIG_SUPERTX + +static void decode_mbmi_block(AV1Decoder *const pbi, MACROBLOCKD *const xd, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif // CONFIG_SUPERTX + int mi_row, int mi_col, aom_reader *r, +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_TYPE partition, +#endif // CONFIG_EXT_PARTITION_TYPES + BLOCK_SIZE bsize) { + AV1_COMMON *const cm = &pbi->common; + const int bw = mi_size_wide[bsize]; + const int bh = mi_size_high[bsize]; + const int x_mis = AOMMIN(bw, cm->mi_cols - mi_col); + const int y_mis = AOMMIN(bh, cm->mi_rows - mi_row); + +#if CONFIG_ACCOUNTING + aom_accounting_set_context(&pbi->accounting, mi_col, mi_row); +#endif +#if CONFIG_SUPERTX + if (supertx_enabled) { + set_mb_offsets(cm, xd, bsize, mi_row, mi_col, bw, bh, x_mis, y_mis); + } else { + set_offsets(cm, xd, bsize, mi_row, mi_col, bw, bh, x_mis, y_mis); + } +#if CONFIG_EXT_PARTITION_TYPES + xd->mi[0]->mbmi.partition = partition; +#endif + av1_read_mode_info(pbi, xd, supertx_enabled, mi_row, mi_col, r, x_mis, y_mis); +#else + set_offsets(cm, xd, bsize, mi_row, mi_col, bw, bh, x_mis, y_mis); +#if CONFIG_EXT_PARTITION_TYPES + xd->mi[0]->mbmi.partition = partition; +#endif + av1_read_mode_info(pbi, xd, mi_row, mi_col, r, x_mis, y_mis); +#endif // CONFIG_SUPERTX + + if (bsize >= BLOCK_8X8 && (cm->subsampling_x || cm->subsampling_y)) { + const BLOCK_SIZE uv_subsize = + ss_size_lookup[bsize][cm->subsampling_x][cm->subsampling_y]; + if (uv_subsize == BLOCK_INVALID) + aom_internal_error(xd->error_info, AOM_CODEC_CORRUPT_FRAME, + "Invalid block size."); + } + +#if CONFIG_SUPERTX + xd->mi[0]->mbmi.segment_id_supertx = MAX_SEGMENTS; +#endif // CONFIG_SUPERTX + + int reader_corrupted_flag = aom_reader_has_error(r); + aom_merge_corrupted_flag(&xd->corrupted, reader_corrupted_flag); +} + +static void decode_token_and_recon_block(AV1Decoder *const pbi, + MACROBLOCKD *const xd, int mi_row, + int mi_col, aom_reader *r, + BLOCK_SIZE bsize) { + AV1_COMMON *const cm = &pbi->common; + const int bw = mi_size_wide[bsize]; + const int bh = mi_size_high[bsize]; + const int x_mis = AOMMIN(bw, cm->mi_cols - mi_col); + const int y_mis = AOMMIN(bh, cm->mi_rows - mi_row); + + set_offsets(cm, xd, bsize, mi_row, mi_col, bw, bh, x_mis, y_mis); + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + +#if CONFIG_DELTA_Q + if (cm->delta_q_present_flag) { + int i; + for (i = 0; i < MAX_SEGMENTS; i++) { +#if CONFIG_EXT_DELTA_Q + xd->plane[0].seg_dequant[i][0] = + av1_dc_quant(av1_get_qindex(&cm->seg, i, xd->current_qindex), + cm->y_dc_delta_q, cm->bit_depth); + xd->plane[0].seg_dequant[i][1] = av1_ac_quant( + av1_get_qindex(&cm->seg, i, xd->current_qindex), 0, cm->bit_depth); + xd->plane[1].seg_dequant[i][0] = + av1_dc_quant(av1_get_qindex(&cm->seg, i, xd->current_qindex), + cm->uv_dc_delta_q, cm->bit_depth); + xd->plane[1].seg_dequant[i][1] = + av1_ac_quant(av1_get_qindex(&cm->seg, i, xd->current_qindex), + cm->uv_ac_delta_q, cm->bit_depth); + xd->plane[2].seg_dequant[i][0] = + av1_dc_quant(av1_get_qindex(&cm->seg, i, xd->current_qindex), + cm->uv_dc_delta_q, cm->bit_depth); + xd->plane[2].seg_dequant[i][1] = + av1_ac_quant(av1_get_qindex(&cm->seg, i, xd->current_qindex), + cm->uv_ac_delta_q, cm->bit_depth); +#else + xd->plane[0].seg_dequant[i][0] = + av1_dc_quant(xd->current_qindex, cm->y_dc_delta_q, cm->bit_depth); + xd->plane[0].seg_dequant[i][1] = + av1_ac_quant(xd->current_qindex, 0, cm->bit_depth); + xd->plane[1].seg_dequant[i][0] = + av1_dc_quant(xd->current_qindex, cm->uv_dc_delta_q, cm->bit_depth); + xd->plane[1].seg_dequant[i][1] = + av1_ac_quant(xd->current_qindex, cm->uv_ac_delta_q, cm->bit_depth); + xd->plane[2].seg_dequant[i][0] = + av1_dc_quant(xd->current_qindex, cm->uv_dc_delta_q, cm->bit_depth); + xd->plane[2].seg_dequant[i][1] = + av1_ac_quant(xd->current_qindex, cm->uv_ac_delta_q, cm->bit_depth); +#endif + } + } +#endif + +#if CONFIG_CB4X4 + if (mbmi->skip) reset_skip_context(xd, bsize); +#else + if (mbmi->skip) reset_skip_context(xd, AOMMAX(BLOCK_8X8, bsize)); +#endif + +#if CONFIG_COEF_INTERLEAVE + { + const struct macroblockd_plane *const pd_y = &xd->plane[0]; + const struct macroblockd_plane *const pd_c = &xd->plane[1]; + const TX_SIZE tx_log2_y = mbmi->tx_size; + const TX_SIZE tx_log2_c = get_uv_tx_size(mbmi, pd_c); + const int tx_sz_y = (1 << tx_log2_y); + const int tx_sz_c = (1 << tx_log2_c); + const int num_4x4_w_y = pd_y->n4_w; + const int num_4x4_h_y = pd_y->n4_h; + const int num_4x4_w_c = pd_c->n4_w; + const int num_4x4_h_c = pd_c->n4_h; + const int max_4x4_w_y = get_max_4x4_size(num_4x4_w_y, xd->mb_to_right_edge, + pd_y->subsampling_x); + const int max_4x4_h_y = get_max_4x4_size(num_4x4_h_y, xd->mb_to_bottom_edge, + pd_y->subsampling_y); + const int max_4x4_w_c = get_max_4x4_size(num_4x4_w_c, xd->mb_to_right_edge, + pd_c->subsampling_x); + const int max_4x4_h_c = get_max_4x4_size(num_4x4_h_c, xd->mb_to_bottom_edge, + pd_c->subsampling_y); + + // The max_4x4_w/h may be smaller than tx_sz under some corner cases, + // i.e. when the SB is splitted by tile boundaries. + const int tu_num_w_y = (max_4x4_w_y + tx_sz_y - 1) / tx_sz_y; + const int tu_num_h_y = (max_4x4_h_y + tx_sz_y - 1) / tx_sz_y; + const int tu_num_w_c = (max_4x4_w_c + tx_sz_c - 1) / tx_sz_c; + const int tu_num_h_c = (max_4x4_h_c + tx_sz_c - 1) / tx_sz_c; + const int tu_num_c = tu_num_w_c * tu_num_h_c; + + if (!is_inter_block(mbmi)) { + int tu_idx_c = 0; + int row_y, col_y, row_c, col_c; + int plane; + +#if CONFIG_PALETTE + for (plane = 0; plane <= 1; ++plane) { + if (mbmi->palette_mode_info.palette_size[plane]) + av1_decode_palette_tokens(xd, plane, r); + } +#endif + + for (row_y = 0; row_y < tu_num_h_y; row_y++) { + for (col_y = 0; col_y < tu_num_w_y; col_y++) { + // luma + predict_and_reconstruct_intra_block( + cm, xd, r, mbmi, 0, row_y * tx_sz_y, col_y * tx_sz_y, tx_log2_y); + // chroma + if (tu_idx_c < tu_num_c) { + row_c = (tu_idx_c / tu_num_w_c) * tx_sz_c; + col_c = (tu_idx_c % tu_num_w_c) * tx_sz_c; + predict_and_reconstruct_intra_block(cm, xd, r, mbmi, 1, row_c, + col_c, tx_log2_c); + predict_and_reconstruct_intra_block(cm, xd, r, mbmi, 2, row_c, + col_c, tx_log2_c); + tu_idx_c++; + } + } + } + + // In 422 case, it's possilbe that Chroma has more TUs than Luma + while (tu_idx_c < tu_num_c) { + row_c = (tu_idx_c / tu_num_w_c) * tx_sz_c; + col_c = (tu_idx_c % tu_num_w_c) * tx_sz_c; + predict_and_reconstruct_intra_block(cm, xd, r, mbmi, 1, row_c, col_c, + tx_log2_c); + predict_and_reconstruct_intra_block(cm, xd, r, mbmi, 2, row_c, col_c, + tx_log2_c); + tu_idx_c++; + } + } else { + // Prediction + av1_build_inter_predictors_sb(xd, mi_row, mi_col, NULL, + AOMMAX(bsize, BLOCK_8X8)); + + // Reconstruction + if (!mbmi->skip) { + int eobtotal = 0; + int tu_idx_c = 0; + int row_y, col_y, row_c, col_c; + + for (row_y = 0; row_y < tu_num_h_y; row_y++) { + for (col_y = 0; col_y < tu_num_w_y; col_y++) { + // luma + eobtotal += reconstruct_inter_block(cm, xd, r, mbmi->segment_id, 0, + row_y * tx_sz_y, + col_y * tx_sz_y, tx_log2_y); + // chroma + if (tu_idx_c < tu_num_c) { + row_c = (tu_idx_c / tu_num_w_c) * tx_sz_c; + col_c = (tu_idx_c % tu_num_w_c) * tx_sz_c; + eobtotal += reconstruct_inter_block(cm, xd, r, mbmi->segment_id, + 1, row_c, col_c, tx_log2_c); + eobtotal += reconstruct_inter_block(cm, xd, r, mbmi->segment_id, + 2, row_c, col_c, tx_log2_c); + tu_idx_c++; + } + } + } + + // In 422 case, it's possilbe that Chroma has more TUs than Luma + while (tu_idx_c < tu_num_c) { + row_c = (tu_idx_c / tu_num_w_c) * tx_sz_c; + col_c = (tu_idx_c % tu_num_w_c) * tx_sz_c; + eobtotal += reconstruct_inter_block(cm, xd, r, mbmi->segment_id, 1, + row_c, col_c, tx_log2_c); + eobtotal += reconstruct_inter_block(cm, xd, r, mbmi->segment_id, 2, + row_c, col_c, tx_log2_c); + tu_idx_c++; + } + + // TODO(CONFIG_COEF_INTERLEAVE owners): bring eob == 0 corner case + // into line with the defaut configuration + if (bsize >= BLOCK_8X8 && eobtotal == 0) mbmi->skip = 1; + } + } + } +#else // CONFIG_COEF_INTERLEAVE + if (!is_inter_block(mbmi)) { + int plane; +#if CONFIG_PALETTE + for (plane = 0; plane <= 1; ++plane) { + if (mbmi->palette_mode_info.palette_size[plane]) + av1_decode_palette_tokens(xd, plane, r); + } +#endif // CONFIG_PALETTE + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const TX_SIZE tx_size = get_tx_size(plane, xd); + const int stepr = tx_size_high_unit[tx_size]; + const int stepc = tx_size_wide_unit[tx_size]; +#if CONFIG_CB4X4 +#if CONFIG_CHROMA_2X2 + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); +#else + const BLOCK_SIZE plane_bsize = + AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd)); +#endif // CONFIG_CHROMA_2X2 +#else + const BLOCK_SIZE plane_bsize = + get_plane_block_size(AOMMAX(BLOCK_8X8, bsize), pd); +#endif + int row, col; + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + const int max_blocks_high = max_block_high(xd, plane_bsize, plane); +#if CONFIG_CB4X4 + if (!is_chroma_reference(mi_row, mi_col, bsize, pd->subsampling_x, + pd->subsampling_y)) + continue; +#endif + + for (row = 0; row < max_blocks_high; row += stepr) + for (col = 0; col < max_blocks_wide; col += stepc) + predict_and_reconstruct_intra_block(cm, xd, r, mbmi, plane, row, col, + tx_size); + } + } else { + int ref; + + for (ref = 0; ref < 1 + has_second_ref(mbmi); ++ref) { + const MV_REFERENCE_FRAME frame = mbmi->ref_frame[ref]; + if (frame < LAST_FRAME) { +#if CONFIG_INTRABC + assert(is_intrabc_block(mbmi)); + assert(frame == INTRA_FRAME); + assert(ref == 0); +#else + assert(0); +#endif // CONFIG_INTRABC + } else { + RefBuffer *ref_buf = &cm->frame_refs[frame - LAST_FRAME]; + + xd->block_refs[ref] = ref_buf; + if ((!av1_is_valid_scale(&ref_buf->sf))) + aom_internal_error(xd->error_info, AOM_CODEC_UNSUP_BITSTREAM, + "Reference frame has invalid dimensions"); + av1_setup_pre_planes(xd, ref, ref_buf->buf, mi_row, mi_col, + &ref_buf->sf); + } + } + +#if CONFIG_CB4X4 + av1_build_inter_predictors_sb(xd, mi_row, mi_col, NULL, bsize); +#else + av1_build_inter_predictors_sb(xd, mi_row, mi_col, NULL, + AOMMAX(bsize, BLOCK_8X8)); +#endif + +#if CONFIG_MOTION_VAR + if (mbmi->motion_mode == OBMC_CAUSAL) { +#if CONFIG_NCOBMC + av1_build_ncobmc_inter_predictors_sb(cm, xd, mi_row, mi_col); +#else + av1_build_obmc_inter_predictors_sb(cm, xd, mi_row, mi_col); +#endif + } +#endif // CONFIG_MOTION_VAR + + // Reconstruction + if (!mbmi->skip) { + int eobtotal = 0; + int plane; + + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + const struct macroblockd_plane *const pd = &xd->plane[plane]; +#if CONFIG_CB4X4 +#if CONFIG_CHROMA_2X2 + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); +#else + const BLOCK_SIZE plane_bsize = + AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd)); +#endif // CONFIG_CHROMA_2X2 +#else + const BLOCK_SIZE plane_bsize = + get_plane_block_size(AOMMAX(BLOCK_8X8, bsize), pd); +#endif + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + const int max_blocks_high = max_block_high(xd, plane_bsize, plane); + int row, col; + +#if CONFIG_CB4X4 + if (!is_chroma_reference(mi_row, mi_col, bsize, pd->subsampling_x, + pd->subsampling_y)) + continue; +#endif + +#if CONFIG_VAR_TX + const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize); + const int bh_var_tx = tx_size_high_unit[max_tx_size]; + const int bw_var_tx = tx_size_wide_unit[max_tx_size]; + for (row = 0; row < max_blocks_high; row += bh_var_tx) + for (col = 0; col < max_blocks_wide; col += bw_var_tx) + decode_reconstruct_tx(cm, xd, r, mbmi, plane, plane_bsize, row, col, + max_tx_size, &eobtotal); +#else + const TX_SIZE tx_size = get_tx_size(plane, xd); + const int stepr = tx_size_high_unit[tx_size]; + const int stepc = tx_size_wide_unit[tx_size]; + for (row = 0; row < max_blocks_high; row += stepr) + for (col = 0; col < max_blocks_wide; col += stepc) + eobtotal += reconstruct_inter_block(cm, xd, r, mbmi->segment_id, + plane, row, col, tx_size); +#endif + } + } + } +#endif // CONFIG_COEF_INTERLEAVE + + int reader_corrupted_flag = aom_reader_has_error(r); + aom_merge_corrupted_flag(&xd->corrupted, reader_corrupted_flag); +} + +#if CONFIG_NCOBMC && CONFIG_MOTION_VAR +static void detoken_and_recon_sb(AV1Decoder *const pbi, MACROBLOCKD *const xd, + int mi_row, int mi_col, aom_reader *r, + BLOCK_SIZE bsize) { + AV1_COMMON *const cm = &pbi->common; + const int hbs = mi_size_wide[bsize] >> 1; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif +#if CONFIG_EXT_PARTITION_TYPES + BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT); +#endif + PARTITION_TYPE partition; + BLOCK_SIZE subsize; + const int has_rows = (mi_row + hbs) < cm->mi_rows; + const int has_cols = (mi_col + hbs) < cm->mi_cols; + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + partition = get_partition(cm, mi_row, mi_col, bsize); + subsize = subsize_lookup[partition][bsize]; + + if (!hbs && !unify_bsize) { + xd->bmode_blocks_wl = 1 >> !!(partition & PARTITION_VERT); + xd->bmode_blocks_hl = 1 >> !!(partition & PARTITION_HORZ); + decode_token_and_recon_block(pbi, xd, mi_row, mi_col, r, subsize); + } else { + switch (partition) { + case PARTITION_NONE: + decode_token_and_recon_block(pbi, xd, mi_row, mi_col, r, bsize); + break; + case PARTITION_HORZ: + decode_token_and_recon_block(pbi, xd, mi_row, mi_col, r, subsize); + if (has_rows) + decode_token_and_recon_block(pbi, xd, mi_row + hbs, mi_col, r, + subsize); + break; + case PARTITION_VERT: + decode_token_and_recon_block(pbi, xd, mi_row, mi_col, r, subsize); + if (has_cols) + decode_token_and_recon_block(pbi, xd, mi_row, mi_col + hbs, r, + subsize); + break; + case PARTITION_SPLIT: + detoken_and_recon_sb(pbi, xd, mi_row, mi_col, r, subsize); + detoken_and_recon_sb(pbi, xd, mi_row, mi_col + hbs, r, subsize); + detoken_and_recon_sb(pbi, xd, mi_row + hbs, mi_col, r, subsize); + detoken_and_recon_sb(pbi, xd, mi_row + hbs, mi_col + hbs, r, subsize); + break; +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_HORZ_A: + decode_token_and_recon_block(pbi, xd, mi_row, mi_col, r, bsize2); + decode_token_and_recon_block(pbi, xd, mi_row, mi_col + hbs, r, bsize2); + decode_token_and_recon_block(pbi, xd, mi_row + hbs, mi_col, r, subsize); + break; + case PARTITION_HORZ_B: + decode_token_and_recon_block(pbi, xd, mi_row, mi_col, r, subsize); + decode_token_and_recon_block(pbi, xd, mi_row + hbs, mi_col, r, bsize2); + decode_token_and_recon_block(pbi, xd, mi_row + hbs, mi_col + hbs, r, + bsize2); + break; + case PARTITION_VERT_A: + decode_token_and_recon_block(pbi, xd, mi_row, mi_col, r, bsize2); + decode_token_and_recon_block(pbi, xd, mi_row + hbs, mi_col, r, bsize2); + decode_token_and_recon_block(pbi, xd, mi_row, mi_col + hbs, r, subsize); + break; + case PARTITION_VERT_B: + decode_token_and_recon_block(pbi, xd, mi_row, mi_col, r, subsize); + decode_token_and_recon_block(pbi, xd, mi_row, mi_col + hbs, r, bsize2); + decode_token_and_recon_block(pbi, xd, mi_row + hbs, mi_col + hbs, r, + bsize2); + break; +#endif + default: assert(0 && "Invalid partition type"); + } + } +} +#endif + +static void decode_block(AV1Decoder *const pbi, MACROBLOCKD *const xd, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif // CONFIG_SUPERTX + int mi_row, int mi_col, aom_reader *r, +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_TYPE partition, +#endif // CONFIG_EXT_PARTITION_TYPES + BLOCK_SIZE bsize) { + decode_mbmi_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row, mi_col, r, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + bsize); +#if !(CONFIG_MOTION_VAR && CONFIG_NCOBMC) +#if CONFIG_SUPERTX + if (!supertx_enabled) +#endif // CONFIG_SUPERTX + decode_token_and_recon_block(pbi, xd, mi_row, mi_col, r, bsize); +#endif +} + +static PARTITION_TYPE read_partition(AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col, aom_reader *r, + int has_rows, int has_cols, + BLOCK_SIZE bsize) { +#if CONFIG_UNPOISON_PARTITION_CTX + const int ctx = + partition_plane_context(xd, mi_row, mi_col, has_rows, has_cols, bsize); + const aom_prob *const probs = + ctx < PARTITION_CONTEXTS ? cm->fc->partition_prob[ctx] : NULL; + FRAME_COUNTS *const counts = ctx < PARTITION_CONTEXTS ? xd->counts : NULL; +#else + const int ctx = partition_plane_context(xd, mi_row, mi_col, bsize); + const aom_prob *const probs = cm->fc->partition_prob[ctx]; + FRAME_COUNTS *const counts = xd->counts; +#endif + PARTITION_TYPE p; +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; + (void)cm; +#elif CONFIG_EC_MULTISYMBOL + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + +#if CONFIG_EC_MULTISYMBOL + aom_cdf_prob *partition_cdf = (ctx >= 0) ? ec_ctx->partition_cdf[ctx] : NULL; +#endif + + if (has_rows && has_cols) +#if CONFIG_EXT_PARTITION_TYPES + if (bsize <= BLOCK_8X8) +#if CONFIG_EC_MULTISYMBOL + p = (PARTITION_TYPE)aom_read_symbol(r, partition_cdf, PARTITION_TYPES, + ACCT_STR); +#else + p = (PARTITION_TYPE)aom_read_tree(r, av1_partition_tree, probs, ACCT_STR); +#endif + else +#if CONFIG_EC_MULTISYMBOL + p = (PARTITION_TYPE)aom_read_symbol(r, partition_cdf, EXT_PARTITION_TYPES, + ACCT_STR); +#else + p = (PARTITION_TYPE)aom_read_tree(r, av1_ext_partition_tree, probs, + ACCT_STR); +#endif +#else +#if CONFIG_EC_MULTISYMBOL + p = (PARTITION_TYPE)aom_read_symbol(r, partition_cdf, PARTITION_TYPES, + ACCT_STR); +#else + p = (PARTITION_TYPE)aom_read_tree(r, av1_partition_tree, probs, ACCT_STR); +#endif +#endif // CONFIG_EXT_PARTITION_TYPES + else if (!has_rows && has_cols) + p = aom_read(r, probs[1], ACCT_STR) ? PARTITION_SPLIT : PARTITION_HORZ; + else if (has_rows && !has_cols) + p = aom_read(r, probs[2], ACCT_STR) ? PARTITION_SPLIT : PARTITION_VERT; + else + p = PARTITION_SPLIT; + + if (counts) ++counts->partition[ctx][p]; + + return p; +} + +#if CONFIG_SUPERTX +static int read_skip(AV1_COMMON *cm, const MACROBLOCKD *xd, int segment_id, + aom_reader *r) { + if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) { + return 1; + } else { + const int ctx = av1_get_skip_context(xd); + const int skip = aom_read(r, cm->fc->skip_probs[ctx], ACCT_STR); + FRAME_COUNTS *counts = xd->counts; + if (counts) ++counts->skip[ctx][skip]; + return skip; + } +} +#endif // CONFIG_SUPERTX + +// TODO(slavarnway): eliminate bsize and subsize in future commits +static void decode_partition(AV1Decoder *const pbi, MACROBLOCKD *const xd, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif + int mi_row, int mi_col, aom_reader *r, + BLOCK_SIZE bsize, int n4x4_l2) { + AV1_COMMON *const cm = &pbi->common; + const int n8x8_l2 = n4x4_l2 - 1; + const int num_8x8_wh = mi_size_wide[bsize]; + const int hbs = num_8x8_wh >> 1; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + PARTITION_TYPE partition; + BLOCK_SIZE subsize; +#if CONFIG_EXT_PARTITION_TYPES + BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT); +#endif + const int has_rows = (mi_row + hbs) < cm->mi_rows; + const int has_cols = (mi_col + hbs) < cm->mi_cols; +#if CONFIG_SUPERTX + const int read_token = !supertx_enabled; + int skip = 0; + TX_SIZE supertx_size = max_txsize_lookup[bsize]; + const TileInfo *const tile = &xd->tile; + int txfm = DCT_DCT; +#endif // CONFIG_SUPERTX + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + partition = (bsize < BLOCK_8X8) ? PARTITION_NONE + : read_partition(cm, xd, mi_row, mi_col, r, + has_rows, has_cols, bsize); + subsize = subsize_lookup[partition][bsize]; // get_subsize(bsize, partition); + +#if CONFIG_PVQ + assert(partition < PARTITION_TYPES); + assert(subsize < BLOCK_SIZES); +#endif +#if CONFIG_SUPERTX + if (!frame_is_intra_only(cm) && partition != PARTITION_NONE && + bsize <= MAX_SUPERTX_BLOCK_SIZE && !supertx_enabled && !xd->lossless[0]) { + const int supertx_context = partition_supertx_context_lookup[partition]; + supertx_enabled = aom_read( + r, cm->fc->supertx_prob[supertx_context][supertx_size], ACCT_STR); + if (xd->counts) + xd->counts->supertx[supertx_context][supertx_size][supertx_enabled]++; +#if CONFIG_VAR_TX + if (supertx_enabled) xd->supertx_size = supertx_size; +#endif + } +#endif // CONFIG_SUPERTX + if (!hbs && !unify_bsize) { + // calculate bmode block dimensions (log 2) + xd->bmode_blocks_wl = 1 >> !!(partition & PARTITION_VERT); + xd->bmode_blocks_hl = 1 >> !!(partition & PARTITION_HORZ); + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif // CONFIG_SUPERTX + mi_row, mi_col, r, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif // CONFIG_EXT_PARTITION_TYPES + subsize); + } else { + switch (partition) { + case PARTITION_NONE: + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif // CONFIG_SUPERTX + mi_row, mi_col, r, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif // CONFIG_EXT_PARTITION_TYPES + subsize); + break; + case PARTITION_HORZ: + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif // CONFIG_SUPERTX + mi_row, mi_col, r, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif // CONFIG_EXT_PARTITION_TYPES + subsize); + if (has_rows) + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif // CONFIG_SUPERTX + mi_row + hbs, mi_col, r, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif // CONFIG_EXT_PARTITION_TYPES + subsize); + break; + case PARTITION_VERT: + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif // CONFIG_SUPERTX + mi_row, mi_col, r, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif // CONFIG_EXT_PARTITION_TYPES + subsize); + if (has_cols) + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif // CONFIG_SUPERTX + mi_row, mi_col + hbs, r, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif // CONFIG_EXT_PARTITION_TYPES + subsize); + break; + case PARTITION_SPLIT: + decode_partition(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif // CONFIG_SUPERTX + mi_row, mi_col, r, subsize, n8x8_l2); + decode_partition(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif // CONFIG_SUPERTX + mi_row, mi_col + hbs, r, subsize, n8x8_l2); + decode_partition(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif // CONFIG_SUPERTX + mi_row + hbs, mi_col, r, subsize, n8x8_l2); + decode_partition(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif // CONFIG_SUPERTX + mi_row + hbs, mi_col + hbs, r, subsize, n8x8_l2); + break; +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_HORZ_A: + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row, mi_col, r, partition, bsize2); + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row, mi_col + hbs, r, partition, bsize2); + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row + hbs, mi_col, r, partition, subsize); + break; + case PARTITION_HORZ_B: + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row, mi_col, r, partition, subsize); + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row + hbs, mi_col, r, partition, bsize2); + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row + hbs, mi_col + hbs, r, partition, bsize2); + break; + case PARTITION_VERT_A: + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row, mi_col, r, partition, bsize2); + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row + hbs, mi_col, r, partition, bsize2); + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row, mi_col + hbs, r, partition, subsize); + break; + case PARTITION_VERT_B: + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row, mi_col, r, partition, subsize); + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row, mi_col + hbs, r, partition, bsize2); + decode_block(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row + hbs, mi_col + hbs, r, partition, bsize2); + break; +#endif + default: assert(0 && "Invalid partition type"); + } + } + +#if CONFIG_SUPERTX + if (supertx_enabled && read_token) { + uint8_t *dst_buf[3]; + int dst_stride[3], i; + int offset = mi_row * cm->mi_stride + mi_col; + + set_segment_id_supertx(cm, mi_row, mi_col, bsize); + +#if CONFIG_DELTA_Q + if (cm->delta_q_present_flag) { + for (i = 0; i < MAX_SEGMENTS; i++) { + xd->plane[0].seg_dequant[i][0] = + av1_dc_quant(xd->current_qindex, cm->y_dc_delta_q, cm->bit_depth); + xd->plane[0].seg_dequant[i][1] = + av1_ac_quant(xd->current_qindex, 0, cm->bit_depth); + xd->plane[1].seg_dequant[i][0] = + av1_dc_quant(xd->current_qindex, cm->uv_dc_delta_q, cm->bit_depth); + xd->plane[1].seg_dequant[i][1] = + av1_ac_quant(xd->current_qindex, cm->uv_ac_delta_q, cm->bit_depth); + xd->plane[2].seg_dequant[i][0] = + av1_dc_quant(xd->current_qindex, cm->uv_dc_delta_q, cm->bit_depth); + xd->plane[2].seg_dequant[i][1] = + av1_ac_quant(xd->current_qindex, cm->uv_ac_delta_q, cm->bit_depth); + } + } +#endif + + xd->mi = cm->mi_grid_visible + offset; + xd->mi[0] = cm->mi + offset; + set_mi_row_col(xd, tile, mi_row, mi_size_high[bsize], mi_col, + mi_size_wide[bsize], +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + set_skip_context(xd, mi_row, mi_col); + skip = read_skip(cm, xd, xd->mi[0]->mbmi.segment_id_supertx, r); + if (skip) { + reset_skip_context(xd, bsize); + } else { +#if CONFIG_EXT_TX + if (get_ext_tx_types(supertx_size, bsize, 1, cm->reduced_tx_set_used) > + 1) { + const int eset = + get_ext_tx_set(supertx_size, bsize, 1, cm->reduced_tx_set_used); + if (eset > 0) { + txfm = aom_read_tree(r, av1_ext_tx_inter_tree[eset], + cm->fc->inter_ext_tx_prob[eset][supertx_size], + ACCT_STR); + if (xd->counts) ++xd->counts->inter_ext_tx[eset][supertx_size][txfm]; + } + } +#else + if (supertx_size < TX_32X32) { + txfm = aom_read_tree(r, av1_ext_tx_tree, + cm->fc->inter_ext_tx_prob[supertx_size], ACCT_STR); + if (xd->counts) ++xd->counts->inter_ext_tx[supertx_size][txfm]; + } +#endif // CONFIG_EXT_TX + } + + av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, + mi_col); + for (i = 0; i < MAX_MB_PLANE; i++) { + dst_buf[i] = xd->plane[i].dst.buf; + dst_stride[i] = xd->plane[i].dst.stride; + } + dec_predict_sb_complex(pbi, xd, tile, mi_row, mi_col, mi_row, mi_col, bsize, + bsize, dst_buf, dst_stride); + + if (!skip) { + int eobtotal = 0; + MB_MODE_INFO *mbmi; + set_offsets_topblock(cm, xd, tile, bsize, mi_row, mi_col); + mbmi = &xd->mi[0]->mbmi; + mbmi->tx_type = txfm; + assert(mbmi->segment_id_supertx != MAX_SEGMENTS); + for (i = 0; i < MAX_MB_PLANE; ++i) { + const struct macroblockd_plane *const pd = &xd->plane[i]; + int row, col; + const TX_SIZE tx_size = get_tx_size(i, xd); + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); + const int stepr = tx_size_high_unit[tx_size]; + const int stepc = tx_size_wide_unit[tx_size]; + const int max_blocks_wide = max_block_wide(xd, plane_bsize, i); + const int max_blocks_high = max_block_high(xd, plane_bsize, i); + + for (row = 0; row < max_blocks_high; row += stepr) + for (col = 0; col < max_blocks_wide; col += stepc) + eobtotal += reconstruct_inter_block( + cm, xd, r, mbmi->segment_id_supertx, i, row, col, tx_size); + } + if ((unify_bsize || !(subsize < BLOCK_8X8)) && eobtotal == 0) skip = 1; + } + set_param_topblock(cm, xd, bsize, mi_row, mi_col, txfm, skip); + } +#endif // CONFIG_SUPERTX + +#if CONFIG_EXT_PARTITION_TYPES + update_ext_partition_context(xd, mi_row, mi_col, subsize, bsize, partition); +#else + // update partition context + if (bsize >= BLOCK_8X8 && + (bsize == BLOCK_8X8 || partition != PARTITION_SPLIT)) + update_partition_context(xd, mi_row, mi_col, subsize, bsize); +#endif // CONFIG_EXT_PARTITION_TYPES + +#if CONFIG_CDEF +#if CONFIG_EXT_PARTITION + if (cm->sb_size == BLOCK_128X128 && bsize == BLOCK_128X128) { + if (!sb_all_skip(cm, mi_row, mi_col)) { + cm->mi_grid_visible[mi_row * cm->mi_stride + mi_col]->mbmi.cdef_strength = + aom_read_literal(r, cm->cdef_bits, ACCT_STR); + } else { + cm->mi_grid_visible[mi_row * cm->mi_stride + mi_col]->mbmi.cdef_strength = + 0; + } + } else if (cm->sb_size == BLOCK_64X64 && bsize == BLOCK_64X64) { +#else + if (bsize == BLOCK_64X64) { +#endif + if (!sb_all_skip(cm, mi_row, mi_col)) { + cm->mi_grid_visible[mi_row * cm->mi_stride + mi_col]->mbmi.cdef_strength = + aom_read_literal(r, cm->cdef_bits, ACCT_STR); + } else { + cm->mi_grid_visible[mi_row * cm->mi_stride + mi_col]->mbmi.cdef_strength = + -1; + } + } +#endif // CONFIG_CDEF +} + +static void setup_bool_decoder(const uint8_t *data, const uint8_t *data_end, + const size_t read_size, + struct aom_internal_error_info *error_info, + aom_reader *r, +#if CONFIG_ANS && ANS_MAX_SYMBOLS + int window_size, +#endif // CONFIG_ANS && ANS_MAX_SYMBOLS + aom_decrypt_cb decrypt_cb, void *decrypt_state) { + // Validate the calculated partition length. If the buffer + // described by the partition can't be fully read, then restrict + // it to the portion that can be (for EC mode) or throw an error. + if (!read_is_valid(data, read_size, data_end)) + aom_internal_error(error_info, AOM_CODEC_CORRUPT_FRAME, + "Truncated packet or corrupt tile length"); + +#if CONFIG_ANS && ANS_MAX_SYMBOLS + r->window_size = window_size; +#endif + if (aom_reader_init(r, data, read_size, decrypt_cb, decrypt_state)) + aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, + "Failed to allocate bool decoder %d", 1); +} + +#if !CONFIG_PVQ && !(CONFIG_EC_ADAPT && CONFIG_NEW_TOKENSET) && !CONFIG_LV_MAP +static void read_coef_probs_common(av1_coeff_probs_model *coef_probs, + aom_reader *r) { + int i, j, k, l, m; +#if CONFIG_EC_ADAPT + const int node_limit = UNCONSTRAINED_NODES - 1; +#else + const int node_limit = UNCONSTRAINED_NODES; +#endif + + if (aom_read_bit(r, ACCT_STR)) + for (i = 0; i < PLANE_TYPES; ++i) + for (j = 0; j < REF_TYPES; ++j) + for (k = 0; k < COEF_BANDS; ++k) + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) + for (m = 0; m < node_limit; ++m) + av1_diff_update_prob(r, &coef_probs[i][j][k][l][m], ACCT_STR); +} + +static void read_coef_probs(FRAME_CONTEXT *fc, TX_MODE tx_mode, aom_reader *r) { + const TX_SIZE max_tx_size = tx_mode_to_biggest_tx_size[tx_mode]; + TX_SIZE tx_size; + for (tx_size = 0; tx_size <= max_tx_size; ++tx_size) + read_coef_probs_common(fc->coef_probs[tx_size], r); +} +#endif + +static void setup_segmentation(AV1_COMMON *const cm, + struct aom_read_bit_buffer *rb) { + struct segmentation *const seg = &cm->seg; + int i, j; + + seg->update_map = 0; + seg->update_data = 0; + + seg->enabled = aom_rb_read_bit(rb); + if (!seg->enabled) return; + + // Segmentation map update + if (frame_is_intra_only(cm) || cm->error_resilient_mode) { + seg->update_map = 1; + } else { + seg->update_map = aom_rb_read_bit(rb); + } + if (seg->update_map) { + if (frame_is_intra_only(cm) || cm->error_resilient_mode) { + seg->temporal_update = 0; + } else { + seg->temporal_update = aom_rb_read_bit(rb); + } + } + + // Segmentation data update + seg->update_data = aom_rb_read_bit(rb); + if (seg->update_data) { + seg->abs_delta = aom_rb_read_bit(rb); + + av1_clearall_segfeatures(seg); + + for (i = 0; i < MAX_SEGMENTS; i++) { + for (j = 0; j < SEG_LVL_MAX; j++) { + int data = 0; + const int feature_enabled = aom_rb_read_bit(rb); + if (feature_enabled) { + av1_enable_segfeature(seg, i, j); + data = decode_unsigned_max(rb, av1_seg_feature_data_max(j)); + if (av1_is_segfeature_signed(j)) + data = aom_rb_read_bit(rb) ? -data : data; + } + av1_set_segdata(seg, i, j, data); + } + } + } +} + +#if CONFIG_LOOP_RESTORATION +static void decode_restoration_mode(AV1_COMMON *cm, + struct aom_read_bit_buffer *rb) { + int p; + RestorationInfo *rsi = &cm->rst_info[0]; + if (aom_rb_read_bit(rb)) { + rsi->frame_restoration_type = + aom_rb_read_bit(rb) ? RESTORE_SGRPROJ : RESTORE_WIENER; + } else { + rsi->frame_restoration_type = + aom_rb_read_bit(rb) ? RESTORE_SWITCHABLE : RESTORE_NONE; + } + for (p = 1; p < MAX_MB_PLANE; ++p) { + cm->rst_info[p].frame_restoration_type = + aom_rb_read_bit(rb) ? RESTORE_WIENER : RESTORE_NONE; + } + + cm->rst_info[0].restoration_tilesize = RESTORATION_TILESIZE_MAX; + cm->rst_info[1].restoration_tilesize = RESTORATION_TILESIZE_MAX; + cm->rst_info[2].restoration_tilesize = RESTORATION_TILESIZE_MAX; + if (cm->rst_info[0].frame_restoration_type != RESTORE_NONE || + cm->rst_info[1].frame_restoration_type != RESTORE_NONE || + cm->rst_info[2].frame_restoration_type != RESTORE_NONE) { + rsi = &cm->rst_info[0]; + rsi->restoration_tilesize >>= aom_rb_read_bit(rb); + if (rsi->restoration_tilesize != RESTORATION_TILESIZE_MAX) { + rsi->restoration_tilesize >>= aom_rb_read_bit(rb); + } + cm->rst_info[1].restoration_tilesize = cm->rst_info[0].restoration_tilesize; + cm->rst_info[2].restoration_tilesize = cm->rst_info[0].restoration_tilesize; + } +} + +static void read_wiener_filter(WienerInfo *wiener_info, + WienerInfo *ref_wiener_info, aom_reader *rb) { + wiener_info->vfilter[0] = wiener_info->vfilter[WIENER_WIN - 1] = + aom_read_primitive_refsubexpfin( + rb, WIENER_FILT_TAP0_MAXV - WIENER_FILT_TAP0_MINV + 1, + WIENER_FILT_TAP0_SUBEXP_K, + ref_wiener_info->vfilter[0] - WIENER_FILT_TAP0_MINV) + + WIENER_FILT_TAP0_MINV; + wiener_info->vfilter[1] = wiener_info->vfilter[WIENER_WIN - 2] = + aom_read_primitive_refsubexpfin( + rb, WIENER_FILT_TAP1_MAXV - WIENER_FILT_TAP1_MINV + 1, + WIENER_FILT_TAP1_SUBEXP_K, + ref_wiener_info->vfilter[1] - WIENER_FILT_TAP1_MINV) + + WIENER_FILT_TAP1_MINV; + wiener_info->vfilter[2] = wiener_info->vfilter[WIENER_WIN - 3] = + aom_read_primitive_refsubexpfin( + rb, WIENER_FILT_TAP2_MAXV - WIENER_FILT_TAP2_MINV + 1, + WIENER_FILT_TAP2_SUBEXP_K, + ref_wiener_info->vfilter[2] - WIENER_FILT_TAP2_MINV) + + WIENER_FILT_TAP2_MINV; + // The central element has an implicit +WIENER_FILT_STEP + wiener_info->vfilter[WIENER_HALFWIN] = + -2 * (wiener_info->vfilter[0] + wiener_info->vfilter[1] + + wiener_info->vfilter[2]); + + wiener_info->hfilter[0] = wiener_info->hfilter[WIENER_WIN - 1] = + aom_read_primitive_refsubexpfin( + rb, WIENER_FILT_TAP0_MAXV - WIENER_FILT_TAP0_MINV + 1, + WIENER_FILT_TAP0_SUBEXP_K, + ref_wiener_info->hfilter[0] - WIENER_FILT_TAP0_MINV) + + WIENER_FILT_TAP0_MINV; + wiener_info->hfilter[1] = wiener_info->hfilter[WIENER_WIN - 2] = + aom_read_primitive_refsubexpfin( + rb, WIENER_FILT_TAP1_MAXV - WIENER_FILT_TAP1_MINV + 1, + WIENER_FILT_TAP1_SUBEXP_K, + ref_wiener_info->hfilter[1] - WIENER_FILT_TAP1_MINV) + + WIENER_FILT_TAP1_MINV; + wiener_info->hfilter[2] = wiener_info->hfilter[WIENER_WIN - 3] = + aom_read_primitive_refsubexpfin( + rb, WIENER_FILT_TAP2_MAXV - WIENER_FILT_TAP2_MINV + 1, + WIENER_FILT_TAP2_SUBEXP_K, + ref_wiener_info->hfilter[2] - WIENER_FILT_TAP2_MINV) + + WIENER_FILT_TAP2_MINV; + // The central element has an implicit +WIENER_FILT_STEP + wiener_info->hfilter[WIENER_HALFWIN] = + -2 * (wiener_info->hfilter[0] + wiener_info->hfilter[1] + + wiener_info->hfilter[2]); + memcpy(ref_wiener_info, wiener_info, sizeof(*wiener_info)); +} + +static void read_sgrproj_filter(SgrprojInfo *sgrproj_info, + SgrprojInfo *ref_sgrproj_info, aom_reader *rb) { + sgrproj_info->ep = aom_read_literal(rb, SGRPROJ_PARAMS_BITS, ACCT_STR); + sgrproj_info->xqd[0] = + aom_read_primitive_refsubexpfin( + rb, SGRPROJ_PRJ_MAX0 - SGRPROJ_PRJ_MIN0 + 1, SGRPROJ_PRJ_SUBEXP_K, + ref_sgrproj_info->xqd[0] - SGRPROJ_PRJ_MIN0) + + SGRPROJ_PRJ_MIN0; + sgrproj_info->xqd[1] = + aom_read_primitive_refsubexpfin( + rb, SGRPROJ_PRJ_MAX1 - SGRPROJ_PRJ_MIN1 + 1, SGRPROJ_PRJ_SUBEXP_K, + ref_sgrproj_info->xqd[1] - SGRPROJ_PRJ_MIN1) + + SGRPROJ_PRJ_MIN1; + memcpy(ref_sgrproj_info, sgrproj_info, sizeof(*sgrproj_info)); +} + +static void decode_restoration(AV1_COMMON *cm, aom_reader *rb) { + int i, p; + SgrprojInfo ref_sgrproj_info; + WienerInfo ref_wiener_info; + set_default_wiener(&ref_wiener_info); + set_default_sgrproj(&ref_sgrproj_info); + const int ntiles = av1_get_rest_ntiles(cm->width, cm->height, + cm->rst_info[0].restoration_tilesize, + NULL, NULL, NULL, NULL); + const int ntiles_uv = av1_get_rest_ntiles( + ROUND_POWER_OF_TWO(cm->width, cm->subsampling_x), + ROUND_POWER_OF_TWO(cm->height, cm->subsampling_y), + cm->rst_info[1].restoration_tilesize, NULL, NULL, NULL, NULL); + RestorationInfo *rsi = &cm->rst_info[0]; + if (rsi->frame_restoration_type != RESTORE_NONE) { + if (rsi->frame_restoration_type == RESTORE_SWITCHABLE) { + for (i = 0; i < ntiles; ++i) { + rsi->restoration_type[i] = + aom_read_tree(rb, av1_switchable_restore_tree, + cm->fc->switchable_restore_prob, ACCT_STR); + if (rsi->restoration_type[i] == RESTORE_WIENER) { + read_wiener_filter(&rsi->wiener_info[i], &ref_wiener_info, rb); + } else if (rsi->restoration_type[i] == RESTORE_SGRPROJ) { + read_sgrproj_filter(&rsi->sgrproj_info[i], &ref_sgrproj_info, rb); + } + } + } else if (rsi->frame_restoration_type == RESTORE_WIENER) { + for (i = 0; i < ntiles; ++i) { + if (aom_read(rb, RESTORE_NONE_WIENER_PROB, ACCT_STR)) { + rsi->restoration_type[i] = RESTORE_WIENER; + read_wiener_filter(&rsi->wiener_info[i], &ref_wiener_info, rb); + } else { + rsi->restoration_type[i] = RESTORE_NONE; + } + } + } else if (rsi->frame_restoration_type == RESTORE_SGRPROJ) { + for (i = 0; i < ntiles; ++i) { + if (aom_read(rb, RESTORE_NONE_SGRPROJ_PROB, ACCT_STR)) { + rsi->restoration_type[i] = RESTORE_SGRPROJ; + read_sgrproj_filter(&rsi->sgrproj_info[i], &ref_sgrproj_info, rb); + } else { + rsi->restoration_type[i] = RESTORE_NONE; + } + } + } + } + for (p = 1; p < MAX_MB_PLANE; ++p) { + set_default_wiener(&ref_wiener_info); + rsi = &cm->rst_info[p]; + if (rsi->frame_restoration_type == RESTORE_WIENER) { + for (i = 0; i < ntiles_uv; ++i) { + if (ntiles_uv > 1) + rsi->restoration_type[i] = + aom_read(rb, RESTORE_NONE_WIENER_PROB, ACCT_STR) ? RESTORE_WIENER + : RESTORE_NONE; + else + rsi->restoration_type[i] = RESTORE_WIENER; + if (rsi->restoration_type[i] == RESTORE_WIENER) { + read_wiener_filter(&rsi->wiener_info[i], &ref_wiener_info, rb); + } + } + } + } +} +#endif // CONFIG_LOOP_RESTORATION + +static void setup_loopfilter(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) { + struct loopfilter *lf = &cm->lf; + lf->filter_level = aom_rb_read_literal(rb, 6); + lf->sharpness_level = aom_rb_read_literal(rb, 3); + + // Read in loop filter deltas applied at the MB level based on mode or ref + // frame. + lf->mode_ref_delta_update = 0; + + lf->mode_ref_delta_enabled = aom_rb_read_bit(rb); + if (lf->mode_ref_delta_enabled) { + lf->mode_ref_delta_update = aom_rb_read_bit(rb); + if (lf->mode_ref_delta_update) { + int i; + + for (i = 0; i < TOTAL_REFS_PER_FRAME; i++) + if (aom_rb_read_bit(rb)) + lf->ref_deltas[i] = aom_rb_read_inv_signed_literal(rb, 6); + + for (i = 0; i < MAX_MODE_LF_DELTAS; i++) + if (aom_rb_read_bit(rb)) + lf->mode_deltas[i] = aom_rb_read_inv_signed_literal(rb, 6); + } + } +} + +#if CONFIG_CDEF +static void setup_cdef(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) { + int i; + cm->cdef_dering_damping = aom_rb_read_literal(rb, 1) + 5; + cm->cdef_clpf_damping = aom_rb_read_literal(rb, 2) + 3; + cm->cdef_bits = aom_rb_read_literal(rb, 2); + cm->nb_cdef_strengths = 1 << cm->cdef_bits; + for (i = 0; i < cm->nb_cdef_strengths; i++) { + cm->cdef_strengths[i] = aom_rb_read_literal(rb, CDEF_STRENGTH_BITS); + cm->cdef_uv_strengths[i] = aom_rb_read_literal(rb, CDEF_STRENGTH_BITS); + } +} +#endif // CONFIG_CDEF + +static INLINE int read_delta_q(struct aom_read_bit_buffer *rb) { + return aom_rb_read_bit(rb) ? aom_rb_read_inv_signed_literal(rb, 6) : 0; +} + +static void setup_quantization(AV1_COMMON *const cm, + struct aom_read_bit_buffer *rb) { + cm->base_qindex = aom_rb_read_literal(rb, QINDEX_BITS); + cm->y_dc_delta_q = read_delta_q(rb); + cm->uv_dc_delta_q = read_delta_q(rb); + cm->uv_ac_delta_q = read_delta_q(rb); + cm->dequant_bit_depth = cm->bit_depth; +#if CONFIG_AOM_QM + cm->using_qmatrix = aom_rb_read_bit(rb); + if (cm->using_qmatrix) { + cm->min_qmlevel = aom_rb_read_literal(rb, QM_LEVEL_BITS); + cm->max_qmlevel = aom_rb_read_literal(rb, QM_LEVEL_BITS); + } else { + cm->min_qmlevel = 0; + cm->max_qmlevel = 0; + } +#endif +} + +static void setup_segmentation_dequant(AV1_COMMON *const cm) { + // Build y/uv dequant values based on segmentation. + int i = 0; +#if CONFIG_AOM_QM + int lossless; + int j = 0; + int qmlevel; + int using_qm = cm->using_qmatrix; + int minqm = cm->min_qmlevel; + int maxqm = cm->max_qmlevel; +#endif +#if CONFIG_NEW_QUANT + int b; + int dq; +#endif // CONFIG_NEW_QUANT + if (cm->seg.enabled) { + for (i = 0; i < MAX_SEGMENTS; ++i) { + const int qindex = av1_get_qindex(&cm->seg, i, cm->base_qindex); + cm->y_dequant[i][0] = + av1_dc_quant(qindex, cm->y_dc_delta_q, cm->bit_depth); + cm->y_dequant[i][1] = av1_ac_quant(qindex, 0, cm->bit_depth); + cm->uv_dequant[i][0] = + av1_dc_quant(qindex, cm->uv_dc_delta_q, cm->bit_depth); + cm->uv_dequant[i][1] = + av1_ac_quant(qindex, cm->uv_ac_delta_q, cm->bit_depth); +#if CONFIG_AOM_QM + lossless = qindex == 0 && cm->y_dc_delta_q == 0 && + cm->uv_dc_delta_q == 0 && cm->uv_ac_delta_q == 0; + // NB: depends on base index so there is only 1 set per frame + // No quant weighting when lossless or signalled not using QM + qmlevel = (lossless || using_qm == 0) + ? NUM_QM_LEVELS - 1 + : aom_get_qmlevel(cm->base_qindex, minqm, maxqm); + for (j = 0; j < TX_SIZES; ++j) { + cm->y_iqmatrix[i][1][j] = aom_iqmatrix(cm, qmlevel, 0, j, 1); + cm->y_iqmatrix[i][0][j] = aom_iqmatrix(cm, qmlevel, 0, j, 0); + cm->uv_iqmatrix[i][1][j] = aom_iqmatrix(cm, qmlevel, 1, j, 1); + cm->uv_iqmatrix[i][0][j] = aom_iqmatrix(cm, qmlevel, 1, j, 0); + } +#endif // CONFIG_AOM_QM +#if CONFIG_NEW_QUANT + for (dq = 0; dq < QUANT_PROFILES; dq++) { + for (b = 0; b < COEF_BANDS; ++b) { + av1_get_dequant_val_nuq(cm->y_dequant[i][b != 0], b, + cm->y_dequant_nuq[i][dq][b], NULL, dq); + av1_get_dequant_val_nuq(cm->uv_dequant[i][b != 0], b, + cm->uv_dequant_nuq[i][dq][b], NULL, dq); + } + } +#endif // CONFIG_NEW_QUANT + } + } else { + const int qindex = cm->base_qindex; + // When segmentation is disabled, only the first value is used. The + // remaining are don't cares. + cm->y_dequant[0][0] = av1_dc_quant(qindex, cm->y_dc_delta_q, cm->bit_depth); + cm->y_dequant[0][1] = av1_ac_quant(qindex, 0, cm->bit_depth); + cm->uv_dequant[0][0] = + av1_dc_quant(qindex, cm->uv_dc_delta_q, cm->bit_depth); + cm->uv_dequant[0][1] = + av1_ac_quant(qindex, cm->uv_ac_delta_q, cm->bit_depth); +#if CONFIG_AOM_QM + lossless = qindex == 0 && cm->y_dc_delta_q == 0 && cm->uv_dc_delta_q == 0 && + cm->uv_ac_delta_q == 0; + // No quant weighting when lossless or signalled not using QM + qmlevel = (lossless || using_qm == 0) + ? NUM_QM_LEVELS - 1 + : aom_get_qmlevel(cm->base_qindex, minqm, maxqm); + for (j = 0; j < TX_SIZES; ++j) { + cm->y_iqmatrix[i][1][j] = aom_iqmatrix(cm, qmlevel, 0, j, 1); + cm->y_iqmatrix[i][0][j] = aom_iqmatrix(cm, qmlevel, 0, j, 0); + cm->uv_iqmatrix[i][1][j] = aom_iqmatrix(cm, qmlevel, 1, j, 1); + cm->uv_iqmatrix[i][0][j] = aom_iqmatrix(cm, qmlevel, 1, j, 0); + } +#endif +#if CONFIG_NEW_QUANT + for (dq = 0; dq < QUANT_PROFILES; dq++) { + for (b = 0; b < COEF_BANDS; ++b) { + av1_get_dequant_val_nuq(cm->y_dequant[0][b != 0], b, + cm->y_dequant_nuq[0][dq][b], NULL, dq); + av1_get_dequant_val_nuq(cm->uv_dequant[0][b != 0], b, + cm->uv_dequant_nuq[0][dq][b], NULL, dq); + } + } +#endif // CONFIG_NEW_QUANT + } +} + +static InterpFilter read_frame_interp_filter(struct aom_read_bit_buffer *rb) { + return aom_rb_read_bit(rb) ? SWITCHABLE + : aom_rb_read_literal(rb, LOG_SWITCHABLE_FILTERS); +} + +static void setup_render_size(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) { + cm->render_width = cm->width; + cm->render_height = cm->height; + if (aom_rb_read_bit(rb)) + av1_read_frame_size(rb, &cm->render_width, &cm->render_height); +} + +#if CONFIG_FRAME_SUPERRES +// TODO(afergs): make "struct aom_read_bit_buffer *const rb"? +static void setup_superres_size(AV1_COMMON *const cm, + struct aom_read_bit_buffer *rb, int *width, + int *height) { + // TODO(afergs): Test this behaviour + // Frame superres is probably in compatible with this render resolution + assert(cm->width == cm->render_width && cm->height == cm->render_height); + + cm->superres_width = cm->width; + cm->superres_height = cm->height; + if (aom_rb_read_bit(rb)) { + cm->superres_scale_numerator = + (uint8_t)aom_rb_read_literal(rb, SUPERRES_SCALE_BITS); + cm->superres_scale_numerator += SUPERRES_SCALE_NUMERATOR_MIN; + // Don't edit cm->width or cm->height directly, or the buffers won't get + // resized correctly + // TODO(afergs): Should the render resolution not be modified? It's the same + // by default (ie. when it isn't sent)... + // resize_context_buffers() will change cm->width to equal cm->render_width, + // then they'll be the same again + *width = cm->render_width = + cm->width * cm->superres_scale_numerator / SUPERRES_SCALE_DENOMINATOR; + *height = cm->render_height = + cm->height * cm->superres_scale_numerator / SUPERRES_SCALE_DENOMINATOR; + } else { + // 1:1 scaling - ie. no scaling, scale not provided + cm->superres_scale_numerator = SUPERRES_SCALE_DENOMINATOR; + } +} +#endif // CONFIG_FRAME_SUPERRES + +static void resize_mv_buffer(AV1_COMMON *cm) { + aom_free(cm->cur_frame->mvs); + cm->cur_frame->mi_rows = cm->mi_rows; + cm->cur_frame->mi_cols = cm->mi_cols; + CHECK_MEM_ERROR(cm, cm->cur_frame->mvs, + (MV_REF *)aom_calloc(cm->mi_rows * cm->mi_cols, + sizeof(*cm->cur_frame->mvs))); +} + +static void resize_context_buffers(AV1_COMMON *cm, int width, int height) { +#if CONFIG_SIZE_LIMIT + if (width > DECODE_WIDTH_LIMIT || height > DECODE_HEIGHT_LIMIT) + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Dimensions of %dx%d beyond allowed size of %dx%d.", + width, height, DECODE_WIDTH_LIMIT, DECODE_HEIGHT_LIMIT); +#endif + if (cm->width != width || cm->height != height) { + const int new_mi_rows = + ALIGN_POWER_OF_TWO(height, MI_SIZE_LOG2) >> MI_SIZE_LOG2; + const int new_mi_cols = + ALIGN_POWER_OF_TWO(width, MI_SIZE_LOG2) >> MI_SIZE_LOG2; + + // Allocations in av1_alloc_context_buffers() depend on individual + // dimensions as well as the overall size. + if (new_mi_cols > cm->mi_cols || new_mi_rows > cm->mi_rows) { + if (av1_alloc_context_buffers(cm, width, height)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate context buffers"); + } else { + av1_set_mb_mi(cm, width, height); + } + av1_init_context_buffers(cm); + cm->width = width; + cm->height = height; + } + if (cm->cur_frame->mvs == NULL || cm->mi_rows > cm->cur_frame->mi_rows || + cm->mi_cols > cm->cur_frame->mi_cols) { + resize_mv_buffer(cm); + } +} + +static void setup_frame_size(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) { + int width, height; + BufferPool *const pool = cm->buffer_pool; + av1_read_frame_size(rb, &width, &height); + setup_render_size(cm, rb); +#if CONFIG_FRAME_SUPERRES + setup_superres_size(cm, rb, &width, &height); +#endif // CONFIG_FRAME_SUPERRES + resize_context_buffers(cm, width, height); + + lock_buffer_pool(pool); + if (aom_realloc_frame_buffer( + get_frame_new_buffer(cm), cm->width, cm->height, cm->subsampling_x, + cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + AOM_BORDER_IN_PIXELS, cm->byte_alignment, + &pool->frame_bufs[cm->new_fb_idx].raw_frame_buffer, pool->get_fb_cb, + pool->cb_priv)) { + unlock_buffer_pool(pool); + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate frame buffer"); + } + unlock_buffer_pool(pool); + + pool->frame_bufs[cm->new_fb_idx].buf.subsampling_x = cm->subsampling_x; + pool->frame_bufs[cm->new_fb_idx].buf.subsampling_y = cm->subsampling_y; + pool->frame_bufs[cm->new_fb_idx].buf.bit_depth = (unsigned int)cm->bit_depth; + pool->frame_bufs[cm->new_fb_idx].buf.color_space = cm->color_space; + pool->frame_bufs[cm->new_fb_idx].buf.color_range = cm->color_range; + pool->frame_bufs[cm->new_fb_idx].buf.render_width = cm->render_width; + pool->frame_bufs[cm->new_fb_idx].buf.render_height = cm->render_height; +} + +static INLINE int valid_ref_frame_img_fmt(aom_bit_depth_t ref_bit_depth, + int ref_xss, int ref_yss, + aom_bit_depth_t this_bit_depth, + int this_xss, int this_yss) { + return ref_bit_depth == this_bit_depth && ref_xss == this_xss && + ref_yss == this_yss; +} + +static void setup_frame_size_with_refs(AV1_COMMON *cm, + struct aom_read_bit_buffer *rb) { + int width, height; + int found = 0, i; + int has_valid_ref_frame = 0; + BufferPool *const pool = cm->buffer_pool; + for (i = 0; i < INTER_REFS_PER_FRAME; ++i) { + if (aom_rb_read_bit(rb)) { + YV12_BUFFER_CONFIG *const buf = cm->frame_refs[i].buf; + width = buf->y_crop_width; + height = buf->y_crop_height; + cm->render_width = buf->render_width; + cm->render_height = buf->render_height; + found = 1; + break; + } + } + + if (!found) { + av1_read_frame_size(rb, &width, &height); + setup_render_size(cm, rb); + } + + if (width <= 0 || height <= 0) + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Invalid frame size"); + + // Check to make sure at least one of frames that this frame references + // has valid dimensions. + for (i = 0; i < INTER_REFS_PER_FRAME; ++i) { + RefBuffer *const ref_frame = &cm->frame_refs[i]; + has_valid_ref_frame |= + valid_ref_frame_size(ref_frame->buf->y_crop_width, + ref_frame->buf->y_crop_height, width, height); + } + if (!has_valid_ref_frame) + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Referenced frame has invalid size"); + for (i = 0; i < INTER_REFS_PER_FRAME; ++i) { + RefBuffer *const ref_frame = &cm->frame_refs[i]; + if (!valid_ref_frame_img_fmt(ref_frame->buf->bit_depth, + ref_frame->buf->subsampling_x, + ref_frame->buf->subsampling_y, cm->bit_depth, + cm->subsampling_x, cm->subsampling_y)) + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Referenced frame has incompatible color format"); + } + + resize_context_buffers(cm, width, height); + + lock_buffer_pool(pool); + if (aom_realloc_frame_buffer( + get_frame_new_buffer(cm), cm->width, cm->height, cm->subsampling_x, + cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + AOM_BORDER_IN_PIXELS, cm->byte_alignment, + &pool->frame_bufs[cm->new_fb_idx].raw_frame_buffer, pool->get_fb_cb, + pool->cb_priv)) { + unlock_buffer_pool(pool); + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate frame buffer"); + } + unlock_buffer_pool(pool); + + pool->frame_bufs[cm->new_fb_idx].buf.subsampling_x = cm->subsampling_x; + pool->frame_bufs[cm->new_fb_idx].buf.subsampling_y = cm->subsampling_y; + pool->frame_bufs[cm->new_fb_idx].buf.bit_depth = (unsigned int)cm->bit_depth; + pool->frame_bufs[cm->new_fb_idx].buf.color_space = cm->color_space; + pool->frame_bufs[cm->new_fb_idx].buf.color_range = cm->color_range; + pool->frame_bufs[cm->new_fb_idx].buf.render_width = cm->render_width; + pool->frame_bufs[cm->new_fb_idx].buf.render_height = cm->render_height; +} + +static void read_tile_info(AV1Decoder *const pbi, + struct aom_read_bit_buffer *const rb) { + AV1_COMMON *const cm = &pbi->common; +#if CONFIG_EXT_TILE + cm->tile_encoding_mode = aom_rb_read_literal(rb, 1); +// Read the tile width/height +#if CONFIG_EXT_PARTITION + if (cm->sb_size == BLOCK_128X128) { + cm->tile_width = aom_rb_read_literal(rb, 5) + 1; + cm->tile_height = aom_rb_read_literal(rb, 5) + 1; + } else +#endif // CONFIG_EXT_PARTITION + { + cm->tile_width = aom_rb_read_literal(rb, 6) + 1; + cm->tile_height = aom_rb_read_literal(rb, 6) + 1; + } + +#if CONFIG_LOOPFILTERING_ACROSS_TILES + cm->loop_filter_across_tiles_enabled = aom_rb_read_bit(rb); +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + + cm->tile_width <<= cm->mib_size_log2; + cm->tile_height <<= cm->mib_size_log2; + + cm->tile_width = AOMMIN(cm->tile_width, cm->mi_cols); + cm->tile_height = AOMMIN(cm->tile_height, cm->mi_rows); + + // Get the number of tiles + cm->tile_cols = 1; + while (cm->tile_cols * cm->tile_width < cm->mi_cols) ++cm->tile_cols; + + cm->tile_rows = 1; + while (cm->tile_rows * cm->tile_height < cm->mi_rows) ++cm->tile_rows; + + if (cm->tile_cols * cm->tile_rows > 1) { + // Read the number of bytes used to store tile size + pbi->tile_col_size_bytes = aom_rb_read_literal(rb, 2) + 1; + pbi->tile_size_bytes = aom_rb_read_literal(rb, 2) + 1; + } + +#if CONFIG_DEPENDENT_HORZTILES + if (cm->tile_rows <= 1) + cm->dependent_horz_tiles = aom_rb_read_bit(rb); + else + cm->dependent_horz_tiles = 0; +#endif +#else + int min_log2_tile_cols, max_log2_tile_cols, max_ones; + av1_get_tile_n_bits(cm->mi_cols, &min_log2_tile_cols, &max_log2_tile_cols); + + // columns + max_ones = max_log2_tile_cols - min_log2_tile_cols; + cm->log2_tile_cols = min_log2_tile_cols; + while (max_ones-- && aom_rb_read_bit(rb)) cm->log2_tile_cols++; + + if (cm->log2_tile_cols > 6) + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Invalid number of tile columns"); + + // rows + cm->log2_tile_rows = aom_rb_read_bit(rb); + if (cm->log2_tile_rows) cm->log2_tile_rows += aom_rb_read_bit(rb); +#if CONFIG_DEPENDENT_HORZTILES + if (cm->log2_tile_rows != 0) + cm->dependent_horz_tiles = aom_rb_read_bit(rb); + else + cm->dependent_horz_tiles = 0; +#endif +#if CONFIG_LOOPFILTERING_ACROSS_TILES + cm->loop_filter_across_tiles_enabled = aom_rb_read_bit(rb); +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + + cm->tile_cols = 1 << cm->log2_tile_cols; + cm->tile_rows = 1 << cm->log2_tile_rows; + + cm->tile_width = ALIGN_POWER_OF_TWO(cm->mi_cols, MAX_MIB_SIZE_LOG2); + cm->tile_width >>= cm->log2_tile_cols; + cm->tile_height = ALIGN_POWER_OF_TWO(cm->mi_rows, MAX_MIB_SIZE_LOG2); + cm->tile_height >>= cm->log2_tile_rows; + + // round to integer multiples of superblock size + cm->tile_width = ALIGN_POWER_OF_TWO(cm->tile_width, MAX_MIB_SIZE_LOG2); + cm->tile_height = ALIGN_POWER_OF_TWO(cm->tile_height, MAX_MIB_SIZE_LOG2); + +// tile size magnitude +#if !CONFIG_TILE_GROUPS + if (cm->tile_rows > 1 || cm->tile_cols > 1) +#endif + pbi->tile_size_bytes = aom_rb_read_literal(rb, 2) + 1; +#endif // CONFIG_EXT_TILE + +#if CONFIG_TILE_GROUPS + // Store an index to the location of the tile group information + pbi->tg_size_bit_offset = rb->bit_offset; + pbi->tg_size = 1 << (cm->log2_tile_rows + cm->log2_tile_cols); + if (cm->log2_tile_rows + cm->log2_tile_cols > 0) { + pbi->tg_start = + aom_rb_read_literal(rb, cm->log2_tile_rows + cm->log2_tile_cols); + pbi->tg_size = + 1 + aom_rb_read_literal(rb, cm->log2_tile_rows + cm->log2_tile_cols); + } +#endif +} + +static int mem_get_varsize(const uint8_t *src, int sz) { + switch (sz) { + case 1: return src[0]; + case 2: return mem_get_le16(src); + case 3: return mem_get_le24(src); + case 4: return mem_get_le32(src); + default: assert("Invalid size" && 0); return -1; + } +} + +#if CONFIG_EXT_TILE +// Reads the next tile returning its size and adjusting '*data' accordingly +// based on 'is_last'. +static void get_tile_buffer(const uint8_t *const data_end, + struct aom_internal_error_info *error_info, + const uint8_t **data, aom_decrypt_cb decrypt_cb, + void *decrypt_state, + TileBufferDec (*const tile_buffers)[MAX_TILE_COLS], + int tile_size_bytes, int col, int row, + unsigned int tile_encoding_mode) { + size_t size; + + size_t copy_size = 0; + const uint8_t *copy_data = NULL; + + if (!read_is_valid(*data, tile_size_bytes, data_end)) + aom_internal_error(error_info, AOM_CODEC_CORRUPT_FRAME, + "Truncated packet or corrupt tile length"); + if (decrypt_cb) { + uint8_t be_data[4]; + decrypt_cb(decrypt_state, *data, be_data, tile_size_bytes); + + // Only read number of bytes in cm->tile_size_bytes. + size = mem_get_varsize(be_data, tile_size_bytes); + } else { + size = mem_get_varsize(*data, tile_size_bytes); + } + + // If cm->tile_encoding_mode = 1 (i.e. TILE_VR), then the top bit of the tile + // header indicates copy mode. + if (tile_encoding_mode && (size >> (tile_size_bytes * 8 - 1)) == 1) { + // The remaining bits in the top byte signal the row offset + int offset = (size >> (tile_size_bytes - 1) * 8) & 0x7f; + + // Currently, only use tiles in same column as reference tiles. + copy_data = tile_buffers[row - offset][col].data; + copy_size = tile_buffers[row - offset][col].size; + size = 0; + } + + *data += tile_size_bytes; + + if (size > (size_t)(data_end - *data)) + aom_internal_error(error_info, AOM_CODEC_CORRUPT_FRAME, + "Truncated packet or corrupt tile size"); + + if (size > 0) { + tile_buffers[row][col].data = *data; + tile_buffers[row][col].size = size; + } else { + tile_buffers[row][col].data = copy_data; + tile_buffers[row][col].size = copy_size; + } + + *data += size; + + tile_buffers[row][col].raw_data_end = *data; +} + +static void get_tile_buffers( + AV1Decoder *pbi, const uint8_t *data, const uint8_t *data_end, + TileBufferDec (*const tile_buffers)[MAX_TILE_COLS]) { + AV1_COMMON *const cm = &pbi->common; + const int tile_cols = cm->tile_cols; + const int tile_rows = cm->tile_rows; + const int have_tiles = tile_cols * tile_rows > 1; + + if (!have_tiles) { + const size_t tile_size = data_end - data; + tile_buffers[0][0].data = data; + tile_buffers[0][0].size = tile_size; + tile_buffers[0][0].raw_data_end = NULL; + } else { + // We locate only the tile buffers that are required, which are the ones + // specified by pbi->dec_tile_col and pbi->dec_tile_row. Also, we always + // need the last (bottom right) tile buffer, as we need to know where the + // end of the compressed frame buffer is for proper superframe decoding. + + const uint8_t *tile_col_data_end[MAX_TILE_COLS]; + const uint8_t *const data_start = data; + + const int dec_tile_row = AOMMIN(pbi->dec_tile_row, tile_rows); + const int single_row = pbi->dec_tile_row >= 0; + const int tile_rows_start = single_row ? dec_tile_row : 0; + const int tile_rows_end = single_row ? tile_rows_start + 1 : tile_rows; + const int dec_tile_col = AOMMIN(pbi->dec_tile_col, tile_cols); + const int single_col = pbi->dec_tile_col >= 0; + const int tile_cols_start = single_col ? dec_tile_col : 0; + const int tile_cols_end = single_col ? tile_cols_start + 1 : tile_cols; + + const int tile_col_size_bytes = pbi->tile_col_size_bytes; + const int tile_size_bytes = pbi->tile_size_bytes; + + size_t tile_col_size; + int r, c; + + // Read tile column sizes for all columns (we need the last tile buffer) + for (c = 0; c < tile_cols; ++c) { + const int is_last = c == tile_cols - 1; + if (!is_last) { + tile_col_size = mem_get_varsize(data, tile_col_size_bytes); + data += tile_col_size_bytes; + tile_col_data_end[c] = data + tile_col_size; + } else { + tile_col_size = data_end - data; + tile_col_data_end[c] = data_end; + } + data += tile_col_size; + } + + data = data_start; + + // Read the required tile sizes. + for (c = tile_cols_start; c < tile_cols_end; ++c) { + const int is_last = c == tile_cols - 1; + + if (c > 0) data = tile_col_data_end[c - 1]; + + if (!is_last) data += tile_col_size_bytes; + + // Get the whole of the last column, otherwise stop at the required tile. + for (r = 0; r < (is_last ? tile_rows : tile_rows_end); ++r) { + tile_buffers[r][c].col = c; + + get_tile_buffer(tile_col_data_end[c], &pbi->common.error, &data, + pbi->decrypt_cb, pbi->decrypt_state, tile_buffers, + tile_size_bytes, c, r, cm->tile_encoding_mode); + } + } + + // If we have not read the last column, then read it to get the last tile. + if (tile_cols_end != tile_cols) { + c = tile_cols - 1; + + data = tile_col_data_end[c - 1]; + + for (r = 0; r < tile_rows; ++r) { + tile_buffers[r][c].col = c; + + get_tile_buffer(tile_col_data_end[c], &pbi->common.error, &data, + pbi->decrypt_cb, pbi->decrypt_state, tile_buffers, + tile_size_bytes, c, r, cm->tile_encoding_mode); + } + } + } +} +#else +// Reads the next tile returning its size and adjusting '*data' accordingly +// based on 'is_last'. +static void get_tile_buffer(const uint8_t *const data_end, + const int tile_size_bytes, int is_last, + struct aom_internal_error_info *error_info, + const uint8_t **data, aom_decrypt_cb decrypt_cb, + void *decrypt_state, TileBufferDec *const buf) { + size_t size; + + if (!is_last) { + if (!read_is_valid(*data, tile_size_bytes, data_end)) + aom_internal_error(error_info, AOM_CODEC_CORRUPT_FRAME, + "Truncated packet or corrupt tile length"); + + if (decrypt_cb) { + uint8_t be_data[4]; + decrypt_cb(decrypt_state, *data, be_data, tile_size_bytes); + size = mem_get_varsize(be_data, tile_size_bytes); + } else { + size = mem_get_varsize(*data, tile_size_bytes); + } + *data += tile_size_bytes; + + if (size > (size_t)(data_end - *data)) + aom_internal_error(error_info, AOM_CODEC_CORRUPT_FRAME, + "Truncated packet or corrupt tile size"); + } else { + size = data_end - *data; + } + + buf->data = *data; + buf->size = size; + + *data += size; +} + +static void get_tile_buffers( + AV1Decoder *pbi, const uint8_t *data, const uint8_t *data_end, + TileBufferDec (*const tile_buffers)[MAX_TILE_COLS]) { + AV1_COMMON *const cm = &pbi->common; +#if CONFIG_TILE_GROUPS + int r, c; + const int tile_cols = cm->tile_cols; + const int tile_rows = cm->tile_rows; + int tc = 0; + int first_tile_in_tg = 0; + struct aom_read_bit_buffer rb_tg_hdr; + uint8_t clear_data[MAX_AV1_HEADER_SIZE]; + const int num_tiles = tile_rows * tile_cols; + const int num_bits = OD_ILOG(num_tiles) - 1; + const size_t hdr_size = pbi->uncomp_hdr_size + pbi->first_partition_size; + const int tg_size_bit_offset = pbi->tg_size_bit_offset; +#if CONFIG_DEPENDENT_HORZTILES + int tile_group_start_col = 0; + int tile_group_start_row = 0; +#endif + + for (r = 0; r < tile_rows; ++r) { + for (c = 0; c < tile_cols; ++c, ++tc) { + TileBufferDec *const buf = &tile_buffers[r][c]; + const int is_last = (r == tile_rows - 1) && (c == tile_cols - 1); + const size_t hdr_offset = (tc && tc == first_tile_in_tg) ? hdr_size : 0; + + buf->col = c; + if (hdr_offset) { + init_read_bit_buffer(pbi, &rb_tg_hdr, data, data_end, clear_data); + rb_tg_hdr.bit_offset = tg_size_bit_offset; + if (num_tiles) { + pbi->tg_start = aom_rb_read_literal(&rb_tg_hdr, num_bits); + pbi->tg_size = 1 + aom_rb_read_literal(&rb_tg_hdr, num_bits); +#if CONFIG_DEPENDENT_HORZTILES + tile_group_start_row = r; + tile_group_start_col = c; +#endif + } + } + first_tile_in_tg += tc == first_tile_in_tg ? pbi->tg_size : 0; + data += hdr_offset; + get_tile_buffer(data_end, pbi->tile_size_bytes, is_last, + &pbi->common.error, &data, pbi->decrypt_cb, + pbi->decrypt_state, buf); +#if CONFIG_DEPENDENT_HORZTILES + cm->tile_group_start_row[r][c] = tile_group_start_row; + cm->tile_group_start_col[r][c] = tile_group_start_col; +#endif + } + } +#else + int r, c; + const int tile_cols = cm->tile_cols; + const int tile_rows = cm->tile_rows; + + for (r = 0; r < tile_rows; ++r) { + for (c = 0; c < tile_cols; ++c) { + const int is_last = (r == tile_rows - 1) && (c == tile_cols - 1); + TileBufferDec *const buf = &tile_buffers[r][c]; + buf->col = c; + get_tile_buffer(data_end, pbi->tile_size_bytes, is_last, &cm->error, + &data, pbi->decrypt_cb, pbi->decrypt_state, buf); + } + } +#endif +} +#endif // CONFIG_EXT_TILE + +#if CONFIG_PVQ +static void daala_dec_init(AV1_COMMON *const cm, daala_dec_ctx *daala_dec, + aom_reader *r) { + daala_dec->r = r; + + // TODO(yushin) : activity masking info needs be signaled by a bitstream + daala_dec->use_activity_masking = AV1_PVQ_ENABLE_ACTIVITY_MASKING; + +#if !CONFIG_DAALA_DIST + daala_dec->use_activity_masking = 0; +#endif + + if (daala_dec->use_activity_masking) + daala_dec->qm = OD_HVS_QM; + else + daala_dec->qm = OD_FLAT_QM; + + od_init_qm(daala_dec->state.qm, daala_dec->state.qm_inv, + daala_dec->qm == OD_HVS_QM ? OD_QM8_Q4_HVS : OD_QM8_Q4_FLAT); + + if (daala_dec->use_activity_masking) { + int pli; + int use_masking = daala_dec->use_activity_masking; + int segment_id = 0; + int qindex = av1_get_qindex(&cm->seg, segment_id, cm->base_qindex); + + for (pli = 0; pli < MAX_MB_PLANE; pli++) { + int i; + int q; + + q = qindex; + if (q <= OD_DEFAULT_QMS[use_masking][0][pli].interp_q << OD_COEFF_SHIFT) { + od_interp_qm(&daala_dec->state.pvq_qm_q4[pli][0], q, + &OD_DEFAULT_QMS[use_masking][0][pli], NULL); + } else { + i = 0; + while (OD_DEFAULT_QMS[use_masking][i + 1][pli].qm_q4 != NULL && + q > OD_DEFAULT_QMS[use_masking][i + 1][pli].interp_q + << OD_COEFF_SHIFT) { + i++; + } + od_interp_qm(&daala_dec->state.pvq_qm_q4[pli][0], q, + &OD_DEFAULT_QMS[use_masking][i][pli], + &OD_DEFAULT_QMS[use_masking][i + 1][pli]); + } + } + } +} +#endif // #if CONFIG_PVQ + +static const uint8_t *decode_tiles(AV1Decoder *pbi, const uint8_t *data, + const uint8_t *data_end) { + AV1_COMMON *const cm = &pbi->common; + const AVxWorkerInterface *const winterface = aom_get_worker_interface(); + const int tile_cols = cm->tile_cols; + const int tile_rows = cm->tile_rows; + const int n_tiles = tile_cols * tile_rows; + TileBufferDec(*const tile_buffers)[MAX_TILE_COLS] = pbi->tile_buffers; +#if CONFIG_EXT_TILE + const int dec_tile_row = AOMMIN(pbi->dec_tile_row, tile_rows); + const int single_row = pbi->dec_tile_row >= 0; + const int tile_rows_start = single_row ? dec_tile_row : 0; + const int tile_rows_end = single_row ? dec_tile_row + 1 : tile_rows; + const int dec_tile_col = AOMMIN(pbi->dec_tile_col, tile_cols); + const int single_col = pbi->dec_tile_col >= 0; + const int tile_cols_start = single_col ? dec_tile_col : 0; + const int tile_cols_end = single_col ? tile_cols_start + 1 : tile_cols; + const int inv_col_order = pbi->inv_tile_order && !single_col; + const int inv_row_order = pbi->inv_tile_order && !single_row; +#else + const int tile_rows_start = 0; + const int tile_rows_end = tile_rows; + const int tile_cols_start = 0; + const int tile_cols_end = tile_cols; + const int inv_col_order = pbi->inv_tile_order; + const int inv_row_order = pbi->inv_tile_order; +#endif // CONFIG_EXT_TILE + int tile_row, tile_col; + +#if CONFIG_SUBFRAME_PROB_UPDATE + cm->do_subframe_update = n_tiles == 1; +#endif // CONFIG_SUBFRAME_PROB_UPDATE + + if (cm->lf.filter_level && !cm->skip_loop_filter && + pbi->lf_worker.data1 == NULL) { + CHECK_MEM_ERROR(cm, pbi->lf_worker.data1, + aom_memalign(32, sizeof(LFWorkerData))); + pbi->lf_worker.hook = (AVxWorkerHook)av1_loop_filter_worker; + if (pbi->max_threads > 1 && !winterface->reset(&pbi->lf_worker)) { + aom_internal_error(&cm->error, AOM_CODEC_ERROR, + "Loop filter thread creation failed"); + } + } + + if (cm->lf.filter_level && !cm->skip_loop_filter) { + LFWorkerData *const lf_data = (LFWorkerData *)pbi->lf_worker.data1; + // Be sure to sync as we might be resuming after a failed frame decode. + winterface->sync(&pbi->lf_worker); + av1_loop_filter_data_reset(lf_data, get_frame_new_buffer(cm), cm, + pbi->mb.plane); + } + + assert(tile_rows <= MAX_TILE_ROWS); + assert(tile_cols <= MAX_TILE_COLS); + + get_tile_buffers(pbi, data, data_end, tile_buffers); + + if (pbi->tile_data == NULL || n_tiles != pbi->allocated_tiles) { + aom_free(pbi->tile_data); + CHECK_MEM_ERROR(cm, pbi->tile_data, + aom_memalign(32, n_tiles * (sizeof(*pbi->tile_data)))); + pbi->allocated_tiles = n_tiles; + } +#if CONFIG_ACCOUNTING + if (pbi->acct_enabled) { + aom_accounting_reset(&pbi->accounting); + } +#endif + // Load all tile information into tile_data. + for (tile_row = tile_rows_start; tile_row < tile_rows_end; ++tile_row) { + for (tile_col = tile_cols_start; tile_col < tile_cols_end; ++tile_col) { + const TileBufferDec *const buf = &tile_buffers[tile_row][tile_col]; + TileData *const td = pbi->tile_data + tile_cols * tile_row + tile_col; + + td->cm = cm; + td->xd = pbi->mb; + td->xd.corrupted = 0; + td->xd.counts = + cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD + ? &cm->counts + : NULL; + av1_zero(td->dqcoeff); +#if CONFIG_PVQ + av1_zero(td->pvq_ref_coeff); +#endif + av1_tile_init(&td->xd.tile, td->cm, tile_row, tile_col); + setup_bool_decoder(buf->data, data_end, buf->size, &cm->error, + &td->bit_reader, +#if CONFIG_ANS && ANS_MAX_SYMBOLS + 1 << cm->ans_window_size_log2, +#endif // CONFIG_ANS && ANS_MAX_SYMBOLS + pbi->decrypt_cb, pbi->decrypt_state); +#if CONFIG_ACCOUNTING + if (pbi->acct_enabled) { + td->bit_reader.accounting = &pbi->accounting; + } else { + td->bit_reader.accounting = NULL; + } +#endif + av1_init_macroblockd(cm, &td->xd, +#if CONFIG_PVQ + td->pvq_ref_coeff, +#endif +#if CONFIG_CFL + &td->cfl, +#endif + td->dqcoeff); + +#if CONFIG_EC_ADAPT + // Initialise the tile context from the frame context + td->tctx = *cm->fc; + td->xd.tile_ctx = &td->tctx; +#endif + +#if CONFIG_PVQ + daala_dec_init(cm, &td->xd.daala_dec, &td->bit_reader); + td->xd.daala_dec.state.adapt = &td->tctx.pvq_context; +#endif + +#if CONFIG_PALETTE + td->xd.plane[0].color_index_map = td->color_index_map[0]; + td->xd.plane[1].color_index_map = td->color_index_map[1]; +#endif // CONFIG_PALETTE + } + } + + for (tile_row = tile_rows_start; tile_row < tile_rows_end; ++tile_row) { + const int row = inv_row_order ? tile_rows - 1 - tile_row : tile_row; + int mi_row = 0; + TileInfo tile_info; + + av1_tile_set_row(&tile_info, cm, row); + + for (tile_col = tile_cols_start; tile_col < tile_cols_end; ++tile_col) { + const int col = inv_col_order ? tile_cols - 1 - tile_col : tile_col; + TileData *const td = pbi->tile_data + tile_cols * row + col; +#if CONFIG_ACCOUNTING + if (pbi->acct_enabled) { + td->bit_reader.accounting->last_tell_frac = + aom_reader_tell_frac(&td->bit_reader); + } +#endif + + av1_tile_set_col(&tile_info, cm, col); + +#if CONFIG_DEPENDENT_HORZTILES +#if CONFIG_TILE_GROUPS + av1_tile_set_tg_boundary(&tile_info, cm, tile_row, tile_col); + if (!cm->dependent_horz_tiles || tile_row == 0 || + tile_info.tg_horz_boundary) { +#else + if (!cm->dependent_horz_tiles || tile_row == 0) { +#endif + av1_zero_above_context(cm, tile_info.mi_col_start, + tile_info.mi_col_end); + } +#else + av1_zero_above_context(cm, tile_info.mi_col_start, tile_info.mi_col_end); +#endif + + for (mi_row = tile_info.mi_row_start; mi_row < tile_info.mi_row_end; + mi_row += cm->mib_size) { + int mi_col; + + av1_zero_left_context(&td->xd); + + for (mi_col = tile_info.mi_col_start; mi_col < tile_info.mi_col_end; + mi_col += cm->mib_size) { + av1_update_boundary_info(cm, &tile_info, mi_row, mi_col); + decode_partition(pbi, &td->xd, +#if CONFIG_SUPERTX + 0, +#endif // CONFIG_SUPERTX + mi_row, mi_col, &td->bit_reader, cm->sb_size, + b_width_log2_lookup[cm->sb_size]); +#if CONFIG_NCOBMC && CONFIG_MOTION_VAR + detoken_and_recon_sb(pbi, &td->xd, mi_row, mi_col, &td->bit_reader, + cm->sb_size); +#endif + } + aom_merge_corrupted_flag(&pbi->mb.corrupted, td->xd.corrupted); + if (pbi->mb.corrupted) + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Failed to decode tile data"); +#if CONFIG_SUBFRAME_PROB_UPDATE + if (cm->do_subframe_update && + cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { + const int mi_rows_per_update = + MI_SIZE * AOMMAX(cm->mi_rows / MI_SIZE / COEF_PROBS_BUFS, 1); + if ((mi_row + MI_SIZE) % mi_rows_per_update == 0 && + mi_row + MI_SIZE < cm->mi_rows && + cm->coef_probs_update_idx < COEF_PROBS_BUFS - 1) { + av1_partial_adapt_probs(cm, mi_row, mi_col); + ++cm->coef_probs_update_idx; + } + } +#endif // CONFIG_SUBFRAME_PROB_UPDATE + } + } + + assert(mi_row > 0); + +// when Parallel deblocking is enabled, deblocking should not +// be interleaved with decoding. Instead, deblocking should be done +// after the entire frame is decoded. +#if !CONFIG_VAR_TX && !CONFIG_PARALLEL_DEBLOCKING && !CONFIG_CB4X4 + // Loopfilter one tile row. + // Note: If out-of-order tile decoding is used(for example, inv_row_order + // = 1), the loopfiltering has be done after all tile rows are decoded. + if (!inv_row_order && cm->lf.filter_level && !cm->skip_loop_filter) { + LFWorkerData *const lf_data = (LFWorkerData *)pbi->lf_worker.data1; + const int lf_start = AOMMAX(0, tile_info.mi_row_start - cm->mib_size); + const int lf_end = tile_info.mi_row_end - cm->mib_size; + + // Delay the loopfilter if the first tile row is only + // a single superblock high. + if (lf_end <= 0) continue; + + // Decoding has completed. Finish up the loop filter in this thread. + if (tile_info.mi_row_end >= cm->mi_rows) continue; + + winterface->sync(&pbi->lf_worker); + lf_data->start = lf_start; + lf_data->stop = lf_end; + if (pbi->max_threads > 1) { + winterface->launch(&pbi->lf_worker); + } else { + winterface->execute(&pbi->lf_worker); + } + } +#endif // !CONFIG_VAR_TX && !CONFIG_PARALLEL_DEBLOCKING + + // After loopfiltering, the last 7 row pixels in each superblock row may + // still be changed by the longest loopfilter of the next superblock row. + if (cm->frame_parallel_decode) + av1_frameworker_broadcast(pbi->cur_buf, mi_row << cm->mib_size_log2); + } + +#if CONFIG_VAR_TX || CONFIG_CB4X4 + // Loopfilter the whole frame. + av1_loop_filter_frame(get_frame_new_buffer(cm), cm, &pbi->mb, + cm->lf.filter_level, 0, 0); +#else +#if CONFIG_PARALLEL_DEBLOCKING + // Loopfilter all rows in the frame in the frame. + if (cm->lf.filter_level && !cm->skip_loop_filter) { + LFWorkerData *const lf_data = (LFWorkerData *)pbi->lf_worker.data1; + winterface->sync(&pbi->lf_worker); + lf_data->start = 0; + lf_data->stop = cm->mi_rows; + winterface->execute(&pbi->lf_worker); + } +#else + // Loopfilter remaining rows in the frame. + if (cm->lf.filter_level && !cm->skip_loop_filter) { + LFWorkerData *const lf_data = (LFWorkerData *)pbi->lf_worker.data1; + winterface->sync(&pbi->lf_worker); + lf_data->start = lf_data->stop; + lf_data->stop = cm->mi_rows; + winterface->execute(&pbi->lf_worker); + } +#endif // CONFIG_PARALLEL_DEBLOCKING +#endif // CONFIG_VAR_TX + if (cm->frame_parallel_decode) + av1_frameworker_broadcast(pbi->cur_buf, INT_MAX); + +#if CONFIG_EXT_TILE + if (n_tiles == 1) { +#if CONFIG_ANS + return data_end; +#else + // Find the end of the single tile buffer + return aom_reader_find_end(&pbi->tile_data->bit_reader); +#endif // CONFIG_ANS + } else { + // Return the end of the last tile buffer + return tile_buffers[tile_rows - 1][tile_cols - 1].raw_data_end; + } +#else +#if CONFIG_ANS + return data_end; +#else + { + // Get last tile data. + TileData *const td = pbi->tile_data + tile_cols * tile_rows - 1; + return aom_reader_find_end(&td->bit_reader); + } +#endif // CONFIG_ANS +#endif // CONFIG_EXT_TILE +} + +static int tile_worker_hook(TileWorkerData *const tile_data, + const TileInfo *const tile) { + AV1Decoder *const pbi = tile_data->pbi; + const AV1_COMMON *const cm = &pbi->common; + int mi_row, mi_col; + + if (setjmp(tile_data->error_info.jmp)) { + tile_data->error_info.setjmp = 0; + aom_merge_corrupted_flag(&tile_data->xd.corrupted, 1); + return 0; + } + + tile_data->error_info.setjmp = 1; + tile_data->xd.error_info = &tile_data->error_info; +#if CONFIG_DEPENDENT_HORZTILES +#if CONFIG_TILE_GROUPS + if (!cm->dependent_horz_tiles || tile->tg_horz_boundary) { +#else + if (!cm->dependent_horz_tiles) { +#endif + av1_zero_above_context(&pbi->common, tile->mi_col_start, tile->mi_col_end); + } +#else + av1_zero_above_context(&pbi->common, tile->mi_col_start, tile->mi_col_end); +#endif + + for (mi_row = tile->mi_row_start; mi_row < tile->mi_row_end; + mi_row += cm->mib_size) { + av1_zero_left_context(&tile_data->xd); + + for (mi_col = tile->mi_col_start; mi_col < tile->mi_col_end; + mi_col += cm->mib_size) { + decode_partition(pbi, &tile_data->xd, +#if CONFIG_SUPERTX + 0, +#endif + mi_row, mi_col, &tile_data->bit_reader, cm->sb_size, + b_width_log2_lookup[cm->sb_size]); +#if CONFIG_NCOBMC && CONFIG_MOTION_VAR + detoken_and_recon_sb(pbi, &tile_data->xd, mi_row, mi_col, + &tile_data->bit_reader, cm->sb_size); +#endif + } + } + return !tile_data->xd.corrupted; +} + +// sorts in descending order +static int compare_tile_buffers(const void *a, const void *b) { + const TileBufferDec *const buf1 = (const TileBufferDec *)a; + const TileBufferDec *const buf2 = (const TileBufferDec *)b; + return (int)(buf2->size - buf1->size); +} + +static const uint8_t *decode_tiles_mt(AV1Decoder *pbi, const uint8_t *data, + const uint8_t *data_end) { + AV1_COMMON *const cm = &pbi->common; + const AVxWorkerInterface *const winterface = aom_get_worker_interface(); + const int tile_cols = cm->tile_cols; + const int tile_rows = cm->tile_rows; + const int num_workers = AOMMIN(pbi->max_threads & ~1, tile_cols); + TileBufferDec(*const tile_buffers)[MAX_TILE_COLS] = pbi->tile_buffers; +#if CONFIG_EXT_TILE + const int dec_tile_row = AOMMIN(pbi->dec_tile_row, tile_rows); + const int single_row = pbi->dec_tile_row >= 0; + const int tile_rows_start = single_row ? dec_tile_row : 0; + const int tile_rows_end = single_row ? dec_tile_row + 1 : tile_rows; + const int dec_tile_col = AOMMIN(pbi->dec_tile_col, tile_cols); + const int single_col = pbi->dec_tile_col >= 0; + const int tile_cols_start = single_col ? dec_tile_col : 0; + const int tile_cols_end = single_col ? tile_cols_start + 1 : tile_cols; +#else + const int tile_rows_start = 0; + const int tile_rows_end = tile_rows; + const int tile_cols_start = 0; + const int tile_cols_end = tile_cols; +#endif // CONFIG_EXT_TILE + int tile_row, tile_col; + int i; + +#if !(CONFIG_ANS || CONFIG_EXT_TILE) + int final_worker = -1; +#endif // !(CONFIG_ANS || CONFIG_EXT_TILE) + + assert(tile_rows <= MAX_TILE_ROWS); + assert(tile_cols <= MAX_TILE_COLS); + + assert(tile_cols * tile_rows > 1); + + // TODO(jzern): See if we can remove the restriction of passing in max + // threads to the decoder. + if (pbi->num_tile_workers == 0) { + const int num_threads = pbi->max_threads & ~1; + CHECK_MEM_ERROR(cm, pbi->tile_workers, + aom_malloc(num_threads * sizeof(*pbi->tile_workers))); + // Ensure tile data offsets will be properly aligned. This may fail on + // platforms without DECLARE_ALIGNED(). + assert((sizeof(*pbi->tile_worker_data) % 16) == 0); + CHECK_MEM_ERROR( + cm, pbi->tile_worker_data, + aom_memalign(32, num_threads * sizeof(*pbi->tile_worker_data))); + CHECK_MEM_ERROR(cm, pbi->tile_worker_info, + aom_malloc(num_threads * sizeof(*pbi->tile_worker_info))); + for (i = 0; i < num_threads; ++i) { + AVxWorker *const worker = &pbi->tile_workers[i]; + ++pbi->num_tile_workers; + + winterface->init(worker); + if (i < num_threads - 1 && !winterface->reset(worker)) { + aom_internal_error(&cm->error, AOM_CODEC_ERROR, + "Tile decoder thread creation failed"); + } + } + } + + // Reset tile decoding hook + for (i = 0; i < num_workers; ++i) { + AVxWorker *const worker = &pbi->tile_workers[i]; + winterface->sync(worker); + worker->hook = (AVxWorkerHook)tile_worker_hook; + worker->data1 = &pbi->tile_worker_data[i]; + worker->data2 = &pbi->tile_worker_info[i]; + } + + // Initialize thread frame counts. + if (cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { + for (i = 0; i < num_workers; ++i) { + TileWorkerData *const twd = (TileWorkerData *)pbi->tile_workers[i].data1; + av1_zero(twd->counts); + } + } + + // Load tile data into tile_buffers + get_tile_buffers(pbi, data, data_end, tile_buffers); + + for (tile_row = tile_rows_start; tile_row < tile_rows_end; ++tile_row) { + // Sort the buffers in this tile row based on size in descending order. + qsort(&tile_buffers[tile_row][tile_cols_start], + tile_cols_end - tile_cols_start, sizeof(tile_buffers[0][0]), + compare_tile_buffers); + + // Rearrange the tile buffers in this tile row such that per-tile group + // the largest, and presumably the most difficult tile will be decoded in + // the main thread. This should help minimize the number of instances + // where the main thread is waiting for a worker to complete. + { + int group_start; + for (group_start = tile_cols_start; group_start < tile_cols_end; + group_start += num_workers) { + const int group_end = AOMMIN(group_start + num_workers, tile_cols); + const TileBufferDec largest = tile_buffers[tile_row][group_start]; + memmove(&tile_buffers[tile_row][group_start], + &tile_buffers[tile_row][group_start + 1], + (group_end - group_start - 1) * sizeof(tile_buffers[0][0])); + tile_buffers[tile_row][group_end - 1] = largest; + } + } + + for (tile_col = tile_cols_start; tile_col < tile_cols_end;) { + // Launch workers for individual columns + for (i = 0; i < num_workers && tile_col < tile_cols_end; + ++i, ++tile_col) { + TileBufferDec *const buf = &tile_buffers[tile_row][tile_col]; + AVxWorker *const worker = &pbi->tile_workers[i]; + TileWorkerData *const twd = (TileWorkerData *)worker->data1; + TileInfo *const tile_info = (TileInfo *)worker->data2; + + twd->pbi = pbi; + twd->xd = pbi->mb; + twd->xd.corrupted = 0; + twd->xd.counts = + cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD + ? &twd->counts + : NULL; + av1_zero(twd->dqcoeff); + av1_tile_init(tile_info, cm, tile_row, buf->col); + av1_tile_init(&twd->xd.tile, cm, tile_row, buf->col); + setup_bool_decoder(buf->data, data_end, buf->size, &cm->error, + &twd->bit_reader, +#if CONFIG_ANS && ANS_MAX_SYMBOLS + 1 << cm->ans_window_size_log2, +#endif // CONFIG_ANS && ANS_MAX_SYMBOLS + pbi->decrypt_cb, pbi->decrypt_state); + av1_init_macroblockd(cm, &twd->xd, +#if CONFIG_PVQ + twd->pvq_ref_coeff, +#endif +#if CONFIG_CFL + &twd->cfl, +#endif + twd->dqcoeff); +#if CONFIG_PVQ + daala_dec_init(cm, &twd->xd.daala_dec, &twd->bit_reader); + twd->xd.daala_dec.state.adapt = &twd->tctx.pvq_context; +#endif +#if CONFIG_EC_ADAPT + // Initialise the tile context from the frame context + twd->tctx = *cm->fc; + twd->xd.tile_ctx = &twd->tctx; +#endif +#if CONFIG_PALETTE + twd->xd.plane[0].color_index_map = twd->color_index_map[0]; + twd->xd.plane[1].color_index_map = twd->color_index_map[1]; +#endif // CONFIG_PALETTE + + worker->had_error = 0; + if (i == num_workers - 1 || tile_col == tile_cols_end - 1) { + winterface->execute(worker); + } else { + winterface->launch(worker); + } + +#if !(CONFIG_ANS || CONFIG_EXT_TILE) + if (tile_row == tile_rows - 1 && buf->col == tile_cols - 1) { + final_worker = i; + } +#endif // !(CONFIG_ANS || CONFIG_EXT_TILE) + } + + // Sync all workers + for (; i > 0; --i) { + AVxWorker *const worker = &pbi->tile_workers[i - 1]; + // TODO(jzern): The tile may have specific error data associated with + // its aom_internal_error_info which could be propagated to the main + // info in cm. Additionally once the threads have been synced and an + // error is detected, there's no point in continuing to decode tiles. + pbi->mb.corrupted |= !winterface->sync(worker); + } + } + } + + // Accumulate thread frame counts. + if (cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { + for (i = 0; i < num_workers; ++i) { + TileWorkerData *const twd = (TileWorkerData *)pbi->tile_workers[i].data1; + av1_accumulate_frame_counts(&cm->counts, &twd->counts); + } + } + +#if CONFIG_EXT_TILE + // Return the end of the last tile buffer + return tile_buffers[tile_rows - 1][tile_cols - 1].raw_data_end; +#else +#if CONFIG_ANS + return data_end; +#else + assert(final_worker != -1); + { + TileWorkerData *const twd = + (TileWorkerData *)pbi->tile_workers[final_worker].data1; + return aom_reader_find_end(&twd->bit_reader); + } +#endif // CONFIG_ANS +#endif // CONFIG_EXT_TILE +} + +static void error_handler(void *data) { + AV1_COMMON *const cm = (AV1_COMMON *)data; + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, "Truncated packet"); +} + +static void read_bitdepth_colorspace_sampling(AV1_COMMON *cm, + struct aom_read_bit_buffer *rb) { + if (cm->profile >= PROFILE_2) { + cm->bit_depth = aom_rb_read_bit(rb) ? AOM_BITS_12 : AOM_BITS_10; + } else { + cm->bit_depth = AOM_BITS_8; + } + +#if CONFIG_HIGHBITDEPTH + if (cm->bit_depth > AOM_BITS_8) { + cm->use_highbitdepth = 1; + } else { +#if CONFIG_LOWBITDEPTH + cm->use_highbitdepth = 0; +#else + cm->use_highbitdepth = 1; +#endif + } +#endif + + cm->color_space = aom_rb_read_literal(rb, 3); + if (cm->color_space != AOM_CS_SRGB) { + // [16,235] (including xvycc) vs [0,255] range + cm->color_range = aom_rb_read_bit(rb); + if (cm->profile == PROFILE_1 || cm->profile == PROFILE_3) { + cm->subsampling_x = aom_rb_read_bit(rb); + cm->subsampling_y = aom_rb_read_bit(rb); + if (cm->subsampling_x == 1 && cm->subsampling_y == 1) + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "4:2:0 color not supported in profile 1 or 3"); + if (aom_rb_read_bit(rb)) + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "Reserved bit set"); + } else { + cm->subsampling_y = cm->subsampling_x = 1; + } + } else { + if (cm->profile == PROFILE_1 || cm->profile == PROFILE_3) { + // Note if colorspace is SRGB then 4:4:4 chroma sampling is assumed. + // 4:2:2 or 4:4:0 chroma sampling is not allowed. + cm->subsampling_y = cm->subsampling_x = 0; + if (aom_rb_read_bit(rb)) + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "Reserved bit set"); + } else { + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "4:4:4 color not supported in profile 0 or 2"); + } + } +} + +#if CONFIG_REFERENCE_BUFFER +void read_sequence_header(SequenceHeader *seq_params) { + /* Placeholder for actually reading from the bitstream */ + seq_params->frame_id_numbers_present_flag = FRAME_ID_NUMBERS_PRESENT_FLAG; + seq_params->frame_id_length_minus7 = FRAME_ID_LENGTH_MINUS7; + seq_params->delta_frame_id_length_minus2 = DELTA_FRAME_ID_LENGTH_MINUS2; +} +#endif + +static size_t read_uncompressed_header(AV1Decoder *pbi, + struct aom_read_bit_buffer *rb) { + AV1_COMMON *const cm = &pbi->common; + MACROBLOCKD *const xd = &pbi->mb; + BufferPool *const pool = cm->buffer_pool; + RefCntBuffer *const frame_bufs = pool->frame_bufs; + int i, mask, ref_index = 0; + size_t sz; + +#if CONFIG_REFERENCE_BUFFER + /* TODO: Move outside frame loop or inside key-frame branch */ + read_sequence_header(&pbi->seq_params); +#endif + + cm->last_frame_type = cm->frame_type; + cm->last_intra_only = cm->intra_only; + +#if CONFIG_EXT_REFS + // NOTE: By default all coded frames to be used as a reference + cm->is_reference_frame = 1; +#endif // CONFIG_EXT_REFS + + if (aom_rb_read_literal(rb, 2) != AOM_FRAME_MARKER) + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "Invalid frame marker"); + + cm->profile = av1_read_profile(rb); + + const BITSTREAM_PROFILE MAX_SUPPORTED_PROFILE = + CONFIG_HIGHBITDEPTH ? MAX_PROFILES : PROFILE_2; + + if (cm->profile >= MAX_SUPPORTED_PROFILE) + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "Unsupported bitstream profile"); + + cm->show_existing_frame = aom_rb_read_bit(rb); + + if (cm->show_existing_frame) { + // Show an existing frame directly. + const int existing_frame_idx = aom_rb_read_literal(rb, 3); + const int frame_to_show = cm->ref_frame_map[existing_frame_idx]; +#if CONFIG_REFERENCE_BUFFER + if (pbi->seq_params.frame_id_numbers_present_flag) { + int frame_id_length = pbi->seq_params.frame_id_length_minus7 + 7; + int display_frame_id = aom_rb_read_literal(rb, frame_id_length); + /* Compare display_frame_id with ref_frame_id and check valid for + * referencing */ + if (display_frame_id != cm->ref_frame_id[existing_frame_idx] || + cm->valid_for_referencing[existing_frame_idx] == 0) + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Reference buffer frame ID mismatch"); + } +#endif + lock_buffer_pool(pool); + if (frame_to_show < 0 || frame_bufs[frame_to_show].ref_count < 1) { + unlock_buffer_pool(pool); + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "Buffer %d does not contain a decoded frame", + frame_to_show); + } + ref_cnt_fb(frame_bufs, &cm->new_fb_idx, frame_to_show); + unlock_buffer_pool(pool); + + cm->lf.filter_level = 0; + cm->show_frame = 1; + pbi->refresh_frame_flags = 0; + + if (cm->frame_parallel_decode) { + for (i = 0; i < REF_FRAMES; ++i) + cm->next_ref_frame_map[i] = cm->ref_frame_map[i]; + } + + return 0; + } + + cm->frame_type = (FRAME_TYPE)aom_rb_read_bit(rb); + cm->show_frame = aom_rb_read_bit(rb); + cm->error_resilient_mode = aom_rb_read_bit(rb); +#if CONFIG_REFERENCE_BUFFER + if (pbi->seq_params.frame_id_numbers_present_flag) { + int frame_id_length = pbi->seq_params.frame_id_length_minus7 + 7; + int diff_len = pbi->seq_params.delta_frame_id_length_minus2 + 2; + int prev_frame_id = 0; + if (cm->frame_type != KEY_FRAME) { + prev_frame_id = cm->current_frame_id; + } + cm->current_frame_id = aom_rb_read_literal(rb, frame_id_length); + + if (cm->frame_type != KEY_FRAME) { + int diff_frame_id; + if (cm->current_frame_id > prev_frame_id) { + diff_frame_id = cm->current_frame_id - prev_frame_id; + } else { + diff_frame_id = + (1 << frame_id_length) + cm->current_frame_id - prev_frame_id; + } + /* Check current_frame_id for conformance */ + if (prev_frame_id == cm->current_frame_id || + diff_frame_id >= (1 << (frame_id_length - 1))) { + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Invalid value of current_frame_id"); + } + } + /* Check if some frames need to be marked as not valid for referencing */ + for (i = 0; i < REF_FRAMES; i++) { + if (cm->frame_type == KEY_FRAME) { + cm->valid_for_referencing[i] = 0; + } else if (cm->current_frame_id - (1 << diff_len) > 0) { + if (cm->ref_frame_id[i] > cm->current_frame_id || + cm->ref_frame_id[i] < cm->current_frame_id - (1 << diff_len)) + cm->valid_for_referencing[i] = 0; + } else { + if (cm->ref_frame_id[i] > cm->current_frame_id && + cm->ref_frame_id[i] < + (1 << frame_id_length) + cm->current_frame_id - (1 << diff_len)) + cm->valid_for_referencing[i] = 0; + } + } + } +#endif + if (cm->frame_type == KEY_FRAME) { + if (!av1_read_sync_code(rb)) + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "Invalid frame sync code"); + + read_bitdepth_colorspace_sampling(cm, rb); + pbi->refresh_frame_flags = (1 << REF_FRAMES) - 1; + + for (i = 0; i < INTER_REFS_PER_FRAME; ++i) { + cm->frame_refs[i].idx = INVALID_IDX; + cm->frame_refs[i].buf = NULL; + } + + setup_frame_size(cm, rb); + if (pbi->need_resync) { + memset(&cm->ref_frame_map, -1, sizeof(cm->ref_frame_map)); + pbi->need_resync = 0; + } +#if CONFIG_ANS && ANS_MAX_SYMBOLS + cm->ans_window_size_log2 = aom_rb_read_literal(rb, 4) + 8; +#endif // CONFIG_ANS && ANS_MAX_SYMBOLS +#if CONFIG_PALETTE + cm->allow_screen_content_tools = aom_rb_read_bit(rb); +#endif // CONFIG_PALETTE + } else { + cm->intra_only = cm->show_frame ? 0 : aom_rb_read_bit(rb); +#if CONFIG_PALETTE + if (cm->intra_only) cm->allow_screen_content_tools = aom_rb_read_bit(rb); +#endif // CONFIG_PALETTE + if (cm->error_resilient_mode) { + cm->reset_frame_context = RESET_FRAME_CONTEXT_ALL; + } else { + if (cm->intra_only) { + cm->reset_frame_context = aom_rb_read_bit(rb) + ? RESET_FRAME_CONTEXT_ALL + : RESET_FRAME_CONTEXT_CURRENT; + } else { + cm->reset_frame_context = aom_rb_read_bit(rb) + ? RESET_FRAME_CONTEXT_CURRENT + : RESET_FRAME_CONTEXT_NONE; + if (cm->reset_frame_context == RESET_FRAME_CONTEXT_CURRENT) + cm->reset_frame_context = aom_rb_read_bit(rb) + ? RESET_FRAME_CONTEXT_ALL + : RESET_FRAME_CONTEXT_CURRENT; + } + } + + if (cm->intra_only) { + if (!av1_read_sync_code(rb)) + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "Invalid frame sync code"); + + read_bitdepth_colorspace_sampling(cm, rb); + + pbi->refresh_frame_flags = aom_rb_read_literal(rb, REF_FRAMES); + setup_frame_size(cm, rb); + if (pbi->need_resync) { + memset(&cm->ref_frame_map, -1, sizeof(cm->ref_frame_map)); + pbi->need_resync = 0; + } +#if CONFIG_ANS && ANS_MAX_SYMBOLS + cm->ans_window_size_log2 = aom_rb_read_literal(rb, 4) + 8; +#endif + } else if (pbi->need_resync != 1) { /* Skip if need resync */ + pbi->refresh_frame_flags = aom_rb_read_literal(rb, REF_FRAMES); + +#if CONFIG_EXT_REFS + if (!pbi->refresh_frame_flags) { + // NOTE: "pbi->refresh_frame_flags == 0" indicates that the coded frame + // will not be used as a reference + cm->is_reference_frame = 0; + } +#endif // CONFIG_EXT_REFS + + for (i = 0; i < INTER_REFS_PER_FRAME; ++i) { + const int ref = aom_rb_read_literal(rb, REF_FRAMES_LOG2); + const int idx = cm->ref_frame_map[ref]; + RefBuffer *const ref_frame = &cm->frame_refs[i]; + ref_frame->idx = idx; + ref_frame->buf = &frame_bufs[idx].buf; + cm->ref_frame_sign_bias[LAST_FRAME + i] = aom_rb_read_bit(rb); +#if CONFIG_REFERENCE_BUFFER + if (pbi->seq_params.frame_id_numbers_present_flag) { + int frame_id_length = pbi->seq_params.frame_id_length_minus7 + 7; + int diff_len = pbi->seq_params.delta_frame_id_length_minus2 + 2; + int delta_frame_id_minus1 = aom_rb_read_literal(rb, diff_len); + int ref_frame_id = + ((cm->current_frame_id - (delta_frame_id_minus1 + 1) + + (1 << frame_id_length)) % + (1 << frame_id_length)); + /* Compare values derived from delta_frame_id_minus1 and + * refresh_frame_flags. Also, check valid for referencing */ + if (ref_frame_id != cm->ref_frame_id[ref] || + cm->valid_for_referencing[ref] == 0) + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Reference buffer frame ID mismatch"); + } +#endif + } + +#if CONFIG_FRAME_SIZE + if (cm->error_resilient_mode == 0) { + setup_frame_size_with_refs(cm, rb); + } else { + setup_frame_size(cm, rb); + } +#else + setup_frame_size_with_refs(cm, rb); +#endif + + cm->allow_high_precision_mv = aom_rb_read_bit(rb); + cm->interp_filter = read_frame_interp_filter(rb); +#if CONFIG_TEMPMV_SIGNALING + if (!cm->error_resilient_mode) { + cm->use_prev_frame_mvs = aom_rb_read_bit(rb); + } +#endif + for (i = 0; i < INTER_REFS_PER_FRAME; ++i) { + RefBuffer *const ref_buf = &cm->frame_refs[i]; +#if CONFIG_HIGHBITDEPTH + av1_setup_scale_factors_for_frame( + &ref_buf->sf, ref_buf->buf->y_crop_width, + ref_buf->buf->y_crop_height, cm->width, cm->height, + cm->use_highbitdepth); +#else + av1_setup_scale_factors_for_frame( + &ref_buf->sf, ref_buf->buf->y_crop_width, + ref_buf->buf->y_crop_height, cm->width, cm->height); +#endif + } + } + } +#if CONFIG_TEMPMV_SIGNALING + cm->cur_frame->intra_only = cm->frame_type == KEY_FRAME || cm->intra_only; +#endif + +#if CONFIG_REFERENCE_BUFFER + if (pbi->seq_params.frame_id_numbers_present_flag) { + /* If bitmask is set, update reference frame id values and + mark frames as valid for reference */ + int refresh_frame_flags = + cm->frame_type == KEY_FRAME ? 0xFF : pbi->refresh_frame_flags; + for (i = 0; i < REF_FRAMES; i++) { + if ((refresh_frame_flags >> i) & 1) { + cm->ref_frame_id[i] = cm->current_frame_id; + cm->valid_for_referencing[i] = 1; + } + } + } +#endif + +#if CONFIG_HIGHBITDEPTH + get_frame_new_buffer(cm)->bit_depth = cm->bit_depth; +#endif + get_frame_new_buffer(cm)->color_space = cm->color_space; + get_frame_new_buffer(cm)->color_range = cm->color_range; + get_frame_new_buffer(cm)->render_width = cm->render_width; + get_frame_new_buffer(cm)->render_height = cm->render_height; + + if (pbi->need_resync) { + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Keyframe / intra-only frame required to reset decoder" + " state"); + } + + if (!cm->error_resilient_mode) { + cm->refresh_frame_context = aom_rb_read_bit(rb) + ? REFRESH_FRAME_CONTEXT_FORWARD + : REFRESH_FRAME_CONTEXT_BACKWARD; + } else { + cm->refresh_frame_context = REFRESH_FRAME_CONTEXT_FORWARD; + } + + // This flag will be overridden by the call to av1_setup_past_independence + // below, forcing the use of context 0 for those frame types. + cm->frame_context_idx = aom_rb_read_literal(rb, FRAME_CONTEXTS_LOG2); + + // Generate next_ref_frame_map. + lock_buffer_pool(pool); + for (mask = pbi->refresh_frame_flags; mask; mask >>= 1) { + if (mask & 1) { + cm->next_ref_frame_map[ref_index] = cm->new_fb_idx; + ++frame_bufs[cm->new_fb_idx].ref_count; + } else { + cm->next_ref_frame_map[ref_index] = cm->ref_frame_map[ref_index]; + } + // Current thread holds the reference frame. + if (cm->ref_frame_map[ref_index] >= 0) + ++frame_bufs[cm->ref_frame_map[ref_index]].ref_count; + ++ref_index; + } + + for (; ref_index < REF_FRAMES; ++ref_index) { + cm->next_ref_frame_map[ref_index] = cm->ref_frame_map[ref_index]; + + // Current thread holds the reference frame. + if (cm->ref_frame_map[ref_index] >= 0) + ++frame_bufs[cm->ref_frame_map[ref_index]].ref_count; + } + unlock_buffer_pool(pool); + pbi->hold_ref_buf = 1; + + if (frame_is_intra_only(cm) || cm->error_resilient_mode) + av1_setup_past_independence(cm); + +#if CONFIG_EXT_PARTITION + set_sb_size(cm, aom_rb_read_bit(rb) ? BLOCK_128X128 : BLOCK_64X64); +#else + set_sb_size(cm, BLOCK_64X64); +#endif // CONFIG_EXT_PARTITION + + setup_loopfilter(cm, rb); +#if CONFIG_CDEF + setup_cdef(cm, rb); +#endif +#if CONFIG_LOOP_RESTORATION + decode_restoration_mode(cm, rb); +#endif // CONFIG_LOOP_RESTORATION + setup_quantization(cm, rb); +#if CONFIG_HIGHBITDEPTH + xd->bd = (int)cm->bit_depth; +#endif + +#if CONFIG_Q_ADAPT_PROBS + av1_default_coef_probs(cm); + if (cm->frame_type == KEY_FRAME || cm->error_resilient_mode || + cm->reset_frame_context == RESET_FRAME_CONTEXT_ALL) { + for (i = 0; i < FRAME_CONTEXTS; ++i) cm->frame_contexts[i] = *cm->fc; + } else if (cm->reset_frame_context == RESET_FRAME_CONTEXT_CURRENT) { + cm->frame_contexts[cm->frame_context_idx] = *cm->fc; + } +#endif // CONFIG_Q_ADAPT_PROBS + + setup_segmentation(cm, rb); + +#if CONFIG_DELTA_Q + { + struct segmentation *const seg = &cm->seg; + int segment_quantizer_active = 0; + for (i = 0; i < MAX_SEGMENTS; i++) { + if (segfeature_active(seg, i, SEG_LVL_ALT_Q)) { + segment_quantizer_active = 1; + } + } + + cm->delta_q_res = 1; +#if CONFIG_EXT_DELTA_Q + cm->delta_lf_res = 1; +#endif + if (segment_quantizer_active == 0 && cm->base_qindex > 0) { + cm->delta_q_present_flag = aom_rb_read_bit(rb); + } else { + cm->delta_q_present_flag = 0; + } + if (cm->delta_q_present_flag) { + xd->prev_qindex = cm->base_qindex; + cm->delta_q_res = 1 << aom_rb_read_literal(rb, 2); +#if CONFIG_EXT_DELTA_Q + if (segment_quantizer_active) { + assert(seg->abs_delta == SEGMENT_DELTADATA); + } + cm->delta_lf_present_flag = aom_rb_read_bit(rb); + if (cm->delta_lf_present_flag) { + xd->prev_delta_lf_from_base = 0; + cm->delta_lf_res = 1 << aom_rb_read_literal(rb, 2); + } else { + cm->delta_lf_present_flag = 0; + } +#endif // CONFIG_EXT_DELTA_Q + } + } +#endif + + for (i = 0; i < MAX_SEGMENTS; ++i) { + const int qindex = cm->seg.enabled + ? av1_get_qindex(&cm->seg, i, cm->base_qindex) + : cm->base_qindex; + xd->lossless[i] = qindex == 0 && cm->y_dc_delta_q == 0 && + cm->uv_dc_delta_q == 0 && cm->uv_ac_delta_q == 0; + xd->qindex[i] = qindex; + } + + setup_segmentation_dequant(cm); + cm->tx_mode = read_tx_mode(cm, xd, rb); + cm->reference_mode = read_frame_reference_mode(cm, rb); + +#if CONFIG_EXT_TX + cm->reduced_tx_set_used = aom_rb_read_bit(rb); +#endif // CONFIG_EXT_TX + + read_tile_info(pbi, rb); + sz = aom_rb_read_literal(rb, 16); + + if (sz == 0) + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Invalid header size"); + return sz; +} + +#if CONFIG_EXT_TX +#if !CONFIG_EC_ADAPT +static void read_ext_tx_probs(FRAME_CONTEXT *fc, aom_reader *r) { + int i, j, k; + int s; + for (s = 1; s < EXT_TX_SETS_INTER; ++s) { + if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) { + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + if (!use_inter_ext_tx_for_txsize[s][i]) continue; + for (j = 0; j < num_ext_tx_set[ext_tx_set_type_inter[s]] - 1; ++j) + av1_diff_update_prob(r, &fc->inter_ext_tx_prob[s][i][j], ACCT_STR); + } + } + } + + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { + if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) { + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + if (!use_intra_ext_tx_for_txsize[s][i]) continue; + for (j = 0; j < INTRA_MODES; ++j) + for (k = 0; k < num_ext_tx_set[ext_tx_set_type_intra[s]] - 1; ++k) + av1_diff_update_prob(r, &fc->intra_ext_tx_prob[s][i][j][k], + ACCT_STR); + } + } + } +} +#endif // !CONFIG_EC_ADAPT +#else + +#endif // CONFIG_EXT_TX +#if CONFIG_SUPERTX +static void read_supertx_probs(FRAME_CONTEXT *fc, aom_reader *r) { + int i, j; + if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) { + for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) { + for (j = TX_8X8; j < TX_SIZES; ++j) { + av1_diff_update_prob(r, &fc->supertx_prob[i][j], ACCT_STR); + } + } + } +} +#endif // CONFIG_SUPERTX + +#if CONFIG_GLOBAL_MOTION +static void read_global_motion_params(WarpedMotionParams *params, + WarpedMotionParams *ref_params, + aom_prob *probs, aom_reader *r, + int allow_hp) { + TransformationType type = + aom_read_tree(r, av1_global_motion_types_tree, probs, ACCT_STR); + int trans_bits; + int trans_dec_factor; + int trans_prec_diff; + set_default_warp_params(params); + params->wmtype = type; + switch (type) { + case HOMOGRAPHY: + case HORTRAPEZOID: + case VERTRAPEZOID: + if (type != HORTRAPEZOID) + params->wmmat[6] = + aom_read_signed_primitive_refsubexpfin( + r, GM_ROW3HOMO_MAX + 1, SUBEXPFIN_K, + (ref_params->wmmat[6] >> GM_ROW3HOMO_PREC_DIFF)) * + GM_ROW3HOMO_DECODE_FACTOR; + if (type != VERTRAPEZOID) + params->wmmat[7] = + aom_read_signed_primitive_refsubexpfin( + r, GM_ROW3HOMO_MAX + 1, SUBEXPFIN_K, + (ref_params->wmmat[7] >> GM_ROW3HOMO_PREC_DIFF)) * + GM_ROW3HOMO_DECODE_FACTOR; + case AFFINE: + case ROTZOOM: + params->wmmat[2] = aom_read_signed_primitive_refsubexpfin( + r, GM_ALPHA_MAX + 1, SUBEXPFIN_K, + (ref_params->wmmat[2] >> GM_ALPHA_PREC_DIFF) - + (1 << GM_ALPHA_PREC_BITS)) * + GM_ALPHA_DECODE_FACTOR + + (1 << WARPEDMODEL_PREC_BITS); + if (type != VERTRAPEZOID) + params->wmmat[3] = aom_read_signed_primitive_refsubexpfin( + r, GM_ALPHA_MAX + 1, SUBEXPFIN_K, + (ref_params->wmmat[3] >> GM_ALPHA_PREC_DIFF)) * + GM_ALPHA_DECODE_FACTOR; + if (type >= AFFINE) { + if (type != HORTRAPEZOID) + params->wmmat[4] = aom_read_signed_primitive_refsubexpfin( + r, GM_ALPHA_MAX + 1, SUBEXPFIN_K, + (ref_params->wmmat[4] >> GM_ALPHA_PREC_DIFF)) * + GM_ALPHA_DECODE_FACTOR; + params->wmmat[5] = aom_read_signed_primitive_refsubexpfin( + r, GM_ALPHA_MAX + 1, SUBEXPFIN_K, + (ref_params->wmmat[5] >> GM_ALPHA_PREC_DIFF) - + (1 << GM_ALPHA_PREC_BITS)) * + GM_ALPHA_DECODE_FACTOR + + (1 << WARPEDMODEL_PREC_BITS); + } else { + params->wmmat[4] = -params->wmmat[3]; + params->wmmat[5] = params->wmmat[2]; + } + // fallthrough intended + case TRANSLATION: + trans_bits = (type == TRANSLATION) ? GM_ABS_TRANS_ONLY_BITS - !allow_hp + : GM_ABS_TRANS_BITS; + trans_dec_factor = (type == TRANSLATION) + ? GM_TRANS_ONLY_DECODE_FACTOR * (1 << !allow_hp) + : GM_TRANS_DECODE_FACTOR; + trans_prec_diff = (type == TRANSLATION) + ? GM_TRANS_ONLY_PREC_DIFF + !allow_hp + : GM_TRANS_PREC_DIFF; + params->wmmat[0] = aom_read_signed_primitive_refsubexpfin( + r, (1 << trans_bits) + 1, SUBEXPFIN_K, + (ref_params->wmmat[0] >> trans_prec_diff)) * + trans_dec_factor; + params->wmmat[1] = aom_read_signed_primitive_refsubexpfin( + r, (1 << trans_bits) + 1, SUBEXPFIN_K, + (ref_params->wmmat[1] >> trans_prec_diff)) * + trans_dec_factor; + case IDENTITY: break; + default: assert(0); + } + if (params->wmtype <= AFFINE) + if (!get_shear_params(params)) assert(0); +} + +static void read_global_motion(AV1_COMMON *cm, aom_reader *r) { + int frame; + for (frame = LAST_FRAME; frame <= ALTREF_FRAME; ++frame) { + read_global_motion_params( + &cm->global_motion[frame], &cm->prev_frame->global_motion[frame], + cm->fc->global_motion_types_prob, r, cm->allow_high_precision_mv); + /* + printf("Dec Ref %d [%d/%d]: %d %d %d %d\n", + frame, cm->current_video_frame, cm->show_frame, + cm->global_motion[frame].wmmat[0], + cm->global_motion[frame].wmmat[1], + cm->global_motion[frame].wmmat[2], + cm->global_motion[frame].wmmat[3]); + */ + } + memcpy(cm->cur_frame->global_motion, cm->global_motion, + TOTAL_REFS_PER_FRAME * sizeof(WarpedMotionParams)); +} +#endif // CONFIG_GLOBAL_MOTION + +static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data, + size_t partition_size) { + AV1_COMMON *const cm = &pbi->common; +#if CONFIG_SUPERTX + MACROBLOCKD *const xd = &pbi->mb; +#endif + FRAME_CONTEXT *const fc = cm->fc; + aom_reader r; + int k, i; +#if !CONFIG_EC_ADAPT || \ + (CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION || CONFIG_EXT_INTER) + int j; +#endif + +#if CONFIG_ANS && ANS_MAX_SYMBOLS + r.window_size = 1 << cm->ans_window_size_log2; +#endif + if (aom_reader_init(&r, data, partition_size, pbi->decrypt_cb, + pbi->decrypt_state)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate bool decoder 0"); + +#if CONFIG_LOOP_RESTORATION + if (cm->rst_info[0].frame_restoration_type != RESTORE_NONE || + cm->rst_info[1].frame_restoration_type != RESTORE_NONE || + cm->rst_info[2].frame_restoration_type != RESTORE_NONE) { + av1_alloc_restoration_buffers(cm); + decode_restoration(cm, &r); + } +#endif + +#if !CONFIG_EC_ADAPT + if (cm->tx_mode == TX_MODE_SELECT) read_tx_size_probs(fc, &r); +#endif + +#if CONFIG_LV_MAP + av1_read_txb_probs(fc, cm->tx_mode, &r); +#else // CONFIG_LV_MAP +#if !CONFIG_PVQ +#if !(CONFIG_EC_ADAPT && CONFIG_NEW_TOKENSET) + read_coef_probs(fc, cm->tx_mode, &r); +#endif // !(CONFIG_EC_ADAPT && CONFIG_NEW_TOKENSET) +#endif // !CONFIG_PVQ +#endif // CONFIG_LV_MAP + +#if CONFIG_VAR_TX + for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k) + av1_diff_update_prob(&r, &fc->txfm_partition_prob[k], ACCT_STR); +#endif // CONFIG_VAR_TX + for (k = 0; k < SKIP_CONTEXTS; ++k) + av1_diff_update_prob(&r, &fc->skip_probs[k], ACCT_STR); + +#if CONFIG_DELTA_Q && !CONFIG_EC_ADAPT +#if CONFIG_EXT_DELTA_Q + if (cm->delta_q_present_flag) { + for (k = 0; k < DELTA_Q_PROBS; ++k) + av1_diff_update_prob(&r, &fc->delta_q_prob[k], ACCT_STR); + } + if (cm->delta_lf_present_flag) { + for (k = 0; k < DELTA_LF_PROBS; ++k) + av1_diff_update_prob(&r, &fc->delta_lf_prob[k], ACCT_STR); + } +#else + for (k = 0; k < DELTA_Q_PROBS; ++k) + av1_diff_update_prob(&r, &fc->delta_q_prob[k], ACCT_STR); +#endif +#endif + +#if !CONFIG_EC_ADAPT + if (cm->seg.enabled && cm->seg.update_map) { + if (cm->seg.temporal_update) { + for (k = 0; k < PREDICTION_PROBS; k++) + av1_diff_update_prob(&r, &cm->fc->seg.pred_probs[k], ACCT_STR); + } + for (k = 0; k < MAX_SEGMENTS - 1; k++) + av1_diff_update_prob(&r, &cm->fc->seg.tree_probs[k], ACCT_STR); + } + + for (j = 0; j < INTRA_MODES; j++) { + for (i = 0; i < INTRA_MODES - 1; ++i) + av1_diff_update_prob(&r, &fc->uv_mode_prob[j][i], ACCT_STR); + } + +#if CONFIG_EXT_PARTITION_TYPES + for (j = 0; j < PARTITION_PLOFFSET; ++j) + for (i = 0; i < PARTITION_TYPES - 1; ++i) + av1_diff_update_prob(&r, &fc->partition_prob[j][i], ACCT_STR); + for (; j < PARTITION_CONTEXTS_PRIMARY; ++j) + for (i = 0; i < EXT_PARTITION_TYPES - 1; ++i) + av1_diff_update_prob(&r, &fc->partition_prob[j][i], ACCT_STR); +#else + for (j = 0; j < PARTITION_CONTEXTS_PRIMARY; ++j) + for (i = 0; i < PARTITION_TYPES - 1; ++i) + av1_diff_update_prob(&r, &fc->partition_prob[j][i], ACCT_STR); +#endif // CONFIG_EXT_PARTITION_TYPES + +#if CONFIG_UNPOISON_PARTITION_CTX + for (; j < PARTITION_CONTEXTS_PRIMARY + PARTITION_BLOCK_SIZES; ++j) + av1_diff_update_prob(&r, &fc->partition_prob[j][PARTITION_VERT], ACCT_STR); + for (; j < PARTITION_CONTEXTS_PRIMARY + 2 * PARTITION_BLOCK_SIZES; ++j) + av1_diff_update_prob(&r, &fc->partition_prob[j][PARTITION_HORZ], ACCT_STR); +#endif // CONFIG_UNPOISON_PARTITION_CTX + +#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP + for (i = 0; i < INTRA_FILTERS + 1; ++i) + for (j = 0; j < INTRA_FILTERS - 1; ++j) + av1_diff_update_prob(&r, &fc->intra_filter_probs[i][j], ACCT_STR); +#endif // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP +#endif // !CONFIG_EC_ADAPT + + if (frame_is_intra_only(cm)) { + av1_copy(cm->kf_y_prob, av1_kf_y_mode_prob); +#if CONFIG_EC_MULTISYMBOL + av1_copy(cm->fc->kf_y_cdf, av1_kf_y_mode_cdf); +#endif +#if !CONFIG_EC_ADAPT + for (k = 0; k < INTRA_MODES; k++) + for (j = 0; j < INTRA_MODES; j++) + for (i = 0; i < INTRA_MODES - 1; ++i) + av1_diff_update_prob(&r, &cm->kf_y_prob[k][j][i], ACCT_STR); +#endif + } else { +#if !CONFIG_REF_MV + nmv_context *const nmvc = &fc->nmvc; +#endif + read_inter_mode_probs(fc, &r); + +#if CONFIG_EXT_INTER + read_inter_compound_mode_probs(fc, &r); + if (cm->reference_mode != COMPOUND_REFERENCE) { + for (i = 0; i < BLOCK_SIZE_GROUPS; i++) { + if (is_interintra_allowed_bsize_group(i)) { + av1_diff_update_prob(&r, &fc->interintra_prob[i], ACCT_STR); + } + } + for (i = 0; i < BLOCK_SIZE_GROUPS; i++) { + for (j = 0; j < INTERINTRA_MODES - 1; j++) + av1_diff_update_prob(&r, &fc->interintra_mode_prob[i][j], ACCT_STR); + } + for (i = 0; i < BLOCK_SIZES; i++) { + if (is_interintra_allowed_bsize(i) && is_interintra_wedge_used(i)) { + av1_diff_update_prob(&r, &fc->wedge_interintra_prob[i], ACCT_STR); + } + } + } +#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE + if (cm->reference_mode != SINGLE_REFERENCE) { + for (i = 0; i < BLOCK_SIZES; i++) { + for (j = 0; j < COMPOUND_TYPES - 1; j++) { + av1_diff_update_prob(&r, &fc->compound_type_prob[i][j], ACCT_STR); + } + } + } +#endif // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE +#endif // CONFIG_EXT_INTER + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + for (i = BLOCK_8X8; i < BLOCK_SIZES; ++i) { + for (j = 0; j < MOTION_MODES - 1; ++j) + av1_diff_update_prob(&r, &fc->motion_mode_prob[i][j], ACCT_STR); + } +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + +#if !CONFIG_EC_ADAPT + if (cm->interp_filter == SWITCHABLE) read_switchable_interp_probs(fc, &r); +#endif + + for (i = 0; i < INTRA_INTER_CONTEXTS; i++) + av1_diff_update_prob(&r, &fc->intra_inter_prob[i], ACCT_STR); + + if (cm->reference_mode != SINGLE_REFERENCE) + setup_compound_reference_mode(cm); + read_frame_reference_mode_probs(cm, &r); + +#if !CONFIG_EC_ADAPT + for (j = 0; j < BLOCK_SIZE_GROUPS; j++) { + for (i = 0; i < INTRA_MODES - 1; ++i) + av1_diff_update_prob(&r, &fc->y_mode_prob[j][i], ACCT_STR); + } +#endif + +#if CONFIG_REF_MV + for (i = 0; i < NMV_CONTEXTS; ++i) + read_mv_probs(&fc->nmvc[i], cm->allow_high_precision_mv, &r); +#else + read_mv_probs(nmvc, cm->allow_high_precision_mv, &r); +#endif +#if !CONFIG_EC_ADAPT + read_ext_tx_probs(fc, &r); +#endif // EC_ADAPT +#if CONFIG_SUPERTX + if (!xd->lossless[0]) read_supertx_probs(fc, &r); +#endif +#if CONFIG_GLOBAL_MOTION + read_global_motion(cm, &r); +#endif // EC_ADAPT, DAALA_EC + } +#if CONFIG_EC_MULTISYMBOL && !CONFIG_EC_ADAPT +#if CONFIG_NEW_TOKENSET + av1_coef_head_cdfs(fc); +#endif + /* Make tail distribution from head */ + av1_coef_pareto_cdfs(fc); +#if CONFIG_REF_MV + for (i = 0; i < NMV_CONTEXTS; ++i) av1_set_mv_cdfs(&fc->nmvc[i]); +#else + av1_set_mv_cdfs(&fc->nmvc); +#endif + av1_set_mode_cdfs(cm); +#endif // CONFIG_EC_MULTISYMBOL && !CONFIG_EC_ADAPT + + return aom_reader_has_error(&r); +} + +#ifdef NDEBUG +#define debug_check_frame_counts(cm) (void)0 +#else // !NDEBUG +// Counts should only be incremented when frame_parallel_decoding_mode and +// error_resilient_mode are disabled. +static void debug_check_frame_counts(const AV1_COMMON *const cm) { + FRAME_COUNTS zero_counts; + av1_zero(zero_counts); + assert(cm->refresh_frame_context != REFRESH_FRAME_CONTEXT_BACKWARD || + cm->error_resilient_mode); + assert(!memcmp(cm->counts.y_mode, zero_counts.y_mode, + sizeof(cm->counts.y_mode))); + assert(!memcmp(cm->counts.uv_mode, zero_counts.uv_mode, + sizeof(cm->counts.uv_mode))); + assert(!memcmp(cm->counts.partition, zero_counts.partition, + sizeof(cm->counts.partition))); + assert(!memcmp(cm->counts.coef, zero_counts.coef, sizeof(cm->counts.coef))); + assert(!memcmp(cm->counts.eob_branch, zero_counts.eob_branch, + sizeof(cm->counts.eob_branch))); +#if CONFIG_EC_MULTISYMBOL + assert(!memcmp(cm->counts.blockz_count, zero_counts.blockz_count, + sizeof(cm->counts.blockz_count))); +#endif + assert(!memcmp(cm->counts.switchable_interp, zero_counts.switchable_interp, + sizeof(cm->counts.switchable_interp))); + assert(!memcmp(cm->counts.inter_mode, zero_counts.inter_mode, + sizeof(cm->counts.inter_mode))); +#if CONFIG_EXT_INTER + assert(!memcmp(cm->counts.inter_compound_mode, + zero_counts.inter_compound_mode, + sizeof(cm->counts.inter_compound_mode))); + assert(!memcmp(cm->counts.interintra, zero_counts.interintra, + sizeof(cm->counts.interintra))); + assert(!memcmp(cm->counts.wedge_interintra, zero_counts.wedge_interintra, + sizeof(cm->counts.wedge_interintra))); + assert(!memcmp(cm->counts.compound_interinter, + zero_counts.compound_interinter, + sizeof(cm->counts.compound_interinter))); +#endif // CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + assert(!memcmp(cm->counts.motion_mode, zero_counts.motion_mode, + sizeof(cm->counts.motion_mode))); +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + assert(!memcmp(cm->counts.intra_inter, zero_counts.intra_inter, + sizeof(cm->counts.intra_inter))); + assert(!memcmp(cm->counts.comp_inter, zero_counts.comp_inter, + sizeof(cm->counts.comp_inter))); + assert(!memcmp(cm->counts.single_ref, zero_counts.single_ref, + sizeof(cm->counts.single_ref))); + assert(!memcmp(cm->counts.comp_ref, zero_counts.comp_ref, + sizeof(cm->counts.comp_ref))); +#if CONFIG_EXT_REFS + assert(!memcmp(cm->counts.comp_bwdref, zero_counts.comp_bwdref, + sizeof(cm->counts.comp_bwdref))); +#endif // CONFIG_EXT_REFS + assert(!memcmp(&cm->counts.tx_size, &zero_counts.tx_size, + sizeof(cm->counts.tx_size))); + assert(!memcmp(cm->counts.skip, zero_counts.skip, sizeof(cm->counts.skip))); +#if CONFIG_REF_MV + assert( + !memcmp(&cm->counts.mv[0], &zero_counts.mv[0], sizeof(cm->counts.mv[0]))); + assert( + !memcmp(&cm->counts.mv[1], &zero_counts.mv[1], sizeof(cm->counts.mv[0]))); +#else + assert(!memcmp(&cm->counts.mv, &zero_counts.mv, sizeof(cm->counts.mv))); +#endif + assert(!memcmp(cm->counts.inter_ext_tx, zero_counts.inter_ext_tx, + sizeof(cm->counts.inter_ext_tx))); + assert(!memcmp(cm->counts.intra_ext_tx, zero_counts.intra_ext_tx, + sizeof(cm->counts.intra_ext_tx))); +} +#endif // NDEBUG + +static struct aom_read_bit_buffer *init_read_bit_buffer( + AV1Decoder *pbi, struct aom_read_bit_buffer *rb, const uint8_t *data, + const uint8_t *data_end, uint8_t clear_data[MAX_AV1_HEADER_SIZE]) { + rb->bit_offset = 0; + rb->error_handler = error_handler; + rb->error_handler_data = &pbi->common; + if (pbi->decrypt_cb) { + const int n = (int)AOMMIN(MAX_AV1_HEADER_SIZE, data_end - data); + pbi->decrypt_cb(pbi->decrypt_state, data, clear_data, n); + rb->bit_buffer = clear_data; + rb->bit_buffer_end = clear_data + n; + } else { + rb->bit_buffer = data; + rb->bit_buffer_end = data_end; + } + return rb; +} + +//------------------------------------------------------------------------------ + +int av1_read_sync_code(struct aom_read_bit_buffer *const rb) { + return aom_rb_read_literal(rb, 8) == AV1_SYNC_CODE_0 && + aom_rb_read_literal(rb, 8) == AV1_SYNC_CODE_1 && + aom_rb_read_literal(rb, 8) == AV1_SYNC_CODE_2; +} + +void av1_read_frame_size(struct aom_read_bit_buffer *rb, int *width, + int *height) { + *width = aom_rb_read_literal(rb, 16) + 1; + *height = aom_rb_read_literal(rb, 16) + 1; +} + +BITSTREAM_PROFILE av1_read_profile(struct aom_read_bit_buffer *rb) { + int profile = aom_rb_read_bit(rb); + profile |= aom_rb_read_bit(rb) << 1; + if (profile > 2) profile += aom_rb_read_bit(rb); + return (BITSTREAM_PROFILE)profile; +} + +#if CONFIG_EC_ADAPT +static void make_update_tile_list_dec(AV1Decoder *pbi, int tile_rows, + int tile_cols, FRAME_CONTEXT *ec_ctxs[]) { + int i; + for (i = 0; i < tile_rows * tile_cols; ++i) + ec_ctxs[i] = &pbi->tile_data[i].tctx; +} +#endif + +void av1_decode_frame(AV1Decoder *pbi, const uint8_t *data, + const uint8_t *data_end, const uint8_t **p_data_end) { + AV1_COMMON *const cm = &pbi->common; + MACROBLOCKD *const xd = &pbi->mb; + struct aom_read_bit_buffer rb; + int context_updated = 0; + uint8_t clear_data[MAX_AV1_HEADER_SIZE]; + size_t first_partition_size; + YV12_BUFFER_CONFIG *new_fb; + +#if CONFIG_ADAPT_SCAN + av1_deliver_eob_threshold(cm, xd); +#endif +#if CONFIG_BITSTREAM_DEBUG + bitstream_queue_set_frame_read(cm->current_video_frame * 2 + cm->show_frame); +#endif + + first_partition_size = read_uncompressed_header( + pbi, init_read_bit_buffer(pbi, &rb, data, data_end, clear_data)); + +#if CONFIG_EXT_TILE + // If cm->tile_encoding_mode == TILE_NORMAL, the independent decoding of a + // single tile or a section of a frame is not allowed. + if (!cm->tile_encoding_mode && + (pbi->dec_tile_row >= 0 || pbi->dec_tile_col >= 0)) { + pbi->dec_tile_row = -1; + pbi->dec_tile_col = -1; + } +#endif // CONFIG_EXT_TILE + +#if CONFIG_TILE_GROUPS + pbi->first_partition_size = first_partition_size; + pbi->uncomp_hdr_size = aom_rb_bytes_read(&rb); +#endif + new_fb = get_frame_new_buffer(cm); + xd->cur_buf = new_fb; +#if CONFIG_GLOBAL_MOTION + int i; + for (i = LAST_FRAME; i <= ALTREF_FRAME; ++i) { + set_default_warp_params(&cm->global_motion[i]); + set_default_warp_params(&cm->cur_frame->global_motion[i]); + } + xd->global_motion = cm->global_motion; +#endif // CONFIG_GLOBAL_MOTION + + if (!first_partition_size) { + // showing a frame directly + *p_data_end = data + aom_rb_bytes_read(&rb); + return; + } + + data += aom_rb_bytes_read(&rb); + if (!read_is_valid(data, first_partition_size, data_end)) + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Truncated packet or corrupt header length"); + +#if CONFIG_REF_MV + cm->setup_mi(cm); +#endif + +#if CONFIG_TEMPMV_SIGNALING + if (cm->use_prev_frame_mvs) { + RefBuffer *last_fb_ref_buf = &cm->frame_refs[LAST_FRAME - LAST_FRAME]; + cm->prev_frame = &cm->buffer_pool->frame_bufs[last_fb_ref_buf->idx]; + assert(!cm->error_resilient_mode && + cm->width == last_fb_ref_buf->buf->y_width && + cm->height == last_fb_ref_buf->buf->y_height && + !cm->prev_frame->intra_only); + } +#else + cm->use_prev_frame_mvs = + !cm->error_resilient_mode && cm->width == cm->last_width && + cm->height == cm->last_height && !cm->last_intra_only && + cm->last_show_frame && (cm->last_frame_type != KEY_FRAME); +#endif +#if CONFIG_EXT_REFS + // NOTE(zoeliu): As cm->prev_frame can take neither a frame of + // show_exisiting_frame=1, nor can it take a frame not used as + // a reference, it is probable that by the time it is being + // referred to, the frame buffer it originally points to may + // already get expired and have been reassigned to the current + // newly coded frame. Hence, we need to check whether this is + // the case, and if yes, we have 2 choices: + // (1) Simply disable the use of previous frame mvs; or + // (2) Have cm->prev_frame point to one reference frame buffer, + // e.g. LAST_FRAME. + if (cm->use_prev_frame_mvs && !dec_is_ref_frame_buf(pbi, cm->prev_frame)) { + // Reassign the LAST_FRAME buffer to cm->prev_frame. + RefBuffer *last_fb_ref_buf = &cm->frame_refs[LAST_FRAME - LAST_FRAME]; + cm->prev_frame = &cm->buffer_pool->frame_bufs[last_fb_ref_buf->idx]; + } +#endif // CONFIG_EXT_REFS + + av1_setup_block_planes(xd, cm->subsampling_x, cm->subsampling_y); + + *cm->fc = cm->frame_contexts[cm->frame_context_idx]; + if (!cm->fc->initialized) + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Uninitialized entropy context."); + + av1_zero(cm->counts); + + xd->corrupted = 0; + new_fb->corrupted = read_compressed_header(pbi, data, first_partition_size); + if (new_fb->corrupted) + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Decode failed. Frame data header is corrupted."); + + if (cm->lf.filter_level && !cm->skip_loop_filter) { + av1_loop_filter_frame_init(cm, cm->lf.filter_level); + } + + // If encoded in frame parallel mode, frame context is ready after decoding + // the frame header. + if (cm->frame_parallel_decode && + cm->refresh_frame_context != REFRESH_FRAME_CONTEXT_BACKWARD) { + AVxWorker *const worker = pbi->frame_worker_owner; + FrameWorkerData *const frame_worker_data = worker->data1; + if (cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_FORWARD) { + context_updated = 1; + cm->frame_contexts[cm->frame_context_idx] = *cm->fc; + } + av1_frameworker_lock_stats(worker); + pbi->cur_buf->row = -1; + pbi->cur_buf->col = -1; + frame_worker_data->frame_context_ready = 1; + // Signal the main thread that context is ready. + av1_frameworker_signal_stats(worker); + av1_frameworker_unlock_stats(worker); + } + +#if CONFIG_SUBFRAME_PROB_UPDATE + av1_copy(cm->starting_coef_probs, cm->fc->coef_probs); + cm->coef_probs_update_idx = 0; +#endif // CONFIG_SUBFRAME_PROB_UPDATE + + if (pbi->max_threads > 1 && !CONFIG_CB4X4 && +#if CONFIG_EXT_TILE + pbi->dec_tile_col < 0 && // Decoding all columns +#endif // CONFIG_EXT_TILE + cm->tile_cols > 1) { + // Multi-threaded tile decoder + *p_data_end = decode_tiles_mt(pbi, data + first_partition_size, data_end); + if (!xd->corrupted) { + if (!cm->skip_loop_filter) { + // If multiple threads are used to decode tiles, then we use those + // threads to do parallel loopfiltering. + av1_loop_filter_frame_mt(new_fb, cm, pbi->mb.plane, cm->lf.filter_level, + 0, 0, pbi->tile_workers, pbi->num_tile_workers, + &pbi->lf_row_sync); + } + } else { + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Decode failed. Frame data is corrupted."); + } + } else { + *p_data_end = decode_tiles(pbi, data + first_partition_size, data_end); + } + +#if CONFIG_CDEF + if (!cm->skip_loop_filter) { + av1_cdef_frame(&pbi->cur_buf->buf, cm, &pbi->mb); + } +#endif // CONFIG_CDEF + +#if CONFIG_LOOP_RESTORATION + if (cm->rst_info[0].frame_restoration_type != RESTORE_NONE || + cm->rst_info[1].frame_restoration_type != RESTORE_NONE || + cm->rst_info[2].frame_restoration_type != RESTORE_NONE) { + av1_loop_restoration_frame(new_fb, cm, cm->rst_info, 7, 0, NULL); + } +#endif // CONFIG_LOOP_RESTORATION + + if (!xd->corrupted) { + if (cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { +#if CONFIG_EC_ADAPT + FRAME_CONTEXT **tile_ctxs = aom_malloc(cm->tile_rows * cm->tile_cols * + sizeof(&pbi->tile_data[0].tctx)); + aom_cdf_prob **cdf_ptrs = + aom_malloc(cm->tile_rows * cm->tile_cols * + sizeof(&pbi->tile_data[0].tctx.partition_cdf[0][0])); + make_update_tile_list_dec(pbi, cm->tile_rows, cm->tile_cols, tile_ctxs); +#endif + +#if CONFIG_SUBFRAME_PROB_UPDATE + cm->partial_prob_update = 0; +#endif // CONFIG_SUBFRAME_PROB_UPDATE + av1_adapt_coef_probs(cm); + av1_adapt_intra_frame_probs(cm); +#if CONFIG_EC_ADAPT + av1_average_tile_coef_cdfs(pbi->common.fc, tile_ctxs, cdf_ptrs, + cm->tile_rows * cm->tile_cols); + av1_average_tile_intra_cdfs(pbi->common.fc, tile_ctxs, cdf_ptrs, + cm->tile_rows * cm->tile_cols); +#if CONFIG_PVQ + av1_average_tile_pvq_cdfs(pbi->common.fc, tile_ctxs, + cm->tile_rows * cm->tile_cols); +#endif // CONFIG_PVQ +#endif // CONFIG_EC_ADAPT +#if CONFIG_ADAPT_SCAN + av1_adapt_scan_order(cm); +#endif // CONFIG_ADAPT_SCAN + + if (!frame_is_intra_only(cm)) { + av1_adapt_inter_frame_probs(cm); + av1_adapt_mv_probs(cm, cm->allow_high_precision_mv); +#if CONFIG_EC_ADAPT + av1_average_tile_inter_cdfs(&pbi->common, pbi->common.fc, tile_ctxs, + cdf_ptrs, cm->tile_rows * cm->tile_cols); + av1_average_tile_mv_cdfs(pbi->common.fc, tile_ctxs, cdf_ptrs, + cm->tile_rows * cm->tile_cols); +#endif + } +#if CONFIG_EC_ADAPT + aom_free(tile_ctxs); + aom_free(cdf_ptrs); +#endif + } else { + debug_check_frame_counts(cm); + } + } else { + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Decode failed. Frame data is corrupted."); + } + +#if CONFIG_INSPECTION + if (pbi->inspect_cb != NULL) { + (*pbi->inspect_cb)(pbi, pbi->inspect_ctx); + } +#endif + + // Non frame parallel update frame context here. + if (!cm->error_resilient_mode && !context_updated) + cm->frame_contexts[cm->frame_context_idx] = *cm->fc; +} diff --git a/third_party/aom/av1/decoder/decodeframe.h b/third_party/aom/av1/decoder/decodeframe.h new file mode 100644 index 0000000000..a904658b08 --- /dev/null +++ b/third_party/aom/av1/decoder/decodeframe.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_DECODER_DECODEFRAME_H_ +#define AV1_DECODER_DECODEFRAME_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +struct AV1Decoder; +struct aom_read_bit_buffer; + +#if CONFIG_REFERENCE_BUFFER +/* Placeholder for now */ +void read_sequence_header(SequenceHeader *seq_params); +#endif + +int av1_read_sync_code(struct aom_read_bit_buffer *const rb); +void av1_read_frame_size(struct aom_read_bit_buffer *rb, int *width, + int *height); +BITSTREAM_PROFILE av1_read_profile(struct aom_read_bit_buffer *rb); + +void av1_decode_frame(struct AV1Decoder *pbi, const uint8_t *data, + const uint8_t *data_end, const uint8_t **p_data_end); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_DECODER_DECODEFRAME_H_ diff --git a/third_party/aom/av1/decoder/decodemv.c b/third_party/aom/av1/decoder/decodemv.c new file mode 100644 index 0000000000..ec0f877512 --- /dev/null +++ b/third_party/aom/av1/decoder/decodemv.c @@ -0,0 +1,2405 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/common.h" +#include "av1/common/entropy.h" +#include "av1/common/entropymode.h" +#include "av1/common/entropymv.h" +#include "av1/common/mvref_common.h" +#include "av1/common/pred_common.h" +#include "av1/common/reconinter.h" +#if CONFIG_EXT_INTRA +#include "av1/common/reconintra.h" +#endif // CONFIG_EXT_INTRA +#include "av1/common/seg_common.h" +#if CONFIG_WARPED_MOTION +#include "av1/common/warped_motion.h" +#endif // CONFIG_WARPED_MOTION + +#include "av1/decoder/decodeframe.h" +#include "av1/decoder/decodemv.h" + +#include "aom_dsp/aom_dsp_common.h" + +#define ACCT_STR __func__ +#if CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA || CONFIG_PALETTE +static INLINE int read_uniform(aom_reader *r, int n) { + const int l = get_unsigned_bits(n); + const int m = (1 << l) - n; + const int v = aom_read_literal(r, l - 1, ACCT_STR); + assert(l != 0); + if (v < m) + return v; + else + return (v << 1) - m + aom_read_literal(r, 1, ACCT_STR); +} +#endif // CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA || CONFIG_PALETTE + +#if CONFIG_EC_MULTISYMBOL +static PREDICTION_MODE read_intra_mode(aom_reader *r, aom_cdf_prob *cdf) { + return (PREDICTION_MODE) + av1_intra_mode_inv[aom_read_symbol(r, cdf, INTRA_MODES, ACCT_STR)]; +} +#else +static PREDICTION_MODE read_intra_mode(aom_reader *r, const aom_prob *p) { + return (PREDICTION_MODE)aom_read_tree(r, av1_intra_mode_tree, p, ACCT_STR); +} +#endif + +#if CONFIG_DELTA_Q +static int read_delta_qindex(AV1_COMMON *cm, MACROBLOCKD *xd, aom_reader *r, + MB_MODE_INFO *const mbmi, int mi_col, int mi_row) { + FRAME_COUNTS *counts = xd->counts; + int sign, abs, reduced_delta_qindex = 0; + BLOCK_SIZE bsize = mbmi->sb_type; + const int b_col = mi_col & MAX_MIB_MASK; + const int b_row = mi_row & MAX_MIB_MASK; + const int read_delta_q_flag = (b_col == 0 && b_row == 0); + int rem_bits, thr; + int i, smallval; +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; + (void)cm; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + + if ((bsize != BLOCK_LARGEST || mbmi->skip == 0) && read_delta_q_flag) { +#if !CONFIG_EC_MULTISYMBOL + int bit = 1; + abs = 0; + while (abs < DELTA_Q_SMALL && bit) { + bit = aom_read(r, ec_ctx->delta_q_prob[abs], ACCT_STR); + abs += bit; + } +#else + abs = aom_read_symbol(r, ec_ctx->delta_q_cdf, DELTA_Q_PROBS + 1, ACCT_STR); +#endif + smallval = (abs < DELTA_Q_SMALL); + if (counts) { + for (i = 0; i < abs; ++i) counts->delta_q[i][1]++; + if (smallval) counts->delta_q[abs][0]++; + } + + if (!smallval) { + rem_bits = aom_read_literal(r, 3, ACCT_STR); + thr = (1 << rem_bits) + 1; + abs = aom_read_literal(r, rem_bits, ACCT_STR) + thr; + } + + if (abs) { + sign = aom_read_bit(r, ACCT_STR); + } else { + sign = 1; + } + + reduced_delta_qindex = sign ? -abs : abs; + } + return reduced_delta_qindex; +} +#if CONFIG_EXT_DELTA_Q +static int read_delta_lflevel(AV1_COMMON *cm, MACROBLOCKD *xd, aom_reader *r, + MB_MODE_INFO *const mbmi, int mi_col, + int mi_row) { + FRAME_COUNTS *counts = xd->counts; + int sign, abs, reduced_delta_lflevel = 0; + BLOCK_SIZE bsize = mbmi->sb_type; + const int b_col = mi_col & MAX_MIB_MASK; + const int b_row = mi_row & MAX_MIB_MASK; + const int read_delta_lf_flag = (b_col == 0 && b_row == 0); + int rem_bits, thr; + int i, smallval; +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; + (void)cm; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + + if ((bsize != BLOCK_64X64 || mbmi->skip == 0) && read_delta_lf_flag) { +#if !CONFIG_EC_MULTISYMBOL + int bit = 1; + abs = 0; + while (abs < DELTA_LF_SMALL && bit) { + bit = aom_read(r, ec_ctx->delta_lf_prob[abs], ACCT_STR); + abs += bit; + } +#else + abs = + aom_read_symbol(r, ec_ctx->delta_lf_cdf, DELTA_LF_PROBS + 1, ACCT_STR); +#endif + smallval = (abs < DELTA_LF_SMALL); + if (counts) { + for (i = 0; i < abs; ++i) counts->delta_lf[i][1]++; + if (smallval) counts->delta_lf[abs][0]++; + } + if (!smallval) { + rem_bits = aom_read_literal(r, 3, ACCT_STR); + thr = (1 << rem_bits) + 1; + abs = aom_read_literal(r, rem_bits, ACCT_STR) + thr; + } + + if (abs) { + sign = aom_read_bit(r, ACCT_STR); + } else { + sign = 1; + } + + reduced_delta_lflevel = sign ? -abs : abs; + } + return reduced_delta_lflevel; +} +#endif +#endif + +static PREDICTION_MODE read_intra_mode_y(AV1_COMMON *cm, MACROBLOCKD *xd, + aom_reader *r, int size_group) { +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#elif CONFIG_EC_MULTISYMBOL + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + + const PREDICTION_MODE y_mode = +#if CONFIG_EC_MULTISYMBOL + read_intra_mode(r, ec_ctx->y_mode_cdf[size_group]); +#else + read_intra_mode(r, cm->fc->y_mode_prob[size_group]); +#endif + FRAME_COUNTS *counts = xd->counts; +#if CONFIG_EC_ADAPT + (void)cm; +#endif + if (counts) ++counts->y_mode[size_group][y_mode]; + return y_mode; +} + +static PREDICTION_MODE read_intra_mode_uv(AV1_COMMON *cm, MACROBLOCKD *xd, + aom_reader *r, + PREDICTION_MODE y_mode) { +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#elif CONFIG_EC_MULTISYMBOL + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + + const PREDICTION_MODE uv_mode = +#if CONFIG_EC_MULTISYMBOL + read_intra_mode(r, ec_ctx->uv_mode_cdf[y_mode]); +#else + read_intra_mode(r, cm->fc->uv_mode_prob[y_mode]); +#endif + FRAME_COUNTS *counts = xd->counts; +#if CONFIG_EC_ADAPT + (void)cm; +#endif + if (counts) ++counts->uv_mode[y_mode][uv_mode]; + return uv_mode; +} + +#if CONFIG_EXT_INTER +static INTERINTRA_MODE read_interintra_mode(AV1_COMMON *cm, MACROBLOCKD *xd, + aom_reader *r, int size_group) { + const INTERINTRA_MODE ii_mode = (INTERINTRA_MODE)aom_read_tree( + r, av1_interintra_mode_tree, cm->fc->interintra_mode_prob[size_group], + ACCT_STR); + FRAME_COUNTS *counts = xd->counts; + if (counts) ++counts->interintra_mode[size_group][ii_mode]; + return ii_mode; +} +#endif // CONFIG_EXT_INTER + +static PREDICTION_MODE read_inter_mode(FRAME_CONTEXT *ec_ctx, MACROBLOCKD *xd, + aom_reader *r, int16_t ctx) { +#if CONFIG_REF_MV + FRAME_COUNTS *counts = xd->counts; + int16_t mode_ctx = ctx & NEWMV_CTX_MASK; + aom_prob mode_prob = ec_ctx->newmv_prob[mode_ctx]; + + if (aom_read(r, mode_prob, ACCT_STR) == 0) { + if (counts) ++counts->newmv_mode[mode_ctx][0]; + return NEWMV; + } + if (counts) ++counts->newmv_mode[mode_ctx][1]; + + if (ctx & (1 << ALL_ZERO_FLAG_OFFSET)) return ZEROMV; + + mode_ctx = (ctx >> ZEROMV_OFFSET) & ZEROMV_CTX_MASK; + + mode_prob = ec_ctx->zeromv_prob[mode_ctx]; + if (aom_read(r, mode_prob, ACCT_STR) == 0) { + if (counts) ++counts->zeromv_mode[mode_ctx][0]; + return ZEROMV; + } + if (counts) ++counts->zeromv_mode[mode_ctx][1]; + + mode_ctx = (ctx >> REFMV_OFFSET) & REFMV_CTX_MASK; + + if (ctx & (1 << SKIP_NEARESTMV_OFFSET)) mode_ctx = 6; + if (ctx & (1 << SKIP_NEARMV_OFFSET)) mode_ctx = 7; + if (ctx & (1 << SKIP_NEARESTMV_SUB8X8_OFFSET)) mode_ctx = 8; + + mode_prob = ec_ctx->refmv_prob[mode_ctx]; + + if (aom_read(r, mode_prob, ACCT_STR) == 0) { + if (counts) ++counts->refmv_mode[mode_ctx][0]; + + return NEARESTMV; + } else { + if (counts) ++counts->refmv_mode[mode_ctx][1]; + return NEARMV; + } + + // Invalid prediction mode. + assert(0); +#else +#if CONFIG_EC_MULTISYMBOL + const int mode = av1_inter_mode_inv[aom_read_symbol( + r, ec_ctx->inter_mode_cdf[ctx], INTER_MODES, ACCT_STR)]; +#else + const int mode = aom_read_tree(r, av1_inter_mode_tree, + ec_ctx->inter_mode_probs[ctx], ACCT_STR); +#endif + FRAME_COUNTS *counts = xd->counts; + if (counts) ++counts->inter_mode[ctx][mode]; + + return NEARESTMV + mode; +#endif +} + +#if CONFIG_REF_MV +static void read_drl_idx(const AV1_COMMON *cm, MACROBLOCKD *xd, + MB_MODE_INFO *mbmi, aom_reader *r) { + uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + mbmi->ref_mv_idx = 0; + +#if CONFIG_EXT_INTER + if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) { +#else + if (mbmi->mode == NEWMV) { +#endif + int idx; + for (idx = 0; idx < 2; ++idx) { + if (xd->ref_mv_count[ref_frame_type] > idx + 1) { + uint8_t drl_ctx = av1_drl_ctx(xd->ref_mv_stack[ref_frame_type], idx); + aom_prob drl_prob = cm->fc->drl_prob[drl_ctx]; + if (!aom_read(r, drl_prob, ACCT_STR)) { + mbmi->ref_mv_idx = idx; + if (xd->counts) ++xd->counts->drl_mode[drl_ctx][0]; + return; + } + mbmi->ref_mv_idx = idx + 1; + if (xd->counts) ++xd->counts->drl_mode[drl_ctx][1]; + } + } + } + + if (have_nearmv_in_inter_mode(mbmi->mode)) { + int idx; + // Offset the NEARESTMV mode. + // TODO(jingning): Unify the two syntax decoding loops after the NEARESTMV + // mode is factored in. + for (idx = 1; idx < 3; ++idx) { + if (xd->ref_mv_count[ref_frame_type] > idx + 1) { + uint8_t drl_ctx = av1_drl_ctx(xd->ref_mv_stack[ref_frame_type], idx); + aom_prob drl_prob = cm->fc->drl_prob[drl_ctx]; + if (!aom_read(r, drl_prob, ACCT_STR)) { + mbmi->ref_mv_idx = idx - 1; + if (xd->counts) ++xd->counts->drl_mode[drl_ctx][0]; + return; + } + mbmi->ref_mv_idx = idx; + if (xd->counts) ++xd->counts->drl_mode[drl_ctx][1]; + } + } + } +} +#endif + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +static MOTION_MODE read_motion_mode(AV1_COMMON *cm, MACROBLOCKD *xd, + MODE_INFO *mi, aom_reader *r) { + MB_MODE_INFO *mbmi = &mi->mbmi; + const MOTION_MODE last_motion_mode_allowed = motion_mode_allowed( +#if CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + 0, xd->global_motion, +#endif // CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + mi); + int motion_mode; + FRAME_COUNTS *counts = xd->counts; + + if (last_motion_mode_allowed == SIMPLE_TRANSLATION) return SIMPLE_TRANSLATION; +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + if (last_motion_mode_allowed == OBMC_CAUSAL) { + motion_mode = aom_read(r, cm->fc->obmc_prob[mbmi->sb_type], ACCT_STR); + if (counts) ++counts->obmc[mbmi->sb_type][motion_mode]; + return (MOTION_MODE)(SIMPLE_TRANSLATION + motion_mode); + } else { +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + motion_mode = + aom_read_tree(r, av1_motion_mode_tree, + cm->fc->motion_mode_prob[mbmi->sb_type], ACCT_STR); + if (counts) ++counts->motion_mode[mbmi->sb_type][motion_mode]; + return (MOTION_MODE)(SIMPLE_TRANSLATION + motion_mode); +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + } +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION +} +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + +#if CONFIG_EXT_INTER +static PREDICTION_MODE read_inter_compound_mode(AV1_COMMON *cm, MACROBLOCKD *xd, + aom_reader *r, int16_t ctx) { + const int mode = + aom_read_tree(r, av1_inter_compound_mode_tree, + cm->fc->inter_compound_mode_probs[ctx], ACCT_STR); + FRAME_COUNTS *counts = xd->counts; + + if (counts) ++counts->inter_compound_mode[ctx][mode]; + + assert(is_inter_compound_mode(NEAREST_NEARESTMV + mode)); + return NEAREST_NEARESTMV + mode; +} +#endif // CONFIG_EXT_INTER + +static int read_segment_id(aom_reader *r, struct segmentation_probs *segp) { +#if CONFIG_EC_MULTISYMBOL + return aom_read_symbol(r, segp->tree_cdf, MAX_SEGMENTS, ACCT_STR); +#else + return aom_read_tree(r, av1_segment_tree, segp->tree_probs, ACCT_STR); +#endif +} + +#if CONFIG_VAR_TX +static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, + MB_MODE_INFO *mbmi, FRAME_COUNTS *counts, + TX_SIZE tx_size, int depth, int blk_row, + int blk_col, aom_reader *r) { + int is_split = 0; + const int tx_row = blk_row >> 1; + const int tx_col = blk_col >> 1; + const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0); + const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0); + int ctx = txfm_partition_context(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, + mbmi->sb_type, tx_size); + TX_SIZE(*const inter_tx_size) + [MAX_MIB_SIZE] = + (TX_SIZE(*)[MAX_MIB_SIZE]) & mbmi->inter_tx_size[tx_row][tx_col]; + if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + + if (depth == MAX_VARTX_DEPTH) { + int idx, idy; + inter_tx_size[0][0] = tx_size; + for (idy = 0; idy < tx_size_high_unit[tx_size] / 2; ++idy) + for (idx = 0; idx < tx_size_wide_unit[tx_size] / 2; ++idx) + inter_tx_size[idy][idx] = tx_size; + mbmi->tx_size = tx_size; + mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size)); + if (counts) ++counts->txfm_partition[ctx][0]; + txfm_partition_update(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, tx_size, tx_size); + return; + } + + is_split = aom_read(r, cm->fc->txfm_partition_prob[ctx], ACCT_STR); + + if (is_split) { + const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; + const int bsl = tx_size_wide_unit[sub_txs]; + int i; + + if (counts) ++counts->txfm_partition[ctx][1]; + + if (tx_size == TX_8X8) { + int idx, idy; + inter_tx_size[0][0] = sub_txs; + for (idy = 0; idy < tx_size_high_unit[tx_size] / 2; ++idy) + for (idx = 0; idx < tx_size_wide_unit[tx_size] / 2; ++idx) + inter_tx_size[idy][idx] = inter_tx_size[0][0]; + mbmi->tx_size = sub_txs; + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); + txfm_partition_update(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, sub_txs, tx_size); + return; + } + + assert(bsl > 0); + for (i = 0; i < 4; ++i) { + int offsetr = blk_row + (i >> 1) * bsl; + int offsetc = blk_col + (i & 0x01) * bsl; + read_tx_size_vartx(cm, xd, mbmi, counts, sub_txs, depth + 1, offsetr, + offsetc, r); + } + } else { + int idx, idy; + inter_tx_size[0][0] = tx_size; + for (idy = 0; idy < tx_size_high_unit[tx_size] / 2; ++idy) + for (idx = 0; idx < tx_size_wide_unit[tx_size] / 2; ++idx) + inter_tx_size[idy][idx] = tx_size; + mbmi->tx_size = tx_size; + mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size)); + if (counts) ++counts->txfm_partition[ctx][0]; + txfm_partition_update(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, tx_size, tx_size); + } +} +#endif + +static TX_SIZE read_selected_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, + int tx_size_cat, aom_reader *r) { + FRAME_COUNTS *counts = xd->counts; + const int ctx = get_tx_size_context(xd); +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; + (void)cm; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + + const int depth = +#if CONFIG_EC_MULTISYMBOL + aom_read_symbol(r, ec_ctx->tx_size_cdf[tx_size_cat][ctx], tx_size_cat + 2, + ACCT_STR); +#else + aom_read_tree(r, av1_tx_size_tree[tx_size_cat], + ec_ctx->tx_size_probs[tx_size_cat][ctx], ACCT_STR); +#endif + const TX_SIZE tx_size = depth_to_tx_size(depth); +#if CONFIG_RECT_TX + assert(!is_rect_tx(tx_size)); +#endif // CONFIG_RECT_TX + if (counts) ++counts->tx_size[tx_size_cat][ctx][depth]; + return tx_size; +} + +static TX_SIZE read_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, int is_inter, + int allow_select_inter, aom_reader *r) { + const TX_MODE tx_mode = cm->tx_mode; + const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + if (xd->lossless[xd->mi[0]->mbmi.segment_id]) return TX_4X4; +#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX + if (bsize > BLOCK_4X4) { +#else + if (bsize >= BLOCK_8X8) { +#endif // CONFIG_CB4X4 && CONFIG_VAR_TX + if ((!is_inter || allow_select_inter) && tx_mode == TX_MODE_SELECT) { + const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize] + : intra_tx_size_cat_lookup[bsize]; + const TX_SIZE coded_tx_size = + read_selected_tx_size(cm, xd, tx_size_cat, r); +#if CONFIG_EXT_TX && CONFIG_RECT_TX + if (coded_tx_size > max_txsize_lookup[bsize]) { + assert(coded_tx_size == max_txsize_lookup[bsize] + 1); + return max_txsize_rect_lookup[bsize]; + } +#else + assert(coded_tx_size <= max_txsize_lookup[bsize]); +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + return coded_tx_size; + } else { + return tx_size_from_tx_mode(bsize, tx_mode, is_inter); + } + } else { +#if CONFIG_EXT_TX && CONFIG_RECT_TX + assert(IMPLIES(tx_mode == ONLY_4X4, bsize == BLOCK_4X4)); + return max_txsize_rect_lookup[bsize]; +#else + return TX_4X4; +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + } +} + +static int dec_get_segment_id(const AV1_COMMON *cm, const uint8_t *segment_ids, + int mi_offset, int x_mis, int y_mis) { + int x, y, segment_id = INT_MAX; + + for (y = 0; y < y_mis; y++) + for (x = 0; x < x_mis; x++) + segment_id = + AOMMIN(segment_id, segment_ids[mi_offset + y * cm->mi_cols + x]); + + assert(segment_id >= 0 && segment_id < MAX_SEGMENTS); + return segment_id; +} + +static void set_segment_id(AV1_COMMON *cm, int mi_offset, int x_mis, int y_mis, + int segment_id) { + int x, y; + + assert(segment_id >= 0 && segment_id < MAX_SEGMENTS); + + for (y = 0; y < y_mis; y++) + for (x = 0; x < x_mis; x++) + cm->current_frame_seg_map[mi_offset + y * cm->mi_cols + x] = segment_id; +} + +static int read_intra_segment_id(AV1_COMMON *const cm, MACROBLOCKD *const xd, + int mi_offset, int x_mis, int y_mis, + aom_reader *r) { + struct segmentation *const seg = &cm->seg; + FRAME_COUNTS *counts = xd->counts; + struct segmentation_probs *const segp = &cm->fc->seg; + int segment_id; + + if (!seg->enabled) return 0; // Default for disabled segmentation + + assert(seg->update_map && !seg->temporal_update); + + segment_id = read_segment_id(r, segp); + if (counts) ++counts->seg.tree_total[segment_id]; + set_segment_id(cm, mi_offset, x_mis, y_mis, segment_id); + return segment_id; +} + +static void copy_segment_id(const AV1_COMMON *cm, + const uint8_t *last_segment_ids, + uint8_t *current_segment_ids, int mi_offset, + int x_mis, int y_mis) { + int x, y; + + for (y = 0; y < y_mis; y++) + for (x = 0; x < x_mis; x++) + current_segment_ids[mi_offset + y * cm->mi_cols + x] = + last_segment_ids ? last_segment_ids[mi_offset + y * cm->mi_cols + x] + : 0; +} + +static int read_inter_segment_id(AV1_COMMON *const cm, MACROBLOCKD *const xd, + int mi_row, int mi_col, aom_reader *r) { + struct segmentation *const seg = &cm->seg; + FRAME_COUNTS *counts = xd->counts; + struct segmentation_probs *const segp = &cm->fc->seg; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + int predicted_segment_id, segment_id; + const int mi_offset = mi_row * cm->mi_cols + mi_col; + const int bw = mi_size_wide[mbmi->sb_type]; + const int bh = mi_size_high[mbmi->sb_type]; + + // TODO(slavarnway): move x_mis, y_mis into xd ????? + const int x_mis = AOMMIN(cm->mi_cols - mi_col, bw); + const int y_mis = AOMMIN(cm->mi_rows - mi_row, bh); + + if (!seg->enabled) return 0; // Default for disabled segmentation + + predicted_segment_id = cm->last_frame_seg_map + ? dec_get_segment_id(cm, cm->last_frame_seg_map, + mi_offset, x_mis, y_mis) + : 0; + + if (!seg->update_map) { + copy_segment_id(cm, cm->last_frame_seg_map, cm->current_frame_seg_map, + mi_offset, x_mis, y_mis); + return predicted_segment_id; + } + + if (seg->temporal_update) { + const int ctx = av1_get_pred_context_seg_id(xd); + const aom_prob pred_prob = segp->pred_probs[ctx]; + mbmi->seg_id_predicted = aom_read(r, pred_prob, ACCT_STR); + if (counts) ++counts->seg.pred[ctx][mbmi->seg_id_predicted]; + if (mbmi->seg_id_predicted) { + segment_id = predicted_segment_id; + } else { + segment_id = read_segment_id(r, segp); + if (counts) ++counts->seg.tree_mispred[segment_id]; + } + } else { + segment_id = read_segment_id(r, segp); + if (counts) ++counts->seg.tree_total[segment_id]; + } + set_segment_id(cm, mi_offset, x_mis, y_mis, segment_id); + return segment_id; +} + +static int read_skip(AV1_COMMON *cm, const MACROBLOCKD *xd, int segment_id, + aom_reader *r) { + if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) { + return 1; + } else { + const int ctx = av1_get_skip_context(xd); + const int skip = aom_read(r, cm->fc->skip_probs[ctx], ACCT_STR); + FRAME_COUNTS *counts = xd->counts; + if (counts) ++counts->skip[ctx][skip]; + return skip; + } +} + +#if CONFIG_PALETTE +static void read_palette_mode_info(AV1_COMMON *const cm, MACROBLOCKD *const xd, + aom_reader *r) { + MODE_INFO *const mi = xd->mi[0]; + MB_MODE_INFO *const mbmi = &mi->mbmi; + const MODE_INFO *const above_mi = xd->above_mi; + const MODE_INFO *const left_mi = xd->left_mi; + const BLOCK_SIZE bsize = mbmi->sb_type; + int i, n; + PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; + + if (mbmi->mode == DC_PRED) { + int palette_y_mode_ctx = 0; + if (above_mi) + palette_y_mode_ctx += + (above_mi->mbmi.palette_mode_info.palette_size[0] > 0); + if (left_mi) + palette_y_mode_ctx += + (left_mi->mbmi.palette_mode_info.palette_size[0] > 0); + if (aom_read(r, av1_default_palette_y_mode_prob[bsize - BLOCK_8X8] + [palette_y_mode_ctx], + ACCT_STR)) { + pmi->palette_size[0] = + aom_read_tree(r, av1_palette_size_tree, + av1_default_palette_y_size_prob[bsize - BLOCK_8X8], + ACCT_STR) + + 2; + n = pmi->palette_size[0]; +#if CONFIG_PALETTE_DELTA_ENCODING + const int min_bits = cm->bit_depth - 3; + int bits = min_bits + aom_read_literal(r, 2, ACCT_STR); + pmi->palette_colors[0] = aom_read_literal(r, cm->bit_depth, ACCT_STR); + for (i = 1; i < n; ++i) { + pmi->palette_colors[i] = pmi->palette_colors[i - 1] + + aom_read_literal(r, bits, ACCT_STR) + 1; + bits = AOMMIN( + bits, av1_ceil_log2((1 << cm->bit_depth) - pmi->palette_colors[i])); + } +#else + for (i = 0; i < n; ++i) + pmi->palette_colors[i] = aom_read_literal(r, cm->bit_depth, ACCT_STR); +#endif // CONFIG_PALETTE_DELTA_ENCODING + xd->plane[0].color_index_map[0] = read_uniform(r, n); + assert(xd->plane[0].color_index_map[0] < n); + } + } + + if (mbmi->uv_mode == DC_PRED) { + const int palette_uv_mode_ctx = (pmi->palette_size[0] > 0); + if (aom_read(r, av1_default_palette_uv_mode_prob[palette_uv_mode_ctx], + ACCT_STR)) { + pmi->palette_size[1] = + aom_read_tree(r, av1_palette_size_tree, + av1_default_palette_uv_size_prob[bsize - BLOCK_8X8], + ACCT_STR) + + 2; + n = pmi->palette_size[1]; +#if CONFIG_PALETTE_DELTA_ENCODING + // U channel colors. + const int min_bits_u = cm->bit_depth - 3; + int bits = min_bits_u + aom_read_literal(r, 2, ACCT_STR); + pmi->palette_colors[PALETTE_MAX_SIZE] = + aom_read_literal(r, cm->bit_depth, ACCT_STR); + for (i = 1; i < n; ++i) { + pmi->palette_colors[PALETTE_MAX_SIZE + i] = + pmi->palette_colors[PALETTE_MAX_SIZE + i - 1] + + aom_read_literal(r, bits, ACCT_STR); + bits = AOMMIN(bits, + av1_ceil_log2(1 + (1 << cm->bit_depth) - + pmi->palette_colors[PALETTE_MAX_SIZE + i])); + } + // V channel colors. + if (aom_read_bit(r, ACCT_STR)) { // Delta encoding. + const int min_bits_v = cm->bit_depth - 4; + const int max_val = 1 << cm->bit_depth; + bits = min_bits_v + aom_read_literal(r, 2, ACCT_STR); + pmi->palette_colors[2 * PALETTE_MAX_SIZE] = + aom_read_literal(r, cm->bit_depth, ACCT_STR); + for (i = 1; i < n; ++i) { + int delta = aom_read_literal(r, bits, ACCT_STR); + if (delta && aom_read_bit(r, ACCT_STR)) delta = -delta; + int val = + (int)pmi->palette_colors[2 * PALETTE_MAX_SIZE + i - 1] + delta; + if (val < 0) val += max_val; + if (val >= max_val) val -= max_val; + pmi->palette_colors[2 * PALETTE_MAX_SIZE + i] = val; + } + } else { + for (i = 0; i < n; ++i) { + pmi->palette_colors[2 * PALETTE_MAX_SIZE + i] = + aom_read_literal(r, cm->bit_depth, ACCT_STR); + } + } +#else + for (i = 0; i < n; ++i) { + pmi->palette_colors[PALETTE_MAX_SIZE + i] = + aom_read_literal(r, cm->bit_depth, ACCT_STR); + pmi->palette_colors[2 * PALETTE_MAX_SIZE + i] = + aom_read_literal(r, cm->bit_depth, ACCT_STR); + } +#endif // CONFIG_PALETTE_DELTA_ENCODING + xd->plane[1].color_index_map[0] = read_uniform(r, n); + assert(xd->plane[1].color_index_map[0] < n); + } + } +} +#endif // CONFIG_PALETTE + +#if CONFIG_FILTER_INTRA +static void read_filter_intra_mode_info(AV1_COMMON *const cm, + MACROBLOCKD *const xd, aom_reader *r) { + MODE_INFO *const mi = xd->mi[0]; + MB_MODE_INFO *const mbmi = &mi->mbmi; + FRAME_COUNTS *counts = xd->counts; + FILTER_INTRA_MODE_INFO *filter_intra_mode_info = + &mbmi->filter_intra_mode_info; + + if (mbmi->mode == DC_PRED +#if CONFIG_PALETTE + && mbmi->palette_mode_info.palette_size[0] == 0 +#endif // CONFIG_PALETTE + ) { + filter_intra_mode_info->use_filter_intra_mode[0] = + aom_read(r, cm->fc->filter_intra_probs[0], ACCT_STR); + if (filter_intra_mode_info->use_filter_intra_mode[0]) { + filter_intra_mode_info->filter_intra_mode[0] = + read_uniform(r, FILTER_INTRA_MODES); + } + if (counts) { + ++counts + ->filter_intra[0][filter_intra_mode_info->use_filter_intra_mode[0]]; + } + } + if (mbmi->uv_mode == DC_PRED +#if CONFIG_PALETTE + && mbmi->palette_mode_info.palette_size[1] == 0 +#endif // CONFIG_PALETTE + ) { + filter_intra_mode_info->use_filter_intra_mode[1] = + aom_read(r, cm->fc->filter_intra_probs[1], ACCT_STR); + if (filter_intra_mode_info->use_filter_intra_mode[1]) { + filter_intra_mode_info->filter_intra_mode[1] = + read_uniform(r, FILTER_INTRA_MODES); + } + if (counts) { + ++counts + ->filter_intra[1][filter_intra_mode_info->use_filter_intra_mode[1]]; + } + } +} +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_EXT_INTRA +static void read_intra_angle_info(AV1_COMMON *const cm, MACROBLOCKD *const xd, + aom_reader *r) { + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const BLOCK_SIZE bsize = mbmi->sb_type; +#if CONFIG_INTRA_INTERP +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *const ec_ctx = xd->tile_ctx; +#else + FRAME_CONTEXT *const ec_ctx = cm->fc; +#endif // CONFIG_EC_ADAPT + const int ctx = av1_get_pred_context_intra_interp(xd); + int p_angle; +#endif // CONFIG_INTRA_INTERP + + (void)cm; + if (bsize < BLOCK_8X8) return; + + if (av1_is_directional_mode(mbmi->mode, bsize)) { + mbmi->angle_delta[0] = + read_uniform(r, 2 * MAX_ANGLE_DELTA + 1) - MAX_ANGLE_DELTA; +#if CONFIG_INTRA_INTERP + p_angle = mode_to_angle_map[mbmi->mode] + mbmi->angle_delta[0] * ANGLE_STEP; + if (av1_is_intra_filter_switchable(p_angle)) { + FRAME_COUNTS *counts = xd->counts; +#if CONFIG_EC_MULTISYMBOL + mbmi->intra_filter = aom_read_symbol(r, ec_ctx->intra_filter_cdf[ctx], + INTRA_FILTERS, ACCT_STR); +#else + mbmi->intra_filter = aom_read_tree( + r, av1_intra_filter_tree, ec_ctx->intra_filter_probs[ctx], ACCT_STR); +#endif // CONFIG_EC_MULTISYMBOL + if (counts) ++counts->intra_filter[ctx][mbmi->intra_filter]; + } else { + mbmi->intra_filter = INTRA_FILTER_LINEAR; + } +#endif // CONFIG_INTRA_INTERP + } + + if (av1_is_directional_mode(mbmi->uv_mode, bsize)) { + mbmi->angle_delta[1] = + read_uniform(r, 2 * MAX_ANGLE_DELTA + 1) - MAX_ANGLE_DELTA; + } +} +#endif // CONFIG_EXT_INTRA + +void av1_read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif +#if CONFIG_TXK_SEL + int block, int plane, +#endif + aom_reader *r) { + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const int inter_block = is_inter_block(mbmi); +#if CONFIG_VAR_TX + const TX_SIZE tx_size = inter_block ? mbmi->min_tx_size : mbmi->tx_size; +#else + const TX_SIZE tx_size = mbmi->tx_size; +#endif +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + +#if !CONFIG_TXK_SEL + TX_TYPE *tx_type = &mbmi->tx_type; +#else + // only y plane's tx_type is transmitted + if (plane > 0) return; + TX_TYPE *tx_type = &mbmi->txk_type[block]; +#endif + + if (!FIXED_TX_TYPE) { +#if CONFIG_EXT_TX + const TX_SIZE square_tx_size = txsize_sqr_map[tx_size]; + if (get_ext_tx_types(tx_size, mbmi->sb_type, inter_block, + cm->reduced_tx_set_used) > 1 && + ((!cm->seg.enabled && cm->base_qindex > 0) || + (cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) && + !mbmi->skip && +#if CONFIG_SUPERTX + !supertx_enabled && +#endif // CONFIG_SUPERTX + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + const int eset = get_ext_tx_set(tx_size, mbmi->sb_type, inter_block, + cm->reduced_tx_set_used); + FRAME_COUNTS *counts = xd->counts; + + if (inter_block) { + if (eset > 0) { +#if CONFIG_EC_MULTISYMBOL + *tx_type = av1_ext_tx_inter_inv[eset][aom_read_symbol( + r, ec_ctx->inter_ext_tx_cdf[eset][square_tx_size], + ext_tx_cnt_inter[eset], ACCT_STR)]; +#else + *tx_type = aom_read_tree( + r, av1_ext_tx_inter_tree[eset], + ec_ctx->inter_ext_tx_prob[eset][square_tx_size], ACCT_STR); +#endif + if (counts) ++counts->inter_ext_tx[eset][square_tx_size][*tx_type]; + } + } else if (ALLOW_INTRA_EXT_TX) { + if (eset > 0) { +#if CONFIG_EC_MULTISYMBOL + *tx_type = av1_ext_tx_intra_inv[eset][aom_read_symbol( + r, ec_ctx->intra_ext_tx_cdf[eset][square_tx_size][mbmi->mode], + ext_tx_cnt_intra[eset], ACCT_STR)]; +#else + *tx_type = aom_read_tree( + r, av1_ext_tx_intra_tree[eset], + ec_ctx->intra_ext_tx_prob[eset][square_tx_size][mbmi->mode], + ACCT_STR); +#endif + if (counts) + ++counts->intra_ext_tx[eset][square_tx_size][mbmi->mode][*tx_type]; + } + } + } else { + *tx_type = DCT_DCT; + } +#else + + if (tx_size < TX_32X32 && + ((!cm->seg.enabled && cm->base_qindex > 0) || + (cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) && + !mbmi->skip && +#if CONFIG_SUPERTX + !supertx_enabled && +#endif // CONFIG_SUPERTX + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + FRAME_COUNTS *counts = xd->counts; + + if (inter_block) { +#if CONFIG_EC_MULTISYMBOL + *tx_type = av1_ext_tx_inv[aom_read_symbol( + r, ec_ctx->inter_ext_tx_cdf[tx_size], TX_TYPES, ACCT_STR)]; +#else + *tx_type = aom_read_tree(r, av1_ext_tx_tree, + ec_ctx->inter_ext_tx_prob[tx_size], ACCT_STR); +#endif + if (counts) ++counts->inter_ext_tx[tx_size][*tx_type]; + } else { + const TX_TYPE tx_type_nom = intra_mode_to_tx_type_context[mbmi->mode]; +#if CONFIG_EC_MULTISYMBOL + *tx_type = av1_ext_tx_inv[aom_read_symbol( + r, ec_ctx->intra_ext_tx_cdf[tx_size][tx_type_nom], TX_TYPES, + ACCT_STR)]; +#else + *tx_type = aom_read_tree( + r, av1_ext_tx_tree, ec_ctx->intra_ext_tx_prob[tx_size][tx_type_nom], + ACCT_STR); +#endif + if (counts) ++counts->intra_ext_tx[tx_size][tx_type_nom][*tx_type]; + } + } else { + *tx_type = DCT_DCT; + } +#endif // CONFIG_EXT_TX + } +} + +#if CONFIG_INTRABC +static INLINE void read_mv(aom_reader *r, MV *mv, const MV *ref, + nmv_context *ctx, nmv_context_counts *counts, + int allow_hp); + +static INLINE int is_mv_valid(const MV *mv); + +static INLINE int assign_dv(AV1_COMMON *cm, MACROBLOCKD *xd, int_mv *mv, + const int_mv *ref_mv, int mi_row, int mi_col, + BLOCK_SIZE bsize, aom_reader *r) { +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; + (void)cm; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + FRAME_COUNTS *counts = xd->counts; + nmv_context_counts *const dv_counts = counts ? &counts->dv : NULL; + read_mv(r, &mv->as_mv, &ref_mv->as_mv, &ec_ctx->ndvc, dv_counts, 0); + int valid = is_mv_valid(&mv->as_mv) && + is_dv_valid(mv->as_mv, &xd->tile, mi_row, mi_col, bsize); + // TODO(aconverse@google.com): additional validation + return valid; +} +#endif // CONFIG_INTRABC + +static void read_intra_frame_mode_info(AV1_COMMON *const cm, + MACROBLOCKD *const xd, int mi_row, + int mi_col, aom_reader *r) { + MODE_INFO *const mi = xd->mi[0]; + MB_MODE_INFO *const mbmi = &mi->mbmi; + const MODE_INFO *above_mi = xd->above_mi; + const MODE_INFO *left_mi = xd->left_mi; + const BLOCK_SIZE bsize = mbmi->sb_type; + int i; + const int mi_offset = mi_row * cm->mi_cols + mi_col; + const int bw = mi_size_wide[bsize]; + const int bh = mi_size_high[bsize]; + + // TODO(slavarnway): move x_mis, y_mis into xd ????? + const int x_mis = AOMMIN(cm->mi_cols - mi_col, bw); + const int y_mis = AOMMIN(cm->mi_rows - mi_row, bh); +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#elif CONFIG_EC_MULTISYMBOL + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + + mbmi->segment_id = read_intra_segment_id(cm, xd, mi_offset, x_mis, y_mis, r); + mbmi->skip = read_skip(cm, xd, mbmi->segment_id, r); + +#if CONFIG_DELTA_Q + if (cm->delta_q_present_flag) { + xd->current_qindex = + xd->prev_qindex + + read_delta_qindex(cm, xd, r, mbmi, mi_col, mi_row) * cm->delta_q_res; + /* Normative: Clamp to [1,MAXQ] to not interfere with lossless mode */ + xd->current_qindex = clamp(xd->current_qindex, 1, MAXQ); + xd->prev_qindex = xd->current_qindex; +#if CONFIG_EXT_DELTA_Q + if (cm->delta_lf_present_flag) { + mbmi->current_delta_lf_from_base = xd->current_delta_lf_from_base = + xd->prev_delta_lf_from_base + + read_delta_lflevel(cm, xd, r, mbmi, mi_col, mi_row) * + cm->delta_lf_res; + xd->prev_delta_lf_from_base = xd->current_delta_lf_from_base; + } +#endif + } +#endif + + mbmi->tx_size = read_tx_size(cm, xd, 0, 1, r); + mbmi->ref_frame[0] = INTRA_FRAME; + mbmi->ref_frame[1] = NONE_FRAME; + +#if CONFIG_INTRABC + if (bsize >= BLOCK_8X8 && cm->allow_screen_content_tools) { + mbmi->use_intrabc = aom_read(r, INTRABC_PROB, ACCT_STR); + if (mbmi->use_intrabc) { + int_mv dv_ref; + mbmi->mode = mbmi->uv_mode = DC_PRED; +#if CONFIG_DUAL_FILTER + for (int idx = 0; idx < 4; ++idx) mbmi->interp_filter[idx] = BILINEAR; +#else + mbmi->interp_filter = BILINEAR; +#endif + av1_find_ref_dv(&dv_ref, mi_row, mi_col); + xd->corrupted |= + !assign_dv(cm, xd, &mbmi->mv[0], &dv_ref, mi_row, mi_col, bsize, r); + return; + } + } +#endif // CONFIG_INTRABC + +#if CONFIG_CB4X4 + (void)i; + mbmi->mode = +#if CONFIG_EC_MULTISYMBOL + read_intra_mode(r, get_y_mode_cdf(ec_ctx, mi, above_mi, left_mi, 0)); +#else + read_intra_mode(r, get_y_mode_probs(cm, mi, above_mi, left_mi, 0)); +#endif +#else + switch (bsize) { + case BLOCK_4X4: + for (i = 0; i < 4; ++i) + mi->bmi[i].as_mode = +#if CONFIG_EC_MULTISYMBOL + read_intra_mode(r, + get_y_mode_cdf(ec_ctx, mi, above_mi, left_mi, i)); +#else + read_intra_mode(r, get_y_mode_probs(cm, mi, above_mi, left_mi, i)); +#endif + mbmi->mode = mi->bmi[3].as_mode; + break; + case BLOCK_4X8: + mi->bmi[0].as_mode = mi->bmi[2].as_mode = +#if CONFIG_EC_MULTISYMBOL + read_intra_mode(r, get_y_mode_cdf(ec_ctx, mi, above_mi, left_mi, 0)); +#else + read_intra_mode(r, get_y_mode_probs(cm, mi, above_mi, left_mi, 0)); +#endif + mi->bmi[1].as_mode = mi->bmi[3].as_mode = mbmi->mode = +#if CONFIG_EC_MULTISYMBOL + read_intra_mode(r, get_y_mode_cdf(ec_ctx, mi, above_mi, left_mi, 1)); +#else + read_intra_mode(r, get_y_mode_probs(cm, mi, above_mi, left_mi, 1)); +#endif + break; + case BLOCK_8X4: + mi->bmi[0].as_mode = mi->bmi[1].as_mode = +#if CONFIG_EC_MULTISYMBOL + read_intra_mode(r, get_y_mode_cdf(ec_ctx, mi, above_mi, left_mi, 0)); +#else + read_intra_mode(r, get_y_mode_probs(cm, mi, above_mi, left_mi, 0)); +#endif + mi->bmi[2].as_mode = mi->bmi[3].as_mode = mbmi->mode = +#if CONFIG_EC_MULTISYMBOL + read_intra_mode(r, get_y_mode_cdf(ec_ctx, mi, above_mi, left_mi, 2)); +#else + read_intra_mode(r, get_y_mode_probs(cm, mi, above_mi, left_mi, 2)); +#endif + break; + default: + mbmi->mode = +#if CONFIG_EC_MULTISYMBOL + read_intra_mode(r, get_y_mode_cdf(ec_ctx, mi, above_mi, left_mi, 0)); +#else + read_intra_mode(r, get_y_mode_probs(cm, mi, above_mi, left_mi, 0)); +#endif + } +#endif + +#if CONFIG_CB4X4 + if (is_chroma_reference(mi_row, mi_col, bsize, xd->plane[1].subsampling_x, + xd->plane[1].subsampling_y)) + mbmi->uv_mode = read_intra_mode_uv(cm, xd, r, mbmi->mode); +#else + mbmi->uv_mode = read_intra_mode_uv(cm, xd, r, mbmi->mode); +#endif + +#if CONFIG_EXT_INTRA + read_intra_angle_info(cm, xd, r); +#endif // CONFIG_EXT_INTRA +#if CONFIG_PALETTE + mbmi->palette_mode_info.palette_size[0] = 0; + mbmi->palette_mode_info.palette_size[1] = 0; + if (bsize >= BLOCK_8X8 && cm->allow_screen_content_tools) + read_palette_mode_info(cm, xd, r); +#endif // CONFIG_PALETTE +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0; + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = 0; + if (bsize >= BLOCK_8X8 || CONFIG_CB4X4) + read_filter_intra_mode_info(cm, xd, r); +#endif // CONFIG_FILTER_INTRA + +#if !CONFIG_TXK_SEL + av1_read_tx_type(cm, xd, +#if CONFIG_SUPERTX + 0, +#endif + r); +#endif // !CONFIG_TXK_SEL +} + +static int read_mv_component(aom_reader *r, nmv_component *mvcomp, int usehp) { + int mag, d, fr, hp; + const int sign = aom_read(r, mvcomp->sign, ACCT_STR); + const int mv_class = +#if CONFIG_EC_MULTISYMBOL + aom_read_symbol(r, mvcomp->class_cdf, MV_CLASSES, ACCT_STR); +#else + aom_read_tree(r, av1_mv_class_tree, mvcomp->classes, ACCT_STR); +#endif + const int class0 = mv_class == MV_CLASS_0; + + // Integer part + if (class0) { + d = aom_read(r, mvcomp->class0[0], ACCT_STR); + mag = 0; + } else { + int i; + const int n = mv_class + CLASS0_BITS - 1; // number of bits + + d = 0; + for (i = 0; i < n; ++i) d |= aom_read(r, mvcomp->bits[i], ACCT_STR) << i; + mag = CLASS0_SIZE << (mv_class + 2); + } + +// Fractional part +#if CONFIG_EC_MULTISYMBOL + fr = aom_read_symbol(r, class0 ? mvcomp->class0_fp_cdf[d] : mvcomp->fp_cdf, + MV_FP_SIZE, ACCT_STR); +#else + fr = aom_read_tree(r, av1_mv_fp_tree, + class0 ? mvcomp->class0_fp[d] : mvcomp->fp, ACCT_STR); +#endif + + // High precision part (if hp is not used, the default value of the hp is 1) + hp = usehp ? aom_read(r, class0 ? mvcomp->class0_hp : mvcomp->hp, ACCT_STR) + : 1; + + // Result + mag += ((d << 3) | (fr << 1) | hp) + 1; + return sign ? -mag : mag; +} + +static INLINE void read_mv(aom_reader *r, MV *mv, const MV *ref, + nmv_context *ctx, nmv_context_counts *counts, + int allow_hp) { + MV_JOINT_TYPE joint_type; + MV diff = { 0, 0 }; + joint_type = +#if CONFIG_EC_MULTISYMBOL + (MV_JOINT_TYPE)aom_read_symbol(r, ctx->joint_cdf, MV_JOINTS, ACCT_STR); +#else + (MV_JOINT_TYPE)aom_read_tree(r, av1_mv_joint_tree, ctx->joints, ACCT_STR); +#endif + + if (mv_joint_vertical(joint_type)) + diff.row = read_mv_component(r, &ctx->comps[0], allow_hp); + + if (mv_joint_horizontal(joint_type)) + diff.col = read_mv_component(r, &ctx->comps[1], allow_hp); + + av1_inc_mv(&diff, counts, allow_hp); + + mv->row = ref->row + diff.row; + mv->col = ref->col + diff.col; +} + +static REFERENCE_MODE read_block_reference_mode(AV1_COMMON *cm, + const MACROBLOCKD *xd, + aom_reader *r) { +#if !SUB8X8_COMP_REF + if (xd->mi[0]->mbmi.sb_type < BLOCK_8X8) return SINGLE_REFERENCE; +#endif + if (cm->reference_mode == REFERENCE_MODE_SELECT) { + const int ctx = av1_get_reference_mode_context(cm, xd); + const REFERENCE_MODE mode = + (REFERENCE_MODE)aom_read(r, cm->fc->comp_inter_prob[ctx], ACCT_STR); + FRAME_COUNTS *counts = xd->counts; + if (counts) ++counts->comp_inter[ctx][mode]; + return mode; // SINGLE_REFERENCE or COMPOUND_REFERENCE + } else { + return cm->reference_mode; + } +} + +// Read the referncence frame +static void read_ref_frames(AV1_COMMON *const cm, MACROBLOCKD *const xd, + aom_reader *r, int segment_id, + MV_REFERENCE_FRAME ref_frame[2]) { + FRAME_CONTEXT *const fc = cm->fc; + FRAME_COUNTS *counts = xd->counts; + + if (segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME)) { + ref_frame[0] = (MV_REFERENCE_FRAME)get_segdata(&cm->seg, segment_id, + SEG_LVL_REF_FRAME); + ref_frame[1] = NONE_FRAME; + } else { + const REFERENCE_MODE mode = read_block_reference_mode(cm, xd, r); + // FIXME(rbultje) I'm pretty sure this breaks segmentation ref frame coding + if (mode == COMPOUND_REFERENCE) { +#if CONFIG_LOWDELAY_COMPOUND // Normative in decoder (for low delay) + const int idx = 1; +#else +#if CONFIG_EXT_REFS + const int idx = cm->ref_frame_sign_bias[cm->comp_bwd_ref[0]]; +#else + const int idx = cm->ref_frame_sign_bias[cm->comp_fixed_ref]; +#endif // CONFIG_EXT_REFS +#endif + const int ctx = av1_get_pred_context_comp_ref_p(cm, xd); + + const int bit = aom_read(r, fc->comp_ref_prob[ctx][0], ACCT_STR); + if (counts) ++counts->comp_ref[ctx][0][bit]; + +#if CONFIG_EXT_REFS + // Decode forward references. + if (!bit) { + const int ctx1 = av1_get_pred_context_comp_ref_p1(cm, xd); + const int bit1 = aom_read(r, fc->comp_ref_prob[ctx1][1], ACCT_STR); + if (counts) ++counts->comp_ref[ctx1][1][bit1]; + ref_frame[!idx] = cm->comp_fwd_ref[bit1 ? 0 : 1]; + } else { + const int ctx2 = av1_get_pred_context_comp_ref_p2(cm, xd); + const int bit2 = aom_read(r, fc->comp_ref_prob[ctx2][2], ACCT_STR); + if (counts) ++counts->comp_ref[ctx2][2][bit2]; + ref_frame[!idx] = cm->comp_fwd_ref[bit2 ? 3 : 2]; + } + + // Decode backward references. + { + const int ctx_bwd = av1_get_pred_context_comp_bwdref_p(cm, xd); + const int bit_bwd = + aom_read(r, fc->comp_bwdref_prob[ctx_bwd][0], ACCT_STR); + if (counts) ++counts->comp_bwdref[ctx_bwd][0][bit_bwd]; + ref_frame[idx] = cm->comp_bwd_ref[bit_bwd]; + } +#else + ref_frame[!idx] = cm->comp_var_ref[bit]; + ref_frame[idx] = cm->comp_fixed_ref; +#endif // CONFIG_EXT_REFS + } else if (mode == SINGLE_REFERENCE) { +#if CONFIG_EXT_REFS + const int ctx0 = av1_get_pred_context_single_ref_p1(xd); + const int bit0 = aom_read(r, fc->single_ref_prob[ctx0][0], ACCT_STR); + if (counts) ++counts->single_ref[ctx0][0][bit0]; + + if (bit0) { + const int ctx1 = av1_get_pred_context_single_ref_p2(xd); + const int bit1 = aom_read(r, fc->single_ref_prob[ctx1][1], ACCT_STR); + if (counts) ++counts->single_ref[ctx1][1][bit1]; + ref_frame[0] = bit1 ? ALTREF_FRAME : BWDREF_FRAME; + } else { + const int ctx2 = av1_get_pred_context_single_ref_p3(xd); + const int bit2 = aom_read(r, fc->single_ref_prob[ctx2][2], ACCT_STR); + if (counts) ++counts->single_ref[ctx2][2][bit2]; + if (bit2) { + const int ctx4 = av1_get_pred_context_single_ref_p5(xd); + const int bit4 = aom_read(r, fc->single_ref_prob[ctx4][4], ACCT_STR); + if (counts) ++counts->single_ref[ctx4][4][bit4]; + ref_frame[0] = bit4 ? GOLDEN_FRAME : LAST3_FRAME; + } else { + const int ctx3 = av1_get_pred_context_single_ref_p4(xd); + const int bit3 = aom_read(r, fc->single_ref_prob[ctx3][3], ACCT_STR); + if (counts) ++counts->single_ref[ctx3][3][bit3]; + ref_frame[0] = bit3 ? LAST2_FRAME : LAST_FRAME; + } + } +#else + const int ctx0 = av1_get_pred_context_single_ref_p1(xd); + const int bit0 = aom_read(r, fc->single_ref_prob[ctx0][0], ACCT_STR); + if (counts) ++counts->single_ref[ctx0][0][bit0]; + + if (bit0) { + const int ctx1 = av1_get_pred_context_single_ref_p2(xd); + const int bit1 = aom_read(r, fc->single_ref_prob[ctx1][1], ACCT_STR); + if (counts) ++counts->single_ref[ctx1][1][bit1]; + ref_frame[0] = bit1 ? ALTREF_FRAME : GOLDEN_FRAME; + } else { + ref_frame[0] = LAST_FRAME; + } +#endif // CONFIG_EXT_REFS + + ref_frame[1] = NONE_FRAME; + } else { + assert(0 && "Invalid prediction mode."); + } + } +} + +static INLINE void read_mb_interp_filter(AV1_COMMON *const cm, + MACROBLOCKD *const xd, + MB_MODE_INFO *const mbmi, + aom_reader *r) { + FRAME_COUNTS *counts = xd->counts; +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + + if (!av1_is_interp_needed(xd)) { + set_default_interp_filters(mbmi, cm->interp_filter); + return; + } + +#if CONFIG_DUAL_FILTER + if (cm->interp_filter != SWITCHABLE) { + int dir; + + for (dir = 0; dir < 4; ++dir) mbmi->interp_filter[dir] = cm->interp_filter; + } else { + int dir; + + for (dir = 0; dir < 2; ++dir) { + const int ctx = av1_get_pred_context_switchable_interp(xd, dir); + mbmi->interp_filter[dir] = EIGHTTAP_REGULAR; + + if (has_subpel_mv_component(xd->mi[0], xd, dir) || + (mbmi->ref_frame[1] > INTRA_FRAME && + has_subpel_mv_component(xd->mi[0], xd, dir + 2))) { +#if CONFIG_EC_MULTISYMBOL + mbmi->interp_filter[dir] = + (InterpFilter)av1_switchable_interp_inv[aom_read_symbol( + r, ec_ctx->switchable_interp_cdf[ctx], SWITCHABLE_FILTERS, + ACCT_STR)]; +#else + mbmi->interp_filter[dir] = (InterpFilter)aom_read_tree( + r, av1_switchable_interp_tree, ec_ctx->switchable_interp_prob[ctx], + ACCT_STR); +#endif + if (counts) ++counts->switchable_interp[ctx][mbmi->interp_filter[dir]]; + } + } + // The index system works as: + // (0, 1) -> (vertical, horizontal) filter types for the first ref frame. + // (2, 3) -> (vertical, horizontal) filter types for the second ref frame. + mbmi->interp_filter[2] = mbmi->interp_filter[0]; + mbmi->interp_filter[3] = mbmi->interp_filter[1]; + } +#else // CONFIG_DUAL_FILTER + if (cm->interp_filter != SWITCHABLE) { + mbmi->interp_filter = cm->interp_filter; + } else { + const int ctx = av1_get_pred_context_switchable_interp(xd); +#if CONFIG_EC_MULTISYMBOL + mbmi->interp_filter = + (InterpFilter)av1_switchable_interp_inv[aom_read_symbol( + r, ec_ctx->switchable_interp_cdf[ctx], SWITCHABLE_FILTERS, + ACCT_STR)]; +#else + mbmi->interp_filter = (InterpFilter)aom_read_tree( + r, av1_switchable_interp_tree, ec_ctx->switchable_interp_prob[ctx], + ACCT_STR); +#endif + if (counts) ++counts->switchable_interp[ctx][mbmi->interp_filter]; + } +#endif // CONFIG_DUAL_FILTER +} + +static void read_intra_block_mode_info(AV1_COMMON *const cm, const int mi_row, + const int mi_col, MACROBLOCKD *const xd, + MODE_INFO *mi, aom_reader *r) { + MB_MODE_INFO *const mbmi = &mi->mbmi; + const BLOCK_SIZE bsize = mi->mbmi.sb_type; + int i; + + mbmi->ref_frame[0] = INTRA_FRAME; + mbmi->ref_frame[1] = NONE_FRAME; + +#if CONFIG_CB4X4 + (void)i; + mbmi->mode = read_intra_mode_y(cm, xd, r, size_group_lookup[bsize]); +#else + switch (bsize) { + case BLOCK_4X4: + for (i = 0; i < 4; ++i) + mi->bmi[i].as_mode = read_intra_mode_y(cm, xd, r, 0); + mbmi->mode = mi->bmi[3].as_mode; + break; + case BLOCK_4X8: + mi->bmi[0].as_mode = mi->bmi[2].as_mode = read_intra_mode_y(cm, xd, r, 0); + mi->bmi[1].as_mode = mi->bmi[3].as_mode = mbmi->mode = + read_intra_mode_y(cm, xd, r, 0); + break; + case BLOCK_8X4: + mi->bmi[0].as_mode = mi->bmi[1].as_mode = read_intra_mode_y(cm, xd, r, 0); + mi->bmi[2].as_mode = mi->bmi[3].as_mode = mbmi->mode = + read_intra_mode_y(cm, xd, r, 0); + break; + default: + mbmi->mode = read_intra_mode_y(cm, xd, r, size_group_lookup[bsize]); + } +#endif + +#if CONFIG_CB4X4 + if (is_chroma_reference(mi_row, mi_col, bsize, xd->plane[1].subsampling_x, + xd->plane[1].subsampling_y)) + mbmi->uv_mode = read_intra_mode_uv(cm, xd, r, mbmi->mode); +#else + mbmi->uv_mode = read_intra_mode_uv(cm, xd, r, mbmi->mode); + (void)mi_row; + (void)mi_col; +#endif + +#if CONFIG_EXT_INTRA + read_intra_angle_info(cm, xd, r); +#endif // CONFIG_EXT_INTRA +#if CONFIG_PALETTE + mbmi->palette_mode_info.palette_size[0] = 0; + mbmi->palette_mode_info.palette_size[1] = 0; + if (bsize >= BLOCK_8X8 && cm->allow_screen_content_tools) + read_palette_mode_info(cm, xd, r); +#endif // CONFIG_PALETTE +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0; + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = 0; + if (bsize >= BLOCK_8X8 || CONFIG_CB4X4) + read_filter_intra_mode_info(cm, xd, r); +#endif // CONFIG_FILTER_INTRA +} + +static INLINE int is_mv_valid(const MV *mv) { + return mv->row > MV_LOW && mv->row < MV_UPP && mv->col > MV_LOW && + mv->col < MV_UPP; +} + +static INLINE int assign_mv(AV1_COMMON *cm, MACROBLOCKD *xd, + PREDICTION_MODE mode, + MV_REFERENCE_FRAME ref_frame[2], int block, + int_mv mv[2], int_mv ref_mv[2], + int_mv nearest_mv[2], int_mv near_mv[2], int mi_row, + int mi_col, int is_compound, int allow_hp, + aom_reader *r) { + int i; + int ret = 1; +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; +#if CONFIG_REF_MV + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; +#if CONFIG_CB4X4 + int_mv *pred_mv = mbmi->pred_mv; + (void)block; +#else + int_mv *pred_mv = + (bsize >= BLOCK_8X8) ? mbmi->pred_mv : xd->mi[0]->bmi[block].pred_mv; +#endif // CONFIG_CB4X4 +#else + (void)block; +#endif // CONFIG_REF_MV + (void)ref_frame; + (void)cm; + (void)mi_row; + (void)mi_col; + (void)bsize; + + switch (mode) { + case NEWMV: { + FRAME_COUNTS *counts = xd->counts; +#if !CONFIG_REF_MV + nmv_context *const nmvc = &ec_ctx->nmvc; + nmv_context_counts *const mv_counts = counts ? &counts->mv : NULL; +#endif + for (i = 0; i < 1 + is_compound; ++i) { +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = + av1_nmv_ctx(xd->ref_mv_count[rf_type], xd->ref_mv_stack[rf_type], i, + mbmi->ref_mv_idx); + nmv_context *const nmvc = &ec_ctx->nmvc[nmv_ctx]; + nmv_context_counts *const mv_counts = + counts ? &counts->mv[nmv_ctx] : NULL; +#endif + read_mv(r, &mv[i].as_mv, &ref_mv[i].as_mv, nmvc, mv_counts, allow_hp); + ret = ret && is_mv_valid(&mv[i].as_mv); + +#if CONFIG_REF_MV + pred_mv[i].as_int = ref_mv[i].as_int; +#endif + } + break; + } + case NEARESTMV: { + mv[0].as_int = nearest_mv[0].as_int; + if (is_compound) mv[1].as_int = nearest_mv[1].as_int; + +#if CONFIG_REF_MV + pred_mv[0].as_int = nearest_mv[0].as_int; + if (is_compound) pred_mv[1].as_int = nearest_mv[1].as_int; +#endif + break; + } + case NEARMV: { + mv[0].as_int = near_mv[0].as_int; + if (is_compound) mv[1].as_int = near_mv[1].as_int; + +#if CONFIG_REF_MV + pred_mv[0].as_int = near_mv[0].as_int; + if (is_compound) pred_mv[1].as_int = near_mv[1].as_int; +#endif + break; + } + case ZEROMV: { +#if CONFIG_GLOBAL_MOTION + mv[0].as_int = gm_get_motion_vector(&cm->global_motion[ref_frame[0]], + cm->allow_high_precision_mv, bsize, + mi_col, mi_row, block) + .as_int; + if (is_compound) + mv[1].as_int = gm_get_motion_vector(&cm->global_motion[ref_frame[1]], + cm->allow_high_precision_mv, bsize, + mi_col, mi_row, block) + .as_int; +#else + mv[0].as_int = 0; + if (is_compound) mv[1].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION + +#if CONFIG_REF_MV + pred_mv[0].as_int = mv[0].as_int; + if (is_compound) pred_mv[1].as_int = mv[1].as_int; +#endif + break; + } +#if CONFIG_EXT_INTER + case NEW_NEWMV: { + FRAME_COUNTS *counts = xd->counts; +#if !CONFIG_REF_MV + nmv_context *const nmvc = &ec_ctx->nmvc; + nmv_context_counts *const mv_counts = counts ? &counts->mv : NULL; +#endif + assert(is_compound); + for (i = 0; i < 2; ++i) { +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = + av1_nmv_ctx(xd->ref_mv_count[rf_type], xd->ref_mv_stack[rf_type], i, + mbmi->ref_mv_idx); + nmv_context *const nmvc = &ec_ctx->nmvc[nmv_ctx]; + nmv_context_counts *const mv_counts = + counts ? &counts->mv[nmv_ctx] : NULL; +#endif + read_mv(r, &mv[i].as_mv, &ref_mv[i].as_mv, nmvc, mv_counts, allow_hp); + ret = ret && is_mv_valid(&mv[i].as_mv); + } + break; + } + case NEAREST_NEARESTMV: { + assert(is_compound); + mv[0].as_int = nearest_mv[0].as_int; + mv[1].as_int = nearest_mv[1].as_int; + break; + } + case NEAREST_NEARMV: { + assert(is_compound); + mv[0].as_int = nearest_mv[0].as_int; + mv[1].as_int = near_mv[1].as_int; + break; + } + case NEAR_NEARESTMV: { + assert(is_compound); + mv[0].as_int = near_mv[0].as_int; + mv[1].as_int = nearest_mv[1].as_int; + break; + } + case NEAR_NEARMV: { + assert(is_compound); + mv[0].as_int = near_mv[0].as_int; + mv[1].as_int = near_mv[1].as_int; + break; + } + case NEW_NEARESTMV: { + FRAME_COUNTS *counts = xd->counts; +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = av1_nmv_ctx(xd->ref_mv_count[rf_type], + xd->ref_mv_stack[rf_type], 0, mbmi->ref_mv_idx); + nmv_context *const nmvc = &ec_ctx->nmvc[nmv_ctx]; + nmv_context_counts *const mv_counts = + counts ? &counts->mv[nmv_ctx] : NULL; +#else + nmv_context *const nmvc = &ec_ctx->nmvc; + nmv_context_counts *const mv_counts = counts ? &counts->mv : NULL; +#endif + read_mv(r, &mv[0].as_mv, &ref_mv[0].as_mv, nmvc, mv_counts, allow_hp); + assert(is_compound); + ret = ret && is_mv_valid(&mv[0].as_mv); + mv[1].as_int = nearest_mv[1].as_int; + break; + } + case NEAREST_NEWMV: { + FRAME_COUNTS *counts = xd->counts; +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = av1_nmv_ctx(xd->ref_mv_count[rf_type], + xd->ref_mv_stack[rf_type], 1, mbmi->ref_mv_idx); + nmv_context_counts *const mv_counts = + counts ? &counts->mv[nmv_ctx] : NULL; + nmv_context *const nmvc = &ec_ctx->nmvc[nmv_ctx]; +#else + nmv_context *const nmvc = &ec_ctx->nmvc; + nmv_context_counts *const mv_counts = counts ? &counts->mv : NULL; +#endif + mv[0].as_int = nearest_mv[0].as_int; + read_mv(r, &mv[1].as_mv, &ref_mv[1].as_mv, nmvc, mv_counts, allow_hp); + assert(is_compound); + ret = ret && is_mv_valid(&mv[1].as_mv); + break; + } + case NEAR_NEWMV: { + FRAME_COUNTS *counts = xd->counts; +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = av1_nmv_ctx(xd->ref_mv_count[rf_type], + xd->ref_mv_stack[rf_type], 1, mbmi->ref_mv_idx); + nmv_context *const nmvc = &ec_ctx->nmvc[nmv_ctx]; + nmv_context_counts *const mv_counts = + counts ? &counts->mv[nmv_ctx] : NULL; +#else + nmv_context *const nmvc = &ec_ctx->nmvc; + nmv_context_counts *const mv_counts = counts ? &counts->mv : NULL; +#endif + mv[0].as_int = near_mv[0].as_int; + read_mv(r, &mv[1].as_mv, &ref_mv[1].as_mv, nmvc, mv_counts, allow_hp); + assert(is_compound); + + ret = ret && is_mv_valid(&mv[1].as_mv); + break; + } + case NEW_NEARMV: { + FRAME_COUNTS *counts = xd->counts; +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = av1_nmv_ctx(xd->ref_mv_count[rf_type], + xd->ref_mv_stack[rf_type], 0, mbmi->ref_mv_idx); + nmv_context *const nmvc = &ec_ctx->nmvc[nmv_ctx]; + nmv_context_counts *const mv_counts = + counts ? &counts->mv[nmv_ctx] : NULL; +#else + nmv_context *const nmvc = &ec_ctx->nmvc; + nmv_context_counts *const mv_counts = counts ? &counts->mv : NULL; +#endif + read_mv(r, &mv[0].as_mv, &ref_mv[0].as_mv, nmvc, mv_counts, allow_hp); + assert(is_compound); + ret = ret && is_mv_valid(&mv[0].as_mv); + mv[1].as_int = near_mv[1].as_int; + break; + } + case ZERO_ZEROMV: { + assert(is_compound); +#if CONFIG_GLOBAL_MOTION + mv[0].as_int = gm_get_motion_vector(&cm->global_motion[ref_frame[0]], + cm->allow_high_precision_mv, bsize, + mi_col, mi_row, block) + .as_int; + mv[1].as_int = gm_get_motion_vector(&cm->global_motion[ref_frame[1]], + cm->allow_high_precision_mv, bsize, + mi_col, mi_row, block) + .as_int; +#else + mv[0].as_int = 0; + mv[1].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION + break; + } +#endif // CONFIG_EXT_INTER + default: { return 0; } + } + return ret; +} + +static int read_is_inter_block(AV1_COMMON *const cm, MACROBLOCKD *const xd, + int segment_id, aom_reader *r) { + if (segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME)) { + return get_segdata(&cm->seg, segment_id, SEG_LVL_REF_FRAME) != INTRA_FRAME; + } else { + const int ctx = av1_get_intra_inter_context(xd); + const int is_inter = aom_read(r, cm->fc->intra_inter_prob[ctx], ACCT_STR); + FRAME_COUNTS *counts = xd->counts; + if (counts) ++counts->intra_inter[ctx][is_inter]; + return is_inter; + } +} + +static void fpm_sync(void *const data, int mi_row) { + AV1Decoder *const pbi = (AV1Decoder *)data; + av1_frameworker_wait(pbi->frame_worker_owner, pbi->common.prev_frame, + mi_row << pbi->common.mib_size_log2); +} + +static void read_inter_block_mode_info(AV1Decoder *const pbi, + MACROBLOCKD *const xd, + MODE_INFO *const mi, +#if (CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION || CONFIG_EXT_INTER) && \ + CONFIG_SUPERTX + int mi_row, int mi_col, aom_reader *r, + int supertx_enabled) { +#else + int mi_row, int mi_col, aom_reader *r) { +#endif // CONFIG_MOTION_VAR && CONFIG_SUPERTX + AV1_COMMON *const cm = &pbi->common; + MB_MODE_INFO *const mbmi = &mi->mbmi; + const BLOCK_SIZE bsize = mbmi->sb_type; + const int allow_hp = cm->allow_high_precision_mv; + const int unify_bsize = CONFIG_CB4X4; + int_mv nearestmv[2], nearmv[2]; + int_mv ref_mvs[MODE_CTX_REF_FRAMES][MAX_MV_REF_CANDIDATES]; + int ref, is_compound; + int16_t inter_mode_ctx[MODE_CTX_REF_FRAMES]; +#if CONFIG_REF_MV && CONFIG_EXT_INTER + int16_t compound_inter_mode_ctx[MODE_CTX_REF_FRAMES]; +#endif // CONFIG_REF_MV && CONFIG_EXT_INTER + int16_t mode_ctx = 0; +#if CONFIG_WARPED_MOTION + int pts[SAMPLES_ARRAY_SIZE], pts_inref[SAMPLES_ARRAY_SIZE]; +#endif // CONFIG_WARPED_MOTION +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + +#if CONFIG_PALETTE + mbmi->palette_mode_info.palette_size[0] = 0; + mbmi->palette_mode_info.palette_size[1] = 0; +#endif // CONFIG_PALETTE + + memset(ref_mvs, 0, sizeof(ref_mvs)); + + read_ref_frames(cm, xd, r, mbmi->segment_id, mbmi->ref_frame); + is_compound = has_second_ref(mbmi); + + for (ref = 0; ref < 1 + is_compound; ++ref) { + MV_REFERENCE_FRAME frame = mbmi->ref_frame[ref]; + + av1_find_mv_refs(cm, xd, mi, frame, +#if CONFIG_REF_MV + &xd->ref_mv_count[frame], xd->ref_mv_stack[frame], +#if CONFIG_EXT_INTER + compound_inter_mode_ctx, +#endif // CONFIG_EXT_INTER +#endif + ref_mvs[frame], mi_row, mi_col, fpm_sync, (void *)pbi, + inter_mode_ctx); + } + +#if CONFIG_REF_MV + if (is_compound) { + MV_REFERENCE_FRAME ref_frame = av1_ref_frame_type(mbmi->ref_frame); + av1_find_mv_refs(cm, xd, mi, ref_frame, &xd->ref_mv_count[ref_frame], + xd->ref_mv_stack[ref_frame], +#if CONFIG_EXT_INTER + compound_inter_mode_ctx, +#endif // CONFIG_EXT_INTER + ref_mvs[ref_frame], mi_row, mi_col, fpm_sync, (void *)pbi, + inter_mode_ctx); + + if (xd->ref_mv_count[ref_frame] < 2) { + MV_REFERENCE_FRAME rf[2]; + int_mv zeromv[2]; + av1_set_ref_frame(rf, ref_frame); +#if CONFIG_GLOBAL_MOTION + zeromv[0].as_int = gm_get_motion_vector(&cm->global_motion[rf[0]], + cm->allow_high_precision_mv, + bsize, mi_col, mi_row, 0) + .as_int; + zeromv[1].as_int = (rf[1] != NONE_FRAME) + ? gm_get_motion_vector(&cm->global_motion[rf[1]], + cm->allow_high_precision_mv, + bsize, mi_col, mi_row, 0) + .as_int + : 0; +#else + zeromv[0].as_int = zeromv[1].as_int = 0; +#endif + for (ref = 0; ref < 2; ++ref) { + if (rf[ref] == NONE_FRAME) continue; + lower_mv_precision(&ref_mvs[rf[ref]][0].as_mv, allow_hp); + lower_mv_precision(&ref_mvs[rf[ref]][1].as_mv, allow_hp); + if (ref_mvs[rf[ref]][0].as_int != zeromv[ref].as_int || + ref_mvs[rf[ref]][1].as_int != zeromv[ref].as_int) + inter_mode_ctx[ref_frame] &= ~(1 << ALL_ZERO_FLAG_OFFSET); + } + } + } + +#if CONFIG_EXT_INTER + if (is_compound) + mode_ctx = compound_inter_mode_ctx[mbmi->ref_frame[0]]; + else +#endif // CONFIG_EXT_INTER + mode_ctx = + av1_mode_context_analyzer(inter_mode_ctx, mbmi->ref_frame, bsize, -1); + mbmi->ref_mv_idx = 0; +#else + mode_ctx = inter_mode_ctx[mbmi->ref_frame[0]]; +#endif + + if (segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + mbmi->mode = ZEROMV; + if (bsize < BLOCK_8X8 && !unify_bsize) { + aom_internal_error(xd->error_info, AOM_CODEC_UNSUP_BITSTREAM, + "Invalid usage of segement feature on small blocks"); + return; + } + } else { + if (bsize >= BLOCK_8X8 || unify_bsize) { +#if CONFIG_EXT_INTER + if (is_compound) + mbmi->mode = read_inter_compound_mode(cm, xd, r, mode_ctx); + else +#endif // CONFIG_EXT_INTER + mbmi->mode = read_inter_mode(ec_ctx, xd, r, mode_ctx); +#if CONFIG_REF_MV +#if CONFIG_EXT_INTER + if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV || + have_nearmv_in_inter_mode(mbmi->mode)) +#else + if (mbmi->mode == NEARMV || mbmi->mode == NEWMV) +#endif + read_drl_idx(cm, xd, mbmi, r); +#endif + } + } + +#if CONFIG_EXT_INTER + if ((bsize < BLOCK_8X8 && unify_bsize) || + (mbmi->mode != ZEROMV && mbmi->mode != ZERO_ZEROMV)) { +#else + if ((bsize < BLOCK_8X8 && !unify_bsize) || mbmi->mode != ZEROMV) { +#endif // CONFIG_EXT_INTER + for (ref = 0; ref < 1 + is_compound; ++ref) { + av1_find_best_ref_mvs(allow_hp, ref_mvs[mbmi->ref_frame[ref]], + &nearestmv[ref], &nearmv[ref]); + } + } + +#if CONFIG_REF_MV + if (mbmi->ref_mv_idx > 0) { + int_mv cur_mv = + xd->ref_mv_stack[mbmi->ref_frame[0]][1 + mbmi->ref_mv_idx].this_mv; + nearmv[0] = cur_mv; + } + +#if CONFIG_EXT_INTER + if (is_compound && (bsize >= BLOCK_8X8 || unify_bsize) && + mbmi->mode != ZERO_ZEROMV) { +#else + if (is_compound && (bsize >= BLOCK_8X8 || unify_bsize) && + mbmi->mode != NEWMV && mbmi->mode != ZEROMV) { +#endif // CONFIG_EXT_INTER + uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + +#if CONFIG_EXT_INTER + if (xd->ref_mv_count[ref_frame_type] > 0) { +#else + if (xd->ref_mv_count[ref_frame_type] == 1 && mbmi->mode == NEARESTMV) { +#endif // CONFIG_EXT_INTER +#if CONFIG_EXT_INTER + if (mbmi->mode == NEAREST_NEARESTMV) { +#endif // CONFIG_EXT_INTER + nearestmv[0] = xd->ref_mv_stack[ref_frame_type][0].this_mv; + nearestmv[1] = xd->ref_mv_stack[ref_frame_type][0].comp_mv; + lower_mv_precision(&nearestmv[0].as_mv, allow_hp); + lower_mv_precision(&nearestmv[1].as_mv, allow_hp); +#if CONFIG_EXT_INTER + } else if (mbmi->mode == NEAREST_NEWMV || mbmi->mode == NEAREST_NEARMV) { + nearestmv[0] = xd->ref_mv_stack[ref_frame_type][0].this_mv; + lower_mv_precision(&nearestmv[0].as_mv, allow_hp); + } else if (mbmi->mode == NEW_NEARESTMV || mbmi->mode == NEAR_NEARESTMV) { + nearestmv[1] = xd->ref_mv_stack[ref_frame_type][0].comp_mv; + lower_mv_precision(&nearestmv[1].as_mv, allow_hp); + } +#endif // CONFIG_EXT_INTER + } + +#if CONFIG_EXT_INTER + if (xd->ref_mv_count[ref_frame_type] > 1) { + int ref_mv_idx = 1 + mbmi->ref_mv_idx; + if (compound_ref0_mode(mbmi->mode) == NEARMV) { + nearmv[0] = xd->ref_mv_stack[ref_frame_type][ref_mv_idx].this_mv; + lower_mv_precision(&nearmv[0].as_mv, allow_hp); + } + + if (compound_ref1_mode(mbmi->mode) == NEARMV) { + nearmv[1] = xd->ref_mv_stack[ref_frame_type][ref_mv_idx].comp_mv; + lower_mv_precision(&nearmv[1].as_mv, allow_hp); + } + } +#else + if (xd->ref_mv_count[ref_frame_type] > 1) { + int ref_mv_idx = 1 + mbmi->ref_mv_idx; + nearestmv[0] = xd->ref_mv_stack[ref_frame_type][0].this_mv; + nearestmv[1] = xd->ref_mv_stack[ref_frame_type][0].comp_mv; + nearmv[0] = xd->ref_mv_stack[ref_frame_type][ref_mv_idx].this_mv; + nearmv[1] = xd->ref_mv_stack[ref_frame_type][ref_mv_idx].comp_mv; + } +#endif // CONFIG_EXT_INTER + } +#endif + +#if !CONFIG_DUAL_FILTER && !CONFIG_WARPED_MOTION && !CONFIG_GLOBAL_MOTION + read_mb_interp_filter(cm, xd, mbmi, r); +#endif // !CONFIG_DUAL_FILTER && !CONFIG_WARPED_MOTION + + if (bsize < BLOCK_8X8 && !unify_bsize) { + const int num_4x4_w = 1 << xd->bmode_blocks_wl; + const int num_4x4_h = 1 << xd->bmode_blocks_hl; + int idx, idy; + PREDICTION_MODE b_mode; + int_mv nearest_sub8x8[2], near_sub8x8[2]; +#if CONFIG_EXT_INTER + int_mv ref_mv[2][2]; +#endif // CONFIG_EXT_INTER + for (idy = 0; idy < 2; idy += num_4x4_h) { + for (idx = 0; idx < 2; idx += num_4x4_w) { + int_mv block[2]; + const int j = idy * 2 + idx; + int_mv ref_mv_s8[2]; +#if CONFIG_REF_MV +#if CONFIG_EXT_INTER + if (!is_compound) +#endif // CONFIG_EXT_INTER + mode_ctx = av1_mode_context_analyzer(inter_mode_ctx, mbmi->ref_frame, + bsize, j); +#endif +#if CONFIG_EXT_INTER + if (is_compound) + b_mode = read_inter_compound_mode(cm, xd, r, mode_ctx); + else +#endif // CONFIG_EXT_INTER + b_mode = read_inter_mode(ec_ctx, xd, r, mode_ctx); + +#if CONFIG_EXT_INTER + if (b_mode != ZEROMV && b_mode != ZERO_ZEROMV) { +#else + if (b_mode != ZEROMV) { +#endif // CONFIG_EXT_INTER +#if CONFIG_REF_MV + CANDIDATE_MV ref_mv_stack[2][MAX_REF_MV_STACK_SIZE]; + uint8_t ref_mv_count[2]; +#endif + for (ref = 0; ref < 1 + is_compound; ++ref) +#if CONFIG_EXT_INTER + { + int_mv mv_ref_list[MAX_MV_REF_CANDIDATES]; + av1_update_mv_context(cm, xd, mi, mbmi->ref_frame[ref], mv_ref_list, + j, mi_row, mi_col, NULL); +#endif // CONFIG_EXT_INTER + av1_append_sub8x8_mvs_for_idx(cm, xd, j, ref, mi_row, mi_col, +#if CONFIG_REF_MV + ref_mv_stack[ref], &ref_mv_count[ref], +#endif +#if CONFIG_EXT_INTER + mv_ref_list, +#endif // CONFIG_EXT_INTER + &nearest_sub8x8[ref], + &near_sub8x8[ref]); +#if CONFIG_EXT_INTER + if (have_newmv_in_inter_mode(b_mode)) { + mv_ref_list[0].as_int = nearest_sub8x8[ref].as_int; + mv_ref_list[1].as_int = near_sub8x8[ref].as_int; + av1_find_best_ref_mvs(allow_hp, mv_ref_list, &ref_mv[0][ref], + &ref_mv[1][ref]); + } + } +#endif // CONFIG_EXT_INTER + } + + for (ref = 0; ref < 1 + is_compound && b_mode != ZEROMV; ++ref) { +#if CONFIG_REF_MV + ref_mv_s8[ref] = nearest_sub8x8[ref]; + lower_mv_precision(&ref_mv_s8[ref].as_mv, allow_hp); +#else + ref_mv_s8[ref] = nearestmv[ref]; +#endif + } +#if CONFIG_EXT_INTER + (void)ref_mv_s8; +#endif + + if (!assign_mv(cm, xd, b_mode, mbmi->ref_frame, j, block, +#if CONFIG_EXT_INTER + ref_mv[0], +#else // !CONFIG_EXT_INTER + ref_mv_s8, +#endif // CONFIG_EXT_INTER + nearest_sub8x8, near_sub8x8, mi_row, mi_col, is_compound, + allow_hp, r)) { + aom_merge_corrupted_flag(&xd->corrupted, 1); + break; + }; + + mi->bmi[j].as_mv[0].as_int = block[0].as_int; + mi->bmi[j].as_mode = b_mode; + if (is_compound) mi->bmi[j].as_mv[1].as_int = block[1].as_int; + + if (num_4x4_h == 2) mi->bmi[j + 2] = mi->bmi[j]; + if (num_4x4_w == 2) mi->bmi[j + 1] = mi->bmi[j]; + } + } + +#if CONFIG_REF_MV + mbmi->pred_mv[0].as_int = mi->bmi[3].pred_mv[0].as_int; + mbmi->pred_mv[1].as_int = mi->bmi[3].pred_mv[1].as_int; +#endif + mi->mbmi.mode = b_mode; + + mbmi->mv[0].as_int = mi->bmi[3].as_mv[0].as_int; + mbmi->mv[1].as_int = mi->bmi[3].as_mv[1].as_int; + } else { + int_mv ref_mv[2]; + ref_mv[0] = nearestmv[0]; + ref_mv[1] = nearestmv[1]; + +#if CONFIG_EXT_INTER + if (is_compound) { +#if CONFIG_REF_MV + int ref_mv_idx = mbmi->ref_mv_idx; + // Special case: NEAR_NEWMV and NEW_NEARMV modes use + // 1 + mbmi->ref_mv_idx (like NEARMV) instead of + // mbmi->ref_mv_idx (like NEWMV) + if (mbmi->mode == NEAR_NEWMV || mbmi->mode == NEW_NEARMV) + ref_mv_idx = 1 + mbmi->ref_mv_idx; +#endif + + if (compound_ref0_mode(mbmi->mode) == NEWMV) { +#if CONFIG_REF_MV + uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + if (xd->ref_mv_count[ref_frame_type] > 1) { + ref_mv[0] = xd->ref_mv_stack[ref_frame_type][ref_mv_idx].this_mv; + clamp_mv_ref(&ref_mv[0].as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + } +#endif + nearestmv[0] = ref_mv[0]; + } + if (compound_ref1_mode(mbmi->mode) == NEWMV) { +#if CONFIG_REF_MV + uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + if (xd->ref_mv_count[ref_frame_type] > 1) { + ref_mv[1] = xd->ref_mv_stack[ref_frame_type][ref_mv_idx].comp_mv; + clamp_mv_ref(&ref_mv[1].as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + } +#endif + nearestmv[1] = ref_mv[1]; + } + } else { +#endif // CONFIG_EXT_INTER + if (mbmi->mode == NEWMV) { + for (ref = 0; ref < 1 + is_compound; ++ref) { +#if CONFIG_REF_MV + uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + if (xd->ref_mv_count[ref_frame_type] > 1) { + ref_mv[ref] = + (ref == 0) + ? xd->ref_mv_stack[ref_frame_type][mbmi->ref_mv_idx].this_mv + : xd->ref_mv_stack[ref_frame_type][mbmi->ref_mv_idx] + .comp_mv; + clamp_mv_ref(&ref_mv[ref].as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + } +#endif + nearestmv[ref] = ref_mv[ref]; + } + } +#if CONFIG_EXT_INTER + } +#endif // CONFIG_EXT_INTER + + int mv_corrupted_flag = + !assign_mv(cm, xd, mbmi->mode, mbmi->ref_frame, 0, mbmi->mv, ref_mv, + nearestmv, nearmv, mi_row, mi_col, is_compound, allow_hp, r); + aom_merge_corrupted_flag(&xd->corrupted, mv_corrupted_flag); + } + +#if CONFIG_EXT_INTER + mbmi->use_wedge_interintra = 0; + if (cm->reference_mode != COMPOUND_REFERENCE && +#if CONFIG_SUPERTX + !supertx_enabled && +#endif + is_interintra_allowed(mbmi)) { + const int bsize_group = size_group_lookup[bsize]; + const int interintra = + aom_read(r, cm->fc->interintra_prob[bsize_group], ACCT_STR); + if (xd->counts) xd->counts->interintra[bsize_group][interintra]++; + assert(mbmi->ref_frame[1] == NONE_FRAME); + if (interintra) { + const INTERINTRA_MODE interintra_mode = + read_interintra_mode(cm, xd, r, bsize_group); + mbmi->ref_frame[1] = INTRA_FRAME; + mbmi->interintra_mode = interintra_mode; +#if CONFIG_EXT_INTRA + mbmi->angle_delta[0] = 0; + mbmi->angle_delta[1] = 0; +#if CONFIG_INTRA_INTERP + mbmi->intra_filter = INTRA_FILTER_LINEAR; +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0; + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = 0; +#endif // CONFIG_FILTER_INTRA + if (is_interintra_wedge_used(bsize)) { + mbmi->use_wedge_interintra = + aom_read(r, cm->fc->wedge_interintra_prob[bsize], ACCT_STR); + if (xd->counts) + xd->counts->wedge_interintra[bsize][mbmi->use_wedge_interintra]++; + if (mbmi->use_wedge_interintra) { + mbmi->interintra_wedge_index = + aom_read_literal(r, get_wedge_bits_lookup(bsize), ACCT_STR); + mbmi->interintra_wedge_sign = 0; + } + } + } + } +#endif // CONFIG_EXT_INTER + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + mbmi->motion_mode = SIMPLE_TRANSLATION; +#if CONFIG_WARPED_MOTION + if (mbmi->sb_type >= BLOCK_8X8 && !has_second_ref(mbmi)) + mbmi->num_proj_ref[0] = findSamples(cm, xd, mi_row, mi_col, pts, pts_inref); +#endif // CONFIG_WARPED_MOTION +#if CONFIG_MOTION_VAR + av1_count_overlappable_neighbors(cm, xd, mi_row, mi_col); +#endif + +#if CONFIG_SUPERTX + if (!supertx_enabled) { +#endif // CONFIG_SUPERTX +#if CONFIG_EXT_INTER + if (mbmi->ref_frame[1] != INTRA_FRAME) +#endif // CONFIG_EXT_INTER + mbmi->motion_mode = read_motion_mode(cm, xd, mi, r); +#if CONFIG_WARPED_MOTION + if (mbmi->motion_mode == WARPED_CAUSAL) { + mbmi->wm_params[0].wmtype = DEFAULT_WMTYPE; + if (find_projection(mbmi->num_proj_ref[0], pts, pts_inref, bsize, + mbmi->mv[0].as_mv.row, mbmi->mv[0].as_mv.col, + &mbmi->wm_params[0], mi_row, mi_col)) { + assert(0 && "Invalid Warped Model."); + } + } +#endif // CONFIG_WARPED_MOTION +#if CONFIG_SUPERTX + } +#endif // CONFIG_SUPERTX +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + +#if CONFIG_EXT_INTER + mbmi->interinter_compound_type = COMPOUND_AVERAGE; + if (cm->reference_mode != SINGLE_REFERENCE && + is_inter_compound_mode(mbmi->mode) +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + && mbmi->motion_mode == SIMPLE_TRANSLATION +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + ) { + if (is_any_masked_compound_used(bsize)) { +#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE + mbmi->interinter_compound_type = + aom_read_tree(r, av1_compound_type_tree, + cm->fc->compound_type_prob[bsize], ACCT_STR); +#endif // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE +#if CONFIG_WEDGE + if (mbmi->interinter_compound_type == COMPOUND_WEDGE) { + mbmi->wedge_index = + aom_read_literal(r, get_wedge_bits_lookup(bsize), ACCT_STR); + mbmi->wedge_sign = aom_read_bit(r, ACCT_STR); + } +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + if (mbmi->interinter_compound_type == COMPOUND_SEG) { + mbmi->mask_type = aom_read_literal(r, MAX_SEG_MASK_BITS, ACCT_STR); + } +#endif // CONFIG_COMPOUND_SEGMENT + } else { + mbmi->interinter_compound_type = COMPOUND_AVERAGE; + } + if (xd->counts) + xd->counts->compound_interinter[bsize][mbmi->interinter_compound_type]++; + } +#endif // CONFIG_EXT_INTER + +#if CONFIG_DUAL_FILTER || CONFIG_WARPED_MOTION || CONFIG_GLOBAL_MOTION + read_mb_interp_filter(cm, xd, mbmi, r); +#endif // CONFIG_DUAL_FILTER || CONFIG_WARPED_MOTION +} + +static void read_inter_frame_mode_info(AV1Decoder *const pbi, + MACROBLOCKD *const xd, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif // CONFIG_SUPERTX + int mi_row, int mi_col, aom_reader *r) { + AV1_COMMON *const cm = &pbi->common; + MODE_INFO *const mi = xd->mi[0]; + MB_MODE_INFO *const mbmi = &mi->mbmi; + int inter_block = 1; +#if CONFIG_VAR_TX + BLOCK_SIZE bsize = mbmi->sb_type; +#endif // CONFIG_VAR_TX + + mbmi->mv[0].as_int = 0; + mbmi->mv[1].as_int = 0; + mbmi->segment_id = read_inter_segment_id(cm, xd, mi_row, mi_col, r); +#if CONFIG_SUPERTX + if (!supertx_enabled) +#endif // CONFIG_SUPERTX + mbmi->skip = read_skip(cm, xd, mbmi->segment_id, r); + +#if CONFIG_DELTA_Q + if (cm->delta_q_present_flag) { + xd->current_qindex = + xd->prev_qindex + + read_delta_qindex(cm, xd, r, mbmi, mi_col, mi_row) * cm->delta_q_res; + /* Normative: Clamp to [1,MAXQ] to not interfere with lossless mode */ + xd->current_qindex = clamp(xd->current_qindex, 1, MAXQ); + xd->prev_qindex = xd->current_qindex; +#if CONFIG_EXT_DELTA_Q + if (cm->delta_lf_present_flag) { + mbmi->current_delta_lf_from_base = xd->current_delta_lf_from_base = + xd->prev_delta_lf_from_base + + read_delta_lflevel(cm, xd, r, mbmi, mi_col, mi_row) * + cm->delta_lf_res; + xd->prev_delta_lf_from_base = xd->current_delta_lf_from_base; + } +#endif + } +#endif + +#if CONFIG_SUPERTX + if (!supertx_enabled) { +#endif // CONFIG_SUPERTX + inter_block = read_is_inter_block(cm, xd, mbmi->segment_id, r); + +#if CONFIG_VAR_TX + xd->above_txfm_context = cm->above_txfm_context + mi_col; + xd->left_txfm_context = + xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK); + + if (cm->tx_mode == TX_MODE_SELECT && +#if CONFIG_CB4X4 + bsize > BLOCK_4X4 && +#else + bsize >= BLOCK_8X8 && +#endif + !mbmi->skip && inter_block) { + const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize]; + const int bh = tx_size_high_unit[max_tx_size]; + const int bw = tx_size_wide_unit[max_tx_size]; + const int width = block_size_wide[bsize] >> tx_size_wide_log2[0]; + const int height = block_size_high[bsize] >> tx_size_wide_log2[0]; + int idx, idy; + + mbmi->min_tx_size = TX_SIZES_ALL; + for (idy = 0; idy < height; idy += bh) + for (idx = 0; idx < width; idx += bw) + read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, + height != width, idy, idx, r); + } else { + mbmi->tx_size = read_tx_size(cm, xd, inter_block, !mbmi->skip, r); + + if (inter_block) { + const int width = block_size_wide[bsize] >> tx_size_wide_log2[0]; + const int height = block_size_high[bsize] >> tx_size_high_log2[0]; + int idx, idy; + for (idy = 0; idy < height; ++idy) + for (idx = 0; idx < width; ++idx) + mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size; + } + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); + set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, mbmi->skip, xd); + } +#else + mbmi->tx_size = read_tx_size(cm, xd, inter_block, !mbmi->skip, r); +#endif // CONFIG_VAR_TX +#if CONFIG_SUPERTX + } +#if CONFIG_VAR_TX + else if (inter_block) { + const int width = num_4x4_blocks_wide_lookup[bsize]; + const int height = num_4x4_blocks_high_lookup[bsize]; + int idx, idy; + xd->mi[0]->mbmi.tx_size = xd->supertx_size; + for (idy = 0; idy < height; ++idy) + for (idx = 0; idx < width; ++idx) + xd->mi[0]->mbmi.inter_tx_size[idy >> 1][idx >> 1] = xd->supertx_size; + } +#endif // CONFIG_VAR_TX +#endif // CONFIG_SUPERTX + + if (inter_block) + read_inter_block_mode_info(pbi, xd, +#if (CONFIG_MOTION_VAR || CONFIG_EXT_INTER || CONFIG_WARPED_MOTION) && \ + CONFIG_SUPERTX + + mi, mi_row, mi_col, r, supertx_enabled); +#else + mi, mi_row, mi_col, r); +#endif // CONFIG_MOTION_VAR && CONFIG_SUPERTX + else + read_intra_block_mode_info(cm, mi_row, mi_col, xd, mi, r); + +#if !CONFIG_TXK_SEL + av1_read_tx_type(cm, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + r); +#endif // !CONFIG_TXK_SEL +} + +void av1_read_mode_info(AV1Decoder *const pbi, MACROBLOCKD *xd, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif // CONFIG_SUPERTX + int mi_row, int mi_col, aom_reader *r, int x_mis, + int y_mis) { + AV1_COMMON *const cm = &pbi->common; + MODE_INFO *const mi = xd->mi[0]; + MV_REF *frame_mvs = cm->cur_frame->mvs + mi_row * cm->mi_cols + mi_col; + int w, h; + +#if CONFIG_INTRABC + mi->mbmi.use_intrabc = 0; +#endif // CONFIG_INTRABC + + if (frame_is_intra_only(cm)) { + read_intra_frame_mode_info(cm, xd, mi_row, mi_col, r); +#if CONFIG_REF_MV + for (h = 0; h < y_mis; ++h) { + MV_REF *const frame_mv = frame_mvs + h * cm->mi_cols; + for (w = 0; w < x_mis; ++w) { + MV_REF *const mv = frame_mv + w; + mv->ref_frame[0] = NONE_FRAME; + mv->ref_frame[1] = NONE_FRAME; + } + } +#endif + } else { + read_inter_frame_mode_info(pbi, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif // CONFIG_SUPERTX + mi_row, mi_col, r); + for (h = 0; h < y_mis; ++h) { + MV_REF *const frame_mv = frame_mvs + h * cm->mi_cols; + for (w = 0; w < x_mis; ++w) { + MV_REF *const mv = frame_mv + w; + mv->ref_frame[0] = mi->mbmi.ref_frame[0]; + mv->ref_frame[1] = mi->mbmi.ref_frame[1]; + mv->mv[0].as_int = mi->mbmi.mv[0].as_int; + mv->mv[1].as_int = mi->mbmi.mv[1].as_int; +#if CONFIG_REF_MV + mv->pred_mv[0].as_int = mi->mbmi.pred_mv[0].as_int; + mv->pred_mv[1].as_int = mi->mbmi.pred_mv[1].as_int; +#endif + } + } + } +} diff --git a/third_party/aom/av1/decoder/decodemv.h b/third_party/aom/av1/decoder/decodemv.h new file mode 100644 index 0000000000..ceaee1d6b0 --- /dev/null +++ b/third_party/aom/av1/decoder/decodemv.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_DECODER_DECODEMV_H_ +#define AV1_DECODER_DECODEMV_H_ + +#include "aom_dsp/bitreader.h" + +#include "av1/decoder/decoder.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_read_mode_info(AV1Decoder *const pbi, MACROBLOCKD *xd, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif + + int mi_row, int mi_col, aom_reader *r, int x_mis, + int y_mis); + +#ifdef __cplusplus +} // extern "C" +#endif + +void av1_read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif +#if CONFIG_TXK_SEL + int block, int plane, +#endif + aom_reader *r); + +#endif // AV1_DECODER_DECODEMV_H_ diff --git a/third_party/aom/av1/decoder/decoder.c b/third_party/aom/av1/decoder/decoder.c new file mode 100644 index 0000000000..1bd91086e5 --- /dev/null +++ b/third_party/aom/av1/decoder/decoder.c @@ -0,0 +1,583 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" +#include "./aom_scale_rtcd.h" + +#include "aom_mem/aom_mem.h" +#include "aom_ports/system_state.h" +#include "aom_ports/aom_once.h" +#include "aom_ports/aom_timer.h" +#include "aom_scale/aom_scale.h" +#include "aom_util/aom_thread.h" + +#include "av1/common/alloccommon.h" +#include "av1/common/av1_loopfilter.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/quant_common.h" +#include "av1/common/reconinter.h" +#include "av1/common/reconintra.h" + +#include "av1/decoder/decodeframe.h" +#include "av1/decoder/decoder.h" + +#if !CONFIG_PVQ +#include "av1/decoder/detokenize.h" +#endif + +static void initialize_dec(void) { + static volatile int init_done = 0; + + if (!init_done) { + av1_rtcd(); + aom_dsp_rtcd(); + aom_scale_rtcd(); + av1_init_intra_predictors(); +#if CONFIG_EXT_INTER + av1_init_wedge_masks(); +#endif // CONFIG_EXT_INTER + init_done = 1; +#if CONFIG_EC_MULTISYMBOL + av1_indices_from_tree(av1_intra_mode_ind, av1_intra_mode_inv, + av1_intra_mode_tree); + av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv, + av1_switchable_interp_tree); +#if CONFIG_EXT_TX + int s; + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) + av1_indices_from_tree(av1_ext_tx_intra_ind[s], av1_ext_tx_intra_inv[s], + av1_ext_tx_intra_tree[s]); + for (s = 1; s < EXT_TX_SETS_INTER; ++s) + av1_indices_from_tree(av1_ext_tx_inter_ind[s], av1_ext_tx_inter_inv[s], + av1_ext_tx_inter_tree[s]); +#else + av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree); +#endif + av1_indices_from_tree(av1_inter_mode_ind, av1_inter_mode_inv, + av1_inter_mode_tree); +#endif + } +} + +static void av1_dec_setup_mi(AV1_COMMON *cm) { + cm->mi = cm->mip + cm->mi_stride + 1; + cm->mi_grid_visible = cm->mi_grid_base + cm->mi_stride + 1; + memset(cm->mi_grid_base, 0, + cm->mi_stride * (cm->mi_rows + 1) * sizeof(*cm->mi_grid_base)); +} + +static int av1_dec_alloc_mi(AV1_COMMON *cm, int mi_size) { + cm->mip = aom_calloc(mi_size, sizeof(*cm->mip)); + if (!cm->mip) return 1; + cm->mi_alloc_size = mi_size; + cm->mi_grid_base = (MODE_INFO **)aom_calloc(mi_size, sizeof(MODE_INFO *)); + if (!cm->mi_grid_base) return 1; + return 0; +} + +static void av1_dec_free_mi(AV1_COMMON *cm) { + aom_free(cm->mip); + cm->mip = NULL; + aom_free(cm->mi_grid_base); + cm->mi_grid_base = NULL; +} + +AV1Decoder *av1_decoder_create(BufferPool *const pool) { + AV1Decoder *volatile const pbi = aom_memalign(32, sizeof(*pbi)); + AV1_COMMON *volatile const cm = pbi ? &pbi->common : NULL; + + if (!cm) return NULL; + + av1_zero(*pbi); + + if (setjmp(cm->error.jmp)) { + cm->error.setjmp = 0; + av1_decoder_remove(pbi); + return NULL; + } + + cm->error.setjmp = 1; + + CHECK_MEM_ERROR(cm, cm->fc, + (FRAME_CONTEXT *)aom_memalign(32, sizeof(*cm->fc))); + CHECK_MEM_ERROR(cm, cm->frame_contexts, + (FRAME_CONTEXT *)aom_memalign( + 32, FRAME_CONTEXTS * sizeof(*cm->frame_contexts))); + memset(cm->fc, 0, sizeof(*cm->fc)); + memset(cm->frame_contexts, 0, FRAME_CONTEXTS * sizeof(*cm->frame_contexts)); + + pbi->need_resync = 1; + once(initialize_dec); + + // Initialize the references to not point to any frame buffers. + memset(&cm->ref_frame_map, -1, sizeof(cm->ref_frame_map)); + memset(&cm->next_ref_frame_map, -1, sizeof(cm->next_ref_frame_map)); + + cm->current_video_frame = 0; + pbi->ready_for_new_data = 1; + pbi->common.buffer_pool = pool; + + cm->bit_depth = AOM_BITS_8; + cm->dequant_bit_depth = AOM_BITS_8; + + cm->alloc_mi = av1_dec_alloc_mi; + cm->free_mi = av1_dec_free_mi; + cm->setup_mi = av1_dec_setup_mi; + + av1_loop_filter_init(cm); + +#if CONFIG_AOM_QM + aom_qm_init(cm); +#endif +#if CONFIG_LOOP_RESTORATION + av1_loop_restoration_precal(); +#endif // CONFIG_LOOP_RESTORATION +#if CONFIG_ACCOUNTING + pbi->acct_enabled = 1; + aom_accounting_init(&pbi->accounting); +#endif + + cm->error.setjmp = 0; + + aom_get_worker_interface()->init(&pbi->lf_worker); + + return pbi; +} + +void av1_decoder_remove(AV1Decoder *pbi) { + int i; + + if (!pbi) return; + + aom_get_worker_interface()->end(&pbi->lf_worker); + aom_free(pbi->lf_worker.data1); + aom_free(pbi->tile_data); + for (i = 0; i < pbi->num_tile_workers; ++i) { + AVxWorker *const worker = &pbi->tile_workers[i]; + aom_get_worker_interface()->end(worker); + } + aom_free(pbi->tile_worker_data); + aom_free(pbi->tile_worker_info); + aom_free(pbi->tile_workers); + + if (pbi->num_tile_workers > 0) { + av1_loop_filter_dealloc(&pbi->lf_row_sync); + } + +#if CONFIG_ACCOUNTING + aom_accounting_clear(&pbi->accounting); +#endif + + aom_free(pbi); +} + +static int equal_dimensions(const YV12_BUFFER_CONFIG *a, + const YV12_BUFFER_CONFIG *b) { + return a->y_height == b->y_height && a->y_width == b->y_width && + a->uv_height == b->uv_height && a->uv_width == b->uv_width; +} + +aom_codec_err_t av1_copy_reference_dec(AV1Decoder *pbi, + AOM_REFFRAME ref_frame_flag, + YV12_BUFFER_CONFIG *sd) { + AV1_COMMON *cm = &pbi->common; + + /* TODO(jkoleszar): The decoder doesn't have any real knowledge of what the + * encoder is using the frame buffers for. This is just a stub to keep the + * aomenc --test-decode functionality working, and will be replaced in a + * later commit that adds AV1-specific controls for this functionality. + */ + if (ref_frame_flag == AOM_LAST_FLAG) { + const YV12_BUFFER_CONFIG *const cfg = get_ref_frame(cm, 0); + if (cfg == NULL) { + aom_internal_error(&cm->error, AOM_CODEC_ERROR, + "No 'last' reference frame"); + return AOM_CODEC_ERROR; + } + if (!equal_dimensions(cfg, sd)) + aom_internal_error(&cm->error, AOM_CODEC_ERROR, + "Incorrect buffer dimensions"); + else + aom_yv12_copy_frame(cfg, sd); + } else { + aom_internal_error(&cm->error, AOM_CODEC_ERROR, "Invalid reference frame"); + } + + return cm->error.error_code; +} + +aom_codec_err_t av1_set_reference_dec(AV1_COMMON *cm, + AOM_REFFRAME ref_frame_flag, + YV12_BUFFER_CONFIG *sd) { + int idx; + YV12_BUFFER_CONFIG *ref_buf = NULL; + + // TODO(jkoleszar): The decoder doesn't have any real knowledge of what the + // encoder is using the frame buffers for. This is just a stub to keep the + // aomenc --test-decode functionality working, and will be replaced in a + // later commit that adds AV1-specific controls for this functionality. + + // (Yunqing) The set_reference control depends on the following setting in + // encoder. + // cpi->lst_fb_idx = 0; + // #if CONFIG_EXT_REFS + // cpi->lst2_fb_idx = 1; + // cpi->lst3_fb_idx = 2; + // cpi->gld_fb_idx = 3; + // cpi->bwd_fb_idx = 4; + // cpi->alt_fb_idx = 5; + // #else // CONFIG_EXT_REFS + // cpi->gld_fb_idx = 1; + // cpi->alt_fb_idx = 2; + // #endif // CONFIG_EXT_REFS + + // TODO(zoeliu): To revisit following code and reconsider what assumption we + // may take on the reference frame buffer virtual indexes + if (ref_frame_flag == AOM_LAST_FLAG) { + idx = cm->ref_frame_map[0]; +#if CONFIG_EXT_REFS + } else if (ref_frame_flag == AOM_LAST2_FLAG) { + idx = cm->ref_frame_map[1]; + } else if (ref_frame_flag == AOM_LAST3_FLAG) { + idx = cm->ref_frame_map[2]; + } else if (ref_frame_flag == AOM_GOLD_FLAG) { + idx = cm->ref_frame_map[3]; + } else if (ref_frame_flag == AOM_BWD_FLAG) { + idx = cm->ref_frame_map[4]; + } else if (ref_frame_flag == AOM_ALT_FLAG) { + idx = cm->ref_frame_map[5]; +#else + } else if (ref_frame_flag == AOM_GOLD_FLAG) { + idx = cm->ref_frame_map[1]; + } else if (ref_frame_flag == AOM_ALT_FLAG) { + idx = cm->ref_frame_map[2]; +#endif // CONFIG_EXT_REFS + } else { + aom_internal_error(&cm->error, AOM_CODEC_ERROR, "Invalid reference frame"); + return cm->error.error_code; + } + + if (idx < 0 || idx >= FRAME_BUFFERS) { + aom_internal_error(&cm->error, AOM_CODEC_ERROR, + "Invalid reference frame map"); + return cm->error.error_code; + } + + // Get the destination reference buffer. + ref_buf = &cm->buffer_pool->frame_bufs[idx].buf; + + if (!equal_dimensions(ref_buf, sd)) { + aom_internal_error(&cm->error, AOM_CODEC_ERROR, + "Incorrect buffer dimensions"); + } else { + // Overwrite the reference frame buffer. + aom_yv12_copy_frame(sd, ref_buf); + } + + return cm->error.error_code; +} + +/* If any buffer updating is signaled it should be done here. */ +static void swap_frame_buffers(AV1Decoder *pbi) { + int ref_index = 0, mask; + AV1_COMMON *const cm = &pbi->common; + BufferPool *const pool = cm->buffer_pool; + RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; + + lock_buffer_pool(pool); + for (mask = pbi->refresh_frame_flags; mask; mask >>= 1) { + const int old_idx = cm->ref_frame_map[ref_index]; + // Current thread releases the holding of reference frame. + decrease_ref_count(old_idx, frame_bufs, pool); + + // Release the reference frame holding in the reference map for the decoding + // of the next frame. + if (mask & 1) decrease_ref_count(old_idx, frame_bufs, pool); + cm->ref_frame_map[ref_index] = cm->next_ref_frame_map[ref_index]; + ++ref_index; + } + + // Current thread releases the holding of reference frame. + for (; ref_index < REF_FRAMES && !cm->show_existing_frame; ++ref_index) { + const int old_idx = cm->ref_frame_map[ref_index]; + decrease_ref_count(old_idx, frame_bufs, pool); + cm->ref_frame_map[ref_index] = cm->next_ref_frame_map[ref_index]; + } + + unlock_buffer_pool(pool); + pbi->hold_ref_buf = 0; + cm->frame_to_show = get_frame_new_buffer(cm); + + // TODO(zoeliu): To fix the ref frame buffer update for the scenario of + // cm->frame_parellel_decode == 1 + if (!cm->frame_parallel_decode || !cm->show_frame) { + lock_buffer_pool(pool); + --frame_bufs[cm->new_fb_idx].ref_count; + unlock_buffer_pool(pool); + } + + // Invalidate these references until the next frame starts. + for (ref_index = 0; ref_index < INTER_REFS_PER_FRAME; ref_index++) { + cm->frame_refs[ref_index].idx = INVALID_IDX; + cm->frame_refs[ref_index].buf = NULL; + } +} + +int av1_receive_compressed_data(AV1Decoder *pbi, size_t size, + const uint8_t **psource) { + AV1_COMMON *volatile const cm = &pbi->common; + BufferPool *volatile const pool = cm->buffer_pool; + RefCntBuffer *volatile const frame_bufs = cm->buffer_pool->frame_bufs; + const uint8_t *source = *psource; + int retcode = 0; + cm->error.error_code = AOM_CODEC_OK; + + if (size == 0) { + // This is used to signal that we are missing frames. + // We do not know if the missing frame(s) was supposed to update + // any of the reference buffers, but we act conservative and + // mark only the last buffer as corrupted. + // + // TODO(jkoleszar): Error concealment is undefined and non-normative + // at this point, but if it becomes so, [0] may not always be the correct + // thing to do here. + if (cm->frame_refs[0].idx > 0) { + assert(cm->frame_refs[0].buf != NULL); + cm->frame_refs[0].buf->corrupted = 1; + } + } + + pbi->ready_for_new_data = 0; + + // Find a free buffer for the new frame, releasing the reference previously + // held. + + // Check if the previous frame was a frame without any references to it. + // Release frame buffer if not decoding in frame parallel mode. + if (!cm->frame_parallel_decode && cm->new_fb_idx >= 0 && + frame_bufs[cm->new_fb_idx].ref_count == 0) + pool->release_fb_cb(pool->cb_priv, + &frame_bufs[cm->new_fb_idx].raw_frame_buffer); + + // Find a free frame buffer. Return error if can not find any. + cm->new_fb_idx = get_free_fb(cm); + if (cm->new_fb_idx == INVALID_IDX) return AOM_CODEC_MEM_ERROR; + + // Assign a MV array to the frame buffer. + cm->cur_frame = &pool->frame_bufs[cm->new_fb_idx]; + + pbi->hold_ref_buf = 0; + if (cm->frame_parallel_decode) { + AVxWorker *const worker = pbi->frame_worker_owner; + av1_frameworker_lock_stats(worker); + frame_bufs[cm->new_fb_idx].frame_worker_owner = worker; + // Reset decoding progress. + pbi->cur_buf = &frame_bufs[cm->new_fb_idx]; + pbi->cur_buf->row = -1; + pbi->cur_buf->col = -1; + av1_frameworker_unlock_stats(worker); + } else { + pbi->cur_buf = &frame_bufs[cm->new_fb_idx]; + } + + if (setjmp(cm->error.jmp)) { + const AVxWorkerInterface *const winterface = aom_get_worker_interface(); + int i; + + cm->error.setjmp = 0; + pbi->ready_for_new_data = 1; + + // Synchronize all threads immediately as a subsequent decode call may + // cause a resize invalidating some allocations. + winterface->sync(&pbi->lf_worker); + for (i = 0; i < pbi->num_tile_workers; ++i) { + winterface->sync(&pbi->tile_workers[i]); + } + + lock_buffer_pool(pool); + // Release all the reference buffers if worker thread is holding them. + if (pbi->hold_ref_buf == 1) { + int ref_index = 0, mask; + for (mask = pbi->refresh_frame_flags; mask; mask >>= 1) { + const int old_idx = cm->ref_frame_map[ref_index]; + // Current thread releases the holding of reference frame. + decrease_ref_count(old_idx, frame_bufs, pool); + + // Release the reference frame holding in the reference map for the + // decoding of the next frame. + if (mask & 1) decrease_ref_count(old_idx, frame_bufs, pool); + ++ref_index; + } + + // Current thread releases the holding of reference frame. + for (; ref_index < REF_FRAMES && !cm->show_existing_frame; ++ref_index) { + const int old_idx = cm->ref_frame_map[ref_index]; + decrease_ref_count(old_idx, frame_bufs, pool); + } + pbi->hold_ref_buf = 0; + } + // Release current frame. + decrease_ref_count(cm->new_fb_idx, frame_bufs, pool); + unlock_buffer_pool(pool); + + aom_clear_system_state(); + return -1; + } + + cm->error.setjmp = 1; + av1_decode_frame(pbi, source, source + size, psource); + + swap_frame_buffers(pbi); + +#if CONFIG_EXT_TILE + // For now, we only extend the frame borders when the whole frame is decoded. + // Later, if needed, extend the border for the decoded tile on the frame + // border. + if (pbi->dec_tile_row == -1 && pbi->dec_tile_col == -1) +#endif // CONFIG_EXT_TILE + aom_extend_frame_inner_borders(cm->frame_to_show); + + aom_clear_system_state(); + + if (!cm->show_existing_frame) { + cm->last_show_frame = cm->show_frame; + +#if CONFIG_EXT_REFS + // NOTE: It is not supposed to ref to any frame not used as reference + if (cm->is_reference_frame) +#endif // CONFIG_EXT_REFS + cm->prev_frame = cm->cur_frame; + + if (cm->seg.enabled && !cm->frame_parallel_decode) + av1_swap_current_and_last_seg_map(cm); + } + + // Update progress in frame parallel decode. + if (cm->frame_parallel_decode) { + // Need to lock the mutex here as another thread may + // be accessing this buffer. + AVxWorker *const worker = pbi->frame_worker_owner; + FrameWorkerData *const frame_worker_data = worker->data1; + av1_frameworker_lock_stats(worker); + + if (cm->show_frame) { + cm->current_video_frame++; + } + frame_worker_data->frame_decoded = 1; + frame_worker_data->frame_context_ready = 1; + av1_frameworker_signal_stats(worker); + av1_frameworker_unlock_stats(worker); + } else { + cm->last_width = cm->width; + cm->last_height = cm->height; + if (cm->show_frame) { + cm->current_video_frame++; + } + } + + cm->error.setjmp = 0; + return retcode; +} + +int av1_get_raw_frame(AV1Decoder *pbi, YV12_BUFFER_CONFIG *sd) { + AV1_COMMON *const cm = &pbi->common; + int ret = -1; + if (pbi->ready_for_new_data == 1) return ret; + + pbi->ready_for_new_data = 1; + + /* no raw frame to show!!! */ + if (!cm->show_frame) return ret; + + pbi->ready_for_new_data = 1; + *sd = *cm->frame_to_show; + ret = 0; + aom_clear_system_state(); + return ret; +} + +int av1_get_frame_to_show(AV1Decoder *pbi, YV12_BUFFER_CONFIG *frame) { + AV1_COMMON *const cm = &pbi->common; + + if (!cm->show_frame || !cm->frame_to_show) return -1; + + *frame = *cm->frame_to_show; + return 0; +} + +aom_codec_err_t av1_parse_superframe_index(const uint8_t *data, size_t data_sz, + uint32_t sizes[8], int *count, + aom_decrypt_cb decrypt_cb, + void *decrypt_state) { + // A chunk ending with a byte matching 0xc0 is an invalid chunk unless + // it is a super frame index. If the last byte of real video compression + // data is 0xc0 the encoder must add a 0 byte. If we have the marker but + // not the associated matching marker byte at the front of the index we have + // an invalid bitstream and need to return an error. + + uint8_t marker; + size_t frame_sz_sum = 0; + + assert(data_sz); + marker = read_marker(decrypt_cb, decrypt_state, data + data_sz - 1); + *count = 0; + + if ((marker & 0xe0) == 0xc0) { + const uint32_t frames = (marker & 0x7) + 1; + const uint32_t mag = ((marker >> 3) & 0x3) + 1; + const size_t index_sz = 2 + mag * (frames - 1); + + // This chunk is marked as having a superframe index but doesn't have + // enough data for it, thus it's an invalid superframe index. + if (data_sz < index_sz) return AOM_CODEC_CORRUPT_FRAME; + + { + const uint8_t marker2 = + read_marker(decrypt_cb, decrypt_state, data + data_sz - index_sz); + + // This chunk is marked as having a superframe index but doesn't have + // the matching marker byte at the front of the index therefore it's an + // invalid chunk. + if (marker != marker2) return AOM_CODEC_CORRUPT_FRAME; + } + + { + // Found a valid superframe index. + uint32_t i, j; + const uint8_t *x = &data[data_sz - index_sz + 1]; + + // Frames has a maximum of 8 and mag has a maximum of 4. + uint8_t clear_buffer[28]; + assert(sizeof(clear_buffer) >= (frames - 1) * mag); + if (decrypt_cb) { + decrypt_cb(decrypt_state, x, clear_buffer, (frames - 1) * mag); + x = clear_buffer; + } + + for (i = 0; i < frames - 1; ++i) { + uint32_t this_sz = 0; + + for (j = 0; j < mag; ++j) this_sz |= (*x++) << (j * 8); + this_sz += 1; + sizes[i] = this_sz; + frame_sz_sum += this_sz; + } + sizes[i] = (uint32_t)(data_sz - index_sz - frame_sz_sum); + *count = frames; + } + } + return AOM_CODEC_OK; +} diff --git a/third_party/aom/av1/decoder/decoder.h b/third_party/aom/av1/decoder/decoder.h new file mode 100644 index 0000000000..4a90b4ad55 --- /dev/null +++ b/third_party/aom/av1/decoder/decoder.h @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_DECODER_DECODER_H_ +#define AV1_DECODER_DECODER_H_ + +#include "./aom_config.h" + +#include "aom/aom_codec.h" +#include "aom_dsp/bitreader.h" +#include "aom_scale/yv12config.h" +#include "aom_util/aom_thread.h" + +#include "av1/common/thread_common.h" +#include "av1/common/onyxc_int.h" +#include "av1/decoder/dthread.h" +#if CONFIG_ACCOUNTING +#include "av1/decoder/accounting.h" +#endif +#if CONFIG_INSPECTION +#include "av1/decoder/inspection.h" +#endif + +#if CONFIG_PVQ +#include "aom_dsp/entdec.h" +#include "av1/decoder/decint.h" +#include "av1/encoder/encodemb.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +// TODO(hkuang): combine this with TileWorkerData. +typedef struct TileData { + AV1_COMMON *cm; + aom_reader bit_reader; + DECLARE_ALIGNED(16, MACROBLOCKD, xd); + /* dqcoeff are shared by all the planes. So planes must be decoded serially */ + DECLARE_ALIGNED(16, tran_low_t, dqcoeff[MAX_TX_SQUARE]); +#if CONFIG_PVQ + /* forward transformed predicted image, a reference for PVQ */ + DECLARE_ALIGNED(16, tran_low_t, pvq_ref_coeff[OD_TXSIZE_MAX * OD_TXSIZE_MAX]); +#endif +#if CONFIG_CFL + CFL_CTX cfl; +#endif +#if CONFIG_EC_ADAPT + DECLARE_ALIGNED(16, FRAME_CONTEXT, tctx); +#endif +#if CONFIG_PALETTE + DECLARE_ALIGNED(16, uint8_t, color_index_map[2][MAX_SB_SQUARE]); +#endif // CONFIG_PALETTE +} TileData; + +typedef struct TileWorkerData { + struct AV1Decoder *pbi; + aom_reader bit_reader; + FRAME_COUNTS counts; + DECLARE_ALIGNED(16, MACROBLOCKD, xd); + /* dqcoeff are shared by all the planes. So planes must be decoded serially */ + DECLARE_ALIGNED(16, tran_low_t, dqcoeff[MAX_TX_SQUARE]); +#if CONFIG_PVQ + /* forward transformed predicted image, a reference for PVQ */ + DECLARE_ALIGNED(16, tran_low_t, pvq_ref_coeff[OD_TXSIZE_MAX * OD_TXSIZE_MAX]); +#endif +#if CONFIG_CFL + CFL_CTX cfl; +#endif +#if CONFIG_EC_ADAPT + FRAME_CONTEXT tctx; +#endif +#if CONFIG_PALETTE + DECLARE_ALIGNED(16, uint8_t, color_index_map[2][MAX_SB_SQUARE]); +#endif // CONFIG_PALETTE + struct aom_internal_error_info error_info; +} TileWorkerData; + +typedef struct TileBufferDec { + const uint8_t *data; + size_t size; + const uint8_t *raw_data_end; // The end of the raw tile buffer in the + // bit stream. + int col; // only used with multi-threaded decoding +} TileBufferDec; + +typedef struct AV1Decoder { + DECLARE_ALIGNED(16, MACROBLOCKD, mb); + + DECLARE_ALIGNED(16, AV1_COMMON, common); + + int ready_for_new_data; + + int refresh_frame_flags; + + // TODO(hkuang): Combine this with cur_buf in macroblockd as they are + // the same. + RefCntBuffer *cur_buf; // Current decoding frame buffer. + + AVxWorker *frame_worker_owner; // frame_worker that owns this pbi. + AVxWorker lf_worker; + AVxWorker *tile_workers; + TileWorkerData *tile_worker_data; + TileInfo *tile_worker_info; + int num_tile_workers; + + TileData *tile_data; + int allocated_tiles; + + TileBufferDec tile_buffers[MAX_TILE_ROWS][MAX_TILE_COLS]; + + AV1LfSync lf_row_sync; + + aom_decrypt_cb decrypt_cb; + void *decrypt_state; + + int max_threads; + int inv_tile_order; + int need_resync; // wait for key/intra-only frame. + int hold_ref_buf; // hold the reference buffer. + + int tile_size_bytes; +#if CONFIG_EXT_TILE + int tile_col_size_bytes; + int dec_tile_row, dec_tile_col; +#endif // CONFIG_EXT_TILE +#if CONFIG_ACCOUNTING + int acct_enabled; + Accounting accounting; +#endif + size_t uncomp_hdr_size; // Size of the uncompressed header + size_t first_partition_size; // Size of the compressed header +#if CONFIG_TILE_GROUPS + int tg_size; // Number of tiles in the current tilegroup + int tg_start; // First tile in the current tilegroup + int tg_size_bit_offset; +#endif +#if CONFIG_REFERENCE_BUFFER + SequenceHeader seq_params; +#endif +#if CONFIG_INSPECTION + aom_inspect_cb inspect_cb; + void *inspect_ctx; +#endif +} AV1Decoder; + +int av1_receive_compressed_data(struct AV1Decoder *pbi, size_t size, + const uint8_t **dest); + +int av1_get_raw_frame(struct AV1Decoder *pbi, YV12_BUFFER_CONFIG *sd); + +int av1_get_frame_to_show(struct AV1Decoder *pbi, YV12_BUFFER_CONFIG *frame); + +aom_codec_err_t av1_copy_reference_dec(struct AV1Decoder *pbi, + AOM_REFFRAME ref_frame_flag, + YV12_BUFFER_CONFIG *sd); + +aom_codec_err_t av1_set_reference_dec(AV1_COMMON *cm, + AOM_REFFRAME ref_frame_flag, + YV12_BUFFER_CONFIG *sd); + +static INLINE uint8_t read_marker(aom_decrypt_cb decrypt_cb, + void *decrypt_state, const uint8_t *data) { + if (decrypt_cb) { + uint8_t marker; + decrypt_cb(decrypt_state, data, &marker, 1); + return marker; + } + return *data; +} + +// This function is exposed for use in tests, as well as the inlined function +// "read_marker". +aom_codec_err_t av1_parse_superframe_index(const uint8_t *data, size_t data_sz, + uint32_t sizes[8], int *count, + aom_decrypt_cb decrypt_cb, + void *decrypt_state); + +struct AV1Decoder *av1_decoder_create(BufferPool *const pool); + +void av1_decoder_remove(struct AV1Decoder *pbi); + +static INLINE void decrease_ref_count(int idx, RefCntBuffer *const frame_bufs, + BufferPool *const pool) { + if (idx >= 0) { + --frame_bufs[idx].ref_count; + // A worker may only get a free framebuffer index when calling get_free_fb. + // But the private buffer is not set up until finish decoding header. + // So any error happens during decoding header, the frame_bufs will not + // have valid priv buffer. + if (frame_bufs[idx].ref_count == 0 && + frame_bufs[idx].raw_frame_buffer.priv) { + pool->release_fb_cb(pool->cb_priv, &frame_bufs[idx].raw_frame_buffer); + } + } +} + +#if CONFIG_EXT_REFS +static INLINE int dec_is_ref_frame_buf(AV1Decoder *const pbi, + RefCntBuffer *frame_buf) { + AV1_COMMON *const cm = &pbi->common; + int i; + for (i = 0; i < INTER_REFS_PER_FRAME; ++i) { + RefBuffer *const ref_frame = &cm->frame_refs[i]; + if (ref_frame->idx == INVALID_IDX) continue; + if (frame_buf == &cm->buffer_pool->frame_bufs[ref_frame->idx]) break; + } + return (i < INTER_REFS_PER_FRAME); +} +#endif // CONFIG_EXT_REFS + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_DECODER_DECODER_H_ diff --git a/third_party/aom/av1/decoder/decodetxb.c b/third_party/aom/av1/decoder/decodetxb.c new file mode 100644 index 0000000000..e1db09775a --- /dev/null +++ b/third_party/aom/av1/decoder/decodetxb.c @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/common/scan.h" +#include "av1/common/idct.h" +#include "av1/common/txb_common.h" +#include "av1/decoder/decodemv.h" +#include "av1/decoder/decodetxb.h" +#include "av1/decoder/dsubexp.h" + +#define ACCT_STR __func__ + +static int read_golomb(MACROBLOCKD *xd, aom_reader *r) { + int x = 1; + int length = 0; + int i = 0; + + while (!i) { + i = aom_read_bit(r, ACCT_STR); + ++length; + if (length >= 32) { + aom_internal_error(xd->error_info, AOM_CODEC_CORRUPT_FRAME, + "Invalid length in read_golomb"); + break; + } + } + + for (i = 0; i < length - 1; ++i) { + x <<= 1; + x += aom_read_bit(r, ACCT_STR); + } + + return x - 1; +} + +uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, + aom_reader *r, int block, int plane, + tran_low_t *tcoeffs, TXB_CTX *txb_ctx, + int16_t *max_scan_line, int *eob) { + FRAME_COUNTS *counts = xd->counts; + TX_SIZE tx_size = get_tx_size(plane, xd); + PLANE_TYPE plane_type = get_plane_type(plane); + aom_prob *nz_map = cm->fc->nz_map[tx_size][plane_type]; + aom_prob *eob_flag = cm->fc->eob_flag[tx_size][plane_type]; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const int seg_eob = tx_size_2d[tx_size]; + int c = 0; + int update_eob = -1; + const int16_t *const dequant = xd->plane[plane].seg_dequant[mbmi->segment_id]; + const int shift = av1_get_tx_scale(tx_size); + const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; + int cul_level = 0; + unsigned int(*nz_map_count)[SIG_COEF_CONTEXTS][2]; + uint8_t txb_mask[32 * 32] = { 0 }; + + nz_map_count = (counts) ? &counts->nz_map[tx_size][plane_type] : NULL; + + memset(tcoeffs, 0, sizeof(*tcoeffs) * seg_eob); + + int all_zero = + aom_read(r, cm->fc->txb_skip[tx_size][txb_ctx->txb_skip_ctx], ACCT_STR); + if (xd->counts) + ++xd->counts->txb_skip[tx_size][txb_ctx->txb_skip_ctx][all_zero]; + + *eob = 0; + if (all_zero) { + *max_scan_line = 0; + return 0; + } + +#if CONFIG_TXK_SEL + av1_read_tx_type(cm, xd, block, plane, r); +#endif + TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + const SCAN_ORDER *const scan_order = + get_scan(cm, tx_size, tx_type, is_inter_block(mbmi)); + const int16_t *scan = scan_order->scan; + + for (c = 0; c < seg_eob; ++c) { + int is_nz; + int coeff_ctx = get_nz_map_ctx(tcoeffs, txb_mask, scan[c], bwl); + int eob_ctx = get_eob_ctx(tcoeffs, scan[c], bwl); + + if (c < seg_eob - 1) + is_nz = aom_read(r, nz_map[coeff_ctx], tx_size); + else + is_nz = 1; + + // set non-zero coefficient map. + tcoeffs[scan[c]] = is_nz; + + if (c == seg_eob - 1) { + ++c; + break; + } + + if (counts) ++(*nz_map_count)[coeff_ctx][is_nz]; + + if (is_nz) { + int is_eob = aom_read(r, eob_flag[eob_ctx], tx_size); + if (counts) ++counts->eob_flag[tx_size][plane_type][eob_ctx][is_eob]; + if (is_eob) break; + } + txb_mask[scan[c]] = 1; + } + + *eob = AOMMIN(seg_eob, c + 1); + *max_scan_line = *eob; + + int i; + for (i = 0; i < NUM_BASE_LEVELS; ++i) { + aom_prob *coeff_base = cm->fc->coeff_base[tx_size][plane_type][i]; + + update_eob = 0; + for (c = *eob - 1; c >= 0; --c) { + tran_low_t *v = &tcoeffs[scan[c]]; + int sign; + int ctx; + + if (*v <= i) continue; + + ctx = get_base_ctx(tcoeffs, scan[c], bwl, i + 1); + + if (aom_read(r, coeff_base[ctx], tx_size)) { + *v = i + 1; + cul_level += i + 1; + + if (counts) ++counts->coeff_base[tx_size][plane_type][i][ctx][1]; + + if (c == 0) { + int dc_sign_ctx = txb_ctx->dc_sign_ctx; + sign = aom_read(r, cm->fc->dc_sign[plane_type][dc_sign_ctx], tx_size); + if (counts) ++counts->dc_sign[plane_type][dc_sign_ctx][sign]; + } else { + sign = aom_read_bit(r, ACCT_STR); + } + if (sign) *v = -(*v); + continue; + } + *v = i + 2; + if (counts) ++counts->coeff_base[tx_size][plane_type][i][ctx][0]; + + // update the eob flag for coefficients with magnitude above 1. + update_eob = AOMMAX(update_eob, c); + } + } + + for (c = update_eob; c >= 0; --c) { + tran_low_t *v = &tcoeffs[scan[c]]; + int sign; + int idx; + int ctx; + + if (*v <= NUM_BASE_LEVELS) continue; + + if (c == 0) { + int dc_sign_ctx = txb_ctx->dc_sign_ctx; + sign = aom_read(r, cm->fc->dc_sign[plane_type][dc_sign_ctx], tx_size); + if (counts) ++counts->dc_sign[plane_type][dc_sign_ctx][sign]; + } else { + sign = aom_read_bit(r, ACCT_STR); + } + + ctx = get_level_ctx(tcoeffs, scan[c], bwl); + + if (cm->fc->coeff_lps[tx_size][plane_type][ctx] == 0) exit(0); + + for (idx = 0; idx < COEFF_BASE_RANGE; ++idx) { + if (aom_read(r, cm->fc->coeff_lps[tx_size][plane_type][ctx], tx_size)) { + *v = (idx + 1 + NUM_BASE_LEVELS); + if (sign) *v = -(*v); + cul_level += abs(*v); + + if (counts) ++counts->coeff_lps[tx_size][plane_type][ctx][1]; + break; + } + if (counts) ++counts->coeff_lps[tx_size][plane_type][ctx][0]; + } + if (idx < COEFF_BASE_RANGE) continue; + + // decode 0-th order Golomb code + *v = read_golomb(xd, r) + COEFF_BASE_RANGE + 1 + NUM_BASE_LEVELS; + if (sign) *v = -(*v); + cul_level += abs(*v); + } + + for (c = 0; c < *eob; ++c) { + int16_t dqv = (c == 0) ? dequant[0] : dequant[1]; + tran_low_t *v = &tcoeffs[scan[c]]; + int sign = (*v) < 0; + *v = (abs(*v) * dqv) >> shift; + if (sign) *v = -(*v); + } + + cul_level = AOMMIN(63, cul_level); + + // DC value + set_dc_sign(&cul_level, tcoeffs[0]); + + return cul_level; +} + +uint8_t av1_read_coeffs_txb_facade(AV1_COMMON *cm, MACROBLOCKD *xd, + aom_reader *r, int row, int col, int block, + int plane, tran_low_t *tcoeffs, + int16_t *max_scan_line, int *eob) { + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + struct macroblockd_plane *pd = &xd->plane[plane]; + + const BLOCK_SIZE bsize = mbmi->sb_type; +#if CONFIG_CB4X4 +#if CONFIG_CHROMA_2X2 + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); +#else + const BLOCK_SIZE plane_bsize = + AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd)); +#endif // CONFIG_CHROMA_2X2 +#else // CONFIG_CB4X4 + const BLOCK_SIZE plane_bsize = + get_plane_block_size(AOMMAX(BLOCK_8X8, bsize), pd); +#endif // CONFIG_CB4X4 + + TX_SIZE tx_size = get_tx_size(plane, xd); + TXB_CTX txb_ctx; + get_txb_ctx(plane_bsize, tx_size, plane, pd->above_context + col, + pd->left_context + row, &txb_ctx); + uint8_t cul_level = av1_read_coeffs_txb(cm, xd, r, block, plane, tcoeffs, + &txb_ctx, max_scan_line, eob); +#if CONFIG_ADAPT_SCAN + PLANE_TYPE plane_type = get_plane_type(plane); + TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + if (xd->counts && *eob > 0) + av1_update_scan_count_facade(cm, xd->counts, tx_size, tx_type, pd->dqcoeff, + *eob); +#endif + av1_set_contexts(xd, pd, plane, tx_size, cul_level, col, row); + return cul_level; +} + +static void read_txb_probs(FRAME_CONTEXT *fc, const TX_SIZE tx_size, + aom_reader *r) { + int plane, ctx, level; + + if (aom_read_bit(r, ACCT_STR) == 0) return; + + for (ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx) + av1_diff_update_prob(r, &fc->txb_skip[tx_size][ctx], ACCT_STR); + + for (plane = 0; plane < PLANE_TYPES; ++plane) + for (ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx) + av1_diff_update_prob(r, &fc->nz_map[tx_size][plane][ctx], ACCT_STR); + + for (plane = 0; plane < PLANE_TYPES; ++plane) + for (ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx) + av1_diff_update_prob(r, &fc->eob_flag[tx_size][plane][ctx], ACCT_STR); + + for (level = 0; level < NUM_BASE_LEVELS; ++level) + for (plane = 0; plane < PLANE_TYPES; ++plane) + for (ctx = 0; ctx < COEFF_BASE_CONTEXTS; ++ctx) + av1_diff_update_prob(r, &fc->coeff_base[tx_size][plane][level][ctx], + ACCT_STR); + + for (plane = 0; plane < PLANE_TYPES; ++plane) + for (ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx) + av1_diff_update_prob(r, &fc->coeff_lps[tx_size][plane][ctx], ACCT_STR); +} + +void av1_read_txb_probs(FRAME_CONTEXT *fc, TX_MODE tx_mode, aom_reader *r) { + const TX_SIZE max_tx_size = tx_mode_to_biggest_tx_size[tx_mode]; + TX_SIZE tx_size; + int ctx, plane; + for (plane = 0; plane < PLANE_TYPES; ++plane) + for (ctx = 0; ctx < DC_SIGN_CONTEXTS; ++ctx) + av1_diff_update_prob(r, &fc->dc_sign[plane][ctx], ACCT_STR); + + for (tx_size = TX_4X4; tx_size <= max_tx_size; ++tx_size) + read_txb_probs(fc, tx_size, r); +} diff --git a/third_party/aom/av1/decoder/decodetxb.h b/third_party/aom/av1/decoder/decodetxb.h new file mode 100644 index 0000000000..ee1bf6a3df --- /dev/null +++ b/third_party/aom/av1/decoder/decodetxb.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef DECODETXB_H_ +#define DECODETXB_H_ + +#include "./aom_config.h" +#include "av1/common/blockd.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/txb_common.h" +#include "aom_dsp/bitreader.h" + +uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, + aom_reader *r, int block, int plane, + tran_low_t *tcoeffs, TXB_CTX *txb_ctx, + int16_t *max_scan_line, int *eob); + +uint8_t av1_read_coeffs_txb_facade(AV1_COMMON *cm, MACROBLOCKD *xd, + aom_reader *r, int row, int col, int block, + int plane, tran_low_t *tcoeffs, + int16_t *max_scan_line, int *eob); +void av1_read_txb_probs(FRAME_CONTEXT *fc, TX_MODE tx_mode, aom_reader *r); +#endif // DECODETXB_H_ diff --git a/third_party/aom/av1/decoder/detokenize.c b/third_party/aom/av1/decoder/detokenize.c new file mode 100644 index 0000000000..494f1681fc --- /dev/null +++ b/third_party/aom/av1/decoder/detokenize.c @@ -0,0 +1,467 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" +#if !CONFIG_PVQ +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" +#endif // !CONFIG_PVQ + +#include "av1/common/blockd.h" + +#define ACCT_STR __func__ + +#if !CONFIG_PVQ || CONFIG_VAR_TX +#include "av1/common/common.h" +#include "av1/common/entropy.h" +#include "av1/common/idct.h" +#include "av1/decoder/detokenize.h" + +#define EOB_CONTEXT_NODE 0 +#define ZERO_CONTEXT_NODE 1 +#define ONE_CONTEXT_NODE 2 +#define LOW_VAL_CONTEXT_NODE 0 +#define TWO_CONTEXT_NODE 1 +#define THREE_CONTEXT_NODE 2 +#define HIGH_LOW_CONTEXT_NODE 3 +#define CAT_ONE_CONTEXT_NODE 4 +#define CAT_THREEFOUR_CONTEXT_NODE 5 +#define CAT_THREE_CONTEXT_NODE 6 +#define CAT_FIVE_CONTEXT_NODE 7 + +#define INCREMENT_COUNT(token) \ + do { \ + if (counts) ++coef_counts[band][ctx][token]; \ + } while (0) + +#if CONFIG_NEW_MULTISYMBOL +#define READ_COEFF(prob_name, cdf_name, num, r) read_coeff(cdf_name, num, r); +static INLINE int read_coeff(const aom_cdf_prob *const *cdf, int n, + aom_reader *r) { + int val = 0; + int i = 0; + int count = 0; + while (count < n) { + const int size = AOMMIN(n - count, 4); + val |= aom_read_cdf(r, cdf[i++], 1 << size, ACCT_STR) << count; + count += size; + } + return val; +} +#else +#define READ_COEFF(prob_name, cdf_name, num, r) read_coeff(prob_name, num, r); +static INLINE int read_coeff(const aom_prob *probs, int n, aom_reader *r) { + int i, val = 0; + for (i = 0; i < n; ++i) val = (val << 1) | aom_read(r, probs[i], ACCT_STR); + return val; +} + +#endif + +static int token_to_value(aom_reader *const r, int token, TX_SIZE tx_size, + int bit_depth) { +#if !CONFIG_HIGHBITDEPTH + assert(bit_depth == 8); +#endif // !CONFIG_HIGHBITDEPTH + + switch (token) { + case ZERO_TOKEN: + case ONE_TOKEN: + case TWO_TOKEN: + case THREE_TOKEN: + case FOUR_TOKEN: return token; + case CATEGORY1_TOKEN: + return CAT1_MIN_VAL + READ_COEFF(av1_cat1_prob, av1_cat1_cdf, 1, r); + case CATEGORY2_TOKEN: + return CAT2_MIN_VAL + READ_COEFF(av1_cat2_prob, av1_cat2_cdf, 2, r); + case CATEGORY3_TOKEN: + return CAT3_MIN_VAL + READ_COEFF(av1_cat3_prob, av1_cat3_cdf, 3, r); + case CATEGORY4_TOKEN: + return CAT4_MIN_VAL + READ_COEFF(av1_cat4_prob, av1_cat4_cdf, 4, r); + case CATEGORY5_TOKEN: + return CAT5_MIN_VAL + READ_COEFF(av1_cat5_prob, av1_cat5_cdf, 5, r); + case CATEGORY6_TOKEN: { + const int skip_bits = (int)sizeof(av1_cat6_prob) - + av1_get_cat6_extrabits_size(tx_size, bit_depth); + return CAT6_MIN_VAL + READ_COEFF(av1_cat6_prob + skip_bits, av1_cat6_cdf, + 18 - skip_bits, r); + } + default: + assert(0); // Invalid token. + return -1; + } +} + +static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff, + TX_SIZE tx_size, TX_TYPE tx_type, const int16_t *dq, +#if CONFIG_NEW_QUANT + dequant_val_type_nuq *dq_val, +#endif // CONFIG_NEW_QUANT +#if CONFIG_AOM_QM + const qm_val_t *iqm[2][TX_SIZES], +#endif // CONFIG_AOM_QM + int ctx, const int16_t *scan, const int16_t *nb, + int16_t *max_scan_line, aom_reader *r) { + FRAME_COUNTS *counts = xd->counts; +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#else + FRAME_CONTEXT *const ec_ctx = xd->fc; +#endif + const int max_eob = tx_size_2d[tx_size]; + const int ref = is_inter_block(&xd->mi[0]->mbmi); +#if CONFIG_AOM_QM + const qm_val_t *iqmatrix = iqm[!ref][tx_size]; +#endif // CONFIG_AOM_QM + int band, c = 0; + const int tx_size_ctx = txsize_sqr_map[tx_size]; +#if CONFIG_NEW_TOKENSET + aom_cdf_prob(*coef_head_cdfs)[COEFF_CONTEXTS][CDF_SIZE(ENTROPY_TOKENS)] = + ec_ctx->coef_head_cdfs[tx_size_ctx][type][ref]; + aom_cdf_prob(*coef_tail_cdfs)[COEFF_CONTEXTS][CDF_SIZE(ENTROPY_TOKENS)] = + ec_ctx->coef_tail_cdfs[tx_size_ctx][type][ref]; + int val = 0; + +#if !CONFIG_EC_ADAPT + unsigned int *blockz_count; + unsigned int(*coef_counts)[COEFF_CONTEXTS][UNCONSTRAINED_NODES + 1] = NULL; + unsigned int(*eob_branch_count)[COEFF_CONTEXTS] = NULL; +#endif +#else + aom_prob(*coef_probs)[COEFF_CONTEXTS][UNCONSTRAINED_NODES] = + ec_ctx->coef_probs[tx_size_ctx][type][ref]; + const aom_prob *prob; +#if CONFIG_EC_MULTISYMBOL + aom_cdf_prob(*coef_cdfs)[COEFF_CONTEXTS][CDF_SIZE(ENTROPY_TOKENS)] = + ec_ctx->coef_cdfs[tx_size_ctx][type][ref]; + aom_cdf_prob(*cdf)[CDF_SIZE(ENTROPY_TOKENS)]; +#endif // CONFIG_EC_MULTISYMBOL + unsigned int(*coef_counts)[COEFF_CONTEXTS][UNCONSTRAINED_NODES + 1] = NULL; + unsigned int(*eob_branch_count)[COEFF_CONTEXTS] = NULL; +#endif // CONFIG_NEW_TOKENSET + uint8_t token_cache[MAX_TX_SQUARE]; + const uint8_t *band_translate = get_band_translate(tx_size); + int dq_shift; + int v, token; + int16_t dqv = dq[0]; +#if CONFIG_NEW_QUANT + const tran_low_t *dqv_val = &dq_val[0][0]; +#endif // CONFIG_NEW_QUANT + (void)tx_type; +#if CONFIG_AOM_QM + (void)iqmatrix; +#endif // CONFIG_AOM_QM + + if (counts) { +#if !CONFIG_NEW_TOKENSET || !CONFIG_EC_ADAPT + coef_counts = counts->coef[tx_size_ctx][type][ref]; + eob_branch_count = counts->eob_branch[tx_size_ctx][type][ref]; +#endif +#if CONFIG_NEW_TOKENSET && !CONFIG_EC_ADAPT + blockz_count = counts->blockz_count[tx_size_ctx][type][ref][ctx]; +#endif + } + + dq_shift = av1_get_tx_scale(tx_size); + +#if CONFIG_NEW_TOKENSET + band = *band_translate++; + + int more_data = 1; + while (more_data) { + int comb_token; + int last_pos = (c + 1 == max_eob); + int first_pos = (c == 0); + +#if CONFIG_NEW_QUANT + dqv_val = &dq_val[band][0]; +#endif // CONFIG_NEW_QUANT + + comb_token = last_pos ? 2 * aom_read_bit(r, ACCT_STR) + 2 + : aom_read_symbol(r, coef_head_cdfs[band][ctx], + HEAD_TOKENS + first_pos, ACCT_STR) + + !first_pos; + if (first_pos) { +#if !CONFIG_EC_ADAPT + if (counts) ++blockz_count[comb_token != 0]; +#endif + if (comb_token == 0) return 0; + } + token = comb_token >> 1; + + while (!token) { + *max_scan_line = AOMMAX(*max_scan_line, scan[c]); + token_cache[scan[c]] = 0; +#if !CONFIG_EC_ADAPT + if (counts && !last_pos) { + ++coef_counts[band][ctx][ZERO_TOKEN]; + } +#endif + ++c; + dqv = dq[1]; + ctx = get_coef_context(nb, token_cache, c); + band = *band_translate++; + + last_pos = (c + 1 == max_eob); + + comb_token = last_pos ? 2 * aom_read_bit(r, ACCT_STR) + 2 + : aom_read_symbol(r, coef_head_cdfs[band][ctx], + HEAD_TOKENS, ACCT_STR) + + 1; + token = comb_token >> 1; + } + + more_data = comb_token & 1; +#if !CONFIG_EC_ADAPT + if (counts && !last_pos) { + ++coef_counts[band][ctx][token]; + ++eob_branch_count[band][ctx]; + if (!more_data) ++coef_counts[band][ctx][EOB_MODEL_TOKEN]; + } +#endif + + if (token > ONE_TOKEN) + token += + aom_read_symbol(r, coef_tail_cdfs[band][ctx], TAIL_TOKENS, ACCT_STR); +#if CONFIG_NEW_QUANT + dqv_val = &dq_val[band][0]; +#endif // CONFIG_NEW_QUANT + + *max_scan_line = AOMMAX(*max_scan_line, scan[c]); + token_cache[scan[c]] = av1_pt_energy_class[token]; + + val = token_to_value(r, token, tx_size, +#if CONFIG_HIGHBITDEPTH + xd->bd); +#else + 8); +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_NEW_QUANT + v = av1_dequant_abscoeff_nuq(val, dqv, dqv_val); + v = dq_shift ? ROUND_POWER_OF_TWO(v, dq_shift) : v; +#else +#if CONFIG_AOM_QM + dqv = ((iqmatrix[scan[c]] * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >> + AOM_QM_BITS; +#endif + v = (val * dqv) >> dq_shift; +#endif + + v = aom_read_bit(r, ACCT_STR) ? -v : v; +#if CONFIG_COEFFICIENT_RANGE_CHECKING +#if CONFIG_HIGHBITDEPTH + check_range(v, xd->bd); +#else + check_range(v, 8); +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_COEFFICIENT_RANGE_CHECKING + + dqcoeff[scan[c]] = v; + + ++c; + more_data &= (c < max_eob); + if (!more_data) break; + dqv = dq[1]; + ctx = get_coef_context(nb, token_cache, c); + band = *band_translate++; + +#else // CONFIG_NEW_TOKENSET + while (c < max_eob) { + int val = -1; + band = *band_translate++; + prob = coef_probs[band][ctx]; + if (counts) ++eob_branch_count[band][ctx]; + if (!aom_read(r, prob[EOB_CONTEXT_NODE], ACCT_STR)) { + INCREMENT_COUNT(EOB_MODEL_TOKEN); + break; + } + +#if CONFIG_NEW_QUANT + dqv_val = &dq_val[band][0]; +#endif // CONFIG_NEW_QUANT + + while (!aom_read(r, prob[ZERO_CONTEXT_NODE], ACCT_STR)) { + INCREMENT_COUNT(ZERO_TOKEN); + dqv = dq[1]; + token_cache[scan[c]] = 0; + ++c; + if (c >= max_eob) return c; // zero tokens at the end (no eob token) + ctx = get_coef_context(nb, token_cache, c); + band = *band_translate++; + prob = coef_probs[band][ctx]; +#if CONFIG_NEW_QUANT + dqv_val = &dq_val[band][0]; +#endif // CONFIG_NEW_QUANT + } + + *max_scan_line = AOMMAX(*max_scan_line, scan[c]); + +#if CONFIG_EC_MULTISYMBOL + cdf = &coef_cdfs[band][ctx]; + token = ONE_TOKEN + + aom_read_symbol(r, *cdf, CATEGORY6_TOKEN - ONE_TOKEN + 1, ACCT_STR); + INCREMENT_COUNT(ONE_TOKEN + (token > ONE_TOKEN)); + assert(token != ZERO_TOKEN); + val = token_to_value(r, token, tx_size, +#if CONFIG_HIGHBITDEPTH + xd->bd); +#else + 8); +#endif // CONFIG_HIGHBITDEPTH +#else // CONFIG_EC_MULTISYMBOL + if (!aom_read(r, prob[ONE_CONTEXT_NODE], ACCT_STR)) { + INCREMENT_COUNT(ONE_TOKEN); + token = ONE_TOKEN; + val = 1; + } else { + INCREMENT_COUNT(TWO_TOKEN); + token = aom_read_tree(r, av1_coef_con_tree, + av1_pareto8_full[prob[PIVOT_NODE] - 1], ACCT_STR); + assert(token != ZERO_TOKEN && token != ONE_TOKEN); + val = token_to_value(r, token, tx_size, +#if CONFIG_HIGHBITDEPTH + xd->bd); +#else + 8); +#endif // CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_EC_MULTISYMBOL +#if CONFIG_NEW_QUANT + v = av1_dequant_abscoeff_nuq(val, dqv, dqv_val); + v = dq_shift ? ROUND_POWER_OF_TWO(v, dq_shift) : v; +#else +#if CONFIG_AOM_QM + dqv = ((iqmatrix[scan[c]] * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >> + AOM_QM_BITS; +#endif + v = (val * dqv) >> dq_shift; +#endif // CONFIG_NEW_QUANT + +#if CONFIG_COEFFICIENT_RANGE_CHECKING +#if CONFIG_HIGHBITDEPTH + dqcoeff[scan[c]] = + highbd_check_range((aom_read_bit(r, ACCT_STR) ? -v : v), xd->bd); +#else + dqcoeff[scan[c]] = check_range(aom_read_bit(r, ACCT_STR) ? -v : v, 8); +#endif // CONFIG_HIGHBITDEPTH +#else + dqcoeff[scan[c]] = aom_read_bit(r, ACCT_STR) ? -v : v; +#endif // CONFIG_COEFFICIENT_RANGE_CHECKING + token_cache[scan[c]] = av1_pt_energy_class[token]; + ++c; + ctx = get_coef_context(nb, token_cache, c); + dqv = dq[1]; +#endif // CONFIG_NEW_TOKENSET + } + + return c; +} +#endif // !CONFIG_PVQ + +#if CONFIG_PALETTE +void av1_decode_palette_tokens(MACROBLOCKD *const xd, int plane, + aom_reader *r) { + const MODE_INFO *const mi = xd->mi[0]; + const MB_MODE_INFO *const mbmi = &mi->mbmi; + uint8_t color_order[PALETTE_MAX_SIZE]; + const int n = mbmi->palette_mode_info.palette_size[plane]; + int i, j; + uint8_t *const color_map = xd->plane[plane].color_index_map; + const aom_prob( + *const prob)[PALETTE_COLOR_INDEX_CONTEXTS][PALETTE_COLORS - 1] = + plane ? av1_default_palette_uv_color_index_prob + : av1_default_palette_y_color_index_prob; + int plane_block_width, plane_block_height, rows, cols; + av1_get_block_dimensions(mbmi->sb_type, plane, xd, &plane_block_width, + &plane_block_height, &rows, &cols); + assert(plane == 0 || plane == 1); + +#if CONFIG_PALETTE_THROUGHPUT + // Run wavefront on the palette map index decoding. + for (i = 1; i < rows + cols - 1; ++i) { + for (j = AOMMIN(i, cols - 1); j >= AOMMAX(0, i - rows + 1); --j) { + const int color_ctx = av1_get_palette_color_index_context( + color_map, plane_block_width, (i - j), j, n, color_order, NULL); + const int color_idx = + aom_read_tree(r, av1_palette_color_index_tree[n - 2], + prob[n - 2][color_ctx], ACCT_STR); + assert(color_idx >= 0 && color_idx < n); + color_map[(i - j) * plane_block_width + j] = color_order[color_idx]; + } + } + // Copy last column to extra columns. + if (cols < plane_block_width) { + for (i = 0; i < plane_block_height; ++i) { + memset(color_map + i * plane_block_width + cols, + color_map[i * plane_block_width + cols - 1], + (plane_block_width - cols)); + } + } +#else + for (i = 0; i < rows; ++i) { + for (j = (i == 0 ? 1 : 0); j < cols; ++j) { + const int color_ctx = av1_get_palette_color_index_context( + color_map, plane_block_width, i, j, n, color_order, NULL); + const int color_idx = + aom_read_tree(r, av1_palette_color_index_tree[n - PALETTE_MIN_SIZE], + prob[n - PALETTE_MIN_SIZE][color_ctx], ACCT_STR); + assert(color_idx >= 0 && color_idx < n); + color_map[i * plane_block_width + j] = color_order[color_idx]; + } + memset(color_map + i * plane_block_width + cols, + color_map[i * plane_block_width + cols - 1], + (plane_block_width - cols)); // Copy last column to extra columns. + } +#endif // CONFIG_PALETTE_THROUGHPUT + // Copy last row to extra rows. + for (i = rows; i < plane_block_height; ++i) { + memcpy(color_map + i * plane_block_width, + color_map + (rows - 1) * plane_block_width, plane_block_width); + } +} +#endif // CONFIG_PALETTE + +#if !CONFIG_PVQ || CONFIG_VAR_TX +int av1_decode_block_tokens(AV1_COMMON *cm, MACROBLOCKD *const xd, int plane, + const SCAN_ORDER *sc, int x, int y, TX_SIZE tx_size, + TX_TYPE tx_type, int16_t *max_scan_line, + aom_reader *r, int seg_id) { + struct macroblockd_plane *const pd = &xd->plane[plane]; + const int16_t *const dequant = pd->seg_dequant[seg_id]; + const int ctx = + get_entropy_context(tx_size, pd->above_context + x, pd->left_context + y); +#if CONFIG_NEW_QUANT + const int ref = is_inter_block(&xd->mi[0]->mbmi); + int dq = + get_dq_profile_from_ctx(xd->qindex[seg_id], ctx, ref, pd->plane_type); +#endif // CONFIG_NEW_QUANT + + const int eob = + decode_coefs(xd, pd->plane_type, pd->dqcoeff, tx_size, tx_type, dequant, +#if CONFIG_NEW_QUANT + pd->seg_dequant_nuq[seg_id][dq], +#endif // CONFIG_NEW_QUANT +#if CONFIG_AOM_QM + pd->seg_iqmatrix[seg_id], +#endif // CONFIG_AOM_QM + ctx, sc->scan, sc->neighbors, max_scan_line, r); + av1_set_contexts(xd, pd, plane, tx_size, eob > 0, x, y); +#if CONFIG_ADAPT_SCAN + if (xd->counts) + av1_update_scan_count_facade(cm, xd->counts, tx_size, tx_type, pd->dqcoeff, + eob); +#else + (void)cm; +#endif + return eob; +} +#endif // !CONFIG_PVQ diff --git a/third_party/aom/av1/decoder/detokenize.h b/third_party/aom/av1/decoder/detokenize.h new file mode 100644 index 0000000000..ba40666037 --- /dev/null +++ b/third_party/aom/av1/decoder/detokenize.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_DECODER_DETOKENIZE_H_ +#define AV1_DECODER_DETOKENIZE_H_ + +#include "./aom_config.h" +#if !CONFIG_PVQ || CONFIG_VAR_TX +#include "av1/decoder/decoder.h" +#include "av1/common/scan.h" +#endif // !CONFIG_PVQ + +#ifdef __cplusplus +extern "C" { +#endif + +#if CONFIG_PALETTE +void av1_decode_palette_tokens(MACROBLOCKD *const xd, int plane, aom_reader *r); +#endif // CONFIG_PALETTE + +#if !CONFIG_PVQ || CONFIG_VAR_TX +int av1_decode_block_tokens(AV1_COMMON *cm, MACROBLOCKD *const xd, int plane, + const SCAN_ORDER *sc, int x, int y, TX_SIZE tx_size, + TX_TYPE tx_type, int16_t *max_scan_line, + aom_reader *r, int seg_id); +#endif // !CONFIG_PVQ +#ifdef __cplusplus +} // extern "C" +#endif +#endif // AV1_DECODER_DETOKENIZE_H_ diff --git a/third_party/aom/av1/decoder/dsubexp.c b/third_party/aom/av1/decoder/dsubexp.c new file mode 100644 index 0000000000..5171f1144e --- /dev/null +++ b/third_party/aom/av1/decoder/dsubexp.c @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/entropy.h" + +#include "av1/decoder/dsubexp.h" + +static int inv_recenter_nonneg(int v, int m) { + if (v > 2 * m) return v; + + return (v & 1) ? m - ((v + 1) >> 1) : m + (v >> 1); +} + +#define decode_uniform(r, ACCT_STR_NAME) \ + decode_uniform_(r ACCT_STR_ARG(ACCT_STR_NAME)) +#define decode_term_subexp(r, ACCT_STR_NAME) \ + decode_term_subexp_(r ACCT_STR_ARG(ACCT_STR_NAME)) + +static int decode_uniform_(aom_reader *r ACCT_STR_PARAM) { + const int l = 8; + const int m = (1 << l) - 190; + const int v = aom_read_literal(r, l - 1, ACCT_STR_NAME); + return v < m ? v : (v << 1) - m + aom_read_bit(r, ACCT_STR_NAME); +} + +static int inv_remap_prob(int v, int m) { + /* clang-format off */ + static uint8_t inv_map_table[MAX_PROB - 1] = { + 7, 20, 33, 46, 59, 72, 85, 98, 111, 124, 137, 150, 163, 176, 189, + 202, 215, 228, 241, 254, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, + 44, 45, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 60, + 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 73, 74, 75, 76, + 77, 78, 79, 80, 81, 82, 83, 84, 86, 87, 88, 89, 90, 91, 92, + 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 125, + 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 138, 139, 140, 141, + 142, 143, 144, 145, 146, 147, 148, 149, 151, 152, 153, 154, 155, 156, 157, + 158, 159, 160, 161, 162, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, + 174, 175, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 190, + 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 203, 204, 205, 206, + 207, 208, 209, 210, 211, 212, 213, 214, 216, 217, 218, 219, 220, 221, 222, + 223, 224, 225, 226, 227, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, + 239, 240, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253 + }; /* clang-format on */ + assert(v < (int)(sizeof(inv_map_table) / sizeof(inv_map_table[0]))); + v = inv_map_table[v]; + m--; + if ((m << 1) <= MAX_PROB) { + return 1 + inv_recenter_nonneg(v, m); + } else { + return MAX_PROB - inv_recenter_nonneg(v, MAX_PROB - 1 - m); + } +} + +static int decode_term_subexp_(aom_reader *r ACCT_STR_PARAM) { + if (!aom_read_bit(r, ACCT_STR_NAME)) + return aom_read_literal(r, 4, ACCT_STR_NAME); + if (!aom_read_bit(r, ACCT_STR_NAME)) + return aom_read_literal(r, 4, ACCT_STR_NAME) + 16; + if (!aom_read_bit(r, ACCT_STR_NAME)) + return aom_read_literal(r, 5, ACCT_STR_NAME) + 32; + return decode_uniform(r, ACCT_STR_NAME) + 64; +} + +void av1_diff_update_prob_(aom_reader *r, aom_prob *p ACCT_STR_PARAM) { + if (aom_read(r, DIFF_UPDATE_PROB, ACCT_STR_NAME)) { + const int delp = decode_term_subexp(r, ACCT_STR_NAME); + *p = (aom_prob)inv_remap_prob(delp, *p); + } +} diff --git a/third_party/aom/av1/decoder/dsubexp.h b/third_party/aom/av1/decoder/dsubexp.h new file mode 100644 index 0000000000..4bc38578c9 --- /dev/null +++ b/third_party/aom/av1/decoder/dsubexp.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_DECODER_DSUBEXP_H_ +#define AV1_DECODER_DSUBEXP_H_ + +#include "aom_dsp/bitreader.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if CONFIG_ACCOUNTING +#define av1_diff_update_prob(r, p, str) av1_diff_update_prob_(r, p, str) +#else +#define av1_diff_update_prob(r, p, str) av1_diff_update_prob_(r, p) +#endif + +void av1_diff_update_prob_(aom_reader *r, aom_prob *p ACCT_STR_PARAM); + +#ifdef __cplusplus +} // extern "C" +#endif +#endif // AV1_DECODER_DSUBEXP_H_ diff --git a/third_party/aom/av1/decoder/dthread.c b/third_party/aom/av1/decoder/dthread.c new file mode 100644 index 0000000000..50f8ed192c --- /dev/null +++ b/third_party/aom/av1/decoder/dthread.c @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./aom_config.h" +#include "aom_mem/aom_mem.h" +#include "av1/common/reconinter.h" +#include "av1/decoder/dthread.h" +#include "av1/decoder/decoder.h" + +// #define DEBUG_THREAD + +// TODO(hkuang): Clean up all the #ifdef in this file. +void av1_frameworker_lock_stats(AVxWorker *const worker) { +#if CONFIG_MULTITHREAD + FrameWorkerData *const worker_data = worker->data1; + pthread_mutex_lock(&worker_data->stats_mutex); +#else + (void)worker; +#endif +} + +void av1_frameworker_unlock_stats(AVxWorker *const worker) { +#if CONFIG_MULTITHREAD + FrameWorkerData *const worker_data = worker->data1; + pthread_mutex_unlock(&worker_data->stats_mutex); +#else + (void)worker; +#endif +} + +void av1_frameworker_signal_stats(AVxWorker *const worker) { +#if CONFIG_MULTITHREAD + FrameWorkerData *const worker_data = worker->data1; + +// TODO(hkuang): Fix the pthread_cond_broadcast in windows wrapper. +#if defined(_WIN32) && !HAVE_PTHREAD_H + pthread_cond_signal(&worker_data->stats_cond); +#else + pthread_cond_broadcast(&worker_data->stats_cond); +#endif + +#else + (void)worker; +#endif +} + +// This macro prevents thread_sanitizer from reporting known concurrent writes. +#if defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define BUILDING_WITH_TSAN +#endif +#endif + +// TODO(hkuang): Remove worker parameter as it is only used in debug code. +void av1_frameworker_wait(AVxWorker *const worker, RefCntBuffer *const ref_buf, + int row) { +#if CONFIG_MULTITHREAD + if (!ref_buf) return; + +#ifndef BUILDING_WITH_TSAN + // The following line of code will get harmless tsan error but it is the key + // to get best performance. + if (ref_buf->row >= row && ref_buf->buf.corrupted != 1) return; +#endif + + { + // Find the worker thread that owns the reference frame. If the reference + // frame has been fully decoded, it may not have owner. + AVxWorker *const ref_worker = ref_buf->frame_worker_owner; + FrameWorkerData *const ref_worker_data = + (FrameWorkerData *)ref_worker->data1; + const AV1Decoder *const pbi = ref_worker_data->pbi; + +#ifdef DEBUG_THREAD + { + FrameWorkerData *const worker_data = (FrameWorkerData *)worker->data1; + printf("%d %p worker is waiting for %d %p worker (%d) ref %d \r\n", + worker_data->worker_id, worker, ref_worker_data->worker_id, + ref_buf->frame_worker_owner, row, ref_buf->row); + } +#endif + + av1_frameworker_lock_stats(ref_worker); + while (ref_buf->row < row && pbi->cur_buf == ref_buf && + ref_buf->buf.corrupted != 1) { + pthread_cond_wait(&ref_worker_data->stats_cond, + &ref_worker_data->stats_mutex); + } + + if (ref_buf->buf.corrupted == 1) { + FrameWorkerData *const worker_data = (FrameWorkerData *)worker->data1; + av1_frameworker_unlock_stats(ref_worker); + aom_internal_error(&worker_data->pbi->common.error, + AOM_CODEC_CORRUPT_FRAME, + "Worker %p failed to decode frame", worker); + } + av1_frameworker_unlock_stats(ref_worker); + } +#else + (void)worker; + (void)ref_buf; + (void)row; + (void)ref_buf; +#endif // CONFIG_MULTITHREAD +} + +void av1_frameworker_broadcast(RefCntBuffer *const buf, int row) { +#if CONFIG_MULTITHREAD + AVxWorker *worker = buf->frame_worker_owner; + +#ifdef DEBUG_THREAD + { + FrameWorkerData *const worker_data = (FrameWorkerData *)worker->data1; + printf("%d %p worker decode to (%d) \r\n", worker_data->worker_id, + buf->frame_worker_owner, row); + } +#endif + + av1_frameworker_lock_stats(worker); + buf->row = row; + av1_frameworker_signal_stats(worker); + av1_frameworker_unlock_stats(worker); +#else + (void)buf; + (void)row; +#endif // CONFIG_MULTITHREAD +} + +void av1_frameworker_copy_context(AVxWorker *const dst_worker, + AVxWorker *const src_worker) { +#if CONFIG_MULTITHREAD + FrameWorkerData *const src_worker_data = (FrameWorkerData *)src_worker->data1; + FrameWorkerData *const dst_worker_data = (FrameWorkerData *)dst_worker->data1; + AV1_COMMON *const src_cm = &src_worker_data->pbi->common; + AV1_COMMON *const dst_cm = &dst_worker_data->pbi->common; + int i; + + // Wait until source frame's context is ready. + av1_frameworker_lock_stats(src_worker); + while (!src_worker_data->frame_context_ready) { + pthread_cond_wait(&src_worker_data->stats_cond, + &src_worker_data->stats_mutex); + } + + dst_cm->last_frame_seg_map = src_cm->seg.enabled + ? src_cm->current_frame_seg_map + : src_cm->last_frame_seg_map; + dst_worker_data->pbi->need_resync = src_worker_data->pbi->need_resync; + av1_frameworker_unlock_stats(src_worker); + + dst_cm->bit_depth = src_cm->bit_depth; +#if CONFIG_HIGHBITDEPTH + dst_cm->use_highbitdepth = src_cm->use_highbitdepth; +#endif +#if CONFIG_EXT_REFS +// TODO(zoeliu): To handle parallel decoding +#endif // CONFIG_EXT_REFS + dst_cm->prev_frame = + src_cm->show_existing_frame ? src_cm->prev_frame : src_cm->cur_frame; + dst_cm->last_width = + !src_cm->show_existing_frame ? src_cm->width : src_cm->last_width; + dst_cm->last_height = + !src_cm->show_existing_frame ? src_cm->height : src_cm->last_height; + dst_cm->subsampling_x = src_cm->subsampling_x; + dst_cm->subsampling_y = src_cm->subsampling_y; + dst_cm->frame_type = src_cm->frame_type; + dst_cm->last_show_frame = !src_cm->show_existing_frame + ? src_cm->show_frame + : src_cm->last_show_frame; + for (i = 0; i < REF_FRAMES; ++i) + dst_cm->ref_frame_map[i] = src_cm->next_ref_frame_map[i]; + + memcpy(dst_cm->lf_info.lfthr, src_cm->lf_info.lfthr, + (MAX_LOOP_FILTER + 1) * sizeof(loop_filter_thresh)); + dst_cm->lf.last_sharpness_level = src_cm->lf.sharpness_level; + dst_cm->lf.filter_level = src_cm->lf.filter_level; + memcpy(dst_cm->lf.ref_deltas, src_cm->lf.ref_deltas, TOTAL_REFS_PER_FRAME); + memcpy(dst_cm->lf.mode_deltas, src_cm->lf.mode_deltas, MAX_MODE_LF_DELTAS); + dst_cm->seg = src_cm->seg; + memcpy(dst_cm->frame_contexts, src_cm->frame_contexts, + FRAME_CONTEXTS * sizeof(dst_cm->frame_contexts[0])); +#else + (void)dst_worker; + (void)src_worker; +#endif // CONFIG_MULTITHREAD +} diff --git a/third_party/aom/av1/decoder/dthread.h b/third_party/aom/av1/decoder/dthread.h new file mode 100644 index 0000000000..c17053d9c2 --- /dev/null +++ b/third_party/aom/av1/decoder/dthread.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_DECODER_DTHREAD_H_ +#define AV1_DECODER_DTHREAD_H_ + +#include "./aom_config.h" +#include "aom_util/aom_thread.h" +#include "aom/internal/aom_codec_internal.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct AV1Common; +struct AV1Decoder; + +// WorkerData for the FrameWorker thread. It contains all the information of +// the worker and decode structures for decoding a frame. +typedef struct FrameWorkerData { + struct AV1Decoder *pbi; + const uint8_t *data; + const uint8_t *data_end; + size_t data_size; + void *user_priv; + int result; + int worker_id; + int received_frame; + + // scratch_buffer is used in frame parallel mode only. + // It is used to make a copy of the compressed data. + uint8_t *scratch_buffer; + size_t scratch_buffer_size; + +#if CONFIG_MULTITHREAD + pthread_mutex_t stats_mutex; + pthread_cond_t stats_cond; +#endif + + int frame_context_ready; // Current frame's context is ready to read. + int frame_decoded; // Finished decoding current frame. +} FrameWorkerData; + +void av1_frameworker_lock_stats(AVxWorker *const worker); +void av1_frameworker_unlock_stats(AVxWorker *const worker); +void av1_frameworker_signal_stats(AVxWorker *const worker); + +// Wait until ref_buf has been decoded to row in real pixel unit. +// Note: worker may already finish decoding ref_buf and release it in order to +// start decoding next frame. So need to check whether worker is still decoding +// ref_buf. +void av1_frameworker_wait(AVxWorker *const worker, RefCntBuffer *const ref_buf, + int row); + +// FrameWorker broadcasts its decoding progress so other workers that are +// waiting on it can resume decoding. +void av1_frameworker_broadcast(RefCntBuffer *const buf, int row); + +// Copy necessary decoding context from src worker to dst worker. +void av1_frameworker_copy_context(AVxWorker *const dst_worker, + AVxWorker *const src_worker); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_DECODER_DTHREAD_H_ diff --git a/third_party/aom/av1/decoder/generic_decoder.c b/third_party/aom/av1/decoder/generic_decoder.c new file mode 100644 index 0000000000..0c7d71b9fc --- /dev/null +++ b/third_party/aom/av1/decoder/generic_decoder.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include + +#include "aom_dsp/bitreader.h" +#include "av1/common/generic_code.h" +#include "av1/common/odintrin.h" +#include "pvq_decoder.h" + +/** Decodes a value from 0 to N-1 (with N up to 16) based on a cdf and adapts + * the cdf accordingly. + * + * @param [in,out] r multi-symbol entropy decoder + * @param [in,out] cdf CDF of the variable (Q15) + * @param [in] n number of values possible + * @param [in,out] count number of symbols encoded with that cdf so far + * @param [in] rate adaptation rate shift (smaller is faster) + * @return decoded variable + */ +int aom_decode_cdf_adapt_q15_(aom_reader *r, uint16_t *cdf, int n, + int *count, int rate ACCT_STR_PARAM) { + int val; + int i; + if (*count == 0) { + int ft; + ft = cdf[n - 1]; + for (i = 0; i < n; i++) { + cdf[i] = AOM_ICDF(cdf[i]*32768/ft); + } + } + val = aom_read_cdf(r, cdf, n, ACCT_STR_NAME); + aom_cdf_adapt_q15(val, cdf, n, count, rate); + return val; +} + +/** Encodes a random variable using a "generic" model, assuming that the + * distribution is one-sided (zero and up), has a single mode, and decays + * exponentially past the model. + * + * @param [in,out] r multi-symbol entropy decoder + * @param [in,out] model generic probability model + * @param [in] x variable being encoded + * @param [in,out] ExQ16 expectation of x (adapted) + * @param [in] integration integration period of ExQ16 (leaky average over + * 1<> 1); + /* Choose the cdf to use: we have two per "octave" of ExQ16. */ + id = OD_MINI(GENERIC_TABLES - 1, lg_q1); + cdf = model->cdf[id]; + xs = aom_read_symbol_pvq(r, cdf, 16, ACCT_STR_NAME); + if (xs == 15) { + int e; + unsigned decay; + /* Estimate decay based on the assumption that the distribution is close + to Laplacian for large values. We should probably have an adaptive + estimate instead. Note: The 2* is a kludge that's not fully understood + yet. */ + OD_ASSERT(*ex_q16 < INT_MAX >> 1); + e = ((2**ex_q16 >> 8) + (1 << shift >> 1)) >> shift; + decay = OD_MAXI(2, OD_MINI(254, 256*e/(e + 256))); + xs += aom_laplace_decode_special(r, decay, ACCT_STR_NAME); + } + if (shift != 0) { + int special; + /* Because of the rounding, there's only half the number of possibilities + for xs=0 */ + special = xs == 0; + if (shift - special > 0) { + lsb = aom_read_literal(r, shift - special, ACCT_STR_NAME); + } + lsb -= !special << (shift - 1); + } + x = (xs << shift) + lsb; + generic_model_update(ex_q16, x, integration); + OD_LOG((OD_LOG_ENTROPY_CODER, OD_LOG_DEBUG, + "dec: %d %d %d %d %d %x", *ex_q16, x, shift, id, xs, dec->rng)); + return x; +} diff --git a/third_party/aom/av1/decoder/inspection.c b/third_party/aom/av1/decoder/inspection.c new file mode 100644 index 0000000000..2e8a610872 --- /dev/null +++ b/third_party/aom/av1/decoder/inspection.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include "av1/decoder/decoder.h" +#include "av1/decoder/inspection.h" +#include "av1/common/enums.h" +#if CONFIG_CDEF +#include "av1/common/cdef.h" +#endif + +void ifd_init(insp_frame_data *fd, int frame_width, int frame_height) { + fd->mi_cols = ALIGN_POWER_OF_TWO(frame_width, 3) >> MI_SIZE_LOG2; + fd->mi_rows = ALIGN_POWER_OF_TWO(frame_height, 3) >> MI_SIZE_LOG2; + fd->mi_grid = (insp_mi_data *)aom_malloc(sizeof(insp_mi_data) * fd->mi_rows * + fd->mi_cols); +} + +void ifd_clear(insp_frame_data *fd) { + aom_free(fd->mi_grid); + fd->mi_grid = NULL; +} + +/* TODO(negge) This function may be called by more than one thread when using + a multi-threaded decoder and this may cause a data race. */ +int ifd_inspect(insp_frame_data *fd, void *decoder) { + struct AV1Decoder *pbi = (struct AV1Decoder *)decoder; + AV1_COMMON *const cm = &pbi->common; + // TODO(negge): Should this function just call ifd_clear() and ifd_init()? + if (fd->mi_rows != cm->mi_rows || fd->mi_cols != cm->mi_cols) { + return 0; + } + fd->show_frame = cm->show_frame; + fd->frame_type = cm->frame_type; + fd->base_qindex = cm->base_qindex; + fd->tile_mi_cols = cm->tile_width; + fd->tile_mi_rows = cm->tile_height; +#if CONFIG_ACCOUNTING + fd->accounting = &pbi->accounting; +#endif +#if CONFIG_CDEF +// TODO(negge): copy per frame CDEF data +#endif + int i, j; + for (i = 0; i < MAX_SEGMENTS; i++) { + for (j = 0; j < 2; j++) { + fd->y_dequant[i][j] = cm->y_dequant[i][j]; + fd->uv_dequant[i][j] = cm->uv_dequant[i][j]; + } + } + for (j = 0; j < cm->mi_rows; j++) { + for (i = 0; i < cm->mi_cols; i++) { + const MB_MODE_INFO *mbmi = + &cm->mi_grid_visible[j * cm->mi_stride + i]->mbmi; + insp_mi_data *mi = &fd->mi_grid[j * cm->mi_cols + i]; + // Segment + mi->segment_id = mbmi->segment_id; + // Motion Vectors + mi->mv[0].row = mbmi->mv[0].as_mv.row; + mi->mv[0].col = mbmi->mv[0].as_mv.col; + mi->mv[1].row = mbmi->mv[1].as_mv.row; + mi->mv[1].col = mbmi->mv[1].as_mv.col; + // Reference Frames + mi->ref_frame[0] = mbmi->ref_frame[0]; + mi->ref_frame[1] = mbmi->ref_frame[1]; + // Prediction Mode + mi->mode = mbmi->mode; + // Prediction Mode for Chromatic planes + if (mi->mode < INTRA_MODES) { + mi->uv_mode = mbmi->uv_mode; + } else { + mi->uv_mode = INTRA_INVALID; + } + // Block Size + mi->sb_type = mbmi->sb_type; + // Skip Flag + mi->skip = mbmi->skip; +#if CONFIG_DUAL_FILTER + mi->filter[0] = mbmi->interp_filter[0]; + mi->filter[1] = mbmi->interp_filter[1]; +#else + mi->filter = mbmi->interp_filter; +#endif + // Transform + mi->tx_type = mbmi->tx_type; + mi->tx_size = mbmi->tx_size; + +#if CONFIG_CDEF + mi->cdef_level = cm->cdef_strengths[mbmi->cdef_strength] / CLPF_STRENGTHS; + mi->cdef_strength = + cm->cdef_strengths[mbmi->cdef_strength] % CLPF_STRENGTHS; + mi->cdef_strength += mi->cdef_strength == 3; +#endif + } + } + return 1; +} diff --git a/third_party/aom/av1/decoder/inspection.h b/third_party/aom/av1/decoder/inspection.h new file mode 100644 index 0000000000..d6cf4319a6 --- /dev/null +++ b/third_party/aom/av1/decoder/inspection.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_INSPECTION_H_ +#define AOM_INSPECTION_H_ + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +#if CONFIG_ACCOUNTING +#include "av1/decoder/accounting.h" +#endif + +typedef void (*aom_inspect_cb)(void *decoder, void *data); + +typedef struct insp_mv insp_mv; + +struct insp_mv { + int16_t row; + int16_t col; +}; + +typedef struct insp_mi_data insp_mi_data; + +struct insp_mi_data { + insp_mv mv[2]; + int8_t ref_frame[2]; + int8_t mode; + int8_t uv_mode; + int8_t sb_type; + int8_t skip; + int8_t segment_id; +#if CONFIG_DUAL_FILTER + int8_t filter[2]; +#else + int8_t filter; +#endif + int8_t tx_type; + int8_t tx_size; +#if CONFIG_CDEF + int8_t cdef_level; + int8_t cdef_strength; +#endif +}; + +typedef struct insp_frame_data insp_frame_data; + +struct insp_frame_data { +#if CONFIG_ACCOUNTING + Accounting *accounting; +#endif + insp_mi_data *mi_grid; + int show_frame; + int frame_type; + int base_qindex; + int mi_rows; + int mi_cols; + int tile_mi_rows; + int tile_mi_cols; + int16_t y_dequant[MAX_SEGMENTS][2]; + int16_t uv_dequant[MAX_SEGMENTS][2]; +#if CONFIG_CDEF +// TODO(negge): add per frame CDEF data +#endif +}; + +void ifd_init(insp_frame_data *fd, int frame_width, int frame_height); +void ifd_clear(insp_frame_data *fd); +int ifd_inspect(insp_frame_data *fd, void *decoder); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus +#endif // AOM_INSPECTION_H_ diff --git a/third_party/aom/av1/decoder/laplace_decoder.c b/third_party/aom/av1/decoder/laplace_decoder.c new file mode 100644 index 0000000000..b6cf50bc7f --- /dev/null +++ b/third_party/aom/av1/decoder/laplace_decoder.c @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +/* clang-format off */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include + +#include "aom_dsp/bitreader.h" +#include "av1/common/pvq.h" +#include "pvq_decoder.h" + +#define aom_decode_pvq_split(r, adapt, sum, ctx, ACCT_STR_NAME) \ + aom_decode_pvq_split_(r, adapt, sum, ctx ACCT_STR_ARG(ACCT_STR_NAME)) + +static int aom_decode_pvq_split_(aom_reader *r, od_pvq_codeword_ctx *adapt, + int sum, int ctx ACCT_STR_PARAM) { + int shift; + int count; + int msbs; + int fctx; + count = 0; + if (sum == 0) return 0; + shift = OD_MAXI(0, OD_ILOG(sum) - 3); + fctx = 7*ctx + (sum >> shift) - 1; + msbs = aom_read_symbol_pvq(r, adapt->pvq_split_cdf[fctx], (sum >> shift) + 1, + ACCT_STR_NAME); + if (shift) count = aom_read_literal(r, shift, ACCT_STR_NAME); + count += msbs << shift; + if (count > sum) { + count = sum; +#if CONFIG_DAALA_EC + r->ec.error = 1; +#else +# error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + } + return count; +} + +void aom_decode_band_pvq_splits(aom_reader *r, od_pvq_codeword_ctx *adapt, + od_coeff *y, int n, int k, int level) { + int mid; + int count_right; + if (n == 1) { + y[0] = k; + } + else if (k == 0) { + OD_CLEAR(y, n); + } + else if (k == 1 && n <= 16) { + int cdf_id; + int pos; + cdf_id = od_pvq_k1_ctx(n, level == 0); + OD_CLEAR(y, n); + pos = aom_read_symbol_pvq(r, adapt->pvq_k1_cdf[cdf_id], n, "pvq:k1"); + y[pos] = 1; + } + else { + mid = n >> 1; + count_right = aom_decode_pvq_split(r, adapt, k, od_pvq_size_ctx(n), + "pvq:split"); + aom_decode_band_pvq_splits(r, adapt, y, mid, k - count_right, level + 1); + aom_decode_band_pvq_splits(r, adapt, y + mid, n - mid, count_right, + level + 1); + } +} + +/** Decodes the tail of a Laplace-distributed variable, i.e. it doesn't + * do anything special for the zero case. + * + * @param [dec] range decoder + * @param [decay] decay factor of the distribution, i.e. pdf ~= decay^x + * + * @retval decoded variable x + */ +int aom_laplace_decode_special_(aom_reader *r, unsigned decay ACCT_STR_PARAM) { + int pos; + int shift; + int xs; + int sym; + const uint16_t *cdf; + shift = 0; + /* We don't want a large decay value because that would require too many + symbols. */ + while (decay > 235) { + decay = (decay*decay + 128) >> 8; + shift++; + } + decay = OD_MINI(decay, 254); + decay = OD_MAXI(decay, 2); + cdf = EXP_CDF_TABLE[(decay + 1) >> 1]; + OD_LOG((OD_LOG_PVQ, OD_LOG_DEBUG, "decay = %d\n", decay)); + xs = 0; + do { + sym = OD_MINI(xs, 15); + { + int i; + OD_LOG((OD_LOG_PVQ, OD_LOG_DEBUG, "%d %d %d", xs, shift, sym)); + for (i = 0; i < 16; i++) { + OD_LOG_PARTIAL((OD_LOG_PVQ, OD_LOG_DEBUG, "%d ", cdf[i])); + } + OD_LOG_PARTIAL((OD_LOG_PVQ, OD_LOG_DEBUG, "\n")); + } + sym = aom_read_cdf(r, cdf, 16, ACCT_STR_NAME); + xs += sym; + } while (sym >= 15); + if (shift) pos = (xs << shift) + aom_read_literal(r, shift, ACCT_STR_NAME); + else pos = xs; + return pos; +} diff --git a/third_party/aom/av1/decoder/pvq_decoder.c b/third_party/aom/av1/decoder/pvq_decoder.c new file mode 100644 index 0000000000..d9a8e80563 --- /dev/null +++ b/third_party/aom/av1/decoder/pvq_decoder.c @@ -0,0 +1,378 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include +#include "./aom_config.h" +#include "aom_dsp/bitreader.h" +#include "aom_dsp/entcode.h" +#include "aom_dsp/entdec.h" +#include "av1/common/odintrin.h" +#include "av1/common/partition.h" +#include "av1/common/pvq_state.h" +#include "av1/decoder/decint.h" +#include "av1/decoder/pvq_decoder.h" +#include "aom_ports/system_state.h" + +int aom_read_symbol_pvq_(aom_reader *r, aom_cdf_prob *cdf, int nsymbs + ACCT_STR_PARAM) { + if (cdf[0] == 0) + aom_cdf_init_q15_1D(cdf, nsymbs, CDF_SIZE(nsymbs)); + return aom_read_symbol(r, cdf, nsymbs, ACCT_STR_NAME); +} + +static void aom_decode_pvq_codeword(aom_reader *r, od_pvq_codeword_ctx *ctx, + od_coeff *y, int n, int k) { + int i; + aom_decode_band_pvq_splits(r, ctx, y, n, k, 0); + for (i = 0; i < n; i++) { + if (y[i] && aom_read_bit(r, "pvq:sign")) y[i] = -y[i]; + } +} + +/** Inverse of neg_interleave; decodes the interleaved gain. + * + * @param [in] x quantized/interleaved gain to decode + * @param [in] ref quantized gain of the reference + * @return original quantized gain value + */ +static int neg_deinterleave(int x, int ref) { + if (x < 2*ref-1) { + if (x & 1) return ref - 1 - (x >> 1); + else return ref + (x >> 1); + } + else return x+1; +} + +/** Synthesizes one parition of coefficient values from a PVQ-encoded + * vector. + * + * @param [out] xcoeff output coefficient partition (x in math doc) + * @param [in] ypulse PVQ-encoded values (y in math doc); in the noref + * case, this vector has n entries, in the + * reference case it contains n-1 entries + * (the m-th entry is not included) + * @param [in] ref reference vector (prediction) + * @param [in] n number of elements in this partition + * @param [in] gr gain of the reference vector (prediction) + * @param [in] noref indicates presence or lack of prediction + * @param [in] g decoded quantized vector gain + * @param [in] theta decoded theta (prediction error) + * @param [in] qm QM with magnitude compensation + * @param [in] qm_inv Inverse of QM with magnitude compensation + */ +static void pvq_synthesis(od_coeff *xcoeff, od_coeff *ypulse, od_val16 *r16, + int n, od_val32 gr, int noref, od_val32 g, od_val32 theta, const int16_t *qm_inv, + int shift) { + int s; + int m; + /* Sign of the Householder reflection vector */ + s = 0; + /* Direction of the Householder reflection vector */ + m = noref ? 0 : od_compute_householder(r16, n, gr, &s, shift); + od_pvq_synthesis_partial(xcoeff, ypulse, r16, n, noref, g, theta, m, s, + qm_inv); +} + +typedef struct { + od_coeff *ref; + int nb_coeffs; + int allow_flip; +} cfl_ctx; + +/** Decodes a single vector of integers (eg, a partition within a + * coefficient block) encoded using PVQ + * + * @param [in,out] ec range encoder + * @param [in] q0 scale/quantizer + * @param [in] n number of coefficients in partition + * @param [in,out] model entropy decoder state + * @param [in,out] adapt adaptation context + * @param [in,out] exg ExQ16 expectation of decoded gain value + * @param [in,out] ext ExQ16 expectation of decoded theta value + * @param [in] ref 'reference' (prediction) vector + * @param [out] out decoded partition + * @param [out] noref boolean indicating absence of reference + * @param [in] beta per-band activity masking beta param + * @param [in] is_keyframe whether we're encoding a keyframe + * @param [in] pli plane index + * @param [in] cdf_ctx selects which cdf context to use + * @param [in,out] skip_rest whether to skip further bands in each direction + * @param [in] band index of the band being decoded + * @param [in] band index of the band being decoded + * @param [out] skip skip flag with range [0,1] + * @param [in] qm QM with magnitude compensation + * @param [in] qm_inv Inverse of QM with magnitude compensation + */ +static void pvq_decode_partition(aom_reader *r, + int q0, + int n, + generic_encoder model[3], + od_adapt_ctx *adapt, + int *exg, + int *ext, + od_coeff *ref, + od_coeff *out, + int *noref, + od_val16 beta, + int is_keyframe, + int pli, + int cdf_ctx, + cfl_ctx *cfl, + int has_skip, + int *skip_rest, + int band, + int *skip, + const int16_t *qm, + const int16_t *qm_inv) { + int k; + od_val32 qcg; + int itheta; + od_val32 theta; + od_val32 gr; + od_val32 gain_offset; + od_coeff y[MAXN]; + int qg; + int id; + int i; + od_val16 ref16[MAXN]; + int rshift; + theta = 0; + gr = 0; + gain_offset = 0; + /* Skip is per-direction. For band=0, we can use any of the flags. */ + if (skip_rest[(band + 2) % 3]) { + qg = 0; + if (is_keyframe) { + itheta = -1; + *noref = 1; + } + else { + itheta = 0; + *noref = 0; + } + } + else { + /* Jointly decode gain, itheta and noref for small values. Then we handle + larger gain. */ + id = aom_read_symbol_pvq(r, &adapt->pvq.pvq_gaintheta_cdf[cdf_ctx][0], + 8 + 7*has_skip, "pvq:gaintheta"); + if (!is_keyframe && id >= 10) id++; + if (is_keyframe && id >= 8) id++; + if (id >= 8) { + id -= 8; + skip_rest[0] = skip_rest[1] = skip_rest[2] = 1; + } + qg = id & 1; + itheta = (id >> 1) - 1; + *noref = (itheta == -1); + } + /* The CfL flip bit is only decoded on the first band that has noref=0. */ + if (cfl->allow_flip && !*noref) { + int flip; + flip = aom_read_bit(r, "cfl:flip"); + if (flip) { + for (i = 0; i < cfl->nb_coeffs; i++) cfl->ref[i] = -cfl->ref[i]; + } + cfl->allow_flip = 0; + } + if (qg > 0) { + int tmp; + tmp = *exg; + qg = 1 + generic_decode(r, &model[!*noref], &tmp, 2, "pvq:gain"); + OD_IIR_DIADIC(*exg, qg << 16, 2); + } + *skip = 0; +#if defined(OD_FLOAT_PVQ) + rshift = 0; +#else + /* Shift needed to make the reference fit in 15 bits, so that the Householder + vector can fit in 16 bits. */ + rshift = OD_MAXI(0, od_vector_log_mag(ref, n) - 14); +#endif + for (i = 0; i < n; i++) { +#if defined(OD_FLOAT_PVQ) + ref16[i] = ref[i]*(double)qm[i]*OD_QM_SCALE_1; +#else + ref16[i] = OD_SHR_ROUND(ref[i]*qm[i], OD_QM_SHIFT + rshift); +#endif + } + if(!*noref){ + /* we have a reference; compute its gain */ + od_val32 cgr; + int icgr; + int cfl_enabled; + cfl_enabled = pli != 0 && is_keyframe && !OD_DISABLE_CFL; + cgr = od_pvq_compute_gain(ref16, n, q0, &gr, beta, rshift); + if (cfl_enabled) cgr = OD_CGAIN_SCALE; +#if defined(OD_FLOAT_PVQ) + icgr = (int)floor(.5 + cgr); +#else + icgr = OD_SHR_ROUND(cgr, OD_CGAIN_SHIFT); +#endif + /* quantized gain is interleave encoded when there's a reference; + deinterleave it now */ + if (is_keyframe) qg = neg_deinterleave(qg, icgr); + else { + qg = neg_deinterleave(qg, icgr + 1) - 1; + if (qg == 0) *skip = (icgr ? OD_PVQ_SKIP_ZERO : OD_PVQ_SKIP_COPY); + } + if (qg == icgr && itheta == 0 && !cfl_enabled) *skip = OD_PVQ_SKIP_COPY; + gain_offset = cgr - OD_SHL(icgr, OD_CGAIN_SHIFT); + qcg = OD_SHL(qg, OD_CGAIN_SHIFT) + gain_offset; + /* read and decode first-stage PVQ error theta */ + if (itheta > 1) { + int tmp; + tmp = *ext; + itheta = 2 + generic_decode(r, &model[2], &tmp, 2, "pvq:theta"); + OD_IIR_DIADIC(*ext, itheta << 16, 2); + } + theta = od_pvq_compute_theta(itheta, od_pvq_compute_max_theta(qcg, beta)); + } + else{ + itheta = 0; + if (!is_keyframe) qg++; + qcg = OD_SHL(qg, OD_CGAIN_SHIFT); + if (qg == 0) *skip = OD_PVQ_SKIP_ZERO; + } + + k = od_pvq_compute_k(qcg, itheta, *noref, n, beta); + if (k != 0) { + /* when noref==0, y is actually size n-1 */ + aom_decode_pvq_codeword(r, &adapt->pvq.pvq_codeword_ctx, y, + n - !*noref, k); + } + else { + OD_CLEAR(y, n); + } + if (*skip) { + if (*skip == OD_PVQ_SKIP_COPY) OD_COPY(out, ref, n); + else OD_CLEAR(out, n); + } + else { + od_val32 g; + g = od_gain_expand(qcg, q0, beta); + pvq_synthesis(out, y, ref16, n, gr, *noref, g, theta, qm_inv, rshift); + } + /* If OD_PVQ_SKIP_ZERO or OD_PVQ_SKIP_COPY, set skip to 1 for visualization */ + if (*skip) *skip = 1; +} + +/** Decodes a coefficient block (except for DC) encoded using PVQ + * + * @param [in,out] dec daala decoder context + * @param [in] ref 'reference' (prediction) vector + * @param [out] out decoded partition + * @param [in] q0 quantizer + * @param [in] pli plane index + * @param [in] bs log of the block size minus two + * @param [in] beta per-band activity masking beta param + * @param [in] is_keyframe whether we're encoding a keyframe + * @param [out] flags bitmask of the per band skip and noref flags + * @param [in] ac_dc_coded skip flag for the block (range 0-3) + * @param [in] qm QM with magnitude compensation + * @param [in] qm_inv Inverse of QM with magnitude compensation + */ +void od_pvq_decode(daala_dec_ctx *dec, + od_coeff *ref, + od_coeff *out, + int q0, + int pli, + int bs, + const od_val16 *beta, + int is_keyframe, + unsigned int *flags, + PVQ_SKIP_TYPE ac_dc_coded, + const int16_t *qm, + const int16_t *qm_inv){ + + int noref[PVQ_MAX_PARTITIONS]; + int skip[PVQ_MAX_PARTITIONS]; + int *exg; + int *ext; + int nb_bands; + int i; + const int *off; + int size[PVQ_MAX_PARTITIONS]; + generic_encoder *model; + int skip_rest[3] = {0}; + cfl_ctx cfl; + const unsigned char *pvq_qm; + int use_masking; + + aom_clear_system_state(); + + /*Default to skip=1 and noref=0 for all bands.*/ + for (i = 0; i < PVQ_MAX_PARTITIONS; i++) { + noref[i] = 0; + skip[i] = 1; + } + + use_masking = dec->use_activity_masking; + + if (use_masking) + pvq_qm = &dec->state.pvq_qm_q4[pli][0]; + else + pvq_qm = 0; + + exg = &dec->state.adapt->pvq.pvq_exg[pli][bs][0]; + ext = dec->state.adapt->pvq.pvq_ext + bs*PVQ_MAX_PARTITIONS; + model = dec->state.adapt->pvq.pvq_param_model; + nb_bands = OD_BAND_OFFSETS[bs][0]; + off = &OD_BAND_OFFSETS[bs][1]; + out[0] = ac_dc_coded & DC_CODED; + if (ac_dc_coded < AC_CODED) { + if (is_keyframe) for (i = 1; i < 1 << (2*bs + 4); i++) out[i] = 0; + else for (i = 1; i < 1 << (2*bs + 4); i++) out[i] = ref[i]; + } + else { + for (i = 0; i < nb_bands; i++) size[i] = off[i+1] - off[i]; + cfl.ref = ref; + cfl.nb_coeffs = off[nb_bands]; + cfl.allow_flip = pli != 0 && is_keyframe; + for (i = 0; i < nb_bands; i++) { + int q; + + if (use_masking) + q = OD_MAXI(1, q0 * pvq_qm[od_qm_get_index(bs, i + 1)] >> 4); + else + q = OD_MAXI(1, q0); + + pvq_decode_partition(dec->r, q, size[i], + model, dec->state.adapt, exg + i, ext + i, ref + off[i], out + off[i], + &noref[i], beta[i], is_keyframe, pli, + (pli != 0)*OD_TXSIZES*PVQ_MAX_PARTITIONS + bs*PVQ_MAX_PARTITIONS + i, + &cfl, i == 0 && (i < nb_bands - 1), skip_rest, i, &skip[i], + qm + off[i], qm_inv + off[i]); + if (i == 0 && !skip_rest[0] && bs > 0) { + int skip_dir; + int j; + skip_dir = aom_read_symbol(dec->r, + &dec->state.adapt->pvq.pvq_skip_dir_cdf[(pli != 0) + 2*(bs - 1)][0], 7, + "pvq:skiprest"); + for (j = 0; j < 3; j++) skip_rest[j] = !!(skip_dir & (1 << j)); + } + } + } + *flags = 0; + for (i = nb_bands - 1; i >= 0; i--) { + *flags <<= 1; + *flags |= noref[i]&1; + *flags <<= 1; + *flags |= skip[i]&1; + } +} diff --git a/third_party/aom/av1/decoder/pvq_decoder.h b/third_party/aom/av1/decoder/pvq_decoder.h new file mode 100644 index 0000000000..98970663be --- /dev/null +++ b/third_party/aom/av1/decoder/pvq_decoder.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#if !defined(_pvq_decoder_H) +# define _pvq_decoder_H (1) +# include "aom_dsp/bitreader.h" +# include "aom_dsp/entdec.h" +# include "av1/common/pvq.h" +# include "av1/decoder/decint.h" + +#define aom_read_symbol_pvq(r, cdf, nsymbs, ACCT_STR_NAME) \ + aom_read_symbol_pvq_(r, cdf, nsymbs ACCT_STR_ARG(ACCT_STR_NAME)) + +int aom_read_symbol_pvq_(aom_reader *r, aom_cdf_prob *cdf, int nsymbs + ACCT_STR_PARAM); + +void aom_decode_band_pvq_splits(aom_reader *r, od_pvq_codeword_ctx *adapt, + od_coeff *y, int n, int k, int level); + +#define aom_laplace_decode_special(r, decay, ACCT_STR_NAME) \ + aom_laplace_decode_special_(r, decay ACCT_STR_ARG(ACCT_STR_NAME)) + +int aom_laplace_decode_special_(aom_reader *r, unsigned decay ACCT_STR_PARAM); + +void od_pvq_decode(daala_dec_ctx *dec, od_coeff *ref, od_coeff *out, int q0, + int pli, int bs, const od_val16 *beta, int is_keyframe, + unsigned int *flags, PVQ_SKIP_TYPE ac_dc_coded, const int16_t *qm, + const int16_t *qm_inv); + +#endif diff --git a/third_party/aom/av1/encoder/aq_complexity.c b/third_party/aom/av1/encoder/aq_complexity.c new file mode 100644 index 0000000000..054b0e062b --- /dev/null +++ b/third_party/aom/av1/encoder/aq_complexity.c @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "av1/encoder/aq_complexity.h" +#include "av1/encoder/aq_variance.h" +#include "av1/encoder/encodeframe.h" +#include "av1/common/seg_common.h" +#include "av1/encoder/segmentation.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/system_state.h" + +#define AQ_C_SEGMENTS 5 +#define DEFAULT_AQ2_SEG 3 // Neutral Q segment +#define AQ_C_STRENGTHS 3 +static const double aq_c_q_adj_factor[AQ_C_STRENGTHS][AQ_C_SEGMENTS] = { + { 1.75, 1.25, 1.05, 1.00, 0.90 }, + { 2.00, 1.50, 1.15, 1.00, 0.85 }, + { 2.50, 1.75, 1.25, 1.00, 0.80 } +}; +static const double aq_c_transitions[AQ_C_STRENGTHS][AQ_C_SEGMENTS] = { + { 0.15, 0.30, 0.55, 2.00, 100.0 }, + { 0.20, 0.40, 0.65, 2.00, 100.0 }, + { 0.25, 0.50, 0.75, 2.00, 100.0 } +}; +static const double aq_c_var_thresholds[AQ_C_STRENGTHS][AQ_C_SEGMENTS] = { + { -4.0, -3.0, -2.0, 100.00, 100.0 }, + { -3.5, -2.5, -1.5, 100.00, 100.0 }, + { -3.0, -2.0, -1.0, 100.00, 100.0 } +}; + +#define DEFAULT_COMPLEXITY 64 + +static int get_aq_c_strength(int q_index, aom_bit_depth_t bit_depth) { + // Approximate base quatizer (truncated to int) + const int base_quant = av1_ac_quant(q_index, 0, bit_depth) / 4; + return (base_quant > 10) + (base_quant > 25); +} + +void av1_setup_in_frame_q_adj(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + struct segmentation *const seg = &cm->seg; + + // Make SURE use of floating point in this function is safe. + aom_clear_system_state(); + + if (frame_is_intra_only(cm) || cm->error_resilient_mode || + cpi->refresh_alt_ref_frame || + (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref)) { + int segment; + const int aq_strength = get_aq_c_strength(cm->base_qindex, cm->bit_depth); + + // Clear down the segment map. + memset(cpi->segmentation_map, DEFAULT_AQ2_SEG, cm->mi_rows * cm->mi_cols); + + av1_clearall_segfeatures(seg); + + // Segmentation only makes sense if the target bits per SB is above a + // threshold. Below this the overheads will usually outweigh any benefit. + if (cpi->rc.sb64_target_rate < 256) { + av1_disable_segmentation(seg); + return; + } + + av1_enable_segmentation(seg); + + // Select delta coding method. + seg->abs_delta = SEGMENT_DELTADATA; + + // Default segment "Q" feature is disabled so it defaults to the baseline Q. + av1_disable_segfeature(seg, DEFAULT_AQ2_SEG, SEG_LVL_ALT_Q); + + // Use some of the segments for in frame Q adjustment. + for (segment = 0; segment < AQ_C_SEGMENTS; ++segment) { + int qindex_delta; + + if (segment == DEFAULT_AQ2_SEG) continue; + + qindex_delta = av1_compute_qdelta_by_rate( + &cpi->rc, cm->frame_type, cm->base_qindex, + aq_c_q_adj_factor[aq_strength][segment], cm->bit_depth); + + // For AQ complexity mode, we dont allow Q0 in a segment if the base + // Q is not 0. Q0 (lossless) implies 4x4 only and in AQ mode 2 a segment + // Q delta is sometimes applied without going back around the rd loop. + // This could lead to an illegal combination of partition size and q. + if ((cm->base_qindex != 0) && ((cm->base_qindex + qindex_delta) == 0)) { + qindex_delta = -cm->base_qindex + 1; + } + if ((cm->base_qindex + qindex_delta) > 0) { + av1_enable_segfeature(seg, segment, SEG_LVL_ALT_Q); + av1_set_segdata(seg, segment, SEG_LVL_ALT_Q, qindex_delta); + } + } + } +} + +#define DEFAULT_LV_THRESH 10.0 +#define MIN_DEFAULT_LV_THRESH 8.0 +#define VAR_STRENGTH_STEP 0.25 +// Select a segment for the current block. +// The choice of segment for a block depends on the ratio of the projected +// bits for the block vs a target average and its spatial complexity. +void av1_caq_select_segment(const AV1_COMP *cpi, MACROBLOCK *mb, BLOCK_SIZE bs, + int mi_row, int mi_col, int projected_rate) { + const AV1_COMMON *const cm = &cpi->common; + + const int mi_offset = mi_row * cm->mi_cols + mi_col; + const int xmis = AOMMIN(cm->mi_cols - mi_col, mi_size_wide[bs]); + const int ymis = AOMMIN(cm->mi_rows - mi_row, mi_size_high[bs]); + int x, y; + int i; + unsigned char segment; + + if (0) { + segment = DEFAULT_AQ2_SEG; + } else { + // Rate depends on fraction of a SB64 in frame (xmis * ymis / bw * bh). + // It is converted to bits * 256 units. + const int64_t num = (int64_t)cpi->rc.sb64_target_rate * xmis * ymis * 256; + const int denom = cm->mib_size * cm->mib_size; + const int target_rate = (int)(num / denom); + double logvar; + double low_var_thresh; + const int aq_strength = get_aq_c_strength(cm->base_qindex, cm->bit_depth); + + aom_clear_system_state(); + low_var_thresh = (cpi->oxcf.pass == 2) ? AOMMAX(cpi->twopass.mb_av_energy, + MIN_DEFAULT_LV_THRESH) + : DEFAULT_LV_THRESH; + + av1_setup_src_planes(mb, cpi->source, mi_row, mi_col); + logvar = av1_log_block_var(cpi, mb, bs); + + segment = AQ_C_SEGMENTS - 1; // Just in case no break out below. + for (i = 0; i < AQ_C_SEGMENTS; ++i) { + // Test rate against a threshold value and variance against a threshold. + // Increasing segment number (higher variance and complexity) = higher Q. + if ((projected_rate < target_rate * aq_c_transitions[aq_strength][i]) && + (logvar < (low_var_thresh + aq_c_var_thresholds[aq_strength][i]))) { + segment = i; + break; + } + } + } + + // Fill in the entires in the segment map corresponding to this SB64. + for (y = 0; y < ymis; y++) { + for (x = 0; x < xmis; x++) { + cpi->segmentation_map[mi_offset + y * cm->mi_cols + x] = segment; + } + } +} diff --git a/third_party/aom/av1/encoder/aq_complexity.h b/third_party/aom/av1/encoder/aq_complexity.h new file mode 100644 index 0000000000..af525b36de --- /dev/null +++ b/third_party/aom/av1/encoder/aq_complexity.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_AQ_COMPLEXITY_H_ +#define AV1_ENCODER_AQ_COMPLEXITY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "av1/common/enums.h" + +struct AV1_COMP; +struct macroblock; + +// Select a segment for the current Block. +void av1_caq_select_segment(const struct AV1_COMP *cpi, struct macroblock *, + BLOCK_SIZE bs, int mi_row, int mi_col, + int projected_rate); + +// This function sets up a set of segments with delta Q values around +// the baseline frame quantizer. +void av1_setup_in_frame_q_adj(struct AV1_COMP *cpi); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_AQ_COMPLEXITY_H_ diff --git a/third_party/aom/av1/encoder/aq_cyclicrefresh.c b/third_party/aom/av1/encoder/aq_cyclicrefresh.c new file mode 100644 index 0000000000..e41c608b64 --- /dev/null +++ b/third_party/aom/av1/encoder/aq_cyclicrefresh.c @@ -0,0 +1,566 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "av1/common/seg_common.h" +#include "av1/encoder/aq_cyclicrefresh.h" +#include "av1/encoder/ratectrl.h" +#include "av1/encoder/segmentation.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/system_state.h" + +struct CYCLIC_REFRESH { + // Percentage of blocks per frame that are targeted as candidates + // for cyclic refresh. + int percent_refresh; + // Maximum q-delta as percentage of base q. + int max_qdelta_perc; + // Superblock starting index for cycling through the frame. + int sb_index; + // Controls how long block will need to wait to be refreshed again, in + // excess of the cycle time, i.e., in the case of all zero motion, block + // will be refreshed every (100/percent_refresh + time_for_refresh) frames. + int time_for_refresh; + // Target number of (8x8) blocks that are set for delta-q. + int target_num_seg_blocks; + // Actual number of (8x8) blocks that were applied delta-q. + int actual_num_seg1_blocks; + int actual_num_seg2_blocks; + // RD mult. parameters for segment 1. + int rdmult; + // Cyclic refresh map. + signed char *map; + // Map of the last q a block was coded at. + uint8_t *last_coded_q_map; + // Thresholds applied to the projected rate/distortion of the coding block, + // when deciding whether block should be refreshed. + int64_t thresh_rate_sb; + int64_t thresh_dist_sb; + // Threshold applied to the motion vector (in units of 1/8 pel) of the + // coding block, when deciding whether block should be refreshed. + int16_t motion_thresh; + // Rate target ratio to set q delta. + double rate_ratio_qdelta; + // Boost factor for rate target ratio, for segment CR_SEGMENT_ID_BOOST2. + int rate_boost_fac; + double low_content_avg; + int qindex_delta[3]; +}; + +CYCLIC_REFRESH *av1_cyclic_refresh_alloc(int mi_rows, int mi_cols) { + size_t last_coded_q_map_size; + CYCLIC_REFRESH *const cr = aom_calloc(1, sizeof(*cr)); + if (cr == NULL) return NULL; + + cr->map = aom_calloc(mi_rows * mi_cols, sizeof(*cr->map)); + if (cr->map == NULL) { + av1_cyclic_refresh_free(cr); + return NULL; + } + last_coded_q_map_size = mi_rows * mi_cols * sizeof(*cr->last_coded_q_map); + cr->last_coded_q_map = aom_malloc(last_coded_q_map_size); + if (cr->last_coded_q_map == NULL) { + av1_cyclic_refresh_free(cr); + return NULL; + } + assert(MAXQ <= 255); + memset(cr->last_coded_q_map, MAXQ, last_coded_q_map_size); + + return cr; +} + +void av1_cyclic_refresh_free(CYCLIC_REFRESH *cr) { + aom_free(cr->map); + aom_free(cr->last_coded_q_map); + aom_free(cr); +} + +// Check if we should turn off cyclic refresh based on bitrate condition. +static int apply_cyclic_refresh_bitrate(const AV1_COMMON *cm, + const RATE_CONTROL *rc) { + // Turn off cyclic refresh if bits available per frame is not sufficiently + // larger than bit cost of segmentation. Segment map bit cost should scale + // with number of seg blocks, so compare available bits to number of blocks. + // Average bits available per frame = avg_frame_bandwidth + // Number of (8x8) blocks in frame = mi_rows * mi_cols; + const float factor = 0.25; + const int number_blocks = cm->mi_rows * cm->mi_cols; + // The condition below corresponds to turning off at target bitrates: + // (at 30fps), ~12kbps for CIF, 36kbps for VGA, 100kps for HD/720p. + // Also turn off at very small frame sizes, to avoid too large fraction of + // superblocks to be refreshed per frame. Threshold below is less than QCIF. + if (rc->avg_frame_bandwidth < factor * number_blocks || + number_blocks / 64 < 5) + return 0; + else + return 1; +} + +// Check if this coding block, of size bsize, should be considered for refresh +// (lower-qp coding). Decision can be based on various factors, such as +// size of the coding block (i.e., below min_block size rejected), coding +// mode, and rate/distortion. +static int candidate_refresh_aq(const CYCLIC_REFRESH *cr, + const MB_MODE_INFO *mbmi, int64_t rate, + int64_t dist, int bsize) { + MV mv = mbmi->mv[0].as_mv; + // Reject the block for lower-qp coding if projected distortion + // is above the threshold, and any of the following is true: + // 1) mode uses large mv + // 2) mode is an intra-mode + // Otherwise accept for refresh. + if (dist > cr->thresh_dist_sb && + (mv.row > cr->motion_thresh || mv.row < -cr->motion_thresh || + mv.col > cr->motion_thresh || mv.col < -cr->motion_thresh || + !is_inter_block(mbmi))) + return CR_SEGMENT_ID_BASE; + else if (bsize >= BLOCK_16X16 && rate < cr->thresh_rate_sb && + is_inter_block(mbmi) && mbmi->mv[0].as_int == 0 && + cr->rate_boost_fac > 10) + // More aggressive delta-q for bigger blocks with zero motion. + return CR_SEGMENT_ID_BOOST2; + else + return CR_SEGMENT_ID_BOOST1; +} + +// Compute delta-q for the segment. +static int compute_deltaq(const AV1_COMP *cpi, int q, double rate_factor) { + const CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + const RATE_CONTROL *const rc = &cpi->rc; + int deltaq = av1_compute_qdelta_by_rate(rc, cpi->common.frame_type, q, + rate_factor, cpi->common.bit_depth); + if ((-deltaq) > cr->max_qdelta_perc * q / 100) { + deltaq = -cr->max_qdelta_perc * q / 100; + } + return deltaq; +} + +// For the just encoded frame, estimate the bits, incorporating the delta-q +// from non-base segment. For now ignore effect of multiple segments +// (with different delta-q). Note this function is called in the postencode +// (called from rc_update_rate_correction_factors()). +int av1_cyclic_refresh_estimate_bits_at_q(const AV1_COMP *cpi, + double correction_factor) { + const AV1_COMMON *const cm = &cpi->common; + const CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + int estimated_bits; + int mbs = cm->MBs; + int num8x8bl = mbs << 2; + // Weight for non-base segments: use actual number of blocks refreshed in + // previous/just encoded frame. Note number of blocks here is in 8x8 units. + double weight_segment1 = (double)cr->actual_num_seg1_blocks / num8x8bl; + double weight_segment2 = (double)cr->actual_num_seg2_blocks / num8x8bl; + // Take segment weighted average for estimated bits. + estimated_bits = + (int)((1.0 - weight_segment1 - weight_segment2) * + av1_estimate_bits_at_q(cm->frame_type, cm->base_qindex, mbs, + correction_factor, cm->bit_depth) + + weight_segment1 * + av1_estimate_bits_at_q(cm->frame_type, + cm->base_qindex + cr->qindex_delta[1], + mbs, correction_factor, cm->bit_depth) + + weight_segment2 * + av1_estimate_bits_at_q(cm->frame_type, + cm->base_qindex + cr->qindex_delta[2], + mbs, correction_factor, cm->bit_depth)); + return estimated_bits; +} + +// Prior to encoding the frame, estimate the bits per mb, for a given q = i and +// a corresponding delta-q (for segment 1). This function is called in the +// rc_regulate_q() to set the base qp index. +// Note: the segment map is set to either 0/CR_SEGMENT_ID_BASE (no refresh) or +// to 1/CR_SEGMENT_ID_BOOST1 (refresh) for each superblock, prior to encoding. +int av1_cyclic_refresh_rc_bits_per_mb(const AV1_COMP *cpi, int i, + double correction_factor) { + const AV1_COMMON *const cm = &cpi->common; + CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + int bits_per_mb; + int num8x8bl = cm->MBs << 2; + // Weight for segment prior to encoding: take the average of the target + // number for the frame to be encoded and the actual from the previous frame. + double weight_segment = + (double)((cr->target_num_seg_blocks + cr->actual_num_seg1_blocks + + cr->actual_num_seg2_blocks) >> + 1) / + num8x8bl; + // Compute delta-q corresponding to qindex i. + int deltaq = compute_deltaq(cpi, i, cr->rate_ratio_qdelta); + // Take segment weighted average for bits per mb. + bits_per_mb = (int)((1.0 - weight_segment) * + av1_rc_bits_per_mb(cm->frame_type, i, + correction_factor, cm->bit_depth) + + weight_segment * + av1_rc_bits_per_mb(cm->frame_type, i + deltaq, + correction_factor, cm->bit_depth)); + return bits_per_mb; +} + +// Prior to coding a given prediction block, of size bsize at (mi_row, mi_col), +// check if we should reset the segment_id, and update the cyclic_refresh map +// and segmentation map. +void av1_cyclic_refresh_update_segment(const AV1_COMP *cpi, + MB_MODE_INFO *const mbmi, int mi_row, + int mi_col, BLOCK_SIZE bsize, + int64_t rate, int64_t dist, int skip) { + const AV1_COMMON *const cm = &cpi->common; + CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + const int bw = mi_size_wide[bsize]; + const int bh = mi_size_high[bsize]; + const int xmis = AOMMIN(cm->mi_cols - mi_col, bw); + const int ymis = AOMMIN(cm->mi_rows - mi_row, bh); + const int block_index = mi_row * cm->mi_cols + mi_col; + const int refresh_this_block = + candidate_refresh_aq(cr, mbmi, rate, dist, bsize); + // Default is to not update the refresh map. + int new_map_value = cr->map[block_index]; + int x = 0; + int y = 0; + + // If this block is labeled for refresh, check if we should reset the + // segment_id. + if (cyclic_refresh_segment_id_boosted(mbmi->segment_id)) { + mbmi->segment_id = refresh_this_block; + // Reset segment_id if will be skipped. + if (skip) mbmi->segment_id = CR_SEGMENT_ID_BASE; + } + + // Update the cyclic refresh map, to be used for setting segmentation map + // for the next frame. If the block will be refreshed this frame, mark it + // as clean. The magnitude of the -ve influences how long before we consider + // it for refresh again. + if (cyclic_refresh_segment_id_boosted(mbmi->segment_id)) { + new_map_value = -cr->time_for_refresh; + } else if (refresh_this_block) { + // Else if it is accepted as candidate for refresh, and has not already + // been refreshed (marked as 1) then mark it as a candidate for cleanup + // for future time (marked as 0), otherwise don't update it. + if (cr->map[block_index] == 1) new_map_value = 0; + } else { + // Leave it marked as block that is not candidate for refresh. + new_map_value = 1; + } + + // Update entries in the cyclic refresh map with new_map_value, and + // copy mbmi->segment_id into global segmentation map. + for (y = 0; y < ymis; y++) + for (x = 0; x < xmis; x++) { + int map_offset = block_index + y * cm->mi_cols + x; + cr->map[map_offset] = new_map_value; + cpi->segmentation_map[map_offset] = mbmi->segment_id; + // Inter skip blocks were clearly not coded at the current qindex, so + // don't update the map for them. For cases where motion is non-zero or + // the reference frame isn't the previous frame, the previous value in + // the map for this spatial location is not entirely correct. + if ((!is_inter_block(mbmi) || !skip) && + mbmi->segment_id <= CR_SEGMENT_ID_BOOST2) { + cr->last_coded_q_map[map_offset] = clamp( + cm->base_qindex + cr->qindex_delta[mbmi->segment_id], 0, MAXQ); + } else if (is_inter_block(mbmi) && skip && + mbmi->segment_id <= CR_SEGMENT_ID_BOOST2) { + cr->last_coded_q_map[map_offset] = + AOMMIN(clamp(cm->base_qindex + cr->qindex_delta[mbmi->segment_id], + 0, MAXQ), + cr->last_coded_q_map[map_offset]); + } + } +} + +// Update the actual number of blocks that were applied the segment delta q. +void av1_cyclic_refresh_postencode(AV1_COMP *const cpi) { + AV1_COMMON *const cm = &cpi->common; + CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + unsigned char *const seg_map = cpi->segmentation_map; + int mi_row, mi_col; + cr->actual_num_seg1_blocks = 0; + cr->actual_num_seg2_blocks = 0; + for (mi_row = 0; mi_row < cm->mi_rows; mi_row++) + for (mi_col = 0; mi_col < cm->mi_cols; mi_col++) { + if (cyclic_refresh_segment_id(seg_map[mi_row * cm->mi_cols + mi_col]) == + CR_SEGMENT_ID_BOOST1) + cr->actual_num_seg1_blocks++; + else if (cyclic_refresh_segment_id( + seg_map[mi_row * cm->mi_cols + mi_col]) == + CR_SEGMENT_ID_BOOST2) + cr->actual_num_seg2_blocks++; + } +} + +// Set golden frame update interval, for 1 pass CBR mode. +void av1_cyclic_refresh_set_golden_update(AV1_COMP *const cpi) { + RATE_CONTROL *const rc = &cpi->rc; + CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + // Set minimum gf_interval for GF update to a multiple (== 2) of refresh + // period. Depending on past encoding stats, GF flag may be reset and update + // may not occur until next baseline_gf_interval. + if (cr->percent_refresh > 0) + rc->baseline_gf_interval = 4 * (100 / cr->percent_refresh); + else + rc->baseline_gf_interval = 40; +} + +// Update some encoding stats (from the just encoded frame). If this frame's +// background has high motion, refresh the golden frame. Otherwise, if the +// golden reference is to be updated check if we should NOT update the golden +// ref. +void av1_cyclic_refresh_check_golden_update(AV1_COMP *const cpi) { + AV1_COMMON *const cm = &cpi->common; + CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + int mi_row, mi_col; + double fraction_low = 0.0; + int low_content_frame = 0; + + MODE_INFO **mi; + RATE_CONTROL *const rc = &cpi->rc; + const int rows = cm->mi_rows, cols = cm->mi_cols; + int cnt1 = 0, cnt2 = 0; + int force_gf_refresh = 0; + + for (mi_row = 0; mi_row < rows; mi_row++) { + mi = cm->mi_grid_visible + mi_row * cm->mi_stride; + + for (mi_col = 0; mi_col < cols; mi_col++) { + int16_t abs_mvr = mi[0]->mbmi.mv[0].as_mv.row >= 0 + ? mi[0]->mbmi.mv[0].as_mv.row + : -1 * mi[0]->mbmi.mv[0].as_mv.row; + int16_t abs_mvc = mi[0]->mbmi.mv[0].as_mv.col >= 0 + ? mi[0]->mbmi.mv[0].as_mv.col + : -1 * mi[0]->mbmi.mv[0].as_mv.col; + + // Calculate the motion of the background. + if (abs_mvr <= 16 && abs_mvc <= 16) { + cnt1++; + if (abs_mvr == 0 && abs_mvc == 0) cnt2++; + } + mi++; + + // Accumulate low_content_frame. + if (cr->map[mi_row * cols + mi_col] < 1) low_content_frame++; + } + } + + // For video conference clips, if the background has high motion in current + // frame because of the camera movement, set this frame as the golden frame. + // Use 70% and 5% as the thresholds for golden frame refreshing. + // Also, force this frame as a golden update frame if this frame will change + // the resolution (resize_pending != 0). + if (cpi->resize_pending != 0 || + (cnt1 * 10 > (70 * rows * cols) && cnt2 * 20 < cnt1)) { + av1_cyclic_refresh_set_golden_update(cpi); + rc->frames_till_gf_update_due = rc->baseline_gf_interval; + + if (rc->frames_till_gf_update_due > rc->frames_to_key) + rc->frames_till_gf_update_due = rc->frames_to_key; + cpi->refresh_golden_frame = 1; + force_gf_refresh = 1; + } + + fraction_low = (double)low_content_frame / (rows * cols); + // Update average. + cr->low_content_avg = (fraction_low + 3 * cr->low_content_avg) / 4; + if (!force_gf_refresh && cpi->refresh_golden_frame == 1) { + // Don't update golden reference if the amount of low_content for the + // current encoded frame is small, or if the recursive average of the + // low_content over the update interval window falls below threshold. + if (fraction_low < 0.8 || cr->low_content_avg < 0.7) + cpi->refresh_golden_frame = 0; + // Reset for next internal. + cr->low_content_avg = fraction_low; + } +} + +// Update the segmentation map, and related quantities: cyclic refresh map, +// refresh sb_index, and target number of blocks to be refreshed. +// The map is set to either 0/CR_SEGMENT_ID_BASE (no refresh) or to +// 1/CR_SEGMENT_ID_BOOST1 (refresh) for each superblock. +// Blocks labeled as BOOST1 may later get set to BOOST2 (during the +// encoding of the superblock). +static void cyclic_refresh_update_map(AV1_COMP *const cpi) { + AV1_COMMON *const cm = &cpi->common; + CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + unsigned char *const seg_map = cpi->segmentation_map; + int i, block_count, bl_index, sb_rows, sb_cols, sbs_in_frame; + int xmis, ymis, x, y; + memset(seg_map, CR_SEGMENT_ID_BASE, cm->mi_rows * cm->mi_cols); + sb_cols = (cm->mi_cols + cm->mib_size - 1) / cm->mib_size; + sb_rows = (cm->mi_rows + cm->mib_size - 1) / cm->mib_size; + sbs_in_frame = sb_cols * sb_rows; + // Number of target blocks to get the q delta (segment 1). + block_count = cr->percent_refresh * cm->mi_rows * cm->mi_cols / 100; + // Set the segmentation map: cycle through the superblocks, starting at + // cr->mb_index, and stopping when either block_count blocks have been found + // to be refreshed, or we have passed through whole frame. + assert(cr->sb_index < sbs_in_frame); + i = cr->sb_index; + cr->target_num_seg_blocks = 0; + do { + int sum_map = 0; + // Get the mi_row/mi_col corresponding to superblock index i. + int sb_row_index = (i / sb_cols); + int sb_col_index = i - sb_row_index * sb_cols; + int mi_row = sb_row_index * cm->mib_size; + int mi_col = sb_col_index * cm->mib_size; + int qindex_thresh = + cpi->oxcf.content == AOM_CONTENT_SCREEN + ? av1_get_qindex(&cm->seg, CR_SEGMENT_ID_BOOST2, cm->base_qindex) + : 0; + assert(mi_row >= 0 && mi_row < cm->mi_rows); + assert(mi_col >= 0 && mi_col < cm->mi_cols); + bl_index = mi_row * cm->mi_cols + mi_col; + // Loop through all MI blocks in superblock and update map. + xmis = AOMMIN(cm->mi_cols - mi_col, cm->mib_size); + ymis = AOMMIN(cm->mi_rows - mi_row, cm->mib_size); + for (y = 0; y < ymis; y++) { + for (x = 0; x < xmis; x++) { + const int bl_index2 = bl_index + y * cm->mi_cols + x; + // If the block is as a candidate for clean up then mark it + // for possible boost/refresh (segment 1). The segment id may get + // reset to 0 later if block gets coded anything other than ZEROMV. + if (cr->map[bl_index2] == 0) { + if (cr->last_coded_q_map[bl_index2] > qindex_thresh) sum_map++; + } else if (cr->map[bl_index2] < 0) { + cr->map[bl_index2]++; + } + } + } + // Enforce constant segment over superblock. + // If segment is at least half of superblock, set to 1. + if (sum_map >= xmis * ymis / 2) { + for (y = 0; y < ymis; y++) + for (x = 0; x < xmis; x++) { + seg_map[bl_index + y * cm->mi_cols + x] = CR_SEGMENT_ID_BOOST1; + } + cr->target_num_seg_blocks += xmis * ymis; + } + i++; + if (i == sbs_in_frame) { + i = 0; + } + } while (cr->target_num_seg_blocks < block_count && i != cr->sb_index); + cr->sb_index = i; +} + +// Set cyclic refresh parameters. +void av1_cyclic_refresh_update_parameters(AV1_COMP *const cpi) { + const RATE_CONTROL *const rc = &cpi->rc; + const AV1_COMMON *const cm = &cpi->common; + CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + cr->percent_refresh = 10; + cr->max_qdelta_perc = 50; + cr->time_for_refresh = 0; + // Use larger delta-qp (increase rate_ratio_qdelta) for first few (~4) + // periods of the refresh cycle, after a key frame. + if (rc->frames_since_key < 4 * cr->percent_refresh) + cr->rate_ratio_qdelta = 3.0; + else + cr->rate_ratio_qdelta = 2.0; + // Adjust some parameters for low resolutions at low bitrates. + if (cm->width <= 352 && cm->height <= 288 && rc->avg_frame_bandwidth < 3400) { + cr->motion_thresh = 4; + cr->rate_boost_fac = 10; + } else { + cr->motion_thresh = 32; + cr->rate_boost_fac = 17; + } +} + +// Setup cyclic background refresh: set delta q and segmentation map. +void av1_cyclic_refresh_setup(AV1_COMP *const cpi) { + AV1_COMMON *const cm = &cpi->common; + const RATE_CONTROL *const rc = &cpi->rc; + CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + struct segmentation *const seg = &cm->seg; + const int apply_cyclic_refresh = apply_cyclic_refresh_bitrate(cm, rc); + if (cm->current_video_frame == 0) cr->low_content_avg = 0.0; + // Don't apply refresh on key frame or enhancement layer frames. + if (!apply_cyclic_refresh || cm->frame_type == KEY_FRAME) { + // Set segmentation map to 0 and disable. + unsigned char *const seg_map = cpi->segmentation_map; + memset(seg_map, 0, cm->mi_rows * cm->mi_cols); + av1_disable_segmentation(&cm->seg); + if (cm->frame_type == KEY_FRAME) { + memset(cr->last_coded_q_map, MAXQ, + cm->mi_rows * cm->mi_cols * sizeof(*cr->last_coded_q_map)); + cr->sb_index = 0; + } + return; + } else { + int qindex_delta = 0; + int qindex2; + const double q = av1_convert_qindex_to_q(cm->base_qindex, cm->bit_depth); + aom_clear_system_state(); + // Set rate threshold to some multiple (set to 2 for now) of the target + // rate (target is given by sb64_target_rate and scaled by 256). + cr->thresh_rate_sb = ((int64_t)(rc->sb64_target_rate) << 8) << 2; + // Distortion threshold, quadratic in Q, scale factor to be adjusted. + // q will not exceed 457, so (q * q) is within 32bit; see: + // av1_convert_qindex_to_q(), av1_ac_quant(), ac_qlookup*[]. + cr->thresh_dist_sb = ((int64_t)(q * q)) << 2; + + // Set up segmentation. + // Clear down the segment map. + av1_enable_segmentation(&cm->seg); + av1_clearall_segfeatures(seg); + // Select delta coding method. + seg->abs_delta = SEGMENT_DELTADATA; + + // Note: setting temporal_update has no effect, as the seg-map coding method + // (temporal or spatial) is determined in + // av1_choose_segmap_coding_method(), + // based on the coding cost of each method. For error_resilient mode on the + // last_frame_seg_map is set to 0, so if temporal coding is used, it is + // relative to 0 previous map. + // seg->temporal_update = 0; + + // Segment BASE "Q" feature is disabled so it defaults to the baseline Q. + av1_disable_segfeature(seg, CR_SEGMENT_ID_BASE, SEG_LVL_ALT_Q); + // Use segment BOOST1 for in-frame Q adjustment. + av1_enable_segfeature(seg, CR_SEGMENT_ID_BOOST1, SEG_LVL_ALT_Q); + // Use segment BOOST2 for more aggressive in-frame Q adjustment. + av1_enable_segfeature(seg, CR_SEGMENT_ID_BOOST2, SEG_LVL_ALT_Q); + + // Set the q delta for segment BOOST1. + qindex_delta = compute_deltaq(cpi, cm->base_qindex, cr->rate_ratio_qdelta); + cr->qindex_delta[1] = qindex_delta; + + // Compute rd-mult for segment BOOST1. + qindex2 = clamp(cm->base_qindex + cm->y_dc_delta_q + qindex_delta, 0, MAXQ); + + cr->rdmult = av1_compute_rd_mult(cpi, qindex2); + + av1_set_segdata(seg, CR_SEGMENT_ID_BOOST1, SEG_LVL_ALT_Q, qindex_delta); + + // Set a more aggressive (higher) q delta for segment BOOST2. + qindex_delta = compute_deltaq( + cpi, cm->base_qindex, + AOMMIN(CR_MAX_RATE_TARGET_RATIO, + 0.1 * cr->rate_boost_fac * cr->rate_ratio_qdelta)); + cr->qindex_delta[2] = qindex_delta; + av1_set_segdata(seg, CR_SEGMENT_ID_BOOST2, SEG_LVL_ALT_Q, qindex_delta); + + // Update the segmentation and refresh map. + cyclic_refresh_update_map(cpi); + } +} + +int av1_cyclic_refresh_get_rdmult(const CYCLIC_REFRESH *cr) { + return cr->rdmult; +} + +void av1_cyclic_refresh_reset_resize(AV1_COMP *const cpi) { + const AV1_COMMON *const cm = &cpi->common; + CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + memset(cr->map, 0, cm->mi_rows * cm->mi_cols); + cr->sb_index = 0; + cpi->refresh_golden_frame = 1; +} diff --git a/third_party/aom/av1/encoder/aq_cyclicrefresh.h b/third_party/aom/av1/encoder/aq_cyclicrefresh.h new file mode 100644 index 0000000000..459ab80b8e --- /dev/null +++ b/third_party/aom/av1/encoder/aq_cyclicrefresh.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_AQ_CYCLICREFRESH_H_ +#define AV1_ENCODER_AQ_CYCLICREFRESH_H_ + +#include "av1/common/blockd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// The segment ids used in cyclic refresh: from base (no boost) to increasing +// boost (higher delta-qp). +#define CR_SEGMENT_ID_BASE 0 +#define CR_SEGMENT_ID_BOOST1 1 +#define CR_SEGMENT_ID_BOOST2 2 + +// Maximum rate target ratio for setting segment delta-qp. +#define CR_MAX_RATE_TARGET_RATIO 4.0 + +struct AV1_COMP; + +struct CYCLIC_REFRESH; +typedef struct CYCLIC_REFRESH CYCLIC_REFRESH; + +CYCLIC_REFRESH *av1_cyclic_refresh_alloc(int mi_rows, int mi_cols); + +void av1_cyclic_refresh_free(CYCLIC_REFRESH *cr); + +// Estimate the bits, incorporating the delta-q from segment 1, after encoding +// the frame. +int av1_cyclic_refresh_estimate_bits_at_q(const struct AV1_COMP *cpi, + double correction_factor); + +// Estimate the bits per mb, for a given q = i and a corresponding delta-q +// (for segment 1), prior to encoding the frame. +int av1_cyclic_refresh_rc_bits_per_mb(const struct AV1_COMP *cpi, int i, + double correction_factor); + +// Prior to coding a given prediction block, of size bsize at (mi_row, mi_col), +// check if we should reset the segment_id, and update the cyclic_refresh map +// and segmentation map. +void av1_cyclic_refresh_update_segment(const struct AV1_COMP *cpi, + MB_MODE_INFO *const mbmi, int mi_row, + int mi_col, BLOCK_SIZE bsize, + int64_t rate, int64_t dist, int skip); + +// Update the segmentation map, and related quantities: cyclic refresh map, +// refresh sb_index, and target number of blocks to be refreshed. +void av1_cyclic_refresh_update__map(struct AV1_COMP *const cpi); + +// Update the actual number of blocks that were applied the segment delta q. +void av1_cyclic_refresh_postencode(struct AV1_COMP *const cpi); + +// Set golden frame update interval, for 1 pass CBR mode. +void av1_cyclic_refresh_set_golden_update(struct AV1_COMP *const cpi); + +// Check if we should not update golden reference, based on past refresh stats. +void av1_cyclic_refresh_check_golden_update(struct AV1_COMP *const cpi); + +// Set/update global/frame level refresh parameters. +void av1_cyclic_refresh_update_parameters(struct AV1_COMP *const cpi); + +// Setup cyclic background refresh: set delta q and segmentation map. +void av1_cyclic_refresh_setup(struct AV1_COMP *const cpi); + +int av1_cyclic_refresh_get_rdmult(const CYCLIC_REFRESH *cr); + +void av1_cyclic_refresh_reset_resize(struct AV1_COMP *const cpi); + +static INLINE int cyclic_refresh_segment_id_boosted(int segment_id) { + return segment_id == CR_SEGMENT_ID_BOOST1 || + segment_id == CR_SEGMENT_ID_BOOST2; +} + +static INLINE int cyclic_refresh_segment_id(int segment_id) { + if (segment_id == CR_SEGMENT_ID_BOOST1) + return CR_SEGMENT_ID_BOOST1; + else if (segment_id == CR_SEGMENT_ID_BOOST2) + return CR_SEGMENT_ID_BOOST2; + else + return CR_SEGMENT_ID_BASE; +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_AQ_CYCLICREFRESH_H_ diff --git a/third_party/aom/av1/encoder/aq_variance.c b/third_party/aom/av1/encoder/aq_variance.c new file mode 100644 index 0000000000..ab9b3790bf --- /dev/null +++ b/third_party/aom/av1/encoder/aq_variance.c @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom_ports/mem.h" + +#include "av1/encoder/aq_variance.h" + +#include "av1/common/seg_common.h" +#include "av1/encoder/ratectrl.h" +#include "av1/encoder/rd.h" +#include "av1/encoder/segmentation.h" +#include "aom_ports/system_state.h" + +#define ENERGY_MIN (-4) +#define ENERGY_MAX (1) +#define ENERGY_SPAN (ENERGY_MAX - ENERGY_MIN + 1) +#define ENERGY_IN_BOUNDS(energy) \ + assert((energy) >= ENERGY_MIN && (energy) <= ENERGY_MAX) + +static const double rate_ratio[MAX_SEGMENTS] = { 2.5, 2.0, 1.5, 1.0, + 0.75, 1.0, 1.0, 1.0 }; +static const int segment_id[ENERGY_SPAN] = { 0, 1, 1, 2, 3, 4 }; + +#define SEGMENT_ID(i) segment_id[(i)-ENERGY_MIN] + +DECLARE_ALIGNED(16, static const uint8_t, av1_all_zeros[MAX_SB_SIZE]) = { 0 }; +#if CONFIG_HIGHBITDEPTH +DECLARE_ALIGNED(16, static const uint16_t, + av1_highbd_all_zeros[MAX_SB_SIZE]) = { 0 }; +#endif + +unsigned int av1_vaq_segment_id(int energy) { + ENERGY_IN_BOUNDS(energy); + return SEGMENT_ID(energy); +} + +void av1_vaq_frame_setup(AV1_COMP *cpi) { + AV1_COMMON *cm = &cpi->common; + struct segmentation *seg = &cm->seg; + int i; + + if (frame_is_intra_only(cm) || cm->error_resilient_mode || + cpi->refresh_alt_ref_frame || + (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref)) { + cpi->vaq_refresh = 1; + + av1_enable_segmentation(seg); + av1_clearall_segfeatures(seg); + + seg->abs_delta = SEGMENT_DELTADATA; + + aom_clear_system_state(); + + for (i = 0; i < MAX_SEGMENTS; ++i) { + int qindex_delta = + av1_compute_qdelta_by_rate(&cpi->rc, cm->frame_type, cm->base_qindex, + rate_ratio[i], cm->bit_depth); + + // We don't allow qindex 0 in a segment if the base value is not 0. + // Q index 0 (lossless) implies 4x4 encoding only and in AQ mode a segment + // Q delta is sometimes applied without going back around the rd loop. + // This could lead to an illegal combination of partition size and q. + if ((cm->base_qindex != 0) && ((cm->base_qindex + qindex_delta) == 0)) { + qindex_delta = -cm->base_qindex + 1; + } + + // No need to enable SEG_LVL_ALT_Q for this segment. + if (rate_ratio[i] == 1.0) { + continue; + } + + av1_set_segdata(seg, i, SEG_LVL_ALT_Q, qindex_delta); + av1_enable_segfeature(seg, i, SEG_LVL_ALT_Q); + } + } +} + +/* TODO(agrange, paulwilkins): The block_variance calls the unoptimized versions + * of variance() and highbd_8_variance(). It should not. + */ +static void aq_variance(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int w, int h, unsigned int *sse, + int *sum) { + int i, j; + + *sum = 0; + *sse = 0; + + for (i = 0; i < h; i++) { + for (j = 0; j < w; j++) { + const int diff = a[j] - b[j]; + *sum += diff; + *sse += diff * diff; + } + + a += a_stride; + b += b_stride; + } +} + +#if CONFIG_HIGHBITDEPTH +static void aq_highbd_variance64(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, int w, int h, + uint64_t *sse, uint64_t *sum) { + int i, j; + + uint16_t *a = CONVERT_TO_SHORTPTR(a8); + uint16_t *b = CONVERT_TO_SHORTPTR(b8); + *sum = 0; + *sse = 0; + + for (i = 0; i < h; i++) { + for (j = 0; j < w; j++) { + const int diff = a[j] - b[j]; + *sum += diff; + *sse += diff * diff; + } + a += a_stride; + b += b_stride; + } +} + +static void aq_highbd_8_variance(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, int w, int h, + unsigned int *sse, int *sum) { + uint64_t sse_long = 0; + uint64_t sum_long = 0; + aq_highbd_variance64(a8, a_stride, b8, b_stride, w, h, &sse_long, &sum_long); + *sse = (unsigned int)sse_long; + *sum = (int)sum_long; +} +#endif // CONFIG_HIGHBITDEPTH + +static unsigned int block_variance(const AV1_COMP *const cpi, MACROBLOCK *x, + BLOCK_SIZE bs) { + MACROBLOCKD *xd = &x->e_mbd; + unsigned int var, sse; + int right_overflow = + (xd->mb_to_right_edge < 0) ? ((-xd->mb_to_right_edge) >> 3) : 0; + int bottom_overflow = + (xd->mb_to_bottom_edge < 0) ? ((-xd->mb_to_bottom_edge) >> 3) : 0; + + if (right_overflow || bottom_overflow) { + const int bw = 8 * mi_size_wide[bs] - right_overflow; + const int bh = 8 * mi_size_high[bs] - bottom_overflow; + int avg; +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + aq_highbd_8_variance(x->plane[0].src.buf, x->plane[0].src.stride, + CONVERT_TO_BYTEPTR(av1_highbd_all_zeros), 0, bw, bh, + &sse, &avg); + sse >>= 2 * (xd->bd - 8); + avg >>= (xd->bd - 8); + } else { + aq_variance(x->plane[0].src.buf, x->plane[0].src.stride, av1_all_zeros, 0, + bw, bh, &sse, &avg); + } +#else + aq_variance(x->plane[0].src.buf, x->plane[0].src.stride, av1_all_zeros, 0, + bw, bh, &sse, &avg); +#endif // CONFIG_HIGHBITDEPTH + var = sse - (unsigned int)(((int64_t)avg * avg) / (bw * bh)); + return (unsigned int)((uint64_t)var * 256) / (bw * bh); + } else { +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + var = + cpi->fn_ptr[bs].vf(x->plane[0].src.buf, x->plane[0].src.stride, + CONVERT_TO_BYTEPTR(av1_highbd_all_zeros), 0, &sse); + } else { + var = cpi->fn_ptr[bs].vf(x->plane[0].src.buf, x->plane[0].src.stride, + av1_all_zeros, 0, &sse); + } +#else + var = cpi->fn_ptr[bs].vf(x->plane[0].src.buf, x->plane[0].src.stride, + av1_all_zeros, 0, &sse); +#endif // CONFIG_HIGHBITDEPTH + return (unsigned int)((uint64_t)var * 256) >> num_pels_log2_lookup[bs]; + } +} + +double av1_log_block_var(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs) { + unsigned int var = block_variance(cpi, x, bs); + aom_clear_system_state(); + return log(var + 1.0); +} + +#define DEFAULT_E_MIDPOINT 10.0 +int av1_block_energy(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs) { + double energy; + double energy_midpoint; + aom_clear_system_state(); + energy_midpoint = + (cpi->oxcf.pass == 2) ? cpi->twopass.mb_av_energy : DEFAULT_E_MIDPOINT; + energy = av1_log_block_var(cpi, x, bs) - energy_midpoint; + return clamp((int)round(energy), ENERGY_MIN, ENERGY_MAX); +} diff --git a/third_party/aom/av1/encoder/aq_variance.h b/third_party/aom/av1/encoder/aq_variance.h new file mode 100644 index 0000000000..05725c5def --- /dev/null +++ b/third_party/aom/av1/encoder/aq_variance.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_AQ_VARIANCE_H_ +#define AV1_ENCODER_AQ_VARIANCE_H_ + +#include "av1/encoder/encoder.h" + +#ifdef __cplusplus +extern "C" { +#endif + +unsigned int av1_vaq_segment_id(int energy); +void av1_vaq_frame_setup(AV1_COMP *cpi); + +int av1_block_energy(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs); +double av1_log_block_var(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_AQ_VARIANCE_H_ diff --git a/third_party/aom/av1/encoder/arm/neon/dct_neon.c b/third_party/aom/av1/encoder/arm/neon/dct_neon.c new file mode 100644 index 0000000000..f6ce24a3dd --- /dev/null +++ b/third_party/aom/av1/encoder/arm/neon/dct_neon.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./av1_rtcd.h" +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "av1/common/blockd.h" +#include "aom_dsp/txfm_common.h" + +void av1_fdct8x8_quant_neon(const int16_t *input, int stride, + int16_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, int16_t *qcoeff_ptr, + int16_t *dqcoeff_ptr, const int16_t *dequant_ptr, + uint16_t *eob_ptr, const int16_t *scan_ptr, + const int16_t *iscan_ptr) { + int16_t temp_buffer[64]; + (void)coeff_ptr; + + aom_fdct8x8_neon(input, temp_buffer, stride); + av1_quantize_fp_neon(temp_buffer, n_coeffs, skip_block, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, + dequant_ptr, eob_ptr, scan_ptr, iscan_ptr); +} diff --git a/third_party/aom/av1/encoder/arm/neon/error_neon.c b/third_party/aom/av1/encoder/arm/neon/error_neon.c new file mode 100644 index 0000000000..fe5233f89f --- /dev/null +++ b/third_party/aom/av1/encoder/arm/neon/error_neon.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./av1_rtcd.h" + +int64_t av1_block_error_fp_neon(const int16_t *coeff, const int16_t *dqcoeff, + int block_size) { + int64x2_t error = vdupq_n_s64(0); + + assert(block_size >= 8); + assert((block_size % 8) == 0); + + do { + const int16x8_t c = vld1q_s16(coeff); + const int16x8_t d = vld1q_s16(dqcoeff); + const int16x8_t diff = vsubq_s16(c, d); + const int16x4_t diff_lo = vget_low_s16(diff); + const int16x4_t diff_hi = vget_high_s16(diff); + // diff is 15-bits, the squares 30, so we can store 2 in 31-bits before + // accumulating them in 64-bits. + const int32x4_t err0 = vmull_s16(diff_lo, diff_lo); + const int32x4_t err1 = vmlal_s16(err0, diff_hi, diff_hi); + const int64x2_t err2 = vaddl_s32(vget_low_s32(err1), vget_high_s32(err1)); + error = vaddq_s64(error, err2); + coeff += 8; + dqcoeff += 8; + block_size -= 8; + } while (block_size != 0); + + return vgetq_lane_s64(error, 0) + vgetq_lane_s64(error, 1); +} diff --git a/third_party/aom/av1/encoder/arm/neon/quantize_neon.c b/third_party/aom/av1/encoder/arm/neon/quantize_neon.c new file mode 100644 index 0000000000..36e7d33702 --- /dev/null +++ b/third_party/aom/av1/encoder/arm/neon/quantize_neon.c @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include + +#include "aom_mem/aom_mem.h" + +#include "av1/common/quant_common.h" +#include "av1/common/seg_common.h" + +#include "av1/encoder/av1_quantize.h" +#include "av1/encoder/encoder.h" +#include "av1/encoder/rd.h" + +void av1_quantize_fp_neon(const int16_t *coeff_ptr, intptr_t count, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, int16_t *qcoeff_ptr, + int16_t *dqcoeff_ptr, const int16_t *dequant_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const int16_t *iscan) { + // TODO(jingning) Decide the need of these arguments after the + // quantization process is completed. + (void)zbin_ptr; + (void)quant_shift_ptr; + (void)scan; + + if (!skip_block) { + // Quantization pass: All coefficients with index >= zero_flag are + // skippable. Note: zero_flag can be zero. + int i; + const int16x8_t v_zero = vdupq_n_s16(0); + const int16x8_t v_one = vdupq_n_s16(1); + int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1); + int16x8_t v_round = vmovq_n_s16(round_ptr[1]); + int16x8_t v_quant = vmovq_n_s16(quant_ptr[1]); + int16x8_t v_dequant = vmovq_n_s16(dequant_ptr[1]); + // adjust for dc + v_round = vsetq_lane_s16(round_ptr[0], v_round, 0); + v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0); + v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0); + // process dc and the first seven ac coeffs + { + const int16x8_t v_iscan = vld1q_s16(&iscan[0]); + const int16x8_t v_coeff = vld1q_s16(&coeff_ptr[0]); + const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15); + const int16x8_t v_tmp = vabaq_s16(v_round, v_coeff, v_zero); + const int32x4_t v_tmp_lo = + vmull_s16(vget_low_s16(v_tmp), vget_low_s16(v_quant)); + const int32x4_t v_tmp_hi = + vmull_s16(vget_high_s16(v_tmp), vget_high_s16(v_quant)); + const int16x8_t v_tmp2 = + vcombine_s16(vshrn_n_s32(v_tmp_lo, 16), vshrn_n_s32(v_tmp_hi, 16)); + const uint16x8_t v_nz_mask = vceqq_s16(v_tmp2, v_zero); + const int16x8_t v_iscan_plus1 = vaddq_s16(v_iscan, v_one); + const int16x8_t v_nz_iscan = vbslq_s16(v_nz_mask, v_zero, v_iscan_plus1); + const int16x8_t v_qcoeff_a = veorq_s16(v_tmp2, v_coeff_sign); + const int16x8_t v_qcoeff = vsubq_s16(v_qcoeff_a, v_coeff_sign); + const int16x8_t v_dqcoeff = vmulq_s16(v_qcoeff, v_dequant); + v_eobmax_76543210 = vmaxq_s16(v_eobmax_76543210, v_nz_iscan); + vst1q_s16(&qcoeff_ptr[0], v_qcoeff); + vst1q_s16(&dqcoeff_ptr[0], v_dqcoeff); + v_round = vmovq_n_s16(round_ptr[1]); + v_quant = vmovq_n_s16(quant_ptr[1]); + v_dequant = vmovq_n_s16(dequant_ptr[1]); + } + // now process the rest of the ac coeffs + for (i = 8; i < count; i += 8) { + const int16x8_t v_iscan = vld1q_s16(&iscan[i]); + const int16x8_t v_coeff = vld1q_s16(&coeff_ptr[i]); + const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15); + const int16x8_t v_tmp = vabaq_s16(v_round, v_coeff, v_zero); + const int32x4_t v_tmp_lo = + vmull_s16(vget_low_s16(v_tmp), vget_low_s16(v_quant)); + const int32x4_t v_tmp_hi = + vmull_s16(vget_high_s16(v_tmp), vget_high_s16(v_quant)); + const int16x8_t v_tmp2 = + vcombine_s16(vshrn_n_s32(v_tmp_lo, 16), vshrn_n_s32(v_tmp_hi, 16)); + const uint16x8_t v_nz_mask = vceqq_s16(v_tmp2, v_zero); + const int16x8_t v_iscan_plus1 = vaddq_s16(v_iscan, v_one); + const int16x8_t v_nz_iscan = vbslq_s16(v_nz_mask, v_zero, v_iscan_plus1); + const int16x8_t v_qcoeff_a = veorq_s16(v_tmp2, v_coeff_sign); + const int16x8_t v_qcoeff = vsubq_s16(v_qcoeff_a, v_coeff_sign); + const int16x8_t v_dqcoeff = vmulq_s16(v_qcoeff, v_dequant); + v_eobmax_76543210 = vmaxq_s16(v_eobmax_76543210, v_nz_iscan); + vst1q_s16(&qcoeff_ptr[i], v_qcoeff); + vst1q_s16(&dqcoeff_ptr[i], v_dqcoeff); + } + { + const int16x4_t v_eobmax_3210 = vmax_s16( + vget_low_s16(v_eobmax_76543210), vget_high_s16(v_eobmax_76543210)); + const int64x1_t v_eobmax_xx32 = + vshr_n_s64(vreinterpret_s64_s16(v_eobmax_3210), 32); + const int16x4_t v_eobmax_tmp = + vmax_s16(v_eobmax_3210, vreinterpret_s16_s64(v_eobmax_xx32)); + const int64x1_t v_eobmax_xxx3 = + vshr_n_s64(vreinterpret_s64_s16(v_eobmax_tmp), 16); + const int16x4_t v_eobmax_final = + vmax_s16(v_eobmax_tmp, vreinterpret_s16_s64(v_eobmax_xxx3)); + + *eob_ptr = (uint16_t)vget_lane_s16(v_eobmax_final, 0); + } + } else { + memset(qcoeff_ptr, 0, count * sizeof(int16_t)); + memset(dqcoeff_ptr, 0, count * sizeof(int16_t)); + *eob_ptr = 0; + } +} diff --git a/third_party/aom/av1/encoder/av1_quantize.c b/third_party/aom/av1/encoder/av1_quantize.c new file mode 100644 index 0000000000..6cffac264b --- /dev/null +++ b/third_party/aom/av1/encoder/av1_quantize.c @@ -0,0 +1,1790 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/quantize.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +#include "av1/common/idct.h" +#include "av1/common/quant_common.h" +#include "av1/common/scan.h" +#include "av1/common/seg_common.h" + +#include "av1/encoder/av1_quantize.h" +#include "av1/encoder/encoder.h" +#include "av1/encoder/rd.h" + +#if CONFIG_NEW_QUANT +static INLINE int quantize_coeff_nuq( + const tran_low_t coeffv, const int16_t quant, const int16_t quant_shift, + const int16_t dequant, const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr) { + const int coeff = coeffv; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int i, q; + int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX); + for (i = 0; i < NUQ_KNOTS; i++) { + if (tmp < cuml_bins_ptr[i]) { + q = i; + break; + } + } + if (i == NUQ_KNOTS) { + tmp -= cuml_bins_ptr[NUQ_KNOTS - 1]; + q = NUQ_KNOTS + (((((tmp * quant) >> 16) + tmp) * quant_shift) >> 16); + } + if (q) { + *dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val); + *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; + *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; + } else { + *qcoeff_ptr = 0; + *dqcoeff_ptr = 0; + } + return (q != 0); +} + +static INLINE int quantize_coeff_bigtx_nuq( + const tran_low_t coeffv, const int16_t quant, const int16_t quant_shift, + const int16_t dequant, const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, int logsizeby16) { + const int coeff = coeffv; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int i, q; + int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX); + for (i = 0; i < NUQ_KNOTS; i++) { + if (tmp < ROUND_POWER_OF_TWO(cuml_bins_ptr[i], logsizeby16)) { + q = i; + break; + } + } + if (i == NUQ_KNOTS) { + tmp -= ROUND_POWER_OF_TWO(cuml_bins_ptr[NUQ_KNOTS - 1], logsizeby16); + q = NUQ_KNOTS + + (((((tmp * quant) >> 16) + tmp) * quant_shift) >> (16 - logsizeby16)); + } + if (q) { + *dqcoeff_ptr = ROUND_POWER_OF_TWO( + av1_dequant_abscoeff_nuq(q, dequant, dequant_val), logsizeby16); + // *dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val) >> + // (logsizeby16); + *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; + *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; + } else { + *qcoeff_ptr = 0; + *dqcoeff_ptr = 0; + } + return (q != 0); +} + +static INLINE int quantize_coeff_fp_nuq( + const tran_low_t coeffv, const int16_t quant, const int16_t dequant, + const tran_low_t *cuml_bins_ptr, const tran_low_t *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr) { + const int coeff = coeffv; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int i, q; + int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX); + for (i = 0; i < NUQ_KNOTS; i++) { + if (tmp < cuml_bins_ptr[i]) { + q = i; + break; + } + } + if (i == NUQ_KNOTS) { + q = NUQ_KNOTS + + ((((int64_t)tmp - cuml_bins_ptr[NUQ_KNOTS - 1]) * quant) >> 16); + } + if (q) { + *dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val); + *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; + *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; + } else { + *qcoeff_ptr = 0; + *dqcoeff_ptr = 0; + } + return (q != 0); +} + +static INLINE int quantize_coeff_bigtx_fp_nuq( + const tran_low_t coeffv, const int16_t quant, const int16_t dequant, + const tran_low_t *cuml_bins_ptr, const tran_low_t *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, int logsizeby16) { + const int coeff = coeffv; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int i, q; + int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX); + for (i = 0; i < NUQ_KNOTS; i++) { + if (tmp < ROUND_POWER_OF_TWO(cuml_bins_ptr[i], logsizeby16)) { + q = i; + break; + } + } + if (i == NUQ_KNOTS) { + q = NUQ_KNOTS + + ((((int64_t)tmp - + ROUND_POWER_OF_TWO(cuml_bins_ptr[NUQ_KNOTS - 1], logsizeby16)) * + quant) >> + (16 - logsizeby16)); + } + if (q) { + *dqcoeff_ptr = ROUND_POWER_OF_TWO( + av1_dequant_abscoeff_nuq(q, dequant, dequant_val), logsizeby16); + // *dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val) >> + // (logsizeby16); + *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; + *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; + } else { + *qcoeff_ptr = 0; + *dqcoeff_ptr = 0; + } + return (q != 0); +} + +void quantize_dc_nuq(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t quant, + const int16_t quant_shift, const int16_t dequant, + const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + const int rc = 0; + if (quantize_coeff_nuq(coeff_ptr[rc], quant, quant_shift, dequant, + cuml_bins_ptr, dequant_val, qcoeff_ptr, dqcoeff_ptr)) + eob = 0; + } + *eob_ptr = eob + 1; +} + +void quantize_dc_fp_nuq(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t quant, + const int16_t dequant, const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + const int rc = 0; + if (quantize_coeff_fp_nuq(coeff_ptr[rc], quant, dequant, cuml_bins_ptr, + dequant_val, qcoeff_ptr, dqcoeff_ptr)) + eob = 0; + } + *eob_ptr = eob + 1; +} + +void quantize_dc_32x32_nuq(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t quant, + const int16_t quant_shift, const int16_t dequant, + const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + const int rc = 0; + if (quantize_coeff_bigtx_nuq(coeff_ptr[rc], quant, quant_shift, dequant, + cuml_bins_ptr, dequant_val, qcoeff_ptr, + dqcoeff_ptr, av1_get_tx_scale(TX_32X32))) + eob = 0; + } + *eob_ptr = eob + 1; +} + +void quantize_dc_32x32_fp_nuq(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t quant, + const int16_t dequant, + const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + const int rc = 0; + if (quantize_coeff_bigtx_fp_nuq(coeff_ptr[rc], quant, dequant, + cuml_bins_ptr, dequant_val, qcoeff_ptr, + dqcoeff_ptr, av1_get_tx_scale(TX_32X32))) + eob = 0; + } + *eob_ptr = eob + 1; +} + +#if CONFIG_TX64X64 +void quantize_dc_64x64_nuq(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t quant, + const int16_t quant_shift, const int16_t dequant, + const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + const int rc = 0; + if (quantize_coeff_bigtx_nuq(coeff_ptr[rc], quant, quant_shift, dequant, + cuml_bins_ptr, dequant_val, qcoeff_ptr, + dqcoeff_ptr, av1_get_tx_scale(TX_64X64))) + eob = 0; + } + *eob_ptr = eob + 1; +} + +void quantize_dc_64x64_fp_nuq(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t quant, + const int16_t dequant, + const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + const int rc = 0; + if (quantize_coeff_bigtx_fp_nuq(coeff_ptr[rc], quant, dequant, + cuml_bins_ptr, dequant_val, qcoeff_ptr, + dqcoeff_ptr, av1_get_tx_scale(TX_64X64))) + eob = 0; + } + *eob_ptr = eob + 1; +} +#endif // CONFIG_TX64X64 + +void quantize_nuq_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const uint8_t *band) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + int i; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + if (quantize_coeff_nuq(coeff_ptr[rc], quant_ptr[rc != 0], + quant_shift_ptr[rc != 0], dequant_ptr[rc != 0], + cuml_bins_ptr[band[i]], dequant_val[band[i]], + &qcoeff_ptr[rc], &dqcoeff_ptr[rc])) + eob = i; + } + } + *eob_ptr = eob + 1; +} + +void quantize_fp_nuq_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *quant_ptr, + const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const uint8_t *band) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + int i; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + if (quantize_coeff_fp_nuq(coeff_ptr[rc], quant_ptr[rc != 0], + dequant_ptr[rc != 0], cuml_bins_ptr[band[i]], + dequant_val[band[i]], &qcoeff_ptr[rc], + &dqcoeff_ptr[rc])) + eob = i; + } + } + *eob_ptr = eob + 1; +} + +void quantize_32x32_nuq_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const uint8_t *band) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + int i; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + if (quantize_coeff_bigtx_nuq( + coeff_ptr[rc], quant_ptr[rc != 0], quant_shift_ptr[rc != 0], + dequant_ptr[rc != 0], cuml_bins_ptr[band[i]], + dequant_val[band[i]], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], + av1_get_tx_scale(TX_32X32))) + eob = i; + } + } + *eob_ptr = eob + 1; +} + +void quantize_32x32_fp_nuq_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *quant_ptr, + const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const uint8_t *band) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + int i; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + if (quantize_coeff_bigtx_fp_nuq( + coeff_ptr[rc], quant_ptr[rc != 0], dequant_ptr[rc != 0], + cuml_bins_ptr[band[i]], dequant_val[band[i]], &qcoeff_ptr[rc], + &dqcoeff_ptr[rc], av1_get_tx_scale(TX_32X32))) + eob = i; + } + } + *eob_ptr = eob + 1; +} + +#if CONFIG_TX64X64 +void quantize_64x64_nuq_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const uint8_t *band) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + int i; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + if (quantize_coeff_bigtx_nuq( + coeff_ptr[rc], quant_ptr[rc != 0], quant_shift_ptr[rc != 0], + dequant_ptr[rc != 0], cuml_bins_ptr[band[i]], + dequant_val[band[i]], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], + av1_get_tx_scale(TX_64X64))) + eob = i; + } + } + *eob_ptr = eob + 1; +} + +void quantize_64x64_fp_nuq_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *quant_ptr, + const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const uint8_t *band) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + int i; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + if (quantize_coeff_bigtx_fp_nuq( + coeff_ptr[rc], quant_ptr[rc != 0], dequant_ptr[rc != 0], + cuml_bins_ptr[band[i]], dequant_val[band[i]], &qcoeff_ptr[rc], + &dqcoeff_ptr[rc], av1_get_tx_scale(TX_64X64))) + eob = i; + } + } + *eob_ptr = eob + 1; +} +#endif // CONFIG_TX64X64 +#endif // CONFIG_NEW_QUANT + +void av1_quantize_skip(intptr_t n_coeffs, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr) { + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + *eob_ptr = 0; +} + +static void quantize_fp_helper_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, +#if CONFIG_AOM_QM + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, +#endif + int log_scale) { + int i, eob = -1; + // TODO(jingning) Decide the need of these arguments after the + // quantization process is completed. + (void)zbin_ptr; + (void)quant_shift_ptr; + (void)iscan; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + // Quantization pass: All coefficients with index >= zero_flag are + // skippable. Note: zero_flag can be zero. + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + const int coeff = coeff_ptr[rc]; +#if CONFIG_AOM_QM + const qm_val_t wt = qm_ptr[rc]; + const qm_val_t iwt = iqm_ptr[rc]; + const int dequant = + (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> + AOM_QM_BITS; +#endif + const int coeff_sign = (coeff >> 31); + int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int tmp32 = 0; +#if CONFIG_AOM_QM + if (abs_coeff * wt >= + (dequant_ptr[rc != 0] << (AOM_QM_BITS - (1 + log_scale)))) { +#else + if (abs_coeff >= (dequant_ptr[rc != 0] >> (1 + log_scale))) { +#endif + abs_coeff += ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale); + abs_coeff = clamp(abs_coeff, INT16_MIN, INT16_MAX); +#if CONFIG_AOM_QM + tmp32 = (int)((abs_coeff * wt * quant_ptr[rc != 0]) >> + ((16 - log_scale) + AOM_QM_BITS)); + qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant / (1 << log_scale); +#else + tmp32 = (int)((abs_coeff * quant_ptr[rc != 0]) >> (16 - log_scale)); + qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = + qcoeff_ptr[rc] * dequant_ptr[rc != 0] / (1 << log_scale); +#endif + } + + if (tmp32) eob = i; + } + } + *eob_ptr = eob + 1; +} + +void av1_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const int16_t *iscan +#if CONFIG_AOM_QM + , + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr +#endif + ) { + quantize_fp_helper_c(coeff_ptr, n_coeffs, skip_block, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, + dequant_ptr, eob_ptr, scan, iscan, +#if CONFIG_AOM_QM + qm_ptr, iqm_ptr, +#endif + 0); +} + +void av1_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan +#if CONFIG_AOM_QM + , + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr +#endif + ) { + quantize_fp_helper_c(coeff_ptr, n_coeffs, skip_block, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, + dequant_ptr, eob_ptr, scan, iscan, +#if CONFIG_AOM_QM + qm_ptr, iqm_ptr, +#endif + 1); +} + +#if CONFIG_TX64X64 +void av1_quantize_fp_64x64_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan +#if CONFIG_AOM_QM + , + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr +#endif + ) { + quantize_fp_helper_c(coeff_ptr, n_coeffs, skip_block, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, + dequant_ptr, eob_ptr, scan, iscan, +#if CONFIG_AOM_QM + qm_ptr, iqm_ptr, +#endif + 2); +} +#endif // CONFIG_TX64X64 + +void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, const QUANT_PARAM *qparam) { + // obsolete skip_block + const int skip_block = 0; +#if CONFIG_AOM_QM + const qm_val_t *qm_ptr = qparam->qmatrix; + const qm_val_t *iqm_ptr = qparam->iqmatrix; +#endif // CONFIG_AOM_QM + + switch (qparam->log_scale) { + case 0: + if (n_coeffs < 16) { + // TODO(jingning): Need SIMD implementation for smaller block size + // quantization. + quantize_fp_helper_c(coeff_ptr, n_coeffs, skip_block, p->zbin, + p->round_fp, p->quant_fp, p->quant_shift, + qcoeff_ptr, dqcoeff_ptr, pd->dequant, eob_ptr, + sc->scan, sc->iscan, +#if CONFIG_AOM_QM + qm_ptr, iqm_ptr, +#endif + qparam->log_scale); + } else { + av1_quantize_fp(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round_fp, + p->quant_fp, p->quant_shift, qcoeff_ptr, dqcoeff_ptr, + pd->dequant, eob_ptr, sc->scan, sc->iscan +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + } + break; + case 1: + av1_quantize_fp_32x32(coeff_ptr, n_coeffs, skip_block, p->zbin, + p->round_fp, p->quant_fp, p->quant_shift, + qcoeff_ptr, dqcoeff_ptr, pd->dequant, eob_ptr, + sc->scan, sc->iscan +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + break; +#if CONFIG_TX64X64 + case 2: + av1_quantize_fp_64x64(coeff_ptr, n_coeffs, skip_block, p->zbin, + p->round_fp, p->quant_fp, p->quant_shift, + qcoeff_ptr, dqcoeff_ptr, pd->dequant, eob_ptr, + sc->scan, sc->iscan +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + break; +#endif // CONFIG_TX64X64 + default: assert(0); + } +} + +void av1_quantize_b_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr, const SCAN_ORDER *sc, + const QUANT_PARAM *qparam) { + // obsolete skip_block + const int skip_block = 0; +#if CONFIG_AOM_QM + const qm_val_t *qm_ptr = qparam->qmatrix; + const qm_val_t *iqm_ptr = qparam->iqmatrix; +#endif // CONFIG_AOM_QM + + switch (qparam->log_scale) { + case 0: + aom_quantize_b(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round, + p->quant, p->quant_shift, qcoeff_ptr, dqcoeff_ptr, + pd->dequant, eob_ptr, sc->scan, sc->iscan +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + break; + case 1: + aom_quantize_b_32x32(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round, + p->quant, p->quant_shift, qcoeff_ptr, dqcoeff_ptr, + pd->dequant, eob_ptr, sc->scan, sc->iscan +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + break; +#if CONFIG_TX64X64 + case 2: + aom_quantize_b_64x64(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round, + p->quant, p->quant_shift, qcoeff_ptr, dqcoeff_ptr, + pd->dequant, eob_ptr, sc->scan, sc->iscan +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + break; +#endif // CONFIG_TX64X64 + default: assert(0); + } +} + +void av1_quantize_dc_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, const QUANT_PARAM *qparam) { + // obsolete skip_block + const int skip_block = 0; +#if CONFIG_AOM_QM + const qm_val_t *qm_ptr = qparam->qmatrix; + const qm_val_t *iqm_ptr = qparam->iqmatrix; +#endif // CONFIG_AOM_QM + + (void)sc; + + switch (qparam->log_scale) { + case 0: + aom_quantize_dc(coeff_ptr, (int)n_coeffs, skip_block, p->round, + p->quant_fp[0], qcoeff_ptr, dqcoeff_ptr, pd->dequant[0], + eob_ptr +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + break; + case 1: + aom_quantize_dc_32x32(coeff_ptr, skip_block, p->round, p->quant_fp[0], + qcoeff_ptr, dqcoeff_ptr, pd->dequant[0], eob_ptr +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + break; +#if CONFIG_TX64X64 + aom_quantize_dc_64x64(coeff_ptr, skip_block, p->round, p->quant_fp[0], + qcoeff_ptr, dqcoeff_ptr, pd->dequant[0], eob_ptr +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + case 2: break; +#endif // CONFIG_TX64X64 + default: assert(0); + } +} + +#if CONFIG_NEW_QUANT +void av1_quantize_b_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, + const QUANT_PARAM *qparam) { + // obsolete skip_block + const int skip_block = 0; + const uint8_t *band = get_band_translate(qparam->tx_size); + int dq = qparam->dq; + + switch (qparam->log_scale) { + case 0: + quantize_nuq(coeff_ptr, n_coeffs, skip_block, p->quant, p->quant_shift, + pd->dequant, + (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], + (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], + qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band); + break; + case 1: + quantize_32x32_nuq(coeff_ptr, n_coeffs, skip_block, p->quant, + p->quant_shift, pd->dequant, + (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], + (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], + qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band); + break; +#if CONFIG_TX64X64 + case 2: + quantize_64x64_nuq(coeff_ptr, n_coeffs, skip_block, p->quant, + p->quant_shift, pd->dequant, + (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], + (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], + qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band); + break; +#endif // CONFIG_TX64X64 + default: assert(0); + } +} + +void av1_quantize_fp_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, + const QUANT_PARAM *qparam) { + // obsolete skip_block + const int skip_block = 0; + const uint8_t *band = get_band_translate(qparam->tx_size); + int dq = qparam->dq; + + switch (qparam->log_scale) { + case 0: + quantize_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant, + (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], + (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], + qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band); + break; + case 1: + quantize_32x32_fp_nuq( + coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant, + (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], + (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr, + dqcoeff_ptr, eob_ptr, sc->scan, band); + break; +#if CONFIG_TX64X64 + case 2: + quantize_64x64_fp_nuq( + coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant, + (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], + (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr, + dqcoeff_ptr, eob_ptr, sc->scan, band); + break; +#endif // CONFIG_TX64X64 + default: assert(0); + } +} + +void av1_quantize_dc_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, + const QUANT_PARAM *qparam) { + // obsolete skip_block + const int skip_block = 0; + int dq = qparam->dq; + (void)sc; + + switch (qparam->log_scale) { + case 0: + quantize_dc_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp[0], + pd->dequant[0], p->cuml_bins_nuq[dq][0], + pd->dequant_val_nuq[dq][0], qcoeff_ptr, dqcoeff_ptr, + eob_ptr); + break; + case 1: + quantize_dc_32x32_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp[0], + pd->dequant[0], p->cuml_bins_nuq[dq][0], + pd->dequant_val_nuq[dq][0], qcoeff_ptr, + dqcoeff_ptr, eob_ptr); + break; +#if CONFIG_TX64X64 + case 2: + quantize_dc_64x64_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp[0], + pd->dequant[0], p->cuml_bins_nuq[dq][0], + pd->dequant_val_nuq[dq][0], qcoeff_ptr, + dqcoeff_ptr, eob_ptr); + break; +#endif // CONFIG_TX64X64 + default: assert(0); + } +} +#endif // CONFIG_NEW_QUANT + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_quantize_fp_facade(const tran_low_t *coeff_ptr, + intptr_t n_coeffs, const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, + const QUANT_PARAM *qparam) { + // obsolete skip_block + const int skip_block = 0; +#if CONFIG_AOM_QM + const qm_val_t *qm_ptr = qparam->qmatrix; + const qm_val_t *iqm_ptr = qparam->iqmatrix; +#endif // CONFIG_AOM_QM + + if (n_coeffs < 16) { + // TODO(jingning): Need SIMD implementation for smaller block size + // quantization. + av1_highbd_quantize_fp_c(coeff_ptr, n_coeffs, skip_block, p->zbin, + p->round_fp, p->quant_fp, p->quant_shift, + qcoeff_ptr, dqcoeff_ptr, pd->dequant, eob_ptr, + sc->scan, sc->iscan, +#if CONFIG_AOM_QM + qm_ptr, iqm_ptr, +#endif + qparam->log_scale); + return; + } + + av1_highbd_quantize_fp(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round_fp, + p->quant_fp, p->quant_shift, qcoeff_ptr, dqcoeff_ptr, + pd->dequant, eob_ptr, sc->scan, sc->iscan, +#if CONFIG_AOM_QM + qm_ptr, iqm_ptr, +#endif + qparam->log_scale); +} + +void av1_highbd_quantize_b_facade(const tran_low_t *coeff_ptr, + intptr_t n_coeffs, const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, + const QUANT_PARAM *qparam) { + // obsolete skip_block + const int skip_block = 0; +#if CONFIG_AOM_QM + const qm_val_t *qm_ptr = qparam->qmatrix; + const qm_val_t *iqm_ptr = qparam->iqmatrix; +#endif // CONFIG_AOM_QM + + switch (qparam->log_scale) { + case 0: + aom_highbd_quantize_b(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round, + p->quant, p->quant_shift, qcoeff_ptr, dqcoeff_ptr, + pd->dequant, eob_ptr, sc->scan, sc->iscan +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + break; + case 1: + aom_highbd_quantize_b_32x32(coeff_ptr, n_coeffs, skip_block, p->zbin, + p->round, p->quant, p->quant_shift, + qcoeff_ptr, dqcoeff_ptr, pd->dequant, eob_ptr, + sc->scan, sc->iscan +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + break; +#if CONFIG_TX64X64 + case 2: + aom_highbd_quantize_b_64x64(coeff_ptr, n_coeffs, skip_block, p->zbin, + p->round, p->quant, p->quant_shift, + qcoeff_ptr, dqcoeff_ptr, pd->dequant, eob_ptr, + sc->scan, sc->iscan +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + break; +#endif // CONFIG_TX64X64 + default: assert(0); + } +} + +#if CONFIG_HIGHBITDEPTH +static INLINE void highbd_quantize_dc( + const tran_low_t *coeff_ptr, int n_coeffs, int skip_block, + const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t dequant_ptr, uint16_t *eob_ptr, +#if CONFIG_AOM_QM + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, +#endif + const int log_scale) { + int eob = -1; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); +#if CONFIG_AOM_QM + (void)qm_ptr; + (void)iqm_ptr; +#endif + if (!skip_block) { + const int coeff = coeff_ptr[0]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp = abs_coeff + round_ptr[0]; + const uint32_t abs_qcoeff = (uint32_t)((tmp * quant) >> (16 - log_scale)); + qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant_ptr / (1 << log_scale); + if (abs_qcoeff) eob = 0; + } + *eob_ptr = eob + 1; +} +#endif // CONFIG_HIGHBITDEPTH + +void av1_highbd_quantize_dc_facade(const tran_low_t *coeff_ptr, + intptr_t n_coeffs, const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, + const QUANT_PARAM *qparam) { + // obsolete skip_block + const int skip_block = 0; +#if CONFIG_AOM_QM + const qm_val_t *qm_ptr = qparam->qmatrix; + const qm_val_t *iqm_ptr = qparam->iqmatrix; +#endif // CONFIG_AOM_QM + + (void)sc; + + highbd_quantize_dc(coeff_ptr, (int)n_coeffs, skip_block, p->round, + p->quant_fp[0], qcoeff_ptr, dqcoeff_ptr, pd->dequant[0], + eob_ptr, +#if CONFIG_AOM_QM + qm_ptr, iqm_ptr, +#endif + qparam->log_scale); +} + +#if CONFIG_NEW_QUANT +static INLINE int highbd_quantize_coeff_nuq( + const tran_low_t coeffv, const int16_t quant, const int16_t quant_shift, + const int16_t dequant, const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr) { + const int coeff = coeffv; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int i, q; + int64_t tmp = clamp(abs_coeff, INT32_MIN, INT32_MAX); + for (i = 0; i < NUQ_KNOTS; i++) { + if (tmp < cuml_bins_ptr[i]) { + q = i; + break; + } + } + if (i == NUQ_KNOTS) { + tmp -= cuml_bins_ptr[NUQ_KNOTS - 1]; + q = NUQ_KNOTS + (int)(((((tmp * quant) >> 16) + tmp) * quant_shift) >> 16); + } + if (q) { + *dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val); + *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; + *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; + } else { + *qcoeff_ptr = 0; + *dqcoeff_ptr = 0; + } + return (q != 0); +} + +static INLINE int highbd_quantize_coeff_fp_nuq( + const tran_low_t coeffv, const int16_t quant, const int16_t dequant, + const tran_low_t *cuml_bins_ptr, const tran_low_t *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr) { + const int coeff = coeffv; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int i, q; + int64_t tmp = clamp(abs_coeff, INT32_MIN, INT32_MAX); + for (i = 0; i < NUQ_KNOTS; i++) { + if (tmp < cuml_bins_ptr[i]) { + q = i; + break; + } + } + if (i == NUQ_KNOTS) { + q = NUQ_KNOTS + (int)(((tmp - cuml_bins_ptr[NUQ_KNOTS - 1]) * quant) >> 16); + } + if (q) { + *dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val); + *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; + *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; + } else { + *qcoeff_ptr = 0; + *dqcoeff_ptr = 0; + } + return (q != 0); +} + +static INLINE int highbd_quantize_coeff_bigtx_fp_nuq( + const tran_low_t coeffv, const int16_t quant, const int16_t dequant, + const tran_low_t *cuml_bins_ptr, const tran_low_t *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, int logsizeby16) { + const int coeff = coeffv; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int i, q; + int64_t tmp = clamp(abs_coeff, INT32_MIN, INT32_MAX); + for (i = 0; i < NUQ_KNOTS; i++) { + if (tmp < ROUND_POWER_OF_TWO(cuml_bins_ptr[i], logsizeby16)) { + q = i; + break; + } + } + if (i == NUQ_KNOTS) { + q = NUQ_KNOTS + + (int)(((tmp - + ROUND_POWER_OF_TWO(cuml_bins_ptr[NUQ_KNOTS - 1], logsizeby16)) * + quant) >> + (16 - logsizeby16)); + } + if (q) { + *dqcoeff_ptr = ROUND_POWER_OF_TWO( + av1_dequant_abscoeff_nuq(q, dequant, dequant_val), logsizeby16); + *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; + *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; + } else { + *qcoeff_ptr = 0; + *dqcoeff_ptr = 0; + } + return (q != 0); +} + +static INLINE int highbd_quantize_coeff_bigtx_nuq( + const tran_low_t coeffv, const int16_t quant, const int16_t quant_shift, + const int16_t dequant, const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, int logsizeby16) { + const int coeff = coeffv; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int i, q; + int64_t tmp = clamp(abs_coeff, INT32_MIN, INT32_MAX); + for (i = 0; i < NUQ_KNOTS; i++) { + if (tmp < ROUND_POWER_OF_TWO(cuml_bins_ptr[i], logsizeby16)) { + q = i; + break; + } + } + if (i == NUQ_KNOTS) { + tmp -= ROUND_POWER_OF_TWO(cuml_bins_ptr[NUQ_KNOTS - 1], logsizeby16); + q = NUQ_KNOTS + (int)(((((tmp * quant) >> 16) + tmp) * quant_shift) >> + (16 - logsizeby16)); + } + if (q) { + *dqcoeff_ptr = ROUND_POWER_OF_TWO( + av1_dequant_abscoeff_nuq(q, dequant, dequant_val), logsizeby16); + *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; + *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; + } else { + *qcoeff_ptr = 0; + *dqcoeff_ptr = 0; + } + return (q != 0); +} + +void highbd_quantize_dc_nuq(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t quant, + const int16_t quant_shift, const int16_t dequant, + const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + const int rc = 0; + if (highbd_quantize_coeff_nuq(coeff_ptr[rc], quant, quant_shift, dequant, + cuml_bins_ptr, dequant_val, qcoeff_ptr, + dqcoeff_ptr)) + eob = 0; + } + *eob_ptr = eob + 1; +} + +void highbd_quantize_dc_fp_nuq(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t quant, + const int16_t dequant, + const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + const int rc = 0; + if (highbd_quantize_coeff_fp_nuq(coeff_ptr[rc], quant, dequant, + cuml_bins_ptr, dequant_val, qcoeff_ptr, + dqcoeff_ptr)) + eob = 0; + } + *eob_ptr = eob + 1; +} + +void highbd_quantize_nuq_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const uint8_t *band) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + int i; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + if (highbd_quantize_coeff_nuq( + coeff_ptr[rc], quant_ptr[rc != 0], quant_shift_ptr[rc != 0], + dequant_ptr[rc != 0], cuml_bins_ptr[band[i]], + dequant_val[band[i]], &qcoeff_ptr[rc], &dqcoeff_ptr[rc])) + eob = i; + } + } + *eob_ptr = eob + 1; +} + +void highbd_quantize_32x32_nuq_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, + tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const int16_t *scan, const uint8_t *band) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + int i; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + if (highbd_quantize_coeff_bigtx_nuq( + coeff_ptr[rc], quant_ptr[rc != 0], quant_shift_ptr[rc != 0], + dequant_ptr[rc != 0], cuml_bins_ptr[band[i]], + dequant_val[band[i]], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], + av1_get_tx_scale(TX_32X32))) + eob = i; + } + } + *eob_ptr = eob + 1; +} + +void highbd_quantize_32x32_fp_nuq_c(const tran_low_t *coeff_ptr, + intptr_t n_coeffs, int skip_block, + const int16_t *quant_ptr, + const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, + tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const int16_t *scan, const uint8_t *band) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + int i; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + if (highbd_quantize_coeff_bigtx_fp_nuq( + coeff_ptr[rc], quant_ptr[rc != 0], dequant_ptr[rc != 0], + cuml_bins_ptr[band[i]], dequant_val[band[i]], &qcoeff_ptr[rc], + &dqcoeff_ptr[rc], av1_get_tx_scale(TX_32X32))) + eob = i; + } + } + *eob_ptr = eob + 1; +} + +#if CONFIG_TX64X64 +void highbd_quantize_64x64_nuq_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, + tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const int16_t *scan, const uint8_t *band) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + int i; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + if (highbd_quantize_coeff_bigtx_nuq( + coeff_ptr[rc], quant_ptr[rc != 0], quant_shift_ptr[rc != 0], + dequant_ptr[rc != 0], cuml_bins_ptr[band[i]], + dequant_val[band[i]], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], + av1_get_tx_scale(TX_64X64))) + eob = i; + } + } + *eob_ptr = eob + 1; +} + +void highbd_quantize_64x64_fp_nuq_c(const tran_low_t *coeff_ptr, + intptr_t n_coeffs, int skip_block, + const int16_t *quant_ptr, + const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, + tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const int16_t *scan, const uint8_t *band) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + int i; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + if (highbd_quantize_coeff_bigtx_fp_nuq( + coeff_ptr[rc], quant_ptr[rc != 0], dequant_ptr[rc != 0], + cuml_bins_ptr[band[i]], dequant_val[band[i]], &qcoeff_ptr[rc], + &dqcoeff_ptr[rc], av1_get_tx_scale(TX_64X64))) + eob = i; + } + } + *eob_ptr = eob + 1; +} +#endif // CONFIG_TX64X64 + +void highbd_quantize_fp_nuq_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *quant_ptr, + const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const uint8_t *band) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + int i; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + if (highbd_quantize_coeff_fp_nuq( + coeff_ptr[rc], quant_ptr[rc != 0], dequant_ptr[rc != 0], + cuml_bins_ptr[band[i]], dequant_val[band[i]], &qcoeff_ptr[rc], + &dqcoeff_ptr[rc])) + eob = i; + } + } + *eob_ptr = eob + 1; +} + +void highbd_quantize_dc_32x32_nuq( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t quant, const int16_t quant_shift, const int16_t dequant, + const tran_low_t *cuml_bins_ptr, const tran_low_t *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + const int rc = 0; + if (highbd_quantize_coeff_bigtx_nuq( + coeff_ptr[rc], quant, quant_shift, dequant, cuml_bins_ptr, + dequant_val, qcoeff_ptr, dqcoeff_ptr, av1_get_tx_scale(TX_32X32))) + eob = 0; + } + *eob_ptr = eob + 1; +} + +void highbd_quantize_dc_32x32_fp_nuq( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t quant, const int16_t dequant, const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + const int rc = 0; + if (highbd_quantize_coeff_bigtx_fp_nuq( + coeff_ptr[rc], quant, dequant, cuml_bins_ptr, dequant_val, + qcoeff_ptr, dqcoeff_ptr, av1_get_tx_scale(TX_32X32))) + eob = 0; + } + *eob_ptr = eob + 1; +} + +#if CONFIG_TX64X64 +void highbd_quantize_dc_64x64_nuq( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t quant, const int16_t quant_shift, const int16_t dequant, + const tran_low_t *cuml_bins_ptr, const tran_low_t *dequant_val, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + const int rc = 0; + if (highbd_quantize_coeff_bigtx_nuq( + coeff_ptr[rc], quant, quant_shift, dequant, cuml_bins_ptr, + dequant_val, qcoeff_ptr, dqcoeff_ptr, av1_get_tx_scale(TX_64X64))) + eob = 0; + } + *eob_ptr = eob + 1; +} + +void highbd_quantize_dc_64x64_fp_nuq( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t quant, const int16_t dequant, const tran_low_t *cuml_bins_ptr, + const tran_low_t *dequant_val, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr) { + int eob = -1; + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + if (!skip_block) { + const int rc = 0; + if (highbd_quantize_coeff_bigtx_fp_nuq( + coeff_ptr[rc], quant, dequant, cuml_bins_ptr, dequant_val, + qcoeff_ptr, dqcoeff_ptr, av1_get_tx_scale(TX_64X64))) + eob = 0; + } + *eob_ptr = eob + 1; +} +#endif // CONFIG_TX64X64 + +void av1_highbd_quantize_b_nuq_facade( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc, + const QUANT_PARAM *qparam) { + // obsolete skip_block + const int skip_block = 0; + const uint8_t *band = get_band_translate(qparam->tx_size); + const int dq = qparam->dq; + + switch (qparam->log_scale) { + case 0: + highbd_quantize_nuq(coeff_ptr, n_coeffs, skip_block, p->quant, + p->quant_shift, pd->dequant, + (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], + (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], + qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band); + break; + case 1: + highbd_quantize_32x32_nuq( + coeff_ptr, n_coeffs, skip_block, p->quant, p->quant_shift, + pd->dequant, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], + (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr, + dqcoeff_ptr, eob_ptr, sc->scan, band); + break; +#if CONFIG_TX64X64 + case 2: + highbd_quantize_64x64_nuq( + coeff_ptr, n_coeffs, skip_block, p->quant, p->quant_shift, + pd->dequant, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], + (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr, + dqcoeff_ptr, eob_ptr, sc->scan, band); + break; +#endif // CONFIG_TX64X64 + default: assert(0); + } +} + +void av1_highbd_quantize_fp_nuq_facade( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc, + const QUANT_PARAM *qparam) { + // obsolete skip_block + const int skip_block = 0; + const uint8_t *band = get_band_translate(qparam->tx_size); + const int dq = qparam->dq; + + switch (qparam->log_scale) { + case 0: + highbd_quantize_fp_nuq( + coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant, + (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], + (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr, + dqcoeff_ptr, eob_ptr, sc->scan, band); + break; + case 1: + highbd_quantize_32x32_fp_nuq( + coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant, + (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], + (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr, + dqcoeff_ptr, eob_ptr, sc->scan, band); + break; +#if CONFIG_TX64X64 + case 2: + highbd_quantize_64x64_fp_nuq( + coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant, + (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], + (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr, + dqcoeff_ptr, eob_ptr, sc->scan, band); + break; +#endif // CONFIG_TX64X64 + default: assert(0); + } +} + +void av1_highbd_quantize_dc_nuq_facade( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc, + const QUANT_PARAM *qparam) { + // obsolete skip_block + const int skip_block = 0; + const int dq = qparam->dq; + (void)sc; + + switch (qparam->log_scale) { + case 0: + highbd_quantize_dc_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp[0], + pd->dequant[0], p->cuml_bins_nuq[dq][0], + pd->dequant_val_nuq[dq][0], qcoeff_ptr, + dqcoeff_ptr, eob_ptr); + break; + case 1: + highbd_quantize_dc_32x32_fp_nuq( + coeff_ptr, n_coeffs, skip_block, p->quant_fp[0], pd->dequant[0], + p->cuml_bins_nuq[dq][0], pd->dequant_val_nuq[dq][0], qcoeff_ptr, + dqcoeff_ptr, eob_ptr); + break; +#if CONFIG_TX64X64 + case 2: + highbd_quantize_dc_64x64_fp_nuq( + coeff_ptr, n_coeffs, skip_block, p->quant_fp[0], pd->dequant[0], + p->cuml_bins_nuq[dq][0], pd->dequant_val_nuq[dq][0], qcoeff_ptr, + dqcoeff_ptr, eob_ptr); + break; +#endif // CONFIG_TX64X64 + default: assert(0); + } +} +#endif // CONFIG_NEW_QUANT +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t count, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, + const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, +#if CONFIG_AOM_QM + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, +#endif + int log_scale) { + int i; + int eob = -1; + const int scale = 1 << log_scale; + const int shift = 16 - log_scale; + // TODO(jingning) Decide the need of these arguments after the + // quantization process is completed. + (void)zbin_ptr; + (void)quant_shift_ptr; + (void)iscan; + + memset(qcoeff_ptr, 0, count * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, count * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + // Quantization pass: All coefficients with index >= zero_flag are + // skippable. Note: zero_flag can be zero. + for (i = 0; i < count; i++) { + const int rc = scan[i]; + const int coeff = coeff_ptr[rc]; +#if CONFIG_AOM_QM + const qm_val_t wt = qm_ptr[rc]; + const qm_val_t iwt = iqm_ptr[rc]; + const int dequant = + (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> + AOM_QM_BITS; +#endif + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp = abs_coeff + round_ptr[rc != 0]; +#if CONFIG_AOM_QM + const uint32_t abs_qcoeff = + (uint32_t)((tmp * quant_ptr[rc != 0] * wt) >> (shift + AOM_QM_BITS)); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant / scale; +#else + const uint32_t abs_qcoeff = + (uint32_t)((tmp * quant_ptr[rc != 0]) >> shift); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / scale; +#endif + if (abs_qcoeff) eob = i; + } + } + *eob_ptr = eob + 1; +} + +#endif // CONFIG_HIGHBITDEPTH + +static void invert_quant(int16_t *quant, int16_t *shift, int d) { + uint32_t t; + int l, m; + t = d; + for (l = 0; t > 1; l++) t >>= 1; + m = 1 + (1 << (16 + l)) / d; + *quant = (int16_t)(m - (1 << 16)); + *shift = 1 << (16 - l); +} + +static int get_qzbin_factor(int q, aom_bit_depth_t bit_depth) { + const int quant = av1_dc_quant(q, 0, bit_depth); +#if CONFIG_HIGHBITDEPTH + switch (bit_depth) { + case AOM_BITS_8: return q == 0 ? 64 : (quant < 148 ? 84 : 80); + case AOM_BITS_10: return q == 0 ? 64 : (quant < 592 ? 84 : 80); + case AOM_BITS_12: return q == 0 ? 64 : (quant < 2368 ? 84 : 80); + default: + assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12"); + return -1; + } +#else + (void)bit_depth; + return q == 0 ? 64 : (quant < 148 ? 84 : 80); +#endif +} + +void av1_init_quantizer(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + QUANTS *const quants = &cpi->quants; + int i, q, quant; +#if CONFIG_NEW_QUANT + int dq; +#endif + + for (q = 0; q < QINDEX_RANGE; q++) { + const int qzbin_factor = get_qzbin_factor(q, cm->bit_depth); + const int qrounding_factor = q == 0 ? 64 : 48; + + for (i = 0; i < 2; ++i) { + int qrounding_factor_fp = 64; + // y + quant = i == 0 ? av1_dc_quant(q, cm->y_dc_delta_q, cm->bit_depth) + : av1_ac_quant(q, 0, cm->bit_depth); + invert_quant(&quants->y_quant[q][i], &quants->y_quant_shift[q][i], quant); + quants->y_quant_fp[q][i] = (1 << 16) / quant; + quants->y_round_fp[q][i] = (qrounding_factor_fp * quant) >> 7; + quants->y_zbin[q][i] = ROUND_POWER_OF_TWO(qzbin_factor * quant, 7); + quants->y_round[q][i] = (qrounding_factor * quant) >> 7; + cpi->y_dequant[q][i] = quant; + + // uv + quant = i == 0 ? av1_dc_quant(q, cm->uv_dc_delta_q, cm->bit_depth) + : av1_ac_quant(q, cm->uv_ac_delta_q, cm->bit_depth); + invert_quant(&quants->uv_quant[q][i], &quants->uv_quant_shift[q][i], + quant); + quants->uv_quant_fp[q][i] = (1 << 16) / quant; + quants->uv_round_fp[q][i] = (qrounding_factor_fp * quant) >> 7; + quants->uv_zbin[q][i] = ROUND_POWER_OF_TWO(qzbin_factor * quant, 7); + quants->uv_round[q][i] = (qrounding_factor * quant) >> 7; + cpi->uv_dequant[q][i] = quant; + } + +#if CONFIG_NEW_QUANT + for (dq = 0; dq < QUANT_PROFILES; dq++) { + for (i = 0; i < COEF_BANDS; i++) { + const int y_quant = cpi->y_dequant[q][i != 0]; + const int uvquant = cpi->uv_dequant[q][i != 0]; + av1_get_dequant_val_nuq(y_quant, i, cpi->y_dequant_val_nuq[dq][q][i], + quants->y_cuml_bins_nuq[dq][q][i], dq); + av1_get_dequant_val_nuq(uvquant, i, cpi->uv_dequant_val_nuq[dq][q][i], + quants->uv_cuml_bins_nuq[dq][q][i], dq); + } + } +#endif // CONFIG_NEW_QUANT + + for (i = 2; i < 8; i++) { // 8: SIMD width + quants->y_quant[q][i] = quants->y_quant[q][1]; + quants->y_quant_fp[q][i] = quants->y_quant_fp[q][1]; + quants->y_round_fp[q][i] = quants->y_round_fp[q][1]; + quants->y_quant_shift[q][i] = quants->y_quant_shift[q][1]; + quants->y_zbin[q][i] = quants->y_zbin[q][1]; + quants->y_round[q][i] = quants->y_round[q][1]; + cpi->y_dequant[q][i] = cpi->y_dequant[q][1]; + + quants->uv_quant[q][i] = quants->uv_quant[q][1]; + quants->uv_quant_fp[q][i] = quants->uv_quant_fp[q][1]; + quants->uv_round_fp[q][i] = quants->uv_round_fp[q][1]; + quants->uv_quant_shift[q][i] = quants->uv_quant_shift[q][1]; + quants->uv_zbin[q][i] = quants->uv_zbin[q][1]; + quants->uv_round[q][i] = quants->uv_round[q][1]; + cpi->uv_dequant[q][i] = cpi->uv_dequant[q][1]; + } + } +} + +void av1_init_plane_quantizers(const AV1_COMP *cpi, MACROBLOCK *x, + int segment_id) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + const QUANTS *const quants = &cpi->quants; + +#if CONFIG_DELTA_Q +#if CONFIG_EXT_DELTA_Q + int current_q_index = AOMMAX( + 0, AOMMIN(QINDEX_RANGE - 1, cpi->oxcf.deltaq_mode != NO_DELTA_Q + ? cm->base_qindex + xd->delta_qindex + : cm->base_qindex)); +#else + int current_q_index = AOMMAX( + 0, AOMMIN(QINDEX_RANGE - 1, cm->delta_q_present_flag + ? cm->base_qindex + xd->delta_qindex + : cm->base_qindex)); +#endif + const int qindex = av1_get_qindex(&cm->seg, segment_id, current_q_index); +#else + const int qindex = av1_get_qindex(&cm->seg, segment_id, cm->base_qindex); +#endif + const int rdmult = av1_compute_rd_mult(cpi, qindex + cm->y_dc_delta_q); + int i; +#if CONFIG_AOM_QM + int minqm = cm->min_qmlevel; + int maxqm = cm->max_qmlevel; + // Quant matrix only depends on the base QP so there is only one set per frame + int qmlevel = (xd->lossless[segment_id] || cm->using_qmatrix == 0) + ? NUM_QM_LEVELS - 1 + : aom_get_qmlevel(cm->base_qindex, minqm, maxqm); +#endif +#if CONFIG_NEW_QUANT + int dq; +#endif + + // Y + x->plane[0].quant = quants->y_quant[qindex]; + x->plane[0].quant_fp = quants->y_quant_fp[qindex]; + x->plane[0].round_fp = quants->y_round_fp[qindex]; + x->plane[0].quant_shift = quants->y_quant_shift[qindex]; + x->plane[0].zbin = quants->y_zbin[qindex]; + x->plane[0].round = quants->y_round[qindex]; +#if CONFIG_AOM_QM + memcpy(&xd->plane[0].seg_qmatrix[segment_id], cm->gqmatrix[qmlevel][0], + sizeof(cm->gqmatrix[qmlevel][0])); + memcpy(&xd->plane[0].seg_iqmatrix[segment_id], cm->giqmatrix[qmlevel][0], + sizeof(cm->giqmatrix[qmlevel][0])); +#endif + xd->plane[0].dequant = cpi->y_dequant[qindex]; +#if CONFIG_NEW_QUANT + for (dq = 0; dq < QUANT_PROFILES; dq++) { + x->plane[0].cuml_bins_nuq[dq] = quants->y_cuml_bins_nuq[dq][qindex]; + xd->plane[0].dequant_val_nuq[dq] = cpi->y_dequant_val_nuq[dq][qindex]; + } +#endif // CONFIG_NEW_QUANT + + // UV + for (i = 1; i < 3; i++) { + x->plane[i].quant = quants->uv_quant[qindex]; + x->plane[i].quant_fp = quants->uv_quant_fp[qindex]; + x->plane[i].round_fp = quants->uv_round_fp[qindex]; + x->plane[i].quant_shift = quants->uv_quant_shift[qindex]; + x->plane[i].zbin = quants->uv_zbin[qindex]; + x->plane[i].round = quants->uv_round[qindex]; +#if CONFIG_AOM_QM + memcpy(&xd->plane[i].seg_qmatrix[segment_id], cm->gqmatrix[qmlevel][1], + sizeof(cm->gqmatrix[qmlevel][1])); + memcpy(&xd->plane[i].seg_iqmatrix[segment_id], cm->giqmatrix[qmlevel][1], + sizeof(cm->giqmatrix[qmlevel][1])); +#endif + xd->plane[i].dequant = cpi->uv_dequant[qindex]; +#if CONFIG_NEW_QUANT + for (dq = 0; dq < QUANT_PROFILES; dq++) { + x->plane[i].cuml_bins_nuq[dq] = quants->uv_cuml_bins_nuq[dq][qindex]; + xd->plane[i].dequant_val_nuq[dq] = cpi->uv_dequant_val_nuq[dq][qindex]; + } +#endif // CONFIG_NEW_QUANT + } + + x->skip_block = segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP); + x->qindex = qindex; + + set_error_per_bit(x, rdmult); + + av1_initialize_me_consts(cpi, x, qindex); +} + +void av1_frame_init_quantizer(AV1_COMP *cpi) { + MACROBLOCK *const x = &cpi->td.mb; + MACROBLOCKD *const xd = &x->e_mbd; + av1_init_plane_quantizers(cpi, x, xd->mi[0]->mbmi.segment_id); +} + +void av1_set_quantizer(AV1_COMMON *cm, int q) { + // quantizer has to be reinitialized with av1_init_quantizer() if any + // delta_q changes. + cm->base_qindex = q; + cm->y_dc_delta_q = 0; + cm->uv_dc_delta_q = 0; + cm->uv_ac_delta_q = 0; +} + +// Table that converts 0-63 Q-range values passed in outside to the Qindex +// range used internally. +static const int quantizer_to_qindex[] = { + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, + 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, + 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, + 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, + 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 249, 255, +}; + +int av1_quantizer_to_qindex(int quantizer) { + return quantizer_to_qindex[quantizer]; +} + +int av1_qindex_to_quantizer(int qindex) { + int quantizer; + + for (quantizer = 0; quantizer < 64; ++quantizer) + if (quantizer_to_qindex[quantizer] >= qindex) return quantizer; + + return 63; +} diff --git a/third_party/aom/av1/encoder/av1_quantize.h b/third_party/aom/av1/encoder/av1_quantize.h new file mode 100644 index 0000000000..c87b6b7dc0 --- /dev/null +++ b/third_party/aom/av1/encoder/av1_quantize.h @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_QUANTIZE_H_ +#define AV1_ENCODER_QUANTIZE_H_ + +#include "./aom_config.h" +#include "av1/common/quant_common.h" +#include "av1/common/scan.h" +#include "av1/encoder/block.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct QUANT_PARAM { + int log_scale; +#if CONFIG_NEW_QUANT + TX_SIZE tx_size; + int dq; +#endif // CONFIG_NEW_QUANT +#if CONFIG_AOM_QM + const qm_val_t *qmatrix; + const qm_val_t *iqmatrix; +#endif // CONFIG_AOM_QM +} QUANT_PARAM; + +typedef void (*AV1_QUANT_FACADE)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, + const QUANT_PARAM *qparam); + +typedef struct { +#if CONFIG_NEW_QUANT + DECLARE_ALIGNED( + 16, tran_low_t, + y_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS][NUQ_KNOTS]); + DECLARE_ALIGNED( + 16, tran_low_t, + uv_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS][NUQ_KNOTS]); +#endif // CONFIG_NEW_QUANT + // 0: dc 1: ac 2-8: ac repeated to SIMD width + DECLARE_ALIGNED(16, int16_t, y_quant[QINDEX_RANGE][8]); + DECLARE_ALIGNED(16, int16_t, y_quant_shift[QINDEX_RANGE][8]); + DECLARE_ALIGNED(16, int16_t, y_zbin[QINDEX_RANGE][8]); + DECLARE_ALIGNED(16, int16_t, y_round[QINDEX_RANGE][8]); + + // TODO(jingning): in progress of re-working the quantization. will decide + // if we want to deprecate the current use of y_quant. + DECLARE_ALIGNED(16, int16_t, y_quant_fp[QINDEX_RANGE][8]); + DECLARE_ALIGNED(16, int16_t, uv_quant_fp[QINDEX_RANGE][8]); + DECLARE_ALIGNED(16, int16_t, y_round_fp[QINDEX_RANGE][8]); + DECLARE_ALIGNED(16, int16_t, uv_round_fp[QINDEX_RANGE][8]); + + DECLARE_ALIGNED(16, int16_t, uv_quant[QINDEX_RANGE][8]); + DECLARE_ALIGNED(16, int16_t, uv_quant_shift[QINDEX_RANGE][8]); + DECLARE_ALIGNED(16, int16_t, uv_zbin[QINDEX_RANGE][8]); + DECLARE_ALIGNED(16, int16_t, uv_round[QINDEX_RANGE][8]); +} QUANTS; + +struct AV1_COMP; +struct AV1Common; + +void av1_frame_init_quantizer(struct AV1_COMP *cpi); + +void av1_init_plane_quantizers(const struct AV1_COMP *cpi, MACROBLOCK *x, + int segment_id); + +void av1_init_quantizer(struct AV1_COMP *cpi); + +void av1_set_quantizer(struct AV1Common *cm, int q); + +int av1_quantizer_to_qindex(int quantizer); + +int av1_qindex_to_quantizer(int qindex); + +void av1_quantize_skip(intptr_t n_coeffs, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr); + +void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, const QUANT_PARAM *qparam); + +void av1_quantize_b_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, tran_low_t *dqcoeff_ptr, + uint16_t *eob_ptr, const SCAN_ORDER *sc, + const QUANT_PARAM *qparam); + +void av1_quantize_dc_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, const QUANT_PARAM *qparam); + +#if CONFIG_NEW_QUANT +void av1_quantize_fp_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, + const QUANT_PARAM *qparam); + +void av1_quantize_b_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, const QUANT_PARAM *qparam); + +void av1_quantize_dc_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, + const QUANT_PARAM *qparam); +#endif // CONFIG_NEW_QUANT + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_quantize_fp_facade(const tran_low_t *coeff_ptr, + intptr_t n_coeffs, const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, + const QUANT_PARAM *qparam); + +void av1_highbd_quantize_b_facade(const tran_low_t *coeff_ptr, + intptr_t n_coeffs, const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, + const QUANT_PARAM *qparam); + +void av1_highbd_quantize_dc_facade(const tran_low_t *coeff_ptr, + intptr_t n_coeffs, const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, + const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, + const SCAN_ORDER *sc, + const QUANT_PARAM *qparam); + +#if CONFIG_NEW_QUANT +void av1_highbd_quantize_fp_nuq_facade( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc, + const QUANT_PARAM *qparam); + +void av1_highbd_quantize_b_nuq_facade( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc, + const QUANT_PARAM *qparam); + +void av1_highbd_quantize_dc_nuq_facade( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p, + tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc, + const QUANT_PARAM *qparam); +#endif // CONFIG_NEW_QUANT +#endif // CONFIG_HIGHBITDEPTH + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_QUANTIZE_H_ diff --git a/third_party/aom/av1/encoder/bitstream.c b/third_party/aom/av1/encoder/bitstream.c new file mode 100644 index 0000000000..7cc6179ead --- /dev/null +++ b/third_party/aom/av1/encoder/bitstream.c @@ -0,0 +1,5399 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "aom/aom_encoder.h" +#include "aom_dsp/bitwriter_buffer.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/binary_codes_writer.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem_ops.h" +#include "aom_ports/system_state.h" +#if CONFIG_BITSTREAM_DEBUG +#include "aom_util/debug_util.h" +#endif // CONFIG_BITSTREAM_DEBUG + +#if CONFIG_CDEF +#include "av1/common/cdef.h" +#include "av1/common/clpf.h" +#endif // CONFIG_CDEF +#include "av1/common/entropy.h" +#include "av1/common/entropymode.h" +#include "av1/common/entropymv.h" +#include "av1/common/mvref_common.h" +#include "av1/common/odintrin.h" +#include "av1/common/pred_common.h" +#include "av1/common/reconinter.h" +#if CONFIG_EXT_INTRA +#include "av1/common/reconintra.h" +#endif // CONFIG_EXT_INTRA +#include "av1/common/seg_common.h" +#include "av1/common/tile_common.h" + +#if CONFIG_ANS +#include "aom_dsp/buf_ans.h" +#endif // CONFIG_ANS +#if CONFIG_LV_MAP +#include "av1/encoder/encodetxb.h" +#endif // CONFIG_LV_MAP +#include "av1/encoder/bitstream.h" +#include "av1/encoder/cost.h" +#include "av1/encoder/encodemv.h" +#include "av1/encoder/mcomp.h" +#if CONFIG_PALETTE && CONFIG_PALETTE_DELTA_ENCODING +#include "av1/encoder/palette.h" +#endif // CONFIG_PALETTE && CONFIG_PALETTE_DELTA_ENCODING +#include "av1/encoder/segmentation.h" +#include "av1/encoder/subexp.h" +#include "av1/encoder/tokenize.h" +#if CONFIG_PVQ +#include "av1/encoder/pvq_encoder.h" +#endif + +static struct av1_token intra_mode_encodings[INTRA_MODES]; +static struct av1_token switchable_interp_encodings[SWITCHABLE_FILTERS]; +#if CONFIG_EXT_PARTITION_TYPES && !CONFIG_EC_MULTISYMBOL +static const struct av1_token ext_partition_encodings[EXT_PARTITION_TYPES] = { + { 0, 1 }, { 4, 3 }, { 12, 4 }, { 7, 3 }, + { 10, 4 }, { 11, 4 }, { 26, 5 }, { 27, 5 } +}; +#endif +static struct av1_token partition_encodings[PARTITION_TYPES]; +#if !CONFIG_REF_MV +static struct av1_token inter_mode_encodings[INTER_MODES]; +#endif +#if CONFIG_EXT_INTER +static const struct av1_token + inter_compound_mode_encodings[INTER_COMPOUND_MODES] = { + { 2, 2 }, { 50, 6 }, { 51, 6 }, { 24, 5 }, { 52, 6 }, + { 53, 6 }, { 54, 6 }, { 55, 6 }, { 0, 1 }, { 7, 3 } + }; +#endif // CONFIG_EXT_INTER +#if CONFIG_PALETTE +static struct av1_token palette_size_encodings[PALETTE_SIZES]; +static struct av1_token palette_color_index_encodings[PALETTE_SIZES] + [PALETTE_COLORS]; +#endif // CONFIG_PALETTE +#if !CONFIG_EC_MULTISYMBOL +static const struct av1_token tx_size_encodings[MAX_TX_DEPTH][TX_SIZES] = { + { { 0, 1 }, { 1, 1 } }, // Max tx_size is 8X8 + { { 0, 1 }, { 2, 2 }, { 3, 2 } }, // Max tx_size is 16X16 + { { 0, 1 }, { 2, 2 }, { 6, 3 }, { 7, 3 } }, // Max tx_size is 32X32 +#if CONFIG_TX64X64 + { { 0, 1 }, { 2, 2 }, { 6, 3 }, { 14, 4 }, { 15, 4 } }, // Max tx_size 64X64 +#endif // CONFIG_TX64X64 +}; +#endif + +#if CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA || CONFIG_PALETTE +static INLINE void write_uniform(aom_writer *w, int n, int v) { + const int l = get_unsigned_bits(n); + const int m = (1 << l) - n; + if (l == 0) return; + if (v < m) { + aom_write_literal(w, v, l - 1); + } else { + aom_write_literal(w, m + ((v - m) >> 1), l - 1); + aom_write_literal(w, (v - m) & 1, 1); + } +} +#endif // CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA || CONFIG_PALETTE + +#if CONFIG_EXT_TX +static struct av1_token ext_tx_inter_encodings[EXT_TX_SETS_INTER][TX_TYPES]; +static struct av1_token ext_tx_intra_encodings[EXT_TX_SETS_INTRA][TX_TYPES]; +#else +static struct av1_token ext_tx_encodings[TX_TYPES]; +#endif // CONFIG_EXT_TX +#if CONFIG_GLOBAL_MOTION +static struct av1_token global_motion_types_encodings[GLOBAL_TRANS_TYPES]; +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP +static struct av1_token intra_filter_encodings[INTRA_FILTERS]; +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA +#if CONFIG_EXT_INTER +static struct av1_token interintra_mode_encodings[INTERINTRA_MODES]; +#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE +static struct av1_token compound_type_encodings[COMPOUND_TYPES]; +#endif // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE +#endif // CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +static struct av1_token motion_mode_encodings[MOTION_MODES]; +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +#if CONFIG_LOOP_RESTORATION +static struct av1_token switchable_restore_encodings[RESTORE_SWITCHABLE_TYPES]; +#endif // CONFIG_LOOP_RESTORATION +static void write_uncompressed_header(AV1_COMP *cpi, + struct aom_write_bit_buffer *wb); +static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data); +static int remux_tiles(const AV1_COMMON *const cm, uint8_t *dst, + const uint32_t data_size, const uint32_t max_tile_size, + const uint32_t max_tile_col_size, + int *const tile_size_bytes, + int *const tile_col_size_bytes); + +void av1_encode_token_init(void) { +#if CONFIG_EXT_TX || CONFIG_PALETTE + int s; +#endif // CONFIG_EXT_TX || CONFIG_PALETTE +#if CONFIG_EXT_TX + for (s = 1; s < EXT_TX_SETS_INTER; ++s) { + av1_tokens_from_tree(ext_tx_inter_encodings[s], av1_ext_tx_inter_tree[s]); + } + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { + av1_tokens_from_tree(ext_tx_intra_encodings[s], av1_ext_tx_intra_tree[s]); + } +#else + av1_tokens_from_tree(ext_tx_encodings, av1_ext_tx_tree); +#endif // CONFIG_EXT_TX + av1_tokens_from_tree(intra_mode_encodings, av1_intra_mode_tree); + av1_tokens_from_tree(switchable_interp_encodings, av1_switchable_interp_tree); + av1_tokens_from_tree(partition_encodings, av1_partition_tree); +#if !CONFIG_REF_MV + av1_tokens_from_tree(inter_mode_encodings, av1_inter_mode_tree); +#endif + +#if CONFIG_PALETTE + av1_tokens_from_tree(palette_size_encodings, av1_palette_size_tree); + for (s = 0; s < PALETTE_SIZES; ++s) { + av1_tokens_from_tree(palette_color_index_encodings[s], + av1_palette_color_index_tree[s]); + } +#endif // CONFIG_PALETTE + +#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP + av1_tokens_from_tree(intra_filter_encodings, av1_intra_filter_tree); +#endif // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP +#if CONFIG_EXT_INTER + av1_tokens_from_tree(interintra_mode_encodings, av1_interintra_mode_tree); +#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE + av1_tokens_from_tree(compound_type_encodings, av1_compound_type_tree); +#endif // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE +#endif // CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + av1_tokens_from_tree(motion_mode_encodings, av1_motion_mode_tree); +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +#if CONFIG_GLOBAL_MOTION + av1_tokens_from_tree(global_motion_types_encodings, + av1_global_motion_types_tree); +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_LOOP_RESTORATION + av1_tokens_from_tree(switchable_restore_encodings, + av1_switchable_restore_tree); +#endif // CONFIG_LOOP_RESTORATION + +#if CONFIG_EC_MULTISYMBOL + /* This hack is necessary when CONFIG_DUAL_FILTER is enabled because the five + SWITCHABLE_FILTERS are not consecutive, e.g., 0, 1, 2, 3, 4, when doing + an in-order traversal of the av1_switchable_interp_tree structure. */ + av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv, + av1_switchable_interp_tree); +/* This hack is necessary because the four TX_TYPES are not consecutive, + e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree + structure. */ +#if CONFIG_EXT_TX + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) + av1_indices_from_tree(av1_ext_tx_intra_ind[s], av1_ext_tx_intra_inv[s], + av1_ext_tx_intra_tree[s]); + for (s = 1; s < EXT_TX_SETS_INTER; ++s) + av1_indices_from_tree(av1_ext_tx_inter_ind[s], av1_ext_tx_inter_inv[s], + av1_ext_tx_inter_tree[s]); +#else + av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree); +#endif + av1_indices_from_tree(av1_intra_mode_ind, av1_intra_mode_inv, + av1_intra_mode_tree); + av1_indices_from_tree(av1_inter_mode_ind, av1_inter_mode_inv, + av1_inter_mode_tree); +#endif +} + +static void write_intra_mode_kf(const AV1_COMMON *cm, FRAME_CONTEXT *frame_ctx, + const MODE_INFO *mi, const MODE_INFO *above_mi, + const MODE_INFO *left_mi, int block, + PREDICTION_MODE mode, aom_writer *w) { +#if CONFIG_INTRABC + assert(!is_intrabc_block(&mi->mbmi)); +#endif // CONFIG_INTRABC +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, av1_intra_mode_ind[mode], + get_y_mode_cdf(frame_ctx, mi, above_mi, left_mi, block), + INTRA_MODES); + (void)cm; +#else + av1_write_token(w, av1_intra_mode_tree, + get_y_mode_probs(cm, mi, above_mi, left_mi, block), + &intra_mode_encodings[mode]); + (void)frame_ctx; +#endif +} + +#if CONFIG_EXT_INTER +static void write_interintra_mode(aom_writer *w, INTERINTRA_MODE mode, + const aom_prob *probs) { + av1_write_token(w, av1_interintra_mode_tree, probs, + &interintra_mode_encodings[mode]); +} +#endif // CONFIG_EXT_INTER + +static void write_inter_mode(aom_writer *w, PREDICTION_MODE mode, + FRAME_CONTEXT *ec_ctx, const int16_t mode_ctx) { +#if CONFIG_REF_MV + const int16_t newmv_ctx = mode_ctx & NEWMV_CTX_MASK; + const aom_prob newmv_prob = ec_ctx->newmv_prob[newmv_ctx]; + +#define IS_NEWMV_MODE(mode) ((mode) == NEWMV) + aom_write(w, !IS_NEWMV_MODE(mode), newmv_prob); + + if (!IS_NEWMV_MODE(mode)) { + const int16_t zeromv_ctx = (mode_ctx >> ZEROMV_OFFSET) & ZEROMV_CTX_MASK; + const aom_prob zeromv_prob = ec_ctx->zeromv_prob[zeromv_ctx]; + + if (mode_ctx & (1 << ALL_ZERO_FLAG_OFFSET)) { + assert(mode == ZEROMV); + return; + } + + aom_write(w, mode != ZEROMV, zeromv_prob); + + if (mode != ZEROMV) { + int16_t refmv_ctx = (mode_ctx >> REFMV_OFFSET) & REFMV_CTX_MASK; + aom_prob refmv_prob; + + if (mode_ctx & (1 << SKIP_NEARESTMV_OFFSET)) refmv_ctx = 6; + if (mode_ctx & (1 << SKIP_NEARMV_OFFSET)) refmv_ctx = 7; + if (mode_ctx & (1 << SKIP_NEARESTMV_SUB8X8_OFFSET)) refmv_ctx = 8; + + refmv_prob = ec_ctx->refmv_prob[refmv_ctx]; + aom_write(w, mode != NEARESTMV, refmv_prob); + } + } + +#undef IS_NEWMV_MODE + +#else // !CONFIG_REF_MV + assert(is_inter_mode(mode)); +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, av1_inter_mode_ind[INTER_OFFSET(mode)], + ec_ctx->inter_mode_cdf[mode_ctx], INTER_MODES); +#else + { + const aom_prob *const inter_probs = ec_ctx->inter_mode_probs[mode_ctx]; + av1_write_token(w, av1_inter_mode_tree, inter_probs, + &inter_mode_encodings[INTER_OFFSET(mode)]); + } +#endif +#endif +} + +#if CONFIG_REF_MV +static void write_drl_idx(const AV1_COMMON *cm, const MB_MODE_INFO *mbmi, + const MB_MODE_INFO_EXT *mbmi_ext, aom_writer *w) { + uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + + assert(mbmi->ref_mv_idx < 3); + +#if CONFIG_EXT_INTER + if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) { +#else + if (mbmi->mode == NEWMV) { +#endif + int idx; + for (idx = 0; idx < 2; ++idx) { + if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) { + uint8_t drl_ctx = + av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx); + aom_prob drl_prob = cm->fc->drl_prob[drl_ctx]; + + aom_write(w, mbmi->ref_mv_idx != idx, drl_prob); + if (mbmi->ref_mv_idx == idx) return; + } + } + return; + } + + if (have_nearmv_in_inter_mode(mbmi->mode)) { + int idx; + // TODO(jingning): Temporary solution to compensate the NEARESTMV offset. + for (idx = 1; idx < 3; ++idx) { + if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) { + uint8_t drl_ctx = + av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx); + aom_prob drl_prob = cm->fc->drl_prob[drl_ctx]; + + aom_write(w, mbmi->ref_mv_idx != (idx - 1), drl_prob); + if (mbmi->ref_mv_idx == (idx - 1)) return; + } + } + return; + } +} +#endif + +#if CONFIG_EXT_INTER +static void write_inter_compound_mode(AV1_COMMON *cm, aom_writer *w, + PREDICTION_MODE mode, + const int16_t mode_ctx) { + const aom_prob *const inter_compound_probs = + cm->fc->inter_compound_mode_probs[mode_ctx]; + + assert(is_inter_compound_mode(mode)); + av1_write_token(w, av1_inter_compound_mode_tree, inter_compound_probs, + &inter_compound_mode_encodings[INTER_COMPOUND_OFFSET(mode)]); +} +#endif // CONFIG_EXT_INTER + +static void encode_unsigned_max(struct aom_write_bit_buffer *wb, int data, + int max) { + aom_wb_write_literal(wb, data, get_unsigned_bits(max)); +} + +#if !CONFIG_EC_ADAPT || \ + (CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION || CONFIG_EXT_INTER) +static void prob_diff_update(const aom_tree_index *tree, + aom_prob probs[/*n - 1*/], + const unsigned int counts[/*n - 1*/], int n, + int probwt, aom_writer *w) { + int i; + unsigned int branch_ct[32][2]; + + // Assuming max number of probabilities <= 32 + assert(n <= 32); + + av1_tree_probs_from_distribution(tree, branch_ct, counts); + for (i = 0; i < n - 1; ++i) + av1_cond_prob_diff_update(w, &probs[i], branch_ct[i], probwt); +} +#endif + +#if CONFIG_EXT_INTER || !CONFIG_EC_ADAPT +static int prob_diff_update_savings(const aom_tree_index *tree, + aom_prob probs[/*n - 1*/], + const unsigned int counts[/*n - 1*/], int n, + int probwt) { + int i; + unsigned int branch_ct[32][2]; + int savings = 0; + + // Assuming max number of probabilities <= 32 + assert(n <= 32); + av1_tree_probs_from_distribution(tree, branch_ct, counts); + for (i = 0; i < n - 1; ++i) { + savings += + av1_cond_prob_diff_update_savings(&probs[i], branch_ct[i], probwt); + } + return savings; +} +#endif // CONFIG_EXT_INTER || !CONFIG_EC_ADAPT + +#if CONFIG_VAR_TX +static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd, + const MB_MODE_INFO *mbmi, TX_SIZE tx_size, + int depth, int blk_row, int blk_col, + aom_writer *w) { + const int tx_row = blk_row >> 1; + const int tx_col = blk_col >> 1; + const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0); + const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0); + + int ctx = txfm_partition_context(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, + mbmi->sb_type, tx_size); + + if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + + if (depth == MAX_VARTX_DEPTH) { + txfm_partition_update(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, tx_size, tx_size); + return; + } + + if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) { + aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]); + txfm_partition_update(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, tx_size, tx_size); + } else { + const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; + const int bsl = tx_size_wide_unit[sub_txs]; + int i; + + aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]); + + if (tx_size == TX_8X8) { + txfm_partition_update(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, sub_txs, tx_size); + return; + } + + assert(bsl > 0); + for (i = 0; i < 4; ++i) { + int offsetr = blk_row + (i >> 1) * bsl; + int offsetc = blk_col + (i & 0x01) * bsl; + write_tx_size_vartx(cm, xd, mbmi, sub_txs, depth + 1, offsetr, offsetc, + w); + } + } +} + +static void update_txfm_partition_probs(AV1_COMMON *cm, aom_writer *w, + FRAME_COUNTS *counts, int probwt) { + int k; + for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k) + av1_cond_prob_diff_update(w, &cm->fc->txfm_partition_prob[k], + counts->txfm_partition[k], probwt); +} +#endif + +static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd, + aom_writer *w) { + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const BLOCK_SIZE bsize = mbmi->sb_type; +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; + (void)cm; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif +// For sub8x8 blocks the tx_size symbol does not need to be sent +#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX + if (bsize > BLOCK_4X4) { +#else + if (bsize >= BLOCK_8X8) { +#endif + const TX_SIZE tx_size = mbmi->tx_size; + const int is_inter = is_inter_block(mbmi); + const int tx_size_ctx = get_tx_size_context(xd); + const int tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize] + : intra_tx_size_cat_lookup[bsize]; + const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size]; + const int depth = tx_size_to_depth(coded_tx_size); +#if CONFIG_EXT_TX && CONFIG_RECT_TX + assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi))); + assert( + IMPLIES(is_rect_tx(tx_size), tx_size == max_txsize_rect_lookup[bsize])); +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, depth, ec_ctx->tx_size_cdf[tx_size_cat][tx_size_ctx], + tx_size_cat + 2); +#else + av1_write_token(w, av1_tx_size_tree[tx_size_cat], + ec_ctx->tx_size_probs[tx_size_cat][tx_size_ctx], + &tx_size_encodings[tx_size_cat][depth]); +#endif + } +} + +#if CONFIG_REF_MV +static void update_inter_mode_probs(AV1_COMMON *cm, aom_writer *w, + FRAME_COUNTS *counts) { + int i; +#if CONFIG_TILE_GROUPS + const int probwt = cm->num_tg; +#else + const int probwt = 1; +#endif + for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i) + av1_cond_prob_diff_update(w, &cm->fc->newmv_prob[i], counts->newmv_mode[i], + probwt); + for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i) + av1_cond_prob_diff_update(w, &cm->fc->zeromv_prob[i], + counts->zeromv_mode[i], probwt); + for (i = 0; i < REFMV_MODE_CONTEXTS; ++i) + av1_cond_prob_diff_update(w, &cm->fc->refmv_prob[i], counts->refmv_mode[i], + probwt); + for (i = 0; i < DRL_MODE_CONTEXTS; ++i) + av1_cond_prob_diff_update(w, &cm->fc->drl_prob[i], counts->drl_mode[i], + probwt); +} +#endif + +#if CONFIG_EXT_INTER +static void update_inter_compound_mode_probs(AV1_COMMON *cm, int probwt, + aom_writer *w) { + const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) - + av1_cost_zero(GROUP_DIFF_UPDATE_PROB); + int i; + int savings = 0; + int do_update = 0; + for (i = 0; i < INTER_MODE_CONTEXTS; ++i) { + savings += prob_diff_update_savings( + av1_inter_compound_mode_tree, cm->fc->inter_compound_mode_probs[i], + cm->counts.inter_compound_mode[i], INTER_COMPOUND_MODES, probwt); + } + do_update = savings > savings_thresh; + aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB); + if (do_update) { + for (i = 0; i < INTER_MODE_CONTEXTS; ++i) { + prob_diff_update( + av1_inter_compound_mode_tree, cm->fc->inter_compound_mode_probs[i], + cm->counts.inter_compound_mode[i], INTER_COMPOUND_MODES, probwt, w); + } + } +} +#endif // CONFIG_EXT_INTER + +static int write_skip(const AV1_COMMON *cm, const MACROBLOCKD *xd, + int segment_id, const MODE_INFO *mi, aom_writer *w) { + if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) { + return 1; + } else { + const int skip = mi->mbmi.skip; + aom_write(w, skip, av1_get_skip_prob(cm, xd)); + return skip; + } +} + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +static void write_motion_mode(const AV1_COMMON *cm, const MODE_INFO *mi, + aom_writer *w) { + const MB_MODE_INFO *mbmi = &mi->mbmi; + MOTION_MODE last_motion_mode_allowed = motion_mode_allowed( +#if CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + 0, cm->global_motion, +#endif // CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + mi); + + if (last_motion_mode_allowed == SIMPLE_TRANSLATION) return; +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + if (last_motion_mode_allowed == OBMC_CAUSAL) { + aom_write(w, mbmi->motion_mode == OBMC_CAUSAL, + cm->fc->obmc_prob[mbmi->sb_type]); + } else { +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + av1_write_token(w, av1_motion_mode_tree, + cm->fc->motion_mode_prob[mbmi->sb_type], + &motion_mode_encodings[mbmi->motion_mode]); +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + } +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION +} +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + +#if CONFIG_DELTA_Q +static void write_delta_qindex(const AV1_COMMON *cm, const MACROBLOCKD *xd, + int delta_qindex, aom_writer *w) { + int sign = delta_qindex < 0; + int abs = sign ? -delta_qindex : delta_qindex; + int rem_bits, thr; + int smallval = abs < DELTA_Q_SMALL ? 1 : 0; +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; + (void)cm; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; + (void)xd; +#endif + +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, AOMMIN(abs, DELTA_Q_SMALL), ec_ctx->delta_q_cdf, + DELTA_Q_PROBS + 1); +#else + int i = 0; + while (i < DELTA_Q_SMALL && i <= abs) { + int bit = (i < abs); + aom_write(w, bit, ec_ctx->delta_q_prob[i]); + i++; + } +#endif + + if (!smallval) { + rem_bits = OD_ILOG_NZ(abs - 1) - 1; + thr = (1 << rem_bits) + 1; + aom_write_literal(w, rem_bits, 3); + aom_write_literal(w, abs - thr, rem_bits); + } + if (abs > 0) { + aom_write_bit(w, sign); + } +} + +#if !CONFIG_EC_ADAPT +static void update_delta_q_probs(AV1_COMMON *cm, aom_writer *w, + FRAME_COUNTS *counts) { + int k; +#if CONFIG_TILE_GROUPS + const int probwt = cm->num_tg; +#else + const int probwt = 1; +#endif +#if CONFIG_EXT_DELTA_Q + if (!cm->delta_q_present_flag) return; +#endif // CONFIG_EXT_DELTA_Q + for (k = 0; k < DELTA_Q_PROBS; ++k) { + av1_cond_prob_diff_update(w, &cm->fc->delta_q_prob[k], counts->delta_q[k], + probwt); + } +} +#endif // CONFIG_EC_ADAPT + +#if CONFIG_EXT_DELTA_Q +static void write_delta_lflevel(const AV1_COMMON *cm, const MACROBLOCKD *xd, + int delta_lflevel, aom_writer *w) { + int sign = delta_lflevel < 0; + int abs = sign ? -delta_lflevel : delta_lflevel; + int rem_bits, thr; + int smallval = abs < DELTA_LF_SMALL ? 1 : 0; +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; + (void)cm; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; + (void)xd; +#endif + +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL), ec_ctx->delta_lf_cdf, + DELTA_LF_PROBS + 1); +#else + int i = 0; + while (i < DELTA_LF_SMALL && i <= abs) { + int bit = (i < abs); + aom_write(w, bit, ec_ctx->delta_lf_prob[i]); + i++; + } +#endif // CONFIG_EC_MULTISYMBOL + + if (!smallval) { + rem_bits = OD_ILOG_NZ(abs - 1) - 1; + thr = (1 << rem_bits) + 1; + aom_write_literal(w, rem_bits, 3); + aom_write_literal(w, abs - thr, rem_bits); + } + if (abs > 0) { + aom_write_bit(w, sign); + } +} + +#if !CONFIG_EC_ADAPT +static void update_delta_lf_probs(AV1_COMMON *cm, aom_writer *w, + FRAME_COUNTS *counts) { + int k; +#if CONFIG_TILE_GROUPS + const int probwt = cm->num_tg; +#else + const int probwt = 1; +#endif + if (!cm->delta_lf_present_flag) return; + for (k = 0; k < DELTA_LF_PROBS; ++k) { + av1_cond_prob_diff_update(w, &cm->fc->delta_lf_prob[k], counts->delta_lf[k], + probwt); + } +} +#endif // CONFIG_EC_ADAPT +#endif // CONFIG_EXT_DELTA_Q +#endif // CONFIG_DELTA_Q + +static void update_skip_probs(AV1_COMMON *cm, aom_writer *w, + FRAME_COUNTS *counts) { + int k; +#if CONFIG_TILE_GROUPS + const int probwt = cm->num_tg; +#else + const int probwt = 1; +#endif + for (k = 0; k < SKIP_CONTEXTS; ++k) { + av1_cond_prob_diff_update(w, &cm->fc->skip_probs[k], counts->skip[k], + probwt); + } +} + +#if !CONFIG_EC_ADAPT +static void update_switchable_interp_probs(AV1_COMMON *cm, aom_writer *w, + FRAME_COUNTS *counts) { + int j; + for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) { +#if CONFIG_TILE_GROUPS + const int probwt = cm->num_tg; +#else + const int probwt = 1; +#endif + prob_diff_update( + av1_switchable_interp_tree, cm->fc->switchable_interp_prob[j], + counts->switchable_interp[j], SWITCHABLE_FILTERS, probwt, w); + } +} +#endif + +#if !CONFIG_EC_ADAPT +#if CONFIG_EXT_TX +static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) { + const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) - + av1_cost_zero(GROUP_DIFF_UPDATE_PROB); + int i, j; + int s; +#if CONFIG_TILE_GROUPS + const int probwt = cm->num_tg; +#else + const int probwt = 1; +#endif + for (s = 1; s < EXT_TX_SETS_INTER; ++s) { + int savings = 0; + int do_update = 0; + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + if (!use_inter_ext_tx_for_txsize[s][i]) continue; + savings += prob_diff_update_savings( + av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i], + cm->counts.inter_ext_tx[s][i], + num_ext_tx_set[ext_tx_set_type_inter[s]], probwt); + } + do_update = savings > savings_thresh; + aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB); + if (do_update) { + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + if (!use_inter_ext_tx_for_txsize[s][i]) continue; + prob_diff_update(av1_ext_tx_inter_tree[s], + cm->fc->inter_ext_tx_prob[s][i], + cm->counts.inter_ext_tx[s][i], + num_ext_tx_set[ext_tx_set_type_inter[s]], probwt, w); + } + } + } + + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { + int savings = 0; + int do_update = 0; + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + if (!use_intra_ext_tx_for_txsize[s][i]) continue; + for (j = 0; j < INTRA_MODES; ++j) + savings += prob_diff_update_savings( + av1_ext_tx_intra_tree[s], cm->fc->intra_ext_tx_prob[s][i][j], + cm->counts.intra_ext_tx[s][i][j], + num_ext_tx_set[ext_tx_set_type_intra[s]], probwt); + } + do_update = savings > savings_thresh; + aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB); + if (do_update) { + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + if (!use_intra_ext_tx_for_txsize[s][i]) continue; + for (j = 0; j < INTRA_MODES; ++j) + prob_diff_update(av1_ext_tx_intra_tree[s], + cm->fc->intra_ext_tx_prob[s][i][j], + cm->counts.intra_ext_tx[s][i][j], + num_ext_tx_set[ext_tx_set_type_intra[s]], probwt, w); + } + } + } +} + +#else +static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) { + const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) - + av1_cost_zero(GROUP_DIFF_UPDATE_PROB); + int i, j; + + int savings = 0; + int do_update = 0; +#if CONFIG_TILE_GROUPS + const int probwt = cm->num_tg; +#else + const int probwt = 1; +#endif + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + for (j = 0; j < TX_TYPES; ++j) + savings += prob_diff_update_savings( + av1_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j], + cm->counts.intra_ext_tx[i][j], TX_TYPES, probwt); + } + do_update = savings > savings_thresh; + aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB); + if (do_update) { + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + for (j = 0; j < TX_TYPES; ++j) { + prob_diff_update(av1_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j], + cm->counts.intra_ext_tx[i][j], TX_TYPES, probwt, w); + } + } + } + + savings = 0; + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + savings += + prob_diff_update_savings(av1_ext_tx_tree, cm->fc->inter_ext_tx_prob[i], + cm->counts.inter_ext_tx[i], TX_TYPES, probwt); + } + do_update = savings > savings_thresh; + aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB); + if (do_update) { + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + prob_diff_update(av1_ext_tx_tree, cm->fc->inter_ext_tx_prob[i], + cm->counts.inter_ext_tx[i], TX_TYPES, probwt, w); + } + } +} +#endif // CONFIG_EXT_TX +#endif // !CONFIG_EC_ADAPT +#if CONFIG_PALETTE +static void pack_palette_tokens(aom_writer *w, const TOKENEXTRA **tp, int n, + int num) { + int i; + const TOKENEXTRA *p = *tp; + + for (i = 0; i < num; ++i) { + av1_write_token( + w, av1_palette_color_index_tree[n - PALETTE_MIN_SIZE], p->context_tree, + &palette_color_index_encodings[n - PALETTE_MIN_SIZE][p->token]); + ++p; + } + + *tp = p; +} +#endif // CONFIG_PALETTE + +#if !CONFIG_PVQ +#if CONFIG_SUPERTX +static void update_supertx_probs(AV1_COMMON *cm, int probwt, aom_writer *w) { + const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) - + av1_cost_zero(GROUP_DIFF_UPDATE_PROB); + int i, j; + int savings = 0; + int do_update = 0; + for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) { + for (j = TX_8X8; j < TX_SIZES; ++j) { + savings += av1_cond_prob_diff_update_savings( + &cm->fc->supertx_prob[i][j], cm->counts.supertx[i][j], probwt); + } + } + do_update = savings > savings_thresh; + aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB); + if (do_update) { + for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) { + for (j = TX_8X8; j < TX_SIZES; ++j) { + av1_cond_prob_diff_update(w, &cm->fc->supertx_prob[i][j], + cm->counts.supertx[i][j], probwt); + } + } + } +} +#endif // CONFIG_SUPERTX + +#if CONFIG_NEW_MULTISYMBOL +static INLINE void write_coeff_extra(const aom_cdf_prob *const *cdf, int val, + int n, aom_writer *w) { + // Code the extra bits from LSB to MSB in groups of 4 + int i = 0; + int count = 0; + while (count < n) { + const int size = AOMMIN(n - count, 4); + const int mask = (1 << size) - 1; + aom_write_cdf(w, val & mask, cdf[i++], 1 << size); + val >>= size; + count += size; + } +} +#else +static INLINE void write_coeff_extra(const aom_prob *pb, int value, + int num_bits, int skip_bits, aom_writer *w, + TOKEN_STATS *token_stats) { + // Code the extra bits from MSB to LSB 1 bit at a time + int index; + for (index = skip_bits; index < num_bits; ++index) { + const int shift = num_bits - index - 1; + const int bb = (value >> shift) & 1; + aom_write_record(w, bb, pb[index], token_stats); + } +} +#endif + +#if CONFIG_NEW_TOKENSET && !CONFIG_LV_MAP +static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp, + const TOKENEXTRA *const stop, + aom_bit_depth_t bit_depth, const TX_SIZE tx_size, + TOKEN_STATS *token_stats) { + const TOKENEXTRA *p = *tp; +#if CONFIG_VAR_TX + int count = 0; + const int seg_eob = tx_size_2d[tx_size]; +#endif + + while (p < stop && p->token != EOSB_TOKEN) { + const int token = p->token; + if (token == BLOCK_Z_TOKEN) { + aom_write_symbol(w, 0, *p->head_cdf, HEAD_TOKENS + 1); + p++; + continue; + } + + const av1_extra_bit *const extra_bits = &av1_extra_bits[token]; + if (p->eob_val == LAST_EOB) { + // Just code a flag indicating whether the value is >1 or 1. + aom_write_bit(w, token != ONE_TOKEN); + } else { + int comb_symb = 2 * AOMMIN(token, TWO_TOKEN) - p->eob_val + p->first_val; + aom_write_symbol(w, comb_symb, *p->head_cdf, HEAD_TOKENS + p->first_val); + } + if (token > ONE_TOKEN) { + aom_write_symbol(w, token - TWO_TOKEN, *p->tail_cdf, TAIL_TOKENS); + } + + if (extra_bits->base_val) { + const int bit_string = p->extra; + const int bit_string_length = extra_bits->len; // Length of extra bits to + const int is_cat6 = (extra_bits->base_val == CAT6_MIN_VAL); + // be written excluding + // the sign bit. + int skip_bits = is_cat6 + ? (int)sizeof(av1_cat6_prob) - + av1_get_cat6_extrabits_size(tx_size, bit_depth) + : 0; + + assert(!(bit_string >> (bit_string_length - skip_bits + 1))); + if (bit_string_length > 0) +#if CONFIG_NEW_MULTISYMBOL + write_coeff_extra(extra_bits->cdf, bit_string >> 1, + bit_string_length - skip_bits, w); +#else + write_coeff_extra(extra_bits->prob, bit_string >> 1, bit_string_length, + skip_bits, w, token_stats); +#endif + + aom_write_bit_record(w, bit_string & 1, token_stats); + } + ++p; + +#if CONFIG_VAR_TX + ++count; + if (token == EOB_TOKEN || count == seg_eob) break; +#endif + } + + *tp = p; +} +#else // CONFIG_NEW_TOKENSET +#if !CONFIG_LV_MAP +static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp, + const TOKENEXTRA *const stop, + aom_bit_depth_t bit_depth, const TX_SIZE tx_size, + TOKEN_STATS *token_stats) { + const TOKENEXTRA *p = *tp; +#if CONFIG_VAR_TX + int count = 0; + const int seg_eob = tx_size_2d[tx_size]; +#endif + + while (p < stop && p->token != EOSB_TOKEN) { + const int token = p->token; +#if !CONFIG_EC_MULTISYMBOL + const struct av1_token *const coef_encoding = &av1_coef_encodings[token]; + int coef_value = coef_encoding->value; + int coef_length = coef_encoding->len; +#endif // !CONFIG_EC_MULTISYMBOL + const av1_extra_bit *const extra_bits = &av1_extra_bits[token]; + +#if CONFIG_EC_MULTISYMBOL + /* skip one or two nodes */ + if (!p->skip_eob_node) + aom_write_record(w, token != EOB_TOKEN, p->context_tree[0], token_stats); + if (token != EOB_TOKEN) { + aom_write_record(w, token != ZERO_TOKEN, p->context_tree[1], token_stats); + if (token != ZERO_TOKEN) { + aom_write_symbol(w, token - ONE_TOKEN, *p->token_cdf, + CATEGORY6_TOKEN - ONE_TOKEN + 1); + } + } +#else + /* skip one or two nodes */ + if (p->skip_eob_node) + coef_length -= p->skip_eob_node; + else + aom_write_record(w, token != EOB_TOKEN, p->context_tree[0], token_stats); + + if (token != EOB_TOKEN) { + aom_write_record(w, token != ZERO_TOKEN, p->context_tree[1], token_stats); + + if (token != ZERO_TOKEN) { + aom_write_record(w, token != ONE_TOKEN, p->context_tree[2], + token_stats); + + if (token != ONE_TOKEN) { + const int unconstrained_len = UNCONSTRAINED_NODES - p->skip_eob_node; + aom_write_tree_record( + w, av1_coef_con_tree, + av1_pareto8_full[p->context_tree[PIVOT_NODE] - 1], coef_value, + coef_length - unconstrained_len, 0, token_stats); + } + } + } +#endif // CONFIG_EC_MULTISYMBOL + + if (extra_bits->base_val) { + const int bit_string = p->extra; + const int bit_string_length = extra_bits->len; // Length of extra bits to + // be written excluding + // the sign bit. + int skip_bits = (extra_bits->base_val == CAT6_MIN_VAL) + ? (int)sizeof(av1_cat6_prob) - + av1_get_cat6_extrabits_size(tx_size, bit_depth) + : 0; + + assert(!(bit_string >> (bit_string_length - skip_bits + 1))); + if (bit_string_length > 0) { +#if CONFIG_NEW_MULTISYMBOL + skip_bits &= ~3; + write_coeff_extra(extra_bits->cdf, bit_string >> 1, + bit_string_length - skip_bits, w); +#else + write_coeff_extra(extra_bits->prob, bit_string >> 1, bit_string_length, + skip_bits, w, token_stats); +#endif + } + aom_write_bit_record(w, bit_string & 1, token_stats); + } + ++p; + +#if CONFIG_VAR_TX + ++count; + if (token == EOB_TOKEN || count == seg_eob) break; +#endif + } + + *tp = p; +} +#endif // !CONFIG_LV_MAP +#endif // CONFIG_NEW_TOKENSET +#else // !CONFIG_PVQ +static PVQ_INFO *get_pvq_block(PVQ_QUEUE *pvq_q) { + PVQ_INFO *pvq; + + assert(pvq_q->curr_pos <= pvq_q->last_pos); + assert(pvq_q->curr_pos < pvq_q->buf_len); + + pvq = pvq_q->buf + pvq_q->curr_pos; + ++pvq_q->curr_pos; + + return pvq; +} + +static void pack_pvq_tokens(aom_writer *w, MACROBLOCK *const x, + MACROBLOCKD *const xd, int plane, BLOCK_SIZE bsize, + const TX_SIZE tx_size) { + PVQ_INFO *pvq; + int idx, idy; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + od_adapt_ctx *adapt; + int max_blocks_wide; + int max_blocks_high; + int step = (1 << tx_size); + const BLOCK_SIZE plane_bsize = + get_plane_block_size(AOMMAX(bsize, BLOCK_8X8), pd); + + adapt = x->daala_enc.state.adapt; + + max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + max_blocks_high = max_block_high(xd, plane_bsize, plane); + + for (idy = 0; idy < max_blocks_high; idy += step) { + for (idx = 0; idx < max_blocks_wide; idx += step) { + const int is_keyframe = 0; + const int encode_flip = 0; + const int flip = 0; + int i; + const int has_dc_skip = 1; + int *exg = &adapt->pvq.pvq_exg[plane][tx_size][0]; + int *ext = adapt->pvq.pvq_ext + tx_size * PVQ_MAX_PARTITIONS; + generic_encoder *model = adapt->pvq.pvq_param_model; + + pvq = get_pvq_block(x->pvq_q); + + // encode block skip info + aom_write_symbol(w, pvq->ac_dc_coded, + adapt->skip_cdf[2 * tx_size + (plane != 0)], 4); + + // AC coeffs coded? + if (pvq->ac_dc_coded & AC_CODED) { + assert(pvq->bs == tx_size); + for (i = 0; i < pvq->nb_bands; i++) { + if (i == 0 || + (!pvq->skip_rest && !(pvq->skip_dir & (1 << ((i - 1) % 3))))) { + pvq_encode_partition( + w, pvq->qg[i], pvq->theta[i], pvq->y + pvq->off[i], + pvq->size[i], pvq->k[i], model, adapt, exg + i, ext + i, + (plane != 0) * OD_TXSIZES * PVQ_MAX_PARTITIONS + + pvq->bs * PVQ_MAX_PARTITIONS + i, + is_keyframe, i == 0 && (i < pvq->nb_bands - 1), pvq->skip_rest, + encode_flip, flip); + } + if (i == 0 && !pvq->skip_rest && pvq->bs > 0) { + aom_write_symbol( + w, pvq->skip_dir, + &adapt->pvq + .pvq_skip_dir_cdf[(plane != 0) + 2 * (pvq->bs - 1)][0], + 7); + } + } + } + // Encode residue of DC coeff, if exist. + if (!has_dc_skip || (pvq->ac_dc_coded & DC_CODED)) { + generic_encode(w, &adapt->model_dc[plane], + abs(pvq->dq_dc_residue) - has_dc_skip, + &adapt->ex_dc[plane][pvq->bs][0], 2); + } + if ((pvq->ac_dc_coded & DC_CODED)) { + aom_write_bit(w, pvq->dq_dc_residue < 0); + } + } + } // for (idy = 0; +} +#endif // !CONFIG_PVG + +#if CONFIG_VAR_TX && !CONFIG_COEF_INTERLEAVE +static void pack_txb_tokens(aom_writer *w, const TOKENEXTRA **tp, + const TOKENEXTRA *const tok_end, +#if CONFIG_PVQ + MACROBLOCK *const x, +#endif + MACROBLOCKD *xd, MB_MODE_INFO *mbmi, int plane, + BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth, + int block, int blk_row, int blk_col, + TX_SIZE tx_size, TOKEN_STATS *token_stats) { + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const BLOCK_SIZE bsize = txsize_to_bsize[tx_size]; + const int tx_row = blk_row >> (1 - pd->subsampling_y); + const int tx_col = blk_col >> (1 - pd->subsampling_x); + TX_SIZE plane_tx_size; + const int max_blocks_high = max_block_high(xd, plane_bsize, plane); + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + + if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + + plane_tx_size = + plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0] + : mbmi->inter_tx_size[tx_row][tx_col]; + + if (tx_size == plane_tx_size) { + TOKEN_STATS tmp_token_stats; + init_token_stats(&tmp_token_stats); +#if !CONFIG_PVQ + pack_mb_tokens(w, tp, tok_end, bit_depth, tx_size, &tmp_token_stats); +#else + pack_pvq_tokens(w, x, xd, plane, bsize, tx_size); +#endif +#if CONFIG_RD_DEBUG + token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost; + token_stats->cost += tmp_token_stats.cost; +#endif + } else { + const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; + const int bsl = tx_size_wide_unit[sub_txs]; + int i; + + assert(bsl > 0); + + for (i = 0; i < 4; ++i) { + const int offsetr = blk_row + (i >> 1) * bsl; + const int offsetc = blk_col + (i & 0x01) * bsl; + const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs]; + + if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue; + + pack_txb_tokens(w, tp, tok_end, +#if CONFIG_PVQ + x, +#endif + xd, mbmi, plane, plane_bsize, bit_depth, block, offsetr, + offsetc, sub_txs, token_stats); + block += step; + } + } +} +#endif + +static void write_segment_id(aom_writer *w, const struct segmentation *seg, + struct segmentation_probs *segp, int segment_id) { + if (seg->enabled && seg->update_map) { +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, segment_id, segp->tree_cdf, MAX_SEGMENTS); +#else + aom_write_tree(w, av1_segment_tree, segp->tree_probs, segment_id, 3, 0); +#endif + } +} + +// This function encodes the reference frame +static void write_ref_frames(const AV1_COMMON *cm, const MACROBLOCKD *xd, + aom_writer *w) { + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const int is_compound = has_second_ref(mbmi); + const int segment_id = mbmi->segment_id; + + // If segment level coding of this signal is disabled... + // or the segment allows multiple reference frame options + if (segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME)) { + assert(!is_compound); + assert(mbmi->ref_frame[0] == + get_segdata(&cm->seg, segment_id, SEG_LVL_REF_FRAME)); + } else { + // does the feature use compound prediction or not + // (if not specified at the frame/segment level) + if (cm->reference_mode == REFERENCE_MODE_SELECT) { +#if SUB8X8_COMP_REF + aom_write(w, is_compound, av1_get_reference_mode_prob(cm, xd)); +#else + if (mbmi->sb_type >= BLOCK_8X8) + aom_write(w, is_compound, av1_get_reference_mode_prob(cm, xd)); +#endif + } else { + assert((!is_compound) == (cm->reference_mode == SINGLE_REFERENCE)); + } + + if (is_compound) { +#if CONFIG_EXT_REFS + const int bit = (mbmi->ref_frame[0] == GOLDEN_FRAME || + mbmi->ref_frame[0] == LAST3_FRAME); + const int bit_bwd = mbmi->ref_frame[1] == ALTREF_FRAME; +#else // CONFIG_EXT_REFS + const int bit = mbmi->ref_frame[0] == GOLDEN_FRAME; +#endif // CONFIG_EXT_REFS + + aom_write(w, bit, av1_get_pred_prob_comp_ref_p(cm, xd)); + +#if CONFIG_EXT_REFS + if (!bit) { + const int bit1 = mbmi->ref_frame[0] == LAST_FRAME; + aom_write(w, bit1, av1_get_pred_prob_comp_ref_p1(cm, xd)); + } else { + const int bit2 = mbmi->ref_frame[0] == GOLDEN_FRAME; + aom_write(w, bit2, av1_get_pred_prob_comp_ref_p2(cm, xd)); + } + aom_write(w, bit_bwd, av1_get_pred_prob_comp_bwdref_p(cm, xd)); +#endif // CONFIG_EXT_REFS + } else { +#if CONFIG_EXT_REFS + const int bit0 = (mbmi->ref_frame[0] == ALTREF_FRAME || + mbmi->ref_frame[0] == BWDREF_FRAME); + aom_write(w, bit0, av1_get_pred_prob_single_ref_p1(cm, xd)); + + if (bit0) { + const int bit1 = mbmi->ref_frame[0] == ALTREF_FRAME; + aom_write(w, bit1, av1_get_pred_prob_single_ref_p2(cm, xd)); + } else { + const int bit2 = (mbmi->ref_frame[0] == LAST3_FRAME || + mbmi->ref_frame[0] == GOLDEN_FRAME); + aom_write(w, bit2, av1_get_pred_prob_single_ref_p3(cm, xd)); + + if (!bit2) { + const int bit3 = mbmi->ref_frame[0] != LAST_FRAME; + aom_write(w, bit3, av1_get_pred_prob_single_ref_p4(cm, xd)); + } else { + const int bit4 = mbmi->ref_frame[0] != LAST3_FRAME; + aom_write(w, bit4, av1_get_pred_prob_single_ref_p5(cm, xd)); + } + } +#else // CONFIG_EXT_REFS + const int bit0 = mbmi->ref_frame[0] != LAST_FRAME; + aom_write(w, bit0, av1_get_pred_prob_single_ref_p1(cm, xd)); + + if (bit0) { + const int bit1 = mbmi->ref_frame[0] != GOLDEN_FRAME; + aom_write(w, bit1, av1_get_pred_prob_single_ref_p2(cm, xd)); + } +#endif // CONFIG_EXT_REFS + } + } +} + +#if CONFIG_FILTER_INTRA +static void write_filter_intra_mode_info(const AV1_COMMON *const cm, + const MB_MODE_INFO *const mbmi, + aom_writer *w) { + if (mbmi->mode == DC_PRED +#if CONFIG_PALETTE + && mbmi->palette_mode_info.palette_size[0] == 0 +#endif // CONFIG_PALETTE + ) { + aom_write(w, mbmi->filter_intra_mode_info.use_filter_intra_mode[0], + cm->fc->filter_intra_probs[0]); + if (mbmi->filter_intra_mode_info.use_filter_intra_mode[0]) { + const FILTER_INTRA_MODE mode = + mbmi->filter_intra_mode_info.filter_intra_mode[0]; + write_uniform(w, FILTER_INTRA_MODES, mode); + } + } + + if (mbmi->uv_mode == DC_PRED +#if CONFIG_PALETTE + && mbmi->palette_mode_info.palette_size[1] == 0 +#endif // CONFIG_PALETTE + ) { + aom_write(w, mbmi->filter_intra_mode_info.use_filter_intra_mode[1], + cm->fc->filter_intra_probs[1]); + if (mbmi->filter_intra_mode_info.use_filter_intra_mode[1]) { + const FILTER_INTRA_MODE mode = + mbmi->filter_intra_mode_info.filter_intra_mode[1]; + write_uniform(w, FILTER_INTRA_MODES, mode); + } + } +} +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_EXT_INTRA +static void write_intra_angle_info(const MACROBLOCKD *xd, + FRAME_CONTEXT *const ec_ctx, aom_writer *w) { + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const BLOCK_SIZE bsize = mbmi->sb_type; +#if CONFIG_INTRA_INTERP + const int intra_filter_ctx = av1_get_pred_context_intra_interp(xd); + int p_angle; +#endif // CONFIG_INTRA_INTERP + + (void)ec_ctx; + if (bsize < BLOCK_8X8) return; + + if (av1_is_directional_mode(mbmi->mode, bsize)) { + write_uniform(w, 2 * MAX_ANGLE_DELTA + 1, + MAX_ANGLE_DELTA + mbmi->angle_delta[0]); +#if CONFIG_INTRA_INTERP + p_angle = mode_to_angle_map[mbmi->mode] + mbmi->angle_delta[0] * ANGLE_STEP; + if (av1_is_intra_filter_switchable(p_angle)) { +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, mbmi->intra_filter, + ec_ctx->intra_filter_cdf[intra_filter_ctx], + INTRA_FILTERS); +#else + av1_write_token(w, av1_intra_filter_tree, + ec_ctx->intra_filter_probs[intra_filter_ctx], + &intra_filter_encodings[mbmi->intra_filter]); +#endif // CONFIG_EC_MULTISYMBOL + } +#endif // CONFIG_INTRA_INTERP + } + + if (av1_is_directional_mode(mbmi->uv_mode, bsize)) { + write_uniform(w, 2 * MAX_ANGLE_DELTA + 1, + MAX_ANGLE_DELTA + mbmi->angle_delta[1]); + } +} +#endif // CONFIG_EXT_INTRA + +static void write_mb_interp_filter(AV1_COMP *cpi, const MACROBLOCKD *xd, + aom_writer *w) { + AV1_COMMON *const cm = &cpi->common; + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + + if (!av1_is_interp_needed(xd)) { +#if CONFIG_DUAL_FILTER + for (int i = 0; i < 4; ++i) + assert(mbmi->interp_filter[i] == (cm->interp_filter == SWITCHABLE + ? EIGHTTAP_REGULAR + : cm->interp_filter)); +#else + assert(mbmi->interp_filter == (cm->interp_filter == SWITCHABLE + ? EIGHTTAP_REGULAR + : cm->interp_filter)); +#endif // CONFIG_DUAL_FILTER + return; + } + if (cm->interp_filter == SWITCHABLE) { +#if CONFIG_DUAL_FILTER + int dir; + for (dir = 0; dir < 2; ++dir) { + if (has_subpel_mv_component(xd->mi[0], xd, dir) || + (mbmi->ref_frame[1] > INTRA_FRAME && + has_subpel_mv_component(xd->mi[0], xd, dir + 2))) { + const int ctx = av1_get_pred_context_switchable_interp(xd, dir); +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, av1_switchable_interp_ind[mbmi->interp_filter[dir]], + ec_ctx->switchable_interp_cdf[ctx], + SWITCHABLE_FILTERS); +#else + av1_write_token(w, av1_switchable_interp_tree, + ec_ctx->switchable_interp_prob[ctx], + &switchable_interp_encodings[mbmi->interp_filter[dir]]); +#endif + ++cpi->interp_filter_selected[0][mbmi->interp_filter[dir]]; + } else { + assert(mbmi->interp_filter[dir] == EIGHTTAP_REGULAR); + } + } +#else + { + const int ctx = av1_get_pred_context_switchable_interp(xd); +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, av1_switchable_interp_ind[mbmi->interp_filter], + ec_ctx->switchable_interp_cdf[ctx], SWITCHABLE_FILTERS); +#else + av1_write_token(w, av1_switchable_interp_tree, + ec_ctx->switchable_interp_prob[ctx], + &switchable_interp_encodings[mbmi->interp_filter]); +#endif + ++cpi->interp_filter_selected[0][mbmi->interp_filter]; + } +#endif // CONFIG_DUAL_FILTER + } +} + +#if CONFIG_PALETTE +#if CONFIG_PALETTE_DELTA_ENCODING +// Write luma palette color values with delta encoding. Write the first value as +// literal, and the deltas between each value and the previous one. The luma +// palette is sorted so each delta is larger than 0. +static void write_palette_colors_y(const PALETTE_MODE_INFO *const pmi, + int bit_depth, aom_writer *w) { + const int n = pmi->palette_size[0]; + int min_bits, i; + int bits = av1_get_palette_delta_bits_y(pmi, bit_depth, &min_bits); + aom_write_literal(w, bits - min_bits, 2); + aom_write_literal(w, pmi->palette_colors[0], bit_depth); + for (i = 1; i < n; ++i) { + aom_write_literal( + w, pmi->palette_colors[i] - pmi->palette_colors[i - 1] - 1, bits); + bits = + AOMMIN(bits, av1_ceil_log2((1 << bit_depth) - pmi->palette_colors[i])); + } +} + +// Write chroma palette color values. Use delta encoding for u channel as its +// palette is sorted. For v channel, either use delta encoding or transmit +// raw values directly, whichever costs less. +static void write_palette_colors_uv(const PALETTE_MODE_INFO *const pmi, + int bit_depth, aom_writer *w) { + int i; + const int n = pmi->palette_size[1]; +#if CONFIG_HIGHBITDEPTH + const uint16_t *colors_u = pmi->palette_colors + PALETTE_MAX_SIZE; + const uint16_t *colors_v = pmi->palette_colors + 2 * PALETTE_MAX_SIZE; +#else + const uint8_t *colors_u = pmi->palette_colors + PALETTE_MAX_SIZE; + const uint8_t *colors_v = pmi->palette_colors + 2 * PALETTE_MAX_SIZE; +#endif // CONFIG_HIGHBITDEPTH + // U channel colors. + int min_bits_u = 0; + int bits_u = av1_get_palette_delta_bits_u(pmi, bit_depth, &min_bits_u); + aom_write_literal(w, bits_u - min_bits_u, 2); + aom_write_literal(w, colors_u[0], bit_depth); + for (i = 1; i < n; ++i) { + aom_write_literal(w, colors_u[i] - colors_u[i - 1], bits_u); + bits_u = AOMMIN(bits_u, av1_ceil_log2(1 + (1 << bit_depth) - colors_u[i])); + } + // V channel colors. + const int max_val = 1 << bit_depth; + int zero_count = 0, min_bits_v = 0; + int bits_v = + av1_get_palette_delta_bits_v(pmi, bit_depth, &zero_count, &min_bits_v); + const int rate_using_delta = + 2 + bit_depth + (bits_v + 1) * (n - 1) - zero_count; + const int rate_using_raw = bit_depth * n; + if (rate_using_delta < rate_using_raw) { // delta encoding + aom_write_bit(w, 1); + aom_write_literal(w, bits_v - min_bits_v, 2); + aom_write_literal(w, colors_v[0], bit_depth); + for (i = 1; i < n; ++i) { + if (colors_v[i] == colors_v[i - 1]) { // No need to signal sign bit. + aom_write_literal(w, 0, bits_v); + continue; + } + const int delta = abs((int)colors_v[i] - colors_v[i - 1]); + const int sign_bit = colors_v[i] < colors_v[i - 1]; + if (delta <= max_val - delta) { + aom_write_literal(w, delta, bits_v); + aom_write_bit(w, sign_bit); + } else { + aom_write_literal(w, max_val - delta, bits_v); + aom_write_bit(w, !sign_bit); + } + } + } else { // Transmit raw values. + aom_write_bit(w, 0); + for (i = 0; i < n; ++i) aom_write_literal(w, colors_v[i], bit_depth); + } +} +#endif // CONFIG_PALETTE_DELTA_ENCODING + +static void write_palette_mode_info(const AV1_COMMON *cm, const MACROBLOCKD *xd, + const MODE_INFO *const mi, aom_writer *w) { + const MB_MODE_INFO *const mbmi = &mi->mbmi; + const MODE_INFO *const above_mi = xd->above_mi; + const MODE_INFO *const left_mi = xd->left_mi; + const BLOCK_SIZE bsize = mbmi->sb_type; + const PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; + + if (mbmi->mode == DC_PRED) { + const int n = pmi->palette_size[0]; + int palette_y_mode_ctx = 0; + if (above_mi) + palette_y_mode_ctx += + (above_mi->mbmi.palette_mode_info.palette_size[0] > 0); + if (left_mi) + palette_y_mode_ctx += + (left_mi->mbmi.palette_mode_info.palette_size[0] > 0); + aom_write( + w, n > 0, + av1_default_palette_y_mode_prob[bsize - BLOCK_8X8][palette_y_mode_ctx]); + if (n > 0) { + av1_write_token(w, av1_palette_size_tree, + av1_default_palette_y_size_prob[bsize - BLOCK_8X8], + &palette_size_encodings[n - PALETTE_MIN_SIZE]); +#if CONFIG_PALETTE_DELTA_ENCODING + write_palette_colors_y(pmi, cm->bit_depth, w); +#else + int i; + for (i = 0; i < n; ++i) + aom_write_literal(w, pmi->palette_colors[i], cm->bit_depth); +#endif // CONFIG_PALETTE_DELTA_ENCODING + write_uniform(w, n, pmi->palette_first_color_idx[0]); + } + } + + if (mbmi->uv_mode == DC_PRED) { + const int n = pmi->palette_size[1]; + const int palette_uv_mode_ctx = (pmi->palette_size[0] > 0); + aom_write(w, n > 0, av1_default_palette_uv_mode_prob[palette_uv_mode_ctx]); + if (n > 0) { + av1_write_token(w, av1_palette_size_tree, + av1_default_palette_uv_size_prob[bsize - BLOCK_8X8], + &palette_size_encodings[n - PALETTE_MIN_SIZE]); +#if CONFIG_PALETTE_DELTA_ENCODING + write_palette_colors_uv(pmi, cm->bit_depth, w); +#else + int i; + for (i = 0; i < n; ++i) { + aom_write_literal(w, pmi->palette_colors[PALETTE_MAX_SIZE + i], + cm->bit_depth); + aom_write_literal(w, pmi->palette_colors[2 * PALETTE_MAX_SIZE + i], + cm->bit_depth); + } +#endif // CONFIG_PALETTE_DELTA_ENCODING + write_uniform(w, n, pmi->palette_first_color_idx[1]); + } + } +} +#endif // CONFIG_PALETTE + +void av1_write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd, +#if CONFIG_SUPERTX + const int supertx_enabled, +#endif +#if CONFIG_TXK_SEL + int block, int plane, +#endif + aom_writer *w) { + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const int is_inter = is_inter_block(mbmi); +#if CONFIG_VAR_TX + const TX_SIZE tx_size = is_inter ? mbmi->min_tx_size : mbmi->tx_size; +#else + const TX_SIZE tx_size = mbmi->tx_size; +#endif // CONFIG_VAR_TX +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + +#if !CONFIG_TXK_SEL + TX_TYPE tx_type = mbmi->tx_type; +#else + // Only y plane's tx_type is transmitted + if (plane > 0) return; + PLANE_TYPE plane_type = get_plane_type(plane); + TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); +#endif + + if (!FIXED_TX_TYPE) { +#if CONFIG_EXT_TX + const TX_SIZE square_tx_size = txsize_sqr_map[tx_size]; + const BLOCK_SIZE bsize = mbmi->sb_type; + if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) > + 1 && + ((!cm->seg.enabled && cm->base_qindex > 0) || + (cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) && + !mbmi->skip && +#if CONFIG_SUPERTX + !supertx_enabled && +#endif // CONFIG_SUPERTX + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + const int eset = + get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used); + if (is_inter) { + assert(ext_tx_used_inter[eset][tx_type]); + if (eset > 0) { +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, av1_ext_tx_inter_ind[eset][tx_type], + ec_ctx->inter_ext_tx_cdf[eset][square_tx_size], + ext_tx_cnt_inter[eset]); +#else + av1_write_token(w, av1_ext_tx_inter_tree[eset], + ec_ctx->inter_ext_tx_prob[eset][square_tx_size], + &ext_tx_inter_encodings[eset][tx_type]); +#endif + } + } else if (ALLOW_INTRA_EXT_TX) { + assert(ext_tx_used_intra[eset][tx_type]); + if (eset > 0) { +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol( + w, av1_ext_tx_intra_ind[eset][tx_type], + ec_ctx->intra_ext_tx_cdf[eset][square_tx_size][mbmi->mode], + ext_tx_cnt_intra[eset]); +#else + av1_write_token( + w, av1_ext_tx_intra_tree[eset], + ec_ctx->intra_ext_tx_prob[eset][square_tx_size][mbmi->mode], + &ext_tx_intra_encodings[eset][tx_type]); +#endif + } + } + } +#else + if (tx_size < TX_32X32 && + ((!cm->seg.enabled && cm->base_qindex > 0) || + (cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) && + !mbmi->skip && +#if CONFIG_SUPERTX + !supertx_enabled && +#endif // CONFIG_SUPERTX + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + if (is_inter) { +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, av1_ext_tx_ind[tx_type], + ec_ctx->inter_ext_tx_cdf[tx_size], TX_TYPES); +#else + av1_write_token(w, av1_ext_tx_tree, ec_ctx->inter_ext_tx_prob[tx_size], + &ext_tx_encodings[tx_type]); +#endif + } else { +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol( + w, av1_ext_tx_ind[tx_type], + ec_ctx->intra_ext_tx_cdf[tx_size] + [intra_mode_to_tx_type_context[mbmi->mode]], + TX_TYPES); +#else + av1_write_token( + w, av1_ext_tx_tree, + ec_ctx + ->intra_ext_tx_prob[tx_size] + [intra_mode_to_tx_type_context[mbmi->mode]], + &ext_tx_encodings[tx_type]); +#endif + } + } +#endif // CONFIG_EXT_TX + } +} + +static void write_intra_mode(FRAME_CONTEXT *frame_ctx, BLOCK_SIZE bsize, + PREDICTION_MODE mode, aom_writer *w) { +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, av1_intra_mode_ind[mode], + frame_ctx->y_mode_cdf[size_group_lookup[bsize]], + INTRA_MODES); +#else + av1_write_token(w, av1_intra_mode_tree, + frame_ctx->y_mode_prob[size_group_lookup[bsize]], + &intra_mode_encodings[mode]); +#endif +} + +static void write_intra_uv_mode(FRAME_CONTEXT *frame_ctx, + PREDICTION_MODE uv_mode, PREDICTION_MODE y_mode, + aom_writer *w) { +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, av1_intra_mode_ind[uv_mode], + frame_ctx->uv_mode_cdf[y_mode], INTRA_MODES); +#else + av1_write_token(w, av1_intra_mode_tree, frame_ctx->uv_mode_prob[y_mode], + &intra_mode_encodings[uv_mode]); +#endif +} + +static void pack_inter_mode_mvs(AV1_COMP *cpi, const int mi_row, + const int mi_col, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif + aom_writer *w) { + AV1_COMMON *const cm = &cpi->common; +#if CONFIG_DELTA_Q || CONFIG_EC_ADAPT + MACROBLOCK *const x = &cpi->td.mb; + MACROBLOCKD *const xd = &x->e_mbd; +#else + const MACROBLOCK *x = &cpi->td.mb; + const MACROBLOCKD *xd = &x->e_mbd; +#endif +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif +#if !CONFIG_REF_MV + nmv_context *nmvc = &ec_ctx->nmvc; +#endif + const MODE_INFO *mi = xd->mi[0]; + + const struct segmentation *const seg = &cm->seg; + struct segmentation_probs *const segp = &cm->fc->seg; + const MB_MODE_INFO *const mbmi = &mi->mbmi; + const MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; + const PREDICTION_MODE mode = mbmi->mode; + const int segment_id = mbmi->segment_id; + const BLOCK_SIZE bsize = mbmi->sb_type; + const int allow_hp = cm->allow_high_precision_mv; + const int is_inter = is_inter_block(mbmi); + const int is_compound = has_second_ref(mbmi); + int skip, ref; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + (void)mi_row; + (void)mi_col; + + if (seg->update_map) { + if (seg->temporal_update) { + const int pred_flag = mbmi->seg_id_predicted; + aom_prob pred_prob = av1_get_pred_prob_seg_id(segp, xd); + aom_write(w, pred_flag, pred_prob); + if (!pred_flag) write_segment_id(w, seg, segp, segment_id); + } else { + write_segment_id(w, seg, segp, segment_id); + } + } + +#if CONFIG_SUPERTX + if (supertx_enabled) + skip = mbmi->skip; + else + skip = write_skip(cm, xd, segment_id, mi, w); +#else + skip = write_skip(cm, xd, segment_id, mi, w); +#endif // CONFIG_SUPERTX +#if CONFIG_DELTA_Q + if (cm->delta_q_present_flag) { + int super_block_upper_left = + ((mi_row & MAX_MIB_MASK) == 0) && ((mi_col & MAX_MIB_MASK) == 0); + if ((bsize != BLOCK_LARGEST || skip == 0) && super_block_upper_left) { + assert(mbmi->current_q_index > 0); + int reduced_delta_qindex = + (mbmi->current_q_index - xd->prev_qindex) / cm->delta_q_res; + write_delta_qindex(cm, xd, reduced_delta_qindex, w); + xd->prev_qindex = mbmi->current_q_index; +#if CONFIG_EXT_DELTA_Q + if (cm->delta_lf_present_flag) { + int reduced_delta_lflevel = + (mbmi->current_delta_lf_from_base - xd->prev_delta_lf_from_base) / + cm->delta_lf_res; + write_delta_lflevel(cm, xd, reduced_delta_lflevel, w); + xd->prev_delta_lf_from_base = mbmi->current_delta_lf_from_base; + } +#endif // CONFIG_EXT_DELTA_Q + } + } +#endif + +#if CONFIG_SUPERTX + if (!supertx_enabled) +#endif // CONFIG_SUPERTX + if (!segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) + aom_write(w, is_inter, av1_get_intra_inter_prob(cm, xd)); + + if (cm->tx_mode == TX_MODE_SELECT && +#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX) +#if CONFIG_RECT_TX + bsize > BLOCK_4X4 && +#else + (bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter)) && +#endif // CONFIG_RECT_TX +#else + bsize >= BLOCK_8X8 && +#endif +#if CONFIG_SUPERTX + !supertx_enabled && +#endif // CONFIG_SUPERTX + !(is_inter && skip) && !xd->lossless[segment_id]) { +#if CONFIG_VAR_TX + if (is_inter) { // This implies skip flag is 0. + const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, bsize); + const int bh = tx_size_high_unit[max_tx_size]; + const int bw = tx_size_wide_unit[max_tx_size]; + const int width = block_size_wide[bsize] >> tx_size_wide_log2[0]; + const int height = block_size_high[bsize] >> tx_size_wide_log2[0]; + int idx, idy; + for (idy = 0; idy < height; idy += bh) + for (idx = 0; idx < width; idx += bw) + write_tx_size_vartx(cm, xd, mbmi, max_tx_size, height != width, idy, + idx, w); + } else { + set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, skip, xd); + write_selected_tx_size(cm, xd, w); + } + } else { + set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, skip, xd); +#else + write_selected_tx_size(cm, xd, w); +#endif + } + + if (!is_inter) { + if (bsize >= BLOCK_8X8 || unify_bsize) { + write_intra_mode(ec_ctx, bsize, mode, w); + } else { + int idx, idy; + const int num_4x4_w = num_4x4_blocks_wide_lookup[bsize]; + const int num_4x4_h = num_4x4_blocks_high_lookup[bsize]; + for (idy = 0; idy < 2; idy += num_4x4_h) { + for (idx = 0; idx < 2; idx += num_4x4_w) { + const PREDICTION_MODE b_mode = mi->bmi[idy * 2 + idx].as_mode; + write_intra_mode(ec_ctx, bsize, b_mode, w); + } + } + } +#if CONFIG_CB4X4 + if (is_chroma_reference(mi_row, mi_col, bsize, xd->plane[1].subsampling_x, + xd->plane[1].subsampling_y)) + write_intra_uv_mode(ec_ctx, mbmi->uv_mode, mode, w); +#else // !CONFIG_CB4X4 + write_intra_uv_mode(ec_ctx, mbmi->uv_mode, mode, w); +#endif // CONFIG_CB4X4 + +#if CONFIG_EXT_INTRA + write_intra_angle_info(xd, ec_ctx, w); +#endif // CONFIG_EXT_INTRA +#if CONFIG_PALETTE + if (bsize >= BLOCK_8X8 && cm->allow_screen_content_tools) + write_palette_mode_info(cm, xd, mi, w); +#endif // CONFIG_PALETTE +#if CONFIG_FILTER_INTRA + if (bsize >= BLOCK_8X8 || unify_bsize) + write_filter_intra_mode_info(cm, mbmi, w); +#endif // CONFIG_FILTER_INTRA + } else { + int16_t mode_ctx; + write_ref_frames(cm, xd, w); + +#if CONFIG_REF_MV +#if CONFIG_EXT_INTER + if (is_compound) + mode_ctx = mbmi_ext->compound_mode_context[mbmi->ref_frame[0]]; + else +#endif // CONFIG_EXT_INTER + mode_ctx = av1_mode_context_analyzer(mbmi_ext->mode_context, + mbmi->ref_frame, bsize, -1); +#else // CONFIG_REF_MV + mode_ctx = mbmi_ext->mode_context[mbmi->ref_frame[0]]; +#endif // CONFIG_REF_MV + + // If segment skip is not enabled code the mode. + if (!segfeature_active(seg, segment_id, SEG_LVL_SKIP)) { + if (bsize >= BLOCK_8X8 || unify_bsize) { +#if CONFIG_EXT_INTER + if (is_inter_compound_mode(mode)) + write_inter_compound_mode(cm, w, mode, mode_ctx); + else if (is_inter_singleref_mode(mode)) +#endif // CONFIG_EXT_INTER + write_inter_mode(w, mode, ec_ctx, mode_ctx); + +#if CONFIG_REF_MV +#if CONFIG_EXT_INTER + if (mode == NEWMV || mode == NEW_NEWMV || + have_nearmv_in_inter_mode(mode)) +#else + if (mode == NEARMV || mode == NEWMV) +#endif + write_drl_idx(cm, mbmi, mbmi_ext, w); + else + assert(mbmi->ref_mv_idx == 0); +#endif + } + } + +#if !CONFIG_DUAL_FILTER && !CONFIG_WARPED_MOTION && !CONFIG_GLOBAL_MOTION + write_mb_interp_filter(cpi, xd, w); +#endif // !CONFIG_DUAL_FILTER && !CONFIG_WARPED_MOTION + + if (bsize < BLOCK_8X8 && !unify_bsize) { + const int num_4x4_w = num_4x4_blocks_wide_lookup[bsize]; + const int num_4x4_h = num_4x4_blocks_high_lookup[bsize]; + int idx, idy; + for (idy = 0; idy < 2; idy += num_4x4_h) { + for (idx = 0; idx < 2; idx += num_4x4_w) { + const int j = idy * 2 + idx; + const PREDICTION_MODE b_mode = mi->bmi[j].as_mode; +#if CONFIG_REF_MV +#if CONFIG_EXT_INTER + if (!is_compound) +#endif // CONFIG_EXT_INTER + mode_ctx = av1_mode_context_analyzer(mbmi_ext->mode_context, + mbmi->ref_frame, bsize, j); +#endif +#if CONFIG_EXT_INTER + if (is_inter_compound_mode(b_mode)) + write_inter_compound_mode(cm, w, b_mode, mode_ctx); + else if (is_inter_singleref_mode(b_mode)) +#endif // CONFIG_EXT_INTER + write_inter_mode(w, b_mode, ec_ctx, mode_ctx); + +#if CONFIG_EXT_INTER + if (b_mode == NEWMV || b_mode == NEW_NEWMV) { +#else + if (b_mode == NEWMV) { +#endif // CONFIG_EXT_INTER + for (ref = 0; ref < 1 + is_compound; ++ref) { +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], ref, + mbmi->ref_mv_idx); + nmv_context *nmvc = &ec_ctx->nmvc[nmv_ctx]; +#endif + av1_encode_mv(cpi, w, &mi->bmi[j].as_mv[ref].as_mv, +#if CONFIG_EXT_INTER + &mi->bmi[j].ref_mv[ref].as_mv, +#else +#if CONFIG_REF_MV + &mi->bmi[j].pred_mv[ref].as_mv, +#else + &mbmi_ext->ref_mvs[mbmi->ref_frame[ref]][0].as_mv, +#endif // CONFIG_REF_MV +#endif // CONFIG_EXT_INTER + nmvc, allow_hp); + } + } +#if CONFIG_EXT_INTER + else if (b_mode == NEAREST_NEWMV || b_mode == NEAR_NEWMV) { +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], 1, + mbmi->ref_mv_idx); + nmv_context *nmvc = &ec_ctx->nmvc[nmv_ctx]; +#endif + av1_encode_mv(cpi, w, &mi->bmi[j].as_mv[1].as_mv, + &mi->bmi[j].ref_mv[1].as_mv, nmvc, allow_hp); + } else if (b_mode == NEW_NEARESTMV || b_mode == NEW_NEARMV) { +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], 0, + mbmi->ref_mv_idx); + nmv_context *nmvc = &ec_ctx->nmvc[nmv_ctx]; +#endif + av1_encode_mv(cpi, w, &mi->bmi[j].as_mv[0].as_mv, + &mi->bmi[j].ref_mv[0].as_mv, nmvc, allow_hp); + } +#endif // CONFIG_EXT_INTER + } + } + } else { +#if CONFIG_EXT_INTER + if (mode == NEWMV || mode == NEW_NEWMV) { +#else + if (mode == NEWMV) { +#endif // CONFIG_EXT_INTER + int_mv ref_mv; + for (ref = 0; ref < 1 + is_compound; ++ref) { +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], ref, + mbmi->ref_mv_idx); + nmv_context *nmvc = &ec_ctx->nmvc[nmv_ctx]; +#endif + ref_mv = mbmi_ext->ref_mvs[mbmi->ref_frame[ref]][0]; + av1_encode_mv(cpi, w, &mbmi->mv[ref].as_mv, &ref_mv.as_mv, nmvc, + allow_hp); + } +#if CONFIG_EXT_INTER + } else if (mode == NEAREST_NEWMV || mode == NEAR_NEWMV) { +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = + av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], 1, mbmi->ref_mv_idx); + nmv_context *nmvc = &ec_ctx->nmvc[nmv_ctx]; +#endif + av1_encode_mv(cpi, w, &mbmi->mv[1].as_mv, + &mbmi_ext->ref_mvs[mbmi->ref_frame[1]][0].as_mv, nmvc, + allow_hp); + } else if (mode == NEW_NEARESTMV || mode == NEW_NEARMV) { +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = + av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], 0, mbmi->ref_mv_idx); + nmv_context *nmvc = &ec_ctx->nmvc[nmv_ctx]; +#endif + av1_encode_mv(cpi, w, &mbmi->mv[0].as_mv, + &mbmi_ext->ref_mvs[mbmi->ref_frame[0]][0].as_mv, nmvc, + allow_hp); +#endif // CONFIG_EXT_INTER + } + } + +#if CONFIG_EXT_INTER + if (cpi->common.reference_mode != COMPOUND_REFERENCE && +#if CONFIG_SUPERTX + !supertx_enabled && +#endif // CONFIG_SUPERTX + is_interintra_allowed(mbmi)) { + const int interintra = mbmi->ref_frame[1] == INTRA_FRAME; + const int bsize_group = size_group_lookup[bsize]; + aom_write(w, interintra, cm->fc->interintra_prob[bsize_group]); + if (interintra) { + write_interintra_mode(w, mbmi->interintra_mode, + cm->fc->interintra_mode_prob[bsize_group]); + if (is_interintra_wedge_used(bsize)) { + aom_write(w, mbmi->use_wedge_interintra, + cm->fc->wedge_interintra_prob[bsize]); + if (mbmi->use_wedge_interintra) { + aom_write_literal(w, mbmi->interintra_wedge_index, + get_wedge_bits_lookup(bsize)); + assert(mbmi->interintra_wedge_sign == 0); + } + } + } + } +#endif // CONFIG_EXT_INTER + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +#if CONFIG_SUPERTX + if (!supertx_enabled) +#endif // CONFIG_SUPERTX +#if CONFIG_EXT_INTER + if (mbmi->ref_frame[1] != INTRA_FRAME) +#endif // CONFIG_EXT_INTER + write_motion_mode(cm, mi, w); +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + +#if CONFIG_EXT_INTER + if (cpi->common.reference_mode != SINGLE_REFERENCE && + is_inter_compound_mode(mbmi->mode) +#if CONFIG_MOTION_VAR + && mbmi->motion_mode == SIMPLE_TRANSLATION +#endif // CONFIG_MOTION_VAR + && is_any_masked_compound_used(bsize)) { +#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE + av1_write_token(w, av1_compound_type_tree, + cm->fc->compound_type_prob[bsize], + &compound_type_encodings[mbmi->interinter_compound_type]); +#endif // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE +#if CONFIG_WEDGE + if (mbmi->interinter_compound_type == COMPOUND_WEDGE) { + aom_write_literal(w, mbmi->wedge_index, get_wedge_bits_lookup(bsize)); + aom_write_bit(w, mbmi->wedge_sign); + } +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + if (mbmi->interinter_compound_type == COMPOUND_SEG) { + aom_write_literal(w, mbmi->mask_type, MAX_SEG_MASK_BITS); + } +#endif // CONFIG_COMPOUND_SEGMENT + } +#endif // CONFIG_EXT_INTER + +#if CONFIG_DUAL_FILTER || CONFIG_WARPED_MOTION || CONFIG_GLOBAL_MOTION + write_mb_interp_filter(cpi, xd, w); +#endif // CONFIG_DUAL_FILTE || CONFIG_WARPED_MOTION + } + +#if !CONFIG_TXK_SEL + av1_write_tx_type(cm, xd, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + w); +#endif // !CONFIG_TXK_SEL +} + +#if CONFIG_DELTA_Q +static void write_mb_modes_kf(AV1_COMMON *cm, MACROBLOCKD *xd, const int mi_row, + const int mi_col, aom_writer *w) { + int skip; +#else +static void write_mb_modes_kf(AV1_COMMON *cm, const MACROBLOCKD *xd, + const int mi_row, const int mi_col, + aom_writer *w) { +#endif + const struct segmentation *const seg = &cm->seg; + struct segmentation_probs *const segp = &cm->fc->seg; + const MODE_INFO *const mi = xd->mi[0]; + const MODE_INFO *const above_mi = xd->above_mi; + const MODE_INFO *const left_mi = xd->left_mi; + const MB_MODE_INFO *const mbmi = &mi->mbmi; + const BLOCK_SIZE bsize = mbmi->sb_type; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + (void)mi_row; + (void)mi_col; + +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + + if (seg->update_map) write_segment_id(w, seg, segp, mbmi->segment_id); + +#if CONFIG_DELTA_Q + skip = write_skip(cm, xd, mbmi->segment_id, mi, w); + if (cm->delta_q_present_flag) { + int super_block_upper_left = + ((mi_row & MAX_MIB_MASK) == 0) && ((mi_col & MAX_MIB_MASK) == 0); + if ((bsize != BLOCK_LARGEST || skip == 0) && super_block_upper_left) { + assert(mbmi->current_q_index > 0); + int reduced_delta_qindex = + (mbmi->current_q_index - xd->prev_qindex) / cm->delta_q_res; + write_delta_qindex(cm, xd, reduced_delta_qindex, w); + xd->prev_qindex = mbmi->current_q_index; +#if CONFIG_EXT_DELTA_Q + if (cm->delta_lf_present_flag) { + int reduced_delta_lflevel = + (mbmi->current_delta_lf_from_base - xd->prev_delta_lf_from_base) / + cm->delta_lf_res; + write_delta_lflevel(cm, xd, reduced_delta_lflevel, w); + xd->prev_delta_lf_from_base = mbmi->current_delta_lf_from_base; + } +#endif // CONFIG_EXT_DELTA_Q + } + } +#else + write_skip(cm, xd, mbmi->segment_id, mi, w); +#endif + + if (cm->tx_mode == TX_MODE_SELECT && +#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX) +#if CONFIG_RECT_TX + bsize > BLOCK_4X4 && +#else + bsize >= BLOCK_8X8 && +#endif // CONFIG_RECT_TX +#else + bsize >= BLOCK_8X8 && +#endif + !xd->lossless[mbmi->segment_id]) + write_selected_tx_size(cm, xd, w); + +#if CONFIG_INTRABC + if (bsize >= BLOCK_8X8 && cm->allow_screen_content_tools) { + int use_intrabc = is_intrabc_block(mbmi); + aom_write(w, use_intrabc, INTRABC_PROB); + if (use_intrabc) { + assert(mbmi->mode == DC_PRED); + assert(mbmi->uv_mode == DC_PRED); + int_mv dv_ref; + av1_find_ref_dv(&dv_ref, mi_row, mi_col); + av1_encode_dv(w, &mbmi->mv[0].as_mv, &dv_ref.as_mv, &ec_ctx->ndvc); + return; + } + } +#endif // CONFIG_INTRABC + + if (bsize >= BLOCK_8X8 || unify_bsize) { + write_intra_mode_kf(cm, ec_ctx, mi, above_mi, left_mi, 0, mbmi->mode, w); + } else { + const int num_4x4_w = num_4x4_blocks_wide_lookup[bsize]; + const int num_4x4_h = num_4x4_blocks_high_lookup[bsize]; + int idx, idy; + + for (idy = 0; idy < 2; idy += num_4x4_h) { + for (idx = 0; idx < 2; idx += num_4x4_w) { + const int block = idy * 2 + idx; + write_intra_mode_kf(cm, ec_ctx, mi, above_mi, left_mi, block, + mi->bmi[block].as_mode, w); + } + } + } + +#if CONFIG_CB4X4 + if (is_chroma_reference(mi_row, mi_col, bsize, xd->plane[1].subsampling_x, + xd->plane[1].subsampling_y)) + write_intra_uv_mode(ec_ctx, mbmi->uv_mode, mbmi->mode, w); +#else // !CONFIG_CB4X4 + write_intra_uv_mode(ec_ctx, mbmi->uv_mode, mbmi->mode, w); +#endif // CONFIG_CB4X4 + +#if CONFIG_EXT_INTRA + write_intra_angle_info(xd, ec_ctx, w); +#endif // CONFIG_EXT_INTRA +#if CONFIG_PALETTE + if (bsize >= BLOCK_8X8 && cm->allow_screen_content_tools) + write_palette_mode_info(cm, xd, mi, w); +#endif // CONFIG_PALETTE +#if CONFIG_FILTER_INTRA + if (bsize >= BLOCK_8X8 || unify_bsize) + write_filter_intra_mode_info(cm, mbmi, w); +#endif // CONFIG_FILTER_INTRA + +#if !CONFIG_TXK_SEL + av1_write_tx_type(cm, xd, +#if CONFIG_SUPERTX + 0, +#endif + w); +#endif // !CONFIG_TXK_SEL +} + +#if CONFIG_SUPERTX +#define write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, \ + mi_row, mi_col) \ + write_modes_b(cpi, tile, w, tok, tok_end, supertx_enabled, mi_row, mi_col) +#else +#define write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, \ + mi_row, mi_col) \ + write_modes_b(cpi, tile, w, tok, tok_end, mi_row, mi_col) +#endif // CONFIG_SUPERTX + +#if CONFIG_RD_DEBUG +static void dump_mode_info(MODE_INFO *mi) { + printf("\nmi->mbmi.mi_row == %d\n", mi->mbmi.mi_row); + printf("&& mi->mbmi.mi_col == %d\n", mi->mbmi.mi_col); + printf("&& mi->mbmi.sb_type == %d\n", mi->mbmi.sb_type); + printf("&& mi->mbmi.tx_size == %d\n", mi->mbmi.tx_size); + if (mi->mbmi.sb_type >= BLOCK_8X8) { + printf("&& mi->mbmi.mode == %d\n", mi->mbmi.mode); + } else { + printf("&& mi->bmi[0].as_mode == %d\n", mi->bmi[0].as_mode); + } +} +static int rd_token_stats_mismatch(RD_STATS *rd_stats, TOKEN_STATS *token_stats, + int plane) { + if (rd_stats->txb_coeff_cost[plane] != token_stats->cost) { +#if CONFIG_VAR_TX + int r, c; +#endif + printf("\nplane %d rd_stats->txb_coeff_cost %d token_stats->cost %d\n", + plane, rd_stats->txb_coeff_cost[plane], token_stats->cost); +#if CONFIG_VAR_TX + printf("rd txb_coeff_cost_map\n"); + for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r) { + for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c) { + printf("%d ", rd_stats->txb_coeff_cost_map[plane][r][c]); + } + printf("\n"); + } + + printf("pack txb_coeff_cost_map\n"); + for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r) { + for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c) { + printf("%d ", token_stats->txb_coeff_cost_map[r][c]); + } + printf("\n"); + } +#endif + return 1; + } + return 0; +} +#endif + +static void write_mbmi_b(AV1_COMP *cpi, const TileInfo *const tile, + aom_writer *w, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif + int mi_row, int mi_col) { + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; + MODE_INFO *m; + int bh, bw; + xd->mi = cm->mi_grid_visible + (mi_row * cm->mi_stride + mi_col); + m = xd->mi[0]; + + assert(m->mbmi.sb_type <= cm->sb_size); + + bh = mi_size_high[m->mbmi.sb_type]; + bw = mi_size_wide[m->mbmi.sb_type]; + + cpi->td.mb.mbmi_ext = cpi->mbmi_ext_base + (mi_row * cm->mi_cols + mi_col); + + set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + + if (frame_is_intra_only(cm)) { + write_mb_modes_kf(cm, xd, mi_row, mi_col, w); + } else { +#if CONFIG_VAR_TX + xd->above_txfm_context = cm->above_txfm_context + mi_col; + xd->left_txfm_context = + xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK); +#endif +#if CONFIG_DUAL_FILTER + // has_subpel_mv_component needs the ref frame buffers set up to look + // up if they are scaled. has_subpel_mv_component is in turn needed by + // write_switchable_interp_filter, which is called by pack_inter_mode_mvs. + set_ref_ptrs(cm, xd, m->mbmi.ref_frame[0], m->mbmi.ref_frame[1]); +#endif // CONFIG_DUAL_FILTER +#if 0 + // NOTE(zoeliu): For debug + if (cm->current_video_frame == FRAME_TO_CHECK && cm->show_frame == 1) { + const PREDICTION_MODE mode = m->mbmi.mode; + const int segment_id = m->mbmi.segment_id; + const BLOCK_SIZE bsize = m->mbmi.sb_type; + + // For sub8x8, simply dump out the first sub8x8 block info + const PREDICTION_MODE b_mode = + (bsize < BLOCK_8X8) ? m->bmi[0].as_mode : -1; + const int mv_x = (bsize < BLOCK_8X8) ? + m->bmi[0].as_mv[0].as_mv.row : m->mbmi.mv[0].as_mv.row; + const int mv_y = (bsize < BLOCK_8X8) ? + m->bmi[0].as_mv[0].as_mv.col : m->mbmi.mv[0].as_mv.col; + + printf("Before pack_inter_mode_mvs(): " + "Frame=%d, (mi_row,mi_col)=(%d,%d), " + "mode=%d, segment_id=%d, bsize=%d, b_mode=%d, " + "mv[0]=(%d, %d), ref[0]=%d, ref[1]=%d\n", + cm->current_video_frame, mi_row, mi_col, + mode, segment_id, bsize, b_mode, mv_x, mv_y, + m->mbmi.ref_frame[0], m->mbmi.ref_frame[1]); + } +#endif // 0 + pack_inter_mode_mvs(cpi, mi_row, mi_col, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + w); + } +} + +static void write_tokens_b(AV1_COMP *cpi, const TileInfo *const tile, + aom_writer *w, const TOKENEXTRA **tok, + const TOKENEXTRA *const tok_end, int mi_row, + int mi_col) { + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; + MODE_INFO *const m = xd->mi[0]; + MB_MODE_INFO *const mbmi = &m->mbmi; + int plane; + int bh, bw; +#if CONFIG_PVQ || CONFIG_LV_MAP + MACROBLOCK *const x = &cpi->td.mb; + (void)tok; + (void)tok_end; +#endif + xd->mi = cm->mi_grid_visible + (mi_row * cm->mi_stride + mi_col); + + assert(mbmi->sb_type <= cm->sb_size); + + bh = mi_size_high[mbmi->sb_type]; + bw = mi_size_wide[mbmi->sb_type]; + cpi->td.mb.mbmi_ext = cpi->mbmi_ext_base + (mi_row * cm->mi_cols + mi_col); + + set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + +#if CONFIG_PALETTE + for (plane = 0; plane <= 1; ++plane) { + const uint8_t palette_size_plane = + mbmi->palette_mode_info.palette_size[plane]; + if (palette_size_plane > 0) { +#if CONFIG_INTRABC + assert(mbmi->use_intrabc == 0); +#endif + int rows, cols; + assert(mbmi->sb_type >= BLOCK_8X8); + av1_get_block_dimensions(mbmi->sb_type, plane, xd, NULL, NULL, &rows, + &cols); + assert(*tok < tok_end); + pack_palette_tokens(w, tok, palette_size_plane, rows * cols - 1); + assert(*tok < tok_end + mbmi->skip); + } + } +#endif // CONFIG_PALETTE + +#if CONFIG_COEF_INTERLEAVE + if (!mbmi->skip) { + const struct macroblockd_plane *const pd_y = &xd->plane[0]; + const struct macroblockd_plane *const pd_c = &xd->plane[1]; + const TX_SIZE tx_log2_y = mbmi->tx_size; + const TX_SIZE tx_log2_c = get_uv_tx_size(mbmi, pd_c); + const int tx_sz_y = (1 << tx_log2_y); + const int tx_sz_c = (1 << tx_log2_c); + + const BLOCK_SIZE plane_bsize_y = + get_plane_block_size(AOMMAX(mbmi->sb_type, 3), pd_y); + const BLOCK_SIZE plane_bsize_c = + get_plane_block_size(AOMMAX(mbmi->sb_type, 3), pd_c); + + const int num_4x4_w_y = num_4x4_blocks_wide_lookup[plane_bsize_y]; + const int num_4x4_w_c = num_4x4_blocks_wide_lookup[plane_bsize_c]; + const int num_4x4_h_y = num_4x4_blocks_high_lookup[plane_bsize_y]; + const int num_4x4_h_c = num_4x4_blocks_high_lookup[plane_bsize_c]; + + const int max_4x4_w_y = get_max_4x4_size(num_4x4_w_y, xd->mb_to_right_edge, + pd_y->subsampling_x); + const int max_4x4_h_y = get_max_4x4_size(num_4x4_h_y, xd->mb_to_bottom_edge, + pd_y->subsampling_y); + const int max_4x4_w_c = get_max_4x4_size(num_4x4_w_c, xd->mb_to_right_edge, + pd_c->subsampling_x); + const int max_4x4_h_c = get_max_4x4_size(num_4x4_h_c, xd->mb_to_bottom_edge, + pd_c->subsampling_y); + + // The max_4x4_w/h may be smaller than tx_sz under some corner cases, + // i.e. when the SB is splitted by tile boundaries. + const int tu_num_w_y = (max_4x4_w_y + tx_sz_y - 1) / tx_sz_y; + const int tu_num_h_y = (max_4x4_h_y + tx_sz_y - 1) / tx_sz_y; + const int tu_num_w_c = (max_4x4_w_c + tx_sz_c - 1) / tx_sz_c; + const int tu_num_h_c = (max_4x4_h_c + tx_sz_c - 1) / tx_sz_c; + const int tu_num_y = tu_num_w_y * tu_num_h_y; + const int tu_num_c = tu_num_w_c * tu_num_h_c; + + int tu_idx_y = 0, tu_idx_c = 0; + TOKEN_STATS token_stats; + init_token_stats(&token_stats); + + assert(*tok < tok_end); + + while (tu_idx_y < tu_num_y) { + pack_mb_tokens(w, tok, tok_end, cm->bit_depth, tx_log2_y, &token_stats); + assert(*tok < tok_end && (*tok)->token == EOSB_TOKEN); + (*tok)++; + tu_idx_y++; + + if (tu_idx_c < tu_num_c) { + pack_mb_tokens(w, tok, tok_end, cm->bit_depth, tx_log2_c, &token_stats); + assert(*tok < tok_end && (*tok)->token == EOSB_TOKEN); + (*tok)++; + + pack_mb_tokens(w, tok, tok_end, cm->bit_depth, tx_log2_c, &token_stats); + assert(*tok < tok_end && (*tok)->token == EOSB_TOKEN); + (*tok)++; + + tu_idx_c++; + } + } + + // In 422 case, it's possilbe that Chroma has more TUs than Luma + while (tu_idx_c < tu_num_c) { + pack_mb_tokens(w, tok, tok_end, cm->bit_depth, tx_log2_c, &token_stats); + assert(*tok < tok_end && (*tok)->token == EOSB_TOKEN); + (*tok)++; + + pack_mb_tokens(w, tok, tok_end, cm->bit_depth, tx_log2_c, &token_stats); + assert(*tok < tok_end && (*tok)->token == EOSB_TOKEN); + (*tok)++; + + tu_idx_c++; + } + } +#else // CONFIG_COEF_INTERLEAVE + if (!mbmi->skip) { +#if !CONFIG_PVQ && !CONFIG_LV_MAP + assert(*tok < tok_end); +#endif + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { +#if CONFIG_CB4X4 + if (!is_chroma_reference(mi_row, mi_col, mbmi->sb_type, + xd->plane[plane].subsampling_x, + xd->plane[plane].subsampling_y)) { + (*tok)++; + continue; + } +#endif +#if CONFIG_VAR_TX + const struct macroblockd_plane *const pd = &xd->plane[plane]; + BLOCK_SIZE bsize = mbmi->sb_type; +#if CONFIG_CB4X4 +#if CONFIG_CHROMA_2X2 + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); +#else + const BLOCK_SIZE plane_bsize = + AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd)); +#endif +#else + const BLOCK_SIZE plane_bsize = + get_plane_block_size(AOMMAX(bsize, BLOCK_8X8), pd); +#endif + + const int num_4x4_w = + block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; + const int num_4x4_h = + block_size_high[plane_bsize] >> tx_size_wide_log2[0]; + int row, col; + TOKEN_STATS token_stats; + init_token_stats(&token_stats); + + if (is_inter_block(mbmi)) { + const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize); + int block = 0; + const int step = + tx_size_wide_unit[max_tx_size] * tx_size_high_unit[max_tx_size]; + const int bkw = tx_size_wide_unit[max_tx_size]; + const int bkh = tx_size_high_unit[max_tx_size]; + for (row = 0; row < num_4x4_h; row += bkh) { + for (col = 0; col < num_4x4_w; col += bkw) { + pack_txb_tokens(w, tok, tok_end, +#if CONFIG_PVQ + x, +#endif + xd, mbmi, plane, plane_bsize, cm->bit_depth, block, + row, col, max_tx_size, &token_stats); + block += step; + } + } +#if CONFIG_RD_DEBUG + if (mbmi->sb_type >= BLOCK_8X8 && + rd_token_stats_mismatch(&mbmi->rd_stats, &token_stats, plane)) { + dump_mode_info(m); + assert(0); + } +#endif // CONFIG_RD_DEBUG + } else { + TX_SIZE tx = get_tx_size(plane, xd); +#if CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + tx = AOMMAX(TX_4X4, tx); +#endif + const int bkw = tx_size_wide_unit[tx]; + const int bkh = tx_size_high_unit[tx]; + for (row = 0; row < num_4x4_h; row += bkh) { + for (col = 0; col < num_4x4_w; col += bkw) { +#if !CONFIG_PVQ + pack_mb_tokens(w, tok, tok_end, cm->bit_depth, tx, &token_stats); +#else + pack_pvq_tokens(w, x, xd, plane, bsize, tx); +#endif + } + } + } +#else + TX_SIZE tx = get_tx_size(plane, xd); + TOKEN_STATS token_stats; +#if !CONFIG_PVQ + init_token_stats(&token_stats); +#if CONFIG_LV_MAP + (void)tx; + av1_write_coeffs_mb(cm, x, w, plane); +#else // CONFIG_LV_MAP + pack_mb_tokens(w, tok, tok_end, cm->bit_depth, tx, &token_stats); +#endif // CONFIG_LV_MAP + +#else + (void)token_stats; + pack_pvq_tokens(w, x, xd, plane, mbmi->sb_type, tx); +#endif +#if CONFIG_RD_DEBUG + if (is_inter_block(mbmi) && mbmi->sb_type >= BLOCK_8X8 && + rd_token_stats_mismatch(&mbmi->rd_stats, &token_stats, plane)) { + dump_mode_info(m); + assert(0); + } +#endif // CONFIG_RD_DEBUG +#endif // CONFIG_VAR_TX + +#if !CONFIG_PVQ && !CONFIG_LV_MAP + assert(*tok < tok_end && (*tok)->token == EOSB_TOKEN); + (*tok)++; +#endif + } + } +#endif // CONFIG_COEF_INTERLEAVE +} + +#if CONFIG_MOTION_VAR && CONFIG_NCOBMC +static void write_tokens_sb(AV1_COMP *cpi, const TileInfo *const tile, + aom_writer *w, const TOKENEXTRA **tok, + const TOKENEXTRA *const tok_end, int mi_row, + int mi_col, BLOCK_SIZE bsize) { + const AV1_COMMON *const cm = &cpi->common; + const int hbs = mi_size_wide[bsize] / 2; + PARTITION_TYPE partition; + BLOCK_SIZE subsize; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + partition = get_partition(cm, mi_row, mi_col, bsize); + subsize = get_subsize(bsize, partition); + + if (subsize < BLOCK_8X8 && !unify_bsize) { + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col); + } else { + switch (partition) { + case PARTITION_NONE: + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col); + break; + case PARTITION_HORZ: + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col); + if (mi_row + hbs < cm->mi_rows) + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row + hbs, mi_col); + break; + case PARTITION_VERT: + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col); + if (mi_col + hbs < cm->mi_cols) + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col + hbs); + break; + case PARTITION_SPLIT: + write_tokens_sb(cpi, tile, w, tok, tok_end, mi_row, mi_col, subsize); + write_tokens_sb(cpi, tile, w, tok, tok_end, mi_row, mi_col + hbs, + subsize); + write_tokens_sb(cpi, tile, w, tok, tok_end, mi_row + hbs, mi_col, + subsize); + write_tokens_sb(cpi, tile, w, tok, tok_end, mi_row + hbs, mi_col + hbs, + subsize); + break; +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_HORZ_A: + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col); + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col + hbs); + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row + hbs, mi_col); + break; + case PARTITION_HORZ_B: + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col); + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row + hbs, mi_col); + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row + hbs, mi_col + hbs); + break; + case PARTITION_VERT_A: + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col); + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row + hbs, mi_col); + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col + hbs); + break; + case PARTITION_VERT_B: + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col); + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col + hbs); + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row + hbs, mi_col + hbs); + break; +#endif // CONFIG_EXT_PARTITION_TYPES + default: assert(0); + } + } +} +#endif + +static void write_modes_b(AV1_COMP *cpi, const TileInfo *const tile, + aom_writer *w, const TOKENEXTRA **tok, + const TOKENEXTRA *const tok_end, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif + int mi_row, int mi_col) { + write_mbmi_b(cpi, tile, w, +#if CONFIG_SUPERTX + supertx_enabled, +#endif + mi_row, mi_col); +#if CONFIG_MOTION_VAR && CONFIG_NCOBMC + (void)tok; + (void)tok_end; +#else +#if !CONFIG_PVQ && CONFIG_SUPERTX + if (!supertx_enabled) +#endif + write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col); +#endif +} + +static void write_partition(const AV1_COMMON *const cm, + const MACROBLOCKD *const xd, int hbs, int mi_row, + int mi_col, PARTITION_TYPE p, BLOCK_SIZE bsize, + aom_writer *w) { + const int has_rows = (mi_row + hbs) < cm->mi_rows; + const int has_cols = (mi_col + hbs) < cm->mi_cols; + const int is_partition_point = bsize >= BLOCK_8X8; + const int ctx = is_partition_point + ? partition_plane_context(xd, mi_row, mi_col, +#if CONFIG_UNPOISON_PARTITION_CTX + has_rows, has_cols, +#endif + bsize) + : 0; +#if CONFIG_UNPOISON_PARTITION_CTX + const aom_prob *const probs = + ctx < PARTITION_CONTEXTS ? cm->fc->partition_prob[ctx] : NULL; +#else + const aom_prob *const probs = cm->fc->partition_prob[ctx]; +#endif + +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; + (void)cm; +#elif CONFIG_EC_MULTISYMBOL + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + + if (!is_partition_point) return; + + if (has_rows && has_cols) { +#if CONFIG_EXT_PARTITION_TYPES + if (bsize <= BLOCK_8X8) +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, p, ec_ctx->partition_cdf[ctx], PARTITION_TYPES); +#else + av1_write_token(w, av1_partition_tree, probs, &partition_encodings[p]); +#endif + else +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, p, ec_ctx->partition_cdf[ctx], EXT_PARTITION_TYPES); +#else + av1_write_token(w, av1_ext_partition_tree, probs, + &ext_partition_encodings[p]); +#endif // CONFIG_EC_MULTISYMBOL +#else +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, p, ec_ctx->partition_cdf[ctx], PARTITION_TYPES); +#else + av1_write_token(w, av1_partition_tree, probs, &partition_encodings[p]); +#endif +#endif // CONFIG_EXT_PARTITION_TYPES + } else if (!has_rows && has_cols) { + assert(p == PARTITION_SPLIT || p == PARTITION_HORZ); + aom_write(w, p == PARTITION_SPLIT, probs[1]); + } else if (has_rows && !has_cols) { + assert(p == PARTITION_SPLIT || p == PARTITION_VERT); + aom_write(w, p == PARTITION_SPLIT, probs[2]); + } else { + assert(p == PARTITION_SPLIT); + } +} + +#if CONFIG_SUPERTX +#define write_modes_sb_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, \ + mi_row, mi_col, bsize) \ + write_modes_sb(cpi, tile, w, tok, tok_end, supertx_enabled, mi_row, mi_col, \ + bsize) +#else +#define write_modes_sb_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, \ + mi_row, mi_col, bsize) \ + write_modes_sb(cpi, tile, w, tok, tok_end, mi_row, mi_col, bsize) +#endif // CONFIG_SUPERTX + +static void write_modes_sb(AV1_COMP *const cpi, const TileInfo *const tile, + aom_writer *const w, const TOKENEXTRA **tok, + const TOKENEXTRA *const tok_end, +#if CONFIG_SUPERTX + int supertx_enabled, +#endif + int mi_row, int mi_col, BLOCK_SIZE bsize) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; + const int hbs = mi_size_wide[bsize] / 2; + const PARTITION_TYPE partition = get_partition(cm, mi_row, mi_col, bsize); + const BLOCK_SIZE subsize = get_subsize(bsize, partition); +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + +#if CONFIG_SUPERTX + const int mi_offset = mi_row * cm->mi_stride + mi_col; + MB_MODE_INFO *mbmi; + const int pack_token = !supertx_enabled; + TX_SIZE supertx_size; + int plane; +#endif + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + write_partition(cm, xd, hbs, mi_row, mi_col, partition, bsize, w); +#if CONFIG_SUPERTX + mbmi = &cm->mi_grid_visible[mi_offset]->mbmi; + xd->mi = cm->mi_grid_visible + mi_offset; + set_mi_row_col(xd, tile, mi_row, mi_size_high[bsize], mi_col, + mi_size_wide[bsize], +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + if (!supertx_enabled && !frame_is_intra_only(cm) && + partition != PARTITION_NONE && bsize <= MAX_SUPERTX_BLOCK_SIZE && + !xd->lossless[0]) { + aom_prob prob; + supertx_size = max_txsize_lookup[bsize]; + prob = cm->fc->supertx_prob[partition_supertx_context_lookup[partition]] + [supertx_size]; + supertx_enabled = (xd->mi[0]->mbmi.tx_size == supertx_size); + aom_write(w, supertx_enabled, prob); + } +#endif // CONFIG_SUPERTX + if (subsize < BLOCK_8X8 && !unify_bsize) { + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, mi_row, + mi_col); + } else { + switch (partition) { + case PARTITION_NONE: + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col); + break; + case PARTITION_HORZ: + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col); + if (mi_row + hbs < cm->mi_rows) + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row + hbs, mi_col); + break; + case PARTITION_VERT: + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col); + if (mi_col + hbs < cm->mi_cols) + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col + hbs); + break; + case PARTITION_SPLIT: + write_modes_sb_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col, subsize); + write_modes_sb_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col + hbs, subsize); + write_modes_sb_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row + hbs, mi_col, subsize); + write_modes_sb_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row + hbs, mi_col + hbs, subsize); + break; +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_HORZ_A: + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col); + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col + hbs); + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row + hbs, mi_col); + break; + case PARTITION_HORZ_B: + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col); + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row + hbs, mi_col); + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row + hbs, mi_col + hbs); + break; + case PARTITION_VERT_A: + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col); + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row + hbs, mi_col); + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col + hbs); + break; + case PARTITION_VERT_B: + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col); + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row, mi_col + hbs); + write_modes_b_wrapper(cpi, tile, w, tok, tok_end, supertx_enabled, + mi_row + hbs, mi_col + hbs); + break; +#endif // CONFIG_EXT_PARTITION_TYPES + default: assert(0); + } + } +#if CONFIG_SUPERTX + if (partition != PARTITION_NONE && supertx_enabled && pack_token) { + int skip; + const int bsw = mi_size_wide[bsize]; + const int bsh = mi_size_high[bsize]; + + xd->mi = cm->mi_grid_visible + mi_offset; + supertx_size = mbmi->tx_size; + set_mi_row_col(xd, tile, mi_row, bsh, mi_col, bsw, +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + + assert(IMPLIES(!cm->seg.enabled, mbmi->segment_id_supertx == 0)); + assert(mbmi->segment_id_supertx < MAX_SEGMENTS); + + skip = write_skip(cm, xd, mbmi->segment_id_supertx, xd->mi[0], w); +#if CONFIG_EXT_TX + if (get_ext_tx_types(supertx_size, bsize, 1, cm->reduced_tx_set_used) > 1 && + !skip) { + const int eset = + get_ext_tx_set(supertx_size, bsize, 1, cm->reduced_tx_set_used); + if (eset > 0) { +#if CONFIG_EC_MULTISYMBOL +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#else + FRAME_CONTEXT *ec_ctx = cm->fc; +#endif + aom_write_symbol(w, av1_ext_tx_inter_ind[eset][mbmi->tx_type], + ec_ctx->inter_ext_tx_cdf[eset][supertx_size], + ext_tx_cnt_inter[eset]); +#else + av1_write_token(w, av1_ext_tx_inter_tree[eset], + cm->fc->inter_ext_tx_prob[eset][supertx_size], + &ext_tx_inter_encodings[eset][mbmi->tx_type]); +#endif + } + } +#else + if (supertx_size < TX_32X32 && !skip) { + av1_write_token(w, av1_ext_tx_tree, + cm->fc->inter_ext_tx_prob[supertx_size], + &ext_tx_encodings[mbmi->tx_type]); + } +#endif // CONFIG_EXT_TX + + if (!skip) { + assert(*tok < tok_end); + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const int mbmi_txb_size = txsize_to_bsize[mbmi->tx_size]; + const BLOCK_SIZE plane_bsize = get_plane_block_size(mbmi_txb_size, pd); + + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + const int max_blocks_high = max_block_high(xd, plane_bsize, plane); + + int row, col; + TX_SIZE tx = get_tx_size(plane, xd); + BLOCK_SIZE txb_size = txsize_to_bsize[tx]; + + const int stepr = tx_size_high_unit[txb_size]; + const int stepc = tx_size_wide_unit[txb_size]; + + TOKEN_STATS token_stats; + token_stats.cost = 0; + for (row = 0; row < max_blocks_high; row += stepr) + for (col = 0; col < max_blocks_wide; col += stepc) + pack_mb_tokens(w, tok, tok_end, cm->bit_depth, tx, &token_stats); + assert(*tok < tok_end && (*tok)->token == EOSB_TOKEN); + (*tok)++; + } + } +#if CONFIG_VAR_TX + xd->above_txfm_context = cm->above_txfm_context + mi_col; + xd->left_txfm_context = + xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK); + set_txfm_ctxs(xd->mi[0]->mbmi.tx_size, bsw, bsh, skip, xd); +#endif + } +#endif // CONFIG_SUPERTX + +// update partition context +#if CONFIG_EXT_PARTITION_TYPES + update_ext_partition_context(xd, mi_row, mi_col, subsize, bsize, partition); +#else + if (bsize >= BLOCK_8X8 && + (bsize == BLOCK_8X8 || partition != PARTITION_SPLIT)) + update_partition_context(xd, mi_row, mi_col, subsize, bsize); +#endif // CONFIG_EXT_PARTITION_TYPES + +#if CONFIG_CDEF +#if CONFIG_EXT_PARTITION + if (cm->sb_size == BLOCK_128X128 && bsize == BLOCK_128X128 && + !sb_all_skip(cm, mi_row, mi_col)) { + aom_write_literal(w, cm->mi_grid_visible[mi_row * cm->mi_stride + mi_col] + ->mbmi.cdef_strength, + cm->cdef_bits); + } else if (cm->sb_size == BLOCK_64X64 && bsize == BLOCK_64X64 && +#else + if (bsize == BLOCK_64X64 && +#endif // CONFIG_EXT_PARTITION + !sb_all_skip(cm, mi_row, mi_col)) { + if (cm->cdef_bits != 0) + aom_write_literal(w, cm->mi_grid_visible[mi_row * cm->mi_stride + mi_col] + ->mbmi.cdef_strength, + cm->cdef_bits); + } +#endif +} + +static void write_modes(AV1_COMP *const cpi, const TileInfo *const tile, + aom_writer *const w, const TOKENEXTRA **tok, + const TOKENEXTRA *const tok_end) { + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; + const int mi_row_start = tile->mi_row_start; + const int mi_row_end = tile->mi_row_end; + const int mi_col_start = tile->mi_col_start; + const int mi_col_end = tile->mi_col_end; + int mi_row, mi_col; + +#if CONFIG_DEPENDENT_HORZTILES +#if CONFIG_TILE_GROUPS + if (!cm->dependent_horz_tiles || mi_row_start == 0 || + tile->tg_horz_boundary) { +#else + if (!cm->dependent_horz_tiles || mi_row_start == 0) { +#endif + av1_zero_above_context(cm, mi_col_start, mi_col_end); + } +#else + av1_zero_above_context(cm, mi_col_start, mi_col_end); +#endif +#if CONFIG_PVQ + assert(cpi->td.mb.pvq_q->curr_pos == 0); +#endif +#if CONFIG_DELTA_Q + if (cpi->common.delta_q_present_flag) { + xd->prev_qindex = cpi->common.base_qindex; +#if CONFIG_EXT_DELTA_Q + if (cpi->common.delta_lf_present_flag) { + xd->prev_delta_lf_from_base = 0; + } +#endif // CONFIG_EXT_DELTA_Q + } +#endif + + for (mi_row = mi_row_start; mi_row < mi_row_end; mi_row += cm->mib_size) { + av1_zero_left_context(xd); + + for (mi_col = mi_col_start; mi_col < mi_col_end; mi_col += cm->mib_size) { + write_modes_sb_wrapper(cpi, tile, w, tok, tok_end, 0, mi_row, mi_col, + cm->sb_size); +#if CONFIG_MOTION_VAR && CONFIG_NCOBMC + write_tokens_sb(cpi, tile, w, tok, tok_end, mi_row, mi_col, cm->sb_size); +#endif + } + } +#if CONFIG_PVQ + // Check that the number of PVQ blocks encoded and written to the bitstream + // are the same + assert(cpi->td.mb.pvq_q->curr_pos == cpi->td.mb.pvq_q->last_pos); + // Reset curr_pos in case we repack the bitstream + cpi->td.mb.pvq_q->curr_pos = 0; +#endif +} + +#if !CONFIG_LV_MAP +#if !CONFIG_PVQ && !(CONFIG_EC_ADAPT && CONFIG_NEW_TOKENSET) +static void build_tree_distribution(AV1_COMP *cpi, TX_SIZE tx_size, + av1_coeff_stats *coef_branch_ct, + av1_coeff_probs_model *coef_probs) { + av1_coeff_count *coef_counts = cpi->td.rd_counts.coef_counts[tx_size]; + unsigned int(*eob_branch_ct)[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS] = + cpi->common.counts.eob_branch[tx_size]; + int i, j, k, l, m; +#if CONFIG_RECT_TX + assert(!is_rect_tx(tx_size)); +#endif // CONFIG_RECT_TX + + for (i = 0; i < PLANE_TYPES; ++i) { + for (j = 0; j < REF_TYPES; ++j) { + for (k = 0; k < COEF_BANDS; ++k) { + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { + av1_tree_probs_from_distribution(av1_coef_tree, + coef_branch_ct[i][j][k][l], + coef_counts[i][j][k][l]); + coef_branch_ct[i][j][k][l][0][1] = + eob_branch_ct[i][j][k][l] - coef_branch_ct[i][j][k][l][0][0]; + for (m = 0; m < UNCONSTRAINED_NODES; ++m) + coef_probs[i][j][k][l][m] = + get_binary_prob(coef_branch_ct[i][j][k][l][m][0], + coef_branch_ct[i][j][k][l][m][1]); + } + } + } + } +} + +#if !(CONFIG_EC_ADAPT && CONFIG_NEW_TOKENSET) +static void update_coef_probs_common(aom_writer *const bc, AV1_COMP *cpi, + TX_SIZE tx_size, + av1_coeff_stats *frame_branch_ct, + av1_coeff_probs_model *new_coef_probs) { + av1_coeff_probs_model *old_coef_probs = cpi->common.fc->coef_probs[tx_size]; + const aom_prob upd = DIFF_UPDATE_PROB; +#if CONFIG_EC_ADAPT + const int entropy_nodes_update = UNCONSTRAINED_NODES - 1; +#else + const int entropy_nodes_update = UNCONSTRAINED_NODES; +#endif + int i, j, k, l, t; + int stepsize = cpi->sf.coeff_prob_appx_step; +#if CONFIG_TILE_GROUPS + const int probwt = cpi->common.num_tg; +#else + const int probwt = 1; +#endif +#if CONFIG_RECT_TX + assert(!is_rect_tx(tx_size)); +#endif // CONFIG_RECT_TX + + switch (cpi->sf.use_fast_coef_updates) { + case TWO_LOOP: { + /* dry run to see if there is any update at all needed */ + int savings = 0; + int update[2] = { 0, 0 }; + for (i = 0; i < PLANE_TYPES; ++i) { + for (j = 0; j < REF_TYPES; ++j) { + for (k = 0; k < COEF_BANDS; ++k) { + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { + for (t = 0; t < entropy_nodes_update; ++t) { + aom_prob newp = new_coef_probs[i][j][k][l][t]; + const aom_prob oldp = old_coef_probs[i][j][k][l][t]; + int s; + int u = 0; + if (t == PIVOT_NODE) + s = av1_prob_diff_update_savings_search_model( + frame_branch_ct[i][j][k][l][0], oldp, &newp, upd, + stepsize, probwt); + else + s = av1_prob_diff_update_savings_search( + frame_branch_ct[i][j][k][l][t], oldp, &newp, upd, probwt); + + if (s > 0 && newp != oldp) u = 1; + if (u) + savings += s - (int)(av1_cost_zero(upd)); + else + savings -= (int)(av1_cost_zero(upd)); + update[u]++; + } + } + } + } + } + + /* Is coef updated at all */ + if (update[1] == 0 || savings < 0) { + aom_write_bit(bc, 0); + return; + } + aom_write_bit(bc, 1); + for (i = 0; i < PLANE_TYPES; ++i) { + for (j = 0; j < REF_TYPES; ++j) { + for (k = 0; k < COEF_BANDS; ++k) { + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { + // calc probs and branch cts for this frame only + for (t = 0; t < entropy_nodes_update; ++t) { + aom_prob newp = new_coef_probs[i][j][k][l][t]; + aom_prob *oldp = old_coef_probs[i][j][k][l] + t; + int s; + int u = 0; + if (t == PIVOT_NODE) + s = av1_prob_diff_update_savings_search_model( + frame_branch_ct[i][j][k][l][0], *oldp, &newp, upd, + stepsize, probwt); + else + s = av1_prob_diff_update_savings_search( + frame_branch_ct[i][j][k][l][t], *oldp, &newp, upd, + probwt); + if (s > 0 && newp != *oldp) u = 1; + aom_write(bc, u, upd); + if (u) { + /* send/use new probability */ + av1_write_prob_diff_update(bc, newp, *oldp); + *oldp = newp; + } + } + } + } + } + } + return; + } + + case ONE_LOOP_REDUCED: { + int updates = 0; + int noupdates_before_first = 0; + for (i = 0; i < PLANE_TYPES; ++i) { + for (j = 0; j < REF_TYPES; ++j) { + for (k = 0; k < COEF_BANDS; ++k) { + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { + // calc probs and branch cts for this frame only + for (t = 0; t < entropy_nodes_update; ++t) { + aom_prob newp = new_coef_probs[i][j][k][l][t]; + aom_prob *oldp = old_coef_probs[i][j][k][l] + t; + int s; + int u = 0; + if (t == PIVOT_NODE) { + s = av1_prob_diff_update_savings_search_model( + frame_branch_ct[i][j][k][l][0], *oldp, &newp, upd, + stepsize, probwt); + } else { + s = av1_prob_diff_update_savings_search( + frame_branch_ct[i][j][k][l][t], *oldp, &newp, upd, + probwt); + } + + if (s > 0 && newp != *oldp) u = 1; + updates += u; + if (u == 0 && updates == 0) { + noupdates_before_first++; + continue; + } + if (u == 1 && updates == 1) { + int v; + // first update + aom_write_bit(bc, 1); + for (v = 0; v < noupdates_before_first; ++v) + aom_write(bc, 0, upd); + } + aom_write(bc, u, upd); + if (u) { + /* send/use new probability */ + av1_write_prob_diff_update(bc, newp, *oldp); + *oldp = newp; + } + } + } + } + } + } + if (updates == 0) { + aom_write_bit(bc, 0); // no updates + } + return; + } + default: assert(0); + } +} +#endif +#if CONFIG_SUBFRAME_PROB_UPDATE +// Calculate the token counts between subsequent subframe updates. +static void get_coef_counts_diff( + AV1_COMP *cpi, int index, + av1_coeff_count coef_counts[TX_SIZES][PLANE_TYPES], + unsigned int eob_counts[TX_SIZES][PLANE_TYPES][REF_TYPES][COEF_BANDS] + [COEFF_CONTEXTS]) { + int i, j, k, l, m, tx_size, val; + const int max_idx = cpi->common.coef_probs_update_idx; + const TX_MODE tx_mode = cpi->common.tx_mode; + const int max_tx_size = tx_mode_to_biggest_tx_size[tx_mode]; + const SUBFRAME_STATS *subframe_stats = &cpi->subframe_stats; + + assert(max_idx < COEF_PROBS_BUFS); + + for (tx_size = 0; tx_size <= max_tx_size; ++tx_size) + for (i = 0; i < PLANE_TYPES; ++i) + for (j = 0; j < REF_TYPES; ++j) + for (k = 0; k < COEF_BANDS; ++k) + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { + if (index == max_idx) { + val = + cpi->common.counts.eob_branch[tx_size][i][j][k][l] - + subframe_stats->eob_counts_buf[max_idx][tx_size][i][j][k][l]; + } else { + val = subframe_stats + ->eob_counts_buf[index + 1][tx_size][i][j][k][l] - + subframe_stats->eob_counts_buf[index][tx_size][i][j][k][l]; + } + assert(val >= 0); + eob_counts[tx_size][i][j][k][l] = val; + + for (m = 0; m < ENTROPY_TOKENS; ++m) { + if (index == max_idx) { + val = cpi->td.rd_counts.coef_counts[tx_size][i][j][k][l][m] - + subframe_stats + ->coef_counts_buf[max_idx][tx_size][i][j][k][l][m]; + } else { + val = subframe_stats + ->coef_counts_buf[index + 1][tx_size][i][j][k][l][m] - + subframe_stats + ->coef_counts_buf[index][tx_size][i][j][k][l][m]; + } + assert(val >= 0); + coef_counts[tx_size][i][j][k][l][m] = val; + } + } +} + +static void update_coef_probs_subframe( + aom_writer *const bc, AV1_COMP *cpi, TX_SIZE tx_size, + av1_coeff_stats branch_ct[COEF_PROBS_BUFS][TX_SIZES][PLANE_TYPES], + av1_coeff_probs_model *new_coef_probs) { + av1_coeff_probs_model *old_coef_probs = cpi->common.fc->coef_probs[tx_size]; + const aom_prob upd = DIFF_UPDATE_PROB; + const int entropy_nodes_update = UNCONSTRAINED_NODES; + int i, j, k, l, t; + int stepsize = cpi->sf.coeff_prob_appx_step; + const int max_idx = cpi->common.coef_probs_update_idx; + int idx; + unsigned int this_branch_ct[ENTROPY_NODES][COEF_PROBS_BUFS][2]; + + switch (cpi->sf.use_fast_coef_updates) { + case TWO_LOOP: { + /* dry run to see if there is any update at all needed */ + int savings = 0; + int update[2] = { 0, 0 }; + for (i = 0; i < PLANE_TYPES; ++i) { + for (j = 0; j < REF_TYPES; ++j) { + for (k = 0; k < COEF_BANDS; ++k) { + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { + for (t = 0; t < ENTROPY_NODES; ++t) { + for (idx = 0; idx <= max_idx; ++idx) { + memcpy(this_branch_ct[t][idx], + branch_ct[idx][tx_size][i][j][k][l][t], + 2 * sizeof(this_branch_ct[t][idx][0])); + } + } + for (t = 0; t < entropy_nodes_update; ++t) { + aom_prob newp = new_coef_probs[i][j][k][l][t]; + const aom_prob oldp = old_coef_probs[i][j][k][l][t]; + int s, u = 0; + + if (t == PIVOT_NODE) + s = av1_prob_update_search_model_subframe( + this_branch_ct, old_coef_probs[i][j][k][l], &newp, upd, + stepsize, max_idx); + else + s = av1_prob_update_search_subframe(this_branch_ct[t], oldp, + &newp, upd, max_idx); + if (s > 0 && newp != oldp) u = 1; + if (u) + savings += s - (int)(av1_cost_zero(upd)); + else + savings -= (int)(av1_cost_zero(upd)); + update[u]++; + } + } + } + } + } + + /* Is coef updated at all */ + if (update[1] == 0 || savings < 0) { + aom_write_bit(bc, 0); + return; + } + aom_write_bit(bc, 1); + for (i = 0; i < PLANE_TYPES; ++i) { + for (j = 0; j < REF_TYPES; ++j) { + for (k = 0; k < COEF_BANDS; ++k) { + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { + for (t = 0; t < ENTROPY_NODES; ++t) { + for (idx = 0; idx <= max_idx; ++idx) { + memcpy(this_branch_ct[t][idx], + branch_ct[idx][tx_size][i][j][k][l][t], + 2 * sizeof(this_branch_ct[t][idx][0])); + } + } + for (t = 0; t < entropy_nodes_update; ++t) { + aom_prob newp = new_coef_probs[i][j][k][l][t]; + aom_prob *oldp = old_coef_probs[i][j][k][l] + t; + int s; + int u = 0; + + if (t == PIVOT_NODE) + s = av1_prob_update_search_model_subframe( + this_branch_ct, old_coef_probs[i][j][k][l], &newp, upd, + stepsize, max_idx); + else + s = av1_prob_update_search_subframe(this_branch_ct[t], *oldp, + &newp, upd, max_idx); + if (s > 0 && newp != *oldp) u = 1; + aom_write(bc, u, upd); + if (u) { + /* send/use new probability */ + av1_write_prob_diff_update(bc, newp, *oldp); + *oldp = newp; + } + } + } + } + } + } + return; + } + + case ONE_LOOP_REDUCED: { + int updates = 0; + int noupdates_before_first = 0; + for (i = 0; i < PLANE_TYPES; ++i) { + for (j = 0; j < REF_TYPES; ++j) { + for (k = 0; k < COEF_BANDS; ++k) { + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { + for (t = 0; t < ENTROPY_NODES; ++t) { + for (idx = 0; idx <= max_idx; ++idx) { + memcpy(this_branch_ct[t][idx], + branch_ct[idx][tx_size][i][j][k][l][t], + 2 * sizeof(this_branch_ct[t][idx][0])); + } + } + for (t = 0; t < entropy_nodes_update; ++t) { + aom_prob newp = new_coef_probs[i][j][k][l][t]; + aom_prob *oldp = old_coef_probs[i][j][k][l] + t; + int s; + int u = 0; + + if (t == PIVOT_NODE) + s = av1_prob_update_search_model_subframe( + this_branch_ct, old_coef_probs[i][j][k][l], &newp, upd, + stepsize, max_idx); + else + s = av1_prob_update_search_subframe(this_branch_ct[t], *oldp, + &newp, upd, max_idx); + if (s > 0 && newp != *oldp) u = 1; + updates += u; + if (u == 0 && updates == 0) { + noupdates_before_first++; + continue; + } + if (u == 1 && updates == 1) { + int v; + // first update + aom_write_bit(bc, 1); + for (v = 0; v < noupdates_before_first; ++v) + aom_write(bc, 0, upd); + } + aom_write(bc, u, upd); + if (u) { + /* send/use new probability */ + av1_write_prob_diff_update(bc, newp, *oldp); + *oldp = newp; + } + } + } + } + } + } + if (updates == 0) { + aom_write_bit(bc, 0); // no updates + } + return; + } + default: assert(0); + } +} +#endif // CONFIG_SUBFRAME_PROB_UPDATE + +#if !(CONFIG_EC_ADAPT && CONFIG_NEW_TOKENSET) +static void update_coef_probs(AV1_COMP *cpi, aom_writer *w) { + const TX_MODE tx_mode = cpi->common.tx_mode; + const TX_SIZE max_tx_size = tx_mode_to_biggest_tx_size[tx_mode]; + TX_SIZE tx_size; +#if CONFIG_SUBFRAME_PROB_UPDATE + AV1_COMMON *cm = &cpi->common; + SUBFRAME_STATS *subframe_stats = &cpi->subframe_stats; + int i; + av1_coeff_probs_model dummy_frame_coef_probs[PLANE_TYPES]; + + if (cm->do_subframe_update && + cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { + av1_copy(cpi->common.fc->coef_probs, + subframe_stats->enc_starting_coef_probs); + for (i = 0; i <= cpi->common.coef_probs_update_idx; ++i) { + get_coef_counts_diff(cpi, i, cpi->wholeframe_stats.coef_counts_buf[i], + cpi->wholeframe_stats.eob_counts_buf[i]); + } + } +#endif // CONFIG_SUBFRAME_PROB_UPDATE + + for (tx_size = 0; tx_size <= max_tx_size; ++tx_size) { + av1_coeff_stats frame_branch_ct[PLANE_TYPES]; + av1_coeff_probs_model frame_coef_probs[PLANE_TYPES]; + if (cpi->td.counts->tx_size_totals[tx_size] <= 20 || CONFIG_RD_DEBUG || + (tx_size >= TX_16X16 && cpi->sf.tx_size_search_method == USE_TX_8X8)) { + aom_write_bit(w, 0); + } else { +#if CONFIG_SUBFRAME_PROB_UPDATE + if (cm->do_subframe_update && + cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { + unsigned int this_eob_counts_copy[PLANE_TYPES][REF_TYPES][COEF_BANDS] + [COEFF_CONTEXTS]; + av1_coeff_count coef_counts_copy[PLANE_TYPES]; + av1_copy(this_eob_counts_copy, cpi->common.counts.eob_branch[tx_size]); + av1_copy(coef_counts_copy, cpi->td.rd_counts.coef_counts[tx_size]); + build_tree_distribution(cpi, tx_size, frame_branch_ct, + frame_coef_probs); + for (i = 0; i <= cpi->common.coef_probs_update_idx; ++i) { + av1_copy(cpi->common.counts.eob_branch[tx_size], + cpi->wholeframe_stats.eob_counts_buf[i][tx_size]); + av1_copy(cpi->td.rd_counts.coef_counts[tx_size], + cpi->wholeframe_stats.coef_counts_buf[i][tx_size]); + build_tree_distribution(cpi, tx_size, cpi->branch_ct_buf[i][tx_size], + dummy_frame_coef_probs); + } + av1_copy(cpi->common.counts.eob_branch[tx_size], this_eob_counts_copy); + av1_copy(cpi->td.rd_counts.coef_counts[tx_size], coef_counts_copy); + + update_coef_probs_subframe(w, cpi, tx_size, cpi->branch_ct_buf, + frame_coef_probs); + } else { +#endif // CONFIG_SUBFRAME_PROB_UPDATE + build_tree_distribution(cpi, tx_size, frame_branch_ct, + frame_coef_probs); + update_coef_probs_common(w, cpi, tx_size, frame_branch_ct, + frame_coef_probs); +#if CONFIG_SUBFRAME_PROB_UPDATE + } +#endif // CONFIG_SUBFRAME_PROB_UPDATE + } + } + +#if CONFIG_SUBFRAME_PROB_UPDATE + av1_copy(cm->starting_coef_probs, cm->fc->coef_probs); + av1_copy(subframe_stats->coef_probs_buf[0], cm->fc->coef_probs); + if (cm->do_subframe_update && + cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { + unsigned int eob_counts_copy[TX_SIZES][PLANE_TYPES][REF_TYPES][COEF_BANDS] + [COEFF_CONTEXTS]; + av1_copy(eob_counts_copy, cm->counts.eob_branch); + for (i = 1; i <= cpi->common.coef_probs_update_idx; ++i) { + for (tx_size = 0; tx_size <= max_tx_size; ++tx_size) + av1_full_to_model_counts(cm->counts.coef[tx_size], + subframe_stats->coef_counts_buf[i][tx_size]); + av1_copy(cm->counts.eob_branch, subframe_stats->eob_counts_buf[i]); + av1_partial_adapt_probs(cm, 0, 0); + av1_copy(subframe_stats->coef_probs_buf[i], cm->fc->coef_probs); + } + av1_copy(cm->fc->coef_probs, subframe_stats->coef_probs_buf[0]); + av1_copy(cm->counts.eob_branch, eob_counts_copy); + } +#endif // CONFIG_SUBFRAME_PROB_UPDATE +} +#endif // !(CONFIG_EC_ADAPT && CONFIG_NEW_TOKENSET) +#endif // !CONFIG_EC_ADAPT +#endif // !CONFIG_LV_MAP + +#if CONFIG_LOOP_RESTORATION +static void encode_restoration_mode(AV1_COMMON *cm, + struct aom_write_bit_buffer *wb) { + int p; + RestorationInfo *rsi = &cm->rst_info[0]; + switch (rsi->frame_restoration_type) { + case RESTORE_NONE: + aom_wb_write_bit(wb, 0); + aom_wb_write_bit(wb, 0); + break; + case RESTORE_WIENER: + aom_wb_write_bit(wb, 1); + aom_wb_write_bit(wb, 0); + break; + case RESTORE_SGRPROJ: + aom_wb_write_bit(wb, 1); + aom_wb_write_bit(wb, 1); + break; + case RESTORE_SWITCHABLE: + aom_wb_write_bit(wb, 0); + aom_wb_write_bit(wb, 1); + break; + default: assert(0); + } + for (p = 1; p < MAX_MB_PLANE; ++p) { + rsi = &cm->rst_info[p]; + switch (rsi->frame_restoration_type) { + case RESTORE_NONE: aom_wb_write_bit(wb, 0); break; + case RESTORE_WIENER: aom_wb_write_bit(wb, 1); break; + default: assert(0); + } + } + if (cm->rst_info[0].frame_restoration_type != RESTORE_NONE || + cm->rst_info[1].frame_restoration_type != RESTORE_NONE || + cm->rst_info[2].frame_restoration_type != RESTORE_NONE) { + rsi = &cm->rst_info[0]; + aom_wb_write_bit(wb, rsi->restoration_tilesize != RESTORATION_TILESIZE_MAX); + if (rsi->restoration_tilesize != RESTORATION_TILESIZE_MAX) { + aom_wb_write_bit( + wb, rsi->restoration_tilesize != (RESTORATION_TILESIZE_MAX >> 1)); + } + } +} + +static void write_wiener_filter(WienerInfo *wiener_info, + WienerInfo *ref_wiener_info, aom_writer *wb) { + aom_write_primitive_refsubexpfin( + wb, WIENER_FILT_TAP0_MAXV - WIENER_FILT_TAP0_MINV + 1, + WIENER_FILT_TAP0_SUBEXP_K, + ref_wiener_info->vfilter[0] - WIENER_FILT_TAP0_MINV, + wiener_info->vfilter[0] - WIENER_FILT_TAP0_MINV); + aom_write_primitive_refsubexpfin( + wb, WIENER_FILT_TAP1_MAXV - WIENER_FILT_TAP1_MINV + 1, + WIENER_FILT_TAP1_SUBEXP_K, + ref_wiener_info->vfilter[1] - WIENER_FILT_TAP1_MINV, + wiener_info->vfilter[1] - WIENER_FILT_TAP1_MINV); + aom_write_primitive_refsubexpfin( + wb, WIENER_FILT_TAP2_MAXV - WIENER_FILT_TAP2_MINV + 1, + WIENER_FILT_TAP2_SUBEXP_K, + ref_wiener_info->vfilter[2] - WIENER_FILT_TAP2_MINV, + wiener_info->vfilter[2] - WIENER_FILT_TAP2_MINV); + aom_write_primitive_refsubexpfin( + wb, WIENER_FILT_TAP0_MAXV - WIENER_FILT_TAP0_MINV + 1, + WIENER_FILT_TAP0_SUBEXP_K, + ref_wiener_info->hfilter[0] - WIENER_FILT_TAP0_MINV, + wiener_info->hfilter[0] - WIENER_FILT_TAP0_MINV); + aom_write_primitive_refsubexpfin( + wb, WIENER_FILT_TAP1_MAXV - WIENER_FILT_TAP1_MINV + 1, + WIENER_FILT_TAP1_SUBEXP_K, + ref_wiener_info->hfilter[1] - WIENER_FILT_TAP1_MINV, + wiener_info->hfilter[1] - WIENER_FILT_TAP1_MINV); + aom_write_primitive_refsubexpfin( + wb, WIENER_FILT_TAP2_MAXV - WIENER_FILT_TAP2_MINV + 1, + WIENER_FILT_TAP2_SUBEXP_K, + ref_wiener_info->hfilter[2] - WIENER_FILT_TAP2_MINV, + wiener_info->hfilter[2] - WIENER_FILT_TAP2_MINV); + memcpy(ref_wiener_info, wiener_info, sizeof(*wiener_info)); +} + +static void write_sgrproj_filter(SgrprojInfo *sgrproj_info, + SgrprojInfo *ref_sgrproj_info, + aom_writer *wb) { + aom_write_literal(wb, sgrproj_info->ep, SGRPROJ_PARAMS_BITS); + aom_write_primitive_refsubexpfin(wb, SGRPROJ_PRJ_MAX0 - SGRPROJ_PRJ_MIN0 + 1, + SGRPROJ_PRJ_SUBEXP_K, + ref_sgrproj_info->xqd[0] - SGRPROJ_PRJ_MIN0, + sgrproj_info->xqd[0] - SGRPROJ_PRJ_MIN0); + aom_write_primitive_refsubexpfin(wb, SGRPROJ_PRJ_MAX1 - SGRPROJ_PRJ_MIN1 + 1, + SGRPROJ_PRJ_SUBEXP_K, + ref_sgrproj_info->xqd[1] - SGRPROJ_PRJ_MIN1, + sgrproj_info->xqd[1] - SGRPROJ_PRJ_MIN1); + memcpy(ref_sgrproj_info, sgrproj_info, sizeof(*sgrproj_info)); +} + +static void encode_restoration(AV1_COMMON *cm, aom_writer *wb) { + int i, p; + const int ntiles = av1_get_rest_ntiles(cm->width, cm->height, + cm->rst_info[0].restoration_tilesize, + NULL, NULL, NULL, NULL); + WienerInfo ref_wiener_info; + SgrprojInfo ref_sgrproj_info; + set_default_wiener(&ref_wiener_info); + set_default_sgrproj(&ref_sgrproj_info); + const int ntiles_uv = av1_get_rest_ntiles( + ROUND_POWER_OF_TWO(cm->width, cm->subsampling_x), + ROUND_POWER_OF_TWO(cm->height, cm->subsampling_y), + cm->rst_info[1].restoration_tilesize, NULL, NULL, NULL, NULL); + RestorationInfo *rsi = &cm->rst_info[0]; + if (rsi->frame_restoration_type != RESTORE_NONE) { + if (rsi->frame_restoration_type == RESTORE_SWITCHABLE) { + // RESTORE_SWITCHABLE + for (i = 0; i < ntiles; ++i) { + av1_write_token( + wb, av1_switchable_restore_tree, cm->fc->switchable_restore_prob, + &switchable_restore_encodings[rsi->restoration_type[i]]); + if (rsi->restoration_type[i] == RESTORE_WIENER) { + write_wiener_filter(&rsi->wiener_info[i], &ref_wiener_info, wb); + } else if (rsi->restoration_type[i] == RESTORE_SGRPROJ) { + write_sgrproj_filter(&rsi->sgrproj_info[i], &ref_sgrproj_info, wb); + } + } + } else if (rsi->frame_restoration_type == RESTORE_WIENER) { + for (i = 0; i < ntiles; ++i) { + aom_write(wb, rsi->restoration_type[i] != RESTORE_NONE, + RESTORE_NONE_WIENER_PROB); + if (rsi->restoration_type[i] != RESTORE_NONE) { + write_wiener_filter(&rsi->wiener_info[i], &ref_wiener_info, wb); + } + } + } else if (rsi->frame_restoration_type == RESTORE_SGRPROJ) { + for (i = 0; i < ntiles; ++i) { + aom_write(wb, rsi->restoration_type[i] != RESTORE_NONE, + RESTORE_NONE_SGRPROJ_PROB); + if (rsi->restoration_type[i] != RESTORE_NONE) { + write_sgrproj_filter(&rsi->sgrproj_info[i], &ref_sgrproj_info, wb); + } + } + } + } + for (p = 1; p < MAX_MB_PLANE; ++p) { + set_default_wiener(&ref_wiener_info); + rsi = &cm->rst_info[p]; + if (rsi->frame_restoration_type == RESTORE_WIENER) { + for (i = 0; i < ntiles_uv; ++i) { + if (ntiles_uv > 1) + aom_write(wb, rsi->restoration_type[i] != RESTORE_NONE, + RESTORE_NONE_WIENER_PROB); + if (rsi->restoration_type[i] != RESTORE_NONE) { + write_wiener_filter(&rsi->wiener_info[i], &ref_wiener_info, wb); + } + } + } else if (rsi->frame_restoration_type != RESTORE_NONE) { + assert(0); + } + } +} +#endif // CONFIG_LOOP_RESTORATION + +static void encode_loopfilter(AV1_COMMON *cm, struct aom_write_bit_buffer *wb) { + int i; + struct loopfilter *lf = &cm->lf; + + // Encode the loop filter level and type + aom_wb_write_literal(wb, lf->filter_level, 6); + aom_wb_write_literal(wb, lf->sharpness_level, 3); + + // Write out loop filter deltas applied at the MB level based on mode or + // ref frame (if they are enabled). + aom_wb_write_bit(wb, lf->mode_ref_delta_enabled); + + if (lf->mode_ref_delta_enabled) { + aom_wb_write_bit(wb, lf->mode_ref_delta_update); + if (lf->mode_ref_delta_update) { + for (i = 0; i < TOTAL_REFS_PER_FRAME; i++) { + const int delta = lf->ref_deltas[i]; + const int changed = delta != lf->last_ref_deltas[i]; + aom_wb_write_bit(wb, changed); + if (changed) { + lf->last_ref_deltas[i] = delta; + aom_wb_write_inv_signed_literal(wb, delta, 6); + } + } + + for (i = 0; i < MAX_MODE_LF_DELTAS; i++) { + const int delta = lf->mode_deltas[i]; + const int changed = delta != lf->last_mode_deltas[i]; + aom_wb_write_bit(wb, changed); + if (changed) { + lf->last_mode_deltas[i] = delta; + aom_wb_write_inv_signed_literal(wb, delta, 6); + } + } + } + } +} + +#if CONFIG_CDEF +static void encode_cdef(const AV1_COMMON *cm, struct aom_write_bit_buffer *wb) { + int i; + aom_wb_write_literal(wb, cm->cdef_dering_damping - 5, 1); + aom_wb_write_literal(wb, cm->cdef_clpf_damping - 3, 2); + aom_wb_write_literal(wb, cm->cdef_bits, 2); + for (i = 0; i < cm->nb_cdef_strengths; i++) { + aom_wb_write_literal(wb, cm->cdef_strengths[i], CDEF_STRENGTH_BITS); + aom_wb_write_literal(wb, cm->cdef_uv_strengths[i], CDEF_STRENGTH_BITS); + } +} +#endif + +static void write_delta_q(struct aom_write_bit_buffer *wb, int delta_q) { + if (delta_q != 0) { + aom_wb_write_bit(wb, 1); + aom_wb_write_inv_signed_literal(wb, delta_q, 6); + } else { + aom_wb_write_bit(wb, 0); + } +} + +static void encode_quantization(const AV1_COMMON *const cm, + struct aom_write_bit_buffer *wb) { + aom_wb_write_literal(wb, cm->base_qindex, QINDEX_BITS); + write_delta_q(wb, cm->y_dc_delta_q); + write_delta_q(wb, cm->uv_dc_delta_q); + write_delta_q(wb, cm->uv_ac_delta_q); +#if CONFIG_AOM_QM + aom_wb_write_bit(wb, cm->using_qmatrix); + if (cm->using_qmatrix) { + aom_wb_write_literal(wb, cm->min_qmlevel, QM_LEVEL_BITS); + aom_wb_write_literal(wb, cm->max_qmlevel, QM_LEVEL_BITS); + } +#endif +} + +static void encode_segmentation(AV1_COMMON *cm, MACROBLOCKD *xd, + struct aom_write_bit_buffer *wb) { + int i, j; + const struct segmentation *seg = &cm->seg; + + aom_wb_write_bit(wb, seg->enabled); + if (!seg->enabled) return; + + // Segmentation map + if (!frame_is_intra_only(cm) && !cm->error_resilient_mode) { + aom_wb_write_bit(wb, seg->update_map); + } else { + assert(seg->update_map == 1); + } + if (seg->update_map) { + // Select the coding strategy (temporal or spatial) + av1_choose_segmap_coding_method(cm, xd); + + // Write out the chosen coding method. + if (!frame_is_intra_only(cm) && !cm->error_resilient_mode) { + aom_wb_write_bit(wb, seg->temporal_update); + } else { + assert(seg->temporal_update == 0); + } + } + + // Segmentation data + aom_wb_write_bit(wb, seg->update_data); + if (seg->update_data) { + aom_wb_write_bit(wb, seg->abs_delta); + + for (i = 0; i < MAX_SEGMENTS; i++) { + for (j = 0; j < SEG_LVL_MAX; j++) { + const int active = segfeature_active(seg, i, j); + aom_wb_write_bit(wb, active); + if (active) { + const int data = get_segdata(seg, i, j); + const int data_max = av1_seg_feature_data_max(j); + + if (av1_is_segfeature_signed(j)) { + encode_unsigned_max(wb, abs(data), data_max); + aom_wb_write_bit(wb, data < 0); + } else { + encode_unsigned_max(wb, data, data_max); + } + } + } + } + } +} + +#if !CONFIG_EC_ADAPT +static void update_seg_probs(AV1_COMP *cpi, aom_writer *w) { + AV1_COMMON *cm = &cpi->common; +#if CONFIG_TILE_GROUPS + const int probwt = cm->num_tg; +#else + const int probwt = 1; +#endif + + if (!cm->seg.enabled || !cm->seg.update_map) return; + + if (cm->seg.temporal_update) { + int i; + + for (i = 0; i < PREDICTION_PROBS; i++) + av1_cond_prob_diff_update(w, &cm->fc->seg.pred_probs[i], + cm->counts.seg.pred[i], probwt); + + prob_diff_update(av1_segment_tree, cm->fc->seg.tree_probs, + cm->counts.seg.tree_mispred, MAX_SEGMENTS, probwt, w); + } else { + prob_diff_update(av1_segment_tree, cm->fc->seg.tree_probs, + cm->counts.seg.tree_total, MAX_SEGMENTS, probwt, w); + } +} +#endif + +static void write_tx_mode(AV1_COMMON *cm, MACROBLOCKD *xd, TX_MODE *mode, + struct aom_write_bit_buffer *wb) { + int i, all_lossless = 1; + + if (cm->seg.enabled) { + for (i = 0; i < MAX_SEGMENTS; ++i) { + if (!xd->lossless[i]) { + all_lossless = 0; + break; + } + } + } else { + all_lossless = xd->lossless[0]; + } + if (all_lossless) { + *mode = ONLY_4X4; + return; + } +#if CONFIG_TX64X64 + aom_wb_write_bit(wb, *mode == TX_MODE_SELECT); + if (*mode != TX_MODE_SELECT) { + aom_wb_write_literal(wb, AOMMIN(*mode, ALLOW_32X32), 2); + if (*mode >= ALLOW_32X32) aom_wb_write_bit(wb, *mode == ALLOW_64X64); + } +#else + aom_wb_write_bit(wb, *mode == TX_MODE_SELECT); + if (*mode != TX_MODE_SELECT) aom_wb_write_literal(wb, *mode, 2); +#endif // CONFIG_TX64X64 +} + +#if !CONFIG_EC_ADAPT +static void update_txfm_probs(AV1_COMMON *cm, aom_writer *w, + FRAME_COUNTS *counts) { +#if CONFIG_TILE_GROUPS + const int probwt = cm->num_tg; +#else + const int probwt = 1; +#endif + if (cm->tx_mode == TX_MODE_SELECT) { + int i, j; + for (i = 0; i < MAX_TX_DEPTH; ++i) + for (j = 0; j < TX_SIZE_CONTEXTS; ++j) + prob_diff_update(av1_tx_size_tree[i], cm->fc->tx_size_probs[i][j], + counts->tx_size[i][j], i + 2, probwt, w); + } +} +#endif + +static void write_frame_interp_filter(InterpFilter filter, + struct aom_write_bit_buffer *wb) { + aom_wb_write_bit(wb, filter == SWITCHABLE); + if (filter != SWITCHABLE) + aom_wb_write_literal(wb, filter, LOG_SWITCHABLE_FILTERS); +} + +static void fix_interp_filter(AV1_COMMON *cm, FRAME_COUNTS *counts) { + if (cm->interp_filter == SWITCHABLE) { + // Check to see if only one of the filters is actually used + int count[SWITCHABLE_FILTERS]; + int i, j, c = 0; + for (i = 0; i < SWITCHABLE_FILTERS; ++i) { + count[i] = 0; + for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) + count[i] += counts->switchable_interp[j][i]; + c += (count[i] > 0); + } + if (c == 1) { + // Only one filter is used. So set the filter at frame level + for (i = 0; i < SWITCHABLE_FILTERS; ++i) { + if (count[i]) { +#if CONFIG_MOTION_VAR && (CONFIG_WARPED_MOTION || CONFIG_GLOBAL_MOTION) +#if CONFIG_WARPED_MOTION + if (i == EIGHTTAP_REGULAR || WARP_WM_NEIGHBORS_WITH_OBMC) +#else + if (i == EIGHTTAP_REGULAR || WARP_GM_NEIGHBORS_WITH_OBMC) +#endif // CONFIG_WARPED_MOTION +#endif // CONFIG_MOTION_VAR && (CONFIG_WARPED_MOTION || CONFIG_GLOBAL_MOTION) + cm->interp_filter = i; + break; + } + } + } + } +} + +static void write_tile_info(const AV1_COMMON *const cm, + struct aom_write_bit_buffer *wb) { +#if CONFIG_EXT_TILE + const int tile_width = + ALIGN_POWER_OF_TWO(cm->tile_width, cm->mib_size_log2) >> + cm->mib_size_log2; + const int tile_height = + ALIGN_POWER_OF_TWO(cm->tile_height, cm->mib_size_log2) >> + cm->mib_size_log2; + + assert(tile_width > 0); + assert(tile_height > 0); + + aom_wb_write_literal(wb, cm->tile_encoding_mode, 1); + +// Write the tile sizes +#if CONFIG_EXT_PARTITION + if (cm->sb_size == BLOCK_128X128) { + assert(tile_width <= 32); + assert(tile_height <= 32); + aom_wb_write_literal(wb, tile_width - 1, 5); + aom_wb_write_literal(wb, tile_height - 1, 5); + } else +#endif // CONFIG_EXT_PARTITION + { + assert(tile_width <= 64); + assert(tile_height <= 64); + aom_wb_write_literal(wb, tile_width - 1, 6); + aom_wb_write_literal(wb, tile_height - 1, 6); + } +#else + int min_log2_tile_cols, max_log2_tile_cols, ones; + av1_get_tile_n_bits(cm->mi_cols, &min_log2_tile_cols, &max_log2_tile_cols); + + // columns + ones = cm->log2_tile_cols - min_log2_tile_cols; + while (ones--) aom_wb_write_bit(wb, 1); + + if (cm->log2_tile_cols < max_log2_tile_cols) aom_wb_write_bit(wb, 0); + + // rows + aom_wb_write_bit(wb, cm->log2_tile_rows != 0); + if (cm->log2_tile_rows != 0) aom_wb_write_bit(wb, cm->log2_tile_rows != 1); +#endif // CONFIG_EXT_TILE + +#if CONFIG_DEPENDENT_HORZTILES + if (cm->log2_tile_rows != 0) aom_wb_write_bit(wb, cm->dependent_horz_tiles); +#endif + +#if CONFIG_LOOPFILTERING_ACROSS_TILES + aom_wb_write_bit(wb, cm->loop_filter_across_tiles_enabled); +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES +} + +static int get_refresh_mask(AV1_COMP *cpi) { + int refresh_mask = 0; + +#if CONFIG_EXT_REFS + // NOTE(zoeliu): When LAST_FRAME is to get refreshed, the decoder will be + // notified to get LAST3_FRAME refreshed and then the virtual indexes for all + // the 3 LAST reference frames will be updated accordingly, i.e.: + // (1) The original virtual index for LAST3_FRAME will become the new virtual + // index for LAST_FRAME; and + // (2) The original virtual indexes for LAST_FRAME and LAST2_FRAME will be + // shifted and become the new virtual indexes for LAST2_FRAME and + // LAST3_FRAME. + refresh_mask |= + (cpi->refresh_last_frame << cpi->lst_fb_idxes[LAST_REF_FRAMES - 1]); + if (cpi->rc.is_bwd_ref_frame && cpi->num_extra_arfs) { + // We have swapped the virtual indices + refresh_mask |= (cpi->refresh_bwd_ref_frame << cpi->arf_map[0]); + } else { + refresh_mask |= (cpi->refresh_bwd_ref_frame << cpi->bwd_fb_idx); + } +#else + refresh_mask |= (cpi->refresh_last_frame << cpi->lst_fb_idx); +#endif // CONFIG_EXT_REFS + + if (av1_preserve_existing_gf(cpi)) { + // We have decided to preserve the previously existing golden frame as our + // new ARF frame. However, in the short term we leave it in the GF slot and, + // if we're updating the GF with the current decoded frame, we save it + // instead to the ARF slot. + // Later, in the function av1_encoder.c:av1_update_reference_frames() we + // will swap gld_fb_idx and alt_fb_idx to achieve our objective. We do it + // there so that it can be done outside of the recode loop. + // Note: This is highly specific to the use of ARF as a forward reference, + // and this needs to be generalized as other uses are implemented + // (like RTC/temporal scalability). + return refresh_mask | (cpi->refresh_golden_frame << cpi->alt_fb_idx); + } else { +#if CONFIG_EXT_REFS + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + int arf_idx = cpi->arf_map[gf_group->arf_update_idx[gf_group->index]]; +#else + int arf_idx = cpi->alt_fb_idx; + if ((cpi->oxcf.pass == 2) && cpi->multi_arf_allowed) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + arf_idx = gf_group->arf_update_idx[gf_group->index]; + } +#endif // CONFIG_EXT_REFS + return refresh_mask | (cpi->refresh_golden_frame << cpi->gld_fb_idx) | + (cpi->refresh_alt_ref_frame << arf_idx); + } +} + +#if CONFIG_EXT_TILE +static INLINE int find_identical_tile( + const int tile_row, const int tile_col, + TileBufferEnc (*const tile_buffers)[1024]) { + const MV32 candidate_offset[1] = { { 1, 0 } }; + const uint8_t *const cur_tile_data = + tile_buffers[tile_row][tile_col].data + 4; + const size_t cur_tile_size = tile_buffers[tile_row][tile_col].size; + + int i; + + if (tile_row == 0) return 0; + + // (TODO: yunqingwang) For now, only above tile is checked and used. + // More candidates such as left tile can be added later. + for (i = 0; i < 1; i++) { + int row_offset = candidate_offset[0].row; + int col_offset = candidate_offset[0].col; + int row = tile_row - row_offset; + int col = tile_col - col_offset; + uint8_t tile_hdr; + const uint8_t *tile_data; + TileBufferEnc *candidate; + + if (row < 0 || col < 0) continue; + + tile_hdr = *(tile_buffers[row][col].data); + + // Read out tcm bit + if ((tile_hdr >> 7) == 1) { + // The candidate is a copy tile itself + row_offset += tile_hdr & 0x7f; + row = tile_row - row_offset; + } + + candidate = &tile_buffers[row][col]; + + if (row_offset >= 128 || candidate->size != cur_tile_size) continue; + + tile_data = candidate->data + 4; + + if (memcmp(tile_data, cur_tile_data, cur_tile_size) != 0) continue; + + // Identical tile found + assert(row_offset > 0); + return row_offset; + } + + // No identical tile found + return 0; +} +#endif // CONFIG_EXT_TILE + +#if CONFIG_TILE_GROUPS +static uint32_t write_tiles(AV1_COMP *const cpi, + struct aom_write_bit_buffer *wb, + unsigned int *max_tile_size, + unsigned int *max_tile_col_size) { +#else +static uint32_t write_tiles(AV1_COMP *const cpi, uint8_t *const dst, + unsigned int *max_tile_size, + unsigned int *max_tile_col_size) { +#endif + const AV1_COMMON *const cm = &cpi->common; +#if CONFIG_ANS + struct BufAnsCoder *buf_ans = &cpi->buf_ans; +#else + aom_writer mode_bc; +#endif // CONFIG_ANS + int tile_row, tile_col; + TOKENEXTRA *(*const tok_buffers)[MAX_TILE_COLS] = cpi->tile_tok; + TileBufferEnc(*const tile_buffers)[MAX_TILE_COLS] = cpi->tile_buffers; + uint32_t total_size = 0; + const int tile_cols = cm->tile_cols; + const int tile_rows = cm->tile_rows; + unsigned int tile_size = 0; +#if CONFIG_TILE_GROUPS + const int n_log2_tiles = cm->log2_tile_rows + cm->log2_tile_cols; + const int have_tiles = n_log2_tiles > 0; + uint32_t comp_hdr_size; + // Fixed size tile groups for the moment + const int num_tg_hdrs = cm->num_tg; + const int tg_size = (tile_rows * tile_cols + num_tg_hdrs - 1) / num_tg_hdrs; + int tile_count = 0; + int tg_count = 1; + int tile_size_bytes = 4; + int tile_col_size_bytes; + uint32_t uncompressed_hdr_size = 0; + uint8_t *dst = NULL; + struct aom_write_bit_buffer comp_hdr_len_wb; + struct aom_write_bit_buffer tg_params_wb; + struct aom_write_bit_buffer tile_size_bytes_wb; + uint32_t saved_offset; + int mtu_size = cpi->oxcf.mtu; + int curr_tg_data_size = 0; + int hdr_size; +#endif +#if CONFIG_EXT_TILE + const int have_tiles = tile_cols * tile_rows > 1; +#endif // CONFIG_EXT_TILE + + *max_tile_size = 0; + *max_tile_col_size = 0; + +// All tile size fields are output on 4 bytes. A call to remux_tiles will +// later compact the data if smaller headers are adequate. + +#if CONFIG_EXT_TILE + for (tile_col = 0; tile_col < tile_cols; tile_col++) { + TileInfo tile_info; + const int is_last_col = (tile_col == tile_cols - 1); + const uint32_t col_offset = total_size; + + av1_tile_set_col(&tile_info, cm, tile_col); + + // The last column does not have a column header + if (!is_last_col) total_size += 4; + + for (tile_row = 0; tile_row < tile_rows; tile_row++) { + TileBufferEnc *const buf = &tile_buffers[tile_row][tile_col]; + const TOKENEXTRA *tok = tok_buffers[tile_row][tile_col]; + const TOKENEXTRA *tok_end = tok + cpi->tok_count[tile_row][tile_col]; + const int data_offset = have_tiles ? 4 : 0; +#if CONFIG_EC_ADAPT + const int tile_idx = tile_row * tile_cols + tile_col; + TileDataEnc *this_tile = &cpi->tile_data[tile_idx]; +#endif + av1_tile_set_row(&tile_info, cm, tile_row); + + buf->data = dst + total_size; + + // Is CONFIG_EXT_TILE = 1, every tile in the row has a header, + // even for the last one, unless no tiling is used at all. + total_size += data_offset; +#if CONFIG_EC_ADAPT + // Initialise tile context from the frame context + this_tile->tctx = *cm->fc; + cpi->td.mb.e_mbd.tile_ctx = &this_tile->tctx; +#endif +#if CONFIG_PVQ + cpi->td.mb.pvq_q = &this_tile->pvq_q; + cpi->td.mb.daala_enc.state.adapt = &this_tile->tctx.pvq_context; +#endif // CONFIG_PVQ +#if !CONFIG_ANS + aom_start_encode(&mode_bc, buf->data + data_offset); + write_modes(cpi, &tile_info, &mode_bc, &tok, tok_end); + assert(tok == tok_end); + aom_stop_encode(&mode_bc); + tile_size = mode_bc.pos; +#else + buf_ans_write_init(buf_ans, buf->data + data_offset); + write_modes(cpi, &tile_info, buf_ans, &tok, tok_end); + assert(tok == tok_end); + aom_buf_ans_flush(buf_ans); + tile_size = buf_ans_write_end(buf_ans); +#endif // !CONFIG_ANS +#if CONFIG_PVQ + cpi->td.mb.pvq_q = NULL; +#endif + buf->size = tile_size; + + // Record the maximum tile size we see, so we can compact headers later. + *max_tile_size = AOMMAX(*max_tile_size, tile_size); + + if (have_tiles) { + // tile header: size of this tile, or copy offset + uint32_t tile_header = tile_size; + + // If the tile_encoding_mode is 1 (i.e. TILE_VR), check if this tile is + // a copy tile. + // Very low chances to have copy tiles on the key frames, so don't + // search on key frames to reduce unnecessary search. + if (cm->frame_type != KEY_FRAME && cm->tile_encoding_mode) { + const int idendical_tile_offset = + find_identical_tile(tile_row, tile_col, tile_buffers); + + if (idendical_tile_offset > 0) { + tile_size = 0; + tile_header = idendical_tile_offset | 0x80; + tile_header <<= 24; + } + } + + mem_put_le32(buf->data, tile_header); + } + + total_size += tile_size; + } + + if (!is_last_col) { + uint32_t col_size = total_size - col_offset - 4; + mem_put_le32(dst + col_offset, col_size); + + // If it is not final packing, record the maximum tile column size we see, + // otherwise, check if the tile size is out of the range. + *max_tile_col_size = AOMMAX(*max_tile_col_size, col_size); + } + } +#else +#if CONFIG_TILE_GROUPS + write_uncompressed_header(cpi, wb); + +#if CONFIG_EXT_REFS + if (cm->show_existing_frame) { + total_size = aom_wb_bytes_written(wb); + return (uint32_t)total_size; + } +#endif // CONFIG_EXT_REFS + + // Write the tile length code + tile_size_bytes_wb = *wb; + aom_wb_write_literal(wb, 3, 2); + + /* Write a placeholder for the number of tiles in each tile group */ + tg_params_wb = *wb; + saved_offset = wb->bit_offset; + if (have_tiles) { + aom_wb_overwrite_literal(wb, 3, n_log2_tiles); + aom_wb_overwrite_literal(wb, (1 << n_log2_tiles) - 1, n_log2_tiles); + } + + /* Write a placeholder for the compressed header length */ + comp_hdr_len_wb = *wb; + aom_wb_write_literal(wb, 0, 16); + + uncompressed_hdr_size = aom_wb_bytes_written(wb); + dst = wb->bit_buffer; + comp_hdr_size = write_compressed_header(cpi, dst + uncompressed_hdr_size); + aom_wb_overwrite_literal(&comp_hdr_len_wb, (int)(comp_hdr_size), 16); + hdr_size = uncompressed_hdr_size + comp_hdr_size; + total_size += hdr_size; +#endif + + for (tile_row = 0; tile_row < tile_rows; tile_row++) { + TileInfo tile_info; + const int is_last_row = (tile_row == tile_rows - 1); + av1_tile_set_row(&tile_info, cm, tile_row); + + for (tile_col = 0; tile_col < tile_cols; tile_col++) { + const int tile_idx = tile_row * tile_cols + tile_col; + TileBufferEnc *const buf = &tile_buffers[tile_row][tile_col]; +#if CONFIG_PVQ || CONFIG_EC_ADAPT + TileDataEnc *this_tile = &cpi->tile_data[tile_idx]; +#endif + const TOKENEXTRA *tok = tok_buffers[tile_row][tile_col]; + const TOKENEXTRA *tok_end = tok + cpi->tok_count[tile_row][tile_col]; + const int is_last_col = (tile_col == tile_cols - 1); + const int is_last_tile = is_last_col && is_last_row; +#if !CONFIG_TILE_GROUPS + (void)tile_idx; +#else + + if ((!mtu_size && tile_count > tg_size) || + (mtu_size && tile_count && curr_tg_data_size >= mtu_size)) { + // New tile group + tg_count++; + // We've exceeded the packet size + if (tile_count > 1) { + /* The last tile exceeded the packet size. The tile group size + should therefore be tile_count-1. + Move the last tile and insert headers before it + */ + uint32_t old_total_size = total_size - tile_size - 4; + memmove(dst + old_total_size + hdr_size, dst + old_total_size, + (tile_size + 4) * sizeof(uint8_t)); + // Copy uncompressed header + memmove(dst + old_total_size, dst, + uncompressed_hdr_size * sizeof(uint8_t)); + // Write the number of tiles in the group into the last uncompressed + // header before the one we've just inserted + aom_wb_overwrite_literal(&tg_params_wb, tile_idx - tile_count, + n_log2_tiles); + aom_wb_overwrite_literal(&tg_params_wb, tile_count - 2, n_log2_tiles); + // Update the pointer to the last TG params + tg_params_wb.bit_offset = saved_offset + 8 * old_total_size; + // Copy compressed header + memmove(dst + old_total_size + uncompressed_hdr_size, + dst + uncompressed_hdr_size, comp_hdr_size * sizeof(uint8_t)); + total_size += hdr_size; + tile_count = 1; + curr_tg_data_size = hdr_size + tile_size + 4; + + } else { + // We exceeded the packet size in just one tile + // Copy uncompressed header + memmove(dst + total_size, dst, + uncompressed_hdr_size * sizeof(uint8_t)); + // Write the number of tiles in the group into the last uncompressed + // header + aom_wb_overwrite_literal(&tg_params_wb, tile_idx - tile_count, + n_log2_tiles); + aom_wb_overwrite_literal(&tg_params_wb, tile_count - 1, n_log2_tiles); + tg_params_wb.bit_offset = saved_offset + 8 * total_size; + // Copy compressed header + memmove(dst + total_size + uncompressed_hdr_size, + dst + uncompressed_hdr_size, comp_hdr_size * sizeof(uint8_t)); + total_size += hdr_size; + tile_count = 0; + curr_tg_data_size = hdr_size; + } + } + tile_count++; +#endif + av1_tile_set_col(&tile_info, cm, tile_col); + +#if CONFIG_DEPENDENT_HORZTILES && CONFIG_TILE_GROUPS + av1_tile_set_tg_boundary(&tile_info, cm, tile_row, tile_col); +#endif + buf->data = dst + total_size; + + // The last tile does not have a header. + if (!is_last_tile) total_size += 4; + +#if CONFIG_EC_ADAPT + // Initialise tile context from the frame context + this_tile->tctx = *cm->fc; + cpi->td.mb.e_mbd.tile_ctx = &this_tile->tctx; +#endif +#if CONFIG_PVQ + cpi->td.mb.pvq_q = &this_tile->pvq_q; + cpi->td.mb.daala_enc.state.adapt = &this_tile->tctx.pvq_context; +#endif // CONFIG_PVQ +#if CONFIG_ANS + buf_ans_write_init(buf_ans, dst + total_size); + write_modes(cpi, &tile_info, buf_ans, &tok, tok_end); + assert(tok == tok_end); + aom_buf_ans_flush(buf_ans); + tile_size = buf_ans_write_end(buf_ans); +#else + aom_start_encode(&mode_bc, dst + total_size); + write_modes(cpi, &tile_info, &mode_bc, &tok, tok_end); +#if !CONFIG_LV_MAP + assert(tok == tok_end); +#endif // !CONFIG_LV_MAP + aom_stop_encode(&mode_bc); + tile_size = mode_bc.pos; +#endif // CONFIG_ANS +#if CONFIG_PVQ + cpi->td.mb.pvq_q = NULL; +#endif + + assert(tile_size > 0); + +#if CONFIG_TILE_GROUPS + curr_tg_data_size += tile_size + 4; +#endif + buf->size = tile_size; + + if (!is_last_tile) { + *max_tile_size = AOMMAX(*max_tile_size, tile_size); + // size of this tile + mem_put_le32(buf->data, tile_size); + } + + total_size += tile_size; + } + } +#if CONFIG_TILE_GROUPS + // Write the final tile group size + if (n_log2_tiles) { + aom_wb_overwrite_literal(&tg_params_wb, (1 << n_log2_tiles) - tile_count, + n_log2_tiles); + aom_wb_overwrite_literal(&tg_params_wb, tile_count - 1, n_log2_tiles); + } + // Remux if possible. TODO (Thomas Davies): do this for more than one tile + // group + if (have_tiles && tg_count == 1) { + int data_size = total_size - (uncompressed_hdr_size + comp_hdr_size); + data_size = remux_tiles(cm, dst + uncompressed_hdr_size + comp_hdr_size, + data_size, *max_tile_size, *max_tile_col_size, + &tile_size_bytes, &tile_col_size_bytes); + total_size = data_size + uncompressed_hdr_size + comp_hdr_size; + aom_wb_overwrite_literal(&tile_size_bytes_wb, tile_size_bytes - 1, 2); + } + +#endif +#endif // CONFIG_EXT_TILE + return (uint32_t)total_size; +} + +static void write_render_size(const AV1_COMMON *cm, + struct aom_write_bit_buffer *wb) { + const int scaling_active = + cm->width != cm->render_width || cm->height != cm->render_height; + aom_wb_write_bit(wb, scaling_active); + if (scaling_active) { + aom_wb_write_literal(wb, cm->render_width - 1, 16); + aom_wb_write_literal(wb, cm->render_height - 1, 16); + } +} + +#if CONFIG_FRAME_SUPERRES +static void write_superres_scale(const AV1_COMMON *const cm, + struct aom_write_bit_buffer *wb) { + // This scaling and frame superres are probably incompatible + assert(cm->width == cm->render_width && cm->height == cm->render_height); + + // First bit is whether to to scale or not + if (cm->superres_scale_numerator == SUPERRES_SCALE_DENOMINATOR) { + aom_wb_write_bit(wb, 0); // no scaling + } else { + aom_wb_write_bit(wb, 1); // scaling, write scale factor + // TODO(afergs): write factor to the compressed header instead + aom_wb_write_literal( + wb, cm->superres_scale_numerator - SUPERRES_SCALE_NUMERATOR_MIN, + SUPERRES_SCALE_BITS); + } +} +#endif // CONFIG_FRAME_SUPERRES + +static void write_frame_size(const AV1_COMMON *cm, + struct aom_write_bit_buffer *wb) { +#if CONFIG_FRAME_SUPERRES + // If SUPERRES scaling is happening, write the full resolution instead of the + // downscaled resolution. The decoder will reduce this resolution itself. + if (cm->superres_scale_numerator != SUPERRES_SCALE_DENOMINATOR) { + aom_wb_write_literal(wb, cm->superres_width - 1, 16); + aom_wb_write_literal(wb, cm->superres_height - 1, 16); + } else { +#endif // CONFIG_FRAME_SUPERRES + aom_wb_write_literal(wb, cm->width - 1, 16); + aom_wb_write_literal(wb, cm->height - 1, 16); +#if CONFIG_FRAME_SUPERRES + } +#endif // CONFIG_FRAME_SUPERRES + + // TODO(afergs): Also write something different to render_size? + // When superres scales, they'll be almost guaranteed to be + // different on the other side. + write_render_size(cm, wb); +#if CONFIG_FRAME_SUPERRES + write_superres_scale(cm, wb); +#endif // CONFIG_FRAME_SUPERRES +} + +static void write_frame_size_with_refs(AV1_COMP *cpi, + struct aom_write_bit_buffer *wb) { + AV1_COMMON *const cm = &cpi->common; + int found = 0; + + MV_REFERENCE_FRAME ref_frame; + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { + YV12_BUFFER_CONFIG *cfg = get_ref_frame_buffer(cpi, ref_frame); + + if (cfg != NULL) { + found = + cm->width == cfg->y_crop_width && cm->height == cfg->y_crop_height; + found &= cm->render_width == cfg->render_width && + cm->render_height == cfg->render_height; + } + aom_wb_write_bit(wb, found); + if (found) { + break; + } + } + + if (!found) { + write_frame_size(cm, wb); + } +} + +static void write_sync_code(struct aom_write_bit_buffer *wb) { + aom_wb_write_literal(wb, AV1_SYNC_CODE_0, 8); + aom_wb_write_literal(wb, AV1_SYNC_CODE_1, 8); + aom_wb_write_literal(wb, AV1_SYNC_CODE_2, 8); +} + +static void write_profile(BITSTREAM_PROFILE profile, + struct aom_write_bit_buffer *wb) { + switch (profile) { + case PROFILE_0: aom_wb_write_literal(wb, 0, 2); break; + case PROFILE_1: aom_wb_write_literal(wb, 2, 2); break; + case PROFILE_2: aom_wb_write_literal(wb, 1, 2); break; + case PROFILE_3: aom_wb_write_literal(wb, 6, 3); break; + default: assert(0); + } +} + +static void write_bitdepth_colorspace_sampling( + AV1_COMMON *const cm, struct aom_write_bit_buffer *wb) { + if (cm->profile >= PROFILE_2) { + assert(cm->bit_depth > AOM_BITS_8); + aom_wb_write_bit(wb, cm->bit_depth == AOM_BITS_10 ? 0 : 1); + } + aom_wb_write_literal(wb, cm->color_space, 3); + if (cm->color_space != AOM_CS_SRGB) { + // 0: [16, 235] (i.e. xvYCC), 1: [0, 255] + aom_wb_write_bit(wb, cm->color_range); + if (cm->profile == PROFILE_1 || cm->profile == PROFILE_3) { + assert(cm->subsampling_x != 1 || cm->subsampling_y != 1); + aom_wb_write_bit(wb, cm->subsampling_x); + aom_wb_write_bit(wb, cm->subsampling_y); + aom_wb_write_bit(wb, 0); // unused + } else { + assert(cm->subsampling_x == 1 && cm->subsampling_y == 1); + } + } else { + assert(cm->profile == PROFILE_1 || cm->profile == PROFILE_3); + aom_wb_write_bit(wb, 0); // unused + } +} + +#if CONFIG_REFERENCE_BUFFER +void write_sequence_header(SequenceHeader *seq_params) { + /* Placeholder for actually writing to the bitstream */ + seq_params->frame_id_numbers_present_flag = FRAME_ID_NUMBERS_PRESENT_FLAG; + seq_params->frame_id_length_minus7 = FRAME_ID_LENGTH_MINUS7; + seq_params->delta_frame_id_length_minus2 = DELTA_FRAME_ID_LENGTH_MINUS2; +} +#endif + +static void write_uncompressed_header(AV1_COMP *cpi, + struct aom_write_bit_buffer *wb) { + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; + +#if CONFIG_REFERENCE_BUFFER + /* TODO: Move outside frame loop or inside key-frame branch */ + write_sequence_header(&cpi->seq_params); +#endif + + aom_wb_write_literal(wb, AOM_FRAME_MARKER, 2); + + write_profile(cm->profile, wb); + +#if CONFIG_EXT_REFS + // NOTE: By default all coded frames to be used as a reference + cm->is_reference_frame = 1; + + if (cm->show_existing_frame) { + RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; + const int frame_to_show = cm->ref_frame_map[cpi->existing_fb_idx_to_show]; + + if (frame_to_show < 0 || frame_bufs[frame_to_show].ref_count < 1) { + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "Buffer %d does not contain a reconstructed frame", + frame_to_show); + } + ref_cnt_fb(frame_bufs, &cm->new_fb_idx, frame_to_show); + + aom_wb_write_bit(wb, 1); // show_existing_frame + aom_wb_write_literal(wb, cpi->existing_fb_idx_to_show, 3); + +#if CONFIG_REFERENCE_BUFFER + if (cpi->seq_params.frame_id_numbers_present_flag) { + int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7; + int display_frame_id = cm->ref_frame_id[cpi->existing_fb_idx_to_show]; + aom_wb_write_literal(wb, display_frame_id, frame_id_len); + /* Add a zero byte to prevent emulation of superframe marker */ + /* Same logic as when when terminating the entropy coder */ + /* Consider to have this logic only one place */ + aom_wb_write_literal(wb, 0, 8); + } +#endif + + return; + } else { +#endif // CONFIG_EXT_REFS + aom_wb_write_bit(wb, 0); // show_existing_frame +#if CONFIG_EXT_REFS + } +#endif // CONFIG_EXT_REFS + + aom_wb_write_bit(wb, cm->frame_type); + aom_wb_write_bit(wb, cm->show_frame); + aom_wb_write_bit(wb, cm->error_resilient_mode); + +#if CONFIG_REFERENCE_BUFFER + cm->invalid_delta_frame_id_minus1 = 0; + if (cpi->seq_params.frame_id_numbers_present_flag) { + int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7; + aom_wb_write_literal(wb, cm->current_frame_id, frame_id_len); + } +#endif + +#if CONFIG_FRAME_SUPERRES + // TODO(afergs): Remove - this is just to stop superres from breaking + cm->superres_scale_numerator = SUPERRES_SCALE_DENOMINATOR; +#endif // CONFIG_FRAME_SUPERRES + + if (cm->frame_type == KEY_FRAME) { + write_sync_code(wb); + write_bitdepth_colorspace_sampling(cm, wb); + write_frame_size(cm, wb); +#if CONFIG_ANS && ANS_MAX_SYMBOLS + assert(cpi->common.ans_window_size_log2 >= 8); + assert(cpi->common.ans_window_size_log2 < 24); + aom_wb_write_literal(wb, cpi->common.ans_window_size_log2 - 8, 4); +#endif // CONFIG_ANS && ANS_MAX_SYMBOLS +#if CONFIG_PALETTE + aom_wb_write_bit(wb, cm->allow_screen_content_tools); +#endif // CONFIG_PALETTE + } else { + if (!cm->show_frame) aom_wb_write_bit(wb, cm->intra_only); +#if CONFIG_PALETTE + if (cm->intra_only) aom_wb_write_bit(wb, cm->allow_screen_content_tools); +#endif // CONFIG_PALETTE + if (!cm->error_resilient_mode) { + if (cm->intra_only) { + aom_wb_write_bit(wb, + cm->reset_frame_context == RESET_FRAME_CONTEXT_ALL); + } else { + aom_wb_write_bit(wb, + cm->reset_frame_context != RESET_FRAME_CONTEXT_NONE); + if (cm->reset_frame_context != RESET_FRAME_CONTEXT_NONE) + aom_wb_write_bit(wb, + cm->reset_frame_context == RESET_FRAME_CONTEXT_ALL); + } + } + +#if CONFIG_EXT_REFS + cpi->refresh_frame_mask = get_refresh_mask(cpi); +#endif // CONFIG_EXT_REFS + + if (cm->intra_only) { + write_sync_code(wb); + write_bitdepth_colorspace_sampling(cm, wb); + +#if CONFIG_EXT_REFS + aom_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES); +#else + aom_wb_write_literal(wb, get_refresh_mask(cpi), REF_FRAMES); +#endif // CONFIG_EXT_REFS + write_frame_size(cm, wb); + +#if CONFIG_ANS && ANS_MAX_SYMBOLS + assert(cpi->common.ans_window_size_log2 >= 8); + assert(cpi->common.ans_window_size_log2 < 24); + aom_wb_write_literal(wb, cpi->common.ans_window_size_log2 - 8, 4); +#endif // CONFIG_ANS && ANS_MAX_SYMBOLS + } else { + MV_REFERENCE_FRAME ref_frame; + +#if CONFIG_EXT_REFS + aom_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES); +#else + aom_wb_write_literal(wb, get_refresh_mask(cpi), REF_FRAMES); +#endif // CONFIG_EXT_REFS + +#if CONFIG_EXT_REFS + if (!cpi->refresh_frame_mask) { + // NOTE: "cpi->refresh_frame_mask == 0" indicates that the coded frame + // will not be used as a reference + cm->is_reference_frame = 0; + } +#endif // CONFIG_EXT_REFS + + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { + assert(get_ref_frame_map_idx(cpi, ref_frame) != INVALID_IDX); + aom_wb_write_literal(wb, get_ref_frame_map_idx(cpi, ref_frame), + REF_FRAMES_LOG2); + aom_wb_write_bit(wb, cm->ref_frame_sign_bias[ref_frame]); +#if CONFIG_REFERENCE_BUFFER + if (cpi->seq_params.frame_id_numbers_present_flag) { + int i = get_ref_frame_map_idx(cpi, ref_frame); + int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7; + int diff_len = cpi->seq_params.delta_frame_id_length_minus2 + 2; + int delta_frame_id_minus1 = + ((cm->current_frame_id - cm->ref_frame_id[i] + + (1 << frame_id_len)) % + (1 << frame_id_len)) - + 1; + if (delta_frame_id_minus1 < 0 || + delta_frame_id_minus1 >= (1 << diff_len)) + cm->invalid_delta_frame_id_minus1 = 1; + aom_wb_write_literal(wb, delta_frame_id_minus1, diff_len); + } +#endif + } + +#if CONFIG_FRAME_SIZE + if (cm->error_resilient_mode == 0) { + write_frame_size_with_refs(cpi, wb); + } else { + write_frame_size(cm, wb); + } +#else + write_frame_size_with_refs(cpi, wb); +#endif + + aom_wb_write_bit(wb, cm->allow_high_precision_mv); + + fix_interp_filter(cm, cpi->td.counts); + write_frame_interp_filter(cm->interp_filter, wb); +#if CONFIG_TEMPMV_SIGNALING + if (!cm->error_resilient_mode) { + aom_wb_write_bit(wb, cm->use_prev_frame_mvs); + } +#endif + } + } + +#if CONFIG_REFERENCE_BUFFER + cm->refresh_mask = cm->frame_type == KEY_FRAME ? 0xFF : get_refresh_mask(cpi); +#endif + + if (!cm->error_resilient_mode) { + aom_wb_write_bit( + wb, cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_FORWARD); + } + + aom_wb_write_literal(wb, cm->frame_context_idx, FRAME_CONTEXTS_LOG2); + + assert(cm->mib_size == mi_size_wide[cm->sb_size]); + assert(cm->mib_size == 1 << cm->mib_size_log2); +#if CONFIG_EXT_PARTITION + assert(cm->sb_size == BLOCK_128X128 || cm->sb_size == BLOCK_64X64); + aom_wb_write_bit(wb, cm->sb_size == BLOCK_128X128 ? 1 : 0); +#else + assert(cm->sb_size == BLOCK_64X64); +#endif // CONFIG_EXT_PARTITION + + encode_loopfilter(cm, wb); +#if CONFIG_CDEF + encode_cdef(cm, wb); +#endif +#if CONFIG_LOOP_RESTORATION + encode_restoration_mode(cm, wb); +#endif // CONFIG_LOOP_RESTORATION + encode_quantization(cm, wb); + encode_segmentation(cm, xd, wb); +#if CONFIG_DELTA_Q + { + int i; + struct segmentation *const seg = &cm->seg; + int segment_quantizer_active = 0; + for (i = 0; i < MAX_SEGMENTS; i++) { + if (segfeature_active(seg, i, SEG_LVL_ALT_Q)) { + segment_quantizer_active = 1; + } + } + + if (cm->delta_q_present_flag) + assert(segment_quantizer_active == 0 && cm->base_qindex > 0); + if (segment_quantizer_active == 0 && cm->base_qindex > 0) { + aom_wb_write_bit(wb, cm->delta_q_present_flag); + if (cm->delta_q_present_flag) { + aom_wb_write_literal(wb, OD_ILOG_NZ(cm->delta_q_res) - 1, 2); + xd->prev_qindex = cm->base_qindex; +#if CONFIG_EXT_DELTA_Q + assert(seg->abs_delta == SEGMENT_DELTADATA); + aom_wb_write_bit(wb, cm->delta_lf_present_flag); + if (cm->delta_lf_present_flag) { + aom_wb_write_literal(wb, OD_ILOG_NZ(cm->delta_lf_res) - 1, 2); + xd->prev_delta_lf_from_base = 0; + } +#endif // CONFIG_EXT_DELTA_Q + } + } + } +#endif + + write_tx_mode(cm, xd, &cm->tx_mode, wb); + + if (cpi->allow_comp_inter_inter) { + const int use_hybrid_pred = cm->reference_mode == REFERENCE_MODE_SELECT; +#if !CONFIG_REF_ADAPT + const int use_compound_pred = cm->reference_mode != SINGLE_REFERENCE; +#endif // !CONFIG_REF_ADAPT + + aom_wb_write_bit(wb, use_hybrid_pred); +#if !CONFIG_REF_ADAPT + if (!use_hybrid_pred) aom_wb_write_bit(wb, use_compound_pred); +#endif // !CONFIG_REF_ADAPT + } + +#if CONFIG_EXT_TX + aom_wb_write_bit(wb, cm->reduced_tx_set_used); +#endif // CONFIG_EXT_TX + + write_tile_info(cm, wb); +} + +#if CONFIG_GLOBAL_MOTION +static void write_global_motion_params(WarpedMotionParams *params, + WarpedMotionParams *ref_params, + aom_prob *probs, aom_writer *w, + int allow_hp) { + TransformationType type = params->wmtype; + int trans_bits; + int trans_prec_diff; + av1_write_token(w, av1_global_motion_types_tree, probs, + &global_motion_types_encodings[type]); + switch (type) { + case HOMOGRAPHY: + case HORTRAPEZOID: + case VERTRAPEZOID: + if (type != HORTRAPEZOID) + aom_write_signed_primitive_refsubexpfin( + w, GM_ROW3HOMO_MAX + 1, SUBEXPFIN_K, + (ref_params->wmmat[6] >> GM_ROW3HOMO_PREC_DIFF), + (params->wmmat[6] >> GM_ROW3HOMO_PREC_DIFF)); + if (type != VERTRAPEZOID) + aom_write_signed_primitive_refsubexpfin( + w, GM_ROW3HOMO_MAX + 1, SUBEXPFIN_K, + (ref_params->wmmat[7] >> GM_ROW3HOMO_PREC_DIFF), + (params->wmmat[7] >> GM_ROW3HOMO_PREC_DIFF)); + // fallthrough intended + case AFFINE: + case ROTZOOM: + aom_write_signed_primitive_refsubexpfin( + w, GM_ALPHA_MAX + 1, SUBEXPFIN_K, + (ref_params->wmmat[2] >> GM_ALPHA_PREC_DIFF) - + (1 << GM_ALPHA_PREC_BITS), + (params->wmmat[2] >> GM_ALPHA_PREC_DIFF) - (1 << GM_ALPHA_PREC_BITS)); + if (type != VERTRAPEZOID) + aom_write_signed_primitive_refsubexpfin( + w, GM_ALPHA_MAX + 1, SUBEXPFIN_K, + (ref_params->wmmat[3] >> GM_ALPHA_PREC_DIFF), + (params->wmmat[3] >> GM_ALPHA_PREC_DIFF)); + if (type >= AFFINE) { + if (type != HORTRAPEZOID) + aom_write_signed_primitive_refsubexpfin( + w, GM_ALPHA_MAX + 1, SUBEXPFIN_K, + (ref_params->wmmat[4] >> GM_ALPHA_PREC_DIFF), + (params->wmmat[4] >> GM_ALPHA_PREC_DIFF)); + aom_write_signed_primitive_refsubexpfin( + w, GM_ALPHA_MAX + 1, SUBEXPFIN_K, + (ref_params->wmmat[5] >> GM_ALPHA_PREC_DIFF) - + (1 << GM_ALPHA_PREC_BITS), + (params->wmmat[5] >> GM_ALPHA_PREC_DIFF) - + (1 << GM_ALPHA_PREC_BITS)); + } + // fallthrough intended + case TRANSLATION: + trans_bits = (type == TRANSLATION) ? GM_ABS_TRANS_ONLY_BITS - !allow_hp + : GM_ABS_TRANS_BITS; + trans_prec_diff = (type == TRANSLATION) + ? GM_TRANS_ONLY_PREC_DIFF + !allow_hp + : GM_TRANS_PREC_DIFF; + aom_write_signed_primitive_refsubexpfin( + w, (1 << trans_bits) + 1, SUBEXPFIN_K, + (ref_params->wmmat[0] >> trans_prec_diff), + (params->wmmat[0] >> trans_prec_diff)); + aom_write_signed_primitive_refsubexpfin( + w, (1 << trans_bits) + 1, SUBEXPFIN_K, + (ref_params->wmmat[1] >> trans_prec_diff), + (params->wmmat[1] >> trans_prec_diff)); + break; + case IDENTITY: break; + default: assert(0); + } +} + +static void write_global_motion(AV1_COMP *cpi, aom_writer *w) { + AV1_COMMON *const cm = &cpi->common; + int frame; + for (frame = LAST_FRAME; frame <= ALTREF_FRAME; ++frame) { +#if !CONFIG_REF_MV + // With ref-mv, clearing unused global motion models here is + // unsafe, and we need to rely on the recode loop to do it + // instead. See av1_find_mv_refs for details. + if (!cpi->td.rd_counts.global_motion_used[frame]) { + set_default_warp_params(&cm->global_motion[frame]); + } +#endif + write_global_motion_params( + &cm->global_motion[frame], &cm->prev_frame->global_motion[frame], + cm->fc->global_motion_types_prob, w, cm->allow_high_precision_mv); + /* + printf("Frame %d/%d: Enc Ref %d (used %d): %d %d %d %d\n", + cm->current_video_frame, cm->show_frame, frame, + cpi->global_motion_used[frame], cm->global_motion[frame].wmmat[0], + cm->global_motion[frame].wmmat[1], cm->global_motion[frame].wmmat[2], + cm->global_motion[frame].wmmat[3]); + */ + } +} +#endif + +static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data) { + AV1_COMMON *const cm = &cpi->common; +#if CONFIG_SUPERTX + MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; +#endif // CONFIG_SUPERTX + FRAME_CONTEXT *const fc = cm->fc; + FRAME_COUNTS *counts = cpi->td.counts; + aom_writer *header_bc; + int i, j; + +#if CONFIG_TILE_GROUPS + const int probwt = cm->num_tg; +#else + const int probwt = 1; +#endif + +#if CONFIG_ANS + int header_size; + header_bc = &cpi->buf_ans; + buf_ans_write_init(header_bc, data); +#else + aom_writer real_header_bc; + header_bc = &real_header_bc; + aom_start_encode(header_bc, data); +#endif + +#if CONFIG_LOOP_RESTORATION + encode_restoration(cm, header_bc); +#endif // CONFIG_LOOP_RESTORATION +#if !CONFIG_EC_ADAPT + update_txfm_probs(cm, header_bc, counts); +#endif +#if CONFIG_LV_MAP + av1_write_txb_probs(cpi, header_bc); +#else +#if !CONFIG_PVQ +#if !(CONFIG_EC_ADAPT && CONFIG_NEW_TOKENSET) + update_coef_probs(cpi, header_bc); +#endif // !(CONFIG_EC_ADAPT && CONFIG_NEW_TOKENSET) +#endif // CONFIG_PVQ +#endif // CONFIG_LV_MAP + +#if CONFIG_VAR_TX + update_txfm_partition_probs(cm, header_bc, counts, probwt); +#endif + + update_skip_probs(cm, header_bc, counts); +#if !CONFIG_EC_ADAPT && CONFIG_DELTA_Q + update_delta_q_probs(cm, header_bc, counts); +#if CONFIG_EXT_DELTA_Q + update_delta_lf_probs(cm, header_bc, counts); +#endif +#endif +#if !CONFIG_EC_ADAPT + update_seg_probs(cpi, header_bc); + + for (i = 0; i < INTRA_MODES; ++i) { + prob_diff_update(av1_intra_mode_tree, fc->uv_mode_prob[i], + counts->uv_mode[i], INTRA_MODES, probwt, header_bc); + } + +#if CONFIG_EXT_PARTITION_TYPES + for (i = 0; i < PARTITION_PLOFFSET; ++i) + prob_diff_update(av1_partition_tree, fc->partition_prob[i], + counts->partition[i], PARTITION_TYPES, probwt, header_bc); + for (; i < PARTITION_CONTEXTS_PRIMARY; ++i) + prob_diff_update(av1_ext_partition_tree, fc->partition_prob[i], + counts->partition[i], EXT_PARTITION_TYPES, probwt, + header_bc); +#else + for (i = 0; i < PARTITION_CONTEXTS_PRIMARY; ++i) + prob_diff_update(av1_partition_tree, fc->partition_prob[i], + counts->partition[i], PARTITION_TYPES, probwt, header_bc); +#endif // CONFIG_EXT_PARTITION_TYPES +#if CONFIG_UNPOISON_PARTITION_CTX + for (; i < PARTITION_CONTEXTS_PRIMARY + PARTITION_BLOCK_SIZES; ++i) { + unsigned int ct[2] = { counts->partition[i][PARTITION_VERT], + counts->partition[i][PARTITION_SPLIT] }; + assert(counts->partition[i][PARTITION_NONE] == 0); + assert(counts->partition[i][PARTITION_HORZ] == 0); + assert(fc->partition_prob[i][PARTITION_NONE] == 0); + assert(fc->partition_prob[i][PARTITION_HORZ] == 0); + av1_cond_prob_diff_update(header_bc, &fc->partition_prob[i][PARTITION_VERT], + ct, probwt); + } + for (; i < PARTITION_CONTEXTS_PRIMARY + 2 * PARTITION_BLOCK_SIZES; ++i) { + unsigned int ct[2] = { counts->partition[i][PARTITION_HORZ], + counts->partition[i][PARTITION_SPLIT] }; + assert(counts->partition[i][PARTITION_NONE] == 0); + assert(counts->partition[i][PARTITION_VERT] == 0); + assert(fc->partition_prob[i][PARTITION_NONE] == 0); + assert(fc->partition_prob[i][PARTITION_VERT] == 0); + av1_cond_prob_diff_update(header_bc, &fc->partition_prob[i][PARTITION_HORZ], + ct, probwt); + } +#endif +#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP + for (i = 0; i < INTRA_FILTERS + 1; ++i) + prob_diff_update(av1_intra_filter_tree, fc->intra_filter_probs[i], + counts->intra_filter[i], INTRA_FILTERS, probwt, header_bc); +#endif // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP +#endif // !CONFIG_EC_ADAPT + + if (frame_is_intra_only(cm)) { + av1_copy(cm->kf_y_prob, av1_kf_y_mode_prob); +#if CONFIG_EC_MULTISYMBOL + av1_copy(cm->fc->kf_y_cdf, av1_kf_y_mode_cdf); +#endif + +#if !CONFIG_EC_ADAPT + for (i = 0; i < INTRA_MODES; ++i) + for (j = 0; j < INTRA_MODES; ++j) + prob_diff_update(av1_intra_mode_tree, cm->kf_y_prob[i][j], + counts->kf_y_mode[i][j], INTRA_MODES, probwt, + header_bc); +#endif // CONFIG_EC_ADAPT + } else { +#if CONFIG_REF_MV + update_inter_mode_probs(cm, header_bc, counts); +#else +#if !CONFIG_EC_ADAPT + for (i = 0; i < INTER_MODE_CONTEXTS; ++i) { + prob_diff_update(av1_inter_mode_tree, cm->fc->inter_mode_probs[i], + counts->inter_mode[i], INTER_MODES, probwt, header_bc); + } +#endif +#endif +#if CONFIG_EXT_INTER + update_inter_compound_mode_probs(cm, probwt, header_bc); + + if (cm->reference_mode != COMPOUND_REFERENCE) { + for (i = 0; i < BLOCK_SIZE_GROUPS; i++) { + if (is_interintra_allowed_bsize_group(i)) { + av1_cond_prob_diff_update(header_bc, &fc->interintra_prob[i], + cm->counts.interintra[i], probwt); + } + } + for (i = 0; i < BLOCK_SIZE_GROUPS; i++) { + prob_diff_update( + av1_interintra_mode_tree, cm->fc->interintra_mode_prob[i], + counts->interintra_mode[i], INTERINTRA_MODES, probwt, header_bc); + } + for (i = 0; i < BLOCK_SIZES; i++) { + if (is_interintra_allowed_bsize(i) && is_interintra_wedge_used(i)) + av1_cond_prob_diff_update(header_bc, &fc->wedge_interintra_prob[i], + cm->counts.wedge_interintra[i], probwt); + } + } +#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE + if (cm->reference_mode != SINGLE_REFERENCE) { + for (i = 0; i < BLOCK_SIZES; i++) + prob_diff_update(av1_compound_type_tree, fc->compound_type_prob[i], + cm->counts.compound_interinter[i], COMPOUND_TYPES, + probwt, header_bc); + } +#endif // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE +#endif // CONFIG_EXT_INTER + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + for (i = BLOCK_8X8; i < BLOCK_SIZES; ++i) + prob_diff_update(av1_motion_mode_tree, fc->motion_mode_prob[i], + counts->motion_mode[i], MOTION_MODES, probwt, header_bc); +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +#if !CONFIG_EC_ADAPT + if (cm->interp_filter == SWITCHABLE) + update_switchable_interp_probs(cm, header_bc, counts); +#endif + + for (i = 0; i < INTRA_INTER_CONTEXTS; i++) + av1_cond_prob_diff_update(header_bc, &fc->intra_inter_prob[i], + counts->intra_inter[i], probwt); + + if (cpi->allow_comp_inter_inter) { + const int use_hybrid_pred = cm->reference_mode == REFERENCE_MODE_SELECT; + if (use_hybrid_pred) + for (i = 0; i < COMP_INTER_CONTEXTS; i++) + av1_cond_prob_diff_update(header_bc, &fc->comp_inter_prob[i], + counts->comp_inter[i], probwt); + } + + if (cm->reference_mode != COMPOUND_REFERENCE) { + for (i = 0; i < REF_CONTEXTS; i++) { + for (j = 0; j < (SINGLE_REFS - 1); j++) { + av1_cond_prob_diff_update(header_bc, &fc->single_ref_prob[i][j], + counts->single_ref[i][j], probwt); + } + } + } + if (cm->reference_mode != SINGLE_REFERENCE) { + for (i = 0; i < REF_CONTEXTS; i++) { +#if CONFIG_EXT_REFS + for (j = 0; j < (FWD_REFS - 1); j++) { + av1_cond_prob_diff_update(header_bc, &fc->comp_ref_prob[i][j], + counts->comp_ref[i][j], probwt); + } + for (j = 0; j < (BWD_REFS - 1); j++) { + av1_cond_prob_diff_update(header_bc, &fc->comp_bwdref_prob[i][j], + counts->comp_bwdref[i][j], probwt); + } +#else + for (j = 0; j < (COMP_REFS - 1); j++) { + av1_cond_prob_diff_update(header_bc, &fc->comp_ref_prob[i][j], + counts->comp_ref[i][j], probwt); + } +#endif // CONFIG_EXT_REFS + } + } + +#if !CONFIG_EC_ADAPT + for (i = 0; i < BLOCK_SIZE_GROUPS; ++i) { + prob_diff_update(av1_intra_mode_tree, cm->fc->y_mode_prob[i], + counts->y_mode[i], INTRA_MODES, probwt, header_bc); + } +#endif + + av1_write_nmv_probs(cm, cm->allow_high_precision_mv, header_bc, +#if CONFIG_REF_MV + counts->mv); +#else + &counts->mv); +#endif +#if !CONFIG_EC_ADAPT + update_ext_tx_probs(cm, header_bc); +#endif +#if CONFIG_SUPERTX + if (!xd->lossless[0]) update_supertx_probs(cm, probwt, header_bc); +#endif // CONFIG_SUPERTX +#if CONFIG_GLOBAL_MOTION + write_global_motion(cpi, header_bc); +#endif // CONFIG_GLOBAL_MOTION + } +#if CONFIG_EC_MULTISYMBOL +#if !CONFIG_EC_ADAPT +#if CONFIG_NEW_TOKENSET + av1_coef_head_cdfs(fc); +#endif + av1_coef_pareto_cdfs(fc); +#if CONFIG_REF_MV + for (i = 0; i < NMV_CONTEXTS; ++i) av1_set_mv_cdfs(&fc->nmvc[i]); +#else + av1_set_mv_cdfs(&fc->nmvc); +#endif +#if CONFIG_EC_MULTISYMBOL + av1_set_mode_cdfs(cm); +#endif +#endif // !CONFIG_EC_ADAPT +#endif +#if CONFIG_ANS + aom_buf_ans_flush(header_bc); + header_size = buf_ans_write_end(header_bc); + assert(header_size <= 0xffff); + return header_size; +#else + aom_stop_encode(header_bc); + assert(header_bc->pos <= 0xffff); + return header_bc->pos; +#endif // CONFIG_ANS +} + +static int choose_size_bytes(uint32_t size, int spare_msbs) { + // Choose the number of bytes required to represent size, without + // using the 'spare_msbs' number of most significant bits. + + // Make sure we will fit in 4 bytes to start with.. + if (spare_msbs > 0 && size >> (32 - spare_msbs) != 0) return -1; + + // Normalise to 32 bits + size <<= spare_msbs; + + if (size >> 24 != 0) + return 4; + else if (size >> 16 != 0) + return 3; + else if (size >> 8 != 0) + return 2; + else + return 1; +} + +static void mem_put_varsize(uint8_t *const dst, const int sz, const int val) { + switch (sz) { + case 1: dst[0] = (uint8_t)(val & 0xff); break; + case 2: mem_put_le16(dst, val); break; + case 3: mem_put_le24(dst, val); break; + case 4: mem_put_le32(dst, val); break; + default: assert(0 && "Invalid size"); break; + } +} +static int remux_tiles(const AV1_COMMON *const cm, uint8_t *dst, + const uint32_t data_size, const uint32_t max_tile_size, + const uint32_t max_tile_col_size, + int *const tile_size_bytes, + int *const tile_col_size_bytes) { +// Choose the tile size bytes (tsb) and tile column size bytes (tcsb) +#if CONFIG_EXT_TILE + // The top bit in the tile size field indicates tile copy mode, so we + // have 1 less bit to code the tile size + const int tsb = choose_size_bytes(max_tile_size, 1); + const int tcsb = choose_size_bytes(max_tile_col_size, 0); +#else + const int tsb = choose_size_bytes(max_tile_size, 0); + const int tcsb = 4; // This is ignored + (void)max_tile_col_size; +#endif // CONFIG_EXT_TILE + + assert(tsb > 0); + assert(tcsb > 0); + + *tile_size_bytes = tsb; + *tile_col_size_bytes = tcsb; + + if (tsb == 4 && tcsb == 4) { + return data_size; + } else { + uint32_t wpos = 0; + uint32_t rpos = 0; + +#if CONFIG_EXT_TILE + int tile_row; + int tile_col; + + for (tile_col = 0; tile_col < cm->tile_cols; tile_col++) { + // All but the last column has a column header + if (tile_col < cm->tile_cols - 1) { + uint32_t tile_col_size = mem_get_le32(dst + rpos); + rpos += 4; + + // Adjust the tile column size by the number of bytes removed + // from the tile size fields. + tile_col_size -= (4 - tsb) * cm->tile_rows; + + mem_put_varsize(dst + wpos, tcsb, tile_col_size); + wpos += tcsb; + } + + for (tile_row = 0; tile_row < cm->tile_rows; tile_row++) { + // All, including the last row has a header + uint32_t tile_header = mem_get_le32(dst + rpos); + rpos += 4; + + // If this is a copy tile, we need to shift the MSB to the + // top bit of the new width, and there is no data to copy. + if (tile_header >> 31 != 0) { + if (tsb < 4) tile_header >>= 32 - 8 * tsb; + mem_put_varsize(dst + wpos, tsb, tile_header); + wpos += tsb; + } else { + mem_put_varsize(dst + wpos, tsb, tile_header); + wpos += tsb; + + memmove(dst + wpos, dst + rpos, tile_header); + rpos += tile_header; + wpos += tile_header; + } + } + } +#else + const int n_tiles = cm->tile_cols * cm->tile_rows; + int n; + + for (n = 0; n < n_tiles; n++) { + int tile_size; + + if (n == n_tiles - 1) { + tile_size = data_size - rpos; + } else { + tile_size = mem_get_le32(dst + rpos); + rpos += 4; + mem_put_varsize(dst + wpos, tsb, tile_size); + wpos += tsb; + } + + memmove(dst + wpos, dst + rpos, tile_size); + + rpos += tile_size; + wpos += tile_size; + } +#endif // CONFIG_EXT_TILE + + assert(rpos > wpos); + assert(rpos == data_size); + + return wpos; + } +} + +void av1_pack_bitstream(AV1_COMP *const cpi, uint8_t *dst, size_t *size) { + uint8_t *data = dst; +#if !CONFIG_TILE_GROUPS + uint32_t compressed_header_size; + uint32_t uncompressed_header_size; + struct aom_write_bit_buffer saved_wb; +#endif + uint32_t data_size; + struct aom_write_bit_buffer wb = { data, 0 }; + + unsigned int max_tile_size; + unsigned int max_tile_col_size; + +#if CONFIG_BITSTREAM_DEBUG + bitstream_queue_reset_write(); +#endif + +#if !CONFIG_TILE_GROUPS + int tile_size_bytes; + int tile_col_size_bytes; + AV1_COMMON *const cm = &cpi->common; + const int have_tiles = cm->tile_cols * cm->tile_rows > 1; + + // Write the uncompressed header + write_uncompressed_header(cpi, &wb); + +#if CONFIG_EXT_REFS + if (cm->show_existing_frame) { + *size = aom_wb_bytes_written(&wb); + return; + } +#endif // CONFIG_EXT_REFS + + // We do not know these in advance. Output placeholder bit. + saved_wb = wb; + // Write tile size magnitudes + if (have_tiles) { +// Note that the last item in the uncompressed header is the data +// describing tile configuration. +#if CONFIG_EXT_TILE + // Number of bytes in tile column size - 1 + aom_wb_write_literal(&wb, 0, 2); +#endif // CONFIG_EXT_TILE + // Number of bytes in tile size - 1 + aom_wb_write_literal(&wb, 0, 2); + } + // Size of compressed header + aom_wb_write_literal(&wb, 0, 16); + + uncompressed_header_size = (uint32_t)aom_wb_bytes_written(&wb); + data += uncompressed_header_size; + + aom_clear_system_state(); + + // Write the compressed header + compressed_header_size = write_compressed_header(cpi, data); + data += compressed_header_size; + + // Write the encoded tile data + data_size = write_tiles(cpi, data, &max_tile_size, &max_tile_col_size); +#else + data_size = write_tiles(cpi, &wb, &max_tile_size, &max_tile_col_size); +#endif +#if !CONFIG_TILE_GROUPS + if (have_tiles) { + data_size = + remux_tiles(cm, data, data_size, max_tile_size, max_tile_col_size, + &tile_size_bytes, &tile_col_size_bytes); + } + + data += data_size; + + // Now fill in the gaps in the uncompressed header. + if (have_tiles) { +#if CONFIG_EXT_TILE + assert(tile_col_size_bytes >= 1 && tile_col_size_bytes <= 4); + aom_wb_write_literal(&saved_wb, tile_col_size_bytes - 1, 2); +#endif // CONFIG_EXT_TILE + assert(tile_size_bytes >= 1 && tile_size_bytes <= 4); + aom_wb_write_literal(&saved_wb, tile_size_bytes - 1, 2); + } + // TODO(jbb): Figure out what to do if compressed_header_size > 16 bits. + assert(compressed_header_size <= 0xffff); + aom_wb_write_literal(&saved_wb, compressed_header_size, 16); +#else + data += data_size; +#endif +#if CONFIG_ANS && ANS_REVERSE + // Avoid aliasing the superframe index + *data++ = 0; +#endif + *size = data - dst; +} diff --git a/third_party/aom/av1/encoder/bitstream.h b/third_party/aom/av1/encoder/bitstream.h new file mode 100644 index 0000000000..c75d80891b --- /dev/null +++ b/third_party/aom/av1/encoder/bitstream.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_BITSTREAM_H_ +#define AV1_ENCODER_BITSTREAM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "av1/encoder/encoder.h" + +#if CONFIG_REFERENCE_BUFFER +void write_sequence_header(SequenceHeader *seq_params); +#endif + +void av1_pack_bitstream(AV1_COMP *const cpi, uint8_t *dest, size_t *size); + +void av1_encode_token_init(void); + +static INLINE int av1_preserve_existing_gf(AV1_COMP *cpi) { +#if CONFIG_EXT_REFS + // Do not swap gf and arf indices for internal overlay frames + return !cpi->multi_arf_allowed && cpi->rc.is_src_frame_alt_ref && + !cpi->rc.is_src_frame_ext_arf; +#else + return !cpi->multi_arf_allowed && cpi->refresh_golden_frame && + cpi->rc.is_src_frame_alt_ref; +#endif // CONFIG_EXT_REFS +} + +void av1_write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd, +#if CONFIG_SUPERTX + const int supertx_enabled, +#endif +#if CONFIG_TXK_SEL + int block, int plane, +#endif + aom_writer *w); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_BITSTREAM_H_ diff --git a/third_party/aom/av1/encoder/block.h b/third_party/aom/av1/encoder/block.h new file mode 100644 index 0000000000..39e08d5b4c --- /dev/null +++ b/third_party/aom/av1/encoder/block.h @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_BLOCK_H_ +#define AV1_ENCODER_BLOCK_H_ + +#include "av1/common/entropymv.h" +#include "av1/common/entropy.h" +#if CONFIG_PVQ +#include "av1/encoder/encint.h" +#endif +#if CONFIG_REF_MV +#include "av1/common/mvref_common.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#if CONFIG_PVQ +// Maximum possible # of tx blocks in luma plane, which is currently 256, +// since there can be 16x16 of 4x4 tx. +#define MAX_PVQ_BLOCKS_IN_SB (MAX_SB_SQUARE >> 2 * OD_LOG_BSIZE0) +#endif + +typedef struct { + unsigned int sse; + int sum; + unsigned int var; +} DIFF; + +typedef struct macroblock_plane { + DECLARE_ALIGNED(16, int16_t, src_diff[MAX_SB_SQUARE]); +#if CONFIG_PVQ + DECLARE_ALIGNED(16, int16_t, src_int16[MAX_SB_SQUARE]); +#endif + tran_low_t *qcoeff; + tran_low_t *coeff; + uint16_t *eobs; +#if CONFIG_LV_MAP + uint8_t *txb_entropy_ctx; +#endif + struct buf_2d src; + + // Quantizer setings + const int16_t *quant_fp; + const int16_t *round_fp; + const int16_t *quant; + const int16_t *quant_shift; + const int16_t *zbin; + const int16_t *round; +#if CONFIG_NEW_QUANT + const cuml_bins_type_nuq *cuml_bins_nuq[QUANT_PROFILES]; +#endif // CONFIG_NEW_QUANT +} MACROBLOCK_PLANE; + +/* The [2] dimension is for whether we skip the EOB node (i.e. if previous + * coefficient in this block was zero) or not. */ +typedef unsigned int av1_coeff_cost[PLANE_TYPES][REF_TYPES][COEF_BANDS][2] + [COEFF_CONTEXTS][ENTROPY_TOKENS]; + +typedef struct { + int_mv ref_mvs[MODE_CTX_REF_FRAMES][MAX_MV_REF_CANDIDATES]; + int16_t mode_context[MODE_CTX_REF_FRAMES]; +#if CONFIG_LV_MAP + // TODO(angiebird): Reduce the buffer size according to sb_type + tran_low_t tcoeff[MAX_MB_PLANE][MAX_SB_SQUARE]; + uint16_t eobs[MAX_MB_PLANE][MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)]; + uint8_t txb_skip_ctx[MAX_MB_PLANE] + [MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)]; + int dc_sign_ctx[MAX_MB_PLANE] + [MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)]; +#endif +#if CONFIG_REF_MV + uint8_t ref_mv_count[MODE_CTX_REF_FRAMES]; + CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE]; +#if CONFIG_EXT_INTER + int16_t compound_mode_context[MODE_CTX_REF_FRAMES]; +#endif // CONFIG_EXT_INTER +#endif +} MB_MODE_INFO_EXT; + +typedef struct { + int col_min; + int col_max; + int row_min; + int row_max; +} MvLimits; + +#if CONFIG_PALETTE +typedef struct { + uint8_t best_palette_color_map[MAX_SB_SQUARE]; + float kmeans_data_buf[2 * MAX_SB_SQUARE]; +} PALETTE_BUFFER; +#endif // CONFIG_PALETTE + +typedef struct macroblock MACROBLOCK; +struct macroblock { + struct macroblock_plane plane[MAX_MB_PLANE]; + + MACROBLOCKD e_mbd; + MB_MODE_INFO_EXT *mbmi_ext; + int skip_block; + int qindex; + + // The equivalent error at the current rdmult of one whole bit (not one + // bitcost unit). + int errorperbit; + // The equivalend SAD error of one (whole) bit at the current quantizer + // for large blocks. + int sadperbit16; + // The equivalend SAD error of one (whole) bit at the current quantizer + // for sub-8x8 blocks. + int sadperbit4; + int rddiv; + int rdmult; + int mb_energy; + int *m_search_count_ptr; + int *ex_search_count_ptr; + +#if CONFIG_VAR_TX + unsigned int txb_split_count; +#endif + + // These are set to their default values at the beginning, and then adjusted + // further in the encoding process. + BLOCK_SIZE min_partition_size; + BLOCK_SIZE max_partition_size; + + int mv_best_ref_index[TOTAL_REFS_PER_FRAME]; + unsigned int max_mv_context[TOTAL_REFS_PER_FRAME]; + unsigned int source_variance; + unsigned int pred_sse[TOTAL_REFS_PER_FRAME]; + int pred_mv_sad[TOTAL_REFS_PER_FRAME]; + +#if CONFIG_REF_MV + int *nmvjointcost; + int nmv_vec_cost[NMV_CONTEXTS][MV_JOINTS]; + int *nmvcost[NMV_CONTEXTS][2]; + int *nmvcost_hp[NMV_CONTEXTS][2]; + int **mv_cost_stack[NMV_CONTEXTS]; + int *nmvjointsadcost; +#else + int nmvjointcost[MV_JOINTS]; + int *nmvcost[2]; + int *nmvcost_hp[2]; + int nmvjointsadcost[MV_JOINTS]; +#endif + + int **mvcost; + int *nmvsadcost[2]; + int *nmvsadcost_hp[2]; + int **mvsadcost; +#if CONFIG_MOTION_VAR + int32_t *wsrc_buf; + int32_t *mask_buf; +#endif // CONFIG_MOTION_VAR + +#if CONFIG_PALETTE + PALETTE_BUFFER *palette_buffer; +#endif // CONFIG_PALETTE + + // These define limits to motion vector components to prevent them + // from extending outside the UMV borders + MvLimits mv_limits; + +#if CONFIG_VAR_TX + uint8_t blk_skip[MAX_MB_PLANE][MAX_MIB_SIZE * MAX_MIB_SIZE * 8]; +#if CONFIG_REF_MV + uint8_t blk_skip_drl[MAX_MB_PLANE][MAX_MIB_SIZE * MAX_MIB_SIZE * 8]; +#endif +#endif + + int skip; + +#if CONFIG_CB4X4 + int skip_chroma_rd; +#endif + + // note that token_costs is the cost when eob node is skipped + av1_coeff_cost token_costs[TX_SIZES]; + + int optimize; + + // Used to store sub partition's choices. + MV pred_mv[TOTAL_REFS_PER_FRAME]; + + // Store the best motion vector during motion search + int_mv best_mv; + // Store the second best motion vector during full-pixel motion search + int_mv second_best_mv; + + // use default transform and skip transform type search for intra modes + int use_default_intra_tx_type; + // use default transform and skip transform type search for inter modes + int use_default_inter_tx_type; +#if CONFIG_PVQ + int rate; + // 1 if neither AC nor DC is coded. Only used during RDO. + int pvq_skip[MAX_MB_PLANE]; + PVQ_QUEUE *pvq_q; + + // Storage for PVQ tx block encodings in a superblock. + // There can be max 16x16 of 4x4 blocks (and YUV) encode by PVQ + // 256 is the max # of 4x4 blocks in a SB (64x64), which comes from: + // 1) Since PVQ is applied to each trasnform-ed block + // 2) 4x4 is the smallest tx size in AV1 + // 3) AV1 allows using smaller tx size than block (i.e. partition) size + // TODO(yushin) : The memory usage could be improved a lot, since this has + // storage for 10 bands and 128 coefficients for every 4x4 block, + PVQ_INFO pvq[MAX_PVQ_BLOCKS_IN_SB][MAX_MB_PLANE]; + daala_enc_ctx daala_enc; + int pvq_speed; + int pvq_coded; // Indicates whether pvq_info needs be stored to tokenize +#endif +#if CONFIG_DAALA_DIST + // Keep rate of each 4x4 block in the current macroblock during RDO + // This is needed when using the 8x8 Daala distortion metric during RDO, + // because it evaluates distortion in a different order than the underlying + // 4x4 blocks are coded. + int rate_4x4[256]; +#endif +#if CONFIG_CFL + // Whether luma needs to be stored during RDO. + int cfl_store_y; +#endif +}; + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_BLOCK_H_ diff --git a/third_party/aom/av1/encoder/blockiness.c b/third_party/aom/av1/encoder/blockiness.c new file mode 100644 index 0000000000..113ceb29d2 --- /dev/null +++ b/third_party/aom/av1/encoder/blockiness.c @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./av1_rtcd.h" +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "av1/common/common.h" +#include "av1/common/filter.h" +#include "aom/aom_integer.h" +#include "aom_dsp/aom_convolve.h" +#include "aom_dsp/aom_filter.h" +#include "aom_ports/mem.h" +#include "aom_ports/system_state.h" + +static int horizontal_filter(const uint8_t *s) { + return (s[1] - s[-2]) * 2 + (s[-1] - s[0]) * 6; +} + +static int vertical_filter(const uint8_t *s, int p) { + return (s[p] - s[-2 * p]) * 2 + (s[-p] - s[0]) * 6; +} + +static int variance(int sum, int sum_squared, int size) { + return sum_squared / size - (sum / size) * (sum / size); +} +// Calculate a blockiness level for a vertical block edge. +// This function returns a new blockiness metric that's defined as + +// p0 p1 p2 p3 +// q0 q1 q2 q3 +// block edge -> +// r0 r1 r2 r3 +// s0 s1 s2 s3 + +// blockiness = p0*-2+q0*6+r0*-6+s0*2 + +// p1*-2+q1*6+r1*-6+s1*2 + +// p2*-2+q2*6+r2*-6+s2*2 + +// p3*-2+q3*6+r3*-6+s3*2 ; + +// reconstructed_blockiness = abs(blockiness from reconstructed buffer - +// blockiness from source buffer,0) +// +// I make the assumption that flat blocks are much more visible than high +// contrast blocks. As such, I scale the result of the blockiness calc +// by dividing the blockiness by the variance of the pixels on either side +// of the edge as follows: +// var_0 = (q0^2+q1^2+q2^2+q3^2) - ((q0 + q1 + q2 + q3) / 4 )^2 +// var_1 = (r0^2+r1^2+r2^2+r3^2) - ((r0 + r1 + r2 + r3) / 4 )^2 +// The returned blockiness is the scaled value +// Reconstructed blockiness / ( 1 + var_0 + var_1 ) ; +static int blockiness_vertical(const uint8_t *s, int sp, const uint8_t *r, + int rp, int size) { + int s_blockiness = 0; + int r_blockiness = 0; + int sum_0 = 0; + int sum_sq_0 = 0; + int sum_1 = 0; + int sum_sq_1 = 0; + int i; + int var_0; + int var_1; + for (i = 0; i < size; ++i, s += sp, r += rp) { + s_blockiness += horizontal_filter(s); + r_blockiness += horizontal_filter(r); + sum_0 += s[0]; + sum_sq_0 += s[0] * s[0]; + sum_1 += s[-1]; + sum_sq_1 += s[-1] * s[-1]; + } + var_0 = variance(sum_0, sum_sq_0, size); + var_1 = variance(sum_1, sum_sq_1, size); + r_blockiness = abs(r_blockiness); + s_blockiness = abs(s_blockiness); + + if (r_blockiness > s_blockiness) + return (r_blockiness - s_blockiness) / (1 + var_0 + var_1); + else + return 0; +} + +// Calculate a blockiness level for a horizontal block edge +// same as above. +static int blockiness_horizontal(const uint8_t *s, int sp, const uint8_t *r, + int rp, int size) { + int s_blockiness = 0; + int r_blockiness = 0; + int sum_0 = 0; + int sum_sq_0 = 0; + int sum_1 = 0; + int sum_sq_1 = 0; + int i; + int var_0; + int var_1; + for (i = 0; i < size; ++i, ++s, ++r) { + s_blockiness += vertical_filter(s, sp); + r_blockiness += vertical_filter(r, rp); + sum_0 += s[0]; + sum_sq_0 += s[0] * s[0]; + sum_1 += s[-sp]; + sum_sq_1 += s[-sp] * s[-sp]; + } + var_0 = variance(sum_0, sum_sq_0, size); + var_1 = variance(sum_1, sum_sq_1, size); + r_blockiness = abs(r_blockiness); + s_blockiness = abs(s_blockiness); + + if (r_blockiness > s_blockiness) + return (r_blockiness - s_blockiness) / (1 + var_0 + var_1); + else + return 0; +} + +// This function returns the blockiness for the entire frame currently by +// looking at all borders in steps of 4. +double av1_get_blockiness(const unsigned char *img1, int img1_pitch, + const unsigned char *img2, int img2_pitch, int width, + int height) { + double blockiness = 0; + int i, j; + aom_clear_system_state(); + for (i = 0; i < height; + i += 4, img1 += img1_pitch * 4, img2 += img2_pitch * 4) { + for (j = 0; j < width; j += 4) { + if (i > 0 && i < height && j > 0 && j < width) { + blockiness += + blockiness_vertical(img1 + j, img1_pitch, img2 + j, img2_pitch, 4); + blockiness += blockiness_horizontal(img1 + j, img1_pitch, img2 + j, + img2_pitch, 4); + } + } + } + blockiness /= width * height / 16; + return blockiness; +} diff --git a/third_party/aom/av1/encoder/context_tree.c b/third_party/aom/av1/encoder/context_tree.c new file mode 100644 index 0000000000..4c7d6ff00a --- /dev/null +++ b/third_party/aom/av1/encoder/context_tree.c @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/encoder/context_tree.h" +#include "av1/encoder/encoder.h" + +static const BLOCK_SIZE square[MAX_SB_SIZE_LOG2 - 1] = { +#if CONFIG_CB4X4 + BLOCK_4X4, +#endif + BLOCK_8X8, BLOCK_16X16, BLOCK_32X32, BLOCK_64X64, +#if CONFIG_EXT_PARTITION + BLOCK_128X128, +#endif // CONFIG_EXT_PARTITION +}; + +static void alloc_mode_context(AV1_COMMON *cm, int num_4x4_blk, +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_TYPE partition, +#endif + PICK_MODE_CONTEXT *ctx) { + const int num_blk = (num_4x4_blk < 4 ? 4 : num_4x4_blk); + const int num_pix = num_blk * tx_size_2d[0]; + int i; +#if CONFIG_CB4X4 && CONFIG_VAR_TX + ctx->num_4x4_blk = num_blk / 4; +#else + ctx->num_4x4_blk = num_blk; +#endif + +#if CONFIG_EXT_PARTITION_TYPES + ctx->partition = partition; +#endif + + for (i = 0; i < MAX_MB_PLANE; ++i) { +#if CONFIG_VAR_TX + CHECK_MEM_ERROR(cm, ctx->blk_skip[i], aom_calloc(num_blk, sizeof(uint8_t))); +#endif + CHECK_MEM_ERROR(cm, ctx->coeff[i], + aom_memalign(32, num_pix * sizeof(*ctx->coeff[i]))); + CHECK_MEM_ERROR(cm, ctx->qcoeff[i], + aom_memalign(32, num_pix * sizeof(*ctx->qcoeff[i]))); + CHECK_MEM_ERROR(cm, ctx->dqcoeff[i], + aom_memalign(32, num_pix * sizeof(*ctx->dqcoeff[i]))); + CHECK_MEM_ERROR(cm, ctx->eobs[i], + aom_memalign(32, num_blk * sizeof(*ctx->eobs[i]))); +#if CONFIG_LV_MAP + CHECK_MEM_ERROR( + cm, ctx->txb_entropy_ctx[i], + aom_memalign(32, num_blk * sizeof(*ctx->txb_entropy_ctx[i]))); +#endif + +#if CONFIG_PVQ + CHECK_MEM_ERROR(cm, ctx->pvq_ref_coeff[i], + aom_memalign(32, num_pix * sizeof(*ctx->pvq_ref_coeff[i]))); +#endif + } + +#if CONFIG_PALETTE + if (cm->allow_screen_content_tools) { + for (i = 0; i < 2; ++i) { + CHECK_MEM_ERROR( + cm, ctx->color_index_map[i], + aom_memalign(32, num_pix * sizeof(*ctx->color_index_map[i]))); + } + } +#endif // CONFIG_PALETTE +} + +static void free_mode_context(PICK_MODE_CONTEXT *ctx) { + int i; + for (i = 0; i < MAX_MB_PLANE; ++i) { +#if CONFIG_VAR_TX + aom_free(ctx->blk_skip[i]); + ctx->blk_skip[i] = 0; +#endif + aom_free(ctx->coeff[i]); + ctx->coeff[i] = 0; + aom_free(ctx->qcoeff[i]); + ctx->qcoeff[i] = 0; + aom_free(ctx->dqcoeff[i]); + ctx->dqcoeff[i] = 0; +#if CONFIG_PVQ + aom_free(ctx->pvq_ref_coeff[i]); + ctx->pvq_ref_coeff[i] = 0; +#endif + aom_free(ctx->eobs[i]); + ctx->eobs[i] = 0; +#if CONFIG_LV_MAP + aom_free(ctx->txb_entropy_ctx[i]); + ctx->txb_entropy_ctx[i] = 0; +#endif + } + +#if CONFIG_PALETTE + for (i = 0; i < 2; ++i) { + aom_free(ctx->color_index_map[i]); + ctx->color_index_map[i] = 0; + } +#endif // CONFIG_PALETTE +} + +static void alloc_tree_contexts(AV1_COMMON *cm, PC_TREE *tree, + int num_4x4_blk) { +#if CONFIG_EXT_PARTITION_TYPES + alloc_mode_context(cm, num_4x4_blk, PARTITION_NONE, &tree->none); + alloc_mode_context(cm, num_4x4_blk / 2, PARTITION_HORZ, &tree->horizontal[0]); + alloc_mode_context(cm, num_4x4_blk / 2, PARTITION_VERT, &tree->vertical[0]); + alloc_mode_context(cm, num_4x4_blk / 2, PARTITION_VERT, &tree->horizontal[1]); + alloc_mode_context(cm, num_4x4_blk / 2, PARTITION_VERT, &tree->vertical[1]); + + alloc_mode_context(cm, num_4x4_blk / 4, PARTITION_HORZ_A, + &tree->horizontala[0]); + alloc_mode_context(cm, num_4x4_blk / 4, PARTITION_HORZ_A, + &tree->horizontala[1]); + alloc_mode_context(cm, num_4x4_blk / 2, PARTITION_HORZ_A, + &tree->horizontala[2]); + alloc_mode_context(cm, num_4x4_blk / 2, PARTITION_HORZ_B, + &tree->horizontalb[0]); + alloc_mode_context(cm, num_4x4_blk / 4, PARTITION_HORZ_B, + &tree->horizontalb[1]); + alloc_mode_context(cm, num_4x4_blk / 4, PARTITION_HORZ_B, + &tree->horizontalb[2]); + alloc_mode_context(cm, num_4x4_blk / 4, PARTITION_VERT_A, + &tree->verticala[0]); + alloc_mode_context(cm, num_4x4_blk / 4, PARTITION_VERT_A, + &tree->verticala[1]); + alloc_mode_context(cm, num_4x4_blk / 2, PARTITION_VERT_A, + &tree->verticala[2]); + alloc_mode_context(cm, num_4x4_blk / 2, PARTITION_VERT_B, + &tree->verticalb[0]); + alloc_mode_context(cm, num_4x4_blk / 4, PARTITION_VERT_B, + &tree->verticalb[1]); + alloc_mode_context(cm, num_4x4_blk / 4, PARTITION_VERT_B, + &tree->verticalb[2]); +#ifdef CONFIG_SUPERTX + alloc_mode_context(cm, num_4x4_blk, PARTITION_HORZ, + &tree->horizontal_supertx); + alloc_mode_context(cm, num_4x4_blk, PARTITION_VERT, &tree->vertical_supertx); + alloc_mode_context(cm, num_4x4_blk, PARTITION_SPLIT, &tree->split_supertx); + alloc_mode_context(cm, num_4x4_blk, PARTITION_HORZ_A, + &tree->horizontala_supertx); + alloc_mode_context(cm, num_4x4_blk, PARTITION_HORZ_B, + &tree->horizontalb_supertx); + alloc_mode_context(cm, num_4x4_blk, PARTITION_VERT_A, + &tree->verticala_supertx); + alloc_mode_context(cm, num_4x4_blk, PARTITION_VERT_B, + &tree->verticalb_supertx); +#endif // CONFIG_SUPERTX +#else + alloc_mode_context(cm, num_4x4_blk, &tree->none); + alloc_mode_context(cm, num_4x4_blk / 2, &tree->horizontal[0]); + alloc_mode_context(cm, num_4x4_blk / 2, &tree->vertical[0]); +#ifdef CONFIG_SUPERTX + alloc_mode_context(cm, num_4x4_blk, &tree->horizontal_supertx); + alloc_mode_context(cm, num_4x4_blk, &tree->vertical_supertx); + alloc_mode_context(cm, num_4x4_blk, &tree->split_supertx); +#endif + + if (num_4x4_blk > 4) { + alloc_mode_context(cm, num_4x4_blk / 2, &tree->horizontal[1]); + alloc_mode_context(cm, num_4x4_blk / 2, &tree->vertical[1]); + } else { + memset(&tree->horizontal[1], 0, sizeof(tree->horizontal[1])); + memset(&tree->vertical[1], 0, sizeof(tree->vertical[1])); + } +#endif // CONFIG_EXT_PARTITION_TYPES +} + +static void free_tree_contexts(PC_TREE *tree) { +#if CONFIG_EXT_PARTITION_TYPES + int i; + for (i = 0; i < 3; i++) { + free_mode_context(&tree->horizontala[i]); + free_mode_context(&tree->horizontalb[i]); + free_mode_context(&tree->verticala[i]); + free_mode_context(&tree->verticalb[i]); + } +#endif // CONFIG_EXT_PARTITION_TYPES + free_mode_context(&tree->none); + free_mode_context(&tree->horizontal[0]); + free_mode_context(&tree->horizontal[1]); + free_mode_context(&tree->vertical[0]); + free_mode_context(&tree->vertical[1]); +#ifdef CONFIG_SUPERTX + free_mode_context(&tree->horizontal_supertx); + free_mode_context(&tree->vertical_supertx); + free_mode_context(&tree->split_supertx); +#if CONFIG_EXT_PARTITION_TYPES + free_mode_context(&tree->horizontala_supertx); + free_mode_context(&tree->horizontalb_supertx); + free_mode_context(&tree->verticala_supertx); + free_mode_context(&tree->verticalb_supertx); +#endif // CONFIG_EXT_PARTITION_TYPES +#endif // CONFIG_SUPERTX +} + +// This function sets up a tree of contexts such that at each square +// partition level. There are contexts for none, horizontal, vertical, and +// split. Along with a block_size value and a selected block_size which +// represents the state of our search. +void av1_setup_pc_tree(AV1_COMMON *cm, ThreadData *td) { + int i, j; +// TODO(jingning): The pc_tree allocation is redundant. We can take out all +// the leaf nodes after cb4x4 mode is enabled. +#if CONFIG_CB4X4 +#if CONFIG_EXT_PARTITION + const int tree_nodes_inc = 1024; +#else + const int tree_nodes_inc = 256; +#endif // CONFIG_EXT_PARTITION + const int leaf_factor = 4; +#else + const int tree_nodes_inc = 0; + const int leaf_factor = 1; +#endif +#if CONFIG_EXT_PARTITION + const int leaf_nodes = 256 * leaf_factor; + const int tree_nodes = tree_nodes_inc + 256 + 64 + 16 + 4 + 1; +#else + const int leaf_nodes = 64 * leaf_factor; + const int tree_nodes = tree_nodes_inc + 64 + 16 + 4 + 1; +#endif // CONFIG_EXT_PARTITION + int pc_tree_index = 0; + PC_TREE *this_pc; + PICK_MODE_CONTEXT *this_leaf; + int square_index = 1; + int nodes; + + aom_free(td->leaf_tree); + CHECK_MEM_ERROR(cm, td->leaf_tree, + aom_calloc(leaf_nodes, sizeof(*td->leaf_tree))); + aom_free(td->pc_tree); + CHECK_MEM_ERROR(cm, td->pc_tree, + aom_calloc(tree_nodes, sizeof(*td->pc_tree))); + + this_pc = &td->pc_tree[0]; + this_leaf = &td->leaf_tree[0]; + + // 4x4 blocks smaller than 8x8 but in the same 8x8 block share the same + // context so we only need to allocate 1 for each 8x8 block. + for (i = 0; i < leaf_nodes; ++i) { +#if CONFIG_EXT_PARTITION_TYPES + alloc_mode_context(cm, 4, PARTITION_NONE, &td->leaf_tree[i]); +#else + alloc_mode_context(cm, 16, &td->leaf_tree[i]); +#endif + } + + // Sets up all the leaf nodes in the tree. + for (pc_tree_index = 0; pc_tree_index < leaf_nodes; ++pc_tree_index) { + PC_TREE *const tree = &td->pc_tree[pc_tree_index]; + tree->block_size = square[0]; +#if CONFIG_CB4X4 + alloc_tree_contexts(cm, tree, 16); +#else + alloc_tree_contexts(cm, tree, 4); +#endif + tree->leaf_split[0] = this_leaf++; + for (j = 1; j < 4; j++) tree->leaf_split[j] = tree->leaf_split[0]; + } + + // Each node has 4 leaf nodes, fill each block_size level of the tree + // from leafs to the root. + for (nodes = leaf_nodes >> 2; nodes > 0; nodes >>= 2) { + for (i = 0; i < nodes; ++i) { + PC_TREE *const tree = &td->pc_tree[pc_tree_index]; +#if CONFIG_CB4X4 + alloc_tree_contexts(cm, tree, 16 << (2 * square_index)); +#else + alloc_tree_contexts(cm, tree, 4 << (2 * square_index)); +#endif + tree->block_size = square[square_index]; + for (j = 0; j < 4; j++) tree->split[j] = this_pc++; + ++pc_tree_index; + } + ++square_index; + } + + // Set up the root node for the largest superblock size + i = MAX_MIB_SIZE_LOG2 - MIN_MIB_SIZE_LOG2; + td->pc_root[i] = &td->pc_tree[tree_nodes - 1]; + td->pc_root[i]->none.best_mode_index = 2; + // Set up the root nodes for the rest of the possible superblock sizes + while (--i >= 0) { + td->pc_root[i] = td->pc_root[i + 1]->split[0]; + td->pc_root[i]->none.best_mode_index = 2; + } +} + +void av1_free_pc_tree(ThreadData *td) { +#if CONFIG_CB4X4 +#if CONFIG_EXT_PARTITION + const int tree_nodes_inc = 1024; +#else + const int tree_nodes_inc = 256; +#endif // CONFIG_EXT_PARTITION + const int leaf_factor = 4; +#else + const int tree_nodes_inc = 0; + const int leaf_factor = 1; +#endif + +#if CONFIG_EXT_PARTITION + const int leaf_nodes = 256 * leaf_factor; + const int tree_nodes = tree_nodes_inc + 256 + 64 + 16 + 4 + 1; +#else + const int leaf_nodes = 64 * leaf_factor; + const int tree_nodes = tree_nodes_inc + 64 + 16 + 4 + 1; +#endif // CONFIG_EXT_PARTITION + int i; + + // Set up all 4x4 mode contexts + for (i = 0; i < leaf_nodes; ++i) free_mode_context(&td->leaf_tree[i]); + + // Sets up all the leaf nodes in the tree. + for (i = 0; i < tree_nodes; ++i) free_tree_contexts(&td->pc_tree[i]); + + aom_free(td->pc_tree); + td->pc_tree = NULL; + aom_free(td->leaf_tree); + td->leaf_tree = NULL; +} diff --git a/third_party/aom/av1/encoder/context_tree.h b/third_party/aom/av1/encoder/context_tree.h new file mode 100644 index 0000000000..67954126c6 --- /dev/null +++ b/third_party/aom/av1/encoder/context_tree.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_CONTEXT_TREE_H_ +#define AV1_ENCODER_CONTEXT_TREE_H_ + +#include "av1/common/blockd.h" +#include "av1/encoder/block.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct AV1_COMP; +struct AV1Common; +struct ThreadData; + +// Structure to hold snapshot of coding context during the mode picking process +typedef struct { + MODE_INFO mic; + MB_MODE_INFO_EXT mbmi_ext; +#if CONFIG_PALETTE + uint8_t *color_index_map[2]; +#endif // CONFIG_PALETTE +#if CONFIG_VAR_TX + uint8_t *blk_skip[MAX_MB_PLANE]; +#endif + + // dual buffer pointers, 0: in use, 1: best in store + tran_low_t *coeff[MAX_MB_PLANE]; + tran_low_t *qcoeff[MAX_MB_PLANE]; + tran_low_t *dqcoeff[MAX_MB_PLANE]; +#if CONFIG_PVQ + tran_low_t *pvq_ref_coeff[MAX_MB_PLANE]; +#endif + uint16_t *eobs[MAX_MB_PLANE]; +#if CONFIG_LV_MAP + uint8_t *txb_entropy_ctx[MAX_MB_PLANE]; +#endif + + int num_4x4_blk; + int skip; + int pred_pixel_ready; + // For current partition, only if all Y, U, and V transform blocks' + // coefficients are quantized to 0, skippable is set to 0. + int skippable; + int best_mode_index; + int hybrid_pred_diff; + int comp_pred_diff; + int single_pred_diff; + + // TODO(jingning) Use RD_COST struct here instead. This involves a boarder + // scope of refactoring. + int rate; + int64_t dist; + + // motion vector cache for adaptive motion search control in partition + // search loop + MV pred_mv[TOTAL_REFS_PER_FRAME]; + InterpFilter pred_interp_filter; +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_TYPE partition; +#endif +} PICK_MODE_CONTEXT; + +typedef struct PC_TREE { + int index; + PARTITION_TYPE partitioning; + BLOCK_SIZE block_size; + PICK_MODE_CONTEXT none; + PICK_MODE_CONTEXT horizontal[2]; + PICK_MODE_CONTEXT vertical[2]; +#if CONFIG_EXT_PARTITION_TYPES + PICK_MODE_CONTEXT horizontala[3]; + PICK_MODE_CONTEXT horizontalb[3]; + PICK_MODE_CONTEXT verticala[3]; + PICK_MODE_CONTEXT verticalb[3]; +#endif + union { + struct PC_TREE *split[4]; + PICK_MODE_CONTEXT *leaf_split[4]; + }; +#ifdef CONFIG_SUPERTX + PICK_MODE_CONTEXT horizontal_supertx; + PICK_MODE_CONTEXT vertical_supertx; + PICK_MODE_CONTEXT split_supertx; +#if CONFIG_EXT_PARTITION_TYPES + PICK_MODE_CONTEXT horizontala_supertx; + PICK_MODE_CONTEXT horizontalb_supertx; + PICK_MODE_CONTEXT verticala_supertx; + PICK_MODE_CONTEXT verticalb_supertx; +#endif +#endif +} PC_TREE; + +void av1_setup_pc_tree(struct AV1Common *cm, struct ThreadData *td); +void av1_free_pc_tree(struct ThreadData *td); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* AV1_ENCODER_CONTEXT_TREE_H_ */ diff --git a/third_party/aom/av1/encoder/corner_detect.c b/third_party/aom/av1/encoder/corner_detect.c new file mode 100644 index 0000000000..e4c59dd9c6 --- /dev/null +++ b/third_party/aom/av1/encoder/corner_detect.c @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include +#include + +#include "third_party/fastfeat/fast.h" + +#include "av1/encoder/corner_detect.h" + +// Fast_9 wrapper +#define FAST_BARRIER 18 +int fast_corner_detect(unsigned char *buf, int width, int height, int stride, + int *points, int max_points) { + int num_points; + xy *const frm_corners_xy = fast9_detect_nonmax(buf, width, height, stride, + FAST_BARRIER, &num_points); + num_points = (num_points <= max_points ? num_points : max_points); + if (num_points > 0 && frm_corners_xy) { + memcpy(points, frm_corners_xy, sizeof(*frm_corners_xy) * num_points); + free(frm_corners_xy); + return num_points; + } + free(frm_corners_xy); + return 0; +} diff --git a/third_party/aom/av1/encoder/corner_detect.h b/third_party/aom/av1/encoder/corner_detect.h new file mode 100644 index 0000000000..0317db5b31 --- /dev/null +++ b/third_party/aom/av1/encoder/corner_detect.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_CORNER_DETECT_H_ +#define AV1_ENCODER_CORNER_DETECT_H_ + +#include +#include +#include + +int fast_corner_detect(unsigned char *buf, int width, int height, int stride, + int *points, int max_points); + +#endif // AV1_ENCODER_CORNER_DETECT_H_ diff --git a/third_party/aom/av1/encoder/corner_match.c b/third_party/aom/av1/encoder/corner_match.c new file mode 100644 index 0000000000..64ee0c5ae1 --- /dev/null +++ b/third_party/aom/av1/encoder/corner_match.c @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include + +#include "av1/encoder/corner_match.h" + +#define MATCH_SZ 13 +#define MATCH_SZ_BY2 ((MATCH_SZ - 1) / 2) +#define MATCH_SZ_SQ (MATCH_SZ * MATCH_SZ) +#define SEARCH_SZ 9 +#define SEARCH_SZ_BY2 ((SEARCH_SZ - 1) / 2) + +#define THRESHOLD_NCC 0.75 + +/* Compute var(im) * MATCH_SZ_SQ over a MATCH_SZ by MATCH_SZ window of im, + centered at (x, y). +*/ +static double compute_variance(unsigned char *im, int stride, int x, int y) { + int sum = 0.0; + int sumsq = 0.0; + int var; + int i, j; + for (i = 0; i < MATCH_SZ; ++i) + for (j = 0; j < MATCH_SZ; ++j) { + sum += im[(i + y - MATCH_SZ_BY2) * stride + (j + x - MATCH_SZ_BY2)]; + sumsq += im[(i + y - MATCH_SZ_BY2) * stride + (j + x - MATCH_SZ_BY2)] * + im[(i + y - MATCH_SZ_BY2) * stride + (j + x - MATCH_SZ_BY2)]; + } + var = sumsq * MATCH_SZ_SQ - sum * sum; + return (double)var; +} + +/* Compute corr(im1, im2) * MATCH_SZ * stddev(im1), where the + correlation/standard deviation are taken over MATCH_SZ by MATCH_SZ windows + of each image, centered at (x1, y1) and (x2, y2) respectively. +*/ +static double compute_cross_correlation(unsigned char *im1, int stride1, int x1, + int y1, unsigned char *im2, int stride2, + int x2, int y2) { + int v1, v2; + int sum1 = 0; + int sum2 = 0; + int sumsq2 = 0; + int cross = 0; + int var2, cov; + int i, j; + for (i = 0; i < MATCH_SZ; ++i) + for (j = 0; j < MATCH_SZ; ++j) { + v1 = im1[(i + y1 - MATCH_SZ_BY2) * stride1 + (j + x1 - MATCH_SZ_BY2)]; + v2 = im2[(i + y2 - MATCH_SZ_BY2) * stride2 + (j + x2 - MATCH_SZ_BY2)]; + sum1 += v1; + sum2 += v2; + sumsq2 += v2 * v2; + cross += v1 * v2; + } + var2 = sumsq2 * MATCH_SZ_SQ - sum2 * sum2; + cov = cross * MATCH_SZ_SQ - sum1 * sum2; + return cov / sqrt((double)var2); +} + +static int is_eligible_point(int pointx, int pointy, int width, int height) { + return (pointx >= MATCH_SZ_BY2 && pointy >= MATCH_SZ_BY2 && + pointx + MATCH_SZ_BY2 < width && pointy + MATCH_SZ_BY2 < height); +} + +static int is_eligible_distance(int point1x, int point1y, int point2x, + int point2y, int width, int height) { + const int thresh = (width < height ? height : width) >> 4; + return ((point1x - point2x) * (point1x - point2x) + + (point1y - point2y) * (point1y - point2y)) <= thresh * thresh; +} + +static void improve_correspondence(unsigned char *frm, unsigned char *ref, + int width, int height, int frm_stride, + int ref_stride, + Correspondence *correspondences, + int num_correspondences) { + int i; + for (i = 0; i < num_correspondences; ++i) { + int x, y, best_x = 0, best_y = 0; + double best_match_ncc = 0.0; + for (y = -SEARCH_SZ_BY2; y <= SEARCH_SZ_BY2; ++y) { + for (x = -SEARCH_SZ_BY2; x <= SEARCH_SZ_BY2; ++x) { + double match_ncc; + if (!is_eligible_point(correspondences[i].rx + x, + correspondences[i].ry + y, width, height)) + continue; + if (!is_eligible_distance(correspondences[i].x, correspondences[i].y, + correspondences[i].rx + x, + correspondences[i].ry + y, width, height)) + continue; + match_ncc = compute_cross_correlation( + frm, frm_stride, correspondences[i].x, correspondences[i].y, ref, + ref_stride, correspondences[i].rx + x, correspondences[i].ry + y); + if (match_ncc > best_match_ncc) { + best_match_ncc = match_ncc; + best_y = y; + best_x = x; + } + } + } + correspondences[i].rx += best_x; + correspondences[i].ry += best_y; + } + for (i = 0; i < num_correspondences; ++i) { + int x, y, best_x = 0, best_y = 0; + double best_match_ncc = 0.0; + for (y = -SEARCH_SZ_BY2; y <= SEARCH_SZ_BY2; ++y) + for (x = -SEARCH_SZ_BY2; x <= SEARCH_SZ_BY2; ++x) { + double match_ncc; + if (!is_eligible_point(correspondences[i].x + x, + correspondences[i].y + y, width, height)) + continue; + if (!is_eligible_distance( + correspondences[i].x + x, correspondences[i].y + y, + correspondences[i].rx, correspondences[i].ry, width, height)) + continue; + match_ncc = compute_cross_correlation( + ref, ref_stride, correspondences[i].rx, correspondences[i].ry, frm, + frm_stride, correspondences[i].x + x, correspondences[i].y + y); + if (match_ncc > best_match_ncc) { + best_match_ncc = match_ncc; + best_y = y; + best_x = x; + } + } + correspondences[i].x += best_x; + correspondences[i].y += best_y; + } +} + +int determine_correspondence(unsigned char *frm, int *frm_corners, + int num_frm_corners, unsigned char *ref, + int *ref_corners, int num_ref_corners, int width, + int height, int frm_stride, int ref_stride, + int *correspondence_pts) { + // TODO(sarahparker) Improve this to include 2-way match + int i, j; + Correspondence *correspondences = (Correspondence *)correspondence_pts; + int num_correspondences = 0; + for (i = 0; i < num_frm_corners; ++i) { + double best_match_ncc = 0.0; + double template_norm; + int best_match_j = -1; + if (!is_eligible_point(frm_corners[2 * i], frm_corners[2 * i + 1], width, + height)) + continue; + for (j = 0; j < num_ref_corners; ++j) { + double match_ncc; + if (!is_eligible_point(ref_corners[2 * j], ref_corners[2 * j + 1], width, + height)) + continue; + if (!is_eligible_distance(frm_corners[2 * i], frm_corners[2 * i + 1], + ref_corners[2 * j], ref_corners[2 * j + 1], + width, height)) + continue; + match_ncc = compute_cross_correlation( + frm, frm_stride, frm_corners[2 * i], frm_corners[2 * i + 1], ref, + ref_stride, ref_corners[2 * j], ref_corners[2 * j + 1]); + if (match_ncc > best_match_ncc) { + best_match_ncc = match_ncc; + best_match_j = j; + } + } + // Note: We want to test if the best correlation is >= THRESHOLD_NCC, + // but need to account for the normalization in compute_cross_correlation. + template_norm = compute_variance(frm, frm_stride, frm_corners[2 * i], + frm_corners[2 * i + 1]); + if (best_match_ncc > THRESHOLD_NCC * sqrt(template_norm)) { + correspondences[num_correspondences].x = frm_corners[2 * i]; + correspondences[num_correspondences].y = frm_corners[2 * i + 1]; + correspondences[num_correspondences].rx = ref_corners[2 * best_match_j]; + correspondences[num_correspondences].ry = + ref_corners[2 * best_match_j + 1]; + num_correspondences++; + } + } + improve_correspondence(frm, ref, width, height, frm_stride, ref_stride, + correspondences, num_correspondences); + return num_correspondences; +} diff --git a/third_party/aom/av1/encoder/corner_match.h b/third_party/aom/av1/encoder/corner_match.h new file mode 100644 index 0000000000..c0458642c1 --- /dev/null +++ b/third_party/aom/av1/encoder/corner_match.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AV1_ENCODER_CORNER_MATCH_H_ +#define AV1_ENCODER_CORNER_MATCH_H_ + +#include +#include +#include + +typedef struct { + int x, y; + int rx, ry; +} Correspondence; + +int determine_correspondence(unsigned char *frm, int *frm_corners, + int num_frm_corners, unsigned char *ref, + int *ref_corners, int num_ref_corners, int width, + int height, int frm_stride, int ref_stride, + int *correspondence_pts); + +#endif // AV1_ENCODER_CORNER_MATCH_H_ diff --git a/third_party/aom/av1/encoder/cost.c b/third_party/aom/av1/encoder/cost.c new file mode 100644 index 0000000000..e3151a5973 --- /dev/null +++ b/third_party/aom/av1/encoder/cost.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include + +#include "av1/encoder/cost.h" +#include "av1/common/entropy.h" + +/* round(-log2(i/256.) * (1 << AV1_PROB_COST_SHIFT)) + Begins with a bogus entry for simpler addressing. */ +const uint16_t av1_prob_cost[256] = { + 4096, 4096, 3584, 3284, 3072, 2907, 2772, 2659, 2560, 2473, 2395, 2325, 2260, + 2201, 2147, 2096, 2048, 2003, 1961, 1921, 1883, 1847, 1813, 1780, 1748, 1718, + 1689, 1661, 1635, 1609, 1584, 1559, 1536, 1513, 1491, 1470, 1449, 1429, 1409, + 1390, 1371, 1353, 1335, 1318, 1301, 1284, 1268, 1252, 1236, 1221, 1206, 1192, + 1177, 1163, 1149, 1136, 1123, 1110, 1097, 1084, 1072, 1059, 1047, 1036, 1024, + 1013, 1001, 990, 979, 968, 958, 947, 937, 927, 917, 907, 897, 887, + 878, 868, 859, 850, 841, 832, 823, 814, 806, 797, 789, 780, 772, + 764, 756, 748, 740, 732, 724, 717, 709, 702, 694, 687, 680, 673, + 665, 658, 651, 644, 637, 631, 624, 617, 611, 604, 598, 591, 585, + 578, 572, 566, 560, 554, 547, 541, 535, 530, 524, 518, 512, 506, + 501, 495, 489, 484, 478, 473, 467, 462, 456, 451, 446, 441, 435, + 430, 425, 420, 415, 410, 405, 400, 395, 390, 385, 380, 375, 371, + 366, 361, 356, 352, 347, 343, 338, 333, 329, 324, 320, 316, 311, + 307, 302, 298, 294, 289, 285, 281, 277, 273, 268, 264, 260, 256, + 252, 248, 244, 240, 236, 232, 228, 224, 220, 216, 212, 209, 205, + 201, 197, 194, 190, 186, 182, 179, 175, 171, 168, 164, 161, 157, + 153, 150, 146, 143, 139, 136, 132, 129, 125, 122, 119, 115, 112, + 109, 105, 102, 99, 95, 92, 89, 86, 82, 79, 76, 73, 70, + 66, 63, 60, 57, 54, 51, 48, 45, 42, 38, 35, 32, 29, + 26, 23, 20, 18, 15, 12, 9, 6, 3 +}; + +static void cost(int *costs, aom_tree tree, const aom_prob *probs, int i, + int c) { + const aom_prob prob = probs[i / 2]; + int b; + + assert(prob != 0); + for (b = 0; b <= 1; ++b) { + const int cc = c + av1_cost_bit(prob, b); + const aom_tree_index ii = tree[i + b]; + + if (ii <= 0) + costs[-ii] = cc; + else + cost(costs, tree, probs, ii, cc); + } +} + +void av1_cost_tokens(int *costs, const aom_prob *probs, aom_tree tree) { + cost(costs, tree, probs, 0, 0); +} + +void av1_cost_tokens_skip(int *costs, const aom_prob *probs, aom_tree tree) { + assert(tree[0] <= 0 && tree[1] > 0); + + costs[-tree[0]] = av1_cost_bit(probs[0], 0); + cost(costs, tree, probs, 2, 0); +} diff --git a/third_party/aom/av1/encoder/cost.h b/third_party/aom/av1/encoder/cost.h new file mode 100644 index 0000000000..d8fb357e6d --- /dev/null +++ b/third_party/aom/av1/encoder/cost.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_COST_H_ +#define AV1_ENCODER_COST_H_ + +#include "aom_dsp/prob.h" +#include "aom/aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern const uint16_t av1_prob_cost[256]; + +// The factor to scale from cost in bits to cost in av1_prob_cost units. +#define AV1_PROB_COST_SHIFT 9 + +#define av1_cost_zero(prob) (av1_prob_cost[prob]) + +#define av1_cost_one(prob) av1_cost_zero(256 - (prob)) + +#define av1_cost_bit(prob, bit) av1_cost_zero((bit) ? 256 - (prob) : (prob)) + +// Cost of coding an n bit literal, using 128 (i.e. 50%) probability +// for each bit. +#define av1_cost_literal(n) ((n) * (1 << AV1_PROB_COST_SHIFT)) + +static INLINE unsigned int cost_branch256(const unsigned int ct[2], + aom_prob p) { + return ct[0] * av1_cost_zero(p) + ct[1] * av1_cost_one(p); +} + +static INLINE int treed_cost(aom_tree tree, const aom_prob *probs, int bits, + int len) { + int cost = 0; + aom_tree_index i = 0; + + do { + const int bit = (bits >> --len) & 1; + cost += av1_cost_bit(probs[i >> 1], bit); + i = tree[i + bit]; + } while (len); + + return cost; +} + +void av1_cost_tokens(int *costs, const aom_prob *probs, aom_tree tree); +void av1_cost_tokens_skip(int *costs, const aom_prob *probs, aom_tree tree); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_COST_H_ diff --git a/third_party/aom/av1/encoder/daala_compat_enc.c b/third_party/aom/av1/encoder/daala_compat_enc.c new file mode 100644 index 0000000000..3df424cac2 --- /dev/null +++ b/third_party/aom/av1/encoder/daala_compat_enc.c @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "encint.h" + +void od_encode_checkpoint(const daala_enc_ctx *enc, od_rollback_buffer *rbuf) { +#if CONFIG_DAALA_EC + od_ec_enc_checkpoint(&rbuf->ec, &enc->w.ec); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + OD_COPY(&rbuf->adapt, enc->state.adapt, 1); +} + +void od_encode_rollback(daala_enc_ctx *enc, const od_rollback_buffer *rbuf) { +#if CONFIG_DAALA_EC + od_ec_enc_rollback(&enc->w.ec, &rbuf->ec); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + OD_COPY(enc->state.adapt, &rbuf->adapt, 1); +} diff --git a/third_party/aom/av1/encoder/dct.c b/third_party/aom/av1/encoder/dct.c new file mode 100644 index 0000000000..09e1b05637 --- /dev/null +++ b/third_party/aom/av1/encoder/dct.c @@ -0,0 +1,2228 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" +#include "aom_dsp/fwd_txfm.h" +#include "aom_ports/mem.h" +#include "av1/common/blockd.h" +#include "av1/common/av1_fwd_txfm1d.h" +#include "av1/common/av1_fwd_txfm2d_cfg.h" +#include "av1/common/idct.h" + +static INLINE void range_check(const tran_low_t *input, const int size, + const int bit) { +#if 0 // CONFIG_COEFFICIENT_RANGE_CHECKING +// TODO(angiebird): the range_check is not used because the bit range +// in fdct# is not correct. Since we are going to merge in a new version +// of fdct# from nextgenv2, we won't fix the incorrect bit range now. + int i; + for (i = 0; i < size; ++i) { + assert(abs(input[i]) < (1 << bit)); + } +#else + (void)input; + (void)size; + (void)bit; +#endif +} + +static void fdct4(const tran_low_t *input, tran_low_t *output) { + tran_high_t temp; + tran_low_t step[4]; + + // stage 0 + range_check(input, 4, 14); + + // stage 1 + output[0] = input[0] + input[3]; + output[1] = input[1] + input[2]; + output[2] = input[1] - input[2]; + output[3] = input[0] - input[3]; + + range_check(output, 4, 15); + + // stage 2 + temp = output[0] * cospi_16_64 + output[1] * cospi_16_64; + step[0] = (tran_low_t)fdct_round_shift(temp); + temp = output[1] * -cospi_16_64 + output[0] * cospi_16_64; + step[1] = (tran_low_t)fdct_round_shift(temp); + temp = output[2] * cospi_24_64 + output[3] * cospi_8_64; + step[2] = (tran_low_t)fdct_round_shift(temp); + temp = output[3] * cospi_24_64 + output[2] * -cospi_8_64; + step[3] = (tran_low_t)fdct_round_shift(temp); + + range_check(step, 4, 16); + + // stage 3 + output[0] = step[0]; + output[1] = step[2]; + output[2] = step[1]; + output[3] = step[3]; + + range_check(output, 4, 16); +} + +static void fdct8(const tran_low_t *input, tran_low_t *output) { + tran_high_t temp; + tran_low_t step[8]; + + // stage 0 + range_check(input, 8, 13); + + // stage 1 + output[0] = input[0] + input[7]; + output[1] = input[1] + input[6]; + output[2] = input[2] + input[5]; + output[3] = input[3] + input[4]; + output[4] = input[3] - input[4]; + output[5] = input[2] - input[5]; + output[6] = input[1] - input[6]; + output[7] = input[0] - input[7]; + + range_check(output, 8, 14); + + // stage 2 + step[0] = output[0] + output[3]; + step[1] = output[1] + output[2]; + step[2] = output[1] - output[2]; + step[3] = output[0] - output[3]; + step[4] = output[4]; + temp = output[5] * -cospi_16_64 + output[6] * cospi_16_64; + step[5] = (tran_low_t)fdct_round_shift(temp); + temp = output[6] * cospi_16_64 + output[5] * cospi_16_64; + step[6] = (tran_low_t)fdct_round_shift(temp); + step[7] = output[7]; + + range_check(step, 8, 15); + + // stage 3 + temp = step[0] * cospi_16_64 + step[1] * cospi_16_64; + output[0] = (tran_low_t)fdct_round_shift(temp); + temp = step[1] * -cospi_16_64 + step[0] * cospi_16_64; + output[1] = (tran_low_t)fdct_round_shift(temp); + temp = step[2] * cospi_24_64 + step[3] * cospi_8_64; + output[2] = (tran_low_t)fdct_round_shift(temp); + temp = step[3] * cospi_24_64 + step[2] * -cospi_8_64; + output[3] = (tran_low_t)fdct_round_shift(temp); + output[4] = step[4] + step[5]; + output[5] = step[4] - step[5]; + output[6] = step[7] - step[6]; + output[7] = step[7] + step[6]; + + range_check(output, 8, 16); + + // stage 4 + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + temp = output[4] * cospi_28_64 + output[7] * cospi_4_64; + step[4] = (tran_low_t)fdct_round_shift(temp); + temp = output[5] * cospi_12_64 + output[6] * cospi_20_64; + step[5] = (tran_low_t)fdct_round_shift(temp); + temp = output[6] * cospi_12_64 + output[5] * -cospi_20_64; + step[6] = (tran_low_t)fdct_round_shift(temp); + temp = output[7] * cospi_28_64 + output[4] * -cospi_4_64; + step[7] = (tran_low_t)fdct_round_shift(temp); + + range_check(step, 8, 16); + + // stage 5 + output[0] = step[0]; + output[1] = step[4]; + output[2] = step[2]; + output[3] = step[6]; + output[4] = step[1]; + output[5] = step[5]; + output[6] = step[3]; + output[7] = step[7]; + + range_check(output, 8, 16); +} + +static void fdct16(const tran_low_t *input, tran_low_t *output) { + tran_high_t temp; + tran_low_t step[16]; + + // stage 0 + range_check(input, 16, 13); + + // stage 1 + output[0] = input[0] + input[15]; + output[1] = input[1] + input[14]; + output[2] = input[2] + input[13]; + output[3] = input[3] + input[12]; + output[4] = input[4] + input[11]; + output[5] = input[5] + input[10]; + output[6] = input[6] + input[9]; + output[7] = input[7] + input[8]; + output[8] = input[7] - input[8]; + output[9] = input[6] - input[9]; + output[10] = input[5] - input[10]; + output[11] = input[4] - input[11]; + output[12] = input[3] - input[12]; + output[13] = input[2] - input[13]; + output[14] = input[1] - input[14]; + output[15] = input[0] - input[15]; + + range_check(output, 16, 14); + + // stage 2 + step[0] = output[0] + output[7]; + step[1] = output[1] + output[6]; + step[2] = output[2] + output[5]; + step[3] = output[3] + output[4]; + step[4] = output[3] - output[4]; + step[5] = output[2] - output[5]; + step[6] = output[1] - output[6]; + step[7] = output[0] - output[7]; + step[8] = output[8]; + step[9] = output[9]; + temp = output[10] * -cospi_16_64 + output[13] * cospi_16_64; + step[10] = (tran_low_t)fdct_round_shift(temp); + temp = output[11] * -cospi_16_64 + output[12] * cospi_16_64; + step[11] = (tran_low_t)fdct_round_shift(temp); + temp = output[12] * cospi_16_64 + output[11] * cospi_16_64; + step[12] = (tran_low_t)fdct_round_shift(temp); + temp = output[13] * cospi_16_64 + output[10] * cospi_16_64; + step[13] = (tran_low_t)fdct_round_shift(temp); + step[14] = output[14]; + step[15] = output[15]; + + range_check(step, 16, 15); + + // stage 3 + output[0] = step[0] + step[3]; + output[1] = step[1] + step[2]; + output[2] = step[1] - step[2]; + output[3] = step[0] - step[3]; + output[4] = step[4]; + temp = step[5] * -cospi_16_64 + step[6] * cospi_16_64; + output[5] = (tran_low_t)fdct_round_shift(temp); + temp = step[6] * cospi_16_64 + step[5] * cospi_16_64; + output[6] = (tran_low_t)fdct_round_shift(temp); + output[7] = step[7]; + output[8] = step[8] + step[11]; + output[9] = step[9] + step[10]; + output[10] = step[9] - step[10]; + output[11] = step[8] - step[11]; + output[12] = step[15] - step[12]; + output[13] = step[14] - step[13]; + output[14] = step[14] + step[13]; + output[15] = step[15] + step[12]; + + range_check(output, 16, 16); + + // stage 4 + temp = output[0] * cospi_16_64 + output[1] * cospi_16_64; + step[0] = (tran_low_t)fdct_round_shift(temp); + temp = output[1] * -cospi_16_64 + output[0] * cospi_16_64; + step[1] = (tran_low_t)fdct_round_shift(temp); + temp = output[2] * cospi_24_64 + output[3] * cospi_8_64; + step[2] = (tran_low_t)fdct_round_shift(temp); + temp = output[3] * cospi_24_64 + output[2] * -cospi_8_64; + step[3] = (tran_low_t)fdct_round_shift(temp); + step[4] = output[4] + output[5]; + step[5] = output[4] - output[5]; + step[6] = output[7] - output[6]; + step[7] = output[7] + output[6]; + step[8] = output[8]; + temp = output[9] * -cospi_8_64 + output[14] * cospi_24_64; + step[9] = (tran_low_t)fdct_round_shift(temp); + temp = output[10] * -cospi_24_64 + output[13] * -cospi_8_64; + step[10] = (tran_low_t)fdct_round_shift(temp); + step[11] = output[11]; + step[12] = output[12]; + temp = output[13] * cospi_24_64 + output[10] * -cospi_8_64; + step[13] = (tran_low_t)fdct_round_shift(temp); + temp = output[14] * cospi_8_64 + output[9] * cospi_24_64; + step[14] = (tran_low_t)fdct_round_shift(temp); + step[15] = output[15]; + + range_check(step, 16, 16); + + // stage 5 + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + temp = step[4] * cospi_28_64 + step[7] * cospi_4_64; + output[4] = (tran_low_t)fdct_round_shift(temp); + temp = step[5] * cospi_12_64 + step[6] * cospi_20_64; + output[5] = (tran_low_t)fdct_round_shift(temp); + temp = step[6] * cospi_12_64 + step[5] * -cospi_20_64; + output[6] = (tran_low_t)fdct_round_shift(temp); + temp = step[7] * cospi_28_64 + step[4] * -cospi_4_64; + output[7] = (tran_low_t)fdct_round_shift(temp); + output[8] = step[8] + step[9]; + output[9] = step[8] - step[9]; + output[10] = step[11] - step[10]; + output[11] = step[11] + step[10]; + output[12] = step[12] + step[13]; + output[13] = step[12] - step[13]; + output[14] = step[15] - step[14]; + output[15] = step[15] + step[14]; + + range_check(output, 16, 16); + + // stage 6 + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + temp = output[8] * cospi_30_64 + output[15] * cospi_2_64; + step[8] = (tran_low_t)fdct_round_shift(temp); + temp = output[9] * cospi_14_64 + output[14] * cospi_18_64; + step[9] = (tran_low_t)fdct_round_shift(temp); + temp = output[10] * cospi_22_64 + output[13] * cospi_10_64; + step[10] = (tran_low_t)fdct_round_shift(temp); + temp = output[11] * cospi_6_64 + output[12] * cospi_26_64; + step[11] = (tran_low_t)fdct_round_shift(temp); + temp = output[12] * cospi_6_64 + output[11] * -cospi_26_64; + step[12] = (tran_low_t)fdct_round_shift(temp); + temp = output[13] * cospi_22_64 + output[10] * -cospi_10_64; + step[13] = (tran_low_t)fdct_round_shift(temp); + temp = output[14] * cospi_14_64 + output[9] * -cospi_18_64; + step[14] = (tran_low_t)fdct_round_shift(temp); + temp = output[15] * cospi_30_64 + output[8] * -cospi_2_64; + step[15] = (tran_low_t)fdct_round_shift(temp); + + range_check(step, 16, 16); + + // stage 7 + output[0] = step[0]; + output[1] = step[8]; + output[2] = step[4]; + output[3] = step[12]; + output[4] = step[2]; + output[5] = step[10]; + output[6] = step[6]; + output[7] = step[14]; + output[8] = step[1]; + output[9] = step[9]; + output[10] = step[5]; + output[11] = step[13]; + output[12] = step[3]; + output[13] = step[11]; + output[14] = step[7]; + output[15] = step[15]; + + range_check(output, 16, 16); +} + +static void fdct32(const tran_low_t *input, tran_low_t *output) { + tran_high_t temp; + tran_low_t step[32]; + + // stage 0 + range_check(input, 32, 14); + + // stage 1 + output[0] = input[0] + input[31]; + output[1] = input[1] + input[30]; + output[2] = input[2] + input[29]; + output[3] = input[3] + input[28]; + output[4] = input[4] + input[27]; + output[5] = input[5] + input[26]; + output[6] = input[6] + input[25]; + output[7] = input[7] + input[24]; + output[8] = input[8] + input[23]; + output[9] = input[9] + input[22]; + output[10] = input[10] + input[21]; + output[11] = input[11] + input[20]; + output[12] = input[12] + input[19]; + output[13] = input[13] + input[18]; + output[14] = input[14] + input[17]; + output[15] = input[15] + input[16]; + output[16] = input[15] - input[16]; + output[17] = input[14] - input[17]; + output[18] = input[13] - input[18]; + output[19] = input[12] - input[19]; + output[20] = input[11] - input[20]; + output[21] = input[10] - input[21]; + output[22] = input[9] - input[22]; + output[23] = input[8] - input[23]; + output[24] = input[7] - input[24]; + output[25] = input[6] - input[25]; + output[26] = input[5] - input[26]; + output[27] = input[4] - input[27]; + output[28] = input[3] - input[28]; + output[29] = input[2] - input[29]; + output[30] = input[1] - input[30]; + output[31] = input[0] - input[31]; + + range_check(output, 32, 15); + + // stage 2 + step[0] = output[0] + output[15]; + step[1] = output[1] + output[14]; + step[2] = output[2] + output[13]; + step[3] = output[3] + output[12]; + step[4] = output[4] + output[11]; + step[5] = output[5] + output[10]; + step[6] = output[6] + output[9]; + step[7] = output[7] + output[8]; + step[8] = output[7] - output[8]; + step[9] = output[6] - output[9]; + step[10] = output[5] - output[10]; + step[11] = output[4] - output[11]; + step[12] = output[3] - output[12]; + step[13] = output[2] - output[13]; + step[14] = output[1] - output[14]; + step[15] = output[0] - output[15]; + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + temp = output[20] * -cospi_16_64 + output[27] * cospi_16_64; + step[20] = (tran_low_t)fdct_round_shift(temp); + temp = output[21] * -cospi_16_64 + output[26] * cospi_16_64; + step[21] = (tran_low_t)fdct_round_shift(temp); + temp = output[22] * -cospi_16_64 + output[25] * cospi_16_64; + step[22] = (tran_low_t)fdct_round_shift(temp); + temp = output[23] * -cospi_16_64 + output[24] * cospi_16_64; + step[23] = (tran_low_t)fdct_round_shift(temp); + temp = output[24] * cospi_16_64 + output[23] * cospi_16_64; + step[24] = (tran_low_t)fdct_round_shift(temp); + temp = output[25] * cospi_16_64 + output[22] * cospi_16_64; + step[25] = (tran_low_t)fdct_round_shift(temp); + temp = output[26] * cospi_16_64 + output[21] * cospi_16_64; + step[26] = (tran_low_t)fdct_round_shift(temp); + temp = output[27] * cospi_16_64 + output[20] * cospi_16_64; + step[27] = (tran_low_t)fdct_round_shift(temp); + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + + range_check(step, 32, 16); + + // stage 3 + output[0] = step[0] + step[7]; + output[1] = step[1] + step[6]; + output[2] = step[2] + step[5]; + output[3] = step[3] + step[4]; + output[4] = step[3] - step[4]; + output[5] = step[2] - step[5]; + output[6] = step[1] - step[6]; + output[7] = step[0] - step[7]; + output[8] = step[8]; + output[9] = step[9]; + temp = step[10] * -cospi_16_64 + step[13] * cospi_16_64; + output[10] = (tran_low_t)fdct_round_shift(temp); + temp = step[11] * -cospi_16_64 + step[12] * cospi_16_64; + output[11] = (tran_low_t)fdct_round_shift(temp); + temp = step[12] * cospi_16_64 + step[11] * cospi_16_64; + output[12] = (tran_low_t)fdct_round_shift(temp); + temp = step[13] * cospi_16_64 + step[10] * cospi_16_64; + output[13] = (tran_low_t)fdct_round_shift(temp); + output[14] = step[14]; + output[15] = step[15]; + output[16] = step[16] + step[23]; + output[17] = step[17] + step[22]; + output[18] = step[18] + step[21]; + output[19] = step[19] + step[20]; + output[20] = step[19] - step[20]; + output[21] = step[18] - step[21]; + output[22] = step[17] - step[22]; + output[23] = step[16] - step[23]; + output[24] = step[31] - step[24]; + output[25] = step[30] - step[25]; + output[26] = step[29] - step[26]; + output[27] = step[28] - step[27]; + output[28] = step[28] + step[27]; + output[29] = step[29] + step[26]; + output[30] = step[30] + step[25]; + output[31] = step[31] + step[24]; + + range_check(output, 32, 17); + + // stage 4 + step[0] = output[0] + output[3]; + step[1] = output[1] + output[2]; + step[2] = output[1] - output[2]; + step[3] = output[0] - output[3]; + step[4] = output[4]; + temp = output[5] * -cospi_16_64 + output[6] * cospi_16_64; + step[5] = (tran_low_t)fdct_round_shift(temp); + temp = output[6] * cospi_16_64 + output[5] * cospi_16_64; + step[6] = (tran_low_t)fdct_round_shift(temp); + step[7] = output[7]; + step[8] = output[8] + output[11]; + step[9] = output[9] + output[10]; + step[10] = output[9] - output[10]; + step[11] = output[8] - output[11]; + step[12] = output[15] - output[12]; + step[13] = output[14] - output[13]; + step[14] = output[14] + output[13]; + step[15] = output[15] + output[12]; + step[16] = output[16]; + step[17] = output[17]; + temp = output[18] * -cospi_8_64 + output[29] * cospi_24_64; + step[18] = (tran_low_t)fdct_round_shift(temp); + temp = output[19] * -cospi_8_64 + output[28] * cospi_24_64; + step[19] = (tran_low_t)fdct_round_shift(temp); + temp = output[20] * -cospi_24_64 + output[27] * -cospi_8_64; + step[20] = (tran_low_t)fdct_round_shift(temp); + temp = output[21] * -cospi_24_64 + output[26] * -cospi_8_64; + step[21] = (tran_low_t)fdct_round_shift(temp); + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + temp = output[26] * cospi_24_64 + output[21] * -cospi_8_64; + step[26] = (tran_low_t)fdct_round_shift(temp); + temp = output[27] * cospi_24_64 + output[20] * -cospi_8_64; + step[27] = (tran_low_t)fdct_round_shift(temp); + temp = output[28] * cospi_8_64 + output[19] * cospi_24_64; + step[28] = (tran_low_t)fdct_round_shift(temp); + temp = output[29] * cospi_8_64 + output[18] * cospi_24_64; + step[29] = (tran_low_t)fdct_round_shift(temp); + step[30] = output[30]; + step[31] = output[31]; + + range_check(step, 32, 18); + + // stage 5 + temp = step[0] * cospi_16_64 + step[1] * cospi_16_64; + output[0] = (tran_low_t)fdct_round_shift(temp); + temp = step[1] * -cospi_16_64 + step[0] * cospi_16_64; + output[1] = (tran_low_t)fdct_round_shift(temp); + temp = step[2] * cospi_24_64 + step[3] * cospi_8_64; + output[2] = (tran_low_t)fdct_round_shift(temp); + temp = step[3] * cospi_24_64 + step[2] * -cospi_8_64; + output[3] = (tran_low_t)fdct_round_shift(temp); + output[4] = step[4] + step[5]; + output[5] = step[4] - step[5]; + output[6] = step[7] - step[6]; + output[7] = step[7] + step[6]; + output[8] = step[8]; + temp = step[9] * -cospi_8_64 + step[14] * cospi_24_64; + output[9] = (tran_low_t)fdct_round_shift(temp); + temp = step[10] * -cospi_24_64 + step[13] * -cospi_8_64; + output[10] = (tran_low_t)fdct_round_shift(temp); + output[11] = step[11]; + output[12] = step[12]; + temp = step[13] * cospi_24_64 + step[10] * -cospi_8_64; + output[13] = (tran_low_t)fdct_round_shift(temp); + temp = step[14] * cospi_8_64 + step[9] * cospi_24_64; + output[14] = (tran_low_t)fdct_round_shift(temp); + output[15] = step[15]; + output[16] = step[16] + step[19]; + output[17] = step[17] + step[18]; + output[18] = step[17] - step[18]; + output[19] = step[16] - step[19]; + output[20] = step[23] - step[20]; + output[21] = step[22] - step[21]; + output[22] = step[22] + step[21]; + output[23] = step[23] + step[20]; + output[24] = step[24] + step[27]; + output[25] = step[25] + step[26]; + output[26] = step[25] - step[26]; + output[27] = step[24] - step[27]; + output[28] = step[31] - step[28]; + output[29] = step[30] - step[29]; + output[30] = step[30] + step[29]; + output[31] = step[31] + step[28]; + + range_check(output, 32, 18); + + // stage 6 + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + temp = output[4] * cospi_28_64 + output[7] * cospi_4_64; + step[4] = (tran_low_t)fdct_round_shift(temp); + temp = output[5] * cospi_12_64 + output[6] * cospi_20_64; + step[5] = (tran_low_t)fdct_round_shift(temp); + temp = output[6] * cospi_12_64 + output[5] * -cospi_20_64; + step[6] = (tran_low_t)fdct_round_shift(temp); + temp = output[7] * cospi_28_64 + output[4] * -cospi_4_64; + step[7] = (tran_low_t)fdct_round_shift(temp); + step[8] = output[8] + output[9]; + step[9] = output[8] - output[9]; + step[10] = output[11] - output[10]; + step[11] = output[11] + output[10]; + step[12] = output[12] + output[13]; + step[13] = output[12] - output[13]; + step[14] = output[15] - output[14]; + step[15] = output[15] + output[14]; + step[16] = output[16]; + temp = output[17] * -cospi_4_64 + output[30] * cospi_28_64; + step[17] = (tran_low_t)fdct_round_shift(temp); + temp = output[18] * -cospi_28_64 + output[29] * -cospi_4_64; + step[18] = (tran_low_t)fdct_round_shift(temp); + step[19] = output[19]; + step[20] = output[20]; + temp = output[21] * -cospi_20_64 + output[26] * cospi_12_64; + step[21] = (tran_low_t)fdct_round_shift(temp); + temp = output[22] * -cospi_12_64 + output[25] * -cospi_20_64; + step[22] = (tran_low_t)fdct_round_shift(temp); + step[23] = output[23]; + step[24] = output[24]; + temp = output[25] * cospi_12_64 + output[22] * -cospi_20_64; + step[25] = (tran_low_t)fdct_round_shift(temp); + temp = output[26] * cospi_20_64 + output[21] * cospi_12_64; + step[26] = (tran_low_t)fdct_round_shift(temp); + step[27] = output[27]; + step[28] = output[28]; + temp = output[29] * cospi_28_64 + output[18] * -cospi_4_64; + step[29] = (tran_low_t)fdct_round_shift(temp); + temp = output[30] * cospi_4_64 + output[17] * cospi_28_64; + step[30] = (tran_low_t)fdct_round_shift(temp); + step[31] = output[31]; + + range_check(step, 32, 18); + + // stage 7 + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + temp = step[8] * cospi_30_64 + step[15] * cospi_2_64; + output[8] = (tran_low_t)fdct_round_shift(temp); + temp = step[9] * cospi_14_64 + step[14] * cospi_18_64; + output[9] = (tran_low_t)fdct_round_shift(temp); + temp = step[10] * cospi_22_64 + step[13] * cospi_10_64; + output[10] = (tran_low_t)fdct_round_shift(temp); + temp = step[11] * cospi_6_64 + step[12] * cospi_26_64; + output[11] = (tran_low_t)fdct_round_shift(temp); + temp = step[12] * cospi_6_64 + step[11] * -cospi_26_64; + output[12] = (tran_low_t)fdct_round_shift(temp); + temp = step[13] * cospi_22_64 + step[10] * -cospi_10_64; + output[13] = (tran_low_t)fdct_round_shift(temp); + temp = step[14] * cospi_14_64 + step[9] * -cospi_18_64; + output[14] = (tran_low_t)fdct_round_shift(temp); + temp = step[15] * cospi_30_64 + step[8] * -cospi_2_64; + output[15] = (tran_low_t)fdct_round_shift(temp); + output[16] = step[16] + step[17]; + output[17] = step[16] - step[17]; + output[18] = step[19] - step[18]; + output[19] = step[19] + step[18]; + output[20] = step[20] + step[21]; + output[21] = step[20] - step[21]; + output[22] = step[23] - step[22]; + output[23] = step[23] + step[22]; + output[24] = step[24] + step[25]; + output[25] = step[24] - step[25]; + output[26] = step[27] - step[26]; + output[27] = step[27] + step[26]; + output[28] = step[28] + step[29]; + output[29] = step[28] - step[29]; + output[30] = step[31] - step[30]; + output[31] = step[31] + step[30]; + + range_check(output, 32, 18); + + // stage 8 + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + temp = output[16] * cospi_31_64 + output[31] * cospi_1_64; + step[16] = (tran_low_t)fdct_round_shift(temp); + temp = output[17] * cospi_15_64 + output[30] * cospi_17_64; + step[17] = (tran_low_t)fdct_round_shift(temp); + temp = output[18] * cospi_23_64 + output[29] * cospi_9_64; + step[18] = (tran_low_t)fdct_round_shift(temp); + temp = output[19] * cospi_7_64 + output[28] * cospi_25_64; + step[19] = (tran_low_t)fdct_round_shift(temp); + temp = output[20] * cospi_27_64 + output[27] * cospi_5_64; + step[20] = (tran_low_t)fdct_round_shift(temp); + temp = output[21] * cospi_11_64 + output[26] * cospi_21_64; + step[21] = (tran_low_t)fdct_round_shift(temp); + temp = output[22] * cospi_19_64 + output[25] * cospi_13_64; + step[22] = (tran_low_t)fdct_round_shift(temp); + temp = output[23] * cospi_3_64 + output[24] * cospi_29_64; + step[23] = (tran_low_t)fdct_round_shift(temp); + temp = output[24] * cospi_3_64 + output[23] * -cospi_29_64; + step[24] = (tran_low_t)fdct_round_shift(temp); + temp = output[25] * cospi_19_64 + output[22] * -cospi_13_64; + step[25] = (tran_low_t)fdct_round_shift(temp); + temp = output[26] * cospi_11_64 + output[21] * -cospi_21_64; + step[26] = (tran_low_t)fdct_round_shift(temp); + temp = output[27] * cospi_27_64 + output[20] * -cospi_5_64; + step[27] = (tran_low_t)fdct_round_shift(temp); + temp = output[28] * cospi_7_64 + output[19] * -cospi_25_64; + step[28] = (tran_low_t)fdct_round_shift(temp); + temp = output[29] * cospi_23_64 + output[18] * -cospi_9_64; + step[29] = (tran_low_t)fdct_round_shift(temp); + temp = output[30] * cospi_15_64 + output[17] * -cospi_17_64; + step[30] = (tran_low_t)fdct_round_shift(temp); + temp = output[31] * cospi_31_64 + output[16] * -cospi_1_64; + step[31] = (tran_low_t)fdct_round_shift(temp); + + range_check(step, 32, 18); + + // stage 9 + output[0] = step[0]; + output[1] = step[16]; + output[2] = step[8]; + output[3] = step[24]; + output[4] = step[4]; + output[5] = step[20]; + output[6] = step[12]; + output[7] = step[28]; + output[8] = step[2]; + output[9] = step[18]; + output[10] = step[10]; + output[11] = step[26]; + output[12] = step[6]; + output[13] = step[22]; + output[14] = step[14]; + output[15] = step[30]; + output[16] = step[1]; + output[17] = step[17]; + output[18] = step[9]; + output[19] = step[25]; + output[20] = step[5]; + output[21] = step[21]; + output[22] = step[13]; + output[23] = step[29]; + output[24] = step[3]; + output[25] = step[19]; + output[26] = step[11]; + output[27] = step[27]; + output[28] = step[7]; + output[29] = step[23]; + output[30] = step[15]; + output[31] = step[31]; + + range_check(output, 32, 18); +} + +#ifndef AV1_DCT_GTEST + +static void fadst4(const tran_low_t *input, tran_low_t *output) { + tran_high_t x0, x1, x2, x3; + tran_high_t s0, s1, s2, s3, s4, s5, s6, s7; + + x0 = input[0]; + x1 = input[1]; + x2 = input[2]; + x3 = input[3]; + + if (!(x0 | x1 | x2 | x3)) { + output[0] = output[1] = output[2] = output[3] = 0; + return; + } + + s0 = sinpi_1_9 * x0; + s1 = sinpi_4_9 * x0; + s2 = sinpi_2_9 * x1; + s3 = sinpi_1_9 * x1; + s4 = sinpi_3_9 * x2; + s5 = sinpi_4_9 * x3; + s6 = sinpi_2_9 * x3; + s7 = x0 + x1 - x3; + + x0 = s0 + s2 + s5; + x1 = sinpi_3_9 * s7; + x2 = s1 - s3 + s6; + x3 = s4; + + s0 = x0 + x3; + s1 = x1; + s2 = x2 - x3; + s3 = x2 - x0 + x3; + + // 1-D transform scaling factor is sqrt(2). + output[0] = (tran_low_t)fdct_round_shift(s0); + output[1] = (tran_low_t)fdct_round_shift(s1); + output[2] = (tran_low_t)fdct_round_shift(s2); + output[3] = (tran_low_t)fdct_round_shift(s3); +} + +static void fadst8(const tran_low_t *input, tran_low_t *output) { + tran_high_t s0, s1, s2, s3, s4, s5, s6, s7; + + tran_high_t x0 = input[7]; + tran_high_t x1 = input[0]; + tran_high_t x2 = input[5]; + tran_high_t x3 = input[2]; + tran_high_t x4 = input[3]; + tran_high_t x5 = input[4]; + tran_high_t x6 = input[1]; + tran_high_t x7 = input[6]; + + // stage 1 + s0 = cospi_2_64 * x0 + cospi_30_64 * x1; + s1 = cospi_30_64 * x0 - cospi_2_64 * x1; + s2 = cospi_10_64 * x2 + cospi_22_64 * x3; + s3 = cospi_22_64 * x2 - cospi_10_64 * x3; + s4 = cospi_18_64 * x4 + cospi_14_64 * x5; + s5 = cospi_14_64 * x4 - cospi_18_64 * x5; + s6 = cospi_26_64 * x6 + cospi_6_64 * x7; + s7 = cospi_6_64 * x6 - cospi_26_64 * x7; + + x0 = s0 + s4; + x1 = s1 + s5; + x2 = s2 + s6; + x3 = s3 + s7; + x4 = fdct_round_shift(s0 - s4); + x5 = fdct_round_shift(s1 - s5); + x6 = fdct_round_shift(s2 - s6); + x7 = fdct_round_shift(s3 - s7); + + // stage 2 + s0 = x0; + s1 = x1; + s2 = x2; + s3 = x3; + s4 = cospi_8_64 * x4 + cospi_24_64 * x5; + s5 = cospi_24_64 * x4 - cospi_8_64 * x5; + s6 = -cospi_24_64 * x6 + cospi_8_64 * x7; + s7 = cospi_8_64 * x6 + cospi_24_64 * x7; + + x0 = fdct_round_shift(s0 + s2); + x1 = fdct_round_shift(s1 + s3); + x2 = fdct_round_shift(s0 - s2); + x3 = fdct_round_shift(s1 - s3); + x4 = fdct_round_shift(s4 + s6); + x5 = fdct_round_shift(s5 + s7); + x6 = fdct_round_shift(s4 - s6); + x7 = fdct_round_shift(s5 - s7); + + // stage 3 + s2 = cospi_16_64 * (x2 + x3); + s3 = cospi_16_64 * (x2 - x3); + s6 = cospi_16_64 * (x6 + x7); + s7 = cospi_16_64 * (x6 - x7); + + x2 = fdct_round_shift(s2); + x3 = fdct_round_shift(s3); + x6 = fdct_round_shift(s6); + x7 = fdct_round_shift(s7); + + output[0] = (tran_low_t)x0; + output[1] = (tran_low_t)-x4; + output[2] = (tran_low_t)x6; + output[3] = (tran_low_t)-x2; + output[4] = (tran_low_t)x3; + output[5] = (tran_low_t)-x7; + output[6] = (tran_low_t)x5; + output[7] = (tran_low_t)-x1; +} + +static void fadst16(const tran_low_t *input, tran_low_t *output) { + tran_high_t s0, s1, s2, s3, s4, s5, s6, s7, s8; + tran_high_t s9, s10, s11, s12, s13, s14, s15; + + tran_high_t x0 = input[15]; + tran_high_t x1 = input[0]; + tran_high_t x2 = input[13]; + tran_high_t x3 = input[2]; + tran_high_t x4 = input[11]; + tran_high_t x5 = input[4]; + tran_high_t x6 = input[9]; + tran_high_t x7 = input[6]; + tran_high_t x8 = input[7]; + tran_high_t x9 = input[8]; + tran_high_t x10 = input[5]; + tran_high_t x11 = input[10]; + tran_high_t x12 = input[3]; + tran_high_t x13 = input[12]; + tran_high_t x14 = input[1]; + tran_high_t x15 = input[14]; + + // stage 1 + s0 = x0 * cospi_1_64 + x1 * cospi_31_64; + s1 = x0 * cospi_31_64 - x1 * cospi_1_64; + s2 = x2 * cospi_5_64 + x3 * cospi_27_64; + s3 = x2 * cospi_27_64 - x3 * cospi_5_64; + s4 = x4 * cospi_9_64 + x5 * cospi_23_64; + s5 = x4 * cospi_23_64 - x5 * cospi_9_64; + s6 = x6 * cospi_13_64 + x7 * cospi_19_64; + s7 = x6 * cospi_19_64 - x7 * cospi_13_64; + s8 = x8 * cospi_17_64 + x9 * cospi_15_64; + s9 = x8 * cospi_15_64 - x9 * cospi_17_64; + s10 = x10 * cospi_21_64 + x11 * cospi_11_64; + s11 = x10 * cospi_11_64 - x11 * cospi_21_64; + s12 = x12 * cospi_25_64 + x13 * cospi_7_64; + s13 = x12 * cospi_7_64 - x13 * cospi_25_64; + s14 = x14 * cospi_29_64 + x15 * cospi_3_64; + s15 = x14 * cospi_3_64 - x15 * cospi_29_64; + + x0 = s0 + s8; + x1 = s1 + s9; + x2 = s2 + s10; + x3 = s3 + s11; + x4 = s4 + s12; + x5 = s5 + s13; + x6 = s6 + s14; + x7 = s7 + s15; + + x8 = fdct_round_shift(s0 - s8); + x9 = fdct_round_shift(s1 - s9); + x10 = fdct_round_shift(s2 - s10); + x11 = fdct_round_shift(s3 - s11); + x12 = fdct_round_shift(s4 - s12); + x13 = fdct_round_shift(s5 - s13); + x14 = fdct_round_shift(s6 - s14); + x15 = fdct_round_shift(s7 - s15); + + // stage 2 + s0 = x0; + s1 = x1; + s2 = x2; + s3 = x3; + s4 = x4; + s5 = x5; + s6 = x6; + s7 = x7; + s8 = x8 * cospi_4_64 + x9 * cospi_28_64; + s9 = x8 * cospi_28_64 - x9 * cospi_4_64; + s10 = x10 * cospi_20_64 + x11 * cospi_12_64; + s11 = x10 * cospi_12_64 - x11 * cospi_20_64; + s12 = -x12 * cospi_28_64 + x13 * cospi_4_64; + s13 = x12 * cospi_4_64 + x13 * cospi_28_64; + s14 = -x14 * cospi_12_64 + x15 * cospi_20_64; + s15 = x14 * cospi_20_64 + x15 * cospi_12_64; + + x0 = s0 + s4; + x1 = s1 + s5; + x2 = s2 + s6; + x3 = s3 + s7; + x4 = fdct_round_shift(s0 - s4); + x5 = fdct_round_shift(s1 - s5); + x6 = fdct_round_shift(s2 - s6); + x7 = fdct_round_shift(s3 - s7); + + x8 = s8 + s12; + x9 = s9 + s13; + x10 = s10 + s14; + x11 = s11 + s15; + x12 = fdct_round_shift(s8 - s12); + x13 = fdct_round_shift(s9 - s13); + x14 = fdct_round_shift(s10 - s14); + x15 = fdct_round_shift(s11 - s15); + + // stage 3 + s0 = x0; + s1 = x1; + s2 = x2; + s3 = x3; + s4 = x4 * cospi_8_64 + x5 * cospi_24_64; + s5 = x4 * cospi_24_64 - x5 * cospi_8_64; + s6 = -x6 * cospi_24_64 + x7 * cospi_8_64; + s7 = x6 * cospi_8_64 + x7 * cospi_24_64; + s8 = x8; + s9 = x9; + s10 = x10; + s11 = x11; + s12 = x12 * cospi_8_64 + x13 * cospi_24_64; + s13 = x12 * cospi_24_64 - x13 * cospi_8_64; + s14 = -x14 * cospi_24_64 + x15 * cospi_8_64; + s15 = x14 * cospi_8_64 + x15 * cospi_24_64; + + x0 = fdct_round_shift(s0 + s2); + x1 = fdct_round_shift(s1 + s3); + x2 = fdct_round_shift(s0 - s2); + x3 = fdct_round_shift(s1 - s3); + + x4 = fdct_round_shift(s4 + s6); + x5 = fdct_round_shift(s5 + s7); + x6 = fdct_round_shift(s4 - s6); + x7 = fdct_round_shift(s5 - s7); + + x8 = fdct_round_shift(s8 + s10); + x9 = fdct_round_shift(s9 + s11); + x10 = fdct_round_shift(s8 - s10); + x11 = fdct_round_shift(s9 - s11); + + x12 = fdct_round_shift(s12 + s14); + x13 = fdct_round_shift(s13 + s15); + x14 = fdct_round_shift(s12 - s14); + x15 = fdct_round_shift(s13 - s15); + + // stage 4 + s2 = (-cospi_16_64) * (x2 + x3); + s3 = cospi_16_64 * (x2 - x3); + s6 = cospi_16_64 * (x6 + x7); + s7 = cospi_16_64 * (-x6 + x7); + s10 = cospi_16_64 * (x10 + x11); + s11 = cospi_16_64 * (-x10 + x11); + s14 = (-cospi_16_64) * (x14 + x15); + s15 = cospi_16_64 * (x14 - x15); + + x2 = fdct_round_shift(s2); + x3 = fdct_round_shift(s3); + x6 = fdct_round_shift(s6); + x7 = fdct_round_shift(s7); + x10 = fdct_round_shift(s10); + x11 = fdct_round_shift(s11); + x14 = fdct_round_shift(s14); + x15 = fdct_round_shift(s15); + + output[0] = (tran_low_t)x0; + output[1] = (tran_low_t)-x8; + output[2] = (tran_low_t)x12; + output[3] = (tran_low_t)-x4; + output[4] = (tran_low_t)x6; + output[5] = (tran_low_t)x14; + output[6] = (tran_low_t)x10; + output[7] = (tran_low_t)x2; + output[8] = (tran_low_t)x3; + output[9] = (tran_low_t)x11; + output[10] = (tran_low_t)x15; + output[11] = (tran_low_t)x7; + output[12] = (tran_low_t)x5; + output[13] = (tran_low_t)-x13; + output[14] = (tran_low_t)x9; + output[15] = (tran_low_t)-x1; +} + +// For use in lieu of ADST +static void fhalfright32(const tran_low_t *input, tran_low_t *output) { + int i; + tran_low_t inputhalf[16]; + for (i = 0; i < 16; ++i) { + output[16 + i] = input[i] * 4; + } + // Multiply input by sqrt(2) + for (i = 0; i < 16; ++i) { + inputhalf[i] = (tran_low_t)fdct_round_shift(input[i + 16] * Sqrt2); + } + fdct16(inputhalf, output); + // Note overall scaling factor is 4 times orthogonal +} + +#if CONFIG_EXT_TX +static void fidtx4(const tran_low_t *input, tran_low_t *output) { + int i; + for (i = 0; i < 4; ++i) + output[i] = (tran_low_t)fdct_round_shift(input[i] * Sqrt2); +} + +static void fidtx8(const tran_low_t *input, tran_low_t *output) { + int i; + for (i = 0; i < 8; ++i) output[i] = input[i] * 2; +} + +static void fidtx16(const tran_low_t *input, tran_low_t *output) { + int i; + for (i = 0; i < 16; ++i) + output[i] = (tran_low_t)fdct_round_shift(input[i] * 2 * Sqrt2); +} + +static void fidtx32(const tran_low_t *input, tran_low_t *output) { + int i; + for (i = 0; i < 32; ++i) output[i] = input[i] * 4; +} + +static void copy_block(const int16_t *src, int src_stride, int l, int w, + int16_t *dest, int dest_stride) { + int i; + for (i = 0; i < l; ++i) { + memcpy(dest + dest_stride * i, src + src_stride * i, w * sizeof(int16_t)); + } +} + +static void fliplr(int16_t *dest, int stride, int l, int w) { + int i, j; + for (i = 0; i < l; ++i) { + for (j = 0; j < w / 2; ++j) { + const int16_t tmp = dest[i * stride + j]; + dest[i * stride + j] = dest[i * stride + w - 1 - j]; + dest[i * stride + w - 1 - j] = tmp; + } + } +} + +static void flipud(int16_t *dest, int stride, int l, int w) { + int i, j; + for (j = 0; j < w; ++j) { + for (i = 0; i < l / 2; ++i) { + const int16_t tmp = dest[i * stride + j]; + dest[i * stride + j] = dest[(l - 1 - i) * stride + j]; + dest[(l - 1 - i) * stride + j] = tmp; + } + } +} + +static void fliplrud(int16_t *dest, int stride, int l, int w) { + int i, j; + for (i = 0; i < l / 2; ++i) { + for (j = 0; j < w; ++j) { + const int16_t tmp = dest[i * stride + j]; + dest[i * stride + j] = dest[(l - 1 - i) * stride + w - 1 - j]; + dest[(l - 1 - i) * stride + w - 1 - j] = tmp; + } + } +} + +static void copy_fliplr(const int16_t *src, int src_stride, int l, int w, + int16_t *dest, int dest_stride) { + copy_block(src, src_stride, l, w, dest, dest_stride); + fliplr(dest, dest_stride, l, w); +} + +static void copy_flipud(const int16_t *src, int src_stride, int l, int w, + int16_t *dest, int dest_stride) { + copy_block(src, src_stride, l, w, dest, dest_stride); + flipud(dest, dest_stride, l, w); +} + +static void copy_fliplrud(const int16_t *src, int src_stride, int l, int w, + int16_t *dest, int dest_stride) { + copy_block(src, src_stride, l, w, dest, dest_stride); + fliplrud(dest, dest_stride, l, w); +} + +static void maybe_flip_input(const int16_t **src, int *src_stride, int l, int w, + int16_t *buff, int tx_type) { + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + case IDTX: + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: break; + case FLIPADST_DCT: + case FLIPADST_ADST: + case V_FLIPADST: + copy_flipud(*src, *src_stride, l, w, buff, w); + *src = buff; + *src_stride = w; + break; + case DCT_FLIPADST: + case ADST_FLIPADST: + case H_FLIPADST: + copy_fliplr(*src, *src_stride, l, w, buff, w); + *src = buff; + *src_stride = w; + break; + case FLIPADST_FLIPADST: + copy_fliplrud(*src, *src_stride, l, w, buff, w); + *src = buff; + *src_stride = w; + break; + default: assert(0); break; + } +} +#endif // CONFIG_EXT_TX + +void av1_fht4x4_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + if (tx_type == DCT_DCT) { + aom_fdct4x4_c(input, output, stride); + } else { + static const transform_2d FHT[] = { + { fdct4, fdct4 }, // DCT_DCT + { fadst4, fdct4 }, // ADST_DCT + { fdct4, fadst4 }, // DCT_ADST + { fadst4, fadst4 }, // ADST_ADST +#if CONFIG_EXT_TX + { fadst4, fdct4 }, // FLIPADST_DCT + { fdct4, fadst4 }, // DCT_FLIPADST + { fadst4, fadst4 }, // FLIPADST_FLIPADST + { fadst4, fadst4 }, // ADST_FLIPADST + { fadst4, fadst4 }, // FLIPADST_ADST + { fidtx4, fidtx4 }, // IDTX + { fdct4, fidtx4 }, // V_DCT + { fidtx4, fdct4 }, // H_DCT + { fadst4, fidtx4 }, // V_ADST + { fidtx4, fadst4 }, // H_ADST + { fadst4, fidtx4 }, // V_FLIPADST + { fidtx4, fadst4 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + const transform_2d ht = FHT[tx_type]; + tran_low_t out[4 * 4]; + int i, j; + tran_low_t temp_in[4], temp_out[4]; + +#if CONFIG_EXT_TX + int16_t flipped_input[4 * 4]; + maybe_flip_input(&input, &stride, 4, 4, flipped_input, tx_type); +#endif + + // Columns + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) temp_in[j] = input[j * stride + i] * 16; + if (i == 0 && temp_in[0]) temp_in[0] += 1; + ht.cols(temp_in, temp_out); + for (j = 0; j < 4; ++j) out[j * 4 + i] = temp_out[j]; + } + + // Rows + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) temp_in[j] = out[j + i * 4]; + ht.rows(temp_in, temp_out); + for (j = 0; j < 4; ++j) output[j + i * 4] = (temp_out[j] + 1) >> 2; + } + } +} + +void av1_fht4x8_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct8, fdct4 }, // DCT_DCT + { fadst8, fdct4 }, // ADST_DCT + { fdct8, fadst4 }, // DCT_ADST + { fadst8, fadst4 }, // ADST_ADST +#if CONFIG_EXT_TX + { fadst8, fdct4 }, // FLIPADST_DCT + { fdct8, fadst4 }, // DCT_FLIPADST + { fadst8, fadst4 }, // FLIPADST_FLIPADST + { fadst8, fadst4 }, // ADST_FLIPADST + { fadst8, fadst4 }, // FLIPADST_ADST + { fidtx8, fidtx4 }, // IDTX + { fdct8, fidtx4 }, // V_DCT + { fidtx8, fdct4 }, // H_DCT + { fadst8, fidtx4 }, // V_ADST + { fidtx8, fadst4 }, // H_ADST + { fadst8, fidtx4 }, // V_FLIPADST + { fidtx8, fadst4 }, // H_FLIPADST +#endif + }; + const transform_2d ht = FHT[tx_type]; + const int n = 4; + const int n2 = 8; + tran_low_t out[8 * 4]; + tran_low_t temp_in[8], temp_out[8]; + int i, j; +#if CONFIG_EXT_TX + int16_t flipped_input[8 * 4]; + maybe_flip_input(&input, &stride, n2, n, flipped_input, tx_type); +#endif + + // Rows + for (i = 0; i < n2; ++i) { + for (j = 0; j < n; ++j) + temp_in[j] = + (tran_low_t)fdct_round_shift(input[i * stride + j] * 4 * Sqrt2); + ht.rows(temp_in, temp_out); + for (j = 0; j < n; ++j) out[j * n2 + i] = temp_out[j]; + } + + // Columns + for (i = 0; i < n; ++i) { + for (j = 0; j < n2; ++j) temp_in[j] = out[j + i * n2]; + ht.cols(temp_in, temp_out); + for (j = 0; j < n2; ++j) + output[i + j * n] = (temp_out[j] + (temp_out[j] < 0)) >> 1; + } + // Note: overall scale factor of transform is 8 times unitary +} + +void av1_fht8x4_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct4, fdct8 }, // DCT_DCT + { fadst4, fdct8 }, // ADST_DCT + { fdct4, fadst8 }, // DCT_ADST + { fadst4, fadst8 }, // ADST_ADST +#if CONFIG_EXT_TX + { fadst4, fdct8 }, // FLIPADST_DCT + { fdct4, fadst8 }, // DCT_FLIPADST + { fadst4, fadst8 }, // FLIPADST_FLIPADST + { fadst4, fadst8 }, // ADST_FLIPADST + { fadst4, fadst8 }, // FLIPADST_ADST + { fidtx4, fidtx8 }, // IDTX + { fdct4, fidtx8 }, // V_DCT + { fidtx4, fdct8 }, // H_DCT + { fadst4, fidtx8 }, // V_ADST + { fidtx4, fadst8 }, // H_ADST + { fadst4, fidtx8 }, // V_FLIPADST + { fidtx4, fadst8 }, // H_FLIPADST +#endif + }; + const transform_2d ht = FHT[tx_type]; + const int n = 4; + const int n2 = 8; + tran_low_t out[8 * 4]; + tran_low_t temp_in[8], temp_out[8]; + int i, j; +#if CONFIG_EXT_TX + int16_t flipped_input[8 * 4]; + maybe_flip_input(&input, &stride, n, n2, flipped_input, tx_type); +#endif + + // Columns + for (i = 0; i < n2; ++i) { + for (j = 0; j < n; ++j) + temp_in[j] = + (tran_low_t)fdct_round_shift(input[j * stride + i] * 4 * Sqrt2); + ht.cols(temp_in, temp_out); + for (j = 0; j < n; ++j) out[j * n2 + i] = temp_out[j]; + } + + // Rows + for (i = 0; i < n; ++i) { + for (j = 0; j < n2; ++j) temp_in[j] = out[j + i * n2]; + ht.rows(temp_in, temp_out); + for (j = 0; j < n2; ++j) + output[j + i * n2] = (temp_out[j] + (temp_out[j] < 0)) >> 1; + } + // Note: overall scale factor of transform is 8 times unitary +} + +void av1_fht4x16_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct16, fdct4 }, // DCT_DCT + { fadst16, fdct4 }, // ADST_DCT + { fdct16, fadst4 }, // DCT_ADST + { fadst16, fadst4 }, // ADST_ADST +#if CONFIG_EXT_TX + { fadst16, fdct4 }, // FLIPADST_DCT + { fdct16, fadst4 }, // DCT_FLIPADST + { fadst16, fadst4 }, // FLIPADST_FLIPADST + { fadst16, fadst4 }, // ADST_FLIPADST + { fadst16, fadst4 }, // FLIPADST_ADST + { fidtx16, fidtx4 }, // IDTX + { fdct16, fidtx4 }, // V_DCT + { fidtx16, fdct4 }, // H_DCT + { fadst16, fidtx4 }, // V_ADST + { fidtx16, fadst4 }, // H_ADST + { fadst16, fidtx4 }, // V_FLIPADST + { fidtx16, fadst4 }, // H_FLIPADST +#endif + }; + const transform_2d ht = FHT[tx_type]; + const int n = 4; + const int n4 = 16; + tran_low_t out[16 * 4]; + tran_low_t temp_in[16], temp_out[16]; + int i, j; +#if CONFIG_EXT_TX + int16_t flipped_input[16 * 4]; + maybe_flip_input(&input, &stride, n4, n, flipped_input, tx_type); +#endif + + // Rows + for (i = 0; i < n4; ++i) { + for (j = 0; j < n; ++j) temp_in[j] = input[i * stride + j] * 4; + ht.rows(temp_in, temp_out); + for (j = 0; j < n; ++j) out[j * n4 + i] = temp_out[j]; + } + + // Columns + for (i = 0; i < n; ++i) { + for (j = 0; j < n4; ++j) temp_in[j] = out[j + i * n4]; + ht.cols(temp_in, temp_out); + for (j = 0; j < n4; ++j) + output[i + j * n] = (temp_out[j] + (temp_out[j] < 0)) >> 1; + } + // Note: overall scale factor of transform is 8 times unitary +} + +void av1_fht16x4_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct4, fdct16 }, // DCT_DCT + { fadst4, fdct16 }, // ADST_DCT + { fdct4, fadst16 }, // DCT_ADST + { fadst4, fadst16 }, // ADST_ADST +#if CONFIG_EXT_TX + { fadst4, fdct16 }, // FLIPADST_DCT + { fdct4, fadst16 }, // DCT_FLIPADST + { fadst4, fadst16 }, // FLIPADST_FLIPADST + { fadst4, fadst16 }, // ADST_FLIPADST + { fadst4, fadst16 }, // FLIPADST_ADST + { fidtx4, fidtx16 }, // IDTX + { fdct4, fidtx16 }, // V_DCT + { fidtx4, fdct16 }, // H_DCT + { fadst4, fidtx16 }, // V_ADST + { fidtx4, fadst16 }, // H_ADST + { fadst4, fidtx16 }, // V_FLIPADST + { fidtx4, fadst16 }, // H_FLIPADST +#endif + }; + const transform_2d ht = FHT[tx_type]; + const int n = 4; + const int n4 = 16; + tran_low_t out[16 * 4]; + tran_low_t temp_in[16], temp_out[16]; + int i, j; +#if CONFIG_EXT_TX + int16_t flipped_input[16 * 4]; + maybe_flip_input(&input, &stride, n, n4, flipped_input, tx_type); +#endif + + // Columns + for (i = 0; i < n4; ++i) { + for (j = 0; j < n; ++j) temp_in[j] = input[j * stride + i] * 4; + ht.cols(temp_in, temp_out); + for (j = 0; j < n; ++j) out[j * n4 + i] = temp_out[j]; + } + + // Rows + for (i = 0; i < n; ++i) { + for (j = 0; j < n4; ++j) temp_in[j] = out[j + i * n4]; + ht.rows(temp_in, temp_out); + for (j = 0; j < n4; ++j) + output[j + i * n4] = (temp_out[j] + (temp_out[j] < 0)) >> 1; + } + // Note: overall scale factor of transform is 8 times unitary +} + +void av1_fht8x16_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct16, fdct8 }, // DCT_DCT + { fadst16, fdct8 }, // ADST_DCT + { fdct16, fadst8 }, // DCT_ADST + { fadst16, fadst8 }, // ADST_ADST +#if CONFIG_EXT_TX + { fadst16, fdct8 }, // FLIPADST_DCT + { fdct16, fadst8 }, // DCT_FLIPADST + { fadst16, fadst8 }, // FLIPADST_FLIPADST + { fadst16, fadst8 }, // ADST_FLIPADST + { fadst16, fadst8 }, // FLIPADST_ADST + { fidtx16, fidtx8 }, // IDTX + { fdct16, fidtx8 }, // V_DCT + { fidtx16, fdct8 }, // H_DCT + { fadst16, fidtx8 }, // V_ADST + { fidtx16, fadst8 }, // H_ADST + { fadst16, fidtx8 }, // V_FLIPADST + { fidtx16, fadst8 }, // H_FLIPADST +#endif + }; + const transform_2d ht = FHT[tx_type]; + const int n = 8; + const int n2 = 16; + tran_low_t out[16 * 8]; + tran_low_t temp_in[16], temp_out[16]; + int i, j; +#if CONFIG_EXT_TX + int16_t flipped_input[16 * 8]; + maybe_flip_input(&input, &stride, n2, n, flipped_input, tx_type); +#endif + + // Rows + for (i = 0; i < n2; ++i) { + for (j = 0; j < n; ++j) + temp_in[j] = + (tran_low_t)fdct_round_shift(input[i * stride + j] * 4 * Sqrt2); + ht.rows(temp_in, temp_out); + for (j = 0; j < n; ++j) + out[j * n2 + i] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 2); + } + + // Columns + for (i = 0; i < n; ++i) { + for (j = 0; j < n2; ++j) temp_in[j] = out[j + i * n2]; + ht.cols(temp_in, temp_out); + for (j = 0; j < n2; ++j) output[i + j * n] = temp_out[j]; + } + // Note: overall scale factor of transform is 8 times unitary +} + +void av1_fht16x8_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct8, fdct16 }, // DCT_DCT + { fadst8, fdct16 }, // ADST_DCT + { fdct8, fadst16 }, // DCT_ADST + { fadst8, fadst16 }, // ADST_ADST +#if CONFIG_EXT_TX + { fadst8, fdct16 }, // FLIPADST_DCT + { fdct8, fadst16 }, // DCT_FLIPADST + { fadst8, fadst16 }, // FLIPADST_FLIPADST + { fadst8, fadst16 }, // ADST_FLIPADST + { fadst8, fadst16 }, // FLIPADST_ADST + { fidtx8, fidtx16 }, // IDTX + { fdct8, fidtx16 }, // V_DCT + { fidtx8, fdct16 }, // H_DCT + { fadst8, fidtx16 }, // V_ADST + { fidtx8, fadst16 }, // H_ADST + { fadst8, fidtx16 }, // V_FLIPADST + { fidtx8, fadst16 }, // H_FLIPADST +#endif + }; + const transform_2d ht = FHT[tx_type]; + const int n = 8; + const int n2 = 16; + tran_low_t out[16 * 8]; + tran_low_t temp_in[16], temp_out[16]; + int i, j; +#if CONFIG_EXT_TX + int16_t flipped_input[16 * 8]; + maybe_flip_input(&input, &stride, n, n2, flipped_input, tx_type); +#endif + + // Columns + for (i = 0; i < n2; ++i) { + for (j = 0; j < n; ++j) + temp_in[j] = + (tran_low_t)fdct_round_shift(input[j * stride + i] * 4 * Sqrt2); + ht.cols(temp_in, temp_out); + for (j = 0; j < n; ++j) + out[j * n2 + i] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 2); + } + + // Rows + for (i = 0; i < n; ++i) { + for (j = 0; j < n2; ++j) temp_in[j] = out[j + i * n2]; + ht.rows(temp_in, temp_out); + for (j = 0; j < n2; ++j) output[j + i * n2] = temp_out[j]; + } + // Note: overall scale factor of transform is 8 times unitary +} + +void av1_fht8x32_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct32, fdct8 }, // DCT_DCT + { fhalfright32, fdct8 }, // ADST_DCT + { fdct32, fadst8 }, // DCT_ADST + { fhalfright32, fadst8 }, // ADST_ADST +#if CONFIG_EXT_TX + { fhalfright32, fdct8 }, // FLIPADST_DCT + { fdct32, fadst8 }, // DCT_FLIPADST + { fhalfright32, fadst8 }, // FLIPADST_FLIPADST + { fhalfright32, fadst8 }, // ADST_FLIPADST + { fhalfright32, fadst8 }, // FLIPADST_ADST + { fidtx32, fidtx8 }, // IDTX + { fdct32, fidtx8 }, // V_DCT + { fidtx32, fdct8 }, // H_DCT + { fhalfright32, fidtx8 }, // V_ADST + { fidtx32, fadst8 }, // H_ADST + { fhalfright32, fidtx8 }, // V_FLIPADST + { fidtx32, fadst8 }, // H_FLIPADST +#endif + }; + const transform_2d ht = FHT[tx_type]; + const int n = 8; + const int n4 = 32; + tran_low_t out[32 * 8]; + tran_low_t temp_in[32], temp_out[32]; + int i, j; +#if CONFIG_EXT_TX + int16_t flipped_input[32 * 8]; + maybe_flip_input(&input, &stride, n4, n, flipped_input, tx_type); +#endif + + // Rows + for (i = 0; i < n4; ++i) { + for (j = 0; j < n; ++j) temp_in[j] = input[i * stride + j] * 4; + ht.rows(temp_in, temp_out); + for (j = 0; j < n; ++j) out[j * n4 + i] = temp_out[j]; + } + + // Columns + for (i = 0; i < n; ++i) { + for (j = 0; j < n4; ++j) temp_in[j] = out[j + i * n4]; + ht.cols(temp_in, temp_out); + for (j = 0; j < n4; ++j) + output[i + j * n] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 2); + } + // Note: overall scale factor of transform is 4 times unitary +} + +void av1_fht32x8_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct8, fdct32 }, // DCT_DCT + { fadst8, fdct32 }, // ADST_DCT + { fdct8, fhalfright32 }, // DCT_ADST + { fadst8, fhalfright32 }, // ADST_ADST +#if CONFIG_EXT_TX + { fadst8, fdct32 }, // FLIPADST_DCT + { fdct8, fhalfright32 }, // DCT_FLIPADST + { fadst8, fhalfright32 }, // FLIPADST_FLIPADST + { fadst8, fhalfright32 }, // ADST_FLIPADST + { fadst8, fhalfright32 }, // FLIPADST_ADST + { fidtx8, fidtx32 }, // IDTX + { fdct8, fidtx32 }, // V_DCT + { fidtx8, fdct32 }, // H_DCT + { fadst8, fidtx32 }, // V_ADST + { fidtx8, fhalfright32 }, // H_ADST + { fadst8, fidtx32 }, // V_FLIPADST + { fidtx8, fhalfright32 }, // H_FLIPADST +#endif + }; + const transform_2d ht = FHT[tx_type]; + const int n = 8; + const int n4 = 32; + tran_low_t out[32 * 8]; + tran_low_t temp_in[32], temp_out[32]; + int i, j; +#if CONFIG_EXT_TX + int16_t flipped_input[32 * 8]; + maybe_flip_input(&input, &stride, n, n4, flipped_input, tx_type); +#endif + + // Columns + for (i = 0; i < n4; ++i) { + for (j = 0; j < n; ++j) temp_in[j] = input[j * stride + i] * 4; + ht.cols(temp_in, temp_out); + for (j = 0; j < n; ++j) out[j * n4 + i] = temp_out[j]; + } + + // Rows + for (i = 0; i < n; ++i) { + for (j = 0; j < n4; ++j) temp_in[j] = out[j + i * n4]; + ht.rows(temp_in, temp_out); + for (j = 0; j < n4; ++j) + output[j + i * n4] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 2); + } + // Note: overall scale factor of transform is 4 times unitary +} + +void av1_fht16x32_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct32, fdct16 }, // DCT_DCT + { fhalfright32, fdct16 }, // ADST_DCT + { fdct32, fadst16 }, // DCT_ADST + { fhalfright32, fadst16 }, // ADST_ADST +#if CONFIG_EXT_TX + { fhalfright32, fdct16 }, // FLIPADST_DCT + { fdct32, fadst16 }, // DCT_FLIPADST + { fhalfright32, fadst16 }, // FLIPADST_FLIPADST + { fhalfright32, fadst16 }, // ADST_FLIPADST + { fhalfright32, fadst16 }, // FLIPADST_ADST + { fidtx32, fidtx16 }, // IDTX + { fdct32, fidtx16 }, // V_DCT + { fidtx32, fdct16 }, // H_DCT + { fhalfright32, fidtx16 }, // V_ADST + { fidtx32, fadst16 }, // H_ADST + { fhalfright32, fidtx16 }, // V_FLIPADST + { fidtx32, fadst16 }, // H_FLIPADST +#endif + }; + const transform_2d ht = FHT[tx_type]; + const int n = 16; + const int n2 = 32; + tran_low_t out[32 * 16]; + tran_low_t temp_in[32], temp_out[32]; + int i, j; +#if CONFIG_EXT_TX + int16_t flipped_input[32 * 16]; + maybe_flip_input(&input, &stride, n2, n, flipped_input, tx_type); +#endif + + // Rows + for (i = 0; i < n2; ++i) { + for (j = 0; j < n; ++j) + temp_in[j] = + (tran_low_t)fdct_round_shift(input[i * stride + j] * 4 * Sqrt2); + ht.rows(temp_in, temp_out); + for (j = 0; j < n; ++j) + out[j * n2 + i] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 4); + } + + // Columns + for (i = 0; i < n; ++i) { + for (j = 0; j < n2; ++j) temp_in[j] = out[j + i * n2]; + ht.cols(temp_in, temp_out); + for (j = 0; j < n2; ++j) output[i + j * n] = temp_out[j]; + } + // Note: overall scale factor of transform is 4 times unitary +} + +void av1_fht32x16_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct16, fdct32 }, // DCT_DCT + { fadst16, fdct32 }, // ADST_DCT + { fdct16, fhalfright32 }, // DCT_ADST + { fadst16, fhalfright32 }, // ADST_ADST +#if CONFIG_EXT_TX + { fadst16, fdct32 }, // FLIPADST_DCT + { fdct16, fhalfright32 }, // DCT_FLIPADST + { fadst16, fhalfright32 }, // FLIPADST_FLIPADST + { fadst16, fhalfright32 }, // ADST_FLIPADST + { fadst16, fhalfright32 }, // FLIPADST_ADST + { fidtx16, fidtx32 }, // IDTX + { fdct16, fidtx32 }, // V_DCT + { fidtx16, fdct32 }, // H_DCT + { fadst16, fidtx32 }, // V_ADST + { fidtx16, fhalfright32 }, // H_ADST + { fadst16, fidtx32 }, // V_FLIPADST + { fidtx16, fhalfright32 }, // H_FLIPADST +#endif + }; + const transform_2d ht = FHT[tx_type]; + const int n = 16; + const int n2 = 32; + tran_low_t out[32 * 16]; + tran_low_t temp_in[32], temp_out[32]; + int i, j; +#if CONFIG_EXT_TX + int16_t flipped_input[32 * 16]; + maybe_flip_input(&input, &stride, n, n2, flipped_input, tx_type); +#endif + + // Columns + for (i = 0; i < n2; ++i) { + for (j = 0; j < n; ++j) + temp_in[j] = + (tran_low_t)fdct_round_shift(input[j * stride + i] * 4 * Sqrt2); + ht.cols(temp_in, temp_out); + for (j = 0; j < n; ++j) + out[j * n2 + i] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 4); + } + + // Rows + for (i = 0; i < n; ++i) { + for (j = 0; j < n2; ++j) temp_in[j] = out[j + i * n2]; + ht.rows(temp_in, temp_out); + for (j = 0; j < n2; ++j) output[j + i * n2] = temp_out[j]; + } + // Note: overall scale factor of transform is 4 times unitary +} + +void av1_fdct8x8_quant_c(const int16_t *input, int stride, + tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, + uint16_t *eob_ptr, const int16_t *scan, + const int16_t *iscan +#if CONFIG_AOM_QM + , + const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr +#endif + ) { + int eob = -1; + + int i, j; + tran_low_t intermediate[64]; + + // Transform columns + { + tran_low_t *output = intermediate; + tran_high_t s0, s1, s2, s3, s4, s5, s6, s7; // canbe16 + tran_high_t t0, t1, t2, t3; // needs32 + tran_high_t x0, x1, x2, x3; // canbe16 + + for (i = 0; i < 8; i++) { + // stage 1 + s0 = (input[0 * stride] + input[7 * stride]) * 4; + s1 = (input[1 * stride] + input[6 * stride]) * 4; + s2 = (input[2 * stride] + input[5 * stride]) * 4; + s3 = (input[3 * stride] + input[4 * stride]) * 4; + s4 = (input[3 * stride] - input[4 * stride]) * 4; + s5 = (input[2 * stride] - input[5 * stride]) * 4; + s6 = (input[1 * stride] - input[6 * stride]) * 4; + s7 = (input[0 * stride] - input[7 * stride]) * 4; + + // fdct4(step, step); + x0 = s0 + s3; + x1 = s1 + s2; + x2 = s1 - s2; + x3 = s0 - s3; + t0 = (x0 + x1) * cospi_16_64; + t1 = (x0 - x1) * cospi_16_64; + t2 = x2 * cospi_24_64 + x3 * cospi_8_64; + t3 = -x2 * cospi_8_64 + x3 * cospi_24_64; + output[0 * 8] = (tran_low_t)fdct_round_shift(t0); + output[2 * 8] = (tran_low_t)fdct_round_shift(t2); + output[4 * 8] = (tran_low_t)fdct_round_shift(t1); + output[6 * 8] = (tran_low_t)fdct_round_shift(t3); + + // stage 2 + t0 = (s6 - s5) * cospi_16_64; + t1 = (s6 + s5) * cospi_16_64; + t2 = fdct_round_shift(t0); + t3 = fdct_round_shift(t1); + + // stage 3 + x0 = s4 + t2; + x1 = s4 - t2; + x2 = s7 - t3; + x3 = s7 + t3; + + // stage 4 + t0 = x0 * cospi_28_64 + x3 * cospi_4_64; + t1 = x1 * cospi_12_64 + x2 * cospi_20_64; + t2 = x2 * cospi_12_64 + x1 * -cospi_20_64; + t3 = x3 * cospi_28_64 + x0 * -cospi_4_64; + output[1 * 8] = (tran_low_t)fdct_round_shift(t0); + output[3 * 8] = (tran_low_t)fdct_round_shift(t2); + output[5 * 8] = (tran_low_t)fdct_round_shift(t1); + output[7 * 8] = (tran_low_t)fdct_round_shift(t3); + input++; + output++; + } + } + + // Rows + for (i = 0; i < 8; ++i) { + fdct8(&intermediate[i * 8], &coeff_ptr[i * 8]); + for (j = 0; j < 8; ++j) coeff_ptr[j + i * 8] /= 2; + } + + // TODO(jingning) Decide the need of these arguments after the + // quantization process is completed. + (void)zbin_ptr; + (void)quant_shift_ptr; + (void)iscan; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + if (!skip_block) { + // Quantization pass: All coefficients with index >= zero_flag are + // skippable. Note: zero_flag can be zero. + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + const int coeff = coeff_ptr[rc]; +#if CONFIG_AOM_QM + const qm_val_t wt = qm_ptr[rc]; + const qm_val_t iwt = iqm_ptr[rc]; + const int dequant = + (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> + AOM_QM_BITS; +#endif + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + + int64_t tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX); + int tmp32; +#if CONFIG_AOM_QM + tmp32 = (int)((tmp * quant_ptr[rc != 0] * wt) >> (16 + AOM_QM_BITS)); + qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant; +#else + tmp32 = (int)((tmp * quant_ptr[rc != 0]) >> 16); + qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0]; +#endif + + if (tmp32) eob = i; + } + } + *eob_ptr = eob + 1; +} + +void av1_fht8x8_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + if (tx_type == DCT_DCT) { + aom_fdct8x8_c(input, output, stride); + } else { + static const transform_2d FHT[] = { + { fdct8, fdct8 }, // DCT_DCT + { fadst8, fdct8 }, // ADST_DCT + { fdct8, fadst8 }, // DCT_ADST + { fadst8, fadst8 }, // ADST_ADST +#if CONFIG_EXT_TX + { fadst8, fdct8 }, // FLIPADST_DCT + { fdct8, fadst8 }, // DCT_FLIPADST + { fadst8, fadst8 }, // FLIPADST_FLIPADST + { fadst8, fadst8 }, // ADST_FLIPADST + { fadst8, fadst8 }, // FLIPADST_ADST + { fidtx8, fidtx8 }, // IDTX + { fdct8, fidtx8 }, // V_DCT + { fidtx8, fdct8 }, // H_DCT + { fadst8, fidtx8 }, // V_ADST + { fidtx8, fadst8 }, // H_ADST + { fadst8, fidtx8 }, // V_FLIPADST + { fidtx8, fadst8 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + const transform_2d ht = FHT[tx_type]; + tran_low_t out[64]; + int i, j; + tran_low_t temp_in[8], temp_out[8]; + +#if CONFIG_EXT_TX + int16_t flipped_input[8 * 8]; + maybe_flip_input(&input, &stride, 8, 8, flipped_input, tx_type); +#endif + + // Columns + for (i = 0; i < 8; ++i) { + for (j = 0; j < 8; ++j) temp_in[j] = input[j * stride + i] * 4; + ht.cols(temp_in, temp_out); + for (j = 0; j < 8; ++j) out[j * 8 + i] = temp_out[j]; + } + + // Rows + for (i = 0; i < 8; ++i) { + for (j = 0; j < 8; ++j) temp_in[j] = out[j + i * 8]; + ht.rows(temp_in, temp_out); + for (j = 0; j < 8; ++j) + output[j + i * 8] = (temp_out[j] + (temp_out[j] < 0)) >> 1; + } + } +} + +/* 4-point reversible, orthonormal Walsh-Hadamard in 3.5 adds, 0.5 shifts per + pixel. */ +void av1_fwht4x4_c(const int16_t *input, tran_low_t *output, int stride) { + int i; + tran_high_t a1, b1, c1, d1, e1; + const int16_t *ip_pass0 = input; + const tran_low_t *ip = NULL; + tran_low_t *op = output; + + for (i = 0; i < 4; i++) { + a1 = ip_pass0[0 * stride]; + b1 = ip_pass0[1 * stride]; + c1 = ip_pass0[2 * stride]; + d1 = ip_pass0[3 * stride]; + + a1 += b1; + d1 = d1 - c1; + e1 = (a1 - d1) >> 1; + b1 = e1 - b1; + c1 = e1 - c1; + a1 -= c1; + d1 += b1; + op[0] = (tran_low_t)a1; + op[4] = (tran_low_t)c1; + op[8] = (tran_low_t)d1; + op[12] = (tran_low_t)b1; + + ip_pass0++; + op++; + } + ip = output; + op = output; + + for (i = 0; i < 4; i++) { + a1 = ip[0]; + b1 = ip[1]; + c1 = ip[2]; + d1 = ip[3]; + + a1 += b1; + d1 -= c1; + e1 = (a1 - d1) >> 1; + b1 = e1 - b1; + c1 = e1 - c1; + a1 -= c1; + d1 += b1; + op[0] = (tran_low_t)(a1 * UNIT_QUANT_FACTOR); + op[1] = (tran_low_t)(c1 * UNIT_QUANT_FACTOR); + op[2] = (tran_low_t)(d1 * UNIT_QUANT_FACTOR); + op[3] = (tran_low_t)(b1 * UNIT_QUANT_FACTOR); + + ip += 4; + op += 4; + } +} + +void av1_fht16x16_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct16, fdct16 }, // DCT_DCT + { fadst16, fdct16 }, // ADST_DCT + { fdct16, fadst16 }, // DCT_ADST + { fadst16, fadst16 }, // ADST_ADST +#if CONFIG_EXT_TX + { fadst16, fdct16 }, // FLIPADST_DCT + { fdct16, fadst16 }, // DCT_FLIPADST + { fadst16, fadst16 }, // FLIPADST_FLIPADST + { fadst16, fadst16 }, // ADST_FLIPADST + { fadst16, fadst16 }, // FLIPADST_ADST + { fidtx16, fidtx16 }, // IDTX + { fdct16, fidtx16 }, // V_DCT + { fidtx16, fdct16 }, // H_DCT + { fadst16, fidtx16 }, // V_ADST + { fidtx16, fadst16 }, // H_ADST + { fadst16, fidtx16 }, // V_FLIPADST + { fidtx16, fadst16 }, // H_FLIPADST +#endif // CONFIG_EXT_TX + }; + + const transform_2d ht = FHT[tx_type]; + tran_low_t out[256]; + int i, j; + tran_low_t temp_in[16], temp_out[16]; + +#if CONFIG_EXT_TX + int16_t flipped_input[16 * 16]; + maybe_flip_input(&input, &stride, 16, 16, flipped_input, tx_type); +#endif + + // Columns + for (i = 0; i < 16; ++i) { + for (j = 0; j < 16; ++j) temp_in[j] = input[j * stride + i] * 4; + ht.cols(temp_in, temp_out); + for (j = 0; j < 16; ++j) + out[j * 16 + i] = (temp_out[j] + 1 + (temp_out[j] < 0)) >> 2; + } + + // Rows + for (i = 0; i < 16; ++i) { + for (j = 0; j < 16; ++j) temp_in[j] = out[j + i * 16]; + ht.rows(temp_in, temp_out); + for (j = 0; j < 16; ++j) output[j + i * 16] = temp_out[j]; + } +} + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_fht4x4_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht4x4_c(input, output, stride, tx_type); +} + +void av1_highbd_fht4x8_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht4x8_c(input, output, stride, tx_type); +} + +void av1_highbd_fht8x4_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht8x4_c(input, output, stride, tx_type); +} + +void av1_highbd_fht8x16_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht8x16_c(input, output, stride, tx_type); +} + +void av1_highbd_fht16x8_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht16x8_c(input, output, stride, tx_type); +} + +void av1_highbd_fht16x32_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht16x32_c(input, output, stride, tx_type); +} + +void av1_highbd_fht32x16_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht32x16_c(input, output, stride, tx_type); +} + +void av1_highbd_fht4x16_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht4x16_c(input, output, stride, tx_type); +} + +void av1_highbd_fht16x4_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht16x4_c(input, output, stride, tx_type); +} + +void av1_highbd_fht8x32_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht8x32_c(input, output, stride, tx_type); +} + +void av1_highbd_fht32x8_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht32x8_c(input, output, stride, tx_type); +} + +void av1_highbd_fht8x8_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht8x8_c(input, output, stride, tx_type); +} + +void av1_highbd_fwht4x4_c(const int16_t *input, tran_low_t *output, + int stride) { + av1_fwht4x4_c(input, output, stride); +} + +void av1_highbd_fht16x16_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht16x16_c(input, output, stride, tx_type); +} +#endif // CONFIG_HIGHBITDEPTH + +void av1_fht32x32_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct32, fdct32 }, // DCT_DCT +#if CONFIG_EXT_TX + { fhalfright32, fdct32 }, // ADST_DCT + { fdct32, fhalfright32 }, // DCT_ADST + { fhalfright32, fhalfright32 }, // ADST_ADST + { fhalfright32, fdct32 }, // FLIPADST_DCT + { fdct32, fhalfright32 }, // DCT_FLIPADST + { fhalfright32, fhalfright32 }, // FLIPADST_FLIPADST + { fhalfright32, fhalfright32 }, // ADST_FLIPADST + { fhalfright32, fhalfright32 }, // FLIPADST_ADST + { fidtx32, fidtx32 }, // IDTX + { fdct32, fidtx32 }, // V_DCT + { fidtx32, fdct32 }, // H_DCT + { fhalfright32, fidtx32 }, // V_ADST + { fidtx32, fhalfright32 }, // H_ADST + { fhalfright32, fidtx32 }, // V_FLIPADST + { fidtx32, fhalfright32 }, // H_FLIPADST +#endif + }; + const transform_2d ht = FHT[tx_type]; + tran_low_t out[1024]; + int i, j; + tran_low_t temp_in[32], temp_out[32]; + +#if CONFIG_EXT_TX + int16_t flipped_input[32 * 32]; + maybe_flip_input(&input, &stride, 32, 32, flipped_input, tx_type); +#endif + + // Columns + for (i = 0; i < 32; ++i) { + for (j = 0; j < 32; ++j) temp_in[j] = input[j * stride + i] * 4; + ht.cols(temp_in, temp_out); + for (j = 0; j < 32; ++j) + out[j * 32 + i] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 4); + } + + // Rows + for (i = 0; i < 32; ++i) { + for (j = 0; j < 32; ++j) temp_in[j] = out[j + i * 32]; + ht.rows(temp_in, temp_out); + for (j = 0; j < 32; ++j) output[j + i * 32] = temp_out[j]; + } +} + +#if CONFIG_TX64X64 +#if CONFIG_EXT_TX +static void fidtx64(const tran_low_t *input, tran_low_t *output) { + int i; + for (i = 0; i < 64; ++i) + output[i] = (tran_low_t)fdct_round_shift(input[i] * 4 * Sqrt2); +} + +// For use in lieu of ADST +static void fhalfright64(const tran_low_t *input, tran_low_t *output) { + int i; + tran_low_t inputhalf[32]; + for (i = 0; i < 32; ++i) { + output[32 + i] = (tran_low_t)fdct_round_shift(input[i] * 4 * Sqrt2); + } + // Multiply input by sqrt(2) + for (i = 0; i < 32; ++i) { + inputhalf[i] = (tran_low_t)fdct_round_shift(input[i + 32] * Sqrt2); + } + fdct32(inputhalf, output); + // Note overall scaling factor is 2 times unitary +} +#endif // CONFIG_EXT_TX + +static void fdct64_col(const tran_low_t *input, tran_low_t *output) { + int32_t in[64], out[64]; + int i; + for (i = 0; i < 64; ++i) in[i] = (int32_t)input[i]; + av1_fdct64_new(in, out, fwd_cos_bit_col_dct_dct_64, + fwd_stage_range_col_dct_dct_64); + for (i = 0; i < 64; ++i) output[i] = (tran_low_t)out[i]; +} + +static void fdct64_row(const tran_low_t *input, tran_low_t *output) { + int32_t in[64], out[64]; + int i; + for (i = 0; i < 64; ++i) in[i] = (int32_t)input[i]; + av1_fdct64_new(in, out, fwd_cos_bit_row_dct_dct_64, + fwd_stage_range_row_dct_dct_64); + for (i = 0; i < 64; ++i) output[i] = (tran_low_t)out[i]; +} + +void av1_fht64x64_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + static const transform_2d FHT[] = { + { fdct64_col, fdct64_row }, // DCT_DCT +#if CONFIG_EXT_TX + { fhalfright64, fdct64_row }, // ADST_DCT + { fdct64_col, fhalfright64 }, // DCT_ADST + { fhalfright64, fhalfright64 }, // ADST_ADST + { fhalfright64, fdct64_row }, // FLIPADST_DCT + { fdct64_col, fhalfright64 }, // DCT_FLIPADST + { fhalfright64, fhalfright64 }, // FLIPADST_FLIPADST + { fhalfright64, fhalfright64 }, // ADST_FLIPADST + { fhalfright64, fhalfright64 }, // FLIPADST_ADST + { fidtx64, fidtx64 }, // IDTX + { fdct64_col, fidtx64 }, // V_DCT + { fidtx64, fdct64_row }, // H_DCT + { fhalfright64, fidtx64 }, // V_ADST + { fidtx64, fhalfright64 }, // H_ADST + { fhalfright64, fidtx64 }, // V_FLIPADST + { fidtx64, fhalfright64 }, // H_FLIPADST +#endif + }; + const transform_2d ht = FHT[tx_type]; + tran_low_t out[4096]; + int i, j; + tran_low_t temp_in[64], temp_out[64]; +#if CONFIG_EXT_TX + int16_t flipped_input[64 * 64]; + maybe_flip_input(&input, &stride, 64, 64, flipped_input, tx_type); +#endif + // Columns + for (i = 0; i < 64; ++i) { + for (j = 0; j < 64; ++j) temp_in[j] = input[j * stride + i]; + ht.cols(temp_in, temp_out); + for (j = 0; j < 64; ++j) + out[j * 64 + i] = (temp_out[j] + 1 + (temp_out[j] > 0)) >> 2; + } + + // Rows + for (i = 0; i < 64; ++i) { + for (j = 0; j < 64; ++j) temp_in[j] = out[j + i * 64]; + ht.rows(temp_in, temp_out); + for (j = 0; j < 64; ++j) + output[j + i * 64] = + (tran_low_t)((temp_out[j] + 1 + (temp_out[j] < 0)) >> 2); + } +} +#endif // CONFIG_TX64X64 + +#if CONFIG_EXT_TX +// Forward identity transform. +void av1_fwd_idtx_c(const int16_t *src_diff, tran_low_t *coeff, int stride, + int bs, int tx_type) { + int r, c; + const int shift = bs < 32 ? 3 : (bs < 64 ? 2 : 1); + if (tx_type == IDTX) { + for (r = 0; r < bs; ++r) { + for (c = 0; c < bs; ++c) coeff[c] = src_diff[c] * (1 << shift); + src_diff += stride; + coeff += bs; + } + } +} +#endif // CONFIG_EXT_TX + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_fht32x32_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht32x32_c(input, output, stride, tx_type); +} + +#if CONFIG_TX64X64 +void av1_highbd_fht64x64_c(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + av1_fht64x64_c(input, output, stride, tx_type); +} +#endif // CONFIG_TX64X64 +#endif // CONFIG_HIGHBITDEPTH +#endif // !AV1_DCT_GTEST diff --git a/third_party/aom/av1/encoder/encint.h b/third_party/aom/av1/encoder/encint.h new file mode 100644 index 0000000000..30ea8521fb --- /dev/null +++ b/third_party/aom/av1/encoder/encint.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +/* clang-format off */ + +#if !defined(_encint_H) +# define _encint_H (1) + +typedef struct daala_enc_ctx od_enc_ctx; +typedef struct od_params_ctx od_params_ctx; +typedef struct od_rollback_buffer od_rollback_buffer; + +# include "aom_dsp/entenc.h" +# include "av1/common/odintrin.h" +# include "av1/common/pvq_state.h" + +struct daala_enc_ctx{ + /* Stores context-adaptive CDFs for PVQ. */ + od_state state; + /* AOM entropy encoder. */ + aom_writer w; + int use_activity_masking; + /* Mode of quantization matrice : FLAT (0) or HVS (1) */ + int qm; + /*Normalized PVQ lambda for use where we've already performed + quantization.*/ + double pvq_norm_lambda; + double pvq_norm_lambda_dc; +}; + +// from daalaenc.h +/**The encoder context.*/ +typedef struct daala_enc_ctx daala_enc_ctx; + +/** Holds important encoder information so we can roll back decisions */ +struct od_rollback_buffer { + od_ec_enc ec; + od_adapt_ctx adapt; +}; + +void od_encode_checkpoint(const daala_enc_ctx *enc, od_rollback_buffer *rbuf); +void od_encode_rollback(daala_enc_ctx *enc, const od_rollback_buffer *rbuf); + +#endif diff --git a/third_party/aom/av1/encoder/encodeframe.c b/third_party/aom/av1/encoder/encodeframe.c new file mode 100644 index 0000000000..d254157e72 --- /dev/null +++ b/third_party/aom/av1/encoder/encodeframe.c @@ -0,0 +1,7160 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" +#include "./aom_config.h" + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/binary_codes_writer.h" +#include "aom_ports/mem.h" +#include "aom_ports/aom_timer.h" +#include "aom_ports/system_state.h" + +#include "av1/common/common.h" +#include "av1/common/entropy.h" +#include "av1/common/entropymode.h" +#include "av1/common/idct.h" +#include "av1/common/mv.h" +#include "av1/common/mvref_common.h" +#include "av1/common/pred_common.h" +#include "av1/common/quant_common.h" +#include "av1/common/reconintra.h" +#include "av1/common/reconinter.h" +#include "av1/common/seg_common.h" +#include "av1/common/tile_common.h" + +#include "av1/encoder/aq_complexity.h" +#include "av1/encoder/aq_cyclicrefresh.h" +#include "av1/encoder/aq_variance.h" +#if CONFIG_SUPERTX +#include "av1/encoder/cost.h" +#endif +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#include "av1/common/warped_motion.h" +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION +#if CONFIG_GLOBAL_MOTION +#include "av1/encoder/global_motion.h" +#endif // CONFIG_GLOBAL_MOTION +#include "av1/encoder/encodeframe.h" +#include "av1/encoder/encodemb.h" +#include "av1/encoder/encodemv.h" +#if CONFIG_LV_MAP +#include "av1/encoder/encodetxb.h" +#endif +#include "av1/encoder/ethread.h" +#include "av1/encoder/extend.h" +#include "av1/encoder/rd.h" +#include "av1/encoder/rdopt.h" +#include "av1/encoder/segmentation.h" +#include "av1/encoder/tokenize.h" +#if CONFIG_PVQ +#include "av1/common/pvq.h" +#include "av1/encoder/pvq_encoder.h" +#endif +#if CONFIG_HIGHBITDEPTH +#define IF_HBD(...) __VA_ARGS__ +#else +#define IF_HBD(...) +#endif // CONFIG_HIGHBITDEPTH + +static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td, + TOKENEXTRA **t, RUN_TYPE dry_run, int mi_row, + int mi_col, BLOCK_SIZE bsize, + PICK_MODE_CONTEXT *ctx, int *rate); + +#if CONFIG_SUPERTX +static int check_intra_b(PICK_MODE_CONTEXT *ctx); + +static int check_intra_sb(const AV1_COMP *cpi, const TileInfo *const tile, + int mi_row, int mi_col, BLOCK_SIZE bsize, + PC_TREE *pc_tree); +static void predict_superblock(const AV1_COMP *const cpi, ThreadData *td, +#if CONFIG_EXT_INTER + int mi_row_ori, int mi_col_ori, +#endif // CONFIG_EXT_INTER + int mi_row_pred, int mi_col_pred, + BLOCK_SIZE bsize_pred, int b_sub8x8, int block); +static int check_supertx_sb(BLOCK_SIZE bsize, TX_SIZE supertx_size, + PC_TREE *pc_tree); +static void predict_sb_complex(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, int mi_row, + int mi_col, int mi_row_ori, int mi_col_ori, + RUN_TYPE dry_run, BLOCK_SIZE bsize, + BLOCK_SIZE top_bsize, uint8_t *dst_buf[3], + int dst_stride[3], PC_TREE *pc_tree); +static void update_state_sb_supertx(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, int mi_row, + int mi_col, BLOCK_SIZE bsize, + RUN_TYPE dry_run, PC_TREE *pc_tree); +static void rd_supertx_sb(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, int mi_row, int mi_col, + BLOCK_SIZE bsize, int *tmp_rate, int64_t *tmp_dist, + TX_TYPE *best_tx, PC_TREE *pc_tree); +#endif // CONFIG_SUPERTX + +// This is used as a reference when computing the source variance for the +// purposes of activity masking. +// Eventually this should be replaced by custom no-reference routines, +// which will be faster. +static const uint8_t AV1_VAR_OFFS[MAX_SB_SIZE] = { + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, +#if CONFIG_EXT_PARTITION + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 +#endif // CONFIG_EXT_PARTITION +}; + +#if CONFIG_HIGHBITDEPTH +static const uint16_t AV1_HIGH_VAR_OFFS_8[MAX_SB_SIZE] = { + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, +#if CONFIG_EXT_PARTITION + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 +#endif // CONFIG_EXT_PARTITION +}; + +static const uint16_t AV1_HIGH_VAR_OFFS_10[MAX_SB_SIZE] = { + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, +#if CONFIG_EXT_PARTITION + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, + 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4 +#endif // CONFIG_EXT_PARTITION +}; + +static const uint16_t AV1_HIGH_VAR_OFFS_12[MAX_SB_SIZE] = { + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, +#if CONFIG_EXT_PARTITION + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, + 128 * 16 +#endif // CONFIG_EXT_PARTITION +}; +#endif // CONFIG_HIGHBITDEPTH + +unsigned int av1_get_sby_perpixel_variance(const AV1_COMP *cpi, + const struct buf_2d *ref, + BLOCK_SIZE bs) { + unsigned int sse; + const unsigned int var = + cpi->fn_ptr[bs].vf(ref->buf, ref->stride, AV1_VAR_OFFS, 0, &sse); + return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]); +} + +#if CONFIG_HIGHBITDEPTH +unsigned int av1_high_get_sby_perpixel_variance(const AV1_COMP *cpi, + const struct buf_2d *ref, + BLOCK_SIZE bs, int bd) { + unsigned int var, sse; + switch (bd) { + case 10: + var = + cpi->fn_ptr[bs].vf(ref->buf, ref->stride, + CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_10), 0, &sse); + break; + case 12: + var = + cpi->fn_ptr[bs].vf(ref->buf, ref->stride, + CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_12), 0, &sse); + break; + case 8: + default: + var = + cpi->fn_ptr[bs].vf(ref->buf, ref->stride, + CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_8), 0, &sse); + break; + } + return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]); +} +#endif // CONFIG_HIGHBITDEPTH + +static unsigned int get_sby_perpixel_diff_variance(const AV1_COMP *const cpi, + const struct buf_2d *ref, + int mi_row, int mi_col, + BLOCK_SIZE bs) { + unsigned int sse, var; + uint8_t *last_y; + const YV12_BUFFER_CONFIG *last = get_ref_frame_buffer(cpi, LAST_FRAME); + + assert(last != NULL); + last_y = + &last->y_buffer[mi_row * MI_SIZE * last->y_stride + mi_col * MI_SIZE]; + var = cpi->fn_ptr[bs].vf(ref->buf, ref->stride, last_y, last->y_stride, &sse); + return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]); +} + +static BLOCK_SIZE get_rd_var_based_fixed_partition(AV1_COMP *cpi, MACROBLOCK *x, + int mi_row, int mi_col) { + unsigned int var = get_sby_perpixel_diff_variance( + cpi, &x->plane[0].src, mi_row, mi_col, BLOCK_64X64); + if (var < 8) + return BLOCK_64X64; + else if (var < 128) + return BLOCK_32X32; + else if (var < 2048) + return BLOCK_16X16; + else + return BLOCK_8X8; +} + +// Lighter version of set_offsets that only sets the mode info +// pointers. +static void set_mode_info_offsets(const AV1_COMP *const cpi, + MACROBLOCK *const x, MACROBLOCKD *const xd, + int mi_row, int mi_col) { + const AV1_COMMON *const cm = &cpi->common; + const int idx_str = xd->mi_stride * mi_row + mi_col; + xd->mi = cm->mi_grid_visible + idx_str; + xd->mi[0] = cm->mi + idx_str; + x->mbmi_ext = cpi->mbmi_ext_base + (mi_row * cm->mi_cols + mi_col); +} + +static void set_offsets_without_segment_id(const AV1_COMP *const cpi, + const TileInfo *const tile, + MACROBLOCK *const x, int mi_row, + int mi_col, BLOCK_SIZE bsize) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + const int mi_width = mi_size_wide[bsize]; + const int mi_height = mi_size_high[bsize]; + + set_skip_context(xd, mi_row, mi_col); + + set_mode_info_offsets(cpi, x, xd, mi_row, mi_col); + +#if CONFIG_VAR_TX + xd->above_txfm_context = cm->above_txfm_context + mi_col; + xd->left_txfm_context = + xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK); + xd->max_tx_size = max_txsize_lookup[bsize]; +#endif + + // Set up destination pointers. + av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, + mi_col); + + // Set up limit values for MV components. + // Mv beyond the range do not produce new/different prediction block. + x->mv_limits.row_min = + -(((mi_row + mi_height) * MI_SIZE) + AOM_INTERP_EXTEND); + x->mv_limits.col_min = -(((mi_col + mi_width) * MI_SIZE) + AOM_INTERP_EXTEND); + x->mv_limits.row_max = (cm->mi_rows - mi_row) * MI_SIZE + AOM_INTERP_EXTEND; + x->mv_limits.col_max = (cm->mi_cols - mi_col) * MI_SIZE + AOM_INTERP_EXTEND; + + set_plane_n4(xd, mi_width, mi_height); + + // Set up distance of MB to edge of frame in 1/8th pel units. + assert(!(mi_col & (mi_width - 1)) && !(mi_row & (mi_height - 1))); + set_mi_row_col(xd, tile, mi_row, mi_height, mi_col, mi_width, +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + + // Set up source buffers. + av1_setup_src_planes(x, cpi->source, mi_row, mi_col); + + // R/D setup. + x->rddiv = cpi->rd.RDDIV; + x->rdmult = cpi->rd.RDMULT; + + // required by av1_append_sub8x8_mvs_for_idx() and av1_find_best_ref_mvs() + xd->tile = *tile; +} + +static void set_offsets(const AV1_COMP *const cpi, const TileInfo *const tile, + MACROBLOCK *const x, int mi_row, int mi_col, + BLOCK_SIZE bsize) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *mbmi; + const struct segmentation *const seg = &cm->seg; + + set_offsets_without_segment_id(cpi, tile, x, mi_row, mi_col, bsize); + + mbmi = &xd->mi[0]->mbmi; + + // Setup segment ID. + if (seg->enabled) { + if (!cpi->vaq_refresh) { + const uint8_t *const map = + seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map; + mbmi->segment_id = get_segment_id(cm, map, bsize, mi_row, mi_col); + } + av1_init_plane_quantizers(cpi, x, mbmi->segment_id); + } else { + mbmi->segment_id = 0; + } + +#if CONFIG_SUPERTX + mbmi->segment_id_supertx = MAX_SEGMENTS; +#endif // CONFIG_SUPERTX +} + +#if CONFIG_SUPERTX +static void set_offsets_supertx(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, int mi_row, + int mi_col, BLOCK_SIZE bsize) { + MACROBLOCK *const x = &td->mb; + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + const int mi_width = mi_size_wide[bsize]; + const int mi_height = mi_size_high[bsize]; +#if CONFIG_DEPENDENT_HORZTILES + set_mode_info_offsets(cpi, x, xd, mi_row, mi_col, cm->dependent_horz_tiles); +#else + set_mode_info_offsets(cpi, x, xd, mi_row, mi_col); +#endif + + // Set up distance of MB to edge of frame in 1/8th pel units. + assert(!(mi_col & (mi_width - 1)) && !(mi_row & (mi_height - 1))); + set_mi_row_col(xd, tile, mi_row, mi_height, mi_col, mi_width, +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); +} + +static void set_offsets_extend(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, int mi_row_pred, + int mi_col_pred, int mi_row_ori, int mi_col_ori, + BLOCK_SIZE bsize_pred) { + // Used in supertx + // (mi_row_ori, mi_col_ori, bsize_ori): region for mv + // (mi_row_pred, mi_col_pred, bsize_pred): region to predict + MACROBLOCK *const x = &td->mb; + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + const int mi_width = mi_size_wide[bsize_pred]; + const int mi_height = mi_size_high[bsize_pred]; + +#if CONFIG_DEPENDENT_HORZTILES + set_mode_info_offsets(cpi, x, xd, mi_row_ori, mi_col_ori, + cm->dependent_horz_tiles); +#else + set_mode_info_offsets(cpi, x, xd, mi_row_ori, mi_col_ori); +#endif + + // Set up limit values for MV components. + // Mv beyond the range do not produce new/different prediction block. + x->mv_limits.row_min = + -(((mi_row_pred + mi_height) * MI_SIZE) + AOM_INTERP_EXTEND); + x->mv_limits.col_min = + -(((mi_col_pred + mi_width) * MI_SIZE) + AOM_INTERP_EXTEND); + x->mv_limits.row_max = + (cm->mi_rows - mi_row_pred) * MI_SIZE + AOM_INTERP_EXTEND; + x->mv_limits.col_max = + (cm->mi_cols - mi_col_pred) * MI_SIZE + AOM_INTERP_EXTEND; + +// Set up distance of MB to edge of frame in 1/8th pel units. +#if !CONFIG_CB4X4 + assert(!(mi_col_pred & (mi_width - mi_size_wide[BLOCK_8X8])) && + !(mi_row_pred & (mi_height - mi_size_high[BLOCK_8X8]))); +#endif + set_mi_row_col(xd, tile, mi_row_pred, mi_height, mi_col_pred, mi_width, +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + xd->up_available = (mi_row_ori > tile->mi_row_start); + xd->left_available = (mi_col_ori > tile->mi_col_start); + + // R/D setup. + x->rddiv = cpi->rd.RDDIV; + x->rdmult = cpi->rd.RDMULT; +} + +static void set_segment_id_supertx(const AV1_COMP *const cpi, + MACROBLOCK *const x, const int mi_row, + const int mi_col, const BLOCK_SIZE bsize) { + const AV1_COMMON *cm = &cpi->common; + const struct segmentation *seg = &cm->seg; + const int miw = AOMMIN(mi_size_wide[bsize], cm->mi_cols - mi_col); + const int mih = AOMMIN(mi_size_high[bsize], cm->mi_rows - mi_row); + const int mi_offset = mi_row * cm->mi_stride + mi_col; + MODE_INFO **const mip = cm->mi_grid_visible + mi_offset; + int r, c; + int seg_id_supertx = MAX_SEGMENTS; + + if (!seg->enabled) { + seg_id_supertx = 0; + } else { + // Find the minimum segment_id + for (r = 0; r < mih; r++) + for (c = 0; c < miw; c++) + seg_id_supertx = + AOMMIN(mip[r * cm->mi_stride + c]->mbmi.segment_id, seg_id_supertx); + assert(0 <= seg_id_supertx && seg_id_supertx < MAX_SEGMENTS); + + // Initialize plane quantisers + av1_init_plane_quantizers(cpi, x, seg_id_supertx); + } + + // Assign the the segment_id back to segment_id_supertx + for (r = 0; r < mih; r++) + for (c = 0; c < miw; c++) + mip[r * cm->mi_stride + c]->mbmi.segment_id_supertx = seg_id_supertx; +} +#endif // CONFIG_SUPERTX + +static void set_block_size(AV1_COMP *const cpi, MACROBLOCK *const x, + MACROBLOCKD *const xd, int mi_row, int mi_col, + BLOCK_SIZE bsize) { + if (cpi->common.mi_cols > mi_col && cpi->common.mi_rows > mi_row) { + const int mi_width = AOMMAX(mi_size_wide[bsize], mi_size_wide[BLOCK_8X8]); + const int mi_height = AOMMAX(mi_size_high[bsize], mi_size_high[BLOCK_8X8]); + for (int r = 0; r < mi_height; ++r) { + for (int c = 0; c < mi_width; ++c) { + set_mode_info_offsets(cpi, x, xd, mi_row + r, mi_col + c); + xd->mi[0]->mbmi.sb_type = bsize; + } + } + } +} + +static void set_vt_partitioning(AV1_COMP *cpi, MACROBLOCK *const x, + MACROBLOCKD *const xd, VAR_TREE *vt, int mi_row, + int mi_col, const int64_t *const threshold, + const BLOCK_SIZE *const bsize_min) { + AV1_COMMON *const cm = &cpi->common; + const int hbw = mi_size_wide[vt->bsize] / 2; + const int hbh = mi_size_high[vt->bsize] / 2; + const int has_cols = mi_col + hbw < cm->mi_cols; + const int has_rows = mi_row + hbh < cm->mi_rows; + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + assert(vt->bsize >= BLOCK_8X8); + + assert(hbh == hbw); + + if (vt->bsize == BLOCK_8X8 && cm->frame_type != KEY_FRAME) { + set_block_size(cpi, x, xd, mi_row, mi_col, BLOCK_8X8); + return; + } + + if (vt->force_split || (!has_cols && !has_rows)) goto split; + + // For bsize=bsize_min (16x16/8x8 for 8x8/4x4 downsampling), select if + // variance is below threshold, otherwise split will be selected. + // No check for vert/horiz split as too few samples for variance. + if (vt->bsize == bsize_min[0]) { + if (has_cols && has_rows && vt->variances.none.variance < threshold[0]) { + set_block_size(cpi, x, xd, mi_row, mi_col, vt->bsize); + return; + } else { + BLOCK_SIZE subsize = get_subsize(vt->bsize, PARTITION_SPLIT); + set_block_size(cpi, x, xd, mi_row, mi_col, subsize); + if (vt->bsize > BLOCK_8X8) { + set_block_size(cpi, x, xd, mi_row, mi_col + hbw, subsize); + set_block_size(cpi, x, xd, mi_row + hbh, mi_col, subsize); + set_block_size(cpi, x, xd, mi_row + hbh, mi_col + hbw, subsize); + } + return; + } + } else if (vt->bsize > bsize_min[0]) { + // For key frame: take split for bsize above 32X32 or very high variance. + if (cm->frame_type == KEY_FRAME && + (vt->bsize > BLOCK_32X32 || + vt->variances.none.variance > (threshold[0] << 4))) { + goto split; + } + // If variance is low, take the bsize (no split). + if (has_cols && has_rows && vt->variances.none.variance < threshold[0]) { + set_block_size(cpi, x, xd, mi_row, mi_col, vt->bsize); + return; + } + + // Check vertical split. + if (has_rows) { + BLOCK_SIZE subsize = get_subsize(vt->bsize, PARTITION_VERT); + if (vt->variances.vert[0].variance < threshold[0] && + vt->variances.vert[1].variance < threshold[0] && + get_plane_block_size(subsize, &xd->plane[1]) < BLOCK_INVALID) { + set_block_size(cpi, x, xd, mi_row, mi_col, subsize); + set_block_size(cpi, x, xd, mi_row, mi_col + hbw, subsize); + return; + } + } + // Check horizontal split. + if (has_cols) { + BLOCK_SIZE subsize = get_subsize(vt->bsize, PARTITION_HORZ); + if (vt->variances.horz[0].variance < threshold[0] && + vt->variances.horz[1].variance < threshold[0] && + get_plane_block_size(subsize, &xd->plane[1]) < BLOCK_INVALID) { + set_block_size(cpi, x, xd, mi_row, mi_col, subsize); + set_block_size(cpi, x, xd, mi_row + hbh, mi_col, subsize); + return; + } + } + } + +split : { + set_vt_partitioning(cpi, x, xd, vt->split[0], mi_row, mi_col, threshold + 1, + bsize_min + 1); + set_vt_partitioning(cpi, x, xd, vt->split[1], mi_row, mi_col + hbw, + threshold + 1, bsize_min + 1); + set_vt_partitioning(cpi, x, xd, vt->split[2], mi_row + hbh, mi_col, + threshold + 1, bsize_min + 1); + set_vt_partitioning(cpi, x, xd, vt->split[3], mi_row + hbh, mi_col + hbw, + threshold + 1, bsize_min + 1); + return; +} +} + +// Set the variance split thresholds for following the block sizes: +// 0 - threshold_64x64, 1 - threshold_32x32, 2 - threshold_16x16, +// 3 - vbp_threshold_8x8. vbp_threshold_8x8 (to split to 4x4 partition) is +// currently only used on key frame. +static void set_vbp_thresholds(AV1_COMP *cpi, int64_t thresholds[], int q) { + AV1_COMMON *const cm = &cpi->common; + const int is_key_frame = (cm->frame_type == KEY_FRAME); + const int threshold_multiplier = is_key_frame ? 20 : 1; + const int64_t threshold_base = + (int64_t)(threshold_multiplier * cpi->y_dequant[q][1]); + if (is_key_frame) { + thresholds[1] = threshold_base; + thresholds[2] = threshold_base >> 2; + thresholds[3] = threshold_base >> 2; + thresholds[4] = threshold_base << 2; + } else { + thresholds[2] = threshold_base; + if (cm->width <= 352 && cm->height <= 288) { + thresholds[1] = threshold_base >> 2; + thresholds[3] = threshold_base << 3; + } else { + thresholds[1] = threshold_base; + thresholds[2] = (5 * threshold_base) >> 2; + if (cm->width >= 1920 && cm->height >= 1080) + thresholds[2] = (7 * threshold_base) >> 2; + thresholds[3] = threshold_base << cpi->oxcf.speed; + } + } + thresholds[0] = INT64_MIN; +} + +void av1_set_variance_partition_thresholds(AV1_COMP *cpi, int q) { + AV1_COMMON *const cm = &cpi->common; + SPEED_FEATURES *const sf = &cpi->sf; + const int is_key_frame = (cm->frame_type == KEY_FRAME); + if (sf->partition_search_type != VAR_BASED_PARTITION && + sf->partition_search_type != REFERENCE_PARTITION) { + return; + } else { + set_vbp_thresholds(cpi, cpi->vbp_thresholds, q); + // The thresholds below are not changed locally. + if (is_key_frame) { + cpi->vbp_threshold_sad = 0; + cpi->vbp_bsize_min = BLOCK_8X8; + } else { + if (cm->width <= 352 && cm->height <= 288) + cpi->vbp_threshold_sad = 100; + else + cpi->vbp_threshold_sad = (cpi->y_dequant[q][1] << 1) > 1000 + ? (cpi->y_dequant[q][1] << 1) + : 1000; + cpi->vbp_bsize_min = BLOCK_16X16; + } + cpi->vbp_threshold_minmax = 15 + (q >> 3); + } +} + +// Compute the minmax over the 8x8 subblocks. +static int compute_minmax_8x8(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, +#if CONFIG_HIGHBITDEPTH + int highbd, +#endif + int pixels_wide, int pixels_high) { + int k; + int minmax_max = 0; + int minmax_min = 255; + // Loop over the 4 8x8 subblocks. + for (k = 0; k < 4; k++) { + const int x8_idx = ((k & 1) << 3); + const int y8_idx = ((k >> 1) << 3); + int min = 0; + int max = 0; + if (x8_idx < pixels_wide && y8_idx < pixels_high) { + const int src_offset = y8_idx * src_stride + x8_idx; + const int ref_offset = y8_idx * ref_stride + x8_idx; +#if CONFIG_HIGHBITDEPTH + if (highbd) { + aom_highbd_minmax_8x8(src + src_offset, src_stride, ref + ref_offset, + ref_stride, &min, &max); + } else { + aom_minmax_8x8(src + src_offset, src_stride, ref + ref_offset, + ref_stride, &min, &max); + } +#else + aom_minmax_8x8(src + src_offset, src_stride, ref + ref_offset, ref_stride, + &min, &max); +#endif + if ((max - min) > minmax_max) minmax_max = (max - min); + if ((max - min) < minmax_min) minmax_min = (max - min); + } + } + return (minmax_max - minmax_min); +} + +#if CONFIG_HIGHBITDEPTH +static INLINE int avg_4x4(const uint8_t *const src, const int stride, + const int highbd) { + if (highbd) { + return aom_highbd_avg_4x4(src, stride); + } else { + return aom_avg_4x4(src, stride); + } +} +#else +static INLINE int avg_4x4(const uint8_t *const src, const int stride) { + return aom_avg_4x4(src, stride); +} +#endif + +#if CONFIG_HIGHBITDEPTH +static INLINE int avg_8x8(const uint8_t *const src, const int stride, + const int highbd) { + if (highbd) { + return aom_highbd_avg_8x8(src, stride); + } else { + return aom_avg_8x8(src, stride); + } +} +#else +static INLINE int avg_8x8(const uint8_t *const src, const int stride) { + return aom_avg_8x8(src, stride); +} +#endif + +static void init_variance_tree(VAR_TREE *const vt, +#if CONFIG_HIGHBITDEPTH + const int highbd, +#endif + BLOCK_SIZE bsize, BLOCK_SIZE leaf_size, + const int width, const int height, + const uint8_t *const src, const int src_stride, + const uint8_t *const ref, const int ref_stride) { + assert(bsize >= leaf_size); + + vt->bsize = bsize; + + vt->force_split = 0; + + vt->src = src; + vt->src_stride = src_stride; + vt->ref = ref; + vt->ref_stride = ref_stride; + + vt->width = width; + vt->height = height; + +#if CONFIG_HIGHBITDEPTH + vt->highbd = highbd; +#endif // CONFIG_HIGHBITDEPTH + + if (bsize > leaf_size) { + const BLOCK_SIZE subsize = get_subsize(bsize, PARTITION_SPLIT); + const int px = block_size_wide[subsize]; + + init_variance_tree(vt->split[0], +#if CONFIG_HIGHBITDEPTH + highbd, +#endif // CONFIG_HIGHBITDEPTH + subsize, leaf_size, AOMMIN(px, width), + AOMMIN(px, height), src, src_stride, ref, ref_stride); + init_variance_tree(vt->split[1], +#if CONFIG_HIGHBITDEPTH + highbd, +#endif // CONFIG_HIGHBITDEPTH + subsize, leaf_size, width - px, AOMMIN(px, height), + src + px, src_stride, ref + px, ref_stride); + init_variance_tree(vt->split[2], +#if CONFIG_HIGHBITDEPTH + highbd, +#endif // CONFIG_HIGHBITDEPTH + subsize, leaf_size, AOMMIN(px, width), height - px, + src + px * src_stride, src_stride, ref + px * ref_stride, + ref_stride); + init_variance_tree(vt->split[3], +#if CONFIG_HIGHBITDEPTH + highbd, +#endif // CONFIG_HIGHBITDEPTH + subsize, leaf_size, width - px, height - px, + src + px * src_stride + px, src_stride, + ref + px * ref_stride + px, ref_stride); + } +} + +// Fill the variance tree based on averaging pixel values (sub-sampling), at +// the leaf node size. +static void fill_variance_tree(VAR_TREE *const vt, const BLOCK_SIZE leaf_size) { + if (vt->bsize > leaf_size) { + fill_variance_tree(vt->split[0], leaf_size); + fill_variance_tree(vt->split[1], leaf_size); + fill_variance_tree(vt->split[2], leaf_size); + fill_variance_tree(vt->split[3], leaf_size); + fill_variance_node(vt); + } else if (vt->width <= 0 || vt->height <= 0) { + fill_variance(0, 0, 0, &vt->variances.none); + } else { + unsigned int sse = 0; + int sum = 0; + int src_avg; + int ref_avg; + assert(leaf_size == BLOCK_4X4 || leaf_size == BLOCK_8X8); + if (leaf_size == BLOCK_4X4) { + src_avg = avg_4x4(vt->src, vt->src_stride IF_HBD(, vt->highbd)); + ref_avg = avg_4x4(vt->ref, vt->ref_stride IF_HBD(, vt->highbd)); + } else { + src_avg = avg_8x8(vt->src, vt->src_stride IF_HBD(, vt->highbd)); + ref_avg = avg_8x8(vt->ref, vt->ref_stride IF_HBD(, vt->highbd)); + } + sum = src_avg - ref_avg; + sse = sum * sum; + fill_variance(sse, sum, 0, &vt->variances.none); + } +} + +static void refine_variance_tree(VAR_TREE *const vt, const int64_t threshold) { + if (vt->bsize >= BLOCK_8X8) { + if (vt->bsize == BLOCK_16X16) { + if (vt->variances.none.variance <= threshold) + return; + else + vt->force_split = 0; + } + + refine_variance_tree(vt->split[0], threshold); + refine_variance_tree(vt->split[1], threshold); + refine_variance_tree(vt->split[2], threshold); + refine_variance_tree(vt->split[3], threshold); + + if (vt->bsize <= BLOCK_16X16) fill_variance_node(vt); + } else if (vt->width <= 0 || vt->height <= 0) { + fill_variance(0, 0, 0, &vt->variances.none); + } else { + const int src_avg = avg_4x4(vt->src, vt->src_stride IF_HBD(, vt->highbd)); + const int ref_avg = avg_4x4(vt->ref, vt->ref_stride IF_HBD(, vt->highbd)); + const int sum = src_avg - ref_avg; + const unsigned int sse = sum * sum; + assert(vt->bsize == BLOCK_4X4); + fill_variance(sse, sum, 0, &vt->variances.none); + } +} + +static int check_split_key_frame(VAR_TREE *const vt, const int64_t threshold) { + if (vt->bsize == BLOCK_32X32) { + vt->force_split = vt->variances.none.variance > threshold; + } else { + vt->force_split |= check_split_key_frame(vt->split[0], threshold); + vt->force_split |= check_split_key_frame(vt->split[1], threshold); + vt->force_split |= check_split_key_frame(vt->split[2], threshold); + vt->force_split |= check_split_key_frame(vt->split[3], threshold); + } + return vt->force_split; +} + +static int check_split(AV1_COMP *const cpi, VAR_TREE *const vt, + const int segment_id, const int64_t *const thresholds) { + if (vt->bsize == BLOCK_16X16) { + vt->force_split = vt->variances.none.variance > thresholds[0]; + if (!vt->force_split && vt->variances.none.variance > thresholds[-1] && + !cyclic_refresh_segment_id_boosted(segment_id)) { + // We have some nominal amount of 16x16 variance (based on average), + // compute the minmax over the 8x8 sub-blocks, and if above threshold, + // force split to 8x8 block for this 16x16 block. + int minmax = + compute_minmax_8x8(vt->src, vt->src_stride, vt->ref, vt->ref_stride, +#if CONFIG_HIGHBITDEPTH + vt->highbd, +#endif + vt->width, vt->height); + vt->force_split = minmax > cpi->vbp_threshold_minmax; + } + } else { + vt->force_split |= + check_split(cpi, vt->split[0], segment_id, thresholds + 1); + vt->force_split |= + check_split(cpi, vt->split[1], segment_id, thresholds + 1); + vt->force_split |= + check_split(cpi, vt->split[2], segment_id, thresholds + 1); + vt->force_split |= + check_split(cpi, vt->split[3], segment_id, thresholds + 1); + + if (vt->bsize == BLOCK_32X32 && !vt->force_split) { + vt->force_split = vt->variances.none.variance > thresholds[0]; + } + } + + return vt->force_split; +} + +// This function chooses partitioning based on the variance between source and +// reconstructed last (or golden), where variance is computed for down-sampled +// inputs. +static void choose_partitioning(AV1_COMP *const cpi, ThreadData *const td, + const TileInfo *const tile, MACROBLOCK *const x, + const int mi_row, const int mi_col) { + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + VAR_TREE *const vt = td->var_root[cm->mib_size_log2 - MIN_MIB_SIZE_LOG2]; +#if CONFIG_DUAL_FILTER + int i; +#endif + const uint8_t *src; + const uint8_t *ref; + int src_stride; + int ref_stride; + int pixels_wide = MI_SIZE * mi_size_wide[cm->sb_size]; + int pixels_high = MI_SIZE * mi_size_high[cm->sb_size]; + int64_t thresholds[5] = { + cpi->vbp_thresholds[0], cpi->vbp_thresholds[1], cpi->vbp_thresholds[2], + cpi->vbp_thresholds[3], cpi->vbp_thresholds[4], + }; + BLOCK_SIZE bsize_min[5] = { BLOCK_16X16, BLOCK_16X16, BLOCK_16X16, + cpi->vbp_bsize_min, BLOCK_8X8 }; + const int start_level = cm->sb_size == BLOCK_64X64 ? 1 : 0; + const int64_t *const thre = thresholds + start_level; + const BLOCK_SIZE *const bmin = bsize_min + start_level; + + const int is_key_frame = (cm->frame_type == KEY_FRAME); + const int low_res = (cm->width <= 352 && cm->height <= 288); + + int segment_id = CR_SEGMENT_ID_BASE; + + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cm->seg.enabled) { + const uint8_t *const map = + cm->seg.update_map ? cpi->segmentation_map : cm->last_frame_seg_map; + segment_id = get_segment_id(cm, map, cm->sb_size, mi_row, mi_col); + + if (cyclic_refresh_segment_id_boosted(segment_id)) { + int q = av1_get_qindex(&cm->seg, segment_id, cm->base_qindex); + set_vbp_thresholds(cpi, thresholds, q); + } + } + + set_offsets(cpi, tile, x, mi_row, mi_col, cm->sb_size); + + if (xd->mb_to_right_edge < 0) pixels_wide += (xd->mb_to_right_edge >> 3); + if (xd->mb_to_bottom_edge < 0) pixels_high += (xd->mb_to_bottom_edge >> 3); + + src = x->plane[0].src.buf; + src_stride = x->plane[0].src.stride; + + if (!is_key_frame) { + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, LAST_FRAME); + const YV12_BUFFER_CONFIG *yv12_g = get_ref_frame_buffer(cpi, GOLDEN_FRAME); + unsigned int y_sad, y_sad_g; + + const int hbs = cm->mib_size / 2; + const int split_vert = mi_col + hbs >= cm->mi_cols; + const int split_horz = mi_row + hbs >= cm->mi_rows; + BLOCK_SIZE bsize; + + if (split_vert && split_horz) + bsize = get_subsize(cm->sb_size, PARTITION_SPLIT); + else if (split_vert) + bsize = get_subsize(cm->sb_size, PARTITION_VERT); + else if (split_horz) + bsize = get_subsize(cm->sb_size, PARTITION_HORZ); + else + bsize = cm->sb_size; + + assert(yv12 != NULL); + + if (yv12_g && yv12_g != yv12) { + av1_setup_pre_planes(xd, 0, yv12_g, mi_row, mi_col, + &cm->frame_refs[GOLDEN_FRAME - 1].sf); + y_sad_g = cpi->fn_ptr[bsize].sdf( + x->plane[0].src.buf, x->plane[0].src.stride, xd->plane[0].pre[0].buf, + xd->plane[0].pre[0].stride); + } else { + y_sad_g = UINT_MAX; + } + + av1_setup_pre_planes(xd, 0, yv12, mi_row, mi_col, + &cm->frame_refs[LAST_FRAME - 1].sf); + mbmi->ref_frame[0] = LAST_FRAME; + mbmi->ref_frame[1] = NONE_FRAME; + mbmi->sb_type = cm->sb_size; + mbmi->mv[0].as_int = 0; +#if CONFIG_DUAL_FILTER + for (i = 0; i < 4; ++i) mbmi->interp_filter[i] = BILINEAR; +#else + mbmi->interp_filter = BILINEAR; +#endif + + y_sad = av1_int_pro_motion_estimation(cpi, x, bsize, mi_row, mi_col); + + if (y_sad_g < y_sad) { + av1_setup_pre_planes(xd, 0, yv12_g, mi_row, mi_col, + &cm->frame_refs[GOLDEN_FRAME - 1].sf); + mbmi->ref_frame[0] = GOLDEN_FRAME; + mbmi->mv[0].as_int = 0; + y_sad = y_sad_g; + } else { + x->pred_mv[LAST_FRAME] = mbmi->mv[0].as_mv; + } + + av1_build_inter_predictors_sb(xd, mi_row, mi_col, NULL, cm->sb_size); + + ref = xd->plane[0].dst.buf; + ref_stride = xd->plane[0].dst.stride; + + // If the y_sad is very small, take the largest partition and exit. + // Don't check on boosted segment for now, as largest is suppressed there. + if (segment_id == CR_SEGMENT_ID_BASE && y_sad < cpi->vbp_threshold_sad) { + if (!split_vert && !split_horz) { + set_block_size(cpi, x, xd, mi_row, mi_col, cm->sb_size); + return; + } + } + } else { + ref = AV1_VAR_OFFS; + ref_stride = 0; +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + switch (xd->bd) { + case 10: ref = CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_10); break; + case 12: ref = CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_12); break; + case 8: + default: ref = CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_8); break; + } + } +#endif // CONFIG_HIGHBITDEPTH + } + + init_variance_tree( + vt, +#if CONFIG_HIGHBITDEPTH + xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH, +#endif // CONFIG_HIGHBITDEPTH + cm->sb_size, (is_key_frame || low_res) ? BLOCK_4X4 : BLOCK_8X8, + pixels_wide, pixels_high, src, src_stride, ref, ref_stride); + + // Fill in the entire tree of variances and compute splits. + if (is_key_frame) { + fill_variance_tree(vt, BLOCK_4X4); + check_split_key_frame(vt, thre[1]); + } else { + fill_variance_tree(vt, BLOCK_8X8); + check_split(cpi, vt, segment_id, thre); + if (low_res) { + refine_variance_tree(vt, thre[1] << 1); + } + } + + vt->force_split |= mi_col + cm->mib_size > cm->mi_cols || + mi_row + cm->mib_size > cm->mi_rows; + + // Now go through the entire structure, splitting every block size until + // we get to one that's got a variance lower than our threshold. + set_vt_partitioning(cpi, x, xd, vt, mi_row, mi_col, thre, bmin); +} + +#if CONFIG_DUAL_FILTER +static void reset_intmv_filter_type(const AV1_COMMON *const cm, MACROBLOCKD *xd, + MB_MODE_INFO *mbmi) { + int dir; + for (dir = 0; dir < 2; ++dir) { + if (!has_subpel_mv_component(xd->mi[0], xd, dir) && + (mbmi->ref_frame[1] == NONE_FRAME || + !has_subpel_mv_component(xd->mi[0], xd, dir + 2))) + mbmi->interp_filter[dir] = (cm->interp_filter == SWITCHABLE) + ? EIGHTTAP_REGULAR + : cm->interp_filter; + mbmi->interp_filter[dir + 2] = mbmi->interp_filter[dir]; + } +} + +static void update_filter_type_count(FRAME_COUNTS *counts, + const MACROBLOCKD *xd, + const MB_MODE_INFO *mbmi) { + int dir; + for (dir = 0; dir < 2; ++dir) { + if (has_subpel_mv_component(xd->mi[0], xd, dir) || + (mbmi->ref_frame[1] > INTRA_FRAME && + has_subpel_mv_component(xd->mi[0], xd, dir + 2))) { + const int ctx = av1_get_pred_context_switchable_interp(xd, dir); + ++counts->switchable_interp[ctx][mbmi->interp_filter[dir]]; + } + } +} +#endif +#if CONFIG_GLOBAL_MOTION +static void update_global_motion_used(PREDICTION_MODE mode, BLOCK_SIZE bsize, + const MB_MODE_INFO *mbmi, + RD_COUNTS *rdc) { + if (mode == ZEROMV +#if CONFIG_EXT_INTER + || mode == ZERO_ZEROMV +#endif + ) { + const int num_4x4s = + num_4x4_blocks_wide_lookup[bsize] * num_4x4_blocks_high_lookup[bsize]; + int ref; + for (ref = 0; ref < 1 + has_second_ref(mbmi); ++ref) { + rdc->global_motion_used[mbmi->ref_frame[ref]] += num_4x4s; + } + } +} +#endif // CONFIG_GLOBAL_MOTION + +static void reset_tx_size(MACROBLOCKD *xd, MB_MODE_INFO *mbmi, + const TX_MODE tx_mode) { + if (xd->lossless[mbmi->segment_id]) { + mbmi->tx_size = TX_4X4; + } else if (tx_mode != TX_MODE_SELECT) { + mbmi->tx_size = + tx_size_from_tx_mode(mbmi->sb_type, tx_mode, is_inter_block(mbmi)); + } +} + +#if CONFIG_REF_MV +static void set_ref_and_pred_mvs(MACROBLOCK *const x, int_mv *const mi_pred_mv, + int8_t rf_type) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + + const int bw = xd->n8_w << MI_SIZE_LOG2; + const int bh = xd->n8_h << MI_SIZE_LOG2; + int ref_mv_idx = mbmi->ref_mv_idx; + MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; + CANDIDATE_MV *const curr_ref_mv_stack = mbmi_ext->ref_mv_stack[rf_type]; + +#if CONFIG_EXT_INTER + if (has_second_ref(mbmi)) { + // Special case: NEAR_NEWMV and NEW_NEARMV modes use 1 + mbmi->ref_mv_idx + // (like NEARMV) instead + if (mbmi->mode == NEAR_NEWMV || mbmi->mode == NEW_NEARMV) ref_mv_idx += 1; + + if (compound_ref0_mode(mbmi->mode) == NEWMV) { + int_mv this_mv = curr_ref_mv_stack[ref_mv_idx].this_mv; + clamp_mv_ref(&this_mv.as_mv, bw, bh, xd); + mbmi_ext->ref_mvs[mbmi->ref_frame[0]][0] = this_mv; + mbmi->pred_mv[0] = this_mv; + mi_pred_mv[0] = this_mv; + } + if (compound_ref1_mode(mbmi->mode) == NEWMV) { + int_mv this_mv = curr_ref_mv_stack[ref_mv_idx].comp_mv; + clamp_mv_ref(&this_mv.as_mv, bw, bh, xd); + mbmi_ext->ref_mvs[mbmi->ref_frame[1]][0] = this_mv; + mbmi->pred_mv[1] = this_mv; + mi_pred_mv[1] = this_mv; + } + } else { +#endif // CONFIG_EXT_INTER + if (mbmi->mode == NEWMV) { + int i; + for (i = 0; i < 1 + has_second_ref(mbmi); ++i) { + int_mv this_mv = (i == 0) ? curr_ref_mv_stack[ref_mv_idx].this_mv + : curr_ref_mv_stack[ref_mv_idx].comp_mv; + clamp_mv_ref(&this_mv.as_mv, bw, bh, xd); + mbmi_ext->ref_mvs[mbmi->ref_frame[i]][0] = this_mv; + mbmi->pred_mv[i] = this_mv; + mi_pred_mv[i] = this_mv; + } + } +#if CONFIG_EXT_INTER + } +#endif // CONFIG_EXT_INTER +} +#endif // CONFIG_REF_MV + +static void update_state(const AV1_COMP *const cpi, ThreadData *td, + PICK_MODE_CONTEXT *ctx, int mi_row, int mi_col, + BLOCK_SIZE bsize, RUN_TYPE dry_run) { + int i, x_idx, y; + const AV1_COMMON *const cm = &cpi->common; + RD_COUNTS *const rdc = &td->rd_counts; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + struct macroblock_plane *const p = x->plane; + struct macroblockd_plane *const pd = xd->plane; + MODE_INFO *mi = &ctx->mic; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + MODE_INFO *mi_addr = xd->mi[0]; + const struct segmentation *const seg = &cm->seg; + const int bw = mi_size_wide[mi->mbmi.sb_type]; + const int bh = mi_size_high[mi->mbmi.sb_type]; + const int x_mis = AOMMIN(bw, cm->mi_cols - mi_col); + const int y_mis = AOMMIN(bh, cm->mi_rows - mi_row); + MV_REF *const frame_mvs = cm->cur_frame->mvs + mi_row * cm->mi_cols + mi_col; + int w, h; + + const int mis = cm->mi_stride; + const int mi_width = mi_size_wide[bsize]; + const int mi_height = mi_size_high[bsize]; + const int unify_bsize = CONFIG_CB4X4; + +#if CONFIG_REF_MV + int8_t rf_type; +#endif + +#if !CONFIG_SUPERTX + assert(mi->mbmi.sb_type == bsize); +#endif + + *mi_addr = *mi; + *x->mbmi_ext = ctx->mbmi_ext; + +#if CONFIG_DUAL_FILTER + reset_intmv_filter_type(cm, xd, mbmi); +#endif + +#if CONFIG_REF_MV + rf_type = av1_ref_frame_type(mbmi->ref_frame); + if (x->mbmi_ext->ref_mv_count[rf_type] > 1 && + (mbmi->sb_type >= BLOCK_8X8 || unify_bsize)) { + set_ref_and_pred_mvs(x, mi->mbmi.pred_mv, rf_type); + } +#endif // CONFIG_REF_MV + + // If segmentation in use + if (seg->enabled) { + // For in frame complexity AQ copy the segment id from the segment map. + if (cpi->oxcf.aq_mode == COMPLEXITY_AQ) { + const uint8_t *const map = + seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map; + mi_addr->mbmi.segment_id = get_segment_id(cm, map, bsize, mi_row, mi_col); + reset_tx_size(xd, &mi_addr->mbmi, cm->tx_mode); + } + // Else for cyclic refresh mode update the segment map, set the segment id + // and then update the quantizer. + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) { + av1_cyclic_refresh_update_segment(cpi, &xd->mi[0]->mbmi, mi_row, mi_col, + bsize, ctx->rate, ctx->dist, x->skip); + reset_tx_size(xd, &mi_addr->mbmi, cm->tx_mode); + } + } + + for (i = 0; i < MAX_MB_PLANE; ++i) { + p[i].coeff = ctx->coeff[i]; + p[i].qcoeff = ctx->qcoeff[i]; + pd[i].dqcoeff = ctx->dqcoeff[i]; +#if CONFIG_PVQ + pd[i].pvq_ref_coeff = ctx->pvq_ref_coeff[i]; +#endif + p[i].eobs = ctx->eobs[i]; +#if CONFIG_LV_MAP + p[i].txb_entropy_ctx = ctx->txb_entropy_ctx[i]; +#endif // CONFIG_LV_MAP + } +#if CONFIG_PALETTE + for (i = 0; i < 2; ++i) pd[i].color_index_map = ctx->color_index_map[i]; +#endif // CONFIG_PALETTE + + // Restore the coding context of the MB to that that was in place + // when the mode was picked for it + for (y = 0; y < mi_height; y++) + for (x_idx = 0; x_idx < mi_width; x_idx++) + if ((xd->mb_to_right_edge >> (3 + MI_SIZE_LOG2)) + mi_width > x_idx && + (xd->mb_to_bottom_edge >> (3 + MI_SIZE_LOG2)) + mi_height > y) { + xd->mi[x_idx + y * mis] = mi_addr; + } + +#if CONFIG_DELTA_Q && !CONFIG_EXT_DELTA_Q + if (cpi->oxcf.aq_mode > NO_AQ && cpi->oxcf.aq_mode < DELTA_AQ) + av1_init_plane_quantizers(cpi, x, xd->mi[0]->mbmi.segment_id); +#else + if (cpi->oxcf.aq_mode) + av1_init_plane_quantizers(cpi, x, xd->mi[0]->mbmi.segment_id); +#endif + + if (is_inter_block(mbmi) && mbmi->sb_type < BLOCK_8X8 && !unify_bsize) { + mbmi->mv[0].as_int = mi->bmi[3].as_mv[0].as_int; + mbmi->mv[1].as_int = mi->bmi[3].as_mv[1].as_int; + } + + x->skip = ctx->skip; + +#if CONFIG_VAR_TX + for (i = 0; i < 1; ++i) + memcpy(x->blk_skip[i], ctx->blk_skip[i], + sizeof(uint8_t) * ctx->num_4x4_blk); +#endif + + if (dry_run) return; + +#if CONFIG_INTERNAL_STATS + { + unsigned int *const mode_chosen_counts = + (unsigned int *)cpi->mode_chosen_counts; // Cast const away. + if (frame_is_intra_only(cm)) { + static const int kf_mode_index[] = { + THR_DC /*DC_PRED*/, + THR_V_PRED /*V_PRED*/, + THR_H_PRED /*H_PRED*/, + THR_D45_PRED /*D45_PRED*/, + THR_D135_PRED /*D135_PRED*/, + THR_D117_PRED /*D117_PRED*/, + THR_D153_PRED /*D153_PRED*/, + THR_D207_PRED /*D207_PRED*/, + THR_D63_PRED /*D63_PRED*/, +#if CONFIG_ALT_INTRA + THR_SMOOTH, /*SMOOTH_PRED*/ +#endif // CONFIG_ALT_INTRA + THR_TM /*TM_PRED*/, + }; + ++mode_chosen_counts[kf_mode_index[mbmi->mode]]; + } else { + // Note how often each mode chosen as best + ++mode_chosen_counts[ctx->best_mode_index]; + } + } +#endif + if (!frame_is_intra_only(cm)) { + if (is_inter_block(mbmi)) { + av1_update_mv_count(td); +#if CONFIG_GLOBAL_MOTION + if (bsize >= BLOCK_8X8) { + // TODO(sarahparker): global motion stats need to be handled per-tile + // to be compatible with tile-based threading. + update_global_motion_used(mbmi->mode, bsize, mbmi, rdc); + } else { + const int num_4x4_w = num_4x4_blocks_wide_lookup[bsize]; + const int num_4x4_h = num_4x4_blocks_high_lookup[bsize]; + int idx, idy; + for (idy = 0; idy < 2; idy += num_4x4_h) { + for (idx = 0; idx < 2; idx += num_4x4_w) { + const int j = idy * 2 + idx; + update_global_motion_used(mi->bmi[j].as_mode, bsize, mbmi, rdc); + } + } + } +#endif // CONFIG_GLOBAL_MOTION + if (cm->interp_filter == SWITCHABLE +#if CONFIG_WARPED_MOTION + && mbmi->motion_mode != WARPED_CAUSAL +#endif // CONFIG_WARPED_MOTION +#if CONFIG_GLOBAL_MOTION + && !is_nontrans_global_motion(xd) +#endif // CONFIG_GLOBAL_MOTION + ) { +#if CONFIG_DUAL_FILTER + update_filter_type_count(td->counts, xd, mbmi); +#else + const int switchable_ctx = av1_get_pred_context_switchable_interp(xd); + ++td->counts->switchable_interp[switchable_ctx][mbmi->interp_filter]; +#endif + } + } + + rdc->comp_pred_diff[SINGLE_REFERENCE] += ctx->single_pred_diff; + rdc->comp_pred_diff[COMPOUND_REFERENCE] += ctx->comp_pred_diff; + rdc->comp_pred_diff[REFERENCE_MODE_SELECT] += ctx->hybrid_pred_diff; + } + + for (h = 0; h < y_mis; ++h) { + MV_REF *const frame_mv = frame_mvs + h * cm->mi_cols; + for (w = 0; w < x_mis; ++w) { + MV_REF *const mv = frame_mv + w; + mv->ref_frame[0] = mi->mbmi.ref_frame[0]; + mv->ref_frame[1] = mi->mbmi.ref_frame[1]; + mv->mv[0].as_int = mi->mbmi.mv[0].as_int; + mv->mv[1].as_int = mi->mbmi.mv[1].as_int; + } + } +} + +#if CONFIG_SUPERTX +static void update_state_supertx(const AV1_COMP *const cpi, ThreadData *td, + PICK_MODE_CONTEXT *ctx, int mi_row, int mi_col, + BLOCK_SIZE bsize, RUN_TYPE dry_run) { + int y, x_idx; +#if CONFIG_VAR_TX + int i; +#endif + const AV1_COMMON *const cm = &cpi->common; + RD_COUNTS *const rdc = &td->rd_counts; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + MODE_INFO *mi = &ctx->mic; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + MODE_INFO *mi_addr = xd->mi[0]; + const struct segmentation *const seg = &cm->seg; + const int mis = cm->mi_stride; + const int mi_width = mi_size_wide[bsize]; + const int mi_height = mi_size_high[bsize]; + const int x_mis = AOMMIN(mi_width, cm->mi_cols - mi_col); + const int y_mis = AOMMIN(mi_height, cm->mi_rows - mi_row); + const int unify_bsize = CONFIG_CB4X4; + MV_REF *const frame_mvs = cm->cur_frame->mvs + mi_row * cm->mi_cols + mi_col; + int w, h; + +#if CONFIG_REF_MV + int8_t rf_type; +#endif + + *mi_addr = *mi; + *x->mbmi_ext = ctx->mbmi_ext; + assert(is_inter_block(mbmi)); + assert(mbmi->tx_size == ctx->mic.mbmi.tx_size); + +#if CONFIG_DUAL_FILTER + reset_intmv_filter_type(cm, xd, mbmi); +#endif + +#if CONFIG_REF_MV + rf_type = av1_ref_frame_type(mbmi->ref_frame); + if (x->mbmi_ext->ref_mv_count[rf_type] > 1 && + (mbmi->sb_type >= BLOCK_8X8 || unify_bsize)) { + set_ref_and_pred_mvs(x, mi->mbmi.pred_mv, rf_type); + } +#endif // CONFIG_REF_MV + + // If segmentation in use + if (seg->enabled) { + if (cpi->vaq_refresh) { + const int energy = + bsize <= BLOCK_16X16 ? x->mb_energy : av1_block_energy(cpi, x, bsize); + mi_addr->mbmi.segment_id = av1_vaq_segment_id(energy); + } else if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) { + // For cyclic refresh mode, now update the segment map + // and set the segment id. + av1_cyclic_refresh_update_segment(cpi, &xd->mi[0]->mbmi, mi_row, mi_col, + bsize, ctx->rate, ctx->dist, 1); + } else { + // Otherwise just set the segment id based on the current segment map + const uint8_t *const map = + seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map; + mi_addr->mbmi.segment_id = get_segment_id(cm, map, bsize, mi_row, mi_col); + } + mi_addr->mbmi.segment_id_supertx = MAX_SEGMENTS; + } + + // Restore the coding context of the MB to that that was in place + // when the mode was picked for it + for (y = 0; y < mi_height; y++) + for (x_idx = 0; x_idx < mi_width; x_idx++) + if ((xd->mb_to_right_edge >> (3 + MI_SIZE_LOG2)) + mi_width > x_idx && + (xd->mb_to_bottom_edge >> (3 + MI_SIZE_LOG2)) + mi_height > y) { + xd->mi[x_idx + y * mis] = mi_addr; + } + +#if !CONFIG_CB4X4 + if (is_inter_block(mbmi) && mbmi->sb_type < BLOCK_8X8) { + mbmi->mv[0].as_int = mi->bmi[3].as_mv[0].as_int; + mbmi->mv[1].as_int = mi->bmi[3].as_mv[1].as_int; + } +#endif + + x->skip = ctx->skip; + +#if CONFIG_VAR_TX + for (i = 0; i < 1; ++i) + memcpy(x->blk_skip[i], ctx->blk_skip[i], + sizeof(uint8_t) * ctx->num_4x4_blk); + + if (!is_inter_block(mbmi) || mbmi->skip) + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); +#endif // CONFIG_VAR_TX + +#if CONFIG_VAR_TX + { + const TX_SIZE mtx = mbmi->tx_size; + const int num_4x4_blocks_wide = tx_size_wide_unit[mtx] >> 1; + const int num_4x4_blocks_high = tx_size_high_unit[mtx] >> 1; + int idy, idx; + mbmi->inter_tx_size[0][0] = mtx; + for (idy = 0; idy < num_4x4_blocks_high; ++idy) + for (idx = 0; idx < num_4x4_blocks_wide; ++idx) + mbmi->inter_tx_size[idy][idx] = mtx; + } +#endif // CONFIG_VAR_TX + // Turn motion variation off for supertx + mbmi->motion_mode = SIMPLE_TRANSLATION; + + if (dry_run) return; + + if (!frame_is_intra_only(cm)) { + av1_update_mv_count(td); + +#if CONFIG_GLOBAL_MOTION + if (is_inter_block(mbmi)) { + if (bsize >= BLOCK_8X8) { + // TODO(sarahparker): global motion stats need to be handled per-tile + // to be compatible with tile-based threading. + update_global_motion_used(mbmi->mode, bsize, mbmi, rdc); + } else { + const int num_4x4_w = num_4x4_blocks_wide_lookup[bsize]; + const int num_4x4_h = num_4x4_blocks_high_lookup[bsize]; + int idx, idy; + for (idy = 0; idy < 2; idy += num_4x4_h) { + for (idx = 0; idx < 2; idx += num_4x4_w) { + const int j = idy * 2 + idx; + update_global_motion_used(mi->bmi[j].as_mode, bsize, mbmi, rdc); + } + } + } + } +#endif // CONFIG_GLOBAL_MOTION + + if (cm->interp_filter == SWITCHABLE +#if CONFIG_GLOBAL_MOTION + && !is_nontrans_global_motion(xd) +#endif // CONFIG_GLOBAL_MOTION + ) { +#if CONFIG_DUAL_FILTER + update_filter_type_count(td->counts, xd, mbmi); +#else + const int pred_ctx = av1_get_pred_context_switchable_interp(xd); + ++td->counts->switchable_interp[pred_ctx][mbmi->interp_filter]; +#endif + } + + rdc->comp_pred_diff[SINGLE_REFERENCE] += ctx->single_pred_diff; + rdc->comp_pred_diff[COMPOUND_REFERENCE] += ctx->comp_pred_diff; + rdc->comp_pred_diff[REFERENCE_MODE_SELECT] += ctx->hybrid_pred_diff; + } + + for (h = 0; h < y_mis; ++h) { + MV_REF *const frame_mv = frame_mvs + h * cm->mi_cols; + for (w = 0; w < x_mis; ++w) { + MV_REF *const mv = frame_mv + w; + mv->ref_frame[0] = mi->mbmi.ref_frame[0]; + mv->ref_frame[1] = mi->mbmi.ref_frame[1]; + mv->mv[0].as_int = mi->mbmi.mv[0].as_int; + mv->mv[1].as_int = mi->mbmi.mv[1].as_int; + } + } +} + +static void update_state_sb_supertx(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, int mi_row, + int mi_col, BLOCK_SIZE bsize, + RUN_TYPE dry_run, PC_TREE *pc_tree) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + struct macroblock_plane *const p = x->plane; + struct macroblockd_plane *const pd = xd->plane; + int hbs = mi_size_wide[bsize] / 2; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + PARTITION_TYPE partition = pc_tree->partitioning; + BLOCK_SIZE subsize = get_subsize(bsize, partition); + int i; +#if CONFIG_EXT_PARTITION_TYPES + BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT); +#endif + PICK_MODE_CONTEXT *pmc = NULL; + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + if (bsize == BLOCK_16X16 && cpi->vaq_refresh) + x->mb_energy = av1_block_energy(cpi, x, bsize); + + switch (partition) { + case PARTITION_NONE: + set_offsets_supertx(cpi, td, tile, mi_row, mi_col, subsize); + update_state_supertx(cpi, td, &pc_tree->none, mi_row, mi_col, subsize, + dry_run); + break; + case PARTITION_VERT: + set_offsets_supertx(cpi, td, tile, mi_row, mi_col, subsize); + update_state_supertx(cpi, td, &pc_tree->vertical[0], mi_row, mi_col, + subsize, dry_run); + if (mi_col + hbs < cm->mi_cols && (bsize > BLOCK_8X8 || unify_bsize)) { + set_offsets_supertx(cpi, td, tile, mi_row, mi_col + hbs, subsize); + update_state_supertx(cpi, td, &pc_tree->vertical[1], mi_row, + mi_col + hbs, subsize, dry_run); + } + pmc = &pc_tree->vertical_supertx; + break; + case PARTITION_HORZ: + set_offsets_supertx(cpi, td, tile, mi_row, mi_col, subsize); + update_state_supertx(cpi, td, &pc_tree->horizontal[0], mi_row, mi_col, + subsize, dry_run); + if (mi_row + hbs < cm->mi_rows && (bsize > BLOCK_8X8 || unify_bsize)) { + set_offsets_supertx(cpi, td, tile, mi_row + hbs, mi_col, subsize); + update_state_supertx(cpi, td, &pc_tree->horizontal[1], mi_row + hbs, + mi_col, subsize, dry_run); + } + pmc = &pc_tree->horizontal_supertx; + break; + case PARTITION_SPLIT: + if (bsize == BLOCK_8X8 && !unify_bsize) { + set_offsets_supertx(cpi, td, tile, mi_row, mi_col, subsize); + update_state_supertx(cpi, td, pc_tree->leaf_split[0], mi_row, mi_col, + subsize, dry_run); + } else { + set_offsets_supertx(cpi, td, tile, mi_row, mi_col, subsize); + update_state_sb_supertx(cpi, td, tile, mi_row, mi_col, subsize, dry_run, + pc_tree->split[0]); + set_offsets_supertx(cpi, td, tile, mi_row, mi_col + hbs, subsize); + update_state_sb_supertx(cpi, td, tile, mi_row, mi_col + hbs, subsize, + dry_run, pc_tree->split[1]); + set_offsets_supertx(cpi, td, tile, mi_row + hbs, mi_col, subsize); + update_state_sb_supertx(cpi, td, tile, mi_row + hbs, mi_col, subsize, + dry_run, pc_tree->split[2]); + set_offsets_supertx(cpi, td, tile, mi_row + hbs, mi_col + hbs, subsize); + update_state_sb_supertx(cpi, td, tile, mi_row + hbs, mi_col + hbs, + subsize, dry_run, pc_tree->split[3]); + } + pmc = &pc_tree->split_supertx; + break; +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_HORZ_A: + set_offsets_supertx(cpi, td, tile, mi_row, mi_col, bsize2); + update_state_supertx(cpi, td, &pc_tree->horizontala[0], mi_row, mi_col, + bsize2, dry_run); + set_offsets_supertx(cpi, td, tile, mi_row, mi_col + hbs, bsize2); + update_state_supertx(cpi, td, &pc_tree->horizontala[1], mi_row, + mi_col + hbs, bsize2, dry_run); + set_offsets_supertx(cpi, td, tile, mi_row + hbs, mi_col, subsize); + update_state_supertx(cpi, td, &pc_tree->horizontala[2], mi_row + hbs, + mi_col, subsize, dry_run); + pmc = &pc_tree->horizontala_supertx; + break; + case PARTITION_HORZ_B: + set_offsets_supertx(cpi, td, tile, mi_row, mi_col, subsize); + update_state_supertx(cpi, td, &pc_tree->horizontalb[0], mi_row, mi_col, + subsize, dry_run); + set_offsets_supertx(cpi, td, tile, mi_row + hbs, mi_col, bsize2); + update_state_supertx(cpi, td, &pc_tree->horizontalb[1], mi_row + hbs, + mi_col, bsize2, dry_run); + set_offsets_supertx(cpi, td, tile, mi_row + hbs, mi_col + hbs, bsize2); + update_state_supertx(cpi, td, &pc_tree->horizontalb[2], mi_row + hbs, + mi_col + hbs, bsize2, dry_run); + pmc = &pc_tree->horizontalb_supertx; + break; + case PARTITION_VERT_A: + set_offsets_supertx(cpi, td, tile, mi_row, mi_col, bsize2); + update_state_supertx(cpi, td, &pc_tree->verticala[0], mi_row, mi_col, + bsize2, dry_run); + set_offsets_supertx(cpi, td, tile, mi_row + hbs, mi_col, bsize2); + update_state_supertx(cpi, td, &pc_tree->verticala[1], mi_row + hbs, + mi_col, bsize2, dry_run); + set_offsets_supertx(cpi, td, tile, mi_row, mi_col + hbs, subsize); + update_state_supertx(cpi, td, &pc_tree->verticala[2], mi_row, + mi_col + hbs, subsize, dry_run); + pmc = &pc_tree->verticala_supertx; + break; + case PARTITION_VERT_B: + set_offsets_supertx(cpi, td, tile, mi_row, mi_col, subsize); + update_state_supertx(cpi, td, &pc_tree->verticalb[0], mi_row, mi_col, + subsize, dry_run); + set_offsets_supertx(cpi, td, tile, mi_row, mi_col + hbs, bsize2); + update_state_supertx(cpi, td, &pc_tree->verticalb[1], mi_row, + mi_col + hbs, bsize2, dry_run); + set_offsets_supertx(cpi, td, tile, mi_row + hbs, mi_col + hbs, bsize2); + update_state_supertx(cpi, td, &pc_tree->verticalb[2], mi_row + hbs, + mi_col + hbs, bsize2, dry_run); + pmc = &pc_tree->verticalb_supertx; + break; +#endif // CONFIG_EXT_PARTITION_TYPES + default: assert(0); + } + + for (i = 0; i < MAX_MB_PLANE; ++i) { + if (pmc != NULL) { + p[i].coeff = pmc->coeff[i]; + p[i].qcoeff = pmc->qcoeff[i]; + pd[i].dqcoeff = pmc->dqcoeff[i]; + p[i].eobs = pmc->eobs[i]; + } else { + // These should never be used + p[i].coeff = NULL; + p[i].qcoeff = NULL; + pd[i].dqcoeff = NULL; + p[i].eobs = NULL; + } + } +} + +static void update_supertx_param(ThreadData *td, PICK_MODE_CONTEXT *ctx, + int best_tx, TX_SIZE supertx_size) { + MACROBLOCK *const x = &td->mb; +#if CONFIG_VAR_TX + int i; + + for (i = 0; i < 1; ++i) + memcpy(ctx->blk_skip[i], x->blk_skip[i], + sizeof(uint8_t) * ctx->num_4x4_blk); + ctx->mic.mbmi.min_tx_size = get_min_tx_size(supertx_size); +#endif // CONFIG_VAR_TX + ctx->mic.mbmi.tx_size = supertx_size; + ctx->skip = x->skip; + ctx->mic.mbmi.tx_type = best_tx; +} + +static void update_supertx_param_sb(const AV1_COMP *const cpi, ThreadData *td, + int mi_row, int mi_col, BLOCK_SIZE bsize, + int best_tx, TX_SIZE supertx_size, + PC_TREE *pc_tree) { + const AV1_COMMON *const cm = &cpi->common; + const int hbs = mi_size_wide[bsize] / 2; + PARTITION_TYPE partition = pc_tree->partitioning; + BLOCK_SIZE subsize = get_subsize(bsize, partition); +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif +#if CONFIG_EXT_PARTITION_TYPES + int i; +#endif + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + switch (partition) { + case PARTITION_NONE: + update_supertx_param(td, &pc_tree->none, best_tx, supertx_size); + break; + case PARTITION_VERT: + update_supertx_param(td, &pc_tree->vertical[0], best_tx, supertx_size); + if (mi_col + hbs < cm->mi_cols && (bsize > BLOCK_8X8 || unify_bsize)) + update_supertx_param(td, &pc_tree->vertical[1], best_tx, supertx_size); + break; + case PARTITION_HORZ: + update_supertx_param(td, &pc_tree->horizontal[0], best_tx, supertx_size); + if (mi_row + hbs < cm->mi_rows && (bsize > BLOCK_8X8 || unify_bsize)) + update_supertx_param(td, &pc_tree->horizontal[1], best_tx, + supertx_size); + break; + case PARTITION_SPLIT: + if (bsize == BLOCK_8X8 && !unify_bsize) { + update_supertx_param(td, pc_tree->leaf_split[0], best_tx, supertx_size); + } else { + update_supertx_param_sb(cpi, td, mi_row, mi_col, subsize, best_tx, + supertx_size, pc_tree->split[0]); + update_supertx_param_sb(cpi, td, mi_row, mi_col + hbs, subsize, best_tx, + supertx_size, pc_tree->split[1]); + update_supertx_param_sb(cpi, td, mi_row + hbs, mi_col, subsize, best_tx, + supertx_size, pc_tree->split[2]); + update_supertx_param_sb(cpi, td, mi_row + hbs, mi_col + hbs, subsize, + best_tx, supertx_size, pc_tree->split[3]); + } + break; +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_HORZ_A: + for (i = 0; i < 3; i++) + update_supertx_param(td, &pc_tree->horizontala[i], best_tx, + supertx_size); + break; + case PARTITION_HORZ_B: + for (i = 0; i < 3; i++) + update_supertx_param(td, &pc_tree->horizontalb[i], best_tx, + supertx_size); + break; + case PARTITION_VERT_A: + for (i = 0; i < 3; i++) + update_supertx_param(td, &pc_tree->verticala[i], best_tx, supertx_size); + break; + case PARTITION_VERT_B: + for (i = 0; i < 3; i++) + update_supertx_param(td, &pc_tree->verticalb[i], best_tx, supertx_size); + break; +#endif // CONFIG_EXT_PARTITION_TYPES + default: assert(0); + } +} +#endif // CONFIG_SUPERTX + +#if CONFIG_MOTION_VAR && CONFIG_NCOBMC +static void set_mode_info_b(const AV1_COMP *const cpi, + const TileInfo *const tile, ThreadData *td, + int mi_row, int mi_col, BLOCK_SIZE bsize, + PICK_MODE_CONTEXT *ctx) { + MACROBLOCK *const x = &td->mb; + set_offsets(cpi, tile, x, mi_row, mi_col, bsize); + update_state(cpi, td, ctx, mi_row, mi_col, bsize, 1); +} + +static void set_mode_info_sb(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, TOKENEXTRA **tp, + int mi_row, int mi_col, BLOCK_SIZE bsize, + PC_TREE *pc_tree) { + const AV1_COMMON *const cm = &cpi->common; + const int hbs = mi_size_wide[bsize] / 2; + const PARTITION_TYPE partition = pc_tree->partitioning; + BLOCK_SIZE subsize = get_subsize(bsize, partition); +#if CONFIG_EXT_PARTITION_TYPES + const BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT); +#endif +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; + assert(bsize >= BLOCK_8X8); +#endif + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + switch (partition) { + case PARTITION_NONE: + set_mode_info_b(cpi, tile, td, mi_row, mi_col, subsize, &pc_tree->none); + break; + case PARTITION_VERT: + set_mode_info_b(cpi, tile, td, mi_row, mi_col, subsize, + &pc_tree->vertical[0]); + if (mi_col + hbs < cm->mi_cols && (bsize > BLOCK_8X8 || unify_bsize)) { + set_mode_info_b(cpi, tile, td, mi_row, mi_col + hbs, subsize, + &pc_tree->vertical[1]); + } + break; + case PARTITION_HORZ: + set_mode_info_b(cpi, tile, td, mi_row, mi_col, subsize, + &pc_tree->horizontal[0]); + if (mi_row + hbs < cm->mi_rows && (bsize > BLOCK_8X8 || unify_bsize)) { + set_mode_info_b(cpi, tile, td, mi_row + hbs, mi_col, subsize, + &pc_tree->horizontal[1]); + } + break; + case PARTITION_SPLIT: + if (bsize == BLOCK_8X8 && !unify_bsize) { + set_mode_info_b(cpi, tile, td, mi_row, mi_col, subsize, + pc_tree->leaf_split[0]); + } else { + set_mode_info_sb(cpi, td, tile, tp, mi_row, mi_col, subsize, + pc_tree->split[0]); + set_mode_info_sb(cpi, td, tile, tp, mi_row, mi_col + hbs, subsize, + pc_tree->split[1]); + set_mode_info_sb(cpi, td, tile, tp, mi_row + hbs, mi_col, subsize, + pc_tree->split[2]); + set_mode_info_sb(cpi, td, tile, tp, mi_row + hbs, mi_col + hbs, subsize, + pc_tree->split[3]); + } + break; +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_HORZ_A: + set_mode_info_b(cpi, tile, td, mi_row, mi_col, bsize2, + &pc_tree->horizontala[0]); + set_mode_info_b(cpi, tile, td, mi_row, mi_col + hbs, bsize2, + &pc_tree->horizontala[1]); + set_mode_info_b(cpi, tile, td, mi_row + hbs, mi_col, subsize, + &pc_tree->horizontala[2]); + break; + case PARTITION_HORZ_B: + set_mode_info_b(cpi, tile, td, mi_row, mi_col, subsize, + &pc_tree->horizontalb[0]); + set_mode_info_b(cpi, tile, td, mi_row + hbs, mi_col, bsize2, + &pc_tree->horizontalb[1]); + set_mode_info_b(cpi, tile, td, mi_row + hbs, mi_col + hbs, bsize2, + &pc_tree->horizontalb[2]); + break; + case PARTITION_VERT_A: + set_mode_info_b(cpi, tile, td, mi_row, mi_col, bsize2, + &pc_tree->verticala[0]); + set_mode_info_b(cpi, tile, td, mi_row + hbs, mi_col, bsize2, + &pc_tree->verticala[1]); + set_mode_info_b(cpi, tile, td, mi_row, mi_col + hbs, subsize, + &pc_tree->verticala[2]); + break; + case PARTITION_VERT_B: + set_mode_info_b(cpi, tile, td, mi_row, mi_col, subsize, + &pc_tree->verticalb[0]); + set_mode_info_b(cpi, tile, td, mi_row, mi_col + hbs, bsize2, + &pc_tree->verticalb[1]); + set_mode_info_b(cpi, tile, td, mi_row + hbs, mi_col + hbs, bsize2, + &pc_tree->verticalb[2]); + break; +#endif // CONFIG_EXT_PARTITION_TYPES + default: assert(0 && "Invalid partition type."); break; + } +} +#endif + +void av1_setup_src_planes(MACROBLOCK *x, const YV12_BUFFER_CONFIG *src, + int mi_row, int mi_col) { + uint8_t *const buffers[3] = { src->y_buffer, src->u_buffer, src->v_buffer }; + const int widths[3] = { src->y_crop_width, src->uv_crop_width, + src->uv_crop_width }; + const int heights[3] = { src->y_crop_height, src->uv_crop_height, + src->uv_crop_height }; + const int strides[3] = { src->y_stride, src->uv_stride, src->uv_stride }; + int i; + + // Set current frame pointer. + x->e_mbd.cur_buf = src; + + for (i = 0; i < MAX_MB_PLANE; i++) + setup_pred_plane(&x->plane[i].src, x->e_mbd.mi[0]->mbmi.sb_type, buffers[i], + widths[i], heights[i], strides[i], mi_row, mi_col, NULL, + x->e_mbd.plane[i].subsampling_x, + x->e_mbd.plane[i].subsampling_y); +} + +static int set_segment_rdmult(const AV1_COMP *const cpi, MACROBLOCK *const x, + int8_t segment_id) { + int segment_qindex; + const AV1_COMMON *const cm = &cpi->common; + av1_init_plane_quantizers(cpi, x, segment_id); + aom_clear_system_state(); + segment_qindex = av1_get_qindex(&cm->seg, segment_id, cm->base_qindex); + return av1_compute_rd_mult(cpi, segment_qindex + cm->y_dc_delta_q); +} + +static void rd_pick_sb_modes(const AV1_COMP *const cpi, TileDataEnc *tile_data, + MACROBLOCK *const x, int mi_row, int mi_col, + RD_STATS *rd_cost, +#if CONFIG_SUPERTX + int *totalrate_nocoef, +#endif +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_TYPE partition, +#endif + BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx, + int64_t best_rd) { + const AV1_COMMON *const cm = &cpi->common; + TileInfo *const tile_info = &tile_data->tile_info; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *mbmi; + struct macroblock_plane *const p = x->plane; + struct macroblockd_plane *const pd = xd->plane; + const AQ_MODE aq_mode = cpi->oxcf.aq_mode; + int i, orig_rdmult; + const int unify_bsize = CONFIG_CB4X4; + + aom_clear_system_state(); + +#if CONFIG_PVQ + x->pvq_speed = 1; + x->pvq_coded = 0; +#endif +#if CONFIG_CFL + // Don't store luma during RDO (we will store the best mode later). + x->cfl_store_y = 0; +#endif + + set_offsets(cpi, tile_info, x, mi_row, mi_col, bsize); + mbmi = &xd->mi[0]->mbmi; + mbmi->sb_type = bsize; +#if CONFIG_RD_DEBUG + mbmi->mi_row = mi_row; + mbmi->mi_col = mi_col; +#endif +#if CONFIG_SUPERTX + // We set tx_size here as skip blocks would otherwise not set it. + // tx_size needs to be set at this point as supertx_enable in + // write_modes_sb is computed based on this, and if the garbage in memory + // just happens to be the supertx_size, then the packer will code this + // block as a supertx block, even if rdopt did not pick it as such. + mbmi->tx_size = max_txsize_lookup[bsize]; +#endif +#if CONFIG_EXT_PARTITION_TYPES + mbmi->partition = partition; +#endif + + for (i = 0; i < MAX_MB_PLANE; ++i) { + p[i].coeff = ctx->coeff[i]; + p[i].qcoeff = ctx->qcoeff[i]; + pd[i].dqcoeff = ctx->dqcoeff[i]; +#if CONFIG_PVQ + pd[i].pvq_ref_coeff = ctx->pvq_ref_coeff[i]; +#endif + p[i].eobs = ctx->eobs[i]; +#if CONFIG_LV_MAP + p[i].txb_entropy_ctx = ctx->txb_entropy_ctx[i]; +#endif + } + +#if CONFIG_PALETTE + for (i = 0; i < 2; ++i) pd[i].color_index_map = ctx->color_index_map[i]; +#endif // CONFIG_PALETTE + + ctx->skippable = 0; + ctx->pred_pixel_ready = 0; + + // Set to zero to make sure we do not use the previous encoded frame stats + mbmi->skip = 0; + +#if CONFIG_CB4X4 + x->skip_chroma_rd = + !is_chroma_reference(mi_row, mi_col, bsize, xd->plane[1].subsampling_x, + xd->plane[1].subsampling_y); +#endif + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + x->source_variance = av1_high_get_sby_perpixel_variance( + cpi, &x->plane[0].src, bsize, xd->bd); + } else { + x->source_variance = + av1_get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize); + } +#else + x->source_variance = + av1_get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize); +#endif // CONFIG_HIGHBITDEPTH + + // Save rdmult before it might be changed, so it can be restored later. + orig_rdmult = x->rdmult; + + if (aq_mode == VARIANCE_AQ) { + if (cpi->vaq_refresh) { + const int energy = + bsize <= BLOCK_16X16 ? x->mb_energy : av1_block_energy(cpi, x, bsize); + mbmi->segment_id = av1_vaq_segment_id(energy); + // Re-initialise quantiser + av1_init_plane_quantizers(cpi, x, mbmi->segment_id); + } + x->rdmult = set_segment_rdmult(cpi, x, mbmi->segment_id); + } else if (aq_mode == COMPLEXITY_AQ) { + x->rdmult = set_segment_rdmult(cpi, x, mbmi->segment_id); + } else if (aq_mode == CYCLIC_REFRESH_AQ) { + // If segment is boosted, use rdmult for that segment. + if (cyclic_refresh_segment_id_boosted(mbmi->segment_id)) + x->rdmult = av1_cyclic_refresh_get_rdmult(cpi->cyclic_refresh); + } + + // Find best coding mode & reconstruct the MB so it is available + // as a predictor for MBs that follow in the SB + if (frame_is_intra_only(cm)) { + av1_rd_pick_intra_mode_sb(cpi, x, rd_cost, bsize, ctx, best_rd); +#if CONFIG_SUPERTX + *totalrate_nocoef = 0; +#endif // CONFIG_SUPERTX + } else { + if (bsize >= BLOCK_8X8 || unify_bsize) { + if (segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + av1_rd_pick_inter_mode_sb_seg_skip(cpi, tile_data, x, mi_row, mi_col, + rd_cost, bsize, ctx, best_rd); +#if CONFIG_SUPERTX + *totalrate_nocoef = rd_cost->rate; +#endif // CONFIG_SUPERTX + } else { + av1_rd_pick_inter_mode_sb(cpi, tile_data, x, mi_row, mi_col, rd_cost, +#if CONFIG_SUPERTX + totalrate_nocoef, +#endif // CONFIG_SUPERTX + bsize, ctx, best_rd); +#if CONFIG_SUPERTX + assert(*totalrate_nocoef >= 0); +#endif // CONFIG_SUPERTX + } + } else { + if (segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + // The decoder rejects sub8x8 partitions when SEG_LVL_SKIP is set. + rd_cost->rate = INT_MAX; + } else { + av1_rd_pick_inter_mode_sub8x8(cpi, tile_data, x, mi_row, mi_col, + rd_cost, +#if CONFIG_SUPERTX + totalrate_nocoef, +#endif // CONFIG_SUPERTX + bsize, ctx, best_rd); +#if CONFIG_SUPERTX + assert(*totalrate_nocoef >= 0); +#endif // CONFIG_SUPERTX + } + } + } + + // Examine the resulting rate and for AQ mode 2 make a segment choice. + if ((rd_cost->rate != INT_MAX) && (aq_mode == COMPLEXITY_AQ) && + (bsize >= BLOCK_16X16) && + (cm->frame_type == KEY_FRAME || cpi->refresh_alt_ref_frame || + (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref))) { + av1_caq_select_segment(cpi, x, bsize, mi_row, mi_col, rd_cost->rate); + } + + x->rdmult = orig_rdmult; + + // TODO(jingning) The rate-distortion optimization flow needs to be + // refactored to provide proper exit/return handle. + if (rd_cost->rate == INT_MAX) rd_cost->rdcost = INT64_MAX; + + ctx->rate = rd_cost->rate; + ctx->dist = rd_cost->dist; +} + +#if CONFIG_REF_MV +static void update_inter_mode_stats(FRAME_COUNTS *counts, PREDICTION_MODE mode, + int16_t mode_context) { + int16_t mode_ctx = mode_context & NEWMV_CTX_MASK; + if (mode == NEWMV) { + ++counts->newmv_mode[mode_ctx][0]; + return; + } else { + ++counts->newmv_mode[mode_ctx][1]; + + if (mode_context & (1 << ALL_ZERO_FLAG_OFFSET)) { + return; + } + + mode_ctx = (mode_context >> ZEROMV_OFFSET) & ZEROMV_CTX_MASK; + if (mode == ZEROMV) { + ++counts->zeromv_mode[mode_ctx][0]; + return; + } else { + ++counts->zeromv_mode[mode_ctx][1]; + mode_ctx = (mode_context >> REFMV_OFFSET) & REFMV_CTX_MASK; + + if (mode_context & (1 << SKIP_NEARESTMV_OFFSET)) mode_ctx = 6; + if (mode_context & (1 << SKIP_NEARMV_OFFSET)) mode_ctx = 7; + if (mode_context & (1 << SKIP_NEARESTMV_SUB8X8_OFFSET)) mode_ctx = 8; + + ++counts->refmv_mode[mode_ctx][mode != NEARESTMV]; + } + } +} +#endif + +static void update_stats(const AV1_COMMON *const cm, ThreadData *td, int mi_row, + int mi_col +#if CONFIG_SUPERTX + , + int supertx_enabled +#endif + ) { +#if CONFIG_DELTA_Q + MACROBLOCK *x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; +#else + const MACROBLOCK *x = &td->mb; + const MACROBLOCKD *const xd = &x->e_mbd; +#endif + const MODE_INFO *const mi = xd->mi[0]; + const MB_MODE_INFO *const mbmi = &mi->mbmi; + const MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; + const BLOCK_SIZE bsize = mbmi->sb_type; + const int unify_bsize = CONFIG_CB4X4; + +#if CONFIG_DELTA_Q + // delta quant applies to both intra and inter + const int super_block_upper_left = ((mi_row & 7) == 0) && ((mi_col & 7) == 0); + + if (cm->delta_q_present_flag && (bsize != BLOCK_64X64 || !mbmi->skip) && + super_block_upper_left) { + const int dq = (mbmi->current_q_index - xd->prev_qindex) / cm->delta_q_res; + const int absdq = abs(dq); + int i; + for (i = 0; i < AOMMIN(absdq, DELTA_Q_SMALL); ++i) { + td->counts->delta_q[i][1]++; + } + if (absdq < DELTA_Q_SMALL) td->counts->delta_q[absdq][0]++; + xd->prev_qindex = mbmi->current_q_index; +#if CONFIG_EXT_DELTA_Q + if (cm->delta_lf_present_flag) { + const int dlf = + (mbmi->current_delta_lf_from_base - xd->prev_delta_lf_from_base) / + cm->delta_lf_res; + const int absdlf = abs(dlf); + for (i = 0; i < AOMMIN(absdlf, DELTA_LF_SMALL); ++i) { + td->counts->delta_lf[i][1]++; + } + if (absdlf < DELTA_LF_SMALL) td->counts->delta_lf[absdlf][0]++; + xd->prev_delta_lf_from_base = mbmi->current_delta_lf_from_base; + } +#endif + } +#else + (void)mi_row; + (void)mi_col; +#endif + if (!frame_is_intra_only(cm)) { + FRAME_COUNTS *const counts = td->counts; + const int inter_block = is_inter_block(mbmi); + const int seg_ref_active = + segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_REF_FRAME); + if (!seg_ref_active) { +#if CONFIG_SUPERTX + if (!supertx_enabled) +#endif + counts->intra_inter[av1_get_intra_inter_context(xd)][inter_block]++; + // If the segment reference feature is enabled we have only a single + // reference frame allowed for the segment so exclude it from + // the reference frame counts used to work out probabilities. + if (inter_block) { + const MV_REFERENCE_FRAME ref0 = mbmi->ref_frame[0]; +#if CONFIG_EXT_REFS + const MV_REFERENCE_FRAME ref1 = mbmi->ref_frame[1]; +#endif // CONFIG_EXT_REFS + + if (cm->reference_mode == REFERENCE_MODE_SELECT) { +#if !SUB8X8_COMP_REF + if (mbmi->sb_type >= BLOCK_8X8) + counts->comp_inter[av1_get_reference_mode_context(cm, xd)] + [has_second_ref(mbmi)]++; +#else + counts->comp_inter[av1_get_reference_mode_context(cm, xd)] + [has_second_ref(mbmi)]++; +#endif + } + + if (has_second_ref(mbmi)) { +#if CONFIG_EXT_REFS + const int bit = (ref0 == GOLDEN_FRAME || ref0 == LAST3_FRAME); + + counts->comp_ref[av1_get_pred_context_comp_ref_p(cm, xd)][0][bit]++; + if (!bit) { + counts->comp_ref[av1_get_pred_context_comp_ref_p1(cm, xd)][1] + [ref0 == LAST_FRAME]++; + } else { + counts->comp_ref[av1_get_pred_context_comp_ref_p2(cm, xd)][2] + [ref0 == GOLDEN_FRAME]++; + } + + counts->comp_bwdref[av1_get_pred_context_comp_bwdref_p(cm, xd)][0] + [ref1 == ALTREF_FRAME]++; +#else + counts->comp_ref[av1_get_pred_context_comp_ref_p(cm, xd)][0] + [ref0 == GOLDEN_FRAME]++; +#endif // CONFIG_EXT_REFS + } else { +#if CONFIG_EXT_REFS + const int bit = (ref0 == ALTREF_FRAME || ref0 == BWDREF_FRAME); + + counts->single_ref[av1_get_pred_context_single_ref_p1(xd)][0][bit]++; + if (bit) { + counts->single_ref[av1_get_pred_context_single_ref_p2(xd)][1] + [ref0 != BWDREF_FRAME]++; + } else { + const int bit1 = !(ref0 == LAST2_FRAME || ref0 == LAST_FRAME); + counts + ->single_ref[av1_get_pred_context_single_ref_p3(xd)][2][bit1]++; + if (!bit1) { + counts->single_ref[av1_get_pred_context_single_ref_p4(xd)][3] + [ref0 != LAST_FRAME]++; + } else { + counts->single_ref[av1_get_pred_context_single_ref_p5(xd)][4] + [ref0 != LAST3_FRAME]++; + } + } +#else + counts->single_ref[av1_get_pred_context_single_ref_p1(xd)][0] + [ref0 != LAST_FRAME]++; + if (ref0 != LAST_FRAME) { + counts->single_ref[av1_get_pred_context_single_ref_p2(xd)][1] + [ref0 != GOLDEN_FRAME]++; + } +#endif // CONFIG_EXT_REFS + } + +#if CONFIG_EXT_INTER + if (cm->reference_mode != COMPOUND_REFERENCE && +#if CONFIG_SUPERTX + !supertx_enabled && +#endif + is_interintra_allowed(mbmi)) { + const int bsize_group = size_group_lookup[bsize]; + if (mbmi->ref_frame[1] == INTRA_FRAME) { + counts->interintra[bsize_group][1]++; + counts->interintra_mode[bsize_group][mbmi->interintra_mode]++; + if (is_interintra_wedge_used(bsize)) + counts->wedge_interintra[bsize][mbmi->use_wedge_interintra]++; + } else { + counts->interintra[bsize_group][0]++; + } + } +#endif // CONFIG_EXT_INTER + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + const MOTION_MODE motion_allowed = motion_mode_allowed( +#if CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + 0, xd->global_motion, +#endif // CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + mi); +#if CONFIG_SUPERTX + if (!supertx_enabled) +#endif // CONFIG_SUPERTX +#if CONFIG_EXT_INTER + if (mbmi->ref_frame[1] != INTRA_FRAME) +#endif // CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + { + if (motion_allowed == WARPED_CAUSAL) + counts->motion_mode[mbmi->sb_type][mbmi->motion_mode]++; + else if (motion_allowed == OBMC_CAUSAL) + counts->obmc[mbmi->sb_type][mbmi->motion_mode == OBMC_CAUSAL]++; + } +#else + if (motion_allowed > SIMPLE_TRANSLATION) + counts->motion_mode[mbmi->sb_type][mbmi->motion_mode]++; +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + +#if CONFIG_EXT_INTER + if (cm->reference_mode != SINGLE_REFERENCE && + is_inter_compound_mode(mbmi->mode) +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + && mbmi->motion_mode == SIMPLE_TRANSLATION +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + ) { + counts->compound_interinter[bsize][mbmi->interinter_compound_type]++; + } +#endif // CONFIG_EXT_INTER + } + } + + if (inter_block && + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + int16_t mode_ctx; +#if !CONFIG_REF_MV + mode_ctx = mbmi_ext->mode_context[mbmi->ref_frame[0]]; +#endif + if (bsize >= BLOCK_8X8 || unify_bsize) { + const PREDICTION_MODE mode = mbmi->mode; +#if CONFIG_REF_MV +#if CONFIG_EXT_INTER + if (has_second_ref(mbmi)) { + mode_ctx = mbmi_ext->compound_mode_context[mbmi->ref_frame[0]]; + ++counts->inter_compound_mode[mode_ctx][INTER_COMPOUND_OFFSET(mode)]; + } else { +#endif // CONFIG_EXT_INTER + mode_ctx = av1_mode_context_analyzer(mbmi_ext->mode_context, + mbmi->ref_frame, bsize, -1); + update_inter_mode_stats(counts, mode, mode_ctx); +#if CONFIG_EXT_INTER + } +#endif // CONFIG_EXT_INTER + +#if CONFIG_EXT_INTER + if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) { +#else + if (mbmi->mode == NEWMV) { +#endif + uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + int idx; + + for (idx = 0; idx < 2; ++idx) { + if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) { + uint8_t drl_ctx = + av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx); + ++counts->drl_mode[drl_ctx][mbmi->ref_mv_idx != idx]; + + if (mbmi->ref_mv_idx == idx) break; + } + } + } + +#if CONFIG_EXT_INTER + if (have_nearmv_in_inter_mode(mbmi->mode)) { +#else + if (mbmi->mode == NEARMV) { +#endif + uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + int idx; + + for (idx = 1; idx < 3; ++idx) { + if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) { + uint8_t drl_ctx = + av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx); + ++counts->drl_mode[drl_ctx][mbmi->ref_mv_idx != idx - 1]; + + if (mbmi->ref_mv_idx == idx - 1) break; + } + } + } +#else +#if CONFIG_EXT_INTER + if (is_inter_compound_mode(mode)) + ++counts->inter_compound_mode[mode_ctx][INTER_COMPOUND_OFFSET(mode)]; + else +#endif // CONFIG_EXT_INTER + ++counts->inter_mode[mode_ctx][INTER_OFFSET(mode)]; +#endif + } else { + const int num_4x4_w = num_4x4_blocks_wide_lookup[bsize]; + const int num_4x4_h = num_4x4_blocks_high_lookup[bsize]; + int idx, idy; + for (idy = 0; idy < 2; idy += num_4x4_h) { + for (idx = 0; idx < 2; idx += num_4x4_w) { + const int j = idy * 2 + idx; + const PREDICTION_MODE b_mode = mi->bmi[j].as_mode; +#if CONFIG_REF_MV +#if CONFIG_EXT_INTER + if (has_second_ref(mbmi)) { + mode_ctx = mbmi_ext->compound_mode_context[mbmi->ref_frame[0]]; + ++counts->inter_compound_mode[mode_ctx] + [INTER_COMPOUND_OFFSET(b_mode)]; + } else { +#endif // CONFIG_EXT_INTER + mode_ctx = av1_mode_context_analyzer(mbmi_ext->mode_context, + mbmi->ref_frame, bsize, j); + update_inter_mode_stats(counts, b_mode, mode_ctx); +#if CONFIG_EXT_INTER + } +#endif // CONFIG_EXT_INTER +#else +#if CONFIG_EXT_INTER + if (is_inter_compound_mode(b_mode)) + ++counts->inter_compound_mode[mode_ctx] + [INTER_COMPOUND_OFFSET(b_mode)]; + else +#endif // CONFIG_EXT_INTER + ++counts->inter_mode[mode_ctx][INTER_OFFSET(b_mode)]; +#endif + } + } + } + } + } +} + +typedef struct { + ENTROPY_CONTEXT a[2 * MAX_MIB_SIZE * MAX_MB_PLANE]; + ENTROPY_CONTEXT l[2 * MAX_MIB_SIZE * MAX_MB_PLANE]; + PARTITION_CONTEXT sa[MAX_MIB_SIZE]; + PARTITION_CONTEXT sl[MAX_MIB_SIZE]; +#if CONFIG_VAR_TX + TXFM_CONTEXT *p_ta; + TXFM_CONTEXT *p_tl; + TXFM_CONTEXT ta[MAX_MIB_SIZE]; + TXFM_CONTEXT tl[MAX_MIB_SIZE]; +#endif +} RD_SEARCH_MACROBLOCK_CONTEXT; + +static void restore_context(MACROBLOCK *x, + const RD_SEARCH_MACROBLOCK_CONTEXT *ctx, int mi_row, + int mi_col, +#if CONFIG_PVQ + od_rollback_buffer *rdo_buf, +#endif + BLOCK_SIZE bsize) { + MACROBLOCKD *xd = &x->e_mbd; + int p; + const int num_4x4_blocks_wide = + block_size_wide[bsize] >> tx_size_wide_log2[0]; + const int num_4x4_blocks_high = + block_size_high[bsize] >> tx_size_high_log2[0]; + int mi_width = mi_size_wide[bsize]; + int mi_height = mi_size_high[bsize]; + for (p = 0; p < MAX_MB_PLANE; p++) { + memcpy(xd->above_context[p] + ((mi_col * 2) >> xd->plane[p].subsampling_x), + ctx->a + num_4x4_blocks_wide * p, + (sizeof(ENTROPY_CONTEXT) * num_4x4_blocks_wide) >> + xd->plane[p].subsampling_x); + memcpy(xd->left_context[p] + + ((mi_row & MAX_MIB_MASK) * 2 >> xd->plane[p].subsampling_y), + ctx->l + num_4x4_blocks_high * p, + (sizeof(ENTROPY_CONTEXT) * num_4x4_blocks_high) >> + xd->plane[p].subsampling_y); + } + memcpy(xd->above_seg_context + mi_col, ctx->sa, + sizeof(*xd->above_seg_context) * mi_width); + memcpy(xd->left_seg_context + (mi_row & MAX_MIB_MASK), ctx->sl, + sizeof(xd->left_seg_context[0]) * mi_height); +#if CONFIG_VAR_TX + xd->above_txfm_context = ctx->p_ta; + xd->left_txfm_context = ctx->p_tl; + memcpy(xd->above_txfm_context, ctx->ta, + sizeof(*xd->above_txfm_context) * mi_width); + memcpy(xd->left_txfm_context, ctx->tl, + sizeof(*xd->left_txfm_context) * mi_height); +#endif +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, rdo_buf); +#endif +} + +static void save_context(const MACROBLOCK *x, RD_SEARCH_MACROBLOCK_CONTEXT *ctx, + int mi_row, int mi_col, +#if CONFIG_PVQ + od_rollback_buffer *rdo_buf, +#endif + BLOCK_SIZE bsize) { + const MACROBLOCKD *xd = &x->e_mbd; + int p; + const int num_4x4_blocks_wide = + block_size_wide[bsize] >> tx_size_wide_log2[0]; + const int num_4x4_blocks_high = + block_size_high[bsize] >> tx_size_high_log2[0]; + int mi_width = mi_size_wide[bsize]; + int mi_height = mi_size_high[bsize]; + + // buffer the above/left context information of the block in search. + for (p = 0; p < MAX_MB_PLANE; ++p) { + memcpy(ctx->a + num_4x4_blocks_wide * p, + xd->above_context[p] + (mi_col * 2 >> xd->plane[p].subsampling_x), + (sizeof(ENTROPY_CONTEXT) * num_4x4_blocks_wide) >> + xd->plane[p].subsampling_x); + memcpy(ctx->l + num_4x4_blocks_high * p, + xd->left_context[p] + + ((mi_row & MAX_MIB_MASK) * 2 >> xd->plane[p].subsampling_y), + (sizeof(ENTROPY_CONTEXT) * num_4x4_blocks_high) >> + xd->plane[p].subsampling_y); + } + memcpy(ctx->sa, xd->above_seg_context + mi_col, + sizeof(*xd->above_seg_context) * mi_width); + memcpy(ctx->sl, xd->left_seg_context + (mi_row & MAX_MIB_MASK), + sizeof(xd->left_seg_context[0]) * mi_height); +#if CONFIG_VAR_TX + memcpy(ctx->ta, xd->above_txfm_context, + sizeof(*xd->above_txfm_context) * mi_width); + memcpy(ctx->tl, xd->left_txfm_context, + sizeof(*xd->left_txfm_context) * mi_height); + ctx->p_ta = xd->above_txfm_context; + ctx->p_tl = xd->left_txfm_context; +#endif +#if CONFIG_PVQ + od_encode_checkpoint(&x->daala_enc, rdo_buf); +#endif +} + +static void encode_b(const AV1_COMP *const cpi, const TileInfo *const tile, + ThreadData *td, TOKENEXTRA **tp, int mi_row, int mi_col, + RUN_TYPE dry_run, BLOCK_SIZE bsize, +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_TYPE partition, +#endif + PICK_MODE_CONTEXT *ctx, int *rate) { + MACROBLOCK *const x = &td->mb; +#if (CONFIG_MOTION_VAR && CONFIG_NCOBMC) | CONFIG_EXT_DELTA_Q + MACROBLOCKD *xd = &x->e_mbd; + MB_MODE_INFO *mbmi; +#if CONFIG_MOTION_VAR && CONFIG_NCOBMC + int check_ncobmc; +#endif +#endif + + set_offsets(cpi, tile, x, mi_row, mi_col, bsize); +#if CONFIG_EXT_PARTITION_TYPES + x->e_mbd.mi[0]->mbmi.partition = partition; +#endif + update_state(cpi, td, ctx, mi_row, mi_col, bsize, dry_run); +#if CONFIG_MOTION_VAR && CONFIG_NCOBMC + mbmi = &xd->mi[0]->mbmi; + const MOTION_MODE motion_allowed = motion_mode_allowed( +#if CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + 0, xd->global_motion, +#endif // CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + xd->mi[0]); + check_ncobmc = is_inter_block(mbmi) && motion_allowed >= OBMC_CAUSAL; + if (!dry_run && check_ncobmc) { + av1_check_ncobmc_rd(cpi, x, mi_row, mi_col); + av1_setup_dst_planes(x->e_mbd.plane, bsize, + get_frame_new_buffer(&cpi->common), mi_row, mi_col); + } +#endif + encode_superblock(cpi, td, tp, dry_run, mi_row, mi_col, bsize, ctx, rate); + + if (!dry_run) { +#if CONFIG_EXT_DELTA_Q + mbmi = &xd->mi[0]->mbmi; + if (bsize == BLOCK_64X64 && mbmi->skip == 1 && is_inter_block(mbmi) && + cpi->common.delta_lf_present_flag) { + mbmi->current_delta_lf_from_base = xd->prev_delta_lf_from_base; + } +#endif +#if CONFIG_SUPERTX + update_stats(&cpi->common, td, mi_row, mi_col, 0); +#else + update_stats(&cpi->common, td, mi_row, mi_col); +#endif + } +} + +static void encode_sb(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, TOKENEXTRA **tp, int mi_row, + int mi_col, RUN_TYPE dry_run, BLOCK_SIZE bsize, + PC_TREE *pc_tree, int *rate) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + const int hbs = mi_size_wide[bsize] / 2; + const int is_partition_root = bsize >= BLOCK_8X8; + const int ctx = is_partition_root + ? partition_plane_context(xd, mi_row, mi_col, +#if CONFIG_UNPOISON_PARTITION_CTX + mi_row + hbs < cm->mi_rows, + mi_col + hbs < cm->mi_cols, +#endif + bsize) + : -1; + const PARTITION_TYPE partition = pc_tree->partitioning; + const BLOCK_SIZE subsize = get_subsize(bsize, partition); +#if CONFIG_EXT_PARTITION_TYPES + const BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT); +#endif + +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; + assert(bsize >= BLOCK_8X8); +#endif + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + if (!dry_run && ctx >= 0) td->counts->partition[ctx][partition]++; + +#if CONFIG_SUPERTX + if (!frame_is_intra_only(cm) && bsize <= MAX_SUPERTX_BLOCK_SIZE && + partition != PARTITION_NONE && !xd->lossless[0]) { + int supertx_enabled; + TX_SIZE supertx_size = max_txsize_lookup[bsize]; + supertx_enabled = check_supertx_sb(bsize, supertx_size, pc_tree); + if (supertx_enabled) { + const int mi_width = mi_size_wide[bsize]; + const int mi_height = mi_size_high[bsize]; + int x_idx, y_idx, i; + uint8_t *dst_buf[3]; + int dst_stride[3]; + set_skip_context(xd, mi_row, mi_col); + set_mode_info_offsets(cpi, x, xd, mi_row, mi_col); + update_state_sb_supertx(cpi, td, tile, mi_row, mi_col, bsize, dry_run, + pc_tree); + + av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, + mi_col); + for (i = 0; i < MAX_MB_PLANE; i++) { + dst_buf[i] = xd->plane[i].dst.buf; + dst_stride[i] = xd->plane[i].dst.stride; + } + predict_sb_complex(cpi, td, tile, mi_row, mi_col, mi_row, mi_col, dry_run, + bsize, bsize, dst_buf, dst_stride, pc_tree); + + set_offsets_without_segment_id(cpi, tile, x, mi_row, mi_col, bsize); + set_segment_id_supertx(cpi, x, mi_row, mi_col, bsize); + + if (!x->skip) { + int this_rate = 0; + av1_encode_sb_supertx((AV1_COMMON *)cm, x, bsize); + av1_tokenize_sb_supertx(cpi, td, tp, dry_run, bsize, rate); + if (rate) *rate += this_rate; + } else { + xd->mi[0]->mbmi.skip = 1; + if (!dry_run) td->counts->skip[av1_get_skip_context(xd)][1]++; + reset_skip_context(xd, bsize); + } + if (!dry_run) { + for (y_idx = 0; y_idx < mi_height; y_idx++) + for (x_idx = 0; x_idx < mi_width; x_idx++) { + if ((xd->mb_to_right_edge >> (3 + MI_SIZE_LOG2)) + mi_width > + x_idx && + (xd->mb_to_bottom_edge >> (3 + MI_SIZE_LOG2)) + mi_height > + y_idx) { + xd->mi[x_idx + y_idx * cm->mi_stride]->mbmi.skip = + xd->mi[0]->mbmi.skip; + } + } + td->counts->supertx[partition_supertx_context_lookup[partition]] + [supertx_size][1]++; + td->counts->supertx_size[supertx_size]++; +#if CONFIG_EXT_TX + if (get_ext_tx_types(supertx_size, bsize, 1, cm->reduced_tx_set_used) > + 1 && + !xd->mi[0]->mbmi.skip) { + const int eset = + get_ext_tx_set(supertx_size, bsize, 1, cm->reduced_tx_set_used); + if (eset > 0) { + ++td->counts + ->inter_ext_tx[eset][supertx_size][xd->mi[0]->mbmi.tx_type]; + } + } +#else + if (supertx_size < TX_32X32 && !xd->mi[0]->mbmi.skip) { + ++td->counts->inter_ext_tx[supertx_size][xd->mi[0]->mbmi.tx_type]; + } +#endif // CONFIG_EXT_TX + } +#if CONFIG_EXT_PARTITION_TYPES + update_ext_partition_context(xd, mi_row, mi_col, subsize, bsize, + partition); +#else + if (partition != PARTITION_SPLIT || bsize == BLOCK_8X8) + update_partition_context(xd, mi_row, mi_col, subsize, bsize); +#endif +#if CONFIG_VAR_TX + set_txfm_ctxs(supertx_size, mi_width, mi_height, xd->mi[0]->mbmi.skip, + xd); +#endif // CONFIG_VAR_TX + return; + } else { + if (!dry_run) { + td->counts->supertx[partition_supertx_context_lookup[partition]] + [supertx_size][0]++; + } + } + } +#endif // CONFIG_SUPERTX + + switch (partition) { + case PARTITION_NONE: + encode_b(cpi, tile, td, tp, mi_row, mi_col, dry_run, subsize, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + &pc_tree->none, rate); + break; + case PARTITION_VERT: + encode_b(cpi, tile, td, tp, mi_row, mi_col, dry_run, subsize, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + &pc_tree->vertical[0], rate); + if (mi_col + hbs < cm->mi_cols && (bsize > BLOCK_8X8 || unify_bsize)) { + encode_b(cpi, tile, td, tp, mi_row, mi_col + hbs, dry_run, subsize, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + &pc_tree->vertical[1], rate); + } + break; + case PARTITION_HORZ: + encode_b(cpi, tile, td, tp, mi_row, mi_col, dry_run, subsize, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + &pc_tree->horizontal[0], rate); + if (mi_row + hbs < cm->mi_rows && (bsize > BLOCK_8X8 || unify_bsize)) { + encode_b(cpi, tile, td, tp, mi_row + hbs, mi_col, dry_run, subsize, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + &pc_tree->horizontal[1], rate); + } + break; + case PARTITION_SPLIT: + if (bsize == BLOCK_8X8 && !unify_bsize) { + encode_b(cpi, tile, td, tp, mi_row, mi_col, dry_run, subsize, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + pc_tree->leaf_split[0], rate); + } else { + encode_sb(cpi, td, tile, tp, mi_row, mi_col, dry_run, subsize, + pc_tree->split[0], rate); + encode_sb(cpi, td, tile, tp, mi_row, mi_col + hbs, dry_run, subsize, + pc_tree->split[1], rate); + encode_sb(cpi, td, tile, tp, mi_row + hbs, mi_col, dry_run, subsize, + pc_tree->split[2], rate); + encode_sb(cpi, td, tile, tp, mi_row + hbs, mi_col + hbs, dry_run, + subsize, pc_tree->split[3], rate); + } + break; +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_HORZ_A: + encode_b(cpi, tile, td, tp, mi_row, mi_col, dry_run, bsize2, partition, + &pc_tree->horizontala[0], rate); + encode_b(cpi, tile, td, tp, mi_row, mi_col + hbs, dry_run, bsize2, + partition, &pc_tree->horizontala[1], rate); + encode_b(cpi, tile, td, tp, mi_row + hbs, mi_col, dry_run, subsize, + partition, &pc_tree->horizontala[2], rate); + break; + case PARTITION_HORZ_B: + encode_b(cpi, tile, td, tp, mi_row, mi_col, dry_run, subsize, partition, + &pc_tree->horizontalb[0], rate); + encode_b(cpi, tile, td, tp, mi_row + hbs, mi_col, dry_run, bsize2, + partition, &pc_tree->horizontalb[1], rate); + encode_b(cpi, tile, td, tp, mi_row + hbs, mi_col + hbs, dry_run, bsize2, + partition, &pc_tree->horizontalb[2], rate); + break; + case PARTITION_VERT_A: + encode_b(cpi, tile, td, tp, mi_row, mi_col, dry_run, bsize2, partition, + &pc_tree->verticala[0], rate); + encode_b(cpi, tile, td, tp, mi_row + hbs, mi_col, dry_run, bsize2, + partition, &pc_tree->verticala[1], rate); + encode_b(cpi, tile, td, tp, mi_row, mi_col + hbs, dry_run, subsize, + partition, &pc_tree->verticala[2], rate); + + break; + case PARTITION_VERT_B: + encode_b(cpi, tile, td, tp, mi_row, mi_col, dry_run, subsize, partition, + &pc_tree->verticalb[0], rate); + encode_b(cpi, tile, td, tp, mi_row, mi_col + hbs, dry_run, bsize2, + partition, &pc_tree->verticalb[1], rate); + encode_b(cpi, tile, td, tp, mi_row + hbs, mi_col + hbs, dry_run, bsize2, + partition, &pc_tree->verticalb[2], rate); + break; +#endif // CONFIG_EXT_PARTITION_TYPES + default: assert(0 && "Invalid partition type."); break; + } + +#if CONFIG_EXT_PARTITION_TYPES + update_ext_partition_context(xd, mi_row, mi_col, subsize, bsize, partition); +#else + if (partition != PARTITION_SPLIT || bsize == BLOCK_8X8) + update_partition_context(xd, mi_row, mi_col, subsize, bsize); +#endif // CONFIG_EXT_PARTITION_TYPES +} + +// Check to see if the given partition size is allowed for a specified number +// of mi block rows and columns remaining in the image. +// If not then return the largest allowed partition size +static BLOCK_SIZE find_partition_size(BLOCK_SIZE bsize, int rows_left, + int cols_left, int *bh, int *bw) { + if (rows_left <= 0 || cols_left <= 0) { + return AOMMIN(bsize, BLOCK_8X8); + } else { + for (; bsize > 0; bsize -= 3) { + *bh = mi_size_high[bsize]; + *bw = mi_size_wide[bsize]; + if ((*bh <= rows_left) && (*bw <= cols_left)) { + break; + } + } + } + return bsize; +} + +static void set_partial_sb_partition(const AV1_COMMON *const cm, MODE_INFO *mi, + int bh_in, int bw_in, + int mi_rows_remaining, + int mi_cols_remaining, BLOCK_SIZE bsize, + MODE_INFO **mib) { + int bh = bh_in; + int r, c; + for (r = 0; r < cm->mib_size; r += bh) { + int bw = bw_in; + for (c = 0; c < cm->mib_size; c += bw) { + const int index = r * cm->mi_stride + c; + mib[index] = mi + index; + mib[index]->mbmi.sb_type = find_partition_size( + bsize, mi_rows_remaining - r, mi_cols_remaining - c, &bh, &bw); + } + } +} + +// This function attempts to set all mode info entries in a given superblock +// to the same block partition size. +// However, at the bottom and right borders of the image the requested size +// may not be allowed in which case this code attempts to choose the largest +// allowable partition. +static void set_fixed_partitioning(AV1_COMP *cpi, const TileInfo *const tile, + MODE_INFO **mib, int mi_row, int mi_col, + BLOCK_SIZE bsize) { + AV1_COMMON *const cm = &cpi->common; + const int mi_rows_remaining = tile->mi_row_end - mi_row; + const int mi_cols_remaining = tile->mi_col_end - mi_col; + int block_row, block_col; + MODE_INFO *const mi_upper_left = cm->mi + mi_row * cm->mi_stride + mi_col; + int bh = mi_size_high[bsize]; + int bw = mi_size_wide[bsize]; + + assert((mi_rows_remaining > 0) && (mi_cols_remaining > 0)); + + // Apply the requested partition size to the SB if it is all "in image" + if ((mi_cols_remaining >= cm->mib_size) && + (mi_rows_remaining >= cm->mib_size)) { + for (block_row = 0; block_row < cm->mib_size; block_row += bh) { + for (block_col = 0; block_col < cm->mib_size; block_col += bw) { + int index = block_row * cm->mi_stride + block_col; + mib[index] = mi_upper_left + index; + mib[index]->mbmi.sb_type = bsize; + } + } + } else { + // Else this is a partial SB. + set_partial_sb_partition(cm, mi_upper_left, bh, bw, mi_rows_remaining, + mi_cols_remaining, bsize, mib); + } +} + +static void rd_use_partition(AV1_COMP *cpi, ThreadData *td, + TileDataEnc *tile_data, MODE_INFO **mib, + TOKENEXTRA **tp, int mi_row, int mi_col, + BLOCK_SIZE bsize, int *rate, int64_t *dist, +#if CONFIG_SUPERTX + int *rate_nocoef, +#endif + int do_recon, PC_TREE *pc_tree) { + AV1_COMMON *const cm = &cpi->common; + TileInfo *const tile_info = &tile_data->tile_info; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + const int bs = mi_size_wide[bsize]; + const int hbs = bs / 2; + int i; + const int pl = (bsize >= BLOCK_8X8) + ? partition_plane_context(xd, mi_row, mi_col, +#if CONFIG_UNPOISON_PARTITION_CTX + mi_row + hbs < cm->mi_rows, + mi_col + hbs < cm->mi_cols, +#endif + bsize) + : 0; + const PARTITION_TYPE partition = + (bsize >= BLOCK_8X8) ? get_partition(cm, mi_row, mi_col, bsize) + : PARTITION_NONE; + const BLOCK_SIZE subsize = get_subsize(bsize, partition); + RD_SEARCH_MACROBLOCK_CONTEXT x_ctx; + RD_STATS last_part_rdc, none_rdc, chosen_rdc; + BLOCK_SIZE sub_subsize = BLOCK_4X4; + int splits_below = 0; + BLOCK_SIZE bs_type = mib[0]->mbmi.sb_type; + int do_partition_search = 1; + PICK_MODE_CONTEXT *ctx_none = &pc_tree->none; + const int unify_bsize = CONFIG_CB4X4; +#if CONFIG_SUPERTX + int last_part_rate_nocoef = INT_MAX; + int none_rate_nocoef = INT_MAX; + int chosen_rate_nocoef = INT_MAX; +#endif +#if CONFIG_PVQ + od_rollback_buffer pre_rdo_buf; +#endif + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + assert(num_4x4_blocks_wide_lookup[bsize] == + num_4x4_blocks_high_lookup[bsize]); + + av1_invalid_rd_stats(&last_part_rdc); + av1_invalid_rd_stats(&none_rdc); + av1_invalid_rd_stats(&chosen_rdc); + + pc_tree->partitioning = partition; + +#if CONFIG_VAR_TX + xd->above_txfm_context = cm->above_txfm_context + mi_col; + xd->left_txfm_context = + xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK); +#endif +#if !CONFIG_PVQ + save_context(x, &x_ctx, mi_row, mi_col, bsize); +#else + save_context(x, &x_ctx, mi_row, mi_col, &pre_rdo_buf, bsize); +#endif + + if (bsize == BLOCK_16X16 && cpi->vaq_refresh) { + set_offsets(cpi, tile_info, x, mi_row, mi_col, bsize); + x->mb_energy = av1_block_energy(cpi, x, bsize); + } + + if (do_partition_search && + cpi->sf.partition_search_type == SEARCH_PARTITION && + cpi->sf.adjust_partitioning_from_last_frame) { + // Check if any of the sub blocks are further split. + if (partition == PARTITION_SPLIT && subsize > BLOCK_8X8) { + sub_subsize = get_subsize(subsize, PARTITION_SPLIT); + splits_below = 1; + for (i = 0; i < 4; i++) { + int jj = i >> 1, ii = i & 0x01; + MODE_INFO *this_mi = mib[jj * hbs * cm->mi_stride + ii * hbs]; + if (this_mi && this_mi->mbmi.sb_type >= sub_subsize) { + splits_below = 0; + } + } + } + + // If partition is not none try none unless each of the 4 splits are split + // even further.. + if (partition != PARTITION_NONE && !splits_below && + mi_row + hbs < cm->mi_rows && mi_col + hbs < cm->mi_cols) { + pc_tree->partitioning = PARTITION_NONE; + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &none_rdc, +#if CONFIG_SUPERTX + &none_rate_nocoef, +#endif +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_NONE, +#endif + bsize, ctx_none, INT64_MAX); + + if (none_rdc.rate < INT_MAX) { + none_rdc.rate += cpi->partition_cost[pl][PARTITION_NONE]; + none_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, none_rdc.rate, none_rdc.dist); +#if CONFIG_SUPERTX + none_rate_nocoef += cpi->partition_cost[pl][PARTITION_NONE]; +#endif + } + +#if !CONFIG_PVQ + restore_context(x, &x_ctx, mi_row, mi_col, bsize); +#else + restore_context(x, &x_ctx, mi_row, mi_col, &pre_rdo_buf, bsize); +#endif + mib[0]->mbmi.sb_type = bs_type; + pc_tree->partitioning = partition; + } + } + + switch (partition) { + case PARTITION_NONE: + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc, +#if CONFIG_SUPERTX + &last_part_rate_nocoef, +#endif +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_NONE, +#endif + bsize, ctx_none, INT64_MAX); + break; + case PARTITION_HORZ: + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc, +#if CONFIG_SUPERTX + &last_part_rate_nocoef, +#endif +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_HORZ, +#endif + subsize, &pc_tree->horizontal[0], INT64_MAX); + if (last_part_rdc.rate != INT_MAX && bsize >= BLOCK_8X8 && + mi_row + hbs < cm->mi_rows) { + RD_STATS tmp_rdc; +#if CONFIG_SUPERTX + int rt_nocoef = 0; +#endif + PICK_MODE_CONTEXT *ctx_h = &pc_tree->horizontal[0]; + av1_init_rd_stats(&tmp_rdc); + update_state(cpi, td, ctx_h, mi_row, mi_col, subsize, 1); + encode_superblock(cpi, td, tp, DRY_RUN_NORMAL, mi_row, mi_col, subsize, + ctx_h, NULL); + rd_pick_sb_modes(cpi, tile_data, x, mi_row + hbs, mi_col, &tmp_rdc, +#if CONFIG_SUPERTX + &rt_nocoef, +#endif +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_HORZ, +#endif + subsize, &pc_tree->horizontal[1], INT64_MAX); + if (tmp_rdc.rate == INT_MAX || tmp_rdc.dist == INT64_MAX) { + av1_invalid_rd_stats(&last_part_rdc); +#if CONFIG_SUPERTX + last_part_rate_nocoef = INT_MAX; +#endif + break; + } + last_part_rdc.rate += tmp_rdc.rate; + last_part_rdc.dist += tmp_rdc.dist; + last_part_rdc.rdcost += tmp_rdc.rdcost; +#if CONFIG_SUPERTX + last_part_rate_nocoef += rt_nocoef; +#endif + } + break; + case PARTITION_VERT: + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc, +#if CONFIG_SUPERTX + &last_part_rate_nocoef, +#endif +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_VERT, +#endif + subsize, &pc_tree->vertical[0], INT64_MAX); + if (last_part_rdc.rate != INT_MAX && bsize >= BLOCK_8X8 && + mi_col + hbs < cm->mi_cols) { + RD_STATS tmp_rdc; +#if CONFIG_SUPERTX + int rt_nocoef = 0; +#endif + PICK_MODE_CONTEXT *ctx_v = &pc_tree->vertical[0]; + av1_init_rd_stats(&tmp_rdc); + update_state(cpi, td, ctx_v, mi_row, mi_col, subsize, 1); + encode_superblock(cpi, td, tp, DRY_RUN_NORMAL, mi_row, mi_col, subsize, + ctx_v, NULL); + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col + hbs, &tmp_rdc, +#if CONFIG_SUPERTX + &rt_nocoef, +#endif +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_VERT, +#endif + subsize, &pc_tree->vertical[bsize > BLOCK_8X8], + INT64_MAX); + if (tmp_rdc.rate == INT_MAX || tmp_rdc.dist == INT64_MAX) { + av1_invalid_rd_stats(&last_part_rdc); +#if CONFIG_SUPERTX + last_part_rate_nocoef = INT_MAX; +#endif + break; + } + last_part_rdc.rate += tmp_rdc.rate; + last_part_rdc.dist += tmp_rdc.dist; + last_part_rdc.rdcost += tmp_rdc.rdcost; +#if CONFIG_SUPERTX + last_part_rate_nocoef += rt_nocoef; +#endif + } + break; + case PARTITION_SPLIT: + if (bsize == BLOCK_8X8 && !unify_bsize) { + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc, +#if CONFIG_SUPERTX + &last_part_rate_nocoef, +#endif +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_SPLIT, +#endif + subsize, pc_tree->leaf_split[0], INT64_MAX); + break; + } + last_part_rdc.rate = 0; + last_part_rdc.dist = 0; + last_part_rdc.rdcost = 0; +#if CONFIG_SUPERTX + last_part_rate_nocoef = 0; +#endif + for (i = 0; i < 4; i++) { + int x_idx = (i & 1) * hbs; + int y_idx = (i >> 1) * hbs; + int jj = i >> 1, ii = i & 0x01; + RD_STATS tmp_rdc; +#if CONFIG_SUPERTX + int rt_nocoef; +#endif + if ((mi_row + y_idx >= cm->mi_rows) || (mi_col + x_idx >= cm->mi_cols)) + continue; + + av1_init_rd_stats(&tmp_rdc); + rd_use_partition(cpi, td, tile_data, + mib + jj * hbs * cm->mi_stride + ii * hbs, tp, + mi_row + y_idx, mi_col + x_idx, subsize, &tmp_rdc.rate, + &tmp_rdc.dist, +#if CONFIG_SUPERTX + &rt_nocoef, +#endif + i != 3, pc_tree->split[i]); + if (tmp_rdc.rate == INT_MAX || tmp_rdc.dist == INT64_MAX) { + av1_invalid_rd_stats(&last_part_rdc); +#if CONFIG_SUPERTX + last_part_rate_nocoef = INT_MAX; +#endif + break; + } + last_part_rdc.rate += tmp_rdc.rate; + last_part_rdc.dist += tmp_rdc.dist; +#if CONFIG_SUPERTX + last_part_rate_nocoef += rt_nocoef; +#endif + } + break; +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_VERT_A: + case PARTITION_VERT_B: + case PARTITION_HORZ_A: + case PARTITION_HORZ_B: assert(0 && "Cannot handle extended partiton types"); +#endif // CONFIG_EXT_PARTITION_TYPES + default: assert(0); break; + } + + if (last_part_rdc.rate < INT_MAX) { + last_part_rdc.rate += cpi->partition_cost[pl][partition]; + last_part_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, last_part_rdc.rate, last_part_rdc.dist); +#if CONFIG_SUPERTX + last_part_rate_nocoef += cpi->partition_cost[pl][partition]; +#endif + } + + if (do_partition_search && cpi->sf.adjust_partitioning_from_last_frame && + cpi->sf.partition_search_type == SEARCH_PARTITION && + partition != PARTITION_SPLIT && bsize > BLOCK_8X8 && + (mi_row + bs < cm->mi_rows || mi_row + hbs == cm->mi_rows) && + (mi_col + bs < cm->mi_cols || mi_col + hbs == cm->mi_cols)) { + BLOCK_SIZE split_subsize = get_subsize(bsize, PARTITION_SPLIT); + chosen_rdc.rate = 0; + chosen_rdc.dist = 0; +#if CONFIG_SUPERTX + chosen_rate_nocoef = 0; +#endif +#if !CONFIG_PVQ + restore_context(x, &x_ctx, mi_row, mi_col, bsize); +#else + restore_context(x, &x_ctx, mi_row, mi_col, &pre_rdo_buf, bsize); +#endif + pc_tree->partitioning = PARTITION_SPLIT; + + // Split partition. + for (i = 0; i < 4; i++) { + int x_idx = (i & 1) * hbs; + int y_idx = (i >> 1) * hbs; + RD_STATS tmp_rdc; +#if CONFIG_SUPERTX + int rt_nocoef = 0; +#endif +#if CONFIG_PVQ + od_rollback_buffer buf; +#endif + if ((mi_row + y_idx >= cm->mi_rows) || (mi_col + x_idx >= cm->mi_cols)) + continue; + +#if !CONFIG_PVQ + save_context(x, &x_ctx, mi_row, mi_col, bsize); +#else + save_context(x, &x_ctx, mi_row, mi_col, &buf, bsize); +#endif + pc_tree->split[i]->partitioning = PARTITION_NONE; + rd_pick_sb_modes(cpi, tile_data, x, mi_row + y_idx, mi_col + x_idx, + &tmp_rdc, +#if CONFIG_SUPERTX + &rt_nocoef, +#endif +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_SPLIT, +#endif + split_subsize, &pc_tree->split[i]->none, INT64_MAX); + +#if !CONFIG_PVQ + restore_context(x, &x_ctx, mi_row, mi_col, bsize); +#else + restore_context(x, &x_ctx, mi_row, mi_col, &buf, bsize); +#endif + if (tmp_rdc.rate == INT_MAX || tmp_rdc.dist == INT64_MAX) { + av1_invalid_rd_stats(&chosen_rdc); +#if CONFIG_SUPERTX + chosen_rate_nocoef = INT_MAX; +#endif + break; + } + + chosen_rdc.rate += tmp_rdc.rate; + chosen_rdc.dist += tmp_rdc.dist; +#if CONFIG_SUPERTX + chosen_rate_nocoef += rt_nocoef; +#endif + + if (i != 3) + encode_sb(cpi, td, tile_info, tp, mi_row + y_idx, mi_col + x_idx, + OUTPUT_ENABLED, split_subsize, pc_tree->split[i], NULL); + + chosen_rdc.rate += cpi->partition_cost[pl][PARTITION_NONE]; +#if CONFIG_SUPERTX + chosen_rate_nocoef += cpi->partition_cost[pl][PARTITION_SPLIT]; +#endif + } + if (chosen_rdc.rate < INT_MAX) { + chosen_rdc.rate += cpi->partition_cost[pl][PARTITION_SPLIT]; + chosen_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, chosen_rdc.rate, chosen_rdc.dist); +#if CONFIG_SUPERTX + chosen_rate_nocoef += cpi->partition_cost[pl][PARTITION_NONE]; +#endif + } + } + + // If last_part is better set the partitioning to that. + if (last_part_rdc.rdcost < chosen_rdc.rdcost) { + mib[0]->mbmi.sb_type = bsize; + if (bsize >= BLOCK_8X8) pc_tree->partitioning = partition; + chosen_rdc = last_part_rdc; +#if CONFIG_SUPERTX + chosen_rate_nocoef = last_part_rate_nocoef; +#endif + } + // If none was better set the partitioning to that. + if (none_rdc.rdcost < chosen_rdc.rdcost) { + if (bsize >= BLOCK_8X8) pc_tree->partitioning = PARTITION_NONE; + chosen_rdc = none_rdc; +#if CONFIG_SUPERTX + chosen_rate_nocoef = none_rate_nocoef; +#endif + } + +#if !CONFIG_PVQ + restore_context(x, &x_ctx, mi_row, mi_col, bsize); +#else + restore_context(x, &x_ctx, mi_row, mi_col, &pre_rdo_buf, bsize); +#endif + + // We must have chosen a partitioning and encoding or we'll fail later on. + // No other opportunities for success. + if (bsize == cm->sb_size) + assert(chosen_rdc.rate < INT_MAX && chosen_rdc.dist < INT64_MAX); + + if (do_recon) { + if (bsize == cm->sb_size) { + // NOTE: To get estimate for rate due to the tokens, use: + // int rate_coeffs = 0; + // encode_sb(cpi, td, tile_info, tp, mi_row, mi_col, DRY_RUN_COSTCOEFFS, + // bsize, pc_tree, &rate_coeffs); + encode_sb(cpi, td, tile_info, tp, mi_row, mi_col, OUTPUT_ENABLED, bsize, + pc_tree, NULL); + } else { + encode_sb(cpi, td, tile_info, tp, mi_row, mi_col, DRY_RUN_NORMAL, bsize, + pc_tree, NULL); + } + } + + *rate = chosen_rdc.rate; + *dist = chosen_rdc.dist; +#if CONFIG_SUPERTX + *rate_nocoef = chosen_rate_nocoef; +#endif +} + +/* clang-format off */ +static const BLOCK_SIZE min_partition_size[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + BLOCK_2X2, BLOCK_2X2, BLOCK_2X2, // 2x2, 2x4, 4x2 +#endif + BLOCK_4X4, // 4x4 + BLOCK_4X4, BLOCK_4X4, BLOCK_4X4, // 4x8, 8x4, 8x8 + BLOCK_4X4, BLOCK_4X4, BLOCK_8X8, // 8x16, 16x8, 16x16 + BLOCK_8X8, BLOCK_8X8, BLOCK_16X16, // 16x32, 32x16, 32x32 + BLOCK_16X16, BLOCK_16X16, BLOCK_16X16, // 32x64, 64x32, 64x64 +#if CONFIG_EXT_PARTITION + BLOCK_16X16, BLOCK_16X16, BLOCK_16X16 // 64x128, 128x64, 128x128 +#endif // CONFIG_EXT_PARTITION +}; + +static const BLOCK_SIZE max_partition_size[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + BLOCK_4X4, BLOCK_4X4, BLOCK_4X4, // 2x2, 2x4, 4x2 +#endif + BLOCK_8X8, // 4x4 + BLOCK_16X16, BLOCK_16X16, BLOCK_16X16, // 4x8, 8x4, 8x8 + BLOCK_32X32, BLOCK_32X32, BLOCK_32X32, // 8x16, 16x8, 16x16 + BLOCK_64X64, BLOCK_64X64, BLOCK_64X64, // 16x32, 32x16, 32x32 + BLOCK_LARGEST, BLOCK_LARGEST, BLOCK_LARGEST, // 32x64, 64x32, 64x64 +#if CONFIG_EXT_PARTITION + BLOCK_LARGEST, BLOCK_LARGEST, BLOCK_LARGEST // 64x128, 128x64, 128x128 +#endif // CONFIG_EXT_PARTITION +}; + +// Next square block size less or equal than current block size. +static const BLOCK_SIZE next_square_size[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + BLOCK_2X2, BLOCK_2X2, BLOCK_2X2, // 2x2, 2x4, 4x2 +#endif + BLOCK_4X4, // 4x4 + BLOCK_4X4, BLOCK_4X4, BLOCK_8X8, // 4x8, 8x4, 8x8 + BLOCK_8X8, BLOCK_8X8, BLOCK_16X16, // 8x16, 16x8, 16x16 + BLOCK_16X16, BLOCK_16X16, BLOCK_32X32, // 16x32, 32x16, 32x32 + BLOCK_32X32, BLOCK_32X32, BLOCK_64X64, // 32x64, 64x32, 64x64 +#if CONFIG_EXT_PARTITION + BLOCK_64X64, BLOCK_64X64, BLOCK_128X128 // 64x128, 128x64, 128x128 +#endif // CONFIG_EXT_PARTITION +}; +/* clang-format on */ + +// Look at all the mode_info entries for blocks that are part of this +// partition and find the min and max values for sb_type. +// At the moment this is designed to work on a superblock but could be +// adjusted to use a size parameter. +// +// The min and max are assumed to have been initialized prior to calling this +// function so repeat calls can accumulate a min and max of more than one +// superblock. +static void get_sb_partition_size_range(const AV1_COMMON *const cm, + MACROBLOCKD *xd, MODE_INFO **mib, + BLOCK_SIZE *min_block_size, + BLOCK_SIZE *max_block_size) { + int i, j; + int index = 0; + + // Check the sb_type for each block that belongs to this region. + for (i = 0; i < cm->mib_size; ++i) { + for (j = 0; j < cm->mib_size; ++j) { + MODE_INFO *mi = mib[index + j]; + BLOCK_SIZE sb_type = mi ? mi->mbmi.sb_type : BLOCK_4X4; + *min_block_size = AOMMIN(*min_block_size, sb_type); + *max_block_size = AOMMAX(*max_block_size, sb_type); + } + index += xd->mi_stride; + } +} + +// Look at neighboring blocks and set a min and max partition size based on +// what they chose. +static void rd_auto_partition_range(AV1_COMP *cpi, const TileInfo *const tile, + MACROBLOCKD *const xd, int mi_row, + int mi_col, BLOCK_SIZE *min_block_size, + BLOCK_SIZE *max_block_size) { + AV1_COMMON *const cm = &cpi->common; + MODE_INFO **mi = xd->mi; + const int left_in_image = xd->left_available && mi[-1]; + const int above_in_image = xd->up_available && mi[-xd->mi_stride]; + const int mi_rows_remaining = tile->mi_row_end - mi_row; + const int mi_cols_remaining = tile->mi_col_end - mi_col; + int bh, bw; + BLOCK_SIZE min_size = BLOCK_4X4; + BLOCK_SIZE max_size = BLOCK_LARGEST; + + // Trap case where we do not have a prediction. + if (left_in_image || above_in_image || cm->frame_type != KEY_FRAME) { + // Default "min to max" and "max to min" + min_size = BLOCK_LARGEST; + max_size = BLOCK_4X4; + + // NOTE: each call to get_sb_partition_size_range() uses the previous + // passed in values for min and max as a starting point. + // Find the min and max partition used in previous frame at this location + if (cm->frame_type != KEY_FRAME) { + MODE_INFO **prev_mi = + &cm->prev_mi_grid_visible[mi_row * xd->mi_stride + mi_col]; + get_sb_partition_size_range(cm, xd, prev_mi, &min_size, &max_size); + } + // Find the min and max partition sizes used in the left superblock + if (left_in_image) { + MODE_INFO **left_sb_mi = &mi[-cm->mib_size]; + get_sb_partition_size_range(cm, xd, left_sb_mi, &min_size, &max_size); + } + // Find the min and max partition sizes used in the above suprblock. + if (above_in_image) { + MODE_INFO **above_sb_mi = &mi[-xd->mi_stride * cm->mib_size]; + get_sb_partition_size_range(cm, xd, above_sb_mi, &min_size, &max_size); + } + + // Adjust observed min and max for "relaxed" auto partition case. + if (cpi->sf.auto_min_max_partition_size == RELAXED_NEIGHBORING_MIN_MAX) { + min_size = min_partition_size[min_size]; + max_size = max_partition_size[max_size]; + } + } + + // Check border cases where max and min from neighbors may not be legal. + max_size = find_partition_size(max_size, mi_rows_remaining, mi_cols_remaining, + &bh, &bw); + min_size = AOMMIN(min_size, max_size); + + // Test for blocks at the edge of the active image. + // This may be the actual edge of the image or where there are formatting + // bars. + if (av1_active_edge_sb(cpi, mi_row, mi_col)) { + min_size = BLOCK_4X4; + } else { + min_size = AOMMIN(cpi->sf.rd_auto_partition_min_limit, min_size); + } + + // When use_square_partition_only is true, make sure at least one square + // partition is allowed by selecting the next smaller square size as + // *min_block_size. + if (cpi->sf.use_square_partition_only) { + min_size = AOMMIN(min_size, next_square_size[max_size]); + } + + *min_block_size = AOMMIN(min_size, cm->sb_size); + *max_block_size = AOMMIN(max_size, cm->sb_size); +} + +// TODO(jingning) refactor functions setting partition search range +static void set_partition_range(const AV1_COMMON *const cm, + const MACROBLOCKD *const xd, int mi_row, + int mi_col, BLOCK_SIZE bsize, + BLOCK_SIZE *const min_bs, + BLOCK_SIZE *const max_bs) { + const int mi_width = mi_size_wide[bsize]; + const int mi_height = mi_size_high[bsize]; + int idx, idy; + + const int idx_str = cm->mi_stride * mi_row + mi_col; + MODE_INFO **const prev_mi = &cm->prev_mi_grid_visible[idx_str]; + BLOCK_SIZE min_size = BLOCK_64X64; // default values + BLOCK_SIZE max_size = BLOCK_4X4; + + if (prev_mi) { + for (idy = 0; idy < mi_height; ++idy) { + for (idx = 0; idx < mi_width; ++idx) { + const MODE_INFO *const mi = prev_mi[idy * cm->mi_stride + idx]; + const BLOCK_SIZE bs = mi ? mi->mbmi.sb_type : bsize; + min_size = AOMMIN(min_size, bs); + max_size = AOMMAX(max_size, bs); + } + } + } + + if (xd->left_available) { + for (idy = 0; idy < mi_height; ++idy) { + const MODE_INFO *const mi = xd->mi[idy * cm->mi_stride - 1]; + const BLOCK_SIZE bs = mi ? mi->mbmi.sb_type : bsize; + min_size = AOMMIN(min_size, bs); + max_size = AOMMAX(max_size, bs); + } + } + + if (xd->up_available) { + for (idx = 0; idx < mi_width; ++idx) { + const MODE_INFO *const mi = xd->mi[idx - cm->mi_stride]; + const BLOCK_SIZE bs = mi ? mi->mbmi.sb_type : bsize; + min_size = AOMMIN(min_size, bs); + max_size = AOMMAX(max_size, bs); + } + } + + if (min_size == max_size) { + min_size = min_partition_size[min_size]; + max_size = max_partition_size[max_size]; + } + + *min_bs = AOMMIN(min_size, cm->sb_size); + *max_bs = AOMMIN(max_size, cm->sb_size); +} + +static INLINE void store_pred_mv(MACROBLOCK *x, PICK_MODE_CONTEXT *ctx) { + memcpy(ctx->pred_mv, x->pred_mv, sizeof(x->pred_mv)); +} + +static INLINE void load_pred_mv(MACROBLOCK *x, PICK_MODE_CONTEXT *ctx) { + memcpy(x->pred_mv, ctx->pred_mv, sizeof(x->pred_mv)); +} + +#if CONFIG_FP_MB_STATS +const int qindex_skip_threshold_lookup[BLOCK_SIZES] = { + 0, + 10, + 10, + 30, + 40, + 40, + 60, + 80, + 80, + 90, + 100, + 100, + 120, +#if CONFIG_EXT_PARTITION + // TODO(debargha): What are the correct numbers here? + 130, + 130, + 150 +#endif // CONFIG_EXT_PARTITION +}; +const int qindex_split_threshold_lookup[BLOCK_SIZES] = { + 0, + 3, + 3, + 7, + 15, + 15, + 30, + 40, + 40, + 60, + 80, + 80, + 120, +#if CONFIG_EXT_PARTITION + // TODO(debargha): What are the correct numbers here? + 160, + 160, + 240 +#endif // CONFIG_EXT_PARTITION +}; +const int complexity_16x16_blocks_threshold[BLOCK_SIZES] = { + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 4, + 4, + 6, +#if CONFIG_EXT_PARTITION + // TODO(debargha): What are the correct numbers here? + 8, + 8, + 10 +#endif // CONFIG_EXT_PARTITION +}; + +typedef enum { + MV_ZERO = 0, + MV_LEFT = 1, + MV_UP = 2, + MV_RIGHT = 3, + MV_DOWN = 4, + MV_INVALID +} MOTION_DIRECTION; + +static INLINE MOTION_DIRECTION get_motion_direction_fp(uint8_t fp_byte) { + if (fp_byte & FPMB_MOTION_ZERO_MASK) { + return MV_ZERO; + } else if (fp_byte & FPMB_MOTION_LEFT_MASK) { + return MV_LEFT; + } else if (fp_byte & FPMB_MOTION_RIGHT_MASK) { + return MV_RIGHT; + } else if (fp_byte & FPMB_MOTION_UP_MASK) { + return MV_UP; + } else { + return MV_DOWN; + } +} + +static INLINE int get_motion_inconsistency(MOTION_DIRECTION this_mv, + MOTION_DIRECTION that_mv) { + if (this_mv == that_mv) { + return 0; + } else { + return abs(this_mv - that_mv) == 2 ? 2 : 1; + } +} +#endif + +#if CONFIG_EXT_PARTITION_TYPES +static void rd_test_partition3( + const AV1_COMP *const cpi, ThreadData *td, TileDataEnc *tile_data, + TOKENEXTRA **tp, PC_TREE *pc_tree, RD_STATS *best_rdc, + PICK_MODE_CONTEXT ctxs[3], PICK_MODE_CONTEXT *ctx, int mi_row, int mi_col, + BLOCK_SIZE bsize, PARTITION_TYPE partition, +#if CONFIG_SUPERTX + int64_t best_rd, int *best_rate_nocoef, RD_SEARCH_MACROBLOCK_CONTEXT *x_ctx, +#endif + int mi_row0, int mi_col0, BLOCK_SIZE subsize0, int mi_row1, int mi_col1, + BLOCK_SIZE subsize1, int mi_row2, int mi_col2, BLOCK_SIZE subsize2) { + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + RD_STATS this_rdc, sum_rdc; +#if CONFIG_SUPERTX + const AV1_COMMON *const cm = &cpi->common; + TileInfo *const tile_info = &tile_data->tile_info; + int this_rate_nocoef, sum_rate_nocoef; + int abort_flag; + const int supertx_allowed = !frame_is_intra_only(cm) && + bsize <= MAX_SUPERTX_BLOCK_SIZE && + !xd->lossless[0]; +#endif + if (cpi->sf.adaptive_motion_search) load_pred_mv(x, ctx); + + rd_pick_sb_modes(cpi, tile_data, x, mi_row0, mi_col0, &sum_rdc, +#if CONFIG_SUPERTX + &sum_rate_nocoef, +#endif +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + subsize0, &ctxs[0], best_rdc->rdcost); +#if CONFIG_SUPERTX + abort_flag = sum_rdc.rdcost >= best_rd; +#endif + +#if CONFIG_SUPERTX + if (sum_rdc.rdcost < INT64_MAX) { +#else + if (sum_rdc.rdcost < best_rdc->rdcost) { +#endif + PICK_MODE_CONTEXT *ctx_0 = &ctxs[0]; + update_state(cpi, td, ctx_0, mi_row0, mi_col0, subsize0, 1); + encode_superblock(cpi, td, tp, DRY_RUN_NORMAL, mi_row0, mi_col0, subsize0, + ctx_0, NULL); + + if (cpi->sf.adaptive_motion_search) load_pred_mv(x, ctx_0); + +#if CONFIG_SUPERTX + rd_pick_sb_modes(cpi, tile_data, x, mi_row1, mi_col1, &this_rdc, + &this_rate_nocoef, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + subsize1, &ctxs[1], INT64_MAX - sum_rdc.rdcost); +#else + rd_pick_sb_modes(cpi, tile_data, x, mi_row1, mi_col1, &this_rdc, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + subsize1, &ctxs[1], best_rdc->rdcost - sum_rdc.rdcost); +#endif // CONFIG_SUPERTX + + if (this_rdc.rate == INT_MAX) { + sum_rdc.rdcost = INT64_MAX; +#if CONFIG_SUPERTX + sum_rate_nocoef = INT_MAX; +#endif + } else { + sum_rdc.rate += this_rdc.rate; + sum_rdc.dist += this_rdc.dist; + sum_rdc.rdcost += this_rdc.rdcost; +#if CONFIG_SUPERTX + sum_rate_nocoef += this_rate_nocoef; +#endif + } + +#if CONFIG_SUPERTX + if (sum_rdc.rdcost < INT64_MAX) { +#else + if (sum_rdc.rdcost < best_rdc->rdcost) { +#endif + PICK_MODE_CONTEXT *ctx_1 = &ctxs[1]; + update_state(cpi, td, ctx_1, mi_row1, mi_col1, subsize1, 1); + encode_superblock(cpi, td, tp, DRY_RUN_NORMAL, mi_row1, mi_col1, subsize1, + ctx_1, NULL); + + if (cpi->sf.adaptive_motion_search) load_pred_mv(x, ctx_1); + +#if CONFIG_SUPERTX + rd_pick_sb_modes(cpi, tile_data, x, mi_row2, mi_col2, &this_rdc, + &this_rate_nocoef, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + subsize2, &ctxs[2], INT64_MAX - sum_rdc.rdcost); +#else + rd_pick_sb_modes(cpi, tile_data, x, mi_row2, mi_col2, &this_rdc, +#if CONFIG_EXT_PARTITION_TYPES + partition, +#endif + subsize2, &ctxs[2], best_rdc->rdcost - sum_rdc.rdcost); +#endif // CONFIG_SUPERTX + + if (this_rdc.rate == INT_MAX) { + sum_rdc.rdcost = INT64_MAX; +#if CONFIG_SUPERTX + sum_rate_nocoef = INT_MAX; +#endif + } else { + sum_rdc.rate += this_rdc.rate; + sum_rdc.dist += this_rdc.dist; + sum_rdc.rdcost += this_rdc.rdcost; +#if CONFIG_SUPERTX + sum_rate_nocoef += this_rate_nocoef; +#endif + } + +#if CONFIG_SUPERTX + if (supertx_allowed && !abort_flag && sum_rdc.rdcost < INT64_MAX) { + TX_SIZE supertx_size = max_txsize_lookup[bsize]; + const PARTITION_TYPE best_partition = pc_tree->partitioning; + pc_tree->partitioning = partition; + sum_rdc.rate += av1_cost_bit( + cm->fc->supertx_prob[partition_supertx_context_lookup[partition]] + [supertx_size], + 0); + sum_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist); + + if (!check_intra_sb(cpi, tile_info, mi_row, mi_col, bsize, pc_tree)) { + TX_TYPE best_tx = DCT_DCT; + RD_STATS tmp_rdc = { sum_rate_nocoef, 0, 0 }; + + restore_context(x, x_ctx, mi_row, mi_col, bsize); + + rd_supertx_sb(cpi, td, tile_info, mi_row, mi_col, bsize, + &tmp_rdc.rate, &tmp_rdc.dist, &best_tx, pc_tree); + + tmp_rdc.rate += av1_cost_bit( + cm->fc->supertx_prob[partition_supertx_context_lookup[partition]] + [supertx_size], + 1); + tmp_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, tmp_rdc.rate, tmp_rdc.dist); + if (tmp_rdc.rdcost < sum_rdc.rdcost) { + sum_rdc = tmp_rdc; + update_supertx_param_sb(cpi, td, mi_row, mi_col, bsize, best_tx, + supertx_size, pc_tree); + } + } + + pc_tree->partitioning = best_partition; + } +#endif // CONFIG_SUPERTX + + if (sum_rdc.rdcost < best_rdc->rdcost) { + int pl = partition_plane_context(xd, mi_row, mi_col, +#if CONFIG_UNPOISON_PARTITION_CTX + has_rows, has_cols, +#endif + bsize); + sum_rdc.rate += cpi->partition_cost[pl][partition]; + sum_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist); +#if CONFIG_SUPERTX + sum_rate_nocoef += cpi->partition_cost[pl][partition]; +#endif + if (sum_rdc.rdcost < best_rdc->rdcost) { +#if CONFIG_SUPERTX + *best_rate_nocoef = sum_rate_nocoef; + assert(*best_rate_nocoef >= 0); +#endif + *best_rdc = sum_rdc; + pc_tree->partitioning = partition; + } + } + } + } +} +#endif // CONFIG_EXT_PARTITION_TYPES + +// TODO(jingning,jimbankoski,rbultje): properly skip partition types that are +// unlikely to be selected depending on previous rate-distortion optimization +// results, for encoding speed-up. +static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td, + TileDataEnc *tile_data, TOKENEXTRA **tp, + int mi_row, int mi_col, BLOCK_SIZE bsize, + RD_STATS *rd_cost, +#if CONFIG_SUPERTX + int *rate_nocoef, +#endif + int64_t best_rd, PC_TREE *pc_tree) { + const AV1_COMMON *const cm = &cpi->common; + TileInfo *const tile_info = &tile_data->tile_info; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + const int mi_step = mi_size_wide[bsize] / 2; + RD_SEARCH_MACROBLOCK_CONTEXT x_ctx; + const TOKENEXTRA *const tp_orig = *tp; + PICK_MODE_CONTEXT *ctx_none = &pc_tree->none; +#if CONFIG_UNPOISON_PARTITION_CTX + const int hbs = mi_size_wide[bsize] / 2; + const int has_rows = mi_row + hbs < cm->mi_rows; + const int has_cols = mi_col + hbs < cm->mi_cols; +#else + int tmp_partition_cost[PARTITION_TYPES]; +#endif + BLOCK_SIZE subsize; + RD_STATS this_rdc, sum_rdc, best_rdc; + const int bsize_at_least_8x8 = (bsize >= BLOCK_8X8); + int do_square_split = bsize_at_least_8x8; +#if CONFIG_CB4X4 + const int unify_bsize = 1; + const int pl = bsize_at_least_8x8 + ? partition_plane_context(xd, mi_row, mi_col, +#if CONFIG_UNPOISON_PARTITION_CTX + has_rows, has_cols, +#endif + bsize) + : 0; +#else + const int unify_bsize = 0; + const int pl = partition_plane_context(xd, mi_row, mi_col, +#if CONFIG_UNPOISON_PARTITION_CTX + has_rows, has_cols, +#endif + bsize); +#endif // CONFIG_CB4X4 + const int *partition_cost = cpi->partition_cost[pl]; +#if CONFIG_SUPERTX + int this_rate_nocoef, sum_rate_nocoef = 0, best_rate_nocoef = INT_MAX; + int abort_flag; + const int supertx_allowed = !frame_is_intra_only(cm) && + bsize <= MAX_SUPERTX_BLOCK_SIZE && + !xd->lossless[0]; +#endif // CONFIG_SUPERTX + + int do_rectangular_split = 1; +#if CONFIG_EXT_PARTITION_TYPES + BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT); +#endif + + // Override skipping rectangular partition operations for edge blocks + const int force_horz_split = (mi_row + mi_step >= cm->mi_rows); + const int force_vert_split = (mi_col + mi_step >= cm->mi_cols); + const int xss = x->e_mbd.plane[1].subsampling_x; + const int yss = x->e_mbd.plane[1].subsampling_y; + + BLOCK_SIZE min_size = x->min_partition_size; + BLOCK_SIZE max_size = x->max_partition_size; + +#if CONFIG_FP_MB_STATS + unsigned int src_diff_var = UINT_MAX; + int none_complexity = 0; +#endif + + int partition_none_allowed = !force_horz_split && !force_vert_split; + int partition_horz_allowed = + !force_vert_split && yss <= xss && bsize_at_least_8x8; + int partition_vert_allowed = + !force_horz_split && xss <= yss && bsize_at_least_8x8; + +#if CONFIG_PVQ + od_rollback_buffer pre_rdo_buf; +#endif + + (void)*tp_orig; + +#if !CONFIG_UNPOISON_PARTITION_CTX + if (force_horz_split || force_vert_split) { + tmp_partition_cost[PARTITION_NONE] = INT_MAX; + + if (!force_vert_split) { // force_horz_split only + tmp_partition_cost[PARTITION_VERT] = INT_MAX; + tmp_partition_cost[PARTITION_HORZ] = + av1_cost_bit(cm->fc->partition_prob[pl][PARTITION_HORZ], 0); + tmp_partition_cost[PARTITION_SPLIT] = + av1_cost_bit(cm->fc->partition_prob[pl][PARTITION_HORZ], 1); + } else if (!force_horz_split) { // force_vert_split only + tmp_partition_cost[PARTITION_HORZ] = INT_MAX; + tmp_partition_cost[PARTITION_VERT] = + av1_cost_bit(cm->fc->partition_prob[pl][PARTITION_VERT], 0); + tmp_partition_cost[PARTITION_SPLIT] = + av1_cost_bit(cm->fc->partition_prob[pl][PARTITION_VERT], 1); + } else { // force_ horz_split && force_vert_split horz_split + tmp_partition_cost[PARTITION_HORZ] = INT_MAX; + tmp_partition_cost[PARTITION_VERT] = INT_MAX; + tmp_partition_cost[PARTITION_SPLIT] = 0; + } + + partition_cost = tmp_partition_cost; + } +#endif + +#if CONFIG_VAR_TX +#ifndef NDEBUG + // Nothing should rely on the default value of this array (which is just + // leftover from encoding the previous block. Setting it to magic number + // when debugging. + memset(x->blk_skip[0], 234, sizeof(x->blk_skip[0])); +#endif // NDEBUG +#endif // CONFIG_VAR_TX + + assert(mi_size_wide[bsize] == mi_size_high[bsize]); + + av1_init_rd_stats(&this_rdc); + av1_init_rd_stats(&sum_rdc); + av1_invalid_rd_stats(&best_rdc); + best_rdc.rdcost = best_rd; + + set_offsets(cpi, tile_info, x, mi_row, mi_col, bsize); + + if (bsize == BLOCK_16X16 && cpi->vaq_refresh) + x->mb_energy = av1_block_energy(cpi, x, bsize); + + if (cpi->sf.cb_partition_search && bsize == BLOCK_16X16) { + const int cb_partition_search_ctrl = + ((pc_tree->index == 0 || pc_tree->index == 3) + + get_chessboard_index(cm->current_video_frame)) & + 0x1; + + if (cb_partition_search_ctrl && bsize > min_size && bsize < max_size) + set_partition_range(cm, xd, mi_row, mi_col, bsize, &min_size, &max_size); + } + + // Determine partition types in search according to the speed features. + // The threshold set here has to be of square block size. + if (cpi->sf.auto_min_max_partition_size) { + const int no_partition_allowed = (bsize <= max_size && bsize >= min_size); + // Note: Further partitioning is NOT allowed when bsize == min_size already. + const int partition_allowed = (bsize <= max_size && bsize > min_size); + partition_none_allowed &= no_partition_allowed; + partition_horz_allowed &= partition_allowed || force_horz_split; + partition_vert_allowed &= partition_allowed || force_vert_split; + do_square_split &= bsize > min_size; + } + if (cpi->sf.use_square_partition_only) { + partition_horz_allowed &= force_horz_split; + partition_vert_allowed &= force_vert_split; + } + +#if CONFIG_VAR_TX + xd->above_txfm_context = cm->above_txfm_context + mi_col; + xd->left_txfm_context = + xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK); +#endif +#if !CONFIG_PVQ + save_context(x, &x_ctx, mi_row, mi_col, bsize); +#else + save_context(x, &x_ctx, mi_row, mi_col, &pre_rdo_buf, bsize); +#endif + +#if CONFIG_FP_MB_STATS + if (cpi->use_fp_mb_stats) { + set_offsets(cpi, tile_info, x, mi_row, mi_col, bsize); + src_diff_var = get_sby_perpixel_diff_variance(cpi, &x->plane[0].src, mi_row, + mi_col, bsize); + } +#endif + +#if CONFIG_FP_MB_STATS + // Decide whether we shall split directly and skip searching NONE by using + // the first pass block statistics + if (cpi->use_fp_mb_stats && bsize >= BLOCK_32X32 && do_square_split && + partition_none_allowed && src_diff_var > 4 && + cm->base_qindex < qindex_split_threshold_lookup[bsize]) { + int mb_row = mi_row >> 1; + int mb_col = mi_col >> 1; + int mb_row_end = + AOMMIN(mb_row + num_16x16_blocks_high_lookup[bsize], cm->mb_rows); + int mb_col_end = + AOMMIN(mb_col + num_16x16_blocks_wide_lookup[bsize], cm->mb_cols); + int r, c; + + // compute a complexity measure, basically measure inconsistency of motion + // vectors obtained from the first pass in the current block + for (r = mb_row; r < mb_row_end; r++) { + for (c = mb_col; c < mb_col_end; c++) { + const int mb_index = r * cm->mb_cols + c; + + MOTION_DIRECTION this_mv; + MOTION_DIRECTION right_mv; + MOTION_DIRECTION bottom_mv; + + this_mv = + get_motion_direction_fp(cpi->twopass.this_frame_mb_stats[mb_index]); + + // to its right + if (c != mb_col_end - 1) { + right_mv = get_motion_direction_fp( + cpi->twopass.this_frame_mb_stats[mb_index + 1]); + none_complexity += get_motion_inconsistency(this_mv, right_mv); + } + + // to its bottom + if (r != mb_row_end - 1) { + bottom_mv = get_motion_direction_fp( + cpi->twopass.this_frame_mb_stats[mb_index + cm->mb_cols]); + none_complexity += get_motion_inconsistency(this_mv, bottom_mv); + } + + // do not count its left and top neighbors to avoid double counting + } + } + + if (none_complexity > complexity_16x16_blocks_threshold[bsize]) { + partition_none_allowed = 0; + } + } +#endif + + // PARTITION_NONE + if (partition_none_allowed) { + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &this_rdc, +#if CONFIG_SUPERTX + &this_rate_nocoef, +#endif +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_NONE, +#endif + bsize, ctx_none, best_rdc.rdcost); + if (this_rdc.rate != INT_MAX) { + if (bsize_at_least_8x8) { + this_rdc.rate += partition_cost[PARTITION_NONE]; + this_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, this_rdc.rate, this_rdc.dist); +#if CONFIG_SUPERTX + this_rate_nocoef += partition_cost[PARTITION_NONE]; +#endif + } + + if (this_rdc.rdcost < best_rdc.rdcost) { + // Adjust dist breakout threshold according to the partition size. + const int64_t dist_breakout_thr = + cpi->sf.partition_search_breakout_dist_thr >> + ((2 * (MAX_SB_SIZE_LOG2 - 2)) - + (b_width_log2_lookup[bsize] + b_height_log2_lookup[bsize])); + const int rate_breakout_thr = + cpi->sf.partition_search_breakout_rate_thr * + num_pels_log2_lookup[bsize]; + + best_rdc = this_rdc; +#if CONFIG_SUPERTX + best_rate_nocoef = this_rate_nocoef; + assert(best_rate_nocoef >= 0); +#endif + if (bsize_at_least_8x8) pc_tree->partitioning = PARTITION_NONE; + + // If all y, u, v transform blocks in this partition are skippable, and + // the dist & rate are within the thresholds, the partition search is + // terminated for current branch of the partition search tree. + // The dist & rate thresholds are set to 0 at speed 0 to disable the + // early termination at that speed. + if (!x->e_mbd.lossless[xd->mi[0]->mbmi.segment_id] && + (ctx_none->skippable && best_rdc.dist < dist_breakout_thr && + best_rdc.rate < rate_breakout_thr)) { + do_square_split = 0; + do_rectangular_split = 0; + } + +#if CONFIG_FP_MB_STATS + // Check if every 16x16 first pass block statistics has zero + // motion and the corresponding first pass residue is small enough. + // If that is the case, check the difference variance between the + // current frame and the last frame. If the variance is small enough, + // stop further splitting in RD optimization + if (cpi->use_fp_mb_stats && do_square_split && + cm->base_qindex > qindex_skip_threshold_lookup[bsize]) { + int mb_row = mi_row >> 1; + int mb_col = mi_col >> 1; + int mb_row_end = + AOMMIN(mb_row + num_16x16_blocks_high_lookup[bsize], cm->mb_rows); + int mb_col_end = + AOMMIN(mb_col + num_16x16_blocks_wide_lookup[bsize], cm->mb_cols); + int r, c; + + int skip = 1; + for (r = mb_row; r < mb_row_end; r++) { + for (c = mb_col; c < mb_col_end; c++) { + const int mb_index = r * cm->mb_cols + c; + if (!(cpi->twopass.this_frame_mb_stats[mb_index] & + FPMB_MOTION_ZERO_MASK) || + !(cpi->twopass.this_frame_mb_stats[mb_index] & + FPMB_ERROR_SMALL_MASK)) { + skip = 0; + break; + } + } + if (skip == 0) { + break; + } + } + if (skip) { + if (src_diff_var == UINT_MAX) { + set_offsets(cpi, tile_info, x, mi_row, mi_col, bsize); + src_diff_var = get_sby_perpixel_diff_variance( + cpi, &x->plane[0].src, mi_row, mi_col, bsize); + } + if (src_diff_var < 8) { + do_square_split = 0; + do_rectangular_split = 0; + } + } + } +#endif + } + } +#if !CONFIG_PVQ + restore_context(x, &x_ctx, mi_row, mi_col, bsize); +#else + restore_context(x, &x_ctx, mi_row, mi_col, &pre_rdo_buf, bsize); +#endif + } + + // store estimated motion vector + if (cpi->sf.adaptive_motion_search) store_pred_mv(x, ctx_none); + + // PARTITION_SPLIT + // TODO(jingning): use the motion vectors given by the above search as + // the starting point of motion search in the following partition type check. + if (do_square_split) { + int reached_last_index = 0; + subsize = get_subsize(bsize, PARTITION_SPLIT); + if (bsize == BLOCK_8X8 && !unify_bsize) { +#if CONFIG_DUAL_FILTER + if (cpi->sf.adaptive_pred_interp_filter && partition_none_allowed) + pc_tree->leaf_split[0]->pred_interp_filter = + ctx_none->mic.mbmi.interp_filter[0]; +#else + if (cpi->sf.adaptive_pred_interp_filter && partition_none_allowed) + pc_tree->leaf_split[0]->pred_interp_filter = + ctx_none->mic.mbmi.interp_filter; +#endif +#if CONFIG_SUPERTX + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &sum_rdc, + &sum_rate_nocoef, +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_SPLIT, +#endif + subsize, pc_tree->leaf_split[0], INT64_MAX); +#else + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &sum_rdc, +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_SPLIT, +#endif + subsize, pc_tree->leaf_split[0], best_rdc.rdcost); +#endif // CONFIG_SUPERTX + if (sum_rdc.rate == INT_MAX) { + sum_rdc.rdcost = INT64_MAX; +#if CONFIG_SUPERTX + sum_rate_nocoef = INT_MAX; +#endif + } +#if CONFIG_SUPERTX + if (supertx_allowed && sum_rdc.rdcost < INT64_MAX) { + TX_SIZE supertx_size = max_txsize_lookup[bsize]; + const PARTITION_TYPE best_partition = pc_tree->partitioning; + + pc_tree->partitioning = PARTITION_SPLIT; + + sum_rdc.rate += av1_cost_bit( + cm->fc->supertx_prob[partition_supertx_context_lookup + [PARTITION_SPLIT]][supertx_size], + 0); + sum_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist); + + if (is_inter_mode(pc_tree->leaf_split[0]->mic.mbmi.mode)) { + TX_TYPE best_tx = DCT_DCT; + RD_STATS tmp_rdc; + av1_init_rd_stats(&tmp_rdc); + tmp_rdc.rate = sum_rate_nocoef; + + restore_context(x, &x_ctx, mi_row, mi_col, bsize); + + rd_supertx_sb(cpi, td, tile_info, mi_row, mi_col, bsize, + &tmp_rdc.rate, &tmp_rdc.dist, &best_tx, pc_tree); + + tmp_rdc.rate += av1_cost_bit( + cm->fc->supertx_prob[partition_supertx_context_lookup + [PARTITION_SPLIT]][supertx_size], + 1); + tmp_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, tmp_rdc.rate, tmp_rdc.dist); + if (tmp_rdc.rdcost < sum_rdc.rdcost) { + sum_rdc = tmp_rdc; + update_supertx_param_sb(cpi, td, mi_row, mi_col, bsize, best_tx, + supertx_size, pc_tree); + } + } + + pc_tree->partitioning = best_partition; + } +#endif // CONFIG_SUPERTX + reached_last_index = 1; + } else { + int idx; +#if CONFIG_SUPERTX + for (idx = 0; idx < 4 && sum_rdc.rdcost < INT64_MAX; ++idx) { +#else + for (idx = 0; idx < 4 && sum_rdc.rdcost < best_rdc.rdcost; ++idx) { +#endif // CONFIG_SUPERTX + const int x_idx = (idx & 1) * mi_step; + const int y_idx = (idx >> 1) * mi_step; + + if (mi_row + y_idx >= cm->mi_rows || mi_col + x_idx >= cm->mi_cols) + continue; + + if (cpi->sf.adaptive_motion_search) load_pred_mv(x, ctx_none); + + pc_tree->split[idx]->index = idx; +#if CONFIG_SUPERTX + rd_pick_partition(cpi, td, tile_data, tp, mi_row + y_idx, + mi_col + x_idx, subsize, &this_rdc, &this_rate_nocoef, + INT64_MAX - sum_rdc.rdcost, pc_tree->split[idx]); +#else + rd_pick_partition( + cpi, td, tile_data, tp, mi_row + y_idx, mi_col + x_idx, subsize, + &this_rdc, best_rdc.rdcost - sum_rdc.rdcost, pc_tree->split[idx]); +#endif // CONFIG_SUPERTX + + if (this_rdc.rate == INT_MAX) { + sum_rdc.rdcost = INT64_MAX; +#if CONFIG_SUPERTX + sum_rate_nocoef = INT_MAX; +#endif // CONFIG_SUPERTX + break; + } else { + sum_rdc.rate += this_rdc.rate; + sum_rdc.dist += this_rdc.dist; + sum_rdc.rdcost += this_rdc.rdcost; +#if CONFIG_SUPERTX + sum_rate_nocoef += this_rate_nocoef; +#endif // CONFIG_SUPERTX + } + } + reached_last_index = (idx == 4); +#if CONFIG_SUPERTX + if (supertx_allowed && sum_rdc.rdcost < INT64_MAX && reached_last_index) { + TX_SIZE supertx_size = max_txsize_lookup[bsize]; + const PARTITION_TYPE best_partition = pc_tree->partitioning; + + pc_tree->partitioning = PARTITION_SPLIT; + + sum_rdc.rate += av1_cost_bit( + cm->fc->supertx_prob[partition_supertx_context_lookup + [PARTITION_SPLIT]][supertx_size], + 0); + sum_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist); + + if (!check_intra_sb(cpi, tile_info, mi_row, mi_col, bsize, pc_tree)) { + TX_TYPE best_tx = DCT_DCT; + RD_STATS tmp_rdc; + av1_init_rd_stats(&tmp_rdc); + tmp_rdc.rate = sum_rate_nocoef; + + restore_context(x, &x_ctx, mi_row, mi_col, bsize); + + rd_supertx_sb(cpi, td, tile_info, mi_row, mi_col, bsize, + &tmp_rdc.rate, &tmp_rdc.dist, &best_tx, pc_tree); + + tmp_rdc.rate += av1_cost_bit( + cm->fc->supertx_prob[partition_supertx_context_lookup + [PARTITION_SPLIT]][supertx_size], + 1); + tmp_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, tmp_rdc.rate, tmp_rdc.dist); + if (tmp_rdc.rdcost < sum_rdc.rdcost) { + sum_rdc = tmp_rdc; + update_supertx_param_sb(cpi, td, mi_row, mi_col, bsize, best_tx, + supertx_size, pc_tree); + } + } + + pc_tree->partitioning = best_partition; + } +#endif // CONFIG_SUPERTX + } + + if (reached_last_index && sum_rdc.rdcost < best_rdc.rdcost) { + sum_rdc.rate += partition_cost[PARTITION_SPLIT]; + sum_rdc.rdcost = RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist); +#if CONFIG_SUPERTX + sum_rate_nocoef += partition_cost[PARTITION_SPLIT]; +#endif // CONFIG_SUPERTX + + if (sum_rdc.rdcost < best_rdc.rdcost) { + best_rdc = sum_rdc; +#if CONFIG_SUPERTX + best_rate_nocoef = sum_rate_nocoef; + assert(best_rate_nocoef >= 0); +#endif // CONFIG_SUPERTX + pc_tree->partitioning = PARTITION_SPLIT; + } + } else if (cpi->sf.less_rectangular_check) { + // skip rectangular partition test when larger block size + // gives better rd cost + do_rectangular_split &= !partition_none_allowed; + } +#if !CONFIG_PVQ + restore_context(x, &x_ctx, mi_row, mi_col, bsize); +#else + restore_context(x, &x_ctx, mi_row, mi_col, &pre_rdo_buf, bsize); +#endif + } // if (do_split) + + // PARTITION_HORZ + if (partition_horz_allowed && + (do_rectangular_split || av1_active_h_edge(cpi, mi_row, mi_step))) { + subsize = get_subsize(bsize, PARTITION_HORZ); + if (cpi->sf.adaptive_motion_search) load_pred_mv(x, ctx_none); +#if CONFIG_DUAL_FILTER + if (cpi->sf.adaptive_pred_interp_filter && bsize == BLOCK_8X8 && + partition_none_allowed) + pc_tree->horizontal[0].pred_interp_filter = + ctx_none->mic.mbmi.interp_filter[0]; +#else + if (cpi->sf.adaptive_pred_interp_filter && bsize == BLOCK_8X8 && + partition_none_allowed) + pc_tree->horizontal[0].pred_interp_filter = + ctx_none->mic.mbmi.interp_filter; +#endif + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &sum_rdc, +#if CONFIG_SUPERTX + &sum_rate_nocoef, +#endif // CONFIG_SUPERTX +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_HORZ, +#endif + subsize, &pc_tree->horizontal[0], best_rdc.rdcost); + +#if CONFIG_SUPERTX + abort_flag = + (sum_rdc.rdcost >= best_rd && (bsize > BLOCK_8X8 || unify_bsize)) || + (sum_rdc.rate == INT_MAX && bsize == BLOCK_8X8); + if (sum_rdc.rdcost < INT64_MAX && +#else + if (sum_rdc.rdcost < best_rdc.rdcost && +#endif // CONFIG_SUPERTX + !force_horz_split && (bsize > BLOCK_8X8 || unify_bsize)) { + PICK_MODE_CONTEXT *ctx_h = &pc_tree->horizontal[0]; + update_state(cpi, td, ctx_h, mi_row, mi_col, subsize, 1); + encode_superblock(cpi, td, tp, DRY_RUN_NORMAL, mi_row, mi_col, subsize, + ctx_h, NULL); + + if (cpi->sf.adaptive_motion_search) load_pred_mv(x, ctx_h); + +#if CONFIG_DUAL_FILTER + if (cpi->sf.adaptive_pred_interp_filter && bsize == BLOCK_8X8 && + partition_none_allowed) + pc_tree->horizontal[1].pred_interp_filter = + ctx_h->mic.mbmi.interp_filter[0]; +#else + if (cpi->sf.adaptive_pred_interp_filter && bsize == BLOCK_8X8 && + partition_none_allowed) + pc_tree->horizontal[1].pred_interp_filter = + ctx_none->mic.mbmi.interp_filter; +#endif +#if CONFIG_SUPERTX + rd_pick_sb_modes(cpi, tile_data, x, mi_row + mi_step, mi_col, &this_rdc, + &this_rate_nocoef, +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_HORZ, +#endif + subsize, &pc_tree->horizontal[1], INT64_MAX); +#else + rd_pick_sb_modes(cpi, tile_data, x, mi_row + mi_step, mi_col, &this_rdc, +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_HORZ, +#endif + subsize, &pc_tree->horizontal[1], + best_rdc.rdcost - sum_rdc.rdcost); +#endif // CONFIG_SUPERTX + if (this_rdc.rate == INT_MAX) { + sum_rdc.rdcost = INT64_MAX; +#if CONFIG_SUPERTX + sum_rate_nocoef = INT_MAX; +#endif // CONFIG_SUPERTX + } else { + sum_rdc.rate += this_rdc.rate; + sum_rdc.dist += this_rdc.dist; + sum_rdc.rdcost += this_rdc.rdcost; +#if CONFIG_SUPERTX + sum_rate_nocoef += this_rate_nocoef; +#endif // CONFIG_SUPERTX + } + } + +#if CONFIG_SUPERTX + if (supertx_allowed && sum_rdc.rdcost < INT64_MAX && !abort_flag) { + TX_SIZE supertx_size = max_txsize_lookup[bsize]; + const PARTITION_TYPE best_partition = pc_tree->partitioning; + + pc_tree->partitioning = PARTITION_HORZ; + + sum_rdc.rate += av1_cost_bit( + cm->fc->supertx_prob[partition_supertx_context_lookup[PARTITION_HORZ]] + [supertx_size], + 0); + sum_rdc.rdcost = RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist); + + if (!check_intra_sb(cpi, tile_info, mi_row, mi_col, bsize, pc_tree)) { + TX_TYPE best_tx = DCT_DCT; + RD_STATS tmp_rdc; + av1_init_rd_stats(&tmp_rdc); + tmp_rdc.rate = sum_rate_nocoef; + + restore_context(x, &x_ctx, mi_row, mi_col, bsize); + + rd_supertx_sb(cpi, td, tile_info, mi_row, mi_col, bsize, &tmp_rdc.rate, + &tmp_rdc.dist, &best_tx, pc_tree); + + tmp_rdc.rate += av1_cost_bit( + cm->fc + ->supertx_prob[partition_supertx_context_lookup[PARTITION_HORZ]] + [supertx_size], + 1); + tmp_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, tmp_rdc.rate, tmp_rdc.dist); + if (tmp_rdc.rdcost < sum_rdc.rdcost) { + sum_rdc = tmp_rdc; + update_supertx_param_sb(cpi, td, mi_row, mi_col, bsize, best_tx, + supertx_size, pc_tree); + } + } + + pc_tree->partitioning = best_partition; + } +#endif // CONFIG_SUPERTX + + if (sum_rdc.rdcost < best_rdc.rdcost) { + sum_rdc.rate += partition_cost[PARTITION_HORZ]; + sum_rdc.rdcost = RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist); +#if CONFIG_SUPERTX + sum_rate_nocoef += partition_cost[PARTITION_HORZ]; +#endif // CONFIG_SUPERTX + if (sum_rdc.rdcost < best_rdc.rdcost) { + best_rdc = sum_rdc; +#if CONFIG_SUPERTX + best_rate_nocoef = sum_rate_nocoef; + assert(best_rate_nocoef >= 0); +#endif // CONFIG_SUPERTX + pc_tree->partitioning = PARTITION_HORZ; + } + } +#if !CONFIG_PVQ + restore_context(x, &x_ctx, mi_row, mi_col, bsize); +#else + restore_context(x, &x_ctx, mi_row, mi_col, &pre_rdo_buf, bsize); +#endif + } + + // PARTITION_VERT + if (partition_vert_allowed && + (do_rectangular_split || av1_active_v_edge(cpi, mi_col, mi_step))) { + subsize = get_subsize(bsize, PARTITION_VERT); + + if (cpi->sf.adaptive_motion_search) load_pred_mv(x, ctx_none); + +#if CONFIG_DUAL_FILTER + if (cpi->sf.adaptive_pred_interp_filter && bsize == BLOCK_8X8 && + partition_none_allowed) + pc_tree->vertical[0].pred_interp_filter = + ctx_none->mic.mbmi.interp_filter[0]; +#else + if (cpi->sf.adaptive_pred_interp_filter && bsize == BLOCK_8X8 && + partition_none_allowed) + pc_tree->vertical[0].pred_interp_filter = + ctx_none->mic.mbmi.interp_filter; +#endif + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &sum_rdc, +#if CONFIG_SUPERTX + &sum_rate_nocoef, +#endif // CONFIG_SUPERTX +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_VERT, +#endif + subsize, &pc_tree->vertical[0], best_rdc.rdcost); +#if CONFIG_SUPERTX + abort_flag = + (sum_rdc.rdcost >= best_rd && (bsize > BLOCK_8X8 || unify_bsize)) || + (sum_rdc.rate == INT_MAX && bsize == BLOCK_8X8); + if (sum_rdc.rdcost < INT64_MAX && +#else + if (sum_rdc.rdcost < best_rdc.rdcost && +#endif // CONFIG_SUPERTX + !force_vert_split && (bsize > BLOCK_8X8 || unify_bsize)) { + update_state(cpi, td, &pc_tree->vertical[0], mi_row, mi_col, subsize, 1); + encode_superblock(cpi, td, tp, DRY_RUN_NORMAL, mi_row, mi_col, subsize, + &pc_tree->vertical[0], NULL); + + if (cpi->sf.adaptive_motion_search) load_pred_mv(x, ctx_none); + +#if CONFIG_DUAL_FILTER + if (cpi->sf.adaptive_pred_interp_filter && bsize == BLOCK_8X8 && + partition_none_allowed) + pc_tree->vertical[1].pred_interp_filter = + ctx_none->mic.mbmi.interp_filter[0]; +#else + if (cpi->sf.adaptive_pred_interp_filter && bsize == BLOCK_8X8 && + partition_none_allowed) + pc_tree->vertical[1].pred_interp_filter = + ctx_none->mic.mbmi.interp_filter; +#endif +#if CONFIG_SUPERTX + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col + mi_step, &this_rdc, + &this_rate_nocoef, +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_VERT, +#endif + subsize, &pc_tree->vertical[1], + INT64_MAX - sum_rdc.rdcost); +#else + rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col + mi_step, &this_rdc, +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_VERT, +#endif + subsize, &pc_tree->vertical[1], + best_rdc.rdcost - sum_rdc.rdcost); +#endif // CONFIG_SUPERTX + if (this_rdc.rate == INT_MAX) { + sum_rdc.rdcost = INT64_MAX; +#if CONFIG_SUPERTX + sum_rate_nocoef = INT_MAX; +#endif // CONFIG_SUPERTX + } else { + sum_rdc.rate += this_rdc.rate; + sum_rdc.dist += this_rdc.dist; + sum_rdc.rdcost += this_rdc.rdcost; +#if CONFIG_SUPERTX + sum_rate_nocoef += this_rate_nocoef; +#endif // CONFIG_SUPERTX + } + } +#if CONFIG_SUPERTX + if (supertx_allowed && sum_rdc.rdcost < INT64_MAX && !abort_flag) { + TX_SIZE supertx_size = max_txsize_lookup[bsize]; + const PARTITION_TYPE best_partition = pc_tree->partitioning; + + pc_tree->partitioning = PARTITION_VERT; + + sum_rdc.rate += av1_cost_bit( + cm->fc->supertx_prob[partition_supertx_context_lookup[PARTITION_VERT]] + [supertx_size], + 0); + sum_rdc.rdcost = RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist); + + if (!check_intra_sb(cpi, tile_info, mi_row, mi_col, bsize, pc_tree)) { + TX_TYPE best_tx = DCT_DCT; + RD_STATS tmp_rdc; + av1_init_rd_stats(&tmp_rdc); + tmp_rdc.rate = sum_rate_nocoef; + + restore_context(x, &x_ctx, mi_row, mi_col, bsize); + + rd_supertx_sb(cpi, td, tile_info, mi_row, mi_col, bsize, &tmp_rdc.rate, + &tmp_rdc.dist, &best_tx, pc_tree); + + tmp_rdc.rate += av1_cost_bit( + cm->fc + ->supertx_prob[partition_supertx_context_lookup[PARTITION_VERT]] + [supertx_size], + 1); + tmp_rdc.rdcost = + RDCOST(x->rdmult, x->rddiv, tmp_rdc.rate, tmp_rdc.dist); + if (tmp_rdc.rdcost < sum_rdc.rdcost) { + sum_rdc = tmp_rdc; + update_supertx_param_sb(cpi, td, mi_row, mi_col, bsize, best_tx, + supertx_size, pc_tree); + } + } + + pc_tree->partitioning = best_partition; + } +#endif // CONFIG_SUPERTX + + if (sum_rdc.rdcost < best_rdc.rdcost) { + sum_rdc.rate += partition_cost[PARTITION_VERT]; + sum_rdc.rdcost = RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist); +#if CONFIG_SUPERTX + sum_rate_nocoef += partition_cost[PARTITION_VERT]; +#endif // CONFIG_SUPERTX + if (sum_rdc.rdcost < best_rdc.rdcost) { + best_rdc = sum_rdc; +#if CONFIG_SUPERTX + best_rate_nocoef = sum_rate_nocoef; + assert(best_rate_nocoef >= 0); +#endif // CONFIG_SUPERTX + pc_tree->partitioning = PARTITION_VERT; + } + } +#if !CONFIG_PVQ + restore_context(x, &x_ctx, mi_row, mi_col, bsize); +#else + restore_context(x, &x_ctx, mi_row, mi_col, &pre_rdo_buf, bsize); +#endif + } + +#if CONFIG_EXT_PARTITION_TYPES + // PARTITION_HORZ_A + if (partition_horz_allowed && do_rectangular_split && bsize > BLOCK_8X8 && + partition_none_allowed) { + subsize = get_subsize(bsize, PARTITION_HORZ_A); + rd_test_partition3(cpi, td, tile_data, tp, pc_tree, &best_rdc, + pc_tree->horizontala, ctx_none, mi_row, mi_col, bsize, + PARTITION_HORZ_A, +#if CONFIG_SUPERTX + best_rd, &best_rate_nocoef, &x_ctx, +#endif + mi_row, mi_col, bsize2, mi_row, mi_col + mi_step, bsize2, + mi_row + mi_step, mi_col, subsize); + restore_context(x, &x_ctx, mi_row, mi_col, bsize); + } + // PARTITION_HORZ_B + if (partition_horz_allowed && do_rectangular_split && bsize > BLOCK_8X8 && + partition_none_allowed) { + subsize = get_subsize(bsize, PARTITION_HORZ_B); + rd_test_partition3(cpi, td, tile_data, tp, pc_tree, &best_rdc, + pc_tree->horizontalb, ctx_none, mi_row, mi_col, bsize, + PARTITION_HORZ_B, +#if CONFIG_SUPERTX + best_rd, &best_rate_nocoef, &x_ctx, +#endif + mi_row, mi_col, subsize, mi_row + mi_step, mi_col, + bsize2, mi_row + mi_step, mi_col + mi_step, bsize2); + restore_context(x, &x_ctx, mi_row, mi_col, bsize); + } + // PARTITION_VERT_A + if (partition_vert_allowed && do_rectangular_split && bsize > BLOCK_8X8 && + partition_none_allowed) { + subsize = get_subsize(bsize, PARTITION_VERT_A); + rd_test_partition3(cpi, td, tile_data, tp, pc_tree, &best_rdc, + pc_tree->verticala, ctx_none, mi_row, mi_col, bsize, + PARTITION_VERT_A, +#if CONFIG_SUPERTX + best_rd, &best_rate_nocoef, &x_ctx, +#endif + mi_row, mi_col, bsize2, mi_row + mi_step, mi_col, bsize2, + mi_row, mi_col + mi_step, subsize); + restore_context(x, &x_ctx, mi_row, mi_col, bsize); + } + // PARTITION_VERT_B + if (partition_vert_allowed && do_rectangular_split && bsize > BLOCK_8X8 && + partition_none_allowed) { + subsize = get_subsize(bsize, PARTITION_VERT_B); + rd_test_partition3(cpi, td, tile_data, tp, pc_tree, &best_rdc, + pc_tree->verticalb, ctx_none, mi_row, mi_col, bsize, + PARTITION_VERT_B, +#if CONFIG_SUPERTX + best_rd, &best_rate_nocoef, &x_ctx, +#endif + mi_row, mi_col, subsize, mi_row, mi_col + mi_step, + bsize2, mi_row + mi_step, mi_col + mi_step, bsize2); + restore_context(x, &x_ctx, mi_row, mi_col, bsize); + } +#endif // CONFIG_EXT_PARTITION_TYPES + + // TODO(jbb): This code added so that we avoid static analysis + // warning related to the fact that best_rd isn't used after this + // point. This code should be refactored so that the duplicate + // checks occur in some sub function and thus are used... + (void)best_rd; + *rd_cost = best_rdc; +#if CONFIG_SUPERTX + *rate_nocoef = best_rate_nocoef; +#endif // CONFIG_SUPERTX + +#if CONFIG_CFL + // Store the luma for the best mode + x->cfl_store_y = 1; +#endif + if (best_rdc.rate < INT_MAX && best_rdc.dist < INT64_MAX && + pc_tree->index != 3) { + if (bsize == cm->sb_size) { +#if CONFIG_MOTION_VAR && CONFIG_NCOBMC + set_mode_info_sb(cpi, td, tile_info, tp, mi_row, mi_col, bsize, pc_tree); +#endif + encode_sb(cpi, td, tile_info, tp, mi_row, mi_col, OUTPUT_ENABLED, bsize, + pc_tree, NULL); + } else { + encode_sb(cpi, td, tile_info, tp, mi_row, mi_col, DRY_RUN_NORMAL, bsize, + pc_tree, NULL); + } + } +#if CONFIG_CFL + x->cfl_store_y = 0; +#endif + + if (bsize == cm->sb_size) { +#if !CONFIG_PVQ && !CONFIG_LV_MAP + assert(tp_orig < *tp || (tp_orig == *tp && xd->mi[0]->mbmi.skip)); +#endif + assert(best_rdc.rate < INT_MAX); + assert(best_rdc.dist < INT64_MAX); + } else { + assert(tp_orig == *tp); + } +} + +static void encode_rd_sb_row(AV1_COMP *cpi, ThreadData *td, + TileDataEnc *tile_data, int mi_row, + TOKENEXTRA **tp) { + AV1_COMMON *const cm = &cpi->common; + const TileInfo *const tile_info = &tile_data->tile_info; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + SPEED_FEATURES *const sf = &cpi->sf; + int mi_col; +#if CONFIG_EXT_PARTITION + const int leaf_nodes = 256; +#else + const int leaf_nodes = 64; +#endif // CONFIG_EXT_PARTITION + + // Initialize the left context for the new SB row + av1_zero_left_context(xd); + +#if CONFIG_DELTA_Q + // Reset delta for every tile + if (cm->delta_q_present_flag) + if (mi_row == tile_info->mi_row_start) xd->prev_qindex = cm->base_qindex; +#if CONFIG_EXT_DELTA_Q + if (cm->delta_lf_present_flag) + if (mi_row == tile_info->mi_row_start) xd->prev_delta_lf_from_base = 0; +#endif +#endif + + // Code each SB in the row + for (mi_col = tile_info->mi_col_start; mi_col < tile_info->mi_col_end; + mi_col += cm->mib_size) { + const struct segmentation *const seg = &cm->seg; + int dummy_rate; + int64_t dummy_dist; + RD_STATS dummy_rdc; +#if CONFIG_SUPERTX + int dummy_rate_nocoef; +#endif // CONFIG_SUPERTX + int i; + int seg_skip = 0; + + const int idx_str = cm->mi_stride * mi_row + mi_col; + MODE_INFO **mi = cm->mi_grid_visible + idx_str; + PC_TREE *const pc_root = td->pc_root[cm->mib_size_log2 - MIN_MIB_SIZE_LOG2]; + + av1_update_boundary_info(cm, tile_info, mi_row, mi_col); + + if (sf->adaptive_pred_interp_filter) { + for (i = 0; i < leaf_nodes; ++i) + td->leaf_tree[i].pred_interp_filter = SWITCHABLE; + + for (i = 0; i < leaf_nodes; ++i) { + td->pc_tree[i].vertical[0].pred_interp_filter = SWITCHABLE; + td->pc_tree[i].vertical[1].pred_interp_filter = SWITCHABLE; + td->pc_tree[i].horizontal[0].pred_interp_filter = SWITCHABLE; + td->pc_tree[i].horizontal[1].pred_interp_filter = SWITCHABLE; + } + } + + av1_zero(x->pred_mv); + pc_root->index = 0; + + if (seg->enabled) { + const uint8_t *const map = + seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map; + int segment_id = get_segment_id(cm, map, cm->sb_size, mi_row, mi_col); + seg_skip = segfeature_active(seg, segment_id, SEG_LVL_SKIP); + } + +#if CONFIG_DELTA_Q + if (cm->delta_q_present_flag) { + // Test mode for delta quantization + int sb_row = mi_row >> 3; + int sb_col = mi_col >> 3; + int sb_stride = (cm->width + MAX_SB_SIZE - 1) >> MAX_SB_SIZE_LOG2; + int index = ((sb_row * sb_stride + sb_col + 8) & 31) - 16; + + // Ensure divisibility of delta_qindex by delta_q_res + int offset_qindex = (index < 0 ? -index - 8 : index - 8); + int qmask = ~(cm->delta_q_res - 1); + int current_qindex = clamp(cm->base_qindex + offset_qindex, + cm->delta_q_res, 256 - cm->delta_q_res); + + current_qindex = + ((current_qindex - cm->base_qindex + cm->delta_q_res / 2) & qmask) + + cm->base_qindex; + assert(current_qindex > 0); + + xd->delta_qindex = current_qindex - cm->base_qindex; + set_offsets(cpi, tile_info, x, mi_row, mi_col, BLOCK_64X64); + xd->mi[0]->mbmi.current_q_index = current_qindex; +#if !CONFIG_EXT_DELTA_Q + xd->mi[0]->mbmi.segment_id = 0; +#endif // CONFIG_EXT_DELTA_Q + av1_init_plane_quantizers(cpi, x, xd->mi[0]->mbmi.segment_id); +#if CONFIG_EXT_DELTA_Q + if (cpi->oxcf.deltaq_mode == DELTA_Q_LF) { + int j, k; + int lfmask = ~(cm->delta_lf_res - 1); + int current_delta_lf_from_base = offset_qindex / 2; + current_delta_lf_from_base = + ((current_delta_lf_from_base + cm->delta_lf_res / 2) & lfmask); + + // pre-set the delta lf for loop filter. Note that this value is set + // before mi is assigned for each block in current superblock + for (j = 0; j < AOMMIN(cm->mib_size, cm->mi_rows - mi_row); j++) { + for (k = 0; k < AOMMIN(cm->mib_size, cm->mi_cols - mi_col); k++) { + cm->mi[(mi_row + j) * cm->mi_stride + (mi_col + k)] + .mbmi.current_delta_lf_from_base = current_delta_lf_from_base; + } + } + } +#endif // CONFIG_EXT_DELTA_Q + } +#endif // CONFIG_DELTA_Q + + x->source_variance = UINT_MAX; + if (sf->partition_search_type == FIXED_PARTITION || seg_skip) { + BLOCK_SIZE bsize; + set_offsets(cpi, tile_info, x, mi_row, mi_col, cm->sb_size); + bsize = seg_skip ? cm->sb_size : sf->always_this_block_size; + set_fixed_partitioning(cpi, tile_info, mi, mi_row, mi_col, bsize); + rd_use_partition(cpi, td, tile_data, mi, tp, mi_row, mi_col, cm->sb_size, + &dummy_rate, &dummy_dist, +#if CONFIG_SUPERTX + &dummy_rate_nocoef, +#endif // CONFIG_SUPERTX + 1, pc_root); + } else if (cpi->partition_search_skippable_frame) { + BLOCK_SIZE bsize; + set_offsets(cpi, tile_info, x, mi_row, mi_col, cm->sb_size); + bsize = get_rd_var_based_fixed_partition(cpi, x, mi_row, mi_col); + set_fixed_partitioning(cpi, tile_info, mi, mi_row, mi_col, bsize); + rd_use_partition(cpi, td, tile_data, mi, tp, mi_row, mi_col, cm->sb_size, + &dummy_rate, &dummy_dist, +#if CONFIG_SUPERTX + &dummy_rate_nocoef, +#endif // CONFIG_SUPERTX + 1, pc_root); + } else if (sf->partition_search_type == VAR_BASED_PARTITION) { + choose_partitioning(cpi, td, tile_info, x, mi_row, mi_col); + rd_use_partition(cpi, td, tile_data, mi, tp, mi_row, mi_col, cm->sb_size, + &dummy_rate, &dummy_dist, +#if CONFIG_SUPERTX + &dummy_rate_nocoef, +#endif // CONFIG_SUPERTX + 1, pc_root); + } else { + // If required set upper and lower partition size limits + if (sf->auto_min_max_partition_size) { + set_offsets(cpi, tile_info, x, mi_row, mi_col, cm->sb_size); + rd_auto_partition_range(cpi, tile_info, xd, mi_row, mi_col, + &x->min_partition_size, &x->max_partition_size); + } + rd_pick_partition(cpi, td, tile_data, tp, mi_row, mi_col, cm->sb_size, + &dummy_rdc, +#if CONFIG_SUPERTX + &dummy_rate_nocoef, +#endif // CONFIG_SUPERTX + INT64_MAX, pc_root); + } + } +#if CONFIG_SUBFRAME_PROB_UPDATE + if (cm->do_subframe_update && + cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { + const int mi_rows_per_update = + MI_SIZE * AOMMAX(cm->mi_rows / MI_SIZE / COEF_PROBS_BUFS, 1); + if ((mi_row + MI_SIZE) % mi_rows_per_update == 0 && + mi_row + MI_SIZE < cm->mi_rows && + cm->coef_probs_update_idx < COEF_PROBS_BUFS - 1) { + TX_SIZE t; + SUBFRAME_STATS *subframe_stats = &cpi->subframe_stats; + + for (t = 0; t < TX_SIZES; ++t) + av1_full_to_model_counts(cpi->td.counts->coef[t], + cpi->td.rd_counts.coef_counts[t]); + av1_partial_adapt_probs(cm, mi_row, mi_col); + ++cm->coef_probs_update_idx; + av1_copy(subframe_stats->coef_probs_buf[cm->coef_probs_update_idx], + cm->fc->coef_probs); + av1_copy(subframe_stats->coef_counts_buf[cm->coef_probs_update_idx], + cpi->td.rd_counts.coef_counts); + av1_copy(subframe_stats->eob_counts_buf[cm->coef_probs_update_idx], + cm->counts.eob_branch); + av1_fill_token_costs(x->token_costs, cm->fc->coef_probs); + } + } +#endif // CONFIG_SUBFRAME_PROB_UPDATE +} + +static void init_encode_frame_mb_context(AV1_COMP *cpi) { + MACROBLOCK *const x = &cpi->td.mb; + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + + // Copy data over into macro block data structures. + av1_setup_src_planes(x, cpi->source, 0, 0); + + av1_setup_block_planes(xd, cm->subsampling_x, cm->subsampling_y); +} + +#if !CONFIG_REF_ADAPT +static int check_dual_ref_flags(AV1_COMP *cpi) { + const int ref_flags = cpi->ref_frame_flags; + + if (segfeature_active(&cpi->common.seg, 1, SEG_LVL_REF_FRAME)) { + return 0; + } else { + return (!!(ref_flags & AOM_GOLD_FLAG) + !!(ref_flags & AOM_LAST_FLAG) + +#if CONFIG_EXT_REFS + !!(ref_flags & AOM_LAST2_FLAG) + !!(ref_flags & AOM_LAST3_FLAG) + + !!(ref_flags & AOM_BWD_FLAG) + +#endif // CONFIG_EXT_REFS + !!(ref_flags & AOM_ALT_FLAG)) >= 2; + } +} +#endif // !CONFIG_REF_ADAPT + +#if !CONFIG_VAR_TX +static void reset_skip_tx_size(AV1_COMMON *cm, TX_SIZE max_tx_size) { + int mi_row, mi_col; + const int mis = cm->mi_stride; + MODE_INFO **mi_ptr = cm->mi_grid_visible; + + for (mi_row = 0; mi_row < cm->mi_rows; ++mi_row, mi_ptr += mis) { + for (mi_col = 0; mi_col < cm->mi_cols; ++mi_col) { + if (txsize_sqr_up_map[mi_ptr[mi_col]->mbmi.tx_size] > max_tx_size) + mi_ptr[mi_col]->mbmi.tx_size = max_tx_size; + } + } +} +#endif + +static MV_REFERENCE_FRAME get_frame_type(const AV1_COMP *cpi) { + if (frame_is_intra_only(&cpi->common)) return INTRA_FRAME; +#if CONFIG_EXT_REFS + // We will not update the golden frame with an internal overlay frame + else if ((cpi->rc.is_src_frame_alt_ref && cpi->refresh_golden_frame) || + cpi->rc.is_src_frame_ext_arf) +#else + else if (cpi->rc.is_src_frame_alt_ref && cpi->refresh_golden_frame) +#endif + return ALTREF_FRAME; + else if (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame) + return GOLDEN_FRAME; + else + // TODO(zoeliu): To investigate whether a frame_type other than + // INTRA/ALTREF/GOLDEN/LAST needs to be specified seperately. + return LAST_FRAME; +} + +static TX_MODE select_tx_mode(const AV1_COMP *cpi, MACROBLOCKD *const xd) { + int i, all_lossless = 1; + + if (cpi->common.seg.enabled) { + for (i = 0; i < MAX_SEGMENTS; ++i) { + if (!xd->lossless[i]) { + all_lossless = 0; + break; + } + } + } else { + all_lossless = xd->lossless[0]; + } + if (all_lossless) return ONLY_4X4; + if (cpi->sf.tx_size_search_method == USE_LARGESTALL) + return ALLOW_32X32 + CONFIG_TX64X64; + else if (cpi->sf.tx_size_search_method == USE_FULL_RD || + cpi->sf.tx_size_search_method == USE_TX_8X8) + return TX_MODE_SELECT; + else + return cpi->common.tx_mode; +} + +void av1_init_tile_data(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + const int tile_cols = cm->tile_cols; + const int tile_rows = cm->tile_rows; + int tile_col, tile_row; + TOKENEXTRA *pre_tok = cpi->tile_tok[0][0]; + unsigned int tile_tok = 0; + + if (cpi->tile_data == NULL || cpi->allocated_tiles < tile_cols * tile_rows) { + if (cpi->tile_data != NULL) aom_free(cpi->tile_data); + CHECK_MEM_ERROR( + cm, cpi->tile_data, + aom_memalign(32, tile_cols * tile_rows * sizeof(*cpi->tile_data))); + cpi->allocated_tiles = tile_cols * tile_rows; + + for (tile_row = 0; tile_row < tile_rows; ++tile_row) + for (tile_col = 0; tile_col < tile_cols; ++tile_col) { + TileDataEnc *const tile_data = + &cpi->tile_data[tile_row * tile_cols + tile_col]; + int i, j; + for (i = 0; i < BLOCK_SIZES; ++i) { + for (j = 0; j < MAX_MODES; ++j) { + tile_data->thresh_freq_fact[i][j] = 32; + tile_data->mode_map[i][j] = j; + } + } +#if CONFIG_PVQ + // This will be dynamically increased as more pvq block is encoded. + tile_data->pvq_q.buf_len = 1000; + CHECK_MEM_ERROR( + cm, tile_data->pvq_q.buf, + aom_malloc(tile_data->pvq_q.buf_len * sizeof(PVQ_INFO))); + tile_data->pvq_q.curr_pos = 0; +#endif + } + } + + for (tile_row = 0; tile_row < tile_rows; ++tile_row) { + for (tile_col = 0; tile_col < tile_cols; ++tile_col) { + TileInfo *const tile_info = + &cpi->tile_data[tile_row * tile_cols + tile_col].tile_info; + av1_tile_init(tile_info, cm, tile_row, tile_col); + + cpi->tile_tok[tile_row][tile_col] = pre_tok + tile_tok; + pre_tok = cpi->tile_tok[tile_row][tile_col]; + tile_tok = allocated_tokens(*tile_info); +#if CONFIG_PVQ + cpi->tile_data[tile_row * tile_cols + tile_col].pvq_q.curr_pos = 0; +#endif + } + } +} + +void av1_encode_tile(AV1_COMP *cpi, ThreadData *td, int tile_row, + int tile_col) { + AV1_COMMON *const cm = &cpi->common; + TileDataEnc *const this_tile = + &cpi->tile_data[tile_row * cm->tile_cols + tile_col]; + const TileInfo *const tile_info = &this_tile->tile_info; + TOKENEXTRA *tok = cpi->tile_tok[tile_row][tile_col]; + int mi_row; + +#if CONFIG_DEPENDENT_HORZTILES +#if CONFIG_TILE_GROUPS + if ((!cm->dependent_horz_tiles) || (tile_row == 0) || + tile_info->tg_horz_boundary) { +#else + if ((!cm->dependent_horz_tiles) || (tile_row == 0)) { +#endif + av1_zero_above_context(cm, tile_info->mi_col_start, tile_info->mi_col_end); + } +#else + av1_zero_above_context(cm, tile_info->mi_col_start, tile_info->mi_col_end); +#endif + + // Set up pointers to per thread motion search counters. + this_tile->m_search_count = 0; // Count of motion search hits. + this_tile->ex_search_count = 0; // Exhaustive mesh search hits. + td->mb.m_search_count_ptr = &this_tile->m_search_count; + td->mb.ex_search_count_ptr = &this_tile->ex_search_count; + +#if CONFIG_PVQ + td->mb.pvq_q = &this_tile->pvq_q; + + // TODO(yushin) : activity masking info needs be signaled by a bitstream + td->mb.daala_enc.use_activity_masking = AV1_PVQ_ENABLE_ACTIVITY_MASKING; + + if (td->mb.daala_enc.use_activity_masking) + td->mb.daala_enc.qm = OD_HVS_QM; // Hard coded. Enc/dec required to sync. + else + td->mb.daala_enc.qm = OD_FLAT_QM; // Hard coded. Enc/dec required to sync. + + { + // FIXME: Multiple segments support + int segment_id = 0; + int rdmult = set_segment_rdmult(cpi, &td->mb, segment_id); + int qindex = av1_get_qindex(&cm->seg, segment_id, cm->base_qindex); +#if CONFIG_HIGHBITDEPTH + const int quantizer_shift = td->mb.e_mbd.bd - 8; +#else + const int quantizer_shift = 0; +#endif // CONFIG_HIGHBITDEPTH + int64_t q_ac = OD_MAXI( + 1, av1_ac_quant(qindex, 0, cpi->common.bit_depth) >> quantizer_shift); + int64_t q_dc = OD_MAXI( + 1, av1_dc_quant(qindex, 0, cpi->common.bit_depth) >> quantizer_shift); + /* td->mb.daala_enc.pvq_norm_lambda = OD_PVQ_LAMBDA; */ + td->mb.daala_enc.pvq_norm_lambda = + (double)rdmult * (64 / 16) / (q_ac * q_ac * (1 << RDDIV_BITS)); + td->mb.daala_enc.pvq_norm_lambda_dc = + (double)rdmult * (64 / 16) / (q_dc * q_dc * (1 << RDDIV_BITS)); + // printf("%f\n", td->mb.daala_enc.pvq_norm_lambda); + } + od_init_qm(td->mb.daala_enc.state.qm, td->mb.daala_enc.state.qm_inv, + td->mb.daala_enc.qm == OD_HVS_QM ? OD_QM8_Q4_HVS : OD_QM8_Q4_FLAT); + + if (td->mb.daala_enc.use_activity_masking) { + int pli; + int use_masking = td->mb.daala_enc.use_activity_masking; + int segment_id = 0; + int qindex = av1_get_qindex(&cm->seg, segment_id, cm->base_qindex); + + for (pli = 0; pli < MAX_MB_PLANE; pli++) { + int i; + int q; + + q = qindex; + if (q <= OD_DEFAULT_QMS[use_masking][0][pli].interp_q << OD_COEFF_SHIFT) { + od_interp_qm(&td->mb.daala_enc.state.pvq_qm_q4[pli][0], q, + &OD_DEFAULT_QMS[use_masking][0][pli], NULL); + } else { + i = 0; + while (OD_DEFAULT_QMS[use_masking][i + 1][pli].qm_q4 != NULL && + q > OD_DEFAULT_QMS[use_masking][i + 1][pli].interp_q + << OD_COEFF_SHIFT) { + i++; + } + od_interp_qm(&td->mb.daala_enc.state.pvq_qm_q4[pli][0], q, + &OD_DEFAULT_QMS[use_masking][i][pli], + &OD_DEFAULT_QMS[use_masking][i + 1][pli]); + } + } + } + +#if CONFIG_DAALA_EC + od_ec_enc_init(&td->mb.daala_enc.w.ec, 65025); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + +#if CONFIG_DAALA_EC + od_ec_enc_reset(&td->mb.daala_enc.w.ec); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif +#endif // #if CONFIG_PVQ + +#if CONFIG_EC_ADAPT + this_tile->tctx = *cm->fc; + td->mb.e_mbd.tile_ctx = &this_tile->tctx; +#endif // #if CONFIG_EC_ADAPT + +#if CONFIG_CFL + MACROBLOCKD *const xd = &td->mb.e_mbd; + xd->cfl = &this_tile->cfl; + cfl_init(xd->cfl, cm, xd->plane[AOM_PLANE_U].subsampling_x, + xd->plane[AOM_PLANE_U].subsampling_y); +#endif + +#if CONFIG_PVQ + td->mb.daala_enc.state.adapt = &this_tile->tctx.pvq_context; +#endif // CONFIG_PVQ + + for (mi_row = tile_info->mi_row_start; mi_row < tile_info->mi_row_end; + mi_row += cm->mib_size) { + encode_rd_sb_row(cpi, td, this_tile, mi_row, &tok); + } + + cpi->tok_count[tile_row][tile_col] = + (unsigned int)(tok - cpi->tile_tok[tile_row][tile_col]); + assert(cpi->tok_count[tile_row][tile_col] <= allocated_tokens(*tile_info)); +#if CONFIG_PVQ +#if CONFIG_DAALA_EC + od_ec_enc_clear(&td->mb.daala_enc.w.ec); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + + td->mb.pvq_q->last_pos = td->mb.pvq_q->curr_pos; + // rewind current position so that bitstream can be written + // from the 1st pvq block + td->mb.pvq_q->curr_pos = 0; + + td->mb.pvq_q = NULL; +#endif +} + +static void encode_tiles(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + int tile_col, tile_row; + + av1_init_tile_data(cpi); + + for (tile_row = 0; tile_row < cm->tile_rows; ++tile_row) + for (tile_col = 0; tile_col < cm->tile_cols; ++tile_col) + av1_encode_tile(cpi, &cpi->td, tile_row, tile_col); +} + +#if CONFIG_FP_MB_STATS +static int input_fpmb_stats(FIRSTPASS_MB_STATS *firstpass_mb_stats, + AV1_COMMON *cm, uint8_t **this_frame_mb_stats) { + uint8_t *mb_stats_in = firstpass_mb_stats->mb_stats_start + + cm->current_video_frame * cm->MBs * sizeof(uint8_t); + + if (mb_stats_in > firstpass_mb_stats->mb_stats_end) return EOF; + + *this_frame_mb_stats = mb_stats_in; + + return 1; +} +#endif + +#if CONFIG_GLOBAL_MOTION +#define GLOBAL_TRANS_TYPES_ENC 3 // highest motion model to search +static int gm_get_params_cost(WarpedMotionParams *gm, + WarpedMotionParams *ref_gm, int allow_hp) { + assert(gm->wmtype < GLOBAL_TRANS_TYPES); + int params_cost = 0; + int trans_bits, trans_prec_diff; + switch (gm->wmtype) { + case HOMOGRAPHY: + case HORTRAPEZOID: + case VERTRAPEZOID: + if (gm->wmtype != HORTRAPEZOID) + params_cost += aom_count_signed_primitive_refsubexpfin( + GM_ROW3HOMO_MAX + 1, SUBEXPFIN_K, + (ref_gm->wmmat[6] >> GM_ROW3HOMO_PREC_DIFF), + (gm->wmmat[6] >> GM_ROW3HOMO_PREC_DIFF)); + if (gm->wmtype != VERTRAPEZOID) + params_cost += aom_count_signed_primitive_refsubexpfin( + GM_ROW3HOMO_MAX + 1, SUBEXPFIN_K, + (ref_gm->wmmat[7] >> GM_ROW3HOMO_PREC_DIFF), + (gm->wmmat[7] >> GM_ROW3HOMO_PREC_DIFF)); + // Fallthrough intended + case AFFINE: + case ROTZOOM: + params_cost += aom_count_signed_primitive_refsubexpfin( + GM_ALPHA_MAX + 1, SUBEXPFIN_K, + (ref_gm->wmmat[2] >> GM_ALPHA_PREC_DIFF) - (1 << GM_ALPHA_PREC_BITS), + (gm->wmmat[2] >> GM_ALPHA_PREC_DIFF) - (1 << GM_ALPHA_PREC_BITS)); + if (gm->wmtype != VERTRAPEZOID) + params_cost += aom_count_signed_primitive_refsubexpfin( + GM_ALPHA_MAX + 1, SUBEXPFIN_K, + (ref_gm->wmmat[3] >> GM_ALPHA_PREC_DIFF), + (gm->wmmat[3] >> GM_ALPHA_PREC_DIFF)); + if (gm->wmtype >= AFFINE) { + if (gm->wmtype != HORTRAPEZOID) + params_cost += aom_count_signed_primitive_refsubexpfin( + GM_ALPHA_MAX + 1, SUBEXPFIN_K, + (ref_gm->wmmat[4] >> GM_ALPHA_PREC_DIFF), + (gm->wmmat[4] >> GM_ALPHA_PREC_DIFF)); + params_cost += aom_count_signed_primitive_refsubexpfin( + GM_ALPHA_MAX + 1, SUBEXPFIN_K, + (ref_gm->wmmat[5] >> GM_ALPHA_PREC_DIFF) - + (1 << GM_ALPHA_PREC_BITS), + (gm->wmmat[5] >> GM_ALPHA_PREC_DIFF) - (1 << GM_ALPHA_PREC_BITS)); + } + // Fallthrough intended + case TRANSLATION: + trans_bits = (gm->wmtype == TRANSLATION) + ? GM_ABS_TRANS_ONLY_BITS - !allow_hp + : GM_ABS_TRANS_BITS; + trans_prec_diff = (gm->wmtype == TRANSLATION) + ? GM_TRANS_ONLY_PREC_DIFF + !allow_hp + : GM_TRANS_PREC_DIFF; + params_cost += aom_count_signed_primitive_refsubexpfin( + (1 << trans_bits) + 1, SUBEXPFIN_K, + (ref_gm->wmmat[0] >> trans_prec_diff), + (gm->wmmat[0] >> trans_prec_diff)); + params_cost += aom_count_signed_primitive_refsubexpfin( + (1 << trans_bits) + 1, SUBEXPFIN_K, + (ref_gm->wmmat[1] >> trans_prec_diff), + (gm->wmmat[1] >> trans_prec_diff)); + // Fallthrough intended + case IDENTITY: break; + default: assert(0); + } + return (params_cost << AV1_PROB_COST_SHIFT); +} +#endif // CONFIG_GLOBAL_MOTION + +static void encode_frame_internal(AV1_COMP *cpi) { + ThreadData *const td = &cpi->td; + MACROBLOCK *const x = &td->mb; + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + RD_COUNTS *const rdc = &cpi->td.rd_counts; + int i; +#if CONFIG_TEMPMV_SIGNALING || CONFIG_EXT_REFS + const int last_fb_buf_idx = get_ref_frame_buf_idx(cpi, LAST_FRAME); +#endif // CONFIG_TEMPMV_SIGNALING || CONFIG_EXT_REFS + +#if CONFIG_ADAPT_SCAN + av1_deliver_eob_threshold(cm, xd); +#endif + + x->min_partition_size = AOMMIN(x->min_partition_size, cm->sb_size); + x->max_partition_size = AOMMIN(x->max_partition_size, cm->sb_size); +#if CONFIG_REF_MV + cm->setup_mi(cm); +#endif + + xd->mi = cm->mi_grid_visible; + xd->mi[0] = cm->mi; + + av1_zero(*td->counts); + av1_zero(rdc->coef_counts); + av1_zero(rdc->comp_pred_diff); + +#if CONFIG_GLOBAL_MOTION + av1_zero(rdc->global_motion_used); + if (cpi->common.frame_type == INTER_FRAME && cpi->source && + !cpi->global_motion_search_done) { + YV12_BUFFER_CONFIG *ref_buf; + int frame; + double params_by_motion[RANSAC_NUM_MOTIONS * (MAX_PARAMDIM - 1)]; + const double *params_this_motion; + int inliers_by_motion[RANSAC_NUM_MOTIONS]; + WarpedMotionParams tmp_wm_params; + static const double kInfiniteErrAdv = 1e12; + static const double kIdentityParams[MAX_PARAMDIM - 1] = { + 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0 + }; + + for (frame = LAST_FRAME; frame <= ALTREF_FRAME; ++frame) { + ref_buf = get_ref_frame_buffer(cpi, frame); + if (ref_buf) { + TransformationType model; + aom_clear_system_state(); + for (model = ROTZOOM; model < GLOBAL_TRANS_TYPES_ENC; ++model) { + double best_erroradvantage = kInfiniteErrAdv; + + // Initially set all params to identity. + for (i = 0; i < RANSAC_NUM_MOTIONS; ++i) { + memcpy(params_by_motion + (MAX_PARAMDIM - 1) * i, kIdentityParams, + (MAX_PARAMDIM - 1) * sizeof(*params_by_motion)); + } + + compute_global_motion_feature_based( + model, cpi->source, ref_buf, +#if CONFIG_HIGHBITDEPTH + cpi->common.bit_depth, +#endif // CONFIG_HIGHBITDEPTH + inliers_by_motion, params_by_motion, RANSAC_NUM_MOTIONS); + + for (i = 0; i < RANSAC_NUM_MOTIONS; ++i) { + if (inliers_by_motion[i] == 0) continue; + + params_this_motion = params_by_motion + (MAX_PARAMDIM - 1) * i; + convert_model_to_params(params_this_motion, &tmp_wm_params); + + if (tmp_wm_params.wmtype != IDENTITY) { + const double erroradv_this_motion = refine_integerized_param( + &tmp_wm_params, tmp_wm_params.wmtype, +#if CONFIG_HIGHBITDEPTH + xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH, xd->bd, +#endif // CONFIG_HIGHBITDEPTH + ref_buf->y_buffer, ref_buf->y_width, ref_buf->y_height, + ref_buf->y_stride, cpi->source->y_buffer, + cpi->source->y_width, cpi->source->y_height, + cpi->source->y_stride, 3); + if (erroradv_this_motion < best_erroradvantage) { + best_erroradvantage = erroradv_this_motion; + // Save the wm_params modified by refine_integerized_param() + // rather than motion index to avoid rerunning refine() below. + memcpy(&(cm->global_motion[frame]), &tmp_wm_params, + sizeof(WarpedMotionParams)); + } + } + } + if (cm->global_motion[frame].wmtype <= AFFINE) + if (!get_shear_params(&cm->global_motion[frame])) + set_default_warp_params(&cm->global_motion[frame]); + + if (cm->global_motion[frame].wmtype == TRANSLATION) { + cm->global_motion[frame].wmmat[0] = + convert_to_trans_prec(cm->allow_high_precision_mv, + cm->global_motion[frame].wmmat[0]) * + GM_TRANS_ONLY_DECODE_FACTOR; + cm->global_motion[frame].wmmat[1] = + convert_to_trans_prec(cm->allow_high_precision_mv, + cm->global_motion[frame].wmmat[1]) * + GM_TRANS_ONLY_DECODE_FACTOR; + } + + // If the best error advantage found doesn't meet the threshold for + // this motion type, revert to IDENTITY. + if (!is_enough_erroradvantage( + best_erroradvantage, + gm_get_params_cost(&cm->global_motion[frame], + &cm->prev_frame->global_motion[frame], + cm->allow_high_precision_mv))) { + set_default_warp_params(&cm->global_motion[frame]); + } + + if (cm->global_motion[frame].wmtype != IDENTITY) break; + } + aom_clear_system_state(); + } + cpi->gmparams_cost[frame] = + gm_get_params_cost(&cm->global_motion[frame], + &cm->prev_frame->global_motion[frame], + cm->allow_high_precision_mv) + + cpi->gmtype_cost[cm->global_motion[frame].wmtype] - + cpi->gmtype_cost[IDENTITY]; + } + cpi->global_motion_search_done = 1; + } + memcpy(cm->cur_frame->global_motion, cm->global_motion, + TOTAL_REFS_PER_FRAME * sizeof(WarpedMotionParams)); +#endif // CONFIG_GLOBAL_MOTION + + for (i = 0; i < MAX_SEGMENTS; ++i) { + const int qindex = cm->seg.enabled + ? av1_get_qindex(&cm->seg, i, cm->base_qindex) + : cm->base_qindex; + xd->lossless[i] = qindex == 0 && cm->y_dc_delta_q == 0 && + cm->uv_dc_delta_q == 0 && cm->uv_ac_delta_q == 0; + xd->qindex[i] = qindex; + } + + if (!cm->seg.enabled && xd->lossless[0]) x->optimize = 0; + + cm->tx_mode = select_tx_mode(cpi, xd); + +#if CONFIG_DELTA_Q + // Fix delta q resolution for the moment + cm->delta_q_res = DEFAULT_DELTA_Q_RES; +// Set delta_q_present_flag before it is used for the first time +#if CONFIG_EXT_DELTA_Q + cm->delta_lf_res = DEFAULT_DELTA_LF_RES; + // update delta_q_present_flag and delta_lf_present_flag based on base_qindex + cm->delta_q_present_flag &= cm->base_qindex > 0; + cm->delta_lf_present_flag &= cm->base_qindex > 0; +#else + cm->delta_q_present_flag = + cpi->oxcf.aq_mode == DELTA_AQ && cm->base_qindex > 0; +#endif // CONFIG_EXT_DELTA_Q +#endif + + av1_frame_init_quantizer(cpi); + + av1_initialize_rd_consts(cpi); + av1_initialize_me_consts(cpi, x, cm->base_qindex); + init_encode_frame_mb_context(cpi); +#if CONFIG_TEMPMV_SIGNALING + if (last_fb_buf_idx != INVALID_IDX) { + cm->prev_frame = &cm->buffer_pool->frame_bufs[last_fb_buf_idx]; + cm->use_prev_frame_mvs &= !cm->error_resilient_mode && + cm->width == cm->prev_frame->buf.y_width && + cm->height == cm->prev_frame->buf.y_height && + !cm->intra_only && !cm->prev_frame->intra_only; + } +#else + cm->use_prev_frame_mvs = + !cm->error_resilient_mode && cm->width == cm->last_width && + cm->height == cm->last_height && !cm->intra_only && cm->last_show_frame; +#endif + +#if CONFIG_EXT_REFS + // NOTE(zoeliu): As cm->prev_frame can take neither a frame of + // show_exisiting_frame=1, nor can it take a frame not used as + // a reference, it is probable that by the time it is being + // referred to, the frame buffer it originally points to may + // already get expired and have been reassigned to the current + // newly coded frame. Hence, we need to check whether this is + // the case, and if yes, we have 2 choices: + // (1) Simply disable the use of previous frame mvs; or + // (2) Have cm->prev_frame point to one reference frame buffer, + // e.g. LAST_FRAME. + if (cm->use_prev_frame_mvs && !enc_is_ref_frame_buf(cpi, cm->prev_frame)) { + // Reassign the LAST_FRAME buffer to cm->prev_frame. + cm->prev_frame = &cm->buffer_pool->frame_bufs[last_fb_buf_idx]; + } +#endif // CONFIG_EXT_REFS + + // Special case: set prev_mi to NULL when the previous mode info + // context cannot be used. + cm->prev_mi = + cm->use_prev_frame_mvs ? cm->prev_mip + cm->mi_stride + 1 : NULL; + +#if CONFIG_VAR_TX + x->txb_split_count = 0; +#if CONFIG_REF_MV + av1_zero(x->blk_skip_drl); +#endif +#endif + + if (cpi->sf.partition_search_type == VAR_BASED_PARTITION && + cpi->td.var_root[0] == NULL) + av1_setup_var_tree(&cpi->common, &cpi->td); + + { + struct aom_usec_timer emr_timer; + aom_usec_timer_start(&emr_timer); + +#if CONFIG_FP_MB_STATS + if (cpi->use_fp_mb_stats) { + input_fpmb_stats(&cpi->twopass.firstpass_mb_stats, cm, + &cpi->twopass.this_frame_mb_stats); + } +#endif + + // If allowed, encoding tiles in parallel with one thread handling one tile. + // TODO(geza.lore): The multi-threaded encoder is not safe with more than + // 1 tile rows, as it uses the single above_context et al arrays from + // cpi->common + if (AOMMIN(cpi->oxcf.max_threads, cm->tile_cols) > 1 && cm->tile_rows == 1) + av1_encode_tiles_mt(cpi); + else + encode_tiles(cpi); + + aom_usec_timer_mark(&emr_timer); + cpi->time_encode_sb_row += aom_usec_timer_elapsed(&emr_timer); + } + +#if 0 + // Keep record of the total distortion this time around for future use + cpi->last_frame_distortion = cpi->frame_distortion; +#endif +} + +void av1_encode_frame(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; +#if CONFIG_EXT_TX + // Indicates whether or not to use a default reduced set for ext-tx + // rather than the potential full set of 16 transforms + cm->reduced_tx_set_used = 0; +#endif // CONFIG_EXT_TX + + // In the longer term the encoder should be generalized to match the + // decoder such that we allow compound where one of the 3 buffers has a + // different sign bias and that buffer is then the fixed ref. However, this + // requires further work in the rd loop. For now the only supported encoder + // side behavior is where the ALT ref buffer has opposite sign bias to + // the other two. + if (!frame_is_intra_only(cm)) { +#if CONFIG_LOWDELAY_COMPOUND // Normative in encoder + cpi->allow_comp_inter_inter = 1; +#if CONFIG_EXT_REFS + cm->comp_fwd_ref[0] = LAST_FRAME; + cm->comp_fwd_ref[1] = LAST2_FRAME; + cm->comp_fwd_ref[2] = LAST3_FRAME; + cm->comp_fwd_ref[3] = GOLDEN_FRAME; + cm->comp_bwd_ref[0] = BWDREF_FRAME; + cm->comp_bwd_ref[1] = ALTREF_FRAME; +#else + cm->comp_fixed_ref = ALTREF_FRAME; + cm->comp_var_ref[0] = LAST_FRAME; + cm->comp_var_ref[1] = GOLDEN_FRAME; +#endif // CONFIG_EXT_REFS +#else + if ((cm->ref_frame_sign_bias[ALTREF_FRAME] == + cm->ref_frame_sign_bias[GOLDEN_FRAME]) || + (cm->ref_frame_sign_bias[ALTREF_FRAME] == + cm->ref_frame_sign_bias[LAST_FRAME])) { + cpi->allow_comp_inter_inter = 0; + } else { + cpi->allow_comp_inter_inter = 1; + +#if CONFIG_EXT_REFS + cm->comp_fwd_ref[0] = LAST_FRAME; + cm->comp_fwd_ref[1] = LAST2_FRAME; + cm->comp_fwd_ref[2] = LAST3_FRAME; + cm->comp_fwd_ref[3] = GOLDEN_FRAME; + cm->comp_bwd_ref[0] = BWDREF_FRAME; + cm->comp_bwd_ref[1] = ALTREF_FRAME; +#else + cm->comp_fixed_ref = ALTREF_FRAME; + cm->comp_var_ref[0] = LAST_FRAME; + cm->comp_var_ref[1] = GOLDEN_FRAME; +#endif // CONFIG_EXT_REFS + } +#endif + } else { + cpi->allow_comp_inter_inter = 0; + } + + if (cpi->sf.frame_parameter_update) { + int i; + RD_OPT *const rd_opt = &cpi->rd; + FRAME_COUNTS *counts = cpi->td.counts; + RD_COUNTS *const rdc = &cpi->td.rd_counts; + + // This code does a single RD pass over the whole frame assuming + // either compound, single or hybrid prediction as per whatever has + // worked best for that type of frame in the past. + // It also predicts whether another coding mode would have worked + // better than this coding mode. If that is the case, it remembers + // that for subsequent frames. + // It does the same analysis for transform size selection also. + // + // TODO(zoeliu): To investigate whether a frame_type other than + // INTRA/ALTREF/GOLDEN/LAST needs to be specified seperately. + const MV_REFERENCE_FRAME frame_type = get_frame_type(cpi); + int64_t *const mode_thrs = rd_opt->prediction_type_threshes[frame_type]; + const int is_alt_ref = frame_type == ALTREF_FRAME; + +/* prediction (compound, single or hybrid) mode selection */ +#if CONFIG_REF_ADAPT + // NOTE(zoeliu): "is_alt_ref" is true only for OVERLAY/INTNL_OVERLAY frames + if (is_alt_ref || !cpi->allow_comp_inter_inter) + cm->reference_mode = SINGLE_REFERENCE; + else + cm->reference_mode = REFERENCE_MODE_SELECT; +#else + if (is_alt_ref || !cpi->allow_comp_inter_inter) + cm->reference_mode = SINGLE_REFERENCE; + else if (mode_thrs[COMPOUND_REFERENCE] > mode_thrs[SINGLE_REFERENCE] && + mode_thrs[COMPOUND_REFERENCE] > mode_thrs[REFERENCE_MODE_SELECT] && + check_dual_ref_flags(cpi) && cpi->static_mb_pct == 100) + cm->reference_mode = COMPOUND_REFERENCE; + else if (mode_thrs[SINGLE_REFERENCE] > mode_thrs[REFERENCE_MODE_SELECT]) + cm->reference_mode = SINGLE_REFERENCE; + else + cm->reference_mode = REFERENCE_MODE_SELECT; +#endif // CONFIG_REF_ADAPT + +#if CONFIG_DUAL_FILTER + cm->interp_filter = SWITCHABLE; +#endif + + encode_frame_internal(cpi); + + for (i = 0; i < REFERENCE_MODES; ++i) + mode_thrs[i] = (mode_thrs[i] + rdc->comp_pred_diff[i] / cm->MBs) / 2; + + if (cm->reference_mode == REFERENCE_MODE_SELECT) { + int single_count_zero = 0; + int comp_count_zero = 0; + + for (i = 0; i < COMP_INTER_CONTEXTS; i++) { + single_count_zero += counts->comp_inter[i][0]; + comp_count_zero += counts->comp_inter[i][1]; + } + + if (comp_count_zero == 0) { + cm->reference_mode = SINGLE_REFERENCE; + av1_zero(counts->comp_inter); +#if !CONFIG_REF_ADAPT + } else if (single_count_zero == 0) { + cm->reference_mode = COMPOUND_REFERENCE; + av1_zero(counts->comp_inter); +#endif // !CONFIG_REF_ADAPT + } + } + +#if CONFIG_VAR_TX + if (cm->tx_mode == TX_MODE_SELECT && cpi->td.mb.txb_split_count == 0) + cm->tx_mode = ALLOW_32X32 + CONFIG_TX64X64; +#else + if (cm->tx_mode == TX_MODE_SELECT) { +#if CONFIG_TX64X64 + int count4x4 = 0; + int count8x8_8x8p = 0, count8x8_lp = 0; + int count16x16_16x16p = 0, count16x16_lp = 0; + int count32x32_32x32p = 0, count32x32_lp = 0; + int count64x64_64x64p = 0; + for (i = 0; i < TX_SIZE_CONTEXTS; ++i) { + // counts->tx_size[max_depth][context_idx][this_depth_level] + count4x4 += counts->tx_size[0][i][0]; + count4x4 += counts->tx_size[1][i][0]; + count4x4 += counts->tx_size[2][i][0]; + count4x4 += counts->tx_size[3][i][0]; + + count8x8_8x8p += counts->tx_size[0][i][1]; + count8x8_lp += counts->tx_size[1][i][1]; + count8x8_lp += counts->tx_size[2][i][1]; + count8x8_lp += counts->tx_size[3][i][1]; + + count16x16_16x16p += counts->tx_size[1][i][2]; + count16x16_lp += counts->tx_size[2][i][2]; + count16x16_lp += counts->tx_size[3][i][2]; + + count32x32_32x32p += counts->tx_size[2][i][3]; + count32x32_lp += counts->tx_size[3][i][3]; + + count64x64_64x64p += counts->tx_size[3][i][4]; + } +#if CONFIG_EXT_TX && CONFIG_RECT_TX + count4x4 += counts->tx_size_implied[0][TX_4X4]; + count4x4 += counts->tx_size_implied[1][TX_4X4]; + count4x4 += counts->tx_size_implied[2][TX_4X4]; + count4x4 += counts->tx_size_implied[3][TX_4X4]; + count8x8_8x8p += counts->tx_size_implied[1][TX_8X8]; + count8x8_lp += counts->tx_size_implied[2][TX_8X8]; + count8x8_lp += counts->tx_size_implied[3][TX_8X8]; + count8x8_lp += counts->tx_size_implied[4][TX_8X8]; + count16x16_16x16p += counts->tx_size_implied[2][TX_16X16]; + count16x16_lp += counts->tx_size_implied[3][TX_16X16]; + count16x16_lp += counts->tx_size_implied[4][TX_16X16]; + count32x32_32x32p += counts->tx_size_implied[3][TX_32X32]; + count32x32_lp += counts->tx_size_implied[4][TX_32X32]; + count64x64_64x64p += counts->tx_size_implied[4][TX_64X64]; +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + if (count4x4 == 0 && count16x16_lp == 0 && count16x16_16x16p == 0 && + count32x32_lp == 0 && count32x32_32x32p == 0 && +#if CONFIG_SUPERTX + cm->counts.supertx_size[TX_16X16] == 0 && + cm->counts.supertx_size[TX_32X32] == 0 && + cm->counts.supertx_size[TX_64X64] == 0 && +#endif + count64x64_64x64p == 0) { + cm->tx_mode = ALLOW_8X8; + reset_skip_tx_size(cm, TX_8X8); + } else if (count8x8_8x8p == 0 && count8x8_lp == 0 && + count16x16_16x16p == 0 && count16x16_lp == 0 && + count32x32_32x32p == 0 && count32x32_lp == 0 && +#if CONFIG_SUPERTX + cm->counts.supertx_size[TX_8X8] == 0 && + cm->counts.supertx_size[TX_16X16] == 0 && + cm->counts.supertx_size[TX_32X32] == 0 && + cm->counts.supertx_size[TX_64X64] == 0 && +#endif + count64x64_64x64p == 0) { + cm->tx_mode = ONLY_4X4; + reset_skip_tx_size(cm, TX_4X4); + } else if (count4x4 == 0 && count8x8_lp == 0 && count16x16_lp == 0 && + count32x32_lp == 0) { + cm->tx_mode = ALLOW_64X64; + } else if (count4x4 == 0 && count8x8_lp == 0 && count16x16_lp == 0 && +#if CONFIG_SUPERTX + cm->counts.supertx_size[TX_64X64] == 0 && +#endif + count64x64_64x64p == 0) { + cm->tx_mode = ALLOW_32X32; + reset_skip_tx_size(cm, TX_32X32); + } else if (count4x4 == 0 && count8x8_lp == 0 && count32x32_lp == 0 && + count32x32_32x32p == 0 && +#if CONFIG_SUPERTX + cm->counts.supertx_size[TX_32X32] == 0 && + cm->counts.supertx_size[TX_64X64] == 0 && +#endif + count64x64_64x64p == 0) { + cm->tx_mode = ALLOW_16X16; + reset_skip_tx_size(cm, TX_16X16); + } + +#else // CONFIG_TX64X64 + + int count4x4 = 0; + int count8x8_lp = 0, count8x8_8x8p = 0; + int count16x16_16x16p = 0, count16x16_lp = 0; + int count32x32 = 0; + for (i = 0; i < TX_SIZE_CONTEXTS; ++i) { + // counts->tx_size[max_depth][context_idx][this_depth_level] + count4x4 += counts->tx_size[0][i][0]; + count4x4 += counts->tx_size[1][i][0]; + count4x4 += counts->tx_size[2][i][0]; + + count8x8_8x8p += counts->tx_size[0][i][1]; + count8x8_lp += counts->tx_size[1][i][1]; + count8x8_lp += counts->tx_size[2][i][1]; + + count16x16_16x16p += counts->tx_size[1][i][2]; + count16x16_lp += counts->tx_size[2][i][2]; + count32x32 += counts->tx_size[2][i][3]; + } +#if CONFIG_EXT_TX && CONFIG_RECT_TX + count4x4 += counts->tx_size_implied[0][TX_4X4]; + count4x4 += counts->tx_size_implied[1][TX_4X4]; + count4x4 += counts->tx_size_implied[2][TX_4X4]; + count4x4 += counts->tx_size_implied[3][TX_4X4]; + count8x8_8x8p += counts->tx_size_implied[1][TX_8X8]; + count8x8_lp += counts->tx_size_implied[2][TX_8X8]; + count8x8_lp += counts->tx_size_implied[3][TX_8X8]; + count16x16_lp += counts->tx_size_implied[3][TX_16X16]; + count16x16_16x16p += counts->tx_size_implied[2][TX_16X16]; + count32x32 += counts->tx_size_implied[3][TX_32X32]; +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + if (count4x4 == 0 && count16x16_lp == 0 && count16x16_16x16p == 0 && +#if CONFIG_SUPERTX + cm->counts.supertx_size[TX_16X16] == 0 && + cm->counts.supertx_size[TX_32X32] == 0 && +#endif // CONFIG_SUPERTX + count32x32 == 0) { + cm->tx_mode = ALLOW_8X8; + reset_skip_tx_size(cm, TX_8X8); + } else if (count8x8_8x8p == 0 && count16x16_16x16p == 0 && + count8x8_lp == 0 && count16x16_lp == 0 && +#if CONFIG_SUPERTX + cm->counts.supertx_size[TX_8X8] == 0 && + cm->counts.supertx_size[TX_16X16] == 0 && + cm->counts.supertx_size[TX_32X32] == 0 && +#endif // CONFIG_SUPERTX + count32x32 == 0) { + cm->tx_mode = ONLY_4X4; + reset_skip_tx_size(cm, TX_4X4); + } else if (count8x8_lp == 0 && count16x16_lp == 0 && count4x4 == 0) { + cm->tx_mode = ALLOW_32X32; + } else if (count32x32 == 0 && count8x8_lp == 0 && +#if CONFIG_SUPERTX + cm->counts.supertx_size[TX_32X32] == 0 && +#endif // CONFIG_SUPERTX + count4x4 == 0) { + cm->tx_mode = ALLOW_16X16; + reset_skip_tx_size(cm, TX_16X16); + } +#endif // CONFIG_TX64X64 + } +#endif + } else { + encode_frame_internal(cpi); + } +} + +static void sum_intra_stats(FRAME_COUNTS *counts, MACROBLOCKD *xd, + const MODE_INFO *mi, const MODE_INFO *above_mi, + const MODE_INFO *left_mi, const int intraonly, + const int mi_row, const int mi_col) { + const MB_MODE_INFO *const mbmi = &mi->mbmi; + const PREDICTION_MODE y_mode = mbmi->mode; + const PREDICTION_MODE uv_mode = mbmi->uv_mode; + const BLOCK_SIZE bsize = mbmi->sb_type; + const int unify_bsize = CONFIG_CB4X4; + + if (bsize < BLOCK_8X8 && !unify_bsize) { + int idx, idy; + const int num_4x4_w = num_4x4_blocks_wide_lookup[bsize]; + const int num_4x4_h = num_4x4_blocks_high_lookup[bsize]; + for (idy = 0; idy < 2; idy += num_4x4_h) + for (idx = 0; idx < 2; idx += num_4x4_w) { + const int bidx = idy * 2 + idx; + const PREDICTION_MODE bmode = mi->bmi[bidx].as_mode; + if (intraonly) { + const PREDICTION_MODE a = av1_above_block_mode(mi, above_mi, bidx); + const PREDICTION_MODE l = av1_left_block_mode(mi, left_mi, bidx); + ++counts->kf_y_mode[a][l][bmode]; + } else { + ++counts->y_mode[0][bmode]; + } + } + } else { + if (intraonly) { + const PREDICTION_MODE above = av1_above_block_mode(mi, above_mi, 0); + const PREDICTION_MODE left = av1_left_block_mode(mi, left_mi, 0); + ++counts->kf_y_mode[above][left][y_mode]; + } else { + ++counts->y_mode[size_group_lookup[bsize]][y_mode]; + } +#if CONFIG_FILTER_INTRA + if (mbmi->mode == DC_PRED +#if CONFIG_PALETTE + && mbmi->palette_mode_info.palette_size[0] == 0 +#endif // CONFIG_PALETTE + ) { + const int use_filter_intra_mode = + mbmi->filter_intra_mode_info.use_filter_intra_mode[0]; + ++counts->filter_intra[0][use_filter_intra_mode]; + } + if (mbmi->uv_mode == DC_PRED +#if CONFIG_PALETTE + && mbmi->palette_mode_info.palette_size[1] == 0 +#endif // CONFIG_PALETTE + ) { + const int use_filter_intra_mode = + mbmi->filter_intra_mode_info.use_filter_intra_mode[1]; + ++counts->filter_intra[1][use_filter_intra_mode]; + } +#endif // CONFIG_FILTER_INTRA +#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP + if (av1_is_directional_mode(mbmi->mode, bsize)) { + const int intra_filter_ctx = av1_get_pred_context_intra_interp(xd); + const int p_angle = + mode_to_angle_map[mbmi->mode] + mbmi->angle_delta[0] * ANGLE_STEP; + if (av1_is_intra_filter_switchable(p_angle)) + ++counts->intra_filter[intra_filter_ctx][mbmi->intra_filter]; + } +#endif // CONFIG_INTRA_INTERP && CONFIG_INTRA_INTERP + } + +#if CONFIG_CB4X4 + if (!is_chroma_reference(mi_row, mi_col, bsize, xd->plane[1].subsampling_x, + xd->plane[1].subsampling_y)) + return; +#else + (void)mi_row; + (void)mi_col; + (void)xd; +#endif + ++counts->uv_mode[y_mode][uv_mode]; +} + +#if CONFIG_VAR_TX +static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd, + FRAME_COUNTS *counts, TX_SIZE tx_size, int depth, + int blk_row, int blk_col) { + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const int tx_row = blk_row >> 1; + const int tx_col = blk_col >> 1; + const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0); + const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0); + int ctx = txfm_partition_context(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, + mbmi->sb_type, tx_size); + const TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_row][tx_col]; + + if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + + if (tx_size == plane_tx_size) { + ++counts->txfm_partition[ctx][0]; + mbmi->tx_size = tx_size; + txfm_partition_update(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, tx_size, tx_size); + } else { + const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; + const int bs = tx_size_wide_unit[sub_txs]; + int i; + + ++counts->txfm_partition[ctx][1]; + ++x->txb_split_count; + + if (tx_size == TX_8X8) { + mbmi->inter_tx_size[tx_row][tx_col] = TX_4X4; + mbmi->tx_size = TX_4X4; + txfm_partition_update(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, TX_4X4, tx_size); + return; + } + + for (i = 0; i < 4; ++i) { + int offsetr = (i >> 1) * bs; + int offsetc = (i & 0x01) * bs; + update_txfm_count(x, xd, counts, sub_txs, depth + 1, blk_row + offsetr, + blk_col + offsetc); + } + } +} + +static void tx_partition_count_update(const AV1_COMMON *const cm, MACROBLOCK *x, + BLOCK_SIZE plane_bsize, int mi_row, + int mi_col, FRAME_COUNTS *td_counts) { + MACROBLOCKD *xd = &x->e_mbd; + const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; + const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0]; + TX_SIZE max_tx_size = get_vartx_max_txsize(&xd->mi[0]->mbmi, plane_bsize); + const int bh = tx_size_high_unit[max_tx_size]; + const int bw = tx_size_wide_unit[max_tx_size]; + int idx, idy; + + xd->above_txfm_context = cm->above_txfm_context + mi_col; + xd->left_txfm_context = + xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK); + + for (idy = 0; idy < mi_height; idy += bh) + for (idx = 0; idx < mi_width; idx += bw) + update_txfm_count(x, xd, td_counts, max_tx_size, mi_width != mi_height, + idy, idx); +} + +static void set_txfm_context(MACROBLOCKD *xd, TX_SIZE tx_size, int blk_row, + int blk_col) { + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const int tx_row = blk_row >> 1; + const int tx_col = blk_col >> 1; + const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0); + const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0); + const TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_row][tx_col]; + + if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + + if (tx_size == plane_tx_size) { + mbmi->tx_size = tx_size; + txfm_partition_update(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, tx_size, tx_size); + + } else { + const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; + const int bsl = tx_size_wide_unit[sub_txs]; + int i; + + if (tx_size == TX_8X8) { + mbmi->inter_tx_size[tx_row][tx_col] = TX_4X4; + mbmi->tx_size = TX_4X4; + txfm_partition_update(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, TX_4X4, tx_size); + return; + } + + assert(bsl > 0); + for (i = 0; i < 4; ++i) { + int offsetr = (i >> 1) * bsl; + int offsetc = (i & 0x01) * bsl; + set_txfm_context(xd, sub_txs, blk_row + offsetr, blk_col + offsetc); + } + } +} + +static void tx_partition_set_contexts(const AV1_COMMON *const cm, + MACROBLOCKD *xd, BLOCK_SIZE plane_bsize, + int mi_row, int mi_col) { + const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; + const int mi_height = block_size_high[plane_bsize] >> tx_size_high_log2[0]; + TX_SIZE max_tx_size = get_vartx_max_txsize(&xd->mi[0]->mbmi, plane_bsize); + const int bh = tx_size_high_unit[max_tx_size]; + const int bw = tx_size_wide_unit[max_tx_size]; + int idx, idy; + + xd->above_txfm_context = cm->above_txfm_context + mi_col; + xd->left_txfm_context = + xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK); + + for (idy = 0; idy < mi_height; idy += bh) + for (idx = 0; idx < mi_width; idx += bw) + set_txfm_context(xd, max_tx_size, idy, idx); +} +#endif + +void av1_update_tx_type_count(const AV1_COMMON *cm, MACROBLOCKD *xd, +#if CONFIG_TXK_SEL + int block, int plane, +#endif + BLOCK_SIZE bsize, TX_SIZE tx_size, + FRAME_COUNTS *counts) { + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + int is_inter = is_inter_block(mbmi); +#if !CONFIG_TXK_SEL + TX_TYPE tx_type = mbmi->tx_type; +#else + // Only y plane's tx_type is updated + if (plane > 0) return; + TX_TYPE tx_type = get_tx_type(PLANE_TYPE_Y, xd, block, tx_size); +#endif +#if CONFIG_EXT_TX + if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) > 1 && + cm->base_qindex > 0 && !mbmi->skip && + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + const int eset = + get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used); + if (eset > 0) { + if (is_inter) { + ++counts->inter_ext_tx[eset][txsize_sqr_map[tx_size]][tx_type]; + } else { + ++counts->intra_ext_tx[eset][txsize_sqr_map[tx_size]][mbmi->mode] + [tx_type]; + } + } + } +#else + (void)bsize; + if (tx_size < TX_32X32 && + ((!cm->seg.enabled && cm->base_qindex > 0) || + (cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) && + !mbmi->skip && + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + if (is_inter) { + ++counts->inter_ext_tx[tx_size][tx_type]; + } else { + ++counts->intra_ext_tx[tx_size][intra_mode_to_tx_type_context[mbmi->mode]] + [tx_type]; + } + } +#endif // CONFIG_EXT_TX +} + +static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td, + TOKENEXTRA **t, RUN_TYPE dry_run, int mi_row, + int mi_col, BLOCK_SIZE bsize, + PICK_MODE_CONTEXT *ctx, int *rate) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + MODE_INFO **mi_8x8 = xd->mi; + MODE_INFO *mi = mi_8x8[0]; + MB_MODE_INFO *mbmi = &mi->mbmi; + const int seg_skip = + segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP); + const int mis = cm->mi_stride; + const int mi_width = mi_size_wide[bsize]; + const int mi_height = mi_size_high[bsize]; + const int is_inter = is_inter_block(mbmi); +#if CONFIG_CB4X4 + const BLOCK_SIZE block_size = bsize; +#else + const BLOCK_SIZE block_size = AOMMAX(bsize, BLOCK_8X8); +#endif + +#if CONFIG_PVQ + x->pvq_speed = 0; + x->pvq_coded = (dry_run == OUTPUT_ENABLED) ? 1 : 0; +#endif +#if CONFIG_CFL + x->cfl_store_y = (dry_run == OUTPUT_ENABLED) ? 1 : 0; +#endif + + if (!is_inter) { + int plane; + mbmi->skip = 1; + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + av1_encode_intra_block_plane((AV1_COMMON *)cm, x, block_size, plane, 1, + mi_row, mi_col); + } + if (!dry_run) { + sum_intra_stats(td->counts, xd, mi, xd->above_mi, xd->left_mi, + frame_is_intra_only(cm), mi_row, mi_col); + } +#if CONFIG_PALETTE + if (bsize >= BLOCK_8X8 && !dry_run) { + for (plane = 0; plane <= 1; ++plane) { + if (mbmi->palette_mode_info.palette_size[plane] > 0) { + mbmi->palette_mode_info.palette_first_color_idx[plane] = + xd->plane[plane].color_index_map[0]; + // TODO(huisu): this increases the use of token buffer. Needs stretch + // test to verify. + av1_tokenize_palette_sb(cpi, td, plane, t, dry_run, bsize, rate); + } + } + } +#endif // CONFIG_PALETTE +#if CONFIG_VAR_TX + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); +#endif +#if CONFIG_LV_MAP + av1_update_txb_context(cpi, td, dry_run, block_size, rate, mi_row, mi_col); +#else // CONFIG_LV_MAP + av1_tokenize_sb(cpi, td, t, dry_run, block_size, rate, mi_row, mi_col); +#endif // CONFIG_LV_MAP + } else { + int ref; + const int is_compound = has_second_ref(mbmi); + + set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]); + for (ref = 0; ref < 1 + is_compound; ++ref) { + YV12_BUFFER_CONFIG *cfg = get_ref_frame_buffer(cpi, mbmi->ref_frame[ref]); +#if CONFIG_INTRABC + assert(IMPLIES(!is_intrabc_block(mbmi), cfg)); +#else + assert(cfg != NULL); +#endif // !CONFIG_INTRABC + av1_setup_pre_planes(xd, ref, cfg, mi_row, mi_col, + &xd->block_refs[ref]->sf); + } + if (!(cpi->sf.reuse_inter_pred_sby && ctx->pred_pixel_ready) || seg_skip) + av1_build_inter_predictors_sby(xd, mi_row, mi_col, NULL, block_size); + + av1_build_inter_predictors_sbuv(xd, mi_row, mi_col, NULL, block_size); +#if CONFIG_MOTION_VAR + if (mbmi->motion_mode == OBMC_CAUSAL) { +#if CONFIG_NCOBMC + if (dry_run == OUTPUT_ENABLED) + av1_build_ncobmc_inter_predictors_sb(cm, xd, mi_row, mi_col); + else +#endif + av1_build_obmc_inter_predictors_sb(cm, xd, mi_row, mi_col); + } +#endif // CONFIG_MOTION_VAR + + av1_encode_sb((AV1_COMMON *)cm, x, block_size, mi_row, mi_col); +#if CONFIG_VAR_TX + if (mbmi->skip) mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); + av1_tokenize_sb_vartx(cpi, td, t, dry_run, mi_row, mi_col, block_size, + rate); +#else +#if CONFIG_LV_MAP + av1_update_txb_context(cpi, td, dry_run, block_size, rate, mi_row, mi_col); +#else // CONFIG_LV_MAP + av1_tokenize_sb(cpi, td, t, dry_run, block_size, rate, mi_row, mi_col); +#endif // CONFIG_LV_MAP +#endif + } + + if (!dry_run) { +#if CONFIG_VAR_TX + TX_SIZE tx_size = + is_inter && !mbmi->skip ? mbmi->min_tx_size : mbmi->tx_size; +#else + TX_SIZE tx_size = mbmi->tx_size; +#endif + if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id] && +#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX + mbmi->sb_type > BLOCK_4X4 && +#else + mbmi->sb_type >= BLOCK_8X8 && +#endif + !(is_inter && (mbmi->skip || seg_skip))) { +#if CONFIG_VAR_TX + if (is_inter) { + tx_partition_count_update(cm, x, bsize, mi_row, mi_col, td->counts); + } else { + const int tx_size_ctx = get_tx_size_context(xd); + const int tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize] + : intra_tx_size_cat_lookup[bsize]; + const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size]; + const int depth = tx_size_to_depth(coded_tx_size); + ++td->counts->tx_size[tx_size_cat][tx_size_ctx][depth]; + if (tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count; + } +#else + const int tx_size_ctx = get_tx_size_context(xd); + const int tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize] + : intra_tx_size_cat_lookup[bsize]; + const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size]; + const int depth = tx_size_to_depth(coded_tx_size); + + ++td->counts->tx_size[tx_size_cat][tx_size_ctx][depth]; +#endif +#if CONFIG_EXT_TX && CONFIG_RECT_TX + assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi))); +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + } else { + int i, j; + TX_SIZE intra_tx_size; + // The new intra coding scheme requires no change of transform size + if (is_inter) { + if (xd->lossless[mbmi->segment_id]) { + intra_tx_size = TX_4X4; + } else { + intra_tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 1); + } + } else { +#if CONFIG_EXT_TX && CONFIG_RECT_TX + intra_tx_size = tx_size; +#else + intra_tx_size = (bsize >= BLOCK_8X8) ? tx_size : TX_4X4; +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + } +#if CONFIG_EXT_TX && CONFIG_RECT_TX + ++td->counts->tx_size_implied[max_txsize_lookup[bsize]] + [txsize_sqr_up_map[tx_size]]; +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + + for (j = 0; j < mi_height; j++) + for (i = 0; i < mi_width; i++) + if (mi_col + i < cm->mi_cols && mi_row + j < cm->mi_rows) + mi_8x8[mis * j + i]->mbmi.tx_size = intra_tx_size; + +#if CONFIG_VAR_TX + mbmi->min_tx_size = get_min_tx_size(intra_tx_size); + if (intra_tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count; +#endif + } + + ++td->counts->tx_size_totals[txsize_sqr_map[tx_size]]; + ++td->counts + ->tx_size_totals[txsize_sqr_map[get_uv_tx_size(mbmi, &xd->plane[1])]]; +#if !CONFIG_TXK_SEL + av1_update_tx_type_count(cm, xd, bsize, tx_size, td->counts); +#endif + } + +#if CONFIG_VAR_TX + if (cm->tx_mode == TX_MODE_SELECT && +#if CONFIG_CB4X4 + mbmi->sb_type > BLOCK_4X4 && +#else + mbmi->sb_type >= BLOCK_8X8 && +#endif + is_inter && !(mbmi->skip || seg_skip)) { + if (dry_run) tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col); + } else { + TX_SIZE tx_size = mbmi->tx_size; + // The new intra coding scheme requires no change of transform size + if (is_inter) + tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, is_inter); + else + tx_size = (bsize > BLOCK_4X4) ? tx_size : TX_4X4; + mbmi->tx_size = tx_size; + set_txfm_ctxs(tx_size, xd->n8_w, xd->n8_h, (mbmi->skip || seg_skip), xd); + } +#endif // CONFIG_VAR_TX +} + +#if CONFIG_SUPERTX +static int check_intra_b(PICK_MODE_CONTEXT *ctx) { + if (!is_inter_mode((&ctx->mic)->mbmi.mode)) return 1; +#if CONFIG_EXT_INTER + if (ctx->mic.mbmi.ref_frame[1] == INTRA_FRAME) return 1; +#endif // CONFIG_EXT_INTER + return 0; +} + +static int check_intra_sb(const AV1_COMP *const cpi, const TileInfo *const tile, + int mi_row, int mi_col, BLOCK_SIZE bsize, + PC_TREE *pc_tree) { + const AV1_COMMON *const cm = &cpi->common; + const int hbs = mi_size_wide[bsize] / 2; + const PARTITION_TYPE partition = pc_tree->partitioning; + const BLOCK_SIZE subsize = get_subsize(bsize, partition); +#if CONFIG_EXT_PARTITION_TYPES + int i; +#endif +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + +#if !CONFIG_CB4X4 + assert(bsize >= BLOCK_8X8); +#endif + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return 1; + + switch (partition) { + case PARTITION_NONE: return check_intra_b(&pc_tree->none); break; + case PARTITION_VERT: + if (check_intra_b(&pc_tree->vertical[0])) return 1; + if (mi_col + hbs < cm->mi_cols && (bsize > BLOCK_8X8 || unify_bsize)) { + if (check_intra_b(&pc_tree->vertical[1])) return 1; + } + break; + case PARTITION_HORZ: + if (check_intra_b(&pc_tree->horizontal[0])) return 1; + if (mi_row + hbs < cm->mi_rows && (bsize > BLOCK_8X8 || unify_bsize)) { + if (check_intra_b(&pc_tree->horizontal[1])) return 1; + } + break; + case PARTITION_SPLIT: + if (bsize == BLOCK_8X8 && !unify_bsize) { + if (check_intra_b(pc_tree->leaf_split[0])) return 1; + } else { + if (check_intra_sb(cpi, tile, mi_row, mi_col, subsize, + pc_tree->split[0])) + return 1; + if (check_intra_sb(cpi, tile, mi_row, mi_col + hbs, subsize, + pc_tree->split[1])) + return 1; + if (check_intra_sb(cpi, tile, mi_row + hbs, mi_col, subsize, + pc_tree->split[2])) + return 1; + if (check_intra_sb(cpi, tile, mi_row + hbs, mi_col + hbs, subsize, + pc_tree->split[3])) + return 1; + } + break; +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_HORZ_A: + for (i = 0; i < 3; i++) { + if (check_intra_b(&pc_tree->horizontala[i])) return 1; + } + break; + case PARTITION_HORZ_B: + for (i = 0; i < 3; i++) { + if (check_intra_b(&pc_tree->horizontalb[i])) return 1; + } + break; + case PARTITION_VERT_A: + for (i = 0; i < 3; i++) { + if (check_intra_b(&pc_tree->verticala[i])) return 1; + } + break; + case PARTITION_VERT_B: + for (i = 0; i < 3; i++) { + if (check_intra_b(&pc_tree->verticalb[i])) return 1; + } + break; +#endif // CONFIG_EXT_PARTITION_TYPES + default: assert(0); + } + return 0; +} + +static int check_supertx_b(TX_SIZE supertx_size, PICK_MODE_CONTEXT *ctx) { + return ctx->mic.mbmi.tx_size == supertx_size; +} + +static int check_supertx_sb(BLOCK_SIZE bsize, TX_SIZE supertx_size, + PC_TREE *pc_tree) { + PARTITION_TYPE partition; + BLOCK_SIZE subsize; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + + partition = pc_tree->partitioning; + subsize = get_subsize(bsize, partition); + switch (partition) { + case PARTITION_NONE: return check_supertx_b(supertx_size, &pc_tree->none); + case PARTITION_VERT: + return check_supertx_b(supertx_size, &pc_tree->vertical[0]); + case PARTITION_HORZ: + return check_supertx_b(supertx_size, &pc_tree->horizontal[0]); + case PARTITION_SPLIT: + if (bsize == BLOCK_8X8 && !unify_bsize) + return check_supertx_b(supertx_size, pc_tree->leaf_split[0]); + else + return check_supertx_sb(subsize, supertx_size, pc_tree->split[0]); +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_HORZ_A: + return check_supertx_b(supertx_size, &pc_tree->horizontala[0]); + case PARTITION_HORZ_B: + return check_supertx_b(supertx_size, &pc_tree->horizontalb[0]); + case PARTITION_VERT_A: + return check_supertx_b(supertx_size, &pc_tree->verticala[0]); + case PARTITION_VERT_B: + return check_supertx_b(supertx_size, &pc_tree->verticalb[0]); +#endif // CONFIG_EXT_PARTITION_TYPES + default: assert(0); return 0; + } +} + +static void predict_superblock(const AV1_COMP *const cpi, ThreadData *td, +#if CONFIG_EXT_INTER + int mi_row_ori, int mi_col_ori, +#endif // CONFIG_EXT_INTER + int mi_row_pred, int mi_col_pred, + BLOCK_SIZE bsize_pred, int b_sub8x8, int block) { + // Used in supertx + // (mi_row_ori, mi_col_ori): location for mv + // (mi_row_pred, mi_col_pred, bsize_pred): region to predict + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + MODE_INFO *mi_8x8 = xd->mi[0]; + MODE_INFO *mi = mi_8x8; + MB_MODE_INFO *mbmi = &mi->mbmi; + int ref; + const int is_compound = has_second_ref(mbmi); + + set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]); + + for (ref = 0; ref < 1 + is_compound; ++ref) { + YV12_BUFFER_CONFIG *cfg = get_ref_frame_buffer(cpi, mbmi->ref_frame[ref]); + av1_setup_pre_planes(xd, ref, cfg, mi_row_pred, mi_col_pred, + &xd->block_refs[ref]->sf); + } + + if (!b_sub8x8) + av1_build_inter_predictors_sb_extend(xd, +#if CONFIG_EXT_INTER + mi_row_ori, mi_col_ori, +#endif // CONFIG_EXT_INTER + mi_row_pred, mi_col_pred, bsize_pred); + else + av1_build_inter_predictors_sb_sub8x8_extend(xd, +#if CONFIG_EXT_INTER + mi_row_ori, mi_col_ori, +#endif // CONFIG_EXT_INTER + mi_row_pred, mi_col_pred, + bsize_pred, block); +} + +static void predict_b_extend(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, int block, + int mi_row_ori, int mi_col_ori, int mi_row_pred, + int mi_col_pred, int mi_row_top, int mi_col_top, + uint8_t *dst_buf[3], int dst_stride[3], + BLOCK_SIZE bsize_top, BLOCK_SIZE bsize_pred, + RUN_TYPE dry_run, int b_sub8x8, int bextend) { + // Used in supertx + // (mi_row_ori, mi_col_ori): location for mv + // (mi_row_pred, mi_col_pred, bsize_pred): region to predict + // (mi_row_top, mi_col_top, bsize_top): region of the top partition size + // block: sub location of sub8x8 blocks + // b_sub8x8: 1: ori is sub8x8; 0: ori is not sub8x8 + // bextend: 1: region to predict is an extension of ori; 0: not + + MACROBLOCK *const x = &td->mb; + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + int r = (mi_row_pred - mi_row_top) * MI_SIZE; + int c = (mi_col_pred - mi_col_top) * MI_SIZE; + const int mi_width_top = mi_size_wide[bsize_top]; + const int mi_height_top = mi_size_high[bsize_top]; + + if (mi_row_pred < mi_row_top || mi_col_pred < mi_col_top || + mi_row_pred >= mi_row_top + mi_height_top || + mi_col_pred >= mi_col_top + mi_width_top || mi_row_pred >= cm->mi_rows || + mi_col_pred >= cm->mi_cols) + return; + + set_offsets_extend(cpi, td, tile, mi_row_pred, mi_col_pred, mi_row_ori, + mi_col_ori, bsize_pred); + xd->plane[0].dst.stride = dst_stride[0]; + xd->plane[1].dst.stride = dst_stride[1]; + xd->plane[2].dst.stride = dst_stride[2]; + xd->plane[0].dst.buf = dst_buf[0] + + (r >> xd->plane[0].subsampling_y) * dst_stride[0] + + (c >> xd->plane[0].subsampling_x); + xd->plane[1].dst.buf = dst_buf[1] + + (r >> xd->plane[1].subsampling_y) * dst_stride[1] + + (c >> xd->plane[1].subsampling_x); + xd->plane[2].dst.buf = dst_buf[2] + + (r >> xd->plane[2].subsampling_y) * dst_stride[2] + + (c >> xd->plane[2].subsampling_x); + + predict_superblock(cpi, td, +#if CONFIG_EXT_INTER + mi_row_ori, mi_col_ori, +#endif // CONFIG_EXT_INTER + mi_row_pred, mi_col_pred, bsize_pred, b_sub8x8, block); + + if (!dry_run && !bextend) + update_stats(&cpi->common, td, mi_row_pred, mi_col_pred, 1); +} + +static void extend_dir(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, int block, BLOCK_SIZE bsize, + BLOCK_SIZE top_bsize, int mi_row, int mi_col, + int mi_row_top, int mi_col_top, RUN_TYPE dry_run, + uint8_t *dst_buf[3], int dst_stride[3], int dir) { + // dir: 0-lower, 1-upper, 2-left, 3-right + // 4-lowerleft, 5-upperleft, 6-lowerright, 7-upperright + MACROBLOCKD *xd = &td->mb.e_mbd; + const int mi_width = mi_size_wide[bsize]; + const int mi_height = mi_size_high[bsize]; + int xss = xd->plane[1].subsampling_x; + int yss = xd->plane[1].subsampling_y; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + int b_sub8x8 = (bsize < BLOCK_8X8) && !unify_bsize ? 1 : 0; + int wide_unit, high_unit; + int i, j; + int ext_offset = 0; + + BLOCK_SIZE extend_bsize; + int mi_row_pred, mi_col_pred; + + if (dir == 0 || dir == 1) { // lower and upper + extend_bsize = + (mi_width == mi_size_wide[BLOCK_8X8] || bsize < BLOCK_8X8 || xss < yss) + ? BLOCK_8X8 + : BLOCK_16X8; + +#if CONFIG_CB4X4 + if (bsize < BLOCK_8X8) { + extend_bsize = BLOCK_4X4; + ext_offset = mi_size_wide[BLOCK_8X8]; + } +#endif + wide_unit = mi_size_wide[extend_bsize]; + high_unit = mi_size_high[extend_bsize]; + + mi_row_pred = mi_row + ((dir == 0) ? mi_height : -(mi_height + ext_offset)); + mi_col_pred = mi_col; + + for (j = 0; j < mi_height + ext_offset; j += high_unit) + for (i = 0; i < mi_width + ext_offset; i += wide_unit) + predict_b_extend(cpi, td, tile, block, mi_row, mi_col, mi_row_pred + j, + mi_col_pred + i, mi_row_top, mi_col_top, dst_buf, + dst_stride, top_bsize, extend_bsize, dry_run, b_sub8x8, + 1); + } else if (dir == 2 || dir == 3) { // left and right + extend_bsize = + (mi_height == mi_size_high[BLOCK_8X8] || bsize < BLOCK_8X8 || yss < xss) + ? BLOCK_8X8 + : BLOCK_8X16; +#if CONFIG_CB4X4 + if (bsize < BLOCK_8X8) { + extend_bsize = BLOCK_4X4; + ext_offset = mi_size_wide[BLOCK_8X8]; + } +#endif + wide_unit = mi_size_wide[extend_bsize]; + high_unit = mi_size_high[extend_bsize]; + + mi_row_pred = mi_row; + mi_col_pred = mi_col + ((dir == 3) ? mi_width : -(mi_width + ext_offset)); + + for (j = 0; j < mi_height + ext_offset; j += high_unit) + for (i = 0; i < mi_width + ext_offset; i += wide_unit) + predict_b_extend(cpi, td, tile, block, mi_row, mi_col, mi_row_pred + j, + mi_col_pred + i, mi_row_top, mi_col_top, dst_buf, + dst_stride, top_bsize, extend_bsize, dry_run, b_sub8x8, + 1); + } else { + extend_bsize = BLOCK_8X8; +#if CONFIG_CB4X4 + if (bsize < BLOCK_8X8) { + extend_bsize = BLOCK_4X4; + ext_offset = mi_size_wide[BLOCK_8X8]; + } +#endif + wide_unit = mi_size_wide[extend_bsize]; + high_unit = mi_size_high[extend_bsize]; + + mi_row_pred = mi_row + ((dir == 4 || dir == 6) ? mi_height + : -(mi_height + ext_offset)); + mi_col_pred = + mi_col + ((dir == 6 || dir == 7) ? mi_width : -(mi_width + ext_offset)); + + for (j = 0; j < mi_height + ext_offset; j += high_unit) + for (i = 0; i < mi_width + ext_offset; i += wide_unit) + predict_b_extend(cpi, td, tile, block, mi_row, mi_col, mi_row_pred + j, + mi_col_pred + i, mi_row_top, mi_col_top, dst_buf, + dst_stride, top_bsize, extend_bsize, dry_run, b_sub8x8, + 1); + } +} + +static void extend_all(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, int block, BLOCK_SIZE bsize, + BLOCK_SIZE top_bsize, int mi_row, int mi_col, + int mi_row_top, int mi_col_top, RUN_TYPE dry_run, + uint8_t *dst_buf[3], int dst_stride[3]) { + assert(block >= 0 && block < 4); + extend_dir(cpi, td, tile, block, bsize, top_bsize, mi_row, mi_col, mi_row_top, + mi_col_top, dry_run, dst_buf, dst_stride, 0); + extend_dir(cpi, td, tile, block, bsize, top_bsize, mi_row, mi_col, mi_row_top, + mi_col_top, dry_run, dst_buf, dst_stride, 1); + extend_dir(cpi, td, tile, block, bsize, top_bsize, mi_row, mi_col, mi_row_top, + mi_col_top, dry_run, dst_buf, dst_stride, 2); + extend_dir(cpi, td, tile, block, bsize, top_bsize, mi_row, mi_col, mi_row_top, + mi_col_top, dry_run, dst_buf, dst_stride, 3); + extend_dir(cpi, td, tile, block, bsize, top_bsize, mi_row, mi_col, mi_row_top, + mi_col_top, dry_run, dst_buf, dst_stride, 4); + extend_dir(cpi, td, tile, block, bsize, top_bsize, mi_row, mi_col, mi_row_top, + mi_col_top, dry_run, dst_buf, dst_stride, 5); + extend_dir(cpi, td, tile, block, bsize, top_bsize, mi_row, mi_col, mi_row_top, + mi_col_top, dry_run, dst_buf, dst_stride, 6); + extend_dir(cpi, td, tile, block, bsize, top_bsize, mi_row, mi_col, mi_row_top, + mi_col_top, dry_run, dst_buf, dst_stride, 7); +} + +// This function generates prediction for multiple blocks, between which +// discontinuity around boundary is reduced by smoothing masks. The basic +// smoothing mask is a soft step function along horz/vert direction. In more +// complicated case when a block is split into 4 subblocks, the basic mask is +// first applied to neighboring subblocks (2 pairs) in horizontal direction and +// then applied to the 2 masked prediction mentioned above in vertical direction +// If the block is split into more than one level, at every stage, masked +// prediction is stored in dst_buf[] passed from higher level. +static void predict_sb_complex(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, int mi_row, + int mi_col, int mi_row_top, int mi_col_top, + RUN_TYPE dry_run, BLOCK_SIZE bsize, + BLOCK_SIZE top_bsize, uint8_t *dst_buf[3], + int dst_stride[3], PC_TREE *pc_tree) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + const int hbs = mi_size_wide[bsize] / 2; + const int is_partition_root = bsize >= BLOCK_8X8; + const int ctx = is_partition_root + ? partition_plane_context(xd, mi_row, mi_col, +#if CONFIG_UNPOISON_PARTITION_CTX + mi_row + hbs < cm->mi_rows, + mi_col + hbs < cm->mi_cols, +#endif + bsize) + : -1; + const PARTITION_TYPE partition = pc_tree->partitioning; + const BLOCK_SIZE subsize = get_subsize(bsize, partition); +#if CONFIG_EXT_PARTITION_TYPES + const BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT); +#endif + + int i; + uint8_t *dst_buf1[3], *dst_buf2[3], *dst_buf3[3]; + DECLARE_ALIGNED(16, uint8_t, tmp_buf1[MAX_MB_PLANE * MAX_TX_SQUARE * 2]); + DECLARE_ALIGNED(16, uint8_t, tmp_buf2[MAX_MB_PLANE * MAX_TX_SQUARE * 2]); + DECLARE_ALIGNED(16, uint8_t, tmp_buf3[MAX_MB_PLANE * MAX_TX_SQUARE * 2]); + int dst_stride1[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE }; + int dst_stride2[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE }; + int dst_stride3[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE }; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; + assert(bsize >= BLOCK_8X8); +#endif + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + int len = sizeof(uint16_t); + dst_buf1[0] = CONVERT_TO_BYTEPTR(tmp_buf1); + dst_buf1[1] = CONVERT_TO_BYTEPTR(tmp_buf1 + MAX_TX_SQUARE * len); + dst_buf1[2] = CONVERT_TO_BYTEPTR(tmp_buf1 + 2 * MAX_TX_SQUARE * len); + dst_buf2[0] = CONVERT_TO_BYTEPTR(tmp_buf2); + dst_buf2[1] = CONVERT_TO_BYTEPTR(tmp_buf2 + MAX_TX_SQUARE * len); + dst_buf2[2] = CONVERT_TO_BYTEPTR(tmp_buf2 + 2 * MAX_TX_SQUARE * len); + dst_buf3[0] = CONVERT_TO_BYTEPTR(tmp_buf3); + dst_buf3[1] = CONVERT_TO_BYTEPTR(tmp_buf3 + MAX_TX_SQUARE * len); + dst_buf3[2] = CONVERT_TO_BYTEPTR(tmp_buf3 + 2 * MAX_TX_SQUARE * len); + } else { +#endif // CONFIG_HIGHBITDEPTH + dst_buf1[0] = tmp_buf1; + dst_buf1[1] = tmp_buf1 + MAX_TX_SQUARE; + dst_buf1[2] = tmp_buf1 + 2 * MAX_TX_SQUARE; + dst_buf2[0] = tmp_buf2; + dst_buf2[1] = tmp_buf2 + MAX_TX_SQUARE; + dst_buf2[2] = tmp_buf2 + 2 * MAX_TX_SQUARE; + dst_buf3[0] = tmp_buf3; + dst_buf3[1] = tmp_buf3 + MAX_TX_SQUARE; + dst_buf3[2] = tmp_buf3 + 2 * MAX_TX_SQUARE; +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + + if (!dry_run && ctx >= 0 && bsize < top_bsize) { + // Explicitly cast away const. + FRAME_COUNTS *const frame_counts = (FRAME_COUNTS *)&cm->counts; + frame_counts->partition[ctx][partition]++; + } + + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + } + + switch (partition) { + case PARTITION_NONE: + assert(bsize < top_bsize); + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, top_bsize, + bsize, dry_run, 0, 0); + extend_all(cpi, td, tile, 0, bsize, top_bsize, mi_row, mi_col, mi_row_top, + mi_col_top, dry_run, dst_buf, dst_stride); + break; + case PARTITION_HORZ: + if (bsize == BLOCK_8X8 && !unify_bsize) { + // Fisrt half + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, top_bsize, + BLOCK_8X8, dry_run, 1, 0); + if (bsize < top_bsize) + extend_all(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride); + + // Second half + predict_b_extend(cpi, td, tile, 2, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf1, dst_stride1, + top_bsize, BLOCK_8X8, dry_run, 1, 1); + if (bsize < top_bsize) + extend_all(cpi, td, tile, 2, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf1, dst_stride1); + + // Smooth + xd->plane[0].dst.buf = dst_buf[0]; + xd->plane[0].dst.stride = dst_stride[0]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[0], dst_stride[0], dst_buf1[0], dst_stride1[0], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_HORZ, + 0); + } else { + // First half + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, top_bsize, + subsize, dry_run, 0, 0); + if (bsize < top_bsize) + extend_all(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride); + else + extend_dir(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride, 0); + + if (mi_row + hbs < cm->mi_rows) { + // Second half + predict_b_extend(cpi, td, tile, 0, mi_row + hbs, mi_col, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dst_buf1, + dst_stride1, top_bsize, subsize, dry_run, 0, 0); + if (bsize < top_bsize) + extend_all(cpi, td, tile, 0, subsize, top_bsize, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dry_run, dst_buf1, + dst_stride1); + else + extend_dir(cpi, td, tile, 0, subsize, top_bsize, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dry_run, dst_buf1, + dst_stride1, 1); + + // Smooth + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_HORZ, i); + } + } + } + break; + case PARTITION_VERT: + if (bsize == BLOCK_8X8 && !unify_bsize) { + // First half + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, top_bsize, + BLOCK_8X8, dry_run, 1, 0); + if (bsize < top_bsize) + extend_all(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride); + + // Second half + predict_b_extend(cpi, td, tile, 1, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf1, dst_stride1, + top_bsize, BLOCK_8X8, dry_run, 1, 1); + if (bsize < top_bsize) + extend_all(cpi, td, tile, 1, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf1, dst_stride1); + + // Smooth + xd->plane[0].dst.buf = dst_buf[0]; + xd->plane[0].dst.stride = dst_stride[0]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[0], dst_stride[0], dst_buf1[0], dst_stride1[0], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_VERT, + 0); + } else { + // bsize: not important, not useful + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, top_bsize, + subsize, dry_run, 0, 0); + if (bsize < top_bsize) + extend_all(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride); + else + extend_dir(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride, 3); + + if (mi_col + hbs < cm->mi_cols) { + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col + hbs, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf1, + dst_stride1, top_bsize, subsize, dry_run, 0, 0); + if (bsize < top_bsize) + extend_all(cpi, td, tile, 0, subsize, top_bsize, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dry_run, dst_buf1, + dst_stride1); + else + extend_dir(cpi, td, tile, 0, subsize, top_bsize, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dry_run, dst_buf1, + dst_stride1, 2); + + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_VERT, i); + } + } + } + break; + case PARTITION_SPLIT: + if (bsize == BLOCK_8X8 && !unify_bsize) { + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, top_bsize, + BLOCK_8X8, dry_run, 1, 0); + predict_b_extend(cpi, td, tile, 1, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf1, dst_stride1, + top_bsize, BLOCK_8X8, dry_run, 1, 1); + predict_b_extend(cpi, td, tile, 2, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf2, dst_stride2, + top_bsize, BLOCK_8X8, dry_run, 1, 1); + predict_b_extend(cpi, td, tile, 3, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf3, dst_stride3, + top_bsize, BLOCK_8X8, dry_run, 1, 1); + + if (bsize < top_bsize) { + extend_all(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride); + extend_all(cpi, td, tile, 1, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf1, dst_stride1); + extend_all(cpi, td, tile, 2, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf2, dst_stride2); + extend_all(cpi, td, tile, 3, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf3, dst_stride3); + } + } else { + predict_sb_complex(cpi, td, tile, mi_row, mi_col, mi_row_top, + mi_col_top, dry_run, subsize, top_bsize, dst_buf, + dst_stride, pc_tree->split[0]); + if (mi_row < cm->mi_rows && mi_col + hbs < cm->mi_cols) + predict_sb_complex(cpi, td, tile, mi_row, mi_col + hbs, mi_row_top, + mi_col_top, dry_run, subsize, top_bsize, dst_buf1, + dst_stride1, pc_tree->split[1]); + if (mi_row + hbs < cm->mi_rows && mi_col < cm->mi_cols) + predict_sb_complex(cpi, td, tile, mi_row + hbs, mi_col, mi_row_top, + mi_col_top, dry_run, subsize, top_bsize, dst_buf2, + dst_stride2, pc_tree->split[2]); + if (mi_row + hbs < cm->mi_rows && mi_col + hbs < cm->mi_cols) + predict_sb_complex(cpi, td, tile, mi_row + hbs, mi_col + hbs, + mi_row_top, mi_col_top, dry_run, subsize, + top_bsize, dst_buf3, dst_stride3, + pc_tree->split[3]); + } + for (i = 0; i < MAX_MB_PLANE; i++) { +#if !CONFIG_CB4X4 + if (bsize == BLOCK_8X8 && i != 0) + continue; // Skip <4x4 chroma smoothing +#endif + if (mi_row < cm->mi_rows && mi_col + hbs < cm->mi_cols) { + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_VERT, i); + if (mi_row + hbs < cm->mi_rows) { + av1_build_masked_inter_predictor_complex( + xd, dst_buf2[i], dst_stride2[i], dst_buf3[i], dst_stride3[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_VERT, i); + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf2[i], dst_stride2[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_HORZ, i); + } + } else if (mi_row + hbs < cm->mi_rows && mi_col < cm->mi_cols) { + if (bsize == BLOCK_8X8 && i != 0) + continue; // Skip <4x4 chroma smoothing + + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf2[i], dst_stride2[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_HORZ, i); + } + } + break; +#if CONFIG_EXT_PARTITION_TYPES + case PARTITION_HORZ_A: + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, top_bsize, + bsize2, dry_run, 0, 0); + extend_all(cpi, td, tile, 0, bsize2, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride); + + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col + hbs, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf1, + dst_stride1, top_bsize, bsize2, dry_run, 0, 0); + extend_all(cpi, td, tile, 0, bsize2, top_bsize, mi_row, mi_col + hbs, + mi_row_top, mi_col_top, dry_run, dst_buf1, dst_stride1); + + predict_b_extend(cpi, td, tile, 0, mi_row + hbs, mi_col, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dst_buf2, dst_stride2, + top_bsize, subsize, dry_run, 0, 0); + if (bsize < top_bsize) + extend_all(cpi, td, tile, 0, subsize, top_bsize, mi_row + hbs, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf2, dst_stride2); + else + extend_dir(cpi, td, tile, 0, subsize, top_bsize, mi_row + hbs, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf2, dst_stride2, 1); + + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_VERT, + i); + } + for (i = 0; i < MAX_MB_PLANE; i++) { + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf2[i], dst_stride2[i], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_HORZ, + i); + } + + break; + case PARTITION_VERT_A: + + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, top_bsize, + bsize2, dry_run, 0, 0); + extend_all(cpi, td, tile, 0, bsize2, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride); + + predict_b_extend(cpi, td, tile, 0, mi_row + hbs, mi_col, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dst_buf1, dst_stride1, + top_bsize, bsize2, dry_run, 0, 0); + extend_all(cpi, td, tile, 0, bsize2, top_bsize, mi_row + hbs, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf1, dst_stride1); + + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col + hbs, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf2, + dst_stride2, top_bsize, subsize, dry_run, 0, 0); + if (bsize < top_bsize) + extend_all(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col + hbs, + mi_row_top, mi_col_top, dry_run, dst_buf2, dst_stride2); + else + extend_dir(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col + hbs, + mi_row_top, mi_col_top, dry_run, dst_buf2, dst_stride2, 2); + + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_HORZ, + i); + } + for (i = 0; i < MAX_MB_PLANE; i++) { + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf2[i], dst_stride2[i], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_VERT, + i); + } + break; + case PARTITION_HORZ_B: + + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, top_bsize, + subsize, dry_run, 0, 0); + if (bsize < top_bsize) + extend_all(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride); + else + extend_dir(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride, 0); + + predict_b_extend(cpi, td, tile, 0, mi_row + hbs, mi_col, mi_row + hbs, + mi_col, mi_row_top, mi_col_top, dst_buf1, dst_stride1, + top_bsize, bsize2, dry_run, 0, 0); + extend_all(cpi, td, tile, 0, bsize2, top_bsize, mi_row + hbs, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf1, dst_stride1); + + predict_b_extend(cpi, td, tile, 0, mi_row + hbs, mi_col + hbs, + mi_row + hbs, mi_col + hbs, mi_row_top, mi_col_top, + dst_buf2, dst_stride2, top_bsize, bsize2, dry_run, 0, 0); + extend_all(cpi, td, tile, 0, bsize2, top_bsize, mi_row + hbs, + mi_col + hbs, mi_row_top, mi_col_top, dry_run, dst_buf2, + dst_stride2); + + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf1[i]; + xd->plane[i].dst.stride = dst_stride1[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf1[i], dst_stride1[i], dst_buf2[i], dst_stride2[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_VERT, i); + } + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_HORZ, + i); + } + break; + case PARTITION_VERT_B: + + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col, mi_row, mi_col, + mi_row_top, mi_col_top, dst_buf, dst_stride, top_bsize, + subsize, dry_run, 0, 0); + if (bsize < top_bsize) + extend_all(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride); + else + extend_dir(cpi, td, tile, 0, subsize, top_bsize, mi_row, mi_col, + mi_row_top, mi_col_top, dry_run, dst_buf, dst_stride, 3); + + predict_b_extend(cpi, td, tile, 0, mi_row, mi_col + hbs, mi_row, + mi_col + hbs, mi_row_top, mi_col_top, dst_buf1, + dst_stride1, top_bsize, bsize2, dry_run, 0, 0); + extend_all(cpi, td, tile, 0, bsize2, top_bsize, mi_row, mi_col + hbs, + mi_row_top, mi_col_top, dry_run, dst_buf1, dst_stride1); + + predict_b_extend(cpi, td, tile, 0, mi_row + hbs, mi_col + hbs, + mi_row + hbs, mi_col + hbs, mi_row_top, mi_col_top, + dst_buf2, dst_stride2, top_bsize, bsize2, dry_run, 0, 0); + extend_all(cpi, td, tile, 0, bsize2, top_bsize, mi_row + hbs, + mi_col + hbs, mi_row_top, mi_col_top, dry_run, dst_buf2, + dst_stride2); + + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf1[i]; + xd->plane[i].dst.stride = dst_stride1[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf1[i], dst_stride1[i], dst_buf2[i], dst_stride2[i], + mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize, + PARTITION_HORZ, i); + } + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst_buf[i]; + xd->plane[i].dst.stride = dst_stride[i]; + av1_build_masked_inter_predictor_complex( + xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i], mi_row, + mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_VERT, + i); + } + break; +#endif // CONFIG_EXT_PARTITION_TYPES + default: assert(0); + } + +#if CONFIG_EXT_PARTITION_TYPES + if (bsize < top_bsize) + update_ext_partition_context(xd, mi_row, mi_col, subsize, bsize, partition); +#else + if (bsize < top_bsize && (partition != PARTITION_SPLIT || bsize == BLOCK_8X8)) + update_partition_context(xd, mi_row, mi_col, subsize, bsize); +#endif // CONFIG_EXT_PARTITION_TYPES +} + +static void rd_supertx_sb(const AV1_COMP *const cpi, ThreadData *td, + const TileInfo *const tile, int mi_row, int mi_col, + BLOCK_SIZE bsize, int *tmp_rate, int64_t *tmp_dist, + TX_TYPE *best_tx, PC_TREE *pc_tree) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + int plane, pnskip, skippable, skippable_uv, rate_uv, this_rate, + base_rate = *tmp_rate; + int64_t sse, pnsse, sse_uv, this_dist, dist_uv; + uint8_t *dst_buf[3]; + int dst_stride[3]; + TX_SIZE tx_size; + MB_MODE_INFO *mbmi; + TX_TYPE tx_type, best_tx_nostx; +#if CONFIG_EXT_TX + int ext_tx_set; +#endif // CONFIG_EXT_TX + int tmp_rate_tx = 0, skip_tx = 0; + int64_t tmp_dist_tx = 0, rd_tx, bestrd_tx = INT64_MAX; + + set_skip_context(xd, mi_row, mi_col); + set_mode_info_offsets(cpi, x, xd, mi_row, mi_col); + update_state_sb_supertx(cpi, td, tile, mi_row, mi_col, bsize, 1, pc_tree); + av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, + mi_col); + for (plane = 0; plane < MAX_MB_PLANE; plane++) { + dst_buf[plane] = xd->plane[plane].dst.buf; + dst_stride[plane] = xd->plane[plane].dst.stride; + } + predict_sb_complex(cpi, td, tile, mi_row, mi_col, mi_row, mi_col, 1, bsize, + bsize, dst_buf, dst_stride, pc_tree); + + set_offsets_without_segment_id(cpi, tile, x, mi_row, mi_col, bsize); + set_segment_id_supertx(cpi, x, mi_row, mi_col, bsize); + + mbmi = &xd->mi[0]->mbmi; + best_tx_nostx = mbmi->tx_type; + + *best_tx = DCT_DCT; + + // chroma + skippable_uv = 1; + rate_uv = 0; + dist_uv = 0; + sse_uv = 0; + for (plane = 1; plane < MAX_MB_PLANE; ++plane) { +#if CONFIG_VAR_TX + ENTROPY_CONTEXT ctxa[2 * MAX_MIB_SIZE]; + ENTROPY_CONTEXT ctxl[2 * MAX_MIB_SIZE]; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + RD_STATS this_rd_stats; + av1_init_rd_stats(&this_rd_stats); + + tx_size = max_txsize_lookup[bsize]; + tx_size = + uv_txsize_lookup[bsize][tx_size][cm->subsampling_x][cm->subsampling_y]; + av1_get_entropy_contexts(bsize, tx_size, pd, ctxa, ctxl); + + av1_subtract_plane(x, bsize, plane); + av1_tx_block_rd_b(cpi, x, tx_size, 0, 0, plane, 0, + get_plane_block_size(bsize, pd), &ctxa[0], &ctxl[0], + &this_rd_stats); + + this_rate = this_rd_stats.rate; + this_dist = this_rd_stats.dist; + pnsse = this_rd_stats.sse; + pnskip = this_rd_stats.skip; +#else + tx_size = max_txsize_lookup[bsize]; + tx_size = + uv_txsize_lookup[bsize][tx_size][cm->subsampling_x][cm->subsampling_y]; + av1_subtract_plane(x, bsize, plane); + av1_txfm_rd_in_plane_supertx(x, cpi, &this_rate, &this_dist, &pnskip, + &pnsse, INT64_MAX, plane, bsize, tx_size, 0); +#endif // CONFIG_VAR_TX + + rate_uv += this_rate; + dist_uv += this_dist; + sse_uv += pnsse; + skippable_uv &= pnskip; + } + + // luma + tx_size = max_txsize_lookup[bsize]; + av1_subtract_plane(x, bsize, 0); +#if CONFIG_EXT_TX + ext_tx_set = get_ext_tx_set(tx_size, bsize, 1, cm->reduced_tx_set_used); +#endif // CONFIG_EXT_TX + for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) { +#if CONFIG_VAR_TX + ENTROPY_CONTEXT ctxa[2 * MAX_MIB_SIZE]; + ENTROPY_CONTEXT ctxl[2 * MAX_MIB_SIZE]; + const struct macroblockd_plane *const pd = &xd->plane[0]; + RD_STATS this_rd_stats; +#endif // CONFIG_VAR_TX + +#if CONFIG_EXT_TX + if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue; +#else + if (tx_size >= TX_32X32 && tx_type != DCT_DCT) continue; +#endif // CONFIG_EXT_TX + mbmi->tx_type = tx_type; + +#if CONFIG_VAR_TX + av1_init_rd_stats(&this_rd_stats); + av1_get_entropy_contexts(bsize, tx_size, pd, ctxa, ctxl); + av1_tx_block_rd_b(cpi, x, tx_size, 0, 0, 0, 0, bsize, &ctxa[0], &ctxl[0], + &this_rd_stats); + + this_rate = this_rd_stats.rate; + this_dist = this_rd_stats.dist; + pnsse = this_rd_stats.sse; + pnskip = this_rd_stats.skip; +#else + av1_txfm_rd_in_plane_supertx(x, cpi, &this_rate, &this_dist, &pnskip, + &pnsse, INT64_MAX, 0, bsize, tx_size, 0); +#endif // CONFIG_VAR_TX + +#if CONFIG_EXT_TX + if (get_ext_tx_types(tx_size, bsize, 1, cm->reduced_tx_set_used) > 1 && + !xd->lossless[xd->mi[0]->mbmi.segment_id] && this_rate != INT_MAX) { + if (ext_tx_set > 0) + this_rate += + cpi->inter_tx_type_costs[ext_tx_set][mbmi->tx_size][mbmi->tx_type]; + } +#else + if (tx_size < TX_32X32 && !xd->lossless[xd->mi[0]->mbmi.segment_id] && + this_rate != INT_MAX) { + this_rate += cpi->inter_tx_type_costs[tx_size][mbmi->tx_type]; + } +#endif // CONFIG_EXT_TX + *tmp_rate = rate_uv + this_rate; + *tmp_dist = dist_uv + this_dist; + sse = sse_uv + pnsse; + skippable = skippable_uv && pnskip; + if (skippable) { + *tmp_rate = av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + x->skip = 1; + } else { + if (RDCOST(x->rdmult, x->rddiv, *tmp_rate, *tmp_dist) < + RDCOST(x->rdmult, x->rddiv, 0, sse)) { + *tmp_rate += av1_cost_bit(av1_get_skip_prob(cm, xd), 0); + x->skip = 0; + } else { + *tmp_dist = sse; + *tmp_rate = av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + x->skip = 1; + } + } + *tmp_rate += base_rate; + rd_tx = RDCOST(x->rdmult, x->rddiv, *tmp_rate, *tmp_dist); + if (rd_tx < bestrd_tx * 0.99 || tx_type == DCT_DCT) { + *best_tx = tx_type; + bestrd_tx = rd_tx; + tmp_rate_tx = *tmp_rate; + tmp_dist_tx = *tmp_dist; + skip_tx = x->skip; + } + } + *tmp_rate = tmp_rate_tx; + *tmp_dist = tmp_dist_tx; + x->skip = skip_tx; +#if CONFIG_VAR_TX + for (plane = 0; plane < 1; ++plane) + memset(x->blk_skip[plane], x->skip, + sizeof(uint8_t) * pc_tree->none.num_4x4_blk); +#endif // CONFIG_VAR_TX + xd->mi[0]->mbmi.tx_type = best_tx_nostx; +} +#endif // CONFIG_SUPERTX diff --git a/third_party/aom/av1/encoder/encodeframe.h b/third_party/aom/av1/encoder/encodeframe.h new file mode 100644 index 0000000000..08d6d20dee --- /dev/null +++ b/third_party/aom/av1/encoder/encodeframe.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_ENCODEFRAME_H_ +#define AV1_ENCODER_ENCODEFRAME_H_ + +#include "aom/aom_integer.h" +#include "av1/common/blockd.h" +#include "av1/common/enums.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct macroblock; +struct yv12_buffer_config; +struct AV1_COMP; +struct ThreadData; + +// Constants used in SOURCE_VAR_BASED_PARTITION +#define VAR_HIST_MAX_BG_VAR 1000 +#define VAR_HIST_FACTOR 10 +#define VAR_HIST_BINS (VAR_HIST_MAX_BG_VAR / VAR_HIST_FACTOR + 1) +#define VAR_HIST_LARGE_CUT_OFF 75 +#define VAR_HIST_SMALL_CUT_OFF 45 + +void av1_setup_src_planes(struct macroblock *x, + const struct yv12_buffer_config *src, int mi_row, + int mi_col); + +void av1_encode_frame(struct AV1_COMP *cpi); + +void av1_init_tile_data(struct AV1_COMP *cpi); +void av1_encode_tile(struct AV1_COMP *cpi, struct ThreadData *td, int tile_row, + int tile_col); + +void av1_set_variance_partition_thresholds(struct AV1_COMP *cpi, int q); + +void av1_update_tx_type_count(const struct AV1Common *cm, MACROBLOCKD *xd, +#if CONFIG_TXK_SEL + int block, int plane, +#endif + BLOCK_SIZE bsize, TX_SIZE tx_size, + FRAME_COUNTS *counts); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_ENCODEFRAME_H_ diff --git a/third_party/aom/av1/encoder/encodemb.c b/third_party/aom/av1/encoder/encodemb.c new file mode 100644 index 0000000000..c450244b1c --- /dev/null +++ b/third_party/aom/av1/encoder/encodemb.c @@ -0,0 +1,1671 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./av1_rtcd.h" +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "aom_dsp/bitwriter.h" +#include "aom_dsp/quantize.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +#include "av1/common/idct.h" +#include "av1/common/reconinter.h" +#include "av1/common/reconintra.h" +#include "av1/common/scan.h" + +#include "av1/encoder/av1_quantize.h" +#include "av1/encoder/encodemb.h" +#if CONFIG_LV_MAP +#include "av1/encoder/encodetxb.h" +#endif +#include "av1/encoder/hybrid_fwd_txfm.h" +#include "av1/encoder/rd.h" +#include "av1/encoder/tokenize.h" + +#if CONFIG_PVQ +#include "av1/encoder/encint.h" +#include "av1/common/partition.h" +#include "av1/encoder/pvq_encoder.h" +#endif + +#if CONFIG_CFL +#include "av1/common/cfl.h" +#endif + +// Check if one needs to use c version subtraction. +static int check_subtract_block_size(int w, int h) { return w < 4 || h < 4; } + +static void subtract_block(const MACROBLOCKD *xd, int rows, int cols, + int16_t *diff, ptrdiff_t diff_stride, + const uint8_t *src8, ptrdiff_t src_stride, + const uint8_t *pred8, ptrdiff_t pred_stride) { +#if !CONFIG_HIGHBITDEPTH + (void)xd; +#endif + + if (check_subtract_block_size(rows, cols)) { +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + aom_highbd_subtract_block_c(rows, cols, diff, diff_stride, src8, + src_stride, pred8, pred_stride, xd->bd); + return; + } +#endif // CONFIG_HIGHBITDEPTH + aom_subtract_block_c(rows, cols, diff, diff_stride, src8, src_stride, pred8, + pred_stride); + + return; + } + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + aom_highbd_subtract_block(rows, cols, diff, diff_stride, src8, src_stride, + pred8, pred_stride, xd->bd); + return; + } +#endif // CONFIG_HIGHBITDEPTH + aom_subtract_block(rows, cols, diff, diff_stride, src8, src_stride, pred8, + pred_stride); +} + +void av1_subtract_txb(MACROBLOCK *x, int plane, BLOCK_SIZE plane_bsize, + int blk_col, int blk_row, TX_SIZE tx_size) { + MACROBLOCKD *const xd = &x->e_mbd; + struct macroblock_plane *const p = &x->plane[plane]; + const struct macroblockd_plane *const pd = &x->e_mbd.plane[plane]; + const int diff_stride = block_size_wide[plane_bsize]; + const int src_stride = p->src.stride; + const int dst_stride = pd->dst.stride; + const int tx1d_width = tx_size_wide[tx_size]; + const int tx1d_height = tx_size_high[tx_size]; + uint8_t *dst = + &pd->dst.buf[(blk_row * dst_stride + blk_col) << tx_size_wide_log2[0]]; + uint8_t *src = + &p->src.buf[(blk_row * src_stride + blk_col) << tx_size_wide_log2[0]]; + int16_t *src_diff = + &p->src_diff[(blk_row * diff_stride + blk_col) << tx_size_wide_log2[0]]; + subtract_block(xd, tx1d_height, tx1d_width, src_diff, diff_stride, src, + src_stride, dst, dst_stride); +} + +void av1_subtract_plane(MACROBLOCK *x, BLOCK_SIZE bsize, int plane) { + struct macroblock_plane *const p = &x->plane[plane]; + const struct macroblockd_plane *const pd = &x->e_mbd.plane[plane]; + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); + const int bw = block_size_wide[plane_bsize]; + const int bh = block_size_high[plane_bsize]; + const MACROBLOCKD *xd = &x->e_mbd; + + subtract_block(xd, bh, bw, p->src_diff, bw, p->src.buf, p->src.stride, + pd->dst.buf, pd->dst.stride); +} + +// These numbers are empirically obtained. +static const int plane_rd_mult[REF_TYPES][PLANE_TYPES] = { +#if CONFIG_EC_ADAPT + { 10, 7 }, { 8, 5 }, +#else + { 10, 6 }, { 8, 5 }, +#endif +}; + +#define UPDATE_RD_COST() \ + { \ + rd_cost0 = RDCOST(rdmult, rddiv, rate0, error0); \ + rd_cost1 = RDCOST(rdmult, rddiv, rate1, error1); \ + } + +static INLINE int64_t +get_token_bit_costs(unsigned int token_costs[2][COEFF_CONTEXTS][ENTROPY_TOKENS], + int skip_eob, int ctx, int token) { +#if CONFIG_NEW_TOKENSET + (void)skip_eob; + return token_costs[token == ZERO_TOKEN || token == EOB_TOKEN][ctx][token]; +#else + return token_costs[skip_eob][ctx][token]; +#endif +} + +#define USE_GREEDY_OPTIMIZE_B 0 + +#if USE_GREEDY_OPTIMIZE_B + +typedef struct av1_token_state { + int16_t token; + tran_low_t qc; + tran_low_t dqc; +} av1_token_state; + +int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block, + TX_SIZE tx_size, int ctx) { +#if !CONFIG_PVQ + MACROBLOCKD *const xd = &mb->e_mbd; + struct macroblock_plane *const p = &mb->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; + const int ref = is_inter_block(&xd->mi[0]->mbmi); + av1_token_state tokens[MAX_TX_SQUARE + 1][2]; + uint8_t token_cache[MAX_TX_SQUARE]; + const tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block); + tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block); + tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); + const int eob = p->eobs[block]; + const PLANE_TYPE plane_type = pd->plane_type; + const int16_t *const dequant_ptr = pd->dequant; + const uint8_t *const band_translate = get_band_translate(tx_size); + TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + const SCAN_ORDER *const scan_order = + get_scan(cm, tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi)); + const int16_t *const scan = scan_order->scan; + const int16_t *const nb = scan_order->neighbors; + int dqv; + const int shift = av1_get_tx_scale(tx_size); +#if CONFIG_AOM_QM + int seg_id = xd->mi[0]->mbmi.segment_id; + const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][!ref][tx_size]; +#endif +#if CONFIG_NEW_QUANT + int dq = get_dq_profile_from_ctx(mb->qindex, ctx, ref, plane_type); + const dequant_val_type_nuq *dequant_val = pd->dequant_val_nuq[dq]; +#elif !CONFIG_AOM_QM + const int dq_step[2] = { dequant_ptr[0] >> shift, dequant_ptr[1] >> shift }; +#endif // CONFIG_NEW_QUANT + int sz = 0; + const int64_t rddiv = mb->rddiv; + int64_t rd_cost0, rd_cost1; + int16_t t0, t1; + int i, final_eob; +#if CONFIG_HIGHBITDEPTH + const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, xd->bd); +#else + const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, 8); +#endif + unsigned int(*token_costs)[2][COEFF_CONTEXTS][ENTROPY_TOKENS] = + mb->token_costs[txsize_sqr_map[tx_size]][plane_type][ref]; + const int default_eob = tx_size_2d[tx_size]; + + assert((mb->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0)); + + assert((!plane_type && !plane) || (plane_type && plane)); + assert(eob <= default_eob); + + int64_t rdmult = (mb->rdmult * plane_rd_mult[ref][plane_type]) >> 1; +/* CpuSpeedTest uses "--min-q=0 --max-q=0" and expects 100dB psnr +* This creates conflict with search for a better EOB position +* The line below is to make sure EOB search is disabled at this corner case. +*/ +#if !CONFIG_NEW_QUANT && !CONFIG_AOM_QM + if (dq_step[1] <= 4) { + rdmult = 1; + } +#endif + + int64_t rate0, rate1; + for (i = 0; i < eob; i++) { + const int rc = scan[i]; + int x = qcoeff[rc]; + t0 = av1_get_token(x); + + tokens[i][0].qc = x; + tokens[i][0].token = t0; + tokens[i][0].dqc = dqcoeff[rc]; + + token_cache[rc] = av1_pt_energy_class[t0]; + } + tokens[eob][0].token = EOB_TOKEN; + tokens[eob][0].qc = 0; + tokens[eob][0].dqc = 0; + tokens[eob][1] = tokens[eob][0]; + + unsigned int(*token_costs_ptr)[2][COEFF_CONTEXTS][ENTROPY_TOKENS] = + token_costs; + + final_eob = 0; + + int64_t eob_cost0, eob_cost1; + + const int ctx0 = ctx; + /* Record the r-d cost */ + int64_t accu_rate = 0; + int64_t accu_error = 0; + + rate0 = get_token_bit_costs(*(token_costs_ptr + band_translate[0]), 0, ctx0, + EOB_TOKEN); + int64_t best_block_rd_cost = RDCOST(rdmult, rddiv, rate0, accu_error); + + // int64_t best_block_rd_cost_all0 = best_block_rd_cost; + + int x_prev = 1; + + for (i = 0; i < eob; i++) { + const int rc = scan[i]; + int x = qcoeff[rc]; + sz = -(x < 0); + + int band_cur = band_translate[i]; + int ctx_cur = (i == 0) ? ctx : get_coef_context(nb, token_cache, i); + int token_tree_sel_cur = (x_prev == 0); + + if (x == 0) { + // no need to search when x == 0 + rate0 = + get_token_bit_costs(*(token_costs_ptr + band_cur), token_tree_sel_cur, + ctx_cur, tokens[i][0].token); + accu_rate += rate0; + x_prev = 0; + // accu_error does not change when x==0 + } else { + /* Computing distortion + */ + // compute the distortion for the first candidate + // and the distortion for quantizing to 0. + int dx0 = (-coeff[rc]) * (1 << shift); +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + dx0 >>= xd->bd - 8; + } +#endif + int64_t d0 = (int64_t)dx0 * dx0; + + int x_a = x - 2 * sz - 1; + int64_t d2, d2_a; + + int dx; + +#if CONFIG_AOM_QM + int iwt = iqmatrix[rc]; + dqv = dequant_ptr[rc != 0]; + dqv = ((iwt * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS; +#else + dqv = dequant_ptr[rc != 0]; +#endif + + dx = (dqcoeff[rc] - coeff[rc]) * (1 << shift); +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + dx >>= xd->bd - 8; + } +#endif // CONFIG_HIGHBITDEPTH + d2 = (int64_t)dx * dx; + + /* compute the distortion for the second candidate + * x_a = x - 2 * sz + 1; + */ + if (x_a != 0) { +#if CONFIG_NEW_QUANT + dx = av1_dequant_coeff_nuq(x, dqv, dequant_val[band_translate[i]]) - + (coeff[rc] << shift); +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + dx >>= xd->bd - 8; + } +#endif // CONFIG_HIGHBITDEPTH +#else // CONFIG_NEW_QUANT +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + dx -= ((dqv >> (xd->bd - 8)) + sz) ^ sz; + } else { + dx -= (dqv + sz) ^ sz; + } +#else + dx -= (dqv + sz) ^ sz; +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_NEW_QUANT + d2_a = (int64_t)dx * dx; + } else { + d2_a = d0; + } + /* Computing rates and r-d cost + */ + + int best_x, best_eob_x; + int64_t base_bits, next_bits0, next_bits1; + int64_t next_eob_bits0, next_eob_bits1; + + // rate cost of x + base_bits = av1_get_token_cost(x, &t0, cat6_bits); + rate0 = base_bits + get_token_bit_costs(*(token_costs_ptr + band_cur), + token_tree_sel_cur, ctx_cur, t0); + + base_bits = av1_get_token_cost(x_a, &t1, cat6_bits); + rate1 = base_bits + get_token_bit_costs(*(token_costs_ptr + band_cur), + token_tree_sel_cur, ctx_cur, t1); + + next_bits0 = 0; + next_bits1 = 0; + next_eob_bits0 = 0; + next_eob_bits1 = 0; + + if (i < default_eob - 1) { + int ctx_next, token_tree_sel_next; + int band_next = band_translate[i + 1]; + + token_cache[rc] = av1_pt_energy_class[t0]; + ctx_next = get_coef_context(nb, token_cache, i + 1); + token_tree_sel_next = (x == 0); + + next_bits0 = get_token_bit_costs(*(token_costs_ptr + band_next), + token_tree_sel_next, ctx_next, + tokens[i + 1][0].token); + next_eob_bits0 = + get_token_bit_costs(*(token_costs_ptr + band_next), + token_tree_sel_next, ctx_next, EOB_TOKEN); + + token_cache[rc] = av1_pt_energy_class[t1]; + ctx_next = get_coef_context(nb, token_cache, i + 1); + token_tree_sel_next = (x_a == 0); + + next_bits1 = get_token_bit_costs(*(token_costs_ptr + band_next), + token_tree_sel_next, ctx_next, + tokens[i + 1][0].token); + + if (x_a != 0) { + next_eob_bits1 = + get_token_bit_costs(*(token_costs_ptr + band_next), + token_tree_sel_next, ctx_next, EOB_TOKEN); + } + } + + rd_cost0 = RDCOST(rdmult, rddiv, (rate0 + next_bits0), d2); + rd_cost1 = RDCOST(rdmult, rddiv, (rate1 + next_bits1), d2_a); + + best_x = (rd_cost1 < rd_cost0); + + eob_cost0 = RDCOST(rdmult, rddiv, (accu_rate + rate0 + next_eob_bits0), + (accu_error + d2 - d0)); + eob_cost1 = eob_cost0; + if (x_a != 0) { + eob_cost1 = RDCOST(rdmult, rddiv, (accu_rate + rate1 + next_eob_bits1), + (accu_error + d2_a - d0)); + best_eob_x = (eob_cost1 < eob_cost0); + } else { + best_eob_x = 0; + } + + int dqc, dqc_a = 0; + + dqc = dqcoeff[rc]; + if (best_x + best_eob_x) { + if (x_a != 0) { +#if CONFIG_NEW_QUANT + dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, + dequant_val[band_translate[i]]); + dqc_a = shift ? ROUND_POWER_OF_TWO(dqc_a, shift) : dqc_a; + if (sz) dqc_a = -dqc_a; +#else +// The 32x32 transform coefficient uses half quantization step size. +// Account for the rounding difference in the dequantized coefficeint +// value when the quantization index is dropped from an even number +// to an odd number. + +#if CONFIG_AOM_QM + tran_low_t offset = dqv >> shift; +#else + tran_low_t offset = dq_step[rc != 0]; +#endif + if (shift & x_a) offset += (dqv & 0x01); + + if (sz == 0) + dqc_a = dqcoeff[rc] - offset; + else + dqc_a = dqcoeff[rc] + offset; +#endif // CONFIG_NEW_QUANT + } else { + dqc_a = 0; + } // if (x_a != 0) + } + + // record the better quantized value + if (best_x) { + qcoeff[rc] = x_a; + dqcoeff[rc] = dqc_a; + + accu_rate += rate1; + accu_error += d2_a - d0; + assert(d2_a <= d0); + + token_cache[rc] = av1_pt_energy_class[t1]; + } else { + accu_rate += rate0; + accu_error += d2 - d0; + assert(d2 <= d0); + + token_cache[rc] = av1_pt_energy_class[t0]; + } + + x_prev = qcoeff[rc]; + + // determine whether to move the eob position to i+1 + int64_t best_eob_cost_i = eob_cost0; + + tokens[i][1].token = t0; + tokens[i][1].qc = x; + tokens[i][1].dqc = dqc; + + if ((x_a != 0) && (best_eob_x)) { + best_eob_cost_i = eob_cost1; + + tokens[i][1].token = t1; + tokens[i][1].qc = x_a; + tokens[i][1].dqc = dqc_a; + } + + if (best_eob_cost_i < best_block_rd_cost) { + best_block_rd_cost = best_eob_cost_i; + final_eob = i + 1; + } + } // if (x==0) + } // for (i) + + assert(final_eob <= eob); + if (final_eob > 0) { + assert(tokens[final_eob - 1][1].qc != 0); + i = final_eob - 1; + int rc = scan[i]; + qcoeff[rc] = tokens[i][1].qc; + dqcoeff[rc] = tokens[i][1].dqc; + } + + for (i = final_eob; i < eob; i++) { + int rc = scan[i]; + qcoeff[rc] = 0; + dqcoeff[rc] = 0; + } + + mb->plane[plane].eobs[block] = final_eob; + return final_eob; + +#else // !CONFIG_PVQ + (void)cm; + (void)tx_size; + (void)ctx; + struct macroblock_plane *const p = &mb->plane[plane]; + return p->eobs[block]; +#endif // !CONFIG_PVQ +} + +#else // USE_GREEDY_OPTIMIZE_B + +typedef struct av1_token_state { + int64_t error; + int rate; + int16_t next; + int16_t token; + tran_low_t qc; + tran_low_t dqc; + uint8_t best_index; +} av1_token_state; + +int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block, + TX_SIZE tx_size, int ctx) { +#if !CONFIG_PVQ + MACROBLOCKD *const xd = &mb->e_mbd; + struct macroblock_plane *const p = &mb->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; + const int ref = is_inter_block(&xd->mi[0]->mbmi); + av1_token_state tokens[MAX_TX_SQUARE + 1][2]; + uint8_t token_cache[MAX_TX_SQUARE]; + const tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block); + tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block); + tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); + const int eob = p->eobs[block]; + const PLANE_TYPE plane_type = pd->plane_type; + const int default_eob = tx_size_2d[tx_size]; + const int16_t *const dequant_ptr = pd->dequant; + const uint8_t *const band_translate = get_band_translate(tx_size); + TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + const SCAN_ORDER *const scan_order = + get_scan(cm, tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi)); + const int16_t *const scan = scan_order->scan; + const int16_t *const nb = scan_order->neighbors; + int dqv; + const int shift = av1_get_tx_scale(tx_size); +#if CONFIG_AOM_QM + int seg_id = xd->mi[0]->mbmi.segment_id; + const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][!ref][tx_size]; +#endif +#if CONFIG_NEW_QUANT + int dq = get_dq_profile_from_ctx(mb->qindex, ctx, ref, plane_type); + const dequant_val_type_nuq *dequant_val = pd->dequant_val_nuq[dq]; +#elif !CONFIG_AOM_QM + const int dq_step[2] = { dequant_ptr[0] >> shift, dequant_ptr[1] >> shift }; +#endif // CONFIG_NEW_QUANT + int next = eob, sz = 0; + const int64_t rdmult = (mb->rdmult * plane_rd_mult[ref][plane_type]) >> 1; + const int64_t rddiv = mb->rddiv; + int64_t rd_cost0, rd_cost1; + int rate0, rate1; + int64_t error0, error1; + int16_t t0, t1; + int best, band = (eob < default_eob) ? band_translate[eob] + : band_translate[eob - 1]; + int pt, i, final_eob; +#if CONFIG_HIGHBITDEPTH + const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, xd->bd); +#else + const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, 8); +#endif + unsigned int(*token_costs)[2][COEFF_CONTEXTS][ENTROPY_TOKENS] = + mb->token_costs[txsize_sqr_map[tx_size]][plane_type][ref]; + const uint16_t *band_counts = &band_count_table[tx_size][band]; + uint16_t band_left = eob - band_cum_count_table[tx_size][band] + 1; + int shortcut = 0; + int next_shortcut = 0; + +#if CONFIG_EXT_DELTA_Q + const int qindex = cm->seg.enabled + ? av1_get_qindex(&cm->seg, xd->mi[0]->mbmi.segment_id, + cm->base_qindex) + : cm->base_qindex; + if (qindex == 0) { + assert((qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0)); + } +#else + assert((mb->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0)); +#endif + + token_costs += band; + + assert((!plane_type && !plane) || (plane_type && plane)); + assert(eob <= default_eob); + + /* Now set up a Viterbi trellis to evaluate alternative roundings. */ + /* Initialize the sentinel node of the trellis. */ + tokens[eob][0].rate = 0; + tokens[eob][0].error = 0; + tokens[eob][0].next = default_eob; + tokens[eob][0].token = EOB_TOKEN; + tokens[eob][0].qc = 0; + tokens[eob][1] = tokens[eob][0]; + + for (i = 0; i < eob; i++) { + const int rc = scan[i]; + tokens[i][0].rate = av1_get_token_cost(qcoeff[rc], &t0, cat6_bits); + tokens[i][0].token = t0; + token_cache[rc] = av1_pt_energy_class[t0]; + } + + for (i = eob; i-- > 0;) { + int base_bits, dx; + int64_t d2; + const int rc = scan[i]; + int x = qcoeff[rc]; +#if CONFIG_AOM_QM + int iwt = iqmatrix[rc]; + dqv = dequant_ptr[rc != 0]; + dqv = ((iwt * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS; +#else + dqv = dequant_ptr[rc != 0]; +#endif + next_shortcut = shortcut; + + /* Only add a trellis state for non-zero coefficients. */ + if (UNLIKELY(x)) { + error0 = tokens[next][0].error; + error1 = tokens[next][1].error; + /* Evaluate the first possibility for this state. */ + rate0 = tokens[next][0].rate; + rate1 = tokens[next][1].rate; + + if (next_shortcut) { + /* Consider both possible successor states. */ + if (next < default_eob) { + pt = get_coef_context(nb, token_cache, i + 1); + rate0 += + get_token_bit_costs(*token_costs, 0, pt, tokens[next][0].token); + rate1 += + get_token_bit_costs(*token_costs, 0, pt, tokens[next][1].token); + } + UPDATE_RD_COST(); + /* And pick the best. */ + best = rd_cost1 < rd_cost0; + } else { + if (next < default_eob) { + pt = get_coef_context(nb, token_cache, i + 1); + rate0 += + get_token_bit_costs(*token_costs, 0, pt, tokens[next][0].token); + } + best = 0; + } + + dx = (dqcoeff[rc] - coeff[rc]) * (1 << shift); +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + dx >>= xd->bd - 8; + } +#endif // CONFIG_HIGHBITDEPTH + d2 = (int64_t)dx * dx; + tokens[i][0].rate += (best ? rate1 : rate0); + tokens[i][0].error = d2 + (best ? error1 : error0); + tokens[i][0].next = next; + tokens[i][0].qc = x; + tokens[i][0].dqc = dqcoeff[rc]; + tokens[i][0].best_index = best; + + /* Evaluate the second possibility for this state. */ + rate0 = tokens[next][0].rate; + rate1 = tokens[next][1].rate; + + // The threshold of 3 is empirically obtained. + if (UNLIKELY(abs(x) > 3)) { + shortcut = 0; + } else { +#if CONFIG_NEW_QUANT + shortcut = ((av1_dequant_abscoeff_nuq(abs(x), dqv, + dequant_val[band_translate[i]]) > + (abs(coeff[rc]) << shift)) && + (av1_dequant_abscoeff_nuq(abs(x) - 1, dqv, + dequant_val[band_translate[i]]) < + (abs(coeff[rc]) << shift))); +#else // CONFIG_NEW_QUANT +#if CONFIG_AOM_QM + if ((abs(x) * dequant_ptr[rc != 0] * iwt > + ((abs(coeff[rc]) << shift) << AOM_QM_BITS)) && + (abs(x) * dequant_ptr[rc != 0] * iwt < + (((abs(coeff[rc]) << shift) + dequant_ptr[rc != 0]) + << AOM_QM_BITS))) +#else + if ((abs(x) * dequant_ptr[rc != 0] > (abs(coeff[rc]) << shift)) && + (abs(x) * dequant_ptr[rc != 0] < + (abs(coeff[rc]) << shift) + dequant_ptr[rc != 0])) +#endif // CONFIG_AOM_QM + shortcut = 1; + else + shortcut = 0; +#endif // CONFIG_NEW_QUANT + } + + if (shortcut) { + sz = -(x < 0); + x -= 2 * sz + 1; + } else { + tokens[i][1] = tokens[i][0]; + next = i; + + if (UNLIKELY(!(--band_left))) { + --band_counts; + band_left = *band_counts; + --token_costs; + } + continue; + } + + /* Consider both possible successor states. */ + if (!x) { + /* If we reduced this coefficient to zero, check to see if + * we need to move the EOB back here. + */ + t0 = tokens[next][0].token == EOB_TOKEN ? EOB_TOKEN : ZERO_TOKEN; + t1 = tokens[next][1].token == EOB_TOKEN ? EOB_TOKEN : ZERO_TOKEN; + base_bits = 0; + } else { + base_bits = av1_get_token_cost(x, &t0, cat6_bits); + t1 = t0; + } + + if (next_shortcut) { + if (LIKELY(next < default_eob)) { + if (t0 != EOB_TOKEN) { + token_cache[rc] = av1_pt_energy_class[t0]; + pt = get_coef_context(nb, token_cache, i + 1); + rate0 += get_token_bit_costs(*token_costs, !x, pt, + tokens[next][0].token); + } + if (t1 != EOB_TOKEN) { + token_cache[rc] = av1_pt_energy_class[t1]; + pt = get_coef_context(nb, token_cache, i + 1); + rate1 += get_token_bit_costs(*token_costs, !x, pt, + tokens[next][1].token); + } + } + + UPDATE_RD_COST(); + /* And pick the best. */ + best = rd_cost1 < rd_cost0; + } else { + // The two states in next stage are identical. + if (next < default_eob && t0 != EOB_TOKEN) { + token_cache[rc] = av1_pt_energy_class[t0]; + pt = get_coef_context(nb, token_cache, i + 1); + rate0 += + get_token_bit_costs(*token_costs, !x, pt, tokens[next][0].token); + } + best = 0; + } + +#if CONFIG_NEW_QUANT + dx = av1_dequant_coeff_nuq(x, dqv, dequant_val[band_translate[i]]) - + (coeff[rc] << shift); +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + dx >>= xd->bd - 8; + } +#endif // CONFIG_HIGHBITDEPTH +#else // CONFIG_NEW_QUANT +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + dx -= ((dqv >> (xd->bd - 8)) + sz) ^ sz; + } else { + dx -= (dqv + sz) ^ sz; + } +#else + dx -= (dqv + sz) ^ sz; +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_NEW_QUANT + d2 = (int64_t)dx * dx; + + tokens[i][1].rate = base_bits + (best ? rate1 : rate0); + tokens[i][1].error = d2 + (best ? error1 : error0); + tokens[i][1].next = next; + tokens[i][1].token = best ? t1 : t0; + tokens[i][1].qc = x; + + if (x) { +#if CONFIG_NEW_QUANT + tokens[i][1].dqc = av1_dequant_abscoeff_nuq( + abs(x), dqv, dequant_val[band_translate[i]]); + tokens[i][1].dqc = shift ? ROUND_POWER_OF_TWO(tokens[i][1].dqc, shift) + : tokens[i][1].dqc; + if (sz) tokens[i][1].dqc = -tokens[i][1].dqc; +#else +// The 32x32 transform coefficient uses half quantization step size. +// Account for the rounding difference in the dequantized coefficeint +// value when the quantization index is dropped from an even number +// to an odd number. + +#if CONFIG_AOM_QM + tran_low_t offset = dqv >> shift; +#else + tran_low_t offset = dq_step[rc != 0]; +#endif + if (shift & x) offset += (dqv & 0x01); + + if (sz == 0) + tokens[i][1].dqc = dqcoeff[rc] - offset; + else + tokens[i][1].dqc = dqcoeff[rc] + offset; +#endif // CONFIG_NEW_QUANT + } else { + tokens[i][1].dqc = 0; + } + + tokens[i][1].best_index = best; + /* Finally, make this the new head of the trellis. */ + next = i; + } else { + /* There's no choice to make for a zero coefficient, so we don't + * add a new trellis node, but we do need to update the costs. + */ + t0 = tokens[next][0].token; + t1 = tokens[next][1].token; + pt = get_coef_context(nb, token_cache, i + 1); + /* Update the cost of each path if we're past the EOB token. */ + if (t0 != EOB_TOKEN) { + tokens[next][0].rate += get_token_bit_costs(*token_costs, 1, pt, t0); + tokens[next][0].token = ZERO_TOKEN; + } + if (t1 != EOB_TOKEN) { + tokens[next][1].rate += get_token_bit_costs(*token_costs, 1, pt, t1); + tokens[next][1].token = ZERO_TOKEN; + } + tokens[i][0].best_index = tokens[i][1].best_index = 0; + shortcut = (tokens[next][0].rate != tokens[next][1].rate); + /* Don't update next, because we didn't add a new node. */ + } + + if (UNLIKELY(!(--band_left))) { + --band_counts; + band_left = *band_counts; + --token_costs; + } + } + + /* Now pick the best path through the whole trellis. */ + rate0 = tokens[next][0].rate; + rate1 = tokens[next][1].rate; + error0 = tokens[next][0].error; + error1 = tokens[next][1].error; + t0 = tokens[next][0].token; + t1 = tokens[next][1].token; + rate0 += get_token_bit_costs(*token_costs, 0, ctx, t0); + rate1 += get_token_bit_costs(*token_costs, 0, ctx, t1); + UPDATE_RD_COST(); + best = rd_cost1 < rd_cost0; + + final_eob = -1; + + for (i = next; i < eob; i = next) { + const int x = tokens[i][best].qc; + const int rc = scan[i]; + if (x) final_eob = i; + qcoeff[rc] = x; + dqcoeff[rc] = tokens[i][best].dqc; + + next = tokens[i][best].next; + best = tokens[i][best].best_index; + } + final_eob++; + + mb->plane[plane].eobs[block] = final_eob; + assert(final_eob <= default_eob); + return final_eob; +#else // !CONFIG_PVQ + (void)cm; + (void)tx_size; + (void)ctx; + struct macroblock_plane *const p = &mb->plane[plane]; + return p->eobs[block]; +#endif // !CONFIG_PVQ +} + +#endif // USE_GREEDY_OPTIMIZE_B + +#if !CONFIG_PVQ +#if CONFIG_HIGHBITDEPTH +typedef enum QUANT_FUNC { + QUANT_FUNC_LOWBD = 0, + QUANT_FUNC_HIGHBD = 1, + QUANT_FUNC_TYPES = 2 +} QUANT_FUNC; + +static AV1_QUANT_FACADE + quant_func_list[AV1_XFORM_QUANT_TYPES][QUANT_FUNC_TYPES] = { +#if !CONFIG_NEW_QUANT + { av1_quantize_fp_facade, av1_highbd_quantize_fp_facade }, + { av1_quantize_b_facade, av1_highbd_quantize_b_facade }, + { av1_quantize_dc_facade, av1_highbd_quantize_dc_facade }, +#else // !CONFIG_NEW_QUANT + { av1_quantize_fp_nuq_facade, av1_highbd_quantize_fp_nuq_facade }, + { av1_quantize_b_nuq_facade, av1_highbd_quantize_b_nuq_facade }, + { av1_quantize_dc_nuq_facade, av1_highbd_quantize_dc_nuq_facade }, +#endif // !CONFIG_NEW_QUANT + { NULL, NULL } + }; + +#else + +typedef enum QUANT_FUNC { + QUANT_FUNC_LOWBD = 0, + QUANT_FUNC_TYPES = 1 +} QUANT_FUNC; + +static AV1_QUANT_FACADE quant_func_list[AV1_XFORM_QUANT_TYPES] + [QUANT_FUNC_TYPES] = { +#if !CONFIG_NEW_QUANT + { av1_quantize_fp_facade }, + { av1_quantize_b_facade }, + { av1_quantize_dc_facade }, +#else // !CONFIG_NEW_QUANT + { av1_quantize_fp_nuq_facade }, + { av1_quantize_b_nuq_facade }, + { av1_quantize_dc_nuq_facade }, +#endif // !CONFIG_NEW_QUANT + { NULL } + }; +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_PVQ + +void av1_xform_quant(const AV1_COMMON *cm, MACROBLOCK *x, int plane, int block, + int blk_row, int blk_col, BLOCK_SIZE plane_bsize, + TX_SIZE tx_size, int ctx, + AV1_XFORM_QUANT xform_quant_idx) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; +#if !(CONFIG_PVQ || CONFIG_DAALA_DIST) + const struct macroblock_plane *const p = &x->plane[plane]; + const struct macroblockd_plane *const pd = &xd->plane[plane]; +#else + struct macroblock_plane *const p = &x->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; +#endif + PLANE_TYPE plane_type = get_plane_type(plane); + TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + const int is_inter = is_inter_block(mbmi); + const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, is_inter); + tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block); + tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block); + tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); + uint16_t *const eob = &p->eobs[block]; + const int diff_stride = block_size_wide[plane_bsize]; +#if CONFIG_AOM_QM + int seg_id = mbmi->segment_id; + const qm_val_t *qmatrix = pd->seg_qmatrix[seg_id][!is_inter][tx_size]; + const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][!is_inter][tx_size]; +#endif + + FWD_TXFM_PARAM fwd_txfm_param; + +#if CONFIG_PVQ || CONFIG_DAALA_DIST + uint8_t *dst; + int16_t *pred; + const int dst_stride = pd->dst.stride; + int tx_blk_size; + int i, j; +#endif + +#if !CONFIG_PVQ + const int tx2d_size = tx_size_2d[tx_size]; + QUANT_PARAM qparam; + const int16_t *src_diff; + + src_diff = + &p->src_diff[(blk_row * diff_stride + blk_col) << tx_size_wide_log2[0]]; + qparam.log_scale = av1_get_tx_scale(tx_size); +#if CONFIG_NEW_QUANT + qparam.tx_size = tx_size; + qparam.dq = get_dq_profile_from_ctx(x->qindex, ctx, is_inter, plane_type); +#endif // CONFIG_NEW_QUANT +#if CONFIG_AOM_QM + qparam.qmatrix = qmatrix; + qparam.iqmatrix = iqmatrix; +#endif // CONFIG_AOM_QM +#else + tran_low_t *ref_coeff = BLOCK_OFFSET(pd->pvq_ref_coeff, block); + int skip = 1; + PVQ_INFO *pvq_info = NULL; + uint8_t *src; + int16_t *src_int16; + const int src_stride = p->src.stride; + + (void)ctx; + (void)scan_order; + (void)qcoeff; + + if (x->pvq_coded) { + assert(block < MAX_PVQ_BLOCKS_IN_SB); + pvq_info = &x->pvq[block][plane]; + } + src = &p->src.buf[(blk_row * src_stride + blk_col) << tx_size_wide_log2[0]]; + src_int16 = + &p->src_int16[(blk_row * diff_stride + blk_col) << tx_size_wide_log2[0]]; + + // transform block size in pixels + tx_blk_size = tx_size_wide[tx_size]; +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + for (j = 0; j < tx_blk_size; j++) + for (i = 0; i < tx_blk_size; i++) + src_int16[diff_stride * j + i] = + CONVERT_TO_SHORTPTR(src)[src_stride * j + i]; + } else { +#endif // CONFIG_HIGHBITDEPTH + for (j = 0; j < tx_blk_size; j++) + for (i = 0; i < tx_blk_size; i++) + src_int16[diff_stride * j + i] = src[src_stride * j + i]; +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH +#endif + +#if CONFIG_PVQ || CONFIG_DAALA_DIST + dst = &pd->dst.buf[(blk_row * dst_stride + blk_col) << tx_size_wide_log2[0]]; + pred = &pd->pred[(blk_row * diff_stride + blk_col) << tx_size_wide_log2[0]]; + + // transform block size in pixels + tx_blk_size = tx_size_wide[tx_size]; + +// copy uint8 orig and predicted block to int16 buffer +// in order to use existing VP10 transform functions +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + for (j = 0; j < tx_blk_size; j++) + for (i = 0; i < tx_blk_size; i++) + pred[diff_stride * j + i] = + CONVERT_TO_SHORTPTR(dst)[dst_stride * j + i]; + } else { +#endif // CONFIG_HIGHBITDEPTH + for (j = 0; j < tx_blk_size; j++) + for (i = 0; i < tx_blk_size; i++) + pred[diff_stride * j + i] = dst[dst_stride * j + i]; +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH +#endif + + (void)ctx; + + fwd_txfm_param.tx_type = tx_type; + fwd_txfm_param.tx_size = tx_size; + fwd_txfm_param.lossless = xd->lossless[mbmi->segment_id]; + +#if !CONFIG_PVQ +#if CONFIG_HIGHBITDEPTH + fwd_txfm_param.bd = xd->bd; + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + av1_highbd_fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param); + if (xform_quant_idx != AV1_XFORM_QUANT_SKIP_QUANT) { + if (LIKELY(!x->skip_block)) { + quant_func_list[xform_quant_idx][QUANT_FUNC_HIGHBD]( + coeff, tx2d_size, p, qcoeff, pd, dqcoeff, eob, scan_order, &qparam); + } else { + av1_quantize_skip(tx2d_size, qcoeff, dqcoeff, eob); + } + } +#if CONFIG_LV_MAP + p->txb_entropy_ctx[block] = + (uint8_t)av1_get_txb_entropy_context(qcoeff, scan_order, *eob); +#endif // CONFIG_LV_MAP + return; + } +#endif // CONFIG_HIGHBITDEPTH + av1_fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param); + if (xform_quant_idx != AV1_XFORM_QUANT_SKIP_QUANT) { + if (LIKELY(!x->skip_block)) { + quant_func_list[xform_quant_idx][QUANT_FUNC_LOWBD]( + coeff, tx2d_size, p, qcoeff, pd, dqcoeff, eob, scan_order, &qparam); + } else { + av1_quantize_skip(tx2d_size, qcoeff, dqcoeff, eob); + } + } +#if CONFIG_LV_MAP + p->txb_entropy_ctx[block] = + (uint8_t)av1_get_txb_entropy_context(qcoeff, scan_order, *eob); +#endif // CONFIG_LV_MAP +#else // #if !CONFIG_PVQ + (void)xform_quant_idx; +#if CONFIG_HIGHBITDEPTH + fwd_txfm_param.bd = xd->bd; + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + av1_highbd_fwd_txfm(src_int16, coeff, diff_stride, &fwd_txfm_param); + av1_highbd_fwd_txfm(pred, ref_coeff, diff_stride, &fwd_txfm_param); + } else { +#endif + av1_fwd_txfm(src_int16, coeff, diff_stride, &fwd_txfm_param); + av1_fwd_txfm(pred, ref_coeff, diff_stride, &fwd_txfm_param); +#if CONFIG_HIGHBITDEPTH + } +#endif + + // PVQ for inter mode block + if (!x->skip_block) { + PVQ_SKIP_TYPE ac_dc_coded = + av1_pvq_encode_helper(x, + coeff, // target original vector + ref_coeff, // reference vector + dqcoeff, // de-quantized vector + eob, // End of Block marker + pd->dequant, // aom's quantizers + plane, // image plane + tx_size, // block size in log_2 - 2 + tx_type, + &x->rate, // rate measured + x->pvq_speed, + pvq_info); // PVQ info for a block + skip = ac_dc_coded == PVQ_SKIP; + } + x->pvq_skip[plane] = skip; + + if (!skip) mbmi->skip = 0; +#endif // #if !CONFIG_PVQ +} + +static void encode_block(int plane, int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) { + struct encode_b_args *const args = arg; + AV1_COMMON *cm = args->cm; + MACROBLOCK *const x = args->x; + MACROBLOCKD *const xd = &x->e_mbd; + int ctx; + struct macroblock_plane *const p = &x->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; + tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); + uint8_t *dst; +#if !CONFIG_PVQ + ENTROPY_CONTEXT *a, *l; +#endif +#if CONFIG_VAR_TX + int bw = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; +#endif + dst = &pd->dst + .buf[(blk_row * pd->dst.stride + blk_col) << tx_size_wide_log2[0]]; + +#if !CONFIG_PVQ + a = &args->ta[blk_col]; + l = &args->tl[blk_row]; +#if CONFIG_VAR_TX + ctx = get_entropy_context(tx_size, a, l); +#else + ctx = combine_entropy_contexts(*a, *l); +#endif +#else + ctx = 0; +#endif // CONFIG_PVQ + +#if CONFIG_VAR_TX + // Assert not magic number (uninitialized). + assert(x->blk_skip[plane][blk_row * bw + blk_col] != 234); + + if (x->blk_skip[plane][blk_row * bw + blk_col] == 0) { +#else + { +#endif + av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, + ctx, AV1_XFORM_QUANT_FP); + } +#if CONFIG_VAR_TX + else { + p->eobs[block] = 0; + } +#endif + +#if !CONFIG_PVQ + if (p->eobs[block] && !xd->lossless[xd->mi[0]->mbmi.segment_id]) + av1_optimize_b(cm, x, plane, block, tx_size, ctx); + + av1_set_txb_context(x, plane, block, tx_size, a, l); + + if (p->eobs[block]) *(args->skip) = 0; + + if (p->eobs[block] == 0) return; +#else + (void)ctx; + if (!x->pvq_skip[plane]) *(args->skip) = 0; + + if (x->pvq_skip[plane]) return; +#endif + TX_TYPE tx_type = get_tx_type(pd->plane_type, xd, block, tx_size); + av1_inverse_transform_block(xd, dqcoeff, tx_type, tx_size, dst, + pd->dst.stride, p->eobs[block]); +} + +#if CONFIG_VAR_TX +static void encode_block_inter(int plane, int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, + void *arg) { + struct encode_b_args *const args = arg; + MACROBLOCK *const x = args->x; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const BLOCK_SIZE bsize = txsize_to_bsize[tx_size]; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const int tx_row = blk_row >> (1 - pd->subsampling_y); + const int tx_col = blk_col >> (1 - pd->subsampling_x); + TX_SIZE plane_tx_size; + const int max_blocks_high = max_block_high(xd, plane_bsize, plane); + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + + if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + + plane_tx_size = + plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0] + : mbmi->inter_tx_size[tx_row][tx_col]; + + if (tx_size == plane_tx_size) { + encode_block(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg); + } else { + const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; + // This is the square transform block partition entry point. + int bsl = tx_size_wide_unit[sub_txs]; + int i; + assert(bsl > 0); + assert(tx_size < TX_SIZES_ALL); + + for (i = 0; i < 4; ++i) { + const int offsetr = blk_row + ((i >> 1) * bsl); + const int offsetc = blk_col + ((i & 0x01) * bsl); + int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs]; + + if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue; + + encode_block_inter(plane, block, offsetr, offsetc, plane_bsize, sub_txs, + arg); + block += step; + } + } +} +#endif + +typedef struct encode_block_pass1_args { + AV1_COMMON *cm; + MACROBLOCK *x; +} encode_block_pass1_args; + +static void encode_block_pass1(int plane, int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, + void *arg) { + encode_block_pass1_args *args = (encode_block_pass1_args *)arg; + AV1_COMMON *cm = args->cm; + MACROBLOCK *const x = args->x; + MACROBLOCKD *const xd = &x->e_mbd; + struct macroblock_plane *const p = &x->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; + tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); + uint8_t *dst; + int ctx = 0; + dst = &pd->dst + .buf[(blk_row * pd->dst.stride + blk_col) << tx_size_wide_log2[0]]; + + av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, + ctx, AV1_XFORM_QUANT_B); +#if !CONFIG_PVQ + if (p->eobs[block] > 0) { +#else + if (!x->pvq_skip[plane]) { + { + int tx_blk_size; + int i, j; + // transform block size in pixels + tx_blk_size = tx_size_wide[tx_size]; + +// Since av1 does not have separate function which does inverse transform +// but av1_inv_txfm_add_*x*() also does addition of predicted image to +// inverse transformed image, +// pass blank dummy image to av1_inv_txfm_add_*x*(), i.e. set dst as zeros +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + for (j = 0; j < tx_blk_size; j++) + for (i = 0; i < tx_blk_size; i++) + CONVERT_TO_SHORTPTR(dst)[j * pd->dst.stride + i] = 0; + } else { +#endif // CONFIG_HIGHBITDEPTH + for (j = 0; j < tx_blk_size; j++) + for (i = 0; i < tx_blk_size; i++) dst[j * pd->dst.stride + i] = 0; +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + } +#endif // !CONFIG_PVQ +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (xd->lossless[xd->mi[0]->mbmi.segment_id]) { + av1_highbd_iwht4x4_add(dqcoeff, dst, pd->dst.stride, p->eobs[block], + xd->bd); + } else { + av1_highbd_idct4x4_add(dqcoeff, dst, pd->dst.stride, p->eobs[block], + xd->bd); + } + return; + } +#endif // CONFIG_HIGHBITDEPTH + if (xd->lossless[xd->mi[0]->mbmi.segment_id]) { + av1_iwht4x4_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]); + } else { + av1_idct4x4_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]); + } + } +} + +void av1_encode_sby_pass1(AV1_COMMON *cm, MACROBLOCK *x, BLOCK_SIZE bsize) { + encode_block_pass1_args args = { cm, x }; + av1_subtract_plane(x, bsize, 0); + av1_foreach_transformed_block_in_plane(&x->e_mbd, bsize, 0, + encode_block_pass1, &args); +} + +void av1_encode_sb(AV1_COMMON *cm, MACROBLOCK *x, BLOCK_SIZE bsize, + const int mi_row, const int mi_col) { + MACROBLOCKD *const xd = &x->e_mbd; + struct optimize_ctx ctx; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + struct encode_b_args arg = { cm, x, &ctx, &mbmi->skip, NULL, NULL, 1 }; + int plane; + + mbmi->skip = 1; + + if (x->skip) return; + + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { +#if CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + const int subsampling_x = xd->plane[plane].subsampling_x; + const int subsampling_y = xd->plane[plane].subsampling_y; + + if (!is_chroma_reference(mi_row, mi_col, bsize, subsampling_x, + subsampling_y)) + continue; + + bsize = scale_chroma_bsize(bsize, subsampling_x, subsampling_y); +#else + (void)mi_row; + (void)mi_col; +#endif + +#if CONFIG_VAR_TX + // TODO(jingning): Clean this up. + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); + const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; + const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0]; + const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize); + const BLOCK_SIZE txb_size = txsize_to_bsize[max_tx_size]; + const int bw = block_size_wide[txb_size] >> tx_size_wide_log2[0]; + const int bh = block_size_high[txb_size] >> tx_size_wide_log2[0]; + int idx, idy; + int block = 0; + int step = tx_size_wide_unit[max_tx_size] * tx_size_high_unit[max_tx_size]; + av1_get_entropy_contexts(bsize, 0, pd, ctx.ta[plane], ctx.tl[plane]); +#else + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const TX_SIZE tx_size = get_tx_size(plane, xd); + av1_get_entropy_contexts(bsize, tx_size, pd, ctx.ta[plane], ctx.tl[plane]); +#endif + +#if !CONFIG_PVQ + av1_subtract_plane(x, bsize, plane); +#endif + arg.ta = ctx.ta[plane]; + arg.tl = ctx.tl[plane]; + +#if CONFIG_VAR_TX + for (idy = 0; idy < mi_height; idy += bh) { + for (idx = 0; idx < mi_width; idx += bw) { + encode_block_inter(plane, block, idy, idx, plane_bsize, max_tx_size, + &arg); + block += step; + } + } +#else + av1_foreach_transformed_block_in_plane(xd, bsize, plane, encode_block, + &arg); +#endif + } +} + +#if CONFIG_SUPERTX +void av1_encode_sb_supertx(AV1_COMMON *cm, MACROBLOCK *x, BLOCK_SIZE bsize) { + MACROBLOCKD *const xd = &x->e_mbd; + struct optimize_ctx ctx; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + struct encode_b_args arg = { cm, x, &ctx, &mbmi->skip, NULL, NULL, 1 }; + int plane; + + mbmi->skip = 1; + if (x->skip) return; + + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + const struct macroblockd_plane *const pd = &xd->plane[plane]; +#if CONFIG_VAR_TX + const TX_SIZE tx_size = TX_4X4; +#else + const TX_SIZE tx_size = get_tx_size(plane, xd); +#endif + av1_subtract_plane(x, bsize, plane); + av1_get_entropy_contexts(bsize, tx_size, pd, ctx.ta[plane], ctx.tl[plane]); + arg.ta = ctx.ta[plane]; + arg.tl = ctx.tl[plane]; + av1_foreach_transformed_block_in_plane(xd, bsize, plane, encode_block, + &arg); + } +} +#endif // CONFIG_SUPERTX + +#if !CONFIG_PVQ +void av1_set_txb_context(MACROBLOCK *x, int plane, int block, TX_SIZE tx_size, + ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) { + (void)tx_size; + struct macroblock_plane *p = &x->plane[plane]; + +#if !CONFIG_LV_MAP + *a = *l = p->eobs[block] > 0; +#else // !CONFIG_LV_MAP + *a = *l = p->txb_entropy_ctx[block]; +#endif // !CONFIG_LV_MAP + +#if CONFIG_VAR_TX || CONFIG_LV_MAP + int i; + for (i = 0; i < tx_size_wide_unit[tx_size]; ++i) a[i] = a[0]; + + for (i = 0; i < tx_size_high_unit[tx_size]; ++i) l[i] = l[0]; +#endif +} +#endif + +static void encode_block_intra_and_set_context(int plane, int block, + int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, + TX_SIZE tx_size, void *arg) { + av1_encode_block_intra(plane, block, blk_row, blk_col, plane_bsize, tx_size, + arg); +#if !CONFIG_PVQ + struct encode_b_args *const args = arg; + MACROBLOCK *x = args->x; + ENTROPY_CONTEXT *a = &args->ta[blk_col]; + ENTROPY_CONTEXT *l = &args->tl[blk_row]; + av1_set_txb_context(x, plane, block, tx_size, a, l); +#endif +} + +void av1_encode_block_intra(int plane, int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, + void *arg) { + struct encode_b_args *const args = arg; + AV1_COMMON *cm = args->cm; + MACROBLOCK *const x = args->x; + MACROBLOCKD *const xd = &x->e_mbd; + struct macroblock_plane *const p = &x->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; + tran_low_t *dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); + PLANE_TYPE plane_type = get_plane_type(plane); + const TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + uint16_t *eob = &p->eobs[block]; + const int dst_stride = pd->dst.stride; + uint8_t *dst = + &pd->dst.buf[(blk_row * dst_stride + blk_col) << tx_size_wide_log2[0]]; + av1_predict_intra_block_facade(xd, plane, block, blk_col, blk_row, tx_size); + av1_subtract_txb(x, plane, plane_bsize, blk_col, blk_row, tx_size); + + const ENTROPY_CONTEXT *a = &args->ta[blk_col]; + const ENTROPY_CONTEXT *l = &args->tl[blk_row]; + int ctx = combine_entropy_contexts(*a, *l); + if (args->enable_optimize_b) { + av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, + ctx, AV1_XFORM_QUANT_FP); + if (p->eobs[block]) { + av1_optimize_b(cm, x, plane, block, tx_size, ctx); + } + } else { + av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, + ctx, AV1_XFORM_QUANT_B); + } + +#if CONFIG_PVQ + // *(args->skip) == mbmi->skip + if (!x->pvq_skip[plane]) *(args->skip) = 0; + + if (x->pvq_skip[plane]) return; +#endif // CONFIG_PVQ + av1_inverse_transform_block(xd, dqcoeff, tx_type, tx_size, dst, dst_stride, + *eob); +#if !CONFIG_PVQ + if (*eob) *(args->skip) = 0; +#else +// Note : *(args->skip) == mbmi->skip +#endif +#if CONFIG_CFL + if (plane == AOM_PLANE_Y && x->cfl_store_y) { + cfl_store(xd->cfl, dst, dst_stride, blk_row, blk_col, tx_size); + } +#endif +} + +void av1_encode_intra_block_plane(AV1_COMMON *cm, MACROBLOCK *x, + BLOCK_SIZE bsize, int plane, + int enable_optimize_b, const int mi_row, + const int mi_col) { + const MACROBLOCKD *const xd = &x->e_mbd; + ENTROPY_CONTEXT ta[2 * MAX_MIB_SIZE] = { 0 }; + ENTROPY_CONTEXT tl[2 * MAX_MIB_SIZE] = { 0 }; + + struct encode_b_args arg = { + cm, x, NULL, &xd->mi[0]->mbmi.skip, ta, tl, enable_optimize_b + }; + +#if CONFIG_CB4X4 + if (!is_chroma_reference(mi_row, mi_col, bsize, + xd->plane[plane].subsampling_x, + xd->plane[plane].subsampling_y)) + return; +#else + (void)mi_row; + (void)mi_col; +#endif + + if (enable_optimize_b) { + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const TX_SIZE tx_size = get_tx_size(plane, xd); + av1_get_entropy_contexts(bsize, tx_size, pd, ta, tl); + } + av1_foreach_transformed_block_in_plane( + xd, bsize, plane, encode_block_intra_and_set_context, &arg); +} + +#if CONFIG_PVQ +PVQ_SKIP_TYPE av1_pvq_encode_helper(MACROBLOCK *x, tran_low_t *const coeff, + tran_low_t *ref_coeff, + tran_low_t *const dqcoeff, uint16_t *eob, + const int16_t *quant, int plane, + int tx_size, TX_TYPE tx_type, int *rate, + int speed, PVQ_INFO *pvq_info) { + const int tx_blk_size = tx_size_wide[tx_size]; + daala_enc_ctx *daala_enc = &x->daala_enc; + PVQ_SKIP_TYPE ac_dc_coded; + int coeff_shift = 3 - av1_get_tx_scale(tx_size); + int hbd_downshift = 0; + int rounding_mask; + int pvq_dc_quant; + int use_activity_masking = daala_enc->use_activity_masking; + int tell; + int has_dc_skip = 1; + int i; + int off = od_qm_offset(tx_size, plane ? 1 : 0); + + DECLARE_ALIGNED(16, tran_low_t, coeff_pvq[OD_TXSIZE_MAX * OD_TXSIZE_MAX]); + DECLARE_ALIGNED(16, tran_low_t, ref_coeff_pvq[OD_TXSIZE_MAX * OD_TXSIZE_MAX]); + DECLARE_ALIGNED(16, tran_low_t, dqcoeff_pvq[OD_TXSIZE_MAX * OD_TXSIZE_MAX]); + + DECLARE_ALIGNED(16, int32_t, in_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX]); + DECLARE_ALIGNED(16, int32_t, ref_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX]); + DECLARE_ALIGNED(16, int32_t, out_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX]); + +#if CONFIG_HIGHBITDEPTH + hbd_downshift = x->e_mbd.bd - 8; +#endif + + assert(OD_COEFF_SHIFT >= 4); + // DC quantizer for PVQ + if (use_activity_masking) + pvq_dc_quant = + OD_MAXI(1, (quant[0] << (OD_COEFF_SHIFT - 3) >> hbd_downshift) * + daala_enc->state + .pvq_qm_q4[plane][od_qm_get_index(tx_size, 0)] >> + 4); + else + pvq_dc_quant = + OD_MAXI(1, quant[0] << (OD_COEFF_SHIFT - 3) >> hbd_downshift); + + *eob = 0; + +#if CONFIG_DAALA_EC + tell = od_ec_enc_tell_frac(&daala_enc->w.ec); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + + // Change coefficient ordering for pvq encoding. + od_raster_to_coding_order(coeff_pvq, tx_blk_size, tx_type, coeff, + tx_blk_size); + od_raster_to_coding_order(ref_coeff_pvq, tx_blk_size, tx_type, ref_coeff, + tx_blk_size); + + // copy int16 inputs to int32 + for (i = 0; i < tx_blk_size * tx_blk_size; i++) { + ref_int32[i] = + AOM_SIGNED_SHL(ref_coeff_pvq[i], OD_COEFF_SHIFT - coeff_shift) >> + hbd_downshift; + in_int32[i] = AOM_SIGNED_SHL(coeff_pvq[i], OD_COEFF_SHIFT - coeff_shift) >> + hbd_downshift; + } + + if (abs(in_int32[0] - ref_int32[0]) < pvq_dc_quant * 141 / 256) { /* 0.55 */ + out_int32[0] = 0; + } else { + out_int32[0] = OD_DIV_R0(in_int32[0] - ref_int32[0], pvq_dc_quant); + } + + ac_dc_coded = + od_pvq_encode(daala_enc, ref_int32, in_int32, out_int32, + OD_MAXI(1, quant[0] << (OD_COEFF_SHIFT - 3) >> + hbd_downshift), // scale/quantizer + OD_MAXI(1, quant[1] << (OD_COEFF_SHIFT - 3) >> + hbd_downshift), // scale/quantizer + plane, + tx_size, OD_PVQ_BETA[use_activity_masking][plane][tx_size], + 0, // is_keyframe, + daala_enc->state.qm + off, daala_enc->state.qm_inv + off, + speed, // speed + pvq_info); + + // Encode residue of DC coeff, if required. + if (!has_dc_skip || out_int32[0]) { + generic_encode(&daala_enc->w, &daala_enc->state.adapt->model_dc[plane], + abs(out_int32[0]) - has_dc_skip, + &daala_enc->state.adapt->ex_dc[plane][tx_size][0], 2); + } + if (out_int32[0]) { + aom_write_bit(&daala_enc->w, out_int32[0] < 0); + } + + // need to save quantized residue of DC coeff + // so that final pvq bitstream writing can know whether DC is coded. + if (pvq_info) pvq_info->dq_dc_residue = out_int32[0]; + + out_int32[0] = out_int32[0] * pvq_dc_quant; + out_int32[0] += ref_int32[0]; + + // copy int32 result back to int16 + assert(OD_COEFF_SHIFT > coeff_shift); + rounding_mask = (1 << (OD_COEFF_SHIFT - coeff_shift - 1)) - 1; + for (i = 0; i < tx_blk_size * tx_blk_size; i++) { + out_int32[i] = AOM_SIGNED_SHL(out_int32[i], hbd_downshift); + dqcoeff_pvq[i] = (out_int32[i] + (out_int32[i] < 0) + rounding_mask) >> + (OD_COEFF_SHIFT - coeff_shift); + } + + // Back to original coefficient order + od_coding_order_to_raster(dqcoeff, tx_blk_size, tx_type, dqcoeff_pvq, + tx_blk_size); + + *eob = tx_blk_size * tx_blk_size; + +#if CONFIG_DAALA_EC + *rate = (od_ec_enc_tell_frac(&daala_enc->w.ec) - tell) + << (AV1_PROB_COST_SHIFT - OD_BITRES); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + assert(*rate >= 0); + + return ac_dc_coded; +} + +void av1_store_pvq_enc_info(PVQ_INFO *pvq_info, int *qg, int *theta, int *k, + od_coeff *y, int nb_bands, const int *off, + int *size, int skip_rest, int skip_dir, + int bs) { // block size in log_2 -2 + int i; + const int tx_blk_size = tx_size_wide[bs]; + + for (i = 0; i < nb_bands; i++) { + pvq_info->qg[i] = qg[i]; + pvq_info->theta[i] = theta[i]; + pvq_info->k[i] = k[i]; + pvq_info->off[i] = off[i]; + pvq_info->size[i] = size[i]; + } + + memcpy(pvq_info->y, y, tx_blk_size * tx_blk_size * sizeof(od_coeff)); + + pvq_info->nb_bands = nb_bands; + pvq_info->skip_rest = skip_rest; + pvq_info->skip_dir = skip_dir; + pvq_info->bs = bs; +} +#endif diff --git a/third_party/aom/av1/encoder/encodemb.h b/third_party/aom/av1/encoder/encodemb.h new file mode 100644 index 0000000000..73fde1d884 --- /dev/null +++ b/third_party/aom/av1/encoder/encodemb.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_ENCODEMB_H_ +#define AV1_ENCODER_ENCODEMB_H_ + +#include "./aom_config.h" +#include "av1/common/onyxc_int.h" +#include "av1/encoder/block.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct optimize_ctx { + ENTROPY_CONTEXT ta[MAX_MB_PLANE][2 * MAX_MIB_SIZE]; + ENTROPY_CONTEXT tl[MAX_MB_PLANE][2 * MAX_MIB_SIZE]; +}; + +struct encode_b_args { + AV1_COMMON *cm; + MACROBLOCK *x; + struct optimize_ctx *ctx; + int8_t *skip; + ENTROPY_CONTEXT *ta; + ENTROPY_CONTEXT *tl; + int8_t enable_optimize_b; +}; + +typedef enum AV1_XFORM_QUANT { + AV1_XFORM_QUANT_FP = 0, + AV1_XFORM_QUANT_B = 1, + AV1_XFORM_QUANT_DC = 2, + AV1_XFORM_QUANT_SKIP_QUANT, + AV1_XFORM_QUANT_TYPES, +} AV1_XFORM_QUANT; + +void av1_encode_sb(AV1_COMMON *cm, MACROBLOCK *x, BLOCK_SIZE bsize, int mi_row, + int mi_col); +#if CONFIG_SUPERTX +void av1_encode_sb_supertx(AV1_COMMON *cm, MACROBLOCK *x, BLOCK_SIZE bsize); +#endif // CONFIG_SUPERTX +void av1_encode_sby_pass1(AV1_COMMON *cm, MACROBLOCK *x, BLOCK_SIZE bsize); +void av1_xform_quant(const AV1_COMMON *cm, MACROBLOCK *x, int plane, int block, + int blk_row, int blk_col, BLOCK_SIZE plane_bsize, + TX_SIZE tx_size, int ctx, AV1_XFORM_QUANT xform_quant_idx); + +int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block, + TX_SIZE tx_size, int ctx); + +void av1_subtract_txb(MACROBLOCK *x, int plane, BLOCK_SIZE plane_bsize, + int blk_col, int blk_row, TX_SIZE tx_size); + +void av1_subtract_plane(MACROBLOCK *x, BLOCK_SIZE bsize, int plane); + +void av1_set_txb_context(MACROBLOCK *x, int plane, int block, TX_SIZE tx_size, + ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l); + +void av1_encode_block_intra(int plane, int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg); + +void av1_encode_intra_block_plane(AV1_COMMON *cm, MACROBLOCK *x, + BLOCK_SIZE bsize, int plane, + int enable_optimize_b, int mi_row, + int mi_col); + +#if CONFIG_PVQ +PVQ_SKIP_TYPE av1_pvq_encode_helper(MACROBLOCK *x, tran_low_t *const coeff, + tran_low_t *ref_coeff, + tran_low_t *const dqcoeff, uint16_t *eob, + const int16_t *quant, int plane, + int tx_size, TX_TYPE tx_type, int *rate, + int speed, PVQ_INFO *pvq_info); + +void av1_store_pvq_enc_info(PVQ_INFO *pvq_info, int *qg, int *theta, int *k, + od_coeff *y, int nb_bands, const int *off, + int *size, int skip_rest, int skip_dir, int bs); +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_ENCODEMB_H_ diff --git a/third_party/aom/av1/encoder/encodemv.c b/third_party/aom/av1/encoder/encodemv.c new file mode 100644 index 0000000000..a2a53f8408 --- /dev/null +++ b/third_party/aom/av1/encoder/encodemv.c @@ -0,0 +1,497 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/common.h" +#include "av1/common/entropymode.h" + +#include "av1/encoder/cost.h" +#include "av1/encoder/encodemv.h" +#include "av1/encoder/subexp.h" + +#include "aom_dsp/aom_dsp_common.h" + +static struct av1_token mv_joint_encodings[MV_JOINTS]; +static struct av1_token mv_class_encodings[MV_CLASSES]; +static struct av1_token mv_fp_encodings[MV_FP_SIZE]; + +void av1_entropy_mv_init(void) { + av1_tokens_from_tree(mv_joint_encodings, av1_mv_joint_tree); + av1_tokens_from_tree(mv_class_encodings, av1_mv_class_tree); + av1_tokens_from_tree(mv_fp_encodings, av1_mv_fp_tree); +} + +static void encode_mv_component(aom_writer *w, int comp, nmv_component *mvcomp, + int usehp) { + int offset; + const int sign = comp < 0; + const int mag = sign ? -comp : comp; + const int mv_class = av1_get_mv_class(mag - 1, &offset); + const int d = offset >> 3; // int mv data + const int fr = (offset >> 1) & 3; // fractional mv data + const int hp = offset & 1; // high precision mv data + + assert(comp != 0); + + // Sign + aom_write(w, sign, mvcomp->sign); + +// Class +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, mv_class, mvcomp->class_cdf, MV_CLASSES); +#else + av1_write_token(w, av1_mv_class_tree, mvcomp->classes, + &mv_class_encodings[mv_class]); +#endif + + // Integer bits + if (mv_class == MV_CLASS_0) { + aom_write(w, d, mvcomp->class0[0]); + } else { + int i; + const int n = mv_class + CLASS0_BITS - 1; // number of bits + for (i = 0; i < n; ++i) aom_write(w, (d >> i) & 1, mvcomp->bits[i]); + } + +// Fractional bits +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol( + w, fr, mv_class == MV_CLASS_0 ? mvcomp->class0_fp_cdf[d] : mvcomp->fp_cdf, + MV_FP_SIZE); +#else + av1_write_token(w, av1_mv_fp_tree, + mv_class == MV_CLASS_0 ? mvcomp->class0_fp[d] : mvcomp->fp, + &mv_fp_encodings[fr]); +#endif + + // High precision bit + if (usehp) + aom_write(w, hp, mv_class == MV_CLASS_0 ? mvcomp->class0_hp : mvcomp->hp); +} + +static void build_nmv_component_cost_table(int *mvcost, + const nmv_component *const mvcomp, + int usehp) { + int i, v; + int sign_cost[2], class_cost[MV_CLASSES], class0_cost[CLASS0_SIZE]; + int bits_cost[MV_OFFSET_BITS][2]; + int class0_fp_cost[CLASS0_SIZE][MV_FP_SIZE], fp_cost[MV_FP_SIZE]; + int class0_hp_cost[2], hp_cost[2]; + + sign_cost[0] = av1_cost_zero(mvcomp->sign); + sign_cost[1] = av1_cost_one(mvcomp->sign); + av1_cost_tokens(class_cost, mvcomp->classes, av1_mv_class_tree); + av1_cost_tokens(class0_cost, mvcomp->class0, av1_mv_class0_tree); + for (i = 0; i < MV_OFFSET_BITS; ++i) { + bits_cost[i][0] = av1_cost_zero(mvcomp->bits[i]); + bits_cost[i][1] = av1_cost_one(mvcomp->bits[i]); + } + + for (i = 0; i < CLASS0_SIZE; ++i) + av1_cost_tokens(class0_fp_cost[i], mvcomp->class0_fp[i], av1_mv_fp_tree); + av1_cost_tokens(fp_cost, mvcomp->fp, av1_mv_fp_tree); + + if (usehp) { + class0_hp_cost[0] = av1_cost_zero(mvcomp->class0_hp); + class0_hp_cost[1] = av1_cost_one(mvcomp->class0_hp); + hp_cost[0] = av1_cost_zero(mvcomp->hp); + hp_cost[1] = av1_cost_one(mvcomp->hp); + } + mvcost[0] = 0; + for (v = 1; v <= MV_MAX; ++v) { + int z, c, o, d, e, f, cost = 0; + z = v - 1; + c = av1_get_mv_class(z, &o); + cost += class_cost[c]; + d = (o >> 3); /* int mv data */ + f = (o >> 1) & 3; /* fractional pel mv data */ + e = (o & 1); /* high precision mv data */ + if (c == MV_CLASS_0) { + cost += class0_cost[d]; + } else { + const int b = c + CLASS0_BITS - 1; /* number of bits */ + for (i = 0; i < b; ++i) cost += bits_cost[i][((d >> i) & 1)]; + } + if (c == MV_CLASS_0) { + cost += class0_fp_cost[d][f]; + } else { + cost += fp_cost[f]; + } + if (usehp) { + if (c == MV_CLASS_0) { + cost += class0_hp_cost[e]; + } else { + cost += hp_cost[e]; + } + } + mvcost[v] = cost + sign_cost[0]; + mvcost[-v] = cost + sign_cost[1]; + } +} + +static void update_mv(aom_writer *w, const unsigned int ct[2], aom_prob *cur_p, + aom_prob upd_p) { + (void)upd_p; +#if CONFIG_TILE_GROUPS + // Just use the default maximum number of tile groups to avoid passing in the + // actual + // number + av1_cond_prob_diff_update(w, cur_p, ct, DEFAULT_MAX_NUM_TG); +#else + av1_cond_prob_diff_update(w, cur_p, ct, 1); +#endif +} + +#if !CONFIG_EC_ADAPT +static void write_mv_update(const aom_tree_index *tree, + aom_prob probs[/*n - 1*/], + const unsigned int counts[/*n - 1*/], int n, + aom_writer *w) { + int i; + unsigned int branch_ct[32][2]; + + // Assuming max number of probabilities <= 32 + assert(n <= 32); + + av1_tree_probs_from_distribution(tree, branch_ct, counts); + for (i = 0; i < n - 1; ++i) + update_mv(w, branch_ct[i], &probs[i], MV_UPDATE_PROB); +} +#endif + +void av1_write_nmv_probs(AV1_COMMON *cm, int usehp, aom_writer *w, + nmv_context_counts *const nmv_counts) { + int i; +#if CONFIG_REF_MV + int nmv_ctx = 0; + for (nmv_ctx = 0; nmv_ctx < NMV_CONTEXTS; ++nmv_ctx) { + nmv_context *const mvc = &cm->fc->nmvc[nmv_ctx]; + nmv_context_counts *const counts = &nmv_counts[nmv_ctx]; +#if !CONFIG_EC_ADAPT + write_mv_update(av1_mv_joint_tree, mvc->joints, counts->joints, MV_JOINTS, + w); + + for (i = 0; i < 2; ++i) { + int j; + nmv_component *comp = &mvc->comps[i]; + nmv_component_counts *comp_counts = &counts->comps[i]; + + update_mv(w, comp_counts->sign, &comp->sign, MV_UPDATE_PROB); + write_mv_update(av1_mv_class_tree, comp->classes, comp_counts->classes, + MV_CLASSES, w); + write_mv_update(av1_mv_class0_tree, comp->class0, comp_counts->class0, + CLASS0_SIZE, w); + for (j = 0; j < MV_OFFSET_BITS; ++j) + update_mv(w, comp_counts->bits[j], &comp->bits[j], MV_UPDATE_PROB); + } + + for (i = 0; i < 2; ++i) { + int j; + for (j = 0; j < CLASS0_SIZE; ++j) + write_mv_update(av1_mv_fp_tree, mvc->comps[i].class0_fp[j], + counts->comps[i].class0_fp[j], MV_FP_SIZE, w); + + write_mv_update(av1_mv_fp_tree, mvc->comps[i].fp, counts->comps[i].fp, + MV_FP_SIZE, w); + } +#endif + + if (usehp) { + for (i = 0; i < 2; ++i) { + update_mv(w, counts->comps[i].class0_hp, &mvc->comps[i].class0_hp, + MV_UPDATE_PROB); + update_mv(w, counts->comps[i].hp, &mvc->comps[i].hp, MV_UPDATE_PROB); + } + } + } +#else + nmv_context *const mvc = &cm->fc->nmvc; + nmv_context_counts *const counts = nmv_counts; + +#if !CONFIG_EC_ADAPT + write_mv_update(av1_mv_joint_tree, mvc->joints, counts->joints, MV_JOINTS, w); + + for (i = 0; i < 2; ++i) { + int j; + nmv_component *comp = &mvc->comps[i]; + nmv_component_counts *comp_counts = &counts->comps[i]; + + update_mv(w, comp_counts->sign, &comp->sign, MV_UPDATE_PROB); + write_mv_update(av1_mv_class_tree, comp->classes, comp_counts->classes, + MV_CLASSES, w); + write_mv_update(av1_mv_class0_tree, comp->class0, comp_counts->class0, + CLASS0_SIZE, w); + for (j = 0; j < MV_OFFSET_BITS; ++j) + update_mv(w, comp_counts->bits[j], &comp->bits[j], MV_UPDATE_PROB); + } + + for (i = 0; i < 2; ++i) { + int j; + for (j = 0; j < CLASS0_SIZE; ++j) { + write_mv_update(av1_mv_fp_tree, mvc->comps[i].class0_fp[j], + counts->comps[i].class0_fp[j], MV_FP_SIZE, w); + } + write_mv_update(av1_mv_fp_tree, mvc->comps[i].fp, counts->comps[i].fp, + MV_FP_SIZE, w); + } +#endif // !CONFIG_EC_ADAPT + + if (usehp) { + for (i = 0; i < 2; ++i) { + update_mv(w, counts->comps[i].class0_hp, &mvc->comps[i].class0_hp, + MV_UPDATE_PROB); + update_mv(w, counts->comps[i].hp, &mvc->comps[i].hp, MV_UPDATE_PROB); + } + } +#endif +} + +void av1_encode_mv(AV1_COMP *cpi, aom_writer *w, const MV *mv, const MV *ref, + nmv_context *mvctx, int usehp) { + const MV diff = { mv->row - ref->row, mv->col - ref->col }; + const MV_JOINT_TYPE j = av1_get_mv_joint(&diff); +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, j, mvctx->joint_cdf, MV_JOINTS); +#else + av1_write_token(w, av1_mv_joint_tree, mvctx->joints, &mv_joint_encodings[j]); +#endif + if (mv_joint_vertical(j)) + encode_mv_component(w, diff.row, &mvctx->comps[0], usehp); + + if (mv_joint_horizontal(j)) + encode_mv_component(w, diff.col, &mvctx->comps[1], usehp); + + // If auto_mv_step_size is enabled then keep track of the largest + // motion vector component used. + if (cpi->sf.mv.auto_mv_step_size) { + unsigned int maxv = AOMMAX(abs(mv->row), abs(mv->col)) >> 3; + cpi->max_mv_magnitude = AOMMAX(maxv, cpi->max_mv_magnitude); + } +} + +#if CONFIG_INTRABC +void av1_encode_dv(aom_writer *w, const MV *mv, const MV *ref, + nmv_context *mvctx) { + const MV diff = { mv->row - ref->row, mv->col - ref->col }; + const MV_JOINT_TYPE j = av1_get_mv_joint(&diff); + +#if CONFIG_EC_MULTISYMBOL + aom_write_symbol(w, j, mvctx->joint_cdf, MV_JOINTS); +#else + av1_write_token(w, av1_mv_joint_tree, mvctx->joints, &mv_joint_encodings[j]); +#endif + if (mv_joint_vertical(j)) + encode_mv_component(w, diff.row, &mvctx->comps[0], 0); + + if (mv_joint_horizontal(j)) + encode_mv_component(w, diff.col, &mvctx->comps[1], 0); +} +#endif // CONFIG_INTRABC + +void av1_build_nmv_cost_table(int *mvjoint, int *mvcost[2], + const nmv_context *ctx, int usehp) { + av1_cost_tokens(mvjoint, ctx->joints, av1_mv_joint_tree); + build_nmv_component_cost_table(mvcost[0], &ctx->comps[0], usehp); + build_nmv_component_cost_table(mvcost[1], &ctx->comps[1], usehp); +} + +#if CONFIG_EXT_INTER +static void inc_mvs(const MB_MODE_INFO *mbmi, const MB_MODE_INFO_EXT *mbmi_ext, + const int_mv mvs[2], +#if CONFIG_REF_MV + const int_mv pred_mvs[2], +#endif + nmv_context_counts *nmv_counts) { + int i; + PREDICTION_MODE mode = mbmi->mode; +#if !CONFIG_REF_MV + nmv_context_counts *counts = nmv_counts; +#endif + + if (mode == NEWMV || mode == NEW_NEWMV) { + for (i = 0; i < 1 + has_second_ref(mbmi); ++i) { + const MV *ref = &mbmi_ext->ref_mvs[mbmi->ref_frame[i]][0].as_mv; + const MV diff = { mvs[i].as_mv.row - ref->row, + mvs[i].as_mv.col - ref->col }; +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = + av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], i, mbmi->ref_mv_idx); + nmv_context_counts *counts = &nmv_counts[nmv_ctx]; + (void)pred_mvs; +#endif + av1_inc_mv(&diff, counts, 1); + } + } else if (mode == NEAREST_NEWMV || mode == NEAR_NEWMV) { + const MV *ref = &mbmi_ext->ref_mvs[mbmi->ref_frame[1]][0].as_mv; + const MV diff = { mvs[1].as_mv.row - ref->row, + mvs[1].as_mv.col - ref->col }; +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = + av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], 1, mbmi->ref_mv_idx); + nmv_context_counts *counts = &nmv_counts[nmv_ctx]; +#endif + av1_inc_mv(&diff, counts, 1); + } else if (mode == NEW_NEARESTMV || mode == NEW_NEARMV) { + const MV *ref = &mbmi_ext->ref_mvs[mbmi->ref_frame[0]][0].as_mv; + const MV diff = { mvs[0].as_mv.row - ref->row, + mvs[0].as_mv.col - ref->col }; +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = + av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], 0, mbmi->ref_mv_idx); + nmv_context_counts *counts = &nmv_counts[nmv_ctx]; +#endif + av1_inc_mv(&diff, counts, 1); + } +} + +static void inc_mvs_sub8x8(const MODE_INFO *mi, int block, const int_mv mvs[2], +#if CONFIG_REF_MV + const MB_MODE_INFO_EXT *mbmi_ext, +#endif + nmv_context_counts *nmv_counts) { + int i; + PREDICTION_MODE mode = mi->bmi[block].as_mode; +#if CONFIG_REF_MV + const MB_MODE_INFO *mbmi = &mi->mbmi; +#else + nmv_context_counts *counts = nmv_counts; +#endif + + if (mode == NEWMV || mode == NEW_NEWMV) { + for (i = 0; i < 1 + has_second_ref(&mi->mbmi); ++i) { + const MV *ref = &mi->bmi[block].ref_mv[i].as_mv; + const MV diff = { mvs[i].as_mv.row - ref->row, + mvs[i].as_mv.col - ref->col }; +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = + av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], i, mbmi->ref_mv_idx); + nmv_context_counts *counts = &nmv_counts[nmv_ctx]; +#endif + av1_inc_mv(&diff, counts, 1); + } + } else if (mode == NEAREST_NEWMV || mode == NEAR_NEWMV) { + const MV *ref = &mi->bmi[block].ref_mv[1].as_mv; + const MV diff = { mvs[1].as_mv.row - ref->row, + mvs[1].as_mv.col - ref->col }; +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = + av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], 1, mbmi->ref_mv_idx); + nmv_context_counts *counts = &nmv_counts[nmv_ctx]; +#endif + av1_inc_mv(&diff, counts, 1); + } else if (mode == NEW_NEARESTMV || mode == NEW_NEARMV) { + const MV *ref = &mi->bmi[block].ref_mv[0].as_mv; + const MV diff = { mvs[0].as_mv.row - ref->row, + mvs[0].as_mv.col - ref->col }; +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = + av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], 0, mbmi->ref_mv_idx); + nmv_context_counts *counts = &nmv_counts[nmv_ctx]; +#endif + av1_inc_mv(&diff, counts, 1); + } +} +#else +static void inc_mvs(const MB_MODE_INFO *mbmi, const MB_MODE_INFO_EXT *mbmi_ext, + const int_mv mvs[2], +#if CONFIG_REF_MV + const int_mv pred_mvs[2], +#endif + nmv_context_counts *nmv_counts) { + int i; +#if !CONFIG_REF_MV + nmv_context_counts *counts = nmv_counts; +#endif + + for (i = 0; i < 1 + has_second_ref(mbmi); ++i) { +#if CONFIG_REF_MV + int8_t rf_type = av1_ref_frame_type(mbmi->ref_frame); + int nmv_ctx = + av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], i, mbmi->ref_mv_idx); + nmv_context_counts *counts = &nmv_counts[nmv_ctx]; + const MV *ref = &pred_mvs[i].as_mv; +#else + const MV *ref = &mbmi_ext->ref_mvs[mbmi->ref_frame[i]][0].as_mv; +#endif + const MV diff = { mvs[i].as_mv.row - ref->row, + mvs[i].as_mv.col - ref->col }; + av1_inc_mv(&diff, counts, 1); + } +} +#endif // CONFIG_EXT_INTER + +void av1_update_mv_count(ThreadData *td) { + const MACROBLOCKD *xd = &td->mb.e_mbd; + const MODE_INFO *mi = xd->mi[0]; + const MB_MODE_INFO *const mbmi = &mi->mbmi; + const MB_MODE_INFO_EXT *mbmi_ext = td->mb.mbmi_ext; +#if CONFIG_CB4X4 + const int unify_bsize = 1; +#else + const int unify_bsize = 0; +#endif + + if (mbmi->sb_type < BLOCK_8X8 && !unify_bsize) { + const int num_4x4_w = num_4x4_blocks_wide_lookup[mbmi->sb_type]; + const int num_4x4_h = num_4x4_blocks_high_lookup[mbmi->sb_type]; + int idx, idy; + + for (idy = 0; idy < 2; idy += num_4x4_h) { + for (idx = 0; idx < 2; idx += num_4x4_w) { + const int i = idy * 2 + idx; + +#if CONFIG_EXT_INTER + if (have_newmv_in_inter_mode(mi->bmi[i].as_mode)) + inc_mvs_sub8x8(mi, i, mi->bmi[i].as_mv, +#if CONFIG_REF_MV + mbmi_ext, td->counts->mv); +#else + &td->counts->mv); +#endif +#else + if (mi->bmi[i].as_mode == NEWMV) + inc_mvs(mbmi, mbmi_ext, mi->bmi[i].as_mv, +#if CONFIG_REF_MV + mi->bmi[i].pred_mv, td->counts->mv); +#else + &td->counts->mv); +#endif +#endif // CONFIG_EXT_INTER + } + } + } else { +#if CONFIG_EXT_INTER + if (have_newmv_in_inter_mode(mbmi->mode)) +#else + if (mbmi->mode == NEWMV) +#endif // CONFIG_EXT_INTER + inc_mvs(mbmi, mbmi_ext, mbmi->mv, +#if CONFIG_REF_MV + mbmi->pred_mv, td->counts->mv); +#else + &td->counts->mv); +#endif + } +} diff --git a/third_party/aom/av1/encoder/encodemv.h b/third_party/aom/av1/encoder/encodemv.h new file mode 100644 index 0000000000..6d442147fe --- /dev/null +++ b/third_party/aom/av1/encoder/encodemv.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_ENCODEMV_H_ +#define AV1_ENCODER_ENCODEMV_H_ + +#include "av1/encoder/encoder.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_entropy_mv_init(void); + +void av1_write_nmv_probs(AV1_COMMON *cm, int usehp, aom_writer *w, + nmv_context_counts *const counts); + +void av1_encode_mv(AV1_COMP *cpi, aom_writer *w, const MV *mv, const MV *ref, + nmv_context *mvctx, int usehp); + +void av1_build_nmv_cost_table(int *mvjoint, int *mvcost[2], + const nmv_context *mvctx, int usehp); + +void av1_update_mv_count(ThreadData *td); + +#if CONFIG_INTRABC +void av1_encode_dv(aom_writer *w, const MV *mv, const MV *ref, + nmv_context *mvctx); +#endif // CONFIG_INTRABC + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_ENCODEMV_H_ diff --git a/third_party/aom/av1/encoder/encoder.c b/third_party/aom/av1/encoder/encoder.c new file mode 100644 index 0000000000..0271091519 --- /dev/null +++ b/third_party/aom/av1/encoder/encoder.c @@ -0,0 +1,5980 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "./aom_config.h" + +#include "av1/common/alloccommon.h" +#if CONFIG_CDEF +#include "av1/common/cdef.h" +#include "av1/common/clpf.h" +#endif // CONFIG_CDEF +#include "av1/common/filter.h" +#include "av1/common/idct.h" +#include "av1/common/reconinter.h" +#include "av1/common/reconintra.h" +#include "av1/common/resize.h" +#include "av1/common/tile_common.h" + +#include "av1/encoder/aq_complexity.h" +#include "av1/encoder/aq_cyclicrefresh.h" +#include "av1/encoder/aq_variance.h" +#include "av1/encoder/bitstream.h" +#if CONFIG_ANS +#include "aom_dsp/buf_ans.h" +#endif +#include "av1/encoder/context_tree.h" +#include "av1/encoder/encodeframe.h" +#include "av1/encoder/encodemv.h" +#include "av1/encoder/encoder.h" +#if CONFIG_LV_MAP +#include "av1/encoder/encodetxb.h" +#endif +#include "av1/encoder/ethread.h" +#include "av1/encoder/firstpass.h" +#include "av1/encoder/mbgraph.h" +#include "av1/encoder/picklpf.h" +#if CONFIG_LOOP_RESTORATION +#include "av1/encoder/pickrst.h" +#endif // CONFIG_LOOP_RESTORATION +#include "av1/encoder/ratectrl.h" +#include "av1/encoder/rd.h" +#include "av1/encoder/segmentation.h" +#include "av1/encoder/speed_features.h" +#include "av1/encoder/temporal_filter.h" + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" +#include "./aom_scale_rtcd.h" +#include "aom_dsp/psnr.h" +#if CONFIG_INTERNAL_STATS +#include "aom_dsp/ssim.h" +#endif +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/aom_filter.h" +#include "aom_ports/aom_timer.h" +#include "aom_ports/mem.h" +#include "aom_ports/system_state.h" +#include "aom_scale/aom_scale.h" +#if CONFIG_BITSTREAM_DEBUG +#include "aom_util/debug_util.h" +#endif // CONFIG_BITSTREAM_DEBUG + +#if CONFIG_ENTROPY_STATS +FRAME_COUNTS aggregate_fc; +#endif // CONFIG_ENTROPY_STATS + +#define AM_SEGMENT_ID_INACTIVE 7 +#define AM_SEGMENT_ID_ACTIVE 0 + +#define SHARP_FILTER_QTHRESH 0 /* Q threshold for 8-tap sharp filter */ + +#define ALTREF_HIGH_PRECISION_MV 1 // Whether to use high precision mv + // for altref computation. +#define HIGH_PRECISION_MV_QTHRESH 200 // Q threshold for high precision + // mv. Choose a very high value for + // now so that HIGH_PRECISION is always + // chosen. +// #define OUTPUT_YUV_REC +#ifdef OUTPUT_YUV_DENOISED +FILE *yuv_denoised_file = NULL; +#endif +#ifdef OUTPUT_YUV_SKINMAP +FILE *yuv_skinmap_file = NULL; +#endif +#ifdef OUTPUT_YUV_REC +FILE *yuv_rec_file; +#define FILE_NAME_LEN 100 +#endif + +#if 0 +FILE *framepsnr; +FILE *kf_list; +FILE *keyfile; +#endif + +#if CONFIG_CFL +CFL_CTX NULL_CFL; +#endif + +#if CONFIG_INTERNAL_STATS +typedef enum { Y, U, V, ALL } STAT_TYPE; +#endif // CONFIG_INTERNAL_STATS + +static INLINE void Scale2Ratio(AOM_SCALING mode, int *hr, int *hs) { + switch (mode) { + case NORMAL: + *hr = 1; + *hs = 1; + break; + case FOURFIVE: + *hr = 4; + *hs = 5; + break; + case THREEFIVE: + *hr = 3; + *hs = 5; + break; + case ONETWO: + *hr = 1; + *hs = 2; + break; + default: + *hr = 1; + *hs = 1; + assert(0); + break; + } +} + +// Mark all inactive blocks as active. Other segmentation features may be set +// so memset cannot be used, instead only inactive blocks should be reset. +static void suppress_active_map(AV1_COMP *cpi) { + unsigned char *const seg_map = cpi->segmentation_map; + int i; + if (cpi->active_map.enabled || cpi->active_map.update) + for (i = 0; i < cpi->common.mi_rows * cpi->common.mi_cols; ++i) + if (seg_map[i] == AM_SEGMENT_ID_INACTIVE) + seg_map[i] = AM_SEGMENT_ID_ACTIVE; +} + +static void apply_active_map(AV1_COMP *cpi) { + struct segmentation *const seg = &cpi->common.seg; + unsigned char *const seg_map = cpi->segmentation_map; + const unsigned char *const active_map = cpi->active_map.map; + int i; + + assert(AM_SEGMENT_ID_ACTIVE == CR_SEGMENT_ID_BASE); + + if (frame_is_intra_only(&cpi->common)) { + cpi->active_map.enabled = 0; + cpi->active_map.update = 1; + } + + if (cpi->active_map.update) { + if (cpi->active_map.enabled) { + for (i = 0; i < cpi->common.mi_rows * cpi->common.mi_cols; ++i) + if (seg_map[i] == AM_SEGMENT_ID_ACTIVE) seg_map[i] = active_map[i]; + av1_enable_segmentation(seg); + av1_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_SKIP); + av1_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF); + // Setting the data to -MAX_LOOP_FILTER will result in the computed loop + // filter level being zero regardless of the value of seg->abs_delta. + av1_set_segdata(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF, + -MAX_LOOP_FILTER); + } else { + av1_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_SKIP); + av1_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF); + if (seg->enabled) { + seg->update_data = 1; + seg->update_map = 1; + } + } + cpi->active_map.update = 0; + } +} + +int av1_set_active_map(AV1_COMP *cpi, unsigned char *new_map_16x16, int rows, + int cols) { + if (rows == cpi->common.mb_rows && cols == cpi->common.mb_cols) { + unsigned char *const active_map_8x8 = cpi->active_map.map; + const int mi_rows = cpi->common.mi_rows; + const int mi_cols = cpi->common.mi_cols; + const int row_scale = mi_size_high[BLOCK_16X16] == 2 ? 1 : 2; + const int col_scale = mi_size_wide[BLOCK_16X16] == 2 ? 1 : 2; + cpi->active_map.update = 1; + if (new_map_16x16) { + int r, c; + for (r = 0; r < mi_rows; ++r) { + for (c = 0; c < mi_cols; ++c) { + active_map_8x8[r * mi_cols + c] = + new_map_16x16[(r >> row_scale) * cols + (c >> col_scale)] + ? AM_SEGMENT_ID_ACTIVE + : AM_SEGMENT_ID_INACTIVE; + } + } + cpi->active_map.enabled = 1; + } else { + cpi->active_map.enabled = 0; + } + return 0; + } else { + return -1; + } +} + +int av1_get_active_map(AV1_COMP *cpi, unsigned char *new_map_16x16, int rows, + int cols) { + if (rows == cpi->common.mb_rows && cols == cpi->common.mb_cols && + new_map_16x16) { + unsigned char *const seg_map_8x8 = cpi->segmentation_map; + const int mi_rows = cpi->common.mi_rows; + const int mi_cols = cpi->common.mi_cols; + const int row_scale = mi_size_high[BLOCK_16X16] == 2 ? 1 : 2; + const int col_scale = mi_size_wide[BLOCK_16X16] == 2 ? 1 : 2; + + memset(new_map_16x16, !cpi->active_map.enabled, rows * cols); + if (cpi->active_map.enabled) { + int r, c; + for (r = 0; r < mi_rows; ++r) { + for (c = 0; c < mi_cols; ++c) { + // Cyclic refresh segments are considered active despite not having + // AM_SEGMENT_ID_ACTIVE + new_map_16x16[(r >> row_scale) * cols + (c >> col_scale)] |= + seg_map_8x8[r * mi_cols + c] != AM_SEGMENT_ID_INACTIVE; + } + } + } + return 0; + } else { + return -1; + } +} + +void av1_set_high_precision_mv(AV1_COMP *cpi, int allow_high_precision_mv) { + MACROBLOCK *const mb = &cpi->td.mb; + cpi->common.allow_high_precision_mv = allow_high_precision_mv; + +#if CONFIG_REF_MV + if (cpi->common.allow_high_precision_mv) { + int i; + for (i = 0; i < NMV_CONTEXTS; ++i) { + mb->mv_cost_stack[i] = mb->nmvcost_hp[i]; + mb->mvsadcost = mb->nmvsadcost_hp; + } + } else { + int i; + for (i = 0; i < NMV_CONTEXTS; ++i) { + mb->mv_cost_stack[i] = mb->nmvcost[i]; + mb->mvsadcost = mb->nmvsadcost; + } + } +#else + if (cpi->common.allow_high_precision_mv) { + mb->mvcost = mb->nmvcost_hp; + mb->mvsadcost = mb->nmvcost_hp; + } else { + mb->mvcost = mb->nmvcost; + mb->mvsadcost = mb->nmvcost; + } +#endif +} + +static BLOCK_SIZE select_sb_size(const AV1_COMP *const cpi) { +#if CONFIG_EXT_PARTITION + if (cpi->oxcf.superblock_size == AOM_SUPERBLOCK_SIZE_64X64) + return BLOCK_64X64; + + if (cpi->oxcf.superblock_size == AOM_SUPERBLOCK_SIZE_128X128) + return BLOCK_128X128; + + assert(cpi->oxcf.superblock_size == AOM_SUPERBLOCK_SIZE_DYNAMIC); + + assert(IMPLIES(cpi->common.tile_cols > 1, + cpi->common.tile_width % MAX_MIB_SIZE == 0)); + assert(IMPLIES(cpi->common.tile_rows > 1, + cpi->common.tile_height % MAX_MIB_SIZE == 0)); + + // TODO(any): Possibly could improve this with a heuristic. + return BLOCK_128X128; +#else + (void)cpi; + return BLOCK_64X64; +#endif // CONFIG_EXT_PARTITION +} + +static void setup_frame(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + // Set up entropy context depending on frame type. The decoder mandates + // the use of the default context, index 0, for keyframes and inter + // frames where the error_resilient_mode or intra_only flag is set. For + // other inter-frames the encoder currently uses only two contexts; + // context 1 for ALTREF frames and context 0 for the others. + if (frame_is_intra_only(cm) || cm->error_resilient_mode) { + av1_setup_past_independence(cm); + } else { +#if CONFIG_EXT_REFS + const GF_GROUP *gf_group = &cpi->twopass.gf_group; + if (gf_group->rf_level[gf_group->index] == GF_ARF_LOW) + cm->frame_context_idx = EXT_ARF_FRAME; + else if (cpi->refresh_alt_ref_frame) + cm->frame_context_idx = ARF_FRAME; +#else + if (cpi->refresh_alt_ref_frame) cm->frame_context_idx = ARF_FRAME; +#endif // CONFIG_EXT_REFS + else if (cpi->rc.is_src_frame_alt_ref) + cm->frame_context_idx = OVERLAY_FRAME; + else if (cpi->refresh_golden_frame) + cm->frame_context_idx = GLD_FRAME; +#if CONFIG_EXT_REFS + else if (cpi->refresh_bwd_ref_frame) + cm->frame_context_idx = BRF_FRAME; +#endif // CONFIG_EXT_REFS + else + cm->frame_context_idx = REGULAR_FRAME; + } + + if (cm->frame_type == KEY_FRAME) { + cpi->refresh_golden_frame = 1; + cpi->refresh_alt_ref_frame = 1; + av1_zero(cpi->interp_filter_selected); + } else { + *cm->fc = cm->frame_contexts[cm->frame_context_idx]; + av1_zero(cpi->interp_filter_selected[0]); + } +#if CONFIG_EXT_REFS +#if CONFIG_LOWDELAY_COMPOUND // No change to bitstream + if (cpi->sf.recode_loop == DISALLOW_RECODE) { + cpi->refresh_bwd_ref_frame = cpi->refresh_last_frame; + cpi->rc.is_bipred_frame = 1; + } +#endif +#endif + + cpi->vaq_refresh = 0; + + set_sb_size(cm, select_sb_size(cpi)); +} + +static void av1_enc_setup_mi(AV1_COMMON *cm) { + int i; + cm->mi = cm->mip + cm->mi_stride + 1; + memset(cm->mip, 0, cm->mi_stride * (cm->mi_rows + 1) * sizeof(*cm->mip)); + cm->prev_mi = cm->prev_mip + cm->mi_stride + 1; + // Clear top border row + memset(cm->prev_mip, 0, sizeof(*cm->prev_mip) * cm->mi_stride); + // Clear left border column + for (i = 1; i < cm->mi_rows + 1; ++i) + memset(&cm->prev_mip[i * cm->mi_stride], 0, sizeof(*cm->prev_mip)); + + cm->mi_grid_visible = cm->mi_grid_base + cm->mi_stride + 1; + cm->prev_mi_grid_visible = cm->prev_mi_grid_base + cm->mi_stride + 1; + + memset(cm->mi_grid_base, 0, + cm->mi_stride * (cm->mi_rows + 1) * sizeof(*cm->mi_grid_base)); +} + +static int av1_enc_alloc_mi(AV1_COMMON *cm, int mi_size) { + cm->mip = aom_calloc(mi_size, sizeof(*cm->mip)); + if (!cm->mip) return 1; + cm->prev_mip = aom_calloc(mi_size, sizeof(*cm->prev_mip)); + if (!cm->prev_mip) return 1; + cm->mi_alloc_size = mi_size; + + cm->mi_grid_base = (MODE_INFO **)aom_calloc(mi_size, sizeof(MODE_INFO *)); + if (!cm->mi_grid_base) return 1; + cm->prev_mi_grid_base = + (MODE_INFO **)aom_calloc(mi_size, sizeof(MODE_INFO *)); + if (!cm->prev_mi_grid_base) return 1; + + return 0; +} + +static void av1_enc_free_mi(AV1_COMMON *cm) { + aom_free(cm->mip); + cm->mip = NULL; + aom_free(cm->prev_mip); + cm->prev_mip = NULL; + aom_free(cm->mi_grid_base); + cm->mi_grid_base = NULL; + aom_free(cm->prev_mi_grid_base); + cm->prev_mi_grid_base = NULL; +} + +static void av1_swap_mi_and_prev_mi(AV1_COMMON *cm) { + // Current mip will be the prev_mip for the next frame. + MODE_INFO **temp_base = cm->prev_mi_grid_base; + MODE_INFO *temp = cm->prev_mip; + cm->prev_mip = cm->mip; + cm->mip = temp; + + // Update the upper left visible macroblock ptrs. + cm->mi = cm->mip + cm->mi_stride + 1; + cm->prev_mi = cm->prev_mip + cm->mi_stride + 1; + + cm->prev_mi_grid_base = cm->mi_grid_base; + cm->mi_grid_base = temp_base; + cm->mi_grid_visible = cm->mi_grid_base + cm->mi_stride + 1; + cm->prev_mi_grid_visible = cm->prev_mi_grid_base + cm->mi_stride + 1; +} + +void av1_initialize_enc(void) { + static volatile int init_done = 0; + + if (!init_done) { + av1_rtcd(); + aom_dsp_rtcd(); + aom_scale_rtcd(); + av1_init_intra_predictors(); + av1_init_me_luts(); +#if !CONFIG_XIPHRC + av1_rc_init_minq_luts(); +#endif + av1_entropy_mv_init(); + av1_encode_token_init(); +#if CONFIG_EXT_INTER + av1_init_wedge_masks(); +#endif + init_done = 1; + } +} + +static void dealloc_compressor_data(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + int i; + + aom_free(cpi->mbmi_ext_base); + cpi->mbmi_ext_base = NULL; + +#if CONFIG_PVQ + if (cpi->oxcf.pass != 1) { + const int tile_cols = cm->tile_cols; + const int tile_rows = cm->tile_rows; + int tile_col, tile_row; + + for (tile_row = 0; tile_row < tile_rows; ++tile_row) + for (tile_col = 0; tile_col < tile_cols; ++tile_col) { + TileDataEnc *tile_data = + &cpi->tile_data[tile_row * tile_cols + tile_col]; + aom_free(tile_data->pvq_q.buf); + } + } +#endif + aom_free(cpi->tile_data); + cpi->tile_data = NULL; + + // Delete sementation map + aom_free(cpi->segmentation_map); + cpi->segmentation_map = NULL; + + av1_cyclic_refresh_free(cpi->cyclic_refresh); + cpi->cyclic_refresh = NULL; + + aom_free(cpi->active_map.map); + cpi->active_map.map = NULL; + + // Free up-sampled reference buffers. + for (i = 0; i < (REF_FRAMES + 1); i++) + aom_free_frame_buffer(&cpi->upsampled_ref_bufs[i].buf); + + av1_free_ref_frame_buffers(cm->buffer_pool); +#if CONFIG_LV_MAP + av1_free_txb_buf(cpi); +#endif + av1_free_context_buffers(cm); + + aom_free_frame_buffer(&cpi->last_frame_uf); +#if CONFIG_LOOP_RESTORATION + av1_free_restoration_buffers(cm); + aom_free_frame_buffer(&cpi->last_frame_db); + aom_free_frame_buffer(&cpi->trial_frame_rst); + aom_free(cpi->extra_rstbuf); + for (i = 0; i < MAX_MB_PLANE; ++i) + av1_free_restoration_struct(&cpi->rst_search[i]); +#endif // CONFIG_LOOP_RESTORATION + aom_free_frame_buffer(&cpi->scaled_source); + aom_free_frame_buffer(&cpi->scaled_last_source); + aom_free_frame_buffer(&cpi->alt_ref_buffer); + av1_lookahead_destroy(cpi->lookahead); + + aom_free(cpi->tile_tok[0][0]); + cpi->tile_tok[0][0] = 0; + + av1_free_pc_tree(&cpi->td); + av1_free_var_tree(&cpi->td); + +#if CONFIG_PALETTE + if (cpi->common.allow_screen_content_tools) + aom_free(cpi->td.mb.palette_buffer); +#endif // CONFIG_PALETTE + + if (cpi->source_diff_var != NULL) { + aom_free(cpi->source_diff_var); + cpi->source_diff_var = NULL; + } +#if CONFIG_ANS + aom_buf_ans_free(&cpi->buf_ans); +#endif // CONFIG_ANS +} + +static void save_coding_context(AV1_COMP *cpi) { + CODING_CONTEXT *const cc = &cpi->coding_context; + AV1_COMMON *cm = &cpi->common; +#if CONFIG_REF_MV + int i; +#endif + +// Stores a snapshot of key state variables which can subsequently be +// restored with a call to av1_restore_coding_context. These functions are +// intended for use in a re-code loop in av1_compress_frame where the +// quantizer value is adjusted between loop iterations. +#if CONFIG_REF_MV + for (i = 0; i < NMV_CONTEXTS; ++i) { + av1_copy(cc->nmv_vec_cost[i], cpi->td.mb.nmv_vec_cost[i]); + av1_copy(cc->nmv_costs, cpi->nmv_costs); + av1_copy(cc->nmv_costs_hp, cpi->nmv_costs_hp); + } +#else + av1_copy(cc->nmvjointcost, cpi->td.mb.nmvjointcost); +#endif + + av1_copy(cc->nmvcosts, cpi->nmvcosts); + av1_copy(cc->nmvcosts_hp, cpi->nmvcosts_hp); + + av1_copy(cc->last_ref_lf_deltas, cm->lf.last_ref_deltas); + av1_copy(cc->last_mode_lf_deltas, cm->lf.last_mode_deltas); + + cc->fc = *cm->fc; +} + +static void restore_coding_context(AV1_COMP *cpi) { + CODING_CONTEXT *const cc = &cpi->coding_context; + AV1_COMMON *cm = &cpi->common; +#if CONFIG_REF_MV + int i; +#endif + +// Restore key state variables to the snapshot state stored in the +// previous call to av1_save_coding_context. +#if CONFIG_REF_MV + for (i = 0; i < NMV_CONTEXTS; ++i) { + av1_copy(cpi->td.mb.nmv_vec_cost[i], cc->nmv_vec_cost[i]); + av1_copy(cpi->nmv_costs, cc->nmv_costs); + av1_copy(cpi->nmv_costs_hp, cc->nmv_costs_hp); + } +#else + av1_copy(cpi->td.mb.nmvjointcost, cc->nmvjointcost); +#endif + + av1_copy(cpi->nmvcosts, cc->nmvcosts); + av1_copy(cpi->nmvcosts_hp, cc->nmvcosts_hp); + + av1_copy(cm->lf.last_ref_deltas, cc->last_ref_lf_deltas); + av1_copy(cm->lf.last_mode_deltas, cc->last_mode_lf_deltas); + + *cm->fc = cc->fc; +} + +static void configure_static_seg_features(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + const RATE_CONTROL *const rc = &cpi->rc; + struct segmentation *const seg = &cm->seg; + + int high_q = (int)(rc->avg_q > 48.0); + int qi_delta; + + // Disable and clear down for KF + if (cm->frame_type == KEY_FRAME) { + // Clear down the global segmentation map + memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols); + seg->update_map = 0; + seg->update_data = 0; + cpi->static_mb_pct = 0; + + // Disable segmentation + av1_disable_segmentation(seg); + + // Clear down the segment features. + av1_clearall_segfeatures(seg); + } else if (cpi->refresh_alt_ref_frame) { + // If this is an alt ref frame + // Clear down the global segmentation map + memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols); + seg->update_map = 0; + seg->update_data = 0; + cpi->static_mb_pct = 0; + + // Disable segmentation and individual segment features by default + av1_disable_segmentation(seg); + av1_clearall_segfeatures(seg); + + // Scan frames from current to arf frame. + // This function re-enables segmentation if appropriate. + av1_update_mbgraph_stats(cpi); + + // If segmentation was enabled set those features needed for the + // arf itself. + if (seg->enabled) { + seg->update_map = 1; + seg->update_data = 1; + + qi_delta = + av1_compute_qdelta(rc, rc->avg_q, rc->avg_q * 0.875, cm->bit_depth); + av1_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta - 2); + av1_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2); + + av1_enable_segfeature(seg, 1, SEG_LVL_ALT_Q); + av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF); + + // Where relevant assume segment data is delta data + seg->abs_delta = SEGMENT_DELTADATA; + } + } else if (seg->enabled) { + // All other frames if segmentation has been enabled + + // First normal frame in a valid gf or alt ref group + if (rc->frames_since_golden == 0) { + // Set up segment features for normal frames in an arf group + if (rc->source_alt_ref_active) { + seg->update_map = 0; + seg->update_data = 1; + seg->abs_delta = SEGMENT_DELTADATA; + + qi_delta = + av1_compute_qdelta(rc, rc->avg_q, rc->avg_q * 1.125, cm->bit_depth); + av1_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta + 2); + av1_enable_segfeature(seg, 1, SEG_LVL_ALT_Q); + + av1_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2); + av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF); + + // Segment coding disabled for compred testing + if (high_q || (cpi->static_mb_pct == 100)) { + av1_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME); + av1_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME); + av1_enable_segfeature(seg, 1, SEG_LVL_SKIP); + } + } else { + // Disable segmentation and clear down features if alt ref + // is not active for this group + + av1_disable_segmentation(seg); + + memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols); + + seg->update_map = 0; + seg->update_data = 0; + + av1_clearall_segfeatures(seg); + } + } else if (rc->is_src_frame_alt_ref) { + // Special case where we are coding over the top of a previous + // alt ref frame. + // Segment coding disabled for compred testing + + // Enable ref frame features for segment 0 as well + av1_enable_segfeature(seg, 0, SEG_LVL_REF_FRAME); + av1_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME); + + // All mbs should use ALTREF_FRAME + av1_clear_segdata(seg, 0, SEG_LVL_REF_FRAME); + av1_set_segdata(seg, 0, SEG_LVL_REF_FRAME, ALTREF_FRAME); + av1_clear_segdata(seg, 1, SEG_LVL_REF_FRAME); + av1_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME); + + // Skip all MBs if high Q (0,0 mv and skip coeffs) + if (high_q) { + av1_enable_segfeature(seg, 0, SEG_LVL_SKIP); + av1_enable_segfeature(seg, 1, SEG_LVL_SKIP); + } + // Enable data update + seg->update_data = 1; + } else { + // All other frames. + + // No updates.. leave things as they are. + seg->update_map = 0; + seg->update_data = 0; + } + } +} + +static void update_reference_segmentation_map(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + MODE_INFO **mi_8x8_ptr = cm->mi_grid_visible; + uint8_t *cache_ptr = cm->last_frame_seg_map; + int row, col; + + for (row = 0; row < cm->mi_rows; row++) { + MODE_INFO **mi_8x8 = mi_8x8_ptr; + uint8_t *cache = cache_ptr; + for (col = 0; col < cm->mi_cols; col++, mi_8x8++, cache++) + cache[0] = mi_8x8[0]->mbmi.segment_id; + mi_8x8_ptr += cm->mi_stride; + cache_ptr += cm->mi_cols; + } +} + +static void alloc_raw_frame_buffers(AV1_COMP *cpi) { + AV1_COMMON *cm = &cpi->common; + const AV1EncoderConfig *oxcf = &cpi->oxcf; + + if (!cpi->lookahead) + cpi->lookahead = av1_lookahead_init(oxcf->width, oxcf->height, + cm->subsampling_x, cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + oxcf->lag_in_frames); + if (!cpi->lookahead) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate lag buffers"); + + // TODO(agrange) Check if ARF is enabled and skip allocation if not. + if (aom_realloc_frame_buffer(&cpi->alt_ref_buffer, oxcf->width, oxcf->height, + cm->subsampling_x, cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + AOM_BORDER_IN_PIXELS, cm->byte_alignment, NULL, + NULL, NULL)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate altref buffer"); +} + +static void alloc_util_frame_buffers(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + if (aom_realloc_frame_buffer(&cpi->last_frame_uf, cm->width, cm->height, + cm->subsampling_x, cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + AOM_BORDER_IN_PIXELS, cm->byte_alignment, NULL, + NULL, NULL)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate last frame buffer"); + +#if CONFIG_LOOP_RESTORATION + if (aom_realloc_frame_buffer(&cpi->last_frame_db, cm->width, cm->height, + cm->subsampling_x, cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + AOM_BORDER_IN_PIXELS, cm->byte_alignment, NULL, + NULL, NULL)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate last frame deblocked buffer"); + if (aom_realloc_frame_buffer(&cpi->trial_frame_rst, cm->width, cm->height, + cm->subsampling_x, cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + AOM_BORDER_IN_PIXELS, cm->byte_alignment, NULL, + NULL, NULL)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate trial restored frame buffer"); + int extra_rstbuf_sz = RESTORATION_EXTBUF_SIZE; + if (extra_rstbuf_sz > 0) { + aom_free(cpi->extra_rstbuf); + CHECK_MEM_ERROR(cm, cpi->extra_rstbuf, + (uint8_t *)aom_malloc(extra_rstbuf_sz)); + } else { + cpi->extra_rstbuf = NULL; + } +#endif // CONFIG_LOOP_RESTORATION + + if (aom_realloc_frame_buffer(&cpi->scaled_source, cm->width, cm->height, + cm->subsampling_x, cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + AOM_BORDER_IN_PIXELS, cm->byte_alignment, NULL, + NULL, NULL)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate scaled source buffer"); + + if (aom_realloc_frame_buffer(&cpi->scaled_last_source, cm->width, cm->height, + cm->subsampling_x, cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + AOM_BORDER_IN_PIXELS, cm->byte_alignment, NULL, + NULL, NULL)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate scaled last source buffer"); +} + +static int alloc_context_buffers_ext(AV1_COMP *cpi) { + AV1_COMMON *cm = &cpi->common; + int mi_size = cm->mi_cols * cm->mi_rows; + + cpi->mbmi_ext_base = aom_calloc(mi_size, sizeof(*cpi->mbmi_ext_base)); + if (!cpi->mbmi_ext_base) return 1; + + return 0; +} + +void av1_alloc_compressor_data(AV1_COMP *cpi) { + AV1_COMMON *cm = &cpi->common; + + av1_alloc_context_buffers(cm, cm->width, cm->height); + +#if CONFIG_LV_MAP + av1_alloc_txb_buf(cpi); +#endif + + alloc_context_buffers_ext(cpi); + + aom_free(cpi->tile_tok[0][0]); + + { + unsigned int tokens = get_token_alloc(cm->mb_rows, cm->mb_cols); + CHECK_MEM_ERROR(cm, cpi->tile_tok[0][0], + aom_calloc(tokens, sizeof(*cpi->tile_tok[0][0]))); +#if CONFIG_ANS && !ANS_MAX_SYMBOLS + aom_buf_ans_alloc(&cpi->buf_ans, &cm->error, (int)tokens); +#endif // CONFIG_ANS + } + + av1_setup_pc_tree(&cpi->common, &cpi->td); +} + +void av1_new_framerate(AV1_COMP *cpi, double framerate) { + cpi->framerate = framerate < 0.1 ? 30 : framerate; +#if CONFIG_XIPHRC + if (!cpi->od_rc.cur_frame) return; + cpi->od_rc.framerate = cpi->framerate; + od_enc_rc_resize(&cpi->od_rc); +#else + av1_rc_update_framerate(cpi); +#endif +} + +static void set_tile_info(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; +#if CONFIG_TILE_GROUPS && CONFIG_DEPENDENT_HORZTILES + int tile_row, tile_col, num_tiles_in_tg; + int tg_row_start, tg_col_start; +#endif +#if CONFIG_EXT_TILE +#if CONFIG_EXT_PARTITION + if (cpi->oxcf.superblock_size != AOM_SUPERBLOCK_SIZE_64X64) { + cm->tile_width = clamp(cpi->oxcf.tile_columns, 1, 32); + cm->tile_height = clamp(cpi->oxcf.tile_rows, 1, 32); + cm->tile_width <<= MAX_MIB_SIZE_LOG2; + cm->tile_height <<= MAX_MIB_SIZE_LOG2; + } else { + cm->tile_width = clamp(cpi->oxcf.tile_columns, 1, 64); + cm->tile_height = clamp(cpi->oxcf.tile_rows, 1, 64); + cm->tile_width <<= MAX_MIB_SIZE_LOG2 - 1; + cm->tile_height <<= MAX_MIB_SIZE_LOG2 - 1; + } +#else + cm->tile_width = clamp(cpi->oxcf.tile_columns, 1, 64); + cm->tile_height = clamp(cpi->oxcf.tile_rows, 1, 64); + cm->tile_width <<= MAX_MIB_SIZE_LOG2; + cm->tile_height <<= MAX_MIB_SIZE_LOG2; +#endif // CONFIG_EXT_PARTITION + + cm->tile_width = AOMMIN(cm->tile_width, cm->mi_cols); + cm->tile_height = AOMMIN(cm->tile_height, cm->mi_rows); + + assert(cm->tile_width >> MAX_MIB_SIZE <= 32); + assert(cm->tile_height >> MAX_MIB_SIZE <= 32); + + // Get the number of tiles + cm->tile_cols = 1; + while (cm->tile_cols * cm->tile_width < cm->mi_cols) ++cm->tile_cols; + + cm->tile_rows = 1; + while (cm->tile_rows * cm->tile_height < cm->mi_rows) ++cm->tile_rows; +#else + int min_log2_tile_cols, max_log2_tile_cols; + av1_get_tile_n_bits(cm->mi_cols, &min_log2_tile_cols, &max_log2_tile_cols); + + cm->log2_tile_cols = + clamp(cpi->oxcf.tile_columns, min_log2_tile_cols, max_log2_tile_cols); + cm->log2_tile_rows = cpi->oxcf.tile_rows; + + cm->tile_cols = 1 << cm->log2_tile_cols; + cm->tile_rows = 1 << cm->log2_tile_rows; + + cm->tile_width = ALIGN_POWER_OF_TWO(cm->mi_cols, MAX_MIB_SIZE_LOG2); + cm->tile_width >>= cm->log2_tile_cols; + cm->tile_height = ALIGN_POWER_OF_TWO(cm->mi_rows, MAX_MIB_SIZE_LOG2); + cm->tile_height >>= cm->log2_tile_rows; + + // round to integer multiples of max superblock size + cm->tile_width = ALIGN_POWER_OF_TWO(cm->tile_width, MAX_MIB_SIZE_LOG2); + cm->tile_height = ALIGN_POWER_OF_TWO(cm->tile_height, MAX_MIB_SIZE_LOG2); +#endif // CONFIG_EXT_TILE + +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles = cpi->oxcf.dependent_horz_tiles; + if (cm->log2_tile_rows == 0) cm->dependent_horz_tiles = 0; +#if CONFIG_TILE_GROUPS + if (cpi->oxcf.mtu == 0) { + cm->num_tg = cpi->oxcf.num_tile_groups; + } else { + // Use a default value for the purposes of weighting costs in probability + // updates + cm->num_tg = DEFAULT_MAX_NUM_TG; + } + num_tiles_in_tg = + (cm->tile_cols * cm->tile_rows + cm->num_tg - 1) / cm->num_tg; + tg_row_start = 0; + tg_col_start = 0; + for (tile_row = 0; tile_row < cm->tile_rows; ++tile_row) { + for (tile_col = 0; tile_col < cm->tile_cols; ++tile_col) { + if ((tile_row * cm->tile_cols + tile_col) % num_tiles_in_tg == 0) { + tg_row_start = tile_row; + tg_col_start = tile_col; + } + cm->tile_group_start_row[tile_row][tile_col] = tg_row_start; + cm->tile_group_start_col[tile_row][tile_col] = tg_col_start; + } + } +#endif +#endif + +#if CONFIG_LOOPFILTERING_ACROSS_TILES + cm->loop_filter_across_tiles_enabled = + cpi->oxcf.loop_filter_across_tiles_enabled; +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES +} + +static void update_frame_size(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; + + av1_set_mb_mi(cm, cm->width, cm->height); + av1_init_context_buffers(cm); + av1_init_macroblockd(cm, xd, +#if CONFIG_PVQ + NULL, +#endif +#if CONFIG_CFL + &NULL_CFL, +#endif + NULL); + memset(cpi->mbmi_ext_base, 0, + cm->mi_rows * cm->mi_cols * sizeof(*cpi->mbmi_ext_base)); + + set_tile_info(cpi); +} + +static void init_buffer_indices(AV1_COMP *cpi) { +#if CONFIG_EXT_REFS + int fb_idx; + for (fb_idx = 0; fb_idx < LAST_REF_FRAMES; ++fb_idx) + cpi->lst_fb_idxes[fb_idx] = fb_idx; + cpi->gld_fb_idx = LAST_REF_FRAMES; + cpi->bwd_fb_idx = LAST_REF_FRAMES + 1; + cpi->alt_fb_idx = LAST_REF_FRAMES + 2; + for (fb_idx = 0; fb_idx < MAX_EXT_ARFS + 1; ++fb_idx) + cpi->arf_map[fb_idx] = LAST_REF_FRAMES + 2 + fb_idx; +#else + cpi->lst_fb_idx = 0; + cpi->gld_fb_idx = 1; + cpi->alt_fb_idx = 2; +#endif // CONFIG_EXT_REFS +} + +static void init_config(struct AV1_COMP *cpi, AV1EncoderConfig *oxcf) { + AV1_COMMON *const cm = &cpi->common; + + cpi->oxcf = *oxcf; + cpi->framerate = oxcf->init_framerate; + + cm->profile = oxcf->profile; + cm->bit_depth = oxcf->bit_depth; +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth = oxcf->use_highbitdepth; +#endif + cm->color_space = oxcf->color_space; + cm->color_range = oxcf->color_range; + + cm->width = oxcf->width; + cm->height = oxcf->height; + av1_alloc_compressor_data(cpi); + + // Single thread case: use counts in common. + cpi->td.counts = &cm->counts; + + // change includes all joint functionality + av1_change_config(cpi, oxcf); + + cpi->static_mb_pct = 0; + cpi->ref_frame_flags = 0; + + init_buffer_indices(cpi); +} + +static void set_rc_buffer_sizes(RATE_CONTROL *rc, + const AV1EncoderConfig *oxcf) { + const int64_t bandwidth = oxcf->target_bandwidth; + const int64_t starting = oxcf->starting_buffer_level_ms; + const int64_t optimal = oxcf->optimal_buffer_level_ms; + const int64_t maximum = oxcf->maximum_buffer_size_ms; + + rc->starting_buffer_level = starting * bandwidth / 1000; + rc->optimal_buffer_level = + (optimal == 0) ? bandwidth / 8 : optimal * bandwidth / 1000; + rc->maximum_buffer_size = + (maximum == 0) ? bandwidth / 8 : maximum * bandwidth / 1000; +} + +#if CONFIG_HIGHBITDEPTH +#define HIGHBD_BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX3F, SDX8F, SDX4DF) \ + cpi->fn_ptr[BT].sdf = SDF; \ + cpi->fn_ptr[BT].sdaf = SDAF; \ + cpi->fn_ptr[BT].vf = VF; \ + cpi->fn_ptr[BT].svf = SVF; \ + cpi->fn_ptr[BT].svaf = SVAF; \ + cpi->fn_ptr[BT].sdx3f = SDX3F; \ + cpi->fn_ptr[BT].sdx8f = SDX8F; \ + cpi->fn_ptr[BT].sdx4df = SDX4DF; + +#define MAKE_BFP_SAD_WRAPPER(fnname) \ + static unsigned int fnname##_bits8(const uint8_t *src_ptr, \ + int source_stride, \ + const uint8_t *ref_ptr, int ref_stride) { \ + return fnname(src_ptr, source_stride, ref_ptr, ref_stride); \ + } \ + static unsigned int fnname##_bits10( \ + const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, \ + int ref_stride) { \ + return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 2; \ + } \ + static unsigned int fnname##_bits12( \ + const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, \ + int ref_stride) { \ + return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 4; \ + } + +#define MAKE_BFP_SADAVG_WRAPPER(fnname) \ + static unsigned int fnname##_bits8( \ + const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, \ + int ref_stride, const uint8_t *second_pred) { \ + return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred); \ + } \ + static unsigned int fnname##_bits10( \ + const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, \ + int ref_stride, const uint8_t *second_pred) { \ + return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred) >> \ + 2; \ + } \ + static unsigned int fnname##_bits12( \ + const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, \ + int ref_stride, const uint8_t *second_pred) { \ + return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred) >> \ + 4; \ + } + +#define MAKE_BFP_SAD3_WRAPPER(fnname) \ + static void fnname##_bits8(const uint8_t *src_ptr, int source_stride, \ + const uint8_t *ref_ptr, int ref_stride, \ + unsigned int *sad_array) { \ + fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \ + } \ + static void fnname##_bits10(const uint8_t *src_ptr, int source_stride, \ + const uint8_t *ref_ptr, int ref_stride, \ + unsigned int *sad_array) { \ + int i; \ + fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \ + for (i = 0; i < 3; i++) sad_array[i] >>= 2; \ + } \ + static void fnname##_bits12(const uint8_t *src_ptr, int source_stride, \ + const uint8_t *ref_ptr, int ref_stride, \ + unsigned int *sad_array) { \ + int i; \ + fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \ + for (i = 0; i < 3; i++) sad_array[i] >>= 4; \ + } + +#define MAKE_BFP_SAD8_WRAPPER(fnname) \ + static void fnname##_bits8(const uint8_t *src_ptr, int source_stride, \ + const uint8_t *ref_ptr, int ref_stride, \ + unsigned int *sad_array) { \ + fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \ + } \ + static void fnname##_bits10(const uint8_t *src_ptr, int source_stride, \ + const uint8_t *ref_ptr, int ref_stride, \ + unsigned int *sad_array) { \ + int i; \ + fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \ + for (i = 0; i < 8; i++) sad_array[i] >>= 2; \ + } \ + static void fnname##_bits12(const uint8_t *src_ptr, int source_stride, \ + const uint8_t *ref_ptr, int ref_stride, \ + unsigned int *sad_array) { \ + int i; \ + fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \ + for (i = 0; i < 8; i++) sad_array[i] >>= 4; \ + } +#define MAKE_BFP_SAD4D_WRAPPER(fnname) \ + static void fnname##_bits8(const uint8_t *src_ptr, int source_stride, \ + const uint8_t *const ref_ptr[], int ref_stride, \ + unsigned int *sad_array) { \ + fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \ + } \ + static void fnname##_bits10(const uint8_t *src_ptr, int source_stride, \ + const uint8_t *const ref_ptr[], int ref_stride, \ + unsigned int *sad_array) { \ + int i; \ + fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \ + for (i = 0; i < 4; i++) sad_array[i] >>= 2; \ + } \ + static void fnname##_bits12(const uint8_t *src_ptr, int source_stride, \ + const uint8_t *const ref_ptr[], int ref_stride, \ + unsigned int *sad_array) { \ + int i; \ + fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \ + for (i = 0; i < 4; i++) sad_array[i] >>= 4; \ + } + +#if CONFIG_EXT_PARTITION +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad128x128) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad128x128_avg) +MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad128x128x3) +MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad128x128x8) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad128x128x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad128x64) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad128x64_avg) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad128x64x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad64x128) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad64x128_avg) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad64x128x4d) +#endif // CONFIG_EXT_PARTITION +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad32x16) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad32x16_avg) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad32x16x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad16x32) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad16x32_avg) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad16x32x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad64x32) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad64x32_avg) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad64x32x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad32x64) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad32x64_avg) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad32x64x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad32x32) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad32x32_avg) +MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad32x32x3) +MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad32x32x8) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad32x32x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad64x64) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad64x64_avg) +MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad64x64x3) +MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad64x64x8) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad64x64x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad16x16) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad16x16_avg) +MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad16x16x3) +MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad16x16x8) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad16x16x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad16x8) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad16x8_avg) +MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad16x8x3) +MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad16x8x8) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad16x8x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad8x16) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad8x16_avg) +MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad8x16x3) +MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad8x16x8) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad8x16x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad8x8) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad8x8_avg) +MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad8x8x3) +MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad8x8x8) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad8x8x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad8x4) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad8x4_avg) +MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad8x4x8) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad8x4x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad4x8) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad4x8_avg) +MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad4x8x8) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad4x8x4d) +MAKE_BFP_SAD_WRAPPER(aom_highbd_sad4x4) +MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad4x4_avg) +MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad4x4x3) +MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad4x4x8) +MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad4x4x4d) + +#if CONFIG_EXT_INTER +#define HIGHBD_MBFP(BT, MSDF, MVF, MSVF) \ + cpi->fn_ptr[BT].msdf = MSDF; \ + cpi->fn_ptr[BT].mvf = MVF; \ + cpi->fn_ptr[BT].msvf = MSVF; + +#define MAKE_MBFP_SAD_WRAPPER(fnname) \ + static unsigned int fnname##_bits8( \ + const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, \ + int ref_stride, const uint8_t *m, int m_stride) { \ + return fnname(src_ptr, source_stride, ref_ptr, ref_stride, m, m_stride); \ + } \ + static unsigned int fnname##_bits10( \ + const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, \ + int ref_stride, const uint8_t *m, int m_stride) { \ + return fnname(src_ptr, source_stride, ref_ptr, ref_stride, m, m_stride) >> \ + 2; \ + } \ + static unsigned int fnname##_bits12( \ + const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, \ + int ref_stride, const uint8_t *m, int m_stride) { \ + return fnname(src_ptr, source_stride, ref_ptr, ref_stride, m, m_stride) >> \ + 4; \ + } + +#if CONFIG_EXT_PARTITION +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad128x128) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad128x64) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad64x128) +#endif // CONFIG_EXT_PARTITION +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad64x64) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad64x32) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad32x64) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad32x32) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad32x16) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad16x32) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad16x16) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad16x8) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad8x16) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad8x8) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad8x4) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad4x8) +MAKE_MBFP_SAD_WRAPPER(aom_highbd_masked_sad4x4) +#endif // CONFIG_EXT_INTER + +#if CONFIG_MOTION_VAR +#define HIGHBD_OBFP(BT, OSDF, OVF, OSVF) \ + cpi->fn_ptr[BT].osdf = OSDF; \ + cpi->fn_ptr[BT].ovf = OVF; \ + cpi->fn_ptr[BT].osvf = OSVF; + +#define MAKE_OBFP_SAD_WRAPPER(fnname) \ + static unsigned int fnname##_bits8(const uint8_t *ref, int ref_stride, \ + const int32_t *wsrc, \ + const int32_t *msk) { \ + return fnname(ref, ref_stride, wsrc, msk); \ + } \ + static unsigned int fnname##_bits10(const uint8_t *ref, int ref_stride, \ + const int32_t *wsrc, \ + const int32_t *msk) { \ + return fnname(ref, ref_stride, wsrc, msk) >> 2; \ + } \ + static unsigned int fnname##_bits12(const uint8_t *ref, int ref_stride, \ + const int32_t *wsrc, \ + const int32_t *msk) { \ + return fnname(ref, ref_stride, wsrc, msk) >> 4; \ + } + +#if CONFIG_EXT_PARTITION +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad128x128) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad128x64) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad64x128) +#endif // CONFIG_EXT_PARTITION +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad64x64) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad64x32) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad32x64) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad32x32) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad32x16) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad16x32) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad16x16) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad16x8) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad8x16) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad8x8) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad8x4) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad4x8) +MAKE_OBFP_SAD_WRAPPER(aom_highbd_obmc_sad4x4) +#endif // CONFIG_MOTION_VAR + +static void highbd_set_var_fns(AV1_COMP *const cpi) { + AV1_COMMON *const cm = &cpi->common; + if (cm->use_highbitdepth) { + switch (cm->bit_depth) { + case AOM_BITS_8: + HIGHBD_BFP(BLOCK_32X16, aom_highbd_sad32x16_bits8, + aom_highbd_sad32x16_avg_bits8, aom_highbd_8_variance32x16, + aom_highbd_8_sub_pixel_variance32x16, + aom_highbd_8_sub_pixel_avg_variance32x16, NULL, NULL, + aom_highbd_sad32x16x4d_bits8) + + HIGHBD_BFP(BLOCK_16X32, aom_highbd_sad16x32_bits8, + aom_highbd_sad16x32_avg_bits8, aom_highbd_8_variance16x32, + aom_highbd_8_sub_pixel_variance16x32, + aom_highbd_8_sub_pixel_avg_variance16x32, NULL, NULL, + aom_highbd_sad16x32x4d_bits8) + + HIGHBD_BFP(BLOCK_64X32, aom_highbd_sad64x32_bits8, + aom_highbd_sad64x32_avg_bits8, aom_highbd_8_variance64x32, + aom_highbd_8_sub_pixel_variance64x32, + aom_highbd_8_sub_pixel_avg_variance64x32, NULL, NULL, + aom_highbd_sad64x32x4d_bits8) + + HIGHBD_BFP(BLOCK_32X64, aom_highbd_sad32x64_bits8, + aom_highbd_sad32x64_avg_bits8, aom_highbd_8_variance32x64, + aom_highbd_8_sub_pixel_variance32x64, + aom_highbd_8_sub_pixel_avg_variance32x64, NULL, NULL, + aom_highbd_sad32x64x4d_bits8) + + HIGHBD_BFP(BLOCK_32X32, aom_highbd_sad32x32_bits8, + aom_highbd_sad32x32_avg_bits8, aom_highbd_8_variance32x32, + aom_highbd_8_sub_pixel_variance32x32, + aom_highbd_8_sub_pixel_avg_variance32x32, + aom_highbd_sad32x32x3_bits8, aom_highbd_sad32x32x8_bits8, + aom_highbd_sad32x32x4d_bits8) + + HIGHBD_BFP(BLOCK_64X64, aom_highbd_sad64x64_bits8, + aom_highbd_sad64x64_avg_bits8, aom_highbd_8_variance64x64, + aom_highbd_8_sub_pixel_variance64x64, + aom_highbd_8_sub_pixel_avg_variance64x64, + aom_highbd_sad64x64x3_bits8, aom_highbd_sad64x64x8_bits8, + aom_highbd_sad64x64x4d_bits8) + + HIGHBD_BFP(BLOCK_16X16, aom_highbd_sad16x16_bits8, + aom_highbd_sad16x16_avg_bits8, aom_highbd_8_variance16x16, + aom_highbd_8_sub_pixel_variance16x16, + aom_highbd_8_sub_pixel_avg_variance16x16, + aom_highbd_sad16x16x3_bits8, aom_highbd_sad16x16x8_bits8, + aom_highbd_sad16x16x4d_bits8) + + HIGHBD_BFP( + BLOCK_16X8, aom_highbd_sad16x8_bits8, aom_highbd_sad16x8_avg_bits8, + aom_highbd_8_variance16x8, aom_highbd_8_sub_pixel_variance16x8, + aom_highbd_8_sub_pixel_avg_variance16x8, aom_highbd_sad16x8x3_bits8, + aom_highbd_sad16x8x8_bits8, aom_highbd_sad16x8x4d_bits8) + + HIGHBD_BFP( + BLOCK_8X16, aom_highbd_sad8x16_bits8, aom_highbd_sad8x16_avg_bits8, + aom_highbd_8_variance8x16, aom_highbd_8_sub_pixel_variance8x16, + aom_highbd_8_sub_pixel_avg_variance8x16, aom_highbd_sad8x16x3_bits8, + aom_highbd_sad8x16x8_bits8, aom_highbd_sad8x16x4d_bits8) + + HIGHBD_BFP( + BLOCK_8X8, aom_highbd_sad8x8_bits8, aom_highbd_sad8x8_avg_bits8, + aom_highbd_8_variance8x8, aom_highbd_8_sub_pixel_variance8x8, + aom_highbd_8_sub_pixel_avg_variance8x8, aom_highbd_sad8x8x3_bits8, + aom_highbd_sad8x8x8_bits8, aom_highbd_sad8x8x4d_bits8) + + HIGHBD_BFP(BLOCK_8X4, aom_highbd_sad8x4_bits8, + aom_highbd_sad8x4_avg_bits8, aom_highbd_8_variance8x4, + aom_highbd_8_sub_pixel_variance8x4, + aom_highbd_8_sub_pixel_avg_variance8x4, NULL, + aom_highbd_sad8x4x8_bits8, aom_highbd_sad8x4x4d_bits8) + + HIGHBD_BFP(BLOCK_4X8, aom_highbd_sad4x8_bits8, + aom_highbd_sad4x8_avg_bits8, aom_highbd_8_variance4x8, + aom_highbd_8_sub_pixel_variance4x8, + aom_highbd_8_sub_pixel_avg_variance4x8, NULL, + aom_highbd_sad4x8x8_bits8, aom_highbd_sad4x8x4d_bits8) + + HIGHBD_BFP( + BLOCK_4X4, aom_highbd_sad4x4_bits8, aom_highbd_sad4x4_avg_bits8, + aom_highbd_8_variance4x4, aom_highbd_8_sub_pixel_variance4x4, + aom_highbd_8_sub_pixel_avg_variance4x4, aom_highbd_sad4x4x3_bits8, + aom_highbd_sad4x4x8_bits8, aom_highbd_sad4x4x4d_bits8) + +#if CONFIG_CB4X4 + HIGHBD_BFP(BLOCK_2X2, NULL, NULL, aom_highbd_8_variance2x2, NULL, NULL, + NULL, NULL, NULL) + HIGHBD_BFP(BLOCK_4X2, NULL, NULL, aom_highbd_8_variance4x2, NULL, NULL, + NULL, NULL, NULL) + HIGHBD_BFP(BLOCK_2X4, NULL, NULL, aom_highbd_8_variance2x4, NULL, NULL, + NULL, NULL, NULL) +#endif + +#if CONFIG_EXT_PARTITION + HIGHBD_BFP(BLOCK_128X128, aom_highbd_sad128x128_bits8, + aom_highbd_sad128x128_avg_bits8, + aom_highbd_8_variance128x128, + aom_highbd_8_sub_pixel_variance128x128, + aom_highbd_8_sub_pixel_avg_variance128x128, + aom_highbd_sad128x128x3_bits8, aom_highbd_sad128x128x8_bits8, + aom_highbd_sad128x128x4d_bits8) + + HIGHBD_BFP(BLOCK_128X64, aom_highbd_sad128x64_bits8, + aom_highbd_sad128x64_avg_bits8, aom_highbd_8_variance128x64, + aom_highbd_8_sub_pixel_variance128x64, + aom_highbd_8_sub_pixel_avg_variance128x64, NULL, NULL, + aom_highbd_sad128x64x4d_bits8) + + HIGHBD_BFP(BLOCK_64X128, aom_highbd_sad64x128_bits8, + aom_highbd_sad64x128_avg_bits8, aom_highbd_8_variance64x128, + aom_highbd_8_sub_pixel_variance64x128, + aom_highbd_8_sub_pixel_avg_variance64x128, NULL, NULL, + aom_highbd_sad64x128x4d_bits8) +#endif // CONFIG_EXT_PARTITION + +#if CONFIG_EXT_INTER +#if CONFIG_EXT_PARTITION + HIGHBD_MBFP(BLOCK_128X128, aom_highbd_masked_sad128x128_bits8, + aom_highbd_masked_variance128x128, + aom_highbd_masked_sub_pixel_variance128x128) + HIGHBD_MBFP(BLOCK_128X64, aom_highbd_masked_sad128x64_bits8, + aom_highbd_masked_variance128x64, + aom_highbd_masked_sub_pixel_variance128x64) + HIGHBD_MBFP(BLOCK_64X128, aom_highbd_masked_sad64x128_bits8, + aom_highbd_masked_variance64x128, + aom_highbd_masked_sub_pixel_variance64x128) +#endif // CONFIG_EXT_PARTITION + HIGHBD_MBFP(BLOCK_64X64, aom_highbd_masked_sad64x64_bits8, + aom_highbd_masked_variance64x64, + aom_highbd_masked_sub_pixel_variance64x64) + HIGHBD_MBFP(BLOCK_64X32, aom_highbd_masked_sad64x32_bits8, + aom_highbd_masked_variance64x32, + aom_highbd_masked_sub_pixel_variance64x32) + HIGHBD_MBFP(BLOCK_32X64, aom_highbd_masked_sad32x64_bits8, + aom_highbd_masked_variance32x64, + aom_highbd_masked_sub_pixel_variance32x64) + HIGHBD_MBFP(BLOCK_32X32, aom_highbd_masked_sad32x32_bits8, + aom_highbd_masked_variance32x32, + aom_highbd_masked_sub_pixel_variance32x32) + HIGHBD_MBFP(BLOCK_32X16, aom_highbd_masked_sad32x16_bits8, + aom_highbd_masked_variance32x16, + aom_highbd_masked_sub_pixel_variance32x16) + HIGHBD_MBFP(BLOCK_16X32, aom_highbd_masked_sad16x32_bits8, + aom_highbd_masked_variance16x32, + aom_highbd_masked_sub_pixel_variance16x32) + HIGHBD_MBFP(BLOCK_16X16, aom_highbd_masked_sad16x16_bits8, + aom_highbd_masked_variance16x16, + aom_highbd_masked_sub_pixel_variance16x16) + HIGHBD_MBFP(BLOCK_8X16, aom_highbd_masked_sad8x16_bits8, + aom_highbd_masked_variance8x16, + aom_highbd_masked_sub_pixel_variance8x16) + HIGHBD_MBFP(BLOCK_16X8, aom_highbd_masked_sad16x8_bits8, + aom_highbd_masked_variance16x8, + aom_highbd_masked_sub_pixel_variance16x8) + HIGHBD_MBFP(BLOCK_8X8, aom_highbd_masked_sad8x8_bits8, + aom_highbd_masked_variance8x8, + aom_highbd_masked_sub_pixel_variance8x8) + HIGHBD_MBFP(BLOCK_4X8, aom_highbd_masked_sad4x8_bits8, + aom_highbd_masked_variance4x8, + aom_highbd_masked_sub_pixel_variance4x8) + HIGHBD_MBFP(BLOCK_8X4, aom_highbd_masked_sad8x4_bits8, + aom_highbd_masked_variance8x4, + aom_highbd_masked_sub_pixel_variance8x4) + HIGHBD_MBFP(BLOCK_4X4, aom_highbd_masked_sad4x4_bits8, + aom_highbd_masked_variance4x4, + aom_highbd_masked_sub_pixel_variance4x4) +#endif // CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR +#if CONFIG_EXT_PARTITION + HIGHBD_OBFP(BLOCK_128X128, aom_highbd_obmc_sad128x128_bits8, + aom_highbd_obmc_variance128x128, + aom_highbd_obmc_sub_pixel_variance128x128) + HIGHBD_OBFP(BLOCK_128X64, aom_highbd_obmc_sad128x64_bits8, + aom_highbd_obmc_variance128x64, + aom_highbd_obmc_sub_pixel_variance128x64) + HIGHBD_OBFP(BLOCK_64X128, aom_highbd_obmc_sad64x128_bits8, + aom_highbd_obmc_variance64x128, + aom_highbd_obmc_sub_pixel_variance64x128) +#endif // CONFIG_EXT_PARTITION + HIGHBD_OBFP(BLOCK_64X64, aom_highbd_obmc_sad64x64_bits8, + aom_highbd_obmc_variance64x64, + aom_highbd_obmc_sub_pixel_variance64x64) + HIGHBD_OBFP(BLOCK_64X32, aom_highbd_obmc_sad64x32_bits8, + aom_highbd_obmc_variance64x32, + aom_highbd_obmc_sub_pixel_variance64x32) + HIGHBD_OBFP(BLOCK_32X64, aom_highbd_obmc_sad32x64_bits8, + aom_highbd_obmc_variance32x64, + aom_highbd_obmc_sub_pixel_variance32x64) + HIGHBD_OBFP(BLOCK_32X32, aom_highbd_obmc_sad32x32_bits8, + aom_highbd_obmc_variance32x32, + aom_highbd_obmc_sub_pixel_variance32x32) + HIGHBD_OBFP(BLOCK_32X16, aom_highbd_obmc_sad32x16_bits8, + aom_highbd_obmc_variance32x16, + aom_highbd_obmc_sub_pixel_variance32x16) + HIGHBD_OBFP(BLOCK_16X32, aom_highbd_obmc_sad16x32_bits8, + aom_highbd_obmc_variance16x32, + aom_highbd_obmc_sub_pixel_variance16x32) + HIGHBD_OBFP(BLOCK_16X16, aom_highbd_obmc_sad16x16_bits8, + aom_highbd_obmc_variance16x16, + aom_highbd_obmc_sub_pixel_variance16x16) + HIGHBD_OBFP(BLOCK_8X16, aom_highbd_obmc_sad8x16_bits8, + aom_highbd_obmc_variance8x16, + aom_highbd_obmc_sub_pixel_variance8x16) + HIGHBD_OBFP(BLOCK_16X8, aom_highbd_obmc_sad16x8_bits8, + aom_highbd_obmc_variance16x8, + aom_highbd_obmc_sub_pixel_variance16x8) + HIGHBD_OBFP(BLOCK_8X8, aom_highbd_obmc_sad8x8_bits8, + aom_highbd_obmc_variance8x8, + aom_highbd_obmc_sub_pixel_variance8x8) + HIGHBD_OBFP(BLOCK_4X8, aom_highbd_obmc_sad4x8_bits8, + aom_highbd_obmc_variance4x8, + aom_highbd_obmc_sub_pixel_variance4x8) + HIGHBD_OBFP(BLOCK_8X4, aom_highbd_obmc_sad8x4_bits8, + aom_highbd_obmc_variance8x4, + aom_highbd_obmc_sub_pixel_variance8x4) + HIGHBD_OBFP(BLOCK_4X4, aom_highbd_obmc_sad4x4_bits8, + aom_highbd_obmc_variance4x4, + aom_highbd_obmc_sub_pixel_variance4x4) +#endif // CONFIG_MOTION_VAR + break; + + case AOM_BITS_10: + HIGHBD_BFP(BLOCK_32X16, aom_highbd_sad32x16_bits10, + aom_highbd_sad32x16_avg_bits10, aom_highbd_10_variance32x16, + aom_highbd_10_sub_pixel_variance32x16, + aom_highbd_10_sub_pixel_avg_variance32x16, NULL, NULL, + aom_highbd_sad32x16x4d_bits10) + + HIGHBD_BFP(BLOCK_16X32, aom_highbd_sad16x32_bits10, + aom_highbd_sad16x32_avg_bits10, aom_highbd_10_variance16x32, + aom_highbd_10_sub_pixel_variance16x32, + aom_highbd_10_sub_pixel_avg_variance16x32, NULL, NULL, + aom_highbd_sad16x32x4d_bits10) + + HIGHBD_BFP(BLOCK_64X32, aom_highbd_sad64x32_bits10, + aom_highbd_sad64x32_avg_bits10, aom_highbd_10_variance64x32, + aom_highbd_10_sub_pixel_variance64x32, + aom_highbd_10_sub_pixel_avg_variance64x32, NULL, NULL, + aom_highbd_sad64x32x4d_bits10) + + HIGHBD_BFP(BLOCK_32X64, aom_highbd_sad32x64_bits10, + aom_highbd_sad32x64_avg_bits10, aom_highbd_10_variance32x64, + aom_highbd_10_sub_pixel_variance32x64, + aom_highbd_10_sub_pixel_avg_variance32x64, NULL, NULL, + aom_highbd_sad32x64x4d_bits10) + + HIGHBD_BFP(BLOCK_32X32, aom_highbd_sad32x32_bits10, + aom_highbd_sad32x32_avg_bits10, aom_highbd_10_variance32x32, + aom_highbd_10_sub_pixel_variance32x32, + aom_highbd_10_sub_pixel_avg_variance32x32, + aom_highbd_sad32x32x3_bits10, aom_highbd_sad32x32x8_bits10, + aom_highbd_sad32x32x4d_bits10) + + HIGHBD_BFP(BLOCK_64X64, aom_highbd_sad64x64_bits10, + aom_highbd_sad64x64_avg_bits10, aom_highbd_10_variance64x64, + aom_highbd_10_sub_pixel_variance64x64, + aom_highbd_10_sub_pixel_avg_variance64x64, + aom_highbd_sad64x64x3_bits10, aom_highbd_sad64x64x8_bits10, + aom_highbd_sad64x64x4d_bits10) + + HIGHBD_BFP(BLOCK_16X16, aom_highbd_sad16x16_bits10, + aom_highbd_sad16x16_avg_bits10, aom_highbd_10_variance16x16, + aom_highbd_10_sub_pixel_variance16x16, + aom_highbd_10_sub_pixel_avg_variance16x16, + aom_highbd_sad16x16x3_bits10, aom_highbd_sad16x16x8_bits10, + aom_highbd_sad16x16x4d_bits10) + + HIGHBD_BFP(BLOCK_16X8, aom_highbd_sad16x8_bits10, + aom_highbd_sad16x8_avg_bits10, aom_highbd_10_variance16x8, + aom_highbd_10_sub_pixel_variance16x8, + aom_highbd_10_sub_pixel_avg_variance16x8, + aom_highbd_sad16x8x3_bits10, aom_highbd_sad16x8x8_bits10, + aom_highbd_sad16x8x4d_bits10) + + HIGHBD_BFP(BLOCK_8X16, aom_highbd_sad8x16_bits10, + aom_highbd_sad8x16_avg_bits10, aom_highbd_10_variance8x16, + aom_highbd_10_sub_pixel_variance8x16, + aom_highbd_10_sub_pixel_avg_variance8x16, + aom_highbd_sad8x16x3_bits10, aom_highbd_sad8x16x8_bits10, + aom_highbd_sad8x16x4d_bits10) + + HIGHBD_BFP( + BLOCK_8X8, aom_highbd_sad8x8_bits10, aom_highbd_sad8x8_avg_bits10, + aom_highbd_10_variance8x8, aom_highbd_10_sub_pixel_variance8x8, + aom_highbd_10_sub_pixel_avg_variance8x8, aom_highbd_sad8x8x3_bits10, + aom_highbd_sad8x8x8_bits10, aom_highbd_sad8x8x4d_bits10) + + HIGHBD_BFP(BLOCK_8X4, aom_highbd_sad8x4_bits10, + aom_highbd_sad8x4_avg_bits10, aom_highbd_10_variance8x4, + aom_highbd_10_sub_pixel_variance8x4, + aom_highbd_10_sub_pixel_avg_variance8x4, NULL, + aom_highbd_sad8x4x8_bits10, aom_highbd_sad8x4x4d_bits10) + + HIGHBD_BFP(BLOCK_4X8, aom_highbd_sad4x8_bits10, + aom_highbd_sad4x8_avg_bits10, aom_highbd_10_variance4x8, + aom_highbd_10_sub_pixel_variance4x8, + aom_highbd_10_sub_pixel_avg_variance4x8, NULL, + aom_highbd_sad4x8x8_bits10, aom_highbd_sad4x8x4d_bits10) + + HIGHBD_BFP( + BLOCK_4X4, aom_highbd_sad4x4_bits10, aom_highbd_sad4x4_avg_bits10, + aom_highbd_10_variance4x4, aom_highbd_10_sub_pixel_variance4x4, + aom_highbd_10_sub_pixel_avg_variance4x4, aom_highbd_sad4x4x3_bits10, + aom_highbd_sad4x4x8_bits10, aom_highbd_sad4x4x4d_bits10) + +#if CONFIG_CB4X4 + HIGHBD_BFP(BLOCK_2X2, NULL, NULL, aom_highbd_10_variance2x2, NULL, NULL, + NULL, NULL, NULL) + HIGHBD_BFP(BLOCK_4X2, NULL, NULL, aom_highbd_10_variance4x2, NULL, NULL, + NULL, NULL, NULL) + HIGHBD_BFP(BLOCK_2X4, NULL, NULL, aom_highbd_10_variance2x4, NULL, NULL, + NULL, NULL, NULL) +#endif + +#if CONFIG_EXT_PARTITION + HIGHBD_BFP( + BLOCK_128X128, aom_highbd_sad128x128_bits10, + aom_highbd_sad128x128_avg_bits10, aom_highbd_10_variance128x128, + aom_highbd_10_sub_pixel_variance128x128, + aom_highbd_10_sub_pixel_avg_variance128x128, + aom_highbd_sad128x128x3_bits10, aom_highbd_sad128x128x8_bits10, + aom_highbd_sad128x128x4d_bits10) + + HIGHBD_BFP(BLOCK_128X64, aom_highbd_sad128x64_bits10, + aom_highbd_sad128x64_avg_bits10, + aom_highbd_10_variance128x64, + aom_highbd_10_sub_pixel_variance128x64, + aom_highbd_10_sub_pixel_avg_variance128x64, NULL, NULL, + aom_highbd_sad128x64x4d_bits10) + + HIGHBD_BFP(BLOCK_64X128, aom_highbd_sad64x128_bits10, + aom_highbd_sad64x128_avg_bits10, + aom_highbd_10_variance64x128, + aom_highbd_10_sub_pixel_variance64x128, + aom_highbd_10_sub_pixel_avg_variance64x128, NULL, NULL, + aom_highbd_sad64x128x4d_bits10) +#endif // CONFIG_EXT_PARTITION + +#if CONFIG_EXT_INTER +#if CONFIG_EXT_PARTITION + HIGHBD_MBFP(BLOCK_128X128, aom_highbd_masked_sad128x128_bits10, + aom_highbd_10_masked_variance128x128, + aom_highbd_10_masked_sub_pixel_variance128x128) + HIGHBD_MBFP(BLOCK_128X64, aom_highbd_masked_sad128x64_bits10, + aom_highbd_10_masked_variance128x64, + aom_highbd_10_masked_sub_pixel_variance128x64) + HIGHBD_MBFP(BLOCK_64X128, aom_highbd_masked_sad64x128_bits10, + aom_highbd_10_masked_variance64x128, + aom_highbd_10_masked_sub_pixel_variance64x128) +#endif // CONFIG_EXT_PARTITION + HIGHBD_MBFP(BLOCK_64X64, aom_highbd_masked_sad64x64_bits10, + aom_highbd_10_masked_variance64x64, + aom_highbd_10_masked_sub_pixel_variance64x64) + HIGHBD_MBFP(BLOCK_64X32, aom_highbd_masked_sad64x32_bits10, + aom_highbd_10_masked_variance64x32, + aom_highbd_10_masked_sub_pixel_variance64x32) + HIGHBD_MBFP(BLOCK_32X64, aom_highbd_masked_sad32x64_bits10, + aom_highbd_10_masked_variance32x64, + aom_highbd_10_masked_sub_pixel_variance32x64) + HIGHBD_MBFP(BLOCK_32X32, aom_highbd_masked_sad32x32_bits10, + aom_highbd_10_masked_variance32x32, + aom_highbd_10_masked_sub_pixel_variance32x32) + HIGHBD_MBFP(BLOCK_32X16, aom_highbd_masked_sad32x16_bits10, + aom_highbd_10_masked_variance32x16, + aom_highbd_10_masked_sub_pixel_variance32x16) + HIGHBD_MBFP(BLOCK_16X32, aom_highbd_masked_sad16x32_bits10, + aom_highbd_10_masked_variance16x32, + aom_highbd_10_masked_sub_pixel_variance16x32) + HIGHBD_MBFP(BLOCK_16X16, aom_highbd_masked_sad16x16_bits10, + aom_highbd_10_masked_variance16x16, + aom_highbd_10_masked_sub_pixel_variance16x16) + HIGHBD_MBFP(BLOCK_8X16, aom_highbd_masked_sad8x16_bits10, + aom_highbd_10_masked_variance8x16, + aom_highbd_10_masked_sub_pixel_variance8x16) + HIGHBD_MBFP(BLOCK_16X8, aom_highbd_masked_sad16x8_bits10, + aom_highbd_10_masked_variance16x8, + aom_highbd_10_masked_sub_pixel_variance16x8) + HIGHBD_MBFP(BLOCK_8X8, aom_highbd_masked_sad8x8_bits10, + aom_highbd_10_masked_variance8x8, + aom_highbd_10_masked_sub_pixel_variance8x8) + HIGHBD_MBFP(BLOCK_4X8, aom_highbd_masked_sad4x8_bits10, + aom_highbd_10_masked_variance4x8, + aom_highbd_10_masked_sub_pixel_variance4x8) + HIGHBD_MBFP(BLOCK_8X4, aom_highbd_masked_sad8x4_bits10, + aom_highbd_10_masked_variance8x4, + aom_highbd_10_masked_sub_pixel_variance8x4) + HIGHBD_MBFP(BLOCK_4X4, aom_highbd_masked_sad4x4_bits10, + aom_highbd_10_masked_variance4x4, + aom_highbd_10_masked_sub_pixel_variance4x4) +#endif // CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR +#if CONFIG_EXT_PARTITION + HIGHBD_OBFP(BLOCK_128X128, aom_highbd_obmc_sad128x128_bits10, + aom_highbd_10_obmc_variance128x128, + aom_highbd_10_obmc_sub_pixel_variance128x128) + HIGHBD_OBFP(BLOCK_128X64, aom_highbd_obmc_sad128x64_bits10, + aom_highbd_10_obmc_variance128x64, + aom_highbd_10_obmc_sub_pixel_variance128x64) + HIGHBD_OBFP(BLOCK_64X128, aom_highbd_obmc_sad64x128_bits10, + aom_highbd_10_obmc_variance64x128, + aom_highbd_10_obmc_sub_pixel_variance64x128) +#endif // CONFIG_EXT_PARTITION + HIGHBD_OBFP(BLOCK_64X64, aom_highbd_obmc_sad64x64_bits10, + aom_highbd_10_obmc_variance64x64, + aom_highbd_10_obmc_sub_pixel_variance64x64) + HIGHBD_OBFP(BLOCK_64X32, aom_highbd_obmc_sad64x32_bits10, + aom_highbd_10_obmc_variance64x32, + aom_highbd_10_obmc_sub_pixel_variance64x32) + HIGHBD_OBFP(BLOCK_32X64, aom_highbd_obmc_sad32x64_bits10, + aom_highbd_10_obmc_variance32x64, + aom_highbd_10_obmc_sub_pixel_variance32x64) + HIGHBD_OBFP(BLOCK_32X32, aom_highbd_obmc_sad32x32_bits10, + aom_highbd_10_obmc_variance32x32, + aom_highbd_10_obmc_sub_pixel_variance32x32) + HIGHBD_OBFP(BLOCK_32X16, aom_highbd_obmc_sad32x16_bits10, + aom_highbd_10_obmc_variance32x16, + aom_highbd_10_obmc_sub_pixel_variance32x16) + HIGHBD_OBFP(BLOCK_16X32, aom_highbd_obmc_sad16x32_bits10, + aom_highbd_10_obmc_variance16x32, + aom_highbd_10_obmc_sub_pixel_variance16x32) + HIGHBD_OBFP(BLOCK_16X16, aom_highbd_obmc_sad16x16_bits10, + aom_highbd_10_obmc_variance16x16, + aom_highbd_10_obmc_sub_pixel_variance16x16) + HIGHBD_OBFP(BLOCK_8X16, aom_highbd_obmc_sad8x16_bits10, + aom_highbd_10_obmc_variance8x16, + aom_highbd_10_obmc_sub_pixel_variance8x16) + HIGHBD_OBFP(BLOCK_16X8, aom_highbd_obmc_sad16x8_bits10, + aom_highbd_10_obmc_variance16x8, + aom_highbd_10_obmc_sub_pixel_variance16x8) + HIGHBD_OBFP(BLOCK_8X8, aom_highbd_obmc_sad8x8_bits10, + aom_highbd_10_obmc_variance8x8, + aom_highbd_10_obmc_sub_pixel_variance8x8) + HIGHBD_OBFP(BLOCK_4X8, aom_highbd_obmc_sad4x8_bits10, + aom_highbd_10_obmc_variance4x8, + aom_highbd_10_obmc_sub_pixel_variance4x8) + HIGHBD_OBFP(BLOCK_8X4, aom_highbd_obmc_sad8x4_bits10, + aom_highbd_10_obmc_variance8x4, + aom_highbd_10_obmc_sub_pixel_variance8x4) + HIGHBD_OBFP(BLOCK_4X4, aom_highbd_obmc_sad4x4_bits10, + aom_highbd_10_obmc_variance4x4, + aom_highbd_10_obmc_sub_pixel_variance4x4) +#endif // CONFIG_MOTION_VAR + break; + + case AOM_BITS_12: + HIGHBD_BFP(BLOCK_32X16, aom_highbd_sad32x16_bits12, + aom_highbd_sad32x16_avg_bits12, aom_highbd_12_variance32x16, + aom_highbd_12_sub_pixel_variance32x16, + aom_highbd_12_sub_pixel_avg_variance32x16, NULL, NULL, + aom_highbd_sad32x16x4d_bits12) + + HIGHBD_BFP(BLOCK_16X32, aom_highbd_sad16x32_bits12, + aom_highbd_sad16x32_avg_bits12, aom_highbd_12_variance16x32, + aom_highbd_12_sub_pixel_variance16x32, + aom_highbd_12_sub_pixel_avg_variance16x32, NULL, NULL, + aom_highbd_sad16x32x4d_bits12) + + HIGHBD_BFP(BLOCK_64X32, aom_highbd_sad64x32_bits12, + aom_highbd_sad64x32_avg_bits12, aom_highbd_12_variance64x32, + aom_highbd_12_sub_pixel_variance64x32, + aom_highbd_12_sub_pixel_avg_variance64x32, NULL, NULL, + aom_highbd_sad64x32x4d_bits12) + + HIGHBD_BFP(BLOCK_32X64, aom_highbd_sad32x64_bits12, + aom_highbd_sad32x64_avg_bits12, aom_highbd_12_variance32x64, + aom_highbd_12_sub_pixel_variance32x64, + aom_highbd_12_sub_pixel_avg_variance32x64, NULL, NULL, + aom_highbd_sad32x64x4d_bits12) + + HIGHBD_BFP(BLOCK_32X32, aom_highbd_sad32x32_bits12, + aom_highbd_sad32x32_avg_bits12, aom_highbd_12_variance32x32, + aom_highbd_12_sub_pixel_variance32x32, + aom_highbd_12_sub_pixel_avg_variance32x32, + aom_highbd_sad32x32x3_bits12, aom_highbd_sad32x32x8_bits12, + aom_highbd_sad32x32x4d_bits12) + + HIGHBD_BFP(BLOCK_64X64, aom_highbd_sad64x64_bits12, + aom_highbd_sad64x64_avg_bits12, aom_highbd_12_variance64x64, + aom_highbd_12_sub_pixel_variance64x64, + aom_highbd_12_sub_pixel_avg_variance64x64, + aom_highbd_sad64x64x3_bits12, aom_highbd_sad64x64x8_bits12, + aom_highbd_sad64x64x4d_bits12) + + HIGHBD_BFP(BLOCK_16X16, aom_highbd_sad16x16_bits12, + aom_highbd_sad16x16_avg_bits12, aom_highbd_12_variance16x16, + aom_highbd_12_sub_pixel_variance16x16, + aom_highbd_12_sub_pixel_avg_variance16x16, + aom_highbd_sad16x16x3_bits12, aom_highbd_sad16x16x8_bits12, + aom_highbd_sad16x16x4d_bits12) + + HIGHBD_BFP(BLOCK_16X8, aom_highbd_sad16x8_bits12, + aom_highbd_sad16x8_avg_bits12, aom_highbd_12_variance16x8, + aom_highbd_12_sub_pixel_variance16x8, + aom_highbd_12_sub_pixel_avg_variance16x8, + aom_highbd_sad16x8x3_bits12, aom_highbd_sad16x8x8_bits12, + aom_highbd_sad16x8x4d_bits12) + + HIGHBD_BFP(BLOCK_8X16, aom_highbd_sad8x16_bits12, + aom_highbd_sad8x16_avg_bits12, aom_highbd_12_variance8x16, + aom_highbd_12_sub_pixel_variance8x16, + aom_highbd_12_sub_pixel_avg_variance8x16, + aom_highbd_sad8x16x3_bits12, aom_highbd_sad8x16x8_bits12, + aom_highbd_sad8x16x4d_bits12) + + HIGHBD_BFP( + BLOCK_8X8, aom_highbd_sad8x8_bits12, aom_highbd_sad8x8_avg_bits12, + aom_highbd_12_variance8x8, aom_highbd_12_sub_pixel_variance8x8, + aom_highbd_12_sub_pixel_avg_variance8x8, aom_highbd_sad8x8x3_bits12, + aom_highbd_sad8x8x8_bits12, aom_highbd_sad8x8x4d_bits12) + + HIGHBD_BFP(BLOCK_8X4, aom_highbd_sad8x4_bits12, + aom_highbd_sad8x4_avg_bits12, aom_highbd_12_variance8x4, + aom_highbd_12_sub_pixel_variance8x4, + aom_highbd_12_sub_pixel_avg_variance8x4, NULL, + aom_highbd_sad8x4x8_bits12, aom_highbd_sad8x4x4d_bits12) + + HIGHBD_BFP(BLOCK_4X8, aom_highbd_sad4x8_bits12, + aom_highbd_sad4x8_avg_bits12, aom_highbd_12_variance4x8, + aom_highbd_12_sub_pixel_variance4x8, + aom_highbd_12_sub_pixel_avg_variance4x8, NULL, + aom_highbd_sad4x8x8_bits12, aom_highbd_sad4x8x4d_bits12) + + HIGHBD_BFP( + BLOCK_4X4, aom_highbd_sad4x4_bits12, aom_highbd_sad4x4_avg_bits12, + aom_highbd_12_variance4x4, aom_highbd_12_sub_pixel_variance4x4, + aom_highbd_12_sub_pixel_avg_variance4x4, aom_highbd_sad4x4x3_bits12, + aom_highbd_sad4x4x8_bits12, aom_highbd_sad4x4x4d_bits12) + +#if CONFIG_CB4X4 + HIGHBD_BFP(BLOCK_2X2, NULL, NULL, aom_highbd_12_variance2x2, NULL, NULL, + NULL, NULL, NULL) + HIGHBD_BFP(BLOCK_4X2, NULL, NULL, aom_highbd_12_variance4x2, NULL, NULL, + NULL, NULL, NULL) + HIGHBD_BFP(BLOCK_2X4, NULL, NULL, aom_highbd_12_variance2x4, NULL, NULL, + NULL, NULL, NULL) +#endif + +#if CONFIG_EXT_PARTITION + HIGHBD_BFP( + BLOCK_128X128, aom_highbd_sad128x128_bits12, + aom_highbd_sad128x128_avg_bits12, aom_highbd_12_variance128x128, + aom_highbd_12_sub_pixel_variance128x128, + aom_highbd_12_sub_pixel_avg_variance128x128, + aom_highbd_sad128x128x3_bits12, aom_highbd_sad128x128x8_bits12, + aom_highbd_sad128x128x4d_bits12) + + HIGHBD_BFP(BLOCK_128X64, aom_highbd_sad128x64_bits12, + aom_highbd_sad128x64_avg_bits12, + aom_highbd_12_variance128x64, + aom_highbd_12_sub_pixel_variance128x64, + aom_highbd_12_sub_pixel_avg_variance128x64, NULL, NULL, + aom_highbd_sad128x64x4d_bits12) + + HIGHBD_BFP(BLOCK_64X128, aom_highbd_sad64x128_bits12, + aom_highbd_sad64x128_avg_bits12, + aom_highbd_12_variance64x128, + aom_highbd_12_sub_pixel_variance64x128, + aom_highbd_12_sub_pixel_avg_variance64x128, NULL, NULL, + aom_highbd_sad64x128x4d_bits12) +#endif // CONFIG_EXT_PARTITION + +#if CONFIG_EXT_INTER +#if CONFIG_EXT_PARTITION + HIGHBD_MBFP(BLOCK_128X128, aom_highbd_masked_sad128x128_bits12, + aom_highbd_12_masked_variance128x128, + aom_highbd_12_masked_sub_pixel_variance128x128) + HIGHBD_MBFP(BLOCK_128X64, aom_highbd_masked_sad128x64_bits12, + aom_highbd_12_masked_variance128x64, + aom_highbd_12_masked_sub_pixel_variance128x64) + HIGHBD_MBFP(BLOCK_64X128, aom_highbd_masked_sad64x128_bits12, + aom_highbd_12_masked_variance64x128, + aom_highbd_12_masked_sub_pixel_variance64x128) +#endif // CONFIG_EXT_PARTITION + HIGHBD_MBFP(BLOCK_64X64, aom_highbd_masked_sad64x64_bits12, + aom_highbd_12_masked_variance64x64, + aom_highbd_12_masked_sub_pixel_variance64x64) + HIGHBD_MBFP(BLOCK_64X32, aom_highbd_masked_sad64x32_bits12, + aom_highbd_12_masked_variance64x32, + aom_highbd_12_masked_sub_pixel_variance64x32) + HIGHBD_MBFP(BLOCK_32X64, aom_highbd_masked_sad32x64_bits12, + aom_highbd_12_masked_variance32x64, + aom_highbd_12_masked_sub_pixel_variance32x64) + HIGHBD_MBFP(BLOCK_32X32, aom_highbd_masked_sad32x32_bits12, + aom_highbd_12_masked_variance32x32, + aom_highbd_12_masked_sub_pixel_variance32x32) + HIGHBD_MBFP(BLOCK_32X16, aom_highbd_masked_sad32x16_bits12, + aom_highbd_12_masked_variance32x16, + aom_highbd_12_masked_sub_pixel_variance32x16) + HIGHBD_MBFP(BLOCK_16X32, aom_highbd_masked_sad16x32_bits12, + aom_highbd_12_masked_variance16x32, + aom_highbd_12_masked_sub_pixel_variance16x32) + HIGHBD_MBFP(BLOCK_16X16, aom_highbd_masked_sad16x16_bits12, + aom_highbd_12_masked_variance16x16, + aom_highbd_12_masked_sub_pixel_variance16x16) + HIGHBD_MBFP(BLOCK_8X16, aom_highbd_masked_sad8x16_bits12, + aom_highbd_12_masked_variance8x16, + aom_highbd_12_masked_sub_pixel_variance8x16) + HIGHBD_MBFP(BLOCK_16X8, aom_highbd_masked_sad16x8_bits12, + aom_highbd_12_masked_variance16x8, + aom_highbd_12_masked_sub_pixel_variance16x8) + HIGHBD_MBFP(BLOCK_8X8, aom_highbd_masked_sad8x8_bits12, + aom_highbd_12_masked_variance8x8, + aom_highbd_12_masked_sub_pixel_variance8x8) + HIGHBD_MBFP(BLOCK_4X8, aom_highbd_masked_sad4x8_bits12, + aom_highbd_12_masked_variance4x8, + aom_highbd_12_masked_sub_pixel_variance4x8) + HIGHBD_MBFP(BLOCK_8X4, aom_highbd_masked_sad8x4_bits12, + aom_highbd_12_masked_variance8x4, + aom_highbd_12_masked_sub_pixel_variance8x4) + HIGHBD_MBFP(BLOCK_4X4, aom_highbd_masked_sad4x4_bits12, + aom_highbd_12_masked_variance4x4, + aom_highbd_12_masked_sub_pixel_variance4x4) +#endif // CONFIG_EXT_INTER + +#if CONFIG_MOTION_VAR +#if CONFIG_EXT_PARTITION + HIGHBD_OBFP(BLOCK_128X128, aom_highbd_obmc_sad128x128_bits12, + aom_highbd_12_obmc_variance128x128, + aom_highbd_12_obmc_sub_pixel_variance128x128) + HIGHBD_OBFP(BLOCK_128X64, aom_highbd_obmc_sad128x64_bits12, + aom_highbd_12_obmc_variance128x64, + aom_highbd_12_obmc_sub_pixel_variance128x64) + HIGHBD_OBFP(BLOCK_64X128, aom_highbd_obmc_sad64x128_bits12, + aom_highbd_12_obmc_variance64x128, + aom_highbd_12_obmc_sub_pixel_variance64x128) +#endif // CONFIG_EXT_PARTITION + HIGHBD_OBFP(BLOCK_64X64, aom_highbd_obmc_sad64x64_bits12, + aom_highbd_12_obmc_variance64x64, + aom_highbd_12_obmc_sub_pixel_variance64x64) + HIGHBD_OBFP(BLOCK_64X32, aom_highbd_obmc_sad64x32_bits12, + aom_highbd_12_obmc_variance64x32, + aom_highbd_12_obmc_sub_pixel_variance64x32) + HIGHBD_OBFP(BLOCK_32X64, aom_highbd_obmc_sad32x64_bits12, + aom_highbd_12_obmc_variance32x64, + aom_highbd_12_obmc_sub_pixel_variance32x64) + HIGHBD_OBFP(BLOCK_32X32, aom_highbd_obmc_sad32x32_bits12, + aom_highbd_12_obmc_variance32x32, + aom_highbd_12_obmc_sub_pixel_variance32x32) + HIGHBD_OBFP(BLOCK_32X16, aom_highbd_obmc_sad32x16_bits12, + aom_highbd_12_obmc_variance32x16, + aom_highbd_12_obmc_sub_pixel_variance32x16) + HIGHBD_OBFP(BLOCK_16X32, aom_highbd_obmc_sad16x32_bits12, + aom_highbd_12_obmc_variance16x32, + aom_highbd_12_obmc_sub_pixel_variance16x32) + HIGHBD_OBFP(BLOCK_16X16, aom_highbd_obmc_sad16x16_bits12, + aom_highbd_12_obmc_variance16x16, + aom_highbd_12_obmc_sub_pixel_variance16x16) + HIGHBD_OBFP(BLOCK_8X16, aom_highbd_obmc_sad8x16_bits12, + aom_highbd_12_obmc_variance8x16, + aom_highbd_12_obmc_sub_pixel_variance8x16) + HIGHBD_OBFP(BLOCK_16X8, aom_highbd_obmc_sad16x8_bits12, + aom_highbd_12_obmc_variance16x8, + aom_highbd_12_obmc_sub_pixel_variance16x8) + HIGHBD_OBFP(BLOCK_8X8, aom_highbd_obmc_sad8x8_bits12, + aom_highbd_12_obmc_variance8x8, + aom_highbd_12_obmc_sub_pixel_variance8x8) + HIGHBD_OBFP(BLOCK_4X8, aom_highbd_obmc_sad4x8_bits12, + aom_highbd_12_obmc_variance4x8, + aom_highbd_12_obmc_sub_pixel_variance4x8) + HIGHBD_OBFP(BLOCK_8X4, aom_highbd_obmc_sad8x4_bits12, + aom_highbd_12_obmc_variance8x4, + aom_highbd_12_obmc_sub_pixel_variance8x4) + HIGHBD_OBFP(BLOCK_4X4, aom_highbd_obmc_sad4x4_bits12, + aom_highbd_12_obmc_variance4x4, + aom_highbd_12_obmc_sub_pixel_variance4x4) +#endif // CONFIG_MOTION_VAR + break; + + default: + assert(0 && + "cm->bit_depth should be AOM_BITS_8, " + "AOM_BITS_10 or AOM_BITS_12"); + } + } +} +#endif // CONFIG_HIGHBITDEPTH + +static void realloc_segmentation_maps(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + + // Create the encoder segmentation map and set all entries to 0 + aom_free(cpi->segmentation_map); + CHECK_MEM_ERROR(cm, cpi->segmentation_map, + aom_calloc(cm->mi_rows * cm->mi_cols, 1)); + + // Create a map used for cyclic background refresh. + if (cpi->cyclic_refresh) av1_cyclic_refresh_free(cpi->cyclic_refresh); + CHECK_MEM_ERROR(cm, cpi->cyclic_refresh, + av1_cyclic_refresh_alloc(cm->mi_rows, cm->mi_cols)); + + // Create a map used to mark inactive areas. + aom_free(cpi->active_map.map); + CHECK_MEM_ERROR(cm, cpi->active_map.map, + aom_calloc(cm->mi_rows * cm->mi_cols, 1)); +} + +void av1_change_config(struct AV1_COMP *cpi, const AV1EncoderConfig *oxcf) { + AV1_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; + + if (cm->profile != oxcf->profile) cm->profile = oxcf->profile; + cm->bit_depth = oxcf->bit_depth; + cm->color_space = oxcf->color_space; + cm->color_range = oxcf->color_range; + + if (cm->profile <= PROFILE_1) + assert(cm->bit_depth == AOM_BITS_8); + else + assert(cm->bit_depth > AOM_BITS_8); + + cpi->oxcf = *oxcf; +#if CONFIG_HIGHBITDEPTH + cpi->td.mb.e_mbd.bd = (int)cm->bit_depth; +#endif // CONFIG_HIGHBITDEPTH +#if CONFIG_GLOBAL_MOTION + cpi->td.mb.e_mbd.global_motion = cm->global_motion; +#endif // CONFIG_GLOBAL_MOTION + + if ((oxcf->pass == 0) && (oxcf->rc_mode == AOM_Q)) { + rc->baseline_gf_interval = FIXED_GF_INTERVAL; + } else { + rc->baseline_gf_interval = (MIN_GF_INTERVAL + MAX_GF_INTERVAL) / 2; + } + + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 0; +#if CONFIG_EXT_REFS + cpi->refresh_bwd_ref_frame = 0; +#endif // CONFIG_EXT_REFS + + cm->refresh_frame_context = + (oxcf->error_resilient_mode || oxcf->frame_parallel_decoding_mode) + ? REFRESH_FRAME_CONTEXT_FORWARD + : REFRESH_FRAME_CONTEXT_BACKWARD; + cm->reset_frame_context = RESET_FRAME_CONTEXT_NONE; + +#if CONFIG_PALETTE + cm->allow_screen_content_tools = (cpi->oxcf.content == AOM_CONTENT_SCREEN); + if (cm->allow_screen_content_tools) { + MACROBLOCK *x = &cpi->td.mb; + if (x->palette_buffer == 0) { + CHECK_MEM_ERROR(cm, x->palette_buffer, + aom_memalign(16, sizeof(*x->palette_buffer))); + } + // Reallocate the pc_tree, as it's contents depends on + // the state of cm->allow_screen_content_tools + av1_free_pc_tree(&cpi->td); + av1_setup_pc_tree(&cpi->common, &cpi->td); + } +#endif // CONFIG_PALETTE + + av1_reset_segment_features(cm); + av1_set_high_precision_mv(cpi, 0); + + set_rc_buffer_sizes(rc, &cpi->oxcf); + + // Under a configuration change, where maximum_buffer_size may change, + // keep buffer level clipped to the maximum allowed buffer size. + rc->bits_off_target = AOMMIN(rc->bits_off_target, rc->maximum_buffer_size); + rc->buffer_level = AOMMIN(rc->buffer_level, rc->maximum_buffer_size); + + // Set up frame rate and related parameters rate control values. + av1_new_framerate(cpi, cpi->framerate); + + // Set absolute upper and lower quality limits + rc->worst_quality = cpi->oxcf.worst_allowed_q; + rc->best_quality = cpi->oxcf.best_allowed_q; + + cm->interp_filter = cpi->sf.default_interp_filter; + + if (cpi->oxcf.render_width > 0 && cpi->oxcf.render_height > 0) { + cm->render_width = cpi->oxcf.render_width; + cm->render_height = cpi->oxcf.render_height; + } else { + cm->render_width = cpi->oxcf.width; + cm->render_height = cpi->oxcf.height; + } + cm->width = cpi->oxcf.width; + cm->height = cpi->oxcf.height; + + if (cpi->initial_width) { + if (cm->width > cpi->initial_width || cm->height > cpi->initial_height) { + av1_free_context_buffers(cm); + av1_alloc_compressor_data(cpi); + realloc_segmentation_maps(cpi); + cpi->initial_width = cpi->initial_height = 0; + } + } + update_frame_size(cpi); + + cpi->alt_ref_source = NULL; + rc->is_src_frame_alt_ref = 0; + +#if CONFIG_EXT_REFS + rc->is_bwd_ref_frame = 0; + rc->is_last_bipred_frame = 0; + rc->is_bipred_frame = 0; +#endif // CONFIG_EXT_REFS + +#if 0 + // Experimental RD Code + cpi->frame_distortion = 0; + cpi->last_frame_distortion = 0; +#endif + + set_tile_info(cpi); + + cpi->ext_refresh_frame_flags_pending = 0; + cpi->ext_refresh_frame_context_pending = 0; + +#if CONFIG_HIGHBITDEPTH + highbd_set_var_fns(cpi); +#endif + +#if CONFIG_ANS && ANS_MAX_SYMBOLS + cpi->common.ans_window_size_log2 = cpi->oxcf.ans_window_size_log2; + if (cpi->buf_ans.size != (1 << cpi->common.ans_window_size_log2)) { + aom_buf_ans_free(&cpi->buf_ans); + aom_buf_ans_alloc(&cpi->buf_ans, &cpi->common.error, + 1 << cpi->common.ans_window_size_log2); + } +#endif // CONFIG_ANS && ANS_MAX_SYMBOLS +} + +#ifndef M_LOG2_E +#define M_LOG2_E 0.693147180559945309417 +#endif +#define log2f(x) (log(x) / (float)M_LOG2_E) + +#if !CONFIG_REF_MV +static void cal_nmvjointsadcost(int *mvjointsadcost) { + mvjointsadcost[0] = 600; + mvjointsadcost[1] = 300; + mvjointsadcost[2] = 300; + mvjointsadcost[3] = 300; +} +#endif + +static void cal_nmvsadcosts(int *mvsadcost[2]) { + int i = 1; + + mvsadcost[0][0] = 0; + mvsadcost[1][0] = 0; + + do { + double z = 256 * (2 * (log2f(8 * i) + .6)); + mvsadcost[0][i] = (int)z; + mvsadcost[1][i] = (int)z; + mvsadcost[0][-i] = (int)z; + mvsadcost[1][-i] = (int)z; + } while (++i <= MV_MAX); +} + +static void cal_nmvsadcosts_hp(int *mvsadcost[2]) { + int i = 1; + + mvsadcost[0][0] = 0; + mvsadcost[1][0] = 0; + + do { + double z = 256 * (2 * (log2f(8 * i) + .6)); + mvsadcost[0][i] = (int)z; + mvsadcost[1][i] = (int)z; + mvsadcost[0][-i] = (int)z; + mvsadcost[1][-i] = (int)z; + } while (++i <= MV_MAX); +} + +static INLINE void init_upsampled_ref_frame_bufs(AV1_COMP *cpi) { + int i; + + for (i = 0; i < (REF_FRAMES + 1); ++i) { + cpi->upsampled_ref_bufs[i].ref_count = 0; + cpi->upsampled_ref_idx[i] = INVALID_IDX; + } +} + +AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf, + BufferPool *const pool) { + unsigned int i; + AV1_COMP *volatile const cpi = aom_memalign(32, sizeof(AV1_COMP)); + AV1_COMMON *volatile const cm = cpi != NULL ? &cpi->common : NULL; + + if (!cm) return NULL; + + av1_zero(*cpi); + + if (setjmp(cm->error.jmp)) { + cm->error.setjmp = 0; + av1_remove_compressor(cpi); + return 0; + } + + cm->error.setjmp = 1; + cm->alloc_mi = av1_enc_alloc_mi; + cm->free_mi = av1_enc_free_mi; + cm->setup_mi = av1_enc_setup_mi; + + CHECK_MEM_ERROR(cm, cm->fc, + (FRAME_CONTEXT *)aom_memalign(32, sizeof(*cm->fc))); + CHECK_MEM_ERROR(cm, cm->frame_contexts, + (FRAME_CONTEXT *)aom_memalign( + 32, FRAME_CONTEXTS * sizeof(*cm->frame_contexts))); + memset(cm->fc, 0, sizeof(*cm->fc)); + memset(cm->frame_contexts, 0, FRAME_CONTEXTS * sizeof(*cm->frame_contexts)); + + cpi->resize_state = 0; + cpi->resize_avg_qp = 0; + cpi->resize_buffer_underflow = 0; + cpi->common.buffer_pool = pool; + + init_config(cpi, oxcf); +#if CONFIG_XIPHRC + cpi->od_rc.framerate = cpi->framerate; + cpi->od_rc.frame_width = cm->render_width; + cpi->od_rc.frame_height = cm->render_height; + cpi->od_rc.keyframe_rate = oxcf->key_freq; + cpi->od_rc.goldenframe_rate = FIXED_GF_INTERVAL; + cpi->od_rc.altref_rate = 25; + cpi->od_rc.firstpass_quant = 1; + cpi->od_rc.bit_depth = cm->bit_depth; + cpi->od_rc.minq = oxcf->best_allowed_q; + cpi->od_rc.maxq = oxcf->worst_allowed_q; + if (cpi->oxcf.rc_mode == AOM_CQ) cpi->od_rc.minq = cpi->od_rc.quality; + cpi->od_rc.quality = cpi->oxcf.rc_mode == AOM_Q ? oxcf->cq_level : -1; + cpi->od_rc.periodic_boosts = oxcf->frame_periodic_boost; + od_enc_rc_init(&cpi->od_rc, + cpi->oxcf.rc_mode == AOM_Q ? -1 : oxcf->target_bandwidth, + oxcf->maximum_buffer_size_ms); +#else + av1_rc_init(&cpi->oxcf, oxcf->pass, &cpi->rc); +#endif + + cm->current_video_frame = 0; + cpi->partition_search_skippable_frame = 0; + cpi->tile_data = NULL; + cpi->last_show_frame_buf_idx = INVALID_IDX; + + realloc_segmentation_maps(cpi); + +#if CONFIG_REF_MV + for (i = 0; i < NMV_CONTEXTS; ++i) { + memset(cpi->nmv_costs, 0, sizeof(cpi->nmv_costs)); + memset(cpi->nmv_costs_hp, 0, sizeof(cpi->nmv_costs_hp)); + } +#endif + + memset(cpi->nmvcosts, 0, sizeof(cpi->nmvcosts)); + memset(cpi->nmvcosts_hp, 0, sizeof(cpi->nmvcosts_hp)); + memset(cpi->nmvsadcosts, 0, sizeof(cpi->nmvsadcosts)); + memset(cpi->nmvsadcosts_hp, 0, sizeof(cpi->nmvsadcosts_hp)); + + for (i = 0; i < (sizeof(cpi->mbgraph_stats) / sizeof(cpi->mbgraph_stats[0])); + i++) { + CHECK_MEM_ERROR( + cm, cpi->mbgraph_stats[i].mb_stats, + aom_calloc(cm->MBs * sizeof(*cpi->mbgraph_stats[i].mb_stats), 1)); + } + +#if CONFIG_FP_MB_STATS + cpi->use_fp_mb_stats = 0; + if (cpi->use_fp_mb_stats) { + // a place holder used to store the first pass mb stats in the first pass + CHECK_MEM_ERROR(cm, cpi->twopass.frame_mb_stats_buf, + aom_calloc(cm->MBs * sizeof(uint8_t), 1)); + } else { + cpi->twopass.frame_mb_stats_buf = NULL; + } +#endif + + cpi->refresh_alt_ref_frame = 0; + cpi->multi_arf_last_grp_enabled = 0; + + cpi->b_calculate_psnr = CONFIG_INTERNAL_STATS; +#if CONFIG_INTERNAL_STATS + cpi->b_calculate_blockiness = 1; + cpi->b_calculate_consistency = 1; + cpi->total_inconsistency = 0; + cpi->psnr.worst = 100.0; + cpi->worst_ssim = 100.0; + + cpi->count = 0; + cpi->bytes = 0; + + if (cpi->b_calculate_psnr) { + cpi->total_sq_error = 0; + cpi->total_samples = 0; + cpi->tot_recode_hits = 0; + cpi->summed_quality = 0; + cpi->summed_weights = 0; + } + + cpi->fastssim.worst = 100.0; + cpi->psnrhvs.worst = 100.0; + + if (cpi->b_calculate_blockiness) { + cpi->total_blockiness = 0; + cpi->worst_blockiness = 0.0; + } + + if (cpi->b_calculate_consistency) { + CHECK_MEM_ERROR(cm, cpi->ssim_vars, + aom_malloc(sizeof(*cpi->ssim_vars) * 4 * + cpi->common.mi_rows * cpi->common.mi_cols)); + cpi->worst_consistency = 100.0; + } +#endif +#if CONFIG_ENTROPY_STATS + av1_zero(aggregate_fc); +#endif // CONFIG_ENTROPY_STATS + + cpi->first_time_stamp_ever = INT64_MAX; + +#if CONFIG_REF_MV + for (i = 0; i < NMV_CONTEXTS; ++i) { + cpi->td.mb.nmvcost[i][0] = &cpi->nmv_costs[i][0][MV_MAX]; + cpi->td.mb.nmvcost[i][1] = &cpi->nmv_costs[i][1][MV_MAX]; + cpi->td.mb.nmvcost_hp[i][0] = &cpi->nmv_costs_hp[i][0][MV_MAX]; + cpi->td.mb.nmvcost_hp[i][1] = &cpi->nmv_costs_hp[i][1][MV_MAX]; + } +#else + cal_nmvjointsadcost(cpi->td.mb.nmvjointsadcost); + cpi->td.mb.nmvcost[0] = &cpi->nmvcosts[0][MV_MAX]; + cpi->td.mb.nmvcost[1] = &cpi->nmvcosts[1][MV_MAX]; + cpi->td.mb.nmvcost_hp[0] = &cpi->nmvcosts_hp[0][MV_MAX]; + cpi->td.mb.nmvcost_hp[1] = &cpi->nmvcosts_hp[1][MV_MAX]; +#endif + cpi->td.mb.nmvsadcost[0] = &cpi->nmvsadcosts[0][MV_MAX]; + cpi->td.mb.nmvsadcost[1] = &cpi->nmvsadcosts[1][MV_MAX]; + cal_nmvsadcosts(cpi->td.mb.nmvsadcost); + + cpi->td.mb.nmvsadcost_hp[0] = &cpi->nmvsadcosts_hp[0][MV_MAX]; + cpi->td.mb.nmvsadcost_hp[1] = &cpi->nmvsadcosts_hp[1][MV_MAX]; + cal_nmvsadcosts_hp(cpi->td.mb.nmvsadcost_hp); + +#ifdef OUTPUT_YUV_SKINMAP + yuv_skinmap_file = fopen("skinmap.yuv", "ab"); +#endif +#ifdef OUTPUT_YUV_REC + yuv_rec_file = fopen("rec.yuv", "wb"); +#endif + +#if 0 + framepsnr = fopen("framepsnr.stt", "a"); + kf_list = fopen("kf_list.stt", "w"); +#endif + +#if CONFIG_XIPHRC + if (oxcf->pass == 2) { + cpi->od_rc.twopass_allframes_buf = oxcf->two_pass_stats_in.buf; + cpi->od_rc.twopass_allframes_buf_size = oxcf->two_pass_stats_in.sz; + } +#else + if (oxcf->pass == 1) { + av1_init_first_pass(cpi); + } else if (oxcf->pass == 2) { + const size_t packet_sz = sizeof(FIRSTPASS_STATS); + const int packets = (int)(oxcf->two_pass_stats_in.sz / packet_sz); + +#if CONFIG_FP_MB_STATS + if (cpi->use_fp_mb_stats) { + const size_t psz = cpi->common.MBs * sizeof(uint8_t); + const int ps = (int)(oxcf->firstpass_mb_stats_in.sz / psz); + + cpi->twopass.firstpass_mb_stats.mb_stats_start = + oxcf->firstpass_mb_stats_in.buf; + cpi->twopass.firstpass_mb_stats.mb_stats_end = + cpi->twopass.firstpass_mb_stats.mb_stats_start + + (ps - 1) * cpi->common.MBs * sizeof(uint8_t); + } +#endif + + cpi->twopass.stats_in_start = oxcf->two_pass_stats_in.buf; + cpi->twopass.stats_in = cpi->twopass.stats_in_start; + cpi->twopass.stats_in_end = &cpi->twopass.stats_in[packets - 1]; + + av1_init_second_pass(cpi); + } +#endif + + init_upsampled_ref_frame_bufs(cpi); + + av1_set_speed_features_framesize_independent(cpi); + av1_set_speed_features_framesize_dependent(cpi); + + // Allocate memory to store variances for a frame. + CHECK_MEM_ERROR(cm, cpi->source_diff_var, + aom_calloc(cm->MBs, sizeof(*cpi->source_diff_var))); + cpi->source_var_thresh = 0; + cpi->frames_till_next_var_check = 0; + +#define BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX3F, SDX8F, SDX4DF) \ + cpi->fn_ptr[BT].sdf = SDF; \ + cpi->fn_ptr[BT].sdaf = SDAF; \ + cpi->fn_ptr[BT].vf = VF; \ + cpi->fn_ptr[BT].svf = SVF; \ + cpi->fn_ptr[BT].svaf = SVAF; \ + cpi->fn_ptr[BT].sdx3f = SDX3F; \ + cpi->fn_ptr[BT].sdx8f = SDX8F; \ + cpi->fn_ptr[BT].sdx4df = SDX4DF; + +#if CONFIG_EXT_PARTITION + BFP(BLOCK_128X128, aom_sad128x128, aom_sad128x128_avg, aom_variance128x128, + aom_sub_pixel_variance128x128, aom_sub_pixel_avg_variance128x128, + aom_sad128x128x3, aom_sad128x128x8, aom_sad128x128x4d) + + BFP(BLOCK_128X64, aom_sad128x64, aom_sad128x64_avg, aom_variance128x64, + aom_sub_pixel_variance128x64, aom_sub_pixel_avg_variance128x64, NULL, + NULL, aom_sad128x64x4d) + + BFP(BLOCK_64X128, aom_sad64x128, aom_sad64x128_avg, aom_variance64x128, + aom_sub_pixel_variance64x128, aom_sub_pixel_avg_variance64x128, NULL, + NULL, aom_sad64x128x4d) +#endif // CONFIG_EXT_PARTITION + + BFP(BLOCK_32X16, aom_sad32x16, aom_sad32x16_avg, aom_variance32x16, + aom_sub_pixel_variance32x16, aom_sub_pixel_avg_variance32x16, NULL, NULL, + aom_sad32x16x4d) + + BFP(BLOCK_16X32, aom_sad16x32, aom_sad16x32_avg, aom_variance16x32, + aom_sub_pixel_variance16x32, aom_sub_pixel_avg_variance16x32, NULL, NULL, + aom_sad16x32x4d) + + BFP(BLOCK_64X32, aom_sad64x32, aom_sad64x32_avg, aom_variance64x32, + aom_sub_pixel_variance64x32, aom_sub_pixel_avg_variance64x32, NULL, NULL, + aom_sad64x32x4d) + + BFP(BLOCK_32X64, aom_sad32x64, aom_sad32x64_avg, aom_variance32x64, + aom_sub_pixel_variance32x64, aom_sub_pixel_avg_variance32x64, NULL, NULL, + aom_sad32x64x4d) + + BFP(BLOCK_32X32, aom_sad32x32, aom_sad32x32_avg, aom_variance32x32, + aom_sub_pixel_variance32x32, aom_sub_pixel_avg_variance32x32, + aom_sad32x32x3, aom_sad32x32x8, aom_sad32x32x4d) + + BFP(BLOCK_64X64, aom_sad64x64, aom_sad64x64_avg, aom_variance64x64, + aom_sub_pixel_variance64x64, aom_sub_pixel_avg_variance64x64, + aom_sad64x64x3, aom_sad64x64x8, aom_sad64x64x4d) + + BFP(BLOCK_16X16, aom_sad16x16, aom_sad16x16_avg, aom_variance16x16, + aom_sub_pixel_variance16x16, aom_sub_pixel_avg_variance16x16, + aom_sad16x16x3, aom_sad16x16x8, aom_sad16x16x4d) + + BFP(BLOCK_16X8, aom_sad16x8, aom_sad16x8_avg, aom_variance16x8, + aom_sub_pixel_variance16x8, aom_sub_pixel_avg_variance16x8, aom_sad16x8x3, + aom_sad16x8x8, aom_sad16x8x4d) + + BFP(BLOCK_8X16, aom_sad8x16, aom_sad8x16_avg, aom_variance8x16, + aom_sub_pixel_variance8x16, aom_sub_pixel_avg_variance8x16, aom_sad8x16x3, + aom_sad8x16x8, aom_sad8x16x4d) + + BFP(BLOCK_8X8, aom_sad8x8, aom_sad8x8_avg, aom_variance8x8, + aom_sub_pixel_variance8x8, aom_sub_pixel_avg_variance8x8, aom_sad8x8x3, + aom_sad8x8x8, aom_sad8x8x4d) + + BFP(BLOCK_8X4, aom_sad8x4, aom_sad8x4_avg, aom_variance8x4, + aom_sub_pixel_variance8x4, aom_sub_pixel_avg_variance8x4, NULL, + aom_sad8x4x8, aom_sad8x4x4d) + + BFP(BLOCK_4X8, aom_sad4x8, aom_sad4x8_avg, aom_variance4x8, + aom_sub_pixel_variance4x8, aom_sub_pixel_avg_variance4x8, NULL, + aom_sad4x8x8, aom_sad4x8x4d) + + BFP(BLOCK_4X4, aom_sad4x4, aom_sad4x4_avg, aom_variance4x4, + aom_sub_pixel_variance4x4, aom_sub_pixel_avg_variance4x4, aom_sad4x4x3, + aom_sad4x4x8, aom_sad4x4x4d) + +#if CONFIG_CB4X4 + BFP(BLOCK_2X2, NULL, NULL, aom_variance2x2, NULL, NULL, NULL, NULL, NULL) + BFP(BLOCK_2X4, NULL, NULL, aom_variance2x4, NULL, NULL, NULL, NULL, NULL) + BFP(BLOCK_4X2, NULL, NULL, aom_variance4x2, NULL, NULL, NULL, NULL, NULL) +#endif + +#if CONFIG_MOTION_VAR +#define OBFP(BT, OSDF, OVF, OSVF) \ + cpi->fn_ptr[BT].osdf = OSDF; \ + cpi->fn_ptr[BT].ovf = OVF; \ + cpi->fn_ptr[BT].osvf = OSVF; + +#if CONFIG_EXT_PARTITION + OBFP(BLOCK_128X128, aom_obmc_sad128x128, aom_obmc_variance128x128, + aom_obmc_sub_pixel_variance128x128) + OBFP(BLOCK_128X64, aom_obmc_sad128x64, aom_obmc_variance128x64, + aom_obmc_sub_pixel_variance128x64) + OBFP(BLOCK_64X128, aom_obmc_sad64x128, aom_obmc_variance64x128, + aom_obmc_sub_pixel_variance64x128) +#endif // CONFIG_EXT_PARTITION + OBFP(BLOCK_64X64, aom_obmc_sad64x64, aom_obmc_variance64x64, + aom_obmc_sub_pixel_variance64x64) + OBFP(BLOCK_64X32, aom_obmc_sad64x32, aom_obmc_variance64x32, + aom_obmc_sub_pixel_variance64x32) + OBFP(BLOCK_32X64, aom_obmc_sad32x64, aom_obmc_variance32x64, + aom_obmc_sub_pixel_variance32x64) + OBFP(BLOCK_32X32, aom_obmc_sad32x32, aom_obmc_variance32x32, + aom_obmc_sub_pixel_variance32x32) + OBFP(BLOCK_32X16, aom_obmc_sad32x16, aom_obmc_variance32x16, + aom_obmc_sub_pixel_variance32x16) + OBFP(BLOCK_16X32, aom_obmc_sad16x32, aom_obmc_variance16x32, + aom_obmc_sub_pixel_variance16x32) + OBFP(BLOCK_16X16, aom_obmc_sad16x16, aom_obmc_variance16x16, + aom_obmc_sub_pixel_variance16x16) + OBFP(BLOCK_16X8, aom_obmc_sad16x8, aom_obmc_variance16x8, + aom_obmc_sub_pixel_variance16x8) + OBFP(BLOCK_8X16, aom_obmc_sad8x16, aom_obmc_variance8x16, + aom_obmc_sub_pixel_variance8x16) + OBFP(BLOCK_8X8, aom_obmc_sad8x8, aom_obmc_variance8x8, + aom_obmc_sub_pixel_variance8x8) + OBFP(BLOCK_4X8, aom_obmc_sad4x8, aom_obmc_variance4x8, + aom_obmc_sub_pixel_variance4x8) + OBFP(BLOCK_8X4, aom_obmc_sad8x4, aom_obmc_variance8x4, + aom_obmc_sub_pixel_variance8x4) + OBFP(BLOCK_4X4, aom_obmc_sad4x4, aom_obmc_variance4x4, + aom_obmc_sub_pixel_variance4x4) +#endif // CONFIG_MOTION_VAR + +#if CONFIG_EXT_INTER +#define MBFP(BT, MSDF, MVF, MSVF) \ + cpi->fn_ptr[BT].msdf = MSDF; \ + cpi->fn_ptr[BT].mvf = MVF; \ + cpi->fn_ptr[BT].msvf = MSVF; + +#if CONFIG_EXT_PARTITION + MBFP(BLOCK_128X128, aom_masked_sad128x128, aom_masked_variance128x128, + aom_masked_sub_pixel_variance128x128) + MBFP(BLOCK_128X64, aom_masked_sad128x64, aom_masked_variance128x64, + aom_masked_sub_pixel_variance128x64) + MBFP(BLOCK_64X128, aom_masked_sad64x128, aom_masked_variance64x128, + aom_masked_sub_pixel_variance64x128) +#endif // CONFIG_EXT_PARTITION + MBFP(BLOCK_64X64, aom_masked_sad64x64, aom_masked_variance64x64, + aom_masked_sub_pixel_variance64x64) + MBFP(BLOCK_64X32, aom_masked_sad64x32, aom_masked_variance64x32, + aom_masked_sub_pixel_variance64x32) + MBFP(BLOCK_32X64, aom_masked_sad32x64, aom_masked_variance32x64, + aom_masked_sub_pixel_variance32x64) + MBFP(BLOCK_32X32, aom_masked_sad32x32, aom_masked_variance32x32, + aom_masked_sub_pixel_variance32x32) + MBFP(BLOCK_32X16, aom_masked_sad32x16, aom_masked_variance32x16, + aom_masked_sub_pixel_variance32x16) + MBFP(BLOCK_16X32, aom_masked_sad16x32, aom_masked_variance16x32, + aom_masked_sub_pixel_variance16x32) + MBFP(BLOCK_16X16, aom_masked_sad16x16, aom_masked_variance16x16, + aom_masked_sub_pixel_variance16x16) + MBFP(BLOCK_16X8, aom_masked_sad16x8, aom_masked_variance16x8, + aom_masked_sub_pixel_variance16x8) + MBFP(BLOCK_8X16, aom_masked_sad8x16, aom_masked_variance8x16, + aom_masked_sub_pixel_variance8x16) + MBFP(BLOCK_8X8, aom_masked_sad8x8, aom_masked_variance8x8, + aom_masked_sub_pixel_variance8x8) + MBFP(BLOCK_4X8, aom_masked_sad4x8, aom_masked_variance4x8, + aom_masked_sub_pixel_variance4x8) + MBFP(BLOCK_8X4, aom_masked_sad8x4, aom_masked_variance8x4, + aom_masked_sub_pixel_variance8x4) + MBFP(BLOCK_4X4, aom_masked_sad4x4, aom_masked_variance4x4, + aom_masked_sub_pixel_variance4x4) +#endif // CONFIG_EXT_INTER + +#if CONFIG_HIGHBITDEPTH + highbd_set_var_fns(cpi); +#endif + + /* av1_init_quantizer() is first called here. Add check in + * av1_frame_init_quantizer() so that av1_init_quantizer is only + * called later when needed. This will avoid unnecessary calls of + * av1_init_quantizer() for every frame. + */ + av1_init_quantizer(cpi); +#if CONFIG_AOM_QM + aom_qm_init(cm); +#endif + + av1_loop_filter_init(cm); +#if CONFIG_LOOP_RESTORATION + av1_loop_restoration_precal(); +#endif // CONFIG_LOOP_RESTORATION + + cm->error.setjmp = 0; + + return cpi; +} + +#define SNPRINT(H, T) snprintf((H) + strlen(H), sizeof(H) - strlen(H), (T)) + +#define SNPRINT2(H, T, V) \ + snprintf((H) + strlen(H), sizeof(H) - strlen(H), (T), (V)) + +void av1_remove_compressor(AV1_COMP *cpi) { + AV1_COMMON *cm; + unsigned int i; + int t; + + if (!cpi) return; + + cm = &cpi->common; + if (cm->current_video_frame > 0) { +#if CONFIG_ENTROPY_STATS + if (cpi->oxcf.pass != 1) { + fprintf(stderr, "Writing counts.stt\n"); + FILE *f = fopen("counts.stt", "wb"); + fwrite(&aggregate_fc, sizeof(aggregate_fc), 1, f); + fclose(f); + } +#endif // CONFIG_ENTROPY_STATS +#if CONFIG_INTERNAL_STATS + aom_clear_system_state(); + + if (cpi->oxcf.pass != 1) { + char headings[512] = { 0 }; + char results[512] = { 0 }; + FILE *f = fopen("opsnr.stt", "a"); + double time_encoded = + (cpi->last_end_time_stamp_seen - cpi->first_time_stamp_ever) / + 10000000.000; + double total_encode_time = + (cpi->time_receive_data + cpi->time_compress_data) / 1000.000; + const double dr = + (double)cpi->bytes * (double)8 / (double)1000 / time_encoded; + const double peak = (double)((1 << cpi->oxcf.input_bit_depth) - 1); + const double target_rate = (double)cpi->oxcf.target_bandwidth / 1000; + const double rate_err = ((100.0 * (dr - target_rate)) / target_rate); + + if (cpi->b_calculate_psnr) { + const double total_psnr = aom_sse_to_psnr( + (double)cpi->total_samples, peak, (double)cpi->total_sq_error); + const double total_ssim = + 100 * pow(cpi->summed_quality / cpi->summed_weights, 8.0); + snprintf(headings, sizeof(headings), + "Bitrate\tAVGPsnr\tGLBPsnr\tAVPsnrP\tGLPsnrP\t" + "AOMSSIM\tVPSSIMP\tFASTSIM\tPSNRHVS\t" + "WstPsnr\tWstSsim\tWstFast\tWstHVS"); + snprintf(results, sizeof(results), + "%7.2f\t%7.3f\t%7.3f\t%7.3f\t%7.3f\t" + "%7.3f\t%7.3f\t%7.3f\t%7.3f\t" + "%7.3f\t%7.3f\t%7.3f\t%7.3f", + dr, cpi->psnr.stat[ALL] / cpi->count, total_psnr, + cpi->psnr.stat[ALL] / cpi->count, total_psnr, total_ssim, + total_ssim, cpi->fastssim.stat[ALL] / cpi->count, + cpi->psnrhvs.stat[ALL] / cpi->count, cpi->psnr.worst, + cpi->worst_ssim, cpi->fastssim.worst, cpi->psnrhvs.worst); + + if (cpi->b_calculate_blockiness) { + SNPRINT(headings, "\t Block\tWstBlck"); + SNPRINT2(results, "\t%7.3f", cpi->total_blockiness / cpi->count); + SNPRINT2(results, "\t%7.3f", cpi->worst_blockiness); + } + + if (cpi->b_calculate_consistency) { + double consistency = + aom_sse_to_psnr((double)cpi->total_samples, peak, + (double)cpi->total_inconsistency); + + SNPRINT(headings, "\tConsist\tWstCons"); + SNPRINT2(results, "\t%7.3f", consistency); + SNPRINT2(results, "\t%7.3f", cpi->worst_consistency); + } + fprintf(f, "%s\t Time\tRcErr\tAbsErr\n", headings); + fprintf(f, "%s\t%8.0f\t%7.2f\t%7.2f\n", results, total_encode_time, + rate_err, fabs(rate_err)); + } + + fclose(f); + } + +#endif + +#if 0 + { + printf("\n_pick_loop_filter_level:%d\n", cpi->time_pick_lpf / 1000); + printf("\n_frames recive_data encod_mb_row compress_frame Total\n"); + printf("%6d %10ld %10ld %10ld %10ld\n", cpi->common.current_video_frame, + cpi->time_receive_data / 1000, cpi->time_encode_sb_row / 1000, + cpi->time_compress_data / 1000, + (cpi->time_receive_data + cpi->time_compress_data) / 1000); + } +#endif + } + + for (t = 0; t < cpi->num_workers; ++t) { + AVxWorker *const worker = &cpi->workers[t]; + EncWorkerData *const thread_data = &cpi->tile_thr_data[t]; + + // Deallocate allocated threads. + aom_get_worker_interface()->end(worker); + + // Deallocate allocated thread data. + if (t < cpi->num_workers - 1) { +#if CONFIG_PALETTE + if (cpi->common.allow_screen_content_tools) + aom_free(thread_data->td->mb.palette_buffer); +#endif // CONFIG_PALETTE + aom_free(thread_data->td->counts); + av1_free_pc_tree(thread_data->td); + av1_free_var_tree(thread_data->td); + aom_free(thread_data->td); + } + } + aom_free(cpi->tile_thr_data); + aom_free(cpi->workers); + + if (cpi->num_workers > 1) av1_loop_filter_dealloc(&cpi->lf_row_sync); + + dealloc_compressor_data(cpi); + + for (i = 0; i < sizeof(cpi->mbgraph_stats) / sizeof(cpi->mbgraph_stats[0]); + ++i) { + aom_free(cpi->mbgraph_stats[i].mb_stats); + } + +#if CONFIG_FP_MB_STATS + if (cpi->use_fp_mb_stats) { + aom_free(cpi->twopass.frame_mb_stats_buf); + cpi->twopass.frame_mb_stats_buf = NULL; + } +#endif +#if CONFIG_INTERNAL_STATS + aom_free(cpi->ssim_vars); + cpi->ssim_vars = NULL; +#endif // CONFIG_INTERNAL_STATS + + av1_remove_common(cm); + av1_free_ref_frame_buffers(cm->buffer_pool); + aom_free(cpi); + +#ifdef OUTPUT_YUV_SKINMAP + fclose(yuv_skinmap_file); +#endif +#ifdef OUTPUT_YUV_REC + fclose(yuv_rec_file); +#endif + +#if 0 + + if (keyfile) + fclose(keyfile); + + if (framepsnr) + fclose(framepsnr); + + if (kf_list) + fclose(kf_list); + +#endif +} + +static void generate_psnr_packet(AV1_COMP *cpi) { + struct aom_codec_cx_pkt pkt; + int i; + PSNR_STATS psnr; +#if CONFIG_HIGHBITDEPTH + aom_calc_highbd_psnr(cpi->source, cpi->common.frame_to_show, &psnr, + cpi->td.mb.e_mbd.bd, cpi->oxcf.input_bit_depth); +#else + aom_calc_psnr(cpi->source, cpi->common.frame_to_show, &psnr); +#endif + + for (i = 0; i < 4; ++i) { + pkt.data.psnr.samples[i] = psnr.samples[i]; + pkt.data.psnr.sse[i] = psnr.sse[i]; + pkt.data.psnr.psnr[i] = psnr.psnr[i]; + } + pkt.kind = AOM_CODEC_PSNR_PKT; + aom_codec_pkt_list_add(cpi->output_pkt_list, &pkt); +} + +int av1_use_as_reference(AV1_COMP *cpi, int ref_frame_flags) { + if (ref_frame_flags > ((1 << INTER_REFS_PER_FRAME) - 1)) return -1; + + cpi->ref_frame_flags = ref_frame_flags; + return 0; +} + +void av1_update_reference(AV1_COMP *cpi, int ref_frame_flags) { + cpi->ext_refresh_golden_frame = (ref_frame_flags & AOM_GOLD_FLAG) != 0; + cpi->ext_refresh_alt_ref_frame = (ref_frame_flags & AOM_ALT_FLAG) != 0; + cpi->ext_refresh_last_frame = (ref_frame_flags & AOM_LAST_FLAG) != 0; + cpi->ext_refresh_frame_flags_pending = 1; +} + +static YV12_BUFFER_CONFIG *get_av1_ref_frame_buffer( + AV1_COMP *cpi, AOM_REFFRAME ref_frame_flag) { + MV_REFERENCE_FRAME ref_frame = NONE_FRAME; + if (ref_frame_flag == AOM_LAST_FLAG) ref_frame = LAST_FRAME; +#if CONFIG_EXT_REFS + else if (ref_frame_flag == AOM_LAST2_FLAG) + ref_frame = LAST2_FRAME; + else if (ref_frame_flag == AOM_LAST3_FLAG) + ref_frame = LAST3_FRAME; +#endif // CONFIG_EXT_REFS + else if (ref_frame_flag == AOM_GOLD_FLAG) + ref_frame = GOLDEN_FRAME; +#if CONFIG_EXT_REFS + else if (ref_frame_flag == AOM_BWD_FLAG) + ref_frame = BWDREF_FRAME; +#endif // CONFIG_EXT_REFS + else if (ref_frame_flag == AOM_ALT_FLAG) + ref_frame = ALTREF_FRAME; + + return ref_frame == NONE_FRAME ? NULL : get_ref_frame_buffer(cpi, ref_frame); +} + +int av1_copy_reference_enc(AV1_COMP *cpi, AOM_REFFRAME ref_frame_flag, + YV12_BUFFER_CONFIG *sd) { + YV12_BUFFER_CONFIG *cfg = get_av1_ref_frame_buffer(cpi, ref_frame_flag); + if (cfg) { + aom_yv12_copy_frame(cfg, sd); + return 0; + } else { + return -1; + } +} + +int av1_set_reference_enc(AV1_COMP *cpi, AOM_REFFRAME ref_frame_flag, + YV12_BUFFER_CONFIG *sd) { + YV12_BUFFER_CONFIG *cfg = get_av1_ref_frame_buffer(cpi, ref_frame_flag); + if (cfg) { + aom_yv12_copy_frame(sd, cfg); + return 0; + } else { + return -1; + } +} + +int av1_update_entropy(AV1_COMP *cpi, int update) { + cpi->ext_refresh_frame_context = update; + cpi->ext_refresh_frame_context_pending = 1; + return 0; +} + +#if defined(OUTPUT_YUV_DENOISED) || defined(OUTPUT_YUV_SKINMAP) +// The denoiser buffer is allocated as a YUV 440 buffer. This function writes it +// as YUV 420. We simply use the top-left pixels of the UV buffers, since we do +// not denoise the UV channels at this time. If ever we implement UV channel +// denoising we will have to modify this. +void aom_write_yuv_frame_420(YV12_BUFFER_CONFIG *s, FILE *f) { + uint8_t *src = s->y_buffer; + int h = s->y_height; + + do { + fwrite(src, s->y_width, 1, f); + src += s->y_stride; + } while (--h); + + src = s->u_buffer; + h = s->uv_height; + + do { + fwrite(src, s->uv_width, 1, f); + src += s->uv_stride; + } while (--h); + + src = s->v_buffer; + h = s->uv_height; + + do { + fwrite(src, s->uv_width, 1, f); + src += s->uv_stride; + } while (--h); +} +#endif + +#if CONFIG_EXT_REFS && !CONFIG_XIPHRC +static void check_show_existing_frame(AV1_COMP *cpi) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + AV1_COMMON *const cm = &cpi->common; + const FRAME_UPDATE_TYPE next_frame_update_type = + gf_group->update_type[gf_group->index]; + const int which_arf = gf_group->arf_update_idx[gf_group->index]; + + if (cm->show_existing_frame == 1) { + cm->show_existing_frame = 0; + } else if (cpi->rc.is_last_bipred_frame) { + // NOTE(zoeliu): If the current frame is a last bi-predictive frame, it is + // needed next to show the BWDREF_FRAME, which is pointed by + // the last_fb_idxes[0] after reference frame buffer update + cpi->rc.is_last_bipred_frame = 0; + cm->show_existing_frame = 1; + cpi->existing_fb_idx_to_show = cpi->lst_fb_idxes[0]; + } else if (cpi->is_arf_filter_off[which_arf] && + (next_frame_update_type == OVERLAY_UPDATE || + next_frame_update_type == INTNL_OVERLAY_UPDATE)) { + // Other parameters related to OVERLAY_UPDATE will be taken care of + // in av1_rc_get_second_pass_params(cpi) + cm->show_existing_frame = 1; + cpi->rc.is_src_frame_alt_ref = 1; + cpi->existing_fb_idx_to_show = cpi->alt_fb_idx; + cpi->is_arf_filter_off[which_arf] = 0; + } + cpi->rc.is_src_frame_ext_arf = 0; +} +#endif // CONFIG_EXT_REFS && !CONFIG_XIPHRC + +#ifdef OUTPUT_YUV_REC +void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + uint8_t *src = s->y_buffer; + int h = cm->height; + +#if CONFIG_HIGHBITDEPTH + if (s->flags & YV12_FLAG_HIGHBITDEPTH) { + uint16_t *src16 = CONVERT_TO_SHORTPTR(s->y_buffer); + + do { + fwrite(src16, s->y_width, 2, yuv_rec_file); + src16 += s->y_stride; + } while (--h); + + src16 = CONVERT_TO_SHORTPTR(s->u_buffer); + h = s->uv_height; + + do { + fwrite(src16, s->uv_width, 2, yuv_rec_file); + src16 += s->uv_stride; + } while (--h); + + src16 = CONVERT_TO_SHORTPTR(s->v_buffer); + h = s->uv_height; + + do { + fwrite(src16, s->uv_width, 2, yuv_rec_file); + src16 += s->uv_stride; + } while (--h); + + fflush(yuv_rec_file); + return; + } +#endif // CONFIG_HIGHBITDEPTH + + do { + fwrite(src, s->y_width, 1, yuv_rec_file); + src += s->y_stride; + } while (--h); + + src = s->u_buffer; + h = s->uv_height; + + do { + fwrite(src, s->uv_width, 1, yuv_rec_file); + src += s->uv_stride; + } while (--h); + + src = s->v_buffer; + h = s->uv_height; + + do { + fwrite(src, s->uv_width, 1, yuv_rec_file); + src += s->uv_stride; + } while (--h); + + fflush(yuv_rec_file); +} +#endif // OUTPUT_YUV_REC + +#if CONFIG_HIGHBITDEPTH +static void scale_and_extend_frame_nonnormative(const YV12_BUFFER_CONFIG *src, + YV12_BUFFER_CONFIG *dst, + int bd) { +#else +static void scale_and_extend_frame_nonnormative(const YV12_BUFFER_CONFIG *src, + YV12_BUFFER_CONFIG *dst) { +#endif // CONFIG_HIGHBITDEPTH + // TODO(dkovalev): replace YV12_BUFFER_CONFIG with aom_image_t + int i; + const uint8_t *const srcs[3] = { src->y_buffer, src->u_buffer, + src->v_buffer }; + const int src_strides[3] = { src->y_stride, src->uv_stride, src->uv_stride }; + const int src_widths[3] = { src->y_crop_width, src->uv_crop_width, + src->uv_crop_width }; + const int src_heights[3] = { src->y_crop_height, src->uv_crop_height, + src->uv_crop_height }; + uint8_t *const dsts[3] = { dst->y_buffer, dst->u_buffer, dst->v_buffer }; + const int dst_strides[3] = { dst->y_stride, dst->uv_stride, dst->uv_stride }; + const int dst_widths[3] = { dst->y_crop_width, dst->uv_crop_width, + dst->uv_crop_width }; + const int dst_heights[3] = { dst->y_crop_height, dst->uv_crop_height, + dst->uv_crop_height }; + + for (i = 0; i < MAX_MB_PLANE; ++i) { +#if CONFIG_HIGHBITDEPTH + if (src->flags & YV12_FLAG_HIGHBITDEPTH) { + av1_highbd_resize_plane(srcs[i], src_heights[i], src_widths[i], + src_strides[i], dsts[i], dst_heights[i], + dst_widths[i], dst_strides[i], bd); + } else { + av1_resize_plane(srcs[i], src_heights[i], src_widths[i], src_strides[i], + dsts[i], dst_heights[i], dst_widths[i], dst_strides[i]); + } +#else + av1_resize_plane(srcs[i], src_heights[i], src_widths[i], src_strides[i], + dsts[i], dst_heights[i], dst_widths[i], dst_strides[i]); +#endif // CONFIG_HIGHBITDEPTH + } + aom_extend_frame_borders(dst); +} + +#if CONFIG_HIGHBITDEPTH +static void scale_and_extend_frame(const YV12_BUFFER_CONFIG *src, + YV12_BUFFER_CONFIG *dst, int planes, + int bd) { +#else +static void scale_and_extend_frame(const YV12_BUFFER_CONFIG *src, + YV12_BUFFER_CONFIG *dst, int planes) { +#endif // CONFIG_HIGHBITDEPTH + const int src_w = src->y_crop_width; + const int src_h = src->y_crop_height; + const int dst_w = dst->y_crop_width; + const int dst_h = dst->y_crop_height; + const uint8_t *const srcs[3] = { src->y_buffer, src->u_buffer, + src->v_buffer }; + const int src_strides[3] = { src->y_stride, src->uv_stride, src->uv_stride }; + uint8_t *const dsts[3] = { dst->y_buffer, dst->u_buffer, dst->v_buffer }; + const int dst_strides[3] = { dst->y_stride, dst->uv_stride, dst->uv_stride }; + const InterpFilterParams interp_filter_params = + av1_get_interp_filter_params(EIGHTTAP_REGULAR); + const int16_t *kernel = interp_filter_params.filter_ptr; + const int taps = interp_filter_params.taps; + int x, y, i; + + assert(planes <= 3); + for (y = 0; y < dst_h; y += 16) { + for (x = 0; x < dst_w; x += 16) { + for (i = 0; i < planes; ++i) { + const int factor = (i == 0 || i == 3 ? 1 : 2); + const int x_q4 = x * (16 / factor) * src_w / dst_w; + const int y_q4 = y * (16 / factor) * src_h / dst_h; + const int src_stride = src_strides[i]; + const int dst_stride = dst_strides[i]; + const uint8_t *src_ptr = srcs[i] + + (y / factor) * src_h / dst_h * src_stride + + (x / factor) * src_w / dst_w; + uint8_t *dst_ptr = dsts[i] + (y / factor) * dst_stride + (x / factor); + +#if CONFIG_HIGHBITDEPTH + if (src->flags & YV12_FLAG_HIGHBITDEPTH) { + aom_highbd_convolve8(src_ptr, src_stride, dst_ptr, dst_stride, + &kernel[(x_q4 & 0xf) * taps], 16 * src_w / dst_w, + &kernel[(y_q4 & 0xf) * taps], 16 * src_h / dst_h, + 16 / factor, 16 / factor, bd); + } else { + aom_scaled_2d(src_ptr, src_stride, dst_ptr, dst_stride, + &kernel[(x_q4 & 0xf) * taps], 16 * src_w / dst_w, + &kernel[(y_q4 & 0xf) * taps], 16 * src_h / dst_h, + 16 / factor, 16 / factor); + } +#else + aom_scaled_2d(src_ptr, src_stride, dst_ptr, dst_stride, + &kernel[(x_q4 & 0xf) * taps], 16 * src_w / dst_w, + &kernel[(y_q4 & 0xf) * taps], 16 * src_h / dst_h, + 16 / factor, 16 / factor); +#endif // CONFIG_HIGHBITDEPTH + } + } + } + + if (planes == 1) + aom_extend_frame_borders_y(dst); + else + aom_extend_frame_borders(dst); +} + +static int scale_down(AV1_COMP *cpi, int q) { + RATE_CONTROL *const rc = &cpi->rc; + GF_GROUP *const gf_group = &cpi->twopass.gf_group; + int scale = 0; + assert(frame_is_kf_gf_arf(cpi)); + + if (rc->frame_size_selector == UNSCALED && + q >= rc->rf_level_maxq[gf_group->rf_level[gf_group->index]]) { + const int max_size_thresh = + (int)(rate_thresh_mult[SCALE_STEP1] * + AOMMAX(rc->this_frame_target, rc->avg_frame_bandwidth)); + scale = rc->projected_frame_size > max_size_thresh ? 1 : 0; + } + return scale; +} + +#if CONFIG_GLOBAL_MOTION +#define GM_RECODE_LOOP_NUM4X4_FACTOR 192 +static int recode_loop_test_global_motion(AV1_COMP *cpi) { + int i; + int recode = 0; + RD_COUNTS *const rdc = &cpi->td.rd_counts; + AV1_COMMON *const cm = &cpi->common; + for (i = LAST_FRAME; i <= ALTREF_FRAME; ++i) { + if (cm->global_motion[i].wmtype != IDENTITY && + rdc->global_motion_used[i] * GM_RECODE_LOOP_NUM4X4_FACTOR < + cpi->gmparams_cost[i]) { + set_default_warp_params(&cm->global_motion[i]); + cpi->gmparams_cost[i] = 0; +#if CONFIG_REF_MV + recode = 1; +#else + recode |= (rdc->global_motion_used[i] > 0); +#endif + } + } + return recode; +} +#endif // CONFIG_GLOBAL_MOTION + +// Function to test for conditions that indicate we should loop +// back and recode a frame. +static int recode_loop_test(AV1_COMP *cpi, int high_limit, int low_limit, int q, + int maxq, int minq) { + const RATE_CONTROL *const rc = &cpi->rc; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + const int frame_is_kfgfarf = frame_is_kf_gf_arf(cpi); + int force_recode = 0; + + if ((rc->projected_frame_size >= rc->max_frame_bandwidth) || + (cpi->sf.recode_loop == ALLOW_RECODE) || + (frame_is_kfgfarf && (cpi->sf.recode_loop == ALLOW_RECODE_KFARFGF))) { + if (frame_is_kfgfarf && (oxcf->resize_mode == RESIZE_DYNAMIC) && + scale_down(cpi, q)) { + // Code this group at a lower resolution. + cpi->resize_pending = 1; + return 1; + } + + // TODO(agrange) high_limit could be greater than the scale-down threshold. + if ((rc->projected_frame_size > high_limit && q < maxq) || + (rc->projected_frame_size < low_limit && q > minq)) { + force_recode = 1; + } else if (cpi->oxcf.rc_mode == AOM_CQ) { + // Deal with frame undershoot and whether or not we are + // below the automatically set cq level. + if (q > oxcf->cq_level && + rc->projected_frame_size < ((rc->this_frame_target * 7) >> 3)) { + force_recode = 1; + } + } + } + return force_recode; +} + +static INLINE int get_free_upsampled_ref_buf(EncRefCntBuffer *ubufs) { + int i; + + for (i = 0; i < (REF_FRAMES + 1); i++) { + if (!ubufs[i].ref_count) { + return i; + } + } + return INVALID_IDX; +} + +// Up-sample 1 reference frame. +static INLINE int upsample_ref_frame(AV1_COMP *cpi, + const YV12_BUFFER_CONFIG *const ref) { + AV1_COMMON *const cm = &cpi->common; + EncRefCntBuffer *ubufs = cpi->upsampled_ref_bufs; + int new_uidx = get_free_upsampled_ref_buf(ubufs); + + if (new_uidx == INVALID_IDX) { + return INVALID_IDX; + } else { + YV12_BUFFER_CONFIG *upsampled_ref = &ubufs[new_uidx].buf; + + // Can allocate buffer for Y plane only. + if (upsampled_ref->buffer_alloc_sz < (ref->buffer_alloc_sz << 6)) + if (aom_realloc_frame_buffer(upsampled_ref, (cm->width << 3), + (cm->height << 3), cm->subsampling_x, + cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + (AOM_BORDER_IN_PIXELS << 3), + cm->byte_alignment, NULL, NULL, NULL)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate up-sampled frame buffer"); + +// Currently, only Y plane is up-sampled, U, V are not used. +#if CONFIG_HIGHBITDEPTH + scale_and_extend_frame(ref, upsampled_ref, 1, (int)cm->bit_depth); +#else + scale_and_extend_frame(ref, upsampled_ref, 1); +#endif + return new_uidx; + } +} + +#define DUMP_REF_FRAME_IMAGES 0 + +#if DUMP_REF_FRAME_IMAGES == 1 +static int dump_one_image(AV1_COMMON *cm, + const YV12_BUFFER_CONFIG *const ref_buf, + char *file_name) { + int h; + FILE *f_ref = NULL; + + if (ref_buf == NULL) { + printf("Frame data buffer is NULL.\n"); + return AOM_CODEC_MEM_ERROR; + } + + if ((f_ref = fopen(file_name, "wb")) == NULL) { + printf("Unable to open file %s to write.\n", file_name); + return AOM_CODEC_MEM_ERROR; + } + + // --- Y --- + for (h = 0; h < cm->height; ++h) { + fwrite(&ref_buf->y_buffer[h * ref_buf->y_stride], 1, cm->width, f_ref); + } + // --- U --- + for (h = 0; h < (cm->height >> 1); ++h) { + fwrite(&ref_buf->u_buffer[h * ref_buf->uv_stride], 1, (cm->width >> 1), + f_ref); + } + // --- V --- + for (h = 0; h < (cm->height >> 1); ++h) { + fwrite(&ref_buf->v_buffer[h * ref_buf->uv_stride], 1, (cm->width >> 1), + f_ref); + } + + fclose(f_ref); + + return AOM_CODEC_OK; +} + +static void dump_ref_frame_images(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + MV_REFERENCE_FRAME ref_frame; + + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { + char file_name[256] = ""; + snprintf(file_name, sizeof(file_name), "/tmp/enc_F%d_ref_%d.yuv", + cm->current_video_frame, ref_frame); + dump_one_image(cm, get_ref_frame_buffer(cpi, ref_frame), file_name); + } +} +#endif // DUMP_REF_FRAME_IMAGES == 1 + +#if CONFIG_EXT_REFS +// This function is used to shift the virtual indices of last reference frames +// as follows: +// LAST_FRAME -> LAST2_FRAME -> LAST3_FRAME +// when the LAST_FRAME is updated. +static INLINE void shift_last_ref_frames(AV1_COMP *cpi) { + int ref_frame; + for (ref_frame = LAST_REF_FRAMES - 1; ref_frame > 0; --ref_frame) { + cpi->lst_fb_idxes[ref_frame] = cpi->lst_fb_idxes[ref_frame - 1]; + + // [0] is allocated to the current coded frame. The statistics for the + // reference frames start at [LAST_FRAME], i.e. [1]. + if (!cpi->rc.is_src_frame_alt_ref) { + memcpy(cpi->interp_filter_selected[ref_frame + LAST_FRAME], + cpi->interp_filter_selected[ref_frame - 1 + LAST_FRAME], + sizeof(cpi->interp_filter_selected[ref_frame - 1 + LAST_FRAME])); + } + } +} +#endif // CONFIG_EXT_REFS + +void av1_update_reference_frames(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + BufferPool *const pool = cm->buffer_pool; + const int use_upsampled_ref = cpi->sf.use_upsampled_references; + int new_uidx = 0; + + // NOTE: Save the new show frame buffer index for --test-code=warn, i.e., + // for the purpose to verify no mismatch between encoder and decoder. + if (cm->show_frame) cpi->last_show_frame_buf_idx = cm->new_fb_idx; + + if (use_upsampled_ref) { +#if CONFIG_EXT_REFS + if (cm->show_existing_frame) { + new_uidx = cpi->upsampled_ref_idx[cpi->existing_fb_idx_to_show]; + // TODO(zoeliu): Once following is confirmed, remove it. + assert(cpi->upsampled_ref_bufs[new_uidx].ref_count > 0); + } else { +#endif // CONFIG_EXT_REFS + // Up-sample the current encoded frame. + RefCntBuffer *bufs = pool->frame_bufs; + const YV12_BUFFER_CONFIG *const ref = &bufs[cm->new_fb_idx].buf; + + new_uidx = upsample_ref_frame(cpi, ref); +#if CONFIG_EXT_REFS + assert(new_uidx != INVALID_IDX); + } +#endif // CONFIG_EXT_REFS + } + // At this point the new frame has been encoded. + // If any buffer copy / swapping is signaled it should be done here. + if (cm->frame_type == KEY_FRAME) { + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx], + cm->new_fb_idx); +#if CONFIG_EXT_REFS + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->bwd_fb_idx], + cm->new_fb_idx); +#endif // CONFIG_EXT_REFS + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->alt_fb_idx], + cm->new_fb_idx); + + if (use_upsampled_ref) { + uref_cnt_fb(cpi->upsampled_ref_bufs, + &cpi->upsampled_ref_idx[cpi->gld_fb_idx], new_uidx); +#if CONFIG_EXT_REFS + uref_cnt_fb(cpi->upsampled_ref_bufs, + &cpi->upsampled_ref_idx[cpi->bwd_fb_idx], new_uidx); +#endif // CONFIG_EXT_REFS + uref_cnt_fb(cpi->upsampled_ref_bufs, + &cpi->upsampled_ref_idx[cpi->alt_fb_idx], new_uidx); + } + } else if (av1_preserve_existing_gf(cpi)) { + // We have decided to preserve the previously existing golden frame as our + // new ARF frame. However, in the short term in function + // av1_bitstream.c::get_refresh_mask() we left it in the GF slot and, if + // we're updating the GF with the current decoded frame, we save it to the + // ARF slot instead. + // We now have to update the ARF with the current frame and swap gld_fb_idx + // and alt_fb_idx so that, overall, we've stored the old GF in the new ARF + // slot and, if we're updating the GF, the current frame becomes the new GF. + int tmp; + + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->alt_fb_idx], + cm->new_fb_idx); + if (use_upsampled_ref) + uref_cnt_fb(cpi->upsampled_ref_bufs, + &cpi->upsampled_ref_idx[cpi->alt_fb_idx], new_uidx); + + tmp = cpi->alt_fb_idx; + cpi->alt_fb_idx = cpi->gld_fb_idx; + cpi->gld_fb_idx = tmp; + +#if CONFIG_EXT_REFS + // We need to modify the mapping accordingly + cpi->arf_map[0] = cpi->alt_fb_idx; +#endif +// TODO(zoeliu): Do we need to copy cpi->interp_filter_selected[0] over to +// cpi->interp_filter_selected[GOLDEN_FRAME]? +#if CONFIG_EXT_REFS + } else if (cpi->rc.is_last_bipred_frame) { + // Refresh the LAST_FRAME with the BWDREF_FRAME and retire the LAST3_FRAME + // by updating the virtual indices. Note that the frame BWDREF_FRAME points + // to now should be retired, and it should not be used before refreshed. + int tmp = cpi->lst_fb_idxes[LAST_REF_FRAMES - 1]; + + shift_last_ref_frames(cpi); + cpi->lst_fb_idxes[0] = cpi->bwd_fb_idx; + cpi->bwd_fb_idx = tmp; + + memcpy(cpi->interp_filter_selected[LAST_FRAME], + cpi->interp_filter_selected[BWDREF_FRAME], + sizeof(cpi->interp_filter_selected[BWDREF_FRAME])); + } else if (cpi->rc.is_src_frame_ext_arf && cm->show_existing_frame) { + // Deal with the special case for showing existing internal ALTREF_FRAME + // Refresh the LAST_FRAME with the ALTREF_FRAME and retire the LAST3_FRAME + // by updating the virtual indices. + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + int which_arf = gf_group->arf_ref_idx[gf_group->index]; + int tmp = cpi->lst_fb_idxes[LAST_REF_FRAMES - 1]; + + shift_last_ref_frames(cpi); + cpi->lst_fb_idxes[0] = cpi->alt_fb_idx; + cpi->alt_fb_idx = tmp; + + // We need to modify the mapping accordingly + cpi->arf_map[which_arf] = cpi->alt_fb_idx; + + memcpy(cpi->interp_filter_selected[LAST_FRAME], + cpi->interp_filter_selected[ALTREF_FRAME + which_arf], + sizeof(cpi->interp_filter_selected[ALTREF_FRAME + which_arf])); +#endif // CONFIG_EXT_REFS + } else { /* For non key/golden frames */ + if (cpi->refresh_alt_ref_frame) { + int arf_idx = cpi->alt_fb_idx; + int which_arf = 0; +#if CONFIG_EXT_REFS + if (cpi->oxcf.pass == 2) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + which_arf = gf_group->arf_update_idx[gf_group->index]; + arf_idx = cpi->arf_map[which_arf]; + } +#else + if ((cpi->oxcf.pass == 2) && cpi->multi_arf_allowed) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + arf_idx = gf_group->arf_update_idx[gf_group->index]; + } +#endif // CONFIG_EXT_REFS + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[arf_idx], cm->new_fb_idx); + if (use_upsampled_ref) + uref_cnt_fb(cpi->upsampled_ref_bufs, &cpi->upsampled_ref_idx[arf_idx], + new_uidx); + + memcpy(cpi->interp_filter_selected[ALTREF_FRAME + which_arf], + cpi->interp_filter_selected[0], + sizeof(cpi->interp_filter_selected[0])); + } + + if (cpi->refresh_golden_frame) { + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx], + cm->new_fb_idx); + if (use_upsampled_ref) + uref_cnt_fb(cpi->upsampled_ref_bufs, + &cpi->upsampled_ref_idx[cpi->gld_fb_idx], new_uidx); + +#if !CONFIG_EXT_REFS + if (!cpi->rc.is_src_frame_alt_ref) +#endif // !CONFIG_EXT_REFS + memcpy(cpi->interp_filter_selected[GOLDEN_FRAME], + cpi->interp_filter_selected[0], + sizeof(cpi->interp_filter_selected[0])); + } + +#if CONFIG_EXT_REFS + if (cpi->refresh_bwd_ref_frame) { + if (cpi->rc.is_bwd_ref_frame && cpi->num_extra_arfs) { + // We have swapped the virtual indices to allow bwd_ref_frame to use + // ALT0 as reference frame. We need to swap them back. + // NOTE: The ALT_REFs' are indexed reversely, and ALT0 refers to the + // farthest ALT_REF from the first frame in the gf group. + int tmp = cpi->arf_map[0]; + cpi->arf_map[0] = cpi->alt_fb_idx; + cpi->alt_fb_idx = cpi->bwd_fb_idx; + cpi->bwd_fb_idx = tmp; + } + + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->bwd_fb_idx], + cm->new_fb_idx); + if (use_upsampled_ref) + uref_cnt_fb(cpi->upsampled_ref_bufs, + &cpi->upsampled_ref_idx[cpi->bwd_fb_idx], new_uidx); + + memcpy(cpi->interp_filter_selected[BWDREF_FRAME], + cpi->interp_filter_selected[0], + sizeof(cpi->interp_filter_selected[0])); + } +#endif // CONFIG_EXT_REFS + } + + if (cpi->refresh_last_frame) { +#if CONFIG_EXT_REFS + // NOTE(zoeliu): We have two layers of mapping (1) from the per-frame + // reference to the reference frame buffer virtual index; and then (2) from + // the virtual index to the reference frame buffer physical index: + // + // LAST_FRAME, ..., LAST3_FRAME, ..., ALTREF_FRAME + // | | | + // v v v + // lst_fb_idxes[0], ..., lst_fb_idxes[2], ..., alt_fb_idx + // | | | + // v v v + // ref_frame_map[], ..., ref_frame_map[], ..., ref_frame_map[] + // + // When refresh_last_frame is set, it is intended to retire LAST3_FRAME, + // have the other 2 LAST reference frames shifted as follows: + // LAST_FRAME -> LAST2_FRAME -> LAST3_FRAME + // , and then have LAST_FRAME refreshed by the newly coded frame. + // + // To fulfill it, the decoder will be notified to execute following 2 steps: + // + // (a) To change ref_frame_map[] and have the virtual index of LAST3_FRAME + // to point to the newly coded frame, i.e. + // ref_frame_map[lst_fb_idexes[2]] => new_fb_idx; + // + // (b) To change the 1st layer mapping to have LAST_FRAME mapped to the + // original virtual index of LAST3_FRAME and have the other mappings + // shifted as follows: + // LAST_FRAME, LAST2_FRAME, LAST3_FRAME + // | | | + // v v v + // lst_fb_idxes[2], lst_fb_idxes[0], lst_fb_idxes[1] + int ref_frame; + + if (cpi->rc.is_bwd_ref_frame && cpi->num_extra_arfs) { + // We have swapped the virtual indices to use ALT0 as BWD_REF + // and we need to swap them back. + int tmp = cpi->arf_map[0]; + cpi->arf_map[0] = cpi->alt_fb_idx; + cpi->alt_fb_idx = cpi->bwd_fb_idx; + cpi->bwd_fb_idx = tmp; + } + + if (cm->frame_type == KEY_FRAME) { + for (ref_frame = 0; ref_frame < LAST_REF_FRAMES; ++ref_frame) { + ref_cnt_fb(pool->frame_bufs, + &cm->ref_frame_map[cpi->lst_fb_idxes[ref_frame]], + cm->new_fb_idx); + + if (use_upsampled_ref) + uref_cnt_fb(cpi->upsampled_ref_bufs, + &cpi->upsampled_ref_idx[cpi->lst_fb_idxes[ref_frame]], + new_uidx); + } + } else { + int tmp; + + ref_cnt_fb(pool->frame_bufs, + &cm->ref_frame_map[cpi->lst_fb_idxes[LAST_REF_FRAMES - 1]], + cm->new_fb_idx); + + if (use_upsampled_ref) + uref_cnt_fb( + cpi->upsampled_ref_bufs, + &cpi->upsampled_ref_idx[cpi->lst_fb_idxes[LAST_REF_FRAMES - 1]], + new_uidx); + + tmp = cpi->lst_fb_idxes[LAST_REF_FRAMES - 1]; + + shift_last_ref_frames(cpi); + cpi->lst_fb_idxes[0] = tmp; + + assert(cm->show_existing_frame == 0); + // NOTE: Currently only LF_UPDATE and INTNL_OVERLAY_UPDATE frames are to + // refresh the LAST_FRAME. + memcpy(cpi->interp_filter_selected[LAST_FRAME], + cpi->interp_filter_selected[0], + sizeof(cpi->interp_filter_selected[0])); + } +#else + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->lst_fb_idx], + cm->new_fb_idx); + if (use_upsampled_ref) + uref_cnt_fb(cpi->upsampled_ref_bufs, + &cpi->upsampled_ref_idx[cpi->lst_fb_idx], new_uidx); + if (!cpi->rc.is_src_frame_alt_ref) { + memcpy(cpi->interp_filter_selected[LAST_FRAME], + cpi->interp_filter_selected[0], + sizeof(cpi->interp_filter_selected[0])); + } +#endif // CONFIG_EXT_REFS + } + +#if DUMP_REF_FRAME_IMAGES == 1 + // Dump out all reference frame images. + dump_ref_frame_images(cpi); +#endif // DUMP_REF_FRAME_IMAGES +} + +static void loopfilter_frame(AV1_COMP *cpi, AV1_COMMON *cm) { + MACROBLOCKD *xd = &cpi->td.mb.e_mbd; + struct loopfilter *lf = &cm->lf; + if (is_lossless_requested(&cpi->oxcf)) { + lf->filter_level = 0; + } else { + struct aom_usec_timer timer; + + aom_clear_system_state(); + + aom_usec_timer_start(&timer); + + av1_pick_filter_level(cpi->source, cpi, cpi->sf.lpf_pick); + + aom_usec_timer_mark(&timer); + cpi->time_pick_lpf += aom_usec_timer_elapsed(&timer); + } + + if (lf->filter_level > 0) { +#if CONFIG_VAR_TX || CONFIG_EXT_PARTITION || CONFIG_CB4X4 + av1_loop_filter_frame(cm->frame_to_show, cm, xd, lf->filter_level, 0, 0); +#else + if (cpi->num_workers > 1) + av1_loop_filter_frame_mt(cm->frame_to_show, cm, xd->plane, + lf->filter_level, 0, 0, cpi->workers, + cpi->num_workers, &cpi->lf_row_sync); + else + av1_loop_filter_frame(cm->frame_to_show, cm, xd, lf->filter_level, 0, 0); +#endif + } +#if CONFIG_CDEF + if (is_lossless_requested(&cpi->oxcf)) { + cm->cdef_bits = 0; + cm->cdef_strengths[0] = 0; + cm->nb_cdef_strengths = 1; + } else { + // Find cm->dering_level, cm->clpf_strength_u and cm->clpf_strength_v + av1_cdef_search(cm->frame_to_show, cpi->source, cm, xd); + + // Apply the filter + av1_cdef_frame(cm->frame_to_show, cm, xd); + } +#endif +#if CONFIG_LOOP_RESTORATION + av1_pick_filter_restoration(cpi->source, cpi, cpi->sf.lpf_pick); + if (cm->rst_info[0].frame_restoration_type != RESTORE_NONE || + cm->rst_info[1].frame_restoration_type != RESTORE_NONE || + cm->rst_info[2].frame_restoration_type != RESTORE_NONE) { + av1_loop_restoration_frame(cm->frame_to_show, cm, cm->rst_info, 7, 0, NULL); + } +#endif // CONFIG_LOOP_RESTORATION + aom_extend_frame_inner_borders(cm->frame_to_show); +} + +static INLINE void alloc_frame_mvs(AV1_COMMON *const cm, int buffer_idx) { + RefCntBuffer *const new_fb_ptr = &cm->buffer_pool->frame_bufs[buffer_idx]; + if (new_fb_ptr->mvs == NULL || new_fb_ptr->mi_rows < cm->mi_rows || + new_fb_ptr->mi_cols < cm->mi_cols) { + aom_free(new_fb_ptr->mvs); + CHECK_MEM_ERROR(cm, new_fb_ptr->mvs, + (MV_REF *)aom_calloc(cm->mi_rows * cm->mi_cols, + sizeof(*new_fb_ptr->mvs))); + new_fb_ptr->mi_rows = cm->mi_rows; + new_fb_ptr->mi_cols = cm->mi_cols; + } +} + +void av1_scale_references(AV1_COMP *cpi) { + AV1_COMMON *cm = &cpi->common; + MV_REFERENCE_FRAME ref_frame; + const AOM_REFFRAME ref_mask[INTER_REFS_PER_FRAME] = { + AOM_LAST_FLAG, +#if CONFIG_EXT_REFS + AOM_LAST2_FLAG, + AOM_LAST3_FLAG, +#endif // CONFIG_EXT_REFS + AOM_GOLD_FLAG, +#if CONFIG_EXT_REFS + AOM_BWD_FLAG, +#endif // CONFIG_EXT_REFS + AOM_ALT_FLAG + }; + + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { + // Need to convert from AOM_REFFRAME to index into ref_mask (subtract 1). + if (cpi->ref_frame_flags & ref_mask[ref_frame - 1]) { + BufferPool *const pool = cm->buffer_pool; + const YV12_BUFFER_CONFIG *const ref = + get_ref_frame_buffer(cpi, ref_frame); + + if (ref == NULL) { + cpi->scaled_ref_idx[ref_frame - 1] = INVALID_IDX; + continue; + } + +#if CONFIG_HIGHBITDEPTH + if (ref->y_crop_width != cm->width || ref->y_crop_height != cm->height) { + RefCntBuffer *new_fb_ptr = NULL; + int force_scaling = 0; + int new_fb = cpi->scaled_ref_idx[ref_frame - 1]; + if (new_fb == INVALID_IDX) { + new_fb = get_free_fb(cm); + force_scaling = 1; + } + if (new_fb == INVALID_IDX) return; + new_fb_ptr = &pool->frame_bufs[new_fb]; + if (force_scaling || new_fb_ptr->buf.y_crop_width != cm->width || + new_fb_ptr->buf.y_crop_height != cm->height) { + if (aom_realloc_frame_buffer( + &new_fb_ptr->buf, cm->width, cm->height, cm->subsampling_x, + cm->subsampling_y, cm->use_highbitdepth, AOM_BORDER_IN_PIXELS, + cm->byte_alignment, NULL, NULL, NULL)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate frame buffer"); + scale_and_extend_frame(ref, &new_fb_ptr->buf, MAX_MB_PLANE, + (int)cm->bit_depth); + cpi->scaled_ref_idx[ref_frame - 1] = new_fb; + alloc_frame_mvs(cm, new_fb); + } +#else + if (ref->y_crop_width != cm->width || ref->y_crop_height != cm->height) { + RefCntBuffer *new_fb_ptr = NULL; + int force_scaling = 0; + int new_fb = cpi->scaled_ref_idx[ref_frame - 1]; + if (new_fb == INVALID_IDX) { + new_fb = get_free_fb(cm); + force_scaling = 1; + } + if (new_fb == INVALID_IDX) return; + new_fb_ptr = &pool->frame_bufs[new_fb]; + if (force_scaling || new_fb_ptr->buf.y_crop_width != cm->width || + new_fb_ptr->buf.y_crop_height != cm->height) { + if (aom_realloc_frame_buffer(&new_fb_ptr->buf, cm->width, cm->height, + cm->subsampling_x, cm->subsampling_y, + AOM_BORDER_IN_PIXELS, cm->byte_alignment, + NULL, NULL, NULL)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate frame buffer"); + scale_and_extend_frame(ref, &new_fb_ptr->buf, MAX_MB_PLANE); + cpi->scaled_ref_idx[ref_frame - 1] = new_fb; + alloc_frame_mvs(cm, new_fb); + } +#endif // CONFIG_HIGHBITDEPTH + + if (cpi->sf.use_upsampled_references && + (force_scaling || new_fb_ptr->buf.y_crop_width != cm->width || + new_fb_ptr->buf.y_crop_height != cm->height)) { + const int map_idx = get_ref_frame_map_idx(cpi, ref_frame); + EncRefCntBuffer *ubuf = + &cpi->upsampled_ref_bufs[cpi->upsampled_ref_idx[map_idx]]; + + if (aom_realloc_frame_buffer(&ubuf->buf, (cm->width << 3), + (cm->height << 3), cm->subsampling_x, + cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + (AOM_BORDER_IN_PIXELS << 3), + cm->byte_alignment, NULL, NULL, NULL)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate up-sampled frame buffer"); +#if CONFIG_HIGHBITDEPTH + scale_and_extend_frame(&new_fb_ptr->buf, &ubuf->buf, 1, + (int)cm->bit_depth); +#else + scale_and_extend_frame(&new_fb_ptr->buf, &ubuf->buf, 1); +#endif + } + } else { + const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame); + RefCntBuffer *const buf = &pool->frame_bufs[buf_idx]; + buf->buf.y_crop_width = ref->y_crop_width; + buf->buf.y_crop_height = ref->y_crop_height; + cpi->scaled_ref_idx[ref_frame - 1] = buf_idx; + ++buf->ref_count; + } + } else { + if (cpi->oxcf.pass != 0) cpi->scaled_ref_idx[ref_frame - 1] = INVALID_IDX; + } + } +} + +static void release_scaled_references(AV1_COMP *cpi) { + AV1_COMMON *cm = &cpi->common; + int i; + if (cpi->oxcf.pass == 0) { + // Only release scaled references under certain conditions: + // if reference will be updated, or if scaled reference has same resolution. + int refresh[INTER_REFS_PER_FRAME]; + refresh[0] = (cpi->refresh_last_frame) ? 1 : 0; +#if CONFIG_EXT_REFS + refresh[1] = refresh[2] = 0; + refresh[3] = (cpi->refresh_golden_frame) ? 1 : 0; + refresh[4] = (cpi->refresh_bwd_ref_frame) ? 1 : 0; + refresh[5] = (cpi->refresh_alt_ref_frame) ? 1 : 0; +#else + refresh[1] = (cpi->refresh_golden_frame) ? 1 : 0; + refresh[2] = (cpi->refresh_alt_ref_frame) ? 1 : 0; +#endif // CONFIG_EXT_REFS + for (i = LAST_FRAME; i <= ALTREF_FRAME; ++i) { + const int idx = cpi->scaled_ref_idx[i - 1]; + RefCntBuffer *const buf = + idx != INVALID_IDX ? &cm->buffer_pool->frame_bufs[idx] : NULL; + const YV12_BUFFER_CONFIG *const ref = get_ref_frame_buffer(cpi, i); + if (buf != NULL && + (refresh[i - 1] || (buf->buf.y_crop_width == ref->y_crop_width && + buf->buf.y_crop_height == ref->y_crop_height))) { + --buf->ref_count; + cpi->scaled_ref_idx[i - 1] = INVALID_IDX; + } + } + } else { + for (i = 0; i < TOTAL_REFS_PER_FRAME; ++i) { + const int idx = cpi->scaled_ref_idx[i]; + RefCntBuffer *const buf = + idx != INVALID_IDX ? &cm->buffer_pool->frame_bufs[idx] : NULL; + if (buf != NULL) { + --buf->ref_count; + cpi->scaled_ref_idx[i] = INVALID_IDX; + } + } + } +} + +static void full_to_model_count(unsigned int *model_count, + unsigned int *full_count) { + int n; + model_count[ZERO_TOKEN] = full_count[ZERO_TOKEN]; + model_count[ONE_TOKEN] = full_count[ONE_TOKEN]; + model_count[TWO_TOKEN] = full_count[TWO_TOKEN]; + for (n = THREE_TOKEN; n < EOB_TOKEN; ++n) + model_count[TWO_TOKEN] += full_count[n]; + model_count[EOB_MODEL_TOKEN] = full_count[EOB_TOKEN]; +} + +void av1_full_to_model_counts(av1_coeff_count_model *model_count, + av1_coeff_count *full_count) { + int i, j, k, l; + + for (i = 0; i < PLANE_TYPES; ++i) + for (j = 0; j < REF_TYPES; ++j) + for (k = 0; k < COEF_BANDS; ++k) + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) + full_to_model_count(model_count[i][j][k][l], full_count[i][j][k][l]); +} + +#if 0 && CONFIG_INTERNAL_STATS +static void output_frame_level_debug_stats(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + FILE *const f = fopen("tmp.stt", cm->current_video_frame ? "a" : "w"); + int64_t recon_err; + + aom_clear_system_state(); + + recon_err = aom_get_y_sse(cpi->source, get_frame_new_buffer(cm)); + + if (cpi->twopass.total_left_stats.coded_error != 0.0) + fprintf(f, "%10u %dx%d %d %d %10d %10d %10d %10d" + "%10"PRId64" %10"PRId64" %5d %5d %10"PRId64" " + "%10"PRId64" %10"PRId64" %10d " + "%7.2lf %7.2lf %7.2lf %7.2lf %7.2lf" + "%6d %6d %5d %5d %5d " + "%10"PRId64" %10.3lf" + "%10lf %8u %10"PRId64" %10d %10d %10d\n", + cpi->common.current_video_frame, + cm->width, cm->height, + cpi->rc.source_alt_ref_pending, + cpi->rc.source_alt_ref_active, + cpi->rc.this_frame_target, + cpi->rc.projected_frame_size, + cpi->rc.projected_frame_size / cpi->common.MBs, + (cpi->rc.projected_frame_size - cpi->rc.this_frame_target), + cpi->rc.vbr_bits_off_target, + cpi->rc.vbr_bits_off_target_fast, + cpi->twopass.extend_minq, + cpi->twopass.extend_minq_fast, + cpi->rc.total_target_vs_actual, + (cpi->rc.starting_buffer_level - cpi->rc.bits_off_target), + cpi->rc.total_actual_bits, cm->base_qindex, + av1_convert_qindex_to_q(cm->base_qindex, cm->bit_depth), + (double)av1_dc_quant(cm->base_qindex, 0, cm->bit_depth) / 4.0, + av1_convert_qindex_to_q(cpi->twopass.active_worst_quality, + cm->bit_depth), + cpi->rc.avg_q, + av1_convert_qindex_to_q(cpi->oxcf.cq_level, cm->bit_depth), + cpi->refresh_last_frame, cpi->refresh_golden_frame, + cpi->refresh_alt_ref_frame, cm->frame_type, cpi->rc.gfu_boost, + cpi->twopass.bits_left, + cpi->twopass.total_left_stats.coded_error, + cpi->twopass.bits_left / + (1 + cpi->twopass.total_left_stats.coded_error), + cpi->tot_recode_hits, recon_err, cpi->rc.kf_boost, + cpi->twopass.kf_zeromotion_pct, + cpi->twopass.fr_content_type); + + fclose(f); + + if (0) { + FILE *const fmodes = fopen("Modes.stt", "a"); + int i; + + fprintf(fmodes, "%6d:%1d:%1d:%1d ", cpi->common.current_video_frame, + cm->frame_type, cpi->refresh_golden_frame, + cpi->refresh_alt_ref_frame); + + for (i = 0; i < MAX_MODES; ++i) + fprintf(fmodes, "%5d ", cpi->mode_chosen_counts[i]); + + fprintf(fmodes, "\n"); + + fclose(fmodes); + } +} +#endif + +static void set_mv_search_params(AV1_COMP *cpi) { + const AV1_COMMON *const cm = &cpi->common; + const unsigned int max_mv_def = AOMMIN(cm->width, cm->height); + + // Default based on max resolution. + cpi->mv_step_param = av1_init_search_range(max_mv_def); + + if (cpi->sf.mv.auto_mv_step_size) { + if (frame_is_intra_only(cm)) { + // Initialize max_mv_magnitude for use in the first INTER frame + // after a key/intra-only frame. + cpi->max_mv_magnitude = max_mv_def; + } else { + if (cm->show_frame) { + // Allow mv_steps to correspond to twice the max mv magnitude found + // in the previous frame, capped by the default max_mv_magnitude based + // on resolution. + cpi->mv_step_param = av1_init_search_range( + AOMMIN(max_mv_def, 2 * cpi->max_mv_magnitude)); + } + cpi->max_mv_magnitude = 0; + } + } +} + +static void set_size_independent_vars(AV1_COMP *cpi) { +#if CONFIG_GLOBAL_MOTION + int i; + for (i = LAST_FRAME; i <= ALTREF_FRAME; ++i) { + set_default_warp_params(&cpi->common.global_motion[i]); + } + cpi->global_motion_search_done = 0; +#endif // CONFIG_GLOBAL_MOTION + av1_set_speed_features_framesize_independent(cpi); + av1_set_rd_speed_thresholds(cpi); + av1_set_rd_speed_thresholds_sub8x8(cpi); + cpi->common.interp_filter = cpi->sf.default_interp_filter; +} + +static void set_size_dependent_vars(AV1_COMP *cpi, int *q, int *bottom_index, + int *top_index) { + AV1_COMMON *const cm = &cpi->common; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + + // Setup variables that depend on the dimensions of the frame. + av1_set_speed_features_framesize_dependent(cpi); + +// Decide q and q bounds. +#if CONFIG_XIPHRC + int frame_type = cm->frame_type == KEY_FRAME ? OD_I_FRAME : OD_P_FRAME; + *q = od_enc_rc_select_quantizers_and_lambdas( + &cpi->od_rc, cpi->refresh_golden_frame, cpi->refresh_alt_ref_frame, + frame_type, bottom_index, top_index); +#else + *q = av1_rc_pick_q_and_bounds(cpi, bottom_index, top_index); +#endif + + if (!frame_is_intra_only(cm)) { + av1_set_high_precision_mv(cpi, (*q) < HIGH_PRECISION_MV_QTHRESH); + } + + // Configure experimental use of segmentation for enhanced coding of + // static regions if indicated. + // Only allowed in the second pass of a two pass encode, as it requires + // lagged coding, and if the relevant speed feature flag is set. + if (oxcf->pass == 2 && cpi->sf.static_segmentation) + configure_static_seg_features(cpi); +} + +static void init_motion_estimation(AV1_COMP *cpi) { + int y_stride = cpi->scaled_source.y_stride; + + if (cpi->sf.mv.search_method == NSTEP) { + av1_init3smotion_compensation(&cpi->ss_cfg, y_stride); + } else if (cpi->sf.mv.search_method == DIAMOND) { + av1_init_dsmotion_compensation(&cpi->ss_cfg, y_stride); + } +} + +#if CONFIG_LOOP_RESTORATION +static void set_restoration_tilesize(int width, int height, + RestorationInfo *rst) { + (void)width; + (void)height; + rst[0].restoration_tilesize = (RESTORATION_TILESIZE_MAX >> 1); + rst[1].restoration_tilesize = rst[0].restoration_tilesize; + rst[2].restoration_tilesize = rst[0].restoration_tilesize; +} +#endif // CONFIG_LOOP_RESTORATION + +static void set_frame_size(AV1_COMP *cpi) { + int ref_frame; + AV1_COMMON *const cm = &cpi->common; + AV1EncoderConfig *const oxcf = &cpi->oxcf; + MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; + + if (oxcf->pass == 2 && oxcf->rc_mode == AOM_VBR && + ((oxcf->resize_mode == RESIZE_FIXED && cm->current_video_frame == 0) || + (oxcf->resize_mode == RESIZE_DYNAMIC && cpi->resize_pending))) { + av1_calculate_coded_size(cpi, &oxcf->scaled_frame_width, + &oxcf->scaled_frame_height); + + // There has been a change in frame size. + av1_set_size_literal(cpi, oxcf->scaled_frame_width, + oxcf->scaled_frame_height); + } + + if (oxcf->pass == 0 && oxcf->rc_mode == AOM_CBR && + oxcf->resize_mode == RESIZE_DYNAMIC) { + if (cpi->resize_pending == 1) { + oxcf->scaled_frame_width = + (cm->width * cpi->resize_scale_num) / cpi->resize_scale_den; + oxcf->scaled_frame_height = + (cm->height * cpi->resize_scale_num) / cpi->resize_scale_den; + } else if (cpi->resize_pending == -1) { + // Go back up to original size. + oxcf->scaled_frame_width = oxcf->width; + oxcf->scaled_frame_height = oxcf->height; + } + if (cpi->resize_pending != 0) { + // There has been a change in frame size. + av1_set_size_literal(cpi, oxcf->scaled_frame_width, + oxcf->scaled_frame_height); + + // TODO(agrange) Scale cpi->max_mv_magnitude if frame-size has changed. + set_mv_search_params(cpi); + } + } + +#if !CONFIG_XIPHRC + if (oxcf->pass == 2) { + av1_set_target_rate(cpi); + } +#endif + + alloc_frame_mvs(cm, cm->new_fb_idx); + + // Reset the frame pointers to the current frame size. + if (aom_realloc_frame_buffer(get_frame_new_buffer(cm), cm->width, cm->height, + cm->subsampling_x, cm->subsampling_y, +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth, +#endif + AOM_BORDER_IN_PIXELS, cm->byte_alignment, NULL, + NULL, NULL)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate frame buffer"); + +#if CONFIG_LOOP_RESTORATION + set_restoration_tilesize(cm->width, cm->height, cm->rst_info); + for (int i = 0; i < MAX_MB_PLANE; ++i) + cm->rst_info[i].frame_restoration_type = RESTORE_NONE; + av1_alloc_restoration_buffers(cm); + for (int i = 0; i < MAX_MB_PLANE; ++i) { + cpi->rst_search[i].restoration_tilesize = + cm->rst_info[i].restoration_tilesize; + av1_alloc_restoration_struct(cm, &cpi->rst_search[i], cm->width, + cm->height); + } +#endif // CONFIG_LOOP_RESTORATION + alloc_util_frame_buffers(cpi); + init_motion_estimation(cpi); + + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { + RefBuffer *const ref_buf = &cm->frame_refs[ref_frame - LAST_FRAME]; + const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame); + + ref_buf->idx = buf_idx; + + if (buf_idx != INVALID_IDX) { + YV12_BUFFER_CONFIG *const buf = &cm->buffer_pool->frame_bufs[buf_idx].buf; + ref_buf->buf = buf; +#if CONFIG_HIGHBITDEPTH + av1_setup_scale_factors_for_frame( + &ref_buf->sf, buf->y_crop_width, buf->y_crop_height, cm->width, + cm->height, (buf->flags & YV12_FLAG_HIGHBITDEPTH) ? 1 : 0); +#else + av1_setup_scale_factors_for_frame(&ref_buf->sf, buf->y_crop_width, + buf->y_crop_height, cm->width, + cm->height); +#endif // CONFIG_HIGHBITDEPTH + if (av1_is_scaled(&ref_buf->sf)) aom_extend_frame_borders(buf); + } else { + ref_buf->buf = NULL; + } + } + + set_ref_ptrs(cm, xd, LAST_FRAME, LAST_FRAME); +} + +static void reset_use_upsampled_references(AV1_COMP *cpi) { + MV_REFERENCE_FRAME ref_frame; + + // reset up-sampled reference buffer structure. + init_upsampled_ref_frame_bufs(cpi); + + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { + const YV12_BUFFER_CONFIG *const ref = get_ref_frame_buffer(cpi, ref_frame); + int new_uidx = upsample_ref_frame(cpi, ref); + + // Update the up-sampled reference index. + cpi->upsampled_ref_idx[get_ref_frame_map_idx(cpi, ref_frame)] = new_uidx; + cpi->upsampled_ref_bufs[new_uidx].ref_count++; + } +} + +static void encode_without_recode_loop(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + int q = 0, bottom_index = 0, top_index = 0; // Dummy variables. + const int use_upsampled_ref = cpi->sf.use_upsampled_references; + + aom_clear_system_state(); + + set_frame_size(cpi); + + // For 1 pass CBR under dynamic resize mode: use faster scaling for source. + // Only for 2x2 scaling for now. + if (cpi->oxcf.pass == 0 && cpi->oxcf.rc_mode == AOM_CBR && + cpi->oxcf.resize_mode == RESIZE_DYNAMIC && + cpi->un_scaled_source->y_width == (cm->width << 1) && + cpi->un_scaled_source->y_height == (cm->height << 1)) { + cpi->source = av1_scale_if_required_fast(cm, cpi->un_scaled_source, + &cpi->scaled_source); + if (cpi->unscaled_last_source != NULL) + cpi->last_source = av1_scale_if_required_fast( + cm, cpi->unscaled_last_source, &cpi->scaled_last_source); + } else { + cpi->source = + av1_scale_if_required(cm, cpi->un_scaled_source, &cpi->scaled_source); + if (cpi->unscaled_last_source != NULL) + cpi->last_source = av1_scale_if_required(cm, cpi->unscaled_last_source, + &cpi->scaled_last_source); + } + + if (frame_is_intra_only(cm) == 0) { + av1_scale_references(cpi); + } + + set_size_independent_vars(cpi); + set_size_dependent_vars(cpi, &q, &bottom_index, &top_index); + + // cpi->sf.use_upsampled_references can be different from frame to frame. + // Every time when cpi->sf.use_upsampled_references is changed from 0 to 1. + // The reference frames for this frame have to be up-sampled before encoding. + if (!use_upsampled_ref && cpi->sf.use_upsampled_references && + cm->frame_type != KEY_FRAME) + reset_use_upsampled_references(cpi); + + av1_set_quantizer(cm, q); + av1_set_variance_partition_thresholds(cpi, q); + + setup_frame(cpi); + +#if CONFIG_SUBFRAME_PROB_UPDATE + cm->do_subframe_update = cm->tile_cols == 1 && cm->tile_rows == 1; + av1_copy(cm->starting_coef_probs, cm->fc->coef_probs); + av1_copy(cpi->subframe_stats.enc_starting_coef_probs, cm->fc->coef_probs); + cm->coef_probs_update_idx = 0; + av1_copy(cpi->subframe_stats.coef_probs_buf[0], cm->fc->coef_probs); +#endif // CONFIG_SUBFRAME_PROB_UPDATE + + suppress_active_map(cpi); + // Variance adaptive and in frame q adjustment experiments are mutually + // exclusive. + if (cpi->oxcf.aq_mode == VARIANCE_AQ) { + av1_vaq_frame_setup(cpi); + } else if (cpi->oxcf.aq_mode == COMPLEXITY_AQ) { + av1_setup_in_frame_q_adj(cpi); + } else if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) { + av1_cyclic_refresh_setup(cpi); + } + apply_active_map(cpi); + + // transform / motion compensation build reconstruction frame + av1_encode_frame(cpi); + + // Update some stats from cyclic refresh, and check if we should not update + // golden reference, for 1 pass CBR. + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cm->frame_type != KEY_FRAME && + (cpi->oxcf.pass == 0 && cpi->oxcf.rc_mode == AOM_CBR)) + av1_cyclic_refresh_check_golden_update(cpi); + + // Update the skip mb flag probabilities based on the distribution + // seen in the last encoder iteration. + // update_base_skip_probs(cpi); + aom_clear_system_state(); +} + +static void encode_with_recode_loop(AV1_COMP *cpi, size_t *size, + uint8_t *dest) { + AV1_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; + int bottom_index, top_index; + int loop_count = 0; + int loop_at_this_size = 0; + int loop = 0; +#if !CONFIG_XIPHRC + int overshoot_seen = 0; + int undershoot_seen = 0; +#endif + int frame_over_shoot_limit; + int frame_under_shoot_limit; + int q = 0, q_low = 0, q_high = 0; + const int use_upsampled_ref = cpi->sf.use_upsampled_references; + + set_size_independent_vars(cpi); + + do { + aom_clear_system_state(); + + set_frame_size(cpi); + + if (loop_count == 0 || cpi->resize_pending != 0) { + set_size_dependent_vars(cpi, &q, &bottom_index, &top_index); + + // cpi->sf.use_upsampled_references can be different from frame to frame. + // Every time when cpi->sf.use_upsampled_references is changed from 0 to + // 1. + // The reference frames for this frame have to be up-sampled before + // encoding. + if (!use_upsampled_ref && cpi->sf.use_upsampled_references && + cm->frame_type != KEY_FRAME) + reset_use_upsampled_references(cpi); + + // TODO(agrange) Scale cpi->max_mv_magnitude if frame-size has changed. + set_mv_search_params(cpi); + +#if !CONFIG_XIPHRC + // Reset the loop state for new frame size. + overshoot_seen = 0; + undershoot_seen = 0; +#endif + + // Reconfiguration for change in frame size has concluded. + cpi->resize_pending = 0; + + q_low = bottom_index; + q_high = top_index; + + loop_at_this_size = 0; + } + + // Decide frame size bounds first time through. + if (loop_count == 0) { + av1_rc_compute_frame_size_bounds(cpi, rc->this_frame_target, + &frame_under_shoot_limit, + &frame_over_shoot_limit); + } + + cpi->source = + av1_scale_if_required(cm, cpi->un_scaled_source, &cpi->scaled_source); + + if (cpi->unscaled_last_source != NULL) + cpi->last_source = av1_scale_if_required(cm, cpi->unscaled_last_source, + &cpi->scaled_last_source); + + if (frame_is_intra_only(cm) == 0) { + if (loop_count > 0) { + release_scaled_references(cpi); + } + av1_scale_references(cpi); + } + + av1_set_quantizer(cm, q); + + if (loop_count == 0) setup_frame(cpi); + +#if CONFIG_Q_ADAPT_PROBS + // Base q-index may have changed, so we need to assign proper default coef + // probs before every iteration. + if (frame_is_intra_only(cm) || cm->error_resilient_mode) { + int i; + av1_default_coef_probs(cm); + if (cm->frame_type == KEY_FRAME || cm->error_resilient_mode || + cm->reset_frame_context == RESET_FRAME_CONTEXT_ALL) { + for (i = 0; i < FRAME_CONTEXTS; ++i) cm->frame_contexts[i] = *cm->fc; + } else if (cm->reset_frame_context == RESET_FRAME_CONTEXT_CURRENT) { + cm->frame_contexts[cm->frame_context_idx] = *cm->fc; + } + } +#endif // CONFIG_Q_ADAPT_PROBS + +#if CONFIG_SUBFRAME_PROB_UPDATE + cm->do_subframe_update = cm->tile_cols == 1 && cm->tile_rows == 1; + if (loop_count == 0 || frame_is_intra_only(cm) || + cm->error_resilient_mode) { + av1_copy(cm->starting_coef_probs, cm->fc->coef_probs); + av1_copy(cpi->subframe_stats.enc_starting_coef_probs, cm->fc->coef_probs); + } else { + if (cm->do_subframe_update) { + av1_copy(cm->fc->coef_probs, + cpi->subframe_stats.enc_starting_coef_probs); + av1_copy(cm->starting_coef_probs, + cpi->subframe_stats.enc_starting_coef_probs); + av1_zero(cpi->subframe_stats.coef_counts_buf); + av1_zero(cpi->subframe_stats.eob_counts_buf); + } + } + cm->coef_probs_update_idx = 0; + av1_copy(cpi->subframe_stats.coef_probs_buf[0], cm->fc->coef_probs); +#endif // CONFIG_SUBFRAME_PROB_UPDATE + + // Variance adaptive and in frame q adjustment experiments are mutually + // exclusive. + if (cpi->oxcf.aq_mode == VARIANCE_AQ) { + av1_vaq_frame_setup(cpi); + } else if (cpi->oxcf.aq_mode == COMPLEXITY_AQ) { + av1_setup_in_frame_q_adj(cpi); + } + + // transform / motion compensation build reconstruction frame + av1_encode_frame(cpi); + + // Update the skip mb flag probabilities based on the distribution + // seen in the last encoder iteration. + // update_base_skip_probs(cpi); + + aom_clear_system_state(); + + // Dummy pack of the bitstream using up to date stats to get an + // accurate estimate of output frame size to determine if we need + // to recode. + if (cpi->sf.recode_loop >= ALLOW_RECODE_KFARFGF) { + save_coding_context(cpi); + + av1_pack_bitstream(cpi, dest, size); + + rc->projected_frame_size = (int)(*size) << 3; + restore_coding_context(cpi); + + if (frame_over_shoot_limit == 0) frame_over_shoot_limit = 1; + } + + if (cpi->oxcf.rc_mode == AOM_Q) { + loop = 0; + } else { + if ((cm->frame_type == KEY_FRAME) && rc->this_key_frame_forced && + (rc->projected_frame_size < rc->max_frame_bandwidth)) { + int last_q = q; + int64_t kf_err; + + int64_t high_err_target = cpi->ambient_err; + int64_t low_err_target = cpi->ambient_err >> 1; + +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) { + kf_err = aom_highbd_get_y_sse(cpi->source, get_frame_new_buffer(cm)); + } else { + kf_err = aom_get_y_sse(cpi->source, get_frame_new_buffer(cm)); + } +#else + kf_err = aom_get_y_sse(cpi->source, get_frame_new_buffer(cm)); +#endif // CONFIG_HIGHBITDEPTH + + // Prevent possible divide by zero error below for perfect KF + kf_err += !kf_err; + + // The key frame is not good enough or we can afford + // to make it better without undue risk of popping. + if ((kf_err > high_err_target && + rc->projected_frame_size <= frame_over_shoot_limit) || + (kf_err > low_err_target && + rc->projected_frame_size <= frame_under_shoot_limit)) { + // Lower q_high + q_high = q > q_low ? q - 1 : q_low; + + // Adjust Q + q = (int)((q * high_err_target) / kf_err); + q = AOMMIN(q, (q_high + q_low) >> 1); + } else if (kf_err < low_err_target && + rc->projected_frame_size >= frame_under_shoot_limit) { + // The key frame is much better than the previous frame + // Raise q_low + q_low = q < q_high ? q + 1 : q_high; + + // Adjust Q + q = (int)((q * low_err_target) / kf_err); + q = AOMMIN(q, (q_high + q_low + 1) >> 1); + } + + // Clamp Q to upper and lower limits: + q = clamp(q, q_low, q_high); + + loop = q != last_q; + } else if (recode_loop_test(cpi, frame_over_shoot_limit, + frame_under_shoot_limit, q, + AOMMAX(q_high, top_index), bottom_index)) { + // Is the projected frame size out of range and are we allowed + // to attempt to recode. + int last_q = q; +#if !CONFIG_XIPHRC + int retries = 0; +#endif + + if (cpi->resize_pending == 1) { + // Change in frame size so go back around the recode loop. + cpi->rc.frame_size_selector = + SCALE_STEP1 - cpi->rc.frame_size_selector; + cpi->rc.next_frame_size_selector = cpi->rc.frame_size_selector; + +#if CONFIG_INTERNAL_STATS + ++cpi->tot_recode_hits; +#endif + ++loop_count; + loop = 1; + continue; + } + +#if !CONFIG_XIPHRC + // Frame size out of permitted range: + // Update correction factor & compute new Q to try... + // Frame is too large + if (rc->projected_frame_size > rc->this_frame_target) { + // Special case if the projected size is > the max allowed. + if (rc->projected_frame_size >= rc->max_frame_bandwidth) + q_high = rc->worst_quality; + + // Raise Qlow as to at least the current value + q_low = q < q_high ? q + 1 : q_high; + + if (undershoot_seen || loop_at_this_size > 1) { + // Update rate_correction_factor unless + av1_rc_update_rate_correction_factors(cpi); + + q = (q_high + q_low + 1) / 2; + } else { + // Update rate_correction_factor unless + av1_rc_update_rate_correction_factors(cpi); + + q = av1_rc_regulate_q(cpi, rc->this_frame_target, bottom_index, + AOMMAX(q_high, top_index)); + + while (q < q_low && retries < 10) { + av1_rc_update_rate_correction_factors(cpi); + q = av1_rc_regulate_q(cpi, rc->this_frame_target, bottom_index, + AOMMAX(q_high, top_index)); + retries++; + } + } + + overshoot_seen = 1; + } else { + // Frame is too small + q_high = q > q_low ? q - 1 : q_low; + + if (overshoot_seen || loop_at_this_size > 1) { + av1_rc_update_rate_correction_factors(cpi); + q = (q_high + q_low) / 2; + } else { + av1_rc_update_rate_correction_factors(cpi); + q = av1_rc_regulate_q(cpi, rc->this_frame_target, bottom_index, + top_index); + // Special case reset for qlow for constrained quality. + // This should only trigger where there is very substantial + // undershoot on a frame and the auto cq level is above + // the user passsed in value. + if (cpi->oxcf.rc_mode == AOM_CQ && q < q_low) { + q_low = q; + } + + while (q > q_high && retries < 10) { + av1_rc_update_rate_correction_factors(cpi); + q = av1_rc_regulate_q(cpi, rc->this_frame_target, bottom_index, + top_index); + retries++; + } + } + + undershoot_seen = 1; + } +#endif + + // Clamp Q to upper and lower limits: + q = clamp(q, q_low, q_high); + + loop = (q != last_q); + } else { + loop = 0; + } + } + + // Special case for overlay frame. + if (rc->is_src_frame_alt_ref && + rc->projected_frame_size < rc->max_frame_bandwidth) + loop = 0; + +#if CONFIG_GLOBAL_MOTION + if (recode_loop_test_global_motion(cpi)) { + loop = 1; + } +#endif // CONFIG_GLOBAL_MOTION + + if (loop) { + ++loop_count; + ++loop_at_this_size; + +#if CONFIG_INTERNAL_STATS + ++cpi->tot_recode_hits; +#endif + } + } while (loop); +} + +static int get_ref_frame_flags(const AV1_COMP *cpi) { + const int *const map = cpi->common.ref_frame_map; + +#if CONFIG_EXT_REFS + const int last2_is_last = + map[cpi->lst_fb_idxes[1]] == map[cpi->lst_fb_idxes[0]]; + const int last3_is_last = + map[cpi->lst_fb_idxes[2]] == map[cpi->lst_fb_idxes[0]]; + const int gld_is_last = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[0]]; +#if CONFIG_LOWDELAY_COMPOUND + const int alt_is_last = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[0]]; + const int last3_is_last2 = + map[cpi->lst_fb_idxes[2]] == map[cpi->lst_fb_idxes[1]]; + const int gld_is_last2 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[1]]; + const int gld_is_last3 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[2]]; +#else + const int bwd_is_last = map[cpi->bwd_fb_idx] == map[cpi->lst_fb_idxes[0]]; + const int alt_is_last = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[0]]; + + const int last3_is_last2 = + map[cpi->lst_fb_idxes[2]] == map[cpi->lst_fb_idxes[1]]; + const int gld_is_last2 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[1]]; + const int bwd_is_last2 = map[cpi->bwd_fb_idx] == map[cpi->lst_fb_idxes[1]]; + + const int gld_is_last3 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[2]]; + const int bwd_is_last3 = map[cpi->bwd_fb_idx] == map[cpi->lst_fb_idxes[2]]; + + const int bwd_is_gld = map[cpi->bwd_fb_idx] == map[cpi->gld_fb_idx]; + +#endif + const int last2_is_alt = map[cpi->lst_fb_idxes[1]] == map[cpi->alt_fb_idx]; + const int last3_is_alt = map[cpi->lst_fb_idxes[2]] == map[cpi->alt_fb_idx]; + const int gld_is_alt = map[cpi->gld_fb_idx] == map[cpi->alt_fb_idx]; + const int bwd_is_alt = map[cpi->bwd_fb_idx] == map[cpi->alt_fb_idx]; +#else + const int gld_is_last = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idx]; + const int gld_is_alt = map[cpi->gld_fb_idx] == map[cpi->alt_fb_idx]; + const int alt_is_last = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idx]; +#endif // CONFIG_EXT_REFS + + int flags = AOM_REFFRAME_ALL; + +#if CONFIG_EXT_REFS + // Disable the use of BWDREF_FRAME for non-bipredictive frames. + if (!(cpi->rc.is_bipred_frame || cpi->rc.is_last_bipred_frame || + (cpi->rc.is_bwd_ref_frame && cpi->num_extra_arfs))) + flags &= ~AOM_BWD_FLAG; +#endif // CONFIG_EXT_REFS + + if (gld_is_last || gld_is_alt) flags &= ~AOM_GOLD_FLAG; + + if (cpi->rc.frames_till_gf_update_due == INT_MAX) flags &= ~AOM_GOLD_FLAG; + + if (alt_is_last) flags &= ~AOM_ALT_FLAG; + +#if CONFIG_EXT_REFS + if (last2_is_last || last2_is_alt) flags &= ~AOM_LAST2_FLAG; + + if (last3_is_last || last3_is_last2 || last3_is_alt) flags &= ~AOM_LAST3_FLAG; + + if (gld_is_last2 || gld_is_last3) flags &= ~AOM_GOLD_FLAG; + +#if CONFIG_LOWDELAY_COMPOUND // Changes LL & HL bitstream + /* Allow biprediction between two identical frames (e.g. bwd_is_last = 1) */ + if (bwd_is_alt && (flags & AOM_BWD_FLAG)) flags &= ~AOM_BWD_FLAG; +#else + if ((bwd_is_last || bwd_is_last2 || bwd_is_last3 || bwd_is_gld || + bwd_is_alt) && + (flags & AOM_BWD_FLAG)) + flags &= ~AOM_BWD_FLAG; +#endif +#endif // CONFIG_EXT_REFS + + return flags; +} + +static void set_ext_overrides(AV1_COMP *cpi) { + // Overrides the defaults with the externally supplied values with + // av1_update_reference() and av1_update_entropy() calls + // Note: The overrides are valid only for the next frame passed + // to encode_frame_to_data_rate() function + if (cpi->ext_refresh_frame_context_pending) { + cpi->common.refresh_frame_context = cpi->ext_refresh_frame_context; + cpi->ext_refresh_frame_context_pending = 0; + } + if (cpi->ext_refresh_frame_flags_pending) { + cpi->refresh_last_frame = cpi->ext_refresh_last_frame; + cpi->refresh_golden_frame = cpi->ext_refresh_golden_frame; + cpi->refresh_alt_ref_frame = cpi->ext_refresh_alt_ref_frame; + cpi->ext_refresh_frame_flags_pending = 0; + } +} + +YV12_BUFFER_CONFIG *av1_scale_if_required_fast(AV1_COMMON *cm, + YV12_BUFFER_CONFIG *unscaled, + YV12_BUFFER_CONFIG *scaled) { + if (cm->mi_cols * MI_SIZE != unscaled->y_width || + cm->mi_rows * MI_SIZE != unscaled->y_height) { + // For 2x2 scaling down. + aom_scale_frame(unscaled, scaled, unscaled->y_buffer, 9, 2, 1, 2, 1, 0); + aom_extend_frame_borders(scaled); + return scaled; + } else { + return unscaled; + } +} + +YV12_BUFFER_CONFIG *av1_scale_if_required(AV1_COMMON *cm, + YV12_BUFFER_CONFIG *unscaled, + YV12_BUFFER_CONFIG *scaled) { + if (cm->mi_cols * MI_SIZE != unscaled->y_width || + cm->mi_rows * MI_SIZE != unscaled->y_height) { +#if CONFIG_HIGHBITDEPTH + scale_and_extend_frame_nonnormative(unscaled, scaled, (int)cm->bit_depth); +#else + scale_and_extend_frame_nonnormative(unscaled, scaled); +#endif // CONFIG_HIGHBITDEPTH + return scaled; + } else { + return unscaled; + } +} + +static void set_arf_sign_bias(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + int arf_sign_bias; +#if CONFIG_EXT_REFS + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + // The arf_sign_bias will be one for internal ARFs' + arf_sign_bias = cpi->rc.source_alt_ref_active && + (!cpi->refresh_alt_ref_frame || + (gf_group->rf_level[gf_group->index] == GF_ARF_LOW)); +#else + if ((cpi->oxcf.pass == 2) && cpi->multi_arf_allowed) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + arf_sign_bias = cpi->rc.source_alt_ref_active && + (!cpi->refresh_alt_ref_frame || + (gf_group->rf_level[gf_group->index] == GF_ARF_LOW)); + } else { + arf_sign_bias = + (cpi->rc.source_alt_ref_active && !cpi->refresh_alt_ref_frame); + } +#endif // CONFIG_EXT_REFS + + cm->ref_frame_sign_bias[ALTREF_FRAME] = arf_sign_bias; +#if CONFIG_EXT_REFS + cm->ref_frame_sign_bias[BWDREF_FRAME] = cm->ref_frame_sign_bias[ALTREF_FRAME]; +#endif // CONFIG_EXT_REFS +} + +static int setup_interp_filter_search_mask(AV1_COMP *cpi) { + InterpFilter ifilter; + int ref_total[TOTAL_REFS_PER_FRAME] = { 0 }; + MV_REFERENCE_FRAME ref; + int mask = 0; + int arf_idx = ALTREF_FRAME; + +#if CONFIG_EXT_REFS + // Get which arf used as ALTREF_FRAME + if (cpi->oxcf.pass == 2) + arf_idx += cpi->twopass.gf_group.arf_ref_idx[cpi->twopass.gf_group.index]; +#endif // CONFIG_EXT_REFS + + if (cpi->common.last_frame_type == KEY_FRAME || cpi->refresh_alt_ref_frame) + return mask; + +#if CONFIG_EXT_REFS + for (ref = LAST_FRAME; ref < ALTREF_FRAME; ++ref) + for (ifilter = EIGHTTAP_REGULAR; ifilter < SWITCHABLE_FILTERS; ++ifilter) + ref_total[ref] += cpi->interp_filter_selected[ref][ifilter]; + + for (ifilter = EIGHTTAP_REGULAR; ifilter < SWITCHABLE_FILTERS; ++ifilter) + ref_total[ref] += cpi->interp_filter_selected[arf_idx][ifilter]; +#else + for (ref = LAST_FRAME; ref <= ALTREF_FRAME; ++ref) + for (ifilter = EIGHTTAP_REGULAR; ifilter < SWITCHABLE_FILTERS; ++ifilter) + ref_total[ref] += cpi->interp_filter_selected[ref][ifilter]; +#endif // CONFIG_EXT_REFS + + for (ifilter = EIGHTTAP_REGULAR; ifilter < SWITCHABLE_FILTERS; ++ifilter) { + if ((ref_total[LAST_FRAME] && + cpi->interp_filter_selected[LAST_FRAME][ifilter] == 0) && +#if CONFIG_EXT_REFS + (ref_total[LAST2_FRAME] == 0 || + cpi->interp_filter_selected[LAST2_FRAME][ifilter] * 50 < + ref_total[LAST2_FRAME]) && + (ref_total[LAST3_FRAME] == 0 || + cpi->interp_filter_selected[LAST3_FRAME][ifilter] * 50 < + ref_total[LAST3_FRAME]) && +#endif // CONFIG_EXT_REFS + (ref_total[GOLDEN_FRAME] == 0 || + cpi->interp_filter_selected[GOLDEN_FRAME][ifilter] * 50 < + ref_total[GOLDEN_FRAME]) && +#if CONFIG_EXT_REFS + (ref_total[BWDREF_FRAME] == 0 || + cpi->interp_filter_selected[BWDREF_FRAME][ifilter] * 50 < + ref_total[BWDREF_FRAME]) && +#endif // CONFIG_EXT_REFS + (ref_total[ALTREF_FRAME] == 0 || + cpi->interp_filter_selected[arf_idx][ifilter] * 50 < + ref_total[ALTREF_FRAME])) + mask |= 1 << ifilter; + } + return mask; +} + +#define DUMP_RECON_FRAMES 0 + +#if DUMP_RECON_FRAMES == 1 +// NOTE(zoeliu): For debug - Output the filtered reconstructed video. +static void dump_filtered_recon_frames(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + const YV12_BUFFER_CONFIG *recon_buf = cm->frame_to_show; + int h; + char file_name[256] = "/tmp/enc_filtered_recon.yuv"; + FILE *f_recon = NULL; + + if (recon_buf == NULL || !cm->show_frame) { + printf("Frame %d is not ready or no show to dump.\n", + cm->current_video_frame); + return; + } + + if (cm->current_video_frame == 0) { + if ((f_recon = fopen(file_name, "wb")) == NULL) { + printf("Unable to open file %s to write.\n", file_name); + return; + } + } else { + if ((f_recon = fopen(file_name, "ab")) == NULL) { + printf("Unable to open file %s to append.\n", file_name); + return; + } + } + printf( + "\nFrame=%5d, encode_update_type[%5d]=%1d, show_existing_frame=%d, " + "y_stride=%4d, uv_stride=%4d, width=%4d, height=%4d\n", + cm->current_video_frame, cpi->twopass.gf_group.index, + cpi->twopass.gf_group.update_type[cpi->twopass.gf_group.index], + cm->show_existing_frame, recon_buf->y_stride, recon_buf->uv_stride, + cm->width, cm->height); + + // --- Y --- + for (h = 0; h < cm->height; ++h) { + fwrite(&recon_buf->y_buffer[h * recon_buf->y_stride], 1, cm->width, + f_recon); + } + // --- U --- + for (h = 0; h < (cm->height >> 1); ++h) { + fwrite(&recon_buf->u_buffer[h * recon_buf->uv_stride], 1, (cm->width >> 1), + f_recon); + } + // --- V --- + for (h = 0; h < (cm->height >> 1); ++h) { + fwrite(&recon_buf->v_buffer[h * recon_buf->uv_stride], 1, (cm->width >> 1), + f_recon); + } + + fclose(f_recon); +} +#endif // DUMP_RECON_FRAMES + +#if CONFIG_EC_ADAPT + +static void make_update_tile_list_enc(AV1_COMP *cpi, const int tile_rows, + const int tile_cols, + FRAME_CONTEXT *ec_ctxs[]) { + int i; + for (i = 0; i < tile_rows * tile_cols; ++i) + ec_ctxs[i] = &cpi->tile_data[i].tctx; +} + +#endif +static void encode_frame_to_data_rate(AV1_COMP *cpi, size_t *size, + uint8_t *dest, int skip_adapt, + unsigned int *frame_flags) { + AV1_COMMON *const cm = &cpi->common; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + struct segmentation *const seg = &cm->seg; + TX_SIZE t; +#if CONFIG_EC_ADAPT + FRAME_CONTEXT **tile_ctxs = aom_malloc(cm->tile_rows * cm->tile_cols * + sizeof(&cpi->tile_data[0].tctx)); + aom_cdf_prob **cdf_ptrs = + aom_malloc(cm->tile_rows * cm->tile_cols * + sizeof(&cpi->tile_data[0].tctx.partition_cdf[0][0])); +#endif +#if CONFIG_XIPHRC + int frame_type; + int drop_this_frame = 0; +#endif // CONFIG_XIPHRC + set_ext_overrides(cpi); + aom_clear_system_state(); + + // Set the arf sign bias for this frame. + set_arf_sign_bias(cpi); +#if CONFIG_TEMPMV_SIGNALING + // frame type has been decided outside of this function call + cm->cur_frame->intra_only = cm->frame_type == KEY_FRAME || cm->intra_only; + cm->use_prev_frame_mvs = + !cpi->oxcf.disable_tempmv && !cm->cur_frame->intra_only; +#endif + +#if CONFIG_EXT_REFS + // NOTE: + // (1) Move the setup of the ref_frame_flags upfront as it would be + // determined by the current frame properties; + // (2) The setup of the ref_frame_flags applies to both show_existing_frame's + // and the other cases. + if (cm->current_video_frame > 0) + cpi->ref_frame_flags = get_ref_frame_flags(cpi); + + if (cm->show_existing_frame) { + // NOTE(zoeliu): In BIDIR_PRED, the existing frame to show is the current + // BWDREF_FRAME in the reference frame buffer. + cm->frame_type = INTER_FRAME; + cm->show_frame = 1; + cpi->frame_flags = *frame_flags; + + // In the case of show_existing frame, we will not send fresh flag + // to decoder. Any change in the reference frame buffer can be done by + // switching the virtual indices. + + cpi->refresh_last_frame = 0; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + + cpi->rc.is_bwd_ref_frame = 0; + cpi->rc.is_last_bipred_frame = 0; + cpi->rc.is_bipred_frame = 0; + + // Build the bitstream + av1_pack_bitstream(cpi, dest, size); + + // Set up frame to show to get ready for stats collection. + cm->frame_to_show = get_frame_new_buffer(cm); + +#if DUMP_RECON_FRAMES == 1 + // NOTE(zoeliu): For debug - Output the filtered reconstructed video. + dump_filtered_recon_frames(cpi); +#endif // DUMP_RECON_FRAMES + + // Update the LAST_FRAME in the reference frame buffer. + av1_update_reference_frames(cpi); + + // Update frame flags + cpi->frame_flags &= ~FRAMEFLAGS_GOLDEN; + cpi->frame_flags &= ~FRAMEFLAGS_BWDREF; + cpi->frame_flags &= ~FRAMEFLAGS_ALTREF; + + *frame_flags = cpi->frame_flags & ~FRAMEFLAGS_KEY; + + // Update the frame type + cm->last_frame_type = cm->frame_type; + + // Since we allocate a spot for the OVERLAY frame in the gf group, we need + // to do post-encoding update accordingly. + if (cpi->rc.is_src_frame_alt_ref) { + av1_set_target_rate(cpi); +#if CONFIG_XIPHRC + frame_type = cm->frame_type == INTER_FRAME ? OD_P_FRAME : OD_I_FRAME; + drop_this_frame = od_enc_rc_update_state( + &cpi->od_rc, *size << 3, cpi->refresh_golden_frame, + cpi->refresh_alt_ref_frame, frame_type, cpi->droppable); +#else + av1_rc_postencode_update(cpi, *size); +#endif + } + + cm->last_width = cm->width; + cm->last_height = cm->height; + + ++cm->current_video_frame; + +#if CONFIG_EC_ADAPT + aom_free(tile_ctxs); + aom_free(cdf_ptrs); +#endif + return; + } +#endif // CONFIG_EXT_REFS + + // Set default state for segment based loop filter update flags. + cm->lf.mode_ref_delta_update = 0; + + if (cpi->oxcf.pass == 2 && cpi->sf.adaptive_interp_filter_search) + cpi->sf.interp_filter_search_mask = setup_interp_filter_search_mask(cpi); + + // Set various flags etc to special state if it is a key frame. + if (frame_is_intra_only(cm)) { + // Reset the loop filter deltas and segmentation map. + av1_reset_segment_features(cm); + + // If segmentation is enabled force a map update for key frames. + if (seg->enabled) { + seg->update_map = 1; + seg->update_data = 1; + } + + // The alternate reference frame cannot be active for a key frame. + cpi->rc.source_alt_ref_active = 0; + + cm->error_resilient_mode = oxcf->error_resilient_mode; + + // By default, encoder assumes decoder can use prev_mi. + if (cm->error_resilient_mode) { + cm->reset_frame_context = RESET_FRAME_CONTEXT_NONE; + cm->refresh_frame_context = REFRESH_FRAME_CONTEXT_FORWARD; + } else if (cm->intra_only) { + // Only reset the current context. + cm->reset_frame_context = RESET_FRAME_CONTEXT_CURRENT; + } + } +#if CONFIG_TILE_GROUPS + if (cpi->oxcf.mtu == 0) { + cm->num_tg = cpi->oxcf.num_tile_groups; + } else { + // Use a default value for the purposes of weighting costs in probability + // updates + cm->num_tg = DEFAULT_MAX_NUM_TG; + } +#endif + +#if CONFIG_EXT_TILE + cm->tile_encoding_mode = cpi->oxcf.tile_encoding_mode; +#endif // CONFIG_EXT_TILE + +#if CONFIG_XIPHRC + if (drop_this_frame) { + av1_rc_postencode_update_drop_frame(cpi); + ++cm->current_video_frame; +#if CONFIG_EC_ADAPT + aom_free(tile_ctxs); + aom_free(cdf_ptrs); +#endif + return; + } +#else + // For 1 pass CBR, check if we are dropping this frame. + // Never drop on key frame. + if (oxcf->pass == 0 && oxcf->rc_mode == AOM_CBR && + cm->frame_type != KEY_FRAME) { + if (av1_rc_drop_frame(cpi)) { + av1_rc_postencode_update_drop_frame(cpi); + ++cm->current_video_frame; +#if CONFIG_EC_ADAPT + aom_free(tile_ctxs); + aom_free(cdf_ptrs); +#endif + return; + } + } +#endif + + aom_clear_system_state(); + +#if CONFIG_INTERNAL_STATS + memset(cpi->mode_chosen_counts, 0, + MAX_MODES * sizeof(*cpi->mode_chosen_counts)); +#endif + +#if CONFIG_REFERENCE_BUFFER + { + /* Non-normative definition of current_frame_id ("frame counter" with + * wraparound) */ + const int frame_id_length = FRAME_ID_LENGTH_MINUS7 + 7; + if (cm->current_frame_id == -1) { + int lsb, msb; +/* quasi-random initialization of current_frame_id for a key frame */ +#if CONFIG_HIGHBITDEPTH + if (cpi->source->flags & YV12_FLAG_HIGHBITDEPTH) { + lsb = CONVERT_TO_SHORTPTR(cpi->source->y_buffer)[0] & 0xff; + msb = CONVERT_TO_SHORTPTR(cpi->source->y_buffer)[1] & 0xff; + } else { +#endif + lsb = cpi->source->y_buffer[0] & 0xff; + msb = cpi->source->y_buffer[1] & 0xff; +#if CONFIG_HIGHBITDEPTH + } +#endif + cm->current_frame_id = ((msb << 8) + lsb) % (1 << frame_id_length); + } else { + cm->current_frame_id = + (cm->current_frame_id + 1 + (1 << frame_id_length)) % + (1 << frame_id_length); + } + } +#endif + +#if CONFIG_EXT_DELTA_Q + cm->delta_q_present_flag = cpi->oxcf.deltaq_mode != NO_DELTA_Q; + cm->delta_lf_present_flag = cpi->oxcf.deltaq_mode == DELTA_Q_LF; +#endif + + if (cpi->sf.recode_loop == DISALLOW_RECODE) { + encode_without_recode_loop(cpi); + } else { + encode_with_recode_loop(cpi, size, dest); + } + +#ifdef OUTPUT_YUV_SKINMAP + if (cpi->common.current_video_frame > 1) { + av1_compute_skin_map(cpi, yuv_skinmap_file); + } +#endif // OUTPUT_YUV_SKINMAP + + // Special case code to reduce pulsing when key frames are forced at a + // fixed interval. Note the reconstruction error if it is the frame before + // the force key frame + if (cpi->rc.next_key_frame_forced && cpi->rc.frames_to_key == 1) { +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) { + cpi->ambient_err = + aom_highbd_get_y_sse(cpi->source, get_frame_new_buffer(cm)); + } else { + cpi->ambient_err = aom_get_y_sse(cpi->source, get_frame_new_buffer(cm)); + } +#else + cpi->ambient_err = aom_get_y_sse(cpi->source, get_frame_new_buffer(cm)); +#endif // CONFIG_HIGHBITDEPTH + } + + // If the encoder forced a KEY_FRAME decision + if (cm->frame_type == KEY_FRAME) { + cpi->refresh_last_frame = 1; + } + + cm->frame_to_show = get_frame_new_buffer(cm); + cm->frame_to_show->color_space = cm->color_space; + cm->frame_to_show->color_range = cm->color_range; + cm->frame_to_show->render_width = cm->render_width; + cm->frame_to_show->render_height = cm->render_height; + +#if CONFIG_EXT_REFS +// TODO(zoeliu): For non-ref frames, loop filtering may need to be turned +// off. +#endif // CONFIG_EXT_REFS + + // Pick the loop filter level for the frame. + loopfilter_frame(cpi, cm); + + // Build the bitstream + av1_pack_bitstream(cpi, dest, size); + + if (skip_adapt) { +#if CONFIG_EC_ADAPT + aom_free(tile_ctxs); + aom_free(cdf_ptrs); +#endif + return; + } + +#if CONFIG_REFERENCE_BUFFER + { + int i; + /* Update reference frame id values based on the value of refresh_mask */ + for (i = 0; i < REF_FRAMES; i++) { + if ((cm->refresh_mask >> i) & 1) { + cm->ref_frame_id[i] = cm->current_frame_id; + } + } + } +#endif + +#if DUMP_RECON_FRAMES == 1 + // NOTE(zoeliu): For debug - Output the filtered reconstructed video. + if (cm->show_frame) dump_filtered_recon_frames(cpi); +#endif // DUMP_RECON_FRAMES + + if (cm->seg.update_map) update_reference_segmentation_map(cpi); + + if (frame_is_intra_only(cm) == 0) { + release_scaled_references(cpi); + } + + av1_update_reference_frames(cpi); + + for (t = 0; t < TX_SIZES; t++) + av1_full_to_model_counts(cpi->td.counts->coef[t], + cpi->td.rd_counts.coef_counts[t]); +#if CONFIG_ENTROPY_STATS + av1_accumulate_frame_counts(&aggregate_fc, &cm->counts); +#endif // CONFIG_ENTROPY_STATS + if (cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { +#if CONFIG_SUBFRAME_PROB_UPDATE + cm->partial_prob_update = 0; +#endif // CONFIG_SUBFRAME_PROB_UPDATE + av1_adapt_coef_probs(cm); + av1_adapt_intra_frame_probs(cm); +#if CONFIG_EC_ADAPT + make_update_tile_list_enc(cpi, cm->tile_rows, cm->tile_cols, tile_ctxs); + av1_average_tile_coef_cdfs(cpi->common.fc, tile_ctxs, cdf_ptrs, + cm->tile_rows * cm->tile_cols); + av1_average_tile_intra_cdfs(cpi->common.fc, tile_ctxs, cdf_ptrs, + cm->tile_rows * cm->tile_cols); +#if CONFIG_PVQ + av1_average_tile_pvq_cdfs(cpi->common.fc, tile_ctxs, + cm->tile_rows * cm->tile_cols); +#endif // CONFIG_PVQ +#endif // CONFIG_EC_ADAPT +#if CONFIG_ADAPT_SCAN + av1_adapt_scan_order(cm); +#endif // CONFIG_ADAPT_SCAN + } + + if (!frame_is_intra_only(cm)) { + if (cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { + av1_adapt_inter_frame_probs(cm); + av1_adapt_mv_probs(cm, cm->allow_high_precision_mv); +#if CONFIG_EC_ADAPT + av1_average_tile_inter_cdfs(&cpi->common, cpi->common.fc, tile_ctxs, + cdf_ptrs, cm->tile_rows * cm->tile_cols); + av1_average_tile_mv_cdfs(cpi->common.fc, tile_ctxs, cdf_ptrs, + cm->tile_rows * cm->tile_cols); +#endif + } + } + + if (cpi->refresh_golden_frame == 1) + cpi->frame_flags |= FRAMEFLAGS_GOLDEN; + else + cpi->frame_flags &= ~FRAMEFLAGS_GOLDEN; + + if (cpi->refresh_alt_ref_frame == 1) + cpi->frame_flags |= FRAMEFLAGS_ALTREF; + else + cpi->frame_flags &= ~FRAMEFLAGS_ALTREF; + +#if CONFIG_EXT_REFS + if (cpi->refresh_bwd_ref_frame == 1) + cpi->frame_flags |= FRAMEFLAGS_BWDREF; + else + cpi->frame_flags &= ~FRAMEFLAGS_BWDREF; +#endif // CONFIG_EXT_REFS + +#if !CONFIG_EXT_REFS + cpi->ref_frame_flags = get_ref_frame_flags(cpi); +#endif // !CONFIG_EXT_REFS + + cm->last_frame_type = cm->frame_type; + +#if CONFIG_XIPHRC + frame_type = cm->frame_type == KEY_FRAME ? OD_I_FRAME : OD_P_FRAME; + + drop_this_frame = + od_enc_rc_update_state(&cpi->od_rc, *size << 3, cpi->refresh_golden_frame, + cpi->refresh_alt_ref_frame, frame_type, 0); + if (drop_this_frame) { + av1_rc_postencode_update_drop_frame(cpi); + ++cm->current_video_frame; +#if CONFIG_EC_ADAPT + aom_free(tile_ctxs); + aom_free(cdf_ptrs); +#endif + return; + } +#else // !CONFIG_XIPHRC + av1_rc_postencode_update(cpi, *size); +#endif // CONFIG_XIPHRC + +#if 0 + output_frame_level_debug_stats(cpi); +#endif + + if (cm->frame_type == KEY_FRAME) { + // Tell the caller that the frame was coded as a key frame + *frame_flags = cpi->frame_flags | FRAMEFLAGS_KEY; + } else { + *frame_flags = cpi->frame_flags & ~FRAMEFLAGS_KEY; + } + + // Clear the one shot update flags for segmentation map and mode/ref loop + // filter deltas. + cm->seg.update_map = 0; + cm->seg.update_data = 0; + cm->lf.mode_ref_delta_update = 0; + + // keep track of the last coded dimensions + cm->last_width = cm->width; + cm->last_height = cm->height; + + // reset to normal state now that we are done. + if (!cm->show_existing_frame) cm->last_show_frame = cm->show_frame; + + if (cm->show_frame) { +#if CONFIG_EXT_REFS +// TODO(zoeliu): We may only swamp mi and prev_mi for those frames that are +// being used as reference. +#endif // CONFIG_EXT_REFS + av1_swap_mi_and_prev_mi(cm); + // Don't increment frame counters if this was an altref buffer + // update not a real frame + ++cm->current_video_frame; + } + +#if CONFIG_EXT_REFS + // NOTE: Shall not refer to any frame not used as reference. + if (cm->is_reference_frame) +#endif // CONFIG_EXT_REFS + cm->prev_frame = cm->cur_frame; +#if CONFIG_EC_ADAPT + aom_free(tile_ctxs); + aom_free(cdf_ptrs); +#endif +} + +static void Pass0Encode(AV1_COMP *cpi, size_t *size, uint8_t *dest, + int skip_adapt, unsigned int *frame_flags) { +#if CONFIG_XIPHRC + int64_t ip_count; + int frame_type, is_golden, is_altref; + + /* Not updated during init so update it here */ + if (cpi->oxcf.rc_mode == AOM_Q) cpi->od_rc.quality = cpi->oxcf.cq_level; + + frame_type = od_frame_type(&cpi->od_rc, cpi->od_rc.cur_frame, &is_golden, + &is_altref, &ip_count); + + if (frame_type == OD_I_FRAME) { + frame_type = KEY_FRAME; + cpi->frame_flags &= FRAMEFLAGS_KEY; + } else if (frame_type == OD_P_FRAME) { + frame_type = INTER_FRAME; + } + + if (is_altref) { + cpi->refresh_alt_ref_frame = 1; + cpi->rc.source_alt_ref_active = 1; + } + + cpi->refresh_golden_frame = is_golden; + cpi->common.frame_type = frame_type; + if (is_golden) cpi->frame_flags &= FRAMEFLAGS_GOLDEN; +#else + if (cpi->oxcf.rc_mode == AOM_CBR) { + av1_rc_get_one_pass_cbr_params(cpi); + } else { + av1_rc_get_one_pass_vbr_params(cpi); + } +#endif + encode_frame_to_data_rate(cpi, size, dest, skip_adapt, frame_flags); +} + +#if !CONFIG_XIPHRC +static void Pass2Encode(AV1_COMP *cpi, size_t *size, uint8_t *dest, + unsigned int *frame_flags) { + encode_frame_to_data_rate(cpi, size, dest, 0, frame_flags); + +#if CONFIG_EXT_REFS + // Do not do post-encoding update for those frames that do not have a spot in + // a gf group, but note that an OVERLAY frame always has a spot in a gf group, + // even when show_existing_frame is used. + if (!cpi->common.show_existing_frame || cpi->rc.is_src_frame_alt_ref) { + av1_twopass_postencode_update(cpi); + } + check_show_existing_frame(cpi); +#else + av1_twopass_postencode_update(cpi); +#endif // CONFIG_EXT_REFS +} +#endif + +static void init_ref_frame_bufs(AV1_COMMON *cm) { + int i; + BufferPool *const pool = cm->buffer_pool; + cm->new_fb_idx = INVALID_IDX; + for (i = 0; i < REF_FRAMES; ++i) { + cm->ref_frame_map[i] = INVALID_IDX; + pool->frame_bufs[i].ref_count = 0; + } +} + +static void check_initial_width(AV1_COMP *cpi, +#if CONFIG_HIGHBITDEPTH + int use_highbitdepth, +#endif + int subsampling_x, int subsampling_y) { + AV1_COMMON *const cm = &cpi->common; + + if (!cpi->initial_width || +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth != use_highbitdepth || +#endif + cm->subsampling_x != subsampling_x || + cm->subsampling_y != subsampling_y) { + cm->subsampling_x = subsampling_x; + cm->subsampling_y = subsampling_y; +#if CONFIG_HIGHBITDEPTH + cm->use_highbitdepth = use_highbitdepth; +#endif + + alloc_raw_frame_buffers(cpi); + init_ref_frame_bufs(cm); + alloc_util_frame_buffers(cpi); + + init_motion_estimation(cpi); // TODO(agrange) This can be removed. + + cpi->initial_width = cm->width; + cpi->initial_height = cm->height; + cpi->initial_mbs = cm->MBs; + } +} + +int av1_receive_raw_frame(AV1_COMP *cpi, aom_enc_frame_flags_t frame_flags, + YV12_BUFFER_CONFIG *sd, int64_t time_stamp, + int64_t end_time) { + AV1_COMMON *const cm = &cpi->common; + struct aom_usec_timer timer; + int res = 0; + const int subsampling_x = sd->subsampling_x; + const int subsampling_y = sd->subsampling_y; +#if CONFIG_HIGHBITDEPTH + const int use_highbitdepth = (sd->flags & YV12_FLAG_HIGHBITDEPTH) != 0; +#endif + +#if CONFIG_HIGHBITDEPTH + check_initial_width(cpi, use_highbitdepth, subsampling_x, subsampling_y); +#else + check_initial_width(cpi, subsampling_x, subsampling_y); +#endif // CONFIG_HIGHBITDEPTH + + aom_usec_timer_start(&timer); + + if (av1_lookahead_push(cpi->lookahead, sd, time_stamp, end_time, +#if CONFIG_HIGHBITDEPTH + use_highbitdepth, +#endif // CONFIG_HIGHBITDEPTH + frame_flags)) + res = -1; + aom_usec_timer_mark(&timer); + cpi->time_receive_data += aom_usec_timer_elapsed(&timer); + + if ((cm->profile == PROFILE_0 || cm->profile == PROFILE_2) && + (subsampling_x != 1 || subsampling_y != 1)) { + aom_internal_error(&cm->error, AOM_CODEC_INVALID_PARAM, + "Non-4:2:0 color format requires profile 1 or 3"); + res = -1; + } + if ((cm->profile == PROFILE_1 || cm->profile == PROFILE_3) && + (subsampling_x == 1 && subsampling_y == 1)) { + aom_internal_error(&cm->error, AOM_CODEC_INVALID_PARAM, + "4:2:0 color format requires profile 0 or 2"); + res = -1; + } + + return res; +} + +static int frame_is_reference(const AV1_COMP *cpi) { + const AV1_COMMON *cm = &cpi->common; + + return cm->frame_type == KEY_FRAME || cpi->refresh_last_frame || + cpi->refresh_golden_frame || +#if CONFIG_EXT_REFS + cpi->refresh_bwd_ref_frame || +#endif // CONFIG_EXT_REFS + cpi->refresh_alt_ref_frame || !cm->error_resilient_mode || + cm->lf.mode_ref_delta_update || cm->seg.update_map || + cm->seg.update_data; +} + +static void adjust_frame_rate(AV1_COMP *cpi, + const struct lookahead_entry *source) { + int64_t this_duration; + int step = 0; + + if (source->ts_start == cpi->first_time_stamp_ever) { + this_duration = source->ts_end - source->ts_start; + step = 1; + } else { + int64_t last_duration = + cpi->last_end_time_stamp_seen - cpi->last_time_stamp_seen; + + this_duration = source->ts_end - cpi->last_end_time_stamp_seen; + + // do a step update if the duration changes by 10% + if (last_duration) + step = (int)((this_duration - last_duration) * 10 / last_duration); + } + + if (this_duration) { + if (step) { + av1_new_framerate(cpi, 10000000.0 / this_duration); + } else { + // Average this frame's rate into the last second's average + // frame rate. If we haven't seen 1 second yet, then average + // over the whole interval seen. + const double interval = AOMMIN( + (double)(source->ts_end - cpi->first_time_stamp_ever), 10000000.0); + double avg_duration = 10000000.0 / cpi->framerate; + avg_duration *= (interval - avg_duration + this_duration); + avg_duration /= interval; + + av1_new_framerate(cpi, 10000000.0 / avg_duration); + } + } + cpi->last_time_stamp_seen = source->ts_start; + cpi->last_end_time_stamp_seen = source->ts_end; +} + +// Returns 0 if this is not an alt ref else the offset of the source frame +// used as the arf midpoint. +static int get_arf_src_index(AV1_COMP *cpi) { + RATE_CONTROL *const rc = &cpi->rc; + int arf_src_index = 0; + if (is_altref_enabled(cpi)) { + if (cpi->oxcf.pass == 2) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + if (gf_group->update_type[gf_group->index] == ARF_UPDATE) { + arf_src_index = gf_group->arf_src_offset[gf_group->index]; + } + } else if (rc->source_alt_ref_pending) { + arf_src_index = rc->frames_till_gf_update_due; + } + } + return arf_src_index; +} + +#if CONFIG_EXT_REFS +static int get_brf_src_index(AV1_COMP *cpi) { + int brf_src_index = 0; + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + + // TODO(zoeliu): We need to add the check on the -bwd_ref command line setup + // flag. + if (gf_group->bidir_pred_enabled[gf_group->index]) { + if (cpi->oxcf.pass == 2) { + if (gf_group->update_type[gf_group->index] == BRF_UPDATE) + brf_src_index = gf_group->brf_src_offset[gf_group->index]; + } else { + // TODO(zoeliu): To re-visit the setup for this scenario + brf_src_index = cpi->rc.bipred_group_interval - 1; + } + } + + return brf_src_index; +} +#endif // CONFIG_EXT_REFS + +static void check_src_altref(AV1_COMP *cpi, + const struct lookahead_entry *source) { + RATE_CONTROL *const rc = &cpi->rc; + + // If pass == 2, the parameters set here will be reset in + // av1_rc_get_second_pass_params() + + if (cpi->oxcf.pass == 2) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + rc->is_src_frame_alt_ref = +#if CONFIG_EXT_REFS + (gf_group->update_type[gf_group->index] == INTNL_OVERLAY_UPDATE) || +#endif // CONFIG_EXT_REFS + (gf_group->update_type[gf_group->index] == OVERLAY_UPDATE); + } else { + rc->is_src_frame_alt_ref = + cpi->alt_ref_source && (source == cpi->alt_ref_source); + } + + if (rc->is_src_frame_alt_ref) { + // Current frame is an ARF overlay frame. + cpi->alt_ref_source = NULL; + + // Don't refresh the last buffer for an ARF overlay frame. It will + // become the GF so preserve last as an alternative prediction option. + cpi->refresh_last_frame = 0; + } +} + +#if CONFIG_INTERNAL_STATS +extern double av1_get_blockiness(const unsigned char *img1, int img1_pitch, + const unsigned char *img2, int img2_pitch, + int width, int height); + +static void adjust_image_stat(double y, double u, double v, double all, + ImageStat *s) { + s->stat[Y] += y; + s->stat[U] += u; + s->stat[V] += v; + s->stat[ALL] += all; + s->worst = AOMMIN(s->worst, all); +} + +static void compute_internal_stats(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + double samples = 0.0; + uint32_t in_bit_depth = 8; + uint32_t bit_depth = 8; + +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) { + in_bit_depth = cpi->oxcf.input_bit_depth; + bit_depth = cm->bit_depth; + } +#endif + if (cm->show_frame) { + const YV12_BUFFER_CONFIG *orig = cpi->source; + const YV12_BUFFER_CONFIG *recon = cpi->common.frame_to_show; + double y, u, v, frame_all; + + cpi->count++; + if (cpi->b_calculate_psnr) { + PSNR_STATS psnr; + double frame_ssim2 = 0.0, weight = 0.0; + aom_clear_system_state(); +// TODO(yaowu): unify these two versions into one. +#if CONFIG_HIGHBITDEPTH + aom_calc_highbd_psnr(orig, recon, &psnr, bit_depth, in_bit_depth); +#else + aom_calc_psnr(orig, recon, &psnr); +#endif // CONFIG_HIGHBITDEPTH + + adjust_image_stat(psnr.psnr[1], psnr.psnr[2], psnr.psnr[3], psnr.psnr[0], + &cpi->psnr); + cpi->total_sq_error += psnr.sse[0]; + cpi->total_samples += psnr.samples[0]; + samples = psnr.samples[0]; +// TODO(yaowu): unify these two versions into one. +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + frame_ssim2 = + aom_highbd_calc_ssim(orig, recon, &weight, bit_depth, in_bit_depth); + else + frame_ssim2 = aom_calc_ssim(orig, recon, &weight); +#else + frame_ssim2 = aom_calc_ssim(orig, recon, &weight); +#endif // CONFIG_HIGHBITDEPTH + + cpi->worst_ssim = AOMMIN(cpi->worst_ssim, frame_ssim2); + cpi->summed_quality += frame_ssim2 * weight; + cpi->summed_weights += weight; + +#if 0 + { + FILE *f = fopen("q_used.stt", "a"); + fprintf(f, "%5d : Y%f7.3:U%f7.3:V%f7.3:F%f7.3:S%7.3f\n", + cpi->common.current_video_frame, y2, u2, v2, + frame_psnr2, frame_ssim2); + fclose(f); + } +#endif + } + if (cpi->b_calculate_blockiness) { +#if CONFIG_HIGHBITDEPTH + if (!cm->use_highbitdepth) +#endif + { + const double frame_blockiness = + av1_get_blockiness(orig->y_buffer, orig->y_stride, recon->y_buffer, + recon->y_stride, orig->y_width, orig->y_height); + cpi->worst_blockiness = AOMMAX(cpi->worst_blockiness, frame_blockiness); + cpi->total_blockiness += frame_blockiness; + } + + if (cpi->b_calculate_consistency) { +#if CONFIG_HIGHBITDEPTH + if (!cm->use_highbitdepth) +#endif + { + const double this_inconsistency = aom_get_ssim_metrics( + orig->y_buffer, orig->y_stride, recon->y_buffer, recon->y_stride, + orig->y_width, orig->y_height, cpi->ssim_vars, &cpi->metrics, 1); + + const double peak = (double)((1 << in_bit_depth) - 1); + const double consistency = + aom_sse_to_psnr(samples, peak, cpi->total_inconsistency); + if (consistency > 0.0) + cpi->worst_consistency = + AOMMIN(cpi->worst_consistency, consistency); + cpi->total_inconsistency += this_inconsistency; + } + } + } + + frame_all = + aom_calc_fastssim(orig, recon, &y, &u, &v, bit_depth, in_bit_depth); + adjust_image_stat(y, u, v, frame_all, &cpi->fastssim); + frame_all = aom_psnrhvs(orig, recon, &y, &u, &v, bit_depth, in_bit_depth); + adjust_image_stat(y, u, v, frame_all, &cpi->psnrhvs); + } +} +#endif // CONFIG_INTERNAL_STATS + +int av1_get_compressed_data(AV1_COMP *cpi, unsigned int *frame_flags, + size_t *size, uint8_t *dest, int64_t *time_stamp, + int64_t *time_end, int flush) { + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + AV1_COMMON *const cm = &cpi->common; + BufferPool *const pool = cm->buffer_pool; + RATE_CONTROL *const rc = &cpi->rc; + struct aom_usec_timer cmptimer; + YV12_BUFFER_CONFIG *force_src_buffer = NULL; + struct lookahead_entry *last_source = NULL; + struct lookahead_entry *source = NULL; + int arf_src_index; +#if CONFIG_EXT_REFS + int brf_src_index; +#endif // CONFIG_EXT_REFS + int i; + +#if CONFIG_XIPHRC + cpi->od_rc.end_of_input = flush; +#endif + +#if CONFIG_BITSTREAM_DEBUG + assert(cpi->oxcf.max_threads == 0 && + "bitstream debug tool does not support multithreading"); + bitstream_queue_record_write(); + bitstream_queue_set_frame_write(cm->current_video_frame * 2 + cm->show_frame); +#endif + + aom_usec_timer_start(&cmptimer); + + av1_set_high_precision_mv(cpi, ALTREF_HIGH_PRECISION_MV); + + // Is multi-arf enabled. + // Note that at the moment multi_arf is only configured for 2 pass VBR + if ((oxcf->pass == 2) && (cpi->oxcf.enable_auto_arf > 1)) + cpi->multi_arf_allowed = 1; + else + cpi->multi_arf_allowed = 0; + + // Normal defaults + cm->reset_frame_context = RESET_FRAME_CONTEXT_NONE; + cm->refresh_frame_context = + (oxcf->error_resilient_mode || oxcf->frame_parallel_decoding_mode) + ? REFRESH_FRAME_CONTEXT_FORWARD + : REFRESH_FRAME_CONTEXT_BACKWARD; + + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 0; +#if CONFIG_EXT_REFS + cpi->refresh_bwd_ref_frame = 0; +#endif // CONFIG_EXT_REFS + cpi->refresh_alt_ref_frame = 0; + +#if CONFIG_EXT_REFS && !CONFIG_XIPHRC + if (oxcf->pass == 2 && cm->show_existing_frame) { + // Manage the source buffer and flush out the source frame that has been + // coded already; Also get prepared for PSNR calculation if needed. + if ((source = av1_lookahead_pop(cpi->lookahead, flush)) == NULL) { + *size = 0; + return -1; + } + cpi->source = &source->img; + // TODO(zoeliu): To track down to determine whether it's needed to adjust + // the frame rate. + *time_stamp = source->ts_start; + *time_end = source->ts_end; + + // We need to adjust frame rate for an overlay frame + if (cpi->rc.is_src_frame_alt_ref) adjust_frame_rate(cpi, source); + + // Find a free buffer for the new frame, releasing the reference previously + // held. + if (cm->new_fb_idx != INVALID_IDX) { + --pool->frame_bufs[cm->new_fb_idx].ref_count; + } + cm->new_fb_idx = get_free_fb(cm); + + if (cm->new_fb_idx == INVALID_IDX) return -1; + + // Clear down mmx registers + aom_clear_system_state(); + + // Start with a 0 size frame. + *size = 0; + + // We need to update the gf_group for show_existing overlay frame + if (cpi->rc.is_src_frame_alt_ref) av1_rc_get_second_pass_params(cpi); + + Pass2Encode(cpi, size, dest, frame_flags); + + if (cpi->b_calculate_psnr) generate_psnr_packet(cpi); + +#if CONFIG_INTERNAL_STATS + compute_internal_stats(cpi); + cpi->bytes += (int)(*size); +#endif // CONFIG_INTERNAL_STATS + + // Clear down mmx registers + aom_clear_system_state(); + + cm->show_existing_frame = 0; + return 0; + } +#endif // CONFIG_EXT_REFS && !CONFIG_XIPHRC + + // Should we encode an arf frame. + arf_src_index = get_arf_src_index(cpi); + if (arf_src_index) { + for (i = 0; i <= arf_src_index; ++i) { + struct lookahead_entry *e = av1_lookahead_peek(cpi->lookahead, i); + // Avoid creating an alt-ref if there's a forced keyframe pending. + if (e == NULL) { + break; + } else if (e->flags == AOM_EFLAG_FORCE_KF) { + arf_src_index = 0; + flush = 1; + break; + } + } + } + + if (arf_src_index) { + assert(arf_src_index <= rc->frames_to_key); + + if ((source = av1_lookahead_peek(cpi->lookahead, arf_src_index)) != NULL) { + cpi->alt_ref_source = source; + + if (oxcf->arnr_max_frames > 0) { + // Produce the filtered ARF frame. + av1_temporal_filter(cpi, arf_src_index); + aom_extend_frame_borders(&cpi->alt_ref_buffer); + force_src_buffer = &cpi->alt_ref_buffer; + } + + cm->show_frame = 0; + cm->intra_only = 0; + cpi->refresh_alt_ref_frame = 1; + cpi->refresh_golden_frame = 0; + cpi->refresh_last_frame = 0; + rc->is_src_frame_alt_ref = 0; + } + rc->source_alt_ref_pending = 0; + } + +#if CONFIG_EXT_REFS + rc->is_bwd_ref_frame = 0; + brf_src_index = get_brf_src_index(cpi); + if (brf_src_index) { + assert(brf_src_index <= rc->frames_to_key); + if ((source = av1_lookahead_peek(cpi->lookahead, brf_src_index)) != NULL) { + cm->show_frame = 0; + cm->intra_only = 0; + + cpi->refresh_bwd_ref_frame = 1; + cpi->refresh_last_frame = 0; + cpi->refresh_golden_frame = 0; + cpi->refresh_alt_ref_frame = 0; + + rc->is_bwd_ref_frame = 1; + } + } +#endif // CONFIG_EXT_REFS + + if (!source) { + // Get last frame source. + if (cm->current_video_frame > 0) { + if ((last_source = av1_lookahead_peek(cpi->lookahead, -1)) == NULL) + return -1; + } + + // Read in the source frame. + source = av1_lookahead_pop(cpi->lookahead, flush); + + if (source != NULL) { + cm->show_frame = 1; + cm->intra_only = 0; + + // Check to see if the frame should be encoded as an arf overlay. + check_src_altref(cpi, source); + } + } + + if (source) { + cpi->un_scaled_source = cpi->source = + force_src_buffer ? force_src_buffer : &source->img; + + cpi->unscaled_last_source = last_source != NULL ? &last_source->img : NULL; + + *time_stamp = source->ts_start; + *time_end = source->ts_end; + *frame_flags = (source->flags & AOM_EFLAG_FORCE_KF) ? FRAMEFLAGS_KEY : 0; + + } else { + *size = 0; + if (flush && oxcf->pass == 1 && !cpi->twopass.first_pass_done) { +#if CONFIG_XIPHRC + od_enc_rc_2pass_out(&cpi->od_rc, cpi->output_pkt_list, 1); +#else + av1_end_first_pass(cpi); /* get last stats packet */ +#endif + cpi->twopass.first_pass_done = 1; + } + return -1; + } + + if (source->ts_start < cpi->first_time_stamp_ever) { + cpi->first_time_stamp_ever = source->ts_start; + cpi->last_end_time_stamp_seen = source->ts_start; + } + + // Clear down mmx registers + aom_clear_system_state(); + + // adjust frame rates based on timestamps given + if (cm->show_frame) adjust_frame_rate(cpi, source); + + // Find a free buffer for the new frame, releasing the reference previously + // held. + if (cm->new_fb_idx != INVALID_IDX) { + --pool->frame_bufs[cm->new_fb_idx].ref_count; + } + cm->new_fb_idx = get_free_fb(cm); + + if (cm->new_fb_idx == INVALID_IDX) return -1; + + cm->cur_frame = &pool->frame_bufs[cm->new_fb_idx]; + +#if CONFIG_EXT_REFS + if (oxcf->pass == 2) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + cpi->alt_fb_idx = cpi->arf_map[gf_group->arf_ref_idx[gf_group->index]]; + } +#else + if (cpi->multi_arf_allowed) { + if (cm->frame_type == KEY_FRAME) { + init_buffer_indices(cpi); + } else if (oxcf->pass == 2) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + cpi->alt_fb_idx = gf_group->arf_ref_idx[gf_group->index]; + } + } +#endif // CONFIG_EXT_REFS + + // Start with a 0 size frame. + *size = 0; + + cpi->frame_flags = *frame_flags; + + if (oxcf->pass == 2) { +#if CONFIG_XIPHRC + if (od_enc_rc_2pass_in(&cpi->od_rc) < 0) return -1; + } +#else + av1_rc_get_second_pass_params(cpi); + } else if (oxcf->pass == 1) { + set_frame_size(cpi); + } +#endif + + if (cpi->oxcf.pass != 0 || frame_is_intra_only(cm) == 1) { + for (i = 0; i < TOTAL_REFS_PER_FRAME; ++i) + cpi->scaled_ref_idx[i] = INVALID_IDX; + } + +#if CONFIG_AOM_QM + cm->using_qmatrix = cpi->oxcf.using_qm; + cm->min_qmlevel = cpi->oxcf.qm_minlevel; + cm->max_qmlevel = cpi->oxcf.qm_maxlevel; +#endif + +#if CONFIG_REFERENCE_BUFFER + if (*time_stamp == 0) { + cpi->common.current_frame_id = -1; + } +#endif + +#if CONFIG_XIPHRC + if (oxcf->pass == 1) { + size_t tmp; + if (cpi->od_rc.cur_frame == 0) Pass0Encode(cpi, &tmp, dest, 1, frame_flags); + cpi->od_rc.firstpass_quant = cpi->od_rc.target_quantizer; + Pass0Encode(cpi, &tmp, dest, 0, frame_flags); + od_enc_rc_2pass_out(&cpi->od_rc, cpi->output_pkt_list, 0); + } else if (oxcf->pass == 2) { + Pass0Encode(cpi, size, dest, 0, frame_flags); + } else { + if (cpi->od_rc.cur_frame == 0) { + size_t tmp; + Pass0Encode(cpi, &tmp, dest, 1, frame_flags); + } + Pass0Encode(cpi, size, dest, 0, frame_flags); + } +#else + if (oxcf->pass == 1) { + cpi->td.mb.e_mbd.lossless[0] = is_lossless_requested(oxcf); + av1_first_pass(cpi, source); + } else if (oxcf->pass == 2) { + Pass2Encode(cpi, size, dest, frame_flags); + } else { + // One pass encode + Pass0Encode(cpi, size, dest, 0, frame_flags); + } +#endif + + if (!cm->error_resilient_mode) + cm->frame_contexts[cm->frame_context_idx] = *cm->fc; + + // No frame encoded, or frame was dropped, release scaled references. + if ((*size == 0) && (frame_is_intra_only(cm) == 0)) { + release_scaled_references(cpi); + } + + if (*size > 0) { + cpi->droppable = !frame_is_reference(cpi); + } + + aom_usec_timer_mark(&cmptimer); + cpi->time_compress_data += aom_usec_timer_elapsed(&cmptimer); + + if (cpi->b_calculate_psnr && oxcf->pass != 1 && cm->show_frame) + generate_psnr_packet(cpi); + +#if CONFIG_INTERNAL_STATS + if (oxcf->pass != 1) { + compute_internal_stats(cpi); + cpi->bytes += (int)(*size); + } +#endif // CONFIG_INTERNAL_STATS + +#if CONFIG_XIPHRC + cpi->od_rc.cur_frame++; +#endif + + aom_clear_system_state(); + + return 0; +} + +int av1_get_preview_raw_frame(AV1_COMP *cpi, YV12_BUFFER_CONFIG *dest) { + AV1_COMMON *cm = &cpi->common; + if (!cm->show_frame) { + return -1; + } else { + int ret; + if (cm->frame_to_show) { + *dest = *cm->frame_to_show; + dest->y_width = cm->width; + dest->y_height = cm->height; + dest->uv_width = cm->width >> cm->subsampling_x; + dest->uv_height = cm->height >> cm->subsampling_y; + ret = 0; + } else { + ret = -1; + } + aom_clear_system_state(); + return ret; + } +} + +int av1_get_last_show_frame(AV1_COMP *cpi, YV12_BUFFER_CONFIG *frame) { + if (cpi->last_show_frame_buf_idx == INVALID_IDX) return -1; + + *frame = + cpi->common.buffer_pool->frame_bufs[cpi->last_show_frame_buf_idx].buf; + return 0; +} + +int av1_set_internal_size(AV1_COMP *cpi, AOM_SCALING horiz_mode, + AOM_SCALING vert_mode) { + AV1_COMMON *cm = &cpi->common; + int hr = 0, hs = 0, vr = 0, vs = 0; + + if (horiz_mode > ONETWO || vert_mode > ONETWO) return -1; + + Scale2Ratio(horiz_mode, &hr, &hs); + Scale2Ratio(vert_mode, &vr, &vs); + + // always go to the next whole number + cm->width = (hs - 1 + cpi->oxcf.width * hr) / hs; + cm->height = (vs - 1 + cpi->oxcf.height * vr) / vs; + assert(cm->width <= cpi->initial_width); + assert(cm->height <= cpi->initial_height); + + update_frame_size(cpi); + + return 0; +} + +int av1_set_size_literal(AV1_COMP *cpi, unsigned int width, + unsigned int height) { + AV1_COMMON *cm = &cpi->common; +#if CONFIG_HIGHBITDEPTH + check_initial_width(cpi, cm->use_highbitdepth, 1, 1); +#else + check_initial_width(cpi, 1, 1); +#endif // CONFIG_HIGHBITDEPTH + + if (width) { + cm->width = width; + if (cm->width > cpi->initial_width) { + cm->width = cpi->initial_width; + printf("Warning: Desired width too large, changed to %d\n", cm->width); + } + } + + if (height) { + cm->height = height; + if (cm->height > cpi->initial_height) { + cm->height = cpi->initial_height; + printf("Warning: Desired height too large, changed to %d\n", cm->height); + } + } + assert(cm->width <= cpi->initial_width); + assert(cm->height <= cpi->initial_height); + + update_frame_size(cpi); + + return 0; +} + +int av1_get_quantizer(AV1_COMP *cpi) { return cpi->common.base_qindex; } + +void av1_apply_encoding_flags(AV1_COMP *cpi, aom_enc_frame_flags_t flags) { + if (flags & + (AOM_EFLAG_NO_REF_LAST | AOM_EFLAG_NO_REF_GF | AOM_EFLAG_NO_REF_ARF)) { + int ref = AOM_REFFRAME_ALL; + + if (flags & AOM_EFLAG_NO_REF_LAST) { + ref ^= AOM_LAST_FLAG; +#if CONFIG_EXT_REFS + ref ^= AOM_LAST2_FLAG; + ref ^= AOM_LAST3_FLAG; +#endif // CONFIG_EXT_REFS + } + + if (flags & AOM_EFLAG_NO_REF_GF) ref ^= AOM_GOLD_FLAG; + + if (flags & AOM_EFLAG_NO_REF_ARF) ref ^= AOM_ALT_FLAG; + + av1_use_as_reference(cpi, ref); + } + + if (flags & + (AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | AOM_EFLAG_NO_UPD_ARF | + AOM_EFLAG_FORCE_GF | AOM_EFLAG_FORCE_ARF)) { + int upd = AOM_REFFRAME_ALL; + + if (flags & AOM_EFLAG_NO_UPD_LAST) { + upd ^= AOM_LAST_FLAG; +#if CONFIG_EXT_REFS + upd ^= AOM_LAST2_FLAG; + upd ^= AOM_LAST3_FLAG; +#endif // CONFIG_EXT_REFS + } + + if (flags & AOM_EFLAG_NO_UPD_GF) upd ^= AOM_GOLD_FLAG; + + if (flags & AOM_EFLAG_NO_UPD_ARF) upd ^= AOM_ALT_FLAG; + + av1_update_reference(cpi, upd); + } + + if (flags & AOM_EFLAG_NO_UPD_ENTROPY) { + av1_update_entropy(cpi, 0); + } +} diff --git a/third_party/aom/av1/encoder/encoder.h b/third_party/aom/av1/encoder/encoder.h new file mode 100644 index 0000000000..4e7aef8fcd --- /dev/null +++ b/third_party/aom/av1/encoder/encoder.h @@ -0,0 +1,883 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_ENCODER_H_ +#define AV1_ENCODER_ENCODER_H_ + +#include + +#include "./aom_config.h" +#include "aom/aomcx.h" + +#include "av1/common/alloccommon.h" +#include "av1/common/entropymode.h" +#include "av1/common/thread_common.h" +#include "av1/common/onyxc_int.h" +#include "av1/encoder/aq_cyclicrefresh.h" +#if CONFIG_ANS +#include "aom_dsp/ans.h" +#include "aom_dsp/buf_ans.h" +#endif +#include "av1/encoder/av1_quantize.h" +#include "av1/encoder/context_tree.h" +#include "av1/encoder/encodemb.h" +#include "av1/encoder/firstpass.h" +#include "av1/encoder/lookahead.h" +#include "av1/encoder/mbgraph.h" +#include "av1/encoder/mcomp.h" +#include "av1/encoder/ratectrl.h" +#include "av1/encoder/rd.h" +#include "av1/encoder/speed_features.h" +#include "av1/encoder/tokenize.h" +#include "av1/encoder/variance_tree.h" +#if CONFIG_XIPHRC +#include "av1/encoder/ratectrl_xiph.h" +#endif + +#if CONFIG_INTERNAL_STATS +#include "aom_dsp/ssim.h" +#endif +#include "aom_dsp/variance.h" +#include "aom/internal/aom_codec_internal.h" +#include "aom_util/aom_thread.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + int nmvjointcost[MV_JOINTS]; + int nmvcosts[2][MV_VALS]; + int nmvcosts_hp[2][MV_VALS]; + +#if CONFIG_REF_MV + int nmv_vec_cost[NMV_CONTEXTS][MV_JOINTS]; + int nmv_costs[NMV_CONTEXTS][2][MV_VALS]; + int nmv_costs_hp[NMV_CONTEXTS][2][MV_VALS]; +#endif + + // 0 = Intra, Last, GF, ARF + signed char last_ref_lf_deltas[TOTAL_REFS_PER_FRAME]; + // 0 = ZERO_MV, MV + signed char last_mode_lf_deltas[MAX_MODE_LF_DELTAS]; + + FRAME_CONTEXT fc; +} CODING_CONTEXT; + +typedef enum { + // regular inter frame + REGULAR_FRAME = 0, + // alternate reference frame + ARF_FRAME = 1, + // overlay frame + OVERLAY_FRAME = 2, + // golden frame + GLD_FRAME = 3, +#if CONFIG_EXT_REFS + // backward reference frame + BRF_FRAME = 4, + // extra alternate reference frame + EXT_ARF_FRAME = 5 +#endif +} FRAME_CONTEXT_INDEX; + +typedef enum { + NORMAL = 0, + FOURFIVE = 1, + THREEFIVE = 2, + ONETWO = 3 +} AOM_SCALING; + +typedef enum { + // Good Quality Fast Encoding. The encoder balances quality with the amount of + // time it takes to encode the output. Speed setting controls how fast. + GOOD +} MODE; + +typedef enum { + FRAMEFLAGS_KEY = 1 << 0, + FRAMEFLAGS_GOLDEN = 1 << 1, +#if CONFIG_EXT_REFS + FRAMEFLAGS_BWDREF = 1 << 2, + FRAMEFLAGS_ALTREF = 1 << 3, +#else + FRAMEFLAGS_ALTREF = 1 << 2, +#endif // CONFIG_EXT_REFS +} FRAMETYPE_FLAGS; + +typedef enum { + NO_AQ = 0, + VARIANCE_AQ = 1, + COMPLEXITY_AQ = 2, + CYCLIC_REFRESH_AQ = 3, +#if CONFIG_DELTA_Q && !CONFIG_EXT_DELTA_Q + DELTA_AQ = 4, +#endif + AQ_MODE_COUNT // This should always be the last member of the enum +} AQ_MODE; +#if CONFIG_EXT_DELTA_Q +typedef enum { + NO_DELTA_Q = 0, + DELTA_Q_ONLY = 1, + DELTA_Q_LF = 2, + DELTAQ_MODE_COUNT // This should always be the last member of the enum +} DELTAQ_MODE; +#endif +typedef enum { + RESIZE_NONE = 0, // No frame resizing allowed. + RESIZE_FIXED = 1, // All frames are coded at the specified dimension. + RESIZE_DYNAMIC = 2 // Coded size of each frame is determined by the codec. +} RESIZE_TYPE; + +typedef struct AV1EncoderConfig { + BITSTREAM_PROFILE profile; + aom_bit_depth_t bit_depth; // Codec bit-depth. + int width; // width of data passed to the compressor + int height; // height of data passed to the compressor + unsigned int input_bit_depth; // Input bit depth. + double init_framerate; // set to passed in framerate + int64_t target_bandwidth; // bandwidth to be used in bits per second + + int noise_sensitivity; // pre processing blur: recommendation 0 + int sharpness; // sharpening output: recommendation 0: + int speed; + // maximum allowed bitrate for any intra frame in % of bitrate target. + unsigned int rc_max_intra_bitrate_pct; + // maximum allowed bitrate for any inter frame in % of bitrate target. + unsigned int rc_max_inter_bitrate_pct; + // percent of rate boost for golden frame in CBR mode. + unsigned int gf_cbr_boost_pct; + + MODE mode; + int pass; + + // Key Framing Operations + int auto_key; // autodetect cut scenes and set the keyframes + int key_freq; // maximum distance to key frame. + + int lag_in_frames; // how many frames lag before we start encoding + + // ---------------------------------------------------------------- + // DATARATE CONTROL OPTIONS + + // vbr, cbr, constrained quality or constant quality + enum aom_rc_mode rc_mode; + + // buffer targeting aggressiveness + int under_shoot_pct; + int over_shoot_pct; + + // buffering parameters + int64_t starting_buffer_level_ms; + int64_t optimal_buffer_level_ms; + int64_t maximum_buffer_size_ms; + + // Frame drop threshold. + int drop_frames_water_mark; + + // controlling quality + int fixed_q; + int worst_allowed_q; + int best_allowed_q; + int cq_level; + AQ_MODE aq_mode; // Adaptive Quantization mode +#if CONFIG_EXT_DELTA_Q + DELTAQ_MODE deltaq_mode; +#endif +#if CONFIG_AOM_QM + int using_qm; + int qm_minlevel; + int qm_maxlevel; +#endif +#if CONFIG_TILE_GROUPS + unsigned int num_tile_groups; + unsigned int mtu; +#endif + +#if CONFIG_TEMPMV_SIGNALING + unsigned int disable_tempmv; +#endif + // Internal frame size scaling. + RESIZE_TYPE resize_mode; + int scaled_frame_width; + int scaled_frame_height; + + // Enable feature to reduce the frame quantization every x frames. + int frame_periodic_boost; + + // two pass datarate control + int two_pass_vbrbias; // two pass datarate control tweaks + int two_pass_vbrmin_section; + int two_pass_vbrmax_section; + // END DATARATE CONTROL OPTIONS + // ---------------------------------------------------------------- + + int enable_auto_arf; +#if CONFIG_EXT_REFS + int enable_auto_brf; // (b)ackward (r)ef (f)rame +#endif // CONFIG_EXT_REFS + + /* Bitfield defining the error resiliency features to enable. + * Can provide decodable frames after losses in previous + * frames and decodable partitions after losses in the same frame. + */ + unsigned int error_resilient_mode; + + /* Bitfield defining the parallel decoding mode where the + * decoding in successive frames may be conducted in parallel + * just by decoding the frame headers. + */ + unsigned int frame_parallel_decoding_mode; + + int arnr_max_frames; + int arnr_strength; + + int min_gf_interval; + int max_gf_interval; + + int tile_columns; + int tile_rows; +#if CONFIG_DEPENDENT_HORZTILES + int dependent_horz_tiles; +#endif +#if CONFIG_LOOPFILTERING_ACROSS_TILES + int loop_filter_across_tiles_enabled; +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + + int max_threads; + + aom_fixed_buf_t two_pass_stats_in; + struct aom_codec_pkt_list *output_pkt_list; + +#if CONFIG_FP_MB_STATS + aom_fixed_buf_t firstpass_mb_stats_in; +#endif + + aom_tune_metric tuning; + aom_tune_content content; +#if CONFIG_HIGHBITDEPTH + int use_highbitdepth; +#endif + aom_color_space_t color_space; + int color_range; + int render_width; + int render_height; + +#if CONFIG_EXT_PARTITION + aom_superblock_size_t superblock_size; +#endif // CONFIG_EXT_PARTITION +#if CONFIG_ANS && ANS_MAX_SYMBOLS + int ans_window_size_log2; +#endif // CONFIG_ANS && ANS_MAX_SYMBOLS +#if CONFIG_EXT_TILE + unsigned int tile_encoding_mode; +#endif // CONFIG_EXT_TILE + + unsigned int motion_vector_unit_test; +} AV1EncoderConfig; + +static INLINE int is_lossless_requested(const AV1EncoderConfig *cfg) { + return cfg->best_allowed_q == 0 && cfg->worst_allowed_q == 0; +} + +// TODO(jingning) All spatially adaptive variables should go to TileDataEnc. +typedef struct TileDataEnc { + TileInfo tile_info; + int thresh_freq_fact[BLOCK_SIZES][MAX_MODES]; + int mode_map[BLOCK_SIZES][MAX_MODES]; + int m_search_count; + int ex_search_count; +#if CONFIG_PVQ + PVQ_QUEUE pvq_q; +#endif +#if CONFIG_CFL + CFL_CTX cfl; +#endif +#if CONFIG_EC_ADAPT + DECLARE_ALIGNED(16, FRAME_CONTEXT, tctx); +#endif +} TileDataEnc; + +typedef struct RD_COUNTS { + av1_coeff_count coef_counts[TX_SIZES][PLANE_TYPES]; + int64_t comp_pred_diff[REFERENCE_MODES]; +#if CONFIG_GLOBAL_MOTION + // Stores number of 4x4 blocks using global motion per reference frame. + int global_motion_used[TOTAL_REFS_PER_FRAME]; +#endif // CONFIG_GLOBAL_MOTION +} RD_COUNTS; + +typedef struct ThreadData { + MACROBLOCK mb; + RD_COUNTS rd_counts; + FRAME_COUNTS *counts; + + PICK_MODE_CONTEXT *leaf_tree; + PC_TREE *pc_tree; + PC_TREE *pc_root[MAX_MIB_SIZE_LOG2 - MIN_MIB_SIZE_LOG2 + 1]; + + VAR_TREE *var_tree; + VAR_TREE *var_root[MAX_MIB_SIZE_LOG2 - MIN_MIB_SIZE_LOG2 + 1]; +} ThreadData; + +struct EncWorkerData; + +typedef struct ActiveMap { + int enabled; + int update; + unsigned char *map; +} ActiveMap; + +#define NUM_STAT_TYPES 4 // types of stats: Y, U, V and ALL + +typedef struct IMAGE_STAT { + double stat[NUM_STAT_TYPES]; + double worst; +} ImageStat; + +#undef NUM_STAT_TYPES + +typedef struct { + int ref_count; + YV12_BUFFER_CONFIG buf; +} EncRefCntBuffer; + +#if CONFIG_SUBFRAME_PROB_UPDATE +typedef struct SUBFRAME_STATS { + av1_coeff_probs_model coef_probs_buf[COEF_PROBS_BUFS][TX_SIZES][PLANE_TYPES]; + av1_coeff_count coef_counts_buf[COEF_PROBS_BUFS][TX_SIZES][PLANE_TYPES]; + unsigned int eob_counts_buf[COEF_PROBS_BUFS][TX_SIZES][PLANE_TYPES][REF_TYPES] + [COEF_BANDS][COEFF_CONTEXTS]; + av1_coeff_probs_model enc_starting_coef_probs[TX_SIZES][PLANE_TYPES]; +} SUBFRAME_STATS; +#endif // CONFIG_SUBFRAME_PROB_UPDATE + +typedef struct TileBufferEnc { + uint8_t *data; + size_t size; +} TileBufferEnc; + +typedef struct AV1_COMP { + QUANTS quants; + ThreadData td; + MB_MODE_INFO_EXT *mbmi_ext_base; + DECLARE_ALIGNED(16, int16_t, y_dequant[QINDEX_RANGE][8]); // 8: SIMD width + DECLARE_ALIGNED(16, int16_t, uv_dequant[QINDEX_RANGE][8]); // 8: SIMD width +#if CONFIG_NEW_QUANT + DECLARE_ALIGNED(16, dequant_val_type_nuq, + y_dequant_val_nuq[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS]); + DECLARE_ALIGNED(16, dequant_val_type_nuq, + uv_dequant_val_nuq[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS]); +#endif // CONFIG_NEW_QUANT + AV1_COMMON common; + AV1EncoderConfig oxcf; + struct lookahead_ctx *lookahead; + struct lookahead_entry *alt_ref_source; + + YV12_BUFFER_CONFIG *source; + YV12_BUFFER_CONFIG *last_source; // NULL for first frame and alt_ref frames + YV12_BUFFER_CONFIG *un_scaled_source; + YV12_BUFFER_CONFIG scaled_source; + YV12_BUFFER_CONFIG *unscaled_last_source; + YV12_BUFFER_CONFIG scaled_last_source; + + // Up-sampled reference buffers + // NOTE(zoeliu): It is needed to allocate sufficient space to the up-sampled + // reference buffers, which should include the up-sampled version of all the + // possibly stored references plus the currently coded frame itself. + EncRefCntBuffer upsampled_ref_bufs[REF_FRAMES + 1]; + int upsampled_ref_idx[REF_FRAMES + 1]; + + // For a still frame, this flag is set to 1 to skip partition search. + int partition_search_skippable_frame; + + int scaled_ref_idx[TOTAL_REFS_PER_FRAME]; +#if CONFIG_EXT_REFS + int lst_fb_idxes[LAST_REF_FRAMES]; +#else + int lst_fb_idx; +#endif // CONFIG_EXT_REFS + int gld_fb_idx; +#if CONFIG_EXT_REFS + int bwd_fb_idx; // BWD_REF_FRAME +#endif // CONFIG_EXT_REFS + int alt_fb_idx; + + int last_show_frame_buf_idx; // last show frame buffer index + + int refresh_last_frame; + int refresh_golden_frame; +#if CONFIG_EXT_REFS + int refresh_bwd_ref_frame; +#endif // CONFIG_EXT_REFS + int refresh_alt_ref_frame; + + int ext_refresh_frame_flags_pending; + int ext_refresh_last_frame; + int ext_refresh_golden_frame; + int ext_refresh_alt_ref_frame; + + int ext_refresh_frame_context_pending; + int ext_refresh_frame_context; + + YV12_BUFFER_CONFIG last_frame_uf; +#if CONFIG_LOOP_RESTORATION + YV12_BUFFER_CONFIG last_frame_db; + YV12_BUFFER_CONFIG trial_frame_rst; + uint8_t *extra_rstbuf; // Extra buffers used in restoration search + RestorationInfo rst_search[MAX_MB_PLANE]; // Used for encoder side search +#endif // CONFIG_LOOP_RESTORATION + + // Ambient reconstruction err target for force key frames + int64_t ambient_err; + + RD_OPT rd; + + CODING_CONTEXT coding_context; + +#if CONFIG_REF_MV + int nmv_costs[NMV_CONTEXTS][2][MV_VALS]; + int nmv_costs_hp[NMV_CONTEXTS][2][MV_VALS]; +#endif + + int nmvcosts[2][MV_VALS]; + int nmvcosts_hp[2][MV_VALS]; + int nmvsadcosts[2][MV_VALS]; + int nmvsadcosts_hp[2][MV_VALS]; + + int64_t last_time_stamp_seen; + int64_t last_end_time_stamp_seen; + int64_t first_time_stamp_ever; + + RATE_CONTROL rc; +#if CONFIG_XIPHRC + od_rc_state od_rc; +#endif + double framerate; + + // NOTE(zoeliu): Any inter frame allows maximum of REF_FRAMES inter + // references; Plus the currently coded frame itself, it is needed to allocate + // sufficient space to the size of the maximum possible number of frames. + int interp_filter_selected[REF_FRAMES + 1][SWITCHABLE]; + + struct aom_codec_pkt_list *output_pkt_list; + + MBGRAPH_FRAME_STATS mbgraph_stats[MAX_LAG_BUFFERS]; + int mbgraph_n_frames; // number of frames filled in the above + int static_mb_pct; // % forced skip mbs by segmentation + int ref_frame_flags; + + SPEED_FEATURES sf; + + unsigned int max_mv_magnitude; + int mv_step_param; + + int allow_comp_inter_inter; + + uint8_t *segmentation_map; + + CYCLIC_REFRESH *cyclic_refresh; + ActiveMap active_map; + + fractional_mv_step_fp *find_fractional_mv_step; + av1_full_search_fn_t full_search_sad; // It is currently unused. + av1_diamond_search_fn_t diamond_search_sad; + aom_variance_fn_ptr_t fn_ptr[BLOCK_SIZES]; + uint64_t time_receive_data; + uint64_t time_compress_data; + uint64_t time_pick_lpf; + uint64_t time_encode_sb_row; + +#if CONFIG_FP_MB_STATS + int use_fp_mb_stats; +#endif + + TWO_PASS twopass; + + YV12_BUFFER_CONFIG alt_ref_buffer; + +#if CONFIG_INTERNAL_STATS + unsigned int mode_chosen_counts[MAX_MODES]; + + int count; + uint64_t total_sq_error; + uint64_t total_samples; + ImageStat psnr; + + double total_blockiness; + double worst_blockiness; + + int bytes; + double summed_quality; + double summed_weights; + unsigned int tot_recode_hits; + double worst_ssim; + + ImageStat fastssim; + ImageStat psnrhvs; + + int b_calculate_blockiness; + int b_calculate_consistency; + + double total_inconsistency; + double worst_consistency; + Ssimv *ssim_vars; + Metrics metrics; +#endif + int b_calculate_psnr; + + int droppable; + + int initial_width; + int initial_height; + int initial_mbs; // Number of MBs in the full-size frame; to be used to + // normalize the firstpass stats. This will differ from the + // number of MBs in the current frame when the frame is + // scaled. + + // Store frame variance info in SOURCE_VAR_BASED_PARTITION search type. + DIFF *source_diff_var; + // The threshold used in SOURCE_VAR_BASED_PARTITION search type. + unsigned int source_var_thresh; + int frames_till_next_var_check; + + int frame_flags; + + search_site_config ss_cfg; + + int mbmode_cost[BLOCK_SIZE_GROUPS][INTRA_MODES]; +#if CONFIG_REF_MV + int newmv_mode_cost[NEWMV_MODE_CONTEXTS][2]; + int zeromv_mode_cost[ZEROMV_MODE_CONTEXTS][2]; + int refmv_mode_cost[REFMV_MODE_CONTEXTS][2]; + int drl_mode_cost0[DRL_MODE_CONTEXTS][2]; +#endif + + unsigned int inter_mode_cost[INTER_MODE_CONTEXTS][INTER_MODES]; +#if CONFIG_EXT_INTER + unsigned int inter_compound_mode_cost[INTER_MODE_CONTEXTS] + [INTER_COMPOUND_MODES]; + unsigned int interintra_mode_cost[BLOCK_SIZE_GROUPS][INTERINTRA_MODES]; +#endif // CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + int motion_mode_cost[BLOCK_SIZES][MOTION_MODES]; +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + int motion_mode_cost1[BLOCK_SIZES][2]; +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + int intra_uv_mode_cost[INTRA_MODES][INTRA_MODES]; + int y_mode_costs[INTRA_MODES][INTRA_MODES][INTRA_MODES]; + int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS]; +#if CONFIG_EXT_PARTITION_TYPES + int partition_cost[PARTITION_CONTEXTS + CONFIG_UNPOISON_PARTITION_CTX] + [EXT_PARTITION_TYPES]; +#else + int partition_cost[PARTITION_CONTEXTS + CONFIG_UNPOISON_PARTITION_CTX] + [PARTITION_TYPES]; +#endif +#if CONFIG_PALETTE + int palette_y_size_cost[PALETTE_BLOCK_SIZES][PALETTE_SIZES]; + int palette_uv_size_cost[PALETTE_BLOCK_SIZES][PALETTE_SIZES]; + int palette_y_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS] + [PALETTE_COLORS]; + int palette_uv_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS] + [PALETTE_COLORS]; +#endif // CONFIG_PALETTE + int tx_size_cost[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES]; +#if CONFIG_EXT_TX + int inter_tx_type_costs[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES]; + int intra_tx_type_costs[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES] + [TX_TYPES]; +#else + int intra_tx_type_costs[EXT_TX_SIZES][TX_TYPES][TX_TYPES]; + int inter_tx_type_costs[EXT_TX_SIZES][TX_TYPES]; +#endif // CONFIG_EXT_TX +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP + int intra_filter_cost[INTRA_FILTERS + 1][INTRA_FILTERS]; +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA +#if CONFIG_LOOP_RESTORATION + int switchable_restore_cost[RESTORE_SWITCHABLE_TYPES]; +#endif // CONFIG_LOOP_RESTORATION +#if CONFIG_GLOBAL_MOTION + int gmtype_cost[TRANS_TYPES]; + int gmparams_cost[TOTAL_REFS_PER_FRAME]; +#endif // CONFIG_GLOBAL_MOTION + + int multi_arf_allowed; + int multi_arf_enabled; + int multi_arf_last_grp_enabled; + + TileDataEnc *tile_data; + int allocated_tiles; // Keep track of memory allocated for tiles. + + TOKENEXTRA *tile_tok[MAX_TILE_ROWS][MAX_TILE_COLS]; + unsigned int tok_count[MAX_TILE_ROWS][MAX_TILE_COLS]; + + TileBufferEnc tile_buffers[MAX_TILE_ROWS][MAX_TILE_COLS]; + + int resize_pending; + int resize_state; + int resize_scale_num; + int resize_scale_den; + int resize_avg_qp; + int resize_buffer_underflow; + int resize_count; + + // VAR_BASED_PARTITION thresholds + // 0 - threshold_128x128; + // 1 - threshold_64x64; + // 2 - threshold_32x32; + // 3 - threshold_16x16; + // 4 - threshold_8x8; + int64_t vbp_thresholds[5]; + int64_t vbp_threshold_minmax; + int64_t vbp_threshold_sad; + BLOCK_SIZE vbp_bsize_min; + + // VARIANCE_AQ segment map refresh + int vaq_refresh; + + // Multi-threading + int num_workers; + AVxWorker *workers; + struct EncWorkerData *tile_thr_data; + AV1LfSync lf_row_sync; +#if CONFIG_SUBFRAME_PROB_UPDATE + SUBFRAME_STATS subframe_stats; + // TODO(yaowu): minimize the size of count buffers + SUBFRAME_STATS wholeframe_stats; + av1_coeff_stats branch_ct_buf[COEF_PROBS_BUFS][TX_SIZES][PLANE_TYPES]; +#endif // CONFIG_SUBFRAME_PROB_UPDATE +#if CONFIG_ANS + struct BufAnsCoder buf_ans; +#endif +#if CONFIG_EXT_REFS + int refresh_frame_mask; + int existing_fb_idx_to_show; + int is_arf_filter_off[MAX_EXT_ARFS + 1]; + int num_extra_arfs; + int arf_map[MAX_EXT_ARFS + 1]; +#endif // CONFIG_EXT_REFS +#if CONFIG_GLOBAL_MOTION + int global_motion_search_done; +#endif +#if CONFIG_REFERENCE_BUFFER + SequenceHeader seq_params; +#endif +#if CONFIG_LV_MAP + tran_low_t *tcoeff_buf[MAX_MB_PLANE]; +#endif +} AV1_COMP; + +void av1_initialize_enc(void); + +struct AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf, + BufferPool *const pool); +void av1_remove_compressor(AV1_COMP *cpi); + +void av1_change_config(AV1_COMP *cpi, const AV1EncoderConfig *oxcf); + +// receive a frames worth of data. caller can assume that a copy of this +// frame is made and not just a copy of the pointer.. +int av1_receive_raw_frame(AV1_COMP *cpi, aom_enc_frame_flags_t frame_flags, + YV12_BUFFER_CONFIG *sd, int64_t time_stamp, + int64_t end_time_stamp); + +int av1_get_compressed_data(AV1_COMP *cpi, unsigned int *frame_flags, + size_t *size, uint8_t *dest, int64_t *time_stamp, + int64_t *time_end, int flush); + +int av1_get_preview_raw_frame(AV1_COMP *cpi, YV12_BUFFER_CONFIG *dest); + +int av1_get_last_show_frame(AV1_COMP *cpi, YV12_BUFFER_CONFIG *frame); + +int av1_use_as_reference(AV1_COMP *cpi, int ref_frame_flags); + +void av1_update_reference(AV1_COMP *cpi, int ref_frame_flags); + +int av1_copy_reference_enc(AV1_COMP *cpi, AOM_REFFRAME ref_frame_flag, + YV12_BUFFER_CONFIG *sd); + +int av1_set_reference_enc(AV1_COMP *cpi, AOM_REFFRAME ref_frame_flag, + YV12_BUFFER_CONFIG *sd); + +int av1_update_entropy(AV1_COMP *cpi, int update); + +int av1_set_active_map(AV1_COMP *cpi, unsigned char *map, int rows, int cols); + +int av1_get_active_map(AV1_COMP *cpi, unsigned char *map, int rows, int cols); + +int av1_set_internal_size(AV1_COMP *cpi, AOM_SCALING horiz_mode, + AOM_SCALING vert_mode); + +int av1_set_size_literal(AV1_COMP *cpi, unsigned int width, + unsigned int height); + +int av1_get_quantizer(struct AV1_COMP *cpi); + +void av1_full_to_model_counts(av1_coeff_count_model *model_count, + av1_coeff_count *full_count); + +static INLINE int frame_is_kf_gf_arf(const AV1_COMP *cpi) { + return frame_is_intra_only(&cpi->common) || cpi->refresh_alt_ref_frame || + (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref); +} + +static INLINE int get_ref_frame_map_idx(const AV1_COMP *cpi, + MV_REFERENCE_FRAME ref_frame) { +#if CONFIG_EXT_REFS + if (ref_frame >= LAST_FRAME && ref_frame <= LAST3_FRAME) + return cpi->lst_fb_idxes[ref_frame - 1]; +#else + if (ref_frame == LAST_FRAME) return cpi->lst_fb_idx; +#endif // CONFIG_EXT_REFS + else if (ref_frame == GOLDEN_FRAME) + return cpi->gld_fb_idx; +#if CONFIG_EXT_REFS + else if (ref_frame == BWDREF_FRAME) + return cpi->bwd_fb_idx; +#endif // CONFIG_EXT_REFS + else + return cpi->alt_fb_idx; +} + +static INLINE int get_ref_frame_buf_idx(const AV1_COMP *cpi, + MV_REFERENCE_FRAME ref_frame) { + const AV1_COMMON *const cm = &cpi->common; + const int map_idx = get_ref_frame_map_idx(cpi, ref_frame); + return (map_idx != INVALID_IDX) ? cm->ref_frame_map[map_idx] : INVALID_IDX; +} + +static INLINE YV12_BUFFER_CONFIG *get_ref_frame_buffer( + const AV1_COMP *cpi, MV_REFERENCE_FRAME ref_frame) { + const AV1_COMMON *const cm = &cpi->common; + const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame); + return buf_idx != INVALID_IDX ? &cm->buffer_pool->frame_bufs[buf_idx].buf + : NULL; +} + +static INLINE const YV12_BUFFER_CONFIG *get_upsampled_ref( + const AV1_COMP *cpi, const MV_REFERENCE_FRAME ref_frame) { + // Use up-sampled reference frames. + const int buf_idx = + cpi->upsampled_ref_idx[get_ref_frame_map_idx(cpi, ref_frame)]; + return &cpi->upsampled_ref_bufs[buf_idx].buf; +} + +#if CONFIG_EXT_REFS +static INLINE int enc_is_ref_frame_buf(AV1_COMP *cpi, RefCntBuffer *frame_buf) { + MV_REFERENCE_FRAME ref_frame; + AV1_COMMON *const cm = &cpi->common; + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { + const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame); + if (buf_idx == INVALID_IDX) continue; + if (frame_buf == &cm->buffer_pool->frame_bufs[buf_idx]) break; + } + return (ref_frame <= ALTREF_FRAME); +} +#endif // CONFIG_EXT_REFS + +static INLINE unsigned int get_token_alloc(int mb_rows, int mb_cols) { + // We assume 3 planes all at full resolution. We assume up to 1 token per + // pixel, and then allow a head room of 1 EOSB token per 4x4 block per plane, + // plus EOSB_TOKEN per plane. + return mb_rows * mb_cols * (16 * 16 + 17) * 3; +} + +// Get the allocated token size for a tile. It does the same calculation as in +// the frame token allocation. +static INLINE unsigned int allocated_tokens(TileInfo tile) { +#if CONFIG_CB4X4 + int tile_mb_rows = (tile.mi_row_end - tile.mi_row_start + 2) >> 2; + int tile_mb_cols = (tile.mi_col_end - tile.mi_col_start + 2) >> 2; +#else + int tile_mb_rows = (tile.mi_row_end - tile.mi_row_start + 1) >> 1; + int tile_mb_cols = (tile.mi_col_end - tile.mi_col_start + 1) >> 1; +#endif + + return get_token_alloc(tile_mb_rows, tile_mb_cols); +} + +void av1_alloc_compressor_data(AV1_COMP *cpi); + +void av1_scale_references(AV1_COMP *cpi); + +void av1_update_reference_frames(AV1_COMP *cpi); + +void av1_set_high_precision_mv(AV1_COMP *cpi, int allow_high_precision_mv); +#if CONFIG_TEMPMV_SIGNALING +void av1_set_temporal_mv_prediction(AV1_COMP *cpi, int allow_tempmv_prediction); +#endif + +YV12_BUFFER_CONFIG *av1_scale_if_required_fast(AV1_COMMON *cm, + YV12_BUFFER_CONFIG *unscaled, + YV12_BUFFER_CONFIG *scaled); + +YV12_BUFFER_CONFIG *av1_scale_if_required(AV1_COMMON *cm, + YV12_BUFFER_CONFIG *unscaled, + YV12_BUFFER_CONFIG *scaled); + +void av1_apply_encoding_flags(AV1_COMP *cpi, aom_enc_frame_flags_t flags); + +static INLINE int is_altref_enabled(const AV1_COMP *const cpi) { + return cpi->oxcf.lag_in_frames > 0 && cpi->oxcf.enable_auto_arf; +} + +// TODO(zoeliu): To set up cpi->oxcf.enable_auto_brf +#if 0 && CONFIG_EXT_REFS +static INLINE int is_bwdref_enabled(const AV1_COMP *const cpi) { + // NOTE(zoeliu): The enabling of bi-predictive frames depends on the use of + // alt_ref, and now will be off when the alt_ref interval is + // not sufficiently large. + return is_altref_enabled(cpi) && cpi->oxcf.enable_auto_brf; +} +#endif // CONFIG_EXT_REFS + +static INLINE void set_ref_ptrs(const AV1_COMMON *cm, MACROBLOCKD *xd, + MV_REFERENCE_FRAME ref0, + MV_REFERENCE_FRAME ref1) { + xd->block_refs[0] = + &cm->frame_refs[ref0 >= LAST_FRAME ? ref0 - LAST_FRAME : 0]; + xd->block_refs[1] = + &cm->frame_refs[ref1 >= LAST_FRAME ? ref1 - LAST_FRAME : 0]; +} + +static INLINE int get_chessboard_index(int frame_index) { + return frame_index & 0x1; +} + +static INLINE int *cond_cost_list(const struct AV1_COMP *cpi, int *cost_list) { + return cpi->sf.mv.subpel_search_method != SUBPEL_TREE ? cost_list : NULL; +} + +void av1_new_framerate(AV1_COMP *cpi, double framerate); + +#define LAYER_IDS_TO_IDX(sl, tl, num_tl) ((sl) * (num_tl) + (tl)) + +// Update up-sampled reference frame index. +static INLINE void uref_cnt_fb(EncRefCntBuffer *ubufs, int *uidx, + int new_uidx) { + const int ref_index = *uidx; + + if (ref_index >= 0 && ubufs[ref_index].ref_count > 0) + ubufs[ref_index].ref_count--; + + *uidx = new_uidx; + ubufs[new_uidx].ref_count++; +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_ENCODER_H_ diff --git a/third_party/aom/av1/encoder/encodetxb.c b/third_party/aom/av1/encoder/encodetxb.c new file mode 100644 index 0000000000..3f71a4472e --- /dev/null +++ b/third_party/aom/av1/encoder/encodetxb.c @@ -0,0 +1,784 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/common/scan.h" +#include "av1/common/blockd.h" +#include "av1/common/idct.h" +#include "av1/common/pred_common.h" +#include "av1/encoder/bitstream.h" +#include "av1/encoder/encodeframe.h" +#include "av1/encoder/cost.h" +#include "av1/encoder/encodetxb.h" +#include "av1/encoder/rdopt.h" +#include "av1/encoder/subexp.h" +#include "av1/encoder/tokenize.h" + +void av1_alloc_txb_buf(AV1_COMP *cpi) { +#if 0 + AV1_COMMON *cm = &cpi->common; + int mi_block_size = 1 << MI_SIZE_LOG2; + // TODO(angiebird): Make sure cm->subsampling_x/y is set correctly, and then + // use precise buffer size according to cm->subsampling_x/y + int pixel_stride = mi_block_size * cm->mi_cols; + int pixel_height = mi_block_size * cm->mi_rows; + int i; + for (i = 0; i < MAX_MB_PLANE; ++i) { + CHECK_MEM_ERROR( + cm, cpi->tcoeff_buf[i], + aom_malloc(sizeof(*cpi->tcoeff_buf[i]) * pixel_stride * pixel_height)); + } +#else + (void)cpi; +#endif +} + +void av1_free_txb_buf(AV1_COMP *cpi) { +#if 0 + int i; + for (i = 0; i < MAX_MB_PLANE; ++i) { + aom_free(cpi->tcoeff_buf[i]); + } +#else + (void)cpi; +#endif +} + +static void write_golomb(aom_writer *w, int level) { + int x = level + 1; + int i = x; + int length = 0; + + while (i) { + i >>= 1; + ++length; + } + assert(length > 0); + + for (i = 0; i < length - 1; ++i) aom_write_bit(w, 0); + + for (i = length - 1; i >= 0; --i) aom_write_bit(w, (x >> i) & 0x01); +} + +void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, + aom_writer *w, int block, int plane, + const tran_low_t *tcoeff, uint16_t eob, + TXB_CTX *txb_ctx) { + aom_prob *nz_map; + aom_prob *eob_flag; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const PLANE_TYPE plane_type = get_plane_type(plane); + const TX_SIZE tx_size = get_tx_size(plane, xd); + const TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + const SCAN_ORDER *const scan_order = + get_scan(cm, tx_size, tx_type, is_inter_block(mbmi)); + const int16_t *scan = scan_order->scan; + int c; + int is_nz; + const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; + const int seg_eob = tx_size_2d[tx_size]; + uint8_t txb_mask[32 * 32] = { 0 }; + uint16_t update_eob = 0; + + aom_write(w, eob == 0, cm->fc->txb_skip[tx_size][txb_ctx->txb_skip_ctx]); + + if (eob == 0) return; +#if CONFIG_TXK_SEL + av1_write_tx_type(cm, xd, block, plane, w); +#endif + + nz_map = cm->fc->nz_map[tx_size][plane_type]; + eob_flag = cm->fc->eob_flag[tx_size][plane_type]; + + for (c = 0; c < eob; ++c) { + int coeff_ctx = get_nz_map_ctx(tcoeff, txb_mask, scan[c], bwl); + int eob_ctx = get_eob_ctx(tcoeff, scan[c], bwl); + + tran_low_t v = tcoeff[scan[c]]; + is_nz = (v != 0); + + if (c == seg_eob - 1) break; + + aom_write(w, is_nz, nz_map[coeff_ctx]); + + if (is_nz) { + aom_write(w, c == (eob - 1), eob_flag[eob_ctx]); + } + txb_mask[scan[c]] = 1; + } + + int i; + for (i = 0; i < NUM_BASE_LEVELS; ++i) { + aom_prob *coeff_base = cm->fc->coeff_base[tx_size][plane_type][i]; + + update_eob = 0; + for (c = eob - 1; c >= 0; --c) { + tran_low_t v = tcoeff[scan[c]]; + tran_low_t level = abs(v); + int sign = (v < 0) ? 1 : 0; + int ctx; + + if (level <= i) continue; + + ctx = get_base_ctx(tcoeff, scan[c], bwl, i + 1); + + if (level == i + 1) { + aom_write(w, 1, coeff_base[ctx]); + if (c == 0) { + aom_write(w, sign, cm->fc->dc_sign[plane_type][txb_ctx->dc_sign_ctx]); + } else { + aom_write_bit(w, sign); + } + continue; + } + aom_write(w, 0, coeff_base[ctx]); + update_eob = AOMMAX(update_eob, c); + } + } + + for (c = update_eob; c >= 0; --c) { + tran_low_t v = tcoeff[scan[c]]; + tran_low_t level = abs(v); + int sign = (v < 0) ? 1 : 0; + int idx; + int ctx; + + if (level <= NUM_BASE_LEVELS) continue; + + if (c == 0) { + aom_write(w, sign, cm->fc->dc_sign[plane_type][txb_ctx->dc_sign_ctx]); + } else { + aom_write_bit(w, sign); + } + + // level is above 1. + ctx = get_level_ctx(tcoeff, scan[c], bwl); + for (idx = 0; idx < COEFF_BASE_RANGE; ++idx) { + if (level == (idx + 1 + NUM_BASE_LEVELS)) { + aom_write(w, 1, cm->fc->coeff_lps[tx_size][plane_type][ctx]); + break; + } + aom_write(w, 0, cm->fc->coeff_lps[tx_size][plane_type][ctx]); + } + if (idx < COEFF_BASE_RANGE) continue; + + // use 0-th order Golomb code to handle the residual level. + write_golomb(w, level - COEFF_BASE_RANGE - 1 - NUM_BASE_LEVELS); + } +} + +void av1_write_coeffs_mb(const AV1_COMMON *const cm, MACROBLOCK *x, + aom_writer *w, int plane) { + MACROBLOCKD *xd = &x->e_mbd; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + BLOCK_SIZE bsize = mbmi->sb_type; + struct macroblockd_plane *pd = &xd->plane[plane]; + +#if CONFIG_CB4X4 + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); +#else + const BLOCK_SIZE plane_bsize = + get_plane_block_size(AOMMAX(bsize, BLOCK_8X8), pd); +#endif + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + const int max_blocks_high = max_block_high(xd, plane_bsize, plane); + TX_SIZE tx_size = get_tx_size(plane, xd); + const int bkw = tx_size_wide_unit[tx_size]; + const int bkh = tx_size_high_unit[tx_size]; + const int step = tx_size_wide_unit[tx_size] * tx_size_high_unit[tx_size]; + int row, col; + int block = 0; + for (row = 0; row < max_blocks_high; row += bkh) { + for (col = 0; col < max_blocks_wide; col += bkw) { + tran_low_t *tcoeff = BLOCK_OFFSET(x->mbmi_ext->tcoeff[plane], block); + uint16_t eob = x->mbmi_ext->eobs[plane][block]; + TXB_CTX txb_ctx = { x->mbmi_ext->txb_skip_ctx[plane][block], + x->mbmi_ext->dc_sign_ctx[plane][block] }; + av1_write_coeffs_txb(cm, xd, w, block, plane, tcoeff, eob, &txb_ctx); + block += step; + } + } +} + +static INLINE void get_base_ctx_set(const tran_low_t *tcoeffs, + int c, // raster order + const int bwl, + int ctx_set[NUM_BASE_LEVELS]) { + const int row = c >> bwl; + const int col = c - (row << bwl); + const int stride = 1 << bwl; + int mag[NUM_BASE_LEVELS] = { 0 }; + int idx; + tran_low_t abs_coeff; + int i; + + for (idx = 0; idx < BASE_CONTEXT_POSITION_NUM; ++idx) { + int ref_row = row + base_ref_offset[idx][0]; + int ref_col = col + base_ref_offset[idx][1]; + int pos = (ref_row << bwl) + ref_col; + + if (ref_row < 0 || ref_col < 0 || ref_row >= stride || ref_col >= stride) + continue; + + abs_coeff = abs(tcoeffs[pos]); + + for (i = 0; i < NUM_BASE_LEVELS; ++i) { + ctx_set[i] += abs_coeff > i; + if (base_ref_offset[idx][0] >= 0 && base_ref_offset[idx][1] >= 0) + mag[i] |= abs_coeff > (i + 1); + } + } + + for (i = 0; i < NUM_BASE_LEVELS; ++i) { + ctx_set[i] = (ctx_set[i] + 1) >> 1; + + if (row == 0 && col == 0) + ctx_set[i] = (ctx_set[i] << 1) + mag[i]; + else if (row == 0) + ctx_set[i] = 8 + (ctx_set[i] << 1) + mag[i]; + else if (col == 0) + ctx_set[i] = 18 + (ctx_set[i] << 1) + mag[i]; + else + ctx_set[i] = 28 + (ctx_set[i] << 1) + mag[i]; + } + return; +} + +int av1_cost_coeffs_txb(const AV1_COMP *const cpi, MACROBLOCK *x, int plane, + int block, TXB_CTX *txb_ctx) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + const TX_SIZE tx_size = get_tx_size(plane, xd); + const PLANE_TYPE plane_type = get_plane_type(plane); + const TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const struct macroblock_plane *p = &x->plane[plane]; + const int eob = p->eobs[block]; + const tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block); + int c, cost; + const int seg_eob = AOMMIN(eob, tx_size_2d[tx_size] - 1); + int txb_skip_ctx = txb_ctx->txb_skip_ctx; + aom_prob *nz_map = xd->fc->nz_map[tx_size][plane_type]; + + const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; + // txb_mask is only initialized for once here. After that, it will be set when + // coding zero map and then reset when coding level 1 info. + uint8_t txb_mask[32 * 32] = { 0 }; + aom_prob(*coeff_base)[COEFF_BASE_CONTEXTS] = + xd->fc->coeff_base[tx_size][plane_type]; + + const SCAN_ORDER *const scan_order = + get_scan(cm, tx_size, tx_type, is_inter_block(mbmi)); + const int16_t *scan = scan_order->scan; + + cost = 0; + + if (eob == 0) { + cost = av1_cost_bit(xd->fc->txb_skip[tx_size][txb_skip_ctx], 1); + return cost; + } + + cost = av1_cost_bit(xd->fc->txb_skip[tx_size][txb_skip_ctx], 0); + +#if CONFIG_TXK_SEL + cost += av1_tx_type_cost(cpi, xd, mbmi->sb_type, plane, tx_size, tx_type); +#endif + + for (c = 0; c < eob; ++c) { + tran_low_t v = qcoeff[scan[c]]; + int is_nz = (v != 0); + int level = abs(v); + + if (c < seg_eob) { + int coeff_ctx = get_nz_map_ctx(qcoeff, txb_mask, scan[c], bwl); + cost += av1_cost_bit(nz_map[coeff_ctx], is_nz); + } + + if (is_nz) { + int ctx_ls[NUM_BASE_LEVELS] = { 0 }; + int sign = (v < 0) ? 1 : 0; + + // sign bit cost + if (c == 0) { + int dc_sign_ctx = txb_ctx->dc_sign_ctx; + + cost += av1_cost_bit(xd->fc->dc_sign[plane_type][dc_sign_ctx], sign); + } else { + cost += av1_cost_bit(128, sign); + } + + get_base_ctx_set(qcoeff, scan[c], bwl, ctx_ls); + + int i; + for (i = 0; i < NUM_BASE_LEVELS; ++i) { + if (level <= i) continue; + + if (level == i + 1) { + cost += av1_cost_bit(coeff_base[i][ctx_ls[i]], 1); + continue; + } + cost += av1_cost_bit(coeff_base[i][ctx_ls[i]], 0); + } + + if (level > NUM_BASE_LEVELS) { + int idx; + int ctx; + + ctx = get_level_ctx(qcoeff, scan[c], bwl); + + for (idx = 0; idx < COEFF_BASE_RANGE; ++idx) { + if (level == (idx + 1 + NUM_BASE_LEVELS)) { + cost += + av1_cost_bit(xd->fc->coeff_lps[tx_size][plane_type][ctx], 1); + break; + } + cost += av1_cost_bit(xd->fc->coeff_lps[tx_size][plane_type][ctx], 0); + } + + if (idx >= COEFF_BASE_RANGE) { + // residual cost + int r = level - COEFF_BASE_RANGE - NUM_BASE_LEVELS; + int ri = r; + int length = 0; + + while (ri) { + ri >>= 1; + ++length; + } + + for (ri = 0; ri < length - 1; ++ri) cost += av1_cost_bit(128, 0); + + for (ri = length - 1; ri >= 0; --ri) + cost += av1_cost_bit(128, (r >> ri) & 0x01); + } + } + + if (c < seg_eob) { + int eob_ctx = get_eob_ctx(qcoeff, scan[c], bwl); + cost += av1_cost_bit(xd->fc->eob_flag[tx_size][plane_type][eob_ctx], + c == (eob - 1)); + } + } + + txb_mask[scan[c]] = 1; + } + + return cost; +} + +typedef struct TxbParams { + const AV1_COMP *cpi; + ThreadData *td; + int rate; +} TxbParams; + +int av1_get_txb_entropy_context(const tran_low_t *qcoeff, + const SCAN_ORDER *scan_order, int eob) { + const int16_t *scan = scan_order->scan; + int cul_level = 0; + int c; + for (c = 0; c < eob; ++c) { + cul_level += abs(qcoeff[scan[c]]); + } + + cul_level = AOMMIN(COEFF_CONTEXT_MASK, cul_level); + set_dc_sign(&cul_level, qcoeff[0]); + + return cul_level; +} + +static void update_txb_context(int plane, int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, + void *arg) { + TxbParams *const args = arg; + const AV1_COMP *cpi = args->cpi; + const AV1_COMMON *cm = &cpi->common; + ThreadData *const td = args->td; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + struct macroblock_plane *p = &x->plane[plane]; + struct macroblockd_plane *pd = &xd->plane[plane]; + const uint16_t eob = p->eobs[block]; + const tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block); + const PLANE_TYPE plane_type = pd->plane_type; + const TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + const SCAN_ORDER *const scan_order = + get_scan(cm, tx_size, tx_type, is_inter_block(mbmi)); + (void)plane_bsize; + + int cul_level = av1_get_txb_entropy_context(qcoeff, scan_order, eob); + av1_set_contexts(xd, pd, plane, tx_size, cul_level, blk_col, blk_row); +} + +static void update_and_record_txb_context(int plane, int block, int blk_row, + int blk_col, BLOCK_SIZE plane_bsize, + TX_SIZE tx_size, void *arg) { + TxbParams *const args = arg; + const AV1_COMP *cpi = args->cpi; + const AV1_COMMON *cm = &cpi->common; + ThreadData *const td = args->td; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + struct macroblock_plane *p = &x->plane[plane]; + struct macroblockd_plane *pd = &xd->plane[plane]; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + int eob = p->eobs[block], update_eob = 0; + const PLANE_TYPE plane_type = pd->plane_type; + const tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block); + tran_low_t *tcoeff = BLOCK_OFFSET(x->mbmi_ext->tcoeff[plane], block); + const int segment_id = mbmi->segment_id; + const TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + const SCAN_ORDER *const scan_order = + get_scan(cm, tx_size, tx_type, is_inter_block(mbmi)); + const int16_t *scan = scan_order->scan; + const int seg_eob = get_tx_eob(&cpi->common.seg, segment_id, tx_size); + int c, i; + TXB_CTX txb_ctx; + get_txb_ctx(plane_bsize, tx_size, plane, pd->above_context + blk_col, + pd->left_context + blk_row, &txb_ctx); + const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; + int cul_level = 0; + unsigned int(*nz_map_count)[SIG_COEF_CONTEXTS][2]; + uint8_t txb_mask[32 * 32] = { 0 }; + + nz_map_count = &td->counts->nz_map[tx_size][plane_type]; + + memcpy(tcoeff, qcoeff, sizeof(*tcoeff) * seg_eob); + + ++td->counts->txb_skip[tx_size][txb_ctx.txb_skip_ctx][eob == 0]; + x->mbmi_ext->txb_skip_ctx[plane][block] = txb_ctx.txb_skip_ctx; + + x->mbmi_ext->eobs[plane][block] = eob; + + if (eob == 0) { + av1_set_contexts(xd, pd, plane, tx_size, 0, blk_col, blk_row); + return; + } + +#if CONFIG_TXK_SEL + av1_update_tx_type_count(cm, xd, block, plane, mbmi->sb_type, tx_size, + td->counts); +#endif + + for (c = 0; c < eob; ++c) { + tran_low_t v = qcoeff[scan[c]]; + int is_nz = (v != 0); + int coeff_ctx = get_nz_map_ctx(tcoeff, txb_mask, scan[c], bwl); + int eob_ctx = get_eob_ctx(tcoeff, scan[c], bwl); + + if (c == seg_eob - 1) break; + + ++(*nz_map_count)[coeff_ctx][is_nz]; + + if (is_nz) { + ++td->counts->eob_flag[tx_size][plane_type][eob_ctx][c == (eob - 1)]; + } + txb_mask[scan[c]] = 1; + } + + // Reverse process order to handle coefficient level and sign. + for (i = 0; i < NUM_BASE_LEVELS; ++i) { + update_eob = 0; + for (c = eob - 1; c >= 0; --c) { + tran_low_t v = qcoeff[scan[c]]; + tran_low_t level = abs(v); + int ctx; + + if (level <= i) continue; + + ctx = get_base_ctx(tcoeff, scan[c], bwl, i + 1); + + if (level == i + 1) { + ++td->counts->coeff_base[tx_size][plane_type][i][ctx][1]; + if (c == 0) { + int dc_sign_ctx = txb_ctx.dc_sign_ctx; + + ++td->counts->dc_sign[plane_type][dc_sign_ctx][v < 0]; + x->mbmi_ext->dc_sign_ctx[plane][block] = dc_sign_ctx; + } + cul_level += level; + continue; + } + ++td->counts->coeff_base[tx_size][plane_type][i][ctx][0]; + update_eob = AOMMAX(update_eob, c); + } + } + + for (c = update_eob; c >= 0; --c) { + tran_low_t v = qcoeff[scan[c]]; + tran_low_t level = abs(v); + int idx; + int ctx; + + if (level <= NUM_BASE_LEVELS) continue; + + cul_level += level; + if (c == 0) { + int dc_sign_ctx = txb_ctx.dc_sign_ctx; + + ++td->counts->dc_sign[plane_type][dc_sign_ctx][v < 0]; + x->mbmi_ext->dc_sign_ctx[plane][block] = dc_sign_ctx; + } + + // level is above 1. + ctx = get_level_ctx(tcoeff, scan[c], bwl); + for (idx = 0; idx < COEFF_BASE_RANGE; ++idx) { + if (level == (idx + 1 + NUM_BASE_LEVELS)) { + ++td->counts->coeff_lps[tx_size][plane_type][ctx][1]; + break; + } + ++td->counts->coeff_lps[tx_size][plane_type][ctx][0]; + } + if (idx < COEFF_BASE_RANGE) continue; + + // use 0-th order Golomb code to handle the residual level. + } + + cul_level = AOMMIN(COEFF_CONTEXT_MASK, cul_level); + + // DC value + set_dc_sign(&cul_level, tcoeff[0]); + av1_set_contexts(xd, pd, plane, tx_size, cul_level, blk_col, blk_row); + +#if CONFIG_ADAPT_SCAN + // Since dqcoeff is not available here, we pass qcoeff into + // av1_update_scan_count_facade(). The update behavior should be the same + // because av1_update_scan_count_facade() only cares if coefficients are zero + // or not. + av1_update_scan_count_facade((AV1_COMMON *)cm, td->counts, tx_size, tx_type, + qcoeff, eob); +#endif +} + +void av1_update_txb_context(const AV1_COMP *cpi, ThreadData *td, + RUN_TYPE dry_run, BLOCK_SIZE bsize, int *rate, + int mi_row, int mi_col) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const int ctx = av1_get_skip_context(xd); + const int skip_inc = + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP); + struct TxbParams arg = { cpi, td, 0 }; + (void)rate; + (void)mi_row; + (void)mi_col; + if (mbmi->skip) { + if (!dry_run) td->counts->skip[ctx][1] += skip_inc; + reset_skip_context(xd, bsize); + return; + } + + if (!dry_run) { + td->counts->skip[ctx][0] += skip_inc; + av1_foreach_transformed_block(xd, bsize, mi_row, mi_col, + update_and_record_txb_context, &arg); + } else if (dry_run == DRY_RUN_NORMAL) { + av1_foreach_transformed_block(xd, bsize, mi_row, mi_col, update_txb_context, + &arg); + } else { + printf("DRY_RUN_COSTCOEFFS is not supported yet\n"); + assert(0); + } +} + +static void find_new_prob(unsigned int *branch_cnt, aom_prob *oldp, + int *savings, int *update, aom_writer *const bc) { + const aom_prob upd = DIFF_UPDATE_PROB; + int u = 0; + aom_prob newp = get_binary_prob(branch_cnt[0], branch_cnt[1]); + int s = av1_prob_diff_update_savings_search(branch_cnt, *oldp, &newp, upd, 1); + + if (s > 0 && newp != *oldp) u = 1; + + if (u) + *savings += s - (int)(av1_cost_zero(upd)); // TODO(jingning): 1? + else + *savings -= (int)(av1_cost_zero(upd)); + + if (update) { + ++update[u]; + return; + } + + aom_write(bc, u, upd); + if (u) { + /* send/use new probability */ + av1_write_prob_diff_update(bc, newp, *oldp); + *oldp = newp; + } +} + +static void write_txb_probs(aom_writer *const bc, AV1_COMP *cpi, + TX_SIZE tx_size) { + FRAME_CONTEXT *fc = cpi->common.fc; + FRAME_COUNTS *counts = cpi->td.counts; + int savings = 0; + int update[2] = { 0, 0 }; + int plane, ctx, level; + + for (ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx) { + find_new_prob(counts->txb_skip[tx_size][ctx], &fc->txb_skip[tx_size][ctx], + &savings, update, bc); + } + + for (plane = 0; plane < PLANE_TYPES; ++plane) { + for (ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx) { + find_new_prob(counts->nz_map[tx_size][plane][ctx], + &fc->nz_map[tx_size][plane][ctx], &savings, update, bc); + } + } + + for (plane = 0; plane < PLANE_TYPES; ++plane) { + for (ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx) { + find_new_prob(counts->eob_flag[tx_size][plane][ctx], + &fc->eob_flag[tx_size][plane][ctx], &savings, update, bc); + } + } + + for (level = 0; level < NUM_BASE_LEVELS; ++level) { + for (plane = 0; plane < PLANE_TYPES; ++plane) { + for (ctx = 0; ctx < COEFF_BASE_CONTEXTS; ++ctx) { + find_new_prob(counts->coeff_base[tx_size][plane][level][ctx], + &fc->coeff_base[tx_size][plane][level][ctx], &savings, + update, bc); + } + } + } + + for (plane = 0; plane < PLANE_TYPES; ++plane) { + for (ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx) { + find_new_prob(counts->coeff_lps[tx_size][plane][ctx], + &fc->coeff_lps[tx_size][plane][ctx], &savings, update, bc); + } + } + + // Decide if to update the model for this tx_size + if (update[1] == 0 || savings < 0) { + aom_write_bit(bc, 0); + return; + } + aom_write_bit(bc, 1); + + for (ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx) { + find_new_prob(counts->txb_skip[tx_size][ctx], &fc->txb_skip[tx_size][ctx], + &savings, NULL, bc); + } + + for (plane = 0; plane < PLANE_TYPES; ++plane) { + for (ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx) { + find_new_prob(counts->nz_map[tx_size][plane][ctx], + &fc->nz_map[tx_size][plane][ctx], &savings, NULL, bc); + } + } + + for (plane = 0; plane < PLANE_TYPES; ++plane) { + for (ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx) { + find_new_prob(counts->eob_flag[tx_size][plane][ctx], + &fc->eob_flag[tx_size][plane][ctx], &savings, NULL, bc); + } + } + + for (level = 0; level < NUM_BASE_LEVELS; ++level) { + for (plane = 0; plane < PLANE_TYPES; ++plane) { + for (ctx = 0; ctx < COEFF_BASE_CONTEXTS; ++ctx) { + find_new_prob(counts->coeff_base[tx_size][plane][level][ctx], + &fc->coeff_base[tx_size][plane][level][ctx], &savings, + NULL, bc); + } + } + } + + for (plane = 0; plane < PLANE_TYPES; ++plane) { + for (ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx) { + find_new_prob(counts->coeff_lps[tx_size][plane][ctx], + &fc->coeff_lps[tx_size][plane][ctx], &savings, NULL, bc); + } + } +} + +void av1_write_txb_probs(AV1_COMP *cpi, aom_writer *w) { + const TX_MODE tx_mode = cpi->common.tx_mode; + const TX_SIZE max_tx_size = tx_mode_to_biggest_tx_size[tx_mode]; + TX_SIZE tx_size; + int ctx, plane; + + for (plane = 0; plane < PLANE_TYPES; ++plane) + for (ctx = 0; ctx < DC_SIGN_CONTEXTS; ++ctx) + av1_cond_prob_diff_update(w, &cpi->common.fc->dc_sign[plane][ctx], + cpi->td.counts->dc_sign[plane][ctx], 1); + + for (tx_size = TX_4X4; tx_size <= max_tx_size; ++tx_size) + write_txb_probs(w, cpi, tx_size); +} + +#if CONFIG_TXK_SEL +int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, + int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, + const ENTROPY_CONTEXT *a, const ENTROPY_CONTEXT *l, + int use_fast_coef_costing, RD_STATS *rd_stats) { + const AV1_COMMON *cm = &cpi->common; + MACROBLOCKD *xd = &x->e_mbd; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + TX_TYPE txk_start = DCT_DCT; + TX_TYPE txk_end = TX_TYPES - 1; + TX_TYPE best_tx_type = txk_start; + int64_t best_rd = INT64_MAX; + const int coeff_ctx = combine_entropy_contexts(*a, *l); + TX_TYPE tx_type; + for (tx_type = txk_start; tx_type <= txk_end; ++tx_type) { + if (plane == 0) mbmi->txk_type[block] = tx_type; + TX_TYPE ref_tx_type = + get_tx_type(get_plane_type(plane), xd, block, tx_size); + if (tx_type != ref_tx_type) { + // use get_tx_type() to check if the tx_type is valid for the current mode + // if it's not, we skip it here. + continue; + } + RD_STATS this_rd_stats; + av1_invalid_rd_stats(&this_rd_stats); + av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, + coeff_ctx, AV1_XFORM_QUANT_FP); + if (x->plane[plane].eobs[block] && !xd->lossless[mbmi->segment_id]) + av1_optimize_b(cm, x, plane, block, tx_size, coeff_ctx); + av1_dist_block(cpi, x, plane, plane_bsize, block, blk_row, blk_col, tx_size, + &this_rd_stats.dist, &this_rd_stats.sse, + OUTPUT_HAS_PREDICTED_PIXELS); + const SCAN_ORDER *scan_order = + get_scan(cm, tx_size, tx_type, is_inter_block(mbmi)); + this_rd_stats.rate = av1_cost_coeffs( + cpi, x, plane, block, tx_size, scan_order, a, l, use_fast_coef_costing); + int rd = + RDCOST(x->rdmult, x->rddiv, this_rd_stats.rate, this_rd_stats.dist); + if (rd < best_rd) { + best_rd = rd; + *rd_stats = this_rd_stats; + best_tx_type = tx_type; + } + } + if (plane == 0) mbmi->txk_type[block] = best_tx_type; + // TODO(angiebird): Instead of re-call av1_xform_quant and av1_optimize_b, + // copy the best result in the above tx_type search for loop + av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, + coeff_ctx, AV1_XFORM_QUANT_FP); + if (x->plane[plane].eobs[block] && !xd->lossless[mbmi->segment_id]) + av1_optimize_b(cm, x, plane, block, tx_size, coeff_ctx); + if (!is_inter_block(mbmi)) { + // intra mode needs decoded result such that the next transform block + // can use it for prediction. + av1_inverse_transform_block_facade(xd, plane, block, blk_row, blk_col, + x->plane[plane].eobs[block]); + } + return best_rd; +} +#endif // CONFIG_TXK_SEL diff --git a/third_party/aom/av1/encoder/encodetxb.h b/third_party/aom/av1/encoder/encodetxb.h new file mode 100644 index 0000000000..552d47b542 --- /dev/null +++ b/third_party/aom/av1/encoder/encodetxb.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef ENCODETXB_H_ +#define ENCODETXB_H_ + +#include "./aom_config.h" +#include "av1/common/blockd.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/txb_common.h" +#include "av1/encoder/block.h" +#include "av1/encoder/encoder.h" +#include "aom_dsp/bitwriter.h" +#ifdef __cplusplus +extern "C" { +#endif +void av1_alloc_txb_buf(AV1_COMP *cpi); +void av1_free_txb_buf(AV1_COMP *cpi); +int av1_cost_coeffs_txb(const AV1_COMP *const cpi, MACROBLOCK *x, int plane, + int block, TXB_CTX *txb_ctx); +void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, + aom_writer *w, int block, int plane, + const tran_low_t *tcoeff, uint16_t eob, + TXB_CTX *txb_ctx); +void av1_write_coeffs_mb(const AV1_COMMON *const cm, MACROBLOCK *x, + aom_writer *w, int plane); +int av1_get_txb_entropy_context(const tran_low_t *qcoeff, + const SCAN_ORDER *scan_order, int eob); +void av1_update_txb_context(const AV1_COMP *cpi, ThreadData *td, + RUN_TYPE dry_run, BLOCK_SIZE bsize, int *rate, + const int mi_row, const int mi_col); +void av1_write_txb_probs(AV1_COMP *cpi, aom_writer *w); + +#if CONFIG_TXK_SEL +int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, + int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, + const ENTROPY_CONTEXT *a, const ENTROPY_CONTEXT *l, + int use_fast_coef_costing, RD_STATS *rd_stats); +#endif +#ifdef __cplusplus +} +#endif + +#endif // COEFFS_CODING_H_ diff --git a/third_party/aom/av1/encoder/ethread.c b/third_party/aom/av1/encoder/ethread.c new file mode 100644 index 0000000000..34f0b95665 --- /dev/null +++ b/third_party/aom/av1/encoder/ethread.c @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/encoder/encodeframe.h" +#include "av1/encoder/encoder.h" +#include "av1/encoder/ethread.h" +#include "aom_dsp/aom_dsp_common.h" + +static void accumulate_rd_opt(ThreadData *td, ThreadData *td_t) { + int i, j, k, l, m, n; + + for (i = 0; i < REFERENCE_MODES; i++) + td->rd_counts.comp_pred_diff[i] += td_t->rd_counts.comp_pred_diff[i]; + +#if CONFIG_GLOBAL_MOTION + for (i = 0; i < TOTAL_REFS_PER_FRAME; i++) + td->rd_counts.global_motion_used[i] += + td_t->rd_counts.global_motion_used[i]; +#endif // CONFIG_GLOBAL_MOTION + + for (i = 0; i < TX_SIZES; i++) + for (j = 0; j < PLANE_TYPES; j++) + for (k = 0; k < REF_TYPES; k++) + for (l = 0; l < COEF_BANDS; l++) + for (m = 0; m < COEFF_CONTEXTS; m++) + for (n = 0; n < ENTROPY_TOKENS; n++) + td->rd_counts.coef_counts[i][j][k][l][m][n] += + td_t->rd_counts.coef_counts[i][j][k][l][m][n]; +} + +static int enc_worker_hook(EncWorkerData *const thread_data, void *unused) { + AV1_COMP *const cpi = thread_data->cpi; + const AV1_COMMON *const cm = &cpi->common; + const int tile_cols = cm->tile_cols; + const int tile_rows = cm->tile_rows; + int t; + + (void)unused; + + for (t = thread_data->start; t < tile_rows * tile_cols; + t += cpi->num_workers) { + int tile_row = t / tile_cols; + int tile_col = t % tile_cols; + + av1_encode_tile(cpi, thread_data->td, tile_row, tile_col); + } + + return 0; +} + +void av1_encode_tiles_mt(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + const int tile_cols = cm->tile_cols; + const AVxWorkerInterface *const winterface = aom_get_worker_interface(); + const int num_workers = AOMMIN(cpi->oxcf.max_threads, tile_cols); + int i; + + av1_init_tile_data(cpi); + + // Only run once to create threads and allocate thread data. + if (cpi->num_workers == 0) { + CHECK_MEM_ERROR(cm, cpi->workers, + aom_malloc(num_workers * sizeof(*cpi->workers))); + + CHECK_MEM_ERROR(cm, cpi->tile_thr_data, + aom_calloc(num_workers, sizeof(*cpi->tile_thr_data))); + + for (i = 0; i < num_workers; i++) { + AVxWorker *const worker = &cpi->workers[i]; + EncWorkerData *const thread_data = &cpi->tile_thr_data[i]; + + ++cpi->num_workers; + winterface->init(worker); + + thread_data->cpi = cpi; + + if (i < num_workers - 1) { + // Allocate thread data. + CHECK_MEM_ERROR(cm, thread_data->td, + aom_memalign(32, sizeof(*thread_data->td))); + av1_zero(*thread_data->td); + + // Set up pc_tree. + thread_data->td->leaf_tree = NULL; + thread_data->td->pc_tree = NULL; + av1_setup_pc_tree(cm, thread_data->td); + + // Set up variance tree if needed. + if (cpi->sf.partition_search_type == VAR_BASED_PARTITION) + av1_setup_var_tree(cm, thread_data->td); + + // Allocate frame counters in thread data. + CHECK_MEM_ERROR(cm, thread_data->td->counts, + aom_calloc(1, sizeof(*thread_data->td->counts))); + + // Create threads + if (!winterface->reset(worker)) + aom_internal_error(&cm->error, AOM_CODEC_ERROR, + "Tile encoder thread creation failed"); + } else { + // Main thread acts as a worker and uses the thread data in cpi. + thread_data->td = &cpi->td; + } + + winterface->sync(worker); + } + } + + for (i = 0; i < num_workers; i++) { + AVxWorker *const worker = &cpi->workers[i]; + EncWorkerData *thread_data; + + worker->hook = (AVxWorkerHook)enc_worker_hook; + worker->data1 = &cpi->tile_thr_data[i]; + worker->data2 = NULL; + thread_data = (EncWorkerData *)worker->data1; + + // Before encoding a frame, copy the thread data from cpi. + if (thread_data->td != &cpi->td) { + thread_data->td->mb = cpi->td.mb; + thread_data->td->rd_counts = cpi->td.rd_counts; + } + if (thread_data->td->counts != &cpi->common.counts) { + memcpy(thread_data->td->counts, &cpi->common.counts, + sizeof(cpi->common.counts)); + } + +#if CONFIG_PALETTE + // Allocate buffers used by palette coding mode. + if (cpi->common.allow_screen_content_tools && i < num_workers - 1) { + MACROBLOCK *x = &thread_data->td->mb; + CHECK_MEM_ERROR(cm, x->palette_buffer, + aom_memalign(16, sizeof(*x->palette_buffer))); + } +#endif // CONFIG_PALETTE + } + + // Encode a frame + for (i = 0; i < num_workers; i++) { + AVxWorker *const worker = &cpi->workers[i]; + EncWorkerData *const thread_data = (EncWorkerData *)worker->data1; + + // Set the starting tile for each thread. + thread_data->start = i; + + if (i == cpi->num_workers - 1) + winterface->execute(worker); + else + winterface->launch(worker); + } + + // Encoding ends. + for (i = 0; i < num_workers; i++) { + AVxWorker *const worker = &cpi->workers[i]; + winterface->sync(worker); + } + + for (i = 0; i < num_workers; i++) { + AVxWorker *const worker = &cpi->workers[i]; + EncWorkerData *const thread_data = (EncWorkerData *)worker->data1; + + // Accumulate counters. + if (i < cpi->num_workers - 1) { + av1_accumulate_frame_counts(&cm->counts, thread_data->td->counts); + accumulate_rd_opt(&cpi->td, thread_data->td); + } + } +} diff --git a/third_party/aom/av1/encoder/ethread.h b/third_party/aom/av1/encoder/ethread.h new file mode 100644 index 0000000000..6c30a3e5cf --- /dev/null +++ b/third_party/aom/av1/encoder/ethread.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_ETHREAD_H_ +#define AV1_ENCODER_ETHREAD_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +struct AV1_COMP; +struct ThreadData; + +typedef struct EncWorkerData { + struct AV1_COMP *cpi; + struct ThreadData *td; + int start; +} EncWorkerData; + +void av1_encode_tiles_mt(struct AV1_COMP *cpi); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_ETHREAD_H_ diff --git a/third_party/aom/av1/encoder/extend.c b/third_party/aom/av1/encoder/extend.c new file mode 100644 index 0000000000..007694a38c --- /dev/null +++ b/third_party/aom/av1/encoder/extend.c @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +#include "av1/common/common.h" +#include "av1/encoder/extend.h" + +static void copy_and_extend_plane(const uint8_t *src, int src_pitch, + uint8_t *dst, int dst_pitch, int w, int h, + int extend_top, int extend_left, + int extend_bottom, int extend_right) { + int i, linesize; + + // copy the left and right most columns out + const uint8_t *src_ptr1 = src; + const uint8_t *src_ptr2 = src + w - 1; + uint8_t *dst_ptr1 = dst - extend_left; + uint8_t *dst_ptr2 = dst + w; + + for (i = 0; i < h; i++) { + memset(dst_ptr1, src_ptr1[0], extend_left); + memcpy(dst_ptr1 + extend_left, src_ptr1, w); + memset(dst_ptr2, src_ptr2[0], extend_right); + src_ptr1 += src_pitch; + src_ptr2 += src_pitch; + dst_ptr1 += dst_pitch; + dst_ptr2 += dst_pitch; + } + + // Now copy the top and bottom lines into each line of the respective + // borders + src_ptr1 = dst - extend_left; + src_ptr2 = dst + dst_pitch * (h - 1) - extend_left; + dst_ptr1 = dst + dst_pitch * (-extend_top) - extend_left; + dst_ptr2 = dst + dst_pitch * (h)-extend_left; + linesize = extend_left + extend_right + w; + + for (i = 0; i < extend_top; i++) { + memcpy(dst_ptr1, src_ptr1, linesize); + dst_ptr1 += dst_pitch; + } + + for (i = 0; i < extend_bottom; i++) { + memcpy(dst_ptr2, src_ptr2, linesize); + dst_ptr2 += dst_pitch; + } +} + +#if CONFIG_HIGHBITDEPTH +static void highbd_copy_and_extend_plane(const uint8_t *src8, int src_pitch, + uint8_t *dst8, int dst_pitch, int w, + int h, int extend_top, int extend_left, + int extend_bottom, int extend_right) { + int i, linesize; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + + // copy the left and right most columns out + const uint16_t *src_ptr1 = src; + const uint16_t *src_ptr2 = src + w - 1; + uint16_t *dst_ptr1 = dst - extend_left; + uint16_t *dst_ptr2 = dst + w; + + for (i = 0; i < h; i++) { + aom_memset16(dst_ptr1, src_ptr1[0], extend_left); + memcpy(dst_ptr1 + extend_left, src_ptr1, w * sizeof(src_ptr1[0])); + aom_memset16(dst_ptr2, src_ptr2[0], extend_right); + src_ptr1 += src_pitch; + src_ptr2 += src_pitch; + dst_ptr1 += dst_pitch; + dst_ptr2 += dst_pitch; + } + + // Now copy the top and bottom lines into each line of the respective + // borders + src_ptr1 = dst - extend_left; + src_ptr2 = dst + dst_pitch * (h - 1) - extend_left; + dst_ptr1 = dst + dst_pitch * (-extend_top) - extend_left; + dst_ptr2 = dst + dst_pitch * (h)-extend_left; + linesize = extend_left + extend_right + w; + + for (i = 0; i < extend_top; i++) { + memcpy(dst_ptr1, src_ptr1, linesize * sizeof(src_ptr1[0])); + dst_ptr1 += dst_pitch; + } + + for (i = 0; i < extend_bottom; i++) { + memcpy(dst_ptr2, src_ptr2, linesize * sizeof(src_ptr2[0])); + dst_ptr2 += dst_pitch; + } +} +#endif // CONFIG_HIGHBITDEPTH + +void av1_copy_and_extend_frame(const YV12_BUFFER_CONFIG *src, + YV12_BUFFER_CONFIG *dst) { + // Extend src frame in buffer + // Altref filtering assumes 16 pixel extension + const int et_y = 16; + const int el_y = 16; + // Motion estimation may use src block variance with the block size up + // to 64x64, so the right and bottom need to be extended to 64 multiple + // or up to 16, whichever is greater. + const int er_y = + AOMMAX(src->y_width + 16, ALIGN_POWER_OF_TWO(src->y_width, 6)) - + src->y_crop_width; + const int eb_y = + AOMMAX(src->y_height + 16, ALIGN_POWER_OF_TWO(src->y_height, 6)) - + src->y_crop_height; + const int uv_width_subsampling = (src->uv_width != src->y_width); + const int uv_height_subsampling = (src->uv_height != src->y_height); + const int et_uv = et_y >> uv_height_subsampling; + const int el_uv = el_y >> uv_width_subsampling; + const int eb_uv = eb_y >> uv_height_subsampling; + const int er_uv = er_y >> uv_width_subsampling; + +#if CONFIG_HIGHBITDEPTH + if (src->flags & YV12_FLAG_HIGHBITDEPTH) { + highbd_copy_and_extend_plane(src->y_buffer, src->y_stride, dst->y_buffer, + dst->y_stride, src->y_crop_width, + src->y_crop_height, et_y, el_y, eb_y, er_y); + + highbd_copy_and_extend_plane( + src->u_buffer, src->uv_stride, dst->u_buffer, dst->uv_stride, + src->uv_crop_width, src->uv_crop_height, et_uv, el_uv, eb_uv, er_uv); + + highbd_copy_and_extend_plane( + src->v_buffer, src->uv_stride, dst->v_buffer, dst->uv_stride, + src->uv_crop_width, src->uv_crop_height, et_uv, el_uv, eb_uv, er_uv); + return; + } +#endif // CONFIG_HIGHBITDEPTH + + copy_and_extend_plane(src->y_buffer, src->y_stride, dst->y_buffer, + dst->y_stride, src->y_crop_width, src->y_crop_height, + et_y, el_y, eb_y, er_y); + + copy_and_extend_plane(src->u_buffer, src->uv_stride, dst->u_buffer, + dst->uv_stride, src->uv_crop_width, src->uv_crop_height, + et_uv, el_uv, eb_uv, er_uv); + + copy_and_extend_plane(src->v_buffer, src->uv_stride, dst->v_buffer, + dst->uv_stride, src->uv_crop_width, src->uv_crop_height, + et_uv, el_uv, eb_uv, er_uv); +} + +void av1_copy_and_extend_frame_with_rect(const YV12_BUFFER_CONFIG *src, + YV12_BUFFER_CONFIG *dst, int srcy, + int srcx, int srch, int srcw) { + // If the side is not touching the bounder then don't extend. + const int et_y = srcy ? 0 : dst->border; + const int el_y = srcx ? 0 : dst->border; + const int eb_y = srcy + srch != src->y_height + ? 0 + : dst->border + dst->y_height - src->y_height; + const int er_y = srcx + srcw != src->y_width + ? 0 + : dst->border + dst->y_width - src->y_width; + const int src_y_offset = srcy * src->y_stride + srcx; + const int dst_y_offset = srcy * dst->y_stride + srcx; + + const int et_uv = ROUND_POWER_OF_TWO(et_y, 1); + const int el_uv = ROUND_POWER_OF_TWO(el_y, 1); + const int eb_uv = ROUND_POWER_OF_TWO(eb_y, 1); + const int er_uv = ROUND_POWER_OF_TWO(er_y, 1); + const int src_uv_offset = ((srcy * src->uv_stride) >> 1) + (srcx >> 1); + const int dst_uv_offset = ((srcy * dst->uv_stride) >> 1) + (srcx >> 1); + const int srch_uv = ROUND_POWER_OF_TWO(srch, 1); + const int srcw_uv = ROUND_POWER_OF_TWO(srcw, 1); + + copy_and_extend_plane(src->y_buffer + src_y_offset, src->y_stride, + dst->y_buffer + dst_y_offset, dst->y_stride, srcw, srch, + et_y, el_y, eb_y, er_y); + + copy_and_extend_plane(src->u_buffer + src_uv_offset, src->uv_stride, + dst->u_buffer + dst_uv_offset, dst->uv_stride, srcw_uv, + srch_uv, et_uv, el_uv, eb_uv, er_uv); + + copy_and_extend_plane(src->v_buffer + src_uv_offset, src->uv_stride, + dst->v_buffer + dst_uv_offset, dst->uv_stride, srcw_uv, + srch_uv, et_uv, el_uv, eb_uv, er_uv); +} diff --git a/third_party/aom/av1/encoder/extend.h b/third_party/aom/av1/encoder/extend.h new file mode 100644 index 0000000000..48178b9647 --- /dev/null +++ b/third_party/aom/av1/encoder/extend.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_EXTEND_H_ +#define AV1_ENCODER_EXTEND_H_ + +#include "aom_scale/yv12config.h" +#include "aom/aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_copy_and_extend_frame(const YV12_BUFFER_CONFIG *src, + YV12_BUFFER_CONFIG *dst); + +void av1_copy_and_extend_frame_with_rect(const YV12_BUFFER_CONFIG *src, + YV12_BUFFER_CONFIG *dst, int srcy, + int srcx, int srch, int srcw); +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_EXTEND_H_ diff --git a/third_party/aom/av1/encoder/firstpass.c b/third_party/aom/av1/encoder/firstpass.c new file mode 100644 index 0000000000..e35a54ef2f --- /dev/null +++ b/third_party/aom/av1/encoder/firstpass.c @@ -0,0 +1,3026 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "./aom_scale_rtcd.h" + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" +#include "aom_ports/system_state.h" +#include "aom_scale/aom_scale.h" +#include "aom_scale/yv12config.h" + +#include "aom_dsp/variance.h" +#include "av1/common/entropymv.h" +#include "av1/common/quant_common.h" +#include "av1/common/reconinter.h" // av1_setup_dst_planes() +#include "av1/encoder/av1_quantize.h" +#include "av1/encoder/aq_variance.h" +#include "av1/encoder/block.h" +#include "av1/encoder/encodeframe.h" +#include "av1/encoder/encodemb.h" +#include "av1/encoder/encodemv.h" +#include "av1/encoder/encoder.h" +#include "av1/encoder/extend.h" +#include "av1/encoder/firstpass.h" +#include "av1/encoder/mcomp.h" +#include "av1/encoder/rd.h" + +#define OUTPUT_FPF 0 +#define ARF_STATS_OUTPUT 0 + +#define GROUP_ADAPTIVE_MAXQ 1 + +#define BOOST_BREAKOUT 12.5 +#define BOOST_FACTOR 12.5 +#define FACTOR_PT_LOW 0.70 +#define FACTOR_PT_HIGH 0.90 +#define FIRST_PASS_Q 10.0 +#define GF_MAX_BOOST 96.0 +#define INTRA_MODE_PENALTY 1024 +#define KF_MAX_BOOST 128.0 +#define MIN_ARF_GF_BOOST 240 +#define MIN_DECAY_FACTOR 0.01 +#define MIN_KF_BOOST 300 +#define NEW_MV_MODE_PENALTY 32 +#define DARK_THRESH 64 +#define DEFAULT_GRP_WEIGHT 1.0 +#define RC_FACTOR_MIN 0.75 +#define RC_FACTOR_MAX 1.75 + +#define NCOUNT_INTRA_THRESH 8192 +#define NCOUNT_INTRA_FACTOR 3 +#define NCOUNT_FRAME_II_THRESH 5.0 + +#define DOUBLE_DIVIDE_CHECK(x) ((x) < 0 ? (x)-0.000001 : (x) + 0.000001) + +#if ARF_STATS_OUTPUT +unsigned int arf_count = 0; +#endif + +// Resets the first pass file to the given position using a relative seek from +// the current position. +static void reset_fpf_position(TWO_PASS *p, const FIRSTPASS_STATS *position) { + p->stats_in = position; +} + +// Read frame stats at an offset from the current position. +static const FIRSTPASS_STATS *read_frame_stats(const TWO_PASS *p, int offset) { + if ((offset >= 0 && p->stats_in + offset >= p->stats_in_end) || + (offset < 0 && p->stats_in + offset < p->stats_in_start)) { + return NULL; + } + + return &p->stats_in[offset]; +} + +static int input_stats(TWO_PASS *p, FIRSTPASS_STATS *fps) { + if (p->stats_in >= p->stats_in_end) return EOF; + + *fps = *p->stats_in; + ++p->stats_in; + return 1; +} + +static void output_stats(FIRSTPASS_STATS *stats, + struct aom_codec_pkt_list *pktlist) { + struct aom_codec_cx_pkt pkt; + pkt.kind = AOM_CODEC_STATS_PKT; + pkt.data.twopass_stats.buf = stats; + pkt.data.twopass_stats.sz = sizeof(FIRSTPASS_STATS); + aom_codec_pkt_list_add(pktlist, &pkt); + +// TEMP debug code +#if OUTPUT_FPF + { + FILE *fpfile; + fpfile = fopen("firstpass.stt", "a"); + + fprintf(fpfile, + "%12.0lf %12.4lf %12.0lf %12.0lf %12.0lf %12.4lf %12.4lf" + "%12.4lf %12.4lf %12.4lf %12.4lf %12.4lf %12.4lf %12.4lf %12.4lf" + "%12.4lf %12.4lf %12.0lf %12.0lf %12.0lf %12.4lf\n", + stats->frame, stats->weight, stats->intra_error, stats->coded_error, + stats->sr_coded_error, stats->pcnt_inter, stats->pcnt_motion, + stats->pcnt_second_ref, stats->pcnt_neutral, stats->intra_skip_pct, + stats->inactive_zone_rows, stats->inactive_zone_cols, stats->MVr, + stats->mvr_abs, stats->MVc, stats->mvc_abs, stats->MVrv, + stats->MVcv, stats->mv_in_out_count, stats->new_mv_count, + stats->count, stats->duration); + fclose(fpfile); + } +#endif +} + +#if CONFIG_FP_MB_STATS +static void output_fpmb_stats(uint8_t *this_frame_mb_stats, int stats_size, + struct aom_codec_pkt_list *pktlist) { + struct aom_codec_cx_pkt pkt; + pkt.kind = AOM_CODEC_FPMB_STATS_PKT; + pkt.data.firstpass_mb_stats.buf = this_frame_mb_stats; + pkt.data.firstpass_mb_stats.sz = stats_size * sizeof(*this_frame_mb_stats); + aom_codec_pkt_list_add(pktlist, &pkt); +} +#endif + +static void zero_stats(FIRSTPASS_STATS *section) { + section->frame = 0.0; + section->weight = 0.0; + section->intra_error = 0.0; + section->coded_error = 0.0; + section->sr_coded_error = 0.0; + section->pcnt_inter = 0.0; + section->pcnt_motion = 0.0; + section->pcnt_second_ref = 0.0; + section->pcnt_neutral = 0.0; + section->intra_skip_pct = 0.0; + section->inactive_zone_rows = 0.0; + section->inactive_zone_cols = 0.0; + section->MVr = 0.0; + section->mvr_abs = 0.0; + section->MVc = 0.0; + section->mvc_abs = 0.0; + section->MVrv = 0.0; + section->MVcv = 0.0; + section->mv_in_out_count = 0.0; + section->new_mv_count = 0.0; + section->count = 0.0; + section->duration = 1.0; +} + +static void accumulate_stats(FIRSTPASS_STATS *section, + const FIRSTPASS_STATS *frame) { + section->frame += frame->frame; + section->weight += frame->weight; + section->intra_error += frame->intra_error; + section->coded_error += frame->coded_error; + section->sr_coded_error += frame->sr_coded_error; + section->pcnt_inter += frame->pcnt_inter; + section->pcnt_motion += frame->pcnt_motion; + section->pcnt_second_ref += frame->pcnt_second_ref; + section->pcnt_neutral += frame->pcnt_neutral; + section->intra_skip_pct += frame->intra_skip_pct; + section->inactive_zone_rows += frame->inactive_zone_rows; + section->inactive_zone_cols += frame->inactive_zone_cols; + section->MVr += frame->MVr; + section->mvr_abs += frame->mvr_abs; + section->MVc += frame->MVc; + section->mvc_abs += frame->mvc_abs; + section->MVrv += frame->MVrv; + section->MVcv += frame->MVcv; + section->mv_in_out_count += frame->mv_in_out_count; + section->new_mv_count += frame->new_mv_count; + section->count += frame->count; + section->duration += frame->duration; +} + +static void subtract_stats(FIRSTPASS_STATS *section, + const FIRSTPASS_STATS *frame) { + section->frame -= frame->frame; + section->weight -= frame->weight; + section->intra_error -= frame->intra_error; + section->coded_error -= frame->coded_error; + section->sr_coded_error -= frame->sr_coded_error; + section->pcnt_inter -= frame->pcnt_inter; + section->pcnt_motion -= frame->pcnt_motion; + section->pcnt_second_ref -= frame->pcnt_second_ref; + section->pcnt_neutral -= frame->pcnt_neutral; + section->intra_skip_pct -= frame->intra_skip_pct; + section->inactive_zone_rows -= frame->inactive_zone_rows; + section->inactive_zone_cols -= frame->inactive_zone_cols; + section->MVr -= frame->MVr; + section->mvr_abs -= frame->mvr_abs; + section->MVc -= frame->MVc; + section->mvc_abs -= frame->mvc_abs; + section->MVrv -= frame->MVrv; + section->MVcv -= frame->MVcv; + section->mv_in_out_count -= frame->mv_in_out_count; + section->new_mv_count -= frame->new_mv_count; + section->count -= frame->count; + section->duration -= frame->duration; +} + +// Calculate the linear size relative to a baseline of 1080P +#define BASE_SIZE 2073600.0 // 1920x1080 +static double get_linear_size_factor(const AV1_COMP *cpi) { + const double this_area = cpi->initial_width * cpi->initial_height; + return pow(this_area / BASE_SIZE, 0.5); +} + +// Calculate an active area of the image that discounts formatting +// bars and partially discounts other 0 energy areas. +#define MIN_ACTIVE_AREA 0.5 +#define MAX_ACTIVE_AREA 1.0 +static double calculate_active_area(const AV1_COMP *cpi, + const FIRSTPASS_STATS *this_frame) { + double active_pct; + + active_pct = + 1.0 - + ((this_frame->intra_skip_pct / 2) + + ((this_frame->inactive_zone_rows * 2) / (double)cpi->common.mb_rows)); + return fclamp(active_pct, MIN_ACTIVE_AREA, MAX_ACTIVE_AREA); +} + +// Calculate a modified Error used in distributing bits between easier and +// harder frames. +#define ACT_AREA_CORRECTION 0.5 +static double calculate_modified_err(const AV1_COMP *cpi, + const TWO_PASS *twopass, + const AV1EncoderConfig *oxcf, + const FIRSTPASS_STATS *this_frame) { + const FIRSTPASS_STATS *const stats = &twopass->total_stats; + const double av_weight = stats->weight / stats->count; + const double av_err = (stats->coded_error * av_weight) / stats->count; + double modified_error = + av_err * pow(this_frame->coded_error * this_frame->weight / + DOUBLE_DIVIDE_CHECK(av_err), + oxcf->two_pass_vbrbias / 100.0); + + // Correction for active area. Frames with a reduced active area + // (eg due to formatting bars) have a higher error per mb for the + // remaining active MBs. The correction here assumes that coding + // 0.5N blocks of complexity 2X is a little easier than coding N + // blocks of complexity X. + modified_error *= + pow(calculate_active_area(cpi, this_frame), ACT_AREA_CORRECTION); + + return fclamp(modified_error, twopass->modified_error_min, + twopass->modified_error_max); +} + +// This function returns the maximum target rate per frame. +static int frame_max_bits(const RATE_CONTROL *rc, + const AV1EncoderConfig *oxcf) { + int64_t max_bits = ((int64_t)rc->avg_frame_bandwidth * + (int64_t)oxcf->two_pass_vbrmax_section) / + 100; + if (max_bits < 0) + max_bits = 0; + else if (max_bits > rc->max_frame_bandwidth) + max_bits = rc->max_frame_bandwidth; + + return (int)max_bits; +} + +void av1_init_first_pass(AV1_COMP *cpi) { + zero_stats(&cpi->twopass.total_stats); +} + +void av1_end_first_pass(AV1_COMP *cpi) { + output_stats(&cpi->twopass.total_stats, cpi->output_pkt_list); +} + +static aom_variance_fn_t get_block_variance_fn(BLOCK_SIZE bsize) { + switch (bsize) { + case BLOCK_8X8: return aom_mse8x8; + case BLOCK_16X8: return aom_mse16x8; + case BLOCK_8X16: return aom_mse8x16; + default: return aom_mse16x16; + } +} + +static unsigned int get_prediction_error(BLOCK_SIZE bsize, + const struct buf_2d *src, + const struct buf_2d *ref) { + unsigned int sse; + const aom_variance_fn_t fn = get_block_variance_fn(bsize); + fn(src->buf, src->stride, ref->buf, ref->stride, &sse); + return sse; +} + +#if CONFIG_HIGHBITDEPTH +static aom_variance_fn_t highbd_get_block_variance_fn(BLOCK_SIZE bsize, + int bd) { + switch (bd) { + default: + switch (bsize) { + case BLOCK_8X8: return aom_highbd_8_mse8x8; + case BLOCK_16X8: return aom_highbd_8_mse16x8; + case BLOCK_8X16: return aom_highbd_8_mse8x16; + default: return aom_highbd_8_mse16x16; + } + break; + case 10: + switch (bsize) { + case BLOCK_8X8: return aom_highbd_10_mse8x8; + case BLOCK_16X8: return aom_highbd_10_mse16x8; + case BLOCK_8X16: return aom_highbd_10_mse8x16; + default: return aom_highbd_10_mse16x16; + } + break; + case 12: + switch (bsize) { + case BLOCK_8X8: return aom_highbd_12_mse8x8; + case BLOCK_16X8: return aom_highbd_12_mse16x8; + case BLOCK_8X16: return aom_highbd_12_mse8x16; + default: return aom_highbd_12_mse16x16; + } + break; + } +} + +static unsigned int highbd_get_prediction_error(BLOCK_SIZE bsize, + const struct buf_2d *src, + const struct buf_2d *ref, + int bd) { + unsigned int sse; + const aom_variance_fn_t fn = highbd_get_block_variance_fn(bsize, bd); + fn(src->buf, src->stride, ref->buf, ref->stride, &sse); + return sse; +} +#endif // CONFIG_HIGHBITDEPTH + +// Refine the motion search range according to the frame dimension +// for first pass test. +static int get_search_range(const AV1_COMP *cpi) { + int sr = 0; + const int dim = AOMMIN(cpi->initial_width, cpi->initial_height); + + while ((dim << sr) < MAX_FULL_PEL_VAL) ++sr; + return sr; +} + +static void first_pass_motion_search(AV1_COMP *cpi, MACROBLOCK *x, + const MV *ref_mv, MV *best_mv, + int *best_motion_err) { + MACROBLOCKD *const xd = &x->e_mbd; + MV tmp_mv = { 0, 0 }; + MV ref_mv_full = { ref_mv->row >> 3, ref_mv->col >> 3 }; + int num00, tmp_err, n; + const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + aom_variance_fn_ptr_t v_fn_ptr = cpi->fn_ptr[bsize]; + const int new_mv_mode_penalty = NEW_MV_MODE_PENALTY; + + int step_param = 3; + int further_steps = (MAX_MVSEARCH_STEPS - 1) - step_param; + const int sr = get_search_range(cpi); + step_param += sr; + further_steps -= sr; + + // Override the default variance function to use MSE. + v_fn_ptr.vf = get_block_variance_fn(bsize); +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + v_fn_ptr.vf = highbd_get_block_variance_fn(bsize, xd->bd); + } +#endif // CONFIG_HIGHBITDEPTH + + // Center the initial step/diamond search on best mv. + tmp_err = cpi->diamond_search_sad(x, &cpi->ss_cfg, &ref_mv_full, &tmp_mv, + step_param, x->sadperbit16, &num00, + &v_fn_ptr, ref_mv); + if (tmp_err < INT_MAX) + tmp_err = av1_get_mvpred_var(x, &tmp_mv, ref_mv, &v_fn_ptr, 1); + if (tmp_err < INT_MAX - new_mv_mode_penalty) tmp_err += new_mv_mode_penalty; + + if (tmp_err < *best_motion_err) { + *best_motion_err = tmp_err; + *best_mv = tmp_mv; + } + + // Carry out further step/diamond searches as necessary. + n = num00; + num00 = 0; + + while (n < further_steps) { + ++n; + + if (num00) { + --num00; + } else { + tmp_err = cpi->diamond_search_sad(x, &cpi->ss_cfg, &ref_mv_full, &tmp_mv, + step_param + n, x->sadperbit16, &num00, + &v_fn_ptr, ref_mv); + if (tmp_err < INT_MAX) + tmp_err = av1_get_mvpred_var(x, &tmp_mv, ref_mv, &v_fn_ptr, 1); + if (tmp_err < INT_MAX - new_mv_mode_penalty) + tmp_err += new_mv_mode_penalty; + + if (tmp_err < *best_motion_err) { + *best_motion_err = tmp_err; + *best_mv = tmp_mv; + } + } + } +} + +static BLOCK_SIZE get_bsize(const AV1_COMMON *cm, int mb_row, int mb_col) { + if (mi_size_wide[BLOCK_16X16] * mb_col + mi_size_wide[BLOCK_8X8] < + cm->mi_cols) { + return mi_size_wide[BLOCK_16X16] * mb_row + mi_size_wide[BLOCK_8X8] < + cm->mi_rows + ? BLOCK_16X16 + : BLOCK_16X8; + } else { + return mi_size_wide[BLOCK_16X16] * mb_row + mi_size_wide[BLOCK_8X8] < + cm->mi_rows + ? BLOCK_8X16 + : BLOCK_8X8; + } +} + +static int find_fp_qindex(aom_bit_depth_t bit_depth) { + int i; + + for (i = 0; i < QINDEX_RANGE; ++i) + if (av1_convert_qindex_to_q(i, bit_depth) >= FIRST_PASS_Q) break; + + if (i == QINDEX_RANGE) i--; + + return i; +} + +static void set_first_pass_params(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + if (!cpi->refresh_alt_ref_frame && + (cm->current_video_frame == 0 || (cpi->frame_flags & FRAMEFLAGS_KEY))) { + cm->frame_type = KEY_FRAME; + } else { + cm->frame_type = INTER_FRAME; + } + // Do not use periodic key frames. + cpi->rc.frames_to_key = INT_MAX; +} + +#define UL_INTRA_THRESH 50 +#define INVALID_ROW -1 +void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { + int mb_row, mb_col; + MACROBLOCK *const x = &cpi->td.mb; + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + TileInfo tile; + struct macroblock_plane *const p = x->plane; + struct macroblockd_plane *const pd = xd->plane; + const PICK_MODE_CONTEXT *ctx = + &cpi->td.pc_root[MAX_MIB_SIZE_LOG2 - MIN_MIB_SIZE_LOG2]->none; + int i; + + int recon_yoffset, recon_uvoffset; + int64_t intra_error = 0; + int64_t coded_error = 0; + int64_t sr_coded_error = 0; + + int sum_mvr = 0, sum_mvc = 0; + int sum_mvr_abs = 0, sum_mvc_abs = 0; + int64_t sum_mvrs = 0, sum_mvcs = 0; + int mvcount = 0; + int intercount = 0; + int second_ref_count = 0; + const int intrapenalty = INTRA_MODE_PENALTY; + double neutral_count; + int intra_skip_count = 0; + int image_data_start_row = INVALID_ROW; + int new_mv_count = 0; + int sum_in_vectors = 0; + MV lastmv = { 0, 0 }; + TWO_PASS *twopass = &cpi->twopass; + const MV zero_mv = { 0, 0 }; + int recon_y_stride, recon_uv_stride, uv_mb_height; + + YV12_BUFFER_CONFIG *const lst_yv12 = get_ref_frame_buffer(cpi, LAST_FRAME); + YV12_BUFFER_CONFIG *gld_yv12 = get_ref_frame_buffer(cpi, GOLDEN_FRAME); + YV12_BUFFER_CONFIG *const new_yv12 = get_frame_new_buffer(cm); + const YV12_BUFFER_CONFIG *first_ref_buf = lst_yv12; + double intra_factor; + double brightness_factor; + BufferPool *const pool = cm->buffer_pool; + const int qindex = find_fp_qindex(cm->bit_depth); + const int mb_scale = mi_size_wide[BLOCK_16X16]; +#if CONFIG_PVQ + PVQ_QUEUE pvq_q; + od_adapt_ctx pvq_context; +#endif + + // First pass code requires valid last and new frame buffers. + assert(new_yv12 != NULL); + assert(frame_is_intra_only(cm) || (lst_yv12 != NULL)); + +#if CONFIG_FP_MB_STATS + if (cpi->use_fp_mb_stats) { + av1_zero_array(cpi->twopass.frame_mb_stats_buf, cpi->initial_mbs); + } +#endif + + aom_clear_system_state(); + + xd->mi = cm->mi_grid_visible; + xd->mi[0] = cm->mi; + x->e_mbd.mi[0]->mbmi.sb_type = BLOCK_16X16; + + intra_factor = 0.0; + brightness_factor = 0.0; + neutral_count = 0.0; + + set_first_pass_params(cpi); + av1_set_quantizer(cm, qindex); + + av1_setup_block_planes(&x->e_mbd, cm->subsampling_x, cm->subsampling_y); + + av1_setup_src_planes(x, cpi->source, 0, 0); + av1_setup_dst_planes(xd->plane, cm->sb_size, new_yv12, 0, 0); + + if (!frame_is_intra_only(cm)) { + av1_setup_pre_planes(xd, 0, first_ref_buf, 0, 0, NULL); + } + + xd->mi = cm->mi_grid_visible; + xd->mi[0] = cm->mi; + +#if CONFIG_CFL + // Don't store luma on the fist pass since chroma is not computed + x->cfl_store_y = 0; +#endif + av1_frame_init_quantizer(cpi); + +#if CONFIG_PVQ + // For pass 1 of 2-pass encoding, init here for PVQ for now. + { + pvq_q.buf_len = 5000; + CHECK_MEM_ERROR(cm, pvq_q.buf, + aom_malloc(pvq_q.buf_len * sizeof(PVQ_INFO))); + pvq_q.curr_pos = 0; + x->pvq_coded = 0; + + x->pvq_q = &pvq_q; + + // TODO(yushin): Since this init step is also called in 2nd pass, + // or 1-pass encoding, consider factoring out it as a function. + // TODO(yushin) + // If activity masking is enabled, change below to OD_HVS_QM + x->daala_enc.qm = OD_FLAT_QM; // Hard coded. Enc/dec required to sync. + x->daala_enc.pvq_norm_lambda = OD_PVQ_LAMBDA; + x->daala_enc.pvq_norm_lambda_dc = OD_PVQ_LAMBDA; + + od_init_qm(x->daala_enc.state.qm, x->daala_enc.state.qm_inv, + x->daala_enc.qm == OD_HVS_QM ? OD_QM8_Q4_HVS : OD_QM8_Q4_FLAT); +#if CONFIG_DAALA_EC + od_ec_enc_init(&x->daala_enc.w.ec, 65025); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + +#if CONFIG_DAALA_EC + od_ec_enc_reset(&x->daala_enc.w.ec); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + } +#endif + + for (i = 0; i < MAX_MB_PLANE; ++i) { + p[i].coeff = ctx->coeff[i]; + p[i].qcoeff = ctx->qcoeff[i]; + pd[i].dqcoeff = ctx->dqcoeff[i]; +#if CONFIG_PVQ + pd[i].pvq_ref_coeff = ctx->pvq_ref_coeff[i]; +#endif + p[i].eobs = ctx->eobs[i]; +#if CONFIG_LV_MAP + p[i].txb_entropy_ctx = ctx->txb_entropy_ctx[i]; +#endif + } + + av1_init_mv_probs(cm); +#if CONFIG_ADAPT_SCAN + av1_init_scan_order(cm); +#endif + av1_convolve_init(cm); +#if CONFIG_PVQ + od_adapt_ctx_reset(&pvq_context, 0); + x->daala_enc.state.adapt = &pvq_context; +#endif // CONFIG_PVQ + av1_initialize_rd_consts(cpi); + + // Tiling is ignored in the first pass. + av1_tile_init(&tile, cm, 0, 0); + + recon_y_stride = new_yv12->y_stride; + recon_uv_stride = new_yv12->uv_stride; + uv_mb_height = 16 >> (new_yv12->y_height > new_yv12->uv_height); + + for (mb_row = 0; mb_row < cm->mb_rows; ++mb_row) { + MV best_ref_mv = { 0, 0 }; + + // Reset above block coeffs. + xd->up_available = (mb_row != 0); + recon_yoffset = (mb_row * recon_y_stride * 16); + recon_uvoffset = (mb_row * recon_uv_stride * uv_mb_height); + + // Set up limit values for motion vectors to prevent them extending + // outside the UMV borders. + x->mv_limits.row_min = -((mb_row * 16) + BORDER_MV_PIXELS_B16); + x->mv_limits.row_max = + ((cm->mb_rows - 1 - mb_row) * 16) + BORDER_MV_PIXELS_B16; + + for (mb_col = 0; mb_col < cm->mb_cols; ++mb_col) { + int this_error; + const int use_dc_pred = (mb_col || mb_row) && (!mb_col || !mb_row); + const BLOCK_SIZE bsize = get_bsize(cm, mb_row, mb_col); + double log_intra; + int level_sample; + +#if CONFIG_FP_MB_STATS + const int mb_index = mb_row * cm->mb_cols + mb_col; +#endif + + aom_clear_system_state(); + + xd->plane[0].dst.buf = new_yv12->y_buffer + recon_yoffset; + xd->plane[1].dst.buf = new_yv12->u_buffer + recon_uvoffset; + xd->plane[2].dst.buf = new_yv12->v_buffer + recon_uvoffset; + xd->left_available = (mb_col != 0); + xd->mi[0]->mbmi.sb_type = bsize; + xd->mi[0]->mbmi.ref_frame[0] = INTRA_FRAME; + set_mi_row_col(xd, &tile, mb_row * mb_scale, mi_size_high[bsize], + mb_col * mb_scale, mi_size_wide[bsize], +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + + set_plane_n4(xd, mi_size_wide[bsize], mi_size_high[bsize]); + + // Do intra 16x16 prediction. + xd->mi[0]->mbmi.segment_id = 0; +#if CONFIG_SUPERTX + xd->mi[0]->mbmi.segment_id_supertx = 0; +#endif // CONFIG_SUPERTX + xd->lossless[xd->mi[0]->mbmi.segment_id] = (qindex == 0); + xd->mi[0]->mbmi.mode = DC_PRED; + xd->mi[0]->mbmi.tx_size = + use_dc_pred ? (bsize >= BLOCK_16X16 ? TX_16X16 : TX_8X8) : TX_4X4; + av1_encode_intra_block_plane(cm, x, bsize, 0, 0, mb_row * 2, mb_col * 2); + this_error = aom_get_mb_ss(x->plane[0].src_diff); + + // Keep a record of blocks that have almost no intra error residual + // (i.e. are in effect completely flat and untextured in the intra + // domain). In natural videos this is uncommon, but it is much more + // common in animations, graphics and screen content, so may be used + // as a signal to detect these types of content. + if (this_error < UL_INTRA_THRESH) { + ++intra_skip_count; + } else if ((mb_col > 0) && (image_data_start_row == INVALID_ROW)) { + image_data_start_row = mb_row; + } + +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) { + switch (cm->bit_depth) { + case AOM_BITS_8: break; + case AOM_BITS_10: this_error >>= 4; break; + case AOM_BITS_12: this_error >>= 8; break; + default: + assert(0 && + "cm->bit_depth should be AOM_BITS_8, " + "AOM_BITS_10 or AOM_BITS_12"); + return; + } + } +#endif // CONFIG_HIGHBITDEPTH + + aom_clear_system_state(); + log_intra = log(this_error + 1.0); + if (log_intra < 10.0) + intra_factor += 1.0 + ((10.0 - log_intra) * 0.05); + else + intra_factor += 1.0; + +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + level_sample = CONVERT_TO_SHORTPTR(x->plane[0].src.buf)[0]; + else + level_sample = x->plane[0].src.buf[0]; +#else + level_sample = x->plane[0].src.buf[0]; +#endif + if ((level_sample < DARK_THRESH) && (log_intra < 9.0)) + brightness_factor += 1.0 + (0.01 * (DARK_THRESH - level_sample)); + else + brightness_factor += 1.0; + + // Intrapenalty below deals with situations where the intra and inter + // error scores are very low (e.g. a plain black frame). + // We do not have special cases in first pass for 0,0 and nearest etc so + // all inter modes carry an overhead cost estimate for the mv. + // When the error score is very low this causes us to pick all or lots of + // INTRA modes and throw lots of key frames. + // This penalty adds a cost matching that of a 0,0 mv to the intra case. + this_error += intrapenalty; + + // Accumulate the intra error. + intra_error += (int64_t)this_error; + +#if CONFIG_FP_MB_STATS + if (cpi->use_fp_mb_stats) { + // initialization + cpi->twopass.frame_mb_stats_buf[mb_index] = 0; + } +#endif + + // Set up limit values for motion vectors to prevent them extending + // outside the UMV borders. + x->mv_limits.col_min = -((mb_col * 16) + BORDER_MV_PIXELS_B16); + x->mv_limits.col_max = + ((cm->mb_cols - 1 - mb_col) * 16) + BORDER_MV_PIXELS_B16; + + if (!frame_is_intra_only(cm)) { // Do a motion search + int tmp_err, motion_error, raw_motion_error; + // Assume 0,0 motion with no mv overhead. + MV mv = { 0, 0 }, tmp_mv = { 0, 0 }; + struct buf_2d unscaled_last_source_buf_2d; + + xd->plane[0].pre[0].buf = first_ref_buf->y_buffer + recon_yoffset; +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + motion_error = highbd_get_prediction_error( + bsize, &x->plane[0].src, &xd->plane[0].pre[0], xd->bd); + } else { + motion_error = get_prediction_error(bsize, &x->plane[0].src, + &xd->plane[0].pre[0]); + } +#else + motion_error = + get_prediction_error(bsize, &x->plane[0].src, &xd->plane[0].pre[0]); +#endif // CONFIG_HIGHBITDEPTH + + // Compute the motion error of the 0,0 motion using the last source + // frame as the reference. Skip the further motion search on + // reconstructed frame if this error is small. + unscaled_last_source_buf_2d.buf = + cpi->unscaled_last_source->y_buffer + recon_yoffset; + unscaled_last_source_buf_2d.stride = + cpi->unscaled_last_source->y_stride; +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + raw_motion_error = highbd_get_prediction_error( + bsize, &x->plane[0].src, &unscaled_last_source_buf_2d, xd->bd); + } else { + raw_motion_error = get_prediction_error(bsize, &x->plane[0].src, + &unscaled_last_source_buf_2d); + } +#else + raw_motion_error = get_prediction_error(bsize, &x->plane[0].src, + &unscaled_last_source_buf_2d); +#endif // CONFIG_HIGHBITDEPTH + + // TODO(pengchong): Replace the hard-coded threshold + if (raw_motion_error > 25) { + // Test last reference frame using the previous best mv as the + // starting point (best reference) for the search. + first_pass_motion_search(cpi, x, &best_ref_mv, &mv, &motion_error); + + // If the current best reference mv is not centered on 0,0 then do a + // 0,0 based search as well. + if (!is_zero_mv(&best_ref_mv)) { + tmp_err = INT_MAX; + first_pass_motion_search(cpi, x, &zero_mv, &tmp_mv, &tmp_err); + + if (tmp_err < motion_error) { + motion_error = tmp_err; + mv = tmp_mv; + } + } + + // Search in an older reference frame. + if ((cm->current_video_frame > 1) && gld_yv12 != NULL) { + // Assume 0,0 motion with no mv overhead. + int gf_motion_error; + + xd->plane[0].pre[0].buf = gld_yv12->y_buffer + recon_yoffset; +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + gf_motion_error = highbd_get_prediction_error( + bsize, &x->plane[0].src, &xd->plane[0].pre[0], xd->bd); + } else { + gf_motion_error = get_prediction_error(bsize, &x->plane[0].src, + &xd->plane[0].pre[0]); + } +#else + gf_motion_error = get_prediction_error(bsize, &x->plane[0].src, + &xd->plane[0].pre[0]); +#endif // CONFIG_HIGHBITDEPTH + + first_pass_motion_search(cpi, x, &zero_mv, &tmp_mv, + &gf_motion_error); + + if (gf_motion_error < motion_error && gf_motion_error < this_error) + ++second_ref_count; + + // Reset to last frame as reference buffer. + xd->plane[0].pre[0].buf = first_ref_buf->y_buffer + recon_yoffset; + xd->plane[1].pre[0].buf = first_ref_buf->u_buffer + recon_uvoffset; + xd->plane[2].pre[0].buf = first_ref_buf->v_buffer + recon_uvoffset; + + // In accumulating a score for the older reference frame take the + // best of the motion predicted score and the intra coded error + // (just as will be done for) accumulation of "coded_error" for + // the last frame. + if (gf_motion_error < this_error) + sr_coded_error += gf_motion_error; + else + sr_coded_error += this_error; + } else { + sr_coded_error += motion_error; + } + } else { + sr_coded_error += motion_error; + } + + // Start by assuming that intra mode is best. + best_ref_mv.row = 0; + best_ref_mv.col = 0; + +#if CONFIG_FP_MB_STATS + if (cpi->use_fp_mb_stats) { + // intra predication statistics + cpi->twopass.frame_mb_stats_buf[mb_index] = 0; + cpi->twopass.frame_mb_stats_buf[mb_index] |= FPMB_DCINTRA_MASK; + cpi->twopass.frame_mb_stats_buf[mb_index] |= FPMB_MOTION_ZERO_MASK; + if (this_error > FPMB_ERROR_LARGE_TH) { + cpi->twopass.frame_mb_stats_buf[mb_index] |= FPMB_ERROR_LARGE_MASK; + } else if (this_error < FPMB_ERROR_SMALL_TH) { + cpi->twopass.frame_mb_stats_buf[mb_index] |= FPMB_ERROR_SMALL_MASK; + } + } +#endif + + if (motion_error <= this_error) { + aom_clear_system_state(); + + // Keep a count of cases where the inter and intra were very close + // and very low. This helps with scene cut detection for example in + // cropped clips with black bars at the sides or top and bottom. + if (((this_error - intrapenalty) * 9 <= motion_error * 10) && + (this_error < (2 * intrapenalty))) { + neutral_count += 1.0; + // Also track cases where the intra is not much worse than the inter + // and use this in limiting the GF/arf group length. + } else if ((this_error > NCOUNT_INTRA_THRESH) && + (this_error < (NCOUNT_INTRA_FACTOR * motion_error))) { + neutral_count += + (double)motion_error / DOUBLE_DIVIDE_CHECK((double)this_error); + } + + mv.row *= 8; + mv.col *= 8; + this_error = motion_error; + xd->mi[0]->mbmi.mode = NEWMV; + xd->mi[0]->mbmi.mv[0].as_mv = mv; + xd->mi[0]->mbmi.tx_size = TX_4X4; + xd->mi[0]->mbmi.ref_frame[0] = LAST_FRAME; + xd->mi[0]->mbmi.ref_frame[1] = NONE_FRAME; + av1_build_inter_predictors_sby(xd, mb_row * mb_scale, + mb_col * mb_scale, NULL, bsize); + av1_encode_sby_pass1(cm, x, bsize); + sum_mvr += mv.row; + sum_mvr_abs += abs(mv.row); + sum_mvc += mv.col; + sum_mvc_abs += abs(mv.col); + sum_mvrs += mv.row * mv.row; + sum_mvcs += mv.col * mv.col; + ++intercount; + + best_ref_mv = mv; + +#if CONFIG_FP_MB_STATS + if (cpi->use_fp_mb_stats) { + // inter predication statistics + cpi->twopass.frame_mb_stats_buf[mb_index] = 0; + cpi->twopass.frame_mb_stats_buf[mb_index] &= ~FPMB_DCINTRA_MASK; + cpi->twopass.frame_mb_stats_buf[mb_index] |= FPMB_MOTION_ZERO_MASK; + if (this_error > FPMB_ERROR_LARGE_TH) { + cpi->twopass.frame_mb_stats_buf[mb_index] |= + FPMB_ERROR_LARGE_MASK; + } else if (this_error < FPMB_ERROR_SMALL_TH) { + cpi->twopass.frame_mb_stats_buf[mb_index] |= + FPMB_ERROR_SMALL_MASK; + } + } +#endif + + if (!is_zero_mv(&mv)) { + ++mvcount; + +#if CONFIG_FP_MB_STATS + if (cpi->use_fp_mb_stats) { + cpi->twopass.frame_mb_stats_buf[mb_index] &= + ~FPMB_MOTION_ZERO_MASK; + // check estimated motion direction + if (mv.col > 0 && mv.col >= abs(mv.row)) { + // right direction + cpi->twopass.frame_mb_stats_buf[mb_index] |= + FPMB_MOTION_RIGHT_MASK; + } else if (mv.row < 0 && abs(mv.row) >= abs(mv.col)) { + // up direction + cpi->twopass.frame_mb_stats_buf[mb_index] |= + FPMB_MOTION_UP_MASK; + } else if (mv.col < 0 && abs(mv.col) >= abs(mv.row)) { + // left direction + cpi->twopass.frame_mb_stats_buf[mb_index] |= + FPMB_MOTION_LEFT_MASK; + } else { + // down direction + cpi->twopass.frame_mb_stats_buf[mb_index] |= + FPMB_MOTION_DOWN_MASK; + } + } +#endif + + // Non-zero vector, was it different from the last non zero vector? + if (!is_equal_mv(&mv, &lastmv)) ++new_mv_count; + lastmv = mv; + + // Does the row vector point inwards or outwards? + if (mb_row < cm->mb_rows / 2) { + if (mv.row > 0) + --sum_in_vectors; + else if (mv.row < 0) + ++sum_in_vectors; + } else if (mb_row > cm->mb_rows / 2) { + if (mv.row > 0) + ++sum_in_vectors; + else if (mv.row < 0) + --sum_in_vectors; + } + + // Does the col vector point inwards or outwards? + if (mb_col < cm->mb_cols / 2) { + if (mv.col > 0) + --sum_in_vectors; + else if (mv.col < 0) + ++sum_in_vectors; + } else if (mb_col > cm->mb_cols / 2) { + if (mv.col > 0) + ++sum_in_vectors; + else if (mv.col < 0) + --sum_in_vectors; + } + } + } + } else { + sr_coded_error += (int64_t)this_error; + } + coded_error += (int64_t)this_error; + + // Adjust to the next column of MBs. + x->plane[0].src.buf += 16; + x->plane[1].src.buf += uv_mb_height; + x->plane[2].src.buf += uv_mb_height; + + recon_yoffset += 16; + recon_uvoffset += uv_mb_height; + } + + // Adjust to the next row of MBs. + x->plane[0].src.buf += 16 * x->plane[0].src.stride - 16 * cm->mb_cols; + x->plane[1].src.buf += + uv_mb_height * x->plane[1].src.stride - uv_mb_height * cm->mb_cols; + x->plane[2].src.buf += + uv_mb_height * x->plane[1].src.stride - uv_mb_height * cm->mb_cols; + + aom_clear_system_state(); + } + +#if CONFIG_PVQ +#if CONFIG_DAALA_EC + od_ec_enc_clear(&x->daala_enc.w.ec); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + + x->pvq_q->last_pos = x->pvq_q->curr_pos; + x->pvq_q->curr_pos = 0; + x->pvq_q = NULL; + + aom_free(pvq_q.buf); +#endif + + // Clamp the image start to rows/2. This number of rows is discarded top + // and bottom as dead data so rows / 2 means the frame is blank. + if ((image_data_start_row > cm->mb_rows / 2) || + (image_data_start_row == INVALID_ROW)) { + image_data_start_row = cm->mb_rows / 2; + } + // Exclude any image dead zone + if (image_data_start_row > 0) { + intra_skip_count = + AOMMAX(0, intra_skip_count - (image_data_start_row * cm->mb_cols * 2)); + } + + { + FIRSTPASS_STATS fps; + // The minimum error here insures some bit allocation to frames even + // in static regions. The allocation per MB declines for larger formats + // where the typical "real" energy per MB also falls. + // Initial estimate here uses sqrt(mbs) to define the min_err, where the + // number of mbs is proportional to the image area. + const int num_mbs = (cpi->oxcf.resize_mode != RESIZE_NONE) + ? cpi->initial_mbs + : cpi->common.MBs; + const double min_err = 200 * sqrt(num_mbs); + + intra_factor = intra_factor / (double)num_mbs; + brightness_factor = brightness_factor / (double)num_mbs; + fps.weight = intra_factor * brightness_factor; + + fps.frame = cm->current_video_frame; + fps.coded_error = (double)(coded_error >> 8) + min_err; + fps.sr_coded_error = (double)(sr_coded_error >> 8) + min_err; + fps.intra_error = (double)(intra_error >> 8) + min_err; + fps.count = 1.0; + fps.pcnt_inter = (double)intercount / num_mbs; + fps.pcnt_second_ref = (double)second_ref_count / num_mbs; + fps.pcnt_neutral = (double)neutral_count / num_mbs; + fps.intra_skip_pct = (double)intra_skip_count / num_mbs; + fps.inactive_zone_rows = (double)image_data_start_row; + fps.inactive_zone_cols = (double)0; // TODO(paulwilkins): fix + + if (mvcount > 0) { + fps.MVr = (double)sum_mvr / mvcount; + fps.mvr_abs = (double)sum_mvr_abs / mvcount; + fps.MVc = (double)sum_mvc / mvcount; + fps.mvc_abs = (double)sum_mvc_abs / mvcount; + fps.MVrv = + ((double)sum_mvrs - ((double)sum_mvr * sum_mvr / mvcount)) / mvcount; + fps.MVcv = + ((double)sum_mvcs - ((double)sum_mvc * sum_mvc / mvcount)) / mvcount; + fps.mv_in_out_count = (double)sum_in_vectors / (mvcount * 2); + fps.new_mv_count = new_mv_count; + fps.pcnt_motion = (double)mvcount / num_mbs; + } else { + fps.MVr = 0.0; + fps.mvr_abs = 0.0; + fps.MVc = 0.0; + fps.mvc_abs = 0.0; + fps.MVrv = 0.0; + fps.MVcv = 0.0; + fps.mv_in_out_count = 0.0; + fps.new_mv_count = 0.0; + fps.pcnt_motion = 0.0; + } + + // TODO(paulwilkins): Handle the case when duration is set to 0, or + // something less than the full time between subsequent values of + // cpi->source_time_stamp. + fps.duration = (double)(source->ts_end - source->ts_start); + + // Don't want to do output stats with a stack variable! + twopass->this_frame_stats = fps; + output_stats(&twopass->this_frame_stats, cpi->output_pkt_list); + accumulate_stats(&twopass->total_stats, &fps); + +#if CONFIG_FP_MB_STATS + if (cpi->use_fp_mb_stats) { + output_fpmb_stats(twopass->frame_mb_stats_buf, cpi->initial_mbs, + cpi->output_pkt_list); + } +#endif + } + + // Copy the previous Last Frame back into gf and and arf buffers if + // the prediction is good enough... but also don't allow it to lag too far. + if ((twopass->sr_update_lag > 3) || + ((cm->current_video_frame > 0) && + (twopass->this_frame_stats.pcnt_inter > 0.20) && + ((twopass->this_frame_stats.intra_error / + DOUBLE_DIVIDE_CHECK(twopass->this_frame_stats.coded_error)) > 2.0))) { + if (gld_yv12 != NULL) { +#if CONFIG_EXT_REFS + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx], + cm->ref_frame_map[cpi->lst_fb_idxes[LAST_FRAME - LAST_FRAME]]); +#else + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx], + cm->ref_frame_map[cpi->lst_fb_idx]); +#endif // CONFIG_EXT_REFS + } + twopass->sr_update_lag = 1; + } else { + ++twopass->sr_update_lag; + } + + aom_extend_frame_borders(new_yv12); + +// The frame we just compressed now becomes the last frame. +#if CONFIG_EXT_REFS + ref_cnt_fb(pool->frame_bufs, + &cm->ref_frame_map[cpi->lst_fb_idxes[LAST_FRAME - LAST_FRAME]], + cm->new_fb_idx); +#else + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->lst_fb_idx], + cm->new_fb_idx); +#endif // CONFIG_EXT_REFS + + // Special case for the first frame. Copy into the GF buffer as a second + // reference. + if (cm->current_video_frame == 0 && cpi->gld_fb_idx != INVALID_IDX) { +#if CONFIG_EXT_REFS + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx], + cm->ref_frame_map[cpi->lst_fb_idxes[LAST_FRAME - LAST_FRAME]]); +#else + ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx], + cm->ref_frame_map[cpi->lst_fb_idx]); +#endif // CONFIG_EXT_REFS + } + + // Use this to see what the first pass reconstruction looks like. + if (0) { + char filename[512]; + FILE *recon_file; + snprintf(filename, sizeof(filename), "enc%04d.yuv", + (int)cm->current_video_frame); + + if (cm->current_video_frame == 0) + recon_file = fopen(filename, "wb"); + else + recon_file = fopen(filename, "ab"); + + (void)fwrite(lst_yv12->buffer_alloc, lst_yv12->frame_size, 1, recon_file); + fclose(recon_file); + } + + ++cm->current_video_frame; +} + +static double calc_correction_factor(double err_per_mb, double err_divisor, + double pt_low, double pt_high, int q, + aom_bit_depth_t bit_depth) { + const double error_term = err_per_mb / err_divisor; + + // Adjustment based on actual quantizer to power term. + const double power_term = + AOMMIN(av1_convert_qindex_to_q(q, bit_depth) * 0.01 + pt_low, pt_high); + + // Calculate correction factor. + if (power_term < 1.0) assert(error_term >= 0.0); + + return fclamp(pow(error_term, power_term), 0.05, 5.0); +} + +#define ERR_DIVISOR 100.0 +static int get_twopass_worst_quality(const AV1_COMP *cpi, + const double section_err, + double inactive_zone, + int section_target_bandwidth, + double group_weight_factor) { + const RATE_CONTROL *const rc = &cpi->rc; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + + inactive_zone = fclamp(inactive_zone, 0.0, 1.0); + + if (section_target_bandwidth <= 0) { + return rc->worst_quality; // Highest value allowed + } else { + const int num_mbs = (cpi->oxcf.resize_mode != RESIZE_NONE) + ? cpi->initial_mbs + : cpi->common.MBs; + const int active_mbs = AOMMAX(1, num_mbs - (int)(num_mbs * inactive_zone)); + const double av_err_per_mb = section_err / active_mbs; + const double speed_term = 1.0 + 0.04 * oxcf->speed; + double ediv_size_correction; + const int target_norm_bits_per_mb = + (int)((uint64_t)section_target_bandwidth << BPER_MB_NORMBITS) / + active_mbs; + int q; + + // Larger image formats are expected to be a little harder to code + // relatively given the same prediction error score. This in part at + // least relates to the increased size and hence coding overheads of + // motion vectors. Some account of this is made through adjustment of + // the error divisor. + ediv_size_correction = + AOMMAX(0.2, AOMMIN(5.0, get_linear_size_factor(cpi))); + if (ediv_size_correction < 1.0) + ediv_size_correction = -(1.0 / ediv_size_correction); + ediv_size_correction *= 4.0; + + // Try and pick a max Q that will be high enough to encode the + // content at the given rate. + for (q = rc->best_quality; q < rc->worst_quality; ++q) { + const double factor = calc_correction_factor( + av_err_per_mb, ERR_DIVISOR - ediv_size_correction, FACTOR_PT_LOW, + FACTOR_PT_HIGH, q, cpi->common.bit_depth); + const int bits_per_mb = av1_rc_bits_per_mb( + INTER_FRAME, q, factor * speed_term * group_weight_factor, + cpi->common.bit_depth); + if (bits_per_mb <= target_norm_bits_per_mb) break; + } + + // Restriction on active max q for constrained quality mode. + if (cpi->oxcf.rc_mode == AOM_CQ) q = AOMMAX(q, oxcf->cq_level); + return q; + } +} + +static void setup_rf_level_maxq(AV1_COMP *cpi) { + int i; + RATE_CONTROL *const rc = &cpi->rc; + for (i = INTER_NORMAL; i < RATE_FACTOR_LEVELS; ++i) { + int qdelta = av1_frame_type_qdelta(cpi, i, rc->worst_quality); + rc->rf_level_maxq[i] = AOMMAX(rc->worst_quality + qdelta, rc->best_quality); + } +} + +void av1_init_subsampling(AV1_COMP *cpi) { + const AV1_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; + const int w = cm->width; + const int h = cm->height; + int i; + + for (i = 0; i < FRAME_SCALE_STEPS; ++i) { + // Note: Frames with odd-sized dimensions may result from this scaling. + rc->frame_width[i] = (w * 16) / frame_scale_factor[i]; + rc->frame_height[i] = (h * 16) / frame_scale_factor[i]; + } + + setup_rf_level_maxq(cpi); +} + +void av1_calculate_coded_size(AV1_COMP *cpi, int *scaled_frame_width, + int *scaled_frame_height) { + RATE_CONTROL *const rc = &cpi->rc; + *scaled_frame_width = rc->frame_width[rc->frame_size_selector]; + *scaled_frame_height = rc->frame_height[rc->frame_size_selector]; +} + +void av1_init_second_pass(AV1_COMP *cpi) { + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + TWO_PASS *const twopass = &cpi->twopass; + double frame_rate; + FIRSTPASS_STATS *stats; + + zero_stats(&twopass->total_stats); + zero_stats(&twopass->total_left_stats); + + if (!twopass->stats_in_end) return; + + stats = &twopass->total_stats; + + *stats = *twopass->stats_in_end; + twopass->total_left_stats = *stats; + + frame_rate = 10000000.0 * stats->count / stats->duration; + // Each frame can have a different duration, as the frame rate in the source + // isn't guaranteed to be constant. The frame rate prior to the first frame + // encoded in the second pass is a guess. However, the sum duration is not. + // It is calculated based on the actual durations of all frames from the + // first pass. + av1_new_framerate(cpi, frame_rate); + twopass->bits_left = + (int64_t)(stats->duration * oxcf->target_bandwidth / 10000000.0); + + // This variable monitors how far behind the second ref update is lagging. + twopass->sr_update_lag = 1; + + // Scan the first pass file and calculate a modified total error based upon + // the bias/power function used to allocate bits. + { + const double avg_error = + stats->coded_error / DOUBLE_DIVIDE_CHECK(stats->count); + const FIRSTPASS_STATS *s = twopass->stats_in; + double modified_error_total = 0.0; + twopass->modified_error_min = + (avg_error * oxcf->two_pass_vbrmin_section) / 100; + twopass->modified_error_max = + (avg_error * oxcf->two_pass_vbrmax_section) / 100; + while (s < twopass->stats_in_end) { + modified_error_total += calculate_modified_err(cpi, twopass, oxcf, s); + ++s; + } + twopass->modified_error_left = modified_error_total; + } + + // Reset the vbr bits off target counters + cpi->rc.vbr_bits_off_target = 0; + cpi->rc.vbr_bits_off_target_fast = 0; + + cpi->rc.rate_error_estimate = 0; + + // Static sequence monitor variables. + twopass->kf_zeromotion_pct = 100; + twopass->last_kfgroup_zeromotion_pct = 100; + + if (oxcf->resize_mode != RESIZE_NONE) { + av1_init_subsampling(cpi); + } +} + +#define SR_DIFF_PART 0.0015 +#define MOTION_AMP_PART 0.003 +#define INTRA_PART 0.005 +#define DEFAULT_DECAY_LIMIT 0.75 +#define LOW_SR_DIFF_TRHESH 0.1 +#define SR_DIFF_MAX 128.0 + +static double get_sr_decay_rate(const AV1_COMP *cpi, + const FIRSTPASS_STATS *frame) { + const int num_mbs = (cpi->oxcf.resize_mode != RESIZE_NONE) ? cpi->initial_mbs + : cpi->common.MBs; + double sr_diff = (frame->sr_coded_error - frame->coded_error) / num_mbs; + double sr_decay = 1.0; + double modified_pct_inter; + double modified_pcnt_intra; + const double motion_amplitude_factor = + frame->pcnt_motion * ((frame->mvc_abs + frame->mvr_abs) / 2); + + modified_pct_inter = frame->pcnt_inter; + if ((frame->intra_error / DOUBLE_DIVIDE_CHECK(frame->coded_error)) < + (double)NCOUNT_FRAME_II_THRESH) { + modified_pct_inter = frame->pcnt_inter - frame->pcnt_neutral; + } + modified_pcnt_intra = 100 * (1.0 - modified_pct_inter); + + if ((sr_diff > LOW_SR_DIFF_TRHESH)) { + sr_diff = AOMMIN(sr_diff, SR_DIFF_MAX); + sr_decay = 1.0 - (SR_DIFF_PART * sr_diff) - + (MOTION_AMP_PART * motion_amplitude_factor) - + (INTRA_PART * modified_pcnt_intra); + } + return AOMMAX(sr_decay, AOMMIN(DEFAULT_DECAY_LIMIT, modified_pct_inter)); +} + +// This function gives an estimate of how badly we believe the prediction +// quality is decaying from frame to frame. +static double get_zero_motion_factor(const AV1_COMP *cpi, + const FIRSTPASS_STATS *frame) { + const double zero_motion_pct = frame->pcnt_inter - frame->pcnt_motion; + double sr_decay = get_sr_decay_rate(cpi, frame); + return AOMMIN(sr_decay, zero_motion_pct); +} + +#define ZM_POWER_FACTOR 0.75 + +static double get_prediction_decay_rate(const AV1_COMP *cpi, + const FIRSTPASS_STATS *next_frame) { + const double sr_decay_rate = get_sr_decay_rate(cpi, next_frame); + const double zero_motion_factor = + (0.95 * pow((next_frame->pcnt_inter - next_frame->pcnt_motion), + ZM_POWER_FACTOR)); + + return AOMMAX(zero_motion_factor, + (sr_decay_rate + ((1.0 - sr_decay_rate) * zero_motion_factor))); +} + +// Function to test for a condition where a complex transition is followed +// by a static section. For example in slide shows where there is a fade +// between slides. This is to help with more optimal kf and gf positioning. +static int detect_transition_to_still(AV1_COMP *cpi, int frame_interval, + int still_interval, + double loop_decay_rate, + double last_decay_rate) { + TWO_PASS *const twopass = &cpi->twopass; + RATE_CONTROL *const rc = &cpi->rc; + + // Break clause to detect very still sections after motion + // For example a static image after a fade or other transition + // instead of a clean scene cut. + if (frame_interval > rc->min_gf_interval && loop_decay_rate >= 0.999 && + last_decay_rate < 0.9) { + int j; + + // Look ahead a few frames to see if static condition persists... + for (j = 0; j < still_interval; ++j) { + const FIRSTPASS_STATS *stats = &twopass->stats_in[j]; + if (stats >= twopass->stats_in_end) break; + + if (stats->pcnt_inter - stats->pcnt_motion < 0.999) break; + } + + // Only if it does do we signal a transition to still. + return j == still_interval; + } + + return 0; +} + +// This function detects a flash through the high relative pcnt_second_ref +// score in the frame following a flash frame. The offset passed in should +// reflect this. +static int detect_flash(const TWO_PASS *twopass, int offset) { + const FIRSTPASS_STATS *const next_frame = read_frame_stats(twopass, offset); + + // What we are looking for here is a situation where there is a + // brief break in prediction (such as a flash) but subsequent frames + // are reasonably well predicted by an earlier (pre flash) frame. + // The recovery after a flash is indicated by a high pcnt_second_ref + // compared to pcnt_inter. + return next_frame != NULL && + next_frame->pcnt_second_ref > next_frame->pcnt_inter && + next_frame->pcnt_second_ref >= 0.5; +} + +// Update the motion related elements to the GF arf boost calculation. +static void accumulate_frame_motion_stats(const FIRSTPASS_STATS *stats, + double *mv_in_out, + double *mv_in_out_accumulator, + double *abs_mv_in_out_accumulator, + double *mv_ratio_accumulator) { + const double pct = stats->pcnt_motion; + + // Accumulate Motion In/Out of frame stats. + *mv_in_out = stats->mv_in_out_count * pct; + *mv_in_out_accumulator += *mv_in_out; + *abs_mv_in_out_accumulator += fabs(*mv_in_out); + + // Accumulate a measure of how uniform (or conversely how random) the motion + // field is (a ratio of abs(mv) / mv). + if (pct > 0.05) { + const double mvr_ratio = + fabs(stats->mvr_abs) / DOUBLE_DIVIDE_CHECK(fabs(stats->MVr)); + const double mvc_ratio = + fabs(stats->mvc_abs) / DOUBLE_DIVIDE_CHECK(fabs(stats->MVc)); + + *mv_ratio_accumulator += + pct * (mvr_ratio < stats->mvr_abs ? mvr_ratio : stats->mvr_abs); + *mv_ratio_accumulator += + pct * (mvc_ratio < stats->mvc_abs ? mvc_ratio : stats->mvc_abs); + } +} + +#define BASELINE_ERR_PER_MB 1000.0 +static double calc_frame_boost(AV1_COMP *cpi, const FIRSTPASS_STATS *this_frame, + double this_frame_mv_in_out, double max_boost) { + double frame_boost; + const double lq = av1_convert_qindex_to_q( + cpi->rc.avg_frame_qindex[INTER_FRAME], cpi->common.bit_depth); + const double boost_q_correction = AOMMIN((0.5 + (lq * 0.015)), 1.5); + int num_mbs = (cpi->oxcf.resize_mode != RESIZE_NONE) ? cpi->initial_mbs + : cpi->common.MBs; + + // Correct for any inactive region in the image + num_mbs = (int)AOMMAX(1, num_mbs * calculate_active_area(cpi, this_frame)); + + // Underlying boost factor is based on inter error ratio. + frame_boost = (BASELINE_ERR_PER_MB * num_mbs) / + DOUBLE_DIVIDE_CHECK(this_frame->coded_error); + frame_boost = frame_boost * BOOST_FACTOR * boost_q_correction; + + // Increase boost for frames where new data coming into frame (e.g. zoom out). + // Slightly reduce boost if there is a net balance of motion out of the frame + // (zoom in). The range for this_frame_mv_in_out is -1.0 to +1.0. + if (this_frame_mv_in_out > 0.0) + frame_boost += frame_boost * (this_frame_mv_in_out * 2.0); + // In the extreme case the boost is halved. + else + frame_boost += frame_boost * (this_frame_mv_in_out / 2.0); + + return AOMMIN(frame_boost, max_boost * boost_q_correction); +} + +static int calc_arf_boost(AV1_COMP *cpi, int offset, int f_frames, int b_frames, + int *f_boost, int *b_boost) { + TWO_PASS *const twopass = &cpi->twopass; + int i; + double boost_score = 0.0; + double mv_ratio_accumulator = 0.0; + double decay_accumulator = 1.0; + double this_frame_mv_in_out = 0.0; + double mv_in_out_accumulator = 0.0; + double abs_mv_in_out_accumulator = 0.0; + int arf_boost; + int flash_detected = 0; + + // Search forward from the proposed arf/next gf position. + for (i = 0; i < f_frames; ++i) { + const FIRSTPASS_STATS *this_frame = read_frame_stats(twopass, i + offset); + if (this_frame == NULL) break; + + // Update the motion related elements to the boost calculation. + accumulate_frame_motion_stats( + this_frame, &this_frame_mv_in_out, &mv_in_out_accumulator, + &abs_mv_in_out_accumulator, &mv_ratio_accumulator); + + // We want to discount the flash frame itself and the recovery + // frame that follows as both will have poor scores. + flash_detected = detect_flash(twopass, i + offset) || + detect_flash(twopass, i + offset + 1); + + // Accumulate the effect of prediction quality decay. + if (!flash_detected) { + decay_accumulator *= get_prediction_decay_rate(cpi, this_frame); + decay_accumulator = decay_accumulator < MIN_DECAY_FACTOR + ? MIN_DECAY_FACTOR + : decay_accumulator; + } + + boost_score += + decay_accumulator * + calc_frame_boost(cpi, this_frame, this_frame_mv_in_out, GF_MAX_BOOST); + } + + *f_boost = (int)boost_score; + + // Reset for backward looking loop. + boost_score = 0.0; + mv_ratio_accumulator = 0.0; + decay_accumulator = 1.0; + this_frame_mv_in_out = 0.0; + mv_in_out_accumulator = 0.0; + abs_mv_in_out_accumulator = 0.0; + + // Search backward towards last gf position. + for (i = -1; i >= -b_frames; --i) { + const FIRSTPASS_STATS *this_frame = read_frame_stats(twopass, i + offset); + if (this_frame == NULL) break; + + // Update the motion related elements to the boost calculation. + accumulate_frame_motion_stats( + this_frame, &this_frame_mv_in_out, &mv_in_out_accumulator, + &abs_mv_in_out_accumulator, &mv_ratio_accumulator); + + // We want to discount the the flash frame itself and the recovery + // frame that follows as both will have poor scores. + flash_detected = detect_flash(twopass, i + offset) || + detect_flash(twopass, i + offset + 1); + + // Cumulative effect of prediction quality decay. + if (!flash_detected) { + decay_accumulator *= get_prediction_decay_rate(cpi, this_frame); + decay_accumulator = decay_accumulator < MIN_DECAY_FACTOR + ? MIN_DECAY_FACTOR + : decay_accumulator; + } + + boost_score += + decay_accumulator * + calc_frame_boost(cpi, this_frame, this_frame_mv_in_out, GF_MAX_BOOST); + } + *b_boost = (int)boost_score; + + arf_boost = (*f_boost + *b_boost); + if (arf_boost < ((b_frames + f_frames) * 20)) + arf_boost = ((b_frames + f_frames) * 20); + arf_boost = AOMMAX(arf_boost, MIN_ARF_GF_BOOST); + + return arf_boost; +} + +// Calculate a section intra ratio used in setting max loop filter. +static int calculate_section_intra_ratio(const FIRSTPASS_STATS *begin, + const FIRSTPASS_STATS *end, + int section_length) { + const FIRSTPASS_STATS *s = begin; + double intra_error = 0.0; + double coded_error = 0.0; + int i = 0; + + while (s < end && i < section_length) { + intra_error += s->intra_error; + coded_error += s->coded_error; + ++s; + ++i; + } + + return (int)(intra_error / DOUBLE_DIVIDE_CHECK(coded_error)); +} + +// Calculate the total bits to allocate in this GF/ARF group. +static int64_t calculate_total_gf_group_bits(AV1_COMP *cpi, + double gf_group_err) { + const RATE_CONTROL *const rc = &cpi->rc; + const TWO_PASS *const twopass = &cpi->twopass; + const int max_bits = frame_max_bits(rc, &cpi->oxcf); + int64_t total_group_bits; + + // Calculate the bits to be allocated to the group as a whole. + if ((twopass->kf_group_bits > 0) && (twopass->kf_group_error_left > 0)) { + total_group_bits = (int64_t)(twopass->kf_group_bits * + (gf_group_err / twopass->kf_group_error_left)); + } else { + total_group_bits = 0; + } + + // Clamp odd edge cases. + total_group_bits = (total_group_bits < 0) + ? 0 + : (total_group_bits > twopass->kf_group_bits) + ? twopass->kf_group_bits + : total_group_bits; + + // Clip based on user supplied data rate variability limit. + if (total_group_bits > (int64_t)max_bits * rc->baseline_gf_interval) + total_group_bits = (int64_t)max_bits * rc->baseline_gf_interval; + + return total_group_bits; +} + +// Calculate the number bits extra to assign to boosted frames in a group. +static int calculate_boost_bits(int frame_count, int boost, + int64_t total_group_bits) { + int allocation_chunks; + + // return 0 for invalid inputs (could arise e.g. through rounding errors) + if (!boost || (total_group_bits <= 0) || (frame_count <= 0)) return 0; + + allocation_chunks = (frame_count * 100) + boost; + + // Prevent overflow. + if (boost > 1023) { + int divisor = boost >> 10; + boost /= divisor; + allocation_chunks /= divisor; + } + + // Calculate the number of extra bits for use in the boosted frame or frames. + return AOMMAX((int)(((int64_t)boost * total_group_bits) / allocation_chunks), + 0); +} + +#if !CONFIG_EXT_REFS +// Current limit on maximum number of active arfs in a GF/ARF group. +#define MAX_ACTIVE_ARFS 2 +#define ARF_SLOT1 2 +#define ARF_SLOT2 3 +// This function indirects the choice of buffers for arfs. +// At the moment the values are fixed but this may change as part of +// the integration process with other codec features that swap buffers around. +static void get_arf_buffer_indices(unsigned char *arf_buffer_indices) { + arf_buffer_indices[0] = ARF_SLOT1; + arf_buffer_indices[1] = ARF_SLOT2; +} +#endif + +static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits, + double group_error, int gf_arf_bits) { + RATE_CONTROL *const rc = &cpi->rc; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + TWO_PASS *const twopass = &cpi->twopass; + GF_GROUP *const gf_group = &twopass->gf_group; + FIRSTPASS_STATS frame_stats; + int i; + int frame_index = 0; + int target_frame_size; + int key_frame; + const int max_bits = frame_max_bits(&cpi->rc, &cpi->oxcf); + int64_t total_group_bits = gf_group_bits; + double modified_err = 0.0; + double err_fraction; + int mid_boost_bits = 0; +#if CONFIG_EXT_REFS + // The use of bi-predictive frames are only enabled when following 3 + // conditions are met: + // (1) Alt-ref is enabled; + // (2) The bi-predictive group interval is at least 2; and + // (3) The bi-predictive group interval is strictly smaller than the + // golden group interval. + const int is_bipred_enabled = + rc->source_alt_ref_pending && rc->bipred_group_interval && + rc->bipred_group_interval <= + (rc->baseline_gf_interval - rc->source_alt_ref_pending); + int bipred_group_end = 0; + int bipred_frame_index = 0; + + int arf_pos[MAX_EXT_ARFS + 1]; + const unsigned char ext_arf_interval = + (unsigned char)(rc->baseline_gf_interval / (cpi->num_extra_arfs + 1) - 1); + int which_arf = cpi->num_extra_arfs; + int subgroup_interval[MAX_EXT_ARFS + 1]; + int ext_arf_boost[MAX_EXT_ARFS]; + int is_sg_bipred_enabled = is_bipred_enabled; + int accumulative_subgroup_interval = 0; +#else + int mid_frame_idx; + unsigned char arf_buffer_indices[MAX_ACTIVE_ARFS]; +#endif // CONFIG_EXT_REFS + +#if CONFIG_EXT_REFS + av1_zero_array(ext_arf_boost, MAX_EXT_ARFS); +#endif // CONFIG_EXT_REFS + + key_frame = cpi->common.frame_type == KEY_FRAME; + +#if !CONFIG_EXT_REFS + get_arf_buffer_indices(arf_buffer_indices); +#endif // !CONFIG_EXT_REFS + + // For key frames the frame target rate is already set and it + // is also the golden frame. + if (!key_frame) { + if (rc->source_alt_ref_active) { + gf_group->update_type[frame_index] = OVERLAY_UPDATE; + gf_group->rf_level[frame_index] = INTER_NORMAL; + gf_group->bit_allocation[frame_index] = 0; + } else { + gf_group->update_type[frame_index] = GF_UPDATE; + gf_group->rf_level[frame_index] = GF_ARF_STD; + gf_group->bit_allocation[frame_index] = gf_arf_bits; + } +#if CONFIG_EXT_REFS + gf_group->arf_update_idx[frame_index] = 0; + gf_group->arf_ref_idx[frame_index] = 0; +#else + gf_group->arf_update_idx[frame_index] = arf_buffer_indices[0]; + gf_group->arf_ref_idx[frame_index] = arf_buffer_indices[0]; +#endif // CONFIG_EXT_REFS + // Step over the golden frame / overlay frame + if (EOF == input_stats(twopass, &frame_stats)) return; + } + +#if CONFIG_EXT_REFS + gf_group->bidir_pred_enabled[frame_index] = 0; + gf_group->brf_src_offset[frame_index] = 0; +#endif // CONFIG_EXT_REFS + + // Deduct the boost bits for arf (or gf if it is not a key frame) + // from the group total. + if (rc->source_alt_ref_pending || !key_frame) total_group_bits -= gf_arf_bits; + + frame_index++; + +#if CONFIG_EXT_REFS + bipred_frame_index++; +#endif // CONFIG_EXT_REFS + + // Store the bits to spend on the ARF if there is one. + if (rc->source_alt_ref_pending) { + gf_group->update_type[frame_index] = ARF_UPDATE; + gf_group->rf_level[frame_index] = GF_ARF_STD; + gf_group->bit_allocation[frame_index] = gf_arf_bits; + + gf_group->arf_src_offset[frame_index] = + (unsigned char)(rc->baseline_gf_interval - 1); + +#if CONFIG_EXT_REFS + gf_group->arf_update_idx[frame_index] = 0; + gf_group->arf_ref_idx[frame_index] = 0; + + gf_group->bidir_pred_enabled[frame_index] = 0; + gf_group->brf_src_offset[frame_index] = 0; +// NOTE: "bidir_pred_frame_index" stays unchanged for ARF_UPDATE frames. +#else + gf_group->arf_update_idx[frame_index] = arf_buffer_indices[0]; + gf_group->arf_ref_idx[frame_index] = + arf_buffer_indices[cpi->multi_arf_last_grp_enabled && + rc->source_alt_ref_active]; +#endif // CONFIG_EXT_REFS + +#if CONFIG_EXT_REFS + // Work out the ARFs' positions in this gf group + // NOTE(weitinglin): ALT_REFs' are indexed inversely, but coded in display + // order (except for the original ARF). In the example of three ALT_REF's, + // We index ALTREF's as: KEY ----- ALT2 ----- ALT1 ----- ALT0 + // but code them in the following order: + // KEY-ALT0-ALT2 ----- OVERLAY2-ALT1 ----- OVERLAY1 ----- OVERLAY0 + arf_pos[0] = + frame_index + cpi->num_extra_arfs + gf_group->arf_src_offset[1] + 1; + for (i = 0; i < cpi->num_extra_arfs; ++i) { + arf_pos[i + 1] = + frame_index + (cpi->num_extra_arfs - i) * (ext_arf_interval + 2); + subgroup_interval[i] = arf_pos[i] - arf_pos[i + 1] - (i == 0 ? 1 : 2); + } + subgroup_interval[cpi->num_extra_arfs] = arf_pos[cpi->num_extra_arfs] - + frame_index - + (cpi->num_extra_arfs == 0 ? 1 : 2); +#endif // CONFIG_EXT_REFS + + ++frame_index; + +#if CONFIG_EXT_REFS + // Insert an extra ARF + if (cpi->num_extra_arfs) { + gf_group->update_type[frame_index] = ARF_UPDATE; + // Note (weitinglin): GF_ARF_LOW is also used as an identifier + // for internal ALT_REF's: + gf_group->rf_level[frame_index] = GF_ARF_LOW; + gf_group->arf_src_offset[frame_index] = ext_arf_interval; + gf_group->arf_update_idx[frame_index] = which_arf; + gf_group->arf_ref_idx[frame_index] = 0; + ++frame_index; + } + accumulative_subgroup_interval += subgroup_interval[cpi->num_extra_arfs]; +#else + if (cpi->multi_arf_enabled) { + // Set aside a slot for a level 1 arf. + gf_group->update_type[frame_index] = ARF_UPDATE; + gf_group->rf_level[frame_index] = GF_ARF_LOW; + gf_group->arf_src_offset[frame_index] = + (unsigned char)((rc->baseline_gf_interval >> 1) - 1); + gf_group->arf_update_idx[frame_index] = arf_buffer_indices[1]; + gf_group->arf_ref_idx[frame_index] = arf_buffer_indices[0]; + ++frame_index; + } +#endif // CONFIG_EXT_ARFS + } + +#if !CONFIG_EXT_REFS + // Define middle frame + mid_frame_idx = frame_index + (rc->baseline_gf_interval >> 1) - 1; +#endif // !CONFIG_EXT_REFS + + // Allocate bits to the other frames in the group. + for (i = 0; i < rc->baseline_gf_interval - rc->source_alt_ref_pending; ++i) { +#if !CONFIG_EXT_REFS + int arf_idx = 0; +#endif // !CONFIG_EXT_REFS + + if (EOF == input_stats(twopass, &frame_stats)) break; + + modified_err = calculate_modified_err(cpi, twopass, oxcf, &frame_stats); + + if (group_error > 0) + err_fraction = modified_err / DOUBLE_DIVIDE_CHECK(group_error); + else + err_fraction = 0.0; + + target_frame_size = (int)((double)total_group_bits * err_fraction); + + if (rc->source_alt_ref_pending && cpi->multi_arf_enabled) { + mid_boost_bits += (target_frame_size >> 4); + target_frame_size -= (target_frame_size >> 4); +#if !CONFIG_EXT_REFS + if (frame_index <= mid_frame_idx) arf_idx = 1; +#endif // !CONFIG_EXT_REFS + } + +#if CONFIG_EXT_REFS + gf_group->arf_update_idx[frame_index] = which_arf; + gf_group->arf_ref_idx[frame_index] = which_arf; +#else + gf_group->arf_update_idx[frame_index] = arf_buffer_indices[arf_idx]; + gf_group->arf_ref_idx[frame_index] = arf_buffer_indices[arf_idx]; +#endif // CONFIG_EXT_REFS + + target_frame_size = + clamp(target_frame_size, 0, AOMMIN(max_bits, (int)total_group_bits)); + +#if CONFIG_EXT_REFS + // If we are going to have ARFs, check if we can have BWDREF in this + // subgroup. + if (rc->source_alt_ref_pending) { + is_sg_bipred_enabled = + is_bipred_enabled && + (subgroup_interval[which_arf] > rc->bipred_group_interval); + } + + // NOTE: BIDIR_PRED is only enabled when the length of the bi-predictive + // frame group interval is strictly smaller than that of the GOLDEN + // FRAME group interval. + // TODO(zoeliu): Currently BIDIR_PRED is only enabled when alt-ref is on. + if (is_sg_bipred_enabled && !bipred_group_end) { + const int cur_brf_src_offset = rc->bipred_group_interval - 1; + + // --- BRF_UPDATE --- + if (bipred_frame_index == 1) { + gf_group->update_type[frame_index] = BRF_UPDATE; + gf_group->bidir_pred_enabled[frame_index] = 1; + gf_group->brf_src_offset[frame_index] = cur_brf_src_offset; + // --- LAST_BIPRED_UPDATE --- + } else if (bipred_frame_index == rc->bipred_group_interval) { + gf_group->update_type[frame_index] = LAST_BIPRED_UPDATE; + gf_group->bidir_pred_enabled[frame_index] = 1; + gf_group->brf_src_offset[frame_index] = 0; + // Reset the bi-predictive frame index. + bipred_frame_index = 0; + // --- BIPRED_UPDATE --- + } else { + gf_group->update_type[frame_index] = BIPRED_UPDATE; + gf_group->bidir_pred_enabled[frame_index] = 1; + gf_group->brf_src_offset[frame_index] = 0; + } + + bipred_frame_index++; + // Check whether the next bi-predictive frame group would entirely be + // included within the current golden frame group. + // In addition, we need to avoid coding a BRF right before an ARF. + if (bipred_frame_index == 1 && + (i + 2 + cur_brf_src_offset) >= accumulative_subgroup_interval) { + bipred_group_end = 1; + } + } else { +#endif // CONFIG_EXT_REFS + gf_group->update_type[frame_index] = LF_UPDATE; +#if CONFIG_EXT_REFS + gf_group->bidir_pred_enabled[frame_index] = 0; + gf_group->brf_src_offset[frame_index] = 0; + } +#endif // CONFIG_EXT_REFS + +#if CONFIG_EXT_REFS + if (gf_group->update_type[frame_index] == BRF_UPDATE) { + // Boost up the allocated bits on BWDREF_FRAME + gf_group->rf_level[frame_index] = GF_ARF_LOW; + gf_group->bit_allocation[frame_index] = + target_frame_size + (target_frame_size >> 2); + } else if (gf_group->update_type[frame_index] == LAST_BIPRED_UPDATE) { + // Press down the allocated bits on LAST_BIPRED_UPDATE frames + gf_group->rf_level[frame_index] = INTER_NORMAL; + gf_group->bit_allocation[frame_index] = + target_frame_size - (target_frame_size >> 1); + } else if (gf_group->update_type[frame_index] == BIPRED_UPDATE) { + // TODO(zoeliu): To investigate whether the allocated bits on + // BIPRED_UPDATE frames need to be further adjusted. + gf_group->rf_level[frame_index] = INTER_NORMAL; + gf_group->bit_allocation[frame_index] = target_frame_size; + } else { +#endif // CONFIG_EXT_REFS + gf_group->rf_level[frame_index] = INTER_NORMAL; + gf_group->bit_allocation[frame_index] = target_frame_size; +#if CONFIG_EXT_REFS + } +#endif // CONFIG_EXT_REFS + + ++frame_index; + +#if CONFIG_EXT_REFS + // Check if we need to update the ARF + if (is_sg_bipred_enabled && cpi->num_extra_arfs && which_arf > 0 && + frame_index > arf_pos[which_arf]) { + --which_arf; + accumulative_subgroup_interval += subgroup_interval[which_arf] + 1; + // Meet the new subgroup. Reset the bipred_group_end flag; + bipred_group_end = 0; + // Insert another extra ARF after the overlay frame + if (which_arf) { + gf_group->update_type[frame_index] = ARF_UPDATE; + gf_group->rf_level[frame_index] = GF_ARF_LOW; + gf_group->arf_src_offset[frame_index] = ext_arf_interval; + gf_group->arf_update_idx[frame_index] = which_arf; + gf_group->arf_ref_idx[frame_index] = 0; + ++frame_index; + } + } +#endif // CONFIG_EXT_REFS + } + +// Note: +// We need to configure the frame at the end of the sequence + 1 that will be +// the start frame for the next group. Otherwise prior to the call to +// av1_rc_get_second_pass_params() the data will be undefined. +#if CONFIG_EXT_REFS + gf_group->arf_update_idx[frame_index] = 0; + gf_group->arf_ref_idx[frame_index] = 0; +#else + gf_group->arf_update_idx[frame_index] = arf_buffer_indices[0]; + gf_group->arf_ref_idx[frame_index] = arf_buffer_indices[0]; +#endif // CONFIG_EXT_REFS + + if (rc->source_alt_ref_pending) { + gf_group->update_type[frame_index] = OVERLAY_UPDATE; + gf_group->rf_level[frame_index] = INTER_NORMAL; + +#if CONFIG_EXT_REFS + if (cpi->num_extra_arfs) { + for (i = cpi->num_extra_arfs; i > 0; --i) { + int arf_pos_in_gf = (i == cpi->num_extra_arfs ? 2 : arf_pos[i + 1] + 1); + gf_group->bit_allocation[arf_pos_in_gf] = + gf_group->bit_allocation[arf_pos[i]]; + gf_group->update_type[arf_pos[i]] = INTNL_OVERLAY_UPDATE; + gf_group->bit_allocation[arf_pos[i]] = 0; + gf_group->rf_level[arf_pos[i]] = INTER_NORMAL; + } + } +#else + // Final setup for second arf and its overlay. + if (cpi->multi_arf_enabled) { + gf_group->bit_allocation[2] = + gf_group->bit_allocation[mid_frame_idx] + mid_boost_bits; + gf_group->update_type[mid_frame_idx] = OVERLAY_UPDATE; + gf_group->bit_allocation[mid_frame_idx] = 0; + } +#endif // CONFIG_EXT_REFS + } else { + gf_group->update_type[frame_index] = GF_UPDATE; + gf_group->rf_level[frame_index] = GF_ARF_STD; + } + +#if CONFIG_EXT_REFS + gf_group->bidir_pred_enabled[frame_index] = 0; + gf_group->brf_src_offset[frame_index] = 0; +#endif // CONFIG_EXT_REFS + + // Note whether multi-arf was enabled this group for next time. + cpi->multi_arf_last_grp_enabled = cpi->multi_arf_enabled; +} + +// Analyse and define a gf/arf group. +static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { + AV1_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; + AV1EncoderConfig *const oxcf = &cpi->oxcf; + TWO_PASS *const twopass = &cpi->twopass; + FIRSTPASS_STATS next_frame; + const FIRSTPASS_STATS *const start_pos = twopass->stats_in; + int i; + + double boost_score = 0.0; + double old_boost_score = 0.0; + double gf_group_err = 0.0; +#if GROUP_ADAPTIVE_MAXQ + double gf_group_raw_error = 0.0; +#endif + double gf_group_skip_pct = 0.0; + double gf_group_inactive_zone_rows = 0.0; + double gf_first_frame_err = 0.0; + double mod_frame_err = 0.0; + + double mv_ratio_accumulator = 0.0; + double decay_accumulator = 1.0; + double zero_motion_accumulator = 1.0; + + double loop_decay_rate = 1.00; + double last_loop_decay_rate = 1.00; + + double this_frame_mv_in_out = 0.0; + double mv_in_out_accumulator = 0.0; + double abs_mv_in_out_accumulator = 0.0; + double mv_ratio_accumulator_thresh; + unsigned int allow_alt_ref = is_altref_enabled(cpi); + + int f_boost = 0; + int b_boost = 0; + int flash_detected; + int active_max_gf_interval; + int active_min_gf_interval; + int64_t gf_group_bits; + double gf_group_error_left; + int gf_arf_bits; + const int is_key_frame = frame_is_intra_only(cm); + const int arf_active_or_kf = is_key_frame || rc->source_alt_ref_active; + + // Reset the GF group data structures unless this is a key + // frame in which case it will already have been done. + if (is_key_frame == 0) { + av1_zero(twopass->gf_group); + } + + aom_clear_system_state(); + av1_zero(next_frame); + + // Load stats for the current frame. + mod_frame_err = calculate_modified_err(cpi, twopass, oxcf, this_frame); + + // Note the error of the frame at the start of the group. This will be + // the GF frame error if we code a normal gf. + gf_first_frame_err = mod_frame_err; + + // If this is a key frame or the overlay from a previous arf then + // the error score / cost of this frame has already been accounted for. + if (arf_active_or_kf) { + gf_group_err -= gf_first_frame_err; +#if GROUP_ADAPTIVE_MAXQ + gf_group_raw_error -= this_frame->coded_error; +#endif + gf_group_skip_pct -= this_frame->intra_skip_pct; + gf_group_inactive_zone_rows -= this_frame->inactive_zone_rows; + } + + // Motion breakout threshold for loop below depends on image size. + mv_ratio_accumulator_thresh = + (cpi->initial_height + cpi->initial_width) / 4.0; + + // Set a maximum and minimum interval for the GF group. + // If the image appears almost completely static we can extend beyond this. + { + int int_max_q = (int)(av1_convert_qindex_to_q(twopass->active_worst_quality, + cpi->common.bit_depth)); + int int_lbq = (int)(av1_convert_qindex_to_q(rc->last_boosted_qindex, + cpi->common.bit_depth)); + + active_min_gf_interval = rc->min_gf_interval + AOMMIN(2, int_max_q / 200); + if (active_min_gf_interval > rc->max_gf_interval) + active_min_gf_interval = rc->max_gf_interval; + + if (cpi->multi_arf_allowed) { + active_max_gf_interval = rc->max_gf_interval; + } else { + // The value chosen depends on the active Q range. At low Q we have + // bits to spare and are better with a smaller interval and smaller boost. + // At high Q when there are few bits to spare we are better with a longer + // interval to spread the cost of the GF. + active_max_gf_interval = 12 + AOMMIN(4, (int_lbq / 6)); + + // We have: active_min_gf_interval <= rc->max_gf_interval + if (active_max_gf_interval < active_min_gf_interval) + active_max_gf_interval = active_min_gf_interval; + else if (active_max_gf_interval > rc->max_gf_interval) + active_max_gf_interval = rc->max_gf_interval; + } + } + + i = 0; + while (i < rc->static_scene_max_gf_interval && i < rc->frames_to_key) { + ++i; + + // Accumulate error score of frames in this gf group. + mod_frame_err = calculate_modified_err(cpi, twopass, oxcf, this_frame); + gf_group_err += mod_frame_err; +#if GROUP_ADAPTIVE_MAXQ + gf_group_raw_error += this_frame->coded_error; +#endif + gf_group_skip_pct += this_frame->intra_skip_pct; + gf_group_inactive_zone_rows += this_frame->inactive_zone_rows; + + if (EOF == input_stats(twopass, &next_frame)) break; + + // Test for the case where there is a brief flash but the prediction + // quality back to an earlier frame is then restored. + flash_detected = detect_flash(twopass, 0); + + // Update the motion related elements to the boost calculation. + accumulate_frame_motion_stats( + &next_frame, &this_frame_mv_in_out, &mv_in_out_accumulator, + &abs_mv_in_out_accumulator, &mv_ratio_accumulator); + + // Accumulate the effect of prediction quality decay. + if (!flash_detected) { + last_loop_decay_rate = loop_decay_rate; + loop_decay_rate = get_prediction_decay_rate(cpi, &next_frame); + + decay_accumulator = decay_accumulator * loop_decay_rate; + + // Monitor for static sections. + zero_motion_accumulator = AOMMIN( + zero_motion_accumulator, get_zero_motion_factor(cpi, &next_frame)); + + // Break clause to detect very still sections after motion. For example, + // a static image after a fade or other transition. + if (detect_transition_to_still(cpi, i, 5, loop_decay_rate, + last_loop_decay_rate)) { + allow_alt_ref = 0; + break; + } + } + + // Calculate a boost number for this frame. + boost_score += + decay_accumulator * + calc_frame_boost(cpi, &next_frame, this_frame_mv_in_out, GF_MAX_BOOST); + + // Break out conditions. + if ( + // Break at active_max_gf_interval unless almost totally static. + (i >= (active_max_gf_interval + arf_active_or_kf) && + zero_motion_accumulator < 0.995) || + ( + // Don't break out with a very short interval. + (i >= active_min_gf_interval + arf_active_or_kf) && + (!flash_detected) && + ((mv_ratio_accumulator > mv_ratio_accumulator_thresh) || + (abs_mv_in_out_accumulator > 3.0) || + (mv_in_out_accumulator < -2.0) || + ((boost_score - old_boost_score) < BOOST_BREAKOUT)))) { + boost_score = old_boost_score; + break; + } + + *this_frame = next_frame; + old_boost_score = boost_score; + } + + twopass->gf_zeromotion_pct = (int)(zero_motion_accumulator * 1000.0); + + // Was the group length constrained by the requirement for a new KF? + rc->constrained_gf_group = (i >= rc->frames_to_key) ? 1 : 0; + + // Should we use the alternate reference frame. + if (allow_alt_ref && (i < cpi->oxcf.lag_in_frames) && + (i >= rc->min_gf_interval)) { + // Calculate the boost for alt ref. + rc->gfu_boost = + calc_arf_boost(cpi, 0, (i - 1), (i - 1), &f_boost, &b_boost); + rc->source_alt_ref_pending = 1; + + // Test to see if multi arf is appropriate. + cpi->multi_arf_enabled = + (cpi->multi_arf_allowed && (rc->baseline_gf_interval >= 6) && + (zero_motion_accumulator < 0.995)) + ? 1 + : 0; + } else { + rc->gfu_boost = AOMMAX((int)boost_score, MIN_ARF_GF_BOOST); + rc->source_alt_ref_pending = 0; + } + + // Set the interval until the next gf. + rc->baseline_gf_interval = i - (is_key_frame || rc->source_alt_ref_pending); + +#if CONFIG_EXT_REFS + // Compute how many extra alt_refs we can have + cpi->num_extra_arfs = get_number_of_extra_arfs(rc->baseline_gf_interval, + rc->source_alt_ref_pending); + // Currently at maximum two extra ARFs' are allowed + assert(cpi->num_extra_arfs <= MAX_EXT_ARFS); +#endif // CONFIG_EXT_REFS + + rc->frames_till_gf_update_due = rc->baseline_gf_interval; + +#if CONFIG_EXT_REFS + rc->bipred_group_interval = BFG_INTERVAL; + // The minimum bi-predictive frame group interval is 2. + if (rc->bipred_group_interval < 2) rc->bipred_group_interval = 0; +#endif // CONFIG_EXT_REFS + + // Reset the file position. + reset_fpf_position(twopass, start_pos); + + // Calculate the bits to be allocated to the gf/arf group as a whole + gf_group_bits = calculate_total_gf_group_bits(cpi, gf_group_err); + +#if GROUP_ADAPTIVE_MAXQ + // Calculate an estimate of the maxq needed for the group. + // We are more agressive about correcting for sections + // where there could be significant overshoot than for easier + // sections where we do not wish to risk creating an overshoot + // of the allocated bit budget. + if ((cpi->oxcf.rc_mode != AOM_Q) && (rc->baseline_gf_interval > 1)) { + const int vbr_group_bits_per_frame = + (int)(gf_group_bits / rc->baseline_gf_interval); + const double group_av_err = gf_group_raw_error / rc->baseline_gf_interval; + const double group_av_skip_pct = + gf_group_skip_pct / rc->baseline_gf_interval; + const double group_av_inactive_zone = + ((gf_group_inactive_zone_rows * 2) / + (rc->baseline_gf_interval * (double)cm->mb_rows)); + + int tmp_q; + // rc factor is a weight factor that corrects for local rate control drift. + double rc_factor = 1.0; + if (rc->rate_error_estimate > 0) { + rc_factor = AOMMAX(RC_FACTOR_MIN, + (double)(100 - rc->rate_error_estimate) / 100.0); + } else { + rc_factor = AOMMIN(RC_FACTOR_MAX, + (double)(100 - rc->rate_error_estimate) / 100.0); + } + tmp_q = get_twopass_worst_quality( + cpi, group_av_err, (group_av_skip_pct + group_av_inactive_zone), + vbr_group_bits_per_frame, twopass->kfgroup_inter_fraction * rc_factor); + twopass->active_worst_quality = + AOMMAX(tmp_q, twopass->active_worst_quality >> 1); + } +#endif + + // Calculate the extra bits to be used for boosted frame(s) + gf_arf_bits = calculate_boost_bits(rc->baseline_gf_interval, rc->gfu_boost, + gf_group_bits); + + // Adjust KF group bits and error remaining. + twopass->kf_group_error_left -= (int64_t)gf_group_err; + + // If this is an arf update we want to remove the score for the overlay + // frame at the end which will usually be very cheap to code. + // The overlay frame has already, in effect, been coded so we want to spread + // the remaining bits among the other frames. + // For normal GFs remove the score for the GF itself unless this is + // also a key frame in which case it has already been accounted for. + if (rc->source_alt_ref_pending) { + gf_group_error_left = gf_group_err - mod_frame_err; + } else if (is_key_frame == 0) { + gf_group_error_left = gf_group_err - gf_first_frame_err; + } else { + gf_group_error_left = gf_group_err; + } + + // Allocate bits to each of the frames in the GF group. + allocate_gf_group_bits(cpi, gf_group_bits, gf_group_error_left, gf_arf_bits); + + // Reset the file position. + reset_fpf_position(twopass, start_pos); + + // Calculate a section intra ratio used in setting max loop filter. + if (cpi->common.frame_type != KEY_FRAME) { + twopass->section_intra_rating = calculate_section_intra_ratio( + start_pos, twopass->stats_in_end, rc->baseline_gf_interval); + } + + if (oxcf->resize_mode == RESIZE_DYNAMIC) { + // Default to starting GF groups at normal frame size. + cpi->rc.next_frame_size_selector = UNSCALED; + } +} + +// Threshold for use of the lagging second reference frame. High second ref +// usage may point to a transient event like a flash or occlusion rather than +// a real scene cut. +#define SECOND_REF_USEAGE_THRESH 0.1 +// Minimum % intra coding observed in first pass (1.0 = 100%) +#define MIN_INTRA_LEVEL 0.25 +// Minimum ratio between the % of intra coding and inter coding in the first +// pass after discounting neutral blocks (discounting neutral blocks in this +// way helps catch scene cuts in clips with very flat areas or letter box +// format clips with image padding. +#define INTRA_VS_INTER_THRESH 2.0 +// Hard threshold where the first pass chooses intra for almost all blocks. +// In such a case even if the frame is not a scene cut coding a key frame +// may be a good option. +#define VERY_LOW_INTER_THRESH 0.05 +// Maximum threshold for the relative ratio of intra error score vs best +// inter error score. +#define KF_II_ERR_THRESHOLD 2.5 +// In real scene cuts there is almost always a sharp change in the intra +// or inter error score. +#define ERR_CHANGE_THRESHOLD 0.4 +// For real scene cuts we expect an improvment in the intra inter error +// ratio in the next frame. +#define II_IMPROVEMENT_THRESHOLD 3.5 +#define KF_II_MAX 128.0 + +static int test_candidate_kf(TWO_PASS *twopass, + const FIRSTPASS_STATS *last_frame, + const FIRSTPASS_STATS *this_frame, + const FIRSTPASS_STATS *next_frame) { + int is_viable_kf = 0; + double pcnt_intra = 1.0 - this_frame->pcnt_inter; + double modified_pcnt_inter = + this_frame->pcnt_inter - this_frame->pcnt_neutral; + + // Does the frame satisfy the primary criteria of a key frame? + // See above for an explanation of the test criteria. + // If so, then examine how well it predicts subsequent frames. + if ((this_frame->pcnt_second_ref < SECOND_REF_USEAGE_THRESH) && + (next_frame->pcnt_second_ref < SECOND_REF_USEAGE_THRESH) && + ((this_frame->pcnt_inter < VERY_LOW_INTER_THRESH) || + ((pcnt_intra > MIN_INTRA_LEVEL) && + (pcnt_intra > (INTRA_VS_INTER_THRESH * modified_pcnt_inter)) && + ((this_frame->intra_error / + DOUBLE_DIVIDE_CHECK(this_frame->coded_error)) < + KF_II_ERR_THRESHOLD) && + ((fabs(last_frame->coded_error - this_frame->coded_error) / + DOUBLE_DIVIDE_CHECK(this_frame->coded_error) > + ERR_CHANGE_THRESHOLD) || + (fabs(last_frame->intra_error - this_frame->intra_error) / + DOUBLE_DIVIDE_CHECK(this_frame->intra_error) > + ERR_CHANGE_THRESHOLD) || + ((next_frame->intra_error / + DOUBLE_DIVIDE_CHECK(next_frame->coded_error)) > + II_IMPROVEMENT_THRESHOLD))))) { + int i; + const FIRSTPASS_STATS *start_pos = twopass->stats_in; + FIRSTPASS_STATS local_next_frame = *next_frame; + double boost_score = 0.0; + double old_boost_score = 0.0; + double decay_accumulator = 1.0; + + // Examine how well the key frame predicts subsequent frames. + for (i = 0; i < 16; ++i) { + double next_iiratio = (BOOST_FACTOR * local_next_frame.intra_error / + DOUBLE_DIVIDE_CHECK(local_next_frame.coded_error)); + + if (next_iiratio > KF_II_MAX) next_iiratio = KF_II_MAX; + + // Cumulative effect of decay in prediction quality. + if (local_next_frame.pcnt_inter > 0.85) + decay_accumulator *= local_next_frame.pcnt_inter; + else + decay_accumulator *= (0.85 + local_next_frame.pcnt_inter) / 2.0; + + // Keep a running total. + boost_score += (decay_accumulator * next_iiratio); + + // Test various breakout clauses. + if ((local_next_frame.pcnt_inter < 0.05) || (next_iiratio < 1.5) || + (((local_next_frame.pcnt_inter - local_next_frame.pcnt_neutral) < + 0.20) && + (next_iiratio < 3.0)) || + ((boost_score - old_boost_score) < 3.0) || + (local_next_frame.intra_error < 200)) { + break; + } + + old_boost_score = boost_score; + + // Get the next frame details + if (EOF == input_stats(twopass, &local_next_frame)) break; + } + + // If there is tolerable prediction for at least the next 3 frames then + // break out else discard this potential key frame and move on + if (boost_score > 30.0 && (i > 3)) { + is_viable_kf = 1; + } else { + // Reset the file position + reset_fpf_position(twopass, start_pos); + + is_viable_kf = 0; + } + } + + return is_viable_kf; +} + +#define FRAMES_TO_CHECK_DECAY 8 + +static void find_next_key_frame(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { + int i, j; + RATE_CONTROL *const rc = &cpi->rc; + TWO_PASS *const twopass = &cpi->twopass; + GF_GROUP *const gf_group = &twopass->gf_group; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + const FIRSTPASS_STATS first_frame = *this_frame; + const FIRSTPASS_STATS *const start_position = twopass->stats_in; + FIRSTPASS_STATS next_frame; + FIRSTPASS_STATS last_frame; + int kf_bits = 0; + int loop_decay_counter = 0; + double decay_accumulator = 1.0; + double av_decay_accumulator = 0.0; + double zero_motion_accumulator = 1.0; + double boost_score = 0.0; + double kf_mod_err = 0.0; + double kf_group_err = 0.0; + double recent_loop_decay[FRAMES_TO_CHECK_DECAY]; + + av1_zero(next_frame); + + cpi->common.frame_type = KEY_FRAME; + + // Reset the GF group data structures. + av1_zero(*gf_group); + + // Is this a forced key frame by interval. + rc->this_key_frame_forced = rc->next_key_frame_forced; + + // Clear the alt ref active flag and last group multi arf flags as they + // can never be set for a key frame. + rc->source_alt_ref_active = 0; + cpi->multi_arf_last_grp_enabled = 0; + + // KF is always a GF so clear frames till next gf counter. + rc->frames_till_gf_update_due = 0; + + rc->frames_to_key = 1; + + twopass->kf_group_bits = 0; // Total bits available to kf group + twopass->kf_group_error_left = 0; // Group modified error score. + + kf_mod_err = calculate_modified_err(cpi, twopass, oxcf, this_frame); + + // Initialize the decay rates for the recent frames to check + for (j = 0; j < FRAMES_TO_CHECK_DECAY; ++j) recent_loop_decay[j] = 1.0; + + // Find the next keyframe. + i = 0; + while (twopass->stats_in < twopass->stats_in_end && + rc->frames_to_key < cpi->oxcf.key_freq) { + // Accumulate kf group error. + kf_group_err += calculate_modified_err(cpi, twopass, oxcf, this_frame); + + // Load the next frame's stats. + last_frame = *this_frame; + input_stats(twopass, this_frame); + + // Provided that we are not at the end of the file... + if (cpi->oxcf.auto_key && twopass->stats_in < twopass->stats_in_end) { + double loop_decay_rate; + + // Check for a scene cut. + if (test_candidate_kf(twopass, &last_frame, this_frame, + twopass->stats_in)) + break; + + // How fast is the prediction quality decaying? + loop_decay_rate = get_prediction_decay_rate(cpi, twopass->stats_in); + + // We want to know something about the recent past... rather than + // as used elsewhere where we are concerned with decay in prediction + // quality since the last GF or KF. + recent_loop_decay[i % FRAMES_TO_CHECK_DECAY] = loop_decay_rate; + decay_accumulator = 1.0; + for (j = 0; j < FRAMES_TO_CHECK_DECAY; ++j) + decay_accumulator *= recent_loop_decay[j]; + + // Special check for transition or high motion followed by a + // static scene. + if (detect_transition_to_still(cpi, i, cpi->oxcf.key_freq - i, + loop_decay_rate, decay_accumulator)) + break; + + // Step on to the next frame. + ++rc->frames_to_key; + + // If we don't have a real key frame within the next two + // key_freq intervals then break out of the loop. + if (rc->frames_to_key >= 2 * cpi->oxcf.key_freq) break; + } else { + ++rc->frames_to_key; + } + ++i; + } + + // If there is a max kf interval set by the user we must obey it. + // We already breakout of the loop above at 2x max. + // This code centers the extra kf if the actual natural interval + // is between 1x and 2x. + if (cpi->oxcf.auto_key && rc->frames_to_key > cpi->oxcf.key_freq) { + FIRSTPASS_STATS tmp_frame = first_frame; + + rc->frames_to_key /= 2; + + // Reset to the start of the group. + reset_fpf_position(twopass, start_position); + + kf_group_err = 0.0; + + // Rescan to get the correct error data for the forced kf group. + for (i = 0; i < rc->frames_to_key; ++i) { + kf_group_err += calculate_modified_err(cpi, twopass, oxcf, &tmp_frame); + input_stats(twopass, &tmp_frame); + } + rc->next_key_frame_forced = 1; + } else if (twopass->stats_in == twopass->stats_in_end || + rc->frames_to_key >= cpi->oxcf.key_freq) { + rc->next_key_frame_forced = 1; + } else { + rc->next_key_frame_forced = 0; + } + + // Special case for the last key frame of the file. + if (twopass->stats_in >= twopass->stats_in_end) { + // Accumulate kf group error. + kf_group_err += calculate_modified_err(cpi, twopass, oxcf, this_frame); + } + + // Calculate the number of bits that should be assigned to the kf group. + if (twopass->bits_left > 0 && twopass->modified_error_left > 0.0) { + // Maximum number of bits for a single normal frame (not key frame). + const int max_bits = frame_max_bits(rc, &cpi->oxcf); + + // Maximum number of bits allocated to the key frame group. + int64_t max_grp_bits; + + // Default allocation based on bits left and relative + // complexity of the section. + twopass->kf_group_bits = (int64_t)( + twopass->bits_left * (kf_group_err / twopass->modified_error_left)); + + // Clip based on maximum per frame rate defined by the user. + max_grp_bits = (int64_t)max_bits * (int64_t)rc->frames_to_key; + if (twopass->kf_group_bits > max_grp_bits) + twopass->kf_group_bits = max_grp_bits; + } else { + twopass->kf_group_bits = 0; + } + twopass->kf_group_bits = AOMMAX(0, twopass->kf_group_bits); + + // Reset the first pass file position. + reset_fpf_position(twopass, start_position); + + // Scan through the kf group collating various stats used to determine + // how many bits to spend on it. + decay_accumulator = 1.0; + boost_score = 0.0; + for (i = 0; i < (rc->frames_to_key - 1); ++i) { + if (EOF == input_stats(twopass, &next_frame)) break; + + // Monitor for static sections. + zero_motion_accumulator = AOMMIN(zero_motion_accumulator, + get_zero_motion_factor(cpi, &next_frame)); + + // Not all frames in the group are necessarily used in calculating boost. + if ((i <= rc->max_gf_interval) || + ((i <= (rc->max_gf_interval * 4)) && (decay_accumulator > 0.5))) { + const double frame_boost = + calc_frame_boost(cpi, this_frame, 0, KF_MAX_BOOST); + + // How fast is prediction quality decaying. + if (!detect_flash(twopass, 0)) { + const double loop_decay_rate = + get_prediction_decay_rate(cpi, &next_frame); + decay_accumulator *= loop_decay_rate; + decay_accumulator = AOMMAX(decay_accumulator, MIN_DECAY_FACTOR); + av_decay_accumulator += decay_accumulator; + ++loop_decay_counter; + } + boost_score += (decay_accumulator * frame_boost); + } + } + av_decay_accumulator /= (double)loop_decay_counter; + + reset_fpf_position(twopass, start_position); + + // Store the zero motion percentage + twopass->kf_zeromotion_pct = (int)(zero_motion_accumulator * 100.0); + + // Calculate a section intra ratio used in setting max loop filter. + twopass->section_intra_rating = calculate_section_intra_ratio( + start_position, twopass->stats_in_end, rc->frames_to_key); + + // Apply various clamps for min and max boost + rc->kf_boost = (int)(av_decay_accumulator * boost_score); + rc->kf_boost = AOMMAX(rc->kf_boost, (rc->frames_to_key * 3)); + rc->kf_boost = AOMMAX(rc->kf_boost, MIN_KF_BOOST); + + // Work out how many bits to allocate for the key frame itself. + kf_bits = calculate_boost_bits((rc->frames_to_key - 1), rc->kf_boost, + twopass->kf_group_bits); + + // Work out the fraction of the kf group bits reserved for the inter frames + // within the group after discounting the bits for the kf itself. + if (twopass->kf_group_bits) { + twopass->kfgroup_inter_fraction = + (double)(twopass->kf_group_bits - kf_bits) / + (double)twopass->kf_group_bits; + } else { + twopass->kfgroup_inter_fraction = 1.0; + } + + twopass->kf_group_bits -= kf_bits; + + // Save the bits to spend on the key frame. + gf_group->bit_allocation[0] = kf_bits; + gf_group->update_type[0] = KF_UPDATE; + gf_group->rf_level[0] = KF_STD; + + // Note the total error score of the kf group minus the key frame itself. + twopass->kf_group_error_left = (int)(kf_group_err - kf_mod_err); + + // Adjust the count of total modified error left. + // The count of bits left is adjusted elsewhere based on real coded frame + // sizes. + twopass->modified_error_left -= kf_group_err; + + if (oxcf->resize_mode == RESIZE_DYNAMIC) { + // Default to normal-sized frame on keyframes. + cpi->rc.next_frame_size_selector = UNSCALED; + } +} + +// Define the reference buffers that will be updated post encode. +static void configure_buffer_updates(AV1_COMP *cpi) { + TWO_PASS *const twopass = &cpi->twopass; + + // Wei-Ting: Should we define another function to take care of + // cpi->rc.is_$Source_Type to make this function as it is in the comment? + + cpi->rc.is_src_frame_alt_ref = 0; +#if CONFIG_EXT_REFS + cpi->rc.is_bwd_ref_frame = 0; + cpi->rc.is_last_bipred_frame = 0; + cpi->rc.is_bipred_frame = 0; + cpi->rc.is_src_frame_ext_arf = 0; +#endif // CONFIG_EXT_REFS + + switch (twopass->gf_group.update_type[twopass->gf_group.index]) { + case KF_UPDATE: +#if CONFIG_EXT_REFS + cpi->refresh_bwd_ref_frame = 1; +#endif // CONFIG_EXT_REFS + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 1; + cpi->refresh_alt_ref_frame = 1; + break; + + case LF_UPDATE: +#if CONFIG_EXT_REFS + // If we have extra ALT_REFs, we can use the farthest ALT (ALT0) as + // the BWD_REF. + if (cpi->num_extra_arfs) { + int tmp = cpi->bwd_fb_idx; + + cpi->bwd_fb_idx = cpi->alt_fb_idx; + cpi->alt_fb_idx = cpi->arf_map[0]; + cpi->arf_map[0] = tmp; + + cpi->rc.is_bwd_ref_frame = 1; + } else { + cpi->rc.is_bwd_ref_frame = 0; + } +#endif // CONFIG_EXT_REFS + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 0; + cpi->refresh_alt_ref_frame = 0; + break; + + case GF_UPDATE: +#if CONFIG_EXT_REFS + cpi->refresh_bwd_ref_frame = 0; +#endif // CONFIG_EXT_REFS + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 1; + cpi->refresh_alt_ref_frame = 0; + break; + + case OVERLAY_UPDATE: + cpi->refresh_last_frame = 0; + cpi->refresh_golden_frame = 1; +#if CONFIG_EXT_REFS + cpi->refresh_bwd_ref_frame = 0; +#endif // CONFIG_EXT_REFS + cpi->refresh_alt_ref_frame = 0; + cpi->rc.is_src_frame_alt_ref = 1; + break; + + case ARF_UPDATE: +#if CONFIG_EXT_REFS + cpi->refresh_bwd_ref_frame = 1; +#endif // CONFIG_EXT_REFS + cpi->refresh_last_frame = 0; + cpi->refresh_golden_frame = 0; + cpi->refresh_alt_ref_frame = 1; + break; + +#if CONFIG_EXT_REFS + case BRF_UPDATE: + cpi->refresh_last_frame = 0; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 1; + cpi->refresh_alt_ref_frame = 0; + cpi->rc.is_bwd_ref_frame = 1; + if (cpi->num_extra_arfs) { + // Allow BRF use the farthest ALT_REF (ALT0) as BWD_REF by swapping + // the virtual indices. + // NOTE: The indices will be swapped back after this frame is encoded + // (in av1_update_reference_frames()). + int tmp = cpi->bwd_fb_idx; + + cpi->bwd_fb_idx = cpi->alt_fb_idx; + cpi->alt_fb_idx = cpi->arf_map[0]; + cpi->arf_map[0] = tmp; + } + break; + + case LAST_BIPRED_UPDATE: + cpi->refresh_last_frame = 0; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + cpi->rc.is_last_bipred_frame = 1; + break; + + case BIPRED_UPDATE: + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + cpi->rc.is_bipred_frame = 1; + break; + + case INTNL_OVERLAY_UPDATE: + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + cpi->rc.is_src_frame_alt_ref = 1; + cpi->rc.is_src_frame_ext_arf = 1; + break; +#endif // CONFIG_EXT_REFS + + default: assert(0); break; + } +} + +static int is_skippable_frame(const AV1_COMP *cpi) { + // If the current frame does not have non-zero motion vector detected in the + // first pass, and so do its previous and forward frames, then this frame + // can be skipped for partition check, and the partition size is assigned + // according to the variance + const TWO_PASS *const twopass = &cpi->twopass; + + return (!frame_is_intra_only(&cpi->common) && + twopass->stats_in - 2 > twopass->stats_in_start && + twopass->stats_in < twopass->stats_in_end && + (twopass->stats_in - 1)->pcnt_inter - + (twopass->stats_in - 1)->pcnt_motion == + 1 && + (twopass->stats_in - 2)->pcnt_inter - + (twopass->stats_in - 2)->pcnt_motion == + 1 && + twopass->stats_in->pcnt_inter - twopass->stats_in->pcnt_motion == 1); +} + +void av1_rc_get_second_pass_params(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; + TWO_PASS *const twopass = &cpi->twopass; + GF_GROUP *const gf_group = &twopass->gf_group; + int frames_left; + FIRSTPASS_STATS this_frame; + + int target_rate; + + frames_left = (int)(twopass->total_stats.count - cm->current_video_frame); + + if (!twopass->stats_in) return; + + // If this is an arf frame then we dont want to read the stats file or + // advance the input pointer as we already have what we need. + if (gf_group->update_type[gf_group->index] == ARF_UPDATE) { + configure_buffer_updates(cpi); + target_rate = gf_group->bit_allocation[gf_group->index]; + target_rate = av1_rc_clamp_pframe_target_size(cpi, target_rate); + rc->base_frame_target = target_rate; + + cm->frame_type = INTER_FRAME; + + // Do the firstpass stats indicate that this frame is skippable for the + // partition search? + if (cpi->sf.allow_partition_search_skip && cpi->oxcf.pass == 2) { + cpi->partition_search_skippable_frame = is_skippable_frame(cpi); + } + + return; + } + + aom_clear_system_state(); + + if (cpi->oxcf.rc_mode == AOM_Q) { + twopass->active_worst_quality = cpi->oxcf.cq_level; + } else if (cm->current_video_frame == 0) { + // Special case code for first frame. + const int section_target_bandwidth = + (int)(twopass->bits_left / frames_left); + const double section_length = twopass->total_left_stats.count; + const double section_error = + twopass->total_left_stats.coded_error / section_length; + const double section_intra_skip = + twopass->total_left_stats.intra_skip_pct / section_length; + const double section_inactive_zone = + (twopass->total_left_stats.inactive_zone_rows * 2) / + ((double)cm->mb_rows * section_length); + const int tmp_q = get_twopass_worst_quality( + cpi, section_error, section_intra_skip + section_inactive_zone, + section_target_bandwidth, DEFAULT_GRP_WEIGHT); + + twopass->active_worst_quality = tmp_q; + twopass->baseline_active_worst_quality = tmp_q; + rc->ni_av_qi = tmp_q; + rc->last_q[INTER_FRAME] = tmp_q; + rc->avg_q = av1_convert_qindex_to_q(tmp_q, cm->bit_depth); + rc->avg_frame_qindex[INTER_FRAME] = tmp_q; + rc->last_q[KEY_FRAME] = (tmp_q + cpi->oxcf.best_allowed_q) / 2; + rc->avg_frame_qindex[KEY_FRAME] = rc->last_q[KEY_FRAME]; + } + + av1_zero(this_frame); + if (EOF == input_stats(twopass, &this_frame)) return; + + // Set the frame content type flag. + if (this_frame.intra_skip_pct >= FC_ANIMATION_THRESH) + twopass->fr_content_type = FC_GRAPHICS_ANIMATION; + else + twopass->fr_content_type = FC_NORMAL; + + // Keyframe and section processing. + if (rc->frames_to_key == 0 || (cpi->frame_flags & FRAMEFLAGS_KEY)) { + FIRSTPASS_STATS this_frame_copy; + this_frame_copy = this_frame; + // Define next KF group and assign bits to it. + find_next_key_frame(cpi, &this_frame); + this_frame = this_frame_copy; + } else { + cm->frame_type = INTER_FRAME; + } + + // Define a new GF/ARF group. (Should always enter here for key frames). + if (rc->frames_till_gf_update_due == 0) { + define_gf_group(cpi, &this_frame); + + rc->frames_till_gf_update_due = rc->baseline_gf_interval; + +#if ARF_STATS_OUTPUT + { + FILE *fpfile; + fpfile = fopen("arf.stt", "a"); + ++arf_count; + fprintf(fpfile, "%10d %10ld %10d %10d %10ld\n", cm->current_video_frame, + rc->frames_till_gf_update_due, rc->kf_boost, arf_count, + rc->gfu_boost); + + fclose(fpfile); + } +#endif + } + + configure_buffer_updates(cpi); + + // Do the firstpass stats indicate that this frame is skippable for the + // partition search? + if (cpi->sf.allow_partition_search_skip && cpi->oxcf.pass == 2) { + cpi->partition_search_skippable_frame = is_skippable_frame(cpi); + } + + target_rate = gf_group->bit_allocation[gf_group->index]; + + if (cpi->common.frame_type == KEY_FRAME) + target_rate = av1_rc_clamp_iframe_target_size(cpi, target_rate); + else + target_rate = av1_rc_clamp_pframe_target_size(cpi, target_rate); + + rc->base_frame_target = target_rate; + + { + const int num_mbs = (cpi->oxcf.resize_mode != RESIZE_NONE) + ? cpi->initial_mbs + : cpi->common.MBs; + // The multiplication by 256 reverses a scaling factor of (>> 8) + // applied when combining MB error values for the frame. + twopass->mb_av_energy = + log(((this_frame.intra_error * 256.0) / num_mbs) + 1.0); + } + + // Update the total stats remaining structure. + subtract_stats(&twopass->total_left_stats, &this_frame); +} + +#define MINQ_ADJ_LIMIT 48 +#define MINQ_ADJ_LIMIT_CQ 20 +#define HIGH_UNDERSHOOT_RATIO 2 +void av1_twopass_postencode_update(AV1_COMP *cpi) { + TWO_PASS *const twopass = &cpi->twopass; + RATE_CONTROL *const rc = &cpi->rc; + const int bits_used = rc->base_frame_target; + + // VBR correction is done through rc->vbr_bits_off_target. Based on the + // sign of this value, a limited % adjustment is made to the target rate + // of subsequent frames, to try and push it back towards 0. This method + // is designed to prevent extreme behaviour at the end of a clip + // or group of frames. + rc->vbr_bits_off_target += rc->base_frame_target - rc->projected_frame_size; + twopass->bits_left = AOMMAX(twopass->bits_left - bits_used, 0); + + // Calculate the pct rc error. + if (rc->total_actual_bits) { + rc->rate_error_estimate = + (int)((rc->vbr_bits_off_target * 100) / rc->total_actual_bits); + rc->rate_error_estimate = clamp(rc->rate_error_estimate, -100, 100); + } else { + rc->rate_error_estimate = 0; + } + + if (cpi->common.frame_type != KEY_FRAME) { + twopass->kf_group_bits -= bits_used; + twopass->last_kfgroup_zeromotion_pct = twopass->kf_zeromotion_pct; + } + twopass->kf_group_bits = AOMMAX(twopass->kf_group_bits, 0); + + // Increment the gf group index ready for the next frame. + ++twopass->gf_group.index; + + // If the rate control is drifting consider adjustment to min or maxq. + if ((cpi->oxcf.rc_mode != AOM_Q) && + (cpi->twopass.gf_zeromotion_pct < VLOW_MOTION_THRESHOLD) && + !cpi->rc.is_src_frame_alt_ref) { + const int maxq_adj_limit = + rc->worst_quality - twopass->active_worst_quality; + const int minq_adj_limit = + (cpi->oxcf.rc_mode == AOM_CQ ? MINQ_ADJ_LIMIT_CQ : MINQ_ADJ_LIMIT); + + // Undershoot. + if (rc->rate_error_estimate > cpi->oxcf.under_shoot_pct) { + --twopass->extend_maxq; + if (rc->rolling_target_bits >= rc->rolling_actual_bits) + ++twopass->extend_minq; + // Overshoot. + } else if (rc->rate_error_estimate < -cpi->oxcf.over_shoot_pct) { + --twopass->extend_minq; + if (rc->rolling_target_bits < rc->rolling_actual_bits) + ++twopass->extend_maxq; + } else { + // Adjustment for extreme local overshoot. + if (rc->projected_frame_size > (2 * rc->base_frame_target) && + rc->projected_frame_size > (2 * rc->avg_frame_bandwidth)) + ++twopass->extend_maxq; + + // Unwind undershoot or overshoot adjustment. + if (rc->rolling_target_bits < rc->rolling_actual_bits) + --twopass->extend_minq; + else if (rc->rolling_target_bits > rc->rolling_actual_bits) + --twopass->extend_maxq; + } + + twopass->extend_minq = clamp(twopass->extend_minq, 0, minq_adj_limit); + twopass->extend_maxq = clamp(twopass->extend_maxq, 0, maxq_adj_limit); + + // If there is a big and undexpected undershoot then feed the extra + // bits back in quickly. One situation where this may happen is if a + // frame is unexpectedly almost perfectly predicted by the ARF or GF + // but not very well predcited by the previous frame. + if (!frame_is_kf_gf_arf(cpi) && !cpi->rc.is_src_frame_alt_ref) { + int fast_extra_thresh = rc->base_frame_target / HIGH_UNDERSHOOT_RATIO; + if (rc->projected_frame_size < fast_extra_thresh) { + rc->vbr_bits_off_target_fast += + fast_extra_thresh - rc->projected_frame_size; + rc->vbr_bits_off_target_fast = + AOMMIN(rc->vbr_bits_off_target_fast, (4 * rc->avg_frame_bandwidth)); + + // Fast adaptation of minQ if necessary to use up the extra bits. + if (rc->avg_frame_bandwidth) { + twopass->extend_minq_fast = + (int)(rc->vbr_bits_off_target_fast * 8 / rc->avg_frame_bandwidth); + } + twopass->extend_minq_fast = AOMMIN( + twopass->extend_minq_fast, minq_adj_limit - twopass->extend_minq); + } else if (rc->vbr_bits_off_target_fast) { + twopass->extend_minq_fast = AOMMIN( + twopass->extend_minq_fast, minq_adj_limit - twopass->extend_minq); + } else { + twopass->extend_minq_fast = 0; + } + } + } +} diff --git a/third_party/aom/av1/encoder/firstpass.h b/third_party/aom/av1/encoder/firstpass.h new file mode 100644 index 0000000000..db459cc229 --- /dev/null +++ b/third_party/aom/av1/encoder/firstpass.h @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_FIRSTPASS_H_ +#define AV1_ENCODER_FIRSTPASS_H_ + +#include "av1/encoder/lookahead.h" +#include "av1/encoder/ratectrl.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if CONFIG_FP_MB_STATS + +#define FPMB_DCINTRA_MASK 0x01 + +#define FPMB_MOTION_ZERO_MASK 0x02 +#define FPMB_MOTION_LEFT_MASK 0x04 +#define FPMB_MOTION_RIGHT_MASK 0x08 +#define FPMB_MOTION_UP_MASK 0x10 +#define FPMB_MOTION_DOWN_MASK 0x20 + +#define FPMB_ERROR_SMALL_MASK 0x40 +#define FPMB_ERROR_LARGE_MASK 0x80 +#define FPMB_ERROR_SMALL_TH 2000 +#define FPMB_ERROR_LARGE_TH 48000 + +typedef struct { + uint8_t *mb_stats_start; + uint8_t *mb_stats_end; +} FIRSTPASS_MB_STATS; +#endif + +#if CONFIG_EXT_REFS +// Length of the bi-predictive frame group (BFG) +// NOTE: Currently each BFG contains one backward ref (BWF) frame plus a certain +// number of bi-predictive frames. +#define BFG_INTERVAL 2 +// The maximum number of extra ALT_REF's +// NOTE: This number cannot be greater than 2 or the reference frame buffer will +// overflow. +#define MAX_EXT_ARFS 2 +#define MIN_EXT_ARF_INTERVAL 4 +#endif // CONFIG_EXT_REFS + +#define VLOW_MOTION_THRESHOLD 950 + +typedef struct { + double frame; + double weight; + double intra_error; + double coded_error; + double sr_coded_error; + double pcnt_inter; + double pcnt_motion; + double pcnt_second_ref; + double pcnt_neutral; + double intra_skip_pct; + double inactive_zone_rows; // Image mask rows top and bottom. + double inactive_zone_cols; // Image mask columns at left and right edges. + double MVr; + double mvr_abs; + double MVc; + double mvc_abs; + double MVrv; + double MVcv; + double mv_in_out_count; + double new_mv_count; + double duration; + double count; +} FIRSTPASS_STATS; + +typedef enum { + KF_UPDATE = 0, + LF_UPDATE = 1, + GF_UPDATE = 2, + ARF_UPDATE = 3, + OVERLAY_UPDATE = 4, +#if CONFIG_EXT_REFS + BRF_UPDATE = 5, // Backward Reference Frame + LAST_BIPRED_UPDATE = 6, // Last Bi-predictive Frame + BIPRED_UPDATE = 7, // Bi-predictive Frame, but not the last one + INTNL_OVERLAY_UPDATE = 8, // Internal Overlay Frame + FRAME_UPDATE_TYPES = 9 +#else + FRAME_UPDATE_TYPES = 5 +#endif // CONFIG_EXT_REFS +} FRAME_UPDATE_TYPE; + +#define FC_ANIMATION_THRESH 0.15 +typedef enum { + FC_NORMAL = 0, + FC_GRAPHICS_ANIMATION = 1, + FRAME_CONTENT_TYPES = 2 +} FRAME_CONTENT_TYPE; + +typedef struct { + unsigned char index; + RATE_FACTOR_LEVEL rf_level[(MAX_LAG_BUFFERS * 2) + 1]; + FRAME_UPDATE_TYPE update_type[(MAX_LAG_BUFFERS * 2) + 1]; + unsigned char arf_src_offset[(MAX_LAG_BUFFERS * 2) + 1]; + unsigned char arf_update_idx[(MAX_LAG_BUFFERS * 2) + 1]; + unsigned char arf_ref_idx[(MAX_LAG_BUFFERS * 2) + 1]; +#if CONFIG_EXT_REFS + unsigned char brf_src_offset[(MAX_LAG_BUFFERS * 2) + 1]; + unsigned char bidir_pred_enabled[(MAX_LAG_BUFFERS * 2) + 1]; +#endif // CONFIG_EXT_REFS + int bit_allocation[(MAX_LAG_BUFFERS * 2) + 1]; +} GF_GROUP; + +typedef struct { + unsigned int section_intra_rating; + FIRSTPASS_STATS total_stats; + FIRSTPASS_STATS this_frame_stats; + const FIRSTPASS_STATS *stats_in; + const FIRSTPASS_STATS *stats_in_start; + const FIRSTPASS_STATS *stats_in_end; + FIRSTPASS_STATS total_left_stats; + int first_pass_done; + int64_t bits_left; + double modified_error_min; + double modified_error_max; + double modified_error_left; + double mb_av_energy; + +#if CONFIG_FP_MB_STATS + uint8_t *frame_mb_stats_buf; + uint8_t *this_frame_mb_stats; + FIRSTPASS_MB_STATS firstpass_mb_stats; +#endif + // An indication of the content type of the current frame + FRAME_CONTENT_TYPE fr_content_type; + + // Projected total bits available for a key frame group of frames + int64_t kf_group_bits; + + // Error score of frames still to be coded in kf group + int64_t kf_group_error_left; + + // The fraction for a kf groups total bits allocated to the inter frames + double kfgroup_inter_fraction; + + int sr_update_lag; + + int kf_zeromotion_pct; + int last_kfgroup_zeromotion_pct; + int gf_zeromotion_pct; + int active_worst_quality; + int baseline_active_worst_quality; + int extend_minq; + int extend_maxq; + int extend_minq_fast; + + GF_GROUP gf_group; +} TWO_PASS; + +struct AV1_COMP; + +void av1_init_first_pass(struct AV1_COMP *cpi); +void av1_rc_get_first_pass_params(struct AV1_COMP *cpi); +void av1_first_pass(struct AV1_COMP *cpi, const struct lookahead_entry *source); +void av1_end_first_pass(struct AV1_COMP *cpi); + +void av1_init_second_pass(struct AV1_COMP *cpi); +void av1_rc_get_second_pass_params(struct AV1_COMP *cpi); +void av1_twopass_postencode_update(struct AV1_COMP *cpi); + +// Post encode update of the rate control parameters for 2-pass +void av1_twopass_postencode_update(struct AV1_COMP *cpi); + +void av1_init_subsampling(struct AV1_COMP *cpi); + +void av1_calculate_coded_size(struct AV1_COMP *cpi, int *scaled_frame_width, + int *scaled_frame_height); + +#if CONFIG_EXT_REFS +static INLINE int get_number_of_extra_arfs(int interval, int arf_pending) { + if (arf_pending && MAX_EXT_ARFS > 0) + return interval >= MIN_EXT_ARF_INTERVAL * (MAX_EXT_ARFS + 1) + ? MAX_EXT_ARFS + : interval >= MIN_EXT_ARF_INTERVAL * MAX_EXT_ARFS + ? MAX_EXT_ARFS - 1 + : 0; + else + return 0; +} +#endif // CONFIG_EXT_REFS + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_FIRSTPASS_H_ diff --git a/third_party/aom/av1/encoder/generic_encoder.c b/third_party/aom/av1/encoder/generic_encoder.c new file mode 100644 index 0000000000..a31bb9ef69 --- /dev/null +++ b/third_party/aom/av1/encoder/generic_encoder.c @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include + +#include "aom_dsp/bitwriter.h" +#include "av1/common/generic_code.h" +#include "av1/common/odintrin.h" +#include "pvq_encoder.h" + +/** Encodes a value from 0 to N-1 (with N up to 16) based on a cdf and adapts + * the cdf accordingly. + * + * @param [in,out] w multi-symbol entropy encoder + * @param [in] val variable being encoded + * @param [in,out] cdf CDF of the variable (Q15) + * @param [in] n number of values possible + * @param [in,out] count number of symbols encoded with that cdf so far + * @param [in] rate adaptation rate shift (smaller is faster) + */ +void aom_encode_cdf_adapt_q15(aom_writer *w, int val, uint16_t *cdf, int n, + int *count, int rate) { + int i; + if (*count == 0) { + /* On the first call, we normalize the cdf to (32768 - n). This should + eventually be moved to the state init, but for now it makes it much + easier to experiment and convert symbols to the Q15 adaptation.*/ + int ft; + ft = cdf[n - 1]; + for (i = 0; i < n; i++) { + cdf[i] = AOM_ICDF(cdf[i]*32768/ft); + } + } + aom_write_cdf(w, val, cdf, n); + aom_cdf_adapt_q15(val, cdf, n, count, rate); +} + +/** Encodes a random variable using a "generic" model, assuming that the + * distribution is one-sided (zero and up), has a single mode, and decays + * exponentially past the model. + * + * @param [in,out] w multi-symbol entropy encoder + * @param [in,out] model generic probability model + * @param [in] x variable being encoded + * @param [in,out] ExQ16 expectation of x (adapted) + * @param [in] integration integration period of ExQ16 (leaky average over + * 1<> 1); + /* Choose the cdf to use: we have two per "octave" of ExQ16 */ + id = OD_MINI(GENERIC_TABLES - 1, lg_q1); + cdf = model->cdf[id]; + xs = (x + (1 << shift >> 1)) >> shift; + aom_write_symbol_pvq(w, OD_MINI(15, xs), cdf, 16); + if (xs >= 15) { + int e; + unsigned decay; + /* Estimate decay based on the assumption that the distribution is close + to Laplacian for large values. We should probably have an adaptive + estimate instead. Note: The 2* is a kludge that's not fully understood + yet. */ + OD_ASSERT(*ex_q16 < INT_MAX >> 1); + e = ((2**ex_q16 >> 8) + (1 << shift >> 1)) >> shift; + decay = OD_MAXI(2, OD_MINI(254, 256*e/(e + 256))); + /* Encode the tail of the distribution assuming exponential decay. */ + aom_laplace_encode_special(w, xs - 15, decay); + } + if (shift != 0) { + int special; + /* Because of the rounding, there's only half the number of possibilities + for xs=0. */ + special = xs == 0; + if (shift - special > 0) { + aom_write_literal(w, x - (xs << shift) + (!special << (shift - 1)), + shift - special); + } + } + generic_model_update(ex_q16, x, integration); + OD_LOG((OD_LOG_ENTROPY_CODER, OD_LOG_DEBUG, + "enc: %d %d %d %d %d %x", *ex_q16, x, shift, id, xs, enc->rng)); +} + +/** Estimates the cost of encoding a value with generic_encode(). + * + * @param [in,out] model generic probability model + * @param [in] x variable being encoded + * @param [in,out] ExQ16 expectation of x (adapted) + * @return number of bits (approximation) + */ +double generic_encode_cost(generic_encoder *model, int x, int *ex_q16) { + int lg_q1; + int shift; + int id; + uint16_t *cdf; + int xs; + int extra; + lg_q1 = log_ex(*ex_q16); + /* If expectation is too large, shift x to ensure that + all we have past xs=15 is the exponentially decaying tail + of the distribution */ + shift = OD_MAXI(0, (lg_q1 - 5) >> 1); + /* Choose the cdf to use: we have two per "octave" of ExQ16 */ + id = OD_MINI(GENERIC_TABLES - 1, lg_q1); + cdf = model->cdf[id]; + xs = (x + (1 << shift >> 1)) >> shift; + extra = 0; + if (shift) extra = shift - (xs == 0); + xs = OD_MINI(15, xs); + /* Shortcut: assume it's going to cost 2 bits for the Laplace coder. */ + if (xs == 15) extra += 2; + return + extra - OD_LOG2((double)(cdf[xs] - (xs == 0 ? 0 : cdf[xs - 1]))/cdf[15]); +} + +/*Estimates the cost of encoding a value with a given CDF.*/ +double od_encode_cdf_cost(int val, uint16_t *cdf, int n) { + int total_prob; + int prev_prob; + double val_prob; + OD_ASSERT(n > 0); + total_prob = cdf[n - 1]; + if (val == 0) { + prev_prob = 0; + } + else { + prev_prob = cdf[val - 1]; + } + val_prob = (cdf[val] - prev_prob) / (double)total_prob; + return -OD_LOG2(val_prob); +} diff --git a/third_party/aom/av1/encoder/global_motion.c b/third_party/aom/av1/encoder/global_motion.c new file mode 100644 index 0000000000..2a62049391 --- /dev/null +++ b/third_party/aom/av1/encoder/global_motion.c @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include +#include + +#include "av1/encoder/global_motion.h" + +#include "av1/common/warped_motion.h" + +#include "av1/encoder/segmentation.h" +#include "av1/encoder/corner_detect.h" +#include "av1/encoder/corner_match.h" +#include "av1/encoder/ransac.h" + +#define MAX_CORNERS 4096 +#define MIN_INLIER_PROB 0.1 + +#define MIN_TRANS_THRESH (1 * GM_TRANS_DECODE_FACTOR) + +// Border over which to compute the global motion +#define ERRORADV_BORDER 0 + +#define ERRORADV_MAX_THRESH 0.995 +#define ERRORADV_COST_PRODUCT_THRESH 26000 + +int is_enough_erroradvantage(double best_erroradvantage, int params_cost) { + return best_erroradvantage < ERRORADV_MAX_THRESH && + best_erroradvantage * params_cost < ERRORADV_COST_PRODUCT_THRESH; +} + +static void convert_to_params(const double *params, int32_t *model) { + int i; + int alpha_present = 0; + model[0] = (int32_t)floor(params[0] * (1 << GM_TRANS_PREC_BITS) + 0.5); + model[1] = (int32_t)floor(params[1] * (1 << GM_TRANS_PREC_BITS) + 0.5); + model[0] = (int32_t)clamp(model[0], GM_TRANS_MIN, GM_TRANS_MAX) * + GM_TRANS_DECODE_FACTOR; + model[1] = (int32_t)clamp(model[1], GM_TRANS_MIN, GM_TRANS_MAX) * + GM_TRANS_DECODE_FACTOR; + + for (i = 2; i < 6; ++i) { + const int diag_value = ((i == 2 || i == 5) ? (1 << GM_ALPHA_PREC_BITS) : 0); + model[i] = (int32_t)floor(params[i] * (1 << GM_ALPHA_PREC_BITS) + 0.5); + model[i] = + (int32_t)clamp(model[i] - diag_value, GM_ALPHA_MIN, GM_ALPHA_MAX); + alpha_present |= (model[i] != 0); + model[i] = (model[i] + diag_value) * GM_ALPHA_DECODE_FACTOR; + } + for (; i < 8; ++i) { + model[i] = (int32_t)floor(params[i] * (1 << GM_ROW3HOMO_PREC_BITS) + 0.5); + model[i] = (int32_t)clamp(model[i], GM_ROW3HOMO_MIN, GM_ROW3HOMO_MAX) * + GM_ROW3HOMO_DECODE_FACTOR; + alpha_present |= (model[i] != 0); + } + + if (!alpha_present) { + if (abs(model[0]) < MIN_TRANS_THRESH && abs(model[1]) < MIN_TRANS_THRESH) { + model[0] = 0; + model[1] = 0; + } + } +} + +void convert_model_to_params(const double *params, WarpedMotionParams *model) { + convert_to_params(params, model->wmmat); + model->wmtype = get_gmtype(model); +} + +// Adds some offset to a global motion parameter and handles +// all of the necessary precision shifts, clamping, and +// zero-centering. +static int32_t add_param_offset(int param_index, int32_t param_value, + int32_t offset) { + const int scale_vals[3] = { GM_TRANS_PREC_DIFF, GM_ALPHA_PREC_DIFF, + GM_ROW3HOMO_PREC_DIFF }; + const int clamp_vals[3] = { GM_TRANS_MAX, GM_ALPHA_MAX, GM_ROW3HOMO_MAX }; + // type of param: 0 - translation, 1 - affine, 2 - homography + const int param_type = (param_index < 2 ? 0 : (param_index < 6 ? 1 : 2)); + const int is_one_centered = (param_index == 2 || param_index == 5); + + // Make parameter zero-centered and offset the shift that was done to make + // it compatible with the warped model + param_value = (param_value - (is_one_centered << WARPEDMODEL_PREC_BITS)) >> + scale_vals[param_type]; + // Add desired offset to the rescaled/zero-centered parameter + param_value += offset; + // Clamp the parameter so it does not overflow the number of bits allotted + // to it in the bitstream + param_value = (int32_t)clamp(param_value, -clamp_vals[param_type], + clamp_vals[param_type]); + // Rescale the parameter to WARPEDMODEL_PRECISION_BITS so it is compatible + // with the warped motion library + param_value *= (1 << scale_vals[param_type]); + + // Undo the zero-centering step if necessary + return param_value + (is_one_centered << WARPEDMODEL_PREC_BITS); +} + +static void force_wmtype(WarpedMotionParams *wm, TransformationType wmtype) { + switch (wmtype) { + case IDENTITY: wm->wmmat[0] = 0; wm->wmmat[1] = 0; + case TRANSLATION: + wm->wmmat[2] = 1 << WARPEDMODEL_PREC_BITS; + wm->wmmat[3] = 0; + case ROTZOOM: wm->wmmat[4] = -wm->wmmat[3]; wm->wmmat[5] = wm->wmmat[2]; + case AFFINE: wm->wmmat[6] = wm->wmmat[7] = 0; break; + case HORTRAPEZOID: wm->wmmat[6] = wm->wmmat[4] = 0; break; + case VERTRAPEZOID: wm->wmmat[7] = wm->wmmat[3] = 0; break; + case HOMOGRAPHY: break; + default: assert(0); + } + wm->wmtype = wmtype; +} + +double refine_integerized_param(WarpedMotionParams *wm, + TransformationType wmtype, +#if CONFIG_HIGHBITDEPTH + int use_hbd, int bd, +#endif // CONFIG_HIGHBITDEPTH + uint8_t *ref, int r_width, int r_height, + int r_stride, uint8_t *dst, int d_width, + int d_height, int d_stride, int n_refinements) { + static const int max_trans_model_params[TRANS_TYPES] = { + 0, 2, 4, 6, 8, 8, 8 + }; + const int border = ERRORADV_BORDER; + int i = 0, p; + int n_params = max_trans_model_params[wmtype]; + int32_t *param_mat = wm->wmmat; + double step_error; + int32_t step; + int32_t *param; + int32_t curr_param; + int32_t best_param; + double best_error; + + force_wmtype(wm, wmtype); + best_error = av1_warp_erroradv(wm, +#if CONFIG_HIGHBITDEPTH + use_hbd, bd, +#endif // CONFIG_HIGHBITDEPTH + ref, r_width, r_height, r_stride, + dst + border * d_stride + border, border, + border, d_width - 2 * border, + d_height - 2 * border, d_stride, 0, 0, 16, 16); + step = 1 << (n_refinements + 1); + for (i = 0; i < n_refinements; i++, step >>= 1) { + for (p = 0; p < n_params; ++p) { + int step_dir = 0; + // Skip searches for parameters that are forced to be 0 + if (wmtype == HORTRAPEZOID && (p == 4 || p == 6)) continue; + if (wmtype == VERTRAPEZOID && (p == 3 || p == 7)) continue; + param = param_mat + p; + curr_param = *param; + best_param = curr_param; + // look to the left + *param = add_param_offset(p, curr_param, -step); + step_error = av1_warp_erroradv( + wm, +#if CONFIG_HIGHBITDEPTH + use_hbd, bd, +#endif // CONFIG_HIGHBITDEPTH + ref, r_width, r_height, r_stride, dst + border * d_stride + border, + border, border, d_width - 2 * border, d_height - 2 * border, d_stride, + 0, 0, 16, 16); + if (step_error < best_error) { + best_error = step_error; + best_param = *param; + step_dir = -1; + } + + // look to the right + *param = add_param_offset(p, curr_param, step); + step_error = av1_warp_erroradv( + wm, +#if CONFIG_HIGHBITDEPTH + use_hbd, bd, +#endif // CONFIG_HIGHBITDEPTH + ref, r_width, r_height, r_stride, dst + border * d_stride + border, + border, border, d_width - 2 * border, d_height - 2 * border, d_stride, + 0, 0, 16, 16); + if (step_error < best_error) { + best_error = step_error; + best_param = *param; + step_dir = 1; + } + *param = best_param; + + // look to the direction chosen above repeatedly until error increases + // for the biggest step size + while (step_dir) { + *param = add_param_offset(p, best_param, step * step_dir); + step_error = av1_warp_erroradv( + wm, +#if CONFIG_HIGHBITDEPTH + use_hbd, bd, +#endif // CONFIG_HIGHBITDEPTH + ref, r_width, r_height, r_stride, dst + border * d_stride + border, + border, border, d_width - 2 * border, d_height - 2 * border, + d_stride, 0, 0, 16, 16); + if (step_error < best_error) { + best_error = step_error; + best_param = *param; + } else { + *param = best_param; + step_dir = 0; + } + } + } + } + force_wmtype(wm, wmtype); + wm->wmtype = get_gmtype(wm); + return best_error; +} + +static INLINE RansacFunc get_ransac_type(TransformationType type) { + switch (type) { + case HOMOGRAPHY: return ransac_homography; + case HORTRAPEZOID: return ransac_hortrapezoid; + case VERTRAPEZOID: return ransac_vertrapezoid; + case AFFINE: return ransac_affine; + case ROTZOOM: return ransac_rotzoom; + case TRANSLATION: return ransac_translation; + default: assert(0); return NULL; + } +} + +#if CONFIG_HIGHBITDEPTH +static unsigned char *downconvert_frame(YV12_BUFFER_CONFIG *frm, + int bit_depth) { + int i, j; + uint16_t *orig_buf = CONVERT_TO_SHORTPTR(frm->y_buffer); + uint8_t *buf = malloc(frm->y_height * frm->y_stride * sizeof(*buf)); + + for (i = 0; i < frm->y_height; ++i) + for (j = 0; j < frm->y_width; ++j) + buf[i * frm->y_stride + j] = + orig_buf[i * frm->y_stride + j] >> (bit_depth - 8); + + return buf; +} +#endif + +int compute_global_motion_feature_based( + TransformationType type, YV12_BUFFER_CONFIG *frm, YV12_BUFFER_CONFIG *ref, +#if CONFIG_HIGHBITDEPTH + int bit_depth, +#endif + int *num_inliers_by_motion, double *params_by_motion, int num_motions) { + int i; + int num_frm_corners, num_ref_corners; + int num_correspondences; + int *correspondences; + int frm_corners[2 * MAX_CORNERS], ref_corners[2 * MAX_CORNERS]; + unsigned char *frm_buffer = frm->y_buffer; + unsigned char *ref_buffer = ref->y_buffer; + RansacFunc ransac = get_ransac_type(type); + +#if CONFIG_HIGHBITDEPTH + if (frm->flags & YV12_FLAG_HIGHBITDEPTH) { + // The frame buffer is 16-bit, so we need to convert to 8 bits for the + // following code. We cache the result until the frame is released. + if (frm->y_buffer_8bit) + frm_buffer = frm->y_buffer_8bit; + else + frm_buffer = frm->y_buffer_8bit = downconvert_frame(frm, bit_depth); + } + if (ref->flags & YV12_FLAG_HIGHBITDEPTH) { + if (ref->y_buffer_8bit) + ref_buffer = ref->y_buffer_8bit; + else + ref_buffer = ref->y_buffer_8bit = downconvert_frame(ref, bit_depth); + } +#endif + + // compute interest points in images using FAST features + num_frm_corners = fast_corner_detect(frm_buffer, frm->y_width, frm->y_height, + frm->y_stride, frm_corners, MAX_CORNERS); + num_ref_corners = fast_corner_detect(ref_buffer, ref->y_width, ref->y_height, + ref->y_stride, ref_corners, MAX_CORNERS); + + // find correspondences between the two images + correspondences = + (int *)malloc(num_frm_corners * 4 * sizeof(*correspondences)); + num_correspondences = determine_correspondence( + frm_buffer, (int *)frm_corners, num_frm_corners, ref_buffer, + (int *)ref_corners, num_ref_corners, frm->y_width, frm->y_height, + frm->y_stride, ref->y_stride, correspondences); + + ransac(correspondences, num_correspondences, num_inliers_by_motion, + params_by_motion, num_motions); + + free(correspondences); + + // Set num_inliers = 0 for motions with too few inliers so they are ignored. + for (i = 0; i < num_motions; ++i) { + if (num_inliers_by_motion[i] < MIN_INLIER_PROB * num_correspondences) { + num_inliers_by_motion[i] = 0; + } + } + + // Return true if any one of the motions has inliers. + for (i = 0; i < num_motions; ++i) { + if (num_inliers_by_motion[i] > 0) return 1; + } + return 0; +} diff --git a/third_party/aom/av1/encoder/global_motion.h b/third_party/aom/av1/encoder/global_motion.h new file mode 100644 index 0000000000..8fc757f387 --- /dev/null +++ b/third_party/aom/av1/encoder/global_motion.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_GLOBAL_MOTION_H_ +#define AV1_ENCODER_GLOBAL_MOTION_H_ + +#include "aom/aom_integer.h" +#include "aom_scale/yv12config.h" +#include "av1/common/mv.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define RANSAC_NUM_MOTIONS 1 + +void convert_model_to_params(const double *params, WarpedMotionParams *model); + +int is_enough_erroradvantage(double erroradv, int params_cost); + +double refine_integerized_param(WarpedMotionParams *wm, + TransformationType wmtype, +#if CONFIG_HIGHBITDEPTH + int use_hbd, int bd, +#endif // CONFIG_HIGHBITDEPTH + uint8_t *ref, int r_width, int r_height, + int r_stride, uint8_t *dst, int d_width, + int d_height, int d_stride, int n_refinements); + +/* + Computes "num_motions" candidate global motion parameters between two frames. + The array "params_by_motion" should be length 8 * "num_motions". The ordering + of each set of parameters is best described by the homography: + + [x' (m2 m3 m0 [x + z . y' = m4 m5 m1 * y + 1] m6 m7 1) 1] + + where m{i} represents the ith value in any given set of parameters. + + "num_inliers" should be length "num_motions", and will be populated with the + number of inlier feature points for each motion. Params for which the + num_inliers entry is 0 should be ignored by the caller. +*/ +int compute_global_motion_feature_based( + TransformationType type, YV12_BUFFER_CONFIG *frm, YV12_BUFFER_CONFIG *ref, +#if CONFIG_HIGHBITDEPTH + int bit_depth, +#endif + int *num_inliers_by_motion, double *params_by_motion, int num_motions); +#ifdef __cplusplus +} // extern "C" +#endif +#endif // AV1_ENCODER_GLOBAL_MOTION_H_ diff --git a/third_party/aom/av1/encoder/hybrid_fwd_txfm.c b/third_party/aom/av1/encoder/hybrid_fwd_txfm.c new file mode 100644 index 0000000000..4fd5631632 --- /dev/null +++ b/third_party/aom/av1/encoder/hybrid_fwd_txfm.c @@ -0,0 +1,499 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./av1_rtcd.h" +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "av1/common/idct.h" +#include "av1/encoder/hybrid_fwd_txfm.h" + +#if CONFIG_CB4X4 +static void fwd_txfm_2x2(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, int lossless) { + tran_high_t a1 = src_diff[0]; + tran_high_t b1 = src_diff[1]; + tran_high_t c1 = src_diff[diff_stride]; + tran_high_t d1 = src_diff[1 + diff_stride]; + + tran_high_t a2 = a1 + c1; + tran_high_t b2 = b1 + d1; + tran_high_t c2 = a1 - c1; + tran_high_t d2 = b1 - d1; + + a1 = a2 + b2; + b1 = a2 - b2; + c1 = c2 + d2; + d1 = c2 - d2; + + coeff[0] = (tran_low_t)(4 * a1); + coeff[1] = (tran_low_t)(4 * b1); + coeff[2] = (tran_low_t)(4 * c1); + coeff[3] = (tran_low_t)(4 * d1); + + (void)tx_type; + (void)lossless; +} +#endif + +static void fwd_txfm_4x4(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, int lossless) { + if (lossless) { + assert(tx_type == DCT_DCT); + av1_fwht4x4(src_diff, coeff, diff_stride); + return; + } + + av1_fht4x4(src_diff, coeff, diff_stride, tx_type); +} + +static void fwd_txfm_4x8(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt) { + (void)fwd_txfm_opt; + av1_fht4x8(src_diff, coeff, diff_stride, tx_type); +} + +static void fwd_txfm_8x4(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt) { + (void)fwd_txfm_opt; + av1_fht8x4(src_diff, coeff, diff_stride, tx_type); +} + +static void fwd_txfm_8x16(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt) { + (void)fwd_txfm_opt; + av1_fht8x16(src_diff, coeff, diff_stride, tx_type); +} + +static void fwd_txfm_16x8(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt) { + (void)fwd_txfm_opt; + av1_fht16x8(src_diff, coeff, diff_stride, tx_type); +} + +static void fwd_txfm_16x32(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt) { + (void)fwd_txfm_opt; + av1_fht16x32(src_diff, coeff, diff_stride, tx_type); +} + +static void fwd_txfm_32x16(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt) { + (void)fwd_txfm_opt; + av1_fht32x16(src_diff, coeff, diff_stride, tx_type); +} + +static void fwd_txfm_8x8(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt) { + (void)fwd_txfm_opt; + av1_fht8x8(src_diff, coeff, diff_stride, tx_type); +} + +static void fwd_txfm_16x16(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt) { + (void)fwd_txfm_opt; + av1_fht16x16(src_diff, coeff, diff_stride, tx_type); +} + +static void fwd_txfm_32x32(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt) { + (void)fwd_txfm_opt; + av1_fht32x32(src_diff, coeff, diff_stride, tx_type); +} + +#if CONFIG_TX64X64 +static void fwd_txfm_64x64(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt) { + (void)fwd_txfm_opt; +#if CONFIG_EXT_TX + if (tx_type == IDTX) + av1_fwd_idtx_c(src_diff, coeff, diff_stride, 64, tx_type); + else +#endif + av1_fht64x64(src_diff, coeff, diff_stride, tx_type); +} +#endif // CONFIG_TX64X64 + +#if CONFIG_HIGHBITDEPTH +#if CONFIG_CB4X4 +static void highbd_fwd_txfm_2x2(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, int lossless, + const int bd) { + tran_high_t a1 = src_diff[0]; + tran_high_t b1 = src_diff[1]; + tran_high_t c1 = src_diff[diff_stride]; + tran_high_t d1 = src_diff[1 + diff_stride]; + + tran_high_t a2 = a1 + c1; + tran_high_t b2 = b1 + d1; + tran_high_t c2 = a1 - c1; + tran_high_t d2 = b1 - d1; + + a1 = a2 + b2; + b1 = a2 - b2; + c1 = c2 + d2; + d1 = c2 - d2; + + coeff[0] = (tran_low_t)(4 * a1); + coeff[1] = (tran_low_t)(4 * b1); + coeff[2] = (tran_low_t)(4 * c1); + coeff[3] = (tran_low_t)(4 * d1); + + (void)tx_type; + (void)lossless; + (void)bd; +} +#endif + +static void highbd_fwd_txfm_4x4(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, int lossless, + const int bd) { + if (lossless) { + assert(tx_type == DCT_DCT); + av1_highbd_fwht4x4(src_diff, coeff, diff_stride); + return; + } + + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + av1_fwd_txfm2d_4x4(src_diff, coeff, diff_stride, tx_type, bd); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + av1_fwd_txfm2d_4x4(src_diff, coeff, diff_stride, tx_type, bd); + break; + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + av1_highbd_fht4x4_c(src_diff, coeff, diff_stride, tx_type); + break; + case IDTX: av1_fwd_idtx_c(src_diff, coeff, diff_stride, 4, tx_type); break; +#endif // CONFIG_EXT_TX + default: assert(0); + } +} + +static void highbd_fwd_txfm_4x8(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt, const int bd) { + (void)fwd_txfm_opt; + (void)bd; + av1_highbd_fht4x8(src_diff, coeff, diff_stride, tx_type); +} + +static void highbd_fwd_txfm_8x4(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt, const int bd) { + (void)fwd_txfm_opt; + (void)bd; + av1_highbd_fht8x4(src_diff, coeff, diff_stride, tx_type); +} + +static void highbd_fwd_txfm_8x16(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt, const int bd) { + (void)fwd_txfm_opt; + (void)bd; + av1_highbd_fht8x16(src_diff, coeff, diff_stride, tx_type); +} + +static void highbd_fwd_txfm_16x8(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt, const int bd) { + (void)fwd_txfm_opt; + (void)bd; + av1_highbd_fht16x8(src_diff, coeff, diff_stride, tx_type); +} + +static void highbd_fwd_txfm_16x32(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt, const int bd) { + (void)fwd_txfm_opt; + (void)bd; + av1_highbd_fht16x32(src_diff, coeff, diff_stride, tx_type); +} + +static void highbd_fwd_txfm_32x16(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt, const int bd) { + (void)fwd_txfm_opt; + (void)bd; + av1_highbd_fht32x16(src_diff, coeff, diff_stride, tx_type); +} + +static void highbd_fwd_txfm_8x8(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt, const int bd) { + (void)fwd_txfm_opt; + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + av1_fwd_txfm2d_8x8(src_diff, coeff, diff_stride, tx_type, bd); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + av1_fwd_txfm2d_8x8(src_diff, coeff, diff_stride, tx_type, bd); + break; + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + // Use C version since DST exists only in C + av1_highbd_fht8x8_c(src_diff, coeff, diff_stride, tx_type); + break; + case IDTX: av1_fwd_idtx_c(src_diff, coeff, diff_stride, 8, tx_type); break; +#endif // CONFIG_EXT_TX + default: assert(0); + } +} + +static void highbd_fwd_txfm_16x16(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt, const int bd) { + (void)fwd_txfm_opt; + switch (tx_type) { + case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + av1_fwd_txfm2d_16x16(src_diff, coeff, diff_stride, tx_type, bd); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + av1_fwd_txfm2d_16x16(src_diff, coeff, diff_stride, tx_type, bd); + break; + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + // Use C version since DST exists only in C + av1_highbd_fht16x16_c(src_diff, coeff, diff_stride, tx_type); + break; + case IDTX: av1_fwd_idtx_c(src_diff, coeff, diff_stride, 16, tx_type); break; +#endif // CONFIG_EXT_TX + default: assert(0); + } +} + +static void highbd_fwd_txfm_32x32(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt, const int bd) { + (void)fwd_txfm_opt; + switch (tx_type) { + case DCT_DCT: + av1_fwd_txfm2d_32x32(src_diff, coeff, diff_stride, tx_type, bd); + break; +#if CONFIG_EXT_TX + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + av1_highbd_fht32x32_c(src_diff, coeff, diff_stride, tx_type); + break; + case IDTX: av1_fwd_idtx_c(src_diff, coeff, diff_stride, 32, tx_type); break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} + +#if CONFIG_TX64X64 +static void highbd_fwd_txfm_64x64(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TX_TYPE tx_type, + FWD_TXFM_OPT fwd_txfm_opt, const int bd) { + (void)fwd_txfm_opt; + (void)bd; + switch (tx_type) { + case DCT_DCT: + av1_highbd_fht64x64(src_diff, coeff, diff_stride, tx_type); + break; +#if CONFIG_EXT_TX + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: + case V_DCT: + case H_DCT: + case V_ADST: + case H_ADST: + case V_FLIPADST: + case H_FLIPADST: + av1_highbd_fht64x64(src_diff, coeff, diff_stride, tx_type); + break; + case IDTX: av1_fwd_idtx_c(src_diff, coeff, diff_stride, 64, tx_type); break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} +#endif // CONFIG_TX64X64 +#endif // CONFIG_HIGHBITDEPTH + +void av1_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff, int diff_stride, + FWD_TXFM_PARAM *fwd_txfm_param) { + const int fwd_txfm_opt = FWD_TXFM_OPT_NORMAL; + const TX_TYPE tx_type = fwd_txfm_param->tx_type; + const TX_SIZE tx_size = fwd_txfm_param->tx_size; + const int lossless = fwd_txfm_param->lossless; + switch (tx_size) { +#if CONFIG_TX64X64 + case TX_64X64: + fwd_txfm_64x64(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt); + break; +#endif // CONFIG_TX64X64 + case TX_32X32: + fwd_txfm_32x32(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt); + break; + case TX_16X16: + fwd_txfm_16x16(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt); + break; + case TX_8X8: + fwd_txfm_8x8(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt); + break; + case TX_4X8: + fwd_txfm_4x8(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt); + break; + case TX_8X4: + fwd_txfm_8x4(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt); + break; + case TX_8X16: + fwd_txfm_8x16(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt); + break; + case TX_16X8: + fwd_txfm_16x8(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt); + break; + case TX_16X32: + fwd_txfm_16x32(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt); + break; + case TX_32X16: + fwd_txfm_32x16(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt); + break; + case TX_4X4: + fwd_txfm_4x4(src_diff, coeff, diff_stride, tx_type, lossless); + break; +#if CONFIG_CB4X4 + case TX_2X2: + fwd_txfm_2x2(src_diff, coeff, diff_stride, tx_type, lossless); + break; +#endif + default: assert(0); break; + } +} + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, FWD_TXFM_PARAM *fwd_txfm_param) { + const int fwd_txfm_opt = FWD_TXFM_OPT_NORMAL; + const TX_TYPE tx_type = fwd_txfm_param->tx_type; + const TX_SIZE tx_size = fwd_txfm_param->tx_size; + const int lossless = fwd_txfm_param->lossless; + const int bd = fwd_txfm_param->bd; + switch (tx_size) { +#if CONFIG_TX64X64 + case TX_64X64: + highbd_fwd_txfm_64x64(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt, + bd); + break; +#endif // CONFIG_TX64X64 + case TX_32X32: + highbd_fwd_txfm_32x32(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt, + bd); + break; + case TX_16X16: + highbd_fwd_txfm_16x16(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt, + bd); + break; + case TX_8X8: + highbd_fwd_txfm_8x8(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt, + bd); + break; + case TX_4X8: + highbd_fwd_txfm_4x8(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt, + bd); + break; + case TX_8X4: + highbd_fwd_txfm_8x4(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt, + bd); + break; + case TX_8X16: + highbd_fwd_txfm_8x16(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt, + bd); + break; + case TX_16X8: + highbd_fwd_txfm_16x8(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt, + bd); + break; + case TX_16X32: + highbd_fwd_txfm_16x32(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt, + bd); + break; + case TX_32X16: + highbd_fwd_txfm_32x16(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt, + bd); + break; + case TX_4X4: + highbd_fwd_txfm_4x4(src_diff, coeff, diff_stride, tx_type, lossless, bd); + break; +#if CONFIG_CB4X4 + case TX_2X2: + highbd_fwd_txfm_2x2(src_diff, coeff, diff_stride, tx_type, lossless, bd); + break; +#endif + default: assert(0); break; + } +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/av1/encoder/hybrid_fwd_txfm.h b/third_party/aom/av1/encoder/hybrid_fwd_txfm.h new file mode 100644 index 0000000000..e6fd17275e --- /dev/null +++ b/third_party/aom/av1/encoder/hybrid_fwd_txfm.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_HYBRID_FWD_TXFM_H_ +#define AV1_ENCODER_HYBRID_FWD_TXFM_H_ + +#include "./aom_config.h" + +typedef enum FWD_TXFM_OPT { FWD_TXFM_OPT_NORMAL } FWD_TXFM_OPT; + +typedef struct FWD_TXFM_PARAM { + TX_TYPE tx_type; + TX_SIZE tx_size; + int lossless; +#if CONFIG_HIGHBITDEPTH + int bd; +#endif // CONFIG_HIGHBITDEPTH +} FWD_TXFM_PARAM; + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff, int diff_stride, + FWD_TXFM_PARAM *fwd_txfm_param); + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, FWD_TXFM_PARAM *fwd_txfm_param); +#endif // CONFIG_HIGHBITDEPTH + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_HYBRID_FWD_TXFM_H_ diff --git a/third_party/aom/av1/encoder/laplace_encoder.c b/third_party/aom/av1/encoder/laplace_encoder.c new file mode 100644 index 0000000000..54ffc88fb5 --- /dev/null +++ b/third_party/aom/av1/encoder/laplace_encoder.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include + +#include "aom_dsp/bitwriter.h" +#include "av1/common/odintrin.h" +#include "av1/common/pvq.h" +#include "pvq_encoder.h" + +static void aom_encode_pvq_split(aom_writer *w, od_pvq_codeword_ctx *adapt, + int count, int sum, int ctx) { + int shift; + int rest; + int fctx; + if (sum == 0) return; + shift = OD_MAXI(0, OD_ILOG(sum) - 3); + if (shift) { + rest = count & ((1 << shift) - 1); + count >>= shift; + sum >>= shift; + } + fctx = 7*ctx + sum - 1; + aom_write_symbol_pvq(w, count, adapt->pvq_split_cdf[fctx], sum + 1); + if (shift) aom_write_literal(w, rest, shift); +} + +void aom_encode_band_pvq_splits(aom_writer *w, od_pvq_codeword_ctx *adapt, + const int *y, int n, int k, int level) { + int mid; + int i; + int count_right; + if (n <= 1 || k == 0) return; + if (k == 1 && n <= 16) { + int cdf_id; + int pos; + cdf_id = od_pvq_k1_ctx(n, level == 0); + for (pos = 0; !y[pos]; pos++); + OD_ASSERT(pos < n); + aom_write_symbol_pvq(w, pos, adapt->pvq_k1_cdf[cdf_id], n); + } + else { + mid = n >> 1; + count_right = k; + for (i = 0; i < mid; i++) count_right -= abs(y[i]); + aom_encode_pvq_split(w, adapt, count_right, k, od_pvq_size_ctx(n)); + aom_encode_band_pvq_splits(w, adapt, y, mid, k - count_right, level + 1); + aom_encode_band_pvq_splits(w, adapt, y + mid, n - mid, count_right, + level + 1); + } +} + +/** Encodes the tail of a Laplace-distributed variable, i.e. it doesn't + * do anything special for the zero case. + * + * @param [in,out] enc range encoder + * @param [in] x variable to encode (has to be positive) + * @param [in] decay decay factor of the distribution in Q8 format, + * i.e. pdf ~= decay^x + */ +void aom_laplace_encode_special(aom_writer *w, int x, unsigned decay) { + int shift; + int xs; + int sym; + const uint16_t *cdf; + shift = 0; + /* We don't want a large decay value because that would require too many + symbols. */ + while (decay > 235) { + decay = (decay*decay + 128) >> 8; + shift++; + } + decay = OD_MINI(decay, 254); + decay = OD_MAXI(decay, 2); + xs = x >> shift; + cdf = EXP_CDF_TABLE[(decay + 1) >> 1]; + OD_LOG((OD_LOG_PVQ, OD_LOG_DEBUG, "decay = %d", decay)); + do { + sym = OD_MINI(xs, 15); + { + int i; + OD_LOG((OD_LOG_PVQ, OD_LOG_DEBUG, "%d %d %d %d %d\n", x, xs, shift, + sym, max)); + for (i = 0; i < 16; i++) { + OD_LOG_PARTIAL((OD_LOG_PVQ, OD_LOG_DEBUG, "%d ", cdf[i])); + } + OD_LOG_PARTIAL((OD_LOG_PVQ, OD_LOG_DEBUG, "\n")); + } + aom_write_cdf(w, sym, cdf, 16); + xs -= 15; + } while (sym >= 15); + if (shift) aom_write_literal(w, x & ((1 << shift) - 1), shift); +} diff --git a/third_party/aom/av1/encoder/lookahead.c b/third_party/aom/av1/encoder/lookahead.c new file mode 100644 index 0000000000..591ca61521 --- /dev/null +++ b/third_party/aom/av1/encoder/lookahead.c @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include +#include + +#include "./aom_config.h" + +#include "av1/common/common.h" + +#include "av1/encoder/encoder.h" +#include "av1/encoder/extend.h" +#include "av1/encoder/lookahead.h" + +/* Return the buffer at the given absolute index and increment the index */ +static struct lookahead_entry *pop(struct lookahead_ctx *ctx, int *idx) { + int index = *idx; + struct lookahead_entry *buf = ctx->buf + index; + + assert(index < ctx->max_sz); + if (++index >= ctx->max_sz) index -= ctx->max_sz; + *idx = index; + return buf; +} + +void av1_lookahead_destroy(struct lookahead_ctx *ctx) { + if (ctx) { + if (ctx->buf) { + int i; + + for (i = 0; i < ctx->max_sz; i++) aom_free_frame_buffer(&ctx->buf[i].img); + free(ctx->buf); + } + free(ctx); + } +} + +struct lookahead_ctx *av1_lookahead_init(unsigned int width, + unsigned int height, + unsigned int subsampling_x, + unsigned int subsampling_y, +#if CONFIG_HIGHBITDEPTH + int use_highbitdepth, +#endif + unsigned int depth) { + struct lookahead_ctx *ctx = NULL; + + // Clamp the lookahead queue depth + depth = clamp(depth, 1, MAX_LAG_BUFFERS); + + // Allocate memory to keep previous source frames available. + depth += MAX_PRE_FRAMES; + + // Allocate the lookahead structures + ctx = calloc(1, sizeof(*ctx)); + if (ctx) { + const int legacy_byte_alignment = 0; + unsigned int i; + ctx->max_sz = depth; + ctx->buf = calloc(depth, sizeof(*ctx->buf)); + if (!ctx->buf) goto bail; + for (i = 0; i < depth; i++) + if (aom_alloc_frame_buffer(&ctx->buf[i].img, width, height, subsampling_x, + subsampling_y, +#if CONFIG_HIGHBITDEPTH + use_highbitdepth, +#endif + AOM_BORDER_IN_PIXELS, legacy_byte_alignment)) + goto bail; + } + return ctx; +bail: + av1_lookahead_destroy(ctx); + return NULL; +} + +#define USE_PARTIAL_COPY 0 + +int av1_lookahead_push(struct lookahead_ctx *ctx, YV12_BUFFER_CONFIG *src, + int64_t ts_start, int64_t ts_end, +#if CONFIG_HIGHBITDEPTH + int use_highbitdepth, +#endif + aom_enc_frame_flags_t flags) { + struct lookahead_entry *buf; +#if USE_PARTIAL_COPY + int row, col, active_end; + int mb_rows = (src->y_height + 15) >> 4; + int mb_cols = (src->y_width + 15) >> 4; +#endif + int width = src->y_crop_width; + int height = src->y_crop_height; + int uv_width = src->uv_crop_width; + int uv_height = src->uv_crop_height; + int subsampling_x = src->subsampling_x; + int subsampling_y = src->subsampling_y; + int larger_dimensions, new_dimensions; + + if (ctx->sz + 1 + MAX_PRE_FRAMES > ctx->max_sz) return 1; + ctx->sz++; + buf = pop(ctx, &ctx->write_idx); + + new_dimensions = width != buf->img.y_crop_width || + height != buf->img.y_crop_height || + uv_width != buf->img.uv_crop_width || + uv_height != buf->img.uv_crop_height; + larger_dimensions = width > buf->img.y_width || height > buf->img.y_height || + uv_width > buf->img.uv_width || + uv_height > buf->img.uv_height; + assert(!larger_dimensions || new_dimensions); + +#if USE_PARTIAL_COPY + // TODO(jkoleszar): This is disabled for now, as + // av1_copy_and_extend_frame_with_rect is not subsampling/alpha aware. + + // Only do this partial copy if the following conditions are all met: + // 1. Lookahead queue has has size of 1. + // 2. Active map is provided. + // 3. This is not a key frame, golden nor altref frame. + if (!new_dimensions && ctx->max_sz == 1 && active_map && !flags) { + for (row = 0; row < mb_rows; ++row) { + col = 0; + + while (1) { + // Find the first active macroblock in this row. + for (; col < mb_cols; ++col) { + if (active_map[col]) break; + } + + // No more active macroblock in this row. + if (col == mb_cols) break; + + // Find the end of active region in this row. + active_end = col; + + for (; active_end < mb_cols; ++active_end) { + if (!active_map[active_end]) break; + } + + // Only copy this active region. + av1_copy_and_extend_frame_with_rect(src, &buf->img, row << 4, col << 4, + 16, (active_end - col) << 4); + + // Start again from the end of this active region. + col = active_end; + } + + active_map += mb_cols; + } + } else { +#endif + if (larger_dimensions) { + YV12_BUFFER_CONFIG new_img; + memset(&new_img, 0, sizeof(new_img)); + if (aom_alloc_frame_buffer(&new_img, width, height, subsampling_x, + subsampling_y, +#if CONFIG_HIGHBITDEPTH + use_highbitdepth, +#endif + AOM_BORDER_IN_PIXELS, 0)) + return 1; + aom_free_frame_buffer(&buf->img); + buf->img = new_img; + } else if (new_dimensions) { + buf->img.y_crop_width = src->y_crop_width; + buf->img.y_crop_height = src->y_crop_height; + buf->img.uv_crop_width = src->uv_crop_width; + buf->img.uv_crop_height = src->uv_crop_height; + buf->img.subsampling_x = src->subsampling_x; + buf->img.subsampling_y = src->subsampling_y; + } + // Partial copy not implemented yet + av1_copy_and_extend_frame(src, &buf->img); +#if USE_PARTIAL_COPY + } +#endif + + buf->ts_start = ts_start; + buf->ts_end = ts_end; + buf->flags = flags; + return 0; +} + +struct lookahead_entry *av1_lookahead_pop(struct lookahead_ctx *ctx, + int drain) { + struct lookahead_entry *buf = NULL; + + if (ctx && ctx->sz && (drain || ctx->sz == ctx->max_sz - MAX_PRE_FRAMES)) { + buf = pop(ctx, &ctx->read_idx); + ctx->sz--; + } + return buf; +} + +struct lookahead_entry *av1_lookahead_peek(struct lookahead_ctx *ctx, + int index) { + struct lookahead_entry *buf = NULL; + + if (index >= 0) { + // Forward peek + if (index < ctx->sz) { + index += ctx->read_idx; + if (index >= ctx->max_sz) index -= ctx->max_sz; + buf = ctx->buf + index; + } + } else if (index < 0) { + // Backward peek + if (-index <= MAX_PRE_FRAMES) { + index += (int)(ctx->read_idx); + if (index < 0) index += (int)(ctx->max_sz); + buf = ctx->buf + index; + } + } + + return buf; +} + +unsigned int av1_lookahead_depth(struct lookahead_ctx *ctx) { return ctx->sz; } diff --git a/third_party/aom/av1/encoder/lookahead.h b/third_party/aom/av1/encoder/lookahead.h new file mode 100644 index 0000000000..19f75d7e45 --- /dev/null +++ b/third_party/aom/av1/encoder/lookahead.h @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_LOOKAHEAD_H_ +#define AV1_ENCODER_LOOKAHEAD_H_ + +#include "aom_scale/yv12config.h" +#include "aom/aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MAX_LAG_BUFFERS 25 + +struct lookahead_entry { + YV12_BUFFER_CONFIG img; + int64_t ts_start; + int64_t ts_end; + aom_enc_frame_flags_t flags; +}; + +// The max of past frames we want to keep in the queue. +#define MAX_PRE_FRAMES 1 + +struct lookahead_ctx { + int max_sz; /* Absolute size of the queue */ + int sz; /* Number of buffers currently in the queue */ + int read_idx; /* Read index */ + int write_idx; /* Write index */ + struct lookahead_entry *buf; /* Buffer list */ +}; + +/**\brief Initializes the lookahead stage + * + * The lookahead stage is a queue of frame buffers on which some analysis + * may be done when buffers are enqueued. + */ +struct lookahead_ctx *av1_lookahead_init(unsigned int width, + unsigned int height, + unsigned int subsampling_x, + unsigned int subsampling_y, +#if CONFIG_HIGHBITDEPTH + int use_highbitdepth, +#endif + unsigned int depth); + +/**\brief Destroys the lookahead stage + */ +void av1_lookahead_destroy(struct lookahead_ctx *ctx); + +/**\brief Enqueue a source buffer + * + * This function will copy the source image into a new framebuffer with + * the expected stride/border. + * + * If active_map is non-NULL and there is only one frame in the queue, then copy + * only active macroblocks. + * + * \param[in] ctx Pointer to the lookahead context + * \param[in] src Pointer to the image to enqueue + * \param[in] ts_start Timestamp for the start of this frame + * \param[in] ts_end Timestamp for the end of this frame + * \param[in] flags Flags set on this frame + * \param[in] active_map Map that specifies which macroblock is active + */ +int av1_lookahead_push(struct lookahead_ctx *ctx, YV12_BUFFER_CONFIG *src, + int64_t ts_start, int64_t ts_end, +#if CONFIG_HIGHBITDEPTH + int use_highbitdepth, +#endif + aom_enc_frame_flags_t flags); + +/**\brief Get the next source buffer to encode + * + * + * \param[in] ctx Pointer to the lookahead context + * \param[in] drain Flag indicating the buffer should be drained + * (return a buffer regardless of the current queue depth) + * + * \retval NULL, if drain set and queue is empty + * \retval NULL, if drain not set and queue not of the configured depth + */ +struct lookahead_entry *av1_lookahead_pop(struct lookahead_ctx *ctx, int drain); + +/**\brief Get a future source buffer to encode + * + * \param[in] ctx Pointer to the lookahead context + * \param[in] index Index of the frame to be returned, 0 == next frame + * + * \retval NULL, if no buffer exists at the specified index + */ +struct lookahead_entry *av1_lookahead_peek(struct lookahead_ctx *ctx, + int index); + +/**\brief Get the number of frames currently in the lookahead queue + * + * \param[in] ctx Pointer to the lookahead context + */ +unsigned int av1_lookahead_depth(struct lookahead_ctx *ctx); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_LOOKAHEAD_H_ diff --git a/third_party/aom/av1/encoder/mbgraph.c b/third_party/aom/av1/encoder/mbgraph.c new file mode 100644 index 0000000000..1296027dcb --- /dev/null +++ b/third_party/aom/av1/encoder/mbgraph.c @@ -0,0 +1,398 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/system_state.h" +#include "av1/encoder/segmentation.h" +#include "av1/encoder/mcomp.h" +#include "av1/common/blockd.h" +#include "av1/common/reconinter.h" +#include "av1/common/reconintra.h" + +static unsigned int do_16x16_motion_iteration(AV1_COMP *cpi, const MV *ref_mv, + int mb_row, int mb_col) { + MACROBLOCK *const x = &cpi->td.mb; + MACROBLOCKD *const xd = &x->e_mbd; + const MV_SPEED_FEATURES *const mv_sf = &cpi->sf.mv; + const aom_variance_fn_ptr_t v_fn_ptr = cpi->fn_ptr[BLOCK_16X16]; + + const MvLimits tmp_mv_limits = x->mv_limits; + MV ref_full; + int cost_list[5]; + + // Further step/diamond searches as necessary + int step_param = mv_sf->reduce_first_step_size; + step_param = AOMMIN(step_param, MAX_MVSEARCH_STEPS - 2); + + av1_set_mv_search_range(&x->mv_limits, ref_mv); + + ref_full.col = ref_mv->col >> 3; + ref_full.row = ref_mv->row >> 3; + + /*cpi->sf.search_method == HEX*/ + av1_hex_search(x, &ref_full, step_param, x->errorperbit, 0, + cond_cost_list(cpi, cost_list), &v_fn_ptr, 0, ref_mv); + + // Try sub-pixel MC + // if (bestsme > error_thresh && bestsme < INT_MAX) + { + int distortion; + unsigned int sse; + cpi->find_fractional_mv_step(x, ref_mv, cpi->common.allow_high_precision_mv, + x->errorperbit, &v_fn_ptr, 0, + mv_sf->subpel_iters_per_step, + cond_cost_list(cpi, cost_list), NULL, NULL, + &distortion, &sse, NULL, 0, 0, 0); + } + +#if CONFIG_EXT_INTER + if (has_second_ref(&xd->mi[0]->mbmi)) + xd->mi[0]->mbmi.mode = NEW_NEWMV; + else +#endif // CONFIG_EXT_INTER + xd->mi[0]->mbmi.mode = NEWMV; + + xd->mi[0]->mbmi.mv[0] = x->best_mv; +#if CONFIG_EXT_INTER + xd->mi[0]->mbmi.ref_frame[1] = NONE_FRAME; +#endif // CONFIG_EXT_INTER + + av1_build_inter_predictors_sby(xd, mb_row, mb_col, NULL, BLOCK_16X16); + + /* restore UMV window */ + x->mv_limits = tmp_mv_limits; + + return aom_sad16x16(x->plane[0].src.buf, x->plane[0].src.stride, + xd->plane[0].dst.buf, xd->plane[0].dst.stride); +} + +static int do_16x16_motion_search(AV1_COMP *cpi, const MV *ref_mv, int mb_row, + int mb_col) { + MACROBLOCK *const x = &cpi->td.mb; + MACROBLOCKD *const xd = &x->e_mbd; + unsigned int err, tmp_err; + MV best_mv; + + // Try zero MV first + // FIXME should really use something like near/nearest MV and/or MV prediction + err = aom_sad16x16(x->plane[0].src.buf, x->plane[0].src.stride, + xd->plane[0].pre[0].buf, xd->plane[0].pre[0].stride); + best_mv.col = best_mv.row = 0; + + // Test last reference frame using the previous best mv as the + // starting point (best reference) for the search + tmp_err = do_16x16_motion_iteration(cpi, ref_mv, mb_row, mb_col); + if (tmp_err < err) { + err = tmp_err; + best_mv = x->best_mv.as_mv; + } + + // If the current best reference mv is not centered on 0,0 then do a 0,0 + // based search as well. + if (ref_mv->row != 0 || ref_mv->col != 0) { + MV zero_ref_mv = { 0, 0 }; + + tmp_err = do_16x16_motion_iteration(cpi, &zero_ref_mv, mb_row, mb_col); + if (tmp_err < err) { + err = tmp_err; + best_mv = x->best_mv.as_mv; + } + } + + x->best_mv.as_mv = best_mv; + return err; +} + +static int do_16x16_zerozero_search(AV1_COMP *cpi, int_mv *dst_mv) { + MACROBLOCK *const x = &cpi->td.mb; + MACROBLOCKD *const xd = &x->e_mbd; + unsigned int err; + + // Try zero MV first + // FIXME should really use something like near/nearest MV and/or MV prediction + err = aom_sad16x16(x->plane[0].src.buf, x->plane[0].src.stride, + xd->plane[0].pre[0].buf, xd->plane[0].pre[0].stride); + + dst_mv->as_int = 0; + + return err; +} +static int find_best_16x16_intra(AV1_COMP *cpi, PREDICTION_MODE *pbest_mode) { + MACROBLOCK *const x = &cpi->td.mb; + MACROBLOCKD *const xd = &x->e_mbd; + PREDICTION_MODE best_mode = -1, mode; + unsigned int best_err = INT_MAX; + + // calculate SATD for each intra prediction mode; + // we're intentionally not doing 4x4, we just want a rough estimate + for (mode = DC_PRED; mode <= TM_PRED; mode++) { + unsigned int err; + + xd->mi[0]->mbmi.mode = mode; + av1_predict_intra_block(xd, 16, 16, BLOCK_16X16, mode, x->plane[0].src.buf, + x->plane[0].src.stride, xd->plane[0].dst.buf, + xd->plane[0].dst.stride, 0, 0, 0); + err = aom_sad16x16(x->plane[0].src.buf, x->plane[0].src.stride, + xd->plane[0].dst.buf, xd->plane[0].dst.stride); + + // find best + if (err < best_err) { + best_err = err; + best_mode = mode; + } + } + + if (pbest_mode) *pbest_mode = best_mode; + + return best_err; +} + +static void update_mbgraph_mb_stats(AV1_COMP *cpi, MBGRAPH_MB_STATS *stats, + YV12_BUFFER_CONFIG *buf, int mb_y_offset, + YV12_BUFFER_CONFIG *golden_ref, + const MV *prev_golden_ref_mv, + YV12_BUFFER_CONFIG *alt_ref, int mb_row, + int mb_col) { + MACROBLOCK *const x = &cpi->td.mb; + MACROBLOCKD *const xd = &x->e_mbd; + int intra_error; + AV1_COMMON *cm = &cpi->common; + + // FIXME in practice we're completely ignoring chroma here + x->plane[0].src.buf = buf->y_buffer + mb_y_offset; + x->plane[0].src.stride = buf->y_stride; + + xd->plane[0].dst.buf = get_frame_new_buffer(cm)->y_buffer + mb_y_offset; + xd->plane[0].dst.stride = get_frame_new_buffer(cm)->y_stride; + + // do intra 16x16 prediction + intra_error = find_best_16x16_intra(cpi, &stats->ref[INTRA_FRAME].m.mode); + if (intra_error <= 0) intra_error = 1; + stats->ref[INTRA_FRAME].err = intra_error; + + // Golden frame MV search, if it exists and is different than last frame + if (golden_ref) { + int g_motion_error; + xd->plane[0].pre[0].buf = golden_ref->y_buffer + mb_y_offset; + xd->plane[0].pre[0].stride = golden_ref->y_stride; + g_motion_error = + do_16x16_motion_search(cpi, prev_golden_ref_mv, mb_row, mb_col); + stats->ref[GOLDEN_FRAME].m.mv = x->best_mv; + stats->ref[GOLDEN_FRAME].err = g_motion_error; + } else { + stats->ref[GOLDEN_FRAME].err = INT_MAX; + stats->ref[GOLDEN_FRAME].m.mv.as_int = 0; + } + + // Do an Alt-ref frame MV search, if it exists and is different than + // last/golden frame. + if (alt_ref) { + int a_motion_error; + xd->plane[0].pre[0].buf = alt_ref->y_buffer + mb_y_offset; + xd->plane[0].pre[0].stride = alt_ref->y_stride; + a_motion_error = + do_16x16_zerozero_search(cpi, &stats->ref[ALTREF_FRAME].m.mv); + + stats->ref[ALTREF_FRAME].err = a_motion_error; + } else { + stats->ref[ALTREF_FRAME].err = INT_MAX; + stats->ref[ALTREF_FRAME].m.mv.as_int = 0; + } +} + +static void update_mbgraph_frame_stats(AV1_COMP *cpi, + MBGRAPH_FRAME_STATS *stats, + YV12_BUFFER_CONFIG *buf, + YV12_BUFFER_CONFIG *golden_ref, + YV12_BUFFER_CONFIG *alt_ref) { + MACROBLOCK *const x = &cpi->td.mb; + MACROBLOCKD *const xd = &x->e_mbd; + AV1_COMMON *const cm = &cpi->common; + + int mb_col, mb_row, offset = 0; + int mb_y_offset = 0, arf_y_offset = 0, gld_y_offset = 0; + MV gld_top_mv = { 0, 0 }; + MODE_INFO mi_local; + + av1_zero(mi_local); + // Set up limit values for motion vectors to prevent them extending outside + // the UMV borders. + x->mv_limits.row_min = -BORDER_MV_PIXELS_B16; + x->mv_limits.row_max = (cm->mb_rows - 1) * 8 + BORDER_MV_PIXELS_B16; + xd->up_available = 0; + xd->plane[0].dst.stride = buf->y_stride; + xd->plane[0].pre[0].stride = buf->y_stride; + xd->plane[1].dst.stride = buf->uv_stride; + xd->mi[0] = &mi_local; + mi_local.mbmi.sb_type = BLOCK_16X16; + mi_local.mbmi.ref_frame[0] = LAST_FRAME; + mi_local.mbmi.ref_frame[1] = NONE_FRAME; + + for (mb_row = 0; mb_row < cm->mb_rows; mb_row++) { + MV gld_left_mv = gld_top_mv; + int mb_y_in_offset = mb_y_offset; + int arf_y_in_offset = arf_y_offset; + int gld_y_in_offset = gld_y_offset; + + // Set up limit values for motion vectors to prevent them extending outside + // the UMV borders. + x->mv_limits.col_min = -BORDER_MV_PIXELS_B16; + x->mv_limits.col_max = (cm->mb_cols - 1) * 8 + BORDER_MV_PIXELS_B16; + xd->left_available = 0; + + for (mb_col = 0; mb_col < cm->mb_cols; mb_col++) { + MBGRAPH_MB_STATS *mb_stats = &stats->mb_stats[offset + mb_col]; + + update_mbgraph_mb_stats(cpi, mb_stats, buf, mb_y_in_offset, golden_ref, + &gld_left_mv, alt_ref, mb_row, mb_col); + gld_left_mv = mb_stats->ref[GOLDEN_FRAME].m.mv.as_mv; + if (mb_col == 0) { + gld_top_mv = gld_left_mv; + } + xd->left_available = 1; + mb_y_in_offset += 16; + gld_y_in_offset += 16; + arf_y_in_offset += 16; + x->mv_limits.col_min -= 16; + x->mv_limits.col_max -= 16; + } + xd->up_available = 1; + mb_y_offset += buf->y_stride * 16; + gld_y_offset += golden_ref->y_stride * 16; + if (alt_ref) arf_y_offset += alt_ref->y_stride * 16; + x->mv_limits.row_min -= 16; + x->mv_limits.row_max -= 16; + offset += cm->mb_cols; + } +} + +// void separate_arf_mbs_byzz +static void separate_arf_mbs(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + int mb_col, mb_row, offset, i; + int mi_row, mi_col; + int ncnt[4] = { 0 }; + int n_frames = cpi->mbgraph_n_frames; + + int *arf_not_zz; + + CHECK_MEM_ERROR( + cm, arf_not_zz, + aom_calloc(cm->mb_rows * cm->mb_cols * sizeof(*arf_not_zz), 1)); + + // We are not interested in results beyond the alt ref itself. + if (n_frames > cpi->rc.frames_till_gf_update_due) + n_frames = cpi->rc.frames_till_gf_update_due; + + // defer cost to reference frames + for (i = n_frames - 1; i >= 0; i--) { + MBGRAPH_FRAME_STATS *frame_stats = &cpi->mbgraph_stats[i]; + + for (offset = 0, mb_row = 0; mb_row < cm->mb_rows; + offset += cm->mb_cols, mb_row++) { + for (mb_col = 0; mb_col < cm->mb_cols; mb_col++) { + MBGRAPH_MB_STATS *mb_stats = &frame_stats->mb_stats[offset + mb_col]; + + int altref_err = mb_stats->ref[ALTREF_FRAME].err; + int intra_err = mb_stats->ref[INTRA_FRAME].err; + int golden_err = mb_stats->ref[GOLDEN_FRAME].err; + + // Test for altref vs intra and gf and that its mv was 0,0. + if (altref_err > 1000 || altref_err > intra_err || + altref_err > golden_err) { + arf_not_zz[offset + mb_col]++; + } + } + } + } + + // arf_not_zz is indexed by MB, but this loop is indexed by MI to avoid out + // of bound access in segmentation_map + for (mi_row = 0; mi_row < cm->mi_rows; mi_row++) { + for (mi_col = 0; mi_col < cm->mi_cols; mi_col++) { + // If any of the blocks in the sequence failed then the MB + // goes in segment 0 + if (arf_not_zz[mi_row / 2 * cm->mb_cols + mi_col / 2]) { + ncnt[0]++; + cpi->segmentation_map[mi_row * cm->mi_cols + mi_col] = 0; + } else { + cpi->segmentation_map[mi_row * cm->mi_cols + mi_col] = 1; + ncnt[1]++; + } + } + } + + // Only bother with segmentation if over 10% of the MBs in static segment + // if ( ncnt[1] && (ncnt[0] / ncnt[1] < 10) ) + if (1) { + // Note % of blocks that are marked as static + if (cm->MBs) + cpi->static_mb_pct = (ncnt[1] * 100) / (cm->mi_rows * cm->mi_cols); + + // This error case should not be reachable as this function should + // never be called with the common data structure uninitialized. + else + cpi->static_mb_pct = 0; + + av1_enable_segmentation(&cm->seg); + } else { + cpi->static_mb_pct = 0; + av1_disable_segmentation(&cm->seg); + } + + // Free localy allocated storage + aom_free(arf_not_zz); +} + +void av1_update_mbgraph_stats(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + int i, n_frames = av1_lookahead_depth(cpi->lookahead); + YV12_BUFFER_CONFIG *golden_ref = get_ref_frame_buffer(cpi, GOLDEN_FRAME); + + assert(golden_ref != NULL); + + // we need to look ahead beyond where the ARF transitions into + // being a GF - so exit if we don't look ahead beyond that + if (n_frames <= cpi->rc.frames_till_gf_update_due) return; + + if (n_frames > MAX_LAG_BUFFERS) n_frames = MAX_LAG_BUFFERS; + + cpi->mbgraph_n_frames = n_frames; + for (i = 0; i < n_frames; i++) { + MBGRAPH_FRAME_STATS *frame_stats = &cpi->mbgraph_stats[i]; + memset(frame_stats->mb_stats, 0, + cm->mb_rows * cm->mb_cols * sizeof(*cpi->mbgraph_stats[i].mb_stats)); + } + + // do motion search to find contribution of each reference to data + // later on in this GF group + // FIXME really, the GF/last MC search should be done forward, and + // the ARF MC search backwards, to get optimal results for MV caching + for (i = 0; i < n_frames; i++) { + MBGRAPH_FRAME_STATS *frame_stats = &cpi->mbgraph_stats[i]; + struct lookahead_entry *q_cur = av1_lookahead_peek(cpi->lookahead, i); + + assert(q_cur != NULL); + + update_mbgraph_frame_stats(cpi, frame_stats, &q_cur->img, golden_ref, + cpi->source); + } + + aom_clear_system_state(); + + separate_arf_mbs(cpi); +} diff --git a/third_party/aom/av1/encoder/mbgraph.h b/third_party/aom/av1/encoder/mbgraph.h new file mode 100644 index 0000000000..758e2ad152 --- /dev/null +++ b/third_party/aom/av1/encoder/mbgraph.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_MBGRAPH_H_ +#define AV1_ENCODER_MBGRAPH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + struct { + int err; + union { + int_mv mv; + PREDICTION_MODE mode; + } m; + } ref[TOTAL_REFS_PER_FRAME]; +} MBGRAPH_MB_STATS; + +typedef struct { MBGRAPH_MB_STATS *mb_stats; } MBGRAPH_FRAME_STATS; + +struct AV1_COMP; + +void av1_update_mbgraph_stats(struct AV1_COMP *cpi); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_MBGRAPH_H_ diff --git a/third_party/aom/av1/encoder/mcomp.c b/third_party/aom/av1/encoder/mcomp.c new file mode 100644 index 0000000000..d069eefb0d --- /dev/null +++ b/third_party/aom/av1/encoder/mcomp.c @@ -0,0 +1,3493 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +#include "av1/common/common.h" +#include "av1/common/mvref_common.h" +#include "av1/common/reconinter.h" + +#include "av1/encoder/encoder.h" +#include "av1/encoder/mcomp.h" +#include "av1/encoder/rdopt.h" + +// #define NEW_DIAMOND_SEARCH + +static INLINE const uint8_t *get_buf_from_mv(const struct buf_2d *buf, + const MV *mv) { + return &buf->buf[mv->row * buf->stride + mv->col]; +} + +void av1_set_mv_search_range(MvLimits *mv_limits, const MV *mv) { + int col_min = (mv->col >> 3) - MAX_FULL_PEL_VAL + (mv->col & 7 ? 1 : 0); + int row_min = (mv->row >> 3) - MAX_FULL_PEL_VAL + (mv->row & 7 ? 1 : 0); + int col_max = (mv->col >> 3) + MAX_FULL_PEL_VAL; + int row_max = (mv->row >> 3) + MAX_FULL_PEL_VAL; + + col_min = AOMMAX(col_min, (MV_LOW >> 3) + 1); + row_min = AOMMAX(row_min, (MV_LOW >> 3) + 1); + col_max = AOMMIN(col_max, (MV_UPP >> 3) - 1); + row_max = AOMMIN(row_max, (MV_UPP >> 3) - 1); + + // Get intersection of UMV window and valid MV window to reduce # of checks + // in diamond search. + if (mv_limits->col_min < col_min) mv_limits->col_min = col_min; + if (mv_limits->col_max > col_max) mv_limits->col_max = col_max; + if (mv_limits->row_min < row_min) mv_limits->row_min = row_min; + if (mv_limits->row_max > row_max) mv_limits->row_max = row_max; +} + +static void av1_set_subpel_mv_search_range(const MvLimits *mv_limits, + int *col_min, int *col_max, + int *row_min, int *row_max, + const MV *ref_mv) { + const int max_mv = MAX_FULL_PEL_VAL * 8; + const int minc = AOMMAX(mv_limits->col_min * 8, ref_mv->col - max_mv); + const int maxc = AOMMIN(mv_limits->col_max * 8, ref_mv->col + max_mv); + const int minr = AOMMAX(mv_limits->row_min * 8, ref_mv->row - max_mv); + const int maxr = AOMMIN(mv_limits->row_max * 8, ref_mv->row + max_mv); + + *col_min = AOMMAX(MV_LOW + 1, minc); + *col_max = AOMMIN(MV_UPP - 1, maxc); + *row_min = AOMMAX(MV_LOW + 1, minr); + *row_max = AOMMIN(MV_UPP - 1, maxr); +} + +int av1_init_search_range(int size) { + int sr = 0; + // Minimum search size no matter what the passed in value. + size = AOMMAX(16, size); + + while ((size << sr) < MAX_FULL_PEL_VAL) sr++; + + sr = AOMMIN(sr, MAX_MVSEARCH_STEPS - 2); + return sr; +} + +static INLINE int mv_cost(const MV *mv, const int *joint_cost, + int *const comp_cost[2]) { + return joint_cost[av1_get_mv_joint(mv)] + comp_cost[0][mv->row] + + comp_cost[1][mv->col]; +} + +int av1_mv_bit_cost(const MV *mv, const MV *ref, const int *mvjcost, + int *mvcost[2], int weight) { + const MV diff = { mv->row - ref->row, mv->col - ref->col }; + return ROUND_POWER_OF_TWO(mv_cost(&diff, mvjcost, mvcost) * weight, 7); +} + +#define PIXEL_TRANSFORM_ERROR_SCALE 4 +static int mv_err_cost(const MV *mv, const MV *ref, const int *mvjcost, + int *mvcost[2], int error_per_bit) { + if (mvcost) { + const MV diff = { mv->row - ref->row, mv->col - ref->col }; + return (int)ROUND_POWER_OF_TWO_64( + (int64_t)mv_cost(&diff, mvjcost, mvcost) * error_per_bit, + RDDIV_BITS + AV1_PROB_COST_SHIFT - RD_EPB_SHIFT + + PIXEL_TRANSFORM_ERROR_SCALE); + } + return 0; +} + +static int mvsad_err_cost(const MACROBLOCK *x, const MV *mv, const MV *ref, + int sad_per_bit) { + const MV diff = { (mv->row - ref->row) * 8, (mv->col - ref->col) * 8 }; + return ROUND_POWER_OF_TWO( + (unsigned)mv_cost(&diff, x->nmvjointsadcost, x->mvsadcost) * sad_per_bit, + AV1_PROB_COST_SHIFT); +} + +void av1_init_dsmotion_compensation(search_site_config *cfg, int stride) { + int len, ss_count = 1; + + cfg->ss[0].mv.col = cfg->ss[0].mv.row = 0; + cfg->ss[0].offset = 0; + + for (len = MAX_FIRST_STEP; len > 0; len /= 2) { + // Generate offsets for 4 search sites per step. + const MV ss_mvs[] = { { -len, 0 }, { len, 0 }, { 0, -len }, { 0, len } }; + int i; + for (i = 0; i < 4; ++i) { + search_site *const ss = &cfg->ss[ss_count++]; + ss->mv = ss_mvs[i]; + ss->offset = ss->mv.row * stride + ss->mv.col; + } + } + + cfg->ss_count = ss_count; + cfg->searches_per_step = 4; +} + +void av1_init3smotion_compensation(search_site_config *cfg, int stride) { + int len, ss_count = 1; + + cfg->ss[0].mv.col = cfg->ss[0].mv.row = 0; + cfg->ss[0].offset = 0; + + for (len = MAX_FIRST_STEP; len > 0; len /= 2) { + // Generate offsets for 8 search sites per step. + const MV ss_mvs[8] = { { -len, 0 }, { len, 0 }, { 0, -len }, + { 0, len }, { -len, -len }, { -len, len }, + { len, -len }, { len, len } }; + int i; + for (i = 0; i < 8; ++i) { + search_site *const ss = &cfg->ss[ss_count++]; + ss->mv = ss_mvs[i]; + ss->offset = ss->mv.row * stride + ss->mv.col; + } + } + + cfg->ss_count = ss_count; + cfg->searches_per_step = 8; +} + +/* + * To avoid the penalty for crossing cache-line read, preload the reference + * area in a small buffer, which is aligned to make sure there won't be crossing + * cache-line read while reading from this buffer. This reduced the cpu + * cycles spent on reading ref data in sub-pixel filter functions. + * TODO: Currently, since sub-pixel search range here is -3 ~ 3, copy 22 rows x + * 32 cols area that is enough for 16x16 macroblock. Later, for SPLITMV, we + * could reduce the area. + */ + +// convert motion vector component to offset for sv[a]f calc +static INLINE int sp(int x) { return x & 7; } + +static INLINE const uint8_t *pre(const uint8_t *buf, int stride, int r, int c) { + return &buf[(r >> 3) * stride + (c >> 3)]; +} + +/* checks if (r, c) has better score than previous best */ +#define CHECK_BETTER(v, r, c) \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + MV this_mv = { r, c }; \ + v = mv_err_cost(&this_mv, ref_mv, mvjcost, mvcost, error_per_bit); \ + if (second_pred == NULL) \ + thismse = vfp->svf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), \ + src_address, src_stride, &sse); \ + else \ + thismse = vfp->svaf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), \ + src_address, src_stride, &sse, second_pred); \ + v += thismse; \ + if (v < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } + +#define CHECK_BETTER0(v, r, c) CHECK_BETTER(v, r, c) + +static INLINE const uint8_t *upre(const uint8_t *buf, int stride, int r, + int c) { + return &buf[(r)*stride + (c)]; +} + +/* checks if (r, c) has better score than previous best */ +#define CHECK_BETTER1(v, r, c) \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + MV this_mv = { r, c }; \ + thismse = upsampled_pref_error(xd, vfp, src_address, src_stride, \ + upre(y, y_stride, r, c), y_stride, \ + second_pred, w, h, &sse); \ + v = mv_err_cost(&this_mv, ref_mv, mvjcost, mvcost, error_per_bit); \ + v += thismse; \ + if (v < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } + +#define FIRST_LEVEL_CHECKS \ + { \ + unsigned int left, right, up, down, diag; \ + CHECK_BETTER(left, tr, tc - hstep); \ + CHECK_BETTER(right, tr, tc + hstep); \ + CHECK_BETTER(up, tr - hstep, tc); \ + CHECK_BETTER(down, tr + hstep, tc); \ + whichdir = (left < right ? 0 : 1) + (up < down ? 0 : 2); \ + switch (whichdir) { \ + case 0: CHECK_BETTER(diag, tr - hstep, tc - hstep); break; \ + case 1: CHECK_BETTER(diag, tr - hstep, tc + hstep); break; \ + case 2: CHECK_BETTER(diag, tr + hstep, tc - hstep); break; \ + case 3: CHECK_BETTER(diag, tr + hstep, tc + hstep); break; \ + } \ + } + +#define SECOND_LEVEL_CHECKS \ + { \ + int kr, kc; \ + unsigned int second; \ + if (tr != br && tc != bc) { \ + kr = br - tr; \ + kc = bc - tc; \ + CHECK_BETTER(second, tr + kr, tc + 2 * kc); \ + CHECK_BETTER(second, tr + 2 * kr, tc + kc); \ + } else if (tr == br && tc != bc) { \ + kc = bc - tc; \ + CHECK_BETTER(second, tr + hstep, tc + 2 * kc); \ + CHECK_BETTER(second, tr - hstep, tc + 2 * kc); \ + switch (whichdir) { \ + case 0: \ + case 1: CHECK_BETTER(second, tr + hstep, tc + kc); break; \ + case 2: \ + case 3: CHECK_BETTER(second, tr - hstep, tc + kc); break; \ + } \ + } else if (tr != br && tc == bc) { \ + kr = br - tr; \ + CHECK_BETTER(second, tr + 2 * kr, tc + hstep); \ + CHECK_BETTER(second, tr + 2 * kr, tc - hstep); \ + switch (whichdir) { \ + case 0: \ + case 2: CHECK_BETTER(second, tr + kr, tc + hstep); break; \ + case 1: \ + case 3: CHECK_BETTER(second, tr + kr, tc - hstep); break; \ + } \ + } \ + } + +// TODO(yunqingwang): SECOND_LEVEL_CHECKS_BEST was a rewrote of +// SECOND_LEVEL_CHECKS, and SECOND_LEVEL_CHECKS should be rewritten +// later in the same way. +#define SECOND_LEVEL_CHECKS_BEST(k) \ + { \ + unsigned int second; \ + int br0 = br; \ + int bc0 = bc; \ + assert(tr == br || tc == bc); \ + if (tr == br && tc != bc) { \ + kc = bc - tc; \ + } else if (tr != br && tc == bc) { \ + kr = br - tr; \ + } \ + CHECK_BETTER##k(second, br0 + kr, bc0); \ + CHECK_BETTER##k(second, br0, bc0 + kc); \ + if (br0 != br || bc0 != bc) { \ + CHECK_BETTER##k(second, br0 + kr, bc0 + kc); \ + } \ + } + +#define SETUP_SUBPEL_SEARCH \ + const uint8_t *const src_address = x->plane[0].src.buf; \ + const int src_stride = x->plane[0].src.stride; \ + const MACROBLOCKD *xd = &x->e_mbd; \ + unsigned int besterr = INT_MAX; \ + unsigned int sse; \ + unsigned int whichdir; \ + int thismse; \ + MV *bestmv = &x->best_mv.as_mv; \ + const unsigned int halfiters = iters_per_step; \ + const unsigned int quarteriters = iters_per_step; \ + const unsigned int eighthiters = iters_per_step; \ + const int y_stride = xd->plane[0].pre[0].stride; \ + const int offset = bestmv->row * y_stride + bestmv->col; \ + const uint8_t *const y = xd->plane[0].pre[0].buf; \ + \ + int br = bestmv->row * 8; \ + int bc = bestmv->col * 8; \ + int hstep = 4; \ + int minc, maxc, minr, maxr; \ + int tr = br; \ + int tc = bc; \ + \ + av1_set_subpel_mv_search_range(&x->mv_limits, &minc, &maxc, &minr, &maxr, \ + ref_mv); \ + \ + bestmv->row *= 8; \ + bestmv->col *= 8; + +static unsigned int setup_center_error( + const MACROBLOCKD *xd, const MV *bestmv, const MV *ref_mv, + int error_per_bit, const aom_variance_fn_ptr_t *vfp, + const uint8_t *const src, const int src_stride, const uint8_t *const y, + int y_stride, const uint8_t *second_pred, int w, int h, int offset, + int *mvjcost, int *mvcost[2], unsigned int *sse1, int *distortion) { + unsigned int besterr; +#if CONFIG_HIGHBITDEPTH + if (second_pred != NULL) { + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + DECLARE_ALIGNED(16, uint16_t, comp_pred16[MAX_SB_SQUARE]); + aom_highbd_comp_avg_pred(comp_pred16, second_pred, w, h, y + offset, + y_stride); + besterr = + vfp->vf(CONVERT_TO_BYTEPTR(comp_pred16), w, src, src_stride, sse1); + } else { + DECLARE_ALIGNED(16, uint8_t, comp_pred[MAX_SB_SQUARE]); + aom_comp_avg_pred(comp_pred, second_pred, w, h, y + offset, y_stride); + besterr = vfp->vf(comp_pred, w, src, src_stride, sse1); + } + } else { + besterr = vfp->vf(y + offset, y_stride, src, src_stride, sse1); + } + *distortion = besterr; + besterr += mv_err_cost(bestmv, ref_mv, mvjcost, mvcost, error_per_bit); +#else + (void)xd; + if (second_pred != NULL) { + DECLARE_ALIGNED(16, uint8_t, comp_pred[MAX_SB_SQUARE]); + aom_comp_avg_pred(comp_pred, second_pred, w, h, y + offset, y_stride); + besterr = vfp->vf(comp_pred, w, src, src_stride, sse1); + } else { + besterr = vfp->vf(y + offset, y_stride, src, src_stride, sse1); + } + *distortion = besterr; + besterr += mv_err_cost(bestmv, ref_mv, mvjcost, mvcost, error_per_bit); +#endif // CONFIG_HIGHBITDEPTH + return besterr; +} + +static INLINE int divide_and_round(int n, int d) { + return ((n < 0) ^ (d < 0)) ? ((n - d / 2) / d) : ((n + d / 2) / d); +} + +static INLINE int is_cost_list_wellbehaved(int *cost_list) { + return cost_list[0] < cost_list[1] && cost_list[0] < cost_list[2] && + cost_list[0] < cost_list[3] && cost_list[0] < cost_list[4]; +} + +// Returns surface minima estimate at given precision in 1/2^n bits. +// Assume a model for the cost surface: S = A(x - x0)^2 + B(y - y0)^2 + C +// For a given set of costs S0, S1, S2, S3, S4 at points +// (y, x) = (0, 0), (0, -1), (1, 0), (0, 1) and (-1, 0) respectively, +// the solution for the location of the minima (x0, y0) is given by: +// x0 = 1/2 (S1 - S3)/(S1 + S3 - 2*S0), +// y0 = 1/2 (S4 - S2)/(S4 + S2 - 2*S0). +// The code below is an integerized version of that. +static void get_cost_surf_min(int *cost_list, int *ir, int *ic, int bits) { + *ic = divide_and_round((cost_list[1] - cost_list[3]) * (1 << (bits - 1)), + (cost_list[1] - 2 * cost_list[0] + cost_list[3])); + *ir = divide_and_round((cost_list[4] - cost_list[2]) * (1 << (bits - 1)), + (cost_list[4] - 2 * cost_list[0] + cost_list[2])); +} + +int av1_find_best_sub_pixel_tree_pruned_evenmore( + MACROBLOCK *x, const MV *ref_mv, int allow_hp, int error_per_bit, + const aom_variance_fn_ptr_t *vfp, int forced_stop, int iters_per_step, + int *cost_list, int *mvjcost, int *mvcost[2], int *distortion, + unsigned int *sse1, const uint8_t *second_pred, int w, int h, + int use_upsampled_ref) { + SETUP_SUBPEL_SEARCH; + besterr = setup_center_error( + xd, bestmv, ref_mv, error_per_bit, vfp, src_address, src_stride, y, + y_stride, second_pred, w, h, offset, mvjcost, mvcost, sse1, distortion); + (void)halfiters; + (void)quarteriters; + (void)eighthiters; + (void)whichdir; + (void)allow_hp; + (void)forced_stop; + (void)hstep; + (void)use_upsampled_ref; + + if (cost_list && cost_list[0] != INT_MAX && cost_list[1] != INT_MAX && + cost_list[2] != INT_MAX && cost_list[3] != INT_MAX && + cost_list[4] != INT_MAX && is_cost_list_wellbehaved(cost_list)) { + int ir, ic; + unsigned int minpt; + get_cost_surf_min(cost_list, &ir, &ic, 2); + if (ir != 0 || ic != 0) { + CHECK_BETTER(minpt, tr + 2 * ir, tc + 2 * ic); + } + } else { + FIRST_LEVEL_CHECKS; + if (halfiters > 1) { + SECOND_LEVEL_CHECKS; + } + + tr = br; + tc = bc; + + // Each subsequent iteration checks at least one point in common with + // the last iteration could be 2 ( if diag selected) 1/4 pel + // Note forced_stop: 0 - full, 1 - qtr only, 2 - half only + if (forced_stop != 2) { + hstep >>= 1; + FIRST_LEVEL_CHECKS; + if (quarteriters > 1) { + SECOND_LEVEL_CHECKS; + } + } + } + + tr = br; + tc = bc; + + if (allow_hp && forced_stop == 0) { + hstep >>= 1; + FIRST_LEVEL_CHECKS; + if (eighthiters > 1) { + SECOND_LEVEL_CHECKS; + } + } + + bestmv->row = br; + bestmv->col = bc; + + return besterr; +} + +int av1_find_best_sub_pixel_tree_pruned_more( + MACROBLOCK *x, const MV *ref_mv, int allow_hp, int error_per_bit, + const aom_variance_fn_ptr_t *vfp, int forced_stop, int iters_per_step, + int *cost_list, int *mvjcost, int *mvcost[2], int *distortion, + unsigned int *sse1, const uint8_t *second_pred, int w, int h, + int use_upsampled_ref) { + SETUP_SUBPEL_SEARCH; + (void)use_upsampled_ref; + + besterr = setup_center_error( + xd, bestmv, ref_mv, error_per_bit, vfp, src_address, src_stride, y, + y_stride, second_pred, w, h, offset, mvjcost, mvcost, sse1, distortion); + if (cost_list && cost_list[0] != INT_MAX && cost_list[1] != INT_MAX && + cost_list[2] != INT_MAX && cost_list[3] != INT_MAX && + cost_list[4] != INT_MAX && is_cost_list_wellbehaved(cost_list)) { + unsigned int minpt; + int ir, ic; + get_cost_surf_min(cost_list, &ir, &ic, 1); + if (ir != 0 || ic != 0) { + CHECK_BETTER(minpt, tr + ir * hstep, tc + ic * hstep); + } + } else { + FIRST_LEVEL_CHECKS; + if (halfiters > 1) { + SECOND_LEVEL_CHECKS; + } + } + + // Each subsequent iteration checks at least one point in common with + // the last iteration could be 2 ( if diag selected) 1/4 pel + + // Note forced_stop: 0 - full, 1 - qtr only, 2 - half only + if (forced_stop != 2) { + tr = br; + tc = bc; + hstep >>= 1; + FIRST_LEVEL_CHECKS; + if (quarteriters > 1) { + SECOND_LEVEL_CHECKS; + } + } + + if (allow_hp && forced_stop == 0) { + tr = br; + tc = bc; + hstep >>= 1; + FIRST_LEVEL_CHECKS; + if (eighthiters > 1) { + SECOND_LEVEL_CHECKS; + } + } + // These lines insure static analysis doesn't warn that + // tr and tc aren't used after the above point. + (void)tr; + (void)tc; + + bestmv->row = br; + bestmv->col = bc; + + return besterr; +} + +int av1_find_best_sub_pixel_tree_pruned( + MACROBLOCK *x, const MV *ref_mv, int allow_hp, int error_per_bit, + const aom_variance_fn_ptr_t *vfp, int forced_stop, int iters_per_step, + int *cost_list, int *mvjcost, int *mvcost[2], int *distortion, + unsigned int *sse1, const uint8_t *second_pred, int w, int h, + int use_upsampled_ref) { + SETUP_SUBPEL_SEARCH; + (void)use_upsampled_ref; + + besterr = setup_center_error( + xd, bestmv, ref_mv, error_per_bit, vfp, src_address, src_stride, y, + y_stride, second_pred, w, h, offset, mvjcost, mvcost, sse1, distortion); + if (cost_list && cost_list[0] != INT_MAX && cost_list[1] != INT_MAX && + cost_list[2] != INT_MAX && cost_list[3] != INT_MAX && + cost_list[4] != INT_MAX) { + unsigned int left, right, up, down, diag; + whichdir = (cost_list[1] < cost_list[3] ? 0 : 1) + + (cost_list[2] < cost_list[4] ? 0 : 2); + switch (whichdir) { + case 0: + CHECK_BETTER(left, tr, tc - hstep); + CHECK_BETTER(down, tr + hstep, tc); + CHECK_BETTER(diag, tr + hstep, tc - hstep); + break; + case 1: + CHECK_BETTER(right, tr, tc + hstep); + CHECK_BETTER(down, tr + hstep, tc); + CHECK_BETTER(diag, tr + hstep, tc + hstep); + break; + case 2: + CHECK_BETTER(left, tr, tc - hstep); + CHECK_BETTER(up, tr - hstep, tc); + CHECK_BETTER(diag, tr - hstep, tc - hstep); + break; + case 3: + CHECK_BETTER(right, tr, tc + hstep); + CHECK_BETTER(up, tr - hstep, tc); + CHECK_BETTER(diag, tr - hstep, tc + hstep); + break; + } + } else { + FIRST_LEVEL_CHECKS; + if (halfiters > 1) { + SECOND_LEVEL_CHECKS; + } + } + + tr = br; + tc = bc; + + // Each subsequent iteration checks at least one point in common with + // the last iteration could be 2 ( if diag selected) 1/4 pel + + // Note forced_stop: 0 - full, 1 - qtr only, 2 - half only + if (forced_stop != 2) { + hstep >>= 1; + FIRST_LEVEL_CHECKS; + if (quarteriters > 1) { + SECOND_LEVEL_CHECKS; + } + tr = br; + tc = bc; + } + + if (allow_hp && forced_stop == 0) { + hstep >>= 1; + FIRST_LEVEL_CHECKS; + if (eighthiters > 1) { + SECOND_LEVEL_CHECKS; + } + tr = br; + tc = bc; + } + // These lines insure static analysis doesn't warn that + // tr and tc aren't used after the above point. + (void)tr; + (void)tc; + + bestmv->row = br; + bestmv->col = bc; + + return besterr; +} + +/* clang-format off */ +static const MV search_step_table[12] = { + // left, right, up, down + { 0, -4 }, { 0, 4 }, { -4, 0 }, { 4, 0 }, + { 0, -2 }, { 0, 2 }, { -2, 0 }, { 2, 0 }, + { 0, -1 }, { 0, 1 }, { -1, 0 }, { 1, 0 } +}; +/* clang-format on */ + +static int upsampled_pref_error(const MACROBLOCKD *xd, + const aom_variance_fn_ptr_t *vfp, + const uint8_t *const src, const int src_stride, + const uint8_t *const y, int y_stride, + const uint8_t *second_pred, int w, int h, + unsigned int *sse) { + unsigned int besterr; +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + DECLARE_ALIGNED(16, uint16_t, pred16[MAX_SB_SQUARE]); + if (second_pred != NULL) + aom_highbd_comp_avg_upsampled_pred(pred16, second_pred, w, h, y, + y_stride); + else + aom_highbd_upsampled_pred(pred16, w, h, y, y_stride); + + besterr = vfp->vf(CONVERT_TO_BYTEPTR(pred16), w, src, src_stride, sse); + } else { + DECLARE_ALIGNED(16, uint8_t, pred[MAX_SB_SQUARE]); +#else + DECLARE_ALIGNED(16, uint8_t, pred[MAX_SB_SQUARE]); + (void)xd; +#endif // CONFIG_HIGHBITDEPTH + if (second_pred != NULL) + aom_comp_avg_upsampled_pred(pred, second_pred, w, h, y, y_stride); + else + aom_upsampled_pred(pred, w, h, y, y_stride); + + besterr = vfp->vf(pred, w, src, src_stride, sse); +#if CONFIG_HIGHBITDEPTH + } +#endif + return besterr; +} + +static unsigned int upsampled_setup_center_error( + const MACROBLOCKD *xd, const MV *bestmv, const MV *ref_mv, + int error_per_bit, const aom_variance_fn_ptr_t *vfp, + const uint8_t *const src, const int src_stride, const uint8_t *const y, + int y_stride, const uint8_t *second_pred, int w, int h, int offset, + int *mvjcost, int *mvcost[2], unsigned int *sse1, int *distortion) { + unsigned int besterr = upsampled_pref_error( + xd, vfp, src, src_stride, y + offset, y_stride, second_pred, w, h, sse1); + *distortion = besterr; + besterr += mv_err_cost(bestmv, ref_mv, mvjcost, mvcost, error_per_bit); + return besterr; +} + +int av1_find_best_sub_pixel_tree(MACROBLOCK *x, const MV *ref_mv, int allow_hp, + int error_per_bit, + const aom_variance_fn_ptr_t *vfp, + int forced_stop, int iters_per_step, + int *cost_list, int *mvjcost, int *mvcost[2], + int *distortion, unsigned int *sse1, + const uint8_t *second_pred, int w, int h, + int use_upsampled_ref) { + const uint8_t *const src_address = x->plane[0].src.buf; + const int src_stride = x->plane[0].src.stride; + const MACROBLOCKD *xd = &x->e_mbd; + unsigned int besterr = INT_MAX; + unsigned int sse; + unsigned int thismse; + const int y_stride = xd->plane[0].pre[0].stride; + MV *bestmv = &x->best_mv.as_mv; + const int offset = bestmv->row * y_stride + bestmv->col; + const uint8_t *const y = xd->plane[0].pre[0].buf; + + int br = bestmv->row * 8; + int bc = bestmv->col * 8; + int hstep = 4; + int iter, round = 3 - forced_stop; + int tr = br; + int tc = bc; + const MV *search_step = search_step_table; + int idx, best_idx = -1; + unsigned int cost_array[5]; + int kr, kc; + int minc, maxc, minr, maxr; + + av1_set_subpel_mv_search_range(&x->mv_limits, &minc, &maxc, &minr, &maxr, + ref_mv); + + if (!allow_hp) + if (round == 3) round = 2; + + bestmv->row *= 8; + bestmv->col *= 8; + + // use_upsampled_ref can be 0 or 1 + if (use_upsampled_ref) + besterr = upsampled_setup_center_error( + xd, bestmv, ref_mv, error_per_bit, vfp, src_address, src_stride, y, + y_stride, second_pred, w, h, (offset * 8), mvjcost, mvcost, sse1, + distortion); + else + besterr = setup_center_error( + xd, bestmv, ref_mv, error_per_bit, vfp, src_address, src_stride, y, + y_stride, second_pred, w, h, offset, mvjcost, mvcost, sse1, distortion); + + (void)cost_list; // to silence compiler warning + + for (iter = 0; iter < round; ++iter) { + // Check vertical and horizontal sub-pixel positions. + for (idx = 0; idx < 4; ++idx) { + tr = br + search_step[idx].row; + tc = bc + search_step[idx].col; + if (tc >= minc && tc <= maxc && tr >= minr && tr <= maxr) { + MV this_mv = { tr, tc }; + + if (use_upsampled_ref) { + const uint8_t *const pre_address = y + tr * y_stride + tc; + + thismse = upsampled_pref_error(xd, vfp, src_address, src_stride, + pre_address, y_stride, second_pred, w, + h, &sse); + } else { + const uint8_t *const pre_address = + y + (tr >> 3) * y_stride + (tc >> 3); + if (second_pred == NULL) + thismse = vfp->svf(pre_address, y_stride, sp(tc), sp(tr), + src_address, src_stride, &sse); + else + thismse = vfp->svaf(pre_address, y_stride, sp(tc), sp(tr), + src_address, src_stride, &sse, second_pred); + } + + cost_array[idx] = thismse + mv_err_cost(&this_mv, ref_mv, mvjcost, + mvcost, error_per_bit); + + if (cost_array[idx] < besterr) { + best_idx = idx; + besterr = cost_array[idx]; + *distortion = thismse; + *sse1 = sse; + } + } else { + cost_array[idx] = INT_MAX; + } + } + + // Check diagonal sub-pixel position + kc = (cost_array[0] <= cost_array[1] ? -hstep : hstep); + kr = (cost_array[2] <= cost_array[3] ? -hstep : hstep); + + tc = bc + kc; + tr = br + kr; + if (tc >= minc && tc <= maxc && tr >= minr && tr <= maxr) { + MV this_mv = { tr, tc }; + + if (use_upsampled_ref) { + const uint8_t *const pre_address = y + tr * y_stride + tc; + + thismse = + upsampled_pref_error(xd, vfp, src_address, src_stride, pre_address, + y_stride, second_pred, w, h, &sse); + } else { + const uint8_t *const pre_address = y + (tr >> 3) * y_stride + (tc >> 3); + + if (second_pred == NULL) + thismse = vfp->svf(pre_address, y_stride, sp(tc), sp(tr), src_address, + src_stride, &sse); + else + thismse = vfp->svaf(pre_address, y_stride, sp(tc), sp(tr), + src_address, src_stride, &sse, second_pred); + } + + cost_array[4] = thismse + mv_err_cost(&this_mv, ref_mv, mvjcost, mvcost, + error_per_bit); + + if (cost_array[4] < besterr) { + best_idx = 4; + besterr = cost_array[4]; + *distortion = thismse; + *sse1 = sse; + } + } else { + cost_array[idx] = INT_MAX; + } + + if (best_idx < 4 && best_idx >= 0) { + br += search_step[best_idx].row; + bc += search_step[best_idx].col; + } else if (best_idx == 4) { + br = tr; + bc = tc; + } + + if (iters_per_step > 1 && best_idx != -1) { + if (use_upsampled_ref) { + SECOND_LEVEL_CHECKS_BEST(1); + } else { + SECOND_LEVEL_CHECKS_BEST(0); + } + } + + search_step += 4; + hstep >>= 1; + best_idx = -1; + } + + // These lines insure static analysis doesn't warn that + // tr and tc aren't used after the above point. + (void)tr; + (void)tc; + + bestmv->row = br; + bestmv->col = bc; + + return besterr; +} + +#undef PRE +#undef CHECK_BETTER + +static INLINE int check_bounds(const MvLimits *mv_limits, int row, int col, + int range) { + return ((row - range) >= mv_limits->row_min) & + ((row + range) <= mv_limits->row_max) & + ((col - range) >= mv_limits->col_min) & + ((col + range) <= mv_limits->col_max); +} + +static INLINE int is_mv_in(const MvLimits *mv_limits, const MV *mv) { + return (mv->col >= mv_limits->col_min) && (mv->col <= mv_limits->col_max) && + (mv->row >= mv_limits->row_min) && (mv->row <= mv_limits->row_max); +} + +#define CHECK_BETTER \ + { \ + if (thissad < bestsad) { \ + if (use_mvcost) \ + thissad += mvsad_err_cost(x, &this_mv, &fcenter_mv, sad_per_bit); \ + if (thissad < bestsad) { \ + bestsad = thissad; \ + best_site = i; \ + } \ + } \ + } + +#define MAX_PATTERN_SCALES 11 +#define MAX_PATTERN_CANDIDATES 8 // max number of canddiates per scale +#define PATTERN_CANDIDATES_REF 3 // number of refinement candidates + +// Calculate and return a sad+mvcost list around an integer best pel. +static INLINE void calc_int_cost_list(const MACROBLOCK *x, + const MV *const ref_mv, int sadpb, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *best_mv, int *cost_list) { + static const MV neighbors[4] = { { 0, -1 }, { 1, 0 }, { 0, 1 }, { -1, 0 } }; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &x->e_mbd.plane[0].pre[0]; + const MV fcenter_mv = { ref_mv->row >> 3, ref_mv->col >> 3 }; + const int br = best_mv->row; + const int bc = best_mv->col; + int i; + unsigned int sse; + const MV this_mv = { br, bc }; + + cost_list[0] = + fn_ptr->vf(what->buf, what->stride, get_buf_from_mv(in_what, &this_mv), + in_what->stride, &sse) + + mvsad_err_cost(x, &this_mv, &fcenter_mv, sadpb); + if (check_bounds(&x->mv_limits, br, bc, 1)) { + for (i = 0; i < 4; i++) { + const MV neighbor_mv = { br + neighbors[i].row, bc + neighbors[i].col }; + cost_list[i + 1] = fn_ptr->vf(what->buf, what->stride, + get_buf_from_mv(in_what, &neighbor_mv), + in_what->stride, &sse) + + mv_err_cost(&neighbor_mv, &fcenter_mv, x->nmvjointcost, + x->mvcost, x->errorperbit); + } + } else { + for (i = 0; i < 4; i++) { + const MV neighbor_mv = { br + neighbors[i].row, bc + neighbors[i].col }; + if (!is_mv_in(&x->mv_limits, &neighbor_mv)) + cost_list[i + 1] = INT_MAX; + else + cost_list[i + 1] = + fn_ptr->vf(what->buf, what->stride, + get_buf_from_mv(in_what, &neighbor_mv), in_what->stride, + &sse) + + mv_err_cost(&neighbor_mv, &fcenter_mv, x->nmvjointcost, x->mvcost, + x->errorperbit); + } + } +} + +static INLINE void calc_int_sad_list(const MACROBLOCK *x, + const MV *const ref_mv, int sadpb, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *best_mv, int *cost_list, + const int use_mvcost, const int bestsad) { + static const MV neighbors[4] = { { 0, -1 }, { 1, 0 }, { 0, 1 }, { -1, 0 } }; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &x->e_mbd.plane[0].pre[0]; + const MV fcenter_mv = { ref_mv->row >> 3, ref_mv->col >> 3 }; + int i; + const int br = best_mv->row; + const int bc = best_mv->col; + + if (cost_list[0] == INT_MAX) { + cost_list[0] = bestsad; + if (check_bounds(&x->mv_limits, br, bc, 1)) { + for (i = 0; i < 4; i++) { + const MV this_mv = { br + neighbors[i].row, bc + neighbors[i].col }; + cost_list[i + 1] = + fn_ptr->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &this_mv), in_what->stride); + } + } else { + for (i = 0; i < 4; i++) { + const MV this_mv = { br + neighbors[i].row, bc + neighbors[i].col }; + if (!is_mv_in(&x->mv_limits, &this_mv)) + cost_list[i + 1] = INT_MAX; + else + cost_list[i + 1] = + fn_ptr->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &this_mv), in_what->stride); + } + } + } else { + if (use_mvcost) { + for (i = 0; i < 4; i++) { + const MV this_mv = { br + neighbors[i].row, bc + neighbors[i].col }; + if (cost_list[i + 1] != INT_MAX) { + cost_list[i + 1] += mvsad_err_cost(x, &this_mv, &fcenter_mv, sadpb); + } + } + } + } +} + +// Generic pattern search function that searches over multiple scales. +// Each scale can have a different number of candidates and shape of +// candidates as indicated in the num_candidates and candidates arrays +// passed into this function +// +static int pattern_search( + MACROBLOCK *x, MV *start_mv, int search_param, int sad_per_bit, + int do_init_search, int *cost_list, const aom_variance_fn_ptr_t *vfp, + int use_mvcost, const MV *center_mv, + const int num_candidates[MAX_PATTERN_SCALES], + const MV candidates[MAX_PATTERN_SCALES][MAX_PATTERN_CANDIDATES]) { + const MACROBLOCKD *const xd = &x->e_mbd; + static const int search_param_to_steps[MAX_MVSEARCH_STEPS] = { + 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + }; + int i, s, t; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &xd->plane[0].pre[0]; + const int last_is_4 = num_candidates[0] == 4; + int br, bc; + int bestsad = INT_MAX; + int thissad; + int k = -1; + const MV fcenter_mv = { center_mv->row >> 3, center_mv->col >> 3 }; + int best_init_s = search_param_to_steps[search_param]; + // adjust ref_mv to make sure it is within MV range + clamp_mv(start_mv, x->mv_limits.col_min, x->mv_limits.col_max, + x->mv_limits.row_min, x->mv_limits.row_max); + br = start_mv->row; + bc = start_mv->col; + if (cost_list != NULL) { + cost_list[0] = cost_list[1] = cost_list[2] = cost_list[3] = cost_list[4] = + INT_MAX; + } + + // Work out the start point for the search + bestsad = vfp->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, start_mv), in_what->stride) + + mvsad_err_cost(x, start_mv, &fcenter_mv, sad_per_bit); + + // Search all possible scales upto the search param around the center point + // pick the scale of the point that is best as the starting scale of + // further steps around it. + if (do_init_search) { + s = best_init_s; + best_init_s = -1; + for (t = 0; t <= s; ++t) { + int best_site = -1; + if (check_bounds(&x->mv_limits, br, bc, 1 << t)) { + for (i = 0; i < num_candidates[t]; i++) { + const MV this_mv = { br + candidates[t][i].row, + bc + candidates[t][i].col }; + thissad = + vfp->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &this_mv), in_what->stride); + CHECK_BETTER + } + } else { + for (i = 0; i < num_candidates[t]; i++) { + const MV this_mv = { br + candidates[t][i].row, + bc + candidates[t][i].col }; + if (!is_mv_in(&x->mv_limits, &this_mv)) continue; + thissad = + vfp->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &this_mv), in_what->stride); + CHECK_BETTER + } + } + if (best_site == -1) { + continue; + } else { + best_init_s = t; + k = best_site; + } + } + if (best_init_s != -1) { + br += candidates[best_init_s][k].row; + bc += candidates[best_init_s][k].col; + } + } + + // If the center point is still the best, just skip this and move to + // the refinement step. + if (best_init_s != -1) { + const int last_s = (last_is_4 && cost_list != NULL); + int best_site = -1; + s = best_init_s; + + for (; s >= last_s; s--) { + // No need to search all points the 1st time if initial search was used + if (!do_init_search || s != best_init_s) { + if (check_bounds(&x->mv_limits, br, bc, 1 << s)) { + for (i = 0; i < num_candidates[s]; i++) { + const MV this_mv = { br + candidates[s][i].row, + bc + candidates[s][i].col }; + thissad = + vfp->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &this_mv), in_what->stride); + CHECK_BETTER + } + } else { + for (i = 0; i < num_candidates[s]; i++) { + const MV this_mv = { br + candidates[s][i].row, + bc + candidates[s][i].col }; + if (!is_mv_in(&x->mv_limits, &this_mv)) continue; + thissad = + vfp->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &this_mv), in_what->stride); + CHECK_BETTER + } + } + + if (best_site == -1) { + continue; + } else { + br += candidates[s][best_site].row; + bc += candidates[s][best_site].col; + k = best_site; + } + } + + do { + int next_chkpts_indices[PATTERN_CANDIDATES_REF]; + best_site = -1; + next_chkpts_indices[0] = (k == 0) ? num_candidates[s] - 1 : k - 1; + next_chkpts_indices[1] = k; + next_chkpts_indices[2] = (k == num_candidates[s] - 1) ? 0 : k + 1; + + if (check_bounds(&x->mv_limits, br, bc, 1 << s)) { + for (i = 0; i < PATTERN_CANDIDATES_REF; i++) { + const MV this_mv = { + br + candidates[s][next_chkpts_indices[i]].row, + bc + candidates[s][next_chkpts_indices[i]].col + }; + thissad = + vfp->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &this_mv), in_what->stride); + CHECK_BETTER + } + } else { + for (i = 0; i < PATTERN_CANDIDATES_REF; i++) { + const MV this_mv = { + br + candidates[s][next_chkpts_indices[i]].row, + bc + candidates[s][next_chkpts_indices[i]].col + }; + if (!is_mv_in(&x->mv_limits, &this_mv)) continue; + thissad = + vfp->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &this_mv), in_what->stride); + CHECK_BETTER + } + } + + if (best_site != -1) { + k = next_chkpts_indices[best_site]; + br += candidates[s][k].row; + bc += candidates[s][k].col; + } + } while (best_site != -1); + } + + // Note: If we enter the if below, then cost_list must be non-NULL. + if (s == 0) { + cost_list[0] = bestsad; + if (!do_init_search || s != best_init_s) { + if (check_bounds(&x->mv_limits, br, bc, 1 << s)) { + for (i = 0; i < num_candidates[s]; i++) { + const MV this_mv = { br + candidates[s][i].row, + bc + candidates[s][i].col }; + cost_list[i + 1] = thissad = + vfp->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &this_mv), in_what->stride); + CHECK_BETTER + } + } else { + for (i = 0; i < num_candidates[s]; i++) { + const MV this_mv = { br + candidates[s][i].row, + bc + candidates[s][i].col }; + if (!is_mv_in(&x->mv_limits, &this_mv)) continue; + cost_list[i + 1] = thissad = + vfp->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &this_mv), in_what->stride); + CHECK_BETTER + } + } + + if (best_site != -1) { + br += candidates[s][best_site].row; + bc += candidates[s][best_site].col; + k = best_site; + } + } + while (best_site != -1) { + int next_chkpts_indices[PATTERN_CANDIDATES_REF]; + best_site = -1; + next_chkpts_indices[0] = (k == 0) ? num_candidates[s] - 1 : k - 1; + next_chkpts_indices[1] = k; + next_chkpts_indices[2] = (k == num_candidates[s] - 1) ? 0 : k + 1; + cost_list[1] = cost_list[2] = cost_list[3] = cost_list[4] = INT_MAX; + cost_list[((k + 2) % 4) + 1] = cost_list[0]; + cost_list[0] = bestsad; + + if (check_bounds(&x->mv_limits, br, bc, 1 << s)) { + for (i = 0; i < PATTERN_CANDIDATES_REF; i++) { + const MV this_mv = { + br + candidates[s][next_chkpts_indices[i]].row, + bc + candidates[s][next_chkpts_indices[i]].col + }; + cost_list[next_chkpts_indices[i] + 1] = thissad = + vfp->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &this_mv), in_what->stride); + CHECK_BETTER + } + } else { + for (i = 0; i < PATTERN_CANDIDATES_REF; i++) { + const MV this_mv = { + br + candidates[s][next_chkpts_indices[i]].row, + bc + candidates[s][next_chkpts_indices[i]].col + }; + if (!is_mv_in(&x->mv_limits, &this_mv)) { + cost_list[next_chkpts_indices[i] + 1] = INT_MAX; + continue; + } + cost_list[next_chkpts_indices[i] + 1] = thissad = + vfp->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &this_mv), in_what->stride); + CHECK_BETTER + } + } + + if (best_site != -1) { + k = next_chkpts_indices[best_site]; + br += candidates[s][k].row; + bc += candidates[s][k].col; + } + } + } + } + + // Returns the one-away integer pel cost/sad around the best as follows: + // cost_list[0]: cost/sad at the best integer pel + // cost_list[1]: cost/sad at delta {0, -1} (left) from the best integer pel + // cost_list[2]: cost/sad at delta { 1, 0} (bottom) from the best integer pel + // cost_list[3]: cost/sad at delta { 0, 1} (right) from the best integer pel + // cost_list[4]: cost/sad at delta {-1, 0} (top) from the best integer pel + if (cost_list) { + const MV best_int_mv = { br, bc }; + if (last_is_4) { + calc_int_sad_list(x, center_mv, sad_per_bit, vfp, &best_int_mv, cost_list, + use_mvcost, bestsad); + } else { + calc_int_cost_list(x, center_mv, sad_per_bit, vfp, &best_int_mv, + cost_list); + } + } + x->best_mv.as_mv.row = br; + x->best_mv.as_mv.col = bc; + return bestsad; +} + +int av1_get_mvpred_var(const MACROBLOCK *x, const MV *best_mv, + const MV *center_mv, const aom_variance_fn_ptr_t *vfp, + int use_mvcost) { + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &xd->plane[0].pre[0]; + const MV mv = { best_mv->row * 8, best_mv->col * 8 }; + unsigned int unused; + + return vfp->vf(what->buf, what->stride, get_buf_from_mv(in_what, best_mv), + in_what->stride, &unused) + + (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmvjointcost, x->mvcost, + x->errorperbit) + : 0); +} + +int av1_get_mvpred_av_var(const MACROBLOCK *x, const MV *best_mv, + const MV *center_mv, const uint8_t *second_pred, + const aom_variance_fn_ptr_t *vfp, int use_mvcost) { + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &xd->plane[0].pre[0]; + const MV mv = { best_mv->row * 8, best_mv->col * 8 }; + unsigned int unused; + + return vfp->svaf(get_buf_from_mv(in_what, best_mv), in_what->stride, 0, 0, + what->buf, what->stride, &unused, second_pred) + + (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmvjointcost, x->mvcost, + x->errorperbit) + : 0); +} + +int av1_hex_search(MACROBLOCK *x, MV *start_mv, int search_param, + int sad_per_bit, int do_init_search, int *cost_list, + const aom_variance_fn_ptr_t *vfp, int use_mvcost, + const MV *center_mv) { + // First scale has 8-closest points, the rest have 6 points in hex shape + // at increasing scales + static const int hex_num_candidates[MAX_PATTERN_SCALES] = { 8, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6 }; + // Note that the largest candidate step at each scale is 2^scale + /* clang-format off */ + static const MV hex_candidates[MAX_PATTERN_SCALES][MAX_PATTERN_CANDIDATES] = { + { { -1, -1 }, { 0, -1 }, { 1, -1 }, { 1, 0 }, { 1, 1 }, { 0, 1 }, { -1, 1 }, + { -1, 0 } }, + { { -1, -2 }, { 1, -2 }, { 2, 0 }, { 1, 2 }, { -1, 2 }, { -2, 0 } }, + { { -2, -4 }, { 2, -4 }, { 4, 0 }, { 2, 4 }, { -2, 4 }, { -4, 0 } }, + { { -4, -8 }, { 4, -8 }, { 8, 0 }, { 4, 8 }, { -4, 8 }, { -8, 0 } }, + { { -8, -16 }, { 8, -16 }, { 16, 0 }, { 8, 16 }, { -8, 16 }, { -16, 0 } }, + { { -16, -32 }, { 16, -32 }, { 32, 0 }, { 16, 32 }, { -16, 32 }, + { -32, 0 } }, + { { -32, -64 }, { 32, -64 }, { 64, 0 }, { 32, 64 }, { -32, 64 }, + { -64, 0 } }, + { { -64, -128 }, { 64, -128 }, { 128, 0 }, { 64, 128 }, { -64, 128 }, + { -128, 0 } }, + { { -128, -256 }, { 128, -256 }, { 256, 0 }, { 128, 256 }, { -128, 256 }, + { -256, 0 } }, + { { -256, -512 }, { 256, -512 }, { 512, 0 }, { 256, 512 }, { -256, 512 }, + { -512, 0 } }, + { { -512, -1024 }, { 512, -1024 }, { 1024, 0 }, { 512, 1024 }, + { -512, 1024 }, { -1024, 0 } }, + }; + /* clang-format on */ + return pattern_search(x, start_mv, search_param, sad_per_bit, do_init_search, + cost_list, vfp, use_mvcost, center_mv, + hex_num_candidates, hex_candidates); +} + +static int bigdia_search(MACROBLOCK *x, MV *start_mv, int search_param, + int sad_per_bit, int do_init_search, int *cost_list, + const aom_variance_fn_ptr_t *vfp, int use_mvcost, + const MV *center_mv) { + // First scale has 4-closest points, the rest have 8 points in diamond + // shape at increasing scales + static const int bigdia_num_candidates[MAX_PATTERN_SCALES] = { + 4, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + }; + // Note that the largest candidate step at each scale is 2^scale + /* clang-format off */ + static const MV + bigdia_candidates[MAX_PATTERN_SCALES][MAX_PATTERN_CANDIDATES] = { + { { 0, -1 }, { 1, 0 }, { 0, 1 }, { -1, 0 } }, + { { -1, -1 }, { 0, -2 }, { 1, -1 }, { 2, 0 }, { 1, 1 }, { 0, 2 }, + { -1, 1 }, { -2, 0 } }, + { { -2, -2 }, { 0, -4 }, { 2, -2 }, { 4, 0 }, { 2, 2 }, { 0, 4 }, + { -2, 2 }, { -4, 0 } }, + { { -4, -4 }, { 0, -8 }, { 4, -4 }, { 8, 0 }, { 4, 4 }, { 0, 8 }, + { -4, 4 }, { -8, 0 } }, + { { -8, -8 }, { 0, -16 }, { 8, -8 }, { 16, 0 }, { 8, 8 }, { 0, 16 }, + { -8, 8 }, { -16, 0 } }, + { { -16, -16 }, { 0, -32 }, { 16, -16 }, { 32, 0 }, { 16, 16 }, + { 0, 32 }, { -16, 16 }, { -32, 0 } }, + { { -32, -32 }, { 0, -64 }, { 32, -32 }, { 64, 0 }, { 32, 32 }, + { 0, 64 }, { -32, 32 }, { -64, 0 } }, + { { -64, -64 }, { 0, -128 }, { 64, -64 }, { 128, 0 }, { 64, 64 }, + { 0, 128 }, { -64, 64 }, { -128, 0 } }, + { { -128, -128 }, { 0, -256 }, { 128, -128 }, { 256, 0 }, { 128, 128 }, + { 0, 256 }, { -128, 128 }, { -256, 0 } }, + { { -256, -256 }, { 0, -512 }, { 256, -256 }, { 512, 0 }, { 256, 256 }, + { 0, 512 }, { -256, 256 }, { -512, 0 } }, + { { -512, -512 }, { 0, -1024 }, { 512, -512 }, { 1024, 0 }, + { 512, 512 }, { 0, 1024 }, { -512, 512 }, { -1024, 0 } }, + }; + /* clang-format on */ + return pattern_search(x, start_mv, search_param, sad_per_bit, do_init_search, + cost_list, vfp, use_mvcost, center_mv, + bigdia_num_candidates, bigdia_candidates); +} + +static int square_search(MACROBLOCK *x, MV *start_mv, int search_param, + int sad_per_bit, int do_init_search, int *cost_list, + const aom_variance_fn_ptr_t *vfp, int use_mvcost, + const MV *center_mv) { + // All scales have 8 closest points in square shape + static const int square_num_candidates[MAX_PATTERN_SCALES] = { + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + }; + // Note that the largest candidate step at each scale is 2^scale + /* clang-format off */ + static const MV + square_candidates[MAX_PATTERN_SCALES][MAX_PATTERN_CANDIDATES] = { + { { -1, -1 }, { 0, -1 }, { 1, -1 }, { 1, 0 }, { 1, 1 }, { 0, 1 }, + { -1, 1 }, { -1, 0 } }, + { { -2, -2 }, { 0, -2 }, { 2, -2 }, { 2, 0 }, { 2, 2 }, { 0, 2 }, + { -2, 2 }, { -2, 0 } }, + { { -4, -4 }, { 0, -4 }, { 4, -4 }, { 4, 0 }, { 4, 4 }, { 0, 4 }, + { -4, 4 }, { -4, 0 } }, + { { -8, -8 }, { 0, -8 }, { 8, -8 }, { 8, 0 }, { 8, 8 }, { 0, 8 }, + { -8, 8 }, { -8, 0 } }, + { { -16, -16 }, { 0, -16 }, { 16, -16 }, { 16, 0 }, { 16, 16 }, + { 0, 16 }, { -16, 16 }, { -16, 0 } }, + { { -32, -32 }, { 0, -32 }, { 32, -32 }, { 32, 0 }, { 32, 32 }, + { 0, 32 }, { -32, 32 }, { -32, 0 } }, + { { -64, -64 }, { 0, -64 }, { 64, -64 }, { 64, 0 }, { 64, 64 }, + { 0, 64 }, { -64, 64 }, { -64, 0 } }, + { { -128, -128 }, { 0, -128 }, { 128, -128 }, { 128, 0 }, { 128, 128 }, + { 0, 128 }, { -128, 128 }, { -128, 0 } }, + { { -256, -256 }, { 0, -256 }, { 256, -256 }, { 256, 0 }, { 256, 256 }, + { 0, 256 }, { -256, 256 }, { -256, 0 } }, + { { -512, -512 }, { 0, -512 }, { 512, -512 }, { 512, 0 }, { 512, 512 }, + { 0, 512 }, { -512, 512 }, { -512, 0 } }, + { { -1024, -1024 }, { 0, -1024 }, { 1024, -1024 }, { 1024, 0 }, + { 1024, 1024 }, { 0, 1024 }, { -1024, 1024 }, { -1024, 0 } }, + }; + /* clang-format on */ + return pattern_search(x, start_mv, search_param, sad_per_bit, do_init_search, + cost_list, vfp, use_mvcost, center_mv, + square_num_candidates, square_candidates); +} + +static int fast_hex_search(MACROBLOCK *x, MV *ref_mv, int search_param, + int sad_per_bit, + int do_init_search, // must be zero for fast_hex + int *cost_list, const aom_variance_fn_ptr_t *vfp, + int use_mvcost, const MV *center_mv) { + return av1_hex_search(x, ref_mv, AOMMAX(MAX_MVSEARCH_STEPS - 2, search_param), + sad_per_bit, do_init_search, cost_list, vfp, use_mvcost, + center_mv); +} + +static int fast_dia_search(MACROBLOCK *x, MV *ref_mv, int search_param, + int sad_per_bit, int do_init_search, int *cost_list, + const aom_variance_fn_ptr_t *vfp, int use_mvcost, + const MV *center_mv) { + return bigdia_search(x, ref_mv, AOMMAX(MAX_MVSEARCH_STEPS - 2, search_param), + sad_per_bit, do_init_search, cost_list, vfp, use_mvcost, + center_mv); +} + +#undef CHECK_BETTER + +// Exhuastive motion search around a given centre position with a given +// step size. +static int exhuastive_mesh_search(MACROBLOCK *x, MV *ref_mv, MV *best_mv, + int range, int step, int sad_per_bit, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv) { + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &xd->plane[0].pre[0]; + MV fcenter_mv = { center_mv->row, center_mv->col }; + unsigned int best_sad = INT_MAX; + int r, c, i; + int start_col, end_col, start_row, end_row; + int col_step = (step > 1) ? step : 4; + + assert(step >= 1); + + clamp_mv(&fcenter_mv, x->mv_limits.col_min, x->mv_limits.col_max, + x->mv_limits.row_min, x->mv_limits.row_max); + *best_mv = fcenter_mv; + best_sad = + fn_ptr->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &fcenter_mv), in_what->stride) + + mvsad_err_cost(x, &fcenter_mv, ref_mv, sad_per_bit); + start_row = AOMMAX(-range, x->mv_limits.row_min - fcenter_mv.row); + start_col = AOMMAX(-range, x->mv_limits.col_min - fcenter_mv.col); + end_row = AOMMIN(range, x->mv_limits.row_max - fcenter_mv.row); + end_col = AOMMIN(range, x->mv_limits.col_max - fcenter_mv.col); + + for (r = start_row; r <= end_row; r += step) { + for (c = start_col; c <= end_col; c += col_step) { + // Step > 1 means we are not checking every location in this pass. + if (step > 1) { + const MV mv = { fcenter_mv.row + r, fcenter_mv.col + c }; + unsigned int sad = + fn_ptr->sdf(what->buf, what->stride, get_buf_from_mv(in_what, &mv), + in_what->stride); + if (sad < best_sad) { + sad += mvsad_err_cost(x, &mv, ref_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + x->second_best_mv.as_mv = *best_mv; + *best_mv = mv; + } + } + } else { + // 4 sads in a single call if we are checking every location + if (c + 3 <= end_col) { + unsigned int sads[4]; + const uint8_t *addrs[4]; + for (i = 0; i < 4; ++i) { + const MV mv = { fcenter_mv.row + r, fcenter_mv.col + c + i }; + addrs[i] = get_buf_from_mv(in_what, &mv); + } + fn_ptr->sdx4df(what->buf, what->stride, addrs, in_what->stride, sads); + + for (i = 0; i < 4; ++i) { + if (sads[i] < best_sad) { + const MV mv = { fcenter_mv.row + r, fcenter_mv.col + c + i }; + const unsigned int sad = + sads[i] + mvsad_err_cost(x, &mv, ref_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + x->second_best_mv.as_mv = *best_mv; + *best_mv = mv; + } + } + } + } else { + for (i = 0; i < end_col - c; ++i) { + const MV mv = { fcenter_mv.row + r, fcenter_mv.col + c + i }; + unsigned int sad = + fn_ptr->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &mv), in_what->stride); + if (sad < best_sad) { + sad += mvsad_err_cost(x, &mv, ref_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + x->second_best_mv.as_mv = *best_mv; + *best_mv = mv; + } + } + } + } + } + } + } + + return best_sad; +} + +int av1_diamond_search_sad_c(MACROBLOCK *x, const search_site_config *cfg, + MV *ref_mv, MV *best_mv, int search_param, + int sad_per_bit, int *num00, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv) { + int i, j, step; + + const MACROBLOCKD *const xd = &x->e_mbd; + uint8_t *what = x->plane[0].src.buf; + const int what_stride = x->plane[0].src.stride; + const uint8_t *in_what; + const int in_what_stride = xd->plane[0].pre[0].stride; + const uint8_t *best_address; + + unsigned int bestsad = INT_MAX; + int best_site = 0; + int last_site = 0; + + int ref_row; + int ref_col; + + // search_param determines the length of the initial step and hence the number + // of iterations. + // 0 = initial step (MAX_FIRST_STEP) pel + // 1 = (MAX_FIRST_STEP/2) pel, + // 2 = (MAX_FIRST_STEP/4) pel... + const search_site *ss = &cfg->ss[search_param * cfg->searches_per_step]; + const int tot_steps = (cfg->ss_count / cfg->searches_per_step) - search_param; + + const MV fcenter_mv = { center_mv->row >> 3, center_mv->col >> 3 }; + clamp_mv(ref_mv, x->mv_limits.col_min, x->mv_limits.col_max, + x->mv_limits.row_min, x->mv_limits.row_max); + ref_row = ref_mv->row; + ref_col = ref_mv->col; + *num00 = 0; + best_mv->row = ref_row; + best_mv->col = ref_col; + + // Work out the start point for the search + in_what = xd->plane[0].pre[0].buf + ref_row * in_what_stride + ref_col; + best_address = in_what; + + // Check the starting position + bestsad = fn_ptr->sdf(what, what_stride, in_what, in_what_stride) + + mvsad_err_cost(x, best_mv, &fcenter_mv, sad_per_bit); + + i = 1; + + for (step = 0; step < tot_steps; step++) { + int all_in = 1, t; + + // All_in is true if every one of the points we are checking are within + // the bounds of the image. + all_in &= ((best_mv->row + ss[i].mv.row) > x->mv_limits.row_min); + all_in &= ((best_mv->row + ss[i + 1].mv.row) < x->mv_limits.row_max); + all_in &= ((best_mv->col + ss[i + 2].mv.col) > x->mv_limits.col_min); + all_in &= ((best_mv->col + ss[i + 3].mv.col) < x->mv_limits.col_max); + + // If all the pixels are within the bounds we don't check whether the + // search point is valid in this loop, otherwise we check each point + // for validity.. + if (all_in) { + unsigned int sad_array[4]; + + for (j = 0; j < cfg->searches_per_step; j += 4) { + unsigned char const *block_offset[4]; + + for (t = 0; t < 4; t++) + block_offset[t] = ss[i + t].offset + best_address; + + fn_ptr->sdx4df(what, what_stride, block_offset, in_what_stride, + sad_array); + + for (t = 0; t < 4; t++, i++) { + if (sad_array[t] < bestsad) { + const MV this_mv = { best_mv->row + ss[i].mv.row, + best_mv->col + ss[i].mv.col }; + sad_array[t] += + mvsad_err_cost(x, &this_mv, &fcenter_mv, sad_per_bit); + if (sad_array[t] < bestsad) { + bestsad = sad_array[t]; + best_site = i; + } + } + } + } + } else { + for (j = 0; j < cfg->searches_per_step; j++) { + // Trap illegal vectors + const MV this_mv = { best_mv->row + ss[i].mv.row, + best_mv->col + ss[i].mv.col }; + + if (is_mv_in(&x->mv_limits, &this_mv)) { + const uint8_t *const check_here = ss[i].offset + best_address; + unsigned int thissad = + fn_ptr->sdf(what, what_stride, check_here, in_what_stride); + + if (thissad < bestsad) { + thissad += mvsad_err_cost(x, &this_mv, &fcenter_mv, sad_per_bit); + if (thissad < bestsad) { + bestsad = thissad; + best_site = i; + } + } + } + i++; + } + } + if (best_site != last_site) { + x->second_best_mv.as_mv = *best_mv; + best_mv->row += ss[best_site].mv.row; + best_mv->col += ss[best_site].mv.col; + best_address += ss[best_site].offset; + last_site = best_site; +#if defined(NEW_DIAMOND_SEARCH) + while (1) { + const MV this_mv = { best_mv->row + ss[best_site].mv.row, + best_mv->col + ss[best_site].mv.col }; + if (is_mv_in(&x->mv_limits, &this_mv)) { + const uint8_t *const check_here = ss[best_site].offset + best_address; + unsigned int thissad = + fn_ptr->sdf(what, what_stride, check_here, in_what_stride); + if (thissad < bestsad) { + thissad += mvsad_err_cost(x, &this_mv, &fcenter_mv, sad_per_bit); + if (thissad < bestsad) { + bestsad = thissad; + best_mv->row += ss[best_site].mv.row; + best_mv->col += ss[best_site].mv.col; + best_address += ss[best_site].offset; + continue; + } + } + } + break; + } +#endif + } else if (best_address == in_what) { + (*num00)++; + } + } + return bestsad; +} + +static int vector_match(int16_t *ref, int16_t *src, int bwl) { + int best_sad = INT_MAX; + int this_sad; + int d; + int center, offset = 0; + int bw = 4 << bwl; // redundant variable, to be changed in the experiments. + for (d = 0; d <= bw; d += 16) { + this_sad = aom_vector_var(&ref[d], src, bwl); + if (this_sad < best_sad) { + best_sad = this_sad; + offset = d; + } + } + center = offset; + + for (d = -8; d <= 8; d += 16) { + int this_pos = offset + d; + // check limit + if (this_pos < 0 || this_pos > bw) continue; + this_sad = aom_vector_var(&ref[this_pos], src, bwl); + if (this_sad < best_sad) { + best_sad = this_sad; + center = this_pos; + } + } + offset = center; + + for (d = -4; d <= 4; d += 8) { + int this_pos = offset + d; + // check limit + if (this_pos < 0 || this_pos > bw) continue; + this_sad = aom_vector_var(&ref[this_pos], src, bwl); + if (this_sad < best_sad) { + best_sad = this_sad; + center = this_pos; + } + } + offset = center; + + for (d = -2; d <= 2; d += 4) { + int this_pos = offset + d; + // check limit + if (this_pos < 0 || this_pos > bw) continue; + this_sad = aom_vector_var(&ref[this_pos], src, bwl); + if (this_sad < best_sad) { + best_sad = this_sad; + center = this_pos; + } + } + offset = center; + + for (d = -1; d <= 1; d += 2) { + int this_pos = offset + d; + // check limit + if (this_pos < 0 || this_pos > bw) continue; + this_sad = aom_vector_var(&ref[this_pos], src, bwl); + if (this_sad < best_sad) { + best_sad = this_sad; + center = this_pos; + } + } + + return (center - (bw >> 1)); +} + +static const MV search_pos[4] = { + { -1, 0 }, { 0, -1 }, { 0, 1 }, { 1, 0 }, +}; + +unsigned int av1_int_pro_motion_estimation(const AV1_COMP *cpi, MACROBLOCK *x, + BLOCK_SIZE bsize, int mi_row, + int mi_col) { + MACROBLOCKD *xd = &x->e_mbd; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + struct buf_2d backup_yv12[MAX_MB_PLANE] = { { 0, 0, 0, 0, 0 } }; + DECLARE_ALIGNED(16, int16_t, hbuf[2 * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, int16_t, vbuf[2 * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, int16_t, src_hbuf[MAX_SB_SQUARE]); + DECLARE_ALIGNED(16, int16_t, src_vbuf[MAX_SB_SQUARE]); + int idx; + const int bw = 4 << b_width_log2_lookup[bsize]; + const int bh = 4 << b_height_log2_lookup[bsize]; + const int search_width = bw << 1; + const int search_height = bh << 1; + const int src_stride = x->plane[0].src.stride; + const int ref_stride = xd->plane[0].pre[0].stride; + uint8_t const *ref_buf, *src_buf; + MV *tmp_mv = &xd->mi[0]->mbmi.mv[0].as_mv; + unsigned int best_sad, tmp_sad, sad_arr[4]; + MV this_mv; + const int norm_factor = 3 + (bw >> 5); + const YV12_BUFFER_CONFIG *scaled_ref_frame = + av1_get_scaled_ref_frame(cpi, mbmi->ref_frame[0]); + + if (scaled_ref_frame) { + int i; + // Swap out the reference frame for a version that's been scaled to + // match the resolution of the current frame, allowing the existing + // motion search code to be used without additional modifications. + for (i = 0; i < MAX_MB_PLANE; i++) backup_yv12[i] = xd->plane[i].pre[0]; + av1_setup_pre_planes(xd, 0, scaled_ref_frame, mi_row, mi_col, NULL); + } + +#if CONFIG_HIGHBITDEPTH + { + unsigned int this_sad; + tmp_mv->row = 0; + tmp_mv->col = 0; + this_sad = cpi->fn_ptr[bsize].sdf(x->plane[0].src.buf, src_stride, + xd->plane[0].pre[0].buf, ref_stride); + + if (scaled_ref_frame) { + int i; + for (i = 0; i < MAX_MB_PLANE; i++) xd->plane[i].pre[0] = backup_yv12[i]; + } + return this_sad; + } +#endif + + // Set up prediction 1-D reference set + ref_buf = xd->plane[0].pre[0].buf - (bw >> 1); + for (idx = 0; idx < search_width; idx += 16) { + aom_int_pro_row(&hbuf[idx], ref_buf, ref_stride, bh); + ref_buf += 16; + } + + ref_buf = xd->plane[0].pre[0].buf - (bh >> 1) * ref_stride; + for (idx = 0; idx < search_height; ++idx) { + vbuf[idx] = aom_int_pro_col(ref_buf, bw) >> norm_factor; + ref_buf += ref_stride; + } + + // Set up src 1-D reference set + for (idx = 0; idx < bw; idx += 16) { + src_buf = x->plane[0].src.buf + idx; + aom_int_pro_row(&src_hbuf[idx], src_buf, src_stride, bh); + } + + src_buf = x->plane[0].src.buf; + for (idx = 0; idx < bh; ++idx) { + src_vbuf[idx] = aom_int_pro_col(src_buf, bw) >> norm_factor; + src_buf += src_stride; + } + + // Find the best match per 1-D search + tmp_mv->col = vector_match(hbuf, src_hbuf, b_width_log2_lookup[bsize]); + tmp_mv->row = vector_match(vbuf, src_vbuf, b_height_log2_lookup[bsize]); + + this_mv = *tmp_mv; + src_buf = x->plane[0].src.buf; + ref_buf = xd->plane[0].pre[0].buf + this_mv.row * ref_stride + this_mv.col; + best_sad = cpi->fn_ptr[bsize].sdf(src_buf, src_stride, ref_buf, ref_stride); + + { + const uint8_t *const pos[4] = { + ref_buf - ref_stride, ref_buf - 1, ref_buf + 1, ref_buf + ref_stride, + }; + + cpi->fn_ptr[bsize].sdx4df(src_buf, src_stride, pos, ref_stride, sad_arr); + } + + for (idx = 0; idx < 4; ++idx) { + if (sad_arr[idx] < best_sad) { + best_sad = sad_arr[idx]; + tmp_mv->row = search_pos[idx].row + this_mv.row; + tmp_mv->col = search_pos[idx].col + this_mv.col; + } + } + + if (sad_arr[0] < sad_arr[3]) + this_mv.row -= 1; + else + this_mv.row += 1; + + if (sad_arr[1] < sad_arr[2]) + this_mv.col -= 1; + else + this_mv.col += 1; + + ref_buf = xd->plane[0].pre[0].buf + this_mv.row * ref_stride + this_mv.col; + + tmp_sad = cpi->fn_ptr[bsize].sdf(src_buf, src_stride, ref_buf, ref_stride); + if (best_sad > tmp_sad) { + *tmp_mv = this_mv; + best_sad = tmp_sad; + } + + tmp_mv->row *= 8; + tmp_mv->col *= 8; + + if (scaled_ref_frame) { + int i; + for (i = 0; i < MAX_MB_PLANE; i++) xd->plane[i].pre[0] = backup_yv12[i]; + } + + return best_sad; +} + +/* do_refine: If last step (1-away) of n-step search doesn't pick the center + point as the best match, we will do a final 1-away diamond + refining search */ +static int full_pixel_diamond(const AV1_COMP *const cpi, MACROBLOCK *x, + MV *mvp_full, int step_param, int sadpb, + int further_steps, int do_refine, int *cost_list, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *ref_mv) { + MV temp_mv; + int thissme, n, num00 = 0; + int bestsme = cpi->diamond_search_sad(x, &cpi->ss_cfg, mvp_full, &temp_mv, + step_param, sadpb, &n, fn_ptr, ref_mv); + if (bestsme < INT_MAX) + bestsme = av1_get_mvpred_var(x, &temp_mv, ref_mv, fn_ptr, 1); + x->best_mv.as_mv = temp_mv; + + // If there won't be more n-step search, check to see if refining search is + // needed. + if (n > further_steps) do_refine = 0; + + while (n < further_steps) { + ++n; + + if (num00) { + num00--; + } else { + thissme = cpi->diamond_search_sad(x, &cpi->ss_cfg, mvp_full, &temp_mv, + step_param + n, sadpb, &num00, fn_ptr, + ref_mv); + if (thissme < INT_MAX) + thissme = av1_get_mvpred_var(x, &temp_mv, ref_mv, fn_ptr, 1); + + // check to see if refining search is needed. + if (num00 > further_steps - n) do_refine = 0; + + if (thissme < bestsme) { + bestsme = thissme; + x->best_mv.as_mv = temp_mv; + } + } + } + + // final 1-away diamond refining search + if (do_refine) { + const int search_range = 8; + MV best_mv = x->best_mv.as_mv; + thissme = av1_refining_search_sad(x, &best_mv, sadpb, search_range, fn_ptr, + ref_mv); + if (thissme < INT_MAX) + thissme = av1_get_mvpred_var(x, &best_mv, ref_mv, fn_ptr, 1); + if (thissme < bestsme) { + bestsme = thissme; + x->best_mv.as_mv = best_mv; + } + } + + // Return cost list. + if (cost_list) { + calc_int_cost_list(x, ref_mv, sadpb, fn_ptr, &x->best_mv.as_mv, cost_list); + } + return bestsme; +} + +#define MIN_RANGE 7 +#define MAX_RANGE 256 +#define MIN_INTERVAL 1 +// Runs an limited range exhaustive mesh search using a pattern set +// according to the encode speed profile. +static int full_pixel_exhaustive(const AV1_COMP *const cpi, MACROBLOCK *x, + const MV *centre_mv_full, int sadpb, + int *cost_list, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *ref_mv, MV *dst_mv) { + const SPEED_FEATURES *const sf = &cpi->sf; + MV temp_mv = { centre_mv_full->row, centre_mv_full->col }; + MV f_ref_mv = { ref_mv->row >> 3, ref_mv->col >> 3 }; + int bestsme; + int i; + int interval = sf->mesh_patterns[0].interval; + int range = sf->mesh_patterns[0].range; + int baseline_interval_divisor; + + // Keep track of number of exhaustive calls (this frame in this thread). + ++(*x->ex_search_count_ptr); + + // Trap illegal values for interval and range for this function. + if ((range < MIN_RANGE) || (range > MAX_RANGE) || (interval < MIN_INTERVAL) || + (interval > range)) + return INT_MAX; + + baseline_interval_divisor = range / interval; + + // Check size of proposed first range against magnitude of the centre + // value used as a starting point. + range = AOMMAX(range, (5 * AOMMAX(abs(temp_mv.row), abs(temp_mv.col))) / 4); + range = AOMMIN(range, MAX_RANGE); + interval = AOMMAX(interval, range / baseline_interval_divisor); + + // initial search + bestsme = exhuastive_mesh_search(x, &f_ref_mv, &temp_mv, range, interval, + sadpb, fn_ptr, &temp_mv); + + if ((interval > MIN_INTERVAL) && (range > MIN_RANGE)) { + // Progressive searches with range and step size decreasing each time + // till we reach a step size of 1. Then break out. + for (i = 1; i < MAX_MESH_STEP; ++i) { + // First pass with coarser step and longer range + bestsme = exhuastive_mesh_search( + x, &f_ref_mv, &temp_mv, sf->mesh_patterns[i].range, + sf->mesh_patterns[i].interval, sadpb, fn_ptr, &temp_mv); + + if (sf->mesh_patterns[i].interval == 1) break; + } + } + + if (bestsme < INT_MAX) + bestsme = av1_get_mvpred_var(x, &temp_mv, ref_mv, fn_ptr, 1); + *dst_mv = temp_mv; + + // Return cost list. + if (cost_list) { + calc_int_cost_list(x, ref_mv, sadpb, fn_ptr, dst_mv, cost_list); + } + return bestsme; +} + +int av1_full_search_sad_c(const MACROBLOCK *x, const MV *ref_mv, + int sad_per_bit, int distance, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv, MV *best_mv) { + int r, c; + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &xd->plane[0].pre[0]; + const int row_min = AOMMAX(ref_mv->row - distance, x->mv_limits.row_min); + const int row_max = AOMMIN(ref_mv->row + distance, x->mv_limits.row_max); + const int col_min = AOMMAX(ref_mv->col - distance, x->mv_limits.col_min); + const int col_max = AOMMIN(ref_mv->col + distance, x->mv_limits.col_max); + const MV fcenter_mv = { center_mv->row >> 3, center_mv->col >> 3 }; + int best_sad = + fn_ptr->sdf(what->buf, what->stride, get_buf_from_mv(in_what, ref_mv), + in_what->stride) + + mvsad_err_cost(x, ref_mv, &fcenter_mv, sad_per_bit); + *best_mv = *ref_mv; + + for (r = row_min; r < row_max; ++r) { + for (c = col_min; c < col_max; ++c) { + const MV mv = { r, c }; + const int sad = + fn_ptr->sdf(what->buf, what->stride, get_buf_from_mv(in_what, &mv), + in_what->stride) + + mvsad_err_cost(x, &mv, &fcenter_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + *best_mv = mv; + } + } + } + return best_sad; +} + +int av1_full_search_sadx3(const MACROBLOCK *x, const MV *ref_mv, + int sad_per_bit, int distance, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv, MV *best_mv) { + int r; + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &xd->plane[0].pre[0]; + const int row_min = AOMMAX(ref_mv->row - distance, x->mv_limits.row_min); + const int row_max = AOMMIN(ref_mv->row + distance, x->mv_limits.row_max); + const int col_min = AOMMAX(ref_mv->col - distance, x->mv_limits.col_min); + const int col_max = AOMMIN(ref_mv->col + distance, x->mv_limits.col_max); + const MV fcenter_mv = { center_mv->row >> 3, center_mv->col >> 3 }; + unsigned int best_sad = + fn_ptr->sdf(what->buf, what->stride, get_buf_from_mv(in_what, ref_mv), + in_what->stride) + + mvsad_err_cost(x, ref_mv, &fcenter_mv, sad_per_bit); + *best_mv = *ref_mv; + + for (r = row_min; r < row_max; ++r) { + int c = col_min; + const uint8_t *check_here = &in_what->buf[r * in_what->stride + c]; + + if (fn_ptr->sdx3f != NULL) { + while ((c + 2) < col_max) { + int i; + DECLARE_ALIGNED(16, uint32_t, sads[3]); + + fn_ptr->sdx3f(what->buf, what->stride, check_here, in_what->stride, + sads); + + for (i = 0; i < 3; ++i) { + unsigned int sad = sads[i]; + if (sad < best_sad) { + const MV mv = { r, c }; + sad += mvsad_err_cost(x, &mv, &fcenter_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + *best_mv = mv; + } + } + ++check_here; + ++c; + } + } + } + + while (c < col_max) { + unsigned int sad = + fn_ptr->sdf(what->buf, what->stride, check_here, in_what->stride); + if (sad < best_sad) { + const MV mv = { r, c }; + sad += mvsad_err_cost(x, &mv, &fcenter_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + *best_mv = mv; + } + } + ++check_here; + ++c; + } + } + + return best_sad; +} + +int av1_full_search_sadx8(const MACROBLOCK *x, const MV *ref_mv, + int sad_per_bit, int distance, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv, MV *best_mv) { + int r; + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &xd->plane[0].pre[0]; + const int row_min = AOMMAX(ref_mv->row - distance, x->mv_limits.row_min); + const int row_max = AOMMIN(ref_mv->row + distance, x->mv_limits.row_max); + const int col_min = AOMMAX(ref_mv->col - distance, x->mv_limits.col_min); + const int col_max = AOMMIN(ref_mv->col + distance, x->mv_limits.col_max); + const MV fcenter_mv = { center_mv->row >> 3, center_mv->col >> 3 }; + unsigned int best_sad = + fn_ptr->sdf(what->buf, what->stride, get_buf_from_mv(in_what, ref_mv), + in_what->stride) + + mvsad_err_cost(x, ref_mv, &fcenter_mv, sad_per_bit); + *best_mv = *ref_mv; + + for (r = row_min; r < row_max; ++r) { + int c = col_min; + const uint8_t *check_here = &in_what->buf[r * in_what->stride + c]; + + if (fn_ptr->sdx8f != NULL) { + while ((c + 7) < col_max) { + int i; + DECLARE_ALIGNED(16, uint32_t, sads[8]); + + fn_ptr->sdx8f(what->buf, what->stride, check_here, in_what->stride, + sads); + + for (i = 0; i < 8; ++i) { + unsigned int sad = sads[i]; + if (sad < best_sad) { + const MV mv = { r, c }; + sad += mvsad_err_cost(x, &mv, &fcenter_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + *best_mv = mv; + } + } + ++check_here; + ++c; + } + } + } + + if (fn_ptr->sdx3f != NULL) { + while ((c + 2) < col_max) { + int i; + DECLARE_ALIGNED(16, uint32_t, sads[3]); + + fn_ptr->sdx3f(what->buf, what->stride, check_here, in_what->stride, + sads); + + for (i = 0; i < 3; ++i) { + unsigned int sad = sads[i]; + if (sad < best_sad) { + const MV mv = { r, c }; + sad += mvsad_err_cost(x, &mv, &fcenter_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + *best_mv = mv; + } + } + ++check_here; + ++c; + } + } + } + + while (c < col_max) { + unsigned int sad = + fn_ptr->sdf(what->buf, what->stride, check_here, in_what->stride); + if (sad < best_sad) { + const MV mv = { r, c }; + sad += mvsad_err_cost(x, &mv, &fcenter_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + *best_mv = mv; + } + } + ++check_here; + ++c; + } + } + + return best_sad; +} + +int av1_refining_search_sad(MACROBLOCK *x, MV *ref_mv, int error_per_bit, + int search_range, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv) { + const MACROBLOCKD *const xd = &x->e_mbd; + const MV neighbors[4] = { { -1, 0 }, { 0, -1 }, { 0, 1 }, { 1, 0 } }; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &xd->plane[0].pre[0]; + const MV fcenter_mv = { center_mv->row >> 3, center_mv->col >> 3 }; + const uint8_t *best_address = get_buf_from_mv(in_what, ref_mv); + unsigned int best_sad = + fn_ptr->sdf(what->buf, what->stride, best_address, in_what->stride) + + mvsad_err_cost(x, ref_mv, &fcenter_mv, error_per_bit); + int i, j; + + for (i = 0; i < search_range; i++) { + int best_site = -1; + const int all_in = ((ref_mv->row - 1) > x->mv_limits.row_min) & + ((ref_mv->row + 1) < x->mv_limits.row_max) & + ((ref_mv->col - 1) > x->mv_limits.col_min) & + ((ref_mv->col + 1) < x->mv_limits.col_max); + + if (all_in) { + unsigned int sads[4]; + const uint8_t *const positions[4] = { best_address - in_what->stride, + best_address - 1, best_address + 1, + best_address + in_what->stride }; + + fn_ptr->sdx4df(what->buf, what->stride, positions, in_what->stride, sads); + + for (j = 0; j < 4; ++j) { + if (sads[j] < best_sad) { + const MV mv = { ref_mv->row + neighbors[j].row, + ref_mv->col + neighbors[j].col }; + sads[j] += mvsad_err_cost(x, &mv, &fcenter_mv, error_per_bit); + if (sads[j] < best_sad) { + best_sad = sads[j]; + best_site = j; + } + } + } + } else { + for (j = 0; j < 4; ++j) { + const MV mv = { ref_mv->row + neighbors[j].row, + ref_mv->col + neighbors[j].col }; + + if (is_mv_in(&x->mv_limits, &mv)) { + unsigned int sad = + fn_ptr->sdf(what->buf, what->stride, + get_buf_from_mv(in_what, &mv), in_what->stride); + if (sad < best_sad) { + sad += mvsad_err_cost(x, &mv, &fcenter_mv, error_per_bit); + if (sad < best_sad) { + best_sad = sad; + best_site = j; + } + } + } + } + } + + if (best_site == -1) { + break; + } else { + x->second_best_mv.as_mv = *ref_mv; + ref_mv->row += neighbors[best_site].row; + ref_mv->col += neighbors[best_site].col; + best_address = get_buf_from_mv(in_what, ref_mv); + } + } + + return best_sad; +} + +// This function is called when we do joint motion search in comp_inter_inter +// mode. +int av1_refining_search_8p_c(MACROBLOCK *x, int error_per_bit, int search_range, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv, const uint8_t *second_pred) { + const MV neighbors[8] = { { -1, 0 }, { 0, -1 }, { 0, 1 }, { 1, 0 }, + { -1, -1 }, { 1, -1 }, { -1, 1 }, { 1, 1 } }; + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &xd->plane[0].pre[0]; + const MV fcenter_mv = { center_mv->row >> 3, center_mv->col >> 3 }; + MV *best_mv = &x->best_mv.as_mv; + unsigned int best_sad = INT_MAX; + int i, j; + + clamp_mv(best_mv, x->mv_limits.col_min, x->mv_limits.col_max, + x->mv_limits.row_min, x->mv_limits.row_max); + best_sad = + fn_ptr->sdaf(what->buf, what->stride, get_buf_from_mv(in_what, best_mv), + in_what->stride, second_pred) + + mvsad_err_cost(x, best_mv, &fcenter_mv, error_per_bit); + + for (i = 0; i < search_range; ++i) { + int best_site = -1; + + for (j = 0; j < 8; ++j) { + const MV mv = { best_mv->row + neighbors[j].row, + best_mv->col + neighbors[j].col }; + + if (is_mv_in(&x->mv_limits, &mv)) { + unsigned int sad = + fn_ptr->sdaf(what->buf, what->stride, get_buf_from_mv(in_what, &mv), + in_what->stride, second_pred); + if (sad < best_sad) { + sad += mvsad_err_cost(x, &mv, &fcenter_mv, error_per_bit); + if (sad < best_sad) { + best_sad = sad; + best_site = j; + } + } + } + } + + if (best_site == -1) { + break; + } else { + best_mv->row += neighbors[best_site].row; + best_mv->col += neighbors[best_site].col; + } + } + return best_sad; +} + +#define MIN_EX_SEARCH_LIMIT 128 +static int is_exhaustive_allowed(const AV1_COMP *const cpi, MACROBLOCK *x) { + const SPEED_FEATURES *const sf = &cpi->sf; + const int max_ex = + AOMMAX(MIN_EX_SEARCH_LIMIT, + (*x->m_search_count_ptr * sf->max_exaustive_pct) / 100); + + return sf->allow_exhaustive_searches && + (sf->exhaustive_searches_thresh < INT_MAX) && + (*x->ex_search_count_ptr <= max_ex) && !cpi->rc.is_src_frame_alt_ref; +} + +int av1_full_pixel_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, + MV *mvp_full, int step_param, int error_per_bit, + int *cost_list, const MV *ref_mv, int var_max, + int rd) { + const SPEED_FEATURES *const sf = &cpi->sf; + const SEARCH_METHODS method = sf->mv.search_method; + const aom_variance_fn_ptr_t *fn_ptr = &cpi->fn_ptr[bsize]; + int var = 0; + + if (cost_list) { + cost_list[0] = INT_MAX; + cost_list[1] = INT_MAX; + cost_list[2] = INT_MAX; + cost_list[3] = INT_MAX; + cost_list[4] = INT_MAX; + } + + // Keep track of number of searches (this frame in this thread). + ++(*x->m_search_count_ptr); + + switch (method) { + case FAST_DIAMOND: + var = fast_dia_search(x, mvp_full, step_param, error_per_bit, 0, + cost_list, fn_ptr, 1, ref_mv); + break; + case FAST_HEX: + var = fast_hex_search(x, mvp_full, step_param, error_per_bit, 0, + cost_list, fn_ptr, 1, ref_mv); + break; + case HEX: + var = av1_hex_search(x, mvp_full, step_param, error_per_bit, 1, cost_list, + fn_ptr, 1, ref_mv); + break; + case SQUARE: + var = square_search(x, mvp_full, step_param, error_per_bit, 1, cost_list, + fn_ptr, 1, ref_mv); + break; + case BIGDIA: + var = bigdia_search(x, mvp_full, step_param, error_per_bit, 1, cost_list, + fn_ptr, 1, ref_mv); + break; + case NSTEP: + var = full_pixel_diamond(cpi, x, mvp_full, step_param, error_per_bit, + MAX_MVSEARCH_STEPS - 1 - step_param, 1, + cost_list, fn_ptr, ref_mv); + + // Should we allow a follow on exhaustive search? + if (is_exhaustive_allowed(cpi, x)) { + int exhuastive_thr = sf->exhaustive_searches_thresh; + exhuastive_thr >>= + 10 - (b_width_log2_lookup[bsize] + b_height_log2_lookup[bsize]); + + // Threshold variance for an exhaustive full search. + if (var > exhuastive_thr) { + int var_ex; + MV tmp_mv_ex; + var_ex = + full_pixel_exhaustive(cpi, x, &x->best_mv.as_mv, error_per_bit, + cost_list, fn_ptr, ref_mv, &tmp_mv_ex); + + if (var_ex < var) { + var = var_ex; + x->best_mv.as_mv = tmp_mv_ex; + } + } + } + break; + + break; + default: assert(0 && "Invalid search method."); + } + + if (method != NSTEP && rd && var < var_max) + var = av1_get_mvpred_var(x, &x->best_mv.as_mv, ref_mv, fn_ptr, 1); + + return var; +} + +#if CONFIG_EXT_INTER +/* returns subpixel variance error function */ +#define DIST(r, c) \ + vfp->msvf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, src_stride, \ + mask, mask_stride, &sse) + +/* checks if (r, c) has better score than previous best */ + +#define MVC(r, c) \ + (mvcost \ + ? ((mvjcost[((r) != rr) * 2 + ((c) != rc)] + mvcost[0][((r)-rr)] + \ + mvcost[1][((c)-rc)]) * \ + error_per_bit + \ + 4096) >> \ + 13 \ + : 0) + +#define CHECK_BETTER(v, r, c) \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + thismse = (DIST(r, c)); \ + if ((v = MVC(r, c) + thismse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } + +#undef CHECK_BETTER0 +#define CHECK_BETTER0(v, r, c) CHECK_BETTER(v, r, c) + +#undef CHECK_BETTER1 +#define CHECK_BETTER1(v, r, c) \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + thismse = upsampled_masked_pref_error(xd, mask, mask_stride, vfp, z, \ + src_stride, upre(y, y_stride, r, c), \ + y_stride, w, h, &sse); \ + if ((v = MVC(r, c) + thismse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } + +int av1_find_best_masked_sub_pixel_tree( + const MACROBLOCK *x, const uint8_t *mask, int mask_stride, MV *bestmv, + const MV *ref_mv, int allow_hp, int error_per_bit, + const aom_variance_fn_ptr_t *vfp, int forced_stop, int iters_per_step, + int *mvjcost, int *mvcost[2], int *distortion, unsigned int *sse1, + int is_second) { + const uint8_t *const z = x->plane[0].src.buf; + const int src_stride = x->plane[0].src.stride; + const MACROBLOCKD *xd = &x->e_mbd; + unsigned int besterr = INT_MAX; + unsigned int sse; + int thismse; + unsigned int whichdir; + unsigned int halfiters = iters_per_step; + unsigned int quarteriters = iters_per_step; + unsigned int eighthiters = iters_per_step; + + const int y_stride = xd->plane[0].pre[is_second].stride; + const int offset = bestmv->row * y_stride + bestmv->col; + const uint8_t *const y = xd->plane[0].pre[is_second].buf; + + int rr = ref_mv->row; + int rc = ref_mv->col; + int br = bestmv->row * 8; + int bc = bestmv->col * 8; + int hstep = 4; + int tr = br; + int tc = bc; + int minc, maxc, minr, maxr; + + av1_set_subpel_mv_search_range(&x->mv_limits, &minc, &maxc, &minr, &maxr, + ref_mv); + + // central mv + bestmv->row *= 8; + bestmv->col *= 8; + + // calculate central point error + besterr = + vfp->mvf(y + offset, y_stride, z, src_stride, mask, mask_stride, sse1); + *distortion = besterr; + besterr += mv_err_cost(bestmv, ref_mv, mvjcost, mvcost, error_per_bit); + + // 1/2 pel + FIRST_LEVEL_CHECKS; + if (halfiters > 1) { + SECOND_LEVEL_CHECKS; + } + tr = br; + tc = bc; + + // Note forced_stop: 0 - full, 1 - qtr only, 2 - half only + if (forced_stop != 2) { + hstep >>= 1; + FIRST_LEVEL_CHECKS; + if (quarteriters > 1) { + SECOND_LEVEL_CHECKS; + } + tr = br; + tc = bc; + } + + if (allow_hp && forced_stop == 0) { + hstep >>= 1; + FIRST_LEVEL_CHECKS; + if (eighthiters > 1) { + SECOND_LEVEL_CHECKS; + } + tr = br; + tc = bc; + } + // These lines insure static analysis doesn't warn that + // tr and tc aren't used after the above point. + (void)tr; + (void)tc; + + bestmv->row = br; + bestmv->col = bc; + + return besterr; +} + +static unsigned int setup_masked_center_error( + const uint8_t *mask, int mask_stride, const MV *bestmv, const MV *ref_mv, + int error_per_bit, const aom_variance_fn_ptr_t *vfp, + const uint8_t *const src, const int src_stride, const uint8_t *const y, + int y_stride, int offset, int *mvjcost, int *mvcost[2], unsigned int *sse1, + int *distortion) { + unsigned int besterr; + besterr = + vfp->mvf(y + offset, y_stride, src, src_stride, mask, mask_stride, sse1); + *distortion = besterr; + besterr += mv_err_cost(bestmv, ref_mv, mvjcost, mvcost, error_per_bit); + return besterr; +} + +static int upsampled_masked_pref_error(const MACROBLOCKD *xd, + const uint8_t *mask, int mask_stride, + const aom_variance_fn_ptr_t *vfp, + const uint8_t *const src, + const int src_stride, + const uint8_t *const y, int y_stride, + int w, int h, unsigned int *sse) { + unsigned int besterr; +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + DECLARE_ALIGNED(16, uint16_t, pred16[MAX_SB_SQUARE]); + aom_highbd_upsampled_pred(pred16, w, h, y, y_stride); + + besterr = vfp->mvf(CONVERT_TO_BYTEPTR(pred16), w, src, src_stride, mask, + mask_stride, sse); + } else { + DECLARE_ALIGNED(16, uint8_t, pred[MAX_SB_SQUARE]); +#else + DECLARE_ALIGNED(16, uint8_t, pred[MAX_SB_SQUARE]); + (void)xd; +#endif // CONFIG_HIGHBITDEPTH + aom_upsampled_pred(pred, w, h, y, y_stride); + + besterr = vfp->mvf(pred, w, src, src_stride, mask, mask_stride, sse); +#if CONFIG_HIGHBITDEPTH + } +#endif + return besterr; +} + +static unsigned int upsampled_setup_masked_center_error( + const MACROBLOCKD *xd, const uint8_t *mask, int mask_stride, + const MV *bestmv, const MV *ref_mv, int error_per_bit, + const aom_variance_fn_ptr_t *vfp, const uint8_t *const src, + const int src_stride, const uint8_t *const y, int y_stride, int w, int h, + int offset, int *mvjcost, int *mvcost[2], unsigned int *sse1, + int *distortion) { + unsigned int besterr = + upsampled_masked_pref_error(xd, mask, mask_stride, vfp, src, src_stride, + y + offset, y_stride, w, h, sse1); + *distortion = besterr; + besterr += mv_err_cost(bestmv, ref_mv, mvjcost, mvcost, error_per_bit); + return besterr; +} + +int av1_find_best_masked_sub_pixel_tree_up( + const AV1_COMP *cpi, MACROBLOCK *x, const uint8_t *mask, int mask_stride, + int mi_row, int mi_col, MV *bestmv, const MV *ref_mv, int allow_hp, + int error_per_bit, const aom_variance_fn_ptr_t *vfp, int forced_stop, + int iters_per_step, int *mvjcost, int *mvcost[2], int *distortion, + unsigned int *sse1, int is_second, int use_upsampled_ref) { + const uint8_t *const z = x->plane[0].src.buf; + const uint8_t *const src_address = z; + const int src_stride = x->plane[0].src.stride; + MACROBLOCKD *xd = &x->e_mbd; + struct macroblockd_plane *const pd = &xd->plane[0]; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + unsigned int besterr = INT_MAX; + unsigned int sse; + unsigned int thismse; + + int rr = ref_mv->row; + int rc = ref_mv->col; + int br = bestmv->row * 8; + int bc = bestmv->col * 8; + int hstep = 4; + int iter; + int round = 3 - forced_stop; + int tr = br; + int tc = bc; + const MV *search_step = search_step_table; + int idx, best_idx = -1; + unsigned int cost_array[5]; + int kr, kc; + const int w = block_size_wide[mbmi->sb_type]; + const int h = block_size_high[mbmi->sb_type]; + int offset; + int y_stride; + const uint8_t *y; + + const struct buf_2d backup_pred = pd->pre[is_second]; + int minc, maxc, minr, maxr; + + av1_set_subpel_mv_search_range(&x->mv_limits, &minc, &maxc, &minr, &maxr, + ref_mv); + + if (use_upsampled_ref) { + int ref = xd->mi[0]->mbmi.ref_frame[is_second]; + const YV12_BUFFER_CONFIG *upsampled_ref = get_upsampled_ref(cpi, ref); + setup_pred_plane(&pd->pre[is_second], mbmi->sb_type, + upsampled_ref->y_buffer, upsampled_ref->y_crop_width, + upsampled_ref->y_crop_height, upsampled_ref->y_stride, + (mi_row << 3), (mi_col << 3), NULL, pd->subsampling_x, + pd->subsampling_y); + } + y = pd->pre[is_second].buf; + y_stride = pd->pre[is_second].stride; + offset = bestmv->row * y_stride + bestmv->col; + + if (!allow_hp) + if (round == 3) round = 2; + + bestmv->row *= 8; + bestmv->col *= 8; + + // use_upsampled_ref can be 0 or 1 + if (use_upsampled_ref) + besterr = upsampled_setup_masked_center_error( + xd, mask, mask_stride, bestmv, ref_mv, error_per_bit, vfp, z, + src_stride, y, y_stride, w, h, (offset * 8), mvjcost, mvcost, sse1, + distortion); + else + besterr = setup_masked_center_error( + mask, mask_stride, bestmv, ref_mv, error_per_bit, vfp, z, src_stride, y, + y_stride, offset, mvjcost, mvcost, sse1, distortion); + + for (iter = 0; iter < round; ++iter) { + // Check vertical and horizontal sub-pixel positions. + for (idx = 0; idx < 4; ++idx) { + tr = br + search_step[idx].row; + tc = bc + search_step[idx].col; + if (tc >= minc && tc <= maxc && tr >= minr && tr <= maxr) { + MV this_mv = { tr, tc }; + + if (use_upsampled_ref) { + const uint8_t *const pre_address = y + tr * y_stride + tc; + + thismse = upsampled_masked_pref_error( + xd, mask, mask_stride, vfp, src_address, src_stride, pre_address, + y_stride, w, h, &sse); + } else { + const uint8_t *const pre_address = + y + (tr >> 3) * y_stride + (tc >> 3); + thismse = vfp->msvf(pre_address, y_stride, sp(tc), sp(tr), + src_address, src_stride, mask, mask_stride, &sse); + } + + cost_array[idx] = thismse + mv_err_cost(&this_mv, ref_mv, mvjcost, + mvcost, error_per_bit); + + if (cost_array[idx] < besterr) { + best_idx = idx; + besterr = cost_array[idx]; + *distortion = thismse; + *sse1 = sse; + } + } else { + cost_array[idx] = INT_MAX; + } + } + + // Check diagonal sub-pixel position + kc = (cost_array[0] <= cost_array[1] ? -hstep : hstep); + kr = (cost_array[2] <= cost_array[3] ? -hstep : hstep); + + tc = bc + kc; + tr = br + kr; + if (tc >= minc && tc <= maxc && tr >= minr && tr <= maxr) { + MV this_mv = { tr, tc }; + + if (use_upsampled_ref) { + const uint8_t *const pre_address = y + tr * y_stride + tc; + + thismse = upsampled_masked_pref_error( + xd, mask, mask_stride, vfp, src_address, src_stride, pre_address, + y_stride, w, h, &sse); + } else { + const uint8_t *const pre_address = y + (tr >> 3) * y_stride + (tc >> 3); + + thismse = vfp->msvf(pre_address, y_stride, sp(tc), sp(tr), src_address, + src_stride, mask, mask_stride, &sse); + } + + cost_array[4] = thismse + mv_err_cost(&this_mv, ref_mv, mvjcost, mvcost, + error_per_bit); + + if (cost_array[4] < besterr) { + best_idx = 4; + besterr = cost_array[4]; + *distortion = thismse; + *sse1 = sse; + } + } else { + cost_array[idx] = INT_MAX; + } + + if (best_idx < 4 && best_idx >= 0) { + br += search_step[best_idx].row; + bc += search_step[best_idx].col; + } else if (best_idx == 4) { + br = tr; + bc = tc; + } + + if (iters_per_step > 1 && best_idx != -1) { + if (use_upsampled_ref) { + SECOND_LEVEL_CHECKS_BEST(1); + } else { + SECOND_LEVEL_CHECKS_BEST(0); + } + } + + tr = br; + tc = bc; + + search_step += 4; + hstep >>= 1; + best_idx = -1; + } + + // These lines insure static analysis doesn't warn that + // tr and tc aren't used after the above point. + (void)tr; + (void)tc; + + bestmv->row = br; + bestmv->col = bc; + + if (use_upsampled_ref) { + pd->pre[is_second] = backup_pred; + } + + return besterr; +} + +#undef DIST +#undef MVC +#undef CHECK_BETTER + +static int get_masked_mvpred_var(const MACROBLOCK *x, const uint8_t *mask, + int mask_stride, const MV *best_mv, + const MV *center_mv, + const aom_variance_fn_ptr_t *vfp, + int use_mvcost, int is_second) { + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &xd->plane[0].pre[is_second]; + const MV mv = { best_mv->row * 8, best_mv->col * 8 }; + unsigned int unused; + + return vfp->mvf(what->buf, what->stride, get_buf_from_mv(in_what, best_mv), + in_what->stride, mask, mask_stride, &unused) + + (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmvjointcost, x->mvcost, + x->errorperbit) + : 0); +} + +int masked_refining_search_sad(const MACROBLOCK *x, const uint8_t *mask, + int mask_stride, MV *ref_mv, int error_per_bit, + int search_range, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv, int is_second) { + const MV neighbors[4] = { { -1, 0 }, { 0, -1 }, { 0, 1 }, { 1, 0 } }; + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &xd->plane[0].pre[is_second]; + const MV fcenter_mv = { center_mv->row >> 3, center_mv->col >> 3 }; + unsigned int best_sad = + fn_ptr->msdf(what->buf, what->stride, get_buf_from_mv(in_what, ref_mv), + in_what->stride, mask, mask_stride) + + mvsad_err_cost(x, ref_mv, &fcenter_mv, error_per_bit); + int i, j; + + for (i = 0; i < search_range; i++) { + int best_site = -1; + + for (j = 0; j < 4; j++) { + const MV mv = { ref_mv->row + neighbors[j].row, + ref_mv->col + neighbors[j].col }; + if (is_mv_in(&x->mv_limits, &mv)) { + unsigned int sad = + fn_ptr->msdf(what->buf, what->stride, get_buf_from_mv(in_what, &mv), + in_what->stride, mask, mask_stride); + if (sad < best_sad) { + sad += mvsad_err_cost(x, &mv, &fcenter_mv, error_per_bit); + if (sad < best_sad) { + best_sad = sad; + best_site = j; + } + } + } + } + + if (best_site == -1) { + break; + } else { + ref_mv->row += neighbors[best_site].row; + ref_mv->col += neighbors[best_site].col; + } + } + return best_sad; +} + +int masked_diamond_search_sad(const MACROBLOCK *x, + const search_site_config *cfg, + const uint8_t *mask, int mask_stride, MV *ref_mv, + MV *best_mv, int search_param, int sad_per_bit, + int *num00, const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv, int is_second) { + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const what = &x->plane[0].src; + const struct buf_2d *const in_what = &xd->plane[0].pre[is_second]; + // search_param determines the length of the initial step and hence the number + // of iterations + // 0 = initial step (MAX_FIRST_STEP) pel : 1 = (MAX_FIRST_STEP/2) pel, 2 = + // (MAX_FIRST_STEP/4) pel... etc. + const search_site *const ss = &cfg->ss[search_param * cfg->searches_per_step]; + const int tot_steps = (cfg->ss_count / cfg->searches_per_step) - search_param; + const MV fcenter_mv = { center_mv->row >> 3, center_mv->col >> 3 }; + const uint8_t *best_address, *in_what_ref; + int best_sad = INT_MAX; + int best_site = 0; + int last_site = 0; + int i, j, step; + + clamp_mv(ref_mv, x->mv_limits.col_min, x->mv_limits.col_max, + x->mv_limits.row_min, x->mv_limits.row_max); + in_what_ref = get_buf_from_mv(in_what, ref_mv); + best_address = in_what_ref; + *num00 = 0; + *best_mv = *ref_mv; + + // Check the starting position + best_sad = fn_ptr->msdf(what->buf, what->stride, best_address, + in_what->stride, mask, mask_stride) + + mvsad_err_cost(x, best_mv, &fcenter_mv, sad_per_bit); + + i = 1; + + for (step = 0; step < tot_steps; step++) { + for (j = 0; j < cfg->searches_per_step; j++) { + const MV mv = { best_mv->row + ss[i].mv.row, + best_mv->col + ss[i].mv.col }; + if (is_mv_in(&x->mv_limits, &mv)) { + int sad = + fn_ptr->msdf(what->buf, what->stride, best_address + ss[i].offset, + in_what->stride, mask, mask_stride); + if (sad < best_sad) { + sad += mvsad_err_cost(x, &mv, &fcenter_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + best_site = i; + } + } + } + + i++; + } + + if (best_site != last_site) { + best_mv->row += ss[best_site].mv.row; + best_mv->col += ss[best_site].mv.col; + best_address += ss[best_site].offset; + last_site = best_site; +#if defined(NEW_DIAMOND_SEARCH) + while (1) { + const MV this_mv = { best_mv->row + ss[best_site].mv.row, + best_mv->col + ss[best_site].mv.col }; + if (is_mv_in(&x->mv_limits, &this_mv)) { + int sad = fn_ptr->msdf(what->buf, what->stride, + best_address + ss[best_site].offset, + in_what->stride, mask, mask_stride); + if (sad < best_sad) { + sad += mvsad_err_cost(x, &this_mv, &fcenter_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + best_mv->row += ss[best_site].mv.row; + best_mv->col += ss[best_site].mv.col; + best_address += ss[best_site].offset; + continue; + } + } + } + break; + } +#endif + } else if (best_address == in_what_ref) { + (*num00)++; + } + } + return best_sad; +} + +int av1_masked_full_pixel_diamond(const AV1_COMP *cpi, MACROBLOCK *x, + const uint8_t *mask, int mask_stride, + MV *mvp_full, int step_param, int sadpb, + int further_steps, int do_refine, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *ref_mv, MV *dst_mv, int is_second) { + MV temp_mv; + int thissme, n, num00 = 0; + int bestsme = masked_diamond_search_sad(x, &cpi->ss_cfg, mask, mask_stride, + mvp_full, &temp_mv, step_param, sadpb, + &n, fn_ptr, ref_mv, is_second); + if (bestsme < INT_MAX) + bestsme = get_masked_mvpred_var(x, mask, mask_stride, &temp_mv, ref_mv, + fn_ptr, 1, is_second); + *dst_mv = temp_mv; + + // If there won't be more n-step search, check to see if refining search is + // needed. + if (n > further_steps) do_refine = 0; + + while (n < further_steps) { + ++n; + + if (num00) { + num00--; + } else { + thissme = masked_diamond_search_sad( + x, &cpi->ss_cfg, mask, mask_stride, mvp_full, &temp_mv, + step_param + n, sadpb, &num00, fn_ptr, ref_mv, is_second); + if (thissme < INT_MAX) + thissme = get_masked_mvpred_var(x, mask, mask_stride, &temp_mv, ref_mv, + fn_ptr, 1, is_second); + + // check to see if refining search is needed. + if (num00 > further_steps - n) do_refine = 0; + + if (thissme < bestsme) { + bestsme = thissme; + *dst_mv = temp_mv; + } + } + } + + // final 1-away diamond refining search + if (do_refine) { + const int search_range = 8; + MV best_mv = *dst_mv; + thissme = + masked_refining_search_sad(x, mask, mask_stride, &best_mv, sadpb, + search_range, fn_ptr, ref_mv, is_second); + if (thissme < INT_MAX) + thissme = get_masked_mvpred_var(x, mask, mask_stride, &best_mv, ref_mv, + fn_ptr, 1, is_second); + if (thissme < bestsme) { + bestsme = thissme; + *dst_mv = best_mv; + } + } + return bestsme; +} +#endif // CONFIG_EXT_INTER + +#if CONFIG_MOTION_VAR +/* returns subpixel variance error function */ +#define DIST(r, c) \ + vfp->osvf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, mask, &sse) + +/* checks if (r, c) has better score than previous best */ +#define MVC(r, c) \ + (mvcost \ + ? ((mvjcost[((r) != rr) * 2 + ((c) != rc)] + mvcost[0][((r)-rr)] + \ + mvcost[1][((c)-rc)]) * \ + error_per_bit + \ + 4096) >> \ + 13 \ + : 0) + +#define CHECK_BETTER(v, r, c) \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + thismse = (DIST(r, c)); \ + if ((v = MVC(r, c) + thismse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } + +#undef CHECK_BETTER0 +#define CHECK_BETTER0(v, r, c) CHECK_BETTER(v, r, c) + +#undef CHECK_BETTER1 +#define CHECK_BETTER1(v, r, c) \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + thismse = upsampled_obmc_pref_error( \ + xd, mask, vfp, z, upre(y, y_stride, r, c), y_stride, w, h, &sse); \ + if ((v = MVC(r, c) + thismse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } + +static unsigned int setup_obmc_center_error( + const int32_t *mask, const MV *bestmv, const MV *ref_mv, int error_per_bit, + const aom_variance_fn_ptr_t *vfp, const int32_t *const wsrc, + const uint8_t *const y, int y_stride, int offset, int *mvjcost, + int *mvcost[2], unsigned int *sse1, int *distortion) { + unsigned int besterr; + besterr = vfp->ovf(y + offset, y_stride, wsrc, mask, sse1); + *distortion = besterr; + besterr += mv_err_cost(bestmv, ref_mv, mvjcost, mvcost, error_per_bit); + return besterr; +} + +static int upsampled_obmc_pref_error(const MACROBLOCKD *xd, const int32_t *mask, + const aom_variance_fn_ptr_t *vfp, + const int32_t *const wsrc, + const uint8_t *const y, int y_stride, + int w, int h, unsigned int *sse) { + unsigned int besterr; +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + DECLARE_ALIGNED(16, uint16_t, pred16[MAX_SB_SQUARE]); + aom_highbd_upsampled_pred(pred16, w, h, y, y_stride); + + besterr = vfp->ovf(CONVERT_TO_BYTEPTR(pred16), w, wsrc, mask, sse); + } else { + DECLARE_ALIGNED(16, uint8_t, pred[MAX_SB_SQUARE]); +#else + DECLARE_ALIGNED(16, uint8_t, pred[MAX_SB_SQUARE]); + (void)xd; +#endif // CONFIG_HIGHBITDEPTH + aom_upsampled_pred(pred, w, h, y, y_stride); + + besterr = vfp->ovf(pred, w, wsrc, mask, sse); +#if CONFIG_HIGHBITDEPTH + } +#endif + return besterr; +} + +static unsigned int upsampled_setup_obmc_center_error( + const MACROBLOCKD *xd, const int32_t *mask, const MV *bestmv, + const MV *ref_mv, int error_per_bit, const aom_variance_fn_ptr_t *vfp, + const int32_t *const wsrc, const uint8_t *const y, int y_stride, int w, + int h, int offset, int *mvjcost, int *mvcost[2], unsigned int *sse1, + int *distortion) { + unsigned int besterr = upsampled_obmc_pref_error( + xd, mask, vfp, wsrc, y + offset, y_stride, w, h, sse1); + *distortion = besterr; + besterr += mv_err_cost(bestmv, ref_mv, mvjcost, mvcost, error_per_bit); + return besterr; +} + +int av1_find_best_obmc_sub_pixel_tree_up( + const AV1_COMP *cpi, MACROBLOCK *x, int mi_row, int mi_col, MV *bestmv, + const MV *ref_mv, int allow_hp, int error_per_bit, + const aom_variance_fn_ptr_t *vfp, int forced_stop, int iters_per_step, + int *mvjcost, int *mvcost[2], int *distortion, unsigned int *sse1, + int is_second, int use_upsampled_ref) { + const int32_t *wsrc = x->wsrc_buf; + const int32_t *mask = x->mask_buf; + const int *const z = wsrc; + const int *const src_address = z; + MACROBLOCKD *xd = &x->e_mbd; + struct macroblockd_plane *const pd = &xd->plane[0]; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + unsigned int besterr = INT_MAX; + unsigned int sse; + unsigned int thismse; + + int rr = ref_mv->row; + int rc = ref_mv->col; + int br = bestmv->row * 8; + int bc = bestmv->col * 8; + int hstep = 4; + int iter; + int round = 3 - forced_stop; + int tr = br; + int tc = bc; + const MV *search_step = search_step_table; + int idx, best_idx = -1; + unsigned int cost_array[5]; + int kr, kc; + const int w = block_size_wide[mbmi->sb_type]; + const int h = block_size_high[mbmi->sb_type]; + int offset; + int y_stride; + const uint8_t *y; + + const struct buf_2d backup_pred = pd->pre[is_second]; + int minc, maxc, minr, maxr; + + av1_set_subpel_mv_search_range(&x->mv_limits, &minc, &maxc, &minr, &maxr, + ref_mv); + + if (use_upsampled_ref) { + int ref = xd->mi[0]->mbmi.ref_frame[is_second]; + const YV12_BUFFER_CONFIG *upsampled_ref = get_upsampled_ref(cpi, ref); + setup_pred_plane(&pd->pre[is_second], mbmi->sb_type, + upsampled_ref->y_buffer, upsampled_ref->y_crop_width, + upsampled_ref->y_crop_height, upsampled_ref->y_stride, + (mi_row << 3), (mi_col << 3), NULL, pd->subsampling_x, + pd->subsampling_y); + } + y = pd->pre[is_second].buf; + y_stride = pd->pre[is_second].stride; + offset = bestmv->row * y_stride + bestmv->col; + + if (!allow_hp) + if (round == 3) round = 2; + + bestmv->row *= 8; + bestmv->col *= 8; + // use_upsampled_ref can be 0 or 1 + if (use_upsampled_ref) + besterr = upsampled_setup_obmc_center_error( + xd, mask, bestmv, ref_mv, error_per_bit, vfp, z, y, y_stride, w, h, + (offset * 8), mvjcost, mvcost, sse1, distortion); + else + besterr = setup_obmc_center_error(mask, bestmv, ref_mv, error_per_bit, vfp, + z, y, y_stride, offset, mvjcost, mvcost, + sse1, distortion); + + for (iter = 0; iter < round; ++iter) { + // Check vertical and horizontal sub-pixel positions. + for (idx = 0; idx < 4; ++idx) { + tr = br + search_step[idx].row; + tc = bc + search_step[idx].col; + if (tc >= minc && tc <= maxc && tr >= minr && tr <= maxr) { + MV this_mv = { tr, tc }; + + if (use_upsampled_ref) { + const uint8_t *const pre_address = y + tr * y_stride + tc; + + thismse = upsampled_obmc_pref_error( + xd, mask, vfp, src_address, pre_address, y_stride, w, h, &sse); + } else { + const uint8_t *const pre_address = + y + (tr >> 3) * y_stride + (tc >> 3); + thismse = vfp->osvf(pre_address, y_stride, sp(tc), sp(tr), + src_address, mask, &sse); + } + + cost_array[idx] = thismse + mv_err_cost(&this_mv, ref_mv, mvjcost, + mvcost, error_per_bit); + if (cost_array[idx] < besterr) { + best_idx = idx; + besterr = cost_array[idx]; + *distortion = thismse; + *sse1 = sse; + } + } else { + cost_array[idx] = INT_MAX; + } + } + + // Check diagonal sub-pixel position + kc = (cost_array[0] <= cost_array[1] ? -hstep : hstep); + kr = (cost_array[2] <= cost_array[3] ? -hstep : hstep); + + tc = bc + kc; + tr = br + kr; + if (tc >= minc && tc <= maxc && tr >= minr && tr <= maxr) { + MV this_mv = { tr, tc }; + + if (use_upsampled_ref) { + const uint8_t *const pre_address = y + tr * y_stride + tc; + + thismse = upsampled_obmc_pref_error(xd, mask, vfp, src_address, + pre_address, y_stride, w, h, &sse); + } else { + const uint8_t *const pre_address = y + (tr >> 3) * y_stride + (tc >> 3); + + thismse = vfp->osvf(pre_address, y_stride, sp(tc), sp(tr), src_address, + mask, &sse); + } + + cost_array[4] = thismse + mv_err_cost(&this_mv, ref_mv, mvjcost, mvcost, + error_per_bit); + + if (cost_array[4] < besterr) { + best_idx = 4; + besterr = cost_array[4]; + *distortion = thismse; + *sse1 = sse; + } + } else { + cost_array[idx] = INT_MAX; + } + + if (best_idx < 4 && best_idx >= 0) { + br += search_step[best_idx].row; + bc += search_step[best_idx].col; + } else if (best_idx == 4) { + br = tr; + bc = tc; + } + + if (iters_per_step > 1 && best_idx != -1) { + if (use_upsampled_ref) { + SECOND_LEVEL_CHECKS_BEST(1); + } else { + SECOND_LEVEL_CHECKS_BEST(0); + } + } + + tr = br; + tc = bc; + + search_step += 4; + hstep >>= 1; + best_idx = -1; + } + + // These lines insure static analysis doesn't warn that + // tr and tc aren't used after the above point. + (void)tr; + (void)tc; + + bestmv->row = br; + bestmv->col = bc; + + if (use_upsampled_ref) { + pd->pre[is_second] = backup_pred; + } + + return besterr; +} + +#undef DIST +#undef MVC +#undef CHECK_BETTER + +static int get_obmc_mvpred_var(const MACROBLOCK *x, const int32_t *wsrc, + const int32_t *mask, const MV *best_mv, + const MV *center_mv, + const aom_variance_fn_ptr_t *vfp, int use_mvcost, + int is_second) { + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const in_what = &xd->plane[0].pre[is_second]; + const MV mv = { best_mv->row * 8, best_mv->col * 8 }; + unsigned int unused; + + return vfp->ovf(get_buf_from_mv(in_what, best_mv), in_what->stride, wsrc, + mask, &unused) + + (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmvjointcost, x->mvcost, + x->errorperbit) + : 0); +} + +int obmc_refining_search_sad(const MACROBLOCK *x, const int32_t *wsrc, + const int32_t *mask, MV *ref_mv, int error_per_bit, + int search_range, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv, int is_second) { + const MV neighbors[4] = { { -1, 0 }, { 0, -1 }, { 0, 1 }, { 1, 0 } }; + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const in_what = &xd->plane[0].pre[is_second]; + const MV fcenter_mv = { center_mv->row >> 3, center_mv->col >> 3 }; + unsigned int best_sad = fn_ptr->osdf(get_buf_from_mv(in_what, ref_mv), + in_what->stride, wsrc, mask) + + mvsad_err_cost(x, ref_mv, &fcenter_mv, error_per_bit); + int i, j; + + for (i = 0; i < search_range; i++) { + int best_site = -1; + + for (j = 0; j < 4; j++) { + const MV mv = { ref_mv->row + neighbors[j].row, + ref_mv->col + neighbors[j].col }; + if (is_mv_in(&x->mv_limits, &mv)) { + unsigned int sad = fn_ptr->osdf(get_buf_from_mv(in_what, &mv), + in_what->stride, wsrc, mask); + if (sad < best_sad) { + sad += mvsad_err_cost(x, &mv, &fcenter_mv, error_per_bit); + if (sad < best_sad) { + best_sad = sad; + best_site = j; + } + } + } + } + + if (best_site == -1) { + break; + } else { + ref_mv->row += neighbors[best_site].row; + ref_mv->col += neighbors[best_site].col; + } + } + return best_sad; +} + +int obmc_diamond_search_sad(const MACROBLOCK *x, const search_site_config *cfg, + const int32_t *wsrc, const int32_t *mask, + MV *ref_mv, MV *best_mv, int search_param, + int sad_per_bit, int *num00, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv, int is_second) { + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const in_what = &xd->plane[0].pre[is_second]; + // search_param determines the length of the initial step and hence the number + // of iterations + // 0 = initial step (MAX_FIRST_STEP) pel : 1 = (MAX_FIRST_STEP/2) pel, 2 = + // (MAX_FIRST_STEP/4) pel... etc. + const search_site *const ss = &cfg->ss[search_param * cfg->searches_per_step]; + const int tot_steps = (cfg->ss_count / cfg->searches_per_step) - search_param; + const MV fcenter_mv = { center_mv->row >> 3, center_mv->col >> 3 }; + const uint8_t *best_address, *in_what_ref; + int best_sad = INT_MAX; + int best_site = 0; + int last_site = 0; + int i, j, step; + + clamp_mv(ref_mv, x->mv_limits.col_min, x->mv_limits.col_max, + x->mv_limits.row_min, x->mv_limits.row_max); + in_what_ref = in_what->buf + ref_mv->row * in_what->stride + ref_mv->col; + best_address = in_what_ref; + *num00 = 0; + *best_mv = *ref_mv; + + // Check the starting position + best_sad = fn_ptr->osdf(best_address, in_what->stride, wsrc, mask) + + mvsad_err_cost(x, best_mv, &fcenter_mv, sad_per_bit); + + i = 1; + + for (step = 0; step < tot_steps; step++) { + for (j = 0; j < cfg->searches_per_step; j++) { + const MV mv = { best_mv->row + ss[i].mv.row, + best_mv->col + ss[i].mv.col }; + if (is_mv_in(&x->mv_limits, &mv)) { + int sad = fn_ptr->osdf(best_address + ss[i].offset, in_what->stride, + wsrc, mask); + if (sad < best_sad) { + sad += mvsad_err_cost(x, &mv, &fcenter_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + best_site = i; + } + } + } + + i++; + } + + if (best_site != last_site) { + best_mv->row += ss[best_site].mv.row; + best_mv->col += ss[best_site].mv.col; + best_address += ss[best_site].offset; + last_site = best_site; +#if defined(NEW_DIAMOND_SEARCH) + while (1) { + const MV this_mv = { best_mv->row + ss[best_site].mv.row, + best_mv->col + ss[best_site].mv.col }; + if (is_mv_in(&x->mv_limits, &this_mv)) { + int sad = fn_ptr->osdf(best_address + ss[best_site].offset, + in_what->stride, wsrc, mask); + if (sad < best_sad) { + sad += mvsad_err_cost(x, &this_mv, &fcenter_mv, sad_per_bit); + if (sad < best_sad) { + best_sad = sad; + best_mv->row += ss[best_site].mv.row; + best_mv->col += ss[best_site].mv.col; + best_address += ss[best_site].offset; + continue; + } + } + } + break; + } +#endif + } else if (best_address == in_what_ref) { + (*num00)++; + } + } + return best_sad; +} + +int av1_obmc_full_pixel_diamond(const AV1_COMP *cpi, MACROBLOCK *x, + MV *mvp_full, int step_param, int sadpb, + int further_steps, int do_refine, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *ref_mv, MV *dst_mv, int is_second) { + const int32_t *wsrc = x->wsrc_buf; + const int32_t *mask = x->mask_buf; + MV temp_mv; + int thissme, n, num00 = 0; + int bestsme = + obmc_diamond_search_sad(x, &cpi->ss_cfg, wsrc, mask, mvp_full, &temp_mv, + step_param, sadpb, &n, fn_ptr, ref_mv, is_second); + if (bestsme < INT_MAX) + bestsme = get_obmc_mvpred_var(x, wsrc, mask, &temp_mv, ref_mv, fn_ptr, 1, + is_second); + *dst_mv = temp_mv; + + // If there won't be more n-step search, check to see if refining search is + // needed. + if (n > further_steps) do_refine = 0; + + while (n < further_steps) { + ++n; + + if (num00) { + num00--; + } else { + thissme = obmc_diamond_search_sad(x, &cpi->ss_cfg, wsrc, mask, mvp_full, + &temp_mv, step_param + n, sadpb, &num00, + fn_ptr, ref_mv, is_second); + if (thissme < INT_MAX) + thissme = get_obmc_mvpred_var(x, wsrc, mask, &temp_mv, ref_mv, fn_ptr, + 1, is_second); + + // check to see if refining search is needed. + if (num00 > further_steps - n) do_refine = 0; + + if (thissme < bestsme) { + bestsme = thissme; + *dst_mv = temp_mv; + } + } + } + + // final 1-away diamond refining search + if (do_refine) { + const int search_range = 8; + MV best_mv = *dst_mv; + thissme = obmc_refining_search_sad(x, wsrc, mask, &best_mv, sadpb, + search_range, fn_ptr, ref_mv, is_second); + if (thissme < INT_MAX) + thissme = get_obmc_mvpred_var(x, wsrc, mask, &best_mv, ref_mv, fn_ptr, 1, + is_second); + if (thissme < bestsme) { + bestsme = thissme; + *dst_mv = best_mv; + } + } + return bestsme; +} +#endif // CONFIG_MOTION_VAR + +// Note(yunqingwang): The following 2 functions are only used in the motion +// vector unit test, which return extreme motion vectors allowed by the MV +// limits. +#define COMMON_MV_TEST \ + SETUP_SUBPEL_SEARCH; \ + \ + (void)error_per_bit; \ + (void)vfp; \ + (void)src_address; \ + (void)src_stride; \ + (void)y; \ + (void)y_stride; \ + (void)second_pred; \ + (void)w; \ + (void)h; \ + (void)use_upsampled_ref; \ + (void)offset; \ + (void)mvjcost; \ + (void)mvcost; \ + (void)sse1; \ + (void)distortion; \ + \ + (void)halfiters; \ + (void)quarteriters; \ + (void)eighthiters; \ + (void)whichdir; \ + (void)forced_stop; \ + (void)hstep; \ + \ + (void)tr; \ + (void)tc; \ + (void)sse; \ + (void)thismse; \ + (void)cost_list; +// Return the maximum MV. +int av1_return_max_sub_pixel_mv(MACROBLOCK *x, const MV *ref_mv, int allow_hp, + int error_per_bit, + const aom_variance_fn_ptr_t *vfp, + int forced_stop, int iters_per_step, + int *cost_list, int *mvjcost, int *mvcost[2], + int *distortion, unsigned int *sse1, + const uint8_t *second_pred, int w, int h, + int use_upsampled_ref) { + COMMON_MV_TEST; + (void)minr; + (void)minc; + bestmv->row = maxr; + bestmv->col = maxc; + besterr = 0; + // In the sub-pel motion search, if hp is not used, then the last bit of mv + // has to be 0. + lower_mv_precision(bestmv, allow_hp); + return besterr; +} +// Return the minimum MV. +int av1_return_min_sub_pixel_mv(MACROBLOCK *x, const MV *ref_mv, int allow_hp, + int error_per_bit, + const aom_variance_fn_ptr_t *vfp, + int forced_stop, int iters_per_step, + int *cost_list, int *mvjcost, int *mvcost[2], + int *distortion, unsigned int *sse1, + const uint8_t *second_pred, int w, int h, + int use_upsampled_ref) { + COMMON_MV_TEST; + (void)maxr; + (void)maxc; + bestmv->row = minr; + bestmv->col = minc; + besterr = 0; + // In the sub-pel motion search, if hp is not used, then the last bit of mv + // has to be 0. + lower_mv_precision(bestmv, allow_hp); + return besterr; +} diff --git a/third_party/aom/av1/encoder/mcomp.h b/third_party/aom/av1/encoder/mcomp.h new file mode 100644 index 0000000000..8465860adf --- /dev/null +++ b/third_party/aom/av1/encoder/mcomp.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_MCOMP_H_ +#define AV1_ENCODER_MCOMP_H_ + +#include "av1/encoder/block.h" +#include "aom_dsp/variance.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// The maximum number of steps in a step search given the largest +// allowed initial step +#define MAX_MVSEARCH_STEPS 11 +// Max full pel mv specified in the unit of full pixel +// Enable the use of motion vector in range [-1023, 1023]. +#define MAX_FULL_PEL_VAL ((1 << (MAX_MVSEARCH_STEPS - 1)) - 1) +// Maximum size of the first step in full pel units +#define MAX_FIRST_STEP (1 << (MAX_MVSEARCH_STEPS - 1)) +// Allowed motion vector pixel distance outside image border +// for Block_16x16 +#define BORDER_MV_PIXELS_B16 (16 + AOM_INTERP_EXTEND) + +// motion search site +typedef struct search_site { + MV mv; + int offset; +} search_site; + +typedef struct search_site_config { + search_site ss[8 * MAX_MVSEARCH_STEPS + 1]; + int ss_count; + int searches_per_step; +} search_site_config; + +void av1_init_dsmotion_compensation(search_site_config *cfg, int stride); +void av1_init3smotion_compensation(search_site_config *cfg, int stride); + +void av1_set_mv_search_range(MvLimits *mv_limits, const MV *mv); + +int av1_mv_bit_cost(const MV *mv, const MV *ref, const int *mvjcost, + int *mvcost[2], int weight); + +// Utility to compute variance + MV rate cost for a given MV +int av1_get_mvpred_var(const MACROBLOCK *x, const MV *best_mv, + const MV *center_mv, const aom_variance_fn_ptr_t *vfp, + int use_mvcost); +int av1_get_mvpred_av_var(const MACROBLOCK *x, const MV *best_mv, + const MV *center_mv, const uint8_t *second_pred, + const aom_variance_fn_ptr_t *vfp, int use_mvcost); + +struct AV1_COMP; +struct SPEED_FEATURES; + +int av1_init_search_range(int size); + +int av1_refining_search_sad(struct macroblock *x, struct mv *ref_mv, + int sad_per_bit, int distance, + const aom_variance_fn_ptr_t *fn_ptr, + const struct mv *center_mv); + +// Runs sequence of diamond searches in smaller steps for RD. +int av1_full_pixel_diamond(const struct AV1_COMP *cpi, MACROBLOCK *x, + MV *mvp_full, int step_param, int sadpb, + int further_steps, int do_refine, int *cost_list, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *ref_mv, MV *dst_mv); + +// Perform integral projection based motion estimation. +unsigned int av1_int_pro_motion_estimation(const struct AV1_COMP *cpi, + MACROBLOCK *x, BLOCK_SIZE bsize, + int mi_row, int mi_col); + +int av1_hex_search(MACROBLOCK *x, MV *start_mv, int search_param, + int sad_per_bit, int do_init_search, int *cost_list, + const aom_variance_fn_ptr_t *vfp, int use_mvcost, + const MV *center_mv); + +typedef int(fractional_mv_step_fp)( + MACROBLOCK *x, const MV *ref_mv, int allow_hp, int error_per_bit, + const aom_variance_fn_ptr_t *vfp, + int forced_stop, // 0 - full, 1 - qtr only, 2 - half only + int iters_per_step, int *cost_list, int *mvjcost, int *mvcost[2], + int *distortion, unsigned int *sse1, const uint8_t *second_pred, int w, + int h, int use_upsampled_ref); + +extern fractional_mv_step_fp av1_find_best_sub_pixel_tree; +extern fractional_mv_step_fp av1_find_best_sub_pixel_tree_pruned; +extern fractional_mv_step_fp av1_find_best_sub_pixel_tree_pruned_more; +extern fractional_mv_step_fp av1_find_best_sub_pixel_tree_pruned_evenmore; +extern fractional_mv_step_fp av1_return_max_sub_pixel_mv; +extern fractional_mv_step_fp av1_return_min_sub_pixel_mv; + +typedef int (*av1_full_search_fn_t)(const MACROBLOCK *x, const MV *ref_mv, + int sad_per_bit, int distance, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv, MV *best_mv); + +typedef int (*av1_diamond_search_fn_t)( + MACROBLOCK *x, const search_site_config *cfg, MV *ref_mv, MV *best_mv, + int search_param, int sad_per_bit, int *num00, + const aom_variance_fn_ptr_t *fn_ptr, const MV *center_mv); + +int av1_refining_search_8p_c(MACROBLOCK *x, int error_per_bit, int search_range, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv, const uint8_t *second_pred); + +struct AV1_COMP; + +int av1_full_pixel_search(const struct AV1_COMP *cpi, MACROBLOCK *x, + BLOCK_SIZE bsize, MV *mvp_full, int step_param, + int error_per_bit, int *cost_list, const MV *ref_mv, + int var_max, int rd); + +#if CONFIG_EXT_INTER +int av1_find_best_masked_sub_pixel_tree( + const MACROBLOCK *x, const uint8_t *mask, int mask_stride, MV *bestmv, + const MV *ref_mv, int allow_hp, int error_per_bit, + const aom_variance_fn_ptr_t *vfp, int forced_stop, int iters_per_step, + int *mvjcost, int *mvcost[2], int *distortion, unsigned int *sse1, + int is_second); +int av1_find_best_masked_sub_pixel_tree_up( + const struct AV1_COMP *cpi, MACROBLOCK *x, const uint8_t *mask, + int mask_stride, int mi_row, int mi_col, MV *bestmv, const MV *ref_mv, + int allow_hp, int error_per_bit, const aom_variance_fn_ptr_t *vfp, + int forced_stop, int iters_per_step, int *mvjcost, int *mvcost[2], + int *distortion, unsigned int *sse1, int is_second, int use_upsampled_ref); +int av1_masked_full_pixel_diamond(const struct AV1_COMP *cpi, MACROBLOCK *x, + const uint8_t *mask, int mask_stride, + MV *mvp_full, int step_param, int sadpb, + int further_steps, int do_refine, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *ref_mv, MV *dst_mv, int is_second); +#endif // CONFIG_EXT_INTER + +#if CONFIG_MOTION_VAR +int av1_obmc_full_pixel_diamond(const struct AV1_COMP *cpi, MACROBLOCK *x, + MV *mvp_full, int step_param, int sadpb, + int further_steps, int do_refine, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *ref_mv, MV *dst_mv, int is_second); +int av1_find_best_obmc_sub_pixel_tree_up( + const struct AV1_COMP *cpi, MACROBLOCK *x, int mi_row, int mi_col, + MV *bestmv, const MV *ref_mv, int allow_hp, int error_per_bit, + const aom_variance_fn_ptr_t *vfp, int forced_stop, int iters_per_step, + int *mvjcost, int *mvcost[2], int *distortion, unsigned int *sse1, + int is_second, int use_upsampled_ref); +#endif // CONFIG_MOTION_VAR +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_MCOMP_H_ diff --git a/third_party/aom/av1/encoder/mips/msa/error_msa.c b/third_party/aom/av1/encoder/mips/msa/error_msa.c new file mode 100644 index 0000000000..8d13af7ad9 --- /dev/null +++ b/third_party/aom/av1/encoder/mips/msa/error_msa.c @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./av1_rtcd.h" +#include "aom_dsp/mips/macros_msa.h" + +#define BLOCK_ERROR_BLOCKSIZE_MSA(BSize) \ + static int64_t block_error_##BSize##size_msa( \ + const int16_t *coeff_ptr, const int16_t *dq_coeff_ptr, int64_t *ssz) { \ + int64_t err = 0; \ + uint32_t loop_cnt; \ + v8i16 coeff, dq_coeff, coeff_r_h, coeff_l_h; \ + v4i32 diff_r, diff_l, coeff_r_w, coeff_l_w; \ + v2i64 sq_coeff_r, sq_coeff_l; \ + v2i64 err0, err_dup0, err1, err_dup1; \ + \ + coeff = LD_SH(coeff_ptr); \ + dq_coeff = LD_SH(dq_coeff_ptr); \ + UNPCK_SH_SW(coeff, coeff_r_w, coeff_l_w); \ + ILVRL_H2_SH(coeff, dq_coeff, coeff_r_h, coeff_l_h); \ + HSUB_UH2_SW(coeff_r_h, coeff_l_h, diff_r, diff_l); \ + DOTP_SW2_SD(coeff_r_w, coeff_l_w, coeff_r_w, coeff_l_w, sq_coeff_r, \ + sq_coeff_l); \ + DOTP_SW2_SD(diff_r, diff_l, diff_r, diff_l, err0, err1); \ + \ + coeff = LD_SH(coeff_ptr + 8); \ + dq_coeff = LD_SH(dq_coeff_ptr + 8); \ + UNPCK_SH_SW(coeff, coeff_r_w, coeff_l_w); \ + ILVRL_H2_SH(coeff, dq_coeff, coeff_r_h, coeff_l_h); \ + HSUB_UH2_SW(coeff_r_h, coeff_l_h, diff_r, diff_l); \ + DPADD_SD2_SD(coeff_r_w, coeff_l_w, sq_coeff_r, sq_coeff_l); \ + DPADD_SD2_SD(diff_r, diff_l, err0, err1); \ + \ + coeff_ptr += 16; \ + dq_coeff_ptr += 16; \ + \ + for (loop_cnt = ((BSize >> 4) - 1); loop_cnt--;) { \ + coeff = LD_SH(coeff_ptr); \ + dq_coeff = LD_SH(dq_coeff_ptr); \ + UNPCK_SH_SW(coeff, coeff_r_w, coeff_l_w); \ + ILVRL_H2_SH(coeff, dq_coeff, coeff_r_h, coeff_l_h); \ + HSUB_UH2_SW(coeff_r_h, coeff_l_h, diff_r, diff_l); \ + DPADD_SD2_SD(coeff_r_w, coeff_l_w, sq_coeff_r, sq_coeff_l); \ + DPADD_SD2_SD(diff_r, diff_l, err0, err1); \ + \ + coeff = LD_SH(coeff_ptr + 8); \ + dq_coeff = LD_SH(dq_coeff_ptr + 8); \ + UNPCK_SH_SW(coeff, coeff_r_w, coeff_l_w); \ + ILVRL_H2_SH(coeff, dq_coeff, coeff_r_h, coeff_l_h); \ + HSUB_UH2_SW(coeff_r_h, coeff_l_h, diff_r, diff_l); \ + DPADD_SD2_SD(coeff_r_w, coeff_l_w, sq_coeff_r, sq_coeff_l); \ + DPADD_SD2_SD(diff_r, diff_l, err0, err1); \ + \ + coeff_ptr += 16; \ + dq_coeff_ptr += 16; \ + } \ + \ + err_dup0 = __msa_splati_d(sq_coeff_r, 1); \ + err_dup1 = __msa_splati_d(sq_coeff_l, 1); \ + sq_coeff_r += err_dup0; \ + sq_coeff_l += err_dup1; \ + *ssz = __msa_copy_s_d(sq_coeff_r, 0); \ + *ssz += __msa_copy_s_d(sq_coeff_l, 0); \ + \ + err_dup0 = __msa_splati_d(err0, 1); \ + err_dup1 = __msa_splati_d(err1, 1); \ + err0 += err_dup0; \ + err1 += err_dup1; \ + err = __msa_copy_s_d(err0, 0); \ + err += __msa_copy_s_d(err1, 0); \ + \ + return err; \ + } + +/* clang-format off */ +BLOCK_ERROR_BLOCKSIZE_MSA(16) +BLOCK_ERROR_BLOCKSIZE_MSA(64) +BLOCK_ERROR_BLOCKSIZE_MSA(256) +BLOCK_ERROR_BLOCKSIZE_MSA(1024) +/* clang-format on */ + +int64_t av1_block_error_msa(const tran_low_t *coeff_ptr, + const tran_low_t *dq_coeff_ptr, intptr_t blk_size, + int64_t *ssz) { + int64_t err; + const int16_t *coeff = (const int16_t *)coeff_ptr; + const int16_t *dq_coeff = (const int16_t *)dq_coeff_ptr; + + switch (blk_size) { + case 16: err = block_error_16size_msa(coeff, dq_coeff, ssz); break; + case 64: err = block_error_64size_msa(coeff, dq_coeff, ssz); break; + case 256: err = block_error_256size_msa(coeff, dq_coeff, ssz); break; + case 1024: err = block_error_1024size_msa(coeff, dq_coeff, ssz); break; + default: + err = av1_block_error_c(coeff_ptr, dq_coeff_ptr, blk_size, ssz); + break; + } + + return err; +} diff --git a/third_party/aom/av1/encoder/mips/msa/fdct16x16_msa.c b/third_party/aom/av1/encoder/mips/msa/fdct16x16_msa.c new file mode 100644 index 0000000000..4b0364d6c8 --- /dev/null +++ b/third_party/aom/av1/encoder/mips/msa/fdct16x16_msa.c @@ -0,0 +1,436 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/enums.h" +#include "av1/encoder/mips/msa/fdct_msa.h" +#include "aom_dsp/mips/fwd_txfm_msa.h" + +static void fadst16_cols_step1_msa(const int16_t *input, int32_t stride, + const int32_t *const0, int16_t *int_buf) { + v8i16 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15; + v8i16 tp0, tp1, tp2, tp3, g0, g1, g2, g3, g8, g9, g10, g11, h0, h1, h2, h3; + v4i32 k0, k1, k2, k3; + + /* load input data */ + r0 = LD_SH(input); + r15 = LD_SH(input + 15 * stride); + r7 = LD_SH(input + 7 * stride); + r8 = LD_SH(input + 8 * stride); + SLLI_4V(r0, r15, r7, r8, 2); + + /* stage 1 */ + LD_SW2(const0, 4, k0, k1); + LD_SW2(const0 + 8, 4, k2, k3); + MADD_BF(r15, r0, r7, r8, k0, k1, k2, k3, g0, g1, g2, g3); + + r3 = LD_SH(input + 3 * stride); + r4 = LD_SH(input + 4 * stride); + r11 = LD_SH(input + 11 * stride); + r12 = LD_SH(input + 12 * stride); + SLLI_4V(r3, r4, r11, r12, 2); + + LD_SW2(const0 + 4 * 4, 4, k0, k1); + LD_SW2(const0 + 4 * 6, 4, k2, k3); + MADD_BF(r11, r4, r3, r12, k0, k1, k2, k3, g8, g9, g10, g11); + + /* stage 2 */ + BUTTERFLY_4(g0, g2, g10, g8, tp0, tp2, tp3, tp1); + ST_SH2(tp0, tp2, int_buf, 8); + ST_SH2(tp1, tp3, int_buf + 4 * 8, 8); + + LD_SW2(const0 + 4 * 8, 4, k0, k1); + k2 = LD_SW(const0 + 4 * 10); + MADD_BF(g1, g3, g9, g11, k0, k1, k2, k0, h0, h1, h2, h3); + + ST_SH2(h0, h1, int_buf + 8 * 8, 8); + ST_SH2(h3, h2, int_buf + 12 * 8, 8); + + r9 = LD_SH(input + 9 * stride); + r6 = LD_SH(input + 6 * stride); + r1 = LD_SH(input + stride); + r14 = LD_SH(input + 14 * stride); + SLLI_4V(r9, r6, r1, r14, 2); + + LD_SW2(const0 + 4 * 11, 4, k0, k1); + LD_SW2(const0 + 4 * 13, 4, k2, k3); + MADD_BF(r9, r6, r1, r14, k0, k1, k2, k3, g0, g1, g2, g3); + + ST_SH2(g1, g3, int_buf + 3 * 8, 4 * 8); + + r13 = LD_SH(input + 13 * stride); + r2 = LD_SH(input + 2 * stride); + r5 = LD_SH(input + 5 * stride); + r10 = LD_SH(input + 10 * stride); + SLLI_4V(r13, r2, r5, r10, 2); + + LD_SW2(const0 + 4 * 15, 4, k0, k1); + LD_SW2(const0 + 4 * 17, 4, k2, k3); + MADD_BF(r13, r2, r5, r10, k0, k1, k2, k3, h0, h1, h2, h3); + + ST_SH2(h1, h3, int_buf + 11 * 8, 4 * 8); + + BUTTERFLY_4(h0, h2, g2, g0, tp0, tp1, tp2, tp3); + ST_SH4(tp0, tp1, tp2, tp3, int_buf + 2 * 8, 4 * 8); +} + +static void fadst16_step2_msa_helper(int16_t *int_buf, const int32_t *const0, + int16_t *out, int16_t *out_ptr) { + v8i16 tp0, tp1, tp2, tp3, g5, g7, g13, g15; + v8i16 h0, h1, h2, h3, h4, h5, h6, h7, h10, h11; + v8i16 out0, out1, out2, out3, out4, out5, out6, out7; + v8i16 out8, out9, out10, out11, out12, out13, out14, out15; + v4i32 k0, k1, k2, k3; + + LD_SH2(int_buf + 3 * 8, 4 * 8, g13, g15); + LD_SH2(int_buf + 11 * 8, 4 * 8, g5, g7); + LD_SW2(const0 + 4 * 19, 4, k0, k1); + k2 = LD_SW(const0 + 4 * 21); + MADD_BF(g7, g5, g15, g13, k0, k1, k2, k0, h4, h5, h6, h7); + + tp0 = LD_SH(int_buf + 4 * 8); + tp1 = LD_SH(int_buf + 5 * 8); + tp3 = LD_SH(int_buf + 10 * 8); + tp2 = LD_SH(int_buf + 14 * 8); + LD_SW2(const0 + 4 * 22, 4, k0, k1); + k2 = LD_SW(const0 + 4 * 24); + MADD_BF(tp0, tp1, tp2, tp3, k0, k1, k2, k0, out4, out6, out5, out7); + out4 = -out4; + ST_SH(out4, (out + 3 * 16)); + ST_SH(out5, (out_ptr + 4 * 16)); + + h1 = LD_SH(int_buf + 9 * 8); + h3 = LD_SH(int_buf + 12 * 8); + MADD_BF(h1, h3, h5, h7, k0, k1, k2, k0, out12, out14, out13, out15); + out13 = -out13; + ST_SH(out12, (out + 2 * 16)); + ST_SH(out13, (out_ptr + 5 * 16)); + + tp0 = LD_SH(int_buf); + tp1 = LD_SH(int_buf + 8); + tp2 = LD_SH(int_buf + 2 * 8); + tp3 = LD_SH(int_buf + 6 * 8); + + BUTTERFLY_4(tp0, tp1, tp3, tp2, out0, out1, h11, h10); + out1 = -out1; + ST_SH(out0, (out)); + ST_SH(out1, (out_ptr + 7 * 16)); + + h0 = LD_SH(int_buf + 8 * 8); + h2 = LD_SH(int_buf + 13 * 8); + + BUTTERFLY_4(h0, h2, h6, h4, out8, out9, out11, out10); + out8 = -out8; + ST_SH(out8, (out + 16)); + ST_SH(out9, (out_ptr + 6 * 16)); + + /* stage 4 */ + LD_SW2(const0 + 4 * 25, 4, k0, k1); + LD_SW2(const0 + 4 * 27, 4, k2, k3); + MADD_SHORT(h10, h11, k1, k2, out2, out3); + ST_SH(out2, (out + 7 * 16)); + ST_SH(out3, (out_ptr)); + + MADD_SHORT(out6, out7, k0, k3, out6, out7); + ST_SH(out6, (out + 4 * 16)); + ST_SH(out7, (out_ptr + 3 * 16)); + + MADD_SHORT(out10, out11, k0, k3, out10, out11); + ST_SH(out10, (out + 6 * 16)); + ST_SH(out11, (out_ptr + 16)); + + MADD_SHORT(out14, out15, k1, k2, out14, out15); + ST_SH(out14, (out + 5 * 16)); + ST_SH(out15, (out_ptr + 2 * 16)); +} + +static void fadst16_cols_step2_msa(int16_t *int_buf, const int32_t *const0, + int16_t *out) { + fadst16_step2_msa_helper(int_buf, const0, out, out + 128); +} + +static void fadst16_transpose_postproc_msa(int16_t *input, int16_t *out) { + v8i16 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15; + v8i16 l0, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15; + + /* load input data */ + LD_SH8(input, 16, l0, l1, l2, l3, l4, l5, l6, l7); + TRANSPOSE8x8_SH_SH(l0, l1, l2, l3, l4, l5, l6, l7, r0, r1, r2, r3, r4, r5, r6, + r7); + FDCT_POSTPROC_2V_NEG_H(r0, r1); + FDCT_POSTPROC_2V_NEG_H(r2, r3); + FDCT_POSTPROC_2V_NEG_H(r4, r5); + FDCT_POSTPROC_2V_NEG_H(r6, r7); + ST_SH8(r0, r1, r2, r3, r4, r5, r6, r7, out, 8); + out += 64; + + LD_SH8(input + 8, 16, l8, l9, l10, l11, l12, l13, l14, l15); + TRANSPOSE8x8_SH_SH(l8, l9, l10, l11, l12, l13, l14, l15, r8, r9, r10, r11, + r12, r13, r14, r15); + FDCT_POSTPROC_2V_NEG_H(r8, r9); + FDCT_POSTPROC_2V_NEG_H(r10, r11); + FDCT_POSTPROC_2V_NEG_H(r12, r13); + FDCT_POSTPROC_2V_NEG_H(r14, r15); + ST_SH8(r8, r9, r10, r11, r12, r13, r14, r15, out, 8); + out += 64; + + /* load input data */ + input += 128; + LD_SH8(input, 16, l0, l1, l2, l3, l4, l5, l6, l7); + TRANSPOSE8x8_SH_SH(l0, l1, l2, l3, l4, l5, l6, l7, r0, r1, r2, r3, r4, r5, r6, + r7); + FDCT_POSTPROC_2V_NEG_H(r0, r1); + FDCT_POSTPROC_2V_NEG_H(r2, r3); + FDCT_POSTPROC_2V_NEG_H(r4, r5); + FDCT_POSTPROC_2V_NEG_H(r6, r7); + ST_SH8(r0, r1, r2, r3, r4, r5, r6, r7, out, 8); + out += 64; + + LD_SH8(input + 8, 16, l8, l9, l10, l11, l12, l13, l14, l15); + TRANSPOSE8x8_SH_SH(l8, l9, l10, l11, l12, l13, l14, l15, r8, r9, r10, r11, + r12, r13, r14, r15); + FDCT_POSTPROC_2V_NEG_H(r8, r9); + FDCT_POSTPROC_2V_NEG_H(r10, r11); + FDCT_POSTPROC_2V_NEG_H(r12, r13); + FDCT_POSTPROC_2V_NEG_H(r14, r15); + ST_SH8(r8, r9, r10, r11, r12, r13, r14, r15, out, 8); +} + +static void fadst16_rows_step1_msa(int16_t *input, const int32_t *const0, + int16_t *int_buf) { + v8i16 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15; + v8i16 tp0, tp1, tp2, tp3, g0, g1, g2, g3, g8, g9, g10, g11, h0, h1, h2, h3; + v4i32 k0, k1, k2, k3; + + /* load input data */ + r0 = LD_SH(input); + r7 = LD_SH(input + 7 * 8); + r8 = LD_SH(input + 8 * 8); + r15 = LD_SH(input + 15 * 8); + + /* stage 1 */ + LD_SW2(const0, 4, k0, k1); + LD_SW2(const0 + 4 * 2, 4, k2, k3); + MADD_BF(r15, r0, r7, r8, k0, k1, k2, k3, g0, g1, g2, g3); + + r3 = LD_SH(input + 3 * 8); + r4 = LD_SH(input + 4 * 8); + r11 = LD_SH(input + 11 * 8); + r12 = LD_SH(input + 12 * 8); + + LD_SW2(const0 + 4 * 4, 4, k0, k1); + LD_SW2(const0 + 4 * 6, 4, k2, k3); + MADD_BF(r11, r4, r3, r12, k0, k1, k2, k3, g8, g9, g10, g11); + + /* stage 2 */ + BUTTERFLY_4(g0, g2, g10, g8, tp0, tp2, tp3, tp1); + ST_SH2(tp0, tp1, int_buf, 4 * 8); + ST_SH2(tp2, tp3, int_buf + 8, 4 * 8); + + LD_SW2(const0 + 4 * 8, 4, k0, k1); + k2 = LD_SW(const0 + 4 * 10); + MADD_BF(g1, g3, g9, g11, k0, k1, k2, k0, h0, h1, h2, h3); + ST_SH2(h0, h3, int_buf + 8 * 8, 4 * 8); + ST_SH2(h1, h2, int_buf + 9 * 8, 4 * 8); + + r1 = LD_SH(input + 8); + r6 = LD_SH(input + 6 * 8); + r9 = LD_SH(input + 9 * 8); + r14 = LD_SH(input + 14 * 8); + + LD_SW2(const0 + 4 * 11, 4, k0, k1); + LD_SW2(const0 + 4 * 13, 4, k2, k3); + MADD_BF(r9, r6, r1, r14, k0, k1, k2, k3, g0, g1, g2, g3); + ST_SH2(g1, g3, int_buf + 3 * 8, 4 * 8); + + r2 = LD_SH(input + 2 * 8); + r5 = LD_SH(input + 5 * 8); + r10 = LD_SH(input + 10 * 8); + r13 = LD_SH(input + 13 * 8); + + LD_SW2(const0 + 4 * 15, 4, k0, k1); + LD_SW2(const0 + 4 * 17, 4, k2, k3); + MADD_BF(r13, r2, r5, r10, k0, k1, k2, k3, h0, h1, h2, h3); + ST_SH2(h1, h3, int_buf + 11 * 8, 4 * 8); + BUTTERFLY_4(h0, h2, g2, g0, tp0, tp1, tp2, tp3); + ST_SH4(tp0, tp1, tp2, tp3, int_buf + 2 * 8, 4 * 8); +} + +static void fadst16_rows_step2_msa(int16_t *int_buf, const int32_t *const0, + int16_t *out) { + fadst16_step2_msa_helper(int_buf, const0, out, out + 8); +} + +static void fadst16_transpose_msa(int16_t *input, int16_t *out) { + v8i16 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15; + v8i16 l0, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15; + + /* load input data */ + LD_SH16(input, 8, l0, l8, l1, l9, l2, l10, l3, l11, l4, l12, l5, l13, l6, l14, + l7, l15); + TRANSPOSE8x8_SH_SH(l0, l1, l2, l3, l4, l5, l6, l7, r0, r1, r2, r3, r4, r5, r6, + r7); + TRANSPOSE8x8_SH_SH(l8, l9, l10, l11, l12, l13, l14, l15, r8, r9, r10, r11, + r12, r13, r14, r15); + ST_SH8(r0, r8, r1, r9, r2, r10, r3, r11, out, 8); + ST_SH8(r4, r12, r5, r13, r6, r14, r7, r15, (out + 64), 8); + out += 16 * 8; + + /* load input data */ + input += 128; + LD_SH16(input, 8, l0, l8, l1, l9, l2, l10, l3, l11, l4, l12, l5, l13, l6, l14, + l7, l15); + TRANSPOSE8x8_SH_SH(l0, l1, l2, l3, l4, l5, l6, l7, r0, r1, r2, r3, r4, r5, r6, + r7); + TRANSPOSE8x8_SH_SH(l8, l9, l10, l11, l12, l13, l14, l15, r8, r9, r10, r11, + r12, r13, r14, r15); + ST_SH8(r0, r8, r1, r9, r2, r10, r3, r11, out, 8); + ST_SH8(r4, r12, r5, r13, r6, r14, r7, r15, (out + 64), 8); +} + +static void postproc_fdct16x8_1d_row(int16_t *intermediate, int16_t *output) { + int16_t *temp = intermediate; + int16_t *out = output; + v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + v8i16 in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11; + v8i16 in12, in13, in14, in15; + + LD_SH8(temp, 16, in0, in1, in2, in3, in4, in5, in6, in7); + temp = intermediate + 8; + LD_SH8(temp, 16, in8, in9, in10, in11, in12, in13, in14, in15); + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + TRANSPOSE8x8_SH_SH(in8, in9, in10, in11, in12, in13, in14, in15, in8, in9, + in10, in11, in12, in13, in14, in15); + FDCT_POSTPROC_2V_NEG_H(in0, in1); + FDCT_POSTPROC_2V_NEG_H(in2, in3); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + FDCT_POSTPROC_2V_NEG_H(in6, in7); + FDCT_POSTPROC_2V_NEG_H(in8, in9); + FDCT_POSTPROC_2V_NEG_H(in10, in11); + FDCT_POSTPROC_2V_NEG_H(in12, in13); + FDCT_POSTPROC_2V_NEG_H(in14, in15); + BUTTERFLY_16(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11, + in12, in13, in14, in15, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, + tmp7, in8, in9, in10, in11, in12, in13, in14, in15); + temp = intermediate; + ST_SH8(in8, in9, in10, in11, in12, in13, in14, in15, temp, 16); + FDCT8x16_EVEN(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp0, tmp1, + tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + temp = intermediate; + LD_SH8(temp, 16, in8, in9, in10, in11, in12, in13, in14, in15); + FDCT8x16_ODD(in8, in9, in10, in11, in12, in13, in14, in15, in0, in1, in2, in3, + in4, in5, in6, in7); + TRANSPOSE8x8_SH_SH(tmp0, in0, tmp1, in1, tmp2, in2, tmp3, in3, tmp0, in0, + tmp1, in1, tmp2, in2, tmp3, in3); + ST_SH8(tmp0, in0, tmp1, in1, tmp2, in2, tmp3, in3, out, 16); + TRANSPOSE8x8_SH_SH(tmp4, in4, tmp5, in5, tmp6, in6, tmp7, in7, tmp4, in4, + tmp5, in5, tmp6, in6, tmp7, in7); + out = output + 8; + ST_SH8(tmp4, in4, tmp5, in5, tmp6, in6, tmp7, in7, out, 16); +} + +void av1_fht16x16_msa(const int16_t *input, int16_t *output, int32_t stride, + int32_t tx_type) { + DECLARE_ALIGNED(32, int16_t, tmp[256]); + DECLARE_ALIGNED(32, int16_t, trans_buf[256]); + DECLARE_ALIGNED(32, int16_t, tmp_buf[128]); + int32_t i; + int16_t *ptmpbuf = &tmp_buf[0]; + int16_t *trans = &trans_buf[0]; + const int32_t const_arr[29 * 4] = { + 52707308, 52707308, 52707308, 52707308, -1072430300, + -1072430300, -1072430300, -1072430300, 795618043, 795618043, + 795618043, 795618043, -721080468, -721080468, -721080468, + -721080468, 459094491, 459094491, 459094491, 459094491, + -970646691, -970646691, -970646691, -970646691, 1010963856, + 1010963856, 1010963856, 1010963856, -361743294, -361743294, + -361743294, -361743294, 209469125, 209469125, 209469125, + 209469125, -1053094788, -1053094788, -1053094788, -1053094788, + 1053160324, 1053160324, 1053160324, 1053160324, 639644520, + 639644520, 639644520, 639644520, -862444000, -862444000, + -862444000, -862444000, 1062144356, 1062144356, 1062144356, + 1062144356, -157532337, -157532337, -157532337, -157532337, + 260914709, 260914709, 260914709, 260914709, -1041559667, + -1041559667, -1041559667, -1041559667, 920985831, 920985831, + 920985831, 920985831, -551995675, -551995675, -551995675, + -551995675, 596522295, 596522295, 596522295, 596522295, + 892853362, 892853362, 892853362, 892853362, -892787826, + -892787826, -892787826, -892787826, 410925857, 410925857, + 410925857, 410925857, -992012162, -992012162, -992012162, + -992012162, 992077698, 992077698, 992077698, 992077698, + 759246145, 759246145, 759246145, 759246145, -759180609, + -759180609, -759180609, -759180609, -759222975, -759222975, + -759222975, -759222975, 759288511, 759288511, 759288511, + 759288511 + }; + + switch (tx_type) { + case DCT_DCT: + /* column transform */ + for (i = 0; i < 2; ++i) { + fdct8x16_1d_column(input + 8 * i, tmp + 8 * i, stride); + } + + /* row transform */ + for (i = 0; i < 2; ++i) { + fdct16x8_1d_row(tmp + (128 * i), output + (128 * i)); + } + break; + case ADST_DCT: + /* column transform */ + for (i = 0; i < 2; ++i) { + fadst16_cols_step1_msa(input + (i << 3), stride, const_arr, ptmpbuf); + fadst16_cols_step2_msa(ptmpbuf, const_arr, tmp + (i << 3)); + } + + /* row transform */ + for (i = 0; i < 2; ++i) { + postproc_fdct16x8_1d_row(tmp + (128 * i), output + (128 * i)); + } + break; + case DCT_ADST: + /* column transform */ + for (i = 0; i < 2; ++i) { + fdct8x16_1d_column(input + 8 * i, tmp + 8 * i, stride); + } + + fadst16_transpose_postproc_msa(tmp, trans); + + /* row transform */ + for (i = 0; i < 2; ++i) { + fadst16_rows_step1_msa(trans + (i << 7), const_arr, ptmpbuf); + fadst16_rows_step2_msa(ptmpbuf, const_arr, tmp + (i << 7)); + } + + fadst16_transpose_msa(tmp, output); + break; + case ADST_ADST: + /* column transform */ + for (i = 0; i < 2; ++i) { + fadst16_cols_step1_msa(input + (i << 3), stride, const_arr, ptmpbuf); + fadst16_cols_step2_msa(ptmpbuf, const_arr, tmp + (i << 3)); + } + + fadst16_transpose_postproc_msa(tmp, trans); + + /* row transform */ + for (i = 0; i < 2; ++i) { + fadst16_rows_step1_msa(trans + (i << 7), const_arr, ptmpbuf); + fadst16_rows_step2_msa(ptmpbuf, const_arr, tmp + (i << 7)); + } + + fadst16_transpose_msa(tmp, output); + break; + default: assert(0); break; + } +} diff --git a/third_party/aom/av1/encoder/mips/msa/fdct4x4_msa.c b/third_party/aom/av1/encoder/mips/msa/fdct4x4_msa.c new file mode 100644 index 0000000000..da1ac74f06 --- /dev/null +++ b/third_party/aom/av1/encoder/mips/msa/fdct4x4_msa.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/enums.h" +#include "av1/encoder/mips/msa/fdct_msa.h" + +void av1_fwht4x4_msa(const int16_t *input, int16_t *output, + int32_t src_stride) { + v8i16 in0, in1, in2, in3, in4; + + LD_SH4(input, src_stride, in0, in1, in2, in3); + + in0 += in1; + in3 -= in2; + in4 = (in0 - in3) >> 1; + SUB2(in4, in1, in4, in2, in1, in2); + in0 -= in2; + in3 += in1; + + TRANSPOSE4x4_SH_SH(in0, in2, in3, in1, in0, in2, in3, in1); + + in0 += in2; + in1 -= in3; + in4 = (in0 - in1) >> 1; + SUB2(in4, in2, in4, in3, in2, in3); + in0 -= in3; + in1 += in2; + + SLLI_4V(in0, in1, in2, in3, 2); + + TRANSPOSE4x4_SH_SH(in0, in3, in1, in2, in0, in3, in1, in2); + + ST4x2_UB(in0, output, 4); + ST4x2_UB(in3, output + 4, 4); + ST4x2_UB(in1, output + 8, 4); + ST4x2_UB(in2, output + 12, 4); +} + +void av1_fht4x4_msa(const int16_t *input, int16_t *output, int32_t stride, + int32_t tx_type) { + v8i16 in0, in1, in2, in3; + + LD_SH4(input, stride, in0, in1, in2, in3); + + /* fdct4 pre-process */ + { + v8i16 temp, mask; + v16i8 zero = { 0 }; + v16i8 one = __msa_ldi_b(1); + + mask = (v8i16)__msa_sldi_b(zero, one, 15); + SLLI_4V(in0, in1, in2, in3, 4); + temp = __msa_ceqi_h(in0, 0); + temp = (v8i16)__msa_xori_b((v16u8)temp, 255); + temp = mask & temp; + in0 += temp; + } + + switch (tx_type) { + case DCT_DCT: + AOM_FDCT4(in0, in1, in2, in3, in0, in1, in2, in3); + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + AOM_FDCT4(in0, in1, in2, in3, in0, in1, in2, in3); + break; + case ADST_DCT: + AOM_FADST4(in0, in1, in2, in3, in0, in1, in2, in3); + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + AOM_FDCT4(in0, in1, in2, in3, in0, in1, in2, in3); + break; + case DCT_ADST: + AOM_FDCT4(in0, in1, in2, in3, in0, in1, in2, in3); + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + AOM_FADST4(in0, in1, in2, in3, in0, in1, in2, in3); + break; + case ADST_ADST: + AOM_FADST4(in0, in1, in2, in3, in0, in1, in2, in3); + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + AOM_FADST4(in0, in1, in2, in3, in0, in1, in2, in3); + break; + default: assert(0); break; + } + + TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, in0, in1, in2, in3); + ADD4(in0, 1, in1, 1, in2, 1, in3, 1, in0, in1, in2, in3); + SRA_4V(in0, in1, in2, in3, 2); + PCKEV_D2_SH(in1, in0, in3, in2, in0, in2); + ST_SH2(in0, in2, output, 8); +} diff --git a/third_party/aom/av1/encoder/mips/msa/fdct8x8_msa.c b/third_party/aom/av1/encoder/mips/msa/fdct8x8_msa.c new file mode 100644 index 0000000000..4cbf60a11d --- /dev/null +++ b/third_party/aom/av1/encoder/mips/msa/fdct8x8_msa.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/common/enums.h" +#include "av1/encoder/mips/msa/fdct_msa.h" + +void av1_fht8x8_msa(const int16_t *input, int16_t *output, int32_t stride, + int32_t tx_type) { + v8i16 in0, in1, in2, in3, in4, in5, in6, in7; + + LD_SH8(input, stride, in0, in1, in2, in3, in4, in5, in6, in7); + SLLI_4V(in0, in1, in2, in3, 2); + SLLI_4V(in4, in5, in6, in7, 2); + + switch (tx_type) { + case DCT_DCT: + AOM_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, + in3, in4, in5, in6, in7); + AOM_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + break; + case ADST_DCT: + AOM_ADST8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, + in3, in4, in5, in6, in7); + AOM_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + break; + case DCT_ADST: + AOM_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, + in3, in4, in5, in6, in7); + AOM_ADST8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + break; + case ADST_ADST: + AOM_ADST8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, + in3, in4, in5, in6, in7); + AOM_ADST8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + break; + default: assert(0); break; + } + + TRANSPOSE8x8_SH_SH(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + SRLI_AVE_S_4V_H(in0, in1, in2, in3, in4, in5, in6, in7); + ST_SH8(in0, in1, in2, in3, in4, in5, in6, in7, output, 8); +} diff --git a/third_party/aom/av1/encoder/mips/msa/fdct_msa.h b/third_party/aom/av1/encoder/mips/msa/fdct_msa.h new file mode 100644 index 0000000000..52bcf790c9 --- /dev/null +++ b/third_party/aom/av1/encoder/mips/msa/fdct_msa.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_MIPS_MSA_AV1_FDCT_MSA_H_ +#define AV1_ENCODER_MIPS_MSA_AV1_FDCT_MSA_H_ + +#include "aom_dsp/mips/fwd_txfm_msa.h" +#include "aom_dsp/mips/txfm_macros_msa.h" +#include "aom_ports/mem.h" + +#define AOM_ADST8(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, \ + out3, out4, out5, out6, out7) \ + { \ + v8i16 cnst0_m, cnst1_m, cnst2_m, cnst3_m, cnst4_m; \ + v8i16 vec0_m, vec1_m, vec2_m, vec3_m, s0_m, s1_m; \ + v8i16 coeff0_m = { cospi_2_64, cospi_6_64, cospi_10_64, cospi_14_64, \ + cospi_18_64, cospi_22_64, cospi_26_64, cospi_30_64 }; \ + v8i16 coeff1_m = { cospi_8_64, -cospi_8_64, cospi_16_64, -cospi_16_64, \ + cospi_24_64, -cospi_24_64, 0, 0 }; \ + \ + SPLATI_H2_SH(coeff0_m, 0, 7, cnst0_m, cnst1_m); \ + cnst2_m = -cnst0_m; \ + ILVEV_H2_SH(cnst0_m, cnst1_m, cnst1_m, cnst2_m, cnst0_m, cnst1_m); \ + SPLATI_H2_SH(coeff0_m, 4, 3, cnst2_m, cnst3_m); \ + cnst4_m = -cnst2_m; \ + ILVEV_H2_SH(cnst2_m, cnst3_m, cnst3_m, cnst4_m, cnst2_m, cnst3_m); \ + \ + ILVRL_H2_SH(in0, in7, vec1_m, vec0_m); \ + ILVRL_H2_SH(in4, in3, vec3_m, vec2_m); \ + DOT_ADD_SUB_SRARI_PCK(vec0_m, vec1_m, vec2_m, vec3_m, cnst0_m, cnst1_m, \ + cnst2_m, cnst3_m, in7, in0, in4, in3); \ + \ + SPLATI_H2_SH(coeff0_m, 2, 5, cnst0_m, cnst1_m); \ + cnst2_m = -cnst0_m; \ + ILVEV_H2_SH(cnst0_m, cnst1_m, cnst1_m, cnst2_m, cnst0_m, cnst1_m); \ + SPLATI_H2_SH(coeff0_m, 6, 1, cnst2_m, cnst3_m); \ + cnst4_m = -cnst2_m; \ + ILVEV_H2_SH(cnst2_m, cnst3_m, cnst3_m, cnst4_m, cnst2_m, cnst3_m); \ + \ + ILVRL_H2_SH(in2, in5, vec1_m, vec0_m); \ + ILVRL_H2_SH(in6, in1, vec3_m, vec2_m); \ + \ + DOT_ADD_SUB_SRARI_PCK(vec0_m, vec1_m, vec2_m, vec3_m, cnst0_m, cnst1_m, \ + cnst2_m, cnst3_m, in5, in2, in6, in1); \ + BUTTERFLY_4(in7, in0, in2, in5, s1_m, s0_m, in2, in5); \ + out7 = -s0_m; \ + out0 = s1_m; \ + \ + SPLATI_H4_SH(coeff1_m, 0, 4, 1, 5, cnst0_m, cnst1_m, cnst2_m, cnst3_m); \ + \ + ILVEV_H2_SH(cnst3_m, cnst0_m, cnst1_m, cnst2_m, cnst3_m, cnst2_m); \ + cnst0_m = __msa_ilvev_h(cnst1_m, cnst0_m); \ + cnst1_m = cnst0_m; \ + \ + ILVRL_H2_SH(in4, in3, vec1_m, vec0_m); \ + ILVRL_H2_SH(in6, in1, vec3_m, vec2_m); \ + DOT_ADD_SUB_SRARI_PCK(vec0_m, vec1_m, vec2_m, vec3_m, cnst0_m, cnst2_m, \ + cnst3_m, cnst1_m, out1, out6, s0_m, s1_m); \ + \ + SPLATI_H2_SH(coeff1_m, 2, 3, cnst0_m, cnst1_m); \ + cnst1_m = __msa_ilvev_h(cnst1_m, cnst0_m); \ + \ + ILVRL_H2_SH(in2, in5, vec1_m, vec0_m); \ + ILVRL_H2_SH(s0_m, s1_m, vec3_m, vec2_m); \ + out3 = DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m); \ + out4 = DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst1_m); \ + out2 = DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst0_m); \ + out5 = DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst1_m); \ + \ + out1 = -out1; \ + out3 = -out3; \ + out5 = -out5; \ + } + +#define AOM_FADST4(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + v4i32 s0_m, s1_m, s2_m, s3_m, constant_m; \ + v4i32 in0_r_m, in1_r_m, in2_r_m, in3_r_m; \ + \ + UNPCK_R_SH_SW(in0, in0_r_m); \ + UNPCK_R_SH_SW(in1, in1_r_m); \ + UNPCK_R_SH_SW(in2, in2_r_m); \ + UNPCK_R_SH_SW(in3, in3_r_m); \ + \ + constant_m = __msa_fill_w(sinpi_4_9); \ + MUL2(in0_r_m, constant_m, in3_r_m, constant_m, s1_m, s0_m); \ + \ + constant_m = __msa_fill_w(sinpi_1_9); \ + s0_m += in0_r_m * constant_m; \ + s1_m -= in1_r_m * constant_m; \ + \ + constant_m = __msa_fill_w(sinpi_2_9); \ + s0_m += in1_r_m * constant_m; \ + s1_m += in3_r_m * constant_m; \ + \ + s2_m = in0_r_m + in1_r_m - in3_r_m; \ + \ + constant_m = __msa_fill_w(sinpi_3_9); \ + MUL2(in2_r_m, constant_m, s2_m, constant_m, s3_m, in1_r_m); \ + \ + in0_r_m = s0_m + s3_m; \ + s2_m = s1_m - s3_m; \ + s3_m = s1_m - s0_m + s3_m; \ + \ + SRARI_W4_SW(in0_r_m, in1_r_m, s2_m, s3_m, DCT_CONST_BITS); \ + PCKEV_H4_SH(in0_r_m, in0_r_m, in1_r_m, in1_r_m, s2_m, s2_m, s3_m, s3_m, \ + out0, out1, out2, out3); \ + } +#endif // AV1_ENCODER_MIPS_MSA_AV1_FDCT_MSA_H_ diff --git a/third_party/aom/av1/encoder/mips/msa/temporal_filter_msa.c b/third_party/aom/av1/encoder/mips/msa/temporal_filter_msa.c new file mode 100644 index 0000000000..4ec679642a --- /dev/null +++ b/third_party/aom/av1/encoder/mips/msa/temporal_filter_msa.c @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./av1_rtcd.h" +#include "aom_dsp/mips/macros_msa.h" + +static void temporal_filter_apply_8size_msa(uint8_t *frm1_ptr, uint32_t stride, + uint8_t *frm2_ptr, int32_t filt_sth, + int32_t filt_wgt, uint32_t *acc, + uint16_t *cnt) { + uint32_t row; + uint64_t f0, f1, f2, f3; + v16i8 frm2, frm1 = { 0 }; + v16i8 frm4, frm3 = { 0 }; + v16u8 frm_r, frm_l; + v8i16 frm2_r, frm2_l; + v8i16 diff0, diff1, mod0_h, mod1_h; + v4i32 cnst3, cnst16, filt_wt, strength; + v4i32 mod0_w, mod1_w, mod2_w, mod3_w; + v4i32 diff0_r, diff0_l, diff1_r, diff1_l; + v4i32 frm2_rr, frm2_rl, frm2_lr, frm2_ll; + v4i32 acc0, acc1, acc2, acc3; + v8i16 cnt0, cnt1; + + filt_wt = __msa_fill_w(filt_wgt); + strength = __msa_fill_w(filt_sth); + cnst3 = __msa_ldi_w(3); + cnst16 = __msa_ldi_w(16); + + for (row = 2; row--;) { + LD4(frm1_ptr, stride, f0, f1, f2, f3); + frm1_ptr += (4 * stride); + + LD_SB2(frm2_ptr, 16, frm2, frm4); + frm2_ptr += 32; + + LD_SW2(acc, 4, acc0, acc1); + LD_SW2(acc + 8, 4, acc2, acc3); + LD_SH2(cnt, 8, cnt0, cnt1); + + INSERT_D2_SB(f0, f1, frm1); + INSERT_D2_SB(f2, f3, frm3); + ILVRL_B2_UB(frm1, frm2, frm_r, frm_l); + HSUB_UB2_SH(frm_r, frm_l, diff0, diff1); + UNPCK_SH_SW(diff0, diff0_r, diff0_l); + UNPCK_SH_SW(diff1, diff1_r, diff1_l); + MUL4(diff0_r, diff0_r, diff0_l, diff0_l, diff1_r, diff1_r, diff1_l, diff1_l, + mod0_w, mod1_w, mod2_w, mod3_w); + MUL4(mod0_w, cnst3, mod1_w, cnst3, mod2_w, cnst3, mod3_w, cnst3, mod0_w, + mod1_w, mod2_w, mod3_w); + SRAR_W4_SW(mod0_w, mod1_w, mod2_w, mod3_w, strength); + + diff0_r = (mod0_w < cnst16); + diff0_l = (mod1_w < cnst16); + diff1_r = (mod2_w < cnst16); + diff1_l = (mod3_w < cnst16); + + SUB4(cnst16, mod0_w, cnst16, mod1_w, cnst16, mod2_w, cnst16, mod3_w, mod0_w, + mod1_w, mod2_w, mod3_w); + + mod0_w = diff0_r & mod0_w; + mod1_w = diff0_l & mod1_w; + mod2_w = diff1_r & mod2_w; + mod3_w = diff1_l & mod3_w; + + MUL4(mod0_w, filt_wt, mod1_w, filt_wt, mod2_w, filt_wt, mod3_w, filt_wt, + mod0_w, mod1_w, mod2_w, mod3_w); + PCKEV_H2_SH(mod1_w, mod0_w, mod3_w, mod2_w, mod0_h, mod1_h); + ADD2(mod0_h, cnt0, mod1_h, cnt1, mod0_h, mod1_h); + ST_SH2(mod0_h, mod1_h, cnt, 8); + cnt += 16; + + UNPCK_UB_SH(frm2, frm2_r, frm2_l); + UNPCK_SH_SW(frm2_r, frm2_rr, frm2_rl); + UNPCK_SH_SW(frm2_l, frm2_lr, frm2_ll); + MUL4(mod0_w, frm2_rr, mod1_w, frm2_rl, mod2_w, frm2_lr, mod3_w, frm2_ll, + mod0_w, mod1_w, mod2_w, mod3_w); + ADD4(mod0_w, acc0, mod1_w, acc1, mod2_w, acc2, mod3_w, acc3, mod0_w, mod1_w, + mod2_w, mod3_w); + + ST_SW2(mod0_w, mod1_w, acc, 4); + acc += 8; + ST_SW2(mod2_w, mod3_w, acc, 4); + acc += 8; + + LD_SW2(acc, 4, acc0, acc1); + LD_SW2(acc + 8, 4, acc2, acc3); + LD_SH2(cnt, 8, cnt0, cnt1); + + ILVRL_B2_UB(frm3, frm4, frm_r, frm_l); + HSUB_UB2_SH(frm_r, frm_l, diff0, diff1); + UNPCK_SH_SW(diff0, diff0_r, diff0_l); + UNPCK_SH_SW(diff1, diff1_r, diff1_l); + MUL4(diff0_r, diff0_r, diff0_l, diff0_l, diff1_r, diff1_r, diff1_l, diff1_l, + mod0_w, mod1_w, mod2_w, mod3_w); + MUL4(mod0_w, cnst3, mod1_w, cnst3, mod2_w, cnst3, mod3_w, cnst3, mod0_w, + mod1_w, mod2_w, mod3_w); + SRAR_W4_SW(mod0_w, mod1_w, mod2_w, mod3_w, strength); + + diff0_r = (mod0_w < cnst16); + diff0_l = (mod1_w < cnst16); + diff1_r = (mod2_w < cnst16); + diff1_l = (mod3_w < cnst16); + + SUB4(cnst16, mod0_w, cnst16, mod1_w, cnst16, mod2_w, cnst16, mod3_w, mod0_w, + mod1_w, mod2_w, mod3_w); + + mod0_w = diff0_r & mod0_w; + mod1_w = diff0_l & mod1_w; + mod2_w = diff1_r & mod2_w; + mod3_w = diff1_l & mod3_w; + + MUL4(mod0_w, filt_wt, mod1_w, filt_wt, mod2_w, filt_wt, mod3_w, filt_wt, + mod0_w, mod1_w, mod2_w, mod3_w); + PCKEV_H2_SH(mod1_w, mod0_w, mod3_w, mod2_w, mod0_h, mod1_h); + ADD2(mod0_h, cnt0, mod1_h, cnt1, mod0_h, mod1_h); + ST_SH2(mod0_h, mod1_h, cnt, 8); + cnt += 16; + UNPCK_UB_SH(frm4, frm2_r, frm2_l); + UNPCK_SH_SW(frm2_r, frm2_rr, frm2_rl); + UNPCK_SH_SW(frm2_l, frm2_lr, frm2_ll); + MUL4(mod0_w, frm2_rr, mod1_w, frm2_rl, mod2_w, frm2_lr, mod3_w, frm2_ll, + mod0_w, mod1_w, mod2_w, mod3_w); + ADD4(mod0_w, acc0, mod1_w, acc1, mod2_w, acc2, mod3_w, acc3, mod0_w, mod1_w, + mod2_w, mod3_w); + + ST_SW2(mod0_w, mod1_w, acc, 4); + acc += 8; + ST_SW2(mod2_w, mod3_w, acc, 4); + acc += 8; + } +} + +static void temporal_filter_apply_16size_msa(uint8_t *frm1_ptr, uint32_t stride, + uint8_t *frm2_ptr, + int32_t filt_sth, int32_t filt_wgt, + uint32_t *acc, uint16_t *cnt) { + uint32_t row; + v16i8 frm1, frm2, frm3, frm4; + v16u8 frm_r, frm_l; + v16i8 zero = { 0 }; + v8u16 frm2_r, frm2_l; + v8i16 diff0, diff1, mod0_h, mod1_h; + v4i32 cnst3, cnst16, filt_wt, strength; + v4i32 mod0_w, mod1_w, mod2_w, mod3_w; + v4i32 diff0_r, diff0_l, diff1_r, diff1_l; + v4i32 frm2_rr, frm2_rl, frm2_lr, frm2_ll; + v4i32 acc0, acc1, acc2, acc3; + v8i16 cnt0, cnt1; + + filt_wt = __msa_fill_w(filt_wgt); + strength = __msa_fill_w(filt_sth); + cnst3 = __msa_ldi_w(3); + cnst16 = __msa_ldi_w(16); + + for (row = 8; row--;) { + LD_SB2(frm1_ptr, stride, frm1, frm3); + frm1_ptr += stride; + + LD_SB2(frm2_ptr, 16, frm2, frm4); + frm2_ptr += 16; + + LD_SW2(acc, 4, acc0, acc1); + LD_SW2(acc, 4, acc2, acc3); + LD_SH2(cnt, 8, cnt0, cnt1); + + ILVRL_B2_UB(frm1, frm2, frm_r, frm_l); + HSUB_UB2_SH(frm_r, frm_l, diff0, diff1); + UNPCK_SH_SW(diff0, diff0_r, diff0_l); + UNPCK_SH_SW(diff1, diff1_r, diff1_l); + MUL4(diff0_r, diff0_r, diff0_l, diff0_l, diff1_r, diff1_r, diff1_l, diff1_l, + mod0_w, mod1_w, mod2_w, mod3_w); + MUL4(mod0_w, cnst3, mod1_w, cnst3, mod2_w, cnst3, mod3_w, cnst3, mod0_w, + mod1_w, mod2_w, mod3_w); + SRAR_W4_SW(mod0_w, mod1_w, mod2_w, mod3_w, strength); + + diff0_r = (mod0_w < cnst16); + diff0_l = (mod1_w < cnst16); + diff1_r = (mod2_w < cnst16); + diff1_l = (mod3_w < cnst16); + + SUB4(cnst16, mod0_w, cnst16, mod1_w, cnst16, mod2_w, cnst16, mod3_w, mod0_w, + mod1_w, mod2_w, mod3_w); + + mod0_w = diff0_r & mod0_w; + mod1_w = diff0_l & mod1_w; + mod2_w = diff1_r & mod2_w; + mod3_w = diff1_l & mod3_w; + + MUL4(mod0_w, filt_wt, mod1_w, filt_wt, mod2_w, filt_wt, mod3_w, filt_wt, + mod0_w, mod1_w, mod2_w, mod3_w); + PCKEV_H2_SH(mod1_w, mod0_w, mod3_w, mod2_w, mod0_h, mod1_h); + ADD2(mod0_h, cnt0, mod1_h, cnt1, mod0_h, mod1_h); + ST_SH2(mod0_h, mod1_h, cnt, 8); + cnt += 16; + + ILVRL_B2_UH(zero, frm2, frm2_r, frm2_l); + UNPCK_SH_SW(frm2_r, frm2_rr, frm2_rl); + UNPCK_SH_SW(frm2_l, frm2_lr, frm2_ll); + MUL4(mod0_w, frm2_rr, mod1_w, frm2_rl, mod2_w, frm2_lr, mod3_w, frm2_ll, + mod0_w, mod1_w, mod2_w, mod3_w); + ADD4(mod0_w, acc0, mod1_w, acc1, mod2_w, acc2, mod3_w, acc3, mod0_w, mod1_w, + mod2_w, mod3_w); + + ST_SW2(mod0_w, mod1_w, acc, 4); + acc += 8; + ST_SW2(mod2_w, mod3_w, acc, 4); + acc += 8; + + LD_SW2(acc, 4, acc0, acc1); + LD_SW2(acc + 8, 4, acc2, acc3); + LD_SH2(cnt, 8, cnt0, cnt1); + + ILVRL_B2_UB(frm3, frm4, frm_r, frm_l); + HSUB_UB2_SH(frm_r, frm_l, diff0, diff1); + UNPCK_SH_SW(diff0, diff0_r, diff0_l); + UNPCK_SH_SW(diff1, diff1_r, diff1_l); + MUL4(diff0_r, diff0_r, diff0_l, diff0_l, diff1_r, diff1_r, diff1_l, diff1_l, + mod0_w, mod1_w, mod2_w, mod3_w); + MUL4(mod0_w, cnst3, mod1_w, cnst3, mod2_w, cnst3, mod3_w, cnst3, mod0_w, + mod1_w, mod2_w, mod3_w); + SRAR_W4_SW(mod0_w, mod1_w, mod2_w, mod3_w, strength); + + diff0_r = (mod0_w < cnst16); + diff0_l = (mod1_w < cnst16); + diff1_r = (mod2_w < cnst16); + diff1_l = (mod3_w < cnst16); + + SUB4(cnst16, mod0_w, cnst16, mod1_w, cnst16, mod2_w, cnst16, mod3_w, mod0_w, + mod1_w, mod2_w, mod3_w); + + mod0_w = diff0_r & mod0_w; + mod1_w = diff0_l & mod1_w; + mod2_w = diff1_r & mod2_w; + mod3_w = diff1_l & mod3_w; + + MUL4(mod0_w, filt_wt, mod1_w, filt_wt, mod2_w, filt_wt, mod3_w, filt_wt, + mod0_w, mod1_w, mod2_w, mod3_w); + PCKEV_H2_SH(mod1_w, mod0_w, mod3_w, mod2_w, mod0_h, mod1_h); + ADD2(mod0_h, cnt0, mod1_h, cnt1, mod0_h, mod1_h); + ST_SH2(mod0_h, mod1_h, cnt, 8); + cnt += 16; + + ILVRL_B2_UH(zero, frm4, frm2_r, frm2_l); + UNPCK_SH_SW(frm2_r, frm2_rr, frm2_rl); + UNPCK_SH_SW(frm2_l, frm2_lr, frm2_ll); + MUL4(mod0_w, frm2_rr, mod1_w, frm2_rl, mod2_w, frm2_lr, mod3_w, frm2_ll, + mod0_w, mod1_w, mod2_w, mod3_w); + ADD4(mod0_w, acc0, mod1_w, acc1, mod2_w, acc2, mod3_w, acc3, mod0_w, mod1_w, + mod2_w, mod3_w); + ST_SW2(mod0_w, mod1_w, acc, 4); + acc += 8; + ST_SW2(mod2_w, mod3_w, acc, 4); + acc += 8; + + frm1_ptr += stride; + frm2_ptr += 16; + } +} + +void av1_temporal_filter_apply_msa(uint8_t *frame1_ptr, uint32_t stride, + uint8_t *frame2_ptr, uint32_t blk_w, + uint32_t blk_h, int32_t strength, + int32_t filt_wgt, uint32_t *accu, + uint16_t *cnt) { + if (8 == (blk_w * blk_h)) { + temporal_filter_apply_8size_msa(frame1_ptr, stride, frame2_ptr, strength, + filt_wgt, accu, cnt); + } else if (16 == (blk_w * blk_h)) { + temporal_filter_apply_16size_msa(frame1_ptr, stride, frame2_ptr, strength, + filt_wgt, accu, cnt); + } else { + av1_temporal_filter_apply_c(frame1_ptr, stride, frame2_ptr, blk_w, blk_h, + strength, filt_wgt, accu, cnt); + } +} diff --git a/third_party/aom/av1/encoder/palette.c b/third_party/aom/av1/encoder/palette.c new file mode 100644 index 0000000000..355141de55 --- /dev/null +++ b/third_party/aom/av1/encoder/palette.c @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "av1/encoder/cost.h" +#include "av1/encoder/palette.h" + +static float calc_dist(const float *p1, const float *p2, int dim) { + float dist = 0; + int i; + for (i = 0; i < dim; ++i) { + const float diff = p1[i] - p2[i]; + dist += diff * diff; + } + return dist; +} + +void av1_calc_indices(const float *data, const float *centroids, + uint8_t *indices, int n, int k, int dim) { + int i, j; + for (i = 0; i < n; ++i) { + float min_dist = calc_dist(data + i * dim, centroids, dim); + indices[i] = 0; + for (j = 1; j < k; ++j) { + const float this_dist = + calc_dist(data + i * dim, centroids + j * dim, dim); + if (this_dist < min_dist) { + min_dist = this_dist; + indices[i] = j; + } + } + } +} + +// Generate a random number in the range [0, 32768). +static unsigned int lcg_rand16(unsigned int *state) { + *state = (unsigned int)(*state * 1103515245ULL + 12345); + return *state / 65536 % 32768; +} + +static void calc_centroids(const float *data, float *centroids, + const uint8_t *indices, int n, int k, int dim) { + int i, j, index; + int count[PALETTE_MAX_SIZE]; + unsigned int rand_state = (unsigned int)data[0]; + + assert(n <= 32768); + + memset(count, 0, sizeof(count[0]) * k); + memset(centroids, 0, sizeof(centroids[0]) * k * dim); + + for (i = 0; i < n; ++i) { + index = indices[i]; + assert(index < k); + ++count[index]; + for (j = 0; j < dim; ++j) { + centroids[index * dim + j] += data[i * dim + j]; + } + } + + for (i = 0; i < k; ++i) { + if (count[i] == 0) { + memcpy(centroids + i * dim, data + (lcg_rand16(&rand_state) % n) * dim, + sizeof(centroids[0]) * dim); + } else { + const float norm = 1.0f / count[i]; + for (j = 0; j < dim; ++j) centroids[i * dim + j] *= norm; + } + } + + // Round to nearest integers. + for (i = 0; i < k * dim; ++i) { + centroids[i] = roundf(centroids[i]); + } +} + +static float calc_total_dist(const float *data, const float *centroids, + const uint8_t *indices, int n, int k, int dim) { + float dist = 0; + int i; + (void)k; + + for (i = 0; i < n; ++i) + dist += calc_dist(data + i * dim, centroids + indices[i] * dim, dim); + + return dist; +} + +void av1_k_means(const float *data, float *centroids, uint8_t *indices, int n, + int k, int dim, int max_itr) { + int i; + float this_dist; + float pre_centroids[2 * PALETTE_MAX_SIZE]; + uint8_t pre_indices[MAX_SB_SQUARE]; + + av1_calc_indices(data, centroids, indices, n, k, dim); + this_dist = calc_total_dist(data, centroids, indices, n, k, dim); + + for (i = 0; i < max_itr; ++i) { + const float pre_dist = this_dist; + memcpy(pre_centroids, centroids, sizeof(pre_centroids[0]) * k * dim); + memcpy(pre_indices, indices, sizeof(pre_indices[0]) * n); + + calc_centroids(data, centroids, indices, n, k, dim); + av1_calc_indices(data, centroids, indices, n, k, dim); + this_dist = calc_total_dist(data, centroids, indices, n, k, dim); + + if (this_dist > pre_dist) { + memcpy(centroids, pre_centroids, sizeof(pre_centroids[0]) * k * dim); + memcpy(indices, pre_indices, sizeof(pre_indices[0]) * n); + break; + } + if (!memcmp(centroids, pre_centroids, sizeof(pre_centroids[0]) * k * dim)) + break; + } +} + +static int float_comparer(const void *a, const void *b) { + const float fa = *(const float *)a; + const float fb = *(const float *)b; + return (fa > fb) - (fa < fb); +} + +int av1_remove_duplicates(float *centroids, int num_centroids) { + int num_unique; // number of unique centroids + int i; + qsort(centroids, num_centroids, sizeof(*centroids), float_comparer); + // Remove duplicates. + num_unique = 1; + for (i = 1; i < num_centroids; ++i) { + if (centroids[i] != centroids[i - 1]) { // found a new unique centroid + centroids[num_unique++] = centroids[i]; + } + } + return num_unique; +} + +int av1_count_colors(const uint8_t *src, int stride, int rows, int cols) { + int n = 0, r, c, i, val_count[256]; + uint8_t val; + memset(val_count, 0, sizeof(val_count)); + + for (r = 0; r < rows; ++r) { + for (c = 0; c < cols; ++c) { + val = src[r * stride + c]; + ++val_count[val]; + } + } + + for (i = 0; i < 256; ++i) { + if (val_count[i]) { + ++n; + } + } + + return n; +} + +#if CONFIG_PALETTE_DELTA_ENCODING +int av1_get_palette_delta_bits_y(const PALETTE_MODE_INFO *const pmi, + int bit_depth, int *min_bits) { + const int n = pmi->palette_size[0]; + int max_d = 0, i; + *min_bits = bit_depth - 3; + for (i = 1; i < n; ++i) { + const int delta = pmi->palette_colors[i] - pmi->palette_colors[i - 1]; + assert(delta > 0); + if (delta > max_d) max_d = delta; + } + return AOMMAX(av1_ceil_log2(max_d), *min_bits); +} + +int av1_get_palette_delta_bits_u(const PALETTE_MODE_INFO *const pmi, + int bit_depth, int *min_bits) { + const int n = pmi->palette_size[1]; + int max_d = 0, i; + *min_bits = bit_depth - 3; + for (i = 1; i < n; ++i) { + const int delta = pmi->palette_colors[PALETTE_MAX_SIZE + i] - + pmi->palette_colors[PALETTE_MAX_SIZE + i - 1]; + assert(delta >= 0); + if (delta > max_d) max_d = delta; + } + return AOMMAX(av1_ceil_log2(max_d + 1), *min_bits); +} + +int av1_get_palette_delta_bits_v(const PALETTE_MODE_INFO *const pmi, + int bit_depth, int *zero_count, + int *min_bits) { + const int n = pmi->palette_size[1]; + const int max_val = 1 << bit_depth; + int max_d = 0, i; + *min_bits = bit_depth - 4; + *zero_count = 0; + for (i = 1; i < n; ++i) { + const int delta = pmi->palette_colors[2 * PALETTE_MAX_SIZE + i] - + pmi->palette_colors[2 * PALETTE_MAX_SIZE + i - 1]; + const int v = abs(delta); + const int d = AOMMIN(v, max_val - v); + if (d > max_d) max_d = d; + if (d == 0) ++(*zero_count); + } + return AOMMAX(av1_ceil_log2(max_d + 1), *min_bits); +} +#endif // CONFIG_PALETTE_DELTA_ENCODING + +int av1_palette_color_cost_y(const PALETTE_MODE_INFO *const pmi, + int bit_depth) { + const int n = pmi->palette_size[0]; +#if CONFIG_PALETTE_DELTA_ENCODING + int min_bits = 0; + const int bits = av1_get_palette_delta_bits_y(pmi, bit_depth, &min_bits); + return av1_cost_bit(128, 0) * (2 + bit_depth + bits * (n - 1)); +#else + return bit_depth * n * av1_cost_bit(128, 0); +#endif // CONFIG_PALETTE_DELTA_ENCODING +} + +int av1_palette_color_cost_uv(const PALETTE_MODE_INFO *const pmi, + int bit_depth) { + const int n = pmi->palette_size[1]; +#if CONFIG_PALETTE_DELTA_ENCODING + int cost = 0; + // U channel palette color cost. + int min_bits_u = 0; + const int bits_u = av1_get_palette_delta_bits_u(pmi, bit_depth, &min_bits_u); + cost += av1_cost_bit(128, 0) * (2 + bit_depth + bits_u * (n - 1)); + // V channel palette color cost. + int zero_count = 0, min_bits_v = 0; + const int bits_v = + av1_get_palette_delta_bits_v(pmi, bit_depth, &zero_count, &min_bits_v); + const int bits_using_delta = + 2 + bit_depth + (bits_v + 1) * (n - 1) - zero_count; + const int bits_using_raw = bit_depth * n; + cost += av1_cost_bit(128, 0) * (1 + AOMMIN(bits_using_delta, bits_using_raw)); + return cost; +#else + return 2 * bit_depth * n * av1_cost_bit(128, 0); +#endif // CONFIG_PALETTE_DELTA_ENCODING +} + +#if CONFIG_HIGHBITDEPTH +int av1_count_colors_highbd(const uint8_t *src8, int stride, int rows, int cols, + int bit_depth) { + int n = 0, r, c, i; + uint16_t val; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + int val_count[1 << 12]; + + assert(bit_depth <= 12); + memset(val_count, 0, (1 << 12) * sizeof(val_count[0])); + for (r = 0; r < rows; ++r) { + for (c = 0; c < cols; ++c) { + val = src[r * stride + c]; + ++val_count[val]; + } + } + + for (i = 0; i < (1 << bit_depth); ++i) { + if (val_count[i]) { + ++n; + } + } + + return n; +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/av1/encoder/palette.h b/third_party/aom/av1/encoder/palette.h new file mode 100644 index 0000000000..5403ac5e60 --- /dev/null +++ b/third_party/aom/av1/encoder/palette.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_PALETTE_H_ +#define AV1_ENCODER_PALETTE_H_ + +#include "av1/common/blockd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Given 'n' 'data' points and 'k' 'centroids' each of dimension 'dim', +// calculate the centroid 'indices' for the data points. +void av1_calc_indices(const float *data, const float *centroids, + uint8_t *indices, int n, int k, int dim); + +// Given 'n' 'data' points and an initial guess of 'k' 'centroids' each of +// dimension 'dim', runs up to 'max_itr' iterations of k-means algorithm to get +// updated 'centroids' and the centroid 'indices' for elements in 'data'. +// Note: the output centroids are rounded off to nearest integers. +void av1_k_means(const float *data, float *centroids, uint8_t *indices, int n, + int k, int dim, int max_itr); + +// Given a list of centroids, returns the unique number of centroids 'k', and +// puts these unique centroids in first 'k' indices of 'centroids' array. +// Ideally, the centroids should be rounded to integers before calling this +// method. +int av1_remove_duplicates(float *centroids, int num_centroids); + +// Returns the number of colors in 'src'. +int av1_count_colors(const uint8_t *src, int stride, int rows, int cols); +#if CONFIG_HIGHBITDEPTH +// Same as av1_count_colors(), but for high-bitdepth mode. +int av1_count_colors_highbd(const uint8_t *src8, int stride, int rows, int cols, + int bit_depth); +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_PALETTE_DELTA_ENCODING +// Return the number of bits used to transmit each luma palette color delta. +int av1_get_palette_delta_bits_y(const PALETTE_MODE_INFO *const pmi, + int bit_depth, int *min_bits); + +// Return the number of bits used to transmit each U palette color delta. +int av1_get_palette_delta_bits_u(const PALETTE_MODE_INFO *const pmi, + int bit_depth, int *min_bits); + +// Return the number of bits used to transmit each v palette color delta; +// assign zero_count with the number of deltas being 0. +int av1_get_palette_delta_bits_v(const PALETTE_MODE_INFO *const pmi, + int bit_depth, int *zero_count, int *min_bits); +#endif // CONFIG_PALETTE_DELTA_ENCODING + +// Return the rate cost for transmitting luma palette color values. +int av1_palette_color_cost_y(const PALETTE_MODE_INFO *const pmi, int bit_depth); + +// Return the rate cost for transmitting chroma palette color values. +int av1_palette_color_cost_uv(const PALETTE_MODE_INFO *const pmi, + int bit_depth); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* AV1_ENCODER_PALETTE_H_ */ diff --git a/third_party/aom/av1/encoder/pickcdef.c b/third_party/aom/av1/encoder/pickcdef.c new file mode 100644 index 0000000000..da64fb48d6 --- /dev/null +++ b/third_party/aom/av1/encoder/pickcdef.c @@ -0,0 +1,490 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_scale_rtcd.h" +#include "aom/aom_integer.h" +#include "av1/common/cdef.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/reconinter.h" +#include "av1/encoder/encoder.h" + +#define TOTAL_STRENGTHS (DERING_STRENGTHS * CLPF_STRENGTHS) + +/* Search for the best strength to add as an option, knowing we + already selected nb_strengths options. */ +static uint64_t search_one(int *lev, int nb_strengths, + uint64_t mse[][TOTAL_STRENGTHS], int sb_count) { + uint64_t tot_mse[TOTAL_STRENGTHS]; + int i, j; + uint64_t best_tot_mse = (uint64_t)1 << 63; + int best_id = 0; + memset(tot_mse, 0, sizeof(tot_mse)); + for (i = 0; i < sb_count; i++) { + int gi; + uint64_t best_mse = (uint64_t)1 << 63; + /* Find best mse among already selected options. */ + for (gi = 0; gi < nb_strengths; gi++) { + if (mse[i][lev[gi]] < best_mse) { + best_mse = mse[i][lev[gi]]; + } + } + /* Find best mse when adding each possible new option. */ + for (j = 0; j < TOTAL_STRENGTHS; j++) { + uint64_t best = best_mse; + if (mse[i][j] < best) best = mse[i][j]; + tot_mse[j] += best; + } + } + for (j = 0; j < TOTAL_STRENGTHS; j++) { + if (tot_mse[j] < best_tot_mse) { + best_tot_mse = tot_mse[j]; + best_id = j; + } + } + lev[nb_strengths] = best_id; + return best_tot_mse; +} + +/* Search for the best luma+chroma strength to add as an option, knowing we + already selected nb_strengths options. */ +static uint64_t search_one_dual(int *lev0, int *lev1, int nb_strengths, + uint64_t (**mse)[TOTAL_STRENGTHS], + int sb_count) { + uint64_t tot_mse[TOTAL_STRENGTHS][TOTAL_STRENGTHS]; + int i, j; + uint64_t best_tot_mse = (uint64_t)1 << 63; + int best_id0 = 0; + int best_id1 = 0; + memset(tot_mse, 0, sizeof(tot_mse)); + for (i = 0; i < sb_count; i++) { + int gi; + uint64_t best_mse = (uint64_t)1 << 63; + /* Find best mse among already selected options. */ + for (gi = 0; gi < nb_strengths; gi++) { + uint64_t curr = mse[0][i][lev0[gi]]; + curr += mse[1][i][lev1[gi]]; + if (curr < best_mse) { + best_mse = curr; + } + } + /* Find best mse when adding each possible new option. */ + for (j = 0; j < TOTAL_STRENGTHS; j++) { + int k; + for (k = 0; k < TOTAL_STRENGTHS; k++) { + uint64_t best = best_mse; + uint64_t curr = mse[0][i][j]; + curr += mse[1][i][k]; + if (curr < best) best = curr; + tot_mse[j][k] += best; + } + } + } + for (j = 0; j < TOTAL_STRENGTHS; j++) { + int k; + for (k = 0; k < TOTAL_STRENGTHS; k++) { + if (tot_mse[j][k] < best_tot_mse) { + best_tot_mse = tot_mse[j][k]; + best_id0 = j; + best_id1 = k; + } + } + } + lev0[nb_strengths] = best_id0; + lev1[nb_strengths] = best_id1; + return best_tot_mse; +} + +/* Search for the set of strengths that minimizes mse. */ +static uint64_t joint_strength_search(int *best_lev, int nb_strengths, + uint64_t mse[][TOTAL_STRENGTHS], + int sb_count) { + uint64_t best_tot_mse; + int i; + best_tot_mse = (uint64_t)1 << 63; + /* Greedy search: add one strength options at a time. */ + for (i = 0; i < nb_strengths; i++) { + best_tot_mse = search_one(best_lev, i, mse, sb_count); + } + /* Trying to refine the greedy search by reconsidering each + already-selected option. */ + for (i = 0; i < 4 * nb_strengths; i++) { + int j; + for (j = 0; j < nb_strengths - 1; j++) best_lev[j] = best_lev[j + 1]; + best_tot_mse = search_one(best_lev, nb_strengths - 1, mse, sb_count); + } + return best_tot_mse; +} + +/* Search for the set of luma+chroma strengths that minimizes mse. */ +static uint64_t joint_strength_search_dual(int *best_lev0, int *best_lev1, + int nb_strengths, + uint64_t (**mse)[TOTAL_STRENGTHS], + int sb_count) { + uint64_t best_tot_mse; + int i; + best_tot_mse = (uint64_t)1 << 63; + /* Greedy search: add one strength options at a time. */ + for (i = 0; i < nb_strengths; i++) { + best_tot_mse = search_one_dual(best_lev0, best_lev1, i, mse, sb_count); + } + /* Trying to refine the greedy search by reconsidering each + already-selected option. */ + for (i = 0; i < 4 * nb_strengths; i++) { + int j; + for (j = 0; j < nb_strengths - 1; j++) { + best_lev0[j] = best_lev0[j + 1]; + best_lev1[j] = best_lev1[j + 1]; + } + best_tot_mse = + search_one_dual(best_lev0, best_lev1, nb_strengths - 1, mse, sb_count); + } + return best_tot_mse; +} + +/* FIXME: SSE-optimize this. */ +static void copy_sb16_16(uint16_t *dst, int dstride, const uint16_t *src, + int src_voffset, int src_hoffset, int sstride, + int vsize, int hsize) { + int r, c; + const uint16_t *base = &src[src_voffset * sstride + src_hoffset]; + for (r = 0; r < vsize; r++) { + for (c = 0; c < hsize; c++) { + dst[r * dstride + c] = base[r * sstride + c]; + } + } +} + +static INLINE uint64_t dist_8x8_16bit(uint16_t *dst, int dstride, uint16_t *src, + int sstride, int coeff_shift) { + uint64_t svar = 0; + uint64_t dvar = 0; + uint64_t sum_s = 0; + uint64_t sum_d = 0; + uint64_t sum_s2 = 0; + uint64_t sum_d2 = 0; + uint64_t sum_sd = 0; + int i, j; + for (i = 0; i < 8; i++) { + for (j = 0; j < 8; j++) { + sum_s += src[i * sstride + j]; + sum_d += dst[i * dstride + j]; + sum_s2 += src[i * sstride + j] * src[i * sstride + j]; + sum_d2 += dst[i * dstride + j] * dst[i * dstride + j]; + sum_sd += src[i * sstride + j] * dst[i * dstride + j]; + } + } + /* Compute the variance -- the calculation cannot go negative. */ + svar = sum_s2 - ((sum_s * sum_s + 32) >> 6); + dvar = sum_d2 - ((sum_d * sum_d + 32) >> 6); + return (uint64_t)floor( + .5 + + (sum_d2 + sum_s2 - 2 * sum_sd) * .5 * + (svar + dvar + (400 << 2 * coeff_shift)) / + (sqrt((20000 << 4 * coeff_shift) + svar * (double)dvar))); +} + +static INLINE uint64_t mse_8x8_16bit(uint16_t *dst, int dstride, uint16_t *src, + int sstride) { + uint64_t sum = 0; + int i, j; + for (i = 0; i < 8; i++) { + for (j = 0; j < 8; j++) { + int e = dst[i * dstride + j] - src[i * sstride + j]; + sum += e * e; + } + } + return sum; +} + +static INLINE uint64_t mse_4x4_16bit(uint16_t *dst, int dstride, uint16_t *src, + int sstride) { + uint64_t sum = 0; + int i, j; + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + int e = dst[i * dstride + j] - src[i * sstride + j]; + sum += e * e; + } + } + return sum; +} + +/* Compute MSE only on the blocks we filtered. */ +uint64_t compute_dering_dist(uint16_t *dst, int dstride, uint16_t *src, + dering_list *dlist, int dering_count, + BLOCK_SIZE bsize, int coeff_shift, int pli) { + uint64_t sum = 0; + int bi, bx, by; + if (bsize == BLOCK_8X8) { + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + if (pli == 0) { + sum += dist_8x8_16bit(&dst[(by << 3) * dstride + (bx << 3)], dstride, + &src[bi << (3 + 3)], 8, coeff_shift); + } else { + sum += mse_8x8_16bit(&dst[(by << 3) * dstride + (bx << 3)], dstride, + &src[bi << (3 + 3)], 8); + } + } + } else if (bsize == BLOCK_4X8) { + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + sum += mse_4x4_16bit(&dst[(by << 3) * dstride + (bx << 2)], dstride, + &src[bi << (3 + 2)], 4); + sum += mse_4x4_16bit(&dst[((by << 3) + 4) * dstride + (bx << 2)], dstride, + &src[(bi << (3 + 2)) + 4 * 4], 4); + } + } else if (bsize == BLOCK_8X4) { + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + sum += mse_4x4_16bit(&dst[(by << 2) * dstride + (bx << 3)], dstride, + &src[bi << (2 + 3)], 8); + sum += mse_4x4_16bit(&dst[(by << 2) * dstride + (bx << 3) + 4], dstride, + &src[(bi << (2 + 3)) + 4], 8); + } + } else { + assert(bsize == BLOCK_4X4); + for (bi = 0; bi < dering_count; bi++) { + by = dlist[bi].by; + bx = dlist[bi].bx; + sum += mse_4x4_16bit(&dst[(by << 2) * dstride + (bx << 2)], dstride, + &src[bi << (2 + 2)], 4); + } + } + return sum >> 2 * coeff_shift; +} + +void av1_cdef_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref, + AV1_COMMON *cm, MACROBLOCKD *xd) { + int r, c; + int sbr, sbc; + uint16_t *src[3]; + uint16_t *ref_coeff[3]; + dering_list dlist[MAX_MIB_SIZE * MAX_MIB_SIZE]; + int dir[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS] = { { 0 } }; + int var[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS] = { { 0 } }; + int stride[3]; + int bsize[3]; + int mi_wide_l2[3]; + int mi_high_l2[3]; + int xdec[3]; + int ydec[3]; + int pli; + int dering_count; + int coeff_shift = AOMMAX(cm->bit_depth - 8, 0); + uint64_t best_tot_mse = (uint64_t)1 << 63; + uint64_t tot_mse; + int sb_count; + int nvsb = (cm->mi_rows + MAX_MIB_SIZE - 1) / MAX_MIB_SIZE; + int nhsb = (cm->mi_cols + MAX_MIB_SIZE - 1) / MAX_MIB_SIZE; + int *sb_index = aom_malloc(nvsb * nhsb * sizeof(*sb_index)); + int *selected_strength = aom_malloc(nvsb * nhsb * sizeof(*sb_index)); + uint64_t(*mse[2])[TOTAL_STRENGTHS]; + int clpf_damping = 3 + (cm->base_qindex >> 6); + int dering_damping = 6; + int i; + int nb_strengths; + int nb_strength_bits; + int quantizer; + double lambda; + int nplanes = 3; + DECLARE_ALIGNED(32, uint16_t, inbuf[OD_DERING_INBUF_SIZE]); + uint16_t *in; + DECLARE_ALIGNED(32, uint16_t, tmp_dst[MAX_SB_SQUARE]); + int chroma_dering = + xd->plane[1].subsampling_x == xd->plane[1].subsampling_y && + xd->plane[2].subsampling_x == xd->plane[2].subsampling_y; + quantizer = + av1_ac_quant(cm->base_qindex, 0, cm->bit_depth) >> (cm->bit_depth - 8); + lambda = .12 * quantizer * quantizer / 256.; + + av1_setup_dst_planes(xd->plane, cm->sb_size, frame, 0, 0); + mse[0] = aom_malloc(sizeof(**mse) * nvsb * nhsb); + mse[1] = aom_malloc(sizeof(**mse) * nvsb * nhsb); + for (pli = 0; pli < nplanes; pli++) { + uint8_t *ref_buffer; + int ref_stride; + switch (pli) { + case 0: + ref_buffer = ref->y_buffer; + ref_stride = ref->y_stride; + break; + case 1: + ref_buffer = ref->u_buffer; + ref_stride = ref->uv_stride; + break; + case 2: + ref_buffer = ref->v_buffer; + ref_stride = ref->uv_stride; + break; + } + src[pli] = aom_memalign( + 32, sizeof(*src) * cm->mi_rows * cm->mi_cols * MI_SIZE * MI_SIZE); + ref_coeff[pli] = aom_memalign( + 32, sizeof(*ref_coeff) * cm->mi_rows * cm->mi_cols * MI_SIZE * MI_SIZE); + xdec[pli] = xd->plane[pli].subsampling_x; + ydec[pli] = xd->plane[pli].subsampling_y; + bsize[pli] = ydec[pli] ? (xdec[pli] ? BLOCK_4X4 : BLOCK_8X4) + : (xdec[pli] ? BLOCK_4X8 : BLOCK_8X8); + stride[pli] = cm->mi_cols << MI_SIZE_LOG2; + mi_wide_l2[pli] = MI_SIZE_LOG2 - xd->plane[pli].subsampling_x; + mi_high_l2[pli] = MI_SIZE_LOG2 - xd->plane[pli].subsampling_y; + + const int frame_height = + (cm->mi_rows * MI_SIZE) >> xd->plane[pli].subsampling_y; + const int frame_width = + (cm->mi_cols * MI_SIZE) >> xd->plane[pli].subsampling_x; + + for (r = 0; r < frame_height; ++r) { + for (c = 0; c < frame_width; ++c) { +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) { + src[pli][r * stride[pli] + c] = CONVERT_TO_SHORTPTR( + xd->plane[pli].dst.buf)[r * xd->plane[pli].dst.stride + c]; + ref_coeff[pli][r * stride[pli] + c] = + CONVERT_TO_SHORTPTR(ref_buffer)[r * ref_stride + c]; + } else { +#endif + src[pli][r * stride[pli] + c] = + xd->plane[pli].dst.buf[r * xd->plane[pli].dst.stride + c]; + ref_coeff[pli][r * stride[pli] + c] = ref_buffer[r * ref_stride + c]; +#if CONFIG_HIGHBITDEPTH + } +#endif + } + } + } + in = inbuf + OD_FILT_VBORDER * OD_FILT_BSTRIDE + OD_FILT_HBORDER; + sb_count = 0; + for (sbr = 0; sbr < nvsb; ++sbr) { + for (sbc = 0; sbc < nhsb; ++sbc) { + int nvb, nhb; + int gi; + int dirinit = 0; + nhb = AOMMIN(MAX_MIB_SIZE, cm->mi_cols - MAX_MIB_SIZE * sbc); + nvb = AOMMIN(MAX_MIB_SIZE, cm->mi_rows - MAX_MIB_SIZE * sbr); + cm->mi_grid_visible[MAX_MIB_SIZE * sbr * cm->mi_stride + + MAX_MIB_SIZE * sbc] + ->mbmi.cdef_strength = -1; + if (sb_all_skip(cm, sbr * MAX_MIB_SIZE, sbc * MAX_MIB_SIZE)) continue; + dering_count = sb_compute_dering_list(cm, sbr * MAX_MIB_SIZE, + sbc * MAX_MIB_SIZE, dlist, 1); + for (pli = 0; pli < nplanes; pli++) { + for (i = 0; i < OD_DERING_INBUF_SIZE; i++) + inbuf[i] = OD_DERING_VERY_LARGE; + for (gi = 0; gi < TOTAL_STRENGTHS; gi++) { + int threshold; + uint64_t curr_mse; + int clpf_strength; + threshold = gi / CLPF_STRENGTHS; + if (pli > 0 && !chroma_dering) threshold = 0; + /* We avoid filtering the pixels for which some of the pixels to + average + are outside the frame. We could change the filter instead, but it + would add special cases for any future vectorization. */ + int yoff = OD_FILT_VBORDER * (sbr != 0); + int xoff = OD_FILT_HBORDER * (sbc != 0); + int ysize = (nvb << mi_high_l2[pli]) + + OD_FILT_VBORDER * (sbr != nvsb - 1) + yoff; + int xsize = (nhb << mi_wide_l2[pli]) + + OD_FILT_HBORDER * (sbc != nhsb - 1) + xoff; + clpf_strength = gi % CLPF_STRENGTHS; + if (clpf_strength == 0) + copy_sb16_16(&in[(-yoff * OD_FILT_BSTRIDE - xoff)], OD_FILT_BSTRIDE, + src[pli], + (sbr * MAX_MIB_SIZE << mi_high_l2[pli]) - yoff, + (sbc * MAX_MIB_SIZE << mi_wide_l2[pli]) - xoff, + stride[pli], ysize, xsize); + od_dering(clpf_strength ? NULL : (uint8_t *)in, OD_FILT_BSTRIDE, + tmp_dst, in, xdec[pli], ydec[pli], dir, &dirinit, var, pli, + dlist, dering_count, threshold, + clpf_strength + (clpf_strength == 3), clpf_damping, + dering_damping, coeff_shift, clpf_strength != 0, 1); + curr_mse = compute_dering_dist( + ref_coeff[pli] + + (sbr * MAX_MIB_SIZE << mi_high_l2[pli]) * stride[pli] + + (sbc * MAX_MIB_SIZE << mi_wide_l2[pli]), + stride[pli], tmp_dst, dlist, dering_count, bsize[pli], + coeff_shift, pli); + if (pli < 2) + mse[pli][sb_count][gi] = curr_mse; + else + mse[1][sb_count][gi] += curr_mse; + sb_index[sb_count] = + MAX_MIB_SIZE * sbr * cm->mi_stride + MAX_MIB_SIZE * sbc; + } + } + sb_count++; + } + } + nb_strength_bits = 0; + /* Search for different number of signalling bits. */ + for (i = 0; i <= 3; i++) { + int j; + int best_lev0[CDEF_MAX_STRENGTHS]; + int best_lev1[CDEF_MAX_STRENGTHS] = { 0 }; + nb_strengths = 1 << i; + if (nplanes >= 3) + tot_mse = joint_strength_search_dual(best_lev0, best_lev1, nb_strengths, + mse, sb_count); + else + tot_mse = + joint_strength_search(best_lev0, nb_strengths, mse[0], sb_count); + /* Count superblock signalling cost. */ + tot_mse += (uint64_t)(sb_count * lambda * i); + /* Count header signalling cost. */ + tot_mse += (uint64_t)(nb_strengths * lambda * CDEF_STRENGTH_BITS); + if (tot_mse < best_tot_mse) { + best_tot_mse = tot_mse; + nb_strength_bits = i; + for (j = 0; j < 1 << nb_strength_bits; j++) { + cm->cdef_strengths[j] = best_lev0[j]; + cm->cdef_uv_strengths[j] = best_lev1[j]; + } + } + } + nb_strengths = 1 << nb_strength_bits; + + cm->cdef_bits = nb_strength_bits; + cm->nb_cdef_strengths = nb_strengths; + for (i = 0; i < sb_count; i++) { + int gi; + int best_gi; + uint64_t best_mse = (uint64_t)1 << 63; + best_gi = 0; + for (gi = 0; gi < cm->nb_cdef_strengths; gi++) { + uint64_t curr = mse[0][i][cm->cdef_strengths[gi]]; + if (nplanes >= 3) curr += mse[1][i][cm->cdef_uv_strengths[gi]]; + if (curr < best_mse) { + best_gi = gi; + best_mse = curr; + } + } + selected_strength[i] = best_gi; + cm->mi_grid_visible[sb_index[i]]->mbmi.cdef_strength = best_gi; + } + cm->cdef_dering_damping = dering_damping; + cm->cdef_clpf_damping = clpf_damping; + aom_free(mse[0]); + aom_free(mse[1]); + for (pli = 0; pli < nplanes; pli++) { + aom_free(src[pli]); + aom_free(ref_coeff[pli]); + } + aom_free(sb_index); + aom_free(selected_strength); +} diff --git a/third_party/aom/av1/encoder/picklpf.c b/third_party/aom/av1/encoder/picklpf.c new file mode 100644 index 0000000000..fc0ea485d8 --- /dev/null +++ b/third_party/aom/av1/encoder/picklpf.c @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_scale_rtcd.h" + +#include "aom_dsp/psnr.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +#include "av1/common/av1_loopfilter.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/quant_common.h" + +#include "av1/encoder/av1_quantize.h" +#include "av1/encoder/encoder.h" +#include "av1/encoder/picklpf.h" + +int av1_get_max_filter_level(const AV1_COMP *cpi) { + if (cpi->oxcf.pass == 2) { + return cpi->twopass.section_intra_rating > 8 ? MAX_LOOP_FILTER * 3 / 4 + : MAX_LOOP_FILTER; + } else { + return MAX_LOOP_FILTER; + } +} + +static int64_t try_filter_frame(const YV12_BUFFER_CONFIG *sd, + AV1_COMP *const cpi, int filt_level, + int partial_frame) { + AV1_COMMON *const cm = &cpi->common; + int64_t filt_err; + +#if CONFIG_VAR_TX || CONFIG_EXT_PARTITION || CONFIG_CB4X4 + av1_loop_filter_frame(cm->frame_to_show, cm, &cpi->td.mb.e_mbd, filt_level, 1, + partial_frame); +#else + if (cpi->num_workers > 1) + av1_loop_filter_frame_mt(cm->frame_to_show, cm, cpi->td.mb.e_mbd.plane, + filt_level, 1, partial_frame, cpi->workers, + cpi->num_workers, &cpi->lf_row_sync); + else + av1_loop_filter_frame(cm->frame_to_show, cm, &cpi->td.mb.e_mbd, filt_level, + 1, partial_frame); +#endif + +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) { + filt_err = aom_highbd_get_y_sse(sd, cm->frame_to_show); + } else { + filt_err = aom_get_y_sse(sd, cm->frame_to_show); + } +#else + filt_err = aom_get_y_sse(sd, cm->frame_to_show); +#endif // CONFIG_HIGHBITDEPTH + + // Re-instate the unfiltered frame + aom_yv12_copy_y(&cpi->last_frame_uf, cm->frame_to_show); + + return filt_err; +} + +int av1_search_filter_level(const YV12_BUFFER_CONFIG *sd, AV1_COMP *cpi, + int partial_frame, double *best_cost_ret) { + const AV1_COMMON *const cm = &cpi->common; + const struct loopfilter *const lf = &cm->lf; + const int min_filter_level = 0; + const int max_filter_level = av1_get_max_filter_level(cpi); + int filt_direction = 0; + int64_t best_err; + int filt_best; + MACROBLOCK *x = &cpi->td.mb; + + // Start the search at the previous frame filter level unless it is now out of + // range. + int filt_mid = clamp(lf->filter_level, min_filter_level, max_filter_level); + int filter_step = filt_mid < 16 ? 4 : filt_mid / 4; + // Sum squared error at each filter level + int64_t ss_err[MAX_LOOP_FILTER + 1]; + + // Set each entry to -1 + memset(ss_err, 0xFF, sizeof(ss_err)); + + // Make a copy of the unfiltered / processed recon buffer + aom_yv12_copy_y(cm->frame_to_show, &cpi->last_frame_uf); + + best_err = try_filter_frame(sd, cpi, filt_mid, partial_frame); + filt_best = filt_mid; + ss_err[filt_mid] = best_err; + + while (filter_step > 0) { + const int filt_high = AOMMIN(filt_mid + filter_step, max_filter_level); + const int filt_low = AOMMAX(filt_mid - filter_step, min_filter_level); + + // Bias against raising loop filter in favor of lowering it. + int64_t bias = (best_err >> (15 - (filt_mid / 8))) * filter_step; + + if ((cpi->oxcf.pass == 2) && (cpi->twopass.section_intra_rating < 20)) + bias = (bias * cpi->twopass.section_intra_rating) / 20; + + // yx, bias less for large block size + if (cm->tx_mode != ONLY_4X4) bias >>= 1; + + if (filt_direction <= 0 && filt_low != filt_mid) { + // Get Low filter error score + if (ss_err[filt_low] < 0) { + ss_err[filt_low] = try_filter_frame(sd, cpi, filt_low, partial_frame); + } + // If value is close to the best so far then bias towards a lower loop + // filter value. + if (ss_err[filt_low] < (best_err + bias)) { + // Was it actually better than the previous best? + if (ss_err[filt_low] < best_err) { + best_err = ss_err[filt_low]; + } + filt_best = filt_low; + } + } + + // Now look at filt_high + if (filt_direction >= 0 && filt_high != filt_mid) { + if (ss_err[filt_high] < 0) { + ss_err[filt_high] = try_filter_frame(sd, cpi, filt_high, partial_frame); + } + // If value is significantly better than previous best, bias added against + // raising filter value + if (ss_err[filt_high] < (best_err - bias)) { + best_err = ss_err[filt_high]; + filt_best = filt_high; + } + } + + // Half the step distance if the best filter value was the same as last time + if (filt_best == filt_mid) { + filter_step /= 2; + filt_direction = 0; + } else { + filt_direction = (filt_best < filt_mid) ? -1 : 1; + filt_mid = filt_best; + } + } + + // Update best error + best_err = ss_err[filt_best]; + + if (best_cost_ret) + *best_cost_ret = RDCOST_DBL(x->rdmult, x->rddiv, 0, best_err); + return filt_best; +} + +void av1_pick_filter_level(const YV12_BUFFER_CONFIG *sd, AV1_COMP *cpi, + LPF_PICK_METHOD method) { + AV1_COMMON *const cm = &cpi->common; + struct loopfilter *const lf = &cm->lf; + + lf->sharpness_level = cm->frame_type == KEY_FRAME ? 0 : cpi->oxcf.sharpness; + + if (method == LPF_PICK_MINIMAL_LPF && lf->filter_level) { + lf->filter_level = 0; + } else if (method >= LPF_PICK_FROM_Q) { + const int min_filter_level = 0; + const int max_filter_level = av1_get_max_filter_level(cpi); + const int q = av1_ac_quant(cm->base_qindex, 0, cm->bit_depth); +// These values were determined by linear fitting the result of the +// searched level, filt_guess = q * 0.316206 + 3.87252 +#if CONFIG_HIGHBITDEPTH + int filt_guess; + switch (cm->bit_depth) { + case AOM_BITS_8: + filt_guess = ROUND_POWER_OF_TWO(q * 20723 + 1015158, 18); + break; + case AOM_BITS_10: + filt_guess = ROUND_POWER_OF_TWO(q * 20723 + 4060632, 20); + break; + case AOM_BITS_12: + filt_guess = ROUND_POWER_OF_TWO(q * 20723 + 16242526, 22); + break; + default: + assert(0 && + "bit_depth should be AOM_BITS_8, AOM_BITS_10 " + "or AOM_BITS_12"); + return; + } +#else + int filt_guess = ROUND_POWER_OF_TWO(q * 20723 + 1015158, 18); +#endif // CONFIG_HIGHBITDEPTH + if (cm->frame_type == KEY_FRAME) filt_guess -= 4; + lf->filter_level = clamp(filt_guess, min_filter_level, max_filter_level); + } else { + lf->filter_level = av1_search_filter_level( + sd, cpi, method == LPF_PICK_FROM_SUBIMAGE, NULL); + } + +#if CONFIG_EXT_TILE + // TODO(any): 0 loopfilter level is only necessary if individual tile + // decoding is required. We need to communicate this requirement to this + // code and force loop filter level 0 only if required. + if (cm->tile_encoding_mode) lf->filter_level = 0; +#endif // CONFIG_EXT_TILE +} diff --git a/third_party/aom/av1/encoder/picklpf.h b/third_party/aom/av1/encoder/picklpf.h new file mode 100644 index 0000000000..3c0a83462b --- /dev/null +++ b/third_party/aom/av1/encoder/picklpf.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_PICKLPF_H_ +#define AV1_ENCODER_PICKLPF_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "av1/encoder/encoder.h" + +struct yv12_buffer_config; +struct AV1_COMP; +int av1_get_max_filter_level(const AV1_COMP *cpi); +int av1_search_filter_level(const YV12_BUFFER_CONFIG *sd, AV1_COMP *cpi, + int partial_frame, double *err); +void av1_pick_filter_level(const struct yv12_buffer_config *sd, + struct AV1_COMP *cpi, LPF_PICK_METHOD method); +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_PICKLPF_H_ diff --git a/third_party/aom/av1/encoder/pickrst.c b/third_party/aom/av1/encoder/pickrst.c new file mode 100644 index 0000000000..21410e0afa --- /dev/null +++ b/third_party/aom/av1/encoder/pickrst.c @@ -0,0 +1,1269 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include + +#include "./aom_scale_rtcd.h" + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/binary_codes_writer.h" +#include "aom_dsp/psnr.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" +#include "aom_ports/system_state.h" + +#include "av1/common/onyxc_int.h" +#include "av1/common/quant_common.h" +#include "av1/common/restoration.h" + +#include "av1/encoder/av1_quantize.h" +#include "av1/encoder/encoder.h" +#include "av1/encoder/picklpf.h" +#include "av1/encoder/pickrst.h" + +// When set to RESTORE_WIENER or RESTORE_SGRPROJ only those are allowed. +// When set to RESTORE_NONE (0) we allow switchable. +const RestorationType force_restore_type = RESTORE_NONE; + +// Number of Wiener iterations +#define NUM_WIENER_ITERS 10 + +typedef double (*search_restore_type)(const YV12_BUFFER_CONFIG *src, + AV1_COMP *cpi, int partial_frame, + RestorationInfo *info, + RestorationType *rest_level, + double *best_tile_cost, + YV12_BUFFER_CONFIG *dst_frame); + +const int frame_level_restore_bits[RESTORE_TYPES] = { 2, 2, 2, 2 }; + +static int64_t sse_restoration_tile(const YV12_BUFFER_CONFIG *src, + const YV12_BUFFER_CONFIG *dst, + const AV1_COMMON *cm, int h_start, + int width, int v_start, int height, + int components_pattern) { + int64_t filt_err = 0; + (void)cm; + // Y and UV components cannot be mixed + assert(components_pattern == 1 || components_pattern == 2 || + components_pattern == 4 || components_pattern == 6); +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) { + if ((components_pattern >> AOM_PLANE_Y) & 1) { + filt_err += + aom_highbd_get_y_sse_part(src, dst, h_start, width, v_start, height); + } + if ((components_pattern >> AOM_PLANE_U) & 1) { + filt_err += + aom_highbd_get_u_sse_part(src, dst, h_start, width, v_start, height); + } + if ((components_pattern >> AOM_PLANE_V) & 1) { + filt_err += + aom_highbd_get_v_sse_part(src, dst, h_start, width, v_start, height); + } + return filt_err; + } +#endif // CONFIG_HIGHBITDEPTH + if ((components_pattern >> AOM_PLANE_Y) & 1) { + filt_err += aom_get_y_sse_part(src, dst, h_start, width, v_start, height); + } + if ((components_pattern >> AOM_PLANE_U) & 1) { + filt_err += aom_get_u_sse_part(src, dst, h_start, width, v_start, height); + } + if ((components_pattern >> AOM_PLANE_V) & 1) { + filt_err += aom_get_v_sse_part(src, dst, h_start, width, v_start, height); + } + return filt_err; +} + +static int64_t sse_restoration_frame(AV1_COMMON *const cm, + const YV12_BUFFER_CONFIG *src, + const YV12_BUFFER_CONFIG *dst, + int components_pattern) { + int64_t filt_err = 0; +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) { + if ((components_pattern >> AOM_PLANE_Y) & 1) { + filt_err += aom_highbd_get_y_sse(src, dst); + } + if ((components_pattern >> AOM_PLANE_U) & 1) { + filt_err += aom_highbd_get_u_sse(src, dst); + } + if ((components_pattern >> AOM_PLANE_V) & 1) { + filt_err += aom_highbd_get_v_sse(src, dst); + } + return filt_err; + } +#else + (void)cm; +#endif // CONFIG_HIGHBITDEPTH + if ((components_pattern >> AOM_PLANE_Y) & 1) { + filt_err = aom_get_y_sse(src, dst); + } + if ((components_pattern >> AOM_PLANE_U) & 1) { + filt_err += aom_get_u_sse(src, dst); + } + if ((components_pattern >> AOM_PLANE_V) & 1) { + filt_err += aom_get_v_sse(src, dst); + } + return filt_err; +} + +static int64_t try_restoration_tile(const YV12_BUFFER_CONFIG *src, + AV1_COMP *const cpi, RestorationInfo *rsi, + int components_pattern, int partial_frame, + int tile_idx, int subtile_idx, + int subtile_bits, + YV12_BUFFER_CONFIG *dst_frame) { + AV1_COMMON *const cm = &cpi->common; + int64_t filt_err; + int tile_width, tile_height, nhtiles, nvtiles; + int h_start, h_end, v_start, v_end; + int ntiles, width, height; + + // Y and UV components cannot be mixed + assert(components_pattern == 1 || components_pattern == 2 || + components_pattern == 4 || components_pattern == 6); + + if (components_pattern == 1) { // Y only + width = src->y_crop_width; + height = src->y_crop_height; + } else { // Color + width = src->uv_crop_width; + height = src->uv_crop_height; + } + ntiles = av1_get_rest_ntiles( + width, height, cm->rst_info[components_pattern > 1].restoration_tilesize, + &tile_width, &tile_height, &nhtiles, &nvtiles); + (void)ntiles; + + av1_loop_restoration_frame(cm->frame_to_show, cm, rsi, components_pattern, + partial_frame, dst_frame); + av1_get_rest_tile_limits(tile_idx, subtile_idx, subtile_bits, nhtiles, + nvtiles, tile_width, tile_height, width, height, 0, + 0, &h_start, &h_end, &v_start, &v_end); + filt_err = sse_restoration_tile(src, dst_frame, cm, h_start, h_end - h_start, + v_start, v_end - v_start, components_pattern); + + return filt_err; +} + +static int64_t try_restoration_frame(const YV12_BUFFER_CONFIG *src, + AV1_COMP *const cpi, RestorationInfo *rsi, + int components_pattern, int partial_frame, + YV12_BUFFER_CONFIG *dst_frame) { + AV1_COMMON *const cm = &cpi->common; + int64_t filt_err; + av1_loop_restoration_frame(cm->frame_to_show, cm, rsi, components_pattern, + partial_frame, dst_frame); + filt_err = sse_restoration_frame(cm, src, dst_frame, components_pattern); + return filt_err; +} + +static int64_t get_pixel_proj_error(uint8_t *src8, int width, int height, + int src_stride, uint8_t *dat8, + int dat_stride, int bit_depth, + int32_t *flt1, int flt1_stride, + int32_t *flt2, int flt2_stride, int *xqd) { + int i, j; + int64_t err = 0; + int xq[2]; + decode_xq(xqd, xq); + if (bit_depth == 8) { + const uint8_t *src = src8; + const uint8_t *dat = dat8; + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const int32_t u = + (int32_t)(dat[i * dat_stride + j] << SGRPROJ_RST_BITS); + const int32_t f1 = (int32_t)flt1[i * flt1_stride + j] - u; + const int32_t f2 = (int32_t)flt2[i * flt2_stride + j] - u; + const int32_t v = xq[0] * f1 + xq[1] * f2 + (u << SGRPROJ_PRJ_BITS); + const int32_t e = + ROUND_POWER_OF_TWO(v, SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS) - + src[i * src_stride + j]; + err += e * e; + } + } + } else { + const uint16_t *src = CONVERT_TO_SHORTPTR(src8); + const uint16_t *dat = CONVERT_TO_SHORTPTR(dat8); + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const int32_t u = + (int32_t)(dat[i * dat_stride + j] << SGRPROJ_RST_BITS); + const int32_t f1 = (int32_t)flt1[i * flt1_stride + j] - u; + const int32_t f2 = (int32_t)flt2[i * flt2_stride + j] - u; + const int32_t v = xq[0] * f1 + xq[1] * f2 + (u << SGRPROJ_PRJ_BITS); + const int32_t e = + ROUND_POWER_OF_TWO(v, SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS) - + src[i * src_stride + j]; + err += e * e; + } + } + } + return err; +} + +static void get_proj_subspace(uint8_t *src8, int width, int height, + int src_stride, uint8_t *dat8, int dat_stride, + int bit_depth, int32_t *flt1, int flt1_stride, + int32_t *flt2, int flt2_stride, int *xq) { + int i, j; + double H[2][2] = { { 0, 0 }, { 0, 0 } }; + double C[2] = { 0, 0 }; + double Det; + double x[2]; + const int size = width * height; + + aom_clear_system_state(); + + // Default + xq[0] = 0; + xq[1] = 0; + if (bit_depth == 8) { + const uint8_t *src = src8; + const uint8_t *dat = dat8; + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const double u = (double)(dat[i * dat_stride + j] << SGRPROJ_RST_BITS); + const double s = + (double)(src[i * src_stride + j] << SGRPROJ_RST_BITS) - u; + const double f1 = (double)flt1[i * flt1_stride + j] - u; + const double f2 = (double)flt2[i * flt2_stride + j] - u; + H[0][0] += f1 * f1; + H[1][1] += f2 * f2; + H[0][1] += f1 * f2; + C[0] += f1 * s; + C[1] += f2 * s; + } + } + } else { + const uint16_t *src = CONVERT_TO_SHORTPTR(src8); + const uint16_t *dat = CONVERT_TO_SHORTPTR(dat8); + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const double u = (double)(dat[i * dat_stride + j] << SGRPROJ_RST_BITS); + const double s = + (double)(src[i * src_stride + j] << SGRPROJ_RST_BITS) - u; + const double f1 = (double)flt1[i * flt1_stride + j] - u; + const double f2 = (double)flt2[i * flt2_stride + j] - u; + H[0][0] += f1 * f1; + H[1][1] += f2 * f2; + H[0][1] += f1 * f2; + C[0] += f1 * s; + C[1] += f2 * s; + } + } + } + H[0][0] /= size; + H[0][1] /= size; + H[1][1] /= size; + H[1][0] = H[0][1]; + C[0] /= size; + C[1] /= size; + Det = (H[0][0] * H[1][1] - H[0][1] * H[1][0]); + if (Det < 1e-8) return; // ill-posed, return default values + x[0] = (H[1][1] * C[0] - H[0][1] * C[1]) / Det; + x[1] = (H[0][0] * C[1] - H[1][0] * C[0]) / Det; + xq[0] = (int)rint(x[0] * (1 << SGRPROJ_PRJ_BITS)); + xq[1] = (int)rint(x[1] * (1 << SGRPROJ_PRJ_BITS)); +} + +void encode_xq(int *xq, int *xqd) { + xqd[0] = xq[0]; + xqd[0] = clamp(xqd[0], SGRPROJ_PRJ_MIN0, SGRPROJ_PRJ_MAX0); + xqd[1] = (1 << SGRPROJ_PRJ_BITS) - xqd[0] - xq[1]; + xqd[1] = clamp(xqd[1], SGRPROJ_PRJ_MIN1, SGRPROJ_PRJ_MAX1); +} + +static void search_selfguided_restoration(uint8_t *dat8, int width, int height, + int dat_stride, uint8_t *src8, + int src_stride, int bit_depth, + int *eps, int *xqd, int32_t *rstbuf) { + int32_t *flt1 = rstbuf; + int32_t *flt2 = flt1 + RESTORATION_TILEPELS_MAX; + int32_t *tmpbuf2 = flt2 + RESTORATION_TILEPELS_MAX; + int ep, bestep = 0; + int64_t err, besterr = -1; + int exqd[2], bestxqd[2] = { 0, 0 }; + + for (ep = 0; ep < SGRPROJ_PARAMS; ep++) { + int exq[2]; +#if CONFIG_HIGHBITDEPTH + if (bit_depth > 8) { + uint16_t *dat = CONVERT_TO_SHORTPTR(dat8); +#if USE_HIGHPASS_IN_SGRPROJ + av1_highpass_filter_highbd(dat, width, height, dat_stride, flt1, width, + sgr_params[ep].corner, sgr_params[ep].edge); +#else + av1_selfguided_restoration_highbd(dat, width, height, dat_stride, flt1, + width, bit_depth, sgr_params[ep].r1, + sgr_params[ep].e1, tmpbuf2); +#endif // USE_HIGHPASS_IN_SGRPROJ + av1_selfguided_restoration_highbd(dat, width, height, dat_stride, flt2, + width, bit_depth, sgr_params[ep].r2, + sgr_params[ep].e2, tmpbuf2); + } else { +#endif +#if USE_HIGHPASS_IN_SGRPROJ + av1_highpass_filter(dat8, width, height, dat_stride, flt1, width, + sgr_params[ep].corner, sgr_params[ep].edge); +#else + av1_selfguided_restoration(dat8, width, height, dat_stride, flt1, width, + sgr_params[ep].r1, sgr_params[ep].e1, tmpbuf2); +#endif // USE_HIGHPASS_IN_SGRPROJ + av1_selfguided_restoration(dat8, width, height, dat_stride, flt2, width, + sgr_params[ep].r2, sgr_params[ep].e2, tmpbuf2); +#if CONFIG_HIGHBITDEPTH + } +#endif + get_proj_subspace(src8, width, height, src_stride, dat8, dat_stride, + bit_depth, flt1, width, flt2, width, exq); + encode_xq(exq, exqd); + err = + get_pixel_proj_error(src8, width, height, src_stride, dat8, dat_stride, + bit_depth, flt1, width, flt2, width, exqd); + if (besterr == -1 || err < besterr) { + bestep = ep; + besterr = err; + bestxqd[0] = exqd[0]; + bestxqd[1] = exqd[1]; + } + } + *eps = bestep; + xqd[0] = bestxqd[0]; + xqd[1] = bestxqd[1]; +} + +static int count_sgrproj_bits(SgrprojInfo *sgrproj_info, + SgrprojInfo *ref_sgrproj_info) { + int bits = SGRPROJ_PARAMS_BITS; + bits += aom_count_primitive_refsubexpfin( + SGRPROJ_PRJ_MAX0 - SGRPROJ_PRJ_MIN0 + 1, SGRPROJ_PRJ_SUBEXP_K, + ref_sgrproj_info->xqd[0] - SGRPROJ_PRJ_MIN0, + sgrproj_info->xqd[0] - SGRPROJ_PRJ_MIN0); + bits += aom_count_primitive_refsubexpfin( + SGRPROJ_PRJ_MAX1 - SGRPROJ_PRJ_MIN1 + 1, SGRPROJ_PRJ_SUBEXP_K, + ref_sgrproj_info->xqd[1] - SGRPROJ_PRJ_MIN1, + sgrproj_info->xqd[1] - SGRPROJ_PRJ_MIN1); + return bits; +} + +static double search_sgrproj(const YV12_BUFFER_CONFIG *src, AV1_COMP *cpi, + int partial_frame, RestorationInfo *info, + RestorationType *type, double *best_tile_cost, + YV12_BUFFER_CONFIG *dst_frame) { + SgrprojInfo *sgrproj_info = info->sgrproj_info; + double err, cost_norestore, cost_sgrproj; + int bits; + MACROBLOCK *x = &cpi->td.mb; + AV1_COMMON *const cm = &cpi->common; + const YV12_BUFFER_CONFIG *dgd = cm->frame_to_show; + RestorationInfo *rsi = &cpi->rst_search[0]; + int tile_idx, tile_width, tile_height, nhtiles, nvtiles; + int h_start, h_end, v_start, v_end; + // Allocate for the src buffer at high precision + const int ntiles = av1_get_rest_ntiles( + cm->width, cm->height, cm->rst_info[0].restoration_tilesize, &tile_width, + &tile_height, &nhtiles, &nvtiles); + SgrprojInfo ref_sgrproj_info; + set_default_sgrproj(&ref_sgrproj_info); + + rsi->frame_restoration_type = RESTORE_SGRPROJ; + + for (tile_idx = 0; tile_idx < ntiles; ++tile_idx) { + rsi->restoration_type[tile_idx] = RESTORE_NONE; + } + // Compute best Sgrproj filters for each tile + for (tile_idx = 0; tile_idx < ntiles; ++tile_idx) { + av1_get_rest_tile_limits(tile_idx, 0, 0, nhtiles, nvtiles, tile_width, + tile_height, cm->width, cm->height, 0, 0, &h_start, + &h_end, &v_start, &v_end); + err = sse_restoration_tile(src, cm->frame_to_show, cm, h_start, + h_end - h_start, v_start, v_end - v_start, 1); + // #bits when a tile is not restored + bits = av1_cost_bit(RESTORE_NONE_SGRPROJ_PROB, 0); + cost_norestore = RDCOST_DBL(x->rdmult, x->rddiv, (bits >> 4), err); + best_tile_cost[tile_idx] = DBL_MAX; + search_selfguided_restoration( + dgd->y_buffer + v_start * dgd->y_stride + h_start, h_end - h_start, + v_end - v_start, dgd->y_stride, + src->y_buffer + v_start * src->y_stride + h_start, src->y_stride, +#if CONFIG_HIGHBITDEPTH + cm->bit_depth, +#else + 8, +#endif // CONFIG_HIGHBITDEPTH + &rsi->sgrproj_info[tile_idx].ep, rsi->sgrproj_info[tile_idx].xqd, + cm->rst_internal.tmpbuf); + rsi->restoration_type[tile_idx] = RESTORE_SGRPROJ; + err = try_restoration_tile(src, cpi, rsi, 1, partial_frame, tile_idx, 0, 0, + dst_frame); + bits = count_sgrproj_bits(&rsi->sgrproj_info[tile_idx], &ref_sgrproj_info) + << AV1_PROB_COST_SHIFT; + bits += av1_cost_bit(RESTORE_NONE_SGRPROJ_PROB, 1); + cost_sgrproj = RDCOST_DBL(x->rdmult, x->rddiv, (bits >> 4), err); + if (cost_sgrproj >= cost_norestore) { + type[tile_idx] = RESTORE_NONE; + } else { + type[tile_idx] = RESTORE_SGRPROJ; + memcpy(&sgrproj_info[tile_idx], &rsi->sgrproj_info[tile_idx], + sizeof(sgrproj_info[tile_idx])); + bits = count_sgrproj_bits(&rsi->sgrproj_info[tile_idx], &ref_sgrproj_info) + << AV1_PROB_COST_SHIFT; + memcpy(&ref_sgrproj_info, &sgrproj_info[tile_idx], + sizeof(ref_sgrproj_info)); + best_tile_cost[tile_idx] = err; + } + rsi->restoration_type[tile_idx] = RESTORE_NONE; + } + // Cost for Sgrproj filtering + set_default_sgrproj(&ref_sgrproj_info); + bits = frame_level_restore_bits[rsi->frame_restoration_type] + << AV1_PROB_COST_SHIFT; + for (tile_idx = 0; tile_idx < ntiles; ++tile_idx) { + bits += + av1_cost_bit(RESTORE_NONE_SGRPROJ_PROB, type[tile_idx] != RESTORE_NONE); + memcpy(&rsi->sgrproj_info[tile_idx], &sgrproj_info[tile_idx], + sizeof(sgrproj_info[tile_idx])); + if (type[tile_idx] == RESTORE_SGRPROJ) { + bits += + count_sgrproj_bits(&rsi->sgrproj_info[tile_idx], &ref_sgrproj_info) + << AV1_PROB_COST_SHIFT; + memcpy(&ref_sgrproj_info, &rsi->sgrproj_info[tile_idx], + sizeof(ref_sgrproj_info)); + } + rsi->restoration_type[tile_idx] = type[tile_idx]; + } + err = try_restoration_frame(src, cpi, rsi, 1, partial_frame, dst_frame); + cost_sgrproj = RDCOST_DBL(x->rdmult, x->rddiv, (bits >> 4), err); + + return cost_sgrproj; +} + +static double find_average(uint8_t *src, int h_start, int h_end, int v_start, + int v_end, int stride) { + uint64_t sum = 0; + double avg = 0; + int i, j; + aom_clear_system_state(); + for (i = v_start; i < v_end; i++) + for (j = h_start; j < h_end; j++) sum += src[i * stride + j]; + avg = (double)sum / ((v_end - v_start) * (h_end - h_start)); + return avg; +} + +static void compute_stats(uint8_t *dgd, uint8_t *src, int h_start, int h_end, + int v_start, int v_end, int dgd_stride, + int src_stride, double *M, double *H) { + int i, j, k, l; + double Y[WIENER_WIN2]; + const double avg = + find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); + + memset(M, 0, sizeof(*M) * WIENER_WIN2); + memset(H, 0, sizeof(*H) * WIENER_WIN2 * WIENER_WIN2); + for (i = v_start; i < v_end; i++) { + for (j = h_start; j < h_end; j++) { + const double X = (double)src[i * src_stride + j] - avg; + int idx = 0; + for (k = -WIENER_HALFWIN; k <= WIENER_HALFWIN; k++) { + for (l = -WIENER_HALFWIN; l <= WIENER_HALFWIN; l++) { + Y[idx] = (double)dgd[(i + l) * dgd_stride + (j + k)] - avg; + idx++; + } + } + for (k = 0; k < WIENER_WIN2; ++k) { + M[k] += Y[k] * X; + H[k * WIENER_WIN2 + k] += Y[k] * Y[k]; + for (l = k + 1; l < WIENER_WIN2; ++l) { + // H is a symmetric matrix, so we only need to fill out the upper + // triangle here. We can copy it down to the lower triangle outside + // the (i, j) loops. + H[k * WIENER_WIN2 + l] += Y[k] * Y[l]; + } + } + } + } + for (k = 0; k < WIENER_WIN2; ++k) { + for (l = k + 1; l < WIENER_WIN2; ++l) { + H[l * WIENER_WIN2 + k] = H[k * WIENER_WIN2 + l]; + } + } +} + +#if CONFIG_HIGHBITDEPTH +static double find_average_highbd(uint16_t *src, int h_start, int h_end, + int v_start, int v_end, int stride) { + uint64_t sum = 0; + double avg = 0; + int i, j; + aom_clear_system_state(); + for (i = v_start; i < v_end; i++) + for (j = h_start; j < h_end; j++) sum += src[i * stride + j]; + avg = (double)sum / ((v_end - v_start) * (h_end - h_start)); + return avg; +} + +static void compute_stats_highbd(uint8_t *dgd8, uint8_t *src8, int h_start, + int h_end, int v_start, int v_end, + int dgd_stride, int src_stride, double *M, + double *H) { + int i, j, k, l; + double Y[WIENER_WIN2]; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + uint16_t *dgd = CONVERT_TO_SHORTPTR(dgd8); + const double avg = + find_average_highbd(dgd, h_start, h_end, v_start, v_end, dgd_stride); + + memset(M, 0, sizeof(*M) * WIENER_WIN2); + memset(H, 0, sizeof(*H) * WIENER_WIN2 * WIENER_WIN2); + for (i = v_start; i < v_end; i++) { + for (j = h_start; j < h_end; j++) { + const double X = (double)src[i * src_stride + j] - avg; + int idx = 0; + for (k = -WIENER_HALFWIN; k <= WIENER_HALFWIN; k++) { + for (l = -WIENER_HALFWIN; l <= WIENER_HALFWIN; l++) { + Y[idx] = (double)dgd[(i + l) * dgd_stride + (j + k)] - avg; + idx++; + } + } + for (k = 0; k < WIENER_WIN2; ++k) { + M[k] += Y[k] * X; + H[k * WIENER_WIN2 + k] += Y[k] * Y[k]; + for (l = k + 1; l < WIENER_WIN2; ++l) { + // H is a symmetric matrix, so we only need to fill out the upper + // triangle here. We can copy it down to the lower triangle outside + // the (i, j) loops. + H[k * WIENER_WIN2 + l] += Y[k] * Y[l]; + } + } + } + } + for (k = 0; k < WIENER_WIN2; ++k) { + for (l = k + 1; l < WIENER_WIN2; ++l) { + H[l * WIENER_WIN2 + k] = H[k * WIENER_WIN2 + l]; + } + } +} +#endif // CONFIG_HIGHBITDEPTH + +// Solves Ax = b, where x and b are column vectors +static int linsolve(int n, double *A, int stride, double *b, double *x) { + int i, j, k; + double c; + + aom_clear_system_state(); + + // Forward elimination + for (k = 0; k < n - 1; k++) { + // Bring the largest magitude to the diagonal position + for (i = n - 1; i > k; i--) { + if (fabs(A[(i - 1) * stride + k]) < fabs(A[i * stride + k])) { + for (j = 0; j < n; j++) { + c = A[i * stride + j]; + A[i * stride + j] = A[(i - 1) * stride + j]; + A[(i - 1) * stride + j] = c; + } + c = b[i]; + b[i] = b[i - 1]; + b[i - 1] = c; + } + } + for (i = k; i < n - 1; i++) { + if (fabs(A[k * stride + k]) < 1e-10) return 0; + c = A[(i + 1) * stride + k] / A[k * stride + k]; + for (j = 0; j < n; j++) A[(i + 1) * stride + j] -= c * A[k * stride + j]; + b[i + 1] -= c * b[k]; + } + } + // Backward substitution + for (i = n - 1; i >= 0; i--) { + if (fabs(A[i * stride + i]) < 1e-10) return 0; + c = 0; + for (j = i + 1; j <= n - 1; j++) c += A[i * stride + j] * x[j]; + x[i] = (b[i] - c) / A[i * stride + i]; + } + + return 1; +} + +static INLINE int wrap_index(int i) { + return (i >= WIENER_HALFWIN1 ? WIENER_WIN - 1 - i : i); +} + +// Fix vector b, update vector a +static void update_a_sep_sym(double **Mc, double **Hc, double *a, double *b) { + int i, j; + double S[WIENER_WIN]; + double A[WIENER_HALFWIN1], B[WIENER_HALFWIN1 * WIENER_HALFWIN1]; + int w, w2; + memset(A, 0, sizeof(A)); + memset(B, 0, sizeof(B)); + for (i = 0; i < WIENER_WIN; i++) { + for (j = 0; j < WIENER_WIN; ++j) { + const int jj = wrap_index(j); + A[jj] += Mc[i][j] * b[i]; + } + } + for (i = 0; i < WIENER_WIN; i++) { + for (j = 0; j < WIENER_WIN; j++) { + int k, l; + for (k = 0; k < WIENER_WIN; ++k) + for (l = 0; l < WIENER_WIN; ++l) { + const int kk = wrap_index(k); + const int ll = wrap_index(l); + B[ll * WIENER_HALFWIN1 + kk] += + Hc[j * WIENER_WIN + i][k * WIENER_WIN2 + l] * b[i] * b[j]; + } + } + } + // Normalization enforcement in the system of equations itself + w = WIENER_WIN; + w2 = (w >> 1) + 1; + for (i = 0; i < w2 - 1; ++i) + A[i] -= + A[w2 - 1] * 2 + B[i * w2 + w2 - 1] - 2 * B[(w2 - 1) * w2 + (w2 - 1)]; + for (i = 0; i < w2 - 1; ++i) + for (j = 0; j < w2 - 1; ++j) + B[i * w2 + j] -= 2 * (B[i * w2 + (w2 - 1)] + B[(w2 - 1) * w2 + j] - + 2 * B[(w2 - 1) * w2 + (w2 - 1)]); + if (linsolve(w2 - 1, B, w2, A, S)) { + S[w2 - 1] = 1.0; + for (i = w2; i < w; ++i) { + S[i] = S[w - 1 - i]; + S[w2 - 1] -= 2 * S[i]; + } + memcpy(a, S, w * sizeof(*a)); + } +} + +// Fix vector a, update vector b +static void update_b_sep_sym(double **Mc, double **Hc, double *a, double *b) { + int i, j; + double S[WIENER_WIN]; + double A[WIENER_HALFWIN1], B[WIENER_HALFWIN1 * WIENER_HALFWIN1]; + int w, w2; + memset(A, 0, sizeof(A)); + memset(B, 0, sizeof(B)); + for (i = 0; i < WIENER_WIN; i++) { + const int ii = wrap_index(i); + for (j = 0; j < WIENER_WIN; j++) A[ii] += Mc[i][j] * a[j]; + } + + for (i = 0; i < WIENER_WIN; i++) { + for (j = 0; j < WIENER_WIN; j++) { + const int ii = wrap_index(i); + const int jj = wrap_index(j); + int k, l; + for (k = 0; k < WIENER_WIN; ++k) + for (l = 0; l < WIENER_WIN; ++l) + B[jj * WIENER_HALFWIN1 + ii] += + Hc[i * WIENER_WIN + j][k * WIENER_WIN2 + l] * a[k] * a[l]; + } + } + // Normalization enforcement in the system of equations itself + w = WIENER_WIN; + w2 = WIENER_HALFWIN1; + for (i = 0; i < w2 - 1; ++i) + A[i] -= + A[w2 - 1] * 2 + B[i * w2 + w2 - 1] - 2 * B[(w2 - 1) * w2 + (w2 - 1)]; + for (i = 0; i < w2 - 1; ++i) + for (j = 0; j < w2 - 1; ++j) + B[i * w2 + j] -= 2 * (B[i * w2 + (w2 - 1)] + B[(w2 - 1) * w2 + j] - + 2 * B[(w2 - 1) * w2 + (w2 - 1)]); + if (linsolve(w2 - 1, B, w2, A, S)) { + S[w2 - 1] = 1.0; + for (i = w2; i < w; ++i) { + S[i] = S[w - 1 - i]; + S[w2 - 1] -= 2 * S[i]; + } + memcpy(b, S, w * sizeof(*b)); + } +} + +static int wiener_decompose_sep_sym(double *M, double *H, double *a, + double *b) { + static const double init_filt[WIENER_WIN] = { + 0.035623, -0.127154, 0.211436, 0.760190, 0.211436, -0.127154, 0.035623, + }; + int i, j, iter; + double *Hc[WIENER_WIN2]; + double *Mc[WIENER_WIN]; + for (i = 0; i < WIENER_WIN; i++) { + Mc[i] = M + i * WIENER_WIN; + for (j = 0; j < WIENER_WIN; j++) { + Hc[i * WIENER_WIN + j] = + H + i * WIENER_WIN * WIENER_WIN2 + j * WIENER_WIN; + } + } + memcpy(a, init_filt, sizeof(*a) * WIENER_WIN); + memcpy(b, init_filt, sizeof(*b) * WIENER_WIN); + + iter = 1; + while (iter < NUM_WIENER_ITERS) { + update_a_sep_sym(Mc, Hc, a, b); + update_b_sep_sym(Mc, Hc, a, b); + iter++; + } + return 1; +} + +// Computes the function x'*H*x - x'*M for the learned 2D filter x, and compares +// against identity filters; Final score is defined as the difference between +// the function values +static double compute_score(double *M, double *H, InterpKernel vfilt, + InterpKernel hfilt) { + double ab[WIENER_WIN * WIENER_WIN]; + int i, k, l; + double P = 0, Q = 0; + double iP = 0, iQ = 0; + double Score, iScore; + double a[WIENER_WIN], b[WIENER_WIN]; + + aom_clear_system_state(); + + a[WIENER_HALFWIN] = b[WIENER_HALFWIN] = 1.0; + for (i = 0; i < WIENER_HALFWIN; ++i) { + a[i] = a[WIENER_WIN - i - 1] = (double)vfilt[i] / WIENER_FILT_STEP; + b[i] = b[WIENER_WIN - i - 1] = (double)hfilt[i] / WIENER_FILT_STEP; + a[WIENER_HALFWIN] -= 2 * a[i]; + b[WIENER_HALFWIN] -= 2 * b[i]; + } + for (k = 0; k < WIENER_WIN; ++k) { + for (l = 0; l < WIENER_WIN; ++l) ab[k * WIENER_WIN + l] = a[l] * b[k]; + } + for (k = 0; k < WIENER_WIN2; ++k) { + P += ab[k] * M[k]; + for (l = 0; l < WIENER_WIN2; ++l) + Q += ab[k] * H[k * WIENER_WIN2 + l] * ab[l]; + } + Score = Q - 2 * P; + + iP = M[WIENER_WIN2 >> 1]; + iQ = H[(WIENER_WIN2 >> 1) * WIENER_WIN2 + (WIENER_WIN2 >> 1)]; + iScore = iQ - 2 * iP; + + return Score - iScore; +} + +static void quantize_sym_filter(double *f, InterpKernel fi) { + int i; + for (i = 0; i < WIENER_HALFWIN; ++i) { + fi[i] = RINT(f[i] * WIENER_FILT_STEP); + } + // Specialize for 7-tap filter + fi[0] = CLIP(fi[0], WIENER_FILT_TAP0_MINV, WIENER_FILT_TAP0_MAXV); + fi[1] = CLIP(fi[1], WIENER_FILT_TAP1_MINV, WIENER_FILT_TAP1_MAXV); + fi[2] = CLIP(fi[2], WIENER_FILT_TAP2_MINV, WIENER_FILT_TAP2_MAXV); + // Satisfy filter constraints + fi[WIENER_WIN - 1] = fi[0]; + fi[WIENER_WIN - 2] = fi[1]; + fi[WIENER_WIN - 3] = fi[2]; + // The central element has an implicit +WIENER_FILT_STEP + fi[3] = -2 * (fi[0] + fi[1] + fi[2]); +} + +static int count_wiener_bits(WienerInfo *wiener_info, + WienerInfo *ref_wiener_info) { + int bits = 0; + bits += aom_count_primitive_refsubexpfin( + WIENER_FILT_TAP0_MAXV - WIENER_FILT_TAP0_MINV + 1, + WIENER_FILT_TAP0_SUBEXP_K, + ref_wiener_info->vfilter[0] - WIENER_FILT_TAP0_MINV, + wiener_info->vfilter[0] - WIENER_FILT_TAP0_MINV); + bits += aom_count_primitive_refsubexpfin( + WIENER_FILT_TAP1_MAXV - WIENER_FILT_TAP1_MINV + 1, + WIENER_FILT_TAP1_SUBEXP_K, + ref_wiener_info->vfilter[1] - WIENER_FILT_TAP1_MINV, + wiener_info->vfilter[1] - WIENER_FILT_TAP1_MINV); + bits += aom_count_primitive_refsubexpfin( + WIENER_FILT_TAP2_MAXV - WIENER_FILT_TAP2_MINV + 1, + WIENER_FILT_TAP2_SUBEXP_K, + ref_wiener_info->vfilter[2] - WIENER_FILT_TAP2_MINV, + wiener_info->vfilter[2] - WIENER_FILT_TAP2_MINV); + bits += aom_count_primitive_refsubexpfin( + WIENER_FILT_TAP0_MAXV - WIENER_FILT_TAP0_MINV + 1, + WIENER_FILT_TAP0_SUBEXP_K, + ref_wiener_info->hfilter[0] - WIENER_FILT_TAP0_MINV, + wiener_info->hfilter[0] - WIENER_FILT_TAP0_MINV); + bits += aom_count_primitive_refsubexpfin( + WIENER_FILT_TAP1_MAXV - WIENER_FILT_TAP1_MINV + 1, + WIENER_FILT_TAP1_SUBEXP_K, + ref_wiener_info->hfilter[1] - WIENER_FILT_TAP1_MINV, + wiener_info->hfilter[1] - WIENER_FILT_TAP1_MINV); + bits += aom_count_primitive_refsubexpfin( + WIENER_FILT_TAP2_MAXV - WIENER_FILT_TAP2_MINV + 1, + WIENER_FILT_TAP2_SUBEXP_K, + ref_wiener_info->hfilter[2] - WIENER_FILT_TAP2_MINV, + wiener_info->hfilter[2] - WIENER_FILT_TAP2_MINV); + return bits; +} + +static double search_wiener_uv(const YV12_BUFFER_CONFIG *src, AV1_COMP *cpi, + int partial_frame, int plane, + RestorationInfo *info, RestorationType *type, + YV12_BUFFER_CONFIG *dst_frame) { + WienerInfo *wiener_info = info->wiener_info; + AV1_COMMON *const cm = &cpi->common; + RestorationInfo *rsi = cpi->rst_search; + int64_t err; + int bits; + double cost_wiener, cost_norestore, cost_wiener_frame, cost_norestore_frame; + MACROBLOCK *x = &cpi->td.mb; + double M[WIENER_WIN2]; + double H[WIENER_WIN2 * WIENER_WIN2]; + double vfilterd[WIENER_WIN], hfilterd[WIENER_WIN]; + const YV12_BUFFER_CONFIG *dgd = cm->frame_to_show; + const int width = src->uv_crop_width; + const int height = src->uv_crop_height; + const int src_stride = src->uv_stride; + const int dgd_stride = dgd->uv_stride; + double score; + int tile_idx, tile_width, tile_height, nhtiles, nvtiles; + int h_start, h_end, v_start, v_end; + const int ntiles = + av1_get_rest_ntiles(width, height, cm->rst_info[1].restoration_tilesize, + &tile_width, &tile_height, &nhtiles, &nvtiles); + WienerInfo ref_wiener_info; + set_default_wiener(&ref_wiener_info); + assert(width == dgd->uv_crop_width); + assert(height == dgd->uv_crop_height); + + rsi[plane].frame_restoration_type = RESTORE_NONE; + err = sse_restoration_frame(cm, src, cm->frame_to_show, (1 << plane)); + bits = 0; + cost_norestore_frame = RDCOST_DBL(x->rdmult, x->rddiv, (bits >> 4), err); + + rsi[plane].frame_restoration_type = RESTORE_WIENER; + + for (tile_idx = 0; tile_idx < ntiles; ++tile_idx) { + rsi[plane].restoration_type[tile_idx] = RESTORE_NONE; + } + + // Compute best Wiener filters for each tile + for (tile_idx = 0; tile_idx < ntiles; ++tile_idx) { + av1_get_rest_tile_limits(tile_idx, 0, 0, nhtiles, nvtiles, tile_width, + tile_height, width, height, 0, 0, &h_start, &h_end, + &v_start, &v_end); + err = sse_restoration_tile(src, cm->frame_to_show, cm, h_start, + h_end - h_start, v_start, v_end - v_start, + 1 << plane); + // #bits when a tile is not restored + bits = av1_cost_bit(RESTORE_NONE_WIENER_PROB, 0); + cost_norestore = RDCOST_DBL(x->rdmult, x->rddiv, (bits >> 4), err); + // best_tile_cost[tile_idx] = DBL_MAX; + + av1_get_rest_tile_limits(tile_idx, 0, 0, nhtiles, nvtiles, tile_width, + tile_height, width, height, WIENER_HALFWIN, + WIENER_HALFWIN, &h_start, &h_end, &v_start, + &v_end); + if (plane == AOM_PLANE_U) { +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + compute_stats_highbd(dgd->u_buffer, src->u_buffer, h_start, h_end, + v_start, v_end, dgd_stride, src_stride, M, H); + else +#endif // CONFIG_HIGHBITDEPTH + compute_stats(dgd->u_buffer, src->u_buffer, h_start, h_end, v_start, + v_end, dgd_stride, src_stride, M, H); + } else if (plane == AOM_PLANE_V) { +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + compute_stats_highbd(dgd->v_buffer, src->v_buffer, h_start, h_end, + v_start, v_end, dgd_stride, src_stride, M, H); + else +#endif // CONFIG_HIGHBITDEPTH + compute_stats(dgd->v_buffer, src->v_buffer, h_start, h_end, v_start, + v_end, dgd_stride, src_stride, M, H); + } else { + assert(0); + } + + type[tile_idx] = RESTORE_WIENER; + + if (!wiener_decompose_sep_sym(M, H, vfilterd, hfilterd)) { + type[tile_idx] = RESTORE_NONE; + continue; + } + quantize_sym_filter(vfilterd, rsi[plane].wiener_info[tile_idx].vfilter); + quantize_sym_filter(hfilterd, rsi[plane].wiener_info[tile_idx].hfilter); + + // Filter score computes the value of the function x'*A*x - x'*b for the + // learned filter and compares it against identity filer. If there is no + // reduction in the function, the filter is reverted back to identity + score = compute_score(M, H, rsi[plane].wiener_info[tile_idx].vfilter, + rsi[plane].wiener_info[tile_idx].hfilter); + if (score > 0.0) { + type[tile_idx] = RESTORE_NONE; + continue; + } + + rsi[plane].restoration_type[tile_idx] = RESTORE_WIENER; + err = try_restoration_tile(src, cpi, rsi, 1 << plane, partial_frame, + tile_idx, 0, 0, dst_frame); + bits = + count_wiener_bits(&rsi[plane].wiener_info[tile_idx], &ref_wiener_info) + << AV1_PROB_COST_SHIFT; + // bits = WIENER_FILT_BITS << AV1_PROB_COST_SHIFT; + bits += av1_cost_bit(RESTORE_NONE_WIENER_PROB, 1); + cost_wiener = RDCOST_DBL(x->rdmult, x->rddiv, (bits >> 4), err); + if (cost_wiener >= cost_norestore) { + type[tile_idx] = RESTORE_NONE; + } else { + type[tile_idx] = RESTORE_WIENER; + memcpy(&wiener_info[tile_idx], &rsi[plane].wiener_info[tile_idx], + sizeof(wiener_info[tile_idx])); + memcpy(&ref_wiener_info, &rsi[plane].wiener_info[tile_idx], + sizeof(ref_wiener_info)); + } + rsi[plane].restoration_type[tile_idx] = RESTORE_NONE; + } + // Cost for Wiener filtering + set_default_wiener(&ref_wiener_info); + bits = 0; + for (tile_idx = 0; tile_idx < ntiles; ++tile_idx) { + bits += + av1_cost_bit(RESTORE_NONE_WIENER_PROB, type[tile_idx] != RESTORE_NONE); + memcpy(&rsi[plane].wiener_info[tile_idx], &wiener_info[tile_idx], + sizeof(wiener_info[tile_idx])); + if (type[tile_idx] == RESTORE_WIENER) { + bits += + count_wiener_bits(&rsi[plane].wiener_info[tile_idx], &ref_wiener_info) + << AV1_PROB_COST_SHIFT; + memcpy(&ref_wiener_info, &rsi[plane].wiener_info[tile_idx], + sizeof(ref_wiener_info)); + } + rsi[plane].restoration_type[tile_idx] = type[tile_idx]; + } + err = try_restoration_frame(src, cpi, rsi, 1 << plane, partial_frame, + dst_frame); + cost_wiener_frame = RDCOST_DBL(x->rdmult, x->rddiv, (bits >> 4), err); + + if (cost_wiener_frame < cost_norestore_frame) { + info->frame_restoration_type = RESTORE_WIENER; + } else { + info->frame_restoration_type = RESTORE_NONE; + } + + return info->frame_restoration_type == RESTORE_WIENER ? cost_wiener_frame + : cost_norestore_frame; +} + +static double search_wiener(const YV12_BUFFER_CONFIG *src, AV1_COMP *cpi, + int partial_frame, RestorationInfo *info, + RestorationType *type, double *best_tile_cost, + YV12_BUFFER_CONFIG *dst_frame) { + WienerInfo *wiener_info = info->wiener_info; + AV1_COMMON *const cm = &cpi->common; + RestorationInfo *rsi = cpi->rst_search; + int64_t err; + int bits; + double cost_wiener, cost_norestore; + MACROBLOCK *x = &cpi->td.mb; + double M[WIENER_WIN2]; + double H[WIENER_WIN2 * WIENER_WIN2]; + double vfilterd[WIENER_WIN], hfilterd[WIENER_WIN]; + const YV12_BUFFER_CONFIG *dgd = cm->frame_to_show; + const int width = cm->width; + const int height = cm->height; + const int src_stride = src->y_stride; + const int dgd_stride = dgd->y_stride; + double score; + int tile_idx, tile_width, tile_height, nhtiles, nvtiles; + int h_start, h_end, v_start, v_end; + const int ntiles = + av1_get_rest_ntiles(width, height, cm->rst_info[0].restoration_tilesize, + &tile_width, &tile_height, &nhtiles, &nvtiles); + WienerInfo ref_wiener_info; + set_default_wiener(&ref_wiener_info); + + assert(width == dgd->y_crop_width); + assert(height == dgd->y_crop_height); + assert(width == src->y_crop_width); + assert(height == src->y_crop_height); + + rsi->frame_restoration_type = RESTORE_WIENER; + + for (tile_idx = 0; tile_idx < ntiles; ++tile_idx) { + rsi->restoration_type[tile_idx] = RESTORE_NONE; + } + +// Construct a (WIENER_HALFWIN)-pixel border around the frame +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + extend_frame_highbd(CONVERT_TO_SHORTPTR(dgd->y_buffer), width, height, + dgd_stride); + else +#endif + extend_frame(dgd->y_buffer, width, height, dgd_stride); + + // Compute best Wiener filters for each tile + for (tile_idx = 0; tile_idx < ntiles; ++tile_idx) { + av1_get_rest_tile_limits(tile_idx, 0, 0, nhtiles, nvtiles, tile_width, + tile_height, width, height, 0, 0, &h_start, &h_end, + &v_start, &v_end); + err = sse_restoration_tile(src, cm->frame_to_show, cm, h_start, + h_end - h_start, v_start, v_end - v_start, 1); + // #bits when a tile is not restored + bits = av1_cost_bit(RESTORE_NONE_WIENER_PROB, 0); + cost_norestore = RDCOST_DBL(x->rdmult, x->rddiv, (bits >> 4), err); + best_tile_cost[tile_idx] = DBL_MAX; + + av1_get_rest_tile_limits(tile_idx, 0, 0, nhtiles, nvtiles, tile_width, + tile_height, width, height, 0, 0, &h_start, &h_end, + &v_start, &v_end); +#if CONFIG_HIGHBITDEPTH + if (cm->use_highbitdepth) + compute_stats_highbd(dgd->y_buffer, src->y_buffer, h_start, h_end, + v_start, v_end, dgd_stride, src_stride, M, H); + else +#endif // CONFIG_HIGHBITDEPTH + compute_stats(dgd->y_buffer, src->y_buffer, h_start, h_end, v_start, + v_end, dgd_stride, src_stride, M, H); + + type[tile_idx] = RESTORE_WIENER; + + if (!wiener_decompose_sep_sym(M, H, vfilterd, hfilterd)) { + type[tile_idx] = RESTORE_NONE; + continue; + } + quantize_sym_filter(vfilterd, rsi->wiener_info[tile_idx].vfilter); + quantize_sym_filter(hfilterd, rsi->wiener_info[tile_idx].hfilter); + + // Filter score computes the value of the function x'*A*x - x'*b for the + // learned filter and compares it against identity filer. If there is no + // reduction in the function, the filter is reverted back to identity + score = compute_score(M, H, rsi->wiener_info[tile_idx].vfilter, + rsi->wiener_info[tile_idx].hfilter); + if (score > 0.0) { + type[tile_idx] = RESTORE_NONE; + continue; + } + + rsi->restoration_type[tile_idx] = RESTORE_WIENER; + err = try_restoration_tile(src, cpi, rsi, 1, partial_frame, tile_idx, 0, 0, + dst_frame); + bits = count_wiener_bits(&rsi->wiener_info[tile_idx], &ref_wiener_info) + << AV1_PROB_COST_SHIFT; + bits += av1_cost_bit(RESTORE_NONE_WIENER_PROB, 1); + cost_wiener = RDCOST_DBL(x->rdmult, x->rddiv, (bits >> 4), err); + if (cost_wiener >= cost_norestore) { + type[tile_idx] = RESTORE_NONE; + } else { + type[tile_idx] = RESTORE_WIENER; + memcpy(&wiener_info[tile_idx], &rsi->wiener_info[tile_idx], + sizeof(wiener_info[tile_idx])); + memcpy(&ref_wiener_info, &rsi->wiener_info[tile_idx], + sizeof(ref_wiener_info)); + bits = count_wiener_bits(&wiener_info[tile_idx], &ref_wiener_info) + << AV1_PROB_COST_SHIFT; + best_tile_cost[tile_idx] = err; + } + rsi->restoration_type[tile_idx] = RESTORE_NONE; + } + // Cost for Wiener filtering + set_default_wiener(&ref_wiener_info); + bits = frame_level_restore_bits[rsi->frame_restoration_type] + << AV1_PROB_COST_SHIFT; + for (tile_idx = 0; tile_idx < ntiles; ++tile_idx) { + bits += + av1_cost_bit(RESTORE_NONE_WIENER_PROB, type[tile_idx] != RESTORE_NONE); + memcpy(&rsi->wiener_info[tile_idx], &wiener_info[tile_idx], + sizeof(wiener_info[tile_idx])); + if (type[tile_idx] == RESTORE_WIENER) { + bits += count_wiener_bits(&rsi->wiener_info[tile_idx], &ref_wiener_info) + << AV1_PROB_COST_SHIFT; + memcpy(&ref_wiener_info, &rsi->wiener_info[tile_idx], + sizeof(ref_wiener_info)); + } + rsi->restoration_type[tile_idx] = type[tile_idx]; + } + err = try_restoration_frame(src, cpi, rsi, 1, partial_frame, dst_frame); + cost_wiener = RDCOST_DBL(x->rdmult, x->rddiv, (bits >> 4), err); + + return cost_wiener; +} + +static double search_norestore(const YV12_BUFFER_CONFIG *src, AV1_COMP *cpi, + int partial_frame, RestorationInfo *info, + RestorationType *type, double *best_tile_cost, + YV12_BUFFER_CONFIG *dst_frame) { + double err, cost_norestore; + int bits; + MACROBLOCK *x = &cpi->td.mb; + AV1_COMMON *const cm = &cpi->common; + int tile_idx, tile_width, tile_height, nhtiles, nvtiles; + int h_start, h_end, v_start, v_end; + const int ntiles = av1_get_rest_ntiles( + cm->width, cm->height, cm->rst_info[0].restoration_tilesize, &tile_width, + &tile_height, &nhtiles, &nvtiles); + (void)info; + (void)dst_frame; + (void)partial_frame; + + for (tile_idx = 0; tile_idx < ntiles; ++tile_idx) { + av1_get_rest_tile_limits(tile_idx, 0, 0, nhtiles, nvtiles, tile_width, + tile_height, cm->width, cm->height, 0, 0, &h_start, + &h_end, &v_start, &v_end); + err = sse_restoration_tile(src, cm->frame_to_show, cm, h_start, + h_end - h_start, v_start, v_end - v_start, 1); + type[tile_idx] = RESTORE_NONE; + best_tile_cost[tile_idx] = err; + } + // RD cost associated with no restoration + err = sse_restoration_tile(src, cm->frame_to_show, cm, 0, cm->width, 0, + cm->height, 1); + bits = frame_level_restore_bits[RESTORE_NONE] << AV1_PROB_COST_SHIFT; + cost_norestore = RDCOST_DBL(x->rdmult, x->rddiv, (bits >> 4), err); + return cost_norestore; +} + +static double search_switchable_restoration( + AV1_COMP *cpi, int partial_frame, RestorationInfo *rsi, + double *tile_cost[RESTORE_SWITCHABLE_TYPES]) { + AV1_COMMON *const cm = &cpi->common; + MACROBLOCK *x = &cpi->td.mb; + double cost_switchable = 0; + int bits, tile_idx; + RestorationType r; + const int ntiles = av1_get_rest_ntiles(cm->width, cm->height, + cm->rst_info[0].restoration_tilesize, + NULL, NULL, NULL, NULL); + SgrprojInfo ref_sgrproj_info; + set_default_sgrproj(&ref_sgrproj_info); + WienerInfo ref_wiener_info; + set_default_wiener(&ref_wiener_info); + (void)partial_frame; + + rsi->frame_restoration_type = RESTORE_SWITCHABLE; + bits = frame_level_restore_bits[rsi->frame_restoration_type] + << AV1_PROB_COST_SHIFT; + cost_switchable = RDCOST_DBL(x->rdmult, x->rddiv, bits >> 4, 0); + for (tile_idx = 0; tile_idx < ntiles; ++tile_idx) { + double best_cost = RDCOST_DBL( + x->rdmult, x->rddiv, (cpi->switchable_restore_cost[RESTORE_NONE] >> 4), + tile_cost[RESTORE_NONE][tile_idx]); + rsi->restoration_type[tile_idx] = RESTORE_NONE; + for (r = 1; r < RESTORE_SWITCHABLE_TYPES; r++) { + if (force_restore_type != 0) + if (r != force_restore_type) continue; + int tilebits = 0; + if (r == RESTORE_WIENER) + tilebits += + count_wiener_bits(&rsi->wiener_info[tile_idx], &ref_wiener_info); + else if (r == RESTORE_SGRPROJ) + tilebits += + count_sgrproj_bits(&rsi->sgrproj_info[tile_idx], &ref_sgrproj_info); + tilebits <<= AV1_PROB_COST_SHIFT; + tilebits += cpi->switchable_restore_cost[r]; + double cost = RDCOST_DBL(x->rdmult, x->rddiv, tilebits >> 4, + tile_cost[r][tile_idx]); + + if (cost < best_cost) { + rsi->restoration_type[tile_idx] = r; + best_cost = cost; + } + } + if (rsi->restoration_type[tile_idx] == RESTORE_WIENER) + memcpy(&ref_wiener_info, &rsi->wiener_info[tile_idx], + sizeof(ref_wiener_info)); + else if (rsi->restoration_type[tile_idx] == RESTORE_SGRPROJ) + memcpy(&ref_sgrproj_info, &rsi->sgrproj_info[tile_idx], + sizeof(ref_sgrproj_info)); + if (force_restore_type != 0) + assert(rsi->restoration_type[tile_idx] == force_restore_type || + rsi->restoration_type[tile_idx] == RESTORE_NONE); + cost_switchable += best_cost; + } + return cost_switchable; +} + +void av1_pick_filter_restoration(const YV12_BUFFER_CONFIG *src, AV1_COMP *cpi, + LPF_PICK_METHOD method) { + static search_restore_type search_restore_fun[RESTORE_SWITCHABLE_TYPES] = { + search_norestore, search_wiener, search_sgrproj, + }; + AV1_COMMON *const cm = &cpi->common; + double cost_restore[RESTORE_TYPES]; + double *tile_cost[RESTORE_SWITCHABLE_TYPES]; + RestorationType *restore_types[RESTORE_SWITCHABLE_TYPES]; + double best_cost_restore; + RestorationType r, best_restore; + + const int ntiles = av1_get_rest_ntiles(cm->width, cm->height, + cm->rst_info[0].restoration_tilesize, + NULL, NULL, NULL, NULL); + + for (r = 0; r < RESTORE_SWITCHABLE_TYPES; r++) { + tile_cost[r] = (double *)aom_malloc(sizeof(*tile_cost[0]) * ntiles); + restore_types[r] = + (RestorationType *)aom_malloc(sizeof(*restore_types[0]) * ntiles); + } + + for (r = 0; r < RESTORE_SWITCHABLE_TYPES; ++r) { + if (force_restore_type != 0) + if (r != RESTORE_NONE && r != force_restore_type) continue; + cost_restore[r] = search_restore_fun[r]( + src, cpi, method == LPF_PICK_FROM_SUBIMAGE, &cm->rst_info[0], + restore_types[r], tile_cost[r], &cpi->trial_frame_rst); + } + cost_restore[RESTORE_SWITCHABLE] = search_switchable_restoration( + cpi, method == LPF_PICK_FROM_SUBIMAGE, &cm->rst_info[0], tile_cost); + + best_cost_restore = DBL_MAX; + best_restore = 0; + for (r = 0; r < RESTORE_TYPES; ++r) { + if (force_restore_type != 0) + if (r != RESTORE_NONE && r != force_restore_type) continue; + if (cost_restore[r] < best_cost_restore) { + best_restore = r; + best_cost_restore = cost_restore[r]; + } + } + cm->rst_info[0].frame_restoration_type = best_restore; + if (force_restore_type != 0) + assert(best_restore == force_restore_type || best_restore == RESTORE_NONE); + if (best_restore != RESTORE_SWITCHABLE) { + memcpy(cm->rst_info[0].restoration_type, restore_types[best_restore], + ntiles * sizeof(restore_types[best_restore][0])); + } + + // Color components + search_wiener_uv(src, cpi, method == LPF_PICK_FROM_SUBIMAGE, AOM_PLANE_U, + &cm->rst_info[AOM_PLANE_U], + cm->rst_info[AOM_PLANE_U].restoration_type, + &cpi->trial_frame_rst); + search_wiener_uv(src, cpi, method == LPF_PICK_FROM_SUBIMAGE, AOM_PLANE_V, + &cm->rst_info[AOM_PLANE_V], + cm->rst_info[AOM_PLANE_V].restoration_type, + &cpi->trial_frame_rst); + /* + printf("Frame %d/%d restore types: %d %d %d\n", + cm->current_video_frame, cm->show_frame, + cm->rst_info[0].frame_restoration_type, + cm->rst_info[1].frame_restoration_type, + cm->rst_info[2].frame_restoration_type); + printf("Frame %d/%d frame_restore_type %d : %f %f %f %f\n", + cm->current_video_frame, cm->show_frame, + cm->rst_info[0].frame_restoration_type, cost_restore[0], + cost_restore[1], cost_restore[2], cost_restore[3]); + */ + + for (r = 0; r < RESTORE_SWITCHABLE_TYPES; r++) { + aom_free(tile_cost[r]); + aom_free(restore_types[r]); + } +} diff --git a/third_party/aom/av1/encoder/pickrst.h b/third_party/aom/av1/encoder/pickrst.h new file mode 100644 index 0000000000..f6096ed1d1 --- /dev/null +++ b/third_party/aom/av1/encoder/pickrst.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AV1_ENCODER_PICKRST_H_ +#define AV1_ENCODER_PICKRST_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "av1/encoder/encoder.h" + +struct yv12_buffer_config; +struct AV1_COMP; + +void av1_pick_filter_restoration(const YV12_BUFFER_CONFIG *sd, AV1_COMP *cpi, + LPF_PICK_METHOD method); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_PICKRST_H_ diff --git a/third_party/aom/av1/encoder/pvq_encoder.c b/third_party/aom/av1/encoder/pvq_encoder.c new file mode 100644 index 0000000000..ab63f1b7dd --- /dev/null +++ b/third_party/aom/av1/encoder/pvq_encoder.c @@ -0,0 +1,988 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include +#include +#include "aom_dsp/entcode.h" +#include "aom_dsp/entenc.h" +#include "av1/common/blockd.h" +#include "av1/common/odintrin.h" +#include "av1/common/partition.h" +#include "av1/common/pvq_state.h" +#include "av1/encoder/encodemb.h" +#include "av1/encoder/pvq_encoder.h" +#include "aom_ports/system_state.h" + +/*Shift to ensure that the upper bound (i.e. for the max blocksize) of the + dot-product of the 1st band of chroma with the luma ref doesn't overflow.*/ +#define OD_CFL_FLIP_SHIFT (OD_LIMIT_BSIZE_MAX + 0) + +void aom_write_symbol_pvq(aom_writer *w, int symb, aom_cdf_prob *cdf, + int nsymbs) { + if (cdf[0] == 0) + aom_cdf_init_q15_1D(cdf, nsymbs, CDF_SIZE(nsymbs)); + aom_write_symbol(w, symb, cdf, nsymbs); +} + +static void aom_encode_pvq_codeword(aom_writer *w, od_pvq_codeword_ctx *adapt, + const od_coeff *in, int n, int k) { + int i; + aom_encode_band_pvq_splits(w, adapt, in, n, k, 0); + for (i = 0; i < n; i++) if (in[i]) aom_write_bit(w, in[i] < 0); +} + +/* Computes 1/sqrt(i) using a table for small values. */ +static double od_rsqrt_table(int i) { + static double table[16] = { + 1.000000, 0.707107, 0.577350, 0.500000, + 0.447214, 0.408248, 0.377964, 0.353553, + 0.333333, 0.316228, 0.301511, 0.288675, + 0.277350, 0.267261, 0.258199, 0.250000}; + if (i <= 16) return table[i-1]; + else return 1./sqrt(i); +} + +/*Computes 1/sqrt(start+2*i+1) using a lookup table containing the results + where 0 <= i < table_size.*/ +static double od_custom_rsqrt_dynamic_table(const double* table, + const int table_size, const double start, const int i) { + if (i < table_size) return table[i]; + else return od_rsqrt_table((int)(start + 2*i + 1)); +} + +/*Fills tables used in od_custom_rsqrt_dynamic_table for a given start.*/ +static void od_fill_dynamic_rsqrt_table(double *table, const int table_size, + const double start) { + int i; + for (i = 0; i < table_size; i++) + table[i] = od_rsqrt_table((int)(start + 2*i + 1)); +} + +/** Find the codepoint on the given PSphere closest to the desired + * vector. Double-precision PVQ search just to make sure our tests + * aren't limited by numerical accuracy. + * + * @param [in] xcoeff input vector to quantize (x in the math doc) + * @param [in] n number of dimensions + * @param [in] k number of pulses + * @param [out] ypulse optimal codevector found (y in the math doc) + * @param [out] g2 multiplier for the distortion (typically squared + * gain units) + * @param [in] pvq_norm_lambda enc->pvq_norm_lambda for quantized RDO + * @param [in] prev_k number of pulses already in ypulse that we should + * reuse for the search (or 0 for a new search) + * @return cosine distance between x and y (between 0 and 1) + */ +double pvq_search_rdo_double_c(const od_val16 *xcoeff, int n, int k, + od_coeff *ypulse, double g2, double pvq_norm_lambda, int prev_k) { + int i, j; + double xy; + double yy; + /* TODO - This blows our 8kB stack space budget and should be fixed when + converting PVQ to fixed point. */ + double x[MAXN]; + double xx; + double lambda; + double norm_1; + int rdo_pulses; + double delta_rate; + xx = xy = yy = 0; + for (j = 0; j < n; j++) { + x[j] = fabs((float)xcoeff[j]); + xx += x[j]*x[j]; + } + norm_1 = 1./sqrt(1e-30 + xx); + lambda = pvq_norm_lambda/(1e-30 + g2); + i = 0; + if (prev_k > 0 && prev_k <= k) { + /* We reuse pulses from a previous search so we don't have to search them + again. */ + for (j = 0; j < n; j++) { + ypulse[j] = abs(ypulse[j]); + xy += x[j]*ypulse[j]; + yy += ypulse[j]*ypulse[j]; + i += ypulse[j]; + } + } + else if (k > 2) { + double l1_norm; + double l1_inv; + l1_norm = 0; + for (j = 0; j < n; j++) l1_norm += x[j]; + l1_inv = 1./OD_MAXF(l1_norm, 1e-100); + for (j = 0; j < n; j++) { + double tmp; + tmp = k*x[j]*l1_inv; + ypulse[j] = OD_MAXI(0, (int)floor(tmp)); + xy += x[j]*ypulse[j]; + yy += ypulse[j]*ypulse[j]; + i += ypulse[j]; + } + } + else OD_CLEAR(ypulse, n); + + /* Only use RDO on the last few pulses. This not only saves CPU, but using + RDO on all pulses actually makes the results worse for reasons I don't + fully understand. */ + rdo_pulses = 1 + k/4; + /* Rough assumption for now, the last position costs about 3 bits more than + the first. */ + delta_rate = 3./n; + /* Search one pulse at a time */ + for (; i < k - rdo_pulses; i++) { + int pos; + double best_xy; + double best_yy; + pos = 0; + best_xy = -10; + best_yy = 1; + for (j = 0; j < n; j++) { + double tmp_xy; + double tmp_yy; + tmp_xy = xy + x[j]; + tmp_yy = yy + 2*ypulse[j] + 1; + tmp_xy *= tmp_xy; + if (j == 0 || tmp_xy*best_yy > best_xy*tmp_yy) { + best_xy = tmp_xy; + best_yy = tmp_yy; + pos = j; + } + } + xy = xy + x[pos]; + yy = yy + 2*ypulse[pos] + 1; + ypulse[pos]++; + } + /* Search last pulses with RDO. Distortion is D = (x-y)^2 = x^2 - 2*x*y + y^2 + and since x^2 and y^2 are constant, we just maximize x*y, plus a + lambda*rate term. Note that since x and y aren't normalized here, + we need to divide by sqrt(x^2)*sqrt(y^2). */ + for (; i < k; i++) { + double rsqrt_table[4]; + int rsqrt_table_size = 4; + int pos; + double best_cost; + pos = 0; + best_cost = -1e5; + /*Fill the small rsqrt lookup table with inputs relative to yy. + Specifically, the table of n values is filled with + rsqrt(yy + 1), rsqrt(yy + 2 + 1) .. rsqrt(yy + 2*(n-1) + 1).*/ + od_fill_dynamic_rsqrt_table(rsqrt_table, rsqrt_table_size, yy); + for (j = 0; j < n; j++) { + double tmp_xy; + double tmp_yy; + tmp_xy = xy + x[j]; + /*Calculate rsqrt(yy + 2*ypulse[j] + 1) using an optimized method.*/ + tmp_yy = od_custom_rsqrt_dynamic_table(rsqrt_table, rsqrt_table_size, + yy, ypulse[j]); + tmp_xy = 2*tmp_xy*norm_1*tmp_yy - lambda*j*delta_rate; + if (j == 0 || tmp_xy > best_cost) { + best_cost = tmp_xy; + pos = j; + } + } + xy = xy + x[pos]; + yy = yy + 2*ypulse[pos] + 1; + ypulse[pos]++; + } + for (i = 0; i < n; i++) { + if (xcoeff[i] < 0) ypulse[i] = -ypulse[i]; + } + return xy/(1e-100 + sqrt(xx*yy)); +} + +/** Encodes the gain so that the return value increases with the + * distance |x-ref|, so that we can encode a zero when x=ref. The + * value x=0 is not covered because it is only allowed in the noref + * case. + * + * @param [in] x quantized gain to encode + * @param [in] ref quantized gain of the reference + * @return interleave-encoded quantized gain value + */ +static int neg_interleave(int x, int ref) { + if (x < ref) return -2*(x - ref) - 1; + else if (x < 2*ref) return 2*(x - ref); + else return x-1; +} + +int od_vector_is_null(const od_coeff *x, int len) { + int i; + for (i = 0; i < len; i++) if (x[i]) return 0; + return 1; +} + +static double od_pvq_rate(int qg, int icgr, int theta, int ts, + const od_adapt_ctx *adapt, const od_coeff *y0, int k, int n, int speed) { + double rate; + if (k == 0) rate = 0; + else if (speed > 0) { + int i; + int sum; + double f; + /* Compute "center of mass" of the pulse vector. */ + sum = 0; + for (i = 0; i < n - (theta != -1); i++) sum += i*abs(y0[i]); + f = sum/(double)(k*n); + /* Estimates the number of bits it will cost to encode K pulses in + N dimensions based on hand-tuned fit for bitrate vs K, N and + "center of mass". */ + rate = (1 + .4*f)*n*OD_LOG2(1 + OD_MAXF(0, log(n*2*(1*f + .025))*k/n)) + 3; + } + else { + aom_writer w; + od_pvq_codeword_ctx cd; + int tell; +#if CONFIG_DAALA_EC + od_ec_enc_init(&w.ec, 1000); +#else +# error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + OD_COPY(&cd, &adapt->pvq.pvq_codeword_ctx, 1); +#if CONFIG_DAALA_EC + tell = od_ec_enc_tell_frac(&w.ec); +#else +# error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + aom_encode_pvq_codeword(&w, &cd, y0, n - (theta != -1), k); +#if CONFIG_DAALA_EC + rate = (od_ec_enc_tell_frac(&w.ec)-tell)/8.; + od_ec_enc_clear(&w.ec); +#else +# error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + } + if (qg > 0 && theta >= 0) { + /* Approximate cost of entropy-coding theta */ + rate += .9*OD_LOG2(ts); + if (qg == icgr) rate -= .5; + } + return rate; +} + +#define MAX_PVQ_ITEMS (20) +/* This stores the information about a PVQ search candidate, so we can sort + based on K. */ +typedef struct { + int gain; + int k; + od_val32 qtheta; + int theta; + int ts; + od_val32 qcg; +} pvq_search_item; + +int items_compare(pvq_search_item *a, pvq_search_item *b) { + /* Break ties in K with gain to ensure a stable sort. + Otherwise, the order depends on qsort implementation. */ + return a->k == b->k ? a->gain - b->gain : a->k - b->k; +} + +/** Perform PVQ quantization with prediction, trying several + * possible gains and angles. See draft-valin-videocodec-pvq and + * http://jmvalin.ca/slides/pvq.pdf for more details. + * + * @param [out] out coefficients after quantization + * @param [in] x0 coefficients before quantization + * @param [in] r0 reference, aka predicted coefficients + * @param [in] n number of dimensions + * @param [in] q0 quantization step size + * @param [out] y pulse vector (i.e. selected PVQ codevector) + * @param [out] itheta angle between input and reference (-1 if noref) + * @param [out] vk total number of pulses + * @param [in] beta per-band activity masking beta param + * @param [out] skip_diff distortion cost of skipping this block + * (accumulated) + * @param [in] is_keyframe whether we're encoding a keyframe + * @param [in] pli plane index + * @param [in] adapt probability adaptation context + * @param [in] qm QM with magnitude compensation + * @param [in] qm_inv Inverse of QM with magnitude compensation + * @param [in] pvq_norm_lambda enc->pvq_norm_lambda for quantized RDO + * @param [in] speed Make search faster by making approximations + * @return gain index of the quatized gain +*/ +static int pvq_theta(od_coeff *out, const od_coeff *x0, const od_coeff *r0, + int n, int q0, od_coeff *y, int *itheta, int *vk, + od_val16 beta, double *skip_diff, int is_keyframe, int pli, + const od_adapt_ctx *adapt, const int16_t *qm, const int16_t *qm_inv, + double pvq_norm_lambda, int speed) { + od_val32 g; + od_val32 gr; + od_coeff y_tmp[MAXN + 3]; + int i; + /* Number of pulses. */ + int k; + /* Companded gain of x and reference, normalized to q. */ + od_val32 cg; + od_val32 cgr; + int icgr; + int qg; + /* Best RDO cost (D + lamdba*R) so far. */ + double best_cost; + double dist0; + /* Distortion (D) that corresponds to the best RDO cost. */ + double best_dist; + double dist; + /* Sign of Householder reflection. */ + int s; + /* Dimension on which Householder reflects. */ + int m; + od_val32 theta; + double corr; + int best_k; + od_val32 best_qtheta; + od_val32 gain_offset; + int noref; + double skip_dist; + int cfl_enabled; + int skip; + double gain_weight; + od_val16 x16[MAXN]; + od_val16 r16[MAXN]; + int xshift; + int rshift; + /* Give more weight to gain error when calculating the total distortion. */ + gain_weight = 1.0; + OD_ASSERT(n > 1); + corr = 0; +#if !defined(OD_FLOAT_PVQ) + /* Shift needed to make x fit in 16 bits even after rotation. + This shift value is not normative (it can be changed without breaking + the bitstream) */ + xshift = OD_MAXI(0, od_vector_log_mag(x0, n) - 15); + /* Shift needed to make the reference fit in 15 bits, so that the Householder + vector can fit in 16 bits. + This shift value *is* normative, and has to match the decoder. */ + rshift = OD_MAXI(0, od_vector_log_mag(r0, n) - 14); +#else + xshift = 0; + rshift = 0; +#endif + for (i = 0; i < n; i++) { +#if defined(OD_FLOAT_PVQ) + /*This is slightly different from the original float PVQ code, + where the qm was applied in the accumulation in od_pvq_compute_gain and + the vectors were od_coeffs, not od_val16 (i.e. double).*/ + x16[i] = x0[i]*(double)qm[i]*OD_QM_SCALE_1; + r16[i] = r0[i]*(double)qm[i]*OD_QM_SCALE_1; +#else + x16[i] = OD_SHR_ROUND(x0[i]*qm[i], OD_QM_SHIFT + xshift); + r16[i] = OD_SHR_ROUND(r0[i]*qm[i], OD_QM_SHIFT + rshift); +#endif + corr += OD_MULT16_16(x16[i], r16[i]); + } + cfl_enabled = is_keyframe && pli != 0 && !OD_DISABLE_CFL; + cg = od_pvq_compute_gain(x16, n, q0, &g, beta, xshift); + cgr = od_pvq_compute_gain(r16, n, q0, &gr, beta, rshift); + if (cfl_enabled) cgr = OD_CGAIN_SCALE; + /* gain_offset is meant to make sure one of the quantized gains has + exactly the same gain as the reference. */ +#if defined(OD_FLOAT_PVQ) + icgr = (int)floor(.5 + cgr); +#else + icgr = OD_SHR_ROUND(cgr, OD_CGAIN_SHIFT); +#endif + gain_offset = cgr - OD_SHL(icgr, OD_CGAIN_SHIFT); + /* Start search with null case: gain=0, no pulse. */ + qg = 0; + dist = gain_weight*cg*cg*OD_CGAIN_SCALE_2; + best_dist = dist; + best_cost = dist + pvq_norm_lambda*od_pvq_rate(0, 0, -1, 0, adapt, NULL, 0, + n, speed); + noref = 1; + best_k = 0; + *itheta = -1; + OD_CLEAR(y, n); + best_qtheta = 0; + m = 0; + s = 1; + corr = corr/(1e-100 + g*(double)gr/OD_SHL(1, xshift + rshift)); + corr = OD_MAXF(OD_MINF(corr, 1.), -1.); + if (is_keyframe) skip_dist = gain_weight*cg*cg*OD_CGAIN_SCALE_2; + else { + skip_dist = gain_weight*(cg - cgr)*(cg - cgr) + + cgr*(double)cg*(2 - 2*corr); + skip_dist *= OD_CGAIN_SCALE_2; + } + if (!is_keyframe) { + /* noref, gain=0 isn't allowed, but skip is allowed. */ + od_val32 scgr; + scgr = OD_MAXF(0,gain_offset); + if (icgr == 0) { + best_dist = gain_weight*(cg - scgr)*(cg - scgr) + + scgr*(double)cg*(2 - 2*corr); + best_dist *= OD_CGAIN_SCALE_2; + } + best_cost = best_dist + pvq_norm_lambda*od_pvq_rate(0, icgr, 0, 0, adapt, + NULL, 0, n, speed); + best_qtheta = 0; + *itheta = 0; + noref = 0; + } + dist0 = best_dist; + if (n <= OD_MAX_PVQ_SIZE && !od_vector_is_null(r0, n) && corr > 0) { + od_val16 xr[MAXN]; + int gain_bound; + int prev_k; + pvq_search_item items[MAX_PVQ_ITEMS]; + int idx; + int nitems; + double cos_dist; + idx = 0; + gain_bound = OD_SHR(cg - gain_offset, OD_CGAIN_SHIFT); + /* Perform theta search only if prediction is useful. */ + theta = OD_ROUND32(OD_THETA_SCALE*acos(corr)); + m = od_compute_householder(r16, n, gr, &s, rshift); + od_apply_householder(xr, x16, r16, n); + prev_k = 0; + for (i = m; i < n - 1; i++) xr[i] = xr[i + 1]; + /* Compute all candidate PVQ searches within a reasonable range of gain + and theta. */ + for (i = OD_MAXI(1, gain_bound - 1); i <= gain_bound + 1; i++) { + int j; + od_val32 qcg; + int ts; + int theta_lower; + int theta_upper; + /* Quantized companded gain */ + qcg = OD_SHL(i, OD_CGAIN_SHIFT) + gain_offset; + /* Set angular resolution (in ra) to match the encoded gain */ + ts = od_pvq_compute_max_theta(qcg, beta); + theta_lower = OD_MAXI(0, (int)floor(.5 + + theta*OD_THETA_SCALE_1*2/M_PI*ts) - 2); + theta_upper = OD_MINI(ts - 1, (int)ceil(theta*OD_THETA_SCALE_1*2/M_PI*ts)); + /* Include the angles within a reasonable range. */ + for (j = theta_lower; j <= theta_upper; j++) { + od_val32 qtheta; + qtheta = od_pvq_compute_theta(j, ts); + k = od_pvq_compute_k(qcg, j, 0, n, beta); + items[idx].gain = i; + items[idx].theta = j; + items[idx].k = k; + items[idx].qcg = qcg; + items[idx].qtheta = qtheta; + items[idx].ts = ts; + idx++; + OD_ASSERT(idx < MAX_PVQ_ITEMS); + } + } + nitems = idx; + cos_dist = 0; + /* Sort PVQ search candidates in ascending order of pulses K so that + we can reuse all the previously searched pulses across searches. */ + qsort(items, nitems, sizeof(items[0]), + (int (*)(const void *, const void *))items_compare); + /* Search for the best gain/theta in order. */ + for (idx = 0; idx < nitems; idx++) { + int j; + od_val32 qcg; + int ts; + double cost; + double dist_theta; + double sin_prod; + od_val32 qtheta; + /* Quantized companded gain */ + qcg = items[idx].qcg; + i = items[idx].gain; + j = items[idx].theta; + /* Set angular resolution (in ra) to match the encoded gain */ + ts = items[idx].ts; + /* Search for the best angle within a reasonable range. */ + qtheta = items[idx].qtheta; + k = items[idx].k; + /* Compute the minimal possible distortion by not taking the PVQ + cos_dist into account. */ + dist_theta = 2 - 2.*od_pvq_cos(theta - qtheta)*OD_TRIG_SCALE_1; + dist = gain_weight*(qcg - cg)*(qcg - cg) + qcg*(double)cg*dist_theta; + dist *= OD_CGAIN_SCALE_2; + /* If we have no hope of beating skip (including a 1-bit worst-case + penalty), stop now. */ + if (dist > dist0 + 1.0*pvq_norm_lambda && k != 0) continue; + sin_prod = od_pvq_sin(theta)*OD_TRIG_SCALE_1*od_pvq_sin(qtheta)* + OD_TRIG_SCALE_1; + /* PVQ search, using a gain of qcg*cg*sin(theta)*sin(qtheta) since + that's the factor by which cos_dist is multiplied to get the + distortion metric. */ + if (k == 0) { + cos_dist = 0; + OD_CLEAR(y_tmp, n-1); + } + else if (k != prev_k) { + cos_dist = pvq_search_rdo_double(xr, n - 1, k, y_tmp, + qcg*(double)cg*sin_prod*OD_CGAIN_SCALE_2, pvq_norm_lambda, prev_k); + } + prev_k = k; + /* See Jmspeex' Journal of Dubious Theoretical Results. */ + dist_theta = 2 - 2.*od_pvq_cos(theta - qtheta)*OD_TRIG_SCALE_1 + + sin_prod*(2 - 2*cos_dist); + dist = gain_weight*(qcg - cg)*(qcg - cg) + qcg*(double)cg*dist_theta; + dist *= OD_CGAIN_SCALE_2; + /* Do approximate RDO. */ + cost = dist + pvq_norm_lambda*od_pvq_rate(i, icgr, j, ts, adapt, y_tmp, + k, n, speed); + if (cost < best_cost) { + best_cost = cost; + best_dist = dist; + qg = i; + best_k = k; + best_qtheta = qtheta; + *itheta = j; + noref = 0; + OD_COPY(y, y_tmp, n - 1); + } + } + } + /* Don't bother with no-reference version if there's a reasonable + correlation. */ + if (n <= OD_MAX_PVQ_SIZE && (corr < .5 + || cg < (od_val32)(OD_SHL(2, OD_CGAIN_SHIFT)))) { + int gain_bound; + int prev_k; + gain_bound = OD_SHR(cg, OD_CGAIN_SHIFT); + prev_k = 0; + /* Search for the best gain (haven't determined reasonable range yet). */ + for (i = OD_MAXI(1, gain_bound); i <= gain_bound + 1; i++) { + double cos_dist; + double cost; + od_val32 qcg; + qcg = OD_SHL(i, OD_CGAIN_SHIFT); + k = od_pvq_compute_k(qcg, -1, 1, n, beta); + /* Compute the minimal possible distortion by not taking the PVQ + cos_dist into account. */ + dist = gain_weight*(qcg - cg)*(qcg - cg); + dist *= OD_CGAIN_SCALE_2; + if (dist > dist0 && k != 0) continue; + cos_dist = pvq_search_rdo_double(x16, n, k, y_tmp, + qcg*(double)cg*OD_CGAIN_SCALE_2, pvq_norm_lambda, prev_k); + prev_k = k; + /* See Jmspeex' Journal of Dubious Theoretical Results. */ + dist = gain_weight*(qcg - cg)*(qcg - cg) + + qcg*(double)cg*(2 - 2*cos_dist); + dist *= OD_CGAIN_SCALE_2; + /* Do approximate RDO. */ + cost = dist + pvq_norm_lambda*od_pvq_rate(i, 0, -1, 0, adapt, y_tmp, k, + n, speed); + if (cost <= best_cost) { + best_cost = cost; + best_dist = dist; + qg = i; + noref = 1; + best_k = k; + *itheta = -1; + OD_COPY(y, y_tmp, n); + } + } + } + k = best_k; + theta = best_qtheta; + skip = 0; + if (noref) { + if (qg == 0) skip = OD_PVQ_SKIP_ZERO; + } + else { + if (!is_keyframe && qg == 0) { + skip = (icgr ? OD_PVQ_SKIP_ZERO : OD_PVQ_SKIP_COPY); + } + if (qg == icgr && *itheta == 0 && !cfl_enabled) skip = OD_PVQ_SKIP_COPY; + } + /* Synthesize like the decoder would. */ + if (skip) { + if (skip == OD_PVQ_SKIP_COPY) OD_COPY(out, r0, n); + else OD_CLEAR(out, n); + } + else { + if (noref) gain_offset = 0; + g = od_gain_expand(OD_SHL(qg, OD_CGAIN_SHIFT) + gain_offset, q0, beta); + od_pvq_synthesis_partial(out, y, r16, n, noref, g, theta, m, s, + qm_inv); + } + *vk = k; + *skip_diff += skip_dist - best_dist; + /* Encode gain differently depending on whether we use prediction or not. + Special encoding on inter frames where qg=0 is allowed for noref=0 + but not noref=1.*/ + if (is_keyframe) return noref ? qg : neg_interleave(qg, icgr); + else return noref ? qg - 1 : neg_interleave(qg + 1, icgr + 1); +} + +/** Encodes a single vector of integers (eg, a partition within a + * coefficient block) using PVQ + * + * @param [in,out] w multi-symbol entropy encoder + * @param [in] qg quantized gain + * @param [in] theta quantized post-prediction theta + * @param [in] in coefficient vector to code + * @param [in] n number of coefficients in partition + * @param [in] k number of pulses in partition + * @param [in,out] model entropy encoder state + * @param [in,out] adapt adaptation context + * @param [in,out] exg ExQ16 expectation of gain value + * @param [in,out] ext ExQ16 expectation of theta value + * @param [in] cdf_ctx selects which cdf context to use + * @param [in] is_keyframe whether we're encoding a keyframe + * @param [in] code_skip whether the "skip rest" flag is allowed + * @param [in] skip_rest when set, we skip all higher bands + * @param [in] encode_flip whether we need to encode the CfL flip flag now + * @param [in] flip value of the CfL flip flag + */ +void pvq_encode_partition(aom_writer *w, + int qg, + int theta, + const od_coeff *in, + int n, + int k, + generic_encoder model[3], + od_adapt_ctx *adapt, + int *exg, + int *ext, + int cdf_ctx, + int is_keyframe, + int code_skip, + int skip_rest, + int encode_flip, + int flip) { + int noref; + int id; + noref = (theta == -1); + id = (qg > 0) + 2*OD_MINI(theta + 1,3) + 8*code_skip*skip_rest; + if (is_keyframe) { + OD_ASSERT(id != 8); + if (id >= 8) id--; + } + else { + OD_ASSERT(id != 10); + if (id >= 10) id--; + } + /* Jointly code gain, theta and noref for small values. Then we handle + larger gain and theta values. For noref, theta = -1. */ + aom_write_symbol_pvq(w, id, &adapt->pvq.pvq_gaintheta_cdf[cdf_ctx][0], + 8 + 7*code_skip); + if (encode_flip) { + /* We could eventually do some smarter entropy coding here, but it would + have to be good enough to overcome the overhead of the entropy coder. + An early attempt using a "toogle" flag with simple adaptation wasn't + worth the trouble. */ + aom_write_bit(w, flip); + } + if (qg > 0) { + int tmp; + tmp = *exg; + generic_encode(w, &model[!noref], qg - 1, &tmp, 2); + OD_IIR_DIADIC(*exg, qg << 16, 2); + } + if (theta > 1) { + int tmp; + tmp = *ext; + generic_encode(w, &model[2], theta - 2, &tmp, 2); + OD_IIR_DIADIC(*ext, theta << 16, 2); + } + aom_encode_pvq_codeword(w, &adapt->pvq.pvq_codeword_ctx, in, + n - (theta != -1), k); +} + +/** Quantizes a scalar with rate-distortion optimization (RDO) + * @param [in] x unquantized value + * @param [in] q quantization step size + * @param [in] delta0 rate increase for encoding a 1 instead of a 0 + * @param [in] pvq_norm_lambda enc->pvq_norm_lambda for quantized RDO + * @retval quantized value + */ +int od_rdo_quant(od_coeff x, int q, double delta0, double pvq_norm_lambda) { + int n; + /* Optimal quantization threshold is 1/2 + lambda*delta_rate/2. See + Jmspeex' Journal of Dubious Theoretical Results for details. */ + n = OD_DIV_R0(abs(x), q); + if ((double)abs(x)/q < (double)n/2 + pvq_norm_lambda*delta0/(2*n)) { + return 0; + } + else { + return OD_DIV_R0(x, q); + } +} + +/** Encode a coefficient block (excepting DC) using PVQ + * + * @param [in,out] enc daala encoder context + * @param [in] ref 'reference' (prediction) vector + * @param [in] in coefficient block to quantize and encode + * @param [out] out quantized coefficient block + * @param [in] q0 scale/quantizer + * @param [in] pli plane index + * @param [in] bs log of the block size minus two + * @param [in] beta per-band activity masking beta param + * @param [in] is_keyframe whether we're encoding a keyframe + * @param [in] qm QM with magnitude compensation + * @param [in] qm_inv Inverse of QM with magnitude compensation + * @param [in] speed Make search faster by making approximations + * @param [in] pvq_info If null, conisdered as RDO search mode + * @return Returns block skip info indicating whether DC/AC are coded. + * bit0: DC is coded, bit1: AC is coded (1 means coded) + * + */ +PVQ_SKIP_TYPE od_pvq_encode(daala_enc_ctx *enc, + od_coeff *ref, + const od_coeff *in, + od_coeff *out, + int q_dc, + int q_ac, + int pli, + int bs, + const od_val16 *beta, + int is_keyframe, + const int16_t *qm, + const int16_t *qm_inv, + int speed, + PVQ_INFO *pvq_info){ + int theta[PVQ_MAX_PARTITIONS]; + int qg[PVQ_MAX_PARTITIONS]; + int k[PVQ_MAX_PARTITIONS]; + od_coeff y[OD_TXSIZE_MAX*OD_TXSIZE_MAX]; + int *exg; + int *ext; + int nb_bands; + int i; + const int *off; + int size[PVQ_MAX_PARTITIONS]; + generic_encoder *model; + double skip_diff; + int tell; + uint16_t *skip_cdf; + od_rollback_buffer buf; + int dc_quant; + int flip; + int cfl_encoded; + int skip_rest; + int skip_dir; + int skip_theta_value; + const unsigned char *pvq_qm; + double dc_rate; + int use_masking; + PVQ_SKIP_TYPE ac_dc_coded; + + aom_clear_system_state(); + + use_masking = enc->use_activity_masking; + + if (use_masking) + pvq_qm = &enc->state.pvq_qm_q4[pli][0]; + else + pvq_qm = 0; + + exg = &enc->state.adapt->pvq.pvq_exg[pli][bs][0]; + ext = enc->state.adapt->pvq.pvq_ext + bs*PVQ_MAX_PARTITIONS; + skip_cdf = enc->state.adapt->skip_cdf[2*bs + (pli != 0)]; + model = enc->state.adapt->pvq.pvq_param_model; + nb_bands = OD_BAND_OFFSETS[bs][0]; + off = &OD_BAND_OFFSETS[bs][1]; + + if (use_masking) + dc_quant = OD_MAXI(1, q_dc * pvq_qm[od_qm_get_index(bs, 0)] >> 4); + else + dc_quant = OD_MAXI(1, q_dc); + + tell = 0; + for (i = 0; i < nb_bands; i++) size[i] = off[i+1] - off[i]; + skip_diff = 0; + flip = 0; + /*If we are coding a chroma block of a keyframe, we are doing CfL.*/ + if (pli != 0 && is_keyframe) { + od_val32 xy; + xy = 0; + /*Compute the dot-product of the first band of chroma with the luma ref.*/ + for (i = off[0]; i < off[1]; i++) { +#if defined(OD_FLOAT_PVQ) + xy += ref[i]*(double)qm[i]*OD_QM_SCALE_1* + (double)in[i]*(double)qm[i]*OD_QM_SCALE_1; +#else + od_val32 rq; + od_val32 inq; + rq = ref[i]*qm[i]; + inq = in[i]*qm[i]; + xy += OD_SHR(rq*(int64_t)inq, OD_SHL(OD_QM_SHIFT + OD_CFL_FLIP_SHIFT, + 1)); +#endif + } + /*If cos(theta) < 0, then |theta| > pi/2 and we should negate the ref.*/ + if (xy < 0) { + flip = 1; + for(i = off[0]; i < off[nb_bands]; i++) ref[i] = -ref[i]; + } + } + for (i = 0; i < nb_bands; i++) { + int q; + + if (use_masking) + q = OD_MAXI(1, q_ac * pvq_qm[od_qm_get_index(bs, i + 1)] >> 4); + else + q = OD_MAXI(1, q_ac); + + qg[i] = pvq_theta(out + off[i], in + off[i], ref + off[i], size[i], + q, y + off[i], &theta[i], &k[i], beta[i], &skip_diff, is_keyframe, + pli, enc->state.adapt, qm + off[i], qm_inv + off[i], + enc->pvq_norm_lambda, speed); + } + od_encode_checkpoint(enc, &buf); + if (is_keyframe) out[0] = 0; + else { + int n; + n = OD_DIV_R0(abs(in[0] - ref[0]), dc_quant); + if (n == 0) { + out[0] = 0; + } else { + int tell2; + od_rollback_buffer dc_buf; + + dc_rate = -OD_LOG2((double)(skip_cdf[3] - skip_cdf[2])/ + (double)(skip_cdf[2] - skip_cdf[1])); + dc_rate += 1; + +#if CONFIG_DAALA_EC + tell2 = od_ec_enc_tell_frac(&enc->w.ec); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + od_encode_checkpoint(enc, &dc_buf); + generic_encode(&enc->w, &enc->state.adapt->model_dc[pli], + n - 1, &enc->state.adapt->ex_dc[pli][bs][0], 2); +#if CONFIG_DAALA_EC + tell2 = od_ec_enc_tell_frac(&enc->w.ec) - tell2; +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + dc_rate += tell2/8.0; + od_encode_rollback(enc, &dc_buf); + + out[0] = od_rdo_quant(in[0] - ref[0], dc_quant, dc_rate, + enc->pvq_norm_lambda); + } + } +#if CONFIG_DAALA_EC + tell = od_ec_enc_tell_frac(&enc->w.ec); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + /* Code as if we're not skipping. */ + aom_write_symbol(&enc->w, 2 + (out[0] != 0), skip_cdf, 4); + ac_dc_coded = AC_CODED + (out[0] != 0); + cfl_encoded = 0; + skip_rest = 1; + skip_theta_value = is_keyframe ? -1 : 0; + for (i = 1; i < nb_bands; i++) { + if (theta[i] != skip_theta_value || qg[i]) skip_rest = 0; + } + skip_dir = 0; + if (nb_bands > 1) { + for (i = 0; i < 3; i++) { + int j; + int tmp; + tmp = 1; + // ToDo(yaowu): figure out better stop condition without gcc warning. + for (j = i + 1; j < nb_bands && j < PVQ_MAX_PARTITIONS; j += 3) { + if (theta[j] != skip_theta_value || qg[j]) tmp = 0; + } + skip_dir |= tmp << i; + } + } + if (theta[0] == skip_theta_value && qg[0] == 0 && skip_rest) nb_bands = 0; + + /* NOTE: There was no other better place to put this function. */ + if (pvq_info) + av1_store_pvq_enc_info(pvq_info, qg, theta, k, y, nb_bands, off, size, + skip_rest, skip_dir, bs); + + for (i = 0; i < nb_bands; i++) { + int encode_flip; + /* Encode CFL flip bit just after the first time it's used. */ + encode_flip = pli != 0 && is_keyframe && theta[i] != -1 && !cfl_encoded; + if (i == 0 || (!skip_rest && !(skip_dir & (1 << ((i - 1)%3))))) { + pvq_encode_partition(&enc->w, qg[i], theta[i], y + off[i], + size[i], k[i], model, enc->state.adapt, exg + i, ext + i, + (pli != 0)*OD_TXSIZES*PVQ_MAX_PARTITIONS + bs*PVQ_MAX_PARTITIONS + i, + is_keyframe, i == 0 && (i < nb_bands - 1), skip_rest, encode_flip, flip); + } + if (i == 0 && !skip_rest && bs > 0) { + aom_write_symbol(&enc->w, skip_dir, + &enc->state.adapt->pvq.pvq_skip_dir_cdf[(pli != 0) + 2*(bs - 1)][0], 7); + } + if (encode_flip) cfl_encoded = 1; + } +#if CONFIG_DAALA_EC + tell = od_ec_enc_tell_frac(&enc->w.ec) - tell; +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + /* Account for the rate of skipping the AC, based on the same DC decision + we made when trying to not skip AC. */ + { + double skip_rate; + if (out[0] != 0) { + skip_rate = -OD_LOG2((skip_cdf[1] - skip_cdf[0])/ + (double)skip_cdf[3]); + } + else { + skip_rate = -OD_LOG2(skip_cdf[0]/ + (double)skip_cdf[3]); + } + tell -= (int)floor(.5+8*skip_rate); + } + if (nb_bands == 0 || skip_diff <= enc->pvq_norm_lambda/8*tell) { + if (is_keyframe) out[0] = 0; + else { + int n; + n = OD_DIV_R0(abs(in[0] - ref[0]), dc_quant); + if (n == 0) { + out[0] = 0; + } else { + int tell2; + od_rollback_buffer dc_buf; + + dc_rate = -OD_LOG2((double)(skip_cdf[1] - skip_cdf[0])/ + (double)skip_cdf[0]); + dc_rate += 1; + +#if CONFIG_DAALA_EC + tell2 = od_ec_enc_tell_frac(&enc->w.ec); +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + od_encode_checkpoint(enc, &dc_buf); + generic_encode(&enc->w, &enc->state.adapt->model_dc[pli], + n - 1, &enc->state.adapt->ex_dc[pli][bs][0], 2); +#if CONFIG_DAALA_EC + tell2 = od_ec_enc_tell_frac(&enc->w.ec) - tell2; +#else +#error "CONFIG_PVQ currently requires CONFIG_DAALA_EC." +#endif + dc_rate += tell2/8.0; + od_encode_rollback(enc, &dc_buf); + + out[0] = od_rdo_quant(in[0] - ref[0], dc_quant, dc_rate, + enc->pvq_norm_lambda); + } + } + /* We decide to skip, roll back everything as it was before. */ + od_encode_rollback(enc, &buf); + aom_write_symbol(&enc->w, out[0] != 0, skip_cdf, 4); + ac_dc_coded = (out[0] != 0); + if (is_keyframe) for (i = 1; i < 1 << (2*bs + 4); i++) out[i] = 0; + else for (i = 1; i < 1 << (2*bs + 4); i++) out[i] = ref[i]; + } + if (pvq_info) + pvq_info->ac_dc_coded = ac_dc_coded; + return ac_dc_coded; +} diff --git a/third_party/aom/av1/encoder/pvq_encoder.h b/third_party/aom/av1/encoder/pvq_encoder.h new file mode 100644 index 0000000000..b84c8961b7 --- /dev/null +++ b/third_party/aom/av1/encoder/pvq_encoder.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* clang-format off */ + +#if !defined(_pvq_encoder_H) +# define _pvq_encoder_H (1) +# include "aom_dsp/bitwriter.h" +# include "aom_dsp/entenc.h" +# include "av1/common/blockd.h" +# include "av1/common/pvq.h" +# include "av1/encoder/encint.h" + +void aom_write_symbol_pvq(aom_writer *w, int symb, aom_cdf_prob *cdf, + int nsymbs); + +void aom_encode_band_pvq_splits(aom_writer *w, od_pvq_codeword_ctx *adapt, + const int *y, int n, int k, int level); + +void aom_laplace_encode_special(aom_writer *w, int x, unsigned decay); + +void pvq_encode_partition(aom_writer *w, + int qg, + int theta, + const od_coeff *in, + int n, + int k, + generic_encoder model[3], + od_adapt_ctx *adapt, + int *exg, + int *ext, + int cdf_ctx, + int is_keyframe, + int code_skip, + int skip_rest, + int encode_flip, + int flip); + +PVQ_SKIP_TYPE od_pvq_encode(daala_enc_ctx *enc, od_coeff *ref, + const od_coeff *in, od_coeff *out, int q_dc, int q_ac, int pli, int bs, + const od_val16 *beta, int is_keyframe, + const int16_t *qm, const int16_t *qm_inv, int speed, + PVQ_INFO *pvq_info); + +#endif diff --git a/third_party/aom/av1/encoder/ransac.c b/third_party/aom/av1/encoder/ransac.c new file mode 100644 index 0000000000..5d5dd75721 --- /dev/null +++ b/third_party/aom/av1/encoder/ransac.c @@ -0,0 +1,1210 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#define _POSIX_C_SOURCE 200112L // rand_r() +#include +#include +#include +#include +#include +#include + +#include "av1/encoder/ransac.h" + +#define MAX_MINPTS 4 +#define MAX_DEGENERATE_ITER 10 +#define MINPTS_MULTIPLIER 5 + +#define INLIER_THRESHOLD 1.0 +#define MIN_TRIALS 20 + +//////////////////////////////////////////////////////////////////////////////// +// ransac +typedef int (*IsDegenerateFunc)(double *p); +typedef void (*NormalizeFunc)(double *p, int np, double *T); +typedef void (*DenormalizeFunc)(double *params, double *T1, double *T2); +typedef int (*FindTransformationFunc)(int points, double *points1, + double *points2, double *params); +typedef void (*ProjectPointsDoubleFunc)(double *mat, double *points, + double *proj, const int n, + const int stride_points, + const int stride_proj); + +static void project_points_double_translation(double *mat, double *points, + double *proj, const int n, + const int stride_points, + const int stride_proj) { + int i; + for (i = 0; i < n; ++i) { + const double x = *(points++), y = *(points++); + *(proj++) = x + mat[0]; + *(proj++) = y + mat[1]; + points += stride_points - 2; + proj += stride_proj - 2; + } +} + +static void project_points_double_rotzoom(double *mat, double *points, + double *proj, const int n, + const int stride_points, + const int stride_proj) { + int i; + for (i = 0; i < n; ++i) { + const double x = *(points++), y = *(points++); + *(proj++) = mat[2] * x + mat[3] * y + mat[0]; + *(proj++) = -mat[3] * x + mat[2] * y + mat[1]; + points += stride_points - 2; + proj += stride_proj - 2; + } +} + +static void project_points_double_affine(double *mat, double *points, + double *proj, const int n, + const int stride_points, + const int stride_proj) { + int i; + for (i = 0; i < n; ++i) { + const double x = *(points++), y = *(points++); + *(proj++) = mat[2] * x + mat[3] * y + mat[0]; + *(proj++) = mat[4] * x + mat[5] * y + mat[1]; + points += stride_points - 2; + proj += stride_proj - 2; + } +} + +static void project_points_double_hortrapezoid(double *mat, double *points, + double *proj, const int n, + const int stride_points, + const int stride_proj) { + int i; + double x, y, Z, Z_inv; + for (i = 0; i < n; ++i) { + x = *(points++), y = *(points++); + Z_inv = mat[7] * y + 1; + assert(fabs(Z_inv) > 0.000001); + Z = 1. / Z_inv; + *(proj++) = (mat[2] * x + mat[3] * y + mat[0]) * Z; + *(proj++) = (mat[5] * y + mat[1]) * Z; + points += stride_points - 2; + proj += stride_proj - 2; + } +} + +static void project_points_double_vertrapezoid(double *mat, double *points, + double *proj, const int n, + const int stride_points, + const int stride_proj) { + int i; + double x, y, Z, Z_inv; + for (i = 0; i < n; ++i) { + x = *(points++), y = *(points++); + Z_inv = mat[6] * x + 1; + assert(fabs(Z_inv) > 0.000001); + Z = 1. / Z_inv; + *(proj++) = (mat[2] * x + mat[0]) * Z; + *(proj++) = (mat[4] * x + mat[5] * y + mat[1]) * Z; + points += stride_points - 2; + proj += stride_proj - 2; + } +} + +static void project_points_double_homography(double *mat, double *points, + double *proj, const int n, + const int stride_points, + const int stride_proj) { + int i; + double x, y, Z, Z_inv; + for (i = 0; i < n; ++i) { + x = *(points++), y = *(points++); + Z_inv = mat[6] * x + mat[7] * y + 1; + assert(fabs(Z_inv) > 0.000001); + Z = 1. / Z_inv; + *(proj++) = (mat[2] * x + mat[3] * y + mat[0]) * Z; + *(proj++) = (mat[4] * x + mat[5] * y + mat[1]) * Z; + points += stride_points - 2; + proj += stride_proj - 2; + } +} + +/////////////////////////////////////////////////////////////////////////////// +// svdcmp +// Adopted from Numerical Recipes in C + +static const double TINY_NEAR_ZERO = 1.0E-12; + +static INLINE double sign(double a, double b) { + return ((b) >= 0 ? fabs(a) : -fabs(a)); +} + +static INLINE double pythag(double a, double b) { + double ct; + const double absa = fabs(a); + const double absb = fabs(b); + + if (absa > absb) { + ct = absb / absa; + return absa * sqrt(1.0 + ct * ct); + } else { + ct = absa / absb; + return (absb == 0) ? 0 : absb * sqrt(1.0 + ct * ct); + } +} + +static void multiply_mat(const double *m1, const double *m2, double *res, + const int m1_rows, const int inner_dim, + const int m2_cols) { + double sum; + + int row, col, inner; + for (row = 0; row < m1_rows; ++row) { + for (col = 0; col < m2_cols; ++col) { + sum = 0; + for (inner = 0; inner < inner_dim; ++inner) + sum += m1[row * inner_dim + inner] * m2[inner * m2_cols + col]; + *(res++) = sum; + } + } +} + +static int svdcmp(double **u, int m, int n, double w[], double **v) { + const int max_its = 30; + int flag, i, its, j, jj, k, l, nm; + double anorm, c, f, g, h, s, scale, x, y, z; + double *rv1 = (double *)aom_malloc(sizeof(*rv1) * (n + 1)); + g = scale = anorm = 0.0; + for (i = 0; i < n; i++) { + l = i + 1; + rv1[i] = scale * g; + g = s = scale = 0.0; + if (i < m) { + for (k = i; k < m; k++) scale += fabs(u[k][i]); + if (scale != 0.) { + for (k = i; k < m; k++) { + u[k][i] /= scale; + s += u[k][i] * u[k][i]; + } + f = u[i][i]; + g = -sign(sqrt(s), f); + h = f * g - s; + u[i][i] = f - g; + for (j = l; j < n; j++) { + for (s = 0.0, k = i; k < m; k++) s += u[k][i] * u[k][j]; + f = s / h; + for (k = i; k < m; k++) u[k][j] += f * u[k][i]; + } + for (k = i; k < m; k++) u[k][i] *= scale; + } + } + w[i] = scale * g; + g = s = scale = 0.0; + if (i < m && i != n - 1) { + for (k = l; k < n; k++) scale += fabs(u[i][k]); + if (scale != 0.) { + for (k = l; k < n; k++) { + u[i][k] /= scale; + s += u[i][k] * u[i][k]; + } + f = u[i][l]; + g = -sign(sqrt(s), f); + h = f * g - s; + u[i][l] = f - g; + for (k = l; k < n; k++) rv1[k] = u[i][k] / h; + for (j = l; j < m; j++) { + for (s = 0.0, k = l; k < n; k++) s += u[j][k] * u[i][k]; + for (k = l; k < n; k++) u[j][k] += s * rv1[k]; + } + for (k = l; k < n; k++) u[i][k] *= scale; + } + } + anorm = fmax(anorm, (fabs(w[i]) + fabs(rv1[i]))); + } + + for (i = n - 1; i >= 0; i--) { + if (i < n - 1) { + if (g != 0.) { + for (j = l; j < n; j++) v[j][i] = (u[i][j] / u[i][l]) / g; + for (j = l; j < n; j++) { + for (s = 0.0, k = l; k < n; k++) s += u[i][k] * v[k][j]; + for (k = l; k < n; k++) v[k][j] += s * v[k][i]; + } + } + for (j = l; j < n; j++) v[i][j] = v[j][i] = 0.0; + } + v[i][i] = 1.0; + g = rv1[i]; + l = i; + } + for (i = AOMMIN(m, n) - 1; i >= 0; i--) { + l = i + 1; + g = w[i]; + for (j = l; j < n; j++) u[i][j] = 0.0; + if (g != 0.) { + g = 1.0 / g; + for (j = l; j < n; j++) { + for (s = 0.0, k = l; k < m; k++) s += u[k][i] * u[k][j]; + f = (s / u[i][i]) * g; + for (k = i; k < m; k++) u[k][j] += f * u[k][i]; + } + for (j = i; j < m; j++) u[j][i] *= g; + } else { + for (j = i; j < m; j++) u[j][i] = 0.0; + } + ++u[i][i]; + } + for (k = n - 1; k >= 0; k--) { + for (its = 0; its < max_its; its++) { + flag = 1; + for (l = k; l >= 0; l--) { + nm = l - 1; + if ((double)(fabs(rv1[l]) + anorm) == anorm || nm < 0) { + flag = 0; + break; + } + if ((double)(fabs(w[nm]) + anorm) == anorm) break; + } + if (flag) { + c = 0.0; + s = 1.0; + for (i = l; i <= k; i++) { + f = s * rv1[i]; + rv1[i] = c * rv1[i]; + if ((double)(fabs(f) + anorm) == anorm) break; + g = w[i]; + h = pythag(f, g); + w[i] = h; + h = 1.0 / h; + c = g * h; + s = -f * h; + for (j = 0; j < m; j++) { + y = u[j][nm]; + z = u[j][i]; + u[j][nm] = y * c + z * s; + u[j][i] = z * c - y * s; + } + } + } + z = w[k]; + if (l == k) { + if (z < 0.0) { + w[k] = -z; + for (j = 0; j < n; j++) v[j][k] = -v[j][k]; + } + break; + } + if (its == max_its - 1) { + aom_free(rv1); + return 1; + } + assert(k > 0); + x = w[l]; + nm = k - 1; + y = w[nm]; + g = rv1[nm]; + h = rv1[k]; + f = ((y - z) * (y + z) + (g - h) * (g + h)) / (2.0 * h * y); + g = pythag(f, 1.0); + f = ((x - z) * (x + z) + h * ((y / (f + sign(g, f))) - h)) / x; + c = s = 1.0; + for (j = l; j <= nm; j++) { + i = j + 1; + g = rv1[i]; + y = w[i]; + h = s * g; + g = c * g; + z = pythag(f, h); + rv1[j] = z; + c = f / z; + s = h / z; + f = x * c + g * s; + g = g * c - x * s; + h = y * s; + y *= c; + for (jj = 0; jj < n; jj++) { + x = v[jj][j]; + z = v[jj][i]; + v[jj][j] = x * c + z * s; + v[jj][i] = z * c - x * s; + } + z = pythag(f, h); + w[j] = z; + if (z != 0.) { + z = 1.0 / z; + c = f * z; + s = h * z; + } + f = c * g + s * y; + x = c * y - s * g; + for (jj = 0; jj < m; jj++) { + y = u[jj][j]; + z = u[jj][i]; + u[jj][j] = y * c + z * s; + u[jj][i] = z * c - y * s; + } + } + rv1[l] = 0.0; + rv1[k] = f; + w[k] = x; + } + } + aom_free(rv1); + return 0; +} + +static int SVD(double *U, double *W, double *V, double *matx, int M, int N) { + // Assumes allocation for U is MxN + double **nrU = (double **)aom_malloc((M) * sizeof(*nrU)); + double **nrV = (double **)aom_malloc((N) * sizeof(*nrV)); + int problem, i; + + problem = !(nrU && nrV); + if (!problem) { + for (i = 0; i < M; i++) { + nrU[i] = &U[i * N]; + } + for (i = 0; i < N; i++) { + nrV[i] = &V[i * N]; + } + } else { + if (nrU) aom_free(nrU); + if (nrV) aom_free(nrV); + return 1; + } + + /* copy from given matx into nrU */ + for (i = 0; i < M; i++) { + memcpy(&(nrU[i][0]), matx + N * i, N * sizeof(*matx)); + } + + /* HERE IT IS: do SVD */ + if (svdcmp(nrU, M, N, W, nrV)) { + aom_free(nrU); + aom_free(nrV); + return 1; + } + + /* aom_free Numerical Recipes arrays */ + aom_free(nrU); + aom_free(nrV); + + return 0; +} + +int pseudo_inverse(double *inv, double *matx, const int M, const int N) { + double ans; + int i, j, k; + double *const U = (double *)aom_malloc(M * N * sizeof(*matx)); + double *const W = (double *)aom_malloc(N * sizeof(*matx)); + double *const V = (double *)aom_malloc(N * N * sizeof(*matx)); + + if (!(U && W && V)) { + return 1; + } + if (SVD(U, W, V, matx, M, N)) { + aom_free(U); + aom_free(W); + aom_free(V); + return 1; + } + for (i = 0; i < N; i++) { + if (fabs(W[i]) < TINY_NEAR_ZERO) { + aom_free(U); + aom_free(W); + aom_free(V); + return 1; + } + } + + for (i = 0; i < N; i++) { + for (j = 0; j < M; j++) { + ans = 0; + for (k = 0; k < N; k++) { + ans += V[k + N * i] * U[k + N * j] / W[k]; + } + inv[j + M * i] = ans; + } + } + aom_free(U); + aom_free(W); + aom_free(V); + return 0; +} + +static void normalize_homography(double *pts, int n, double *T) { + double *p = pts; + double mean[2] = { 0, 0 }; + double msqe = 0; + double scale; + int i; + for (i = 0; i < n; ++i, p += 2) { + mean[0] += p[0]; + mean[1] += p[1]; + } + mean[0] /= n; + mean[1] /= n; + for (p = pts, i = 0; i < n; ++i, p += 2) { + p[0] -= mean[0]; + p[1] -= mean[1]; + msqe += sqrt(p[0] * p[0] + p[1] * p[1]); + } + msqe /= n; + scale = (msqe == 0 ? 1.0 : sqrt(2) / msqe); + T[0] = scale; + T[1] = 0; + T[2] = -scale * mean[0]; + T[3] = 0; + T[4] = scale; + T[5] = -scale * mean[1]; + T[6] = 0; + T[7] = 0; + T[8] = 1; + for (p = pts, i = 0; i < n; ++i, p += 2) { + p[0] *= scale; + p[1] *= scale; + } +} + +static void invnormalize_mat(double *T, double *iT) { + double is = 1.0 / T[0]; + double m0 = -T[2] * is; + double m1 = -T[5] * is; + iT[0] = is; + iT[1] = 0; + iT[2] = m0; + iT[3] = 0; + iT[4] = is; + iT[5] = m1; + iT[6] = 0; + iT[7] = 0; + iT[8] = 1; +} + +static void denormalize_homography(double *params, double *T1, double *T2) { + double iT2[9]; + double params2[9]; + invnormalize_mat(T2, iT2); + multiply_mat(params, T1, params2, 3, 3, 3); + multiply_mat(iT2, params2, params, 3, 3, 3); +} + +static void denormalize_homography_reorder(double *params, double *T1, + double *T2) { + double params_denorm[MAX_PARAMDIM]; + memcpy(params_denorm, params, sizeof(*params) * 8); + params_denorm[8] = 1.0; + denormalize_homography(params_denorm, T1, T2); + params[0] = params_denorm[2]; + params[1] = params_denorm[5]; + params[2] = params_denorm[0]; + params[3] = params_denorm[1]; + params[4] = params_denorm[3]; + params[5] = params_denorm[4]; + params[6] = params_denorm[6]; + params[7] = params_denorm[7]; +} + +static void denormalize_affine_reorder(double *params, double *T1, double *T2) { + double params_denorm[MAX_PARAMDIM]; + params_denorm[0] = params[0]; + params_denorm[1] = params[1]; + params_denorm[2] = params[4]; + params_denorm[3] = params[2]; + params_denorm[4] = params[3]; + params_denorm[5] = params[5]; + params_denorm[6] = params_denorm[7] = 0; + params_denorm[8] = 1; + denormalize_homography(params_denorm, T1, T2); + params[0] = params_denorm[2]; + params[1] = params_denorm[5]; + params[2] = params_denorm[0]; + params[3] = params_denorm[1]; + params[4] = params_denorm[3]; + params[5] = params_denorm[4]; + params[6] = params[7] = 0; +} + +static void denormalize_rotzoom_reorder(double *params, double *T1, + double *T2) { + double params_denorm[MAX_PARAMDIM]; + params_denorm[0] = params[0]; + params_denorm[1] = params[1]; + params_denorm[2] = params[2]; + params_denorm[3] = -params[1]; + params_denorm[4] = params[0]; + params_denorm[5] = params[3]; + params_denorm[6] = params_denorm[7] = 0; + params_denorm[8] = 1; + denormalize_homography(params_denorm, T1, T2); + params[0] = params_denorm[2]; + params[1] = params_denorm[5]; + params[2] = params_denorm[0]; + params[3] = params_denorm[1]; + params[4] = -params[3]; + params[5] = params[2]; + params[6] = params[7] = 0; +} + +static void denormalize_translation_reorder(double *params, double *T1, + double *T2) { + double params_denorm[MAX_PARAMDIM]; + params_denorm[0] = 1; + params_denorm[1] = 0; + params_denorm[2] = params[0]; + params_denorm[3] = 0; + params_denorm[4] = 1; + params_denorm[5] = params[1]; + params_denorm[6] = params_denorm[7] = 0; + params_denorm[8] = 1; + denormalize_homography(params_denorm, T1, T2); + params[0] = params_denorm[2]; + params[1] = params_denorm[5]; + params[2] = params[5] = 1; + params[3] = params[4] = 0; + params[6] = params[7] = 0; +} + +static int find_translation(int np, double *pts1, double *pts2, double *mat) { + int i; + double sx, sy, dx, dy; + double sumx, sumy; + + double T1[9], T2[9]; + normalize_homography(pts1, np, T1); + normalize_homography(pts2, np, T2); + + sumx = 0; + sumy = 0; + for (i = 0; i < np; ++i) { + dx = *(pts2++); + dy = *(pts2++); + sx = *(pts1++); + sy = *(pts1++); + + sumx += dx - sx; + sumy += dy - sy; + } + mat[0] = sumx / np; + mat[1] = sumy / np; + denormalize_translation_reorder(mat, T1, T2); + return 0; +} + +static int find_rotzoom(int np, double *pts1, double *pts2, double *mat) { + const int np2 = np * 2; + double *a = (double *)aom_malloc(sizeof(*a) * np2 * 9); + double *b = a + np2 * 4; + double *temp = b + np2; + int i; + double sx, sy, dx, dy; + + double T1[9], T2[9]; + normalize_homography(pts1, np, T1); + normalize_homography(pts2, np, T2); + + for (i = 0; i < np; ++i) { + dx = *(pts2++); + dy = *(pts2++); + sx = *(pts1++); + sy = *(pts1++); + + a[i * 2 * 4 + 0] = sx; + a[i * 2 * 4 + 1] = sy; + a[i * 2 * 4 + 2] = 1; + a[i * 2 * 4 + 3] = 0; + a[(i * 2 + 1) * 4 + 0] = sy; + a[(i * 2 + 1) * 4 + 1] = -sx; + a[(i * 2 + 1) * 4 + 2] = 0; + a[(i * 2 + 1) * 4 + 3] = 1; + + b[2 * i] = dx; + b[2 * i + 1] = dy; + } + if (pseudo_inverse(temp, a, np2, 4)) { + aom_free(a); + return 1; + } + multiply_mat(temp, b, mat, 4, np2, 1); + denormalize_rotzoom_reorder(mat, T1, T2); + aom_free(a); + return 0; +} + +static int find_affine(int np, double *pts1, double *pts2, double *mat) { + const int np2 = np * 2; + double *a = (double *)aom_malloc(sizeof(*a) * np2 * 13); + double *b = a + np2 * 6; + double *temp = b + np2; + int i; + double sx, sy, dx, dy; + + double T1[9], T2[9]; + normalize_homography(pts1, np, T1); + normalize_homography(pts2, np, T2); + + for (i = 0; i < np; ++i) { + dx = *(pts2++); + dy = *(pts2++); + sx = *(pts1++); + sy = *(pts1++); + + a[i * 2 * 6 + 0] = sx; + a[i * 2 * 6 + 1] = sy; + a[i * 2 * 6 + 2] = 0; + a[i * 2 * 6 + 3] = 0; + a[i * 2 * 6 + 4] = 1; + a[i * 2 * 6 + 5] = 0; + a[(i * 2 + 1) * 6 + 0] = 0; + a[(i * 2 + 1) * 6 + 1] = 0; + a[(i * 2 + 1) * 6 + 2] = sx; + a[(i * 2 + 1) * 6 + 3] = sy; + a[(i * 2 + 1) * 6 + 4] = 0; + a[(i * 2 + 1) * 6 + 5] = 1; + + b[2 * i] = dx; + b[2 * i + 1] = dy; + } + if (pseudo_inverse(temp, a, np2, 6)) { + aom_free(a); + return 1; + } + multiply_mat(temp, b, mat, 6, np2, 1); + denormalize_affine_reorder(mat, T1, T2); + aom_free(a); + return 0; +} + +static int find_vertrapezoid(int np, double *pts1, double *pts2, double *mat) { + const int np3 = np * 3; + double *a = (double *)aom_malloc(sizeof(*a) * np3 * 14); + double *U = a + np3 * 7; + double S[7], V[7 * 7], H[9]; + int i, mini; + double sx, sy, dx, dy; + double T1[9], T2[9]; + + normalize_homography(pts1, np, T1); + normalize_homography(pts2, np, T2); + + for (i = 0; i < np; ++i) { + dx = *(pts2++); + dy = *(pts2++); + sx = *(pts1++); + sy = *(pts1++); + + a[i * 3 * 7 + 0] = a[i * 3 * 7 + 1] = 0; + a[i * 3 * 7 + 2] = -sx; + a[i * 3 * 7 + 3] = -sy; + a[i * 3 * 7 + 4] = -1; + a[i * 3 * 7 + 5] = dy * sx; + a[i * 3 * 7 + 6] = dy; + + a[(i * 3 + 1) * 7 + 0] = sx; + a[(i * 3 + 1) * 7 + 1] = 1; + a[(i * 3 + 1) * 7 + 2] = a[(i * 3 + 1) * 7 + 3] = a[(i * 3 + 1) * 7 + 4] = + 0; + a[(i * 3 + 1) * 7 + 5] = -dx * sx; + a[(i * 3 + 1) * 7 + 6] = -dx; + + a[(i * 3 + 2) * 7 + 0] = -dy * sx; + a[(i * 3 + 2) * 7 + 1] = -dy; + a[(i * 3 + 2) * 7 + 2] = dx * sx; + a[(i * 3 + 2) * 7 + 3] = dx * sy; + a[(i * 3 + 2) * 7 + 4] = dx; + a[(i * 3 + 2) * 7 + 5] = a[(i * 3 + 2) * 7 + 6] = 0; + } + if (SVD(U, S, V, a, np3, 7)) { + aom_free(a); + return 1; + } else { + double minS = 1e12; + mini = -1; + for (i = 0; i < 7; ++i) { + if (S[i] < minS) { + minS = S[i]; + mini = i; + } + } + } + H[1] = H[7] = 0; + for (i = 0; i < 1; i++) H[i] = V[i * 7 + mini]; + for (; i < 6; i++) H[i + 1] = V[i * 7 + mini]; + for (; i < 7; i++) H[i + 2] = V[i * 7 + mini]; + + denormalize_homography_reorder(H, T1, T2); + aom_free(a); + if (H[8] == 0.0) { + return 1; + } else { + // normalize + double f = 1.0 / H[8]; + for (i = 0; i < 8; i++) mat[i] = f * H[i]; + } + return 0; +} + +static int find_hortrapezoid(int np, double *pts1, double *pts2, double *mat) { + const int np3 = np * 3; + double *a = (double *)aom_malloc(sizeof(*a) * np3 * 14); + double *U = a + np3 * 7; + double S[7], V[7 * 7], H[9]; + int i, mini; + double sx, sy, dx, dy; + double T1[9], T2[9]; + + normalize_homography(pts1, np, T1); + normalize_homography(pts2, np, T2); + + for (i = 0; i < np; ++i) { + dx = *(pts2++); + dy = *(pts2++); + sx = *(pts1++); + sy = *(pts1++); + + a[i * 3 * 7 + 0] = a[i * 3 * 7 + 1] = a[i * 3 * 7 + 2] = 0; + a[i * 3 * 7 + 3] = -sy; + a[i * 3 * 7 + 4] = -1; + a[i * 3 * 7 + 5] = dy * sy; + a[i * 3 * 7 + 6] = dy; + + a[(i * 3 + 1) * 7 + 0] = sx; + a[(i * 3 + 1) * 7 + 1] = sy; + a[(i * 3 + 1) * 7 + 2] = 1; + a[(i * 3 + 1) * 7 + 3] = a[(i * 3 + 1) * 7 + 4] = 0; + a[(i * 3 + 1) * 7 + 5] = -dx * sy; + a[(i * 3 + 1) * 7 + 6] = -dx; + + a[(i * 3 + 2) * 7 + 0] = -dy * sx; + a[(i * 3 + 2) * 7 + 1] = -dy * sy; + a[(i * 3 + 2) * 7 + 2] = -dy; + a[(i * 3 + 2) * 7 + 3] = dx * sy; + a[(i * 3 + 2) * 7 + 4] = dx; + a[(i * 3 + 2) * 7 + 5] = a[(i * 3 + 2) * 7 + 6] = 0; + } + + if (SVD(U, S, V, a, np3, 7)) { + aom_free(a); + return 1; + } else { + double minS = 1e12; + mini = -1; + for (i = 0; i < 7; ++i) { + if (S[i] < minS) { + minS = S[i]; + mini = i; + } + } + } + H[3] = H[6] = 0; + for (i = 0; i < 3; i++) H[i] = V[i * 7 + mini]; + for (; i < 5; i++) H[i + 1] = V[i * 7 + mini]; + for (; i < 7; i++) H[i + 2] = V[i * 7 + mini]; + + denormalize_homography_reorder(H, T1, T2); + aom_free(a); + if (H[8] == 0.0) { + return 1; + } else { + // normalize + double f = 1.0 / H[8]; + for (i = 0; i < 8; i++) mat[i] = f * H[i]; + } + return 0; +} + +static int find_homography(int np, double *pts1, double *pts2, double *mat) { + // Implemented from Peter Kovesi's normalized implementation + const int np3 = np * 3; + double *a = (double *)aom_malloc(sizeof(*a) * np3 * 18); + double *U = a + np3 * 9; + double S[9], V[9 * 9], H[9]; + int i, mini; + double sx, sy, dx, dy; + double T1[9], T2[9]; + + normalize_homography(pts1, np, T1); + normalize_homography(pts2, np, T2); + + for (i = 0; i < np; ++i) { + dx = *(pts2++); + dy = *(pts2++); + sx = *(pts1++); + sy = *(pts1++); + + a[i * 3 * 9 + 0] = a[i * 3 * 9 + 1] = a[i * 3 * 9 + 2] = 0; + a[i * 3 * 9 + 3] = -sx; + a[i * 3 * 9 + 4] = -sy; + a[i * 3 * 9 + 5] = -1; + a[i * 3 * 9 + 6] = dy * sx; + a[i * 3 * 9 + 7] = dy * sy; + a[i * 3 * 9 + 8] = dy; + + a[(i * 3 + 1) * 9 + 0] = sx; + a[(i * 3 + 1) * 9 + 1] = sy; + a[(i * 3 + 1) * 9 + 2] = 1; + a[(i * 3 + 1) * 9 + 3] = a[(i * 3 + 1) * 9 + 4] = a[(i * 3 + 1) * 9 + 5] = + 0; + a[(i * 3 + 1) * 9 + 6] = -dx * sx; + a[(i * 3 + 1) * 9 + 7] = -dx * sy; + a[(i * 3 + 1) * 9 + 8] = -dx; + + a[(i * 3 + 2) * 9 + 0] = -dy * sx; + a[(i * 3 + 2) * 9 + 1] = -dy * sy; + a[(i * 3 + 2) * 9 + 2] = -dy; + a[(i * 3 + 2) * 9 + 3] = dx * sx; + a[(i * 3 + 2) * 9 + 4] = dx * sy; + a[(i * 3 + 2) * 9 + 5] = dx; + a[(i * 3 + 2) * 9 + 6] = a[(i * 3 + 2) * 9 + 7] = a[(i * 3 + 2) * 9 + 8] = + 0; + } + + if (SVD(U, S, V, a, np3, 9)) { + aom_free(a); + return 1; + } else { + double minS = 1e12; + mini = -1; + for (i = 0; i < 9; ++i) { + if (S[i] < minS) { + minS = S[i]; + mini = i; + } + } + } + + for (i = 0; i < 9; i++) H[i] = V[i * 9 + mini]; + denormalize_homography_reorder(H, T1, T2); + aom_free(a); + if (H[8] == 0.0) { + return 1; + } else { + // normalize + double f = 1.0 / H[8]; + for (i = 0; i < 8; i++) mat[i] = f * H[i]; + } + return 0; +} + +static int get_rand_indices(int npoints, int minpts, int *indices, + unsigned int *seed) { + int i, j; + int ptr = rand_r(seed) % npoints; + if (minpts > npoints) return 0; + indices[0] = ptr; + ptr = (ptr == npoints - 1 ? 0 : ptr + 1); + i = 1; + while (i < minpts) { + int index = rand_r(seed) % npoints; + while (index) { + ptr = (ptr == npoints - 1 ? 0 : ptr + 1); + for (j = 0; j < i; ++j) { + if (indices[j] == ptr) break; + } + if (j == i) index--; + } + indices[i++] = ptr; + } + return 1; +} + +typedef struct { + int num_inliers; + double variance; + int *inlier_indices; +} RANSAC_MOTION; + +// Return -1 if 'a' is a better motion, 1 if 'b' is better, 0 otherwise. +static int compare_motions(const void *arg_a, const void *arg_b) { + const RANSAC_MOTION *motion_a = (RANSAC_MOTION *)arg_a; + const RANSAC_MOTION *motion_b = (RANSAC_MOTION *)arg_b; + + if (motion_a->num_inliers > motion_b->num_inliers) return -1; + if (motion_a->num_inliers < motion_b->num_inliers) return 1; + if (motion_a->variance < motion_b->variance) return -1; + if (motion_a->variance > motion_b->variance) return 1; + return 0; +} + +static int is_better_motion(const RANSAC_MOTION *motion_a, + const RANSAC_MOTION *motion_b) { + return compare_motions(motion_a, motion_b) < 0; +} + +static void copy_points_at_indices(double *dest, const double *src, + const int *indices, int num_points) { + for (int i = 0; i < num_points; ++i) { + const int index = indices[i]; + dest[i * 2] = src[index * 2]; + dest[i * 2 + 1] = src[index * 2 + 1]; + } +} + +static const double kInfiniteVariance = 1e12; + +static void clear_motion(RANSAC_MOTION *motion, int num_points) { + motion->num_inliers = 0; + motion->variance = kInfiniteVariance; + memset(motion->inlier_indices, 0, + sizeof(*motion->inlier_indices * num_points)); +} + +static int ransac(const int *matched_points, int npoints, + int *num_inliers_by_motion, double *params_by_motion, + int num_desired_motions, const int minpts, + IsDegenerateFunc is_degenerate, + FindTransformationFunc find_transformation, + ProjectPointsDoubleFunc projectpoints) { + static const double PROBABILITY_REQUIRED = 0.9; + static const double EPS = 1e-12; + + int N = 10000, trial_count = 0; + int i = 0; + int ret_val = 0; + + unsigned int seed = (unsigned int)npoints; + + int indices[MAX_MINPTS] = { 0 }; + + double *points1, *points2; + double *corners1, *corners2; + double *image1_coord; + + // Store information for the num_desired_motions best transformations found + // and the worst motion among them, as well as the motion currently under + // consideration. + RANSAC_MOTION *motions, *worst_kept_motion = NULL; + RANSAC_MOTION current_motion; + + // Store the parameters and the indices of the inlier points for the motion + // currently under consideration. + double params_this_motion[MAX_PARAMDIM]; + + double *cnp1, *cnp2; + + if (npoints < minpts * MINPTS_MULTIPLIER || npoints == 0) { + return 1; + } + + points1 = (double *)aom_malloc(sizeof(*points1) * npoints * 2); + points2 = (double *)aom_malloc(sizeof(*points2) * npoints * 2); + corners1 = (double *)aom_malloc(sizeof(*corners1) * npoints * 2); + corners2 = (double *)aom_malloc(sizeof(*corners2) * npoints * 2); + image1_coord = (double *)aom_malloc(sizeof(*image1_coord) * npoints * 2); + + motions = + (RANSAC_MOTION *)aom_malloc(sizeof(RANSAC_MOTION) * num_desired_motions); + for (i = 0; i < num_desired_motions; ++i) { + motions[i].inlier_indices = + (int *)aom_malloc(sizeof(*motions->inlier_indices) * npoints); + clear_motion(motions + i, npoints); + } + current_motion.inlier_indices = + (int *)aom_malloc(sizeof(*current_motion.inlier_indices) * npoints); + clear_motion(¤t_motion, npoints); + + worst_kept_motion = motions; + + if (!(points1 && points2 && corners1 && corners2 && image1_coord && motions && + current_motion.inlier_indices)) { + ret_val = 1; + goto finish_ransac; + } + + cnp1 = corners1; + cnp2 = corners2; + for (i = 0; i < npoints; ++i) { + *(cnp1++) = *(matched_points++); + *(cnp1++) = *(matched_points++); + *(cnp2++) = *(matched_points++); + *(cnp2++) = *(matched_points++); + } + + while (N > trial_count) { + double sum_distance = 0.0; + double sum_distance_squared = 0.0; + + clear_motion(¤t_motion, npoints); + + int degenerate = 1; + int num_degenerate_iter = 0; + + while (degenerate) { + num_degenerate_iter++; + if (!get_rand_indices(npoints, minpts, indices, &seed)) { + ret_val = 1; + goto finish_ransac; + } + + copy_points_at_indices(points1, corners1, indices, minpts); + copy_points_at_indices(points2, corners2, indices, minpts); + + degenerate = is_degenerate(points1); + if (num_degenerate_iter > MAX_DEGENERATE_ITER) { + ret_val = 1; + goto finish_ransac; + } + } + + if (find_transformation(minpts, points1, points2, params_this_motion)) { + trial_count++; + continue; + } + + projectpoints(params_this_motion, corners1, image1_coord, npoints, 2, 2); + + for (i = 0; i < npoints; ++i) { + double dx = image1_coord[i * 2] - corners2[i * 2]; + double dy = image1_coord[i * 2 + 1] - corners2[i * 2 + 1]; + double distance = sqrt(dx * dx + dy * dy); + + if (distance < INLIER_THRESHOLD) { + current_motion.inlier_indices[current_motion.num_inliers++] = i; + sum_distance += distance; + sum_distance_squared += distance * distance; + } + } + + if (current_motion.num_inliers >= worst_kept_motion->num_inliers && + current_motion.num_inliers > 1) { + int temp; + double fracinliers, pNoOutliers, mean_distance; + mean_distance = sum_distance / ((double)current_motion.num_inliers); + current_motion.variance = + sum_distance_squared / ((double)current_motion.num_inliers - 1.0) - + mean_distance * mean_distance * ((double)current_motion.num_inliers) / + ((double)current_motion.num_inliers - 1.0); + if (is_better_motion(¤t_motion, worst_kept_motion)) { + // This motion is better than the worst currently kept motion. Remember + // the inlier points and variance. The parameters for each kept motion + // will be recomputed later using only the inliers. + worst_kept_motion->num_inliers = current_motion.num_inliers; + worst_kept_motion->variance = current_motion.variance; + memcpy(worst_kept_motion->inlier_indices, current_motion.inlier_indices, + sizeof(*current_motion.inlier_indices) * npoints); + + assert(npoints > 0); + fracinliers = (double)current_motion.num_inliers / (double)npoints; + pNoOutliers = 1 - pow(fracinliers, minpts); + pNoOutliers = fmax(EPS, pNoOutliers); + pNoOutliers = fmin(1 - EPS, pNoOutliers); + temp = (int)(log(1.0 - PROBABILITY_REQUIRED) / log(pNoOutliers)); + + if (temp > 0 && temp < N) { + N = AOMMAX(temp, MIN_TRIALS); + } + + // Determine the new worst kept motion and its num_inliers and variance. + for (i = 0; i < num_desired_motions; ++i) { + if (is_better_motion(worst_kept_motion, &motions[i])) { + worst_kept_motion = &motions[i]; + } + } + } + } + trial_count++; + } + + // Sort the motions, best first. + qsort(motions, num_desired_motions, sizeof(RANSAC_MOTION), compare_motions); + + // Recompute the motions using only the inliers. + for (i = 0; i < num_desired_motions; ++i) { + copy_points_at_indices(points1, corners1, motions[i].inlier_indices, + motions[i].num_inliers); + copy_points_at_indices(points2, corners2, motions[i].inlier_indices, + motions[i].num_inliers); + + find_transformation(motions[i].num_inliers, points1, points2, + params_by_motion + (MAX_PARAMDIM - 1) * i); + num_inliers_by_motion[i] = motions[i].num_inliers; + } + +finish_ransac: + aom_free(points1); + aom_free(points2); + aom_free(corners1); + aom_free(corners2); + aom_free(image1_coord); + aom_free(current_motion.inlier_indices); + for (i = 0; i < num_desired_motions; ++i) { + aom_free(motions[i].inlier_indices); + } + aom_free(motions); + + return ret_val; +} + +static int is_collinear3(double *p1, double *p2, double *p3) { + static const double collinear_eps = 1e-3; + const double v = + (p2[0] - p1[0]) * (p3[1] - p1[1]) - (p2[1] - p1[1]) * (p3[0] - p1[0]); + return fabs(v) < collinear_eps; +} + +static int is_degenerate_translation(double *p) { + return (p[0] - p[2]) * (p[0] - p[2]) + (p[1] - p[3]) * (p[1] - p[3]) <= 2; +} + +static int is_degenerate_affine(double *p) { + return is_collinear3(p, p + 2, p + 4); +} + +static int is_degenerate_homography(double *p) { + return is_collinear3(p, p + 2, p + 4) || is_collinear3(p, p + 2, p + 6) || + is_collinear3(p, p + 4, p + 6) || is_collinear3(p + 2, p + 4, p + 6); +} + +int ransac_translation(int *matched_points, int npoints, + int *num_inliers_by_motion, double *params_by_motion, + int num_desired_motions) { + return ransac(matched_points, npoints, num_inliers_by_motion, + params_by_motion, num_desired_motions, 3, + is_degenerate_translation, find_translation, + project_points_double_translation); +} + +int ransac_rotzoom(int *matched_points, int npoints, int *num_inliers_by_motion, + double *params_by_motion, int num_desired_motions) { + return ransac(matched_points, npoints, num_inliers_by_motion, + params_by_motion, num_desired_motions, 3, is_degenerate_affine, + find_rotzoom, project_points_double_rotzoom); +} + +int ransac_affine(int *matched_points, int npoints, int *num_inliers_by_motion, + double *params_by_motion, int num_desired_motions) { + return ransac(matched_points, npoints, num_inliers_by_motion, + params_by_motion, num_desired_motions, 3, is_degenerate_affine, + find_affine, project_points_double_affine); +} + +int ransac_homography(int *matched_points, int npoints, + int *num_inliers_by_motion, double *params_by_motion, + int num_desired_motions) { + return ransac(matched_points, npoints, num_inliers_by_motion, + params_by_motion, num_desired_motions, 4, + is_degenerate_homography, find_homography, + project_points_double_homography); +} + +int ransac_hortrapezoid(int *matched_points, int npoints, + int *num_inliers_by_motion, double *params_by_motion, + int num_desired_motions) { + return ransac(matched_points, npoints, num_inliers_by_motion, + params_by_motion, num_desired_motions, 4, + is_degenerate_homography, find_hortrapezoid, + project_points_double_hortrapezoid); +} + +int ransac_vertrapezoid(int *matched_points, int npoints, + int *num_inliers_by_motion, double *params_by_motion, + int num_desired_motions) { + return ransac(matched_points, npoints, num_inliers_by_motion, + params_by_motion, num_desired_motions, 4, + is_degenerate_homography, find_vertrapezoid, + project_points_double_vertrapezoid); +} diff --git a/third_party/aom/av1/encoder/ransac.h b/third_party/aom/av1/encoder/ransac.h new file mode 100644 index 0000000000..f611add369 --- /dev/null +++ b/third_party/aom/av1/encoder/ransac.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_RANSAC_H_ +#define AV1_ENCODER_RANSAC_H_ + +#include +#include +#include +#include + +#include "av1/common/warped_motion.h" + +typedef int (*RansacFunc)(int *matched_points, int npoints, + int *num_inliers_by_motion, double *params_by_motion, + int num_motions); + +/* Each of these functions fits a motion model from a set of + corresponding points in 2 frames using RANSAC. */ +int ransac_homography(int *matched_points, int npoints, + int *num_inliers_by_motion, double *params_by_motion, + int num_motions); +int ransac_affine(int *matched_points, int npoints, int *num_inliers_by_motion, + double *params_by_motion, int num_motions); +int ransac_hortrapezoid(int *matched_points, int npoints, + int *num_inliers_by_motion, double *params_by_motion, + int num_motions); +int ransac_vertrapezoid(int *matched_points, int npoints, + int *num_inliers_by_motion, double *params_by_motion, + int num_motions); +int ransac_rotzoom(int *matched_points, int npoints, int *num_inliers_by_motion, + double *params_by_motion, int num_motions); +int ransac_translation(int *matched_points, int npoints, + int *num_inliers_by_motion, double *params_by_motion, + int num_motions); +#endif // AV1_ENCODER_RANSAC_H_ diff --git a/third_party/aom/av1/encoder/ratectrl.c b/third_party/aom/av1/encoder/ratectrl.c new file mode 100644 index 0000000000..1f2ea3606f --- /dev/null +++ b/third_party/aom/av1/encoder/ratectrl.c @@ -0,0 +1,1759 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include +#include +#include + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" +#include "aom_ports/system_state.h" + +#include "av1/common/alloccommon.h" +#include "av1/encoder/aq_cyclicrefresh.h" +#include "av1/common/common.h" +#include "av1/common/entropymode.h" +#include "av1/common/quant_common.h" +#include "av1/common/seg_common.h" + +#include "av1/encoder/encodemv.h" +#include "av1/encoder/ratectrl.h" + +// Max rate target for 1080P and below encodes under normal circumstances +// (1920 * 1080 / (16 * 16)) * MAX_MB_RATE bits per MB +#define MAX_MB_RATE 250 +#define MAXRATE_1080P 2025000 + +#define DEFAULT_KF_BOOST 2000 +#define DEFAULT_GF_BOOST 2000 + +#define MIN_BPB_FACTOR 0.005 +#define MAX_BPB_FACTOR 50 + +#define FRAME_OVERHEAD_BITS 200 +#if CONFIG_HIGHBITDEPTH +#define ASSIGN_MINQ_TABLE(bit_depth, name) \ + do { \ + switch (bit_depth) { \ + case AOM_BITS_8: name = name##_8; break; \ + case AOM_BITS_10: name = name##_10; break; \ + case AOM_BITS_12: name = name##_12; break; \ + default: \ + assert(0 && \ + "bit_depth should be AOM_BITS_8, AOM_BITS_10" \ + " or AOM_BITS_12"); \ + name = NULL; \ + } \ + } while (0) +#else +#define ASSIGN_MINQ_TABLE(bit_depth, name) \ + do { \ + (void)bit_depth; \ + name = name##_8; \ + } while (0) +#endif + +// Tables relating active max Q to active min Q +static int kf_low_motion_minq_8[QINDEX_RANGE]; +static int kf_high_motion_minq_8[QINDEX_RANGE]; +static int arfgf_low_motion_minq_8[QINDEX_RANGE]; +static int arfgf_high_motion_minq_8[QINDEX_RANGE]; +static int inter_minq_8[QINDEX_RANGE]; +static int rtc_minq_8[QINDEX_RANGE]; + +#if CONFIG_HIGHBITDEPTH +static int kf_low_motion_minq_10[QINDEX_RANGE]; +static int kf_high_motion_minq_10[QINDEX_RANGE]; +static int arfgf_low_motion_minq_10[QINDEX_RANGE]; +static int arfgf_high_motion_minq_10[QINDEX_RANGE]; +static int inter_minq_10[QINDEX_RANGE]; +static int rtc_minq_10[QINDEX_RANGE]; +static int kf_low_motion_minq_12[QINDEX_RANGE]; +static int kf_high_motion_minq_12[QINDEX_RANGE]; +static int arfgf_low_motion_minq_12[QINDEX_RANGE]; +static int arfgf_high_motion_minq_12[QINDEX_RANGE]; +static int inter_minq_12[QINDEX_RANGE]; +static int rtc_minq_12[QINDEX_RANGE]; +#endif + +static int gf_high = 2000; +static int gf_low = 400; +static int kf_high = 5000; +static int kf_low = 400; + +// Functions to compute the active minq lookup table entries based on a +// formulaic approach to facilitate easier adjustment of the Q tables. +// The formulae were derived from computing a 3rd order polynomial best +// fit to the original data (after plotting real maxq vs minq (not q index)) +static int get_minq_index(double maxq, double x3, double x2, double x1, + aom_bit_depth_t bit_depth) { + int i; + const double minqtarget = AOMMIN(((x3 * maxq + x2) * maxq + x1) * maxq, maxq); + + // Special case handling to deal with the step from q2.0 + // down to lossless mode represented by q 1.0. + if (minqtarget <= 2.0) return 0; + + for (i = 0; i < QINDEX_RANGE; i++) { + if (minqtarget <= av1_convert_qindex_to_q(i, bit_depth)) return i; + } + + return QINDEX_RANGE - 1; +} + +static void init_minq_luts(int *kf_low_m, int *kf_high_m, int *arfgf_low, + int *arfgf_high, int *inter, int *rtc, + aom_bit_depth_t bit_depth) { + int i; + for (i = 0; i < QINDEX_RANGE; i++) { + const double maxq = av1_convert_qindex_to_q(i, bit_depth); + kf_low_m[i] = get_minq_index(maxq, 0.000001, -0.0004, 0.150, bit_depth); + kf_high_m[i] = get_minq_index(maxq, 0.0000021, -0.00125, 0.55, bit_depth); + arfgf_low[i] = get_minq_index(maxq, 0.0000015, -0.0009, 0.30, bit_depth); + arfgf_high[i] = get_minq_index(maxq, 0.0000021, -0.00125, 0.55, bit_depth); + inter[i] = get_minq_index(maxq, 0.00000271, -0.00113, 0.90, bit_depth); + rtc[i] = get_minq_index(maxq, 0.00000271, -0.00113, 0.70, bit_depth); + } +} + +void av1_rc_init_minq_luts(void) { + init_minq_luts(kf_low_motion_minq_8, kf_high_motion_minq_8, + arfgf_low_motion_minq_8, arfgf_high_motion_minq_8, + inter_minq_8, rtc_minq_8, AOM_BITS_8); +#if CONFIG_HIGHBITDEPTH + init_minq_luts(kf_low_motion_minq_10, kf_high_motion_minq_10, + arfgf_low_motion_minq_10, arfgf_high_motion_minq_10, + inter_minq_10, rtc_minq_10, AOM_BITS_10); + init_minq_luts(kf_low_motion_minq_12, kf_high_motion_minq_12, + arfgf_low_motion_minq_12, arfgf_high_motion_minq_12, + inter_minq_12, rtc_minq_12, AOM_BITS_12); +#endif +} + +// These functions use formulaic calculations to make playing with the +// quantizer tables easier. If necessary they can be replaced by lookup +// tables if and when things settle down in the experimental bitstream +double av1_convert_qindex_to_q(int qindex, aom_bit_depth_t bit_depth) { +// Convert the index to a real Q value (scaled down to match old Q values) +#if CONFIG_HIGHBITDEPTH + switch (bit_depth) { + case AOM_BITS_8: return av1_ac_quant(qindex, 0, bit_depth) / 4.0; + case AOM_BITS_10: return av1_ac_quant(qindex, 0, bit_depth) / 16.0; + case AOM_BITS_12: return av1_ac_quant(qindex, 0, bit_depth) / 64.0; + default: + assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12"); + return -1.0; + } +#else + return av1_ac_quant(qindex, 0, bit_depth) / 4.0; +#endif +} + +int av1_rc_bits_per_mb(FRAME_TYPE frame_type, int qindex, + double correction_factor, aom_bit_depth_t bit_depth) { + const double q = av1_convert_qindex_to_q(qindex, bit_depth); + int enumerator = frame_type == KEY_FRAME ? 2700000 : 1800000; + + assert(correction_factor <= MAX_BPB_FACTOR && + correction_factor >= MIN_BPB_FACTOR); + + // q based adjustment to baseline enumerator + enumerator += (int)(enumerator * q) >> 12; + return (int)(enumerator * correction_factor / q); +} + +int av1_estimate_bits_at_q(FRAME_TYPE frame_type, int q, int mbs, + double correction_factor, + aom_bit_depth_t bit_depth) { + const int bpm = + (int)(av1_rc_bits_per_mb(frame_type, q, correction_factor, bit_depth)); + return AOMMAX(FRAME_OVERHEAD_BITS, + (int)((uint64_t)bpm * mbs) >> BPER_MB_NORMBITS); +} + +int av1_rc_clamp_pframe_target_size(const AV1_COMP *const cpi, int target) { + const RATE_CONTROL *rc = &cpi->rc; + const AV1EncoderConfig *oxcf = &cpi->oxcf; + const int min_frame_target = + AOMMAX(rc->min_frame_bandwidth, rc->avg_frame_bandwidth >> 5); +// Clip the frame target to the minimum setup value. +#if CONFIG_EXT_REFS + if (cpi->rc.is_src_frame_alt_ref) { +#else + if (cpi->refresh_golden_frame && rc->is_src_frame_alt_ref) { +#endif // CONFIG_EXT_REFS + // If there is an active ARF at this location use the minimum + // bits on this frame even if it is a constructed arf. + // The active maximum quantizer insures that an appropriate + // number of bits will be spent if needed for constructed ARFs. + target = min_frame_target; + } else if (target < min_frame_target) { + target = min_frame_target; + } + + // Clip the frame target to the maximum allowed value. + if (target > rc->max_frame_bandwidth) target = rc->max_frame_bandwidth; + if (oxcf->rc_max_inter_bitrate_pct) { + const int max_rate = + rc->avg_frame_bandwidth * oxcf->rc_max_inter_bitrate_pct / 100; + target = AOMMIN(target, max_rate); + } + + return target; +} + +int av1_rc_clamp_iframe_target_size(const AV1_COMP *const cpi, int target) { + const RATE_CONTROL *rc = &cpi->rc; + const AV1EncoderConfig *oxcf = &cpi->oxcf; + if (oxcf->rc_max_intra_bitrate_pct) { + const int max_rate = + rc->avg_frame_bandwidth * oxcf->rc_max_intra_bitrate_pct / 100; + target = AOMMIN(target, max_rate); + } + if (target > rc->max_frame_bandwidth) target = rc->max_frame_bandwidth; + return target; +} + +// Update the buffer level: leaky bucket model. +static void update_buffer_level(AV1_COMP *cpi, int encoded_frame_size) { + const AV1_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; + +// Non-viewable frames are a special case and are treated as pure overhead. +#if CONFIG_EXT_REFS + // TODO(zoeliu): To further explore whether we should treat BWDREF_FRAME + // differently, since it is a no-show frame. + if (!cm->show_frame && !rc->is_bwd_ref_frame) +#else + if (!cm->show_frame) +#endif // CONFIG_EXT_REFS + rc->bits_off_target -= encoded_frame_size; + else + rc->bits_off_target += rc->avg_frame_bandwidth - encoded_frame_size; + + // Clip the buffer level to the maximum specified buffer size. + rc->bits_off_target = AOMMIN(rc->bits_off_target, rc->maximum_buffer_size); + rc->buffer_level = rc->bits_off_target; +} + +int av1_rc_get_default_min_gf_interval(int width, int height, + double framerate) { + // Assume we do not need any constraint lower than 4K 20 fps + static const double factor_safe = 3840 * 2160 * 20.0; + const double factor = width * height * framerate; + const int default_interval = + clamp((int)(framerate * 0.125), MIN_GF_INTERVAL, MAX_GF_INTERVAL); + + if (factor <= factor_safe) + return default_interval; + else + return AOMMAX(default_interval, + (int)(MIN_GF_INTERVAL * factor / factor_safe + 0.5)); + // Note this logic makes: + // 4K24: 5 + // 4K30: 6 + // 4K60: 12 +} + +int av1_rc_get_default_max_gf_interval(double framerate, int min_gf_interval) { + int interval = AOMMIN(MAX_GF_INTERVAL, (int)(framerate * 0.75)); + interval += (interval & 0x01); // Round to even value + return AOMMAX(interval, min_gf_interval); +} + +void av1_rc_init(const AV1EncoderConfig *oxcf, int pass, RATE_CONTROL *rc) { + int i; + + if (pass == 0 && oxcf->rc_mode == AOM_CBR) { + rc->avg_frame_qindex[KEY_FRAME] = oxcf->worst_allowed_q; + rc->avg_frame_qindex[INTER_FRAME] = oxcf->worst_allowed_q; + } else { + rc->avg_frame_qindex[KEY_FRAME] = + (oxcf->worst_allowed_q + oxcf->best_allowed_q) / 2; + rc->avg_frame_qindex[INTER_FRAME] = + (oxcf->worst_allowed_q + oxcf->best_allowed_q) / 2; + } + + rc->last_q[KEY_FRAME] = oxcf->best_allowed_q; + rc->last_q[INTER_FRAME] = oxcf->worst_allowed_q; + + rc->buffer_level = rc->starting_buffer_level; + rc->bits_off_target = rc->starting_buffer_level; + + rc->rolling_target_bits = rc->avg_frame_bandwidth; + rc->rolling_actual_bits = rc->avg_frame_bandwidth; + rc->long_rolling_target_bits = rc->avg_frame_bandwidth; + rc->long_rolling_actual_bits = rc->avg_frame_bandwidth; + + rc->total_actual_bits = 0; + rc->total_target_bits = 0; + rc->total_target_vs_actual = 0; + + rc->frames_since_key = 8; // Sensible default for first frame. + rc->this_key_frame_forced = 0; + rc->next_key_frame_forced = 0; + rc->source_alt_ref_pending = 0; + rc->source_alt_ref_active = 0; + + rc->frames_till_gf_update_due = 0; + rc->ni_av_qi = oxcf->worst_allowed_q; + rc->ni_tot_qi = 0; + rc->ni_frames = 0; + + rc->tot_q = 0.0; + rc->avg_q = av1_convert_qindex_to_q(oxcf->worst_allowed_q, oxcf->bit_depth); + + for (i = 0; i < RATE_FACTOR_LEVELS; ++i) { + rc->rate_correction_factors[i] = 1.0; + } + + rc->min_gf_interval = oxcf->min_gf_interval; + rc->max_gf_interval = oxcf->max_gf_interval; + if (rc->min_gf_interval == 0) + rc->min_gf_interval = av1_rc_get_default_min_gf_interval( + oxcf->width, oxcf->height, oxcf->init_framerate); + if (rc->max_gf_interval == 0) + rc->max_gf_interval = av1_rc_get_default_max_gf_interval( + oxcf->init_framerate, rc->min_gf_interval); + rc->baseline_gf_interval = (rc->min_gf_interval + rc->max_gf_interval) / 2; +} + +int av1_rc_drop_frame(AV1_COMP *cpi) { + const AV1EncoderConfig *oxcf = &cpi->oxcf; + RATE_CONTROL *const rc = &cpi->rc; + + if (!oxcf->drop_frames_water_mark) { + return 0; + } else { + if (rc->buffer_level < 0) { + // Always drop if buffer is below 0. + return 1; + } else { + // If buffer is below drop_mark, for now just drop every other frame + // (starting with the next frame) until it increases back over drop_mark. + int drop_mark = + (int)(oxcf->drop_frames_water_mark * rc->optimal_buffer_level / 100); + if ((rc->buffer_level > drop_mark) && (rc->decimation_factor > 0)) { + --rc->decimation_factor; + } else if (rc->buffer_level <= drop_mark && rc->decimation_factor == 0) { + rc->decimation_factor = 1; + } + if (rc->decimation_factor > 0) { + if (rc->decimation_count > 0) { + --rc->decimation_count; + return 1; + } else { + rc->decimation_count = rc->decimation_factor; + return 0; + } + } else { + rc->decimation_count = 0; + return 0; + } + } + } +} + +static double get_rate_correction_factor(const AV1_COMP *cpi) { + const RATE_CONTROL *const rc = &cpi->rc; + double rcf; + + if (cpi->common.frame_type == KEY_FRAME) { + rcf = rc->rate_correction_factors[KF_STD]; + } else if (cpi->oxcf.pass == 2) { + RATE_FACTOR_LEVEL rf_lvl = + cpi->twopass.gf_group.rf_level[cpi->twopass.gf_group.index]; + rcf = rc->rate_correction_factors[rf_lvl]; + } else { + if ((cpi->refresh_alt_ref_frame || cpi->refresh_golden_frame) && + !rc->is_src_frame_alt_ref && + (cpi->oxcf.rc_mode != AOM_CBR || cpi->oxcf.gf_cbr_boost_pct > 20)) + rcf = rc->rate_correction_factors[GF_ARF_STD]; + else + rcf = rc->rate_correction_factors[INTER_NORMAL]; + } + rcf *= rcf_mult[rc->frame_size_selector]; + return fclamp(rcf, MIN_BPB_FACTOR, MAX_BPB_FACTOR); +} + +static void set_rate_correction_factor(AV1_COMP *cpi, double factor) { + RATE_CONTROL *const rc = &cpi->rc; + + // Normalize RCF to account for the size-dependent scaling factor. + factor /= rcf_mult[cpi->rc.frame_size_selector]; + + factor = fclamp(factor, MIN_BPB_FACTOR, MAX_BPB_FACTOR); + + if (cpi->common.frame_type == KEY_FRAME) { + rc->rate_correction_factors[KF_STD] = factor; + } else if (cpi->oxcf.pass == 2) { + RATE_FACTOR_LEVEL rf_lvl = + cpi->twopass.gf_group.rf_level[cpi->twopass.gf_group.index]; + rc->rate_correction_factors[rf_lvl] = factor; + } else { + if ((cpi->refresh_alt_ref_frame || cpi->refresh_golden_frame) && + !rc->is_src_frame_alt_ref && + (cpi->oxcf.rc_mode != AOM_CBR || cpi->oxcf.gf_cbr_boost_pct > 20)) + rc->rate_correction_factors[GF_ARF_STD] = factor; + else + rc->rate_correction_factors[INTER_NORMAL] = factor; + } +} + +void av1_rc_update_rate_correction_factors(AV1_COMP *cpi) { + const AV1_COMMON *const cm = &cpi->common; + int correction_factor = 100; + double rate_correction_factor = get_rate_correction_factor(cpi); + double adjustment_limit; + + int projected_size_based_on_q = 0; + + // Do not update the rate factors for arf overlay frames. + if (cpi->rc.is_src_frame_alt_ref) return; + + // Clear down mmx registers to allow floating point in what follows + aom_clear_system_state(); + + // Work out how big we would have expected the frame to be at this Q given + // the current correction factor. + // Stay in double to avoid int overflow when values are large + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cpi->common.seg.enabled) { + projected_size_based_on_q = + av1_cyclic_refresh_estimate_bits_at_q(cpi, rate_correction_factor); + } else { + projected_size_based_on_q = + av1_estimate_bits_at_q(cpi->common.frame_type, cm->base_qindex, cm->MBs, + rate_correction_factor, cm->bit_depth); + } + // Work out a size correction factor. + if (projected_size_based_on_q > FRAME_OVERHEAD_BITS) + correction_factor = (int)((100 * (int64_t)cpi->rc.projected_frame_size) / + projected_size_based_on_q); + + // More heavily damped adjustment used if we have been oscillating either side + // of target. + if (correction_factor > 0) { + adjustment_limit = + 0.25 + 0.5 * AOMMIN(1, fabs(log10(0.01 * correction_factor))); + } else { + adjustment_limit = 0.75; + } + + cpi->rc.q_2_frame = cpi->rc.q_1_frame; + cpi->rc.q_1_frame = cm->base_qindex; + cpi->rc.rc_2_frame = cpi->rc.rc_1_frame; + if (correction_factor > 110) + cpi->rc.rc_1_frame = -1; + else if (correction_factor < 90) + cpi->rc.rc_1_frame = 1; + else + cpi->rc.rc_1_frame = 0; + + if (correction_factor > 102) { + // We are not already at the worst allowable quality + correction_factor = + (int)(100 + ((correction_factor - 100) * adjustment_limit)); + rate_correction_factor = (rate_correction_factor * correction_factor) / 100; + // Keep rate_correction_factor within limits + if (rate_correction_factor > MAX_BPB_FACTOR) + rate_correction_factor = MAX_BPB_FACTOR; + } else if (correction_factor < 99) { + // We are not already at the best allowable quality + correction_factor = + (int)(100 - ((100 - correction_factor) * adjustment_limit)); + rate_correction_factor = (rate_correction_factor * correction_factor) / 100; + + // Keep rate_correction_factor within limits + if (rate_correction_factor < MIN_BPB_FACTOR) + rate_correction_factor = MIN_BPB_FACTOR; + } + + set_rate_correction_factor(cpi, rate_correction_factor); +} + +int av1_rc_regulate_q(const AV1_COMP *cpi, int target_bits_per_frame, + int active_best_quality, int active_worst_quality) { + const AV1_COMMON *const cm = &cpi->common; + int q = active_worst_quality; + int last_error = INT_MAX; + int i, target_bits_per_mb, bits_per_mb_at_this_q; + const double correction_factor = get_rate_correction_factor(cpi); + + // Calculate required scaling factor based on target frame size and size of + // frame produced using previous Q. + target_bits_per_mb = + (int)((uint64_t)target_bits_per_frame << BPER_MB_NORMBITS) / cm->MBs; + + i = active_best_quality; + + do { + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cm->seg.enabled) { + bits_per_mb_at_this_q = + (int)av1_cyclic_refresh_rc_bits_per_mb(cpi, i, correction_factor); + } else { + bits_per_mb_at_this_q = (int)av1_rc_bits_per_mb( + cm->frame_type, i, correction_factor, cm->bit_depth); + } + + if (bits_per_mb_at_this_q <= target_bits_per_mb) { + if ((target_bits_per_mb - bits_per_mb_at_this_q) <= last_error) + q = i; + else + q = i - 1; + + break; + } else { + last_error = bits_per_mb_at_this_q - target_bits_per_mb; + } + } while (++i <= active_worst_quality); + + // In CBR mode, this makes sure q is between oscillating Qs to prevent + // resonance. + if (cpi->oxcf.rc_mode == AOM_CBR && + (cpi->rc.rc_1_frame * cpi->rc.rc_2_frame == -1) && + cpi->rc.q_1_frame != cpi->rc.q_2_frame) { + q = clamp(q, AOMMIN(cpi->rc.q_1_frame, cpi->rc.q_2_frame), + AOMMAX(cpi->rc.q_1_frame, cpi->rc.q_2_frame)); + } + return q; +} + +static int get_active_quality(int q, int gfu_boost, int low, int high, + int *low_motion_minq, int *high_motion_minq) { + if (gfu_boost > high) { + return low_motion_minq[q]; + } else if (gfu_boost < low) { + return high_motion_minq[q]; + } else { + const int gap = high - low; + const int offset = high - gfu_boost; + const int qdiff = high_motion_minq[q] - low_motion_minq[q]; + const int adjustment = ((offset * qdiff) + (gap >> 1)) / gap; + return low_motion_minq[q] + adjustment; + } +} + +static int get_kf_active_quality(const RATE_CONTROL *const rc, int q, + aom_bit_depth_t bit_depth) { + int *kf_low_motion_minq; + int *kf_high_motion_minq; + ASSIGN_MINQ_TABLE(bit_depth, kf_low_motion_minq); + ASSIGN_MINQ_TABLE(bit_depth, kf_high_motion_minq); + return get_active_quality(q, rc->kf_boost, kf_low, kf_high, + kf_low_motion_minq, kf_high_motion_minq); +} + +static int get_gf_active_quality(const RATE_CONTROL *const rc, int q, + aom_bit_depth_t bit_depth) { + int *arfgf_low_motion_minq; + int *arfgf_high_motion_minq; + ASSIGN_MINQ_TABLE(bit_depth, arfgf_low_motion_minq); + ASSIGN_MINQ_TABLE(bit_depth, arfgf_high_motion_minq); + return get_active_quality(q, rc->gfu_boost, gf_low, gf_high, + arfgf_low_motion_minq, arfgf_high_motion_minq); +} + +static int calc_active_worst_quality_one_pass_vbr(const AV1_COMP *cpi) { + const RATE_CONTROL *const rc = &cpi->rc; + const unsigned int curr_frame = cpi->common.current_video_frame; + int active_worst_quality; + + if (cpi->common.frame_type == KEY_FRAME) { + active_worst_quality = + curr_frame == 0 ? rc->worst_quality : rc->last_q[KEY_FRAME] * 2; + } else { + if (!rc->is_src_frame_alt_ref && + (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame)) { + active_worst_quality = curr_frame == 1 ? rc->last_q[KEY_FRAME] * 5 / 4 + : rc->last_q[INTER_FRAME]; + } else { + active_worst_quality = curr_frame == 1 ? rc->last_q[KEY_FRAME] * 2 + : rc->last_q[INTER_FRAME] * 2; + } + } + return AOMMIN(active_worst_quality, rc->worst_quality); +} + +// Adjust active_worst_quality level based on buffer level. +static int calc_active_worst_quality_one_pass_cbr(const AV1_COMP *cpi) { + // Adjust active_worst_quality: If buffer is above the optimal/target level, + // bring active_worst_quality down depending on fullness of buffer. + // If buffer is below the optimal level, let the active_worst_quality go from + // ambient Q (at buffer = optimal level) to worst_quality level + // (at buffer = critical level). + const AV1_COMMON *const cm = &cpi->common; + const RATE_CONTROL *rc = &cpi->rc; + // Buffer level below which we push active_worst to worst_quality. + int64_t critical_level = rc->optimal_buffer_level >> 3; + int64_t buff_lvl_step = 0; + int adjustment = 0; + int active_worst_quality; + int ambient_qp; + if (cm->frame_type == KEY_FRAME) return rc->worst_quality; + // For ambient_qp we use minimum of avg_frame_qindex[KEY_FRAME/INTER_FRAME] + // for the first few frames following key frame. These are both initialized + // to worst_quality and updated with (3/4, 1/4) average in postencode_update. + // So for first few frames following key, the qp of that key frame is weighted + // into the active_worst_quality setting. + ambient_qp = (cm->current_video_frame < 5) + ? AOMMIN(rc->avg_frame_qindex[INTER_FRAME], + rc->avg_frame_qindex[KEY_FRAME]) + : rc->avg_frame_qindex[INTER_FRAME]; + active_worst_quality = AOMMIN(rc->worst_quality, ambient_qp * 5 / 4); + if (rc->buffer_level > rc->optimal_buffer_level) { + // Adjust down. + // Maximum limit for down adjustment, ~30%. + int max_adjustment_down = active_worst_quality / 3; + if (max_adjustment_down) { + buff_lvl_step = ((rc->maximum_buffer_size - rc->optimal_buffer_level) / + max_adjustment_down); + if (buff_lvl_step) + adjustment = (int)((rc->buffer_level - rc->optimal_buffer_level) / + buff_lvl_step); + active_worst_quality -= adjustment; + } + } else if (rc->buffer_level > critical_level) { + // Adjust up from ambient Q. + if (critical_level) { + buff_lvl_step = (rc->optimal_buffer_level - critical_level); + if (buff_lvl_step) { + adjustment = (int)((rc->worst_quality - ambient_qp) * + (rc->optimal_buffer_level - rc->buffer_level) / + buff_lvl_step); + } + active_worst_quality = ambient_qp + adjustment; + } + } else { + // Set to worst_quality if buffer is below critical level. + active_worst_quality = rc->worst_quality; + } + return active_worst_quality; +} + +static int rc_pick_q_and_bounds_one_pass_cbr(const AV1_COMP *cpi, + int *bottom_index, + int *top_index) { + const AV1_COMMON *const cm = &cpi->common; + const RATE_CONTROL *const rc = &cpi->rc; + int active_best_quality; + int active_worst_quality = calc_active_worst_quality_one_pass_cbr(cpi); + int q; + int *rtc_minq; + ASSIGN_MINQ_TABLE(cm->bit_depth, rtc_minq); + + if (frame_is_intra_only(cm)) { + active_best_quality = rc->best_quality; + // Handle the special case for key frames forced when we have reached + // the maximum key frame interval. Here force the Q to a range + // based on the ambient Q to reduce the risk of popping. + if (rc->this_key_frame_forced) { + int qindex = rc->last_boosted_qindex; + double last_boosted_q = av1_convert_qindex_to_q(qindex, cm->bit_depth); + int delta_qindex = av1_compute_qdelta( + rc, last_boosted_q, (last_boosted_q * 0.75), cm->bit_depth); + active_best_quality = AOMMAX(qindex + delta_qindex, rc->best_quality); + } else if (cm->current_video_frame > 0) { + // not first frame of one pass and kf_boost is set + double q_adj_factor = 1.0; + double q_val; + + active_best_quality = get_kf_active_quality( + rc, rc->avg_frame_qindex[KEY_FRAME], cm->bit_depth); + + // Allow somewhat lower kf minq with small image formats. + if ((cm->width * cm->height) <= (352 * 288)) { + q_adj_factor -= 0.25; + } + + // Convert the adjustment factor to a qindex delta + // on active_best_quality. + q_val = av1_convert_qindex_to_q(active_best_quality, cm->bit_depth); + active_best_quality += + av1_compute_qdelta(rc, q_val, q_val * q_adj_factor, cm->bit_depth); + } + } else if (!rc->is_src_frame_alt_ref && + (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame)) { + // Use the lower of active_worst_quality and recent + // average Q as basis for GF/ARF best Q limit unless last frame was + // a key frame. + if (rc->frames_since_key > 1 && + rc->avg_frame_qindex[INTER_FRAME] < active_worst_quality) { + q = rc->avg_frame_qindex[INTER_FRAME]; + } else { + q = active_worst_quality; + } + active_best_quality = get_gf_active_quality(rc, q, cm->bit_depth); + } else { + // Use the lower of active_worst_quality and recent/average Q. + if (cm->current_video_frame > 1) { + if (rc->avg_frame_qindex[INTER_FRAME] < active_worst_quality) + active_best_quality = rtc_minq[rc->avg_frame_qindex[INTER_FRAME]]; + else + active_best_quality = rtc_minq[active_worst_quality]; + } else { + if (rc->avg_frame_qindex[KEY_FRAME] < active_worst_quality) + active_best_quality = rtc_minq[rc->avg_frame_qindex[KEY_FRAME]]; + else + active_best_quality = rtc_minq[active_worst_quality]; + } + } + + // Clip the active best and worst quality values to limits + active_best_quality = + clamp(active_best_quality, rc->best_quality, rc->worst_quality); + active_worst_quality = + clamp(active_worst_quality, active_best_quality, rc->worst_quality); + + *top_index = active_worst_quality; + *bottom_index = active_best_quality; + + // Limit Q range for the adaptive loop. + if (cm->frame_type == KEY_FRAME && !rc->this_key_frame_forced && + !(cm->current_video_frame == 0)) { + int qdelta = 0; + aom_clear_system_state(); + qdelta = av1_compute_qdelta_by_rate( + &cpi->rc, cm->frame_type, active_worst_quality, 2.0, cm->bit_depth); + *top_index = active_worst_quality + qdelta; + *top_index = AOMMAX(*top_index, *bottom_index); + } + + // Special case code to try and match quality with forced key frames + if (cm->frame_type == KEY_FRAME && rc->this_key_frame_forced) { + q = rc->last_boosted_qindex; + } else { + q = av1_rc_regulate_q(cpi, rc->this_frame_target, active_best_quality, + active_worst_quality); + if (q > *top_index) { + // Special case when we are targeting the max allowed rate + if (rc->this_frame_target >= rc->max_frame_bandwidth) + *top_index = q; + else + q = *top_index; + } + } + + assert(*top_index <= rc->worst_quality && *top_index >= rc->best_quality); + assert(*bottom_index <= rc->worst_quality && + *bottom_index >= rc->best_quality); + assert(q <= rc->worst_quality && q >= rc->best_quality); + return q; +} + +static int get_active_cq_level(const RATE_CONTROL *rc, + const AV1EncoderConfig *const oxcf) { + static const double cq_adjust_threshold = 0.1; + int active_cq_level = oxcf->cq_level; + if (oxcf->rc_mode == AOM_CQ && rc->total_target_bits > 0) { + const double x = (double)rc->total_actual_bits / rc->total_target_bits; + if (x < cq_adjust_threshold) { + active_cq_level = (int)(active_cq_level * x / cq_adjust_threshold); + } + } + return active_cq_level; +} + +static int rc_pick_q_and_bounds_one_pass_vbr(const AV1_COMP *cpi, + int *bottom_index, + int *top_index) { + const AV1_COMMON *const cm = &cpi->common; + const RATE_CONTROL *const rc = &cpi->rc; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + const int cq_level = get_active_cq_level(rc, oxcf); + int active_best_quality; + int active_worst_quality = calc_active_worst_quality_one_pass_vbr(cpi); + int q; + int *inter_minq; + ASSIGN_MINQ_TABLE(cm->bit_depth, inter_minq); + + if (frame_is_intra_only(cm)) { + if (oxcf->rc_mode == AOM_Q) { + const int qindex = cq_level; + const double q_val = av1_convert_qindex_to_q(qindex, cm->bit_depth); + const int delta_qindex = + av1_compute_qdelta(rc, q_val, q_val * 0.25, cm->bit_depth); + active_best_quality = AOMMAX(qindex + delta_qindex, rc->best_quality); + } else if (rc->this_key_frame_forced) { + const int qindex = rc->last_boosted_qindex; + const double last_boosted_q = + av1_convert_qindex_to_q(qindex, cm->bit_depth); + const int delta_qindex = av1_compute_qdelta( + rc, last_boosted_q, last_boosted_q * 0.75, cm->bit_depth); + active_best_quality = AOMMAX(qindex + delta_qindex, rc->best_quality); + } else { // not first frame of one pass and kf_boost is set + double q_adj_factor = 1.0; + + active_best_quality = get_kf_active_quality( + rc, rc->avg_frame_qindex[KEY_FRAME], cm->bit_depth); + + // Allow somewhat lower kf minq with small image formats. + if ((cm->width * cm->height) <= (352 * 288)) { + q_adj_factor -= 0.25; + } + + // Convert the adjustment factor to a qindex delta on active_best_quality. + { + const double q_val = + av1_convert_qindex_to_q(active_best_quality, cm->bit_depth); + active_best_quality += + av1_compute_qdelta(rc, q_val, q_val * q_adj_factor, cm->bit_depth); + } + } + } else if (!rc->is_src_frame_alt_ref && + (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame)) { + // Use the lower of active_worst_quality and recent + // average Q as basis for GF/ARF best Q limit unless last frame was + // a key frame. + q = (rc->frames_since_key > 1 && + rc->avg_frame_qindex[INTER_FRAME] < active_worst_quality) + ? rc->avg_frame_qindex[INTER_FRAME] + : rc->avg_frame_qindex[KEY_FRAME]; + // For constrained quality dont allow Q less than the cq level + if (oxcf->rc_mode == AOM_CQ) { + if (q < cq_level) q = cq_level; + active_best_quality = get_gf_active_quality(rc, q, cm->bit_depth); + // Constrained quality use slightly lower active best. + active_best_quality = active_best_quality * 15 / 16; + } else if (oxcf->rc_mode == AOM_Q) { + const int qindex = cq_level; + const double q_val = av1_convert_qindex_to_q(qindex, cm->bit_depth); + const int delta_qindex = + (cpi->refresh_alt_ref_frame) + ? av1_compute_qdelta(rc, q_val, q_val * 0.40, cm->bit_depth) + : av1_compute_qdelta(rc, q_val, q_val * 0.50, cm->bit_depth); + active_best_quality = AOMMAX(qindex + delta_qindex, rc->best_quality); + } else { + active_best_quality = get_gf_active_quality(rc, q, cm->bit_depth); + } + } else { + if (oxcf->rc_mode == AOM_Q) { + const int qindex = cq_level; + const double q_val = av1_convert_qindex_to_q(qindex, cm->bit_depth); + const double delta_rate[FIXED_GF_INTERVAL] = { 0.50, 1.0, 0.85, 1.0, + 0.70, 1.0, 0.85, 1.0 }; + const int delta_qindex = av1_compute_qdelta( + rc, q_val, + q_val * delta_rate[cm->current_video_frame % FIXED_GF_INTERVAL], + cm->bit_depth); + active_best_quality = AOMMAX(qindex + delta_qindex, rc->best_quality); + } else { + // Use the lower of active_worst_quality and recent/average Q. + active_best_quality = (cm->current_video_frame > 1) + ? inter_minq[rc->avg_frame_qindex[INTER_FRAME]] + : inter_minq[rc->avg_frame_qindex[KEY_FRAME]]; + // For the constrained quality mode we don't want + // q to fall below the cq level. + if ((oxcf->rc_mode == AOM_CQ) && (active_best_quality < cq_level)) { + active_best_quality = cq_level; + } + } + } + + // Clip the active best and worst quality values to limits + active_best_quality = + clamp(active_best_quality, rc->best_quality, rc->worst_quality); + active_worst_quality = + clamp(active_worst_quality, active_best_quality, rc->worst_quality); + + *top_index = active_worst_quality; + *bottom_index = active_best_quality; + + // Limit Q range for the adaptive loop. + { + int qdelta = 0; + aom_clear_system_state(); + if (cm->frame_type == KEY_FRAME && !rc->this_key_frame_forced && + !(cm->current_video_frame == 0)) { + qdelta = av1_compute_qdelta_by_rate( + &cpi->rc, cm->frame_type, active_worst_quality, 2.0, cm->bit_depth); + } else if (!rc->is_src_frame_alt_ref && + (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame)) { + qdelta = av1_compute_qdelta_by_rate( + &cpi->rc, cm->frame_type, active_worst_quality, 1.75, cm->bit_depth); + } + *top_index = active_worst_quality + qdelta; + *top_index = AOMMAX(*top_index, *bottom_index); + } + + if (oxcf->rc_mode == AOM_Q) { + q = active_best_quality; + // Special case code to try and match quality with forced key frames + } else if ((cm->frame_type == KEY_FRAME) && rc->this_key_frame_forced) { + q = rc->last_boosted_qindex; + } else { + q = av1_rc_regulate_q(cpi, rc->this_frame_target, active_best_quality, + active_worst_quality); + if (q > *top_index) { + // Special case when we are targeting the max allowed rate + if (rc->this_frame_target >= rc->max_frame_bandwidth) + *top_index = q; + else + q = *top_index; + } + } + + assert(*top_index <= rc->worst_quality && *top_index >= rc->best_quality); + assert(*bottom_index <= rc->worst_quality && + *bottom_index >= rc->best_quality); + assert(q <= rc->worst_quality && q >= rc->best_quality); + return q; +} + +int av1_frame_type_qdelta(const AV1_COMP *cpi, int rf_level, int q) { + static const double rate_factor_deltas[RATE_FACTOR_LEVELS] = { + 1.00, // INTER_NORMAL +#if CONFIG_EXT_REFS + 0.80, // INTER_LOW + 1.50, // INTER_HIGH + 1.25, // GF_ARF_LOW +#else + 1.00, // INTER_HIGH + 1.50, // GF_ARF_LOW +#endif // CONFIG_EXT_REFS + 2.00, // GF_ARF_STD + 2.00, // KF_STD + }; + static const FRAME_TYPE frame_type[RATE_FACTOR_LEVELS] = +#if CONFIG_EXT_REFS + { INTER_FRAME, INTER_FRAME, INTER_FRAME, + INTER_FRAME, INTER_FRAME, KEY_FRAME }; +#else + { INTER_FRAME, INTER_FRAME, INTER_FRAME, INTER_FRAME, KEY_FRAME }; +#endif // CONFIG_EXT_REFS + const AV1_COMMON *const cm = &cpi->common; + int qdelta = + av1_compute_qdelta_by_rate(&cpi->rc, frame_type[rf_level], q, + rate_factor_deltas[rf_level], cm->bit_depth); + return qdelta; +} + +#define STATIC_MOTION_THRESH 95 +static int rc_pick_q_and_bounds_two_pass(const AV1_COMP *cpi, int *bottom_index, + int *top_index) { + const AV1_COMMON *const cm = &cpi->common; + const RATE_CONTROL *const rc = &cpi->rc; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + const GF_GROUP *gf_group = &cpi->twopass.gf_group; + const int cq_level = get_active_cq_level(rc, oxcf); + int active_best_quality; + int active_worst_quality = cpi->twopass.active_worst_quality; + int q; + int *inter_minq; + ASSIGN_MINQ_TABLE(cm->bit_depth, inter_minq); + + if (frame_is_intra_only(cm)) { + // Handle the special case for key frames forced when we have reached + // the maximum key frame interval. Here force the Q to a range + // based on the ambient Q to reduce the risk of popping. + if (rc->this_key_frame_forced) { + double last_boosted_q; + int delta_qindex; + int qindex; + + if (cpi->twopass.last_kfgroup_zeromotion_pct >= STATIC_MOTION_THRESH) { + qindex = AOMMIN(rc->last_kf_qindex, rc->last_boosted_qindex); + active_best_quality = qindex; + last_boosted_q = av1_convert_qindex_to_q(qindex, cm->bit_depth); + delta_qindex = av1_compute_qdelta(rc, last_boosted_q, + last_boosted_q * 1.25, cm->bit_depth); + active_worst_quality = + AOMMIN(qindex + delta_qindex, active_worst_quality); + } else { + qindex = rc->last_boosted_qindex; + last_boosted_q = av1_convert_qindex_to_q(qindex, cm->bit_depth); + delta_qindex = av1_compute_qdelta(rc, last_boosted_q, + last_boosted_q * 0.75, cm->bit_depth); + active_best_quality = AOMMAX(qindex + delta_qindex, rc->best_quality); + } + } else { + // Not forced keyframe. + double q_adj_factor = 1.0; + double q_val; + + // Baseline value derived from cpi->active_worst_quality and kf boost. + active_best_quality = + get_kf_active_quality(rc, active_worst_quality, cm->bit_depth); + + // Allow somewhat lower kf minq with small image formats. + if ((cm->width * cm->height) <= (352 * 288)) { + q_adj_factor -= 0.25; + } + + // Make a further adjustment based on the kf zero motion measure. + q_adj_factor += 0.05 - (0.001 * (double)cpi->twopass.kf_zeromotion_pct); + + // Convert the adjustment factor to a qindex delta + // on active_best_quality. + q_val = av1_convert_qindex_to_q(active_best_quality, cm->bit_depth); + active_best_quality += + av1_compute_qdelta(rc, q_val, q_val * q_adj_factor, cm->bit_depth); + } + } else if (!rc->is_src_frame_alt_ref && + (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame)) { + // Use the lower of active_worst_quality and recent + // average Q as basis for GF/ARF best Q limit unless last frame was + // a key frame. + if (rc->frames_since_key > 1 && + rc->avg_frame_qindex[INTER_FRAME] < active_worst_quality) { + q = rc->avg_frame_qindex[INTER_FRAME]; + } else { + q = active_worst_quality; + } + // For constrained quality dont allow Q less than the cq level + if (oxcf->rc_mode == AOM_CQ) { + if (q < cq_level) q = cq_level; + + active_best_quality = get_gf_active_quality(rc, q, cm->bit_depth); + + // Constrained quality use slightly lower active best. + active_best_quality = active_best_quality * 15 / 16; + + } else if (oxcf->rc_mode == AOM_Q) { + if (!cpi->refresh_alt_ref_frame) { + active_best_quality = cq_level; + } else { + active_best_quality = get_gf_active_quality(rc, q, cm->bit_depth); + + // Modify best quality for second level arfs. For mode AOM_Q this + // becomes the baseline frame q. + if (gf_group->rf_level[gf_group->index] == GF_ARF_LOW) + active_best_quality = (active_best_quality + cq_level + 1) / 2; + } + } else { + active_best_quality = get_gf_active_quality(rc, q, cm->bit_depth); + } + } else { + if (oxcf->rc_mode == AOM_Q) { + active_best_quality = cq_level; + } else { + active_best_quality = inter_minq[active_worst_quality]; + + // For the constrained quality mode we don't want + // q to fall below the cq level. + if ((oxcf->rc_mode == AOM_CQ) && (active_best_quality < cq_level)) { + active_best_quality = cq_level; + } + } + } + + // Extension to max or min Q if undershoot or overshoot is outside + // the permitted range. + if ((cpi->oxcf.rc_mode != AOM_Q) && + (cpi->twopass.gf_zeromotion_pct < VLOW_MOTION_THRESHOLD)) { + if (frame_is_intra_only(cm) || + (!rc->is_src_frame_alt_ref && + (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame))) { + active_best_quality -= + (cpi->twopass.extend_minq + cpi->twopass.extend_minq_fast); + active_worst_quality += (cpi->twopass.extend_maxq / 2); + } else { + active_best_quality -= + (cpi->twopass.extend_minq + cpi->twopass.extend_minq_fast) / 2; + active_worst_quality += cpi->twopass.extend_maxq; + } + } + + aom_clear_system_state(); + // Static forced key frames Q restrictions dealt with elsewhere. + if (!(frame_is_intra_only(cm)) || !rc->this_key_frame_forced || + (cpi->twopass.last_kfgroup_zeromotion_pct < STATIC_MOTION_THRESH)) { + int qdelta = av1_frame_type_qdelta(cpi, gf_group->rf_level[gf_group->index], + active_worst_quality); + active_worst_quality = + AOMMAX(active_worst_quality + qdelta, active_best_quality); + } + + // Modify active_best_quality for downscaled normal frames. + if (rc->frame_size_selector != UNSCALED && !frame_is_kf_gf_arf(cpi)) { + int qdelta = av1_compute_qdelta_by_rate( + rc, cm->frame_type, active_best_quality, 2.0, cm->bit_depth); + active_best_quality = + AOMMAX(active_best_quality + qdelta, rc->best_quality); + } + + active_best_quality = + clamp(active_best_quality, rc->best_quality, rc->worst_quality); + active_worst_quality = + clamp(active_worst_quality, active_best_quality, rc->worst_quality); + + if (oxcf->rc_mode == AOM_Q) { + q = active_best_quality; + // Special case code to try and match quality with forced key frames. + } else if (frame_is_intra_only(cm) && rc->this_key_frame_forced) { + // If static since last kf use better of last boosted and last kf q. + if (cpi->twopass.last_kfgroup_zeromotion_pct >= STATIC_MOTION_THRESH) { + q = AOMMIN(rc->last_kf_qindex, rc->last_boosted_qindex); + } else { + q = rc->last_boosted_qindex; + } + } else { + q = av1_rc_regulate_q(cpi, rc->this_frame_target, active_best_quality, + active_worst_quality); + if (q > active_worst_quality) { + // Special case when we are targeting the max allowed rate. + if (rc->this_frame_target >= rc->max_frame_bandwidth) + active_worst_quality = q; + else + q = active_worst_quality; + } + } + clamp(q, active_best_quality, active_worst_quality); + + *top_index = active_worst_quality; + *bottom_index = active_best_quality; + + assert(*top_index <= rc->worst_quality && *top_index >= rc->best_quality); + assert(*bottom_index <= rc->worst_quality && + *bottom_index >= rc->best_quality); + assert(q <= rc->worst_quality && q >= rc->best_quality); + return q; +} + +int av1_rc_pick_q_and_bounds(const AV1_COMP *cpi, int *bottom_index, + int *top_index) { + int q; + if (cpi->oxcf.pass == 0) { + if (cpi->oxcf.rc_mode == AOM_CBR) + q = rc_pick_q_and_bounds_one_pass_cbr(cpi, bottom_index, top_index); + else + q = rc_pick_q_and_bounds_one_pass_vbr(cpi, bottom_index, top_index); + } else { + q = rc_pick_q_and_bounds_two_pass(cpi, bottom_index, top_index); + } + + return q; +} + +void av1_rc_compute_frame_size_bounds(const AV1_COMP *cpi, int frame_target, + int *frame_under_shoot_limit, + int *frame_over_shoot_limit) { + if (cpi->oxcf.rc_mode == AOM_Q) { + *frame_under_shoot_limit = 0; + *frame_over_shoot_limit = INT_MAX; + } else { + // For very small rate targets where the fractional adjustment + // may be tiny make sure there is at least a minimum range. + const int tolerance = (cpi->sf.recode_tolerance * frame_target) / 100; + *frame_under_shoot_limit = AOMMAX(frame_target - tolerance - 200, 0); + *frame_over_shoot_limit = + AOMMIN(frame_target + tolerance + 200, cpi->rc.max_frame_bandwidth); + } +} + +void av1_rc_set_frame_target(AV1_COMP *cpi, int target) { + const AV1_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; + + rc->this_frame_target = target; + + // Modify frame size target when down-scaling. + if (cpi->oxcf.resize_mode == RESIZE_DYNAMIC && + rc->frame_size_selector != UNSCALED) + rc->this_frame_target = (int)(rc->this_frame_target * + rate_thresh_mult[rc->frame_size_selector]); + + // Target rate per SB64 (including partial SB64s. + rc->sb64_target_rate = (int)((int64_t)rc->this_frame_target * 64 * 64) / + (cm->width * cm->height); +} + +static void update_alt_ref_frame_stats(AV1_COMP *cpi) { + // this frame refreshes means next frames don't unless specified by user + RATE_CONTROL *const rc = &cpi->rc; + rc->frames_since_golden = 0; + + // Mark the alt ref as done (setting to 0 means no further alt refs pending). + rc->source_alt_ref_pending = 0; + + // Set the alternate reference frame active flag + rc->source_alt_ref_active = 1; +} + +static void update_golden_frame_stats(AV1_COMP *cpi) { + RATE_CONTROL *const rc = &cpi->rc; + +#if CONFIG_EXT_REFS + // Update the Golden frame usage counts. + // NOTE(weitinglin): If we use show_existing_frame for an OVERLAY frame, + // only the virtual indices for the reference frame will be + // updated and cpi->refresh_golden_frame will still be zero. + if (cpi->refresh_golden_frame || rc->is_src_frame_alt_ref) { +#else + // Update the Golden frame usage counts. + if (cpi->refresh_golden_frame) { +#endif // CONFIG_EXT_REFS + +#if CONFIG_EXT_REFS + // We will not use internal overlay frames to replace the golden frame + if (!rc->is_src_frame_ext_arf) +#endif // CONFIG_EXT_REFS + // this frame refreshes means next frames don't unless specified by user + rc->frames_since_golden = 0; + + // If we are not using alt ref in the up and coming group clear the arf + // active flag. In multi arf group case, if the index is not 0 then + // we are overlaying a mid group arf so should not reset the flag. + if (cpi->oxcf.pass == 2) { + if (!rc->source_alt_ref_pending && (cpi->twopass.gf_group.index == 0)) + rc->source_alt_ref_active = 0; + } else if (!rc->source_alt_ref_pending) { + rc->source_alt_ref_active = 0; + } + + // Decrement count down till next gf + if (rc->frames_till_gf_update_due > 0) rc->frames_till_gf_update_due--; + + } else if (!cpi->refresh_alt_ref_frame) { + // Decrement count down till next gf + if (rc->frames_till_gf_update_due > 0) rc->frames_till_gf_update_due--; + + rc->frames_since_golden++; + } +} + +void av1_rc_postencode_update(AV1_COMP *cpi, uint64_t bytes_used) { + const AV1_COMMON *const cm = &cpi->common; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + RATE_CONTROL *const rc = &cpi->rc; + const int qindex = cm->base_qindex; + + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cm->seg.enabled) { + av1_cyclic_refresh_postencode(cpi); + } + + // Update rate control heuristics + rc->projected_frame_size = (int)(bytes_used << 3); + + // Post encode loop adjustment of Q prediction. + av1_rc_update_rate_correction_factors(cpi); + + // Keep a record of last Q and ambient average Q. + if (cm->frame_type == KEY_FRAME) { + rc->last_q[KEY_FRAME] = qindex; + rc->avg_frame_qindex[KEY_FRAME] = + ROUND_POWER_OF_TWO(3 * rc->avg_frame_qindex[KEY_FRAME] + qindex, 2); + } else { + if (!rc->is_src_frame_alt_ref && + !(cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame)) { + rc->last_q[INTER_FRAME] = qindex; + rc->avg_frame_qindex[INTER_FRAME] = + ROUND_POWER_OF_TWO(3 * rc->avg_frame_qindex[INTER_FRAME] + qindex, 2); + rc->ni_frames++; + rc->tot_q += av1_convert_qindex_to_q(qindex, cm->bit_depth); + rc->avg_q = rc->tot_q / rc->ni_frames; + // Calculate the average Q for normal inter frames (not key or GFU + // frames). + rc->ni_tot_qi += qindex; + rc->ni_av_qi = rc->ni_tot_qi / rc->ni_frames; + } + } + + // Keep record of last boosted (KF/GF/ARF) Q value. + // If the current frame is coded at a lower Q then we also update it. + // If all mbs in this group are skipped only update if the Q value is + // better than that already stored. + // This is used to help set quality in forced key frames to reduce popping + if ((qindex < rc->last_boosted_qindex) || (cm->frame_type == KEY_FRAME) || + (!rc->constrained_gf_group && + (cpi->refresh_alt_ref_frame || + (cpi->refresh_golden_frame && !rc->is_src_frame_alt_ref)))) { + rc->last_boosted_qindex = qindex; + } + if (cm->frame_type == KEY_FRAME) rc->last_kf_qindex = qindex; + + update_buffer_level(cpi, rc->projected_frame_size); + + // Rolling monitors of whether we are over or underspending used to help + // regulate min and Max Q in two pass. + if (cm->frame_type != KEY_FRAME) { + rc->rolling_target_bits = ROUND_POWER_OF_TWO( + rc->rolling_target_bits * 3 + rc->this_frame_target, 2); + rc->rolling_actual_bits = ROUND_POWER_OF_TWO( + rc->rolling_actual_bits * 3 + rc->projected_frame_size, 2); + rc->long_rolling_target_bits = ROUND_POWER_OF_TWO( + rc->long_rolling_target_bits * 31 + rc->this_frame_target, 5); + rc->long_rolling_actual_bits = ROUND_POWER_OF_TWO( + rc->long_rolling_actual_bits * 31 + rc->projected_frame_size, 5); + } + + // Actual bits spent + rc->total_actual_bits += rc->projected_frame_size; +#if CONFIG_EXT_REFS + rc->total_target_bits += + (cm->show_frame || rc->is_bwd_ref_frame) ? rc->avg_frame_bandwidth : 0; +#else + rc->total_target_bits += cm->show_frame ? rc->avg_frame_bandwidth : 0; +#endif // CONFIG_EXT_REFS + + rc->total_target_vs_actual = rc->total_actual_bits - rc->total_target_bits; + + if (is_altref_enabled(cpi) && cpi->refresh_alt_ref_frame && + (cm->frame_type != KEY_FRAME)) + // Update the alternate reference frame stats as appropriate. + update_alt_ref_frame_stats(cpi); + else + // Update the Golden frame stats as appropriate. + update_golden_frame_stats(cpi); + + if (cm->frame_type == KEY_FRAME) rc->frames_since_key = 0; + +#if CONFIG_EXT_REFS + if (cm->show_frame || rc->is_bwd_ref_frame) { +#else + if (cm->show_frame) { +#endif // CONFIG_EXT_REFS + rc->frames_since_key++; + rc->frames_to_key--; + } + + // Trigger the resizing of the next frame if it is scaled. + if (oxcf->pass != 0) { + cpi->resize_pending = + rc->next_frame_size_selector != rc->frame_size_selector; + rc->frame_size_selector = rc->next_frame_size_selector; + } +} + +void av1_rc_postencode_update_drop_frame(AV1_COMP *cpi) { + // Update buffer level with zero size, update frame counters, and return. + update_buffer_level(cpi, 0); + cpi->rc.frames_since_key++; + cpi->rc.frames_to_key--; + cpi->rc.rc_2_frame = 0; + cpi->rc.rc_1_frame = 0; +} + +// Use this macro to turn on/off use of alt-refs in one-pass mode. +#define USE_ALTREF_FOR_ONE_PASS 1 + +static int calc_pframe_target_size_one_pass_vbr(const AV1_COMP *const cpi) { + static const int af_ratio = 10; + const RATE_CONTROL *const rc = &cpi->rc; + int target; +#if USE_ALTREF_FOR_ONE_PASS + target = + (!rc->is_src_frame_alt_ref && + (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame)) + ? (rc->avg_frame_bandwidth * rc->baseline_gf_interval * af_ratio) / + (rc->baseline_gf_interval + af_ratio - 1) + : (rc->avg_frame_bandwidth * rc->baseline_gf_interval) / + (rc->baseline_gf_interval + af_ratio - 1); +#else + target = rc->avg_frame_bandwidth; +#endif + return av1_rc_clamp_pframe_target_size(cpi, target); +} + +static int calc_iframe_target_size_one_pass_vbr(const AV1_COMP *const cpi) { + static const int kf_ratio = 25; + const RATE_CONTROL *rc = &cpi->rc; + const int target = rc->avg_frame_bandwidth * kf_ratio; + return av1_rc_clamp_iframe_target_size(cpi, target); +} + +void av1_rc_get_one_pass_vbr_params(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; + int target; + // TODO(yaowu): replace the "auto_key && 0" below with proper decision logic. + if (!cpi->refresh_alt_ref_frame && + (cm->current_video_frame == 0 || (cpi->frame_flags & FRAMEFLAGS_KEY) || + rc->frames_to_key == 0 || (cpi->oxcf.auto_key && 0))) { + cm->frame_type = KEY_FRAME; + rc->this_key_frame_forced = + cm->current_video_frame != 0 && rc->frames_to_key == 0; + rc->frames_to_key = cpi->oxcf.key_freq; + rc->kf_boost = DEFAULT_KF_BOOST; + rc->source_alt_ref_active = 0; + } else { + cm->frame_type = INTER_FRAME; + } + if (rc->frames_till_gf_update_due == 0) { + rc->baseline_gf_interval = (rc->min_gf_interval + rc->max_gf_interval) / 2; + rc->frames_till_gf_update_due = rc->baseline_gf_interval; + // NOTE: frames_till_gf_update_due must be <= frames_to_key. + if (rc->frames_till_gf_update_due > rc->frames_to_key) { + rc->frames_till_gf_update_due = rc->frames_to_key; + rc->constrained_gf_group = 1; + } else { + rc->constrained_gf_group = 0; + } + cpi->refresh_golden_frame = 1; + rc->source_alt_ref_pending = USE_ALTREF_FOR_ONE_PASS; + rc->gfu_boost = DEFAULT_GF_BOOST; + } + if (cm->frame_type == KEY_FRAME) + target = calc_iframe_target_size_one_pass_vbr(cpi); + else + target = calc_pframe_target_size_one_pass_vbr(cpi); + av1_rc_set_frame_target(cpi, target); +} + +static int calc_pframe_target_size_one_pass_cbr(const AV1_COMP *cpi) { + const AV1EncoderConfig *oxcf = &cpi->oxcf; + const RATE_CONTROL *rc = &cpi->rc; + const int64_t diff = rc->optimal_buffer_level - rc->buffer_level; + const int64_t one_pct_bits = 1 + rc->optimal_buffer_level / 100; + int min_frame_target = + AOMMAX(rc->avg_frame_bandwidth >> 4, FRAME_OVERHEAD_BITS); + int target; + + if (oxcf->gf_cbr_boost_pct) { + const int af_ratio_pct = oxcf->gf_cbr_boost_pct + 100; + target = cpi->refresh_golden_frame + ? (rc->avg_frame_bandwidth * rc->baseline_gf_interval * + af_ratio_pct) / + (rc->baseline_gf_interval * 100 + af_ratio_pct - 100) + : (rc->avg_frame_bandwidth * rc->baseline_gf_interval * 100) / + (rc->baseline_gf_interval * 100 + af_ratio_pct - 100); + } else { + target = rc->avg_frame_bandwidth; + } + + if (diff > 0) { + // Lower the target bandwidth for this frame. + const int pct_low = (int)AOMMIN(diff / one_pct_bits, oxcf->under_shoot_pct); + target -= (target * pct_low) / 200; + } else if (diff < 0) { + // Increase the target bandwidth for this frame. + const int pct_high = + (int)AOMMIN(-diff / one_pct_bits, oxcf->over_shoot_pct); + target += (target * pct_high) / 200; + } + if (oxcf->rc_max_inter_bitrate_pct) { + const int max_rate = + rc->avg_frame_bandwidth * oxcf->rc_max_inter_bitrate_pct / 100; + target = AOMMIN(target, max_rate); + } + return AOMMAX(min_frame_target, target); +} + +static int calc_iframe_target_size_one_pass_cbr(const AV1_COMP *cpi) { + const RATE_CONTROL *rc = &cpi->rc; + int target; + if (cpi->common.current_video_frame == 0) { + target = ((rc->starting_buffer_level / 2) > INT_MAX) + ? INT_MAX + : (int)(rc->starting_buffer_level / 2); + } else { + int kf_boost = 32; + double framerate = cpi->framerate; + + kf_boost = AOMMAX(kf_boost, (int)(2 * framerate - 16)); + if (rc->frames_since_key < framerate / 2) { + kf_boost = (int)(kf_boost * rc->frames_since_key / (framerate / 2)); + } + target = ((16 + kf_boost) * rc->avg_frame_bandwidth) >> 4; + } + return av1_rc_clamp_iframe_target_size(cpi, target); +} + +void av1_rc_get_one_pass_cbr_params(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; + int target; + // TODO(yaowu): replace the "auto_key && 0" below with proper decision logic. + if ((cm->current_video_frame == 0 || (cpi->frame_flags & FRAMEFLAGS_KEY) || + rc->frames_to_key == 0 || (cpi->oxcf.auto_key && 0))) { + cm->frame_type = KEY_FRAME; + rc->this_key_frame_forced = + cm->current_video_frame != 0 && rc->frames_to_key == 0; + rc->frames_to_key = cpi->oxcf.key_freq; + rc->kf_boost = DEFAULT_KF_BOOST; + rc->source_alt_ref_active = 0; + } else { + cm->frame_type = INTER_FRAME; + } + if (rc->frames_till_gf_update_due == 0) { + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) + av1_cyclic_refresh_set_golden_update(cpi); + else + rc->baseline_gf_interval = + (rc->min_gf_interval + rc->max_gf_interval) / 2; + rc->frames_till_gf_update_due = rc->baseline_gf_interval; + // NOTE: frames_till_gf_update_due must be <= frames_to_key. + if (rc->frames_till_gf_update_due > rc->frames_to_key) + rc->frames_till_gf_update_due = rc->frames_to_key; + cpi->refresh_golden_frame = 1; + rc->gfu_boost = DEFAULT_GF_BOOST; + } + + // Any update/change of global cyclic refresh parameters (amount/delta-qp) + // should be done here, before the frame qp is selected. + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) + av1_cyclic_refresh_update_parameters(cpi); + + if (cm->frame_type == KEY_FRAME) + target = calc_iframe_target_size_one_pass_cbr(cpi); + else + target = calc_pframe_target_size_one_pass_cbr(cpi); + + av1_rc_set_frame_target(cpi, target); + if (cpi->oxcf.resize_mode == RESIZE_DYNAMIC) + cpi->resize_pending = av1_resize_one_pass_cbr(cpi); + else + cpi->resize_pending = 0; +} + +int av1_compute_qdelta(const RATE_CONTROL *rc, double qstart, double qtarget, + aom_bit_depth_t bit_depth) { + int start_index = rc->worst_quality; + int target_index = rc->worst_quality; + int i; + + // Convert the average q value to an index. + for (i = rc->best_quality; i < rc->worst_quality; ++i) { + start_index = i; + if (av1_convert_qindex_to_q(i, bit_depth) >= qstart) break; + } + + // Convert the q target to an index + for (i = rc->best_quality; i < rc->worst_quality; ++i) { + target_index = i; + if (av1_convert_qindex_to_q(i, bit_depth) >= qtarget) break; + } + + return target_index - start_index; +} + +int av1_compute_qdelta_by_rate(const RATE_CONTROL *rc, FRAME_TYPE frame_type, + int qindex, double rate_target_ratio, + aom_bit_depth_t bit_depth) { + int target_index = rc->worst_quality; + int i; + + // Look up the current projected bits per block for the base index + const int base_bits_per_mb = + av1_rc_bits_per_mb(frame_type, qindex, 1.0, bit_depth); + + // Find the target bits per mb based on the base value and given ratio. + const int target_bits_per_mb = (int)(rate_target_ratio * base_bits_per_mb); + + // Convert the q target to an index + for (i = rc->best_quality; i < rc->worst_quality; ++i) { + if (av1_rc_bits_per_mb(frame_type, i, 1.0, bit_depth) <= + target_bits_per_mb) { + target_index = i; + break; + } + } + return target_index - qindex; +} + +void av1_rc_set_gf_interval_range(const AV1_COMP *const cpi, + RATE_CONTROL *const rc) { + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + + // Special case code for 1 pass fixed Q mode tests + if ((oxcf->pass == 0) && (oxcf->rc_mode == AOM_Q)) { + rc->max_gf_interval = FIXED_GF_INTERVAL; + rc->min_gf_interval = FIXED_GF_INTERVAL; + rc->static_scene_max_gf_interval = FIXED_GF_INTERVAL; + } else { + // Set Maximum gf/arf interval + rc->max_gf_interval = oxcf->max_gf_interval; + rc->min_gf_interval = oxcf->min_gf_interval; + if (rc->min_gf_interval == 0) + rc->min_gf_interval = av1_rc_get_default_min_gf_interval( + oxcf->width, oxcf->height, cpi->framerate); + if (rc->max_gf_interval == 0) + rc->max_gf_interval = av1_rc_get_default_max_gf_interval( + cpi->framerate, rc->min_gf_interval); + + // Extended interval for genuinely static scenes + rc->static_scene_max_gf_interval = MAX_LAG_BUFFERS * 2; + + if (is_altref_enabled(cpi)) { + if (rc->static_scene_max_gf_interval > oxcf->lag_in_frames - 1) + rc->static_scene_max_gf_interval = oxcf->lag_in_frames - 1; + } + + if (rc->max_gf_interval > rc->static_scene_max_gf_interval) + rc->max_gf_interval = rc->static_scene_max_gf_interval; + + // Clamp min to max + rc->min_gf_interval = AOMMIN(rc->min_gf_interval, rc->max_gf_interval); + } +} + +void av1_rc_update_framerate(AV1_COMP *cpi) { + const AV1_COMMON *const cm = &cpi->common; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + RATE_CONTROL *const rc = &cpi->rc; + int vbr_max_bits; + + rc->avg_frame_bandwidth = (int)(oxcf->target_bandwidth / cpi->framerate); + rc->min_frame_bandwidth = + (int)(rc->avg_frame_bandwidth * oxcf->two_pass_vbrmin_section / 100); + + rc->min_frame_bandwidth = + AOMMAX(rc->min_frame_bandwidth, FRAME_OVERHEAD_BITS); + + // A maximum bitrate for a frame is defined. + // The baseline for this aligns with HW implementations that + // can support decode of 1080P content up to a bitrate of MAX_MB_RATE bits + // per 16x16 MB (averaged over a frame). However this limit is extended if + // a very high rate is given on the command line or the the rate cannnot + // be acheived because of a user specificed max q (e.g. when the user + // specifies lossless encode. + vbr_max_bits = + (int)(((int64_t)rc->avg_frame_bandwidth * oxcf->two_pass_vbrmax_section) / + 100); + rc->max_frame_bandwidth = + AOMMAX(AOMMAX((cm->MBs * MAX_MB_RATE), MAXRATE_1080P), vbr_max_bits); + + av1_rc_set_gf_interval_range(cpi, rc); +} + +#define VBR_PCT_ADJUSTMENT_LIMIT 50 +// For VBR...adjustment to the frame target based on error from previous frames +static void vbr_rate_correction(AV1_COMP *cpi, int *this_frame_target) { + RATE_CONTROL *const rc = &cpi->rc; + int64_t vbr_bits_off_target = rc->vbr_bits_off_target; + int max_delta; + double position_factor = 1.0; + + // How far through the clip are we. + // This number is used to damp the per frame rate correction. + // Range 0 - 1.0 + if (cpi->twopass.total_stats.count != 0.) { + position_factor = sqrt((double)cpi->common.current_video_frame / + cpi->twopass.total_stats.count); + } + max_delta = (int)(position_factor * + ((*this_frame_target * VBR_PCT_ADJUSTMENT_LIMIT) / 100)); + + // vbr_bits_off_target > 0 means we have extra bits to spend + if (vbr_bits_off_target > 0) { + *this_frame_target += (vbr_bits_off_target > max_delta) + ? max_delta + : (int)vbr_bits_off_target; + } else { + *this_frame_target -= (vbr_bits_off_target < -max_delta) + ? max_delta + : (int)-vbr_bits_off_target; + } + + // Fast redistribution of bits arising from massive local undershoot. + // Dont do it for kf,arf,gf or overlay frames. + if (!frame_is_kf_gf_arf(cpi) && !rc->is_src_frame_alt_ref && + rc->vbr_bits_off_target_fast) { + int one_frame_bits = AOMMAX(rc->avg_frame_bandwidth, *this_frame_target); + int fast_extra_bits; + fast_extra_bits = (int)AOMMIN(rc->vbr_bits_off_target_fast, one_frame_bits); + fast_extra_bits = (int)AOMMIN( + fast_extra_bits, + AOMMAX(one_frame_bits / 8, rc->vbr_bits_off_target_fast / 8)); + *this_frame_target += (int)fast_extra_bits; + rc->vbr_bits_off_target_fast -= fast_extra_bits; + } +} + +void av1_set_target_rate(AV1_COMP *cpi) { + RATE_CONTROL *const rc = &cpi->rc; + int target_rate = rc->base_frame_target; + + // Correction to rate target based on prior over or under shoot. + if (cpi->oxcf.rc_mode == AOM_VBR || cpi->oxcf.rc_mode == AOM_CQ) + vbr_rate_correction(cpi, &target_rate); + av1_rc_set_frame_target(cpi, target_rate); +} + +// Check if we should resize, based on average QP from past x frames. +// Only allow for resize at most one scale down for now, scaling factor is 2. +int av1_resize_one_pass_cbr(AV1_COMP *cpi) { + const AV1_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; + int resize_now = 0; + cpi->resize_scale_num = 1; + cpi->resize_scale_den = 1; + // Don't resize on key frame; reset the counters on key frame. + if (cm->frame_type == KEY_FRAME) { + cpi->resize_avg_qp = 0; + cpi->resize_count = 0; + return 0; + } + // Resize based on average buffer underflow and QP over some window. + // Ignore samples close to key frame, since QP is usually high after key. + if (cpi->rc.frames_since_key > 2 * cpi->framerate) { + const int window = (int)(5 * cpi->framerate); + cpi->resize_avg_qp += cm->base_qindex; + if (cpi->rc.buffer_level < (int)(30 * rc->optimal_buffer_level / 100)) + ++cpi->resize_buffer_underflow; + ++cpi->resize_count; + // Check for resize action every "window" frames. + if (cpi->resize_count >= window) { + int avg_qp = cpi->resize_avg_qp / cpi->resize_count; + // Resize down if buffer level has underflowed sufficent amount in past + // window, and we are at original resolution. + // Resize back up if average QP is low, and we are currently in a resized + // down state. + if (cpi->resize_state == 0 && + cpi->resize_buffer_underflow > (cpi->resize_count >> 2)) { + resize_now = 1; + cpi->resize_state = 1; + } else if (cpi->resize_state == 1 && + avg_qp < 40 * cpi->rc.worst_quality / 100) { + resize_now = -1; + cpi->resize_state = 0; + } + // Reset for next window measurement. + cpi->resize_avg_qp = 0; + cpi->resize_count = 0; + cpi->resize_buffer_underflow = 0; + } + } + // If decision is to resize, reset some quantities, and check is we should + // reduce rate correction factor, + if (resize_now != 0) { + int target_bits_per_frame; + int active_worst_quality; + int qindex; + int tot_scale_change; + // For now, resize is by 1/2 x 1/2. + cpi->resize_scale_num = 1; + cpi->resize_scale_den = 2; + tot_scale_change = (cpi->resize_scale_den * cpi->resize_scale_den) / + (cpi->resize_scale_num * cpi->resize_scale_num); + // Reset buffer level to optimal, update target size. + rc->buffer_level = rc->optimal_buffer_level; + rc->bits_off_target = rc->optimal_buffer_level; + rc->this_frame_target = calc_pframe_target_size_one_pass_cbr(cpi); + // Reset cyclic refresh parameters. + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cm->seg.enabled) + av1_cyclic_refresh_reset_resize(cpi); + // Get the projected qindex, based on the scaled target frame size (scaled + // so target_bits_per_mb in av1_rc_regulate_q will be correct target). + target_bits_per_frame = (resize_now == 1) + ? rc->this_frame_target * tot_scale_change + : rc->this_frame_target / tot_scale_change; + active_worst_quality = calc_active_worst_quality_one_pass_cbr(cpi); + qindex = av1_rc_regulate_q(cpi, target_bits_per_frame, rc->best_quality, + active_worst_quality); + // If resize is down, check if projected q index is close to worst_quality, + // and if so, reduce the rate correction factor (since likely can afford + // lower q for resized frame). + if (resize_now == 1 && qindex > 90 * cpi->rc.worst_quality / 100) { + rc->rate_correction_factors[INTER_NORMAL] *= 0.85; + } + // If resize is back up, check if projected q index is too much above the + // current base_qindex, and if so, reduce the rate correction factor + // (since prefer to keep q for resized frame at least close to previous q). + if (resize_now == -1 && qindex > 130 * cm->base_qindex / 100) { + rc->rate_correction_factors[INTER_NORMAL] *= 0.9; + } + } + return resize_now; +} diff --git a/third_party/aom/av1/encoder/ratectrl.h b/third_party/aom/av1/encoder/ratectrl.h new file mode 100644 index 0000000000..93a9b49397 --- /dev/null +++ b/third_party/aom/av1/encoder/ratectrl.h @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_RATECTRL_H_ +#define AV1_ENCODER_RATECTRL_H_ + +#include "aom/aom_codec.h" +#include "aom/aom_integer.h" + +#include "av1/common/blockd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Bits Per MB at different Q (Multiplied by 512) +#define BPER_MB_NORMBITS 9 + +#define MIN_GF_INTERVAL 4 +#define MAX_GF_INTERVAL 16 +#define FIXED_GF_INTERVAL 8 // Used in some testing modes only + +#if CONFIG_EXT_REFS +typedef enum { + INTER_NORMAL = 0, + INTER_LOW = 1, + INTER_HIGH = 2, + GF_ARF_LOW = 3, + GF_ARF_STD = 4, + KF_STD = 5, + RATE_FACTOR_LEVELS = 6 +} RATE_FACTOR_LEVEL; +#else +typedef enum { + INTER_NORMAL = 0, + INTER_HIGH = 1, + GF_ARF_LOW = 2, + GF_ARF_STD = 3, + KF_STD = 4, + RATE_FACTOR_LEVELS = 5 +} RATE_FACTOR_LEVEL; +#endif // CONFIG_EXT_REFS + +// Internal frame scaling level. +typedef enum { + UNSCALED = 0, // Frame is unscaled. + SCALE_STEP1 = 1, // First-level down-scaling. + FRAME_SCALE_STEPS +} FRAME_SCALE_LEVEL; + +// Frame dimensions multiplier wrt the native frame size, in 1/16ths, +// specified for the scale-up case. +// e.g. 24 => 16/24 = 2/3 of native size. The restriction to 1/16th is +// intended to match the capabilities of the normative scaling filters, +// giving precedence to the up-scaling accuracy. +static const int frame_scale_factor[FRAME_SCALE_STEPS] = { 16, 24 }; + +// Multiplier of the target rate to be used as threshold for triggering scaling. +static const double rate_thresh_mult[FRAME_SCALE_STEPS] = { 1.0, 2.0 }; + +// Scale dependent Rate Correction Factor multipliers. Compensates for the +// greater number of bits per pixel generated in down-scaled frames. +static const double rcf_mult[FRAME_SCALE_STEPS] = { 1.0, 2.0 }; + +typedef struct { + // Rate targetting variables + int base_frame_target; // A baseline frame target before adjustment + // for previous under or over shoot. + int this_frame_target; // Actual frame target after rc adjustment. + int projected_frame_size; + int sb64_target_rate; + int last_q[FRAME_TYPES]; // Separate values for Intra/Inter + int last_boosted_qindex; // Last boosted GF/KF/ARF q + int last_kf_qindex; // Q index of the last key frame coded. + + int gfu_boost; + int last_boost; + int kf_boost; + + double rate_correction_factors[RATE_FACTOR_LEVELS]; + + int frames_since_golden; + int frames_till_gf_update_due; + int min_gf_interval; + int max_gf_interval; + int static_scene_max_gf_interval; + int baseline_gf_interval; + int constrained_gf_group; + int frames_to_key; + int frames_since_key; + int this_key_frame_forced; + int next_key_frame_forced; + int source_alt_ref_pending; + int source_alt_ref_active; + int is_src_frame_alt_ref; + +#if CONFIG_EXT_REFS + // Length of the bi-predictive frame group interval + int bipred_group_interval; + + // NOTE: Different types of frames may have different bits allocated + // accordingly, aiming to achieve the overall optimal RD performance. + int is_bwd_ref_frame; + int is_last_bipred_frame; + int is_bipred_frame; + int is_src_frame_ext_arf; +#endif // CONFIG_EXT_REFS + + int avg_frame_bandwidth; // Average frame size target for clip + int min_frame_bandwidth; // Minimum allocation used for any frame + int max_frame_bandwidth; // Maximum burst rate allowed for a frame. + + int ni_av_qi; + int ni_tot_qi; + int ni_frames; + int avg_frame_qindex[FRAME_TYPES]; + double tot_q; + double avg_q; + + int64_t buffer_level; + int64_t bits_off_target; + int64_t vbr_bits_off_target; + int64_t vbr_bits_off_target_fast; + + int decimation_factor; + int decimation_count; + + int rolling_target_bits; + int rolling_actual_bits; + + int long_rolling_target_bits; + int long_rolling_actual_bits; + + int rate_error_estimate; + + int64_t total_actual_bits; + int64_t total_target_bits; + int64_t total_target_vs_actual; + + int worst_quality; + int best_quality; + + int64_t starting_buffer_level; + int64_t optimal_buffer_level; + int64_t maximum_buffer_size; + + // rate control history for last frame(1) and the frame before(2). + // -1: undershot + // 1: overshoot + // 0: not initialized. + int rc_1_frame; + int rc_2_frame; + int q_1_frame; + int q_2_frame; + + // Auto frame-scaling variables. + FRAME_SCALE_LEVEL frame_size_selector; + FRAME_SCALE_LEVEL next_frame_size_selector; + int frame_width[FRAME_SCALE_STEPS]; + int frame_height[FRAME_SCALE_STEPS]; + int rf_level_maxq[RATE_FACTOR_LEVELS]; +} RATE_CONTROL; + +struct AV1_COMP; +struct AV1EncoderConfig; + +void av1_rc_init(const struct AV1EncoderConfig *oxcf, int pass, + RATE_CONTROL *rc); + +int av1_estimate_bits_at_q(FRAME_TYPE frame_kind, int q, int mbs, + double correction_factor, aom_bit_depth_t bit_depth); + +double av1_convert_qindex_to_q(int qindex, aom_bit_depth_t bit_depth); + +void av1_rc_init_minq_luts(void); + +int av1_rc_get_default_min_gf_interval(int width, int height, double framerate); +// Note av1_rc_get_default_max_gf_interval() requires the min_gf_interval to +// be passed in to ensure that the max_gf_interval returned is at least as bis +// as that. +int av1_rc_get_default_max_gf_interval(double framerate, int min_frame_rate); + +// Generally at the high level, the following flow is expected +// to be enforced for rate control: +// First call per frame, one of: +// av1_rc_get_one_pass_vbr_params() +// av1_rc_get_one_pass_cbr_params() +// av1_rc_get_first_pass_params() +// av1_rc_get_second_pass_params() +// depending on the usage to set the rate control encode parameters desired. +// +// Then, call encode_frame_to_data_rate() to perform the +// actual encode. This function will in turn call encode_frame() +// one or more times, followed by one of: +// av1_rc_postencode_update() +// av1_rc_postencode_update_drop_frame() +// +// The majority of rate control parameters are only expected +// to be set in the av1_rc_get_..._params() functions and +// updated during the av1_rc_postencode_update...() functions. +// The only exceptions are av1_rc_drop_frame() and +// av1_rc_update_rate_correction_factors() functions. + +// Functions to set parameters for encoding before the actual +// encode_frame_to_data_rate() function. +void av1_rc_get_one_pass_vbr_params(struct AV1_COMP *cpi); +void av1_rc_get_one_pass_cbr_params(struct AV1_COMP *cpi); + +// Post encode update of the rate control parameters based +// on bytes used +void av1_rc_postencode_update(struct AV1_COMP *cpi, uint64_t bytes_used); +// Post encode update of the rate control parameters for dropped frames +void av1_rc_postencode_update_drop_frame(struct AV1_COMP *cpi); + +// Updates rate correction factors +// Changes only the rate correction factors in the rate control structure. +void av1_rc_update_rate_correction_factors(struct AV1_COMP *cpi); + +// Decide if we should drop this frame: For 1-pass CBR. +// Changes only the decimation count in the rate control structure +int av1_rc_drop_frame(struct AV1_COMP *cpi); + +// Computes frame size bounds. +void av1_rc_compute_frame_size_bounds(const struct AV1_COMP *cpi, + int this_frame_target, + int *frame_under_shoot_limit, + int *frame_over_shoot_limit); + +// Picks q and q bounds given the target for bits +int av1_rc_pick_q_and_bounds(const struct AV1_COMP *cpi, int *bottom_index, + int *top_index); + +// Estimates q to achieve a target bits per frame +int av1_rc_regulate_q(const struct AV1_COMP *cpi, int target_bits_per_frame, + int active_best_quality, int active_worst_quality); + +// Estimates bits per mb for a given qindex and correction factor. +int av1_rc_bits_per_mb(FRAME_TYPE frame_type, int qindex, + double correction_factor, aom_bit_depth_t bit_depth); + +// Clamping utilities for bitrate targets for iframes and pframes. +int av1_rc_clamp_iframe_target_size(const struct AV1_COMP *const cpi, + int target); +int av1_rc_clamp_pframe_target_size(const struct AV1_COMP *const cpi, + int target); +// Utility to set frame_target into the RATE_CONTROL structure +// This function is called only from the av1_rc_get_..._params() functions. +void av1_rc_set_frame_target(struct AV1_COMP *cpi, int target); + +// Computes a q delta (in "q index" terms) to get from a starting q value +// to a target q value +int av1_compute_qdelta(const RATE_CONTROL *rc, double qstart, double qtarget, + aom_bit_depth_t bit_depth); + +// Computes a q delta (in "q index" terms) to get from a starting q value +// to a value that should equate to the given rate ratio. +int av1_compute_qdelta_by_rate(const RATE_CONTROL *rc, FRAME_TYPE frame_type, + int qindex, double rate_target_ratio, + aom_bit_depth_t bit_depth); + +int av1_frame_type_qdelta(const struct AV1_COMP *cpi, int rf_level, int q); + +void av1_rc_update_framerate(struct AV1_COMP *cpi); + +void av1_rc_set_gf_interval_range(const struct AV1_COMP *const cpi, + RATE_CONTROL *const rc); + +void av1_set_target_rate(struct AV1_COMP *cpi); + +int av1_resize_one_pass_cbr(struct AV1_COMP *cpi); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_RATECTRL_H_ diff --git a/third_party/aom/av1/encoder/ratectrl_xiph.c b/third_party/aom/av1/encoder/ratectrl_xiph.c new file mode 100644 index 0000000000..b9f827528f --- /dev/null +++ b/third_party/aom/av1/encoder/ratectrl_xiph.c @@ -0,0 +1,1244 @@ +/* + * Copyright (c) 2001-2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include +#include "av1/common/odintrin.h" +#include "av1/encoder/ratectrl_xiph.h" + +#define OD_Q57(v) ((int64_t)((uint64_t)(v) << 57)) +#define OD_F_Q45(v) ((int64_t)(((v) * ((int64_t)1 << 45)))) +#define OD_F_Q12(v) ((int32_t)(((v) * ((int32_t)1 << 12)))) + +/*A rough lookup table for tan(x), 0 <= x < pi/2. + The values are Q12 fixed-point and spaced at 5 degree intervals. + These decisions are somewhat arbitrary, but sufficient for the 2nd order + Bessel follower below. + Values of x larger than 85 degrees are extrapolated from the last interval, + which is way off, but "good enough".*/ +static uint16_t OD_ROUGH_TAN_LOOKUP[18] = { 0, 358, 722, 1098, 1491, + 1910, 2365, 2868, 3437, 4096, + 4881, 5850, 7094, 8784, 11254, + 15286, 23230, 46817 }; + +/*alpha is Q24 in the range [0,0.5). + The return values is 5.12.*/ +static int od_warp_alpha(int alpha) { + int i; + int d; + int t0; + int t1; + i = alpha * 36 >> 24; + if (i >= 17) i = 16; + t0 = OD_ROUGH_TAN_LOOKUP[i]; + t1 = OD_ROUGH_TAN_LOOKUP[i + 1]; + d = alpha * 36 - (i << 24); + return (int)((((int64_t)t0 << 32) + ((t1 - t0) << 8) * (int64_t)d) >> 32); +} + +static const int64_t OD_ATANH_LOG2[32] = { + 0x32B803473F7AD0F4LL, 0x2F2A71BD4E25E916LL, 0x2E68B244BB93BA06LL, + 0x2E39FB9198CE62E4LL, 0x2E2E683F68565C8FLL, 0x2E2B850BE2077FC1LL, + 0x2E2ACC58FE7B78DBLL, 0x2E2A9E2DE52FD5F2LL, 0x2E2A92A338D53EECLL, + 0x2E2A8FC08F5E19B6LL, 0x2E2A8F07E51A485ELL, 0x2E2A8ED9BA8AF388LL, + 0x2E2A8ECE2FE7384ALL, 0x2E2A8ECB4D3E4B1ALL, 0x2E2A8ECA94940FE8LL, + 0x2E2A8ECA6669811DLL, 0x2E2A8ECA5ADEDD6ALL, 0x2E2A8ECA57FC347ELL, + 0x2E2A8ECA57438A43LL, 0x2E2A8ECA57155FB4LL, 0x2E2A8ECA5709D510LL, + 0x2E2A8ECA5706F267LL, 0x2E2A8ECA570639BDLL, 0x2E2A8ECA57060B92LL, + 0x2E2A8ECA57060008LL, 0x2E2A8ECA5705FD25LL, 0x2E2A8ECA5705FC6CLL, + 0x2E2A8ECA5705FC3ELL, 0x2E2A8ECA5705FC33LL, 0x2E2A8ECA5705FC30LL, + 0x2E2A8ECA5705FC2FLL, 0x2E2A8ECA5705FC2FLL +}; + +static int od_ilog64(int64_t v) { + static const unsigned char OD_DEBRUIJN_IDX64[64] = { + 0, 1, 2, 7, 3, 13, 8, 19, 4, 25, 14, 28, 9, 34, 20, 40, + 5, 17, 26, 38, 15, 46, 29, 48, 10, 31, 35, 54, 21, 50, 41, 57, + 63, 6, 12, 18, 24, 27, 33, 39, 16, 37, 45, 47, 30, 53, 49, 56, + 62, 11, 23, 32, 36, 44, 52, 55, 61, 22, 43, 51, 60, 42, 59, 58 + }; + int ret; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + ret = (int)v & 1; + v = (v >> 1) + 1; + ret += OD_DEBRUIJN_IDX64[v * UINT64_C(0x218A392CD3D5DBF) >> 58 & 0x3F]; + return ret; +} + +/*Computes the binary exponential of logq57. + input: a log base 2 in Q57 format + output: a 64 bit integer in Q0 (no fraction) */ +static int64_t od_bexp64(int64_t logq57) { + int64_t w; + int64_t z; + int ipart; + ipart = (int)(logq57 >> 57); + if (ipart < 0) return 0; + if (ipart >= 63) return 0x7FFFFFFFFFFFFFFFLL; + z = logq57 - OD_Q57(ipart); + if (z) { + int64_t mask; + int64_t wlo; + int i; + /*C doesn't give us 64x64->128 muls, so we use CORDIC. + This is not particularly fast, but it's not being used in time-critical + code; it is very accurate.*/ + /*z is the fractional part of the log in Q62 format. + We need 1 bit of headroom since the magnitude can get larger than 1 + during the iteration, and a sign bit.*/ + z <<= 5; + /*w is the exponential in Q61 format (since it also needs headroom and can + get as large as 2.0); we could get another bit if we dropped the sign, + but we'll recover that bit later anyway. + Ideally this should start out as + \lim_{n->\infty} 2^{61}/\product_{i=1}^n \sqrt{1-2^{-2i}} + but in order to guarantee convergence we have to repeat iterations 4, + 13 (=3*4+1), and 40 (=3*13+1, etc.), so it winds up somewhat larger.*/ + w = 0x26A3D0E401DD846DLL; + for (i = 0;; i++) { + mask = -(z < 0); + w += ((w >> (i + 1)) + mask) ^ mask; + z -= (OD_ATANH_LOG2[i] + mask) ^ mask; + /*Repeat iteration 4.*/ + if (i >= 3) break; + z *= 2; + } + for (;; i++) { + mask = -(z < 0); + w += ((w >> (i + 1)) + mask) ^ mask; + z -= (OD_ATANH_LOG2[i] + mask) ^ mask; + /*Repeat iteration 13.*/ + if (i >= 12) break; + z *= 2; + } + for (; i < 32; i++) { + mask = -(z < 0); + w += ((w >> (i + 1)) + mask) ^ mask; + z = (z - ((OD_ATANH_LOG2[i] + mask) ^ mask)) * 2; + } + wlo = 0; + /*Skip the remaining iterations unless we really require that much + precision. + We could have bailed out earlier for smaller iparts, but that would + require initializing w from a table, as the limit doesn't converge to + 61-bit precision until n=30.*/ + if (ipart > 30) { + /*For these iterations, we just update the low bits, as the high bits + can't possibly be affected. + OD_ATANH_LOG2 has also converged (it actually did so one iteration + earlier, but that's no reason for an extra special case).*/ + for (;; i++) { + mask = -(z < 0); + wlo += ((w >> i) + mask) ^ mask; + z -= (OD_ATANH_LOG2[31] + mask) ^ mask; + /*Repeat iteration 40.*/ + if (i >= 39) break; + z <<= 1; + } + for (; i < 61; i++) { + mask = -(z < 0); + wlo += ((w >> i) + mask) ^ mask; + z = (z - ((OD_ATANH_LOG2[31] + mask) ^ mask)) << 1; + } + } + w = (w << 1) + wlo; + } else { + w = (int64_t)1 << 62; + } + if (ipart < 62) { + w = ((w >> (61 - ipart)) + 1) >> 1; + } + return w; +} + +/*Computes the binary log of w + input: a 64-bit integer in Q0 (no fraction) + output: a 64-bit log in Q57 */ +static int64_t od_blog64(int64_t w) { + int64_t z; + int ipart; + if (w <= 0) return -1; + ipart = od_ilog64(w) - 1; + if (ipart > 61) { + w >>= ipart - 61; + } else { + w <<= 61 - ipart; + } + z = 0; + if (w & (w - 1)) { + int64_t x; + int64_t y; + int64_t u; + int64_t mask; + int i; + /*C doesn't give us 64x64->128 muls, so we use CORDIC. + This is not particularly fast, but it's not being used in time-critical + code; it is very accurate.*/ + /*z is the fractional part of the log in Q61 format.*/ + /*x and y are the cosh() and sinh(), respectively, in Q61 format. + We are computing z = 2*atanh(y/x) = 2*atanh((w - 1)/(w + 1)).*/ + x = w + ((int64_t)1 << 61); + y = w - ((int64_t)1 << 61); + for (i = 0; i < 4; i++) { + mask = -(y < 0); + z += ((OD_ATANH_LOG2[i] >> i) + mask) ^ mask; + u = x >> (i + 1); + x -= ((y >> (i + 1)) + mask) ^ mask; + y -= (u + mask) ^ mask; + } + /*Repeat iteration 4.*/ + for (i--; i < 13; i++) { + mask = -(y < 0); + z += ((OD_ATANH_LOG2[i] >> i) + mask) ^ mask; + u = x >> (i + 1); + x -= ((y >> (i + 1)) + mask) ^ mask; + y -= (u + mask) ^ mask; + } + /*Repeat iteration 13.*/ + for (i--; i < 32; i++) { + mask = -(y < 0); + z += ((OD_ATANH_LOG2[i] >> i) + mask) ^ mask; + u = x >> (i + 1); + x -= ((y >> (i + 1)) + mask) ^ mask; + y -= (u + mask) ^ mask; + } + /*OD_ATANH_LOG2 has converged.*/ + for (; i < 40; i++) { + mask = -(y < 0); + z += ((OD_ATANH_LOG2[31] >> i) + mask) ^ mask; + u = x >> (i + 1); + x -= ((y >> (i + 1)) + mask) ^ mask; + y -= (u + mask) ^ mask; + } + /*Repeat iteration 40.*/ + for (i--; i < 62; i++) { + mask = -(y < 0); + z += ((OD_ATANH_LOG2[31] >> i) + mask) ^ mask; + u = x >> (i + 1); + x -= ((y >> (i + 1)) + mask) ^ mask; + y -= (u + mask) ^ mask; + } + z = (z + 8) >> 4; + } + return OD_Q57(ipart) + z; +} + +/*Convenience function converts Q57 value to a clamped 32-bit Q24 value + in: input in Q57 format. + Return: same number in Q24 */ +static int32_t od_q57_to_q24(int64_t in) { + int64_t ret; + ret = (in + ((int64_t)1 << 32)) >> 33; + /*0x80000000 is automatically converted to unsigned on 32-bit systems. + -0x7FFFFFFF-1 is needed to avoid "promoting" the whole expression to + unsigned.*/ + return (int32_t)OD_CLAMPI(-0x7FFFFFFF - 1, ret, 0x7FFFFFFF); +} + +/*Binary exponential of log_scale with 24-bit fractional precision and + saturation. + log_scale: A binary logarithm in Q57 format. + Return: The binary exponential in Q24 format, saturated to 2**31-1 if + log_scale was too large.*/ +static int32_t od_bexp64_q24(int64_t log_scale) { + if (log_scale < OD_Q57(8)) { + int64_t ret; + ret = od_bexp64(log_scale + OD_Q57(24)); + return ret < 0x7FFFFFFF ? (int32_t)ret : 0x7FFFFFFF; + } + return 0x7FFFFFFF; +} + +/*Re-initialize Bessel filter coefficients with the specified delay. + This does not alter the x/y state, but changes the reaction time of the + filter. + Altering the time constant of a reactive filter without alterning internal + state is something that has to be done carefuly, but our design operates at + high enough delays and with small enough time constant changes to make it + safe.*/ +static void od_iir_bessel2_reinit(od_iir_bessel2 *f, int delay) { + int alpha; + int64_t one48; + int64_t warp; + int64_t k1; + int64_t k2; + int64_t d; + int64_t a; + int64_t ik2; + int64_t b1; + int64_t b2; + /*This borrows some code from an unreleased version of Postfish. + See the recipe at http://unicorn.us.com/alex/2polefilters.html for details + on deriving the filter coefficients.*/ + /*alpha is Q24*/ + alpha = (1 << 24) / delay; + one48 = (int64_t)1 << 48; + /*warp is 7.12*/ + warp = OD_MAXI(od_warp_alpha(alpha), 1); + /*k1 is 9.12*/ + k1 = 3 * warp; + /*k2 is 16.24.*/ + k2 = k1 * warp; + /*d is 16.15.*/ + d = ((((1 << 12) + k1) << 12) + k2 + 256) >> 9; + /*a is 0.32, since d is larger than both 1.0 and k2.*/ + a = (k2 << 23) / d; + /*ik2 is 25.24.*/ + ik2 = one48 / k2; + /*b1 is Q56; in practice, the integer ranges between -2 and 2.*/ + b1 = 2 * a * (ik2 - (1 << 24)); + /*b2 is Q56; in practice, the integer ranges between -2 and 2.*/ + b2 = (one48 << 8) - ((4 * a) << 24) - b1; + /*All of the filter parameters are Q24.*/ + f->c[0] = (int32_t)((b1 + ((int64_t)1 << 31)) >> 32); + f->c[1] = (int32_t)((b2 + ((int64_t)1 << 31)) >> 32); + f->g = (int32_t)((a + 128) >> 8); +} + +/*Initialize a 2nd order low-pass Bessel filter with the corresponding delay + and initial value. + value is Q24.*/ +static void od_iir_bessel2_init(od_iir_bessel2 *f, int delay, int32_t value) { + od_iir_bessel2_reinit(f, delay); + f->y[1] = f->y[0] = f->x[1] = f->x[0] = value; +} + +static int64_t od_iir_bessel2_update(od_iir_bessel2 *f, int32_t x) { + int64_t c0; + int64_t c1; + int64_t g; + int64_t x0; + int64_t x1; + int64_t y0; + int64_t y1; + int64_t ya; + c0 = f->c[0]; + c1 = f->c[1]; + g = f->g; + x0 = f->x[0]; + x1 = f->x[1]; + y0 = f->y[0]; + y1 = f->y[1]; + ya = ((x + x0 * 2 + x1) * g + y0 * c0 + y1 * c1 + (1 << 23)) >> 24; + f->x[1] = (int32_t)x0; + f->x[0] = x; + f->y[1] = (int32_t)y0; + f->y[0] = (int32_t)ya; + return ya; +} + +static void od_enc_rc_reset(od_rc_state *rc) { + int64_t npixels; + int64_t ibpp; + rc->bits_per_frame = (int64_t)(rc->target_bitrate / rc->framerate); + /*Insane framerates or frame sizes mean insane bitrates. + Let's not get carried away.*/ + if (rc->bits_per_frame > 0x400000000000LL) { + rc->bits_per_frame = (int64_t)0x400000000000LL; + } else { + if (rc->bits_per_frame < 32) { + rc->bits_per_frame = 32; + } + } + rc->reservoir_frame_delay = OD_MAXI(rc->reservoir_frame_delay, 12); + rc->reservoir_max = rc->bits_per_frame * rc->reservoir_frame_delay; + /*Start with a buffer fullness and fullness target of 50% */ + rc->reservoir_target = (rc->reservoir_max + 1) >> 1; + rc->reservoir_fullness = rc->reservoir_target; + /*Pick exponents and initial scales for quantizer selection.*/ + npixels = rc->frame_width * (int64_t)rc->frame_height; + rc->log_npixels = od_blog64(npixels); + ibpp = npixels / rc->bits_per_frame; + /*All of these initial scale/exp values are from Theora, and have not yet + been adapted to Daala, so they're certainly wrong. + The B-frame values especially are simply copies of the P-frame values.*/ + if (ibpp < 1) { + rc->exp[OD_I_FRAME] = 59; + rc->log_scale[OD_I_FRAME] = od_blog64(1997) - OD_Q57(OD_COEFF_SHIFT); + } else if (ibpp < 2) { + rc->exp[OD_I_FRAME] = 55; + rc->log_scale[OD_I_FRAME] = od_blog64(1604) - OD_Q57(OD_COEFF_SHIFT); + } else { + rc->exp[OD_I_FRAME] = 48; + rc->log_scale[OD_I_FRAME] = od_blog64(834) - OD_Q57(OD_COEFF_SHIFT); + } + if (ibpp < 4) { + rc->exp[OD_P_FRAME] = 100; + rc->log_scale[OD_P_FRAME] = od_blog64(2249) - OD_Q57(OD_COEFF_SHIFT); + } else if (ibpp < 8) { + rc->exp[OD_P_FRAME] = 95; + rc->log_scale[OD_P_FRAME] = od_blog64(1751) - OD_Q57(OD_COEFF_SHIFT); + } else { + rc->exp[OD_P_FRAME] = 73; + rc->log_scale[OD_P_FRAME] = od_blog64(1260) - OD_Q57(OD_COEFF_SHIFT); + } + /*Golden P-frames both use the same log_scale and exp modeling + values as regular P-frames and the same scale follower. + For convenience in the rate calculation code, we maintain a copy of + the scale and exp values in OD_GOLDEN_P_FRAME.*/ + rc->exp[OD_GOLDEN_P_FRAME] = rc->exp[OD_P_FRAME]; + rc->log_scale[OD_GOLDEN_P_FRAME] = rc->log_scale[OD_P_FRAME]; + rc->exp[OD_ALTREF_P_FRAME] = rc->exp[OD_P_FRAME]; + rc->log_scale[OD_ALTREF_P_FRAME] = rc->log_scale[OD_P_FRAME]; + /*We clamp the actual I and B frame delays to a minimum of 10 to work within + the range of values where later incrementing the delay works as designed. + 10 is not an exact choice, but rather a good working trade-off.*/ + rc->inter_p_delay = 10; + rc->inter_delay_target = rc->reservoir_frame_delay >> 1; + memset(rc->frame_count, 0, sizeof(rc->frame_count)); + /*Drop-frame tracking is concerned with more than just the basic three frame + types. + It needs to track boosted and cut subtypes (of which there is only one + right now, OD_GOLDEN_P_FRAME). */ + rc->prev_drop_count[OD_I_FRAME] = 0; + rc->log_drop_scale[OD_I_FRAME] = OD_Q57(0); + rc->prev_drop_count[OD_P_FRAME] = 0; + rc->log_drop_scale[OD_P_FRAME] = OD_Q57(0); + rc->prev_drop_count[OD_GOLDEN_P_FRAME] = 0; + rc->log_drop_scale[OD_GOLDEN_P_FRAME] = OD_Q57(0); + rc->prev_drop_count[OD_ALTREF_P_FRAME] = 0; + rc->log_drop_scale[OD_ALTREF_P_FRAME] = OD_Q57(0); + /*Set up second order followers, initialized according to corresponding + time constants.*/ + od_iir_bessel2_init(&rc->scalefilter[OD_I_FRAME], 4, + od_q57_to_q24(rc->log_scale[OD_I_FRAME])); + od_iir_bessel2_init(&rc->scalefilter[OD_P_FRAME], rc->inter_p_delay, + od_q57_to_q24(rc->log_scale[OD_P_FRAME])); + od_iir_bessel2_init(&rc->vfrfilter[OD_I_FRAME], 4, + od_bexp64_q24(rc->log_drop_scale[OD_I_FRAME])); + od_iir_bessel2_init(&rc->vfrfilter[OD_P_FRAME], 4, + od_bexp64_q24(rc->log_drop_scale[OD_P_FRAME])); + od_iir_bessel2_init(&rc->vfrfilter[OD_GOLDEN_P_FRAME], 4, + od_bexp64_q24(rc->log_drop_scale[OD_GOLDEN_P_FRAME])); + od_iir_bessel2_init(&rc->vfrfilter[OD_ALTREF_P_FRAME], 4, + od_bexp64_q24(rc->log_drop_scale[OD_ALTREF_P_FRAME])); +} + +int od_enc_rc_resize(od_rc_state *rc) { + /*If encoding has not yet begun, reset the buffer state.*/ + if (rc->cur_frame == 0) { + od_enc_rc_reset(rc); + } else { + int idt; + /*Otherwise, update the bounds on the buffer, but not the current + fullness.*/ + rc->bits_per_frame = (int64_t)(rc->target_bitrate / rc->framerate); + /*Insane framerates or frame sizes mean insane bitrates. + Let's not get carried away.*/ + if (rc->bits_per_frame > 0x400000000000LL) { + rc->bits_per_frame = (int64_t)0x400000000000LL; + } else { + if (rc->bits_per_frame < 32) { + rc->bits_per_frame = 32; + } + } + rc->reservoir_frame_delay = OD_MAXI(rc->reservoir_frame_delay, 12); + rc->reservoir_max = rc->bits_per_frame * rc->reservoir_frame_delay; + rc->reservoir_target = + ((rc->reservoir_max + 1) >> 1) + + ((rc->bits_per_frame + 2) >> 2) * + OD_MINI(rc->keyframe_rate, rc->reservoir_frame_delay); + /*Update the INTER-frame scale filter delay. + We jump to it immediately if we've already seen enough frames; otherwise + it is simply set as the new target.*/ + rc->inter_delay_target = idt = OD_MAXI(rc->reservoir_frame_delay >> 1, 10); + if (idt < OD_MINI(rc->inter_p_delay, rc->frame_count[OD_P_FRAME])) { + od_iir_bessel2_init(&rc->scalefilter[OD_P_FRAME], idt, + rc->scalefilter[OD_P_FRAME].y[0]); + rc->inter_p_delay = idt; + } + } + return 0; +} + +int od_enc_rc_init(od_rc_state *rc, int64_t bitrate, int delay_ms) { + if (rc->framerate <= 0) return 1; + if (rc->target_bitrate > 0) { + /*State has already been initialized; rather than reinitialize, + adjust the buffering for the new target rate. */ + rc->target_bitrate = bitrate; + return od_enc_rc_resize(rc); + } + rc->target_quantizer = 0; + rc->target_bitrate = bitrate; + rc->rate_bias = 0; + if (bitrate > 0) { + /* The buffer size is clamped between [12, 256], this interval is short + enough to + allow reaction, but long enough to allow looking into the next GOP + (avoiding + the case where the last frames before an I-frame get starved). + The 12 frame minimum gives us some chance to distribute bit estimation + errors in the worst case. The 256 frame maximum means we'll require 8-10 + seconds + of pre-buffering at 24-30 fps, which is not unreasonable.*/ + rc->reservoir_frame_delay = + (int)OD_MINI((delay_ms / 1000) * rc->framerate, 256); + rc->drop_frames = 1; + rc->cap_overflow = 1; + rc->cap_underflow = 0; + rc->twopass_state = 0; + od_enc_rc_reset(rc); + } + return 0; +} + +/*Scale the number of frames by the number of expected drops/duplicates.*/ +static int od_rc_scale_drop(od_rc_state *rc, int frame_type, int nframes) { + if (rc->prev_drop_count[frame_type] > 0 || + rc->log_drop_scale[frame_type] > OD_Q57(0)) { + int64_t dup_scale; + dup_scale = od_bexp64(((rc->log_drop_scale[frame_type] + + od_blog64(rc->prev_drop_count[frame_type] + 1)) >> + 1) + + OD_Q57(8)); + if (dup_scale < nframes << 8) { + int dup_scalei; + dup_scalei = (int)dup_scale; + if (dup_scalei > 0) { + nframes = ((nframes << 8) + dup_scalei - 1) / dup_scalei; + } + } else { + nframes = !!nframes; + } + } + return nframes; +} + +/*Closed form version of frame determination code. + Used by rate control to predict frame types and subtypes into the future. + No side effects, may be called any number of times. + Note that it ignores end-of-file conditions; one-pass planning *should* + ignore end-of-file. */ +int od_frame_type(od_rc_state *rc, int64_t coding_frame_count, int *is_golden, + int *is_altref, int64_t *ip_count) { + int frame_type; + if (coding_frame_count == 0) { + *is_golden = 1; + *is_altref = 1; + *ip_count = 0; + frame_type = OD_I_FRAME; + } else { + int keyrate = rc->keyframe_rate; + if (rc->closed_gop) { + int ip_per_gop; + int gop_n; + int gop_i; + ip_per_gop = (keyrate - 1) / 2; + gop_n = coding_frame_count / keyrate; + gop_i = coding_frame_count - gop_n * keyrate; + *ip_count = gop_n * ip_per_gop + (gop_i > 0) + (gop_i - 1); + frame_type = gop_i == 0 ? OD_I_FRAME : OD_P_FRAME; + } else { + int ip_per_gop; + int gop_n; + int gop_i; + ip_per_gop = (keyrate); + gop_n = (coding_frame_count - 1) / keyrate; + gop_i = coding_frame_count - gop_n * keyrate - 1; + *ip_count = (coding_frame_count > 0) + gop_n * ip_per_gop + (gop_i); + frame_type = gop_i / 1 < ip_per_gop - 1 ? OD_P_FRAME : OD_I_FRAME; + } + } + *is_golden = + (*ip_count % rc->goldenframe_rate) == 0 || frame_type == OD_I_FRAME; + *is_altref = (*ip_count % rc->altref_rate) == 0 || frame_type == OD_I_FRAME; + return frame_type; +} + +/*Count frames types forward from the current frame up to but not including + the last I-frame in reservoir_frame_delay. + If reservoir_frame_delay contains no I-frames (or the current frame is the + only I-frame), count all reservoir_frame_delay frames. + Returns the number of frames counted. + Right now, this implementation is simple, brute-force, and expensive. + It is also easy to understand and debug. + TODO: replace with a virtual FIFO that keeps running totals as + repeating the counting over-and-over will have a performance impact on + whole-file 2pass usage.*/ +static int frame_type_count(od_rc_state *rc, int nframes[OD_FRAME_NSUBTYPES]) { + int i; + int j; + int acc[OD_FRAME_NSUBTYPES]; + int count; + int reservoir_frames; + int reservoir_frame_delay; + memset(nframes, 0, OD_FRAME_NSUBTYPES * sizeof(*nframes)); + memset(acc, 0, sizeof(acc)); + count = 0; + reservoir_frames = 0; +#if 1 + /*Go ahead and count past end-of-stream. + We won't nail the exact bitrate on short files that end with a partial + GOP, but we also won't [potentially] destroy the quality of the last few + frames in that same case when we suddenly find out the stream is ending + before the original planning horizon.*/ + reservoir_frame_delay = rc->reservoir_frame_delay; +#else + /*Don't count past the end of the stream (once we know where end-of-stream + is).*/ + reservoir_frame_delay = + rc->end_of_input ? rc->input_size + 1 : rc->reservoir_frame_delay; +#endif + for (i = 0; i < reservoir_frame_delay; i++) { + int frame_type; + int is_golden; + int is_altref; + int64_t dummy; + frame_type = + od_frame_type(rc, rc->cur_frame + i, &is_golden, &is_altref, &dummy); + switch (frame_type) { + case OD_I_FRAME: { + for (j = 0; j < OD_FRAME_NSUBTYPES; j++) nframes[j] += acc[j]; + reservoir_frames += count; + memset(acc, 0, sizeof(acc)); + acc[OD_I_FRAME] = 1; + count = 1; + break; + } + case OD_P_FRAME: { + if (is_golden) { + ++acc[OD_GOLDEN_P_FRAME]; + ++count; + } else if (is_altref) { + ++acc[OD_ALTREF_P_FRAME]; + ++count; + } else { + ++acc[OD_P_FRAME]; + ++count; + } + break; + } + } + } + /*If there were no I-frames at all, or only the first frame was an I-frame, + the accumulators never flushed and still contain the counts for the + entire buffer. + In both these cases, we return these counts. + Otherwise, we discard what remains in the accumulators as they contain + the counts from and past the last I-frame.*/ + if (reservoir_frames == 0) { + for (i = 0; i < OD_FRAME_NSUBTYPES; i++) nframes[i] = acc[i]; + reservoir_frames += count; + } + return reservoir_frames; +} + +static int convert_to_ac_quant(int q, int bit_depth) { + return lrint(av1_convert_qindex_to_q(q, bit_depth)); +} + +int od_enc_rc_select_quantizers_and_lambdas(od_rc_state *rc, + int is_golden_frame, + int is_altref_frame, int frame_type, + int *bottom_idx, int *top_idx) { + int frame_subtype; + int64_t log_cur_scale; + int lossy_quantizer_min; + int lossy_quantizer_max; + double mqp_i = OD_MQP_I; + double mqp_p = OD_MQP_P; + double mqp_gp = OD_MQP_GP; + double mqp_ap = OD_MQP_AP; + int reservoir_frames; + int nframes[OD_FRAME_NSUBTYPES]; + int32_t mqp_Q12[OD_FRAME_NSUBTYPES]; + int64_t dqp_Q45[OD_FRAME_NSUBTYPES]; + /*Verify the closed-form frame type determination code matches what the + input queue set.*/ + /*One pseudo-non-closed-form caveat: + Once we've seen end-of-input, the batched frame determination code + suppresses the last open-GOP's I-frame (since it would only be + useful for the next GOP, which doesn't exist). + Thus, don't check one the input queue is drained.*/ + if (!rc->end_of_input) { + int closed_form_type; + int closed_form_golden; + int closed_form_altref; + int64_t closed_form_cur_frame; + closed_form_type = + od_frame_type(rc, rc->cur_frame, &closed_form_golden, + &closed_form_altref, &closed_form_cur_frame); + OD_UNUSED(closed_form_type); + OD_UNUSED(is_altref_frame); + assert(closed_form_type == frame_type); + assert(closed_form_cur_frame == rc->cur_frame); + assert(closed_form_altref == is_altref_frame); + assert(closed_form_golden == is_golden_frame); + } + + log_cur_scale = (int64_t)rc->scalefilter[frame_type].y[0] << 33; + + /*Count the various types and classes of frames.*/ + reservoir_frames = frame_type_count(rc, nframes); + nframes[OD_I_FRAME] = od_rc_scale_drop(rc, OD_I_FRAME, nframes[OD_I_FRAME]); + nframes[OD_P_FRAME] = od_rc_scale_drop(rc, OD_P_FRAME, nframes[OD_P_FRAME]); + nframes[OD_GOLDEN_P_FRAME] = + od_rc_scale_drop(rc, OD_GOLDEN_P_FRAME, nframes[OD_GOLDEN_P_FRAME]); + nframes[OD_ALTREF_P_FRAME] = + od_rc_scale_drop(rc, OD_ALTREF_P_FRAME, nframes[OD_ALTREF_P_FRAME]); + + switch (rc->twopass_state) { + default: break; + case 1: { + /*Pass 1 mode: use a fixed qi value.*/ + return rc->firstpass_quant; + } break; + case 2: { + int i; + int64_t scale_sum[OD_FRAME_NSUBTYPES]; + int qti; + /*Pass 2 mode: we know exactly how much of each frame type there is in + the current buffer window, and have estimates for the scales.*/ + for (i = 0; i < OD_FRAME_NSUBTYPES; i++) { + nframes[i] = rc->nframes[i]; + nframes[i] = rc->nframes[i]; + scale_sum[i] = rc->scale_sum[i]; + } + /*If we're not using the same frame type as in pass 1 (because someone + changed the keyframe interval), remove that scale estimate. + We'll add in a replacement for the correct frame type below.*/ + qti = rc->cur_metrics.frame_type; + if (qti != frame_type) { + nframes[qti]--; + scale_sum[qti] -= od_bexp64_q24(rc->cur_metrics.log_scale); + } + /*Compute log_scale estimates for each frame type from the pass-1 scales + we measured in the current window.*/ + for (qti = 0; qti < OD_FRAME_NSUBTYPES; qti++) { + rc->log_scale[qti] = nframes[qti] > 0 + ? od_blog64(scale_sum[qti]) - + od_blog64(nframes[qti]) - OD_Q57(24) + : -rc->log_npixels; + } + /*If we're not using the same frame type as in pass 1, add a scale + estimate for the corresponding frame using the current low-pass + filter value. + This is mostly to ensure we have a valid estimate even when pass 1 had + no frames of this type in the buffer window. + TODO: We could also plan ahead and figure out how many keyframes we'll + be forced to add in the current buffer window.*/ + qti = rc->cur_metrics.frame_type; + if (qti != frame_type) { + int64_t scale; + scale = rc->log_scale[frame_type] < OD_Q57(23) + ? od_bexp64(rc->log_scale[frame_type] + OD_Q57(24)) + : 0x7FFFFFFFFFFFLL; + scale *= nframes[frame_type]; + nframes[frame_type]++; + scale += od_bexp64_q24(log_cur_scale >> 33); + rc->log_scale[frame_type] = + od_blog64(scale) - od_blog64(nframes[qti]) - OD_Q57(24); + } else { + log_cur_scale = (int64_t)rc->cur_metrics.log_scale << 33; + } + } break; + } + + /*Quantizer selection sticks to the codable, lossy portion of the quantizer + range.*/ + lossy_quantizer_min = convert_to_ac_quant(rc->minq, rc->bit_depth); + lossy_quantizer_max = convert_to_ac_quant(rc->maxq, rc->bit_depth); + frame_subtype = frame_type; + /*Stash quantizer modulation by frame type.*/ + mqp_Q12[OD_I_FRAME] = OD_F_Q12(mqp_i); + mqp_Q12[OD_P_FRAME] = OD_F_Q12(mqp_p); + mqp_Q12[OD_GOLDEN_P_FRAME] = OD_F_Q12(mqp_gp); + mqp_Q12[OD_ALTREF_P_FRAME] = OD_F_Q12(mqp_ap); + dqp_Q45[OD_I_FRAME] = OD_F_Q45(OD_DQP_I); + dqp_Q45[OD_P_FRAME] = OD_F_Q45(OD_DQP_P); + dqp_Q45[OD_GOLDEN_P_FRAME] = OD_F_Q45(OD_DQP_GP); + dqp_Q45[OD_ALTREF_P_FRAME] = OD_F_Q45(OD_DQP_AP); + /*Is rate control active?*/ + if (rc->target_bitrate <= 0) { + /*Rate control is not active; derive quantizer directly from + quality parameter and frame type. */ + /*Can't use the OD_LOSSLESS macro, as it uses state.quantizer to intuit, + and we've not set it yet.*/ + if (rc->quality == 0) { + /*Lossless coding requested.*/ + rc->base_quantizer = 0; + rc->target_quantizer = 0; + } else { + int64_t log_quantizer; + + /* Adjust the modulation constants using the last frame's quantizer. */ + double mqp_delta = (255 - rc->target_quantizer) / 2000.0f; + mqp_i -= mqp_delta; + mqp_p += mqp_delta; + mqp_gp -= mqp_delta; + mqp_Q12[OD_I_FRAME] = OD_F_Q12(mqp_i); + mqp_Q12[OD_P_FRAME] = OD_F_Q12(mqp_p); + mqp_Q12[OD_GOLDEN_P_FRAME] = OD_F_Q12(mqp_gp); + mqp_Q12[OD_ALTREF_P_FRAME] = OD_F_Q12(mqp_ap); + + if (rc->quality == -1) { + /*A quality of -1 means quality was unset; use a default.*/ + rc->base_quantizer = convert_to_ac_quant(10, rc->bit_depth); + } else { + rc->base_quantizer = convert_to_ac_quant(rc->quality, rc->bit_depth); + } + + if (rc->periodic_boosts && !is_golden_frame) { + int pattern_rate = (rc->goldenframe_rate >> 1); + int dist_to_golden = rc->cur_frame % pattern_rate; + int dist_away_golden = pattern_rate - dist_to_golden; + int boost = dist_to_golden; + if (dist_away_golden > dist_to_golden) boost = dist_away_golden; + boost -= pattern_rate; + boost *= (rc->base_quantizer) / OD_PERIODIC_BOOST_DIV; + rc->base_quantizer = rc->base_quantizer + boost; + } + + /*As originally written, qp modulation is applied to the coded quantizer. + Because we now have and use a more precise target quantizer for various + calculation, that needs to be modulated as well. + Calculate what is, effectively, a fractional coded quantizer. */ + /*Get the log2 quantizer in Q57 (normalized for coefficient shift).*/ + log_quantizer = od_blog64(rc->base_quantizer) - OD_Q57(OD_COEFF_SHIFT); + /*log_quantizer to Q21.*/ + log_quantizer >>= 36; + /*scale log quantizer, result is Q33.*/ + log_quantizer *= OD_LOG_QUANTIZER_BASE_Q12; + /*Add Q33 offset to Q33 log_quantizer.*/ + log_quantizer += OD_LOG_QUANTIZER_OFFSET_Q45 >> 12; + /*Modulate quantizer according to frame type; result is Q45.*/ + log_quantizer *= mqp_Q12[frame_subtype]; + /*Add Q45 boost/cut to Q45 fractional coded quantizer.*/ + log_quantizer += dqp_Q45[frame_subtype]; + /*Back to log2 quantizer in Q57.*/ + log_quantizer = (log_quantizer - OD_LOG_QUANTIZER_OFFSET_Q45) * + OD_LOG_QUANTIZER_EXP_Q12 + + OD_Q57(OD_COEFF_SHIFT); + /*Convert Q57 log2 quantizer to unclamped linear target quantizer value.*/ + rc->target_quantizer = od_bexp64(log_quantizer); + } + } else { + int clamp; + int64_t rate_bias; + int64_t rate_total; + int base_quantizer; + int64_t log_quantizer; + int qlo; + int qhi; + int i; + /*We clamp the allowed amount of qi change (after initialization).*/ + clamp = rc->cur_frame > 0; + /*Figure out how to re-distribute bits so that we hit our fullness target + before the last keyframe in our current buffer window (after the current + frame), or the end of the buffer window, whichever comes first.*/ + /*Single pass only right now.*/ + /*If we've been missing our target, add a penalty term.*/ + rate_bias = (rc->rate_bias / (rc->cur_frame + 1000)) * reservoir_frames; + /*rate_total is the total bits available over the next + reservoir_frames frames.*/ + rate_total = rc->reservoir_fullness - rc->reservoir_target + rate_bias + + reservoir_frames * rc->bits_per_frame; + /*Find a target quantizer that meets our rate target for the specific mix + of frame types we'll have over the next frame_delay frames. + We model the rate<->quantizer relationship as: + rate = scale*(quantizer**-exp) + In this case, we have our desired rate, an exponent selected in setup, + and a scale that's been measured over our frame history, so we're + solving for the quantizer. + Exponentiation with arbitrary exponents is expensive, so we work in + the binary log domain (binary exp and log aren't too bad): + rate = e2(log2_scale - log2_quantizer * exp) + There's no easy closed form solution, so we bisection search for it.*/ + /*We do not currently allow rate control to select lossless encoding.*/ + qlo = 1; + /*If there's a quality specified, it's used to select the + coarsest base quantizer we can select. + Otherwise we can use up to and including the coarsest codable + quantizer.*/ + if (rc->quality > 0) + qhi = convert_to_ac_quant(rc->quality, rc->bit_depth); + else + qhi = lossy_quantizer_max; + base_quantizer = (qlo + qhi) >> 1; + while (qlo < qhi) { + volatile int64_t log_base_quantizer; + int64_t diff; + int64_t bits; + /*Count bits contributed by each frame type using the model.*/ + bits = 0; + log_base_quantizer = od_blog64(base_quantizer); + for (i = 0; i < OD_FRAME_NSUBTYPES; i++) { + /*Modulate base quantizer by frame type.*/ + /*Get the log2 quantizer in Q57 (normalized for coefficient shift).*/ + log_quantizer = log_base_quantizer - OD_Q57(OD_COEFF_SHIFT); + /*log_quantizer to Q21.*/ + log_quantizer >>= 36; + /*scale log quantizer, result is Q33.*/ + log_quantizer *= OD_LOG_QUANTIZER_BASE_Q12; + /*Add Q33 offset to Q33 log_quantizer.*/ + log_quantizer += OD_LOG_QUANTIZER_OFFSET_Q45 >> 12; + /*Modulate quantizer according to frame type; result is Q45.*/ + log_quantizer *= mqp_Q12[i]; + /*Add Q45 boost/cut to Q45 fractional coded quantizer.*/ + log_quantizer += dqp_Q45[i]; + /*Back to log2 quantizer in Q57.*/ + log_quantizer = (log_quantizer - OD_LOG_QUANTIZER_OFFSET_Q45) * + OD_LOG_QUANTIZER_EXP_Q12 + + OD_Q57(OD_COEFF_SHIFT); + /*Clamp modulated quantizer values.*/ + log_quantizer = OD_CLAMPI(od_blog64(lossy_quantizer_min), log_quantizer, + od_blog64(lossy_quantizer_max)); + /* All the fields here are Q57 except for the exponent which is Q6.*/ + bits += nframes[i] * od_bexp64(rc->log_scale[i] + rc->log_npixels - + (log_quantizer >> 6) * rc->exp[i]); + } + diff = bits - rate_total; + if (diff > 0) { + qlo = base_quantizer + 1; + } else if (diff < 0) { + qhi = base_quantizer - 1; + } else { + break; + } + base_quantizer = (qlo + qhi) >> 1; + } + /*If this was not one of the initial frames, limit the change in base + quantizer to within [0.8*Q,1.2*Q], where Q is the previous frame's + base quantizer.*/ + if (clamp) { + base_quantizer = OD_CLAMPI((rc->base_quantizer * 0x0CCCD + 0x8000) >> 16, + base_quantizer, + (rc->base_quantizer * 0x13333 + 0x8000) >> 16); + } + /*Modulate chosen base quantizer to produce target quantizer.*/ + log_quantizer = od_blog64(base_quantizer); + /*Get the log2 quantizer in Q57 (normalized for coefficient shift).*/ + log_quantizer -= OD_Q57(OD_COEFF_SHIFT); + /*log_quantizer to Q21.*/ + log_quantizer >>= 36; + /*scale log quantizer, result is Q33.*/ + log_quantizer *= OD_LOG_QUANTIZER_BASE_Q12; + /*Add Q33 offset to Q33 log_quantizer.*/ + log_quantizer += OD_LOG_QUANTIZER_OFFSET_Q45 >> 12; + /*Modulate quantizer according to frame type; result is Q45.*/ + log_quantizer *= mqp_Q12[frame_subtype]; + /*Add Q45 boost/cut to Q45 fractional coded quantizer.*/ + log_quantizer += dqp_Q45[frame_subtype]; + /*Back to log2 quantizer in Q57.*/ + log_quantizer = (log_quantizer - OD_LOG_QUANTIZER_OFFSET_Q45) * + OD_LOG_QUANTIZER_EXP_Q12 + + OD_Q57(OD_COEFF_SHIFT); + /*Clamp modulated quantizer values.*/ + log_quantizer = OD_CLAMPI(od_blog64(lossy_quantizer_min), log_quantizer, + od_blog64(lossy_quantizer_max)); + /*The above allocation looks only at the total rate we'll accumulate in + the next reservoir_frame_delay frames. + However we could overflow the bit reservoir on the very next frame, so + check for that here if we're not using a soft target.*/ + if (rc->cap_overflow) { + int64_t margin; + int64_t soft_limit; + int64_t log_soft_limit; + int64_t log_scale_pixels; + int64_t exp; + int64_t log_qexp; + /*Allow 3% of the buffer for prediction error. + This should be plenty, and we don't mind if we go a bit over; we only + want to keep these bits from being completely wasted.*/ + margin = (rc->reservoir_max + 31) >> 5; + /*We want to use at least this many bits next frame.*/ + soft_limit = rc->reservoir_fullness + rc->bits_per_frame - + (rc->reservoir_max - margin); + log_soft_limit = od_blog64(soft_limit); + /*If we're predicting we won't use that many bits...*/ + log_scale_pixels = rc->log_scale[frame_subtype] + rc->log_npixels; + exp = rc->exp[frame_subtype]; + log_qexp = (log_quantizer >> 6) * exp; + if (log_scale_pixels - log_qexp < log_soft_limit) { + /*Scale the adjustment based on how far into the margin we are.*/ + log_qexp += ((log_scale_pixels - log_soft_limit - log_qexp) >> 32) * + (OD_MINI(margin, soft_limit) << 32) / margin; + log_quantizer = (((log_qexp + (exp >> 1)) / exp) << 6); + } + } + /*We just checked we don't overflow the reservoir next frame, now check + we don't underflow and bust the budget (when not using a soft target). + Disabled when a quality bound is set; if we saturate quantizer to the + maximum possible size when we have a limiting max quality, the + resulting lambda can cause strange behavior.*/ + if (rc->quality == -1) { + int64_t exp; + int64_t log_qexp; + int64_t log_scale_pixels; + int64_t log_hard_limit; + /*Compute the maximum number of bits we can use in the next frame. + Allow 50% of the rate for a single frame for prediction error. + This may not be enough for keyframes or sudden changes in + complexity.*/ + log_hard_limit = + od_blog64(rc->reservoir_fullness + (rc->bits_per_frame >> 1)); + /*If we're predicting we'll use more than this...*/ + log_scale_pixels = rc->log_scale[frame_subtype] + rc->log_npixels; + exp = rc->exp[frame_subtype]; + log_qexp = (log_quantizer >> 6) * exp; + if (log_scale_pixels - log_qexp > log_hard_limit) { + /*Force the target to hit our limit exactly.*/ + log_qexp = log_scale_pixels - log_hard_limit; + log_quantizer = (log_qexp + (exp >> 1)) / exp << 6; + /*If that target is unreasonable, oh well; we'll have to drop.*/ + log_quantizer = OD_MAXI(log_quantizer, od_blog64(lossy_quantizer_max)); + } + } + /*Compute a final estimate of the number of bits we plan to use, update + the running rate bias measurement.*/ + { + int64_t log_qexp; + int64_t log_scale_pixels; + log_scale_pixels = rc->log_scale[frame_subtype] + rc->log_npixels; + log_qexp = (log_quantizer >> 6) * rc->exp[frame_subtype]; + rc->rate_bias += od_bexp64(log_scale_pixels - log_qexp); + } + rc->target_quantizer = od_bexp64(log_quantizer); + /*The various cappings and adjustments may have altered the log_quantizer + target significantly. + We can either update the base quantizer to be consistent with the + target or let it track separately. + Theora behavior effectively keeps them consistent, as it regenerates + the effective base quantizer from the target each frame rather than + saving both. + For Daala, it's easier to allow them to track separately. + For now, allow them to track separately and see how it behaves.*/ + rc->base_quantizer = base_quantizer; + } + *bottom_idx = lossy_quantizer_min; + *top_idx = lossy_quantizer_max; + rc->target_quantizer = av1_qindex_from_ac( + OD_CLAMPI(lossy_quantizer_min, rc->target_quantizer, lossy_quantizer_max), + rc->bit_depth); + return rc->target_quantizer; +} + +int od_enc_rc_update_state(od_rc_state *rc, int64_t bits, int is_golden_frame, + int is_altref_frame, int frame_type, int droppable) { + int dropped; + dropped = 0; + /*Update rate control only if rate control is active.*/ + if (rc->target_bitrate > 0) { + int64_t log_scale; + int frame_subtype; + frame_subtype = frame_type; + /*Track non-golden and golden P frame drops separately.*/ + if (is_golden_frame && frame_type == OD_P_FRAME) + frame_subtype = OD_GOLDEN_P_FRAME; + else if (is_altref_frame && frame_type == OD_P_FRAME) + frame_subtype = OD_ALTREF_P_FRAME; + if (bits <= 0) { + /*We didn't code any blocks in this frame.*/ + log_scale = OD_Q57(-64); + bits = 0; + ++rc->prev_drop_count[frame_subtype]; + } else { + int64_t log_bits; + int64_t log_qexp; + /*Compute the estimated scale factor for this frame type.*/ + log_bits = od_blog64(bits); + log_qexp = od_blog64(rc->target_quantizer); + log_qexp = (log_qexp >> 6) * (rc->exp[frame_type]); + log_scale = OD_MINI(log_bits - rc->log_npixels + log_qexp, OD_Q57(16)); + } + + switch (rc->twopass_state) { + case 1: { + int golden, altref; + int64_t ipc; + rc->cur_metrics.frame_type = + od_frame_type(rc, rc->cur_frame, &golden, &altref, &ipc); + /*Pass 1 mode: save the metrics for this frame.*/ + rc->cur_metrics.log_scale = od_q57_to_q24(log_scale); + } break; + case 2: { + /*Pass 2 mode:*/ + int m_frame_type = rc->cur_metrics.frame_type; + rc->nframes[m_frame_type]--; + rc->scale_sum[m_frame_type] -= od_bexp64_q24(rc->cur_metrics.log_scale); + } break; + } + + if (bits > 0) { + od_iir_bessel2 *f; + /*If this is the first example of the given frame type we've + seen, we immediately replace the default scale factor guess + with the estimate we just computed using the first frame.*/ + if (rc->frame_count[frame_type] == 0) { + f = rc->scalefilter + frame_type; + f->y[1] = f->y[0] = f->x[1] = f->x[0] = od_q57_to_q24(log_scale); + rc->log_scale[frame_type] = log_scale; + } else { + /*Lengthen the time constant for the inter filters as we collect more + frame statistics, until we reach our target.*/ + if (frame_type != OD_I_FRAME && + rc->inter_p_delay < rc->inter_delay_target && + rc->frame_count[frame_type] >= rc->inter_p_delay) { + od_iir_bessel2_reinit(&rc->scalefilter[frame_type], + ++rc->inter_p_delay); + } + /*Update the low-pass scale filter for this frame type + regardless of whether or not we drop this frame.*/ + rc->log_scale[frame_type] = + od_iir_bessel2_update(rc->scalefilter + frame_type, + od_q57_to_q24(log_scale)) + << 33; + } + /*If this frame busts our budget, it must be dropped.*/ + if (droppable && rc->reservoir_fullness + rc->bits_per_frame < bits) { + ++rc->prev_drop_count[frame_subtype]; + bits = 0; + dropped = 1; + } else { + uint32_t drop_count; + /*Update a low-pass filter to estimate the "real" frame rate taking + drops into account. + This is only done if the frame is coded, as it needs the final + count of dropped frames.*/ + drop_count = rc->prev_drop_count[frame_subtype] + 1; + if (drop_count > 0x7F) { + drop_count = 0x7FFFFFFF; + } else { + drop_count <<= 24; + } + rc->log_drop_scale[frame_subtype] = + od_blog64(od_iir_bessel2_update(rc->vfrfilter + frame_subtype, + drop_count)) - + OD_Q57(24); + /*Zero the drop count for this frame. + It will be increased if we drop frames.*/ + rc->prev_drop_count[frame_subtype] = 0; + } + /*Increment the frame count for filter adaptation purposes.*/ + if (!rc->twopass_state) rc->frame_count[frame_type]++; + } + rc->reservoir_fullness += rc->bits_per_frame - bits; + /*If we're too quick filling the buffer and overflow is capped, + that rate is lost forever.*/ + if (rc->cap_overflow && rc->reservoir_fullness > rc->reservoir_max) { + rc->reservoir_fullness = rc->reservoir_max; + } + /*If we're too quick draining the buffer and underflow is capped, + don't try to make up that rate later.*/ + if (rc->cap_underflow && rc->reservoir_fullness < 0) { + rc->reservoir_fullness = 0; + } + /*Adjust the bias for the real bits we've used.*/ + rc->rate_bias -= bits; + } + return dropped; +} + +static INLINE void od_rc_buffer_val(od_rc_state *rc, int64_t val, int bytes) { + while (bytes-- > 0) { + rc->twopass_buffer[rc->twopass_buffer_bytes++] = (uint8_t)(val & 0xFF); + val >>= 8; + } +} + +static INLINE int64_t od_rc_unbuffer_val(od_rc_state *rc, int bytes) { + int64_t ret = 0; + int shift = 0; + while (bytes-- > 0) { + ret |= ((int64_t)rc->twopass_buffer[rc->twopass_buffer_bytes++]) << shift; + shift += 8; + } + return ret; +} + +int od_enc_rc_2pass_out(od_rc_state *rc, struct aom_codec_pkt_list *pkt_list, + int summary) { + int i; + struct aom_codec_cx_pkt pkt; + rc->twopass_buffer = rc->firstpass_buffer; + rc->twopass_buffer_bytes = 0; + if (!rc->twopass_state) { + rc->twopass_state = 1; + for (i = 0; i < OD_FRAME_NSUBTYPES; i++) { + rc->frame_count[i] = 0; + rc->exp[i] = 0; + rc->scale_sum[i] = 0; + } + } + if (summary) { + od_rc_buffer_val(rc, OD_RC_2PASS_MAGIC, 4); + od_rc_buffer_val(rc, OD_RC_2PASS_VERSION, 1); + for (i = 0; i < OD_FRAME_NSUBTYPES; i++) { + od_rc_buffer_val(rc, rc->frame_count[i], 4); + od_rc_buffer_val(rc, rc->exp[i], 4); + od_rc_buffer_val(rc, rc->scale_sum[i], 8); + } + } else { + int frame_type = rc->cur_metrics.frame_type; + rc->scale_sum[frame_type] += od_bexp64_q24(rc->cur_metrics.log_scale); + rc->frame_count[frame_type]++; + od_rc_buffer_val(rc, rc->cur_metrics.frame_type, 1); + od_rc_buffer_val(rc, rc->cur_metrics.log_scale, 4); + } + pkt.data.twopass_stats.buf = rc->firstpass_buffer; + pkt.data.twopass_stats.sz = rc->twopass_buffer_bytes; + pkt.kind = AOM_CODEC_STATS_PKT; + aom_codec_pkt_list_add(pkt_list, &pkt); + return 0; +} + +int od_enc_rc_2pass_in(od_rc_state *rc) { + /* Enable pass 2 mode if this is the first call. */ + if (rc->twopass_state == 0) { + uint32_t i, total_frames = 0; + + if (!rc->twopass_allframes_buf || + rc->twopass_allframes_buf_size < OD_RC_2PASS_MIN) + return -1; + + /* Find summary packet at the end */ + rc->twopass_buffer = rc->twopass_allframes_buf; + rc->twopass_buffer += + rc->twopass_allframes_buf_size - OD_RC_2PASS_SUMMARY_SZ; + rc->twopass_buffer_bytes = 0; + + if (od_rc_unbuffer_val(rc, 4) != OD_RC_2PASS_MAGIC) return -1; + if (od_rc_unbuffer_val(rc, 1) != OD_RC_2PASS_VERSION) return -1; + + for (i = 0; i < OD_FRAME_NSUBTYPES; i++) { + rc->frame_count[i] = od_rc_unbuffer_val(rc, 4); + rc->exp[i] = od_rc_unbuffer_val(rc, 4); + rc->scale_sum[i] = od_rc_unbuffer_val(rc, 8); + rc->nframes[i] = rc->frame_count[i]; + total_frames += rc->frame_count[i]; + } + + if (total_frames < 1) return -1; + + if (total_frames * OD_RC_2PASS_PACKET_SZ > rc->twopass_allframes_buf_size) + return -1; + + od_enc_rc_reset(rc); + + /* Everything looks ok */ + rc->twopass_buffer = rc->twopass_allframes_buf; + rc->twopass_state = 2; + rc->twopass_buffer_bytes = 0; + } + + rc->cur_metrics.frame_type = od_rc_unbuffer_val(rc, 1); + rc->cur_metrics.log_scale = od_rc_unbuffer_val(rc, 4); + + return 0; +} diff --git a/third_party/aom/av1/encoder/ratectrl_xiph.h b/third_party/aom/av1/encoder/ratectrl_xiph.h new file mode 100644 index 0000000000..a4a9052faa --- /dev/null +++ b/third_party/aom/av1/encoder/ratectrl_xiph.h @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2001-2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if !defined(_ratectrl_xiph_H) +#define _ratectrl_xiph_H (1) + +#include "av1/encoder/ratectrl.h" +#include "aom/internal/aom_codec_internal.h" + +/*Frame types.*/ +#define OD_I_FRAME (0) +#define OD_P_FRAME (1) +#define OD_GOLDEN_P_FRAME (2) +#define OD_ALTREF_P_FRAME (3) + +#define OD_FRAME_NSUBTYPES (OD_ALTREF_P_FRAME + 1) + +/* Periodic boost (in between golden frames) strength - lower is more */ +#define OD_PERIODIC_BOOST_DIV (10) + +/* Constants for frame QP modulation <- tweak these + * Adjusts how the rate control system decides the quantizers per frame + * (sub)type */ +#define OD_MQP_I (0.98) +#define OD_MQP_P (1.06) +#define OD_MQP_GP (0.99) +#define OD_MQP_AP (0.92) +#define OD_DQP_I (-2) +#define OD_DQP_P (0) +#define OD_DQP_GP (-2) +#define OD_DQP_AP (-2) + +/*Fractional_coded_quantizer ~= + log2(quantizer / (1 << OD_COEFF_SHIFT))*6.307 + 6.235*/ +/*Base/scale factor for linear quantizer to fractional coded quantizer + conversion (6.307 * 2^12) */ +#define OD_LOG_QUANTIZER_BASE_Q12 (0x0064EB) +/*Inverse of above scale factor.*/ +#define OD_LOG_QUANTIZER_EXP_Q12 (0x000289) +/*Offset for linear quantizer to fractional coded quantizer + conversion (6.235 * 2^45) */ +#define OD_LOG_QUANTIZER_OFFSET_Q45 (0x0000C7851EB851ECLL) + +#define OD_RC_2PASS_MAGIC (0x53015641) /* [A, V, 1, S] in little endian */ +#define OD_RC_2PASS_SUMMARY_SZ (4 + 1 + (4 + 4 + 8) * OD_FRAME_NSUBTYPES) +#define OD_RC_2PASS_PACKET_SZ (1 + 4) +#define OD_RC_2PASS_MIN (OD_RC_2PASS_PACKET_SZ + OD_RC_2PASS_SUMMARY_SZ) +#define OD_RC_2PASS_VERSION (1) + +/*A 2nd order low-pass Bessel follower. + We use this for rate control because it has fast reaction time, but is + critically damped.*/ +typedef struct od_iir_bessel2 { + int32_t c[2]; + int64_t g; + int32_t x[2]; + int32_t y[2]; +} od_iir_bessel2; + +/* The 2-pass metrics associated with a single frame. */ +typedef struct od_frame_metrics { + /*The log base 2 of the scale factor for this frame in Q24 format.*/ + int64_t log_scale; + /*The frame type from pass 1.*/ + unsigned frame_type : 1; +} od_frame_metrics; + +/*Rate control setup and working state information.*/ +typedef struct od_rc_state { + /* Image format */ + int frame_width; + int frame_height; + int bit_depth; + + /* Framerate */ + double framerate; + /* Keyframe rate */ + int keyframe_rate; + /* Golden frame period */ + int goldenframe_rate; + /* Altref frame period */ + int altref_rate; + /*The target bit-rate in bits per second.*/ + int64_t target_bitrate; + /* Quality level for non-bitrate-targeting */ + int quality; + /* Copied from oxcf->frame_periodic_boost */ + int periodic_boosts; + /* Max Q */ + int maxq; + /* Min Q */ + int minq; + /* Quantizer to use for the first pass */ + int firstpass_quant; + + /* 2-pass metrics */ + od_frame_metrics cur_metrics; + + /* 2-pass state */ + int64_t scale_sum[OD_FRAME_NSUBTYPES]; + int nframes[OD_FRAME_NSUBTYPES]; + + /* 2-pass bytestream reader/writer context */ + uint8_t *twopass_buffer; + int twopass_buffer_bytes; + + /* Pass 1 stats packet storage */ + uint8_t firstpass_buffer[OD_RC_2PASS_SUMMARY_SZ]; + + /* Every state packet from the first pass in a single buffer */ + uint8_t *twopass_allframes_buf; + size_t twopass_allframes_buf_size; + + /* Actual returned quantizer */ + int target_quantizer; + /*The full-precision, unmodulated quantizer upon which + our modulated quantizers are based.*/ + int base_quantizer; + + /* Increments by 1 for each frame. */ + int64_t cur_frame; + + /* End of input flag */ + int end_of_input; + /* Closed GOP flag */ + int closed_gop; + /*The number of frames over which to distribute the reservoir usage.*/ + int reservoir_frame_delay; + /*Will we drop frames to meet bitrate target?*/ + unsigned char drop_frames; + /*Do we respect the maximum reservoir fullness?*/ + unsigned char cap_overflow; + /*Can the reservoir go negative?*/ + unsigned char cap_underflow; + /*Two-pass mode state. + 0 => 1-pass encoding. + 1 => 1st pass of 2-pass encoding. + 2 => 2nd pass of 2-pass encoding.*/ + int twopass_state; + /*The log of the number of pixels in a frame in Q57 format.*/ + int64_t log_npixels; + /*The target average bits per frame.*/ + int64_t bits_per_frame; + /*The current bit reservoir fullness (bits available to be used).*/ + int64_t reservoir_fullness; + /*The target buffer fullness. + This is where we'd like to be by the last keyframe the appears in the next + buf_delay frames.*/ + int64_t reservoir_target; + /*The maximum buffer fullness (total size of the buffer).*/ + int64_t reservoir_max; + /*The log of estimated scale factor for the rate model in Q57 format.*/ + int64_t log_scale[OD_FRAME_NSUBTYPES]; + /*The exponent used in the rate model in Q8 format.*/ + unsigned exp[OD_FRAME_NSUBTYPES]; + /*The log of an estimated scale factor used to obtain the real framerate, for + VFR sources or, e.g., 12 fps content doubled to 24 fps, etc.*/ + int64_t log_drop_scale[OD_FRAME_NSUBTYPES]; + /*The total drop count from the previous frame.*/ + uint32_t prev_drop_count[OD_FRAME_NSUBTYPES]; + /*Second-order lowpass filters to track scale and VFR/drops.*/ + od_iir_bessel2 scalefilter[OD_FRAME_NSUBTYPES]; + od_iir_bessel2 vfrfilter[OD_FRAME_NSUBTYPES]; + int frame_count[OD_FRAME_NSUBTYPES]; + int inter_p_delay; + int inter_delay_target; + /*The total accumulated estimation bias.*/ + int64_t rate_bias; +} od_rc_state; + +int od_enc_rc_init(od_rc_state *rc, int64_t bitrate, int delay_ms); + +int od_enc_rc_select_quantizers_and_lambdas(od_rc_state *rc, + int is_golden_frame, + int is_altref_frame, int frame_type, + int *bottom_idx, int *top_idx); + +/* Returns 1 if the frame should be dropped */ +int od_enc_rc_update_state(od_rc_state *rc, int64_t bits, int is_golden_frame, + int is_altref_frame, int frame_type, int droppable); + +int od_frame_type(od_rc_state *rc, int64_t coding_frame_count, int *is_golden, + int *is_altref, int64_t *ip_count); + +int od_enc_rc_resize(od_rc_state *rc); + +int od_enc_rc_2pass_out(od_rc_state *rc, struct aom_codec_pkt_list *pkt_list, + int summary); + +int od_enc_rc_2pass_in(od_rc_state *rc); + +#endif diff --git a/third_party/aom/av1/encoder/rd.c b/third_party/aom/av1/encoder/rd.c new file mode 100644 index 0000000000..f06e569e7f --- /dev/null +++ b/third_party/aom/av1/encoder/rd.c @@ -0,0 +1,1204 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "./av1_rtcd.h" + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/bitops.h" +#include "aom_ports/mem.h" +#include "aom_ports/system_state.h" + +#include "av1/common/common.h" +#include "av1/common/entropy.h" +#include "av1/common/entropymode.h" +#include "av1/common/mvref_common.h" +#include "av1/common/pred_common.h" +#include "av1/common/quant_common.h" +#include "av1/common/reconinter.h" +#include "av1/common/reconintra.h" +#include "av1/common/seg_common.h" + +#include "av1/encoder/av1_quantize.h" +#include "av1/encoder/cost.h" +#include "av1/encoder/encodemb.h" +#include "av1/encoder/encodemv.h" +#include "av1/encoder/encoder.h" +#include "av1/encoder/mcomp.h" +#include "av1/encoder/ratectrl.h" +#include "av1/encoder/rd.h" +#include "av1/encoder/tokenize.h" + +#define RD_THRESH_POW 1.25 + +// Factor to weigh the rate for switchable interp filters. +#define SWITCHABLE_INTERP_RATE_FACTOR 1 + +// The baseline rd thresholds for breaking out of the rd loop for +// certain modes are assumed to be based on 8x8 blocks. +// This table is used to correct for block size. +// The factors here are << 2 (2 = x0.5, 32 = x8 etc). +static const uint8_t rd_thresh_block_size_factor[BLOCK_SIZES] = { +#if CONFIG_CB4X4 + 2, 2, 2, +#endif + 2, 3, 3, 4, 6, 6, 8, 12, 12, 16, 24, 24, 32, +#if CONFIG_EXT_PARTITION + 48, 48, 64 +#endif // CONFIG_EXT_PARTITION +}; + +static void fill_mode_costs(AV1_COMP *cpi) { + const FRAME_CONTEXT *const fc = cpi->common.fc; + int i, j; + + for (i = 0; i < INTRA_MODES; ++i) + for (j = 0; j < INTRA_MODES; ++j) + av1_cost_tokens(cpi->y_mode_costs[i][j], av1_kf_y_mode_prob[i][j], + av1_intra_mode_tree); + + for (i = 0; i < BLOCK_SIZE_GROUPS; ++i) + av1_cost_tokens(cpi->mbmode_cost[i], fc->y_mode_prob[i], + av1_intra_mode_tree); + + for (i = 0; i < INTRA_MODES; ++i) + av1_cost_tokens(cpi->intra_uv_mode_cost[i], fc->uv_mode_prob[i], + av1_intra_mode_tree); + + for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i) + av1_cost_tokens(cpi->switchable_interp_costs[i], + fc->switchable_interp_prob[i], av1_switchable_interp_tree); + +#if CONFIG_PALETTE + for (i = 0; i < PALETTE_BLOCK_SIZES; ++i) { + av1_cost_tokens(cpi->palette_y_size_cost[i], + av1_default_palette_y_size_prob[i], av1_palette_size_tree); + av1_cost_tokens(cpi->palette_uv_size_cost[i], + av1_default_palette_uv_size_prob[i], av1_palette_size_tree); + } + + for (i = 0; i < PALETTE_SIZES; ++i) { + for (j = 0; j < PALETTE_COLOR_INDEX_CONTEXTS; ++j) { + av1_cost_tokens(cpi->palette_y_color_cost[i][j], + av1_default_palette_y_color_index_prob[i][j], + av1_palette_color_index_tree[i]); + av1_cost_tokens(cpi->palette_uv_color_cost[i][j], + av1_default_palette_uv_color_index_prob[i][j], + av1_palette_color_index_tree[i]); + } + } +#endif // CONFIG_PALETTE + + for (i = 0; i < MAX_TX_DEPTH; ++i) + for (j = 0; j < TX_SIZE_CONTEXTS; ++j) + av1_cost_tokens(cpi->tx_size_cost[i][j], fc->tx_size_probs[i][j], + av1_tx_size_tree[i]); + +#if CONFIG_EXT_TX + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + int s; + for (s = 1; s < EXT_TX_SETS_INTER; ++s) { + if (use_inter_ext_tx_for_txsize[s][i]) { + av1_cost_tokens(cpi->inter_tx_type_costs[s][i], + fc->inter_ext_tx_prob[s][i], av1_ext_tx_inter_tree[s]); + } + } + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { + if (use_intra_ext_tx_for_txsize[s][i]) { + for (j = 0; j < INTRA_MODES; ++j) + av1_cost_tokens(cpi->intra_tx_type_costs[s][i][j], + fc->intra_ext_tx_prob[s][i][j], + av1_ext_tx_intra_tree[s]); + } + } + } +#else + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + for (j = 0; j < TX_TYPES; ++j) + av1_cost_tokens(cpi->intra_tx_type_costs[i][j], + fc->intra_ext_tx_prob[i][j], av1_ext_tx_tree); + } + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + av1_cost_tokens(cpi->inter_tx_type_costs[i], fc->inter_ext_tx_prob[i], + av1_ext_tx_tree); + } +#endif // CONFIG_EXT_TX +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP + for (i = 0; i < INTRA_FILTERS + 1; ++i) + av1_cost_tokens(cpi->intra_filter_cost[i], fc->intra_filter_probs[i], + av1_intra_filter_tree); +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA +#if CONFIG_LOOP_RESTORATION + av1_cost_tokens(cpi->switchable_restore_cost, fc->switchable_restore_prob, + av1_switchable_restore_tree); +#endif // CONFIG_LOOP_RESTORATION +#if CONFIG_GLOBAL_MOTION + av1_cost_tokens(cpi->gmtype_cost, fc->global_motion_types_prob, + av1_global_motion_types_tree); +#endif // CONFIG_GLOBAL_MOTION +} + +void av1_fill_token_costs(av1_coeff_cost *c, + av1_coeff_probs_model (*p)[PLANE_TYPES]) { + int i, j, k, l; + TX_SIZE t; + for (t = 0; t < TX_SIZES; ++t) + for (i = 0; i < PLANE_TYPES; ++i) + for (j = 0; j < REF_TYPES; ++j) + for (k = 0; k < COEF_BANDS; ++k) + for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { + aom_prob probs[ENTROPY_NODES]; + av1_model_to_full_probs(p[t][i][j][k][l], probs); + av1_cost_tokens((int *)c[t][i][j][k][0][l], probs, av1_coef_tree); + av1_cost_tokens_skip((int *)c[t][i][j][k][1][l], probs, + av1_coef_tree); + assert(c[t][i][j][k][0][l][EOB_TOKEN] == + c[t][i][j][k][1][l][EOB_TOKEN]); + } +} + +// Values are now correlated to quantizer. +static int sad_per_bit16lut_8[QINDEX_RANGE]; +static int sad_per_bit4lut_8[QINDEX_RANGE]; + +#if CONFIG_HIGHBITDEPTH +static int sad_per_bit16lut_10[QINDEX_RANGE]; +static int sad_per_bit4lut_10[QINDEX_RANGE]; +static int sad_per_bit16lut_12[QINDEX_RANGE]; +static int sad_per_bit4lut_12[QINDEX_RANGE]; +#endif + +static void init_me_luts_bd(int *bit16lut, int *bit4lut, int range, + aom_bit_depth_t bit_depth) { + int i; + // Initialize the sad lut tables using a formulaic calculation for now. + // This is to make it easier to resolve the impact of experimental changes + // to the quantizer tables. + for (i = 0; i < range; i++) { + const double q = av1_convert_qindex_to_q(i, bit_depth); + bit16lut[i] = (int)(0.0418 * q + 2.4107); + bit4lut[i] = (int)(0.063 * q + 2.742); + } +} + +void av1_init_me_luts(void) { + init_me_luts_bd(sad_per_bit16lut_8, sad_per_bit4lut_8, QINDEX_RANGE, + AOM_BITS_8); +#if CONFIG_HIGHBITDEPTH + init_me_luts_bd(sad_per_bit16lut_10, sad_per_bit4lut_10, QINDEX_RANGE, + AOM_BITS_10); + init_me_luts_bd(sad_per_bit16lut_12, sad_per_bit4lut_12, QINDEX_RANGE, + AOM_BITS_12); +#endif +} + +static const int rd_boost_factor[16] = { 64, 32, 32, 32, 24, 16, 12, 12, + 8, 8, 4, 4, 2, 2, 1, 0 }; +static const int rd_frame_type_factor[FRAME_UPDATE_TYPES] = { + 128, 144, 128, 128, 144, +#if CONFIG_EXT_REFS + // TODO(zoeliu): To adjust further following factor values. + 128, 128, 128 + // TODO(weitinglin): We should investigate if the values should be the same + // as the value used by OVERLAY frame + , + 144 +#endif // CONFIG_EXT_REFS +}; + +int av1_compute_rd_mult(const AV1_COMP *cpi, int qindex) { + const int64_t q = av1_dc_quant(qindex, 0, cpi->common.bit_depth); +#if CONFIG_HIGHBITDEPTH + int64_t rdmult = 0; + switch (cpi->common.bit_depth) { + case AOM_BITS_8: rdmult = 88 * q * q / 24; break; + case AOM_BITS_10: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 4); break; + case AOM_BITS_12: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 8); break; + default: + assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12"); + return -1; + } +#else + int64_t rdmult = 88 * q * q / 24; +#endif // CONFIG_HIGHBITDEPTH + if (cpi->oxcf.pass == 2 && (cpi->common.frame_type != KEY_FRAME)) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + const FRAME_UPDATE_TYPE frame_type = gf_group->update_type[gf_group->index]; + const int boost_index = AOMMIN(15, (cpi->rc.gfu_boost / 100)); + + rdmult = (rdmult * rd_frame_type_factor[frame_type]) >> 7; + rdmult += ((rdmult * rd_boost_factor[boost_index]) >> 7); + } + if (rdmult < 1) rdmult = 1; + return (int)rdmult; +} + +static int compute_rd_thresh_factor(int qindex, aom_bit_depth_t bit_depth) { + double q; +#if CONFIG_HIGHBITDEPTH + switch (bit_depth) { + case AOM_BITS_8: q = av1_dc_quant(qindex, 0, AOM_BITS_8) / 4.0; break; + case AOM_BITS_10: q = av1_dc_quant(qindex, 0, AOM_BITS_10) / 16.0; break; + case AOM_BITS_12: q = av1_dc_quant(qindex, 0, AOM_BITS_12) / 64.0; break; + default: + assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12"); + return -1; + } +#else + (void)bit_depth; + q = av1_dc_quant(qindex, 0, AOM_BITS_8) / 4.0; +#endif // CONFIG_HIGHBITDEPTH + // TODO(debargha): Adjust the function below. + return AOMMAX((int)(pow(q, RD_THRESH_POW) * 5.12), 8); +} + +void av1_initialize_me_consts(const AV1_COMP *cpi, MACROBLOCK *x, int qindex) { +#if CONFIG_HIGHBITDEPTH + switch (cpi->common.bit_depth) { + case AOM_BITS_8: + x->sadperbit16 = sad_per_bit16lut_8[qindex]; + x->sadperbit4 = sad_per_bit4lut_8[qindex]; + break; + case AOM_BITS_10: + x->sadperbit16 = sad_per_bit16lut_10[qindex]; + x->sadperbit4 = sad_per_bit4lut_10[qindex]; + break; + case AOM_BITS_12: + x->sadperbit16 = sad_per_bit16lut_12[qindex]; + x->sadperbit4 = sad_per_bit4lut_12[qindex]; + break; + default: + assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12"); + } +#else + (void)cpi; + x->sadperbit16 = sad_per_bit16lut_8[qindex]; + x->sadperbit4 = sad_per_bit4lut_8[qindex]; +#endif // CONFIG_HIGHBITDEPTH +} + +static void set_block_thresholds(const AV1_COMMON *cm, RD_OPT *rd) { + int i, bsize, segment_id; + + for (segment_id = 0; segment_id < MAX_SEGMENTS; ++segment_id) { + const int qindex = + clamp(av1_get_qindex(&cm->seg, segment_id, cm->base_qindex) + + cm->y_dc_delta_q, + 0, MAXQ); + const int q = compute_rd_thresh_factor(qindex, cm->bit_depth); + + for (bsize = 0; bsize < BLOCK_SIZES; ++bsize) { + // Threshold here seems unnecessarily harsh but fine given actual + // range of values used for cpi->sf.thresh_mult[]. + const int t = q * rd_thresh_block_size_factor[bsize]; + const int thresh_max = INT_MAX / t; + +#if CONFIG_CB4X4 + for (i = 0; i < MAX_MODES; ++i) + rd->threshes[segment_id][bsize][i] = rd->thresh_mult[i] < thresh_max + ? rd->thresh_mult[i] * t / 4 + : INT_MAX; +#else + if (bsize >= BLOCK_8X8) { + for (i = 0; i < MAX_MODES; ++i) + rd->threshes[segment_id][bsize][i] = rd->thresh_mult[i] < thresh_max + ? rd->thresh_mult[i] * t / 4 + : INT_MAX; + } else { + for (i = 0; i < MAX_REFS; ++i) + rd->threshes[segment_id][bsize][i] = + rd->thresh_mult_sub8x8[i] < thresh_max + ? rd->thresh_mult_sub8x8[i] * t / 4 + : INT_MAX; + } +#endif + } + } +} + +#if CONFIG_REF_MV +void av1_set_mvcost(MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame, int ref, + int ref_mv_idx) { + MB_MODE_INFO_EXT *mbmi_ext = x->mbmi_ext; + int8_t rf_type = av1_ref_frame_type(x->e_mbd.mi[0]->mbmi.ref_frame); + int nmv_ctx = av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type], + mbmi_ext->ref_mv_stack[rf_type], ref, ref_mv_idx); + (void)ref_frame; + x->mvcost = x->mv_cost_stack[nmv_ctx]; + x->nmvjointcost = x->nmv_vec_cost[nmv_ctx]; + x->mvsadcost = x->mvcost; + x->nmvjointsadcost = x->nmvjointcost; +} +#endif + +void av1_initialize_rd_consts(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + MACROBLOCK *const x = &cpi->td.mb; + RD_OPT *const rd = &cpi->rd; + int i; +#if CONFIG_REF_MV + int nmv_ctx; +#endif + + aom_clear_system_state(); + + rd->RDDIV = RDDIV_BITS; // In bits (to multiply D by 128). + rd->RDMULT = av1_compute_rd_mult(cpi, cm->base_qindex + cm->y_dc_delta_q); + + set_error_per_bit(x, rd->RDMULT); + + set_block_thresholds(cm, rd); + +#if CONFIG_REF_MV + for (nmv_ctx = 0; nmv_ctx < NMV_CONTEXTS; ++nmv_ctx) { + av1_build_nmv_cost_table( + x->nmv_vec_cost[nmv_ctx], + cm->allow_high_precision_mv ? x->nmvcost_hp[nmv_ctx] + : x->nmvcost[nmv_ctx], + &cm->fc->nmvc[nmv_ctx], cm->allow_high_precision_mv); + } + x->mvcost = x->mv_cost_stack[0]; + x->nmvjointcost = x->nmv_vec_cost[0]; + x->mvsadcost = x->mvcost; + x->nmvjointsadcost = x->nmvjointcost; +#else + av1_build_nmv_cost_table( + x->nmvjointcost, cm->allow_high_precision_mv ? x->nmvcost_hp : x->nmvcost, + &cm->fc->nmvc, cm->allow_high_precision_mv); +#endif + + if (cpi->oxcf.pass != 1) { + av1_fill_token_costs(x->token_costs, cm->fc->coef_probs); + + if (cpi->sf.partition_search_type != VAR_BASED_PARTITION || + cm->frame_type == KEY_FRAME) { +#if CONFIG_EXT_PARTITION_TYPES + for (i = 0; i < PARTITION_PLOFFSET; ++i) + av1_cost_tokens(cpi->partition_cost[i], cm->fc->partition_prob[i], + av1_partition_tree); + for (; i < PARTITION_CONTEXTS_PRIMARY; ++i) + av1_cost_tokens(cpi->partition_cost[i], cm->fc->partition_prob[i], + av1_ext_partition_tree); +#else + for (i = 0; i < PARTITION_CONTEXTS_PRIMARY; ++i) + av1_cost_tokens(cpi->partition_cost[i], cm->fc->partition_prob[i], + av1_partition_tree); +#endif // CONFIG_EXT_PARTITION_TYPES +#if CONFIG_UNPOISON_PARTITION_CTX + for (; i < PARTITION_CONTEXTS_PRIMARY + PARTITION_BLOCK_SIZES; ++i) { + aom_prob p = cm->fc->partition_prob[i][PARTITION_VERT]; + assert(p > 0); + cpi->partition_cost[i][PARTITION_NONE] = INT_MAX; + cpi->partition_cost[i][PARTITION_HORZ] = INT_MAX; + cpi->partition_cost[i][PARTITION_VERT] = av1_cost_bit(p, 0); + cpi->partition_cost[i][PARTITION_SPLIT] = av1_cost_bit(p, 1); + } + for (; i < PARTITION_CONTEXTS_PRIMARY + 2 * PARTITION_BLOCK_SIZES; ++i) { + aom_prob p = cm->fc->partition_prob[i][PARTITION_HORZ]; + assert(p > 0); + cpi->partition_cost[i][PARTITION_NONE] = INT_MAX; + cpi->partition_cost[i][PARTITION_HORZ] = av1_cost_bit(p, 0); + cpi->partition_cost[i][PARTITION_VERT] = INT_MAX; + cpi->partition_cost[i][PARTITION_SPLIT] = av1_cost_bit(p, 1); + } + cpi->partition_cost[PARTITION_CONTEXTS][PARTITION_NONE] = INT_MAX; + cpi->partition_cost[PARTITION_CONTEXTS][PARTITION_HORZ] = INT_MAX; + cpi->partition_cost[PARTITION_CONTEXTS][PARTITION_VERT] = INT_MAX; + cpi->partition_cost[PARTITION_CONTEXTS][PARTITION_SPLIT] = 0; +#endif // CONFIG_UNPOISON_PARTITION_CTX + } + + fill_mode_costs(cpi); + + if (!frame_is_intra_only(cm)) { +#if CONFIG_REF_MV + for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i) { + cpi->newmv_mode_cost[i][0] = av1_cost_bit(cm->fc->newmv_prob[i], 0); + cpi->newmv_mode_cost[i][1] = av1_cost_bit(cm->fc->newmv_prob[i], 1); + } + + for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i) { + cpi->zeromv_mode_cost[i][0] = av1_cost_bit(cm->fc->zeromv_prob[i], 0); + cpi->zeromv_mode_cost[i][1] = av1_cost_bit(cm->fc->zeromv_prob[i], 1); + } + + for (i = 0; i < REFMV_MODE_CONTEXTS; ++i) { + cpi->refmv_mode_cost[i][0] = av1_cost_bit(cm->fc->refmv_prob[i], 0); + cpi->refmv_mode_cost[i][1] = av1_cost_bit(cm->fc->refmv_prob[i], 1); + } + + for (i = 0; i < DRL_MODE_CONTEXTS; ++i) { + cpi->drl_mode_cost0[i][0] = av1_cost_bit(cm->fc->drl_prob[i], 0); + cpi->drl_mode_cost0[i][1] = av1_cost_bit(cm->fc->drl_prob[i], 1); + } +#else + for (i = 0; i < INTER_MODE_CONTEXTS; ++i) + av1_cost_tokens((int *)cpi->inter_mode_cost[i], + cm->fc->inter_mode_probs[i], av1_inter_mode_tree); +#endif // CONFIG_REF_MV +#if CONFIG_EXT_INTER + for (i = 0; i < INTER_MODE_CONTEXTS; ++i) + av1_cost_tokens((int *)cpi->inter_compound_mode_cost[i], + cm->fc->inter_compound_mode_probs[i], + av1_inter_compound_mode_tree); + for (i = 0; i < BLOCK_SIZE_GROUPS; ++i) + av1_cost_tokens((int *)cpi->interintra_mode_cost[i], + cm->fc->interintra_mode_prob[i], + av1_interintra_mode_tree); +#endif // CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + for (i = BLOCK_8X8; i < BLOCK_SIZES; i++) { + av1_cost_tokens((int *)cpi->motion_mode_cost[i], + cm->fc->motion_mode_prob[i], av1_motion_mode_tree); + } +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + for (i = BLOCK_8X8; i < BLOCK_SIZES; i++) { + cpi->motion_mode_cost1[i][0] = av1_cost_bit(cm->fc->obmc_prob[i], 0); + cpi->motion_mode_cost1[i][1] = av1_cost_bit(cm->fc->obmc_prob[i], 1); + } +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + } + } +} + +static void model_rd_norm(int xsq_q10, int *r_q10, int *d_q10) { + // NOTE: The tables below must be of the same size. + + // The functions described below are sampled at the four most significant + // bits of x^2 + 8 / 256. + + // Normalized rate: + // This table models the rate for a Laplacian source with given variance + // when quantized with a uniform quantizer with given stepsize. The + // closed form expression is: + // Rn(x) = H(sqrt(r)) + sqrt(r)*[1 + H(r)/(1 - r)], + // where r = exp(-sqrt(2) * x) and x = qpstep / sqrt(variance), + // and H(x) is the binary entropy function. + static const int rate_tab_q10[] = { + 65536, 6086, 5574, 5275, 5063, 4899, 4764, 4651, 4553, 4389, 4255, 4142, + 4044, 3958, 3881, 3811, 3748, 3635, 3538, 3453, 3376, 3307, 3244, 3186, + 3133, 3037, 2952, 2877, 2809, 2747, 2690, 2638, 2589, 2501, 2423, 2353, + 2290, 2232, 2179, 2130, 2084, 2001, 1928, 1862, 1802, 1748, 1698, 1651, + 1608, 1530, 1460, 1398, 1342, 1290, 1243, 1199, 1159, 1086, 1021, 963, + 911, 864, 821, 781, 745, 680, 623, 574, 530, 490, 455, 424, + 395, 345, 304, 269, 239, 213, 190, 171, 154, 126, 104, 87, + 73, 61, 52, 44, 38, 28, 21, 16, 12, 10, 8, 6, + 5, 3, 2, 1, 1, 1, 0, 0, + }; + // Normalized distortion: + // This table models the normalized distortion for a Laplacian source + // with given variance when quantized with a uniform quantizer + // with given stepsize. The closed form expression is: + // Dn(x) = 1 - 1/sqrt(2) * x / sinh(x/sqrt(2)) + // where x = qpstep / sqrt(variance). + // Note the actual distortion is Dn * variance. + static const int dist_tab_q10[] = { + 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5, + 5, 6, 7, 7, 8, 9, 11, 12, 13, 15, 16, 17, + 18, 21, 24, 26, 29, 31, 34, 36, 39, 44, 49, 54, + 59, 64, 69, 73, 78, 88, 97, 106, 115, 124, 133, 142, + 151, 167, 184, 200, 215, 231, 245, 260, 274, 301, 327, 351, + 375, 397, 418, 439, 458, 495, 528, 559, 587, 613, 637, 659, + 680, 717, 749, 777, 801, 823, 842, 859, 874, 899, 919, 936, + 949, 960, 969, 977, 983, 994, 1001, 1006, 1010, 1013, 1015, 1017, + 1018, 1020, 1022, 1022, 1023, 1023, 1023, 1024, + }; + static const int xsq_iq_q10[] = { + 0, 4, 8, 12, 16, 20, 24, 28, 32, + 40, 48, 56, 64, 72, 80, 88, 96, 112, + 128, 144, 160, 176, 192, 208, 224, 256, 288, + 320, 352, 384, 416, 448, 480, 544, 608, 672, + 736, 800, 864, 928, 992, 1120, 1248, 1376, 1504, + 1632, 1760, 1888, 2016, 2272, 2528, 2784, 3040, 3296, + 3552, 3808, 4064, 4576, 5088, 5600, 6112, 6624, 7136, + 7648, 8160, 9184, 10208, 11232, 12256, 13280, 14304, 15328, + 16352, 18400, 20448, 22496, 24544, 26592, 28640, 30688, 32736, + 36832, 40928, 45024, 49120, 53216, 57312, 61408, 65504, 73696, + 81888, 90080, 98272, 106464, 114656, 122848, 131040, 147424, 163808, + 180192, 196576, 212960, 229344, 245728, + }; + const int tmp = (xsq_q10 >> 2) + 8; + const int k = get_msb(tmp) - 3; + const int xq = (k << 3) + ((tmp >> k) & 0x7); + const int one_q10 = 1 << 10; + const int a_q10 = ((xsq_q10 - xsq_iq_q10[xq]) << 10) >> (2 + k); + const int b_q10 = one_q10 - a_q10; + *r_q10 = (rate_tab_q10[xq] * b_q10 + rate_tab_q10[xq + 1] * a_q10) >> 10; + *d_q10 = (dist_tab_q10[xq] * b_q10 + dist_tab_q10[xq + 1] * a_q10) >> 10; +} + +void av1_model_rd_from_var_lapndz(int64_t var, unsigned int n_log2, + unsigned int qstep, int *rate, + int64_t *dist) { + // This function models the rate and distortion for a Laplacian + // source with given variance when quantized with a uniform quantizer + // with given stepsize. The closed form expressions are in: + // Hang and Chen, "Source Model for transform video coder and its + // application - Part I: Fundamental Theory", IEEE Trans. Circ. + // Sys. for Video Tech., April 1997. + if (var == 0) { + *rate = 0; + *dist = 0; + } else { + int d_q10, r_q10; + static const uint32_t MAX_XSQ_Q10 = 245727; + const uint64_t xsq_q10_64 = + (((uint64_t)qstep * qstep << (n_log2 + 10)) + (var >> 1)) / var; + const int xsq_q10 = (int)AOMMIN(xsq_q10_64, MAX_XSQ_Q10); + model_rd_norm(xsq_q10, &r_q10, &d_q10); + *rate = ROUND_POWER_OF_TWO(r_q10 << n_log2, 10 - AV1_PROB_COST_SHIFT); + *dist = (var * (int64_t)d_q10 + 512) >> 10; + } +} + +static void get_entropy_contexts_plane( + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, const struct macroblockd_plane *pd, + ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE], + ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]) { + const int num_4x4_w = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; + const int num_4x4_h = block_size_high[plane_bsize] >> tx_size_high_log2[0]; + const ENTROPY_CONTEXT *const above = pd->above_context; + const ENTROPY_CONTEXT *const left = pd->left_context; + + int i; + +#if CONFIG_CB4X4 + switch (tx_size) { + case TX_2X2: + memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w); + memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h); + break; + case TX_4X4: + for (i = 0; i < num_4x4_w; i += 2) + t_above[i] = !!*(const uint16_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 2) + t_left[i] = !!*(const uint16_t *)&left[i]; + break; + case TX_8X8: + for (i = 0; i < num_4x4_w; i += 4) + t_above[i] = !!*(const uint32_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 4) + t_left[i] = !!*(const uint32_t *)&left[i]; + break; + case TX_16X16: + for (i = 0; i < num_4x4_w; i += 8) + t_above[i] = !!*(const uint64_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 8) + t_left[i] = !!*(const uint64_t *)&left[i]; + break; + case TX_32X32: + for (i = 0; i < num_4x4_w; i += 16) + t_above[i] = + !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]); + for (i = 0; i < num_4x4_h; i += 16) + t_left[i] = + !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]); + break; + case TX_4X8: + for (i = 0; i < num_4x4_w; i += 2) + t_above[i] = !!*(const uint16_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 4) + t_left[i] = !!*(const uint32_t *)&left[i]; + break; + case TX_8X4: + for (i = 0; i < num_4x4_w; i += 4) + t_above[i] = !!*(const uint32_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 2) + t_left[i] = !!*(const uint16_t *)&left[i]; + break; + case TX_8X16: + for (i = 0; i < num_4x4_w; i += 4) + t_above[i] = !!*(const uint32_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 8) + t_left[i] = !!*(const uint64_t *)&left[i]; + break; + case TX_16X8: + for (i = 0; i < num_4x4_w; i += 8) + t_above[i] = !!*(const uint64_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 4) + t_left[i] = !!*(const uint32_t *)&left[i]; + break; + case TX_16X32: + for (i = 0; i < num_4x4_w; i += 8) + t_above[i] = !!*(const uint64_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 16) + t_left[i] = + !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]); + break; + case TX_32X16: + for (i = 0; i < num_4x4_w; i += 16) + t_above[i] = + !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]); + for (i = 0; i < num_4x4_h; i += 8) + t_left[i] = !!*(const uint64_t *)&left[i]; + break; + + default: assert(0 && "Invalid transform size."); break; + } + return; +#endif + + switch (tx_size) { + case TX_4X4: + memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w); + memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h); + break; + case TX_8X8: + for (i = 0; i < num_4x4_w; i += 2) + t_above[i] = !!*(const uint16_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 2) + t_left[i] = !!*(const uint16_t *)&left[i]; + break; + case TX_16X16: + for (i = 0; i < num_4x4_w; i += 4) + t_above[i] = !!*(const uint32_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 4) + t_left[i] = !!*(const uint32_t *)&left[i]; + break; + case TX_32X32: + for (i = 0; i < num_4x4_w; i += 8) + t_above[i] = !!*(const uint64_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 8) + t_left[i] = !!*(const uint64_t *)&left[i]; + break; +#if CONFIG_TX64X64 + case TX_64X64: + for (i = 0; i < num_4x4_w; i += 16) + t_above[i] = + !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]); + for (i = 0; i < num_4x4_h; i += 16) + t_left[i] = + !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]); + break; +#endif // CONFIG_TX64X64 + case TX_4X8: + memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w); + for (i = 0; i < num_4x4_h; i += 2) + t_left[i] = !!*(const uint16_t *)&left[i]; + break; + case TX_8X4: + for (i = 0; i < num_4x4_w; i += 2) + t_above[i] = !!*(const uint16_t *)&above[i]; + memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h); + break; + case TX_8X16: + for (i = 0; i < num_4x4_w; i += 2) + t_above[i] = !!*(const uint16_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 4) + t_left[i] = !!*(const uint32_t *)&left[i]; + break; + case TX_16X8: + for (i = 0; i < num_4x4_w; i += 4) + t_above[i] = !!*(const uint32_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 2) + t_left[i] = !!*(const uint16_t *)&left[i]; + break; + case TX_16X32: + for (i = 0; i < num_4x4_w; i += 4) + t_above[i] = !!*(const uint32_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 8) + t_left[i] = !!*(const uint64_t *)&left[i]; + break; + case TX_32X16: + for (i = 0; i < num_4x4_w; i += 8) + t_above[i] = !!*(const uint64_t *)&above[i]; + for (i = 0; i < num_4x4_h; i += 4) + t_left[i] = !!*(const uint32_t *)&left[i]; + break; + default: assert(0 && "Invalid transform size."); break; + } +} + +void av1_get_entropy_contexts(BLOCK_SIZE bsize, TX_SIZE tx_size, + const struct macroblockd_plane *pd, + ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE], + ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]) { + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); + get_entropy_contexts_plane(plane_bsize, tx_size, pd, t_above, t_left); +} + +void av1_mv_pred(const AV1_COMP *cpi, MACROBLOCK *x, uint8_t *ref_y_buffer, + int ref_y_stride, int ref_frame, BLOCK_SIZE block_size) { + int i; + int zero_seen = 0; + int best_index = 0; + int best_sad = INT_MAX; + int this_sad = INT_MAX; + int max_mv = 0; + int near_same_nearest; + uint8_t *src_y_ptr = x->plane[0].src.buf; + uint8_t *ref_y_ptr; + const int num_mv_refs = + MAX_MV_REF_CANDIDATES + + (cpi->sf.adaptive_motion_search && block_size < x->max_partition_size); + + MV pred_mv[3]; + pred_mv[0] = x->mbmi_ext->ref_mvs[ref_frame][0].as_mv; + pred_mv[1] = x->mbmi_ext->ref_mvs[ref_frame][1].as_mv; + pred_mv[2] = x->pred_mv[ref_frame]; + assert(num_mv_refs <= (int)(sizeof(pred_mv) / sizeof(pred_mv[0]))); + + near_same_nearest = x->mbmi_ext->ref_mvs[ref_frame][0].as_int == + x->mbmi_ext->ref_mvs[ref_frame][1].as_int; + // Get the sad for each candidate reference mv. + for (i = 0; i < num_mv_refs; ++i) { + const MV *this_mv = &pred_mv[i]; + int fp_row, fp_col; + + if (i == 1 && near_same_nearest) continue; + fp_row = (this_mv->row + 3 + (this_mv->row >= 0)) >> 3; + fp_col = (this_mv->col + 3 + (this_mv->col >= 0)) >> 3; + max_mv = AOMMAX(max_mv, AOMMAX(abs(this_mv->row), abs(this_mv->col)) >> 3); + + if (fp_row == 0 && fp_col == 0 && zero_seen) continue; + zero_seen |= (fp_row == 0 && fp_col == 0); + + ref_y_ptr = &ref_y_buffer[ref_y_stride * fp_row + fp_col]; + // Find sad for current vector. + this_sad = cpi->fn_ptr[block_size].sdf(src_y_ptr, x->plane[0].src.stride, + ref_y_ptr, ref_y_stride); + // Note if it is the best so far. + if (this_sad < best_sad) { + best_sad = this_sad; + best_index = i; + } + } + + // Note the index of the mv that worked best in the reference list. + x->mv_best_ref_index[ref_frame] = best_index; + x->max_mv_context[ref_frame] = max_mv; + x->pred_mv_sad[ref_frame] = best_sad; +} + +void av1_setup_pred_block(const MACROBLOCKD *xd, + struct buf_2d dst[MAX_MB_PLANE], + const YV12_BUFFER_CONFIG *src, int mi_row, int mi_col, + const struct scale_factors *scale, + const struct scale_factors *scale_uv) { + int i; + + dst[0].buf = src->y_buffer; + dst[0].stride = src->y_stride; + dst[1].buf = src->u_buffer; + dst[2].buf = src->v_buffer; + dst[1].stride = dst[2].stride = src->uv_stride; + + for (i = 0; i < MAX_MB_PLANE; ++i) { + setup_pred_plane(dst + i, xd->mi[0]->mbmi.sb_type, dst[i].buf, + i ? src->uv_crop_width : src->y_crop_width, + i ? src->uv_crop_height : src->y_crop_height, + dst[i].stride, mi_row, mi_col, i ? scale_uv : scale, + xd->plane[i].subsampling_x, xd->plane[i].subsampling_y); + } +} + +int av1_raster_block_offset(BLOCK_SIZE plane_bsize, int raster_block, + int stride) { + const int bw = b_width_log2_lookup[plane_bsize]; + const int y = 4 * (raster_block >> bw); + const int x = 4 * (raster_block & ((1 << bw) - 1)); + return y * stride + x; +} + +int16_t *av1_raster_block_offset_int16(BLOCK_SIZE plane_bsize, int raster_block, + int16_t *base) { + const int stride = block_size_wide[plane_bsize]; + return base + av1_raster_block_offset(plane_bsize, raster_block, stride); +} + +YV12_BUFFER_CONFIG *av1_get_scaled_ref_frame(const AV1_COMP *cpi, + int ref_frame) { + const AV1_COMMON *const cm = &cpi->common; + const int scaled_idx = cpi->scaled_ref_idx[ref_frame - 1]; + const int ref_idx = get_ref_frame_buf_idx(cpi, ref_frame); + return (scaled_idx != ref_idx && scaled_idx != INVALID_IDX) + ? &cm->buffer_pool->frame_bufs[scaled_idx].buf + : NULL; +} + +#if CONFIG_DUAL_FILTER +int av1_get_switchable_rate(const AV1_COMP *cpi, const MACROBLOCKD *xd) { + const AV1_COMMON *const cm = &cpi->common; + if (cm->interp_filter == SWITCHABLE) { + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + int inter_filter_cost = 0; + int dir; + + for (dir = 0; dir < 2; ++dir) { + if (has_subpel_mv_component(xd->mi[0], xd, dir) || + (mbmi->ref_frame[1] > INTRA_FRAME && + has_subpel_mv_component(xd->mi[0], xd, dir + 2))) { + const int ctx = av1_get_pred_context_switchable_interp(xd, dir); + inter_filter_cost += + cpi->switchable_interp_costs[ctx][mbmi->interp_filter[dir]]; + } + } + return SWITCHABLE_INTERP_RATE_FACTOR * inter_filter_cost; + } else { + return 0; + } +} +#else +int av1_get_switchable_rate(const AV1_COMP *cpi, const MACROBLOCKD *xd) { + const AV1_COMMON *const cm = &cpi->common; + if (cm->interp_filter == SWITCHABLE) { + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const int ctx = av1_get_pred_context_switchable_interp(xd); + return SWITCHABLE_INTERP_RATE_FACTOR * + cpi->switchable_interp_costs[ctx][mbmi->interp_filter]; + } + return 0; +} +#endif + +void av1_set_rd_speed_thresholds(AV1_COMP *cpi) { + int i; + RD_OPT *const rd = &cpi->rd; + SPEED_FEATURES *const sf = &cpi->sf; + + // Set baseline threshold values. + for (i = 0; i < MAX_MODES; ++i) rd->thresh_mult[i] = cpi->oxcf.mode == 0; + + if (sf->adaptive_rd_thresh) { + rd->thresh_mult[THR_NEARESTMV] = 300; +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_NEARESTL2] = 300; + rd->thresh_mult[THR_NEARESTL3] = 300; + rd->thresh_mult[THR_NEARESTB] = 300; +#endif // CONFIG_EXT_REFS + rd->thresh_mult[THR_NEARESTA] = 300; + rd->thresh_mult[THR_NEARESTG] = 300; + } else { + rd->thresh_mult[THR_NEARESTMV] = 0; +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_NEARESTL2] = 0; + rd->thresh_mult[THR_NEARESTL3] = 0; + rd->thresh_mult[THR_NEARESTB] = 0; +#endif // CONFIG_EXT_REFS + rd->thresh_mult[THR_NEARESTA] = 0; + rd->thresh_mult[THR_NEARESTG] = 0; + } + + rd->thresh_mult[THR_DC] += 1000; + + rd->thresh_mult[THR_NEWMV] += 1000; +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_NEWL2] += 1000; + rd->thresh_mult[THR_NEWL3] += 1000; + rd->thresh_mult[THR_NEWB] += 1000; +#endif // CONFIG_EXT_REFS + rd->thresh_mult[THR_NEWA] += 1000; + rd->thresh_mult[THR_NEWG] += 1000; + + rd->thresh_mult[THR_NEARMV] += 1000; +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_NEARL2] += 1000; + rd->thresh_mult[THR_NEARL3] += 1000; + rd->thresh_mult[THR_NEARB] += 1000; +#endif // CONFIG_EXT_REFS + rd->thresh_mult[THR_NEARA] += 1000; + rd->thresh_mult[THR_NEARG] += 1000; + + rd->thresh_mult[THR_ZEROMV] += 2000; +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_ZEROL2] += 2000; + rd->thresh_mult[THR_ZEROL3] += 2000; + rd->thresh_mult[THR_ZEROB] += 2000; +#endif // CONFIG_EXT_REFS + rd->thresh_mult[THR_ZEROG] += 2000; + rd->thresh_mult[THR_ZEROA] += 2000; + + rd->thresh_mult[THR_TM] += 1000; + +#if CONFIG_EXT_INTER + + rd->thresh_mult[THR_COMP_NEAREST_NEARESTLA] += 1000; +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_NEAREST_NEARESTL2A] += 1000; + rd->thresh_mult[THR_COMP_NEAREST_NEARESTL3A] += 1000; +#endif // CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_NEAREST_NEARESTGA] += 1000; +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_NEAREST_NEARESTLB] += 1000; + rd->thresh_mult[THR_COMP_NEAREST_NEARESTL2B] += 1000; + rd->thresh_mult[THR_COMP_NEAREST_NEARESTL3B] += 1000; + rd->thresh_mult[THR_COMP_NEAREST_NEARESTGB] += 1000; +#endif // CONFIG_EXT_REFS + +#else // CONFIG_EXT_INTER + + rd->thresh_mult[THR_COMP_NEARESTLA] += 1000; +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_NEARESTL2A] += 1000; + rd->thresh_mult[THR_COMP_NEARESTL3A] += 1000; +#endif // CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_NEARESTGA] += 1000; +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_NEARESTLB] += 1000; + rd->thresh_mult[THR_COMP_NEARESTL2B] += 1000; + rd->thresh_mult[THR_COMP_NEARESTL3B] += 1000; + rd->thresh_mult[THR_COMP_NEARESTGB] += 1000; +#endif // CONFIG_EXT_REFS + +#endif // CONFIG_EXT_INTER + +#if CONFIG_EXT_INTER + + rd->thresh_mult[THR_COMP_NEAREST_NEARLA] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARESTLA] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARLA] += 1200; + rd->thresh_mult[THR_COMP_NEAREST_NEWLA] += 1500; + rd->thresh_mult[THR_COMP_NEW_NEARESTLA] += 1500; + rd->thresh_mult[THR_COMP_NEAR_NEWLA] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEARLA] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEWLA] += 2000; + rd->thresh_mult[THR_COMP_ZERO_ZEROLA] += 2500; + +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_NEAREST_NEARL2A] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARESTL2A] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARL2A] += 1200; + rd->thresh_mult[THR_COMP_NEAREST_NEWL2A] += 1500; + rd->thresh_mult[THR_COMP_NEW_NEARESTL2A] += 1500; + rd->thresh_mult[THR_COMP_NEAR_NEWL2A] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEARL2A] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEWL2A] += 2000; + rd->thresh_mult[THR_COMP_ZERO_ZEROL2A] += 2500; + + rd->thresh_mult[THR_COMP_NEAREST_NEARL3A] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARESTL3A] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARL3A] += 1200; + rd->thresh_mult[THR_COMP_NEAREST_NEWL3A] += 1500; + rd->thresh_mult[THR_COMP_NEW_NEARESTL3A] += 1500; + rd->thresh_mult[THR_COMP_NEAR_NEWL3A] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEARL3A] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEWL3A] += 2000; + rd->thresh_mult[THR_COMP_ZERO_ZEROL3A] += 2500; +#endif // CONFIG_EXT_REFS + + rd->thresh_mult[THR_COMP_NEAREST_NEARGA] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARESTGA] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARGA] += 1200; + rd->thresh_mult[THR_COMP_NEAREST_NEWGA] += 1500; + rd->thresh_mult[THR_COMP_NEW_NEARESTGA] += 1500; + rd->thresh_mult[THR_COMP_NEAR_NEWGA] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEARGA] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEWGA] += 2000; + rd->thresh_mult[THR_COMP_ZERO_ZEROGA] += 2500; + +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_NEAREST_NEARLB] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARESTLB] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARLB] += 1200; + rd->thresh_mult[THR_COMP_NEAREST_NEWLB] += 1500; + rd->thresh_mult[THR_COMP_NEW_NEARESTLB] += 1500; + rd->thresh_mult[THR_COMP_NEAR_NEWLB] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEARLB] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEWLB] += 2000; + rd->thresh_mult[THR_COMP_ZERO_ZEROLB] += 2500; + + rd->thresh_mult[THR_COMP_NEAREST_NEARL2B] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARESTL2B] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARL2B] += 1200; + rd->thresh_mult[THR_COMP_NEAREST_NEWL2B] += 1500; + rd->thresh_mult[THR_COMP_NEW_NEARESTL2B] += 1500; + rd->thresh_mult[THR_COMP_NEAR_NEWL2B] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEARL2B] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEWL2B] += 2000; + rd->thresh_mult[THR_COMP_ZERO_ZEROL2B] += 2500; + + rd->thresh_mult[THR_COMP_NEAREST_NEARL3B] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARESTL3B] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARL3B] += 1200; + rd->thresh_mult[THR_COMP_NEAREST_NEWL3B] += 1500; + rd->thresh_mult[THR_COMP_NEW_NEARESTL3B] += 1500; + rd->thresh_mult[THR_COMP_NEAR_NEWL3B] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEARL3B] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEWL3B] += 2000; + rd->thresh_mult[THR_COMP_ZERO_ZEROL3B] += 2500; + + rd->thresh_mult[THR_COMP_NEAREST_NEARGB] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARESTGB] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARGB] += 1200; + rd->thresh_mult[THR_COMP_NEAREST_NEWGB] += 1500; + rd->thresh_mult[THR_COMP_NEW_NEARESTGB] += 1500; + rd->thresh_mult[THR_COMP_NEAR_NEWGB] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEARGB] += 1700; + rd->thresh_mult[THR_COMP_NEW_NEWGB] += 2000; + rd->thresh_mult[THR_COMP_ZERO_ZEROGB] += 2500; +#endif // CONFIG_EXT_REFS + +#else // CONFIG_EXT_INTER + + rd->thresh_mult[THR_COMP_NEARLA] += 1500; + rd->thresh_mult[THR_COMP_NEWLA] += 2000; +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_NEARL2A] += 1500; + rd->thresh_mult[THR_COMP_NEWL2A] += 2000; + rd->thresh_mult[THR_COMP_NEARL3A] += 1500; + rd->thresh_mult[THR_COMP_NEWL3A] += 2000; +#endif // CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_NEARGA] += 1500; + rd->thresh_mult[THR_COMP_NEWGA] += 2000; + +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_NEARLB] += 1500; + rd->thresh_mult[THR_COMP_NEWLB] += 2000; + rd->thresh_mult[THR_COMP_NEARL2B] += 1500; + rd->thresh_mult[THR_COMP_NEWL2B] += 2000; + rd->thresh_mult[THR_COMP_NEARL3B] += 1500; + rd->thresh_mult[THR_COMP_NEWL3B] += 2000; + rd->thresh_mult[THR_COMP_NEARGB] += 1500; + rd->thresh_mult[THR_COMP_NEWGB] += 2000; +#endif // CONFIG_EXT_REFS + + rd->thresh_mult[THR_COMP_ZEROLA] += 2500; +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_ZEROL2A] += 2500; + rd->thresh_mult[THR_COMP_ZEROL3A] += 2500; +#endif // CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_ZEROGA] += 2500; + +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_ZEROLB] += 2500; + rd->thresh_mult[THR_COMP_ZEROL2B] += 2500; + rd->thresh_mult[THR_COMP_ZEROL3B] += 2500; + rd->thresh_mult[THR_COMP_ZEROGB] += 2500; +#endif // CONFIG_EXT_REFS + +#endif // CONFIG_EXT_INTER + + rd->thresh_mult[THR_H_PRED] += 2000; + rd->thresh_mult[THR_V_PRED] += 2000; + rd->thresh_mult[THR_D135_PRED] += 2500; + rd->thresh_mult[THR_D207_PRED] += 2500; + rd->thresh_mult[THR_D153_PRED] += 2500; + rd->thresh_mult[THR_D63_PRED] += 2500; + rd->thresh_mult[THR_D117_PRED] += 2500; + rd->thresh_mult[THR_D45_PRED] += 2500; + +#if CONFIG_EXT_INTER + rd->thresh_mult[THR_COMP_INTERINTRA_ZEROL] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTL] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEARL] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEWL] += 2000; + +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_INTERINTRA_ZEROL2] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTL2] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEARL2] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEWL2] += 2000; + + rd->thresh_mult[THR_COMP_INTERINTRA_ZEROL3] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTL3] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEARL3] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEWL3] += 2000; +#endif // CONFIG_EXT_REFS + + rd->thresh_mult[THR_COMP_INTERINTRA_ZEROG] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTG] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEARG] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEWG] += 2000; + +#if CONFIG_EXT_REFS + rd->thresh_mult[THR_COMP_INTERINTRA_ZEROB] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTB] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEARB] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEWB] += 2000; +#endif // CONFIG_EXT_REFS + + rd->thresh_mult[THR_COMP_INTERINTRA_ZEROA] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTA] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEARA] += 1500; + rd->thresh_mult[THR_COMP_INTERINTRA_NEWA] += 2000; +#endif // CONFIG_EXT_INTER +} + +void av1_set_rd_speed_thresholds_sub8x8(AV1_COMP *cpi) { + static const int thresh_mult[MAX_REFS] = { +#if CONFIG_EXT_REFS + 2500, + 2500, + 2500, + 2500, + 2500, + 2500, + 4500, + 4500, + 4500, + 4500, + 4500, + 4500, + 4500, + 4500, + 2500 +#else + 2500, + 2500, + 2500, + 4500, + 4500, + 2500 +#endif // CONFIG_EXT_REFS + }; + RD_OPT *const rd = &cpi->rd; + memcpy(rd->thresh_mult_sub8x8, thresh_mult, sizeof(thresh_mult)); +} + +void av1_update_rd_thresh_fact(const AV1_COMMON *const cm, + int (*factor_buf)[MAX_MODES], int rd_thresh, + int bsize, int best_mode_index) { + if (rd_thresh > 0) { +#if CONFIG_CB4X4 + const int top_mode = MAX_MODES; +#else + const int top_mode = bsize < BLOCK_8X8 ? MAX_REFS : MAX_MODES; +#endif + int mode; + for (mode = 0; mode < top_mode; ++mode) { + const BLOCK_SIZE min_size = AOMMAX(bsize - 1, BLOCK_4X4); + const BLOCK_SIZE max_size = AOMMIN(bsize + 2, (int)cm->sb_size); + BLOCK_SIZE bs; + for (bs = min_size; bs <= max_size; ++bs) { + int *const fact = &factor_buf[bs][mode]; + if (mode == best_mode_index) { + *fact -= (*fact >> 4); + } else { + *fact = AOMMIN(*fact + RD_THRESH_INC, rd_thresh * RD_THRESH_MAX_FACT); + } + } + } + } +} + +int av1_get_intra_cost_penalty(int qindex, int qdelta, + aom_bit_depth_t bit_depth) { + const int q = av1_dc_quant(qindex, qdelta, bit_depth); +#if CONFIG_HIGHBITDEPTH + switch (bit_depth) { + case AOM_BITS_8: return 20 * q; + case AOM_BITS_10: return 5 * q; + case AOM_BITS_12: return ROUND_POWER_OF_TWO(5 * q, 2); + default: + assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12"); + return -1; + } +#else + return 20 * q; +#endif // CONFIG_HIGHBITDEPTH +} diff --git a/third_party/aom/av1/encoder/rd.h b/third_party/aom/av1/encoder/rd.h new file mode 100644 index 0000000000..c0ac1f7e75 --- /dev/null +++ b/third_party/aom/av1/encoder/rd.h @@ -0,0 +1,505 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_RD_H_ +#define AV1_ENCODER_RD_H_ + +#include + +#if CONFIG_ANS +#include "aom_dsp/ans.h" +#endif // CONFIG_ANS +#include "av1/common/blockd.h" + +#include "av1/encoder/block.h" +#include "av1/encoder/context_tree.h" +#include "av1/encoder/cost.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define RDDIV_BITS 7 +#define RD_EPB_SHIFT 6 + +#define RDCOST(RM, DM, R, D) \ + (ROUND_POWER_OF_TWO(((int64_t)R) * (RM), AV1_PROB_COST_SHIFT) + (D << DM)) + +#define RDCOST_DBL(RM, DM, R, D) \ + (((((double)(R)) * (RM)) / (double)(1 << AV1_PROB_COST_SHIFT)) + \ + ((double)(D) * (1 << (DM)))) + +#define QIDX_SKIP_THRESH 115 + +#define MV_COST_WEIGHT 108 +#define MV_COST_WEIGHT_SUB 120 + +#define INVALID_MV 0x80008000 + +#if CONFIG_EXT_REFS +#define MAX_REFS 15 +#else +#define MAX_REFS 6 +#endif // CONFIG_EXT_REFS + +#define RD_THRESH_MAX_FACT 64 +#define RD_THRESH_INC 1 + +// This enumerator type needs to be kept aligned with the mode order in +// const MODE_DEFINITION av1_mode_order[MAX_MODES] used in the rd code. +typedef enum { + THR_NEARESTMV, +#if CONFIG_EXT_REFS + THR_NEARESTL2, + THR_NEARESTL3, + THR_NEARESTB, +#endif // CONFIG_EXT_REFS + THR_NEARESTA, + THR_NEARESTG, + + THR_DC, + + THR_NEWMV, +#if CONFIG_EXT_REFS + THR_NEWL2, + THR_NEWL3, + THR_NEWB, +#endif // CONFIG_EXT_REFS + THR_NEWA, + THR_NEWG, + + THR_NEARMV, +#if CONFIG_EXT_REFS + THR_NEARL2, + THR_NEARL3, + THR_NEARB, +#endif // CONFIG_EXT_REFS + THR_NEARA, + THR_NEARG, + + THR_ZEROMV, +#if CONFIG_EXT_REFS + THR_ZEROL2, + THR_ZEROL3, + THR_ZEROB, +#endif // CONFIG_EXT_REFS + THR_ZEROG, + THR_ZEROA, + +#if CONFIG_EXT_INTER + + THR_COMP_NEAREST_NEARESTLA, +#if CONFIG_EXT_REFS + THR_COMP_NEAREST_NEARESTL2A, + THR_COMP_NEAREST_NEARESTL3A, +#endif // CONFIG_EXT_REFS + THR_COMP_NEAREST_NEARESTGA, +#if CONFIG_EXT_REFS + THR_COMP_NEAREST_NEARESTLB, + THR_COMP_NEAREST_NEARESTL2B, + THR_COMP_NEAREST_NEARESTL3B, + THR_COMP_NEAREST_NEARESTGB, +#endif // CONFIG_EXT_REFS + +#else // CONFIG_EXT_INTER + + THR_COMP_NEARESTLA, +#if CONFIG_EXT_REFS + THR_COMP_NEARESTL2A, + THR_COMP_NEARESTL3A, +#endif // CONFIG_EXT_REFS + THR_COMP_NEARESTGA, +#if CONFIG_EXT_REFS + THR_COMP_NEARESTLB, + THR_COMP_NEARESTL2B, + THR_COMP_NEARESTL3B, + THR_COMP_NEARESTGB, +#endif // CONFIG_EXT_REFS + +#endif // CONFIG_EXT_INTER + + THR_TM, + +#if CONFIG_ALT_INTRA + THR_SMOOTH, +#endif // CONFIG_ALT_INTRA + +#if CONFIG_EXT_INTER + + THR_COMP_NEAR_NEARESTLA, + THR_COMP_NEAREST_NEARLA, + THR_COMP_NEAR_NEARLA, + THR_COMP_NEW_NEARESTLA, + THR_COMP_NEAREST_NEWLA, + THR_COMP_NEW_NEARLA, + THR_COMP_NEAR_NEWLA, + THR_COMP_NEW_NEWLA, + THR_COMP_ZERO_ZEROLA, + +#if CONFIG_EXT_REFS + THR_COMP_NEAR_NEARESTL2A, + THR_COMP_NEAREST_NEARL2A, + THR_COMP_NEAR_NEARL2A, + THR_COMP_NEW_NEARESTL2A, + THR_COMP_NEAREST_NEWL2A, + THR_COMP_NEW_NEARL2A, + THR_COMP_NEAR_NEWL2A, + THR_COMP_NEW_NEWL2A, + THR_COMP_ZERO_ZEROL2A, + + THR_COMP_NEAR_NEARESTL3A, + THR_COMP_NEAREST_NEARL3A, + THR_COMP_NEAR_NEARL3A, + THR_COMP_NEW_NEARESTL3A, + THR_COMP_NEAREST_NEWL3A, + THR_COMP_NEW_NEARL3A, + THR_COMP_NEAR_NEWL3A, + THR_COMP_NEW_NEWL3A, + THR_COMP_ZERO_ZEROL3A, +#endif // CONFIG_EXT_REFS + + THR_COMP_NEAR_NEARESTGA, + THR_COMP_NEAREST_NEARGA, + THR_COMP_NEAR_NEARGA, + THR_COMP_NEW_NEARESTGA, + THR_COMP_NEAREST_NEWGA, + THR_COMP_NEW_NEARGA, + THR_COMP_NEAR_NEWGA, + THR_COMP_NEW_NEWGA, + THR_COMP_ZERO_ZEROGA, + +#if CONFIG_EXT_REFS + THR_COMP_NEAR_NEARESTLB, + THR_COMP_NEAREST_NEARLB, + THR_COMP_NEAR_NEARLB, + THR_COMP_NEW_NEARESTLB, + THR_COMP_NEAREST_NEWLB, + THR_COMP_NEW_NEARLB, + THR_COMP_NEAR_NEWLB, + THR_COMP_NEW_NEWLB, + THR_COMP_ZERO_ZEROLB, + + THR_COMP_NEAR_NEARESTL2B, + THR_COMP_NEAREST_NEARL2B, + THR_COMP_NEAR_NEARL2B, + THR_COMP_NEW_NEARESTL2B, + THR_COMP_NEAREST_NEWL2B, + THR_COMP_NEW_NEARL2B, + THR_COMP_NEAR_NEWL2B, + THR_COMP_NEW_NEWL2B, + THR_COMP_ZERO_ZEROL2B, + + THR_COMP_NEAR_NEARESTL3B, + THR_COMP_NEAREST_NEARL3B, + THR_COMP_NEAR_NEARL3B, + THR_COMP_NEW_NEARESTL3B, + THR_COMP_NEAREST_NEWL3B, + THR_COMP_NEW_NEARL3B, + THR_COMP_NEAR_NEWL3B, + THR_COMP_NEW_NEWL3B, + THR_COMP_ZERO_ZEROL3B, + + THR_COMP_NEAR_NEARESTGB, + THR_COMP_NEAREST_NEARGB, + THR_COMP_NEAR_NEARGB, + THR_COMP_NEW_NEARESTGB, + THR_COMP_NEAREST_NEWGB, + THR_COMP_NEW_NEARGB, + THR_COMP_NEAR_NEWGB, + THR_COMP_NEW_NEWGB, + THR_COMP_ZERO_ZEROGB, +#endif // CONFIG_EXT_REFS + +#else // CONFIG_EXT_INTER + + THR_COMP_NEARLA, + THR_COMP_NEWLA, +#if CONFIG_EXT_REFS + THR_COMP_NEARL2A, + THR_COMP_NEWL2A, + THR_COMP_NEARL3A, + THR_COMP_NEWL3A, +#endif // CONFIG_EXT_REFS + THR_COMP_NEARGA, + THR_COMP_NEWGA, + +#if CONFIG_EXT_REFS + THR_COMP_NEARLB, + THR_COMP_NEWLB, + THR_COMP_NEARL2B, + THR_COMP_NEWL2B, + THR_COMP_NEARL3B, + THR_COMP_NEWL3B, + THR_COMP_NEARGB, + THR_COMP_NEWGB, +#endif // CONFIG_EXT_REFS + + THR_COMP_ZEROLA, +#if CONFIG_EXT_REFS + THR_COMP_ZEROL2A, + THR_COMP_ZEROL3A, +#endif // CONFIG_EXT_REFS + THR_COMP_ZEROGA, + +#if CONFIG_EXT_REFS + THR_COMP_ZEROLB, + THR_COMP_ZEROL2B, + THR_COMP_ZEROL3B, + THR_COMP_ZEROGB, +#endif // CONFIG_EXT_REFS + +#endif // CONFIG_EXT_INTER + + THR_H_PRED, + THR_V_PRED, + THR_D135_PRED, + THR_D207_PRED, + THR_D153_PRED, + THR_D63_PRED, + THR_D117_PRED, + THR_D45_PRED, + +#if CONFIG_EXT_INTER + THR_COMP_INTERINTRA_ZEROL, + THR_COMP_INTERINTRA_NEARESTL, + THR_COMP_INTERINTRA_NEARL, + THR_COMP_INTERINTRA_NEWL, + +#if CONFIG_EXT_REFS + THR_COMP_INTERINTRA_ZEROL2, + THR_COMP_INTERINTRA_NEARESTL2, + THR_COMP_INTERINTRA_NEARL2, + THR_COMP_INTERINTRA_NEWL2, + + THR_COMP_INTERINTRA_ZEROL3, + THR_COMP_INTERINTRA_NEARESTL3, + THR_COMP_INTERINTRA_NEARL3, + THR_COMP_INTERINTRA_NEWL3, +#endif // CONFIG_EXT_REFS + + THR_COMP_INTERINTRA_ZEROG, + THR_COMP_INTERINTRA_NEARESTG, + THR_COMP_INTERINTRA_NEARG, + THR_COMP_INTERINTRA_NEWG, + +#if CONFIG_EXT_REFS + THR_COMP_INTERINTRA_ZEROB, + THR_COMP_INTERINTRA_NEARESTB, + THR_COMP_INTERINTRA_NEARB, + THR_COMP_INTERINTRA_NEWB, +#endif // CONFIG_EXT_REFS + + THR_COMP_INTERINTRA_ZEROA, + THR_COMP_INTERINTRA_NEARESTA, + THR_COMP_INTERINTRA_NEARA, + THR_COMP_INTERINTRA_NEWA, +#endif // CONFIG_EXT_INTER + MAX_MODES +} THR_MODES; + +typedef enum { + THR_LAST, +#if CONFIG_EXT_REFS + THR_LAST2, + THR_LAST3, + THR_BWDR, +#endif // CONFIG_EXT_REFS + THR_GOLD, + THR_ALTR, + + THR_COMP_LA, +#if CONFIG_EXT_REFS + THR_COMP_L2A, + THR_COMP_L3A, +#endif // CONFIG_EXT_REFS + THR_COMP_GA, + +#if CONFIG_EXT_REFS + THR_COMP_LB, + THR_COMP_L2B, + THR_COMP_L3B, + THR_COMP_GB, +#endif // CONFIG_EXT_REFS + + THR_INTRA, +} THR_MODES_SUB8X8; + +typedef struct RD_OPT { + // Thresh_mult is used to set a threshold for the rd score. A higher value + // means that we will accept the best mode so far more often. This number + // is used in combination with the current block size, and thresh_freq_fact + // to pick a threshold. + int thresh_mult[MAX_MODES]; + int thresh_mult_sub8x8[MAX_REFS]; + + int threshes[MAX_SEGMENTS][BLOCK_SIZES][MAX_MODES]; + + int64_t prediction_type_threshes[TOTAL_REFS_PER_FRAME][REFERENCE_MODES]; + + int RDMULT; + int RDDIV; +} RD_OPT; + +static INLINE void av1_init_rd_stats(RD_STATS *rd_stats) { +#if CONFIG_RD_DEBUG + int plane; +#endif + rd_stats->rate = 0; + rd_stats->dist = 0; + rd_stats->rdcost = 0; + rd_stats->sse = 0; + rd_stats->skip = 1; +#if CONFIG_RD_DEBUG + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + rd_stats->txb_coeff_cost[plane] = 0; +#if CONFIG_VAR_TX + { + int r, c; + for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r) + for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c) + rd_stats->txb_coeff_cost_map[plane][r][c] = 0; + } +#endif + } +#endif +} + +static INLINE void av1_invalid_rd_stats(RD_STATS *rd_stats) { +#if CONFIG_RD_DEBUG + int plane; +#endif + rd_stats->rate = INT_MAX; + rd_stats->dist = INT64_MAX; + rd_stats->rdcost = INT64_MAX; + rd_stats->sse = INT64_MAX; + rd_stats->skip = 0; +#if CONFIG_RD_DEBUG + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + rd_stats->txb_coeff_cost[plane] = INT_MAX; +#if CONFIG_VAR_TX + { + int r, c; + for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r) + for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c) + rd_stats->txb_coeff_cost_map[plane][r][c] = INT_MAX; + } +#endif + } +#endif +} + +static INLINE void av1_merge_rd_stats(RD_STATS *rd_stats_dst, + const RD_STATS *rd_stats_src) { +#if CONFIG_RD_DEBUG + int plane; +#endif + rd_stats_dst->rate += rd_stats_src->rate; + rd_stats_dst->dist += rd_stats_src->dist; + rd_stats_dst->sse += rd_stats_src->sse; + rd_stats_dst->skip &= rd_stats_src->skip; +#if CONFIG_RD_DEBUG + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + rd_stats_dst->txb_coeff_cost[plane] += rd_stats_src->txb_coeff_cost[plane]; +#if CONFIG_VAR_TX + { + // TODO(angiebird): optimize this part + int r, c; + int ref_txb_coeff_cost = 0; + for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r) + for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c) { + rd_stats_dst->txb_coeff_cost_map[plane][r][c] += + rd_stats_src->txb_coeff_cost_map[plane][r][c]; + ref_txb_coeff_cost += rd_stats_dst->txb_coeff_cost_map[plane][r][c]; + } + assert(ref_txb_coeff_cost == rd_stats_dst->txb_coeff_cost[plane]); + } +#endif + } +#endif +} + +struct TileInfo; +struct TileDataEnc; +struct AV1_COMP; +struct macroblock; + +int av1_compute_rd_mult(const struct AV1_COMP *cpi, int qindex); + +void av1_initialize_rd_consts(struct AV1_COMP *cpi); + +void av1_initialize_me_consts(const struct AV1_COMP *cpi, MACROBLOCK *x, + int qindex); + +void av1_model_rd_from_var_lapndz(int64_t var, unsigned int n, + unsigned int qstep, int *rate, int64_t *dist); + +int av1_get_switchable_rate(const struct AV1_COMP *cpi, const MACROBLOCKD *xd); + +int av1_raster_block_offset(BLOCK_SIZE plane_bsize, int raster_block, + int stride); + +int16_t *av1_raster_block_offset_int16(BLOCK_SIZE plane_bsize, int raster_block, + int16_t *base); + +YV12_BUFFER_CONFIG *av1_get_scaled_ref_frame(const struct AV1_COMP *cpi, + int ref_frame); + +void av1_init_me_luts(void); + +#if CONFIG_REF_MV +void av1_set_mvcost(MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame, int ref, + int ref_mv_idx); +#endif + +void av1_get_entropy_contexts(BLOCK_SIZE bsize, TX_SIZE tx_size, + const struct macroblockd_plane *pd, + ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE], + ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]); + +void av1_set_rd_speed_thresholds(struct AV1_COMP *cpi); + +void av1_set_rd_speed_thresholds_sub8x8(struct AV1_COMP *cpi); + +void av1_update_rd_thresh_fact(const AV1_COMMON *const cm, + int (*fact)[MAX_MODES], int rd_thresh, int bsize, + int best_mode_index); + +void av1_fill_token_costs(av1_coeff_cost *c, + av1_coeff_probs_model (*p)[PLANE_TYPES]); + +static INLINE int rd_less_than_thresh(int64_t best_rd, int thresh, + int thresh_fact) { + return best_rd < ((int64_t)thresh * thresh_fact >> 5) || thresh == INT_MAX; +} + +void av1_mv_pred(const struct AV1_COMP *cpi, MACROBLOCK *x, + uint8_t *ref_y_buffer, int ref_y_stride, int ref_frame, + BLOCK_SIZE block_size); + +static INLINE void set_error_per_bit(MACROBLOCK *x, int rdmult) { + x->errorperbit = rdmult >> RD_EPB_SHIFT; + x->errorperbit += (x->errorperbit == 0); +} + +void av1_setup_pred_block(const MACROBLOCKD *xd, + struct buf_2d dst[MAX_MB_PLANE], + const YV12_BUFFER_CONFIG *src, int mi_row, int mi_col, + const struct scale_factors *scale, + const struct scale_factors *scale_uv); + +int av1_get_intra_cost_penalty(int qindex, int qdelta, + aom_bit_depth_t bit_depth); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_RD_H_ diff --git a/third_party/aom/av1/encoder/rdopt.c b/third_party/aom/av1/encoder/rdopt.c new file mode 100644 index 0000000000..a1096f782d --- /dev/null +++ b/third_party/aom/av1/encoder/rdopt.c @@ -0,0 +1,12713 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/blend.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" +#include "aom_ports/system_state.h" + +#include "av1/common/common.h" +#include "av1/common/common_data.h" +#include "av1/common/entropy.h" +#include "av1/common/entropymode.h" +#include "av1/common/idct.h" +#include "av1/common/mvref_common.h" +#include "av1/common/pred_common.h" +#include "av1/common/quant_common.h" +#include "av1/common/reconinter.h" +#include "av1/common/reconintra.h" +#include "av1/common/scan.h" +#include "av1/common/seg_common.h" +#if CONFIG_LV_MAP +#include "av1/common/txb_common.h" +#endif +#if CONFIG_WARPED_MOTION +#include "av1/common/warped_motion.h" +#endif // CONFIG_WARPED_MOTION + +#include "av1/encoder/aq_variance.h" +#include "av1/encoder/av1_quantize.h" +#include "av1/encoder/cost.h" +#include "av1/encoder/encodemb.h" +#include "av1/encoder/encodemv.h" +#include "av1/encoder/encoder.h" +#if CONFIG_LV_MAP +#include "av1/encoder/encodetxb.h" +#endif +#include "av1/encoder/hybrid_fwd_txfm.h" +#include "av1/encoder/mcomp.h" +#if CONFIG_PALETTE +#include "av1/encoder/palette.h" +#endif // CONFIG_PALETTE +#include "av1/encoder/ratectrl.h" +#include "av1/encoder/rd.h" +#include "av1/encoder/rdopt.h" +#include "av1/encoder/tokenize.h" +#if CONFIG_PVQ +#include "av1/encoder/pvq_encoder.h" +#endif // CONFIG_PVQ +#if CONFIG_PVQ || CONFIG_DAALA_DIST +#include "av1/common/pvq.h" +#endif // CONFIG_PVQ || CONFIG_DAALA_DIST +#if CONFIG_DUAL_FILTER +#define DUAL_FILTER_SET_SIZE (SWITCHABLE_FILTERS * SWITCHABLE_FILTERS) +static const int filter_sets[DUAL_FILTER_SET_SIZE][2] = { + { 0, 0 }, { 0, 1 }, { 0, 2 }, { 0, 3 }, { 1, 0 }, { 1, 1 }, + { 1, 2 }, { 1, 3 }, { 2, 0 }, { 2, 1 }, { 2, 2 }, { 2, 3 }, + { 3, 0 }, { 3, 1 }, { 3, 2 }, { 3, 3 }, +}; +#endif // CONFIG_DUAL_FILTER + +#if CONFIG_EXT_REFS + +#define LAST_FRAME_MODE_MASK \ + ((1 << INTRA_FRAME) | (1 << LAST2_FRAME) | (1 << LAST3_FRAME) | \ + (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME)) +#define LAST2_FRAME_MODE_MASK \ + ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST3_FRAME) | \ + (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME)) +#define LAST3_FRAME_MODE_MASK \ + ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) | \ + (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME)) +#define GOLDEN_FRAME_MODE_MASK \ + ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) | \ + (1 << LAST3_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME)) +#define BWDREF_FRAME_MODE_MASK \ + ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) | \ + (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME)) +#define ALTREF_FRAME_MODE_MASK \ + ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) | \ + (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME)) + +#else + +#define LAST_FRAME_MODE_MASK \ + ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | (1 << INTRA_FRAME)) +#define GOLDEN_FRAME_MODE_MASK \ + ((1 << LAST_FRAME) | (1 << ALTREF_FRAME) | (1 << INTRA_FRAME)) +#define ALTREF_FRAME_MODE_MASK \ + ((1 << LAST_FRAME) | (1 << GOLDEN_FRAME) | (1 << INTRA_FRAME)) + +#endif // CONFIG_EXT_REFS + +#if CONFIG_EXT_REFS +#define SECOND_REF_FRAME_MASK ((1 << ALTREF_FRAME) | (1 << BWDREF_FRAME) | 0x01) +#else +#define SECOND_REF_FRAME_MASK ((1 << ALTREF_FRAME) | 0x01) +#endif // CONFIG_EXT_REFS + +#define MIN_EARLY_TERM_INDEX 3 +#define NEW_MV_DISCOUNT_FACTOR 8 + +#if CONFIG_EXT_INTRA +#define ANGLE_SKIP_THRESH 10 +#define FILTER_FAST_SEARCH 1 +#endif // CONFIG_EXT_INTRA + +const double ADST_FLIP_SVM[8] = { -6.6623, -2.8062, -3.2531, 3.1671, // vert + -7.7051, -3.2234, -3.6193, 3.4533 }; // horz + +typedef struct { + PREDICTION_MODE mode; + MV_REFERENCE_FRAME ref_frame[2]; +} MODE_DEFINITION; + +typedef struct { MV_REFERENCE_FRAME ref_frame[2]; } REF_DEFINITION; + +struct rdcost_block_args { + const AV1_COMP *cpi; + MACROBLOCK *x; + ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE]; + ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]; + RD_STATS rd_stats; + int64_t this_rd; + int64_t best_rd; + int exit_early; + int use_fast_coef_costing; +}; + +#define LAST_NEW_MV_INDEX 6 +static const MODE_DEFINITION av1_mode_order[MAX_MODES] = { + { NEARESTMV, { LAST_FRAME, NONE_FRAME } }, +#if CONFIG_EXT_REFS + { NEARESTMV, { LAST2_FRAME, NONE_FRAME } }, + { NEARESTMV, { LAST3_FRAME, NONE_FRAME } }, + { NEARESTMV, { BWDREF_FRAME, NONE_FRAME } }, +#endif // CONFIG_EXT_REFS + { NEARESTMV, { ALTREF_FRAME, NONE_FRAME } }, + { NEARESTMV, { GOLDEN_FRAME, NONE_FRAME } }, + + { DC_PRED, { INTRA_FRAME, NONE_FRAME } }, + + { NEWMV, { LAST_FRAME, NONE_FRAME } }, +#if CONFIG_EXT_REFS + { NEWMV, { LAST2_FRAME, NONE_FRAME } }, + { NEWMV, { LAST3_FRAME, NONE_FRAME } }, + { NEWMV, { BWDREF_FRAME, NONE_FRAME } }, +#endif // CONFIG_EXT_REFS + { NEWMV, { ALTREF_FRAME, NONE_FRAME } }, + { NEWMV, { GOLDEN_FRAME, NONE_FRAME } }, + + { NEARMV, { LAST_FRAME, NONE_FRAME } }, +#if CONFIG_EXT_REFS + { NEARMV, { LAST2_FRAME, NONE_FRAME } }, + { NEARMV, { LAST3_FRAME, NONE_FRAME } }, + { NEARMV, { BWDREF_FRAME, NONE_FRAME } }, +#endif // CONFIG_EXT_REFS + { NEARMV, { ALTREF_FRAME, NONE_FRAME } }, + { NEARMV, { GOLDEN_FRAME, NONE_FRAME } }, + + { ZEROMV, { LAST_FRAME, NONE_FRAME } }, +#if CONFIG_EXT_REFS + { ZEROMV, { LAST2_FRAME, NONE_FRAME } }, + { ZEROMV, { LAST3_FRAME, NONE_FRAME } }, + { ZEROMV, { BWDREF_FRAME, NONE_FRAME } }, +#endif // CONFIG_EXT_REFS + { ZEROMV, { GOLDEN_FRAME, NONE_FRAME } }, + { ZEROMV, { ALTREF_FRAME, NONE_FRAME } }, + +// TODO(zoeliu): May need to reconsider the order on the modes to check + +#if CONFIG_EXT_INTER + { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } }, +#if CONFIG_EXT_REFS + { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } }, +#endif // CONFIG_EXT_REFS + { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } }, +#if CONFIG_EXT_REFS + { NEAREST_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEAREST_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEAREST_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEAREST_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } }, +#endif // CONFIG_EXT_REFS + +#else // CONFIG_EXT_INTER + + { NEARESTMV, { LAST_FRAME, ALTREF_FRAME } }, +#if CONFIG_EXT_REFS + { NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } }, +#endif // CONFIG_EXT_REFS + { NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } }, +#if CONFIG_EXT_REFS + { NEARESTMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } }, +#endif // CONFIG_EXT_REFS +#endif // CONFIG_EXT_INTER + + { TM_PRED, { INTRA_FRAME, NONE_FRAME } }, + +#if CONFIG_ALT_INTRA + { SMOOTH_PRED, { INTRA_FRAME, NONE_FRAME } }, +#endif // CONFIG_ALT_INTRA + +#if CONFIG_EXT_INTER + { NEAR_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEAREST_NEARMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEAR_NEARMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEW_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEAREST_NEWMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEW_NEARMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEAR_NEWMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEW_NEWMV, { LAST_FRAME, ALTREF_FRAME } }, + { ZERO_ZEROMV, { LAST_FRAME, ALTREF_FRAME } }, + +#if CONFIG_EXT_REFS + { NEAR_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEAREST_NEARMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEAR_NEARMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEW_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEAREST_NEWMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEW_NEARMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEAR_NEWMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEW_NEWMV, { LAST2_FRAME, ALTREF_FRAME } }, + { ZERO_ZEROMV, { LAST2_FRAME, ALTREF_FRAME } }, + + { NEAR_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEAREST_NEARMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEAR_NEARMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEW_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEAREST_NEWMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEW_NEARMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEAR_NEWMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEW_NEWMV, { LAST3_FRAME, ALTREF_FRAME } }, + { ZERO_ZEROMV, { LAST3_FRAME, ALTREF_FRAME } }, +#endif // CONFIG_EXT_REFS + + { NEAR_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEAREST_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEW_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEW_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { ZERO_ZEROMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + +#if CONFIG_EXT_REFS + { NEAR_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEAREST_NEARMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEAR_NEARMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEW_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEAREST_NEWMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEW_NEARMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEAR_NEWMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEW_NEWMV, { LAST_FRAME, BWDREF_FRAME } }, + { ZERO_ZEROMV, { LAST_FRAME, BWDREF_FRAME } }, + + { NEAR_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEAREST_NEARMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEAR_NEARMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEW_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEAREST_NEWMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEW_NEARMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEAR_NEWMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEW_NEWMV, { LAST2_FRAME, BWDREF_FRAME } }, + { ZERO_ZEROMV, { LAST2_FRAME, BWDREF_FRAME } }, + + { NEAR_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEAREST_NEARMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEAR_NEARMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEW_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEAREST_NEWMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEW_NEARMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEAR_NEWMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEW_NEWMV, { LAST3_FRAME, BWDREF_FRAME } }, + { ZERO_ZEROMV, { LAST3_FRAME, BWDREF_FRAME } }, + + { NEAR_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEAREST_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEAR_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEW_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEAREST_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEW_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEAR_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEW_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { ZERO_ZEROMV, { GOLDEN_FRAME, BWDREF_FRAME } }, +#endif // CONFIG_EXT_REFS + +#else // CONFIG_EXT_INTER + + { NEARMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEWMV, { LAST_FRAME, ALTREF_FRAME } }, +#if CONFIG_EXT_REFS + { NEARMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEWMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEARMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEWMV, { LAST3_FRAME, ALTREF_FRAME } }, +#endif // CONFIG_EXT_REFS + { NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + +#if CONFIG_EXT_REFS + { NEARMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEWMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEARMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEWMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEARMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEWMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } }, +#endif // CONFIG_EXT_REFS + + { ZEROMV, { LAST_FRAME, ALTREF_FRAME } }, +#if CONFIG_EXT_REFS + { ZEROMV, { LAST2_FRAME, ALTREF_FRAME } }, + { ZEROMV, { LAST3_FRAME, ALTREF_FRAME } }, +#endif // CONFIG_EXT_REFS + { ZEROMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + +#if CONFIG_EXT_REFS + { ZEROMV, { LAST_FRAME, BWDREF_FRAME } }, + { ZEROMV, { LAST2_FRAME, BWDREF_FRAME } }, + { ZEROMV, { LAST3_FRAME, BWDREF_FRAME } }, + { ZEROMV, { GOLDEN_FRAME, BWDREF_FRAME } }, +#endif // CONFIG_EXT_REFS + +#endif // CONFIG_EXT_INTER + + { H_PRED, { INTRA_FRAME, NONE_FRAME } }, + { V_PRED, { INTRA_FRAME, NONE_FRAME } }, + { D135_PRED, { INTRA_FRAME, NONE_FRAME } }, + { D207_PRED, { INTRA_FRAME, NONE_FRAME } }, + { D153_PRED, { INTRA_FRAME, NONE_FRAME } }, + { D63_PRED, { INTRA_FRAME, NONE_FRAME } }, + { D117_PRED, { INTRA_FRAME, NONE_FRAME } }, + { D45_PRED, { INTRA_FRAME, NONE_FRAME } }, + +#if CONFIG_EXT_INTER + { ZEROMV, { LAST_FRAME, INTRA_FRAME } }, + { NEARESTMV, { LAST_FRAME, INTRA_FRAME } }, + { NEARMV, { LAST_FRAME, INTRA_FRAME } }, + { NEWMV, { LAST_FRAME, INTRA_FRAME } }, + +#if CONFIG_EXT_REFS + { ZEROMV, { LAST2_FRAME, INTRA_FRAME } }, + { NEARESTMV, { LAST2_FRAME, INTRA_FRAME } }, + { NEARMV, { LAST2_FRAME, INTRA_FRAME } }, + { NEWMV, { LAST2_FRAME, INTRA_FRAME } }, + + { ZEROMV, { LAST3_FRAME, INTRA_FRAME } }, + { NEARESTMV, { LAST3_FRAME, INTRA_FRAME } }, + { NEARMV, { LAST3_FRAME, INTRA_FRAME } }, + { NEWMV, { LAST3_FRAME, INTRA_FRAME } }, +#endif // CONFIG_EXT_REFS + + { ZEROMV, { GOLDEN_FRAME, INTRA_FRAME } }, + { NEARESTMV, { GOLDEN_FRAME, INTRA_FRAME } }, + { NEARMV, { GOLDEN_FRAME, INTRA_FRAME } }, + { NEWMV, { GOLDEN_FRAME, INTRA_FRAME } }, + +#if CONFIG_EXT_REFS + { ZEROMV, { BWDREF_FRAME, INTRA_FRAME } }, + { NEARESTMV, { BWDREF_FRAME, INTRA_FRAME } }, + { NEARMV, { BWDREF_FRAME, INTRA_FRAME } }, + { NEWMV, { BWDREF_FRAME, INTRA_FRAME } }, +#endif // CONFIG_EXT_REFS + + { ZEROMV, { ALTREF_FRAME, INTRA_FRAME } }, + { NEARESTMV, { ALTREF_FRAME, INTRA_FRAME } }, + { NEARMV, { ALTREF_FRAME, INTRA_FRAME } }, + { NEWMV, { ALTREF_FRAME, INTRA_FRAME } }, +#endif // CONFIG_EXT_INTER +}; + +static const REF_DEFINITION av1_ref_order[MAX_REFS] = { + { { LAST_FRAME, NONE_FRAME } }, +#if CONFIG_EXT_REFS + { { LAST2_FRAME, NONE_FRAME } }, { { LAST3_FRAME, NONE_FRAME } }, + { { BWDREF_FRAME, NONE_FRAME } }, +#endif // CONFIG_EXT_REFS + { { GOLDEN_FRAME, NONE_FRAME } }, { { ALTREF_FRAME, NONE_FRAME } }, + + { { LAST_FRAME, ALTREF_FRAME } }, +#if CONFIG_EXT_REFS + { { LAST2_FRAME, ALTREF_FRAME } }, { { LAST3_FRAME, ALTREF_FRAME } }, +#endif // CONFIG_EXT_REFS + { { GOLDEN_FRAME, ALTREF_FRAME } }, + +#if CONFIG_EXT_REFS + { { LAST_FRAME, BWDREF_FRAME } }, { { LAST2_FRAME, BWDREF_FRAME } }, + { { LAST3_FRAME, BWDREF_FRAME } }, { { GOLDEN_FRAME, BWDREF_FRAME } }, +#endif // CONFIG_EXT_REFS + + { { INTRA_FRAME, NONE_FRAME } }, +}; + +#if CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA || CONFIG_PALETTE +static INLINE int write_uniform_cost(int n, int v) { + const int l = get_unsigned_bits(n); + const int m = (1 << l) - n; + if (l == 0) return 0; + if (v < m) + return (l - 1) * av1_cost_bit(128, 0); + else + return l * av1_cost_bit(128, 0); +} +#endif // CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA || CONFIG_PALETTE + +// constants for prune 1 and prune 2 decision boundaries +#define FAST_EXT_TX_CORR_MID 0.0 +#define FAST_EXT_TX_EDST_MID 0.1 +#define FAST_EXT_TX_CORR_MARGIN 0.5 +#define FAST_EXT_TX_EDST_MARGIN 0.3 + +static const TX_TYPE_1D vtx_tab[TX_TYPES] = { + DCT_1D, ADST_1D, DCT_1D, ADST_1D, +#if CONFIG_EXT_TX + FLIPADST_1D, DCT_1D, FLIPADST_1D, ADST_1D, FLIPADST_1D, IDTX_1D, + DCT_1D, IDTX_1D, ADST_1D, IDTX_1D, FLIPADST_1D, IDTX_1D, +#endif // CONFIG_EXT_TX +}; + +static const TX_TYPE_1D htx_tab[TX_TYPES] = { + DCT_1D, DCT_1D, ADST_1D, ADST_1D, +#if CONFIG_EXT_TX + DCT_1D, FLIPADST_1D, FLIPADST_1D, FLIPADST_1D, ADST_1D, IDTX_1D, + IDTX_1D, DCT_1D, IDTX_1D, ADST_1D, IDTX_1D, FLIPADST_1D, +#endif // CONFIG_EXT_TX +}; + +#if CONFIG_DAALA_DIST +static int od_compute_var_4x4(od_coeff *x, int stride) { + int sum; + int s2; + int i; + sum = 0; + s2 = 0; + for (i = 0; i < 4; i++) { + int j; + for (j = 0; j < 4; j++) { + int t; + + t = x[i * stride + j]; + sum += t; + s2 += t * t; + } + } + // TODO(yushin) : Check wheter any changes are required for high bit depth. + return (s2 - (sum * sum >> 4)) >> 4; +} + +/* OD_DIST_LP_MID controls the frequency weighting filter used for computing + the distortion. For a value X, the filter is [1 X 1]/(X + 2) and + is applied both horizontally and vertically. For X=5, the filter is + a good approximation for the OD_QM8_Q4_HVS quantization matrix. */ +#define OD_DIST_LP_MID (5) +#define OD_DIST_LP_NORM (OD_DIST_LP_MID + 2) + +static double od_compute_dist_8x8(int qm, int use_activity_masking, od_coeff *x, + od_coeff *y, od_coeff *e_lp, int stride) { + double sum; + int min_var; + double mean_var; + double var_stat; + double activity; + double calibration; + int i; + int j; + double vardist; + + vardist = 0; + OD_ASSERT(qm != OD_FLAT_QM); + (void)qm; +#if 1 + min_var = INT_MAX; + mean_var = 0; + for (i = 0; i < 3; i++) { + for (j = 0; j < 3; j++) { + int varx; + int vary; + varx = od_compute_var_4x4(x + 2 * i * stride + 2 * j, stride); + vary = od_compute_var_4x4(y + 2 * i * stride + 2 * j, stride); + min_var = OD_MINI(min_var, varx); + mean_var += 1. / (1 + varx); + /* The cast to (double) is to avoid an overflow before the sqrt.*/ + vardist += varx - 2 * sqrt(varx * (double)vary) + vary; + } + } + /* We use a different variance statistic depending on whether activity + masking is used, since the harmonic mean appeared slghtly worse with + masking off. The calibration constant just ensures that we preserve the + rate compared to activity=1. */ + if (use_activity_masking) { + calibration = 1.95; + var_stat = 9. / mean_var; + } else { + calibration = 1.62; + var_stat = min_var; + } + /* 1.62 is a calibration constant, 0.25 is a noise floor and 1/6 is the + activity masking constant. */ + activity = calibration * pow(.25 + var_stat, -1. / 6); +#else + activity = 1; +#endif // 1 + sum = 0; + for (i = 0; i < 8; i++) { + for (j = 0; j < 8; j++) + sum += e_lp[i * stride + j] * (double)e_lp[i * stride + j]; + } + /* Normalize the filter to unit DC response. */ + sum *= 1. / (OD_DIST_LP_NORM * OD_DIST_LP_NORM * OD_DIST_LP_NORM * + OD_DIST_LP_NORM); + return activity * activity * (sum + vardist); +} + +// Note : Inputs x and y are in a pixel domain +static double od_compute_dist(int qm, int activity_masking, od_coeff *x, + od_coeff *y, int bsize_w, int bsize_h, + int qindex) { + int i; + double sum; + sum = 0; + + assert(bsize_w >= 8 && bsize_h >= 8); + + if (qm == OD_FLAT_QM) { + for (i = 0; i < bsize_w * bsize_h; i++) { + double tmp; + tmp = x[i] - y[i]; + sum += tmp * tmp; + } + } else { + int j; + DECLARE_ALIGNED(16, od_coeff, e[MAX_TX_SQUARE]); + DECLARE_ALIGNED(16, od_coeff, tmp[MAX_TX_SQUARE]); + DECLARE_ALIGNED(16, od_coeff, e_lp[MAX_TX_SQUARE]); + int mid = OD_DIST_LP_MID; + for (i = 0; i < bsize_h; i++) { + for (j = 0; j < bsize_w; j++) { + e[i * bsize_w + j] = x[i * bsize_w + j] - y[i * bsize_w + j]; + } + } + for (i = 0; i < bsize_h; i++) { + tmp[i * bsize_w] = mid * e[i * bsize_w] + 2 * e[i * bsize_w + 1]; + tmp[i * bsize_w + bsize_w - 1] = + mid * e[i * bsize_w + bsize_w - 1] + 2 * e[i * bsize_w + bsize_w - 2]; + for (j = 1; j < bsize_w - 1; j++) { + tmp[i * bsize_w + j] = mid * e[i * bsize_w + j] + + e[i * bsize_w + j - 1] + e[i * bsize_w + j + 1]; + } + } + for (j = 0; j < bsize_w; j++) { + e_lp[j] = mid * tmp[j] + 2 * tmp[bsize_w + j]; + e_lp[(bsize_h - 1) * bsize_w + j] = + mid * tmp[(bsize_h - 1) * bsize_w + j] + + 2 * tmp[(bsize_h - 2) * bsize_w + j]; + } + for (i = 1; i < bsize_h - 1; i++) { + for (j = 0; j < bsize_w; j++) { + e_lp[i * bsize_w + j] = mid * tmp[i * bsize_w + j] + + tmp[(i - 1) * bsize_w + j] + + tmp[(i + 1) * bsize_w + j]; + } + } + for (i = 0; i < bsize_h; i += 8) { + for (j = 0; j < bsize_w; j += 8) { + sum += od_compute_dist_8x8(qm, activity_masking, &x[i * bsize_w + j], + &y[i * bsize_w + j], &e_lp[i * bsize_w + j], + bsize_w); + } + } + /* Scale according to linear regression against SSE, for 8x8 blocks. */ + if (activity_masking) { + sum *= 2.2 + (1.7 - 2.2) * (qindex - 99) / (210 - 99) + + (qindex < 99 ? 2.5 * (qindex - 99) / 99 * (qindex - 99) / 99 : 0); + } else { + sum *= qindex >= 128 + ? 1.4 + (0.9 - 1.4) * (qindex - 128) / (209 - 128) + : qindex <= 43 + ? 1.5 + (2.0 - 1.5) * (qindex - 43) / (16 - 43) + : 1.5 + (1.4 - 1.5) * (qindex - 43) / (128 - 43); + } + } + return sum; +} + +static int64_t av1_daala_dist(const uint8_t *src, int src_stride, + const uint8_t *dst, int dst_stride, int bsw, + int bsh, int qm, int use_activity_masking, + int qindex) { + int i, j; + int64_t d; + DECLARE_ALIGNED(16, od_coeff, orig[MAX_TX_SQUARE]); + DECLARE_ALIGNED(16, od_coeff, rec[MAX_TX_SQUARE]); + + assert(qm == OD_HVS_QM); + + for (j = 0; j < bsh; j++) + for (i = 0; i < bsw; i++) orig[j * bsw + i] = src[j * src_stride + i]; + + for (j = 0; j < bsh; j++) + for (i = 0; i < bsw; i++) rec[j * bsw + i] = dst[j * dst_stride + i]; + + d = (int64_t)od_compute_dist(qm, use_activity_masking, orig, rec, bsw, bsh, + qindex); + return d; +} +#endif // CONFIG_DAALA_DIST + +static void get_energy_distribution_fine(const AV1_COMP *cpi, BLOCK_SIZE bsize, + const uint8_t *src, int src_stride, + const uint8_t *dst, int dst_stride, + double *hordist, double *verdist) { + const int bw = block_size_wide[bsize]; + const int bh = block_size_high[bsize]; + unsigned int esq[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + + const int f_index = bsize - BLOCK_16X16; + if (f_index < 0) { + const int w_shift = bw == 8 ? 1 : 2; + const int h_shift = bh == 8 ? 1 : 2; +#if CONFIG_HIGHBITDEPTH + if (cpi->common.use_highbitdepth) { + const uint16_t *src16 = CONVERT_TO_SHORTPTR(src); + const uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst); + for (int i = 0; i < bh; ++i) + for (int j = 0; j < bw; ++j) { + const int index = (j >> w_shift) + ((i >> h_shift) << 2); + esq[index] += + (src16[j + i * src_stride] - dst16[j + i * dst_stride]) * + (src16[j + i * src_stride] - dst16[j + i * dst_stride]); + } + } else { +#endif // CONFIG_HIGHBITDEPTH + + for (int i = 0; i < bh; ++i) + for (int j = 0; j < bw; ++j) { + const int index = (j >> w_shift) + ((i >> h_shift) << 2); + esq[index] += (src[j + i * src_stride] - dst[j + i * dst_stride]) * + (src[j + i * src_stride] - dst[j + i * dst_stride]); + } +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + } else { + cpi->fn_ptr[f_index].vf(src, src_stride, dst, dst_stride, &esq[0]); + cpi->fn_ptr[f_index].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride, + &esq[1]); + cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride, + &esq[2]); + cpi->fn_ptr[f_index].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4, + dst_stride, &esq[3]); + src += bh / 4 * src_stride; + dst += bh / 4 * dst_stride; + + cpi->fn_ptr[f_index].vf(src, src_stride, dst, dst_stride, &esq[4]); + cpi->fn_ptr[f_index].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride, + &esq[5]); + cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride, + &esq[6]); + cpi->fn_ptr[f_index].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4, + dst_stride, &esq[7]); + src += bh / 4 * src_stride; + dst += bh / 4 * dst_stride; + + cpi->fn_ptr[f_index].vf(src, src_stride, dst, dst_stride, &esq[8]); + cpi->fn_ptr[f_index].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride, + &esq[9]); + cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride, + &esq[10]); + cpi->fn_ptr[f_index].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4, + dst_stride, &esq[11]); + src += bh / 4 * src_stride; + dst += bh / 4 * dst_stride; + + cpi->fn_ptr[f_index].vf(src, src_stride, dst, dst_stride, &esq[12]); + cpi->fn_ptr[f_index].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride, + &esq[13]); + cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride, + &esq[14]); + cpi->fn_ptr[f_index].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4, + dst_stride, &esq[15]); + } + + double total = (double)esq[0] + esq[1] + esq[2] + esq[3] + esq[4] + esq[5] + + esq[6] + esq[7] + esq[8] + esq[9] + esq[10] + esq[11] + + esq[12] + esq[13] + esq[14] + esq[15]; + if (total > 0) { + const double e_recip = 1.0 / total; + hordist[0] = ((double)esq[0] + esq[4] + esq[8] + esq[12]) * e_recip; + hordist[1] = ((double)esq[1] + esq[5] + esq[9] + esq[13]) * e_recip; + hordist[2] = ((double)esq[2] + esq[6] + esq[10] + esq[14]) * e_recip; + verdist[0] = ((double)esq[0] + esq[1] + esq[2] + esq[3]) * e_recip; + verdist[1] = ((double)esq[4] + esq[5] + esq[6] + esq[7]) * e_recip; + verdist[2] = ((double)esq[8] + esq[9] + esq[10] + esq[11]) * e_recip; + } else { + hordist[0] = verdist[0] = 0.25; + hordist[1] = verdist[1] = 0.25; + hordist[2] = verdist[2] = 0.25; + } +} + +static int adst_vs_flipadst(const AV1_COMP *cpi, BLOCK_SIZE bsize, + const uint8_t *src, int src_stride, + const uint8_t *dst, int dst_stride) { + int prune_bitmask = 0; + double svm_proj_h = 0, svm_proj_v = 0; + double hdist[3] = { 0, 0, 0 }, vdist[3] = { 0, 0, 0 }; + get_energy_distribution_fine(cpi, bsize, src, src_stride, dst, dst_stride, + hdist, vdist); + + svm_proj_v = vdist[0] * ADST_FLIP_SVM[0] + vdist[1] * ADST_FLIP_SVM[1] + + vdist[2] * ADST_FLIP_SVM[2] + ADST_FLIP_SVM[3]; + svm_proj_h = hdist[0] * ADST_FLIP_SVM[4] + hdist[1] * ADST_FLIP_SVM[5] + + hdist[2] * ADST_FLIP_SVM[6] + ADST_FLIP_SVM[7]; + if (svm_proj_v > FAST_EXT_TX_EDST_MID + FAST_EXT_TX_EDST_MARGIN) + prune_bitmask |= 1 << FLIPADST_1D; + else if (svm_proj_v < FAST_EXT_TX_EDST_MID - FAST_EXT_TX_EDST_MARGIN) + prune_bitmask |= 1 << ADST_1D; + + if (svm_proj_h > FAST_EXT_TX_EDST_MID + FAST_EXT_TX_EDST_MARGIN) + prune_bitmask |= 1 << (FLIPADST_1D + 8); + else if (svm_proj_h < FAST_EXT_TX_EDST_MID - FAST_EXT_TX_EDST_MARGIN) + prune_bitmask |= 1 << (ADST_1D + 8); + + return prune_bitmask; +} + +#if CONFIG_EXT_TX +static void get_horver_correlation(const int16_t *diff, int stride, int w, + int h, double *hcorr, double *vcorr) { + // Returns hor/ver correlation coefficient + const int num = (h - 1) * (w - 1); + double num_r; + int i, j; + int64_t xy_sum = 0, xz_sum = 0; + int64_t x_sum = 0, y_sum = 0, z_sum = 0; + int64_t x2_sum = 0, y2_sum = 0, z2_sum = 0; + double x_var_n, y_var_n, z_var_n, xy_var_n, xz_var_n; + *hcorr = *vcorr = 1; + + assert(num > 0); + num_r = 1.0 / num; + for (i = 1; i < h; ++i) { + for (j = 1; j < w; ++j) { + const int16_t x = diff[i * stride + j]; + const int16_t y = diff[i * stride + j - 1]; + const int16_t z = diff[(i - 1) * stride + j]; + xy_sum += x * y; + xz_sum += x * z; + x_sum += x; + y_sum += y; + z_sum += z; + x2_sum += x * x; + y2_sum += y * y; + z2_sum += z * z; + } + } + x_var_n = x2_sum - (x_sum * x_sum) * num_r; + y_var_n = y2_sum - (y_sum * y_sum) * num_r; + z_var_n = z2_sum - (z_sum * z_sum) * num_r; + xy_var_n = xy_sum - (x_sum * y_sum) * num_r; + xz_var_n = xz_sum - (x_sum * z_sum) * num_r; + if (x_var_n > 0 && y_var_n > 0) { + *hcorr = xy_var_n / sqrt(x_var_n * y_var_n); + *hcorr = *hcorr < 0 ? 0 : *hcorr; + } + if (x_var_n > 0 && z_var_n > 0) { + *vcorr = xz_var_n / sqrt(x_var_n * z_var_n); + *vcorr = *vcorr < 0 ? 0 : *vcorr; + } +} + +int dct_vs_idtx(const int16_t *diff, int stride, int w, int h) { + double hcorr, vcorr; + int prune_bitmask = 0; + get_horver_correlation(diff, stride, w, h, &hcorr, &vcorr); + + if (vcorr > FAST_EXT_TX_CORR_MID + FAST_EXT_TX_CORR_MARGIN) + prune_bitmask |= 1 << IDTX_1D; + else if (vcorr < FAST_EXT_TX_CORR_MID - FAST_EXT_TX_CORR_MARGIN) + prune_bitmask |= 1 << DCT_1D; + + if (hcorr > FAST_EXT_TX_CORR_MID + FAST_EXT_TX_CORR_MARGIN) + prune_bitmask |= 1 << (IDTX_1D + 8); + else if (hcorr < FAST_EXT_TX_CORR_MID - FAST_EXT_TX_CORR_MARGIN) + prune_bitmask |= 1 << (DCT_1D + 8); + return prune_bitmask; +} + +// Performance drop: 0.5%, Speed improvement: 24% +static int prune_two_for_sby(const AV1_COMP *cpi, BLOCK_SIZE bsize, + MACROBLOCK *x, const MACROBLOCKD *xd, + int adst_flipadst, int dct_idtx) { + int prune = 0; + + if (adst_flipadst) { + const struct macroblock_plane *const p = &x->plane[0]; + const struct macroblockd_plane *const pd = &xd->plane[0]; + prune |= adst_vs_flipadst(cpi, bsize, p->src.buf, p->src.stride, + pd->dst.buf, pd->dst.stride); + } + if (dct_idtx) { + av1_subtract_plane(x, bsize, 0); + const struct macroblock_plane *const p = &x->plane[0]; + const int bw = 4 << (b_width_log2_lookup[bsize]); + const int bh = 4 << (b_height_log2_lookup[bsize]); + prune |= dct_vs_idtx(p->src_diff, bw, bw, bh); + } + + return prune; +} +#endif // CONFIG_EXT_TX + +// Performance drop: 0.3%, Speed improvement: 5% +static int prune_one_for_sby(const AV1_COMP *cpi, BLOCK_SIZE bsize, + const MACROBLOCK *x, const MACROBLOCKD *xd) { + const struct macroblock_plane *const p = &x->plane[0]; + const struct macroblockd_plane *const pd = &xd->plane[0]; + return adst_vs_flipadst(cpi, bsize, p->src.buf, p->src.stride, pd->dst.buf, + pd->dst.stride); +} + +static int prune_tx_types(const AV1_COMP *cpi, BLOCK_SIZE bsize, MACROBLOCK *x, + const MACROBLOCKD *const xd, int tx_set) { +#if CONFIG_EXT_TX + const int *tx_set_1D = ext_tx_used_inter_1D[tx_set]; +#else + const int tx_set_1D[TX_TYPES_1D] = { 0 }; +#endif // CONFIG_EXT_TX + + switch (cpi->sf.tx_type_search.prune_mode) { + case NO_PRUNE: return 0; break; + case PRUNE_ONE: + if ((tx_set >= 0) && !(tx_set_1D[FLIPADST_1D] & tx_set_1D[ADST_1D])) + return 0; + return prune_one_for_sby(cpi, bsize, x, xd); + break; +#if CONFIG_EXT_TX + case PRUNE_TWO: + if ((tx_set >= 0) && !(tx_set_1D[FLIPADST_1D] & tx_set_1D[ADST_1D])) { + if (!(tx_set_1D[DCT_1D] & tx_set_1D[IDTX_1D])) return 0; + return prune_two_for_sby(cpi, bsize, x, xd, 0, 1); + } + if ((tx_set >= 0) && !(tx_set_1D[DCT_1D] & tx_set_1D[IDTX_1D])) + return prune_two_for_sby(cpi, bsize, x, xd, 1, 0); + return prune_two_for_sby(cpi, bsize, x, xd, 1, 1); + break; +#endif // CONFIG_EXT_TX + } + assert(0); + return 0; +} + +static int do_tx_type_search(TX_TYPE tx_type, int prune) { +// TODO(sarahparker) implement for non ext tx +#if CONFIG_EXT_TX + return !(((prune >> vtx_tab[tx_type]) & 1) | + ((prune >> (htx_tab[tx_type] + 8)) & 1)); +#else + // temporary to avoid compiler warnings + (void)vtx_tab; + (void)htx_tab; + (void)tx_type; + (void)prune; + return 1; +#endif // CONFIG_EXT_TX +} + +static void model_rd_from_sse(const AV1_COMP *const cpi, + const MACROBLOCKD *const xd, BLOCK_SIZE bsize, + int plane, int64_t sse, int *rate, + int64_t *dist) { + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const int dequant_shift = +#if CONFIG_HIGHBITDEPTH + (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd - 5 : +#endif // CONFIG_HIGHBITDEPTH + 3; + + // Fast approximate the modelling function. + if (cpi->sf.simple_model_rd_from_var) { + const int64_t square_error = sse; + int quantizer = (pd->dequant[1] >> dequant_shift); + + if (quantizer < 120) + *rate = (int)((square_error * (280 - quantizer)) >> + (16 - AV1_PROB_COST_SHIFT)); + else + *rate = 0; + *dist = (square_error * quantizer) >> 8; + } else { + av1_model_rd_from_var_lapndz(sse, num_pels_log2_lookup[bsize], + pd->dequant[1] >> dequant_shift, rate, dist); + } + + *dist <<= 4; +} + +static void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize, + MACROBLOCK *x, MACROBLOCKD *xd, int plane_from, + int plane_to, int *out_rate_sum, + int64_t *out_dist_sum, int *skip_txfm_sb, + int64_t *skip_sse_sb) { + // Note our transform coeffs are 8 times an orthogonal transform. + // Hence quantizer step is also 8 times. To get effective quantizer + // we need to divide by 8 before sending to modeling function. + int plane; + const int ref = xd->mi[0]->mbmi.ref_frame[0]; + + int64_t rate_sum = 0; + int64_t dist_sum = 0; + int64_t total_sse = 0; + + x->pred_sse[ref] = 0; + + for (plane = plane_from; plane <= plane_to; ++plane) { + struct macroblock_plane *const p = &x->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; +#if CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + const BLOCK_SIZE bs = AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd)); +#else + const BLOCK_SIZE bs = get_plane_block_size(bsize, pd); +#endif // CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + + unsigned int sse; + int rate; + int64_t dist; + +#if CONFIG_CB4X4 + if (x->skip_chroma_rd && plane) continue; +#endif // CONFIG_CB4X4 + + // TODO(geza): Write direct sse functions that do not compute + // variance as well. + cpi->fn_ptr[bs].vf(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, + &sse); + + if (plane == 0) x->pred_sse[ref] = sse; + + total_sse += sse; + + model_rd_from_sse(cpi, xd, bs, plane, sse, &rate, &dist); + + rate_sum += rate; + dist_sum += dist; + } + + *skip_txfm_sb = total_sse == 0; + *skip_sse_sb = total_sse << 4; + *out_rate_sum = (int)rate_sum; + *out_dist_sum = dist_sum; +} + +int64_t av1_block_error_c(const tran_low_t *coeff, const tran_low_t *dqcoeff, + intptr_t block_size, int64_t *ssz) { + int i; + int64_t error = 0, sqcoeff = 0; + + for (i = 0; i < block_size; i++) { + const int diff = coeff[i] - dqcoeff[i]; + error += diff * diff; + sqcoeff += coeff[i] * coeff[i]; + } + + *ssz = sqcoeff; + return error; +} + +int64_t av1_block_error_fp_c(const int16_t *coeff, const int16_t *dqcoeff, + int block_size) { + int i; + int64_t error = 0; + + for (i = 0; i < block_size; i++) { + const int diff = coeff[i] - dqcoeff[i]; + error += diff * diff; + } + + return error; +} + +#if CONFIG_HIGHBITDEPTH +int64_t av1_highbd_block_error_c(const tran_low_t *coeff, + const tran_low_t *dqcoeff, intptr_t block_size, + int64_t *ssz, int bd) { + int i; + int64_t error = 0, sqcoeff = 0; + int shift = 2 * (bd - 8); + int rounding = shift > 0 ? 1 << (shift - 1) : 0; + + for (i = 0; i < block_size; i++) { + const int64_t diff = coeff[i] - dqcoeff[i]; + error += diff * diff; + sqcoeff += (int64_t)coeff[i] * (int64_t)coeff[i]; + } + assert(error >= 0 && sqcoeff >= 0); + error = (error + rounding) >> shift; + sqcoeff = (sqcoeff + rounding) >> shift; + + *ssz = sqcoeff; + return error; +} +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_PVQ +// Without PVQ, av1_block_error_c() return two kind of errors, +// 1) reconstruction (i.e. decoded) error and +// 2) Squared sum of transformed residue (i.e. 'coeff') +// However, if PVQ is enabled, coeff does not keep the transformed residue +// but instead a transformed original is kept. +// Hence, new parameter ref vector (i.e. transformed predicted signal) +// is required to derive the residue signal, +// i.e. coeff - ref = residue (all transformed). + +#if CONFIG_HIGHBITDEPTH +static int64_t av1_highbd_block_error2_c(const tran_low_t *coeff, + const tran_low_t *dqcoeff, + const tran_low_t *ref, + intptr_t block_size, int64_t *ssz, + int bd) { + int64_t error; + int64_t sqcoeff; + int shift = 2 * (bd - 8); + int rounding = shift > 0 ? 1 << (shift - 1) : 0; + // Use the existing sse codes for calculating distortion of decoded signal: + // i.e. (orig - decoded)^2 + // For high bit depth, throw away ssz until a 32-bit version of + // av1_block_error_fp is written. + int64_t ssz_trash; + error = av1_block_error(coeff, dqcoeff, block_size, &ssz_trash); + // prediction residue^2 = (orig - ref)^2 + sqcoeff = av1_block_error(coeff, ref, block_size, &ssz_trash); + error = (error + rounding) >> shift; + sqcoeff = (sqcoeff + rounding) >> shift; + *ssz = sqcoeff; + return error; +} +#else +// TODO(yushin) : Since 4x4 case does not need ssz, better to refactor into +// a separate function that does not do the extra computations for ssz. +static int64_t av1_block_error2_c(const tran_low_t *coeff, + const tran_low_t *dqcoeff, + const tran_low_t *ref, intptr_t block_size, + int64_t *ssz) { + int64_t error; + // Use the existing sse codes for calculating distortion of decoded signal: + // i.e. (orig - decoded)^2 + error = av1_block_error_fp(coeff, dqcoeff, block_size); + // prediction residue^2 = (orig - ref)^2 + *ssz = av1_block_error_fp(coeff, ref, block_size); + return error; +} +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_PVQ + +#if !CONFIG_PVQ || CONFIG_VAR_TX +/* The trailing '0' is a terminator which is used inside av1_cost_coeffs() to + * decide whether to include cost of a trailing EOB node or not (i.e. we + * can skip this if the last coefficient in this transform block, e.g. the + * 16th coefficient in a 4x4 block or the 64th coefficient in a 8x8 block, + * were non-zero). */ +#if !CONFIG_LV_MAP +static int cost_coeffs(const AV1_COMMON *const cm, MACROBLOCK *x, int plane, + int block, TX_SIZE tx_size, const SCAN_ORDER *scan_order, + const ENTROPY_CONTEXT *a, const ENTROPY_CONTEXT *l, + int use_fast_coef_costing) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const struct macroblock_plane *p = &x->plane[plane]; + const struct macroblockd_plane *pd = &xd->plane[plane]; + const PLANE_TYPE type = pd->plane_type; + const uint16_t *band_count = &band_count_table[tx_size][1]; + const int eob = p->eobs[block]; + const tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block); + const int tx_size_ctx = txsize_sqr_map[tx_size]; + unsigned int(*token_costs)[2][COEFF_CONTEXTS][ENTROPY_TOKENS] = + x->token_costs[tx_size_ctx][type][is_inter_block(mbmi)]; + uint8_t token_cache[MAX_TX_SQUARE]; + int pt = combine_entropy_contexts(*a, *l); + int c, cost; + const int16_t *scan = scan_order->scan; + const int16_t *nb = scan_order->neighbors; +#if CONFIG_NEW_TOKENSET + const int ref = is_inter_block(mbmi); + aom_prob *blockz_probs = + cm->fc->blockzero_probs[txsize_sqr_map[tx_size]][type][ref]; + +#endif // CONFIG_NEW_TOKENSET + +#if CONFIG_HIGHBITDEPTH + const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, xd->bd); +#else + const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, 8); +#endif // CONFIG_HIGHBITDEPTH + +#if !CONFIG_VAR_TX && !CONFIG_SUPERTX + // Check for consistency of tx_size with mode info + assert(tx_size == get_tx_size(plane, xd)); +#endif // !CONFIG_VAR_TX && !CONFIG_SUPERTX + (void)cm; + + if (eob == 0) { +#if CONFIG_NEW_TOKENSET + // single eob token + cost = av1_cost_bit(blockz_probs[pt], 0); +#else + cost = token_costs[0][0][pt][EOB_TOKEN]; +#endif // CONFIG_NEW_TOKENSET + } else { + if (use_fast_coef_costing) { + int band_left = *band_count++; + + // dc token + int v = qcoeff[0]; + int16_t prev_t; + cost = av1_get_token_cost(v, &prev_t, cat6_bits); +#if CONFIG_NEW_TOKENSET + cost += (*token_costs)[!prev_t][pt][prev_t]; +#else + cost += (*token_costs)[0][pt][prev_t]; +#endif + + token_cache[0] = av1_pt_energy_class[prev_t]; + ++token_costs; + + // ac tokens + for (c = 1; c < eob; c++) { + const int rc = scan[c]; + int16_t t; + + v = qcoeff[rc]; + cost += av1_get_token_cost(v, &t, cat6_bits); +#if CONFIG_NEW_TOKENSET + cost += (*token_costs)[!t][!prev_t][t]; +#else + cost += (*token_costs)[!prev_t][!prev_t][t]; +#endif + prev_t = t; + if (!--band_left) { + band_left = *band_count++; + ++token_costs; + } + } + + // eob token + if (band_left || CONFIG_NEW_TOKENSET) + cost += (*token_costs)[0][!prev_t][EOB_TOKEN]; + + } else { // !use_fast_coef_costing + int band_left = *band_count++; + + // dc token + int v = qcoeff[0]; + int16_t tok; +#if !CONFIG_NEW_TOKENSET + unsigned int(*tok_cost_ptr)[COEFF_CONTEXTS][ENTROPY_TOKENS]; +#endif + cost = av1_get_token_cost(v, &tok, cat6_bits); +#if CONFIG_NEW_TOKENSET + cost += (*token_costs)[!tok][pt][tok]; +#else + cost += (*token_costs)[0][pt][tok]; +#endif + + token_cache[0] = av1_pt_energy_class[tok]; + ++token_costs; + +#if !CONFIG_NEW_TOKENSET + tok_cost_ptr = &((*token_costs)[!tok]); +#endif + + // ac tokens + for (c = 1; c < eob; c++) { + const int rc = scan[c]; + + v = qcoeff[rc]; + cost += av1_get_token_cost(v, &tok, cat6_bits); + pt = get_coef_context(nb, token_cache, c); +#if CONFIG_NEW_TOKENSET + cost += (*token_costs)[!tok][pt][tok]; +#else + cost += (*tok_cost_ptr)[pt][tok]; +#endif + token_cache[rc] = av1_pt_energy_class[tok]; + if (!--band_left) { + band_left = *band_count++; + ++token_costs; + } +#if !CONFIG_NEW_TOKENSET + tok_cost_ptr = &((*token_costs)[!tok]); +#endif + } + + // eob token + if (band_left || CONFIG_NEW_TOKENSET) { + pt = get_coef_context(nb, token_cache, c); + cost += (*token_costs)[0][pt][EOB_TOKEN]; + } + } + } + + return cost; +} +#endif // !CONFIG_LV_MAP + +int av1_cost_coeffs(const AV1_COMP *const cpi, MACROBLOCK *x, int plane, + int block, TX_SIZE tx_size, const SCAN_ORDER *scan_order, + const ENTROPY_CONTEXT *a, const ENTROPY_CONTEXT *l, + int use_fast_coef_costing) { +#if !CONFIG_LV_MAP + const AV1_COMMON *const cm = &cpi->common; + return cost_coeffs(cm, x, plane, block, tx_size, scan_order, a, l, + use_fast_coef_costing); +#else // !CONFIG_LV_MAP + (void)scan_order; + (void)use_fast_coef_costing; + const MACROBLOCKD *xd = &x->e_mbd; + const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const struct macroblockd_plane *pd = &xd->plane[plane]; + const BLOCK_SIZE bsize = mbmi->sb_type; +#if CONFIG_CB4X4 +#if CONFIG_CHROMA_2X2 + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); +#else + const BLOCK_SIZE plane_bsize = + AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd)); +#endif // CONFIG_CHROMA_2X2 +#else // CONFIG_CB4X4 + const BLOCK_SIZE plane_bsize = + get_plane_block_size(AOMMAX(BLOCK_8X8, bsize), pd); +#endif // CONFIG_CB4X4 + + TXB_CTX txb_ctx; + get_txb_ctx(plane_bsize, tx_size, plane, a, l, &txb_ctx); + return av1_cost_coeffs_txb(cpi, x, plane, block, &txb_ctx); +#endif // !CONFIG_LV_MAP +} +#endif // !CONFIG_PVQ || CONFIG_VAR_TX + +// Get transform block visible dimensions cropped to the MI units. +static void get_txb_dimensions(const MACROBLOCKD *xd, int plane, + BLOCK_SIZE plane_bsize, int blk_row, int blk_col, + BLOCK_SIZE tx_bsize, int *width, int *height, + int *visible_width, int *visible_height) { + assert(tx_bsize <= plane_bsize); + int txb_height = block_size_high[tx_bsize]; + int txb_width = block_size_wide[tx_bsize]; + const int block_height = block_size_high[plane_bsize]; + const int block_width = block_size_wide[plane_bsize]; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + // TODO(aconverse@google.com): Investigate using crop_width/height here rather + // than the MI size + const int block_rows = + (xd->mb_to_bottom_edge >= 0) + ? block_height + : (xd->mb_to_bottom_edge >> (3 + pd->subsampling_y)) + block_height; + const int block_cols = + (xd->mb_to_right_edge >= 0) + ? block_width + : (xd->mb_to_right_edge >> (3 + pd->subsampling_x)) + block_width; + const int tx_unit_size = tx_size_wide_log2[0]; + if (width) *width = txb_width; + if (height) *height = txb_height; + *visible_width = clamp(block_cols - (blk_col << tx_unit_size), 0, txb_width); + *visible_height = + clamp(block_rows - (blk_row << tx_unit_size), 0, txb_height); +} + +// Compute the pixel domain sum square error on all visible 4x4s in the +// transform block. +static unsigned pixel_sse(const AV1_COMP *const cpi, const MACROBLOCKD *xd, + int plane, const uint8_t *src, const int src_stride, + const uint8_t *dst, const int dst_stride, int blk_row, + int blk_col, const BLOCK_SIZE plane_bsize, + const BLOCK_SIZE tx_bsize) { + int txb_rows, txb_cols, visible_rows, visible_cols; + get_txb_dimensions(xd, plane, plane_bsize, blk_row, blk_col, tx_bsize, + &txb_cols, &txb_rows, &visible_cols, &visible_rows); + assert(visible_rows > 0); + assert(visible_cols > 0); + if (txb_rows == visible_rows && txb_cols == visible_cols) { + unsigned sse; + cpi->fn_ptr[tx_bsize].vf(src, src_stride, dst, dst_stride, &sse); + return sse; + } +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + uint64_t sse = aom_highbd_sse_odd_size(src, src_stride, dst, dst_stride, + visible_cols, visible_rows); + return (unsigned int)ROUND_POWER_OF_TWO(sse, (xd->bd - 8) * 2); + } +#endif // CONFIG_HIGHBITDEPTH + unsigned sse = aom_sse_odd_size(src, src_stride, dst, dst_stride, + visible_cols, visible_rows); + return sse; +} + +// Compute the squares sum squares on all visible 4x4s in the transform block. +static int64_t sum_squares_visible(const MACROBLOCKD *xd, int plane, + const int16_t *diff, const int diff_stride, + int blk_row, int blk_col, + const BLOCK_SIZE plane_bsize, + const BLOCK_SIZE tx_bsize) { + int visible_rows, visible_cols; + get_txb_dimensions(xd, plane, plane_bsize, blk_row, blk_col, tx_bsize, NULL, + NULL, &visible_cols, &visible_rows); + return aom_sum_squares_2d_i16(diff, diff_stride, visible_cols, visible_rows); +} + +void av1_dist_block(const AV1_COMP *cpi, MACROBLOCK *x, int plane, + BLOCK_SIZE plane_bsize, int block, int blk_row, int blk_col, + TX_SIZE tx_size, int64_t *out_dist, int64_t *out_sse, + OUTPUT_STATUS output_status) { + MACROBLOCKD *const xd = &x->e_mbd; + const struct macroblock_plane *const p = &x->plane[plane]; +#if CONFIG_DAALA_DIST + int qm = OD_HVS_QM; + int use_activity_masking = 0; +#if CONFIG_PVQ + use_activity_masking = x->daala_enc.use_activity_masking; +#endif // CONFIG_PVQ + struct macroblockd_plane *const pd = &xd->plane[plane]; +#else // CONFIG_DAALA_DIST + const struct macroblockd_plane *const pd = &xd->plane[plane]; +#endif // CONFIG_DAALA_DIST + + if (cpi->sf.use_transform_domain_distortion && !CONFIG_DAALA_DIST) { + // Transform domain distortion computation is more efficient as it does + // not involve an inverse transform, but it is less accurate. + const int buffer_length = tx_size_2d[tx_size]; + int64_t this_sse; + int shift = (MAX_TX_SCALE - av1_get_tx_scale(tx_size)) * 2; + tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block); + tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); +#if CONFIG_PVQ + tran_low_t *ref_coeff = BLOCK_OFFSET(pd->pvq_ref_coeff, block); + +#if CONFIG_HIGHBITDEPTH + const int bd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd : 8; + *out_dist = av1_highbd_block_error2_c(coeff, dqcoeff, ref_coeff, + buffer_length, &this_sse, bd) >> + shift; +#else + *out_dist = av1_block_error2_c(coeff, dqcoeff, ref_coeff, buffer_length, + &this_sse) >> + shift; +#endif // CONFIG_HIGHBITDEPTH +#elif CONFIG_HIGHBITDEPTH + const int bd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd : 8; + *out_dist = + av1_highbd_block_error(coeff, dqcoeff, buffer_length, &this_sse, bd) >> + shift; +#else + *out_dist = + av1_block_error(coeff, dqcoeff, buffer_length, &this_sse) >> shift; +#endif // CONFIG_PVQ + *out_sse = this_sse >> shift; + } else { + const BLOCK_SIZE tx_bsize = txsize_to_bsize[tx_size]; +#if !CONFIG_PVQ || CONFIG_DAALA_DIST + const int bsw = block_size_wide[tx_bsize]; + const int bsh = block_size_high[tx_bsize]; +#endif + const int src_stride = x->plane[plane].src.stride; + const int dst_stride = xd->plane[plane].dst.stride; + // Scale the transform block index to pixel unit. + const int src_idx = (blk_row * src_stride + blk_col) + << tx_size_wide_log2[0]; + const int dst_idx = (blk_row * dst_stride + blk_col) + << tx_size_wide_log2[0]; + const uint8_t *src = &x->plane[plane].src.buf[src_idx]; + const uint8_t *dst = &xd->plane[plane].dst.buf[dst_idx]; + const tran_low_t *dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); + const uint16_t eob = p->eobs[block]; + + assert(cpi != NULL); + assert(tx_size_wide_log2[0] == tx_size_high_log2[0]); + +#if CONFIG_DAALA_DIST + if (plane == 0 && bsw >= 8 && bsh >= 8) { + if (output_status == OUTPUT_HAS_DECODED_PIXELS) { + const int pred_stride = block_size_wide[plane_bsize]; + const int pred_idx = (blk_row * pred_stride + blk_col) + << tx_size_wide_log2[0]; + const int16_t *pred = &pd->pred[pred_idx]; + int i, j; + DECLARE_ALIGNED(16, uint8_t, pred8[MAX_TX_SQUARE]); + + for (j = 0; j < bsh; j++) + for (i = 0; i < bsw; i++) + pred8[j * bsw + i] = pred[j * pred_stride + i]; + *out_sse = av1_daala_dist(src, src_stride, pred8, bsw, bsw, bsh, qm, + use_activity_masking, x->qindex); + } else { + *out_sse = av1_daala_dist(src, src_stride, dst, dst_stride, bsw, bsh, + qm, use_activity_masking, x->qindex); + } + } else +#endif // CONFIG_DAALA_DIST + { + const int diff_stride = block_size_wide[plane_bsize]; + const int diff_idx = (blk_row * diff_stride + blk_col) + << tx_size_wide_log2[0]; + const int16_t *diff = &p->src_diff[diff_idx]; + *out_sse = sum_squares_visible(xd, plane, diff, diff_stride, blk_row, + blk_col, plane_bsize, tx_bsize); +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + *out_sse = ROUND_POWER_OF_TWO(*out_sse, (xd->bd - 8) * 2); +#endif // CONFIG_HIGHBITDEPTH + } + *out_sse *= 16; + + if (eob) { + if (output_status == OUTPUT_HAS_DECODED_PIXELS) { +#if CONFIG_DAALA_DIST + if (plane == 0 && bsw >= 8 && bsh >= 8) + *out_dist = av1_daala_dist(src, src_stride, dst, dst_stride, bsw, bsh, + qm, use_activity_masking, x->qindex); + else +#endif // CONFIG_DAALA_DIST + *out_dist = + pixel_sse(cpi, xd, plane, src, src_stride, dst, dst_stride, + blk_row, blk_col, plane_bsize, tx_bsize); + } else { +#if CONFIG_HIGHBITDEPTH + uint8_t *recon; + DECLARE_ALIGNED(16, uint16_t, recon16[MAX_TX_SQUARE]); + + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + recon = CONVERT_TO_BYTEPTR(recon16); + else + recon = (uint8_t *)recon16; +#else + DECLARE_ALIGNED(16, uint8_t, recon[MAX_TX_SQUARE]); +#endif // CONFIG_HIGHBITDEPTH + +#if !CONFIG_PVQ +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + aom_highbd_convolve_copy(dst, dst_stride, recon, MAX_TX_SIZE, NULL, 0, + NULL, 0, bsw, bsh, xd->bd); + } else { +#endif // CONFIG_HIGHBITDEPTH + aom_convolve_copy(dst, dst_stride, recon, MAX_TX_SIZE, NULL, 0, NULL, + 0, bsw, bsh); +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH +#else + (void)dst; +#endif // !CONFIG_PVQ + + const PLANE_TYPE plane_type = get_plane_type(plane); + TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + + av1_inverse_transform_block(xd, dqcoeff, tx_type, tx_size, recon, + MAX_TX_SIZE, eob); + +#if CONFIG_DAALA_DIST + if (plane == 0 && bsw >= 8 && bsh >= 8) { + *out_dist = av1_daala_dist(src, src_stride, recon, MAX_TX_SIZE, bsw, + bsh, qm, use_activity_masking, x->qindex); + } else { + if (plane == 0) { + // Save decoded pixels for inter block in pd->pred to avoid + // block_8x8_rd_txfm_daala_dist() need to produce them + // by calling av1_inverse_transform_block() again. + const int pred_stride = block_size_wide[plane_bsize]; + const int pred_idx = (blk_row * pred_stride + blk_col) + << tx_size_wide_log2[0]; + int16_t *pred = &pd->pred[pred_idx]; + int i, j; + + for (j = 0; j < bsh; j++) + for (i = 0; i < bsw; i++) + pred[j * pred_stride + i] = recon[j * MAX_TX_SIZE + i]; + } +#endif // CONFIG_DAALA_DIST + *out_dist = + pixel_sse(cpi, xd, plane, src, src_stride, recon, MAX_TX_SIZE, + blk_row, blk_col, plane_bsize, tx_bsize); +#if CONFIG_DAALA_DIST + } +#endif // CONFIG_DAALA_DIST + } + *out_dist *= 16; + } else { + *out_dist = *out_sse; + } + } +} + +static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) { + struct rdcost_block_args *args = arg; + MACROBLOCK *const x = args->x; + MACROBLOCKD *const xd = &x->e_mbd; + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const AV1_COMP *cpi = args->cpi; + ENTROPY_CONTEXT *a = args->t_above + blk_col; + ENTROPY_CONTEXT *l = args->t_left + blk_row; +#if !CONFIG_TXK_SEL + const AV1_COMMON *cm = &cpi->common; +#endif + int64_t rd1, rd2, rd; + RD_STATS this_rd_stats; + + assert(tx_size == get_tx_size(plane, xd)); + + av1_init_rd_stats(&this_rd_stats); + + if (args->exit_early) return; + + if (!is_inter_block(mbmi)) { + av1_predict_intra_block_facade(xd, plane, block, blk_col, blk_row, tx_size); + av1_subtract_txb(x, plane, plane_bsize, blk_col, blk_row, tx_size); + } + +#if !CONFIG_TXK_SEL + // full forward transform and quantization + const int coeff_ctx = combine_entropy_contexts(*a, *l); + av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, + coeff_ctx, AV1_XFORM_QUANT_FP); + if (x->plane[plane].eobs[block] && !xd->lossless[mbmi->segment_id]) + av1_optimize_b(cm, x, plane, block, tx_size, coeff_ctx); + + if (!is_inter_block(mbmi)) { + struct macroblock_plane *const p = &x->plane[plane]; + av1_inverse_transform_block_facade(xd, plane, block, blk_row, blk_col, + p->eobs[block]); + av1_dist_block(args->cpi, x, plane, plane_bsize, block, blk_row, blk_col, + tx_size, &this_rd_stats.dist, &this_rd_stats.sse, + OUTPUT_HAS_DECODED_PIXELS); + } else { + av1_dist_block(args->cpi, x, plane, plane_bsize, block, blk_row, blk_col, + tx_size, &this_rd_stats.dist, &this_rd_stats.sse, + OUTPUT_HAS_PREDICTED_PIXELS); + } +#if CONFIG_CFL + if (plane == AOM_PLANE_Y && x->cfl_store_y) { + struct macroblockd_plane *const pd = &xd->plane[plane]; + const int dst_stride = pd->dst.stride; + uint8_t *dst = + &pd->dst.buf[(blk_row * dst_stride + blk_col) << tx_size_wide_log2[0]]; + cfl_store(xd->cfl, dst, dst_stride, blk_row, blk_col, tx_size); + } +#endif + rd = RDCOST(x->rdmult, x->rddiv, 0, this_rd_stats.dist); + if (args->this_rd + rd > args->best_rd) { + args->exit_early = 1; + return; + } +#if !CONFIG_PVQ + const PLANE_TYPE plane_type = get_plane_type(plane); + const TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + const SCAN_ORDER *scan_order = + get_scan(cm, tx_size, tx_type, is_inter_block(mbmi)); + this_rd_stats.rate = + av1_cost_coeffs(cpi, x, plane, block, tx_size, scan_order, a, l, + args->use_fast_coef_costing); +#else // !CONFIG_PVQ + this_rd_stats.rate = x->rate; +#endif // !CONFIG_PVQ +#else // !CONFIG_TXK_SEL + av1_search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize, + tx_size, a, l, args->use_fast_coef_costing, + &this_rd_stats); +#endif // !CONFIG_TXK_SEL + +#if !CONFIG_PVQ +#if CONFIG_RD_DEBUG + av1_update_txb_coeff_cost(&this_rd_stats, plane, tx_size, blk_row, blk_col, + this_rd_stats.rate); +#endif // CONFIG_RD_DEBUG + av1_set_txb_context(x, plane, block, tx_size, a, l); +#endif // !CONFIG_PVQ + + rd1 = RDCOST(x->rdmult, x->rddiv, this_rd_stats.rate, this_rd_stats.dist); + rd2 = RDCOST(x->rdmult, x->rddiv, 0, this_rd_stats.sse); + + // TODO(jingning): temporarily enabled only for luma component + rd = AOMMIN(rd1, rd2); + +#if CONFIG_DAALA_DIST + if (plane == 0 && + (tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4)) { + this_rd_stats.dist = 0; + this_rd_stats.sse = 0; + rd = 0; + x->rate_4x4[block] = this_rd_stats.rate; + } +#endif // CONFIG_DAALA_DIST + +#if !CONFIG_PVQ + this_rd_stats.skip &= !x->plane[plane].eobs[block]; +#else + this_rd_stats.skip &= x->pvq_skip[plane]; +#endif // !CONFIG_PVQ + av1_merge_rd_stats(&args->rd_stats, &this_rd_stats); + + args->this_rd += rd; + + if (args->this_rd > args->best_rd) { + args->exit_early = 1; + return; + } +} + +#if CONFIG_DAALA_DIST +static void block_8x8_rd_txfm_daala_dist(int plane, int block, int blk_row, + int blk_col, BLOCK_SIZE plane_bsize, + TX_SIZE tx_size, void *arg) { + struct rdcost_block_args *args = arg; + MACROBLOCK *const x = args->x; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + int64_t rd, rd1, rd2; + RD_STATS this_rd_stats; + int qm = OD_HVS_QM; + int use_activity_masking = 0; + + (void)tx_size; +#if CONFIG_PVQ + use_activity_masking = x->daala_enc.use_activity_masking; +#endif // CONFIG_PVQ + av1_init_rd_stats(&this_rd_stats); + + if (args->exit_early) return; + + { + const struct macroblock_plane *const p = &x->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; + + const int src_stride = p->src.stride; + const int dst_stride = pd->dst.stride; + const int diff_stride = block_size_wide[plane_bsize]; + + const uint8_t *src = + &p->src.buf[(blk_row * src_stride + blk_col) << tx_size_wide_log2[0]]; + const uint8_t *dst = + &pd->dst.buf[(blk_row * dst_stride + blk_col) << tx_size_wide_log2[0]]; + + unsigned int tmp1, tmp2; + int qindex = x->qindex; + const int pred_stride = block_size_wide[plane_bsize]; + const int pred_idx = (blk_row * pred_stride + blk_col) + << tx_size_wide_log2[0]; + int16_t *pred = &pd->pred[pred_idx]; + int i, j; + const int tx_blk_size = 8; + + DECLARE_ALIGNED(16, uint8_t, pred8[8 * 8]); + + for (j = 0; j < tx_blk_size; j++) + for (i = 0; i < tx_blk_size; i++) + pred8[j * tx_blk_size + i] = pred[j * diff_stride + i]; + + tmp1 = av1_daala_dist(src, src_stride, pred8, tx_blk_size, 8, 8, qm, + use_activity_masking, qindex); + tmp2 = av1_daala_dist(src, src_stride, dst, dst_stride, 8, 8, qm, + use_activity_masking, qindex); + + if (!is_inter_block(mbmi)) { + this_rd_stats.sse = (int64_t)tmp1 * 16; + this_rd_stats.dist = (int64_t)tmp2 * 16; + } else { + // For inter mode, the decoded pixels are provided in pd->pred, + // while the predicted pixels are in dst. + this_rd_stats.sse = (int64_t)tmp2 * 16; + this_rd_stats.dist = (int64_t)tmp1 * 16; + } + } + + rd = RDCOST(x->rdmult, x->rddiv, 0, this_rd_stats.dist); + if (args->this_rd + rd > args->best_rd) { + args->exit_early = 1; + return; + } + + { + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + // The rate of the current 8x8 block is the sum of four 4x4 blocks in it. + this_rd_stats.rate = x->rate_4x4[block - max_blocks_wide - 1] + + x->rate_4x4[block - max_blocks_wide] + + x->rate_4x4[block - 1] + x->rate_4x4[block]; + } + rd1 = RDCOST(x->rdmult, x->rddiv, this_rd_stats.rate, this_rd_stats.dist); + rd2 = RDCOST(x->rdmult, x->rddiv, 0, this_rd_stats.sse); + rd = AOMMIN(rd1, rd2); + + args->rd_stats.dist += this_rd_stats.dist; + args->rd_stats.sse += this_rd_stats.sse; + + args->this_rd += rd; + + if (args->this_rd > args->best_rd) { + args->exit_early = 1; + return; + } +} +#endif // CONFIG_DAALA_DIST + +static void txfm_rd_in_plane(MACROBLOCK *x, const AV1_COMP *cpi, + RD_STATS *rd_stats, int64_t ref_best_rd, int plane, + BLOCK_SIZE bsize, TX_SIZE tx_size, + int use_fast_coef_casting) { + MACROBLOCKD *const xd = &x->e_mbd; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + struct rdcost_block_args args; + av1_zero(args); + args.x = x; + args.cpi = cpi; + args.best_rd = ref_best_rd; + args.use_fast_coef_costing = use_fast_coef_casting; + av1_init_rd_stats(&args.rd_stats); + + if (plane == 0) xd->mi[0]->mbmi.tx_size = tx_size; + + av1_get_entropy_contexts(bsize, tx_size, pd, args.t_above, args.t_left); + +#if CONFIG_DAALA_DIST + if (plane == 0 && + (tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4)) + av1_foreach_8x8_transformed_block_in_plane( + xd, bsize, plane, block_rd_txfm, block_8x8_rd_txfm_daala_dist, &args); + else +#endif // CONFIG_DAALA_DIST + av1_foreach_transformed_block_in_plane(xd, bsize, plane, block_rd_txfm, + &args); + + if (args.exit_early) { + av1_invalid_rd_stats(rd_stats); + } else { + *rd_stats = args.rd_stats; + } +} + +#if CONFIG_SUPERTX +void av1_txfm_rd_in_plane_supertx(MACROBLOCK *x, const AV1_COMP *cpi, int *rate, + int64_t *distortion, int *skippable, + int64_t *sse, int64_t ref_best_rd, int plane, + BLOCK_SIZE bsize, TX_SIZE tx_size, + int use_fast_coef_casting) { + MACROBLOCKD *const xd = &x->e_mbd; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + struct rdcost_block_args args; + av1_zero(args); + args.cpi = cpi; + args.x = x; + args.best_rd = ref_best_rd; + args.use_fast_coef_costing = use_fast_coef_casting; + +#if CONFIG_EXT_TX + assert(tx_size < TX_SIZES); +#endif // CONFIG_EXT_TX + + if (plane == 0) xd->mi[0]->mbmi.tx_size = tx_size; + + av1_get_entropy_contexts(bsize, tx_size, pd, args.t_above, args.t_left); + + block_rd_txfm(plane, 0, 0, 0, get_plane_block_size(bsize, pd), tx_size, + &args); + + if (args.exit_early) { + *rate = INT_MAX; + *distortion = INT64_MAX; + *sse = INT64_MAX; + *skippable = 0; + } else { + *distortion = args.rd_stats.dist; + *rate = args.rd_stats.rate; + *sse = args.rd_stats.sse; + *skippable = !x->plane[plane].eobs[0]; + } +} +#endif // CONFIG_SUPERTX + +static int tx_size_cost(const AV1_COMP *const cpi, const MACROBLOCK *const x, + BLOCK_SIZE bsize, TX_SIZE tx_size) { + const AV1_COMMON *const cm = &cpi->common; + const MACROBLOCKD *const xd = &x->e_mbd; + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + + const int tx_select = + cm->tx_mode == TX_MODE_SELECT && mbmi->sb_type >= BLOCK_8X8; + + if (tx_select) { + const int is_inter = is_inter_block(mbmi); + const int tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize] + : intra_tx_size_cat_lookup[bsize]; + const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size]; + const int depth = tx_size_to_depth(coded_tx_size); + const int tx_size_ctx = get_tx_size_context(xd); + const int r_tx_size = cpi->tx_size_cost[tx_size_cat][tx_size_ctx][depth]; + return r_tx_size; + } else { + return 0; + } +} + +// #TODO(angiebird): use this function whenever it's possible +int av1_tx_type_cost(const AV1_COMP *cpi, const MACROBLOCKD *xd, + BLOCK_SIZE bsize, int plane, TX_SIZE tx_size, + TX_TYPE tx_type) { + if (plane > 0) return 0; + + const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const int is_inter = is_inter_block(mbmi); +#if CONFIG_EXT_TX + const AV1_COMMON *cm = &cpi->common; + if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) > 1 && + !xd->lossless[xd->mi[0]->mbmi.segment_id]) { + const int ext_tx_set = + get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used); + if (is_inter) { + if (ext_tx_set > 0) + return cpi + ->inter_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]][tx_type]; + } else { + if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX) + return cpi->intra_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]] + [mbmi->mode][tx_type]; + } + } +#else + (void)bsize; + if (tx_size < TX_32X32 && !xd->lossless[xd->mi[0]->mbmi.segment_id] && + !FIXED_TX_TYPE) { + if (is_inter) { + return cpi->inter_tx_type_costs[tx_size][tx_type]; + } else { + return cpi->intra_tx_type_costs[tx_size] + [intra_mode_to_tx_type_context[mbmi->mode]] + [tx_type]; + } + } +#endif // CONFIG_EXT_TX + return 0; +} +static int64_t txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x, + RD_STATS *rd_stats, int64_t ref_best_rd, BLOCK_SIZE bs, + TX_TYPE tx_type, int tx_size) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + int64_t rd = INT64_MAX; + aom_prob skip_prob = av1_get_skip_prob(cm, xd); + int s0, s1; + const int is_inter = is_inter_block(mbmi); + const int tx_select = + cm->tx_mode == TX_MODE_SELECT && mbmi->sb_type >= BLOCK_8X8; + + const int r_tx_size = tx_size_cost(cpi, x, bs, tx_size); + + assert(skip_prob > 0); +#if CONFIG_EXT_TX && CONFIG_RECT_TX + assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed_bsize(bs))); +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + + s0 = av1_cost_bit(skip_prob, 0); + s1 = av1_cost_bit(skip_prob, 1); + + mbmi->tx_type = tx_type; + mbmi->tx_size = tx_size; + txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, tx_size, + cpi->sf.use_fast_coef_costing); + if (rd_stats->rate == INT_MAX) return INT64_MAX; +#if !CONFIG_TXK_SEL + int plane = 0; + rd_stats->rate += av1_tx_type_cost(cpi, xd, bs, plane, tx_size, tx_type); +#endif + + if (rd_stats->skip) { + if (is_inter) { + rd = RDCOST(x->rdmult, x->rddiv, s1, rd_stats->sse); + } else { + rd = RDCOST(x->rdmult, x->rddiv, s1 + r_tx_size * tx_select, + rd_stats->sse); + } + } else { + rd = RDCOST(x->rdmult, x->rddiv, + rd_stats->rate + s0 + r_tx_size * tx_select, rd_stats->dist); + } + + if (tx_select) rd_stats->rate += r_tx_size; + + if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] && + !(rd_stats->skip)) + rd = AOMMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, rd_stats->sse)); + + return rd; +} + +static int skip_txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs, + TX_TYPE tx_type, TX_SIZE tx_size) { + const MACROBLOCKD *const xd = &x->e_mbd; + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const TX_SIZE max_tx_size = max_txsize_lookup[bs]; + const int is_inter = is_inter_block(mbmi); + int prune = 0; + if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE) + // passing -1 in for tx_type indicates that all 1D + // transforms should be considered for pruning + prune = prune_tx_types(cpi, bs, x, xd, -1); + +#if CONFIG_REF_MV + if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) return 1; +#endif // CONFIG_REF_MV + if (FIXED_TX_TYPE && tx_type != get_default_tx_type(0, xd, 0, tx_size)) + return 1; + if (!is_inter && x->use_default_intra_tx_type && + tx_type != get_default_tx_type(0, xd, 0, tx_size)) + return 1; + if (is_inter && x->use_default_inter_tx_type && + tx_type != get_default_tx_type(0, xd, 0, tx_size)) + return 1; + if (max_tx_size >= TX_32X32 && tx_size == TX_4X4) return 1; +#if CONFIG_EXT_TX + const AV1_COMMON *const cm = &cpi->common; + int ext_tx_set = + get_ext_tx_set(tx_size, bs, is_inter, cm->reduced_tx_set_used); + if (is_inter) { + if (!ext_tx_used_inter[ext_tx_set][tx_type]) return 1; + if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) { + if (!do_tx_type_search(tx_type, prune)) return 1; + } + } else { + if (!ALLOW_INTRA_EXT_TX && bs >= BLOCK_8X8) { + if (tx_type != intra_mode_to_tx_type_context[mbmi->mode]) return 1; + } + if (!ext_tx_used_intra[ext_tx_set][tx_type]) return 1; + } +#else // CONFIG_EXT_TX + if (tx_size >= TX_32X32 && tx_type != DCT_DCT) return 1; + if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE && + !do_tx_type_search(tx_type, prune)) + return 1; +#endif // CONFIG_EXT_TX + return 0; +} + +#if CONFIG_EXT_INTER +static int64_t estimate_yrd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bs, + MACROBLOCK *x, int *r, int64_t *d, int *s, + int64_t *sse, int64_t ref_best_rd) { + RD_STATS rd_stats; + int64_t rd = txfm_yrd(cpi, x, &rd_stats, ref_best_rd, bs, DCT_DCT, + max_txsize_lookup[bs]); + *r = rd_stats.rate; + *d = rd_stats.dist; + *s = rd_stats.skip; + *sse = rd_stats.sse; + return rd; +} +#endif // CONFIG_EXT_INTER + +static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x, + RD_STATS *rd_stats, int64_t ref_best_rd, + BLOCK_SIZE bs) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + TX_TYPE tx_type, best_tx_type = DCT_DCT; + int64_t this_rd, best_rd = INT64_MAX; + aom_prob skip_prob = av1_get_skip_prob(cm, xd); + int s0 = av1_cost_bit(skip_prob, 0); + int s1 = av1_cost_bit(skip_prob, 1); + const int is_inter = is_inter_block(mbmi); + int prune = 0; + const int plane = 0; +#if CONFIG_EXT_TX + int ext_tx_set; +#endif // CONFIG_EXT_TX + av1_invalid_rd_stats(rd_stats); + + mbmi->tx_size = tx_size_from_tx_mode(bs, cm->tx_mode, is_inter); +#if CONFIG_VAR_TX + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); +#endif // CONFIG_VAR_TX +#if CONFIG_EXT_TX + ext_tx_set = + get_ext_tx_set(mbmi->tx_size, bs, is_inter, cm->reduced_tx_set_used); +#endif // CONFIG_EXT_TX + + if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE) +#if CONFIG_EXT_TX + prune = prune_tx_types(cpi, bs, x, xd, ext_tx_set); +#else + prune = prune_tx_types(cpi, bs, x, xd, 0); +#endif // CONFIG_EXT_TX +#if CONFIG_EXT_TX + if (get_ext_tx_types(mbmi->tx_size, bs, is_inter, cm->reduced_tx_set_used) > + 1 && + !xd->lossless[mbmi->segment_id]) { +#if CONFIG_PVQ + od_rollback_buffer pre_buf, post_buf; + + od_encode_checkpoint(&x->daala_enc, &pre_buf); + od_encode_checkpoint(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + + for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) { + RD_STATS this_rd_stats; + if (is_inter) { + if (x->use_default_inter_tx_type && + tx_type != get_default_tx_type(0, xd, 0, mbmi->tx_size)) + continue; + if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue; + if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) { + if (!do_tx_type_search(tx_type, prune)) continue; + } + } else { + if (x->use_default_intra_tx_type && + tx_type != get_default_tx_type(0, xd, 0, mbmi->tx_size)) + continue; + if (!ALLOW_INTRA_EXT_TX && bs >= BLOCK_8X8) { + if (tx_type != intra_mode_to_tx_type_context[mbmi->mode]) continue; + } + if (!ext_tx_used_intra[ext_tx_set][tx_type]) continue; + } + + mbmi->tx_type = tx_type; + + txfm_rd_in_plane(x, cpi, &this_rd_stats, ref_best_rd, 0, bs, + mbmi->tx_size, cpi->sf.use_fast_coef_costing); +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &pre_buf); +#endif // CONFIG_PVQ + if (this_rd_stats.rate == INT_MAX) continue; + av1_tx_type_cost(cpi, xd, bs, plane, mbmi->tx_size, tx_type); + + if (this_rd_stats.skip) + this_rd = RDCOST(x->rdmult, x->rddiv, s1, this_rd_stats.sse); + else + this_rd = RDCOST(x->rdmult, x->rddiv, this_rd_stats.rate + s0, + this_rd_stats.dist); + if (is_inter_block(mbmi) && !xd->lossless[mbmi->segment_id] && + !this_rd_stats.skip) + this_rd = + AOMMIN(this_rd, RDCOST(x->rdmult, x->rddiv, s1, this_rd_stats.sse)); + + if (this_rd < best_rd) { + best_rd = this_rd; + best_tx_type = mbmi->tx_type; + *rd_stats = this_rd_stats; +#if CONFIG_PVQ + od_encode_checkpoint(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + } + } +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + } else { + mbmi->tx_type = DCT_DCT; + txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, mbmi->tx_size, + cpi->sf.use_fast_coef_costing); + } +#else // CONFIG_EXT_TX + if (mbmi->tx_size < TX_32X32 && !xd->lossless[mbmi->segment_id]) { + for (tx_type = 0; tx_type < TX_TYPES; ++tx_type) { + RD_STATS this_rd_stats; + if (!is_inter && x->use_default_intra_tx_type && + tx_type != get_default_tx_type(0, xd, 0, mbmi->tx_size)) + continue; + if (is_inter && x->use_default_inter_tx_type && + tx_type != get_default_tx_type(0, xd, 0, mbmi->tx_size)) + continue; + mbmi->tx_type = tx_type; + txfm_rd_in_plane(x, cpi, &this_rd_stats, ref_best_rd, 0, bs, + mbmi->tx_size, cpi->sf.use_fast_coef_costing); + if (this_rd_stats.rate == INT_MAX) continue; + + av1_tx_type_cost(cpi, xd, bs, plane, mbmi->tx_size, tx_type); + if (is_inter) { + if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE && + !do_tx_type_search(tx_type, prune)) + continue; + } + if (this_rd_stats.skip) + this_rd = RDCOST(x->rdmult, x->rddiv, s1, this_rd_stats.sse); + else + this_rd = RDCOST(x->rdmult, x->rddiv, this_rd_stats.rate + s0, + this_rd_stats.dist); + if (is_inter && !xd->lossless[mbmi->segment_id] && !this_rd_stats.skip) + this_rd = + AOMMIN(this_rd, RDCOST(x->rdmult, x->rddiv, s1, this_rd_stats.sse)); + + if (this_rd < best_rd) { + best_rd = this_rd; + best_tx_type = mbmi->tx_type; + *rd_stats = this_rd_stats; + } + } + } else { + mbmi->tx_type = DCT_DCT; + txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, mbmi->tx_size, + cpi->sf.use_fast_coef_costing); + } +#endif // CONFIG_EXT_TX + mbmi->tx_type = best_tx_type; +} + +static void choose_smallest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x, + RD_STATS *rd_stats, int64_t ref_best_rd, + BLOCK_SIZE bs) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + + mbmi->tx_size = TX_4X4; + mbmi->tx_type = DCT_DCT; +#if CONFIG_VAR_TX + mbmi->min_tx_size = get_min_tx_size(TX_4X4); +#endif // CONFIG_VAR_TX + + txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, mbmi->tx_size, + cpi->sf.use_fast_coef_costing); +} + +#if CONFIG_TXK_SEL || CONFIG_VAR_TX +static INLINE int bsize_to_num_blk(BLOCK_SIZE bsize) { + int num_blk = 1 << (num_pels_log2_lookup[bsize] - 2 * tx_size_wide_log2[0]); + return num_blk; +} +#endif // CONFIG_TXK_SEL || CONFIG_VAR_TX + +static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi, + MACROBLOCK *x, RD_STATS *rd_stats, + int64_t ref_best_rd, BLOCK_SIZE bs) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + int64_t rd = INT64_MAX; + int n; + int start_tx, end_tx; + int64_t best_rd = INT64_MAX, last_rd = INT64_MAX; + const TX_SIZE max_tx_size = max_txsize_lookup[bs]; + TX_SIZE best_tx_size = max_tx_size; + TX_TYPE best_tx_type = DCT_DCT; +#if CONFIG_TXK_SEL + TX_TYPE best_txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)]; + const int num_blk = bsize_to_num_blk(bs); +#endif // CONFIG_TXK_SEL + const int tx_select = cm->tx_mode == TX_MODE_SELECT; + const int is_inter = is_inter_block(mbmi); +#if CONFIG_PVQ + od_rollback_buffer buf; + od_encode_checkpoint(&x->daala_enc, &buf); +#endif // CONFIG_PVQ + + av1_invalid_rd_stats(rd_stats); + +#if CONFIG_EXT_TX && CONFIG_RECT_TX + int evaluate_rect_tx = 0; + if (tx_select) { + evaluate_rect_tx = is_rect_tx_allowed(xd, mbmi); + } else { + const TX_SIZE chosen_tx_size = + tx_size_from_tx_mode(bs, cm->tx_mode, is_inter); + evaluate_rect_tx = is_rect_tx(chosen_tx_size); + assert(IMPLIES(evaluate_rect_tx, is_rect_tx_allowed(xd, mbmi))); + } + if (evaluate_rect_tx) { + TX_TYPE tx_start = DCT_DCT; + TX_TYPE tx_end = TX_TYPES; +#if CONFIG_TXK_SEL + // The tx_type becomes dummy when lv_map is on. The tx_type search will be + // performed in av1_search_txk_type() + tx_end = DCT_DCT + 1; +#endif + TX_TYPE tx_type; + for (tx_type = tx_start; tx_type < tx_end; ++tx_type) { +#if CONFIG_REF_MV + if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue; +#endif // CONFIG_REF_MV + const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs]; + RD_STATS this_rd_stats; + int ext_tx_set = + get_ext_tx_set(rect_tx_size, bs, is_inter, cm->reduced_tx_set_used); + if ((is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) || + (!is_inter && ext_tx_used_intra[ext_tx_set][tx_type])) { + rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, + rect_tx_size); + if (rd < best_rd) { +#if CONFIG_TXK_SEL + memcpy(best_txk_type, mbmi->txk_type, + sizeof(best_txk_type[0]) * num_blk); +#endif + best_tx_type = tx_type; + best_tx_size = rect_tx_size; + best_rd = rd; + *rd_stats = this_rd_stats; + } + } +#if CONFIG_CB4X4 && !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 + const int is_inter = is_inter_block(mbmi); + if (mbmi->sb_type < BLOCK_8X8 && is_inter) break; +#endif // CONFIG_CB4X4 && !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 + } + } +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + + if (tx_select) { + start_tx = max_tx_size; + end_tx = (max_tx_size >= TX_32X32) ? TX_8X8 : TX_4X4; + } else { + const TX_SIZE chosen_tx_size = + tx_size_from_tx_mode(bs, cm->tx_mode, is_inter); + start_tx = chosen_tx_size; + end_tx = chosen_tx_size; + } + + last_rd = INT64_MAX; + for (n = start_tx; n >= end_tx; --n) { +#if CONFIG_EXT_TX && CONFIG_RECT_TX + if (is_rect_tx(n)) break; +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + TX_TYPE tx_start = DCT_DCT; + TX_TYPE tx_end = TX_TYPES; +#if CONFIG_TXK_SEL + // The tx_type becomes dummy when lv_map is on. The tx_type search will be + // performed in av1_search_txk_type() + tx_end = DCT_DCT + 1; +#endif + TX_TYPE tx_type; + for (tx_type = tx_start; tx_type < tx_end; ++tx_type) { + RD_STATS this_rd_stats; + if (skip_txfm_search(cpi, x, bs, tx_type, n)) continue; + rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, n); +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &buf); +#endif // CONFIG_PVQ + // Early termination in transform size search. + if (cpi->sf.tx_size_search_breakout && + (rd == INT64_MAX || + (this_rd_stats.skip == 1 && tx_type != DCT_DCT && n < start_tx) || + (n < (int)max_tx_size && rd > last_rd))) + break; + + last_rd = rd; + if (rd < best_rd) { +#if CONFIG_TXK_SEL + memcpy(best_txk_type, mbmi->txk_type, + sizeof(best_txk_type[0]) * num_blk); +#endif + best_tx_type = tx_type; + best_tx_size = n; + best_rd = rd; + *rd_stats = this_rd_stats; + } +#if CONFIG_CB4X4 && !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 + const int is_inter = is_inter_block(mbmi); + if (mbmi->sb_type < BLOCK_8X8 && is_inter) break; +#endif // CONFIG_CB4X4 && !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 + } + } + mbmi->tx_size = best_tx_size; + mbmi->tx_type = best_tx_type; +#if CONFIG_TXK_SEL + memcpy(mbmi->txk_type, best_txk_type, sizeof(best_txk_type[0]) * num_blk); +#endif + +#if CONFIG_VAR_TX + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); +#endif // CONFIG_VAR_TX + +#if !CONFIG_EXT_TX + if (mbmi->tx_size >= TX_32X32) assert(mbmi->tx_type == DCT_DCT); +#endif // !CONFIG_EXT_TX +#if CONFIG_PVQ + if (best_rd != INT64_MAX) { + txfm_yrd(cpi, x, rd_stats, ref_best_rd, bs, best_tx_type, best_tx_size); + } +#endif // CONFIG_PVQ +} + +static void super_block_yrd(const AV1_COMP *const cpi, MACROBLOCK *x, + RD_STATS *rd_stats, BLOCK_SIZE bs, + int64_t ref_best_rd) { + MACROBLOCKD *xd = &x->e_mbd; + av1_init_rd_stats(rd_stats); + + assert(bs == xd->mi[0]->mbmi.sb_type); + + if (xd->lossless[xd->mi[0]->mbmi.segment_id]) { + choose_smallest_tx_size(cpi, x, rd_stats, ref_best_rd, bs); + } else if (cpi->sf.tx_size_search_method == USE_LARGESTALL) { + choose_largest_tx_size(cpi, x, rd_stats, ref_best_rd, bs); + } else { + choose_tx_size_type_from_rd(cpi, x, rd_stats, ref_best_rd, bs); + } +} + +static int conditional_skipintra(PREDICTION_MODE mode, + PREDICTION_MODE best_intra_mode) { + if (mode == D117_PRED && best_intra_mode != V_PRED && + best_intra_mode != D135_PRED) + return 1; + if (mode == D63_PRED && best_intra_mode != V_PRED && + best_intra_mode != D45_PRED) + return 1; + if (mode == D207_PRED && best_intra_mode != H_PRED && + best_intra_mode != D45_PRED) + return 1; + if (mode == D153_PRED && best_intra_mode != H_PRED && + best_intra_mode != D135_PRED) + return 1; + return 0; +} + +// Model based RD estimation for luma intra blocks. +static int64_t intra_model_yrd(const AV1_COMP *const cpi, MACROBLOCK *const x, + BLOCK_SIZE bsize, int mode_cost) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + RD_STATS this_rd_stats; + int row, col; + int64_t temp_sse, this_rd; + const TX_SIZE tx_size = tx_size_from_tx_mode(bsize, cpi->common.tx_mode, 0); + const int stepr = tx_size_high_unit[tx_size]; + const int stepc = tx_size_wide_unit[tx_size]; + const int max_blocks_wide = max_block_wide(xd, bsize, 0); + const int max_blocks_high = max_block_high(xd, bsize, 0); + mbmi->tx_size = tx_size; + // Prediction. + const int step = stepr * stepc; + int block = 0; + for (row = 0; row < max_blocks_high; row += stepr) { + for (col = 0; col < max_blocks_wide; col += stepc) { + av1_predict_intra_block_facade(xd, 0, block, col, row, tx_size); + block += step; + } + } + // RD estimation. + model_rd_for_sb(cpi, bsize, x, xd, 0, 0, &this_rd_stats.rate, + &this_rd_stats.dist, &this_rd_stats.skip, &temp_sse); +#if CONFIG_EXT_INTRA + if (av1_is_directional_mode(mbmi->mode, bsize)) { + mode_cost += write_uniform_cost(2 * MAX_ANGLE_DELTA + 1, + MAX_ANGLE_DELTA + mbmi->angle_delta[0]); + } +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + if (mbmi->mode == DC_PRED) { + const aom_prob prob = cpi->common.fc->filter_intra_probs[0]; + if (mbmi->filter_intra_mode_info.use_filter_intra_mode[0]) { + const int mode = mbmi->filter_intra_mode_info.filter_intra_mode[0]; + mode_cost += (av1_cost_bit(prob, 1) + + write_uniform_cost(FILTER_INTRA_MODES, mode)); + } else { + mode_cost += av1_cost_bit(prob, 0); + } + } +#endif // CONFIG_FILTER_INTRA + this_rd = RDCOST(x->rdmult, x->rddiv, this_rd_stats.rate + mode_cost, + this_rd_stats.dist); + return this_rd; +} + +#if CONFIG_PALETTE +// Extends 'color_map' array from 'orig_width x orig_height' to 'new_width x +// new_height'. Extra rows and columns are filled in by copying last valid +// row/column. +static void extend_palette_color_map(uint8_t *const color_map, int orig_width, + int orig_height, int new_width, + int new_height) { + int j; + assert(new_width >= orig_width); + assert(new_height >= orig_height); + if (new_width == orig_width && new_height == orig_height) return; + + for (j = orig_height - 1; j >= 0; --j) { + memmove(color_map + j * new_width, color_map + j * orig_width, orig_width); + // Copy last column to extra columns. + memset(color_map + j * new_width + orig_width, + color_map[j * new_width + orig_width - 1], new_width - orig_width); + } + // Copy last row to extra rows. + for (j = orig_height; j < new_height; ++j) { + memcpy(color_map + j * new_width, color_map + (orig_height - 1) * new_width, + new_width); + } +} + +static int rd_pick_palette_intra_sby(const AV1_COMP *const cpi, MACROBLOCK *x, + BLOCK_SIZE bsize, int palette_ctx, + int dc_mode_cost, MB_MODE_INFO *best_mbmi, + uint8_t *best_palette_color_map, + int64_t *best_rd, int64_t *best_model_rd, + int *rate, int *rate_tokenonly, + int64_t *distortion, int *skippable) { + int rate_overhead = 0; + MACROBLOCKD *const xd = &x->e_mbd; + MODE_INFO *const mic = xd->mi[0]; + MB_MODE_INFO *const mbmi = &mic->mbmi; + int this_rate, colors, n; + const int src_stride = x->plane[0].src.stride; + const uint8_t *const src = x->plane[0].src.buf; + uint8_t *const color_map = xd->plane[0].color_index_map; + int block_width, block_height, rows, cols; + av1_get_block_dimensions(bsize, 0, xd, &block_width, &block_height, &rows, + &cols); + + assert(cpi->common.allow_screen_content_tools); + +#if CONFIG_HIGHBITDEPTH + if (cpi->common.use_highbitdepth) + colors = av1_count_colors_highbd(src, src_stride, rows, cols, + cpi->common.bit_depth); + else +#endif // CONFIG_HIGHBITDEPTH + colors = av1_count_colors(src, src_stride, rows, cols); +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0; +#endif // CONFIG_FILTER_INTRA + + if (colors > 1 && colors <= 64) { + int r, c, i, j, k, palette_mode_cost; + const int max_itr = 50; + uint8_t color_order[PALETTE_MAX_SIZE]; + float *const data = x->palette_buffer->kmeans_data_buf; + float centroids[PALETTE_MAX_SIZE]; + float lb, ub, val; + RD_STATS tokenonly_rd_stats; + int64_t this_rd, this_model_rd; + PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; +#if CONFIG_HIGHBITDEPTH + uint16_t *src16 = CONVERT_TO_SHORTPTR(src); + if (cpi->common.use_highbitdepth) + lb = ub = src16[0]; + else +#endif // CONFIG_HIGHBITDEPTH + lb = ub = src[0]; + +#if CONFIG_HIGHBITDEPTH + if (cpi->common.use_highbitdepth) { + for (r = 0; r < rows; ++r) { + for (c = 0; c < cols; ++c) { + val = src16[r * src_stride + c]; + data[r * cols + c] = val; + if (val < lb) + lb = val; + else if (val > ub) + ub = val; + } + } + } else { +#endif // CONFIG_HIGHBITDEPTH + for (r = 0; r < rows; ++r) { + for (c = 0; c < cols; ++c) { + val = src[r * src_stride + c]; + data[r * cols + c] = val; + if (val < lb) + lb = val; + else if (val > ub) + ub = val; + } + } +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + + mbmi->mode = DC_PRED; +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0; +#endif // CONFIG_FILTER_INTRA + + if (rows * cols > PALETTE_MAX_BLOCK_SIZE) return 0; + + for (n = colors > PALETTE_MAX_SIZE ? PALETTE_MAX_SIZE : colors; n >= 2; + --n) { + for (i = 0; i < n; ++i) + centroids[i] = lb + (2 * i + 1) * (ub - lb) / n / 2; + av1_k_means(data, centroids, color_map, rows * cols, n, 1, max_itr); + k = av1_remove_duplicates(centroids, n); + +#if CONFIG_HIGHBITDEPTH + if (cpi->common.use_highbitdepth) + for (i = 0; i < k; ++i) + pmi->palette_colors[i] = + clip_pixel_highbd((int)centroids[i], cpi->common.bit_depth); + else +#endif // CONFIG_HIGHBITDEPTH + for (i = 0; i < k; ++i) + pmi->palette_colors[i] = clip_pixel((int)centroids[i]); + pmi->palette_size[0] = k; + + av1_calc_indices(data, centroids, color_map, rows * cols, k, 1); + extend_palette_color_map(color_map, cols, rows, block_width, + block_height); + palette_mode_cost = + dc_mode_cost + + cpi->palette_y_size_cost[bsize - BLOCK_8X8][k - PALETTE_MIN_SIZE] + + write_uniform_cost(k, color_map[0]) + + av1_cost_bit( + av1_default_palette_y_mode_prob[bsize - BLOCK_8X8][palette_ctx], + 1); + palette_mode_cost += av1_palette_color_cost_y(pmi, cpi->common.bit_depth); + for (i = 0; i < rows; ++i) { + for (j = (i == 0 ? 1 : 0); j < cols; ++j) { + int color_idx; + const int color_ctx = av1_get_palette_color_index_context( + color_map, block_width, i, j, k, color_order, &color_idx); + assert(color_idx >= 0 && color_idx < k); + palette_mode_cost += cpi->palette_y_color_cost[k - PALETTE_MIN_SIZE] + [color_ctx][color_idx]; + } + } + this_model_rd = intra_model_yrd(cpi, x, bsize, palette_mode_cost); + if (*best_model_rd != INT64_MAX && + this_model_rd > *best_model_rd + (*best_model_rd >> 1)) + continue; + if (this_model_rd < *best_model_rd) *best_model_rd = this_model_rd; + super_block_yrd(cpi, x, &tokenonly_rd_stats, bsize, *best_rd); + if (tokenonly_rd_stats.rate == INT_MAX) continue; + this_rate = tokenonly_rd_stats.rate + palette_mode_cost; + this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, tokenonly_rd_stats.dist); + if (!xd->lossless[mbmi->segment_id] && mbmi->sb_type >= BLOCK_8X8) { + tokenonly_rd_stats.rate -= tx_size_cost(cpi, x, bsize, mbmi->tx_size); + } + if (this_rd < *best_rd) { + *best_rd = this_rd; + memcpy(best_palette_color_map, color_map, + block_width * block_height * sizeof(color_map[0])); + *best_mbmi = *mbmi; + rate_overhead = this_rate - tokenonly_rd_stats.rate; + if (rate) *rate = this_rate; + if (rate_tokenonly) *rate_tokenonly = tokenonly_rd_stats.rate; + if (distortion) *distortion = tokenonly_rd_stats.dist; + if (skippable) *skippable = tokenonly_rd_stats.skip; + } + } + } + + if (best_mbmi->palette_mode_info.palette_size[0] > 0) { + memcpy(color_map, best_palette_color_map, + rows * cols * sizeof(best_palette_color_map[0])); + } + *mbmi = *best_mbmi; + return rate_overhead; +} +#endif // CONFIG_PALETTE + +static int64_t rd_pick_intra_sub_8x8_y_subblock_mode( + const AV1_COMP *const cpi, MACROBLOCK *x, int row, int col, + PREDICTION_MODE *best_mode, const int *bmode_costs, ENTROPY_CONTEXT *a, + ENTROPY_CONTEXT *l, int *bestrate, int *bestratey, int64_t *bestdistortion, + BLOCK_SIZE bsize, TX_SIZE tx_size, int *y_skip, int64_t rd_thresh) { + const AV1_COMMON *const cm = &cpi->common; + PREDICTION_MODE mode; + MACROBLOCKD *const xd = &x->e_mbd; + int64_t best_rd = rd_thresh; + struct macroblock_plane *p = &x->plane[0]; + struct macroblockd_plane *pd = &xd->plane[0]; + const int src_stride = p->src.stride; + const int dst_stride = pd->dst.stride; + const uint8_t *src_init = &p->src.buf[row * 4 * src_stride + col * 4]; + uint8_t *dst_init = &pd->dst.buf[row * 4 * dst_stride + col * 4]; +#if CONFIG_CB4X4 + // TODO(jingning): This is a temporal change. The whole function should be + // out when cb4x4 is enabled. + ENTROPY_CONTEXT ta[4], tempa[4]; + ENTROPY_CONTEXT tl[4], templ[4]; +#else + ENTROPY_CONTEXT ta[2], tempa[2]; + ENTROPY_CONTEXT tl[2], templ[2]; +#endif // CONFIG_CB4X4 + + const int pred_width_in_4x4_blocks = num_4x4_blocks_wide_lookup[bsize]; + const int pred_height_in_4x4_blocks = num_4x4_blocks_high_lookup[bsize]; + const int tx_width_unit = tx_size_wide_unit[tx_size]; + const int tx_height_unit = tx_size_high_unit[tx_size]; + const int pred_block_width = block_size_wide[bsize]; + const int pred_block_height = block_size_high[bsize]; + const int tx_width = tx_size_wide[tx_size]; + const int tx_height = tx_size_high[tx_size]; + const int pred_width_in_transform_blocks = pred_block_width / tx_width; + const int pred_height_in_transform_blocks = pred_block_height / tx_height; + int idx, idy; + int best_can_skip = 0; + uint8_t best_dst[8 * 8]; +#if CONFIG_HIGHBITDEPTH + uint16_t best_dst16[8 * 8]; +#endif // CONFIG_HIGHBITDEPTH + const int is_lossless = xd->lossless[xd->mi[0]->mbmi.segment_id]; +#if CONFIG_EXT_TX && CONFIG_RECT_TX + const int sub_bsize = bsize; +#else + const int sub_bsize = BLOCK_4X4; +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + +#if CONFIG_PVQ + od_rollback_buffer pre_buf, post_buf; + od_encode_checkpoint(&x->daala_enc, &pre_buf); + od_encode_checkpoint(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + + assert(bsize < BLOCK_8X8); + assert(tx_width < 8 || tx_height < 8); +#if CONFIG_EXT_TX && CONFIG_RECT_TX + if (is_lossless) + assert(tx_width == 4 && tx_height == 4); + else + assert(tx_width == pred_block_width && tx_height == pred_block_height); +#else + assert(tx_width == 4 && tx_height == 4); +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + + memcpy(ta, a, pred_width_in_transform_blocks * sizeof(a[0])); + memcpy(tl, l, pred_height_in_transform_blocks * sizeof(l[0])); + + xd->mi[0]->mbmi.tx_size = tx_size; + +#if CONFIG_PALETTE + xd->mi[0]->mbmi.palette_mode_info.palette_size[0] = 0; +#endif // CONFIG_PALETTE + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { +#if CONFIG_PVQ + od_encode_checkpoint(&x->daala_enc, &pre_buf); +#endif + for (mode = DC_PRED; mode <= TM_PRED; ++mode) { + int64_t this_rd; + int ratey = 0; + int64_t distortion = 0; + int rate = bmode_costs[mode]; + int can_skip = 1; + + if (!(cpi->sf.intra_y_mode_mask[txsize_sqr_up_map[tx_size]] & + (1 << mode))) + continue; + + // Only do the oblique modes if the best so far is + // one of the neighboring directional modes + if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) { + if (conditional_skipintra(mode, *best_mode)) continue; + } + + memcpy(tempa, ta, pred_width_in_transform_blocks * sizeof(ta[0])); + memcpy(templ, tl, pred_height_in_transform_blocks * sizeof(tl[0])); + + for (idy = 0; idy < pred_height_in_transform_blocks; ++idy) { + for (idx = 0; idx < pred_width_in_transform_blocks; ++idx) { + const int block_raster_idx = (row + idy) * 2 + (col + idx); + const int block = + av1_raster_order_to_block_index(tx_size, block_raster_idx); + const uint8_t *const src = &src_init[idx * 4 + idy * 4 * src_stride]; + uint8_t *const dst = &dst_init[idx * 4 + idy * 4 * dst_stride]; +#if !CONFIG_PVQ + int16_t *const src_diff = av1_raster_block_offset_int16( + BLOCK_8X8, block_raster_idx, p->src_diff); +#endif + int skip; + assert(block < 4); + assert(IMPLIES(tx_size == TX_4X8 || tx_size == TX_8X4, + idx == 0 && idy == 0)); + assert(IMPLIES(tx_size == TX_4X8 || tx_size == TX_8X4, + block == 0 || block == 2)); + xd->mi[0]->bmi[block_raster_idx].as_mode = mode; + av1_predict_intra_block( + xd, pd->width, pd->height, txsize_to_bsize[tx_size], mode, dst, + dst_stride, dst, dst_stride, col + idx, row + idy, 0); +#if !CONFIG_PVQ + aom_highbd_subtract_block(tx_height, tx_width, src_diff, 8, src, + src_stride, dst, dst_stride, xd->bd); +#endif + if (is_lossless) { + TX_TYPE tx_type = get_tx_type(PLANE_TYPE_Y, xd, block, tx_size); + const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 0); + const int coeff_ctx = + combine_entropy_contexts(tempa[idx], templ[idy]); +#if !CONFIG_PVQ + av1_xform_quant(cm, x, 0, block, row + idy, col + idx, BLOCK_8X8, + tx_size, coeff_ctx, AV1_XFORM_QUANT_FP); + ratey += av1_cost_coeffs(cpi, x, 0, block, tx_size, scan_order, + tempa + idx, templ + idy, + cpi->sf.use_fast_coef_costing); + skip = (p->eobs[block] == 0); + can_skip &= skip; + tempa[idx] = !skip; + templ[idy] = !skip; +#if CONFIG_EXT_TX + if (tx_size == TX_8X4) { + tempa[idx + 1] = tempa[idx]; + } else if (tx_size == TX_4X8) { + templ[idy + 1] = templ[idy]; + } +#endif // CONFIG_EXT_TX +#else + (void)scan_order; + + av1_xform_quant(cm, x, 0, block, row + idy, col + idx, BLOCK_8X8, + tx_size, coeff_ctx, AV1_XFORM_QUANT_B); + + ratey += x->rate; + skip = x->pvq_skip[0]; + tempa[idx] = !skip; + templ[idy] = !skip; + can_skip &= skip; +#endif + if (RDCOST(x->rdmult, x->rddiv, ratey, distortion) >= best_rd) + goto next_highbd; +#if CONFIG_PVQ + if (!skip) +#endif + av1_inverse_transform_block(xd, BLOCK_OFFSET(pd->dqcoeff, block), + DCT_DCT, tx_size, dst, dst_stride, + p->eobs[block]); + } else { + int64_t dist; + unsigned int tmp; + TX_TYPE tx_type = get_tx_type(PLANE_TYPE_Y, xd, block, tx_size); + const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 0); + const int coeff_ctx = + combine_entropy_contexts(tempa[idx], templ[idy]); +#if !CONFIG_PVQ + av1_xform_quant(cm, x, 0, block, row + idy, col + idx, BLOCK_8X8, + tx_size, coeff_ctx, AV1_XFORM_QUANT_FP); + av1_optimize_b(cm, x, 0, block, tx_size, coeff_ctx); + ratey += av1_cost_coeffs(cpi, x, 0, block, tx_size, scan_order, + tempa + idx, templ + idy, + cpi->sf.use_fast_coef_costing); + skip = (p->eobs[block] == 0); + can_skip &= skip; + tempa[idx] = !skip; + templ[idy] = !skip; +#if CONFIG_EXT_TX + if (tx_size == TX_8X4) { + tempa[idx + 1] = tempa[idx]; + } else if (tx_size == TX_4X8) { + templ[idy + 1] = templ[idy]; + } +#endif // CONFIG_EXT_TX +#else + (void)scan_order; + + av1_xform_quant(cm, x, 0, block, row + idy, col + idx, BLOCK_8X8, + tx_size, coeff_ctx, AV1_XFORM_QUANT_FP); + ratey += x->rate; + skip = x->pvq_skip[0]; + tempa[idx] = !skip; + templ[idy] = !skip; + can_skip &= skip; +#endif +#if CONFIG_PVQ + if (!skip) +#endif + av1_inverse_transform_block(xd, BLOCK_OFFSET(pd->dqcoeff, block), + tx_type, tx_size, dst, dst_stride, + p->eobs[block]); + cpi->fn_ptr[sub_bsize].vf(src, src_stride, dst, dst_stride, &tmp); + dist = (int64_t)tmp << 4; + distortion += dist; + if (RDCOST(x->rdmult, x->rddiv, ratey, distortion) >= best_rd) + goto next_highbd; + } + } + } + + rate += ratey; + this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion); + + if (this_rd < best_rd) { + *bestrate = rate; + *bestratey = ratey; + *bestdistortion = distortion; + best_rd = this_rd; + best_can_skip = can_skip; + *best_mode = mode; + memcpy(a, tempa, pred_width_in_transform_blocks * sizeof(tempa[0])); + memcpy(l, templ, pred_height_in_transform_blocks * sizeof(templ[0])); +#if CONFIG_PVQ + od_encode_checkpoint(&x->daala_enc, &post_buf); +#endif + for (idy = 0; idy < pred_height_in_transform_blocks * 4; ++idy) { + memcpy(best_dst16 + idy * 8, + CONVERT_TO_SHORTPTR(dst_init + idy * dst_stride), + pred_width_in_transform_blocks * 4 * sizeof(uint16_t)); + } + } + next_highbd : {} +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &pre_buf); +#endif + } + + if (best_rd >= rd_thresh) return best_rd; + +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &post_buf); +#endif + + if (y_skip) *y_skip &= best_can_skip; + + for (idy = 0; idy < pred_height_in_transform_blocks * 4; ++idy) { + memcpy(CONVERT_TO_SHORTPTR(dst_init + idy * dst_stride), + best_dst16 + idy * 8, + pred_width_in_transform_blocks * 4 * sizeof(uint16_t)); + } + + return best_rd; + } +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_PVQ + od_encode_checkpoint(&x->daala_enc, &pre_buf); +#endif // CONFIG_PVQ + + for (mode = DC_PRED; mode <= TM_PRED; ++mode) { + int64_t this_rd; + int ratey = 0; + int64_t distortion = 0; + int rate = bmode_costs[mode]; + int can_skip = 1; + + if (!(cpi->sf.intra_y_mode_mask[txsize_sqr_up_map[tx_size]] & + (1 << mode))) { + continue; + } + + // Only do the oblique modes if the best so far is + // one of the neighboring directional modes + if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) { + if (conditional_skipintra(mode, *best_mode)) continue; + } + + memcpy(tempa, ta, pred_width_in_transform_blocks * sizeof(ta[0])); + memcpy(templ, tl, pred_height_in_transform_blocks * sizeof(tl[0])); + + for (idy = 0; idy < pred_height_in_4x4_blocks; idy += tx_height_unit) { + for (idx = 0; idx < pred_width_in_4x4_blocks; idx += tx_width_unit) { + const int block_raster_idx = (row + idy) * 2 + (col + idx); + int block = av1_raster_order_to_block_index(tx_size, block_raster_idx); + const uint8_t *const src = &src_init[idx * 4 + idy * 4 * src_stride]; + uint8_t *const dst = &dst_init[idx * 4 + idy * 4 * dst_stride]; +#if !CONFIG_PVQ + int16_t *const src_diff = av1_raster_block_offset_int16( + BLOCK_8X8, block_raster_idx, p->src_diff); +#endif // !CONFIG_PVQ + int skip; + assert(block < 4); + assert(IMPLIES(tx_size == TX_4X8 || tx_size == TX_8X4, + idx == 0 && idy == 0)); + assert(IMPLIES(tx_size == TX_4X8 || tx_size == TX_8X4, + block == 0 || block == 2)); + xd->mi[0]->bmi[block_raster_idx].as_mode = mode; + av1_predict_intra_block(xd, pd->width, pd->height, + txsize_to_bsize[tx_size], mode, dst, dst_stride, + dst, dst_stride, +#if CONFIG_CB4X4 + 2 * (col + idx), 2 * (row + idy), +#else + col + idx, row + idy, +#endif // CONFIG_CB4X4 + 0); +#if !CONFIG_PVQ + aom_subtract_block(tx_height, tx_width, src_diff, 8, src, src_stride, + dst, dst_stride); +#endif // !CONFIG_PVQ + + TX_TYPE tx_type = get_tx_type(PLANE_TYPE_Y, xd, block, tx_size); + const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 0); + const int coeff_ctx = combine_entropy_contexts(tempa[idx], templ[idy]); +#if CONFIG_CB4X4 + block = 4 * block; +#endif // CONFIG_CB4X4 +#if !CONFIG_PVQ + const AV1_XFORM_QUANT xform_quant = + is_lossless ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP; + av1_xform_quant(cm, x, 0, block, +#if CONFIG_CB4X4 + 2 * (row + idy), 2 * (col + idx), +#else + row + idy, col + idx, +#endif // CONFIG_CB4X4 + BLOCK_8X8, tx_size, coeff_ctx, xform_quant); + + if (!is_lossless) { + av1_optimize_b(cm, x, 0, block, tx_size, coeff_ctx); + } + + ratey += + av1_cost_coeffs(cpi, x, 0, block, tx_size, scan_order, tempa + idx, + templ + idy, cpi->sf.use_fast_coef_costing); + skip = (p->eobs[block] == 0); + can_skip &= skip; + tempa[idx] = !skip; + templ[idy] = !skip; +#if CONFIG_EXT_TX + if (tx_size == TX_8X4) { + tempa[idx + 1] = tempa[idx]; + } else if (tx_size == TX_4X8) { + templ[idy + 1] = templ[idy]; + } +#endif // CONFIG_EXT_TX +#else + (void)scan_order; + + av1_xform_quant(cm, x, 0, block, +#if CONFIG_CB4X4 + 2 * (row + idy), 2 * (col + idx), +#else + row + idy, col + idx, +#endif // CONFIG_CB4X4 + BLOCK_8X8, tx_size, coeff_ctx, AV1_XFORM_QUANT_FP); + + ratey += x->rate; + skip = x->pvq_skip[0]; + tempa[idx] = !skip; + templ[idy] = !skip; + can_skip &= skip; +#endif // !CONFIG_PVQ + + if (!is_lossless) { // To use the pixel domain distortion, we need to + // calculate inverse txfm *before* calculating RD + // cost. Compared to calculating the distortion in + // the frequency domain, the overhead of encoding + // effort is low. +#if CONFIG_PVQ + if (!skip) +#endif // CONFIG_PVQ + av1_inverse_transform_block(xd, BLOCK_OFFSET(pd->dqcoeff, block), + tx_type, tx_size, dst, dst_stride, + p->eobs[block]); + unsigned int tmp; + cpi->fn_ptr[sub_bsize].vf(src, src_stride, dst, dst_stride, &tmp); + const int64_t dist = (int64_t)tmp << 4; + distortion += dist; + } + + if (RDCOST(x->rdmult, x->rddiv, ratey, distortion) >= best_rd) + goto next; + + if (is_lossless) { // Calculate inverse txfm *after* RD cost. +#if CONFIG_PVQ + if (!skip) +#endif // CONFIG_PVQ + av1_inverse_transform_block(xd, BLOCK_OFFSET(pd->dqcoeff, block), + DCT_DCT, tx_size, dst, dst_stride, + p->eobs[block]); + } + } + } + + rate += ratey; + this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion); + + if (this_rd < best_rd) { + *bestrate = rate; + *bestratey = ratey; + *bestdistortion = distortion; + best_rd = this_rd; + best_can_skip = can_skip; + *best_mode = mode; + memcpy(a, tempa, pred_width_in_transform_blocks * sizeof(tempa[0])); + memcpy(l, templ, pred_height_in_transform_blocks * sizeof(templ[0])); +#if CONFIG_PVQ + od_encode_checkpoint(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + for (idy = 0; idy < pred_height_in_transform_blocks * 4; ++idy) + memcpy(best_dst + idy * 8, dst_init + idy * dst_stride, + pred_width_in_transform_blocks * 4); + } + next : {} +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &pre_buf); +#endif // CONFIG_PVQ + } // mode decision loop + + if (best_rd >= rd_thresh) return best_rd; + +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + + if (y_skip) *y_skip &= best_can_skip; + + for (idy = 0; idy < pred_height_in_transform_blocks * 4; ++idy) + memcpy(dst_init + idy * dst_stride, best_dst + idy * 8, + pred_width_in_transform_blocks * 4); + + return best_rd; +} + +static int64_t rd_pick_intra_sub_8x8_y_mode(const AV1_COMP *const cpi, + MACROBLOCK *mb, int *rate, + int *rate_y, int64_t *distortion, + int *y_skip, int64_t best_rd) { + const MACROBLOCKD *const xd = &mb->e_mbd; + MODE_INFO *const mic = xd->mi[0]; + const MODE_INFO *above_mi = xd->above_mi; + const MODE_INFO *left_mi = xd->left_mi; + MB_MODE_INFO *const mbmi = &mic->mbmi; + const BLOCK_SIZE bsize = mbmi->sb_type; + const int pred_width_in_4x4_blocks = num_4x4_blocks_wide_lookup[bsize]; + const int pred_height_in_4x4_blocks = num_4x4_blocks_high_lookup[bsize]; + int idx, idy; + int cost = 0; + int64_t total_distortion = 0; + int tot_rate_y = 0; + int64_t total_rd = 0; + const int *bmode_costs = cpi->mbmode_cost[0]; + const int is_lossless = xd->lossless[mbmi->segment_id]; +#if CONFIG_EXT_TX && CONFIG_RECT_TX + const TX_SIZE tx_size = is_lossless ? TX_4X4 : max_txsize_rect_lookup[bsize]; +#else + const TX_SIZE tx_size = TX_4X4; +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + +#if CONFIG_EXT_INTRA +#if CONFIG_INTRA_INTERP + mbmi->intra_filter = INTRA_FILTER_LINEAR; +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0; +#endif // CONFIG_FILTER_INTRA + + // TODO(any): Add search of the tx_type to improve rd performance at the + // expense of speed. + mbmi->tx_type = DCT_DCT; + mbmi->tx_size = tx_size; + + if (y_skip) *y_skip = 1; + + // Pick modes for each prediction sub-block (of size 4x4, 4x8, or 8x4) in this + // 8x8 coding block. + for (idy = 0; idy < 2; idy += pred_height_in_4x4_blocks) { + for (idx = 0; idx < 2; idx += pred_width_in_4x4_blocks) { + PREDICTION_MODE best_mode = DC_PRED; + int r = INT_MAX, ry = INT_MAX; + int64_t d = INT64_MAX, this_rd = INT64_MAX; + int j; + const int pred_block_idx = idy * 2 + idx; + if (cpi->common.frame_type == KEY_FRAME) { + const PREDICTION_MODE A = + av1_above_block_mode(mic, above_mi, pred_block_idx); + const PREDICTION_MODE L = + av1_left_block_mode(mic, left_mi, pred_block_idx); + + bmode_costs = cpi->y_mode_costs[A][L]; + } + this_rd = rd_pick_intra_sub_8x8_y_subblock_mode( + cpi, mb, idy, idx, &best_mode, bmode_costs, + xd->plane[0].above_context + idx, xd->plane[0].left_context + idy, &r, + &ry, &d, bsize, tx_size, y_skip, best_rd - total_rd); +#if !CONFIG_DAALA_DIST + if (this_rd >= best_rd - total_rd) return INT64_MAX; +#endif // !CONFIG_DAALA_DIST + total_rd += this_rd; + cost += r; + total_distortion += d; + tot_rate_y += ry; + + mic->bmi[pred_block_idx].as_mode = best_mode; + for (j = 1; j < pred_height_in_4x4_blocks; ++j) + mic->bmi[pred_block_idx + j * 2].as_mode = best_mode; + for (j = 1; j < pred_width_in_4x4_blocks; ++j) + mic->bmi[pred_block_idx + j].as_mode = best_mode; + + if (total_rd >= best_rd) return INT64_MAX; + } + } + mbmi->mode = mic->bmi[3].as_mode; + +#if CONFIG_DAALA_DIST + { + const struct macroblock_plane *p = &mb->plane[0]; + const struct macroblockd_plane *pd = &xd->plane[0]; + const int src_stride = p->src.stride; + const int dst_stride = pd->dst.stride; + uint8_t *src = p->src.buf; + uint8_t *dst = pd->dst.buf; + int use_activity_masking = 0; + int qm = OD_HVS_QM; + +#if CONFIG_PVQ + use_activity_masking = mb->daala_enc.use_activity_masking; +#endif // CONFIG_PVQ + // Daala-defined distortion computed for the block of 8x8 pixels + total_distortion = av1_daala_dist(src, src_stride, dst, dst_stride, 8, 8, + qm, use_activity_masking, mb->qindex) + << 4; + } +#endif // CONFIG_DAALA_DIST + // Add in the cost of the transform type + if (!is_lossless) { + int rate_tx_type = 0; +#if CONFIG_EXT_TX + if (get_ext_tx_types(tx_size, bsize, 0, cpi->common.reduced_tx_set_used) > + 1) { + const int eset = + get_ext_tx_set(tx_size, bsize, 0, cpi->common.reduced_tx_set_used); + rate_tx_type = cpi->intra_tx_type_costs[eset][txsize_sqr_map[tx_size]] + [mbmi->mode][mbmi->tx_type]; + } +#else + rate_tx_type = + cpi->intra_tx_type_costs[txsize_sqr_map[tx_size]] + [intra_mode_to_tx_type_context[mbmi->mode]] + [mbmi->tx_type]; +#endif // CONFIG_EXT_TX + assert(mbmi->tx_size == tx_size); + cost += rate_tx_type; + tot_rate_y += rate_tx_type; + } + + *rate = cost; + *rate_y = tot_rate_y; + *distortion = total_distortion; + + return RDCOST(mb->rdmult, mb->rddiv, cost, total_distortion); +} + +#if CONFIG_FILTER_INTRA +// Return 1 if an filter intra mode is selected; return 0 otherwise. +static int rd_pick_filter_intra_sby(const AV1_COMP *const cpi, MACROBLOCK *x, + int *rate, int *rate_tokenonly, + int64_t *distortion, int *skippable, + BLOCK_SIZE bsize, int mode_cost, + int64_t *best_rd, int64_t *best_model_rd, + uint16_t skip_mask) { + MACROBLOCKD *const xd = &x->e_mbd; + MODE_INFO *const mic = xd->mi[0]; + MB_MODE_INFO *mbmi = &mic->mbmi; + int filter_intra_selected_flag = 0; + FILTER_INTRA_MODE mode; + TX_SIZE best_tx_size = TX_4X4; + FILTER_INTRA_MODE_INFO filter_intra_mode_info; + TX_TYPE best_tx_type; + + av1_zero(filter_intra_mode_info); + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 1; + mbmi->mode = DC_PRED; +#if CONFIG_PALETTE + mbmi->palette_mode_info.palette_size[0] = 0; +#endif // CONFIG_PALETTE + + for (mode = 0; mode < FILTER_INTRA_MODES; ++mode) { + int this_rate; + int64_t this_rd, this_model_rd; + RD_STATS tokenonly_rd_stats; + if (skip_mask & (1 << mode)) continue; + mbmi->filter_intra_mode_info.filter_intra_mode[0] = mode; + this_model_rd = intra_model_yrd(cpi, x, bsize, mode_cost); + if (*best_model_rd != INT64_MAX && + this_model_rd > *best_model_rd + (*best_model_rd >> 1)) + continue; + if (this_model_rd < *best_model_rd) *best_model_rd = this_model_rd; + super_block_yrd(cpi, x, &tokenonly_rd_stats, bsize, *best_rd); + if (tokenonly_rd_stats.rate == INT_MAX) continue; + this_rate = tokenonly_rd_stats.rate + + av1_cost_bit(cpi->common.fc->filter_intra_probs[0], 1) + + write_uniform_cost(FILTER_INTRA_MODES, mode) + mode_cost; + this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, tokenonly_rd_stats.dist); + + if (this_rd < *best_rd) { + *best_rd = this_rd; + best_tx_size = mic->mbmi.tx_size; + filter_intra_mode_info = mbmi->filter_intra_mode_info; + best_tx_type = mic->mbmi.tx_type; + *rate = this_rate; + *rate_tokenonly = tokenonly_rd_stats.rate; + *distortion = tokenonly_rd_stats.dist; + *skippable = tokenonly_rd_stats.skip; + filter_intra_selected_flag = 1; + } + } + + if (filter_intra_selected_flag) { + mbmi->mode = DC_PRED; + mbmi->tx_size = best_tx_size; + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = + filter_intra_mode_info.use_filter_intra_mode[0]; + mbmi->filter_intra_mode_info.filter_intra_mode[0] = + filter_intra_mode_info.filter_intra_mode[0]; + mbmi->tx_type = best_tx_type; + return 1; + } else { + return 0; + } +} +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_EXT_INTRA +// Run RD calculation with given luma intra prediction angle., and return +// the RD cost. Update the best mode info. if the RD cost is the best so far. +static int64_t calc_rd_given_intra_angle( + const AV1_COMP *const cpi, MACROBLOCK *x, BLOCK_SIZE bsize, int mode_cost, + int64_t best_rd_in, int8_t angle_delta, int max_angle_delta, int *rate, + RD_STATS *rd_stats, int *best_angle_delta, TX_SIZE *best_tx_size, + TX_TYPE *best_tx_type, +#if CONFIG_INTRA_INTERP + INTRA_FILTER *best_filter, +#endif // CONFIG_INTRA_INTERP + int64_t *best_rd, int64_t *best_model_rd) { + int this_rate; + RD_STATS tokenonly_rd_stats; + int64_t this_rd, this_model_rd; + MB_MODE_INFO *mbmi = &x->e_mbd.mi[0]->mbmi; + + mbmi->angle_delta[0] = angle_delta; + this_model_rd = intra_model_yrd(cpi, x, bsize, mode_cost); + if (*best_model_rd != INT64_MAX && + this_model_rd > *best_model_rd + (*best_model_rd >> 1)) + return INT64_MAX; + if (this_model_rd < *best_model_rd) *best_model_rd = this_model_rd; + super_block_yrd(cpi, x, &tokenonly_rd_stats, bsize, best_rd_in); + if (tokenonly_rd_stats.rate == INT_MAX) return INT64_MAX; + + this_rate = tokenonly_rd_stats.rate + mode_cost + + write_uniform_cost(2 * max_angle_delta + 1, + mbmi->angle_delta[0] + max_angle_delta); + this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, tokenonly_rd_stats.dist); + + if (this_rd < *best_rd) { + *best_rd = this_rd; + *best_angle_delta = mbmi->angle_delta[0]; + *best_tx_size = mbmi->tx_size; +#if CONFIG_INTRA_INTERP + *best_filter = mbmi->intra_filter; +#endif // CONFIG_INTRA_INTERP + *best_tx_type = mbmi->tx_type; + *rate = this_rate; + rd_stats->rate = tokenonly_rd_stats.rate; + rd_stats->dist = tokenonly_rd_stats.dist; + rd_stats->skip = tokenonly_rd_stats.skip; + } + return this_rd; +} + +// With given luma directional intra prediction mode, pick the best angle delta +// Return the RD cost corresponding to the best angle delta. +static int64_t rd_pick_intra_angle_sby(const AV1_COMP *const cpi, MACROBLOCK *x, + int *rate, RD_STATS *rd_stats, + BLOCK_SIZE bsize, int mode_cost, + int64_t best_rd, + int64_t *best_model_rd) { + MACROBLOCKD *const xd = &x->e_mbd; + MODE_INFO *const mic = xd->mi[0]; + MB_MODE_INFO *mbmi = &mic->mbmi; + int i, angle_delta, best_angle_delta = 0; + int first_try = 1; +#if CONFIG_INTRA_INTERP + int p_angle; + const int intra_filter_ctx = av1_get_pred_context_intra_interp(xd); + INTRA_FILTER filter, best_filter = INTRA_FILTER_LINEAR; +#endif // CONFIG_INTRA_INTERP + int64_t this_rd, best_rd_in, rd_cost[2 * (MAX_ANGLE_DELTA + 2)]; + TX_SIZE best_tx_size = mic->mbmi.tx_size; + TX_TYPE best_tx_type = mbmi->tx_type; + + for (i = 0; i < 2 * (MAX_ANGLE_DELTA + 2); ++i) rd_cost[i] = INT64_MAX; + + for (angle_delta = 0; angle_delta <= MAX_ANGLE_DELTA; angle_delta += 2) { +#if CONFIG_INTRA_INTERP + for (filter = INTRA_FILTER_LINEAR; filter < INTRA_FILTERS; ++filter) { + if (FILTER_FAST_SEARCH && filter != INTRA_FILTER_LINEAR) continue; + mic->mbmi.intra_filter = filter; +#endif // CONFIG_INTRA_INTERP + for (i = 0; i < 2; ++i) { + best_rd_in = (best_rd == INT64_MAX) + ? INT64_MAX + : (best_rd + (best_rd >> (first_try ? 3 : 5))); + this_rd = calc_rd_given_intra_angle( + cpi, x, bsize, +#if CONFIG_INTRA_INTERP + mode_cost + cpi->intra_filter_cost[intra_filter_ctx][filter], +#else + mode_cost, +#endif // CONFIG_INTRA_INTERP + best_rd_in, (1 - 2 * i) * angle_delta, MAX_ANGLE_DELTA, rate, + rd_stats, &best_angle_delta, &best_tx_size, &best_tx_type, +#if CONFIG_INTRA_INTERP + &best_filter, +#endif // CONFIG_INTRA_INTERP + &best_rd, best_model_rd); + rd_cost[2 * angle_delta + i] = this_rd; + if (first_try && this_rd == INT64_MAX) return best_rd; + first_try = 0; + if (angle_delta == 0) { + rd_cost[1] = this_rd; + break; + } + } +#if CONFIG_INTRA_INTERP + } +#endif // CONFIG_INTRA_INTERP + } + + assert(best_rd != INT64_MAX); + for (angle_delta = 1; angle_delta <= MAX_ANGLE_DELTA; angle_delta += 2) { + int64_t rd_thresh; +#if CONFIG_INTRA_INTERP + for (filter = INTRA_FILTER_LINEAR; filter < INTRA_FILTERS; ++filter) { + if (FILTER_FAST_SEARCH && filter != INTRA_FILTER_LINEAR) continue; + mic->mbmi.intra_filter = filter; +#endif // CONFIG_INTRA_INTERP + for (i = 0; i < 2; ++i) { + int skip_search = 0; + rd_thresh = best_rd + (best_rd >> 5); + if (rd_cost[2 * (angle_delta + 1) + i] > rd_thresh && + rd_cost[2 * (angle_delta - 1) + i] > rd_thresh) + skip_search = 1; + if (!skip_search) { + calc_rd_given_intra_angle( + cpi, x, bsize, +#if CONFIG_INTRA_INTERP + mode_cost + cpi->intra_filter_cost[intra_filter_ctx][filter], +#else + mode_cost, +#endif // CONFIG_INTRA_INTERP + best_rd, (1 - 2 * i) * angle_delta, MAX_ANGLE_DELTA, rate, + rd_stats, &best_angle_delta, &best_tx_size, &best_tx_type, +#if CONFIG_INTRA_INTERP + &best_filter, +#endif // CONFIG_INTRA_INTERP + &best_rd, best_model_rd); + } + } +#if CONFIG_INTRA_INTERP + } +#endif // CONFIG_INTRA_INTERP + } + +#if CONFIG_INTRA_INTERP + if (FILTER_FAST_SEARCH && rd_stats->rate < INT_MAX) { + p_angle = mode_to_angle_map[mbmi->mode] + best_angle_delta * ANGLE_STEP; + if (av1_is_intra_filter_switchable(p_angle)) { + for (filter = INTRA_FILTER_LINEAR + 1; filter < INTRA_FILTERS; ++filter) { + mic->mbmi.intra_filter = filter; + this_rd = calc_rd_given_intra_angle( + cpi, x, bsize, + mode_cost + cpi->intra_filter_cost[intra_filter_ctx][filter], + best_rd, best_angle_delta, MAX_ANGLE_DELTA, rate, rd_stats, + &best_angle_delta, &best_tx_size, &best_tx_type, &best_filter, + &best_rd, best_model_rd); + } + } + } +#endif // CONFIG_INTRA_INTERP + + mbmi->tx_size = best_tx_size; + mbmi->angle_delta[0] = best_angle_delta; +#if CONFIG_INTRA_INTERP + mic->mbmi.intra_filter = best_filter; +#endif // CONFIG_INTRA_INTERP + mbmi->tx_type = best_tx_type; + return best_rd; +} + +// Indices are sign, integer, and fractional part of the gradient value +static const uint8_t gradient_to_angle_bin[2][7][16] = { + { + { 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1 }, + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, + { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }, + { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }, + }, + { + { 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4 }, + { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3 }, + { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }, + { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }, + { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }, + { 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2 }, + { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }, + }, +}; + +static const uint8_t mode_to_angle_bin[INTRA_MODES] = { + 0, 2, 6, 0, 4, 3, 5, 7, 1, 0, +}; + +static void angle_estimation(const uint8_t *src, int src_stride, int rows, + int cols, uint8_t *directional_mode_skip_mask) { + int i, r, c, index, dx, dy, temp, sn, remd, quot; + uint64_t hist[DIRECTIONAL_MODES]; + uint64_t hist_sum = 0; + + memset(hist, 0, DIRECTIONAL_MODES * sizeof(hist[0])); + src += src_stride; + for (r = 1; r < rows; ++r) { + for (c = 1; c < cols; ++c) { + dx = src[c] - src[c - 1]; + dy = src[c] - src[c - src_stride]; + temp = dx * dx + dy * dy; + if (dy == 0) { + index = 2; + } else { + sn = (dx > 0) ^ (dy > 0); + dx = abs(dx); + dy = abs(dy); + remd = dx % dy; + quot = dx / dy; + remd = remd * 16 / dy; + index = gradient_to_angle_bin[sn][AOMMIN(quot, 6)][AOMMIN(remd, 15)]; + } + hist[index] += temp; + } + src += src_stride; + } + + for (i = 0; i < DIRECTIONAL_MODES; ++i) hist_sum += hist[i]; + for (i = 0; i < INTRA_MODES; ++i) { + if (i != DC_PRED && i != TM_PRED) { + const uint8_t angle_bin = mode_to_angle_bin[i]; + uint64_t score = 2 * hist[angle_bin]; + int weight = 2; + if (angle_bin > 0) { + score += hist[angle_bin - 1]; + ++weight; + } + if (angle_bin < DIRECTIONAL_MODES - 1) { + score += hist[angle_bin + 1]; + ++weight; + } + if (score * ANGLE_SKIP_THRESH < hist_sum * weight) + directional_mode_skip_mask[i] = 1; + } + } +} + +#if CONFIG_HIGHBITDEPTH +static void highbd_angle_estimation(const uint8_t *src8, int src_stride, + int rows, int cols, + uint8_t *directional_mode_skip_mask) { + int i, r, c, index, dx, dy, temp, sn, remd, quot; + uint64_t hist[DIRECTIONAL_MODES]; + uint64_t hist_sum = 0; + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + + memset(hist, 0, DIRECTIONAL_MODES * sizeof(hist[0])); + src += src_stride; + for (r = 1; r < rows; ++r) { + for (c = 1; c < cols; ++c) { + dx = src[c] - src[c - 1]; + dy = src[c] - src[c - src_stride]; + temp = dx * dx + dy * dy; + if (dy == 0) { + index = 2; + } else { + sn = (dx > 0) ^ (dy > 0); + dx = abs(dx); + dy = abs(dy); + remd = dx % dy; + quot = dx / dy; + remd = remd * 16 / dy; + index = gradient_to_angle_bin[sn][AOMMIN(quot, 6)][AOMMIN(remd, 15)]; + } + hist[index] += temp; + } + src += src_stride; + } + + for (i = 0; i < DIRECTIONAL_MODES; ++i) hist_sum += hist[i]; + for (i = 0; i < INTRA_MODES; ++i) { + if (i != DC_PRED && i != TM_PRED) { + const uint8_t angle_bin = mode_to_angle_bin[i]; + uint64_t score = 2 * hist[angle_bin]; + int weight = 2; + if (angle_bin > 0) { + score += hist[angle_bin - 1]; + ++weight; + } + if (angle_bin < DIRECTIONAL_MODES - 1) { + score += hist[angle_bin + 1]; + ++weight; + } + if (score * ANGLE_SKIP_THRESH < hist_sum * weight) + directional_mode_skip_mask[i] = 1; + } + } +} +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_EXT_INTRA + +// This function is used only for intra_only frames +static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x, + int *rate, int *rate_tokenonly, + int64_t *distortion, int *skippable, + BLOCK_SIZE bsize, int64_t best_rd) { + uint8_t mode_idx; + MACROBLOCKD *const xd = &x->e_mbd; + MODE_INFO *const mic = xd->mi[0]; + MB_MODE_INFO *const mbmi = &mic->mbmi; + MB_MODE_INFO best_mbmi = *mbmi; + int64_t best_model_rd = INT64_MAX; +#if CONFIG_EXT_INTRA + const int rows = block_size_high[bsize]; + const int cols = block_size_wide[bsize]; +#if CONFIG_INTRA_INTERP + const int intra_filter_ctx = av1_get_pred_context_intra_interp(xd); +#endif // CONFIG_INTRA_INTERP + int is_directional_mode; + uint8_t directional_mode_skip_mask[INTRA_MODES]; + const int src_stride = x->plane[0].src.stride; + const uint8_t *src = x->plane[0].src.buf; +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + int beat_best_rd = 0; + uint16_t filter_intra_mode_skip_mask = (1 << FILTER_INTRA_MODES) - 1; +#endif // CONFIG_FILTER_INTRA + const int *bmode_costs; +#if CONFIG_PALETTE + PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; + uint8_t *best_palette_color_map = + cpi->common.allow_screen_content_tools + ? x->palette_buffer->best_palette_color_map + : NULL; + int palette_y_mode_ctx = 0; + const int try_palette = + cpi->common.allow_screen_content_tools && bsize >= BLOCK_8X8; +#endif // CONFIG_PALETTE + const MODE_INFO *above_mi = xd->above_mi; + const MODE_INFO *left_mi = xd->left_mi; + const PREDICTION_MODE A = av1_above_block_mode(mic, above_mi, 0); + const PREDICTION_MODE L = av1_left_block_mode(mic, left_mi, 0); + const PREDICTION_MODE FINAL_MODE_SEARCH = TM_PRED + 1; +#if CONFIG_PVQ + od_rollback_buffer pre_buf, post_buf; + + od_encode_checkpoint(&x->daala_enc, &pre_buf); + od_encode_checkpoint(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + bmode_costs = cpi->y_mode_costs[A][L]; + +#if CONFIG_EXT_INTRA + mbmi->angle_delta[0] = 0; + memset(directional_mode_skip_mask, 0, + sizeof(directional_mode_skip_mask[0]) * INTRA_MODES); +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + highbd_angle_estimation(src, src_stride, rows, cols, + directional_mode_skip_mask); + else +#endif // CONFIG_HIGHBITDEPTH + angle_estimation(src, src_stride, rows, cols, directional_mode_skip_mask); +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0; +#endif // CONFIG_FILTER_INTRA +#if CONFIG_PALETTE + pmi->palette_size[0] = 0; + if (above_mi) + palette_y_mode_ctx += + (above_mi->mbmi.palette_mode_info.palette_size[0] > 0); + if (left_mi) + palette_y_mode_ctx += (left_mi->mbmi.palette_mode_info.palette_size[0] > 0); +#endif // CONFIG_PALETTE + + if (cpi->sf.tx_type_search.fast_intra_tx_type_search) + x->use_default_intra_tx_type = 1; + else + x->use_default_intra_tx_type = 0; + + /* Y Search for intra prediction mode */ + for (mode_idx = DC_PRED; mode_idx <= FINAL_MODE_SEARCH; ++mode_idx) { + RD_STATS this_rd_stats; + int this_rate, this_rate_tokenonly, s; + int64_t this_distortion, this_rd, this_model_rd; + if (mode_idx == FINAL_MODE_SEARCH) { + if (x->use_default_intra_tx_type == 0) break; + mbmi->mode = best_mbmi.mode; + x->use_default_intra_tx_type = 0; + } else { + mbmi->mode = mode_idx; + } +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &pre_buf); +#endif // CONFIG_PVQ +#if CONFIG_EXT_INTRA + mbmi->angle_delta[0] = 0; +#endif // CONFIG_EXT_INTRA + this_model_rd = intra_model_yrd(cpi, x, bsize, bmode_costs[mbmi->mode]); + if (best_model_rd != INT64_MAX && + this_model_rd > best_model_rd + (best_model_rd >> 1)) + continue; + if (this_model_rd < best_model_rd) best_model_rd = this_model_rd; +#if CONFIG_EXT_INTRA + is_directional_mode = av1_is_directional_mode(mbmi->mode, bsize); + if (is_directional_mode && directional_mode_skip_mask[mbmi->mode]) continue; + if (is_directional_mode) { + this_rd_stats.rate = INT_MAX; + rd_pick_intra_angle_sby(cpi, x, &this_rate, &this_rd_stats, bsize, + bmode_costs[mbmi->mode], best_rd, &best_model_rd); + } else { + super_block_yrd(cpi, x, &this_rd_stats, bsize, best_rd); + } +#else + super_block_yrd(cpi, x, &this_rd_stats, bsize, best_rd); +#endif // CONFIG_EXT_INTRA + this_rate_tokenonly = this_rd_stats.rate; + this_distortion = this_rd_stats.dist; + s = this_rd_stats.skip; + + if (this_rate_tokenonly == INT_MAX) continue; + + this_rate = this_rate_tokenonly + bmode_costs[mbmi->mode]; + + if (!xd->lossless[mbmi->segment_id] && mbmi->sb_type >= BLOCK_8X8) { + // super_block_yrd above includes the cost of the tx_size in the + // tokenonly rate, but for intra blocks, tx_size is always coded + // (prediction granularity), so we account for it in the full rate, + // not the tokenonly rate. + this_rate_tokenonly -= tx_size_cost(cpi, x, bsize, mbmi->tx_size); + } +#if CONFIG_PALETTE + if (try_palette && mbmi->mode == DC_PRED) { + this_rate += + av1_cost_bit(av1_default_palette_y_mode_prob[bsize - BLOCK_8X8] + [palette_y_mode_ctx], + 0); + } +#endif // CONFIG_PALETTE +#if CONFIG_FILTER_INTRA + if (mbmi->mode == DC_PRED) + this_rate += av1_cost_bit(cpi->common.fc->filter_intra_probs[0], 0); +#endif // CONFIG_FILTER_INTRA +#if CONFIG_EXT_INTRA + if (is_directional_mode) { +#if CONFIG_INTRA_INTERP + const int p_angle = + mode_to_angle_map[mbmi->mode] + mbmi->angle_delta[0] * ANGLE_STEP; + if (av1_is_intra_filter_switchable(p_angle)) + this_rate += + cpi->intra_filter_cost[intra_filter_ctx][mbmi->intra_filter]; +#endif // CONFIG_INTRA_INTERP + this_rate += write_uniform_cost(2 * MAX_ANGLE_DELTA + 1, + MAX_ANGLE_DELTA + mbmi->angle_delta[0]); + } +#endif // CONFIG_EXT_INTRA + this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, this_distortion); +#if CONFIG_FILTER_INTRA + if (best_rd == INT64_MAX || this_rd - best_rd < (best_rd >> 4)) { + filter_intra_mode_skip_mask ^= (1 << mbmi->mode); + } +#endif // CONFIG_FILTER_INTRA + + if (this_rd < best_rd) { + best_mbmi = *mbmi; + best_rd = this_rd; +#if CONFIG_FILTER_INTRA + beat_best_rd = 1; +#endif // CONFIG_FILTER_INTRA + *rate = this_rate; + *rate_tokenonly = this_rate_tokenonly; + *distortion = this_distortion; + *skippable = s; +#if CONFIG_PVQ + od_encode_checkpoint(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + } + } + +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + +#if CONFIG_CFL + // Perform one extra txfm_rd_in_plane() call, this time with the best value so + // we can store reconstructed luma values + RD_STATS this_rd_stats; + x->cfl_store_y = 1; + txfm_rd_in_plane(x, cpi, &this_rd_stats, INT64_MAX, 0, bsize, + mic->mbmi.tx_size, cpi->sf.use_fast_coef_costing); + x->cfl_store_y = 0; +#endif + +#if CONFIG_PALETTE + if (try_palette) { + rd_pick_palette_intra_sby(cpi, x, bsize, palette_y_mode_ctx, + bmode_costs[DC_PRED], &best_mbmi, + best_palette_color_map, &best_rd, &best_model_rd, + rate, rate_tokenonly, distortion, skippable); + } +#endif // CONFIG_PALETTE + +#if CONFIG_FILTER_INTRA + if (beat_best_rd) { + if (rd_pick_filter_intra_sby(cpi, x, rate, rate_tokenonly, distortion, + skippable, bsize, bmode_costs[DC_PRED], + &best_rd, &best_model_rd, + filter_intra_mode_skip_mask)) { + best_mbmi = *mbmi; + } + } +#endif // CONFIG_FILTER_INTRA + + *mbmi = best_mbmi; + return best_rd; +} + +// Return value 0: early termination triggered, no valid rd cost available; +// 1: rd cost values are valid. +static int super_block_uvrd(const AV1_COMP *const cpi, MACROBLOCK *x, + RD_STATS *rd_stats, BLOCK_SIZE bsize, + int64_t ref_best_rd) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const TX_SIZE uv_tx_size = get_uv_tx_size(mbmi, &xd->plane[1]); + int plane; + int is_cost_valid = 1; + av1_init_rd_stats(rd_stats); + + if (ref_best_rd < 0) is_cost_valid = 0; + +#if CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + if (x->skip_chroma_rd) return is_cost_valid; + + bsize = scale_chroma_bsize(bsize, xd->plane[1].subsampling_x, + xd->plane[1].subsampling_y); +#endif // CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + +#if !CONFIG_PVQ + if (is_inter_block(mbmi) && is_cost_valid) { + for (plane = 1; plane < MAX_MB_PLANE; ++plane) + av1_subtract_plane(x, bsize, plane); + } +#endif // !CONFIG_PVQ + + if (is_cost_valid) { + for (plane = 1; plane < MAX_MB_PLANE; ++plane) { + RD_STATS pn_rd_stats; + txfm_rd_in_plane(x, cpi, &pn_rd_stats, ref_best_rd, plane, bsize, + uv_tx_size, cpi->sf.use_fast_coef_costing); + if (pn_rd_stats.rate == INT_MAX) { + is_cost_valid = 0; + break; + } + av1_merge_rd_stats(rd_stats, &pn_rd_stats); + if (RDCOST(x->rdmult, x->rddiv, rd_stats->rate, rd_stats->dist) > + ref_best_rd && + RDCOST(x->rdmult, x->rddiv, 0, rd_stats->sse) > ref_best_rd) { + is_cost_valid = 0; + break; + } + } + } + + if (!is_cost_valid) { + // reset cost value + av1_invalid_rd_stats(rd_stats); + } + + return is_cost_valid; +} + +#if CONFIG_VAR_TX +// FIXME crop these calls +static uint64_t sum_squares_2d(const int16_t *diff, int diff_stride, + TX_SIZE tx_size) { + return aom_sum_squares_2d_i16(diff, diff_stride, tx_size_wide[tx_size], + tx_size_high[tx_size]); +} + +void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, + int blk_row, int blk_col, int plane, int block, + int plane_bsize, const ENTROPY_CONTEXT *a, + const ENTROPY_CONTEXT *l, RD_STATS *rd_stats) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *xd = &x->e_mbd; + const struct macroblock_plane *const p = &x->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; + int64_t tmp; + tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); + PLANE_TYPE plane_type = get_plane_type(plane); + TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + const SCAN_ORDER *const scan_order = + get_scan(cm, tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi)); + BLOCK_SIZE txm_bsize = txsize_to_bsize[tx_size]; + int bh = block_size_high[txm_bsize]; + int bw = block_size_wide[txm_bsize]; + int txb_h = tx_size_high_unit[tx_size]; + int txb_w = tx_size_wide_unit[tx_size]; + + int src_stride = p->src.stride; + uint8_t *src = + &p->src.buf[(blk_row * src_stride + blk_col) << tx_size_wide_log2[0]]; + uint8_t *dst = + &pd->dst + .buf[(blk_row * pd->dst.stride + blk_col) << tx_size_wide_log2[0]]; +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, rec_buffer16[MAX_TX_SQUARE]); + uint8_t *rec_buffer; +#else + DECLARE_ALIGNED(16, uint8_t, rec_buffer[MAX_TX_SQUARE]); +#endif // CONFIG_HIGHBITDEPTH + int max_blocks_high = block_size_high[plane_bsize]; + int max_blocks_wide = block_size_wide[plane_bsize]; + const int diff_stride = max_blocks_wide; + const int16_t *diff = + &p->src_diff[(blk_row * diff_stride + blk_col) << tx_size_wide_log2[0]]; + int txb_coeff_cost; + + assert(tx_size < TX_SIZES_ALL); + + if (xd->mb_to_bottom_edge < 0) + max_blocks_high += xd->mb_to_bottom_edge >> (3 + pd->subsampling_y); + if (xd->mb_to_right_edge < 0) + max_blocks_wide += xd->mb_to_right_edge >> (3 + pd->subsampling_x); + + max_blocks_high >>= tx_size_wide_log2[0]; + max_blocks_wide >>= tx_size_wide_log2[0]; + + int coeff_ctx = get_entropy_context(tx_size, a, l); + + av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, + coeff_ctx, AV1_XFORM_QUANT_FP); + + av1_optimize_b(cm, x, plane, block, tx_size, coeff_ctx); + +// TODO(any): Use av1_dist_block to compute distortion +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + rec_buffer = CONVERT_TO_BYTEPTR(rec_buffer16); + aom_highbd_convolve_copy(dst, pd->dst.stride, rec_buffer, MAX_TX_SIZE, NULL, + 0, NULL, 0, bw, bh, xd->bd); + } else { + rec_buffer = (uint8_t *)rec_buffer16; + aom_convolve_copy(dst, pd->dst.stride, rec_buffer, MAX_TX_SIZE, NULL, 0, + NULL, 0, bw, bh); + } +#else + aom_convolve_copy(dst, pd->dst.stride, rec_buffer, MAX_TX_SIZE, NULL, 0, NULL, + 0, bw, bh); +#endif // CONFIG_HIGHBITDEPTH + + if (blk_row + txb_h > max_blocks_high || blk_col + txb_w > max_blocks_wide) { + int idx, idy; + int blocks_height = AOMMIN(txb_h, max_blocks_high - blk_row); + int blocks_width = AOMMIN(txb_w, max_blocks_wide - blk_col); + tmp = 0; + for (idy = 0; idy < blocks_height; ++idy) { + for (idx = 0; idx < blocks_width; ++idx) { + const int16_t *d = + diff + ((idy * diff_stride + idx) << tx_size_wide_log2[0]); + tmp += sum_squares_2d(d, diff_stride, 0); + } + } + } else { + tmp = sum_squares_2d(diff, diff_stride, tx_size); + } + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + tmp = ROUND_POWER_OF_TWO(tmp, (xd->bd - 8) * 2); +#endif // CONFIG_HIGHBITDEPTH + rd_stats->sse += tmp * 16; + const int eob = p->eobs[block]; + + av1_inverse_transform_block(xd, dqcoeff, tx_type, tx_size, rec_buffer, + MAX_TX_SIZE, eob); + if (eob > 0) { + if (txb_w + blk_col > max_blocks_wide || + txb_h + blk_row > max_blocks_high) { + int idx, idy; + unsigned int this_dist; + int blocks_height = AOMMIN(txb_h, max_blocks_high - blk_row); + int blocks_width = AOMMIN(txb_w, max_blocks_wide - blk_col); + tmp = 0; + for (idy = 0; idy < blocks_height; ++idy) { + for (idx = 0; idx < blocks_width; ++idx) { + uint8_t *const s = + src + ((idy * src_stride + idx) << tx_size_wide_log2[0]); + uint8_t *const r = + rec_buffer + ((idy * MAX_TX_SIZE + idx) << tx_size_wide_log2[0]); + cpi->fn_ptr[0].vf(s, src_stride, r, MAX_TX_SIZE, &this_dist); + tmp += this_dist; + } + } + } else { + uint32_t this_dist; + cpi->fn_ptr[txm_bsize].vf(src, src_stride, rec_buffer, MAX_TX_SIZE, + &this_dist); + tmp = this_dist; + } + } + rd_stats->dist += tmp * 16; + txb_coeff_cost = + av1_cost_coeffs(cpi, x, plane, block, tx_size, scan_order, a, l, 0); + rd_stats->rate += txb_coeff_cost; + rd_stats->skip &= (eob == 0); + +#if CONFIG_RD_DEBUG + av1_update_txb_coeff_cost(rd_stats, plane, tx_size, blk_row, blk_col, + txb_coeff_cost); +#endif // CONFIG_RD_DEBUG +} + +static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, + int blk_col, int plane, int block, int block32, + TX_SIZE tx_size, int depth, BLOCK_SIZE plane_bsize, + ENTROPY_CONTEXT *ta, ENTROPY_CONTEXT *tl, + TXFM_CONTEXT *tx_above, TXFM_CONTEXT *tx_left, + RD_STATS *rd_stats, int64_t ref_best_rd, + int *is_cost_valid, RD_STATS *rd_stats_stack) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + struct macroblock_plane *const p = &x->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; + const int tx_row = blk_row >> (1 - pd->subsampling_y); + const int tx_col = blk_col >> (1 - pd->subsampling_x); + TX_SIZE(*const inter_tx_size) + [MAX_MIB_SIZE] = + (TX_SIZE(*)[MAX_MIB_SIZE]) & mbmi->inter_tx_size[tx_row][tx_col]; + const int max_blocks_high = max_block_high(xd, plane_bsize, plane); + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + const int bw = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; + int64_t this_rd = INT64_MAX; + ENTROPY_CONTEXT *pta = ta + blk_col; + ENTROPY_CONTEXT *ptl = tl + blk_row; + int coeff_ctx, i; + int ctx = + txfm_partition_context(tx_above + (blk_col >> 1), + tx_left + (blk_row >> 1), mbmi->sb_type, tx_size); + int64_t sum_rd = INT64_MAX; + int tmp_eob = 0; + int zero_blk_rate; + RD_STATS sum_rd_stats; + const int tx_size_ctx = txsize_sqr_map[tx_size]; + + av1_init_rd_stats(&sum_rd_stats); + + assert(tx_size < TX_SIZES_ALL); + + if (ref_best_rd < 0) { + *is_cost_valid = 0; + return; + } + + coeff_ctx = get_entropy_context(tx_size, pta, ptl); + + av1_init_rd_stats(rd_stats); + + if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + + zero_blk_rate = x->token_costs[tx_size_ctx][pd->plane_type][1][0][0] + [coeff_ctx][EOB_TOKEN]; + + if (cpi->common.tx_mode == TX_MODE_SELECT || tx_size == TX_4X4) { + inter_tx_size[0][0] = tx_size; + + if (tx_size == TX_32X32 && mbmi->tx_type != DCT_DCT && + rd_stats_stack[block32].rate != INT_MAX) { + *rd_stats = rd_stats_stack[block32]; + p->eobs[block] = !rd_stats->skip; + x->blk_skip[plane][blk_row * bw + blk_col] = rd_stats->skip; + } else { + av1_tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block, + plane_bsize, pta, ptl, rd_stats); + if (tx_size == TX_32X32) { + rd_stats_stack[block32] = *rd_stats; + } + } + + if ((RDCOST(x->rdmult, x->rddiv, rd_stats->rate, rd_stats->dist) >= + RDCOST(x->rdmult, x->rddiv, zero_blk_rate, rd_stats->sse) || + rd_stats->skip == 1) && + !xd->lossless[mbmi->segment_id]) { +#if CONFIG_RD_DEBUG + av1_update_txb_coeff_cost(rd_stats, plane, tx_size, blk_row, blk_col, + zero_blk_rate - rd_stats->rate); +#endif // CONFIG_RD_DEBUG + rd_stats->rate = zero_blk_rate; + rd_stats->dist = rd_stats->sse; + rd_stats->skip = 1; + x->blk_skip[plane][blk_row * bw + blk_col] = 1; + p->eobs[block] = 0; + } else { + x->blk_skip[plane][blk_row * bw + blk_col] = 0; + rd_stats->skip = 0; + } + + if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH) + rd_stats->rate += + av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0); + this_rd = RDCOST(x->rdmult, x->rddiv, rd_stats->rate, rd_stats->dist); + tmp_eob = p->eobs[block]; + } + + if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH) { + const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; + const int bsl = tx_size_wide_unit[sub_txs]; + int sub_step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs]; + RD_STATS this_rd_stats; + int this_cost_valid = 1; + int64_t tmp_rd = 0; + + sum_rd_stats.rate = + av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 1); + + assert(tx_size < TX_SIZES_ALL); + + for (i = 0; i < 4 && this_cost_valid; ++i) { + int offsetr = blk_row + (i >> 1) * bsl; + int offsetc = blk_col + (i & 0x01) * bsl; + + if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue; + + select_tx_block(cpi, x, offsetr, offsetc, plane, block, block32, sub_txs, + depth + 1, plane_bsize, ta, tl, tx_above, tx_left, + &this_rd_stats, ref_best_rd - tmp_rd, &this_cost_valid, + rd_stats_stack); + + av1_merge_rd_stats(&sum_rd_stats, &this_rd_stats); + + tmp_rd = + RDCOST(x->rdmult, x->rddiv, sum_rd_stats.rate, sum_rd_stats.dist); + if (this_rd < tmp_rd) break; + block += sub_step; + } + if (this_cost_valid) sum_rd = tmp_rd; + } + + if (this_rd < sum_rd) { + int idx, idy; + for (i = 0; i < tx_size_wide_unit[tx_size]; ++i) pta[i] = !(tmp_eob == 0); + for (i = 0; i < tx_size_high_unit[tx_size]; ++i) ptl[i] = !(tmp_eob == 0); + txfm_partition_update(tx_above + (blk_col >> 1), tx_left + (blk_row >> 1), + tx_size, tx_size); + inter_tx_size[0][0] = tx_size; + for (idy = 0; idy < tx_size_high_unit[tx_size] / 2; ++idy) + for (idx = 0; idx < tx_size_wide_unit[tx_size] / 2; ++idx) + inter_tx_size[idy][idx] = tx_size; + mbmi->tx_size = tx_size; + if (this_rd == INT64_MAX) *is_cost_valid = 0; + x->blk_skip[plane][blk_row * bw + blk_col] = rd_stats->skip; + } else { + *rd_stats = sum_rd_stats; + if (sum_rd == INT64_MAX) *is_cost_valid = 0; + } +} + +static void inter_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, + RD_STATS *rd_stats, BLOCK_SIZE bsize, + int64_t ref_best_rd, RD_STATS *rd_stats_stack) { + MACROBLOCKD *const xd = &x->e_mbd; + int is_cost_valid = 1; + int64_t this_rd = 0; + + if (ref_best_rd < 0) is_cost_valid = 0; + + av1_init_rd_stats(rd_stats); + + if (is_cost_valid) { + const struct macroblockd_plane *const pd = &xd->plane[0]; + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); + const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; + const int mi_height = block_size_high[plane_bsize] >> tx_size_high_log2[0]; + const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize]; + const int bh = tx_size_high_unit[max_tx_size]; + const int bw = tx_size_wide_unit[max_tx_size]; + int idx, idy; + int block = 0; + int block32 = 0; + int step = tx_size_wide_unit[max_tx_size] * tx_size_high_unit[max_tx_size]; + ENTROPY_CONTEXT ctxa[2 * MAX_MIB_SIZE]; + ENTROPY_CONTEXT ctxl[2 * MAX_MIB_SIZE]; + TXFM_CONTEXT tx_above[MAX_MIB_SIZE]; + TXFM_CONTEXT tx_left[MAX_MIB_SIZE]; + + RD_STATS pn_rd_stats; + av1_init_rd_stats(&pn_rd_stats); + + av1_get_entropy_contexts(bsize, 0, pd, ctxa, ctxl); + memcpy(tx_above, xd->above_txfm_context, + sizeof(TXFM_CONTEXT) * (mi_width >> 1)); + memcpy(tx_left, xd->left_txfm_context, + sizeof(TXFM_CONTEXT) * (mi_height >> 1)); + + for (idy = 0; idy < mi_height; idy += bh) { + for (idx = 0; idx < mi_width; idx += bw) { + select_tx_block(cpi, x, idy, idx, 0, block, block32, max_tx_size, + mi_height != mi_width, plane_bsize, ctxa, ctxl, + tx_above, tx_left, &pn_rd_stats, ref_best_rd - this_rd, + &is_cost_valid, rd_stats_stack); + av1_merge_rd_stats(rd_stats, &pn_rd_stats); + this_rd += AOMMIN( + RDCOST(x->rdmult, x->rddiv, pn_rd_stats.rate, pn_rd_stats.dist), + RDCOST(x->rdmult, x->rddiv, 0, pn_rd_stats.sse)); + block += step; + ++block32; + } + } + } + + this_rd = AOMMIN(RDCOST(x->rdmult, x->rddiv, rd_stats->rate, rd_stats->dist), + RDCOST(x->rdmult, x->rddiv, 0, rd_stats->sse)); + if (this_rd > ref_best_rd) is_cost_valid = 0; + + if (!is_cost_valid) { + // reset cost value + av1_invalid_rd_stats(rd_stats); + } +} + +static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x, + RD_STATS *rd_stats, BLOCK_SIZE bsize, + int64_t ref_best_rd, TX_TYPE tx_type, + RD_STATS *rd_stats_stack) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const int is_inter = is_inter_block(mbmi); + aom_prob skip_prob = av1_get_skip_prob(cm, xd); + int s0 = av1_cost_bit(skip_prob, 0); + int s1 = av1_cost_bit(skip_prob, 1); + int64_t rd; + int row, col; + const int max_blocks_high = max_block_high(xd, bsize, 0); + const int max_blocks_wide = max_block_wide(xd, bsize, 0); + + mbmi->tx_type = tx_type; + mbmi->min_tx_size = TX_SIZES_ALL; + inter_block_yrd(cpi, x, rd_stats, bsize, ref_best_rd, rd_stats_stack); + + if (rd_stats->rate == INT_MAX) return INT64_MAX; + + for (row = 0; row < max_blocks_high / 2; ++row) + for (col = 0; col < max_blocks_wide / 2; ++col) + mbmi->min_tx_size = AOMMIN( + mbmi->min_tx_size, get_min_tx_size(mbmi->inter_tx_size[row][col])); + +#if CONFIG_EXT_TX + if (get_ext_tx_types(mbmi->min_tx_size, bsize, is_inter, + cm->reduced_tx_set_used) > 1 && + !xd->lossless[xd->mi[0]->mbmi.segment_id]) { + const int ext_tx_set = get_ext_tx_set(mbmi->min_tx_size, bsize, is_inter, + cm->reduced_tx_set_used); + if (is_inter) { + if (ext_tx_set > 0) + rd_stats->rate += + cpi->inter_tx_type_costs[ext_tx_set] + [txsize_sqr_map[mbmi->min_tx_size]] + [mbmi->tx_type]; + } else { + if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX) + rd_stats->rate += + cpi->intra_tx_type_costs[ext_tx_set][mbmi->min_tx_size][mbmi->mode] + [mbmi->tx_type]; + } + } +#else // CONFIG_EXT_TX + if (mbmi->min_tx_size < TX_32X32 && !xd->lossless[xd->mi[0]->mbmi.segment_id]) + rd_stats->rate += + cpi->inter_tx_type_costs[mbmi->min_tx_size][mbmi->tx_type]; +#endif // CONFIG_EXT_TX + + if (rd_stats->skip) + rd = RDCOST(x->rdmult, x->rddiv, s1, rd_stats->sse); + else + rd = RDCOST(x->rdmult, x->rddiv, rd_stats->rate + s0, rd_stats->dist); + + if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] && + !(rd_stats->skip)) + rd = AOMMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, rd_stats->sse)); + + return rd; +} + +static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, + RD_STATS *rd_stats, BLOCK_SIZE bsize, + int64_t ref_best_rd) { + const AV1_COMMON *cm = &cpi->common; + const TX_SIZE max_tx_size = max_txsize_lookup[bsize]; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + int64_t rd = INT64_MAX; + int64_t best_rd = INT64_MAX; + TX_TYPE tx_type, best_tx_type = DCT_DCT; + const int is_inter = is_inter_block(mbmi); + TX_SIZE best_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE]; + TX_SIZE best_tx = max_txsize_lookup[bsize]; + TX_SIZE best_min_tx_size = TX_SIZES_ALL; + uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE * 8]; + const int n4 = bsize_to_num_blk(bsize); + int idx, idy; + int prune = 0; + const int count32 = + 1 << (2 * (cm->mib_size_log2 - mi_width_log2_lookup[BLOCK_32X32])); +#if CONFIG_EXT_PARTITION + RD_STATS rd_stats_stack[16]; +#else + RD_STATS rd_stats_stack[4]; +#endif // CONFIG_EXT_PARTITION +#if CONFIG_EXT_TX + const int ext_tx_set = + get_ext_tx_set(max_tx_size, bsize, is_inter, cm->reduced_tx_set_used); +#endif // CONFIG_EXT_TX + + if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE) +#if CONFIG_EXT_TX + prune = prune_tx_types(cpi, bsize, x, xd, ext_tx_set); +#else + prune = prune_tx_types(cpi, bsize, x, xd, 0); +#endif // CONFIG_EXT_TX + + av1_invalid_rd_stats(rd_stats); + + for (idx = 0; idx < count32; ++idx) + av1_invalid_rd_stats(&rd_stats_stack[idx]); + + for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) { + RD_STATS this_rd_stats; + av1_init_rd_stats(&this_rd_stats); +#if CONFIG_EXT_TX + if (is_inter) { + if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue; + if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) { + if (!do_tx_type_search(tx_type, prune)) continue; + } + } else { + if (!ALLOW_INTRA_EXT_TX && bsize >= BLOCK_8X8) { + if (tx_type != intra_mode_to_tx_type_context[mbmi->mode]) continue; + } + if (!ext_tx_used_intra[ext_tx_set][tx_type]) continue; + } +#else // CONFIG_EXT_TX + if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE && + !do_tx_type_search(tx_type, prune)) + continue; +#endif // CONFIG_EXT_TX + if (is_inter && x->use_default_inter_tx_type && + tx_type != get_default_tx_type(0, xd, 0, max_tx_size)) + continue; + + if (xd->lossless[mbmi->segment_id]) + if (tx_type != DCT_DCT) continue; + + rd = select_tx_size_fix_type(cpi, x, &this_rd_stats, bsize, ref_best_rd, + tx_type, rd_stats_stack); + + if (rd < best_rd) { + best_rd = rd; + *rd_stats = this_rd_stats; + best_tx_type = mbmi->tx_type; + best_tx = mbmi->tx_size; + best_min_tx_size = mbmi->min_tx_size; + memcpy(best_blk_skip, x->blk_skip[0], sizeof(best_blk_skip[0]) * n4); + for (idy = 0; idy < xd->n8_h; ++idy) + for (idx = 0; idx < xd->n8_w; ++idx) + best_tx_size[idy][idx] = mbmi->inter_tx_size[idy][idx]; + } + } + + mbmi->tx_type = best_tx_type; + for (idy = 0; idy < xd->n8_h; ++idy) + for (idx = 0; idx < xd->n8_w; ++idx) + mbmi->inter_tx_size[idy][idx] = best_tx_size[idy][idx]; + mbmi->tx_size = best_tx; + mbmi->min_tx_size = best_min_tx_size; + memcpy(x->blk_skip[0], best_blk_skip, sizeof(best_blk_skip[0]) * n4); +} + +static void tx_block_rd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, + int blk_col, int plane, int block, TX_SIZE tx_size, + BLOCK_SIZE plane_bsize, ENTROPY_CONTEXT *above_ctx, + ENTROPY_CONTEXT *left_ctx, RD_STATS *rd_stats) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + struct macroblock_plane *const p = &x->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; + BLOCK_SIZE bsize = txsize_to_bsize[tx_size]; + const int tx_row = blk_row >> (1 - pd->subsampling_y); + const int tx_col = blk_col >> (1 - pd->subsampling_x); + TX_SIZE plane_tx_size; + const int max_blocks_high = max_block_high(xd, plane_bsize, plane); + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + + assert(tx_size < TX_SIZES_ALL); + + if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + + plane_tx_size = + plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0] + : mbmi->inter_tx_size[tx_row][tx_col]; + + if (tx_size == plane_tx_size) { + int i; + ENTROPY_CONTEXT *ta = above_ctx + blk_col; + ENTROPY_CONTEXT *tl = left_ctx + blk_row; + av1_tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block, + plane_bsize, ta, tl, rd_stats); + + for (i = 0; i < tx_size_wide_unit[tx_size]; ++i) + ta[i] = !(p->eobs[block] == 0); + for (i = 0; i < tx_size_high_unit[tx_size]; ++i) + tl[i] = !(p->eobs[block] == 0); + } else { + const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; + const int bsl = tx_size_wide_unit[sub_txs]; + int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs]; + int i; + + assert(bsl > 0); + + for (i = 0; i < 4; ++i) { + int offsetr = blk_row + (i >> 1) * bsl; + int offsetc = blk_col + (i & 0x01) * bsl; + + if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue; + + tx_block_rd(cpi, x, offsetr, offsetc, plane, block, sub_txs, plane_bsize, + above_ctx, left_ctx, rd_stats); + block += step; + } + } +} + +// Return value 0: early termination triggered, no valid rd cost available; +// 1: rd cost values are valid. +static int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, + RD_STATS *rd_stats, BLOCK_SIZE bsize, + int64_t ref_best_rd) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + int plane; + int is_cost_valid = 1; + int64_t this_rd; + + if (ref_best_rd < 0) is_cost_valid = 0; + + av1_init_rd_stats(rd_stats); + +#if CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + if (x->skip_chroma_rd) return is_cost_valid; + bsize = AOMMAX(BLOCK_8X8, bsize); +#endif // CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + +#if CONFIG_EXT_TX && CONFIG_RECT_TX + if (is_rect_tx(mbmi->tx_size)) { + return super_block_uvrd(cpi, x, rd_stats, bsize, ref_best_rd); + } +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + + if (is_inter_block(mbmi) && is_cost_valid) { + for (plane = 1; plane < MAX_MB_PLANE; ++plane) + av1_subtract_plane(x, bsize, plane); + } + + for (plane = 1; plane < MAX_MB_PLANE; ++plane) { + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); + const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; + const int mi_height = block_size_high[plane_bsize] >> tx_size_high_log2[0]; + const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize]; + const int bh = tx_size_high_unit[max_tx_size]; + const int bw = tx_size_wide_unit[max_tx_size]; + int idx, idy; + int block = 0; + const int step = bh * bw; + ENTROPY_CONTEXT ta[2 * MAX_MIB_SIZE]; + ENTROPY_CONTEXT tl[2 * MAX_MIB_SIZE]; + RD_STATS pn_rd_stats; + av1_init_rd_stats(&pn_rd_stats); + + av1_get_entropy_contexts(bsize, 0, pd, ta, tl); + + for (idy = 0; idy < mi_height; idy += bh) { + for (idx = 0; idx < mi_width; idx += bw) { + tx_block_rd(cpi, x, idy, idx, plane, block, max_tx_size, plane_bsize, + ta, tl, &pn_rd_stats); + block += step; + } + } + + if (pn_rd_stats.rate == INT_MAX) { + is_cost_valid = 0; + break; + } + + av1_merge_rd_stats(rd_stats, &pn_rd_stats); + + this_rd = + AOMMIN(RDCOST(x->rdmult, x->rddiv, rd_stats->rate, rd_stats->dist), + RDCOST(x->rdmult, x->rddiv, 0, rd_stats->sse)); + + if (this_rd > ref_best_rd) { + is_cost_valid = 0; + break; + } + } + + if (!is_cost_valid) { + // reset cost value + av1_invalid_rd_stats(rd_stats); + } + + return is_cost_valid; +} +#endif // CONFIG_VAR_TX + +#if CONFIG_PALETTE +static void rd_pick_palette_intra_sbuv(const AV1_COMP *const cpi, MACROBLOCK *x, + int dc_mode_cost, + uint8_t *best_palette_color_map, + MB_MODE_INFO *const best_mbmi, + int64_t *best_rd, int *rate, + int *rate_tokenonly, int64_t *distortion, + int *skippable) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; + const BLOCK_SIZE bsize = mbmi->sb_type; + int this_rate; + int64_t this_rd; + int colors_u, colors_v, colors; + const int src_stride = x->plane[1].src.stride; + const uint8_t *const src_u = x->plane[1].src.buf; + const uint8_t *const src_v = x->plane[2].src.buf; + uint8_t *const color_map = xd->plane[1].color_index_map; + RD_STATS tokenonly_rd_stats; + int plane_block_width, plane_block_height, rows, cols; + av1_get_block_dimensions(bsize, 1, xd, &plane_block_width, + &plane_block_height, &rows, &cols); + if (rows * cols > PALETTE_MAX_BLOCK_SIZE) return; + + mbmi->uv_mode = DC_PRED; +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = 0; +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_HIGHBITDEPTH + if (cpi->common.use_highbitdepth) { + colors_u = av1_count_colors_highbd(src_u, src_stride, rows, cols, + cpi->common.bit_depth); + colors_v = av1_count_colors_highbd(src_v, src_stride, rows, cols, + cpi->common.bit_depth); + } else { +#endif // CONFIG_HIGHBITDEPTH + colors_u = av1_count_colors(src_u, src_stride, rows, cols); + colors_v = av1_count_colors(src_v, src_stride, rows, cols); +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + + colors = colors_u > colors_v ? colors_u : colors_v; + if (colors > 1 && colors <= 64) { + int r, c, n, i, j; + const int max_itr = 50; + uint8_t color_order[PALETTE_MAX_SIZE]; + float lb_u, ub_u, val_u; + float lb_v, ub_v, val_v; + float *const data = x->palette_buffer->kmeans_data_buf; + float centroids[2 * PALETTE_MAX_SIZE]; + +#if CONFIG_HIGHBITDEPTH + uint16_t *src_u16 = CONVERT_TO_SHORTPTR(src_u); + uint16_t *src_v16 = CONVERT_TO_SHORTPTR(src_v); + if (cpi->common.use_highbitdepth) { + lb_u = src_u16[0]; + ub_u = src_u16[0]; + lb_v = src_v16[0]; + ub_v = src_v16[0]; + } else { +#endif // CONFIG_HIGHBITDEPTH + lb_u = src_u[0]; + ub_u = src_u[0]; + lb_v = src_v[0]; + ub_v = src_v[0]; +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + + for (r = 0; r < rows; ++r) { + for (c = 0; c < cols; ++c) { +#if CONFIG_HIGHBITDEPTH + if (cpi->common.use_highbitdepth) { + val_u = src_u16[r * src_stride + c]; + val_v = src_v16[r * src_stride + c]; + data[(r * cols + c) * 2] = val_u; + data[(r * cols + c) * 2 + 1] = val_v; + } else { +#endif // CONFIG_HIGHBITDEPTH + val_u = src_u[r * src_stride + c]; + val_v = src_v[r * src_stride + c]; + data[(r * cols + c) * 2] = val_u; + data[(r * cols + c) * 2 + 1] = val_v; +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + if (val_u < lb_u) + lb_u = val_u; + else if (val_u > ub_u) + ub_u = val_u; + if (val_v < lb_v) + lb_v = val_v; + else if (val_v > ub_v) + ub_v = val_v; + } + } + + for (n = colors > PALETTE_MAX_SIZE ? PALETTE_MAX_SIZE : colors; n >= 2; + --n) { + for (i = 0; i < n; ++i) { + centroids[i * 2] = lb_u + (2 * i + 1) * (ub_u - lb_u) / n / 2; + centroids[i * 2 + 1] = lb_v + (2 * i + 1) * (ub_v - lb_v) / n / 2; + } + av1_k_means(data, centroids, color_map, rows * cols, n, 2, max_itr); +#if CONFIG_PALETTE_DELTA_ENCODING + // Sort the U channel colors in ascending order. + for (i = 0; i < 2 * (n - 1); i += 2) { + int min_idx = i; + float min_val = centroids[i]; + for (j = i + 2; j < 2 * n; j += 2) + if (centroids[j] < min_val) min_val = centroids[j], min_idx = j; + if (min_idx != i) { + float temp_u = centroids[i], temp_v = centroids[i + 1]; + centroids[i] = centroids[min_idx]; + centroids[i + 1] = centroids[min_idx + 1]; + centroids[min_idx] = temp_u, centroids[min_idx + 1] = temp_v; + } + } + av1_calc_indices(data, centroids, color_map, rows * cols, n, 2); +#endif // CONFIG_PALETTE_DELTA_ENCODING + extend_palette_color_map(color_map, cols, rows, plane_block_width, + plane_block_height); + pmi->palette_size[1] = n; + for (i = 1; i < 3; ++i) { + for (j = 0; j < n; ++j) { +#if CONFIG_HIGHBITDEPTH + if (cpi->common.use_highbitdepth) + pmi->palette_colors[i * PALETTE_MAX_SIZE + j] = clip_pixel_highbd( + (int)centroids[j * 2 + i - 1], cpi->common.bit_depth); + else +#endif // CONFIG_HIGHBITDEPTH + pmi->palette_colors[i * PALETTE_MAX_SIZE + j] = + clip_pixel((int)centroids[j * 2 + i - 1]); + } + } + + super_block_uvrd(cpi, x, &tokenonly_rd_stats, bsize, *best_rd); + if (tokenonly_rd_stats.rate == INT_MAX) continue; + this_rate = + tokenonly_rd_stats.rate + dc_mode_cost + + cpi->palette_uv_size_cost[bsize - BLOCK_8X8][n - PALETTE_MIN_SIZE] + + write_uniform_cost(n, color_map[0]) + + av1_cost_bit( + av1_default_palette_uv_mode_prob[pmi->palette_size[0] > 0], 1); + this_rate += av1_palette_color_cost_uv(pmi, cpi->common.bit_depth); + for (i = 0; i < rows; ++i) { + for (j = (i == 0 ? 1 : 0); j < cols; ++j) { + int color_idx; + const int color_ctx = av1_get_palette_color_index_context( + color_map, plane_block_width, i, j, n, color_order, &color_idx); + assert(color_idx >= 0 && color_idx < n); + this_rate += cpi->palette_uv_color_cost[n - PALETTE_MIN_SIZE] + [color_ctx][color_idx]; + } + } + + this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, tokenonly_rd_stats.dist); + if (this_rd < *best_rd) { + *best_rd = this_rd; + *best_mbmi = *mbmi; + memcpy(best_palette_color_map, color_map, + plane_block_width * plane_block_height * + sizeof(best_palette_color_map[0])); + *rate = this_rate; + *distortion = tokenonly_rd_stats.dist; + *rate_tokenonly = tokenonly_rd_stats.rate; + *skippable = tokenonly_rd_stats.skip; + } + } + } + if (best_mbmi->palette_mode_info.palette_size[1] > 0) { + memcpy(color_map, best_palette_color_map, + rows * cols * sizeof(best_palette_color_map[0])); + } +} +#endif // CONFIG_PALETTE + +#if CONFIG_FILTER_INTRA +// Return 1 if an filter intra mode is selected; return 0 otherwise. +static int rd_pick_filter_intra_sbuv(const AV1_COMP *const cpi, MACROBLOCK *x, + int *rate, int *rate_tokenonly, + int64_t *distortion, int *skippable, + BLOCK_SIZE bsize, int64_t *best_rd) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + int filter_intra_selected_flag = 0; + int this_rate; + int64_t this_rd; + FILTER_INTRA_MODE mode; + FILTER_INTRA_MODE_INFO filter_intra_mode_info; + RD_STATS tokenonly_rd_stats; + + av1_zero(filter_intra_mode_info); + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = 1; + mbmi->uv_mode = DC_PRED; +#if CONFIG_PALETTE + mbmi->palette_mode_info.palette_size[1] = 0; +#endif // CONFIG_PALETTE + + for (mode = 0; mode < FILTER_INTRA_MODES; ++mode) { + mbmi->filter_intra_mode_info.filter_intra_mode[1] = mode; + if (!super_block_uvrd(cpi, x, &tokenonly_rd_stats, bsize, *best_rd)) + continue; + + this_rate = tokenonly_rd_stats.rate + + av1_cost_bit(cpi->common.fc->filter_intra_probs[1], 1) + + cpi->intra_uv_mode_cost[mbmi->mode][mbmi->uv_mode] + + write_uniform_cost(FILTER_INTRA_MODES, mode); + this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, tokenonly_rd_stats.dist); + if (this_rd < *best_rd) { + *best_rd = this_rd; + *rate = this_rate; + *rate_tokenonly = tokenonly_rd_stats.rate; + *distortion = tokenonly_rd_stats.dist; + *skippable = tokenonly_rd_stats.skip; + filter_intra_mode_info = mbmi->filter_intra_mode_info; + filter_intra_selected_flag = 1; + } + } + + if (filter_intra_selected_flag) { + mbmi->uv_mode = DC_PRED; + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = + filter_intra_mode_info.use_filter_intra_mode[1]; + mbmi->filter_intra_mode_info.filter_intra_mode[1] = + filter_intra_mode_info.filter_intra_mode[1]; + return 1; + } else { + return 0; + } +} +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_EXT_INTRA +// Run RD calculation with given chroma intra prediction angle., and return +// the RD cost. Update the best mode info. if the RD cost is the best so far. +static int64_t pick_intra_angle_routine_sbuv( + const AV1_COMP *const cpi, MACROBLOCK *x, BLOCK_SIZE bsize, + int rate_overhead, int64_t best_rd_in, int *rate, RD_STATS *rd_stats, + int *best_angle_delta, int64_t *best_rd) { + MB_MODE_INFO *mbmi = &x->e_mbd.mi[0]->mbmi; + int this_rate; + int64_t this_rd; + RD_STATS tokenonly_rd_stats; + + if (!super_block_uvrd(cpi, x, &tokenonly_rd_stats, bsize, best_rd_in)) + return INT64_MAX; + this_rate = tokenonly_rd_stats.rate + rate_overhead; + this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, tokenonly_rd_stats.dist); + if (this_rd < *best_rd) { + *best_rd = this_rd; + *best_angle_delta = mbmi->angle_delta[1]; + *rate = this_rate; + rd_stats->rate = tokenonly_rd_stats.rate; + rd_stats->dist = tokenonly_rd_stats.dist; + rd_stats->skip = tokenonly_rd_stats.skip; + } + return this_rd; +} + +// With given chroma directional intra prediction mode, pick the best angle +// delta. Return true if a RD cost that is smaller than the input one is found. +static int rd_pick_intra_angle_sbuv(const AV1_COMP *const cpi, MACROBLOCK *x, + BLOCK_SIZE bsize, int rate_overhead, + int64_t best_rd, int *rate, + RD_STATS *rd_stats) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + int i, angle_delta, best_angle_delta = 0; + int64_t this_rd, best_rd_in, rd_cost[2 * (MAX_ANGLE_DELTA + 2)]; + + rd_stats->rate = INT_MAX; + rd_stats->skip = 0; + rd_stats->dist = INT64_MAX; + for (i = 0; i < 2 * (MAX_ANGLE_DELTA + 2); ++i) rd_cost[i] = INT64_MAX; + + for (angle_delta = 0; angle_delta <= MAX_ANGLE_DELTA; angle_delta += 2) { + for (i = 0; i < 2; ++i) { + best_rd_in = (best_rd == INT64_MAX) + ? INT64_MAX + : (best_rd + (best_rd >> ((angle_delta == 0) ? 3 : 5))); + mbmi->angle_delta[1] = (1 - 2 * i) * angle_delta; + this_rd = pick_intra_angle_routine_sbuv(cpi, x, bsize, rate_overhead, + best_rd_in, rate, rd_stats, + &best_angle_delta, &best_rd); + rd_cost[2 * angle_delta + i] = this_rd; + if (angle_delta == 0) { + if (this_rd == INT64_MAX) return 0; + rd_cost[1] = this_rd; + break; + } + } + } + + assert(best_rd != INT64_MAX); + for (angle_delta = 1; angle_delta <= MAX_ANGLE_DELTA; angle_delta += 2) { + int64_t rd_thresh; + for (i = 0; i < 2; ++i) { + int skip_search = 0; + rd_thresh = best_rd + (best_rd >> 5); + if (rd_cost[2 * (angle_delta + 1) + i] > rd_thresh && + rd_cost[2 * (angle_delta - 1) + i] > rd_thresh) + skip_search = 1; + if (!skip_search) { + mbmi->angle_delta[1] = (1 - 2 * i) * angle_delta; + pick_intra_angle_routine_sbuv(cpi, x, bsize, rate_overhead, best_rd, + rate, rd_stats, &best_angle_delta, + &best_rd); + } + } + } + + mbmi->angle_delta[1] = best_angle_delta; + return rd_stats->rate != INT_MAX; +} +#endif // CONFIG_EXT_INTRA + +static int64_t rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x, + int *rate, int *rate_tokenonly, + int64_t *distortion, int *skippable, + BLOCK_SIZE bsize, TX_SIZE max_tx_size) { + MACROBLOCKD *xd = &x->e_mbd; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + MB_MODE_INFO best_mbmi = *mbmi; + PREDICTION_MODE mode; + int64_t best_rd = INT64_MAX, this_rd; + int this_rate; + RD_STATS tokenonly_rd_stats; +#if CONFIG_PVQ + od_rollback_buffer buf; + od_encode_checkpoint(&x->daala_enc, &buf); +#endif // CONFIG_PVQ +#if CONFIG_PALETTE + PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; + uint8_t *best_palette_color_map = NULL; +#endif // CONFIG_PALETTE + +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = 0; +#endif // CONFIG_FILTER_INTRA +#if CONFIG_PALETTE + pmi->palette_size[1] = 0; +#endif // CONFIG_PALETTE + for (mode = DC_PRED; mode <= TM_PRED; ++mode) { +#if CONFIG_EXT_INTRA + const int is_directional_mode = + av1_is_directional_mode(mode, mbmi->sb_type); +#endif // CONFIG_EXT_INTRA + if (!(cpi->sf.intra_uv_mode_mask[txsize_sqr_up_map[max_tx_size]] & + (1 << mode))) + continue; + + mbmi->uv_mode = mode; +#if CONFIG_EXT_INTRA + mbmi->angle_delta[1] = 0; + if (is_directional_mode) { + const int rate_overhead = cpi->intra_uv_mode_cost[mbmi->mode][mode] + + write_uniform_cost(2 * MAX_ANGLE_DELTA + 1, 0); + if (!rd_pick_intra_angle_sbuv(cpi, x, bsize, rate_overhead, best_rd, + &this_rate, &tokenonly_rd_stats)) + continue; + } else { +#endif // CONFIG_EXT_INTRA + if (!super_block_uvrd(cpi, x, &tokenonly_rd_stats, bsize, best_rd)) { +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &buf); +#endif // CONFIG_PVQ + continue; + } +#if CONFIG_EXT_INTRA + } +#endif // CONFIG_EXT_INTRA + this_rate = + tokenonly_rd_stats.rate + cpi->intra_uv_mode_cost[mbmi->mode][mode]; + +#if CONFIG_EXT_INTRA + if (is_directional_mode) { + this_rate += write_uniform_cost(2 * MAX_ANGLE_DELTA + 1, + MAX_ANGLE_DELTA + mbmi->angle_delta[1]); + } +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + if (mbmi->sb_type >= BLOCK_8X8 && mode == DC_PRED) + this_rate += av1_cost_bit(cpi->common.fc->filter_intra_probs[1], 0); +#endif // CONFIG_FILTER_INTRA +#if CONFIG_PALETTE + if (cpi->common.allow_screen_content_tools && mbmi->sb_type >= BLOCK_8X8 && + mode == DC_PRED) + this_rate += av1_cost_bit( + av1_default_palette_uv_mode_prob[pmi->palette_size[0] > 0], 0); +#endif // CONFIG_PALETTE + +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &buf); +#endif // CONFIG_PVQ + this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, tokenonly_rd_stats.dist); + + if (this_rd < best_rd) { + best_mbmi = *mbmi; + best_rd = this_rd; + *rate = this_rate; + *rate_tokenonly = tokenonly_rd_stats.rate; + *distortion = tokenonly_rd_stats.dist; + *skippable = tokenonly_rd_stats.skip; + } + } + +#if CONFIG_PALETTE + if (cpi->common.allow_screen_content_tools && mbmi->sb_type >= BLOCK_8X8) { + best_palette_color_map = x->palette_buffer->best_palette_color_map; + rd_pick_palette_intra_sbuv(cpi, x, + cpi->intra_uv_mode_cost[mbmi->mode][DC_PRED], + best_palette_color_map, &best_mbmi, &best_rd, + rate, rate_tokenonly, distortion, skippable); + } +#endif // CONFIG_PALETTE + +#if CONFIG_FILTER_INTRA + if (mbmi->sb_type >= BLOCK_8X8) { + if (rd_pick_filter_intra_sbuv(cpi, x, rate, rate_tokenonly, distortion, + skippable, bsize, &best_rd)) + best_mbmi = *mbmi; + } +#endif // CONFIG_FILTER_INTRA + + *mbmi = best_mbmi; + // Make sure we actually chose a mode + assert(best_rd < INT64_MAX); + return best_rd; +} + +static void choose_intra_uv_mode(const AV1_COMP *const cpi, MACROBLOCK *const x, + PICK_MODE_CONTEXT *ctx, BLOCK_SIZE bsize, + TX_SIZE max_tx_size, int *rate_uv, + int *rate_uv_tokenonly, int64_t *dist_uv, + int *skip_uv, PREDICTION_MODE *mode_uv) { + // Use an estimated rd for uv_intra based on DC_PRED if the + // appropriate speed flag is set. + (void)ctx; +#if CONFIG_CB4X4 +#if CONFIG_CHROMA_2X2 + rd_pick_intra_sbuv_mode(cpi, x, rate_uv, rate_uv_tokenonly, dist_uv, skip_uv, + bsize, max_tx_size); +#else + max_tx_size = AOMMAX(max_tx_size, TX_4X4); + if (x->skip_chroma_rd) { + *rate_uv = 0; + *rate_uv_tokenonly = 0; + *dist_uv = 0; + *skip_uv = 1; + *mode_uv = DC_PRED; + return; + } + BLOCK_SIZE bs = scale_chroma_bsize(bsize, x->e_mbd.plane[1].subsampling_x, + x->e_mbd.plane[1].subsampling_y); + rd_pick_intra_sbuv_mode(cpi, x, rate_uv, rate_uv_tokenonly, dist_uv, skip_uv, + bs, max_tx_size); +#endif // CONFIG_CHROMA_2X2 +#else + rd_pick_intra_sbuv_mode(cpi, x, rate_uv, rate_uv_tokenonly, dist_uv, skip_uv, + bsize < BLOCK_8X8 ? BLOCK_8X8 : bsize, max_tx_size); +#endif // CONFIG_CB4X4 + *mode_uv = x->e_mbd.mi[0]->mbmi.uv_mode; +} + +static int cost_mv_ref(const AV1_COMP *const cpi, PREDICTION_MODE mode, + int16_t mode_context) { +#if CONFIG_EXT_INTER + if (is_inter_compound_mode(mode)) { + return cpi + ->inter_compound_mode_cost[mode_context][INTER_COMPOUND_OFFSET(mode)]; + } +#endif + +#if CONFIG_REF_MV + int mode_cost = 0; + int16_t mode_ctx = mode_context & NEWMV_CTX_MASK; + int16_t is_all_zero_mv = mode_context & (1 << ALL_ZERO_FLAG_OFFSET); + + assert(is_inter_mode(mode)); + + if (mode == NEWMV) { + mode_cost = cpi->newmv_mode_cost[mode_ctx][0]; + return mode_cost; + } else { + mode_cost = cpi->newmv_mode_cost[mode_ctx][1]; + mode_ctx = (mode_context >> ZEROMV_OFFSET) & ZEROMV_CTX_MASK; + + if (is_all_zero_mv) return mode_cost; + + if (mode == ZEROMV) { + mode_cost += cpi->zeromv_mode_cost[mode_ctx][0]; + return mode_cost; + } else { + mode_cost += cpi->zeromv_mode_cost[mode_ctx][1]; + mode_ctx = (mode_context >> REFMV_OFFSET) & REFMV_CTX_MASK; + + if (mode_context & (1 << SKIP_NEARESTMV_OFFSET)) mode_ctx = 6; + if (mode_context & (1 << SKIP_NEARMV_OFFSET)) mode_ctx = 7; + if (mode_context & (1 << SKIP_NEARESTMV_SUB8X8_OFFSET)) mode_ctx = 8; + + mode_cost += cpi->refmv_mode_cost[mode_ctx][mode != NEARESTMV]; + return mode_cost; + } + } +#else + assert(is_inter_mode(mode)); + return cpi->inter_mode_cost[mode_context][INTER_OFFSET(mode)]; +#endif // CONFIG_REF_MV +} + +#if CONFIG_EXT_INTER +static int get_interinter_compound_type_bits(BLOCK_SIZE bsize, + COMPOUND_TYPE comp_type) { + (void)bsize; + switch (comp_type) { + case COMPOUND_AVERAGE: return 0; +#if CONFIG_WEDGE + case COMPOUND_WEDGE: return get_interinter_wedge_bits(bsize); +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + case COMPOUND_SEG: return 1; +#endif // CONFIG_COMPOUND_SEGMENT + default: assert(0); return 0; + } +} +#endif // CONFIG_EXT_INTER + +static int set_and_cost_bmi_mvs( + const AV1_COMP *const cpi, MACROBLOCK *x, MACROBLOCKD *xd, int i, + PREDICTION_MODE mode, int_mv this_mv[2], + int_mv frame_mv[MB_MODE_COUNT][TOTAL_REFS_PER_FRAME], + int_mv seg_mvs[TOTAL_REFS_PER_FRAME], +#if CONFIG_EXT_INTER + int_mv compound_seg_newmvs[2], +#endif // CONFIG_EXT_INTER + int_mv *best_ref_mv[2], const int *mvjcost, int *mvcost[2], int mi_row, + int mi_col) { + MODE_INFO *const mic = xd->mi[0]; + const MB_MODE_INFO *const mbmi = &mic->mbmi; + const MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; + int thismvcost = 0; + int idx, idy; + const int num_4x4_blocks_wide = num_4x4_blocks_wide_lookup[mbmi->sb_type]; + const int num_4x4_blocks_high = num_4x4_blocks_high_lookup[mbmi->sb_type]; + const int is_compound = has_second_ref(mbmi); + int mode_ctx; + (void)mi_row; + (void)mi_col; + + switch (mode) { + case NEWMV: this_mv[0].as_int = seg_mvs[mbmi->ref_frame[0]].as_int; +#if CONFIG_EXT_INTER + if (!cpi->common.allow_high_precision_mv) + lower_mv_precision(&this_mv[0].as_mv, 0); +#endif // CONFIG_EXT_INTER + +#if CONFIG_REF_MV + for (idx = 0; idx < 1 + is_compound; ++idx) { + this_mv[idx] = seg_mvs[mbmi->ref_frame[idx]]; + av1_set_mvcost(x, mbmi->ref_frame[idx], idx, mbmi->ref_mv_idx); + thismvcost += + av1_mv_bit_cost(&this_mv[idx].as_mv, &best_ref_mv[idx]->as_mv, + x->nmvjointcost, x->mvcost, MV_COST_WEIGHT_SUB); + } + (void)mvjcost; + (void)mvcost; +#else + thismvcost += av1_mv_bit_cost(&this_mv[0].as_mv, &best_ref_mv[0]->as_mv, + mvjcost, mvcost, MV_COST_WEIGHT_SUB); +#if !CONFIG_EXT_INTER + if (is_compound) { + this_mv[1].as_int = seg_mvs[mbmi->ref_frame[1]].as_int; + thismvcost += av1_mv_bit_cost(&this_mv[1].as_mv, &best_ref_mv[1]->as_mv, + mvjcost, mvcost, MV_COST_WEIGHT_SUB); + } +#endif // !CONFIG_EXT_INTER +#endif // CONFIG_REF_MV + break; + case NEARMV: + case NEARESTMV: + this_mv[0].as_int = frame_mv[mode][mbmi->ref_frame[0]].as_int; + if (is_compound) + this_mv[1].as_int = frame_mv[mode][mbmi->ref_frame[1]].as_int; + break; + case ZEROMV: { + int ref; + for (ref = 0; ref < 1 + is_compound; ++ref) { +#if CONFIG_GLOBAL_MOTION + this_mv[ref].as_int = + gm_get_motion_vector( + &cpi->common.global_motion[mbmi->ref_frame[ref]], + cpi->common.allow_high_precision_mv, mbmi->sb_type, mi_col, + mi_row, i) + .as_int; +#else + this_mv[ref].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION + } + break; + } +#if CONFIG_EXT_INTER + case NEW_NEWMV: + if (compound_seg_newmvs[0].as_int == INVALID_MV || + compound_seg_newmvs[1].as_int == INVALID_MV) { + this_mv[0].as_int = seg_mvs[mbmi->ref_frame[0]].as_int; + this_mv[1].as_int = seg_mvs[mbmi->ref_frame[1]].as_int; + } else { + this_mv[0].as_int = compound_seg_newmvs[0].as_int; + this_mv[1].as_int = compound_seg_newmvs[1].as_int; + } + if (!cpi->common.allow_high_precision_mv) + lower_mv_precision(&this_mv[0].as_mv, 0); + if (!cpi->common.allow_high_precision_mv) + lower_mv_precision(&this_mv[1].as_mv, 0); +#if CONFIG_REF_MV + av1_set_mvcost(x, mbmi->ref_frame[0], 0, mbmi->ref_mv_idx); +#endif + thismvcost += av1_mv_bit_cost(&this_mv[0].as_mv, &best_ref_mv[0]->as_mv, + mvjcost, mvcost, MV_COST_WEIGHT_SUB); +#if CONFIG_REF_MV + av1_set_mvcost(x, mbmi->ref_frame[1], 1, mbmi->ref_mv_idx); +#endif + thismvcost += av1_mv_bit_cost(&this_mv[1].as_mv, &best_ref_mv[1]->as_mv, + mvjcost, mvcost, MV_COST_WEIGHT_SUB); + break; + case NEW_NEARMV: + case NEW_NEARESTMV: + this_mv[0].as_int = seg_mvs[mbmi->ref_frame[0]].as_int; + if (!cpi->common.allow_high_precision_mv) + lower_mv_precision(&this_mv[0].as_mv, 0); +#if CONFIG_REF_MV + av1_set_mvcost(x, mbmi->ref_frame[0], 0, mbmi->ref_mv_idx); +#endif + thismvcost += av1_mv_bit_cost(&this_mv[0].as_mv, &best_ref_mv[0]->as_mv, + mvjcost, mvcost, MV_COST_WEIGHT_SUB); + this_mv[1].as_int = frame_mv[mode][mbmi->ref_frame[1]].as_int; + break; + case NEAR_NEWMV: + case NEAREST_NEWMV: + this_mv[0].as_int = frame_mv[mode][mbmi->ref_frame[0]].as_int; + this_mv[1].as_int = seg_mvs[mbmi->ref_frame[1]].as_int; + if (!cpi->common.allow_high_precision_mv) + lower_mv_precision(&this_mv[1].as_mv, 0); +#if CONFIG_REF_MV + av1_set_mvcost(x, mbmi->ref_frame[1], 1, mbmi->ref_mv_idx); +#endif + thismvcost += av1_mv_bit_cost(&this_mv[1].as_mv, &best_ref_mv[1]->as_mv, + mvjcost, mvcost, MV_COST_WEIGHT_SUB); + break; + case NEAREST_NEARMV: + case NEAR_NEARESTMV: + case NEAREST_NEARESTMV: + case NEAR_NEARMV: + this_mv[0].as_int = frame_mv[mode][mbmi->ref_frame[0]].as_int; + this_mv[1].as_int = frame_mv[mode][mbmi->ref_frame[1]].as_int; + break; + case ZERO_ZEROMV: +#if CONFIG_GLOBAL_MOTION + this_mv[0].as_int = + gm_get_motion_vector(&cpi->common.global_motion[mbmi->ref_frame[0]], + cpi->common.allow_high_precision_mv, + mbmi->sb_type, mi_col, mi_row, i) + .as_int; + this_mv[1].as_int = + gm_get_motion_vector(&cpi->common.global_motion[mbmi->ref_frame[1]], + cpi->common.allow_high_precision_mv, + mbmi->sb_type, mi_col, mi_row, i) + .as_int; +#else + this_mv[0].as_int = 0; + this_mv[1].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION + break; +#endif // CONFIG_EXT_INTER + default: break; + } + + mic->bmi[i].as_mv[0].as_int = this_mv[0].as_int; + if (is_compound) mic->bmi[i].as_mv[1].as_int = this_mv[1].as_int; + + mic->bmi[i].as_mode = mode; + +#if CONFIG_REF_MV + if (mode == NEWMV) { + mic->bmi[i].pred_mv[0].as_int = + mbmi_ext->ref_mvs[mbmi->ref_frame[0]][0].as_int; + if (is_compound) + mic->bmi[i].pred_mv[1].as_int = + mbmi_ext->ref_mvs[mbmi->ref_frame[1]][0].as_int; + } else { + mic->bmi[i].pred_mv[0].as_int = this_mv[0].as_int; + if (is_compound) mic->bmi[i].pred_mv[1].as_int = this_mv[1].as_int; + } +#endif // CONFIG_REF_MV + + for (idy = 0; idy < num_4x4_blocks_high; ++idy) + for (idx = 0; idx < num_4x4_blocks_wide; ++idx) + memmove(&mic->bmi[i + idy * 2 + idx], &mic->bmi[i], sizeof(mic->bmi[i])); + +#if CONFIG_REF_MV +#if CONFIG_EXT_INTER + if (is_compound) + mode_ctx = mbmi_ext->compound_mode_context[mbmi->ref_frame[0]]; + else +#endif // CONFIG_EXT_INTER + mode_ctx = av1_mode_context_analyzer(mbmi_ext->mode_context, + mbmi->ref_frame, mbmi->sb_type, i); +#else // CONFIG_REF_MV + mode_ctx = mbmi_ext->mode_context[mbmi->ref_frame[0]]; +#endif // CONFIG_REF_MV + return cost_mv_ref(cpi, mode, mode_ctx) + thismvcost; +} + +static int64_t encode_inter_mb_segment_sub8x8( + const AV1_COMP *const cpi, MACROBLOCK *x, int64_t best_yrd, int i, + int *labelyrate, int64_t *distortion, int64_t *sse, ENTROPY_CONTEXT *ta, + ENTROPY_CONTEXT *tl, int ir, int ic, int mi_row, int mi_col) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *xd = &x->e_mbd; + struct macroblockd_plane *const pd = &xd->plane[0]; + struct macroblock_plane *const p = &x->plane[0]; + MODE_INFO *const mi = xd->mi[0]; + const BLOCK_SIZE plane_bsize = get_plane_block_size(mi->mbmi.sb_type, pd); + const int txb_width = max_block_wide(xd, plane_bsize, 0); + const int txb_height = max_block_high(xd, plane_bsize, 0); + const int width = block_size_wide[plane_bsize]; + const int height = block_size_high[plane_bsize]; + int idx, idy; + const uint8_t *const src = + &p->src.buf[av1_raster_block_offset(BLOCK_8X8, i, p->src.stride)]; + uint8_t *const dst = + &pd->dst.buf[av1_raster_block_offset(BLOCK_8X8, i, pd->dst.stride)]; + int64_t thisdistortion = 0, thissse = 0; + int thisrate = 0; + TX_SIZE tx_size = mi->mbmi.tx_size; + TX_TYPE tx_type = get_tx_type(PLANE_TYPE_Y, xd, i, tx_size); + const int num_4x4_w = tx_size_wide_unit[tx_size]; + const int num_4x4_h = tx_size_high_unit[tx_size]; +#if !CONFIG_PVQ + const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 1); +#else + (void)cpi; + (void)ta; + (void)tl; + (void)tx_type; +#endif // !CONFIG_PVQ + +#if CONFIG_EXT_TX && CONFIG_RECT_TX + assert(IMPLIES(xd->lossless[mi->mbmi.segment_id], tx_size == TX_4X4)); + assert(IMPLIES(!xd->lossless[mi->mbmi.segment_id], + tx_size == max_txsize_rect_lookup[mi->mbmi.sb_type])); +#else + assert(tx_size == TX_4X4); +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + + assert(tx_type == DCT_DCT); + + av1_build_inter_predictor_sub8x8(xd, 0, i, ir, ic, mi_row, mi_col); + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + aom_highbd_subtract_block( + height, width, av1_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff), + 8, src, p->src.stride, dst, pd->dst.stride, xd->bd); + } else { + aom_subtract_block(height, width, + av1_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff), + 8, src, p->src.stride, dst, pd->dst.stride); + } +#else + aom_subtract_block(height, width, + av1_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff), + 8, src, p->src.stride, dst, pd->dst.stride); +#endif // CONFIG_HIGHBITDEPTH + + for (idy = 0; idy < txb_height; idy += num_4x4_h) { + for (idx = 0; idx < txb_width; idx += num_4x4_w) { + int64_t dist, ssz, rd, rd1, rd2; + int coeff_ctx; + const int k = i + (idy * 2 + idx); + const int block = av1_raster_order_to_block_index(tx_size, k); + assert(IMPLIES(tx_size == TX_4X8 || tx_size == TX_8X4, + idx == 0 && idy == 0)); + coeff_ctx = combine_entropy_contexts(*(ta + (k & 1)), *(tl + (k >> 1))); + av1_xform_quant(cm, x, 0, block, idy + (i >> 1), idx + (i & 0x01), + BLOCK_8X8, tx_size, coeff_ctx, AV1_XFORM_QUANT_FP); + if (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0) + av1_optimize_b(cm, x, 0, block, tx_size, coeff_ctx); + av1_dist_block(cpi, x, 0, BLOCK_8X8, block, idy + (i >> 1), + idx + (i & 0x1), tx_size, &dist, &ssz, + OUTPUT_HAS_PREDICTED_PIXELS); + thisdistortion += dist; + thissse += ssz; +#if !CONFIG_PVQ + thisrate += + av1_cost_coeffs(cpi, x, 0, block, tx_size, scan_order, (ta + (k & 1)), + (tl + (k >> 1)), cpi->sf.use_fast_coef_costing); + *(ta + (k & 1)) = !(p->eobs[block] == 0); + *(tl + (k >> 1)) = !(p->eobs[block] == 0); +#else + thisrate += x->rate; +#endif // !CONFIG_PVQ +#if CONFIG_EXT_TX + if (tx_size == TX_8X4) { + *(ta + (k & 1) + 1) = *(ta + (k & 1)); + } + if (tx_size == TX_4X8) { + *(tl + (k >> 1) + 1) = *(tl + (k >> 1)); + } +#endif // CONFIG_EXT_TX + rd1 = RDCOST(x->rdmult, x->rddiv, thisrate, thisdistortion); + rd2 = RDCOST(x->rdmult, x->rddiv, 0, thissse); + rd = AOMMIN(rd1, rd2); + if (rd >= best_yrd) return INT64_MAX; + } + } + + *distortion = thisdistortion; + *labelyrate = thisrate; + *sse = thissse; + + return RDCOST(x->rdmult, x->rddiv, *labelyrate, *distortion); +} + +typedef struct { + int eobs; + int brate; + int byrate; + int64_t bdist; + int64_t bsse; + int64_t brdcost; + int_mv mvs[2]; +#if CONFIG_REF_MV + int_mv pred_mv[2]; +#endif // CONFIG_REF_MV +#if CONFIG_EXT_INTER + int_mv ref_mv[2]; +#endif // CONFIG_EXT_INTER + +#if CONFIG_CB4X4 + ENTROPY_CONTEXT ta[4]; + ENTROPY_CONTEXT tl[4]; +#else + ENTROPY_CONTEXT ta[2]; + ENTROPY_CONTEXT tl[2]; +#endif // CONFIG_CB4X4 +} SEG_RDSTAT; + +typedef struct { + int_mv *ref_mv[2]; + int_mv mvp; + + int64_t segment_rd; + int r; + int64_t d; + int64_t sse; + int segment_yrate; + PREDICTION_MODE modes[4]; +#if CONFIG_EXT_INTER + SEG_RDSTAT rdstat[4][INTER_MODES + INTER_COMPOUND_MODES]; +#else + SEG_RDSTAT rdstat[4][INTER_MODES]; +#endif // CONFIG_EXT_INTER + int mvthresh; +} BEST_SEG_INFO; + +static INLINE int mv_check_bounds(const MvLimits *mv_limits, const MV *mv) { + return (mv->row >> 3) < mv_limits->row_min || + (mv->row >> 3) > mv_limits->row_max || + (mv->col >> 3) < mv_limits->col_min || + (mv->col >> 3) > mv_limits->col_max; +} + +static INLINE void mi_buf_shift(MACROBLOCK *x, int i) { + MB_MODE_INFO *const mbmi = &x->e_mbd.mi[0]->mbmi; + struct macroblock_plane *const p = &x->plane[0]; + struct macroblockd_plane *const pd = &x->e_mbd.plane[0]; + + p->src.buf = + &p->src.buf[av1_raster_block_offset(BLOCK_8X8, i, p->src.stride)]; + assert(((intptr_t)pd->pre[0].buf & 0x7) == 0); + pd->pre[0].buf = + &pd->pre[0].buf[av1_raster_block_offset(BLOCK_8X8, i, pd->pre[0].stride)]; + if (has_second_ref(mbmi)) + pd->pre[1].buf = + &pd->pre[1] + .buf[av1_raster_block_offset(BLOCK_8X8, i, pd->pre[1].stride)]; +} + +static INLINE void mi_buf_restore(MACROBLOCK *x, struct buf_2d orig_src, + struct buf_2d orig_pre[2]) { + MB_MODE_INFO *mbmi = &x->e_mbd.mi[0]->mbmi; + x->plane[0].src = orig_src; + x->e_mbd.plane[0].pre[0] = orig_pre[0]; + if (has_second_ref(mbmi)) x->e_mbd.plane[0].pre[1] = orig_pre[1]; +} + +// Check if NEARESTMV/NEARMV/ZEROMV is the cheapest way encode zero motion. +// TODO(aconverse): Find out if this is still productive then clean up or remove +static int check_best_zero_mv( + const AV1_COMP *const cpi, const int16_t mode_context[TOTAL_REFS_PER_FRAME], +#if CONFIG_REF_MV && CONFIG_EXT_INTER + const int16_t compound_mode_context[TOTAL_REFS_PER_FRAME], +#endif // CONFIG_REF_MV && CONFIG_EXT_INTER + int_mv frame_mv[MB_MODE_COUNT][TOTAL_REFS_PER_FRAME], int this_mode, + const MV_REFERENCE_FRAME ref_frames[2], const BLOCK_SIZE bsize, int block, + int mi_row, int mi_col) { + int_mv zeromv[2]; + int comp_pred_mode = ref_frames[1] > INTRA_FRAME; + int cur_frm; + (void)mi_row; + (void)mi_col; + for (cur_frm = 0; cur_frm < 1 + comp_pred_mode; cur_frm++) { +#if CONFIG_GLOBAL_MOTION + if (this_mode == ZEROMV +#if CONFIG_EXT_INTER + || this_mode == ZERO_ZEROMV +#endif // CONFIG_EXT_INTER + ) + zeromv[cur_frm].as_int = + gm_get_motion_vector(&cpi->common.global_motion[ref_frames[cur_frm]], + cpi->common.allow_high_precision_mv, bsize, + mi_col, mi_row, block) + .as_int; + else +#endif // CONFIG_GLOBAL_MOTION + zeromv[cur_frm].as_int = 0; + } +#if !CONFIG_EXT_INTER + assert(ref_frames[1] != INTRA_FRAME); // Just sanity check +#endif // !CONFIG_EXT_INTER + if ((this_mode == NEARMV || this_mode == NEARESTMV || this_mode == ZEROMV) && + frame_mv[this_mode][ref_frames[0]].as_int == zeromv[0].as_int && + (ref_frames[1] <= INTRA_FRAME || + frame_mv[this_mode][ref_frames[1]].as_int == zeromv[1].as_int)) { +#if CONFIG_REF_MV + int16_t rfc = + av1_mode_context_analyzer(mode_context, ref_frames, bsize, block); +#else + int16_t rfc = mode_context[ref_frames[0]]; +#endif // CONFIG_REF_MV + int c1 = cost_mv_ref(cpi, NEARMV, rfc); + int c2 = cost_mv_ref(cpi, NEARESTMV, rfc); + int c3 = cost_mv_ref(cpi, ZEROMV, rfc); + +#if !CONFIG_REF_MV + (void)bsize; + (void)block; +#endif // !CONFIG_REF_MV + + if (this_mode == NEARMV) { + if (c1 > c3) return 0; + } else if (this_mode == NEARESTMV) { + if (c2 > c3) return 0; + } else { + assert(this_mode == ZEROMV); + if (ref_frames[1] <= INTRA_FRAME) { + if ((c3 >= c2 && frame_mv[NEARESTMV][ref_frames[0]].as_int == 0) || + (c3 >= c1 && frame_mv[NEARMV][ref_frames[0]].as_int == 0)) + return 0; + } else { + if ((c3 >= c2 && frame_mv[NEARESTMV][ref_frames[0]].as_int == 0 && + frame_mv[NEARESTMV][ref_frames[1]].as_int == 0) || + (c3 >= c1 && frame_mv[NEARMV][ref_frames[0]].as_int == 0 && + frame_mv[NEARMV][ref_frames[1]].as_int == 0)) + return 0; + } + } + } +#if CONFIG_EXT_INTER + else if ((this_mode == NEAREST_NEARESTMV || this_mode == NEAREST_NEARMV || + this_mode == NEAR_NEARESTMV || this_mode == NEAR_NEARMV || + this_mode == ZERO_ZEROMV) && + frame_mv[this_mode][ref_frames[0]].as_int == zeromv[0].as_int && + frame_mv[this_mode][ref_frames[1]].as_int == zeromv[1].as_int) { +#if CONFIG_REF_MV + int16_t rfc = compound_mode_context[ref_frames[0]]; +#else + int16_t rfc = mode_context[ref_frames[0]]; +#endif // CONFIG_REF_MV + int c1 = cost_mv_ref(cpi, NEAREST_NEARMV, rfc); + int c2 = cost_mv_ref(cpi, NEAREST_NEARESTMV, rfc); + int c3 = cost_mv_ref(cpi, ZERO_ZEROMV, rfc); + int c4 = cost_mv_ref(cpi, NEAR_NEARESTMV, rfc); + int c5 = cost_mv_ref(cpi, NEAR_NEARMV, rfc); + + if (this_mode == NEAREST_NEARMV) { + if (c1 > c3) return 0; + } else if (this_mode == NEAREST_NEARESTMV) { + if (c2 > c3) return 0; + } else if (this_mode == NEAR_NEARESTMV) { + if (c4 > c3) return 0; + } else if (this_mode == NEAR_NEARMV) { + if (c5 > c3) return 0; + } else { + assert(this_mode == ZERO_ZEROMV); + if ((c3 >= c2 && frame_mv[NEAREST_NEARESTMV][ref_frames[0]].as_int == 0 && + frame_mv[NEAREST_NEARESTMV][ref_frames[1]].as_int == 0) || + (c3 >= c1 && frame_mv[NEAREST_NEARMV][ref_frames[0]].as_int == 0 && + frame_mv[NEAREST_NEARMV][ref_frames[1]].as_int == 0) || + (c3 >= c5 && frame_mv[NEAR_NEARMV][ref_frames[0]].as_int == 0 && + frame_mv[NEAR_NEARMV][ref_frames[1]].as_int == 0) || + (c3 >= c4 && frame_mv[NEAR_NEARESTMV][ref_frames[0]].as_int == 0 && + frame_mv[NEAR_NEARESTMV][ref_frames[1]].as_int == 0)) + return 0; + } + } +#endif // CONFIG_EXT_INTER + return 1; +} + +static void joint_motion_search(const AV1_COMP *cpi, MACROBLOCK *x, + BLOCK_SIZE bsize, int_mv *frame_mv, int mi_row, + int mi_col, +#if CONFIG_EXT_INTER + int_mv *ref_mv_sub8x8[2], +#endif // CONFIG_EXT_INTER + int *rate_mv, const int block) { + const AV1_COMMON *const cm = &cpi->common; + const int pw = block_size_wide[bsize]; + const int ph = block_size_high[bsize]; + MACROBLOCKD *xd = &x->e_mbd; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + // This function should only ever be called for compound modes + assert(has_second_ref(mbmi)); + const int refs[2] = { mbmi->ref_frame[0], mbmi->ref_frame[1] }; + int_mv ref_mv[2]; + int ite, ref; +#if CONFIG_DUAL_FILTER + InterpFilter interp_filter[4] = { + mbmi->interp_filter[0], mbmi->interp_filter[1], mbmi->interp_filter[2], + mbmi->interp_filter[3], + }; +#else + const InterpFilter interp_filter = mbmi->interp_filter; +#endif // CONFIG_DUAL_FILTER + struct scale_factors sf; + struct macroblockd_plane *const pd = &xd->plane[0]; +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + // ic and ir are the 4x4 coordiantes of the sub8x8 at index "block" + const int ic = block & 1; + const int ir = (block - ic) >> 1; + const int p_col = ((mi_col * MI_SIZE) >> pd->subsampling_x) + 4 * ic; + const int p_row = ((mi_row * MI_SIZE) >> pd->subsampling_y) + 4 * ir; +#if CONFIG_GLOBAL_MOTION + int is_global[2]; + for (ref = 0; ref < 2; ++ref) { + WarpedMotionParams *const wm = + &xd->global_motion[xd->mi[0]->mbmi.ref_frame[ref]]; + is_global[ref] = is_global_mv_block(xd->mi[0], block, wm->wmtype); + } +#endif // CONFIG_GLOBAL_MOTION +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + + // Do joint motion search in compound mode to get more accurate mv. + struct buf_2d backup_yv12[2][MAX_MB_PLANE]; + int last_besterr[2] = { INT_MAX, INT_MAX }; + const YV12_BUFFER_CONFIG *const scaled_ref_frame[2] = { + av1_get_scaled_ref_frame(cpi, refs[0]), + av1_get_scaled_ref_frame(cpi, refs[1]) + }; + +// Prediction buffer from second frame. +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, second_pred_alloc_16[MAX_SB_SQUARE]); + uint8_t *second_pred; +#else + DECLARE_ALIGNED(16, uint8_t, second_pred[MAX_SB_SQUARE]); +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_EXT_INTER && CONFIG_CB4X4 + (void)ref_mv_sub8x8; +#endif // CONFIG_EXT_INTER && CONFIG_CB4X4 + + for (ref = 0; ref < 2; ++ref) { +#if CONFIG_EXT_INTER && !CONFIG_CB4X4 + if (bsize < BLOCK_8X8 && ref_mv_sub8x8 != NULL) + ref_mv[ref].as_int = ref_mv_sub8x8[ref]->as_int; + else +#endif // CONFIG_EXT_INTER && !CONFIG_CB4X4 + ref_mv[ref] = x->mbmi_ext->ref_mvs[refs[ref]][0]; + + if (scaled_ref_frame[ref]) { + int i; + // Swap out the reference frame for a version that's been scaled to + // match the resolution of the current frame, allowing the existing + // motion search code to be used without additional modifications. + for (i = 0; i < MAX_MB_PLANE; i++) + backup_yv12[ref][i] = xd->plane[i].pre[ref]; + av1_setup_pre_planes(xd, ref, scaled_ref_frame[ref], mi_row, mi_col, + NULL); + } + } + +// Since we have scaled the reference frames to match the size of the current +// frame we must use a unit scaling factor during mode selection. +#if CONFIG_HIGHBITDEPTH + av1_setup_scale_factors_for_frame(&sf, cm->width, cm->height, cm->width, + cm->height, cm->use_highbitdepth); +#else + av1_setup_scale_factors_for_frame(&sf, cm->width, cm->height, cm->width, + cm->height); +#endif // CONFIG_HIGHBITDEPTH + + // Allow joint search multiple times iteratively for each reference frame + // and break out of the search loop if it couldn't find a better mv. + for (ite = 0; ite < 4; ite++) { + struct buf_2d ref_yv12[2]; + int bestsme = INT_MAX; + int sadpb = x->sadperbit16; + MV *const best_mv = &x->best_mv.as_mv; + int search_range = 3; + + MvLimits tmp_mv_limits = x->mv_limits; + int id = ite % 2; // Even iterations search in the first reference frame, + // odd iterations search in the second. The predictor + // found for the 'other' reference frame is factored in. + const int plane = 0; + ConvolveParams conv_params = get_conv_params(0, plane); +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + WarpTypesAllowed warp_types; +#if CONFIG_GLOBAL_MOTION + warp_types.global_warp_allowed = is_global[!id]; +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_WARPED_MOTION + warp_types.local_warp_allowed = mbmi->motion_mode == WARPED_CAUSAL; +#endif // CONFIG_WARPED_MOTION +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + + // Initialized here because of compiler problem in Visual Studio. + ref_yv12[0] = xd->plane[plane].pre[0]; + ref_yv12[1] = xd->plane[plane].pre[1]; + +#if CONFIG_DUAL_FILTER + // reload the filter types + interp_filter[0] = + (id == 0) ? mbmi->interp_filter[2] : mbmi->interp_filter[0]; + interp_filter[1] = + (id == 0) ? mbmi->interp_filter[3] : mbmi->interp_filter[1]; +#endif // CONFIG_DUAL_FILTER + +// Get the prediction block from the 'other' reference frame. +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + second_pred = CONVERT_TO_BYTEPTR(second_pred_alloc_16); + av1_highbd_build_inter_predictor( + ref_yv12[!id].buf, ref_yv12[!id].stride, second_pred, pw, + &frame_mv[refs[!id]].as_mv, &sf, pw, ph, 0, interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, p_col, p_row, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + plane, MV_PRECISION_Q3, mi_col * MI_SIZE, mi_row * MI_SIZE, xd); + } else { + second_pred = (uint8_t *)second_pred_alloc_16; +#endif // CONFIG_HIGHBITDEPTH + av1_build_inter_predictor( + ref_yv12[!id].buf, ref_yv12[!id].stride, second_pred, pw, + &frame_mv[refs[!id]].as_mv, &sf, pw, ph, &conv_params, interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, p_col, p_row, plane, !id, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + MV_PRECISION_Q3, mi_col * MI_SIZE, mi_row * MI_SIZE, xd); +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + + // Do compound motion search on the current reference frame. + if (id) xd->plane[plane].pre[0] = ref_yv12[id]; + av1_set_mv_search_range(&x->mv_limits, &ref_mv[id].as_mv); + + // Use the mv result from the single mode as mv predictor. + *best_mv = frame_mv[refs[id]].as_mv; + + best_mv->col >>= 3; + best_mv->row >>= 3; + +#if CONFIG_REF_MV + av1_set_mvcost(x, refs[id], id, mbmi->ref_mv_idx); +#endif // CONFIG_REF_MV + + // Small-range full-pixel motion search. + bestsme = + av1_refining_search_8p_c(x, sadpb, search_range, &cpi->fn_ptr[bsize], + &ref_mv[id].as_mv, second_pred); + if (bestsme < INT_MAX) + bestsme = av1_get_mvpred_av_var(x, best_mv, &ref_mv[id].as_mv, + second_pred, &cpi->fn_ptr[bsize], 1); + + x->mv_limits = tmp_mv_limits; + + if (bestsme < INT_MAX) { + int dis; /* TODO: use dis in distortion calculation later. */ + unsigned int sse; + if (cpi->sf.use_upsampled_references) { + // Use up-sampled reference frames. + struct buf_2d backup_pred = pd->pre[0]; + const YV12_BUFFER_CONFIG *upsampled_ref = + get_upsampled_ref(cpi, refs[id]); + + // Set pred for Y plane + setup_pred_plane(&pd->pre[0], bsize, upsampled_ref->y_buffer, + upsampled_ref->y_crop_width, + upsampled_ref->y_crop_height, upsampled_ref->y_stride, + (mi_row << 3), (mi_col << 3), NULL, pd->subsampling_x, + pd->subsampling_y); + +// If bsize < BLOCK_8X8, adjust pred pointer for this block +#if !CONFIG_CB4X4 + if (bsize < BLOCK_8X8) + pd->pre[0].buf = + &pd->pre[0].buf[(av1_raster_block_offset(BLOCK_8X8, block, + pd->pre[0].stride)) + << 3]; +#endif // !CONFIG_CB4X4 + + bestsme = cpi->find_fractional_mv_step( + x, &ref_mv[id].as_mv, cpi->common.allow_high_precision_mv, + x->errorperbit, &cpi->fn_ptr[bsize], 0, + cpi->sf.mv.subpel_iters_per_step, NULL, x->nmvjointcost, x->mvcost, + &dis, &sse, second_pred, pw, ph, 1); + + // Restore the reference frames. + pd->pre[0] = backup_pred; + } else { + (void)block; + bestsme = cpi->find_fractional_mv_step( + x, &ref_mv[id].as_mv, cpi->common.allow_high_precision_mv, + x->errorperbit, &cpi->fn_ptr[bsize], 0, + cpi->sf.mv.subpel_iters_per_step, NULL, x->nmvjointcost, x->mvcost, + &dis, &sse, second_pred, pw, ph, 0); + } + } + + // Restore the pointer to the first (possibly scaled) prediction buffer. + if (id) xd->plane[plane].pre[0] = ref_yv12[0]; + + if (bestsme < last_besterr[id]) { + frame_mv[refs[id]].as_mv = *best_mv; + last_besterr[id] = bestsme; + } else { + break; + } + } + + *rate_mv = 0; + + for (ref = 0; ref < 2; ++ref) { + if (scaled_ref_frame[ref]) { + // Restore the prediction frame pointers to their unscaled versions. + int i; + for (i = 0; i < MAX_MB_PLANE; i++) + xd->plane[i].pre[ref] = backup_yv12[ref][i]; + } +#if CONFIG_REF_MV + av1_set_mvcost(x, refs[ref], ref, mbmi->ref_mv_idx); +#endif // CONFIG_REF_MV +#if CONFIG_EXT_INTER && !CONFIG_CB4X4 + if (bsize >= BLOCK_8X8) +#endif // CONFIG_EXT_INTER && !CONFIG_CB4X4 + *rate_mv += av1_mv_bit_cost(&frame_mv[refs[ref]].as_mv, + &x->mbmi_ext->ref_mvs[refs[ref]][0].as_mv, + x->nmvjointcost, x->mvcost, MV_COST_WEIGHT); +#if CONFIG_EXT_INTER && !CONFIG_CB4X4 + else + *rate_mv += av1_mv_bit_cost(&frame_mv[refs[ref]].as_mv, + &ref_mv_sub8x8[ref]->as_mv, x->nmvjointcost, + x->mvcost, MV_COST_WEIGHT); +#endif // CONFIG_EXT_INTER && !CONFIG_CB4X4 + } +} + +#if CONFIG_REF_MV && !CONFIG_EXT_INTER +static void update_mv_search_and_seg_mvs( + int *const run_mv_search, int_mv *const seg_mvs, int has_second_rf, + const MV_REFERENCE_FRAME *const ref_frame, + const SEG_RDSTAT *const ref_rdstat, int_mv *const bsi_ref_mv[2]) { + if (has_second_rf) { + if (seg_mvs[ref_frame[0]].as_int == ref_rdstat->mvs[0].as_int && + ref_rdstat->mvs[0].as_int != INVALID_MV) + if (bsi_ref_mv[0]->as_int == ref_rdstat->pred_mv[0].as_int) + --*run_mv_search; + + if (seg_mvs[ref_frame[1]].as_int == ref_rdstat->mvs[1].as_int && + ref_rdstat->mvs[1].as_int != INVALID_MV) + if (bsi_ref_mv[1]->as_int == ref_rdstat->pred_mv[1].as_int) + --*run_mv_search; + } else { + if (bsi_ref_mv[0]->as_int == ref_rdstat->pred_mv[0].as_int && + ref_rdstat->mvs[0].as_int != INVALID_MV) { + *run_mv_search = 0; + seg_mvs[ref_frame[0]].as_int = ref_rdstat->mvs[0].as_int; + } + } +} +#endif // CONFIG_REF_MV && !CONFIG_EXT_INTER + +static int64_t rd_pick_inter_best_sub8x8_mode( + const AV1_COMP *const cpi, MACROBLOCK *x, int_mv *best_ref_mv, + int_mv *second_best_ref_mv, int64_t best_rd, int *returntotrate, + int *returnyrate, int64_t *returndistortion, int *skippable, int64_t *psse, + int mvthresh, int_mv seg_mvs[4][TOTAL_REFS_PER_FRAME], +#if CONFIG_EXT_INTER + int_mv compound_seg_newmvs[4][2], +#endif // CONFIG_EXT_INTER + BEST_SEG_INFO *bsi_buf, int filter_idx, int mi_row, int mi_col) { + BEST_SEG_INFO *bsi = bsi_buf + filter_idx; +#if CONFIG_REF_MV + int_mv tmp_ref_mv[2]; +#endif // CONFIG_REF_MV + MACROBLOCKD *xd = &x->e_mbd; + MODE_INFO *mi = xd->mi[0]; + MB_MODE_INFO *mbmi = &mi->mbmi; + int mode_idx; + int k, br = 0, idx, idy; + int64_t bd = 0, block_sse = 0; + PREDICTION_MODE this_mode; + const AV1_COMMON *cm = &cpi->common; + struct macroblock_plane *const p = &x->plane[0]; + struct macroblockd_plane *const pd = &xd->plane[0]; + const int label_count = 4; + int64_t this_segment_rd = 0; + int label_mv_thresh; + int segmentyrate = 0; + const BLOCK_SIZE bsize = mbmi->sb_type; + const int num_4x4_blocks_wide = num_4x4_blocks_wide_lookup[bsize]; + const int num_4x4_blocks_high = num_4x4_blocks_high_lookup[bsize]; +#if CONFIG_CB4X4 + ENTROPY_CONTEXT t_above[4], t_left[4]; +#else + ENTROPY_CONTEXT t_above[2], t_left[2]; +#endif // CONFIG_CB4X4 + int subpelmv = 1, have_ref = 0; + const int has_second_rf = has_second_ref(mbmi); + const int inter_mode_mask = cpi->sf.inter_mode_mask[bsize]; + MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; +#if CONFIG_PVQ + od_rollback_buffer pre_buf; + + od_encode_checkpoint(&x->daala_enc, &pre_buf); +#endif // CONFIG_PVQ +#if CONFIG_EXT_TX && CONFIG_RECT_TX + mbmi->tx_size = + xd->lossless[mbmi->segment_id] ? TX_4X4 : max_txsize_rect_lookup[bsize]; +#else + mbmi->tx_size = TX_4X4; +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX + + av1_zero(*bsi); + + bsi->segment_rd = best_rd; + bsi->ref_mv[0] = best_ref_mv; + bsi->ref_mv[1] = second_best_ref_mv; + bsi->mvp.as_int = best_ref_mv->as_int; + bsi->mvthresh = mvthresh; + + for (idx = 0; idx < 4; ++idx) bsi->modes[idx] = ZEROMV; + +#if CONFIG_REF_MV + for (idx = 0; idx < 4; ++idx) { + for (k = NEARESTMV; k <= NEWMV; ++k) { + bsi->rdstat[idx][INTER_OFFSET(k)].pred_mv[0].as_int = INVALID_MV; + bsi->rdstat[idx][INTER_OFFSET(k)].pred_mv[1].as_int = INVALID_MV; + + bsi->rdstat[idx][INTER_OFFSET(k)].mvs[0].as_int = INVALID_MV; + bsi->rdstat[idx][INTER_OFFSET(k)].mvs[1].as_int = INVALID_MV; + } + } +#endif // CONFIG_REF_MV + + memcpy(t_above, pd->above_context, sizeof(t_above)); + memcpy(t_left, pd->left_context, sizeof(t_left)); + + // 64 makes this threshold really big effectively + // making it so that we very rarely check mvs on + // segments. setting this to 1 would make mv thresh + // roughly equal to what it is for macroblocks + label_mv_thresh = 1 * bsi->mvthresh / label_count; + + // Segmentation method overheads + for (idy = 0; idy < 2; idy += num_4x4_blocks_high) { + for (idx = 0; idx < 2; idx += num_4x4_blocks_wide) { + // TODO(jingning,rbultje): rewrite the rate-distortion optimization + // loop for 4x4/4x8/8x4 block coding. to be replaced with new rd loop + int_mv mode_mv[MB_MODE_COUNT][2]; + int_mv frame_mv[MB_MODE_COUNT][TOTAL_REFS_PER_FRAME]; + PREDICTION_MODE mode_selected = ZEROMV; + int64_t new_best_rd = INT64_MAX; + const int index = idy * 2 + idx; + int ref; +#if CONFIG_REF_MV + CANDIDATE_MV ref_mv_stack[2][MAX_REF_MV_STACK_SIZE]; + uint8_t ref_mv_count[2]; +#endif // CONFIG_REF_MV +#if CONFIG_EXT_INTER + int_mv ref_mvs_sub8x8[2][2]; +#endif // CONFIG_EXT_INTER +#if CONFIG_PVQ + od_rollback_buffer idx_buf, post_buf; + od_encode_checkpoint(&x->daala_enc, &idx_buf); + od_encode_checkpoint(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + + for (ref = 0; ref < 1 + has_second_rf; ++ref) { + const MV_REFERENCE_FRAME frame = mbmi->ref_frame[ref]; +#if CONFIG_EXT_INTER + int_mv mv_ref_list[MAX_MV_REF_CANDIDATES]; + av1_update_mv_context(cm, xd, mi, frame, mv_ref_list, index, mi_row, + mi_col, NULL); +#endif // CONFIG_EXT_INTER +#if CONFIG_GLOBAL_MOTION + frame_mv[ZEROMV][frame].as_int = + gm_get_motion_vector(&cm->global_motion[frame], + cm->allow_high_precision_mv, mbmi->sb_type, + mi_col, mi_row, index) + .as_int; +#else // CONFIG_GLOBAL_MOTION + frame_mv[ZEROMV][frame].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION + av1_append_sub8x8_mvs_for_idx(cm, xd, index, ref, mi_row, mi_col, +#if CONFIG_REF_MV + ref_mv_stack[ref], &ref_mv_count[ref], +#endif // CONFIG_REF_MV +#if CONFIG_EXT_INTER + mv_ref_list, +#endif // CONFIG_EXT_INTER + &frame_mv[NEARESTMV][frame], + &frame_mv[NEARMV][frame]); + +#if CONFIG_REF_MV + tmp_ref_mv[ref] = frame_mv[NEARESTMV][mbmi->ref_frame[ref]]; + lower_mv_precision(&tmp_ref_mv[ref].as_mv, cm->allow_high_precision_mv); + bsi->ref_mv[ref] = &tmp_ref_mv[ref]; + mbmi_ext->ref_mvs[frame][0] = tmp_ref_mv[ref]; +#endif // CONFIG_REF_MV + +#if CONFIG_EXT_INTER + mv_ref_list[0].as_int = frame_mv[NEARESTMV][frame].as_int; + mv_ref_list[1].as_int = frame_mv[NEARMV][frame].as_int; + av1_find_best_ref_mvs(cm->allow_high_precision_mv, mv_ref_list, + &ref_mvs_sub8x8[0][ref], &ref_mvs_sub8x8[1][ref]); + + if (has_second_rf) { +#if CONFIG_GLOBAL_MOTION + frame_mv[ZERO_ZEROMV][frame].as_int = + gm_get_motion_vector(&cm->global_motion[frame], + cm->allow_high_precision_mv, mbmi->sb_type, + mi_col, mi_row, index) + .as_int; +#else + frame_mv[ZERO_ZEROMV][frame].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION + frame_mv[NEAREST_NEARESTMV][frame].as_int = + frame_mv[NEARESTMV][frame].as_int; + + if (ref == 0) { + frame_mv[NEAREST_NEARMV][frame].as_int = + frame_mv[NEARESTMV][frame].as_int; + frame_mv[NEAR_NEARESTMV][frame].as_int = + frame_mv[NEARMV][frame].as_int; + frame_mv[NEAREST_NEWMV][frame].as_int = + frame_mv[NEARESTMV][frame].as_int; + frame_mv[NEAR_NEWMV][frame].as_int = frame_mv[NEARMV][frame].as_int; + frame_mv[NEAR_NEARMV][frame].as_int = + frame_mv[NEARMV][frame].as_int; + } else if (ref == 1) { + frame_mv[NEAREST_NEARMV][frame].as_int = + frame_mv[NEARMV][frame].as_int; + frame_mv[NEAR_NEARESTMV][frame].as_int = + frame_mv[NEARESTMV][frame].as_int; + frame_mv[NEW_NEARESTMV][frame].as_int = + frame_mv[NEARESTMV][frame].as_int; + frame_mv[NEW_NEARMV][frame].as_int = frame_mv[NEARMV][frame].as_int; + frame_mv[NEAR_NEARMV][frame].as_int = + frame_mv[NEARMV][frame].as_int; + } + } +#endif // CONFIG_EXT_INTER + } + +// search for the best motion vector on this segment +#if CONFIG_EXT_INTER + for (this_mode = (has_second_rf ? NEAREST_NEARESTMV : NEARESTMV); + this_mode <= (has_second_rf ? NEW_NEWMV : NEWMV); ++this_mode) +#else + for (this_mode = NEARESTMV; this_mode <= NEWMV; ++this_mode) +#endif // CONFIG_EXT_INTER + { + const struct buf_2d orig_src = x->plane[0].src; + struct buf_2d orig_pre[2]; + // This flag controls if the motion estimation will kick off. When it + // is set to a non-zero value, the encoder will force motion estimation. + int run_mv_search = 0; + + mode_idx = INTER_OFFSET(this_mode); +#if CONFIG_EXT_INTER + for (ref = 0; ref < 1 + has_second_rf; ++ref) + bsi->ref_mv[ref]->as_int = ref_mvs_sub8x8[0][ref].as_int; +#endif // CONFIG_EXT_INTER + bsi->rdstat[index][mode_idx].brdcost = INT64_MAX; + if (!(inter_mode_mask & (1 << this_mode))) continue; + +#if CONFIG_REF_MV + run_mv_search = 2; +#if !CONFIG_EXT_INTER + if (filter_idx > 0 && this_mode == NEWMV) { + const BEST_SEG_INFO *ref_bsi = bsi_buf; + const SEG_RDSTAT *ref_rdstat = &ref_bsi->rdstat[index][mode_idx]; + + update_mv_search_and_seg_mvs(&run_mv_search, seg_mvs[index], + has_second_rf, mbmi->ref_frame, + ref_rdstat, bsi->ref_mv); + + if (run_mv_search != 0 && filter_idx > 1) { + ref_bsi = bsi_buf + 1; + ref_rdstat = &ref_bsi->rdstat[index][mode_idx]; + run_mv_search = 2; + update_mv_search_and_seg_mvs(&run_mv_search, seg_mvs[index], + has_second_rf, mbmi->ref_frame, + ref_rdstat, bsi->ref_mv); + } + } +#endif // !CONFIG_EXT_INTER +#endif // CONFIG_REF_MV + +#if CONFIG_GLOBAL_MOTION + if (cm->global_motion[mbmi->ref_frame[0]].wmtype == IDENTITY && + (!has_second_rf || + cm->global_motion[mbmi->ref_frame[1]].wmtype == IDENTITY)) +#endif // CONFIG_GLOBAL_MOTION + + if (!check_best_zero_mv(cpi, mbmi_ext->mode_context, +#if CONFIG_REF_MV && CONFIG_EXT_INTER + mbmi_ext->compound_mode_context, +#endif // CONFIG_REF_MV && CONFIG_EXT_INTER + frame_mv, this_mode, mbmi->ref_frame, bsize, + index, mi_row, mi_col)) + continue; + + memcpy(orig_pre, pd->pre, sizeof(orig_pre)); + memcpy(bsi->rdstat[index][mode_idx].ta, t_above, + sizeof(bsi->rdstat[index][mode_idx].ta)); + memcpy(bsi->rdstat[index][mode_idx].tl, t_left, + sizeof(bsi->rdstat[index][mode_idx].tl)); +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &idx_buf); +#endif // CONFIG_PVQ + + // motion search for newmv (single predictor case only) + if (!has_second_rf && +#if CONFIG_EXT_INTER + have_newmv_in_inter_mode(this_mode) && + (seg_mvs[index][mbmi->ref_frame[0]].as_int == INVALID_MV) +#else + this_mode == NEWMV && + (seg_mvs[index][mbmi->ref_frame[0]].as_int == INVALID_MV || + run_mv_search) +#endif // CONFIG_EXT_INTER + ) { + int step_param = 0; + int bestsme = INT_MAX; + int sadpb = x->sadperbit4; + MV mvp_full; + int max_mv; + int cost_list[5]; + MvLimits tmp_mv_limits = x->mv_limits; + + /* Is the best so far sufficiently good that we cant justify doing + * and new motion search. */ + if (new_best_rd < label_mv_thresh) break; + +#if CONFIG_EXT_INTER + bsi->mvp.as_int = bsi->ref_mv[0]->as_int; +#else +// use previous block's result as next block's MV predictor. +#if !CONFIG_REF_MV + if (index > 0) { + bsi->mvp.as_int = mi->bmi[index - 1].as_mv[0].as_int; + if (index == 2) + bsi->mvp.as_int = mi->bmi[index - 2].as_mv[0].as_int; + } +#endif // !CONFIG_REF_MV +#endif // CONFIG_EXT_INTER + max_mv = (index == 0) ? (int)x->max_mv_context[mbmi->ref_frame[0]] + : AOMMAX(abs(bsi->mvp.as_mv.row), + abs(bsi->mvp.as_mv.col)) >> + 3; + + if (cpi->sf.mv.auto_mv_step_size && cm->show_frame) { + // Take wtd average of the step_params based on the last frame's + // max mv magnitude and the best ref mvs of the current block for + // the given reference. + step_param = + (av1_init_search_range(max_mv) + cpi->mv_step_param) / 2; + } else { + step_param = cpi->mv_step_param; + } + +#if CONFIG_REF_MV + mvp_full.row = bsi->ref_mv[0]->as_mv.row >> 3; + mvp_full.col = bsi->ref_mv[0]->as_mv.col >> 3; +#else + mvp_full.row = bsi->mvp.as_mv.row >> 3; + mvp_full.col = bsi->mvp.as_mv.col >> 3; +#endif // CONFIG_REF_MV + + if (cpi->sf.adaptive_motion_search) { + mvp_full.row = x->pred_mv[mbmi->ref_frame[0]].row >> 3; + mvp_full.col = x->pred_mv[mbmi->ref_frame[0]].col >> 3; + step_param = AOMMAX(step_param, 8); + } + + // adjust src pointer for this block + mi_buf_shift(x, index); + + av1_set_mv_search_range(&x->mv_limits, &bsi->ref_mv[0]->as_mv); + + x->best_mv.as_int = x->second_best_mv.as_int = INVALID_MV; + +#if CONFIG_REF_MV + av1_set_mvcost(x, mbmi->ref_frame[0], 0, mbmi->ref_mv_idx); +#endif // CONFIG_REF_MV + bestsme = av1_full_pixel_search( + cpi, x, bsize, &mvp_full, step_param, sadpb, + cpi->sf.mv.subpel_search_method != SUBPEL_TREE ? cost_list : NULL, + &bsi->ref_mv[0]->as_mv, INT_MAX, 1); + + x->mv_limits = tmp_mv_limits; + + if (bestsme < INT_MAX) { + int distortion; + if (cpi->sf.use_upsampled_references) { + int best_mv_var; + const int try_second = + x->second_best_mv.as_int != INVALID_MV && + x->second_best_mv.as_int != x->best_mv.as_int; + const int pw = block_size_wide[bsize]; + const int ph = block_size_high[bsize]; + // Use up-sampled reference frames. + struct buf_2d backup_pred = pd->pre[0]; + const YV12_BUFFER_CONFIG *upsampled_ref = + get_upsampled_ref(cpi, mbmi->ref_frame[0]); + + // Set pred for Y plane + setup_pred_plane( + &pd->pre[0], bsize, upsampled_ref->y_buffer, + upsampled_ref->y_crop_width, upsampled_ref->y_crop_height, + upsampled_ref->y_stride, (mi_row << 3), (mi_col << 3), NULL, + pd->subsampling_x, pd->subsampling_y); + + // adjust pred pointer for this block + pd->pre[0].buf = + &pd->pre[0].buf[(av1_raster_block_offset(BLOCK_8X8, index, + pd->pre[0].stride)) + << 3]; + + best_mv_var = cpi->find_fractional_mv_step( + x, &bsi->ref_mv[0]->as_mv, cm->allow_high_precision_mv, + x->errorperbit, &cpi->fn_ptr[bsize], + cpi->sf.mv.subpel_force_stop, + cpi->sf.mv.subpel_iters_per_step, + cond_cost_list(cpi, cost_list), x->nmvjointcost, x->mvcost, + &distortion, &x->pred_sse[mbmi->ref_frame[0]], NULL, pw, ph, + 1); + + if (try_second) { + int this_var; + MV best_mv = x->best_mv.as_mv; + const MV ref_mv = bsi->ref_mv[0]->as_mv; + const int minc = + AOMMAX(x->mv_limits.col_min * 8, ref_mv.col - MV_MAX); + const int maxc = + AOMMIN(x->mv_limits.col_max * 8, ref_mv.col + MV_MAX); + const int minr = + AOMMAX(x->mv_limits.row_min * 8, ref_mv.row - MV_MAX); + const int maxr = + AOMMIN(x->mv_limits.row_max * 8, ref_mv.row + MV_MAX); + + x->best_mv = x->second_best_mv; + if (x->best_mv.as_mv.row * 8 <= maxr && + x->best_mv.as_mv.row * 8 >= minr && + x->best_mv.as_mv.col * 8 <= maxc && + x->best_mv.as_mv.col * 8 >= minc) { + this_var = cpi->find_fractional_mv_step( + x, &bsi->ref_mv[0]->as_mv, cm->allow_high_precision_mv, + x->errorperbit, &cpi->fn_ptr[bsize], + cpi->sf.mv.subpel_force_stop, + cpi->sf.mv.subpel_iters_per_step, + cond_cost_list(cpi, cost_list), x->nmvjointcost, + x->mvcost, &distortion, &x->pred_sse[mbmi->ref_frame[0]], + NULL, pw, ph, 1); + if (this_var < best_mv_var) best_mv = x->best_mv.as_mv; + x->best_mv.as_mv = best_mv; + } + } + + // Restore the reference frames. + pd->pre[0] = backup_pred; + } else { + cpi->find_fractional_mv_step( + x, &bsi->ref_mv[0]->as_mv, cm->allow_high_precision_mv, + x->errorperbit, &cpi->fn_ptr[bsize], + cpi->sf.mv.subpel_force_stop, + cpi->sf.mv.subpel_iters_per_step, + cond_cost_list(cpi, cost_list), x->nmvjointcost, x->mvcost, + &distortion, &x->pred_sse[mbmi->ref_frame[0]], NULL, 0, 0, 0); + } + +// save motion search result for use in compound prediction +#if CONFIG_EXT_INTER + seg_mvs[index][mbmi->ref_frame[0]].as_mv = x->best_mv.as_mv; +#else + seg_mvs[index][mbmi->ref_frame[0]].as_mv = x->best_mv.as_mv; +#endif // CONFIG_EXT_INTER + } + + if (cpi->sf.adaptive_motion_search) + x->pred_mv[mbmi->ref_frame[0]] = x->best_mv.as_mv; + +#if CONFIG_EXT_INTER + mode_mv[this_mode][0] = x->best_mv; +#else + mode_mv[NEWMV][0] = x->best_mv; +#endif // CONFIG_EXT_INTER + + // restore src pointers + mi_buf_restore(x, orig_src, orig_pre); + } + + if (has_second_rf) { +#if CONFIG_EXT_INTER + if (seg_mvs[index][mbmi->ref_frame[1]].as_int == INVALID_MV || + seg_mvs[index][mbmi->ref_frame[0]].as_int == INVALID_MV) +#else + if (seg_mvs[index][mbmi->ref_frame[1]].as_int == INVALID_MV || + seg_mvs[index][mbmi->ref_frame[0]].as_int == INVALID_MV) +#endif // CONFIG_EXT_INTER + continue; + } + +#if CONFIG_DUAL_FILTER + (void)run_mv_search; +#endif // CONFIG_DUAL_FILTER + + if (has_second_rf && +#if CONFIG_EXT_INTER + this_mode == NEW_NEWMV && +#else + this_mode == NEWMV && +#endif // CONFIG_EXT_INTER +#if CONFIG_DUAL_FILTER + (mbmi->interp_filter[0] == EIGHTTAP_REGULAR || run_mv_search)) +#else + (mbmi->interp_filter == EIGHTTAP_REGULAR || run_mv_search)) +#endif // CONFIG_DUAL_FILTER + { + // adjust src pointers + mi_buf_shift(x, index); + if (cpi->sf.comp_inter_joint_search_thresh <= bsize) { + int rate_mv; + frame_mv[this_mode][mbmi->ref_frame[0]].as_int = + seg_mvs[index][mbmi->ref_frame[0]].as_int; + frame_mv[this_mode][mbmi->ref_frame[1]].as_int = + seg_mvs[index][mbmi->ref_frame[1]].as_int; + joint_motion_search(cpi, x, bsize, frame_mv[this_mode], mi_row, + mi_col, +#if CONFIG_EXT_INTER + bsi->ref_mv, +#endif // CONFIG_EXT_INTER + &rate_mv, index); +#if CONFIG_EXT_INTER + compound_seg_newmvs[index][0].as_int = + frame_mv[this_mode][mbmi->ref_frame[0]].as_int; + compound_seg_newmvs[index][1].as_int = + frame_mv[this_mode][mbmi->ref_frame[1]].as_int; +#else + seg_mvs[index][mbmi->ref_frame[0]].as_int = + frame_mv[this_mode][mbmi->ref_frame[0]].as_int; + seg_mvs[index][mbmi->ref_frame[1]].as_int = + frame_mv[this_mode][mbmi->ref_frame[1]].as_int; +#endif // CONFIG_EXT_INTER + } + // restore src pointers + mi_buf_restore(x, orig_src, orig_pre); + } + + bsi->rdstat[index][mode_idx].brate = set_and_cost_bmi_mvs( + cpi, x, xd, index, this_mode, mode_mv[this_mode], frame_mv, + seg_mvs[index], +#if CONFIG_EXT_INTER + compound_seg_newmvs[index], +#endif // CONFIG_EXT_INTER + bsi->ref_mv, x->nmvjointcost, x->mvcost, mi_row, mi_col); + + for (ref = 0; ref < 1 + has_second_rf; ++ref) { + bsi->rdstat[index][mode_idx].mvs[ref].as_int = + mode_mv[this_mode][ref].as_int; + if (num_4x4_blocks_wide > 1) + bsi->rdstat[index + 1][mode_idx].mvs[ref].as_int = + mode_mv[this_mode][ref].as_int; + if (num_4x4_blocks_high > 1) + bsi->rdstat[index + 2][mode_idx].mvs[ref].as_int = + mode_mv[this_mode][ref].as_int; +#if CONFIG_REF_MV + bsi->rdstat[index][mode_idx].pred_mv[ref].as_int = + mi->bmi[index].pred_mv[ref].as_int; + if (num_4x4_blocks_wide > 1) + bsi->rdstat[index + 1][mode_idx].pred_mv[ref].as_int = + mi->bmi[index].pred_mv[ref].as_int; + if (num_4x4_blocks_high > 1) + bsi->rdstat[index + 2][mode_idx].pred_mv[ref].as_int = + mi->bmi[index].pred_mv[ref].as_int; +#endif // CONFIG_REF_MV +#if CONFIG_EXT_INTER + bsi->rdstat[index][mode_idx].ref_mv[ref].as_int = + bsi->ref_mv[ref]->as_int; + if (num_4x4_blocks_wide > 1) + bsi->rdstat[index + 1][mode_idx].ref_mv[ref].as_int = + bsi->ref_mv[ref]->as_int; + if (num_4x4_blocks_high > 1) + bsi->rdstat[index + 2][mode_idx].ref_mv[ref].as_int = + bsi->ref_mv[ref]->as_int; +#endif // CONFIG_EXT_INTER + } + + // Trap vectors that reach beyond the UMV borders + if (mv_check_bounds(&x->mv_limits, &mode_mv[this_mode][0].as_mv) || + (has_second_rf && + mv_check_bounds(&x->mv_limits, &mode_mv[this_mode][1].as_mv))) + continue; + + if (filter_idx > 0) { + BEST_SEG_INFO *ref_bsi = bsi_buf; + subpelmv = 0; + have_ref = 1; + + for (ref = 0; ref < 1 + has_second_rf; ++ref) { + subpelmv |= mv_has_subpel(&mode_mv[this_mode][ref].as_mv); +#if CONFIG_EXT_INTER + if (have_newmv_in_inter_mode(this_mode)) + have_ref &= + ((mode_mv[this_mode][ref].as_int == + ref_bsi->rdstat[index][mode_idx].mvs[ref].as_int) && + (bsi->ref_mv[ref]->as_int == + ref_bsi->rdstat[index][mode_idx].ref_mv[ref].as_int)); + else +#endif // CONFIG_EXT_INTER + have_ref &= mode_mv[this_mode][ref].as_int == + ref_bsi->rdstat[index][mode_idx].mvs[ref].as_int; + } + + have_ref &= ref_bsi->rdstat[index][mode_idx].brate > 0; + + if (filter_idx > 1 && !subpelmv && !have_ref) { + ref_bsi = bsi_buf + 1; + have_ref = 1; + for (ref = 0; ref < 1 + has_second_rf; ++ref) +#if CONFIG_EXT_INTER + if (have_newmv_in_inter_mode(this_mode)) + have_ref &= + ((mode_mv[this_mode][ref].as_int == + ref_bsi->rdstat[index][mode_idx].mvs[ref].as_int) && + (bsi->ref_mv[ref]->as_int == + ref_bsi->rdstat[index][mode_idx].ref_mv[ref].as_int)); + else +#endif // CONFIG_EXT_INTER + have_ref &= mode_mv[this_mode][ref].as_int == + ref_bsi->rdstat[index][mode_idx].mvs[ref].as_int; + + have_ref &= ref_bsi->rdstat[index][mode_idx].brate > 0; + } + + if (!subpelmv && have_ref && + ref_bsi->rdstat[index][mode_idx].brdcost < INT64_MAX) { +#if CONFIG_REF_MV + bsi->rdstat[index][mode_idx].byrate = + ref_bsi->rdstat[index][mode_idx].byrate; + bsi->rdstat[index][mode_idx].bdist = + ref_bsi->rdstat[index][mode_idx].bdist; + bsi->rdstat[index][mode_idx].bsse = + ref_bsi->rdstat[index][mode_idx].bsse; + bsi->rdstat[index][mode_idx].brate += + ref_bsi->rdstat[index][mode_idx].byrate; + bsi->rdstat[index][mode_idx].eobs = + ref_bsi->rdstat[index][mode_idx].eobs; + + bsi->rdstat[index][mode_idx].brdcost = + RDCOST(x->rdmult, x->rddiv, bsi->rdstat[index][mode_idx].brate, + bsi->rdstat[index][mode_idx].bdist); + + memcpy(bsi->rdstat[index][mode_idx].ta, + ref_bsi->rdstat[index][mode_idx].ta, + sizeof(bsi->rdstat[index][mode_idx].ta)); + memcpy(bsi->rdstat[index][mode_idx].tl, + ref_bsi->rdstat[index][mode_idx].tl, + sizeof(bsi->rdstat[index][mode_idx].tl)); +#else + memcpy(&bsi->rdstat[index][mode_idx], + &ref_bsi->rdstat[index][mode_idx], sizeof(SEG_RDSTAT)); +#endif // CONFIG_REF_MV + if (num_4x4_blocks_wide > 1) + bsi->rdstat[index + 1][mode_idx].eobs = + ref_bsi->rdstat[index + 1][mode_idx].eobs; + if (num_4x4_blocks_high > 1) + bsi->rdstat[index + 2][mode_idx].eobs = + ref_bsi->rdstat[index + 2][mode_idx].eobs; + + if (bsi->rdstat[index][mode_idx].brdcost < new_best_rd) { +#if CONFIG_REF_MV + // If the NEWMV mode is using the same motion vector as the + // NEARESTMV mode, skip the rest rate-distortion calculations + // and use the inferred motion vector modes. + if (this_mode == NEWMV) { + if (has_second_rf) { + if (bsi->rdstat[index][mode_idx].mvs[0].as_int == + bsi->ref_mv[0]->as_int && + bsi->rdstat[index][mode_idx].mvs[1].as_int == + bsi->ref_mv[1]->as_int) + continue; + } else { + if (bsi->rdstat[index][mode_idx].mvs[0].as_int == + bsi->ref_mv[0]->as_int) + continue; + } + } +#endif // CONFIG_REF_MV + mode_selected = this_mode; + new_best_rd = bsi->rdstat[index][mode_idx].brdcost; +#if CONFIG_PVQ + od_encode_checkpoint(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + } + continue; + } + } + + bsi->rdstat[index][mode_idx].brdcost = encode_inter_mb_segment_sub8x8( + cpi, x, bsi->segment_rd - this_segment_rd, index, + &bsi->rdstat[index][mode_idx].byrate, + &bsi->rdstat[index][mode_idx].bdist, + &bsi->rdstat[index][mode_idx].bsse, bsi->rdstat[index][mode_idx].ta, + bsi->rdstat[index][mode_idx].tl, idy, idx, mi_row, mi_col); + + if (bsi->rdstat[index][mode_idx].brdcost < INT64_MAX) { + bsi->rdstat[index][mode_idx].brdcost += RDCOST( + x->rdmult, x->rddiv, bsi->rdstat[index][mode_idx].brate, 0); + bsi->rdstat[index][mode_idx].brate += + bsi->rdstat[index][mode_idx].byrate; + bsi->rdstat[index][mode_idx].eobs = p->eobs[index]; + if (num_4x4_blocks_wide > 1) + bsi->rdstat[index + 1][mode_idx].eobs = p->eobs[index + 1]; + if (num_4x4_blocks_high > 1) + bsi->rdstat[index + 2][mode_idx].eobs = p->eobs[index + 2]; + } + + if (bsi->rdstat[index][mode_idx].brdcost < new_best_rd) { +#if CONFIG_REF_MV + // If the NEWMV mode is using the same motion vector as the + // NEARESTMV mode, skip the rest rate-distortion calculations + // and use the inferred motion vector modes. + if (this_mode == NEWMV) { + if (has_second_rf) { + if (bsi->rdstat[index][mode_idx].mvs[0].as_int == + bsi->ref_mv[0]->as_int && + bsi->rdstat[index][mode_idx].mvs[1].as_int == + bsi->ref_mv[1]->as_int) + continue; + } else { + if (bsi->rdstat[index][mode_idx].mvs[0].as_int == + bsi->ref_mv[0]->as_int) + continue; + } + } +#endif // CONFIG_REF_MV + mode_selected = this_mode; + new_best_rd = bsi->rdstat[index][mode_idx].brdcost; + +#if CONFIG_PVQ + od_encode_checkpoint(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + } + } /*for each 4x4 mode*/ + + if (new_best_rd == INT64_MAX) { + int iy, midx; + for (iy = index + 1; iy < 4; ++iy) +#if CONFIG_EXT_INTER + for (midx = 0; midx < INTER_MODES + INTER_COMPOUND_MODES; ++midx) +#else + for (midx = 0; midx < INTER_MODES; ++midx) +#endif // CONFIG_EXT_INTER + bsi->rdstat[iy][midx].brdcost = INT64_MAX; + bsi->segment_rd = INT64_MAX; +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &pre_buf); +#endif // CONFIG_PVQ + return INT64_MAX; + } + + mode_idx = INTER_OFFSET(mode_selected); + memcpy(t_above, bsi->rdstat[index][mode_idx].ta, sizeof(t_above)); + memcpy(t_left, bsi->rdstat[index][mode_idx].tl, sizeof(t_left)); +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &post_buf); +#endif // CONFIG_PVQ + +#if CONFIG_EXT_INTER + bsi->ref_mv[0]->as_int = bsi->rdstat[index][mode_idx].ref_mv[0].as_int; + if (has_second_rf) + bsi->ref_mv[1]->as_int = bsi->rdstat[index][mode_idx].ref_mv[1].as_int; +#endif // CONFIG_EXT_INTER + set_and_cost_bmi_mvs(cpi, x, xd, index, mode_selected, + mode_mv[mode_selected], frame_mv, seg_mvs[index], +#if CONFIG_EXT_INTER + compound_seg_newmvs[index], +#endif // CONFIG_EXT_INTER + bsi->ref_mv, x->nmvjointcost, x->mvcost, mi_row, + mi_col); + + br += bsi->rdstat[index][mode_idx].brate; + bd += bsi->rdstat[index][mode_idx].bdist; + block_sse += bsi->rdstat[index][mode_idx].bsse; + segmentyrate += bsi->rdstat[index][mode_idx].byrate; + this_segment_rd += bsi->rdstat[index][mode_idx].brdcost; + + if (this_segment_rd > bsi->segment_rd) { + int iy, midx; + for (iy = index + 1; iy < 4; ++iy) +#if CONFIG_EXT_INTER + for (midx = 0; midx < INTER_MODES + INTER_COMPOUND_MODES; ++midx) +#else + for (midx = 0; midx < INTER_MODES; ++midx) +#endif // CONFIG_EXT_INTER + bsi->rdstat[iy][midx].brdcost = INT64_MAX; + bsi->segment_rd = INT64_MAX; +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &pre_buf); +#endif // CONFIG_PVQ + return INT64_MAX; + } + } + } /* for each label */ +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &pre_buf); +#endif // CONFIG_PVQ + + bsi->r = br; + bsi->d = bd; + bsi->segment_yrate = segmentyrate; + bsi->segment_rd = this_segment_rd; + bsi->sse = block_sse; + + // update the coding decisions + for (k = 0; k < 4; ++k) bsi->modes[k] = mi->bmi[k].as_mode; + +#if CONFIG_DAALA_DIST + // Compute prediction (i.e. skip) and decoded distortion by daala-distortion. + { + const int src_stride = p->src.stride; + const int dst_stride = pd->dst.stride; + uint8_t *src = p->src.buf; + uint8_t *dst = pd->dst.buf; + const BLOCK_SIZE plane_bsize = get_plane_block_size(mi->mbmi.sb_type, pd); + const int use_activity_masking = 0; + const int qm = OD_HVS_QM; + const int bsw = block_size_wide[plane_bsize]; + const int bsh = block_size_high[plane_bsize]; + int64_t rd1, rd2; + int64_t daala_sse, daala_dist; + TX_SIZE tx_size = mbmi->tx_size; + +#if CONFIG_HIGHBITDEPTH + uint8_t *recon_8x8; + DECLARE_ALIGNED(16, uint16_t, recon16[8 * 8]); + + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + recon_8x8 = CONVERT_TO_BYTEPTR(recon16); + else + recon_8x8 = (uint8_t *)recon16; +#else + DECLARE_ALIGNED(16, uint8_t, recon_8x8[8 * 8]); +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_PVQ + use_activity_masking = x->daala_enc.use_activity_masking; +#endif // CONFIG_PVQ + + // For each of sub8x8 prediction block in a 8x8 block + for (idy = 0; idy < 2; idy += num_4x4_blocks_high) { + for (idx = 0; idx < 2; idx += num_4x4_blocks_wide) { + int i = idy * 2 + idx; + const uint8_t *const src_sub8x8 = + src + av1_raster_block_offset(BLOCK_8X8, i, p->src.stride); + uint8_t *const dst_sub8x8 = + dst + av1_raster_block_offset(BLOCK_8X8, i, pd->dst.stride); + uint8_t *recon_sub8x8 = recon_8x8 + (idy * 8 + idx) * 4; + const int txb_width = max_block_wide(xd, plane_bsize, 0); + const int txb_height = max_block_high(xd, plane_bsize, 0); + int idx_, idy_; + + av1_build_inter_predictor_sub8x8(xd, 0, i, idy, idx, mi_row, mi_col); +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + aom_highbd_subtract_block( + height, width, + av1_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff), 8, + src_sub8x8, p->src.stride, dst_sub8x8, pd->dst.stride, xd->bd); + } else { + aom_subtract_block( + height, width, + av1_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff), 8, + src_sub8x8, p->src.stride, dst_sub8x8, pd->dst.stride); + } +#else + aom_subtract_block( + bsh, bsw, av1_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff), + 8, src_sub8x8, p->src.stride, dst_sub8x8, pd->dst.stride); +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + aom_highbd_convolve_copy(dst_sub8x8, dst_stride, recon_sub8x8, 8, + NULL, 0, NULL, 0, bsw, bsh, xd->bd); + } else { +#endif // CONFIG_HIGHBITDEPTH + aom_convolve_copy(dst_sub8x8, dst_stride, recon_sub8x8, 8, NULL, 0, + NULL, 0, bsw, bsh); +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + + // To get decoded pixels, do 4x4 xform and quant for each 4x4 block + // in a sub8x8 prediction block. In case remaining parts of + // sub8x8 inter mode rdo assume pd->dst stores predicted pixels, + // use local buffer to store decoded pixels. + for (idy_ = 0; idy_ < txb_height; idy_++) { + for (idx_ = 0; idx_ < txb_width; idx_++) { + int coeff_ctx = 0; + const tran_low_t *dqcoeff; + uint16_t eob; + const PLANE_TYPE plane_type = PLANE_TYPE_Y; + uint8_t *recon_4x4 = recon_sub8x8 + (idy_ * 8 + idx_) * 4; + const int block_raster_idx = (idy + idy_) * 2 + (idx + idx_); + const int block = + av1_raster_order_to_block_index(tx_size, block_raster_idx); + TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size); + + dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); + av1_xform_quant(cm, x, 0, block, idy + idy_, idx + idx_, BLOCK_8X8, + tx_size, coeff_ctx, AV1_XFORM_QUANT_FP); + if (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0) + av1_optimize_b(cm, x, 0, block, tx_size, coeff_ctx); + + eob = p->eobs[block]; + av1_inverse_transform_block(xd, dqcoeff, tx_type, tx_size, + recon_4x4, 8, eob); + } + } + } + } + // Compute daala-distortion for a 8x8 block + daala_sse = av1_daala_dist(src, src_stride, pd->dst.buf, dst_stride, 8, 8, + qm, use_activity_masking, x->qindex) + << 4; + + daala_dist = av1_daala_dist(src, src_stride, recon_8x8, 8, 8, 8, qm, + use_activity_masking, x->qindex) + << 4; + + bsi->sse = daala_sse; + bsi->d = daala_dist; + + rd1 = RDCOST(x->rdmult, x->rddiv, bsi->r, bsi->d); + rd2 = RDCOST(x->rdmult, x->rddiv, 0, bsi->sse); + bsi->segment_rd = AOMMIN(rd1, rd2); + } +#endif // CONFIG_DAALA_DIST + + if (bsi->segment_rd > best_rd) return INT64_MAX; + /* set it to the best */ + for (idx = 0; idx < 4; idx++) { + mode_idx = INTER_OFFSET(bsi->modes[idx]); + mi->bmi[idx].as_mv[0].as_int = bsi->rdstat[idx][mode_idx].mvs[0].as_int; + if (has_second_ref(mbmi)) + mi->bmi[idx].as_mv[1].as_int = bsi->rdstat[idx][mode_idx].mvs[1].as_int; +#if CONFIG_REF_MV + mi->bmi[idx].pred_mv[0] = bsi->rdstat[idx][mode_idx].pred_mv[0]; + if (has_second_ref(mbmi)) + mi->bmi[idx].pred_mv[1] = bsi->rdstat[idx][mode_idx].pred_mv[1]; +#endif // CONFIG_REF_MV +#if CONFIG_EXT_INTER + mi->bmi[idx].ref_mv[0].as_int = bsi->rdstat[idx][mode_idx].ref_mv[0].as_int; + if (has_second_rf) + mi->bmi[idx].ref_mv[1].as_int = + bsi->rdstat[idx][mode_idx].ref_mv[1].as_int; +#endif // CONFIG_EXT_INTER + x->plane[0].eobs[idx] = bsi->rdstat[idx][mode_idx].eobs; + mi->bmi[idx].as_mode = bsi->modes[idx]; + } + + /* + * used to set mbmi->mv.as_int + */ + *returntotrate = bsi->r; + *returndistortion = bsi->d; + *returnyrate = bsi->segment_yrate; + *skippable = av1_is_skippable_in_plane(x, BLOCK_8X8, 0); + *psse = bsi->sse; + mbmi->mode = bsi->modes[3]; + + return bsi->segment_rd; +} + +static void estimate_ref_frame_costs(const AV1_COMMON *cm, + const MACROBLOCKD *xd, int segment_id, + unsigned int *ref_costs_single, + unsigned int *ref_costs_comp, + aom_prob *comp_mode_p) { + int seg_ref_active = + segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME); + if (seg_ref_active) { + memset(ref_costs_single, 0, + TOTAL_REFS_PER_FRAME * sizeof(*ref_costs_single)); + memset(ref_costs_comp, 0, TOTAL_REFS_PER_FRAME * sizeof(*ref_costs_comp)); + *comp_mode_p = 128; + } else { + aom_prob intra_inter_p = av1_get_intra_inter_prob(cm, xd); + aom_prob comp_inter_p = 128; + + if (cm->reference_mode == REFERENCE_MODE_SELECT) { + comp_inter_p = av1_get_reference_mode_prob(cm, xd); + *comp_mode_p = comp_inter_p; + } else { + *comp_mode_p = 128; + } + + ref_costs_single[INTRA_FRAME] = av1_cost_bit(intra_inter_p, 0); + + if (cm->reference_mode != COMPOUND_REFERENCE) { + aom_prob ref_single_p1 = av1_get_pred_prob_single_ref_p1(cm, xd); + aom_prob ref_single_p2 = av1_get_pred_prob_single_ref_p2(cm, xd); +#if CONFIG_EXT_REFS + aom_prob ref_single_p3 = av1_get_pred_prob_single_ref_p3(cm, xd); + aom_prob ref_single_p4 = av1_get_pred_prob_single_ref_p4(cm, xd); + aom_prob ref_single_p5 = av1_get_pred_prob_single_ref_p5(cm, xd); +#endif // CONFIG_EXT_REFS + + unsigned int base_cost = av1_cost_bit(intra_inter_p, 1); + + ref_costs_single[LAST_FRAME] = +#if CONFIG_EXT_REFS + ref_costs_single[LAST2_FRAME] = ref_costs_single[LAST3_FRAME] = + ref_costs_single[BWDREF_FRAME] = +#endif // CONFIG_EXT_REFS + ref_costs_single[GOLDEN_FRAME] = + ref_costs_single[ALTREF_FRAME] = base_cost; + +#if CONFIG_EXT_REFS + ref_costs_single[LAST_FRAME] += av1_cost_bit(ref_single_p1, 0); + ref_costs_single[LAST2_FRAME] += av1_cost_bit(ref_single_p1, 0); + ref_costs_single[LAST3_FRAME] += av1_cost_bit(ref_single_p1, 0); + ref_costs_single[GOLDEN_FRAME] += av1_cost_bit(ref_single_p1, 0); + ref_costs_single[BWDREF_FRAME] += av1_cost_bit(ref_single_p1, 1); + ref_costs_single[ALTREF_FRAME] += av1_cost_bit(ref_single_p1, 1); + + ref_costs_single[LAST_FRAME] += av1_cost_bit(ref_single_p3, 0); + ref_costs_single[LAST2_FRAME] += av1_cost_bit(ref_single_p3, 0); + ref_costs_single[LAST3_FRAME] += av1_cost_bit(ref_single_p3, 1); + ref_costs_single[GOLDEN_FRAME] += av1_cost_bit(ref_single_p3, 1); + + ref_costs_single[BWDREF_FRAME] += av1_cost_bit(ref_single_p2, 0); + ref_costs_single[ALTREF_FRAME] += av1_cost_bit(ref_single_p2, 1); + + ref_costs_single[LAST_FRAME] += av1_cost_bit(ref_single_p4, 0); + ref_costs_single[LAST2_FRAME] += av1_cost_bit(ref_single_p4, 1); + + ref_costs_single[LAST3_FRAME] += av1_cost_bit(ref_single_p5, 0); + ref_costs_single[GOLDEN_FRAME] += av1_cost_bit(ref_single_p5, 1); +#else + ref_costs_single[LAST_FRAME] += av1_cost_bit(ref_single_p1, 0); + ref_costs_single[GOLDEN_FRAME] += av1_cost_bit(ref_single_p1, 1); + ref_costs_single[ALTREF_FRAME] += av1_cost_bit(ref_single_p1, 1); + + ref_costs_single[GOLDEN_FRAME] += av1_cost_bit(ref_single_p2, 0); + ref_costs_single[ALTREF_FRAME] += av1_cost_bit(ref_single_p2, 1); +#endif // CONFIG_EXT_REFS + } else { + ref_costs_single[LAST_FRAME] = 512; +#if CONFIG_EXT_REFS + ref_costs_single[LAST2_FRAME] = 512; + ref_costs_single[LAST3_FRAME] = 512; + ref_costs_single[BWDREF_FRAME] = 512; +#endif // CONFIG_EXT_REFS + ref_costs_single[GOLDEN_FRAME] = 512; + ref_costs_single[ALTREF_FRAME] = 512; + } + + if (cm->reference_mode != SINGLE_REFERENCE) { + aom_prob ref_comp_p = av1_get_pred_prob_comp_ref_p(cm, xd); +#if CONFIG_EXT_REFS + aom_prob ref_comp_p1 = av1_get_pred_prob_comp_ref_p1(cm, xd); + aom_prob ref_comp_p2 = av1_get_pred_prob_comp_ref_p2(cm, xd); + aom_prob bwdref_comp_p = av1_get_pred_prob_comp_bwdref_p(cm, xd); +#endif // CONFIG_EXT_REFS + + unsigned int base_cost = av1_cost_bit(intra_inter_p, 1); + + ref_costs_comp[LAST_FRAME] = +#if CONFIG_EXT_REFS + ref_costs_comp[LAST2_FRAME] = ref_costs_comp[LAST3_FRAME] = +#endif // CONFIG_EXT_REFS + ref_costs_comp[GOLDEN_FRAME] = base_cost; + +#if CONFIG_EXT_REFS + ref_costs_comp[BWDREF_FRAME] = ref_costs_comp[ALTREF_FRAME] = 0; +#endif // CONFIG_EXT_REFS + +#if CONFIG_EXT_REFS + ref_costs_comp[LAST_FRAME] += av1_cost_bit(ref_comp_p, 0); + ref_costs_comp[LAST2_FRAME] += av1_cost_bit(ref_comp_p, 0); + ref_costs_comp[LAST3_FRAME] += av1_cost_bit(ref_comp_p, 1); + ref_costs_comp[GOLDEN_FRAME] += av1_cost_bit(ref_comp_p, 1); + + ref_costs_comp[LAST_FRAME] += av1_cost_bit(ref_comp_p1, 1); + ref_costs_comp[LAST2_FRAME] += av1_cost_bit(ref_comp_p1, 0); + + ref_costs_comp[LAST3_FRAME] += av1_cost_bit(ref_comp_p2, 0); + ref_costs_comp[GOLDEN_FRAME] += av1_cost_bit(ref_comp_p2, 1); + + // NOTE(zoeliu): BWDREF and ALTREF each add an extra cost by coding 1 + // more bit. + ref_costs_comp[BWDREF_FRAME] += av1_cost_bit(bwdref_comp_p, 0); + ref_costs_comp[ALTREF_FRAME] += av1_cost_bit(bwdref_comp_p, 1); +#else + ref_costs_comp[LAST_FRAME] += av1_cost_bit(ref_comp_p, 0); + ref_costs_comp[GOLDEN_FRAME] += av1_cost_bit(ref_comp_p, 1); +#endif // CONFIG_EXT_REFS + } else { + ref_costs_comp[LAST_FRAME] = 512; +#if CONFIG_EXT_REFS + ref_costs_comp[LAST2_FRAME] = 512; + ref_costs_comp[LAST3_FRAME] = 512; + ref_costs_comp[BWDREF_FRAME] = 512; + ref_costs_comp[ALTREF_FRAME] = 512; +#endif // CONFIG_EXT_REFS + ref_costs_comp[GOLDEN_FRAME] = 512; + } + } +} + +static void store_coding_context(MACROBLOCK *x, PICK_MODE_CONTEXT *ctx, + int mode_index, + int64_t comp_pred_diff[REFERENCE_MODES], + int skippable) { + MACROBLOCKD *const xd = &x->e_mbd; + + // Take a snapshot of the coding context so it can be + // restored if we decide to encode this way + ctx->skip = x->skip; + ctx->skippable = skippable; + ctx->best_mode_index = mode_index; + ctx->mic = *xd->mi[0]; + ctx->mbmi_ext = *x->mbmi_ext; + ctx->single_pred_diff = (int)comp_pred_diff[SINGLE_REFERENCE]; + ctx->comp_pred_diff = (int)comp_pred_diff[COMPOUND_REFERENCE]; + ctx->hybrid_pred_diff = (int)comp_pred_diff[REFERENCE_MODE_SELECT]; +} + +static void setup_buffer_inter( + const AV1_COMP *const cpi, MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame, + BLOCK_SIZE block_size, int mi_row, int mi_col, + int_mv frame_nearest_mv[TOTAL_REFS_PER_FRAME], + int_mv frame_near_mv[TOTAL_REFS_PER_FRAME], + struct buf_2d yv12_mb[TOTAL_REFS_PER_FRAME][MAX_MB_PLANE]) { + const AV1_COMMON *cm = &cpi->common; + const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame); + MACROBLOCKD *const xd = &x->e_mbd; + MODE_INFO *const mi = xd->mi[0]; + int_mv *const candidates = x->mbmi_ext->ref_mvs[ref_frame]; + const struct scale_factors *const sf = &cm->frame_refs[ref_frame - 1].sf; + MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; + + assert(yv12 != NULL); + + // TODO(jkoleszar): Is the UV buffer ever used here? If so, need to make this + // use the UV scaling factors. + av1_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col, sf, sf); + + // Gets an initial list of candidate vectors from neighbours and orders them + av1_find_mv_refs( + cm, xd, mi, ref_frame, +#if CONFIG_REF_MV + &mbmi_ext->ref_mv_count[ref_frame], mbmi_ext->ref_mv_stack[ref_frame], +#if CONFIG_EXT_INTER + mbmi_ext->compound_mode_context, +#endif // CONFIG_EXT_INTER +#endif // CONFIG_REF_MV + candidates, mi_row, mi_col, NULL, NULL, mbmi_ext->mode_context); + + // Candidate refinement carried out at encoder and decoder + av1_find_best_ref_mvs(cm->allow_high_precision_mv, candidates, + &frame_nearest_mv[ref_frame], + &frame_near_mv[ref_frame]); + +// Further refinement that is encode side only to test the top few candidates +// in full and choose the best as the centre point for subsequent searches. +// The current implementation doesn't support scaling. +#if CONFIG_CB4X4 + av1_mv_pred(cpi, x, yv12_mb[ref_frame][0].buf, yv12->y_stride, ref_frame, + block_size); +#else + if (!av1_is_scaled(sf) && block_size >= BLOCK_8X8) + av1_mv_pred(cpi, x, yv12_mb[ref_frame][0].buf, yv12->y_stride, ref_frame, + block_size); +#endif // CONFIG_CB4X4 +} + +static void single_motion_search(const AV1_COMP *const cpi, MACROBLOCK *x, + BLOCK_SIZE bsize, int mi_row, int mi_col, +#if CONFIG_EXT_INTER + int ref_idx, +#endif // CONFIG_EXT_INTER + int *rate_mv) { + MACROBLOCKD *xd = &x->e_mbd; + const AV1_COMMON *cm = &cpi->common; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + struct buf_2d backup_yv12[MAX_MB_PLANE] = { { 0, 0, 0, 0, 0 } }; + int bestsme = INT_MAX; + int step_param; + int sadpb = x->sadperbit16; + MV mvp_full; +#if CONFIG_EXT_INTER + int ref = mbmi->ref_frame[ref_idx]; +#else + int ref = mbmi->ref_frame[0]; + int ref_idx = 0; +#endif // CONFIG_EXT_INTER + MV ref_mv = x->mbmi_ext->ref_mvs[ref][0].as_mv; + + MvLimits tmp_mv_limits = x->mv_limits; + int cost_list[5]; + + const YV12_BUFFER_CONFIG *scaled_ref_frame = + av1_get_scaled_ref_frame(cpi, ref); + + MV pred_mv[3]; + pred_mv[0] = x->mbmi_ext->ref_mvs[ref][0].as_mv; + pred_mv[1] = x->mbmi_ext->ref_mvs[ref][1].as_mv; + pred_mv[2] = x->pred_mv[ref]; + + if (scaled_ref_frame) { + int i; + // Swap out the reference frame for a version that's been scaled to + // match the resolution of the current frame, allowing the existing + // motion search code to be used without additional modifications. + for (i = 0; i < MAX_MB_PLANE; i++) + backup_yv12[i] = xd->plane[i].pre[ref_idx]; + + av1_setup_pre_planes(xd, ref_idx, scaled_ref_frame, mi_row, mi_col, NULL); + } + + av1_set_mv_search_range(&x->mv_limits, &ref_mv); + +#if CONFIG_REF_MV + av1_set_mvcost(x, ref, ref_idx, mbmi->ref_mv_idx); +#endif // CONFIG_REF_MV + + // Work out the size of the first step in the mv step search. + // 0 here is maximum length first step. 1 is AOMMAX >> 1 etc. + if (cpi->sf.mv.auto_mv_step_size && cm->show_frame) { + // Take wtd average of the step_params based on the last frame's + // max mv magnitude and that based on the best ref mvs of the current + // block for the given reference. + step_param = + (av1_init_search_range(x->max_mv_context[ref]) + cpi->mv_step_param) / + 2; + } else { + step_param = cpi->mv_step_param; + } + + if (cpi->sf.adaptive_motion_search && bsize < cm->sb_size) { + int boffset = + 2 * (b_width_log2_lookup[cm->sb_size] - + AOMMIN(b_height_log2_lookup[bsize], b_width_log2_lookup[bsize])); + step_param = AOMMAX(step_param, boffset); + } + + if (cpi->sf.adaptive_motion_search) { + int bwl = b_width_log2_lookup[bsize]; + int bhl = b_height_log2_lookup[bsize]; + int tlevel = x->pred_mv_sad[ref] >> (bwl + bhl + 4); + + if (tlevel < 5) step_param += 2; + + // prev_mv_sad is not setup for dynamically scaled frames. + if (cpi->oxcf.resize_mode != RESIZE_DYNAMIC) { + int i; + for (i = LAST_FRAME; i <= ALTREF_FRAME && cm->show_frame; ++i) { + if ((x->pred_mv_sad[ref] >> 3) > x->pred_mv_sad[i]) { + x->pred_mv[ref].row = 0; + x->pred_mv[ref].col = 0; + x->best_mv.as_int = INVALID_MV; + + if (scaled_ref_frame) { + int j; + for (j = 0; j < MAX_MB_PLANE; ++j) + xd->plane[j].pre[ref_idx] = backup_yv12[j]; + } + return; + } + } + } + } + + av1_set_mv_search_range(&x->mv_limits, &ref_mv); + +#if CONFIG_MOTION_VAR + if (mbmi->motion_mode != SIMPLE_TRANSLATION) + mvp_full = mbmi->mv[0].as_mv; + else +#endif // CONFIG_MOTION_VAR + mvp_full = pred_mv[x->mv_best_ref_index[ref]]; + + mvp_full.col >>= 3; + mvp_full.row >>= 3; + + x->best_mv.as_int = x->second_best_mv.as_int = INVALID_MV; + +#if CONFIG_MOTION_VAR + switch (mbmi->motion_mode) { + case SIMPLE_TRANSLATION: +#endif // CONFIG_MOTION_VAR + bestsme = av1_full_pixel_search(cpi, x, bsize, &mvp_full, step_param, + sadpb, cond_cost_list(cpi, cost_list), + &ref_mv, INT_MAX, 1); +#if CONFIG_MOTION_VAR + break; + case OBMC_CAUSAL: + bestsme = av1_obmc_full_pixel_diamond( + cpi, x, &mvp_full, step_param, sadpb, + MAX_MVSEARCH_STEPS - 1 - step_param, 1, &cpi->fn_ptr[bsize], &ref_mv, + &(x->best_mv.as_mv), 0); + break; + default: assert("Invalid motion mode!\n"); + } +#endif // CONFIG_MOTION_VAR + + x->mv_limits = tmp_mv_limits; + + if (bestsme < INT_MAX) { + int dis; /* TODO: use dis in distortion calculation later. */ +#if CONFIG_MOTION_VAR + switch (mbmi->motion_mode) { + case SIMPLE_TRANSLATION: +#endif // CONFIG_MOTION_VAR + if (cpi->sf.use_upsampled_references) { + int best_mv_var; + const int try_second = x->second_best_mv.as_int != INVALID_MV && + x->second_best_mv.as_int != x->best_mv.as_int; + const int pw = block_size_wide[bsize]; + const int ph = block_size_high[bsize]; + // Use up-sampled reference frames. + struct macroblockd_plane *const pd = &xd->plane[0]; + struct buf_2d backup_pred = pd->pre[ref_idx]; + const YV12_BUFFER_CONFIG *upsampled_ref = get_upsampled_ref(cpi, ref); + + // Set pred for Y plane + setup_pred_plane( + &pd->pre[ref_idx], bsize, upsampled_ref->y_buffer, + upsampled_ref->y_crop_width, upsampled_ref->y_crop_height, + upsampled_ref->y_stride, (mi_row << 3), (mi_col << 3), NULL, + pd->subsampling_x, pd->subsampling_y); + + best_mv_var = cpi->find_fractional_mv_step( + x, &ref_mv, cm->allow_high_precision_mv, x->errorperbit, + &cpi->fn_ptr[bsize], cpi->sf.mv.subpel_force_stop, + cpi->sf.mv.subpel_iters_per_step, cond_cost_list(cpi, cost_list), + x->nmvjointcost, x->mvcost, &dis, &x->pred_sse[ref], NULL, pw, ph, + 1); + + if (try_second) { + const int minc = + AOMMAX(x->mv_limits.col_min * 8, ref_mv.col - MV_MAX); + const int maxc = + AOMMIN(x->mv_limits.col_max * 8, ref_mv.col + MV_MAX); + const int minr = + AOMMAX(x->mv_limits.row_min * 8, ref_mv.row - MV_MAX); + const int maxr = + AOMMIN(x->mv_limits.row_max * 8, ref_mv.row + MV_MAX); + int this_var; + MV best_mv = x->best_mv.as_mv; + + x->best_mv = x->second_best_mv; + if (x->best_mv.as_mv.row * 8 <= maxr && + x->best_mv.as_mv.row * 8 >= minr && + x->best_mv.as_mv.col * 8 <= maxc && + x->best_mv.as_mv.col * 8 >= minc) { + this_var = cpi->find_fractional_mv_step( + x, &ref_mv, cm->allow_high_precision_mv, x->errorperbit, + &cpi->fn_ptr[bsize], cpi->sf.mv.subpel_force_stop, + cpi->sf.mv.subpel_iters_per_step, + cond_cost_list(cpi, cost_list), x->nmvjointcost, x->mvcost, + &dis, &x->pred_sse[ref], NULL, pw, ph, 1); + if (this_var < best_mv_var) best_mv = x->best_mv.as_mv; + x->best_mv.as_mv = best_mv; + } + } + + // Restore the reference frames. + pd->pre[ref_idx] = backup_pred; + } else { + cpi->find_fractional_mv_step( + x, &ref_mv, cm->allow_high_precision_mv, x->errorperbit, + &cpi->fn_ptr[bsize], cpi->sf.mv.subpel_force_stop, + cpi->sf.mv.subpel_iters_per_step, cond_cost_list(cpi, cost_list), + x->nmvjointcost, x->mvcost, &dis, &x->pred_sse[ref], NULL, 0, 0, + 0); + } +#if CONFIG_MOTION_VAR + break; + case OBMC_CAUSAL: + av1_find_best_obmc_sub_pixel_tree_up( + cpi, x, mi_row, mi_col, &x->best_mv.as_mv, &ref_mv, + cm->allow_high_precision_mv, x->errorperbit, &cpi->fn_ptr[bsize], + cpi->sf.mv.subpel_force_stop, cpi->sf.mv.subpel_iters_per_step, + x->nmvjointcost, x->mvcost, &dis, &x->pred_sse[ref], 0, + cpi->sf.use_upsampled_references); + break; + default: assert("Invalid motion mode!\n"); + } +#endif // CONFIG_MOTION_VAR + } + *rate_mv = av1_mv_bit_cost(&x->best_mv.as_mv, &ref_mv, x->nmvjointcost, + x->mvcost, MV_COST_WEIGHT); + +#if CONFIG_MOTION_VAR + if (cpi->sf.adaptive_motion_search && mbmi->motion_mode == SIMPLE_TRANSLATION) +#else + if (cpi->sf.adaptive_motion_search) +#endif // CONFIG_MOTION_VAR + x->pred_mv[ref] = x->best_mv.as_mv; + + if (scaled_ref_frame) { + int i; + for (i = 0; i < MAX_MB_PLANE; i++) + xd->plane[i].pre[ref_idx] = backup_yv12[i]; + } +} + +static INLINE void restore_dst_buf(MACROBLOCKD *xd, BUFFER_SET dst) { + int i; + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].dst.buf = dst.plane[i]; + xd->plane[i].dst.stride = dst.stride[i]; + } +} + +#if CONFIG_EXT_INTER +#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE +static void do_masked_motion_search(const AV1_COMP *const cpi, MACROBLOCK *x, + const uint8_t *mask, int mask_stride, + BLOCK_SIZE bsize, int mi_row, int mi_col, + int_mv *tmp_mv, int *rate_mv, int ref_idx) { + MACROBLOCKD *xd = &x->e_mbd; + const AV1_COMMON *cm = &cpi->common; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + struct buf_2d backup_yv12[MAX_MB_PLANE] = { { 0, 0, 0, 0, 0 } }; + int bestsme = INT_MAX; + int step_param; + int sadpb = x->sadperbit16; + MV mvp_full; + int ref = mbmi->ref_frame[ref_idx]; + MV ref_mv = x->mbmi_ext->ref_mvs[ref][0].as_mv; + + MvLimits tmp_mv_limits = x->mv_limits; + + const YV12_BUFFER_CONFIG *scaled_ref_frame = + av1_get_scaled_ref_frame(cpi, ref); + int i; + + MV pred_mv[3]; + pred_mv[0] = x->mbmi_ext->ref_mvs[ref][0].as_mv; + pred_mv[1] = x->mbmi_ext->ref_mvs[ref][1].as_mv; + pred_mv[2] = x->pred_mv[ref]; + +#if CONFIG_REF_MV + av1_set_mvcost(x, ref, ref_idx, mbmi->ref_mv_idx); +#endif // CONFIG_REF_MV + + if (scaled_ref_frame) { + // Swap out the reference frame for a version that's been scaled to + // match the resolution of the current frame, allowing the existing + // motion search code to be used without additional modifications. + for (i = 0; i < MAX_MB_PLANE; i++) + backup_yv12[i] = xd->plane[i].pre[ref_idx]; + + av1_setup_pre_planes(xd, ref_idx, scaled_ref_frame, mi_row, mi_col, NULL); + } + + av1_set_mv_search_range(&x->mv_limits, &ref_mv); + + // Work out the size of the first step in the mv step search. + // 0 here is maximum length first step. 1 is MAX >> 1 etc. + if (cpi->sf.mv.auto_mv_step_size && cm->show_frame) { + // Take wtd average of the step_params based on the last frame's + // max mv magnitude and that based on the best ref mvs of the current + // block for the given reference. + step_param = + (av1_init_search_range(x->max_mv_context[ref]) + cpi->mv_step_param) / + 2; + } else { + step_param = cpi->mv_step_param; + } + + // TODO(debargha): is show_frame needed here? + if (cpi->sf.adaptive_motion_search && bsize < cm->sb_size && cm->show_frame) { + int boffset = + 2 * (b_width_log2_lookup[cm->sb_size] - + AOMMIN(b_height_log2_lookup[bsize], b_width_log2_lookup[bsize])); + step_param = AOMMAX(step_param, boffset); + } + + if (cpi->sf.adaptive_motion_search) { + int bwl = b_width_log2_lookup[bsize]; + int bhl = b_height_log2_lookup[bsize]; + int tlevel = x->pred_mv_sad[ref] >> (bwl + bhl + 4); + + if (tlevel < 5) step_param += 2; + + // prev_mv_sad is not setup for dynamically scaled frames. + if (cpi->oxcf.resize_mode != RESIZE_DYNAMIC) { + for (i = LAST_FRAME; i <= ALTREF_FRAME && cm->show_frame; ++i) { + if ((x->pred_mv_sad[ref] >> 3) > x->pred_mv_sad[i]) { + x->pred_mv[ref].row = 0; + x->pred_mv[ref].col = 0; + tmp_mv->as_int = INVALID_MV; + + if (scaled_ref_frame) { + int j; + for (j = 0; j < MAX_MB_PLANE; ++j) + xd->plane[j].pre[ref_idx] = backup_yv12[j]; + } + return; + } + } + } + } + + mvp_full = pred_mv[x->mv_best_ref_index[ref]]; + + mvp_full.col >>= 3; + mvp_full.row >>= 3; + + bestsme = av1_masked_full_pixel_diamond( + cpi, x, mask, mask_stride, &mvp_full, step_param, sadpb, + MAX_MVSEARCH_STEPS - 1 - step_param, 1, &cpi->fn_ptr[bsize], &ref_mv, + &tmp_mv->as_mv, ref_idx); + + x->mv_limits = tmp_mv_limits; + + if (bestsme < INT_MAX) { + int dis; /* TODO: use dis in distortion calculation later. */ + av1_find_best_masked_sub_pixel_tree_up( + cpi, x, mask, mask_stride, mi_row, mi_col, &tmp_mv->as_mv, &ref_mv, + cm->allow_high_precision_mv, x->errorperbit, &cpi->fn_ptr[bsize], + cpi->sf.mv.subpel_force_stop, cpi->sf.mv.subpel_iters_per_step, + x->nmvjointcost, x->mvcost, &dis, &x->pred_sse[ref], ref_idx, + cpi->sf.use_upsampled_references); + } + *rate_mv = av1_mv_bit_cost(&tmp_mv->as_mv, &ref_mv, x->nmvjointcost, + x->mvcost, MV_COST_WEIGHT); + + if (cpi->sf.adaptive_motion_search && cm->show_frame) + x->pred_mv[ref] = tmp_mv->as_mv; + + if (scaled_ref_frame) { + for (i = 0; i < MAX_MB_PLANE; i++) + xd->plane[i].pre[ref_idx] = backup_yv12[i]; + } +} + +static void do_masked_motion_search_indexed( + const AV1_COMP *const cpi, MACROBLOCK *x, + const INTERINTER_COMPOUND_DATA *const comp_data, BLOCK_SIZE bsize, + int mi_row, int mi_col, int_mv *tmp_mv, int *rate_mv, int which) { + // NOTE: which values: 0 - 0 only, 1 - 1 only, 2 - both + MACROBLOCKD *xd = &x->e_mbd; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + BLOCK_SIZE sb_type = mbmi->sb_type; + const uint8_t *mask; + const int mask_stride = block_size_wide[bsize]; + + mask = av1_get_compound_type_mask(comp_data, sb_type); + + if (which == 0 || which == 2) + do_masked_motion_search(cpi, x, mask, mask_stride, bsize, mi_row, mi_col, + &tmp_mv[0], &rate_mv[0], 0); + + if (which == 1 || which == 2) { +// get the negative mask +#if CONFIG_COMPOUND_SEGMENT + uint8_t inv_mask_buf[2 * MAX_SB_SQUARE]; + const int h = block_size_high[bsize]; + mask = av1_get_compound_type_mask_inverse( + comp_data, inv_mask_buf, h, mask_stride, mask_stride, sb_type); +#else + mask = av1_get_compound_type_mask_inverse(comp_data, sb_type); +#endif // CONFIG_COMPOUND_SEGMENT + do_masked_motion_search(cpi, x, mask, mask_stride, bsize, mi_row, mi_col, + &tmp_mv[1], &rate_mv[1], 1); + } +} +#endif // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE +#endif // CONFIG_EXT_INTER + +// In some situations we want to discount tha pparent cost of a new motion +// vector. Where there is a subtle motion field and especially where there is +// low spatial complexity then it can be hard to cover the cost of a new motion +// vector in a single block, even if that motion vector reduces distortion. +// However, once established that vector may be usable through the nearest and +// near mv modes to reduce distortion in subsequent blocks and also improve +// visual quality. +static int discount_newmv_test(const AV1_COMP *const cpi, int this_mode, + int_mv this_mv, + int_mv (*mode_mv)[TOTAL_REFS_PER_FRAME], + int ref_frame) { + return (!cpi->rc.is_src_frame_alt_ref && (this_mode == NEWMV) && + (this_mv.as_int != 0) && + ((mode_mv[NEARESTMV][ref_frame].as_int == 0) || + (mode_mv[NEARESTMV][ref_frame].as_int == INVALID_MV)) && + ((mode_mv[NEARMV][ref_frame].as_int == 0) || + (mode_mv[NEARMV][ref_frame].as_int == INVALID_MV))); +} + +#define LEFT_TOP_MARGIN ((AOM_BORDER_IN_PIXELS - AOM_INTERP_EXTEND) << 3) +#define RIGHT_BOTTOM_MARGIN ((AOM_BORDER_IN_PIXELS - AOM_INTERP_EXTEND) << 3) + +// TODO(jingning): this mv clamping function should be block size dependent. +static INLINE void clamp_mv2(MV *mv, const MACROBLOCKD *xd) { + clamp_mv(mv, xd->mb_to_left_edge - LEFT_TOP_MARGIN, + xd->mb_to_right_edge + RIGHT_BOTTOM_MARGIN, + xd->mb_to_top_edge - LEFT_TOP_MARGIN, + xd->mb_to_bottom_edge + RIGHT_BOTTOM_MARGIN); +} + +#if CONFIG_EXT_INTER +#if CONFIG_WEDGE +static int estimate_wedge_sign(const AV1_COMP *cpi, const MACROBLOCK *x, + const BLOCK_SIZE bsize, const uint8_t *pred0, + int stride0, const uint8_t *pred1, int stride1) { + const struct macroblock_plane *const p = &x->plane[0]; + const uint8_t *src = p->src.buf; + int src_stride = p->src.stride; + const int f_index = bsize - BLOCK_8X8; + const int bw = block_size_wide[bsize]; + const int bh = block_size_high[bsize]; + uint32_t esq[2][4], var; + int64_t tl, br; + +#if CONFIG_HIGHBITDEPTH + if (x->e_mbd.cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + pred0 = CONVERT_TO_BYTEPTR(pred0); + pred1 = CONVERT_TO_BYTEPTR(pred1); + } +#endif // CONFIG_HIGHBITDEPTH + + var = cpi->fn_ptr[f_index].vf(src, src_stride, pred0, stride0, &esq[0][0]); + var = cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, pred0 + bw / 2, + stride0, &esq[0][1]); + var = cpi->fn_ptr[f_index].vf(src + bh / 2 * src_stride, src_stride, + pred0 + bh / 2 * stride0, stride0, &esq[0][2]); + var = cpi->fn_ptr[f_index].vf(src + bh / 2 * src_stride + bw / 2, src_stride, + pred0 + bh / 2 * stride0 + bw / 2, stride0, + &esq[0][3]); + var = cpi->fn_ptr[f_index].vf(src, src_stride, pred1, stride1, &esq[1][0]); + var = cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, pred1 + bw / 2, + stride1, &esq[1][1]); + var = cpi->fn_ptr[f_index].vf(src + bh / 2 * src_stride, src_stride, + pred1 + bh / 2 * stride1, stride0, &esq[1][2]); + var = cpi->fn_ptr[f_index].vf(src + bh / 2 * src_stride + bw / 2, src_stride, + pred1 + bh / 2 * stride1 + bw / 2, stride0, + &esq[1][3]); + (void)var; + + tl = (int64_t)(esq[0][0] + esq[0][1] + esq[0][2]) - + (int64_t)(esq[1][0] + esq[1][1] + esq[1][2]); + br = (int64_t)(esq[1][3] + esq[1][1] + esq[1][2]) - + (int64_t)(esq[0][3] + esq[0][1] + esq[0][2]); + return (tl + br > 0); +} +#endif // CONFIG_WEDGE +#endif // CONFIG_EXT_INTER + +#if !CONFIG_DUAL_FILTER +static InterpFilter predict_interp_filter( + const AV1_COMP *cpi, const MACROBLOCK *x, const BLOCK_SIZE bsize, + const int mi_row, const int mi_col, + InterpFilter (*single_filter)[TOTAL_REFS_PER_FRAME]) { + InterpFilter best_filter = SWITCHABLE; + const AV1_COMMON *cm = &cpi->common; + const MACROBLOCKD *xd = &x->e_mbd; + int bsl = mi_width_log2_lookup[bsize]; + int pred_filter_search = + cpi->sf.cb_pred_filter_search + ? (((mi_row + mi_col) >> bsl) + + get_chessboard_index(cm->current_video_frame)) & + 0x1 + : 0; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + const int is_comp_pred = has_second_ref(mbmi); + const int this_mode = mbmi->mode; + int refs[2] = { mbmi->ref_frame[0], + (mbmi->ref_frame[1] < 0 ? 0 : mbmi->ref_frame[1]) }; + if (pred_filter_search) { + InterpFilter af = SWITCHABLE, lf = SWITCHABLE; + if (xd->up_available) af = xd->mi[-xd->mi_stride]->mbmi.interp_filter; + if (xd->left_available) lf = xd->mi[-1]->mbmi.interp_filter; + +#if CONFIG_EXT_INTER + if ((this_mode != NEWMV && this_mode != NEW_NEWMV) || (af == lf)) +#else + if ((this_mode != NEWMV) || (af == lf)) +#endif // CONFIG_EXT_INTER + best_filter = af; + } + if (is_comp_pred) { + if (cpi->sf.adaptive_mode_search) { +#if CONFIG_EXT_INTER + switch (this_mode) { + case NEAREST_NEARESTMV: + if (single_filter[NEARESTMV][refs[0]] == + single_filter[NEARESTMV][refs[1]]) + best_filter = single_filter[NEARESTMV][refs[0]]; + break; + case NEAREST_NEARMV: + if (single_filter[NEARESTMV][refs[0]] == + single_filter[NEARMV][refs[1]]) + best_filter = single_filter[NEARESTMV][refs[0]]; + break; + case NEAR_NEARESTMV: + if (single_filter[NEARMV][refs[0]] == + single_filter[NEARESTMV][refs[1]]) + best_filter = single_filter[NEARMV][refs[0]]; + break; + case NEAR_NEARMV: + if (single_filter[NEARMV][refs[0]] == single_filter[NEARMV][refs[1]]) + best_filter = single_filter[NEARMV][refs[0]]; + break; + case ZERO_ZEROMV: + if (single_filter[ZEROMV][refs[0]] == single_filter[ZEROMV][refs[1]]) + best_filter = single_filter[ZEROMV][refs[0]]; + break; + case NEW_NEWMV: + if (single_filter[NEWMV][refs[0]] == single_filter[NEWMV][refs[1]]) + best_filter = single_filter[NEWMV][refs[0]]; + break; + case NEAREST_NEWMV: + if (single_filter[NEARESTMV][refs[0]] == + single_filter[NEWMV][refs[1]]) + best_filter = single_filter[NEARESTMV][refs[0]]; + break; + case NEAR_NEWMV: + if (single_filter[NEARMV][refs[0]] == single_filter[NEWMV][refs[1]]) + best_filter = single_filter[NEARMV][refs[0]]; + break; + case NEW_NEARESTMV: + if (single_filter[NEWMV][refs[0]] == + single_filter[NEARESTMV][refs[1]]) + best_filter = single_filter[NEWMV][refs[0]]; + break; + case NEW_NEARMV: + if (single_filter[NEWMV][refs[0]] == single_filter[NEARMV][refs[1]]) + best_filter = single_filter[NEWMV][refs[0]]; + break; + default: + if (single_filter[this_mode][refs[0]] == + single_filter[this_mode][refs[1]]) + best_filter = single_filter[this_mode][refs[0]]; + break; + } +#else + if (single_filter[this_mode][refs[0]] == + single_filter[this_mode][refs[1]]) + best_filter = single_filter[this_mode][refs[0]]; +#endif // CONFIG_EXT_INTER + } + } + if (x->source_variance < cpi->sf.disable_filter_search_var_thresh) { + best_filter = EIGHTTAP_REGULAR; + } + return best_filter; +} +#endif // !CONFIG_DUAL_FILTER + +#if CONFIG_EXT_INTER +// Choose the best wedge index and sign +#if CONFIG_WEDGE +static int64_t pick_wedge(const AV1_COMP *const cpi, const MACROBLOCK *const x, + const BLOCK_SIZE bsize, const uint8_t *const p0, + const uint8_t *const p1, int *const best_wedge_sign, + int *const best_wedge_index) { + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const src = &x->plane[0].src; + const int bw = block_size_wide[bsize]; + const int bh = block_size_high[bsize]; + const int N = bw * bh; + int rate; + int64_t dist; + int64_t rd, best_rd = INT64_MAX; + int wedge_index; + int wedge_sign; + int wedge_types = (1 << get_wedge_bits_lookup(bsize)); + const uint8_t *mask; + uint64_t sse; +#if CONFIG_HIGHBITDEPTH + const int hbd = xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH; + const int bd_round = hbd ? (xd->bd - 8) * 2 : 0; +#else + const int bd_round = 0; +#endif // CONFIG_HIGHBITDEPTH + + DECLARE_ALIGNED(32, int16_t, r0[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, r1[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, d10[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, ds[MAX_SB_SQUARE]); + + int64_t sign_limit; + +#if CONFIG_HIGHBITDEPTH + if (hbd) { + aom_highbd_subtract_block(bh, bw, r0, bw, src->buf, src->stride, + CONVERT_TO_BYTEPTR(p0), bw, xd->bd); + aom_highbd_subtract_block(bh, bw, r1, bw, src->buf, src->stride, + CONVERT_TO_BYTEPTR(p1), bw, xd->bd); + aom_highbd_subtract_block(bh, bw, d10, bw, CONVERT_TO_BYTEPTR(p1), bw, + CONVERT_TO_BYTEPTR(p0), bw, xd->bd); + } else // NOLINT +#endif // CONFIG_HIGHBITDEPTH + { + aom_subtract_block(bh, bw, r0, bw, src->buf, src->stride, p0, bw); + aom_subtract_block(bh, bw, r1, bw, src->buf, src->stride, p1, bw); + aom_subtract_block(bh, bw, d10, bw, p1, bw, p0, bw); + } + + sign_limit = ((int64_t)aom_sum_squares_i16(r0, N) - + (int64_t)aom_sum_squares_i16(r1, N)) * + (1 << WEDGE_WEIGHT_BITS) / 2; + + if (N < 64) + av1_wedge_compute_delta_squares_c(ds, r0, r1, N); + else + av1_wedge_compute_delta_squares(ds, r0, r1, N); + + for (wedge_index = 0; wedge_index < wedge_types; ++wedge_index) { + mask = av1_get_contiguous_soft_mask(wedge_index, 0, bsize); + + // TODO(jingning): Make sse2 functions support N = 16 case + if (N < 64) + wedge_sign = av1_wedge_sign_from_residuals_c(ds, mask, N, sign_limit); + else + wedge_sign = av1_wedge_sign_from_residuals(ds, mask, N, sign_limit); + + mask = av1_get_contiguous_soft_mask(wedge_index, wedge_sign, bsize); + if (N < 64) + sse = av1_wedge_sse_from_residuals_c(r1, d10, mask, N); + else + sse = av1_wedge_sse_from_residuals(r1, d10, mask, N); + sse = ROUND_POWER_OF_TWO(sse, bd_round); + + model_rd_from_sse(cpi, xd, bsize, 0, sse, &rate, &dist); + rd = RDCOST(x->rdmult, x->rddiv, rate, dist); + + if (rd < best_rd) { + *best_wedge_index = wedge_index; + *best_wedge_sign = wedge_sign; + best_rd = rd; + } + } + + return best_rd; +} + +// Choose the best wedge index the specified sign +static int64_t pick_wedge_fixed_sign( + const AV1_COMP *const cpi, const MACROBLOCK *const x, + const BLOCK_SIZE bsize, const uint8_t *const p0, const uint8_t *const p1, + const int wedge_sign, int *const best_wedge_index) { + const MACROBLOCKD *const xd = &x->e_mbd; + const struct buf_2d *const src = &x->plane[0].src; + const int bw = block_size_wide[bsize]; + const int bh = block_size_high[bsize]; + const int N = bw * bh; + int rate; + int64_t dist; + int64_t rd, best_rd = INT64_MAX; + int wedge_index; + int wedge_types = (1 << get_wedge_bits_lookup(bsize)); + const uint8_t *mask; + uint64_t sse; +#if CONFIG_HIGHBITDEPTH + const int hbd = xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH; + const int bd_round = hbd ? (xd->bd - 8) * 2 : 0; +#else + const int bd_round = 0; +#endif // CONFIG_HIGHBITDEPTH + + DECLARE_ALIGNED(32, int16_t, r1[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, d10[MAX_SB_SQUARE]); + +#if CONFIG_HIGHBITDEPTH + if (hbd) { + aom_highbd_subtract_block(bh, bw, r1, bw, src->buf, src->stride, + CONVERT_TO_BYTEPTR(p1), bw, xd->bd); + aom_highbd_subtract_block(bh, bw, d10, bw, CONVERT_TO_BYTEPTR(p1), bw, + CONVERT_TO_BYTEPTR(p0), bw, xd->bd); + } else // NOLINT +#endif // CONFIG_HIGHBITDEPTH + { + aom_subtract_block(bh, bw, r1, bw, src->buf, src->stride, p1, bw); + aom_subtract_block(bh, bw, d10, bw, p1, bw, p0, bw); + } + + for (wedge_index = 0; wedge_index < wedge_types; ++wedge_index) { + mask = av1_get_contiguous_soft_mask(wedge_index, wedge_sign, bsize); + if (N < 64) + sse = av1_wedge_sse_from_residuals_c(r1, d10, mask, N); + else + sse = av1_wedge_sse_from_residuals(r1, d10, mask, N); + sse = ROUND_POWER_OF_TWO(sse, bd_round); + + model_rd_from_sse(cpi, xd, bsize, 0, sse, &rate, &dist); + rd = RDCOST(x->rdmult, x->rddiv, rate, dist); + + if (rd < best_rd) { + *best_wedge_index = wedge_index; + best_rd = rd; + } + } + + return best_rd; +} + +static int64_t pick_interinter_wedge(const AV1_COMP *const cpi, + MACROBLOCK *const x, + const BLOCK_SIZE bsize, + const uint8_t *const p0, + const uint8_t *const p1) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const int bw = block_size_wide[bsize]; + + int64_t rd; + int wedge_index = -1; + int wedge_sign = 0; + + assert(is_interinter_compound_used(COMPOUND_WEDGE, bsize)); + + if (cpi->sf.fast_wedge_sign_estimate) { + wedge_sign = estimate_wedge_sign(cpi, x, bsize, p0, bw, p1, bw); + rd = pick_wedge_fixed_sign(cpi, x, bsize, p0, p1, wedge_sign, &wedge_index); + } else { + rd = pick_wedge(cpi, x, bsize, p0, p1, &wedge_sign, &wedge_index); + } + + mbmi->wedge_sign = wedge_sign; + mbmi->wedge_index = wedge_index; + return rd; +} +#endif // CONFIG_WEDGE + +#if CONFIG_COMPOUND_SEGMENT +static int64_t pick_interinter_seg(const AV1_COMP *const cpi, + MACROBLOCK *const x, const BLOCK_SIZE bsize, + const uint8_t *const p0, + const uint8_t *const p1) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const struct buf_2d *const src = &x->plane[0].src; + const int bw = block_size_wide[bsize]; + const int bh = block_size_high[bsize]; + const int N = bw * bh; + int rate; + uint64_t sse; + int64_t dist; + int64_t rd0; + SEG_MASK_TYPE cur_mask_type; + int64_t best_rd = INT64_MAX; + SEG_MASK_TYPE best_mask_type = 0; +#if CONFIG_HIGHBITDEPTH + const int hbd = xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH; + const int bd_round = hbd ? (xd->bd - 8) * 2 : 0; +#else + const int bd_round = 0; +#endif // CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(32, int16_t, r0[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, r1[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, d10[MAX_SB_SQUARE]); + +#if CONFIG_HIGHBITDEPTH + if (hbd) { + aom_highbd_subtract_block(bh, bw, r0, bw, src->buf, src->stride, + CONVERT_TO_BYTEPTR(p0), bw, xd->bd); + aom_highbd_subtract_block(bh, bw, r1, bw, src->buf, src->stride, + CONVERT_TO_BYTEPTR(p1), bw, xd->bd); + aom_highbd_subtract_block(bh, bw, d10, bw, CONVERT_TO_BYTEPTR(p1), bw, + CONVERT_TO_BYTEPTR(p0), bw, xd->bd); + } else // NOLINT +#endif // CONFIG_HIGHBITDEPTH + { + aom_subtract_block(bh, bw, r0, bw, src->buf, src->stride, p0, bw); + aom_subtract_block(bh, bw, r1, bw, src->buf, src->stride, p1, bw); + aom_subtract_block(bh, bw, d10, bw, p1, bw, p0, bw); + } + + // try each mask type and its inverse + for (cur_mask_type = 0; cur_mask_type < SEG_MASK_TYPES; cur_mask_type++) { +// build mask and inverse +#if CONFIG_HIGHBITDEPTH + if (hbd) + build_compound_seg_mask_highbd( + xd->seg_mask, cur_mask_type, CONVERT_TO_BYTEPTR(p0), bw, + CONVERT_TO_BYTEPTR(p1), bw, bsize, bh, bw, xd->bd); + else +#endif // CONFIG_HIGHBITDEPTH + build_compound_seg_mask(xd->seg_mask, cur_mask_type, p0, bw, p1, bw, + bsize, bh, bw); + + // compute rd for mask + sse = av1_wedge_sse_from_residuals(r1, d10, xd->seg_mask, N); + sse = ROUND_POWER_OF_TWO(sse, bd_round); + + model_rd_from_sse(cpi, xd, bsize, 0, sse, &rate, &dist); + rd0 = RDCOST(x->rdmult, x->rddiv, rate, dist); + + if (rd0 < best_rd) { + best_mask_type = cur_mask_type; + best_rd = rd0; + } + } + + // make final mask + mbmi->mask_type = best_mask_type; +#if CONFIG_HIGHBITDEPTH + if (hbd) + build_compound_seg_mask_highbd( + xd->seg_mask, mbmi->mask_type, CONVERT_TO_BYTEPTR(p0), bw, + CONVERT_TO_BYTEPTR(p1), bw, bsize, bh, bw, xd->bd); + else +#endif // CONFIG_HIGHBITDEPTH + build_compound_seg_mask(xd->seg_mask, mbmi->mask_type, p0, bw, p1, bw, + bsize, bh, bw); + + return best_rd; +} +#endif // CONFIG_COMPOUND_SEGMENT + +#if CONFIG_WEDGE && CONFIG_INTERINTRA +static int64_t pick_interintra_wedge(const AV1_COMP *const cpi, + const MACROBLOCK *const x, + const BLOCK_SIZE bsize, + const uint8_t *const p0, + const uint8_t *const p1) { + const MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + + int64_t rd; + int wedge_index = -1; + + assert(is_interintra_wedge_used(bsize)); + + rd = pick_wedge_fixed_sign(cpi, x, bsize, p0, p1, 0, &wedge_index); + + mbmi->interintra_wedge_sign = 0; + mbmi->interintra_wedge_index = wedge_index; + return rd; +} +#endif // CONFIG_WEDGE && CONFIG_INTERINTRA + +#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE +static int64_t pick_interinter_mask(const AV1_COMP *const cpi, MACROBLOCK *x, + const BLOCK_SIZE bsize, + const uint8_t *const p0, + const uint8_t *const p1) { + const COMPOUND_TYPE compound_type = + x->e_mbd.mi[0]->mbmi.interinter_compound_type; + switch (compound_type) { +#if CONFIG_WEDGE + case COMPOUND_WEDGE: return pick_interinter_wedge(cpi, x, bsize, p0, p1); +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + case COMPOUND_SEG: return pick_interinter_seg(cpi, x, bsize, p0, p1); +#endif // CONFIG_COMPOUND_SEGMENT + default: assert(0); return 0; + } +} + +static int interinter_compound_motion_search(const AV1_COMP *const cpi, + MACROBLOCK *x, + const BLOCK_SIZE bsize, + const int this_mode, int mi_row, + int mi_col) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + int_mv tmp_mv[2]; + int rate_mvs[2], tmp_rate_mv = 0; + const INTERINTER_COMPOUND_DATA compound_data = { +#if CONFIG_WEDGE + mbmi->wedge_index, + mbmi->wedge_sign, +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + mbmi->mask_type, + xd->seg_mask, +#endif // CONFIG_COMPOUND_SEGMENT + mbmi->interinter_compound_type + }; + if (this_mode == NEW_NEWMV) { + do_masked_motion_search_indexed(cpi, x, &compound_data, bsize, mi_row, + mi_col, tmp_mv, rate_mvs, 2); + tmp_rate_mv = rate_mvs[0] + rate_mvs[1]; + mbmi->mv[0].as_int = tmp_mv[0].as_int; + mbmi->mv[1].as_int = tmp_mv[1].as_int; + } else if (this_mode == NEW_NEARESTMV || this_mode == NEW_NEARMV) { + do_masked_motion_search_indexed(cpi, x, &compound_data, bsize, mi_row, + mi_col, tmp_mv, rate_mvs, 0); + tmp_rate_mv = rate_mvs[0]; + mbmi->mv[0].as_int = tmp_mv[0].as_int; + } else if (this_mode == NEAREST_NEWMV || this_mode == NEAR_NEWMV) { + do_masked_motion_search_indexed(cpi, x, &compound_data, bsize, mi_row, + mi_col, tmp_mv, rate_mvs, 1); + tmp_rate_mv = rate_mvs[1]; + mbmi->mv[1].as_int = tmp_mv[1].as_int; + } + return tmp_rate_mv; +} + +static int64_t build_and_cost_compound_type( + const AV1_COMP *const cpi, MACROBLOCK *x, const int_mv *const cur_mv, + const BLOCK_SIZE bsize, const int this_mode, int rs2, int rate_mv, + BUFFER_SET *ctx, int *out_rate_mv, uint8_t **preds0, uint8_t **preds1, + int *strides, int mi_row, int mi_col) { + MACROBLOCKD *xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + int rate_sum; + int64_t dist_sum; + int64_t best_rd_cur = INT64_MAX; + int64_t rd = INT64_MAX; + int tmp_skip_txfm_sb; + int64_t tmp_skip_sse_sb; + const COMPOUND_TYPE compound_type = mbmi->interinter_compound_type; + + best_rd_cur = pick_interinter_mask(cpi, x, bsize, *preds0, *preds1); + best_rd_cur += RDCOST(x->rdmult, x->rddiv, rs2 + rate_mv, 0); + + if (have_newmv_in_inter_mode(this_mode) && + use_masked_motion_search(compound_type)) { + *out_rate_mv = interinter_compound_motion_search(cpi, x, bsize, this_mode, + mi_row, mi_col); + av1_build_inter_predictors_sby(xd, mi_row, mi_col, ctx, bsize); + model_rd_for_sb(cpi, bsize, x, xd, 0, 0, &rate_sum, &dist_sum, + &tmp_skip_txfm_sb, &tmp_skip_sse_sb); + rd = RDCOST(x->rdmult, x->rddiv, rs2 + *out_rate_mv + rate_sum, dist_sum); + if (rd >= best_rd_cur) { + mbmi->mv[0].as_int = cur_mv[0].as_int; + mbmi->mv[1].as_int = cur_mv[1].as_int; + *out_rate_mv = rate_mv; + av1_build_wedge_inter_predictor_from_buf(xd, bsize, 0, 0, +#if CONFIG_SUPERTX + 0, 0, +#endif // CONFIG_SUPERTX + preds0, strides, preds1, + strides); + } + av1_subtract_plane(x, bsize, 0); + rd = estimate_yrd_for_sb(cpi, bsize, x, &rate_sum, &dist_sum, + &tmp_skip_txfm_sb, &tmp_skip_sse_sb, INT64_MAX); + if (rd != INT64_MAX) + rd = RDCOST(x->rdmult, x->rddiv, rs2 + *out_rate_mv + rate_sum, dist_sum); + best_rd_cur = rd; + + } else { + av1_build_wedge_inter_predictor_from_buf(xd, bsize, 0, 0, +#if CONFIG_SUPERTX + 0, 0, +#endif // CONFIG_SUPERTX + preds0, strides, preds1, strides); + av1_subtract_plane(x, bsize, 0); + rd = estimate_yrd_for_sb(cpi, bsize, x, &rate_sum, &dist_sum, + &tmp_skip_txfm_sb, &tmp_skip_sse_sb, INT64_MAX); + if (rd != INT64_MAX) + rd = RDCOST(x->rdmult, x->rddiv, rs2 + rate_mv + rate_sum, dist_sum); + best_rd_cur = rd; + } + return best_rd_cur; +} +#endif // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE +#endif // CONFIG_EXT_INTER + +typedef struct { +#if CONFIG_MOTION_VAR + // Inter prediction buffers and respective strides + uint8_t *above_pred_buf[MAX_MB_PLANE]; + int above_pred_stride[MAX_MB_PLANE]; + uint8_t *left_pred_buf[MAX_MB_PLANE]; + int left_pred_stride[MAX_MB_PLANE]; +#endif // CONFIG_MOTION_VAR + int_mv *single_newmv; +#if CONFIG_EXT_INTER + // Pointer to array of motion vectors to use for each ref and their rates + // Should point to first of 2 arrays in 2D array + int *single_newmv_rate; + // Pointers costs of compound inter-intra and inter-inter predictions + int *compmode_interintra_cost; + int *compmode_interinter_cost; + // Pointer to array of predicted rate-distortion + // Should point to first of 2 arrays in 2D array + int64_t (*modelled_rd)[TOTAL_REFS_PER_FRAME]; +#endif // CONFIG_EXT_INTER + InterpFilter single_filter[MB_MODE_COUNT][TOTAL_REFS_PER_FRAME]; +} HandleInterModeArgs; + +static int64_t handle_newmv(const AV1_COMP *const cpi, MACROBLOCK *const x, + const BLOCK_SIZE bsize, + int_mv (*const mode_mv)[TOTAL_REFS_PER_FRAME], + const int mi_row, const int mi_col, + int *const rate_mv, int_mv *const single_newmv, + HandleInterModeArgs *const args) { + const MACROBLOCKD *const xd = &x->e_mbd; + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; + const int is_comp_pred = has_second_ref(mbmi); + const PREDICTION_MODE this_mode = mbmi->mode; +#if CONFIG_EXT_INTER + const int is_comp_interintra_pred = (mbmi->ref_frame[1] == INTRA_FRAME); +#endif // CONFIG_EXT_INTER + int_mv *const frame_mv = mode_mv[this_mode]; + const int refs[2] = { mbmi->ref_frame[0], + mbmi->ref_frame[1] < 0 ? 0 : mbmi->ref_frame[1] }; + int i; + + (void)args; + + if (is_comp_pred) { +#if CONFIG_EXT_INTER + for (i = 0; i < 2; ++i) { + single_newmv[refs[i]].as_int = args->single_newmv[refs[i]].as_int; + } + + if (this_mode == NEW_NEWMV) { + frame_mv[refs[0]].as_int = single_newmv[refs[0]].as_int; + frame_mv[refs[1]].as_int = single_newmv[refs[1]].as_int; + + if (cpi->sf.comp_inter_joint_search_thresh <= bsize) { + joint_motion_search(cpi, x, bsize, frame_mv, mi_row, mi_col, NULL, + rate_mv, 0); + } else { + *rate_mv = 0; + for (i = 0; i < 2; ++i) { +#if CONFIG_REF_MV + av1_set_mvcost(x, refs[i], i, mbmi->ref_mv_idx); +#endif // CONFIG_REF_MV + *rate_mv += av1_mv_bit_cost( + &frame_mv[refs[i]].as_mv, &mbmi_ext->ref_mvs[refs[i]][0].as_mv, + x->nmvjointcost, x->mvcost, MV_COST_WEIGHT); + } + } + } else if (this_mode == NEAREST_NEWMV || this_mode == NEAR_NEWMV) { + frame_mv[refs[1]].as_int = single_newmv[refs[1]].as_int; +#if CONFIG_REF_MV + av1_set_mvcost(x, refs[1], 1, mbmi->ref_mv_idx); +#endif // CONFIG_REF_MV + *rate_mv = av1_mv_bit_cost(&frame_mv[refs[1]].as_mv, + &mbmi_ext->ref_mvs[refs[1]][0].as_mv, + x->nmvjointcost, x->mvcost, MV_COST_WEIGHT); + } else { + assert(this_mode == NEW_NEARESTMV || this_mode == NEW_NEARMV); + frame_mv[refs[0]].as_int = single_newmv[refs[0]].as_int; +#if CONFIG_REF_MV + av1_set_mvcost(x, refs[0], 0, mbmi->ref_mv_idx); +#endif // CONFIG_REF_MV + *rate_mv = av1_mv_bit_cost(&frame_mv[refs[0]].as_mv, + &mbmi_ext->ref_mvs[refs[0]][0].as_mv, + x->nmvjointcost, x->mvcost, MV_COST_WEIGHT); + } +#else + // Initialize mv using single prediction mode result. + frame_mv[refs[0]].as_int = single_newmv[refs[0]].as_int; + frame_mv[refs[1]].as_int = single_newmv[refs[1]].as_int; + + if (cpi->sf.comp_inter_joint_search_thresh <= bsize) { + joint_motion_search(cpi, x, bsize, frame_mv, mi_row, mi_col, rate_mv, 0); + } else { + *rate_mv = 0; + for (i = 0; i < 2; ++i) { +#if CONFIG_REF_MV + av1_set_mvcost(x, refs[i], i, mbmi->ref_mv_idx); +#endif // CONFIG_REF_MV + *rate_mv += av1_mv_bit_cost(&frame_mv[refs[i]].as_mv, + &mbmi_ext->ref_mvs[refs[i]][0].as_mv, + x->nmvjointcost, x->mvcost, MV_COST_WEIGHT); + } + } +#endif // CONFIG_EXT_INTER + } else { +#if CONFIG_EXT_INTER + if (is_comp_interintra_pred) { + x->best_mv = args->single_newmv[refs[0]]; + *rate_mv = args->single_newmv_rate[refs[0]]; + } else { + single_motion_search(cpi, x, bsize, mi_row, mi_col, 0, rate_mv); + args->single_newmv[refs[0]] = x->best_mv; + args->single_newmv_rate[refs[0]] = *rate_mv; + } +#else + single_motion_search(cpi, x, bsize, mi_row, mi_col, rate_mv); + single_newmv[refs[0]] = x->best_mv; +#endif // CONFIG_EXT_INTER + + if (x->best_mv.as_int == INVALID_MV) return INT64_MAX; + + frame_mv[refs[0]] = x->best_mv; + xd->mi[0]->bmi[0].as_mv[0] = x->best_mv; + + // Estimate the rate implications of a new mv but discount this + // under certain circumstances where we want to help initiate a weak + // motion field, where the distortion gain for a single block may not + // be enough to overcome the cost of a new mv. + if (discount_newmv_test(cpi, this_mode, x->best_mv, mode_mv, refs[0])) { + *rate_mv = AOMMAX(*rate_mv / NEW_MV_DISCOUNT_FACTOR, 1); + } + } + + return 0; +} + +int64_t interpolation_filter_search( + MACROBLOCK *const x, const AV1_COMP *const cpi, BLOCK_SIZE bsize, + int mi_row, int mi_col, const BUFFER_SET *const tmp_dst, + BUFFER_SET *const orig_dst, + InterpFilter (*const single_filter)[TOTAL_REFS_PER_FRAME], + int64_t *const rd, int *const switchable_rate, int *const skip_txfm_sb, + int64_t *const skip_sse_sb) { + const AV1_COMMON *cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + int i; + int tmp_rate; + int64_t tmp_dist; + + (void)single_filter; + + InterpFilter assign_filter = SWITCHABLE; + + if (cm->interp_filter == SWITCHABLE) { +#if !CONFIG_DUAL_FILTER + assign_filter = av1_is_interp_needed(xd) + ? predict_interp_filter(cpi, x, bsize, mi_row, mi_col, + single_filter) + : cm->interp_filter; +#endif // !CONFIG_DUAL_FILTER + } else { + assign_filter = cm->interp_filter; + } + + set_default_interp_filters(mbmi, assign_filter); + + *switchable_rate = av1_get_switchable_rate(cpi, xd); + av1_build_inter_predictors_sb(xd, mi_row, mi_col, orig_dst, bsize); + model_rd_for_sb(cpi, bsize, x, xd, 0, MAX_MB_PLANE - 1, &tmp_rate, &tmp_dist, + skip_txfm_sb, skip_sse_sb); + *rd = RDCOST(x->rdmult, x->rddiv, *switchable_rate + tmp_rate, tmp_dist); + + if (assign_filter == SWITCHABLE) { + // do interp_filter search + if (av1_is_interp_needed(xd) && av1_is_interp_search_needed(xd)) { +#if CONFIG_DUAL_FILTER + const int filter_set_size = DUAL_FILTER_SET_SIZE; +#else + const int filter_set_size = SWITCHABLE_FILTERS; +#endif // CONFIG_DUAL_FILTER + int best_in_temp = 0; +#if CONFIG_DUAL_FILTER + InterpFilter best_filter[4]; + av1_copy(best_filter, mbmi->interp_filter); +#else + InterpFilter best_filter = mbmi->interp_filter; +#endif // CONFIG_DUAL_FILTER + restore_dst_buf(xd, *tmp_dst); + // EIGHTTAP_REGULAR mode is calculated beforehand + for (i = 1; i < filter_set_size; ++i) { + int tmp_skip_sb = 0; + int64_t tmp_skip_sse = INT64_MAX; + int tmp_rs; + int64_t tmp_rd; +#if CONFIG_DUAL_FILTER + mbmi->interp_filter[0] = filter_sets[i][0]; + mbmi->interp_filter[1] = filter_sets[i][1]; + mbmi->interp_filter[2] = filter_sets[i][0]; + mbmi->interp_filter[3] = filter_sets[i][1]; +#else + mbmi->interp_filter = (InterpFilter)i; +#endif // CONFIG_DUAL_FILTER + tmp_rs = av1_get_switchable_rate(cpi, xd); + av1_build_inter_predictors_sb(xd, mi_row, mi_col, orig_dst, bsize); + model_rd_for_sb(cpi, bsize, x, xd, 0, MAX_MB_PLANE - 1, &tmp_rate, + &tmp_dist, &tmp_skip_sb, &tmp_skip_sse); + tmp_rd = RDCOST(x->rdmult, x->rddiv, tmp_rs + tmp_rate, tmp_dist); + + if (tmp_rd < *rd) { + *rd = tmp_rd; + *switchable_rate = av1_get_switchable_rate(cpi, xd); +#if CONFIG_DUAL_FILTER + av1_copy(best_filter, mbmi->interp_filter); +#else + best_filter = mbmi->interp_filter; +#endif // CONFIG_DUAL_FILTER + *skip_txfm_sb = tmp_skip_sb; + *skip_sse_sb = tmp_skip_sse; + best_in_temp = !best_in_temp; + if (best_in_temp) { + restore_dst_buf(xd, *orig_dst); + } else { + restore_dst_buf(xd, *tmp_dst); + } + } + } + if (best_in_temp) { + restore_dst_buf(xd, *tmp_dst); + } else { + restore_dst_buf(xd, *orig_dst); + } +#if CONFIG_DUAL_FILTER + av1_copy(mbmi->interp_filter, best_filter); +#else + mbmi->interp_filter = best_filter; +#endif // CONFIG_DUAL_FILTER + } else { +#if CONFIG_DUAL_FILTER + for (i = 0; i < 4; ++i) + assert(mbmi->interp_filter[i] == EIGHTTAP_REGULAR); +#else + assert(mbmi->interp_filter == EIGHTTAP_REGULAR); +#endif // CONFIG_DUAL_FILTER + } + } + + return 0; +} + +// TODO(afergs): Refactor the MBMI references in here - there's four +// TODO(afergs): Refactor optional args - add them to a struct or remove +static int64_t motion_mode_rd( + const AV1_COMP *const cpi, MACROBLOCK *const x, BLOCK_SIZE bsize, + RD_STATS *rd_stats, RD_STATS *rd_stats_y, RD_STATS *rd_stats_uv, + int *disable_skip, int_mv (*mode_mv)[TOTAL_REFS_PER_FRAME], int mi_row, + int mi_col, HandleInterModeArgs *const args, const int64_t ref_best_rd, + const int *refs, int rate_mv, +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +#if CONFIG_EXT_INTER + int rate2_bmc_nocoeff, MB_MODE_INFO *best_bmc_mbmi, +#if CONFIG_MOTION_VAR + int rate_mv_bmc, +#endif // CONFIG_MOTION_VAR +#endif // CONFIG_EXT_INTER +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + int rs, int *skip_txfm_sb, int64_t *skip_sse_sb, BUFFER_SET *orig_dst) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *xd = &x->e_mbd; + MODE_INFO *mi = xd->mi[0]; + MB_MODE_INFO *mbmi = &mi->mbmi; + const int is_comp_pred = has_second_ref(mbmi); + const PREDICTION_MODE this_mode = mbmi->mode; + + (void)mode_mv; + (void)mi_row; + (void)mi_col; + (void)args; + (void)refs; + (void)rate_mv; + (void)is_comp_pred; + (void)this_mode; + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + MOTION_MODE motion_mode, last_motion_mode_allowed; + int rate2_nocoeff = 0, best_xskip, best_disable_skip = 0; + RD_STATS best_rd_stats, best_rd_stats_y, best_rd_stats_uv; + MB_MODE_INFO base_mbmi, best_mbmi; +#if CONFIG_VAR_TX + uint8_t best_blk_skip[MAX_MB_PLANE][MAX_MIB_SIZE * MAX_MIB_SIZE * 4]; +#endif // CONFIG_VAR_TX +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + +#if CONFIG_WARPED_MOTION + int pts[SAMPLES_ARRAY_SIZE], pts_inref[SAMPLES_ARRAY_SIZE]; +#endif // CONFIG_WARPED_MOTION + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + av1_invalid_rd_stats(&best_rd_stats); +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + + if (cm->interp_filter == SWITCHABLE) rd_stats->rate += rs; +#if CONFIG_WARPED_MOTION + aom_clear_system_state(); + mbmi->num_proj_ref[0] = findSamples(cm, xd, mi_row, mi_col, pts, pts_inref); +#if CONFIG_EXT_INTER + best_bmc_mbmi->num_proj_ref[0] = mbmi->num_proj_ref[0]; +#endif // CONFIG_EXT_INTER +#endif // CONFIG_WARPED_MOTION +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + rate2_nocoeff = rd_stats->rate; + last_motion_mode_allowed = motion_mode_allowed( +#if CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + 0, xd->global_motion, +#endif // CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + mi); + base_mbmi = *mbmi; +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + int64_t best_rd = INT64_MAX; + for (motion_mode = SIMPLE_TRANSLATION; + motion_mode <= last_motion_mode_allowed; motion_mode++) { + int64_t tmp_rd = INT64_MAX; + int tmp_rate; + int64_t tmp_dist; +#if CONFIG_EXT_INTER + int tmp_rate2 = + motion_mode != SIMPLE_TRANSLATION ? rate2_bmc_nocoeff : rate2_nocoeff; +#else + int tmp_rate2 = rate2_nocoeff; +#endif // CONFIG_EXT_INTER + + *mbmi = base_mbmi; + mbmi->motion_mode = motion_mode; +#if CONFIG_MOTION_VAR + if (mbmi->motion_mode == OBMC_CAUSAL) { +#if CONFIG_EXT_INTER + *mbmi = *best_bmc_mbmi; + mbmi->motion_mode = OBMC_CAUSAL; +#endif // CONFIG_EXT_INTER + if (!is_comp_pred && have_newmv_in_inter_mode(this_mode)) { + int tmp_rate_mv = 0; + + single_motion_search(cpi, x, bsize, mi_row, mi_col, +#if CONFIG_EXT_INTER + 0, +#endif // CONFIG_EXT_INTER + &tmp_rate_mv); + mbmi->mv[0].as_int = x->best_mv.as_int; + if (discount_newmv_test(cpi, this_mode, mbmi->mv[0], mode_mv, + refs[0])) { + tmp_rate_mv = AOMMAX((tmp_rate_mv / NEW_MV_DISCOUNT_FACTOR), 1); + } +#if CONFIG_EXT_INTER + tmp_rate2 = rate2_bmc_nocoeff - rate_mv_bmc + tmp_rate_mv; +#else + tmp_rate2 = rate2_nocoeff - rate_mv + tmp_rate_mv; +#endif // CONFIG_EXT_INTER +#if CONFIG_DUAL_FILTER + if (!has_subpel_mv_component(xd->mi[0], xd, 0)) + mbmi->interp_filter[0] = EIGHTTAP_REGULAR; + if (!has_subpel_mv_component(xd->mi[0], xd, 1)) + mbmi->interp_filter[1] = EIGHTTAP_REGULAR; +#endif // CONFIG_DUAL_FILTER + av1_build_inter_predictors_sb(xd, mi_row, mi_col, orig_dst, bsize); +#if CONFIG_EXT_INTER + } else { + av1_build_inter_predictors_sb(xd, mi_row, mi_col, orig_dst, bsize); +#endif // CONFIG_EXT_INTER + } + av1_build_obmc_inter_prediction( + cm, xd, mi_row, mi_col, args->above_pred_buf, args->above_pred_stride, + args->left_pred_buf, args->left_pred_stride); + model_rd_for_sb(cpi, bsize, x, xd, 0, MAX_MB_PLANE - 1, &tmp_rate, + &tmp_dist, skip_txfm_sb, skip_sse_sb); + } +#endif // CONFIG_MOTION_VAR + +#if CONFIG_WARPED_MOTION + if (mbmi->motion_mode == WARPED_CAUSAL) { +#if CONFIG_EXT_INTER + *mbmi = *best_bmc_mbmi; + mbmi->motion_mode = WARPED_CAUSAL; +#endif // CONFIG_EXT_INTER + mbmi->wm_params[0].wmtype = DEFAULT_WMTYPE; +#if CONFIG_DUAL_FILTER + for (int dir = 0; dir < 4; ++dir) + mbmi->interp_filter[dir] = cm->interp_filter == SWITCHABLE + ? EIGHTTAP_REGULAR + : cm->interp_filter; +#else + mbmi->interp_filter = cm->interp_filter == SWITCHABLE ? EIGHTTAP_REGULAR + : cm->interp_filter; +#endif // CONFIG_DUAL_FILTER + + if (find_projection(mbmi->num_proj_ref[0], pts, pts_inref, bsize, + mbmi->mv[0].as_mv.row, mbmi->mv[0].as_mv.col, + &mbmi->wm_params[0], mi_row, mi_col) == 0) { + av1_build_inter_predictors_sb(xd, mi_row, mi_col, NULL, bsize); + model_rd_for_sb(cpi, bsize, x, xd, 0, MAX_MB_PLANE - 1, &tmp_rate, + &tmp_dist, skip_txfm_sb, skip_sse_sb); + } else { + continue; + } + } +#endif // CONFIG_WARPED_MOTION + x->skip = 0; + + rd_stats->dist = 0; + rd_stats->sse = 0; + rd_stats->skip = 1; + rd_stats->rate = tmp_rate2; + if (last_motion_mode_allowed > SIMPLE_TRANSLATION) { +#if CONFIG_WARPED_MOTION && CONFIG_MOTION_VAR + if (last_motion_mode_allowed == WARPED_CAUSAL) +#endif // CONFIG_WARPED_MOTION && CONFIG_MOTION_VAR + rd_stats->rate += cpi->motion_mode_cost[bsize][mbmi->motion_mode]; +#if CONFIG_WARPED_MOTION && CONFIG_MOTION_VAR + else + rd_stats->rate += cpi->motion_mode_cost1[bsize][mbmi->motion_mode]; +#endif // CONFIG_WARPED_MOTION && CONFIG_MOTION_VAR + } +#if CONFIG_WARPED_MOTION + if (mbmi->motion_mode == WARPED_CAUSAL) { + rd_stats->rate -= rs; + } +#endif // CONFIG_WARPED_MOTION +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + if (!*skip_txfm_sb) { + int64_t rdcosty = INT64_MAX; + int is_cost_valid_uv = 0; + + // cost and distortion + av1_subtract_plane(x, bsize, 0); +#if CONFIG_VAR_TX + if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id]) { + select_tx_type_yrd(cpi, x, rd_stats_y, bsize, ref_best_rd); + } else { + int idx, idy; + super_block_yrd(cpi, x, rd_stats_y, bsize, ref_best_rd); + for (idy = 0; idy < xd->n8_h; ++idy) + for (idx = 0; idx < xd->n8_w; ++idx) + mbmi->inter_tx_size[idy][idx] = mbmi->tx_size; + memset(x->blk_skip[0], rd_stats_y->skip, + sizeof(uint8_t) * xd->n8_h * xd->n8_w * 4); + } +#else + /* clang-format off */ + super_block_yrd(cpi, x, rd_stats_y, bsize, ref_best_rd); +/* clang-format on */ +#endif // CONFIG_VAR_TX + + if (rd_stats_y->rate == INT_MAX) { + av1_invalid_rd_stats(rd_stats); +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + if (mbmi->motion_mode != SIMPLE_TRANSLATION) { + continue; + } else { +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + restore_dst_buf(xd, *orig_dst); + return INT64_MAX; +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + } +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + } + + av1_merge_rd_stats(rd_stats, rd_stats_y); + + rdcosty = RDCOST(x->rdmult, x->rddiv, rd_stats->rate, rd_stats->dist); + rdcosty = AOMMIN(rdcosty, RDCOST(x->rdmult, x->rddiv, 0, rd_stats->sse)); +/* clang-format off */ +#if CONFIG_VAR_TX + is_cost_valid_uv = + inter_block_uvrd(cpi, x, rd_stats_uv, bsize, ref_best_rd - rdcosty); +#else + is_cost_valid_uv = + super_block_uvrd(cpi, x, rd_stats_uv, bsize, ref_best_rd - rdcosty); +#endif // CONFIG_VAR_TX + if (!is_cost_valid_uv) { +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + continue; +#else + restore_dst_buf(xd, *orig_dst); + return INT64_MAX; +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + } + /* clang-format on */ + av1_merge_rd_stats(rd_stats, rd_stats_uv); +#if CONFIG_RD_DEBUG + // record transform block coefficient cost + // TODO(angiebird): So far rd_debug tool only detects discrepancy of + // coefficient cost. Therefore, it is fine to copy rd_stats into mbmi + // here because we already collect the coefficient cost. Move this part to + // other place when we need to compare non-coefficient cost. + mbmi->rd_stats = *rd_stats; +#endif // CONFIG_RD_DEBUG +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + if (rd_stats->skip) { + rd_stats->rate -= rd_stats_uv->rate + rd_stats_y->rate; + rd_stats_y->rate = 0; + rd_stats_uv->rate = 0; + rd_stats->rate += av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + mbmi->skip = 0; + // here mbmi->skip temporarily plays a role as what this_skip2 does + } else if (!xd->lossless[mbmi->segment_id] && + (RDCOST(x->rdmult, x->rddiv, + rd_stats_y->rate + rd_stats_uv->rate + + av1_cost_bit(av1_get_skip_prob(cm, xd), 0), + rd_stats->dist) >= + RDCOST(x->rdmult, x->rddiv, + av1_cost_bit(av1_get_skip_prob(cm, xd), 1), + rd_stats->sse))) { + rd_stats->rate -= rd_stats_uv->rate + rd_stats_y->rate; + rd_stats->rate += av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + rd_stats->dist = rd_stats->sse; + rd_stats_y->rate = 0; + rd_stats_uv->rate = 0; + mbmi->skip = 1; + } else { + rd_stats->rate += av1_cost_bit(av1_get_skip_prob(cm, xd), 0); + mbmi->skip = 0; + } + *disable_skip = 0; +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + } else { + x->skip = 1; + *disable_skip = 1; + mbmi->tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 1); + +// The cost of skip bit needs to be added. +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + mbmi->skip = 0; +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + rd_stats->rate += av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + + rd_stats->dist = *skip_sse_sb; + rd_stats->sse = *skip_sse_sb; + rd_stats_y->rate = 0; + rd_stats_uv->rate = 0; + rd_stats->skip = 1; + } + +#if CONFIG_GLOBAL_MOTION + if (this_mode == ZEROMV +#if CONFIG_EXT_INTER + || this_mode == ZERO_ZEROMV +#endif // CONFIG_EXT_INTER + ) { + if (is_nontrans_global_motion(xd)) { + rd_stats->rate -= rs; +#if CONFIG_DUAL_FILTER + mbmi->interp_filter[0] = cm->interp_filter == SWITCHABLE + ? EIGHTTAP_REGULAR + : cm->interp_filter; + mbmi->interp_filter[1] = cm->interp_filter == SWITCHABLE + ? EIGHTTAP_REGULAR + : cm->interp_filter; +#else + mbmi->interp_filter = cm->interp_filter == SWITCHABLE + ? EIGHTTAP_REGULAR + : cm->interp_filter; +#endif // CONFIG_DUAL_FILTER + } + } +#endif // CONFIG_GLOBAL_MOTION + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + tmp_rd = RDCOST(x->rdmult, x->rddiv, rd_stats->rate, rd_stats->dist); + if (mbmi->motion_mode == SIMPLE_TRANSLATION || (tmp_rd < best_rd)) { + best_mbmi = *mbmi; + best_rd = tmp_rd; + best_rd_stats = *rd_stats; + best_rd_stats_y = *rd_stats_y; + best_rd_stats_uv = *rd_stats_uv; +#if CONFIG_VAR_TX + for (int i = 0; i < MAX_MB_PLANE; ++i) + memcpy(best_blk_skip[i], x->blk_skip[i], + sizeof(uint8_t) * xd->n8_h * xd->n8_w * 4); +#endif // CONFIG_VAR_TX + best_xskip = x->skip; + best_disable_skip = *disable_skip; + } + } + + if (best_rd == INT64_MAX) { + av1_invalid_rd_stats(rd_stats); + restore_dst_buf(xd, *orig_dst); + return INT64_MAX; + } + *mbmi = best_mbmi; + *rd_stats = best_rd_stats; + *rd_stats_y = best_rd_stats_y; + *rd_stats_uv = best_rd_stats_uv; +#if CONFIG_VAR_TX + for (int i = 0; i < MAX_MB_PLANE; ++i) + memcpy(x->blk_skip[i], best_blk_skip[i], + sizeof(uint8_t) * xd->n8_h * xd->n8_w * 4); +#endif // CONFIG_VAR_TX + x->skip = best_xskip; + *disable_skip = best_disable_skip; +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + + restore_dst_buf(xd, *orig_dst); + return 0; +} + +static int64_t handle_inter_mode( + const AV1_COMP *const cpi, MACROBLOCK *x, BLOCK_SIZE bsize, + RD_STATS *rd_stats, RD_STATS *rd_stats_y, RD_STATS *rd_stats_uv, + int *disable_skip, int_mv (*mode_mv)[TOTAL_REFS_PER_FRAME], int mi_row, + int mi_col, HandleInterModeArgs *args, const int64_t ref_best_rd) { + const AV1_COMMON *cm = &cpi->common; + (void)cm; + MACROBLOCKD *xd = &x->e_mbd; + MODE_INFO *mi = xd->mi[0]; + MB_MODE_INFO *mbmi = &mi->mbmi; + MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; + const int is_comp_pred = has_second_ref(mbmi); + const int this_mode = mbmi->mode; + int_mv *frame_mv = mode_mv[this_mode]; + int i; + int refs[2] = { mbmi->ref_frame[0], + (mbmi->ref_frame[1] < 0 ? 0 : mbmi->ref_frame[1]) }; + int_mv cur_mv[2]; + int rate_mv = 0; +#if CONFIG_EXT_INTER + int pred_exists = 1; + const int bw = block_size_wide[bsize]; + int_mv single_newmv[TOTAL_REFS_PER_FRAME]; +#if CONFIG_INTERINTRA + const unsigned int *const interintra_mode_cost = + cpi->interintra_mode_cost[size_group_lookup[bsize]]; +#endif // CONFIG_INTERINTRA + const int is_comp_interintra_pred = (mbmi->ref_frame[1] == INTRA_FRAME); +#if CONFIG_REF_MV + uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); +#endif // CONFIG_REF_MV +#else + int_mv *const single_newmv = args->single_newmv; +#endif // CONFIG_EXT_INTER +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint8_t, tmp_buf_[2 * MAX_MB_PLANE * MAX_SB_SQUARE]); +#else + DECLARE_ALIGNED(16, uint8_t, tmp_buf_[MAX_MB_PLANE * MAX_SB_SQUARE]); +#endif // CONFIG_HIGHBITDEPTH + uint8_t *tmp_buf; + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +#if CONFIG_EXT_INTER + int rate2_bmc_nocoeff; + MB_MODE_INFO best_bmc_mbmi; +#if CONFIG_MOTION_VAR + int rate_mv_bmc; +#endif // CONFIG_MOTION_VAR +#endif // CONFIG_EXT_INTER +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + int64_t rd = INT64_MAX; + BUFFER_SET orig_dst, tmp_dst; + int rs = 0; + + int skip_txfm_sb = 0; + int64_t skip_sse_sb = INT64_MAX; + int16_t mode_ctx; + +#if CONFIG_EXT_INTER + *args->compmode_interintra_cost = 0; + mbmi->use_wedge_interintra = 0; + *args->compmode_interinter_cost = 0; + mbmi->interinter_compound_type = COMPOUND_AVERAGE; + + // is_comp_interintra_pred implies !is_comp_pred + assert(!is_comp_interintra_pred || (!is_comp_pred)); + // is_comp_interintra_pred implies is_interintra_allowed(mbmi->sb_type) + assert(!is_comp_interintra_pred || is_interintra_allowed(mbmi)); +#endif // CONFIG_EXT_INTER + +#if CONFIG_REF_MV +#if CONFIG_EXT_INTER + if (is_comp_pred) + mode_ctx = mbmi_ext->compound_mode_context[refs[0]]; + else +#endif // CONFIG_EXT_INTER + mode_ctx = av1_mode_context_analyzer(mbmi_ext->mode_context, + mbmi->ref_frame, bsize, -1); +#else // CONFIG_REF_MV + mode_ctx = mbmi_ext->mode_context[refs[0]]; +#endif // CONFIG_REF_MV + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + tmp_buf = CONVERT_TO_BYTEPTR(tmp_buf_); + else +#endif // CONFIG_HIGHBITDEPTH + tmp_buf = tmp_buf_; + // Make sure that we didn't leave the plane destination buffers set + // to tmp_buf at the end of the last iteration + assert(xd->plane[0].dst.buf != tmp_buf); + +#if CONFIG_WARPED_MOTION + mbmi->num_proj_ref[0] = 0; + mbmi->num_proj_ref[1] = 0; +#endif // CONFIG_WARPED_MOTION + + if (is_comp_pred) { + if (frame_mv[refs[0]].as_int == INVALID_MV || + frame_mv[refs[1]].as_int == INVALID_MV) + return INT64_MAX; + } + + mbmi->motion_mode = SIMPLE_TRANSLATION; + if (have_newmv_in_inter_mode(this_mode)) { + const int64_t ret_val = handle_newmv(cpi, x, bsize, mode_mv, mi_row, mi_col, + &rate_mv, single_newmv, args); + if (ret_val != 0) + return ret_val; + else + rd_stats->rate += rate_mv; + } + for (i = 0; i < is_comp_pred + 1; ++i) { + cur_mv[i] = frame_mv[refs[i]]; + // Clip "next_nearest" so that it does not extend to far out of image + if (this_mode != NEWMV) clamp_mv2(&cur_mv[i].as_mv, xd); + if (mv_check_bounds(&x->mv_limits, &cur_mv[i].as_mv)) return INT64_MAX; + mbmi->mv[i].as_int = cur_mv[i].as_int; + } + +#if CONFIG_REF_MV +#if CONFIG_EXT_INTER + if (this_mode == NEAREST_NEARESTMV) +#else + if (this_mode == NEARESTMV && is_comp_pred) +#endif // CONFIG_EXT_INTER + { +#if !CONFIG_EXT_INTER + uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); +#endif // !CONFIG_EXT_INTER + if (mbmi_ext->ref_mv_count[ref_frame_type] > 0) { + cur_mv[0] = mbmi_ext->ref_mv_stack[ref_frame_type][0].this_mv; + cur_mv[1] = mbmi_ext->ref_mv_stack[ref_frame_type][0].comp_mv; + + for (i = 0; i < 2; ++i) { + clamp_mv2(&cur_mv[i].as_mv, xd); + if (mv_check_bounds(&x->mv_limits, &cur_mv[i].as_mv)) return INT64_MAX; + mbmi->mv[i].as_int = cur_mv[i].as_int; + } + } + } + +#if CONFIG_EXT_INTER + if (mbmi_ext->ref_mv_count[ref_frame_type] > 0) { + if (this_mode == NEAREST_NEWMV || this_mode == NEAREST_NEARMV) { + cur_mv[0] = mbmi_ext->ref_mv_stack[ref_frame_type][0].this_mv; + + lower_mv_precision(&cur_mv[0].as_mv, cm->allow_high_precision_mv); + clamp_mv2(&cur_mv[0].as_mv, xd); + if (mv_check_bounds(&x->mv_limits, &cur_mv[0].as_mv)) return INT64_MAX; + mbmi->mv[0].as_int = cur_mv[0].as_int; + } + + if (this_mode == NEW_NEARESTMV || this_mode == NEAR_NEARESTMV) { + cur_mv[1] = mbmi_ext->ref_mv_stack[ref_frame_type][0].comp_mv; + + lower_mv_precision(&cur_mv[1].as_mv, cm->allow_high_precision_mv); + clamp_mv2(&cur_mv[1].as_mv, xd); + if (mv_check_bounds(&x->mv_limits, &cur_mv[1].as_mv)) return INT64_MAX; + mbmi->mv[1].as_int = cur_mv[1].as_int; + } + } + + if (mbmi_ext->ref_mv_count[ref_frame_type] > 1) { + int ref_mv_idx = mbmi->ref_mv_idx + 1; + if (this_mode == NEAR_NEWMV || this_mode == NEAR_NEARESTMV || + this_mode == NEAR_NEARMV) { + cur_mv[0] = mbmi_ext->ref_mv_stack[ref_frame_type][ref_mv_idx].this_mv; + + lower_mv_precision(&cur_mv[0].as_mv, cm->allow_high_precision_mv); + clamp_mv2(&cur_mv[0].as_mv, xd); + if (mv_check_bounds(&x->mv_limits, &cur_mv[0].as_mv)) return INT64_MAX; + mbmi->mv[0].as_int = cur_mv[0].as_int; + } + + if (this_mode == NEW_NEARMV || this_mode == NEAREST_NEARMV || + this_mode == NEAR_NEARMV) { + cur_mv[1] = mbmi_ext->ref_mv_stack[ref_frame_type][ref_mv_idx].comp_mv; + + lower_mv_precision(&cur_mv[1].as_mv, cm->allow_high_precision_mv); + clamp_mv2(&cur_mv[1].as_mv, xd); + if (mv_check_bounds(&x->mv_limits, &cur_mv[1].as_mv)) return INT64_MAX; + mbmi->mv[1].as_int = cur_mv[1].as_int; + } + } +#else + if (this_mode == NEARMV && is_comp_pred) { + uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + if (mbmi_ext->ref_mv_count[ref_frame_type] > 1) { + int ref_mv_idx = mbmi->ref_mv_idx + 1; + cur_mv[0] = mbmi_ext->ref_mv_stack[ref_frame_type][ref_mv_idx].this_mv; + cur_mv[1] = mbmi_ext->ref_mv_stack[ref_frame_type][ref_mv_idx].comp_mv; + + for (i = 0; i < 2; ++i) { + clamp_mv2(&cur_mv[i].as_mv, xd); + if (mv_check_bounds(&x->mv_limits, &cur_mv[i].as_mv)) return INT64_MAX; + mbmi->mv[i].as_int = cur_mv[i].as_int; + } + } + } +#endif // CONFIG_EXT_INTER +#endif // CONFIG_REF_MV + + // do first prediction into the destination buffer. Do the next + // prediction into a temporary buffer. Then keep track of which one + // of these currently holds the best predictor, and use the other + // one for future predictions. In the end, copy from tmp_buf to + // dst if necessary. + for (i = 0; i < MAX_MB_PLANE; i++) { + tmp_dst.plane[i] = tmp_buf + i * MAX_SB_SQUARE; + tmp_dst.stride[i] = MAX_SB_SIZE; + } + for (i = 0; i < MAX_MB_PLANE; i++) { + orig_dst.plane[i] = xd->plane[i].dst.buf; + orig_dst.stride[i] = xd->plane[i].dst.stride; + } + + // We don't include the cost of the second reference here, because there + // are only three options: Last/Golden, ARF/Last or Golden/ARF, or in other + // words if you present them in that order, the second one is always known + // if the first is known. + // + // Under some circumstances we discount the cost of new mv mode to encourage + // initiation of a motion field. + if (discount_newmv_test(cpi, this_mode, frame_mv[refs[0]], mode_mv, + refs[0])) { +#if CONFIG_EXT_INTER + rd_stats->rate += + AOMMIN(cost_mv_ref(cpi, this_mode, mode_ctx), + cost_mv_ref(cpi, is_comp_pred ? NEAREST_NEARESTMV : NEARESTMV, + mode_ctx)); +#else + rd_stats->rate += AOMMIN(cost_mv_ref(cpi, this_mode, mode_ctx), + cost_mv_ref(cpi, NEARESTMV, mode_ctx)); +#endif // CONFIG_REF_MV && CONFIG_EXT_INTER + } else { + rd_stats->rate += cost_mv_ref(cpi, this_mode, mode_ctx); + } + + if (RDCOST(x->rdmult, x->rddiv, rd_stats->rate, 0) > ref_best_rd && +#if CONFIG_EXT_INTER + mbmi->mode != NEARESTMV && mbmi->mode != NEAREST_NEARESTMV +#else + mbmi->mode != NEARESTMV +#endif // CONFIG_EXT_INTER + ) + return INT64_MAX; + + int64_t ret_val = interpolation_filter_search( + x, cpi, bsize, mi_row, mi_col, &tmp_dst, &orig_dst, args->single_filter, + &rd, &rs, &skip_txfm_sb, &skip_sse_sb); + if (ret_val != 0) return ret_val; + +#if CONFIG_EXT_INTER +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + best_bmc_mbmi = *mbmi; + rate2_bmc_nocoeff = rd_stats->rate; + if (cm->interp_filter == SWITCHABLE) rate2_bmc_nocoeff += rs; +#if CONFIG_MOTION_VAR + rate_mv_bmc = rate_mv; +#endif // CONFIG_MOTION_VAR +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + + if (is_comp_pred) { + int rate_sum, rs2; + int64_t dist_sum; + int64_t best_rd_compound = INT64_MAX, best_rd_cur = INT64_MAX; + INTERINTER_COMPOUND_DATA best_compound_data; + int_mv best_mv[2]; + int best_tmp_rate_mv = rate_mv; + int tmp_skip_txfm_sb; + int64_t tmp_skip_sse_sb; + int compound_type_cost[COMPOUND_TYPES]; + uint8_t pred0[2 * MAX_SB_SQUARE]; + uint8_t pred1[2 * MAX_SB_SQUARE]; + uint8_t *preds0[1] = { pred0 }; + uint8_t *preds1[1] = { pred1 }; + int strides[1] = { bw }; + int tmp_rate_mv; + int masked_compound_used = is_any_masked_compound_used(bsize); + COMPOUND_TYPE cur_type; + + best_mv[0].as_int = cur_mv[0].as_int; + best_mv[1].as_int = cur_mv[1].as_int; + memset(&best_compound_data, 0, sizeof(best_compound_data)); +#if CONFIG_COMPOUND_SEGMENT + uint8_t tmp_mask_buf[2 * MAX_SB_SQUARE]; + best_compound_data.seg_mask = tmp_mask_buf; +#endif // CONFIG_COMPOUND_SEGMENT + av1_cost_tokens(compound_type_cost, cm->fc->compound_type_prob[bsize], + av1_compound_type_tree); + + if (masked_compound_used) { + av1_cost_tokens(compound_type_cost, cm->fc->compound_type_prob[bsize], + av1_compound_type_tree); + // get inter predictors to use for masked compound modes + av1_build_inter_predictors_for_planes_single_buf( + xd, bsize, 0, 0, mi_row, mi_col, 0, preds0, strides); + av1_build_inter_predictors_for_planes_single_buf( + xd, bsize, 0, 0, mi_row, mi_col, 1, preds1, strides); + } + + for (cur_type = COMPOUND_AVERAGE; cur_type < COMPOUND_TYPES; cur_type++) { + if (!is_interinter_compound_used(cur_type, bsize)) break; + tmp_rate_mv = rate_mv; + best_rd_cur = INT64_MAX; + mbmi->interinter_compound_type = cur_type; + rs2 = av1_cost_literal(get_interinter_compound_type_bits( + bsize, mbmi->interinter_compound_type)) + + (masked_compound_used + ? compound_type_cost[mbmi->interinter_compound_type] + : 0); + + switch (cur_type) { + case COMPOUND_AVERAGE: + av1_build_inter_predictors_sby(xd, mi_row, mi_col, &orig_dst, bsize); + av1_subtract_plane(x, bsize, 0); + rd = estimate_yrd_for_sb(cpi, bsize, x, &rate_sum, &dist_sum, + &tmp_skip_txfm_sb, &tmp_skip_sse_sb, + INT64_MAX); + if (rd != INT64_MAX) + best_rd_cur = + RDCOST(x->rdmult, x->rddiv, rs2 + rate_mv + rate_sum, dist_sum); + best_rd_compound = best_rd_cur; + break; +#if CONFIG_WEDGE + case COMPOUND_WEDGE: + if (x->source_variance > cpi->sf.disable_wedge_search_var_thresh && + best_rd_compound / 3 < ref_best_rd) { + best_rd_cur = build_and_cost_compound_type( + cpi, x, cur_mv, bsize, this_mode, rs2, rate_mv, &orig_dst, + &tmp_rate_mv, preds0, preds1, strides, mi_row, mi_col); + } + break; +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + case COMPOUND_SEG: + if (x->source_variance > cpi->sf.disable_wedge_search_var_thresh && + best_rd_compound / 3 < ref_best_rd) { + best_rd_cur = build_and_cost_compound_type( + cpi, x, cur_mv, bsize, this_mode, rs2, rate_mv, &orig_dst, + &tmp_rate_mv, preds0, preds1, strides, mi_row, mi_col); + } + break; +#endif // CONFIG_COMPOUND_SEGMENT + default: assert(0); return 0; + } + + if (best_rd_cur < best_rd_compound) { + best_rd_compound = best_rd_cur; +#if CONFIG_WEDGE + best_compound_data.wedge_index = mbmi->wedge_index; + best_compound_data.wedge_sign = mbmi->wedge_sign; +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + best_compound_data.mask_type = mbmi->mask_type; + memcpy(best_compound_data.seg_mask, xd->seg_mask, + 2 * MAX_SB_SQUARE * sizeof(uint8_t)); +#endif // CONFIG_COMPOUND_SEGMENT + best_compound_data.interinter_compound_type = + mbmi->interinter_compound_type; + if (have_newmv_in_inter_mode(this_mode)) { + if (use_masked_motion_search(cur_type)) { + best_tmp_rate_mv = tmp_rate_mv; + best_mv[0].as_int = mbmi->mv[0].as_int; + best_mv[1].as_int = mbmi->mv[1].as_int; + } else { + best_mv[0].as_int = cur_mv[0].as_int; + best_mv[1].as_int = cur_mv[1].as_int; + } + } + } + // reset to original mvs for next iteration + mbmi->mv[0].as_int = cur_mv[0].as_int; + mbmi->mv[1].as_int = cur_mv[1].as_int; + } +#if CONFIG_WEDGE + mbmi->wedge_index = best_compound_data.wedge_index; + mbmi->wedge_sign = best_compound_data.wedge_sign; +#endif // CONFIG_WEDGE +#if CONFIG_COMPOUND_SEGMENT + mbmi->mask_type = best_compound_data.mask_type; + memcpy(xd->seg_mask, best_compound_data.seg_mask, + 2 * MAX_SB_SQUARE * sizeof(uint8_t)); +#endif // CONFIG_COMPOUND_SEGMENT + mbmi->interinter_compound_type = + best_compound_data.interinter_compound_type; + if (have_newmv_in_inter_mode(this_mode)) { + mbmi->mv[0].as_int = best_mv[0].as_int; + mbmi->mv[1].as_int = best_mv[1].as_int; + xd->mi[0]->bmi[0].as_mv[0].as_int = mbmi->mv[0].as_int; + xd->mi[0]->bmi[0].as_mv[1].as_int = mbmi->mv[1].as_int; + if (use_masked_motion_search(mbmi->interinter_compound_type)) { + rd_stats->rate += best_tmp_rate_mv - rate_mv; + rate_mv = best_tmp_rate_mv; + } + } + + if (ref_best_rd < INT64_MAX && best_rd_compound / 3 > ref_best_rd) { + restore_dst_buf(xd, orig_dst); + return INT64_MAX; + } + + pred_exists = 0; + + *args->compmode_interinter_cost = + av1_cost_literal(get_interinter_compound_type_bits( + bsize, mbmi->interinter_compound_type)) + + (masked_compound_used + ? compound_type_cost[mbmi->interinter_compound_type] + : 0); + } + +#if CONFIG_INTERINTRA + if (is_comp_interintra_pred) { + INTERINTRA_MODE best_interintra_mode = II_DC_PRED; + int64_t best_interintra_rd = INT64_MAX; + int rmode, rate_sum; + int64_t dist_sum; + int j; + int tmp_rate_mv = 0; + int tmp_skip_txfm_sb; + int64_t tmp_skip_sse_sb; + DECLARE_ALIGNED(16, uint8_t, intrapred_[2 * MAX_SB_SQUARE]); + uint8_t *intrapred; + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + intrapred = CONVERT_TO_BYTEPTR(intrapred_); + else +#endif // CONFIG_HIGHBITDEPTH + intrapred = intrapred_; + + mbmi->ref_frame[1] = NONE_FRAME; + for (j = 0; j < MAX_MB_PLANE; j++) { + xd->plane[j].dst.buf = tmp_buf + j * MAX_SB_SQUARE; + xd->plane[j].dst.stride = bw; + } + av1_build_inter_predictors_sby(xd, mi_row, mi_col, &orig_dst, bsize); + restore_dst_buf(xd, orig_dst); + mbmi->ref_frame[1] = INTRA_FRAME; + mbmi->use_wedge_interintra = 0; + + for (j = 0; j < INTERINTRA_MODES; ++j) { + mbmi->interintra_mode = (INTERINTRA_MODE)j; + rmode = interintra_mode_cost[mbmi->interintra_mode]; + av1_build_intra_predictors_for_interintra(xd, bsize, 0, &orig_dst, + intrapred, bw); + av1_combine_interintra(xd, bsize, 0, tmp_buf, bw, intrapred, bw); + model_rd_for_sb(cpi, bsize, x, xd, 0, 0, &rate_sum, &dist_sum, + &tmp_skip_txfm_sb, &tmp_skip_sse_sb); + rd = RDCOST(x->rdmult, x->rddiv, rs + tmp_rate_mv + rate_sum, dist_sum); + if (rd < best_interintra_rd) { + best_interintra_rd = rd; + best_interintra_mode = mbmi->interintra_mode; + } + } + mbmi->interintra_mode = best_interintra_mode; + rmode = interintra_mode_cost[mbmi->interintra_mode]; + av1_build_intra_predictors_for_interintra(xd, bsize, 0, &orig_dst, + intrapred, bw); + av1_combine_interintra(xd, bsize, 0, tmp_buf, bw, intrapred, bw); + av1_subtract_plane(x, bsize, 0); + rd = estimate_yrd_for_sb(cpi, bsize, x, &rate_sum, &dist_sum, + &tmp_skip_txfm_sb, &tmp_skip_sse_sb, INT64_MAX); + if (rd != INT64_MAX) + rd = RDCOST(x->rdmult, x->rddiv, rate_mv + rmode + rate_sum, dist_sum); + best_interintra_rd = rd; + + if (ref_best_rd < INT64_MAX && best_interintra_rd > 2 * ref_best_rd) { + // Don't need to call restore_dst_buf here + return INT64_MAX; + } +#if CONFIG_WEDGE + if (is_interintra_wedge_used(bsize)) { + int64_t best_interintra_rd_nowedge = INT64_MAX; + int64_t best_interintra_rd_wedge = INT64_MAX; + int_mv tmp_mv; + int rwedge = av1_cost_bit(cm->fc->wedge_interintra_prob[bsize], 0); + if (rd != INT64_MAX) + rd = RDCOST(x->rdmult, x->rddiv, rmode + rate_mv + rwedge + rate_sum, + dist_sum); + best_interintra_rd_nowedge = rd; + + // Disable wedge search if source variance is small + if (x->source_variance > cpi->sf.disable_wedge_search_var_thresh) { + mbmi->use_wedge_interintra = 1; + + rwedge = av1_cost_literal(get_interintra_wedge_bits(bsize)) + + av1_cost_bit(cm->fc->wedge_interintra_prob[bsize], 1); + + best_interintra_rd_wedge = + pick_interintra_wedge(cpi, x, bsize, intrapred_, tmp_buf_); + + best_interintra_rd_wedge += + RDCOST(x->rdmult, x->rddiv, rmode + rate_mv + rwedge, 0); + // Refine motion vector. + if (have_newmv_in_inter_mode(this_mode)) { + // get negative of mask + const uint8_t *mask = av1_get_contiguous_soft_mask( + mbmi->interintra_wedge_index, 1, bsize); + do_masked_motion_search(cpi, x, mask, bw, bsize, mi_row, mi_col, + &tmp_mv, &tmp_rate_mv, 0); + mbmi->mv[0].as_int = tmp_mv.as_int; + av1_build_inter_predictors_sby(xd, mi_row, mi_col, &orig_dst, bsize); + model_rd_for_sb(cpi, bsize, x, xd, 0, 0, &rate_sum, &dist_sum, + &tmp_skip_txfm_sb, &tmp_skip_sse_sb); + rd = RDCOST(x->rdmult, x->rddiv, + rmode + tmp_rate_mv + rwedge + rate_sum, dist_sum); + if (rd < best_interintra_rd_wedge) { + best_interintra_rd_wedge = rd; + } else { + tmp_mv.as_int = cur_mv[0].as_int; + tmp_rate_mv = rate_mv; + } + } else { + tmp_mv.as_int = cur_mv[0].as_int; + tmp_rate_mv = rate_mv; + av1_combine_interintra(xd, bsize, 0, tmp_buf, bw, intrapred, bw); + } + // Evaluate closer to true rd + av1_subtract_plane(x, bsize, 0); + rd = + estimate_yrd_for_sb(cpi, bsize, x, &rate_sum, &dist_sum, + &tmp_skip_txfm_sb, &tmp_skip_sse_sb, INT64_MAX); + if (rd != INT64_MAX) + rd = RDCOST(x->rdmult, x->rddiv, + rmode + tmp_rate_mv + rwedge + rate_sum, dist_sum); + best_interintra_rd_wedge = rd; + if (best_interintra_rd_wedge < best_interintra_rd_nowedge) { + mbmi->use_wedge_interintra = 1; + best_interintra_rd = best_interintra_rd_wedge; + mbmi->mv[0].as_int = tmp_mv.as_int; + rd_stats->rate += tmp_rate_mv - rate_mv; + rate_mv = tmp_rate_mv; + } else { + mbmi->use_wedge_interintra = 0; + best_interintra_rd = best_interintra_rd_nowedge; + mbmi->mv[0].as_int = cur_mv[0].as_int; + } + } else { + mbmi->use_wedge_interintra = 0; + best_interintra_rd = best_interintra_rd_nowedge; + } + } +#endif // CONFIG_WEDGE + + pred_exists = 0; + *args->compmode_interintra_cost = + av1_cost_bit(cm->fc->interintra_prob[size_group_lookup[bsize]], 1); + *args->compmode_interintra_cost += + interintra_mode_cost[mbmi->interintra_mode]; + if (is_interintra_wedge_used(bsize)) { + *args->compmode_interintra_cost += av1_cost_bit( + cm->fc->wedge_interintra_prob[bsize], mbmi->use_wedge_interintra); + if (mbmi->use_wedge_interintra) { + *args->compmode_interintra_cost += + av1_cost_literal(get_interintra_wedge_bits(bsize)); + } + } + } else if (is_interintra_allowed(mbmi)) { + *args->compmode_interintra_cost = + av1_cost_bit(cm->fc->interintra_prob[size_group_lookup[bsize]], 0); + } +#endif // CONFIG_INTERINTRA + + if (pred_exists == 0) { + int tmp_rate; + int64_t tmp_dist; + av1_build_inter_predictors_sb(xd, mi_row, mi_col, &orig_dst, bsize); + model_rd_for_sb(cpi, bsize, x, xd, 0, MAX_MB_PLANE - 1, &tmp_rate, + &tmp_dist, &skip_txfm_sb, &skip_sse_sb); + rd = RDCOST(x->rdmult, x->rddiv, rs + tmp_rate, tmp_dist); + } +#endif // CONFIG_EXT_INTER + + if (!is_comp_pred) +#if CONFIG_DUAL_FILTER + args->single_filter[this_mode][refs[0]] = mbmi->interp_filter[0]; +#else + args->single_filter[this_mode][refs[0]] = mbmi->interp_filter; +#endif // CONFIG_DUAL_FILTER + +#if CONFIG_EXT_INTER + if (args->modelled_rd != NULL) { + if (is_comp_pred) { + const int mode0 = compound_ref0_mode(this_mode); + const int mode1 = compound_ref1_mode(this_mode); + const int64_t mrd = AOMMIN(args->modelled_rd[mode0][refs[0]], + args->modelled_rd[mode1][refs[1]]); + if (rd / 4 * 3 > mrd && ref_best_rd < INT64_MAX) { + restore_dst_buf(xd, orig_dst); + return INT64_MAX; + } + } else if (!is_comp_interintra_pred) { + args->modelled_rd[this_mode][refs[0]] = rd; + } + } +#endif // CONFIG_EXT_INTER + + if (cpi->sf.use_rd_breakout && ref_best_rd < INT64_MAX) { + // if current pred_error modeled rd is substantially more than the best + // so far, do not bother doing full rd + if (rd / 2 > ref_best_rd) { + restore_dst_buf(xd, orig_dst); + return INT64_MAX; + } + } + + ret_val = motion_mode_rd(cpi, x, bsize, rd_stats, rd_stats_y, rd_stats_uv, + disable_skip, mode_mv, mi_row, mi_col, args, + ref_best_rd, refs, rate_mv, +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +#if CONFIG_EXT_INTER + rate2_bmc_nocoeff, &best_bmc_mbmi, +#if CONFIG_MOTION_VAR + rate_mv_bmc, +#endif // CONFIG_MOTION_VAR +#endif // CONFIG_EXT_INTER +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + rs, &skip_txfm_sb, &skip_sse_sb, &orig_dst); + if (ret_val != 0) return ret_val; + + return 0; // The rate-distortion cost will be re-calculated by caller. +} + +#if CONFIG_INTRABC +static int64_t rd_pick_intrabc_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x, + RD_STATS *rd_cost, BLOCK_SIZE bsize, + int64_t best_rd) { + const AV1_COMMON *const cm = &cpi->common; + if (bsize < BLOCK_8X8 || !cm->allow_screen_content_tools) return INT64_MAX; + + MACROBLOCKD *const xd = &x->e_mbd; + const TileInfo *tile = &xd->tile; + MODE_INFO *const mi = xd->mi[0]; + const int mi_row = -xd->mb_to_top_edge / (8 * MI_SIZE); + const int mi_col = -xd->mb_to_left_edge / (8 * MI_SIZE); + const int w = block_size_wide[bsize]; + const int h = block_size_high[bsize]; + const int sb_row = mi_row / MAX_MIB_SIZE; + + int_mv dv_ref; + av1_find_ref_dv(&dv_ref, mi_row, mi_col); + + const MvLimits tmp_mv_limits = x->mv_limits; + + // TODO(aconverse@google.com): Handle same row DV. + x->mv_limits.col_min = (tile->mi_col_start - mi_col) * MI_SIZE; + x->mv_limits.col_max = (tile->mi_col_end - mi_col) * MI_SIZE - w; + x->mv_limits.row_min = (tile->mi_row_start - mi_row) * MI_SIZE; + x->mv_limits.row_max = (sb_row * MAX_MIB_SIZE - mi_row) * MI_SIZE - h; + assert(x->mv_limits.col_min >= tmp_mv_limits.col_min); + assert(x->mv_limits.col_max <= tmp_mv_limits.col_max); + assert(x->mv_limits.row_min >= tmp_mv_limits.row_min); + assert(x->mv_limits.row_max <= tmp_mv_limits.row_max); + av1_set_mv_search_range(&x->mv_limits, &dv_ref.as_mv); + + if (x->mv_limits.col_max < x->mv_limits.col_min || + x->mv_limits.row_max < x->mv_limits.row_min) { + x->mv_limits = tmp_mv_limits; + return INT64_MAX; + } + + struct buf_2d yv12_mb[MAX_MB_PLANE]; + av1_setup_pred_block(xd, yv12_mb, xd->cur_buf, mi_row, mi_col, NULL, NULL); + for (int i = 0; i < MAX_MB_PLANE; ++i) { + xd->plane[i].pre[0] = yv12_mb[i]; + } + + int step_param = cpi->mv_step_param; + MV mvp_full = dv_ref.as_mv; + mvp_full.col >>= 3; + mvp_full.row >>= 3; + int sadpb = x->sadperbit16; + int cost_list[5]; + int bestsme = av1_full_pixel_search(cpi, x, bsize, &mvp_full, step_param, + sadpb, cond_cost_list(cpi, cost_list), + &dv_ref.as_mv, INT_MAX, 1); + + x->mv_limits = tmp_mv_limits; + if (bestsme == INT_MAX) return INT64_MAX; + mvp_full = x->best_mv.as_mv; + MV dv = {.row = mvp_full.row * 8, .col = mvp_full.col * 8 }; + if (mv_check_bounds(&x->mv_limits, &dv)) return INT64_MAX; + if (!is_dv_valid(dv, tile, mi_row, mi_col, bsize)) return INT64_MAX; + MB_MODE_INFO *mbmi = &mi->mbmi; + MB_MODE_INFO best_mbmi = *mbmi; + RD_STATS best_rdcost = *rd_cost; + int best_skip = x->skip; +#if CONFIG_PALETTE + memset(&mbmi->palette_mode_info, 0, sizeof(mbmi->palette_mode_info)); +#endif + mbmi->use_intrabc = 1; + mbmi->mode = DC_PRED; + mbmi->uv_mode = DC_PRED; + mbmi->mv[0].as_mv = dv; +#if CONFIG_DUAL_FILTER + for (int idx = 0; idx < 4; ++idx) mbmi->interp_filter[idx] = BILINEAR; +#else + mbmi->interp_filter = BILINEAR; +#endif + mbmi->skip = 0; + x->skip = 0; + av1_build_inter_predictors_sb(xd, mi_row, mi_col, NULL, bsize); + + int rate_mv = av1_mv_bit_cost(&dv, &dv_ref.as_mv, x->nmvjointcost, x->mvcost, + MV_COST_WEIGHT); + const PREDICTION_MODE A = av1_above_block_mode(mi, xd->above_mi, 0); + const PREDICTION_MODE L = av1_left_block_mode(mi, xd->left_mi, 0); + const int rate_mode = + cpi->y_mode_costs[A][L][DC_PRED] + av1_cost_bit(INTRABC_PROB, 1); + + RD_STATS rd_stats, rd_stats_uv; + av1_subtract_plane(x, bsize, 0); + super_block_yrd(cpi, x, &rd_stats, bsize, INT64_MAX); + super_block_uvrd(cpi, x, &rd_stats_uv, bsize, INT64_MAX); + av1_merge_rd_stats(&rd_stats, &rd_stats_uv); +#if CONFIG_RD_DEBUG + mbmi->rd_stats = rd_stats; +#endif + + const aom_prob skip_prob = av1_get_skip_prob(cm, xd); + + RD_STATS rdc_noskip; + av1_init_rd_stats(&rdc_noskip); + rdc_noskip.rate = + rate_mode + rate_mv + rd_stats.rate + av1_cost_bit(skip_prob, 0); + rdc_noskip.dist = rd_stats.dist; + rdc_noskip.rdcost = + RDCOST(x->rdmult, x->rddiv, rdc_noskip.rate, rdc_noskip.dist); + if (rdc_noskip.rdcost < best_rd) { + best_rd = rdc_noskip.rdcost; + best_mbmi = *mbmi; + best_skip = x->skip; + best_rdcost = rdc_noskip; + } + + x->skip = 1; + mbmi->skip = 1; + RD_STATS rdc_skip; + av1_init_rd_stats(&rdc_skip); + rdc_skip.rate = rate_mode + rate_mv + av1_cost_bit(skip_prob, 1); + rdc_skip.dist = rd_stats.sse; + rdc_skip.rdcost = RDCOST(x->rdmult, x->rddiv, rdc_skip.rate, rdc_skip.dist); + if (rdc_skip.rdcost < best_rd) { + best_rd = rdc_skip.rdcost; + best_mbmi = *mbmi; + best_skip = x->skip; + best_rdcost = rdc_skip; + } + *mbmi = best_mbmi; + *rd_cost = best_rdcost; + x->skip = best_skip; + return best_rd; +} +#endif // CONFIG_INTRABC + +void av1_rd_pick_intra_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x, + RD_STATS *rd_cost, BLOCK_SIZE bsize, + PICK_MODE_CONTEXT *ctx, int64_t best_rd) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + struct macroblockd_plane *const pd = xd->plane; + int rate_y = 0, rate_uv = 0, rate_y_tokenonly = 0, rate_uv_tokenonly = 0; + int y_skip = 0, uv_skip = 0; + int64_t dist_y = 0, dist_uv = 0; + TX_SIZE max_uv_tx_size; + const int unify_bsize = CONFIG_CB4X4; + + ctx->skip = 0; + xd->mi[0]->mbmi.ref_frame[0] = INTRA_FRAME; + xd->mi[0]->mbmi.ref_frame[1] = NONE_FRAME; +#if CONFIG_INTRABC + xd->mi[0]->mbmi.use_intrabc = 0; +#endif // CONFIG_INTRABC + + const int64_t intra_yrd = + (bsize >= BLOCK_8X8 || unify_bsize) + ? rd_pick_intra_sby_mode(cpi, x, &rate_y, &rate_y_tokenonly, &dist_y, + &y_skip, bsize, best_rd) + : rd_pick_intra_sub_8x8_y_mode(cpi, x, &rate_y, &rate_y_tokenonly, + &dist_y, &y_skip, best_rd); + + if (intra_yrd < best_rd) { + max_uv_tx_size = uv_txsize_lookup[bsize][xd->mi[0]->mbmi.tx_size] + [pd[1].subsampling_x][pd[1].subsampling_y]; + +#if CONFIG_CB4X4 +#if !CONFIG_CHROMA_2X2 + max_uv_tx_size = AOMMAX(max_uv_tx_size, TX_4X4); +#endif // !CONFIG_CHROMA_2X2 + if (!x->skip_chroma_rd) + rd_pick_intra_sbuv_mode(cpi, x, &rate_uv, &rate_uv_tokenonly, &dist_uv, + &uv_skip, bsize, max_uv_tx_size); +#else + rd_pick_intra_sbuv_mode(cpi, x, &rate_uv, &rate_uv_tokenonly, &dist_uv, + &uv_skip, AOMMAX(BLOCK_8X8, bsize), max_uv_tx_size); +#endif // CONFIG_CB4X4 + + if (y_skip && uv_skip) { + rd_cost->rate = rate_y + rate_uv - rate_y_tokenonly - rate_uv_tokenonly + + av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + rd_cost->dist = dist_y + dist_uv; + } else { + rd_cost->rate = + rate_y + rate_uv + av1_cost_bit(av1_get_skip_prob(cm, xd), 0); + rd_cost->dist = dist_y + dist_uv; + } + rd_cost->rdcost = RDCOST(x->rdmult, x->rddiv, rd_cost->rate, rd_cost->dist); + } else { + rd_cost->rate = INT_MAX; + } + +#if CONFIG_INTRABC + if (rd_cost->rate != INT_MAX && rd_cost->rdcost < best_rd) + best_rd = rd_cost->rdcost; + if (rd_pick_intrabc_mode_sb(cpi, x, rd_cost, bsize, best_rd) < best_rd) { + ctx->skip = x->skip; // FIXME where is the proper place to set this?! + assert(rd_cost->rate != INT_MAX); + rd_cost->rdcost = RDCOST(x->rdmult, x->rddiv, rd_cost->rate, rd_cost->dist); + } +#endif + if (rd_cost->rate == INT_MAX) return; + + ctx->mic = *xd->mi[0]; + ctx->mbmi_ext = *x->mbmi_ext; +} + +// Do we have an internal image edge (e.g. formatting bars). +int av1_internal_image_edge(const AV1_COMP *cpi) { + return (cpi->oxcf.pass == 2) && + ((cpi->twopass.this_frame_stats.inactive_zone_rows > 0) || + (cpi->twopass.this_frame_stats.inactive_zone_cols > 0)); +} + +// Checks to see if a super block is on a horizontal image edge. +// In most cases this is the "real" edge unless there are formatting +// bars embedded in the stream. +int av1_active_h_edge(const AV1_COMP *cpi, int mi_row, int mi_step) { + int top_edge = 0; + int bottom_edge = cpi->common.mi_rows; + int is_active_h_edge = 0; + + // For two pass account for any formatting bars detected. + if (cpi->oxcf.pass == 2) { + const TWO_PASS *const twopass = &cpi->twopass; + + // The inactive region is specified in MBs not mi units. + // The image edge is in the following MB row. + top_edge += (int)(twopass->this_frame_stats.inactive_zone_rows * 2); + + bottom_edge -= (int)(twopass->this_frame_stats.inactive_zone_rows * 2); + bottom_edge = AOMMAX(top_edge, bottom_edge); + } + + if (((top_edge >= mi_row) && (top_edge < (mi_row + mi_step))) || + ((bottom_edge >= mi_row) && (bottom_edge < (mi_row + mi_step)))) { + is_active_h_edge = 1; + } + return is_active_h_edge; +} + +// Checks to see if a super block is on a vertical image edge. +// In most cases this is the "real" edge unless there are formatting +// bars embedded in the stream. +int av1_active_v_edge(const AV1_COMP *cpi, int mi_col, int mi_step) { + int left_edge = 0; + int right_edge = cpi->common.mi_cols; + int is_active_v_edge = 0; + + // For two pass account for any formatting bars detected. + if (cpi->oxcf.pass == 2) { + const TWO_PASS *const twopass = &cpi->twopass; + + // The inactive region is specified in MBs not mi units. + // The image edge is in the following MB row. + left_edge += (int)(twopass->this_frame_stats.inactive_zone_cols * 2); + + right_edge -= (int)(twopass->this_frame_stats.inactive_zone_cols * 2); + right_edge = AOMMAX(left_edge, right_edge); + } + + if (((left_edge >= mi_col) && (left_edge < (mi_col + mi_step))) || + ((right_edge >= mi_col) && (right_edge < (mi_col + mi_step)))) { + is_active_v_edge = 1; + } + return is_active_v_edge; +} + +// Checks to see if a super block is at the edge of the active image. +// In most cases this is the "real" edge unless there are formatting +// bars embedded in the stream. +int av1_active_edge_sb(const AV1_COMP *cpi, int mi_row, int mi_col) { + return av1_active_h_edge(cpi, mi_row, cpi->common.mib_size) || + av1_active_v_edge(cpi, mi_col, cpi->common.mib_size); +} + +#if CONFIG_PALETTE +static void restore_uv_color_map(const AV1_COMP *const cpi, MACROBLOCK *x) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; + const BLOCK_SIZE bsize = mbmi->sb_type; + int src_stride = x->plane[1].src.stride; + const uint8_t *const src_u = x->plane[1].src.buf; + const uint8_t *const src_v = x->plane[2].src.buf; + float *const data = x->palette_buffer->kmeans_data_buf; + float centroids[2 * PALETTE_MAX_SIZE]; + uint8_t *const color_map = xd->plane[1].color_index_map; + int r, c; +#if CONFIG_HIGHBITDEPTH + const uint16_t *const src_u16 = CONVERT_TO_SHORTPTR(src_u); + const uint16_t *const src_v16 = CONVERT_TO_SHORTPTR(src_v); +#endif // CONFIG_HIGHBITDEPTH + int plane_block_width, plane_block_height, rows, cols; + av1_get_block_dimensions(bsize, 1, xd, &plane_block_width, + &plane_block_height, &rows, &cols); + (void)cpi; + + for (r = 0; r < rows; ++r) { + for (c = 0; c < cols; ++c) { +#if CONFIG_HIGHBITDEPTH + if (cpi->common.use_highbitdepth) { + data[(r * cols + c) * 2] = src_u16[r * src_stride + c]; + data[(r * cols + c) * 2 + 1] = src_v16[r * src_stride + c]; + } else { +#endif // CONFIG_HIGHBITDEPTH + data[(r * cols + c) * 2] = src_u[r * src_stride + c]; + data[(r * cols + c) * 2 + 1] = src_v[r * src_stride + c]; +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH + } + } + + for (r = 1; r < 3; ++r) { + for (c = 0; c < pmi->palette_size[1]; ++c) { + centroids[c * 2 + r - 1] = pmi->palette_colors[r * PALETTE_MAX_SIZE + c]; + } + } + + av1_calc_indices(data, centroids, color_map, rows * cols, + pmi->palette_size[1], 2); + extend_palette_color_map(color_map, cols, rows, plane_block_width, + plane_block_height); +} +#endif // CONFIG_PALETTE + +#if CONFIG_FILTER_INTRA +static void pick_filter_intra_interframe( + const AV1_COMP *cpi, MACROBLOCK *x, PICK_MODE_CONTEXT *ctx, + BLOCK_SIZE bsize, int mi_row, int mi_col, int *rate_uv_intra, + int *rate_uv_tokenonly, int64_t *dist_uv, int *skip_uv, + PREDICTION_MODE *mode_uv, FILTER_INTRA_MODE_INFO *filter_intra_mode_info_uv, +#if CONFIG_EXT_INTRA + int8_t *uv_angle_delta, +#endif // CONFIG_EXT_INTRA +#if CONFIG_PALETTE + PALETTE_MODE_INFO *pmi_uv, int palette_ctx, +#endif // CONFIG_PALETTE + int skip_mask, unsigned int *ref_costs_single, int64_t *best_rd, + int64_t *best_intra_rd, PREDICTION_MODE *best_intra_mode, + int *best_mode_index, int *best_skip2, int *best_mode_skippable, +#if CONFIG_SUPERTX + int *returnrate_nocoef, +#endif // CONFIG_SUPERTX + int64_t *best_pred_rd, MB_MODE_INFO *best_mbmode, RD_STATS *rd_cost) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; +#if CONFIG_PALETTE + PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; +#endif // CONFIG_PALETTE + int rate2 = 0, rate_y = INT_MAX, skippable = 0, rate_uv, rate_dummy, i; + int dc_mode_index; + const int *const intra_mode_cost = cpi->mbmode_cost[size_group_lookup[bsize]]; + int64_t distortion2 = 0, distortion_y = 0, this_rd = *best_rd; + int64_t distortion_uv, model_rd = INT64_MAX; + TX_SIZE uv_tx; + + for (i = 0; i < MAX_MODES; ++i) + if (av1_mode_order[i].mode == DC_PRED && + av1_mode_order[i].ref_frame[0] == INTRA_FRAME) + break; + dc_mode_index = i; + assert(i < MAX_MODES); + + // TODO(huisu): use skip_mask for further speedup. + (void)skip_mask; + mbmi->mode = DC_PRED; + mbmi->uv_mode = DC_PRED; + mbmi->ref_frame[0] = INTRA_FRAME; + mbmi->ref_frame[1] = NONE_FRAME; + if (!rd_pick_filter_intra_sby(cpi, x, &rate_dummy, &rate_y, &distortion_y, + &skippable, bsize, intra_mode_cost[mbmi->mode], + &this_rd, &model_rd, 0)) { + return; + } + if (rate_y == INT_MAX) return; + + uv_tx = uv_txsize_lookup[bsize][mbmi->tx_size][xd->plane[1].subsampling_x] + [xd->plane[1].subsampling_y]; + if (rate_uv_intra[uv_tx] == INT_MAX) { + choose_intra_uv_mode(cpi, x, ctx, bsize, uv_tx, &rate_uv_intra[uv_tx], + &rate_uv_tokenonly[uv_tx], &dist_uv[uv_tx], + &skip_uv[uv_tx], &mode_uv[uv_tx]); +#if CONFIG_PALETTE + if (cm->allow_screen_content_tools) pmi_uv[uv_tx] = *pmi; +#endif // CONFIG_PALETTE + filter_intra_mode_info_uv[uv_tx] = mbmi->filter_intra_mode_info; +#if CONFIG_EXT_INTRA + uv_angle_delta[uv_tx] = mbmi->angle_delta[1]; +#endif // CONFIG_EXT_INTRA + } + + rate_uv = rate_uv_tokenonly[uv_tx]; + distortion_uv = dist_uv[uv_tx]; + skippable = skippable && skip_uv[uv_tx]; + mbmi->uv_mode = mode_uv[uv_tx]; +#if CONFIG_PALETTE + if (cm->allow_screen_content_tools) { + pmi->palette_size[1] = pmi_uv[uv_tx].palette_size[1]; + memcpy(pmi->palette_colors + PALETTE_MAX_SIZE, + pmi_uv[uv_tx].palette_colors + PALETTE_MAX_SIZE, + 2 * PALETTE_MAX_SIZE * sizeof(pmi->palette_colors[0])); + } +#endif // CONFIG_PALETTE +#if CONFIG_EXT_INTRA + mbmi->angle_delta[1] = uv_angle_delta[uv_tx]; +#endif // CONFIG_EXT_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = + filter_intra_mode_info_uv[uv_tx].use_filter_intra_mode[1]; + if (filter_intra_mode_info_uv[uv_tx].use_filter_intra_mode[1]) { + mbmi->filter_intra_mode_info.filter_intra_mode[1] = + filter_intra_mode_info_uv[uv_tx].filter_intra_mode[1]; + } + + rate2 = rate_y + intra_mode_cost[mbmi->mode] + rate_uv + + cpi->intra_uv_mode_cost[mbmi->mode][mbmi->uv_mode]; +#if CONFIG_PALETTE + if (cpi->common.allow_screen_content_tools && mbmi->mode == DC_PRED && + bsize >= BLOCK_8X8) + rate2 += av1_cost_bit( + av1_default_palette_y_mode_prob[bsize - BLOCK_8X8][palette_ctx], 0); +#endif // CONFIG_PALETTE + + if (!xd->lossless[mbmi->segment_id]) { + // super_block_yrd above includes the cost of the tx_size in the + // tokenonly rate, but for intra blocks, tx_size is always coded + // (prediction granularity), so we account for it in the full rate, + // not the tokenonly rate. + rate_y -= tx_size_cost(cpi, x, bsize, mbmi->tx_size); + } + + rate2 += av1_cost_bit(cm->fc->filter_intra_probs[0], + mbmi->filter_intra_mode_info.use_filter_intra_mode[0]); + rate2 += write_uniform_cost( + FILTER_INTRA_MODES, mbmi->filter_intra_mode_info.filter_intra_mode[0]); +#if CONFIG_EXT_INTRA + if (av1_is_directional_mode(mbmi->uv_mode, bsize)) { + rate2 += write_uniform_cost(2 * MAX_ANGLE_DELTA + 1, + MAX_ANGLE_DELTA + mbmi->angle_delta[1]); + } +#endif // CONFIG_EXT_INTRA + if (mbmi->mode == DC_PRED) { + rate2 += + av1_cost_bit(cpi->common.fc->filter_intra_probs[1], + mbmi->filter_intra_mode_info.use_filter_intra_mode[1]); + if (mbmi->filter_intra_mode_info.use_filter_intra_mode[1]) + rate2 += + write_uniform_cost(FILTER_INTRA_MODES, + mbmi->filter_intra_mode_info.filter_intra_mode[1]); + } + distortion2 = distortion_y + distortion_uv; + av1_encode_intra_block_plane((AV1_COMMON *)cm, x, bsize, 0, 0, mi_row, + mi_col); + + rate2 += ref_costs_single[INTRA_FRAME]; + + if (skippable) { + rate2 -= (rate_y + rate_uv); + rate_y = 0; + rate_uv = 0; + rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + } else { + rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 0); + } + this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2); + + if (this_rd < *best_intra_rd) { + *best_intra_rd = this_rd; + *best_intra_mode = mbmi->mode; + } + for (i = 0; i < REFERENCE_MODES; ++i) + best_pred_rd[i] = AOMMIN(best_pred_rd[i], this_rd); + + if (this_rd < *best_rd) { + *best_mode_index = dc_mode_index; + mbmi->mv[0].as_int = 0; + rd_cost->rate = rate2; +#if CONFIG_SUPERTX + if (x->skip) + *returnrate_nocoef = rate2; + else + *returnrate_nocoef = rate2 - rate_y - rate_uv; + *returnrate_nocoef -= av1_cost_bit(av1_get_skip_prob(cm, xd), skippable); + *returnrate_nocoef -= av1_cost_bit(av1_get_intra_inter_prob(cm, xd), + mbmi->ref_frame[0] != INTRA_FRAME); +#endif // CONFIG_SUPERTX + rd_cost->dist = distortion2; + rd_cost->rdcost = this_rd; + *best_rd = this_rd; + *best_mbmode = *mbmi; + *best_skip2 = 0; + *best_mode_skippable = skippable; + } +} +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_MOTION_VAR +static void calc_target_weighted_pred(const AV1_COMMON *cm, const MACROBLOCK *x, + const MACROBLOCKD *xd, int mi_row, + int mi_col, const uint8_t *above, + int above_stride, const uint8_t *left, + int left_stride); +#endif // CONFIG_MOTION_VAR + +void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data, + MACROBLOCK *x, int mi_row, int mi_col, + RD_STATS *rd_cost, +#if CONFIG_SUPERTX + int *returnrate_nocoef, +#endif // CONFIG_SUPERTX + BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx, + int64_t best_rd_so_far) { + const AV1_COMMON *const cm = &cpi->common; + const RD_OPT *const rd_opt = &cpi->rd; + const SPEED_FEATURES *const sf = &cpi->sf; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; +#if CONFIG_PALETTE + const int try_palette = + cpi->common.allow_screen_content_tools && bsize >= BLOCK_8X8; + PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; +#endif // CONFIG_PALETTE + MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; + const struct segmentation *const seg = &cm->seg; + PREDICTION_MODE this_mode; + MV_REFERENCE_FRAME ref_frame, second_ref_frame; + unsigned char segment_id = mbmi->segment_id; + int comp_pred, i, k; + int_mv frame_mv[MB_MODE_COUNT][TOTAL_REFS_PER_FRAME]; + struct buf_2d yv12_mb[TOTAL_REFS_PER_FRAME][MAX_MB_PLANE]; + int_mv single_newmv[TOTAL_REFS_PER_FRAME] = { { 0 } }; +#if CONFIG_EXT_INTER + int single_newmv_rate[TOTAL_REFS_PER_FRAME] = { 0 }; + int64_t modelled_rd[MB_MODE_COUNT][TOTAL_REFS_PER_FRAME]; +#endif // CONFIG_EXT_INTER + static const int flag_list[TOTAL_REFS_PER_FRAME] = { + 0, + AOM_LAST_FLAG, +#if CONFIG_EXT_REFS + AOM_LAST2_FLAG, + AOM_LAST3_FLAG, +#endif // CONFIG_EXT_REFS + AOM_GOLD_FLAG, +#if CONFIG_EXT_REFS + AOM_BWD_FLAG, +#endif // CONFIG_EXT_REFS + AOM_ALT_FLAG + }; + int64_t best_rd = best_rd_so_far; + int best_rate_y = INT_MAX, best_rate_uv = INT_MAX; + int64_t best_pred_diff[REFERENCE_MODES]; + int64_t best_pred_rd[REFERENCE_MODES]; + MB_MODE_INFO best_mbmode; +#if CONFIG_REF_MV + int rate_skip0 = av1_cost_bit(av1_get_skip_prob(cm, xd), 0); + int rate_skip1 = av1_cost_bit(av1_get_skip_prob(cm, xd), 1); +#endif // CONFIG_REF_MV + int best_mode_skippable = 0; + int midx, best_mode_index = -1; + unsigned int ref_costs_single[TOTAL_REFS_PER_FRAME]; + unsigned int ref_costs_comp[TOTAL_REFS_PER_FRAME]; + aom_prob comp_mode_p; + int64_t best_intra_rd = INT64_MAX; + unsigned int best_pred_sse = UINT_MAX; + PREDICTION_MODE best_intra_mode = DC_PRED; + int rate_uv_intra[TX_SIZES_ALL], rate_uv_tokenonly[TX_SIZES_ALL]; + int64_t dist_uvs[TX_SIZES_ALL]; + int skip_uvs[TX_SIZES_ALL]; + PREDICTION_MODE mode_uv[TX_SIZES_ALL]; +#if CONFIG_PALETTE + PALETTE_MODE_INFO pmi_uv[TX_SIZES_ALL]; +#endif // CONFIG_PALETTE +#if CONFIG_EXT_INTRA + int8_t uv_angle_delta[TX_SIZES_ALL]; + int is_directional_mode, angle_stats_ready = 0; + uint8_t directional_mode_skip_mask[INTRA_MODES]; +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + int8_t dc_skipped = 1; + FILTER_INTRA_MODE_INFO filter_intra_mode_info_uv[TX_SIZES_ALL]; +#endif // CONFIG_FILTER_INTRA + const int intra_cost_penalty = av1_get_intra_cost_penalty( + cm->base_qindex, cm->y_dc_delta_q, cm->bit_depth); + const int *const intra_mode_cost = cpi->mbmode_cost[size_group_lookup[bsize]]; + int best_skip2 = 0; + uint8_t ref_frame_skip_mask[2] = { 0 }; +#if CONFIG_EXT_INTER + uint32_t mode_skip_mask[TOTAL_REFS_PER_FRAME] = { 0 }; + MV_REFERENCE_FRAME best_single_inter_ref = LAST_FRAME; + int64_t best_single_inter_rd = INT64_MAX; +#else + uint16_t mode_skip_mask[TOTAL_REFS_PER_FRAME] = { 0 }; +#endif // CONFIG_EXT_INTER + int mode_skip_start = sf->mode_skip_start + 1; + const int *const rd_threshes = rd_opt->threshes[segment_id][bsize]; + const int *const rd_thresh_freq_fact = tile_data->thresh_freq_fact[bsize]; + int64_t mode_threshold[MAX_MODES]; + int *mode_map = tile_data->mode_map[bsize]; + const int mode_search_skip_flags = sf->mode_search_skip_flags; +#if CONFIG_PVQ + od_rollback_buffer pre_buf; +#endif // CONFIG_PVQ + + HandleInterModeArgs args = { +#if CONFIG_MOTION_VAR + { NULL }, + { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }, + { NULL }, + { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }, +#endif // CONFIG_MOTION_VAR +#if CONFIG_EXT_INTER + NULL, + NULL, + NULL, + NULL, + NULL, +#else // CONFIG_EXT_INTER + NULL, +#endif // CONFIG_EXT_INTER + { { 0 } }, + }; + +#if CONFIG_PALETTE || CONFIG_EXT_INTRA + const int rows = block_size_high[bsize]; + const int cols = block_size_wide[bsize]; +#endif // CONFIG_PALETTE || CONFIG_EXT_INTRA +#if CONFIG_PALETTE + int palette_ctx = 0; + const MODE_INFO *above_mi = xd->above_mi; + const MODE_INFO *left_mi = xd->left_mi; +#endif // CONFIG_PALETTE +#if CONFIG_MOTION_VAR +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint8_t, tmp_buf1[2 * MAX_MB_PLANE * MAX_SB_SQUARE]); + DECLARE_ALIGNED(16, uint8_t, tmp_buf2[2 * MAX_MB_PLANE * MAX_SB_SQUARE]); +#else + DECLARE_ALIGNED(16, uint8_t, tmp_buf1[MAX_MB_PLANE * MAX_SB_SQUARE]); + DECLARE_ALIGNED(16, uint8_t, tmp_buf2[MAX_MB_PLANE * MAX_SB_SQUARE]); +#endif // CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, int32_t, weighted_src_buf[MAX_SB_SQUARE]); + DECLARE_ALIGNED(16, int32_t, mask2d_buf[MAX_SB_SQUARE]); + int dst_width1[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_width2[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_height1[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + int dst_height2[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + int len = sizeof(uint16_t); + args.above_pred_buf[0] = CONVERT_TO_BYTEPTR(tmp_buf1); + args.above_pred_buf[1] = CONVERT_TO_BYTEPTR(tmp_buf1 + MAX_SB_SQUARE * len); + args.above_pred_buf[2] = + CONVERT_TO_BYTEPTR(tmp_buf1 + 2 * MAX_SB_SQUARE * len); + args.left_pred_buf[0] = CONVERT_TO_BYTEPTR(tmp_buf2); + args.left_pred_buf[1] = CONVERT_TO_BYTEPTR(tmp_buf2 + MAX_SB_SQUARE * len); + args.left_pred_buf[2] = + CONVERT_TO_BYTEPTR(tmp_buf2 + 2 * MAX_SB_SQUARE * len); + } else { +#endif // CONFIG_HIGHBITDEPTH + args.above_pred_buf[0] = tmp_buf1; + args.above_pred_buf[1] = tmp_buf1 + MAX_SB_SQUARE; + args.above_pred_buf[2] = tmp_buf1 + 2 * MAX_SB_SQUARE; + args.left_pred_buf[0] = tmp_buf2; + args.left_pred_buf[1] = tmp_buf2 + MAX_SB_SQUARE; + args.left_pred_buf[2] = tmp_buf2 + 2 * MAX_SB_SQUARE; +#if CONFIG_HIGHBITDEPTH + } +#endif // CONFIG_HIGHBITDEPTH +#endif // CONFIG_MOTION_VAR + + av1_zero(best_mbmode); + +#if CONFIG_PALETTE + av1_zero(pmi_uv); + if (try_palette) { + if (above_mi) + palette_ctx += (above_mi->mbmi.palette_mode_info.palette_size[0] > 0); + if (left_mi) + palette_ctx += (left_mi->mbmi.palette_mode_info.palette_size[0] > 0); + } +#endif // CONFIG_PALETTE + +#if CONFIG_EXT_INTRA + memset(directional_mode_skip_mask, 0, + sizeof(directional_mode_skip_mask[0]) * INTRA_MODES); +#endif // CONFIG_EXT_INTRA + + estimate_ref_frame_costs(cm, xd, segment_id, ref_costs_single, ref_costs_comp, + &comp_mode_p); + + for (i = 0; i < REFERENCE_MODES; ++i) best_pred_rd[i] = INT64_MAX; + for (i = 0; i < TX_SIZES_ALL; i++) rate_uv_intra[i] = INT_MAX; + for (i = 0; i < TOTAL_REFS_PER_FRAME; ++i) x->pred_sse[i] = INT_MAX; + for (i = 0; i < MB_MODE_COUNT; ++i) { + for (k = 0; k < TOTAL_REFS_PER_FRAME; ++k) { + args.single_filter[i][k] = SWITCHABLE; + } + } + + rd_cost->rate = INT_MAX; +#if CONFIG_SUPERTX + *returnrate_nocoef = INT_MAX; +#endif // CONFIG_SUPERTX + + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { + x->pred_mv_sad[ref_frame] = INT_MAX; + x->mbmi_ext->mode_context[ref_frame] = 0; +#if CONFIG_REF_MV && CONFIG_EXT_INTER + x->mbmi_ext->compound_mode_context[ref_frame] = 0; +#endif // CONFIG_REF_MV && CONFIG_EXT_INTER + if (cpi->ref_frame_flags & flag_list[ref_frame]) { + assert(get_ref_frame_buffer(cpi, ref_frame) != NULL); + setup_buffer_inter(cpi, x, ref_frame, bsize, mi_row, mi_col, + frame_mv[NEARESTMV], frame_mv[NEARMV], yv12_mb); + } + frame_mv[NEWMV][ref_frame].as_int = INVALID_MV; +#if CONFIG_GLOBAL_MOTION + frame_mv[ZEROMV][ref_frame].as_int = + gm_get_motion_vector(&cm->global_motion[ref_frame], + cm->allow_high_precision_mv, bsize, mi_col, mi_row, + 0) + .as_int; +#else // CONFIG_GLOBAL_MOTION + frame_mv[ZEROMV][ref_frame].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_EXT_INTER + frame_mv[NEW_NEWMV][ref_frame].as_int = INVALID_MV; +#if CONFIG_GLOBAL_MOTION + frame_mv[ZERO_ZEROMV][ref_frame].as_int = + gm_get_motion_vector(&cm->global_motion[ref_frame], + cm->allow_high_precision_mv, bsize, mi_col, mi_row, + 0) + .as_int; +#else // CONFIG_GLOBAL_MOTION + frame_mv[ZERO_ZEROMV][ref_frame].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION +#endif // CONFIG_EXT_INTER + } + +#if CONFIG_REF_MV + for (; ref_frame < MODE_CTX_REF_FRAMES; ++ref_frame) { + MODE_INFO *const mi = xd->mi[0]; + int_mv *const candidates = x->mbmi_ext->ref_mvs[ref_frame]; + x->mbmi_ext->mode_context[ref_frame] = 0; + av1_find_mv_refs(cm, xd, mi, ref_frame, &mbmi_ext->ref_mv_count[ref_frame], + mbmi_ext->ref_mv_stack[ref_frame], +#if CONFIG_EXT_INTER + mbmi_ext->compound_mode_context, +#endif // CONFIG_EXT_INTER + candidates, mi_row, mi_col, NULL, NULL, + mbmi_ext->mode_context); + if (mbmi_ext->ref_mv_count[ref_frame] < 2) { + MV_REFERENCE_FRAME rf[2]; + av1_set_ref_frame(rf, ref_frame); + if (mbmi_ext->ref_mvs[rf[0]][0].as_int != + frame_mv[ZEROMV][rf[0]].as_int || + mbmi_ext->ref_mvs[rf[0]][1].as_int != + frame_mv[ZEROMV][rf[0]].as_int || + mbmi_ext->ref_mvs[rf[1]][0].as_int != + frame_mv[ZEROMV][rf[1]].as_int || + mbmi_ext->ref_mvs[rf[1]][1].as_int != frame_mv[ZEROMV][rf[1]].as_int) + mbmi_ext->mode_context[ref_frame] &= ~(1 << ALL_ZERO_FLAG_OFFSET); + } + } +#endif // CONFIG_REF_MV + +#if CONFIG_MOTION_VAR + av1_count_overlappable_neighbors(cm, xd, mi_row, mi_col); + if (check_num_overlappable_neighbors(mbmi) && + is_motion_variation_allowed_bsize(bsize)) { + av1_build_prediction_by_above_preds(cm, xd, mi_row, mi_col, + args.above_pred_buf, dst_width1, + dst_height1, args.above_pred_stride); + av1_build_prediction_by_left_preds(cm, xd, mi_row, mi_col, + args.left_pred_buf, dst_width2, + dst_height2, args.left_pred_stride); + av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, + mi_col); + x->mask_buf = mask2d_buf; + x->wsrc_buf = weighted_src_buf; + calc_target_weighted_pred(cm, x, xd, mi_row, mi_col, args.above_pred_buf[0], + args.above_pred_stride[0], args.left_pred_buf[0], + args.left_pred_stride[0]); + } +#endif // CONFIG_MOTION_VAR + + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { + if (!(cpi->ref_frame_flags & flag_list[ref_frame])) { +// Skip checking missing references in both single and compound reference +// modes. Note that a mode will be skipped iff both reference frames +// are masked out. +#if CONFIG_EXT_REFS + if (ref_frame == BWDREF_FRAME || ref_frame == ALTREF_FRAME) { + ref_frame_skip_mask[0] |= (1 << ref_frame); + ref_frame_skip_mask[1] |= ((1 << ref_frame) | 0x01); + } else { +#endif // CONFIG_EXT_REFS + ref_frame_skip_mask[0] |= (1 << ref_frame); + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; +#if CONFIG_EXT_REFS + } +#endif // CONFIG_EXT_REFS + } else { + for (i = LAST_FRAME; i <= ALTREF_FRAME; ++i) { + // Skip fixed mv modes for poor references + if ((x->pred_mv_sad[ref_frame] >> 2) > x->pred_mv_sad[i]) { + mode_skip_mask[ref_frame] |= INTER_NEAREST_NEAR_ZERO; + break; + } + } + } + // If the segment reference frame feature is enabled.... + // then do nothing if the current ref frame is not allowed.. + if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME) && + get_segdata(seg, segment_id, SEG_LVL_REF_FRAME) != (int)ref_frame) { + ref_frame_skip_mask[0] |= (1 << ref_frame); + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; + } + } + + // Disable this drop out case if the ref frame + // segment level feature is enabled for this segment. This is to + // prevent the possibility that we end up unable to pick any mode. + if (!segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) { + // Only consider ZEROMV/ALTREF_FRAME for alt ref frame, + // unless ARNR filtering is enabled in which case we want + // an unfiltered alternative. We allow near/nearest as well + // because they may result in zero-zero MVs but be cheaper. + if (cpi->rc.is_src_frame_alt_ref && (cpi->oxcf.arnr_max_frames == 0)) { + int_mv zeromv; + ref_frame_skip_mask[0] = (1 << LAST_FRAME) | +#if CONFIG_EXT_REFS + (1 << LAST2_FRAME) | (1 << LAST3_FRAME) | + (1 << BWDREF_FRAME) | +#endif // CONFIG_EXT_REFS + (1 << GOLDEN_FRAME); + ref_frame_skip_mask[1] = SECOND_REF_FRAME_MASK; + // TODO(zoeliu): To further explore whether following needs to be done for + // BWDREF_FRAME as well. + mode_skip_mask[ALTREF_FRAME] = ~INTER_NEAREST_NEAR_ZERO; +#if CONFIG_GLOBAL_MOTION + zeromv.as_int = gm_get_motion_vector(&cm->global_motion[ALTREF_FRAME], + cm->allow_high_precision_mv, bsize, + mi_col, mi_row, 0) + .as_int; +#else + zeromv.as_int = 0; +#endif // CONFIG_GLOBAL_MOTION + if (frame_mv[NEARMV][ALTREF_FRAME].as_int != zeromv.as_int) + mode_skip_mask[ALTREF_FRAME] |= (1 << NEARMV); + if (frame_mv[NEARESTMV][ALTREF_FRAME].as_int != zeromv.as_int) + mode_skip_mask[ALTREF_FRAME] |= (1 << NEARESTMV); +#if CONFIG_EXT_INTER + if (frame_mv[NEAREST_NEARESTMV][ALTREF_FRAME].as_int != zeromv.as_int) + mode_skip_mask[ALTREF_FRAME] |= (1 << NEAREST_NEARESTMV); + if (frame_mv[NEAREST_NEARMV][ALTREF_FRAME].as_int != zeromv.as_int) + mode_skip_mask[ALTREF_FRAME] |= (1 << NEAREST_NEARMV); + if (frame_mv[NEAR_NEARESTMV][ALTREF_FRAME].as_int != zeromv.as_int) + mode_skip_mask[ALTREF_FRAME] |= (1 << NEAR_NEARESTMV); + if (frame_mv[NEAR_NEARMV][ALTREF_FRAME].as_int != zeromv.as_int) + mode_skip_mask[ALTREF_FRAME] |= (1 << NEAR_NEARMV); +#endif // CONFIG_EXT_INTER + } + } + + if (cpi->rc.is_src_frame_alt_ref) { + if (sf->alt_ref_search_fp) { + assert(cpi->ref_frame_flags & flag_list[ALTREF_FRAME]); + mode_skip_mask[ALTREF_FRAME] = 0; + ref_frame_skip_mask[0] = ~(1 << ALTREF_FRAME); + ref_frame_skip_mask[1] = SECOND_REF_FRAME_MASK; + } + } + + if (sf->alt_ref_search_fp) + if (!cm->show_frame && x->pred_mv_sad[GOLDEN_FRAME] < INT_MAX) + if (x->pred_mv_sad[ALTREF_FRAME] > (x->pred_mv_sad[GOLDEN_FRAME] << 1)) + mode_skip_mask[ALTREF_FRAME] |= INTER_ALL; + + if (sf->adaptive_mode_search) { + if (cm->show_frame && !cpi->rc.is_src_frame_alt_ref && + cpi->rc.frames_since_golden >= 3) + if (x->pred_mv_sad[GOLDEN_FRAME] > (x->pred_mv_sad[LAST_FRAME] << 1)) + mode_skip_mask[GOLDEN_FRAME] |= INTER_ALL; + } + + if (bsize > sf->max_intra_bsize) { + ref_frame_skip_mask[0] |= (1 << INTRA_FRAME); + ref_frame_skip_mask[1] |= (1 << INTRA_FRAME); + } + + mode_skip_mask[INTRA_FRAME] |= + ~(sf->intra_y_mode_mask[max_txsize_lookup[bsize]]); + + for (i = 0; i <= LAST_NEW_MV_INDEX; ++i) mode_threshold[i] = 0; + for (i = LAST_NEW_MV_INDEX + 1; i < MAX_MODES; ++i) + mode_threshold[i] = ((int64_t)rd_threshes[i] * rd_thresh_freq_fact[i]) >> 5; + + midx = sf->schedule_mode_search ? mode_skip_start : 0; + while (midx > 4) { + uint8_t end_pos = 0; + for (i = 5; i < midx; ++i) { + if (mode_threshold[mode_map[i - 1]] > mode_threshold[mode_map[i]]) { + uint8_t tmp = mode_map[i]; + mode_map[i] = mode_map[i - 1]; + mode_map[i - 1] = tmp; + end_pos = i; + } + } + midx = end_pos; + } + + if (cpi->sf.tx_type_search.fast_intra_tx_type_search) + x->use_default_intra_tx_type = 1; + else + x->use_default_intra_tx_type = 0; + + if (cpi->sf.tx_type_search.fast_inter_tx_type_search) + x->use_default_inter_tx_type = 1; + else + x->use_default_inter_tx_type = 0; +#if CONFIG_PVQ + od_encode_checkpoint(&x->daala_enc, &pre_buf); +#endif // CONFIG_PVQ +#if CONFIG_EXT_INTER + for (i = 0; i < MB_MODE_COUNT; ++i) + for (ref_frame = 0; ref_frame < TOTAL_REFS_PER_FRAME; ++ref_frame) + modelled_rd[i][ref_frame] = INT64_MAX; +#endif // CONFIG_EXT_INTER + + for (midx = 0; midx < MAX_MODES; ++midx) { + int mode_index; + int mode_excluded = 0; + int64_t this_rd = INT64_MAX; + int disable_skip = 0; + int compmode_cost = 0; +#if CONFIG_EXT_INTER + int compmode_interintra_cost = 0; + int compmode_interinter_cost = 0; +#endif // CONFIG_EXT_INTER + int rate2 = 0, rate_y = 0, rate_uv = 0; + int64_t distortion2 = 0, distortion_y = 0, distortion_uv = 0; + int skippable = 0; + int this_skip2 = 0; + int64_t total_sse = INT64_MAX; +#if CONFIG_REF_MV + uint8_t ref_frame_type; +#endif // CONFIG_REF_MV +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &pre_buf); +#endif // CONFIG_PVQ + mode_index = mode_map[midx]; + this_mode = av1_mode_order[mode_index].mode; + ref_frame = av1_mode_order[mode_index].ref_frame[0]; + second_ref_frame = av1_mode_order[mode_index].ref_frame[1]; +#if CONFIG_REF_MV + mbmi->ref_mv_idx = 0; +#endif // CONFIG_REF_MV + +#if CONFIG_EXT_INTER + if (ref_frame > INTRA_FRAME && second_ref_frame == INTRA_FRAME) { + // Mode must by compatible + if (!is_interintra_allowed_mode(this_mode)) continue; + if (!is_interintra_allowed_bsize(bsize)) continue; + } + + if (is_inter_compound_mode(this_mode)) { + frame_mv[this_mode][ref_frame].as_int = + frame_mv[compound_ref0_mode(this_mode)][ref_frame].as_int; + frame_mv[this_mode][second_ref_frame].as_int = + frame_mv[compound_ref1_mode(this_mode)][second_ref_frame].as_int; + } +#endif // CONFIG_EXT_INTER + + // Look at the reference frame of the best mode so far and set the + // skip mask to look at a subset of the remaining modes. + if (midx == mode_skip_start && best_mode_index >= 0) { + switch (best_mbmode.ref_frame[0]) { + case INTRA_FRAME: break; + case LAST_FRAME: + ref_frame_skip_mask[0] |= LAST_FRAME_MODE_MASK; + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; + break; +#if CONFIG_EXT_REFS + case LAST2_FRAME: + ref_frame_skip_mask[0] |= LAST2_FRAME_MODE_MASK; + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; + break; + case LAST3_FRAME: + ref_frame_skip_mask[0] |= LAST3_FRAME_MODE_MASK; + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; + break; +#endif // CONFIG_EXT_REFS + case GOLDEN_FRAME: + ref_frame_skip_mask[0] |= GOLDEN_FRAME_MODE_MASK; + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; + break; +#if CONFIG_EXT_REFS + case BWDREF_FRAME: + ref_frame_skip_mask[0] |= BWDREF_FRAME_MODE_MASK; + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; + break; +#endif // CONFIG_EXT_REFS + case ALTREF_FRAME: ref_frame_skip_mask[0] |= ALTREF_FRAME_MODE_MASK; +#if CONFIG_EXT_REFS + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; +#endif // CONFIG_EXT_REFS + break; + case NONE_FRAME: + case TOTAL_REFS_PER_FRAME: + assert(0 && "Invalid Reference frame"); + break; + } + } + + if ((ref_frame_skip_mask[0] & (1 << ref_frame)) && + (ref_frame_skip_mask[1] & (1 << AOMMAX(0, second_ref_frame)))) + continue; + + if (mode_skip_mask[ref_frame] & (1 << this_mode)) continue; + + // Test best rd so far against threshold for trying this mode. + if (best_mode_skippable && sf->schedule_mode_search) + mode_threshold[mode_index] <<= 1; + + if (best_rd < mode_threshold[mode_index]) continue; + + // This is only used in motion vector unit test. + if (cpi->oxcf.motion_vector_unit_test && ref_frame == INTRA_FRAME) continue; + +#if CONFIG_LOWDELAY_COMPOUND // Changes LL bitstream +#if CONFIG_EXT_REFS + if (cpi->oxcf.pass == 0) { + // Complexity-compression trade-offs + // if (ref_frame == ALTREF_FRAME) continue; + // if (ref_frame == BWDREF_FRAME) continue; + if (second_ref_frame == ALTREF_FRAME) continue; + // if (second_ref_frame == BWDREF_FRAME) continue; + } +#endif +#endif + comp_pred = second_ref_frame > INTRA_FRAME; + if (comp_pred) { + if (!cpi->allow_comp_inter_inter) continue; + + // Skip compound inter modes if ARF is not available. + if (!(cpi->ref_frame_flags & flag_list[second_ref_frame])) continue; + + // Do not allow compound prediction if the segment level reference frame + // feature is in use as in this case there can only be one reference. + if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) continue; + + if ((mode_search_skip_flags & FLAG_SKIP_COMP_BESTINTRA) && + best_mode_index >= 0 && best_mbmode.ref_frame[0] == INTRA_FRAME) + continue; + + mode_excluded = cm->reference_mode == SINGLE_REFERENCE; + } else { + if (ref_frame != INTRA_FRAME) + mode_excluded = cm->reference_mode == COMPOUND_REFERENCE; + } + + if (ref_frame == INTRA_FRAME) { + if (sf->adaptive_mode_search) + if ((x->source_variance << num_pels_log2_lookup[bsize]) > best_pred_sse) + continue; + + if (this_mode != DC_PRED) { + // Disable intra modes other than DC_PRED for blocks with low variance + // Threshold for intra skipping based on source variance + // TODO(debargha): Specialize the threshold for super block sizes + const unsigned int skip_intra_var_thresh = 64; + if ((mode_search_skip_flags & FLAG_SKIP_INTRA_LOWVAR) && + x->source_variance < skip_intra_var_thresh) + continue; + // Only search the oblique modes if the best so far is + // one of the neighboring directional modes + if ((mode_search_skip_flags & FLAG_SKIP_INTRA_BESTINTER) && + (this_mode >= D45_PRED && this_mode <= TM_PRED)) { + if (best_mode_index >= 0 && best_mbmode.ref_frame[0] > INTRA_FRAME) + continue; + } + if (mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) { + if (conditional_skipintra(this_mode, best_intra_mode)) continue; + } + } +#if CONFIG_GLOBAL_MOTION + } else if (cm->global_motion[ref_frame].wmtype == IDENTITY && + (!comp_pred || + cm->global_motion[second_ref_frame].wmtype == IDENTITY)) { +#else // CONFIG_GLOBAL_MOTION + } else { +#endif // CONFIG_GLOBAL_MOTION + const MV_REFERENCE_FRAME ref_frames[2] = { ref_frame, second_ref_frame }; + if (!check_best_zero_mv(cpi, mbmi_ext->mode_context, +#if CONFIG_REF_MV && CONFIG_EXT_INTER + mbmi_ext->compound_mode_context, +#endif // CONFIG_REF_MV && CONFIG_EXT_INTER + frame_mv, this_mode, ref_frames, bsize, -1, + mi_row, mi_col)) + continue; + } + + mbmi->mode = this_mode; + mbmi->uv_mode = DC_PRED; + mbmi->ref_frame[0] = ref_frame; + mbmi->ref_frame[1] = second_ref_frame; +#if CONFIG_PALETTE + pmi->palette_size[0] = 0; + pmi->palette_size[1] = 0; +#endif // CONFIG_PALETTE +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0; + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = 0; +#endif // CONFIG_FILTER_INTRA + // Evaluate all sub-pel filters irrespective of whether we can use + // them for this frame. + + set_default_interp_filters(mbmi, cm->interp_filter); + + mbmi->mv[0].as_int = mbmi->mv[1].as_int = 0; + mbmi->motion_mode = SIMPLE_TRANSLATION; + + x->skip = 0; + set_ref_ptrs(cm, xd, ref_frame, second_ref_frame); + + // Select prediction reference frames. + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].pre[0] = yv12_mb[ref_frame][i]; + if (comp_pred) xd->plane[i].pre[1] = yv12_mb[second_ref_frame][i]; + } + +#if CONFIG_EXT_INTER + mbmi->interintra_mode = (INTERINTRA_MODE)(II_DC_PRED - 1); +#endif // CONFIG_EXT_INTER + + if (ref_frame == INTRA_FRAME) { + RD_STATS rd_stats_y; + TX_SIZE uv_tx; + struct macroblockd_plane *const pd = &xd->plane[1]; +#if CONFIG_EXT_INTRA + is_directional_mode = av1_is_directional_mode(mbmi->mode, bsize); + if (is_directional_mode) { + int rate_dummy; + int64_t model_rd = INT64_MAX; + if (!angle_stats_ready) { + const int src_stride = x->plane[0].src.stride; + const uint8_t *src = x->plane[0].src.buf; +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + highbd_angle_estimation(src, src_stride, rows, cols, + directional_mode_skip_mask); + else +#endif // CONFIG_HIGHBITDEPTH + angle_estimation(src, src_stride, rows, cols, + directional_mode_skip_mask); + angle_stats_ready = 1; + } + if (directional_mode_skip_mask[mbmi->mode]) continue; + rd_stats_y.rate = INT_MAX; + rd_pick_intra_angle_sby(cpi, x, &rate_dummy, &rd_stats_y, bsize, + intra_mode_cost[mbmi->mode], best_rd, + &model_rd); + } else { + mbmi->angle_delta[0] = 0; + super_block_yrd(cpi, x, &rd_stats_y, bsize, best_rd); + } +#else + super_block_yrd(cpi, x, &rd_stats_y, bsize, best_rd); +#endif // CONFIG_EXT_INTRA + rate_y = rd_stats_y.rate; + distortion_y = rd_stats_y.dist; + skippable = rd_stats_y.skip; + + if (rate_y == INT_MAX) continue; + +#if CONFIG_FILTER_INTRA + if (mbmi->mode == DC_PRED) dc_skipped = 0; +#endif // CONFIG_FILTER_INTRA + + uv_tx = uv_txsize_lookup[bsize][mbmi->tx_size][pd->subsampling_x] + [pd->subsampling_y]; + if (rate_uv_intra[uv_tx] == INT_MAX) { + choose_intra_uv_mode(cpi, x, ctx, bsize, uv_tx, &rate_uv_intra[uv_tx], + &rate_uv_tokenonly[uv_tx], &dist_uvs[uv_tx], + &skip_uvs[uv_tx], &mode_uv[uv_tx]); +#if CONFIG_PALETTE + if (try_palette) pmi_uv[uv_tx] = *pmi; +#endif // CONFIG_PALETTE + +#if CONFIG_EXT_INTRA + uv_angle_delta[uv_tx] = mbmi->angle_delta[1]; +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + filter_intra_mode_info_uv[uv_tx] = mbmi->filter_intra_mode_info; +#endif // CONFIG_FILTER_INTRA + } + + rate_uv = rate_uv_tokenonly[uv_tx]; + distortion_uv = dist_uvs[uv_tx]; + skippable = skippable && skip_uvs[uv_tx]; + mbmi->uv_mode = mode_uv[uv_tx]; +#if CONFIG_PALETTE + if (try_palette) { + pmi->palette_size[1] = pmi_uv[uv_tx].palette_size[1]; + memcpy(pmi->palette_colors + PALETTE_MAX_SIZE, + pmi_uv[uv_tx].palette_colors + PALETTE_MAX_SIZE, + 2 * PALETTE_MAX_SIZE * sizeof(pmi->palette_colors[0])); + } +#endif // CONFIG_PALETTE + +#if CONFIG_EXT_INTRA + mbmi->angle_delta[1] = uv_angle_delta[uv_tx]; +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = + filter_intra_mode_info_uv[uv_tx].use_filter_intra_mode[1]; + if (filter_intra_mode_info_uv[uv_tx].use_filter_intra_mode[1]) { + mbmi->filter_intra_mode_info.filter_intra_mode[1] = + filter_intra_mode_info_uv[uv_tx].filter_intra_mode[1]; + } +#endif // CONFIG_FILTER_INTRA + +#if CONFIG_CB4X4 + rate2 = rate_y + intra_mode_cost[mbmi->mode]; + if (!x->skip_chroma_rd) + rate2 += rate_uv + cpi->intra_uv_mode_cost[mbmi->mode][mbmi->uv_mode]; +#else + rate2 = rate_y + intra_mode_cost[mbmi->mode] + rate_uv + + cpi->intra_uv_mode_cost[mbmi->mode][mbmi->uv_mode]; +#endif // CONFIG_CB4X4 + +#if CONFIG_PALETTE + if (try_palette && mbmi->mode == DC_PRED) { + rate2 += av1_cost_bit( + av1_default_palette_y_mode_prob[bsize - BLOCK_8X8][palette_ctx], 0); + } +#endif // CONFIG_PALETTE + + if (!xd->lossless[mbmi->segment_id] && bsize >= BLOCK_8X8) { + // super_block_yrd above includes the cost of the tx_size in the + // tokenonly rate, but for intra blocks, tx_size is always coded + // (prediction granularity), so we account for it in the full rate, + // not the tokenonly rate. + rate_y -= tx_size_cost(cpi, x, bsize, mbmi->tx_size); + } +#if CONFIG_EXT_INTRA + if (is_directional_mode) { +#if CONFIG_INTRA_INTERP + const int intra_filter_ctx = av1_get_pred_context_intra_interp(xd); + const int p_angle = + mode_to_angle_map[mbmi->mode] + mbmi->angle_delta[0] * ANGLE_STEP; + if (av1_is_intra_filter_switchable(p_angle)) + rate2 += cpi->intra_filter_cost[intra_filter_ctx][mbmi->intra_filter]; +#endif // CONFIG_INTRA_INTERP + rate2 += write_uniform_cost(2 * MAX_ANGLE_DELTA + 1, + MAX_ANGLE_DELTA + mbmi->angle_delta[0]); + } + if (mbmi->uv_mode != DC_PRED && mbmi->uv_mode != TM_PRED) { + rate2 += write_uniform_cost(2 * MAX_ANGLE_DELTA + 1, + MAX_ANGLE_DELTA + mbmi->angle_delta[1]); + } +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + if (mbmi->mode == DC_PRED) { + rate2 += + av1_cost_bit(cm->fc->filter_intra_probs[0], + mbmi->filter_intra_mode_info.use_filter_intra_mode[0]); + if (mbmi->filter_intra_mode_info.use_filter_intra_mode[0]) { + rate2 += write_uniform_cost( + FILTER_INTRA_MODES, + mbmi->filter_intra_mode_info.filter_intra_mode[0]); + } + } + if (mbmi->uv_mode == DC_PRED) { + rate2 += + av1_cost_bit(cpi->common.fc->filter_intra_probs[1], + mbmi->filter_intra_mode_info.use_filter_intra_mode[1]); + if (mbmi->filter_intra_mode_info.use_filter_intra_mode[1]) + rate2 += write_uniform_cost( + FILTER_INTRA_MODES, + mbmi->filter_intra_mode_info.filter_intra_mode[1]); + } +#endif // CONFIG_FILTER_INTRA + if (mbmi->mode != DC_PRED && mbmi->mode != TM_PRED) + rate2 += intra_cost_penalty; + distortion2 = distortion_y + distortion_uv; + } else { +#if CONFIG_REF_MV + int_mv backup_ref_mv[2]; + +#if !SUB8X8_COMP_REF + if (bsize < BLOCK_8X8 && mbmi->ref_frame[1] > INTRA_FRAME) continue; +#endif // !SUB8X8_COMP_REF + + backup_ref_mv[0] = mbmi_ext->ref_mvs[ref_frame][0]; + if (comp_pred) backup_ref_mv[1] = mbmi_ext->ref_mvs[second_ref_frame][0]; +#endif // CONFIG_REF_MV +#if CONFIG_EXT_INTER + if (second_ref_frame == INTRA_FRAME) { + if (best_single_inter_ref != ref_frame) continue; + mbmi->interintra_mode = intra_to_interintra_mode[best_intra_mode]; +// TODO(debargha|geza.lore): +// Should we use ext_intra modes for interintra? +#if CONFIG_EXT_INTRA + mbmi->angle_delta[0] = 0; + mbmi->angle_delta[1] = 0; +#if CONFIG_INTRA_INTERP + mbmi->intra_filter = INTRA_FILTER_LINEAR; +#endif // CONFIG_INTRA_INTERP +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0; + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = 0; +#endif // CONFIG_FILTER_INTRA + } +#endif // CONFIG_EXT_INTER +#if CONFIG_REF_MV + mbmi->ref_mv_idx = 0; + ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + +#if CONFIG_EXT_INTER + if (comp_pred) { + if (mbmi_ext->ref_mv_count[ref_frame_type] > 1) { + int ref_mv_idx = 0; + // Special case: NEAR_NEWMV and NEW_NEARMV modes use + // 1 + mbmi->ref_mv_idx (like NEARMV) instead of + // mbmi->ref_mv_idx (like NEWMV) + if (mbmi->mode == NEAR_NEWMV || mbmi->mode == NEW_NEARMV) + ref_mv_idx = 1; + + if (compound_ref0_mode(mbmi->mode) == NEWMV) { + int_mv this_mv = + mbmi_ext->ref_mv_stack[ref_frame_type][ref_mv_idx].this_mv; + clamp_mv_ref(&this_mv.as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + mbmi_ext->ref_mvs[mbmi->ref_frame[0]][0] = this_mv; + } + if (compound_ref1_mode(mbmi->mode) == NEWMV) { + int_mv this_mv = + mbmi_ext->ref_mv_stack[ref_frame_type][ref_mv_idx].comp_mv; + clamp_mv_ref(&this_mv.as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + mbmi_ext->ref_mvs[mbmi->ref_frame[1]][0] = this_mv; + } + } + } else { +#endif // CONFIG_EXT_INTER + if (mbmi->mode == NEWMV && mbmi_ext->ref_mv_count[ref_frame_type] > 1) { + int ref; + for (ref = 0; ref < 1 + comp_pred; ++ref) { + int_mv this_mv = + (ref == 0) ? mbmi_ext->ref_mv_stack[ref_frame_type][0].this_mv + : mbmi_ext->ref_mv_stack[ref_frame_type][0].comp_mv; + clamp_mv_ref(&this_mv.as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + mbmi_ext->ref_mvs[mbmi->ref_frame[ref]][0] = this_mv; + } + } +#if CONFIG_EXT_INTER + } +#endif // CONFIG_EXT_INTER +#endif // CONFIG_REF_MV + { + RD_STATS rd_stats, rd_stats_y, rd_stats_uv; + av1_init_rd_stats(&rd_stats); + rd_stats.rate = rate2; + + // Point to variables that are maintained between loop iterations + args.single_newmv = single_newmv; +#if CONFIG_EXT_INTER + args.single_newmv_rate = single_newmv_rate; + args.compmode_interintra_cost = &compmode_interintra_cost; + args.compmode_interinter_cost = &compmode_interinter_cost; + args.modelled_rd = modelled_rd; +#endif // CONFIG_EXT_INTER + this_rd = handle_inter_mode(cpi, x, bsize, &rd_stats, &rd_stats_y, + &rd_stats_uv, &disable_skip, frame_mv, + mi_row, mi_col, &args, best_rd); +// Prevent pointers from escaping local scope +#if CONFIG_EXT_INTER + args.compmode_interintra_cost = NULL; + args.compmode_interinter_cost = NULL; +#endif // CONFIG_EXT_INTER + + rate2 = rd_stats.rate; + skippable = rd_stats.skip; + distortion2 = rd_stats.dist; + total_sse = rd_stats.sse; + rate_y = rd_stats_y.rate; + rate_uv = rd_stats_uv.rate; + } + +#if CONFIG_REF_MV +// TODO(jingning): This needs some refactoring to improve code quality +// and reduce redundant steps. +#if CONFIG_EXT_INTER + if ((have_nearmv_in_inter_mode(mbmi->mode) && + mbmi_ext->ref_mv_count[ref_frame_type] > 2) || + ((mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) && + mbmi_ext->ref_mv_count[ref_frame_type] > 1)) { +#else + if ((mbmi->mode == NEARMV && + mbmi_ext->ref_mv_count[ref_frame_type] > 2) || + (mbmi->mode == NEWMV && mbmi_ext->ref_mv_count[ref_frame_type] > 1)) { +#endif + int_mv backup_mv = frame_mv[NEARMV][ref_frame]; + MB_MODE_INFO backup_mbmi = *mbmi; + int backup_skip = x->skip; + int64_t tmp_ref_rd = this_rd; + int ref_idx; + +// TODO(jingning): This should be deprecated shortly. +#if CONFIG_EXT_INTER + int idx_offset = have_nearmv_in_inter_mode(mbmi->mode) ? 1 : 0; +#else + int idx_offset = (mbmi->mode == NEARMV) ? 1 : 0; +#endif // CONFIG_EXT_INTER + int ref_set = + AOMMIN(2, mbmi_ext->ref_mv_count[ref_frame_type] - 1 - idx_offset); + + uint8_t drl_ctx = + av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx_offset); + // Dummy + int_mv backup_fmv[2]; + backup_fmv[0] = frame_mv[NEWMV][ref_frame]; + if (comp_pred) backup_fmv[1] = frame_mv[NEWMV][second_ref_frame]; + + rate2 += (rate2 < INT_MAX ? cpi->drl_mode_cost0[drl_ctx][0] : 0); + + if (this_rd < INT64_MAX) { + if (RDCOST(x->rdmult, x->rddiv, rate_y + rate_uv, distortion2) < + RDCOST(x->rdmult, x->rddiv, 0, total_sse)) + tmp_ref_rd = + RDCOST(x->rdmult, x->rddiv, + rate2 + av1_cost_bit(av1_get_skip_prob(cm, xd), 0), + distortion2); + else + tmp_ref_rd = + RDCOST(x->rdmult, x->rddiv, + rate2 + av1_cost_bit(av1_get_skip_prob(cm, xd), 1) - + rate_y - rate_uv, + total_sse); + } +#if CONFIG_VAR_TX + for (i = 0; i < MAX_MB_PLANE; ++i) + memcpy(x->blk_skip_drl[i], x->blk_skip[i], + sizeof(uint8_t) * ctx->num_4x4_blk); +#endif // CONFIG_VAR_TX + + for (ref_idx = 0; ref_idx < ref_set; ++ref_idx) { + int64_t tmp_alt_rd = INT64_MAX; + int dummy_disable_skip = 0; + int ref; + int_mv cur_mv; + RD_STATS tmp_rd_stats, tmp_rd_stats_y, tmp_rd_stats_uv; +#if CONFIG_EXT_INTER + int tmp_compmode_interintra_cost = 0; + int tmp_compmode_interinter_cost = 0; +#endif // CONFIG_EXT_INTER + + av1_invalid_rd_stats(&tmp_rd_stats); + x->skip = 0; + + mbmi->ref_mv_idx = 1 + ref_idx; + +#if CONFIG_EXT_INTER + if (comp_pred) { + int ref_mv_idx = mbmi->ref_mv_idx; + // Special case: NEAR_NEWMV and NEW_NEARMV modes use + // 1 + mbmi->ref_mv_idx (like NEARMV) instead of + // mbmi->ref_mv_idx (like NEWMV) + if (mbmi->mode == NEAR_NEWMV || mbmi->mode == NEW_NEARMV) + ref_mv_idx = 1 + mbmi->ref_mv_idx; + + if (compound_ref0_mode(mbmi->mode) == NEWMV) { + int_mv this_mv = + mbmi_ext->ref_mv_stack[ref_frame_type][ref_mv_idx].this_mv; + clamp_mv_ref(&this_mv.as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + mbmi_ext->ref_mvs[mbmi->ref_frame[0]][0] = this_mv; + } else if (compound_ref0_mode(mbmi->mode) == NEARESTMV) { + int_mv this_mv = + mbmi_ext->ref_mv_stack[ref_frame_type][0].this_mv; + clamp_mv_ref(&this_mv.as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + mbmi_ext->ref_mvs[mbmi->ref_frame[0]][0] = this_mv; + } + + if (compound_ref1_mode(mbmi->mode) == NEWMV) { + int_mv this_mv = + mbmi_ext->ref_mv_stack[ref_frame_type][ref_mv_idx].comp_mv; + clamp_mv_ref(&this_mv.as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + mbmi_ext->ref_mvs[mbmi->ref_frame[1]][0] = this_mv; + } else if (compound_ref1_mode(mbmi->mode) == NEARESTMV) { + int_mv this_mv = + mbmi_ext->ref_mv_stack[ref_frame_type][0].comp_mv; + clamp_mv_ref(&this_mv.as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + mbmi_ext->ref_mvs[mbmi->ref_frame[1]][0] = this_mv; + } + } else { +#endif // CONFIG_EXT_INTER + for (ref = 0; ref < 1 + comp_pred; ++ref) { + int_mv this_mv = + (ref == 0) + ? mbmi_ext->ref_mv_stack[ref_frame_type][mbmi->ref_mv_idx] + .this_mv + : mbmi_ext->ref_mv_stack[ref_frame_type][mbmi->ref_mv_idx] + .comp_mv; + clamp_mv_ref(&this_mv.as_mv, xd->n8_w << MI_SIZE_LOG2, + xd->n8_h << MI_SIZE_LOG2, xd); + mbmi_ext->ref_mvs[mbmi->ref_frame[ref]][0] = this_mv; + } +#if CONFIG_EXT_INTER + } +#endif + + cur_mv = + mbmi_ext->ref_mv_stack[ref_frame][mbmi->ref_mv_idx + idx_offset] + .this_mv; + clamp_mv2(&cur_mv.as_mv, xd); + + if (!mv_check_bounds(&x->mv_limits, &cur_mv.as_mv)) { + int_mv dummy_single_newmv[TOTAL_REFS_PER_FRAME] = { { 0 } }; +#if CONFIG_EXT_INTER + int dummy_single_newmv_rate[TOTAL_REFS_PER_FRAME] = { 0 }; +#endif // CONFIG_EXT_INTER + + frame_mv[NEARMV][ref_frame] = cur_mv; + av1_init_rd_stats(&tmp_rd_stats); + + // Point to variables that are not maintained between iterations + args.single_newmv = dummy_single_newmv; +#if CONFIG_EXT_INTER + args.single_newmv_rate = dummy_single_newmv_rate; + args.compmode_interintra_cost = &tmp_compmode_interintra_cost; + args.compmode_interinter_cost = &tmp_compmode_interinter_cost; + args.modelled_rd = NULL; +#endif // CONFIG_EXT_INTER + tmp_alt_rd = handle_inter_mode( + cpi, x, bsize, &tmp_rd_stats, &tmp_rd_stats_y, &tmp_rd_stats_uv, + &dummy_disable_skip, frame_mv, mi_row, mi_col, &args, best_rd); + // Prevent pointers from escaping local scope + args.single_newmv = NULL; +#if CONFIG_EXT_INTER + args.single_newmv_rate = NULL; + args.compmode_interintra_cost = NULL; + args.compmode_interinter_cost = NULL; +#endif // CONFIG_EXT_INTER + } + + for (i = 0; i < mbmi->ref_mv_idx; ++i) { + uint8_t drl1_ctx = 0; + drl1_ctx = av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], + i + idx_offset); + tmp_rd_stats.rate += + (tmp_rd_stats.rate < INT_MAX ? cpi->drl_mode_cost0[drl1_ctx][1] + : 0); + } + + if (mbmi_ext->ref_mv_count[ref_frame_type] > + mbmi->ref_mv_idx + idx_offset + 1 && + ref_idx < ref_set - 1) { + uint8_t drl1_ctx = + av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], + mbmi->ref_mv_idx + idx_offset); + tmp_rd_stats.rate += + (tmp_rd_stats.rate < INT_MAX ? cpi->drl_mode_cost0[drl1_ctx][0] + : 0); + } + + if (tmp_alt_rd < INT64_MAX) { +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + tmp_alt_rd = RDCOST(x->rdmult, x->rddiv, tmp_rd_stats.rate, + tmp_rd_stats.dist); +#else + if (RDCOST(x->rdmult, x->rddiv, + tmp_rd_stats_y.rate + tmp_rd_stats_uv.rate, + tmp_rd_stats.dist) < + RDCOST(x->rdmult, x->rddiv, 0, tmp_rd_stats.sse)) + tmp_alt_rd = + RDCOST(x->rdmult, x->rddiv, + tmp_rd_stats.rate + + av1_cost_bit(av1_get_skip_prob(cm, xd), 0), + tmp_rd_stats.dist); + else + tmp_alt_rd = + RDCOST(x->rdmult, x->rddiv, + tmp_rd_stats.rate + + av1_cost_bit(av1_get_skip_prob(cm, xd), 1) - + tmp_rd_stats_y.rate - tmp_rd_stats_uv.rate, + tmp_rd_stats.sse); +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + } + + if (tmp_ref_rd > tmp_alt_rd) { + rate2 = tmp_rd_stats.rate; + disable_skip = dummy_disable_skip; + distortion2 = tmp_rd_stats.dist; + skippable = tmp_rd_stats.skip; + rate_y = tmp_rd_stats_y.rate; + rate_uv = tmp_rd_stats_uv.rate; + total_sse = tmp_rd_stats.sse; + this_rd = tmp_alt_rd; + tmp_ref_rd = tmp_alt_rd; + backup_mbmi = *mbmi; + backup_skip = x->skip; +#if CONFIG_VAR_TX + for (i = 0; i < MAX_MB_PLANE; ++i) + memcpy(x->blk_skip_drl[i], x->blk_skip[i], + sizeof(uint8_t) * ctx->num_4x4_blk); +#endif // CONFIG_VAR_TX +#if CONFIG_EXT_INTER + compmode_interintra_cost = tmp_compmode_interintra_cost; + compmode_interinter_cost = tmp_compmode_interinter_cost; +#endif // CONFIG_EXT_INTER + } else { + *mbmi = backup_mbmi; + x->skip = backup_skip; + } + } + + frame_mv[NEARMV][ref_frame] = backup_mv; + frame_mv[NEWMV][ref_frame] = backup_fmv[0]; + if (comp_pred) frame_mv[NEWMV][second_ref_frame] = backup_fmv[1]; +#if CONFIG_VAR_TX + for (i = 0; i < MAX_MB_PLANE; ++i) + memcpy(x->blk_skip[i], x->blk_skip_drl[i], + sizeof(uint8_t) * ctx->num_4x4_blk); +#endif // CONFIG_VAR_TX + } + mbmi_ext->ref_mvs[ref_frame][0] = backup_ref_mv[0]; + if (comp_pred) mbmi_ext->ref_mvs[second_ref_frame][0] = backup_ref_mv[1]; +#endif // CONFIG_REF_MV + + if (this_rd == INT64_MAX) continue; + +#if SUB8X8_COMP_REF + compmode_cost = av1_cost_bit(comp_mode_p, comp_pred); +#else + if (mbmi->sb_type >= BLOCK_8X8) + compmode_cost = av1_cost_bit(comp_mode_p, comp_pred); +#endif // SUB8X8_COMP_REF + + if (cm->reference_mode == REFERENCE_MODE_SELECT) rate2 += compmode_cost; + } + +#if CONFIG_EXT_INTER + rate2 += compmode_interintra_cost; + if (cm->reference_mode != SINGLE_REFERENCE && comp_pred) +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + if (mbmi->motion_mode == SIMPLE_TRANSLATION) +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + rate2 += compmode_interinter_cost; +#endif // CONFIG_EXT_INTER + + // Estimate the reference frame signaling cost and add it + // to the rolling cost variable. + if (comp_pred) { + rate2 += ref_costs_comp[ref_frame]; +#if CONFIG_EXT_REFS + rate2 += ref_costs_comp[second_ref_frame]; +#endif // CONFIG_EXT_REFS + } else { + rate2 += ref_costs_single[ref_frame]; + } + +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + if (ref_frame == INTRA_FRAME) { +#else + if (!disable_skip) { +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + if (skippable) { + // Back out the coefficient coding costs + rate2 -= (rate_y + rate_uv); + rate_y = 0; + rate_uv = 0; + // Cost the skip mb case + rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + } else if (ref_frame != INTRA_FRAME && !xd->lossless[mbmi->segment_id]) { +#if CONFIG_REF_MV + if (RDCOST(x->rdmult, x->rddiv, rate_y + rate_uv + rate_skip0, + distortion2) < + RDCOST(x->rdmult, x->rddiv, rate_skip1, total_sse)) { +#else + if (RDCOST(x->rdmult, x->rddiv, rate_y + rate_uv, distortion2) < + RDCOST(x->rdmult, x->rddiv, 0, total_sse)) { +#endif // CONFIG_REF_MV + // Add in the cost of the no skip flag. + rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 0); + } else { + // FIXME(rbultje) make this work for splitmv also + rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + distortion2 = total_sse; + assert(total_sse >= 0); + rate2 -= (rate_y + rate_uv); + this_skip2 = 1; + rate_y = 0; + rate_uv = 0; + } + } else { + // Add in the cost of the no skip flag. + rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 0); + } + + // Calculate the final RD estimate for this mode. + this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2); +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + } else { + this_skip2 = mbmi->skip; + this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2); + if (this_skip2) { + rate_y = 0; + rate_uv = 0; + } +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION + } + + if (ref_frame == INTRA_FRAME) { + // Keep record of best intra rd + if (this_rd < best_intra_rd) { + best_intra_rd = this_rd; + best_intra_mode = mbmi->mode; + } +#if CONFIG_EXT_INTER + } else if (second_ref_frame == NONE_FRAME) { + if (this_rd < best_single_inter_rd) { + best_single_inter_rd = this_rd; + best_single_inter_ref = mbmi->ref_frame[0]; + } +#endif // CONFIG_EXT_INTER + } + + if (!disable_skip && ref_frame == INTRA_FRAME) { + for (i = 0; i < REFERENCE_MODES; ++i) + best_pred_rd[i] = AOMMIN(best_pred_rd[i], this_rd); + } + + // Did this mode help.. i.e. is it the new best mode + if (this_rd < best_rd || x->skip) { + if (!mode_excluded) { + // Note index of best mode so far + best_mode_index = mode_index; + + if (ref_frame == INTRA_FRAME) { + /* required for left and above block mv */ + mbmi->mv[0].as_int = 0; + } else { + best_pred_sse = x->pred_sse[ref_frame]; + } + + rd_cost->rate = rate2; +#if CONFIG_SUPERTX + if (x->skip) + *returnrate_nocoef = rate2; + else + *returnrate_nocoef = rate2 - rate_y - rate_uv; + *returnrate_nocoef -= av1_cost_bit( + av1_get_skip_prob(cm, xd), disable_skip || skippable || this_skip2); + *returnrate_nocoef -= av1_cost_bit(av1_get_intra_inter_prob(cm, xd), + mbmi->ref_frame[0] != INTRA_FRAME); +#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION + MODE_INFO *const mi = xd->mi[0]; + const MOTION_MODE motion_allowed = motion_mode_allowed( +#if CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + 0, xd->global_motion, +#endif // CONFIG_GLOBAL_MOTION && SEPARATE_GLOBAL_MOTION + mi); + if (motion_allowed == WARPED_CAUSAL) + *returnrate_nocoef -= cpi->motion_mode_cost[bsize][mbmi->motion_mode]; + else if (motion_allowed == OBMC_CAUSAL) + *returnrate_nocoef -= + cpi->motion_mode_cost1[bsize][mbmi->motion_mode]; +#else + *returnrate_nocoef -= cpi->motion_mode_cost[bsize][mbmi->motion_mode]; +#endif // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION +#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION +#endif // CONFIG_SUPERTX + rd_cost->dist = distortion2; + rd_cost->rdcost = this_rd; + best_rd = this_rd; + best_mbmode = *mbmi; + best_skip2 = this_skip2; + best_mode_skippable = skippable; + best_rate_y = rate_y + av1_cost_bit(av1_get_skip_prob(cm, xd), + this_skip2 || skippable); + best_rate_uv = rate_uv; + +#if CONFIG_VAR_TX + for (i = 0; i < MAX_MB_PLANE; ++i) + memcpy(ctx->blk_skip[i], x->blk_skip[i], + sizeof(uint8_t) * ctx->num_4x4_blk); +#endif // CONFIG_VAR_TX + } + } + + /* keep record of best compound/single-only prediction */ + if (!disable_skip && ref_frame != INTRA_FRAME) { + int64_t single_rd, hybrid_rd, single_rate, hybrid_rate; + + if (cm->reference_mode == REFERENCE_MODE_SELECT) { + single_rate = rate2 - compmode_cost; + hybrid_rate = rate2; + } else { + single_rate = rate2; + hybrid_rate = rate2 + compmode_cost; + } + + single_rd = RDCOST(x->rdmult, x->rddiv, single_rate, distortion2); + hybrid_rd = RDCOST(x->rdmult, x->rddiv, hybrid_rate, distortion2); + + if (!comp_pred) { + if (single_rd < best_pred_rd[SINGLE_REFERENCE]) + best_pred_rd[SINGLE_REFERENCE] = single_rd; + } else { + if (single_rd < best_pred_rd[COMPOUND_REFERENCE]) + best_pred_rd[COMPOUND_REFERENCE] = single_rd; + } + if (hybrid_rd < best_pred_rd[REFERENCE_MODE_SELECT]) + best_pred_rd[REFERENCE_MODE_SELECT] = hybrid_rd; + } + + if (x->skip && !comp_pred) break; + } + + if (xd->lossless[mbmi->segment_id] == 0 && best_mode_index >= 0 && + ((sf->tx_type_search.fast_inter_tx_type_search == 1 && + is_inter_mode(best_mbmode.mode)) || + (sf->tx_type_search.fast_intra_tx_type_search == 1 && + !is_inter_mode(best_mbmode.mode)))) { + int skip_blk = 0; + RD_STATS rd_stats_y, rd_stats_uv; + + x->use_default_inter_tx_type = 0; + x->use_default_intra_tx_type = 0; + + *mbmi = best_mbmode; + + set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]); + + // Select prediction reference frames. + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].pre[0] = yv12_mb[mbmi->ref_frame[0]][i]; + if (has_second_ref(mbmi)) + xd->plane[i].pre[1] = yv12_mb[mbmi->ref_frame[1]][i]; + } + + if (is_inter_mode(mbmi->mode)) { + av1_build_inter_predictors_sb(xd, mi_row, mi_col, NULL, bsize); +#if CONFIG_MOTION_VAR + if (mbmi->motion_mode == OBMC_CAUSAL) { + av1_build_obmc_inter_prediction( + cm, xd, mi_row, mi_col, args.above_pred_buf, args.above_pred_stride, + args.left_pred_buf, args.left_pred_stride); + } +#endif // CONFIG_MOTION_VAR + av1_subtract_plane(x, bsize, 0); +#if CONFIG_VAR_TX + if (cm->tx_mode == TX_MODE_SELECT || xd->lossless[mbmi->segment_id]) { + select_tx_type_yrd(cpi, x, &rd_stats_y, bsize, INT64_MAX); + } else { + int idx, idy; + super_block_yrd(cpi, x, &rd_stats_y, bsize, INT64_MAX); + for (idy = 0; idy < xd->n8_h; ++idy) + for (idx = 0; idx < xd->n8_w; ++idx) + mbmi->inter_tx_size[idy][idx] = mbmi->tx_size; + memset(x->blk_skip[0], rd_stats_y.skip, + sizeof(uint8_t) * xd->n8_h * xd->n8_w * 4); + } + + inter_block_uvrd(cpi, x, &rd_stats_uv, bsize, INT64_MAX); +#else + super_block_yrd(cpi, x, &rd_stats_y, bsize, INT64_MAX); + super_block_uvrd(cpi, x, &rd_stats_uv, bsize, INT64_MAX); +#endif // CONFIG_VAR_TX + } else { + super_block_yrd(cpi, x, &rd_stats_y, bsize, INT64_MAX); + super_block_uvrd(cpi, x, &rd_stats_uv, bsize, INT64_MAX); + } + + if (RDCOST(x->rdmult, x->rddiv, rd_stats_y.rate + rd_stats_uv.rate, + (rd_stats_y.dist + rd_stats_uv.dist)) > + RDCOST(x->rdmult, x->rddiv, 0, (rd_stats_y.sse + rd_stats_uv.sse))) { + skip_blk = 1; + rd_stats_y.rate = av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + rd_stats_uv.rate = 0; + rd_stats_y.dist = rd_stats_y.sse; + rd_stats_uv.dist = rd_stats_uv.sse; + } else { + skip_blk = 0; + rd_stats_y.rate += av1_cost_bit(av1_get_skip_prob(cm, xd), 0); + } + + if (RDCOST(x->rdmult, x->rddiv, best_rate_y + best_rate_uv, rd_cost->dist) > + RDCOST(x->rdmult, x->rddiv, rd_stats_y.rate + rd_stats_uv.rate, + (rd_stats_y.dist + rd_stats_uv.dist))) { +#if CONFIG_VAR_TX + int idx, idy; +#endif // CONFIG_VAR_TX + best_mbmode.tx_type = mbmi->tx_type; + best_mbmode.tx_size = mbmi->tx_size; +#if CONFIG_VAR_TX + for (idy = 0; idy < xd->n8_h; ++idy) + for (idx = 0; idx < xd->n8_w; ++idx) + best_mbmode.inter_tx_size[idy][idx] = mbmi->inter_tx_size[idy][idx]; + + for (i = 0; i < MAX_MB_PLANE; ++i) + memcpy(ctx->blk_skip[i], x->blk_skip[i], + sizeof(uint8_t) * ctx->num_4x4_blk); + + best_mbmode.min_tx_size = mbmi->min_tx_size; +#endif // CONFIG_VAR_TX + rd_cost->rate += + (rd_stats_y.rate + rd_stats_uv.rate - best_rate_y - best_rate_uv); + rd_cost->dist = rd_stats_y.dist + rd_stats_uv.dist; + rd_cost->rdcost = + RDCOST(x->rdmult, x->rddiv, rd_cost->rate, rd_cost->dist); + best_skip2 = skip_blk; + } + } + +#if CONFIG_PALETTE + // Only try palette mode when the best mode so far is an intra mode. + if (try_palette && !is_inter_mode(best_mbmode.mode)) { + int rate2 = 0; +#if CONFIG_SUPERTX + int best_rate_nocoef; +#endif // CONFIG_SUPERTX + int64_t distortion2 = 0, best_rd_palette = best_rd, this_rd, + best_model_rd_palette = INT64_MAX; + int skippable = 0, rate_overhead_palette = 0; + RD_STATS rd_stats_y; + TX_SIZE uv_tx; + uint8_t *const best_palette_color_map = + x->palette_buffer->best_palette_color_map; + uint8_t *const color_map = xd->plane[0].color_index_map; + MB_MODE_INFO best_mbmi_palette = best_mbmode; + + mbmi->mode = DC_PRED; + mbmi->uv_mode = DC_PRED; + mbmi->ref_frame[0] = INTRA_FRAME; + mbmi->ref_frame[1] = NONE_FRAME; + rate_overhead_palette = rd_pick_palette_intra_sby( + cpi, x, bsize, palette_ctx, intra_mode_cost[DC_PRED], + &best_mbmi_palette, best_palette_color_map, &best_rd_palette, + &best_model_rd_palette, NULL, NULL, NULL, NULL); + if (pmi->palette_size[0] == 0) goto PALETTE_EXIT; + memcpy(color_map, best_palette_color_map, + rows * cols * sizeof(best_palette_color_map[0])); + super_block_yrd(cpi, x, &rd_stats_y, bsize, best_rd); + if (rd_stats_y.rate == INT_MAX) goto PALETTE_EXIT; + uv_tx = uv_txsize_lookup[bsize][mbmi->tx_size][xd->plane[1].subsampling_x] + [xd->plane[1].subsampling_y]; + if (rate_uv_intra[uv_tx] == INT_MAX) { + choose_intra_uv_mode(cpi, x, ctx, bsize, uv_tx, &rate_uv_intra[uv_tx], + &rate_uv_tokenonly[uv_tx], &dist_uvs[uv_tx], + &skip_uvs[uv_tx], &mode_uv[uv_tx]); + pmi_uv[uv_tx] = *pmi; +#if CONFIG_EXT_INTRA + uv_angle_delta[uv_tx] = mbmi->angle_delta[1]; +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + filter_intra_mode_info_uv[uv_tx] = mbmi->filter_intra_mode_info; +#endif // CONFIG_FILTER_INTRA + } + mbmi->uv_mode = mode_uv[uv_tx]; + pmi->palette_size[1] = pmi_uv[uv_tx].palette_size[1]; + if (pmi->palette_size[1] > 0) { + memcpy(pmi->palette_colors + PALETTE_MAX_SIZE, + pmi_uv[uv_tx].palette_colors + PALETTE_MAX_SIZE, + 2 * PALETTE_MAX_SIZE * sizeof(pmi->palette_colors[0])); + } +#if CONFIG_EXT_INTRA + mbmi->angle_delta[1] = uv_angle_delta[uv_tx]; +#endif // CONFIG_EXT_INTRA +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = + filter_intra_mode_info_uv[uv_tx].use_filter_intra_mode[1]; + if (filter_intra_mode_info_uv[uv_tx].use_filter_intra_mode[1]) { + mbmi->filter_intra_mode_info.filter_intra_mode[1] = + filter_intra_mode_info_uv[uv_tx].filter_intra_mode[1]; + } +#endif // CONFIG_FILTER_INTRA + skippable = rd_stats_y.skip && skip_uvs[uv_tx]; + distortion2 = rd_stats_y.dist + dist_uvs[uv_tx]; + rate2 = rd_stats_y.rate + rate_overhead_palette + rate_uv_intra[uv_tx]; + rate2 += ref_costs_single[INTRA_FRAME]; + + if (skippable) { + rate2 -= (rd_stats_y.rate + rate_uv_tokenonly[uv_tx]); +#if CONFIG_SUPERTX + best_rate_nocoef = rate2; +#endif // CONFIG_SUPERTX + rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + } else { +#if CONFIG_SUPERTX + best_rate_nocoef = rate2 - (rd_stats_y.rate + rate_uv_tokenonly[uv_tx]); +#endif // CONFIG_SUPERTX + rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 0); + } + this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2); + if (this_rd < best_rd) { + best_mode_index = 3; + mbmi->mv[0].as_int = 0; + rd_cost->rate = rate2; +#if CONFIG_SUPERTX + *returnrate_nocoef = best_rate_nocoef; +#endif // CONFIG_SUPERTX + rd_cost->dist = distortion2; + rd_cost->rdcost = this_rd; + best_rd = this_rd; + best_mbmode = *mbmi; + best_skip2 = 0; + best_mode_skippable = skippable; + } + } +PALETTE_EXIT: +#endif // CONFIG_PALETTE + +#if CONFIG_FILTER_INTRA + // TODO(huisu): filter-intra is turned off in lossless mode for now to + // avoid a unit test failure + if (!xd->lossless[mbmi->segment_id] && +#if CONFIG_PALETTE + pmi->palette_size[0] == 0 && +#endif // CONFIG_PALETTE + !dc_skipped && best_mode_index >= 0 && + best_intra_rd < (best_rd + (best_rd >> 3))) { + pick_filter_intra_interframe( + cpi, x, ctx, bsize, mi_row, mi_col, rate_uv_intra, rate_uv_tokenonly, + dist_uvs, skip_uvs, mode_uv, filter_intra_mode_info_uv, +#if CONFIG_EXT_INTRA + uv_angle_delta, +#endif // CONFIG_EXT_INTRA +#if CONFIG_PALETTE + pmi_uv, palette_ctx, +#endif // CONFIG_PALETTE + 0, ref_costs_single, &best_rd, &best_intra_rd, &best_intra_mode, + &best_mode_index, &best_skip2, &best_mode_skippable, +#if CONFIG_SUPERTX + returnrate_nocoef, +#endif // CONFIG_SUPERTX + best_pred_rd, &best_mbmode, rd_cost); + } +#endif // CONFIG_FILTER_INTRA + + // The inter modes' rate costs are not calculated precisely in some cases. + // Therefore, sometimes, NEWMV is chosen instead of NEARESTMV, NEARMV, and + // ZEROMV. Here, checks are added for those cases, and the mode decisions + // are corrected. + if (best_mbmode.mode == NEWMV +#if CONFIG_EXT_INTER + || best_mbmode.mode == NEW_NEWMV +#endif // CONFIG_EXT_INTER + ) { + const MV_REFERENCE_FRAME refs[2] = { best_mbmode.ref_frame[0], + best_mbmode.ref_frame[1] }; + int comp_pred_mode = refs[1] > INTRA_FRAME; + int_mv zeromv[2]; +#if CONFIG_REF_MV + const uint8_t rf_type = av1_ref_frame_type(best_mbmode.ref_frame); +#endif // CONFIG_REF_MV +#if CONFIG_GLOBAL_MOTION + zeromv[0].as_int = gm_get_motion_vector(&cm->global_motion[refs[0]], + cm->allow_high_precision_mv, bsize, + mi_col, mi_row, 0) + .as_int; + zeromv[1].as_int = comp_pred_mode + ? gm_get_motion_vector(&cm->global_motion[refs[1]], + cm->allow_high_precision_mv, + bsize, mi_col, mi_row, 0) + .as_int + : 0; +#else + zeromv[0].as_int = 0; + zeromv[1].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION +#if CONFIG_REF_MV + if (!comp_pred_mode) { + int ref_set = (mbmi_ext->ref_mv_count[rf_type] >= 2) + ? AOMMIN(2, mbmi_ext->ref_mv_count[rf_type] - 2) + : INT_MAX; + + for (i = 0; i <= ref_set && ref_set != INT_MAX; ++i) { + int_mv cur_mv = mbmi_ext->ref_mv_stack[rf_type][i + 1].this_mv; + if (cur_mv.as_int == best_mbmode.mv[0].as_int) { + best_mbmode.mode = NEARMV; + best_mbmode.ref_mv_idx = i; + } + } + + if (frame_mv[NEARESTMV][refs[0]].as_int == best_mbmode.mv[0].as_int) + best_mbmode.mode = NEARESTMV; + else if (best_mbmode.mv[0].as_int == zeromv[0].as_int) + best_mbmode.mode = ZEROMV; + } else { + int_mv nearestmv[2]; + int_mv nearmv[2]; + +#if CONFIG_EXT_INTER + if (mbmi_ext->ref_mv_count[rf_type] > 1) { + nearmv[0] = mbmi_ext->ref_mv_stack[rf_type][1].this_mv; + nearmv[1] = mbmi_ext->ref_mv_stack[rf_type][1].comp_mv; + } else { + nearmv[0] = frame_mv[NEARMV][refs[0]]; + nearmv[1] = frame_mv[NEARMV][refs[1]]; + } +#else + int ref_set = (mbmi_ext->ref_mv_count[rf_type] >= 2) + ? AOMMIN(2, mbmi_ext->ref_mv_count[rf_type] - 2) + : INT_MAX; + + for (i = 0; i <= ref_set && ref_set != INT_MAX; ++i) { + nearmv[0] = mbmi_ext->ref_mv_stack[rf_type][i + 1].this_mv; + nearmv[1] = mbmi_ext->ref_mv_stack[rf_type][i + 1].comp_mv; + + if (nearmv[0].as_int == best_mbmode.mv[0].as_int && + nearmv[1].as_int == best_mbmode.mv[1].as_int) { + best_mbmode.mode = NEARMV; + best_mbmode.ref_mv_idx = i; + } + } +#endif // CONFIG_EXT_INTER + if (mbmi_ext->ref_mv_count[rf_type] >= 1) { + nearestmv[0] = mbmi_ext->ref_mv_stack[rf_type][0].this_mv; + nearestmv[1] = mbmi_ext->ref_mv_stack[rf_type][0].comp_mv; + } else { + nearestmv[0] = frame_mv[NEARESTMV][refs[0]]; + nearestmv[1] = frame_mv[NEARESTMV][refs[1]]; + } + + if (nearestmv[0].as_int == best_mbmode.mv[0].as_int && + nearestmv[1].as_int == best_mbmode.mv[1].as_int) { +#if CONFIG_EXT_INTER + best_mbmode.mode = NEAREST_NEARESTMV; + } else { + int ref_set = (mbmi_ext->ref_mv_count[rf_type] >= 2) + ? AOMMIN(2, mbmi_ext->ref_mv_count[rf_type] - 2) + : INT_MAX; + + for (i = 0; i <= ref_set && ref_set != INT_MAX; ++i) { + nearmv[0] = mbmi_ext->ref_mv_stack[rf_type][i + 1].this_mv; + nearmv[1] = mbmi_ext->ref_mv_stack[rf_type][i + 1].comp_mv; + + // Try switching to the NEAR_NEAREST type modes first + if (nearestmv[0].as_int == best_mbmode.mv[0].as_int && + nearmv[1].as_int == best_mbmode.mv[1].as_int) { + best_mbmode.mode = NEAREST_NEARMV; + best_mbmode.ref_mv_idx = i; + } else if (nearmv[0].as_int == best_mbmode.mv[0].as_int && + nearestmv[1].as_int == best_mbmode.mv[1].as_int) { + best_mbmode.mode = NEAR_NEARESTMV; + best_mbmode.ref_mv_idx = i; + } else if (nearmv[0].as_int == best_mbmode.mv[0].as_int && + nearmv[1].as_int == best_mbmode.mv[1].as_int) { + best_mbmode.mode = NEAR_NEARMV; + best_mbmode.ref_mv_idx = i; + } + } + + if (best_mbmode.mode == NEW_NEWMV && + best_mbmode.mv[0].as_int == zeromv[0].as_int && + best_mbmode.mv[1].as_int == zeromv[1].as_int) + best_mbmode.mode = ZERO_ZEROMV; + } +#else + best_mbmode.mode = NEARESTMV; + } else if (best_mbmode.mv[0].as_int == zeromv[0].as_int && + best_mbmode.mv[1].as_int == zeromv[1].as_int) { + best_mbmode.mode = ZEROMV; + } +#endif // CONFIG_EXT_INTER + } +#else +#if CONFIG_EXT_INTER + if (!comp_pred_mode) { +#endif // CONFIG_EXT_INTER + if (frame_mv[NEARESTMV][refs[0]].as_int == best_mbmode.mv[0].as_int && + ((comp_pred_mode && + frame_mv[NEARESTMV][refs[1]].as_int == best_mbmode.mv[1].as_int) || + !comp_pred_mode)) + best_mbmode.mode = NEARESTMV; + else if (frame_mv[NEARMV][refs[0]].as_int == best_mbmode.mv[0].as_int && + ((comp_pred_mode && + frame_mv[NEARMV][refs[1]].as_int == + best_mbmode.mv[1].as_int) || + !comp_pred_mode)) + best_mbmode.mode = NEARMV; + else if (best_mbmode.mv[0].as_int == zeromv[0].as_int && + ((comp_pred_mode && + best_mbmode.mv[1].as_int == zeromv[1].as_int) || + !comp_pred_mode)) + best_mbmode.mode = ZEROMV; +#if CONFIG_EXT_INTER + } else { +#if CONFIG_GLOBAL_MOTION + zeromv[0].as_int = gm_get_motion_vector(&cm->global_motion[refs[0]], + cm->allow_high_precision_mv, + bsize, mi_col, mi_row, 0) + .as_int; + zeromv[1].as_int = comp_pred_mode + ? gm_get_motion_vector(&cm->global_motion[refs[1]], + cm->allow_high_precision_mv, + bsize, mi_col, mi_row, 0) + .as_int + : 0; +#else + zeromv[0].as_int = 0; + zeromv[1].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION + if (frame_mv[NEAREST_NEARESTMV][refs[0]].as_int == + best_mbmode.mv[0].as_int && + frame_mv[NEAREST_NEARESTMV][refs[1]].as_int == + best_mbmode.mv[1].as_int) + best_mbmode.mode = NEAREST_NEARESTMV; + else if (frame_mv[NEAREST_NEARMV][refs[0]].as_int == + best_mbmode.mv[0].as_int && + frame_mv[NEAREST_NEARMV][refs[1]].as_int == + best_mbmode.mv[1].as_int) + best_mbmode.mode = NEAREST_NEARMV; + else if (frame_mv[NEAR_NEARESTMV][refs[0]].as_int == + best_mbmode.mv[0].as_int && + frame_mv[NEAR_NEARESTMV][refs[1]].as_int == + best_mbmode.mv[1].as_int) + best_mbmode.mode = NEAR_NEARESTMV; + else if (frame_mv[NEAR_NEARMV][refs[0]].as_int == + best_mbmode.mv[0].as_int && + frame_mv[NEAR_NEARMV][refs[1]].as_int == + best_mbmode.mv[1].as_int) + best_mbmode.mode = NEAR_NEARMV; + else if (best_mbmode.mv[0].as_int == zeromv[0].as_int && + best_mbmode.mv[1].as_int == zeromv[1].as_int) + best_mbmode.mode = ZERO_ZEROMV; + } +#endif // CONFIG_EXT_INTER +#endif // CONFIG_REF_MV + } + +#if CONFIG_REF_MV + // Make sure that the ref_mv_idx is only nonzero when we're + // using a mode which can support ref_mv_idx + if (best_mbmode.ref_mv_idx != 0 && +#if CONFIG_EXT_INTER + !(best_mbmode.mode == NEWMV || best_mbmode.mode == NEW_NEWMV || + have_nearmv_in_inter_mode(best_mbmode.mode))) { +#else + !(best_mbmode.mode == NEARMV || best_mbmode.mode == NEWMV)) { +#endif + best_mbmode.ref_mv_idx = 0; + } + + { + int8_t ref_frame_type = av1_ref_frame_type(best_mbmode.ref_frame); + int16_t mode_ctx = mbmi_ext->mode_context[ref_frame_type]; + if (mode_ctx & (1 << ALL_ZERO_FLAG_OFFSET)) { + int_mv zeromv[2]; +#if CONFIG_GLOBAL_MOTION + const MV_REFERENCE_FRAME refs[2] = { best_mbmode.ref_frame[0], + best_mbmode.ref_frame[1] }; + zeromv[0].as_int = gm_get_motion_vector(&cm->global_motion[refs[0]], + cm->allow_high_precision_mv, + bsize, mi_col, mi_row, 0) + .as_int; + zeromv[1].as_int = (refs[1] != NONE_FRAME) + ? gm_get_motion_vector(&cm->global_motion[refs[1]], + cm->allow_high_precision_mv, + bsize, mi_col, mi_row, 0) + .as_int + : 0; + lower_mv_precision(&zeromv[0].as_mv, cm->allow_high_precision_mv); + lower_mv_precision(&zeromv[1].as_mv, cm->allow_high_precision_mv); +#else + zeromv[0].as_int = zeromv[1].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION + if (best_mbmode.ref_frame[0] > INTRA_FRAME && + best_mbmode.mv[0].as_int == zeromv[0].as_int && +#if CONFIG_EXT_INTER + (best_mbmode.ref_frame[1] <= INTRA_FRAME) +#else + (best_mbmode.ref_frame[1] == NONE_FRAME || + best_mbmode.mv[1].as_int == zeromv[1].as_int) +#endif // CONFIG_EXT_INTER + ) { + best_mbmode.mode = ZEROMV; + } + } + } +#endif // CONFIG_REF_MV + + if (best_mode_index < 0 || best_rd >= best_rd_so_far) { + rd_cost->rate = INT_MAX; + rd_cost->rdcost = INT64_MAX; + return; + } + +#if CONFIG_DUAL_FILTER + assert((cm->interp_filter == SWITCHABLE) || + (cm->interp_filter == best_mbmode.interp_filter[0]) || + !is_inter_block(&best_mbmode)); + assert((cm->interp_filter == SWITCHABLE) || + (cm->interp_filter == best_mbmode.interp_filter[1]) || + !is_inter_block(&best_mbmode)); + if (best_mbmode.ref_frame[1] > INTRA_FRAME) { + assert((cm->interp_filter == SWITCHABLE) || + (cm->interp_filter == best_mbmode.interp_filter[2]) || + !is_inter_block(&best_mbmode)); + assert((cm->interp_filter == SWITCHABLE) || + (cm->interp_filter == best_mbmode.interp_filter[3]) || + !is_inter_block(&best_mbmode)); + } +#else + assert((cm->interp_filter == SWITCHABLE) || + (cm->interp_filter == best_mbmode.interp_filter) || + !is_inter_block(&best_mbmode)); +#endif // CONFIG_DUAL_FILTER + + if (!cpi->rc.is_src_frame_alt_ref) + av1_update_rd_thresh_fact(cm, tile_data->thresh_freq_fact, + sf->adaptive_rd_thresh, bsize, best_mode_index); + + // macroblock modes + *mbmi = best_mbmode; + x->skip |= best_skip2; + +// Note: this section is needed since the mode may have been forced to +// ZEROMV by the all-zero mode handling of ref-mv. +#if CONFIG_GLOBAL_MOTION + if (mbmi->mode == ZEROMV +#if CONFIG_EXT_INTER + || mbmi->mode == ZERO_ZEROMV +#endif // CONFIG_EXT_INTER + ) { +#if CONFIG_WARPED_MOTION || CONFIG_MOTION_VAR + // Correct the motion mode for ZEROMV + const MOTION_MODE last_motion_mode_allowed = motion_mode_allowed( +#if SEPARATE_GLOBAL_MOTION + 0, xd->global_motion, +#endif // SEPARATE_GLOBAL_MOTION + xd->mi[0]); + if (mbmi->motion_mode > last_motion_mode_allowed) + mbmi->motion_mode = last_motion_mode_allowed; +#endif // CONFIG_WARPED_MOTION || CONFIG_MOTION_VAR + + // Correct the interpolation filter for ZEROMV + if (is_nontrans_global_motion(xd)) { +#if CONFIG_DUAL_FILTER + mbmi->interp_filter[0] = cm->interp_filter == SWITCHABLE + ? EIGHTTAP_REGULAR + : cm->interp_filter; + mbmi->interp_filter[1] = cm->interp_filter == SWITCHABLE + ? EIGHTTAP_REGULAR + : cm->interp_filter; +#else + mbmi->interp_filter = cm->interp_filter == SWITCHABLE ? EIGHTTAP_REGULAR + : cm->interp_filter; +#endif // CONFIG_DUAL_FILTER + } + } +#endif // CONFIG_GLOBAL_MOTION + +#if CONFIG_REF_MV + for (i = 0; i < 1 + has_second_ref(mbmi); ++i) { + if (mbmi->mode != NEWMV) + mbmi->pred_mv[i].as_int = mbmi->mv[i].as_int; + else + mbmi->pred_mv[i].as_int = mbmi_ext->ref_mvs[mbmi->ref_frame[i]][0].as_int; + } +#endif // CONFIG_REF_MV + + for (i = 0; i < REFERENCE_MODES; ++i) { + if (best_pred_rd[i] == INT64_MAX) + best_pred_diff[i] = INT_MIN; + else + best_pred_diff[i] = best_rd - best_pred_rd[i]; + } + + x->skip |= best_mode_skippable; + + assert(best_mode_index >= 0); + + store_coding_context(x, ctx, best_mode_index, best_pred_diff, + best_mode_skippable); + +#if CONFIG_PALETTE + if (cm->allow_screen_content_tools && pmi->palette_size[1] > 0) { + restore_uv_color_map(cpi, x); + } +#endif // CONFIG_PALETTE +} + +void av1_rd_pick_inter_mode_sb_seg_skip(const AV1_COMP *cpi, + TileDataEnc *tile_data, MACROBLOCK *x, + int mi_row, int mi_col, + RD_STATS *rd_cost, BLOCK_SIZE bsize, + PICK_MODE_CONTEXT *ctx, + int64_t best_rd_so_far) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + unsigned char segment_id = mbmi->segment_id; + const int comp_pred = 0; + int i; + int64_t best_pred_diff[REFERENCE_MODES]; + unsigned int ref_costs_single[TOTAL_REFS_PER_FRAME]; + unsigned int ref_costs_comp[TOTAL_REFS_PER_FRAME]; + aom_prob comp_mode_p; + InterpFilter best_filter = SWITCHABLE; + int64_t this_rd = INT64_MAX; + int rate2 = 0; + const int64_t distortion2 = 0; + (void)mi_row; + (void)mi_col; + + estimate_ref_frame_costs(cm, xd, segment_id, ref_costs_single, ref_costs_comp, + &comp_mode_p); + + for (i = 0; i < TOTAL_REFS_PER_FRAME; ++i) x->pred_sse[i] = INT_MAX; + for (i = LAST_FRAME; i < TOTAL_REFS_PER_FRAME; ++i) + x->pred_mv_sad[i] = INT_MAX; + + rd_cost->rate = INT_MAX; + + assert(segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)); + +#if CONFIG_PALETTE + mbmi->palette_mode_info.palette_size[0] = 0; + mbmi->palette_mode_info.palette_size[1] = 0; +#endif // CONFIG_PALETTE + +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0; + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = 0; +#endif // CONFIG_FILTER_INTRA + mbmi->mode = ZEROMV; + mbmi->motion_mode = SIMPLE_TRANSLATION; + mbmi->uv_mode = DC_PRED; + mbmi->ref_frame[0] = LAST_FRAME; + mbmi->ref_frame[1] = NONE_FRAME; +#if CONFIG_GLOBAL_MOTION + mbmi->mv[0].as_int = + gm_get_motion_vector(&cm->global_motion[mbmi->ref_frame[0]], + cm->allow_high_precision_mv, bsize, mi_col, mi_row, + 0) + .as_int; +#else // CONFIG_GLOBAL_MOTION + mbmi->mv[0].as_int = 0; +#endif // CONFIG_GLOBAL_MOTION + mbmi->tx_size = max_txsize_lookup[bsize]; + x->skip = 1; + +#if CONFIG_REF_MV + mbmi->ref_mv_idx = 0; + mbmi->pred_mv[0].as_int = 0; +#endif // CONFIG_REF_MV + + mbmi->motion_mode = SIMPLE_TRANSLATION; +#if CONFIG_MOTION_VAR + av1_count_overlappable_neighbors(cm, xd, mi_row, mi_col); +#endif +#if CONFIG_WARPED_MOTION + if (is_motion_variation_allowed_bsize(bsize) && !has_second_ref(mbmi)) { + int pts[SAMPLES_ARRAY_SIZE], pts_inref[SAMPLES_ARRAY_SIZE]; + mbmi->num_proj_ref[0] = findSamples(cm, xd, mi_row, mi_col, pts, pts_inref); + } +#endif + + set_default_interp_filters(mbmi, cm->interp_filter); + + if (cm->interp_filter != SWITCHABLE) { + best_filter = cm->interp_filter; + } else { + best_filter = EIGHTTAP_REGULAR; + if (av1_is_interp_needed(xd) && av1_is_interp_search_needed(xd) && + x->source_variance >= cpi->sf.disable_filter_search_var_thresh) { + int rs; + int best_rs = INT_MAX; + for (i = 0; i < SWITCHABLE_FILTERS; ++i) { +#if CONFIG_DUAL_FILTER + int k; + for (k = 0; k < 4; ++k) mbmi->interp_filter[k] = i; +#else + mbmi->interp_filter = i; +#endif // CONFIG_DUAL_FILTER + rs = av1_get_switchable_rate(cpi, xd); + if (rs < best_rs) { + best_rs = rs; +#if CONFIG_DUAL_FILTER + best_filter = mbmi->interp_filter[0]; +#else + best_filter = mbmi->interp_filter; +#endif // CONFIG_DUAL_FILTER + } + } + } + } +// Set the appropriate filter +#if CONFIG_DUAL_FILTER + for (i = 0; i < 4; ++i) mbmi->interp_filter[i] = best_filter; +#else + mbmi->interp_filter = best_filter; +#endif // CONFIG_DUAL_FILTER + rate2 += av1_get_switchable_rate(cpi, xd); + + if (cm->reference_mode == REFERENCE_MODE_SELECT) + rate2 += av1_cost_bit(comp_mode_p, comp_pred); + + // Estimate the reference frame signaling cost and add it + // to the rolling cost variable. + rate2 += ref_costs_single[LAST_FRAME]; + this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2); + + rd_cost->rate = rate2; + rd_cost->dist = distortion2; + rd_cost->rdcost = this_rd; + + if (this_rd >= best_rd_so_far) { + rd_cost->rate = INT_MAX; + rd_cost->rdcost = INT64_MAX; + return; + } + +#if CONFIG_DUAL_FILTER + assert((cm->interp_filter == SWITCHABLE) || + (cm->interp_filter == mbmi->interp_filter[0])); +#else + assert((cm->interp_filter == SWITCHABLE) || + (cm->interp_filter == mbmi->interp_filter)); +#endif // CONFIG_DUAL_FILTER + + av1_update_rd_thresh_fact(cm, tile_data->thresh_freq_fact, + cpi->sf.adaptive_rd_thresh, bsize, THR_ZEROMV); + + av1_zero(best_pred_diff); + + store_coding_context(x, ctx, THR_ZEROMV, best_pred_diff, 0); +} + +void av1_rd_pick_inter_mode_sub8x8(const struct AV1_COMP *cpi, + TileDataEnc *tile_data, struct macroblock *x, + int mi_row, int mi_col, + struct RD_STATS *rd_cost, +#if CONFIG_SUPERTX + int *returnrate_nocoef, +#endif // CONFIG_SUPERTX + BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx, + int64_t best_rd_so_far) { + const AV1_COMMON *const cm = &cpi->common; + const RD_OPT *const rd_opt = &cpi->rd; + const SPEED_FEATURES *const sf = &cpi->sf; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const struct segmentation *const seg = &cm->seg; + MV_REFERENCE_FRAME ref_frame, second_ref_frame; + unsigned char segment_id = mbmi->segment_id; + int comp_pred, i; + int_mv frame_mv[MB_MODE_COUNT][TOTAL_REFS_PER_FRAME]; + struct buf_2d yv12_mb[TOTAL_REFS_PER_FRAME][MAX_MB_PLANE]; + static const int flag_list[TOTAL_REFS_PER_FRAME] = { + 0, + AOM_LAST_FLAG, +#if CONFIG_EXT_REFS + AOM_LAST2_FLAG, + AOM_LAST3_FLAG, +#endif // CONFIG_EXT_REFS + AOM_GOLD_FLAG, +#if CONFIG_EXT_REFS + AOM_BWD_FLAG, +#endif // CONFIG_EXT_REFS + AOM_ALT_FLAG + }; + int64_t best_rd = best_rd_so_far; + int64_t best_yrd = best_rd_so_far; // FIXME(rbultje) more precise + int64_t best_pred_diff[REFERENCE_MODES]; + int64_t best_pred_rd[REFERENCE_MODES]; + MB_MODE_INFO best_mbmode; + int ref_index, best_ref_index = 0; + unsigned int ref_costs_single[TOTAL_REFS_PER_FRAME]; + unsigned int ref_costs_comp[TOTAL_REFS_PER_FRAME]; + aom_prob comp_mode_p; +#if CONFIG_DUAL_FILTER + InterpFilter tmp_best_filter[4] = { 0 }; +#else + InterpFilter tmp_best_filter = SWITCHABLE; +#endif // CONFIG_DUAL_FILTER + int rate_uv_intra, rate_uv_tokenonly = INT_MAX; + int64_t dist_uv = INT64_MAX; + int skip_uv; + PREDICTION_MODE mode_uv = DC_PRED; + const int intra_cost_penalty = av1_get_intra_cost_penalty( + cm->base_qindex, cm->y_dc_delta_q, cm->bit_depth); + int_mv seg_mvs[4][TOTAL_REFS_PER_FRAME]; + b_mode_info best_bmodes[4]; + int best_skip2 = 0; + int ref_frame_skip_mask[2] = { 0 }; + int internal_active_edge = + av1_active_edge_sb(cpi, mi_row, mi_col) && av1_internal_image_edge(cpi); +#if CONFIG_PVQ + od_rollback_buffer pre_buf; + + od_encode_checkpoint(&x->daala_enc, &pre_buf); +#endif // CONFIG_PVQ + +#if CONFIG_SUPERTX + best_rd_so_far = INT64_MAX; + best_rd = best_rd_so_far; + best_yrd = best_rd_so_far; +#endif // CONFIG_SUPERTX + av1_zero(best_mbmode); + +#if CONFIG_FILTER_INTRA + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0; + mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = 0; +#endif // CONFIG_FILTER_INTRA + mbmi->motion_mode = SIMPLE_TRANSLATION; +#if CONFIG_EXT_INTER + mbmi->interinter_compound_type = COMPOUND_AVERAGE; + mbmi->use_wedge_interintra = 0; +#endif // CONFIG_EXT_INTER +#if CONFIG_WARPED_MOTION + mbmi->num_proj_ref[0] = 0; + mbmi->num_proj_ref[1] = 0; +#endif // CONFIG_WARPED_MOTION + + for (i = 0; i < 4; i++) { + int j; + for (j = 0; j < TOTAL_REFS_PER_FRAME; j++) + seg_mvs[i][j].as_int = INVALID_MV; + } + + estimate_ref_frame_costs(cm, xd, segment_id, ref_costs_single, ref_costs_comp, + &comp_mode_p); + + for (i = 0; i < REFERENCE_MODES; ++i) best_pred_rd[i] = INT64_MAX; + rate_uv_intra = INT_MAX; + + rd_cost->rate = INT_MAX; +#if CONFIG_SUPERTX + *returnrate_nocoef = INT_MAX; +#endif // CONFIG_SUPERTX + + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) { + x->mbmi_ext->mode_context[ref_frame] = 0; +#if CONFIG_REF_MV && CONFIG_EXT_INTER + x->mbmi_ext->compound_mode_context[ref_frame] = 0; +#endif // CONFIG_REF_MV && CONFIG_EXT_INTER + if (cpi->ref_frame_flags & flag_list[ref_frame]) { + setup_buffer_inter(cpi, x, ref_frame, bsize, mi_row, mi_col, + frame_mv[NEARESTMV], frame_mv[NEARMV], yv12_mb); + } else { + ref_frame_skip_mask[0] |= (1 << ref_frame); + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; + } + frame_mv[NEWMV][ref_frame].as_int = INVALID_MV; +#if CONFIG_EXT_INTER +#endif // CONFIG_EXT_INTER + frame_mv[ZEROMV][ref_frame].as_int = 0; + } + +#if CONFIG_PALETTE + mbmi->palette_mode_info.palette_size[0] = 0; + mbmi->palette_mode_info.palette_size[1] = 0; +#endif // CONFIG_PALETTE + + for (ref_index = 0; ref_index < MAX_REFS; ++ref_index) { + int mode_excluded = 0; + int64_t this_rd = INT64_MAX; + int disable_skip = 0; + int compmode_cost = 0; + int rate2 = 0, rate_y = 0, rate_uv = 0; + int64_t distortion2 = 0, distortion_y = 0, distortion_uv = 0; + int skippable = 0; + int this_skip2 = 0; + int64_t total_sse = INT_MAX; + +#if CONFIG_PVQ + od_encode_rollback(&x->daala_enc, &pre_buf); +#endif // CONFIG_PVQ + + ref_frame = av1_ref_order[ref_index].ref_frame[0]; + second_ref_frame = av1_ref_order[ref_index].ref_frame[1]; + +#if CONFIG_REF_MV + mbmi->ref_mv_idx = 0; +#endif // CONFIG_REF_MV + + // Look at the reference frame of the best mode so far and set the + // skip mask to look at a subset of the remaining modes. + if (ref_index > 2 && sf->mode_skip_start < MAX_MODES) { + if (ref_index == 3) { + switch (best_mbmode.ref_frame[0]) { + case INTRA_FRAME: break; + case LAST_FRAME: + ref_frame_skip_mask[0] |= (1 << GOLDEN_FRAME) | +#if CONFIG_EXT_REFS + (1 << LAST2_FRAME) | (1 << LAST3_FRAME) | + (1 << BWDREF_FRAME) | +#endif // CONFIG_EXT_REFS + (1 << ALTREF_FRAME); + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; + break; +#if CONFIG_EXT_REFS + case LAST2_FRAME: + ref_frame_skip_mask[0] |= (1 << LAST_FRAME) | (1 << LAST3_FRAME) | + (1 << GOLDEN_FRAME) | + (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME); + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; + break; + case LAST3_FRAME: + ref_frame_skip_mask[0] |= (1 << LAST_FRAME) | (1 << LAST2_FRAME) | + (1 << GOLDEN_FRAME) | + (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME); + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; + break; +#endif // CONFIG_EXT_REFS + case GOLDEN_FRAME: + ref_frame_skip_mask[0] |= (1 << LAST_FRAME) | +#if CONFIG_EXT_REFS + (1 << LAST2_FRAME) | (1 << LAST3_FRAME) | + (1 << BWDREF_FRAME) | +#endif // CONFIG_EXT_REFS + (1 << ALTREF_FRAME); + ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; + break; +#if CONFIG_EXT_REFS + case BWDREF_FRAME: + ref_frame_skip_mask[0] |= (1 << LAST_FRAME) | (1 << LAST2_FRAME) | + (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | + (1 << ALTREF_FRAME); + ref_frame_skip_mask[1] |= (1 << ALTREF_FRAME) | 0x01; + break; +#endif // CONFIG_EXT_REFS + case ALTREF_FRAME: + ref_frame_skip_mask[0] |= (1 << LAST_FRAME) | +#if CONFIG_EXT_REFS + (1 << LAST2_FRAME) | (1 << LAST3_FRAME) | + (1 << BWDREF_FRAME) | +#endif // CONFIG_EXT_REFS + (1 << GOLDEN_FRAME); +#if CONFIG_EXT_REFS + ref_frame_skip_mask[1] |= (1 << BWDREF_FRAME) | 0x01; +#endif // CONFIG_EXT_REFS + break; + case NONE_FRAME: + case TOTAL_REFS_PER_FRAME: + assert(0 && "Invalid Reference frame"); + break; + } + } + } + + if ((ref_frame_skip_mask[0] & (1 << ref_frame)) && + (ref_frame_skip_mask[1] & (1 << AOMMAX(0, second_ref_frame)))) + continue; + + // Test best rd so far against threshold for trying this mode. + if (!internal_active_edge && + rd_less_than_thresh(best_rd, + rd_opt->threshes[segment_id][bsize][ref_index], + tile_data->thresh_freq_fact[bsize][ref_index])) + continue; + + // This is only used in motion vector unit test. + if (cpi->oxcf.motion_vector_unit_test && ref_frame == INTRA_FRAME) continue; + +#if CONFIG_LOWDELAY_COMPOUND // Changes LL bitstream +#if CONFIG_EXT_REFS + if (cpi->oxcf.pass == 0) { + // Complexity-compression trade-offs + // if (ref_frame == ALTREF_FRAME) continue; + // if (ref_frame == BWDREF_FRAME) continue; + if (second_ref_frame == ALTREF_FRAME) continue; + // if (second_ref_frame == BWDREF_FRAME) continue; + } +#endif +#endif + comp_pred = second_ref_frame > INTRA_FRAME; + if (comp_pred) { + if (!cpi->allow_comp_inter_inter) continue; + if (!(cpi->ref_frame_flags & flag_list[second_ref_frame])) continue; + // Do not allow compound prediction if the segment level reference frame + // feature is in use as in this case there can only be one reference. + if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) continue; + + if ((sf->mode_search_skip_flags & FLAG_SKIP_COMP_BESTINTRA) && + best_mbmode.ref_frame[0] == INTRA_FRAME) + continue; + } + + // TODO(jingning, jkoleszar): scaling reference frame not supported for + // sub8x8 blocks. + if (ref_frame > INTRA_FRAME && + av1_is_scaled(&cm->frame_refs[ref_frame - 1].sf)) + continue; + + if (second_ref_frame > INTRA_FRAME && + av1_is_scaled(&cm->frame_refs[second_ref_frame - 1].sf)) + continue; + + if (comp_pred) + mode_excluded = cm->reference_mode == SINGLE_REFERENCE; + else if (ref_frame != INTRA_FRAME) + mode_excluded = cm->reference_mode == COMPOUND_REFERENCE; + + // If the segment reference frame feature is enabled.... + // then do nothing if the current ref frame is not allowed.. + if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME) && + get_segdata(seg, segment_id, SEG_LVL_REF_FRAME) != (int)ref_frame) { + continue; + // Disable this drop out case if the ref frame + // segment level feature is enabled for this segment. This is to + // prevent the possibility that we end up unable to pick any mode. + } else if (!segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) { + // Only consider ZEROMV/ALTREF_FRAME for alt ref frame, + // unless ARNR filtering is enabled in which case we want + // an unfiltered alternative. We allow near/nearest as well + // because they may result in zero-zero MVs but be cheaper. + if (cpi->rc.is_src_frame_alt_ref && (cpi->oxcf.arnr_max_frames == 0)) + continue; + } + + mbmi->tx_size = TX_4X4; + mbmi->uv_mode = DC_PRED; + mbmi->ref_frame[0] = ref_frame; + mbmi->ref_frame[1] = second_ref_frame; +// Evaluate all sub-pel filters irrespective of whether we can use +// them for this frame. +#if CONFIG_DUAL_FILTER + for (i = 0; i < 4; ++i) + mbmi->interp_filter[i] = cm->interp_filter == SWITCHABLE + ? EIGHTTAP_REGULAR + : cm->interp_filter; +#else + mbmi->interp_filter = + cm->interp_filter == SWITCHABLE ? EIGHTTAP_REGULAR : cm->interp_filter; +#endif // CONFIG_DUAL_FILTER + x->skip = 0; + set_ref_ptrs(cm, xd, ref_frame, second_ref_frame); + + // Select prediction reference frames. + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].pre[0] = yv12_mb[ref_frame][i]; + if (comp_pred) xd->plane[i].pre[1] = yv12_mb[second_ref_frame][i]; + } + +#if CONFIG_VAR_TX + mbmi->inter_tx_size[0][0] = mbmi->tx_size; + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); +#endif // CONFIG_VAR_TX + + if (ref_frame == INTRA_FRAME) { + int rate; + if (rd_pick_intra_sub_8x8_y_mode(cpi, x, &rate, &rate_y, &distortion_y, + NULL, best_rd) >= best_rd) + continue; + rate2 += rate; + rate2 += intra_cost_penalty; + distortion2 += distortion_y; + + if (rate_uv_intra == INT_MAX) { + choose_intra_uv_mode(cpi, x, ctx, bsize, TX_4X4, &rate_uv_intra, + &rate_uv_tokenonly, &dist_uv, &skip_uv, &mode_uv); + } + rate2 += rate_uv_intra; + rate_uv = rate_uv_tokenonly; + distortion2 += dist_uv; + distortion_uv = dist_uv; + mbmi->uv_mode = mode_uv; + } else { + int rate; + int64_t distortion; + int64_t this_rd_thresh; + int64_t tmp_rd, tmp_best_rd = INT64_MAX, tmp_best_rdu = INT64_MAX; + int tmp_best_rate = INT_MAX, tmp_best_ratey = INT_MAX; + int64_t tmp_best_distortion = INT_MAX, tmp_best_sse, uv_sse; + int tmp_best_skippable = 0; + int switchable_filter_index; + int_mv *second_ref = + comp_pred ? &x->mbmi_ext->ref_mvs[second_ref_frame][0] : NULL; + b_mode_info tmp_best_bmodes[16]; // Should this be 4 ? + MB_MODE_INFO tmp_best_mbmode; +#if CONFIG_DUAL_FILTER + BEST_SEG_INFO bsi[DUAL_FILTER_SET_SIZE]; +#else + BEST_SEG_INFO bsi[SWITCHABLE_FILTERS]; +#endif // CONFIG_DUAL_FILTER + int pred_exists = 0; + int uv_skippable; +#if CONFIG_EXT_INTER + int_mv compound_seg_newmvs[4][2]; + for (i = 0; i < 4; i++) { + compound_seg_newmvs[i][0].as_int = INVALID_MV; + compound_seg_newmvs[i][1].as_int = INVALID_MV; + } +#endif // CONFIG_EXT_INTER + + this_rd_thresh = (ref_frame == LAST_FRAME) + ? rd_opt->threshes[segment_id][bsize][THR_LAST] + : rd_opt->threshes[segment_id][bsize][THR_ALTR]; +#if CONFIG_EXT_REFS + this_rd_thresh = (ref_frame == LAST2_FRAME) + ? rd_opt->threshes[segment_id][bsize][THR_LAST2] + : this_rd_thresh; + this_rd_thresh = (ref_frame == LAST3_FRAME) + ? rd_opt->threshes[segment_id][bsize][THR_LAST3] + : this_rd_thresh; + this_rd_thresh = (ref_frame == BWDREF_FRAME) + ? rd_opt->threshes[segment_id][bsize][THR_BWDR] + : this_rd_thresh; +#endif // CONFIG_EXT_REFS + this_rd_thresh = (ref_frame == GOLDEN_FRAME) + ? rd_opt->threshes[segment_id][bsize][THR_GOLD] + : this_rd_thresh; + + // TODO(any): Add search of the tx_type to improve rd performance at the + // expense of speed. + mbmi->tx_type = DCT_DCT; + + if (cm->interp_filter != BILINEAR) { +#if CONFIG_DUAL_FILTER + tmp_best_filter[0] = EIGHTTAP_REGULAR; + tmp_best_filter[1] = EIGHTTAP_REGULAR; + tmp_best_filter[2] = EIGHTTAP_REGULAR; + tmp_best_filter[3] = EIGHTTAP_REGULAR; +#else + tmp_best_filter = EIGHTTAP_REGULAR; +#endif // CONFIG_DUAL_FILTER + if (x->source_variance < sf->disable_filter_search_var_thresh) { +#if CONFIG_DUAL_FILTER + tmp_best_filter[0] = EIGHTTAP_REGULAR; +#else + tmp_best_filter = EIGHTTAP_REGULAR; +#endif // CONFIG_DUAL_FILTER + } else if (sf->adaptive_pred_interp_filter == 1 && + ctx->pred_interp_filter < SWITCHABLE) { +#if CONFIG_DUAL_FILTER + tmp_best_filter[0] = ctx->pred_interp_filter; +#else + tmp_best_filter = ctx->pred_interp_filter; +#endif // CONFIG_DUAL_FILTER + } else if (sf->adaptive_pred_interp_filter == 2) { +#if CONFIG_DUAL_FILTER + tmp_best_filter[0] = ctx->pred_interp_filter < SWITCHABLE + ? ctx->pred_interp_filter + : 0; +#else + tmp_best_filter = ctx->pred_interp_filter < SWITCHABLE + ? ctx->pred_interp_filter + : 0; +#endif // CONFIG_DUAL_FILTER + } else { +#if CONFIG_DUAL_FILTER + const int filter_set_size = DUAL_FILTER_SET_SIZE; +#else + const int filter_set_size = SWITCHABLE_FILTERS; +#endif // CONFIG_DUAL_FILTER + for (switchable_filter_index = 0; + switchable_filter_index < filter_set_size; + ++switchable_filter_index) { + int newbest, rs; + int64_t rs_rd; + MB_MODE_INFO_EXT *mbmi_ext = x->mbmi_ext; +#if CONFIG_DUAL_FILTER + mbmi->interp_filter[0] = filter_sets[switchable_filter_index][0]; + mbmi->interp_filter[1] = filter_sets[switchable_filter_index][1]; + mbmi->interp_filter[2] = filter_sets[switchable_filter_index][0]; + mbmi->interp_filter[3] = filter_sets[switchable_filter_index][1]; +#else + mbmi->interp_filter = switchable_filter_index; +#endif // CONFIG_DUAL_FILTER + tmp_rd = rd_pick_inter_best_sub8x8_mode( + cpi, x, &mbmi_ext->ref_mvs[ref_frame][0], second_ref, best_yrd, + &rate, &rate_y, &distortion, &skippable, &total_sse, + (int)this_rd_thresh, seg_mvs, +#if CONFIG_EXT_INTER + compound_seg_newmvs, +#endif // CONFIG_EXT_INTER + bsi, switchable_filter_index, mi_row, mi_col); + if (tmp_rd == INT64_MAX) continue; + rs = av1_get_switchable_rate(cpi, xd); + rs_rd = RDCOST(x->rdmult, x->rddiv, rs, 0); + if (cm->interp_filter == SWITCHABLE) tmp_rd += rs_rd; + + newbest = (tmp_rd < tmp_best_rd); + if (newbest) { +#if CONFIG_DUAL_FILTER + tmp_best_filter[0] = mbmi->interp_filter[0]; + tmp_best_filter[1] = mbmi->interp_filter[1]; + tmp_best_filter[2] = mbmi->interp_filter[2]; + tmp_best_filter[3] = mbmi->interp_filter[3]; +#else + tmp_best_filter = mbmi->interp_filter; +#endif // CONFIG_DUAL_FILTER + tmp_best_rd = tmp_rd; + } + if ((newbest && cm->interp_filter == SWITCHABLE) || + ( +#if CONFIG_DUAL_FILTER + mbmi->interp_filter[0] == cm->interp_filter +#else + mbmi->interp_filter == cm->interp_filter +#endif // CONFIG_DUAL_FILTER + && cm->interp_filter != SWITCHABLE)) { + tmp_best_rdu = tmp_rd; + tmp_best_rate = rate; + tmp_best_ratey = rate_y; + tmp_best_distortion = distortion; + tmp_best_sse = total_sse; + tmp_best_skippable = skippable; + tmp_best_mbmode = *mbmi; + for (i = 0; i < 4; i++) { + tmp_best_bmodes[i] = xd->mi[0]->bmi[i]; + } + pred_exists = 1; + } + } // switchable_filter_index loop + } + } + + if (tmp_best_rdu == INT64_MAX && pred_exists) continue; + +#if CONFIG_DUAL_FILTER + mbmi->interp_filter[0] = + (cm->interp_filter == SWITCHABLE ? tmp_best_filter[0] + : cm->interp_filter); + mbmi->interp_filter[1] = + (cm->interp_filter == SWITCHABLE ? tmp_best_filter[1] + : cm->interp_filter); + mbmi->interp_filter[2] = + (cm->interp_filter == SWITCHABLE ? tmp_best_filter[2] + : cm->interp_filter); + mbmi->interp_filter[3] = + (cm->interp_filter == SWITCHABLE ? tmp_best_filter[3] + : cm->interp_filter); +#else + mbmi->interp_filter = + (cm->interp_filter == SWITCHABLE ? tmp_best_filter + : cm->interp_filter); +#endif // CONFIG_DUAL_FILTER + + if (!pred_exists) { + // Handles the special case when a filter that is not in the + // switchable list (bilinear) is indicated at the frame level + tmp_rd = rd_pick_inter_best_sub8x8_mode( + cpi, x, &x->mbmi_ext->ref_mvs[ref_frame][0], second_ref, best_yrd, + &rate, &rate_y, &distortion, &skippable, &total_sse, + (int)this_rd_thresh, seg_mvs, +#if CONFIG_EXT_INTER + compound_seg_newmvs, +#endif // CONFIG_EXT_INTER + bsi, 0, mi_row, mi_col); + if (tmp_rd == INT64_MAX) continue; + } else { + total_sse = tmp_best_sse; + rate = tmp_best_rate; + rate_y = tmp_best_ratey; + distortion = tmp_best_distortion; + skippable = tmp_best_skippable; + *mbmi = tmp_best_mbmode; + for (i = 0; i < 4; i++) xd->mi[0]->bmi[i] = tmp_best_bmodes[i]; + } + // Add in the cost of the transform type + if (!xd->lossless[mbmi->segment_id]) { + int rate_tx_type = 0; +#if CONFIG_EXT_TX + if (get_ext_tx_types(mbmi->tx_size, bsize, 1, cm->reduced_tx_set_used) > + 1) { + const int eset = + get_ext_tx_set(mbmi->tx_size, bsize, 1, cm->reduced_tx_set_used); + rate_tx_type = + cpi->inter_tx_type_costs[eset][mbmi->tx_size][mbmi->tx_type]; + } +#else + if (mbmi->tx_size < TX_32X32) { + rate_tx_type = cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type]; + } +#endif // CONFIG_EXT_TX + rate += rate_tx_type; + rate_y += rate_tx_type; + } + + rate2 += rate; + distortion2 += distortion; + + if (cm->interp_filter == SWITCHABLE) + rate2 += av1_get_switchable_rate(cpi, xd); + + if (!mode_excluded) + mode_excluded = comp_pred ? cm->reference_mode == SINGLE_REFERENCE + : cm->reference_mode == COMPOUND_REFERENCE; + + compmode_cost = av1_cost_bit(comp_mode_p, comp_pred); + + tmp_best_rdu = + best_rd - AOMMIN(RDCOST(x->rdmult, x->rddiv, rate2, distortion2), + RDCOST(x->rdmult, x->rddiv, 0, total_sse)); + + if (tmp_best_rdu > 0) { + // If even the 'Y' rd value of split is higher than best so far + // then dont bother looking at UV + int is_cost_valid_uv; + RD_STATS rd_stats_uv; + av1_build_inter_predictors_sbuv(&x->e_mbd, mi_row, mi_col, NULL, + BLOCK_8X8); +#if CONFIG_VAR_TX + is_cost_valid_uv = + inter_block_uvrd(cpi, x, &rd_stats_uv, BLOCK_8X8, tmp_best_rdu); +#else + is_cost_valid_uv = + super_block_uvrd(cpi, x, &rd_stats_uv, BLOCK_8X8, tmp_best_rdu); +#endif // CONFIG_VAR_TX + rate_uv = rd_stats_uv.rate; + distortion_uv = rd_stats_uv.dist; + uv_skippable = rd_stats_uv.skip; + uv_sse = rd_stats_uv.sse; + + if (!is_cost_valid_uv) continue; + rate2 += rate_uv; + distortion2 += distortion_uv; + skippable = skippable && uv_skippable; + total_sse += uv_sse; + } else { + continue; + } + } + + if (cm->reference_mode == REFERENCE_MODE_SELECT) rate2 += compmode_cost; + + // Estimate the reference frame signaling cost and add it + // to the rolling cost variable. + if (second_ref_frame > INTRA_FRAME) { + rate2 += ref_costs_comp[ref_frame]; +#if CONFIG_EXT_REFS + rate2 += ref_costs_comp[second_ref_frame]; +#endif // CONFIG_EXT_REFS + } else { + rate2 += ref_costs_single[ref_frame]; + } + + if (!disable_skip) { + // Skip is never coded at the segment level for sub8x8 blocks and instead + // always coded in the bitstream at the mode info level. + + if (ref_frame != INTRA_FRAME && !xd->lossless[mbmi->segment_id]) { + if (RDCOST(x->rdmult, x->rddiv, rate_y + rate_uv, distortion2) < + RDCOST(x->rdmult, x->rddiv, 0, total_sse)) { + // Add in the cost of the no skip flag. + rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 0); + } else { + // FIXME(rbultje) make this work for splitmv also + rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + distortion2 = total_sse; + assert(total_sse >= 0); + rate2 -= (rate_y + rate_uv); + rate_y = 0; + rate_uv = 0; + this_skip2 = 1; + } + } else { + // Add in the cost of the no skip flag. + rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 0); + } + + // Calculate the final RD estimate for this mode. + this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2); + } + + if (!disable_skip && ref_frame == INTRA_FRAME) { + for (i = 0; i < REFERENCE_MODES; ++i) + best_pred_rd[i] = AOMMIN(best_pred_rd[i], this_rd); + } + + // Did this mode help.. i.e. is it the new best mode + if (this_rd < best_rd || x->skip) { + if (!mode_excluded) { + // Note index of best mode so far + best_ref_index = ref_index; + + if (ref_frame == INTRA_FRAME) { + /* required for left and above block mv */ + mbmi->mv[0].as_int = 0; + } + + rd_cost->rate = rate2; +#if CONFIG_SUPERTX + *returnrate_nocoef = rate2 - rate_y - rate_uv; + if (!disable_skip) + *returnrate_nocoef -= + av1_cost_bit(av1_get_skip_prob(cm, xd), this_skip2); + *returnrate_nocoef -= av1_cost_bit(av1_get_intra_inter_prob(cm, xd), + mbmi->ref_frame[0] != INTRA_FRAME); + assert(*returnrate_nocoef > 0); +#endif // CONFIG_SUPERTX + rd_cost->dist = distortion2; + rd_cost->rdcost = this_rd; + best_rd = this_rd; + best_yrd = + best_rd - RDCOST(x->rdmult, x->rddiv, rate_uv, distortion_uv); + best_mbmode = *mbmi; + best_skip2 = this_skip2; + +#if CONFIG_VAR_TX + for (i = 0; i < MAX_MB_PLANE; ++i) + memset(ctx->blk_skip[i], 0, sizeof(uint8_t) * ctx->num_4x4_blk); +#endif // CONFIG_VAR_TX + + for (i = 0; i < 4; i++) best_bmodes[i] = xd->mi[0]->bmi[i]; + } + } + + /* keep record of best compound/single-only prediction */ + if (!disable_skip && ref_frame != INTRA_FRAME) { + int64_t single_rd, hybrid_rd, single_rate, hybrid_rate; + + if (cm->reference_mode == REFERENCE_MODE_SELECT) { + single_rate = rate2 - compmode_cost; + hybrid_rate = rate2; + } else { + single_rate = rate2; + hybrid_rate = rate2 + compmode_cost; + } + + single_rd = RDCOST(x->rdmult, x->rddiv, single_rate, distortion2); + hybrid_rd = RDCOST(x->rdmult, x->rddiv, hybrid_rate, distortion2); + + if (!comp_pred && single_rd < best_pred_rd[SINGLE_REFERENCE]) + best_pred_rd[SINGLE_REFERENCE] = single_rd; + else if (comp_pred && single_rd < best_pred_rd[COMPOUND_REFERENCE]) + best_pred_rd[COMPOUND_REFERENCE] = single_rd; + + if (hybrid_rd < best_pred_rd[REFERENCE_MODE_SELECT]) + best_pred_rd[REFERENCE_MODE_SELECT] = hybrid_rd; + } + + if (x->skip && !comp_pred) break; + } + + if (best_rd >= best_rd_so_far) { + rd_cost->rate = INT_MAX; + rd_cost->rdcost = INT64_MAX; +#if CONFIG_SUPERTX + *returnrate_nocoef = INT_MAX; +#endif // CONFIG_SUPERTX + return; + } + + if (best_rd == INT64_MAX) { + rd_cost->rate = INT_MAX; + rd_cost->dist = INT64_MAX; + rd_cost->rdcost = INT64_MAX; +#if CONFIG_SUPERTX + *returnrate_nocoef = INT_MAX; +#endif // CONFIG_SUPERTX + return; + } + +#if CONFIG_DUAL_FILTER + assert((cm->interp_filter == SWITCHABLE) || + (cm->interp_filter == best_mbmode.interp_filter[0]) || + !is_inter_block(&best_mbmode)); +#else + assert((cm->interp_filter == SWITCHABLE) || + (cm->interp_filter == best_mbmode.interp_filter) || + !is_inter_block(&best_mbmode)); +#endif // CONFIG_DUAL_FILTER + + av1_update_rd_thresh_fact(cm, tile_data->thresh_freq_fact, + sf->adaptive_rd_thresh, bsize, best_ref_index); + + // macroblock modes + *mbmi = best_mbmode; +#if CONFIG_VAR_TX + mbmi->inter_tx_size[0][0] = mbmi->tx_size; +#endif // CONFIG_VAR_TX + + x->skip |= best_skip2; + if (!is_inter_block(&best_mbmode)) { + for (i = 0; i < 4; i++) xd->mi[0]->bmi[i].as_mode = best_bmodes[i].as_mode; + } else { + for (i = 0; i < 4; ++i) + memcpy(&xd->mi[0]->bmi[i], &best_bmodes[i], sizeof(b_mode_info)); + +#if CONFIG_REF_MV + mbmi->pred_mv[0].as_int = xd->mi[0]->bmi[3].pred_mv[0].as_int; + mbmi->pred_mv[1].as_int = xd->mi[0]->bmi[3].pred_mv[1].as_int; +#endif // CONFIG_REF_MV + mbmi->mv[0].as_int = xd->mi[0]->bmi[3].as_mv[0].as_int; + mbmi->mv[1].as_int = xd->mi[0]->bmi[3].as_mv[1].as_int; + } + +// Note: this section is needed since the mode may have been forced to ZEROMV +#if CONFIG_GLOBAL_MOTION + if (mbmi->mode == ZEROMV +#if CONFIG_EXT_INTER + || mbmi->mode == ZERO_ZEROMV +#endif // CONFIG_EXT_INTER + ) { + if (is_nontrans_global_motion(xd)) { +#if CONFIG_DUAL_FILTER + mbmi->interp_filter[0] = cm->interp_filter == SWITCHABLE + ? EIGHTTAP_REGULAR + : cm->interp_filter; + mbmi->interp_filter[1] = cm->interp_filter == SWITCHABLE + ? EIGHTTAP_REGULAR + : cm->interp_filter; +#else + mbmi->interp_filter = cm->interp_filter == SWITCHABLE ? EIGHTTAP_REGULAR + : cm->interp_filter; +#endif // CONFIG_DUAL_FILTER + } + } +#endif // CONFIG_GLOBAL_MOTION + + for (i = 0; i < REFERENCE_MODES; ++i) { + if (best_pred_rd[i] == INT64_MAX) + best_pred_diff[i] = INT_MIN; + else + best_pred_diff[i] = best_rd - best_pred_rd[i]; + } + + store_coding_context(x, ctx, best_ref_index, best_pred_diff, 0); +} + +#if CONFIG_MOTION_VAR +// This function has a structure similar to av1_build_obmc_inter_prediction +// +// The OBMC predictor is computed as: +// +// PObmc(x,y) = +// AOM_BLEND_A64(Mh(x), +// AOM_BLEND_A64(Mv(y), P(x,y), PAbove(x,y)), +// PLeft(x, y)) +// +// Scaling up by AOM_BLEND_A64_MAX_ALPHA ** 2 and omitting the intermediate +// rounding, this can be written as: +// +// AOM_BLEND_A64_MAX_ALPHA * AOM_BLEND_A64_MAX_ALPHA * Pobmc(x,y) = +// Mh(x) * Mv(y) * P(x,y) + +// Mh(x) * Cv(y) * Pabove(x,y) + +// AOM_BLEND_A64_MAX_ALPHA * Ch(x) * PLeft(x, y) +// +// Where : +// +// Cv(y) = AOM_BLEND_A64_MAX_ALPHA - Mv(y) +// Ch(y) = AOM_BLEND_A64_MAX_ALPHA - Mh(y) +// +// This function computes 'wsrc' and 'mask' as: +// +// wsrc(x, y) = +// AOM_BLEND_A64_MAX_ALPHA * AOM_BLEND_A64_MAX_ALPHA * src(x, y) - +// Mh(x) * Cv(y) * Pabove(x,y) + +// AOM_BLEND_A64_MAX_ALPHA * Ch(x) * PLeft(x, y) +// +// mask(x, y) = Mh(x) * Mv(y) +// +// These can then be used to efficiently approximate the error for any +// predictor P in the context of the provided neighbouring predictors by +// computing: +// +// error(x, y) = +// wsrc(x, y) - mask(x, y) * P(x, y) / (AOM_BLEND_A64_MAX_ALPHA ** 2) +// +static void calc_target_weighted_pred(const AV1_COMMON *cm, const MACROBLOCK *x, + const MACROBLOCKD *xd, int mi_row, + int mi_col, const uint8_t *above, + int above_stride, const uint8_t *left, + int left_stride) { + const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + int row, col, i; + const int bw = xd->n8_w << MI_SIZE_LOG2; + const int bh = xd->n8_h << MI_SIZE_LOG2; + int32_t *mask_buf = x->mask_buf; + int32_t *wsrc_buf = x->wsrc_buf; + const int wsrc_stride = bw; + const int mask_stride = bw; + const int src_scale = AOM_BLEND_A64_MAX_ALPHA * AOM_BLEND_A64_MAX_ALPHA; +#if CONFIG_HIGHBITDEPTH + const int is_hbd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? 1 : 0; +#else + const int is_hbd = 0; +#endif // CONFIG_HIGHBITDEPTH + + // plane 0 should not be subsampled + assert(xd->plane[0].subsampling_x == 0); + assert(xd->plane[0].subsampling_y == 0); + + av1_zero_array(wsrc_buf, bw * bh); + for (i = 0; i < bw * bh; ++i) mask_buf[i] = AOM_BLEND_A64_MAX_ALPHA; + + // handle above row + if (xd->up_available) { + const int overlap = num_4x4_blocks_high_lookup[bsize] * 2; + const int miw = AOMMIN(xd->n8_w, cm->mi_cols - mi_col); + const int mi_row_offset = -1; + const uint8_t *const mask1d = av1_get_obmc_mask(overlap); + const int neighbor_limit = max_neighbor_obmc[b_width_log2_lookup[bsize]]; + int neighbor_count = 0; + + assert(miw > 0); + + i = 0; + do { // for each mi in the above row + const int mi_col_offset = i; + const MB_MODE_INFO *const above_mbmi = + &xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]->mbmi; + const BLOCK_SIZE a_bsize = above_mbmi->sb_type; + const int mi_step = AOMMIN(xd->n8_w, mi_size_wide[a_bsize]); + const int neighbor_bw = mi_step * MI_SIZE; + + if (is_neighbor_overlappable(above_mbmi)) { + if (!CONFIG_CB4X4 && (a_bsize == BLOCK_4X4 || a_bsize == BLOCK_4X8)) + neighbor_count += 2; + else + neighbor_count++; + if (neighbor_count > neighbor_limit) break; + + const int tmp_stride = above_stride; + int32_t *wsrc = wsrc_buf + (i * MI_SIZE); + int32_t *mask = mask_buf + (i * MI_SIZE); + + if (!is_hbd) { + const uint8_t *tmp = above; + + for (row = 0; row < overlap; ++row) { + const uint8_t m0 = mask1d[row]; + const uint8_t m1 = AOM_BLEND_A64_MAX_ALPHA - m0; + for (col = 0; col < neighbor_bw; ++col) { + wsrc[col] = m1 * tmp[col]; + mask[col] = m0; + } + wsrc += wsrc_stride; + mask += mask_stride; + tmp += tmp_stride; + } +#if CONFIG_HIGHBITDEPTH + } else { + const uint16_t *tmp = CONVERT_TO_SHORTPTR(above); + + for (row = 0; row < overlap; ++row) { + const uint8_t m0 = mask1d[row]; + const uint8_t m1 = AOM_BLEND_A64_MAX_ALPHA - m0; + for (col = 0; col < neighbor_bw; ++col) { + wsrc[col] = m1 * tmp[col]; + mask[col] = m0; + } + wsrc += wsrc_stride; + mask += mask_stride; + tmp += tmp_stride; + } +#endif // CONFIG_HIGHBITDEPTH + } + } + + above += neighbor_bw; + i += mi_step; + } while (i < miw); + } + + for (i = 0; i < bw * bh; ++i) { + wsrc_buf[i] *= AOM_BLEND_A64_MAX_ALPHA; + mask_buf[i] *= AOM_BLEND_A64_MAX_ALPHA; + } + + // handle left column + if (xd->left_available) { + const int overlap = num_4x4_blocks_wide_lookup[bsize] * 2; + const int mih = AOMMIN(xd->n8_h, cm->mi_rows - mi_row); + const int mi_col_offset = -1; + const uint8_t *const mask1d = av1_get_obmc_mask(overlap); + const int neighbor_limit = max_neighbor_obmc[b_height_log2_lookup[bsize]]; + int neighbor_count = 0; + + assert(mih > 0); + + i = 0; + do { // for each mi in the left column + const int mi_row_offset = i; + const MB_MODE_INFO *const left_mbmi = + &xd->mi[mi_col_offset + mi_row_offset * xd->mi_stride]->mbmi; + const BLOCK_SIZE l_bsize = left_mbmi->sb_type; + const int mi_step = AOMMIN(xd->n8_h, mi_size_high[l_bsize]); + const int neighbor_bh = mi_step * MI_SIZE; + + if (is_neighbor_overlappable(left_mbmi)) { + if (!CONFIG_CB4X4 && (l_bsize == BLOCK_4X4 || l_bsize == BLOCK_8X4)) + neighbor_count += 2; + else + neighbor_count++; + if (neighbor_count > neighbor_limit) break; + + const int tmp_stride = left_stride; + int32_t *wsrc = wsrc_buf + (i * MI_SIZE * wsrc_stride); + int32_t *mask = mask_buf + (i * MI_SIZE * mask_stride); + + if (!is_hbd) { + const uint8_t *tmp = left; + + for (row = 0; row < neighbor_bh; ++row) { + for (col = 0; col < overlap; ++col) { + const uint8_t m0 = mask1d[col]; + const uint8_t m1 = AOM_BLEND_A64_MAX_ALPHA - m0; + wsrc[col] = (wsrc[col] >> AOM_BLEND_A64_ROUND_BITS) * m0 + + (tmp[col] << AOM_BLEND_A64_ROUND_BITS) * m1; + mask[col] = (mask[col] >> AOM_BLEND_A64_ROUND_BITS) * m0; + } + wsrc += wsrc_stride; + mask += mask_stride; + tmp += tmp_stride; + } +#if CONFIG_HIGHBITDEPTH + } else { + const uint16_t *tmp = CONVERT_TO_SHORTPTR(left); + + for (row = 0; row < neighbor_bh; ++row) { + for (col = 0; col < overlap; ++col) { + const uint8_t m0 = mask1d[col]; + const uint8_t m1 = AOM_BLEND_A64_MAX_ALPHA - m0; + wsrc[col] = (wsrc[col] >> AOM_BLEND_A64_ROUND_BITS) * m0 + + (tmp[col] << AOM_BLEND_A64_ROUND_BITS) * m1; + mask[col] = (mask[col] >> AOM_BLEND_A64_ROUND_BITS) * m0; + } + wsrc += wsrc_stride; + mask += mask_stride; + tmp += tmp_stride; + } +#endif // CONFIG_HIGHBITDEPTH + } + } + + left += neighbor_bh * left_stride; + i += mi_step; + } while (i < mih); + } + + if (!is_hbd) { + const uint8_t *src = x->plane[0].src.buf; + + for (row = 0; row < bh; ++row) { + for (col = 0; col < bw; ++col) { + wsrc_buf[col] = src[col] * src_scale - wsrc_buf[col]; + } + wsrc_buf += wsrc_stride; + src += x->plane[0].src.stride; + } +#if CONFIG_HIGHBITDEPTH + } else { + const uint16_t *src = CONVERT_TO_SHORTPTR(x->plane[0].src.buf); + + for (row = 0; row < bh; ++row) { + for (col = 0; col < bw; ++col) { + wsrc_buf[col] = src[col] * src_scale - wsrc_buf[col]; + } + wsrc_buf += wsrc_stride; + src += x->plane[0].src.stride; + } +#endif // CONFIG_HIGHBITDEPTH + } +} + +#if CONFIG_NCOBMC +void av1_check_ncobmc_rd(const struct AV1_COMP *cpi, struct macroblock *x, + int mi_row, int mi_col) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + MB_MODE_INFO backup_mbmi; + BLOCK_SIZE bsize = mbmi->sb_type; + int ref, skip_blk, backup_skip = x->skip; + int64_t rd_causal; + RD_STATS rd_stats_y, rd_stats_uv; + int rate_skip0 = av1_cost_bit(av1_get_skip_prob(cm, xd), 0); + int rate_skip1 = av1_cost_bit(av1_get_skip_prob(cm, xd), 1); + + // Recompute the best causal predictor and rd + mbmi->motion_mode = SIMPLE_TRANSLATION; + set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]); + for (ref = 0; ref < 1 + has_second_ref(mbmi); ++ref) { + YV12_BUFFER_CONFIG *cfg = get_ref_frame_buffer(cpi, mbmi->ref_frame[ref]); + assert(cfg != NULL); + av1_setup_pre_planes(xd, ref, cfg, mi_row, mi_col, + &xd->block_refs[ref]->sf); + } + av1_setup_dst_planes(x->e_mbd.plane, bsize, + get_frame_new_buffer(&cpi->common), mi_row, mi_col); + + av1_build_inter_predictors_sb(xd, mi_row, mi_col, NULL, bsize); + + av1_subtract_plane(x, bsize, 0); + super_block_yrd(cpi, x, &rd_stats_y, bsize, INT64_MAX); + super_block_uvrd(cpi, x, &rd_stats_uv, bsize, INT64_MAX); + assert(rd_stats_y.rate != INT_MAX && rd_stats_uv.rate != INT_MAX); + if (rd_stats_y.skip && rd_stats_uv.skip) { + rd_stats_y.rate = rate_skip1; + rd_stats_uv.rate = 0; + rd_stats_y.dist = rd_stats_y.sse; + rd_stats_uv.dist = rd_stats_uv.sse; + skip_blk = 0; + } else if (RDCOST(x->rdmult, x->rddiv, + (rd_stats_y.rate + rd_stats_uv.rate + rate_skip0), + (rd_stats_y.dist + rd_stats_uv.dist)) > + RDCOST(x->rdmult, x->rddiv, rate_skip1, + (rd_stats_y.sse + rd_stats_uv.sse))) { + rd_stats_y.rate = rate_skip1; + rd_stats_uv.rate = 0; + rd_stats_y.dist = rd_stats_y.sse; + rd_stats_uv.dist = rd_stats_uv.sse; + skip_blk = 1; + } else { + rd_stats_y.rate += rate_skip0; + skip_blk = 0; + } + backup_skip = skip_blk; + backup_mbmi = *mbmi; + rd_causal = RDCOST(x->rdmult, x->rddiv, (rd_stats_y.rate + rd_stats_uv.rate), + (rd_stats_y.dist + rd_stats_uv.dist)); + rd_causal += RDCOST(x->rdmult, x->rddiv, + av1_cost_bit(cm->fc->motion_mode_prob[bsize][0], 0), 0); + + // Check non-causal mode + mbmi->motion_mode = OBMC_CAUSAL; + av1_build_ncobmc_inter_predictors_sb(cm, xd, mi_row, mi_col); + + av1_subtract_plane(x, bsize, 0); + super_block_yrd(cpi, x, &rd_stats_y, bsize, INT64_MAX); + super_block_uvrd(cpi, x, &rd_stats_uv, bsize, INT64_MAX); + assert(rd_stats_y.rate != INT_MAX && rd_stats_uv.rate != INT_MAX); + if (rd_stats_y.skip && rd_stats_uv.skip) { + rd_stats_y.rate = rate_skip1; + rd_stats_uv.rate = 0; + rd_stats_y.dist = rd_stats_y.sse; + rd_stats_uv.dist = rd_stats_uv.sse; + skip_blk = 0; + } else if (RDCOST(x->rdmult, x->rddiv, + (rd_stats_y.rate + rd_stats_uv.rate + rate_skip0), + (rd_stats_y.dist + rd_stats_uv.dist)) > + RDCOST(x->rdmult, x->rddiv, rate_skip1, + (rd_stats_y.sse + rd_stats_uv.sse))) { + rd_stats_y.rate = rate_skip1; + rd_stats_uv.rate = 0; + rd_stats_y.dist = rd_stats_y.sse; + rd_stats_uv.dist = rd_stats_uv.sse; + skip_blk = 1; + } else { + rd_stats_y.rate += rate_skip0; + skip_blk = 0; + } + + if (rd_causal > + RDCOST(x->rdmult, x->rddiv, + rd_stats_y.rate + rd_stats_uv.rate + + av1_cost_bit(cm->fc->motion_mode_prob[bsize][0], 1), + (rd_stats_y.dist + rd_stats_uv.dist))) { + x->skip = skip_blk; + } else { + *mbmi = backup_mbmi; + x->skip = backup_skip; + } +} +#endif // CONFIG_NCOBMC +#endif // CONFIG_MOTION_VAR diff --git a/third_party/aom/av1/encoder/rdopt.h b/third_party/aom/av1/encoder/rdopt.h new file mode 100644 index 0000000000..a7053b2897 --- /dev/null +++ b/third_party/aom/av1/encoder/rdopt.h @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_RDOPT_H_ +#define AV1_ENCODER_RDOPT_H_ + +#include "av1/common/blockd.h" + +#include "av1/encoder/block.h" +#include "av1/encoder/context_tree.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct TileInfo; +struct AV1_COMP; +struct macroblock; +struct RD_STATS; + +#if CONFIG_RD_DEBUG +static INLINE void av1_update_txb_coeff_cost(RD_STATS *rd_stats, int plane, + TX_SIZE tx_size, int blk_row, + int blk_col, int txb_coeff_cost) { + (void)blk_row; + (void)blk_col; + (void)tx_size; + rd_stats->txb_coeff_cost[plane] += txb_coeff_cost; + +#if CONFIG_VAR_TX + { + const int txb_h = tx_size_high_unit[tx_size]; + const int txb_w = tx_size_wide_unit[tx_size]; + int idx, idy; + for (idy = 0; idy < txb_h; ++idy) + for (idx = 0; idx < txb_w; ++idx) + rd_stats->txb_coeff_cost_map[plane][blk_row + idy][blk_col + idx] = 0; + + rd_stats->txb_coeff_cost_map[plane][blk_row][blk_col] = txb_coeff_cost; + } + assert(blk_row < TXB_COEFF_COST_MAP_SIZE); + assert(blk_col < TXB_COEFF_COST_MAP_SIZE); +#endif +} +#endif + +typedef enum OUTPUT_STATUS { + OUTPUT_HAS_PREDICTED_PIXELS, + OUTPUT_HAS_DECODED_PIXELS +} OUTPUT_STATUS; + +void av1_dist_block(const AV1_COMP *cpi, MACROBLOCK *x, int plane, + BLOCK_SIZE plane_bsize, int block, int blk_row, int blk_col, + TX_SIZE tx_size, int64_t *out_dist, int64_t *out_sse, + OUTPUT_STATUS output_status); + +#if !CONFIG_PVQ || CONFIG_VAR_TX +int av1_cost_coeffs(const AV1_COMP *const cpi, MACROBLOCK *x, int plane, + int block, TX_SIZE tx_size, const SCAN_ORDER *scan_order, + const ENTROPY_CONTEXT *a, const ENTROPY_CONTEXT *l, + int use_fast_coef_costing); +#endif +void av1_rd_pick_intra_mode_sb(const struct AV1_COMP *cpi, struct macroblock *x, + struct RD_STATS *rd_cost, BLOCK_SIZE bsize, + PICK_MODE_CONTEXT *ctx, int64_t best_rd); + +unsigned int av1_get_sby_perpixel_variance(const AV1_COMP *cpi, + const struct buf_2d *ref, + BLOCK_SIZE bs); +#if CONFIG_HIGHBITDEPTH +unsigned int av1_high_get_sby_perpixel_variance(const AV1_COMP *cpi, + const struct buf_2d *ref, + BLOCK_SIZE bs, int bd); +#endif + +void av1_rd_pick_inter_mode_sb(const struct AV1_COMP *cpi, + struct TileDataEnc *tile_data, + struct macroblock *x, int mi_row, int mi_col, + struct RD_STATS *rd_cost, +#if CONFIG_SUPERTX + int *returnrate_nocoef, +#endif // CONFIG_SUPERTX + BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx, + int64_t best_rd_so_far); + +void av1_rd_pick_inter_mode_sb_seg_skip( + const struct AV1_COMP *cpi, struct TileDataEnc *tile_data, + struct macroblock *x, int mi_row, int mi_col, struct RD_STATS *rd_cost, + BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx, int64_t best_rd_so_far); + +int av1_internal_image_edge(const struct AV1_COMP *cpi); +int av1_active_h_edge(const struct AV1_COMP *cpi, int mi_row, int mi_step); +int av1_active_v_edge(const struct AV1_COMP *cpi, int mi_col, int mi_step); +int av1_active_edge_sb(const struct AV1_COMP *cpi, int mi_row, int mi_col); + +void av1_rd_pick_inter_mode_sub8x8(const struct AV1_COMP *cpi, + struct TileDataEnc *tile_data, + struct macroblock *x, int mi_row, int mi_col, + struct RD_STATS *rd_cost, +#if CONFIG_SUPERTX + int *returnrate_nocoef, +#endif // CONFIG_SUPERTX + BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx, + int64_t best_rd_so_far); + +#if CONFIG_MOTION_VAR && CONFIG_NCOBMC +void av1_check_ncobmc_rd(const struct AV1_COMP *cpi, struct macroblock *x, + int mi_row, int mi_col); +#endif // CONFIG_MOTION_VAR && CONFIG_NCOBMC + +#if CONFIG_SUPERTX +#if CONFIG_VAR_TX +void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, + int blk_row, int blk_col, int plane, int block, + int plane_bsize, const ENTROPY_CONTEXT *a, + const ENTROPY_CONTEXT *l, RD_STATS *rd_stats); +#endif + +void av1_txfm_rd_in_plane_supertx(MACROBLOCK *x, const AV1_COMP *cpi, int *rate, + int64_t *distortion, int *skippable, + int64_t *sse, int64_t ref_best_rd, int plane, + BLOCK_SIZE bsize, TX_SIZE tx_size, + int use_fast_coef_casting); +#endif // CONFIG_SUPERTX + +#ifdef __cplusplus +} // extern "C" +#endif + +int av1_tx_type_cost(const AV1_COMP *cpi, const MACROBLOCKD *xd, + BLOCK_SIZE bsize, int plane, TX_SIZE tx_size, + TX_TYPE tx_type); + +#endif // AV1_ENCODER_RDOPT_H_ diff --git a/third_party/aom/av1/encoder/segmentation.c b/third_party/aom/av1/encoder/segmentation.c new file mode 100644 index 0000000000..b581a61d0a --- /dev/null +++ b/third_party/aom/av1/encoder/segmentation.c @@ -0,0 +1,394 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom_mem/aom_mem.h" + +#include "av1/common/pred_common.h" +#include "av1/common/tile_common.h" + +#include "av1/encoder/cost.h" +#include "av1/encoder/segmentation.h" +#include "av1/encoder/subexp.h" + +void av1_enable_segmentation(struct segmentation *seg) { + seg->enabled = 1; + seg->update_map = 1; + seg->update_data = 1; +} + +void av1_disable_segmentation(struct segmentation *seg) { + seg->enabled = 0; + seg->update_map = 0; + seg->update_data = 0; +} + +void av1_set_segment_data(struct segmentation *seg, signed char *feature_data, + unsigned char abs_delta) { + seg->abs_delta = abs_delta; + + memcpy(seg->feature_data, feature_data, sizeof(seg->feature_data)); +} +void av1_disable_segfeature(struct segmentation *seg, int segment_id, + SEG_LVL_FEATURES feature_id) { + seg->feature_mask[segment_id] &= ~(1 << feature_id); +} + +void av1_clear_segdata(struct segmentation *seg, int segment_id, + SEG_LVL_FEATURES feature_id) { + seg->feature_data[segment_id][feature_id] = 0; +} + +// Based on set of segment counts calculate a probability tree +static void calc_segtree_probs(unsigned *segcounts, + aom_prob *segment_tree_probs, + const aom_prob *cur_tree_probs, + const int probwt) { + // Work out probabilities of each segment + const unsigned cc[4] = { segcounts[0] + segcounts[1], + segcounts[2] + segcounts[3], + segcounts[4] + segcounts[5], + segcounts[6] + segcounts[7] }; + const unsigned ccc[2] = { cc[0] + cc[1], cc[2] + cc[3] }; + int i; + + segment_tree_probs[0] = get_binary_prob(ccc[0], ccc[1]); + segment_tree_probs[1] = get_binary_prob(cc[0], cc[1]); + segment_tree_probs[2] = get_binary_prob(cc[2], cc[3]); + segment_tree_probs[3] = get_binary_prob(segcounts[0], segcounts[1]); + segment_tree_probs[4] = get_binary_prob(segcounts[2], segcounts[3]); + segment_tree_probs[5] = get_binary_prob(segcounts[4], segcounts[5]); + segment_tree_probs[6] = get_binary_prob(segcounts[6], segcounts[7]); + + for (i = 0; i < 7; i++) { + const unsigned *ct = + i == 0 ? ccc : i < 3 ? cc + (i & 2) : segcounts + (i - 3) * 2; + av1_prob_diff_update_savings_search(ct, cur_tree_probs[i], + &segment_tree_probs[i], + DIFF_UPDATE_PROB, probwt); + } +} + +// Based on set of segment counts and probabilities calculate a cost estimate +static int cost_segmap(unsigned *segcounts, aom_prob *probs) { + const int c01 = segcounts[0] + segcounts[1]; + const int c23 = segcounts[2] + segcounts[3]; + const int c45 = segcounts[4] + segcounts[5]; + const int c67 = segcounts[6] + segcounts[7]; + const int c0123 = c01 + c23; + const int c4567 = c45 + c67; + + // Cost the top node of the tree + int cost = c0123 * av1_cost_zero(probs[0]) + c4567 * av1_cost_one(probs[0]); + + // Cost subsequent levels + if (c0123 > 0) { + cost += c01 * av1_cost_zero(probs[1]) + c23 * av1_cost_one(probs[1]); + + if (c01 > 0) + cost += segcounts[0] * av1_cost_zero(probs[3]) + + segcounts[1] * av1_cost_one(probs[3]); + if (c23 > 0) + cost += segcounts[2] * av1_cost_zero(probs[4]) + + segcounts[3] * av1_cost_one(probs[4]); + } + + if (c4567 > 0) { + cost += c45 * av1_cost_zero(probs[2]) + c67 * av1_cost_one(probs[2]); + + if (c45 > 0) + cost += segcounts[4] * av1_cost_zero(probs[5]) + + segcounts[5] * av1_cost_one(probs[5]); + if (c67 > 0) + cost += segcounts[6] * av1_cost_zero(probs[6]) + + segcounts[7] * av1_cost_one(probs[6]); + } + + return cost; +} + +static void count_segs(const AV1_COMMON *cm, MACROBLOCKD *xd, + const TileInfo *tile, MODE_INFO **mi, + unsigned *no_pred_segcounts, + unsigned (*temporal_predictor_count)[2], + unsigned *t_unpred_seg_counts, int bw, int bh, + int mi_row, int mi_col) { + int segment_id; + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + xd->mi = mi; + segment_id = xd->mi[0]->mbmi.segment_id; + + set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, +#if CONFIG_DEPENDENT_HORZTILES + cm->dependent_horz_tiles, +#endif // CONFIG_DEPENDENT_HORZTILES + cm->mi_rows, cm->mi_cols); + + // Count the number of hits on each segment with no prediction + no_pred_segcounts[segment_id]++; + + // Temporal prediction not allowed on key frames + if (cm->frame_type != KEY_FRAME) { + const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; + // Test to see if the segment id matches the predicted value. + const int pred_segment_id = + get_segment_id(cm, cm->last_frame_seg_map, bsize, mi_row, mi_col); + const int pred_flag = pred_segment_id == segment_id; + const int pred_context = av1_get_pred_context_seg_id(xd); + + // Store the prediction status for this mb and update counts + // as appropriate + xd->mi[0]->mbmi.seg_id_predicted = pred_flag; + temporal_predictor_count[pred_context][pred_flag]++; + + // Update the "unpredicted" segment count + if (!pred_flag) t_unpred_seg_counts[segment_id]++; + } +} + +static void count_segs_sb(const AV1_COMMON *cm, MACROBLOCKD *xd, + const TileInfo *tile, MODE_INFO **mi, + unsigned *no_pred_segcounts, + unsigned (*temporal_predictor_count)[2], + unsigned *t_unpred_seg_counts, int mi_row, int mi_col, + BLOCK_SIZE bsize) { + const int mis = cm->mi_stride; + const int bs = mi_size_wide[bsize], hbs = bs / 2; +#if CONFIG_EXT_PARTITION_TYPES + PARTITION_TYPE partition; +#else + int bw, bh; +#endif // CONFIG_EXT_PARTITION_TYPES + + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + +#if CONFIG_EXT_PARTITION_TYPES + if (bsize == BLOCK_8X8) + partition = PARTITION_NONE; + else + partition = get_partition(cm, mi_row, mi_col, bsize); + switch (partition) { + case PARTITION_NONE: + count_segs(cm, xd, tile, mi, no_pred_segcounts, temporal_predictor_count, + t_unpred_seg_counts, bs, bs, mi_row, mi_col); + break; + case PARTITION_HORZ: + count_segs(cm, xd, tile, mi, no_pred_segcounts, temporal_predictor_count, + t_unpred_seg_counts, bs, hbs, mi_row, mi_col); + count_segs(cm, xd, tile, mi + hbs * mis, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, bs, hbs, + mi_row + hbs, mi_col); + break; + case PARTITION_VERT: + count_segs(cm, xd, tile, mi, no_pred_segcounts, temporal_predictor_count, + t_unpred_seg_counts, hbs, bs, mi_row, mi_col); + count_segs(cm, xd, tile, mi + hbs, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, hbs, bs, mi_row, + mi_col + hbs); + break; + case PARTITION_HORZ_A: + count_segs(cm, xd, tile, mi, no_pred_segcounts, temporal_predictor_count, + t_unpred_seg_counts, hbs, hbs, mi_row, mi_col); + count_segs(cm, xd, tile, mi + hbs, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, hbs, hbs, + mi_row, mi_col + hbs); + count_segs(cm, xd, tile, mi + hbs * mis, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, bs, hbs, + mi_row + hbs, mi_col); + break; + case PARTITION_HORZ_B: + count_segs(cm, xd, tile, mi, no_pred_segcounts, temporal_predictor_count, + t_unpred_seg_counts, bs, hbs, mi_row, mi_col); + count_segs(cm, xd, tile, mi + hbs * mis, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, hbs, hbs, + mi_row + hbs, mi_col); + count_segs(cm, xd, tile, mi + hbs + hbs * mis, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, hbs, hbs, + mi_row + hbs, mi_col + hbs); + break; + case PARTITION_VERT_A: + count_segs(cm, xd, tile, mi, no_pred_segcounts, temporal_predictor_count, + t_unpred_seg_counts, hbs, hbs, mi_row, mi_col); + count_segs(cm, xd, tile, mi + hbs * mis, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, hbs, hbs, + mi_row + hbs, mi_col); + count_segs(cm, xd, tile, mi + hbs, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, hbs, bs, mi_row, + mi_col + hbs); + break; + case PARTITION_VERT_B: + count_segs(cm, xd, tile, mi, no_pred_segcounts, temporal_predictor_count, + t_unpred_seg_counts, hbs, bs, mi_row, mi_col); + count_segs(cm, xd, tile, mi + hbs, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, hbs, hbs, + mi_row, mi_col + hbs); + count_segs(cm, xd, tile, mi + hbs + hbs * mis, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, hbs, hbs, + mi_row + hbs, mi_col + hbs); + break; + case PARTITION_SPLIT: { + const BLOCK_SIZE subsize = subsize_lookup[PARTITION_SPLIT][bsize]; + int n; + + assert(num_8x8_blocks_wide_lookup[mi[0]->mbmi.sb_type] < bs && + num_8x8_blocks_high_lookup[mi[0]->mbmi.sb_type] < bs); + + for (n = 0; n < 4; n++) { + const int mi_dc = hbs * (n & 1); + const int mi_dr = hbs * (n >> 1); + + count_segs_sb(cm, xd, tile, &mi[mi_dr * mis + mi_dc], no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, + mi_row + mi_dr, mi_col + mi_dc, subsize); + } + } break; + default: assert(0); + } +#else + bw = mi_size_wide[mi[0]->mbmi.sb_type]; + bh = mi_size_high[mi[0]->mbmi.sb_type]; + + if (bw == bs && bh == bs) { + count_segs(cm, xd, tile, mi, no_pred_segcounts, temporal_predictor_count, + t_unpred_seg_counts, bs, bs, mi_row, mi_col); + } else if (bw == bs && bh < bs) { + count_segs(cm, xd, tile, mi, no_pred_segcounts, temporal_predictor_count, + t_unpred_seg_counts, bs, hbs, mi_row, mi_col); + count_segs(cm, xd, tile, mi + hbs * mis, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, bs, hbs, + mi_row + hbs, mi_col); + } else if (bw < bs && bh == bs) { + count_segs(cm, xd, tile, mi, no_pred_segcounts, temporal_predictor_count, + t_unpred_seg_counts, hbs, bs, mi_row, mi_col); + count_segs(cm, xd, tile, mi + hbs, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, hbs, bs, mi_row, + mi_col + hbs); + } else { + const BLOCK_SIZE subsize = subsize_lookup[PARTITION_SPLIT][bsize]; + int n; + + assert(bw < bs && bh < bs); + + for (n = 0; n < 4; n++) { + const int mi_dc = hbs * (n & 1); + const int mi_dr = hbs * (n >> 1); + + count_segs_sb(cm, xd, tile, &mi[mi_dr * mis + mi_dc], no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, + mi_row + mi_dr, mi_col + mi_dc, subsize); + } + } +#endif // CONFIG_EXT_PARTITION_TYPES +} + +void av1_choose_segmap_coding_method(AV1_COMMON *cm, MACROBLOCKD *xd) { + struct segmentation *seg = &cm->seg; + struct segmentation_probs *segp = &cm->fc->seg; + + int no_pred_cost; + int t_pred_cost = INT_MAX; + + int i, tile_col, tile_row, mi_row, mi_col; +#if CONFIG_TILE_GROUPS + const int probwt = cm->num_tg; +#else + const int probwt = 1; +#endif + + unsigned(*temporal_predictor_count)[2] = cm->counts.seg.pred; + unsigned *no_pred_segcounts = cm->counts.seg.tree_total; + unsigned *t_unpred_seg_counts = cm->counts.seg.tree_mispred; + + aom_prob no_pred_tree[SEG_TREE_PROBS]; + aom_prob t_pred_tree[SEG_TREE_PROBS]; + aom_prob t_nopred_prob[PREDICTION_PROBS]; + + (void)xd; + + // We are about to recompute all the segment counts, so zero the accumulators. + av1_zero(cm->counts.seg); + + // First of all generate stats regarding how well the last segment map + // predicts this one + for (tile_row = 0; tile_row < cm->tile_rows; tile_row++) { + TileInfo tile_info; + av1_tile_set_row(&tile_info, cm, tile_row); + for (tile_col = 0; tile_col < cm->tile_cols; tile_col++) { + MODE_INFO **mi_ptr; + av1_tile_set_col(&tile_info, cm, tile_col); +#if CONFIG_TILE_GROUPS && CONFIG_DEPENDENT_HORZTILES + av1_tile_set_tg_boundary(&tile_info, cm, tile_row, tile_col); +#endif + mi_ptr = cm->mi_grid_visible + tile_info.mi_row_start * cm->mi_stride + + tile_info.mi_col_start; + for (mi_row = tile_info.mi_row_start; mi_row < tile_info.mi_row_end; + mi_row += cm->mib_size, mi_ptr += cm->mib_size * cm->mi_stride) { + MODE_INFO **mi = mi_ptr; + for (mi_col = tile_info.mi_col_start; mi_col < tile_info.mi_col_end; + mi_col += cm->mib_size, mi += cm->mib_size) { + count_segs_sb(cm, xd, &tile_info, mi, no_pred_segcounts, + temporal_predictor_count, t_unpred_seg_counts, mi_row, + mi_col, cm->sb_size); + } + } + } + } + + // Work out probability tree for coding segments without prediction + // and the cost. + calc_segtree_probs(no_pred_segcounts, no_pred_tree, segp->tree_probs, probwt); + no_pred_cost = cost_segmap(no_pred_segcounts, no_pred_tree); + + // Key frames cannot use temporal prediction + if (!frame_is_intra_only(cm) && !cm->error_resilient_mode) { + // Work out probability tree for coding those segments not + // predicted using the temporal method and the cost. + calc_segtree_probs(t_unpred_seg_counts, t_pred_tree, segp->tree_probs, + probwt); + t_pred_cost = cost_segmap(t_unpred_seg_counts, t_pred_tree); + + // Add in the cost of the signaling for each prediction context. + for (i = 0; i < PREDICTION_PROBS; i++) { + const int count0 = temporal_predictor_count[i][0]; + const int count1 = temporal_predictor_count[i][1]; + + t_nopred_prob[i] = get_binary_prob(count0, count1); + av1_prob_diff_update_savings_search( + temporal_predictor_count[i], segp->pred_probs[i], &t_nopred_prob[i], + DIFF_UPDATE_PROB, probwt); + + // Add in the predictor signaling cost + t_pred_cost += count0 * av1_cost_zero(t_nopred_prob[i]) + + count1 * av1_cost_one(t_nopred_prob[i]); + } + } + + // Now choose which coding method to use. + if (t_pred_cost < no_pred_cost) { + assert(!cm->error_resilient_mode); + seg->temporal_update = 1; + } else { + seg->temporal_update = 0; + } +} + +void av1_reset_segment_features(AV1_COMMON *cm) { + struct segmentation *seg = &cm->seg; + + // Set up default state for MB feature flags + seg->enabled = 0; + seg->update_map = 0; + seg->update_data = 0; + av1_clearall_segfeatures(seg); +} diff --git a/third_party/aom/av1/encoder/segmentation.h b/third_party/aom/av1/encoder/segmentation.h new file mode 100644 index 0000000000..c1491ca2af --- /dev/null +++ b/third_party/aom/av1/encoder/segmentation.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_SEGMENTATION_H_ +#define AV1_ENCODER_SEGMENTATION_H_ + +#include "av1/common/blockd.h" +#include "av1/encoder/encoder.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_enable_segmentation(struct segmentation *seg); +void av1_disable_segmentation(struct segmentation *seg); + +void av1_disable_segfeature(struct segmentation *seg, int segment_id, + SEG_LVL_FEATURES feature_id); +void av1_clear_segdata(struct segmentation *seg, int segment_id, + SEG_LVL_FEATURES feature_id); + +// The values given for each segment can be either deltas (from the default +// value chosen for the frame) or absolute values. +// +// Valid range for abs values is (0-127 for MB_LVL_ALT_Q), (0-63 for +// SEGMENT_ALT_LF) +// Valid range for delta values are (+/-127 for MB_LVL_ALT_Q), (+/-63 for +// SEGMENT_ALT_LF) +// +// abs_delta = SEGMENT_DELTADATA (deltas) abs_delta = SEGMENT_ABSDATA (use +// the absolute values given). +void av1_set_segment_data(struct segmentation *seg, signed char *feature_data, + unsigned char abs_delta); + +void av1_choose_segmap_coding_method(AV1_COMMON *cm, MACROBLOCKD *xd); + +void av1_reset_segment_features(AV1_COMMON *cm); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_SEGMENTATION_H_ diff --git a/third_party/aom/av1/encoder/speed_features.c b/third_party/aom/av1/encoder/speed_features.c new file mode 100644 index 0000000000..20c96761b7 --- /dev/null +++ b/third_party/aom/av1/encoder/speed_features.c @@ -0,0 +1,506 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/encoder/encoder.h" +#include "av1/encoder/speed_features.h" +#include "av1/encoder/rdopt.h" + +#include "aom_dsp/aom_dsp_common.h" + +#define MAX_MESH_SPEED 5 // Max speed setting for mesh motion method +static MESH_PATTERN + good_quality_mesh_patterns[MAX_MESH_SPEED + 1][MAX_MESH_STEP] = { + { { 64, 8 }, { 28, 4 }, { 15, 1 }, { 7, 1 } }, + { { 64, 8 }, { 28, 4 }, { 15, 1 }, { 7, 1 } }, + { { 64, 8 }, { 14, 2 }, { 7, 1 }, { 7, 1 } }, + { { 64, 16 }, { 24, 8 }, { 12, 4 }, { 7, 1 } }, + { { 64, 16 }, { 24, 8 }, { 12, 4 }, { 7, 1 } }, + { { 64, 16 }, { 24, 8 }, { 12, 4 }, { 7, 1 } }, + }; +static unsigned char good_quality_max_mesh_pct[MAX_MESH_SPEED + 1] = { + 50, 25, 15, 5, 1, 1 +}; + +#if CONFIG_INTRABC +// TODO(aconverse@google.com): These settings are pretty relaxed, tune them for +// each speed setting +static MESH_PATTERN intrabc_mesh_patterns[MAX_MESH_SPEED + 1][MAX_MESH_STEP] = { + { { 64, 1 }, { 64, 1 }, { 0, 0 }, { 0, 0 } }, + { { 64, 1 }, { 64, 1 }, { 0, 0 }, { 0, 0 } }, + { { 64, 1 }, { 64, 1 }, { 0, 0 }, { 0, 0 } }, + { { 64, 4 }, { 16, 1 }, { 0, 0 }, { 0, 0 } }, + { { 64, 4 }, { 16, 1 }, { 0, 0 }, { 0, 0 } }, + { { 64, 4 }, { 16, 1 }, { 0, 0 }, { 0, 0 } }, +}; +static uint8_t intrabc_max_mesh_pct[MAX_MESH_SPEED + 1] = { 100, 100, 100, + 25, 25, 10 }; +#endif + +// Intra only frames, golden frames (except alt ref overlays) and +// alt ref frames tend to be coded at a higher than ambient quality +static int frame_is_boosted(const AV1_COMP *cpi) { + return frame_is_kf_gf_arf(cpi); +} + +// Sets a partition size down to which the auto partition code will always +// search (can go lower), based on the image dimensions. The logic here +// is that the extent to which ringing artefacts are offensive, depends +// partly on the screen area that over which they propogate. Propogation is +// limited by transform block size but the screen area take up by a given block +// size will be larger for a small image format stretched to full screen. +static BLOCK_SIZE set_partition_min_limit(AV1_COMMON *const cm) { + unsigned int screen_area = (cm->width * cm->height); + + // Select block size based on image format size. + if (screen_area < 1280 * 720) { + // Formats smaller in area than 720P + return BLOCK_4X4; + } else if (screen_area < 1920 * 1080) { + // Format >= 720P and < 1080P + return BLOCK_8X8; + } else { + // Formats 1080P and up + return BLOCK_16X16; + } +} + +static void set_good_speed_feature_framesize_dependent(AV1_COMP *cpi, + SPEED_FEATURES *sf, + int speed) { + AV1_COMMON *const cm = &cpi->common; + + if (speed >= 1) { + if (AOMMIN(cm->width, cm->height) >= 720) { + sf->disable_split_mask = + cm->show_frame ? DISABLE_ALL_SPLIT : DISABLE_ALL_INTER_SPLIT; + sf->partition_search_breakout_dist_thr = (1 << 23); + } else { + sf->disable_split_mask = DISABLE_COMPOUND_SPLIT; + sf->partition_search_breakout_dist_thr = (1 << 21); + } + } + + if (speed >= 2) { + if (AOMMIN(cm->width, cm->height) >= 720) { + sf->disable_split_mask = + cm->show_frame ? DISABLE_ALL_SPLIT : DISABLE_ALL_INTER_SPLIT; + sf->adaptive_pred_interp_filter = 0; + sf->partition_search_breakout_dist_thr = (1 << 24); + sf->partition_search_breakout_rate_thr = 120; + } else { + sf->disable_split_mask = LAST_AND_INTRA_SPLIT_ONLY; + sf->partition_search_breakout_dist_thr = (1 << 22); + sf->partition_search_breakout_rate_thr = 100; + } + sf->rd_auto_partition_min_limit = set_partition_min_limit(cm); + } + + if (speed >= 3) { + if (AOMMIN(cm->width, cm->height) >= 720) { + sf->disable_split_mask = DISABLE_ALL_SPLIT; + sf->schedule_mode_search = cm->base_qindex < 220 ? 1 : 0; + sf->partition_search_breakout_dist_thr = (1 << 25); + sf->partition_search_breakout_rate_thr = 200; + } else { + sf->max_intra_bsize = BLOCK_32X32; + sf->disable_split_mask = DISABLE_ALL_INTER_SPLIT; + sf->schedule_mode_search = cm->base_qindex < 175 ? 1 : 0; + sf->partition_search_breakout_dist_thr = (1 << 23); + sf->partition_search_breakout_rate_thr = 120; + } + } + + // If this is a two pass clip that fits the criteria for animated or + // graphics content then reset disable_split_mask for speeds 1-4. + // Also if the image edge is internal to the coded area. + if ((speed >= 1) && (cpi->oxcf.pass == 2) && + ((cpi->twopass.fr_content_type == FC_GRAPHICS_ANIMATION) || + (av1_internal_image_edge(cpi)))) { + sf->disable_split_mask = DISABLE_COMPOUND_SPLIT; + } + + if (speed >= 4) { + if (AOMMIN(cm->width, cm->height) >= 720) { + sf->partition_search_breakout_dist_thr = (1 << 26); + } else { + sf->partition_search_breakout_dist_thr = (1 << 24); + } + sf->disable_split_mask = DISABLE_ALL_SPLIT; + } +} + +static void set_good_speed_feature(AV1_COMP *cpi, AV1_COMMON *cm, + SPEED_FEATURES *sf, int speed) { + const int boosted = frame_is_boosted(cpi); + + if (speed >= 1) { + sf->tx_type_search.fast_intra_tx_type_search = 1; + sf->tx_type_search.fast_inter_tx_type_search = 1; + } + + if (speed >= 2) { + if ((cpi->twopass.fr_content_type == FC_GRAPHICS_ANIMATION) || + av1_internal_image_edge(cpi)) { + sf->use_square_partition_only = !frame_is_boosted(cpi); + } else { + sf->use_square_partition_only = !frame_is_intra_only(cm); + } + + sf->less_rectangular_check = 1; + + sf->use_rd_breakout = 1; + sf->adaptive_motion_search = 1; + sf->mv.auto_mv_step_size = 1; + sf->adaptive_rd_thresh = 1; + sf->mv.subpel_iters_per_step = 1; + sf->mode_skip_start = 10; + sf->adaptive_pred_interp_filter = 1; + + sf->recode_loop = ALLOW_RECODE_KFARFGF; +#if CONFIG_TX64X64 + sf->intra_y_mode_mask[TX_64X64] = INTRA_DC_H_V; + sf->intra_uv_mode_mask[TX_64X64] = INTRA_DC_H_V; +#endif // CONFIG_TX64X64 + sf->intra_y_mode_mask[TX_32X32] = INTRA_DC_H_V; + sf->intra_uv_mode_mask[TX_32X32] = INTRA_DC_H_V; + sf->intra_y_mode_mask[TX_16X16] = INTRA_DC_H_V; + sf->intra_uv_mode_mask[TX_16X16] = INTRA_DC_H_V; + + sf->tx_size_search_breakout = 1; + sf->partition_search_breakout_rate_thr = 80; + sf->tx_type_search.prune_mode = PRUNE_ONE; + // Use transform domain distortion. + // Note var-tx expt always uses pixel domain distortion. + sf->use_transform_domain_distortion = 1; +#if CONFIG_EXT_INTER + sf->disable_wedge_search_var_thresh = 100; + sf->fast_wedge_sign_estimate = 1; +#endif // CONFIG_EXT_INTER + } + + if (speed >= 3) { + sf->tx_size_search_method = + frame_is_boosted(cpi) ? USE_FULL_RD : USE_LARGESTALL; + sf->mode_search_skip_flags = + (cm->frame_type == KEY_FRAME) + ? 0 + : FLAG_SKIP_INTRA_DIRMISMATCH | FLAG_SKIP_INTRA_BESTINTER | + FLAG_SKIP_COMP_BESTINTRA | FLAG_SKIP_INTRA_LOWVAR; + sf->disable_filter_search_var_thresh = 100; + sf->comp_inter_joint_search_thresh = BLOCK_SIZES; + sf->auto_min_max_partition_size = RELAXED_NEIGHBORING_MIN_MAX; + sf->allow_partition_search_skip = 1; + sf->use_upsampled_references = 0; + sf->adaptive_rd_thresh = 2; +#if CONFIG_EXT_TX + sf->tx_type_search.prune_mode = PRUNE_TWO; +#endif + } + + if (speed >= 4) { + sf->use_square_partition_only = !frame_is_intra_only(cm); + sf->tx_size_search_method = + frame_is_intra_only(cm) ? USE_FULL_RD : USE_LARGESTALL; + sf->mv.subpel_search_method = SUBPEL_TREE_PRUNED; + sf->adaptive_pred_interp_filter = 0; + sf->adaptive_mode_search = 1; + sf->cb_partition_search = !boosted; + sf->cb_pred_filter_search = 1; + sf->alt_ref_search_fp = 1; + sf->recode_loop = ALLOW_RECODE_KFMAXBW; + sf->adaptive_rd_thresh = 3; + sf->mode_skip_start = 6; +#if CONFIG_TX64X64 + sf->intra_y_mode_mask[TX_64X64] = INTRA_DC; + sf->intra_uv_mode_mask[TX_64X64] = INTRA_DC; +#endif // CONFIG_TX64X64 + sf->intra_y_mode_mask[TX_32X32] = INTRA_DC; + sf->intra_uv_mode_mask[TX_32X32] = INTRA_DC; + sf->adaptive_interp_filter_search = 1; + } + + if (speed >= 5) { + sf->use_square_partition_only = 1; + sf->tx_size_search_method = USE_LARGESTALL; + sf->mv.search_method = BIGDIA; + sf->mv.subpel_search_method = SUBPEL_TREE_PRUNED_MORE; + sf->adaptive_rd_thresh = 4; + if (cm->frame_type != KEY_FRAME) + sf->mode_search_skip_flags |= FLAG_EARLY_TERMINATE; + sf->disable_filter_search_var_thresh = 200; + sf->use_fast_coef_updates = ONE_LOOP_REDUCED; + sf->use_fast_coef_costing = 1; + sf->partition_search_breakout_rate_thr = 300; + } + + if (speed >= 6) { + int i; + sf->optimize_coefficients = 0; + sf->mv.search_method = HEX; + sf->disable_filter_search_var_thresh = 500; + for (i = 0; i < TX_SIZES; ++i) { + sf->intra_y_mode_mask[i] = INTRA_DC; + sf->intra_uv_mode_mask[i] = INTRA_DC; + } + sf->partition_search_breakout_rate_thr = 500; + sf->mv.reduce_first_step_size = 1; + sf->simple_model_rd_from_var = 1; + } + if (speed >= 7) { + const int is_keyframe = cm->frame_type == KEY_FRAME; + const int frames_since_key = is_keyframe ? 0 : cpi->rc.frames_since_key; + sf->default_max_partition_size = BLOCK_32X32; + sf->default_min_partition_size = BLOCK_8X8; +#if CONFIG_TX64X64 + sf->intra_y_mode_mask[TX_64X64] = INTRA_DC; +#endif // CONFIG_TX64X64 + sf->intra_y_mode_mask[TX_32X32] = INTRA_DC; + sf->frame_parameter_update = 0; + sf->mv.search_method = FAST_HEX; + sf->inter_mode_mask[BLOCK_32X32] = INTER_NEAREST_NEAR_NEW; + sf->inter_mode_mask[BLOCK_32X64] = INTER_NEAREST; + sf->inter_mode_mask[BLOCK_64X32] = INTER_NEAREST; + sf->inter_mode_mask[BLOCK_64X64] = INTER_NEAREST; +#if CONFIG_EXT_PARTITION + sf->inter_mode_mask[BLOCK_64X128] = INTER_NEAREST; + sf->inter_mode_mask[BLOCK_128X64] = INTER_NEAREST; + sf->inter_mode_mask[BLOCK_128X128] = INTER_NEAREST; +#endif // CONFIG_EXT_PARTITION + sf->partition_search_type = REFERENCE_PARTITION; + sf->default_min_partition_size = BLOCK_8X8; + sf->reuse_inter_pred_sby = 1; + sf->force_frame_boost = + is_keyframe || + (frames_since_key % (sf->last_partitioning_redo_frequency << 1) == 1); + sf->max_delta_qindex = is_keyframe ? 20 : 15; + sf->coeff_prob_appx_step = 4; + sf->mode_search_skip_flags |= FLAG_SKIP_INTRA_DIRMISMATCH; + } +} + +void av1_set_speed_features_framesize_dependent(AV1_COMP *cpi) { + SPEED_FEATURES *const sf = &cpi->sf; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + AV1_COMMON *const cm = &cpi->common; + RD_OPT *const rd = &cpi->rd; + int i; + +// Limit memory usage for high resolutions +#if CONFIG_EXT_REFS + // TODO(zoeliu): Temporary solution to resolve the insufficient RAM issue for + // ext-refs. Need to work with @yunqingwang to have a more + // effective solution. + if (AOMMIN(cm->width, cm->height) > 720) { + // Turn off the use of upsampled references for HD resolution + sf->use_upsampled_references = 0; + } else if ((AOMMIN(cm->width, cm->height) > 540) && + (oxcf->profile != PROFILE_0)) { + sf->use_upsampled_references = 0; + } +#else + if (AOMMIN(cm->width, cm->height) > 1080) { + sf->use_upsampled_references = 0; + } else if ((AOMMIN(cm->width, cm->height) > 720) && + (oxcf->profile != PROFILE_0)) { + sf->use_upsampled_references = 0; + } +#endif // CONFIG_EXT_REFS + + if (oxcf->mode == GOOD) { + set_good_speed_feature_framesize_dependent(cpi, sf, oxcf->speed); + } + + if (sf->disable_split_mask == DISABLE_ALL_SPLIT) { + sf->adaptive_pred_interp_filter = 0; + } + + // Check for masked out split cases. + for (i = 0; i < MAX_REFS; ++i) { + if (sf->disable_split_mask & (1 << i)) { + rd->thresh_mult_sub8x8[i] = INT_MAX; + } + } + + // This is only used in motion vector unit test. + if (cpi->oxcf.motion_vector_unit_test == 1) + cpi->find_fractional_mv_step = av1_return_max_sub_pixel_mv; + else if (cpi->oxcf.motion_vector_unit_test == 2) + cpi->find_fractional_mv_step = av1_return_min_sub_pixel_mv; +} + +void av1_set_speed_features_framesize_independent(AV1_COMP *cpi) { + SPEED_FEATURES *const sf = &cpi->sf; + AV1_COMMON *const cm = &cpi->common; + MACROBLOCK *const x = &cpi->td.mb; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + int i; + + // best quality defaults + sf->frame_parameter_update = 1; + sf->mv.search_method = NSTEP; + sf->recode_loop = ALLOW_RECODE; + sf->mv.subpel_search_method = SUBPEL_TREE; + sf->mv.subpel_iters_per_step = 2; + sf->mv.subpel_force_stop = 0; + sf->optimize_coefficients = !is_lossless_requested(&cpi->oxcf); + sf->mv.reduce_first_step_size = 0; + sf->coeff_prob_appx_step = 1; + sf->mv.auto_mv_step_size = 0; + sf->mv.fullpel_search_step_param = 6; + sf->comp_inter_joint_search_thresh = BLOCK_4X4; + sf->adaptive_rd_thresh = 0; + sf->tx_size_search_method = USE_FULL_RD; + sf->adaptive_motion_search = 0; + sf->adaptive_pred_interp_filter = 0; + sf->adaptive_mode_search = 0; + sf->cb_pred_filter_search = 0; + sf->cb_partition_search = 0; + sf->alt_ref_search_fp = 0; + sf->partition_search_type = SEARCH_PARTITION; + sf->tx_type_search.prune_mode = NO_PRUNE; + sf->tx_type_search.fast_intra_tx_type_search = 0; + sf->tx_type_search.fast_inter_tx_type_search = 0; + sf->less_rectangular_check = 0; + sf->use_square_partition_only = 0; + sf->auto_min_max_partition_size = NOT_IN_USE; + sf->rd_auto_partition_min_limit = BLOCK_4X4; + sf->default_max_partition_size = BLOCK_LARGEST; + sf->default_min_partition_size = BLOCK_4X4; + sf->adjust_partitioning_from_last_frame = 0; + sf->last_partitioning_redo_frequency = 4; + sf->disable_split_mask = 0; + sf->mode_search_skip_flags = 0; + sf->force_frame_boost = 0; + sf->max_delta_qindex = 0; + sf->disable_filter_search_var_thresh = 0; + sf->adaptive_interp_filter_search = 0; + sf->allow_partition_search_skip = 0; + sf->use_upsampled_references = 1; +#if CONFIG_EXT_INTER + sf->disable_wedge_search_var_thresh = 0; + sf->fast_wedge_sign_estimate = 0; +#endif // CONFIG_EXT_INTER + + for (i = 0; i < TX_SIZES; i++) { + sf->intra_y_mode_mask[i] = INTRA_ALL; + sf->intra_uv_mode_mask[i] = INTRA_ALL; + } + sf->use_rd_breakout = 0; + sf->lpf_pick = LPF_PICK_FROM_FULL_IMAGE; + sf->use_fast_coef_updates = TWO_LOOP; + sf->use_fast_coef_costing = 0; + sf->mode_skip_start = MAX_MODES; // Mode index at which mode skip mask set + sf->schedule_mode_search = 0; + for (i = 0; i < BLOCK_SIZES; ++i) sf->inter_mode_mask[i] = INTER_ALL; + sf->max_intra_bsize = BLOCK_LARGEST; + sf->reuse_inter_pred_sby = 0; + // This setting only takes effect when partition_search_type is set + // to FIXED_PARTITION. + sf->always_this_block_size = BLOCK_16X16; + sf->search_type_check_frequency = 50; + // Recode loop tolerance %. + sf->recode_tolerance = 25; + sf->default_interp_filter = SWITCHABLE; + sf->tx_size_search_breakout = 0; + sf->partition_search_breakout_dist_thr = 0; + sf->partition_search_breakout_rate_thr = 0; + sf->simple_model_rd_from_var = 0; + + // Set this at the appropriate speed levels + sf->use_transform_domain_distortion = 0; + + if (oxcf->mode == GOOD +#if CONFIG_XIPHRC + || oxcf->pass == 1 +#endif + ) + set_good_speed_feature(cpi, cm, sf, oxcf->speed); + + // sf->partition_search_breakout_dist_thr is set assuming max 64x64 + // blocks. Normalise this if the blocks are bigger. + if (MAX_SB_SIZE_LOG2 > 6) { + sf->partition_search_breakout_dist_thr <<= 2 * (MAX_SB_SIZE_LOG2 - 6); + } + + cpi->full_search_sad = av1_full_search_sad; + cpi->diamond_search_sad = av1_diamond_search_sad; + + sf->allow_exhaustive_searches = 1; + int speed = (oxcf->speed > MAX_MESH_SPEED) ? MAX_MESH_SPEED : oxcf->speed; + if (cpi->twopass.fr_content_type == FC_GRAPHICS_ANIMATION) + sf->exhaustive_searches_thresh = (1 << 24); + else + sf->exhaustive_searches_thresh = (1 << 25); + sf->max_exaustive_pct = good_quality_max_mesh_pct[speed]; + if (speed > 0) + sf->exhaustive_searches_thresh = sf->exhaustive_searches_thresh << 1; + + for (i = 0; i < MAX_MESH_STEP; ++i) { + sf->mesh_patterns[i].range = good_quality_mesh_patterns[speed][i].range; + sf->mesh_patterns[i].interval = + good_quality_mesh_patterns[speed][i].interval; + } +#if CONFIG_INTRABC + if ((frame_is_intra_only(cm) && cm->allow_screen_content_tools) && + (cpi->twopass.fr_content_type == FC_GRAPHICS_ANIMATION || + cpi->oxcf.content == AOM_CONTENT_SCREEN)) { + for (i = 0; i < MAX_MESH_STEP; ++i) { + sf->mesh_patterns[i].range = intrabc_mesh_patterns[speed][i].range; + sf->mesh_patterns[i].interval = intrabc_mesh_patterns[speed][i].interval; + } + sf->max_exaustive_pct = intrabc_max_mesh_pct[speed]; + } +#endif // CONFIG_INTRABC + +#if !CONFIG_XIPHRC + // Slow quant, dct and trellis not worthwhile for first pass + // so make sure they are always turned off. + if (oxcf->pass == 1) sf->optimize_coefficients = 0; +#endif + + // No recode for 1 pass. + if (oxcf->pass == 0) { + sf->recode_loop = DISALLOW_RECODE; + sf->optimize_coefficients = 0; + } + + if (sf->mv.subpel_search_method == SUBPEL_TREE) { + cpi->find_fractional_mv_step = av1_find_best_sub_pixel_tree; + } else if (sf->mv.subpel_search_method == SUBPEL_TREE_PRUNED) { + cpi->find_fractional_mv_step = av1_find_best_sub_pixel_tree_pruned; + } else if (sf->mv.subpel_search_method == SUBPEL_TREE_PRUNED_MORE) { + cpi->find_fractional_mv_step = av1_find_best_sub_pixel_tree_pruned_more; + } else if (sf->mv.subpel_search_method == SUBPEL_TREE_PRUNED_EVENMORE) { + cpi->find_fractional_mv_step = av1_find_best_sub_pixel_tree_pruned_evenmore; + } + +#if !CONFIG_AOM_QM + x->optimize = sf->optimize_coefficients == 1 && oxcf->pass != 1; +#else + // FIXME: trellis not very efficient for quantisation matrices + x->optimize = 0; +#endif + + x->min_partition_size = sf->default_min_partition_size; + x->max_partition_size = sf->default_max_partition_size; + + if (!cpi->oxcf.frame_periodic_boost) { + sf->max_delta_qindex = 0; + } + + // This is only used in motion vector unit test. + if (cpi->oxcf.motion_vector_unit_test == 1) + cpi->find_fractional_mv_step = av1_return_max_sub_pixel_mv; + else if (cpi->oxcf.motion_vector_unit_test == 2) + cpi->find_fractional_mv_step = av1_return_min_sub_pixel_mv; +} diff --git a/third_party/aom/av1/encoder/speed_features.h b/third_party/aom/av1/encoder/speed_features.h new file mode 100644 index 0000000000..af54a1a9ae --- /dev/null +++ b/third_party/aom/av1/encoder/speed_features.h @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_SPEED_FEATURES_H_ +#define AV1_ENCODER_SPEED_FEATURES_H_ + +#include "av1/common/enums.h" + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + INTRA_ALL = (1 << DC_PRED) | (1 << V_PRED) | (1 << H_PRED) | (1 << D45_PRED) | + (1 << D135_PRED) | (1 << D117_PRED) | (1 << D153_PRED) | + (1 << D207_PRED) | (1 << D63_PRED) | +#if CONFIG_ALT_INTRA + (1 << SMOOTH_PRED) | +#endif // CONFIG_ALT_INTRA + (1 << TM_PRED), + INTRA_DC = (1 << DC_PRED), + INTRA_DC_TM = (1 << DC_PRED) | (1 << TM_PRED), + INTRA_DC_H_V = (1 << DC_PRED) | (1 << V_PRED) | (1 << H_PRED), + INTRA_DC_TM_H_V = + (1 << DC_PRED) | (1 << TM_PRED) | (1 << V_PRED) | (1 << H_PRED) +}; + +#if CONFIG_EXT_INTER +enum { + INTER_ALL = (1 << NEARESTMV) | (1 << NEARMV) | (1 << ZEROMV) | (1 << NEWMV) | + (1 << NEAREST_NEARESTMV) | (1 << NEAR_NEARMV) | + (1 << NEAREST_NEARMV) | (1 << NEAR_NEARESTMV) | (1 << NEW_NEWMV) | + (1 << NEAREST_NEWMV) | (1 << NEAR_NEWMV) | (1 << NEW_NEARMV) | + (1 << NEW_NEARESTMV) | (1 << ZERO_ZEROMV), + INTER_NEAREST = (1 << NEARESTMV) | (1 << NEAREST_NEARESTMV) | + (1 << NEAREST_NEARMV) | (1 << NEAR_NEARESTMV) | + (1 << NEW_NEARESTMV) | (1 << NEAREST_NEWMV), + INTER_NEAREST_NEW = (1 << NEARESTMV) | (1 << NEWMV) | + (1 << NEAREST_NEARESTMV) | (1 << NEW_NEWMV) | + (1 << NEAR_NEARESTMV) | (1 << NEAREST_NEARMV) | + (1 << NEW_NEARESTMV) | (1 << NEAREST_NEWMV) | + (1 << NEW_NEARMV) | (1 << NEAR_NEWMV), + INTER_NEAREST_ZERO = (1 << NEARESTMV) | (1 << ZEROMV) | + (1 << NEAREST_NEARESTMV) | (1 << ZERO_ZEROMV) | + (1 << NEAREST_NEARMV) | (1 << NEAR_NEARESTMV) | + (1 << NEAREST_NEWMV) | (1 << NEW_NEARESTMV), + INTER_NEAREST_NEW_ZERO = + (1 << NEARESTMV) | (1 << ZEROMV) | (1 << NEWMV) | + (1 << NEAREST_NEARESTMV) | (1 << ZERO_ZEROMV) | (1 << NEW_NEWMV) | + (1 << NEAREST_NEARMV) | (1 << NEAR_NEARESTMV) | (1 << NEW_NEARESTMV) | + (1 << NEAREST_NEWMV) | (1 << NEW_NEARMV) | (1 << NEAR_NEWMV), + INTER_NEAREST_NEAR_NEW = + (1 << NEARESTMV) | (1 << NEARMV) | (1 << NEWMV) | + (1 << NEAREST_NEARESTMV) | (1 << NEW_NEWMV) | (1 << NEAREST_NEARMV) | + (1 << NEAR_NEARESTMV) | (1 << NEW_NEARESTMV) | (1 << NEAREST_NEWMV) | + (1 << NEW_NEARMV) | (1 << NEAR_NEWMV) | (1 << NEAR_NEARMV), + INTER_NEAREST_NEAR_ZERO = + (1 << NEARESTMV) | (1 << NEARMV) | (1 << ZEROMV) | + (1 << NEAREST_NEARESTMV) | (1 << ZERO_ZEROMV) | (1 << NEAREST_NEARMV) | + (1 << NEAR_NEARESTMV) | (1 << NEAREST_NEWMV) | (1 << NEW_NEARESTMV) | + (1 << NEW_NEARMV) | (1 << NEAR_NEWMV) | (1 << NEAR_NEARMV), +}; +#else +enum { + INTER_ALL = (1 << NEARESTMV) | (1 << NEARMV) | (1 << ZEROMV) | (1 << NEWMV), + INTER_NEAREST = (1 << NEARESTMV), + INTER_NEAREST_NEW = (1 << NEARESTMV) | (1 << NEWMV), + INTER_NEAREST_ZERO = (1 << NEARESTMV) | (1 << ZEROMV), + INTER_NEAREST_NEW_ZERO = (1 << NEARESTMV) | (1 << ZEROMV) | (1 << NEWMV), + INTER_NEAREST_NEAR_NEW = (1 << NEARESTMV) | (1 << NEARMV) | (1 << NEWMV), + INTER_NEAREST_NEAR_ZERO = (1 << NEARESTMV) | (1 << NEARMV) | (1 << ZEROMV), +}; +#endif // CONFIG_EXT_INTER + +enum { + DISABLE_ALL_INTER_SPLIT = (1 << THR_COMP_GA) | (1 << THR_COMP_LA) | + (1 << THR_ALTR) | (1 << THR_GOLD) | (1 << THR_LAST), + + DISABLE_ALL_SPLIT = (1 << THR_INTRA) | DISABLE_ALL_INTER_SPLIT, + + DISABLE_COMPOUND_SPLIT = (1 << THR_COMP_GA) | (1 << THR_COMP_LA), + + LAST_AND_INTRA_SPLIT_ONLY = (1 << THR_COMP_GA) | (1 << THR_COMP_LA) | + (1 << THR_ALTR) | (1 << THR_GOLD) +}; + +typedef enum { + DIAMOND = 0, + NSTEP = 1, + HEX = 2, + BIGDIA = 3, + SQUARE = 4, + FAST_HEX = 5, + FAST_DIAMOND = 6 +} SEARCH_METHODS; + +typedef enum { + // No recode. + DISALLOW_RECODE = 0, + // Allow recode for KF and exceeding maximum frame bandwidth. + ALLOW_RECODE_KFMAXBW = 1, + // Allow recode only for KF/ARF/GF frames. + ALLOW_RECODE_KFARFGF = 2, + // Allow recode for all frames based on bitrate constraints. + ALLOW_RECODE = 3, +} RECODE_LOOP_TYPE; + +typedef enum { + SUBPEL_TREE = 0, + SUBPEL_TREE_PRUNED = 1, // Prunes 1/2-pel searches + SUBPEL_TREE_PRUNED_MORE = 2, // Prunes 1/2-pel searches more aggressively + SUBPEL_TREE_PRUNED_EVENMORE = 3, // Prunes 1/2- and 1/4-pel searches + // Other methods to come +} SUBPEL_SEARCH_METHODS; + +typedef enum { + NO_MOTION_THRESHOLD = 0, + LOW_MOTION_THRESHOLD = 7 +} MOTION_THRESHOLD; + +typedef enum { + USE_FULL_RD = 0, + USE_LARGESTALL, + USE_TX_8X8 +} TX_SIZE_SEARCH_METHOD; + +typedef enum { + NOT_IN_USE = 0, + RELAXED_NEIGHBORING_MIN_MAX = 1 +} AUTO_MIN_MAX_MODE; + +typedef enum { + // Try the full image with different values. + LPF_PICK_FROM_FULL_IMAGE, + // Try a small portion of the image with different values. + LPF_PICK_FROM_SUBIMAGE, + // Estimate the level based on quantizer and frame type + LPF_PICK_FROM_Q, + // Pick 0 to disable LPF if LPF was enabled last frame + LPF_PICK_MINIMAL_LPF +} LPF_PICK_METHOD; + +typedef enum { + // Terminate search early based on distortion so far compared to + // qp step, distortion in the neighborhood of the frame, etc. + FLAG_EARLY_TERMINATE = 1 << 0, + + // Skips comp inter modes if the best so far is an intra mode. + FLAG_SKIP_COMP_BESTINTRA = 1 << 1, + + // Skips oblique intra modes if the best so far is an inter mode. + FLAG_SKIP_INTRA_BESTINTER = 1 << 3, + + // Skips oblique intra modes at angles 27, 63, 117, 153 if the best + // intra so far is not one of the neighboring directions. + FLAG_SKIP_INTRA_DIRMISMATCH = 1 << 4, + + // Skips intra modes other than DC_PRED if the source variance is small + FLAG_SKIP_INTRA_LOWVAR = 1 << 5, +} MODE_SEARCH_SKIP_LOGIC; + +typedef enum { + FLAG_SKIP_EIGHTTAP_REGULAR = 1 << EIGHTTAP_REGULAR, + FLAG_SKIP_EIGHTTAP_SMOOTH = 1 << EIGHTTAP_SMOOTH, + FLAG_SKIP_MULTITAP_SHARP = 1 << MULTITAP_SHARP, +} INTERP_FILTER_MASK; + +typedef enum { + NO_PRUNE = 0, + // eliminates one tx type in vertical and horizontal direction + PRUNE_ONE = 1, +#if CONFIG_EXT_TX + // eliminates two tx types in each direction + PRUNE_TWO = 2, +#endif +} TX_TYPE_PRUNE_MODE; + +typedef struct { + TX_TYPE_PRUNE_MODE prune_mode; + int fast_intra_tx_type_search; + int fast_inter_tx_type_search; +} TX_TYPE_SEARCH; + +typedef enum { + // Search partitions using RD criterion + SEARCH_PARTITION, + + // Always use a fixed size partition + FIXED_PARTITION, + + REFERENCE_PARTITION, + + // Use an arbitrary partitioning scheme based on source variance within + // a 64X64 SB + VAR_BASED_PARTITION, + + // Use non-fixed partitions based on source variance + SOURCE_VAR_BASED_PARTITION +} PARTITION_SEARCH_TYPE; + +typedef enum { + // Does a dry run to see if any of the contexts need to be updated or not, + // before the final run. + TWO_LOOP = 0, + + // No dry run, also only half the coef contexts and bands are updated. + // The rest are not updated at all. + ONE_LOOP_REDUCED = 1 +} FAST_COEFF_UPDATE; + +typedef struct MV_SPEED_FEATURES { + // Motion search method (Diamond, NSTEP, Hex, Big Diamond, Square, etc). + SEARCH_METHODS search_method; + + // This parameter controls which step in the n-step process we start at. + // It's changed adaptively based on circumstances. + int reduce_first_step_size; + + // If this is set to 1, we limit the motion search range to 2 times the + // largest motion vector found in the last frame. + int auto_mv_step_size; + + // Subpel_search_method can only be subpel_tree which does a subpixel + // logarithmic search that keeps stepping at 1/2 pixel units until + // you stop getting a gain, and then goes on to 1/4 and repeats + // the same process. Along the way it skips many diagonals. + SUBPEL_SEARCH_METHODS subpel_search_method; + + // Maximum number of steps in logarithmic subpel search before giving up. + int subpel_iters_per_step; + + // Control when to stop subpel search + int subpel_force_stop; + + // This variable sets the step_param used in full pel motion search. + int fullpel_search_step_param; +} MV_SPEED_FEATURES; + +#define MAX_MESH_STEP 4 + +typedef struct MESH_PATTERN { + int range; + int interval; +} MESH_PATTERN; + +typedef struct SPEED_FEATURES { + MV_SPEED_FEATURES mv; + + // Frame level coding parameter update + int frame_parameter_update; + + RECODE_LOOP_TYPE recode_loop; + + // Trellis (dynamic programming) optimization of quantized values (+1, 0). + int optimize_coefficients; + + // Always set to 0. If on it enables 0 cost background transmission + // (except for the initial transmission of the segmentation). The feature is + // disabled because the addition of very large block sizes make the + // backgrounds very to cheap to encode, and the segmentation we have + // adds overhead. + int static_segmentation; + + // If 1 we iterate finding a best reference for 2 ref frames together - via + // a log search that iterates 4 times (check around mv for last for best + // error of combined predictor then check around mv for alt). If 0 we + // we just use the best motion vector found for each frame by itself. + BLOCK_SIZE comp_inter_joint_search_thresh; + + // This variable is used to cap the maximum number of times we skip testing a + // mode to be evaluated. A high value means we will be faster. + int adaptive_rd_thresh; + + // Coefficient probability model approximation step size + int coeff_prob_appx_step; + + // The threshold is to determine how slow the motino is, it is used when + // use_lastframe_partitioning is set to LAST_FRAME_PARTITION_LOW_MOTION + MOTION_THRESHOLD lf_motion_threshold; + + // Determine which method we use to determine transform size. We can choose + // between options like full rd, largest for prediction size, largest + // for intra and model coefs for the rest. + TX_SIZE_SEARCH_METHOD tx_size_search_method; + + // After looking at the first set of modes (set by index here), skip + // checking modes for reference frames that don't match the reference frame + // of the best so far. + int mode_skip_start; + + PARTITION_SEARCH_TYPE partition_search_type; + + TX_TYPE_SEARCH tx_type_search; + + // Used if partition_search_type = FIXED_SIZE_PARTITION + BLOCK_SIZE always_this_block_size; + + // Skip rectangular partition test when partition type none gives better + // rd than partition type split. + int less_rectangular_check; + + // Disable testing non square partitions. (eg 16x32) + int use_square_partition_only; + + // Sets min and max partition sizes for this superblock based on the + // same superblock in last encoded frame, and the left and above neighbor. + AUTO_MIN_MAX_MODE auto_min_max_partition_size; + // Ensures the rd based auto partition search will always + // go down at least to the specified level. + BLOCK_SIZE rd_auto_partition_min_limit; + + // Min and max partition size we enable (block_size) as per auto + // min max, but also used by adjust partitioning, and pick_partitioning. + BLOCK_SIZE default_min_partition_size; + BLOCK_SIZE default_max_partition_size; + + // Whether or not we allow partitions one smaller or one greater than the last + // frame's partitioning. Only used if use_lastframe_partitioning is set. + int adjust_partitioning_from_last_frame; + + // How frequently we re do the partitioning from scratch. Only used if + // use_lastframe_partitioning is set. + int last_partitioning_redo_frequency; + + // Disables sub 8x8 blocksizes in different scenarios: Choices are to disable + // it always, to allow it for only Last frame and Intra, disable it for all + // inter modes or to enable it always. + int disable_split_mask; + + // TODO(jingning): combine the related motion search speed features + // This allows us to use motion search at other sizes as a starting + // point for this motion search and limits the search range around it. + int adaptive_motion_search; + + // Flag for allowing some use of exhaustive searches; + int allow_exhaustive_searches; + + // Threshold for allowing exhaistive motion search. + int exhaustive_searches_thresh; + + // Maximum number of exhaustive searches for a frame. + int max_exaustive_pct; + + // Pattern to be used for any exhaustive mesh searches. + MESH_PATTERN mesh_patterns[MAX_MESH_STEP]; + + int schedule_mode_search; + + // Allows sub 8x8 modes to use the prediction filter that was determined + // best for 8x8 mode. If set to 0 we always re check all the filters for + // sizes less than 8x8, 1 means we check all filter modes if no 8x8 filter + // was selected, and 2 means we use 8 tap if no 8x8 filter mode was selected. + int adaptive_pred_interp_filter; + + // Adaptive prediction mode search + int adaptive_mode_search; + + // Chessboard pattern prediction filter type search + int cb_pred_filter_search; + + int cb_partition_search; + + int alt_ref_search_fp; + + // Use finer quantizer in every other few frames that run variable block + // partition type search. + int force_frame_boost; + + // Maximally allowed base quantization index fluctuation. + int max_delta_qindex; + + // Implements various heuristics to skip searching modes + // The heuristics selected are based on flags + // defined in the MODE_SEARCH_SKIP_HEURISTICS enum + unsigned int mode_search_skip_flags; + + // A source variance threshold below which filter search is disabled + // Choose a very large value (UINT_MAX) to use 8-tap always + unsigned int disable_filter_search_var_thresh; + +#if CONFIG_EXT_INTER + // A source variance threshold below which wedge search is disabled + unsigned int disable_wedge_search_var_thresh; + + // Whether fast wedge sign estimate is used + int fast_wedge_sign_estimate; +#endif // CONFIG_EXT_INTER + + // These bit masks allow you to enable or disable intra modes for each + // transform size separately. + int intra_y_mode_mask[TX_SIZES]; + int intra_uv_mode_mask[TX_SIZES]; + + // These bit masks allow you to enable or disable intra modes for each + // prediction block size separately. + int intra_y_mode_bsize_mask[BLOCK_SIZES]; + + // This variable enables an early break out of mode testing if the model for + // rd built from the prediction signal indicates a value that's much + // higher than the best rd we've seen so far. + int use_rd_breakout; + + // This feature controls how the loop filter level is determined. + LPF_PICK_METHOD lpf_pick; + + // This feature limits the number of coefficients updates we actually do + // by only looking at counts from 1/2 the bands. + FAST_COEFF_UPDATE use_fast_coef_updates; + + // A binary mask indicating if NEARESTMV, NEARMV, ZEROMV, NEWMV + // modes are used in order from LSB to MSB for each BLOCK_SIZE. + int inter_mode_mask[BLOCK_SIZES]; + + // This feature controls whether we do the expensive context update and + // calculation in the rd coefficient costing loop. + int use_fast_coef_costing; + + // This feature controls the tolerence vs target used in deciding whether to + // recode a frame. It has no meaning if recode is disabled. + int recode_tolerance; + + // This variable controls the maximum block size where intra blocks can be + // used in inter frames. + // TODO(aconverse): Fold this into one of the other many mode skips + BLOCK_SIZE max_intra_bsize; + + // The frequency that we check if SOURCE_VAR_BASED_PARTITION or + // FIXED_PARTITION search type should be used. + int search_type_check_frequency; + + // When partition is pre-set, the inter prediction result from pick_inter_mode + // can be reused in final block encoding process. It is enabled only for real- + // time mode speed 6. + int reuse_inter_pred_sby; + + // default interp filter choice + InterpFilter default_interp_filter; + + // Early termination in transform size search, which only applies while + // tx_size_search_method is USE_FULL_RD. + int tx_size_search_breakout; + + // adaptive interp_filter search to allow skip of certain filter types. + int adaptive_interp_filter_search; + + // mask for skip evaluation of certain interp_filter type. + INTERP_FILTER_MASK interp_filter_search_mask; + + // Partition search early breakout thresholds. + int64_t partition_search_breakout_dist_thr; + int partition_search_breakout_rate_thr; + + // Allow skipping partition search for still image frame + int allow_partition_search_skip; + + // Fast approximation of av1_model_rd_from_var_lapndz + int simple_model_rd_from_var; + + // Do sub-pixel search in up-sampled reference frames + int use_upsampled_references; + + // Whether to compute distortion in the image domain (slower but + // more accurate), or in the transform domain (faster but less acurate). + int use_transform_domain_distortion; +} SPEED_FEATURES; + +struct AV1_COMP; + +void av1_set_speed_features_framesize_independent(struct AV1_COMP *cpi); +void av1_set_speed_features_framesize_dependent(struct AV1_COMP *cpi); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_SPEED_FEATURES_H_ diff --git a/third_party/aom/av1/encoder/subexp.c b/third_party/aom/av1/encoder/subexp.c new file mode 100644 index 0000000000..8960d33414 --- /dev/null +++ b/third_party/aom/av1/encoder/subexp.c @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include "aom_dsp/bitwriter.h" + +#include "av1/common/common.h" +#include "av1/common/entropy.h" +#include "av1/encoder/cost.h" +#include "av1/encoder/subexp.h" + +static const uint8_t update_bits[255] = { + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 0, +}; +#define MIN_DELP_BITS 5 + +static int recenter_nonneg(int v, int m) { + if (v > (m << 1)) + return v; + else if (v >= m) + return ((v - m) << 1); + else + return ((m - v) << 1) - 1; +} + +static int remap_prob(int v, int m) { + int i; + static const uint8_t map_table[MAX_PROB - 1] = { + // generated by: + // map_table[j] = split_index(j, MAX_PROB - 1, MODULUS_PARAM); + 20, 21, 22, 23, 24, 25, 0, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 1, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 2, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 3, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 4, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 5, 86, 87, 88, + 89, 90, 91, 92, 93, 94, 95, 96, 97, 6, 98, 99, 100, 101, 102, + 103, 104, 105, 106, 107, 108, 109, 7, 110, 111, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 121, 8, 122, 123, 124, 125, 126, 127, 128, 129, 130, + 131, 132, 133, 9, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, + 145, 10, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 11, + 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 12, 170, 171, + 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 13, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 14, 194, 195, 196, 197, 198, 199, + 200, 201, 202, 203, 204, 205, 15, 206, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, 217, 16, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 17, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, + 18, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 19, + }; + v--; + m--; + if ((m << 1) <= MAX_PROB) + i = recenter_nonneg(v, m) - 1; + else + i = recenter_nonneg(MAX_PROB - 1 - v, MAX_PROB - 1 - m) - 1; + + i = map_table[i]; + return i; +} + +static int prob_diff_update_cost(aom_prob newp, aom_prob oldp) { + int delp = remap_prob(newp, oldp); + return update_bits[delp] << AV1_PROB_COST_SHIFT; +} + +static void encode_uniform(aom_writer *w, int v) { + const int l = 8; + const int m = (1 << l) - 190; + if (v < m) { + aom_write_literal(w, v, l - 1); + } else { + aom_write_literal(w, m + ((v - m) >> 1), l - 1); + aom_write_literal(w, (v - m) & 1, 1); + } +} + +static INLINE int write_bit_gte(aom_writer *w, int word, int test) { + aom_write_literal(w, word >= test, 1); + return word >= test; +} + +static void encode_term_subexp(aom_writer *w, int word) { + if (!write_bit_gte(w, word, 16)) { + aom_write_literal(w, word, 4); + } else if (!write_bit_gte(w, word, 32)) { + aom_write_literal(w, word - 16, 4); + } else if (!write_bit_gte(w, word, 64)) { + aom_write_literal(w, word - 32, 5); + } else { + encode_uniform(w, word - 64); + } +} + +void av1_write_prob_diff_update(aom_writer *w, aom_prob newp, aom_prob oldp) { + const int delp = remap_prob(newp, oldp); + encode_term_subexp(w, delp); +} + +int av1_prob_diff_update_savings_search(const unsigned int *ct, aom_prob oldp, + aom_prob *bestp, aom_prob upd, + int probwt) { + const uint32_t old_b = cost_branch256(ct, oldp); + int bestsavings = 0; + aom_prob newp, bestnewp = oldp; + const int step = *bestp > oldp ? -1 : 1; + const int upd_cost = av1_cost_one(upd) - av1_cost_zero(upd); + + if (old_b > (uint32_t)upd_cost + (MIN_DELP_BITS << AV1_PROB_COST_SHIFT)) { + for (newp = *bestp; newp != oldp; newp += step) { + const int new_b = cost_branch256(ct, newp); + const int update_b = prob_diff_update_cost(newp, oldp) + upd_cost; + const int savings = (int)((int64_t)old_b - new_b - update_b * probwt); + if (savings > bestsavings) { + bestsavings = savings; + bestnewp = newp; + } + } + } + *bestp = bestnewp; + return bestsavings; +} + +int av1_prob_diff_update_savings_search_model(const unsigned int *ct, + const aom_prob oldp, + aom_prob *bestp, aom_prob upd, + int stepsize, int probwt) { + int i, old_b, new_b, update_b, savings, bestsavings; + int newp; + const int step_sign = *bestp > oldp ? -1 : 1; + const int step = stepsize * step_sign; + const int upd_cost = av1_cost_one(upd) - av1_cost_zero(upd); + const aom_prob *newplist, *oldplist; + aom_prob bestnewp; + oldplist = av1_pareto8_full[oldp - 1]; + old_b = cost_branch256(ct + 2 * PIVOT_NODE, oldp); + for (i = UNCONSTRAINED_NODES; i < ENTROPY_NODES; ++i) + old_b += cost_branch256(ct + 2 * i, oldplist[i - UNCONSTRAINED_NODES]); + + bestsavings = 0; + bestnewp = oldp; + + assert(stepsize > 0); + + if (old_b > upd_cost + (MIN_DELP_BITS << AV1_PROB_COST_SHIFT)) { + for (newp = *bestp; (newp - oldp) * step_sign < 0; newp += step) { + if (newp < 1 || newp > 255) continue; + newplist = av1_pareto8_full[newp - 1]; + new_b = cost_branch256(ct + 2 * PIVOT_NODE, newp); + for (i = UNCONSTRAINED_NODES; i < ENTROPY_NODES; ++i) + new_b += cost_branch256(ct + 2 * i, newplist[i - UNCONSTRAINED_NODES]); + update_b = prob_diff_update_cost(newp, oldp) + upd_cost; + savings = old_b - new_b - update_b * probwt; + if (savings > bestsavings) { + bestsavings = savings; + bestnewp = newp; + } + } + } + + *bestp = bestnewp; + return bestsavings; +} + +#if CONFIG_SUBFRAME_PROB_UPDATE +static int get_cost(unsigned int ct[][2], aom_prob p, int n) { + int i, p0 = p; + unsigned int total_ct[2] = { 0, 0 }; + int cost = 0; + + for (i = 0; i <= n; ++i) { + cost += cost_branch256(ct[i], p); + total_ct[0] += ct[i][0]; + total_ct[1] += ct[i][1]; + if (i < n) + p = av1_merge_probs(p0, total_ct, COEF_COUNT_SAT, COEF_MAX_UPDATE_FACTOR); + } + return cost; +} + +int av1_prob_update_search_subframe(unsigned int ct[][2], aom_prob oldp, + aom_prob *bestp, aom_prob upd, int n) { + const int old_b = get_cost(ct, oldp, n); + int bestsavings = 0; + const int upd_cost = av1_cost_one(upd) - av1_cost_zero(upd); + aom_prob newp, bestnewp = oldp; + const int step = *bestp > oldp ? -1 : 1; + + for (newp = *bestp; newp != oldp; newp += step) { + const int new_b = get_cost(ct, newp, n); + const int update_b = prob_diff_update_cost(newp, oldp) + upd_cost; + const int savings = old_b - new_b - update_b; + if (savings > bestsavings) { + bestsavings = savings; + bestnewp = newp; + } + } + *bestp = bestnewp; + return bestsavings; +} + +int av1_prob_update_search_model_subframe( + unsigned int ct[ENTROPY_NODES][COEF_PROBS_BUFS][2], const aom_prob *oldp, + aom_prob *bestp, aom_prob upd, int stepsize, int n) { + int i, old_b, new_b, update_b, savings, bestsavings; + int newp; + const int step_sign = *bestp > oldp[PIVOT_NODE] ? -1 : 1; + const int step = stepsize * step_sign; + const int upd_cost = av1_cost_one(upd) - av1_cost_zero(upd); + aom_prob bestnewp, newplist[ENTROPY_NODES], oldplist[ENTROPY_NODES]; + av1_model_to_full_probs(oldp, oldplist); + memcpy(newplist, oldp, sizeof(aom_prob) * UNCONSTRAINED_NODES); + for (i = UNCONSTRAINED_NODES, old_b = 0; i < ENTROPY_NODES; ++i) + old_b += get_cost(ct[i], oldplist[i], n); + old_b += get_cost(ct[PIVOT_NODE], oldplist[PIVOT_NODE], n); + + bestsavings = 0; + bestnewp = oldp[PIVOT_NODE]; + + assert(stepsize > 0); + + for (newp = *bestp; (newp - oldp[PIVOT_NODE]) * step_sign < 0; newp += step) { + if (newp < 1 || newp > 255) continue; + newplist[PIVOT_NODE] = newp; + av1_model_to_full_probs(newplist, newplist); + for (i = UNCONSTRAINED_NODES, new_b = 0; i < ENTROPY_NODES; ++i) + new_b += get_cost(ct[i], newplist[i], n); + new_b += get_cost(ct[PIVOT_NODE], newplist[PIVOT_NODE], n); + update_b = prob_diff_update_cost(newp, oldp[PIVOT_NODE]) + upd_cost; + savings = old_b - new_b - update_b; + if (savings > bestsavings) { + bestsavings = savings; + bestnewp = newp; + } + } + + *bestp = bestnewp; + return bestsavings; +} +#endif // CONFIG_SUBFRAME_PROB_UPDATE + +void av1_cond_prob_diff_update(aom_writer *w, aom_prob *oldp, + const unsigned int ct[2], int probwt) { + const aom_prob upd = DIFF_UPDATE_PROB; + aom_prob newp = get_binary_prob(ct[0], ct[1]); + const int savings = + av1_prob_diff_update_savings_search(ct, *oldp, &newp, upd, probwt); + assert(newp >= 1); + if (savings > 0) { + aom_write(w, 1, upd); + av1_write_prob_diff_update(w, newp, *oldp); + *oldp = newp; + } else { + aom_write(w, 0, upd); + } +} + +int av1_cond_prob_diff_update_savings(aom_prob *oldp, const unsigned int ct[2], + int probwt) { + const aom_prob upd = DIFF_UPDATE_PROB; + aom_prob newp = get_binary_prob(ct[0], ct[1]); + const int savings = + av1_prob_diff_update_savings_search(ct, *oldp, &newp, upd, probwt); + return savings; +} diff --git a/third_party/aom/av1/encoder/subexp.h b/third_party/aom/av1/encoder/subexp.h new file mode 100644 index 0000000000..049265cb88 --- /dev/null +++ b/third_party/aom/av1/encoder/subexp.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_SUBEXP_H_ +#define AV1_ENCODER_SUBEXP_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "aom_dsp/bitwriter.h" +#include "aom_dsp/prob.h" + +void av1_write_prob_diff_update(aom_writer *w, aom_prob newp, aom_prob oldpm); + +void av1_cond_prob_diff_update(aom_writer *w, aom_prob *oldp, + const unsigned int ct[2], int probwt); + +int av1_prob_diff_update_savings_search(const unsigned int *ct, aom_prob oldp, + aom_prob *bestp, aom_prob upd, + int probwt); + +int av1_prob_diff_update_savings_search_model(const unsigned int *ct, + const aom_prob oldp, + aom_prob *bestp, aom_prob upd, + int stepsize, int probwt); + +int av1_cond_prob_diff_update_savings(aom_prob *oldp, const unsigned int ct[2], + int probwt); +#if CONFIG_SUBFRAME_PROB_UPDATE +int av1_prob_update_search_subframe(unsigned int ct[][2], aom_prob oldp, + aom_prob *bestp, aom_prob upd, int n); +int av1_prob_update_search_model_subframe( + unsigned int ct[ENTROPY_NODES][COEF_PROBS_BUFS][2], const aom_prob *oldp, + aom_prob *bestp, aom_prob upd, int stepsize, int n); +#endif // CONFIG_SUBFRAME_PROB_UPDATE +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_SUBEXP_H_ diff --git a/third_party/aom/av1/encoder/temporal_filter.c b/third_party/aom/av1/encoder/temporal_filter.c new file mode 100644 index 0000000000..de962fe84d --- /dev/null +++ b/third_party/aom/av1/encoder/temporal_filter.c @@ -0,0 +1,719 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./aom_config.h" +#include "av1/common/alloccommon.h" +#include "av1/common/onyxc_int.h" +#include "av1/common/quant_common.h" +#include "av1/common/reconinter.h" +#include "av1/common/odintrin.h" +#include "av1/encoder/av1_quantize.h" +#include "av1/encoder/extend.h" +#include "av1/encoder/firstpass.h" +#include "av1/encoder/mcomp.h" +#include "av1/encoder/encoder.h" +#include "av1/encoder/ratectrl.h" +#include "av1/encoder/segmentation.h" +#include "av1/encoder/temporal_filter.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" +#include "aom_ports/aom_timer.h" +#include "aom_scale/aom_scale.h" + +static void temporal_filter_predictors_mb_c( + MACROBLOCKD *xd, uint8_t *y_mb_ptr, uint8_t *u_mb_ptr, uint8_t *v_mb_ptr, + int stride, int uv_block_width, int uv_block_height, int mv_row, int mv_col, + uint8_t *pred, struct scale_factors *scale, int x, int y) { + const int which_mv = 0; + const MV mv = { mv_row, mv_col }; + enum mv_precision mv_precision_uv; + int uv_stride; + // TODO(angiebird): change plane setting accordingly + ConvolveParams conv_params = get_conv_params(which_mv, 0); + +#if USE_TEMPORALFILTER_12TAP +#if CONFIG_DUAL_FILTER + const InterpFilter interp_filter[4] = { TEMPORALFILTER_12TAP, + TEMPORALFILTER_12TAP, + TEMPORALFILTER_12TAP, + TEMPORALFILTER_12TAP }; +#else + const InterpFilter interp_filter = TEMPORALFILTER_12TAP; +#endif + (void)xd; +#else + const InterpFilter interp_filter = xd->mi[0]->mbmi.interp_filter; +#endif // USE_TEMPORALFILTER_12TAP +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + WarpTypesAllowed warp_types; + memset(&warp_types, 0, sizeof(WarpTypesAllowed)); +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + + if (uv_block_width == 8) { + uv_stride = (stride + 1) >> 1; + mv_precision_uv = MV_PRECISION_Q4; + } else { + uv_stride = stride; + mv_precision_uv = MV_PRECISION_Q3; + } + +#if CONFIG_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + av1_highbd_build_inter_predictor(y_mb_ptr, stride, &pred[0], 16, &mv, scale, + 16, 16, which_mv, interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, x, y, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + 0, MV_PRECISION_Q3, x, y, xd); + + av1_highbd_build_inter_predictor(u_mb_ptr, uv_stride, &pred[256], + uv_block_width, &mv, scale, uv_block_width, + uv_block_height, which_mv, interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, x, y, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + 1, mv_precision_uv, x, y, xd); + + av1_highbd_build_inter_predictor(v_mb_ptr, uv_stride, &pred[512], + uv_block_width, &mv, scale, uv_block_width, + uv_block_height, which_mv, interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, x, y, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + 2, mv_precision_uv, x, y, xd); + return; + } +#endif // CONFIG_HIGHBITDEPTH + av1_build_inter_predictor(y_mb_ptr, stride, &pred[0], 16, &mv, scale, 16, 16, + &conv_params, interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, x, y, 0, 0, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + MV_PRECISION_Q3, x, y, xd); + + av1_build_inter_predictor(u_mb_ptr, uv_stride, &pred[256], uv_block_width, + &mv, scale, uv_block_width, uv_block_height, + &conv_params, interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, x, y, 1, 0, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + mv_precision_uv, x, y, xd); + + av1_build_inter_predictor(v_mb_ptr, uv_stride, &pred[512], uv_block_width, + &mv, scale, uv_block_width, uv_block_height, + &conv_params, interp_filter, +#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + &warp_types, x, y, 2, 0, +#endif // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION + mv_precision_uv, x, y, xd); +} + +void av1_temporal_filter_apply_c(uint8_t *frame1, unsigned int stride, + uint8_t *frame2, unsigned int block_width, + unsigned int block_height, int strength, + int filter_weight, unsigned int *accumulator, + uint16_t *count) { + unsigned int i, j, k; + int modifier; + int byte = 0; + const int rounding = strength > 0 ? 1 << (strength - 1) : 0; + + for (i = 0, k = 0; i < block_height; i++) { + for (j = 0; j < block_width; j++, k++) { + int pixel_value = *frame2; + + // non-local mean approach + int diff_sse[9] = { 0 }; + int idx, idy, index = 0; + + for (idy = -1; idy <= 1; ++idy) { + for (idx = -1; idx <= 1; ++idx) { + int row = (int)i + idy; + int col = (int)j + idx; + + if (row >= 0 && row < (int)block_height && col >= 0 && + col < (int)block_width) { + int diff = frame1[byte + idy * (int)stride + idx] - + frame2[idy * (int)block_width + idx]; + diff_sse[index] = diff * diff; + ++index; + } + } + } + + assert(index > 0); + + modifier = 0; + for (idx = 0; idx < 9; ++idx) modifier += diff_sse[idx]; + + modifier *= 3; + modifier /= index; + + ++frame2; + + modifier += rounding; + modifier >>= strength; + + if (modifier > 16) modifier = 16; + + modifier = 16 - modifier; + modifier *= filter_weight; + + count[k] += modifier; + accumulator[k] += modifier * pixel_value; + + byte++; + } + + byte += stride - block_width; + } +} + +#if CONFIG_HIGHBITDEPTH +void av1_highbd_temporal_filter_apply_c( + uint8_t *frame1_8, unsigned int stride, uint8_t *frame2_8, + unsigned int block_width, unsigned int block_height, int strength, + int filter_weight, unsigned int *accumulator, uint16_t *count) { + uint16_t *frame1 = CONVERT_TO_SHORTPTR(frame1_8); + uint16_t *frame2 = CONVERT_TO_SHORTPTR(frame2_8); + unsigned int i, j, k; + int modifier; + int byte = 0; + const int rounding = strength > 0 ? 1 << (strength - 1) : 0; + + for (i = 0, k = 0; i < block_height; i++) { + for (j = 0; j < block_width; j++, k++) { + int pixel_value = *frame2; + + // non-local mean approach + int diff_sse[9] = { 0 }; + int idx, idy, index = 0; + + for (idy = -1; idy <= 1; ++idy) { + for (idx = -1; idx <= 1; ++idx) { + int row = (int)i + idy; + int col = (int)j + idx; + + if (row >= 0 && row < (int)block_height && col >= 0 && + col < (int)block_width) { + int diff = frame1[byte + idy * (int)stride + idx] - + frame2[idy * (int)block_width + idx]; + diff_sse[index] = diff * diff; + ++index; + } + } + } + + assert(index > 0); + + modifier = 0; + for (idx = 0; idx < 9; ++idx) modifier += diff_sse[idx]; + + modifier *= 3; + modifier /= index; + + ++frame2; + + modifier += rounding; + modifier >>= strength; + + if (modifier > 16) modifier = 16; + + modifier = 16 - modifier; + modifier *= filter_weight; + + count[k] += modifier; + accumulator[k] += modifier * pixel_value; + + byte++; + } + + byte += stride - block_width; + } +} +#endif // CONFIG_HIGHBITDEPTH + +static int temporal_filter_find_matching_mb_c(AV1_COMP *cpi, + uint8_t *arf_frame_buf, + uint8_t *frame_ptr_buf, + int stride) { + MACROBLOCK *const x = &cpi->td.mb; + MACROBLOCKD *const xd = &x->e_mbd; + const MV_SPEED_FEATURES *const mv_sf = &cpi->sf.mv; + int step_param; + int sadpb = x->sadperbit16; + int bestsme = INT_MAX; + int distortion; + unsigned int sse; + int cost_list[5]; + MvLimits tmp_mv_limits = x->mv_limits; + + MV best_ref_mv1 = { 0, 0 }; + MV best_ref_mv1_full; /* full-pixel value of best_ref_mv1 */ + + // Save input state + struct buf_2d src = x->plane[0].src; + struct buf_2d pre = xd->plane[0].pre[0]; + + best_ref_mv1_full.col = best_ref_mv1.col >> 3; + best_ref_mv1_full.row = best_ref_mv1.row >> 3; + + // Setup frame pointers + x->plane[0].src.buf = arf_frame_buf; + x->plane[0].src.stride = stride; + xd->plane[0].pre[0].buf = frame_ptr_buf; + xd->plane[0].pre[0].stride = stride; + + step_param = mv_sf->reduce_first_step_size; + step_param = AOMMIN(step_param, MAX_MVSEARCH_STEPS - 2); + + av1_set_mv_search_range(&x->mv_limits, &best_ref_mv1); + +#if CONFIG_REF_MV + x->mvcost = x->mv_cost_stack[0]; + x->nmvjointcost = x->nmv_vec_cost[0]; + x->mvsadcost = x->mvcost; + x->nmvjointsadcost = x->nmvjointcost; +#endif + + // Ignore mv costing by sending NULL pointer instead of cost arrays + av1_hex_search(x, &best_ref_mv1_full, step_param, sadpb, 1, + cond_cost_list(cpi, cost_list), &cpi->fn_ptr[BLOCK_16X16], 0, + &best_ref_mv1); + + x->mv_limits = tmp_mv_limits; + + // Ignore mv costing by sending NULL pointer instead of cost array + bestsme = cpi->find_fractional_mv_step( + x, &best_ref_mv1, cpi->common.allow_high_precision_mv, x->errorperbit, + &cpi->fn_ptr[BLOCK_16X16], 0, mv_sf->subpel_iters_per_step, + cond_cost_list(cpi, cost_list), NULL, NULL, &distortion, &sse, NULL, 0, 0, + 0); + + x->e_mbd.mi[0]->bmi[0].as_mv[0] = x->best_mv; + + // Restore input state + x->plane[0].src = src; + xd->plane[0].pre[0] = pre; + + return bestsme; +} + +static void temporal_filter_iterate_c(AV1_COMP *cpi, + YV12_BUFFER_CONFIG **frames, + int frame_count, int alt_ref_index, + int strength, + struct scale_factors *scale) { + int byte; + int frame; + int mb_col, mb_row; + unsigned int filter_weight; + int mb_cols = (frames[alt_ref_index]->y_crop_width + 15) >> 4; + int mb_rows = (frames[alt_ref_index]->y_crop_height + 15) >> 4; + int mb_y_offset = 0; + int mb_uv_offset = 0; + DECLARE_ALIGNED(16, unsigned int, accumulator[16 * 16 * 3]); + DECLARE_ALIGNED(16, uint16_t, count[16 * 16 * 3]); + MACROBLOCKD *mbd = &cpi->td.mb.e_mbd; + YV12_BUFFER_CONFIG *f = frames[alt_ref_index]; + uint8_t *dst1, *dst2; +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, predictor16[16 * 16 * 3]); + DECLARE_ALIGNED(16, uint8_t, predictor8[16 * 16 * 3]); + uint8_t *predictor; +#else + DECLARE_ALIGNED(16, uint8_t, predictor[16 * 16 * 3]); +#endif + const int mb_uv_height = 16 >> mbd->plane[1].subsampling_y; + const int mb_uv_width = 16 >> mbd->plane[1].subsampling_x; + + // Save input state + uint8_t *input_buffer[MAX_MB_PLANE]; + int i; +#if CONFIG_HIGHBITDEPTH + if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + predictor = CONVERT_TO_BYTEPTR(predictor16); + } else { + predictor = predictor8; + } +#endif + + for (i = 0; i < MAX_MB_PLANE; i++) input_buffer[i] = mbd->plane[i].pre[0].buf; + + for (mb_row = 0; mb_row < mb_rows; mb_row++) { + // Source frames are extended to 16 pixels. This is different than + // L/A/G reference frames that have a border of 32 (AV1ENCBORDERINPIXELS) + // A 6/8 tap filter is used for motion search. This requires 2 pixels + // before and 3 pixels after. So the largest Y mv on a border would + // then be 16 - AOM_INTERP_EXTEND. The UV blocks are half the size of the + // Y and therefore only extended by 8. The largest mv that a UV block + // can support is 8 - AOM_INTERP_EXTEND. A UV mv is half of a Y mv. + // (16 - AOM_INTERP_EXTEND) >> 1 which is greater than + // 8 - AOM_INTERP_EXTEND. + // To keep the mv in play for both Y and UV planes the max that it + // can be on a border is therefore 16 - (2*AOM_INTERP_EXTEND+1). + cpi->td.mb.mv_limits.row_min = + -((mb_row * 16) + (17 - 2 * AOM_INTERP_EXTEND)); + cpi->td.mb.mv_limits.row_max = + ((mb_rows - 1 - mb_row) * 16) + (17 - 2 * AOM_INTERP_EXTEND); + + for (mb_col = 0; mb_col < mb_cols; mb_col++) { + int j, k; + int stride; + + memset(accumulator, 0, 16 * 16 * 3 * sizeof(accumulator[0])); + memset(count, 0, 16 * 16 * 3 * sizeof(count[0])); + + cpi->td.mb.mv_limits.col_min = + -((mb_col * 16) + (17 - 2 * AOM_INTERP_EXTEND)); + cpi->td.mb.mv_limits.col_max = + ((mb_cols - 1 - mb_col) * 16) + (17 - 2 * AOM_INTERP_EXTEND); + + for (frame = 0; frame < frame_count; frame++) { + const int thresh_low = 10000; + const int thresh_high = 20000; + + if (frames[frame] == NULL) continue; + + mbd->mi[0]->bmi[0].as_mv[0].as_mv.row = 0; + mbd->mi[0]->bmi[0].as_mv[0].as_mv.col = 0; + + if (frame == alt_ref_index) { + filter_weight = 2; + } else { + // Find best match in this frame by MC + int err = temporal_filter_find_matching_mb_c( + cpi, frames[alt_ref_index]->y_buffer + mb_y_offset, + frames[frame]->y_buffer + mb_y_offset, frames[frame]->y_stride); + + // Assign higher weight to matching MB if it's error + // score is lower. If not applying MC default behavior + // is to weight all MBs equal. + filter_weight = err < thresh_low ? 2 : err < thresh_high ? 1 : 0; + } + + if (filter_weight != 0) { + // Construct the predictors + temporal_filter_predictors_mb_c( + mbd, frames[frame]->y_buffer + mb_y_offset, + frames[frame]->u_buffer + mb_uv_offset, + frames[frame]->v_buffer + mb_uv_offset, frames[frame]->y_stride, + mb_uv_width, mb_uv_height, mbd->mi[0]->bmi[0].as_mv[0].as_mv.row, + mbd->mi[0]->bmi[0].as_mv[0].as_mv.col, predictor, scale, + mb_col * 16, mb_row * 16); + +#if CONFIG_HIGHBITDEPTH + if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + int adj_strength = strength + 2 * (mbd->bd - 8); + // Apply the filter (YUV) + av1_highbd_temporal_filter_apply( + f->y_buffer + mb_y_offset, f->y_stride, predictor, 16, 16, + adj_strength, filter_weight, accumulator, count); + av1_highbd_temporal_filter_apply( + f->u_buffer + mb_uv_offset, f->uv_stride, predictor + 256, + mb_uv_width, mb_uv_height, adj_strength, filter_weight, + accumulator + 256, count + 256); + av1_highbd_temporal_filter_apply( + f->v_buffer + mb_uv_offset, f->uv_stride, predictor + 512, + mb_uv_width, mb_uv_height, adj_strength, filter_weight, + accumulator + 512, count + 512); + } else { + // Apply the filter (YUV) + av1_temporal_filter_apply_c(f->y_buffer + mb_y_offset, f->y_stride, + predictor, 16, 16, strength, + filter_weight, accumulator, count); + av1_temporal_filter_apply_c( + f->u_buffer + mb_uv_offset, f->uv_stride, predictor + 256, + mb_uv_width, mb_uv_height, strength, filter_weight, + accumulator + 256, count + 256); + av1_temporal_filter_apply_c( + f->v_buffer + mb_uv_offset, f->uv_stride, predictor + 512, + mb_uv_width, mb_uv_height, strength, filter_weight, + accumulator + 512, count + 512); + } +#else + // Apply the filter (YUV) + av1_temporal_filter_apply_c(f->y_buffer + mb_y_offset, f->y_stride, + predictor, 16, 16, strength, + filter_weight, accumulator, count); + av1_temporal_filter_apply_c(f->u_buffer + mb_uv_offset, f->uv_stride, + predictor + 256, mb_uv_width, + mb_uv_height, strength, filter_weight, + accumulator + 256, count + 256); + av1_temporal_filter_apply_c(f->v_buffer + mb_uv_offset, f->uv_stride, + predictor + 512, mb_uv_width, + mb_uv_height, strength, filter_weight, + accumulator + 512, count + 512); +#endif // CONFIG_HIGHBITDEPTH + } + } + +#if CONFIG_HIGHBITDEPTH + if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + uint16_t *dst1_16; + uint16_t *dst2_16; + // Normalize filter output to produce AltRef frame + dst1 = cpi->alt_ref_buffer.y_buffer; + dst1_16 = CONVERT_TO_SHORTPTR(dst1); + stride = cpi->alt_ref_buffer.y_stride; + byte = mb_y_offset; + for (i = 0, k = 0; i < 16; i++) { + for (j = 0; j < 16; j++, k++) { + dst1_16[byte] = + (uint16_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]); + + // move to next pixel + byte++; + } + + byte += stride - 16; + } + + dst1 = cpi->alt_ref_buffer.u_buffer; + dst2 = cpi->alt_ref_buffer.v_buffer; + dst1_16 = CONVERT_TO_SHORTPTR(dst1); + dst2_16 = CONVERT_TO_SHORTPTR(dst2); + stride = cpi->alt_ref_buffer.uv_stride; + byte = mb_uv_offset; + for (i = 0, k = 256; i < mb_uv_height; i++) { + for (j = 0; j < mb_uv_width; j++, k++) { + int m = k + 256; + + // U + dst1_16[byte] = + (uint16_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]); + + // V + dst2_16[byte] = + (uint16_t)OD_DIVU(accumulator[m] + (count[m] >> 1), count[m]); + + // move to next pixel + byte++; + } + + byte += stride - mb_uv_width; + } + } else { + // Normalize filter output to produce AltRef frame + dst1 = cpi->alt_ref_buffer.y_buffer; + stride = cpi->alt_ref_buffer.y_stride; + byte = mb_y_offset; + for (i = 0, k = 0; i < 16; i++) { + for (j = 0; j < 16; j++, k++) { + dst1[byte] = + (uint8_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]); + + // move to next pixel + byte++; + } + byte += stride - 16; + } + + dst1 = cpi->alt_ref_buffer.u_buffer; + dst2 = cpi->alt_ref_buffer.v_buffer; + stride = cpi->alt_ref_buffer.uv_stride; + byte = mb_uv_offset; + for (i = 0, k = 256; i < mb_uv_height; i++) { + for (j = 0; j < mb_uv_width; j++, k++) { + int m = k + 256; + + // U + dst1[byte] = + (uint8_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]); + + // V + dst2[byte] = + (uint8_t)OD_DIVU(accumulator[m] + (count[m] >> 1), count[m]); + + // move to next pixel + byte++; + } + byte += stride - mb_uv_width; + } + } +#else + // Normalize filter output to produce AltRef frame + dst1 = cpi->alt_ref_buffer.y_buffer; + stride = cpi->alt_ref_buffer.y_stride; + byte = mb_y_offset; + for (i = 0, k = 0; i < 16; i++) { + for (j = 0; j < 16; j++, k++) { + dst1[byte] = + (uint8_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]); + + // move to next pixel + byte++; + } + byte += stride - 16; + } + + dst1 = cpi->alt_ref_buffer.u_buffer; + dst2 = cpi->alt_ref_buffer.v_buffer; + stride = cpi->alt_ref_buffer.uv_stride; + byte = mb_uv_offset; + for (i = 0, k = 256; i < mb_uv_height; i++) { + for (j = 0; j < mb_uv_width; j++, k++) { + int m = k + 256; + + // U + dst1[byte] = + (uint8_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]); + + // V + dst2[byte] = + (uint8_t)OD_DIVU(accumulator[m] + (count[m] >> 1), count[m]); + + // move to next pixel + byte++; + } + byte += stride - mb_uv_width; + } +#endif // CONFIG_HIGHBITDEPTH + mb_y_offset += 16; + mb_uv_offset += mb_uv_width; + } + mb_y_offset += 16 * (f->y_stride - mb_cols); + mb_uv_offset += mb_uv_height * f->uv_stride - mb_uv_width * mb_cols; + } + + // Restore input state + for (i = 0; i < MAX_MB_PLANE; i++) mbd->plane[i].pre[0].buf = input_buffer[i]; +} + +// Apply buffer limits and context specific adjustments to arnr filter. +static void adjust_arnr_filter(AV1_COMP *cpi, int distance, int group_boost, + int *arnr_frames, int *arnr_strength) { + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + const int frames_after_arf = + av1_lookahead_depth(cpi->lookahead) - distance - 1; + int frames_fwd = (cpi->oxcf.arnr_max_frames - 1) >> 1; + int frames_bwd; + int q, frames, strength; + + // Define the forward and backwards filter limits for this arnr group. + if (frames_fwd > frames_after_arf) frames_fwd = frames_after_arf; + if (frames_fwd > distance) frames_fwd = distance; + + frames_bwd = frames_fwd; + + // For even length filter there is one more frame backward + // than forward: e.g. len=6 ==> bbbAff, len=7 ==> bbbAfff. + if (frames_bwd < distance) frames_bwd += (oxcf->arnr_max_frames + 1) & 0x1; + + // Set the baseline active filter size. + frames = frames_bwd + 1 + frames_fwd; + + // Adjust the strength based on active max q. + if (cpi->common.current_video_frame > 1) + q = ((int)av1_convert_qindex_to_q(cpi->rc.avg_frame_qindex[INTER_FRAME], + cpi->common.bit_depth)); + else + q = ((int)av1_convert_qindex_to_q(cpi->rc.avg_frame_qindex[KEY_FRAME], + cpi->common.bit_depth)); + if (q > 16) { + strength = oxcf->arnr_strength; + } else { + strength = oxcf->arnr_strength - ((16 - q) / 2); + if (strength < 0) strength = 0; + } + + // Adjust number of frames in filter and strength based on gf boost level. + if (frames > group_boost / 150) { + frames = group_boost / 150; + frames += !(frames & 1); + } + + if (strength > group_boost / 300) { + strength = group_boost / 300; + } + + // Adjustments for second level arf in multi arf case. + if (cpi->oxcf.pass == 2 && cpi->multi_arf_allowed) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + if (gf_group->rf_level[gf_group->index] != GF_ARF_STD) { + strength >>= 1; + } + } + + *arnr_frames = frames; + *arnr_strength = strength; +} + +void av1_temporal_filter(AV1_COMP *cpi, int distance) { + RATE_CONTROL *const rc = &cpi->rc; + int frame; + int frames_to_blur; + int start_frame; + int strength; + int frames_to_blur_backward; + int frames_to_blur_forward; + struct scale_factors sf; + YV12_BUFFER_CONFIG *frames[MAX_LAG_BUFFERS] = { NULL }; +#if CONFIG_EXT_REFS + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; +#endif + + // Apply context specific adjustments to the arnr filter parameters. + adjust_arnr_filter(cpi, distance, rc->gfu_boost, &frames_to_blur, &strength); +// TODO(weitinglin): Currently, we enforce the filtering strength on +// extra ARFs' to be zeros. We should investigate in which +// case it is more beneficial to use non-zero strength +// filtering. +#if CONFIG_EXT_REFS + if (gf_group->rf_level[gf_group->index] == GF_ARF_LOW) { + strength = 0; + frames_to_blur = 1; + } +#endif + +#if CONFIG_EXT_REFS + if (strength == 0 && frames_to_blur == 1) { + cpi->is_arf_filter_off[gf_group->arf_update_idx[gf_group->index]] = 1; + } else { + cpi->is_arf_filter_off[gf_group->arf_update_idx[gf_group->index]] = 0; + } +#endif + + frames_to_blur_backward = (frames_to_blur / 2); + frames_to_blur_forward = ((frames_to_blur - 1) / 2); + start_frame = distance + frames_to_blur_forward; + + // Setup frame pointers, NULL indicates frame not included in filter. + for (frame = 0; frame < frames_to_blur; ++frame) { + const int which_buffer = start_frame - frame; + struct lookahead_entry *buf = + av1_lookahead_peek(cpi->lookahead, which_buffer); + frames[frames_to_blur - 1 - frame] = &buf->img; + } + + if (frames_to_blur > 0) { +// Setup scaling factors. Scaling on each of the arnr frames is not +// supported. +// ARF is produced at the native frame size and resized when coded. +#if CONFIG_HIGHBITDEPTH + av1_setup_scale_factors_for_frame( + &sf, frames[0]->y_crop_width, frames[0]->y_crop_height, + frames[0]->y_crop_width, frames[0]->y_crop_height, + cpi->common.use_highbitdepth); +#else + av1_setup_scale_factors_for_frame( + &sf, frames[0]->y_crop_width, frames[0]->y_crop_height, + frames[0]->y_crop_width, frames[0]->y_crop_height); +#endif // CONFIG_HIGHBITDEPTH + } + + temporal_filter_iterate_c(cpi, frames, frames_to_blur, + frames_to_blur_backward, strength, &sf); +} diff --git a/third_party/aom/av1/encoder/temporal_filter.h b/third_party/aom/av1/encoder/temporal_filter.h new file mode 100644 index 0000000000..bc0863a638 --- /dev/null +++ b/third_party/aom/av1/encoder/temporal_filter.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_TEMPORAL_FILTER_H_ +#define AV1_ENCODER_TEMPORAL_FILTER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_temporal_filter(AV1_COMP *cpi, int distance); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_TEMPORAL_FILTER_H_ diff --git a/third_party/aom/av1/encoder/tokenize.c b/third_party/aom/av1/encoder/tokenize.c new file mode 100644 index 0000000000..f48493bf89 --- /dev/null +++ b/third_party/aom/av1/encoder/tokenize.c @@ -0,0 +1,887 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include + +#include "aom_mem/aom_mem.h" + +#include "av1/common/entropy.h" +#include "av1/common/pred_common.h" +#include "av1/common/scan.h" +#include "av1/common/seg_common.h" + +#include "av1/encoder/cost.h" +#include "av1/encoder/encoder.h" +#include "av1/encoder/rdopt.h" +#include "av1/encoder/tokenize.h" + +static const TOKENVALUE dct_cat_lt_10_value_tokens[] = { + { 9, 63 }, { 9, 61 }, { 9, 59 }, { 9, 57 }, { 9, 55 }, { 9, 53 }, { 9, 51 }, + { 9, 49 }, { 9, 47 }, { 9, 45 }, { 9, 43 }, { 9, 41 }, { 9, 39 }, { 9, 37 }, + { 9, 35 }, { 9, 33 }, { 9, 31 }, { 9, 29 }, { 9, 27 }, { 9, 25 }, { 9, 23 }, + { 9, 21 }, { 9, 19 }, { 9, 17 }, { 9, 15 }, { 9, 13 }, { 9, 11 }, { 9, 9 }, + { 9, 7 }, { 9, 5 }, { 9, 3 }, { 9, 1 }, { 8, 31 }, { 8, 29 }, { 8, 27 }, + { 8, 25 }, { 8, 23 }, { 8, 21 }, { 8, 19 }, { 8, 17 }, { 8, 15 }, { 8, 13 }, + { 8, 11 }, { 8, 9 }, { 8, 7 }, { 8, 5 }, { 8, 3 }, { 8, 1 }, { 7, 15 }, + { 7, 13 }, { 7, 11 }, { 7, 9 }, { 7, 7 }, { 7, 5 }, { 7, 3 }, { 7, 1 }, + { 6, 7 }, { 6, 5 }, { 6, 3 }, { 6, 1 }, { 5, 3 }, { 5, 1 }, { 4, 1 }, + { 3, 1 }, { 2, 1 }, { 1, 1 }, { 0, 0 }, { 1, 0 }, { 2, 0 }, { 3, 0 }, + { 4, 0 }, { 5, 0 }, { 5, 2 }, { 6, 0 }, { 6, 2 }, { 6, 4 }, { 6, 6 }, + { 7, 0 }, { 7, 2 }, { 7, 4 }, { 7, 6 }, { 7, 8 }, { 7, 10 }, { 7, 12 }, + { 7, 14 }, { 8, 0 }, { 8, 2 }, { 8, 4 }, { 8, 6 }, { 8, 8 }, { 8, 10 }, + { 8, 12 }, { 8, 14 }, { 8, 16 }, { 8, 18 }, { 8, 20 }, { 8, 22 }, { 8, 24 }, + { 8, 26 }, { 8, 28 }, { 8, 30 }, { 9, 0 }, { 9, 2 }, { 9, 4 }, { 9, 6 }, + { 9, 8 }, { 9, 10 }, { 9, 12 }, { 9, 14 }, { 9, 16 }, { 9, 18 }, { 9, 20 }, + { 9, 22 }, { 9, 24 }, { 9, 26 }, { 9, 28 }, { 9, 30 }, { 9, 32 }, { 9, 34 }, + { 9, 36 }, { 9, 38 }, { 9, 40 }, { 9, 42 }, { 9, 44 }, { 9, 46 }, { 9, 48 }, + { 9, 50 }, { 9, 52 }, { 9, 54 }, { 9, 56 }, { 9, 58 }, { 9, 60 }, { 9, 62 } +}; +const TOKENVALUE *av1_dct_cat_lt_10_value_tokens = + dct_cat_lt_10_value_tokens + + (sizeof(dct_cat_lt_10_value_tokens) / sizeof(*dct_cat_lt_10_value_tokens)) / + 2; +// The corresponding costs of the extrabits for the tokens in the above table +// are stored in the table below. The values are obtained from looking up the +// entry for the specified extrabits in the table corresponding to the token +// (as defined in cost element av1_extra_bits) +// e.g. {9, 63} maps to cat5_cost[63 >> 1], {1, 1} maps to sign_cost[1 >> 1] +static const int dct_cat_lt_10_value_cost[] = { + 3773, 3750, 3704, 3681, 3623, 3600, 3554, 3531, 3432, 3409, 3363, 3340, 3282, + 3259, 3213, 3190, 3136, 3113, 3067, 3044, 2986, 2963, 2917, 2894, 2795, 2772, + 2726, 2703, 2645, 2622, 2576, 2553, 3197, 3116, 3058, 2977, 2881, 2800, 2742, + 2661, 2615, 2534, 2476, 2395, 2299, 2218, 2160, 2079, 2566, 2427, 2334, 2195, + 2023, 1884, 1791, 1652, 1893, 1696, 1453, 1256, 1229, 864, 512, 512, 512, + 512, 0, 512, 512, 512, 512, 864, 1229, 1256, 1453, 1696, 1893, 1652, + 1791, 1884, 2023, 2195, 2334, 2427, 2566, 2079, 2160, 2218, 2299, 2395, 2476, + 2534, 2615, 2661, 2742, 2800, 2881, 2977, 3058, 3116, 3197, 2553, 2576, 2622, + 2645, 2703, 2726, 2772, 2795, 2894, 2917, 2963, 2986, 3044, 3067, 3113, 3136, + 3190, 3213, 3259, 3282, 3340, 3363, 3409, 3432, 3531, 3554, 3600, 3623, 3681, + 3704, 3750, 3773, +}; +const int *av1_dct_cat_lt_10_value_cost = + dct_cat_lt_10_value_cost + + (sizeof(dct_cat_lt_10_value_cost) / sizeof(*dct_cat_lt_10_value_cost)) / 2; + +// Array indices are identical to previously-existing CONTEXT_NODE indices +/* clang-format off */ +const aom_tree_index av1_coef_tree[TREE_SIZE(ENTROPY_TOKENS)] = { + -EOB_TOKEN, 2, // 0 = EOB + -ZERO_TOKEN, 4, // 1 = ZERO + -ONE_TOKEN, 6, // 2 = ONE + 8, 12, // 3 = LOW_VAL + -TWO_TOKEN, 10, // 4 = TWO + -THREE_TOKEN, -FOUR_TOKEN, // 5 = THREE + 14, 16, // 6 = HIGH_LOW + -CATEGORY1_TOKEN, -CATEGORY2_TOKEN, // 7 = CAT_ONE + 18, 20, // 8 = CAT_THREEFOUR + -CATEGORY3_TOKEN, -CATEGORY4_TOKEN, // 9 = CAT_THREE + -CATEGORY5_TOKEN, -CATEGORY6_TOKEN // 10 = CAT_FIVE +}; +/* clang-format on */ + +static const int16_t zero_cost[] = { 0 }; +static const int16_t sign_cost[1] = { 512 }; +static const int16_t cat1_cost[1 << 1] = { 864, 1229 }; +static const int16_t cat2_cost[1 << 2] = { 1256, 1453, 1696, 1893 }; +static const int16_t cat3_cost[1 << 3] = { 1652, 1791, 1884, 2023, + 2195, 2334, 2427, 2566 }; +static const int16_t cat4_cost[1 << 4] = { 2079, 2160, 2218, 2299, 2395, 2476, + 2534, 2615, 2661, 2742, 2800, 2881, + 2977, 3058, 3116, 3197 }; +static const int16_t cat5_cost[1 << 5] = { + 2553, 2576, 2622, 2645, 2703, 2726, 2772, 2795, 2894, 2917, 2963, + 2986, 3044, 3067, 3113, 3136, 3190, 3213, 3259, 3282, 3340, 3363, + 3409, 3432, 3531, 3554, 3600, 3623, 3681, 3704, 3750, 3773 +}; +const int16_t av1_cat6_low_cost[256] = { + 3378, 3390, 3401, 3413, 3435, 3447, 3458, 3470, 3517, 3529, 3540, 3552, 3574, + 3586, 3597, 3609, 3671, 3683, 3694, 3706, 3728, 3740, 3751, 3763, 3810, 3822, + 3833, 3845, 3867, 3879, 3890, 3902, 3973, 3985, 3996, 4008, 4030, 4042, 4053, + 4065, 4112, 4124, 4135, 4147, 4169, 4181, 4192, 4204, 4266, 4278, 4289, 4301, + 4323, 4335, 4346, 4358, 4405, 4417, 4428, 4440, 4462, 4474, 4485, 4497, 4253, + 4265, 4276, 4288, 4310, 4322, 4333, 4345, 4392, 4404, 4415, 4427, 4449, 4461, + 4472, 4484, 4546, 4558, 4569, 4581, 4603, 4615, 4626, 4638, 4685, 4697, 4708, + 4720, 4742, 4754, 4765, 4777, 4848, 4860, 4871, 4883, 4905, 4917, 4928, 4940, + 4987, 4999, 5010, 5022, 5044, 5056, 5067, 5079, 5141, 5153, 5164, 5176, 5198, + 5210, 5221, 5233, 5280, 5292, 5303, 5315, 5337, 5349, 5360, 5372, 4988, 5000, + 5011, 5023, 5045, 5057, 5068, 5080, 5127, 5139, 5150, 5162, 5184, 5196, 5207, + 5219, 5281, 5293, 5304, 5316, 5338, 5350, 5361, 5373, 5420, 5432, 5443, 5455, + 5477, 5489, 5500, 5512, 5583, 5595, 5606, 5618, 5640, 5652, 5663, 5675, 5722, + 5734, 5745, 5757, 5779, 5791, 5802, 5814, 5876, 5888, 5899, 5911, 5933, 5945, + 5956, 5968, 6015, 6027, 6038, 6050, 6072, 6084, 6095, 6107, 5863, 5875, 5886, + 5898, 5920, 5932, 5943, 5955, 6002, 6014, 6025, 6037, 6059, 6071, 6082, 6094, + 6156, 6168, 6179, 6191, 6213, 6225, 6236, 6248, 6295, 6307, 6318, 6330, 6352, + 6364, 6375, 6387, 6458, 6470, 6481, 6493, 6515, 6527, 6538, 6550, 6597, 6609, + 6620, 6632, 6654, 6666, 6677, 6689, 6751, 6763, 6774, 6786, 6808, 6820, 6831, + 6843, 6890, 6902, 6913, 6925, 6947, 6959, 6970, 6982 +}; +const int av1_cat6_high_cost[CAT6_HIGH_COST_ENTRIES] = { + 100, 2263, 2739, 4902, 3160, 5323, 5799, 7962, 3678, 5841, 6317, + 8480, 6738, 8901, 9377, 11540, 3678, 5841, 6317, 8480, 6738, 8901, + 9377, 11540, 7256, 9419, 9895, 12058, 10316, 12479, 12955, 15118, 3678, + 5841, 6317, 8480, 6738, 8901, 9377, 11540, 7256, 9419, 9895, 12058, + 10316, 12479, 12955, 15118, 7256, 9419, 9895, 12058, 10316, 12479, 12955, + 15118, 10834, 12997, 13473, 15636, 13894, 16057, 16533, 18696, +#if CONFIG_HIGHBITDEPTH + 4193, 6356, 6832, 8995, 7253, 9416, 9892, 12055, 7771, 9934, 10410, + 12573, 10831, 12994, 13470, 15633, 7771, 9934, 10410, 12573, 10831, 12994, + 13470, 15633, 11349, 13512, 13988, 16151, 14409, 16572, 17048, 19211, 7771, + 9934, 10410, 12573, 10831, 12994, 13470, 15633, 11349, 13512, 13988, 16151, + 14409, 16572, 17048, 19211, 11349, 13512, 13988, 16151, 14409, 16572, 17048, + 19211, 14927, 17090, 17566, 19729, 17987, 20150, 20626, 22789, 4193, 6356, + 6832, 8995, 7253, 9416, 9892, 12055, 7771, 9934, 10410, 12573, 10831, + 12994, 13470, 15633, 7771, 9934, 10410, 12573, 10831, 12994, 13470, 15633, + 11349, 13512, 13988, 16151, 14409, 16572, 17048, 19211, 7771, 9934, 10410, + 12573, 10831, 12994, 13470, 15633, 11349, 13512, 13988, 16151, 14409, 16572, + 17048, 19211, 11349, 13512, 13988, 16151, 14409, 16572, 17048, 19211, 14927, + 17090, 17566, 19729, 17987, 20150, 20626, 22789, 8286, 10449, 10925, 13088, + 11346, 13509, 13985, 16148, 11864, 14027, 14503, 16666, 14924, 17087, 17563, + 19726, 11864, 14027, 14503, 16666, 14924, 17087, 17563, 19726, 15442, 17605, + 18081, 20244, 18502, 20665, 21141, 23304, 11864, 14027, 14503, 16666, 14924, + 17087, 17563, 19726, 15442, 17605, 18081, 20244, 18502, 20665, 21141, 23304, + 15442, 17605, 18081, 20244, 18502, 20665, 21141, 23304, 19020, 21183, 21659, + 23822, 22080, 24243, 24719, 26882, 4193, 6356, 6832, 8995, 7253, 9416, + 9892, 12055, 7771, 9934, 10410, 12573, 10831, 12994, 13470, 15633, 7771, + 9934, 10410, 12573, 10831, 12994, 13470, 15633, 11349, 13512, 13988, 16151, + 14409, 16572, 17048, 19211, 7771, 9934, 10410, 12573, 10831, 12994, 13470, + 15633, 11349, 13512, 13988, 16151, 14409, 16572, 17048, 19211, 11349, 13512, + 13988, 16151, 14409, 16572, 17048, 19211, 14927, 17090, 17566, 19729, 17987, + 20150, 20626, 22789, 8286, 10449, 10925, 13088, 11346, 13509, 13985, 16148, + 11864, 14027, 14503, 16666, 14924, 17087, 17563, 19726, 11864, 14027, 14503, + 16666, 14924, 17087, 17563, 19726, 15442, 17605, 18081, 20244, 18502, 20665, + 21141, 23304, 11864, 14027, 14503, 16666, 14924, 17087, 17563, 19726, 15442, + 17605, 18081, 20244, 18502, 20665, 21141, 23304, 15442, 17605, 18081, 20244, + 18502, 20665, 21141, 23304, 19020, 21183, 21659, 23822, 22080, 24243, 24719, + 26882, 8286, 10449, 10925, 13088, 11346, 13509, 13985, 16148, 11864, 14027, + 14503, 16666, 14924, 17087, 17563, 19726, 11864, 14027, 14503, 16666, 14924, + 17087, 17563, 19726, 15442, 17605, 18081, 20244, 18502, 20665, 21141, 23304, + 11864, 14027, 14503, 16666, 14924, 17087, 17563, 19726, 15442, 17605, 18081, + 20244, 18502, 20665, 21141, 23304, 15442, 17605, 18081, 20244, 18502, 20665, + 21141, 23304, 19020, 21183, 21659, 23822, 22080, 24243, 24719, 26882, 12379, + 14542, 15018, 17181, 15439, 17602, 18078, 20241, 15957, 18120, 18596, 20759, + 19017, 21180, 21656, 23819, 15957, 18120, 18596, 20759, 19017, 21180, 21656, + 23819, 19535, 21698, 22174, 24337, 22595, 24758, 25234, 27397, 15957, 18120, + 18596, 20759, 19017, 21180, 21656, 23819, 19535, 21698, 22174, 24337, 22595, + 24758, 25234, 27397, 19535, 21698, 22174, 24337, 22595, 24758, 25234, 27397, + 23113, 25276, 25752, 27915, 26173, 28336, 28812, 30975, 4193, 6356, 6832, + 8995, 7253, 9416, 9892, 12055, 7771, 9934, 10410, 12573, 10831, 12994, + 13470, 15633, 7771, 9934, 10410, 12573, 10831, 12994, 13470, 15633, 11349, + 13512, 13988, 16151, 14409, 16572, 17048, 19211, 7771, 9934, 10410, 12573, + 10831, 12994, 13470, 15633, 11349, 13512, 13988, 16151, 14409, 16572, 17048, + 19211, 11349, 13512, 13988, 16151, 14409, 16572, 17048, 19211, 14927, 17090, + 17566, 19729, 17987, 20150, 20626, 22789, 8286, 10449, 10925, 13088, 11346, + 13509, 13985, 16148, 11864, 14027, 14503, 16666, 14924, 17087, 17563, 19726, + 11864, 14027, 14503, 16666, 14924, 17087, 17563, 19726, 15442, 17605, 18081, + 20244, 18502, 20665, 21141, 23304, 11864, 14027, 14503, 16666, 14924, 17087, + 17563, 19726, 15442, 17605, 18081, 20244, 18502, 20665, 21141, 23304, 15442, + 17605, 18081, 20244, 18502, 20665, 21141, 23304, 19020, 21183, 21659, 23822, + 22080, 24243, 24719, 26882, 8286, 10449, 10925, 13088, 11346, 13509, 13985, + 16148, 11864, 14027, 14503, 16666, 14924, 17087, 17563, 19726, 11864, 14027, + 14503, 16666, 14924, 17087, 17563, 19726, 15442, 17605, 18081, 20244, 18502, + 20665, 21141, 23304, 11864, 14027, 14503, 16666, 14924, 17087, 17563, 19726, + 15442, 17605, 18081, 20244, 18502, 20665, 21141, 23304, 15442, 17605, 18081, + 20244, 18502, 20665, 21141, 23304, 19020, 21183, 21659, 23822, 22080, 24243, + 24719, 26882, 12379, 14542, 15018, 17181, 15439, 17602, 18078, 20241, 15957, + 18120, 18596, 20759, 19017, 21180, 21656, 23819, 15957, 18120, 18596, 20759, + 19017, 21180, 21656, 23819, 19535, 21698, 22174, 24337, 22595, 24758, 25234, + 27397, 15957, 18120, 18596, 20759, 19017, 21180, 21656, 23819, 19535, 21698, + 22174, 24337, 22595, 24758, 25234, 27397, 19535, 21698, 22174, 24337, 22595, + 24758, 25234, 27397, 23113, 25276, 25752, 27915, 26173, 28336, 28812, 30975, + 8286, 10449, 10925, 13088, 11346, 13509, 13985, 16148, 11864, 14027, 14503, + 16666, 14924, 17087, 17563, 19726, 11864, 14027, 14503, 16666, 14924, 17087, + 17563, 19726, 15442, 17605, 18081, 20244, 18502, 20665, 21141, 23304, 11864, + 14027, 14503, 16666, 14924, 17087, 17563, 19726, 15442, 17605, 18081, 20244, + 18502, 20665, 21141, 23304, 15442, 17605, 18081, 20244, 18502, 20665, 21141, + 23304, 19020, 21183, 21659, 23822, 22080, 24243, 24719, 26882, 12379, 14542, + 15018, 17181, 15439, 17602, 18078, 20241, 15957, 18120, 18596, 20759, 19017, + 21180, 21656, 23819, 15957, 18120, 18596, 20759, 19017, 21180, 21656, 23819, + 19535, 21698, 22174, 24337, 22595, 24758, 25234, 27397, 15957, 18120, 18596, + 20759, 19017, 21180, 21656, 23819, 19535, 21698, 22174, 24337, 22595, 24758, + 25234, 27397, 19535, 21698, 22174, 24337, 22595, 24758, 25234, 27397, 23113, + 25276, 25752, 27915, 26173, 28336, 28812, 30975, 12379, 14542, 15018, 17181, + 15439, 17602, 18078, 20241, 15957, 18120, 18596, 20759, 19017, 21180, 21656, + 23819, 15957, 18120, 18596, 20759, 19017, 21180, 21656, 23819, 19535, 21698, + 22174, 24337, 22595, 24758, 25234, 27397, 15957, 18120, 18596, 20759, 19017, + 21180, 21656, 23819, 19535, 21698, 22174, 24337, 22595, 24758, 25234, 27397, + 19535, 21698, 22174, 24337, 22595, 24758, 25234, 27397, 23113, 25276, 25752, + 27915, 26173, 28336, 28812, 30975, 16472, 18635, 19111, 21274, 19532, 21695, + 22171, 24334, 20050, 22213, 22689, 24852, 23110, 25273, 25749, 27912, 20050, + 22213, 22689, 24852, 23110, 25273, 25749, 27912, 23628, 25791, 26267, 28430, + 26688, 28851, 29327, 31490, 20050, 22213, 22689, 24852, 23110, 25273, 25749, + 27912, 23628, 25791, 26267, 28430, 26688, 28851, 29327, 31490, 23628, 25791, + 26267, 28430, 26688, 28851, 29327, 31490, 27206, 29369, 29845, 32008, 30266, + 32429, 32905, 35068 +#endif +}; + +const uint8_t av1_cat6_skipped_bits_discount[8] = { + 0, 3, 6, 9, 12, 18, 24, 30 +}; + +#if CONFIG_NEW_MULTISYMBOL +const av1_extra_bit av1_extra_bits[ENTROPY_TOKENS] = { + { 0, 0, 0, zero_cost }, // ZERO_TOKEN + { 0, 0, 1, sign_cost }, // ONE_TOKEN + { 0, 0, 2, sign_cost }, // TWO_TOKEN + { 0, 0, 3, sign_cost }, // THREE_TOKEN + { 0, 0, 4, sign_cost }, // FOUR_TOKEN + { av1_cat1_cdf, 1, CAT1_MIN_VAL, cat1_cost }, // CATEGORY1_TOKEN + { av1_cat2_cdf, 2, CAT2_MIN_VAL, cat2_cost }, // CATEGORY2_TOKEN + { av1_cat3_cdf, 3, CAT3_MIN_VAL, cat3_cost }, // CATEGORY3_TOKEN + { av1_cat4_cdf, 4, CAT4_MIN_VAL, cat4_cost }, // CATEGORY4_TOKEN + { av1_cat5_cdf, 5, CAT5_MIN_VAL, cat5_cost }, // CATEGORY5_TOKEN + { av1_cat6_cdf, 18, CAT6_MIN_VAL, 0 }, // CATEGORY6_TOKEN + { 0, 0, 0, zero_cost } // EOB_TOKEN +}; +#else +const av1_extra_bit av1_extra_bits[ENTROPY_TOKENS] = { + { 0, 0, 0, zero_cost }, // ZERO_TOKEN + { 0, 0, 1, sign_cost }, // ONE_TOKEN + { 0, 0, 2, sign_cost }, // TWO_TOKEN + { 0, 0, 3, sign_cost }, // THREE_TOKEN + { 0, 0, 4, sign_cost }, // FOUR_TOKEN + { av1_cat1_prob, 1, CAT1_MIN_VAL, cat1_cost }, // CATEGORY1_TOKEN + { av1_cat2_prob, 2, CAT2_MIN_VAL, cat2_cost }, // CATEGORY2_TOKEN + { av1_cat3_prob, 3, CAT3_MIN_VAL, cat3_cost }, // CATEGORY3_TOKEN + { av1_cat4_prob, 4, CAT4_MIN_VAL, cat4_cost }, // CATEGORY4_TOKEN + { av1_cat5_prob, 5, CAT5_MIN_VAL, cat5_cost }, // CATEGORY5_TOKEN + { av1_cat6_prob, 18, CAT6_MIN_VAL, 0 }, // CATEGORY6_TOKEN + { 0, 0, 0, zero_cost } // EOB_TOKEN +}; +#endif + +#if !CONFIG_EC_MULTISYMBOL +const struct av1_token av1_coef_encodings[ENTROPY_TOKENS] = { + { 2, 2 }, { 6, 3 }, { 28, 5 }, { 58, 6 }, { 59, 6 }, { 60, 6 }, + { 61, 6 }, { 124, 7 }, { 125, 7 }, { 126, 7 }, { 127, 7 }, { 0, 1 } +}; +#endif // !CONFIG_EC_MULTISYMBOL + +struct tokenize_b_args { + const AV1_COMP *cpi; + ThreadData *td; + TOKENEXTRA **tp; + int this_rate; +}; + +#if !CONFIG_PVQ || CONFIG_VAR_TX +static void cost_coeffs_b(int plane, int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) { + struct tokenize_b_args *const args = arg; + const AV1_COMP *const cpi = args->cpi; + const AV1_COMMON *cm = &cpi->common; + ThreadData *const td = args->td; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + struct macroblock_plane *p = &x->plane[plane]; + struct macroblockd_plane *pd = &xd->plane[plane]; + const PLANE_TYPE type = pd->plane_type; + const int ref = is_inter_block(mbmi); + const TX_TYPE tx_type = get_tx_type(type, xd, block, tx_size); + const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, ref); + const int rate = av1_cost_coeffs(cpi, x, plane, block, tx_size, scan_order, + pd->above_context + blk_col, + pd->left_context + blk_row, 0); + args->this_rate += rate; + (void)plane_bsize; + av1_set_contexts(xd, pd, plane, tx_size, p->eobs[block] > 0, blk_col, + blk_row); +} + +static void set_entropy_context_b(int plane, int block, int blk_row, + int blk_col, BLOCK_SIZE plane_bsize, + TX_SIZE tx_size, void *arg) { + struct tokenize_b_args *const args = arg; + ThreadData *const td = args->td; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + struct macroblock_plane *p = &x->plane[plane]; + struct macroblockd_plane *pd = &xd->plane[plane]; + (void)plane_bsize; + av1_set_contexts(xd, pd, plane, tx_size, p->eobs[block] > 0, blk_col, + blk_row); +} + +#if CONFIG_NEW_TOKENSET +static INLINE void add_token(TOKENEXTRA **t, + aom_cdf_prob (*tail_cdf)[CDF_SIZE(ENTROPY_TOKENS)], + aom_cdf_prob (*head_cdf)[CDF_SIZE(ENTROPY_TOKENS)], + int eob_val, int first_val, int32_t extra, + uint8_t token) { + (*t)->token = token; + (*t)->extra = extra; + (*t)->tail_cdf = tail_cdf; + (*t)->head_cdf = head_cdf; + (*t)->eob_val = eob_val; + (*t)->first_val = first_val; + (*t)++; +} + +#else // CONFIG_NEW_TOKENSET +static INLINE void add_token( + TOKENEXTRA **t, const aom_prob *context_tree, +#if CONFIG_EC_MULTISYMBOL + aom_cdf_prob (*token_cdf)[CDF_SIZE(ENTROPY_TOKENS)], +#endif // CONFIG_EC_MULTISYMBOL + int32_t extra, uint8_t token, uint8_t skip_eob_node, unsigned int *counts) { + (*t)->token = token; + (*t)->extra = extra; + (*t)->context_tree = context_tree; +#if CONFIG_EC_MULTISYMBOL + (*t)->token_cdf = token_cdf; +#endif // CONFIG_EC_MULTISYMBOL + (*t)->skip_eob_node = skip_eob_node; + (*t)++; + ++counts[token]; +} +#endif // CONFIG_NEW_TOKENSET +#endif // !CONFIG_PVQ || CONFIG_VAR_TX + +#if CONFIG_PALETTE +void av1_tokenize_palette_sb(const AV1_COMP *cpi, + const struct ThreadData *const td, int plane, + TOKENEXTRA **t, RUN_TYPE dry_run, BLOCK_SIZE bsize, + int *rate) { + const MACROBLOCK *const x = &td->mb; + const MACROBLOCKD *const xd = &x->e_mbd; + const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const uint8_t *const color_map = xd->plane[plane].color_index_map; + const PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; + const int n = pmi->palette_size[plane]; + int i, j; + int this_rate = 0; + uint8_t color_order[PALETTE_MAX_SIZE]; + const aom_prob( + *const probs)[PALETTE_COLOR_INDEX_CONTEXTS][PALETTE_COLORS - 1] = + plane == 0 ? av1_default_palette_y_color_index_prob + : av1_default_palette_uv_color_index_prob; + int plane_block_width, rows, cols; + av1_get_block_dimensions(bsize, plane, xd, &plane_block_width, NULL, &rows, + &cols); + assert(plane == 0 || plane == 1); + +#if CONFIG_PALETTE_THROUGHPUT + int k; + for (k = 1; k < rows + cols - 1; ++k) { + for (j = AOMMIN(k, cols - 1); j >= AOMMAX(0, k - rows + 1); --j) { + i = k - j; +#else + for (i = 0; i < rows; ++i) { + for (j = (i == 0 ? 1 : 0); j < cols; ++j) { +#endif // CONFIG_PALETTE_THROUGHPUT + int color_new_idx; + const int color_ctx = av1_get_palette_color_index_context( + color_map, plane_block_width, i, j, n, color_order, &color_new_idx); + assert(color_new_idx >= 0 && color_new_idx < n); + if (dry_run == DRY_RUN_COSTCOEFFS) + this_rate += cpi->palette_y_color_cost[n - PALETTE_MIN_SIZE][color_ctx] + [color_new_idx]; + (*t)->token = color_new_idx; + (*t)->context_tree = probs[n - PALETTE_MIN_SIZE][color_ctx]; + (*t)->skip_eob_node = 0; + ++(*t); + } + } + if (rate) *rate += this_rate; +} +#endif // CONFIG_PALETTE + +#if CONFIG_PVQ +static void add_pvq_block(AV1_COMMON *const cm, MACROBLOCK *const x, + PVQ_INFO *pvq) { + PVQ_QUEUE *q = x->pvq_q; + if (q->curr_pos >= q->buf_len) { + int new_buf_len = 2 * q->buf_len + 1; + PVQ_INFO *new_buf; + CHECK_MEM_ERROR(cm, new_buf, aom_malloc(new_buf_len * sizeof(PVQ_INFO))); + memcpy(new_buf, q->buf, q->buf_len * sizeof(PVQ_INFO)); + aom_free(q->buf); + q->buf = new_buf; + q->buf_len = new_buf_len; + } + OD_COPY(q->buf + q->curr_pos, pvq, 1); + ++q->curr_pos; +} + +// NOTE: This does not actually generate tokens, instead we store the encoding +// decisions made for PVQ in a queue that we will read from when +// actually writing the bitstream in write_modes_b +static void tokenize_pvq(int plane, int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) { + struct tokenize_b_args *const args = arg; + const AV1_COMP *cpi = args->cpi; + const AV1_COMMON *const cm = &cpi->common; + ThreadData *const td = args->td; + MACROBLOCK *const x = &td->mb; + PVQ_INFO *pvq_info; + + (void)block; + (void)blk_row; + (void)blk_col; + (void)plane_bsize; + (void)tx_size; + + assert(block < MAX_PVQ_BLOCKS_IN_SB); + pvq_info = &x->pvq[block][plane]; + add_pvq_block((AV1_COMMON * const)cm, x, pvq_info); +} +#endif // CONFIG_PVQ + +static void tokenize_b(int plane, int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) { +#if !CONFIG_PVQ + struct tokenize_b_args *const args = arg; + const AV1_COMP *cpi = args->cpi; + const AV1_COMMON *const cm = &cpi->common; + ThreadData *const td = args->td; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + TOKENEXTRA **tp = args->tp; + uint8_t token_cache[MAX_TX_SQUARE]; + struct macroblock_plane *p = &x->plane[plane]; + struct macroblockd_plane *pd = &xd->plane[plane]; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + int pt; /* near block/prev token context index */ + int c; + TOKENEXTRA *t = *tp; /* store tokens starting here */ + const int eob = p->eobs[block]; + const PLANE_TYPE type = pd->plane_type; + const tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block); +#if CONFIG_SUPERTX + const int segment_id = AOMMIN(mbmi->segment_id, mbmi->segment_id_supertx); +#else + const int segment_id = mbmi->segment_id; +#endif // CONFIG_SUEPRTX + const int16_t *scan, *nb; + const TX_TYPE tx_type = get_tx_type(type, xd, block, tx_size); + const SCAN_ORDER *const scan_order = + get_scan(cm, tx_size, tx_type, is_inter_block(mbmi)); + const int ref = is_inter_block(mbmi); + unsigned int(*const counts)[COEFF_CONTEXTS][ENTROPY_TOKENS] = + td->rd_counts.coef_counts[txsize_sqr_map[tx_size]][type][ref]; +#if !CONFIG_NEW_TOKENSET +#if CONFIG_SUBFRAME_PROB_UPDATE + const aom_prob(*coef_probs)[COEFF_CONTEXTS][UNCONSTRAINED_NODES] = + cpi->subframe_stats.coef_probs_buf[cpi->common.coef_probs_update_idx] + [txsize_sqr_map[tx_size]][type][ref]; +#else + aom_prob(*const coef_probs)[COEFF_CONTEXTS][UNCONSTRAINED_NODES] = + cpi->common.fc->coef_probs[txsize_sqr_map[tx_size]][type][ref]; +#endif // CONFIG_SUBFRAME_PROB_UPDATE +#endif // !CONFIG_NEW_TOKENSET +#if CONFIG_EC_ADAPT + FRAME_CONTEXT *ec_ctx = xd->tile_ctx; +#elif CONFIG_EC_MULTISYMBOL + FRAME_CONTEXT *ec_ctx = cpi->common.fc; +#endif +#if CONFIG_NEW_TOKENSET + aom_cdf_prob( + *const coef_head_cdfs)[COEFF_CONTEXTS][CDF_SIZE(ENTROPY_TOKENS)] = + ec_ctx->coef_head_cdfs[txsize_sqr_map[tx_size]][type][ref]; + aom_cdf_prob( + *const coef_tail_cdfs)[COEFF_CONTEXTS][CDF_SIZE(ENTROPY_TOKENS)] = + ec_ctx->coef_tail_cdfs[txsize_sqr_map[tx_size]][type][ref]; + unsigned int(*const blockz_count)[2] = + td->counts->blockz_count[txsize_sqr_map[tx_size]][type][ref]; + int eob_val; + int first_val = 1; +#else +#if CONFIG_EC_MULTISYMBOL + aom_cdf_prob(*const coef_cdfs)[COEFF_CONTEXTS][CDF_SIZE(ENTROPY_TOKENS)] = + ec_ctx->coef_cdfs[txsize_sqr_map[tx_size]][type][ref]; +#endif + int skip_eob = 0; +#endif + const int seg_eob = get_tx_eob(&cpi->common.seg, segment_id, tx_size); + unsigned int(*const eob_branch)[COEFF_CONTEXTS] = + td->counts->eob_branch[txsize_sqr_map[tx_size]][type][ref]; + const uint8_t *const band = get_band_translate(tx_size); + int16_t token; + EXTRABIT extra; + (void)plane_bsize; + pt = get_entropy_context(tx_size, pd->above_context + blk_col, + pd->left_context + blk_row); + scan = scan_order->scan; + nb = scan_order->neighbors; + c = 0; + +#if CONFIG_NEW_TOKENSET + if (eob == 0) + add_token(&t, &coef_tail_cdfs[band[c]][pt], &coef_head_cdfs[band[c]][pt], 1, + 1, 0, BLOCK_Z_TOKEN); + + ++blockz_count[pt][eob != 0]; + + while (c < eob) { + int v = qcoeff[scan[c]]; + first_val = (c == 0); + + if (!v) { + add_token(&t, &coef_tail_cdfs[band[c]][pt], &coef_head_cdfs[band[c]][pt], + 0, first_val, 0, ZERO_TOKEN); + ++counts[band[c]][pt][ZERO_TOKEN]; + token_cache[scan[c]] = 0; + } else { + eob_val = + (c + 1 == eob) ? (c + 1 == seg_eob ? LAST_EOB : EARLY_EOB) : NO_EOB; + + av1_get_token_extra(v, &token, &extra); + + add_token(&t, &coef_tail_cdfs[band[c]][pt], &coef_head_cdfs[band[c]][pt], + eob_val, first_val, extra, (uint8_t)token); + + if (eob_val != LAST_EOB) { + ++counts[band[c]][pt][token]; + ++eob_branch[band[c]][pt]; + counts[band[c]][pt][EOB_TOKEN] += eob_val != NO_EOB; + } + + token_cache[scan[c]] = av1_pt_energy_class[token]; + } + ++c; + pt = get_coef_context(nb, token_cache, AOMMIN(c, eob - 1)); + } +#else + while (c < eob) { + const int v = qcoeff[scan[c]]; + eob_branch[band[c]][pt] += !skip_eob; + + av1_get_token_extra(v, &token, &extra); + + add_token(&t, coef_probs[band[c]][pt], +#if CONFIG_EC_MULTISYMBOL + &coef_cdfs[band[c]][pt], +#endif + extra, (uint8_t)token, (uint8_t)skip_eob, counts[band[c]][pt]); + + token_cache[scan[c]] = av1_pt_energy_class[token]; + ++c; + pt = get_coef_context(nb, token_cache, c); + skip_eob = (token == ZERO_TOKEN); + } + if (c < seg_eob) { + add_token(&t, coef_probs[band[c]][pt], +#if CONFIG_EC_MULTISYMBOL + NULL, +#endif + 0, EOB_TOKEN, 0, counts[band[c]][pt]); + ++eob_branch[band[c]][pt]; + } +#endif // CONFIG_NEW_TOKENSET + +#if CONFIG_COEF_INTERLEAVE + t->token = EOSB_TOKEN; + t++; +#endif + + *tp = t; + +#if CONFIG_ADAPT_SCAN + // Since dqcoeff is not available here, we pass qcoeff into + // av1_update_scan_count_facade(). The update behavior should be the same + // because av1_update_scan_count_facade() only cares if coefficients are zero + // or not. + av1_update_scan_count_facade((AV1_COMMON *)cm, td->counts, tx_size, tx_type, + qcoeff, c); +#endif + + av1_set_contexts(xd, pd, plane, tx_size, c > 0, blk_col, blk_row); +#else // !CONFIG_PVQ + tokenize_pvq(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg); +#endif // !CONFIG_PVQ +} + +struct is_skippable_args { + uint16_t *eobs; + int *skippable; +}; +static void is_skippable(int plane, int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *argv) { + struct is_skippable_args *args = argv; + (void)plane; + (void)plane_bsize; + (void)tx_size; + (void)blk_row; + (void)blk_col; + args->skippable[0] &= (!args->eobs[block]); +} + +// TODO(yaowu): rewrite and optimize this function to remove the usage of +// av1_foreach_transform_block() and simplify is_skippable(). +int av1_is_skippable_in_plane(MACROBLOCK *x, BLOCK_SIZE bsize, int plane) { + int result = 1; + struct is_skippable_args args = { x->plane[plane].eobs, &result }; + av1_foreach_transformed_block_in_plane(&x->e_mbd, bsize, plane, is_skippable, + &args); + return result; +} + +#if CONFIG_VAR_TX +void tokenize_vartx(ThreadData *td, TOKENEXTRA **t, RUN_TYPE dry_run, + TX_SIZE tx_size, BLOCK_SIZE plane_bsize, int blk_row, + int blk_col, int block, int plane, void *arg) { + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const BLOCK_SIZE bsize = txsize_to_bsize[tx_size]; + const int tx_row = blk_row >> (1 - pd->subsampling_y); + const int tx_col = blk_col >> (1 - pd->subsampling_x); + const int max_blocks_high = max_block_high(xd, plane_bsize, plane); + const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); + TX_SIZE plane_tx_size; + + if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + + plane_tx_size = + plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0] + : mbmi->inter_tx_size[tx_row][tx_col]; + + if (tx_size == plane_tx_size) { + plane_bsize = get_plane_block_size(mbmi->sb_type, pd); + if (!dry_run) + tokenize_b(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg); + else if (dry_run == DRY_RUN_NORMAL) + set_entropy_context_b(plane, block, blk_row, blk_col, plane_bsize, + tx_size, arg); + else if (dry_run == DRY_RUN_COSTCOEFFS) + cost_coeffs_b(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg); + } else { + // Half the block size in transform block unit. + const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; + const int bsl = tx_size_wide_unit[sub_txs]; + int i; + + assert(bsl > 0); + + for (i = 0; i < 4; ++i) { + const int offsetr = blk_row + ((i >> 1) * bsl); + const int offsetc = blk_col + ((i & 0x01) * bsl); + + int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs]; + + if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue; + + tokenize_vartx(td, t, dry_run, sub_txs, plane_bsize, offsetr, offsetc, + block, plane, arg); + block += step; + } + } +} + +void av1_tokenize_sb_vartx(const AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t, + RUN_TYPE dry_run, int mi_row, int mi_col, + BLOCK_SIZE bsize, int *rate) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + TOKENEXTRA *t_backup = *t; + const int ctx = av1_get_skip_context(xd); + const int skip_inc = + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP); + struct tokenize_b_args arg = { cpi, td, t, 0 }; + int plane; + if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + + if (mbmi->skip) { + if (!dry_run) td->counts->skip[ctx][1] += skip_inc; + reset_skip_context(xd, bsize); + if (dry_run) *t = t_backup; + return; + } + + if (!dry_run) + td->counts->skip[ctx][0] += skip_inc; + else + *t = t_backup; + + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { +#if CONFIG_CB4X4 + if (!is_chroma_reference(mi_row, mi_col, bsize, + xd->plane[plane].subsampling_x, + xd->plane[plane].subsampling_y)) { +#if !CONFIG_PVQ + if (!dry_run) { + (*t)->token = EOSB_TOKEN; + (*t)++; + } +#endif + continue; + } +#endif + const struct macroblockd_plane *const pd = &xd->plane[plane]; +#if CONFIG_CB4X4 && !CONFIG_CHROMA_2X2 + const BLOCK_SIZE plane_bsize = + AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd)); +#else + const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); +#endif + const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; + const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0]; + const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize); + const BLOCK_SIZE txb_size = txsize_to_bsize[max_tx_size]; + int bw = block_size_wide[txb_size] >> tx_size_wide_log2[0]; + int bh = block_size_high[txb_size] >> tx_size_wide_log2[0]; + int idx, idy; + int block = 0; + int step = tx_size_wide_unit[max_tx_size] * tx_size_high_unit[max_tx_size]; + for (idy = 0; idy < mi_height; idy += bh) { + for (idx = 0; idx < mi_width; idx += bw) { + tokenize_vartx(td, t, dry_run, max_tx_size, plane_bsize, idy, idx, + block, plane, &arg); + block += step; + } + } + + if (!dry_run) { + (*t)->token = EOSB_TOKEN; + (*t)++; + } + } + if (rate) *rate += arg.this_rate; +} +#endif // CONFIG_VAR_TX + +void av1_tokenize_sb(const AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t, + RUN_TYPE dry_run, BLOCK_SIZE bsize, int *rate, + const int mi_row, const int mi_col) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + const int ctx = av1_get_skip_context(xd); + const int skip_inc = + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP); + struct tokenize_b_args arg = { cpi, td, t, 0 }; + if (mbmi->skip) { + if (!dry_run) td->counts->skip[ctx][1] += skip_inc; + reset_skip_context(xd, bsize); + return; + } + + if (!dry_run) { +#if CONFIG_COEF_INTERLEAVE + td->counts->skip[ctx][0] += skip_inc; + av1_foreach_transformed_block_interleave(xd, bsize, tokenize_b, &arg); +#else + int plane; + + td->counts->skip[ctx][0] += skip_inc; + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { +#if CONFIG_CB4X4 + if (!is_chroma_reference(mi_row, mi_col, bsize, + xd->plane[plane].subsampling_x, + xd->plane[plane].subsampling_y)) { +#if !CONFIG_PVQ + (*t)->token = EOSB_TOKEN; + (*t)++; +#endif + continue; + } +#else + (void)mi_row; + (void)mi_col; +#endif + av1_foreach_transformed_block_in_plane(xd, bsize, plane, tokenize_b, + &arg); +#if !CONFIG_PVQ + (*t)->token = EOSB_TOKEN; + (*t)++; +#endif // !CONFIG_PVQ + } +#endif + } +#if !CONFIG_PVQ + else if (dry_run == DRY_RUN_NORMAL) { + int plane; + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { +#if CONFIG_CB4X4 + if (!is_chroma_reference(mi_row, mi_col, bsize, + xd->plane[plane].subsampling_x, + xd->plane[plane].subsampling_y)) + continue; +#else + (void)mi_row; + (void)mi_col; +#endif + av1_foreach_transformed_block_in_plane(xd, bsize, plane, + set_entropy_context_b, &arg); + } + } else if (dry_run == DRY_RUN_COSTCOEFFS) { + int plane; + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { +#if CONFIG_CB4X4 + if (!is_chroma_reference(mi_row, mi_col, bsize, + xd->plane[plane].subsampling_x, + xd->plane[plane].subsampling_y)) + continue; +#else + (void)mi_row; + (void)mi_col; +#endif + av1_foreach_transformed_block_in_plane(xd, bsize, plane, cost_coeffs_b, + &arg); + } + } +#endif // !CONFIG_PVQ + + if (rate) *rate += arg.this_rate; +} + +#if CONFIG_SUPERTX +void av1_tokenize_sb_supertx(const AV1_COMP *cpi, ThreadData *td, + TOKENEXTRA **t, RUN_TYPE dry_run, BLOCK_SIZE bsize, + int *rate) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &td->mb.e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + TOKENEXTRA *t_backup = *t; + const int ctx = av1_get_skip_context(xd); + const int skip_inc = + !segfeature_active(&cm->seg, mbmi->segment_id_supertx, SEG_LVL_SKIP); + struct tokenize_b_args arg = { cpi, td, t, 0 }; + if (mbmi->skip) { + if (!dry_run) td->counts->skip[ctx][1] += skip_inc; + reset_skip_context(xd, bsize); + if (dry_run) *t = t_backup; + return; + } + + if (!dry_run) { + int plane; + td->counts->skip[ctx][0] += skip_inc; + + for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + av1_foreach_transformed_block_in_plane(xd, bsize, plane, tokenize_b, + &arg); + (*t)->token = EOSB_TOKEN; + (*t)++; + } + } else if (dry_run == DRY_RUN_NORMAL) { + int plane; + for (plane = 0; plane < MAX_MB_PLANE; ++plane) + av1_foreach_transformed_block_in_plane(xd, bsize, plane, + set_entropy_context_b, &arg); + *t = t_backup; + } else if (dry_run == DRY_RUN_COSTCOEFFS) { + int plane; + for (plane = 0; plane < MAX_MB_PLANE; ++plane) + av1_foreach_transformed_block_in_plane(xd, bsize, plane, cost_coeffs_b, + &arg); + } + if (rate) *rate += arg.this_rate; +} +#endif // CONFIG_SUPERTX diff --git a/third_party/aom/av1/encoder/tokenize.h b/third_party/aom/av1/encoder/tokenize.h new file mode 100644 index 0000000000..3928111d6f --- /dev/null +++ b/third_party/aom/av1/encoder/tokenize.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_TOKENIZE_H_ +#define AV1_ENCODER_TOKENIZE_H_ + +#include "av1/common/entropy.h" + +#include "av1/encoder/block.h" +#include "av1/encoder/treewriter.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define EOSB_TOKEN 127 // Not signalled, encoder only + +#if CONFIG_HIGHBITDEPTH +typedef int32_t EXTRABIT; +#else +typedef int16_t EXTRABIT; +#endif + +typedef struct { + int16_t token; + EXTRABIT extra; +} TOKENVALUE; + +typedef struct { +#if CONFIG_NEW_TOKENSET + aom_cdf_prob (*tail_cdf)[CDF_SIZE(ENTROPY_TOKENS)]; + aom_cdf_prob (*head_cdf)[CDF_SIZE(ENTROPY_TOKENS)]; + int eob_val; + int first_val; +#elif CONFIG_EC_MULTISYMBOL + aom_cdf_prob (*token_cdf)[CDF_SIZE(ENTROPY_TOKENS)]; +#endif + const aom_prob *context_tree; + EXTRABIT extra; + uint8_t token; + uint8_t skip_eob_node; +} TOKENEXTRA; + +extern const aom_tree_index av1_coef_tree[]; +extern const aom_tree_index av1_coef_con_tree[]; +#if !CONFIG_EC_MULTISYMBOL +extern const struct av1_token av1_coef_encodings[]; +#endif // !CONFIG_EC_MULTISYMBOL + +int av1_is_skippable_in_plane(MACROBLOCK *x, BLOCK_SIZE bsize, int plane); + +struct AV1_COMP; +struct ThreadData; + +typedef enum { + OUTPUT_ENABLED = 0, + DRY_RUN_NORMAL, + DRY_RUN_COSTCOEFFS, +} RUN_TYPE; + +// Note in all the tokenize functions rate if non NULL is incremented +// with the coefficient token cost only if dry_run = DRY_RUN_COSTCOEFS, +// otherwise rate is not incremented. +#if CONFIG_VAR_TX +void av1_tokenize_sb_vartx(const struct AV1_COMP *cpi, struct ThreadData *td, + TOKENEXTRA **t, RUN_TYPE dry_run, int mi_row, + int mi_col, BLOCK_SIZE bsize, int *rate); +#endif +#if CONFIG_PALETTE +void av1_tokenize_palette_sb(const struct AV1_COMP *cpi, + const struct ThreadData *const td, int plane, + TOKENEXTRA **t, RUN_TYPE dry_run, BLOCK_SIZE bsize, + int *rate); +#endif // CONFIG_PALETTE +void av1_tokenize_sb(const struct AV1_COMP *cpi, struct ThreadData *td, + TOKENEXTRA **t, RUN_TYPE dry_run, BLOCK_SIZE bsize, + int *rate, const int mi_row, const int mi_col); +#if CONFIG_SUPERTX +void av1_tokenize_sb_supertx(const struct AV1_COMP *cpi, struct ThreadData *td, + TOKENEXTRA **t, RUN_TYPE dry_run, BLOCK_SIZE bsize, + int *rate); +#endif + +extern const int16_t *av1_dct_value_cost_ptr; +/* TODO: The Token field should be broken out into a separate char array to + * improve cache locality, since it's needed for costing when the rest of the + * fields are not. + */ +extern const TOKENVALUE *av1_dct_value_tokens_ptr; +extern const TOKENVALUE *av1_dct_cat_lt_10_value_tokens; +extern const int *av1_dct_cat_lt_10_value_cost; +extern const int16_t av1_cat6_low_cost[256]; +#if CONFIG_HIGHBITDEPTH +#define CAT6_HIGH_COST_ENTRIES 1024 +#else +#define CAT6_HIGH_COST_ENTRIES 64 +#endif +extern const int av1_cat6_high_cost[CAT6_HIGH_COST_ENTRIES]; +extern const uint8_t av1_cat6_skipped_bits_discount[8]; + +static INLINE void av1_get_token_extra(int v, int16_t *token, EXTRABIT *extra) { + if (v >= CAT6_MIN_VAL || v <= -CAT6_MIN_VAL) { + *token = CATEGORY6_TOKEN; + if (v >= CAT6_MIN_VAL) + *extra = 2 * v - 2 * CAT6_MIN_VAL; + else + *extra = -2 * v - 2 * CAT6_MIN_VAL + 1; + return; + } + *token = av1_dct_cat_lt_10_value_tokens[v].token; + *extra = av1_dct_cat_lt_10_value_tokens[v].extra; +} +static INLINE int16_t av1_get_token(int v) { + if (v >= CAT6_MIN_VAL || v <= -CAT6_MIN_VAL) return 10; + return av1_dct_cat_lt_10_value_tokens[v].token; +} + +static INLINE int av1_get_token_cost(int v, int16_t *token, int cat6_bits) { + if (v >= CAT6_MIN_VAL || v <= -CAT6_MIN_VAL) { + EXTRABIT extrabits; + *token = CATEGORY6_TOKEN; + extrabits = abs(v) - CAT6_MIN_VAL; + return av1_cat6_low_cost[extrabits & 0xff] + + av1_cat6_high_cost[extrabits >> 8] - + av1_cat6_skipped_bits_discount[18 - cat6_bits]; + } + *token = av1_dct_cat_lt_10_value_tokens[v].token; + return av1_dct_cat_lt_10_value_cost[v]; +} + +#if !CONFIG_PVQ || CONFIG_VAR_TX +static INLINE int get_tx_eob(const struct segmentation *seg, int segment_id, + TX_SIZE tx_size) { + const int eob_max = tx_size_2d[tx_size]; + return segfeature_active(seg, segment_id, SEG_LVL_SKIP) ? 0 : eob_max; +} +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_TOKENIZE_H_ diff --git a/third_party/aom/av1/encoder/treewriter.c b/third_party/aom/av1/encoder/treewriter.c new file mode 100644 index 0000000000..50be72413b --- /dev/null +++ b/third_party/aom/av1/encoder/treewriter.c @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/encoder/treewriter.h" + +static void tree2tok(struct av1_token *tokens, const aom_tree_index *tree, + int i, int v, int l) { + v += v; + ++l; + + do { + const aom_tree_index j = tree[i++]; + if (j <= 0) { + tokens[-j].value = v; + tokens[-j].len = l; + } else { + tree2tok(tokens, tree, j, v, l); + } + } while (++v & 1); +} + +void av1_tokens_from_tree(struct av1_token *tokens, + const aom_tree_index *tree) { + tree2tok(tokens, tree, 0, 0, 0); +} + +static unsigned int convert_distribution(unsigned int i, aom_tree tree, + unsigned int branch_ct[][2], + const unsigned int num_events[]) { + unsigned int left, right; + + if (tree[i] <= 0) + left = num_events[-tree[i]]; + else + left = convert_distribution(tree[i], tree, branch_ct, num_events); + + if (tree[i + 1] <= 0) + right = num_events[-tree[i + 1]]; + else + right = convert_distribution(tree[i + 1], tree, branch_ct, num_events); + + branch_ct[i >> 1][0] = left; + branch_ct[i >> 1][1] = right; + return left + right; +} + +void av1_tree_probs_from_distribution(aom_tree tree, + unsigned int branch_ct[/* n-1 */][2], + const unsigned int num_events[/* n */]) { + convert_distribution(0, tree, branch_ct, num_events); +} diff --git a/third_party/aom/av1/encoder/treewriter.h b/third_party/aom/av1/encoder/treewriter.h new file mode 100644 index 0000000000..9a4cb86cb2 --- /dev/null +++ b/third_party/aom/av1/encoder/treewriter.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_TREEWRITER_H_ +#define AV1_ENCODER_TREEWRITER_H_ + +#include "aom_dsp/bitwriter.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_tree_probs_from_distribution(aom_tree tree, + unsigned int branch_ct[/* n - 1 */][2], + const unsigned int num_events[/* n */]); + +struct av1_token { + int value; + int len; +}; + +void av1_tokens_from_tree(struct av1_token *, const aom_tree_index *); + +static INLINE void av1_write_token(aom_writer *w, const aom_tree_index *tree, + const aom_prob *probs, + const struct av1_token *token) { + aom_write_tree(w, tree, probs, token->value, token->len, 0); +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AV1_ENCODER_TREEWRITER_H_ diff --git a/third_party/aom/av1/encoder/variance_tree.c b/third_party/aom/av1/encoder/variance_tree.c new file mode 100644 index 0000000000..9384cd78ef --- /dev/null +++ b/third_party/aom/av1/encoder/variance_tree.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/encoder/variance_tree.h" +#include "av1/encoder/encoder.h" + +void av1_setup_var_tree(struct AV1Common *cm, ThreadData *td) { + int i, j; +#if CONFIG_EXT_PARTITION + const int leaf_nodes = 1024; + const int tree_nodes = 1024 + 256 + 64 + 16 + 4 + 1; +#else + const int leaf_nodes = 256; + const int tree_nodes = 256 + 64 + 16 + 4 + 1; +#endif // CONFIG_EXT_PARTITION + int index = 0; + VAR_TREE *this_var; + int nodes; + + aom_free(td->var_tree); + CHECK_MEM_ERROR(cm, td->var_tree, + aom_calloc(tree_nodes, sizeof(*td->var_tree))); + + this_var = &td->var_tree[0]; + + // Sets up all the leaf nodes in the tree. + for (index = 0; index < leaf_nodes; ++index) { + VAR_TREE *const leaf = &td->var_tree[index]; + leaf->split[0] = NULL; + } + + // Each node has 4 leaf nodes, fill in the child pointers + // from leafs to the root. + for (nodes = leaf_nodes >> 2; nodes > 0; nodes >>= 2) { + for (i = 0; i < nodes; ++i, ++index) { + VAR_TREE *const node = &td->var_tree[index]; + for (j = 0; j < 4; j++) node->split[j] = this_var++; + } + } + + // Set up the root node for the largest superblock size + i = MAX_MIB_SIZE_LOG2 - MIN_MIB_SIZE_LOG2; + td->var_root[i] = &td->var_tree[tree_nodes - 1]; + // Set up the root nodes for the rest of the possible superblock sizes + while (--i >= 0) { + td->var_root[i] = td->var_root[i + 1]->split[0]; + } +} + +void av1_free_var_tree(ThreadData *td) { + aom_free(td->var_tree); + td->var_tree = NULL; +} diff --git a/third_party/aom/av1/encoder/variance_tree.h b/third_party/aom/av1/encoder/variance_tree.h new file mode 100644 index 0000000000..a9f27302e9 --- /dev/null +++ b/third_party/aom/av1/encoder/variance_tree.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_VARIANCE_TREE_H_ +#define AV1_ENCODER_VARIANCE_TREE_H_ + +#include + +#include "./aom_config.h" + +#include "aom/aom_integer.h" + +#include "av1/common/enums.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct AV1Common; +struct ThreadData; + +typedef struct { + int64_t sum_square_error; + int64_t sum_error; + int log2_count; + int variance; +} VAR; + +typedef struct { + VAR none; + VAR horz[2]; + VAR vert[2]; +} partition_variance; + +typedef struct VAR_TREE { + int force_split; + partition_variance variances; + struct VAR_TREE *split[4]; + BLOCK_SIZE bsize; + const uint8_t *src; + const uint8_t *ref; + int src_stride; + int ref_stride; + int width; + int height; +#if CONFIG_HIGHBITDEPTH + int highbd; +#endif // CONFIG_HIGHBITDEPTH +} VAR_TREE; + +void av1_setup_var_tree(struct AV1Common *cm, struct ThreadData *td); +void av1_free_var_tree(struct ThreadData *td); + +// Set variance values given sum square error, sum error, count. +static INLINE void fill_variance(int64_t s2, int64_t s, int c, VAR *v) { + v->sum_square_error = s2; + v->sum_error = s; + v->log2_count = c; + v->variance = + (int)(256 * (v->sum_square_error - + ((v->sum_error * v->sum_error) >> v->log2_count)) >> + v->log2_count); +} + +static INLINE void sum_2_variances(const VAR *a, const VAR *b, VAR *r) { + assert(a->log2_count == b->log2_count); + fill_variance(a->sum_square_error + b->sum_square_error, + a->sum_error + b->sum_error, a->log2_count + 1, r); +} + +static INLINE void fill_variance_node(VAR_TREE *vt) { + sum_2_variances(&vt->split[0]->variances.none, &vt->split[1]->variances.none, + &vt->variances.horz[0]); + sum_2_variances(&vt->split[2]->variances.none, &vt->split[3]->variances.none, + &vt->variances.horz[1]); + sum_2_variances(&vt->split[0]->variances.none, &vt->split[2]->variances.none, + &vt->variances.vert[0]); + sum_2_variances(&vt->split[1]->variances.none, &vt->split[3]->variances.none, + &vt->variances.vert[1]); + sum_2_variances(&vt->variances.vert[0], &vt->variances.vert[1], + &vt->variances.none); +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* AV1_ENCODER_VARIANCE_TREE_H_ */ diff --git a/third_party/aom/av1/encoder/wedge_utils.c b/third_party/aom/av1/encoder/wedge_utils.c new file mode 100644 index 0000000000..e6edbb6af0 --- /dev/null +++ b/third_party/aom/av1/encoder/wedge_utils.c @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "aom/aom_integer.h" + +#include "aom_ports/mem.h" + +#include "aom_dsp/aom_dsp_common.h" + +#include "av1/common/reconinter.h" + +#define MAX_MASK_VALUE (1 << WEDGE_WEIGHT_BITS) + +/** + * Computes SSE of a compound predictor constructed from 2 fundamental + * predictors p0 and p1 using blending with mask. + * + * r1: Residuals of p1. + * (source - p1) + * d: Difference of p1 and p0. + * (p1 - p0) + * m: The blending mask + * N: Number of pixels + * + * 'r1', 'd', and 'm' are contiguous. + * + * Computes: + * Sum((MAX_MASK_VALUE*r1 + mask*d)**2), which is equivalent to: + * Sum((mask*r0 + (MAX_MASK_VALUE-mask)*r1)**2), + * where r0 is (source - p0), and r1 is (source - p1), which is in turn + * is equivalent to: + * Sum((source*MAX_MASK_VALUE - (mask*p0 + (MAX_MASK_VALUE-mask)*p1))**2), + * which is the SSE of the residuals of the compound predictor scaled up by + * MAX_MASK_VALUE**2. + * + * Note that we clamp the partial term in the loop to 16 bits signed. This is + * to facilitate equivalent SIMD implementation. It should have no effect if + * residuals are within 16 - WEDGE_WEIGHT_BITS (=10) signed, which always + * holds for 8 bit input, and on real input, it should hold practically always, + * as residuals are expected to be small. + */ +uint64_t av1_wedge_sse_from_residuals_c(const int16_t *r1, const int16_t *d, + const uint8_t *m, int N) { + uint64_t csse = 0; + int i; + + for (i = 0; i < N; i++) { + int32_t t = MAX_MASK_VALUE * r1[i] + m[i] * d[i]; + t = clamp(t, INT16_MIN, INT16_MAX); + csse += t * t; + } + return ROUND_POWER_OF_TWO(csse, 2 * WEDGE_WEIGHT_BITS); +} + +/** + * Choose the mask sign for a compound predictor. + * + * ds: Difference of the squares of the residuals. + * r0**2 - r1**2 + * m: The blending mask + * N: Number of pixels + * limit: Pre-computed threshold value. + * MAX_MASK_VALUE/2 * (sum(r0**2) - sum(r1**2)) + * + * 'ds' and 'm' are contiguous. + * + * Returns true if the negated mask has lower SSE compared to the positive + * mask. Computation is based on: + * Sum((mask*r0 + (MAX_MASK_VALUE-mask)*r1)**2) + * > + * Sum(((MAX_MASK_VALUE-mask)*r0 + mask*r1)**2) + * + * which can be simplified to: + * + * Sum(mask*(r0**2 - r1**2)) > MAX_MASK_VALUE/2 * (sum(r0**2) - sum(r1**2)) + * + * The right hand side does not depend on the mask, and needs to be passed as + * the 'limit' parameter. + * + * After pre-computing (r0**2 - r1**2), which is passed in as 'ds', the left + * hand side is simply a scalar product between an int16_t and uint8_t vector. + * + * Note that for efficiency, ds is stored on 16 bits. Real input residuals + * being small, this should not cause a noticeable issue. + */ +int av1_wedge_sign_from_residuals_c(const int16_t *ds, const uint8_t *m, int N, + int64_t limit) { + int64_t acc = 0; + + do { + acc += *ds++ * *m++; + } while (--N); + + return acc > limit; +} + +/** + * Compute the element-wise difference of the squares of 2 arrays. + * + * d: Difference of the squares of the inputs: a**2 - b**2 + * a: First input array + * b: Second input array + * N: Number of elements + * + * 'd', 'a', and 'b' are contiguous. + * + * The result is saturated to signed 16 bits. + */ +void av1_wedge_compute_delta_squares_c(int16_t *d, const int16_t *a, + const int16_t *b, int N) { + int i; + + for (i = 0; i < N; i++) + d[i] = clamp(a[i] * a[i] - b[i] * b[i], INT16_MIN, INT16_MAX); +} diff --git a/third_party/aom/av1/encoder/x86/av1_highbd_quantize_sse4.c b/third_party/aom/av1/encoder/x86/av1_highbd_quantize_sse4.c new file mode 100644 index 0000000000..fa5626002f --- /dev/null +++ b/third_party/aom/av1/encoder/x86/av1_highbd_quantize_sse4.c @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./av1_rtcd.h" +#include "aom_dsp/aom_dsp_common.h" + +// Coefficient quantization phase 1 +// param[0-2] : rounding/quan/dequan constants +static INLINE void quantize_coeff_phase1(__m128i *coeff, const __m128i *param, + const int shift, const int scale, + __m128i *qcoeff, __m128i *dquan, + __m128i *sign) { + const __m128i zero = _mm_setzero_si128(); + const __m128i one = _mm_set1_epi32(1); + + *sign = _mm_cmplt_epi32(*coeff, zero); + *sign = _mm_or_si128(*sign, one); + *coeff = _mm_abs_epi32(*coeff); + + qcoeff[0] = _mm_add_epi32(*coeff, param[0]); + qcoeff[1] = _mm_unpackhi_epi32(qcoeff[0], zero); + qcoeff[0] = _mm_unpacklo_epi32(qcoeff[0], zero); + + qcoeff[0] = _mm_mul_epi32(qcoeff[0], param[1]); + qcoeff[0] = _mm_srli_epi64(qcoeff[0], shift); + dquan[0] = _mm_mul_epi32(qcoeff[0], param[2]); + dquan[0] = _mm_srli_epi64(dquan[0], scale); +} + +// Coefficient quantization phase 2 +static INLINE void quantize_coeff_phase2(__m128i *qcoeff, __m128i *dquan, + const __m128i *sign, + const __m128i *param, const int shift, + const int scale, tran_low_t *qAddr, + tran_low_t *dqAddr) { + __m128i mask0L = _mm_set_epi32(-1, -1, 0, 0); + __m128i mask0H = _mm_set_epi32(0, 0, -1, -1); + + qcoeff[1] = _mm_mul_epi32(qcoeff[1], param[1]); + qcoeff[1] = _mm_srli_epi64(qcoeff[1], shift); + dquan[1] = _mm_mul_epi32(qcoeff[1], param[2]); + dquan[1] = _mm_srli_epi64(dquan[1], scale); + + // combine L&H + qcoeff[0] = _mm_shuffle_epi32(qcoeff[0], 0xd8); + qcoeff[1] = _mm_shuffle_epi32(qcoeff[1], 0x8d); + + qcoeff[0] = _mm_and_si128(qcoeff[0], mask0H); + qcoeff[1] = _mm_and_si128(qcoeff[1], mask0L); + + dquan[0] = _mm_shuffle_epi32(dquan[0], 0xd8); + dquan[1] = _mm_shuffle_epi32(dquan[1], 0x8d); + + dquan[0] = _mm_and_si128(dquan[0], mask0H); + dquan[1] = _mm_and_si128(dquan[1], mask0L); + + qcoeff[0] = _mm_or_si128(qcoeff[0], qcoeff[1]); + dquan[0] = _mm_or_si128(dquan[0], dquan[1]); + + qcoeff[0] = _mm_sign_epi32(qcoeff[0], *sign); + dquan[0] = _mm_sign_epi32(dquan[0], *sign); + + _mm_storeu_si128((__m128i *)qAddr, qcoeff[0]); + _mm_storeu_si128((__m128i *)dqAddr, dquan[0]); +} + +static INLINE void find_eob(tran_low_t *qcoeff_ptr, const int16_t *iscan, + __m128i *eob) { + const __m128i zero = _mm_setzero_si128(); + __m128i mask, iscanIdx; + const __m128i q0 = _mm_loadu_si128((__m128i const *)qcoeff_ptr); + const __m128i q1 = _mm_loadu_si128((__m128i const *)(qcoeff_ptr + 4)); + __m128i nz_flag0 = _mm_cmpeq_epi32(q0, zero); + __m128i nz_flag1 = _mm_cmpeq_epi32(q1, zero); + + nz_flag0 = _mm_cmpeq_epi32(nz_flag0, zero); + nz_flag1 = _mm_cmpeq_epi32(nz_flag1, zero); + + mask = _mm_packs_epi32(nz_flag0, nz_flag1); + iscanIdx = _mm_loadu_si128((__m128i const *)iscan); + iscanIdx = _mm_sub_epi16(iscanIdx, mask); + iscanIdx = _mm_and_si128(iscanIdx, mask); + *eob = _mm_max_epi16(*eob, iscanIdx); +} + +static INLINE uint16_t get_accumulated_eob(__m128i *eob) { + __m128i eob_shuffled; + uint16_t eobValue; + eob_shuffled = _mm_shuffle_epi32(*eob, 0xe); + *eob = _mm_max_epi16(*eob, eob_shuffled); + eob_shuffled = _mm_shufflelo_epi16(*eob, 0xe); + *eob = _mm_max_epi16(*eob, eob_shuffled); + eob_shuffled = _mm_shufflelo_epi16(*eob, 0x1); + *eob = _mm_max_epi16(*eob, eob_shuffled); + eobValue = _mm_extract_epi16(*eob, 0); + return eobValue; +} + +void av1_highbd_quantize_fp_sse4_1( + const tran_low_t *coeff_ptr, intptr_t count, int skip_block, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, int log_scale) { + __m128i coeff[2], qcoeff[2], dequant[2], qparam[3], coeff_sign; + __m128i eob = _mm_setzero_si128(); + const tran_low_t *src = coeff_ptr; + tran_low_t *quanAddr = qcoeff_ptr; + tran_low_t *dquanAddr = dqcoeff_ptr; + const int shift = 16 - log_scale; + const int coeff_stride = 4; + const int quan_stride = coeff_stride; + (void)skip_block; + (void)zbin_ptr; + (void)quant_shift_ptr; + (void)scan; + + memset(quanAddr, 0, count * sizeof(quanAddr[0])); + memset(dquanAddr, 0, count * sizeof(dquanAddr[0])); + + if (!skip_block) { + coeff[0] = _mm_loadu_si128((__m128i const *)src); + + qparam[0] = + _mm_set_epi32(round_ptr[1], round_ptr[1], round_ptr[1], round_ptr[0]); + qparam[1] = _mm_set_epi64x(quant_ptr[1], quant_ptr[0]); + qparam[2] = _mm_set_epi64x(dequant_ptr[1], dequant_ptr[0]); + + // DC and first 3 AC + quantize_coeff_phase1(&coeff[0], qparam, shift, log_scale, qcoeff, dequant, + &coeff_sign); + + // update round/quan/dquan for AC + qparam[0] = _mm_unpackhi_epi64(qparam[0], qparam[0]); + qparam[1] = _mm_set_epi64x(quant_ptr[1], quant_ptr[1]); + qparam[2] = _mm_set_epi64x(dequant_ptr[1], dequant_ptr[1]); + + quantize_coeff_phase2(qcoeff, dequant, &coeff_sign, qparam, shift, + log_scale, quanAddr, dquanAddr); + + // next 4 AC + coeff[1] = _mm_loadu_si128((__m128i const *)(src + coeff_stride)); + quantize_coeff_phase1(&coeff[1], qparam, shift, log_scale, qcoeff, dequant, + &coeff_sign); + quantize_coeff_phase2(qcoeff, dequant, &coeff_sign, qparam, shift, + log_scale, quanAddr + quan_stride, + dquanAddr + quan_stride); + + find_eob(quanAddr, iscan, &eob); + + count -= 8; + + // loop for the rest of AC + while (count > 0) { + src += coeff_stride << 1; + quanAddr += quan_stride << 1; + dquanAddr += quan_stride << 1; + iscan += quan_stride << 1; + + coeff[0] = _mm_loadu_si128((__m128i const *)src); + coeff[1] = _mm_loadu_si128((__m128i const *)(src + coeff_stride)); + + quantize_coeff_phase1(&coeff[0], qparam, shift, log_scale, qcoeff, + dequant, &coeff_sign); + quantize_coeff_phase2(qcoeff, dequant, &coeff_sign, qparam, shift, + log_scale, quanAddr, dquanAddr); + + quantize_coeff_phase1(&coeff[1], qparam, shift, log_scale, qcoeff, + dequant, &coeff_sign); + quantize_coeff_phase2(qcoeff, dequant, &coeff_sign, qparam, shift, + log_scale, quanAddr + quan_stride, + dquanAddr + quan_stride); + + find_eob(quanAddr, iscan, &eob); + + count -= 8; + } + *eob_ptr = get_accumulated_eob(&eob); + } else { + *eob_ptr = 0; + } +} diff --git a/third_party/aom/av1/encoder/x86/av1_quantize_sse2.c b/third_party/aom/av1/encoder/x86/av1_quantize_sse2.c new file mode 100644 index 0000000000..f9c95b6cb2 --- /dev/null +++ b/third_party/aom/av1/encoder/x86/av1_quantize_sse2.c @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./av1_rtcd.h" +#include "aom/aom_integer.h" + +void av1_quantize_fp_sse2(const int16_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, int16_t *qcoeff_ptr, + int16_t *dqcoeff_ptr, const int16_t *dequant_ptr, + uint16_t *eob_ptr, const int16_t *scan_ptr, + const int16_t *iscan_ptr) { + __m128i zero; + __m128i thr; + int16_t nzflag; + (void)scan_ptr; + (void)zbin_ptr; + (void)quant_shift_ptr; + + coeff_ptr += n_coeffs; + iscan_ptr += n_coeffs; + qcoeff_ptr += n_coeffs; + dqcoeff_ptr += n_coeffs; + n_coeffs = -n_coeffs; + zero = _mm_setzero_si128(); + + if (!skip_block) { + __m128i eob; + __m128i round, quant, dequant; + { + __m128i coeff0, coeff1; + + // Setup global values + { + round = _mm_load_si128((const __m128i *)round_ptr); + quant = _mm_load_si128((const __m128i *)quant_ptr); + dequant = _mm_load_si128((const __m128i *)dequant_ptr); + } + + { + __m128i coeff0_sign, coeff1_sign; + __m128i qcoeff0, qcoeff1; + __m128i qtmp0, qtmp1; + // Do DC and first 15 AC + coeff0 = _mm_load_si128((const __m128i *)(coeff_ptr + n_coeffs)); + coeff1 = _mm_load_si128((const __m128i *)(coeff_ptr + n_coeffs) + 1); + + // Poor man's sign extract + coeff0_sign = _mm_srai_epi16(coeff0, 15); + coeff1_sign = _mm_srai_epi16(coeff1, 15); + qcoeff0 = _mm_xor_si128(coeff0, coeff0_sign); + qcoeff1 = _mm_xor_si128(coeff1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + qcoeff0 = _mm_adds_epi16(qcoeff0, round); + round = _mm_unpackhi_epi64(round, round); + qcoeff1 = _mm_adds_epi16(qcoeff1, round); + qtmp0 = _mm_mulhi_epi16(qcoeff0, quant); + quant = _mm_unpackhi_epi64(quant, quant); + qtmp1 = _mm_mulhi_epi16(qcoeff1, quant); + + // Reinsert signs + qcoeff0 = _mm_xor_si128(qtmp0, coeff0_sign); + qcoeff1 = _mm_xor_si128(qtmp1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), qcoeff0); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, qcoeff1); + + coeff0 = _mm_mullo_epi16(qcoeff0, dequant); + dequant = _mm_unpackhi_epi64(dequant, dequant); + coeff1 = _mm_mullo_epi16(qcoeff1, dequant); + + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), coeff0); + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, coeff1); + } + + { + // Scan for eob + __m128i zero_coeff0, zero_coeff1; + __m128i nzero_coeff0, nzero_coeff1; + __m128i iscan0, iscan1; + __m128i eob1; + zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero); + zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero); + nzero_coeff0 = _mm_cmpeq_epi16(zero_coeff0, zero); + nzero_coeff1 = _mm_cmpeq_epi16(zero_coeff1, zero); + iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs)); + iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs) + 1); + // Add one to convert from indices to counts + iscan0 = _mm_sub_epi16(iscan0, nzero_coeff0); + iscan1 = _mm_sub_epi16(iscan1, nzero_coeff1); + eob = _mm_and_si128(iscan0, nzero_coeff0); + eob1 = _mm_and_si128(iscan1, nzero_coeff1); + eob = _mm_max_epi16(eob, eob1); + } + n_coeffs += 8 * 2; + } + + thr = _mm_srai_epi16(dequant, 1); + + // AC only loop + while (n_coeffs < 0) { + __m128i coeff0, coeff1; + { + __m128i coeff0_sign, coeff1_sign; + __m128i qcoeff0, qcoeff1; + __m128i qtmp0, qtmp1; + + coeff0 = _mm_load_si128((const __m128i *)(coeff_ptr + n_coeffs)); + coeff1 = _mm_load_si128((const __m128i *)(coeff_ptr + n_coeffs) + 1); + + // Poor man's sign extract + coeff0_sign = _mm_srai_epi16(coeff0, 15); + coeff1_sign = _mm_srai_epi16(coeff1, 15); + qcoeff0 = _mm_xor_si128(coeff0, coeff0_sign); + qcoeff1 = _mm_xor_si128(coeff1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + nzflag = _mm_movemask_epi8(_mm_cmpgt_epi16(qcoeff0, thr)) | + _mm_movemask_epi8(_mm_cmpgt_epi16(qcoeff1, thr)); + + if (nzflag) { + qcoeff0 = _mm_adds_epi16(qcoeff0, round); + qcoeff1 = _mm_adds_epi16(qcoeff1, round); + qtmp0 = _mm_mulhi_epi16(qcoeff0, quant); + qtmp1 = _mm_mulhi_epi16(qcoeff1, quant); + + // Reinsert signs + qcoeff0 = _mm_xor_si128(qtmp0, coeff0_sign); + qcoeff1 = _mm_xor_si128(qtmp1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), qcoeff0); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, qcoeff1); + + coeff0 = _mm_mullo_epi16(qcoeff0, dequant); + coeff1 = _mm_mullo_epi16(qcoeff1, dequant); + + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), coeff0); + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, coeff1); + } else { + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, zero); + + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, zero); + } + } + + if (nzflag) { + // Scan for eob + __m128i zero_coeff0, zero_coeff1; + __m128i nzero_coeff0, nzero_coeff1; + __m128i iscan0, iscan1; + __m128i eob0, eob1; + zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero); + zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero); + nzero_coeff0 = _mm_cmpeq_epi16(zero_coeff0, zero); + nzero_coeff1 = _mm_cmpeq_epi16(zero_coeff1, zero); + iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs)); + iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs) + 1); + // Add one to convert from indices to counts + iscan0 = _mm_sub_epi16(iscan0, nzero_coeff0); + iscan1 = _mm_sub_epi16(iscan1, nzero_coeff1); + eob0 = _mm_and_si128(iscan0, nzero_coeff0); + eob1 = _mm_and_si128(iscan1, nzero_coeff1); + eob0 = _mm_max_epi16(eob0, eob1); + eob = _mm_max_epi16(eob, eob0); + } + n_coeffs += 8 * 2; + } + + // Accumulate EOB + { + __m128i eob_shuffled; + eob_shuffled = _mm_shuffle_epi32(eob, 0xe); + eob = _mm_max_epi16(eob, eob_shuffled); + eob_shuffled = _mm_shufflelo_epi16(eob, 0xe); + eob = _mm_max_epi16(eob, eob_shuffled); + eob_shuffled = _mm_shufflelo_epi16(eob, 0x1); + eob = _mm_max_epi16(eob, eob_shuffled); + *eob_ptr = _mm_extract_epi16(eob, 1); + } + } else { + do { + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, zero); + n_coeffs += 8 * 2; + } while (n_coeffs < 0); + *eob_ptr = 0; + } +} diff --git a/third_party/aom/av1/encoder/x86/av1_quantize_ssse3_x86_64.asm b/third_party/aom/av1/encoder/x86/av1_quantize_ssse3_x86_64.asm new file mode 100644 index 0000000000..ad4ae274e2 --- /dev/null +++ b/third_party/aom/av1/encoder/x86/av1_quantize_ssse3_x86_64.asm @@ -0,0 +1,204 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%define private_prefix av1 + +%include "third_party/x86inc/x86inc.asm" + +SECTION_RODATA +pw_1: times 8 dw 1 + +SECTION .text + +%macro QUANTIZE_FP 2 +cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, skip, zbin, round, quant, \ + shift, qcoeff, dqcoeff, dequant, \ + eob, scan, iscan + cmp dword skipm, 0 + jne .blank + + ; actual quantize loop - setup pointers, rounders, etc. + movifnidn coeffq, coeffmp + movifnidn ncoeffq, ncoeffmp + mov r2, dequantmp + movifnidn zbinq, zbinmp + movifnidn roundq, roundmp + movifnidn quantq, quantmp + mova m1, [roundq] ; m1 = round + mova m2, [quantq] ; m2 = quant +%ifidn %1, fp_32x32 + pcmpeqw m5, m5 + psrlw m5, 15 + paddw m1, m5 + psrlw m1, 1 ; m1 = (m1 + 1) / 2 +%endif + mova m3, [r2q] ; m3 = dequant + mov r3, qcoeffmp + mov r4, dqcoeffmp + mov r5, iscanmp +%ifidn %1, fp_32x32 + psllw m2, 1 +%endif + pxor m5, m5 ; m5 = dedicated zero + + lea coeffq, [ coeffq+ncoeffq*2] + lea r5q, [ r5q+ncoeffq*2] + lea r3q, [ r3q+ncoeffq*2] + lea r4q, [r4q+ncoeffq*2] + neg ncoeffq + + ; get DC and first 15 AC coeffs + mova m9, [ coeffq+ncoeffq*2+ 0] ; m9 = c[i] + mova m10, [ coeffq+ncoeffq*2+16] ; m10 = c[i] + pabsw m6, m9 ; m6 = abs(m9) + pabsw m11, m10 ; m11 = abs(m10) + pcmpeqw m7, m7 + + paddsw m6, m1 ; m6 += round + punpckhqdq m1, m1 + paddsw m11, m1 ; m11 += round + pmulhw m8, m6, m2 ; m8 = m6*q>>16 + punpckhqdq m2, m2 + pmulhw m13, m11, m2 ; m13 = m11*q>>16 + psignw m8, m9 ; m8 = reinsert sign + psignw m13, m10 ; m13 = reinsert sign + mova [r3q+ncoeffq*2+ 0], m8 + mova [r3q+ncoeffq*2+16], m13 +%ifidn %1, fp_32x32 + pabsw m8, m8 + pabsw m13, m13 +%endif + pmullw m8, m3 ; r4[i] = r3[i] * q + punpckhqdq m3, m3 + pmullw m13, m3 ; r4[i] = r3[i] * q +%ifidn %1, fp_32x32 + psrlw m8, 1 + psrlw m13, 1 + psignw m8, m9 + psignw m13, m10 + psrlw m0, m3, 2 +%else + psrlw m0, m3, 1 +%endif + mova [r4q+ncoeffq*2+ 0], m8 + mova [r4q+ncoeffq*2+16], m13 + pcmpeqw m8, m5 ; m8 = c[i] == 0 + pcmpeqw m13, m5 ; m13 = c[i] == 0 + mova m6, [ r5q+ncoeffq*2+ 0] ; m6 = scan[i] + mova m11, [ r5q+ncoeffq*2+16] ; m11 = scan[i] + psubw m6, m7 ; m6 = scan[i] + 1 + psubw m11, m7 ; m11 = scan[i] + 1 + pandn m8, m6 ; m8 = max(eob) + pandn m13, m11 ; m13 = max(eob) + pmaxsw m8, m13 + add ncoeffq, mmsize + jz .accumulate_eob + +.ac_only_loop: + mova m9, [ coeffq+ncoeffq*2+ 0] ; m9 = c[i] + mova m10, [ coeffq+ncoeffq*2+16] ; m10 = c[i] + pabsw m6, m9 ; m6 = abs(m9) + pabsw m11, m10 ; m11 = abs(m10) + + pcmpgtw m7, m6, m0 + pcmpgtw m12, m11, m0 + pmovmskb r6d, m7 + pmovmskb r2d, m12 + + or r6, r2 + jz .skip_iter + + pcmpeqw m7, m7 + + paddsw m6, m1 ; m6 += round + paddsw m11, m1 ; m11 += round + pmulhw m14, m6, m2 ; m14 = m6*q>>16 + pmulhw m13, m11, m2 ; m13 = m11*q>>16 + psignw m14, m9 ; m14 = reinsert sign + psignw m13, m10 ; m13 = reinsert sign + mova [r3q+ncoeffq*2+ 0], m14 + mova [r3q+ncoeffq*2+16], m13 +%ifidn %1, fp_32x32 + pabsw m14, m14 + pabsw m13, m13 +%endif + pmullw m14, m3 ; r4[i] = r3[i] * q + pmullw m13, m3 ; r4[i] = r3[i] * q +%ifidn %1, fp_32x32 + psrlw m14, 1 + psrlw m13, 1 + psignw m14, m9 + psignw m13, m10 +%endif + mova [r4q+ncoeffq*2+ 0], m14 + mova [r4q+ncoeffq*2+16], m13 + pcmpeqw m14, m5 ; m14 = c[i] == 0 + pcmpeqw m13, m5 ; m13 = c[i] == 0 + mova m6, [ r5q+ncoeffq*2+ 0] ; m6 = scan[i] + mova m11, [ r5q+ncoeffq*2+16] ; m11 = scan[i] + psubw m6, m7 ; m6 = scan[i] + 1 + psubw m11, m7 ; m11 = scan[i] + 1 + pandn m14, m6 ; m14 = max(eob) + pandn m13, m11 ; m13 = max(eob) + pmaxsw m8, m14 + pmaxsw m8, m13 + add ncoeffq, mmsize + jl .ac_only_loop + + jmp .accumulate_eob +.skip_iter: + mova [r3q+ncoeffq*2+ 0], m5 + mova [r3q+ncoeffq*2+16], m5 + mova [r4q+ncoeffq*2+ 0], m5 + mova [r4q+ncoeffq*2+16], m5 + add ncoeffq, mmsize + jl .ac_only_loop + +.accumulate_eob: + ; horizontally accumulate/max eobs and write into [eob] memory pointer + mov r2, eobmp + pshufd m7, m8, 0xe + pmaxsw m8, m7 + pshuflw m7, m8, 0xe + pmaxsw m8, m7 + pshuflw m7, m8, 0x1 + pmaxsw m8, m7 + pextrw r6, m8, 0 + mov [r2], r6 + RET + + ; skip-block, i.e. just write all zeroes +.blank: + mov r0, dqcoeffmp + movifnidn ncoeffq, ncoeffmp + mov r2, qcoeffmp + mov r3, eobmp + + lea r0q, [r0q+ncoeffq*2] + lea r2q, [r2q+ncoeffq*2] + neg ncoeffq + pxor m7, m7 +.blank_loop: + mova [r0q+ncoeffq*2+ 0], m7 + mova [r0q+ncoeffq*2+16], m7 + mova [r2q+ncoeffq*2+ 0], m7 + mova [r2q+ncoeffq*2+16], m7 + add ncoeffq, mmsize + jl .blank_loop + mov word [r3q], 0 + RET +%endmacro + +INIT_XMM ssse3 +QUANTIZE_FP fp, 7 +QUANTIZE_FP fp_32x32, 7 diff --git a/third_party/aom/av1/encoder/x86/av1_ssim_opt_x86_64.asm b/third_party/aom/av1/encoder/x86/av1_ssim_opt_x86_64.asm new file mode 100644 index 0000000000..dcc697ba30 --- /dev/null +++ b/third_party/aom/av1/encoder/x86/av1_ssim_opt_x86_64.asm @@ -0,0 +1,219 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "aom_ports/x86_abi_support.asm" + +; tabulate_ssim - sums sum_s,sum_r,sum_sq_s,sum_sq_r, sum_sxr +%macro TABULATE_SSIM 0 + paddusw xmm15, xmm3 ; sum_s + paddusw xmm14, xmm4 ; sum_r + movdqa xmm1, xmm3 + pmaddwd xmm1, xmm1 + paddd xmm13, xmm1 ; sum_sq_s + movdqa xmm2, xmm4 + pmaddwd xmm2, xmm2 + paddd xmm12, xmm2 ; sum_sq_r + pmaddwd xmm3, xmm4 + paddd xmm11, xmm3 ; sum_sxr +%endmacro + +; Sum across the register %1 starting with q words +%macro SUM_ACROSS_Q 1 + movdqa xmm2,%1 + punpckldq %1,xmm0 + punpckhdq xmm2,xmm0 + paddq %1,xmm2 + movdqa xmm2,%1 + punpcklqdq %1,xmm0 + punpckhqdq xmm2,xmm0 + paddq %1,xmm2 +%endmacro + +; Sum across the register %1 starting with q words +%macro SUM_ACROSS_W 1 + movdqa xmm1, %1 + punpcklwd %1,xmm0 + punpckhwd xmm1,xmm0 + paddd %1, xmm1 + SUM_ACROSS_Q %1 +%endmacro +;void ssim_parms_sse2( +; unsigned char *s, +; int sp, +; unsigned char *r, +; int rp +; unsigned long *sum_s, +; unsigned long *sum_r, +; unsigned long *sum_sq_s, +; unsigned long *sum_sq_r, +; unsigned long *sum_sxr); +; +; TODO: Use parm passing through structure, probably don't need the pxors +; ( calling app will initialize to 0 ) could easily fit everything in sse2 +; without too much hastle, and can probably do better estimates with psadw +; or pavgb At this point this is just meant to be first pass for calculating +; all the parms needed for 16x16 ssim so we can play with dssim as distortion +; in mode selection code. +global sym(av1_ssim_parms_16x16_sse2) PRIVATE +sym(av1_ssim_parms_16x16_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 9 + SAVE_XMM 15 + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;s + mov rcx, arg(1) ;sp + mov rdi, arg(2) ;r + mov rax, arg(3) ;rp + + pxor xmm0, xmm0 + pxor xmm15,xmm15 ;sum_s + pxor xmm14,xmm14 ;sum_r + pxor xmm13,xmm13 ;sum_sq_s + pxor xmm12,xmm12 ;sum_sq_r + pxor xmm11,xmm11 ;sum_sxr + + mov rdx, 16 ;row counter +.NextRow: + + ;grab source and reference pixels + movdqu xmm5, [rsi] + movdqu xmm6, [rdi] + movdqa xmm3, xmm5 + movdqa xmm4, xmm6 + punpckhbw xmm3, xmm0 ; high_s + punpckhbw xmm4, xmm0 ; high_r + + TABULATE_SSIM + + movdqa xmm3, xmm5 + movdqa xmm4, xmm6 + punpcklbw xmm3, xmm0 ; low_s + punpcklbw xmm4, xmm0 ; low_r + + TABULATE_SSIM + + add rsi, rcx ; next s row + add rdi, rax ; next r row + + dec rdx ; counter + jnz .NextRow + + SUM_ACROSS_W xmm15 + SUM_ACROSS_W xmm14 + SUM_ACROSS_Q xmm13 + SUM_ACROSS_Q xmm12 + SUM_ACROSS_Q xmm11 + + mov rdi,arg(4) + movd [rdi], xmm15; + mov rdi,arg(5) + movd [rdi], xmm14; + mov rdi,arg(6) + movd [rdi], xmm13; + mov rdi,arg(7) + movd [rdi], xmm12; + mov rdi,arg(8) + movd [rdi], xmm11; + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void ssim_parms_sse2( +; unsigned char *s, +; int sp, +; unsigned char *r, +; int rp +; unsigned long *sum_s, +; unsigned long *sum_r, +; unsigned long *sum_sq_s, +; unsigned long *sum_sq_r, +; unsigned long *sum_sxr); +; +; TODO: Use parm passing through structure, probably don't need the pxors +; ( calling app will initialize to 0 ) could easily fit everything in sse2 +; without too much hastle, and can probably do better estimates with psadw +; or pavgb At this point this is just meant to be first pass for calculating +; all the parms needed for 16x16 ssim so we can play with dssim as distortion +; in mode selection code. +global sym(av1_ssim_parms_8x8_sse2) PRIVATE +sym(av1_ssim_parms_8x8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 9 + SAVE_XMM 15 + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;s + mov rcx, arg(1) ;sp + mov rdi, arg(2) ;r + mov rax, arg(3) ;rp + + pxor xmm0, xmm0 + pxor xmm15,xmm15 ;sum_s + pxor xmm14,xmm14 ;sum_r + pxor xmm13,xmm13 ;sum_sq_s + pxor xmm12,xmm12 ;sum_sq_r + pxor xmm11,xmm11 ;sum_sxr + + mov rdx, 8 ;row counter +.NextRow: + + ;grab source and reference pixels + movq xmm3, [rsi] + movq xmm4, [rdi] + punpcklbw xmm3, xmm0 ; low_s + punpcklbw xmm4, xmm0 ; low_r + + TABULATE_SSIM + + add rsi, rcx ; next s row + add rdi, rax ; next r row + + dec rdx ; counter + jnz .NextRow + + SUM_ACROSS_W xmm15 + SUM_ACROSS_W xmm14 + SUM_ACROSS_Q xmm13 + SUM_ACROSS_Q xmm12 + SUM_ACROSS_Q xmm11 + + mov rdi,arg(4) + movd [rdi], xmm15; + mov rdi,arg(5) + movd [rdi], xmm14; + mov rdi,arg(6) + movd [rdi], xmm13; + mov rdi,arg(7) + movd [rdi], xmm12; + mov rdi,arg(8) + movd [rdi], xmm11; + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret diff --git a/third_party/aom/av1/encoder/x86/dct_intrin_sse2.c b/third_party/aom/av1/encoder/x86/dct_intrin_sse2.c new file mode 100644 index 0000000000..37c4b0d888 --- /dev/null +++ b/third_party/aom/av1/encoder/x86/dct_intrin_sse2.c @@ -0,0 +1,3884 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include // SSE2 + +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" +#include "aom_dsp/txfm_common.h" +#include "aom_dsp/x86/fwd_txfm_sse2.h" +#include "aom_dsp/x86/synonyms.h" +#include "aom_dsp/x86/txfm_common_sse2.h" +#include "aom_ports/mem.h" + +static INLINE void load_buffer_4x4(const int16_t *input, __m128i *in, + int stride, int flipud, int fliplr) { + const __m128i k__nonzero_bias_a = _mm_setr_epi16(0, 1, 1, 1, 1, 1, 1, 1); + const __m128i k__nonzero_bias_b = _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0); + __m128i mask; + + if (!flipud) { + in[0] = _mm_loadl_epi64((const __m128i *)(input + 0 * stride)); + in[1] = _mm_loadl_epi64((const __m128i *)(input + 1 * stride)); + in[2] = _mm_loadl_epi64((const __m128i *)(input + 2 * stride)); + in[3] = _mm_loadl_epi64((const __m128i *)(input + 3 * stride)); + } else { + in[0] = _mm_loadl_epi64((const __m128i *)(input + 3 * stride)); + in[1] = _mm_loadl_epi64((const __m128i *)(input + 2 * stride)); + in[2] = _mm_loadl_epi64((const __m128i *)(input + 1 * stride)); + in[3] = _mm_loadl_epi64((const __m128i *)(input + 0 * stride)); + } + + if (fliplr) { + in[0] = _mm_shufflelo_epi16(in[0], 0x1b); + in[1] = _mm_shufflelo_epi16(in[1], 0x1b); + in[2] = _mm_shufflelo_epi16(in[2], 0x1b); + in[3] = _mm_shufflelo_epi16(in[3], 0x1b); + } + + in[0] = _mm_slli_epi16(in[0], 4); + in[1] = _mm_slli_epi16(in[1], 4); + in[2] = _mm_slli_epi16(in[2], 4); + in[3] = _mm_slli_epi16(in[3], 4); + + mask = _mm_cmpeq_epi16(in[0], k__nonzero_bias_a); + in[0] = _mm_add_epi16(in[0], mask); + in[0] = _mm_add_epi16(in[0], k__nonzero_bias_b); +} + +static INLINE void write_buffer_4x4(tran_low_t *output, __m128i *res) { + const __m128i kOne = _mm_set1_epi16(1); + __m128i in01 = _mm_unpacklo_epi64(res[0], res[1]); + __m128i in23 = _mm_unpacklo_epi64(res[2], res[3]); + __m128i out01 = _mm_add_epi16(in01, kOne); + __m128i out23 = _mm_add_epi16(in23, kOne); + out01 = _mm_srai_epi16(out01, 2); + out23 = _mm_srai_epi16(out23, 2); + store_output(&out01, (output + 0 * 8)); + store_output(&out23, (output + 1 * 8)); +} + +static INLINE void transpose_4x4(__m128i *res) { + // Combine and transpose + // 00 01 02 03 20 21 22 23 + // 10 11 12 13 30 31 32 33 + const __m128i tr0_0 = _mm_unpacklo_epi16(res[0], res[1]); + const __m128i tr0_1 = _mm_unpackhi_epi16(res[0], res[1]); + + // 00 10 01 11 02 12 03 13 + // 20 30 21 31 22 32 23 33 + res[0] = _mm_unpacklo_epi32(tr0_0, tr0_1); + res[2] = _mm_unpackhi_epi32(tr0_0, tr0_1); + + // 00 10 20 30 01 11 21 31 + // 02 12 22 32 03 13 23 33 + // only use the first 4 16-bit integers + res[1] = _mm_unpackhi_epi64(res[0], res[0]); + res[3] = _mm_unpackhi_epi64(res[2], res[2]); +} + +static void fdct4_sse2(__m128i *in) { + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64); + const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + + __m128i u[4], v[4]; + u[0] = _mm_unpacklo_epi16(in[0], in[1]); + u[1] = _mm_unpacklo_epi16(in[3], in[2]); + + v[0] = _mm_add_epi16(u[0], u[1]); + v[1] = _mm_sub_epi16(u[0], u[1]); + + u[0] = _mm_madd_epi16(v[0], k__cospi_p16_p16); // 0 + u[1] = _mm_madd_epi16(v[0], k__cospi_p16_m16); // 2 + u[2] = _mm_madd_epi16(v[1], k__cospi_p08_p24); // 1 + u[3] = _mm_madd_epi16(v[1], k__cospi_p24_m08); // 3 + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + + in[0] = _mm_packs_epi32(u[0], u[1]); + in[1] = _mm_packs_epi32(u[2], u[3]); + transpose_4x4(in); +} + +static void fadst4_sse2(__m128i *in) { + const __m128i k__sinpi_p01_p02 = pair_set_epi16(sinpi_1_9, sinpi_2_9); + const __m128i k__sinpi_p04_m01 = pair_set_epi16(sinpi_4_9, -sinpi_1_9); + const __m128i k__sinpi_p03_p04 = pair_set_epi16(sinpi_3_9, sinpi_4_9); + const __m128i k__sinpi_m03_p02 = pair_set_epi16(-sinpi_3_9, sinpi_2_9); + const __m128i k__sinpi_p03_p03 = _mm_set1_epi16((int16_t)sinpi_3_9); + const __m128i kZero = _mm_set1_epi16(0); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + __m128i u[8], v[8]; + __m128i in7 = _mm_add_epi16(in[0], in[1]); + + u[0] = _mm_unpacklo_epi16(in[0], in[1]); + u[1] = _mm_unpacklo_epi16(in[2], in[3]); + u[2] = _mm_unpacklo_epi16(in7, kZero); + u[3] = _mm_unpacklo_epi16(in[2], kZero); + u[4] = _mm_unpacklo_epi16(in[3], kZero); + + v[0] = _mm_madd_epi16(u[0], k__sinpi_p01_p02); // s0 + s2 + v[1] = _mm_madd_epi16(u[1], k__sinpi_p03_p04); // s4 + s5 + v[2] = _mm_madd_epi16(u[2], k__sinpi_p03_p03); // x1 + v[3] = _mm_madd_epi16(u[0], k__sinpi_p04_m01); // s1 - s3 + v[4] = _mm_madd_epi16(u[1], k__sinpi_m03_p02); // -s4 + s6 + v[5] = _mm_madd_epi16(u[3], k__sinpi_p03_p03); // s4 + v[6] = _mm_madd_epi16(u[4], k__sinpi_p03_p03); + + u[0] = _mm_add_epi32(v[0], v[1]); + u[1] = _mm_sub_epi32(v[2], v[6]); + u[2] = _mm_add_epi32(v[3], v[4]); + u[3] = _mm_sub_epi32(u[2], u[0]); + u[4] = _mm_slli_epi32(v[5], 2); + u[5] = _mm_sub_epi32(u[4], v[5]); + u[6] = _mm_add_epi32(u[3], u[5]); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + + in[0] = _mm_packs_epi32(u[0], u[2]); + in[1] = _mm_packs_epi32(u[1], u[3]); + transpose_4x4(in); +} + +#if CONFIG_EXT_TX +static void fidtx4_sse2(__m128i *in) { + const __m128i k__zero_epi16 = _mm_set1_epi16((int16_t)0); + const __m128i k__sqrt2_epi16 = _mm_set1_epi16((int16_t)Sqrt2); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + + __m128i v0, v1, v2, v3; + __m128i u0, u1, u2, u3; + + v0 = _mm_unpacklo_epi16(in[0], k__zero_epi16); + v1 = _mm_unpacklo_epi16(in[1], k__zero_epi16); + v2 = _mm_unpacklo_epi16(in[2], k__zero_epi16); + v3 = _mm_unpacklo_epi16(in[3], k__zero_epi16); + + u0 = _mm_madd_epi16(v0, k__sqrt2_epi16); + u1 = _mm_madd_epi16(v1, k__sqrt2_epi16); + u2 = _mm_madd_epi16(v2, k__sqrt2_epi16); + u3 = _mm_madd_epi16(v3, k__sqrt2_epi16); + + v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); + v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); + v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); + v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); + + u0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + u1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + u2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + u3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + + in[0] = _mm_packs_epi32(u0, u2); + in[1] = _mm_packs_epi32(u1, u3); + transpose_4x4(in); +} +#endif // CONFIG_EXT_TX + +void av1_fht4x4_sse2(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + __m128i in[4]; + + switch (tx_type) { + case DCT_DCT: aom_fdct4x4_sse2(input, output, stride); break; + case ADST_DCT: + load_buffer_4x4(input, in, stride, 0, 0); + fadst4_sse2(in); + fdct4_sse2(in); + write_buffer_4x4(output, in); + break; + case DCT_ADST: + load_buffer_4x4(input, in, stride, 0, 0); + fdct4_sse2(in); + fadst4_sse2(in); + write_buffer_4x4(output, in); + break; + case ADST_ADST: + load_buffer_4x4(input, in, stride, 0, 0); + fadst4_sse2(in); + fadst4_sse2(in); + write_buffer_4x4(output, in); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + load_buffer_4x4(input, in, stride, 1, 0); + fadst4_sse2(in); + fdct4_sse2(in); + write_buffer_4x4(output, in); + break; + case DCT_FLIPADST: + load_buffer_4x4(input, in, stride, 0, 1); + fdct4_sse2(in); + fadst4_sse2(in); + write_buffer_4x4(output, in); + break; + case FLIPADST_FLIPADST: + load_buffer_4x4(input, in, stride, 1, 1); + fadst4_sse2(in); + fadst4_sse2(in); + write_buffer_4x4(output, in); + break; + case ADST_FLIPADST: + load_buffer_4x4(input, in, stride, 0, 1); + fadst4_sse2(in); + fadst4_sse2(in); + write_buffer_4x4(output, in); + break; + case FLIPADST_ADST: + load_buffer_4x4(input, in, stride, 1, 0); + fadst4_sse2(in); + fadst4_sse2(in); + write_buffer_4x4(output, in); + break; + case IDTX: + load_buffer_4x4(input, in, stride, 0, 0); + fidtx4_sse2(in); + fidtx4_sse2(in); + write_buffer_4x4(output, in); + break; + case V_DCT: + load_buffer_4x4(input, in, stride, 0, 0); + fdct4_sse2(in); + fidtx4_sse2(in); + write_buffer_4x4(output, in); + break; + case H_DCT: + load_buffer_4x4(input, in, stride, 0, 0); + fidtx4_sse2(in); + fdct4_sse2(in); + write_buffer_4x4(output, in); + break; + case V_ADST: + load_buffer_4x4(input, in, stride, 0, 0); + fadst4_sse2(in); + fidtx4_sse2(in); + write_buffer_4x4(output, in); + break; + case H_ADST: + load_buffer_4x4(input, in, stride, 0, 0); + fidtx4_sse2(in); + fadst4_sse2(in); + write_buffer_4x4(output, in); + break; + case V_FLIPADST: + load_buffer_4x4(input, in, stride, 1, 0); + fadst4_sse2(in); + fidtx4_sse2(in); + write_buffer_4x4(output, in); + break; + case H_FLIPADST: + load_buffer_4x4(input, in, stride, 0, 1); + fidtx4_sse2(in); + fadst4_sse2(in); + write_buffer_4x4(output, in); + break; +#endif // CONFIG_EXT_TX + default: assert(0); + } +} + +void av1_fdct8x8_quant_sse2(const int16_t *input, int stride, + int16_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, int16_t *qcoeff_ptr, + int16_t *dqcoeff_ptr, const int16_t *dequant_ptr, + uint16_t *eob_ptr, const int16_t *scan_ptr, + const int16_t *iscan_ptr) { + __m128i zero; + int pass; + // Constants + // When we use them, in one case, they are all the same. In all others + // it's a pair of them that we need to repeat four times. This is done + // by constructing the 32 bit constant corresponding to that pair. + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + // Load input + __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride)); + __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride)); + __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride)); + __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride)); + __m128i in4 = _mm_load_si128((const __m128i *)(input + 4 * stride)); + __m128i in5 = _mm_load_si128((const __m128i *)(input + 5 * stride)); + __m128i in6 = _mm_load_si128((const __m128i *)(input + 6 * stride)); + __m128i in7 = _mm_load_si128((const __m128i *)(input + 7 * stride)); + __m128i *in[8]; + int index = 0; + + (void)scan_ptr; + (void)zbin_ptr; + (void)quant_shift_ptr; + (void)coeff_ptr; + + // Pre-condition input (shift by two) + in0 = _mm_slli_epi16(in0, 2); + in1 = _mm_slli_epi16(in1, 2); + in2 = _mm_slli_epi16(in2, 2); + in3 = _mm_slli_epi16(in3, 2); + in4 = _mm_slli_epi16(in4, 2); + in5 = _mm_slli_epi16(in5, 2); + in6 = _mm_slli_epi16(in6, 2); + in7 = _mm_slli_epi16(in7, 2); + + in[0] = &in0; + in[1] = &in1; + in[2] = &in2; + in[3] = &in3; + in[4] = &in4; + in[5] = &in5; + in[6] = &in6; + in[7] = &in7; + + // We do two passes, first the columns, then the rows. The results of the + // first pass are transposed so that the same column code can be reused. The + // results of the second pass are also transposed so that the rows (processed + // as columns) are put back in row positions. + for (pass = 0; pass < 2; pass++) { + // To store results of each pass before the transpose. + __m128i res0, res1, res2, res3, res4, res5, res6, res7; + // Add/subtract + const __m128i q0 = _mm_add_epi16(in0, in7); + const __m128i q1 = _mm_add_epi16(in1, in6); + const __m128i q2 = _mm_add_epi16(in2, in5); + const __m128i q3 = _mm_add_epi16(in3, in4); + const __m128i q4 = _mm_sub_epi16(in3, in4); + const __m128i q5 = _mm_sub_epi16(in2, in5); + const __m128i q6 = _mm_sub_epi16(in1, in6); + const __m128i q7 = _mm_sub_epi16(in0, in7); + // Work on first four results + { + // Add/subtract + const __m128i r0 = _mm_add_epi16(q0, q3); + const __m128i r1 = _mm_add_epi16(q1, q2); + const __m128i r2 = _mm_sub_epi16(q1, q2); + const __m128i r3 = _mm_sub_epi16(q0, q3); + // Interleave to do the multiply by constants which gets us into 32bits + const __m128i t0 = _mm_unpacklo_epi16(r0, r1); + const __m128i t1 = _mm_unpackhi_epi16(r0, r1); + const __m128i t2 = _mm_unpacklo_epi16(r2, r3); + const __m128i t3 = _mm_unpackhi_epi16(r2, r3); + const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16); + const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16); + const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16); + const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16); + const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08); + const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08); + const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24); + const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24); + // dct_const_round_shift + const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); + const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); + const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); + const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); + const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); + const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); + const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); + const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); + const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS); + const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS); + const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS); + const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS); + // Combine + res0 = _mm_packs_epi32(w0, w1); + res4 = _mm_packs_epi32(w2, w3); + res2 = _mm_packs_epi32(w4, w5); + res6 = _mm_packs_epi32(w6, w7); + } + // Work on next four results + { + // Interleave to do the multiply by constants which gets us into 32bits + const __m128i d0 = _mm_unpacklo_epi16(q6, q5); + const __m128i d1 = _mm_unpackhi_epi16(q6, q5); + const __m128i e0 = _mm_madd_epi16(d0, k__cospi_p16_m16); + const __m128i e1 = _mm_madd_epi16(d1, k__cospi_p16_m16); + const __m128i e2 = _mm_madd_epi16(d0, k__cospi_p16_p16); + const __m128i e3 = _mm_madd_epi16(d1, k__cospi_p16_p16); + // dct_const_round_shift + const __m128i f0 = _mm_add_epi32(e0, k__DCT_CONST_ROUNDING); + const __m128i f1 = _mm_add_epi32(e1, k__DCT_CONST_ROUNDING); + const __m128i f2 = _mm_add_epi32(e2, k__DCT_CONST_ROUNDING); + const __m128i f3 = _mm_add_epi32(e3, k__DCT_CONST_ROUNDING); + const __m128i s0 = _mm_srai_epi32(f0, DCT_CONST_BITS); + const __m128i s1 = _mm_srai_epi32(f1, DCT_CONST_BITS); + const __m128i s2 = _mm_srai_epi32(f2, DCT_CONST_BITS); + const __m128i s3 = _mm_srai_epi32(f3, DCT_CONST_BITS); + // Combine + const __m128i r0 = _mm_packs_epi32(s0, s1); + const __m128i r1 = _mm_packs_epi32(s2, s3); + // Add/subtract + const __m128i x0 = _mm_add_epi16(q4, r0); + const __m128i x1 = _mm_sub_epi16(q4, r0); + const __m128i x2 = _mm_sub_epi16(q7, r1); + const __m128i x3 = _mm_add_epi16(q7, r1); + // Interleave to do the multiply by constants which gets us into 32bits + const __m128i t0 = _mm_unpacklo_epi16(x0, x3); + const __m128i t1 = _mm_unpackhi_epi16(x0, x3); + const __m128i t2 = _mm_unpacklo_epi16(x1, x2); + const __m128i t3 = _mm_unpackhi_epi16(x1, x2); + const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04); + const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04); + const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28); + const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28); + const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20); + const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20); + const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12); + const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12); + // dct_const_round_shift + const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); + const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); + const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); + const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); + const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); + const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); + const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); + const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); + const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS); + const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS); + const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS); + const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS); + // Combine + res1 = _mm_packs_epi32(w0, w1); + res7 = _mm_packs_epi32(w2, w3); + res5 = _mm_packs_epi32(w4, w5); + res3 = _mm_packs_epi32(w6, w7); + } + // Transpose the 8x8. + { + // 00 01 02 03 04 05 06 07 + // 10 11 12 13 14 15 16 17 + // 20 21 22 23 24 25 26 27 + // 30 31 32 33 34 35 36 37 + // 40 41 42 43 44 45 46 47 + // 50 51 52 53 54 55 56 57 + // 60 61 62 63 64 65 66 67 + // 70 71 72 73 74 75 76 77 + const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1); + const __m128i tr0_1 = _mm_unpacklo_epi16(res2, res3); + const __m128i tr0_2 = _mm_unpackhi_epi16(res0, res1); + const __m128i tr0_3 = _mm_unpackhi_epi16(res2, res3); + const __m128i tr0_4 = _mm_unpacklo_epi16(res4, res5); + const __m128i tr0_5 = _mm_unpacklo_epi16(res6, res7); + const __m128i tr0_6 = _mm_unpackhi_epi16(res4, res5); + const __m128i tr0_7 = _mm_unpackhi_epi16(res6, res7); + // 00 10 01 11 02 12 03 13 + // 20 30 21 31 22 32 23 33 + // 04 14 05 15 06 16 07 17 + // 24 34 25 35 26 36 27 37 + // 40 50 41 51 42 52 43 53 + // 60 70 61 71 62 72 63 73 + // 54 54 55 55 56 56 57 57 + // 64 74 65 75 66 76 67 77 + const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); + const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3); + const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); + const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3); + const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); + const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); + const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); + const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); + // 00 10 20 30 01 11 21 31 + // 40 50 60 70 41 51 61 71 + // 02 12 22 32 03 13 23 33 + // 42 52 62 72 43 53 63 73 + // 04 14 24 34 05 15 21 36 + // 44 54 64 74 45 55 61 76 + // 06 16 26 36 07 17 27 37 + // 46 56 66 76 47 57 67 77 + in0 = _mm_unpacklo_epi64(tr1_0, tr1_4); + in1 = _mm_unpackhi_epi64(tr1_0, tr1_4); + in2 = _mm_unpacklo_epi64(tr1_2, tr1_6); + in3 = _mm_unpackhi_epi64(tr1_2, tr1_6); + in4 = _mm_unpacklo_epi64(tr1_1, tr1_5); + in5 = _mm_unpackhi_epi64(tr1_1, tr1_5); + in6 = _mm_unpacklo_epi64(tr1_3, tr1_7); + in7 = _mm_unpackhi_epi64(tr1_3, tr1_7); + // 00 10 20 30 40 50 60 70 + // 01 11 21 31 41 51 61 71 + // 02 12 22 32 42 52 62 72 + // 03 13 23 33 43 53 63 73 + // 04 14 24 34 44 54 64 74 + // 05 15 25 35 45 55 65 75 + // 06 16 26 36 46 56 66 76 + // 07 17 27 37 47 57 67 77 + } + } + // Post-condition output and store it + { + // Post-condition (division by two) + // division of two 16 bits signed numbers using shifts + // n / 2 = (n - (n >> 15)) >> 1 + const __m128i sign_in0 = _mm_srai_epi16(in0, 15); + const __m128i sign_in1 = _mm_srai_epi16(in1, 15); + const __m128i sign_in2 = _mm_srai_epi16(in2, 15); + const __m128i sign_in3 = _mm_srai_epi16(in3, 15); + const __m128i sign_in4 = _mm_srai_epi16(in4, 15); + const __m128i sign_in5 = _mm_srai_epi16(in5, 15); + const __m128i sign_in6 = _mm_srai_epi16(in6, 15); + const __m128i sign_in7 = _mm_srai_epi16(in7, 15); + in0 = _mm_sub_epi16(in0, sign_in0); + in1 = _mm_sub_epi16(in1, sign_in1); + in2 = _mm_sub_epi16(in2, sign_in2); + in3 = _mm_sub_epi16(in3, sign_in3); + in4 = _mm_sub_epi16(in4, sign_in4); + in5 = _mm_sub_epi16(in5, sign_in5); + in6 = _mm_sub_epi16(in6, sign_in6); + in7 = _mm_sub_epi16(in7, sign_in7); + in0 = _mm_srai_epi16(in0, 1); + in1 = _mm_srai_epi16(in1, 1); + in2 = _mm_srai_epi16(in2, 1); + in3 = _mm_srai_epi16(in3, 1); + in4 = _mm_srai_epi16(in4, 1); + in5 = _mm_srai_epi16(in5, 1); + in6 = _mm_srai_epi16(in6, 1); + in7 = _mm_srai_epi16(in7, 1); + } + + iscan_ptr += n_coeffs; + qcoeff_ptr += n_coeffs; + dqcoeff_ptr += n_coeffs; + n_coeffs = -n_coeffs; + zero = _mm_setzero_si128(); + + if (!skip_block) { + __m128i eob; + __m128i round, quant, dequant; + { + __m128i coeff0, coeff1; + + // Setup global values + { + round = _mm_load_si128((const __m128i *)round_ptr); + quant = _mm_load_si128((const __m128i *)quant_ptr); + dequant = _mm_load_si128((const __m128i *)dequant_ptr); + } + + { + __m128i coeff0_sign, coeff1_sign; + __m128i qcoeff0, qcoeff1; + __m128i qtmp0, qtmp1; + // Do DC and first 15 AC + coeff0 = *in[0]; + coeff1 = *in[1]; + + // Poor man's sign extract + coeff0_sign = _mm_srai_epi16(coeff0, 15); + coeff1_sign = _mm_srai_epi16(coeff1, 15); + qcoeff0 = _mm_xor_si128(coeff0, coeff0_sign); + qcoeff1 = _mm_xor_si128(coeff1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + qcoeff0 = _mm_adds_epi16(qcoeff0, round); + round = _mm_unpackhi_epi64(round, round); + qcoeff1 = _mm_adds_epi16(qcoeff1, round); + qtmp0 = _mm_mulhi_epi16(qcoeff0, quant); + quant = _mm_unpackhi_epi64(quant, quant); + qtmp1 = _mm_mulhi_epi16(qcoeff1, quant); + + // Reinsert signs + qcoeff0 = _mm_xor_si128(qtmp0, coeff0_sign); + qcoeff1 = _mm_xor_si128(qtmp1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), qcoeff0); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, qcoeff1); + + coeff0 = _mm_mullo_epi16(qcoeff0, dequant); + dequant = _mm_unpackhi_epi64(dequant, dequant); + coeff1 = _mm_mullo_epi16(qcoeff1, dequant); + + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), coeff0); + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, coeff1); + } + + { + // Scan for eob + __m128i zero_coeff0, zero_coeff1; + __m128i nzero_coeff0, nzero_coeff1; + __m128i iscan0, iscan1; + __m128i eob1; + zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero); + zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero); + nzero_coeff0 = _mm_cmpeq_epi16(zero_coeff0, zero); + nzero_coeff1 = _mm_cmpeq_epi16(zero_coeff1, zero); + iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs)); + iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs) + 1); + // Add one to convert from indices to counts + iscan0 = _mm_sub_epi16(iscan0, nzero_coeff0); + iscan1 = _mm_sub_epi16(iscan1, nzero_coeff1); + eob = _mm_and_si128(iscan0, nzero_coeff0); + eob1 = _mm_and_si128(iscan1, nzero_coeff1); + eob = _mm_max_epi16(eob, eob1); + } + n_coeffs += 8 * 2; + } + + // AC only loop + index = 2; + while (n_coeffs < 0) { + __m128i coeff0, coeff1; + { + __m128i coeff0_sign, coeff1_sign; + __m128i qcoeff0, qcoeff1; + __m128i qtmp0, qtmp1; + + assert(index < (int)(sizeof(in) / sizeof(in[0])) - 1); + coeff0 = *in[index]; + coeff1 = *in[index + 1]; + + // Poor man's sign extract + coeff0_sign = _mm_srai_epi16(coeff0, 15); + coeff1_sign = _mm_srai_epi16(coeff1, 15); + qcoeff0 = _mm_xor_si128(coeff0, coeff0_sign); + qcoeff1 = _mm_xor_si128(coeff1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + qcoeff0 = _mm_adds_epi16(qcoeff0, round); + qcoeff1 = _mm_adds_epi16(qcoeff1, round); + qtmp0 = _mm_mulhi_epi16(qcoeff0, quant); + qtmp1 = _mm_mulhi_epi16(qcoeff1, quant); + + // Reinsert signs + qcoeff0 = _mm_xor_si128(qtmp0, coeff0_sign); + qcoeff1 = _mm_xor_si128(qtmp1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), qcoeff0); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, qcoeff1); + + coeff0 = _mm_mullo_epi16(qcoeff0, dequant); + coeff1 = _mm_mullo_epi16(qcoeff1, dequant); + + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), coeff0); + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, coeff1); + } + + { + // Scan for eob + __m128i zero_coeff0, zero_coeff1; + __m128i nzero_coeff0, nzero_coeff1; + __m128i iscan0, iscan1; + __m128i eob0, eob1; + zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero); + zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero); + nzero_coeff0 = _mm_cmpeq_epi16(zero_coeff0, zero); + nzero_coeff1 = _mm_cmpeq_epi16(zero_coeff1, zero); + iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs)); + iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs) + 1); + // Add one to convert from indices to counts + iscan0 = _mm_sub_epi16(iscan0, nzero_coeff0); + iscan1 = _mm_sub_epi16(iscan1, nzero_coeff1); + eob0 = _mm_and_si128(iscan0, nzero_coeff0); + eob1 = _mm_and_si128(iscan1, nzero_coeff1); + eob0 = _mm_max_epi16(eob0, eob1); + eob = _mm_max_epi16(eob, eob0); + } + n_coeffs += 8 * 2; + index += 2; + } + + // Accumulate EOB + { + __m128i eob_shuffled; + eob_shuffled = _mm_shuffle_epi32(eob, 0xe); + eob = _mm_max_epi16(eob, eob_shuffled); + eob_shuffled = _mm_shufflelo_epi16(eob, 0xe); + eob = _mm_max_epi16(eob, eob_shuffled); + eob_shuffled = _mm_shufflelo_epi16(eob, 0x1); + eob = _mm_max_epi16(eob, eob_shuffled); + *eob_ptr = _mm_extract_epi16(eob, 1); + } + } else { + do { + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, zero); + n_coeffs += 8 * 2; + } while (n_coeffs < 0); + *eob_ptr = 0; + } +} + +// load 8x8 array +static INLINE void load_buffer_8x8(const int16_t *input, __m128i *in, + int stride, int flipud, int fliplr) { + if (!flipud) { + in[0] = _mm_load_si128((const __m128i *)(input + 0 * stride)); + in[1] = _mm_load_si128((const __m128i *)(input + 1 * stride)); + in[2] = _mm_load_si128((const __m128i *)(input + 2 * stride)); + in[3] = _mm_load_si128((const __m128i *)(input + 3 * stride)); + in[4] = _mm_load_si128((const __m128i *)(input + 4 * stride)); + in[5] = _mm_load_si128((const __m128i *)(input + 5 * stride)); + in[6] = _mm_load_si128((const __m128i *)(input + 6 * stride)); + in[7] = _mm_load_si128((const __m128i *)(input + 7 * stride)); + } else { + in[0] = _mm_load_si128((const __m128i *)(input + 7 * stride)); + in[1] = _mm_load_si128((const __m128i *)(input + 6 * stride)); + in[2] = _mm_load_si128((const __m128i *)(input + 5 * stride)); + in[3] = _mm_load_si128((const __m128i *)(input + 4 * stride)); + in[4] = _mm_load_si128((const __m128i *)(input + 3 * stride)); + in[5] = _mm_load_si128((const __m128i *)(input + 2 * stride)); + in[6] = _mm_load_si128((const __m128i *)(input + 1 * stride)); + in[7] = _mm_load_si128((const __m128i *)(input + 0 * stride)); + } + + if (fliplr) { + in[0] = mm_reverse_epi16(in[0]); + in[1] = mm_reverse_epi16(in[1]); + in[2] = mm_reverse_epi16(in[2]); + in[3] = mm_reverse_epi16(in[3]); + in[4] = mm_reverse_epi16(in[4]); + in[5] = mm_reverse_epi16(in[5]); + in[6] = mm_reverse_epi16(in[6]); + in[7] = mm_reverse_epi16(in[7]); + } + + in[0] = _mm_slli_epi16(in[0], 2); + in[1] = _mm_slli_epi16(in[1], 2); + in[2] = _mm_slli_epi16(in[2], 2); + in[3] = _mm_slli_epi16(in[3], 2); + in[4] = _mm_slli_epi16(in[4], 2); + in[5] = _mm_slli_epi16(in[5], 2); + in[6] = _mm_slli_epi16(in[6], 2); + in[7] = _mm_slli_epi16(in[7], 2); +} + +// right shift and rounding +static INLINE void right_shift_8x8(__m128i *res, const int bit) { + __m128i sign0 = _mm_srai_epi16(res[0], 15); + __m128i sign1 = _mm_srai_epi16(res[1], 15); + __m128i sign2 = _mm_srai_epi16(res[2], 15); + __m128i sign3 = _mm_srai_epi16(res[3], 15); + __m128i sign4 = _mm_srai_epi16(res[4], 15); + __m128i sign5 = _mm_srai_epi16(res[5], 15); + __m128i sign6 = _mm_srai_epi16(res[6], 15); + __m128i sign7 = _mm_srai_epi16(res[7], 15); + + if (bit == 2) { + const __m128i const_rounding = _mm_set1_epi16(1); + res[0] = _mm_adds_epi16(res[0], const_rounding); + res[1] = _mm_adds_epi16(res[1], const_rounding); + res[2] = _mm_adds_epi16(res[2], const_rounding); + res[3] = _mm_adds_epi16(res[3], const_rounding); + res[4] = _mm_adds_epi16(res[4], const_rounding); + res[5] = _mm_adds_epi16(res[5], const_rounding); + res[6] = _mm_adds_epi16(res[6], const_rounding); + res[7] = _mm_adds_epi16(res[7], const_rounding); + } + + res[0] = _mm_sub_epi16(res[0], sign0); + res[1] = _mm_sub_epi16(res[1], sign1); + res[2] = _mm_sub_epi16(res[2], sign2); + res[3] = _mm_sub_epi16(res[3], sign3); + res[4] = _mm_sub_epi16(res[4], sign4); + res[5] = _mm_sub_epi16(res[5], sign5); + res[6] = _mm_sub_epi16(res[6], sign6); + res[7] = _mm_sub_epi16(res[7], sign7); + + if (bit == 1) { + res[0] = _mm_srai_epi16(res[0], 1); + res[1] = _mm_srai_epi16(res[1], 1); + res[2] = _mm_srai_epi16(res[2], 1); + res[3] = _mm_srai_epi16(res[3], 1); + res[4] = _mm_srai_epi16(res[4], 1); + res[5] = _mm_srai_epi16(res[5], 1); + res[6] = _mm_srai_epi16(res[6], 1); + res[7] = _mm_srai_epi16(res[7], 1); + } else { + res[0] = _mm_srai_epi16(res[0], 2); + res[1] = _mm_srai_epi16(res[1], 2); + res[2] = _mm_srai_epi16(res[2], 2); + res[3] = _mm_srai_epi16(res[3], 2); + res[4] = _mm_srai_epi16(res[4], 2); + res[5] = _mm_srai_epi16(res[5], 2); + res[6] = _mm_srai_epi16(res[6], 2); + res[7] = _mm_srai_epi16(res[7], 2); + } +} + +// write 8x8 array +static INLINE void write_buffer_8x8(tran_low_t *output, __m128i *res, + int stride) { + store_output(&res[0], (output + 0 * stride)); + store_output(&res[1], (output + 1 * stride)); + store_output(&res[2], (output + 2 * stride)); + store_output(&res[3], (output + 3 * stride)); + store_output(&res[4], (output + 4 * stride)); + store_output(&res[5], (output + 5 * stride)); + store_output(&res[6], (output + 6 * stride)); + store_output(&res[7], (output + 7 * stride)); +} + +// perform in-place transpose +static INLINE void array_transpose_8x8(__m128i *in, __m128i *res) { + const __m128i tr0_0 = _mm_unpacklo_epi16(in[0], in[1]); + const __m128i tr0_1 = _mm_unpacklo_epi16(in[2], in[3]); + const __m128i tr0_2 = _mm_unpackhi_epi16(in[0], in[1]); + const __m128i tr0_3 = _mm_unpackhi_epi16(in[2], in[3]); + const __m128i tr0_4 = _mm_unpacklo_epi16(in[4], in[5]); + const __m128i tr0_5 = _mm_unpacklo_epi16(in[6], in[7]); + const __m128i tr0_6 = _mm_unpackhi_epi16(in[4], in[5]); + const __m128i tr0_7 = _mm_unpackhi_epi16(in[6], in[7]); + // 00 10 01 11 02 12 03 13 + // 20 30 21 31 22 32 23 33 + // 04 14 05 15 06 16 07 17 + // 24 34 25 35 26 36 27 37 + // 40 50 41 51 42 52 43 53 + // 60 70 61 71 62 72 63 73 + // 44 54 45 55 46 56 47 57 + // 64 74 65 75 66 76 67 77 + const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); + const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_4, tr0_5); + const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); + const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_4, tr0_5); + const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_2, tr0_3); + const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); + const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_2, tr0_3); + const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); + // 00 10 20 30 01 11 21 31 + // 40 50 60 70 41 51 61 71 + // 02 12 22 32 03 13 23 33 + // 42 52 62 72 43 53 63 73 + // 04 14 24 34 05 15 25 35 + // 44 54 64 74 45 55 65 75 + // 06 16 26 36 07 17 27 37 + // 46 56 66 76 47 57 67 77 + res[0] = _mm_unpacklo_epi64(tr1_0, tr1_1); + res[1] = _mm_unpackhi_epi64(tr1_0, tr1_1); + res[2] = _mm_unpacklo_epi64(tr1_2, tr1_3); + res[3] = _mm_unpackhi_epi64(tr1_2, tr1_3); + res[4] = _mm_unpacklo_epi64(tr1_4, tr1_5); + res[5] = _mm_unpackhi_epi64(tr1_4, tr1_5); + res[6] = _mm_unpacklo_epi64(tr1_6, tr1_7); + res[7] = _mm_unpackhi_epi64(tr1_6, tr1_7); + // 00 10 20 30 40 50 60 70 + // 01 11 21 31 41 51 61 71 + // 02 12 22 32 42 52 62 72 + // 03 13 23 33 43 53 63 73 + // 04 14 24 34 44 54 64 74 + // 05 15 25 35 45 55 65 75 + // 06 16 26 36 46 56 66 76 + // 07 17 27 37 47 57 67 77 +} + +static void fdct8_sse2(__m128i *in) { + // constants + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + __m128i u0, u1, u2, u3, u4, u5, u6, u7; + __m128i v0, v1, v2, v3, v4, v5, v6, v7; + __m128i s0, s1, s2, s3, s4, s5, s6, s7; + + // stage 1 + s0 = _mm_add_epi16(in[0], in[7]); + s1 = _mm_add_epi16(in[1], in[6]); + s2 = _mm_add_epi16(in[2], in[5]); + s3 = _mm_add_epi16(in[3], in[4]); + s4 = _mm_sub_epi16(in[3], in[4]); + s5 = _mm_sub_epi16(in[2], in[5]); + s6 = _mm_sub_epi16(in[1], in[6]); + s7 = _mm_sub_epi16(in[0], in[7]); + + u0 = _mm_add_epi16(s0, s3); + u1 = _mm_add_epi16(s1, s2); + u2 = _mm_sub_epi16(s1, s2); + u3 = _mm_sub_epi16(s0, s3); + // interleave and perform butterfly multiplication/addition + v0 = _mm_unpacklo_epi16(u0, u1); + v1 = _mm_unpackhi_epi16(u0, u1); + v2 = _mm_unpacklo_epi16(u2, u3); + v3 = _mm_unpackhi_epi16(u2, u3); + + u0 = _mm_madd_epi16(v0, k__cospi_p16_p16); + u1 = _mm_madd_epi16(v1, k__cospi_p16_p16); + u2 = _mm_madd_epi16(v0, k__cospi_p16_m16); + u3 = _mm_madd_epi16(v1, k__cospi_p16_m16); + u4 = _mm_madd_epi16(v2, k__cospi_p24_p08); + u5 = _mm_madd_epi16(v3, k__cospi_p24_p08); + u6 = _mm_madd_epi16(v2, k__cospi_m08_p24); + u7 = _mm_madd_epi16(v3, k__cospi_m08_p24); + + // shift and rounding + v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); + v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); + v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); + v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); + v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); + v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); + v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); + v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); + + u0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + u1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + u2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + u3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + u4 = _mm_srai_epi32(v4, DCT_CONST_BITS); + u5 = _mm_srai_epi32(v5, DCT_CONST_BITS); + u6 = _mm_srai_epi32(v6, DCT_CONST_BITS); + u7 = _mm_srai_epi32(v7, DCT_CONST_BITS); + + in[0] = _mm_packs_epi32(u0, u1); + in[2] = _mm_packs_epi32(u4, u5); + in[4] = _mm_packs_epi32(u2, u3); + in[6] = _mm_packs_epi32(u6, u7); + + // stage 2 + // interleave and perform butterfly multiplication/addition + u0 = _mm_unpacklo_epi16(s6, s5); + u1 = _mm_unpackhi_epi16(s6, s5); + v0 = _mm_madd_epi16(u0, k__cospi_p16_m16); + v1 = _mm_madd_epi16(u1, k__cospi_p16_m16); + v2 = _mm_madd_epi16(u0, k__cospi_p16_p16); + v3 = _mm_madd_epi16(u1, k__cospi_p16_p16); + + // shift and rounding + u0 = _mm_add_epi32(v0, k__DCT_CONST_ROUNDING); + u1 = _mm_add_epi32(v1, k__DCT_CONST_ROUNDING); + u2 = _mm_add_epi32(v2, k__DCT_CONST_ROUNDING); + u3 = _mm_add_epi32(v3, k__DCT_CONST_ROUNDING); + + v0 = _mm_srai_epi32(u0, DCT_CONST_BITS); + v1 = _mm_srai_epi32(u1, DCT_CONST_BITS); + v2 = _mm_srai_epi32(u2, DCT_CONST_BITS); + v3 = _mm_srai_epi32(u3, DCT_CONST_BITS); + + u0 = _mm_packs_epi32(v0, v1); + u1 = _mm_packs_epi32(v2, v3); + + // stage 3 + s0 = _mm_add_epi16(s4, u0); + s1 = _mm_sub_epi16(s4, u0); + s2 = _mm_sub_epi16(s7, u1); + s3 = _mm_add_epi16(s7, u1); + + // stage 4 + u0 = _mm_unpacklo_epi16(s0, s3); + u1 = _mm_unpackhi_epi16(s0, s3); + u2 = _mm_unpacklo_epi16(s1, s2); + u3 = _mm_unpackhi_epi16(s1, s2); + + v0 = _mm_madd_epi16(u0, k__cospi_p28_p04); + v1 = _mm_madd_epi16(u1, k__cospi_p28_p04); + v2 = _mm_madd_epi16(u2, k__cospi_p12_p20); + v3 = _mm_madd_epi16(u3, k__cospi_p12_p20); + v4 = _mm_madd_epi16(u2, k__cospi_m20_p12); + v5 = _mm_madd_epi16(u3, k__cospi_m20_p12); + v6 = _mm_madd_epi16(u0, k__cospi_m04_p28); + v7 = _mm_madd_epi16(u1, k__cospi_m04_p28); + + // shift and rounding + u0 = _mm_add_epi32(v0, k__DCT_CONST_ROUNDING); + u1 = _mm_add_epi32(v1, k__DCT_CONST_ROUNDING); + u2 = _mm_add_epi32(v2, k__DCT_CONST_ROUNDING); + u3 = _mm_add_epi32(v3, k__DCT_CONST_ROUNDING); + u4 = _mm_add_epi32(v4, k__DCT_CONST_ROUNDING); + u5 = _mm_add_epi32(v5, k__DCT_CONST_ROUNDING); + u6 = _mm_add_epi32(v6, k__DCT_CONST_ROUNDING); + u7 = _mm_add_epi32(v7, k__DCT_CONST_ROUNDING); + + v0 = _mm_srai_epi32(u0, DCT_CONST_BITS); + v1 = _mm_srai_epi32(u1, DCT_CONST_BITS); + v2 = _mm_srai_epi32(u2, DCT_CONST_BITS); + v3 = _mm_srai_epi32(u3, DCT_CONST_BITS); + v4 = _mm_srai_epi32(u4, DCT_CONST_BITS); + v5 = _mm_srai_epi32(u5, DCT_CONST_BITS); + v6 = _mm_srai_epi32(u6, DCT_CONST_BITS); + v7 = _mm_srai_epi32(u7, DCT_CONST_BITS); + + in[1] = _mm_packs_epi32(v0, v1); + in[3] = _mm_packs_epi32(v4, v5); + in[5] = _mm_packs_epi32(v2, v3); + in[7] = _mm_packs_epi32(v6, v7); + + // transpose + array_transpose_8x8(in, in); +} + +static void fadst8_sse2(__m128i *in) { + // Constants + const __m128i k__cospi_p02_p30 = pair_set_epi16(cospi_2_64, cospi_30_64); + const __m128i k__cospi_p30_m02 = pair_set_epi16(cospi_30_64, -cospi_2_64); + const __m128i k__cospi_p10_p22 = pair_set_epi16(cospi_10_64, cospi_22_64); + const __m128i k__cospi_p22_m10 = pair_set_epi16(cospi_22_64, -cospi_10_64); + const __m128i k__cospi_p18_p14 = pair_set_epi16(cospi_18_64, cospi_14_64); + const __m128i k__cospi_p14_m18 = pair_set_epi16(cospi_14_64, -cospi_18_64); + const __m128i k__cospi_p26_p06 = pair_set_epi16(cospi_26_64, cospi_6_64); + const __m128i k__cospi_p06_m26 = pair_set_epi16(cospi_6_64, -cospi_26_64); + const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64); + const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i k__cospi_m24_p08 = pair_set_epi16(-cospi_24_64, cospi_8_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__const_0 = _mm_set1_epi16(0); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + + __m128i u0, u1, u2, u3, u4, u5, u6, u7, u8, u9, u10, u11, u12, u13, u14, u15; + __m128i v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15; + __m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; + __m128i s0, s1, s2, s3, s4, s5, s6, s7; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + + // properly aligned for butterfly input + in0 = in[7]; + in1 = in[0]; + in2 = in[5]; + in3 = in[2]; + in4 = in[3]; + in5 = in[4]; + in6 = in[1]; + in7 = in[6]; + + // column transformation + // stage 1 + // interleave and multiply/add into 32-bit integer + s0 = _mm_unpacklo_epi16(in0, in1); + s1 = _mm_unpackhi_epi16(in0, in1); + s2 = _mm_unpacklo_epi16(in2, in3); + s3 = _mm_unpackhi_epi16(in2, in3); + s4 = _mm_unpacklo_epi16(in4, in5); + s5 = _mm_unpackhi_epi16(in4, in5); + s6 = _mm_unpacklo_epi16(in6, in7); + s7 = _mm_unpackhi_epi16(in6, in7); + + u0 = _mm_madd_epi16(s0, k__cospi_p02_p30); + u1 = _mm_madd_epi16(s1, k__cospi_p02_p30); + u2 = _mm_madd_epi16(s0, k__cospi_p30_m02); + u3 = _mm_madd_epi16(s1, k__cospi_p30_m02); + u4 = _mm_madd_epi16(s2, k__cospi_p10_p22); + u5 = _mm_madd_epi16(s3, k__cospi_p10_p22); + u6 = _mm_madd_epi16(s2, k__cospi_p22_m10); + u7 = _mm_madd_epi16(s3, k__cospi_p22_m10); + u8 = _mm_madd_epi16(s4, k__cospi_p18_p14); + u9 = _mm_madd_epi16(s5, k__cospi_p18_p14); + u10 = _mm_madd_epi16(s4, k__cospi_p14_m18); + u11 = _mm_madd_epi16(s5, k__cospi_p14_m18); + u12 = _mm_madd_epi16(s6, k__cospi_p26_p06); + u13 = _mm_madd_epi16(s7, k__cospi_p26_p06); + u14 = _mm_madd_epi16(s6, k__cospi_p06_m26); + u15 = _mm_madd_epi16(s7, k__cospi_p06_m26); + + // addition + w0 = _mm_add_epi32(u0, u8); + w1 = _mm_add_epi32(u1, u9); + w2 = _mm_add_epi32(u2, u10); + w3 = _mm_add_epi32(u3, u11); + w4 = _mm_add_epi32(u4, u12); + w5 = _mm_add_epi32(u5, u13); + w6 = _mm_add_epi32(u6, u14); + w7 = _mm_add_epi32(u7, u15); + w8 = _mm_sub_epi32(u0, u8); + w9 = _mm_sub_epi32(u1, u9); + w10 = _mm_sub_epi32(u2, u10); + w11 = _mm_sub_epi32(u3, u11); + w12 = _mm_sub_epi32(u4, u12); + w13 = _mm_sub_epi32(u5, u13); + w14 = _mm_sub_epi32(u6, u14); + w15 = _mm_sub_epi32(u7, u15); + + // shift and rounding + v8 = _mm_add_epi32(w8, k__DCT_CONST_ROUNDING); + v9 = _mm_add_epi32(w9, k__DCT_CONST_ROUNDING); + v10 = _mm_add_epi32(w10, k__DCT_CONST_ROUNDING); + v11 = _mm_add_epi32(w11, k__DCT_CONST_ROUNDING); + v12 = _mm_add_epi32(w12, k__DCT_CONST_ROUNDING); + v13 = _mm_add_epi32(w13, k__DCT_CONST_ROUNDING); + v14 = _mm_add_epi32(w14, k__DCT_CONST_ROUNDING); + v15 = _mm_add_epi32(w15, k__DCT_CONST_ROUNDING); + + u8 = _mm_srai_epi32(v8, DCT_CONST_BITS); + u9 = _mm_srai_epi32(v9, DCT_CONST_BITS); + u10 = _mm_srai_epi32(v10, DCT_CONST_BITS); + u11 = _mm_srai_epi32(v11, DCT_CONST_BITS); + u12 = _mm_srai_epi32(v12, DCT_CONST_BITS); + u13 = _mm_srai_epi32(v13, DCT_CONST_BITS); + u14 = _mm_srai_epi32(v14, DCT_CONST_BITS); + u15 = _mm_srai_epi32(v15, DCT_CONST_BITS); + + // back to 16-bit and pack 8 integers into __m128i + v0 = _mm_add_epi32(w0, w4); + v1 = _mm_add_epi32(w1, w5); + v2 = _mm_add_epi32(w2, w6); + v3 = _mm_add_epi32(w3, w7); + v4 = _mm_sub_epi32(w0, w4); + v5 = _mm_sub_epi32(w1, w5); + v6 = _mm_sub_epi32(w2, w6); + v7 = _mm_sub_epi32(w3, w7); + + w0 = _mm_add_epi32(v0, k__DCT_CONST_ROUNDING); + w1 = _mm_add_epi32(v1, k__DCT_CONST_ROUNDING); + w2 = _mm_add_epi32(v2, k__DCT_CONST_ROUNDING); + w3 = _mm_add_epi32(v3, k__DCT_CONST_ROUNDING); + w4 = _mm_add_epi32(v4, k__DCT_CONST_ROUNDING); + w5 = _mm_add_epi32(v5, k__DCT_CONST_ROUNDING); + w6 = _mm_add_epi32(v6, k__DCT_CONST_ROUNDING); + w7 = _mm_add_epi32(v7, k__DCT_CONST_ROUNDING); + + v0 = _mm_srai_epi32(w0, DCT_CONST_BITS); + v1 = _mm_srai_epi32(w1, DCT_CONST_BITS); + v2 = _mm_srai_epi32(w2, DCT_CONST_BITS); + v3 = _mm_srai_epi32(w3, DCT_CONST_BITS); + v4 = _mm_srai_epi32(w4, DCT_CONST_BITS); + v5 = _mm_srai_epi32(w5, DCT_CONST_BITS); + v6 = _mm_srai_epi32(w6, DCT_CONST_BITS); + v7 = _mm_srai_epi32(w7, DCT_CONST_BITS); + + in[4] = _mm_packs_epi32(u8, u9); + in[5] = _mm_packs_epi32(u10, u11); + in[6] = _mm_packs_epi32(u12, u13); + in[7] = _mm_packs_epi32(u14, u15); + + // stage 2 + s0 = _mm_packs_epi32(v0, v1); + s1 = _mm_packs_epi32(v2, v3); + s2 = _mm_packs_epi32(v4, v5); + s3 = _mm_packs_epi32(v6, v7); + + u0 = _mm_unpacklo_epi16(in[4], in[5]); + u1 = _mm_unpackhi_epi16(in[4], in[5]); + u2 = _mm_unpacklo_epi16(in[6], in[7]); + u3 = _mm_unpackhi_epi16(in[6], in[7]); + + v0 = _mm_madd_epi16(u0, k__cospi_p08_p24); + v1 = _mm_madd_epi16(u1, k__cospi_p08_p24); + v2 = _mm_madd_epi16(u0, k__cospi_p24_m08); + v3 = _mm_madd_epi16(u1, k__cospi_p24_m08); + v4 = _mm_madd_epi16(u2, k__cospi_m24_p08); + v5 = _mm_madd_epi16(u3, k__cospi_m24_p08); + v6 = _mm_madd_epi16(u2, k__cospi_p08_p24); + v7 = _mm_madd_epi16(u3, k__cospi_p08_p24); + + w0 = _mm_add_epi32(v0, v4); + w1 = _mm_add_epi32(v1, v5); + w2 = _mm_add_epi32(v2, v6); + w3 = _mm_add_epi32(v3, v7); + w4 = _mm_sub_epi32(v0, v4); + w5 = _mm_sub_epi32(v1, v5); + w6 = _mm_sub_epi32(v2, v6); + w7 = _mm_sub_epi32(v3, v7); + + v0 = _mm_add_epi32(w0, k__DCT_CONST_ROUNDING); + v1 = _mm_add_epi32(w1, k__DCT_CONST_ROUNDING); + v2 = _mm_add_epi32(w2, k__DCT_CONST_ROUNDING); + v3 = _mm_add_epi32(w3, k__DCT_CONST_ROUNDING); + v4 = _mm_add_epi32(w4, k__DCT_CONST_ROUNDING); + v5 = _mm_add_epi32(w5, k__DCT_CONST_ROUNDING); + v6 = _mm_add_epi32(w6, k__DCT_CONST_ROUNDING); + v7 = _mm_add_epi32(w7, k__DCT_CONST_ROUNDING); + + u0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + u1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + u2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + u3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + u4 = _mm_srai_epi32(v4, DCT_CONST_BITS); + u5 = _mm_srai_epi32(v5, DCT_CONST_BITS); + u6 = _mm_srai_epi32(v6, DCT_CONST_BITS); + u7 = _mm_srai_epi32(v7, DCT_CONST_BITS); + + // back to 16-bit intergers + s4 = _mm_packs_epi32(u0, u1); + s5 = _mm_packs_epi32(u2, u3); + s6 = _mm_packs_epi32(u4, u5); + s7 = _mm_packs_epi32(u6, u7); + + // stage 3 + u0 = _mm_unpacklo_epi16(s2, s3); + u1 = _mm_unpackhi_epi16(s2, s3); + u2 = _mm_unpacklo_epi16(s6, s7); + u3 = _mm_unpackhi_epi16(s6, s7); + + v0 = _mm_madd_epi16(u0, k__cospi_p16_p16); + v1 = _mm_madd_epi16(u1, k__cospi_p16_p16); + v2 = _mm_madd_epi16(u0, k__cospi_p16_m16); + v3 = _mm_madd_epi16(u1, k__cospi_p16_m16); + v4 = _mm_madd_epi16(u2, k__cospi_p16_p16); + v5 = _mm_madd_epi16(u3, k__cospi_p16_p16); + v6 = _mm_madd_epi16(u2, k__cospi_p16_m16); + v7 = _mm_madd_epi16(u3, k__cospi_p16_m16); + + u0 = _mm_add_epi32(v0, k__DCT_CONST_ROUNDING); + u1 = _mm_add_epi32(v1, k__DCT_CONST_ROUNDING); + u2 = _mm_add_epi32(v2, k__DCT_CONST_ROUNDING); + u3 = _mm_add_epi32(v3, k__DCT_CONST_ROUNDING); + u4 = _mm_add_epi32(v4, k__DCT_CONST_ROUNDING); + u5 = _mm_add_epi32(v5, k__DCT_CONST_ROUNDING); + u6 = _mm_add_epi32(v6, k__DCT_CONST_ROUNDING); + u7 = _mm_add_epi32(v7, k__DCT_CONST_ROUNDING); + + v0 = _mm_srai_epi32(u0, DCT_CONST_BITS); + v1 = _mm_srai_epi32(u1, DCT_CONST_BITS); + v2 = _mm_srai_epi32(u2, DCT_CONST_BITS); + v3 = _mm_srai_epi32(u3, DCT_CONST_BITS); + v4 = _mm_srai_epi32(u4, DCT_CONST_BITS); + v5 = _mm_srai_epi32(u5, DCT_CONST_BITS); + v6 = _mm_srai_epi32(u6, DCT_CONST_BITS); + v7 = _mm_srai_epi32(u7, DCT_CONST_BITS); + + s2 = _mm_packs_epi32(v0, v1); + s3 = _mm_packs_epi32(v2, v3); + s6 = _mm_packs_epi32(v4, v5); + s7 = _mm_packs_epi32(v6, v7); + + // FIXME(jingning): do subtract using bit inversion? + in[0] = s0; + in[1] = _mm_sub_epi16(k__const_0, s4); + in[2] = s6; + in[3] = _mm_sub_epi16(k__const_0, s2); + in[4] = s3; + in[5] = _mm_sub_epi16(k__const_0, s7); + in[6] = s5; + in[7] = _mm_sub_epi16(k__const_0, s1); + + // transpose + array_transpose_8x8(in, in); +} + +#if CONFIG_EXT_TX +static void fidtx8_sse2(__m128i *in) { + in[0] = _mm_slli_epi16(in[0], 1); + in[1] = _mm_slli_epi16(in[1], 1); + in[2] = _mm_slli_epi16(in[2], 1); + in[3] = _mm_slli_epi16(in[3], 1); + in[4] = _mm_slli_epi16(in[4], 1); + in[5] = _mm_slli_epi16(in[5], 1); + in[6] = _mm_slli_epi16(in[6], 1); + in[7] = _mm_slli_epi16(in[7], 1); + + array_transpose_8x8(in, in); +} +#endif // CONFIG_EXT_TX + +void av1_fht8x8_sse2(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + __m128i in[8]; + + switch (tx_type) { + case DCT_DCT: aom_fdct8x8_sse2(input, output, stride); break; + case ADST_DCT: + load_buffer_8x8(input, in, stride, 0, 0); + fadst8_sse2(in); + fdct8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case DCT_ADST: + load_buffer_8x8(input, in, stride, 0, 0); + fdct8_sse2(in); + fadst8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case ADST_ADST: + load_buffer_8x8(input, in, stride, 0, 0); + fadst8_sse2(in); + fadst8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + load_buffer_8x8(input, in, stride, 1, 0); + fadst8_sse2(in); + fdct8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case DCT_FLIPADST: + load_buffer_8x8(input, in, stride, 0, 1); + fdct8_sse2(in); + fadst8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case FLIPADST_FLIPADST: + load_buffer_8x8(input, in, stride, 1, 1); + fadst8_sse2(in); + fadst8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case ADST_FLIPADST: + load_buffer_8x8(input, in, stride, 0, 1); + fadst8_sse2(in); + fadst8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case FLIPADST_ADST: + load_buffer_8x8(input, in, stride, 1, 0); + fadst8_sse2(in); + fadst8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case IDTX: + load_buffer_8x8(input, in, stride, 0, 0); + fidtx8_sse2(in); + fidtx8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case V_DCT: + load_buffer_8x8(input, in, stride, 0, 0); + fdct8_sse2(in); + fidtx8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case H_DCT: + load_buffer_8x8(input, in, stride, 0, 0); + fidtx8_sse2(in); + fdct8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case V_ADST: + load_buffer_8x8(input, in, stride, 0, 0); + fadst8_sse2(in); + fidtx8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case H_ADST: + load_buffer_8x8(input, in, stride, 0, 0); + fidtx8_sse2(in); + fadst8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case V_FLIPADST: + load_buffer_8x8(input, in, stride, 1, 0); + fadst8_sse2(in); + fidtx8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case H_FLIPADST: + load_buffer_8x8(input, in, stride, 0, 1); + fidtx8_sse2(in); + fadst8_sse2(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; +#endif // CONFIG_EXT_TX + default: assert(0); + } +} + +static INLINE void load_buffer_16x16(const int16_t *input, __m128i *in0, + __m128i *in1, int stride, int flipud, + int fliplr) { + // Load 4 8x8 blocks + const int16_t *topL = input; + const int16_t *topR = input + 8; + const int16_t *botL = input + 8 * stride; + const int16_t *botR = input + 8 * stride + 8; + + const int16_t *tmp; + + if (flipud) { + // Swap left columns + tmp = topL; + topL = botL; + botL = tmp; + // Swap right columns + tmp = topR; + topR = botR; + botR = tmp; + } + + if (fliplr) { + // Swap top rows + tmp = topL; + topL = topR; + topR = tmp; + // Swap bottom rows + tmp = botL; + botL = botR; + botR = tmp; + } + + // load first 8 columns + load_buffer_8x8(topL, in0, stride, flipud, fliplr); + load_buffer_8x8(botL, in0 + 8, stride, flipud, fliplr); + + // load second 8 columns + load_buffer_8x8(topR, in1, stride, flipud, fliplr); + load_buffer_8x8(botR, in1 + 8, stride, flipud, fliplr); +} + +static INLINE void write_buffer_16x16(tran_low_t *output, __m128i *in0, + __m128i *in1, int stride) { + // write first 8 columns + write_buffer_8x8(output, in0, stride); + write_buffer_8x8(output + 8 * stride, in0 + 8, stride); + // write second 8 columns + output += 8; + write_buffer_8x8(output, in1, stride); + write_buffer_8x8(output + 8 * stride, in1 + 8, stride); +} + +static INLINE void array_transpose_16x16(__m128i *res0, __m128i *res1) { + __m128i tbuf[8]; + array_transpose_8x8(res0, res0); + array_transpose_8x8(res1, tbuf); + array_transpose_8x8(res0 + 8, res1); + array_transpose_8x8(res1 + 8, res1 + 8); + + res0[8] = tbuf[0]; + res0[9] = tbuf[1]; + res0[10] = tbuf[2]; + res0[11] = tbuf[3]; + res0[12] = tbuf[4]; + res0[13] = tbuf[5]; + res0[14] = tbuf[6]; + res0[15] = tbuf[7]; +} + +static INLINE void right_shift_16x16(__m128i *res0, __m128i *res1) { + // perform rounding operations + right_shift_8x8(res0, 2); + right_shift_8x8(res0 + 8, 2); + right_shift_8x8(res1, 2); + right_shift_8x8(res1 + 8, 2); +} + +static void fdct16_8col(__m128i *in) { + // perform 16x16 1-D DCT for 8 columns + __m128i i[8], s[8], p[8], t[8], u[16], v[16]; + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64); + const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i k__cospi_p30_p02 = pair_set_epi16(cospi_30_64, cospi_2_64); + const __m128i k__cospi_p14_p18 = pair_set_epi16(cospi_14_64, cospi_18_64); + const __m128i k__cospi_m02_p30 = pair_set_epi16(-cospi_2_64, cospi_30_64); + const __m128i k__cospi_m18_p14 = pair_set_epi16(-cospi_18_64, cospi_14_64); + const __m128i k__cospi_p22_p10 = pair_set_epi16(cospi_22_64, cospi_10_64); + const __m128i k__cospi_p06_p26 = pair_set_epi16(cospi_6_64, cospi_26_64); + const __m128i k__cospi_m10_p22 = pair_set_epi16(-cospi_10_64, cospi_22_64); + const __m128i k__cospi_m26_p06 = pair_set_epi16(-cospi_26_64, cospi_6_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + + // stage 1 + i[0] = _mm_add_epi16(in[0], in[15]); + i[1] = _mm_add_epi16(in[1], in[14]); + i[2] = _mm_add_epi16(in[2], in[13]); + i[3] = _mm_add_epi16(in[3], in[12]); + i[4] = _mm_add_epi16(in[4], in[11]); + i[5] = _mm_add_epi16(in[5], in[10]); + i[6] = _mm_add_epi16(in[6], in[9]); + i[7] = _mm_add_epi16(in[7], in[8]); + + s[0] = _mm_sub_epi16(in[7], in[8]); + s[1] = _mm_sub_epi16(in[6], in[9]); + s[2] = _mm_sub_epi16(in[5], in[10]); + s[3] = _mm_sub_epi16(in[4], in[11]); + s[4] = _mm_sub_epi16(in[3], in[12]); + s[5] = _mm_sub_epi16(in[2], in[13]); + s[6] = _mm_sub_epi16(in[1], in[14]); + s[7] = _mm_sub_epi16(in[0], in[15]); + + p[0] = _mm_add_epi16(i[0], i[7]); + p[1] = _mm_add_epi16(i[1], i[6]); + p[2] = _mm_add_epi16(i[2], i[5]); + p[3] = _mm_add_epi16(i[3], i[4]); + p[4] = _mm_sub_epi16(i[3], i[4]); + p[5] = _mm_sub_epi16(i[2], i[5]); + p[6] = _mm_sub_epi16(i[1], i[6]); + p[7] = _mm_sub_epi16(i[0], i[7]); + + u[0] = _mm_add_epi16(p[0], p[3]); + u[1] = _mm_add_epi16(p[1], p[2]); + u[2] = _mm_sub_epi16(p[1], p[2]); + u[3] = _mm_sub_epi16(p[0], p[3]); + + v[0] = _mm_unpacklo_epi16(u[0], u[1]); + v[1] = _mm_unpackhi_epi16(u[0], u[1]); + v[2] = _mm_unpacklo_epi16(u[2], u[3]); + v[3] = _mm_unpackhi_epi16(u[2], u[3]); + + u[0] = _mm_madd_epi16(v[0], k__cospi_p16_p16); + u[1] = _mm_madd_epi16(v[1], k__cospi_p16_p16); + u[2] = _mm_madd_epi16(v[0], k__cospi_p16_m16); + u[3] = _mm_madd_epi16(v[1], k__cospi_p16_m16); + u[4] = _mm_madd_epi16(v[2], k__cospi_p24_p08); + u[5] = _mm_madd_epi16(v[3], k__cospi_p24_p08); + u[6] = _mm_madd_epi16(v[2], k__cospi_m08_p24); + u[7] = _mm_madd_epi16(v[3], k__cospi_m08_p24); + + v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + + u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); + u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); + u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); + u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); + u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS); + u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS); + u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS); + u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS); + + in[0] = _mm_packs_epi32(u[0], u[1]); + in[4] = _mm_packs_epi32(u[4], u[5]); + in[8] = _mm_packs_epi32(u[2], u[3]); + in[12] = _mm_packs_epi32(u[6], u[7]); + + u[0] = _mm_unpacklo_epi16(p[5], p[6]); + u[1] = _mm_unpackhi_epi16(p[5], p[6]); + v[0] = _mm_madd_epi16(u[0], k__cospi_m16_p16); + v[1] = _mm_madd_epi16(u[1], k__cospi_m16_p16); + v[2] = _mm_madd_epi16(u[0], k__cospi_p16_p16); + v[3] = _mm_madd_epi16(u[1], k__cospi_p16_p16); + + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + + v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + + u[0] = _mm_packs_epi32(v[0], v[1]); + u[1] = _mm_packs_epi32(v[2], v[3]); + + t[0] = _mm_add_epi16(p[4], u[0]); + t[1] = _mm_sub_epi16(p[4], u[0]); + t[2] = _mm_sub_epi16(p[7], u[1]); + t[3] = _mm_add_epi16(p[7], u[1]); + + u[0] = _mm_unpacklo_epi16(t[0], t[3]); + u[1] = _mm_unpackhi_epi16(t[0], t[3]); + u[2] = _mm_unpacklo_epi16(t[1], t[2]); + u[3] = _mm_unpackhi_epi16(t[1], t[2]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_p28_p04); + v[1] = _mm_madd_epi16(u[1], k__cospi_p28_p04); + v[2] = _mm_madd_epi16(u[2], k__cospi_p12_p20); + v[3] = _mm_madd_epi16(u[3], k__cospi_p12_p20); + v[4] = _mm_madd_epi16(u[2], k__cospi_m20_p12); + v[5] = _mm_madd_epi16(u[3], k__cospi_m20_p12); + v[6] = _mm_madd_epi16(u[0], k__cospi_m04_p28); + v[7] = _mm_madd_epi16(u[1], k__cospi_m04_p28); + + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); + + v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + + in[2] = _mm_packs_epi32(v[0], v[1]); + in[6] = _mm_packs_epi32(v[4], v[5]); + in[10] = _mm_packs_epi32(v[2], v[3]); + in[14] = _mm_packs_epi32(v[6], v[7]); + + // stage 2 + u[0] = _mm_unpacklo_epi16(s[2], s[5]); + u[1] = _mm_unpackhi_epi16(s[2], s[5]); + u[2] = _mm_unpacklo_epi16(s[3], s[4]); + u[3] = _mm_unpackhi_epi16(s[3], s[4]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_m16_p16); + v[1] = _mm_madd_epi16(u[1], k__cospi_m16_p16); + v[2] = _mm_madd_epi16(u[2], k__cospi_m16_p16); + v[3] = _mm_madd_epi16(u[3], k__cospi_m16_p16); + v[4] = _mm_madd_epi16(u[2], k__cospi_p16_p16); + v[5] = _mm_madd_epi16(u[3], k__cospi_p16_p16); + v[6] = _mm_madd_epi16(u[0], k__cospi_p16_p16); + v[7] = _mm_madd_epi16(u[1], k__cospi_p16_p16); + + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); + + v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + + t[2] = _mm_packs_epi32(v[0], v[1]); + t[3] = _mm_packs_epi32(v[2], v[3]); + t[4] = _mm_packs_epi32(v[4], v[5]); + t[5] = _mm_packs_epi32(v[6], v[7]); + + // stage 3 + p[0] = _mm_add_epi16(s[0], t[3]); + p[1] = _mm_add_epi16(s[1], t[2]); + p[2] = _mm_sub_epi16(s[1], t[2]); + p[3] = _mm_sub_epi16(s[0], t[3]); + p[4] = _mm_sub_epi16(s[7], t[4]); + p[5] = _mm_sub_epi16(s[6], t[5]); + p[6] = _mm_add_epi16(s[6], t[5]); + p[7] = _mm_add_epi16(s[7], t[4]); + + // stage 4 + u[0] = _mm_unpacklo_epi16(p[1], p[6]); + u[1] = _mm_unpackhi_epi16(p[1], p[6]); + u[2] = _mm_unpacklo_epi16(p[2], p[5]); + u[3] = _mm_unpackhi_epi16(p[2], p[5]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_m08_p24); + v[1] = _mm_madd_epi16(u[1], k__cospi_m08_p24); + v[2] = _mm_madd_epi16(u[2], k__cospi_m24_m08); + v[3] = _mm_madd_epi16(u[3], k__cospi_m24_m08); + v[4] = _mm_madd_epi16(u[2], k__cospi_m08_p24); + v[5] = _mm_madd_epi16(u[3], k__cospi_m08_p24); + v[6] = _mm_madd_epi16(u[0], k__cospi_p24_p08); + v[7] = _mm_madd_epi16(u[1], k__cospi_p24_p08); + + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); + + v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + + t[1] = _mm_packs_epi32(v[0], v[1]); + t[2] = _mm_packs_epi32(v[2], v[3]); + t[5] = _mm_packs_epi32(v[4], v[5]); + t[6] = _mm_packs_epi32(v[6], v[7]); + + // stage 5 + s[0] = _mm_add_epi16(p[0], t[1]); + s[1] = _mm_sub_epi16(p[0], t[1]); + s[2] = _mm_sub_epi16(p[3], t[2]); + s[3] = _mm_add_epi16(p[3], t[2]); + s[4] = _mm_add_epi16(p[4], t[5]); + s[5] = _mm_sub_epi16(p[4], t[5]); + s[6] = _mm_sub_epi16(p[7], t[6]); + s[7] = _mm_add_epi16(p[7], t[6]); + + // stage 6 + u[0] = _mm_unpacklo_epi16(s[0], s[7]); + u[1] = _mm_unpackhi_epi16(s[0], s[7]); + u[2] = _mm_unpacklo_epi16(s[1], s[6]); + u[3] = _mm_unpackhi_epi16(s[1], s[6]); + u[4] = _mm_unpacklo_epi16(s[2], s[5]); + u[5] = _mm_unpackhi_epi16(s[2], s[5]); + u[6] = _mm_unpacklo_epi16(s[3], s[4]); + u[7] = _mm_unpackhi_epi16(s[3], s[4]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_p30_p02); + v[1] = _mm_madd_epi16(u[1], k__cospi_p30_p02); + v[2] = _mm_madd_epi16(u[2], k__cospi_p14_p18); + v[3] = _mm_madd_epi16(u[3], k__cospi_p14_p18); + v[4] = _mm_madd_epi16(u[4], k__cospi_p22_p10); + v[5] = _mm_madd_epi16(u[5], k__cospi_p22_p10); + v[6] = _mm_madd_epi16(u[6], k__cospi_p06_p26); + v[7] = _mm_madd_epi16(u[7], k__cospi_p06_p26); + v[8] = _mm_madd_epi16(u[6], k__cospi_m26_p06); + v[9] = _mm_madd_epi16(u[7], k__cospi_m26_p06); + v[10] = _mm_madd_epi16(u[4], k__cospi_m10_p22); + v[11] = _mm_madd_epi16(u[5], k__cospi_m10_p22); + v[12] = _mm_madd_epi16(u[2], k__cospi_m18_p14); + v[13] = _mm_madd_epi16(u[3], k__cospi_m18_p14); + v[14] = _mm_madd_epi16(u[0], k__cospi_m02_p30); + v[15] = _mm_madd_epi16(u[1], k__cospi_m02_p30); + + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); + u[8] = _mm_add_epi32(v[8], k__DCT_CONST_ROUNDING); + u[9] = _mm_add_epi32(v[9], k__DCT_CONST_ROUNDING); + u[10] = _mm_add_epi32(v[10], k__DCT_CONST_ROUNDING); + u[11] = _mm_add_epi32(v[11], k__DCT_CONST_ROUNDING); + u[12] = _mm_add_epi32(v[12], k__DCT_CONST_ROUNDING); + u[13] = _mm_add_epi32(v[13], k__DCT_CONST_ROUNDING); + u[14] = _mm_add_epi32(v[14], k__DCT_CONST_ROUNDING); + u[15] = _mm_add_epi32(v[15], k__DCT_CONST_ROUNDING); + + v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + v[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS); + v[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS); + v[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS); + v[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS); + v[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS); + v[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS); + v[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS); + v[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS); + + in[1] = _mm_packs_epi32(v[0], v[1]); + in[9] = _mm_packs_epi32(v[2], v[3]); + in[5] = _mm_packs_epi32(v[4], v[5]); + in[13] = _mm_packs_epi32(v[6], v[7]); + in[3] = _mm_packs_epi32(v[8], v[9]); + in[11] = _mm_packs_epi32(v[10], v[11]); + in[7] = _mm_packs_epi32(v[12], v[13]); + in[15] = _mm_packs_epi32(v[14], v[15]); +} + +static void fadst16_8col(__m128i *in) { + // perform 16x16 1-D ADST for 8 columns + __m128i s[16], x[16], u[32], v[32]; + const __m128i k__cospi_p01_p31 = pair_set_epi16(cospi_1_64, cospi_31_64); + const __m128i k__cospi_p31_m01 = pair_set_epi16(cospi_31_64, -cospi_1_64); + const __m128i k__cospi_p05_p27 = pair_set_epi16(cospi_5_64, cospi_27_64); + const __m128i k__cospi_p27_m05 = pair_set_epi16(cospi_27_64, -cospi_5_64); + const __m128i k__cospi_p09_p23 = pair_set_epi16(cospi_9_64, cospi_23_64); + const __m128i k__cospi_p23_m09 = pair_set_epi16(cospi_23_64, -cospi_9_64); + const __m128i k__cospi_p13_p19 = pair_set_epi16(cospi_13_64, cospi_19_64); + const __m128i k__cospi_p19_m13 = pair_set_epi16(cospi_19_64, -cospi_13_64); + const __m128i k__cospi_p17_p15 = pair_set_epi16(cospi_17_64, cospi_15_64); + const __m128i k__cospi_p15_m17 = pair_set_epi16(cospi_15_64, -cospi_17_64); + const __m128i k__cospi_p21_p11 = pair_set_epi16(cospi_21_64, cospi_11_64); + const __m128i k__cospi_p11_m21 = pair_set_epi16(cospi_11_64, -cospi_21_64); + const __m128i k__cospi_p25_p07 = pair_set_epi16(cospi_25_64, cospi_7_64); + const __m128i k__cospi_p07_m25 = pair_set_epi16(cospi_7_64, -cospi_25_64); + const __m128i k__cospi_p29_p03 = pair_set_epi16(cospi_29_64, cospi_3_64); + const __m128i k__cospi_p03_m29 = pair_set_epi16(cospi_3_64, -cospi_29_64); + const __m128i k__cospi_p04_p28 = pair_set_epi16(cospi_4_64, cospi_28_64); + const __m128i k__cospi_p28_m04 = pair_set_epi16(cospi_28_64, -cospi_4_64); + const __m128i k__cospi_p20_p12 = pair_set_epi16(cospi_20_64, cospi_12_64); + const __m128i k__cospi_p12_m20 = pair_set_epi16(cospi_12_64, -cospi_20_64); + const __m128i k__cospi_m28_p04 = pair_set_epi16(-cospi_28_64, cospi_4_64); + const __m128i k__cospi_m12_p20 = pair_set_epi16(-cospi_12_64, cospi_20_64); + const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64); + const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64); + const __m128i k__cospi_m24_p08 = pair_set_epi16(-cospi_24_64, cospi_8_64); + const __m128i k__cospi_m16_m16 = _mm_set1_epi16((int16_t)-cospi_16_64); + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + const __m128i kZero = _mm_set1_epi16(0); + + u[0] = _mm_unpacklo_epi16(in[15], in[0]); + u[1] = _mm_unpackhi_epi16(in[15], in[0]); + u[2] = _mm_unpacklo_epi16(in[13], in[2]); + u[3] = _mm_unpackhi_epi16(in[13], in[2]); + u[4] = _mm_unpacklo_epi16(in[11], in[4]); + u[5] = _mm_unpackhi_epi16(in[11], in[4]); + u[6] = _mm_unpacklo_epi16(in[9], in[6]); + u[7] = _mm_unpackhi_epi16(in[9], in[6]); + u[8] = _mm_unpacklo_epi16(in[7], in[8]); + u[9] = _mm_unpackhi_epi16(in[7], in[8]); + u[10] = _mm_unpacklo_epi16(in[5], in[10]); + u[11] = _mm_unpackhi_epi16(in[5], in[10]); + u[12] = _mm_unpacklo_epi16(in[3], in[12]); + u[13] = _mm_unpackhi_epi16(in[3], in[12]); + u[14] = _mm_unpacklo_epi16(in[1], in[14]); + u[15] = _mm_unpackhi_epi16(in[1], in[14]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_p01_p31); + v[1] = _mm_madd_epi16(u[1], k__cospi_p01_p31); + v[2] = _mm_madd_epi16(u[0], k__cospi_p31_m01); + v[3] = _mm_madd_epi16(u[1], k__cospi_p31_m01); + v[4] = _mm_madd_epi16(u[2], k__cospi_p05_p27); + v[5] = _mm_madd_epi16(u[3], k__cospi_p05_p27); + v[6] = _mm_madd_epi16(u[2], k__cospi_p27_m05); + v[7] = _mm_madd_epi16(u[3], k__cospi_p27_m05); + v[8] = _mm_madd_epi16(u[4], k__cospi_p09_p23); + v[9] = _mm_madd_epi16(u[5], k__cospi_p09_p23); + v[10] = _mm_madd_epi16(u[4], k__cospi_p23_m09); + v[11] = _mm_madd_epi16(u[5], k__cospi_p23_m09); + v[12] = _mm_madd_epi16(u[6], k__cospi_p13_p19); + v[13] = _mm_madd_epi16(u[7], k__cospi_p13_p19); + v[14] = _mm_madd_epi16(u[6], k__cospi_p19_m13); + v[15] = _mm_madd_epi16(u[7], k__cospi_p19_m13); + v[16] = _mm_madd_epi16(u[8], k__cospi_p17_p15); + v[17] = _mm_madd_epi16(u[9], k__cospi_p17_p15); + v[18] = _mm_madd_epi16(u[8], k__cospi_p15_m17); + v[19] = _mm_madd_epi16(u[9], k__cospi_p15_m17); + v[20] = _mm_madd_epi16(u[10], k__cospi_p21_p11); + v[21] = _mm_madd_epi16(u[11], k__cospi_p21_p11); + v[22] = _mm_madd_epi16(u[10], k__cospi_p11_m21); + v[23] = _mm_madd_epi16(u[11], k__cospi_p11_m21); + v[24] = _mm_madd_epi16(u[12], k__cospi_p25_p07); + v[25] = _mm_madd_epi16(u[13], k__cospi_p25_p07); + v[26] = _mm_madd_epi16(u[12], k__cospi_p07_m25); + v[27] = _mm_madd_epi16(u[13], k__cospi_p07_m25); + v[28] = _mm_madd_epi16(u[14], k__cospi_p29_p03); + v[29] = _mm_madd_epi16(u[15], k__cospi_p29_p03); + v[30] = _mm_madd_epi16(u[14], k__cospi_p03_m29); + v[31] = _mm_madd_epi16(u[15], k__cospi_p03_m29); + + u[0] = _mm_add_epi32(v[0], v[16]); + u[1] = _mm_add_epi32(v[1], v[17]); + u[2] = _mm_add_epi32(v[2], v[18]); + u[3] = _mm_add_epi32(v[3], v[19]); + u[4] = _mm_add_epi32(v[4], v[20]); + u[5] = _mm_add_epi32(v[5], v[21]); + u[6] = _mm_add_epi32(v[6], v[22]); + u[7] = _mm_add_epi32(v[7], v[23]); + u[8] = _mm_add_epi32(v[8], v[24]); + u[9] = _mm_add_epi32(v[9], v[25]); + u[10] = _mm_add_epi32(v[10], v[26]); + u[11] = _mm_add_epi32(v[11], v[27]); + u[12] = _mm_add_epi32(v[12], v[28]); + u[13] = _mm_add_epi32(v[13], v[29]); + u[14] = _mm_add_epi32(v[14], v[30]); + u[15] = _mm_add_epi32(v[15], v[31]); + u[16] = _mm_sub_epi32(v[0], v[16]); + u[17] = _mm_sub_epi32(v[1], v[17]); + u[18] = _mm_sub_epi32(v[2], v[18]); + u[19] = _mm_sub_epi32(v[3], v[19]); + u[20] = _mm_sub_epi32(v[4], v[20]); + u[21] = _mm_sub_epi32(v[5], v[21]); + u[22] = _mm_sub_epi32(v[6], v[22]); + u[23] = _mm_sub_epi32(v[7], v[23]); + u[24] = _mm_sub_epi32(v[8], v[24]); + u[25] = _mm_sub_epi32(v[9], v[25]); + u[26] = _mm_sub_epi32(v[10], v[26]); + u[27] = _mm_sub_epi32(v[11], v[27]); + u[28] = _mm_sub_epi32(v[12], v[28]); + u[29] = _mm_sub_epi32(v[13], v[29]); + u[30] = _mm_sub_epi32(v[14], v[30]); + u[31] = _mm_sub_epi32(v[15], v[31]); + + v[16] = _mm_add_epi32(u[16], k__DCT_CONST_ROUNDING); + v[17] = _mm_add_epi32(u[17], k__DCT_CONST_ROUNDING); + v[18] = _mm_add_epi32(u[18], k__DCT_CONST_ROUNDING); + v[19] = _mm_add_epi32(u[19], k__DCT_CONST_ROUNDING); + v[20] = _mm_add_epi32(u[20], k__DCT_CONST_ROUNDING); + v[21] = _mm_add_epi32(u[21], k__DCT_CONST_ROUNDING); + v[22] = _mm_add_epi32(u[22], k__DCT_CONST_ROUNDING); + v[23] = _mm_add_epi32(u[23], k__DCT_CONST_ROUNDING); + v[24] = _mm_add_epi32(u[24], k__DCT_CONST_ROUNDING); + v[25] = _mm_add_epi32(u[25], k__DCT_CONST_ROUNDING); + v[26] = _mm_add_epi32(u[26], k__DCT_CONST_ROUNDING); + v[27] = _mm_add_epi32(u[27], k__DCT_CONST_ROUNDING); + v[28] = _mm_add_epi32(u[28], k__DCT_CONST_ROUNDING); + v[29] = _mm_add_epi32(u[29], k__DCT_CONST_ROUNDING); + v[30] = _mm_add_epi32(u[30], k__DCT_CONST_ROUNDING); + v[31] = _mm_add_epi32(u[31], k__DCT_CONST_ROUNDING); + + u[16] = _mm_srai_epi32(v[16], DCT_CONST_BITS); + u[17] = _mm_srai_epi32(v[17], DCT_CONST_BITS); + u[18] = _mm_srai_epi32(v[18], DCT_CONST_BITS); + u[19] = _mm_srai_epi32(v[19], DCT_CONST_BITS); + u[20] = _mm_srai_epi32(v[20], DCT_CONST_BITS); + u[21] = _mm_srai_epi32(v[21], DCT_CONST_BITS); + u[22] = _mm_srai_epi32(v[22], DCT_CONST_BITS); + u[23] = _mm_srai_epi32(v[23], DCT_CONST_BITS); + u[24] = _mm_srai_epi32(v[24], DCT_CONST_BITS); + u[25] = _mm_srai_epi32(v[25], DCT_CONST_BITS); + u[26] = _mm_srai_epi32(v[26], DCT_CONST_BITS); + u[27] = _mm_srai_epi32(v[27], DCT_CONST_BITS); + u[28] = _mm_srai_epi32(v[28], DCT_CONST_BITS); + u[29] = _mm_srai_epi32(v[29], DCT_CONST_BITS); + u[30] = _mm_srai_epi32(v[30], DCT_CONST_BITS); + u[31] = _mm_srai_epi32(v[31], DCT_CONST_BITS); + + v[0] = _mm_add_epi32(u[0], u[8]); + v[1] = _mm_add_epi32(u[1], u[9]); + v[2] = _mm_add_epi32(u[2], u[10]); + v[3] = _mm_add_epi32(u[3], u[11]); + v[4] = _mm_add_epi32(u[4], u[12]); + v[5] = _mm_add_epi32(u[5], u[13]); + v[6] = _mm_add_epi32(u[6], u[14]); + v[7] = _mm_add_epi32(u[7], u[15]); + + v[16] = _mm_add_epi32(v[0], v[4]); + v[17] = _mm_add_epi32(v[1], v[5]); + v[18] = _mm_add_epi32(v[2], v[6]); + v[19] = _mm_add_epi32(v[3], v[7]); + v[20] = _mm_sub_epi32(v[0], v[4]); + v[21] = _mm_sub_epi32(v[1], v[5]); + v[22] = _mm_sub_epi32(v[2], v[6]); + v[23] = _mm_sub_epi32(v[3], v[7]); + v[16] = _mm_add_epi32(v[16], k__DCT_CONST_ROUNDING); + v[17] = _mm_add_epi32(v[17], k__DCT_CONST_ROUNDING); + v[18] = _mm_add_epi32(v[18], k__DCT_CONST_ROUNDING); + v[19] = _mm_add_epi32(v[19], k__DCT_CONST_ROUNDING); + v[20] = _mm_add_epi32(v[20], k__DCT_CONST_ROUNDING); + v[21] = _mm_add_epi32(v[21], k__DCT_CONST_ROUNDING); + v[22] = _mm_add_epi32(v[22], k__DCT_CONST_ROUNDING); + v[23] = _mm_add_epi32(v[23], k__DCT_CONST_ROUNDING); + v[16] = _mm_srai_epi32(v[16], DCT_CONST_BITS); + v[17] = _mm_srai_epi32(v[17], DCT_CONST_BITS); + v[18] = _mm_srai_epi32(v[18], DCT_CONST_BITS); + v[19] = _mm_srai_epi32(v[19], DCT_CONST_BITS); + v[20] = _mm_srai_epi32(v[20], DCT_CONST_BITS); + v[21] = _mm_srai_epi32(v[21], DCT_CONST_BITS); + v[22] = _mm_srai_epi32(v[22], DCT_CONST_BITS); + v[23] = _mm_srai_epi32(v[23], DCT_CONST_BITS); + s[0] = _mm_packs_epi32(v[16], v[17]); + s[1] = _mm_packs_epi32(v[18], v[19]); + s[2] = _mm_packs_epi32(v[20], v[21]); + s[3] = _mm_packs_epi32(v[22], v[23]); + + v[8] = _mm_sub_epi32(u[0], u[8]); + v[9] = _mm_sub_epi32(u[1], u[9]); + v[10] = _mm_sub_epi32(u[2], u[10]); + v[11] = _mm_sub_epi32(u[3], u[11]); + v[12] = _mm_sub_epi32(u[4], u[12]); + v[13] = _mm_sub_epi32(u[5], u[13]); + v[14] = _mm_sub_epi32(u[6], u[14]); + v[15] = _mm_sub_epi32(u[7], u[15]); + + v[8] = _mm_add_epi32(v[8], k__DCT_CONST_ROUNDING); + v[9] = _mm_add_epi32(v[9], k__DCT_CONST_ROUNDING); + v[10] = _mm_add_epi32(v[10], k__DCT_CONST_ROUNDING); + v[11] = _mm_add_epi32(v[11], k__DCT_CONST_ROUNDING); + v[12] = _mm_add_epi32(v[12], k__DCT_CONST_ROUNDING); + v[13] = _mm_add_epi32(v[13], k__DCT_CONST_ROUNDING); + v[14] = _mm_add_epi32(v[14], k__DCT_CONST_ROUNDING); + v[15] = _mm_add_epi32(v[15], k__DCT_CONST_ROUNDING); + + v[8] = _mm_srai_epi32(v[8], DCT_CONST_BITS); + v[9] = _mm_srai_epi32(v[9], DCT_CONST_BITS); + v[10] = _mm_srai_epi32(v[10], DCT_CONST_BITS); + v[11] = _mm_srai_epi32(v[11], DCT_CONST_BITS); + v[12] = _mm_srai_epi32(v[12], DCT_CONST_BITS); + v[13] = _mm_srai_epi32(v[13], DCT_CONST_BITS); + v[14] = _mm_srai_epi32(v[14], DCT_CONST_BITS); + v[15] = _mm_srai_epi32(v[15], DCT_CONST_BITS); + + s[4] = _mm_packs_epi32(v[8], v[9]); + s[5] = _mm_packs_epi32(v[10], v[11]); + s[6] = _mm_packs_epi32(v[12], v[13]); + s[7] = _mm_packs_epi32(v[14], v[15]); + // + + s[8] = _mm_packs_epi32(u[16], u[17]); + s[9] = _mm_packs_epi32(u[18], u[19]); + s[10] = _mm_packs_epi32(u[20], u[21]); + s[11] = _mm_packs_epi32(u[22], u[23]); + s[12] = _mm_packs_epi32(u[24], u[25]); + s[13] = _mm_packs_epi32(u[26], u[27]); + s[14] = _mm_packs_epi32(u[28], u[29]); + s[15] = _mm_packs_epi32(u[30], u[31]); + + // stage 2 + u[0] = _mm_unpacklo_epi16(s[8], s[9]); + u[1] = _mm_unpackhi_epi16(s[8], s[9]); + u[2] = _mm_unpacklo_epi16(s[10], s[11]); + u[3] = _mm_unpackhi_epi16(s[10], s[11]); + u[4] = _mm_unpacklo_epi16(s[12], s[13]); + u[5] = _mm_unpackhi_epi16(s[12], s[13]); + u[6] = _mm_unpacklo_epi16(s[14], s[15]); + u[7] = _mm_unpackhi_epi16(s[14], s[15]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_p04_p28); + v[1] = _mm_madd_epi16(u[1], k__cospi_p04_p28); + v[2] = _mm_madd_epi16(u[0], k__cospi_p28_m04); + v[3] = _mm_madd_epi16(u[1], k__cospi_p28_m04); + v[4] = _mm_madd_epi16(u[2], k__cospi_p20_p12); + v[5] = _mm_madd_epi16(u[3], k__cospi_p20_p12); + v[6] = _mm_madd_epi16(u[2], k__cospi_p12_m20); + v[7] = _mm_madd_epi16(u[3], k__cospi_p12_m20); + v[8] = _mm_madd_epi16(u[4], k__cospi_m28_p04); + v[9] = _mm_madd_epi16(u[5], k__cospi_m28_p04); + v[10] = _mm_madd_epi16(u[4], k__cospi_p04_p28); + v[11] = _mm_madd_epi16(u[5], k__cospi_p04_p28); + v[12] = _mm_madd_epi16(u[6], k__cospi_m12_p20); + v[13] = _mm_madd_epi16(u[7], k__cospi_m12_p20); + v[14] = _mm_madd_epi16(u[6], k__cospi_p20_p12); + v[15] = _mm_madd_epi16(u[7], k__cospi_p20_p12); + + u[0] = _mm_add_epi32(v[0], v[8]); + u[1] = _mm_add_epi32(v[1], v[9]); + u[2] = _mm_add_epi32(v[2], v[10]); + u[3] = _mm_add_epi32(v[3], v[11]); + u[4] = _mm_add_epi32(v[4], v[12]); + u[5] = _mm_add_epi32(v[5], v[13]); + u[6] = _mm_add_epi32(v[6], v[14]); + u[7] = _mm_add_epi32(v[7], v[15]); + u[8] = _mm_sub_epi32(v[0], v[8]); + u[9] = _mm_sub_epi32(v[1], v[9]); + u[10] = _mm_sub_epi32(v[2], v[10]); + u[11] = _mm_sub_epi32(v[3], v[11]); + u[12] = _mm_sub_epi32(v[4], v[12]); + u[13] = _mm_sub_epi32(v[5], v[13]); + u[14] = _mm_sub_epi32(v[6], v[14]); + u[15] = _mm_sub_epi32(v[7], v[15]); + + v[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); + v[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); + v[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); + v[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); + v[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); + v[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); + v[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); + v[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + u[8] = _mm_srai_epi32(v[8], DCT_CONST_BITS); + u[9] = _mm_srai_epi32(v[9], DCT_CONST_BITS); + u[10] = _mm_srai_epi32(v[10], DCT_CONST_BITS); + u[11] = _mm_srai_epi32(v[11], DCT_CONST_BITS); + u[12] = _mm_srai_epi32(v[12], DCT_CONST_BITS); + u[13] = _mm_srai_epi32(v[13], DCT_CONST_BITS); + u[14] = _mm_srai_epi32(v[14], DCT_CONST_BITS); + u[15] = _mm_srai_epi32(v[15], DCT_CONST_BITS); + + v[8] = _mm_add_epi32(u[0], u[4]); + v[9] = _mm_add_epi32(u[1], u[5]); + v[10] = _mm_add_epi32(u[2], u[6]); + v[11] = _mm_add_epi32(u[3], u[7]); + v[12] = _mm_sub_epi32(u[0], u[4]); + v[13] = _mm_sub_epi32(u[1], u[5]); + v[14] = _mm_sub_epi32(u[2], u[6]); + v[15] = _mm_sub_epi32(u[3], u[7]); + + v[8] = _mm_add_epi32(v[8], k__DCT_CONST_ROUNDING); + v[9] = _mm_add_epi32(v[9], k__DCT_CONST_ROUNDING); + v[10] = _mm_add_epi32(v[10], k__DCT_CONST_ROUNDING); + v[11] = _mm_add_epi32(v[11], k__DCT_CONST_ROUNDING); + v[12] = _mm_add_epi32(v[12], k__DCT_CONST_ROUNDING); + v[13] = _mm_add_epi32(v[13], k__DCT_CONST_ROUNDING); + v[14] = _mm_add_epi32(v[14], k__DCT_CONST_ROUNDING); + v[15] = _mm_add_epi32(v[15], k__DCT_CONST_ROUNDING); + v[8] = _mm_srai_epi32(v[8], DCT_CONST_BITS); + v[9] = _mm_srai_epi32(v[9], DCT_CONST_BITS); + v[10] = _mm_srai_epi32(v[10], DCT_CONST_BITS); + v[11] = _mm_srai_epi32(v[11], DCT_CONST_BITS); + v[12] = _mm_srai_epi32(v[12], DCT_CONST_BITS); + v[13] = _mm_srai_epi32(v[13], DCT_CONST_BITS); + v[14] = _mm_srai_epi32(v[14], DCT_CONST_BITS); + v[15] = _mm_srai_epi32(v[15], DCT_CONST_BITS); + s[8] = _mm_packs_epi32(v[8], v[9]); + s[9] = _mm_packs_epi32(v[10], v[11]); + s[10] = _mm_packs_epi32(v[12], v[13]); + s[11] = _mm_packs_epi32(v[14], v[15]); + + x[12] = _mm_packs_epi32(u[8], u[9]); + x[13] = _mm_packs_epi32(u[10], u[11]); + x[14] = _mm_packs_epi32(u[12], u[13]); + x[15] = _mm_packs_epi32(u[14], u[15]); + + // stage 3 + u[0] = _mm_unpacklo_epi16(s[4], s[5]); + u[1] = _mm_unpackhi_epi16(s[4], s[5]); + u[2] = _mm_unpacklo_epi16(s[6], s[7]); + u[3] = _mm_unpackhi_epi16(s[6], s[7]); + u[4] = _mm_unpacklo_epi16(x[12], x[13]); + u[5] = _mm_unpackhi_epi16(x[12], x[13]); + u[6] = _mm_unpacklo_epi16(x[14], x[15]); + u[7] = _mm_unpackhi_epi16(x[14], x[15]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_p08_p24); + v[1] = _mm_madd_epi16(u[1], k__cospi_p08_p24); + v[2] = _mm_madd_epi16(u[0], k__cospi_p24_m08); + v[3] = _mm_madd_epi16(u[1], k__cospi_p24_m08); + v[4] = _mm_madd_epi16(u[2], k__cospi_m24_p08); + v[5] = _mm_madd_epi16(u[3], k__cospi_m24_p08); + v[6] = _mm_madd_epi16(u[2], k__cospi_p08_p24); + v[7] = _mm_madd_epi16(u[3], k__cospi_p08_p24); + v[8] = _mm_madd_epi16(u[4], k__cospi_p08_p24); + v[9] = _mm_madd_epi16(u[5], k__cospi_p08_p24); + v[10] = _mm_madd_epi16(u[4], k__cospi_p24_m08); + v[11] = _mm_madd_epi16(u[5], k__cospi_p24_m08); + v[12] = _mm_madd_epi16(u[6], k__cospi_m24_p08); + v[13] = _mm_madd_epi16(u[7], k__cospi_m24_p08); + v[14] = _mm_madd_epi16(u[6], k__cospi_p08_p24); + v[15] = _mm_madd_epi16(u[7], k__cospi_p08_p24); + + u[0] = _mm_add_epi32(v[0], v[4]); + u[1] = _mm_add_epi32(v[1], v[5]); + u[2] = _mm_add_epi32(v[2], v[6]); + u[3] = _mm_add_epi32(v[3], v[7]); + u[4] = _mm_sub_epi32(v[0], v[4]); + u[5] = _mm_sub_epi32(v[1], v[5]); + u[6] = _mm_sub_epi32(v[2], v[6]); + u[7] = _mm_sub_epi32(v[3], v[7]); + u[8] = _mm_add_epi32(v[8], v[12]); + u[9] = _mm_add_epi32(v[9], v[13]); + u[10] = _mm_add_epi32(v[10], v[14]); + u[11] = _mm_add_epi32(v[11], v[15]); + u[12] = _mm_sub_epi32(v[8], v[12]); + u[13] = _mm_sub_epi32(v[9], v[13]); + u[14] = _mm_sub_epi32(v[10], v[14]); + u[15] = _mm_sub_epi32(v[11], v[15]); + + u[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); + u[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); + u[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); + u[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); + u[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); + u[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); + u[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); + u[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); + u[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); + + v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + v[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS); + v[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS); + v[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS); + v[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS); + v[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS); + v[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS); + v[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS); + v[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS); + + s[4] = _mm_packs_epi32(v[0], v[1]); + s[5] = _mm_packs_epi32(v[2], v[3]); + s[6] = _mm_packs_epi32(v[4], v[5]); + s[7] = _mm_packs_epi32(v[6], v[7]); + + s[12] = _mm_packs_epi32(v[8], v[9]); + s[13] = _mm_packs_epi32(v[10], v[11]); + s[14] = _mm_packs_epi32(v[12], v[13]); + s[15] = _mm_packs_epi32(v[14], v[15]); + + // stage 4 + u[0] = _mm_unpacklo_epi16(s[2], s[3]); + u[1] = _mm_unpackhi_epi16(s[2], s[3]); + u[2] = _mm_unpacklo_epi16(s[6], s[7]); + u[3] = _mm_unpackhi_epi16(s[6], s[7]); + u[4] = _mm_unpacklo_epi16(s[10], s[11]); + u[5] = _mm_unpackhi_epi16(s[10], s[11]); + u[6] = _mm_unpacklo_epi16(s[14], s[15]); + u[7] = _mm_unpackhi_epi16(s[14], s[15]); + + v[0] = _mm_madd_epi16(u[0], k__cospi_m16_m16); + v[1] = _mm_madd_epi16(u[1], k__cospi_m16_m16); + v[2] = _mm_madd_epi16(u[0], k__cospi_p16_m16); + v[3] = _mm_madd_epi16(u[1], k__cospi_p16_m16); + v[4] = _mm_madd_epi16(u[2], k__cospi_p16_p16); + v[5] = _mm_madd_epi16(u[3], k__cospi_p16_p16); + v[6] = _mm_madd_epi16(u[2], k__cospi_m16_p16); + v[7] = _mm_madd_epi16(u[3], k__cospi_m16_p16); + v[8] = _mm_madd_epi16(u[4], k__cospi_p16_p16); + v[9] = _mm_madd_epi16(u[5], k__cospi_p16_p16); + v[10] = _mm_madd_epi16(u[4], k__cospi_m16_p16); + v[11] = _mm_madd_epi16(u[5], k__cospi_m16_p16); + v[12] = _mm_madd_epi16(u[6], k__cospi_m16_m16); + v[13] = _mm_madd_epi16(u[7], k__cospi_m16_m16); + v[14] = _mm_madd_epi16(u[6], k__cospi_p16_m16); + v[15] = _mm_madd_epi16(u[7], k__cospi_p16_m16); + + u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); + u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); + u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); + u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); + u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); + u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); + u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); + u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); + u[8] = _mm_add_epi32(v[8], k__DCT_CONST_ROUNDING); + u[9] = _mm_add_epi32(v[9], k__DCT_CONST_ROUNDING); + u[10] = _mm_add_epi32(v[10], k__DCT_CONST_ROUNDING); + u[11] = _mm_add_epi32(v[11], k__DCT_CONST_ROUNDING); + u[12] = _mm_add_epi32(v[12], k__DCT_CONST_ROUNDING); + u[13] = _mm_add_epi32(v[13], k__DCT_CONST_ROUNDING); + u[14] = _mm_add_epi32(v[14], k__DCT_CONST_ROUNDING); + u[15] = _mm_add_epi32(v[15], k__DCT_CONST_ROUNDING); + + v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); + v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); + v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); + v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); + v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); + v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); + v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); + v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); + v[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS); + v[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS); + v[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS); + v[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS); + v[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS); + v[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS); + v[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS); + v[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS); + + in[0] = s[0]; + in[1] = _mm_sub_epi16(kZero, s[8]); + in[2] = s[12]; + in[3] = _mm_sub_epi16(kZero, s[4]); + in[4] = _mm_packs_epi32(v[4], v[5]); + in[5] = _mm_packs_epi32(v[12], v[13]); + in[6] = _mm_packs_epi32(v[8], v[9]); + in[7] = _mm_packs_epi32(v[0], v[1]); + in[8] = _mm_packs_epi32(v[2], v[3]); + in[9] = _mm_packs_epi32(v[10], v[11]); + in[10] = _mm_packs_epi32(v[14], v[15]); + in[11] = _mm_packs_epi32(v[6], v[7]); + in[12] = s[5]; + in[13] = _mm_sub_epi16(kZero, s[13]); + in[14] = s[9]; + in[15] = _mm_sub_epi16(kZero, s[1]); +} + +static void fdct16_sse2(__m128i *in0, __m128i *in1) { + fdct16_8col(in0); + fdct16_8col(in1); + array_transpose_16x16(in0, in1); +} + +static void fadst16_sse2(__m128i *in0, __m128i *in1) { + fadst16_8col(in0); + fadst16_8col(in1); + array_transpose_16x16(in0, in1); +} + +#if CONFIG_EXT_TX +static void fidtx16_sse2(__m128i *in0, __m128i *in1) { + idtx16_8col(in0); + idtx16_8col(in1); + array_transpose_16x16(in0, in1); +} +#endif // CONFIG_EXT_TX + +void av1_fht16x16_sse2(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + __m128i in0[16], in1[16]; + + switch (tx_type) { + case DCT_DCT: + load_buffer_16x16(input, in0, in1, stride, 0, 0); + fdct16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fdct16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case ADST_DCT: + load_buffer_16x16(input, in0, in1, stride, 0, 0); + fadst16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fdct16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case DCT_ADST: + load_buffer_16x16(input, in0, in1, stride, 0, 0); + fdct16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fadst16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case ADST_ADST: + load_buffer_16x16(input, in0, in1, stride, 0, 0); + fadst16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fadst16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + load_buffer_16x16(input, in0, in1, stride, 1, 0); + fadst16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fdct16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case DCT_FLIPADST: + load_buffer_16x16(input, in0, in1, stride, 0, 1); + fdct16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fadst16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case FLIPADST_FLIPADST: + load_buffer_16x16(input, in0, in1, stride, 1, 1); + fadst16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fadst16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case ADST_FLIPADST: + load_buffer_16x16(input, in0, in1, stride, 0, 1); + fadst16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fadst16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case FLIPADST_ADST: + load_buffer_16x16(input, in0, in1, stride, 1, 0); + fadst16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fadst16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case IDTX: + load_buffer_16x16(input, in0, in1, stride, 0, 0); + fidtx16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fidtx16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case V_DCT: + load_buffer_16x16(input, in0, in1, stride, 0, 0); + fdct16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fidtx16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case H_DCT: + load_buffer_16x16(input, in0, in1, stride, 0, 0); + fidtx16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fdct16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case V_ADST: + load_buffer_16x16(input, in0, in1, stride, 0, 0); + fadst16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fidtx16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case H_ADST: + load_buffer_16x16(input, in0, in1, stride, 0, 0); + fidtx16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fadst16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case V_FLIPADST: + load_buffer_16x16(input, in0, in1, stride, 1, 0); + fadst16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fidtx16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case H_FLIPADST: + load_buffer_16x16(input, in0, in1, stride, 0, 1); + fidtx16_sse2(in0, in1); + right_shift_16x16(in0, in1); + fadst16_sse2(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } +} + +static INLINE void prepare_4x8_row_first(__m128i *in) { + in[0] = _mm_unpacklo_epi64(in[0], in[2]); + in[1] = _mm_unpacklo_epi64(in[1], in[3]); + transpose_4x4(in); + in[4] = _mm_unpacklo_epi64(in[4], in[6]); + in[5] = _mm_unpacklo_epi64(in[5], in[7]); + transpose_4x4(in + 4); +} + +// Load input into the left-hand half of in (ie, into lanes 0..3 of +// each element of in). The right hand half (lanes 4..7) should be +// treated as being filled with "don't care" values. +static INLINE void load_buffer_4x8(const int16_t *input, __m128i *in, + int stride, int flipud, int fliplr) { + const int shift = 2; + if (!flipud) { + in[0] = _mm_loadl_epi64((const __m128i *)(input + 0 * stride)); + in[1] = _mm_loadl_epi64((const __m128i *)(input + 1 * stride)); + in[2] = _mm_loadl_epi64((const __m128i *)(input + 2 * stride)); + in[3] = _mm_loadl_epi64((const __m128i *)(input + 3 * stride)); + in[4] = _mm_loadl_epi64((const __m128i *)(input + 4 * stride)); + in[5] = _mm_loadl_epi64((const __m128i *)(input + 5 * stride)); + in[6] = _mm_loadl_epi64((const __m128i *)(input + 6 * stride)); + in[7] = _mm_loadl_epi64((const __m128i *)(input + 7 * stride)); + } else { + in[0] = _mm_loadl_epi64((const __m128i *)(input + 7 * stride)); + in[1] = _mm_loadl_epi64((const __m128i *)(input + 6 * stride)); + in[2] = _mm_loadl_epi64((const __m128i *)(input + 5 * stride)); + in[3] = _mm_loadl_epi64((const __m128i *)(input + 4 * stride)); + in[4] = _mm_loadl_epi64((const __m128i *)(input + 3 * stride)); + in[5] = _mm_loadl_epi64((const __m128i *)(input + 2 * stride)); + in[6] = _mm_loadl_epi64((const __m128i *)(input + 1 * stride)); + in[7] = _mm_loadl_epi64((const __m128i *)(input + 0 * stride)); + } + + if (fliplr) { + in[0] = _mm_shufflelo_epi16(in[0], 0x1b); + in[1] = _mm_shufflelo_epi16(in[1], 0x1b); + in[2] = _mm_shufflelo_epi16(in[2], 0x1b); + in[3] = _mm_shufflelo_epi16(in[3], 0x1b); + in[4] = _mm_shufflelo_epi16(in[4], 0x1b); + in[5] = _mm_shufflelo_epi16(in[5], 0x1b); + in[6] = _mm_shufflelo_epi16(in[6], 0x1b); + in[7] = _mm_shufflelo_epi16(in[7], 0x1b); + } + + in[0] = _mm_slli_epi16(in[0], shift); + in[1] = _mm_slli_epi16(in[1], shift); + in[2] = _mm_slli_epi16(in[2], shift); + in[3] = _mm_slli_epi16(in[3], shift); + in[4] = _mm_slli_epi16(in[4], shift); + in[5] = _mm_slli_epi16(in[5], shift); + in[6] = _mm_slli_epi16(in[6], shift); + in[7] = _mm_slli_epi16(in[7], shift); + + scale_sqrt2_8x4(in); + scale_sqrt2_8x4(in + 4); + prepare_4x8_row_first(in); +} + +static INLINE void write_buffer_4x8(tran_low_t *output, __m128i *res) { + __m128i in01, in23, in45, in67, sign01, sign23, sign45, sign67; + const int shift = 1; + + // revert the 8x8 txfm's transpose + array_transpose_8x8(res, res); + + in01 = _mm_unpacklo_epi64(res[0], res[1]); + in23 = _mm_unpacklo_epi64(res[2], res[3]); + in45 = _mm_unpacklo_epi64(res[4], res[5]); + in67 = _mm_unpacklo_epi64(res[6], res[7]); + + sign01 = _mm_srai_epi16(in01, 15); + sign23 = _mm_srai_epi16(in23, 15); + sign45 = _mm_srai_epi16(in45, 15); + sign67 = _mm_srai_epi16(in67, 15); + + in01 = _mm_sub_epi16(in01, sign01); + in23 = _mm_sub_epi16(in23, sign23); + in45 = _mm_sub_epi16(in45, sign45); + in67 = _mm_sub_epi16(in67, sign67); + + in01 = _mm_srai_epi16(in01, shift); + in23 = _mm_srai_epi16(in23, shift); + in45 = _mm_srai_epi16(in45, shift); + in67 = _mm_srai_epi16(in67, shift); + + store_output(&in01, (output + 0 * 8)); + store_output(&in23, (output + 1 * 8)); + store_output(&in45, (output + 2 * 8)); + store_output(&in67, (output + 3 * 8)); +} + +void av1_fht4x8_sse2(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + __m128i in[8]; + + switch (tx_type) { + case DCT_DCT: + load_buffer_4x8(input, in, stride, 0, 0); + fdct4_sse2(in); + fdct4_sse2(in + 4); + fdct8_sse2(in); + break; + case ADST_DCT: + load_buffer_4x8(input, in, stride, 0, 0); + fdct4_sse2(in); + fdct4_sse2(in + 4); + fadst8_sse2(in); + break; + case DCT_ADST: + load_buffer_4x8(input, in, stride, 0, 0); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fdct8_sse2(in); + break; + case ADST_ADST: + load_buffer_4x8(input, in, stride, 0, 0); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fadst8_sse2(in); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + load_buffer_4x8(input, in, stride, 1, 0); + fdct4_sse2(in); + fdct4_sse2(in + 4); + fadst8_sse2(in); + break; + case DCT_FLIPADST: + load_buffer_4x8(input, in, stride, 0, 1); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fdct8_sse2(in); + break; + case FLIPADST_FLIPADST: + load_buffer_4x8(input, in, stride, 1, 1); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fadst8_sse2(in); + break; + case ADST_FLIPADST: + load_buffer_4x8(input, in, stride, 0, 1); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fadst8_sse2(in); + break; + case FLIPADST_ADST: + load_buffer_4x8(input, in, stride, 1, 0); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fadst8_sse2(in); + break; + case IDTX: + load_buffer_4x8(input, in, stride, 0, 0); + fidtx4_sse2(in); + fidtx4_sse2(in + 4); + fidtx8_sse2(in); + break; + case V_DCT: + load_buffer_4x8(input, in, stride, 0, 0); + fidtx4_sse2(in); + fidtx4_sse2(in + 4); + fdct8_sse2(in); + break; + case H_DCT: + load_buffer_4x8(input, in, stride, 0, 0); + fdct4_sse2(in); + fdct4_sse2(in + 4); + fidtx8_sse2(in); + break; + case V_ADST: + load_buffer_4x8(input, in, stride, 0, 0); + fidtx4_sse2(in); + fidtx4_sse2(in + 4); + fadst8_sse2(in); + break; + case H_ADST: + load_buffer_4x8(input, in, stride, 0, 0); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fidtx8_sse2(in); + break; + case V_FLIPADST: + load_buffer_4x8(input, in, stride, 1, 0); + fidtx4_sse2(in); + fidtx4_sse2(in + 4); + fadst8_sse2(in); + break; + case H_FLIPADST: + load_buffer_4x8(input, in, stride, 0, 1); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fidtx8_sse2(in); + break; +#endif + default: assert(0); break; + } + write_buffer_4x8(output, in); +} + +// Load input into the left-hand half of in (ie, into lanes 0..3 of +// each element of in). The right hand half (lanes 4..7) should be +// treated as being filled with "don't care" values. +// The input is split horizontally into two 4x4 +// chunks 'l' and 'r'. Then 'l' is stored in the top-left 4x4 +// block of 'in' and 'r' is stored in the bottom-left block. +// This is to allow us to reuse 4x4 transforms. +static INLINE void load_buffer_8x4(const int16_t *input, __m128i *in, + int stride, int flipud, int fliplr) { + const int shift = 2; + if (!flipud) { + in[0] = _mm_loadu_si128((const __m128i *)(input + 0 * stride)); + in[1] = _mm_loadu_si128((const __m128i *)(input + 1 * stride)); + in[2] = _mm_loadu_si128((const __m128i *)(input + 2 * stride)); + in[3] = _mm_loadu_si128((const __m128i *)(input + 3 * stride)); + } else { + in[0] = _mm_loadu_si128((const __m128i *)(input + 3 * stride)); + in[1] = _mm_loadu_si128((const __m128i *)(input + 2 * stride)); + in[2] = _mm_loadu_si128((const __m128i *)(input + 1 * stride)); + in[3] = _mm_loadu_si128((const __m128i *)(input + 0 * stride)); + } + + if (fliplr) { + in[0] = mm_reverse_epi16(in[0]); + in[1] = mm_reverse_epi16(in[1]); + in[2] = mm_reverse_epi16(in[2]); + in[3] = mm_reverse_epi16(in[3]); + } + + in[0] = _mm_slli_epi16(in[0], shift); + in[1] = _mm_slli_epi16(in[1], shift); + in[2] = _mm_slli_epi16(in[2], shift); + in[3] = _mm_slli_epi16(in[3], shift); + + scale_sqrt2_8x4(in); + + in[4] = _mm_shuffle_epi32(in[0], 0xe); + in[5] = _mm_shuffle_epi32(in[1], 0xe); + in[6] = _mm_shuffle_epi32(in[2], 0xe); + in[7] = _mm_shuffle_epi32(in[3], 0xe); +} + +static INLINE void write_buffer_8x4(tran_low_t *output, __m128i *res) { + __m128i out0, out1, out2, out3, sign0, sign1, sign2, sign3; + const int shift = 1; + sign0 = _mm_srai_epi16(res[0], 15); + sign1 = _mm_srai_epi16(res[1], 15); + sign2 = _mm_srai_epi16(res[2], 15); + sign3 = _mm_srai_epi16(res[3], 15); + + out0 = _mm_sub_epi16(res[0], sign0); + out1 = _mm_sub_epi16(res[1], sign1); + out2 = _mm_sub_epi16(res[2], sign2); + out3 = _mm_sub_epi16(res[3], sign3); + + out0 = _mm_srai_epi16(out0, shift); + out1 = _mm_srai_epi16(out1, shift); + out2 = _mm_srai_epi16(out2, shift); + out3 = _mm_srai_epi16(out3, shift); + + store_output(&out0, (output + 0 * 8)); + store_output(&out1, (output + 1 * 8)); + store_output(&out2, (output + 2 * 8)); + store_output(&out3, (output + 3 * 8)); +} + +void av1_fht8x4_sse2(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + __m128i in[8]; + + switch (tx_type) { + case DCT_DCT: + load_buffer_8x4(input, in, stride, 0, 0); + fdct4_sse2(in); + fdct4_sse2(in + 4); + fdct8_sse2(in); + break; + case ADST_DCT: + load_buffer_8x4(input, in, stride, 0, 0); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fdct8_sse2(in); + break; + case DCT_ADST: + load_buffer_8x4(input, in, stride, 0, 0); + fdct4_sse2(in); + fdct4_sse2(in + 4); + fadst8_sse2(in); + break; + case ADST_ADST: + load_buffer_8x4(input, in, stride, 0, 0); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fadst8_sse2(in); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + load_buffer_8x4(input, in, stride, 1, 0); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fdct8_sse2(in); + break; + case DCT_FLIPADST: + load_buffer_8x4(input, in, stride, 0, 1); + fdct4_sse2(in); + fdct4_sse2(in + 4); + fadst8_sse2(in); + break; + case FLIPADST_FLIPADST: + load_buffer_8x4(input, in, stride, 1, 1); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fadst8_sse2(in); + break; + case ADST_FLIPADST: + load_buffer_8x4(input, in, stride, 0, 1); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fadst8_sse2(in); + break; + case FLIPADST_ADST: + load_buffer_8x4(input, in, stride, 1, 0); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fadst8_sse2(in); + break; + case IDTX: + load_buffer_8x4(input, in, stride, 0, 0); + fidtx4_sse2(in); + fidtx4_sse2(in + 4); + fidtx8_sse2(in); + break; + case V_DCT: + load_buffer_8x4(input, in, stride, 0, 0); + fdct4_sse2(in); + fdct4_sse2(in + 4); + fidtx8_sse2(in); + break; + case H_DCT: + load_buffer_8x4(input, in, stride, 0, 0); + fidtx4_sse2(in); + fidtx4_sse2(in + 4); + fdct8_sse2(in); + break; + case V_ADST: + load_buffer_8x4(input, in, stride, 0, 0); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fidtx8_sse2(in); + break; + case H_ADST: + load_buffer_8x4(input, in, stride, 0, 0); + fidtx4_sse2(in); + fidtx4_sse2(in + 4); + fadst8_sse2(in); + break; + case V_FLIPADST: + load_buffer_8x4(input, in, stride, 1, 0); + fadst4_sse2(in); + fadst4_sse2(in + 4); + fidtx8_sse2(in); + break; + case H_FLIPADST: + load_buffer_8x4(input, in, stride, 0, 1); + fidtx4_sse2(in); + fidtx4_sse2(in + 4); + fadst8_sse2(in); + break; +#endif + default: assert(0); break; + } + write_buffer_8x4(output, in); +} + +static INLINE void load_buffer_8x16(const int16_t *input, __m128i *in, + int stride, int flipud, int fliplr) { + // Load 2 8x8 blocks + const int16_t *t = input; + const int16_t *b = input + 8 * stride; + + if (flipud) { + const int16_t *const tmp = t; + t = b; + b = tmp; + } + + load_buffer_8x8(t, in, stride, flipud, fliplr); + scale_sqrt2_8x8(in); + load_buffer_8x8(b, in + 8, stride, flipud, fliplr); + scale_sqrt2_8x8(in + 8); +} + +static INLINE void round_power_of_two_signed(__m128i *x, int n) { + const __m128i rounding = _mm_set1_epi16((1 << n) >> 1); + const __m128i sign = _mm_srai_epi16(*x, 15); + const __m128i res = _mm_add_epi16(_mm_add_epi16(*x, rounding), sign); + *x = _mm_srai_epi16(res, n); +} + +static void row_8x16_rounding(__m128i *in, int bits) { + int i; + for (i = 0; i < 16; i++) { + round_power_of_two_signed(&in[i], bits); + } +} + +void av1_fht8x16_sse2(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + __m128i in[16]; + + __m128i *const t = in; // Alias to top 8x8 sub block + __m128i *const b = in + 8; // Alias to bottom 8x8 sub block + + switch (tx_type) { + case DCT_DCT: + load_buffer_8x16(input, in, stride, 0, 0); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fdct8_sse2(t); + fdct8_sse2(b); + row_8x16_rounding(in, 2); + fdct16_8col(in); + break; + case ADST_DCT: + load_buffer_8x16(input, in, stride, 0, 0); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fdct8_sse2(t); + fdct8_sse2(b); + row_8x16_rounding(in, 2); + fadst16_8col(in); + break; + case DCT_ADST: + load_buffer_8x16(input, in, stride, 0, 0); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fadst8_sse2(t); + fadst8_sse2(b); + row_8x16_rounding(in, 2); + fdct16_8col(in); + break; + case ADST_ADST: + load_buffer_8x16(input, in, stride, 0, 0); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fadst8_sse2(t); + fadst8_sse2(b); + row_8x16_rounding(in, 2); + fadst16_8col(in); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + load_buffer_8x16(input, in, stride, 1, 0); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fdct8_sse2(t); + fdct8_sse2(b); + row_8x16_rounding(in, 2); + fadst16_8col(in); + break; + case DCT_FLIPADST: + load_buffer_8x16(input, in, stride, 0, 1); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fadst8_sse2(t); + fadst8_sse2(b); + row_8x16_rounding(in, 2); + fdct16_8col(in); + break; + case FLIPADST_FLIPADST: + load_buffer_8x16(input, in, stride, 1, 1); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fadst8_sse2(t); + fadst8_sse2(b); + row_8x16_rounding(in, 2); + fadst16_8col(in); + break; + case ADST_FLIPADST: + load_buffer_8x16(input, in, stride, 0, 1); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fadst8_sse2(t); + fadst8_sse2(b); + row_8x16_rounding(in, 2); + fadst16_8col(in); + break; + case FLIPADST_ADST: + load_buffer_8x16(input, in, stride, 1, 0); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fadst8_sse2(t); + fadst8_sse2(b); + row_8x16_rounding(in, 2); + fadst16_8col(in); + break; + case IDTX: + load_buffer_8x16(input, in, stride, 0, 0); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fidtx8_sse2(t); + fidtx8_sse2(b); + row_8x16_rounding(in, 2); + idtx16_8col(in); + break; + case V_DCT: + load_buffer_8x16(input, in, stride, 0, 0); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fidtx8_sse2(t); + fidtx8_sse2(b); + row_8x16_rounding(in, 2); + fdct16_8col(in); + break; + case H_DCT: + load_buffer_8x16(input, in, stride, 0, 0); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fdct8_sse2(t); + fdct8_sse2(b); + row_8x16_rounding(in, 2); + idtx16_8col(in); + break; + case V_ADST: + load_buffer_8x16(input, in, stride, 0, 0); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fidtx8_sse2(t); + fidtx8_sse2(b); + row_8x16_rounding(in, 2); + fadst16_8col(in); + break; + case H_ADST: + load_buffer_8x16(input, in, stride, 0, 0); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fadst8_sse2(t); + fadst8_sse2(b); + row_8x16_rounding(in, 2); + idtx16_8col(in); + break; + case V_FLIPADST: + load_buffer_8x16(input, in, stride, 1, 0); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fidtx8_sse2(t); + fidtx8_sse2(b); + row_8x16_rounding(in, 2); + fadst16_8col(in); + break; + case H_FLIPADST: + load_buffer_8x16(input, in, stride, 0, 1); + array_transpose_8x8(t, t); + array_transpose_8x8(b, b); + fadst8_sse2(t); + fadst8_sse2(b); + row_8x16_rounding(in, 2); + idtx16_8col(in); + break; +#endif + default: assert(0); break; + } + write_buffer_8x8(output, t, 8); + write_buffer_8x8(output + 64, b, 8); +} + +static INLINE void load_buffer_16x8(const int16_t *input, __m128i *in, + int stride, int flipud, int fliplr) { + // Load 2 8x8 blocks + const int16_t *l = input; + const int16_t *r = input + 8; + + if (fliplr) { + const int16_t *const tmp = l; + l = r; + r = tmp; + } + + // load first 8 columns + load_buffer_8x8(l, in, stride, flipud, fliplr); + scale_sqrt2_8x8(in); + load_buffer_8x8(r, in + 8, stride, flipud, fliplr); + scale_sqrt2_8x8(in + 8); +} + +#define col_16x8_rounding row_8x16_rounding + +void av1_fht16x8_sse2(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + __m128i in[16]; + + __m128i *const l = in; // Alias to left 8x8 sub block + __m128i *const r = in + 8; // Alias to right 8x8 sub block, which we store + // in the second half of the array + + switch (tx_type) { + case DCT_DCT: + load_buffer_16x8(input, in, stride, 0, 0); + fdct8_sse2(l); + fdct8_sse2(r); + col_16x8_rounding(in, 2); + fdct16_8col(in); + break; + case ADST_DCT: + load_buffer_16x8(input, in, stride, 0, 0); + fadst8_sse2(l); + fadst8_sse2(r); + col_16x8_rounding(in, 2); + fdct16_8col(in); + break; + case DCT_ADST: + load_buffer_16x8(input, in, stride, 0, 0); + fdct8_sse2(l); + fdct8_sse2(r); + col_16x8_rounding(in, 2); + fadst16_8col(in); + break; + case ADST_ADST: + load_buffer_16x8(input, in, stride, 0, 0); + fadst8_sse2(l); + fadst8_sse2(r); + col_16x8_rounding(in, 2); + fadst16_8col(in); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + load_buffer_16x8(input, in, stride, 1, 0); + fadst8_sse2(l); + fadst8_sse2(r); + col_16x8_rounding(in, 2); + fdct16_8col(in); + break; + case DCT_FLIPADST: + load_buffer_16x8(input, in, stride, 0, 1); + fdct8_sse2(l); + fdct8_sse2(r); + col_16x8_rounding(in, 2); + fadst16_8col(in); + break; + case FLIPADST_FLIPADST: + load_buffer_16x8(input, in, stride, 1, 1); + fadst8_sse2(l); + fadst8_sse2(r); + col_16x8_rounding(in, 2); + fadst16_8col(in); + break; + case ADST_FLIPADST: + load_buffer_16x8(input, in, stride, 0, 1); + fadst8_sse2(l); + fadst8_sse2(r); + col_16x8_rounding(in, 2); + fadst16_8col(in); + break; + case FLIPADST_ADST: + load_buffer_16x8(input, in, stride, 1, 0); + fadst8_sse2(l); + fadst8_sse2(r); + col_16x8_rounding(in, 2); + fadst16_8col(in); + break; + case IDTX: + load_buffer_16x8(input, in, stride, 0, 0); + fidtx8_sse2(l); + fidtx8_sse2(r); + col_16x8_rounding(in, 2); + idtx16_8col(in); + break; + case V_DCT: + load_buffer_16x8(input, in, stride, 0, 0); + fdct8_sse2(l); + fdct8_sse2(r); + col_16x8_rounding(in, 2); + idtx16_8col(in); + break; + case H_DCT: + load_buffer_16x8(input, in, stride, 0, 0); + fidtx8_sse2(l); + fidtx8_sse2(r); + col_16x8_rounding(in, 2); + fdct16_8col(in); + break; + case V_ADST: + load_buffer_16x8(input, in, stride, 0, 0); + fadst8_sse2(l); + fadst8_sse2(r); + col_16x8_rounding(in, 2); + idtx16_8col(in); + break; + case H_ADST: + load_buffer_16x8(input, in, stride, 0, 0); + fidtx8_sse2(l); + fidtx8_sse2(r); + col_16x8_rounding(in, 2); + fadst16_8col(in); + break; + case V_FLIPADST: + load_buffer_16x8(input, in, stride, 1, 0); + fadst8_sse2(l); + fadst8_sse2(r); + col_16x8_rounding(in, 2); + idtx16_8col(in); + break; + case H_FLIPADST: + load_buffer_16x8(input, in, stride, 0, 1); + fidtx8_sse2(l); + fidtx8_sse2(r); + col_16x8_rounding(in, 2); + fadst16_8col(in); + break; +#endif + default: assert(0); break; + } + array_transpose_8x8(l, l); + array_transpose_8x8(r, r); + write_buffer_8x8(output, l, 16); + write_buffer_8x8(output + 8, r, 16); +} + +// Note: The 16-column 32-element transforms expect their input to be +// split up into a 2x2 grid of 8x16 blocks +static INLINE void fdct32_16col(__m128i *tl, __m128i *tr, __m128i *bl, + __m128i *br) { + fdct32_8col(tl, bl); + fdct32_8col(tr, br); + array_transpose_16x16(tl, tr); + array_transpose_16x16(bl, br); +} + +#if CONFIG_EXT_TX +static INLINE void fidtx32_16col(__m128i *tl, __m128i *tr, __m128i *bl, + __m128i *br) { + int i; + for (i = 0; i < 16; ++i) { + tl[i] = _mm_slli_epi16(tl[i], 2); + tr[i] = _mm_slli_epi16(tr[i], 2); + bl[i] = _mm_slli_epi16(bl[i], 2); + br[i] = _mm_slli_epi16(br[i], 2); + } + array_transpose_16x16(tl, tr); + array_transpose_16x16(bl, br); +} +#endif + +static INLINE void load_buffer_16x32(const int16_t *input, __m128i *intl, + __m128i *intr, __m128i *inbl, + __m128i *inbr, int stride, int flipud, + int fliplr) { + int i; + if (flipud) { + input = input + 31 * stride; + stride = -stride; + } + + for (i = 0; i < 16; ++i) { + intl[i] = _mm_slli_epi16( + _mm_load_si128((const __m128i *)(input + i * stride + 0)), 2); + intr[i] = _mm_slli_epi16( + _mm_load_si128((const __m128i *)(input + i * stride + 8)), 2); + inbl[i] = _mm_slli_epi16( + _mm_load_si128((const __m128i *)(input + (i + 16) * stride + 0)), 2); + inbr[i] = _mm_slli_epi16( + _mm_load_si128((const __m128i *)(input + (i + 16) * stride + 8)), 2); + } + + if (fliplr) { + __m128i tmp; + for (i = 0; i < 16; ++i) { + tmp = intl[i]; + intl[i] = mm_reverse_epi16(intr[i]); + intr[i] = mm_reverse_epi16(tmp); + tmp = inbl[i]; + inbl[i] = mm_reverse_epi16(inbr[i]); + inbr[i] = mm_reverse_epi16(tmp); + } + } + + scale_sqrt2_8x16(intl); + scale_sqrt2_8x16(intr); + scale_sqrt2_8x16(inbl); + scale_sqrt2_8x16(inbr); +} + +static INLINE void write_buffer_16x32(tran_low_t *output, __m128i *restl, + __m128i *restr, __m128i *resbl, + __m128i *resbr) { + int i; + for (i = 0; i < 16; ++i) { + store_output(&restl[i], output + i * 16 + 0); + store_output(&restr[i], output + i * 16 + 8); + store_output(&resbl[i], output + (i + 16) * 16 + 0); + store_output(&resbr[i], output + (i + 16) * 16 + 8); + } +} + +static INLINE void round_signed_8x8(__m128i *in, const int bit) { + const __m128i rounding = _mm_set1_epi16((1 << bit) >> 1); + __m128i sign0 = _mm_srai_epi16(in[0], 15); + __m128i sign1 = _mm_srai_epi16(in[1], 15); + __m128i sign2 = _mm_srai_epi16(in[2], 15); + __m128i sign3 = _mm_srai_epi16(in[3], 15); + __m128i sign4 = _mm_srai_epi16(in[4], 15); + __m128i sign5 = _mm_srai_epi16(in[5], 15); + __m128i sign6 = _mm_srai_epi16(in[6], 15); + __m128i sign7 = _mm_srai_epi16(in[7], 15); + + in[0] = _mm_add_epi16(_mm_add_epi16(in[0], rounding), sign0); + in[1] = _mm_add_epi16(_mm_add_epi16(in[1], rounding), sign1); + in[2] = _mm_add_epi16(_mm_add_epi16(in[2], rounding), sign2); + in[3] = _mm_add_epi16(_mm_add_epi16(in[3], rounding), sign3); + in[4] = _mm_add_epi16(_mm_add_epi16(in[4], rounding), sign4); + in[5] = _mm_add_epi16(_mm_add_epi16(in[5], rounding), sign5); + in[6] = _mm_add_epi16(_mm_add_epi16(in[6], rounding), sign6); + in[7] = _mm_add_epi16(_mm_add_epi16(in[7], rounding), sign7); + + in[0] = _mm_srai_epi16(in[0], bit); + in[1] = _mm_srai_epi16(in[1], bit); + in[2] = _mm_srai_epi16(in[2], bit); + in[3] = _mm_srai_epi16(in[3], bit); + in[4] = _mm_srai_epi16(in[4], bit); + in[5] = _mm_srai_epi16(in[5], bit); + in[6] = _mm_srai_epi16(in[6], bit); + in[7] = _mm_srai_epi16(in[7], bit); +} + +static INLINE void round_signed_16x16(__m128i *in0, __m128i *in1) { + const int bit = 4; + round_signed_8x8(in0, bit); + round_signed_8x8(in0 + 8, bit); + round_signed_8x8(in1, bit); + round_signed_8x8(in1 + 8, bit); +} + +// Note: +// suffix "t" indicates the transpose operation comes first +static void fdct16t_sse2(__m128i *in0, __m128i *in1) { + array_transpose_16x16(in0, in1); + fdct16_8col(in0); + fdct16_8col(in1); +} + +static void fadst16t_sse2(__m128i *in0, __m128i *in1) { + array_transpose_16x16(in0, in1); + fadst16_8col(in0); + fadst16_8col(in1); +} + +static INLINE void fdct32t_16col(__m128i *tl, __m128i *tr, __m128i *bl, + __m128i *br) { + array_transpose_16x16(tl, tr); + array_transpose_16x16(bl, br); + fdct32_8col(tl, bl); + fdct32_8col(tr, br); +} + +typedef enum transpose_indicator_ { + transpose, + no_transpose, +} transpose_indicator; + +static INLINE void fhalfright32_16col(__m128i *tl, __m128i *tr, __m128i *bl, + __m128i *br, transpose_indicator t) { + __m128i tmpl[16], tmpr[16]; + int i; + + // Copy the bottom half of the input to temporary storage + for (i = 0; i < 16; ++i) { + tmpl[i] = bl[i]; + tmpr[i] = br[i]; + } + + // Generate the bottom half of the output + for (i = 0; i < 16; ++i) { + bl[i] = _mm_slli_epi16(tl[i], 2); + br[i] = _mm_slli_epi16(tr[i], 2); + } + array_transpose_16x16(bl, br); + + // Copy the temporary storage back to the top half of the input + for (i = 0; i < 16; ++i) { + tl[i] = tmpl[i]; + tr[i] = tmpr[i]; + } + + // Generate the top half of the output + scale_sqrt2_8x16(tl); + scale_sqrt2_8x16(tr); + if (t == transpose) + fdct16t_sse2(tl, tr); + else + fdct16_sse2(tl, tr); +} + +// Note on data layout, for both this and the 32x16 transforms: +// So that we can reuse the 16-element transforms easily, +// we want to split the input into 8x16 blocks. +// For 16x32, this means the input is a 2x2 grid of such blocks. +// For 32x16, it means the input is a 4x1 grid. +void av1_fht16x32_sse2(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + __m128i intl[16], intr[16], inbl[16], inbr[16]; + + switch (tx_type) { + case DCT_DCT: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 0, 0); + fdct16t_sse2(intl, intr); + fdct16t_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fdct32t_16col(intl, intr, inbl, inbr); + break; + case ADST_DCT: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 0, 0); + fdct16t_sse2(intl, intr); + fdct16t_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fhalfright32_16col(intl, intr, inbl, inbr, transpose); + break; + case DCT_ADST: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 0, 0); + fadst16t_sse2(intl, intr); + fadst16t_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fdct32t_16col(intl, intr, inbl, inbr); + break; + case ADST_ADST: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 0, 0); + fadst16t_sse2(intl, intr); + fadst16t_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fhalfright32_16col(intl, intr, inbl, inbr, transpose); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 1, 0); + fdct16t_sse2(intl, intr); + fdct16t_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fhalfright32_16col(intl, intr, inbl, inbr, transpose); + break; + case DCT_FLIPADST: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 0, 1); + fadst16t_sse2(intl, intr); + fadst16t_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fdct32t_16col(intl, intr, inbl, inbr); + break; + case FLIPADST_FLIPADST: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 1, 1); + fadst16t_sse2(intl, intr); + fadst16t_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fhalfright32_16col(intl, intr, inbl, inbr, transpose); + break; + case ADST_FLIPADST: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 0, 1); + fadst16t_sse2(intl, intr); + fadst16t_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fhalfright32_16col(intl, intr, inbl, inbr, transpose); + break; + case FLIPADST_ADST: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 1, 0); + fadst16t_sse2(intl, intr); + fadst16t_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fhalfright32_16col(intl, intr, inbl, inbr, transpose); + break; + case IDTX: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 0, 0); + fidtx16_sse2(intl, intr); + fidtx16_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fidtx32_16col(intl, intr, inbl, inbr); + break; + case V_DCT: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 0, 0); + fidtx16_sse2(intl, intr); + fidtx16_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fdct32t_16col(intl, intr, inbl, inbr); + break; + case H_DCT: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 0, 0); + fdct16t_sse2(intl, intr); + fdct16t_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fidtx32_16col(intl, intr, inbl, inbr); + break; + case V_ADST: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 0, 0); + fidtx16_sse2(intl, intr); + fidtx16_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fhalfright32_16col(intl, intr, inbl, inbr, transpose); + break; + case H_ADST: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 0, 0); + fadst16t_sse2(intl, intr); + fadst16t_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fidtx32_16col(intl, intr, inbl, inbr); + break; + case V_FLIPADST: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 1, 0); + fidtx16_sse2(intl, intr); + fidtx16_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fhalfright32_16col(intl, intr, inbl, inbr, transpose); + break; + case H_FLIPADST: + load_buffer_16x32(input, intl, intr, inbl, inbr, stride, 0, 1); + fadst16t_sse2(intl, intr); + fadst16t_sse2(inbl, inbr); + round_signed_16x16(intl, intr); + round_signed_16x16(inbl, inbr); + fidtx32_16col(intl, intr, inbl, inbr); + break; +#endif + default: assert(0); break; + } + write_buffer_16x32(output, intl, intr, inbl, inbr); +} + +static INLINE void load_buffer_32x16(const int16_t *input, __m128i *in0, + __m128i *in1, __m128i *in2, __m128i *in3, + int stride, int flipud, int fliplr) { + int i; + if (flipud) { + input += 15 * stride; + stride = -stride; + } + + for (i = 0; i < 16; ++i) { + in0[i] = _mm_slli_epi16( + _mm_load_si128((const __m128i *)(input + i * stride + 0)), 2); + in1[i] = _mm_slli_epi16( + _mm_load_si128((const __m128i *)(input + i * stride + 8)), 2); + in2[i] = _mm_slli_epi16( + _mm_load_si128((const __m128i *)(input + i * stride + 16)), 2); + in3[i] = _mm_slli_epi16( + _mm_load_si128((const __m128i *)(input + i * stride + 24)), 2); + } + + if (fliplr) { + for (i = 0; i < 16; ++i) { + __m128i tmp1 = in0[i]; + __m128i tmp2 = in1[i]; + in0[i] = mm_reverse_epi16(in3[i]); + in1[i] = mm_reverse_epi16(in2[i]); + in2[i] = mm_reverse_epi16(tmp2); + in3[i] = mm_reverse_epi16(tmp1); + } + } + + scale_sqrt2_8x16(in0); + scale_sqrt2_8x16(in1); + scale_sqrt2_8x16(in2); + scale_sqrt2_8x16(in3); +} + +static INLINE void write_buffer_32x16(tran_low_t *output, __m128i *res0, + __m128i *res1, __m128i *res2, + __m128i *res3) { + int i; + for (i = 0; i < 16; ++i) { + store_output(&res0[i], output + i * 32 + 0); + store_output(&res1[i], output + i * 32 + 8); + store_output(&res2[i], output + i * 32 + 16); + store_output(&res3[i], output + i * 32 + 24); + } +} + +void av1_fht32x16_sse2(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + __m128i in0[16], in1[16], in2[16], in3[16]; + + load_buffer_32x16(input, in0, in1, in2, in3, stride, 0, 0); + switch (tx_type) { + case DCT_DCT: + fdct16_sse2(in0, in1); + fdct16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fdct32_16col(in0, in1, in2, in3); + break; + case ADST_DCT: + fadst16_sse2(in0, in1); + fadst16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fdct32_16col(in0, in1, in2, in3); + break; + case DCT_ADST: + fdct16_sse2(in0, in1); + fdct16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fhalfright32_16col(in0, in1, in2, in3, no_transpose); + break; + case ADST_ADST: + fadst16_sse2(in0, in1); + fadst16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fhalfright32_16col(in0, in1, in2, in3, no_transpose); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + load_buffer_32x16(input, in0, in1, in2, in3, stride, 1, 0); + fadst16_sse2(in0, in1); + fadst16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fdct32_16col(in0, in1, in2, in3); + break; + case DCT_FLIPADST: + load_buffer_32x16(input, in0, in1, in2, in3, stride, 0, 1); + fdct16_sse2(in0, in1); + fdct16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fhalfright32_16col(in0, in1, in2, in3, no_transpose); + break; + case FLIPADST_FLIPADST: + load_buffer_32x16(input, in0, in1, in2, in3, stride, 1, 1); + fadst16_sse2(in0, in1); + fadst16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fhalfright32_16col(in0, in1, in2, in3, no_transpose); + break; + case ADST_FLIPADST: + load_buffer_32x16(input, in0, in1, in2, in3, stride, 0, 1); + fadst16_sse2(in0, in1); + fadst16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fhalfright32_16col(in0, in1, in2, in3, no_transpose); + break; + case FLIPADST_ADST: + load_buffer_32x16(input, in0, in1, in2, in3, stride, 1, 0); + fadst16_sse2(in0, in1); + fadst16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fhalfright32_16col(in0, in1, in2, in3, no_transpose); + break; + case IDTX: + load_buffer_32x16(input, in0, in1, in2, in3, stride, 0, 0); + fidtx16_sse2(in0, in1); + fidtx16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fidtx32_16col(in0, in1, in2, in3); + break; + case V_DCT: + load_buffer_32x16(input, in0, in1, in2, in3, stride, 0, 0); + fdct16_sse2(in0, in1); + fdct16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fidtx32_16col(in0, in1, in2, in3); + break; + case H_DCT: + load_buffer_32x16(input, in0, in1, in2, in3, stride, 0, 0); + fidtx16_sse2(in0, in1); + fidtx16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fdct32_16col(in0, in1, in2, in3); + break; + case V_ADST: + load_buffer_32x16(input, in0, in1, in2, in3, stride, 0, 0); + fadst16_sse2(in0, in1); + fadst16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fidtx32_16col(in0, in1, in2, in3); + break; + case H_ADST: + load_buffer_32x16(input, in0, in1, in2, in3, stride, 0, 0); + fidtx16_sse2(in0, in1); + fidtx16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fhalfright32_16col(in0, in1, in2, in3, no_transpose); + break; + case V_FLIPADST: + load_buffer_32x16(input, in0, in1, in2, in3, stride, 1, 0); + fadst16_sse2(in0, in1); + fadst16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fidtx32_16col(in0, in1, in2, in3); + break; + case H_FLIPADST: + load_buffer_32x16(input, in0, in1, in2, in3, stride, 0, 1); + fidtx16_sse2(in0, in1); + fidtx16_sse2(in2, in3); + round_signed_16x16(in0, in1); + round_signed_16x16(in2, in3); + fhalfright32_16col(in0, in1, in2, in3, no_transpose); + break; +#endif + default: assert(0); break; + } + write_buffer_32x16(output, in0, in1, in2, in3); +} + +// Note: +// 32x32 hybrid fwd txfm +// 4x2 grids of 8x16 block. Each block is represented by __m128i in[16] +static INLINE void load_buffer_32x32(const int16_t *input, + __m128i *in0 /*in0[32]*/, + __m128i *in1 /*in1[32]*/, + __m128i *in2 /*in2[32]*/, + __m128i *in3 /*in3[32]*/, int stride, + int flipud, int fliplr) { + if (flipud) { + input += 31 * stride; + stride = -stride; + } + + int i; + for (i = 0; i < 32; ++i) { + in0[i] = _mm_slli_epi16( + _mm_load_si128((const __m128i *)(input + i * stride + 0)), 2); + in1[i] = _mm_slli_epi16( + _mm_load_si128((const __m128i *)(input + i * stride + 8)), 2); + in2[i] = _mm_slli_epi16( + _mm_load_si128((const __m128i *)(input + i * stride + 16)), 2); + in3[i] = _mm_slli_epi16( + _mm_load_si128((const __m128i *)(input + i * stride + 24)), 2); + } + + if (fliplr) { + for (i = 0; i < 32; ++i) { + __m128i tmp1 = in0[i]; + __m128i tmp2 = in1[i]; + in0[i] = mm_reverse_epi16(in3[i]); + in1[i] = mm_reverse_epi16(in2[i]); + in2[i] = mm_reverse_epi16(tmp2); + in3[i] = mm_reverse_epi16(tmp1); + } + } +} + +static INLINE void swap_16x16(__m128i *b0l /*b0l[16]*/, + __m128i *b0r /*b0r[16]*/, + __m128i *b1l /*b1l[16]*/, + __m128i *b1r /*b1r[16]*/) { + int i; + for (i = 0; i < 16; ++i) { + __m128i tmp0 = b1l[i]; + __m128i tmp1 = b1r[i]; + b1l[i] = b0l[i]; + b1r[i] = b0r[i]; + b0l[i] = tmp0; + b0r[i] = tmp1; + } +} + +static INLINE void fdct32(__m128i *in0, __m128i *in1, __m128i *in2, + __m128i *in3) { + fdct32_8col(in0, &in0[16]); + fdct32_8col(in1, &in1[16]); + fdct32_8col(in2, &in2[16]); + fdct32_8col(in3, &in3[16]); + + array_transpose_16x16(in0, in1); + array_transpose_16x16(&in0[16], &in1[16]); + array_transpose_16x16(in2, in3); + array_transpose_16x16(&in2[16], &in3[16]); + + swap_16x16(&in0[16], &in1[16], in2, in3); +} + +static INLINE void fhalfright32(__m128i *in0, __m128i *in1, __m128i *in2, + __m128i *in3) { + fhalfright32_16col(in0, in1, &in0[16], &in1[16], no_transpose); + fhalfright32_16col(in2, in3, &in2[16], &in3[16], no_transpose); + swap_16x16(&in0[16], &in1[16], in2, in3); +} + +#if CONFIG_EXT_TX +static INLINE void fidtx32(__m128i *in0, __m128i *in1, __m128i *in2, + __m128i *in3) { + fidtx32_16col(in0, in1, &in0[16], &in1[16]); + fidtx32_16col(in2, in3, &in2[16], &in3[16]); + swap_16x16(&in0[16], &in1[16], in2, in3); +} +#endif + +static INLINE void round_signed_32x32(__m128i *in0, __m128i *in1, __m128i *in2, + __m128i *in3) { + round_signed_16x16(in0, in1); + round_signed_16x16(&in0[16], &in1[16]); + round_signed_16x16(in2, in3); + round_signed_16x16(&in2[16], &in3[16]); +} + +static INLINE void write_buffer_32x32(__m128i *in0, __m128i *in1, __m128i *in2, + __m128i *in3, tran_low_t *output) { + int i; + for (i = 0; i < 32; ++i) { + store_output(&in0[i], output + i * 32 + 0); + store_output(&in1[i], output + i * 32 + 8); + store_output(&in2[i], output + i * 32 + 16); + store_output(&in3[i], output + i * 32 + 24); + } +} + +void av1_fht32x32_sse2(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + __m128i in0[32], in1[32], in2[32], in3[32]; + + load_buffer_32x32(input, in0, in1, in2, in3, stride, 0, 0); + switch (tx_type) { + case DCT_DCT: + fdct32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fdct32(in0, in1, in2, in3); + break; + case ADST_DCT: + fhalfright32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fdct32(in0, in1, in2, in3); + break; + case DCT_ADST: + fdct32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fhalfright32(in0, in1, in2, in3); + break; + case ADST_ADST: + fhalfright32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fhalfright32(in0, in1, in2, in3); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + load_buffer_32x32(input, in0, in1, in2, in3, stride, 1, 0); + fhalfright32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fdct32(in0, in1, in2, in3); + break; + case DCT_FLIPADST: + load_buffer_32x32(input, in0, in1, in2, in3, stride, 0, 1); + fdct32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fhalfright32(in0, in1, in2, in3); + break; + case FLIPADST_FLIPADST: + load_buffer_32x32(input, in0, in1, in2, in3, stride, 1, 1); + fhalfright32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fhalfright32(in0, in1, in2, in3); + break; + case ADST_FLIPADST: + load_buffer_32x32(input, in0, in1, in2, in3, stride, 0, 1); + fhalfright32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fhalfright32(in0, in1, in2, in3); + break; + case FLIPADST_ADST: + load_buffer_32x32(input, in0, in1, in2, in3, stride, 1, 0); + fhalfright32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fhalfright32(in0, in1, in2, in3); + break; + case IDTX: + fidtx32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fidtx32(in0, in1, in2, in3); + break; + case V_DCT: + fdct32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fidtx32(in0, in1, in2, in3); + break; + case H_DCT: + fidtx32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fdct32(in0, in1, in2, in3); + break; + case V_ADST: + fhalfright32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fidtx32(in0, in1, in2, in3); + break; + case H_ADST: + fidtx32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fhalfright32(in0, in1, in2, in3); + break; + case V_FLIPADST: + load_buffer_32x32(input, in0, in1, in2, in3, stride, 1, 0); + fhalfright32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fidtx32(in0, in1, in2, in3); + break; + case H_FLIPADST: + load_buffer_32x32(input, in0, in1, in2, in3, stride, 0, 1); + fidtx32(in0, in1, in2, in3); + round_signed_32x32(in0, in1, in2, in3); + fhalfright32(in0, in1, in2, in3); + break; +#endif + default: assert(0); + } + write_buffer_32x32(in0, in1, in2, in3, output); +} diff --git a/third_party/aom/av1/encoder/x86/dct_sse2.asm b/third_party/aom/av1/encoder/x86/dct_sse2.asm new file mode 100644 index 0000000000..a99db3d6ed --- /dev/null +++ b/third_party/aom/av1/encoder/x86/dct_sse2.asm @@ -0,0 +1,87 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +%define private_prefix av1 + +%include "third_party/x86inc/x86inc.asm" + +SECTION .text + +%macro TRANSFORM_COLS 0 + paddw m0, m1 + movq m4, m0 + psubw m3, m2 + psubw m4, m3 + psraw m4, 1 + movq m5, m4 + psubw m5, m1 ;b1 + psubw m4, m2 ;c1 + psubw m0, m4 + paddw m3, m5 + ; m0 a0 + SWAP 1, 4 ; m1 c1 + SWAP 2, 3 ; m2 d1 + SWAP 3, 5 ; m3 b1 +%endmacro + +%macro TRANSPOSE_4X4 0 + ; 00 01 02 03 + ; 10 11 12 13 + ; 20 21 22 23 + ; 30 31 32 33 + punpcklwd m0, m1 ; 00 10 01 11 02 12 03 13 + punpcklwd m2, m3 ; 20 30 21 31 22 32 23 33 + mova m1, m0 + punpckldq m0, m2 ; 00 10 20 30 01 11 21 31 + punpckhdq m1, m2 ; 02 12 22 32 03 13 23 33 +%endmacro + +INIT_XMM sse2 +cglobal fwht4x4, 3, 4, 8, input, output, stride + lea r3q, [inputq + strideq*4] + movq m0, [inputq] ;a1 + movq m1, [inputq + strideq*2] ;b1 + movq m2, [r3q] ;c1 + movq m3, [r3q + strideq*2] ;d1 + + TRANSFORM_COLS + TRANSPOSE_4X4 + SWAP 1, 2 + psrldq m1, m0, 8 + psrldq m3, m2, 8 + TRANSFORM_COLS + TRANSPOSE_4X4 + + psllw m0, 2 + psllw m1, 2 + +%if CONFIG_HIGHBITDEPTH + ; sign extension + mova m2, m0 + mova m3, m1 + punpcklwd m0, m0 + punpcklwd m1, m1 + punpckhwd m2, m2 + punpckhwd m3, m3 + psrad m0, 16 + psrad m1, 16 + psrad m2, 16 + psrad m3, 16 + mova [outputq], m0 + mova [outputq + 16], m2 + mova [outputq + 32], m1 + mova [outputq + 48], m3 +%else + mova [outputq], m0 + mova [outputq + 16], m1 +%endif + + RET diff --git a/third_party/aom/av1/encoder/x86/dct_ssse3.c b/third_party/aom/av1/encoder/x86/dct_ssse3.c new file mode 100644 index 0000000000..717a99af8f --- /dev/null +++ b/third_party/aom/av1/encoder/x86/dct_ssse3.c @@ -0,0 +1,469 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#if defined(_MSC_VER) && _MSC_VER <= 1500 +// Need to include math.h before calling tmmintrin.h/intrin.h +// in certain versions of MSVS. +#include +#endif +#include // SSSE3 + +#include "./av1_rtcd.h" +#include "aom_dsp/x86/inv_txfm_sse2.h" +#include "aom_dsp/x86/txfm_common_sse2.h" + +void av1_fdct8x8_quant_ssse3( + const int16_t *input, int stride, int16_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, + int16_t *qcoeff_ptr, int16_t *dqcoeff_ptr, const int16_t *dequant_ptr, + uint16_t *eob_ptr, const int16_t *scan_ptr, const int16_t *iscan_ptr) { + __m128i zero; + int pass; + // Constants + // When we use them, in one case, they are all the same. In all others + // it's a pair of them that we need to repeat four times. This is done + // by constructing the 32 bit constant corresponding to that pair. + const __m128i k__dual_p16_p16 = dual_set_epi16(23170, 23170); + const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); + const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64); + const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); + const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64); + const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); + const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); + // Load input + __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride)); + __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride)); + __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride)); + __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride)); + __m128i in4 = _mm_load_si128((const __m128i *)(input + 4 * stride)); + __m128i in5 = _mm_load_si128((const __m128i *)(input + 5 * stride)); + __m128i in6 = _mm_load_si128((const __m128i *)(input + 6 * stride)); + __m128i in7 = _mm_load_si128((const __m128i *)(input + 7 * stride)); + __m128i *in[8]; + int index = 0; + + (void)scan_ptr; + (void)zbin_ptr; + (void)quant_shift_ptr; + (void)coeff_ptr; + + // Pre-condition input (shift by two) + in0 = _mm_slli_epi16(in0, 2); + in1 = _mm_slli_epi16(in1, 2); + in2 = _mm_slli_epi16(in2, 2); + in3 = _mm_slli_epi16(in3, 2); + in4 = _mm_slli_epi16(in4, 2); + in5 = _mm_slli_epi16(in5, 2); + in6 = _mm_slli_epi16(in6, 2); + in7 = _mm_slli_epi16(in7, 2); + + in[0] = &in0; + in[1] = &in1; + in[2] = &in2; + in[3] = &in3; + in[4] = &in4; + in[5] = &in5; + in[6] = &in6; + in[7] = &in7; + + // We do two passes, first the columns, then the rows. The results of the + // first pass are transposed so that the same column code can be reused. The + // results of the second pass are also transposed so that the rows (processed + // as columns) are put back in row positions. + for (pass = 0; pass < 2; pass++) { + // To store results of each pass before the transpose. + __m128i res0, res1, res2, res3, res4, res5, res6, res7; + // Add/subtract + const __m128i q0 = _mm_add_epi16(in0, in7); + const __m128i q1 = _mm_add_epi16(in1, in6); + const __m128i q2 = _mm_add_epi16(in2, in5); + const __m128i q3 = _mm_add_epi16(in3, in4); + const __m128i q4 = _mm_sub_epi16(in3, in4); + const __m128i q5 = _mm_sub_epi16(in2, in5); + const __m128i q6 = _mm_sub_epi16(in1, in6); + const __m128i q7 = _mm_sub_epi16(in0, in7); + // Work on first four results + { + // Add/subtract + const __m128i r0 = _mm_add_epi16(q0, q3); + const __m128i r1 = _mm_add_epi16(q1, q2); + const __m128i r2 = _mm_sub_epi16(q1, q2); + const __m128i r3 = _mm_sub_epi16(q0, q3); + // Interleave to do the multiply by constants which gets us into 32bits + const __m128i t0 = _mm_unpacklo_epi16(r0, r1); + const __m128i t1 = _mm_unpackhi_epi16(r0, r1); + const __m128i t2 = _mm_unpacklo_epi16(r2, r3); + const __m128i t3 = _mm_unpackhi_epi16(r2, r3); + + const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16); + const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16); + const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16); + const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16); + + const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08); + const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08); + const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24); + const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24); + // dct_const_round_shift + + const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); + const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); + const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); + const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); + + const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); + const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); + const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); + const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); + + const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + + const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS); + const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS); + const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS); + const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS); + // Combine + + res0 = _mm_packs_epi32(w0, w1); + res4 = _mm_packs_epi32(w2, w3); + res2 = _mm_packs_epi32(w4, w5); + res6 = _mm_packs_epi32(w6, w7); + } + // Work on next four results + { + // Interleave to do the multiply by constants which gets us into 32bits + const __m128i d0 = _mm_sub_epi16(q6, q5); + const __m128i d1 = _mm_add_epi16(q6, q5); + const __m128i r0 = _mm_mulhrs_epi16(d0, k__dual_p16_p16); + const __m128i r1 = _mm_mulhrs_epi16(d1, k__dual_p16_p16); + + // Add/subtract + const __m128i x0 = _mm_add_epi16(q4, r0); + const __m128i x1 = _mm_sub_epi16(q4, r0); + const __m128i x2 = _mm_sub_epi16(q7, r1); + const __m128i x3 = _mm_add_epi16(q7, r1); + // Interleave to do the multiply by constants which gets us into 32bits + const __m128i t0 = _mm_unpacklo_epi16(x0, x3); + const __m128i t1 = _mm_unpackhi_epi16(x0, x3); + const __m128i t2 = _mm_unpacklo_epi16(x1, x2); + const __m128i t3 = _mm_unpackhi_epi16(x1, x2); + const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04); + const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04); + const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28); + const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28); + const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20); + const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20); + const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12); + const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12); + // dct_const_round_shift + const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); + const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); + const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); + const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); + const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); + const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); + const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); + const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); + const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); + const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); + const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); + const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); + const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS); + const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS); + const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS); + const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS); + // Combine + res1 = _mm_packs_epi32(w0, w1); + res7 = _mm_packs_epi32(w2, w3); + res5 = _mm_packs_epi32(w4, w5); + res3 = _mm_packs_epi32(w6, w7); + } + // Transpose the 8x8. + { + // 00 01 02 03 04 05 06 07 + // 10 11 12 13 14 15 16 17 + // 20 21 22 23 24 25 26 27 + // 30 31 32 33 34 35 36 37 + // 40 41 42 43 44 45 46 47 + // 50 51 52 53 54 55 56 57 + // 60 61 62 63 64 65 66 67 + // 70 71 72 73 74 75 76 77 + const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1); + const __m128i tr0_1 = _mm_unpacklo_epi16(res2, res3); + const __m128i tr0_2 = _mm_unpackhi_epi16(res0, res1); + const __m128i tr0_3 = _mm_unpackhi_epi16(res2, res3); + const __m128i tr0_4 = _mm_unpacklo_epi16(res4, res5); + const __m128i tr0_5 = _mm_unpacklo_epi16(res6, res7); + const __m128i tr0_6 = _mm_unpackhi_epi16(res4, res5); + const __m128i tr0_7 = _mm_unpackhi_epi16(res6, res7); + // 00 10 01 11 02 12 03 13 + // 20 30 21 31 22 32 23 33 + // 04 14 05 15 06 16 07 17 + // 24 34 25 35 26 36 27 37 + // 40 50 41 51 42 52 43 53 + // 60 70 61 71 62 72 63 73 + // 54 54 55 55 56 56 57 57 + // 64 74 65 75 66 76 67 77 + const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); + const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3); + const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); + const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3); + const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); + const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); + const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); + const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); + // 00 10 20 30 01 11 21 31 + // 40 50 60 70 41 51 61 71 + // 02 12 22 32 03 13 23 33 + // 42 52 62 72 43 53 63 73 + // 04 14 24 34 05 15 21 36 + // 44 54 64 74 45 55 61 76 + // 06 16 26 36 07 17 27 37 + // 46 56 66 76 47 57 67 77 + in0 = _mm_unpacklo_epi64(tr1_0, tr1_4); + in1 = _mm_unpackhi_epi64(tr1_0, tr1_4); + in2 = _mm_unpacklo_epi64(tr1_2, tr1_6); + in3 = _mm_unpackhi_epi64(tr1_2, tr1_6); + in4 = _mm_unpacklo_epi64(tr1_1, tr1_5); + in5 = _mm_unpackhi_epi64(tr1_1, tr1_5); + in6 = _mm_unpacklo_epi64(tr1_3, tr1_7); + in7 = _mm_unpackhi_epi64(tr1_3, tr1_7); + // 00 10 20 30 40 50 60 70 + // 01 11 21 31 41 51 61 71 + // 02 12 22 32 42 52 62 72 + // 03 13 23 33 43 53 63 73 + // 04 14 24 34 44 54 64 74 + // 05 15 25 35 45 55 65 75 + // 06 16 26 36 46 56 66 76 + // 07 17 27 37 47 57 67 77 + } + } + // Post-condition output and store it + { + // Post-condition (division by two) + // division of two 16 bits signed numbers using shifts + // n / 2 = (n - (n >> 15)) >> 1 + const __m128i sign_in0 = _mm_srai_epi16(in0, 15); + const __m128i sign_in1 = _mm_srai_epi16(in1, 15); + const __m128i sign_in2 = _mm_srai_epi16(in2, 15); + const __m128i sign_in3 = _mm_srai_epi16(in3, 15); + const __m128i sign_in4 = _mm_srai_epi16(in4, 15); + const __m128i sign_in5 = _mm_srai_epi16(in5, 15); + const __m128i sign_in6 = _mm_srai_epi16(in6, 15); + const __m128i sign_in7 = _mm_srai_epi16(in7, 15); + in0 = _mm_sub_epi16(in0, sign_in0); + in1 = _mm_sub_epi16(in1, sign_in1); + in2 = _mm_sub_epi16(in2, sign_in2); + in3 = _mm_sub_epi16(in3, sign_in3); + in4 = _mm_sub_epi16(in4, sign_in4); + in5 = _mm_sub_epi16(in5, sign_in5); + in6 = _mm_sub_epi16(in6, sign_in6); + in7 = _mm_sub_epi16(in7, sign_in7); + in0 = _mm_srai_epi16(in0, 1); + in1 = _mm_srai_epi16(in1, 1); + in2 = _mm_srai_epi16(in2, 1); + in3 = _mm_srai_epi16(in3, 1); + in4 = _mm_srai_epi16(in4, 1); + in5 = _mm_srai_epi16(in5, 1); + in6 = _mm_srai_epi16(in6, 1); + in7 = _mm_srai_epi16(in7, 1); + } + + iscan_ptr += n_coeffs; + qcoeff_ptr += n_coeffs; + dqcoeff_ptr += n_coeffs; + n_coeffs = -n_coeffs; + zero = _mm_setzero_si128(); + + if (!skip_block) { + __m128i eob; + __m128i round, quant, dequant, thr; + int16_t nzflag; + { + __m128i coeff0, coeff1; + + // Setup global values + { + round = _mm_load_si128((const __m128i *)round_ptr); + quant = _mm_load_si128((const __m128i *)quant_ptr); + dequant = _mm_load_si128((const __m128i *)dequant_ptr); + } + + { + __m128i coeff0_sign, coeff1_sign; + __m128i qcoeff0, qcoeff1; + __m128i qtmp0, qtmp1; + // Do DC and first 15 AC + coeff0 = *in[0]; + coeff1 = *in[1]; + + // Poor man's sign extract + coeff0_sign = _mm_srai_epi16(coeff0, 15); + coeff1_sign = _mm_srai_epi16(coeff1, 15); + qcoeff0 = _mm_xor_si128(coeff0, coeff0_sign); + qcoeff1 = _mm_xor_si128(coeff1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + qcoeff0 = _mm_adds_epi16(qcoeff0, round); + round = _mm_unpackhi_epi64(round, round); + qcoeff1 = _mm_adds_epi16(qcoeff1, round); + qtmp0 = _mm_mulhi_epi16(qcoeff0, quant); + quant = _mm_unpackhi_epi64(quant, quant); + qtmp1 = _mm_mulhi_epi16(qcoeff1, quant); + + // Reinsert signs + qcoeff0 = _mm_xor_si128(qtmp0, coeff0_sign); + qcoeff1 = _mm_xor_si128(qtmp1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), qcoeff0); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, qcoeff1); + + coeff0 = _mm_mullo_epi16(qcoeff0, dequant); + dequant = _mm_unpackhi_epi64(dequant, dequant); + coeff1 = _mm_mullo_epi16(qcoeff1, dequant); + + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), coeff0); + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, coeff1); + } + + { + // Scan for eob + __m128i zero_coeff0, zero_coeff1; + __m128i nzero_coeff0, nzero_coeff1; + __m128i iscan0, iscan1; + __m128i eob1; + zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero); + zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero); + nzero_coeff0 = _mm_cmpeq_epi16(zero_coeff0, zero); + nzero_coeff1 = _mm_cmpeq_epi16(zero_coeff1, zero); + iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs)); + iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs) + 1); + // Add one to convert from indices to counts + iscan0 = _mm_sub_epi16(iscan0, nzero_coeff0); + iscan1 = _mm_sub_epi16(iscan1, nzero_coeff1); + eob = _mm_and_si128(iscan0, nzero_coeff0); + eob1 = _mm_and_si128(iscan1, nzero_coeff1); + eob = _mm_max_epi16(eob, eob1); + } + n_coeffs += 8 * 2; + } + + // AC only loop + index = 2; + thr = _mm_srai_epi16(dequant, 1); + while (n_coeffs < 0) { + __m128i coeff0, coeff1; + { + __m128i coeff0_sign, coeff1_sign; + __m128i qcoeff0, qcoeff1; + __m128i qtmp0, qtmp1; + + assert(index < (int)(sizeof(in) / sizeof(in[0])) - 1); + coeff0 = *in[index]; + coeff1 = *in[index + 1]; + + // Poor man's sign extract + coeff0_sign = _mm_srai_epi16(coeff0, 15); + coeff1_sign = _mm_srai_epi16(coeff1, 15); + qcoeff0 = _mm_xor_si128(coeff0, coeff0_sign); + qcoeff1 = _mm_xor_si128(coeff1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + nzflag = _mm_movemask_epi8(_mm_cmpgt_epi16(qcoeff0, thr)) | + _mm_movemask_epi8(_mm_cmpgt_epi16(qcoeff1, thr)); + + if (nzflag) { + qcoeff0 = _mm_adds_epi16(qcoeff0, round); + qcoeff1 = _mm_adds_epi16(qcoeff1, round); + qtmp0 = _mm_mulhi_epi16(qcoeff0, quant); + qtmp1 = _mm_mulhi_epi16(qcoeff1, quant); + + // Reinsert signs + qcoeff0 = _mm_xor_si128(qtmp0, coeff0_sign); + qcoeff1 = _mm_xor_si128(qtmp1, coeff1_sign); + qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign); + qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign); + + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), qcoeff0); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, qcoeff1); + + coeff0 = _mm_mullo_epi16(qcoeff0, dequant); + coeff1 = _mm_mullo_epi16(qcoeff1, dequant); + + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), coeff0); + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, coeff1); + } else { + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, zero); + + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, zero); + } + } + + if (nzflag) { + // Scan for eob + __m128i zero_coeff0, zero_coeff1; + __m128i nzero_coeff0, nzero_coeff1; + __m128i iscan0, iscan1; + __m128i eob0, eob1; + zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero); + zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero); + nzero_coeff0 = _mm_cmpeq_epi16(zero_coeff0, zero); + nzero_coeff1 = _mm_cmpeq_epi16(zero_coeff1, zero); + iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs)); + iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs) + 1); + // Add one to convert from indices to counts + iscan0 = _mm_sub_epi16(iscan0, nzero_coeff0); + iscan1 = _mm_sub_epi16(iscan1, nzero_coeff1); + eob0 = _mm_and_si128(iscan0, nzero_coeff0); + eob1 = _mm_and_si128(iscan1, nzero_coeff1); + eob0 = _mm_max_epi16(eob0, eob1); + eob = _mm_max_epi16(eob, eob0); + } + n_coeffs += 8 * 2; + index += 2; + } + + // Accumulate EOB + { + __m128i eob_shuffled; + eob_shuffled = _mm_shuffle_epi32(eob, 0xe); + eob = _mm_max_epi16(eob, eob_shuffled); + eob_shuffled = _mm_shufflelo_epi16(eob, 0xe); + eob = _mm_max_epi16(eob, eob_shuffled); + eob_shuffled = _mm_shufflelo_epi16(eob, 0x1); + eob = _mm_max_epi16(eob, eob_shuffled); + *eob_ptr = _mm_extract_epi16(eob, 1); + } + } else { + do { + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, zero); + n_coeffs += 8 * 2; + } while (n_coeffs < 0); + *eob_ptr = 0; + } +} diff --git a/third_party/aom/av1/encoder/x86/error_intrin_avx2.c b/third_party/aom/av1/encoder/x86/error_intrin_avx2.c new file mode 100644 index 0000000000..ae733a1ce8 --- /dev/null +++ b/third_party/aom/av1/encoder/x86/error_intrin_avx2.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // AVX2 + +#include "./av1_rtcd.h" +#include "aom/aom_integer.h" + +int64_t av1_block_error_avx2(const int16_t *coeff, const int16_t *dqcoeff, + intptr_t block_size, int64_t *ssz) { + __m256i sse_reg, ssz_reg, coeff_reg, dqcoeff_reg; + __m256i exp_dqcoeff_lo, exp_dqcoeff_hi, exp_coeff_lo, exp_coeff_hi; + __m256i sse_reg_64hi, ssz_reg_64hi; + __m128i sse_reg128, ssz_reg128; + int64_t sse; + int i; + const __m256i zero_reg = _mm256_set1_epi16(0); + + // init sse and ssz registerd to zero + sse_reg = _mm256_set1_epi16(0); + ssz_reg = _mm256_set1_epi16(0); + + for (i = 0; i < block_size; i += 16) { + // load 32 bytes from coeff and dqcoeff + coeff_reg = _mm256_loadu_si256((const __m256i *)(coeff + i)); + dqcoeff_reg = _mm256_loadu_si256((const __m256i *)(dqcoeff + i)); + // dqcoeff - coeff + dqcoeff_reg = _mm256_sub_epi16(dqcoeff_reg, coeff_reg); + // madd (dqcoeff - coeff) + dqcoeff_reg = _mm256_madd_epi16(dqcoeff_reg, dqcoeff_reg); + // madd coeff + coeff_reg = _mm256_madd_epi16(coeff_reg, coeff_reg); + // expand each double word of madd (dqcoeff - coeff) to quad word + exp_dqcoeff_lo = _mm256_unpacklo_epi32(dqcoeff_reg, zero_reg); + exp_dqcoeff_hi = _mm256_unpackhi_epi32(dqcoeff_reg, zero_reg); + // expand each double word of madd (coeff) to quad word + exp_coeff_lo = _mm256_unpacklo_epi32(coeff_reg, zero_reg); + exp_coeff_hi = _mm256_unpackhi_epi32(coeff_reg, zero_reg); + // add each quad word of madd (dqcoeff - coeff) and madd (coeff) + sse_reg = _mm256_add_epi64(sse_reg, exp_dqcoeff_lo); + ssz_reg = _mm256_add_epi64(ssz_reg, exp_coeff_lo); + sse_reg = _mm256_add_epi64(sse_reg, exp_dqcoeff_hi); + ssz_reg = _mm256_add_epi64(ssz_reg, exp_coeff_hi); + } + // save the higher 64 bit of each 128 bit lane + sse_reg_64hi = _mm256_srli_si256(sse_reg, 8); + ssz_reg_64hi = _mm256_srli_si256(ssz_reg, 8); + // add the higher 64 bit to the low 64 bit + sse_reg = _mm256_add_epi64(sse_reg, sse_reg_64hi); + ssz_reg = _mm256_add_epi64(ssz_reg, ssz_reg_64hi); + + // add each 64 bit from each of the 128 bit lane of the 256 bit + sse_reg128 = _mm_add_epi64(_mm256_castsi256_si128(sse_reg), + _mm256_extractf128_si256(sse_reg, 1)); + + ssz_reg128 = _mm_add_epi64(_mm256_castsi256_si128(ssz_reg), + _mm256_extractf128_si256(ssz_reg, 1)); + + // store the results + _mm_storel_epi64((__m128i *)(&sse), sse_reg128); + + _mm_storel_epi64((__m128i *)(ssz), ssz_reg128); + _mm256_zeroupper(); + return sse; +} diff --git a/third_party/aom/av1/encoder/x86/error_sse2.asm b/third_party/aom/av1/encoder/x86/error_sse2.asm new file mode 100644 index 0000000000..4680f1fabd --- /dev/null +++ b/third_party/aom/av1/encoder/x86/error_sse2.asm @@ -0,0 +1,125 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%define private_prefix av1 + +%include "third_party/x86inc/x86inc.asm" + +SECTION .text + +; int64_t av1_block_error(int16_t *coeff, int16_t *dqcoeff, intptr_t block_size, +; int64_t *ssz) + +INIT_XMM sse2 +cglobal block_error, 3, 3, 8, uqc, dqc, size, ssz + pxor m4, m4 ; sse accumulator + pxor m6, m6 ; ssz accumulator + pxor m5, m5 ; dedicated zero register + lea uqcq, [uqcq+sizeq*2] + lea dqcq, [dqcq+sizeq*2] + neg sizeq +.loop: + mova m2, [uqcq+sizeq*2] + mova m0, [dqcq+sizeq*2] + mova m3, [uqcq+sizeq*2+mmsize] + mova m1, [dqcq+sizeq*2+mmsize] + psubw m0, m2 + psubw m1, m3 + ; individual errors are max. 15bit+sign, so squares are 30bit, and + ; thus the sum of 2 should fit in a 31bit integer (+ unused sign bit) + pmaddwd m0, m0 + pmaddwd m1, m1 + pmaddwd m2, m2 + pmaddwd m3, m3 + ; accumulate in 64bit + punpckldq m7, m0, m5 + punpckhdq m0, m5 + paddq m4, m7 + punpckldq m7, m1, m5 + paddq m4, m0 + punpckhdq m1, m5 + paddq m4, m7 + punpckldq m7, m2, m5 + paddq m4, m1 + punpckhdq m2, m5 + paddq m6, m7 + punpckldq m7, m3, m5 + paddq m6, m2 + punpckhdq m3, m5 + paddq m6, m7 + paddq m6, m3 + add sizeq, mmsize + jl .loop + + ; accumulate horizontally and store in return value + movhlps m5, m4 + movhlps m7, m6 + paddq m4, m5 + paddq m6, m7 +%if ARCH_X86_64 + movq rax, m4 + movq [sszq], m6 +%else + mov eax, sszm + pshufd m5, m4, 0x1 + movq [eax], m6 + movd eax, m4 + movd edx, m5 +%endif + RET + +; Compute the sum of squared difference between two int16_t vectors. +; int64_t av1_block_error_fp(int16_t *coeff, int16_t *dqcoeff, +; intptr_t block_size) + +INIT_XMM sse2 +cglobal block_error_fp, 3, 3, 6, uqc, dqc, size + pxor m4, m4 ; sse accumulator + pxor m5, m5 ; dedicated zero register + lea uqcq, [uqcq+sizeq*2] + lea dqcq, [dqcq+sizeq*2] + neg sizeq +.loop: + mova m2, [uqcq+sizeq*2] + mova m0, [dqcq+sizeq*2] + mova m3, [uqcq+sizeq*2+mmsize] + mova m1, [dqcq+sizeq*2+mmsize] + psubw m0, m2 + psubw m1, m3 + ; individual errors are max. 15bit+sign, so squares are 30bit, and + ; thus the sum of 2 should fit in a 31bit integer (+ unused sign bit) + pmaddwd m0, m0 + pmaddwd m1, m1 + ; accumulate in 64bit + punpckldq m3, m0, m5 + punpckhdq m0, m5 + paddq m4, m3 + punpckldq m3, m1, m5 + paddq m4, m0 + punpckhdq m1, m5 + paddq m4, m3 + paddq m4, m1 + add sizeq, mmsize + jl .loop + + ; accumulate horizontally and store in return value + movhlps m5, m4 + paddq m4, m5 +%if ARCH_X86_64 + movq rax, m4 +%else + pshufd m5, m4, 0x1 + movd eax, m4 + movd edx, m5 +%endif + RET diff --git a/third_party/aom/av1/encoder/x86/highbd_block_error_intrin_sse2.c b/third_party/aom/av1/encoder/x86/highbd_block_error_intrin_sse2.c new file mode 100644 index 0000000000..777304ace7 --- /dev/null +++ b/third_party/aom/av1/encoder/x86/highbd_block_error_intrin_sse2.c @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "av1/common/common.h" + +int64_t av1_highbd_block_error_sse2(tran_low_t *coeff, tran_low_t *dqcoeff, + intptr_t block_size, int64_t *ssz, + int bps) { + int i, j, test; + uint32_t temp[4]; + __m128i max, min, cmp0, cmp1, cmp2, cmp3; + int64_t error = 0, sqcoeff = 0; + const int shift = 2 * (bps - 8); + const int rounding = shift > 0 ? 1 << (shift - 1) : 0; + + for (i = 0; i < block_size; i += 8) { + // Load the data into xmm registers + __m128i mm_coeff = _mm_load_si128((__m128i *)(coeff + i)); + __m128i mm_coeff2 = _mm_load_si128((__m128i *)(coeff + i + 4)); + __m128i mm_dqcoeff = _mm_load_si128((__m128i *)(dqcoeff + i)); + __m128i mm_dqcoeff2 = _mm_load_si128((__m128i *)(dqcoeff + i + 4)); + // Check if any values require more than 15 bit + max = _mm_set1_epi32(0x3fff); + min = _mm_set1_epi32(0xffffc000); + cmp0 = _mm_xor_si128(_mm_cmpgt_epi32(mm_coeff, max), + _mm_cmplt_epi32(mm_coeff, min)); + cmp1 = _mm_xor_si128(_mm_cmpgt_epi32(mm_coeff2, max), + _mm_cmplt_epi32(mm_coeff2, min)); + cmp2 = _mm_xor_si128(_mm_cmpgt_epi32(mm_dqcoeff, max), + _mm_cmplt_epi32(mm_dqcoeff, min)); + cmp3 = _mm_xor_si128(_mm_cmpgt_epi32(mm_dqcoeff2, max), + _mm_cmplt_epi32(mm_dqcoeff2, min)); + test = _mm_movemask_epi8( + _mm_or_si128(_mm_or_si128(cmp0, cmp1), _mm_or_si128(cmp2, cmp3))); + + if (!test) { + __m128i mm_diff, error_sse2, sqcoeff_sse2; + mm_coeff = _mm_packs_epi32(mm_coeff, mm_coeff2); + mm_dqcoeff = _mm_packs_epi32(mm_dqcoeff, mm_dqcoeff2); + mm_diff = _mm_sub_epi16(mm_coeff, mm_dqcoeff); + error_sse2 = _mm_madd_epi16(mm_diff, mm_diff); + sqcoeff_sse2 = _mm_madd_epi16(mm_coeff, mm_coeff); + _mm_storeu_si128((__m128i *)temp, error_sse2); + error = error + temp[0] + temp[1] + temp[2] + temp[3]; + _mm_storeu_si128((__m128i *)temp, sqcoeff_sse2); + sqcoeff += temp[0] + temp[1] + temp[2] + temp[3]; + } else { + for (j = 0; j < 8; j++) { + const int64_t diff = coeff[i + j] - dqcoeff[i + j]; + error += diff * diff; + sqcoeff += (int64_t)coeff[i + j] * (int64_t)coeff[i + j]; + } + } + } + assert(error >= 0 && sqcoeff >= 0); + error = (error + rounding) >> shift; + sqcoeff = (sqcoeff + rounding) >> shift; + + *ssz = sqcoeff; + return error; +} diff --git a/third_party/aom/av1/encoder/x86/highbd_fwd_txfm_sse4.c b/third_party/aom/av1/encoder/x86/highbd_fwd_txfm_sse4.c new file mode 100644 index 0000000000..f201a29aaa --- /dev/null +++ b/third_party/aom/av1/encoder/x86/highbd_fwd_txfm_sse4.c @@ -0,0 +1,1895 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include +#include /* SSE4.1 */ + +#include "./av1_rtcd.h" +#include "./aom_config.h" +#include "av1/common/av1_fwd_txfm2d_cfg.h" +#include "av1/common/av1_txfm.h" +#include "av1/common/x86/highbd_txfm_utility_sse4.h" +#include "aom_dsp/txfm_common.h" +#include "aom_dsp/x86/txfm_common_sse2.h" +#include "aom_ports/mem.h" + +static INLINE void load_buffer_4x4(const int16_t *input, __m128i *in, + int stride, int flipud, int fliplr, + int shift) { + if (!flipud) { + in[0] = _mm_loadl_epi64((const __m128i *)(input + 0 * stride)); + in[1] = _mm_loadl_epi64((const __m128i *)(input + 1 * stride)); + in[2] = _mm_loadl_epi64((const __m128i *)(input + 2 * stride)); + in[3] = _mm_loadl_epi64((const __m128i *)(input + 3 * stride)); + } else { + in[0] = _mm_loadl_epi64((const __m128i *)(input + 3 * stride)); + in[1] = _mm_loadl_epi64((const __m128i *)(input + 2 * stride)); + in[2] = _mm_loadl_epi64((const __m128i *)(input + 1 * stride)); + in[3] = _mm_loadl_epi64((const __m128i *)(input + 0 * stride)); + } + + if (fliplr) { + in[0] = _mm_shufflelo_epi16(in[0], 0x1b); + in[1] = _mm_shufflelo_epi16(in[1], 0x1b); + in[2] = _mm_shufflelo_epi16(in[2], 0x1b); + in[3] = _mm_shufflelo_epi16(in[3], 0x1b); + } + + in[0] = _mm_cvtepi16_epi32(in[0]); + in[1] = _mm_cvtepi16_epi32(in[1]); + in[2] = _mm_cvtepi16_epi32(in[2]); + in[3] = _mm_cvtepi16_epi32(in[3]); + + in[0] = _mm_slli_epi32(in[0], shift); + in[1] = _mm_slli_epi32(in[1], shift); + in[2] = _mm_slli_epi32(in[2], shift); + in[3] = _mm_slli_epi32(in[3], shift); +} + +// We only use stage-2 bit; +// shift[0] is used in load_buffer_4x4() +// shift[1] is used in txfm_func_col() +// shift[2] is used in txfm_func_row() +static void fdct4x4_sse4_1(__m128i *in, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m128i cospi32 = _mm_set1_epi32(cospi[32]); + const __m128i cospi48 = _mm_set1_epi32(cospi[48]); + const __m128i cospi16 = _mm_set1_epi32(cospi[16]); + const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); + __m128i s0, s1, s2, s3; + __m128i u0, u1, u2, u3; + __m128i v0, v1, v2, v3; + + s0 = _mm_add_epi32(in[0], in[3]); + s1 = _mm_add_epi32(in[1], in[2]); + s2 = _mm_sub_epi32(in[1], in[2]); + s3 = _mm_sub_epi32(in[0], in[3]); + + // btf_32_sse4_1_type0(cospi32, cospi32, s[01], u[02], bit); + u0 = _mm_mullo_epi32(s0, cospi32); + u1 = _mm_mullo_epi32(s1, cospi32); + u2 = _mm_add_epi32(u0, u1); + v0 = _mm_sub_epi32(u0, u1); + + u3 = _mm_add_epi32(u2, rnding); + v1 = _mm_add_epi32(v0, rnding); + + u0 = _mm_srai_epi32(u3, bit); + u2 = _mm_srai_epi32(v1, bit); + + // btf_32_sse4_1_type1(cospi48, cospi16, s[23], u[13], bit); + v0 = _mm_mullo_epi32(s2, cospi48); + v1 = _mm_mullo_epi32(s3, cospi16); + v2 = _mm_add_epi32(v0, v1); + + v3 = _mm_add_epi32(v2, rnding); + u1 = _mm_srai_epi32(v3, bit); + + v0 = _mm_mullo_epi32(s2, cospi16); + v1 = _mm_mullo_epi32(s3, cospi48); + v2 = _mm_sub_epi32(v1, v0); + + v3 = _mm_add_epi32(v2, rnding); + u3 = _mm_srai_epi32(v3, bit); + + // Note: shift[1] and shift[2] are zeros + + // Transpose 4x4 32-bit + v0 = _mm_unpacklo_epi32(u0, u1); + v1 = _mm_unpackhi_epi32(u0, u1); + v2 = _mm_unpacklo_epi32(u2, u3); + v3 = _mm_unpackhi_epi32(u2, u3); + + in[0] = _mm_unpacklo_epi64(v0, v2); + in[1] = _mm_unpackhi_epi64(v0, v2); + in[2] = _mm_unpacklo_epi64(v1, v3); + in[3] = _mm_unpackhi_epi64(v1, v3); +} + +static INLINE void write_buffer_4x4(__m128i *res, tran_low_t *output) { + _mm_store_si128((__m128i *)(output + 0 * 4), res[0]); + _mm_store_si128((__m128i *)(output + 1 * 4), res[1]); + _mm_store_si128((__m128i *)(output + 2 * 4), res[2]); + _mm_store_si128((__m128i *)(output + 3 * 4), res[3]); +} + +// Note: +// We implement av1_fwd_txfm2d_4x4(). This function is kept here since +// av1_highbd_fht4x4_c() is not removed yet +void av1_highbd_fht4x4_sse4_1(const int16_t *input, tran_low_t *output, + int stride, int tx_type) { + (void)input; + (void)output; + (void)stride; + (void)tx_type; + assert(0); +} + +static void fadst4x4_sse4_1(__m128i *in, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m128i cospi8 = _mm_set1_epi32(cospi[8]); + const __m128i cospi56 = _mm_set1_epi32(cospi[56]); + const __m128i cospi40 = _mm_set1_epi32(cospi[40]); + const __m128i cospi24 = _mm_set1_epi32(cospi[24]); + const __m128i cospi32 = _mm_set1_epi32(cospi[32]); + const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); + const __m128i kZero = _mm_setzero_si128(); + __m128i s0, s1, s2, s3; + __m128i u0, u1, u2, u3; + __m128i v0, v1, v2, v3; + + // stage 0 + // stage 1 + // stage 2 + u0 = _mm_mullo_epi32(in[3], cospi8); + u1 = _mm_mullo_epi32(in[0], cospi56); + u2 = _mm_add_epi32(u0, u1); + s0 = _mm_add_epi32(u2, rnding); + s0 = _mm_srai_epi32(s0, bit); + + v0 = _mm_mullo_epi32(in[3], cospi56); + v1 = _mm_mullo_epi32(in[0], cospi8); + v2 = _mm_sub_epi32(v0, v1); + s1 = _mm_add_epi32(v2, rnding); + s1 = _mm_srai_epi32(s1, bit); + + u0 = _mm_mullo_epi32(in[1], cospi40); + u1 = _mm_mullo_epi32(in[2], cospi24); + u2 = _mm_add_epi32(u0, u1); + s2 = _mm_add_epi32(u2, rnding); + s2 = _mm_srai_epi32(s2, bit); + + v0 = _mm_mullo_epi32(in[1], cospi24); + v1 = _mm_mullo_epi32(in[2], cospi40); + v2 = _mm_sub_epi32(v0, v1); + s3 = _mm_add_epi32(v2, rnding); + s3 = _mm_srai_epi32(s3, bit); + + // stage 3 + u0 = _mm_add_epi32(s0, s2); + u2 = _mm_sub_epi32(s0, s2); + u1 = _mm_add_epi32(s1, s3); + u3 = _mm_sub_epi32(s1, s3); + + // stage 4 + v0 = _mm_mullo_epi32(u2, cospi32); + v1 = _mm_mullo_epi32(u3, cospi32); + v2 = _mm_add_epi32(v0, v1); + s2 = _mm_add_epi32(v2, rnding); + u2 = _mm_srai_epi32(s2, bit); + + v2 = _mm_sub_epi32(v0, v1); + s3 = _mm_add_epi32(v2, rnding); + u3 = _mm_srai_epi32(s3, bit); + + // u0, u1, u2, u3 + u2 = _mm_sub_epi32(kZero, u2); + u1 = _mm_sub_epi32(kZero, u1); + + // u0, u2, u3, u1 + // Transpose 4x4 32-bit + v0 = _mm_unpacklo_epi32(u0, u2); + v1 = _mm_unpackhi_epi32(u0, u2); + v2 = _mm_unpacklo_epi32(u3, u1); + v3 = _mm_unpackhi_epi32(u3, u1); + + in[0] = _mm_unpacklo_epi64(v0, v2); + in[1] = _mm_unpackhi_epi64(v0, v2); + in[2] = _mm_unpacklo_epi64(v1, v3); + in[3] = _mm_unpackhi_epi64(v1, v3); +} + +void av1_fwd_txfm2d_4x4_sse4_1(const int16_t *input, int32_t *coeff, + int input_stride, int tx_type, int bd) { + __m128i in[4]; + const TXFM_2D_CFG *cfg = NULL; + + switch (tx_type) { + case DCT_DCT: + cfg = &fwd_txfm_2d_cfg_dct_dct_4; + load_buffer_4x4(input, in, input_stride, 0, 0, cfg->shift[0]); + fdct4x4_sse4_1(in, cfg->cos_bit_col[2]); + fdct4x4_sse4_1(in, cfg->cos_bit_row[2]); + write_buffer_4x4(in, coeff); + break; + case ADST_DCT: + cfg = &fwd_txfm_2d_cfg_adst_dct_4; + load_buffer_4x4(input, in, input_stride, 0, 0, cfg->shift[0]); + fadst4x4_sse4_1(in, cfg->cos_bit_col[2]); + fdct4x4_sse4_1(in, cfg->cos_bit_row[2]); + write_buffer_4x4(in, coeff); + break; + case DCT_ADST: + cfg = &fwd_txfm_2d_cfg_dct_adst_4; + load_buffer_4x4(input, in, input_stride, 0, 0, cfg->shift[0]); + fdct4x4_sse4_1(in, cfg->cos_bit_col[2]); + fadst4x4_sse4_1(in, cfg->cos_bit_row[2]); + write_buffer_4x4(in, coeff); + break; + case ADST_ADST: + cfg = &fwd_txfm_2d_cfg_adst_adst_4; + load_buffer_4x4(input, in, input_stride, 0, 0, cfg->shift[0]); + fadst4x4_sse4_1(in, cfg->cos_bit_col[2]); + fadst4x4_sse4_1(in, cfg->cos_bit_row[2]); + write_buffer_4x4(in, coeff); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + cfg = &fwd_txfm_2d_cfg_adst_dct_4; + load_buffer_4x4(input, in, input_stride, 1, 0, cfg->shift[0]); + fadst4x4_sse4_1(in, cfg->cos_bit_col[2]); + fdct4x4_sse4_1(in, cfg->cos_bit_row[2]); + write_buffer_4x4(in, coeff); + break; + case DCT_FLIPADST: + cfg = &fwd_txfm_2d_cfg_dct_adst_4; + load_buffer_4x4(input, in, input_stride, 0, 1, cfg->shift[0]); + fdct4x4_sse4_1(in, cfg->cos_bit_col[2]); + fadst4x4_sse4_1(in, cfg->cos_bit_row[2]); + write_buffer_4x4(in, coeff); + break; + case FLIPADST_FLIPADST: + cfg = &fwd_txfm_2d_cfg_adst_adst_4; + load_buffer_4x4(input, in, input_stride, 1, 1, cfg->shift[0]); + fadst4x4_sse4_1(in, cfg->cos_bit_col[2]); + fadst4x4_sse4_1(in, cfg->cos_bit_row[2]); + write_buffer_4x4(in, coeff); + break; + case ADST_FLIPADST: + cfg = &fwd_txfm_2d_cfg_adst_adst_4; + load_buffer_4x4(input, in, input_stride, 0, 1, cfg->shift[0]); + fadst4x4_sse4_1(in, cfg->cos_bit_col[2]); + fadst4x4_sse4_1(in, cfg->cos_bit_row[2]); + write_buffer_4x4(in, coeff); + break; + case FLIPADST_ADST: + cfg = &fwd_txfm_2d_cfg_adst_adst_4; + load_buffer_4x4(input, in, input_stride, 1, 0, cfg->shift[0]); + fadst4x4_sse4_1(in, cfg->cos_bit_col[2]); + fadst4x4_sse4_1(in, cfg->cos_bit_row[2]); + write_buffer_4x4(in, coeff); + break; +#endif + default: assert(0); + } + (void)bd; +} + +static INLINE void load_buffer_8x8(const int16_t *input, __m128i *in, + int stride, int flipud, int fliplr, + int shift) { + __m128i u; + if (!flipud) { + in[0] = _mm_load_si128((const __m128i *)(input + 0 * stride)); + in[1] = _mm_load_si128((const __m128i *)(input + 1 * stride)); + in[2] = _mm_load_si128((const __m128i *)(input + 2 * stride)); + in[3] = _mm_load_si128((const __m128i *)(input + 3 * stride)); + in[4] = _mm_load_si128((const __m128i *)(input + 4 * stride)); + in[5] = _mm_load_si128((const __m128i *)(input + 5 * stride)); + in[6] = _mm_load_si128((const __m128i *)(input + 6 * stride)); + in[7] = _mm_load_si128((const __m128i *)(input + 7 * stride)); + } else { + in[0] = _mm_load_si128((const __m128i *)(input + 7 * stride)); + in[1] = _mm_load_si128((const __m128i *)(input + 6 * stride)); + in[2] = _mm_load_si128((const __m128i *)(input + 5 * stride)); + in[3] = _mm_load_si128((const __m128i *)(input + 4 * stride)); + in[4] = _mm_load_si128((const __m128i *)(input + 3 * stride)); + in[5] = _mm_load_si128((const __m128i *)(input + 2 * stride)); + in[6] = _mm_load_si128((const __m128i *)(input + 1 * stride)); + in[7] = _mm_load_si128((const __m128i *)(input + 0 * stride)); + } + + if (fliplr) { + in[0] = mm_reverse_epi16(in[0]); + in[1] = mm_reverse_epi16(in[1]); + in[2] = mm_reverse_epi16(in[2]); + in[3] = mm_reverse_epi16(in[3]); + in[4] = mm_reverse_epi16(in[4]); + in[5] = mm_reverse_epi16(in[5]); + in[6] = mm_reverse_epi16(in[6]); + in[7] = mm_reverse_epi16(in[7]); + } + + u = _mm_unpackhi_epi64(in[4], in[4]); + in[8] = _mm_cvtepi16_epi32(in[4]); + in[9] = _mm_cvtepi16_epi32(u); + + u = _mm_unpackhi_epi64(in[5], in[5]); + in[10] = _mm_cvtepi16_epi32(in[5]); + in[11] = _mm_cvtepi16_epi32(u); + + u = _mm_unpackhi_epi64(in[6], in[6]); + in[12] = _mm_cvtepi16_epi32(in[6]); + in[13] = _mm_cvtepi16_epi32(u); + + u = _mm_unpackhi_epi64(in[7], in[7]); + in[14] = _mm_cvtepi16_epi32(in[7]); + in[15] = _mm_cvtepi16_epi32(u); + + u = _mm_unpackhi_epi64(in[3], in[3]); + in[6] = _mm_cvtepi16_epi32(in[3]); + in[7] = _mm_cvtepi16_epi32(u); + + u = _mm_unpackhi_epi64(in[2], in[2]); + in[4] = _mm_cvtepi16_epi32(in[2]); + in[5] = _mm_cvtepi16_epi32(u); + + u = _mm_unpackhi_epi64(in[1], in[1]); + in[2] = _mm_cvtepi16_epi32(in[1]); + in[3] = _mm_cvtepi16_epi32(u); + + u = _mm_unpackhi_epi64(in[0], in[0]); + in[0] = _mm_cvtepi16_epi32(in[0]); + in[1] = _mm_cvtepi16_epi32(u); + + in[0] = _mm_slli_epi32(in[0], shift); + in[1] = _mm_slli_epi32(in[1], shift); + in[2] = _mm_slli_epi32(in[2], shift); + in[3] = _mm_slli_epi32(in[3], shift); + in[4] = _mm_slli_epi32(in[4], shift); + in[5] = _mm_slli_epi32(in[5], shift); + in[6] = _mm_slli_epi32(in[6], shift); + in[7] = _mm_slli_epi32(in[7], shift); + + in[8] = _mm_slli_epi32(in[8], shift); + in[9] = _mm_slli_epi32(in[9], shift); + in[10] = _mm_slli_epi32(in[10], shift); + in[11] = _mm_slli_epi32(in[11], shift); + in[12] = _mm_slli_epi32(in[12], shift); + in[13] = _mm_slli_epi32(in[13], shift); + in[14] = _mm_slli_epi32(in[14], shift); + in[15] = _mm_slli_epi32(in[15], shift); +} + +static INLINE void col_txfm_8x8_rounding(__m128i *in, int shift) { + const __m128i rounding = _mm_set1_epi32(1 << (shift - 1)); + + in[0] = _mm_add_epi32(in[0], rounding); + in[1] = _mm_add_epi32(in[1], rounding); + in[2] = _mm_add_epi32(in[2], rounding); + in[3] = _mm_add_epi32(in[3], rounding); + in[4] = _mm_add_epi32(in[4], rounding); + in[5] = _mm_add_epi32(in[5], rounding); + in[6] = _mm_add_epi32(in[6], rounding); + in[7] = _mm_add_epi32(in[7], rounding); + in[8] = _mm_add_epi32(in[8], rounding); + in[9] = _mm_add_epi32(in[9], rounding); + in[10] = _mm_add_epi32(in[10], rounding); + in[11] = _mm_add_epi32(in[11], rounding); + in[12] = _mm_add_epi32(in[12], rounding); + in[13] = _mm_add_epi32(in[13], rounding); + in[14] = _mm_add_epi32(in[14], rounding); + in[15] = _mm_add_epi32(in[15], rounding); + + in[0] = _mm_srai_epi32(in[0], shift); + in[1] = _mm_srai_epi32(in[1], shift); + in[2] = _mm_srai_epi32(in[2], shift); + in[3] = _mm_srai_epi32(in[3], shift); + in[4] = _mm_srai_epi32(in[4], shift); + in[5] = _mm_srai_epi32(in[5], shift); + in[6] = _mm_srai_epi32(in[6], shift); + in[7] = _mm_srai_epi32(in[7], shift); + in[8] = _mm_srai_epi32(in[8], shift); + in[9] = _mm_srai_epi32(in[9], shift); + in[10] = _mm_srai_epi32(in[10], shift); + in[11] = _mm_srai_epi32(in[11], shift); + in[12] = _mm_srai_epi32(in[12], shift); + in[13] = _mm_srai_epi32(in[13], shift); + in[14] = _mm_srai_epi32(in[14], shift); + in[15] = _mm_srai_epi32(in[15], shift); +} + +static INLINE void write_buffer_8x8(const __m128i *res, tran_low_t *output) { + _mm_store_si128((__m128i *)(output + 0 * 4), res[0]); + _mm_store_si128((__m128i *)(output + 1 * 4), res[1]); + _mm_store_si128((__m128i *)(output + 2 * 4), res[2]); + _mm_store_si128((__m128i *)(output + 3 * 4), res[3]); + + _mm_store_si128((__m128i *)(output + 4 * 4), res[4]); + _mm_store_si128((__m128i *)(output + 5 * 4), res[5]); + _mm_store_si128((__m128i *)(output + 6 * 4), res[6]); + _mm_store_si128((__m128i *)(output + 7 * 4), res[7]); + + _mm_store_si128((__m128i *)(output + 8 * 4), res[8]); + _mm_store_si128((__m128i *)(output + 9 * 4), res[9]); + _mm_store_si128((__m128i *)(output + 10 * 4), res[10]); + _mm_store_si128((__m128i *)(output + 11 * 4), res[11]); + + _mm_store_si128((__m128i *)(output + 12 * 4), res[12]); + _mm_store_si128((__m128i *)(output + 13 * 4), res[13]); + _mm_store_si128((__m128i *)(output + 14 * 4), res[14]); + _mm_store_si128((__m128i *)(output + 15 * 4), res[15]); +} + +static void fdct8x8_sse4_1(__m128i *in, __m128i *out, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m128i cospi32 = _mm_set1_epi32(cospi[32]); + const __m128i cospim32 = _mm_set1_epi32(-cospi[32]); + const __m128i cospi48 = _mm_set1_epi32(cospi[48]); + const __m128i cospi16 = _mm_set1_epi32(cospi[16]); + const __m128i cospi56 = _mm_set1_epi32(cospi[56]); + const __m128i cospi8 = _mm_set1_epi32(cospi[8]); + const __m128i cospi24 = _mm_set1_epi32(cospi[24]); + const __m128i cospi40 = _mm_set1_epi32(cospi[40]); + const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); + __m128i u[8], v[8]; + + // Even 8 points 0, 2, ..., 14 + // stage 0 + // stage 1 + u[0] = _mm_add_epi32(in[0], in[14]); + v[7] = _mm_sub_epi32(in[0], in[14]); // v[7] + u[1] = _mm_add_epi32(in[2], in[12]); + u[6] = _mm_sub_epi32(in[2], in[12]); + u[2] = _mm_add_epi32(in[4], in[10]); + u[5] = _mm_sub_epi32(in[4], in[10]); + u[3] = _mm_add_epi32(in[6], in[8]); + v[4] = _mm_sub_epi32(in[6], in[8]); // v[4] + + // stage 2 + v[0] = _mm_add_epi32(u[0], u[3]); + v[3] = _mm_sub_epi32(u[0], u[3]); + v[1] = _mm_add_epi32(u[1], u[2]); + v[2] = _mm_sub_epi32(u[1], u[2]); + + v[5] = _mm_mullo_epi32(u[5], cospim32); + v[6] = _mm_mullo_epi32(u[6], cospi32); + v[5] = _mm_add_epi32(v[5], v[6]); + v[5] = _mm_add_epi32(v[5], rnding); + v[5] = _mm_srai_epi32(v[5], bit); + + u[0] = _mm_mullo_epi32(u[5], cospi32); + v[6] = _mm_mullo_epi32(u[6], cospim32); + v[6] = _mm_sub_epi32(u[0], v[6]); + v[6] = _mm_add_epi32(v[6], rnding); + v[6] = _mm_srai_epi32(v[6], bit); + + // stage 3 + // type 0 + v[0] = _mm_mullo_epi32(v[0], cospi32); + v[1] = _mm_mullo_epi32(v[1], cospi32); + u[0] = _mm_add_epi32(v[0], v[1]); + u[0] = _mm_add_epi32(u[0], rnding); + u[0] = _mm_srai_epi32(u[0], bit); + + u[1] = _mm_sub_epi32(v[0], v[1]); + u[1] = _mm_add_epi32(u[1], rnding); + u[1] = _mm_srai_epi32(u[1], bit); + + // type 1 + v[0] = _mm_mullo_epi32(v[2], cospi48); + v[1] = _mm_mullo_epi32(v[3], cospi16); + u[2] = _mm_add_epi32(v[0], v[1]); + u[2] = _mm_add_epi32(u[2], rnding); + u[2] = _mm_srai_epi32(u[2], bit); + + v[0] = _mm_mullo_epi32(v[2], cospi16); + v[1] = _mm_mullo_epi32(v[3], cospi48); + u[3] = _mm_sub_epi32(v[1], v[0]); + u[3] = _mm_add_epi32(u[3], rnding); + u[3] = _mm_srai_epi32(u[3], bit); + + u[4] = _mm_add_epi32(v[4], v[5]); + u[5] = _mm_sub_epi32(v[4], v[5]); + u[6] = _mm_sub_epi32(v[7], v[6]); + u[7] = _mm_add_epi32(v[7], v[6]); + + // stage 4 + // stage 5 + v[0] = _mm_mullo_epi32(u[4], cospi56); + v[1] = _mm_mullo_epi32(u[7], cospi8); + v[0] = _mm_add_epi32(v[0], v[1]); + v[0] = _mm_add_epi32(v[0], rnding); + out[2] = _mm_srai_epi32(v[0], bit); // buf0[4] + + v[0] = _mm_mullo_epi32(u[4], cospi8); + v[1] = _mm_mullo_epi32(u[7], cospi56); + v[0] = _mm_sub_epi32(v[1], v[0]); + v[0] = _mm_add_epi32(v[0], rnding); + out[14] = _mm_srai_epi32(v[0], bit); // buf0[7] + + v[0] = _mm_mullo_epi32(u[5], cospi24); + v[1] = _mm_mullo_epi32(u[6], cospi40); + v[0] = _mm_add_epi32(v[0], v[1]); + v[0] = _mm_add_epi32(v[0], rnding); + out[10] = _mm_srai_epi32(v[0], bit); // buf0[5] + + v[0] = _mm_mullo_epi32(u[5], cospi40); + v[1] = _mm_mullo_epi32(u[6], cospi24); + v[0] = _mm_sub_epi32(v[1], v[0]); + v[0] = _mm_add_epi32(v[0], rnding); + out[6] = _mm_srai_epi32(v[0], bit); // buf0[6] + + out[0] = u[0]; // buf0[0] + out[8] = u[1]; // buf0[1] + out[4] = u[2]; // buf0[2] + out[12] = u[3]; // buf0[3] + + // Odd 8 points: 1, 3, ..., 15 + // stage 0 + // stage 1 + u[0] = _mm_add_epi32(in[1], in[15]); + v[7] = _mm_sub_epi32(in[1], in[15]); // v[7] + u[1] = _mm_add_epi32(in[3], in[13]); + u[6] = _mm_sub_epi32(in[3], in[13]); + u[2] = _mm_add_epi32(in[5], in[11]); + u[5] = _mm_sub_epi32(in[5], in[11]); + u[3] = _mm_add_epi32(in[7], in[9]); + v[4] = _mm_sub_epi32(in[7], in[9]); // v[4] + + // stage 2 + v[0] = _mm_add_epi32(u[0], u[3]); + v[3] = _mm_sub_epi32(u[0], u[3]); + v[1] = _mm_add_epi32(u[1], u[2]); + v[2] = _mm_sub_epi32(u[1], u[2]); + + v[5] = _mm_mullo_epi32(u[5], cospim32); + v[6] = _mm_mullo_epi32(u[6], cospi32); + v[5] = _mm_add_epi32(v[5], v[6]); + v[5] = _mm_add_epi32(v[5], rnding); + v[5] = _mm_srai_epi32(v[5], bit); + + u[0] = _mm_mullo_epi32(u[5], cospi32); + v[6] = _mm_mullo_epi32(u[6], cospim32); + v[6] = _mm_sub_epi32(u[0], v[6]); + v[6] = _mm_add_epi32(v[6], rnding); + v[6] = _mm_srai_epi32(v[6], bit); + + // stage 3 + // type 0 + v[0] = _mm_mullo_epi32(v[0], cospi32); + v[1] = _mm_mullo_epi32(v[1], cospi32); + u[0] = _mm_add_epi32(v[0], v[1]); + u[0] = _mm_add_epi32(u[0], rnding); + u[0] = _mm_srai_epi32(u[0], bit); + + u[1] = _mm_sub_epi32(v[0], v[1]); + u[1] = _mm_add_epi32(u[1], rnding); + u[1] = _mm_srai_epi32(u[1], bit); + + // type 1 + v[0] = _mm_mullo_epi32(v[2], cospi48); + v[1] = _mm_mullo_epi32(v[3], cospi16); + u[2] = _mm_add_epi32(v[0], v[1]); + u[2] = _mm_add_epi32(u[2], rnding); + u[2] = _mm_srai_epi32(u[2], bit); + + v[0] = _mm_mullo_epi32(v[2], cospi16); + v[1] = _mm_mullo_epi32(v[3], cospi48); + u[3] = _mm_sub_epi32(v[1], v[0]); + u[3] = _mm_add_epi32(u[3], rnding); + u[3] = _mm_srai_epi32(u[3], bit); + + u[4] = _mm_add_epi32(v[4], v[5]); + u[5] = _mm_sub_epi32(v[4], v[5]); + u[6] = _mm_sub_epi32(v[7], v[6]); + u[7] = _mm_add_epi32(v[7], v[6]); + + // stage 4 + // stage 5 + v[0] = _mm_mullo_epi32(u[4], cospi56); + v[1] = _mm_mullo_epi32(u[7], cospi8); + v[0] = _mm_add_epi32(v[0], v[1]); + v[0] = _mm_add_epi32(v[0], rnding); + out[3] = _mm_srai_epi32(v[0], bit); // buf0[4] + + v[0] = _mm_mullo_epi32(u[4], cospi8); + v[1] = _mm_mullo_epi32(u[7], cospi56); + v[0] = _mm_sub_epi32(v[1], v[0]); + v[0] = _mm_add_epi32(v[0], rnding); + out[15] = _mm_srai_epi32(v[0], bit); // buf0[7] + + v[0] = _mm_mullo_epi32(u[5], cospi24); + v[1] = _mm_mullo_epi32(u[6], cospi40); + v[0] = _mm_add_epi32(v[0], v[1]); + v[0] = _mm_add_epi32(v[0], rnding); + out[11] = _mm_srai_epi32(v[0], bit); // buf0[5] + + v[0] = _mm_mullo_epi32(u[5], cospi40); + v[1] = _mm_mullo_epi32(u[6], cospi24); + v[0] = _mm_sub_epi32(v[1], v[0]); + v[0] = _mm_add_epi32(v[0], rnding); + out[7] = _mm_srai_epi32(v[0], bit); // buf0[6] + + out[1] = u[0]; // buf0[0] + out[9] = u[1]; // buf0[1] + out[5] = u[2]; // buf0[2] + out[13] = u[3]; // buf0[3] +} + +static void fadst8x8_sse4_1(__m128i *in, __m128i *out, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m128i cospi4 = _mm_set1_epi32(cospi[4]); + const __m128i cospi60 = _mm_set1_epi32(cospi[60]); + const __m128i cospi20 = _mm_set1_epi32(cospi[20]); + const __m128i cospi44 = _mm_set1_epi32(cospi[44]); + const __m128i cospi36 = _mm_set1_epi32(cospi[36]); + const __m128i cospi28 = _mm_set1_epi32(cospi[28]); + const __m128i cospi52 = _mm_set1_epi32(cospi[52]); + const __m128i cospi12 = _mm_set1_epi32(cospi[12]); + const __m128i cospi16 = _mm_set1_epi32(cospi[16]); + const __m128i cospi48 = _mm_set1_epi32(cospi[48]); + const __m128i cospim48 = _mm_set1_epi32(-cospi[48]); + const __m128i cospi32 = _mm_set1_epi32(cospi[32]); + const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); + const __m128i kZero = _mm_setzero_si128(); + __m128i u[8], v[8], x; + + // Even 8 points: 0, 2, ..., 14 + // stage 0 + // stage 1 + // stage 2 + // (1) + u[0] = _mm_mullo_epi32(in[14], cospi4); + x = _mm_mullo_epi32(in[0], cospi60); + u[0] = _mm_add_epi32(u[0], x); + u[0] = _mm_add_epi32(u[0], rnding); + u[0] = _mm_srai_epi32(u[0], bit); + + u[1] = _mm_mullo_epi32(in[14], cospi60); + x = _mm_mullo_epi32(in[0], cospi4); + u[1] = _mm_sub_epi32(u[1], x); + u[1] = _mm_add_epi32(u[1], rnding); + u[1] = _mm_srai_epi32(u[1], bit); + + // (2) + u[2] = _mm_mullo_epi32(in[10], cospi20); + x = _mm_mullo_epi32(in[4], cospi44); + u[2] = _mm_add_epi32(u[2], x); + u[2] = _mm_add_epi32(u[2], rnding); + u[2] = _mm_srai_epi32(u[2], bit); + + u[3] = _mm_mullo_epi32(in[10], cospi44); + x = _mm_mullo_epi32(in[4], cospi20); + u[3] = _mm_sub_epi32(u[3], x); + u[3] = _mm_add_epi32(u[3], rnding); + u[3] = _mm_srai_epi32(u[3], bit); + + // (3) + u[4] = _mm_mullo_epi32(in[6], cospi36); + x = _mm_mullo_epi32(in[8], cospi28); + u[4] = _mm_add_epi32(u[4], x); + u[4] = _mm_add_epi32(u[4], rnding); + u[4] = _mm_srai_epi32(u[4], bit); + + u[5] = _mm_mullo_epi32(in[6], cospi28); + x = _mm_mullo_epi32(in[8], cospi36); + u[5] = _mm_sub_epi32(u[5], x); + u[5] = _mm_add_epi32(u[5], rnding); + u[5] = _mm_srai_epi32(u[5], bit); + + // (4) + u[6] = _mm_mullo_epi32(in[2], cospi52); + x = _mm_mullo_epi32(in[12], cospi12); + u[6] = _mm_add_epi32(u[6], x); + u[6] = _mm_add_epi32(u[6], rnding); + u[6] = _mm_srai_epi32(u[6], bit); + + u[7] = _mm_mullo_epi32(in[2], cospi12); + x = _mm_mullo_epi32(in[12], cospi52); + u[7] = _mm_sub_epi32(u[7], x); + u[7] = _mm_add_epi32(u[7], rnding); + u[7] = _mm_srai_epi32(u[7], bit); + + // stage 3 + v[0] = _mm_add_epi32(u[0], u[4]); + v[4] = _mm_sub_epi32(u[0], u[4]); + v[1] = _mm_add_epi32(u[1], u[5]); + v[5] = _mm_sub_epi32(u[1], u[5]); + v[2] = _mm_add_epi32(u[2], u[6]); + v[6] = _mm_sub_epi32(u[2], u[6]); + v[3] = _mm_add_epi32(u[3], u[7]); + v[7] = _mm_sub_epi32(u[3], u[7]); + + // stage 4 + u[0] = v[0]; + u[1] = v[1]; + u[2] = v[2]; + u[3] = v[3]; + + u[4] = _mm_mullo_epi32(v[4], cospi16); + x = _mm_mullo_epi32(v[5], cospi48); + u[4] = _mm_add_epi32(u[4], x); + u[4] = _mm_add_epi32(u[4], rnding); + u[4] = _mm_srai_epi32(u[4], bit); + + u[5] = _mm_mullo_epi32(v[4], cospi48); + x = _mm_mullo_epi32(v[5], cospi16); + u[5] = _mm_sub_epi32(u[5], x); + u[5] = _mm_add_epi32(u[5], rnding); + u[5] = _mm_srai_epi32(u[5], bit); + + u[6] = _mm_mullo_epi32(v[6], cospim48); + x = _mm_mullo_epi32(v[7], cospi16); + u[6] = _mm_add_epi32(u[6], x); + u[6] = _mm_add_epi32(u[6], rnding); + u[6] = _mm_srai_epi32(u[6], bit); + + u[7] = _mm_mullo_epi32(v[6], cospi16); + x = _mm_mullo_epi32(v[7], cospim48); + u[7] = _mm_sub_epi32(u[7], x); + u[7] = _mm_add_epi32(u[7], rnding); + u[7] = _mm_srai_epi32(u[7], bit); + + // stage 5 + v[0] = _mm_add_epi32(u[0], u[2]); + v[2] = _mm_sub_epi32(u[0], u[2]); + v[1] = _mm_add_epi32(u[1], u[3]); + v[3] = _mm_sub_epi32(u[1], u[3]); + v[4] = _mm_add_epi32(u[4], u[6]); + v[6] = _mm_sub_epi32(u[4], u[6]); + v[5] = _mm_add_epi32(u[5], u[7]); + v[7] = _mm_sub_epi32(u[5], u[7]); + + // stage 6 + u[0] = v[0]; + u[1] = v[1]; + u[4] = v[4]; + u[5] = v[5]; + + v[0] = _mm_mullo_epi32(v[2], cospi32); + x = _mm_mullo_epi32(v[3], cospi32); + u[2] = _mm_add_epi32(v[0], x); + u[2] = _mm_add_epi32(u[2], rnding); + u[2] = _mm_srai_epi32(u[2], bit); + + u[3] = _mm_sub_epi32(v[0], x); + u[3] = _mm_add_epi32(u[3], rnding); + u[3] = _mm_srai_epi32(u[3], bit); + + v[0] = _mm_mullo_epi32(v[6], cospi32); + x = _mm_mullo_epi32(v[7], cospi32); + u[6] = _mm_add_epi32(v[0], x); + u[6] = _mm_add_epi32(u[6], rnding); + u[6] = _mm_srai_epi32(u[6], bit); + + u[7] = _mm_sub_epi32(v[0], x); + u[7] = _mm_add_epi32(u[7], rnding); + u[7] = _mm_srai_epi32(u[7], bit); + + // stage 7 + out[0] = u[0]; + out[2] = _mm_sub_epi32(kZero, u[4]); + out[4] = u[6]; + out[6] = _mm_sub_epi32(kZero, u[2]); + out[8] = u[3]; + out[10] = _mm_sub_epi32(kZero, u[7]); + out[12] = u[5]; + out[14] = _mm_sub_epi32(kZero, u[1]); + + // Odd 8 points: 1, 3, ..., 15 + // stage 0 + // stage 1 + // stage 2 + // (1) + u[0] = _mm_mullo_epi32(in[15], cospi4); + x = _mm_mullo_epi32(in[1], cospi60); + u[0] = _mm_add_epi32(u[0], x); + u[0] = _mm_add_epi32(u[0], rnding); + u[0] = _mm_srai_epi32(u[0], bit); + + u[1] = _mm_mullo_epi32(in[15], cospi60); + x = _mm_mullo_epi32(in[1], cospi4); + u[1] = _mm_sub_epi32(u[1], x); + u[1] = _mm_add_epi32(u[1], rnding); + u[1] = _mm_srai_epi32(u[1], bit); + + // (2) + u[2] = _mm_mullo_epi32(in[11], cospi20); + x = _mm_mullo_epi32(in[5], cospi44); + u[2] = _mm_add_epi32(u[2], x); + u[2] = _mm_add_epi32(u[2], rnding); + u[2] = _mm_srai_epi32(u[2], bit); + + u[3] = _mm_mullo_epi32(in[11], cospi44); + x = _mm_mullo_epi32(in[5], cospi20); + u[3] = _mm_sub_epi32(u[3], x); + u[3] = _mm_add_epi32(u[3], rnding); + u[3] = _mm_srai_epi32(u[3], bit); + + // (3) + u[4] = _mm_mullo_epi32(in[7], cospi36); + x = _mm_mullo_epi32(in[9], cospi28); + u[4] = _mm_add_epi32(u[4], x); + u[4] = _mm_add_epi32(u[4], rnding); + u[4] = _mm_srai_epi32(u[4], bit); + + u[5] = _mm_mullo_epi32(in[7], cospi28); + x = _mm_mullo_epi32(in[9], cospi36); + u[5] = _mm_sub_epi32(u[5], x); + u[5] = _mm_add_epi32(u[5], rnding); + u[5] = _mm_srai_epi32(u[5], bit); + + // (4) + u[6] = _mm_mullo_epi32(in[3], cospi52); + x = _mm_mullo_epi32(in[13], cospi12); + u[6] = _mm_add_epi32(u[6], x); + u[6] = _mm_add_epi32(u[6], rnding); + u[6] = _mm_srai_epi32(u[6], bit); + + u[7] = _mm_mullo_epi32(in[3], cospi12); + x = _mm_mullo_epi32(in[13], cospi52); + u[7] = _mm_sub_epi32(u[7], x); + u[7] = _mm_add_epi32(u[7], rnding); + u[7] = _mm_srai_epi32(u[7], bit); + + // stage 3 + v[0] = _mm_add_epi32(u[0], u[4]); + v[4] = _mm_sub_epi32(u[0], u[4]); + v[1] = _mm_add_epi32(u[1], u[5]); + v[5] = _mm_sub_epi32(u[1], u[5]); + v[2] = _mm_add_epi32(u[2], u[6]); + v[6] = _mm_sub_epi32(u[2], u[6]); + v[3] = _mm_add_epi32(u[3], u[7]); + v[7] = _mm_sub_epi32(u[3], u[7]); + + // stage 4 + u[0] = v[0]; + u[1] = v[1]; + u[2] = v[2]; + u[3] = v[3]; + + u[4] = _mm_mullo_epi32(v[4], cospi16); + x = _mm_mullo_epi32(v[5], cospi48); + u[4] = _mm_add_epi32(u[4], x); + u[4] = _mm_add_epi32(u[4], rnding); + u[4] = _mm_srai_epi32(u[4], bit); + + u[5] = _mm_mullo_epi32(v[4], cospi48); + x = _mm_mullo_epi32(v[5], cospi16); + u[5] = _mm_sub_epi32(u[5], x); + u[5] = _mm_add_epi32(u[5], rnding); + u[5] = _mm_srai_epi32(u[5], bit); + + u[6] = _mm_mullo_epi32(v[6], cospim48); + x = _mm_mullo_epi32(v[7], cospi16); + u[6] = _mm_add_epi32(u[6], x); + u[6] = _mm_add_epi32(u[6], rnding); + u[6] = _mm_srai_epi32(u[6], bit); + + u[7] = _mm_mullo_epi32(v[6], cospi16); + x = _mm_mullo_epi32(v[7], cospim48); + u[7] = _mm_sub_epi32(u[7], x); + u[7] = _mm_add_epi32(u[7], rnding); + u[7] = _mm_srai_epi32(u[7], bit); + + // stage 5 + v[0] = _mm_add_epi32(u[0], u[2]); + v[2] = _mm_sub_epi32(u[0], u[2]); + v[1] = _mm_add_epi32(u[1], u[3]); + v[3] = _mm_sub_epi32(u[1], u[3]); + v[4] = _mm_add_epi32(u[4], u[6]); + v[6] = _mm_sub_epi32(u[4], u[6]); + v[5] = _mm_add_epi32(u[5], u[7]); + v[7] = _mm_sub_epi32(u[5], u[7]); + + // stage 6 + u[0] = v[0]; + u[1] = v[1]; + u[4] = v[4]; + u[5] = v[5]; + + v[0] = _mm_mullo_epi32(v[2], cospi32); + x = _mm_mullo_epi32(v[3], cospi32); + u[2] = _mm_add_epi32(v[0], x); + u[2] = _mm_add_epi32(u[2], rnding); + u[2] = _mm_srai_epi32(u[2], bit); + + u[3] = _mm_sub_epi32(v[0], x); + u[3] = _mm_add_epi32(u[3], rnding); + u[3] = _mm_srai_epi32(u[3], bit); + + v[0] = _mm_mullo_epi32(v[6], cospi32); + x = _mm_mullo_epi32(v[7], cospi32); + u[6] = _mm_add_epi32(v[0], x); + u[6] = _mm_add_epi32(u[6], rnding); + u[6] = _mm_srai_epi32(u[6], bit); + + u[7] = _mm_sub_epi32(v[0], x); + u[7] = _mm_add_epi32(u[7], rnding); + u[7] = _mm_srai_epi32(u[7], bit); + + // stage 7 + out[1] = u[0]; + out[3] = _mm_sub_epi32(kZero, u[4]); + out[5] = u[6]; + out[7] = _mm_sub_epi32(kZero, u[2]); + out[9] = u[3]; + out[11] = _mm_sub_epi32(kZero, u[7]); + out[13] = u[5]; + out[15] = _mm_sub_epi32(kZero, u[1]); +} + +void av1_fwd_txfm2d_8x8_sse4_1(const int16_t *input, int32_t *coeff, int stride, + int tx_type, int bd) { + __m128i in[16], out[16]; + const TXFM_2D_CFG *cfg = NULL; + + switch (tx_type) { + case DCT_DCT: + cfg = &fwd_txfm_2d_cfg_dct_dct_8; + load_buffer_8x8(input, in, stride, 0, 0, cfg->shift[0]); + fdct8x8_sse4_1(in, out, cfg->cos_bit_col[2]); + col_txfm_8x8_rounding(out, -cfg->shift[1]); + transpose_8x8(out, in); + fdct8x8_sse4_1(in, out, cfg->cos_bit_row[2]); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case ADST_DCT: + cfg = &fwd_txfm_2d_cfg_adst_dct_8; + load_buffer_8x8(input, in, stride, 0, 0, cfg->shift[0]); + fadst8x8_sse4_1(in, out, cfg->cos_bit_col[2]); + col_txfm_8x8_rounding(out, -cfg->shift[1]); + transpose_8x8(out, in); + fdct8x8_sse4_1(in, out, cfg->cos_bit_row[2]); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case DCT_ADST: + cfg = &fwd_txfm_2d_cfg_dct_adst_8; + load_buffer_8x8(input, in, stride, 0, 0, cfg->shift[0]); + fdct8x8_sse4_1(in, out, cfg->cos_bit_col[2]); + col_txfm_8x8_rounding(out, -cfg->shift[1]); + transpose_8x8(out, in); + fadst8x8_sse4_1(in, out, cfg->cos_bit_row[2]); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case ADST_ADST: + cfg = &fwd_txfm_2d_cfg_adst_adst_8; + load_buffer_8x8(input, in, stride, 0, 0, cfg->shift[0]); + fadst8x8_sse4_1(in, out, cfg->cos_bit_col[2]); + col_txfm_8x8_rounding(out, -cfg->shift[1]); + transpose_8x8(out, in); + fadst8x8_sse4_1(in, out, cfg->cos_bit_row[2]); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + cfg = &fwd_txfm_2d_cfg_adst_dct_8; + load_buffer_8x8(input, in, stride, 1, 0, cfg->shift[0]); + fadst8x8_sse4_1(in, out, cfg->cos_bit_col[2]); + col_txfm_8x8_rounding(out, -cfg->shift[1]); + transpose_8x8(out, in); + fdct8x8_sse4_1(in, out, cfg->cos_bit_row[2]); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case DCT_FLIPADST: + cfg = &fwd_txfm_2d_cfg_dct_adst_8; + load_buffer_8x8(input, in, stride, 0, 1, cfg->shift[0]); + fdct8x8_sse4_1(in, out, cfg->cos_bit_col[2]); + col_txfm_8x8_rounding(out, -cfg->shift[1]); + transpose_8x8(out, in); + fadst8x8_sse4_1(in, out, cfg->cos_bit_row[2]); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case FLIPADST_FLIPADST: + cfg = &fwd_txfm_2d_cfg_adst_adst_8; + load_buffer_8x8(input, in, stride, 1, 1, cfg->shift[0]); + fadst8x8_sse4_1(in, out, cfg->cos_bit_col[2]); + col_txfm_8x8_rounding(out, -cfg->shift[1]); + transpose_8x8(out, in); + fadst8x8_sse4_1(in, out, cfg->cos_bit_row[2]); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case ADST_FLIPADST: + cfg = &fwd_txfm_2d_cfg_adst_adst_8; + load_buffer_8x8(input, in, stride, 0, 1, cfg->shift[0]); + fadst8x8_sse4_1(in, out, cfg->cos_bit_col[2]); + col_txfm_8x8_rounding(out, -cfg->shift[1]); + transpose_8x8(out, in); + fadst8x8_sse4_1(in, out, cfg->cos_bit_row[2]); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case FLIPADST_ADST: + cfg = &fwd_txfm_2d_cfg_adst_adst_8; + load_buffer_8x8(input, in, stride, 1, 0, cfg->shift[0]); + fadst8x8_sse4_1(in, out, cfg->cos_bit_col[2]); + col_txfm_8x8_rounding(out, -cfg->shift[1]); + transpose_8x8(out, in); + fadst8x8_sse4_1(in, out, cfg->cos_bit_row[2]); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; +#endif // CONFIG_EXT_TX + default: assert(0); + } + (void)bd; +} + +// Hybrid Transform 16x16 + +static INLINE void convert_8x8_to_16x16(const __m128i *in, __m128i *out) { + int row_index = 0; + int dst_index = 0; + int src_index = 0; + + // row 0, 1, .., 7 + do { + out[dst_index] = in[src_index]; + out[dst_index + 1] = in[src_index + 1]; + out[dst_index + 2] = in[src_index + 16]; + out[dst_index + 3] = in[src_index + 17]; + dst_index += 4; + src_index += 2; + row_index += 1; + } while (row_index < 8); + + // row 8, 9, ..., 15 + src_index += 16; + do { + out[dst_index] = in[src_index]; + out[dst_index + 1] = in[src_index + 1]; + out[dst_index + 2] = in[src_index + 16]; + out[dst_index + 3] = in[src_index + 17]; + dst_index += 4; + src_index += 2; + row_index += 1; + } while (row_index < 16); +} + +static INLINE void load_buffer_16x16(const int16_t *input, __m128i *out, + int stride, int flipud, int fliplr, + int shift) { + __m128i in[64]; + // Load 4 8x8 blocks + const int16_t *topL = input; + const int16_t *topR = input + 8; + const int16_t *botL = input + 8 * stride; + const int16_t *botR = input + 8 * stride + 8; + + const int16_t *tmp; + + if (flipud) { + // Swap left columns + tmp = topL; + topL = botL; + botL = tmp; + // Swap right columns + tmp = topR; + topR = botR; + botR = tmp; + } + + if (fliplr) { + // Swap top rows + tmp = topL; + topL = topR; + topR = tmp; + // Swap bottom rows + tmp = botL; + botL = botR; + botR = tmp; + } + + // load first 8 columns + load_buffer_8x8(topL, &in[0], stride, flipud, fliplr, shift); + load_buffer_8x8(botL, &in[32], stride, flipud, fliplr, shift); + + // load second 8 columns + load_buffer_8x8(topR, &in[16], stride, flipud, fliplr, shift); + load_buffer_8x8(botR, &in[48], stride, flipud, fliplr, shift); + + convert_8x8_to_16x16(in, out); +} + +static void fdct16x16_sse4_1(__m128i *in, __m128i *out, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m128i cospi32 = _mm_set1_epi32(cospi[32]); + const __m128i cospim32 = _mm_set1_epi32(-cospi[32]); + const __m128i cospi48 = _mm_set1_epi32(cospi[48]); + const __m128i cospi16 = _mm_set1_epi32(cospi[16]); + const __m128i cospim48 = _mm_set1_epi32(-cospi[48]); + const __m128i cospim16 = _mm_set1_epi32(-cospi[16]); + const __m128i cospi56 = _mm_set1_epi32(cospi[56]); + const __m128i cospi8 = _mm_set1_epi32(cospi[8]); + const __m128i cospi24 = _mm_set1_epi32(cospi[24]); + const __m128i cospi40 = _mm_set1_epi32(cospi[40]); + const __m128i cospi60 = _mm_set1_epi32(cospi[60]); + const __m128i cospi4 = _mm_set1_epi32(cospi[4]); + const __m128i cospi28 = _mm_set1_epi32(cospi[28]); + const __m128i cospi36 = _mm_set1_epi32(cospi[36]); + const __m128i cospi44 = _mm_set1_epi32(cospi[44]); + const __m128i cospi20 = _mm_set1_epi32(cospi[20]); + const __m128i cospi12 = _mm_set1_epi32(cospi[12]); + const __m128i cospi52 = _mm_set1_epi32(cospi[52]); + const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); + __m128i u[16], v[16], x; + const int col_num = 4; + int col; + + // Calculate the column 0, 1, 2, 3 + for (col = 0; col < col_num; ++col) { + // stage 0 + // stage 1 + u[0] = _mm_add_epi32(in[0 * col_num + col], in[15 * col_num + col]); + u[15] = _mm_sub_epi32(in[0 * col_num + col], in[15 * col_num + col]); + u[1] = _mm_add_epi32(in[1 * col_num + col], in[14 * col_num + col]); + u[14] = _mm_sub_epi32(in[1 * col_num + col], in[14 * col_num + col]); + u[2] = _mm_add_epi32(in[2 * col_num + col], in[13 * col_num + col]); + u[13] = _mm_sub_epi32(in[2 * col_num + col], in[13 * col_num + col]); + u[3] = _mm_add_epi32(in[3 * col_num + col], in[12 * col_num + col]); + u[12] = _mm_sub_epi32(in[3 * col_num + col], in[12 * col_num + col]); + u[4] = _mm_add_epi32(in[4 * col_num + col], in[11 * col_num + col]); + u[11] = _mm_sub_epi32(in[4 * col_num + col], in[11 * col_num + col]); + u[5] = _mm_add_epi32(in[5 * col_num + col], in[10 * col_num + col]); + u[10] = _mm_sub_epi32(in[5 * col_num + col], in[10 * col_num + col]); + u[6] = _mm_add_epi32(in[6 * col_num + col], in[9 * col_num + col]); + u[9] = _mm_sub_epi32(in[6 * col_num + col], in[9 * col_num + col]); + u[7] = _mm_add_epi32(in[7 * col_num + col], in[8 * col_num + col]); + u[8] = _mm_sub_epi32(in[7 * col_num + col], in[8 * col_num + col]); + + // stage 2 + v[0] = _mm_add_epi32(u[0], u[7]); + v[7] = _mm_sub_epi32(u[0], u[7]); + v[1] = _mm_add_epi32(u[1], u[6]); + v[6] = _mm_sub_epi32(u[1], u[6]); + v[2] = _mm_add_epi32(u[2], u[5]); + v[5] = _mm_sub_epi32(u[2], u[5]); + v[3] = _mm_add_epi32(u[3], u[4]); + v[4] = _mm_sub_epi32(u[3], u[4]); + v[8] = u[8]; + v[9] = u[9]; + + v[10] = _mm_mullo_epi32(u[10], cospim32); + x = _mm_mullo_epi32(u[13], cospi32); + v[10] = _mm_add_epi32(v[10], x); + v[10] = _mm_add_epi32(v[10], rnding); + v[10] = _mm_srai_epi32(v[10], bit); + + v[13] = _mm_mullo_epi32(u[10], cospi32); + x = _mm_mullo_epi32(u[13], cospim32); + v[13] = _mm_sub_epi32(v[13], x); + v[13] = _mm_add_epi32(v[13], rnding); + v[13] = _mm_srai_epi32(v[13], bit); + + v[11] = _mm_mullo_epi32(u[11], cospim32); + x = _mm_mullo_epi32(u[12], cospi32); + v[11] = _mm_add_epi32(v[11], x); + v[11] = _mm_add_epi32(v[11], rnding); + v[11] = _mm_srai_epi32(v[11], bit); + + v[12] = _mm_mullo_epi32(u[11], cospi32); + x = _mm_mullo_epi32(u[12], cospim32); + v[12] = _mm_sub_epi32(v[12], x); + v[12] = _mm_add_epi32(v[12], rnding); + v[12] = _mm_srai_epi32(v[12], bit); + v[14] = u[14]; + v[15] = u[15]; + + // stage 3 + u[0] = _mm_add_epi32(v[0], v[3]); + u[3] = _mm_sub_epi32(v[0], v[3]); + u[1] = _mm_add_epi32(v[1], v[2]); + u[2] = _mm_sub_epi32(v[1], v[2]); + u[4] = v[4]; + + u[5] = _mm_mullo_epi32(v[5], cospim32); + x = _mm_mullo_epi32(v[6], cospi32); + u[5] = _mm_add_epi32(u[5], x); + u[5] = _mm_add_epi32(u[5], rnding); + u[5] = _mm_srai_epi32(u[5], bit); + + u[6] = _mm_mullo_epi32(v[5], cospi32); + x = _mm_mullo_epi32(v[6], cospim32); + u[6] = _mm_sub_epi32(u[6], x); + u[6] = _mm_add_epi32(u[6], rnding); + u[6] = _mm_srai_epi32(u[6], bit); + + u[7] = v[7]; + u[8] = _mm_add_epi32(v[8], v[11]); + u[11] = _mm_sub_epi32(v[8], v[11]); + u[9] = _mm_add_epi32(v[9], v[10]); + u[10] = _mm_sub_epi32(v[9], v[10]); + u[12] = _mm_sub_epi32(v[15], v[12]); + u[15] = _mm_add_epi32(v[15], v[12]); + u[13] = _mm_sub_epi32(v[14], v[13]); + u[14] = _mm_add_epi32(v[14], v[13]); + + // stage 4 + u[0] = _mm_mullo_epi32(u[0], cospi32); + u[1] = _mm_mullo_epi32(u[1], cospi32); + v[0] = _mm_add_epi32(u[0], u[1]); + v[0] = _mm_add_epi32(v[0], rnding); + v[0] = _mm_srai_epi32(v[0], bit); + + v[1] = _mm_sub_epi32(u[0], u[1]); + v[1] = _mm_add_epi32(v[1], rnding); + v[1] = _mm_srai_epi32(v[1], bit); + + v[2] = _mm_mullo_epi32(u[2], cospi48); + x = _mm_mullo_epi32(u[3], cospi16); + v[2] = _mm_add_epi32(v[2], x); + v[2] = _mm_add_epi32(v[2], rnding); + v[2] = _mm_srai_epi32(v[2], bit); + + v[3] = _mm_mullo_epi32(u[2], cospi16); + x = _mm_mullo_epi32(u[3], cospi48); + v[3] = _mm_sub_epi32(x, v[3]); + v[3] = _mm_add_epi32(v[3], rnding); + v[3] = _mm_srai_epi32(v[3], bit); + + v[4] = _mm_add_epi32(u[4], u[5]); + v[5] = _mm_sub_epi32(u[4], u[5]); + v[6] = _mm_sub_epi32(u[7], u[6]); + v[7] = _mm_add_epi32(u[7], u[6]); + v[8] = u[8]; + + v[9] = _mm_mullo_epi32(u[9], cospim16); + x = _mm_mullo_epi32(u[14], cospi48); + v[9] = _mm_add_epi32(v[9], x); + v[9] = _mm_add_epi32(v[9], rnding); + v[9] = _mm_srai_epi32(v[9], bit); + + v[14] = _mm_mullo_epi32(u[9], cospi48); + x = _mm_mullo_epi32(u[14], cospim16); + v[14] = _mm_sub_epi32(v[14], x); + v[14] = _mm_add_epi32(v[14], rnding); + v[14] = _mm_srai_epi32(v[14], bit); + + v[10] = _mm_mullo_epi32(u[10], cospim48); + x = _mm_mullo_epi32(u[13], cospim16); + v[10] = _mm_add_epi32(v[10], x); + v[10] = _mm_add_epi32(v[10], rnding); + v[10] = _mm_srai_epi32(v[10], bit); + + v[13] = _mm_mullo_epi32(u[10], cospim16); + x = _mm_mullo_epi32(u[13], cospim48); + v[13] = _mm_sub_epi32(v[13], x); + v[13] = _mm_add_epi32(v[13], rnding); + v[13] = _mm_srai_epi32(v[13], bit); + + v[11] = u[11]; + v[12] = u[12]; + v[15] = u[15]; + + // stage 5 + u[0] = v[0]; + u[1] = v[1]; + u[2] = v[2]; + u[3] = v[3]; + + u[4] = _mm_mullo_epi32(v[4], cospi56); + x = _mm_mullo_epi32(v[7], cospi8); + u[4] = _mm_add_epi32(u[4], x); + u[4] = _mm_add_epi32(u[4], rnding); + u[4] = _mm_srai_epi32(u[4], bit); + + u[7] = _mm_mullo_epi32(v[4], cospi8); + x = _mm_mullo_epi32(v[7], cospi56); + u[7] = _mm_sub_epi32(x, u[7]); + u[7] = _mm_add_epi32(u[7], rnding); + u[7] = _mm_srai_epi32(u[7], bit); + + u[5] = _mm_mullo_epi32(v[5], cospi24); + x = _mm_mullo_epi32(v[6], cospi40); + u[5] = _mm_add_epi32(u[5], x); + u[5] = _mm_add_epi32(u[5], rnding); + u[5] = _mm_srai_epi32(u[5], bit); + + u[6] = _mm_mullo_epi32(v[5], cospi40); + x = _mm_mullo_epi32(v[6], cospi24); + u[6] = _mm_sub_epi32(x, u[6]); + u[6] = _mm_add_epi32(u[6], rnding); + u[6] = _mm_srai_epi32(u[6], bit); + + u[8] = _mm_add_epi32(v[8], v[9]); + u[9] = _mm_sub_epi32(v[8], v[9]); + u[10] = _mm_sub_epi32(v[11], v[10]); + u[11] = _mm_add_epi32(v[11], v[10]); + u[12] = _mm_add_epi32(v[12], v[13]); + u[13] = _mm_sub_epi32(v[12], v[13]); + u[14] = _mm_sub_epi32(v[15], v[14]); + u[15] = _mm_add_epi32(v[15], v[14]); + + // stage 6 + v[0] = u[0]; + v[1] = u[1]; + v[2] = u[2]; + v[3] = u[3]; + v[4] = u[4]; + v[5] = u[5]; + v[6] = u[6]; + v[7] = u[7]; + + v[8] = _mm_mullo_epi32(u[8], cospi60); + x = _mm_mullo_epi32(u[15], cospi4); + v[8] = _mm_add_epi32(v[8], x); + v[8] = _mm_add_epi32(v[8], rnding); + v[8] = _mm_srai_epi32(v[8], bit); + + v[15] = _mm_mullo_epi32(u[8], cospi4); + x = _mm_mullo_epi32(u[15], cospi60); + v[15] = _mm_sub_epi32(x, v[15]); + v[15] = _mm_add_epi32(v[15], rnding); + v[15] = _mm_srai_epi32(v[15], bit); + + v[9] = _mm_mullo_epi32(u[9], cospi28); + x = _mm_mullo_epi32(u[14], cospi36); + v[9] = _mm_add_epi32(v[9], x); + v[9] = _mm_add_epi32(v[9], rnding); + v[9] = _mm_srai_epi32(v[9], bit); + + v[14] = _mm_mullo_epi32(u[9], cospi36); + x = _mm_mullo_epi32(u[14], cospi28); + v[14] = _mm_sub_epi32(x, v[14]); + v[14] = _mm_add_epi32(v[14], rnding); + v[14] = _mm_srai_epi32(v[14], bit); + + v[10] = _mm_mullo_epi32(u[10], cospi44); + x = _mm_mullo_epi32(u[13], cospi20); + v[10] = _mm_add_epi32(v[10], x); + v[10] = _mm_add_epi32(v[10], rnding); + v[10] = _mm_srai_epi32(v[10], bit); + + v[13] = _mm_mullo_epi32(u[10], cospi20); + x = _mm_mullo_epi32(u[13], cospi44); + v[13] = _mm_sub_epi32(x, v[13]); + v[13] = _mm_add_epi32(v[13], rnding); + v[13] = _mm_srai_epi32(v[13], bit); + + v[11] = _mm_mullo_epi32(u[11], cospi12); + x = _mm_mullo_epi32(u[12], cospi52); + v[11] = _mm_add_epi32(v[11], x); + v[11] = _mm_add_epi32(v[11], rnding); + v[11] = _mm_srai_epi32(v[11], bit); + + v[12] = _mm_mullo_epi32(u[11], cospi52); + x = _mm_mullo_epi32(u[12], cospi12); + v[12] = _mm_sub_epi32(x, v[12]); + v[12] = _mm_add_epi32(v[12], rnding); + v[12] = _mm_srai_epi32(v[12], bit); + + out[0 * col_num + col] = v[0]; + out[1 * col_num + col] = v[8]; + out[2 * col_num + col] = v[4]; + out[3 * col_num + col] = v[12]; + out[4 * col_num + col] = v[2]; + out[5 * col_num + col] = v[10]; + out[6 * col_num + col] = v[6]; + out[7 * col_num + col] = v[14]; + out[8 * col_num + col] = v[1]; + out[9 * col_num + col] = v[9]; + out[10 * col_num + col] = v[5]; + out[11 * col_num + col] = v[13]; + out[12 * col_num + col] = v[3]; + out[13 * col_num + col] = v[11]; + out[14 * col_num + col] = v[7]; + out[15 * col_num + col] = v[15]; + } +} + +static void fadst16x16_sse4_1(__m128i *in, __m128i *out, int bit) { + const int32_t *cospi = cospi_arr[bit - cos_bit_min]; + const __m128i cospi2 = _mm_set1_epi32(cospi[2]); + const __m128i cospi62 = _mm_set1_epi32(cospi[62]); + const __m128i cospi10 = _mm_set1_epi32(cospi[10]); + const __m128i cospi54 = _mm_set1_epi32(cospi[54]); + const __m128i cospi18 = _mm_set1_epi32(cospi[18]); + const __m128i cospi46 = _mm_set1_epi32(cospi[46]); + const __m128i cospi26 = _mm_set1_epi32(cospi[26]); + const __m128i cospi38 = _mm_set1_epi32(cospi[38]); + const __m128i cospi34 = _mm_set1_epi32(cospi[34]); + const __m128i cospi30 = _mm_set1_epi32(cospi[30]); + const __m128i cospi42 = _mm_set1_epi32(cospi[42]); + const __m128i cospi22 = _mm_set1_epi32(cospi[22]); + const __m128i cospi50 = _mm_set1_epi32(cospi[50]); + const __m128i cospi14 = _mm_set1_epi32(cospi[14]); + const __m128i cospi58 = _mm_set1_epi32(cospi[58]); + const __m128i cospi6 = _mm_set1_epi32(cospi[6]); + const __m128i cospi8 = _mm_set1_epi32(cospi[8]); + const __m128i cospi56 = _mm_set1_epi32(cospi[56]); + const __m128i cospi40 = _mm_set1_epi32(cospi[40]); + const __m128i cospi24 = _mm_set1_epi32(cospi[24]); + const __m128i cospim56 = _mm_set1_epi32(-cospi[56]); + const __m128i cospim24 = _mm_set1_epi32(-cospi[24]); + const __m128i cospi48 = _mm_set1_epi32(cospi[48]); + const __m128i cospi16 = _mm_set1_epi32(cospi[16]); + const __m128i cospim48 = _mm_set1_epi32(-cospi[48]); + const __m128i cospi32 = _mm_set1_epi32(cospi[32]); + const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); + __m128i u[16], v[16], x, y; + const int col_num = 4; + int col; + + // Calculate the column 0, 1, 2, 3 + for (col = 0; col < col_num; ++col) { + // stage 0 + // stage 1 + // stage 2 + v[0] = _mm_mullo_epi32(in[15 * col_num + col], cospi2); + x = _mm_mullo_epi32(in[0 * col_num + col], cospi62); + v[0] = _mm_add_epi32(v[0], x); + v[0] = _mm_add_epi32(v[0], rnding); + v[0] = _mm_srai_epi32(v[0], bit); + + v[1] = _mm_mullo_epi32(in[15 * col_num + col], cospi62); + x = _mm_mullo_epi32(in[0 * col_num + col], cospi2); + v[1] = _mm_sub_epi32(v[1], x); + v[1] = _mm_add_epi32(v[1], rnding); + v[1] = _mm_srai_epi32(v[1], bit); + + v[2] = _mm_mullo_epi32(in[13 * col_num + col], cospi10); + x = _mm_mullo_epi32(in[2 * col_num + col], cospi54); + v[2] = _mm_add_epi32(v[2], x); + v[2] = _mm_add_epi32(v[2], rnding); + v[2] = _mm_srai_epi32(v[2], bit); + + v[3] = _mm_mullo_epi32(in[13 * col_num + col], cospi54); + x = _mm_mullo_epi32(in[2 * col_num + col], cospi10); + v[3] = _mm_sub_epi32(v[3], x); + v[3] = _mm_add_epi32(v[3], rnding); + v[3] = _mm_srai_epi32(v[3], bit); + + v[4] = _mm_mullo_epi32(in[11 * col_num + col], cospi18); + x = _mm_mullo_epi32(in[4 * col_num + col], cospi46); + v[4] = _mm_add_epi32(v[4], x); + v[4] = _mm_add_epi32(v[4], rnding); + v[4] = _mm_srai_epi32(v[4], bit); + + v[5] = _mm_mullo_epi32(in[11 * col_num + col], cospi46); + x = _mm_mullo_epi32(in[4 * col_num + col], cospi18); + v[5] = _mm_sub_epi32(v[5], x); + v[5] = _mm_add_epi32(v[5], rnding); + v[5] = _mm_srai_epi32(v[5], bit); + + v[6] = _mm_mullo_epi32(in[9 * col_num + col], cospi26); + x = _mm_mullo_epi32(in[6 * col_num + col], cospi38); + v[6] = _mm_add_epi32(v[6], x); + v[6] = _mm_add_epi32(v[6], rnding); + v[6] = _mm_srai_epi32(v[6], bit); + + v[7] = _mm_mullo_epi32(in[9 * col_num + col], cospi38); + x = _mm_mullo_epi32(in[6 * col_num + col], cospi26); + v[7] = _mm_sub_epi32(v[7], x); + v[7] = _mm_add_epi32(v[7], rnding); + v[7] = _mm_srai_epi32(v[7], bit); + + v[8] = _mm_mullo_epi32(in[7 * col_num + col], cospi34); + x = _mm_mullo_epi32(in[8 * col_num + col], cospi30); + v[8] = _mm_add_epi32(v[8], x); + v[8] = _mm_add_epi32(v[8], rnding); + v[8] = _mm_srai_epi32(v[8], bit); + + v[9] = _mm_mullo_epi32(in[7 * col_num + col], cospi30); + x = _mm_mullo_epi32(in[8 * col_num + col], cospi34); + v[9] = _mm_sub_epi32(v[9], x); + v[9] = _mm_add_epi32(v[9], rnding); + v[9] = _mm_srai_epi32(v[9], bit); + + v[10] = _mm_mullo_epi32(in[5 * col_num + col], cospi42); + x = _mm_mullo_epi32(in[10 * col_num + col], cospi22); + v[10] = _mm_add_epi32(v[10], x); + v[10] = _mm_add_epi32(v[10], rnding); + v[10] = _mm_srai_epi32(v[10], bit); + + v[11] = _mm_mullo_epi32(in[5 * col_num + col], cospi22); + x = _mm_mullo_epi32(in[10 * col_num + col], cospi42); + v[11] = _mm_sub_epi32(v[11], x); + v[11] = _mm_add_epi32(v[11], rnding); + v[11] = _mm_srai_epi32(v[11], bit); + + v[12] = _mm_mullo_epi32(in[3 * col_num + col], cospi50); + x = _mm_mullo_epi32(in[12 * col_num + col], cospi14); + v[12] = _mm_add_epi32(v[12], x); + v[12] = _mm_add_epi32(v[12], rnding); + v[12] = _mm_srai_epi32(v[12], bit); + + v[13] = _mm_mullo_epi32(in[3 * col_num + col], cospi14); + x = _mm_mullo_epi32(in[12 * col_num + col], cospi50); + v[13] = _mm_sub_epi32(v[13], x); + v[13] = _mm_add_epi32(v[13], rnding); + v[13] = _mm_srai_epi32(v[13], bit); + + v[14] = _mm_mullo_epi32(in[1 * col_num + col], cospi58); + x = _mm_mullo_epi32(in[14 * col_num + col], cospi6); + v[14] = _mm_add_epi32(v[14], x); + v[14] = _mm_add_epi32(v[14], rnding); + v[14] = _mm_srai_epi32(v[14], bit); + + v[15] = _mm_mullo_epi32(in[1 * col_num + col], cospi6); + x = _mm_mullo_epi32(in[14 * col_num + col], cospi58); + v[15] = _mm_sub_epi32(v[15], x); + v[15] = _mm_add_epi32(v[15], rnding); + v[15] = _mm_srai_epi32(v[15], bit); + + // stage 3 + u[0] = _mm_add_epi32(v[0], v[8]); + u[8] = _mm_sub_epi32(v[0], v[8]); + u[1] = _mm_add_epi32(v[1], v[9]); + u[9] = _mm_sub_epi32(v[1], v[9]); + u[2] = _mm_add_epi32(v[2], v[10]); + u[10] = _mm_sub_epi32(v[2], v[10]); + u[3] = _mm_add_epi32(v[3], v[11]); + u[11] = _mm_sub_epi32(v[3], v[11]); + u[4] = _mm_add_epi32(v[4], v[12]); + u[12] = _mm_sub_epi32(v[4], v[12]); + u[5] = _mm_add_epi32(v[5], v[13]); + u[13] = _mm_sub_epi32(v[5], v[13]); + u[6] = _mm_add_epi32(v[6], v[14]); + u[14] = _mm_sub_epi32(v[6], v[14]); + u[7] = _mm_add_epi32(v[7], v[15]); + u[15] = _mm_sub_epi32(v[7], v[15]); + + // stage 4 + v[0] = u[0]; + v[1] = u[1]; + v[2] = u[2]; + v[3] = u[3]; + v[4] = u[4]; + v[5] = u[5]; + v[6] = u[6]; + v[7] = u[7]; + + v[8] = _mm_mullo_epi32(u[8], cospi8); + x = _mm_mullo_epi32(u[9], cospi56); + v[8] = _mm_add_epi32(v[8], x); + v[8] = _mm_add_epi32(v[8], rnding); + v[8] = _mm_srai_epi32(v[8], bit); + + v[9] = _mm_mullo_epi32(u[8], cospi56); + x = _mm_mullo_epi32(u[9], cospi8); + v[9] = _mm_sub_epi32(v[9], x); + v[9] = _mm_add_epi32(v[9], rnding); + v[9] = _mm_srai_epi32(v[9], bit); + + v[10] = _mm_mullo_epi32(u[10], cospi40); + x = _mm_mullo_epi32(u[11], cospi24); + v[10] = _mm_add_epi32(v[10], x); + v[10] = _mm_add_epi32(v[10], rnding); + v[10] = _mm_srai_epi32(v[10], bit); + + v[11] = _mm_mullo_epi32(u[10], cospi24); + x = _mm_mullo_epi32(u[11], cospi40); + v[11] = _mm_sub_epi32(v[11], x); + v[11] = _mm_add_epi32(v[11], rnding); + v[11] = _mm_srai_epi32(v[11], bit); + + v[12] = _mm_mullo_epi32(u[12], cospim56); + x = _mm_mullo_epi32(u[13], cospi8); + v[12] = _mm_add_epi32(v[12], x); + v[12] = _mm_add_epi32(v[12], rnding); + v[12] = _mm_srai_epi32(v[12], bit); + + v[13] = _mm_mullo_epi32(u[12], cospi8); + x = _mm_mullo_epi32(u[13], cospim56); + v[13] = _mm_sub_epi32(v[13], x); + v[13] = _mm_add_epi32(v[13], rnding); + v[13] = _mm_srai_epi32(v[13], bit); + + v[14] = _mm_mullo_epi32(u[14], cospim24); + x = _mm_mullo_epi32(u[15], cospi40); + v[14] = _mm_add_epi32(v[14], x); + v[14] = _mm_add_epi32(v[14], rnding); + v[14] = _mm_srai_epi32(v[14], bit); + + v[15] = _mm_mullo_epi32(u[14], cospi40); + x = _mm_mullo_epi32(u[15], cospim24); + v[15] = _mm_sub_epi32(v[15], x); + v[15] = _mm_add_epi32(v[15], rnding); + v[15] = _mm_srai_epi32(v[15], bit); + + // stage 5 + u[0] = _mm_add_epi32(v[0], v[4]); + u[4] = _mm_sub_epi32(v[0], v[4]); + u[1] = _mm_add_epi32(v[1], v[5]); + u[5] = _mm_sub_epi32(v[1], v[5]); + u[2] = _mm_add_epi32(v[2], v[6]); + u[6] = _mm_sub_epi32(v[2], v[6]); + u[3] = _mm_add_epi32(v[3], v[7]); + u[7] = _mm_sub_epi32(v[3], v[7]); + u[8] = _mm_add_epi32(v[8], v[12]); + u[12] = _mm_sub_epi32(v[8], v[12]); + u[9] = _mm_add_epi32(v[9], v[13]); + u[13] = _mm_sub_epi32(v[9], v[13]); + u[10] = _mm_add_epi32(v[10], v[14]); + u[14] = _mm_sub_epi32(v[10], v[14]); + u[11] = _mm_add_epi32(v[11], v[15]); + u[15] = _mm_sub_epi32(v[11], v[15]); + + // stage 6 + v[0] = u[0]; + v[1] = u[1]; + v[2] = u[2]; + v[3] = u[3]; + + v[4] = _mm_mullo_epi32(u[4], cospi16); + x = _mm_mullo_epi32(u[5], cospi48); + v[4] = _mm_add_epi32(v[4], x); + v[4] = _mm_add_epi32(v[4], rnding); + v[4] = _mm_srai_epi32(v[4], bit); + + v[5] = _mm_mullo_epi32(u[4], cospi48); + x = _mm_mullo_epi32(u[5], cospi16); + v[5] = _mm_sub_epi32(v[5], x); + v[5] = _mm_add_epi32(v[5], rnding); + v[5] = _mm_srai_epi32(v[5], bit); + + v[6] = _mm_mullo_epi32(u[6], cospim48); + x = _mm_mullo_epi32(u[7], cospi16); + v[6] = _mm_add_epi32(v[6], x); + v[6] = _mm_add_epi32(v[6], rnding); + v[6] = _mm_srai_epi32(v[6], bit); + + v[7] = _mm_mullo_epi32(u[6], cospi16); + x = _mm_mullo_epi32(u[7], cospim48); + v[7] = _mm_sub_epi32(v[7], x); + v[7] = _mm_add_epi32(v[7], rnding); + v[7] = _mm_srai_epi32(v[7], bit); + + v[8] = u[8]; + v[9] = u[9]; + v[10] = u[10]; + v[11] = u[11]; + + v[12] = _mm_mullo_epi32(u[12], cospi16); + x = _mm_mullo_epi32(u[13], cospi48); + v[12] = _mm_add_epi32(v[12], x); + v[12] = _mm_add_epi32(v[12], rnding); + v[12] = _mm_srai_epi32(v[12], bit); + + v[13] = _mm_mullo_epi32(u[12], cospi48); + x = _mm_mullo_epi32(u[13], cospi16); + v[13] = _mm_sub_epi32(v[13], x); + v[13] = _mm_add_epi32(v[13], rnding); + v[13] = _mm_srai_epi32(v[13], bit); + + v[14] = _mm_mullo_epi32(u[14], cospim48); + x = _mm_mullo_epi32(u[15], cospi16); + v[14] = _mm_add_epi32(v[14], x); + v[14] = _mm_add_epi32(v[14], rnding); + v[14] = _mm_srai_epi32(v[14], bit); + + v[15] = _mm_mullo_epi32(u[14], cospi16); + x = _mm_mullo_epi32(u[15], cospim48); + v[15] = _mm_sub_epi32(v[15], x); + v[15] = _mm_add_epi32(v[15], rnding); + v[15] = _mm_srai_epi32(v[15], bit); + + // stage 7 + u[0] = _mm_add_epi32(v[0], v[2]); + u[2] = _mm_sub_epi32(v[0], v[2]); + u[1] = _mm_add_epi32(v[1], v[3]); + u[3] = _mm_sub_epi32(v[1], v[3]); + u[4] = _mm_add_epi32(v[4], v[6]); + u[6] = _mm_sub_epi32(v[4], v[6]); + u[5] = _mm_add_epi32(v[5], v[7]); + u[7] = _mm_sub_epi32(v[5], v[7]); + u[8] = _mm_add_epi32(v[8], v[10]); + u[10] = _mm_sub_epi32(v[8], v[10]); + u[9] = _mm_add_epi32(v[9], v[11]); + u[11] = _mm_sub_epi32(v[9], v[11]); + u[12] = _mm_add_epi32(v[12], v[14]); + u[14] = _mm_sub_epi32(v[12], v[14]); + u[13] = _mm_add_epi32(v[13], v[15]); + u[15] = _mm_sub_epi32(v[13], v[15]); + + // stage 8 + v[0] = u[0]; + v[1] = u[1]; + + y = _mm_mullo_epi32(u[2], cospi32); + x = _mm_mullo_epi32(u[3], cospi32); + v[2] = _mm_add_epi32(y, x); + v[2] = _mm_add_epi32(v[2], rnding); + v[2] = _mm_srai_epi32(v[2], bit); + + v[3] = _mm_sub_epi32(y, x); + v[3] = _mm_add_epi32(v[3], rnding); + v[3] = _mm_srai_epi32(v[3], bit); + + v[4] = u[4]; + v[5] = u[5]; + + y = _mm_mullo_epi32(u[6], cospi32); + x = _mm_mullo_epi32(u[7], cospi32); + v[6] = _mm_add_epi32(y, x); + v[6] = _mm_add_epi32(v[6], rnding); + v[6] = _mm_srai_epi32(v[6], bit); + + v[7] = _mm_sub_epi32(y, x); + v[7] = _mm_add_epi32(v[7], rnding); + v[7] = _mm_srai_epi32(v[7], bit); + + v[8] = u[8]; + v[9] = u[9]; + + y = _mm_mullo_epi32(u[10], cospi32); + x = _mm_mullo_epi32(u[11], cospi32); + v[10] = _mm_add_epi32(y, x); + v[10] = _mm_add_epi32(v[10], rnding); + v[10] = _mm_srai_epi32(v[10], bit); + + v[11] = _mm_sub_epi32(y, x); + v[11] = _mm_add_epi32(v[11], rnding); + v[11] = _mm_srai_epi32(v[11], bit); + + v[12] = u[12]; + v[13] = u[13]; + + y = _mm_mullo_epi32(u[14], cospi32); + x = _mm_mullo_epi32(u[15], cospi32); + v[14] = _mm_add_epi32(y, x); + v[14] = _mm_add_epi32(v[14], rnding); + v[14] = _mm_srai_epi32(v[14], bit); + + v[15] = _mm_sub_epi32(y, x); + v[15] = _mm_add_epi32(v[15], rnding); + v[15] = _mm_srai_epi32(v[15], bit); + + // stage 9 + out[0 * col_num + col] = v[0]; + out[1 * col_num + col] = _mm_sub_epi32(_mm_set1_epi32(0), v[8]); + out[2 * col_num + col] = v[12]; + out[3 * col_num + col] = _mm_sub_epi32(_mm_set1_epi32(0), v[4]); + out[4 * col_num + col] = v[6]; + out[5 * col_num + col] = _mm_sub_epi32(_mm_set1_epi32(0), v[14]); + out[6 * col_num + col] = v[10]; + out[7 * col_num + col] = _mm_sub_epi32(_mm_set1_epi32(0), v[2]); + out[8 * col_num + col] = v[3]; + out[9 * col_num + col] = _mm_sub_epi32(_mm_set1_epi32(0), v[11]); + out[10 * col_num + col] = v[15]; + out[11 * col_num + col] = _mm_sub_epi32(_mm_set1_epi32(0), v[7]); + out[12 * col_num + col] = v[5]; + out[13 * col_num + col] = _mm_sub_epi32(_mm_set1_epi32(0), v[13]); + out[14 * col_num + col] = v[9]; + out[15 * col_num + col] = _mm_sub_epi32(_mm_set1_epi32(0), v[1]); + } +} + +static void col_txfm_16x16_rounding(__m128i *in, int shift) { + // Note: + // We split 16x16 rounding into 4 sections of 8x8 rounding, + // instead of 4 columns + col_txfm_8x8_rounding(&in[0], shift); + col_txfm_8x8_rounding(&in[16], shift); + col_txfm_8x8_rounding(&in[32], shift); + col_txfm_8x8_rounding(&in[48], shift); +} + +static void write_buffer_16x16(const __m128i *in, tran_low_t *output) { + const int size_8x8 = 16 * 4; + write_buffer_8x8(&in[0], output); + output += size_8x8; + write_buffer_8x8(&in[16], output); + output += size_8x8; + write_buffer_8x8(&in[32], output); + output += size_8x8; + write_buffer_8x8(&in[48], output); +} + +void av1_fwd_txfm2d_16x16_sse4_1(const int16_t *input, int32_t *coeff, + int stride, int tx_type, int bd) { + __m128i in[64], out[64]; + const TXFM_2D_CFG *cfg = NULL; + + switch (tx_type) { + case DCT_DCT: + cfg = &fwd_txfm_2d_cfg_dct_dct_16; + load_buffer_16x16(input, in, stride, 0, 0, cfg->shift[0]); + fdct16x16_sse4_1(in, out, cfg->cos_bit_col[0]); + col_txfm_16x16_rounding(out, -cfg->shift[1]); + transpose_16x16(out, in); + fdct16x16_sse4_1(in, out, cfg->cos_bit_row[0]); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case ADST_DCT: + cfg = &fwd_txfm_2d_cfg_adst_dct_16; + load_buffer_16x16(input, in, stride, 0, 0, cfg->shift[0]); + fadst16x16_sse4_1(in, out, cfg->cos_bit_col[0]); + col_txfm_16x16_rounding(out, -cfg->shift[1]); + transpose_16x16(out, in); + fdct16x16_sse4_1(in, out, cfg->cos_bit_row[0]); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case DCT_ADST: + cfg = &fwd_txfm_2d_cfg_dct_adst_16; + load_buffer_16x16(input, in, stride, 0, 0, cfg->shift[0]); + fdct16x16_sse4_1(in, out, cfg->cos_bit_col[0]); + col_txfm_16x16_rounding(out, -cfg->shift[1]); + transpose_16x16(out, in); + fadst16x16_sse4_1(in, out, cfg->cos_bit_row[0]); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case ADST_ADST: + cfg = &fwd_txfm_2d_cfg_adst_adst_16; + load_buffer_16x16(input, in, stride, 0, 0, cfg->shift[0]); + fadst16x16_sse4_1(in, out, cfg->cos_bit_col[0]); + col_txfm_16x16_rounding(out, -cfg->shift[1]); + transpose_16x16(out, in); + fadst16x16_sse4_1(in, out, cfg->cos_bit_row[0]); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + cfg = &fwd_txfm_2d_cfg_adst_dct_16; + load_buffer_16x16(input, in, stride, 1, 0, cfg->shift[0]); + fadst16x16_sse4_1(in, out, cfg->cos_bit_col[0]); + col_txfm_16x16_rounding(out, -cfg->shift[1]); + transpose_16x16(out, in); + fdct16x16_sse4_1(in, out, cfg->cos_bit_row[0]); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case DCT_FLIPADST: + cfg = &fwd_txfm_2d_cfg_dct_adst_16; + load_buffer_16x16(input, in, stride, 0, 1, cfg->shift[0]); + fdct16x16_sse4_1(in, out, cfg->cos_bit_col[0]); + col_txfm_16x16_rounding(out, -cfg->shift[1]); + transpose_16x16(out, in); + fadst16x16_sse4_1(in, out, cfg->cos_bit_row[0]); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case FLIPADST_FLIPADST: + cfg = &fwd_txfm_2d_cfg_adst_adst_16; + load_buffer_16x16(input, in, stride, 1, 1, cfg->shift[0]); + fadst16x16_sse4_1(in, out, cfg->cos_bit_col[0]); + col_txfm_16x16_rounding(out, -cfg->shift[1]); + transpose_16x16(out, in); + fadst16x16_sse4_1(in, out, cfg->cos_bit_row[0]); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case ADST_FLIPADST: + cfg = &fwd_txfm_2d_cfg_adst_adst_16; + load_buffer_16x16(input, in, stride, 0, 1, cfg->shift[0]); + fadst16x16_sse4_1(in, out, cfg->cos_bit_col[0]); + col_txfm_16x16_rounding(out, -cfg->shift[1]); + transpose_16x16(out, in); + fadst16x16_sse4_1(in, out, cfg->cos_bit_row[0]); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case FLIPADST_ADST: + cfg = &fwd_txfm_2d_cfg_adst_adst_16; + load_buffer_16x16(input, in, stride, 1, 0, cfg->shift[0]); + fadst16x16_sse4_1(in, out, cfg->cos_bit_col[0]); + col_txfm_16x16_rounding(out, -cfg->shift[1]); + transpose_16x16(out, in); + fadst16x16_sse4_1(in, out, cfg->cos_bit_row[0]); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; +#endif // CONFIG_EXT_TX + default: assert(0); + } + (void)bd; +} diff --git a/third_party/aom/av1/encoder/x86/hybrid_fwd_txfm_avx2.c b/third_party/aom/av1/encoder/x86/hybrid_fwd_txfm_avx2.c new file mode 100644 index 0000000000..198e4e4c4c --- /dev/null +++ b/third_party/aom/av1/encoder/x86/hybrid_fwd_txfm_avx2.c @@ -0,0 +1,1678 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // avx2 + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" + +#include "aom_dsp/x86/fwd_txfm_avx2.h" +#include "aom_dsp/txfm_common.h" +#include "aom_dsp/x86/txfm_common_avx2.h" + +static int32_t get_16x16_sum(const int16_t *input, int stride) { + __m256i r0, r1, r2, r3, u0, u1; + __m256i zero = _mm256_setzero_si256(); + __m256i sum = _mm256_setzero_si256(); + const int16_t *blockBound = input + (stride << 4); + __m128i v0, v1; + + while (input < blockBound) { + r0 = _mm256_loadu_si256((__m256i const *)input); + r1 = _mm256_loadu_si256((__m256i const *)(input + stride)); + r2 = _mm256_loadu_si256((__m256i const *)(input + 2 * stride)); + r3 = _mm256_loadu_si256((__m256i const *)(input + 3 * stride)); + + u0 = _mm256_add_epi16(r0, r1); + u1 = _mm256_add_epi16(r2, r3); + sum = _mm256_add_epi16(sum, u0); + sum = _mm256_add_epi16(sum, u1); + + input += stride << 2; + } + + // unpack 16 int16_t into 2x8 int32_t + u0 = _mm256_unpacklo_epi16(zero, sum); + u1 = _mm256_unpackhi_epi16(zero, sum); + u0 = _mm256_srai_epi32(u0, 16); + u1 = _mm256_srai_epi32(u1, 16); + sum = _mm256_add_epi32(u0, u1); + + u0 = _mm256_srli_si256(sum, 8); + u1 = _mm256_add_epi32(sum, u0); + + v0 = _mm_add_epi32(_mm256_extracti128_si256(u1, 1), + _mm256_castsi256_si128(u1)); + v1 = _mm_srli_si128(v0, 4); + v0 = _mm_add_epi32(v0, v1); + return (int32_t)_mm_extract_epi32(v0, 0); +} + +void aom_fdct16x16_1_avx2(const int16_t *input, tran_low_t *output, + int stride) { + int32_t dc = get_16x16_sum(input, stride); + output[0] = (tran_low_t)(dc >> 1); + _mm256_zeroupper(); +} + +static INLINE void load_buffer_16x16(const int16_t *input, int stride, + int flipud, int fliplr, __m256i *in) { + if (!flipud) { + in[0] = _mm256_loadu_si256((const __m256i *)(input + 0 * stride)); + in[1] = _mm256_loadu_si256((const __m256i *)(input + 1 * stride)); + in[2] = _mm256_loadu_si256((const __m256i *)(input + 2 * stride)); + in[3] = _mm256_loadu_si256((const __m256i *)(input + 3 * stride)); + in[4] = _mm256_loadu_si256((const __m256i *)(input + 4 * stride)); + in[5] = _mm256_loadu_si256((const __m256i *)(input + 5 * stride)); + in[6] = _mm256_loadu_si256((const __m256i *)(input + 6 * stride)); + in[7] = _mm256_loadu_si256((const __m256i *)(input + 7 * stride)); + in[8] = _mm256_loadu_si256((const __m256i *)(input + 8 * stride)); + in[9] = _mm256_loadu_si256((const __m256i *)(input + 9 * stride)); + in[10] = _mm256_loadu_si256((const __m256i *)(input + 10 * stride)); + in[11] = _mm256_loadu_si256((const __m256i *)(input + 11 * stride)); + in[12] = _mm256_loadu_si256((const __m256i *)(input + 12 * stride)); + in[13] = _mm256_loadu_si256((const __m256i *)(input + 13 * stride)); + in[14] = _mm256_loadu_si256((const __m256i *)(input + 14 * stride)); + in[15] = _mm256_loadu_si256((const __m256i *)(input + 15 * stride)); + } else { + in[0] = _mm256_loadu_si256((const __m256i *)(input + 15 * stride)); + in[1] = _mm256_loadu_si256((const __m256i *)(input + 14 * stride)); + in[2] = _mm256_loadu_si256((const __m256i *)(input + 13 * stride)); + in[3] = _mm256_loadu_si256((const __m256i *)(input + 12 * stride)); + in[4] = _mm256_loadu_si256((const __m256i *)(input + 11 * stride)); + in[5] = _mm256_loadu_si256((const __m256i *)(input + 10 * stride)); + in[6] = _mm256_loadu_si256((const __m256i *)(input + 9 * stride)); + in[7] = _mm256_loadu_si256((const __m256i *)(input + 8 * stride)); + in[8] = _mm256_loadu_si256((const __m256i *)(input + 7 * stride)); + in[9] = _mm256_loadu_si256((const __m256i *)(input + 6 * stride)); + in[10] = _mm256_loadu_si256((const __m256i *)(input + 5 * stride)); + in[11] = _mm256_loadu_si256((const __m256i *)(input + 4 * stride)); + in[12] = _mm256_loadu_si256((const __m256i *)(input + 3 * stride)); + in[13] = _mm256_loadu_si256((const __m256i *)(input + 2 * stride)); + in[14] = _mm256_loadu_si256((const __m256i *)(input + 1 * stride)); + in[15] = _mm256_loadu_si256((const __m256i *)(input + 0 * stride)); + } + + if (fliplr) { + mm256_reverse_epi16(&in[0]); + mm256_reverse_epi16(&in[1]); + mm256_reverse_epi16(&in[2]); + mm256_reverse_epi16(&in[3]); + mm256_reverse_epi16(&in[4]); + mm256_reverse_epi16(&in[5]); + mm256_reverse_epi16(&in[6]); + mm256_reverse_epi16(&in[7]); + mm256_reverse_epi16(&in[8]); + mm256_reverse_epi16(&in[9]); + mm256_reverse_epi16(&in[10]); + mm256_reverse_epi16(&in[11]); + mm256_reverse_epi16(&in[12]); + mm256_reverse_epi16(&in[13]); + mm256_reverse_epi16(&in[14]); + mm256_reverse_epi16(&in[15]); + } + + in[0] = _mm256_slli_epi16(in[0], 2); + in[1] = _mm256_slli_epi16(in[1], 2); + in[2] = _mm256_slli_epi16(in[2], 2); + in[3] = _mm256_slli_epi16(in[3], 2); + in[4] = _mm256_slli_epi16(in[4], 2); + in[5] = _mm256_slli_epi16(in[5], 2); + in[6] = _mm256_slli_epi16(in[6], 2); + in[7] = _mm256_slli_epi16(in[7], 2); + in[8] = _mm256_slli_epi16(in[8], 2); + in[9] = _mm256_slli_epi16(in[9], 2); + in[10] = _mm256_slli_epi16(in[10], 2); + in[11] = _mm256_slli_epi16(in[11], 2); + in[12] = _mm256_slli_epi16(in[12], 2); + in[13] = _mm256_slli_epi16(in[13], 2); + in[14] = _mm256_slli_epi16(in[14], 2); + in[15] = _mm256_slli_epi16(in[15], 2); +} + +static INLINE void write_buffer_16x16(const __m256i *in, tran_low_t *output) { + int i; + for (i = 0; i < 16; ++i) { + storeu_output_avx2(&in[i], output + (i << 4)); + } +} + +static void right_shift_16x16(__m256i *in) { + const __m256i one = _mm256_set1_epi16(1); + __m256i s0 = _mm256_srai_epi16(in[0], 15); + __m256i s1 = _mm256_srai_epi16(in[1], 15); + __m256i s2 = _mm256_srai_epi16(in[2], 15); + __m256i s3 = _mm256_srai_epi16(in[3], 15); + __m256i s4 = _mm256_srai_epi16(in[4], 15); + __m256i s5 = _mm256_srai_epi16(in[5], 15); + __m256i s6 = _mm256_srai_epi16(in[6], 15); + __m256i s7 = _mm256_srai_epi16(in[7], 15); + __m256i s8 = _mm256_srai_epi16(in[8], 15); + __m256i s9 = _mm256_srai_epi16(in[9], 15); + __m256i s10 = _mm256_srai_epi16(in[10], 15); + __m256i s11 = _mm256_srai_epi16(in[11], 15); + __m256i s12 = _mm256_srai_epi16(in[12], 15); + __m256i s13 = _mm256_srai_epi16(in[13], 15); + __m256i s14 = _mm256_srai_epi16(in[14], 15); + __m256i s15 = _mm256_srai_epi16(in[15], 15); + + in[0] = _mm256_add_epi16(in[0], one); + in[1] = _mm256_add_epi16(in[1], one); + in[2] = _mm256_add_epi16(in[2], one); + in[3] = _mm256_add_epi16(in[3], one); + in[4] = _mm256_add_epi16(in[4], one); + in[5] = _mm256_add_epi16(in[5], one); + in[6] = _mm256_add_epi16(in[6], one); + in[7] = _mm256_add_epi16(in[7], one); + in[8] = _mm256_add_epi16(in[8], one); + in[9] = _mm256_add_epi16(in[9], one); + in[10] = _mm256_add_epi16(in[10], one); + in[11] = _mm256_add_epi16(in[11], one); + in[12] = _mm256_add_epi16(in[12], one); + in[13] = _mm256_add_epi16(in[13], one); + in[14] = _mm256_add_epi16(in[14], one); + in[15] = _mm256_add_epi16(in[15], one); + + in[0] = _mm256_sub_epi16(in[0], s0); + in[1] = _mm256_sub_epi16(in[1], s1); + in[2] = _mm256_sub_epi16(in[2], s2); + in[3] = _mm256_sub_epi16(in[3], s3); + in[4] = _mm256_sub_epi16(in[4], s4); + in[5] = _mm256_sub_epi16(in[5], s5); + in[6] = _mm256_sub_epi16(in[6], s6); + in[7] = _mm256_sub_epi16(in[7], s7); + in[8] = _mm256_sub_epi16(in[8], s8); + in[9] = _mm256_sub_epi16(in[9], s9); + in[10] = _mm256_sub_epi16(in[10], s10); + in[11] = _mm256_sub_epi16(in[11], s11); + in[12] = _mm256_sub_epi16(in[12], s12); + in[13] = _mm256_sub_epi16(in[13], s13); + in[14] = _mm256_sub_epi16(in[14], s14); + in[15] = _mm256_sub_epi16(in[15], s15); + + in[0] = _mm256_srai_epi16(in[0], 2); + in[1] = _mm256_srai_epi16(in[1], 2); + in[2] = _mm256_srai_epi16(in[2], 2); + in[3] = _mm256_srai_epi16(in[3], 2); + in[4] = _mm256_srai_epi16(in[4], 2); + in[5] = _mm256_srai_epi16(in[5], 2); + in[6] = _mm256_srai_epi16(in[6], 2); + in[7] = _mm256_srai_epi16(in[7], 2); + in[8] = _mm256_srai_epi16(in[8], 2); + in[9] = _mm256_srai_epi16(in[9], 2); + in[10] = _mm256_srai_epi16(in[10], 2); + in[11] = _mm256_srai_epi16(in[11], 2); + in[12] = _mm256_srai_epi16(in[12], 2); + in[13] = _mm256_srai_epi16(in[13], 2); + in[14] = _mm256_srai_epi16(in[14], 2); + in[15] = _mm256_srai_epi16(in[15], 2); +} + +static void fdct16_avx2(__m256i *in) { + // sequence: cospi_L_H = pairs(L, H) and L first + const __m256i cospi_p16_m16 = pair256_set_epi16(cospi_16_64, -cospi_16_64); + const __m256i cospi_p16_p16 = pair256_set_epi16(cospi_16_64, cospi_16_64); + const __m256i cospi_p24_p08 = pair256_set_epi16(cospi_24_64, cospi_8_64); + const __m256i cospi_m08_p24 = pair256_set_epi16(-cospi_8_64, cospi_24_64); + const __m256i cospi_m24_m08 = pair256_set_epi16(-cospi_24_64, -cospi_8_64); + + const __m256i cospi_p28_p04 = pair256_set_epi16(cospi_28_64, cospi_4_64); + const __m256i cospi_m04_p28 = pair256_set_epi16(-cospi_4_64, cospi_28_64); + const __m256i cospi_p12_p20 = pair256_set_epi16(cospi_12_64, cospi_20_64); + const __m256i cospi_m20_p12 = pair256_set_epi16(-cospi_20_64, cospi_12_64); + + const __m256i cospi_p30_p02 = pair256_set_epi16(cospi_30_64, cospi_2_64); + const __m256i cospi_m02_p30 = pair256_set_epi16(-cospi_2_64, cospi_30_64); + + const __m256i cospi_p14_p18 = pair256_set_epi16(cospi_14_64, cospi_18_64); + const __m256i cospi_m18_p14 = pair256_set_epi16(-cospi_18_64, cospi_14_64); + + const __m256i cospi_p22_p10 = pair256_set_epi16(cospi_22_64, cospi_10_64); + const __m256i cospi_m10_p22 = pair256_set_epi16(-cospi_10_64, cospi_22_64); + + const __m256i cospi_p06_p26 = pair256_set_epi16(cospi_6_64, cospi_26_64); + const __m256i cospi_m26_p06 = pair256_set_epi16(-cospi_26_64, cospi_6_64); + + __m256i u0, u1, u2, u3, u4, u5, u6, u7; + __m256i s0, s1, s2, s3, s4, s5, s6, s7; + __m256i t0, t1, t2, t3, t4, t5, t6, t7; + __m256i v0, v1, v2, v3; + __m256i x0, x1; + + // 0, 4, 8, 12 + u0 = _mm256_add_epi16(in[0], in[15]); + u1 = _mm256_add_epi16(in[1], in[14]); + u2 = _mm256_add_epi16(in[2], in[13]); + u3 = _mm256_add_epi16(in[3], in[12]); + u4 = _mm256_add_epi16(in[4], in[11]); + u5 = _mm256_add_epi16(in[5], in[10]); + u6 = _mm256_add_epi16(in[6], in[9]); + u7 = _mm256_add_epi16(in[7], in[8]); + + s0 = _mm256_add_epi16(u0, u7); + s1 = _mm256_add_epi16(u1, u6); + s2 = _mm256_add_epi16(u2, u5); + s3 = _mm256_add_epi16(u3, u4); + + // 0, 8 + v0 = _mm256_add_epi16(s0, s3); + v1 = _mm256_add_epi16(s1, s2); + + x0 = _mm256_unpacklo_epi16(v0, v1); + x1 = _mm256_unpackhi_epi16(v0, v1); + + t0 = butter_fly(x0, x1, cospi_p16_p16); + t1 = butter_fly(x0, x1, cospi_p16_m16); + + // 4, 12 + v0 = _mm256_sub_epi16(s1, s2); + v1 = _mm256_sub_epi16(s0, s3); + + x0 = _mm256_unpacklo_epi16(v0, v1); + x1 = _mm256_unpackhi_epi16(v0, v1); + + t2 = butter_fly(x0, x1, cospi_p24_p08); + t3 = butter_fly(x0, x1, cospi_m08_p24); + + // 2, 6, 10, 14 + s0 = _mm256_sub_epi16(u3, u4); + s1 = _mm256_sub_epi16(u2, u5); + s2 = _mm256_sub_epi16(u1, u6); + s3 = _mm256_sub_epi16(u0, u7); + + v0 = s0; // output[4] + v3 = s3; // output[7] + + x0 = _mm256_unpacklo_epi16(s2, s1); + x1 = _mm256_unpackhi_epi16(s2, s1); + + v2 = butter_fly(x0, x1, cospi_p16_p16); // output[5] + v1 = butter_fly(x0, x1, cospi_p16_m16); // output[6] + + s0 = _mm256_add_epi16(v0, v1); // step[4] + s1 = _mm256_sub_epi16(v0, v1); // step[5] + s2 = _mm256_sub_epi16(v3, v2); // step[6] + s3 = _mm256_add_epi16(v3, v2); // step[7] + + // 2, 14 + x0 = _mm256_unpacklo_epi16(s0, s3); + x1 = _mm256_unpackhi_epi16(s0, s3); + + t4 = butter_fly(x0, x1, cospi_p28_p04); + t5 = butter_fly(x0, x1, cospi_m04_p28); + + // 10, 6 + x0 = _mm256_unpacklo_epi16(s1, s2); + x1 = _mm256_unpackhi_epi16(s1, s2); + t6 = butter_fly(x0, x1, cospi_p12_p20); + t7 = butter_fly(x0, x1, cospi_m20_p12); + + // 1, 3, 5, 7, 9, 11, 13, 15 + s0 = _mm256_sub_epi16(in[7], in[8]); // step[8] + s1 = _mm256_sub_epi16(in[6], in[9]); // step[9] + u2 = _mm256_sub_epi16(in[5], in[10]); + u3 = _mm256_sub_epi16(in[4], in[11]); + u4 = _mm256_sub_epi16(in[3], in[12]); + u5 = _mm256_sub_epi16(in[2], in[13]); + s6 = _mm256_sub_epi16(in[1], in[14]); // step[14] + s7 = _mm256_sub_epi16(in[0], in[15]); // step[15] + + in[0] = t0; + in[8] = t1; + in[4] = t2; + in[12] = t3; + in[2] = t4; + in[14] = t5; + in[10] = t6; + in[6] = t7; + + x0 = _mm256_unpacklo_epi16(u5, u2); + x1 = _mm256_unpackhi_epi16(u5, u2); + + s2 = butter_fly(x0, x1, cospi_p16_p16); // step[13] + s5 = butter_fly(x0, x1, cospi_p16_m16); // step[10] + + x0 = _mm256_unpacklo_epi16(u4, u3); + x1 = _mm256_unpackhi_epi16(u4, u3); + + s3 = butter_fly(x0, x1, cospi_p16_p16); // step[12] + s4 = butter_fly(x0, x1, cospi_p16_m16); // step[11] + + u0 = _mm256_add_epi16(s0, s4); // output[8] + u1 = _mm256_add_epi16(s1, s5); + u2 = _mm256_sub_epi16(s1, s5); + u3 = _mm256_sub_epi16(s0, s4); + u4 = _mm256_sub_epi16(s7, s3); + u5 = _mm256_sub_epi16(s6, s2); + u6 = _mm256_add_epi16(s6, s2); + u7 = _mm256_add_epi16(s7, s3); + + // stage 4 + s0 = u0; + s3 = u3; + s4 = u4; + s7 = u7; + + x0 = _mm256_unpacklo_epi16(u1, u6); + x1 = _mm256_unpackhi_epi16(u1, u6); + + s1 = butter_fly(x0, x1, cospi_m08_p24); + s6 = butter_fly(x0, x1, cospi_p24_p08); + + x0 = _mm256_unpacklo_epi16(u2, u5); + x1 = _mm256_unpackhi_epi16(u2, u5); + + s2 = butter_fly(x0, x1, cospi_m24_m08); + s5 = butter_fly(x0, x1, cospi_m08_p24); + + // stage 5 + u0 = _mm256_add_epi16(s0, s1); + u1 = _mm256_sub_epi16(s0, s1); + u2 = _mm256_sub_epi16(s3, s2); + u3 = _mm256_add_epi16(s3, s2); + u4 = _mm256_add_epi16(s4, s5); + u5 = _mm256_sub_epi16(s4, s5); + u6 = _mm256_sub_epi16(s7, s6); + u7 = _mm256_add_epi16(s7, s6); + + // stage 6 + x0 = _mm256_unpacklo_epi16(u0, u7); + x1 = _mm256_unpackhi_epi16(u0, u7); + in[1] = butter_fly(x0, x1, cospi_p30_p02); + in[15] = butter_fly(x0, x1, cospi_m02_p30); + + x0 = _mm256_unpacklo_epi16(u1, u6); + x1 = _mm256_unpackhi_epi16(u1, u6); + in[9] = butter_fly(x0, x1, cospi_p14_p18); + in[7] = butter_fly(x0, x1, cospi_m18_p14); + + x0 = _mm256_unpacklo_epi16(u2, u5); + x1 = _mm256_unpackhi_epi16(u2, u5); + in[5] = butter_fly(x0, x1, cospi_p22_p10); + in[11] = butter_fly(x0, x1, cospi_m10_p22); + + x0 = _mm256_unpacklo_epi16(u3, u4); + x1 = _mm256_unpackhi_epi16(u3, u4); + in[13] = butter_fly(x0, x1, cospi_p06_p26); + in[3] = butter_fly(x0, x1, cospi_m26_p06); +} + +void fadst16_avx2(__m256i *in) { + const __m256i cospi_p01_p31 = pair256_set_epi16(cospi_1_64, cospi_31_64); + const __m256i cospi_p31_m01 = pair256_set_epi16(cospi_31_64, -cospi_1_64); + const __m256i cospi_p05_p27 = pair256_set_epi16(cospi_5_64, cospi_27_64); + const __m256i cospi_p27_m05 = pair256_set_epi16(cospi_27_64, -cospi_5_64); + const __m256i cospi_p09_p23 = pair256_set_epi16(cospi_9_64, cospi_23_64); + const __m256i cospi_p23_m09 = pair256_set_epi16(cospi_23_64, -cospi_9_64); + const __m256i cospi_p13_p19 = pair256_set_epi16(cospi_13_64, cospi_19_64); + const __m256i cospi_p19_m13 = pair256_set_epi16(cospi_19_64, -cospi_13_64); + const __m256i cospi_p17_p15 = pair256_set_epi16(cospi_17_64, cospi_15_64); + const __m256i cospi_p15_m17 = pair256_set_epi16(cospi_15_64, -cospi_17_64); + const __m256i cospi_p21_p11 = pair256_set_epi16(cospi_21_64, cospi_11_64); + const __m256i cospi_p11_m21 = pair256_set_epi16(cospi_11_64, -cospi_21_64); + const __m256i cospi_p25_p07 = pair256_set_epi16(cospi_25_64, cospi_7_64); + const __m256i cospi_p07_m25 = pair256_set_epi16(cospi_7_64, -cospi_25_64); + const __m256i cospi_p29_p03 = pair256_set_epi16(cospi_29_64, cospi_3_64); + const __m256i cospi_p03_m29 = pair256_set_epi16(cospi_3_64, -cospi_29_64); + const __m256i cospi_p04_p28 = pair256_set_epi16(cospi_4_64, cospi_28_64); + const __m256i cospi_p28_m04 = pair256_set_epi16(cospi_28_64, -cospi_4_64); + const __m256i cospi_p20_p12 = pair256_set_epi16(cospi_20_64, cospi_12_64); + const __m256i cospi_p12_m20 = pair256_set_epi16(cospi_12_64, -cospi_20_64); + const __m256i cospi_m28_p04 = pair256_set_epi16(-cospi_28_64, cospi_4_64); + const __m256i cospi_m12_p20 = pair256_set_epi16(-cospi_12_64, cospi_20_64); + const __m256i cospi_p08_p24 = pair256_set_epi16(cospi_8_64, cospi_24_64); + const __m256i cospi_p24_m08 = pair256_set_epi16(cospi_24_64, -cospi_8_64); + const __m256i cospi_m24_p08 = pair256_set_epi16(-cospi_24_64, cospi_8_64); + const __m256i cospi_m16_m16 = _mm256_set1_epi16((int16_t)-cospi_16_64); + const __m256i cospi_p16_p16 = _mm256_set1_epi16((int16_t)cospi_16_64); + const __m256i cospi_p16_m16 = pair256_set_epi16(cospi_16_64, -cospi_16_64); + const __m256i cospi_m16_p16 = pair256_set_epi16(-cospi_16_64, cospi_16_64); + const __m256i zero = _mm256_setzero_si256(); + const __m256i dct_rounding = _mm256_set1_epi32(DCT_CONST_ROUNDING); + __m256i s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, s15; + __m256i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + __m256i u0, u1, u2, u3, u4, u5, u6, u7, u8, u9, u10, u11, u12, u13, u14, u15; + __m256i v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15; + __m256i y0, y1; + + // stage 1, s takes low 256 bits; x takes high 256 bits + y0 = _mm256_unpacklo_epi16(in[15], in[0]); + y1 = _mm256_unpackhi_epi16(in[15], in[0]); + s0 = _mm256_madd_epi16(y0, cospi_p01_p31); + x0 = _mm256_madd_epi16(y1, cospi_p01_p31); + s1 = _mm256_madd_epi16(y0, cospi_p31_m01); + x1 = _mm256_madd_epi16(y1, cospi_p31_m01); + + y0 = _mm256_unpacklo_epi16(in[13], in[2]); + y1 = _mm256_unpackhi_epi16(in[13], in[2]); + s2 = _mm256_madd_epi16(y0, cospi_p05_p27); + x2 = _mm256_madd_epi16(y1, cospi_p05_p27); + s3 = _mm256_madd_epi16(y0, cospi_p27_m05); + x3 = _mm256_madd_epi16(y1, cospi_p27_m05); + + y0 = _mm256_unpacklo_epi16(in[11], in[4]); + y1 = _mm256_unpackhi_epi16(in[11], in[4]); + s4 = _mm256_madd_epi16(y0, cospi_p09_p23); + x4 = _mm256_madd_epi16(y1, cospi_p09_p23); + s5 = _mm256_madd_epi16(y0, cospi_p23_m09); + x5 = _mm256_madd_epi16(y1, cospi_p23_m09); + + y0 = _mm256_unpacklo_epi16(in[9], in[6]); + y1 = _mm256_unpackhi_epi16(in[9], in[6]); + s6 = _mm256_madd_epi16(y0, cospi_p13_p19); + x6 = _mm256_madd_epi16(y1, cospi_p13_p19); + s7 = _mm256_madd_epi16(y0, cospi_p19_m13); + x7 = _mm256_madd_epi16(y1, cospi_p19_m13); + + y0 = _mm256_unpacklo_epi16(in[7], in[8]); + y1 = _mm256_unpackhi_epi16(in[7], in[8]); + s8 = _mm256_madd_epi16(y0, cospi_p17_p15); + x8 = _mm256_madd_epi16(y1, cospi_p17_p15); + s9 = _mm256_madd_epi16(y0, cospi_p15_m17); + x9 = _mm256_madd_epi16(y1, cospi_p15_m17); + + y0 = _mm256_unpacklo_epi16(in[5], in[10]); + y1 = _mm256_unpackhi_epi16(in[5], in[10]); + s10 = _mm256_madd_epi16(y0, cospi_p21_p11); + x10 = _mm256_madd_epi16(y1, cospi_p21_p11); + s11 = _mm256_madd_epi16(y0, cospi_p11_m21); + x11 = _mm256_madd_epi16(y1, cospi_p11_m21); + + y0 = _mm256_unpacklo_epi16(in[3], in[12]); + y1 = _mm256_unpackhi_epi16(in[3], in[12]); + s12 = _mm256_madd_epi16(y0, cospi_p25_p07); + x12 = _mm256_madd_epi16(y1, cospi_p25_p07); + s13 = _mm256_madd_epi16(y0, cospi_p07_m25); + x13 = _mm256_madd_epi16(y1, cospi_p07_m25); + + y0 = _mm256_unpacklo_epi16(in[1], in[14]); + y1 = _mm256_unpackhi_epi16(in[1], in[14]); + s14 = _mm256_madd_epi16(y0, cospi_p29_p03); + x14 = _mm256_madd_epi16(y1, cospi_p29_p03); + s15 = _mm256_madd_epi16(y0, cospi_p03_m29); + x15 = _mm256_madd_epi16(y1, cospi_p03_m29); + + // u takes low 256 bits; v takes high 256 bits + u0 = _mm256_add_epi32(s0, s8); + u1 = _mm256_add_epi32(s1, s9); + u2 = _mm256_add_epi32(s2, s10); + u3 = _mm256_add_epi32(s3, s11); + u4 = _mm256_add_epi32(s4, s12); + u5 = _mm256_add_epi32(s5, s13); + u6 = _mm256_add_epi32(s6, s14); + u7 = _mm256_add_epi32(s7, s15); + + u8 = _mm256_sub_epi32(s0, s8); + u9 = _mm256_sub_epi32(s1, s9); + u10 = _mm256_sub_epi32(s2, s10); + u11 = _mm256_sub_epi32(s3, s11); + u12 = _mm256_sub_epi32(s4, s12); + u13 = _mm256_sub_epi32(s5, s13); + u14 = _mm256_sub_epi32(s6, s14); + u15 = _mm256_sub_epi32(s7, s15); + + v0 = _mm256_add_epi32(x0, x8); + v1 = _mm256_add_epi32(x1, x9); + v2 = _mm256_add_epi32(x2, x10); + v3 = _mm256_add_epi32(x3, x11); + v4 = _mm256_add_epi32(x4, x12); + v5 = _mm256_add_epi32(x5, x13); + v6 = _mm256_add_epi32(x6, x14); + v7 = _mm256_add_epi32(x7, x15); + + v8 = _mm256_sub_epi32(x0, x8); + v9 = _mm256_sub_epi32(x1, x9); + v10 = _mm256_sub_epi32(x2, x10); + v11 = _mm256_sub_epi32(x3, x11); + v12 = _mm256_sub_epi32(x4, x12); + v13 = _mm256_sub_epi32(x5, x13); + v14 = _mm256_sub_epi32(x6, x14); + v15 = _mm256_sub_epi32(x7, x15); + + // low 256 bits rounding + u8 = _mm256_add_epi32(u8, dct_rounding); + u9 = _mm256_add_epi32(u9, dct_rounding); + u10 = _mm256_add_epi32(u10, dct_rounding); + u11 = _mm256_add_epi32(u11, dct_rounding); + u12 = _mm256_add_epi32(u12, dct_rounding); + u13 = _mm256_add_epi32(u13, dct_rounding); + u14 = _mm256_add_epi32(u14, dct_rounding); + u15 = _mm256_add_epi32(u15, dct_rounding); + + u8 = _mm256_srai_epi32(u8, DCT_CONST_BITS); + u9 = _mm256_srai_epi32(u9, DCT_CONST_BITS); + u10 = _mm256_srai_epi32(u10, DCT_CONST_BITS); + u11 = _mm256_srai_epi32(u11, DCT_CONST_BITS); + u12 = _mm256_srai_epi32(u12, DCT_CONST_BITS); + u13 = _mm256_srai_epi32(u13, DCT_CONST_BITS); + u14 = _mm256_srai_epi32(u14, DCT_CONST_BITS); + u15 = _mm256_srai_epi32(u15, DCT_CONST_BITS); + + // high 256 bits rounding + v8 = _mm256_add_epi32(v8, dct_rounding); + v9 = _mm256_add_epi32(v9, dct_rounding); + v10 = _mm256_add_epi32(v10, dct_rounding); + v11 = _mm256_add_epi32(v11, dct_rounding); + v12 = _mm256_add_epi32(v12, dct_rounding); + v13 = _mm256_add_epi32(v13, dct_rounding); + v14 = _mm256_add_epi32(v14, dct_rounding); + v15 = _mm256_add_epi32(v15, dct_rounding); + + v8 = _mm256_srai_epi32(v8, DCT_CONST_BITS); + v9 = _mm256_srai_epi32(v9, DCT_CONST_BITS); + v10 = _mm256_srai_epi32(v10, DCT_CONST_BITS); + v11 = _mm256_srai_epi32(v11, DCT_CONST_BITS); + v12 = _mm256_srai_epi32(v12, DCT_CONST_BITS); + v13 = _mm256_srai_epi32(v13, DCT_CONST_BITS); + v14 = _mm256_srai_epi32(v14, DCT_CONST_BITS); + v15 = _mm256_srai_epi32(v15, DCT_CONST_BITS); + + // Saturation pack 32-bit to 16-bit + x8 = _mm256_packs_epi32(u8, v8); + x9 = _mm256_packs_epi32(u9, v9); + x10 = _mm256_packs_epi32(u10, v10); + x11 = _mm256_packs_epi32(u11, v11); + x12 = _mm256_packs_epi32(u12, v12); + x13 = _mm256_packs_epi32(u13, v13); + x14 = _mm256_packs_epi32(u14, v14); + x15 = _mm256_packs_epi32(u15, v15); + + // stage 2 + y0 = _mm256_unpacklo_epi16(x8, x9); + y1 = _mm256_unpackhi_epi16(x8, x9); + s8 = _mm256_madd_epi16(y0, cospi_p04_p28); + x8 = _mm256_madd_epi16(y1, cospi_p04_p28); + s9 = _mm256_madd_epi16(y0, cospi_p28_m04); + x9 = _mm256_madd_epi16(y1, cospi_p28_m04); + + y0 = _mm256_unpacklo_epi16(x10, x11); + y1 = _mm256_unpackhi_epi16(x10, x11); + s10 = _mm256_madd_epi16(y0, cospi_p20_p12); + x10 = _mm256_madd_epi16(y1, cospi_p20_p12); + s11 = _mm256_madd_epi16(y0, cospi_p12_m20); + x11 = _mm256_madd_epi16(y1, cospi_p12_m20); + + y0 = _mm256_unpacklo_epi16(x12, x13); + y1 = _mm256_unpackhi_epi16(x12, x13); + s12 = _mm256_madd_epi16(y0, cospi_m28_p04); + x12 = _mm256_madd_epi16(y1, cospi_m28_p04); + s13 = _mm256_madd_epi16(y0, cospi_p04_p28); + x13 = _mm256_madd_epi16(y1, cospi_p04_p28); + + y0 = _mm256_unpacklo_epi16(x14, x15); + y1 = _mm256_unpackhi_epi16(x14, x15); + s14 = _mm256_madd_epi16(y0, cospi_m12_p20); + x14 = _mm256_madd_epi16(y1, cospi_m12_p20); + s15 = _mm256_madd_epi16(y0, cospi_p20_p12); + x15 = _mm256_madd_epi16(y1, cospi_p20_p12); + + x0 = _mm256_add_epi32(u0, u4); + s0 = _mm256_add_epi32(v0, v4); + x1 = _mm256_add_epi32(u1, u5); + s1 = _mm256_add_epi32(v1, v5); + x2 = _mm256_add_epi32(u2, u6); + s2 = _mm256_add_epi32(v2, v6); + x3 = _mm256_add_epi32(u3, u7); + s3 = _mm256_add_epi32(v3, v7); + + v8 = _mm256_sub_epi32(u0, u4); + v9 = _mm256_sub_epi32(v0, v4); + v10 = _mm256_sub_epi32(u1, u5); + v11 = _mm256_sub_epi32(v1, v5); + v12 = _mm256_sub_epi32(u2, u6); + v13 = _mm256_sub_epi32(v2, v6); + v14 = _mm256_sub_epi32(u3, u7); + v15 = _mm256_sub_epi32(v3, v7); + + v8 = _mm256_add_epi32(v8, dct_rounding); + v9 = _mm256_add_epi32(v9, dct_rounding); + v10 = _mm256_add_epi32(v10, dct_rounding); + v11 = _mm256_add_epi32(v11, dct_rounding); + v12 = _mm256_add_epi32(v12, dct_rounding); + v13 = _mm256_add_epi32(v13, dct_rounding); + v14 = _mm256_add_epi32(v14, dct_rounding); + v15 = _mm256_add_epi32(v15, dct_rounding); + + v8 = _mm256_srai_epi32(v8, DCT_CONST_BITS); + v9 = _mm256_srai_epi32(v9, DCT_CONST_BITS); + v10 = _mm256_srai_epi32(v10, DCT_CONST_BITS); + v11 = _mm256_srai_epi32(v11, DCT_CONST_BITS); + v12 = _mm256_srai_epi32(v12, DCT_CONST_BITS); + v13 = _mm256_srai_epi32(v13, DCT_CONST_BITS); + v14 = _mm256_srai_epi32(v14, DCT_CONST_BITS); + v15 = _mm256_srai_epi32(v15, DCT_CONST_BITS); + + x4 = _mm256_packs_epi32(v8, v9); + x5 = _mm256_packs_epi32(v10, v11); + x6 = _mm256_packs_epi32(v12, v13); + x7 = _mm256_packs_epi32(v14, v15); + + u8 = _mm256_add_epi32(s8, s12); + u9 = _mm256_add_epi32(s9, s13); + u10 = _mm256_add_epi32(s10, s14); + u11 = _mm256_add_epi32(s11, s15); + u12 = _mm256_sub_epi32(s8, s12); + u13 = _mm256_sub_epi32(s9, s13); + u14 = _mm256_sub_epi32(s10, s14); + u15 = _mm256_sub_epi32(s11, s15); + + v8 = _mm256_add_epi32(x8, x12); + v9 = _mm256_add_epi32(x9, x13); + v10 = _mm256_add_epi32(x10, x14); + v11 = _mm256_add_epi32(x11, x15); + v12 = _mm256_sub_epi32(x8, x12); + v13 = _mm256_sub_epi32(x9, x13); + v14 = _mm256_sub_epi32(x10, x14); + v15 = _mm256_sub_epi32(x11, x15); + + u12 = _mm256_add_epi32(u12, dct_rounding); + u13 = _mm256_add_epi32(u13, dct_rounding); + u14 = _mm256_add_epi32(u14, dct_rounding); + u15 = _mm256_add_epi32(u15, dct_rounding); + + u12 = _mm256_srai_epi32(u12, DCT_CONST_BITS); + u13 = _mm256_srai_epi32(u13, DCT_CONST_BITS); + u14 = _mm256_srai_epi32(u14, DCT_CONST_BITS); + u15 = _mm256_srai_epi32(u15, DCT_CONST_BITS); + + v12 = _mm256_add_epi32(v12, dct_rounding); + v13 = _mm256_add_epi32(v13, dct_rounding); + v14 = _mm256_add_epi32(v14, dct_rounding); + v15 = _mm256_add_epi32(v15, dct_rounding); + + v12 = _mm256_srai_epi32(v12, DCT_CONST_BITS); + v13 = _mm256_srai_epi32(v13, DCT_CONST_BITS); + v14 = _mm256_srai_epi32(v14, DCT_CONST_BITS); + v15 = _mm256_srai_epi32(v15, DCT_CONST_BITS); + + x12 = _mm256_packs_epi32(u12, v12); + x13 = _mm256_packs_epi32(u13, v13); + x14 = _mm256_packs_epi32(u14, v14); + x15 = _mm256_packs_epi32(u15, v15); + + // stage 3 + y0 = _mm256_unpacklo_epi16(x4, x5); + y1 = _mm256_unpackhi_epi16(x4, x5); + s4 = _mm256_madd_epi16(y0, cospi_p08_p24); + x4 = _mm256_madd_epi16(y1, cospi_p08_p24); + s5 = _mm256_madd_epi16(y0, cospi_p24_m08); + x5 = _mm256_madd_epi16(y1, cospi_p24_m08); + + y0 = _mm256_unpacklo_epi16(x6, x7); + y1 = _mm256_unpackhi_epi16(x6, x7); + s6 = _mm256_madd_epi16(y0, cospi_m24_p08); + x6 = _mm256_madd_epi16(y1, cospi_m24_p08); + s7 = _mm256_madd_epi16(y0, cospi_p08_p24); + x7 = _mm256_madd_epi16(y1, cospi_p08_p24); + + y0 = _mm256_unpacklo_epi16(x12, x13); + y1 = _mm256_unpackhi_epi16(x12, x13); + s12 = _mm256_madd_epi16(y0, cospi_p08_p24); + x12 = _mm256_madd_epi16(y1, cospi_p08_p24); + s13 = _mm256_madd_epi16(y0, cospi_p24_m08); + x13 = _mm256_madd_epi16(y1, cospi_p24_m08); + + y0 = _mm256_unpacklo_epi16(x14, x15); + y1 = _mm256_unpackhi_epi16(x14, x15); + s14 = _mm256_madd_epi16(y0, cospi_m24_p08); + x14 = _mm256_madd_epi16(y1, cospi_m24_p08); + s15 = _mm256_madd_epi16(y0, cospi_p08_p24); + x15 = _mm256_madd_epi16(y1, cospi_p08_p24); + + u0 = _mm256_add_epi32(x0, x2); + v0 = _mm256_add_epi32(s0, s2); + u1 = _mm256_add_epi32(x1, x3); + v1 = _mm256_add_epi32(s1, s3); + u2 = _mm256_sub_epi32(x0, x2); + v2 = _mm256_sub_epi32(s0, s2); + u3 = _mm256_sub_epi32(x1, x3); + v3 = _mm256_sub_epi32(s1, s3); + + u0 = _mm256_add_epi32(u0, dct_rounding); + v0 = _mm256_add_epi32(v0, dct_rounding); + u1 = _mm256_add_epi32(u1, dct_rounding); + v1 = _mm256_add_epi32(v1, dct_rounding); + u2 = _mm256_add_epi32(u2, dct_rounding); + v2 = _mm256_add_epi32(v2, dct_rounding); + u3 = _mm256_add_epi32(u3, dct_rounding); + v3 = _mm256_add_epi32(v3, dct_rounding); + + u0 = _mm256_srai_epi32(u0, DCT_CONST_BITS); + v0 = _mm256_srai_epi32(v0, DCT_CONST_BITS); + u1 = _mm256_srai_epi32(u1, DCT_CONST_BITS); + v1 = _mm256_srai_epi32(v1, DCT_CONST_BITS); + u2 = _mm256_srai_epi32(u2, DCT_CONST_BITS); + v2 = _mm256_srai_epi32(v2, DCT_CONST_BITS); + u3 = _mm256_srai_epi32(u3, DCT_CONST_BITS); + v3 = _mm256_srai_epi32(v3, DCT_CONST_BITS); + + in[0] = _mm256_packs_epi32(u0, v0); + x1 = _mm256_packs_epi32(u1, v1); + x2 = _mm256_packs_epi32(u2, v2); + x3 = _mm256_packs_epi32(u3, v3); + + // Rounding on s4 + s6, s5 + s7, s4 - s6, s5 - s7 + u4 = _mm256_add_epi32(s4, s6); + u5 = _mm256_add_epi32(s5, s7); + u6 = _mm256_sub_epi32(s4, s6); + u7 = _mm256_sub_epi32(s5, s7); + + v4 = _mm256_add_epi32(x4, x6); + v5 = _mm256_add_epi32(x5, x7); + v6 = _mm256_sub_epi32(x4, x6); + v7 = _mm256_sub_epi32(x5, x7); + + u4 = _mm256_add_epi32(u4, dct_rounding); + u5 = _mm256_add_epi32(u5, dct_rounding); + u6 = _mm256_add_epi32(u6, dct_rounding); + u7 = _mm256_add_epi32(u7, dct_rounding); + + u4 = _mm256_srai_epi32(u4, DCT_CONST_BITS); + u5 = _mm256_srai_epi32(u5, DCT_CONST_BITS); + u6 = _mm256_srai_epi32(u6, DCT_CONST_BITS); + u7 = _mm256_srai_epi32(u7, DCT_CONST_BITS); + + v4 = _mm256_add_epi32(v4, dct_rounding); + v5 = _mm256_add_epi32(v5, dct_rounding); + v6 = _mm256_add_epi32(v6, dct_rounding); + v7 = _mm256_add_epi32(v7, dct_rounding); + + v4 = _mm256_srai_epi32(v4, DCT_CONST_BITS); + v5 = _mm256_srai_epi32(v5, DCT_CONST_BITS); + v6 = _mm256_srai_epi32(v6, DCT_CONST_BITS); + v7 = _mm256_srai_epi32(v7, DCT_CONST_BITS); + + x4 = _mm256_packs_epi32(u4, v4); + in[12] = _mm256_packs_epi32(u5, v5); + x6 = _mm256_packs_epi32(u6, v6); + x7 = _mm256_packs_epi32(u7, v7); + + u0 = _mm256_add_epi32(u8, u10); + v0 = _mm256_add_epi32(v8, v10); + u1 = _mm256_add_epi32(u9, u11); + v1 = _mm256_add_epi32(v9, v11); + u2 = _mm256_sub_epi32(u8, u10); + v2 = _mm256_sub_epi32(v8, v10); + u3 = _mm256_sub_epi32(u9, u11); + v3 = _mm256_sub_epi32(v9, v11); + + u0 = _mm256_add_epi32(u0, dct_rounding); + v0 = _mm256_add_epi32(v0, dct_rounding); + u1 = _mm256_add_epi32(u1, dct_rounding); + v1 = _mm256_add_epi32(v1, dct_rounding); + u2 = _mm256_add_epi32(u2, dct_rounding); + v2 = _mm256_add_epi32(v2, dct_rounding); + u3 = _mm256_add_epi32(u3, dct_rounding); + v3 = _mm256_add_epi32(v3, dct_rounding); + + u0 = _mm256_srai_epi32(u0, DCT_CONST_BITS); + v0 = _mm256_srai_epi32(v0, DCT_CONST_BITS); + u1 = _mm256_srai_epi32(u1, DCT_CONST_BITS); + v1 = _mm256_srai_epi32(v1, DCT_CONST_BITS); + u2 = _mm256_srai_epi32(u2, DCT_CONST_BITS); + v2 = _mm256_srai_epi32(v2, DCT_CONST_BITS); + u3 = _mm256_srai_epi32(u3, DCT_CONST_BITS); + v3 = _mm256_srai_epi32(v3, DCT_CONST_BITS); + + x8 = _mm256_packs_epi32(u0, v0); + in[14] = _mm256_packs_epi32(u1, v1); + x10 = _mm256_packs_epi32(u2, v2); + x11 = _mm256_packs_epi32(u3, v3); + + // Rounding on s12 + s14, s13 + s15, s12 - s14, s13 - s15 + u12 = _mm256_add_epi32(s12, s14); + u13 = _mm256_add_epi32(s13, s15); + u14 = _mm256_sub_epi32(s12, s14); + u15 = _mm256_sub_epi32(s13, s15); + + v12 = _mm256_add_epi32(x12, x14); + v13 = _mm256_add_epi32(x13, x15); + v14 = _mm256_sub_epi32(x12, x14); + v15 = _mm256_sub_epi32(x13, x15); + + u12 = _mm256_add_epi32(u12, dct_rounding); + u13 = _mm256_add_epi32(u13, dct_rounding); + u14 = _mm256_add_epi32(u14, dct_rounding); + u15 = _mm256_add_epi32(u15, dct_rounding); + + u12 = _mm256_srai_epi32(u12, DCT_CONST_BITS); + u13 = _mm256_srai_epi32(u13, DCT_CONST_BITS); + u14 = _mm256_srai_epi32(u14, DCT_CONST_BITS); + u15 = _mm256_srai_epi32(u15, DCT_CONST_BITS); + + v12 = _mm256_add_epi32(v12, dct_rounding); + v13 = _mm256_add_epi32(v13, dct_rounding); + v14 = _mm256_add_epi32(v14, dct_rounding); + v15 = _mm256_add_epi32(v15, dct_rounding); + + v12 = _mm256_srai_epi32(v12, DCT_CONST_BITS); + v13 = _mm256_srai_epi32(v13, DCT_CONST_BITS); + v14 = _mm256_srai_epi32(v14, DCT_CONST_BITS); + v15 = _mm256_srai_epi32(v15, DCT_CONST_BITS); + + x12 = _mm256_packs_epi32(u12, v12); + x13 = _mm256_packs_epi32(u13, v13); + x14 = _mm256_packs_epi32(u14, v14); + x15 = _mm256_packs_epi32(u15, v15); + in[2] = x12; + + // stage 4 + y0 = _mm256_unpacklo_epi16(x2, x3); + y1 = _mm256_unpackhi_epi16(x2, x3); + s2 = _mm256_madd_epi16(y0, cospi_m16_m16); + x2 = _mm256_madd_epi16(y1, cospi_m16_m16); + s3 = _mm256_madd_epi16(y0, cospi_p16_m16); + x3 = _mm256_madd_epi16(y1, cospi_p16_m16); + + y0 = _mm256_unpacklo_epi16(x6, x7); + y1 = _mm256_unpackhi_epi16(x6, x7); + s6 = _mm256_madd_epi16(y0, cospi_p16_p16); + x6 = _mm256_madd_epi16(y1, cospi_p16_p16); + s7 = _mm256_madd_epi16(y0, cospi_m16_p16); + x7 = _mm256_madd_epi16(y1, cospi_m16_p16); + + y0 = _mm256_unpacklo_epi16(x10, x11); + y1 = _mm256_unpackhi_epi16(x10, x11); + s10 = _mm256_madd_epi16(y0, cospi_p16_p16); + x10 = _mm256_madd_epi16(y1, cospi_p16_p16); + s11 = _mm256_madd_epi16(y0, cospi_m16_p16); + x11 = _mm256_madd_epi16(y1, cospi_m16_p16); + + y0 = _mm256_unpacklo_epi16(x14, x15); + y1 = _mm256_unpackhi_epi16(x14, x15); + s14 = _mm256_madd_epi16(y0, cospi_m16_m16); + x14 = _mm256_madd_epi16(y1, cospi_m16_m16); + s15 = _mm256_madd_epi16(y0, cospi_p16_m16); + x15 = _mm256_madd_epi16(y1, cospi_p16_m16); + + // Rounding + u2 = _mm256_add_epi32(s2, dct_rounding); + u3 = _mm256_add_epi32(s3, dct_rounding); + u6 = _mm256_add_epi32(s6, dct_rounding); + u7 = _mm256_add_epi32(s7, dct_rounding); + + u10 = _mm256_add_epi32(s10, dct_rounding); + u11 = _mm256_add_epi32(s11, dct_rounding); + u14 = _mm256_add_epi32(s14, dct_rounding); + u15 = _mm256_add_epi32(s15, dct_rounding); + + u2 = _mm256_srai_epi32(u2, DCT_CONST_BITS); + u3 = _mm256_srai_epi32(u3, DCT_CONST_BITS); + u6 = _mm256_srai_epi32(u6, DCT_CONST_BITS); + u7 = _mm256_srai_epi32(u7, DCT_CONST_BITS); + + u10 = _mm256_srai_epi32(u10, DCT_CONST_BITS); + u11 = _mm256_srai_epi32(u11, DCT_CONST_BITS); + u14 = _mm256_srai_epi32(u14, DCT_CONST_BITS); + u15 = _mm256_srai_epi32(u15, DCT_CONST_BITS); + + v2 = _mm256_add_epi32(x2, dct_rounding); + v3 = _mm256_add_epi32(x3, dct_rounding); + v6 = _mm256_add_epi32(x6, dct_rounding); + v7 = _mm256_add_epi32(x7, dct_rounding); + + v10 = _mm256_add_epi32(x10, dct_rounding); + v11 = _mm256_add_epi32(x11, dct_rounding); + v14 = _mm256_add_epi32(x14, dct_rounding); + v15 = _mm256_add_epi32(x15, dct_rounding); + + v2 = _mm256_srai_epi32(v2, DCT_CONST_BITS); + v3 = _mm256_srai_epi32(v3, DCT_CONST_BITS); + v6 = _mm256_srai_epi32(v6, DCT_CONST_BITS); + v7 = _mm256_srai_epi32(v7, DCT_CONST_BITS); + + v10 = _mm256_srai_epi32(v10, DCT_CONST_BITS); + v11 = _mm256_srai_epi32(v11, DCT_CONST_BITS); + v14 = _mm256_srai_epi32(v14, DCT_CONST_BITS); + v15 = _mm256_srai_epi32(v15, DCT_CONST_BITS); + + in[7] = _mm256_packs_epi32(u2, v2); + in[8] = _mm256_packs_epi32(u3, v3); + + in[4] = _mm256_packs_epi32(u6, v6); + in[11] = _mm256_packs_epi32(u7, v7); + + in[6] = _mm256_packs_epi32(u10, v10); + in[9] = _mm256_packs_epi32(u11, v11); + + in[5] = _mm256_packs_epi32(u14, v14); + in[10] = _mm256_packs_epi32(u15, v15); + + in[1] = _mm256_sub_epi16(zero, x8); + in[3] = _mm256_sub_epi16(zero, x4); + in[13] = _mm256_sub_epi16(zero, x13); + in[15] = _mm256_sub_epi16(zero, x1); +} + +#if CONFIG_EXT_TX +static void fidtx16_avx2(__m256i *in) { txfm_scaling16_avx2(Sqrt2, in); } +#endif + +void av1_fht16x16_avx2(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + __m256i in[16]; + + switch (tx_type) { + case DCT_DCT: + load_buffer_16x16(input, stride, 0, 0, in); + fdct16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fdct16_avx2(in); + break; + case ADST_DCT: + load_buffer_16x16(input, stride, 0, 0, in); + fadst16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fdct16_avx2(in); + break; + case DCT_ADST: + load_buffer_16x16(input, stride, 0, 0, in); + fdct16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fadst16_avx2(in); + break; + case ADST_ADST: + load_buffer_16x16(input, stride, 0, 0, in); + fadst16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fadst16_avx2(in); + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + load_buffer_16x16(input, stride, 1, 0, in); + fadst16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fdct16_avx2(in); + break; + case DCT_FLIPADST: + load_buffer_16x16(input, stride, 0, 1, in); + fdct16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fadst16_avx2(in); + break; + case FLIPADST_FLIPADST: + load_buffer_16x16(input, stride, 1, 1, in); + fadst16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fadst16_avx2(in); + break; + case ADST_FLIPADST: + load_buffer_16x16(input, stride, 0, 1, in); + fadst16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fadst16_avx2(in); + break; + case FLIPADST_ADST: + load_buffer_16x16(input, stride, 1, 0, in); + fadst16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fadst16_avx2(in); + break; + case IDTX: + load_buffer_16x16(input, stride, 0, 0, in); + fidtx16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fidtx16_avx2(in); + break; + case V_DCT: + load_buffer_16x16(input, stride, 0, 0, in); + fdct16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fidtx16_avx2(in); + break; + case H_DCT: + load_buffer_16x16(input, stride, 0, 0, in); + fidtx16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fdct16_avx2(in); + break; + case V_ADST: + load_buffer_16x16(input, stride, 0, 0, in); + fadst16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fidtx16_avx2(in); + break; + case H_ADST: + load_buffer_16x16(input, stride, 0, 0, in); + fidtx16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fadst16_avx2(in); + break; + case V_FLIPADST: + load_buffer_16x16(input, stride, 1, 0, in); + fadst16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fidtx16_avx2(in); + break; + case H_FLIPADST: + load_buffer_16x16(input, stride, 0, 1, in); + fidtx16_avx2(in); + mm256_transpose_16x16(in); + right_shift_16x16(in); + fadst16_avx2(in); + break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } + mm256_transpose_16x16(in); + write_buffer_16x16(in, output); + _mm256_zeroupper(); +} + +void aom_fdct32x32_1_avx2(const int16_t *input, tran_low_t *output, + int stride) { + // left and upper corner + int32_t sum = get_16x16_sum(input, stride); + // right and upper corner + sum += get_16x16_sum(input + 16, stride); + // left and lower corner + sum += get_16x16_sum(input + (stride << 4), stride); + // right and lower corner + sum += get_16x16_sum(input + (stride << 4) + 16, stride); + + sum >>= 3; + output[0] = (tran_low_t)sum; + _mm256_zeroupper(); +} + +static void mm256_vectors_swap(__m256i *a0, __m256i *a1, const int size) { + int i = 0; + __m256i temp; + while (i < size) { + temp = a0[i]; + a0[i] = a1[i]; + a1[i] = temp; + i++; + } +} + +static void mm256_transpose_32x32(__m256i *in0, __m256i *in1) { + mm256_transpose_16x16(in0); + mm256_transpose_16x16(&in0[16]); + mm256_transpose_16x16(in1); + mm256_transpose_16x16(&in1[16]); + mm256_vectors_swap(&in0[16], in1, 16); +} + +static void prepare_16x16_even(const __m256i *in, __m256i *even) { + even[0] = _mm256_add_epi16(in[0], in[31]); + even[1] = _mm256_add_epi16(in[1], in[30]); + even[2] = _mm256_add_epi16(in[2], in[29]); + even[3] = _mm256_add_epi16(in[3], in[28]); + even[4] = _mm256_add_epi16(in[4], in[27]); + even[5] = _mm256_add_epi16(in[5], in[26]); + even[6] = _mm256_add_epi16(in[6], in[25]); + even[7] = _mm256_add_epi16(in[7], in[24]); + even[8] = _mm256_add_epi16(in[8], in[23]); + even[9] = _mm256_add_epi16(in[9], in[22]); + even[10] = _mm256_add_epi16(in[10], in[21]); + even[11] = _mm256_add_epi16(in[11], in[20]); + even[12] = _mm256_add_epi16(in[12], in[19]); + even[13] = _mm256_add_epi16(in[13], in[18]); + even[14] = _mm256_add_epi16(in[14], in[17]); + even[15] = _mm256_add_epi16(in[15], in[16]); +} + +static void prepare_16x16_odd(const __m256i *in, __m256i *odd) { + odd[0] = _mm256_sub_epi16(in[15], in[16]); + odd[1] = _mm256_sub_epi16(in[14], in[17]); + odd[2] = _mm256_sub_epi16(in[13], in[18]); + odd[3] = _mm256_sub_epi16(in[12], in[19]); + odd[4] = _mm256_sub_epi16(in[11], in[20]); + odd[5] = _mm256_sub_epi16(in[10], in[21]); + odd[6] = _mm256_sub_epi16(in[9], in[22]); + odd[7] = _mm256_sub_epi16(in[8], in[23]); + odd[8] = _mm256_sub_epi16(in[7], in[24]); + odd[9] = _mm256_sub_epi16(in[6], in[25]); + odd[10] = _mm256_sub_epi16(in[5], in[26]); + odd[11] = _mm256_sub_epi16(in[4], in[27]); + odd[12] = _mm256_sub_epi16(in[3], in[28]); + odd[13] = _mm256_sub_epi16(in[2], in[29]); + odd[14] = _mm256_sub_epi16(in[1], in[30]); + odd[15] = _mm256_sub_epi16(in[0], in[31]); +} + +static void collect_16col(const __m256i *even, const __m256i *odd, + __m256i *out) { + // fdct16_avx2() already maps the output + out[0] = even[0]; + out[2] = even[1]; + out[4] = even[2]; + out[6] = even[3]; + out[8] = even[4]; + out[10] = even[5]; + out[12] = even[6]; + out[14] = even[7]; + out[16] = even[8]; + out[18] = even[9]; + out[20] = even[10]; + out[22] = even[11]; + out[24] = even[12]; + out[26] = even[13]; + out[28] = even[14]; + out[30] = even[15]; + + out[1] = odd[0]; + out[17] = odd[1]; + out[9] = odd[2]; + out[25] = odd[3]; + out[5] = odd[4]; + out[21] = odd[5]; + out[13] = odd[6]; + out[29] = odd[7]; + out[3] = odd[8]; + out[19] = odd[9]; + out[11] = odd[10]; + out[27] = odd[11]; + out[7] = odd[12]; + out[23] = odd[13]; + out[15] = odd[14]; + out[31] = odd[15]; +} + +static void collect_coeffs(const __m256i *first_16col_even, + const __m256i *first_16col_odd, + const __m256i *second_16col_even, + const __m256i *second_16col_odd, __m256i *in0, + __m256i *in1) { + collect_16col(first_16col_even, first_16col_odd, in0); + collect_16col(second_16col_even, second_16col_odd, in1); +} + +static void fdct16_odd_avx2(__m256i *in) { + // sequence: cospi_L_H = pairs(L, H) and L first + const __m256i cospi_p16_p16 = pair256_set_epi16(cospi_16_64, cospi_16_64); + const __m256i cospi_m16_p16 = pair256_set_epi16(-cospi_16_64, cospi_16_64); + const __m256i cospi_m08_p24 = pair256_set_epi16(-cospi_8_64, cospi_24_64); + const __m256i cospi_p24_p08 = pair256_set_epi16(cospi_24_64, cospi_8_64); + const __m256i cospi_m24_m08 = pair256_set_epi16(-cospi_24_64, -cospi_8_64); + const __m256i cospi_m04_p28 = pair256_set_epi16(-cospi_4_64, cospi_28_64); + const __m256i cospi_p28_p04 = pair256_set_epi16(cospi_28_64, cospi_4_64); + const __m256i cospi_m28_m04 = pair256_set_epi16(-cospi_28_64, -cospi_4_64); + const __m256i cospi_m20_p12 = pair256_set_epi16(-cospi_20_64, cospi_12_64); + const __m256i cospi_p12_p20 = pair256_set_epi16(cospi_12_64, cospi_20_64); + const __m256i cospi_m12_m20 = pair256_set_epi16(-cospi_12_64, -cospi_20_64); + + const __m256i cospi_p31_p01 = pair256_set_epi16(cospi_31_64, cospi_1_64); + const __m256i cospi_m01_p31 = pair256_set_epi16(-cospi_1_64, cospi_31_64); + const __m256i cospi_p15_p17 = pair256_set_epi16(cospi_15_64, cospi_17_64); + const __m256i cospi_m17_p15 = pair256_set_epi16(-cospi_17_64, cospi_15_64); + const __m256i cospi_p23_p09 = pair256_set_epi16(cospi_23_64, cospi_9_64); + const __m256i cospi_m09_p23 = pair256_set_epi16(-cospi_9_64, cospi_23_64); + const __m256i cospi_p07_p25 = pair256_set_epi16(cospi_7_64, cospi_25_64); + const __m256i cospi_m25_p07 = pair256_set_epi16(-cospi_25_64, cospi_7_64); + const __m256i cospi_p27_p05 = pair256_set_epi16(cospi_27_64, cospi_5_64); + const __m256i cospi_m05_p27 = pair256_set_epi16(-cospi_5_64, cospi_27_64); + const __m256i cospi_p11_p21 = pair256_set_epi16(cospi_11_64, cospi_21_64); + const __m256i cospi_m21_p11 = pair256_set_epi16(-cospi_21_64, cospi_11_64); + const __m256i cospi_p19_p13 = pair256_set_epi16(cospi_19_64, cospi_13_64); + const __m256i cospi_m13_p19 = pair256_set_epi16(-cospi_13_64, cospi_19_64); + const __m256i cospi_p03_p29 = pair256_set_epi16(cospi_3_64, cospi_29_64); + const __m256i cospi_m29_p03 = pair256_set_epi16(-cospi_29_64, cospi_3_64); + + __m256i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15; + __m256i u0, u1; + + // stage 1 is in prepare_16x16_odd() + + // stage 2 + y0 = in[0]; + y1 = in[1]; + y2 = in[2]; + y3 = in[3]; + + u0 = _mm256_unpacklo_epi16(in[4], in[11]); + u1 = _mm256_unpackhi_epi16(in[4], in[11]); + y4 = butter_fly(u0, u1, cospi_m16_p16); + y11 = butter_fly(u0, u1, cospi_p16_p16); + + u0 = _mm256_unpacklo_epi16(in[5], in[10]); + u1 = _mm256_unpackhi_epi16(in[5], in[10]); + y5 = butter_fly(u0, u1, cospi_m16_p16); + y10 = butter_fly(u0, u1, cospi_p16_p16); + + u0 = _mm256_unpacklo_epi16(in[6], in[9]); + u1 = _mm256_unpackhi_epi16(in[6], in[9]); + y6 = butter_fly(u0, u1, cospi_m16_p16); + y9 = butter_fly(u0, u1, cospi_p16_p16); + + u0 = _mm256_unpacklo_epi16(in[7], in[8]); + u1 = _mm256_unpackhi_epi16(in[7], in[8]); + y7 = butter_fly(u0, u1, cospi_m16_p16); + y8 = butter_fly(u0, u1, cospi_p16_p16); + + y12 = in[12]; + y13 = in[13]; + y14 = in[14]; + y15 = in[15]; + + // stage 3 + x0 = _mm256_add_epi16(y0, y7); + x1 = _mm256_add_epi16(y1, y6); + x2 = _mm256_add_epi16(y2, y5); + x3 = _mm256_add_epi16(y3, y4); + x4 = _mm256_sub_epi16(y3, y4); + x5 = _mm256_sub_epi16(y2, y5); + x6 = _mm256_sub_epi16(y1, y6); + x7 = _mm256_sub_epi16(y0, y7); + x8 = _mm256_sub_epi16(y15, y8); + x9 = _mm256_sub_epi16(y14, y9); + x10 = _mm256_sub_epi16(y13, y10); + x11 = _mm256_sub_epi16(y12, y11); + x12 = _mm256_add_epi16(y12, y11); + x13 = _mm256_add_epi16(y13, y10); + x14 = _mm256_add_epi16(y14, y9); + x15 = _mm256_add_epi16(y15, y8); + + // stage 4 + y0 = x0; + y1 = x1; + y6 = x6; + y7 = x7; + y8 = x8; + y9 = x9; + y14 = x14; + y15 = x15; + + u0 = _mm256_unpacklo_epi16(x2, x13); + u1 = _mm256_unpackhi_epi16(x2, x13); + y2 = butter_fly(u0, u1, cospi_m08_p24); + y13 = butter_fly(u0, u1, cospi_p24_p08); + + u0 = _mm256_unpacklo_epi16(x3, x12); + u1 = _mm256_unpackhi_epi16(x3, x12); + y3 = butter_fly(u0, u1, cospi_m08_p24); + y12 = butter_fly(u0, u1, cospi_p24_p08); + + u0 = _mm256_unpacklo_epi16(x4, x11); + u1 = _mm256_unpackhi_epi16(x4, x11); + y4 = butter_fly(u0, u1, cospi_m24_m08); + y11 = butter_fly(u0, u1, cospi_m08_p24); + + u0 = _mm256_unpacklo_epi16(x5, x10); + u1 = _mm256_unpackhi_epi16(x5, x10); + y5 = butter_fly(u0, u1, cospi_m24_m08); + y10 = butter_fly(u0, u1, cospi_m08_p24); + + // stage 5 + x0 = _mm256_add_epi16(y0, y3); + x1 = _mm256_add_epi16(y1, y2); + x2 = _mm256_sub_epi16(y1, y2); + x3 = _mm256_sub_epi16(y0, y3); + x4 = _mm256_sub_epi16(y7, y4); + x5 = _mm256_sub_epi16(y6, y5); + x6 = _mm256_add_epi16(y6, y5); + x7 = _mm256_add_epi16(y7, y4); + + x8 = _mm256_add_epi16(y8, y11); + x9 = _mm256_add_epi16(y9, y10); + x10 = _mm256_sub_epi16(y9, y10); + x11 = _mm256_sub_epi16(y8, y11); + x12 = _mm256_sub_epi16(y15, y12); + x13 = _mm256_sub_epi16(y14, y13); + x14 = _mm256_add_epi16(y14, y13); + x15 = _mm256_add_epi16(y15, y12); + + // stage 6 + y0 = x0; + y3 = x3; + y4 = x4; + y7 = x7; + y8 = x8; + y11 = x11; + y12 = x12; + y15 = x15; + + u0 = _mm256_unpacklo_epi16(x1, x14); + u1 = _mm256_unpackhi_epi16(x1, x14); + y1 = butter_fly(u0, u1, cospi_m04_p28); + y14 = butter_fly(u0, u1, cospi_p28_p04); + + u0 = _mm256_unpacklo_epi16(x2, x13); + u1 = _mm256_unpackhi_epi16(x2, x13); + y2 = butter_fly(u0, u1, cospi_m28_m04); + y13 = butter_fly(u0, u1, cospi_m04_p28); + + u0 = _mm256_unpacklo_epi16(x5, x10); + u1 = _mm256_unpackhi_epi16(x5, x10); + y5 = butter_fly(u0, u1, cospi_m20_p12); + y10 = butter_fly(u0, u1, cospi_p12_p20); + + u0 = _mm256_unpacklo_epi16(x6, x9); + u1 = _mm256_unpackhi_epi16(x6, x9); + y6 = butter_fly(u0, u1, cospi_m12_m20); + y9 = butter_fly(u0, u1, cospi_m20_p12); + + // stage 7 + x0 = _mm256_add_epi16(y0, y1); + x1 = _mm256_sub_epi16(y0, y1); + x2 = _mm256_sub_epi16(y3, y2); + x3 = _mm256_add_epi16(y3, y2); + x4 = _mm256_add_epi16(y4, y5); + x5 = _mm256_sub_epi16(y4, y5); + x6 = _mm256_sub_epi16(y7, y6); + x7 = _mm256_add_epi16(y7, y6); + + x8 = _mm256_add_epi16(y8, y9); + x9 = _mm256_sub_epi16(y8, y9); + x10 = _mm256_sub_epi16(y11, y10); + x11 = _mm256_add_epi16(y11, y10); + x12 = _mm256_add_epi16(y12, y13); + x13 = _mm256_sub_epi16(y12, y13); + x14 = _mm256_sub_epi16(y15, y14); + x15 = _mm256_add_epi16(y15, y14); + + // stage 8 + u0 = _mm256_unpacklo_epi16(x0, x15); + u1 = _mm256_unpackhi_epi16(x0, x15); + in[0] = butter_fly(u0, u1, cospi_p31_p01); + in[15] = butter_fly(u0, u1, cospi_m01_p31); + + u0 = _mm256_unpacklo_epi16(x1, x14); + u1 = _mm256_unpackhi_epi16(x1, x14); + in[1] = butter_fly(u0, u1, cospi_p15_p17); + in[14] = butter_fly(u0, u1, cospi_m17_p15); + + u0 = _mm256_unpacklo_epi16(x2, x13); + u1 = _mm256_unpackhi_epi16(x2, x13); + in[2] = butter_fly(u0, u1, cospi_p23_p09); + in[13] = butter_fly(u0, u1, cospi_m09_p23); + + u0 = _mm256_unpacklo_epi16(x3, x12); + u1 = _mm256_unpackhi_epi16(x3, x12); + in[3] = butter_fly(u0, u1, cospi_p07_p25); + in[12] = butter_fly(u0, u1, cospi_m25_p07); + + u0 = _mm256_unpacklo_epi16(x4, x11); + u1 = _mm256_unpackhi_epi16(x4, x11); + in[4] = butter_fly(u0, u1, cospi_p27_p05); + in[11] = butter_fly(u0, u1, cospi_m05_p27); + + u0 = _mm256_unpacklo_epi16(x5, x10); + u1 = _mm256_unpackhi_epi16(x5, x10); + in[5] = butter_fly(u0, u1, cospi_p11_p21); + in[10] = butter_fly(u0, u1, cospi_m21_p11); + + u0 = _mm256_unpacklo_epi16(x6, x9); + u1 = _mm256_unpackhi_epi16(x6, x9); + in[6] = butter_fly(u0, u1, cospi_p19_p13); + in[9] = butter_fly(u0, u1, cospi_m13_p19); + + u0 = _mm256_unpacklo_epi16(x7, x8); + u1 = _mm256_unpackhi_epi16(x7, x8); + in[7] = butter_fly(u0, u1, cospi_p03_p29); + in[8] = butter_fly(u0, u1, cospi_m29_p03); +} + +static void fdct32_avx2(__m256i *in0, __m256i *in1) { + __m256i even0[16], even1[16], odd0[16], odd1[16]; + prepare_16x16_even(in0, even0); + fdct16_avx2(even0); + + prepare_16x16_odd(in0, odd0); + fdct16_odd_avx2(odd0); + + prepare_16x16_even(in1, even1); + fdct16_avx2(even1); + + prepare_16x16_odd(in1, odd1); + fdct16_odd_avx2(odd1); + + collect_coeffs(even0, odd0, even1, odd1, in0, in1); + + mm256_transpose_32x32(in0, in1); +} + +static INLINE void write_buffer_32x32(const __m256i *in0, const __m256i *in1, + tran_low_t *output) { + int i = 0; + const int stride = 32; + tran_low_t *coeff = output; + while (i < 32) { + storeu_output_avx2(&in0[i], coeff); + storeu_output_avx2(&in1[i], coeff + 16); + coeff += stride; + i += 1; + } +} + +#if CONFIG_EXT_TX +static void fhalfright32_16col_avx2(__m256i *in) { + int i = 0; + const __m256i zero = _mm256_setzero_si256(); + const __m256i sqrt2 = _mm256_set1_epi16(Sqrt2); + const __m256i dct_rounding = _mm256_set1_epi32(DCT_CONST_ROUNDING); + __m256i x0, x1; + + while (i < 16) { + in[i] = _mm256_slli_epi16(in[i], 2); + x0 = _mm256_unpacklo_epi16(in[i + 16], zero); + x1 = _mm256_unpackhi_epi16(in[i + 16], zero); + x0 = _mm256_madd_epi16(x0, sqrt2); + x1 = _mm256_madd_epi16(x1, sqrt2); + x0 = _mm256_add_epi32(x0, dct_rounding); + x1 = _mm256_add_epi32(x1, dct_rounding); + x0 = _mm256_srai_epi32(x0, DCT_CONST_BITS); + x1 = _mm256_srai_epi32(x1, DCT_CONST_BITS); + in[i + 16] = _mm256_packs_epi32(x0, x1); + i += 1; + } + fdct16_avx2(&in[16]); +} + +static void fhalfright32_avx2(__m256i *in0, __m256i *in1) { + fhalfright32_16col_avx2(in0); + fhalfright32_16col_avx2(in1); + mm256_vectors_swap(in0, &in0[16], 16); + mm256_vectors_swap(in1, &in1[16], 16); + mm256_transpose_32x32(in0, in1); +} +#endif // CONFIG_EXT_TX + +static INLINE void load_buffer_32x32(const int16_t *input, int stride, + int flipud, int fliplr, __m256i *in0, + __m256i *in1) { + // Load 4 16x16 blocks + const int16_t *topL = input; + const int16_t *topR = input + 16; + const int16_t *botL = input + 16 * stride; + const int16_t *botR = input + 16 * stride + 16; + + const int16_t *tmp; + + if (flipud) { + // Swap left columns + tmp = topL; + topL = botL; + botL = tmp; + // Swap right columns + tmp = topR; + topR = botR; + botR = tmp; + } + + if (fliplr) { + // Swap top rows + tmp = topL; + topL = topR; + topR = tmp; + // Swap bottom rows + tmp = botL; + botL = botR; + botR = tmp; + } + + // load first 16 columns + load_buffer_16x16(topL, stride, flipud, fliplr, in0); + load_buffer_16x16(botL, stride, flipud, fliplr, in0 + 16); + + // load second 16 columns + load_buffer_16x16(topR, stride, flipud, fliplr, in1); + load_buffer_16x16(botR, stride, flipud, fliplr, in1 + 16); +} + +static INLINE void right_shift_32x32_16col(int bit, __m256i *in) { + int i = 0; + const __m256i rounding = _mm256_set1_epi16((1 << bit) >> 1); + __m256i sign; + while (i < 32) { + sign = _mm256_srai_epi16(in[i], 15); + in[i] = _mm256_add_epi16(in[i], rounding); + in[i] = _mm256_add_epi16(in[i], sign); + in[i] = _mm256_srai_epi16(in[i], bit); + i += 1; + } +} + +// Positive rounding +static INLINE void right_shift_32x32(__m256i *in0, __m256i *in1) { + const int bit = 4; + right_shift_32x32_16col(bit, in0); + right_shift_32x32_16col(bit, in1); +} + +#if CONFIG_EXT_TX +static void fidtx32_avx2(__m256i *in0, __m256i *in1) { + int i = 0; + while (i < 32) { + in0[i] = _mm256_slli_epi16(in0[i], 2); + in1[i] = _mm256_slli_epi16(in1[i], 2); + i += 1; + } + mm256_transpose_32x32(in0, in1); +} +#endif + +void av1_fht32x32_avx2(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + __m256i in0[32]; // left 32 columns + __m256i in1[32]; // right 32 columns + + switch (tx_type) { + case DCT_DCT: + load_buffer_32x32(input, stride, 0, 0, in0, in1); + fdct32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fdct32_avx2(in0, in1); + break; +#if CONFIG_EXT_TX + case ADST_DCT: + load_buffer_32x32(input, stride, 0, 0, in0, in1); + fhalfright32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fdct32_avx2(in0, in1); + break; + case DCT_ADST: + load_buffer_32x32(input, stride, 0, 0, in0, in1); + fdct32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fhalfright32_avx2(in0, in1); + break; + case ADST_ADST: + load_buffer_32x32(input, stride, 0, 0, in0, in1); + fhalfright32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fhalfright32_avx2(in0, in1); + break; + case FLIPADST_DCT: + load_buffer_32x32(input, stride, 1, 0, in0, in1); + fhalfright32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fdct32_avx2(in0, in1); + break; + case DCT_FLIPADST: + load_buffer_32x32(input, stride, 0, 1, in0, in1); + fdct32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fhalfright32_avx2(in0, in1); + break; + case FLIPADST_FLIPADST: + load_buffer_32x32(input, stride, 1, 1, in0, in1); + fhalfright32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fhalfright32_avx2(in0, in1); + break; + case ADST_FLIPADST: + load_buffer_32x32(input, stride, 0, 1, in0, in1); + fhalfright32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fhalfright32_avx2(in0, in1); + break; + case FLIPADST_ADST: + load_buffer_32x32(input, stride, 1, 0, in0, in1); + fhalfright32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fhalfright32_avx2(in0, in1); + break; + case IDTX: + load_buffer_32x32(input, stride, 0, 0, in0, in1); + fidtx32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fidtx32_avx2(in0, in1); + break; + case V_DCT: + load_buffer_32x32(input, stride, 0, 0, in0, in1); + fdct32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fidtx32_avx2(in0, in1); + break; + case H_DCT: + load_buffer_32x32(input, stride, 0, 0, in0, in1); + fidtx32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fdct32_avx2(in0, in1); + break; + case V_ADST: + load_buffer_32x32(input, stride, 0, 0, in0, in1); + fhalfright32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fidtx32_avx2(in0, in1); + break; + case H_ADST: + load_buffer_32x32(input, stride, 0, 0, in0, in1); + fidtx32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fhalfright32_avx2(in0, in1); + break; + case V_FLIPADST: + load_buffer_32x32(input, stride, 1, 0, in0, in1); + fhalfright32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fidtx32_avx2(in0, in1); + break; + case H_FLIPADST: + load_buffer_32x32(input, stride, 0, 1, in0, in1); + fidtx32_avx2(in0, in1); + right_shift_32x32(in0, in1); + fhalfright32_avx2(in0, in1); + break; +#endif // CONFIG_EXT_TX + default: assert(0); break; + } + write_buffer_32x32(in0, in1, output); + _mm256_zeroupper(); +} diff --git a/third_party/aom/av1/encoder/x86/temporal_filter_apply_sse2.asm b/third_party/aom/av1/encoder/x86/temporal_filter_apply_sse2.asm new file mode 100644 index 0000000000..7186b6b924 --- /dev/null +++ b/third_party/aom/av1/encoder/x86/temporal_filter_apply_sse2.asm @@ -0,0 +1,215 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + + +%include "aom_ports/x86_abi_support.asm" + +; void av1_temporal_filter_apply_sse2 | arg +; (unsigned char *frame1, | 0 +; unsigned int stride, | 1 +; unsigned char *frame2, | 2 +; unsigned int block_width, | 3 +; unsigned int block_height, | 4 +; int strength, | 5 +; int filter_weight, | 6 +; unsigned int *accumulator, | 7 +; unsigned short *count) | 8 +global sym(av1_temporal_filter_apply_sse2) PRIVATE +sym(av1_temporal_filter_apply_sse2): + + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 9 + SAVE_XMM 7 + GET_GOT rbx + push rsi + push rdi + ALIGN_STACK 16, rax + %define block_width 0 + %define block_height 16 + %define strength 32 + %define filter_weight 48 + %define rounding_bit 64 + %define rbp_backup 80 + %define stack_size 96 + sub rsp, stack_size + mov [rsp + rbp_backup], rbp + ; end prolog + + mov edx, arg(3) + mov [rsp + block_width], rdx + mov edx, arg(4) + mov [rsp + block_height], rdx + movd xmm6, arg(5) + movdqa [rsp + strength], xmm6 ; where strength is used, all 16 bytes are read + + ; calculate the rounding bit outside the loop + ; 0x8000 >> (16 - strength) + mov rdx, 16 + sub rdx, arg(5) ; 16 - strength + movq xmm4, rdx ; can't use rdx w/ shift + movdqa xmm5, [GLOBAL(_const_top_bit)] + psrlw xmm5, xmm4 + movdqa [rsp + rounding_bit], xmm5 + + mov rsi, arg(0) ; src/frame1 + mov rdx, arg(2) ; predictor frame + mov rdi, arg(7) ; accumulator + mov rax, arg(8) ; count + + ; dup the filter weight and store for later + movd xmm0, arg(6) ; filter_weight + pshuflw xmm0, xmm0, 0 + punpcklwd xmm0, xmm0 + movdqa [rsp + filter_weight], xmm0 + + mov rbp, arg(1) ; stride + pxor xmm7, xmm7 ; zero for extraction + + mov rcx, [rsp + block_width] + imul rcx, [rsp + block_height] + add rcx, rdx + cmp dword ptr [rsp + block_width], 8 + jne .temporal_filter_apply_load_16 + +.temporal_filter_apply_load_8: + movq xmm0, [rsi] ; first row + lea rsi, [rsi + rbp] ; += stride + punpcklbw xmm0, xmm7 ; src[ 0- 7] + movq xmm1, [rsi] ; second row + lea rsi, [rsi + rbp] ; += stride + punpcklbw xmm1, xmm7 ; src[ 8-15] + jmp .temporal_filter_apply_load_finished + +.temporal_filter_apply_load_16: + movdqa xmm0, [rsi] ; src (frame1) + lea rsi, [rsi + rbp] ; += stride + movdqa xmm1, xmm0 + punpcklbw xmm0, xmm7 ; src[ 0- 7] + punpckhbw xmm1, xmm7 ; src[ 8-15] + +.temporal_filter_apply_load_finished: + movdqa xmm2, [rdx] ; predictor (frame2) + movdqa xmm3, xmm2 + punpcklbw xmm2, xmm7 ; pred[ 0- 7] + punpckhbw xmm3, xmm7 ; pred[ 8-15] + + ; modifier = src_byte - pixel_value + psubw xmm0, xmm2 ; src - pred[ 0- 7] + psubw xmm1, xmm3 ; src - pred[ 8-15] + + ; modifier *= modifier + pmullw xmm0, xmm0 ; modifer[ 0- 7]^2 + pmullw xmm1, xmm1 ; modifer[ 8-15]^2 + + ; modifier *= 3 + pmullw xmm0, [GLOBAL(_const_3w)] + pmullw xmm1, [GLOBAL(_const_3w)] + + ; modifer += 0x8000 >> (16 - strength) + paddw xmm0, [rsp + rounding_bit] + paddw xmm1, [rsp + rounding_bit] + + ; modifier >>= strength + psrlw xmm0, [rsp + strength] + psrlw xmm1, [rsp + strength] + + ; modifier = 16 - modifier + ; saturation takes care of modifier > 16 + movdqa xmm3, [GLOBAL(_const_16w)] + movdqa xmm2, [GLOBAL(_const_16w)] + psubusw xmm3, xmm1 + psubusw xmm2, xmm0 + + ; modifier *= filter_weight + pmullw xmm2, [rsp + filter_weight] + pmullw xmm3, [rsp + filter_weight] + + ; count + movdqa xmm4, [rax] + movdqa xmm5, [rax+16] + ; += modifier + paddw xmm4, xmm2 + paddw xmm5, xmm3 + ; write back + movdqa [rax], xmm4 + movdqa [rax+16], xmm5 + lea rax, [rax + 16*2] ; count += 16*(sizeof(short)) + + ; load and extract the predictor up to shorts + pxor xmm7, xmm7 + movdqa xmm0, [rdx] + lea rdx, [rdx + 16*1] ; pred += 16*(sizeof(char)) + movdqa xmm1, xmm0 + punpcklbw xmm0, xmm7 ; pred[ 0- 7] + punpckhbw xmm1, xmm7 ; pred[ 8-15] + + ; modifier *= pixel_value + pmullw xmm0, xmm2 + pmullw xmm1, xmm3 + + ; expand to double words + movdqa xmm2, xmm0 + punpcklwd xmm0, xmm7 ; [ 0- 3] + punpckhwd xmm2, xmm7 ; [ 4- 7] + movdqa xmm3, xmm1 + punpcklwd xmm1, xmm7 ; [ 8-11] + punpckhwd xmm3, xmm7 ; [12-15] + + ; accumulator + movdqa xmm4, [rdi] + movdqa xmm5, [rdi+16] + movdqa xmm6, [rdi+32] + movdqa xmm7, [rdi+48] + ; += modifier + paddd xmm4, xmm0 + paddd xmm5, xmm2 + paddd xmm6, xmm1 + paddd xmm7, xmm3 + ; write back + movdqa [rdi], xmm4 + movdqa [rdi+16], xmm5 + movdqa [rdi+32], xmm6 + movdqa [rdi+48], xmm7 + lea rdi, [rdi + 16*4] ; accumulator += 16*(sizeof(int)) + + cmp rdx, rcx + je .temporal_filter_apply_epilog + pxor xmm7, xmm7 ; zero for extraction + cmp dword ptr [rsp + block_width], 16 + je .temporal_filter_apply_load_16 + jmp .temporal_filter_apply_load_8 + +.temporal_filter_apply_epilog: + ; begin epilog + mov rbp, [rsp + rbp_backup] + add rsp, stack_size + pop rsp + pop rdi + pop rsi + RESTORE_GOT + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +SECTION_RODATA +align 16 +_const_3w: + times 8 dw 3 +align 16 +_const_top_bit: + times 8 dw 1<<15 +align 16 +_const_16w: + times 8 dw 16 diff --git a/third_party/aom/av1/encoder/x86/wedge_utils_sse2.c b/third_party/aom/av1/encoder/x86/wedge_utils_sse2.c new file mode 100644 index 0000000000..bf233ca4d9 --- /dev/null +++ b/third_party/aom/av1/encoder/x86/wedge_utils_sse2.c @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "aom_dsp/x86/synonyms.h" + +#include "aom/aom_integer.h" + +#include "av1/common/reconinter.h" + +#define MAX_MASK_VALUE (1 << WEDGE_WEIGHT_BITS) + +/** + * See av1_wedge_sse_from_residuals_c + */ +uint64_t av1_wedge_sse_from_residuals_sse2(const int16_t *r1, const int16_t *d, + const uint8_t *m, int N) { + int n = -N; + int n8 = n + 8; + + uint64_t csse; + + const __m128i v_mask_max_w = _mm_set1_epi16(MAX_MASK_VALUE); + const __m128i v_zext_q = _mm_set_epi32(0, 0xffffffff, 0, 0xffffffff); + + __m128i v_acc0_q = _mm_setzero_si128(); + + assert(N % 64 == 0); + + r1 += N; + d += N; + m += N; + + do { + const __m128i v_r0_w = xx_load_128(r1 + n); + const __m128i v_r1_w = xx_load_128(r1 + n8); + const __m128i v_d0_w = xx_load_128(d + n); + const __m128i v_d1_w = xx_load_128(d + n8); + const __m128i v_m01_b = xx_load_128(m + n); + + const __m128i v_rd0l_w = _mm_unpacklo_epi16(v_d0_w, v_r0_w); + const __m128i v_rd0h_w = _mm_unpackhi_epi16(v_d0_w, v_r0_w); + const __m128i v_rd1l_w = _mm_unpacklo_epi16(v_d1_w, v_r1_w); + const __m128i v_rd1h_w = _mm_unpackhi_epi16(v_d1_w, v_r1_w); + const __m128i v_m0_w = _mm_unpacklo_epi8(v_m01_b, _mm_setzero_si128()); + const __m128i v_m1_w = _mm_unpackhi_epi8(v_m01_b, _mm_setzero_si128()); + + const __m128i v_m0l_w = _mm_unpacklo_epi16(v_m0_w, v_mask_max_w); + const __m128i v_m0h_w = _mm_unpackhi_epi16(v_m0_w, v_mask_max_w); + const __m128i v_m1l_w = _mm_unpacklo_epi16(v_m1_w, v_mask_max_w); + const __m128i v_m1h_w = _mm_unpackhi_epi16(v_m1_w, v_mask_max_w); + + const __m128i v_t0l_d = _mm_madd_epi16(v_rd0l_w, v_m0l_w); + const __m128i v_t0h_d = _mm_madd_epi16(v_rd0h_w, v_m0h_w); + const __m128i v_t1l_d = _mm_madd_epi16(v_rd1l_w, v_m1l_w); + const __m128i v_t1h_d = _mm_madd_epi16(v_rd1h_w, v_m1h_w); + + const __m128i v_t0_w = _mm_packs_epi32(v_t0l_d, v_t0h_d); + const __m128i v_t1_w = _mm_packs_epi32(v_t1l_d, v_t1h_d); + + const __m128i v_sq0_d = _mm_madd_epi16(v_t0_w, v_t0_w); + const __m128i v_sq1_d = _mm_madd_epi16(v_t1_w, v_t1_w); + + const __m128i v_sum0_q = _mm_add_epi64(_mm_and_si128(v_sq0_d, v_zext_q), + _mm_srli_epi64(v_sq0_d, 32)); + const __m128i v_sum1_q = _mm_add_epi64(_mm_and_si128(v_sq1_d, v_zext_q), + _mm_srli_epi64(v_sq1_d, 32)); + + v_acc0_q = _mm_add_epi64(v_acc0_q, v_sum0_q); + v_acc0_q = _mm_add_epi64(v_acc0_q, v_sum1_q); + + n8 += 16; + n += 16; + } while (n); + + v_acc0_q = _mm_add_epi64(v_acc0_q, _mm_srli_si128(v_acc0_q, 8)); + +#if ARCH_X86_64 + csse = (uint64_t)_mm_cvtsi128_si64(v_acc0_q); +#else + xx_storel_64(&csse, v_acc0_q); +#endif + + return ROUND_POWER_OF_TWO(csse, 2 * WEDGE_WEIGHT_BITS); +} + +/** + * See av1_wedge_sign_from_residuals_c + */ +int av1_wedge_sign_from_residuals_sse2(const int16_t *ds, const uint8_t *m, + int N, int64_t limit) { + int64_t acc; + + __m128i v_sign_d; + __m128i v_acc0_d = _mm_setzero_si128(); + __m128i v_acc1_d = _mm_setzero_si128(); + __m128i v_acc_q; + + // Input size limited to 8192 by the use of 32 bit accumulators and m + // being between [0, 64]. Overflow might happen at larger sizes, + // though it is practically impossible on real video input. + assert(N < 8192); + assert(N % 64 == 0); + + do { + const __m128i v_m01_b = xx_load_128(m); + const __m128i v_m23_b = xx_load_128(m + 16); + const __m128i v_m45_b = xx_load_128(m + 32); + const __m128i v_m67_b = xx_load_128(m + 48); + + const __m128i v_d0_w = xx_load_128(ds); + const __m128i v_d1_w = xx_load_128(ds + 8); + const __m128i v_d2_w = xx_load_128(ds + 16); + const __m128i v_d3_w = xx_load_128(ds + 24); + const __m128i v_d4_w = xx_load_128(ds + 32); + const __m128i v_d5_w = xx_load_128(ds + 40); + const __m128i v_d6_w = xx_load_128(ds + 48); + const __m128i v_d7_w = xx_load_128(ds + 56); + + const __m128i v_m0_w = _mm_unpacklo_epi8(v_m01_b, _mm_setzero_si128()); + const __m128i v_m1_w = _mm_unpackhi_epi8(v_m01_b, _mm_setzero_si128()); + const __m128i v_m2_w = _mm_unpacklo_epi8(v_m23_b, _mm_setzero_si128()); + const __m128i v_m3_w = _mm_unpackhi_epi8(v_m23_b, _mm_setzero_si128()); + const __m128i v_m4_w = _mm_unpacklo_epi8(v_m45_b, _mm_setzero_si128()); + const __m128i v_m5_w = _mm_unpackhi_epi8(v_m45_b, _mm_setzero_si128()); + const __m128i v_m6_w = _mm_unpacklo_epi8(v_m67_b, _mm_setzero_si128()); + const __m128i v_m7_w = _mm_unpackhi_epi8(v_m67_b, _mm_setzero_si128()); + + const __m128i v_p0_d = _mm_madd_epi16(v_d0_w, v_m0_w); + const __m128i v_p1_d = _mm_madd_epi16(v_d1_w, v_m1_w); + const __m128i v_p2_d = _mm_madd_epi16(v_d2_w, v_m2_w); + const __m128i v_p3_d = _mm_madd_epi16(v_d3_w, v_m3_w); + const __m128i v_p4_d = _mm_madd_epi16(v_d4_w, v_m4_w); + const __m128i v_p5_d = _mm_madd_epi16(v_d5_w, v_m5_w); + const __m128i v_p6_d = _mm_madd_epi16(v_d6_w, v_m6_w); + const __m128i v_p7_d = _mm_madd_epi16(v_d7_w, v_m7_w); + + const __m128i v_p01_d = _mm_add_epi32(v_p0_d, v_p1_d); + const __m128i v_p23_d = _mm_add_epi32(v_p2_d, v_p3_d); + const __m128i v_p45_d = _mm_add_epi32(v_p4_d, v_p5_d); + const __m128i v_p67_d = _mm_add_epi32(v_p6_d, v_p7_d); + + const __m128i v_p0123_d = _mm_add_epi32(v_p01_d, v_p23_d); + const __m128i v_p4567_d = _mm_add_epi32(v_p45_d, v_p67_d); + + v_acc0_d = _mm_add_epi32(v_acc0_d, v_p0123_d); + v_acc1_d = _mm_add_epi32(v_acc1_d, v_p4567_d); + + ds += 64; + m += 64; + + N -= 64; + } while (N); + + v_sign_d = _mm_cmplt_epi32(v_acc0_d, _mm_setzero_si128()); + v_acc0_d = _mm_add_epi64(_mm_unpacklo_epi32(v_acc0_d, v_sign_d), + _mm_unpackhi_epi32(v_acc0_d, v_sign_d)); + + v_sign_d = _mm_cmplt_epi32(v_acc1_d, _mm_setzero_si128()); + v_acc1_d = _mm_add_epi64(_mm_unpacklo_epi32(v_acc1_d, v_sign_d), + _mm_unpackhi_epi32(v_acc1_d, v_sign_d)); + + v_acc_q = _mm_add_epi64(v_acc0_d, v_acc1_d); + + v_acc_q = _mm_add_epi64(v_acc_q, _mm_srli_si128(v_acc_q, 8)); + +#if ARCH_X86_64 + acc = (uint64_t)_mm_cvtsi128_si64(v_acc_q); +#else + xx_storel_64(&acc, v_acc_q); +#endif + + return acc > limit; +} + +// Negate under mask +static INLINE __m128i negm_epi16(__m128i v_v_w, __m128i v_mask_w) { + return _mm_sub_epi16(_mm_xor_si128(v_v_w, v_mask_w), v_mask_w); +} + +/** + * av1_wedge_compute_delta_squares_c + */ +void av1_wedge_compute_delta_squares_sse2(int16_t *d, const int16_t *a, + const int16_t *b, int N) { + const __m128i v_neg_w = + _mm_set_epi16(0xffff, 0, 0xffff, 0, 0xffff, 0, 0xffff, 0); + + assert(N % 64 == 0); + + do { + const __m128i v_a0_w = xx_load_128(a); + const __m128i v_b0_w = xx_load_128(b); + const __m128i v_a1_w = xx_load_128(a + 8); + const __m128i v_b1_w = xx_load_128(b + 8); + const __m128i v_a2_w = xx_load_128(a + 16); + const __m128i v_b2_w = xx_load_128(b + 16); + const __m128i v_a3_w = xx_load_128(a + 24); + const __m128i v_b3_w = xx_load_128(b + 24); + + const __m128i v_ab0l_w = _mm_unpacklo_epi16(v_a0_w, v_b0_w); + const __m128i v_ab0h_w = _mm_unpackhi_epi16(v_a0_w, v_b0_w); + const __m128i v_ab1l_w = _mm_unpacklo_epi16(v_a1_w, v_b1_w); + const __m128i v_ab1h_w = _mm_unpackhi_epi16(v_a1_w, v_b1_w); + const __m128i v_ab2l_w = _mm_unpacklo_epi16(v_a2_w, v_b2_w); + const __m128i v_ab2h_w = _mm_unpackhi_epi16(v_a2_w, v_b2_w); + const __m128i v_ab3l_w = _mm_unpacklo_epi16(v_a3_w, v_b3_w); + const __m128i v_ab3h_w = _mm_unpackhi_epi16(v_a3_w, v_b3_w); + + // Negate top word of pairs + const __m128i v_abl0n_w = negm_epi16(v_ab0l_w, v_neg_w); + const __m128i v_abh0n_w = negm_epi16(v_ab0h_w, v_neg_w); + const __m128i v_abl1n_w = negm_epi16(v_ab1l_w, v_neg_w); + const __m128i v_abh1n_w = negm_epi16(v_ab1h_w, v_neg_w); + const __m128i v_abl2n_w = negm_epi16(v_ab2l_w, v_neg_w); + const __m128i v_abh2n_w = negm_epi16(v_ab2h_w, v_neg_w); + const __m128i v_abl3n_w = negm_epi16(v_ab3l_w, v_neg_w); + const __m128i v_abh3n_w = negm_epi16(v_ab3h_w, v_neg_w); + + const __m128i v_r0l_w = _mm_madd_epi16(v_ab0l_w, v_abl0n_w); + const __m128i v_r0h_w = _mm_madd_epi16(v_ab0h_w, v_abh0n_w); + const __m128i v_r1l_w = _mm_madd_epi16(v_ab1l_w, v_abl1n_w); + const __m128i v_r1h_w = _mm_madd_epi16(v_ab1h_w, v_abh1n_w); + const __m128i v_r2l_w = _mm_madd_epi16(v_ab2l_w, v_abl2n_w); + const __m128i v_r2h_w = _mm_madd_epi16(v_ab2h_w, v_abh2n_w); + const __m128i v_r3l_w = _mm_madd_epi16(v_ab3l_w, v_abl3n_w); + const __m128i v_r3h_w = _mm_madd_epi16(v_ab3h_w, v_abh3n_w); + + const __m128i v_r0_w = _mm_packs_epi32(v_r0l_w, v_r0h_w); + const __m128i v_r1_w = _mm_packs_epi32(v_r1l_w, v_r1h_w); + const __m128i v_r2_w = _mm_packs_epi32(v_r2l_w, v_r2h_w); + const __m128i v_r3_w = _mm_packs_epi32(v_r3l_w, v_r3h_w); + + xx_store_128(d, v_r0_w); + xx_store_128(d + 8, v_r1_w); + xx_store_128(d + 16, v_r2_w); + xx_store_128(d + 24, v_r3_w); + + a += 32; + b += 32; + d += 32; + N -= 32; + } while (N); +} diff --git a/third_party/aom/av1/exports_dec b/third_party/aom/av1/exports_dec new file mode 100644 index 0000000000..05860e8c0a --- /dev/null +++ b/third_party/aom/av1/exports_dec @@ -0,0 +1,2 @@ +data aom_codec_av1_dx_algo +text aom_codec_av1_dx diff --git a/third_party/aom/av1/exports_enc b/third_party/aom/av1/exports_enc new file mode 100644 index 0000000000..dc4a9eae79 --- /dev/null +++ b/third_party/aom/av1/exports_enc @@ -0,0 +1,2 @@ +data aom_codec_av1_cx_algo +text aom_codec_av1_cx diff --git a/third_party/aom/build/cmake/aom_config.c.cmake b/third_party/aom/build/cmake/aom_config.c.cmake new file mode 100644 index 0000000000..70bf950371 --- /dev/null +++ b/third_party/aom/build/cmake/aom_config.c.cmake @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include "aom/aom_codec.h" +static const char* const cfg = "${AOM_CMAKE_CONFIG}"; +static const char* const aom_git_hash = "${AOM_GIT_HASH}"; +const char *aom_codec_build_config(void) {return cfg;} +const char *aom_codec_git_hash(void) {return aom_git_hash;} diff --git a/third_party/aom/build/cmake/aom_config_defaults.cmake b/third_party/aom/build/cmake/aom_config_defaults.cmake new file mode 100644 index 0000000000..5c2bc5801f --- /dev/null +++ b/third_party/aom/build/cmake/aom_config_defaults.cmake @@ -0,0 +1,167 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +# Defaults for every libaom configuration variable. Here we add all libaom +# config variables to the cmake variable cache, but omit the FORCE parameter to +# allow users to specify values when executing cmake to generate build files. +# Values here are used only if not set by the user. +set(RESTRICT "" CACHE STRING "Sets RESTRICT value for current target.") +set(INLINE "" CACHE STRING "Sets INLINE value for current target.") +set(ARCH_ARM 0 CACHE BOOL "Enables ARM architecture.") +set(ARCH_MIPS 0 CACHE BOOL "Enables MIPS architecture.") +set(ARCH_X86 0 CACHE BOOL "Enables X86 architecture.") +set(ARCH_X86_64 0 CACHE BOOL "Enables X86_64 architecture.") +set(HAVE_EDSP 0 CACHE BOOL "Enables EDSP optimizations.") +set(HAVE_MEDIA 0 CACHE BOOL "Enables MEDIA optimizations.") +set(HAVE_NEON 0 CACHE BOOL "Enables NEON intrinsics optimizations.") +set(HAVE_NEON_ASM 0 CACHE BOOL "Enables NEON assembly optimizations.") +set(HAVE_MIPS32 0 CACHE BOOL "Enables MIPS32 optimizations.") +set(HAVE_DSPR2 0 CACHE BOOL "Enables DSPR2 optimizations.") +set(HAVE_MSA 0 CACHE BOOL "Enables MSA optimizations.") +set(HAVE_MIPS64 0 CACHE BOOL "Enables MIPS64 optimizations. ") +set(HAVE_MMX 0 CACHE BOOL "Enables MMX optimizations. ") +set(HAVE_SSE 0 CACHE BOOL "Enables SSE optimizations.") +set(HAVE_SSE2 0 CACHE BOOL "Enables SSE2 optimizations.") +set(HAVE_SSE3 0 CACHE BOOL "Enables SSE3 optimizations.") +set(HAVE_SSSE3 0 CACHE BOOL "Enables SSSE3 optimizations.") +set(HAVE_SSE4_1 0 CACHE BOOL "Enables SSE 4.1 optimizations.") +set(HAVE_AVX 0 CACHE BOOL "Enables AVX optimizations.") +set(HAVE_AVX2 0 CACHE BOOL "Enables AVX2 optimizations.") +set(HAVE_AOM_PORTS 0 CACHE BOOL "Internal flag, deprecated.") +set(HAVE_FEXCEPT 0 CACHE BOOL "Internal flag, GNU fenv.h present for target.") +set(HAVE_PTHREAD_H 0 CACHE BOOL "Internal flag, target pthread support.") +set(HAVE_UNISTD_H 0 CACHE BOOL "Internal flag, unistd.h present for target.") +set(HAVE_WXWIDGETS 0 CACHE BOOL "WxWidgets present.") +set(CONFIG_DEPENDENCY_TRACKING 1 CACHE BOOL "Internal flag.") +set(CONFIG_EXTERNAL_BUILD 0 CACHE BOOL "Internal flag.") +set(CONFIG_INSTALL_DOCS 0 CACHE BOOL "Internal flag.") +set(CONFIG_INSTALL_BINS 0 CACHE BOOL "Internal flag.") +set(CONFIG_INSTALL_LIBS 0 CACHE BOOL "Internal flag.") +set(CONFIG_INSTALL_SRCS 0 CACHE BOOL "Internal flag.") +set(CONFIG_DEBUG 0 CACHE BOOL "Internal flag.") +set(CONFIG_GPROF 0 CACHE BOOL "Internal flag.") +set(CONFIG_GCOV 0 CACHE BOOL "Internal flag.") +set(CONFIG_RVCT 0 CACHE BOOL "Internal flag.") +set(CONFIG_GCC 0 CACHE BOOL "Internal flag.") +set(CONFIG_MSVS 0 CACHE BOOL "Internal flag.") +set(CONFIG_PIC 0 CACHE BOOL "Internal flag.") +set(CONFIG_BIG_ENDIAN 0 CACHE BOOL "Internal flag.") +set(CONFIG_CODEC_SRCS 0 CACHE BOOL "Internal flag.") +set(CONFIG_DEBUG_LIBS 0 CACHE BOOL "Internal flag.") +set(CONFIG_RUNTIME_CPU_DETECT 1 CACHE BOOL "Internal flag.") +set(CONFIG_POSTPROC 1 CACHE BOOL "Internal flag.") +set(CONFIG_MULTITHREAD 1 CACHE BOOL "Internal flag.") +set(CONFIG_INTERNAL_STATS 0 CACHE BOOL "Internal flag.") +set(CONFIG_AV1_ENCODER 1 CACHE BOOL "Enable AV1 encoder.") +set(CONFIG_AV1_DECODER 1 CACHE BOOL "Enable AV1 decoder.") +set(CONFIG_AV1 1 CACHE BOOL "Internal flag.") +set(CONFIG_ENCODERS 1 CACHE BOOL "Enable encoding.") +set(CONFIG_DECODERS 1 CACHE BOOL "Enable decoding.") +set(CONFIG_STATIC_MSVCRT 0 CACHE BOOL "Internal flag.") +set(CONFIG_SPATIAL_RESAMPLING 1 CACHE BOOL "Internal flag.") +set(CONFIG_REALTIME_ONLY 0 CACHE BOOL "Internal flag.") +set(CONFIG_ONTHEFLY_BITPACKING 0 CACHE BOOL "Internal flag.") +set(CONFIG_ERROR_CONCEALMENT 0 CACHE BOOL "Internal flag.") +set(CONFIG_SHARED 0 CACHE BOOL "Internal flag.") +set(CONFIG_STATIC 1 CACHE BOOL "Internal flag.") +set(CONFIG_SMALL 0 CACHE BOOL "Internal flag.") +set(CONFIG_POSTPROC_VISUALIZER 0 CACHE BOOL "Internal flag.") +set(CONFIG_OS_SUPPORT 0 CACHE BOOL "Internal flag.") +set(CONFIG_UNIT_TESTS 1 CACHE BOOL "Internal flag.") +set(CONFIG_WEBM_IO 1 CACHE BOOL "Enables WebM support.") +set(CONFIG_LIBYUV 1 CACHE BOOL "Enables libyuv scaling and conversion support.") +set(CONFIG_ACCOUNTING 0 CACHE BOOL "Enables bit accounting.") +set(CONFIG_INSPECTION 0 CACHE BOOL "Enables bitstream inspection.") +set(CONFIG_DECODE_PERF_TESTS 0 CACHE BOOL "Internal flag.") +set(CONFIG_ENCODE_PERF_TESTS 0 CACHE BOOL "Internal flag.") +set(CONFIG_COEFFICIENT_RANGE_CHECKING 0 CACHE BOOL "Internal flag.") +set(CONFIG_LOWBITDEPTH 1 CACHE BOOL "Internal flag.") +set(CONFIG_HIGHBITDEPTH 0 CACHE BOOL "Internal flag.") +set(CONFIG_EXPERIMENTAL 0 CACHE BOOL "Internal flag.") +set(CONFIG_SIZE_LIMIT 0 CACHE BOOL "Internal flag.") +set(CONFIG_AOM_QM 0 CACHE BOOL "Internal flag.") +set(CONFIG_FP_MB_STATS 0 CACHE BOOL "Internal flag.") +set(CONFIG_CDEF 1 CACHE BOOL "Internal flag.") +set(CONFIG_VAR_TX 0 CACHE BOOL "Internal flag.") +set(CONFIG_RECT_TX 1 CACHE BOOL "Internal flag.") +set(CONFIG_REF_MV 1 CACHE BOOL "Internal flag.") +set(CONFIG_TPL_MV 0 CACHE BOOL "Internal flag.") +set(CONFIG_DUAL_FILTER 1 CACHE BOOL "Internal flag.") +set(CONFIG_CONVOLVE_ROUND 0 CACHE BOOL "Internal flag.") +set(CONFIG_COMPOUND_ROUND 0 CACHE BOOL "Internal flag.") +set(CONFIG_EXT_TX 0 CACHE BOOL "Internal flag.") +set(CONFIG_TX64X64 0 CACHE BOOL "Internal flag.") +set(CONFIG_SUB8X8_MC 0 CACHE BOOL "Internal flag.") +set(CONFIG_EXT_INTRA 1 CACHE BOOL "Internal flag.") +set(CONFIG_INTRA_INTERP 0 CACHE BOOL "Internal flag.") +set(CONFIG_FILTER_INTRA 0 CACHE BOOL "Internal flag.") +set(CONFIG_INTRABC 0 CACHE BOOL "Internal flag.") +set(CONFIG_EXT_INTER 0 CACHE BOOL "Internal flag.") +set(CONFIG_INTERINTRA 0 CACHE BOOL "Internal flag.") +set(CONFIG_WEDGE 0 CACHE BOOL "Internal flag.") +set(CONFIG_COMPOUND_SEGMENT 0 CACHE BOOL "Internal flag.") +set(CONFIG_EXT_REFS 1 CACHE BOOL "Internal flag.") +set(CONFIG_GLOBAL_MOTION 0 CACHE BOOL "Internal flag.") +set(CONFIG_NEW_QUANT 0 CACHE BOOL "Internal flag.") +set(CONFIG_SUPERTX 0 CACHE BOOL "Internal flag.") +set(CONFIG_ANS 0 CACHE BOOL "Internal flag.") +set(CONFIG_EC_MULTISYMBOL 1 CACHE BOOL "Internal flag.") +set(CONFIG_NEW_TOKENSET 1 CACHE BOOL "Internal flag.") +set(CONFIG_LOOP_RESTORATION 0 CACHE BOOL "Internal flag.") +set(CONFIG_EXT_PARTITION 0 CACHE BOOL "Internal flag.") +set(CONFIG_EXT_PARTITION_TYPES 0 CACHE BOOL "Internal flag.") +set(CONFIG_UNPOISON_PARTITION_CTX 0 CACHE BOOL "Internal flag.") +set(CONFIG_EXT_TILE 0 CACHE BOOL "Internal flag.") +set(CONFIG_MOTION_VAR 0 CACHE BOOL "Internal flag.") +set(CONFIG_NCOBMC 0 CACHE BOOL "Internal flag.") +set(CONFIG_WARPED_MOTION 0 CACHE BOOL "Internal flag.") +set(CONFIG_Q_ADAPT_PROBS 0 CACHE BOOL "Internal flag.") +set(CONFIG_SUBFRAME_PROB_UPDATE 0 CACHE BOOL "Internal flag.") +set(CONFIG_BITSTREAM_DEBUG 0 CACHE BOOL "Internal flag.") +set(CONFIG_ALT_INTRA 1 CACHE BOOL "Internal flag.") +set(CONFIG_PALETTE 1 CACHE BOOL "Internal flag.") +set(CONFIG_PALETTE_DELTA_ENCODING 0 CACHE BOOL "Internal flag.") +set(CONFIG_DAALA_EC 1 CACHE BOOL "Internal flag.") +set(CONFIG_RAWBITS 0 CACHE BOOL "Internal flag.") +set(CONFIG_EC_SMALLMUL 0 CACHE BOOL "Internal flag.") +set(CONFIG_PVQ 0 CACHE BOOL "Internal flag.") +set(CONFIG_CFL 0 CACHE BOOL "Internal flag.") +set(CONFIG_XIPHRC 0 CACHE BOOL "Internal flag.") +set(CONFIG_CB4X4 1 CACHE BOOL "Internal flag.") +set(CONFIG_CHROMA_2X2 0 CACHE BOOL "Internal flag.") +set(CONFIG_FRAME_SIZE 0 CACHE BOOL "Internal flag.") +set(CONFIG_DELTA_Q 1 CACHE BOOL "Internal flag.") +set(CONFIG_EXT_DELTA_Q 0 CACHE BOOL "Internal flag.") +set(CONFIG_ADAPT_SCAN 0 CACHE BOOL "Internal flag.") +set(CONFIG_FILTER_7BIT 1 CACHE BOOL "Internal flag.") +set(CONFIG_PARALLEL_DEBLOCKING 0 CACHE BOOL "Internal flag.") +set(CONFIG_PARALLEL_DEBLOCKING_15TAP 0 CACHE BOOL "Internal flag.") +set(CONFIG_LOOPFILTERING_ACROSS_TILES 0 CACHE BOOL "Internal flag.") +set(CONFIG_TILE_GROUPS 1 CACHE BOOL "Internal flag.") +set(CONFIG_EC_ADAPT 1 CACHE BOOL "Internal flag.") +set(CONFIG_TEMPMV_SIGNALING 0 CACHE BOOL "Internal flag.") +set(CONFIG_RD_DEBUG 0 CACHE BOOL "Internal flag.") +set(CONFIG_REFERENCE_BUFFER 1 CACHE BOOL "Internal flag.") +set(CONFIG_COEF_INTERLEAVE 0 CACHE BOOL "Internal flag.") +set(CONFIG_ENTROPY_STATS 0 CACHE BOOL "Internal flag.") +set(CONFIG_MASKED_TX 0 CACHE BOOL "Internal flag.") +set(CONFIG_DEPENDENT_HORZTILES 0 CACHE BOOL "Internal flag.") +set(CONFIG_DAALA_DIST 0 CACHE BOOL "Internal flag.") +set(CONFIG_TRIPRED 0 CACHE BOOL "Internal flag.") +set(CONFIG_PALETTE_THROUGHPUT 1 CACHE BOOL "Internal flag.") +set(CONFIG_REF_ADAPT 0 CACHE BOOL "Internal flag.") +set(CONFIG_LV_MAP 0 CACHE BOOL "Internal flag.") +set(CONFIG_TXK_SEL 0 CACHE BOOL "Internal flag.") +set(CONFIG_MV_COMPRESS 1 CACHE BOOL "Internal flag.") +set(CONFIG_FRAME_SUPERRES 0 CACHE BOOL "Internal flag.") +set(CONFIG_NEW_MULTISYMBOL 0 CACHE BOOL "Internal flag.") +set(CONFIG_COMPOUND_SINGLEREF 0 CACHE BOOL "Internal flag.") +set(CONFIG_ANALYZER 0 CACHE BOOL "Internal flag.") diff --git a/third_party/aom/build/cmake/aom_configure.cmake b/third_party/aom/build/cmake/aom_configure.cmake new file mode 100644 index 0000000000..3c9402d0bd --- /dev/null +++ b/third_party/aom/build/cmake/aom_configure.cmake @@ -0,0 +1,306 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +include(FindGit) +include(FindPerl) +include(FindThreads) +include(FindwxWidgets) + +set(AOM_SUPPORTED_CPU_TARGETS + "arm64 armv7 armv7s generic mips32 mips64 x86 x86_64") + +# Generate the user config settings. This must occur before include of +# aom_config_defaults.cmake (because it turns every config variable into a cache +# variable with its own help string). +get_cmake_property(cmake_cache_vars CACHE_VARIABLES) +foreach(cache_var ${cmake_cache_vars}) + get_property(cache_var_helpstring CACHE ${cache_var} PROPERTY HELPSTRING) + set(cmdline_helpstring "No help, variable specified on the command line.") + if("${cache_var_helpstring}" STREQUAL "${cmdline_helpstring}") + set(AOM_CMAKE_CONFIG "${AOM_CMAKE_CONFIG} -D${cache_var}=${${cache_var}}") + endif () +endforeach() +string(STRIP "${AOM_CMAKE_CONFIG}" AOM_CMAKE_CONFIG) + +include("${AOM_ROOT}/build/cmake/aom_config_defaults.cmake") +include("${AOM_ROOT}/build/cmake/aom_optimization.cmake") +include("${AOM_ROOT}/build/cmake/compiler_flags.cmake") +include("${AOM_ROOT}/build/cmake/compiler_tests.cmake") + +# Build a list of all configurable variables. +get_cmake_property(cmake_cache_vars CACHE_VARIABLES) +foreach (var ${cmake_cache_vars}) + if ("${var}" MATCHES "^CONFIG_") + list(APPEND AOM_CONFIG_VARS ${var}) + endif () +endforeach () + +# Detect target CPU. +if (NOT AOM_TARGET_CPU) + if ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "AMD64" OR + "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64") + if (${CMAKE_SIZEOF_VOID_P} EQUAL 4) + set(AOM_TARGET_CPU "x86") + elseif (${CMAKE_SIZEOF_VOID_P} EQUAL 8) + set(AOM_TARGET_CPU "x86_64") + else () + message(FATAL_ERROR + "--- Unexpected pointer size (${CMAKE_SIZEOF_VOID_P}) for\n" + " CMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME}\n" + " CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}\n" + " CMAKE_GENERATOR=${CMAKE_GENERATOR}\n") + endif () + elseif ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "i386" OR + "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86") + set(AOM_TARGET_CPU "x86") + elseif ("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "^arm" OR + "${CMAKE_SYSTEM_PROCESSOR}" MATCHES "^mips") + set(AOM_TARGET_CPU "${CMAKE_SYSTEM_PROCESSOR}") + endif () +endif () + +if (CMAKE_TOOLCHAIN_FILE) + # Add toolchain file to config string. + set(toolchain_string "-DCMAKE_TOOLCHAIN_FILE=\\\"${CMAKE_TOOLCHAIN_FILE}\\\"") + set(AOM_CMAKE_CONFIG "${toolchain_string} ${AOM_CMAKE_CONFIG}") +else () + # Add detected CPU to the config string. + set(AOM_CMAKE_CONFIG "-DAOM_TARGET_CPU=${AOM_TARGET_CPU} ${AOM_CMAKE_CONFIG}") +endif () +set(AOM_CMAKE_CONFIG "-G \\\"${CMAKE_GENERATOR}\\\" ${AOM_CMAKE_CONFIG}") +string(STRIP "${AOM_CMAKE_CONFIG}" AOM_CMAKE_CONFIG) + +message("--- aom_configure: Detected CPU: ${AOM_TARGET_CPU}") +set(AOM_TARGET_SYSTEM ${CMAKE_SYSTEM_NAME}) + +if (NOT "${AOM_SUPPORTED_CPU_TARGETS}" MATCHES "${AOM_TARGET_CPU}") + message(FATAL_ERROR "No RTCD support for ${AOM_TARGET_CPU}. Create it, or " + "add -DAOM_TARGET_CPU=generic to your cmake command line for a " + "generic build of libaom and tools.") +endif () + +if ("${AOM_TARGET_CPU}" STREQUAL "x86" OR "${AOM_TARGET_CPU}" STREQUAL "x86_64") + # TODO(tomfinegan): Support nasm at least as well as the existing build + # system. + find_program(AS_EXECUTABLE yasm $ENV{YASM_PATH}) + if (NOT AS_EXECUTABLE) + message(FATAL_ERROR "Unable to find yasm. To build without optimizations, " + "add -DAOM_TARGET_CPU=generic to your cmake command line.") + endif () + get_asm_obj_format("objformat") + set(AOM_AS_FLAGS -f ${objformat} ${AOM_AS_FLAGS}) + string(STRIP "${AOM_AS_FLAGS}" AOM_AS_FLAGS) +elseif ("${AOM_TARGET_CPU}" MATCHES "arm") + if ("${AOM_TARGET_SYSTEM}" STREQUAL "Darwin") + set(AS_EXECUTABLE as) + set(AOM_AS_FLAGS -arch ${AOM_TARGET_CPU} -isysroot ${CMAKE_OSX_SYSROOT}) + elseif ("${AOM_TARGET_SYSTEM}" STREQUAL "Linux") + # arm linux assembler settings controlled by + # build/cmake/toolchains/arm*-linux*.cmake + endif () + if (NOT AS_EXECUTABLE) + message(FATAL_ERROR + "Unknown assembler for: ${AOM_TARGET_CPU}-${AOM_TARGET_SYSTEM}") + endif () + + string(STRIP "${AOM_AS_FLAGS}" AOM_AS_FLAGS) +endif () + +include("${AOM_ROOT}/build/cmake/cpu.cmake") + +# Test compiler flags. +if (MSVC) + add_compiler_flag_if_supported("/W3") + # Disable MSVC warnings that suggest making code non-portable. + add_compiler_flag_if_supported("/wd4996") + if (ENABLE_WERROR) + add_compiler_flag_if_supported("/WX") + endif () +else () + require_c_flag("-std=c99" YES) + add_compiler_flag_if_supported("-Wall") + add_compiler_flag_if_supported("-Wdisabled-optimization") + add_compiler_flag_if_supported("-Wextra") + add_compiler_flag_if_supported("-Wfloat-conversion") + add_compiler_flag_if_supported("-Wimplicit-function-declaration") + add_compiler_flag_if_supported("-Wpointer-arith") + add_compiler_flag_if_supported("-Wsign-compare") + add_compiler_flag_if_supported("-Wtype-limits") + add_compiler_flag_if_supported("-Wuninitialized") + add_compiler_flag_if_supported("-Wunused") + add_compiler_flag_if_supported("-Wvla") + # TODO(jzern): this could be added as a cxx flags for test/*.cc only, + # avoiding third_party. + add_c_flag_if_supported("-Wshorten-64-to-32") + + # Add -Wshadow only for C files to avoid massive gtest warning spam. + add_c_flag_if_supported("-Wshadow") + + # Add -Wundef only for C files to avoid massive gtest warning spam. + add_c_flag_if_supported("-Wundef") + + if (ENABLE_WERROR) + add_compiler_flag_if_supported("-Werror") + endif () + # Flag(s) added here negate CMake defaults and produce build output similar + # to the existing configure/make build system. + add_compiler_flag_if_supported("-Wno-unused-function") + + if (CMAKE_C_COMPILER_ID MATCHES "GNU\|Clang") + set(CONFIG_GCC 1) + endif () + + if ("${CMAKE_BUILD_TYPE}" MATCHES "Rel") + add_compiler_flag_if_supported("-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0") + endif () + add_compiler_flag_if_supported("-D_LARGEFILE_SOURCE") + add_compiler_flag_if_supported("-D_FILE_OFFSET_BITS=64") +endif () + +if ("${AOM_TARGET_SYSTEM}" MATCHES "Darwin\|Linux\|Windows") + set(CONFIG_OS_SUPPORT 1) +endif () + +# Test compiler support. +aom_get_inline("INLINE") + +# TODO(tomfinegan): aom_ports_check is legacy; HAVE_AOM_PORTS is not used +# anywhere in the aom sources. To be removed after parity with the legacy +# build system stops being important. +aom_check_source_compiles("aom_ports_check" + "#include \"${AOM_ROOT}/aom/aom_integer.h\"" + HAVE_AOM_PORTS) +aom_check_source_compiles("pthread_check" "#include " HAVE_PTHREAD_H) +aom_check_source_compiles("unistd_check" "#include " HAVE_UNISTD_H) + +if (CONFIG_ANALYZER) + find_package(wxWidgets REQUIRED adv base core) + include(${wxWidgets_USE_FILE}) + + if (NOT CONFIG_INSPECTION) + set(CONFIG_INSPECTION 1) + message(WARNING + "--- Enabled CONFIG_INSPECTION, required for CONFIG_ANALYZER.") + endif () +endif () + +if (CONFIG_ANS AND CONFIG_DAALA_EC) + message(FATAL_ERROR + "CONFIG_ANS and CONFIG_DAALA_EC cannot be enabled together.") +endif () + +if (NOT MSVC) + aom_push_var(CMAKE_REQUIRED_LIBRARIES "m") + aom_check_c_compiles("fenv_check" + "#define _GNU_SOURCE + #include + void unused(void) { + (void)feenableexcept(FE_DIVBYZERO | FE_INVALID); + }" HAVE_FEXCEPT) + aom_pop_var(CMAKE_REQUIRED_LIBRARIES) +endif() + +# Generate aom_config templates. +set(aom_config_asm_template "${AOM_CONFIG_DIR}/aom_config.asm.cmake") +set(aom_config_h_template "${AOM_CONFIG_DIR}/aom_config.h.cmake") +execute_process(COMMAND ${CMAKE_COMMAND} + -DAOM_CONFIG_DIR=${AOM_CONFIG_DIR} + -DAOM_ROOT=${AOM_ROOT} + -P "${AOM_ROOT}/build/cmake/generate_aom_config_templates.cmake") + +# Generate aom_config.{asm,h}. +configure_file("${aom_config_asm_template}" "${AOM_CONFIG_DIR}/aom_config.asm") +configure_file("${aom_config_h_template}" "${AOM_CONFIG_DIR}/aom_config.h") + +# Read the current git hash. +find_package(Git) +set(AOM_GIT_DESCRIPTION) +set(AOM_GIT_HASH) +if (GIT_FOUND) + # TODO(tomfinegan): Add build rule so users don't have to re-run cmake to + # create accurately versioned cmake builds. + execute_process(COMMAND ${GIT_EXECUTABLE} + --git-dir=${AOM_ROOT}/.git rev-parse HEAD + OUTPUT_VARIABLE AOM_GIT_HASH) + execute_process(COMMAND ${GIT_EXECUTABLE} --git-dir=${AOM_ROOT}/.git describe + OUTPUT_VARIABLE AOM_GIT_DESCRIPTION ERROR_QUIET) + # Consume the newline at the end of the git output. + string(STRIP "${AOM_GIT_HASH}" AOM_GIT_HASH) + string(STRIP "${AOM_GIT_DESCRIPTION}" AOM_GIT_DESCRIPTION) +endif () + +configure_file("${AOM_ROOT}/build/cmake/aom_config.c.cmake" + "${AOM_CONFIG_DIR}/aom_config.c") + +# Find Perl and generate the RTCD sources. +find_package(Perl) +if (NOT PERL_FOUND) + message(FATAL_ERROR "Perl is required to build libaom.") +endif () +configure_file( + "${AOM_ROOT}/build/cmake/rtcd_config.cmake" + "${AOM_CONFIG_DIR}/${AOM_TARGET_CPU}_rtcd_config.rtcd") + +set(AOM_RTCD_CONFIG_FILE_LIST + "${AOM_ROOT}/aom_dsp/aom_dsp_rtcd_defs.pl" + "${AOM_ROOT}/aom_scale/aom_scale_rtcd.pl" + "${AOM_ROOT}/av1/common/av1_rtcd_defs.pl") +set(AOM_RTCD_HEADER_FILE_LIST + "${AOM_CONFIG_DIR}/aom_dsp_rtcd.h" + "${AOM_CONFIG_DIR}/aom_scale_rtcd.h" + "${AOM_CONFIG_DIR}/av1_rtcd.h") +set(AOM_RTCD_SOURCE_FILE_LIST + "${AOM_ROOT}/aom_dsp/aom_dsp_rtcd.c" + "${AOM_ROOT}/aom_scale/aom_scale_rtcd.c" + "${AOM_ROOT}/av1/common/av1_rtcd.c") +set(AOM_RTCD_SYMBOL_LIST aom_dsp_rtcd aom_scale_rtcd av1_rtcd) +list(LENGTH AOM_RTCD_SYMBOL_LIST AOM_RTCD_CUSTOM_COMMAND_COUNT) +math(EXPR AOM_RTCD_CUSTOM_COMMAND_COUNT "${AOM_RTCD_CUSTOM_COMMAND_COUNT} - 1") + +foreach(NUM RANGE ${AOM_RTCD_CUSTOM_COMMAND_COUNT}) + list(GET AOM_RTCD_CONFIG_FILE_LIST ${NUM} AOM_RTCD_CONFIG_FILE) + list(GET AOM_RTCD_HEADER_FILE_LIST ${NUM} AOM_RTCD_HEADER_FILE) + list(GET AOM_RTCD_SOURCE_FILE_LIST ${NUM} AOM_RTCD_SOURCE_FILE) + list(GET AOM_RTCD_SYMBOL_LIST ${NUM} AOM_RTCD_SYMBOL) + execute_process( + COMMAND ${PERL_EXECUTABLE} "${AOM_ROOT}/build/make/rtcd.pl" + --arch=${AOM_TARGET_CPU} --sym=${AOM_RTCD_SYMBOL} ${AOM_RTCD_FLAGS} + --config=${AOM_CONFIG_DIR}/${AOM_TARGET_CPU}_rtcd_config.rtcd + ${AOM_RTCD_CONFIG_FILE} + OUTPUT_FILE ${AOM_RTCD_HEADER_FILE}) +endforeach() + +function (add_rtcd_build_step config output source symbol) + add_custom_command( + OUTPUT ${output} + COMMAND ${PERL_EXECUTABLE} + ARGS "${AOM_ROOT}/build/make/rtcd.pl" + --arch=${AOM_TARGET_CPU} + --sym=${symbol} + ${AOM_RTCD_FLAGS} + --config=${AOM_CONFIG_DIR}/${AOM_TARGET_CPU}_rtcd_config.rtcd + ${config} + > ${output} + DEPENDS ${config} + COMMENT "Generating ${output}" + WORKING_DIRECTORY ${AOM_CONFIG_DIR} + VERBATIM) + set_property(SOURCE ${source} PROPERTY OBJECT_DEPENDS ${output}) + set_property(SOURCE ${output} PROPERTY GENERATED) +endfunction () + +# Generate aom_version.h. +if ("${AOM_GIT_DESCRIPTION}" STREQUAL "") + set(AOM_GIT_DESCRIPTION "${AOM_ROOT}/CHANGELOG") +endif () +execute_process( + COMMAND ${PERL_EXECUTABLE} "${AOM_ROOT}/build/cmake/aom_version.pl" + --version_data=${AOM_GIT_DESCRIPTION} + --version_filename=${AOM_CONFIG_DIR}/aom_version.h) diff --git a/third_party/aom/build/cmake/aom_optimization.cmake b/third_party/aom/build/cmake/aom_optimization.cmake new file mode 100644 index 0000000000..e2b0ba07e3 --- /dev/null +++ b/third_party/aom/build/cmake/aom_optimization.cmake @@ -0,0 +1,204 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_BUILD_CMAKE_AOM_OPTIMIZATION_CMAKE_) +set(AOM_BUILD_CMAKE_AOM_OPTIMIZATION_CMAKE_ 1) + +# Translate $flag to one which MSVC understands, and write the new flag to the +# variable named by $translated_flag (or unset it, when MSVC needs no flag). +function (get_msvc_intrinsic_flag flag translated_flag) + if ("${flag}" STREQUAL "-mavx") + set(${translated_flag} "/arch:AVX" PARENT_SCOPE) + elseif ("${flag}" STREQUAL "-mavx2") + set(${translated_flag} "/arch:AVX2" PARENT_SCOPE) + else () + # MSVC does not need flags for intrinsics flavors other than AVX/AVX2. + unset(${translated_flag} PARENT_SCOPE) + endif () +endfunction () + +# Adds an object library target. Terminates generation if $flag is not supported +# by the current compiler. $flag is the intrinsics flag required by the current +# compiler, and is added to the compile flags for all sources in $sources. +# $opt_name is used to name the target. $target_to_update is made +# dependent upon the created target. +# +# Note: the libaom target is always updated because OBJECT libraries have rules +# that disallow the direct addition of .o files to them as dependencies. Static +# libraries do not have this limitation. +function (add_intrinsics_object_library flag opt_name target_to_update sources) + set(target_name ${target_to_update}_${opt_name}_intrinsics) + add_library(${target_name} OBJECT ${${sources}}) + + if (MSVC) + get_msvc_intrinsic_flag(${flag} "flag") + endif () + + if (flag) + target_compile_options(${target_name} PUBLIC ${flag}) + endif () + + target_sources(aom PUBLIC $) + + # Add the new lib target to the global list of aom library targets. + list(APPEND AOM_LIB_TARGETS ${target_name}) + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} PARENT_SCOPE) +endfunction () + +# Adds sources in list named by $sources to $target and adds $flag to the +# compile flags for each source file. +function (add_intrinsics_source_to_target flag target sources) + target_sources(${target} PUBLIC ${${sources}}) + if (MSVC) + get_msvc_intrinsic_flag(${flag} "flag") + endif () + if (flag) + foreach (source ${${sources}}) + set_property(SOURCE ${source} APPEND PROPERTY COMPILE_FLAGS ${flag}) + endforeach () + endif () +endfunction () + +# Writes object format for the current target to the var named by $out_format, +# or terminates the build when the object format for the current target is +# unknown. +function (get_asm_obj_format out_format) + if ("${AOM_TARGET_CPU}" STREQUAL "x86_64") + if ("${AOM_TARGET_SYSTEM}" STREQUAL "Darwin") + set(objformat "macho64") + elseif ("${AOM_TARGET_SYSTEM}" STREQUAL "Linux") + set(objformat "elf64") + elseif ("${AOM_TARGET_SYSTEM}" STREQUAL "Windows") + set(objformat "win64") + else () + message(FATAL_ERROR "Unknown obj format: ${AOM_TARGET_SYSTEM}") + endif () + elseif ("${AOM_TARGET_CPU}" STREQUAL "x86") + if ("${AOM_TARGET_SYSTEM}" STREQUAL "Darwin") + set(objformat "macho32") + elseif ("${AOM_TARGET_SYSTEM}" STREQUAL "Linux") + set(objformat "elf32") + elseif ("${AOM_TARGET_SYSTEM}" STREQUAL "Windows") + set(objformat "win32") + else () + message(FATAL_ERROR "Unknown obj format: ${AOM_TARGET_SYSTEM}") + endif () + else () + message(FATAL_ERROR + "Unknown obj format: ${AOM_TARGET_CPU}-${AOM_TARGET_SYSTEM}") + endif () + + set(${out_format} ${objformat} PARENT_SCOPE) +endfunction () + +# Adds library target named $lib_name for ASM files in variable named by +# $asm_sources. Builds an output directory path from $lib_name. Links $lib_name +# into $dependent_target. Generates a dummy C file with a dummy function to +# ensure that all cmake generators can determine the linker language, and that +# build tools don't complain that an object exposes no symbols. +function (add_asm_library lib_name asm_sources dependent_target) + set(asm_lib_obj_dir "${AOM_CONFIG_DIR}/asm_objects/${lib_name}") + if (NOT EXISTS "${asm_lib_obj_dir}") + file(MAKE_DIRECTORY "${asm_lib_obj_dir}") + endif () + + # TODO(tomfinegan): If cmake ever allows addition of .o files to OBJECT lib + # targets, make this OBJECT instead of STATIC to hide the target from + # consumers of the AOM cmake build. + add_library(${lib_name} STATIC ${${asm_sources}}) + + foreach (asm_source ${${asm_sources}}) + get_filename_component(asm_source_name "${asm_source}" NAME) + set(asm_object "${asm_lib_obj_dir}/${asm_source_name}.o") + add_custom_command(OUTPUT "${asm_object}" + COMMAND ${AS_EXECUTABLE} + ARGS ${AOM_AS_FLAGS} + -I${AOM_ROOT} -I${AOM_CONFIG_DIR} + -o "${asm_object}" "${asm_source}" + DEPENDS "${asm_source}" + COMMENT "Building ASM object ${asm_object}" + WORKING_DIRECTORY "${AOM_CONFIG_DIR}" + VERBATIM) + target_sources(${lib_name} PRIVATE "${asm_object}") + endforeach () + + # The above created a target containing only ASM sources. Cmake needs help + # here to determine the linker language. Add a dummy C file to force the + # linker language to C. We don't bother with setting the LINKER_LANGUAGE + # property on the library target because not all generators obey it (looking + # at you, xcode generator). + set(dummy_c_file "${AOM_CONFIG_DIR}/${lib_name}_dummy.c") + file(WRITE "${dummy_c_file}" + "// Generated file. DO NOT EDIT!\n" + "// ${lib_name} needs C file to force link language, ignore me.\n" + "void ${lib_name}_dummy_function(void) {}\n") + target_sources(${lib_name} PUBLIC ${dummy_c_file}) + + target_link_libraries(${dependent_target} PRIVATE ${lib_name}) + + # Add the new lib target to the global list of aom library targets. + list(APPEND AOM_LIB_TARGETS ${lib_name}) + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} PARENT_SCOPE) +endfunction () + +# Converts asm sources in $asm_sources using $AOM_ADS2GAS and calls +# add_asm_library() to create a library from the converted sources. At +# generation time the converted sources are created, and a custom rule is added +# to ensure the sources are reconverted when the original asm source is updated. +# See add_asm_library() for more information. +function (add_gas_asm_library lib_name asm_sources dependent_target) + set(asm_converted_source_dir "${AOM_CONFIG_DIR}/asm_gas/${lib_name}") + if (NOT EXISTS "${asm_converted_source_dir}") + file(MAKE_DIRECTORY "${asm_converted_source_dir}") + endif () + + # Create the converted version of each assembly source at generation time. + unset(gas_target_sources) + foreach (neon_asm_source ${${asm_sources}}) + get_filename_component(output_asm_source "${neon_asm_source}" NAME) + set(output_asm_source "${asm_converted_source_dir}/${output_asm_source}") + set(output_asm_source "${output_asm_source}.${AOM_GAS_EXT}") + execute_process(COMMAND "${PERL_EXECUTABLE}" "${AOM_ADS2GAS}" + INPUT_FILE "${neon_asm_source}" + OUTPUT_FILE "${output_asm_source}") + list(APPEND gas_target_sources "${output_asm_source}") + endforeach () + + add_asm_library("${lib_name}" "gas_target_sources" "${dependent_target}") + + # For each of the converted sources, create a custom rule that will regenerate + # the converted source when its input is touched. + list(LENGTH gas_target_sources num_asm_files) + math(EXPR num_asm_files "${num_asm_files} - 1") + foreach(NUM RANGE ${num_asm_files}) + list(GET ${asm_sources} ${NUM} neon_asm_source) + list(GET gas_target_sources ${NUM} gas_asm_source) + + # Grab only the filename for the custom command output to keep build output + # reasonably sane. + get_filename_component(neon_name "${neon_asm_source}" NAME) + get_filename_component(gas_name "${gas_asm_source}" NAME) + + add_custom_command( + OUTPUT "${gas_asm_source}" + COMMAND ${PERL_EXECUTABLE} + ARGS "${AOM_ADS2GAS}" < "${neon_asm_source}" > "${gas_asm_source}" + DEPENDS "${neon_asm_source}" + COMMENT "ads2gas conversion ${neon_name} -> ${gas_name}" + WORKING_DIRECTORY "${AOM_CONFIG_DIR}" + VERBATIM) + endforeach () + + # Update the sources list passed in to include the converted asm source files. + list(APPEND asm_sources ${gas_target_sources}) + set(${asm_sources} ${${asm_sources}} PARENT_SCOPE) +endfunction () + +endif () # AOM_BUILD_CMAKE_AOM_OPTIMIZATION_CMAKE_ diff --git a/third_party/aom/build/cmake/aom_version.pl b/third_party/aom/build/cmake/aom_version.pl new file mode 100755 index 0000000000..3412feebdf --- /dev/null +++ b/third_party/aom/build/cmake/aom_version.pl @@ -0,0 +1,93 @@ +#!/usr/bin/env perl +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +use strict; +use warnings; +use 5.010; +use Getopt::Long; + +my $version_data; +my $version_filename; +GetOptions('version_data=s' => \$version_data, + 'version_filename=s' => \$version_filename) or + die("Invalid arg(s): $!"); + +if (!defined $version_data || length($version_data) == 0 || + !defined $version_filename || length($version_filename) == 0) { + die("--version_data and --version_filename are required."); +} + +# Determine if $version_data is a filename or a git tag/description. +my $version_string; +if (-r $version_data) { + # $version_data is the path to the CHANGELOG. Parse the most recent version. + my $changelog_filename = $version_data; + open(my $changelog_file, '<', $changelog_filename) or + die("Unable to open CHANGELOG @ $changelog_filename: $!."); + + while (my $line = <$changelog_file>) { + my @split_line = split(" ", $line, 3); + next if @split_line < 2; + $version_string = $split_line[1]; + last if substr($version_string, 0, 1) eq "v"; + } + close($changelog_file); +} else { + # $version_data is either a tag name or a full git description, one of: + # tagName OR tagName-commitsSinceTag-shortCommitHash + # In either case we want the first element of the array returned by split. + $version_string = (split("-", $version_data))[0]; +} + +if (substr($version_string, 0, 1) eq "v") { + $version_string = substr($version_string, 1); +} + +my @version_components = split('\.', $version_string, 4); +my $version_major = $version_components[0]; +my $version_minor = $version_components[1]; +my $version_patch = $version_components[2]; + +my $version_extra = ""; +if (@version_components > 3) { + $version_extra = $version_components[3]; +} + +open(my $version_file, '>', $version_filename) or + die("Cannot open $version_filename: $!"); + +my $version_packed = "((VERSION_MAJOR<<16)|(VERSION_MINOR<<8)|(VERSION_PATCH))"; +my $year = (localtime)[5] + 1900; +my $lic_block = << "EOF"; +/* + * Copyright (c) $year, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +EOF + +select $version_file; +print << "EOF"; +$lic_block +#define VERSION_MAJOR $version_major +#define VERSION_MINOR $version_minor +#define VERSION_PATCH $version_patch +#define VERSION_EXTRA \"$version_extra\" +#define VERSION_PACKED $version_packed +#define VERSION_STRING_NOSP \"v$version_string\" +#define VERSION_STRING \" v$version_string\" +EOF +close($version_file); diff --git a/third_party/aom/build/cmake/compiler_flags.cmake b/third_party/aom/build/cmake/compiler_flags.cmake new file mode 100644 index 0000000000..beb217abcb --- /dev/null +++ b/third_party/aom/build/cmake/compiler_flags.cmake @@ -0,0 +1,218 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +include(CheckCCompilerFlag) +include(CheckCXXCompilerFlag) + +# Strings used to cache failed C/CXX flags. +set(AOM_FAILED_C_FLAGS) +set(AOM_FAILED_CXX_FLAGS) + +# Checks C compiler for support of $c_flag. Adds $c_flag to $CMAKE_C_FLAGS when +# the compile test passes. Caches $c_flag in $AOM_FAILED_C_FLAGS when the test +# fails. +function (add_c_flag_if_supported c_flag) + unset(C_FLAG_FOUND CACHE) + string(FIND "${CMAKE_C_FLAGS}" "${c_flag}" C_FLAG_FOUND) + unset(C_FLAG_FAILED CACHE) + string(FIND "${AOM_FAILED_C_FLAGS}" "${c_flag}" C_FLAG_FAILED) + + if (${C_FLAG_FOUND} EQUAL -1 AND ${C_FLAG_FAILED} EQUAL -1) + unset(C_FLAG_SUPPORTED CACHE) + message("Checking C compiler flag support for: " ${c_flag}) + check_c_compiler_flag("${c_flag}" C_FLAG_SUPPORTED) + if (C_FLAG_SUPPORTED) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${c_flag}" CACHE STRING "" FORCE) + else () + set(AOM_FAILED_C_FLAGS "${AOM_FAILED_C_FLAGS} ${c_flag}" CACHE STRING "" + FORCE) + endif () + endif () +endfunction () + +# Checks C++ compiler for support of $cxx_flag. Adds $cxx_flag to +# $CMAKE_CXX_FLAGS when the compile test passes. Caches $c_flag in +# $AOM_FAILED_CXX_FLAGS when the test fails. +function (add_cxx_flag_if_supported cxx_flag) + unset(CXX_FLAG_FOUND CACHE) + string(FIND "${CMAKE_CXX_FLAGS}" "${cxx_flag}" CXX_FLAG_FOUND) + unset(CXX_FLAG_FAILED CACHE) + string(FIND "${AOM_FAILED_CXX_FLAGS}" "${cxx_flag}" CXX_FLAG_FAILED) + + if (${CXX_FLAG_FOUND} EQUAL -1 AND ${CXX_FLAG_FAILED} EQUAL -1) + unset(CXX_FLAG_SUPPORTED CACHE) + message("Checking CXX compiler flag support for: " ${cxx_flag}) + check_cxx_compiler_flag("${cxx_flag}" CXX_FLAG_SUPPORTED) + if (CXX_FLAG_SUPPORTED) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${cxx_flag}" CACHE STRING "" + FORCE) + else() + set(AOM_FAILED_CXX_FLAGS "${AOM_FAILED_CXX_FLAGS} ${cxx_flag}" CACHE + STRING "" FORCE) + endif () + endif () +endfunction () + +# Convenience method for adding a flag to both the C and C++ compiler command +# lines. +function (add_compiler_flag_if_supported flag) + add_c_flag_if_supported(${flag}) + add_cxx_flag_if_supported(${flag}) +endfunction () + +# Checks C compiler for support of $c_flag and terminates generation when +# support is not present. +function (require_c_flag c_flag update_c_flags) + unset(C_FLAG_FOUND CACHE) + string(FIND "${CMAKE_C_FLAGS}" "${c_flag}" C_FLAG_FOUND) + + if (${C_FLAG_FOUND} EQUAL -1) + unset(HAVE_C_FLAG CACHE) + message("Checking C compiler flag support for: " ${c_flag}) + check_c_compiler_flag("${c_flag}" HAVE_C_FLAG) + if (NOT HAVE_C_FLAG) + message(FATAL_ERROR + "${PROJECT_NAME} requires support for C flag: ${c_flag}.") + endif () + if (update_c_flags) + set(CMAKE_C_FLAGS "${c_flag} ${CMAKE_C_FLAGS}" CACHE STRING "" FORCE) + endif () + endif () +endfunction () + +# Checks CXX compiler for support of $cxx_flag and terminates generation when +# support is not present. +function (require_cxx_flag cxx_flag update_cxx_flags) + unset(CXX_FLAG_FOUND CACHE) + string(FIND "${CMAKE_CXX_FLAGS}" "${cxx_flag}" CXX_FLAG_FOUND) + + if (${CXX_FLAG_FOUND} EQUAL -1) + unset(HAVE_CXX_FLAG CACHE) + message("Checking CXX compiler flag support for: " ${cxx_flag}) + check_cxx_compiler_flag("${cxx_flag}" HAVE_CXX_FLAG) + if (NOT HAVE_CXX_FLAG) + message(FATAL_ERROR + "${PROJECT_NAME} requires support for CXX flag: ${cxx_flag}.") + endif () + if (update_cxx_flags) + set(CMAKE_CXX_FLAGS "${cxx_flag} ${CMAKE_CXX_FLAGS}" CACHE STRING "" + FORCE) + endif () + endif () +endfunction () + +# Checks for support of $flag by both the C and CXX compilers. Terminates +# generation when support is not present in both compilers. +function (require_flag flag update_cmake_flags) + require_c_flag(${flag} ${update_cmake_flags}) + require_cxx_flag(${flag} ${update_cmake_flags}) +endfunction () + +# Checks only non-MSVC targets for support of $c_flag and terminates generation +# when support is not present. +function (require_c_flag_nomsvc c_flag update_c_flags) + if (NOT MSVC) + require_c_flag(${c_flag} ${update_c_flags}) + endif () +endfunction () + +# Checks only non-MSVC targets for support of $cxx_flag and terminates +# generation when support is not present. +function (require_cxx_flag_nomsvc cxx_flag update_cxx_flags) + if (NOT MSVC) + require_cxx_flag(${cxx_flag} ${update_cxx_flags}) + endif () +endfunction () + +# Checks only non-MSVC targets for support of $flag by both the C and CXX +# compilers. Terminates generation when support is not present in both +# compilers. +function (require_flag_nomsvc flag update_cmake_flags) + require_c_flag_nomsvc(${flag} ${update_cmake_flags}) + require_cxx_flag_nomsvc(${flag} ${update_cmake_flags}) +endfunction () + +# Adds $preproc_def to C compiler command line (as -D$preproc_def) if not +# already present. +function (add_c_preproc_definition preproc_def) + unset(PREPROC_DEF_FOUND CACHE) + string(FIND "${CMAKE_C_FLAGS}" "${preproc_def}" PREPROC_DEF_FOUND) + + if (${PREPROC_DEF_FOUND} EQUAL -1) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D${preproc_def}" CACHE STRING "" + FORCE) + endif () +endfunction () + +# Adds $preproc_def to CXX compiler command line (as -D$preproc_def) if not +# already present. +function (add_cxx_preproc_definition preproc_def) + unset(PREPROC_DEF_FOUND CACHE) + string(FIND "${CMAKE_CXX_FLAGS}" "${preproc_def}" PREPROC_DEF_FOUND) + + if (${PREPROC_DEF_FOUND} EQUAL -1) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D${preproc_def}" CACHE STRING "" + FORCE) + endif () +endfunction () + +# Adds $preproc_def to C and CXX compiler command line (as -D$preproc_def) if +# not already present. +function (add_preproc_definition preproc_def) + add_c_preproc_definition(${preproc_def}) + add_cxx_preproc_definition(${preproc_def}) +endfunction () + +# Adds $flag to assembler command line. +function (append_as_flag flag) + unset(AS_FLAG_FOUND CACHE) + string(FIND "${AOM_AS_FLAGS}" "${flag}" AS_FLAG_FOUND) + + if (${AS_FLAG_FOUND} EQUAL -1) + set(AOM_AS_FLAGS "${AOM_AS_FLAGS} ${flag}" CACHE STRING "" FORCE) + endif () +endfunction () + +# Adds $flag to the C compiler command line. +function (append_c_flag flag) + unset(C_FLAG_FOUND CACHE) + string(FIND "${CMAKE_C_FLAGS}" "${flag}" C_FLAG_FOUND) + + if (${C_FLAG_FOUND} EQUAL -1) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${flag}" CACHE STRING "" FORCE) + endif () +endfunction () + +# Adds $flag to the CXX compiler command line. +function (append_cxx_flag flag) + unset(CXX_FLAG_FOUND CACHE) + string(FIND "${CMAKE_CXX_FLAGS}" "${flag}" CXX_FLAG_FOUND) + + if (${CXX_FLAG_FOUND} EQUAL -1) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${flag}" CACHE STRING "" FORCE) + endif () +endfunction () + +# Adds $flag to the C and CXX compiler command lines. +function (append_compiler_flag flag) + append_c_flag(${flag}) + append_cxx_flag(${flag}) +endfunction () + +# Adds $flag to the executable linker command line. +function (append_exe_linker_flag flag) + unset(LINKER_FLAG_FOUND CACHE) + string(FIND "${CMAKE_EXE_LINKER_FLAGS}" "${flag}" LINKER_FLAG_FOUND) + + if (${LINKER_FLAG_FOUND} EQUAL -1) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${flag}" CACHE STRING + "" FORCE) + endif () +endfunction () diff --git a/third_party/aom/build/cmake/compiler_tests.cmake b/third_party/aom/build/cmake/compiler_tests.cmake new file mode 100644 index 0000000000..e763597a20 --- /dev/null +++ b/third_party/aom/build/cmake/compiler_tests.cmake @@ -0,0 +1,133 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +include(CheckCSourceCompiles) +include(CheckCXXSourceCompiles) + +# The basic main() function used in all compile tests. +set(AOM_C_MAIN "\nint main(void) { return 0; }") +set(AOM_CXX_MAIN "\nint main() { return 0; }") + +# Strings containing the names of passed and failed tests. +set(AOM_C_PASSED_TESTS) +set(AOM_C_FAILED_TESTS) +set(AOM_CXX_PASSED_TESTS) +set(AOM_CXX_FAILED_TESTS) + +function(aom_push_var var new_value) + set(SAVED_${var} ${var} PARENT_SCOPE) + set(${var} ${new_value} PARENT_SCOPE) +endfunction () + +function(aom_pop_var var) + set(var ${SAVED_${var}} PARENT_SCOPE) + unset(SAVED_${var} PARENT_SCOPE) +endfunction () + +# Confirms $test_source compiles and stores $test_name in one of +# $AOM_C_PASSED_TESTS or $AOM_C_FAILED_TESTS depending on out come. When the +# test passes $result_var is set to 1. When it fails $result_var is unset. +# The test is not run if the test name is found in either of the passed or +# failed test variables. +function(aom_check_c_compiles test_name test_source result_var) + unset(C_TEST_PASSED CACHE) + unset(C_TEST_FAILED CACHE) + string(FIND "${AOM_C_PASSED_TESTS}" "${test_name}" C_TEST_PASSED) + string(FIND "${AOM_C_FAILED_TESTS}" "${test_name}" C_TEST_FAILED) + if (${C_TEST_PASSED} EQUAL -1 AND ${C_TEST_FAILED} EQUAL -1) + unset(C_TEST_COMPILED CACHE) + message("Running C compiler test: ${test_name}") + check_c_source_compiles("${test_source} ${AOM_C_MAIN}" C_TEST_COMPILED) + set(${result_var} ${C_TEST_COMPILED} PARENT_SCOPE) + + if (C_TEST_COMPILED) + set(AOM_C_PASSED_TESTS "${AOM_C_PASSED_TESTS} ${test_name}" CACHE STRING + "" FORCE) + else () + set(AOM_C_FAILED_TESTS "${AOM_C_FAILED_TESTS} ${test_name}" CACHE STRING + "" FORCE) + message("C Compiler test ${test_name} failed.") + endif () + elseif (NOT ${C_TEST_PASSED} EQUAL -1) + set(${result_var} 1 PARENT_SCOPE) + else () # ${C_TEST_FAILED} NOT EQUAL -1 + unset(${result_var} PARENT_SCOPE) + endif () +endfunction () + +# Confirms $test_source compiles and stores $test_name in one of +# $AOM_CXX_PASSED_TESTS or $AOM_CXX_FAILED_TESTS depending on out come. When the +# test passes $result_var is set to 1. When it fails $result_var is unset. +# The test is not run if the test name is found in either of the passed or +# failed test variables. +function(aom_check_cxx_compiles test_name test_source result_var) + unset(CXX_TEST_PASSED CACHE) + unset(CXX_TEST_FAILED CACHE) + string(FIND "${AOM_CXX_PASSED_TESTS}" "${test_name}" CXX_TEST_PASSED) + string(FIND "${AOM_CXX_FAILED_TESTS}" "${test_name}" CXX_TEST_FAILED) + if (${CXX_TEST_PASSED} EQUAL -1 AND ${CXX_TEST_FAILED} EQUAL -1) + unset(CXX_TEST_COMPILED CACHE) + message("Running CXX compiler test: ${test_name}") + check_cxx_source_compiles("${test_source} ${AOM_CXX_MAIN}" + CXX_TEST_COMPILED) + set(${result_var} ${CXX_TEST_COMPILED} PARENT_SCOPE) + + if (CXX_TEST_COMPILED) + set(AOM_CXX_PASSED_TESTS "${AOM_CXX_PASSED_TESTS} ${test_name}" CACHE + STRING "" FORCE) + else () + set(AOM_CXX_FAILED_TESTS "${AOM_CXX_FAILED_TESTS} ${test_name}" CACHE + STRING "" FORCE) + message("CXX Compiler test ${test_name} failed.") + endif () + elseif (NOT ${CXX_TEST_PASSED} EQUAL -1) + set(${result_var} 1 PARENT_SCOPE) + else () # ${CXX_TEST_FAILED} NOT EQUAL -1 + unset(${result_var} PARENT_SCOPE) + endif () +endfunction () + +# Convenience function that confirms $test_source compiles as C and C++. +# $result_var is set to 1 when both tests are successful, and 0 when one or both +# tests fail. +# Note: This function is intended to be used to write to result variables that are +# expanded via configure_file(). $result_var is set to 1 or 0 to allow direct +# usage of the value in generated source files. +function(aom_check_source_compiles test_name test_source result_var) + unset(C_PASSED) + unset(CXX_PASSED) + aom_check_c_compiles(${test_name} ${test_source} C_PASSED) + aom_check_cxx_compiles(${test_name} ${test_source} CXX_PASSED) + if (C_PASSED AND CXX_PASSED) + set(${result_var} 1 PARENT_SCOPE) + else () + set(${result_var} 0 PARENT_SCOPE) + endif () +endfunction () + +# When inline support is detected for the current compiler the supported +# inlining keyword is written to $result in caller scope. +function (aom_get_inline result) + aom_check_source_compiles("inline_check_1" + "static inline void function(void) {}" + HAVE_INLINE_1) + if (HAVE_INLINE_1 EQUAL 1) + set(${result} "inline" PARENT_SCOPE) + return() + endif () + + # Check __inline. + aom_check_source_compiles("inline_check_2" + "static __inline void function(void) {}" + HAVE_INLINE_2) + if (HAVE_INLINE_2 EQUAL 1) + set(${result} "__inline" PARENT_SCOPE) + endif () +endfunction () diff --git a/third_party/aom/build/cmake/cpu.cmake b/third_party/aom/build/cmake/cpu.cmake new file mode 100644 index 0000000000..5d0b1a6e8a --- /dev/null +++ b/third_party/aom/build/cmake/cpu.cmake @@ -0,0 +1,72 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if ("${AOM_TARGET_CPU}" STREQUAL "arm64") + set(ARCH_ARM 1) + set(HAVE_NEON 1) + set(RTCD_ARCH_ARM "yes") + set(RTCD_HAVE_NEON "yes") +elseif ("${AOM_TARGET_CPU}" MATCHES "^armv7") + set(ARCH_ARM 1) + set(HAVE_NEON 1) + set(HAVE_NEON_ASM 1) + set(RTCD_ARCH_ARM "yes") + set(RTCD_HAVE_NEON "yes") + set(RTCD_HAVE_NEON_ASM "yes") +elseif ("${AOM_TARGET_CPU}" MATCHES "^mips") + set(ARCH_MIPS 1) + + if ("${AOM_TARGET_CPU}" STREQUAL "mips32") + set(HAVE_MIPS32 1) + elseif ("${AOM_TARGET_CPU}" STREQUAL "mips64") + set(HAVE_MIPS64 1) + endif () + + set(RTCD_ARCH_MIPS "yes") + + if (HAVE_DSPR2) + set(RTCD_HAVE_DSPR2 "yes") + endif () + + if (HAVE_MSA) + set(RTCD_HAVE_MSA "yes") + endif () +elseif ("${AOM_TARGET_CPU}" MATCHES "^x86") + if ("${AOM_TARGET_CPU}" STREQUAL "x86") + set(ARCH_X86 1) + set(RTCD_ARCH_X86 "yes") + elseif ("${AOM_TARGET_CPU}" STREQUAL "x86_64") + set(ARCH_X86_64 1) + set(RTCD_ARCH_X86_64 "yes") + endif () + + set(HAVE_MMX 1) + set(HAVE_SSE 1) + set(HAVE_SSE2 1) + set(HAVE_SSE3 1) + set(HAVE_SSSE3 1) + set(HAVE_SSE4_1 1) + set(HAVE_AVX 1) + set(HAVE_AVX2 1) + set(RTCD_HAVE_MMX "yes") + set(RTCD_HAVE_SSE "yes") + set(RTCD_HAVE_SSE2 "yes") + set(RTCD_HAVE_SSE3 "yes") + set(RTCD_HAVE_SSSE3 "yes") + set(RTCD_HAVE_SSE4_1 "yes") + set(RTCD_HAVE_AVX "yes") + set(RTCD_HAVE_AVX2 "yes") +endif () + +foreach (config_var ${AOM_CONFIG_VARS}) + if (${${config_var}}) + set(RTCD_${config_var} yes) + endif () +endforeach () diff --git a/third_party/aom/build/cmake/generate_aom_config_templates.cmake b/third_party/aom/build/cmake/generate_aom_config_templates.cmake new file mode 100644 index 0000000000..aea4253bbe --- /dev/null +++ b/third_party/aom/build/cmake/generate_aom_config_templates.cmake @@ -0,0 +1,84 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +cmake_minimum_required(VERSION 3.5) + +string(TIMESTAMP year "%Y") +set(asm_file_header_block +"\; +\; Copyright (c) ${year}, Alliance for Open Media. All rights reserved +\; +\; This source code is subject to the terms of the BSD 2 Clause License and +\; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +\; was not distributed with this source code in the LICENSE file, you can +\; obtain it at www.aomedia.org/license/software. If the Alliance for Open +\; Media Patent License 1.0 was not distributed with this source code in the +\; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +\; +") +set(h_file_header_block +"/* + * Copyright (c) ${year}, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +\#ifndef AOM_CONFIG_H_ +\#define AOM_CONFIG_H_ +") + +# Terminates cmake execution when $var_name is an empty string, or the variable +# name it contains does not expand to an existing directory. +function (check_directory_var var_name) + if ("${var_name}" STREQUAL "") + message(FATAL_ERROR "The CMake variable ${var_name} must be defined.") + endif () + + if (NOT EXISTS "${${var_name}}") + message(FATAL_ERROR "${${var_name}} (${var_name}) missing.") + endif () +endfunction () + +check_directory_var(AOM_CONFIG_DIR) +check_directory_var(AOM_ROOT) + +set(AOM_DEFAULTS "${AOM_ROOT}/build/cmake/aom_config_defaults.cmake") +if (NOT EXISTS "${AOM_DEFAULTS}") + message(FATAL_ERROR + "Configuration default values file (${AOM_DEFAULTS}) missing.") +endif () + +include("${AOM_ROOT}/build/cmake/aom_config_defaults.cmake") +get_cmake_property(cmake_cache_vars CACHE_VARIABLES) + +set(aom_config_h_template "${AOM_CONFIG_DIR}/aom_config.h.cmake") +file(WRITE "${aom_config_h_template}" ${h_file_header_block}) +foreach(cache_var ${cmake_cache_vars}) + if (NOT "${cache_var}" MATCHES "AOM_CONFIG_DIR\|AOM_ROOT\|^CMAKE_") + file(APPEND + "${aom_config_h_template}" "\#define ${cache_var} \${${cache_var}}\n") + endif () +endforeach() +file(APPEND "${aom_config_h_template}" "\#endif /* AOM_CONFIG_H_ */") + +set(aom_asm_config_template "${AOM_CONFIG_DIR}/aom_config.asm.cmake") +file(WRITE "${aom_asm_config_template}" ${asm_file_header_block}) +foreach(cache_var ${cmake_cache_vars}) + if (NOT "${cache_var}" MATCHES + "AOM_CONFIG_DIR\|AOM_ROOT\|^CMAKE_\|INLINE\|RESTRICT") + file(APPEND "${aom_asm_config_template}" + "${cache_var} equ \${${cache_var}}\n") + endif () +endforeach () + diff --git a/third_party/aom/build/cmake/msvc_runtime.cmake b/third_party/aom/build/cmake/msvc_runtime.cmake new file mode 100644 index 0000000000..0327217b4d --- /dev/null +++ b/third_party/aom/build/cmake/msvc_runtime.cmake @@ -0,0 +1,26 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +if (MSVC) + # CMake defaults to producing code linked to the DLL MSVC runtime. That will + # not work with googletest, and isn't what we want anyway. + if (NOT "${MSVC_RUNTIME}" STREQUAL "dll") + foreach (flag_var + CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE + CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO + CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE + CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO) + if (${flag_var} MATCHES "/MD") + string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}") + endif (${flag_var} MATCHES "/MD") + endforeach (flag_var) + endif () +endif () diff --git a/third_party/aom/build/cmake/rtcd_config.cmake b/third_party/aom/build/cmake/rtcd_config.cmake new file mode 100644 index 0000000000..cdea3452a7 --- /dev/null +++ b/third_party/aom/build/cmake/rtcd_config.cmake @@ -0,0 +1,137 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +ARCH_ARM=${RTCD_ARCH_ARM} +ARCH_MIPS=${RTCD_ARCH_MIPS} +ARCH_X86=${RTCD_ARCH_X86} +ARCH_X86_64=${RTCD_ARCH_X86_64} +HAVE_NEON=${RTCD_HAVE_NEON} +HAVE_NEON_ASM=${RTCD_HAVE_NEON_ASM} +HAVE_MIPS32=${RTCD_HAVE_MIPS32} +HAVE_DSPR2=${RTCD_HAVE_DSPR2} +HAVE_MSA=${RTCD_HAVE_MSA} +HAVE_MIPS64=${RTCD_HAVE_MIPS64} +HAVE_MMX=${RTCD_HAVE_MMX} +HAVE_SSE=${RTCD_HAVE_SSE} +HAVE_SSE2=${RTCD_HAVE_SSE2} +HAVE_SSE3=${RTCD_HAVE_SSE3} +HAVE_SSSE3=${RTCD_HAVE_SSSE3} +HAVE_SSE4_1=${RTCD_HAVE_SSE4_1} +HAVE_AVX=${RTCD_HAVE_AVX} +HAVE_AVX2=${RTCD_HAVE_AVX2} +CONFIG_ACCOUNTING=${RTCD_CONFIG_ACCOUNTING} +CONFIG_INSPECTION=${RTCD_CONFIG_INSPECTION} +CONFIG_ADAPT_SCAN=${RTCD_CONFIG_ADAPT_SCAN} +CONFIG_ALT_INTRA=${RTCD_CONFIG_ALT_INTRA} +CONFIG_ANS=${RTCD_CONFIG_ANS} +CONFIG_HIGHBITDEPTH=${RTCD_CONFIG_HIGHBITDEPTH} +CONFIG_AOM_QM=${RTCD_CONFIG_AOM_QM} +CONFIG_AV1=${RTCD_CONFIG_AV1} +CONFIG_AV1_DECODER=${RTCD_CONFIG_AV1_DECODER} +CONFIG_AV1_ENCODER=${RTCD_CONFIG_AV1_ENCODER} +CONFIG_BIG_ENDIAN=${RTCD_CONFIG_BIG_ENDIAN} +CONFIG_BITSTREAM_DEBUG=${RTCD_CONFIG_BITSTREAM_DEBUG} +CONFIG_CB4X4=${RTCD_CONFIG_CB4X4} +CONFIG_CDEF=${RTCD_CONFIG_CDEF} +CONFIG_CHROMA_2X2=${RTCD_CONFIG_CHROMA_2X2} +CONFIG_CODEC_SRCS=${RTCD_CONFIG_CODEC_SRCS} +CONFIG_COEFFICIENT_RANGE_CHECKING=${RTCD_CONFIG_COEFFICIENT_RANGE_CHECKING} +CONFIG_COEF_INTERLEAVE=${RTCD_CONFIG_COEF_INTERLEAVE} +CONFIG_COMPOUND_SEGMENT=${RTCD_CONFIG_COMPOUND_SEGMENT} +CONFIG_CONVOLVE_ROUND=${RTCD_CONFIG_CONVOLVE_ROUND} +CONFIG_DAALA_DIST=${RTCD_CONFIG_DAALA_DIST} +CONFIG_DAALA_EC=${RTCD_CONFIG_DAALA_EC} +CONFIG_DEBUG=${RTCD_CONFIG_DEBUG} +CONFIG_DEBUG_LIBS=${RTCD_CONFIG_DEBUG_LIBS} +CONFIG_DECODERS=${RTCD_CONFIG_DECODERS} +CONFIG_DECODE_PERF_TESTS=${RTCD_CONFIG_DECODE_PERF_TESTS} +CONFIG_DELTA_Q=${RTCD_CONFIG_DELTA_Q} +CONFIG_DEPENDENCY_TRACKING=${RTCD_CONFIG_DEPENDENCY_TRACKING} +CONFIG_DEPENDENT_HORZTILES=${RTCD_CONFIG_DEPENDENT_HORZTILES} +CONFIG_DUAL_FILTER=${RTCD_CONFIG_DUAL_FILTER} +CONFIG_EC_ADAPT=${RTCD_CONFIG_EC_ADAPT} +CONFIG_EC_MULTISYMBOL=${RTCD_CONFIG_EC_MULTISYMBOL} +CONFIG_ENCODERS=${RTCD_CONFIG_ENCODERS} +CONFIG_ENCODE_PERF_TESTS=${RTCD_CONFIG_ENCODE_PERF_TESTS} +CONFIG_ENTROPY_STATS=${RTCD_CONFIG_ENTROPY_STATS} +CONFIG_ERROR_CONCEALMENT=${RTCD_CONFIG_ERROR_CONCEALMENT} +CONFIG_EXPERIMENTAL=${RTCD_CONFIG_EXPERIMENTAL} +CONFIG_EXTERNAL_BUILD=${RTCD_CONFIG_EXTERNAL_BUILD} +CONFIG_EXT_INTER=${RTCD_CONFIG_EXT_INTER} +CONFIG_EXT_INTRA=${RTCD_CONFIG_EXT_INTRA} +CONFIG_EXT_PARTITION=${RTCD_CONFIG_EXT_PARTITION} +CONFIG_EXT_PARTITION_TYPES=${RTCD_CONFIG_EXT_PARTITION_TYPES} +CONFIG_EXT_REFS=${RTCD_CONFIG_EXT_REFS} +CONFIG_EXT_TILE=${RTCD_CONFIG_EXT_TILE} +CONFIG_EXT_TX=${RTCD_CONFIG_EXT_TX} +CONFIG_FILTER_7BIT=${RTCD_CONFIG_FILTER_7BIT} +CONFIG_FILTER_INTRA=${RTCD_CONFIG_FILTER_INTRA} +CONFIG_FP_MB_STATS=${RTCD_CONFIG_FP_MB_STATS} +CONFIG_FRAME_SIZE=${RTCD_CONFIG_FRAME_SIZE} +CONFIG_GCC=${RTCD_CONFIG_GCC} +CONFIG_GCOV=${RTCD_CONFIG_GCOV} +CONFIG_GLOBAL_MOTION=${RTCD_CONFIG_GLOBAL_MOTION} +CONFIG_GPROF=${RTCD_CONFIG_GPROF} +CONFIG_INSTALL_BINS=${RTCD_CONFIG_INSTALL_BINS} +CONFIG_INSTALL_DOCS=${RTCD_CONFIG_INSTALL_DOCS} +CONFIG_INSTALL_LIBS=${RTCD_CONFIG_INSTALL_LIBS} +CONFIG_INSTALL_SRCS=${RTCD_CONFIG_INSTALL_SRCS} +CONFIG_INTERNAL_STATS=${RTCD_CONFIG_INTERNAL_STATS} +CONFIG_INTRA_INTERP=${RTCD_CONFIG_INTRA_INTERP} +CONFIG_LIBYUV=${RTCD_CONFIG_LIBYUV} +CONFIG_LOOPFILTERING_ACROSS_TILES=${RTCD_CONFIG_LOOPFILTERING_ACROSS_TILES} +CONFIG_LOOP_RESTORATION=${RTCD_CONFIG_LOOP_RESTORATION} +CONFIG_LOWBITDEPTH=${RTCD_CONFIG_LOWBITDEPTH} +CONFIG_LV_MAP=${RTCD_CONFIG_LV_MAP} +CONFIG_MASKED_TX=${RTCD_CONFIG_MASKED_TX} +CONFIG_MOTION_VAR=${RTCD_CONFIG_MOTION_VAR} +CONFIG_MSVS=${RTCD_CONFIG_MSVS} +CONFIG_MULTITHREAD=${RTCD_CONFIG_MULTITHREAD} +CONFIG_MV_COMPRESS=${RTCD_CONFIG_MV_COMPRESS} +CONFIG_NCOBMC=${RTCD_CONFIG_NCOBMC} +CONFIG_NEW_QUANT=${RTCD_CONFIG_NEW_QUANT} +CONFIG_NEW_TOKENSET=${RTCD_CONFIG_NEW_TOKENSET} +CONFIG_ONTHEFLY_BITPACKING=${RTCD_CONFIG_ONTHEFLY_BITPACKING} +CONFIG_OS_SUPPORT=${RTCD_CONFIG_OS_SUPPORT} +CONFIG_PALETTE=${RTCD_CONFIG_PALETTE} +CONFIG_PALETTE_THROUGHPUT=${RTCD_CONFIG_PALETTE_THROUGHPUT} +CONFIG_PARALLEL_DEBLOCKING=${RTCD_CONFIG_PARALLEL_DEBLOCKING} +CONFIG_PIC=${RTCD_CONFIG_PIC} +CONFIG_POSTPROC=${RTCD_CONFIG_POSTPROC} +CONFIG_POSTPROC_VISUALIZER=${RTCD_CONFIG_POSTPROC_VISUALIZER} +CONFIG_PVQ=${RTCD_CONFIG_PVQ} +CONFIG_RAWBITS=${RTCD_CONFIG_RAWBITS} +CONFIG_RD_DEBUG=${RTCD_CONFIG_RD_DEBUG} +CONFIG_REALTIME_ONLY=${RTCD_CONFIG_REALTIME_ONLY} +CONFIG_RECT_TX=${RTCD_CONFIG_RECT_TX} +CONFIG_REFERENCE_BUFFER=${RTCD_CONFIG_REFERENCE_BUFFER} +CONFIG_REF_ADAPT=${RTCD_CONFIG_REF_ADAPT} +CONFIG_REF_MV=${RTCD_CONFIG_REF_MV} +CONFIG_RUNTIME_CPU_DETECT=${RTCD_CONFIG_RUNTIME_CPU_DETECT} +CONFIG_RVCT=${RTCD_CONFIG_RVCT} +CONFIG_SHARED=${RTCD_CONFIG_SHARED} +CONFIG_SIZE_LIMIT=${RTCD_CONFIG_SIZE_LIMIT} +CONFIG_SMALL=${RTCD_CONFIG_SMALL} +CONFIG_SPATIAL_RESAMPLING=${RTCD_CONFIG_SPATIAL_RESAMPLING} +CONFIG_STATIC=${RTCD_CONFIG_STATIC} +CONFIG_STATIC_MSVCRT=${RTCD_CONFIG_STATIC_MSVCRT} +CONFIG_SUB8X8_MC=${RTCD_CONFIG_SUB8X8_MC} +CONFIG_SUPERTX=${RTCD_CONFIG_SUPERTX} +CONFIG_TEMPMV_SIGNALING=${RTCD_CONFIG_TEMPMV_SIGNALING} +CONFIG_TILE_GROUPS=${RTCD_CONFIG_TILE_GROUPS} +CONFIG_TPL_MV=${RTCD_CONFIG_TPL_MV} +CONFIG_TRIPRED=${RTCD_CONFIG_TRIPRED} +CONFIG_TX64X64=${RTCD_CONFIG_TX64X64} +CONFIG_UNIT_TESTS=${RTCD_CONFIG_UNIT_TESTS} +CONFIG_UNPOISON_PARTITION_CTX=${RTCD_CONFIG_UNPOISON_PARTITION_CTX} +CONFIG_VAR_TX=${RTCD_CONFIG_VAR_TX} +CONFIG_WARPED_MOTION=${RTCD_CONFIG_WARPED_MOTION} +CONFIG_WEBM_IO=${RTCD_CONFIG_WEBM_IO} +CONFIG_XIPHRC=${RTCD_CONFIG_XIPHRC} diff --git a/third_party/aom/build/cmake/toolchains/arm-ios-common.cmake b/third_party/aom/build/cmake/toolchains/arm-ios-common.cmake new file mode 100644 index 0000000000..8317ae2725 --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/arm-ios-common.cmake @@ -0,0 +1,31 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_BUILD_CMAKE_TOOLCHAINS_ARM_IOS_COMMON_CMAKE_) +set(AOM_BUILD_CMAKE_ARM_IOS_COMMON_CMAKE_ 1) + +set(CMAKE_SYSTEM_NAME "Darwin") +set(CMAKE_OSX_SYSROOT iphoneos) +set(CMAKE_C_COMPILER clang) +set(CMAKE_C_COMPILER_ARG1 "-arch ${CMAKE_SYSTEM_PROCESSOR}") +set(CMAKE_CXX_COMPILER clang++) +set(CMAKE_CXX_COMPILER_ARG1 "-arch ${CMAKE_SYSTEM_PROCESSOR}") + +# Assembler sources must be converted for ARM iOS targets. +set(AOM_ADS2GAS_REQUIRED 1) +set(AOM_ADS2GAS "${CMAKE_CURRENT_SOURCE_DIR}/build/make/ads2gas_apple.pl") +set(AOM_GAS_EXT "S") + +# No runtime cpu detect for arm*-ios targets. +set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE BOOL "") + +# TODO(tomfinegan): Handle bit code embedding. + +endif () # AOM_BUILD_CMAKE_TOOLCHAINS_ARM_IOS_COMMON_CMAKE_ diff --git a/third_party/aom/build/cmake/toolchains/arm64-ios.cmake b/third_party/aom/build/cmake/toolchains/arm64-ios.cmake new file mode 100644 index 0000000000..434809db91 --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/arm64-ios.cmake @@ -0,0 +1,24 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_BUILD_CMAKE_TOOLCHAINS_ARM64_IOS_CMAKE_) +set(AOM_BUILD_CMAKE_TOOLCHAINS_ARM64_IOS_CMAKE_ 1) + +if (XCODE) + # TODO(tomfinegan): Handle arm builds in Xcode. + message(FATAL_ERROR "This toolchain does not support Xcode.") +endif () + +set(CMAKE_SYSTEM_PROCESSOR "arm64") +set(CMAKE_OSX_ARCHITECTURES "arm64") + +include("${CMAKE_CURRENT_LIST_DIR}/arm-ios-common.cmake") + +endif () # AOM_BUILD_CMAKE_TOOLCHAINS_ARM64_IOS_CMAKE_ diff --git a/third_party/aom/build/cmake/toolchains/arm64-linux-gcc.cmake b/third_party/aom/build/cmake/toolchains/arm64-linux-gcc.cmake new file mode 100644 index 0000000000..b8efe6be66 --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/arm64-linux-gcc.cmake @@ -0,0 +1,35 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_BUILD_CMAKE_TOOLCHAINS_ARM64_LINUX_GCC_CMAKE_) +set(AOM_BUILD_CMAKE_TOOLCHAINS_ARM64_LINUX_GCC_CMAKE_ 1) + +set(CMAKE_SYSTEM_NAME "Linux") + +if ("${CROSS}" STREQUAL "") + # Default the cross compiler prefix to something known to work. + set(CROSS aarch64-linux-gnu-) +endif () + +set(CMAKE_C_COMPILER ${CROSS}gcc) +set(CMAKE_CXX_COMPILER ${CROSS}g++) +set(AS_EXECUTABLE ${CROSS}as) +set(CMAKE_C_COMPILER_ARG1 "-march=armv8-a") +set(CMAKE_CXX_COMPILER_ARG1 "-march=armv8-a") +set(AOM_AS_FLAGS "-march=armv8-a") +set(CMAKE_SYSTEM_PROCESSOR "arm64") + +# No intrinsics flag required for arm64-linux-gcc. +set(AOM_NEON_INTRIN_FLAG "") + +# No runtime cpu detect for arm64-linux-gcc. +set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE BOOL "") + +endif () # AOM_BUILD_CMAKE_TOOLCHAINS_ARM64_LINUX_GCC_CMAKE_ diff --git a/third_party/aom/build/cmake/toolchains/armv7-ios.cmake b/third_party/aom/build/cmake/toolchains/armv7-ios.cmake new file mode 100644 index 0000000000..bcd37a06d7 --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/armv7-ios.cmake @@ -0,0 +1,34 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_BUILD_CMAKE_TOOLCHAINS_ARMV7_IOS_CMAKE_) +set(AOM_BUILD_CMAKE_TOOLCHAINS_ARMV7_IOS_CMAKE_ 1) + +if (XCODE) + # TODO(tomfinegan): Handle arm builds in Xcode. + message(FATAL_ERROR "This toolchain does not support Xcode.") +endif () + +set(CMAKE_SYSTEM_PROCESSOR "armv7") +set(CMAKE_OSX_ARCHITECTURES "armv7") + +include("${CMAKE_CURRENT_LIST_DIR}/arm-ios-common.cmake") + +# No intrinsics flag required for armv7s-ios. +set(AOM_NEON_INTRIN_FLAG "") + +# No runtime cpu detect for armv7s-ios. +set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE BOOL "") + +# RTCD generation requires --disable-media for armv7s-ios. +set(AOM_RTCD_FLAGS ${AOM_RTCD_FLAGS} --disable-media) +string(STRIP AOM_RTCD_FLAGS ${AOM_RTCD_FLAGS}) + +endif () # AOM_BUILD_CMAKE_TOOLCHAINS_ARMV7_IOS_CMAKE_ diff --git a/third_party/aom/build/cmake/toolchains/armv7-linux-gcc.cmake b/third_party/aom/build/cmake/toolchains/armv7-linux-gcc.cmake new file mode 100644 index 0000000000..eedfda4641 --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/armv7-linux-gcc.cmake @@ -0,0 +1,48 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_BUILD_CMAKE_TOOLCHAINS_ARMV7_LINUX_GCC_CMAKE_) +set(AOM_BUILD_CMAKE_TOOLCHAINS_ARMV7_LINUX_GCC_CMAKE_ 1) + +set(CMAKE_SYSTEM_NAME "Linux") + +if ("${CROSS}" STREQUAL "") + # Default the cross compiler prefix to something known to work. + set(CROSS arm-linux-gnueabihf-) +endif () + +if (NOT ${CROSS} MATCHES hf-$) + set(AOM_EXTRA_TOOLCHAIN_FLAGS "-mfloat-abi=softfp") +endif () + +set(CMAKE_C_COMPILER ${CROSS}gcc) +set(CMAKE_CXX_COMPILER ${CROSS}g++) +set(AS_EXECUTABLE ${CROSS}as) +set(CMAKE_C_COMPILER_ARG1 + "-march=armv7-a -mfpu=neon ${AOM_EXTRA_TOOLCHAIN_FLAGS}") +set(CMAKE_CXX_COMPILER_ARG1 + "-march=armv7-a -mfpu=neon ${AOM_EXTRA_TOOLCHAIN_FLAGS}") +set(AOM_AS_FLAGS + --defsym ARCHITECTURE=7 -march=armv7-a -mfpu=neon + ${AOM_EXTRA_TOOLCHAIN_FLAGS}) +set(CMAKE_SYSTEM_PROCESSOR "armv7") + +# No intrinsics flag required for armv7-linux-gcc. +set(AOM_NEON_INTRIN_FLAG "") + +# Assembler sources must be converted for armv7-linux-gcc targets. +set(AOM_ADS2GAS_REQUIRED 1) +set(AOM_ADS2GAS "${CMAKE_CURRENT_SOURCE_DIR}/build/make/ads2gas.pl") +set(AOM_GAS_EXT "S") + +# No runtime cpu detect for armv7-linux-gcc. +set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE BOOL "") + +endif () # AOM_BUILD_CMAKE_TOOLCHAINS_ARMV7_LINUX_GCC_CMAKE_ diff --git a/third_party/aom/build/cmake/toolchains/armv7s-ios.cmake b/third_party/aom/build/cmake/toolchains/armv7s-ios.cmake new file mode 100644 index 0000000000..08a0a37ee7 --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/armv7s-ios.cmake @@ -0,0 +1,34 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_BUILD_CMAKE_TOOLCHAINS_ARMV7S_IOS_CMAKE_) +set(AOM_BUILD_CMAKE_TOOLCHAINS_ARMV7S_IOS_CMAKE_ 1) + +if (XCODE) + # TODO(tomfinegan): Handle arm builds in Xcode. + message(FATAL_ERROR "This toolchain does not support Xcode.") +endif () + +set(CMAKE_SYSTEM_PROCESSOR "armv7s") +set(CMAKE_OSX_ARCHITECTURES "armv7s") + +include("${CMAKE_CURRENT_LIST_DIR}/arm-ios-common.cmake") + +# No intrinsics flag required for armv7s-ios. +set(AOM_NEON_INTRIN_FLAG "") + +# No runtime cpu detect for armv7s-ios. +set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE BOOL "") + +# RTCD generation requires --disable-media for armv7s-ios. +set(AOM_RTCD_FLAGS ${AOM_RTCD_FLAGS} --disable-media) +string(STRIP AOM_RTCD_FLAGS ${AOM_RTCD_FLAGS}) + +endif () # AOM_BUILD_CMAKE_TOOLCHAINS_ARMV7S_IOS_CMAKE_ diff --git a/third_party/aom/build/cmake/toolchains/ios-simulator-common.cmake b/third_party/aom/build/cmake/toolchains/ios-simulator-common.cmake new file mode 100644 index 0000000000..7a28e329c6 --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/ios-simulator-common.cmake @@ -0,0 +1,23 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_BUILD_CMAKE_TOOLCHAINS_IOS_SIMULATOR_COMMON_CMAKE_) +set(AOM_BUILD_CMAKE_IOS_SIMULATOR_COMMON_CMAKE_ 1) + +set(CMAKE_SYSTEM_NAME "Darwin") +set(CMAKE_OSX_SYSROOT iphonesimulator) +set(CMAKE_C_COMPILER clang) +set(CMAKE_C_COMPILER_ARG1 "-arch ${CMAKE_SYSTEM_PROCESSOR}") +set(CMAKE_CXX_COMPILER clang++) +set(CMAKE_CXX_COMPILER_ARG1 "-arch ${CMAKE_SYSTEM_PROCESSOR}") + +# TODO(tomfinegan): Handle bit code embedding. + +endif () # AOM_BUILD_CMAKE_TOOLCHAINS_IOS_SIMULATOR_COMMON_CMAKE_ diff --git a/third_party/aom/build/cmake/toolchains/mips32-linux-gcc.cmake b/third_party/aom/build/cmake/toolchains/mips32-linux-gcc.cmake new file mode 100644 index 0000000000..a55c41115f --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/mips32-linux-gcc.cmake @@ -0,0 +1,71 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_BUILD_CMAKE_TOOLCHAINS_MIPS32_LINUX_GCC_CMAKE_) +set(AOM_BUILD_CMAKE_TOOLCHAINS_MIPS32_LINUX_GCC_CMAKE_ 1) + +set(CMAKE_SYSTEM_NAME "Linux") + +if (ENABLE_DSPR2 AND ENABLE_MSA) + message(FATAL_ERROR "ENABLE_DSPR2 and ENABLE_MSA cannot be combined.") +endif () + +if (ENABLE_DSPR2) + set(HAVE_DSPR2 1 CACHE BOOL "" FORCE) + + if ("${CROSS}" STREQUAL "") + # Default the cross compiler prefix to something known to work. + set(CROSS mips-linux-gnu-) + endif () + + set(MIPS_CFLAGS "-mdspr2") + set(MIPS_CXXFLAGS "-mdspr2") +elseif (ENABLE_MSA) + set(HAVE_MSA 1 CACHE BOOL "" FORCE) + + if ("${CROSS}" STREQUAL "") + # Default the cross compiler prefix to something known to work. + set(CROSS mips-mti-linux-gnu-) + endif () + + set(MIPS_CFLAGS "-mmsa") + set(MIPS_CXXFLAGS "-mmsa") +endif () + +if ("${CROSS}" STREQUAL "") + # TODO(tomfinegan): Make it possible to turn this off. The $CROSS prefix + # won't be desired on a mips host. + # Default cross compiler prefix to something that might work for an + # unoptimized build. + set(CROSS mips-linux-gnu-) +endif () + +if ("${MIPS_CPU}" STREQUAL "") + set(MIPS_CFLAGS "${MIPS_CFLAGS} -mips32r2") + set(MIPS_CXXFLAGS "${MIPS_CXXFLAGS} -mips32r2") +elseif ("${MIPS_CPU}" STREQUAL "p5600") + set(P56_FLAGS + "-mips32r5 -mload-store-pairs -msched-weight -mhard-float -mfp64") + set(MIPS_CFLAGS "${MIPS_CFLAGS} ${P56_FLAGS}") + set(MIPS_CXXFLAGS "${MIPS_CXXFLAGS} ${P56_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "-mfp64 ${CMAKE_EXE_LINKER_FLAGS}") +endif () + +set(CMAKE_C_COMPILER ${CROSS}gcc) +set(CMAKE_CXX_COMPILER ${CROSS}g++) +set(AS_EXECUTABLE ${CROSS}as) +set(CMAKE_C_COMPILER_ARG1 "-EL ${MIPS_CFLAGS}") +set(CMAKE_CXX_COMPILER_ARG1 "-EL ${MIPS_CXXFLAGS}") +set(CMAKE_SYSTEM_PROCESSOR "mips32") + +# No runtime cpu detect for mips32-linux-gcc. +set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE BOOL "") + +endif () # AOM_BUILD_CMAKE_TOOLCHAINS_MIPS32_LINUX_GCC_CMAKE_ diff --git a/third_party/aom/build/cmake/toolchains/mips64-linux-gcc.cmake b/third_party/aom/build/cmake/toolchains/mips64-linux-gcc.cmake new file mode 100644 index 0000000000..28b1582cc3 --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/mips64-linux-gcc.cmake @@ -0,0 +1,48 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_BUILD_CMAKE_TOOLCHAINS_MIPS64_LINUX_GCC_CMAKE_) +set(AOM_BUILD_CMAKE_TOOLCHAINS_MIPS64_LINUX_GCC_CMAKE_ 1) + +set(CMAKE_SYSTEM_NAME "Linux") + +if ("${CROSS}" STREQUAL "") + # TODO(tomfinegan): Make it possible to turn this off. The $CROSS prefix + # won't be desired on a mips host. + # Default the cross compiler prefix to something known to work. + set(CROSS mips-img-linux-gnu-) +endif () + +if (ENABLE_MSA) + set(HAVE_MSA 1 CACHE BOOL "" FORCE) + set(MIPS_CFLAGS "-mmsa") + set(MIPS_CXXFLAGS "-mmsa") +endif () + +if ("${MIPS_CPU}" STREQUAL "i6400" OR "${MIPS_CPU}" STREQUAL "p6600") + set(MIPS_CPU_FLAGS "-mips64r6 -mabi=64 -mload-store-pairs -msched-weight") + set(MIPS_CPU_FLAGS "${MIPS_CPU_FLAGS} -mhard-float -mfp64") + set(MIPS_CFLAGS "${MIPS_CFLAGS} ${MIPS_CPU_FLAGS}") + set(MIPS_CXXFLAGS "${MIPS_CXXFLAGS} ${MIPS_CPU_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS + "-mips64r6 -mabi64 -mfp64 ${CMAKE_EXE_LINKER_FLAGS}") +endif () + +set(CMAKE_C_COMPILER ${CROSS}gcc) +set(CMAKE_CXX_COMPILER ${CROSS}g++) +set(AS_EXECUTABLE ${CROSS}as) +set(CMAKE_C_COMPILER_ARG1 "-EL ${MIPS_CFLAGS}") +set(CMAKE_CXX_COMPILER_ARG1 "-EL ${MIPS_CXXFLAGS}") +set(CMAKE_SYSTEM_PROCESSOR "mips64") + +# No runtime cpu detect for mips64-linux-gcc. +set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE BOOL "") + +endif () # AOM_BUILD_CMAKE_TOOLCHAINS_MIPS64_LINUX_GCC_CMAKE_ diff --git a/third_party/aom/build/cmake/toolchains/x86-ios-simulator.cmake b/third_party/aom/build/cmake/toolchains/x86-ios-simulator.cmake new file mode 100644 index 0000000000..4e4ebc0348 --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/x86-ios-simulator.cmake @@ -0,0 +1,27 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_BUILD_CMAKE_TOOLCHAINS_X86_IOS_SIMULATOR_CMAKE_) +set(AOM_BUILD_CMAKE_TOOLCHAINS_X86_IOS_SIMULATOR_CMAKE_ 1) + +if (XCODE) + # TODO(tomfinegan): Handle ios sim builds in Xcode. + message(FATAL_ERROR "This toolchain does not support Xcode.") +endif () + +set(CMAKE_SYSTEM_PROCESSOR "i386") +set(CMAKE_OSX_ARCHITECTURES "i386") + +# Avoid noisy PIC/PIE warnings. +set(CONFIG_PIC 1 CACHE BOOL "") + +include("${CMAKE_CURRENT_LIST_DIR}/ios-simulator-common.cmake") + +endif () # AOM_BUILD_CMAKE_TOOLCHAINS_X86_IOS_SIMULATOR_CMAKE_ diff --git a/third_party/aom/build/cmake/toolchains/x86-linux.cmake b/third_party/aom/build/cmake/toolchains/x86-linux.cmake new file mode 100644 index 0000000000..077c8f325b --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/x86-linux.cmake @@ -0,0 +1,14 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +set(CMAKE_SYSTEM_PROCESSOR "x86") +set(CMAKE_SYSTEM_NAME "Linux") +set(CMAKE_C_COMPILER_ARG1 "-m32") +set(CMAKE_CXX_COMPILER_ARG1 "-m32") diff --git a/third_party/aom/build/cmake/toolchains/x86-macos.cmake b/third_party/aom/build/cmake/toolchains/x86-macos.cmake new file mode 100644 index 0000000000..22d0171a71 --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/x86-macos.cmake @@ -0,0 +1,18 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +set(CMAKE_SYSTEM_PROCESSOR "x86") +set(CMAKE_SYSTEM_NAME "Darwin") +set(CMAKE_OSX_ARCHITECTURES "i386") +set(CMAKE_C_COMPILER_ARG1 "-arch i386") +set(CMAKE_CXX_COMPILER_ARG1 "-arch i386") + +# Apple tools always complain in 32 bit mode without PIC. +set(CONFIG_PIC 1 CACHE STRING "") diff --git a/third_party/aom/build/cmake/toolchains/x86_64-ios-simulator.cmake b/third_party/aom/build/cmake/toolchains/x86_64-ios-simulator.cmake new file mode 100644 index 0000000000..884540a9d2 --- /dev/null +++ b/third_party/aom/build/cmake/toolchains/x86_64-ios-simulator.cmake @@ -0,0 +1,24 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_BUILD_CMAKE_TOOLCHAINS_X86_64_IOS_SIMULATOR_CMAKE_) +set(AOM_BUILD_CMAKE_TOOLCHAINS_X86_64_IOS_SIMULATOR_CMAKE_ 1) + +if (XCODE) + # TODO(tomfinegan): Handle ios sim builds in Xcode. + message(FATAL_ERROR "This toolchain does not support Xcode.") +endif () + +set(CMAKE_SYSTEM_PROCESSOR "x86_64") +set(CMAKE_OSX_ARCHITECTURES "x86_64") + +include("${CMAKE_CURRENT_LIST_DIR}/ios-simulator-common.cmake") + +endif () # AOM_BUILD_CMAKE_TOOLCHAINS_X86_64_IOS_SIMULATOR_CMAKE_ diff --git a/third_party/aom/build/make/Android.mk b/third_party/aom/build/make/Android.mk new file mode 100644 index 0000000000..6757b1f595 --- /dev/null +++ b/third_party/aom/build/make/Android.mk @@ -0,0 +1,201 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + +# +# This file is to be used for compiling libaom for Android using the NDK. +# In an Android project place a libaom checkout in the jni directory. +# Run the configure script from the jni directory. Base libaom +# encoder/decoder configuration will look similar to: +# ./libaom/configure --target=armv7-android-gcc --disable-examples \ +# --sdk-path=/opt/android-ndk-r6b/ +# +# When targeting Android, realtime-only is enabled by default. This can +# be overridden by adding the command line flag: +# --disable-realtime-only +# +# This will create .mk files that contain variables that contain the +# source files to compile. +# +# Place an Android.mk file in the jni directory that references the +# Android.mk file in the libaom directory: +# LOCAL_PATH := $(call my-dir) +# include $(CLEAR_VARS) +# include jni/libaom/build/make/Android.mk +# +# There are currently two TARGET_ARCH_ABI targets for ARM. +# armeabi and armeabi-v7a. armeabi-v7a is selected by creating an +# Application.mk in the jni directory that contains: +# APP_ABI := armeabi-v7a +# +# By default libaom will detect at runtime the existance of NEON extension. +# For this we import the 'cpufeatures' module from the NDK sources. +# libaom can also be configured without this runtime detection method. +# Configuring with --disable-runtime-cpu-detect will assume presence of NEON. +# Configuring with --disable-runtime-cpu-detect --disable-neon \ +# --disable-neon-asm +# will remove any NEON dependency. + +# To change to building armeabi, run ./libaom/configure again, but with +# --target=armv6-android-gcc and modify the Application.mk file to +# set APP_ABI := armeabi +# +# Running ndk-build will build libaom and include it in your project. +# + +CONFIG_DIR := $(LOCAL_PATH)/ +LIBAOM_PATH := $(LOCAL_PATH)/libaom +ASM_CNV_PATH_LOCAL := $(TARGET_ARCH_ABI)/ads2gas +ASM_CNV_PATH := $(LOCAL_PATH)/$(ASM_CNV_PATH_LOCAL) + +# Use the makefiles generated by upstream configure to determine which files to +# build. Also set any architecture-specific flags. +ifeq ($(TARGET_ARCH_ABI),armeabi-v7a) + include $(CONFIG_DIR)libs-armv7-android-gcc.mk + LOCAL_ARM_MODE := arm +else ifeq ($(TARGET_ARCH_ABI),armeabi) + include $(CONFIG_DIR)libs-armv6-android-gcc.mk + LOCAL_ARM_MODE := arm +else ifeq ($(TARGET_ARCH_ABI),arm64-v8a) + include $(CONFIG_DIR)libs-armv8-android-gcc.mk + LOCAL_ARM_MODE := arm +else ifeq ($(TARGET_ARCH_ABI),x86) + include $(CONFIG_DIR)libs-x86-android-gcc.mk +else ifeq ($(TARGET_ARCH_ABI),x86_64) + include $(CONFIG_DIR)libs-x86_64-android-gcc.mk +else ifeq ($(TARGET_ARCH_ABI),mips) + include $(CONFIG_DIR)libs-mips-android-gcc.mk +else + $(error Not a supported TARGET_ARCH_ABI: $(TARGET_ARCH_ABI)) +endif + +# Rule that is normally in Makefile created by libaom +# configure. Used to filter out source files based on configuration. +enabled=$(filter-out $($(1)-no),$($(1)-yes)) + +# Override the relative path that is defined by the libaom +# configure process +SRC_PATH_BARE := $(LIBAOM_PATH) + +# Include the list of files to be built +include $(LIBAOM_PATH)/libs.mk + +# Optimise the code. May want to revisit this setting in the future. +LOCAL_CFLAGS := -O3 + +# For x86, include the source code in the search path so it will find files +# like x86inc.asm and x86_abi_support.asm +LOCAL_ASMFLAGS := -I$(LIBAOM_PATH) + +.PRECIOUS: %.asm.s +$(ASM_CNV_PATH)/libaom/%.asm.s: $(LIBAOM_PATH)/%.asm + @mkdir -p $(dir $@) + @$(CONFIG_DIR)$(ASM_CONVERSION) <$< > $@ + +# For building *_rtcd.h, which have rules in libs.mk +TGT_ISA:=$(word 1, $(subst -, ,$(TOOLCHAIN))) +target := libs + +LOCAL_SRC_FILES += aom_config.c + +# Remove duplicate entries +CODEC_SRCS_UNIQUE = $(sort $(CODEC_SRCS)) + +# Pull out C files. aom_config.c is in the immediate directory and +# so it does not need libaom/ prefixed like the rest of the source files. +# The neon files with intrinsics need to have .neon appended so the proper +# flags are applied. +CODEC_SRCS_C = $(filter %.c, $(CODEC_SRCS_UNIQUE)) +LOCAL_NEON_SRCS_C = $(filter %_neon.c, $(CODEC_SRCS_C)) +LOCAL_CODEC_SRCS_C = $(filter-out aom_config.c %_neon.c, $(CODEC_SRCS_C)) + +LOCAL_SRC_FILES += $(foreach file, $(LOCAL_CODEC_SRCS_C), libaom/$(file)) +ifeq ($(TARGET_ARCH_ABI),armeabi-v7a) + LOCAL_SRC_FILES += $(foreach file, $(LOCAL_NEON_SRCS_C), libaom/$(file).neon) +else # If there are neon sources then we are building for arm64 and do not need to specify .neon + LOCAL_SRC_FILES += $(foreach file, $(LOCAL_NEON_SRCS_C), libaom/$(file)) +endif + +# Pull out assembly files, splitting NEON from the rest. This is +# done to specify that the NEON assembly files use NEON assembler flags. +# x86 assembly matches %.asm, arm matches %.asm.s + +# x86: + +CODEC_SRCS_ASM_X86 = $(filter %.asm, $(CODEC_SRCS_UNIQUE)) +LOCAL_SRC_FILES += $(foreach file, $(CODEC_SRCS_ASM_X86), libaom/$(file)) + +# arm: +CODEC_SRCS_ASM_ARM_ALL = $(filter %.asm.s, $(CODEC_SRCS_UNIQUE)) +CODEC_SRCS_ASM_ARM = $(foreach v, \ + $(CODEC_SRCS_ASM_ARM_ALL), \ + $(if $(findstring neon,$(v)),,$(v))) +CODEC_SRCS_ASM_ADS2GAS = $(patsubst %.s, \ + $(ASM_CNV_PATH_LOCAL)/libaom/%.s, \ + $(CODEC_SRCS_ASM_ARM)) +LOCAL_SRC_FILES += $(CODEC_SRCS_ASM_ADS2GAS) + +ifeq ($(TARGET_ARCH_ABI),armeabi-v7a) + CODEC_SRCS_ASM_NEON = $(foreach v, \ + $(CODEC_SRCS_ASM_ARM_ALL),\ + $(if $(findstring neon,$(v)),$(v),)) + CODEC_SRCS_ASM_NEON_ADS2GAS = $(patsubst %.s, \ + $(ASM_CNV_PATH_LOCAL)/libaom/%.s, \ + $(CODEC_SRCS_ASM_NEON)) + LOCAL_SRC_FILES += $(patsubst %.s, \ + %.s.neon, \ + $(CODEC_SRCS_ASM_NEON_ADS2GAS)) +endif + +LOCAL_CFLAGS += \ + -DHAVE_CONFIG_H=aom_config.h \ + -I$(LIBAOM_PATH) \ + -I$(ASM_CNV_PATH) + +LOCAL_MODULE := libaom + +ifeq ($(CONFIG_RUNTIME_CPU_DETECT),yes) + LOCAL_STATIC_LIBRARIES := cpufeatures +endif + +# Add a dependency to force generation of the RTCD files. +define rtcd_dep_template +rtcd_dep_template_SRCS := $(addprefix $(LOCAL_PATH)/, $(LOCAL_SRC_FILES)) +rtcd_dep_template_SRCS := $$(rtcd_dep_template_SRCS:.neon=) +ifeq ($(CONFIG_AV1), yes) +$$(rtcd_dep_template_SRCS): av1_rtcd.h +endif +$$(rtcd_dep_template_SRCS): aom_scale_rtcd.h +$$(rtcd_dep_template_SRCS): aom_dsp_rtcd.h + +ifneq ($(findstring $(TARGET_ARCH_ABI),x86 x86_64),) +$$(rtcd_dep_template_SRCS): aom_config.asm +endif +endef + +$(eval $(call rtcd_dep_template)) + +.PHONY: clean +clean: + @echo "Clean: ads2gas files [$(TARGET_ARCH_ABI)]" + @$(RM) $(CODEC_SRCS_ASM_ADS2GAS) $(CODEC_SRCS_ASM_NEON_ADS2GAS) + @$(RM) -r $(ASM_CNV_PATH) + @$(RM) $(CLEAN-OBJS) + +ifeq ($(ENABLE_SHARED),1) + include $(BUILD_SHARED_LIBRARY) +else + include $(BUILD_STATIC_LIBRARY) +endif + +ifeq ($(CONFIG_RUNTIME_CPU_DETECT),yes) +$(call import-module,cpufeatures) +endif diff --git a/third_party/aom/build/make/Makefile b/third_party/aom/build/make/Makefile new file mode 100644 index 0000000000..0b869db0a1 --- /dev/null +++ b/third_party/aom/build/make/Makefile @@ -0,0 +1,466 @@ +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + +include config.mk +quiet?=true +ifeq ($(target),) +# If a target wasn't specified, invoke for all enabled targets. +.DEFAULT: + @for t in $(ALL_TARGETS); do \ + $(MAKE) --no-print-directory target=$$t $(MAKECMDGOALS) || exit $$?;\ + done +all: .DEFAULT +clean:: .DEFAULT +exampletest: .DEFAULT +install:: .DEFAULT +test:: .DEFAULT +test-no-data-check:: .DEFAULT +testdata:: .DEFAULT +utiltest: .DEFAULT +exampletest-no-data-check utiltest-no-data-check: .DEFAULT +test_%: .DEFAULT ; + +# Note: md5sum is not installed on OS X, but openssl is. Openssl may not be +# installed on cygwin, so we need to autodetect here. +md5sum := $(firstword $(wildcard \ + $(foreach e,md5sum openssl,\ + $(foreach p,$(subst :, ,$(PATH)),$(p)/$(e)*))\ + )) +md5sum := $(if $(filter %openssl,$(md5sum)),$(md5sum) dgst -md5,$(md5sum)) + +TGT_CC:=$(word 3, $(subst -, ,$(TOOLCHAIN))) +dist: + @for t in $(ALL_TARGETS); do \ + $(MAKE) --no-print-directory target=$$t $(MAKECMDGOALS) || exit $$?;\ + done + # Run configure for the user with the current toolchain. + @if [ -d "$(DIST_DIR)/src" ]; then \ + mkdir -p "$(DIST_DIR)/build"; \ + cd "$(DIST_DIR)/build"; \ + echo "Rerunning configure $(CONFIGURE_ARGS)"; \ + ../src/configure $(CONFIGURE_ARGS); \ + $(if $(filter vs%,$(TGT_CC)),make NO_LAUNCH_DEVENV=1;) \ + fi + @if [ -d "$(DIST_DIR)" ]; then \ + echo " [MD5SUM] $(DIST_DIR)"; \ + cd $(DIST_DIR) && \ + $(md5sum) `find . -name md5sums.txt -prune -o -type f -print` \ + | sed -e 's/MD5(\(.*\))= \([0-9a-f]\{32\}\)/\2 \1/' \ + > md5sums.txt;\ + fi +endif + +# Since we invoke make recursively for multiple targets we need to include the +# .mk file for the correct target, but only when $(target) is non-empty. +ifneq ($(target),) +include $(target)-$(TOOLCHAIN).mk +endif +BUILD_ROOT?=. +VPATH=$(SRC_PATH_BARE) +CFLAGS+=-I$(BUILD_PFX)$(BUILD_ROOT) -I$(SRC_PATH) +CXXFLAGS+=-I$(BUILD_PFX)$(BUILD_ROOT) -I$(SRC_PATH) +ASFLAGS+=-I$(BUILD_PFX)$(BUILD_ROOT)/ -I$(SRC_PATH)/ +DIST_DIR?=dist +HOSTCC?=gcc +TGT_ISA:=$(word 1, $(subst -, ,$(TOOLCHAIN))) +TGT_OS:=$(word 2, $(subst -, ,$(TOOLCHAIN))) +TGT_CC:=$(word 3, $(subst -, ,$(TOOLCHAIN))) +quiet:=$(if $(or $(verbose), $(V)),, yes) +qexec=$(if $(quiet),@) + +# Cancel built-in implicit rules +%: %.o +%.asm: +%.a: +%: %.cc + +# +# Common rules" +# +.PHONY: all +all: + +.PHONY: clean +clean:: + rm -f $(OBJS-yes) $(OBJS-yes:.o=.d) $(OBJS-yes:.asm.s.o=.asm.s) + rm -f $(CLEAN-OBJS) + +.PHONY: clean +distclean: clean + if [ -z "$(target)" ]; then \ + rm -f Makefile; \ + rm -f config.log config.mk; \ + rm -f aom_config.[hc] aom_config.asm; \ + else \ + rm -f $(target)-$(TOOLCHAIN).mk; \ + fi + +.PHONY: dist +dist: +.PHONY: exampletest +exampletest: +.PHONY: install +install:: +.PHONY: test +test:: +.PHONY: testdata +testdata:: +.PHONY: utiltest +utiltest: +.PHONY: test-no-data-check exampletest-no-data-check utiltest-no-data-check +test-no-data-check:: +exampletest-no-data-check utiltest-no-data-check: + +# Force to realign stack always on OS/2 +ifeq ($(TOOLCHAIN), x86-os2-gcc) +CFLAGS += -mstackrealign +endif + +$(BUILD_PFX)%_mmx.c.d: CFLAGS += -mmmx +$(BUILD_PFX)%_mmx.c.o: CFLAGS += -mmmx +$(BUILD_PFX)%_sse2.c.d: CFLAGS += -msse2 +$(BUILD_PFX)%_sse2.c.o: CFLAGS += -msse2 +$(BUILD_PFX)%_sse3.c.d: CFLAGS += -msse3 +$(BUILD_PFX)%_sse3.c.o: CFLAGS += -msse3 +$(BUILD_PFX)%_ssse3.c.d: CFLAGS += -mssse3 +$(BUILD_PFX)%_ssse3.c.o: CFLAGS += -mssse3 +$(BUILD_PFX)%_sse4.c.d: CFLAGS += -msse4.1 +$(BUILD_PFX)%_sse4.c.o: CFLAGS += -msse4.1 +$(BUILD_PFX)%_avx.c.d: CFLAGS += -mavx +$(BUILD_PFX)%_avx.c.o: CFLAGS += -mavx +$(BUILD_PFX)%_avx2.c.d: CFLAGS += -mavx2 +$(BUILD_PFX)%_avx2.c.o: CFLAGS += -mavx2 +$(BUILD_PFX)%_mmx.cc.d: CXXFLAGS += -mmmx +$(BUILD_PFX)%_mmx.cc.o: CXXFLAGS += -mmmx +$(BUILD_PFX)%_sse2.cc.d: CXXFLAGS += -msse2 +$(BUILD_PFX)%_sse2.cc.o: CXXFLAGS += -msse2 +$(BUILD_PFX)%_sse3.cc.d: CXXFLAGS += -msse3 +$(BUILD_PFX)%_sse3.cc.o: CXXFLAGS += -msse3 +$(BUILD_PFX)%_ssse3.cc.d: CXXFLAGS += -mssse3 +$(BUILD_PFX)%_ssse3.cc.o: CXXFLAGS += -mssse3 +$(BUILD_PFX)%_sse4.cc.d: CXXFLAGS += -msse4.1 +$(BUILD_PFX)%_sse4.cc.o: CXXFLAGS += -msse4.1 +$(BUILD_PFX)%_avx.cc.d: CXXFLAGS += -mavx +$(BUILD_PFX)%_avx.cc.o: CXXFLAGS += -mavx +$(BUILD_PFX)%_avx2.cc.d: CXXFLAGS += -mavx2 +$(BUILD_PFX)%_avx2.cc.o: CXXFLAGS += -mavx2 + +$(BUILD_PFX)%.c.d: %.c + $(if $(quiet),@echo " [DEP] $@") + $(qexec)mkdir -p $(dir $@) + $(qexec)$(CC) $(INTERNAL_CFLAGS) $(CFLAGS) -M $< | $(fmt_deps) > $@ + +$(BUILD_PFX)%.c.o: %.c + $(if $(quiet),@echo " [CC] $@") + $(qexec)$(if $(CONFIG_DEPENDENCY_TRACKING),,mkdir -p $(dir $@)) + $(qexec)$(CC) $(INTERNAL_CFLAGS) $(CFLAGS) -c -o $@ $< + +$(BUILD_PFX)%.cc.d: %.cc + $(if $(quiet),@echo " [DEP] $@") + $(qexec)mkdir -p $(dir $@) + $(qexec)$(CXX) $(INTERNAL_CFLAGS) $(CXXFLAGS) -M $< | $(fmt_deps) > $@ + +$(BUILD_PFX)%.cc.o: %.cc + $(if $(quiet),@echo " [CXX] $@") + $(qexec)$(if $(CONFIG_DEPENDENCY_TRACKING),,mkdir -p $(dir $@)) + $(qexec)$(CXX) $(INTERNAL_CFLAGS) $(CXXFLAGS) -c -o $@ $< + +$(BUILD_PFX)%.cpp.d: %.cpp + $(if $(quiet),@echo " [DEP] $@") + $(qexec)mkdir -p $(dir $@) + $(qexec)$(CXX) $(INTERNAL_CFLAGS) $(CXXFLAGS) -M $< | $(fmt_deps) > $@ + +$(BUILD_PFX)%.cpp.o: %.cpp + $(if $(quiet),@echo " [CXX] $@") + $(qexec)$(if $(CONFIG_DEPENDENCY_TRACKING),,mkdir -p $(dir $@)) + $(qexec)$(CXX) $(INTERNAL_CFLAGS) $(CXXFLAGS) -c -o $@ $< + +$(BUILD_PFX)%.asm.d: %.asm + $(if $(quiet),@echo " [DEP] $@") + $(qexec)mkdir -p $(dir $@) + $(qexec)$(SRC_PATH_BARE)/build/make/gen_asm_deps.sh \ + --build-pfx=$(BUILD_PFX) --depfile=$@ $(ASFLAGS) $< > $@ + +$(BUILD_PFX)%.asm.o: %.asm + $(if $(quiet),@echo " [AS] $@") + $(qexec)$(if $(CONFIG_DEPENDENCY_TRACKING),,mkdir -p $(dir $@)) + $(qexec)$(AS) $(ASFLAGS) -o $@ $< + +$(BUILD_PFX)%.s.d: %.s + $(if $(quiet),@echo " [DEP] $@") + $(qexec)mkdir -p $(dir $@) + $(qexec)$(SRC_PATH_BARE)/build/make/gen_asm_deps.sh \ + --build-pfx=$(BUILD_PFX) --depfile=$@ $(ASFLAGS) $< > $@ + +$(BUILD_PFX)%.s.o: %.s + $(if $(quiet),@echo " [AS] $@") + $(qexec)$(if $(CONFIG_DEPENDENCY_TRACKING),,mkdir -p $(dir $@)) + $(qexec)$(AS) $(ASFLAGS) -o $@ $< + +.PRECIOUS: %.c.S +%.c.S: CFLAGS += -DINLINE_ASM +$(BUILD_PFX)%.c.S: %.c + $(if $(quiet),@echo " [GEN] $@") + $(qexec)$(if $(CONFIG_DEPENDENCY_TRACKING),,mkdir -p $(dir $@)) + $(qexec)$(CC) -S $(CFLAGS) -o $@ $< + +.PRECIOUS: %.asm.s +$(BUILD_PFX)%.asm.s: %.asm + $(if $(quiet),@echo " [ASM CONVERSION] $@") + $(qexec)mkdir -p $(dir $@) + $(qexec)$(ASM_CONVERSION) <$< >$@ + +# If we're in debug mode, pretend we don't have GNU strip, to fall back to +# the copy implementation +HAVE_GNU_STRIP := $(if $(CONFIG_DEBUG),,$(HAVE_GNU_STRIP)) +ifeq ($(HAVE_GNU_STRIP),yes) +# Older binutils strip global symbols not needed for relocation processing +# when given --strip-unneeded. Using nm and awk to identify globals and +# keep them caused command line length issues under mingw and segfaults in +# test_libaom were observed under OS/2: simply use --strip-debug. +%.a: %_g.a + $(if $(quiet),@echo " [STRIP] $@ < $<") + $(qexec)$(STRIP) --strip-debug \ + -o $@ $< +else +%.a: %_g.a + $(if $(quiet),@echo " [CP] $@ < $<") + $(qexec)cp $< $@ +endif + +# +# Utility functions +# +pairmap=$(if $(strip $(2)),\ + $(call $(1),$(word 1,$(2)),$(word 2,$(2)))\ + $(call pairmap,$(1),$(wordlist 3,$(words $(2)),$(2)))\ +) + +enabled=$(filter-out $($(1)-no),$($(1)-yes)) +cond_enabled=$(if $(filter yes,$($(1))), $(call enabled,$(2))) + +find_file1=$(word 1,$(wildcard $(subst //,/,$(addsuffix /$(1),$(2))))) +find_file=$(foreach f,$(1),$(call find_file1,$(strip $(f)),$(strip $(2))) ) +obj_pats=.c=.c.o $(AS_SFX)=$(AS_SFX).o .cc=.cc.o .cpp=.cpp.o +objs=$(addprefix $(BUILD_PFX),$(foreach p,$(obj_pats),$(filter %.o,$(1:$(p))) )) + +install_map_templates=$(eval $(call install_map_template,$(1),$(2))) + +not=$(subst yes,no,$(1)) + +ifeq ($(CONFIG_MSVS),yes) +lib_file_name=$(1).lib +else +lib_file_name=lib$(1).a +endif +# +# Rule Templates +# +define linker_template +$(1): $(filter-out -%,$(2)) +$(1): + $(if $(quiet),@echo " [LD] $$@") + $(qexec)$$(LD) $$(strip $$(INTERNAL_LDFLAGS) $$(LDFLAGS) -o $$@ $(2) $(3) $$(extralibs)) +endef +define linkerxx_template +$(1): $(filter-out -%,$(2)) +$(1): + $(if $(quiet),@echo " [LD] $$@") + $(qexec)$$(CXX) $$(strip $$(INTERNAL_LDFLAGS) $$(LDFLAGS) -o $$@ $(2) $(3) $$(extralibs)) +endef +# make-3.80 has a bug with expanding large input strings to the eval function, +# which was triggered in some cases by the following component of +# linker_template: +# $(1): $$(call find_file, $(patsubst -l%,lib%.a,$(filter -l%,$(2))),\ +# $$(patsubst -L%,%,$$(filter -L%,$$(LDFLAGS) $(2)))) +# This may be useful to revisit in the future (it tries to locate libraries +# in a search path and add them as prerequisites + +define install_map_template +$(DIST_DIR)/$(1): $(2) + $(if $(quiet),@echo " [INSTALL] $$@") + $(qexec)mkdir -p $$(dir $$@) + $(qexec)cp -p $$< $$@ +endef + +define archive_template +# Not using a pattern rule here because we don't want to generate empty +# archives when they are listed as a dependency in files not responsible +# for creating them. +$(1): + $(if $(quiet),@echo " [AR] $$@") + $(qexec)$$(AR) $$(ARFLAGS) $$@ $$^ +endef + +define so_template +# Not using a pattern rule here because we don't want to generate empty +# archives when they are listed as a dependency in files not responsible +# for creating them. +# +# This needs further abstraction for dealing with non-GNU linkers. +$(1): + $(if $(quiet),@echo " [LD] $$@") + $(qexec)$$(LD) -shared $$(LDFLAGS) \ + -Wl,--no-undefined -Wl,-soname,$$(SONAME) \ + -Wl,--version-script,$$(EXPORTS_FILE) -o $$@ \ + $$(filter %.o,$$^) $$(extralibs) +endef + +define dl_template +# Not using a pattern rule here because we don't want to generate empty +# archives when they are listed as a dependency in files not responsible +# for creating them. +$(1): + $(if $(quiet),@echo " [LD] $$@") + $(qexec)$$(LD) -dynamiclib $$(LDFLAGS) \ + -exported_symbols_list $$(EXPORTS_FILE) \ + -Wl,-headerpad_max_install_names,-compatibility_version,1.0,-current_version,$$(VERSION_MAJOR) \ + -o $$@ \ + $$(filter %.o,$$^) $$(extralibs) +endef + +define dll_template +# Not using a pattern rule here because we don't want to generate empty +# archives when they are listed as a dependency in files not responsible +# for creating them. +$(1): + $(if $(quiet),@echo " [LD] $$@") + $(qexec)$$(LD) -Zdll $$(LDFLAGS) \ + -o $$@ \ + $$(filter %.o,$$^) $$(extralibs) $$(EXPORTS_FILE) +endef + + +# +# Get current configuration +# +ifneq ($(target),) +include $(SRC_PATH_BARE)/$(target:-$(TOOLCHAIN)=).mk +endif + +skip_deps := $(filter %clean,$(MAKECMDGOALS)) +skip_deps += $(findstring testdata,$(MAKECMDGOALS)) +ifeq ($(strip $(skip_deps)),) + ifeq ($(CONFIG_DEPENDENCY_TRACKING),yes) + # Older versions of make don't like -include directives with no arguments + ifneq ($(filter %.d,$(OBJS-yes:.o=.d)),) + -include $(filter %.d,$(OBJS-yes:.o=.d)) + endif + endif +endif + +# +# Configuration dependent rules +# +$(call pairmap,install_map_templates,$(INSTALL_MAPS)) + +DOCS=$(call cond_enabled,CONFIG_INSTALL_DOCS,DOCS) +.docs: $(DOCS) + @touch $@ + +INSTALL-DOCS=$(call cond_enabled,CONFIG_INSTALL_DOCS,INSTALL-DOCS) +ifeq ($(MAKECMDGOALS),dist) +INSTALL-DOCS+=$(call cond_enabled,CONFIG_INSTALL_DOCS,DIST-DOCS) +endif +.install-docs: .docs $(addprefix $(DIST_DIR)/,$(INSTALL-DOCS)) + @touch $@ + +clean:: + rm -f .docs .install-docs $(DOCS) + +BINS=$(call enabled,BINS) +.bins: $(BINS) + @touch $@ + +INSTALL-BINS=$(call cond_enabled,CONFIG_INSTALL_BINS,INSTALL-BINS) +ifeq ($(MAKECMDGOALS),dist) +INSTALL-BINS+=$(call cond_enabled,CONFIG_INSTALL_BINS,DIST-BINS) +endif +.install-bins: .bins $(addprefix $(DIST_DIR)/,$(INSTALL-BINS)) + @touch $@ + +clean:: + rm -f .bins .install-bins $(BINS) + +LIBS=$(call enabled,LIBS) +.libs: $(LIBS) + @touch $@ +$(foreach lib,$(filter %_g.a,$(LIBS)),$(eval $(call archive_template,$(lib)))) +$(foreach lib,$(filter %so.$(SO_VERSION_MAJOR).$(SO_VERSION_MINOR).$(SO_VERSION_PATCH),$(LIBS)),$(eval $(call so_template,$(lib)))) +$(foreach lib,$(filter %$(SO_VERSION_MAJOR).dylib,$(LIBS)),$(eval $(call dl_template,$(lib)))) +$(foreach lib,$(filter %$(SO_VERSION_MAJOR).dll,$(LIBS)),$(eval $(call dll_template,$(lib)))) + +INSTALL-LIBS=$(call cond_enabled,CONFIG_INSTALL_LIBS,INSTALL-LIBS) +ifeq ($(MAKECMDGOALS),dist) +INSTALL-LIBS+=$(call cond_enabled,CONFIG_INSTALL_LIBS,DIST-LIBS) +endif +.install-libs: .libs $(addprefix $(DIST_DIR)/,$(INSTALL-LIBS)) + @touch $@ + +clean:: + rm -f .libs .install-libs $(LIBS) + +ifeq ($(CONFIG_EXTERNAL_BUILD),yes) +PROJECTS=$(call enabled,PROJECTS) +.projects: $(PROJECTS) + @touch $@ + +INSTALL-PROJECTS=$(call cond_enabled,CONFIG_INSTALL_PROJECTS,INSTALL-PROJECTS) +ifeq ($(MAKECMDGOALS),dist) +INSTALL-PROJECTS+=$(call cond_enabled,CONFIG_INSTALL_PROJECTS,DIST-PROJECTS) +endif +.install-projects: .projects $(addprefix $(DIST_DIR)/,$(INSTALL-PROJECTS)) + @touch $@ + +clean:: + rm -f .projects .install-projects $(PROJECTS) +endif + +# If there are any source files to be distributed, then include the build +# system too. +ifneq ($(call enabled,DIST-SRCS),) + DIST-SRCS-yes += configure + DIST-SRCS-yes += build/make/configure.sh + DIST-SRCS-yes += build/make/gen_asm_deps.sh + DIST-SRCS-yes += build/make/Makefile + DIST-SRCS-$(CONFIG_MSVS) += build/make/gen_msvs_def.sh + DIST-SRCS-$(CONFIG_MSVS) += build/make/gen_msvs_sln.sh + DIST-SRCS-$(CONFIG_MSVS) += build/make/gen_msvs_vcxproj.sh + DIST-SRCS-$(CONFIG_MSVS) += build/make/msvs_common.sh + DIST-SRCS-$(CONFIG_RVCT) += build/make/armlink_adapter.sh + DIST-SRCS-$(ARCH_ARM) += build/make/ads2gas.pl + DIST-SRCS-$(ARCH_ARM) += build/make/ads2gas_apple.pl + DIST-SRCS-$(ARCH_ARM) += build/make/ads2armasm_ms.pl + DIST-SRCS-$(ARCH_ARM) += build/make/thumb.pm + DIST-SRCS-yes += $(target:-$(TOOLCHAIN)=).mk +endif +INSTALL-SRCS := $(call cond_enabled,CONFIG_INSTALL_SRCS,INSTALL-SRCS) +ifeq ($(MAKECMDGOALS),dist) +INSTALL-SRCS += $(call cond_enabled,CONFIG_INSTALL_SRCS,DIST-SRCS) +endif +.install-srcs: $(addprefix $(DIST_DIR)/src/,$(INSTALL-SRCS)) + @touch $@ + +clean:: + rm -f .install-srcs + +ifeq ($(CONFIG_EXTERNAL_BUILD),yes) + BUILD_TARGETS += .projects + INSTALL_TARGETS += .install-projects +endif +BUILD_TARGETS += .docs .libs .bins +INSTALL_TARGETS += .install-docs .install-srcs .install-libs .install-bins +all: $(BUILD_TARGETS) +install:: $(INSTALL_TARGETS) +dist: $(INSTALL_TARGETS) +test:: + +.SUFFIXES: # Delete default suffix rules diff --git a/third_party/aom/build/make/ads2armasm_ms.pl b/third_party/aom/build/make/ads2armasm_ms.pl new file mode 100755 index 0000000000..8568a2dadf --- /dev/null +++ b/third_party/aom/build/make/ads2armasm_ms.pl @@ -0,0 +1,39 @@ +#!/usr/bin/env perl +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +use FindBin; +use lib $FindBin::Bin; +use thumb; + +print "; This file was created from a .asm file\n"; +print "; using the ads2armasm_ms.pl script.\n"; + +while () +{ + undef $comment; + undef $line; + + s/REQUIRE8//; + s/PRESERVE8//; + s/^\s*ARM\s*$//; + s/AREA\s+\|\|(.*)\|\|/AREA |$1|/; + s/qsubaddx/qsax/i; + s/qaddsubx/qasx/i; + + thumb::FixThumbInstructions($_, 1); + + s/ldrneb/ldrbne/i; + s/ldrneh/ldrhne/i; + s/^(\s*)ENDP.*/$&\n$1ALIGN 4/; + + print; +} + diff --git a/third_party/aom/build/make/ads2gas.pl b/third_party/aom/build/make/ads2gas.pl new file mode 100755 index 0000000000..adf45a3c91 --- /dev/null +++ b/third_party/aom/build/make/ads2gas.pl @@ -0,0 +1,236 @@ +#!/usr/bin/env perl +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + +# ads2gas.pl +# Author: Eric Fung (efung (at) acm.org) +# +# Convert ARM Developer Suite 1.0.1 syntax assembly source to GNU as format +# +# Usage: cat inputfile | perl ads2gas.pl > outputfile +# + +use FindBin; +use lib $FindBin::Bin; +use thumb; + +my $thumb = 0; + +foreach my $arg (@ARGV) { + $thumb = 1 if ($arg eq "-thumb"); +} + +print "@ This file was created from a .asm file\n"; +print "@ using the ads2gas.pl script.\n"; +print "\t.equ DO1STROUNDING, 0\n"; +if ($thumb) { + print "\t.syntax unified\n"; + print "\t.thumb\n"; +} + +# Stack of procedure names. +@proc_stack = (); + +while () +{ + undef $comment; + undef $line; + $comment_char = ";"; + $comment_sub = "@"; + + # Handle comments. + if (/$comment_char/) + { + $comment = ""; + ($line, $comment) = /(.*?)$comment_char(.*)/; + $_ = $line; + } + + # Load and store alignment + s/@/,:/g; + + # Hexadecimal constants prefaced by 0x + s/#&/#0x/g; + + # Convert :OR: to | + s/:OR:/ | /g; + + # Convert :AND: to & + s/:AND:/ & /g; + + # Convert :NOT: to ~ + s/:NOT:/ ~ /g; + + # Convert :SHL: to << + s/:SHL:/ << /g; + + # Convert :SHR: to >> + s/:SHR:/ >> /g; + + # Convert ELSE to .else + s/\bELSE\b/.else/g; + + # Convert ENDIF to .endif + s/\bENDIF\b/.endif/g; + + # Convert ELSEIF to .elseif + s/\bELSEIF\b/.elseif/g; + + # Convert LTORG to .ltorg + s/\bLTORG\b/.ltorg/g; + + # Convert endfunc to nothing. + s/\bendfunc\b//ig; + + # Convert FUNCTION to nothing. + s/\bFUNCTION\b//g; + s/\bfunction\b//g; + + s/\bENTRY\b//g; + s/\bMSARMASM\b/0/g; + s/^\s+end\s+$//g; + + # Convert IF :DEF:to .if + # gcc doesn't have the ability to do a conditional + # if defined variable that is set by IF :DEF: on + # armasm, so convert it to a normal .if and then + # make sure to define a value elesewhere + if (s/\bIF :DEF:\b/.if /g) + { + s/=/==/g; + } + + # Convert IF to .if + if (s/\bIF\b/.if/g) + { + s/=+/==/g; + } + + # Convert INCLUDE to .INCLUDE "file" + s/INCLUDE(\s*)(.*)$/.include $1\"$2\"/; + + # Code directive (ARM vs Thumb) + s/CODE([0-9][0-9])/.code $1/; + + # No AREA required + # But ALIGNs in AREA must be obeyed + s/^\s*AREA.*ALIGN=([0-9])$/.text\n.p2align $1/; + # If no ALIGN, strip the AREA and align to 4 bytes + s/^\s*AREA.*$/.text\n.p2align 2/; + + # DCD to .word + # This one is for incoming symbols + s/DCD\s+\|(\w*)\|/.long $1/; + + # DCW to .short + s/DCW\s+\|(\w*)\|/.short $1/; + s/DCW(.*)/.short $1/; + + # Constants defined in scope + s/DCD(.*)/.long $1/; + s/DCB(.*)/.byte $1/; + + # RN to .req + if (s/RN\s+([Rr]\d+|lr)/.req $1/) + { + print; + print "$comment_sub$comment\n" if defined $comment; + next; + } + + # Make function visible to linker, and make additional symbol with + # prepended underscore + s/EXPORT\s+\|([\$\w]*)\|/.global $1 \n\t.type $1, function/; + s/IMPORT\s+\|([\$\w]*)\|/.global $1/; + + s/EXPORT\s+([\$\w]*)/.global $1/; + s/export\s+([\$\w]*)/.global $1/; + + # No vertical bars required; make additional symbol with prepended + # underscore + s/^\|(\$?\w+)\|/_$1\n\t$1:/g; + + # Labels need trailing colon +# s/^(\w+)/$1:/ if !/EQU/; + # put the colon at the end of the line in the macro + s/^([a-zA-Z_0-9\$]+)/$1:/ if !/EQU/; + + # ALIGN directive + s/\bALIGN\b/.balign/g; + + if ($thumb) { + # ARM code - we force everything to thumb with the declaration in the header + s/\sARM//g; + } else { + # ARM code + s/\sARM/.arm/g; + } + + # push/pop + s/(push\s+)(r\d+)/stmdb sp\!, \{$2\}/g; + s/(pop\s+)(r\d+)/ldmia sp\!, \{$2\}/g; + + # NEON code + s/(vld1.\d+\s+)(q\d+)/$1\{$2\}/g; + s/(vtbl.\d+\s+[^,]+),([^,]+)/$1,\{$2\}/g; + + if ($thumb) { + thumb::FixThumbInstructions($_, 0); + } + + # eabi_attributes numerical equivalents can be found in the + # "ARM IHI 0045C" document. + + # REQUIRE8 Stack is required to be 8-byte aligned + s/\sREQUIRE8/.eabi_attribute 24, 1 \@Tag_ABI_align_needed/g; + + # PRESERVE8 Stack 8-byte align is preserved + s/\sPRESERVE8/.eabi_attribute 25, 1 \@Tag_ABI_align_preserved/g; + + # Use PROC and ENDP to give the symbols a .size directive. + # This makes them show up properly in debugging tools like gdb and valgrind. + if (/\bPROC\b/) + { + my $proc; + /^_([\.0-9A-Z_a-z]\w+)\b/; + $proc = $1; + push(@proc_stack, $proc) if ($proc); + s/\bPROC\b/@ $&/; + } + if (/\bENDP\b/) + { + my $proc; + s/\bENDP\b/@ $&/; + $proc = pop(@proc_stack); + $_ = "\t.size $proc, .-$proc".$_ if ($proc); + } + + # EQU directive + s/(\S+\s+)EQU(\s+\S+)/.equ $1, $2/; + + # Begin macro definition + if (/\bMACRO\b/) { + $_ = ; + s/^/.macro/; + s/\$//g; # remove formal param reference + s/;/@/g; # change comment characters + } + + # For macros, use \ to reference formal params + s/\$/\\/g; # End macro definition + s/\bMEND\b/.endm/; # No need to tell it where to stop assembling + next if /^\s*END\s*$/; + print; + print "$comment_sub$comment\n" if defined $comment; +} + +# Mark that this object doesn't need an executable stack. +printf ("\t.section\t.note.GNU-stack,\"\",\%\%progbits\n"); diff --git a/third_party/aom/build/make/ads2gas_apple.pl b/third_party/aom/build/make/ads2gas_apple.pl new file mode 100755 index 0000000000..31ec91d567 --- /dev/null +++ b/third_party/aom/build/make/ads2gas_apple.pl @@ -0,0 +1,235 @@ +#!/usr/bin/env perl +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + +# ads2gas_apple.pl +# Author: Eric Fung (efung (at) acm.org) +# +# Convert ARM Developer Suite 1.0.1 syntax assembly source to GNU as format +# +# Usage: cat inputfile | perl ads2gas_apple.pl > outputfile +# + +my $chromium = 0; + +foreach my $arg (@ARGV) { + $chromium = 1 if ($arg eq "-chromium"); +} + +print "@ This file was created from a .asm file\n"; +print "@ using the ads2gas_apple.pl script.\n\n"; +print "\t.set WIDE_REFERENCE, 0\n"; +print "\t.set ARCHITECTURE, 5\n"; +print "\t.set DO1STROUNDING, 0\n"; + +my %register_aliases; +my %macro_aliases; + +my @mapping_list = ("\$0", "\$1", "\$2", "\$3", "\$4", "\$5", "\$6", "\$7", "\$8", "\$9"); + +my @incoming_array; + +my @imported_functions; + +# Perl trim function to remove whitespace from the start and end of the string +sub trim($) +{ + my $string = shift; + $string =~ s/^\s+//; + $string =~ s/\s+$//; + return $string; +} + +while () +{ + # Load and store alignment + s/@/,:/g; + + # Comment character + s/;/ @/g; + + # Hexadecimal constants prefaced by 0x + s/#&/#0x/g; + + # Convert :OR: to | + s/:OR:/ | /g; + + # Convert :AND: to & + s/:AND:/ & /g; + + # Convert :NOT: to ~ + s/:NOT:/ ~ /g; + + # Convert :SHL: to << + s/:SHL:/ << /g; + + # Convert :SHR: to >> + s/:SHR:/ >> /g; + + # Convert ELSE to .else + s/\bELSE\b/.else/g; + + # Convert ENDIF to .endif + s/\bENDIF\b/.endif/g; + + # Convert ELSEIF to .elseif + s/\bELSEIF\b/.elseif/g; + + # Convert LTORG to .ltorg + s/\bLTORG\b/.ltorg/g; + + # Convert IF :DEF:to .if + # gcc doesn't have the ability to do a conditional + # if defined variable that is set by IF :DEF: on + # armasm, so convert it to a normal .if and then + # make sure to define a value elesewhere + if (s/\bIF :DEF:\b/.if /g) + { + s/=/==/g; + } + + # Convert IF to .if + if (s/\bIF\b/.if/g) + { + s/=/==/g; + } + + # Convert INCLUDE to .INCLUDE "file" + s/INCLUDE(\s*)(.*)$/.include $1\"$2\"/; + + # Code directive (ARM vs Thumb) + s/CODE([0-9][0-9])/.code $1/; + + # No AREA required + # But ALIGNs in AREA must be obeyed + s/^\s*AREA.*ALIGN=([0-9])$/.text\n.p2align $1/; + # If no ALIGN, strip the AREA and align to 4 bytes + s/^\s*AREA.*$/.text\n.p2align 2/; + + # DCD to .word + # This one is for incoming symbols + s/DCD\s+\|(\w*)\|/.long $1/; + + # DCW to .short + s/DCW\s+\|(\w*)\|/.short $1/; + s/DCW(.*)/.short $1/; + + # Constants defined in scope + s/DCD(.*)/.long $1/; + s/DCB(.*)/.byte $1/; + + # Build a hash of all the register - alias pairs. + if (s/(.*)RN(.*)/$1 .req $2/g) + { + $register_aliases{trim($1)} = trim($2); + next; + } + + while (($key, $value) = each(%register_aliases)) + { + s/\b$key\b/$value/g; + } + + # Make function visible to linker, and make additional symbol with + # prepended underscore + s/EXPORT\s+\|([\$\w]*)\|/.globl _$1\n\t.globl $1/; + + # Prepend imported functions with _ + if (s/IMPORT\s+\|([\$\w]*)\|/.globl $1/) + { + $function = trim($1); + push(@imported_functions, $function); + } + + foreach $function (@imported_functions) + { + s/$function/_$function/; + } + + # No vertical bars required; make additional symbol with prepended + # underscore + s/^\|(\$?\w+)\|/_$1\n\t$1:/g; + + # Labels need trailing colon +# s/^(\w+)/$1:/ if !/EQU/; + # put the colon at the end of the line in the macro + s/^([a-zA-Z_0-9\$]+)/$1:/ if !/EQU/; + + # ALIGN directive + s/\bALIGN\b/.balign/g; + + # Strip ARM + s/\sARM/@ ARM/g; + + # Strip REQUIRE8 + #s/\sREQUIRE8/@ REQUIRE8/g; + s/\sREQUIRE8/@ /g; + + # Strip PRESERVE8 + s/\sPRESERVE8/@ PRESERVE8/g; + + # Strip PROC and ENDPROC + s/\bPROC\b/@/g; + s/\bENDP\b/@/g; + + # EQU directive + s/(.*)EQU(.*)/.set $1, $2/; + + # Begin macro definition + if (/\bMACRO\b/) + { + # Process next line down, which will be the macro definition + $_ = ; + + $trimmed = trim($_); + + # remove commas that are separating list + $trimmed =~ s/,//g; + + # string to array + @incoming_array = split(/\s+/, $trimmed); + + print ".macro @incoming_array[0]\n"; + + # remove the first element, as that is the name of the macro + shift (@incoming_array); + + @macro_aliases{@incoming_array} = @mapping_list; + + next; + } + + while (($key, $value) = each(%macro_aliases)) + { + $key =~ s/\$/\\\$/; + s/$key\b/$value/g; + } + + # For macros, use \ to reference formal params +# s/\$/\\/g; # End macro definition + s/\bMEND\b/.endm/; # No need to tell it where to stop assembling + next if /^\s*END\s*$/; + + # Clang used by Chromium differs slightly from clang in XCode in what it + # will accept in the assembly. + if ($chromium) { + s/qsubaddx/qsax/i; + s/qaddsubx/qasx/i; + s/ldrneb/ldrbne/i; + s/ldrneh/ldrhne/i; + s/(vqshrun\.s16 .*, \#)0$/${1}8/i; + + # http://llvm.org/bugs/show_bug.cgi?id=16022 + s/\.include/#include/; + } + + print; +} diff --git a/third_party/aom/build/make/armlink_adapter.sh b/third_party/aom/build/make/armlink_adapter.sh new file mode 100755 index 0000000000..85c6c96c13 --- /dev/null +++ b/third_party/aom/build/make/armlink_adapter.sh @@ -0,0 +1,53 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +verbose=0 +set -- $* +for i; do + if [ "$i" = "-o" ]; then + on_of=1 + elif [ "$i" = "-v" ]; then + verbose=1 + elif [ "$i" = "-g" ]; then + args="${args} --debug" + elif [ "$on_of" = "1" ]; then + outfile=$i + on_of=0 + elif [ -f "$i" ]; then + infiles="$infiles $i" + elif [ "${i#-l}" != "$i" ]; then + libs="$libs ${i#-l}" + elif [ "${i#-L}" != "$i" ]; then + libpaths="${libpaths} ${i#-L}" + else + args="${args} ${i}" + fi + shift +done + +# Absolutize library file names +for f in $libs; do + found=0 + for d in $libpaths; do + [ -f "$d/$f" ] && infiles="$infiles $d/$f" && found=1 && break + [ -f "$d/lib${f}.so" ] && infiles="$infiles $d/lib${f}.so" && found=1 && break + [ -f "$d/lib${f}.a" ] && infiles="$infiles $d/lib${f}.a" && found=1 && break + done + [ $found -eq 0 ] && infiles="$infiles $f" +done +for d in $libpaths; do + [ -n "$libsearchpath" ] && libsearchpath="${libsearchpath}," + libsearchpath="${libsearchpath}$d" +done + +cmd="armlink $args --userlibpath=$libsearchpath --output=$outfile $infiles" +[ $verbose -eq 1 ] && echo $cmd +$cmd diff --git a/third_party/aom/build/make/configure.sh b/third_party/aom/build/make/configure.sh new file mode 100644 index 0000000000..4ece17aee4 --- /dev/null +++ b/third_party/aom/build/make/configure.sh @@ -0,0 +1,1556 @@ +#!/bin/sh +## +## configure.sh +## +## This script is sourced by the main configure script and contains +## utility functions and other common bits that aren't strictly libaom +## related. +## +## This build system is based in part on the FFmpeg configure script. +## + + +# +# Logging / Output Functions +# +die_unknown(){ + echo "Unknown option \"$1\"." + echo "See $0 --help for available options." + clean_temp_files + exit 1 +} + +die() { + echo "$@" + echo + echo "Configuration failed. This could reflect a misconfiguration of your" + echo "toolchains, improper options selected, or another problem. If you" + echo "don't see any useful error messages above, the next step is to look" + echo "at the configure error log file ($logfile) to determine what" + echo "configure was trying to do when it died." + clean_temp_files + exit 1 +} + +log(){ + echo "$@" >>$logfile +} + +log_file(){ + log BEGIN $1 + cat -n $1 >>$logfile + log END $1 +} + +log_echo() { + echo "$@" + log "$@" +} + +fwrite () { + outfile=$1 + shift + echo "$@" >> ${outfile} +} + +show_help_pre(){ + for opt in ${CMDLINE_SELECT}; do + opt2=`echo $opt | sed -e 's;_;-;g'` + if enabled $opt; then + eval "toggle_${opt}=\"--disable-${opt2}\"" + else + eval "toggle_${opt}=\"--enable-${opt2} \"" + fi + done + + cat <>${logfile} 2>&1 +} + +check_cc() { + log check_cc "$@" + cat >${TMP_C} + log_file ${TMP_C} + check_cmd ${CC} ${CFLAGS} "$@" -c -o ${TMP_O} ${TMP_C} +} + +check_cxx() { + log check_cxx "$@" + cat >${TMP_CC} + log_file ${TMP_CC} + check_cmd ${CXX} ${CXXFLAGS} "$@" -c -o ${TMP_O} ${TMP_CC} +} + +check_cpp() { + log check_cpp "$@" + cat > ${TMP_C} + log_file ${TMP_C} + check_cmd ${CC} ${CFLAGS} "$@" -E -o ${TMP_O} ${TMP_C} +} + +check_ld() { + log check_ld "$@" + check_cc $@ \ + && check_cmd ${LD} ${LDFLAGS} "$@" -o ${TMP_X} ${TMP_O} ${extralibs} +} + +check_header(){ + log check_header "$@" + header=$1 + shift + var=`echo $header | sed 's/[^A-Za-z0-9_]/_/g'` + disable_feature $var + check_cpp "$@" <${TMP_ASM} <${TMP_X} + log_file ${TMP_X} + if ! grep -q '\.rodata .* 16$' ${TMP_X}; then + die "${AS} ${ASFLAGS} does not support section alignment (nasm <=2.08?)" + fi +} + +# tests for -m$1 toggling the feature given in $2. If $2 is empty $1 is used. +check_gcc_machine_option() { + opt="$1" + feature="$2" + [ -n "$feature" ] || feature="$opt" + + if enabled gcc && ! disabled "$feature" && ! check_cflags "-m$opt"; then + RTCD_OPTIONS="${RTCD_OPTIONS}--disable-$feature " + else + soft_enable "$feature" + fi +} + +write_common_config_banner() { + print_webm_license config.mk "##" "" + echo '# This file automatically generated by configure. Do not edit!' >> config.mk + echo "TOOLCHAIN := ${toolchain}" >> config.mk + + case ${toolchain} in + *-linux-rvct) + echo "ALT_LIBC := ${alt_libc}" >> config.mk + ;; + esac +} + +write_common_config_targets() { + for t in ${all_targets}; do + if enabled ${t}; then + if enabled child; then + fwrite config.mk "ALL_TARGETS += ${t}-${toolchain}" + else + fwrite config.mk "ALL_TARGETS += ${t}" + fi + fi + true; + done + true +} + +write_common_target_config_mk() { + saved_CC="${CC}" + saved_CXX="${CXX}" + enabled ccache && CC="ccache ${CC}" + enabled ccache && CXX="ccache ${CXX}" + print_webm_license $1 "##" "" + + cat >> $1 << EOF +# This file automatically generated by configure. Do not edit! +SRC_PATH="$source_path" +SRC_PATH_BARE=$source_path +BUILD_PFX=${BUILD_PFX} +TOOLCHAIN=${toolchain} +ASM_CONVERSION=${asm_conversion_cmd:-${source_path}/build/make/ads2gas.pl} +GEN_VCPROJ=${gen_vcproj_cmd} +MSVS_ARCH_DIR=${msvs_arch_dir} + +CC=${CC} +CXX=${CXX} +AR=${AR} +LD=${LD} +AS=${AS} +STRIP=${STRIP} +NM=${NM} + +CFLAGS = ${CFLAGS} +CXXFLAGS = ${CXXFLAGS} +ARFLAGS = crs\$(if \$(quiet),,v) +LDFLAGS = ${LDFLAGS} +ASFLAGS = ${ASFLAGS} +extralibs = ${extralibs} +AS_SFX = ${AS_SFX:-.asm} +EXE_SFX = ${EXE_SFX} +VCPROJ_SFX = ${VCPROJ_SFX} +RTCD_OPTIONS = ${RTCD_OPTIONS} +WX_CXXFLAGS = ${WX_CXXFLAGS} +WX_LDFLAGS = ${WX_LDFLAGS} +EOF + + if enabled rvct; then cat >> $1 << EOF +fmt_deps = sed -e 's;^__image.axf;\${@:.d=.o} \$@;' #hide +EOF + else cat >> $1 << EOF +fmt_deps = sed -e 's;^\([a-zA-Z0-9_]*\)\.o;\${@:.d=.o} \$@;' +EOF + fi + + print_config_mk ARCH "${1}" ${ARCH_LIST} + print_config_mk HAVE "${1}" ${HAVE_LIST} + print_config_mk CONFIG "${1}" ${CONFIG_LIST} + print_config_mk HAVE "${1}" gnu_strip + + enabled msvs && echo "CONFIG_VS_VERSION=${vs_version}" >> "${1}" + + CC="${saved_CC}" + CXX="${saved_CXX}" +} + +write_common_target_config_h() { + print_webm_license ${TMP_H} "/*" " */" + cat >> ${TMP_H} << EOF +/* This file automatically generated by configure. Do not edit! */ +#ifndef AOM_CONFIG_H +#define AOM_CONFIG_H +#define RESTRICT ${RESTRICT} +#define INLINE ${INLINE} +EOF + print_config_h ARCH "${TMP_H}" ${ARCH_LIST} + print_config_h HAVE "${TMP_H}" ${HAVE_LIST} + print_config_h CONFIG "${TMP_H}" ${CONFIG_LIST} + print_config_vars_h "${TMP_H}" ${VAR_LIST} + echo "#endif /* AOM_CONFIG_H */" >> ${TMP_H} + mkdir -p `dirname "$1"` + cmp "$1" ${TMP_H} >/dev/null 2>&1 || mv ${TMP_H} "$1" +} + +process_common_cmdline() { + for opt in "$@"; do + optval="${opt#*=}" + case "$opt" in + --child) + enable_feature child + ;; + --log*) + logging="$optval" + if ! disabled logging ; then + enabled logging || logfile="$logging" + else + logfile=/dev/null + fi + ;; + --target=*) + toolchain="${toolchain:-${optval}}" + ;; + --force-target=*) + toolchain="${toolchain:-${optval}}" + enable_feature force_toolchain + ;; + --cpu=*) + tune_cpu="$optval" + ;; + --extra-cflags=*) + extra_cflags="${optval}" + ;; + --extra-cxxflags=*) + extra_cxxflags="${optval}" + ;; + --enable-?*|--disable-?*) + eval `echo "$opt" | sed 's/--/action=/;s/-/ option=/;s/-/_/g'` + if is_in ${option} ${ARCH_EXT_LIST}; then + [ $action = "disable" ] && RTCD_OPTIONS="${RTCD_OPTIONS}--disable-${option} " + elif [ $action = "disable" ] && ! disabled $option ; then + is_in ${option} ${CMDLINE_SELECT} || die_unknown $opt + log_echo " disabling $option" + elif [ $action = "enable" ] && ! enabled $option ; then + is_in ${option} ${CMDLINE_SELECT} || die_unknown $opt + log_echo " enabling $option" + fi + ${action}_feature $option + ;; + --require-?*) + eval `echo "$opt" | sed 's/--/action=/;s/-/ option=/;s/-/_/g'` + if is_in ${option} ${ARCH_EXT_LIST}; then + RTCD_OPTIONS="${RTCD_OPTIONS}${opt} " + else + die_unknown $opt + fi + ;; + --force-enable-?*|--force-disable-?*) + eval `echo "$opt" | sed 's/--force-/action=/;s/-/ option=/;s/-/_/g'` + ${action}_feature $option + ;; + --libc=*) + [ -d "${optval}" ] || die "Not a directory: ${optval}" + disable_feature builtin_libc + alt_libc="${optval}" + ;; + --as=*) + [ "${optval}" = yasm ] || [ "${optval}" = nasm ] \ + || [ "${optval}" = auto ] \ + || die "Must be yasm, nasm or auto: ${optval}" + alt_as="${optval}" + ;; + --size-limit=*) + w="${optval%%x*}" + h="${optval##*x}" + VAR_LIST="DECODE_WIDTH_LIMIT ${w} DECODE_HEIGHT_LIMIT ${h}" + [ ${w} -gt 0 ] && [ ${h} -gt 0 ] || die "Invalid size-limit: too small." + [ ${w} -lt 65536 ] && [ ${h} -lt 65536 ] \ + || die "Invalid size-limit: too big." + enable_feature size_limit + ;; + --prefix=*) + prefix="${optval}" + ;; + --libdir=*) + libdir="${optval}" + ;; + --sdk-path=*) + [ -d "${optval}" ] || die "Not a directory: ${optval}" + sdk_path="${optval}" + ;; + --libc|--as|--prefix|--libdir|--sdk-path) + die "Option ${opt} requires argument" + ;; + --help|-h) + show_help + ;; + *) + die_unknown $opt + ;; + esac + done +} + +process_cmdline() { + for opt do + optval="${opt#*=}" + case "$opt" in + *) + process_common_cmdline $opt + ;; + esac + done +} + +post_process_common_cmdline() { + prefix="${prefix:-/usr/local}" + prefix="${prefix%/}" + libdir="${libdir:-${prefix}/lib}" + libdir="${libdir%/}" + if [ "${libdir#${prefix}}" = "${libdir}" ]; then + die "Libdir ${libdir} must be a subdirectory of ${prefix}" + fi +} + +post_process_cmdline() { + true; +} + +setup_gnu_toolchain() { + CC=${CC:-${CROSS}gcc} + CXX=${CXX:-${CROSS}g++} + AR=${AR:-${CROSS}ar} + LD=${LD:-${CROSS}${link_with_cc:-ld}} + AS=${AS:-${CROSS}as} + STRIP=${STRIP:-${CROSS}strip} + NM=${NM:-${CROSS}nm} + AS_SFX=.s + EXE_SFX= +} + +# Reliably find the newest available Darwin SDKs. (Older versions of +# xcrun don't support --show-sdk-path.) +show_darwin_sdk_path() { + xcrun --sdk $1 --show-sdk-path 2>/dev/null || + xcodebuild -sdk $1 -version Path 2>/dev/null +} + +# Print the major version number of the Darwin SDK specified by $1. +show_darwin_sdk_major_version() { + xcrun --sdk $1 --show-sdk-version 2>/dev/null | cut -d. -f1 +} + +# Print the Xcode version. +show_xcode_version() { + xcodebuild -version | head -n1 | cut -d' ' -f2 +} + +# Fails when Xcode version is less than 6.3. +check_xcode_minimum_version() { + xcode_major=$(show_xcode_version | cut -f1 -d.) + xcode_minor=$(show_xcode_version | cut -f2 -d.) + xcode_min_major=6 + xcode_min_minor=3 + if [ ${xcode_major} -lt ${xcode_min_major} ]; then + return 1 + fi + if [ ${xcode_major} -eq ${xcode_min_major} ] \ + && [ ${xcode_minor} -lt ${xcode_min_minor} ]; then + return 1 + fi +} + +process_common_toolchain() { + case "$toolchain" in + *-vs*) ;; + *) add_cflags_only -std=c99 ;; + esac + + if [ -z "$toolchain" ]; then + gcctarget="${CHOST:-$(gcc -dumpmachine 2> /dev/null)}" + + # detect tgt_isa + case "$gcctarget" in + aarch64*) + tgt_isa=arm64 + ;; + armv6*) + tgt_isa=armv6 + ;; + armv7*-hardfloat* | armv7*-gnueabihf | arm-*-gnueabihf) + tgt_isa=armv7 + float_abi=hard + ;; + armv7*) + tgt_isa=armv7 + float_abi=softfp + ;; + *x86_64*|*amd64*) + tgt_isa=x86_64 + ;; + *i[3456]86*) + tgt_isa=x86 + ;; + *sparc*) + tgt_isa=sparc + ;; + esac + + # detect tgt_os + case "$gcctarget" in + *darwin10*) + tgt_isa=x86_64 + tgt_os=darwin10 + ;; + *darwin11*) + tgt_isa=x86_64 + tgt_os=darwin11 + ;; + *darwin12*) + tgt_isa=x86_64 + tgt_os=darwin12 + ;; + *darwin13*) + tgt_isa=x86_64 + tgt_os=darwin13 + ;; + *darwin14*) + tgt_isa=x86_64 + tgt_os=darwin14 + ;; + *darwin15*) + tgt_isa=x86_64 + tgt_os=darwin15 + ;; + *darwin16*) + tgt_isa=x86_64 + tgt_os=darwin16 + ;; + x86_64*mingw32*) + tgt_os=win64 + ;; + *mingw32*|*cygwin*) + [ -z "$tgt_isa" ] && tgt_isa=x86 + tgt_os=win32 + ;; + *linux*|*bsd*) + tgt_os=linux + ;; + *solaris2.10) + tgt_os=solaris + ;; + *os2*) + tgt_os=os2 + ;; + esac + + if [ -n "$tgt_isa" ] && [ -n "$tgt_os" ]; then + toolchain=${tgt_isa}-${tgt_os}-gcc + fi + fi + + toolchain=${toolchain:-generic-gnu} + + is_in ${toolchain} ${all_platforms} || enabled force_toolchain \ + || die "Unrecognized toolchain '${toolchain}'" + + enabled child || log_echo "Configuring for target '${toolchain}'" + + # + # Set up toolchain variables + # + tgt_isa=$(echo ${toolchain} | awk 'BEGIN{FS="-"}{print $1}') + tgt_os=$(echo ${toolchain} | awk 'BEGIN{FS="-"}{print $2}') + tgt_cc=$(echo ${toolchain} | awk 'BEGIN{FS="-"}{print $3}') + + # Mark the specific ISA requested as enabled + soft_enable ${tgt_isa} + enable_feature ${tgt_os} + enable_feature ${tgt_cc} + + # Enable the architecture family + case ${tgt_isa} in + arm*) + enable_feature arm + ;; + mips*) + enable_feature mips + ;; + esac + + # PIC is probably what we want when building shared libs + enabled shared && soft_enable pic + + # Minimum iOS version for all target platforms (darwin and iphonesimulator). + # Shared library framework builds are only possible on iOS 8 and later. + if enabled shared; then + IOS_VERSION_OPTIONS="--enable-shared" + IOS_VERSION_MIN="8.0" + else + IOS_VERSION_OPTIONS="" + IOS_VERSION_MIN="6.0" + fi + + # Handle darwin variants. Newer SDKs allow targeting older + # platforms, so use the newest one available. + case ${toolchain} in + arm*-darwin*) + add_cflags "-miphoneos-version-min=${IOS_VERSION_MIN}" + iphoneos_sdk_dir="$(show_darwin_sdk_path iphoneos)" + if [ -d "${iphoneos_sdk_dir}" ]; then + add_cflags "-isysroot ${iphoneos_sdk_dir}" + add_ldflags "-isysroot ${iphoneos_sdk_dir}" + fi + ;; + x86*-darwin*) + osx_sdk_dir="$(show_darwin_sdk_path macosx)" + if [ -d "${osx_sdk_dir}" ]; then + add_cflags "-isysroot ${osx_sdk_dir}" + add_ldflags "-isysroot ${osx_sdk_dir}" + fi + ;; + esac + + case ${toolchain} in + *-darwin8-*) + add_cflags "-mmacosx-version-min=10.4" + add_ldflags "-mmacosx-version-min=10.4" + ;; + *-darwin9-*) + add_cflags "-mmacosx-version-min=10.5" + add_ldflags "-mmacosx-version-min=10.5" + ;; + *-darwin10-*) + add_cflags "-mmacosx-version-min=10.6" + add_ldflags "-mmacosx-version-min=10.6" + ;; + *-darwin11-*) + add_cflags "-mmacosx-version-min=10.7" + add_ldflags "-mmacosx-version-min=10.7" + ;; + *-darwin12-*) + add_cflags "-mmacosx-version-min=10.8" + add_ldflags "-mmacosx-version-min=10.8" + ;; + *-darwin13-*) + add_cflags "-mmacosx-version-min=10.9" + add_ldflags "-mmacosx-version-min=10.9" + ;; + *-darwin14-*) + add_cflags "-mmacosx-version-min=10.10" + add_ldflags "-mmacosx-version-min=10.10" + ;; + *-darwin15-*) + add_cflags "-mmacosx-version-min=10.11" + add_ldflags "-mmacosx-version-min=10.11" + ;; + *-darwin16-*) + add_cflags "-mmacosx-version-min=10.12" + add_ldflags "-mmacosx-version-min=10.12" + ;; + *-iphonesimulator-*) + add_cflags "-miphoneos-version-min=${IOS_VERSION_MIN}" + add_ldflags "-miphoneos-version-min=${IOS_VERSION_MIN}" + iossim_sdk_dir="$(show_darwin_sdk_path iphonesimulator)" + if [ -d "${iossim_sdk_dir}" ]; then + add_cflags "-isysroot ${iossim_sdk_dir}" + add_ldflags "-isysroot ${iossim_sdk_dir}" + fi + ;; + esac + + # Handle Solaris variants. Solaris 10 needs -lposix4 + case ${toolchain} in + sparc-solaris-*) + add_extralibs -lposix4 + ;; + *-solaris-*) + add_extralibs -lposix4 + ;; + esac + + # Process ARM architecture variants + case ${toolchain} in + arm*) + # on arm, isa versions are supersets + case ${tgt_isa} in + arm64|armv8) + soft_enable neon + ;; + armv7|armv7s) + soft_enable neon + # Only enable neon_asm when neon is also enabled. + enabled neon && soft_enable neon_asm + # If someone tries to force it through, die. + if disabled neon && enabled neon_asm; then + die "Disabling neon while keeping neon-asm is not supported" + fi + case ${toolchain} in + # Apple iOS SDKs no longer support armv6 as of the version 9 + # release (coincides with release of Xcode 7). Only enable media + # when using earlier SDK releases. + *-darwin*) + if [ "$(show_darwin_sdk_major_version iphoneos)" -lt 9 ]; then + soft_enable media + else + soft_disable media + RTCD_OPTIONS="${RTCD_OPTIONS}--disable-media " + fi + ;; + *) + soft_enable media + ;; + esac + ;; + armv6) + case ${toolchain} in + *-darwin*) + if [ "$(show_darwin_sdk_major_version iphoneos)" -lt 9 ]; then + soft_enable media + else + die "Your iOS SDK does not support armv6." + fi + ;; + *) + soft_enable media + ;; + esac + ;; + esac + + asm_conversion_cmd="cat" + + case ${tgt_cc} in + gcc) + link_with_cc=gcc + setup_gnu_toolchain + arch_int=${tgt_isa##armv} + arch_int=${arch_int%%te} + check_add_asflags --defsym ARCHITECTURE=${arch_int} + tune_cflags="-mtune=" + if [ ${tgt_isa} = "armv7" ] || [ ${tgt_isa} = "armv7s" ]; then + if [ -z "${float_abi}" ]; then + check_cpp <&- || \ + die "Couldn't find CodeSourcery GCC from PATH" + + # Use armcc as a linker to enable translation of + # some gcc specific options such as -lm and -lpthread. + LD="armcc --translate_gcc" + + # create configuration file (uses path to CodeSourcery GCC) + armcc --arm_linux_configure --arm_linux_config_file=arm_linux.cfg + + add_cflags --arm_linux_paths --arm_linux_config_file=arm_linux.cfg + add_asflags --no_hide_all --apcs=/interwork + add_ldflags --arm_linux_paths --arm_linux_config_file=arm_linux.cfg + enabled pic && add_cflags --apcs=/fpic + enabled pic && add_asflags --apcs=/fpic + enabled shared && add_cflags --shared + fi + ;; + esac + ;; + mips*) + link_with_cc=gcc + setup_gnu_toolchain + tune_cflags="-mtune=" + if enabled dspr2; then + check_add_cflags -mips32r2 -mdspr2 + fi + + if enabled runtime_cpu_detect; then + disable_feature runtime_cpu_detect + fi + + if [ -n "${tune_cpu}" ]; then + case ${tune_cpu} in + p5600) + check_add_cflags -mips32r5 -mload-store-pairs + check_add_cflags -msched-weight -mhard-float -mfp64 + check_add_asflags -mips32r5 -mhard-float -mfp64 + check_add_ldflags -mfp64 + ;; + i6400|p6600) + check_add_cflags -mips64r6 -mabi=64 -msched-weight + check_add_cflags -mload-store-pairs -mhard-float -mfp64 + check_add_asflags -mips64r6 -mabi=64 -mhard-float -mfp64 + check_add_ldflags -mips64r6 -mabi=64 -mfp64 + ;; + esac + + if enabled msa; then + add_cflags -mmsa + add_asflags -mmsa + add_ldflags -mmsa + fi + fi + + check_add_cflags -march=${tgt_isa} + check_add_asflags -march=${tgt_isa} + check_add_asflags -KPIC + ;; + x86*) + case ${tgt_os} in + win*) + enabled gcc && add_cflags -fno-common + ;; + solaris*) + CC=${CC:-${CROSS}gcc} + CXX=${CXX:-${CROSS}g++} + LD=${LD:-${CROSS}gcc} + CROSS=${CROSS-g} + ;; + os2) + disable_feature pic + AS=${AS:-nasm} + add_ldflags -Zhigh-mem + ;; + esac + + AS="${alt_as:-${AS:-auto}}" + case ${tgt_cc} in + icc*) + CC=${CC:-icc} + LD=${LD:-icc} + setup_gnu_toolchain + add_cflags -use-msasm # remove -use-msasm too? + # add -no-intel-extensions to suppress warning #10237 + # refer to http://software.intel.com/en-us/forums/topic/280199 + add_ldflags -i-static -no-intel-extensions + enabled x86_64 && add_cflags -ipo -static -O3 -no-prec-div + enabled x86_64 && AR=xiar + case ${tune_cpu} in + atom*) + tune_cflags="-x" + tune_cpu="SSE3_ATOM" + ;; + *) + tune_cflags="-march=" + ;; + esac + ;; + gcc*) + link_with_cc=gcc + tune_cflags="-march=" + setup_gnu_toolchain + #for 32 bit x86 builds, -O3 did not turn on this flag + enabled optimizations && disabled gprof && check_add_cflags -fomit-frame-pointer + ;; + vs*) + # When building with Microsoft Visual Studio the assembler is + # invoked directly. Checking at configure time is unnecessary. + # Skip the check by setting AS arbitrarily + AS=msvs + msvs_arch_dir=x86-msvs + vc_version=${tgt_cc##vs} + ;; + esac + + bits=32 + enabled x86_64 && bits=64 + check_cpp < sse4 + check_gcc_machine_option ${ext%_*} $ext + fi + done + + if enabled external_build; then + log_echo " skipping assembler detection" + else + case "${AS}" in + auto|"") + which nasm >/dev/null 2>&1 && AS=nasm + which yasm >/dev/null 2>&1 && AS=yasm + if [ "${AS}" = nasm ] ; then + # Apple ships version 0.98 of nasm through at least Xcode 6. Revisit + # this check if they start shipping a compatible version. + apple=`nasm -v | grep "Apple"` + [ -n "${apple}" ] \ + && echo "Unsupported version of nasm: ${apple}" \ + && AS="" + fi + [ "${AS}" = auto ] || [ -z "${AS}" ] \ + && die "Neither yasm nor nasm have been found." \ + "See the prerequisites section in the README for more info." + ;; + esac + log_echo " using $AS" + fi + [ "${AS##*/}" = nasm ] && add_asflags -Ox + AS_SFX=.asm + case ${tgt_os} in + win32) + add_asflags -f win32 + enabled debug && add_asflags -g cv8 + EXE_SFX=.exe + ;; + win64) + add_asflags -f x64 + enabled debug && add_asflags -g cv8 + EXE_SFX=.exe + ;; + linux*|solaris*|android*) + add_asflags -f elf${bits} + enabled debug && [ "${AS}" = yasm ] && add_asflags -g dwarf2 + enabled debug && [ "${AS}" = nasm ] && add_asflags -g + [ "${AS##*/}" = nasm ] && check_asm_align + ;; + darwin*) + add_asflags -f macho${bits} + enabled x86 && darwin_arch="-arch i386" || darwin_arch="-arch x86_64" + add_cflags ${darwin_arch} + add_ldflags ${darwin_arch} + # -mdynamic-no-pic is still a bit of voodoo -- it was required at + # one time, but does not seem to be now, and it breaks some of the + # code that still relies on inline assembly. + # enabled icc && ! enabled pic && add_cflags -fno-pic -mdynamic-no-pic + enabled icc && ! enabled pic && add_cflags -fno-pic + ;; + iphonesimulator) + add_asflags -f macho${bits} + enabled x86 && sim_arch="-arch i386" || sim_arch="-arch x86_64" + add_cflags ${sim_arch} + add_ldflags ${sim_arch} + + if [ "$(show_darwin_sdk_major_version iphonesimulator)" -gt 8 ]; then + # yasm v1.3.0 doesn't know what -fembed-bitcode means, so turning it + # on is pointless (unless building a C-only lib). Warn the user, but + # do nothing here. + log "Warning: Bitcode embed disabled for simulator targets." + fi + ;; + os2) + add_asflags -f aout + enabled debug && add_asflags -g + EXE_SFX=.exe + ;; + *) + log "Warning: Unknown os $tgt_os while setting up $AS flags" + ;; + esac + ;; + *-gcc|generic-gnu) + link_with_cc=gcc + enable_feature gcc + setup_gnu_toolchain + ;; + esac + + # Try to enable CPU specific tuning + if [ -n "${tune_cpu}" ]; then + if [ -n "${tune_cflags}" ]; then + check_add_cflags ${tune_cflags}${tune_cpu} || \ + die "Requested CPU '${tune_cpu}' not supported by compiler" + fi + if [ -n "${tune_asflags}" ]; then + check_add_asflags ${tune_asflags}${tune_cpu} || \ + die "Requested CPU '${tune_cpu}' not supported by assembler" + fi + if [ -z "${tune_cflags}${tune_asflags}" ]; then + log_echo "Warning: CPU tuning not supported by this toolchain" + fi + fi + + if enabled debug; then + check_add_cflags -g && check_add_ldflags -g + else + check_add_cflags -DNDEBUG + fi + + enabled gprof && check_add_cflags -pg && check_add_ldflags -pg + enabled gcov && + check_add_cflags -fprofile-arcs -ftest-coverage && + check_add_ldflags -fprofile-arcs -ftest-coverage + + if enabled optimizations; then + if enabled rvct; then + enabled small && check_add_cflags -Ospace || check_add_cflags -Otime + else + enabled small && check_add_cflags -O2 || check_add_cflags -O3 + fi + fi + + # Position Independent Code (PIC) support, for building relocatable + # shared objects + enabled gcc && enabled pic && check_add_cflags -fPIC + + # Work around longjmp interception on glibc >= 2.11, to improve binary + # compatibility. See http://code.google.com/p/webm/issues/detail?id=166 + enabled linux && check_add_cflags -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0 + + # Check for strip utility variant + ${STRIP} -V 2>/dev/null | grep GNU >/dev/null && enable_feature gnu_strip + + # Try to determine target endianness + check_cc </dev/null 2>&1 && enable_feature big_endian + + # Try to find which inline keywords are supported + check_cc <> $makefile + fi + done + prefix="${saved_prefix}" +} + +print_config_h() { + saved_prefix="${prefix}" + prefix=$1 + header=$2 + shift 2 + for cfg; do + upname="`toupper $cfg`" + if enabled $cfg; then + echo "#define ${prefix}_${upname} 1" >> $header + else + echo "#define ${prefix}_${upname} 0" >> $header + fi + done + prefix="${saved_prefix}" +} + +print_config_vars_h() { + header=$1 + shift + while [ $# -gt 0 ]; do + upname="`toupper $1`" + echo "#define ${upname} $2" >> $header + shift 2 + done +} + +print_webm_license() { + saved_prefix="${prefix}" + destination=$1 + prefix="$2" + suffix="$3" + shift 3 + cat < ${destination} +${prefix} Copyright (c) 2016, Alliance for Open Media. All rights reserved.${suffix} +${prefix} ${suffix} +${prefix} This source code is subject to the terms of the BSD 2 Clause License and${suffix} +${prefix} the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License${suffix} +${prefix} was not distributed with this source code in the LICENSE file, you can${suffix} +${prefix} obtain it at www.aomedia.org/license/software. If the Alliance for Open${suffix} +${prefix} Media Patent License 1.0 was not distributed with this source code in the${suffix} +${prefix} PATENTS file, you can obtain it at www.aomedia.org/license/patent.${suffix} +EOF + prefix="${saved_prefix}" +} + +process_targets() { + true; +} + +process_detect() { + true; +} + +enable_feature logging +logfile="config.log" +self=$0 +process() { + cmdline_args="$@" + process_cmdline "$@" + if enabled child; then + echo "# ${self} $@" >> ${logfile} + else + echo "# ${self} $@" > ${logfile} + fi + post_process_common_cmdline + post_process_cmdline + process_toolchain + process_detect + process_targets + + OOT_INSTALLS="${OOT_INSTALLS}" + if enabled source_path_used; then + # Prepare the PWD for building. + for f in ${OOT_INSTALLS}; do + install -D "${source_path}/$f" "$f" + done + fi + cp "${source_path}/build/make/Makefile" . + + clean_temp_files + true +} diff --git a/third_party/aom/build/make/gen_asm_deps.sh b/third_party/aom/build/make/gen_asm_deps.sh new file mode 100755 index 0000000000..c867cc2bf5 --- /dev/null +++ b/third_party/aom/build/make/gen_asm_deps.sh @@ -0,0 +1,65 @@ +#!/bin/sh +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + +self=$0 +show_help() { + echo "usage: $self [options] " + echo + echo "Generate Makefile dependency information from assembly code source" + echo + exit 1 +} +die_unknown(){ + echo "Unknown option \"$1\"." + echo "See $0 --help for available options." + exit 1 +} +for opt do + optval="${opt#*=}" + case "$opt" in + --build-pfx=*) pfx="${optval}" + ;; + --depfile=*) out="${optval}" + ;; + -I*) raw_inc_paths="${raw_inc_paths} ${opt}" + inc_path="${inc_path} ${opt#-I}" + ;; + -h|--help) show_help + ;; + *) [ -f "$opt" ] && srcfile="$opt" + ;; + esac +done + +[ -n "$srcfile" ] || show_help +sfx=${sfx:-asm} +includes=$(LC_ALL=C egrep -i "include +\"?[a-z0-9_/]+\.${sfx}" $srcfile | + perl -p -e "s;.*?([a-z0-9_/]+.${sfx}).*;\1;") +#" restore editor state +for inc in ${includes}; do + found_inc_path= + for idir in ${inc_path}; do + [ -f "${idir}/${inc}" ] && found_inc_path="${idir}" && break + done + if [ -f `dirname $srcfile`/$inc ]; then + # Handle include files in the same directory as the source + $self --build-pfx=$pfx --depfile=$out ${raw_inc_paths} `dirname $srcfile`/$inc + elif [ -n "${found_inc_path}" ]; then + # Handle include files on the include path + $self --build-pfx=$pfx --depfile=$out ${raw_inc_paths} "${found_inc_path}/$inc" + else + # Handle generated includes in the build root (which may not exist yet) + echo ${out} ${out%d}o: "${pfx}${inc}" + fi +done +echo ${out} ${out%d}o: $srcfile diff --git a/third_party/aom/build/make/gen_msvs_def.sh b/third_party/aom/build/make/gen_msvs_def.sh new file mode 100755 index 0000000000..dbb2674ac5 --- /dev/null +++ b/third_party/aom/build/make/gen_msvs_def.sh @@ -0,0 +1,82 @@ +#!/bin/bash +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +self=$0 +self_basename=${self##*/} +EOL=$'\n' + +show_help() { + cat < symbol1 [symbol2, symbol3, ...] + +where is either 'text' or 'data' + + +Options: + --help Print this message + --out=filename Write output to a file [stdout] + --name=project_name Name of the library (required) +EOF + exit 1 +} + +die() { + echo "${self_basename}: $@" + exit 1 +} + +die_unknown(){ + echo "Unknown option \"$1\"." + echo "See ${self_basename} --help for available options." + exit 1 +} + +text() { + for sym in "$@"; do + echo " $sym" >> ${outfile} + done +} + +data() { + for sym in "$@"; do + printf " %-40s DATA\n" "$sym" >> ${outfile} + done +} + +# Process command line +for opt in "$@"; do + optval="${opt#*=}" + case "$opt" in + --help|-h) show_help + ;; + --out=*) outfile="$optval" + ;; + --name=*) name="${optval}" + ;; + -*) die_unknown $opt + ;; + *) file_list[${#file_list[@]}]="$opt" + esac +done +outfile=${outfile:-/dev/stdout} +[ -n "$name" ] || die "Library name (--name) must be specified!" + +echo "LIBRARY ${name}" > ${outfile} +echo "EXPORTS" >> ${outfile} +for f in "${file_list[@]}"; do + . $f +done diff --git a/third_party/aom/build/make/gen_msvs_sln.sh b/third_party/aom/build/make/gen_msvs_sln.sh new file mode 100755 index 0000000000..77c68f1da2 --- /dev/null +++ b/third_party/aom/build/make/gen_msvs_sln.sh @@ -0,0 +1,254 @@ +#!/bin/bash +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + +self=$0 +self_basename=${self##*/} +EOL=$'\n' +EOLDOS=$'\r' + +show_help() { + cat <&2 + [ -f "${outfile}" ] && rm -f ${outfile}{,.mk} + exit 1 +} + +die_unknown(){ + echo "Unknown option \"$1\"." >&2 + echo "See ${self_basename} --help for available options." >&2 + [ -f "${outfile}" ] && rm -f ${outfile}{,.mk} + exit 1 +} + +indent1=$'\t' +indent="" +indent_push() { + indent="${indent}${indent1}" +} +indent_pop() { + indent="${indent%${indent1}}" +} + +parse_project() { + local file=$1 + local name=`grep RootNamespace "$file" | sed 's,.*<.*>\(.*\).*,\1,'` + local guid=`grep ProjectGuid "$file" | sed 's,.*<.*>\(.*\).*,\1,'` + + # save the project GUID to a varaible, normalizing to the basename of the + # vcxproj file without the extension + local var + var=${file##*/} + var=${var%%.${sfx}} + eval "${var}_file=\"$1\"" + eval "${var}_name=$name" + eval "${var}_guid=$guid" + + cur_config_list=`grep -B1 'Label="Configuration"' $file | + grep Condition | cut -d\' -f4` + new_config_list=$(for i in $config_list $cur_config_list; do + echo $i + done | sort | uniq) + if [ "$config_list" != "" ] && [ "$config_list" != "$new_config_list" ]; then + mixed_platforms=1 + fi + config_list="$new_config_list" + eval "${var}_config_list=\"$cur_config_list\"" + proj_list="${proj_list} ${var}" +} + +process_project() { + eval "local file=\${$1_file}" + eval "local name=\${$1_name}" + eval "local guid=\${$1_guid}" + + # save the project GUID to a varaible, normalizing to the basename of the + # vcproj file without the extension + local var + var=${file##*/} + var=${var%%.${sfx}} + eval "${var}_guid=$guid" + + echo "Project(\"{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}\") = \"$name\", \"$file\", \"$guid\"" + echo "EndProject" +} + +process_global() { + echo "Global" + indent_push + + # + # Solution Configuration Platforms + # + echo "${indent}GlobalSection(SolutionConfigurationPlatforms) = preSolution" + indent_push + IFS_bak=${IFS} + IFS=$'\r'$'\n' + if [ "$mixed_platforms" != "" ]; then + config_list=" +Release|Mixed Platforms +Debug|Mixed Platforms" + fi + for config in ${config_list}; do + echo "${indent}$config = $config" + done + IFS=${IFS_bak} + indent_pop + echo "${indent}EndGlobalSection" + + # + # Project Configuration Platforms + # + echo "${indent}GlobalSection(ProjectConfigurationPlatforms) = postSolution" + indent_push + for proj in ${proj_list}; do + eval "local proj_guid=\${${proj}_guid}" + eval "local proj_config_list=\${${proj}_config_list}" + IFS=$'\r'$'\n' + for config in ${proj_config_list}; do + if [ "$mixed_platforms" != "" ]; then + local c=${config%%|*} + echo "${indent}${proj_guid}.${c}|Mixed Platforms.ActiveCfg = ${config}" + echo "${indent}${proj_guid}.${c}|Mixed Platforms.Build.0 = ${config}" + else + echo "${indent}${proj_guid}.${config}.ActiveCfg = ${config}" + echo "${indent}${proj_guid}.${config}.Build.0 = ${config}" + fi + + done + IFS=${IFS_bak} + done + indent_pop + echo "${indent}EndGlobalSection" + + # + # Solution Properties + # + echo "${indent}GlobalSection(SolutionProperties) = preSolution" + indent_push + echo "${indent}HideSolutionNode = FALSE" + indent_pop + echo "${indent}EndGlobalSection" + + indent_pop + echo "EndGlobal" +} + +process_makefile() { + IFS_bak=${IFS} + IFS=$'\r'$'\n' + local TAB=$'\t' + cat </dev/null 2>&1 && echo yes) +.nodevenv.once: +${TAB}@echo " * \$(MSBUILD_TOOL) not found in path." +${TAB}@echo " * " +${TAB}@echo " * You will have to build all configurations manually using the" +${TAB}@echo " * Visual Studio IDE. To allow make to build them automatically," +${TAB}@echo " * add the Common7/IDE directory of your Visual Studio" +${TAB}@echo " * installation to your path, eg:" +${TAB}@echo " * C:\Program Files\Microsoft Visual Studio 12.0\Common7\IDE" +${TAB}@echo " * " +${TAB}@touch \$@ +CLEAN-OBJS += \$(if \$(found_devenv),,.nodevenv.once) + +EOF + + for sln_config in ${config_list}; do + local config=${sln_config%%|*} + local platform=${sln_config##*|} + local nows_sln_config=`echo $sln_config | sed -e 's/[^a-zA-Z0-9]/_/g'` + cat <${outfile} <>${outfile} +done +process_global >>${outfile} +process_makefile >${mkoutfile} diff --git a/third_party/aom/build/make/gen_msvs_vcxproj.sh b/third_party/aom/build/make/gen_msvs_vcxproj.sh new file mode 100755 index 0000000000..7543bda172 --- /dev/null +++ b/third_party/aom/build/make/gen_msvs_vcxproj.sh @@ -0,0 +1,474 @@ +#!/bin/bash +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +self=$0 +self_basename=${self##*/} +self_dirname=$(dirname "$0") + +. "$self_dirname/msvs_common.sh"|| exit 127 + +show_help() { + cat <${content}" + indent_pop + else + echo "${indent}<${tag}>${content}" + fi +} + +generate_filter() { + local name=$1 + local pats=$2 + local file_list_sz + local i + local f + local saveIFS="$IFS" + local pack + echo "generating filter '$name' from ${#file_list[@]} files" >&2 + IFS=* + + file_list_sz=${#file_list[@]} + for i in ${!file_list[@]}; do + f=${file_list[i]} + for pat in ${pats//;/$IFS}; do + if [ "${f##*.}" == "$pat" ]; then + unset file_list[i] + + objf=$(echo ${f%.*}.obj \ + | sed -e "s,$src_path_bare,," \ + -e 's/^[\./]\+//g' -e 's,[:/ ],_,g') + + if ([ "$pat" == "asm" ] || [ "$pat" == "s" ]) && $asm_use_custom_step; then + # Avoid object file name collisions, i.e. aom_config.c and + # aom_config.asm produce the same object file without + # this additional suffix. + objf=${objf%.obj}_asm.obj + open_tag CustomBuild \ + Include="$f" + for plat in "${platforms[@]}"; do + for cfg in Debug Release; do + tag_content Message "Assembling %(Filename)%(Extension)" \ + Condition="'\$(Configuration)|\$(Platform)'=='$cfg|$plat'" + tag_content Command "$(eval echo \$asm_${cfg}_cmdline) -o \$(IntDir)$objf" \ + Condition="'\$(Configuration)|\$(Platform)'=='$cfg|$plat'" + tag_content Outputs "\$(IntDir)$objf" \ + Condition="'\$(Configuration)|\$(Platform)'=='$cfg|$plat'" + done + done + close_tag CustomBuild + elif [ "$pat" == "c" ] || \ + [ "$pat" == "cc" ] || [ "$pat" == "cpp" ]; then + open_tag ClCompile \ + Include="$f" + # Separate file names with Condition? + tag_content ObjectFileName "\$(IntDir)$objf" + # Check for AVX and turn it on to avoid warnings. + if [[ $f =~ avx.?\.c$ ]]; then + tag_content AdditionalOptions "/arch:AVX" + fi + close_tag ClCompile + elif [ "$pat" == "h" ] ; then + tag ClInclude \ + Include="$f" + elif [ "$pat" == "vcxproj" ] ; then + open_tag ProjectReference \ + Include="$f" + depguid=`grep ProjectGuid "$f" | sed 's,.*<.*>\(.*\).*,\1,'` + tag_content Project "$depguid" + tag_content ReferenceOutputAssembly false + close_tag ProjectReference + else + tag None \ + Include="$f" + fi + + break + fi + done + done + + IFS="$saveIFS" +} + +# Process command line +unset target +for opt in "$@"; do + optval="${opt#*=}" + case "$opt" in + --help|-h) show_help + ;; + --target=*) target="${optval}" + ;; + --out=*) outfile="$optval" + ;; + --name=*) name="${optval}" + ;; + --proj-guid=*) guid="${optval}" + ;; + --module-def=*) module_def="${optval}" + ;; + --exe) proj_kind="exe" + ;; + --dll) proj_kind="dll" + ;; + --lib) proj_kind="lib" + ;; + --src-path-bare=*) + src_path_bare=$(fix_path "$optval") + src_path_bare=${src_path_bare%/} + ;; + --static-crt) use_static_runtime=true + ;; + --enable-werror) werror=true + ;; + --ver=*) + vs_ver="$optval" + case "$optval" in + 12|14) + ;; + *) die Unrecognized Visual Studio Version in $opt + ;; + esac + ;; + -I*) + opt=${opt##-I} + opt=$(fix_path "$opt") + opt="${opt%/}" + incs="${incs}${incs:+;}"${opt}"" + yasmincs="${yasmincs} -I"${opt}"" + ;; + -D*) defines="${defines}${defines:+;}${opt##-D}" + ;; + -L*) # fudge . to $(OutDir) + if [ "${opt##-L}" == "." ]; then + libdirs="${libdirs}${libdirs:+;}"\$(OutDir)"" + else + # Also try directories for this platform/configuration + opt=${opt##-L} + opt=$(fix_path "$opt") + libdirs="${libdirs}${libdirs:+;}"${opt}"" + libdirs="${libdirs}${libdirs:+;}"${opt}/\$(PlatformName)/\$(Configuration)"" + libdirs="${libdirs}${libdirs:+;}"${opt}/\$(PlatformName)"" + fi + ;; + -l*) libs="${libs}${libs:+ }${opt##-l}.lib" + ;; + -*) die_unknown $opt + ;; + *) + # The paths in file_list are fixed outside of the loop. + file_list[${#file_list[@]}]="$opt" + case "$opt" in + *.asm|*.s) uses_asm=true + ;; + esac + ;; + esac +done + +# Make one call to fix_path for file_list to improve performance. +fix_file_list file_list + +outfile=${outfile:-/dev/stdout} +guid=${guid:-`generate_uuid`} +asm_use_custom_step=false +uses_asm=${uses_asm:-false} +case "${vs_ver:-12}" in + 12|14) + asm_use_custom_step=$uses_asm + ;; +esac + +[ -n "$name" ] || die "Project name (--name) must be specified!" +[ -n "$target" ] || die "Target (--target) must be specified!" + +if ${use_static_runtime:-false}; then + release_runtime=MultiThreaded + debug_runtime=MultiThreadedDebug + lib_sfx=mt +else + release_runtime=MultiThreadedDLL + debug_runtime=MultiThreadedDebugDLL + lib_sfx=md +fi + +# Calculate debug lib names: If a lib ends in ${lib_sfx}.lib, then rename +# it to ${lib_sfx}d.lib. This precludes linking to release libs from a +# debug exe, so this may need to be refactored later. +for lib in ${libs}; do + if [ "$lib" != "${lib%${lib_sfx}.lib}" ]; then + lib=${lib%.lib}d.lib + fi + debug_libs="${debug_libs}${debug_libs:+ }${lib}" +done +debug_libs=${debug_libs// /;} +libs=${libs// /;} + + +# List of all platforms supported for this target +case "$target" in + x86_64*) + platforms[0]="x64" + asm_Debug_cmdline="yasm -Xvc -g cv8 -f win64 ${yasmincs} "%(FullPath)"" + asm_Release_cmdline="yasm -Xvc -f win64 ${yasmincs} "%(FullPath)"" + ;; + x86*) + platforms[0]="Win32" + asm_Debug_cmdline="yasm -Xvc -g cv8 -f win32 ${yasmincs} "%(FullPath)"" + asm_Release_cmdline="yasm -Xvc -f win32 ${yasmincs} "%(FullPath)"" + ;; + arm*) + platforms[0]="ARM" + asm_Debug_cmdline="armasm -nologo -oldit "%(FullPath)"" + asm_Release_cmdline="armasm -nologo -oldit "%(FullPath)"" + ;; + *) die "Unsupported target $target!" + ;; +esac + +generate_vcxproj() { + echo "" + open_tag Project \ + DefaultTargets="Build" \ + ToolsVersion="4.0" \ + xmlns="http://schemas.microsoft.com/developer/msbuild/2003" \ + + open_tag ItemGroup \ + Label="ProjectConfigurations" + for plat in "${platforms[@]}"; do + for config in Debug Release; do + open_tag ProjectConfiguration \ + Include="$config|$plat" + tag_content Configuration $config + tag_content Platform $plat + close_tag ProjectConfiguration + done + done + close_tag ItemGroup + + open_tag PropertyGroup \ + Label="Globals" + tag_content ProjectGuid "{${guid}}" + tag_content RootNamespace ${name} + tag_content Keyword ManagedCProj + if [ "${platforms[0]}" = "ARM" ]; then + tag_content AppContainerApplication true + # The application type can be one of "Windows Store", + # "Windows Phone" or "Windows Phone Silverlight". The + # actual value doesn't matter from the libaom point of view, + # since a static library built for one works on the others. + # The PlatformToolset field needs to be set in sync with this; + # for Windows Store and Windows Phone Silverlight it should be + # v120 while it should be v120_wp81 if the type is Windows Phone. + tag_content ApplicationType "Windows Store" + tag_content ApplicationTypeRevision 8.1 + fi + close_tag PropertyGroup + + tag Import \ + Project="\$(VCTargetsPath)\\Microsoft.Cpp.Default.props" + + for plat in "${platforms[@]}"; do + for config in Release Debug; do + open_tag PropertyGroup \ + Condition="'\$(Configuration)|\$(Platform)'=='$config|$plat'" \ + Label="Configuration" + if [ "$proj_kind" = "exe" ]; then + tag_content ConfigurationType Application + elif [ "$proj_kind" = "dll" ]; then + tag_content ConfigurationType DynamicLibrary + else + tag_content ConfigurationType StaticLibrary + fi + if [ "$vs_ver" = "12" ]; then + # Setting a PlatformToolset indicating windows phone isn't + # enough to build code for arm with MSVC 2013, one strictly + # has to enable AppContainerApplication as well. + tag_content PlatformToolset v120 + fi + if [ "$vs_ver" = "14" ]; then + tag_content PlatformToolset v140 + fi + tag_content CharacterSet Unicode + if [ "$config" = "Release" ]; then + tag_content WholeProgramOptimization true + fi + close_tag PropertyGroup + done + done + + tag Import \ + Project="\$(VCTargetsPath)\\Microsoft.Cpp.props" + + open_tag ImportGroup \ + Label="PropertySheets" + tag Import \ + Project="\$(UserRootDir)\\Microsoft.Cpp.\$(Platform).user.props" \ + Condition="exists('\$(UserRootDir)\\Microsoft.Cpp.\$(Platform).user.props')" \ + Label="LocalAppDataPlatform" + close_tag ImportGroup + + tag PropertyGroup \ + Label="UserMacros" + + for plat in "${platforms[@]}"; do + plat_no_ws=`echo $plat | sed 's/[^A-Za-z0-9_]/_/g'` + for config in Debug Release; do + open_tag PropertyGroup \ + Condition="'\$(Configuration)|\$(Platform)'=='$config|$plat'" + tag_content OutDir "\$(SolutionDir)$plat_no_ws\\\$(Configuration)\\" + tag_content IntDir "$plat_no_ws\\\$(Configuration)\\${name}\\" + if [ "$proj_kind" == "lib" ]; then + if [ "$config" == "Debug" ]; then + config_suffix=d + else + config_suffix="" + fi + tag_content TargetName "${name}${lib_sfx}${config_suffix}" + fi + close_tag PropertyGroup + done + done + + for plat in "${platforms[@]}"; do + for config in Debug Release; do + open_tag ItemDefinitionGroup \ + Condition="'\$(Configuration)|\$(Platform)'=='$config|$plat'" + if [ "$name" == "aom" ]; then + hostplat=$plat + if [ "$hostplat" == "ARM" ]; then + hostplat=Win32 + fi + fi + open_tag ClCompile + if [ "$config" = "Debug" ]; then + opt=Disabled + runtime=$debug_runtime + curlibs=$debug_libs + debug=_DEBUG + else + opt=MaxSpeed + runtime=$release_runtime + curlibs=$libs + tag_content FavorSizeOrSpeed Speed + debug=NDEBUG + fi + extradefines=";$defines" + tag_content Optimization $opt + tag_content AdditionalIncludeDirectories "$incs;%(AdditionalIncludeDirectories)" + tag_content PreprocessorDefinitions "WIN32;$debug;_CRT_SECURE_NO_WARNINGS;_CRT_SECURE_NO_DEPRECATE$extradefines;%(PreprocessorDefinitions)" + tag_content RuntimeLibrary $runtime + tag_content WarningLevel Level3 + if ${werror:-false}; then + tag_content TreatWarningAsError true + fi + # We need to override the defaults for these settings + # if AppContainerApplication is set. + tag_content CompileAsWinRT false + tag_content PrecompiledHeader NotUsing + tag_content SDLCheck false + close_tag ClCompile + case "$proj_kind" in + exe) + open_tag Link + tag_content GenerateDebugInformation true + # Console is the default normally, but if + # AppContainerApplication is set, we need to override it. + tag_content SubSystem Console + close_tag Link + ;; + dll) + open_tag Link + tag_content GenerateDebugInformation true + tag_content ModuleDefinitionFile $module_def + close_tag Link + ;; + lib) + ;; + esac + close_tag ItemDefinitionGroup + done + + done + + open_tag ItemGroup + generate_filter "Source Files" "c;cc;cpp;def;odl;idl;hpj;bat;asm;asmx;s" + close_tag ItemGroup + open_tag ItemGroup + generate_filter "Header Files" "h;hm;inl;inc;xsd" + close_tag ItemGroup + open_tag ItemGroup + generate_filter "Build Files" "mk" + close_tag ItemGroup + open_tag ItemGroup + generate_filter "References" "vcxproj" + close_tag ItemGroup + + tag Import \ + Project="\$(VCTargetsPath)\\Microsoft.Cpp.targets" + + open_tag ImportGroup \ + Label="ExtensionTargets" + close_tag ImportGroup + + close_tag Project + + # This must be done from within the {} subshell + echo "Ignored files list (${#file_list[@]} items) is:" >&2 + for f in "${file_list[@]}"; do + echo " $f" >&2 + done +} + +# This regexp doesn't catch most of the strings in the vcxproj format, +# since they're like path instead of +# as previously. It still seems to work ok despite this. +generate_vcxproj | + sed -e '/"/s;\([^ "]\)/;\1\\;g' | + sed -e '/xmlns/s;\\;/;g' > ${outfile} + +exit diff --git a/third_party/aom/build/make/ios-Info.plist b/third_party/aom/build/make/ios-Info.plist new file mode 100644 index 0000000000..300e3e310d --- /dev/null +++ b/third_party/aom/build/make/ios-Info.plist @@ -0,0 +1,37 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + AOM + CFBundleIdentifier + org.webmproject.AOM + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + AOM + CFBundlePackageType + FMWK + CFBundleShortVersionString + ${VERSION} + CFBundleSignature + ???? + CFBundleSupportedPlatforms + + iPhoneOS + + CFBundleVersion + ${VERSION} + MinimumOSVersion + ${IOS_VERSION_MIN} + UIDeviceFamily + + 1 + 2 + + AOMFullVersion + ${FULLVERSION} + + diff --git a/third_party/aom/build/make/iosbuild.sh b/third_party/aom/build/make/iosbuild.sh new file mode 100755 index 0000000000..ca8214b622 --- /dev/null +++ b/third_party/aom/build/make/iosbuild.sh @@ -0,0 +1,383 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This script generates 'AOM.framework'. An iOS app can encode and decode AVx +## video by including 'AOM.framework'. +## +## Run iosbuild.sh to create 'AOM.framework' in the current directory. +## +set -e +devnull='> /dev/null 2>&1' + +BUILD_ROOT="_iosbuild" +CONFIGURE_ARGS="--disable-docs + --disable-examples + --disable-libyuv + --disable-unit-tests" +DIST_DIR="_dist" +FRAMEWORK_DIR="AOM.framework" +FRAMEWORK_LIB="AOM.framework/AOM" +HEADER_DIR="${FRAMEWORK_DIR}/Headers/aom" +SCRIPT_DIR=$(dirname "$0") +LIBAOM_SOURCE_DIR=$(cd ${SCRIPT_DIR}/../..; pwd) +LIPO=$(xcrun -sdk iphoneos${SDK} -find lipo) +ORIG_PWD="$(pwd)" +ARM_TARGETS="arm64-darwin-gcc + armv7-darwin-gcc + armv7s-darwin-gcc" +SIM_TARGETS="x86-iphonesimulator-gcc + x86_64-iphonesimulator-gcc" +OSX_TARGETS="x86-darwin16-gcc + x86_64-darwin16-gcc" +TARGETS="${ARM_TARGETS} ${SIM_TARGETS}" + +# Configures for the target specified by $1, and invokes make with the dist +# target using $ as the distribution output directory. +build_target() { + local target="$1" + local old_pwd="$(pwd)" + local target_specific_flags="" + + vlog "***Building target: ${target}***" + + case "${target}" in + x86-*) + target_specific_flags="--enable-pic" + vlog "Enabled PIC for ${target}" + ;; + esac + + mkdir "${target}" + cd "${target}" + eval "${LIBAOM_SOURCE_DIR}/configure" --target="${target}" \ + ${CONFIGURE_ARGS} ${EXTRA_CONFIGURE_ARGS} ${target_specific_flags} \ + ${devnull} + export DIST_DIR + eval make dist ${devnull} + cd "${old_pwd}" + + vlog "***Done building target: ${target}***" +} + +# Returns the preprocessor symbol for the target specified by $1. +target_to_preproc_symbol() { + target="$1" + case "${target}" in + arm64-*) + echo "__aarch64__" + ;; + armv7-*) + echo "__ARM_ARCH_7A__" + ;; + armv7s-*) + echo "__ARM_ARCH_7S__" + ;; + x86-*) + echo "__i386__" + ;; + x86_64-*) + echo "__x86_64__" + ;; + *) + echo "#error ${target} unknown/unsupported" + return 1 + ;; + esac +} + +# Create a aom_config.h shim that, based on preprocessor settings for the +# current target CPU, includes the real aom_config.h for the current target. +# $1 is the list of targets. +create_aom_framework_config_shim() { + local targets="$1" + local config_file="${HEADER_DIR}/aom_config.h" + local preproc_symbol="" + local target="" + local include_guard="AOM_FRAMEWORK_HEADERS_AOM_AOM_CONFIG_H_" + + local file_header="/* + * Copyright (c) $(date +%Y), Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* GENERATED FILE: DO NOT EDIT! */ + +#ifndef ${include_guard} +#define ${include_guard} + +#if defined" + + printf "%s" "${file_header}" > "${config_file}" + for target in ${targets}; do + preproc_symbol=$(target_to_preproc_symbol "${target}") + printf " ${preproc_symbol}\n" >> "${config_file}" + printf "#define AOM_FRAMEWORK_TARGET \"${target}\"\n" >> "${config_file}" + printf "#include \"AOM/aom/${target}/aom_config.h\"\n" >> "${config_file}" + printf "#elif defined" >> "${config_file}" + mkdir "${HEADER_DIR}/${target}" + cp -p "${BUILD_ROOT}/${target}/aom_config.h" "${HEADER_DIR}/${target}" + done + + # Consume the last line of output from the loop: We don't want it. + sed -i '' -e '$d' "${config_file}" + + printf "#endif\n\n" >> "${config_file}" + printf "#endif // ${include_guard}" >> "${config_file}" +} + +# Verifies that $FRAMEWORK_LIB fat library contains requested builds. +verify_framework_targets() { + local requested_cpus="" + local cpu="" + + # Extract CPU from full target name. + for target; do + cpu="${target%%-*}" + if [ "${cpu}" = "x86" ]; then + # lipo -info outputs i386 for libaom x86 targets. + cpu="i386" + fi + requested_cpus="${requested_cpus}${cpu} " + done + + # Get target CPUs present in framework library. + local targets_built=$(${LIPO} -info ${FRAMEWORK_LIB}) + + # $LIPO -info outputs a string like the following: + # Architectures in the fat file: $FRAMEWORK_LIB + # Capture only the architecture strings. + targets_built=${targets_built##*: } + + # Sort CPU strings to make the next step a simple string compare. + local actual=$(echo ${targets_built} | tr " " "\n" | sort | tr "\n" " ") + local requested=$(echo ${requested_cpus} | tr " " "\n" | sort | tr "\n" " ") + + vlog "Requested ${FRAMEWORK_LIB} CPUs: ${requested}" + vlog "Actual ${FRAMEWORK_LIB} CPUs: ${actual}" + + if [ "${requested}" != "${actual}" ]; then + elog "Actual ${FRAMEWORK_LIB} targets do not match requested target list." + elog " Requested target CPUs: ${requested}" + elog " Actual target CPUs: ${actual}" + return 1 + fi +} + +# Configures and builds each target specified by $1, and then builds +# AOM.framework. +build_framework() { + local lib_list="" + local targets="$1" + local target="" + local target_dist_dir="" + + # Clean up from previous build(s). + rm -rf "${BUILD_ROOT}" "${FRAMEWORK_DIR}" + + # Create output dirs. + mkdir -p "${BUILD_ROOT}" + mkdir -p "${HEADER_DIR}" + + cd "${BUILD_ROOT}" + + for target in ${targets}; do + build_target "${target}" + target_dist_dir="${BUILD_ROOT}/${target}/${DIST_DIR}" + if [ "${ENABLE_SHARED}" = "yes" ]; then + local suffix="dylib" + else + local suffix="a" + fi + lib_list="${lib_list} ${target_dist_dir}/lib/libaom.${suffix}" + done + + cd "${ORIG_PWD}" + + # The basic libaom API includes are all the same; just grab the most recent + # set. + cp -p "${target_dist_dir}"/include/aom/* "${HEADER_DIR}" + + # Build the fat library. + ${LIPO} -create ${lib_list} -output ${FRAMEWORK_DIR}/AOM + + # Create the aom_config.h shim that allows usage of aom_config.h from + # within AOM.framework. + create_aom_framework_config_shim "${targets}" + + # Copy in aom_version.h. + cp -p "${BUILD_ROOT}/${target}/aom_version.h" "${HEADER_DIR}" + + if [ "${ENABLE_SHARED}" = "yes" ]; then + # Adjust the dylib's name so dynamic linking in apps works as expected. + install_name_tool -id '@rpath/AOM.framework/AOM' ${FRAMEWORK_DIR}/AOM + + # Copy in Info.plist. + cat "${SCRIPT_DIR}/ios-Info.plist" \ + | sed "s/\${FULLVERSION}/${FULLVERSION}/g" \ + | sed "s/\${VERSION}/${VERSION}/g" \ + | sed "s/\${IOS_VERSION_MIN}/${IOS_VERSION_MIN}/g" \ + > "${FRAMEWORK_DIR}/Info.plist" + fi + + # Confirm AOM.framework/AOM contains the targets requested. + verify_framework_targets ${targets} + + vlog "Created fat library ${FRAMEWORK_LIB} containing:" + for lib in ${lib_list}; do + vlog " $(echo ${lib} | awk -F / '{print $2, $NF}')" + done +} + +# Trap function. Cleans up the subtree used to build all targets contained in +# $TARGETS. +cleanup() { + local readonly res=$? + cd "${ORIG_PWD}" + + if [ $res -ne 0 ]; then + elog "build exited with error ($res)" + fi + + if [ "${PRESERVE_BUILD_OUTPUT}" != "yes" ]; then + rm -rf "${BUILD_ROOT}" + fi +} + +print_list() { + local indent="$1" + shift + local list="$@" + for entry in ${list}; do + echo "${indent}${entry}" + done +} + +iosbuild_usage() { +cat << EOF + Usage: ${0##*/} [arguments] + --help: Display this message and exit. + --enable-shared: Build a dynamic framework for use on iOS 8 or later. + --extra-configure-args : Extra args to pass when configuring libaom. + --macosx: Uses darwin16 targets instead of iphonesimulator targets for x86 + and x86_64. Allows linking to framework when builds target MacOSX + instead of iOS. + --preserve-build-output: Do not delete the build directory. + --show-build-output: Show output from each library build. + --targets : Override default target list. Defaults: +$(print_list " " ${TARGETS}) + --test-link: Confirms all targets can be linked. Functionally identical to + passing --enable-examples via --extra-configure-args. + --verbose: Output information about the environment and each stage of the + build. +EOF +} + +elog() { + echo "${0##*/} failed because: $@" 1>&2 +} + +vlog() { + if [ "${VERBOSE}" = "yes" ]; then + echo "$@" + fi +} + +trap cleanup EXIT + +# Parse the command line. +while [ -n "$1" ]; do + case "$1" in + --extra-configure-args) + EXTRA_CONFIGURE_ARGS="$2" + shift + ;; + --help) + iosbuild_usage + exit + ;; + --enable-shared) + ENABLE_SHARED=yes + ;; + --preserve-build-output) + PRESERVE_BUILD_OUTPUT=yes + ;; + --show-build-output) + devnull= + ;; + --test-link) + EXTRA_CONFIGURE_ARGS="${EXTRA_CONFIGURE_ARGS} --enable-examples" + ;; + --targets) + TARGETS="$2" + shift + ;; + --macosx) + TARGETS="${ARM_TARGETS} ${OSX_TARGETS}" + ;; + --verbose) + VERBOSE=yes + ;; + *) + iosbuild_usage + exit 1 + ;; + esac + shift +done + +if [ "${ENABLE_SHARED}" = "yes" ]; then + CONFIGURE_ARGS="--enable-shared ${CONFIGURE_ARGS}" +fi + +FULLVERSION=$("${SCRIPT_DIR}"/version.sh --bare "${LIBAOM_SOURCE_DIR}") +VERSION=$(echo "${FULLVERSION}" | sed -E 's/^v([0-9]+\.[0-9]+\.[0-9]+).*$/\1/') + +if [ "$ENABLE_SHARED" = "yes" ]; then + IOS_VERSION_OPTIONS="--enable-shared" + IOS_VERSION_MIN="8.0" +else + IOS_VERSION_OPTIONS="" + IOS_VERSION_MIN="6.0" +fi + +if [ "${VERBOSE}" = "yes" ]; then +cat << EOF + BUILD_ROOT=${BUILD_ROOT} + DIST_DIR=${DIST_DIR} + CONFIGURE_ARGS=${CONFIGURE_ARGS} + EXTRA_CONFIGURE_ARGS=${EXTRA_CONFIGURE_ARGS} + FRAMEWORK_DIR=${FRAMEWORK_DIR} + FRAMEWORK_LIB=${FRAMEWORK_LIB} + HEADER_DIR=${HEADER_DIR} + LIBAOM_SOURCE_DIR=${LIBAOM_SOURCE_DIR} + LIPO=${LIPO} + MAKEFLAGS=${MAKEFLAGS} + ORIG_PWD=${ORIG_PWD} + PRESERVE_BUILD_OUTPUT=${PRESERVE_BUILD_OUTPUT} + TARGETS="$(print_list "" ${TARGETS})" + ENABLE_SHARED=${ENABLE_SHARED} + OSX_TARGETS="${OSX_TARGETS}" + SIM_TARGETS="${SIM_TARGETS}" + SCRIPT_DIR="${SCRIPT_DIR}" + FULLVERSION="${FULLVERSION}" + VERSION="${VERSION}" + IOS_VERSION_MIN="${IOS_VERSION_MIN}" +EOF +fi + +build_framework "${TARGETS}" +echo "Successfully built '${FRAMEWORK_DIR}' for:" +print_list "" ${TARGETS} diff --git a/third_party/aom/build/make/msvs_common.sh b/third_party/aom/build/make/msvs_common.sh new file mode 100644 index 0000000000..2df27df8de --- /dev/null +++ b/third_party/aom/build/make/msvs_common.sh @@ -0,0 +1,114 @@ +#!/bin/bash +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +if [ "$(uname -o 2>/dev/null)" = "Cygwin" ] \ + && cygpath --help >/dev/null 2>&1; then + FIXPATH='cygpath -m' +else + FIXPATH='echo_path' +fi + +die() { + echo "${self_basename}: $@" >&2 + exit 1 +} + +die_unknown(){ + echo "Unknown option \"$1\"." >&2 + echo "See ${self_basename} --help for available options." >&2 + exit 1 +} + +echo_path() { + for path; do + echo "$path" + done +} + +# Output one, possibly changed based on the system, path per line. +fix_path() { + $FIXPATH "$@" +} + +# Corrects the paths in file_list in one pass for efficiency. +# $1 is the name of the array to be modified. +fix_file_list() { + declare -n array_ref=$1 + files=$(fix_path "${array_ref[@]}") + local IFS=$'\n' + array_ref=($files) +} + +generate_uuid() { + local hex="0123456789ABCDEF" + local i + local uuid="" + local j + #93995380-89BD-4b04-88EB-625FBE52EBFB + for ((i=0; i<32; i++)); do + (( j = $RANDOM % 16 )) + uuid="${uuid}${hex:$j:1}" + done + echo "${uuid:0:8}-${uuid:8:4}-${uuid:12:4}-${uuid:16:4}-${uuid:20:12}" +} + +indent1=" " +indent="" +indent_push() { + indent="${indent}${indent1}" +} +indent_pop() { + indent="${indent%${indent1}}" +} + +tag_attributes() { + for opt in "$@"; do + optval="${opt#*=}" + [ -n "${optval}" ] || + die "Missing attribute value in '$opt' while generating $tag tag" + echo "${indent}${opt%%=*}=\"${optval}\"" + done +} + +open_tag() { + local tag=$1 + shift + if [ $# -ne 0 ]; then + echo "${indent}<${tag}" + indent_push + tag_attributes "$@" + echo "${indent}>" + else + echo "${indent}<${tag}>" + indent_push + fi +} + +close_tag() { + local tag=$1 + indent_pop + echo "${indent}" +} + +tag() { + local tag=$1 + shift + if [ $# -ne 0 ]; then + echo "${indent}<${tag}" + indent_push + tag_attributes "$@" + indent_pop + echo "${indent}/>" + else + echo "${indent}<${tag}/>" + fi +} + diff --git a/third_party/aom/build/make/rtcd.pl b/third_party/aom/build/make/rtcd.pl new file mode 100755 index 0000000000..354ae51766 --- /dev/null +++ b/third_party/aom/build/make/rtcd.pl @@ -0,0 +1,424 @@ +#!/usr/bin/env perl + +no strict 'refs'; +use warnings; +use Getopt::Long; +Getopt::Long::Configure("auto_help") if $Getopt::Long::VERSION > 2.32; + +my %ALL_FUNCS = (); +my @ALL_ARCHS; +my @ALL_FORWARD_DECLS; +my @REQUIRES; + +my %opts = (); +my %disabled = (); +my %required = (); + +my @argv; +foreach (@ARGV) { + $disabled{$1} = 1, next if /--disable-(.*)/; + $required{$1} = 1, next if /--require-(.*)/; + push @argv, $_; +} + +# NB: use GetOptions() instead of GetOptionsFromArray() for compatibility. +@ARGV = @argv; +GetOptions( + \%opts, + 'arch=s', + 'sym=s', + 'config=s', +); + +foreach my $opt (qw/arch config/) { + if (!defined($opts{$opt})) { + warn "--$opt is required!\n"; + Getopt::Long::HelpMessage('-exit' => 1); + } +} + +foreach my $defs_file (@ARGV) { + if (!-f $defs_file) { + warn "$defs_file: $!\n"; + Getopt::Long::HelpMessage('-exit' => 1); + } +} + +open CONFIG_FILE, $opts{config} or + die "Error opening config file '$opts{config}': $!\n"; + +my %config = (); +while () { + next if !/^(?:CONFIG_|HAVE_)/; + chomp; + s/\r$//; + my @pair = split /=/; + $config{$pair[0]} = $pair[1]; +} +close CONFIG_FILE; + +# +# Routines for the RTCD DSL to call +# +sub aom_config($) { + return (defined $config{$_[0]}) ? $config{$_[0]} : ""; +} + +sub specialize { + if (@_ <= 1) { + die "'specialize' must be called with a function name and at least one ", + "architecture ('C' is implied): \n@_\n"; + } + my $fn=$_[0]; + shift; + foreach my $opt (@_) { + eval "\$${fn}_${opt}=${fn}_${opt}"; + } +} + +sub add_proto { + my $fn = splice(@_, -2, 1); + $ALL_FUNCS{$fn} = \@_; + specialize $fn, "c"; +} + +sub require { + foreach my $fn (keys %ALL_FUNCS) { + foreach my $opt (@_) { + my $ofn = eval "\$${fn}_${opt}"; + next if !$ofn; + + # if we already have a default, then we can disable it, as we know + # we can do better. + my $best = eval "\$${fn}_default"; + if ($best) { + my $best_ofn = eval "\$${best}"; + if ($best_ofn && "$best_ofn" ne "$ofn") { + eval "\$${best}_link = 'false'"; + } + } + eval "\$${fn}_default=${fn}_${opt}"; + eval "\$${fn}_${opt}_link='true'"; + } + } +} + +sub forward_decls { + push @ALL_FORWARD_DECLS, @_; +} + +# +# Include the user's directives +# +foreach my $f (@ARGV) { + open FILE, "<", $f or die "cannot open $f: $!\n"; + my $contents = join('', ); + close FILE; + eval $contents or warn "eval failed: $@\n"; +} + +# +# Process the directives according to the command line +# +sub process_forward_decls() { + foreach (@ALL_FORWARD_DECLS) { + $_->(); + } +} + +sub determine_indirection { + aom_config("CONFIG_RUNTIME_CPU_DETECT") eq "yes" or &require(@ALL_ARCHS); + foreach my $fn (keys %ALL_FUNCS) { + my $n = ""; + my @val = @{$ALL_FUNCS{$fn}}; + my $args = pop @val; + my $rtyp = "@val"; + my $dfn = eval "\$${fn}_default"; + $dfn = eval "\$${dfn}"; + foreach my $opt (@_) { + my $ofn = eval "\$${fn}_${opt}"; + next if !$ofn; + my $link = eval "\$${fn}_${opt}_link"; + next if $link && $link eq "false"; + $n .= "x"; + } + if ($n eq "x") { + eval "\$${fn}_indirect = 'false'"; + } else { + eval "\$${fn}_indirect = 'true'"; + } + } +} + +sub declare_function_pointers { + foreach my $fn (sort keys %ALL_FUNCS) { + my @val = @{$ALL_FUNCS{$fn}}; + my $args = pop @val; + my $rtyp = "@val"; + my $dfn = eval "\$${fn}_default"; + $dfn = eval "\$${dfn}"; + foreach my $opt (@_) { + my $ofn = eval "\$${fn}_${opt}"; + next if !$ofn; + print "$rtyp ${ofn}($args);\n"; + } + if (eval "\$${fn}_indirect" eq "false") { + print "#define ${fn} ${dfn}\n"; + } else { + print "RTCD_EXTERN $rtyp (*${fn})($args);\n"; + } + print "\n"; + } +} + +sub set_function_pointers { + foreach my $fn (sort keys %ALL_FUNCS) { + my @val = @{$ALL_FUNCS{$fn}}; + my $args = pop @val; + my $rtyp = "@val"; + my $dfn = eval "\$${fn}_default"; + $dfn = eval "\$${dfn}"; + if (eval "\$${fn}_indirect" eq "true") { + print " $fn = $dfn;\n"; + foreach my $opt (@_) { + my $ofn = eval "\$${fn}_${opt}"; + next if !$ofn; + next if "$ofn" eq "$dfn"; + my $link = eval "\$${fn}_${opt}_link"; + next if $link && $link eq "false"; + my $cond = eval "\$have_${opt}"; + print " if (${cond}) $fn = $ofn;\n" + } + } + } +} + +sub filter { + my @filtered; + foreach (@_) { push @filtered, $_ unless $disabled{$_}; } + return @filtered; +} + +# +# Helper functions for generating the arch specific RTCD files +# +sub common_top() { + my $include_guard = uc($opts{sym})."_H_"; + print <) { + if (/HAVE_DSPR2=yes/) { + @ALL_ARCHS = filter("$opts{arch}", qw/dspr2/); + last; + } + if (/HAVE_MSA=yes/) { + @ALL_ARCHS = filter("$opts{arch}", qw/msa/); + last; + } + } + close CONFIG_FILE; + mips; +} elsif ($opts{arch} eq 'armv6') { + @ALL_ARCHS = filter(qw/media/); + arm; +} elsif ($opts{arch} =~ /armv7\w?/) { + @ALL_ARCHS = filter(qw/media neon_asm neon/); + @REQUIRES = filter(keys %required ? keys %required : qw/media/); + &require(@REQUIRES); + arm; +} elsif ($opts{arch} eq 'armv8' || $opts{arch} eq 'arm64' ) { + @ALL_ARCHS = filter(qw/neon/); + arm; +} else { + unoptimized; +} + +__END__ + +=head1 NAME + +rtcd - + +=head1 SYNOPSIS + +Usage: rtcd.pl [options] FILE + +See 'perldoc rtcd.pl' for more details. + +=head1 DESCRIPTION + +Reads the Run Time CPU Detections definitions from FILE and generates a +C header file on stdout. + +=head1 OPTIONS + +Options: + --arch=ARCH Architecture to generate defs for (required) + --disable-EXT Disable support for EXT extensions + --require-EXT Require support for EXT extensions + --sym=SYMBOL Unique symbol to use for RTCD initialization function + --config=FILE File with CONFIG_FOO=yes lines to parse diff --git a/third_party/aom/build/make/thumb.pm b/third_party/aom/build/make/thumb.pm new file mode 100644 index 0000000000..8248694e94 --- /dev/null +++ b/third_party/aom/build/make/thumb.pm @@ -0,0 +1,71 @@ +#!/usr/bin/env perl +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +package thumb; + +sub FixThumbInstructions($$) +{ + my $short_branches = $_[1]; + my $branch_shift_offset = $short_branches ? 1 : 0; + + # Write additions with shifts, such as "add r10, r11, lsl #8", + # in three operand form, "add r10, r10, r11, lsl #8". + s/(add\s+)(r\d+),\s*(r\d+),\s*(lsl #\d+)/$1$2, $2, $3, $4/g; + + # Convert additions with a non-constant shift into a sequence + # with left shift, addition and a right shift (to restore the + # register to the original value). Currently the right shift + # isn't necessary in the code base since the values in these + # registers aren't used, but doing the shift for consistency. + # This converts instructions such as "add r12, r12, r5, lsl r4" + # into the sequence "lsl r5, r4", "add r12, r12, r5", "lsr r5, r4". + s/^(\s*)(add)(\s+)(r\d+),\s*(r\d+),\s*(r\d+),\s*lsl (r\d+)/$1lsl$3$6, $7\n$1$2$3$4, $5, $6\n$1lsr$3$6, $7/g; + + # Convert loads with right shifts in the indexing into a + # sequence of an add, load and sub. This converts + # "ldrb r4, [r9, lr, asr #1]" into "add r9, r9, lr, asr #1", + # "ldrb r9, [r9]", "sub r9, r9, lr, asr #1". + s/^(\s*)(ldrb)(\s+)(r\d+),\s*\[(\w+),\s*(\w+),\s*(asr #\d+)\]/$1add $3$5, $5, $6, $7\n$1$2$3$4, [$5]\n$1sub $3$5, $5, $6, $7/g; + + # Convert register indexing with writeback into a separate add + # instruction. This converts "ldrb r12, [r1, r2]!" into + # "ldrb r12, [r1, r2]", "add r1, r1, r2". + s/^(\s*)(ldrb)(\s+)(r\d+),\s*\[(\w+),\s*(\w+)\]!/$1$2$3$4, [$5, $6]\n$1add $3$5, $6/g; + + # Convert negative register indexing into separate sub/add instructions. + # This converts "ldrne r4, [src, -pstep, lsl #1]" into + # "subne src, src, pstep, lsl #1", "ldrne r4, [src]", + # "addne src, src, pstep, lsl #1". In a couple of cases where + # this is used, it's used for two subsequent load instructions, + # where a hand-written version of it could merge two subsequent + # add and sub instructions. + s/^(\s*)((ldr|str|pld)(ne)?)(\s+)(r\d+,\s*)?\[(\w+), -([^\]]+)\]/$1sub$4$5$7, $7, $8\n$1$2$5$6\[$7\]\n$1add$4$5$7, $7, $8/g; + + # Convert register post indexing to a separate add instruction. + # This converts "ldrneb r9, [r0], r2" into "ldrneb r9, [r0]", + # "addne r0, r0, r2". + s/^(\s*)((ldr|str)(ne)?[bhd]?)(\s+)(\w+),(\s*\w+,)?\s*\[(\w+)\],\s*(\w+)/$1$2$5$6,$7 [$8]\n$1add$4$5$8, $8, $9/g; + + # Convert a conditional addition to the pc register into a series of + # instructions. This converts "addlt pc, pc, r3, lsl #2" into + # "itttt lt", "movlt.n r12, pc", "addlt.w r12, #12", + # "addlt.w r12, r12, r3, lsl #2", "movlt.n pc, r12". + # This assumes that r12 is free at this point. + s/^(\s*)addlt(\s+)pc,\s*pc,\s*(\w+),\s*lsl\s*#(\d+)/$1itttt$2lt\n$1movlt.n$2r12, pc\n$1addlt.w$2r12, #12\n$1addlt.w$2r12, r12, $3, lsl #($4-$branch_shift_offset)\n$1movlt.n$2pc, r12/g; + + # Convert "mov pc, lr" into "bx lr", since the former only works + # for switching from arm to thumb (and only in armv7), but not + # from thumb to arm. + s/mov(\s*)pc\s*,\s*lr/bx$1lr/g; +} + +1; diff --git a/third_party/aom/build/make/version.sh b/third_party/aom/build/make/version.sh new file mode 100755 index 0000000000..c0eef9f580 --- /dev/null +++ b/third_party/aom/build/make/version.sh @@ -0,0 +1,77 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + + +for opt in "$@"; do + optval="${opt#*=}" + case "$opt" in + --bare) bare=true ;; + *) break ;; + esac + shift +done +source_path=${1:-.} +out_file=${2} +id=${3:-VERSION_STRING} + +git_version_id="" +if [ -e "${source_path}/.git" ]; then + # Source Path is a git working copy. Check for local modifications. + # Note that git submodules may have a file as .git, not a directory. + export GIT_DIR="${source_path}/.git" + git_version_id=`git describe --match=v[0-9]* 2>/dev/null` +fi + +changelog_version="" +for p in "${source_path}" "${source_path}/.."; do + if [ -z "$git_version_id" -a -f "${p}/CHANGELOG" ]; then + changelog_version=`head -n1 "${p}/CHANGELOG" | awk '{print $2}'` + changelog_version="${changelog_version}" + break + fi +done +version_str="${changelog_version}${git_version_id}" +bare_version=${version_str#v} +major_version=${bare_version%%.*} +bare_version=${bare_version#*.} +minor_version=${bare_version%%.*} +bare_version=${bare_version#*.} +patch_version=${bare_version%%-*} +bare_version=${bare_version#${patch_version}} +extra_version=${bare_version##-} + +#since they'll be used as integers below make sure they are or force to 0 +for v in major_version minor_version patch_version; do + if eval echo \$$v |grep -E -q '[^[:digit:]]'; then + eval $v=0 + fi +done + +if [ ${bare} ]; then + echo "${changelog_version}${git_version_id}" > $$.tmp +else + cat<$$.tmp +#define VERSION_MAJOR $major_version +#define VERSION_MINOR $minor_version +#define VERSION_PATCH $patch_version +#define VERSION_EXTRA "$extra_version" +#define VERSION_PACKED ((VERSION_MAJOR<<16)|(VERSION_MINOR<<8)|(VERSION_PATCH)) +#define ${id}_NOSP "${version_str}" +#define ${id} " ${version_str}" +EOF +fi +if [ -n "$out_file" ]; then +diff $$.tmp ${out_file} >/dev/null 2>&1 || cat $$.tmp > ${out_file} +else +cat $$.tmp +fi +rm $$.tmp diff --git a/third_party/aom/codereview.settings b/third_party/aom/codereview.settings new file mode 100644 index 0000000000..0f36904869 --- /dev/null +++ b/third_party/aom/codereview.settings @@ -0,0 +1,5 @@ +# This file is used by gcl to get repository specific information. +GERRIT_HOST: aomedia-review.googlesource.com +GERRIT_PORT: 29418 +GERRIT_SQUASH_UPLOADS: false +CODE_REVIEW_SERVER: aomedia-review.googlesource.com diff --git a/third_party/aom/configure b/third_party/aom/configure new file mode 100755 index 0000000000..83ec996ce8 --- /dev/null +++ b/third_party/aom/configure @@ -0,0 +1,865 @@ +#!/bin/sh +## +## configure +## +## This script is the front-end to the build system. It provides a similar +## interface to standard configure scripts with some extra bits for dealing +## with toolchains that differ from the standard POSIX interface and +## for extracting subsets of the source tree. In theory, reusable parts +## of this script were intended to live in build/make/configure.sh, +## but in practice, the line is pretty blurry. +## +## This build system is based in part on the FFmpeg configure script. +## + +#source_path="`dirname \"$0\"`" +source_path=${0%/*} +. "${source_path}/build/make/configure.sh" + +show_help(){ + show_help_pre + cat << EOF +Advanced options: + ${toggle_libs} libraries + ${toggle_examples} examples + ${toggle_analyzer} analyzer + ${toggle_docs} documentation + ${toggle_unit_tests} unit tests + ${toggle_decode_perf_tests} build decoder perf tests with unit tests + ${toggle_encode_perf_tests} build encoder perf tests with unit tests + --cpu=CPU tune for the specified CPU (ARM: cortex-a8, X86: sse3) + --libc=PATH path to alternate libc + --size-limit=WxH max size to allow in the decoder + --as={yasm|nasm|auto} use specified assembler [auto, yasm preferred] + --sdk-path=PATH path to root of sdk (android builds only) + ${toggle_codec_srcs} in/exclude codec library source code + ${toggle_debug_libs} in/exclude debug version of libraries + ${toggle_static_msvcrt} use static MSVCRT (VS builds only) + ${toggle_highbitdepth} enable 16-bit generic pixel pipeline (used by high bitdepth profiles) + ${toggle_lowbitdepth} enable 8-bit optimized pixel pipeline + ${toggle_av1} AV1 codec support + ${toggle_internal_stats} output of encoder internal stats for debug, if supported (encoders) + ${toggle_postproc} postprocessing + ${toggle_multithread} multithreaded encoding and decoding + ${toggle_spatial_resampling} spatial sampling (scaling) support + ${toggle_realtime_only} enable this option while building for real-time encoding + ${toggle_onthefly_bitpacking} enable on-the-fly bitpacking in real-time encoding + ${toggle_error_concealment} enable this option to get a decoder which is able to conceal losses + ${toggle_coefficient_range_checking} + enable decoder to check if intermediate + transform coefficients are in valid range + ${toggle_runtime_cpu_detect} runtime cpu detection + ${toggle_shared} shared library support + ${toggle_static} static library support + ${toggle_small} favor smaller size over speed + ${toggle_postproc_visualizer} macro block / block level visualizers + ${toggle_webm_io} enable input from and output to WebM container + ${toggle_libyuv} enable libyuv + ${toggle_accounting} enable bit accounting + ${toggle_inspection} enable bitstream inspection + +Codecs: + Codecs can be selectively enabled or disabled individually, or by family: + --disable- + is equivalent to: + --disable--encoder + --disable--decoder + + Codecs available in this distribution: +EOF +#restore editor state ' + + family=""; + last_family=""; + c=""; + str=""; + for c in ${CODECS}; do + family=${c%_*} + if [ "${family}" != "${last_family}" ]; then + [ -z "${str}" ] || echo "${str}" + str="$(printf ' %10s:' ${family})" + fi + str="${str} $(printf '%10s' ${c#*_})" + last_family=${family} + done + echo "${str}" + show_help_post +} + +## +## BEGIN APPLICATION SPECIFIC CONFIGURATION +## + +# all_platforms is a list of all supported target platforms. Maintain +# alphabetically by architecture, generic-gnu last. +all_platforms="${all_platforms} arm64-darwin-gcc" +all_platforms="${all_platforms} arm64-linux-gcc" +all_platforms="${all_platforms} armv6-linux-rvct" +all_platforms="${all_platforms} armv6-linux-gcc" +all_platforms="${all_platforms} armv6-none-rvct" +all_platforms="${all_platforms} armv7-android-gcc" #neon Cortex-A8 +all_platforms="${all_platforms} armv7-darwin-gcc" #neon Cortex-A8 +all_platforms="${all_platforms} armv7-linux-rvct" #neon Cortex-A8 +all_platforms="${all_platforms} armv7-linux-gcc" #neon Cortex-A8 +all_platforms="${all_platforms} armv7-none-rvct" #neon Cortex-A8 +all_platforms="${all_platforms} armv7-win32-vs12" +all_platforms="${all_platforms} armv7-win32-vs14" +all_platforms="${all_platforms} armv7s-darwin-gcc" +all_platforms="${all_platforms} armv8-linux-gcc" +all_platforms="${all_platforms} mips32-linux-gcc" +all_platforms="${all_platforms} mips64-linux-gcc" +all_platforms="${all_platforms} sparc-solaris-gcc" +all_platforms="${all_platforms} x86-android-gcc" +all_platforms="${all_platforms} x86-darwin8-gcc" +all_platforms="${all_platforms} x86-darwin8-icc" +all_platforms="${all_platforms} x86-darwin9-gcc" +all_platforms="${all_platforms} x86-darwin9-icc" +all_platforms="${all_platforms} x86-darwin10-gcc" +all_platforms="${all_platforms} x86-darwin11-gcc" +all_platforms="${all_platforms} x86-darwin12-gcc" +all_platforms="${all_platforms} x86-darwin13-gcc" +all_platforms="${all_platforms} x86-darwin14-gcc" +all_platforms="${all_platforms} x86-darwin15-gcc" +all_platforms="${all_platforms} x86-darwin16-gcc" +all_platforms="${all_platforms} x86-iphonesimulator-gcc" +all_platforms="${all_platforms} x86-linux-gcc" +all_platforms="${all_platforms} x86-linux-icc" +all_platforms="${all_platforms} x86-os2-gcc" +all_platforms="${all_platforms} x86-solaris-gcc" +all_platforms="${all_platforms} x86-win32-gcc" +all_platforms="${all_platforms} x86-win32-vs12" +all_platforms="${all_platforms} x86-win32-vs14" +all_platforms="${all_platforms} x86_64-android-gcc" +all_platforms="${all_platforms} x86_64-darwin9-gcc" +all_platforms="${all_platforms} x86_64-darwin10-gcc" +all_platforms="${all_platforms} x86_64-darwin11-gcc" +all_platforms="${all_platforms} x86_64-darwin12-gcc" +all_platforms="${all_platforms} x86_64-darwin13-gcc" +all_platforms="${all_platforms} x86_64-darwin14-gcc" +all_platforms="${all_platforms} x86_64-darwin15-gcc" +all_platforms="${all_platforms} x86_64-darwin16-gcc" +all_platforms="${all_platforms} x86_64-iphonesimulator-gcc" +all_platforms="${all_platforms} x86_64-linux-gcc" +all_platforms="${all_platforms} x86_64-linux-icc" +all_platforms="${all_platforms} x86_64-solaris-gcc" +all_platforms="${all_platforms} x86_64-win64-gcc" +all_platforms="${all_platforms} x86_64-win64-vs12" +all_platforms="${all_platforms} x86_64-win64-vs14" +all_platforms="${all_platforms} generic-gnu" + +# all_targets is a list of all targets that can be configured +# note that these should be in dependency order for now. +all_targets="libs examples docs" + +# all targets available are enabled, by default. +for t in ${all_targets}; do + [ -f "${source_path}/${t}.mk" ] && enable_feature ${t} +done + +if ! perl --version >/dev/null; then + die "Perl is required to build" +fi + + +if [ "`cd \"${source_path}\" && pwd`" != "`pwd`" ]; then + # test to see if source_path already configured + if [ -f "${source_path}/aom_config.h" ]; then + die "source directory already configured; run 'make distclean' there first" + fi +fi + +# check installed doxygen version +doxy_version=$(doxygen --version 2>/dev/null) +doxy_major=${doxy_version%%.*} +if [ ${doxy_major:-0} -ge 1 ]; then + doxy_version=${doxy_version#*.} + doxy_minor=${doxy_version%%.*} + doxy_patch=${doxy_version##*.} + + [ $doxy_major -gt 1 ] && enable_feature doxygen + [ $doxy_minor -gt 5 ] && enable_feature doxygen + [ $doxy_minor -eq 5 ] && [ $doxy_patch -ge 3 ] && enable_feature doxygen +fi + +# disable codecs when their source directory does not exist +[ -d "${source_path}/av1" ] || disable_codec av1 + +# install everything except the sources, by default. sources will have +# to be enabled when doing dist builds, since that's no longer a common +# case. +enabled doxygen && enable_feature install_docs +enable_feature install_bins +enable_feature install_libs + +enable_feature static +enable_feature optimizations +enable_feature dependency_tracking +enable_feature spatial_resampling +enable_feature multithread +enable_feature os_support + +CODECS=" + av1_encoder + av1_decoder +" +CODEC_FAMILIES=" + av1 +" + +ARCH_LIST=" + arm + mips + x86 + x86_64 +" +ARCH_EXT_LIST_X86=" + mmx + sse + sse2 + sse3 + ssse3 + sse4_1 + avx + avx2 +" +ARCH_EXT_LIST=" + edsp + media + neon + neon_asm + + mips32 + dspr2 + msa + mips64 + + ${ARCH_EXT_LIST_X86} +" +HAVE_LIST=" + ${ARCH_EXT_LIST} + aom_ports + fexcept + pthread_h + unistd_h + wxwidgets +" +EXPERIMENT_LIST=" + fp_mb_stats + cdef + var_tx + rect_tx + ref_mv + tpl_mv + dual_filter + convolve_round + compound_round + ext_tx + tx64x64 + sub8x8_mc + ext_intra + intra_interp + filter_intra + intrabc + ext_inter + interintra + wedge + compound_segment + ext_refs + global_motion + new_quant + supertx + ans + ec_multisymbol + new_tokenset + loop_restoration + ext_partition + ext_partition_types + unpoison_partition_ctx + ext_tile + motion_var + ncobmc + warped_motion + q_adapt_probs + subframe_prob_update + bitstream_debug + alt_intra + palette + palette_delta_encoding + daala_ec + rawbits + ec_smallmul + pvq + cfl + xiphrc + cb4x4 + chroma_2x2 + chroma_sub8x8 + frame_size + delta_q + ext_delta_q + adapt_scan + filter_7bit + parallel_deblocking + parallel_deblocking_15tap + loopfiltering_across_tiles + tile_groups + ec_adapt + tempmv_signaling + rd_debug + reference_buffer + coef_interleave + entropy_stats + masked_tx + dependent_horztiles + daala_dist + tripred + palette_throughput + ref_adapt + lv_map + txk_sel + mv_compress + frame_superres + new_multisymbol + compound_singleref + aom_qm + lowdelay_compound +" +CONFIG_LIST=" + dependency_tracking + external_build + install_docs + install_bins + install_libs + install_srcs + debug + gprof + gcov + rvct + gcc + msvs + pic + big_endian + + codec_srcs + debug_libs + + runtime_cpu_detect + postproc + multithread + internal_stats + ${CODECS} + ${CODEC_FAMILIES} + encoders + decoders + static_msvcrt + spatial_resampling + realtime_only + onthefly_bitpacking + error_concealment + shared + static + small + postproc_visualizer + os_support + unit_tests + webm_io + libyuv + accounting + inspection + decode_perf_tests + encode_perf_tests + coefficient_range_checking + lowbitdepth + highbitdepth + experimental + size_limit + ${EXPERIMENT_LIST} + analyzer +" +CMDLINE_SELECT=" + dependency_tracking + external_build + extra_warnings + werror + install_docs + install_bins + install_libs + install_srcs + debug + gprof + gcov + pic + optimizations + ccache + runtime_cpu_detect + thumb + + libs + examples + analyzer + docs + libc + as + size_limit + codec_srcs + debug_libs + + postproc + multithread + internal_stats + ${CODECS} + ${CODEC_FAMILIES} + static_msvcrt + spatial_resampling + realtime_only + onthefly_bitpacking + error_concealment + shared + static + small + postproc_visualizer + unit_tests + webm_io + libyuv + accounting + inspection + decode_perf_tests + encode_perf_tests + coefficient_range_checking + lowbitdepth + aom_highbitdepth + highbitdepth + experimental +" + +process_cmdline() { + for opt do + optval="${opt#*=}" + case "$opt" in + --disable-codecs) + for c in ${CODEC_FAMILIES}; do disable_codec $c; done + ;; + --enable-?*|--disable-?*) + eval `echo "$opt" | sed 's/--/action=/;s/-/ option=/;s/-/_/g'` + if is_in ${option} ${EXPERIMENT_LIST}; then + if enabled experimental; then + ${action}_feature $option + else + log_echo "Ignoring $opt -- not in experimental mode." + fi + elif is_in ${option} "${CODECS} ${CODEC_FAMILIES}"; then + ${action}_codec ${option} + else + process_common_cmdline $opt + fi + ;; + *) process_common_cmdline "$opt" + ;; + esac + done +} + +post_process_cmdline() { + c="" + + # Enable all detected codecs, if they haven't been disabled + for c in ${CODECS}; do soft_enable $c; done + + # Enable the codec family if any component of that family is enabled + for c in ${CODECS}; do + enabled $c && enable_feature ${c%_*} + done + + # Set the {en,de}coders variable if any algorithm in that class is enabled + for c in ${CODECS}; do + enabled ${c} && enable_feature ${c##*_}s + done + + # Enable adopted experiments by default + soft_enable ref_mv + soft_enable cb4x4 + soft_enable filter_7bit + soft_enable reference_buffer + soft_enable delta_q + soft_enable tile_groups + soft_enable rect_tx + soft_enable global_motion + soft_enable cdef + soft_enable ec_adapt + soft_enable new_tokenset + soft_enable ext_intra + soft_enable mv_compress + soft_enable ext_refs + soft_enable dual_filter + soft_enable motion_var + soft_enable warped_motion + soft_enable ext_delta_q + + # Backwards/jenkins compatibility with --enable-aom-highbitdepth + enabled aom_highbitdepth && enable_feature highbitdepth + + # Enable low-bitdepth pixel pipeline by default + soft_enable lowbitdepth + + ! enabled ans && soft_enable daala_ec + soft_enable ec_multisymbol + soft_enable palette + soft_enable alt_intra + soft_enable palette_throughput + + # Fix up experiment dependencies + enabled pvq && enable_feature ec_adapt + enabled ec_adapt && enable_feature ec_multisymbol + enabled new_tokenset && enable_feature ec_multisymbol + enabled new_multisymbol && enable_feature ec_multisymbol + enabled ec_multisymbol && ! enabled ans && soft_enable daala_ec + enabled ec_multisymbol && ! enabled daala_ec && soft_enable ans + enabled palette_throughput && soft_enable palette + enabled ext_delta_q && soft_enable delta_q + enabled txk_sel && soft_enable lv_map + enabled compound_round && soft_enable convolve_round + if enabled rawbits && ! enabled daala_ec; then + log_echo "rawbits requires daala_ec, so disabling rawbits" + disable_feature rawbits + fi + if enabled ec_smallmul && ! enabled daala_ec; then + log_echo "ec_smallmul requires daala_ec, so disabling ec_smallmul" + disable_feature ec_smallmul + fi + if enabled ext_tile; then + log_echo "ext_tile not compatible with reference_buffer, so" + log_echo "disabling reference_buffer" + disable_feature reference_buffer + fi + if enabled ext_tile; then + log_echo "ext_tile not compatible with tile_groups, so" + log_echo "disabling tile_groups" + disable_feature tile_groups + fi + # Enable accounting and inspection when building the analyzer + if enabled analyzer; then + soft_enable accounting + soft_enable inspection + fi +} + +process_targets() { + enabled child || write_common_config_banner + write_common_target_config_h ${BUILD_PFX}aom_config.h + write_common_config_targets + + # Calculate the default distribution name, based on the enabled features + cf="" + DIST_DIR=aom + for cf in $CODEC_FAMILIES; do + if enabled ${cf}_encoder && enabled ${cf}_decoder; then + DIST_DIR="${DIST_DIR}-${cf}" + elif enabled ${cf}_encoder; then + DIST_DIR="${DIST_DIR}-${cf}cx" + elif enabled ${cf}_decoder; then + DIST_DIR="${DIST_DIR}-${cf}dx" + fi + done + enabled debug_libs && DIST_DIR="${DIST_DIR}-debug" + enabled codec_srcs && DIST_DIR="${DIST_DIR}-src" + ! enabled postproc && DIST_DIR="${DIST_DIR}-nopost" + ! enabled multithread && DIST_DIR="${DIST_DIR}-nomt" + ! enabled install_docs && DIST_DIR="${DIST_DIR}-nodocs" + DIST_DIR="${DIST_DIR}-${tgt_isa}-${tgt_os}" + case "${tgt_os}" in + win*) enabled static_msvcrt && DIST_DIR="${DIST_DIR}mt" || DIST_DIR="${DIST_DIR}md" + DIST_DIR="${DIST_DIR}-${tgt_cc}" + ;; + esac + if [ -f "${source_path}/build/make/version.sh" ]; then + ver=`"$source_path/build/make/version.sh" --bare "$source_path"` + DIST_DIR="${DIST_DIR}-${ver}" + VERSION_STRING=${ver} + ver=${ver%%-*} + VERSION_PATCH=${ver##*.} + ver=${ver%.*} + VERSION_MINOR=${ver##*.} + ver=${ver#v} + VERSION_MAJOR=${ver%.*} + fi + enabled child || cat <> config.mk + +PREFIX=${prefix} +ifeq (\$(MAKECMDGOALS),dist) +DIST_DIR?=${DIST_DIR} +else +DIST_DIR?=\$(DESTDIR)${prefix} +endif +LIBSUBDIR=${libdir##${prefix}/} + +VERSION_STRING=${VERSION_STRING} + +VERSION_MAJOR=${VERSION_MAJOR} +VERSION_MINOR=${VERSION_MINOR} +VERSION_PATCH=${VERSION_PATCH} + +CONFIGURE_ARGS=${CONFIGURE_ARGS} +EOF + enabled child || echo "CONFIGURE_ARGS?=${CONFIGURE_ARGS}" >> config.mk + + # + # Write makefiles for all enabled targets + # + for tgt in libs examples docs solution; do + tgt_fn="$tgt-$toolchain.mk" + + if enabled $tgt; then + echo "Creating makefiles for ${toolchain} ${tgt}" + write_common_target_config_mk $tgt_fn ${BUILD_PFX}aom_config.h + #write_${tgt}_config + fi + done + +} + +process_detect() { + if enabled shared; then + # Can only build shared libs on a subset of platforms. Doing this check + # here rather than at option parse time because the target auto-detect + # magic happens after the command line has been parsed. + case "${tgt_os}" in + linux|os2|darwin*|iphonesimulator*) + # Supported platforms + ;; + *) + if enabled gnu; then + echo "--enable-shared is only supported on ELF; assuming this is OK" + else + die "--enable-shared only supported on ELF, OS/2, and Darwin for now" + fi + ;; + esac + fi + if [ -z "$CC" ] || enabled external_build; then + echo "Bypassing toolchain for environment detection." + enable_feature external_build + check_header() { + log fake_check_header "$@" + header=$1 + shift + var=`echo $header | sed 's/[^A-Za-z0-9_]/_/g'` + disable_feature $var + # Headers common to all environments + case $header in + stdio.h) + true; + ;; + *) + result=false + for d in "$@"; do + [ -f "${d##-I}/$header" ] && result=true && break + done + ${result:-true} + esac && enable_feature $var + + # Specialize windows and POSIX environments. + case $toolchain in + *-win*-*) + # Don't check for any headers in Windows builds. + false + ;; + *) + case $header in + pthread.h) true;; + unistd.h) true;; + *) false;; + esac && enable_feature $var + esac + enabled $var + } + check_ld() { + true + } + fi + check_header stdio.h || die "Unable to invoke compiler: ${CC} ${CFLAGS}" + check_ld < +int main(void) { (void)feenableexcept(FE_DIVBYZERO | FE_INVALID); return 0; } +EOF +} + +process_toolchain() { + process_common_toolchain + + # Enable some useful compiler flags + if enabled gcc; then + enabled werror && check_add_cflags -Werror + check_add_cflags -Wall + check_add_cflags -Wdisabled-optimization + check_add_cflags -Wfloat-conversion + check_add_cflags -Wpointer-arith + check_add_cflags -Wtype-limits + check_add_cflags -Wvla + check_add_cflags -Wimplicit-function-declaration + check_add_cflags -Wuninitialized + check_add_cflags -Wunused + check_add_cflags -Wsign-compare + check_add_cflags -Wlogical-op + check_add_cflags -Wstack-usage=320000 + # Enabling the following warning (in combination with -Wunused above) + # for C++ generates errors in third_party code including googletest and + # libyuv. So enable it only for C code. + check_cflags "-Wextra" && add_cflags_only "-Wextra" + # Enabling the following warning for C++ generates some useless warnings + # about some function parameters shadowing class member function names. + # So, only enable this warning for C code. + check_cflags "-Wshadow" && add_cflags_only "-Wshadow" + if enabled mips || [ -z "${INLINE}" ]; then + enabled extra_warnings || check_add_cflags -Wno-unused-function + fi + # gtest makes heavy use of undefined pre-processor symbols + check_cflags "-Wundef" && add_cflags_only "-Wundef" + # Avoid this warning for third_party C++ sources. Some reorganization + # would be needed to apply this only to test/*.cc. + check_cflags -Wshorten-64-to-32 && add_cflags_only -Wshorten-64-to-32 + fi + + if enabled icc; then + enabled werror && check_add_cflags -Werror + check_add_cflags -Wall + check_add_cflags -Wpointer-arith + + # ICC has a number of floating point optimizations that we disable + # in favor of deterministic output WRT to other compilers + add_cflags -fp-model precise + fi + + if enabled analyzer; then + soft_enable wxwidgets + if ! wx-config --version > /dev/null; then + die "Couldn't find wx-config" + fi + + add_cxxflags_only $(wx-config --cppflags) + add_extralibs $(wx-config --libs) + fi + + # Enable extra, harmless warnings. These might provide additional insight + # to what the compiler is doing and why, but in general, but they shouldn't + # be treated as fatal, even if we're treating warnings as errors. + GCC_EXTRA_WARNINGS=" + -Wdisabled-optimization + -Winline + " + enabled gcc && EXTRA_WARNINGS="${GCC_EXTRA_WARNINGS}" + RVCT_EXTRA_WARNINGS=" + --remarks + " + enabled rvct && EXTRA_WARNINGS="${RVCT_EXTRA_WARNINGS}" + if enabled extra_warnings; then + for w in ${EXTRA_WARNINGS}; do + check_add_cflags ${w} + enabled gcc && enabled werror && check_add_cflags -Wno-error=${w} + done + fi + + # ccache only really works on gcc toolchains + enabled gcc || soft_disable ccache + + # Enable the postbuild target if building for visual studio. + case "$tgt_cc" in + vs*) enable_feature msvs + enable_feature solution + vs_version=${tgt_cc##vs} + VCPROJ_SFX=vcxproj + gen_vcproj_cmd=${source_path}/build/make/gen_msvs_vcxproj.sh + enabled werror && gen_vcproj_cmd="${gen_vcproj_cmd} --enable-werror" + all_targets="${all_targets} solution" + INLINE="__forceinline" + ;; + esac + + # Other toolchain specific defaults + case $toolchain in x86*) soft_enable postproc;; esac + + if enabled postproc_visualizer; then + enabled postproc || die "postproc_visualizer requires postproc to be enabled" + fi + + # Enable unit tests by default if we have a working C++ compiler. + case "$toolchain" in + *-vs*) + soft_enable unit_tests + soft_enable webm_io + soft_enable libyuv + ;; + *-android-*) + soft_enable webm_io + soft_enable libyuv + # GTestLog must be modified to use Android logging utilities. + ;; + *-darwin-*) + # iOS/ARM builds do not work with gtest. This does not match + # x86 targets. + ;; + *-iphonesimulator-*) + soft_enable webm_io + soft_enable libyuv + ;; + *-win*) + # Some mingw toolchains don't have pthread available by default. + # Treat these more like visual studio where threading in gtest + # would be disabled for the same reason. + check_cxx "$@" <> ${BUILD_PFX}aom_config.c +#include "aom/aom_codec.h" +static const char* const cfg = "$CONFIGURE_ARGS"; +const char *aom_codec_build_config(void) {return cfg;} +EOF diff --git a/third_party/aom/docs.mk b/third_party/aom/docs.mk new file mode 100644 index 0000000000..0dfc65b75f --- /dev/null +++ b/third_party/aom/docs.mk @@ -0,0 +1,50 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + + +INSTALL_MAPS += docs/% docs/% +INSTALL_MAPS += src/% % +INSTALL_MAPS += % % + +# Static documentation authored in doxygen +CODEC_DOX := mainpage.dox \ + keywords.dox \ + usage.dox \ + usage_cx.dox \ + usage_dx.dox \ + +# Other doxy files sourced in Markdown +TXT_DOX = $(call enabled,TXT_DOX) + +EXAMPLE_PATH += $(SRC_PATH_BARE) #for CHANGELOG, README, etc +EXAMPLE_PATH += $(SRC_PATH_BARE)/examples + +doxyfile: $(if $(findstring examples, $(ALL_TARGETS)),examples.doxy) +doxyfile: libs.doxy_template libs.doxy + @echo " [CREATE] $@" + @cat $^ > $@ + @echo "STRIP_FROM_PATH += $(SRC_PATH_BARE) $(BUILD_ROOT)" >> $@ + @echo "INPUT += $(addprefix $(SRC_PATH_BARE)/,$(CODEC_DOX))" >> $@; + @echo "INPUT += $(TXT_DOX)" >> $@; + @echo "EXAMPLE_PATH += $(EXAMPLE_PATH)" >> $@ + +CLEAN-OBJS += doxyfile $(wildcard docs/html/*) +docs/html/index.html: doxyfile $(CODEC_DOX) $(TXT_DOX) + @echo " [DOXYGEN] $<" + @doxygen $< +DOCS-yes += docs/html/index.html + +DIST-DOCS-yes = $(wildcard docs/html/*) +DIST-DOCS-$(CONFIG_CODEC_SRCS) += $(addprefix src/,$(CODEC_DOX)) +DIST-DOCS-$(CONFIG_CODEC_SRCS) += src/libs.doxy_template +DIST-DOCS-yes += CHANGELOG +DIST-DOCS-yes += README diff --git a/third_party/aom/examples.mk b/third_party/aom/examples.mk new file mode 100644 index 0000000000..95206924ec --- /dev/null +++ b/third_party/aom/examples.mk @@ -0,0 +1,383 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + +LIBYUV_SRCS += third_party/libyuv/include/libyuv/basic_types.h \ + third_party/libyuv/include/libyuv/convert.h \ + third_party/libyuv/include/libyuv/convert_argb.h \ + third_party/libyuv/include/libyuv/convert_from.h \ + third_party/libyuv/include/libyuv/cpu_id.h \ + third_party/libyuv/include/libyuv/planar_functions.h \ + third_party/libyuv/include/libyuv/rotate.h \ + third_party/libyuv/include/libyuv/row.h \ + third_party/libyuv/include/libyuv/scale.h \ + third_party/libyuv/include/libyuv/scale_row.h \ + third_party/libyuv/source/cpu_id.cc \ + third_party/libyuv/source/planar_functions.cc \ + third_party/libyuv/source/row_any.cc \ + third_party/libyuv/source/row_common.cc \ + third_party/libyuv/source/row_gcc.cc \ + third_party/libyuv/source/row_mips.cc \ + third_party/libyuv/source/row_neon.cc \ + third_party/libyuv/source/row_neon64.cc \ + third_party/libyuv/source/row_win.cc \ + third_party/libyuv/source/scale.cc \ + third_party/libyuv/source/scale_any.cc \ + third_party/libyuv/source/scale_common.cc \ + third_party/libyuv/source/scale_gcc.cc \ + third_party/libyuv/source/scale_mips.cc \ + third_party/libyuv/source/scale_neon.cc \ + third_party/libyuv/source/scale_neon64.cc \ + third_party/libyuv/source/scale_win.cc \ + +LIBWEBM_COMMON_SRCS += third_party/libwebm/common/hdr_util.cc \ + third_party/libwebm/common/hdr_util.h \ + third_party/libwebm/common/webmids.h + +LIBWEBM_MUXER_SRCS += third_party/libwebm/mkvmuxer/mkvmuxer.cc \ + third_party/libwebm/mkvmuxer/mkvmuxerutil.cc \ + third_party/libwebm/mkvmuxer/mkvwriter.cc \ + third_party/libwebm/mkvmuxer/mkvmuxer.h \ + third_party/libwebm/mkvmuxer/mkvmuxertypes.h \ + third_party/libwebm/mkvmuxer/mkvmuxerutil.h \ + third_party/libwebm/mkvparser/mkvparser.h \ + third_party/libwebm/mkvmuxer/mkvwriter.h + +LIBWEBM_PARSER_SRCS = third_party/libwebm/mkvparser/mkvparser.cc \ + third_party/libwebm/mkvparser/mkvreader.cc \ + third_party/libwebm/mkvparser/mkvparser.h \ + third_party/libwebm/mkvparser/mkvreader.h + +# Add compile flags and include path for libwebm sources. +ifeq ($(CONFIG_WEBM_IO),yes) + CXXFLAGS += -D__STDC_CONSTANT_MACROS -D__STDC_LIMIT_MACROS + INC_PATH-yes += $(SRC_PATH_BARE)/third_party/libwebm +endif + +# List of examples to build. UTILS are tools meant for distribution +# while EXAMPLES demonstrate specific portions of the API. +UTILS-$(CONFIG_DECODERS) += aomdec.c +aomdec.SRCS += md5_utils.c md5_utils.h +aomdec.SRCS += aom_ports/mem_ops.h +aomdec.SRCS += aom_ports/mem_ops_aligned.h +aomdec.SRCS += aom_ports/msvc.h +aomdec.SRCS += aom_ports/aom_timer.h +aomdec.SRCS += aom/aom_integer.h +aomdec.SRCS += args.c args.h +aomdec.SRCS += ivfdec.c ivfdec.h +aomdec.SRCS += tools_common.c tools_common.h +aomdec.SRCS += y4menc.c y4menc.h +ifeq ($(CONFIG_LIBYUV),yes) + aomdec.SRCS += $(LIBYUV_SRCS) +endif +ifeq ($(CONFIG_WEBM_IO),yes) + aomdec.SRCS += $(LIBWEBM_COMMON_SRCS) + aomdec.SRCS += $(LIBWEBM_MUXER_SRCS) + aomdec.SRCS += $(LIBWEBM_PARSER_SRCS) + aomdec.SRCS += webmdec.cc webmdec.h +endif +aomdec.GUID = BA5FE66F-38DD-E034-F542-B1578C5FB950 +aomdec.DESCRIPTION = Full featured decoder +UTILS-$(CONFIG_ENCODERS) += aomenc.c +aomenc.SRCS += args.c args.h y4minput.c y4minput.h aomenc.h +aomenc.SRCS += ivfdec.c ivfdec.h +aomenc.SRCS += ivfenc.c ivfenc.h +aomenc.SRCS += rate_hist.c rate_hist.h +aomenc.SRCS += tools_common.c tools_common.h +aomenc.SRCS += examples/encoder_util.h examples/encoder_util.c +aomenc.SRCS += warnings.c warnings.h +aomenc.SRCS += aom_ports/mem_ops.h +aomenc.SRCS += aom_ports/mem_ops_aligned.h +aomenc.SRCS += aom_ports/msvc.h +aomenc.SRCS += aom_ports/aom_timer.h +aomenc.SRCS += aomstats.c aomstats.h +ifeq ($(CONFIG_LIBYUV),yes) + aomenc.SRCS += $(LIBYUV_SRCS) +endif +ifeq ($(CONFIG_WEBM_IO),yes) + aomenc.SRCS += $(LIBWEBM_COMMON_SRCS) + aomenc.SRCS += $(LIBWEBM_MUXER_SRCS) + aomenc.SRCS += $(LIBWEBM_PARSER_SRCS) + aomenc.SRCS += webmenc.cc webmenc.h +endif +aomenc.GUID = 548DEC74-7A15-4B2B-AFC3-AA102E7C25C1 +aomenc.DESCRIPTION = Full featured encoder + +ifeq ($(CONFIG_ANALYZER),yes) + EXAMPLES-$(CONFIG_DECODERS) += analyzer.cc + analyzer.GUID = 83827a8c-e3c3-4b19-8832-0cfc206c4496 + analyzer.SRCS += ivfdec.h ivfdec.c + analyzer.SRCS += av1/decoder/inspection.h + analyzer.SRCS += av1/decoder/inspection.c + analyzer.SRCS += video_reader.h video_reader.c + analyzer.SRCS += tools_common.h tools_common.c + analyzer.DESCRIPTION = Bitstream analyzer +endif + +ifeq ($(CONFIG_INSPECTION),yes) +EXAMPLES-$(CONFIG_DECODERS) += inspect.c +inspect.GUID = FA46A420-3356-441F-B0FD-60AA1345C181 +inspect.SRCS += ivfdec.h ivfdec.c +inspect.SRCS += args.c args.h +inspect.SRCS += tools_common.h tools_common.c +inspect.SRCS += video_common.h +inspect.SRCS += video_reader.h video_reader.c +inspect.SRCS += aom_ports/mem_ops.h +inspect.SRCS += aom_ports/mem_ops_aligned.h +inspect.SRCS += aom_ports/msvc.h +inspect.DESCRIPTION = Dump inspection data +endif + +EXAMPLES-$(CONFIG_DECODERS) += simple_decoder.c +simple_decoder.GUID = D3BBF1E9-2427-450D-BBFF-B2843C1D44CC +simple_decoder.SRCS += ivfdec.h ivfdec.c +simple_decoder.SRCS += tools_common.h tools_common.c +simple_decoder.SRCS += video_common.h +simple_decoder.SRCS += video_reader.h video_reader.c +simple_decoder.SRCS += aom_ports/mem_ops.h +simple_decoder.SRCS += aom_ports/mem_ops_aligned.h +simple_decoder.SRCS += aom_ports/msvc.h +simple_decoder.DESCRIPTION = Simplified decoder loop +EXAMPLES-$(CONFIG_DECODERS) += decode_to_md5.c +decode_to_md5.SRCS += md5_utils.h md5_utils.c +decode_to_md5.SRCS += ivfdec.h ivfdec.c +decode_to_md5.SRCS += tools_common.h tools_common.c +decode_to_md5.SRCS += video_common.h +decode_to_md5.SRCS += video_reader.h video_reader.c +decode_to_md5.SRCS += aom_ports/mem_ops.h +decode_to_md5.SRCS += aom_ports/mem_ops_aligned.h +decode_to_md5.SRCS += aom_ports/msvc.h +decode_to_md5.GUID = 59120B9B-2735-4BFE-B022-146CA340FE42 +decode_to_md5.DESCRIPTION = Frame by frame MD5 checksum +EXAMPLES-$(CONFIG_ENCODERS) += simple_encoder.c +simple_encoder.SRCS += ivfenc.h ivfenc.c +simple_encoder.SRCS += tools_common.h tools_common.c +simple_encoder.SRCS += video_common.h +simple_encoder.SRCS += video_writer.h video_writer.c +simple_encoder.SRCS += aom_ports/msvc.h +simple_encoder.GUID = 4607D299-8A71-4D2C-9B1D-071899B6FBFD +simple_encoder.DESCRIPTION = Simplified encoder loop +EXAMPLES-$(CONFIG_AV1_ENCODER) += lossless_encoder.c +lossless_encoder.SRCS += ivfenc.h ivfenc.c +lossless_encoder.SRCS += tools_common.h tools_common.c +lossless_encoder.SRCS += video_common.h +lossless_encoder.SRCS += video_writer.h video_writer.c +lossless_encoder.SRCS += aom_ports/msvc.h +lossless_encoder.GUID = B63C7C88-5348-46DC-A5A6-CC151EF93366 +lossless_encoder.DESCRIPTION = Simplified lossless encoder +EXAMPLES-$(CONFIG_ENCODERS) += twopass_encoder.c +twopass_encoder.SRCS += ivfenc.h ivfenc.c +twopass_encoder.SRCS += tools_common.h tools_common.c +twopass_encoder.SRCS += video_common.h +twopass_encoder.SRCS += video_writer.h video_writer.c +twopass_encoder.SRCS += aom_ports/msvc.h +twopass_encoder.GUID = 73494FA6-4AF9-4763-8FBB-265C92402FD8 +twopass_encoder.DESCRIPTION = Two-pass encoder loop +EXAMPLES-$(CONFIG_DECODERS) += decode_with_drops.c +decode_with_drops.SRCS += ivfdec.h ivfdec.c +decode_with_drops.SRCS += tools_common.h tools_common.c +decode_with_drops.SRCS += video_common.h +decode_with_drops.SRCS += video_reader.h video_reader.c +decode_with_drops.SRCS += aom_ports/mem_ops.h +decode_with_drops.SRCS += aom_ports/mem_ops_aligned.h +decode_with_drops.SRCS += aom_ports/msvc.h +decode_with_drops.GUID = CE5C53C4-8DDA-438A-86ED-0DDD3CDB8D26 +decode_with_drops.DESCRIPTION = Drops frames while decoding +EXAMPLES-$(CONFIG_ENCODERS) += set_maps.c +set_maps.SRCS += ivfenc.h ivfenc.c +set_maps.SRCS += tools_common.h tools_common.c +set_maps.SRCS += video_common.h +set_maps.SRCS += video_writer.h video_writer.c +set_maps.SRCS += aom_ports/msvc.h +set_maps.GUID = ECB2D24D-98B8-4015-A465-A4AF3DCC145F +set_maps.DESCRIPTION = Set active and ROI maps +ifeq ($(CONFIG_ENCODERS),yes) +ifeq ($(CONFIG_DECODERS),yes) +EXAMPLES-$(CONFIG_ENCODERS) += aom_cx_set_ref.c +aom_cx_set_ref.SRCS += ivfenc.h ivfenc.c +aom_cx_set_ref.SRCS += tools_common.h tools_common.c +aom_cx_set_ref.SRCS += examples/encoder_util.h +aom_cx_set_ref.SRCS += examples/encoder_util.c +aom_cx_set_ref.SRCS += video_common.h +aom_cx_set_ref.SRCS += video_writer.h video_writer.c +aom_cx_set_ref.SRCS += aom_ports/msvc.h +aom_cx_set_ref.GUID = C5E31F7F-96F6-48BD-BD3E-10EBF6E8057A +aom_cx_set_ref.DESCRIPTION = AV1 set encoder reference frame +endif +endif + +# Handle extra library flags depending on codec configuration + +# We should not link to math library (libm) on RVCT +# when building for bare-metal targets +ifeq ($(CONFIG_OS_SUPPORT), yes) +CODEC_EXTRA_LIBS-$(CONFIG_AV1) += m +else + ifeq ($(CONFIG_GCC), yes) + CODEC_EXTRA_LIBS-$(CONFIG_AV1) += m + endif +endif +# +# End of specified files. The rest of the build rules should happen +# automagically from here. +# + + +# Examples need different flags based on whether we're building +# from an installed tree or a version controlled tree. Determine +# the proper paths. +ifeq ($(HAVE_ALT_TREE_LAYOUT),yes) + LIB_PATH-yes := $(SRC_PATH_BARE)/../lib + INC_PATH-yes := $(SRC_PATH_BARE)/../include +else + LIB_PATH-yes += $(if $(BUILD_PFX),$(BUILD_PFX),.) + INC_PATH-$(CONFIG_AV1_DECODER) += $(SRC_PATH_BARE)/av1 + INC_PATH-$(CONFIG_AV1_ENCODER) += $(SRC_PATH_BARE)/av1 +endif +INC_PATH-$(CONFIG_LIBYUV) += $(SRC_PATH_BARE)/third_party/libyuv/include +LIB_PATH := $(call enabled,LIB_PATH) +INC_PATH := $(call enabled,INC_PATH) +INTERNAL_CFLAGS = $(addprefix -I,$(INC_PATH)) +INTERNAL_LDFLAGS += $(addprefix -L,$(LIB_PATH)) + + +# Expand list of selected examples to build (as specified above) +UTILS = $(call enabled,UTILS) +EXAMPLES = $(addprefix examples/,$(call enabled,EXAMPLES)) +ALL_EXAMPLES = $(UTILS) $(EXAMPLES) +UTIL_SRCS = $(foreach ex,$(UTILS),$($(ex:.c=).SRCS) $($(ex:.cc=).SRCS)) +ALL_SRCS = $(foreach ex, $(ALL_EXAMPLES), \ + $($(notdir $(ex:.c=)).SRCS) \ + $($(notdir $(ex:.cc=)).SRCS)) +CODEC_EXTRA_LIBS=$(sort $(call enabled,CODEC_EXTRA_LIBS)) + + +# Expand all example sources into a variable containing all sources +# for that example (not just them main one specified in UTILS/EXAMPLES) +# and add this file to the list (for MSVS workspace generation) +EXAMPLES_C = $(filter-out %.cc, $(ALL_EXAMPLES)) +$(foreach ex,$(EXAMPLES_C), \ + $(eval $(notdir $(ex:.c=)).SRCS += $(ex) examples.mk)) +EXAMPLES_CXX = $(filter-out %.c, $(ALL_EXAMPLES)) +$(foreach ex,$(EXAMPLES_CXX), \ + $(eval $(notdir $(ex:.cc=)).SRCS += $(ex) examples.mk)) + +# Create build/install dependencies for all examples. The common case +# is handled here. The MSVS case is handled below. +NOT_MSVS = $(if $(CONFIG_MSVS),,yes) +DIST-BINS-$(NOT_MSVS) += $(addprefix bin/,$(EXAMPLES_C:.c=$(EXE_SFX))) +DIST-BINS-$(NOT_MSVS) += $(addprefix bin/,$(EXAMPLES_CXX:.cc=$(EXE_SFX))) +INSTALL-BINS-$(NOT_MSVS) += $(addprefix bin/,$(UTILS:.c=$(EXE_SFX))) +DIST-SRCS-yes += $(ALL_SRCS) +INSTALL-SRCS-yes += $(UTIL_SRCS) +OBJS-$(NOT_MSVS) += $(call objs,$(ALL_SRCS)) +BINS-$(NOT_MSVS) += $(addprefix $(BUILD_PFX), \ + $(EXAMPLES_C:.c=$(EXE_SFX))) +BINS-$(NOT_MSVS) += $(addprefix $(BUILD_PFX), \ + $(EXAMPLES_CXX:.cc=$(EXE_SFX))) + +# Instantiate linker template for all examples. +CODEC_LIB=$(if $(CONFIG_DEBUG_LIBS),aom_g,aom) +ifneq ($(filter darwin%,$(TGT_OS)),) +SHARED_LIB_SUF=.dylib +else +ifneq ($(filter os2%,$(TGT_OS)),) +SHARED_LIB_SUF=_dll.a +else +SHARED_LIB_SUF=.so +endif +endif +CODEC_LIB_SUF=$(if $(CONFIG_SHARED),$(SHARED_LIB_SUF),.a) +$(foreach bin,$(BINS-yes),\ + $(eval $(bin):$(LIB_PATH)/lib$(CODEC_LIB)$(CODEC_LIB_SUF))\ + $(eval $(call linker_template,$(bin),\ + $(call objs,$($(notdir $(bin:$(EXE_SFX)=)).SRCS)) \ + -l$(CODEC_LIB) $(addprefix -l,$(CODEC_EXTRA_LIBS))\ + ))) + +# The following pairs define a mapping of locations in the distribution +# tree to locations in the source/build trees. +INSTALL_MAPS += src/%.c %.c +INSTALL_MAPS += src/% $(SRC_PATH_BARE)/% +INSTALL_MAPS += bin/% % +INSTALL_MAPS += % % + + +# Set up additional MSVS environment +ifeq ($(CONFIG_MSVS),yes) +CODEC_LIB=$(if $(CONFIG_SHARED),aom,$(if $(CONFIG_STATIC_MSVCRT),aommt,aommd)) +# This variable uses deferred expansion intentionally, since the results of +# $(wildcard) may change during the course of the Make. +VS_PLATFORMS = $(foreach d,$(wildcard */Release/$(CODEC_LIB).lib),$(word 1,$(subst /, ,$(d)))) +INSTALL_MAPS += $(foreach p,$(VS_PLATFORMS),bin/$(p)/% $(p)/Release/%) +endif + +# Build Visual Studio Projects. We use a template here to instantiate +# explicit rules rather than using an implicit rule because we want to +# leverage make's VPATH searching rather than specifying the paths on +# each file in ALL_EXAMPLES. This has the unfortunate side effect that +# touching the source files trigger a rebuild of the project files +# even though there is no real dependency there (the dependency is on +# the makefiles). We may want to revisit this. +define vcproj_template +$(1): $($(1:.$(VCPROJ_SFX)=).SRCS) aom.$(VCPROJ_SFX) + $(if $(quiet),@echo " [vcproj] $$@") + $(qexec)$$(GEN_VCPROJ)\ + --exe\ + --target=$$(TOOLCHAIN)\ + --name=$$(@:.$(VCPROJ_SFX)=)\ + --ver=$$(CONFIG_VS_VERSION)\ + --proj-guid=$$($$(@:.$(VCPROJ_SFX)=).GUID)\ + --src-path-bare="$(SRC_PATH_BARE)" \ + $$(if $$(CONFIG_STATIC_MSVCRT),--static-crt) \ + --out=$$@ $$(INTERNAL_CFLAGS) $$(CFLAGS) \ + $$(INTERNAL_LDFLAGS) $$(LDFLAGS) -l$$(CODEC_LIB) $$^ +endef +ALL_EXAMPLES_BASENAME := $(notdir $(ALL_EXAMPLES)) +PROJECTS-$(CONFIG_MSVS) += $(ALL_EXAMPLES_BASENAME:.c=.$(VCPROJ_SFX)) +INSTALL-BINS-$(CONFIG_MSVS) += $(foreach p,$(VS_PLATFORMS),\ + $(addprefix bin/$(p)/,$(ALL_EXAMPLES_BASENAME:.c=.exe))) +$(foreach proj,$(call enabled,PROJECTS),\ + $(eval $(call vcproj_template,$(proj)))) + +# +# Documentation Rules +# +%.dox: %.c + @echo " [DOXY] $@" + @mkdir -p $(dir $@) + @echo "/*!\page example_$(@F:.dox=) $(@F:.dox=)" > $@ + @echo " \includelineno $(> $@ + @echo "*/" >> $@ + +samples.dox: examples.mk + @echo " [DOXY] $@" + @echo "/*!\page samples Sample Code" > $@ + @echo " This SDK includes a number of sample applications."\ + "Each sample documents a feature of the SDK in both prose"\ + "and the associated C code."\ + "The following samples are included: ">>$@ + @$(foreach ex,$(sort $(notdir $(EXAMPLES:.c=))),\ + echo " - \subpage example_$(ex) $($(ex).DESCRIPTION)" >> $@;) + @echo >> $@ + @echo " In addition, the SDK contains a number of utilities."\ + "Since these utilities are built upon the concepts described"\ + "in the sample code listed above, they are not documented in"\ + "pieces like the samples are. Their source is included here"\ + "for reference. The following utilities are included:" >> $@ + @$(foreach ex,$(sort $(UTILS:.c=)),\ + echo " - \subpage example_$(ex) $($(ex).DESCRIPTION)" >> $@;) + @echo "*/" >> $@ + +CLEAN-OBJS += examples.doxy samples.dox $(ALL_EXAMPLES:.c=.dox) +DOCS-yes += examples.doxy samples.dox +examples.doxy: samples.dox $(ALL_EXAMPLES:.c=.dox) + @echo "INPUT += $^" > $@ diff --git a/third_party/aom/examples/analyzer.cc b/third_party/aom/examples/analyzer.cc new file mode 100644 index 0000000000..591aaf25e7 --- /dev/null +++ b/third_party/aom/examples/analyzer.cc @@ -0,0 +1,695 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include +#include +#include +#include +#include "./tools_common.h" +#include "./video_reader.h" +#include "aom/aom_decoder.h" +#include "aom/aomdx.h" +#include "av1/decoder/accounting.h" +#include "av1/common/onyxc_int.h" +#include "av1/decoder/inspection.h" + +#define OD_SIGNMASK(a) (-((a) < 0)) +#define OD_FLIPSIGNI(a, b) (((a) + OD_SIGNMASK(b)) ^ OD_SIGNMASK(b)) +#define OD_DIV_ROUND(x, y) (((x) + OD_FLIPSIGNI((y) >> 1, x)) / (y)) + +enum { + OD_LUMA_MASK = 1 << 0, + OD_CB_MASK = 1 << 1, + OD_CR_MASK = 1 << 2, + OD_ALL_MASK = OD_LUMA_MASK | OD_CB_MASK | OD_CR_MASK +}; + +class AV1Decoder { + private: + FILE *input; + wxString path; + + AvxVideoReader *reader; + const AvxVideoInfo *info; + const AvxInterface *decoder; + + insp_frame_data frame_data; + + aom_codec_ctx_t codec; + bool show_padding; + + public: + aom_image_t *image; + int frame; + + int plane_mask; + + AV1Decoder(); + ~AV1Decoder(); + + bool open(const wxString &path); + void close(); + bool step(); + + int getWidthPadding() const; + int getHeightPadding() const; + void togglePadding(); + int getWidth() const; + int getHeight() const; + + bool getAccountingStruct(Accounting **acct); + bool setInspectionCallback(); + + static void inspect(void *decoder, void *data); +}; + +AV1Decoder::AV1Decoder() + : reader(NULL), info(NULL), decoder(NULL), show_padding(false), image(NULL), + frame(0) {} + +AV1Decoder::~AV1Decoder() {} + +void AV1Decoder::togglePadding() { show_padding = !show_padding; } + +bool AV1Decoder::open(const wxString &path) { + reader = aom_video_reader_open(path.mb_str()); + if (!reader) { + fprintf(stderr, "Failed to open %s for reading.", path.mb_str().data()); + return false; + } + this->path = path; + info = aom_video_reader_get_info(reader); + decoder = get_aom_decoder_by_fourcc(info->codec_fourcc); + if (!decoder) { + fprintf(stderr, "Unknown input codec."); + return false; + } + printf("Using %s\n", aom_codec_iface_name(decoder->codec_interface())); + if (aom_codec_dec_init(&codec, decoder->codec_interface(), NULL, 0)) { + fprintf(stderr, "Failed to initialize decoder."); + return false; + } + ifd_init(&frame_data, info->frame_width, info->frame_height); + setInspectionCallback(); + return true; +} + +void AV1Decoder::close() {} + +bool AV1Decoder::step() { + if (aom_video_reader_read_frame(reader)) { + size_t frame_size; + const unsigned char *frame_data; + frame_data = aom_video_reader_get_frame(reader, &frame_size); + if (aom_codec_decode(&codec, frame_data, frame_size, NULL, 0)) { + fprintf(stderr, "Failed to decode frame."); + return false; + } else { + aom_codec_iter_t iter = NULL; + image = aom_codec_get_frame(&codec, &iter); + if (image != NULL) { + frame++; + return true; + } + return false; + } + } + return false; +} + +int AV1Decoder::getWidth() const { + return info->frame_width + 2 * getWidthPadding(); +} + +int AV1Decoder::getWidthPadding() const { + return show_padding + ? AOMMAX(info->frame_width + 16, + ALIGN_POWER_OF_TWO(info->frame_width, 6)) - + info->frame_width + : 0; +} + +int AV1Decoder::getHeight() const { + return info->frame_height + 2 * getHeightPadding(); +} + +int AV1Decoder::getHeightPadding() const { + return show_padding + ? AOMMAX(info->frame_height + 16, + ALIGN_POWER_OF_TWO(info->frame_height, 6)) - + info->frame_height + : 0; +} + +bool AV1Decoder::getAccountingStruct(Accounting **accounting) { + return aom_codec_control(&codec, AV1_GET_ACCOUNTING, accounting) == + AOM_CODEC_OK; +} + +bool AV1Decoder::setInspectionCallback() { + aom_inspect_init ii; + ii.inspect_cb = AV1Decoder::inspect; + ii.inspect_ctx = (void *)this; + return aom_codec_control(&codec, AV1_SET_INSPECTION_CALLBACK, &ii) == + AOM_CODEC_OK; +} + +void AV1Decoder::inspect(void *pbi, void *data) { + AV1Decoder *decoder = (AV1Decoder *)data; + ifd_inspect(&decoder->frame_data, pbi); +} + +#define MIN_ZOOM (1) +#define MAX_ZOOM (4) + +class AnalyzerPanel : public wxPanel { + DECLARE_EVENT_TABLE() + + private: + AV1Decoder decoder; + const wxString path; + + int zoom; + unsigned char *pixels; + + const bool bit_accounting; + double *bpp_q3; + + int plane_mask; + + // The display size is the decode size, scaled by the zoom. + int getDisplayWidth() const; + int getDisplayHeight() const; + + bool updateDisplaySize(); + + void computeBitsPerPixel(); + + public: + AnalyzerPanel(wxWindow *parent, const wxString &path, + const bool bit_accounting); + ~AnalyzerPanel(); + + bool open(const wxString &path); + void close(); + void render(); + void togglePadding(); + bool nextFrame(); + void refresh(); + + int getZoom() const; + bool setZoom(int zoom); + + void setShowPlane(bool show_plane, int mask); + + void onPaint(wxPaintEvent &event); // NOLINT +}; + +BEGIN_EVENT_TABLE(AnalyzerPanel, wxPanel) +EVT_PAINT(AnalyzerPanel::onPaint) +END_EVENT_TABLE() + +AnalyzerPanel::AnalyzerPanel(wxWindow *parent, const wxString &path, + const bool bit_accounting) + : wxPanel(parent), path(path), zoom(0), pixels(NULL), + bit_accounting(bit_accounting), bpp_q3(NULL), plane_mask(OD_ALL_MASK) {} + +AnalyzerPanel::~AnalyzerPanel() { close(); } + +void AnalyzerPanel::setShowPlane(bool show_plane, int mask) { + if (show_plane) { + plane_mask |= mask; + } else { + plane_mask &= ~mask; + } +} + +void AnalyzerPanel::render() { + aom_image_t *img = decoder.image; + int y_stride = img->stride[0]; + int cb_stride = img->stride[1]; + int cr_stride = img->stride[2]; + int p_stride = 3 * getDisplayWidth(); + unsigned char *y_row = img->planes[0]; + unsigned char *cb_row = img->planes[1]; + unsigned char *cr_row = img->planes[2]; + unsigned char *p_row = pixels; + int y_width_padding = decoder.getWidthPadding(); + int cb_width_padding = y_width_padding >> 1; + int cr_width_padding = y_width_padding >> 1; + int y_height_padding = decoder.getHeightPadding(); + int cb_height_padding = y_height_padding >> 1; + int cr_height_padding = y_height_padding >> 1; + for (int j = 0; j < decoder.getHeight(); j++) { + unsigned char *y = y_row - y_stride * y_height_padding; + unsigned char *cb = cb_row - cb_stride * cb_height_padding; + unsigned char *cr = cr_row - cr_stride * cr_height_padding; + unsigned char *p = p_row; + for (int i = 0; i < decoder.getWidth(); i++) { + int64_t yval; + int64_t cbval; + int64_t crval; + int pmask; + unsigned rval; + unsigned gval; + unsigned bval; + yval = *(y - y_width_padding); + cbval = *(cb - cb_width_padding); + crval = *(cr - cr_width_padding); + pmask = plane_mask; + if (pmask & OD_LUMA_MASK) { + yval -= 16; + } else { + yval = 128; + } + cbval = ((pmask & OD_CB_MASK) >> 1) * (cbval - 128); + crval = ((pmask & OD_CR_MASK) >> 2) * (crval - 128); + /*This is intentionally slow and very accurate.*/ + rval = OD_CLAMPI(0, (int32_t)OD_DIV_ROUND( + 2916394880000LL * yval + 4490222169144LL * crval, + 9745792000LL), + 65535); + gval = OD_CLAMPI(0, (int32_t)OD_DIV_ROUND(2916394880000LL * yval - + 534117096223LL * cbval - + 1334761232047LL * crval, + 9745792000LL), + 65535); + bval = OD_CLAMPI(0, (int32_t)OD_DIV_ROUND( + 2916394880000LL * yval + 5290866304968LL * cbval, + 9745792000LL), + 65535); + unsigned char *px_row = p; + for (int v = 0; v < zoom; v++) { + unsigned char *px = px_row; + for (int u = 0; u < zoom; u++) { + *(px + 0) = (unsigned char)(rval >> 8); + *(px + 1) = (unsigned char)(gval >> 8); + *(px + 2) = (unsigned char)(bval >> 8); + px += 3; + } + px_row += p_stride; + } + int dc = ((y - y_row) & 1) | (1 - img->x_chroma_shift); + y++; + cb += dc; + cr += dc; + p += zoom * 3; + } + int dc = -((j & 1) | (1 - img->y_chroma_shift)); + y_row += y_stride; + cb_row += dc & cb_stride; + cr_row += dc & cr_stride; + p_row += zoom * p_stride; + } +} + +void AnalyzerPanel::computeBitsPerPixel() { + Accounting *acct; + double bpp_total; + int totals_q3[MAX_SYMBOL_TYPES] = { 0 }; + int sym_count[MAX_SYMBOL_TYPES] = { 0 }; + decoder.getAccountingStruct(&acct); + for (int j = 0; j < decoder.getHeight(); j++) { + for (int i = 0; i < decoder.getWidth(); i++) { + bpp_q3[j * decoder.getWidth() + i] = 0.0; + } + } + bpp_total = 0; + for (int i = 0; i < acct->syms.num_syms; i++) { + AccountingSymbol *s; + s = &acct->syms.syms[i]; + totals_q3[s->id] += s->bits; + sym_count[s->id] += s->samples; + } + printf("=== Frame: %-3i ===\n", decoder.frame - 1); + for (int i = 0; i < acct->syms.dictionary.num_strs; i++) { + if (totals_q3[i]) { + printf("%30s = %10.3f (%f bit/symbol)\n", acct->syms.dictionary.strs[i], + (float)totals_q3[i] / 8, (float)totals_q3[i] / 8 / sym_count[i]); + } + } + printf("\n"); +} + +void AnalyzerPanel::togglePadding() { + decoder.togglePadding(); + updateDisplaySize(); +} + +bool AnalyzerPanel::nextFrame() { + if (decoder.step()) { + refresh(); + return true; + } + return false; +} + +void AnalyzerPanel::refresh() { + if (bit_accounting) { + computeBitsPerPixel(); + } + render(); +} + +int AnalyzerPanel::getDisplayWidth() const { return zoom * decoder.getWidth(); } + +int AnalyzerPanel::getDisplayHeight() const { + return zoom * decoder.getHeight(); +} + +bool AnalyzerPanel::updateDisplaySize() { + unsigned char *p = (unsigned char *)malloc( + sizeof(*p) * 3 * getDisplayWidth() * getDisplayHeight()); + if (p == NULL) { + return false; + } + free(pixels); + pixels = p; + SetSize(getDisplayWidth(), getDisplayHeight()); + return true; +} + +bool AnalyzerPanel::open(const wxString &path) { + if (!decoder.open(path)) { + return false; + } + if (!setZoom(MIN_ZOOM)) { + return false; + } + if (bit_accounting) { + bpp_q3 = (double *)malloc(sizeof(*bpp_q3) * decoder.getWidth() * + decoder.getHeight()); + if (bpp_q3 == NULL) { + fprintf(stderr, "Could not allocate memory for bit accounting\n"); + close(); + return false; + } + } + if (!nextFrame()) { + close(); + return false; + } + SetFocus(); + return true; +} + +void AnalyzerPanel::close() { + decoder.close(); + free(pixels); + pixels = NULL; + free(bpp_q3); + bpp_q3 = NULL; +} + +int AnalyzerPanel::getZoom() const { return zoom; } + +bool AnalyzerPanel::setZoom(int z) { + if (z <= MAX_ZOOM && z >= MIN_ZOOM && zoom != z) { + int old_zoom = zoom; + zoom = z; + if (!updateDisplaySize()) { + zoom = old_zoom; + return false; + } + return true; + } + return false; +} + +void AnalyzerPanel::onPaint(wxPaintEvent &) { + wxBitmap bmp(wxImage(getDisplayWidth(), getDisplayHeight(), pixels, true)); + wxBufferedPaintDC dc(this, bmp); +} + +class AnalyzerFrame : public wxFrame { + DECLARE_EVENT_TABLE() + + private: + AnalyzerPanel *panel; + const bool bit_accounting; + + wxMenu *fileMenu; + wxMenu *viewMenu; + wxMenu *playbackMenu; + + public: + AnalyzerFrame(const bool bit_accounting); // NOLINT + + void onOpen(wxCommandEvent &event); // NOLINT + void onClose(wxCommandEvent &event); // NOLINT + void onQuit(wxCommandEvent &event); // NOLINT + + void onTogglePadding(wxCommandEvent &event); // NOLINT + void onZoomIn(wxCommandEvent &event); // NOLINT + void onZoomOut(wxCommandEvent &event); // NOLINT + void onActualSize(wxCommandEvent &event); // NOLINT + + void onToggleViewMenuCheckBox(wxCommandEvent &event); // NOLINT + void onResetAndToggleViewMenuCheckBox(wxCommandEvent &event); // NOLINT + + void onNextFrame(wxCommandEvent &event); // NOLINT + void onGotoFrame(wxCommandEvent &event); // NOLINT + void onRestart(wxCommandEvent &event); // NOLINT + + void onAbout(wxCommandEvent &event); // NOLINT + + bool open(const wxString &path); + bool setZoom(int zoom); + void updateViewMenu(); +}; + +enum { + wxID_NEXT_FRAME = 6000, + wxID_SHOW_Y, + wxID_SHOW_U, + wxID_SHOW_V, + wxID_GOTO_FRAME, + wxID_RESTART, + wxID_ACTUAL_SIZE, + wxID_PADDING +}; + +BEGIN_EVENT_TABLE(AnalyzerFrame, wxFrame) +EVT_MENU(wxID_OPEN, AnalyzerFrame::onOpen) +EVT_MENU(wxID_CLOSE, AnalyzerFrame::onClose) +EVT_MENU(wxID_EXIT, AnalyzerFrame::onQuit) +EVT_MENU(wxID_PADDING, AnalyzerFrame::onTogglePadding) +EVT_MENU(wxID_ZOOM_IN, AnalyzerFrame::onZoomIn) +EVT_MENU(wxID_ZOOM_OUT, AnalyzerFrame::onZoomOut) +EVT_MENU(wxID_ACTUAL_SIZE, AnalyzerFrame::onActualSize) +EVT_MENU(wxID_SHOW_Y, AnalyzerFrame::onResetAndToggleViewMenuCheckBox) +EVT_MENU(wxID_SHOW_U, AnalyzerFrame::onResetAndToggleViewMenuCheckBox) +EVT_MENU(wxID_SHOW_V, AnalyzerFrame::onResetAndToggleViewMenuCheckBox) +EVT_MENU(wxID_NEXT_FRAME, AnalyzerFrame::onNextFrame) +EVT_MENU(wxID_GOTO_FRAME, AnalyzerFrame::onGotoFrame) +EVT_MENU(wxID_RESTART, AnalyzerFrame::onRestart) +EVT_MENU(wxID_ABOUT, AnalyzerFrame::onAbout) +END_EVENT_TABLE() + +AnalyzerFrame::AnalyzerFrame(const bool bit_accounting) + : wxFrame(NULL, wxID_ANY, _("AV1 Stream Analyzer"), wxDefaultPosition, + wxDefaultSize, wxDEFAULT_FRAME_STYLE), + panel(NULL), bit_accounting(bit_accounting) { + wxMenuBar *mb = new wxMenuBar(); + + fileMenu = new wxMenu(); + fileMenu->Append(wxID_OPEN, _("&Open...\tCtrl-O"), _("Open daala file")); + fileMenu->Append(wxID_CLOSE, _("&Close\tCtrl-W"), _("Close daala file")); + fileMenu->Enable(wxID_CLOSE, false); + fileMenu->Append(wxID_EXIT, _("E&xit\tCtrl-Q"), _("Quit this program")); + mb->Append(fileMenu, _("&File")); + + wxAcceleratorEntry entries[2]; + entries[0].Set(wxACCEL_CTRL, (int)'=', wxID_ZOOM_IN); + entries[1].Set(wxACCEL_CTRL | wxACCEL_SHIFT, (int)'-', wxID_ZOOM_OUT); + wxAcceleratorTable accel(2, entries); + this->SetAcceleratorTable(accel); + + viewMenu = new wxMenu(); + +viewMenu->Append(wxID_PADDING, _("Toggle padding\tCtrl-p"), + _("Show padding")); + viewMenu->Append(wxID_ZOOM_IN, _("Zoom-In\tCtrl-+"), _("Double image size")); + viewMenu->Append(wxID_ZOOM_OUT, _("Zoom-Out\tCtrl--"), _("Half image size")); + viewMenu->Append(wxID_ACTUAL_SIZE, _("Actual size\tCtrl-0"), + _("Actual size of the frame")); + viewMenu->AppendSeparator(); + viewMenu->AppendCheckItem(wxID_SHOW_Y, _("&Y plane\tCtrl-Y"), + _("Show Y plane")); + viewMenu->AppendCheckItem(wxID_SHOW_U, _("&U plane\tCtrl-U"), + _("Show U plane")); + viewMenu->AppendCheckItem(wxID_SHOW_V, _("&V plane\tCtrl-V"), + _("Show V plane")); + mb->Append(viewMenu, _("&View")); + + playbackMenu = new wxMenu(); + playbackMenu->Append(wxID_NEXT_FRAME, _("Next frame\tCtrl-."), + _("Go to next frame")); + /*playbackMenu->Append(wxID_RESTART, _("&Restart\tCtrl-R"), + _("Set video to frame 0")); + playbackMenu->Append(wxID_GOTO_FRAME, _("Jump to Frame\tCtrl-J"), + _("Go to frame number"));*/ + mb->Append(playbackMenu, _("&Playback")); + + wxMenu *helpMenu = new wxMenu(); + helpMenu->Append(wxID_ABOUT, _("&About...\tF1"), _("Show about dialog")); + mb->Append(helpMenu, _("&Help")); + + SetMenuBar(mb); + + CreateStatusBar(1); +} + +void AnalyzerFrame::onOpen(wxCommandEvent &WXUNUSED(event)) { + wxFileDialog openFileDialog(this, _("Open file"), wxEmptyString, + wxEmptyString, _("AV1 files (*.ivf)|*.ivf"), + wxFD_OPEN | wxFD_FILE_MUST_EXIST); + if (openFileDialog.ShowModal() != wxID_CANCEL) { + open(openFileDialog.GetPath()); + } +} + +void AnalyzerFrame::onClose(wxCommandEvent &WXUNUSED(event)) {} + +void AnalyzerFrame::onQuit(wxCommandEvent &WXUNUSED(event)) { Close(true); } + +void AnalyzerFrame::onTogglePadding(wxCommandEvent &WXUNUSED(event)) { + panel->togglePadding(); + SetClientSize(panel->GetSize()); + panel->render(); + panel->Refresh(); +} + +void AnalyzerFrame::onZoomIn(wxCommandEvent &WXUNUSED(event)) { + setZoom(panel->getZoom() + 1); +} + +void AnalyzerFrame::onZoomOut(wxCommandEvent &WXUNUSED(event)) { + setZoom(panel->getZoom() - 1); +} + +void AnalyzerFrame::onActualSize(wxCommandEvent &WXUNUSED(event)) { + setZoom(MIN_ZOOM); +} + +void AnalyzerFrame::onToggleViewMenuCheckBox(wxCommandEvent &event) { // NOLINT + GetMenuBar()->Check(event.GetId(), event.IsChecked()); + updateViewMenu(); +} + +void AnalyzerFrame::onResetAndToggleViewMenuCheckBox( + wxCommandEvent &event) { // NOLINT + int id = event.GetId(); + if (id != wxID_SHOW_Y && id != wxID_SHOW_U && id != wxID_SHOW_V) { + GetMenuBar()->Check(wxID_SHOW_Y, true); + GetMenuBar()->Check(wxID_SHOW_U, true); + GetMenuBar()->Check(wxID_SHOW_V, true); + } + onToggleViewMenuCheckBox(event); +} + +void AnalyzerFrame::onNextFrame(wxCommandEvent &WXUNUSED(event)) { + panel->nextFrame(); + panel->Refresh(false); +} + +void AnalyzerFrame::onGotoFrame(wxCommandEvent &WXUNUSED(event)) {} + +void AnalyzerFrame::onRestart(wxCommandEvent &WXUNUSED(event)) {} + +void AnalyzerFrame::onAbout(wxCommandEvent &WXUNUSED(event)) { + wxAboutDialogInfo info; + info.SetName(_("AV1 Bitstream Analyzer")); + info.SetVersion(_("0.1-beta")); + info.SetDescription( + _("This program implements a bitstream analyzer for AV1")); + info.SetCopyright( + wxT("(C) 2017 Alliance for Open Media ")); + wxAboutBox(info); +} + +bool AnalyzerFrame::open(const wxString &path) { + panel = new AnalyzerPanel(this, path, bit_accounting); + if (panel->open(path)) { + SetClientSize(panel->GetSize()); + return true; + } else { + delete panel; + return false; + } +} + +bool AnalyzerFrame::setZoom(int zoom) { + if (panel->setZoom(zoom)) { + GetMenuBar()->Enable(wxID_ACTUAL_SIZE, zoom != MIN_ZOOM); + GetMenuBar()->Enable(wxID_ZOOM_IN, zoom != MAX_ZOOM); + GetMenuBar()->Enable(wxID_ZOOM_OUT, zoom != MIN_ZOOM); + SetClientSize(panel->GetSize()); + panel->render(); + panel->Refresh(); + return true; + } + return false; +} + +void AnalyzerFrame::updateViewMenu() { + panel->setShowPlane(GetMenuBar()->IsChecked(wxID_SHOW_Y), OD_LUMA_MASK); + panel->setShowPlane(GetMenuBar()->IsChecked(wxID_SHOW_U), OD_CB_MASK); + panel->setShowPlane(GetMenuBar()->IsChecked(wxID_SHOW_V), OD_CR_MASK); + SetClientSize(panel->GetSize()); + panel->render(); + panel->Refresh(false); +} + +class Analyzer : public wxApp { + private: + AnalyzerFrame *frame; + + public: + void OnInitCmdLine(wxCmdLineParser &parser); // NOLINT + bool OnCmdLineParsed(wxCmdLineParser &parser); // NOLINT +}; + +static const wxCmdLineEntryDesc CMD_LINE_DESC[] = { + { wxCMD_LINE_SWITCH, _("h"), _("help"), _("Display this help and exit."), + wxCMD_LINE_VAL_NONE, wxCMD_LINE_OPTION_HELP }, + { wxCMD_LINE_SWITCH, _("a"), _("bit-accounting"), _("Enable bit accounting"), + wxCMD_LINE_VAL_NONE, wxCMD_LINE_PARAM_OPTIONAL }, + { wxCMD_LINE_PARAM, NULL, NULL, _("input.ivf"), wxCMD_LINE_VAL_STRING, + wxCMD_LINE_PARAM_OPTIONAL }, + { wxCMD_LINE_NONE } +}; + +void Analyzer::OnInitCmdLine(wxCmdLineParser &parser) { // NOLINT + parser.SetDesc(CMD_LINE_DESC); + parser.SetSwitchChars(_("-")); +} + +bool Analyzer::OnCmdLineParsed(wxCmdLineParser &parser) { // NOLINT + bool bit_accounting = parser.Found(_("a")); + if (bit_accounting && !CONFIG_ACCOUNTING) { + fprintf(stderr, + "Bit accounting support not found. " + "Recompile with:\n./configure --enable-accounting\n"); + return false; + } + frame = new AnalyzerFrame(parser.Found(_("a"))); + frame->Show(); + if (parser.GetParamCount() > 0) { + return frame->open(parser.GetParam(0)); + } + return true; +} + +void usage_exit(void) { + fprintf(stderr, "uhh\n"); + exit(EXIT_FAILURE); +} + +IMPLEMENT_APP(Analyzer) diff --git a/third_party/aom/examples/aom_cx_set_ref.c b/third_party/aom/examples/aom_cx_set_ref.c new file mode 100644 index 0000000000..ff24fa14a3 --- /dev/null +++ b/third_party/aom/examples/aom_cx_set_ref.c @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// AV1 Set Reference Frame +// ============================ +// +// This is an example demonstrating how to overwrite the AV1 encoder's +// internal reference frame. In the sample we set the last frame to the +// current frame. This technique could be used to bounce between two cameras. +// +// The decoder would also have to set the reference frame to the same value +// on the same frame, or the video will become corrupt. The 'test_decode' +// variable is set to 1 in this example that tests if the encoder and decoder +// results are matching. +// +// Usage +// ----- +// This example encodes a raw video. And the last argument passed in specifies +// the frame number to update the reference frame on. For example, run +// examples/aom_cx_set_ref av1 352 288 in.yuv out.ivf 4 30 +// The parameter is parsed as follows: +// +// +// Extra Variables +// --------------- +// This example maintains the frame number passed on the command line +// in the `update_frame_num` variable. +// +// +// Configuration +// ------------- +// +// The reference frame is updated on the frame specified on the command +// line. +// +// Observing The Effects +// --------------------- +// The encoder and decoder results should be matching when the same reference +// frame setting operation is done in both encoder and decoder. Otherwise, +// the encoder/decoder mismatch would be seen. + +#include +#include +#include + +#include "aom/aomcx.h" +#include "aom/aom_decoder.h" +#include "aom/aom_encoder.h" +#include "examples/encoder_util.h" +#include "./tools_common.h" +#include "./video_writer.h" + +static const char *exec_name; + +void usage_exit() { + fprintf(stderr, + "Usage: %s " + " \n", + exec_name); + exit(EXIT_FAILURE); +} + +static void testing_decode(aom_codec_ctx_t *encoder, aom_codec_ctx_t *decoder, + unsigned int frame_out, int *mismatch_seen) { + aom_image_t enc_img, dec_img; + struct av1_ref_frame ref_enc, ref_dec; + + if (*mismatch_seen) return; + + ref_enc.idx = 0; + ref_dec.idx = 0; + if (aom_codec_control(encoder, AV1_GET_REFERENCE, &ref_enc)) + die_codec(encoder, "Failed to get encoder reference frame"); + enc_img = ref_enc.img; + if (aom_codec_control(decoder, AV1_GET_REFERENCE, &ref_dec)) + die_codec(decoder, "Failed to get decoder reference frame"); + dec_img = ref_dec.img; + + if (!aom_compare_img(&enc_img, &dec_img)) { + int y[4], u[4], v[4]; + + *mismatch_seen = 1; + + aom_find_mismatch(&enc_img, &dec_img, y, u, v); + printf( + "Encode/decode mismatch on frame %d at" + " Y[%d, %d] {%d/%d}," + " U[%d, %d] {%d/%d}," + " V[%d, %d] {%d/%d}", + frame_out, y[0], y[1], y[2], y[3], u[0], u[1], u[2], u[3], v[0], v[1], + v[2], v[3]); + } + + aom_img_free(&enc_img); + aom_img_free(&dec_img); +} + +static int encode_frame(aom_codec_ctx_t *ecodec, aom_image_t *img, + unsigned int frame_in, AvxVideoWriter *writer, + int test_decode, aom_codec_ctx_t *dcodec, + unsigned int *frame_out, int *mismatch_seen) { + int got_pkts = 0; + aom_codec_iter_t iter = NULL; + const aom_codec_cx_pkt_t *pkt = NULL; + int got_data; + const aom_codec_err_t res = + aom_codec_encode(ecodec, img, frame_in, 1, 0, AOM_DL_GOOD_QUALITY); + if (res != AOM_CODEC_OK) die_codec(ecodec, "Failed to encode frame"); + + got_data = 0; + + while ((pkt = aom_codec_get_cx_data(ecodec, &iter)) != NULL) { + got_pkts = 1; + + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + const int keyframe = (pkt->data.frame.flags & AOM_FRAME_IS_KEY) != 0; + + if (!(pkt->data.frame.flags & AOM_FRAME_IS_FRAGMENT)) { + *frame_out += 1; + } + + if (!aom_video_writer_write_frame(writer, pkt->data.frame.buf, + pkt->data.frame.sz, + pkt->data.frame.pts)) { + die_codec(ecodec, "Failed to write compressed frame"); + } + printf(keyframe ? "K" : "."); + fflush(stdout); + got_data = 1; + + // Decode 1 frame. + if (test_decode) { + if (aom_codec_decode(dcodec, pkt->data.frame.buf, + (unsigned int)pkt->data.frame.sz, NULL, 0)) + die_codec(dcodec, "Failed to decode frame."); + } + } + } + + // Mismatch checking + if (got_data && test_decode) { + testing_decode(ecodec, dcodec, *frame_out, mismatch_seen); + } + + return got_pkts; +} + +int main(int argc, char **argv) { + FILE *infile = NULL; + // Encoder + aom_codec_ctx_t ecodec; + aom_codec_enc_cfg_t cfg; + unsigned int frame_in = 0; + aom_image_t raw; + aom_codec_err_t res; + AvxVideoInfo info; + AvxVideoWriter *writer = NULL; + const AvxInterface *encoder = NULL; + + // Test encoder/decoder mismatch. + int test_decode = 1; + // Decoder + aom_codec_ctx_t dcodec; + unsigned int frame_out = 0; + + // The frame number to set reference frame on + unsigned int update_frame_num = 0; + int mismatch_seen = 0; + + const int fps = 30; + const int bitrate = 500; + + const char *codec_arg = NULL; + const char *width_arg = NULL; + const char *height_arg = NULL; + const char *infile_arg = NULL; + const char *outfile_arg = NULL; + const char *update_frame_num_arg = NULL; + unsigned int limit = 0; + exec_name = argv[0]; + + // Clear explicitly, as simply assigning "{ 0 }" generates + // "missing-field-initializers" warning in some compilers. + memset(&ecodec, 0, sizeof(ecodec)); + memset(&cfg, 0, sizeof(cfg)); + memset(&info, 0, sizeof(info)); + + if (argc < 7) die("Invalid number of arguments"); + + codec_arg = argv[1]; + width_arg = argv[2]; + height_arg = argv[3]; + infile_arg = argv[4]; + outfile_arg = argv[5]; + update_frame_num_arg = argv[6]; + + encoder = get_aom_encoder_by_name(codec_arg); + if (!encoder) die("Unsupported codec."); + + update_frame_num = (unsigned int)strtoul(update_frame_num_arg, NULL, 0); + // In AV1, the reference buffers (cm->buffer_pool->frame_bufs[i].buf) are + // allocated while calling aom_codec_encode(), thus, setting reference for + // 1st frame isn't supported. + if (update_frame_num <= 1) { + die("Couldn't parse frame number '%s'\n", update_frame_num_arg); + } + + if (argc > 7) { + limit = (unsigned int)strtoul(argv[7], NULL, 0); + if (update_frame_num > limit) + die("Update frame number couldn't larger than limit\n"); + } + + info.codec_fourcc = encoder->fourcc; + info.frame_width = (int)strtol(width_arg, NULL, 0); + info.frame_height = (int)strtol(height_arg, NULL, 0); + info.time_base.numerator = 1; + info.time_base.denominator = fps; + + if (info.frame_width <= 0 || info.frame_height <= 0 || + (info.frame_width % 2) != 0 || (info.frame_height % 2) != 0) { + die("Invalid frame size: %dx%d", info.frame_width, info.frame_height); + } + + if (!aom_img_alloc(&raw, AOM_IMG_FMT_I420, info.frame_width, + info.frame_height, 1)) { + die("Failed to allocate image."); + } + + printf("Using %s\n", aom_codec_iface_name(encoder->codec_interface())); + + res = aom_codec_enc_config_default(encoder->codec_interface(), &cfg, 0); + if (res) die_codec(&ecodec, "Failed to get default codec config."); + + cfg.g_w = info.frame_width; + cfg.g_h = info.frame_height; + cfg.g_timebase.num = info.time_base.numerator; + cfg.g_timebase.den = info.time_base.denominator; + cfg.rc_target_bitrate = bitrate; + cfg.g_lag_in_frames = 3; + + writer = aom_video_writer_open(outfile_arg, kContainerIVF, &info); + if (!writer) die("Failed to open %s for writing.", outfile_arg); + + if (!(infile = fopen(infile_arg, "rb"))) + die("Failed to open %s for reading.", infile_arg); + + if (aom_codec_enc_init(&ecodec, encoder->codec_interface(), &cfg, 0)) + die_codec(&ecodec, "Failed to initialize encoder"); + + // Disable alt_ref. + if (aom_codec_control(&ecodec, AOME_SET_ENABLEAUTOALTREF, 0)) + die_codec(&ecodec, "Failed to set enable auto alt ref"); + + if (test_decode) { + const AvxInterface *decoder = get_aom_decoder_by_name(codec_arg); + if (aom_codec_dec_init(&dcodec, decoder->codec_interface(), NULL, 0)) + die_codec(&dcodec, "Failed to initialize decoder."); + } + + // Encode frames. + while (aom_img_read(&raw, infile)) { + if (limit && frame_in >= limit) break; + if (update_frame_num > 1 && frame_out + 1 == update_frame_num) { + aom_ref_frame_t ref; + ref.frame_type = AOM_LAST_FRAME; + ref.img = raw; + // Set reference frame in encoder. + if (aom_codec_control(&ecodec, AOM_SET_REFERENCE, &ref)) + die_codec(&ecodec, "Failed to set reference frame"); + printf(" "); + + // If set_reference in decoder is commented out, the enc/dec mismatch + // would be seen. + if (test_decode) { + if (aom_codec_control(&dcodec, AOM_SET_REFERENCE, &ref)) + die_codec(&dcodec, "Failed to set reference frame"); + } + } + + encode_frame(&ecodec, &raw, frame_in, writer, test_decode, &dcodec, + &frame_out, &mismatch_seen); + frame_in++; + if (mismatch_seen) break; + } + + // Flush encoder. + if (!mismatch_seen) + while (encode_frame(&ecodec, NULL, frame_in, writer, test_decode, &dcodec, + &frame_out, &mismatch_seen)) { + } + + printf("\n"); + fclose(infile); + printf("Processed %d frames.\n", frame_out); + + if (test_decode) { + if (!mismatch_seen) + printf("Encoder/decoder results are matching.\n"); + else + printf("Encoder/decoder results are NOT matching.\n"); + } + + if (test_decode) + if (aom_codec_destroy(&dcodec)) + die_codec(&dcodec, "Failed to destroy decoder"); + + aom_img_free(&raw); + if (aom_codec_destroy(&ecodec)) + die_codec(&ecodec, "Failed to destroy encoder."); + + aom_video_writer_close(writer); + + return EXIT_SUCCESS; +} diff --git a/third_party/aom/examples/decode_to_md5.c b/third_party/aom/examples/decode_to_md5.c new file mode 100644 index 0000000000..5ab2532096 --- /dev/null +++ b/third_party/aom/examples/decode_to_md5.c @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// Frame-by-frame MD5 Checksum +// =========================== +// +// This example builds upon the simple decoder loop to show how checksums +// of the decoded output can be generated. These are used for validating +// decoder implementations against the reference implementation, for example. +// +// MD5 algorithm +// ------------- +// The Message-Digest 5 (MD5) is a well known hash function. We have provided +// an implementation derived from the RSA Data Security, Inc. MD5 Message-Digest +// Algorithm for your use. Our implmentation only changes the interface of this +// reference code. You must include the `md5_utils.h` header for access to these +// functions. +// +// Processing The Decoded Data +// --------------------------- +// Each row of the image is passed to the MD5 accumulator. First the Y plane +// is processed, then U, then V. It is important to honor the image's `stride` +// values. + +#include +#include +#include + +#include "aom/aomdx.h" +#include "aom/aom_decoder.h" + +#include "../md5_utils.h" +#include "../tools_common.h" +#include "../video_reader.h" +#include "./aom_config.h" + +static void get_image_md5(const aom_image_t *img, unsigned char digest[16]) { + int plane, y; + MD5Context md5; + + MD5Init(&md5); + + for (plane = 0; plane < 3; ++plane) { + const unsigned char *buf = img->planes[plane]; + const int stride = img->stride[plane]; + const int w = plane ? (img->d_w + 1) >> 1 : img->d_w; + const int h = plane ? (img->d_h + 1) >> 1 : img->d_h; + + for (y = 0; y < h; ++y) { + MD5Update(&md5, buf, w); + buf += stride; + } + } + + MD5Final(digest, &md5); +} + +static void print_md5(FILE *stream, unsigned char digest[16]) { + int i; + + for (i = 0; i < 16; ++i) fprintf(stream, "%02x", digest[i]); +} + +static const char *exec_name; + +void usage_exit(void) { + fprintf(stderr, "Usage: %s \n", exec_name); + exit(EXIT_FAILURE); +} + +int main(int argc, char **argv) { + int frame_cnt = 0; + FILE *outfile = NULL; + aom_codec_ctx_t codec; + AvxVideoReader *reader = NULL; + const AvxVideoInfo *info = NULL; + const AvxInterface *decoder = NULL; + + exec_name = argv[0]; + + if (argc != 3) die("Invalid number of arguments."); + + reader = aom_video_reader_open(argv[1]); + if (!reader) die("Failed to open %s for reading.", argv[1]); + + if (!(outfile = fopen(argv[2], "wb"))) + die("Failed to open %s for writing.", argv[2]); + + info = aom_video_reader_get_info(reader); + + decoder = get_aom_decoder_by_fourcc(info->codec_fourcc); + if (!decoder) die("Unknown input codec."); + + printf("Using %s\n", aom_codec_iface_name(decoder->codec_interface())); + + if (aom_codec_dec_init(&codec, decoder->codec_interface(), NULL, 0)) + die_codec(&codec, "Failed to initialize decoder"); + + while (aom_video_reader_read_frame(reader)) { + aom_codec_iter_t iter = NULL; + aom_image_t *img = NULL; + size_t frame_size = 0; + const unsigned char *frame = + aom_video_reader_get_frame(reader, &frame_size); + if (aom_codec_decode(&codec, frame, (unsigned int)frame_size, NULL, 0)) + die_codec(&codec, "Failed to decode frame"); + + while ((img = aom_codec_get_frame(&codec, &iter)) != NULL) { + unsigned char digest[16]; + + get_image_md5(img, digest); + print_md5(outfile, digest); + fprintf(outfile, " img-%dx%d-%04d.i420\n", img->d_w, img->d_h, + ++frame_cnt); + } + } + + printf("Processed %d frames.\n", frame_cnt); + if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec."); + + aom_video_reader_close(reader); + + fclose(outfile); + return EXIT_SUCCESS; +} diff --git a/third_party/aom/examples/decode_with_drops.c b/third_party/aom/examples/decode_with_drops.c new file mode 100644 index 0000000000..45e0fb027e --- /dev/null +++ b/third_party/aom/examples/decode_with_drops.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// Decode With Drops Example +// ========================= +// +// This is an example utility which drops a series of frames, as specified +// on the command line. This is useful for observing the error recovery +// features of the codec. +// +// Usage +// ----- +// This example adds a single argument to the `simple_decoder` example, +// which specifies the range or pattern of frames to drop. The parameter is +// parsed as follows: +// +// Dropping A Range Of Frames +// -------------------------- +// To drop a range of frames, specify the starting frame and the ending +// frame to drop, separated by a dash. The following command will drop +// frames 5 through 10 (base 1). +// +// $ ./decode_with_drops in.ivf out.i420 5-10 +// +// +// Dropping A Pattern Of Frames +// ---------------------------- +// To drop a pattern of frames, specify the number of frames to drop and +// the number of frames after which to repeat the pattern, separated by +// a forward-slash. The following command will drop 3 of 7 frames. +// Specifically, it will decode 4 frames, then drop 3 frames, and then +// repeat. +// +// $ ./decode_with_drops in.ivf out.i420 3/7 +// +// +// Extra Variables +// --------------- +// This example maintains the pattern passed on the command line in the +// `n`, `m`, and `is_range` variables: +// +// +// Making The Drop Decision +// ------------------------ +// The example decides whether to drop the frame based on the current +// frame number, immediately before decoding the frame. + +#include +#include +#include + +#include "aom/aomdx.h" +#include "aom/aom_decoder.h" + +#include "../tools_common.h" +#include "../video_reader.h" +#include "./aom_config.h" + +static const char *exec_name; + +void usage_exit(void) { + fprintf(stderr, "Usage: %s \n", exec_name); + exit(EXIT_FAILURE); +} + +int main(int argc, char **argv) { + int frame_cnt = 0; + FILE *outfile = NULL; + aom_codec_ctx_t codec; + const AvxInterface *decoder = NULL; + AvxVideoReader *reader = NULL; + const AvxVideoInfo *info = NULL; + int n = 0; + int m = 0; + int is_range = 0; + char *nptr = NULL; + + exec_name = argv[0]; + + if (argc != 4) die("Invalid number of arguments."); + + reader = aom_video_reader_open(argv[1]); + if (!reader) die("Failed to open %s for reading.", argv[1]); + + if (!(outfile = fopen(argv[2], "wb"))) + die("Failed to open %s for writing.", argv[2]); + + n = (int)strtol(argv[3], &nptr, 0); + m = (int)strtol(nptr + 1, NULL, 0); + is_range = (*nptr == '-'); + if (!n || !m || (*nptr != '-' && *nptr != '/')) + die("Couldn't parse pattern %s.\n", argv[3]); + + info = aom_video_reader_get_info(reader); + + decoder = get_aom_decoder_by_fourcc(info->codec_fourcc); + if (!decoder) die("Unknown input codec."); + + printf("Using %s\n", aom_codec_iface_name(decoder->codec_interface())); + + if (aom_codec_dec_init(&codec, decoder->codec_interface(), NULL, 0)) + die_codec(&codec, "Failed to initialize decoder."); + + while (aom_video_reader_read_frame(reader)) { + aom_codec_iter_t iter = NULL; + aom_image_t *img = NULL; + size_t frame_size = 0; + int skip; + const unsigned char *frame = + aom_video_reader_get_frame(reader, &frame_size); + if (aom_codec_decode(&codec, frame, (unsigned int)frame_size, NULL, 0)) + die_codec(&codec, "Failed to decode frame."); + + ++frame_cnt; + + skip = (is_range && frame_cnt >= n && frame_cnt <= m) || + (!is_range && m - (frame_cnt - 1) % m <= n); + + if (!skip) { + putc('.', stdout); + + while ((img = aom_codec_get_frame(&codec, &iter)) != NULL) + aom_img_write(img, outfile); + } else { + putc('X', stdout); + } + + fflush(stdout); + } + + printf("Processed %d frames.\n", frame_cnt); + if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec."); + + printf("Play: ffplay -f rawvideo -pix_fmt yuv420p -s %dx%d %s\n", + info->frame_width, info->frame_height, argv[2]); + + aom_video_reader_close(reader); + fclose(outfile); + + return EXIT_SUCCESS; +} diff --git a/third_party/aom/examples/encoder_util.c b/third_party/aom/examples/encoder_util.c new file mode 100644 index 0000000000..1aa3a7eef4 --- /dev/null +++ b/third_party/aom/examples/encoder_util.c @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// Utility functions used by encoder binaries. + +#include +#include + +#include "./encoder_util.h" +#include "aom/aom_integer.h" + +#define mmin(a, b) ((a) < (b) ? (a) : (b)) + +static void find_mismatch_plane(const aom_image_t *const img1, + const aom_image_t *const img2, int plane, + int use_highbitdepth, int loc[4]) { + const unsigned char *const p1 = img1->planes[plane]; + const int p1_stride = img1->stride[plane] >> use_highbitdepth; + const unsigned char *const p2 = img2->planes[plane]; + const int p2_stride = img2->stride[plane] >> use_highbitdepth; + const uint32_t bsize = 64; + const int is_y_plane = (plane == AOM_PLANE_Y); + const uint32_t bsizex = is_y_plane ? bsize : bsize >> img1->x_chroma_shift; + const uint32_t bsizey = is_y_plane ? bsize : bsize >> img1->y_chroma_shift; + const uint32_t c_w = + is_y_plane ? img1->d_w + : (img1->d_w + img1->x_chroma_shift) >> img1->x_chroma_shift; + const uint32_t c_h = + is_y_plane ? img1->d_h + : (img1->d_h + img1->y_chroma_shift) >> img1->y_chroma_shift; + assert(img1->d_w == img2->d_w && img1->d_h == img2->d_h); + assert(img1->x_chroma_shift == img2->x_chroma_shift && + img1->y_chroma_shift == img2->y_chroma_shift); + loc[0] = loc[1] = loc[2] = loc[3] = -1; + int match = 1; + uint32_t i, j; + for (i = 0; match && i < c_h; i += bsizey) { + for (j = 0; match && j < c_w; j += bsizex) { + const int si = + is_y_plane ? mmin(i + bsizey, c_h) - i : mmin(i + bsizey, c_h - i); + const int sj = + is_y_plane ? mmin(j + bsizex, c_w) - j : mmin(j + bsizex, c_w - j); + int k, l; + for (k = 0; match && k < si; ++k) { + for (l = 0; match && l < sj; ++l) { + const int row = i + k; + const int col = j + l; + const int offset1 = row * p1_stride + col; + const int offset2 = row * p2_stride + col; + const int val1 = use_highbitdepth + ? p1[2 * offset1] | (p1[2 * offset1 + 1] << 8) + : p1[offset1]; + const int val2 = use_highbitdepth + ? p2[2 * offset2] | (p2[2 * offset2 + 1] << 8) + : p2[offset2]; + if (val1 != val2) { + loc[0] = row; + loc[1] = col; + loc[2] = val1; + loc[3] = val2; + match = 0; + break; + } + } + } + } + } +} + +static void find_mismatch_helper(const aom_image_t *const img1, + const aom_image_t *const img2, + int use_highbitdepth, int yloc[4], int uloc[4], + int vloc[4]) { +#if !CONFIG_HIGHBITDEPTH + assert(!use_highbitdepth); +#endif // !CONFIG_HIGHBITDEPTH + find_mismatch_plane(img1, img2, AOM_PLANE_Y, use_highbitdepth, yloc); + find_mismatch_plane(img1, img2, AOM_PLANE_U, use_highbitdepth, uloc); + find_mismatch_plane(img1, img2, AOM_PLANE_V, use_highbitdepth, vloc); +} + +#if CONFIG_HIGHBITDEPTH +void aom_find_mismatch_high(const aom_image_t *const img1, + const aom_image_t *const img2, int yloc[4], + int uloc[4], int vloc[4]) { + find_mismatch_helper(img1, img2, 1, yloc, uloc, vloc); +} +#endif + +void aom_find_mismatch(const aom_image_t *const img1, + const aom_image_t *const img2, int yloc[4], int uloc[4], + int vloc[4]) { + find_mismatch_helper(img1, img2, 0, yloc, uloc, vloc); +} + +int aom_compare_img(const aom_image_t *const img1, + const aom_image_t *const img2) { + uint32_t l_w = img1->d_w; + uint32_t c_w = (img1->d_w + img1->x_chroma_shift) >> img1->x_chroma_shift; + const uint32_t c_h = + (img1->d_h + img1->y_chroma_shift) >> img1->y_chroma_shift; + uint32_t i; + int match = 1; + + match &= (img1->fmt == img2->fmt); + match &= (img1->d_w == img2->d_w); + match &= (img1->d_h == img2->d_h); +#if CONFIG_HIGHBITDEPTH + if (img1->fmt & AOM_IMG_FMT_HIGHBITDEPTH) { + l_w *= 2; + c_w *= 2; + } +#endif + + for (i = 0; i < img1->d_h; ++i) + match &= (memcmp(img1->planes[AOM_PLANE_Y] + i * img1->stride[AOM_PLANE_Y], + img2->planes[AOM_PLANE_Y] + i * img2->stride[AOM_PLANE_Y], + l_w) == 0); + + for (i = 0; i < c_h; ++i) + match &= (memcmp(img1->planes[AOM_PLANE_U] + i * img1->stride[AOM_PLANE_U], + img2->planes[AOM_PLANE_U] + i * img2->stride[AOM_PLANE_U], + c_w) == 0); + + for (i = 0; i < c_h; ++i) + match &= (memcmp(img1->planes[AOM_PLANE_V] + i * img1->stride[AOM_PLANE_V], + img2->planes[AOM_PLANE_V] + i * img2->stride[AOM_PLANE_V], + c_w) == 0); + + return match; +} diff --git a/third_party/aom/examples/encoder_util.h b/third_party/aom/examples/encoder_util.h new file mode 100644 index 0000000000..38deef03d4 --- /dev/null +++ b/third_party/aom/examples/encoder_util.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// Utility functions used by encoder binaries. + +#ifndef EXAMPLES_ENCODER_UTIL_H_ +#define EXAMPLES_ENCODER_UTIL_H_ + +#include "./aom_config.h" +#include "aom/aom_image.h" + +// Returns mismatch location (?loc[0],?loc[1]) and the values at that location +// in img1 (?loc[2]) and img2 (?loc[3]). +#if CONFIG_HIGHBITDEPTH +void aom_find_mismatch_high(const aom_image_t *const img1, + const aom_image_t *const img2, int yloc[4], + int uloc[4], int vloc[4]); +#endif // CONFIG_HIGHBITDEPTH + +void aom_find_mismatch(const aom_image_t *const img1, + const aom_image_t *const img2, int yloc[4], int uloc[4], + int vloc[4]); + +// Returns 1 if the two images match. +int aom_compare_img(const aom_image_t *const img1, + const aom_image_t *const img2); + +#endif // EXAMPLES_ENCODER_UTIL_H_ diff --git a/third_party/aom/examples/inspect.c b/third_party/aom/examples/inspect.c new file mode 100644 index 0000000000..345c0884d1 --- /dev/null +++ b/third_party/aom/examples/inspect.c @@ -0,0 +1,678 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// Inspect Decoder +// ================ +// +// This is a simple decoder loop that writes JSON stats to stdout. This tool +// can also be compiled with Emscripten and used as a library. + +#include +#include +#include + +#include "./args.h" +#ifdef __EMSCRIPTEN__ +#include +#else +#define EMSCRIPTEN_KEEPALIVE +#endif + +#include "aom/aom_decoder.h" +#include "aom/aomdx.h" + +#include "../tools_common.h" +#include "../video_reader.h" +#include "./aom_config.h" +// #include "av1/av1_dx_iface.c" +#include "../av1/common/onyxc_int.h" +#if CONFIG_ACCOUNTING +#include "../av1/decoder/accounting.h" +#endif +#include "../av1/decoder/inspection.h" + +#include "../video_common.h" + +// Max JSON buffer size. +const int MAX_BUFFER = 1024 * 1024 * 32; + +typedef enum { + ACCOUNTING_LAYER = 1, + BLOCK_SIZE_LAYER = 1 << 1, + TRANSFORM_SIZE_LAYER = 1 << 2, + TRANSFORM_TYPE_LAYER = 1 << 3, + MODE_LAYER = 1 << 4, + SKIP_LAYER = 1 << 5, + FILTER_LAYER = 1 << 6, + CDEF_LAYER = 1 << 7, + REFERENCE_FRAME_LAYER = 1 << 8, + MOTION_VECTORS_LAYER = 1 << 9, + UV_MODE_LAYER = 1 << 10, + ALL_LAYERS = (1 << 11) - 1 +} LayerType; + +static LayerType layers = 0; + +static int stop_after = 0; +static int compress = 0; + +static const arg_def_t limit_arg = + ARG_DEF(NULL, "limit", 1, "Stop decoding after n frames"); +static const arg_def_t dump_all_arg = ARG_DEF("A", "all", 0, "Dump All"); +static const arg_def_t compress_arg = + ARG_DEF("x", "compress", 0, "Compress JSON using RLE"); +static const arg_def_t dump_accounting_arg = + ARG_DEF("a", "accounting", 0, "Dump Accounting"); +static const arg_def_t dump_block_size_arg = + ARG_DEF("bs", "blockSize", 0, "Dump Block Size"); +static const arg_def_t dump_motion_vectors_arg = + ARG_DEF("mv", "motionVectors", 0, "Dump Motion Vectors"); +static const arg_def_t dump_transform_size_arg = + ARG_DEF("ts", "transformSize", 0, "Dump Transform Size"); +static const arg_def_t dump_transform_type_arg = + ARG_DEF("tt", "transformType", 0, "Dump Transform Type"); +static const arg_def_t dump_mode_arg = ARG_DEF("m", "mode", 0, "Dump Mode"); +static const arg_def_t dump_uv_mode_arg = + ARG_DEF("uvm", "uv_mode", 0, "Dump UV Intra Prediction Modes"); +static const arg_def_t dump_skip_arg = ARG_DEF("s", "skip", 0, "Dump Skip"); +static const arg_def_t dump_filter_arg = + ARG_DEF("f", "filter", 0, "Dump Filter"); +static const arg_def_t dump_cdef_arg = ARG_DEF("c", "cdef", 0, "Dump CDEF"); +static const arg_def_t dump_reference_frame_arg = + ARG_DEF("r", "referenceFrame", 0, "Dump Reference Frame"); +static const arg_def_t usage_arg = ARG_DEF("h", "help", 0, "Help"); + +static const arg_def_t *main_args[] = { &limit_arg, + &dump_all_arg, + &compress_arg, +#if CONFIG_ACCOUNTING + &dump_accounting_arg, +#endif + &dump_block_size_arg, + &dump_transform_size_arg, + &dump_transform_type_arg, + &dump_mode_arg, + &dump_uv_mode_arg, + &dump_skip_arg, + &dump_filter_arg, +#if CONFIG_CDEF + &dump_cdef_arg, +#endif + &dump_reference_frame_arg, + &dump_motion_vectors_arg, + &usage_arg, + NULL }; +#define ENUM(name) \ + { #name, name } +#define LAST_ENUM \ + { NULL, 0 } +typedef struct map_entry { + const char *name; + int value; +} map_entry; + +const map_entry refs_map[] = { ENUM(INTRA_FRAME), ENUM(LAST_FRAME), +#if CONFIG_EXT_REFS + ENUM(LAST2_FRAME), ENUM(LAST3_FRAME), + ENUM(GOLDEN_FRAME), ENUM(BWDREF_FRAME), + ENUM(ALTREF_FRAME), +#else + ENUM(GOLDEN_FRAME), ENUM(ALTREF_FRAME), +#endif + LAST_ENUM }; + +const map_entry block_size_map[] = { +#if CONFIG_CB4X4 + ENUM(BLOCK_2X2), ENUM(BLOCK_2X4), ENUM(BLOCK_4X2), +#endif + ENUM(BLOCK_4X4), ENUM(BLOCK_4X8), ENUM(BLOCK_8X4), + ENUM(BLOCK_8X8), ENUM(BLOCK_8X16), ENUM(BLOCK_16X8), + ENUM(BLOCK_16X16), ENUM(BLOCK_16X32), ENUM(BLOCK_32X16), + ENUM(BLOCK_32X32), ENUM(BLOCK_32X64), ENUM(BLOCK_64X32), + ENUM(BLOCK_64X64), +#if CONFIG_EXT_PARTITION + ENUM(BLOCK_64X128), ENUM(BLOCK_128X64), ENUM(BLOCK_128X128), +#endif + LAST_ENUM +}; + +const map_entry tx_size_map[] = { +#if CONFIG_CB4X4 + ENUM(TX_2X2), +#endif + ENUM(TX_4X4), ENUM(TX_8X8), ENUM(TX_16X16), ENUM(TX_32X32), +#if CONFIG_TX64X64 + ENUM(TX_64X64), +#endif + ENUM(TX_4X8), ENUM(TX_8X4), ENUM(TX_8X16), ENUM(TX_16X8), + ENUM(TX_16X32), ENUM(TX_32X16), ENUM(TX_4X16), ENUM(TX_16X4), + ENUM(TX_8X32), ENUM(TX_32X8), LAST_ENUM +}; + +const map_entry tx_type_map[] = { ENUM(DCT_DCT), + ENUM(ADST_DCT), + ENUM(DCT_ADST), + ENUM(ADST_ADST), +#if CONFIG_EXT_TX + ENUM(FLIPADST_DCT), + ENUM(DCT_FLIPADST), + ENUM(FLIPADST_FLIPADST), + ENUM(ADST_FLIPADST), + ENUM(FLIPADST_ADST), + ENUM(IDTX), + ENUM(V_DCT), + ENUM(H_DCT), + ENUM(V_ADST), + ENUM(H_ADST), + ENUM(V_FLIPADST), + ENUM(H_FLIPADST), +#endif + LAST_ENUM }; + +const map_entry prediction_mode_map[] = { + ENUM(DC_PRED), ENUM(V_PRED), + ENUM(H_PRED), ENUM(D45_PRED), + ENUM(D135_PRED), ENUM(D117_PRED), + ENUM(D153_PRED), ENUM(D207_PRED), + ENUM(D63_PRED), +#if CONFIG_ALT_INTRA + ENUM(SMOOTH_PRED), +#endif + ENUM(TM_PRED), ENUM(NEARESTMV), + ENUM(NEARMV), ENUM(ZEROMV), + ENUM(NEWMV), +#if CONFIG_EXT_INTER + ENUM(NEWFROMNEARMV), ENUM(NEAREST_NEARESTMV), + ENUM(NEAREST_NEARMV), ENUM(NEAR_NEARESTMV), + ENUM(NEAR_NEARMV), ENUM(NEAREST_NEWMV), + ENUM(NEW_NEARESTMV), ENUM(NEAR_NEWMV), + ENUM(NEW_NEARMV), ENUM(ZERO_ZEROMV), + ENUM(NEW_NEWMV), +#endif + ENUM(INTRA_INVALID), LAST_ENUM +}; + +#define NO_SKIP 0 +#define SKIP 1 + +const map_entry skip_map[] = { ENUM(SKIP), ENUM(NO_SKIP), LAST_ENUM }; + +const map_entry config_map[] = { ENUM(MI_SIZE), LAST_ENUM }; + +static const char *exec_name; + +insp_frame_data frame_data; +int frame_count = 0; +int decoded_frame_count = 0; +aom_codec_ctx_t codec; +AvxVideoReader *reader = NULL; +const AvxVideoInfo *info = NULL; +aom_image_t *img = NULL; + +void on_frame_decoded_dump(char *json) { +#ifdef __EMSCRIPTEN__ + EM_ASM_({ Module.on_frame_decoded_json($0); }, json); +#else + printf("%s", json); +#endif +} + +// Writing out the JSON buffer using snprintf is very slow, especially when +// compiled with emscripten, these functions speed things up quite a bit. +int put_str(char *buffer, const char *str) { + int i; + for (i = 0; str[i] != '\0'; i++) { + buffer[i] = str[i]; + } + return i; +} + +int put_num(char *buffer, char prefix, int num, char suffix) { + int i = 0; + char *buf = buffer; + int is_neg = 0; + if (prefix) { + buf[i++] = prefix; + } + if (num == 0) { + buf[i++] = '0'; + } else { + if (num < 0) { + num = -num; + is_neg = 1; + } + int s = i; + while (num != 0) { + buf[i++] = '0' + (num % 10); + num = num / 10; + } + if (is_neg) { + buf[i++] = '-'; + } + int e = i - 1; + while (s < e) { + int t = buf[s]; + buf[s] = buf[e]; + buf[e] = t; + s++; + e--; + } + } + if (suffix) { + buf[i++] = suffix; + } + return i; +} + +int put_map(char *buffer, const map_entry *map) { + char *buf = buffer; + const map_entry *entry = map; + while (entry->name != NULL) { + *(buf++) = '"'; + buf += put_str(buf, entry->name); + *(buf++) = '"'; + buf += put_num(buf, ':', entry->value, 0); + entry++; + if (entry->name != NULL) { + *(buf++) = ','; + } + } + return buf - buffer; +} + +int put_reference_frame(char *buffer) { + const int mi_rows = frame_data.mi_rows; + const int mi_cols = frame_data.mi_cols; + char *buf = buffer; + int r, c, t; + buf += put_str(buf, " \"referenceFrameMap\": {"); + buf += put_map(buf, refs_map); + buf += put_str(buf, "},\n"); + buf += put_str(buf, " \"referenceFrame\": ["); + for (r = 0; r < mi_rows; ++r) { + *(buf++) = '['; + for (c = 0; c < mi_cols; ++c) { + insp_mi_data *mi = &frame_data.mi_grid[r * mi_cols + c]; + buf += put_num(buf, '[', mi->ref_frame[0], 0); + buf += put_num(buf, ',', mi->ref_frame[1], ']'); + if (compress) { // RLE + for (t = c + 1; t < mi_cols; ++t) { + insp_mi_data *next_mi = &frame_data.mi_grid[r * mi_cols + t]; + if (mi->ref_frame[0] != next_mi->ref_frame[0] || + mi->ref_frame[1] != next_mi->ref_frame[1]) { + break; + } + } + if (t - c > 1) { + *(buf++) = ','; + buf += put_num(buf, '[', t - c - 1, ']'); + c = t - 1; + } + } + if (c < mi_cols - 1) *(buf++) = ','; + } + *(buf++) = ']'; + if (r < mi_rows - 1) *(buf++) = ','; + } + buf += put_str(buf, "],\n"); + return buf - buffer; +} + +int put_motion_vectors(char *buffer) { + const int mi_rows = frame_data.mi_rows; + const int mi_cols = frame_data.mi_cols; + char *buf = buffer; + int r, c, t; + buf += put_str(buf, " \"motionVectors\": ["); + for (r = 0; r < mi_rows; ++r) { + *(buf++) = '['; + for (c = 0; c < mi_cols; ++c) { + insp_mi_data *mi = &frame_data.mi_grid[r * mi_cols + c]; + buf += put_num(buf, '[', mi->mv[0].col, 0); + buf += put_num(buf, ',', mi->mv[0].row, 0); + buf += put_num(buf, ',', mi->mv[1].col, 0); + buf += put_num(buf, ',', mi->mv[1].row, ']'); + if (compress) { // RLE + for (t = c + 1; t < mi_cols; ++t) { + insp_mi_data *next_mi = &frame_data.mi_grid[r * mi_cols + t]; + if (mi->mv[0].col != next_mi->mv[0].col || + mi->mv[0].row != next_mi->mv[0].row || + mi->mv[1].col != next_mi->mv[1].col || + mi->mv[1].row != next_mi->mv[1].row) { + break; + } + } + if (t - c > 1) { + *(buf++) = ','; + buf += put_num(buf, '[', t - c - 1, ']'); + c = t - 1; + } + } + if (c < mi_cols - 1) *(buf++) = ','; + } + *(buf++) = ']'; + if (r < mi_rows - 1) *(buf++) = ','; + } + buf += put_str(buf, "],\n"); + return buf - buffer; +} + +int put_block_info(char *buffer, const map_entry *map, const char *name, + size_t offset) { + const int mi_rows = frame_data.mi_rows; + const int mi_cols = frame_data.mi_cols; + char *buf = buffer; + int r, c, t, v; + if (map) { + buf += snprintf(buf, MAX_BUFFER, " \"%sMap\": {", name); + buf += put_map(buf, map); + buf += put_str(buf, "},\n"); + } + buf += snprintf(buf, MAX_BUFFER, " \"%s\": [", name); + for (r = 0; r < mi_rows; ++r) { + *(buf++) = '['; + for (c = 0; c < mi_cols; ++c) { + insp_mi_data *curr_mi = &frame_data.mi_grid[r * mi_cols + c]; + v = *(((int8_t *)curr_mi) + offset); + buf += put_num(buf, 0, v, 0); + if (compress) { // RLE + for (t = c + 1; t < mi_cols; ++t) { + insp_mi_data *next_mi = &frame_data.mi_grid[r * mi_cols + t]; + if (v != *(((int8_t *)next_mi) + offset)) { + break; + } + } + if (t - c > 1) { + *(buf++) = ','; + buf += put_num(buf, '[', t - c - 1, ']'); + c = t - 1; + } + } + if (c < mi_cols - 1) *(buf++) = ','; + } + *(buf++) = ']'; + if (r < mi_rows - 1) *(buf++) = ','; + } + buf += put_str(buf, "],\n"); + return buf - buffer; +} + +#if CONFIG_ACCOUNTING +int put_accounting(char *buffer) { + char *buf = buffer; + int i; + const Accounting *accounting = frame_data.accounting; + if (accounting == NULL) { + printf("XXX\n"); + return 0; + } + const int num_syms = accounting->syms.num_syms; + const int num_strs = accounting->syms.dictionary.num_strs; + buf += put_str(buf, " \"symbolsMap\": ["); + for (i = 0; i < num_strs; i++) { + buf += snprintf(buf, MAX_BUFFER, "\"%s\"", + accounting->syms.dictionary.strs[i]); + if (i < num_strs - 1) *(buf++) = ','; + } + buf += put_str(buf, "],\n"); + buf += put_str(buf, " \"symbols\": [\n "); + AccountingSymbolContext context; + context.x = -2; + context.y = -2; + AccountingSymbol *sym; + for (i = 0; i < num_syms; i++) { + sym = &accounting->syms.syms[i]; + if (memcmp(&context, &sym->context, sizeof(AccountingSymbolContext)) != 0) { + buf += put_num(buf, '[', sym->context.x, 0); + buf += put_num(buf, ',', sym->context.y, ']'); + } else { + buf += put_num(buf, '[', sym->id, 0); + buf += put_num(buf, ',', sym->bits, 0); + buf += put_num(buf, ',', sym->samples, ']'); + } + context = sym->context; + if (i < num_syms - 1) *(buf++) = ','; + } + buf += put_str(buf, "],\n"); + return buf - buffer; +} +#endif + +void inspect(void *pbi, void *data) { + /* Fetch frame data. */ + ifd_inspect(&frame_data, pbi); + (void)data; + // We allocate enough space and hope we don't write out of bounds. Totally + // unsafe but this speeds things up, especially when compiled to Javascript. + char *buffer = aom_malloc(MAX_BUFFER); + char *buf = buffer; + buf += put_str(buf, "{\n"); + if (layers & BLOCK_SIZE_LAYER) { + buf += put_block_info(buf, block_size_map, "blockSize", + offsetof(insp_mi_data, sb_type)); + } + if (layers & TRANSFORM_SIZE_LAYER) { + buf += put_block_info(buf, tx_size_map, "transformSize", + offsetof(insp_mi_data, tx_size)); + } + if (layers & TRANSFORM_TYPE_LAYER) { + buf += put_block_info(buf, tx_type_map, "transformType", + offsetof(insp_mi_data, tx_type)); + } + if (layers & MODE_LAYER) { + buf += put_block_info(buf, prediction_mode_map, "mode", + offsetof(insp_mi_data, mode)); + } + if (layers & UV_MODE_LAYER) { + buf += put_block_info(buf, prediction_mode_map, "uv_mode", + offsetof(insp_mi_data, uv_mode)); + } + if (layers & SKIP_LAYER) { + buf += put_block_info(buf, skip_map, "skip", offsetof(insp_mi_data, skip)); + } + if (layers & FILTER_LAYER) { + buf += put_block_info(buf, NULL, "filter", offsetof(insp_mi_data, filter)); + } +#if CONFIG_CDEF + if (layers & CDEF_LAYER) { + buf += put_block_info(buf, NULL, "cdef_level", + offsetof(insp_mi_data, cdef_level)); + buf += put_block_info(buf, NULL, "cdef_strength", + offsetof(insp_mi_data, cdef_strength)); + } +#endif + if (layers & MOTION_VECTORS_LAYER) { + buf += put_motion_vectors(buf); + } + if (layers & REFERENCE_FRAME_LAYER) { + buf += put_reference_frame(buf); + } +#if CONFIG_ACCOUNTING + if (layers & ACCOUNTING_LAYER) { + buf += put_accounting(buf); + } +#endif + buf += snprintf(buf, MAX_BUFFER, " \"frame\": %d,\n", decoded_frame_count); + buf += snprintf(buf, MAX_BUFFER, " \"showFrame\": %d,\n", + frame_data.show_frame); + buf += snprintf(buf, MAX_BUFFER, " \"frameType\": %d,\n", + frame_data.frame_type); + buf += snprintf(buf, MAX_BUFFER, " \"baseQIndex\": %d,\n", + frame_data.base_qindex); + buf += snprintf(buf, MAX_BUFFER, " \"tileCols\": %d,\n", + frame_data.tile_mi_cols); + buf += snprintf(buf, MAX_BUFFER, " \"tileRows\": %d,\n", + frame_data.tile_mi_rows); + buf += put_str(buf, " \"config\": {"); + buf += put_map(buf, config_map); + buf += put_str(buf, "},\n"); + buf += snprintf(buf, MAX_BUFFER, " \"configString\": \"%s\"\n", + aom_codec_build_config()); + decoded_frame_count++; + buf += put_str(buf, "},\n"); + *(buf++) = 0; + on_frame_decoded_dump(buffer); + aom_free(buffer); +} + +void ifd_init_cb() { + aom_inspect_init ii; + ii.inspect_cb = inspect; + ii.inspect_ctx = NULL; + aom_codec_control(&codec, AV1_SET_INSPECTION_CALLBACK, &ii); +} + +EMSCRIPTEN_KEEPALIVE +int open_file(char *file) { + if (file == NULL) { + // The JS analyzer puts the .ivf file at this location. + file = "/tmp/input.ivf"; + } + reader = aom_video_reader_open(file); + if (!reader) die("Failed to open %s for reading.", file); + info = aom_video_reader_get_info(reader); + const AvxInterface *decoder = get_aom_decoder_by_fourcc(info->codec_fourcc); + if (!decoder) die("Unknown input codec."); + fprintf(stderr, "Using %s\n", + aom_codec_iface_name(decoder->codec_interface())); + if (aom_codec_dec_init(&codec, decoder->codec_interface(), NULL, 0)) + die_codec(&codec, "Failed to initialize decoder."); + ifd_init(&frame_data, info->frame_width, info->frame_height); + ifd_init_cb(); + return EXIT_SUCCESS; +} + +EMSCRIPTEN_KEEPALIVE +int read_frame() { + if (!aom_video_reader_read_frame(reader)) return EXIT_FAILURE; + img = NULL; + aom_codec_iter_t iter = NULL; + size_t frame_size = 0; + const unsigned char *frame = aom_video_reader_get_frame(reader, &frame_size); + if (aom_codec_decode(&codec, frame, (unsigned int)frame_size, NULL, 0) != + AOM_CODEC_OK) { + die_codec(&codec, "Failed to decode frame."); + } + img = aom_codec_get_frame(&codec, &iter); + if (img == NULL) { + return EXIT_FAILURE; + } + ++frame_count; + return EXIT_SUCCESS; +} + +EMSCRIPTEN_KEEPALIVE +const char *get_aom_codec_build_config() { return aom_codec_build_config(); } + +EMSCRIPTEN_KEEPALIVE +int get_bit_depth() { return img->bit_depth; } + +EMSCRIPTEN_KEEPALIVE +unsigned char *get_plane(int plane) { return img->planes[plane]; } + +EMSCRIPTEN_KEEPALIVE +int get_plane_stride(int plane) { return img->stride[plane]; } + +EMSCRIPTEN_KEEPALIVE +int get_plane_width(int plane) { return aom_img_plane_width(img, plane); } + +EMSCRIPTEN_KEEPALIVE +int get_plane_height(int plane) { return aom_img_plane_height(img, plane); } + +EMSCRIPTEN_KEEPALIVE +int get_frame_width() { return info->frame_width; } + +EMSCRIPTEN_KEEPALIVE +int get_frame_height() { return info->frame_height; } + +static void parse_args(char **argv) { + char **argi, **argj; + struct arg arg; + (void)dump_accounting_arg; + (void)dump_cdef_arg; + for (argi = argj = argv; (*argj = *argi); argi += arg.argv_step) { + arg.argv_step = 1; + if (arg_match(&arg, &dump_block_size_arg, argi)) layers |= BLOCK_SIZE_LAYER; +#if CONFIG_ACCOUNTING + else if (arg_match(&arg, &dump_accounting_arg, argi)) + layers |= ACCOUNTING_LAYER; +#endif + else if (arg_match(&arg, &dump_transform_size_arg, argi)) + layers |= TRANSFORM_SIZE_LAYER; + else if (arg_match(&arg, &dump_transform_type_arg, argi)) + layers |= TRANSFORM_TYPE_LAYER; + else if (arg_match(&arg, &dump_mode_arg, argi)) + layers |= MODE_LAYER; + else if (arg_match(&arg, &dump_uv_mode_arg, argi)) + layers |= UV_MODE_LAYER; + else if (arg_match(&arg, &dump_skip_arg, argi)) + layers |= SKIP_LAYER; + else if (arg_match(&arg, &dump_filter_arg, argi)) + layers |= FILTER_LAYER; +#if CONFIG_CDEF + else if (arg_match(&arg, &dump_cdef_arg, argi)) + layers |= CDEF_LAYER; +#endif + else if (arg_match(&arg, &dump_reference_frame_arg, argi)) + layers |= REFERENCE_FRAME_LAYER; + else if (arg_match(&arg, &dump_motion_vectors_arg, argi)) + layers |= MOTION_VECTORS_LAYER; + else if (arg_match(&arg, &dump_all_arg, argi)) + layers |= ALL_LAYERS; + else if (arg_match(&arg, &compress_arg, argi)) + compress = 1; + else if (arg_match(&arg, &usage_arg, argi)) + usage_exit(); + else if (arg_match(&arg, &limit_arg, argi)) + stop_after = arg_parse_uint(&arg); + else + argj++; + } +} + +static const char *exec_name; + +void usage_exit(void) { + fprintf(stderr, "Usage: %s src_filename \n", exec_name); + fprintf(stderr, "\nOptions:\n"); + arg_show_usage(stderr, main_args); + exit(EXIT_FAILURE); +} + +EMSCRIPTEN_KEEPALIVE +int main(int argc, char **argv) { + exec_name = argv[0]; + parse_args(argv); + if (argc >= 2) { + open_file(argv[1]); + printf("[\n"); + while (1) { + if (stop_after && (decoded_frame_count >= stop_after)) break; + if (read_frame()) break; + } + printf("null\n"); + printf("]"); + } else { + usage_exit(); + } +} + +EMSCRIPTEN_KEEPALIVE +void quit() { + if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec"); + aom_video_reader_close(reader); +} + +EMSCRIPTEN_KEEPALIVE +void set_layers(LayerType v) { layers = v; } + +EMSCRIPTEN_KEEPALIVE +void set_compress(int v) { compress = v; } diff --git a/third_party/aom/examples/lossless_encoder.c b/third_party/aom/examples/lossless_encoder.c new file mode 100644 index 0000000000..32ab18a163 --- /dev/null +++ b/third_party/aom/examples/lossless_encoder.c @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "aom/aom_encoder.h" +#include "aom/aomcx.h" + +#include "../tools_common.h" +#include "../video_writer.h" + +static const char *exec_name; + +void usage_exit(void) { + fprintf(stderr, + "lossless_encoder: Example demonstrating lossless " + "encoding feature. Supports raw input only.\n"); + fprintf(stderr, "Usage: %s \n", exec_name); + exit(EXIT_FAILURE); +} + +static int encode_frame(aom_codec_ctx_t *codec, aom_image_t *img, + int frame_index, int flags, AvxVideoWriter *writer) { + int got_pkts = 0; + aom_codec_iter_t iter = NULL; + const aom_codec_cx_pkt_t *pkt = NULL; + const aom_codec_err_t res = + aom_codec_encode(codec, img, frame_index, 1, flags, AOM_DL_GOOD_QUALITY); + if (res != AOM_CODEC_OK) die_codec(codec, "Failed to encode frame"); + + while ((pkt = aom_codec_get_cx_data(codec, &iter)) != NULL) { + got_pkts = 1; + + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + const int keyframe = (pkt->data.frame.flags & AOM_FRAME_IS_KEY) != 0; + if (!aom_video_writer_write_frame(writer, pkt->data.frame.buf, + pkt->data.frame.sz, + pkt->data.frame.pts)) { + die_codec(codec, "Failed to write compressed frame"); + } + printf(keyframe ? "K" : "."); + fflush(stdout); + } + } + + return got_pkts; +} + +int main(int argc, char **argv) { + FILE *infile = NULL; + aom_codec_ctx_t codec; + aom_codec_enc_cfg_t cfg; + int frame_count = 0; + aom_image_t raw; + aom_codec_err_t res; + AvxVideoInfo info; + AvxVideoWriter *writer = NULL; + const AvxInterface *encoder = NULL; + const int fps = 30; + + exec_name = argv[0]; + + // Clear explicitly, as simply assigning "{ 0 }" generates + // "missing-field-initializers" warning in some compilers. + memset(&info, 0, sizeof(info)); + + if (argc < 5) die("Invalid number of arguments"); + + encoder = get_aom_encoder_by_name("av1"); + if (!encoder) die("Unsupported codec."); + + info.codec_fourcc = encoder->fourcc; + info.frame_width = (int)strtol(argv[1], NULL, 0); + info.frame_height = (int)strtol(argv[2], NULL, 0); + info.time_base.numerator = 1; + info.time_base.denominator = fps; + + if (info.frame_width <= 0 || info.frame_height <= 0 || + (info.frame_width % 2) != 0 || (info.frame_height % 2) != 0) { + die("Invalid frame size: %dx%d", info.frame_width, info.frame_height); + } + + if (!aom_img_alloc(&raw, AOM_IMG_FMT_I420, info.frame_width, + info.frame_height, 1)) { + die("Failed to allocate image."); + } + + printf("Using %s\n", aom_codec_iface_name(encoder->codec_interface())); + + res = aom_codec_enc_config_default(encoder->codec_interface(), &cfg, 0); + if (res) die_codec(&codec, "Failed to get default codec config."); + + cfg.g_w = info.frame_width; + cfg.g_h = info.frame_height; + cfg.g_timebase.num = info.time_base.numerator; + cfg.g_timebase.den = info.time_base.denominator; + + writer = aom_video_writer_open(argv[4], kContainerIVF, &info); + if (!writer) die("Failed to open %s for writing.", argv[4]); + + if (!(infile = fopen(argv[3], "rb"))) + die("Failed to open %s for reading.", argv[3]); + + if (aom_codec_enc_init(&codec, encoder->codec_interface(), &cfg, 0)) + die_codec(&codec, "Failed to initialize encoder"); + + if (aom_codec_control_(&codec, AV1E_SET_LOSSLESS, 1)) + die_codec(&codec, "Failed to use lossless mode"); + + // Encode frames. + while (aom_img_read(&raw, infile)) { + encode_frame(&codec, &raw, frame_count++, 0, writer); + } + + // Flush encoder. + while (encode_frame(&codec, NULL, -1, 0, writer)) { + } + + printf("\n"); + fclose(infile); + printf("Processed %d frames.\n", frame_count); + + aom_img_free(&raw); + if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec."); + + aom_video_writer_close(writer); + + return EXIT_SUCCESS; +} diff --git a/third_party/aom/examples/resize_util.c b/third_party/aom/examples/resize_util.c new file mode 100644 index 0000000000..5485691a8b --- /dev/null +++ b/third_party/aom/examples/resize_util.c @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include +#include +#include + +#include "../tools_common.h" +#include "../av1/encoder/av1_resize.h" + +static const char *exec_name = NULL; + +static void usage() { + printf("Usage:\n"); + printf("%s x x ", + exec_name); + printf(" []\n"); +} + +void usage_exit(void) { + usage(); + exit(EXIT_FAILURE); +} + +static int parse_dim(char *v, int *width, int *height) { + char *x = strchr(v, 'x'); + if (x == NULL) x = strchr(v, 'X'); + if (x == NULL) return 0; + *width = atoi(v); + *height = atoi(&x[1]); + if (*width <= 0 || *height <= 0) + return 0; + else + return 1; +} + +int main(int argc, char *argv[]) { + char *fin, *fout; + FILE *fpin, *fpout; + uint8_t *inbuf, *outbuf; + uint8_t *inbuf_u, *outbuf_u; + uint8_t *inbuf_v, *outbuf_v; + int f, frames; + int width, height, target_width, target_height; + + exec_name = argv[0]; + + if (argc < 5) { + printf("Incorrect parameters:\n"); + usage(); + return 1; + } + + fin = argv[1]; + fout = argv[4]; + if (!parse_dim(argv[2], &width, &height)) { + printf("Incorrect parameters: %s\n", argv[2]); + usage(); + return 1; + } + if (!parse_dim(argv[3], &target_width, &target_height)) { + printf("Incorrect parameters: %s\n", argv[3]); + usage(); + return 1; + } + + fpin = fopen(fin, "rb"); + if (fpin == NULL) { + printf("Can't open file %s to read\n", fin); + usage(); + return 1; + } + fpout = fopen(fout, "wb"); + if (fpout == NULL) { + printf("Can't open file %s to write\n", fout); + usage(); + return 1; + } + if (argc >= 6) + frames = atoi(argv[5]); + else + frames = INT_MAX; + + printf("Input size: %dx%d\n", width, height); + printf("Target size: %dx%d, Frames: ", target_width, target_height); + if (frames == INT_MAX) + printf("All\n"); + else + printf("%d\n", frames); + + inbuf = (uint8_t *)malloc(width * height * 3 / 2); + outbuf = (uint8_t *)malloc(target_width * target_height * 3 / 2); + inbuf_u = inbuf + width * height; + inbuf_v = inbuf_u + width * height / 4; + outbuf_u = outbuf + target_width * target_height; + outbuf_v = outbuf_u + target_width * target_height / 4; + f = 0; + while (f < frames) { + if (fread(inbuf, width * height * 3 / 2, 1, fpin) != 1) break; + av1_resize_frame420(inbuf, width, inbuf_u, inbuf_v, width / 2, height, + width, outbuf, target_width, outbuf_u, outbuf_v, + target_width / 2, target_height, target_width); + fwrite(outbuf, target_width * target_height * 3 / 2, 1, fpout); + f++; + } + printf("%d frames processed\n", f); + fclose(fpin); + fclose(fpout); + + free(inbuf); + free(outbuf); + return 0; +} diff --git a/third_party/aom/examples/set_maps.c b/third_party/aom/examples/set_maps.c new file mode 100644 index 0000000000..e88cd426f1 --- /dev/null +++ b/third_party/aom/examples/set_maps.c @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// AOM Set Active and ROI Maps +// =========================== +// +// This is an example demonstrating how to control the AOM encoder's +// ROI and Active maps. +// +// ROI (Reigon of Interest) maps are a way for the application to assign +// each macroblock in the image to a region, and then set quantizer and +// filtering parameters on that image. +// +// Active maps are a way for the application to specify on a +// macroblock-by-macroblock basis whether there is any activity in that +// macroblock. +// +// +// Configuration +// ------------- +// An ROI map is set on frame 22. If the width of the image in macroblocks +// is evenly divisble by 4, then the output will appear to have distinct +// columns, where the quantizer, loopfilter, and static threshold differ +// from column to column. +// +// An active map is set on frame 33. If the width of the image in macroblocks +// is evenly divisble by 4, then the output will appear to have distinct +// columns, where one column will have motion and the next will not. +// +// The active map is cleared on frame 44. +// +// Observing The Effects +// --------------------- +// Use the `simple_decoder` example to decode this sample, and observe +// the change in the image at frames 22, 33, and 44. + +#include +#include +#include +#include + +#include "aom/aomcx.h" +#include "aom/aom_encoder.h" + +#include "../tools_common.h" +#include "../video_writer.h" + +static const char *exec_name; + +void usage_exit(void) { + fprintf(stderr, "Usage: %s \n", + exec_name); + exit(EXIT_FAILURE); +} + +static void set_active_map(const aom_codec_enc_cfg_t *cfg, + aom_codec_ctx_t *codec) { + unsigned int i; + aom_active_map_t map = { 0, 0, 0 }; + + map.rows = (cfg->g_h + 15) / 16; + map.cols = (cfg->g_w + 15) / 16; + + map.active_map = (uint8_t *)malloc(map.rows * map.cols); + for (i = 0; i < map.rows * map.cols; ++i) map.active_map[i] = i % 2; + + if (aom_codec_control(codec, AOME_SET_ACTIVEMAP, &map)) + die_codec(codec, "Failed to set active map"); + + free(map.active_map); +} + +static void unset_active_map(const aom_codec_enc_cfg_t *cfg, + aom_codec_ctx_t *codec) { + aom_active_map_t map = { 0, 0, 0 }; + + map.rows = (cfg->g_h + 15) / 16; + map.cols = (cfg->g_w + 15) / 16; + map.active_map = NULL; + + if (aom_codec_control(codec, AOME_SET_ACTIVEMAP, &map)) + die_codec(codec, "Failed to set active map"); +} + +static int encode_frame(aom_codec_ctx_t *codec, aom_image_t *img, + int frame_index, AvxVideoWriter *writer) { + int got_pkts = 0; + aom_codec_iter_t iter = NULL; + const aom_codec_cx_pkt_t *pkt = NULL; + const aom_codec_err_t res = + aom_codec_encode(codec, img, frame_index, 1, 0, AOM_DL_GOOD_QUALITY); + if (res != AOM_CODEC_OK) die_codec(codec, "Failed to encode frame"); + + while ((pkt = aom_codec_get_cx_data(codec, &iter)) != NULL) { + got_pkts = 1; + + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + const int keyframe = (pkt->data.frame.flags & AOM_FRAME_IS_KEY) != 0; + if (!aom_video_writer_write_frame(writer, pkt->data.frame.buf, + pkt->data.frame.sz, + pkt->data.frame.pts)) { + die_codec(codec, "Failed to write compressed frame"); + } + + printf(keyframe ? "K" : "."); + fflush(stdout); + } + } + + return got_pkts; +} + +int main(int argc, char **argv) { + FILE *infile = NULL; + aom_codec_ctx_t codec; + aom_codec_enc_cfg_t cfg; + int frame_count = 0; + const int limit = 30; + aom_image_t raw; + aom_codec_err_t res; + AvxVideoInfo info; + AvxVideoWriter *writer = NULL; + const AvxInterface *encoder = NULL; + const int fps = 2; // TODO(dkovalev) add command line argument + const double bits_per_pixel_per_frame = 0.067; + + exec_name = argv[0]; + if (argc != 6) die("Invalid number of arguments"); + + memset(&info, 0, sizeof(info)); + + encoder = get_aom_encoder_by_name(argv[1]); + if (encoder == NULL) { + die("Unsupported codec."); + } + assert(encoder != NULL); + info.codec_fourcc = encoder->fourcc; + info.frame_width = (int)strtol(argv[2], NULL, 0); + info.frame_height = (int)strtol(argv[3], NULL, 0); + info.time_base.numerator = 1; + info.time_base.denominator = fps; + + if (info.frame_width <= 0 || info.frame_height <= 0 || + (info.frame_width % 2) != 0 || (info.frame_height % 2) != 0) { + die("Invalid frame size: %dx%d", info.frame_width, info.frame_height); + } + + if (!aom_img_alloc(&raw, AOM_IMG_FMT_I420, info.frame_width, + info.frame_height, 1)) { + die("Failed to allocate image."); + } + + printf("Using %s\n", aom_codec_iface_name(encoder->codec_interface())); + + res = aom_codec_enc_config_default(encoder->codec_interface(), &cfg, 0); + if (res) die_codec(&codec, "Failed to get default codec config."); + + cfg.g_w = info.frame_width; + cfg.g_h = info.frame_height; + cfg.g_timebase.num = info.time_base.numerator; + cfg.g_timebase.den = info.time_base.denominator; + cfg.rc_target_bitrate = + (unsigned int)(bits_per_pixel_per_frame * cfg.g_w * cfg.g_h * fps / 1000); + cfg.g_lag_in_frames = 0; + + writer = aom_video_writer_open(argv[5], kContainerIVF, &info); + if (!writer) die("Failed to open %s for writing.", argv[5]); + + if (!(infile = fopen(argv[4], "rb"))) + die("Failed to open %s for reading.", argv[4]); + + if (aom_codec_enc_init(&codec, encoder->codec_interface(), &cfg, 0)) + die_codec(&codec, "Failed to initialize encoder"); + + // Encode frames. + while (aom_img_read(&raw, infile) && frame_count < limit) { + ++frame_count; + + if (frame_count == 11) { + set_active_map(&cfg, &codec); + } else if (frame_count == 22) { + unset_active_map(&cfg, &codec); + } + + encode_frame(&codec, &raw, frame_count, writer); + } + + // Flush encoder. + while (encode_frame(&codec, NULL, -1, writer)) { + } + + printf("\n"); + fclose(infile); + printf("Processed %d frames.\n", frame_count); + + aom_img_free(&raw); + if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec."); + + aom_video_writer_close(writer); + + return EXIT_SUCCESS; +} diff --git a/third_party/aom/examples/simple_decoder.c b/third_party/aom/examples/simple_decoder.c new file mode 100644 index 0000000000..33a8945395 --- /dev/null +++ b/third_party/aom/examples/simple_decoder.c @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// Simple Decoder +// ============== +// +// This is an example of a simple decoder loop. It takes an input file +// containing the compressed data (in IVF format), passes it through the +// decoder, and writes the decompressed frames to disk. Other decoder +// examples build upon this one. +// +// The details of the IVF format have been elided from this example for +// simplicity of presentation, as IVF files will not generally be used by +// your application. In general, an IVF file consists of a file header, +// followed by a variable number of frames. Each frame consists of a frame +// header followed by a variable length payload. The length of the payload +// is specified in the first four bytes of the frame header. The payload is +// the raw compressed data. +// +// Standard Includes +// ----------------- +// For decoders, you only have to include `aom_decoder.h` and then any +// header files for the specific codecs you use. In this case, we're using +// aom. +// +// Initializing The Codec +// ---------------------- +// The libaom decoder is initialized by the call to aom_codec_dec_init(). +// Determining the codec interface to use is handled by AvxVideoReader and the +// functions prefixed with aom_video_reader_. Discussion of those functions is +// beyond the scope of this example, but the main gist is to open the input file +// and parse just enough of it to determine if it's a AVx file and which AVx +// codec is contained within the file. +// Note the NULL pointer passed to aom_codec_dec_init(). We do that in this +// example because we want the algorithm to determine the stream configuration +// (width/height) and allocate memory automatically. +// +// Decoding A Frame +// ---------------- +// Once the frame has been read into memory, it is decoded using the +// `aom_codec_decode` function. The call takes a pointer to the data +// (`frame`) and the length of the data (`frame_size`). No application data +// is associated with the frame in this example, so the `user_priv` +// parameter is NULL. The `deadline` parameter is left at zero for this +// example. This parameter is generally only used when doing adaptive post +// processing. +// +// Codecs may produce a variable number of output frames for every call to +// `aom_codec_decode`. These frames are retrieved by the +// `aom_codec_get_frame` iterator function. The iterator variable `iter` is +// initialized to NULL each time `aom_codec_decode` is called. +// `aom_codec_get_frame` is called in a loop, returning a pointer to a +// decoded image or NULL to indicate the end of list. +// +// Processing The Decoded Data +// --------------------------- +// In this example, we simply write the encoded data to disk. It is +// important to honor the image's `stride` values. +// +// Cleanup +// ------- +// The `aom_codec_destroy` call frees any memory allocated by the codec. +// +// Error Handling +// -------------- +// This example does not special case any error return codes. If there was +// an error, a descriptive message is printed and the program exits. With +// few exceptions, aom_codec functions return an enumerated error status, +// with the value `0` indicating success. + +#include +#include +#include + +#include "aom/aom_decoder.h" + +#include "../tools_common.h" +#include "../video_reader.h" +#include "./aom_config.h" + +static const char *exec_name; + +void usage_exit(void) { + fprintf(stderr, "Usage: %s \n", exec_name); + exit(EXIT_FAILURE); +} + +int main(int argc, char **argv) { + int frame_cnt = 0; + FILE *outfile = NULL; + aom_codec_ctx_t codec; + AvxVideoReader *reader = NULL; + const AvxInterface *decoder = NULL; + const AvxVideoInfo *info = NULL; + + exec_name = argv[0]; + + if (argc != 3) die("Invalid number of arguments."); + + reader = aom_video_reader_open(argv[1]); + if (!reader) die("Failed to open %s for reading.", argv[1]); + + if (!(outfile = fopen(argv[2], "wb"))) + die("Failed to open %s for writing.", argv[2]); + + info = aom_video_reader_get_info(reader); + + decoder = get_aom_decoder_by_fourcc(info->codec_fourcc); + if (!decoder) die("Unknown input codec."); + + printf("Using %s\n", aom_codec_iface_name(decoder->codec_interface())); + + if (aom_codec_dec_init(&codec, decoder->codec_interface(), NULL, 0)) + die_codec(&codec, "Failed to initialize decoder."); + + while (aom_video_reader_read_frame(reader)) { + aom_codec_iter_t iter = NULL; + aom_image_t *img = NULL; + size_t frame_size = 0; + const unsigned char *frame = + aom_video_reader_get_frame(reader, &frame_size); + if (aom_codec_decode(&codec, frame, (unsigned int)frame_size, NULL, 0)) + die_codec(&codec, "Failed to decode frame."); + + while ((img = aom_codec_get_frame(&codec, &iter)) != NULL) { + aom_img_write(img, outfile); + ++frame_cnt; + } + } + + printf("Processed %d frames.\n", frame_cnt); + if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec"); + + printf("Play: ffplay -f rawvideo -pix_fmt yuv420p -s %dx%d %s\n", + info->frame_width, info->frame_height, argv[2]); + + aom_video_reader_close(reader); + + fclose(outfile); + + return EXIT_SUCCESS; +} diff --git a/third_party/aom/examples/simple_encoder.c b/third_party/aom/examples/simple_encoder.c new file mode 100644 index 0000000000..996f6dacf1 --- /dev/null +++ b/third_party/aom/examples/simple_encoder.c @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// Simple Encoder +// ============== +// +// This is an example of a simple encoder loop. It takes an input file in +// YV12 format, passes it through the encoder, and writes the compressed +// frames to disk in IVF format. Other decoder examples build upon this +// one. +// +// The details of the IVF format have been elided from this example for +// simplicity of presentation, as IVF files will not generally be used by +// your application. In general, an IVF file consists of a file header, +// followed by a variable number of frames. Each frame consists of a frame +// header followed by a variable length payload. The length of the payload +// is specified in the first four bytes of the frame header. The payload is +// the raw compressed data. +// +// Standard Includes +// ----------------- +// For encoders, you only have to include `aom_encoder.h` and then any +// header files for the specific codecs you use. In this case, we're using +// aom. +// +// Getting The Default Configuration +// --------------------------------- +// Encoders have the notion of "usage profiles." For example, an encoder +// may want to publish default configurations for both a video +// conferencing application and a best quality offline encoder. These +// obviously have very different default settings. Consult the +// documentation for your codec to see if it provides any default +// configurations. All codecs provide a default configuration, number 0, +// which is valid for material in the vacinity of QCIF/QVGA. +// +// Updating The Configuration +// --------------------------------- +// Almost all applications will want to update the default configuration +// with settings specific to their usage. Here we set the width and height +// of the video file to that specified on the command line. We also scale +// the default bitrate based on the ratio between the default resolution +// and the resolution specified on the command line. +// +// Initializing The Codec +// ---------------------- +// The encoder is initialized by the following code. +// +// Encoding A Frame +// ---------------- +// The frame is read as a continuous block (size width * height * 3 / 2) +// from the input file. If a frame was read (the input file has not hit +// EOF) then the frame is passed to the encoder. Otherwise, a NULL +// is passed, indicating the End-Of-Stream condition to the encoder. The +// `frame_cnt` is reused as the presentation time stamp (PTS) and each +// frame is shown for one frame-time in duration. The flags parameter is +// unused in this example. + +// Forced Keyframes +// ---------------- +// Keyframes can be forced by setting the AOM_EFLAG_FORCE_KF bit of the +// flags passed to `aom_codec_control()`. In this example, we force a +// keyframe every frames. Note, the output stream can +// contain additional keyframes beyond those that have been forced using the +// AOM_EFLAG_FORCE_KF flag because of automatic keyframe placement by the +// encoder. +// +// Processing The Encoded Data +// --------------------------- +// Each packet of type `AOM_CODEC_CX_FRAME_PKT` contains the encoded data +// for this frame. We write a IVF frame header, followed by the raw data. +// +// Cleanup +// ------- +// The `aom_codec_destroy` call frees any memory allocated by the codec. +// +// Error Handling +// -------------- +// This example does not special case any error return codes. If there was +// an error, a descriptive message is printed and the program exits. With +// few exeptions, aom_codec functions return an enumerated error status, +// with the value `0` indicating success. +// +// Error Resiliency Features +// ------------------------- +// Error resiliency is controlled by the g_error_resilient member of the +// configuration structure. Use the `decode_with_drops` example to decode with +// frames 5-10 dropped. Compare the output for a file encoded with this example +// versus one encoded with the `simple_encoder` example. + +#include +#include +#include + +#include "aom/aom_encoder.h" + +#include "../tools_common.h" +#include "../video_writer.h" + +static const char *exec_name; + +void usage_exit(void) { + fprintf(stderr, + "Usage: %s " + " \n" + "See comments in simple_encoder.c for more information.\n", + exec_name); + exit(EXIT_FAILURE); +} + +static int encode_frame(aom_codec_ctx_t *codec, aom_image_t *img, + int frame_index, int flags, AvxVideoWriter *writer) { + int got_pkts = 0; + aom_codec_iter_t iter = NULL; + const aom_codec_cx_pkt_t *pkt = NULL; + const aom_codec_err_t res = + aom_codec_encode(codec, img, frame_index, 1, flags, AOM_DL_GOOD_QUALITY); + if (res != AOM_CODEC_OK) die_codec(codec, "Failed to encode frame"); + + while ((pkt = aom_codec_get_cx_data(codec, &iter)) != NULL) { + got_pkts = 1; + + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + const int keyframe = (pkt->data.frame.flags & AOM_FRAME_IS_KEY) != 0; + if (!aom_video_writer_write_frame(writer, pkt->data.frame.buf, + pkt->data.frame.sz, + pkt->data.frame.pts)) { + die_codec(codec, "Failed to write compressed frame"); + } + printf(keyframe ? "K" : "."); + fflush(stdout); + } + } + + return got_pkts; +} + +// TODO(tomfinegan): Improve command line parsing and add args for bitrate/fps. +int main(int argc, char **argv) { + FILE *infile = NULL; + aom_codec_ctx_t codec; + aom_codec_enc_cfg_t cfg; + int frame_count = 0; + aom_image_t raw; + aom_codec_err_t res; + AvxVideoInfo info; + AvxVideoWriter *writer = NULL; + const AvxInterface *encoder = NULL; + const int fps = 30; + const int bitrate = 200; + int keyframe_interval = 0; + int max_frames = 0; + int frames_encoded = 0; + const char *codec_arg = NULL; + const char *width_arg = NULL; + const char *height_arg = NULL; + const char *infile_arg = NULL; + const char *outfile_arg = NULL; + const char *keyframe_interval_arg = NULL; + + exec_name = argv[0]; + + // Clear explicitly, as simply assigning "{ 0 }" generates + // "missing-field-initializers" warning in some compilers. + memset(&info, 0, sizeof(info)); + + if (argc != 9) die("Invalid number of arguments"); + + codec_arg = argv[1]; + width_arg = argv[2]; + height_arg = argv[3]; + infile_arg = argv[4]; + outfile_arg = argv[5]; + keyframe_interval_arg = argv[6]; + max_frames = (int)strtol(argv[8], NULL, 0); + + encoder = get_aom_encoder_by_name(codec_arg); + if (!encoder) die("Unsupported codec."); + + info.codec_fourcc = encoder->fourcc; + info.frame_width = (int)strtol(width_arg, NULL, 0); + info.frame_height = (int)strtol(height_arg, NULL, 0); + info.time_base.numerator = 1; + info.time_base.denominator = fps; + + if (info.frame_width <= 0 || info.frame_height <= 0 || + (info.frame_width % 2) != 0 || (info.frame_height % 2) != 0) { + die("Invalid frame size: %dx%d", info.frame_width, info.frame_height); + } + + if (!aom_img_alloc(&raw, AOM_IMG_FMT_I420, info.frame_width, + info.frame_height, 1)) { + die("Failed to allocate image."); + } + + keyframe_interval = (int)strtol(keyframe_interval_arg, NULL, 0); + if (keyframe_interval < 0) die("Invalid keyframe interval value."); + + printf("Using %s\n", aom_codec_iface_name(encoder->codec_interface())); + + res = aom_codec_enc_config_default(encoder->codec_interface(), &cfg, 0); + if (res) die_codec(&codec, "Failed to get default codec config."); + + cfg.g_w = info.frame_width; + cfg.g_h = info.frame_height; + cfg.g_timebase.num = info.time_base.numerator; + cfg.g_timebase.den = info.time_base.denominator; + cfg.rc_target_bitrate = bitrate; + cfg.g_error_resilient = (aom_codec_er_flags_t)strtoul(argv[7], NULL, 0); + + writer = aom_video_writer_open(outfile_arg, kContainerIVF, &info); + if (!writer) die("Failed to open %s for writing.", outfile_arg); + + if (!(infile = fopen(infile_arg, "rb"))) + die("Failed to open %s for reading.", infile_arg); + + if (aom_codec_enc_init(&codec, encoder->codec_interface(), &cfg, 0)) + die_codec(&codec, "Failed to initialize encoder"); + + // Encode frames. + while (aom_img_read(&raw, infile)) { + int flags = 0; + if (keyframe_interval > 0 && frame_count % keyframe_interval == 0) + flags |= AOM_EFLAG_FORCE_KF; + encode_frame(&codec, &raw, frame_count++, flags, writer); + frames_encoded++; + if (max_frames > 0 && frames_encoded >= max_frames) break; + } + + // Flush encoder. + while (encode_frame(&codec, NULL, -1, 0, writer)) continue; + + printf("\n"); + fclose(infile); + printf("Processed %d frames.\n", frame_count); + + aom_img_free(&raw); + if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec."); + + aom_video_writer_close(writer); + + return EXIT_SUCCESS; +} diff --git a/third_party/aom/examples/twopass_encoder.c b/third_party/aom/examples/twopass_encoder.c new file mode 100644 index 0000000000..e767bb5d72 --- /dev/null +++ b/third_party/aom/examples/twopass_encoder.c @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// Two Pass Encoder +// ================ +// +// This is an example of a two pass encoder loop. It takes an input file in +// YV12 format, passes it through the encoder twice, and writes the compressed +// frames to disk in IVF format. It builds upon the simple_encoder example. +// +// Twopass Variables +// ----------------- +// Twopass mode needs to track the current pass number and the buffer of +// statistics packets. +// +// Updating The Configuration +// --------------------------------- +// In two pass mode, the configuration has to be updated on each pass. The +// statistics buffer is passed on the last pass. +// +// Encoding A Frame +// ---------------- +// Encoding a frame in two pass mode is identical to the simple encoder +// example. +// +// Processing Statistics Packets +// ----------------------------- +// Each packet of type `AOM_CODEC_CX_FRAME_PKT` contains the encoded data +// for this frame. We write a IVF frame header, followed by the raw data. +// +// +// Pass Progress Reporting +// ----------------------------- +// It's sometimes helpful to see when each pass completes. +// +// +// Clean-up +// ----------------------------- +// Destruction of the encoder instance must be done on each pass. The +// raw image should be destroyed at the end as usual. + +#include +#include +#include + +#include "aom/aom_encoder.h" + +#include "../tools_common.h" +#include "../video_writer.h" + +static const char *exec_name; + +void usage_exit(void) { + fprintf(stderr, + "Usage: %s " + "\n", + exec_name); + exit(EXIT_FAILURE); +} + +static int get_frame_stats(aom_codec_ctx_t *ctx, const aom_image_t *img, + aom_codec_pts_t pts, unsigned int duration, + aom_enc_frame_flags_t flags, unsigned int deadline, + aom_fixed_buf_t *stats) { + int got_pkts = 0; + aom_codec_iter_t iter = NULL; + const aom_codec_cx_pkt_t *pkt = NULL; + const aom_codec_err_t res = + aom_codec_encode(ctx, img, pts, duration, flags, deadline); + if (res != AOM_CODEC_OK) die_codec(ctx, "Failed to get frame stats."); + + while ((pkt = aom_codec_get_cx_data(ctx, &iter)) != NULL) { + got_pkts = 1; + + if (pkt->kind == AOM_CODEC_STATS_PKT) { + const uint8_t *const pkt_buf = pkt->data.twopass_stats.buf; + const size_t pkt_size = pkt->data.twopass_stats.sz; + stats->buf = realloc(stats->buf, stats->sz + pkt_size); + memcpy((uint8_t *)stats->buf + stats->sz, pkt_buf, pkt_size); + stats->sz += pkt_size; + } + } + + return got_pkts; +} + +static int encode_frame(aom_codec_ctx_t *ctx, const aom_image_t *img, + aom_codec_pts_t pts, unsigned int duration, + aom_enc_frame_flags_t flags, unsigned int deadline, + AvxVideoWriter *writer) { + int got_pkts = 0; + aom_codec_iter_t iter = NULL; + const aom_codec_cx_pkt_t *pkt = NULL; + const aom_codec_err_t res = + aom_codec_encode(ctx, img, pts, duration, flags, deadline); + if (res != AOM_CODEC_OK) die_codec(ctx, "Failed to encode frame."); + + while ((pkt = aom_codec_get_cx_data(ctx, &iter)) != NULL) { + got_pkts = 1; + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + const int keyframe = (pkt->data.frame.flags & AOM_FRAME_IS_KEY) != 0; + + if (!aom_video_writer_write_frame(writer, pkt->data.frame.buf, + pkt->data.frame.sz, + pkt->data.frame.pts)) + die_codec(ctx, "Failed to write compressed frame."); + printf(keyframe ? "K" : "."); + fflush(stdout); + } + } + + return got_pkts; +} + +static aom_fixed_buf_t pass0(aom_image_t *raw, FILE *infile, + const AvxInterface *encoder, + const aom_codec_enc_cfg_t *cfg, int limit) { + aom_codec_ctx_t codec; + int frame_count = 0; + aom_fixed_buf_t stats = { NULL, 0 }; + + if (aom_codec_enc_init(&codec, encoder->codec_interface(), cfg, 0)) + die_codec(&codec, "Failed to initialize encoder"); + + // Calculate frame statistics. + while (aom_img_read(raw, infile) && frame_count < limit) { + ++frame_count; + get_frame_stats(&codec, raw, frame_count, 1, 0, AOM_DL_GOOD_QUALITY, + &stats); + } + + // Flush encoder. + while (get_frame_stats(&codec, NULL, frame_count, 1, 0, AOM_DL_GOOD_QUALITY, + &stats)) { + } + + printf("Pass 0 complete. Processed %d frames.\n", frame_count); + if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec."); + + return stats; +} + +static void pass1(aom_image_t *raw, FILE *infile, const char *outfile_name, + const AvxInterface *encoder, const aom_codec_enc_cfg_t *cfg, + int limit) { + AvxVideoInfo info = { encoder->fourcc, + cfg->g_w, + cfg->g_h, + { cfg->g_timebase.num, cfg->g_timebase.den } }; + AvxVideoWriter *writer = NULL; + aom_codec_ctx_t codec; + int frame_count = 0; + + writer = aom_video_writer_open(outfile_name, kContainerIVF, &info); + if (!writer) die("Failed to open %s for writing", outfile_name); + + if (aom_codec_enc_init(&codec, encoder->codec_interface(), cfg, 0)) + die_codec(&codec, "Failed to initialize encoder"); + + // Encode frames. + while (aom_img_read(raw, infile) && frame_count < limit) { + ++frame_count; + encode_frame(&codec, raw, frame_count, 1, 0, AOM_DL_GOOD_QUALITY, writer); + } + + // Flush encoder. + while (encode_frame(&codec, NULL, -1, 1, 0, AOM_DL_GOOD_QUALITY, writer)) { + } + + printf("\n"); + + if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec."); + + aom_video_writer_close(writer); + + printf("Pass 1 complete. Processed %d frames.\n", frame_count); +} + +int main(int argc, char **argv) { + FILE *infile = NULL; + int w, h; + aom_codec_ctx_t codec; + aom_codec_enc_cfg_t cfg; + aom_image_t raw; + aom_codec_err_t res; + aom_fixed_buf_t stats; + + const AvxInterface *encoder = NULL; + const int fps = 30; // TODO(dkovalev) add command line argument + const int bitrate = 200; // kbit/s TODO(dkovalev) add command line argument + const char *const codec_arg = argv[1]; + const char *const width_arg = argv[2]; + const char *const height_arg = argv[3]; + const char *const infile_arg = argv[4]; + const char *const outfile_arg = argv[5]; + int limit = 0; + exec_name = argv[0]; + + if (argc < 6) die("Invalid number of arguments"); + + if (argc > 6) limit = (int)strtol(argv[6], NULL, 0); + + if (limit == 0) limit = 100; + + encoder = get_aom_encoder_by_name(codec_arg); + if (!encoder) die("Unsupported codec."); + + w = (int)strtol(width_arg, NULL, 0); + h = (int)strtol(height_arg, NULL, 0); + + if (w <= 0 || h <= 0 || (w % 2) != 0 || (h % 2) != 0) + die("Invalid frame size: %dx%d", w, h); + + if (!aom_img_alloc(&raw, AOM_IMG_FMT_I420, w, h, 1)) + die("Failed to allocate image", w, h); + + printf("Using %s\n", aom_codec_iface_name(encoder->codec_interface())); + + // Configuration + res = aom_codec_enc_config_default(encoder->codec_interface(), &cfg, 0); + if (res) die_codec(&codec, "Failed to get default codec config."); + + cfg.g_w = w; + cfg.g_h = h; + cfg.g_timebase.num = 1; + cfg.g_timebase.den = fps; + cfg.rc_target_bitrate = bitrate; + + if (!(infile = fopen(infile_arg, "rb"))) + die("Failed to open %s for reading", infile_arg); + + // Pass 0 + cfg.g_pass = AOM_RC_FIRST_PASS; + stats = pass0(&raw, infile, encoder, &cfg, limit); + + // Pass 1 + rewind(infile); + cfg.g_pass = AOM_RC_LAST_PASS; + cfg.rc_twopass_stats_in = stats; + pass1(&raw, infile, outfile_arg, encoder, &cfg, limit); + free(stats.buf); + + aom_img_free(&raw); + fclose(infile); + + return EXIT_SUCCESS; +} diff --git a/third_party/aom/ivfdec.c b/third_party/aom/ivfdec.c new file mode 100644 index 0000000000..fc11b95440 --- /dev/null +++ b/third_party/aom/ivfdec.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "aom_ports/mem_ops.h" + +#include "./ivfdec.h" + +static const char *IVF_SIGNATURE = "DKIF"; + +static void fix_framerate(int *num, int *den) { + if (*den <= 0 || *den >= 1000000000 || *num <= 0 || *num >= 1000) { + // framerate seems to be invalid, just default to 30fps. + *num = 30; + *den = 1; + } +} + +int file_is_ivf(struct AvxInputContext *input_ctx) { + char raw_hdr[32]; + int is_ivf = 0; + + if (fread(raw_hdr, 1, 32, input_ctx->file) == 32) { + if (memcmp(IVF_SIGNATURE, raw_hdr, 4) == 0) { + is_ivf = 1; + + if (mem_get_le16(raw_hdr + 4) != 0) { + fprintf(stderr, + "Error: Unrecognized IVF version! This file may not" + " decode properly."); + } + + input_ctx->fourcc = mem_get_le32(raw_hdr + 8); + input_ctx->width = mem_get_le16(raw_hdr + 12); + input_ctx->height = mem_get_le16(raw_hdr + 14); + input_ctx->framerate.numerator = mem_get_le32(raw_hdr + 16); + input_ctx->framerate.denominator = mem_get_le32(raw_hdr + 20); + fix_framerate(&input_ctx->framerate.numerator, + &input_ctx->framerate.denominator); + } + } + + if (!is_ivf) { + rewind(input_ctx->file); + input_ctx->detect.buf_read = 0; + } else { + input_ctx->detect.position = 4; + } + return is_ivf; +} + +int ivf_read_frame(FILE *infile, uint8_t **buffer, size_t *bytes_read, + size_t *buffer_size) { + char raw_header[IVF_FRAME_HDR_SZ] = { 0 }; + size_t frame_size = 0; + + if (fread(raw_header, IVF_FRAME_HDR_SZ, 1, infile) != 1) { + if (!feof(infile)) warn("Failed to read frame size\n"); + } else { + frame_size = mem_get_le32(raw_header); + + if (frame_size > 256 * 1024 * 1024) { + warn("Read invalid frame size (%u)\n", (unsigned int)frame_size); + frame_size = 0; + } + + if (frame_size > *buffer_size) { + uint8_t *new_buffer = (uint8_t *)realloc(*buffer, 2 * frame_size); + + if (new_buffer) { + *buffer = new_buffer; + *buffer_size = 2 * frame_size; + } else { + warn("Failed to allocate compressed data buffer\n"); + frame_size = 0; + } + } + } + + if (!feof(infile)) { + if (fread(*buffer, 1, frame_size, infile) != frame_size) { + warn("Failed to read full frame\n"); + return 1; + } + + *bytes_read = frame_size; + return 0; + } + + return 1; +} diff --git a/third_party/aom/ivfdec.h b/third_party/aom/ivfdec.h new file mode 100644 index 0000000000..36a6fb84e6 --- /dev/null +++ b/third_party/aom/ivfdec.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef IVFDEC_H_ +#define IVFDEC_H_ + +#include "./tools_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int file_is_ivf(struct AvxInputContext *input); + +int ivf_read_frame(FILE *infile, uint8_t **buffer, size_t *bytes_read, + size_t *buffer_size); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif // IVFDEC_H_ diff --git a/third_party/aom/ivfenc.c b/third_party/aom/ivfenc.c new file mode 100644 index 0000000000..80f4d14e39 --- /dev/null +++ b/third_party/aom/ivfenc.c @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./ivfenc.h" + +#include "aom/aom_encoder.h" +#include "aom_ports/mem_ops.h" + +void ivf_write_file_header(FILE *outfile, const struct aom_codec_enc_cfg *cfg, + unsigned int fourcc, int frame_cnt) { + char header[32]; + + header[0] = 'D'; + header[1] = 'K'; + header[2] = 'I'; + header[3] = 'F'; + mem_put_le16(header + 4, 0); // version + mem_put_le16(header + 6, 32); // header size + mem_put_le32(header + 8, fourcc); // fourcc + mem_put_le16(header + 12, cfg->g_w); // width + mem_put_le16(header + 14, cfg->g_h); // height + mem_put_le32(header + 16, cfg->g_timebase.den); // rate + mem_put_le32(header + 20, cfg->g_timebase.num); // scale + mem_put_le32(header + 24, frame_cnt); // length + mem_put_le32(header + 28, 0); // unused + + fwrite(header, 1, 32, outfile); +} + +void ivf_write_frame_header(FILE *outfile, int64_t pts, size_t frame_size) { + char header[12]; + + mem_put_le32(header, (int)frame_size); + mem_put_le32(header + 4, (int)(pts & 0xFFFFFFFF)); + mem_put_le32(header + 8, (int)(pts >> 32)); + fwrite(header, 1, 12, outfile); +} + +void ivf_write_frame_size(FILE *outfile, size_t frame_size) { + char header[4]; + + mem_put_le32(header, (int)frame_size); + fwrite(header, 1, 4, outfile); +} diff --git a/third_party/aom/ivfenc.h b/third_party/aom/ivfenc.h new file mode 100644 index 0000000000..62b3a91501 --- /dev/null +++ b/third_party/aom/ivfenc.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef IVFENC_H_ +#define IVFENC_H_ + +#include "./tools_common.h" + +struct aom_codec_enc_cfg; +struct aom_codec_cx_pkt; + +#ifdef __cplusplus +extern "C" { +#endif + +void ivf_write_file_header(FILE *outfile, const struct aom_codec_enc_cfg *cfg, + uint32_t fourcc, int frame_cnt); + +void ivf_write_frame_header(FILE *outfile, int64_t pts, size_t frame_size); + +void ivf_write_frame_size(FILE *outfile, size_t frame_size); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif // IVFENC_H_ diff --git a/third_party/aom/keywords.dox b/third_party/aom/keywords.dox new file mode 100644 index 0000000000..56f5368900 --- /dev/null +++ b/third_party/aom/keywords.dox @@ -0,0 +1,51 @@ +/*!\page rfc2119 RFC2119 Keywords + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL + NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and + "OPTIONAL" in this document are to be interpreted as described in + RFC 2119. + +Specifically, the following definitions are used: + +\section MUST +\anchor REQUIRED +\anchor SHALL + This word, or the terms "REQUIRED" or "SHALL", mean that the + definition is an absolute requirement of the specification. + +\section MUSTNOT MUST NOT +\anchor SHALLNOT + This phrase, or the phrase "SHALL NOT", mean that the + definition is an absolute prohibition of the specification. + +\section SHOULD +\anchor RECOMMENDED + This word, or the adjective "RECOMMENDED", mean that there + may exist valid reasons in particular circumstances to ignore a + particular item, but the full implications must be understood and + carefully weighed before choosing a different course. + +\section SHOULDNOT SHOULD NOT +\anchor NOTRECOMMENDED + This phrase, or the phrase "NOT RECOMMENDED" mean that + there may exist valid reasons in particular circumstances when the + particular behavior is acceptable or even useful, but the full + implications should be understood and the case carefully weighed + before implementing any behavior described with this label. + +\section MAY +\anchor OPTIONAL + This word, or the adjective "OPTIONAL", mean that an item is + truly optional. One vendor may choose to include the item because a + particular marketplace requires it or because the vendor feels that + it enhances the product while another vendor may omit the same item. + An implementation which does not include a particular option \ref MUST be + prepared to interoperate with another implementation which does + include the option, though perhaps with reduced functionality. In the + same vein an implementation which does include a particular option + \ref MUST be prepared to interoperate with another implementation which + does not include the option (except, of course, for the feature the + option provides.) + + +*/ diff --git a/third_party/aom/libs.doxy_template b/third_party/aom/libs.doxy_template new file mode 100644 index 0000000000..23400b4299 --- /dev/null +++ b/third_party/aom/libs.doxy_template @@ -0,0 +1,1295 @@ +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +# Doxyfile 1.5.4 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file that +# follow. The default is UTF-8 which is also the encoding used for all text before +# the first occurrence of this tag. Doxygen uses libiconv (or the iconv built into +# libc) for the transcoding. See http://www.gnu.org/software/libiconv for the list of +# possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = "AOMedia Codec SDK" + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = docs + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Finnish, French, German, Greek, Hungarian, +# Italian, Japanese, Japanese-en (Japanese with English messages), Korean, +# Korean-en, Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, +# Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to java_doc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = YES + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a java_doc-style +# comment as the brief description. If set to NO, the java_doc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = YES + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for Java. +# For instance, namespaces will be presented as packages, qualified scopes +# will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to +# include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the defqault) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct (or union) is +# documented as struct with the name of the typedef. So +# typedef struct type_s {} type_t, will appear in the documentation as a struct +# with name type_t. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named type_s. This can typically +# be useful for C code where the coding convention is that all structs are +# typedef'ed and only the typedef is referenced never the struct's name. + +TYPEDEF_HIDES_STRUCT = NO + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be extracted +# and appear in the documentation as a namespace called 'anonymous_namespace{file}', +# where file will be replaced with the base name of the file that contains the anonymous +# namespace. By default anonymous namespace are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = NO + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from the +# version control system). Doxygen will invoke the program by executing (via +# popen()) the command , where is the value of +# the FILE_VERSION_FILTER tag, and is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = YES + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = + +# This tag can be used to specify the character encoding of the source files that +# doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default +# input encoding. Doxygen uses libiconv (or the iconv built into libc) for the transcoding. +# See http://www.gnu.org/software/libiconv for the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 + +FILE_PATTERNS = + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = NO + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the output. +# The symbol name can be a fully qualified name, a word, or if the wildcard * is used, +# a substring. Examples: ANamespace, AClass, AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command , where +# is the value of the INPUT_FILTER tag, and is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. If you have enabled CALL_GRAPH or CALLER_GRAPH +# then you must also enable this option. If you don't then doxygen will produce +# a warning and turn it on anyway + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES (the default) +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = YES + +# If the REFERENCES_RELATION tag is set to YES (the default) +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = YES + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. Otherwise they will link to the documentstion. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = NO + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. For this to work a browser that supports +# java_script and DHTML is required (for instance Mozilla 1.0+, Firefox +# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). + +HTML_DYNAMIC_SECTIONS = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be +# generated containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# java_script, DHTML, CSS and frames is required (for instance Mozilla 1.0+, +# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are +# probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +#--------------------------------------------------------------------------- +# configuration options related to the la_te_x output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = YES + +# The LATEX_OUTPUT tag is used to specify where the la_te_x docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the la_te_x command name to be +# invoked. If left blank `latex' will be used as the default command name. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for la_te_x. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# la_te_x documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = YES + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = letter + +# The EXTRA_PACKAGES tag can be to specify one or more names of la_te_x +# packages that should be included in the la_te_x output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal la_te_x header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the la_te_x that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated la_te_x files. This will instruct la_te_x to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the auto_gen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an auto_gen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and la_te_x code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = YES + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = *.h + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and la_te_x) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see http://www.mcternan.me.uk/mscgen/) to +# produce the chart and insert it in the documentation. The MSCGEN_PATH tag allows you to +# specify the directory where the mscgen tool resides. If left empty the tool is assumed to +# be found in the default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will +# generate a call dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will +# generate a caller dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable caller graphs for selected +# functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MAX_DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the number +# of direct children of the root node in a graph is already larger than +# MAX_DOT_GRAPH_NOTES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, which results in a white background. +# Warning: Depending on the platform used, enabling this option may lead to +# badly anti-aliased labels on the edges of a graph (i.e. they become hard to +# read). + +DOT_TRANSPARENT = YES + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to the search engine +#--------------------------------------------------------------------------- + +# The SEARCHENGINE tag specifies whether or not a search engine should be +# used. If set to NO the values of all tags below this one will be ignored. + +SEARCHENGINE = NO diff --git a/third_party/aom/libs.mk b/third_party/aom/libs.mk new file mode 100644 index 0000000000..4f2e5cab55 --- /dev/null +++ b/third_party/aom/libs.mk @@ -0,0 +1,591 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + +# ARM assembly files are written in RVCT-style. We use some make magic to +# filter those files to allow GCC compilation +ifeq ($(ARCH_ARM),yes) + ASM:=$(if $(filter yes,$(CONFIG_GCC)$(CONFIG_MSVS)),.asm.s,.asm) +else + ASM:=.asm +endif + +# +# Rule to generate runtime cpu detection files +# +define rtcd_h_template +$$(BUILD_PFX)$(1).h: $$(SRC_PATH_BARE)/$(2) + @echo " [CREATE] $$@" + $$(qexec)$$(SRC_PATH_BARE)/build/make/rtcd.pl --arch=$$(TGT_ISA) \ + --sym=$(1) \ + --config=$$(CONFIG_DIR)$$(target)-$$(TOOLCHAIN).mk \ + $$(RTCD_OPTIONS) $$^ > $$@ +CLEAN-OBJS += $$(BUILD_PFX)$(1).h +RTCD += $$(BUILD_PFX)$(1).h +endef + +CODEC_SRCS-yes += CHANGELOG +CODEC_SRCS-yes += libs.mk + +include $(SRC_PATH_BARE)/aom/aom_codec.mk +CODEC_SRCS-yes += $(addprefix aom/,$(call enabled,API_SRCS)) +CODEC_DOC_SRCS += $(addprefix aom/,$(call enabled,API_DOC_SRCS)) + +include $(SRC_PATH_BARE)/aom_mem/aom_mem.mk +CODEC_SRCS-yes += $(addprefix aom_mem/,$(call enabled,MEM_SRCS)) + +include $(SRC_PATH_BARE)/aom_scale/aom_scale.mk +CODEC_SRCS-yes += $(addprefix aom_scale/,$(call enabled,SCALE_SRCS)) + +include $(SRC_PATH_BARE)/aom_ports/aom_ports.mk +CODEC_SRCS-yes += $(addprefix aom_ports/,$(call enabled,PORTS_SRCS)) + +include $(SRC_PATH_BARE)/aom_dsp/aom_dsp.mk +CODEC_SRCS-yes += $(addprefix aom_dsp/,$(call enabled,DSP_SRCS)) + +include $(SRC_PATH_BARE)/aom_util/aom_util.mk +CODEC_SRCS-yes += $(addprefix aom_util/,$(call enabled,UTIL_SRCS)) + +# AV1 make file +ifeq ($(CONFIG_AV1),yes) + AV1_PREFIX=av1/ + include $(SRC_PATH_BARE)/$(AV1_PREFIX)av1_common.mk +endif + +ifeq ($(CONFIG_AV1_ENCODER),yes) + AV1_PREFIX=av1/ + include $(SRC_PATH_BARE)/$(AV1_PREFIX)av1_cx.mk + CODEC_SRCS-yes += $(addprefix $(AV1_PREFIX),$(call enabled,AV1_CX_SRCS)) + CODEC_EXPORTS-yes += $(addprefix $(AV1_PREFIX),$(AV1_CX_EXPORTS)) + CODEC_SRCS-yes += $(AV1_PREFIX)av1_cx.mk aom/aom.h aom/aomcx.h + INSTALL-LIBS-yes += include/aom/aom.h include/aom/aomcx.h + INSTALL_MAPS += include/aom/% $(SRC_PATH_BARE)/$(AV1_PREFIX)/% + CODEC_DOC_SRCS += aom/aom.h aom/aomcx.h + CODEC_DOC_SECTIONS += av1 av1_encoder +endif + +ifeq ($(CONFIG_AV1_DECODER),yes) + AV1_PREFIX=av1/ + include $(SRC_PATH_BARE)/$(AV1_PREFIX)av1_dx.mk + CODEC_SRCS-yes += $(addprefix $(AV1_PREFIX),$(call enabled,AV1_DX_SRCS)) + CODEC_EXPORTS-yes += $(addprefix $(AV1_PREFIX),$(AV1_DX_EXPORTS)) + CODEC_SRCS-yes += $(AV1_PREFIX)av1_dx.mk aom/aom.h aom/aomdx.h + INSTALL-LIBS-yes += include/aom/aom.h include/aom/aomdx.h + INSTALL_MAPS += include/aom/% $(SRC_PATH_BARE)/$(AV1_PREFIX)/% + CODEC_DOC_SRCS += aom/aom.h aom/aomdx.h + CODEC_DOC_SECTIONS += av1 av1_decoder +endif + +AV1_PREFIX=av1/ +$(BUILD_PFX)$(AV1_PREFIX)%.c.o: CFLAGS += -Wextra + +ifeq ($(CONFIG_ENCODERS),yes) + CODEC_DOC_SECTIONS += encoder +endif +ifeq ($(CONFIG_DECODERS),yes) + CODEC_DOC_SECTIONS += decoder +endif + + +ifeq ($(CONFIG_MSVS),yes) +CODEC_LIB=$(if $(CONFIG_STATIC_MSVCRT),aommt,aommd) +GTEST_LIB=$(if $(CONFIG_STATIC_MSVCRT),gtestmt,gtestmd) +# This variable uses deferred expansion intentionally, since the results of +# $(wildcard) may change during the course of the Make. +VS_PLATFORMS = $(foreach d,$(wildcard */Release/$(CODEC_LIB).lib),$(word 1,$(subst /, ,$(d)))) +endif + +# The following pairs define a mapping of locations in the distribution +# tree to locations in the source/build trees. +INSTALL_MAPS += include/aom/% $(SRC_PATH_BARE)/aom/% +INSTALL_MAPS += include/aom/% $(SRC_PATH_BARE)/aom_ports/% +INSTALL_MAPS += $(LIBSUBDIR)/% % +INSTALL_MAPS += src/% $(SRC_PATH_BARE)/% +ifeq ($(CONFIG_MSVS),yes) +INSTALL_MAPS += $(foreach p,$(VS_PLATFORMS),$(LIBSUBDIR)/$(p)/% $(p)/Release/%) +INSTALL_MAPS += $(foreach p,$(VS_PLATFORMS),$(LIBSUBDIR)/$(p)/% $(p)/Debug/%) +endif + +CODEC_SRCS-yes += build/make/version.sh +CODEC_SRCS-yes += build/make/rtcd.pl +CODEC_SRCS-yes += aom_ports/emmintrin_compat.h +CODEC_SRCS-yes += aom_ports/mem_ops.h +CODEC_SRCS-yes += aom_ports/mem_ops_aligned.h +CODEC_SRCS-yes += aom_ports/aom_once.h +CODEC_SRCS-yes += $(BUILD_PFX)aom_config.c +INSTALL-SRCS-no += $(BUILD_PFX)aom_config.c +ifeq ($(ARCH_X86)$(ARCH_X86_64),yes) +INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += third_party/x86inc/x86inc.asm +endif +CODEC_EXPORTS-yes += aom/exports_com +CODEC_EXPORTS-$(CONFIG_ENCODERS) += aom/exports_enc +CODEC_EXPORTS-$(CONFIG_DECODERS) += aom/exports_dec + +INSTALL-LIBS-yes += include/aom/aom_codec.h +INSTALL-LIBS-yes += include/aom/aom_frame_buffer.h +INSTALL-LIBS-yes += include/aom/aom_image.h +INSTALL-LIBS-yes += include/aom/aom_integer.h +INSTALL-LIBS-$(CONFIG_DECODERS) += include/aom/aom_decoder.h +INSTALL-LIBS-$(CONFIG_ENCODERS) += include/aom/aom_encoder.h +ifeq ($(CONFIG_EXTERNAL_BUILD),yes) +ifeq ($(CONFIG_MSVS),yes) +INSTALL-LIBS-yes += $(foreach p,$(VS_PLATFORMS),$(LIBSUBDIR)/$(p)/$(CODEC_LIB).lib) +INSTALL-LIBS-$(CONFIG_DEBUG_LIBS) += $(foreach p,$(VS_PLATFORMS),$(LIBSUBDIR)/$(p)/$(CODEC_LIB)d.lib) +INSTALL-LIBS-$(CONFIG_SHARED) += $(foreach p,$(VS_PLATFORMS),$(LIBSUBDIR)/$(p)/aom.dll) +INSTALL-LIBS-$(CONFIG_SHARED) += $(foreach p,$(VS_PLATFORMS),$(LIBSUBDIR)/$(p)/aom.exp) +endif +else +INSTALL-LIBS-$(CONFIG_STATIC) += $(LIBSUBDIR)/libaom.a +INSTALL-LIBS-$(CONFIG_DEBUG_LIBS) += $(LIBSUBDIR)/libaom_g.a +endif + +CODEC_SRCS=$(call enabled,CODEC_SRCS) +INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(CODEC_SRCS) +INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(call enabled,CODEC_EXPORTS) + + +# Generate a list of all enabled sources, in particular for exporting to gyp +# based build systems. +libaom_srcs.txt: + @echo " [CREATE] $@" + @echo $(CODEC_SRCS) | xargs -n1 echo | LC_ALL=C sort -u > $@ +CLEAN-OBJS += libaom_srcs.txt + + +ifeq ($(CONFIG_EXTERNAL_BUILD),yes) +ifeq ($(CONFIG_MSVS),yes) + +aom.def: $(call enabled,CODEC_EXPORTS) + @echo " [CREATE] $@" + $(qexec)$(SRC_PATH_BARE)/build/make/gen_msvs_def.sh\ + --name=aom\ + --out=$@ $^ +CLEAN-OBJS += aom.def + +# Assembly files that are included, but don't define symbols themselves. +# Filtered out to avoid Visual Studio build warnings. +ASM_INCLUDES := \ + third_party/x86inc/x86inc.asm \ + aom_config.asm \ + aom_ports/x86_abi_support.asm \ + +aom.$(VCPROJ_SFX): $(CODEC_SRCS) aom.def + @echo " [CREATE] $@" + $(qexec)$(GEN_VCPROJ) \ + $(if $(CONFIG_SHARED),--dll,--lib) \ + --target=$(TOOLCHAIN) \ + $(if $(CONFIG_STATIC_MSVCRT),--static-crt) \ + --name=aom \ + --proj-guid=DCE19DAF-69AC-46DB-B14A-39F0FAA5DB74 \ + --module-def=aom.def \ + --ver=$(CONFIG_VS_VERSION) \ + --src-path-bare="$(SRC_PATH_BARE)" \ + --out=$@ $(CFLAGS) \ + $(filter-out $(addprefix %, $(ASM_INCLUDES)), $^) \ + --src-path-bare="$(SRC_PATH_BARE)" \ + +PROJECTS-yes += aom.$(VCPROJ_SFX) + +aom.$(VCPROJ_SFX): aom_config.asm +aom.$(VCPROJ_SFX): $(RTCD) + +endif +else +LIBAOM_OBJS=$(call objs,$(CODEC_SRCS)) +OBJS-yes += $(LIBAOM_OBJS) +LIBS-$(if yes,$(CONFIG_STATIC)) += $(BUILD_PFX)libaom.a $(BUILD_PFX)libaom_g.a +$(BUILD_PFX)libaom_g.a: $(LIBAOM_OBJS) + +SO_VERSION_MAJOR := 3 +SO_VERSION_MINOR := 0 +SO_VERSION_PATCH := 0 +ifeq ($(filter darwin%,$(TGT_OS)),$(TGT_OS)) +LIBAOM_SO := libaom.$(SO_VERSION_MAJOR).dylib +SHARED_LIB_SUF := .dylib +EXPORT_FILE := libaom.syms +LIBAOM_SO_SYMLINKS := $(addprefix $(LIBSUBDIR)/, \ + libaom.dylib ) +else +ifeq ($(filter iphonesimulator%,$(TGT_OS)),$(TGT_OS)) +LIBAOM_SO := libaom.$(SO_VERSION_MAJOR).dylib +SHARED_LIB_SUF := .dylib +EXPORT_FILE := libaom.syms +LIBAOM_SO_SYMLINKS := $(addprefix $(LIBSUBDIR)/, libaom.dylib) +else +ifeq ($(filter os2%,$(TGT_OS)),$(TGT_OS)) +LIBAOM_SO := libaom$(SO_VERSION_MAJOR).dll +SHARED_LIB_SUF := _dll.a +EXPORT_FILE := libaom.def +LIBAOM_SO_SYMLINKS := +LIBAOM_SO_IMPLIB := libaom_dll.a +else +LIBAOM_SO := libaom.so.$(SO_VERSION_MAJOR).$(SO_VERSION_MINOR).$(SO_VERSION_PATCH) +SHARED_LIB_SUF := .so +EXPORT_FILE := libaom.ver +LIBAOM_SO_SYMLINKS := $(addprefix $(LIBSUBDIR)/, \ + libaom.so libaom.so.$(SO_VERSION_MAJOR) \ + libaom.so.$(SO_VERSION_MAJOR).$(SO_VERSION_MINOR)) +endif +endif +endif + +LIBS-$(CONFIG_SHARED) += $(BUILD_PFX)$(LIBAOM_SO)\ + $(notdir $(LIBAOM_SO_SYMLINKS)) \ + $(if $(LIBAOM_SO_IMPLIB), $(BUILD_PFX)$(LIBAOM_SO_IMPLIB)) +$(BUILD_PFX)$(LIBAOM_SO): $(LIBAOM_OBJS) $(EXPORT_FILE) +$(BUILD_PFX)$(LIBAOM_SO): extralibs += -lm +$(BUILD_PFX)$(LIBAOM_SO): SONAME = libaom.so.$(SO_VERSION_MAJOR) +$(BUILD_PFX)$(LIBAOM_SO): EXPORTS_FILE = $(EXPORT_FILE) + +libaom.ver: $(call enabled,CODEC_EXPORTS) + @echo " [CREATE] $@" + $(qexec)echo "{ global:" > $@ + $(qexec)for f in $?; do awk '{print $$2";"}' < $$f >>$@; done + $(qexec)echo "local: *; };" >> $@ +CLEAN-OBJS += libaom.ver + +libaom.syms: $(call enabled,CODEC_EXPORTS) + @echo " [CREATE] $@" + $(qexec)awk '{print "_"$$2}' $^ >$@ +CLEAN-OBJS += libaom.syms + +libaom.def: $(call enabled,CODEC_EXPORTS) + @echo " [CREATE] $@" + $(qexec)echo LIBRARY $(LIBAOM_SO:.dll=) INITINSTANCE TERMINSTANCE > $@ + $(qexec)echo "DATA MULTIPLE NONSHARED" >> $@ + $(qexec)echo "EXPORTS" >> $@ + $(qexec)awk '{print "_"$$2}' $^ >>$@ +CLEAN-OBJS += libaom.def + +libaom_dll.a: $(LIBAOM_SO) + @echo " [IMPLIB] $@" + $(qexec)emximp -o $@ $< +CLEAN-OBJS += libaom_dll.a + +define libaom_symlink_template +$(1): $(2) + @echo " [LN] $(2) $$@" + $(qexec)mkdir -p $$(dir $$@) + $(qexec)ln -sf $(2) $$@ +endef + +$(eval $(call libaom_symlink_template,\ + $(addprefix $(BUILD_PFX),$(notdir $(LIBAOM_SO_SYMLINKS))),\ + $(BUILD_PFX)$(LIBAOM_SO))) +$(eval $(call libaom_symlink_template,\ + $(addprefix $(DIST_DIR)/,$(LIBAOM_SO_SYMLINKS)),\ + $(LIBAOM_SO))) + + +INSTALL-LIBS-$(CONFIG_SHARED) += $(LIBAOM_SO_SYMLINKS) +INSTALL-LIBS-$(CONFIG_SHARED) += $(LIBSUBDIR)/$(LIBAOM_SO) +INSTALL-LIBS-$(CONFIG_SHARED) += $(if $(LIBAOM_SO_IMPLIB),$(LIBSUBDIR)/$(LIBAOM_SO_IMPLIB)) + + +LIBS-yes += aom.pc +aom.pc: config.mk libs.mk + @echo " [CREATE] $@" + $(qexec)echo '# pkg-config file from libaom $(VERSION_STRING)' > $@ + $(qexec)echo 'prefix=$(PREFIX)' >> $@ + $(qexec)echo 'exec_prefix=$${prefix}' >> $@ + $(qexec)echo 'libdir=$${prefix}/$(LIBSUBDIR)' >> $@ + $(qexec)echo 'includedir=$${prefix}/include' >> $@ + $(qexec)echo '' >> $@ + $(qexec)echo 'Name: aom' >> $@ + $(qexec)echo 'Description: WebM Project AVx codec implementation' >> $@ + $(qexec)echo 'Version: $(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_PATCH)' >> $@ + $(qexec)echo 'Requires:' >> $@ + $(qexec)echo 'Conflicts:' >> $@ + $(qexec)echo 'Libs: -L$${libdir} -laom -lm' >> $@ +ifeq ($(HAVE_PTHREAD_H),yes) + $(qexec)echo 'Libs.private: -lm -lpthread' >> $@ +else + $(qexec)echo 'Libs.private: -lm' >> $@ +endif + $(qexec)echo 'Cflags: -I$${includedir}' >> $@ +INSTALL-LIBS-yes += $(LIBSUBDIR)/pkgconfig/aom.pc +INSTALL_MAPS += $(LIBSUBDIR)/pkgconfig/%.pc %.pc +CLEAN-OBJS += aom.pc +endif + +# +# Rule to make assembler configuration file from C configuration file +# +ifeq ($(ARCH_X86)$(ARCH_X86_64),yes) +# YASM +$(BUILD_PFX)aom_config.asm: $(BUILD_PFX)aom_config.h + @echo " [CREATE] $@" + @egrep "#define [A-Z0-9_]+ [01]" $< \ + | awk '{print $$2 " equ " $$3}' > $@ +else +ADS2GAS=$(if $(filter yes,$(CONFIG_GCC)),| $(ASM_CONVERSION)) +$(BUILD_PFX)aom_config.asm: $(BUILD_PFX)aom_config.h + @echo " [CREATE] $@" + @egrep "#define [A-Z0-9_]+ [01]" $< \ + | awk '{print $$2 " EQU " $$3}' $(ADS2GAS) > $@ + @echo " END" $(ADS2GAS) >> $@ +CLEAN-OBJS += $(BUILD_PFX)aom_config.asm +endif + +# +# Add assembler dependencies for configuration. +# +$(filter %.s.o,$(OBJS-yes)): $(BUILD_PFX)aom_config.asm +$(filter %$(ASM).o,$(OBJS-yes)): $(BUILD_PFX)aom_config.asm + + +$(shell $(SRC_PATH_BARE)/build/make/version.sh "$(SRC_PATH_BARE)" $(BUILD_PFX)aom_version.h) +CLEAN-OBJS += $(BUILD_PFX)aom_version.h + +# +# Add include path for libwebm sources. +# +ifeq ($(CONFIG_WEBM_IO),yes) + CXXFLAGS += -I$(SRC_PATH_BARE)/third_party/libwebm +endif + +## +## libaom test directives +## +ifeq ($(CONFIG_UNIT_TESTS),yes) +LIBAOM_TEST_DATA_PATH ?= . + +include $(SRC_PATH_BARE)/test/test.mk +LIBAOM_TEST_SRCS=$(addprefix test/,$(call enabled,LIBAOM_TEST_SRCS)) +LIBAOM_TEST_BIN=./test_libaom$(EXE_SFX) +LIBAOM_TEST_DATA=$(addprefix $(LIBAOM_TEST_DATA_PATH)/,\ + $(call enabled,LIBAOM_TEST_DATA)) +libaom_test_data_url=https://storage.googleapis.com/downloads.webmproject.org/test_data/libvpx/$(1) + +TEST_INTRA_PRED_SPEED_BIN=./test_intra_pred_speed$(EXE_SFX) +TEST_INTRA_PRED_SPEED_SRCS=$(addprefix test/,$(call enabled,TEST_INTRA_PRED_SPEED_SRCS)) +TEST_INTRA_PRED_SPEED_OBJS := $(sort $(call objs,$(TEST_INTRA_PRED_SPEED_SRCS))) + +libaom_test_srcs.txt: + @echo " [CREATE] $@" + @echo $(LIBAOM_TEST_SRCS) | xargs -n1 echo | LC_ALL=C sort -u > $@ +CLEAN-OBJS += libaom_test_srcs.txt + +$(LIBAOM_TEST_DATA): $(SRC_PATH_BARE)/test/test-data.sha1 + @echo " [DOWNLOAD] $@" + $(qexec)trap 'rm -f $@' INT TERM &&\ + curl -L -o $@ $(call libaom_test_data_url,$(@F)) + +testdata:: $(LIBAOM_TEST_DATA) + $(qexec)[ -x "$$(which sha1sum)" ] && sha1sum=sha1sum;\ + [ -x "$$(which shasum)" ] && sha1sum=shasum;\ + [ -x "$$(which sha1)" ] && sha1sum=sha1;\ + if [ -n "$${sha1sum}" ]; then\ + set -e;\ + echo "Checking test data:";\ + for f in $(call enabled,LIBAOM_TEST_DATA); do\ + grep $$f $(SRC_PATH_BARE)/test/test-data.sha1 |\ + (cd $(LIBAOM_TEST_DATA_PATH); $${sha1sum} -c);\ + done; \ + else\ + echo "Skipping test data integrity check, sha1sum not found.";\ + fi + +ifeq ($(CONFIG_EXTERNAL_BUILD),yes) +ifeq ($(CONFIG_MSVS),yes) +gtest.$(VCPROJ_SFX): $(SRC_PATH_BARE)/third_party/googletest/src/googletest/src/gtest-all.cc + @echo " [CREATE] $@" + $(qexec)$(GEN_VCPROJ) \ + --lib \ + --target=$(TOOLCHAIN) \ + $(if $(CONFIG_STATIC_MSVCRT),--static-crt) \ + --name=gtest \ + --proj-guid=EC00E1EC-AF68-4D92-A255-181690D1C9B1 \ + --ver=$(CONFIG_VS_VERSION) \ + --src-path-bare="$(SRC_PATH_BARE)" \ + -D_VARIADIC_MAX=10 \ + --out=gtest.$(VCPROJ_SFX) \ + $(SRC_PATH_BARE)/third_party/googletest/src/googletest/src/gtest-all.cc \ + -I. -I"$(SRC_PATH_BARE)/third_party/googletest/src/googletest/include" \ + -I"$(SRC_PATH_BARE)/third_party/googletest/src/googletest" + +PROJECTS-$(CONFIG_MSVS) += gtest.$(VCPROJ_SFX) + +test_libaom.$(VCPROJ_SFX): $(LIBAOM_TEST_SRCS) aom.$(VCPROJ_SFX) gtest.$(VCPROJ_SFX) + @echo " [CREATE] $@" + $(qexec)$(GEN_VCPROJ) \ + --exe \ + --target=$(TOOLCHAIN) \ + --name=test_libaom \ + -D_VARIADIC_MAX=10 \ + --proj-guid=CD837F5F-52D8-4314-A370-895D614166A7 \ + --ver=$(CONFIG_VS_VERSION) \ + --src-path-bare="$(SRC_PATH_BARE)" \ + $(if $(CONFIG_STATIC_MSVCRT),--static-crt) \ + --out=$@ $(INTERNAL_CFLAGS) $(CFLAGS) \ + -I. -I"$(SRC_PATH_BARE)/third_party/googletest/src/googletest/include" \ + $(if $(CONFIG_WEBM_IO),-I"$(SRC_PATH_BARE)/third_party/libwebm") \ + -L. -l$(CODEC_LIB) -l$(GTEST_LIB) $^ + +PROJECTS-$(CONFIG_MSVS) += test_libaom.$(VCPROJ_SFX) + +LIBAOM_TEST_BIN := $(addprefix $(TGT_OS:win64=x64)/Release/,$(notdir $(LIBAOM_TEST_BIN))) + +ifneq ($(strip $(TEST_INTRA_PRED_SPEED_OBJS)),) +PROJECTS-$(CONFIG_MSVS) += test_intra_pred_speed.$(VCPROJ_SFX) +test_intra_pred_speed.$(VCPROJ_SFX): $(TEST_INTRA_PRED_SPEED_SRCS) aom.$(VCPROJ_SFX) gtest.$(VCPROJ_SFX) + @echo " [CREATE] $@" + $(qexec)$(GEN_VCPROJ) \ + --exe \ + --target=$(TOOLCHAIN) \ + --name=test_intra_pred_speed \ + -D_VARIADIC_MAX=10 \ + --proj-guid=CD837F5F-52D8-4314-A370-895D614166A7 \ + --ver=$(CONFIG_VS_VERSION) \ + --src-path-bare="$(SRC_PATH_BARE)" \ + $(if $(CONFIG_STATIC_MSVCRT),--static-crt) \ + --out=$@ $(INTERNAL_CFLAGS) $(CFLAGS) \ + -I. -I"$(SRC_PATH_BARE)/third_party/googletest/src/googletest/include" \ + -L. -l$(CODEC_LIB) -l$(GTEST_LIB) $^ +endif # TEST_INTRA_PRED_SPEED +endif +else + +include $(SRC_PATH_BARE)/third_party/googletest/gtest.mk +GTEST_SRCS := $(addprefix third_party/,$(call enabled,GTEST_SRCS)) +GTEST_OBJS=$(call objs,$(GTEST_SRCS)) +ifeq ($(filter win%,$(TGT_OS)),$(TGT_OS)) +# Disabling pthreads globally will cause issues on darwin and possibly elsewhere +$(GTEST_OBJS) $(GTEST_OBJS:.o=.d): CXXFLAGS += -DGTEST_HAS_PTHREAD=0 +endif +GTEST_INCLUDES := -I$(SRC_PATH_BARE)/third_party/googletest/src/googletest +GTEST_INCLUDES += -I$(SRC_PATH_BARE)/third_party/googletest/src/googletest/include +$(GTEST_OBJS) $(GTEST_OBJS:.o=.d): CXXFLAGS += $(GTEST_INCLUDES) +OBJS-yes += $(GTEST_OBJS) +LIBS-yes += $(BUILD_PFX)libgtest.a $(BUILD_PFX)libgtest_g.a +$(BUILD_PFX)libgtest_g.a: $(GTEST_OBJS) + +LIBAOM_TEST_OBJS=$(sort $(call objs,$(LIBAOM_TEST_SRCS))) +$(LIBAOM_TEST_OBJS) $(LIBAOM_TEST_OBJS:.o=.d): CXXFLAGS += $(GTEST_INCLUDES) +OBJS-yes += $(LIBAOM_TEST_OBJS) +BINS-yes += $(LIBAOM_TEST_BIN) + +CODEC_LIB=$(if $(CONFIG_DEBUG_LIBS),aom_g,aom) +CODEC_LIB_SUF=$(if $(CONFIG_SHARED),$(SHARED_LIB_SUF),.a) +TEST_LIBS := lib$(CODEC_LIB)$(CODEC_LIB_SUF) libgtest.a +$(LIBAOM_TEST_BIN): $(TEST_LIBS) +$(eval $(call linkerxx_template,$(LIBAOM_TEST_BIN), \ + $(LIBAOM_TEST_OBJS) \ + -L. -laom -lgtest $(extralibs) -lm)) + +ifneq ($(strip $(TEST_INTRA_PRED_SPEED_OBJS)),) +$(TEST_INTRA_PRED_SPEED_OBJS) $(TEST_INTRA_PRED_SPEED_OBJS:.o=.d): CXXFLAGS += $(GTEST_INCLUDES) +OBJS-yes += $(TEST_INTRA_PRED_SPEED_OBJS) +BINS-yes += $(TEST_INTRA_PRED_SPEED_BIN) + +$(TEST_INTRA_PRED_SPEED_BIN): $(TEST_LIBS) +$(eval $(call linkerxx_template,$(TEST_INTRA_PRED_SPEED_BIN), \ + $(TEST_INTRA_PRED_SPEED_OBJS) \ + -L. -laom -lgtest $(extralibs) -lm)) +endif # TEST_INTRA_PRED_SPEED + +endif # CONFIG_UNIT_TESTS + +# Install test sources only if codec source is included +INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(patsubst $(SRC_PATH_BARE)/%,%,\ + $(shell find $(SRC_PATH_BARE)/third_party/googletest -type f)) +INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(LIBAOM_TEST_SRCS) +INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(TEST_INTRA_PRED_SPEED_SRCS) + +define test_shard_template +test:: test_shard.$(1) +test-no-data-check:: test_shard_ndc.$(1) +test_shard.$(1) test_shard_ndc.$(1): $(LIBAOM_TEST_BIN) + @set -e; \ + export GTEST_SHARD_INDEX=$(1); \ + export GTEST_TOTAL_SHARDS=$(2); \ + $(LIBAOM_TEST_BIN) +test_shard.$(1): testdata +.PHONY: test_shard.$(1) +endef + +NUM_SHARDS := 10 +SHARDS := 0 1 2 3 4 5 6 7 8 9 +$(foreach s,$(SHARDS),$(eval $(call test_shard_template,$(s),$(NUM_SHARDS)))) + +endif + +## +## documentation directives +## +CLEAN-OBJS += libs.doxy +DOCS-yes += libs.doxy +libs.doxy: $(CODEC_DOC_SRCS) + @echo " [CREATE] $@" + @rm -f $@ + @echo "INPUT += $^" >> $@ + @echo "INCLUDE_PATH += ." >> $@; + @echo "ENABLED_SECTIONS += $(sort $(CODEC_DOC_SECTIONS))" >> $@ + +## Generate rtcd.h for all objects +ifeq ($(CONFIG_DEPENDENCY_TRACKING),yes) +$(OBJS-yes:.o=.d): $(RTCD) +else +$(OBJS-yes): $(RTCD) +endif + +## Update the global src list +SRCS += $(CODEC_SRCS) $(LIBAOM_TEST_SRCS) $(GTEST_SRCS) + +## +## aomdec/aomenc tests. +## +ifeq ($(CONFIG_UNIT_TESTS),yes) +TEST_BIN_PATH = . +ifeq ($(CONFIG_MSVS),yes) +# MSVC will build both Debug and Release configurations of tools in a +# sub directory named for the current target. Assume the user wants to +# run the Release tools, and assign TEST_BIN_PATH accordingly. +# TODO(tomfinegan): Is this adequate for ARM? +# TODO(tomfinegan): Support running the debug versions of tools? +TEST_BIN_PATH := $(addsuffix /$(TGT_OS:win64=x64)/Release, $(TEST_BIN_PATH)) +endif +utiltest utiltest-no-data-check: + $(qexec)$(SRC_PATH_BARE)/test/aomdec.sh \ + --test-data-path $(LIBAOM_TEST_DATA_PATH) \ + --bin-path $(TEST_BIN_PATH) + $(qexec)$(SRC_PATH_BARE)/test/aomenc.sh \ + --test-data-path $(LIBAOM_TEST_DATA_PATH) \ + --bin-path $(TEST_BIN_PATH) +utiltest: testdata +else +utiltest utiltest-no-data-check: + @echo Unit tests must be enabled to make the utiltest target. +endif + +## +## Example tests. +## +ifeq ($(CONFIG_UNIT_TESTS),yes) +# All non-MSVC targets output example targets in a sub dir named examples. +EXAMPLES_BIN_PATH = examples +ifeq ($(CONFIG_MSVS),yes) +# MSVC will build both Debug and Release configurations of the examples in a +# sub directory named for the current target. Assume the user wants to +# run the Release tools, and assign EXAMPLES_BIN_PATH accordingly. +# TODO(tomfinegan): Is this adequate for ARM? +# TODO(tomfinegan): Support running the debug versions of tools? +EXAMPLES_BIN_PATH := $(TGT_OS:win64=x64)/Release +endif +exampletest exampletest-no-data-check: examples + $(qexec)$(SRC_PATH_BARE)/test/examples.sh \ + --test-data-path $(LIBAOM_TEST_DATA_PATH) \ + --bin-path $(EXAMPLES_BIN_PATH) +exampletest: testdata +else +exampletest exampletest-no-data-check: + @echo Unit tests must be enabled to make the exampletest target. +endif diff --git a/third_party/aom/mainpage.dox b/third_party/aom/mainpage.dox new file mode 100644 index 0000000000..9a82f43603 --- /dev/null +++ b/third_party/aom/mainpage.dox @@ -0,0 +1,52 @@ +/*!\mainpage AMedia Codec SDK + + \section main_contents Page Contents + - \ref main_intro + - \ref main_startpoints + - \ref main_support + + \section main_intro Introduction + Welcome to the AMedia Codec SDK. This SDK allows you to integrate your + applications with the AOM and AV1 video codecs. + + This distribution of the AOMedia Codec SDK includes the following support: + + \if aom_encoder + - \ref aom_encoder + \endif + \if aom_decoder + - \ref aom_decoder + \endif + + + \section main_startpoints Starting Points + - Consult the \ref changelog for a complete list of improvements in this + release. + - The \ref readme contains instructions on recompiling the sample applications. + - Read the \ref usage "usage" for a narrative on codec usage. + - Read the \ref samples "sample code" for examples of how to interact with the + codec. + - \ref codec reference + \if encoder + - \ref encoder reference + \endif + \if decoder + - \ref decoder reference + \endif + + \section main_support Support Options & FAQ + The AOMedia project is an open source project supported by its community. For + questions about this SDK, please mail the apps-devel@webmproject.org list. + To contribute, see http://www.webmproject.org/code/contribute and mail + codec-devel@webmproject.org. +*/ + +/*!\page changelog CHANGELOG + \verbinclude CHANGELOG +*/ + +/*!\page readme README + \verbinclude README +*/ + +/*!\defgroup codecs Supported Codecs */ diff --git a/third_party/aom/md5_utils.c b/third_party/aom/md5_utils.c new file mode 100644 index 0000000000..34012b205b --- /dev/null +++ b/third_party/aom/md5_utils.c @@ -0,0 +1,249 @@ +/* + * This code implements the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + * + * Changed so as no longer to depend on Colin Plumb's `usual.h' header + * definitions + * - Ian Jackson . + * Still in the public domain. + */ + +#include /* for memcpy() */ + +#include "md5_utils.h" + +static void byteSwap(UWORD32 *buf, unsigned words) { + md5byte *p; + + /* Only swap bytes for big endian machines */ + int i = 1; + + if (*(char *)&i == 1) return; + + p = (md5byte *)buf; + + do { + *buf++ = (UWORD32)((unsigned)p[3] << 8 | p[2]) << 16 | + ((unsigned)p[1] << 8 | p[0]); + p += 4; + } while (--words); +} + +/* + * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious + * initialization constants. + */ +void MD5Init(struct MD5Context *ctx) { + ctx->buf[0] = 0x67452301; + ctx->buf[1] = 0xefcdab89; + ctx->buf[2] = 0x98badcfe; + ctx->buf[3] = 0x10325476; + + ctx->bytes[0] = 0; + ctx->bytes[1] = 0; +} + +/* + * Update context to reflect the concatenation of another buffer full + * of bytes. + */ +void MD5Update(struct MD5Context *ctx, md5byte const *buf, unsigned len) { + UWORD32 t; + + /* Update byte count */ + + t = ctx->bytes[0]; + + if ((ctx->bytes[0] = t + len) < t) + ctx->bytes[1]++; /* Carry from low to high */ + + t = 64 - (t & 0x3f); /* Space available in ctx->in (at least 1) */ + + if (t > len) { + memcpy((md5byte *)ctx->in + 64 - t, buf, len); + return; + } + + /* First chunk is an odd size */ + memcpy((md5byte *)ctx->in + 64 - t, buf, t); + byteSwap(ctx->in, 16); + MD5Transform(ctx->buf, ctx->in); + buf += t; + len -= t; + + /* Process data in 64-byte chunks */ + while (len >= 64) { + memcpy(ctx->in, buf, 64); + byteSwap(ctx->in, 16); + MD5Transform(ctx->buf, ctx->in); + buf += 64; + len -= 64; + } + + /* Handle any remaining bytes of data. */ + memcpy(ctx->in, buf, len); +} + +/* + * Final wrapup - pad to 64-byte boundary with the bit pattern + * 1 0* (64-bit count of bits processed, MSB-first) + */ +void MD5Final(md5byte digest[16], struct MD5Context *ctx) { + int count = ctx->bytes[0] & 0x3f; /* Number of bytes in ctx->in */ + md5byte *p = (md5byte *)ctx->in + count; + + /* Set the first char of padding to 0x80. There is always room. */ + *p++ = 0x80; + + /* Bytes of padding needed to make 56 bytes (-8..55) */ + count = 56 - 1 - count; + + if (count < 0) { /* Padding forces an extra block */ + memset(p, 0, count + 8); + byteSwap(ctx->in, 16); + MD5Transform(ctx->buf, ctx->in); + p = (md5byte *)ctx->in; + count = 56; + } + + memset(p, 0, count); + byteSwap(ctx->in, 14); + + /* Append length in bits and transform */ + ctx->in[14] = ctx->bytes[0] << 3; + ctx->in[15] = ctx->bytes[1] << 3 | ctx->bytes[0] >> 29; + MD5Transform(ctx->buf, ctx->in); + + byteSwap(ctx->buf, 4); + memcpy(digest, ctx->buf, 16); + memset(ctx, 0, sizeof(*ctx)); /* In case it's sensitive */ +} + +#ifndef ASM_MD5 + +/* The four core functions - F1 is optimized somewhat */ + +/* #define F1(x, y, z) (x & y | ~x & z) */ +#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F2(x, y, z) F1(z, x, y) +#define F3(x, y, z) (x ^ y ^ z) +#define F4(x, y, z) (y ^ (x | ~z)) + +/* This is the central step in the MD5 algorithm. */ +#define MD5STEP(f, w, x, y, z, in, s) \ + (w += f(x, y, z) + in, w = (w << s | w >> (32 - s)) + x) + +#if defined(__clang__) && defined(__has_attribute) +#if __has_attribute(no_sanitize) +#define AOM_NO_UNSIGNED_OVERFLOW_CHECK \ + __attribute__((no_sanitize("unsigned-integer-overflow"))) +#endif +#endif + +#ifndef AOM_NO_UNSIGNED_OVERFLOW_CHECK +#define AOM_NO_UNSIGNED_OVERFLOW_CHECK +#endif + +/* + * The core of the MD5 algorithm, this alters an existing MD5 hash to + * reflect the addition of 16 longwords of new data. MD5Update blocks + * the data and converts bytes into longwords for this routine. + */ +AOM_NO_UNSIGNED_OVERFLOW_CHECK void MD5Transform(UWORD32 buf[4], + UWORD32 const in[16]) { + register UWORD32 a, b, c, d; + + a = buf[0]; + b = buf[1]; + c = buf[2]; + d = buf[3]; + + MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7); + MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12); + MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17); + MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22); + MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7); + MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12); + MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17); + MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22); + MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7); + MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12); + MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17); + MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22); + MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7); + MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12); + MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17); + MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22); + + MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5); + MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9); + MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14); + MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20); + MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5); + MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9); + MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14); + MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20); + MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5); + MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9); + MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14); + MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20); + MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5); + MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9); + MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14); + MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20); + + MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4); + MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11); + MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16); + MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23); + MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4); + MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11); + MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16); + MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23); + MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4); + MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11); + MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16); + MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23); + MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4); + MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11); + MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16); + MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23); + + MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6); + MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10); + MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15); + MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21); + MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6); + MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10); + MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15); + MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21); + MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6); + MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10); + MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15); + MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21); + MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6); + MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10); + MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15); + MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21); + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; +} + +#undef AOM_NO_UNSIGNED_OVERFLOW_CHECK + +#endif diff --git a/third_party/aom/md5_utils.h b/third_party/aom/md5_utils.h new file mode 100644 index 0000000000..bd4991b3ad --- /dev/null +++ b/third_party/aom/md5_utils.h @@ -0,0 +1,49 @@ +/* + * This is the header file for the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + * + * Changed so as no longer to depend on Colin Plumb's `usual.h' + * header definitions + * - Ian Jackson . + * Still in the public domain. + */ + +#ifndef MD5_UTILS_H_ +#define MD5_UTILS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define md5byte unsigned char +#define UWORD32 unsigned int + +typedef struct MD5Context MD5Context; +struct MD5Context { + UWORD32 buf[4]; + UWORD32 bytes[2]; + UWORD32 in[16]; +}; + +void MD5Init(struct MD5Context *context); +void MD5Update(struct MD5Context *context, md5byte const *buf, unsigned len); +void MD5Final(unsigned char digest[16], struct MD5Context *context); +void MD5Transform(UWORD32 buf[4], UWORD32 const in[16]); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // MD5_UTILS_H_ diff --git a/third_party/aom/rate_hist.c b/third_party/aom/rate_hist.c new file mode 100644 index 0000000000..ffc7b8997e --- /dev/null +++ b/third_party/aom/rate_hist.c @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include +#include + +#include "./rate_hist.h" + +#define RATE_BINS 100 +#define HIST_BAR_MAX 40 + +struct hist_bucket { + int low; + int high; + int count; +}; + +struct rate_hist { + int64_t *pts; + int *sz; + int samples; + int frames; + struct hist_bucket bucket[RATE_BINS]; + int total; +}; + +struct rate_hist *init_rate_histogram(const aom_codec_enc_cfg_t *cfg, + const aom_rational_t *fps) { + int i; + struct rate_hist *hist = malloc(sizeof(*hist)); + + // Determine the number of samples in the buffer. Use the file's framerate + // to determine the number of frames in rc_buf_sz milliseconds, with an + // adjustment (5/4) to account for alt-refs + hist->samples = cfg->rc_buf_sz * 5 / 4 * fps->num / fps->den / 1000; + + // prevent division by zero + if (hist->samples == 0) hist->samples = 1; + + hist->frames = 0; + hist->total = 0; + + hist->pts = calloc(hist->samples, sizeof(*hist->pts)); + hist->sz = calloc(hist->samples, sizeof(*hist->sz)); + for (i = 0; i < RATE_BINS; i++) { + hist->bucket[i].low = INT_MAX; + hist->bucket[i].high = 0; + hist->bucket[i].count = 0; + } + + return hist; +} + +void destroy_rate_histogram(struct rate_hist *hist) { + if (hist) { + free(hist->pts); + free(hist->sz); + free(hist); + } +} + +void update_rate_histogram(struct rate_hist *hist, + const aom_codec_enc_cfg_t *cfg, + const aom_codec_cx_pkt_t *pkt) { + int i; + int64_t then = 0; + int64_t avg_bitrate = 0; + int64_t sum_sz = 0; + const int64_t now = pkt->data.frame.pts * 1000 * + (uint64_t)cfg->g_timebase.num / + (uint64_t)cfg->g_timebase.den; + + int idx = hist->frames++ % hist->samples; + hist->pts[idx] = now; + hist->sz[idx] = (int)pkt->data.frame.sz; + + if (now < cfg->rc_buf_initial_sz) return; + + if (!cfg->rc_target_bitrate) return; + + then = now; + + /* Sum the size over the past rc_buf_sz ms */ + for (i = hist->frames; i > 0 && hist->frames - i < hist->samples; i--) { + const int i_idx = (i - 1) % hist->samples; + + then = hist->pts[i_idx]; + if (now - then > cfg->rc_buf_sz) break; + sum_sz += hist->sz[i_idx]; + } + + if (now == then) return; + + avg_bitrate = sum_sz * 8 * 1000 / (now - then); + idx = (int)(avg_bitrate * (RATE_BINS / 2) / (cfg->rc_target_bitrate * 1000)); + if (idx < 0) idx = 0; + if (idx > RATE_BINS - 1) idx = RATE_BINS - 1; + if (hist->bucket[idx].low > avg_bitrate) + hist->bucket[idx].low = (int)avg_bitrate; + if (hist->bucket[idx].high < avg_bitrate) + hist->bucket[idx].high = (int)avg_bitrate; + hist->bucket[idx].count++; + hist->total++; +} + +static int merge_hist_buckets(struct hist_bucket *bucket, int max_buckets, + int *num_buckets) { + int small_bucket = 0, merge_bucket = INT_MAX, big_bucket = 0; + int buckets = *num_buckets; + int i; + + /* Find the extrema for this list of buckets */ + big_bucket = small_bucket = 0; + for (i = 0; i < buckets; i++) { + if (bucket[i].count < bucket[small_bucket].count) small_bucket = i; + if (bucket[i].count > bucket[big_bucket].count) big_bucket = i; + } + + /* If we have too many buckets, merge the smallest with an adjacent + * bucket. + */ + while (buckets > max_buckets) { + int last_bucket = buckets - 1; + + /* merge the small bucket with an adjacent one. */ + if (small_bucket == 0) + merge_bucket = 1; + else if (small_bucket == last_bucket) + merge_bucket = last_bucket - 1; + else if (bucket[small_bucket - 1].count < bucket[small_bucket + 1].count) + merge_bucket = small_bucket - 1; + else + merge_bucket = small_bucket + 1; + + assert(abs(merge_bucket - small_bucket) <= 1); + assert(small_bucket < buckets); + assert(big_bucket < buckets); + assert(merge_bucket < buckets); + + if (merge_bucket < small_bucket) { + bucket[merge_bucket].high = bucket[small_bucket].high; + bucket[merge_bucket].count += bucket[small_bucket].count; + } else { + bucket[small_bucket].high = bucket[merge_bucket].high; + bucket[small_bucket].count += bucket[merge_bucket].count; + merge_bucket = small_bucket; + } + + assert(bucket[merge_bucket].low != bucket[merge_bucket].high); + + buckets--; + + /* Remove the merge_bucket from the list, and find the new small + * and big buckets while we're at it + */ + big_bucket = small_bucket = 0; + for (i = 0; i < buckets; i++) { + if (i > merge_bucket) bucket[i] = bucket[i + 1]; + + if (bucket[i].count < bucket[small_bucket].count) small_bucket = i; + if (bucket[i].count > bucket[big_bucket].count) big_bucket = i; + } + } + + *num_buckets = buckets; + return bucket[big_bucket].count; +} + +static void show_histogram(const struct hist_bucket *bucket, int buckets, + int total, int scale) { + const char *pat1, *pat2; + int i; + + switch ((int)(log(bucket[buckets - 1].high) / log(10)) + 1) { + case 1: + case 2: + pat1 = "%4d %2s: "; + pat2 = "%4d-%2d: "; + break; + case 3: + pat1 = "%5d %3s: "; + pat2 = "%5d-%3d: "; + break; + case 4: + pat1 = "%6d %4s: "; + pat2 = "%6d-%4d: "; + break; + case 5: + pat1 = "%7d %5s: "; + pat2 = "%7d-%5d: "; + break; + case 6: + pat1 = "%8d %6s: "; + pat2 = "%8d-%6d: "; + break; + case 7: + pat1 = "%9d %7s: "; + pat2 = "%9d-%7d: "; + break; + default: + pat1 = "%12d %10s: "; + pat2 = "%12d-%10d: "; + break; + } + + for (i = 0; i < buckets; i++) { + int len; + int j; + float pct; + + pct = (float)(100.0 * bucket[i].count / total); + len = HIST_BAR_MAX * bucket[i].count / scale; + if (len < 1) len = 1; + assert(len <= HIST_BAR_MAX); + + if (bucket[i].low == bucket[i].high) + fprintf(stderr, pat1, bucket[i].low, ""); + else + fprintf(stderr, pat2, bucket[i].low, bucket[i].high); + + for (j = 0; j < HIST_BAR_MAX; j++) fprintf(stderr, j < len ? "=" : " "); + fprintf(stderr, "\t%5d (%6.2f%%)\n", bucket[i].count, pct); + } +} + +void show_q_histogram(const int counts[64], int max_buckets) { + struct hist_bucket bucket[64]; + int buckets = 0; + int total = 0; + int scale; + int i; + + for (i = 0; i < 64; i++) { + if (counts[i]) { + bucket[buckets].low = bucket[buckets].high = i; + bucket[buckets].count = counts[i]; + buckets++; + total += counts[i]; + } + } + + fprintf(stderr, "\nQuantizer Selection:\n"); + scale = merge_hist_buckets(bucket, max_buckets, &buckets); + show_histogram(bucket, buckets, total, scale); +} + +void show_rate_histogram(struct rate_hist *hist, const aom_codec_enc_cfg_t *cfg, + int max_buckets) { + int i, scale; + int buckets = 0; + + for (i = 0; i < RATE_BINS; i++) { + if (hist->bucket[i].low == INT_MAX) continue; + hist->bucket[buckets++] = hist->bucket[i]; + } + + fprintf(stderr, "\nRate (over %dms window):\n", cfg->rc_buf_sz); + scale = merge_hist_buckets(hist->bucket, max_buckets, &buckets); + show_histogram(hist->bucket, buckets, hist->total, scale); +} diff --git a/third_party/aom/rate_hist.h b/third_party/aom/rate_hist.h new file mode 100644 index 0000000000..e6aa149aef --- /dev/null +++ b/third_party/aom/rate_hist.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef RATE_HIST_H_ +#define RATE_HIST_H_ + +#include "aom/aom_encoder.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct rate_hist; + +struct rate_hist *init_rate_histogram(const aom_codec_enc_cfg_t *cfg, + const aom_rational_t *fps); + +void destroy_rate_histogram(struct rate_hist *hist); + +void update_rate_histogram(struct rate_hist *hist, + const aom_codec_enc_cfg_t *cfg, + const aom_codec_cx_pkt_t *pkt); + +void show_q_histogram(const int counts[64], int max_buckets); + +void show_rate_histogram(struct rate_hist *hist, const aom_codec_enc_cfg_t *cfg, + int max_buckets); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // RATE_HIST_H_ diff --git a/third_party/aom/solution.mk b/third_party/aom/solution.mk new file mode 100644 index 0000000000..caa8bc17bf --- /dev/null +++ b/third_party/aom/solution.mk @@ -0,0 +1,33 @@ +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + + +# libaom reverse dependencies (targets that depend on libaom) +AOM_NONDEPS=$(addsuffix .$(VCPROJ_SFX),aom gtest) +AOM_RDEPS=$(foreach vcp,\ + $(filter-out $(AOM_NONDEPS),$^), --dep=$(vcp:.$(VCPROJ_SFX)=):aom) + +aom.sln: $(wildcard *.$(VCPROJ_SFX)) + @echo " [CREATE] $@" + $(SRC_PATH_BARE)/build/make/gen_msvs_sln.sh \ + $(if $(filter aom.$(VCPROJ_SFX),$^),$(AOM_RDEPS)) \ + --dep=test_libaom:gtest \ + --ver=$(CONFIG_VS_VERSION)\ + --out=$@ $^ +aom.sln.mk: aom.sln + @true + +PROJECTS-yes += aom.sln aom.sln.mk +-include aom.sln.mk + +# Always install this file, as it is an unconditional post-build rule. +INSTALL_MAPS += src/% $(SRC_PATH_BARE)/% +INSTALL-SRCS-yes += $(target).mk diff --git a/third_party/aom/test/accounting_test.cc b/third_party/aom/test/accounting_test.cc new file mode 100644 index 0000000000..e8387d0dc0 --- /dev/null +++ b/third_party/aom/test/accounting_test.cc @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/acm_random.h" +#include "aom/aom_integer.h" +#include "aom_dsp/bitreader.h" +#include "aom_dsp/bitwriter.h" + +using libaom_test::ACMRandom; + +TEST(AV1, TestAccounting) { + const int kBufferSize = 10000; + const int kSymbols = 1024; + aom_writer bw; + uint8_t bw_buffer[kBufferSize]; + aom_start_encode(&bw, bw_buffer); + for (int i = 0; i < kSymbols; i++) { + aom_write(&bw, 0, 32); + aom_write(&bw, 0, 32); + aom_write(&bw, 0, 32); + } + aom_stop_encode(&bw); + aom_reader br; +#if CONFIG_ANS && ANS_MAX_SYMBOLS + br.window_size = 1 << 16; +#endif + aom_reader_init(&br, bw_buffer, bw.pos, NULL, NULL); + + Accounting accounting; + aom_accounting_init(&accounting); + br.accounting = &accounting; + for (int i = 0; i < kSymbols; i++) { + aom_read(&br, 32, "A"); + } + // Consecutive symbols that are the same are coalesced. + GTEST_ASSERT_EQ(accounting.syms.num_syms, 1); + GTEST_ASSERT_EQ(accounting.syms.syms[0].samples, (unsigned int)kSymbols); + + aom_accounting_reset(&accounting); + GTEST_ASSERT_EQ(accounting.syms.num_syms, 0); + + // Should record 2 * kSymbols accounting symbols. + aom_reader_init(&br, bw_buffer, bw.pos, NULL, NULL); + br.accounting = &accounting; + for (int i = 0; i < kSymbols; i++) { + aom_read(&br, 32, "A"); + aom_read(&br, 32, "B"); + aom_read(&br, 32, "B"); + } + GTEST_ASSERT_EQ(accounting.syms.num_syms, kSymbols * 2); + uint32_t tell_frac = aom_reader_tell_frac(&br); + for (int i = 0; i < accounting.syms.num_syms; i++) { + tell_frac -= accounting.syms.syms[i].bits; + } + GTEST_ASSERT_EQ(tell_frac, 0U); + + GTEST_ASSERT_EQ(aom_accounting_dictionary_lookup(&accounting, "A"), + aom_accounting_dictionary_lookup(&accounting, "A")); + + // Check for collisions. The current aom_accounting_hash function returns + // the same hash code for AB and BA. + GTEST_ASSERT_NE(aom_accounting_dictionary_lookup(&accounting, "AB"), + aom_accounting_dictionary_lookup(&accounting, "BA")); +} diff --git a/third_party/aom/test/acm_random.h b/third_party/aom/test/acm_random.h new file mode 100644 index 0000000000..4842345ff6 --- /dev/null +++ b/third_party/aom/test/acm_random.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef TEST_ACM_RANDOM_H_ +#define TEST_ACM_RANDOM_H_ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "aom/aom_integer.h" + +namespace libaom_test { + +class ACMRandom { + public: + ACMRandom() : random_(DeterministicSeed()) {} + + explicit ACMRandom(int seed) : random_(seed) {} + + void Reset(int seed) { random_.Reseed(seed); } + + uint32_t Rand31(void) { + return random_.Generate(testing::internal::Random::kMaxRange); + } + + uint16_t Rand16(void) { + const uint32_t value = + random_.Generate(testing::internal::Random::kMaxRange); + return (value >> 15) & 0xffff; + } + + int16_t Rand9Signed(void) { + // Use 9 bits: values between 255 (0x0FF) and -256 (0x100). + const uint32_t value = random_.Generate(512); + return static_cast(value) - 256; + } + + uint8_t Rand8(void) { + const uint32_t value = + random_.Generate(testing::internal::Random::kMaxRange); + // There's a bit more entropy in the upper bits of this implementation. + return (value >> 23) & 0xff; + } + + uint8_t Rand8Extremes(void) { + // Returns a random value near 0 or near 255, to better exercise + // saturation behavior. + const uint8_t r = Rand8(); + return r < 128 ? r << 4 : r >> 4; + } + + int PseudoUniform(int range) { return random_.Generate(range); } + + int operator()(int n) { return PseudoUniform(n); } + + static int DeterministicSeed(void) { return 0xbaba; } + + private: + testing::internal::Random random_; +}; + +} // namespace libaom_test + +#endif // TEST_ACM_RANDOM_H_ diff --git a/third_party/aom/test/active_map_refresh_test.cc b/third_party/aom/test/active_map_refresh_test.cc new file mode 100644 index 0000000000..7ee86e7e67 --- /dev/null +++ b/third_party/aom/test/active_map_refresh_test.cc @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/util.h" +#include "test/y4m_video_source.h" + +namespace { + +// Check if any pixel in a 16x16 macroblock varies between frames. +int CheckMb(const aom_image_t ¤t, const aom_image_t &previous, int mb_r, + int mb_c) { + for (int plane = 0; plane < 3; plane++) { + int r = 16 * mb_r; + int c0 = 16 * mb_c; + int r_top = std::min(r + 16, static_cast(current.d_h)); + int c_top = std::min(c0 + 16, static_cast(current.d_w)); + r = std::max(r, 0); + c0 = std::max(c0, 0); + if (plane > 0 && current.x_chroma_shift) { + c_top = (c_top + 1) >> 1; + c0 >>= 1; + } + if (plane > 0 && current.y_chroma_shift) { + r_top = (r_top + 1) >> 1; + r >>= 1; + } + for (; r < r_top; ++r) { + for (int c = c0; c < c_top; ++c) { + if (current.planes[plane][current.stride[plane] * r + c] != + previous.planes[plane][previous.stride[plane] * r + c]) + return 1; + } + } + } + return 0; +} + +void GenerateMap(int mb_rows, int mb_cols, const aom_image_t ¤t, + const aom_image_t &previous, uint8_t *map) { + for (int mb_r = 0; mb_r < mb_rows; ++mb_r) { + for (int mb_c = 0; mb_c < mb_cols; ++mb_c) { + map[mb_r * mb_cols + mb_c] = CheckMb(current, previous, mb_r, mb_c); + } + } +} + +const int kAqModeCyclicRefresh = 3; + +class ActiveMapRefreshTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params { + protected: + ActiveMapRefreshTest() : EncoderTest(GET_PARAM(0)) {} + virtual ~ActiveMapRefreshTest() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(GET_PARAM(1)); + cpu_used_ = GET_PARAM(2); + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + ::libaom_test::Y4mVideoSource *y4m_video = + static_cast(video); + if (video->frame() == 1) { + encoder->Control(AOME_SET_CPUUSED, cpu_used_); + encoder->Control(AV1E_SET_AQ_MODE, kAqModeCyclicRefresh); + } else if (video->frame() >= 2 && video->img()) { + aom_image_t *current = video->img(); + aom_image_t *previous = y4m_holder_->img(); + ASSERT_TRUE(previous != NULL); + aom_active_map_t map = aom_active_map_t(); + const int width = static_cast(current->d_w); + const int height = static_cast(current->d_h); + const int mb_width = (width + 15) / 16; + const int mb_height = (height + 15) / 16; + uint8_t *active_map = new uint8_t[mb_width * mb_height]; + GenerateMap(mb_height, mb_width, *current, *previous, active_map); + map.cols = mb_width; + map.rows = mb_height; + map.active_map = active_map; + encoder->Control(AOME_SET_ACTIVEMAP, &map); + delete[] active_map; + } + if (video->img()) { + y4m_video->SwapBuffers(y4m_holder_); + } + } + + int cpu_used_; + ::libaom_test::Y4mVideoSource *y4m_holder_; +}; + +TEST_P(ActiveMapRefreshTest, Test) { + cfg_.g_lag_in_frames = 0; + cfg_.g_profile = 1; + cfg_.rc_target_bitrate = 600; + cfg_.rc_resize_allowed = 0; + cfg_.rc_min_quantizer = 8; + cfg_.rc_max_quantizer = 30; + cfg_.g_pass = AOM_RC_ONE_PASS; + cfg_.rc_end_usage = AOM_CBR; + cfg_.kf_max_dist = 90000; + + ::libaom_test::Y4mVideoSource video("desktop_credits.y4m", 0, 10); + ::libaom_test::Y4mVideoSource video_holder("desktop_credits.y4m", 0, 10); + video_holder.Begin(); + y4m_holder_ = &video_holder; + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} + +AV1_INSTANTIATE_TEST_CASE(ActiveMapRefreshTest, + ::testing::Values(::libaom_test::kRealTime), + ::testing::Range(5, 6)); +} // namespace diff --git a/third_party/aom/test/active_map_test.cc b/third_party/aom/test/active_map_test.cc new file mode 100644 index 0000000000..a926b0faf3 --- /dev/null +++ b/third_party/aom/test/active_map_test.cc @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" + +namespace { + +class ActiveMapTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params { + protected: + static const int kWidth = 208; + static const int kHeight = 144; + + ActiveMapTest() : EncoderTest(GET_PARAM(0)) {} + virtual ~ActiveMapTest() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(GET_PARAM(1)); + cpu_used_ = GET_PARAM(2); + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 1) { + encoder->Control(AOME_SET_CPUUSED, cpu_used_); + } else if (video->frame() == 3) { + aom_active_map_t map = aom_active_map_t(); + /* clang-format off */ + uint8_t active_map[9 * 13] = { + 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, + 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, + 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, + 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, + 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1, + 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, + }; + /* clang-format on */ + map.cols = (kWidth + 15) / 16; + map.rows = (kHeight + 15) / 16; + ASSERT_EQ(map.cols, 13u); + ASSERT_EQ(map.rows, 9u); + map.active_map = active_map; + encoder->Control(AOME_SET_ACTIVEMAP, &map); + } else if (video->frame() == 15) { + aom_active_map_t map = aom_active_map_t(); + map.cols = (kWidth + 15) / 16; + map.rows = (kHeight + 15) / 16; + map.active_map = NULL; + encoder->Control(AOME_SET_ACTIVEMAP, &map); + } + } + + void DoTest() { + // Validate that this non multiple of 64 wide clip encodes + cfg_.g_lag_in_frames = 0; + cfg_.rc_target_bitrate = 400; + cfg_.rc_resize_allowed = 0; + cfg_.g_pass = AOM_RC_ONE_PASS; + cfg_.rc_end_usage = AOM_CBR; + cfg_.kf_max_dist = 90000; + ::libaom_test::I420VideoSource video("hantro_odd.yuv", kWidth, kHeight, 30, + 1, 0, 20); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + int cpu_used_; +}; + +TEST_P(ActiveMapTest, Test) { DoTest(); } + +class ActiveMapTestLarge : public ActiveMapTest {}; + +TEST_P(ActiveMapTestLarge, Test) { DoTest(); } + +AV1_INSTANTIATE_TEST_CASE(ActiveMapTestLarge, + ::testing::Values(::libaom_test::kRealTime), + ::testing::Range(0, 5)); + +AV1_INSTANTIATE_TEST_CASE(ActiveMapTest, + ::testing::Values(::libaom_test::kRealTime), + ::testing::Range(5, 9)); + +} // namespace diff --git a/third_party/aom/test/altref_test.cc b/third_party/aom/test/altref_test.cc new file mode 100644 index 0000000000..6dd8b5186f --- /dev/null +++ b/third_party/aom/test/altref_test.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" +namespace { + +class AltRefForcedKeyTestLarge + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params { + protected: + AltRefForcedKeyTestLarge() + : EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)), + cpu_used_(GET_PARAM(2)), forced_kf_frame_num_(1), frame_num_(0) {} + virtual ~AltRefForcedKeyTestLarge() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + cfg_.rc_end_usage = AOM_VBR; + cfg_.g_threads = 0; + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 0) { + encoder->Control(AOME_SET_CPUUSED, cpu_used_); + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); +#if CONFIG_AV1_ENCODER + // override test default for tile columns if necessary. + if (GET_PARAM(0) == &libaom_test::kAV1) { + encoder->Control(AV1E_SET_TILE_COLUMNS, 6); + } +#endif + } + frame_flags_ = + (video->frame() == forced_kf_frame_num_) ? AOM_EFLAG_FORCE_KF : 0; + } + + virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) { + if (frame_num_ == forced_kf_frame_num_) { + ASSERT_TRUE(!!(pkt->data.frame.flags & AOM_FRAME_IS_KEY)) + << "Frame #" << frame_num_ << " isn't a keyframe!"; + } + ++frame_num_; + } + + ::libaom_test::TestMode encoding_mode_; + int cpu_used_; + unsigned int forced_kf_frame_num_; + unsigned int frame_num_; +}; + +TEST_P(AltRefForcedKeyTestLarge, Frame1IsKey) { + const aom_rational timebase = { 1, 30 }; + const int lag_values[] = { 3, 15, 25, -1 }; + + forced_kf_frame_num_ = 1; + for (int i = 0; lag_values[i] != -1; ++i) { + frame_num_ = 0; + cfg_.g_lag_in_frames = lag_values[i]; + libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + timebase.den, timebase.num, 0, 30); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } +} + +TEST_P(AltRefForcedKeyTestLarge, ForcedFrameIsKey) { + const aom_rational timebase = { 1, 30 }; + const int lag_values[] = { 3, 15, 25, -1 }; + + for (int i = 0; lag_values[i] != -1; ++i) { + frame_num_ = 0; + forced_kf_frame_num_ = lag_values[i] - 1; + cfg_.g_lag_in_frames = lag_values[i]; + libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + timebase.den, timebase.num, 0, 30); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } +} + +AV1_INSTANTIATE_TEST_CASE(AltRefForcedKeyTestLarge, + ::testing::Values(::libaom_test::kOnePassGood), + ::testing::Range(0, 9)); + +} // namespace diff --git a/third_party/aom/test/android/Android.mk b/third_party/aom/test/android/Android.mk new file mode 100644 index 0000000000..74f9d7cbaf --- /dev/null +++ b/third_party/aom/test/android/Android.mk @@ -0,0 +1,58 @@ +# +# Copyright (c) 2016, Alliance for Open Media. All rights reserved +# +# This source code is subject to the terms of the BSD 2 Clause License and +# the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +# was not distributed with this source code in the LICENSE file, you can +# obtain it at www.aomedia.org/license/software. If the Alliance for Open +# Media Patent License 1.0 was not distributed with this source code in the +# PATENTS file, you can obtain it at www.aomedia.org/license/patent. +# +# This make file builds aom_test app for android. +# The test app itself runs on the command line through adb shell +# The paths are really messed up as the libaom make file +# expects to be made from a parent directory. +CUR_WD := $(call my-dir) +BINDINGS_DIR := $(CUR_WD)/../../.. +LOCAL_PATH := $(CUR_WD)/../../.. + +#libwebm +include $(CLEAR_VARS) +include $(BINDINGS_DIR)/libaom/third_party/libwebm/Android.mk +LOCAL_PATH := $(CUR_WD)/../../.. + +#libaom +include $(CLEAR_VARS) +LOCAL_STATIC_LIBRARIES := libwebm +include $(BINDINGS_DIR)/libaom/build/make/Android.mk +LOCAL_PATH := $(CUR_WD)/../.. + +#libgtest +include $(CLEAR_VARS) +LOCAL_ARM_MODE := arm +LOCAL_CPP_EXTENSION := .cc +LOCAL_MODULE := gtest +LOCAL_C_INCLUDES := $(LOCAL_PATH)/third_party/googletest/src/googletest/src +LOCAL_C_INCLUDES += $(LOCAL_PATH)/third_party/googletest/src/googletest/include +LOCAL_SRC_FILES := ./third_party/googletest/src/googletest/src/gtest-all.cc +include $(BUILD_STATIC_LIBRARY) + +#libaom_test +include $(CLEAR_VARS) +LOCAL_ARM_MODE := arm +LOCAL_MODULE := libaom_test +LOCAL_STATIC_LIBRARIES := gtest libwebm + +ifeq ($(ENABLE_SHARED),1) + LOCAL_SHARED_LIBRARIES := aom +else + LOCAL_STATIC_LIBRARIES += aom +endif + +include $(LOCAL_PATH)/test/test.mk +LOCAL_C_INCLUDES := $(BINDINGS_DIR) +FILTERED_SRC := $(sort $(filter %.cc %.c, $(LIBAOM_TEST_SRCS-yes))) +LOCAL_SRC_FILES := $(addprefix ./test/, $(FILTERED_SRC)) +# some test files depend on *_rtcd.h, ensure they're generated first. +$(eval $(call rtcd_dep_template)) +include $(BUILD_EXECUTABLE) diff --git a/third_party/aom/test/android/README b/third_party/aom/test/android/README new file mode 100644 index 0000000000..35c8297386 --- /dev/null +++ b/third_party/aom/test/android/README @@ -0,0 +1,32 @@ +Android.mk will build aom unittests on android. +1) Configure libaom from the parent directory: +./libaom/configure --target=armv7-android-gcc --enable-external-build \ + --enable-postproc --disable-install-srcs --enable-multi-res-encoding \ + --enable-temporal-denoising --disable-unit-tests --disable-install-docs \ + --disable-examples --disable-runtime-cpu-detect --sdk-path=$NDK + +2) From the parent directory, invoke ndk-build: +NDK_PROJECT_PATH=. ndk-build APP_BUILD_SCRIPT=./libaom/test/android/Android.mk \ + APP_ABI=armeabi-v7a APP_PLATFORM=android-18 APP_OPTIM=release \ + APP_STL=gnustl_static + +Note: Both adb and ndk-build are available prebuilt at: + https://chromium.googlesource.com/android_tools + +3) Run get_files.py to download the test files: +python get_files.py -i /path/to/test-data.sha1 -o /path/to/put/files \ + -u http://downloads.webmproject.org/test_data/libaom + +4) Transfer files to device using adb. Ensure you have proper permissions for +the target + +adb push /path/to/test_files /data/local/tmp +adb push /path/to/built_libs /data/local/tmp + +NOTE: Built_libs defaults to parent_dir/libs/armeabi-v7a + +5) Run tests: +adb shell +(on device) +cd /data/local/tmp +LD_LIBRARY_PATH=. ./aom_test diff --git a/third_party/aom/test/android/get_files.py b/third_party/aom/test/android/get_files.py new file mode 100644 index 0000000000..bdae9a315e --- /dev/null +++ b/third_party/aom/test/android/get_files.py @@ -0,0 +1,120 @@ +# +# Copyright (c) 2016, Alliance for Open Media. All rights reserved +# +# This source code is subject to the terms of the BSD 2 Clause License and +# the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +# was not distributed with this source code in the LICENSE file, you can +# obtain it at www.aomedia.org/license/software. If the Alliance for Open +# Media Patent License 1.0 was not distributed with this source code in the +# PATENTS file, you can obtain it at www.aomedia.org/license/patent. +# +# This simple script pulls test files from the webm homepage +# It is intelligent enough to only pull files if +# 1) File / test_data folder does not exist +# 2) SHA mismatch + +import pycurl +import csv +import hashlib +import re +import os.path +import time +import itertools +import sys +import getopt + +#globals +url = '' +file_list_path = '' +local_resource_path = '' + +# Helper functions: +# A simple function which returns the sha hash of a file in hex +def get_file_sha(filename): + try: + sha_hash = hashlib.sha1() + with open(filename, 'rb') as file: + buf = file.read(HASH_CHUNK) + while len(buf) > 0: + sha_hash.update(buf) + buf = file.read(HASH_CHUNK) + return sha_hash.hexdigest() + except IOError: + print "Error reading " + filename + +# Downloads a file from a url, and then checks the sha against the passed +# in sha +def download_and_check_sha(url, filename, sha): + path = os.path.join(local_resource_path, filename) + fp = open(path, "wb") + curl = pycurl.Curl() + curl.setopt(pycurl.URL, url + "/" + filename) + curl.setopt(pycurl.WRITEDATA, fp) + curl.perform() + curl.close() + fp.close() + return get_file_sha(path) == sha + +#constants +ftp_retries = 3 + +SHA_COL = 0 +NAME_COL = 1 +EXPECTED_COL = 2 +HASH_CHUNK = 65536 + +# Main script +try: + opts, args = \ + getopt.getopt(sys.argv[1:], \ + "u:i:o:", ["url=", "input_csv=", "output_dir="]) +except: + print 'get_files.py -u -i -o ' + sys.exit(2) + +for opt, arg in opts: + if opt == '-u': + url = arg + elif opt in ("-i", "--input_csv"): + file_list_path = os.path.join(arg) + elif opt in ("-o", "--output_dir"): + local_resource_path = os.path.join(arg) + +if len(sys.argv) != 7: + print "Expects two paths and a url!" + exit(1) + +if not os.path.isdir(local_resource_path): + os.makedirs(local_resource_path) + +file_list_csv = open(file_list_path, "rb") + +# Our 'csv' file uses multiple spaces as a delimiter, python's +# csv class only uses single character delimiters, so we convert them below +file_list_reader = csv.reader((re.sub(' +', ' ', line) \ + for line in file_list_csv), delimiter = ' ') + +file_shas = [] +file_names = [] + +for row in file_list_reader: + if len(row) != EXPECTED_COL: + continue + file_shas.append(row[SHA_COL]) + file_names.append(row[NAME_COL]) + +file_list_csv.close() + +# Download files, only if they don't already exist and have correct shas +for filename, sha in itertools.izip(file_names, file_shas): + path = os.path.join(local_resource_path, filename) + if os.path.isfile(path) \ + and get_file_sha(path) == sha: + print path + ' exists, skipping' + continue + for retry in range(0, ftp_retries): + print "Downloading " + path + if not download_and_check_sha(url, filename, sha): + print "Sha does not match, retrying..." + else: + break diff --git a/third_party/aom/test/android/scrape_gtest_log.py b/third_party/aom/test/android/scrape_gtest_log.py new file mode 100644 index 0000000000..e0c929a5d5 --- /dev/null +++ b/third_party/aom/test/android/scrape_gtest_log.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2016, Alliance for Open Media. All rights reserved +# +# This source code is subject to the terms of the BSD 2 Clause License and +# the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +# was not distributed with this source code in the LICENSE file, you can +# obtain it at www.aomedia.org/license/software. If the Alliance for Open +# Media Patent License 1.0 was not distributed with this source code in the +# PATENTS file, you can obtain it at www.aomedia.org/license/patent. +# + +"""Standalone script which parses a gtest log for json. + +Json is returned returns as an array. This script is used by the libaom +waterfall to gather json results mixed in with gtest logs. This is +dubious software engineering. +""" + +import getopt +import json +import os +import re +import sys + + +def main(): + if len(sys.argv) != 3: + print "Expects a file to write json to!" + exit(1) + + try: + opts, _ = \ + getopt.getopt(sys.argv[1:], \ + 'o:', ['output-json=']) + except getopt.GetOptError: + print 'scrape_gtest_log.py -o ' + sys.exit(2) + + output_json = '' + for opt, arg in opts: + if opt in ('-o', '--output-json'): + output_json = os.path.join(arg) + + blob = sys.stdin.read() + json_string = '[' + ','.join('{' + x + '}' for x in + re.findall(r'{([^}]*.?)}', blob)) + ']' + print blob + + output = json.dumps(json.loads(json_string), indent=4, sort_keys=True) + print output + + path = os.path.dirname(output_json) + if path and not os.path.exists(path): + os.makedirs(path) + + outfile = open(output_json, 'w') + outfile.write(output) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/third_party/aom/test/ans_codec_test.cc b/third_party/aom/test/ans_codec_test.cc new file mode 100644 index 0000000000..a1b25fbdac --- /dev/null +++ b/third_party/aom/test/ans_codec_test.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/util.h" +#include "test/y4m_video_source.h" +#include "aom_dsp/ans.h" +#include "av1/av1_dx_iface.c" + +// A note on ANS_MAX_SYMBOLS == 0: +// Fused gtest doesn't work with EXPECT_FATAL_FAILURE [1]. Just run with a +// single iteration and don't try to check the window size if we are unwindowed. +// [1] https://github.com/google/googletest/issues/356 + +namespace { + +const char kTestVideoName[] = "niklas_1280_720_30.y4m"; +const int kTestVideoFrames = 10; + +class AnsCodecTest : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWithParam { + protected: + AnsCodecTest() + : EncoderTest(GET_PARAM(0)), ans_window_size_log2_(GET_PARAM(1)) {} + + virtual ~AnsCodecTest() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(::libaom_test::kOnePassGood); + cfg_.g_lag_in_frames = 25; + cfg_.rc_end_usage = AOM_CQ; + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 1) { +#if ANS_MAX_SYMBOLS + encoder->Control(AV1E_SET_ANS_WINDOW_SIZE_LOG2, ans_window_size_log2_); +#endif + // Try to push a high symbol count through the codec + encoder->Control(AOME_SET_CQ_LEVEL, 8); + encoder->Control(AOME_SET_CPUUSED, 2); + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); + encoder->Control(AOME_SET_ARNR_MAXFRAMES, 7); + encoder->Control(AOME_SET_ARNR_STRENGTH, 5); + encoder->Control(AV1E_SET_TILE_COLUMNS, 0); + encoder->Control(AV1E_SET_TILE_ROWS, 0); + } + } + + virtual bool HandleDecodeResult(const aom_codec_err_t res_dec, + libaom_test::Decoder *decoder) { + aom_codec_ctx_t *const av1_decoder = decoder->GetDecoder(); +#if ANS_MAX_SYMBOLS + aom_codec_alg_priv_t *const priv = + reinterpret_cast(av1_decoder->priv); + FrameWorkerData *const worker_data = + reinterpret_cast(priv->frame_workers[0].data1); + AV1_COMMON *const common = &worker_data->pbi->common; + + EXPECT_EQ(ans_window_size_log2_, common->ans_window_size_log2); +#endif + + EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError(); + return AOM_CODEC_OK == res_dec; + } + + private: + int ans_window_size_log2_; +}; + +TEST_P(AnsCodecTest, BitstreamParms) { + testing::internal::scoped_ptr video( + new libaom_test::Y4mVideoSource(kTestVideoName, 0, kTestVideoFrames)); + ASSERT_TRUE(video.get() != NULL); + + ASSERT_NO_FATAL_FAILURE(RunLoop(video.get())); +} + +#if ANS_MAX_SYMBOLS +AV1_INSTANTIATE_TEST_CASE(AnsCodecTest, ::testing::Range(8, 24)); +#else +AV1_INSTANTIATE_TEST_CASE(AnsCodecTest, ::testing::Range(0, 1)); +#endif +} // namespace diff --git a/third_party/aom/test/ans_test.cc b/third_party/aom/test/ans_test.cc new file mode 100644 index 0000000000..a553a9e842 --- /dev/null +++ b/third_party/aom/test/ans_test.cc @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/acm_random.h" +#include "aom_dsp/ansreader.h" +#include "aom_dsp/buf_ans.h" + +namespace { +typedef std::vector > PvVec; + +const int kPrintStats = 0; +// Use a small buffer size to exercise ANS window spills or buffer growth +const int kBufAnsSize = 1 << 8; + +PvVec abs_encode_build_vals(int iters) { + PvVec ret; + libaom_test::ACMRandom gen(0x30317076); + double entropy = 0; + for (int i = 0; i < iters; ++i) { + uint8_t p; + do { + p = gen.Rand8(); + } while (p == 0); // zero is not a valid coding probability + bool b = gen.Rand8() < p; + ret.push_back(std::make_pair(static_cast(p), b)); + if (kPrintStats) { + double d = p / 256.; + entropy += -d * log2(d) - (1 - d) * log2(1 - d); + } + } + if (kPrintStats) printf("entropy %f\n", entropy); + return ret; +} + +bool check_rabs(const PvVec &pv_vec, uint8_t *buf) { + BufAnsCoder a; + aom_buf_ans_alloc(&a, NULL, kBufAnsSize); + buf_ans_write_init(&a, buf); + + std::clock_t start = std::clock(); + for (PvVec::const_iterator it = pv_vec.begin(); it != pv_vec.end(); ++it) { + buf_rabs_write(&a, it->second, 256 - it->first); + } + aom_buf_ans_flush(&a); + std::clock_t enc_time = std::clock() - start; + int offset = buf_ans_write_end(&a); + aom_buf_ans_free(&a); + bool okay = true; + AnsDecoder d; +#if ANS_MAX_SYMBOLS + d.window_size = kBufAnsSize; +#endif + if (ans_read_init(&d, buf, offset)) return false; + start = std::clock(); + for (PvVec::const_iterator it = pv_vec.begin(); it != pv_vec.end(); ++it) { + okay = okay && (rabs_read(&d, 256 - it->first) != 0) == it->second; + } + std::clock_t dec_time = std::clock() - start; + if (!okay) return false; + if (kPrintStats) + printf("uABS size %d enc_time %f dec_time %f\n", offset, + static_cast(enc_time) / CLOCKS_PER_SEC, + static_cast(dec_time) / CLOCKS_PER_SEC); + return ans_read_end(&d) != 0; +} + +const aom_cdf_prob spareto65[] = { 8320, 6018, 4402, 3254, 4259, + 3919, 2057, 492, 45, 2 }; + +const int kRansSymbols = + static_cast(sizeof(spareto65) / sizeof(spareto65[0])); + +struct rans_sym { + aom_cdf_prob prob; + aom_cdf_prob cum_prob; // not-inclusive +}; + +std::vector ans_encode_build_vals(rans_sym *const tab, int iters) { + aom_cdf_prob sum = 0; + for (int i = 0; i < kRansSymbols; ++i) { + tab[i].cum_prob = sum; + tab[i].prob = spareto65[i]; + sum += spareto65[i]; + } + std::vector p_to_sym; + for (int i = 0; i < kRansSymbols; ++i) { + p_to_sym.insert(p_to_sym.end(), tab[i].prob, i); + } + assert(p_to_sym.size() == RANS_PRECISION); + std::vector ret; + libaom_test::ACMRandom gen(18543637); + for (int i = 0; i < iters; ++i) { + int sym = + p_to_sym[((gen.Rand8() << 8) + gen.Rand8()) & (RANS_PRECISION - 1)]; + ret.push_back(sym); + } + return ret; +} + +void rans_build_dec_tab(const struct rans_sym sym_tab[], + aom_cdf_prob *dec_tab) { + unsigned int sum = 0; + for (int i = 0; sum < RANS_PRECISION; ++i) { + dec_tab[i] = sum += sym_tab[i].prob; + } +} + +bool check_rans(const std::vector &sym_vec, const rans_sym *const tab, + uint8_t *buf) { + BufAnsCoder a; + aom_buf_ans_alloc(&a, NULL, kBufAnsSize); + buf_ans_write_init(&a, buf); + aom_cdf_prob dec_tab[kRansSymbols]; + rans_build_dec_tab(tab, dec_tab); + + std::clock_t start = std::clock(); + for (std::vector::const_iterator it = sym_vec.begin(); + it != sym_vec.end(); ++it) { + buf_rans_write(&a, tab[*it].cum_prob, tab[*it].prob); + } + aom_buf_ans_flush(&a); + std::clock_t enc_time = std::clock() - start; + int offset = buf_ans_write_end(&a); + aom_buf_ans_free(&a); + bool okay = true; + AnsDecoder d; +#if ANS_MAX_SYMBOLS + d.window_size = kBufAnsSize; +#endif + if (ans_read_init(&d, buf, offset)) return false; + start = std::clock(); + for (std::vector::const_iterator it = sym_vec.begin(); + it != sym_vec.end(); ++it) { + okay &= rans_read(&d, dec_tab) == *it; + } + std::clock_t dec_time = std::clock() - start; + if (!okay) return false; + if (kPrintStats) + printf("rANS size %d enc_time %f dec_time %f\n", offset, + static_cast(enc_time) / CLOCKS_PER_SEC, + static_cast(dec_time) / CLOCKS_PER_SEC); + return ans_read_end(&d) != 0; +} + +class AbsTestFix : public ::testing::Test { + protected: + static void SetUpTestCase() { pv_vec_ = abs_encode_build_vals(kNumBools); } + virtual void SetUp() { buf_ = new uint8_t[kNumBools / 8]; } + virtual void TearDown() { delete[] buf_; } + static const int kNumBools = 100000000; + static PvVec pv_vec_; + uint8_t *buf_; +}; +PvVec AbsTestFix::pv_vec_; + +class AnsTestFix : public ::testing::Test { + protected: + static void SetUpTestCase() { + sym_vec_ = ans_encode_build_vals(rans_sym_tab_, kNumSyms); + } + virtual void SetUp() { buf_ = new uint8_t[kNumSyms / 2]; } + virtual void TearDown() { delete[] buf_; } + static const int kNumSyms = 25000000; + static std::vector sym_vec_; + static rans_sym rans_sym_tab_[kRansSymbols]; + uint8_t *buf_; +}; +std::vector AnsTestFix::sym_vec_; +rans_sym AnsTestFix::rans_sym_tab_[kRansSymbols]; + +TEST_F(AbsTestFix, Rabs) { EXPECT_TRUE(check_rabs(pv_vec_, buf_)); } +TEST_F(AnsTestFix, Rans) { + EXPECT_TRUE(check_rans(sym_vec_, rans_sym_tab_, buf_)); +} +TEST(AnsTest, FinalStateSerialization) { + for (unsigned i = L_BASE; i < L_BASE * IO_BASE; ++i) { + uint8_t buf[8]; + AnsCoder c; + ans_write_init(&c, buf); + c.state = i; + const int written_size = ans_write_end(&c); + ASSERT_LT(static_cast(written_size), sizeof(buf)); + AnsDecoder d; +#if ANS_MAX_SYMBOLS + // There is no real data window here because no symbols are sent through + // ans (only synthetic states), so use a dummy value + d.window_size = 1024; +#endif + const int read_init_status = ans_read_init(&d, buf, written_size); + EXPECT_EQ(read_init_status, 0); + EXPECT_EQ(d.state, i); + } +} +} // namespace diff --git a/third_party/aom/test/aomcx_set_ref.sh b/third_party/aom/test/aomcx_set_ref.sh new file mode 100755 index 0000000000..f51b73c58e --- /dev/null +++ b/third_party/aom/test/aomcx_set_ref.sh @@ -0,0 +1,58 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This file tests the libaom aom_cx_set_ref example. To add new tests to this +## file, do the following: +## 1. Write a shell function (this is your test). +## 2. Add the function to aom_cx_set_ref_tests (on a new line). +## +. $(dirname $0)/tools_common.sh + +# Environment check: $YUV_RAW_INPUT is required. +aom_cx_set_ref_verify_environment() { + if [ ! -e "${YUV_RAW_INPUT}" ]; then + echo "Libaom test data must exist in LIBAOM_TEST_DATA_PATH." + return 1 + fi +} + +# Runs aom_cx_set_ref and updates the reference frame before encoding frame 90. +# $1 is the codec name, which aom_cx_set_ref does not support at present: It's +# currently used only to name the output file. +# TODO(tomfinegan): Pass the codec param once the example is updated to support +# AV1. +aom_set_ref() { + local encoder="${LIBAOM_BIN_PATH}/aom_cx_set_ref${AOM_TEST_EXE_SUFFIX}" + local codec="$1" + local output_file="${AOM_TEST_OUTPUT_DIR}/aom_cx_set_ref_${codec}.ivf" + local ref_frame_num=4 + local limit=10 + if [ ! -x "${encoder}" ]; then + elog "${encoder} does not exist or is not executable." + return 1 + fi + + eval "${AOM_TEST_PREFIX}" "${encoder}" "${codec}" "${YUV_RAW_INPUT_WIDTH}" \ + "${YUV_RAW_INPUT_HEIGHT}" "${YUV_RAW_INPUT}" "${output_file}" \ + "${ref_frame_num}" "${limit}" ${devnull} + + [ -e "${output_file}" ] || return 1 +} + +aom_cx_set_ref_av1() { + if [ "$(av1_encode_available)" = "yes" ]; then + aom_set_ref av1 || return 1 + fi +} + +aom_cx_set_ref_tests="aom_cx_set_ref_av1" + +run_tests aom_cx_set_ref_verify_environment "${aom_cx_set_ref_tests}" + diff --git a/third_party/aom/test/aomdec.sh b/third_party/aom/test/aomdec.sh new file mode 100755 index 0000000000..28901ed1bd --- /dev/null +++ b/third_party/aom/test/aomdec.sh @@ -0,0 +1,124 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This file tests aomdec. To add new tests to this file, do the following: +## 1. Write a shell function (this is your test). +## 2. Add the function to aomdec_tests (on a new line). +## +. $(dirname $0)/tools_common.sh + +# Environment check: Make sure input is available. +aomdec_verify_environment() { + if [ "$(av1_encode_available)" != "yes" ] ; then + if [ ! -e "${AV1_WEBM_FILE}" ] || \ + [ ! -e "${AV1_FPM_WEBM_FILE}" ] || \ + [ ! -e "${AV1_LT_50_FRAMES_WEBM_FILE}" ] ; then + elog "Libaom test data must exist in LIBAOM_TEST_DATA_PATH." + return 1 + fi + fi + if [ -z "$(aom_tool_path aomdec)" ]; then + elog "aomdec not found. It must exist in LIBAOM_BIN_PATH or its parent." + return 1 + fi +} + +# Wrapper function for running aomdec with pipe input. Requires that +# LIBAOM_BIN_PATH points to the directory containing aomdec. $1 is used as the +# input file path and shifted away. All remaining parameters are passed through +# to aomdec. +aomdec_pipe() { + local readonly input="$1" + shift + if [ ! -e "${input}" ]; then + local file="${AOM_TEST_OUTPUT_DIR}/test_encode.ivf" + encode_yuv_raw_input_av1 "${file}" --ivf + else + local file="${input}" + fi + cat "${file}" | aomdec - "$@" ${devnull} +} + + +# Wrapper function for running aomdec. Requires that LIBAOM_BIN_PATH points to +# the directory containing aomdec. $1 one is used as the input file path and +# shifted away. All remaining parameters are passed through to aomdec. +aomdec() { + local readonly decoder="$(aom_tool_path aomdec)" + local readonly input="$1" + shift + eval "${AOM_TEST_PREFIX}" "${decoder}" "$input" "$@" ${devnull} +} + +aomdec_can_decode_av1() { + if [ "$(av1_decode_available)" = "yes" ]; then + echo yes + fi +} + +aomdec_aom_ivf_pipe_input() { + if [ "$(aomdec_can_decode_av1)" = "yes" ]; then + aomdec_pipe "${AOM_IVF_FILE}" --summary --noblit + fi +} + +aomdec_av1_webm() { + if [ "$(aomdec_can_decode_av1)" = "yes" ] && \ + [ "$(webm_io_available)" = "yes" ]; then + if [ ! -e "${AV1_WEBM_FILE}" ]; then + local file="${AOM_TEST_OUTPUT_DIR}/test_encode.webm" + encode_yuv_raw_input_av1 "${file}" + else + aomdec "${AV1_WEBM_FILE}" --summary --noblit + fi + fi +} + +aomdec_av1_webm_frame_parallel() { + if [ "$(aomdec_can_decode_av1)" = "yes" ] && \ + [ "$(webm_io_available)" = "yes" ]; then + local file + if [ ! -e "${AV1_WEBM_FILE}" ]; then + file="${AOM_TEST_OUTPUT_DIR}/test_encode.webm" + encode_yuv_raw_input_av1 "${file}" "--ivf --error-resilient=1 " + else + file="${AV1_FPM_WEBM_FILE}" + fi + for threads in 2 3 4 5 6 7 8; do + aomdec "${file}" --summary --noblit --threads=$threads \ + --frame-parallel + done + fi +} + +# TODO(vigneshv): Enable or remove this test and associated code. +DISABLED_aomdec_av1_webm_less_than_50_frames() { + # ensure that reaching eof in webm_guess_framerate doesn't result in invalid + # frames in actual webm_read_frame calls. + if [ "$(aomdec_can_decode_av1)" = "yes" ] && \ + [ "$(webm_io_available)" = "yes" ]; then + local readonly decoder="$(aom_tool_path aomdec)" + local readonly expected=10 + local readonly num_frames=$(${AOM_TEST_PREFIX} "${decoder}" \ + "${AV1_LT_50_FRAMES_WEBM_FILE}" --summary --noblit 2>&1 \ + | awk '/^[0-9]+ decoded frames/ { print $1 }') + if [ "$num_frames" -ne "$expected" ]; then + elog "Output frames ($num_frames) != expected ($expected)" + return 1 + fi + fi +} + +aomdec_tests="aomdec_av1_webm + aomdec_av1_webm_frame_parallel + aomdec_aom_ivf_pipe_input + DISABLED_aomdec_av1_webm_less_than_50_frames" + +run_tests aomdec_verify_environment "${aomdec_tests}" diff --git a/third_party/aom/test/aomenc.sh b/third_party/aom/test/aomenc.sh new file mode 100755 index 0000000000..57a4c28a5a --- /dev/null +++ b/third_party/aom/test/aomenc.sh @@ -0,0 +1,241 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This file tests aomenc using hantro_collage_w352h288.yuv as input. To add +## new tests to this file, do the following: +## 1. Write a shell function (this is your test). +## 2. Add the function to aomenc_tests (on a new line). +## +. $(dirname $0)/tools_common.sh + +readonly TEST_FRAMES=5 + +# Environment check: Make sure input is available. +aomenc_verify_environment() { + if [ ! -e "${YUV_RAW_INPUT}" ]; then + elog "The file ${YUV_RAW_INPUT##*/} must exist in LIBAOM_TEST_DATA_PATH." + return 1 + fi + if [ "$(aomenc_can_encode_av1)" = "yes" ]; then + if [ ! -e "${Y4M_NOSQ_PAR_INPUT}" ]; then + elog "The file ${Y4M_NOSQ_PAR_INPUT##*/} must exist in" + elog "LIBAOM_TEST_DATA_PATH." + return 1 + fi + fi + if [ -z "$(aom_tool_path aomenc)" ]; then + elog "aomenc not found. It must exist in LIBAOM_BIN_PATH or its parent." + return 1 + fi +} + +aomenc_can_encode_av1() { + if [ "$(av1_encode_available)" = "yes" ]; then + echo yes + fi +} + +aomenc_can_encode_av1() { + if [ "$(av1_encode_available)" = "yes" ]; then + echo yes + fi +} + +# Utilities that echo aomenc input file parameters. +y4m_input_non_square_par() { + echo ""${Y4M_NOSQ_PAR_INPUT}"" +} + +y4m_input_720p() { + echo ""${Y4M_720P_INPUT}"" +} + +# Echo default aomenc real time encoding params. $1 is the codec, which defaults +# to av1 if unspecified. +aomenc_rt_params() { + local readonly codec="${1:-av1}" + echo "--codec=${codec} + --buf-initial-sz=500 + --buf-optimal-sz=600 + --buf-sz=1000 + --cpu-used=-6 + --end-usage=cbr + --error-resilient=1 + --kf-max-dist=90000 + --lag-in-frames=0 + --max-intra-rate=300 + --max-q=56 + --min-q=2 + --noise-sensitivity=0 + --overshoot-pct=50 + --passes=1 + --profile=0 + --resize-allowed=0 + --rt + --static-thresh=0 + --undershoot-pct=50" +} + +# Wrapper function for running aomenc with pipe input. Requires that +# LIBAOM_BIN_PATH points to the directory containing aomenc. $1 is used as the +# input file path and shifted away. All remaining parameters are passed through +# to aomenc. +aomenc_pipe() { + local readonly encoder="$(aom_tool_path aomenc)" + local readonly input="$1" + shift + cat "${input}" | eval "${AOM_TEST_PREFIX}" "${encoder}" - \ + --test-decode=fatal \ + "$@" ${devnull} +} + +# Wrapper function for running aomenc. Requires that LIBAOM_BIN_PATH points to +# the directory containing aomenc. $1 one is used as the input file path and +# shifted away. All remaining parameters are passed through to aomenc. +aomenc() { + local readonly encoder="$(aom_tool_path aomenc)" + local readonly input="$1" + shift + eval "${AOM_TEST_PREFIX}" "${encoder}" "${input}" \ + --test-decode=fatal \ + "$@" ${devnull} +} + +aomenc_av1_ivf() { + if [ "$(aomenc_can_encode_av1)" = "yes" ]; then + local readonly output="${AOM_TEST_OUTPUT_DIR}/av1.ivf" + aomenc $(yuv_raw_input) \ + --codec=av1 \ + --limit="${TEST_FRAMES}" \ + --ivf \ + --output="${output}" + + if [ ! -e "${output}" ]; then + elog "Output file does not exist." + return 1 + fi + fi +} + +aomenc_av1_webm() { + if [ "$(aomenc_can_encode_av1)" = "yes" ] && \ + [ "$(webm_io_available)" = "yes" ]; then + local readonly output="${AOM_TEST_OUTPUT_DIR}/av1.webm" + aomenc $(yuv_raw_input) \ + --codec=av1 \ + --limit="${TEST_FRAMES}" \ + --output="${output}" + + if [ ! -e "${output}" ]; then + elog "Output file does not exist." + return 1 + fi + fi +} + +aomenc_av1_webm_2pass() { + if [ "$(aomenc_can_encode_av1)" = "yes" ] && \ + [ "$(webm_io_available)" = "yes" ]; then + local readonly output="${AOM_TEST_OUTPUT_DIR}/av1.webm" + aomenc $(yuv_raw_input) \ + --codec=av1 \ + --limit="${TEST_FRAMES}" \ + --output="${output}" \ + --passes=2 + + if [ ! -e "${output}" ]; then + elog "Output file does not exist." + return 1 + fi + fi +} + +aomenc_av1_ivf_lossless() { + if [ "$(aomenc_can_encode_av1)" = "yes" ]; then + local readonly output="${AOM_TEST_OUTPUT_DIR}/av1_lossless.ivf" + aomenc $(yuv_raw_input) \ + --codec=av1 \ + --limit="${TEST_FRAMES}" \ + --ivf \ + --output="${output}" \ + --lossless=1 + + if [ ! -e "${output}" ]; then + elog "Output file does not exist." + return 1 + fi + fi +} + +aomenc_av1_ivf_minq0_maxq0() { + if [ "$(aomenc_can_encode_av1)" = "yes" ]; then + local readonly output="${AOM_TEST_OUTPUT_DIR}/av1_lossless_minq0_maxq0.ivf" + aomenc $(yuv_raw_input) \ + --codec=av1 \ + --limit="${TEST_FRAMES}" \ + --ivf \ + --output="${output}" \ + --min-q=0 \ + --max-q=0 + + if [ ! -e "${output}" ]; then + elog "Output file does not exist." + return 1 + fi + fi +} + +aomenc_av1_webm_lag5_frames10() { + if [ "$(aomenc_can_encode_av1)" = "yes" ] && \ + [ "$(webm_io_available)" = "yes" ]; then + local readonly lag_total_frames=10 + local readonly lag_frames=5 + local readonly output="${AOM_TEST_OUTPUT_DIR}/av1_lag5_frames10.webm" + aomenc $(yuv_raw_input) \ + --codec=av1 \ + --limit="${lag_total_frames}" \ + --lag-in-frames="${lag_frames}" \ + --output="${output}" \ + --passes=2 \ + --auto-alt-ref=1 + + if [ ! -e "${output}" ]; then + elog "Output file does not exist." + return 1 + fi + fi +} + +# TODO(fgalligan): Test that DisplayWidth is different than video width. +aomenc_av1_webm_non_square_par() { + if [ "$(aomenc_can_encode_av1)" = "yes" ] && \ + [ "$(webm_io_available)" = "yes" ]; then + local readonly output="${AOM_TEST_OUTPUT_DIR}/av1_non_square_par.webm" + aomenc $(y4m_input_non_square_par) \ + --codec=av1 \ + --limit="${TEST_FRAMES}" \ + --output="${output}" + + if [ ! -e "${output}" ]; then + elog "Output file does not exist." + return 1 + fi + fi +} + +aomenc_tests="aomenc_av1_ivf + aomenc_av1_webm + aomenc_av1_webm_2pass + aomenc_av1_ivf_lossless + aomenc_av1_ivf_minq0_maxq0 + aomenc_av1_webm_lag5_frames10 + aomenc_av1_webm_non_square_par" + +run_tests aomenc_verify_environment "${aomenc_tests}" diff --git a/third_party/aom/test/aq_segment_test.cc b/third_party/aom/test/aq_segment_test.cc new file mode 100644 index 0000000000..5dc93ec793 --- /dev/null +++ b/third_party/aom/test/aq_segment_test.cc @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "./aom_config.h" +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" + +namespace { + +class AqSegmentTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params { + protected: + AqSegmentTest() : EncoderTest(GET_PARAM(0)) {} + virtual ~AqSegmentTest() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(GET_PARAM(1)); + set_cpu_used_ = GET_PARAM(2); + aq_mode_ = 0; + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 1) { + encoder->Control(AOME_SET_CPUUSED, set_cpu_used_); + encoder->Control(AV1E_SET_AQ_MODE, aq_mode_); +#if CONFIG_EXT_DELTA_Q + encoder->Control(AV1E_SET_DELTAQ_MODE, deltaq_mode_); +#endif + encoder->Control(AOME_SET_MAX_INTRA_BITRATE_PCT, 100); + } + } + + void DoTest(int aq_mode) { + aq_mode_ = aq_mode; +#if CONFIG_EXT_DELTA_Q + deltaq_mode_ = 0; +#endif + cfg_.kf_max_dist = 12; + cfg_.rc_min_quantizer = 8; + cfg_.rc_max_quantizer = 56; + cfg_.rc_end_usage = AOM_CBR; + cfg_.g_lag_in_frames = 6; + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 500; + cfg_.rc_buf_sz = 1000; + cfg_.rc_target_bitrate = 300; + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, + 288, 30, 1, 0, 15); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + int set_cpu_used_; + int aq_mode_; +#if CONFIG_EXT_DELTA_Q + int deltaq_mode_; +#endif +}; + +// Validate that this AQ segmentation mode (AQ=1, variance_ap) +// encodes and decodes without a mismatch. +TEST_P(AqSegmentTest, TestNoMisMatchAQ1) { DoTest(1); } + +// Validate that this AQ segmentation mode (AQ=2, complexity_aq) +// encodes and decodes without a mismatch. +TEST_P(AqSegmentTest, TestNoMisMatchAQ2) { DoTest(2); } + +// Validate that this AQ segmentation mode (AQ=3, cyclic_refresh_aq) +// encodes and decodes without a mismatch. +TEST_P(AqSegmentTest, TestNoMisMatchAQ3) { DoTest(3); } + +class AqSegmentTestLarge : public AqSegmentTest {}; + +TEST_P(AqSegmentTestLarge, TestNoMisMatchAQ1) { DoTest(1); } + +TEST_P(AqSegmentTestLarge, TestNoMisMatchAQ2) { DoTest(2); } + +TEST_P(AqSegmentTestLarge, TestNoMisMatchAQ3) { DoTest(3); } + +#if CONFIG_DELTA_Q & !CONFIG_EXT_DELTA_Q +// Validate that this AQ mode (AQ=4, delta q) +// encodes and decodes without a mismatch. +TEST_P(AqSegmentTest, TestNoMisMatchAQ4) { + cfg_.rc_end_usage = AOM_CQ; + aq_mode_ = 4; + + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 100); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} +#endif + +#if CONFIG_EXT_DELTA_Q +// Validate that this delta q mode +// encodes and decodes without a mismatch. +TEST_P(AqSegmentTest, TestNoMisMatchExtDeltaQ) { + cfg_.rc_end_usage = AOM_CQ; + aq_mode_ = 0; + deltaq_mode_ = 2; + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 100); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} +#endif + +AV1_INSTANTIATE_TEST_CASE(AqSegmentTest, + ::testing::Values(::libaom_test::kRealTime, + ::libaom_test::kOnePassGood), + ::testing::Range(5, 9)); +AV1_INSTANTIATE_TEST_CASE(AqSegmentTestLarge, + ::testing::Values(::libaom_test::kRealTime, + ::libaom_test::kOnePassGood), + ::testing::Range(3, 5)); +} // namespace diff --git a/third_party/aom/test/arf_freq_test.cc b/third_party/aom/test/arf_freq_test.cc new file mode 100644 index 0000000000..bef58b3e85 --- /dev/null +++ b/third_party/aom/test/arf_freq_test.cc @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/util.h" +#include "test/y4m_video_source.h" +#include "test/yuv_video_source.h" +#include "av1/encoder/ratectrl.h" + +namespace { + +const unsigned int kFrames = 100; +const int kBitrate = 500; + +#define ARF_NOT_SEEN 1000001 +#define ARF_SEEN_ONCE 1000000 + +typedef struct { + const char *filename; + unsigned int width; + unsigned int height; + unsigned int framerate_num; + unsigned int framerate_den; + unsigned int input_bit_depth; + aom_img_fmt fmt; + aom_bit_depth_t bit_depth; + unsigned int profile; +} TestVideoParam; + +typedef struct { + libaom_test::TestMode mode; + int cpu_used; +} TestEncodeParam; + +const TestVideoParam kTestVectors[] = { + // artificially increase framerate to trigger default check + { "hantro_collage_w352h288.yuv", 352, 288, 5000, 1, 8, AOM_IMG_FMT_I420, + AOM_BITS_8, 0 }, + { "hantro_collage_w352h288.yuv", 352, 288, 30, 1, 8, AOM_IMG_FMT_I420, + AOM_BITS_8, 0 }, + { "rush_hour_444.y4m", 352, 288, 30, 1, 8, AOM_IMG_FMT_I444, AOM_BITS_8, 1 }, +#if CONFIG_HIGHBITDEPTH +// Add list of profile 2/3 test videos here ... +#endif // CONFIG_HIGHBITDEPTH +}; + +const TestEncodeParam kEncodeVectors[] = { + { ::libaom_test::kOnePassGood, 2 }, { ::libaom_test::kOnePassGood, 5 }, + { ::libaom_test::kTwoPassGood, 1 }, { ::libaom_test::kTwoPassGood, 2 }, + { ::libaom_test::kTwoPassGood, 5 }, { ::libaom_test::kRealTime, 5 }, +}; + +const int kMinArfVectors[] = { + // NOTE: 0 refers to the default built-in logic in: + // av1_rc_get_default_min_gf_interval(...) + 0, 4, 8, 12, 15 +}; + +int is_extension_y4m(const char *filename) { + const char *dot = strrchr(filename, '.'); + if (!dot || dot == filename) + return 0; + else + return !strcmp(dot, ".y4m"); +} + +class ArfFreqTestLarge + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith3Params { + protected: + ArfFreqTestLarge() + : EncoderTest(GET_PARAM(0)), test_video_param_(GET_PARAM(1)), + test_encode_param_(GET_PARAM(2)), min_arf_requested_(GET_PARAM(3)) {} + + virtual ~ArfFreqTestLarge() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(test_encode_param_.mode); + if (test_encode_param_.mode != ::libaom_test::kRealTime) { + cfg_.g_lag_in_frames = 25; + cfg_.rc_end_usage = AOM_VBR; + } else { + cfg_.g_lag_in_frames = 0; + cfg_.rc_end_usage = AOM_CBR; + cfg_.rc_buf_sz = 1000; + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 600; + } + dec_cfg_.threads = 4; + } + + virtual void BeginPassHook(unsigned int) { + min_run_ = ARF_NOT_SEEN; + run_of_visible_frames_ = 0; + } + + int GetNumFramesInPkt(const aom_codec_cx_pkt_t *pkt) { + const uint8_t *buffer = reinterpret_cast(pkt->data.frame.buf); + const uint8_t marker = buffer[pkt->data.frame.sz - 1]; + const int mag = ((marker >> 3) & 3) + 1; + int frames = (marker & 0x7) + 1; + const unsigned int index_sz = 2 + mag * frames; + // Check for superframe or not. + // Assume superframe has only one visible frame, the rest being + // invisible. If superframe index is not found, then there is only + // one frame. + if (!((marker & 0xe0) == 0xc0 && pkt->data.frame.sz >= index_sz && + buffer[pkt->data.frame.sz - index_sz] == marker)) { + frames = 1; + } + return frames; + } + + virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) { + if (pkt->kind != AOM_CODEC_CX_FRAME_PKT) return; + const int frames = GetNumFramesInPkt(pkt); + if (frames == 1) { + run_of_visible_frames_++; + } else if (frames == 2) { + if (min_run_ == ARF_NOT_SEEN) { + min_run_ = ARF_SEEN_ONCE; + } else if (min_run_ == ARF_SEEN_ONCE || + run_of_visible_frames_ < min_run_) { + min_run_ = run_of_visible_frames_; + } + run_of_visible_frames_ = 1; + } else { + min_run_ = 0; + run_of_visible_frames_ = 1; + } + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 0) { + encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1); + encoder->Control(AV1E_SET_TILE_COLUMNS, 4); + encoder->Control(AOME_SET_CPUUSED, test_encode_param_.cpu_used); + encoder->Control(AV1E_SET_MIN_GF_INTERVAL, min_arf_requested_); + if (test_encode_param_.mode != ::libaom_test::kRealTime) { + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); + encoder->Control(AOME_SET_ARNR_MAXFRAMES, 7); + encoder->Control(AOME_SET_ARNR_STRENGTH, 5); + } + } + } + + int GetMinVisibleRun() const { return min_run_; } + + int GetMinArfDistanceRequested() const { + if (min_arf_requested_) + return min_arf_requested_; + else + return av1_rc_get_default_min_gf_interval( + test_video_param_.width, test_video_param_.height, + (double)test_video_param_.framerate_num / + test_video_param_.framerate_den); + } + + TestVideoParam test_video_param_; + TestEncodeParam test_encode_param_; + + private: + int min_arf_requested_; + int min_run_; + int run_of_visible_frames_; +}; + +TEST_P(ArfFreqTestLarge, MinArfFreqTest) { + cfg_.rc_target_bitrate = kBitrate; + cfg_.g_error_resilient = 0; + cfg_.g_profile = test_video_param_.profile; + cfg_.g_input_bit_depth = test_video_param_.input_bit_depth; + cfg_.g_bit_depth = test_video_param_.bit_depth; + init_flags_ = AOM_CODEC_USE_PSNR; + if (cfg_.g_bit_depth > 8) init_flags_ |= AOM_CODEC_USE_HIGHBITDEPTH; + + testing::internal::scoped_ptr video; + if (is_extension_y4m(test_video_param_.filename)) { + video.reset(new libaom_test::Y4mVideoSource(test_video_param_.filename, 0, + kFrames)); + } else { + video.reset(new libaom_test::YUVVideoSource( + test_video_param_.filename, test_video_param_.fmt, + test_video_param_.width, test_video_param_.height, + test_video_param_.framerate_num, test_video_param_.framerate_den, 0, + kFrames)); + } + + ASSERT_NO_FATAL_FAILURE(RunLoop(video.get())); + const int min_run = GetMinVisibleRun(); + const int min_arf_dist_requested = GetMinArfDistanceRequested(); + if (min_run != ARF_NOT_SEEN && min_run != ARF_SEEN_ONCE) { + const int min_arf_dist = min_run + 1; + EXPECT_GE(min_arf_dist, min_arf_dist_requested); + } +} + +#if CONFIG_HIGHBITDEPTH || CONFIG_EXT_REFS +#if CONFIG_AV1_ENCODER +// TODO(angiebird): 25-29 fail in high bitdepth mode. +// TODO(zoeliu): This ArfFreqTest does not work with BWDREF_FRAME, as +// BWDREF_FRAME is also a non-show frame, and the minimum run between two +// consecutive BWDREF_FRAME's may vary between 1 and any arbitrary positive +// number as long as it does not exceed the gf_group interval. +INSTANTIATE_TEST_CASE_P( + DISABLED_AV1, ArfFreqTestLarge, + ::testing::Combine( + ::testing::Values( + static_cast(&libaom_test::kAV1)), + ::testing::ValuesIn(kTestVectors), ::testing::ValuesIn(kEncodeVectors), + ::testing::ValuesIn(kMinArfVectors))); +#endif // CONFIG_AV1_ENCODER +#else +AV1_INSTANTIATE_TEST_CASE(ArfFreqTestLarge, ::testing::ValuesIn(kTestVectors), + ::testing::ValuesIn(kEncodeVectors), + ::testing::ValuesIn(kMinArfVectors)); +#endif // CONFIG_HIGHBITDEPTH || CONFIG_EXT_REFS +} // namespace diff --git a/third_party/aom/test/av1_convolve_optimz_test.cc b/third_party/aom/test/av1_convolve_optimz_test.cc new file mode 100644 index 0000000000..fd0f6dbce2 --- /dev/null +++ b/third_party/aom/test/av1_convolve_optimz_test.cc @@ -0,0 +1,405 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" + +namespace { + +using std::tr1::tuple; +using libaom_test::ACMRandom; + +typedef void (*ConvInit)(); +typedef void (*conv_filter_t)(const uint8_t *, int, uint8_t *, int, int, int, + const InterpFilterParams, int, int, + ConvolveParams *); +#if CONFIG_HIGHBITDEPTH +typedef void (*hbd_conv_filter_t)(const uint16_t *, int, uint16_t *, int, int, + int, const InterpFilterParams, int, int, int, + int); +#endif + +// Test parameter list: +// , filter_params, subpel_x_q4, avg> +typedef tuple BlockDimension; +typedef tuple + ConvParams; +#if CONFIG_HIGHBITDEPTH +// Test parameter list: +// , filter_params, subpel_x_q4, avg, bit_dpeth> +typedef tuple + HbdConvParams; +#endif + +// Note: +// src_ and src_ref_ have special boundary requirement +// dst_ and dst_ref_ don't +const size_t maxWidth = 256; +const size_t maxHeight = 256; +const size_t maxBlockSize = maxWidth * maxHeight; +const int horizOffset = 32; +const int vertiOffset = 32; +const int stride = 128; +const int x_step_q4 = 16; + +class AV1ConvolveOptimzTest : public ::testing::TestWithParam { + public: + virtual ~AV1ConvolveOptimzTest() {} + virtual void SetUp() { + ConvInit conv_init = GET_PARAM(0); + conv_init(); + conv_horiz_ = GET_PARAM(1); + conv_vert_ = GET_PARAM(2); + BlockDimension block = GET_PARAM(3); + width_ = std::tr1::get<0>(block); + height_ = std::tr1::get<1>(block); + filter_ = GET_PARAM(4); + subpel_ = GET_PARAM(5); + int ref = GET_PARAM(6); + const int plane = 0; + conv_params_ = get_conv_params(ref, plane); + + alloc_ = new uint8_t[maxBlockSize * 4]; + src_ = alloc_ + (vertiOffset * maxWidth); + src_ += horizOffset; + src_ref_ = src_ + maxBlockSize; + + dst_ = alloc_ + 2 * maxBlockSize; + dst_ref_ = alloc_ + 3 * maxBlockSize; + } + + virtual void TearDown() { + delete[] alloc_; + libaom_test::ClearSystemState(); + } + + protected: + void RunHorizFilterBitExactCheck(); + void RunVertFilterBitExactCheck(); + + private: + void PrepFilterBuffer(); + void DiffFilterBuffer(); + conv_filter_t conv_horiz_; + conv_filter_t conv_vert_; + uint8_t *alloc_; + uint8_t *src_; + uint8_t *dst_; + uint8_t *src_ref_; + uint8_t *dst_ref_; + int width_; + int height_; + InterpFilter filter_; + int subpel_; + ConvolveParams conv_params_; +}; + +void AV1ConvolveOptimzTest::PrepFilterBuffer() { + int r, c; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + + memset(alloc_, 0, 4 * maxBlockSize * sizeof(alloc_[0])); + + uint8_t *src_ptr = src_; + uint8_t *dst_ptr = dst_; + uint8_t *src_ref_ptr = src_ref_; + uint8_t *dst_ref_ptr = dst_ref_; + + for (r = 0; r < height_; ++r) { + for (c = 0; c < width_; ++c) { + src_ptr[c] = rnd.Rand8(); + src_ref_ptr[c] = src_ptr[c]; + dst_ptr[c] = rnd.Rand8(); + dst_ref_ptr[c] = dst_ptr[c]; + } + src_ptr += stride; + src_ref_ptr += stride; + dst_ptr += stride; + dst_ref_ptr += stride; + } +} + +void AV1ConvolveOptimzTest::DiffFilterBuffer() { + int r, c; + const uint8_t *dst_ptr = dst_; + const uint8_t *dst_ref_ptr = dst_ref_; + for (r = 0; r < height_; ++r) { + for (c = 0; c < width_; ++c) { + EXPECT_EQ((uint8_t)dst_ref_ptr[c], (uint8_t)dst_ptr[c]) + << "Error at row: " << r << " col: " << c << " " + << "w = " << width_ << " " + << "h = " << height_ << " " + << "filter group index = " << filter_ << " " + << "filter index = " << subpel_; + } + dst_ptr += stride; + dst_ref_ptr += stride; + } +} + +void AV1ConvolveOptimzTest::RunHorizFilterBitExactCheck() { + PrepFilterBuffer(); + + InterpFilterParams filter_params = av1_get_interp_filter_params(filter_); + + av1_convolve_horiz_c(src_ref_, stride, dst_ref_, stride, width_, height_, + filter_params, subpel_, x_step_q4, &conv_params_); + + conv_horiz_(src_, stride, dst_, stride, width_, height_, filter_params, + subpel_, x_step_q4, &conv_params_); + + DiffFilterBuffer(); + + // Note: + // Here we need calculate a height which is different from the specified one + // and test again. + int intermediate_height = + (((height_ - 1) * 16 + subpel_) >> SUBPEL_BITS) + filter_params.taps; + PrepFilterBuffer(); + + av1_convolve_horiz_c(src_ref_, stride, dst_ref_, stride, width_, + intermediate_height, filter_params, subpel_, x_step_q4, + &conv_params_); + + conv_horiz_(src_, stride, dst_, stride, width_, intermediate_height, + filter_params, subpel_, x_step_q4, &conv_params_); + + DiffFilterBuffer(); +} + +void AV1ConvolveOptimzTest::RunVertFilterBitExactCheck() { + PrepFilterBuffer(); + + InterpFilterParams filter_params = av1_get_interp_filter_params(filter_); + + av1_convolve_vert_c(src_ref_, stride, dst_ref_, stride, width_, height_, + filter_params, subpel_, x_step_q4, &conv_params_); + + conv_vert_(src_, stride, dst_, stride, width_, height_, filter_params, + subpel_, x_step_q4, &conv_params_); + + DiffFilterBuffer(); +} + +TEST_P(AV1ConvolveOptimzTest, HorizBitExactCheck) { + RunHorizFilterBitExactCheck(); +} +TEST_P(AV1ConvolveOptimzTest, VerticalBitExactCheck) { + RunVertFilterBitExactCheck(); +} + +using std::tr1::make_tuple; + +#if (HAVE_SSSE3 || HAVE_SSE4_1) && CONFIG_DUAL_FILTER +const BlockDimension kBlockDim[] = { + make_tuple(2, 2), make_tuple(2, 4), make_tuple(4, 4), + make_tuple(4, 8), make_tuple(8, 4), make_tuple(8, 8), + make_tuple(8, 16), make_tuple(16, 8), make_tuple(16, 16), + make_tuple(16, 32), make_tuple(32, 16), make_tuple(32, 32), + make_tuple(32, 64), make_tuple(64, 32), make_tuple(64, 64), + make_tuple(64, 128), make_tuple(128, 64), make_tuple(128, 128), +}; + +// 10/12-tap filters +const InterpFilter kFilter[] = { FILTER_REGULAR_UV, BILINEAR, MULTITAP_SHARP }; + +const int kSubpelQ4[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; + +const int kAvg[] = { 0, 1 }; +#endif + +#if HAVE_SSSE3 && CONFIG_DUAL_FILTER +INSTANTIATE_TEST_CASE_P( + SSSE3, AV1ConvolveOptimzTest, + ::testing::Combine(::testing::Values(av1_lowbd_convolve_init_ssse3), + ::testing::Values(av1_convolve_horiz_ssse3), + ::testing::Values(av1_convolve_vert_ssse3), + ::testing::ValuesIn(kBlockDim), + ::testing::ValuesIn(kFilter), + ::testing::ValuesIn(kSubpelQ4), + ::testing::ValuesIn(kAvg))); +#endif // HAVE_SSSE3 && CONFIG_DUAL_FILTER + +#if CONFIG_HIGHBITDEPTH +typedef ::testing::TestWithParam TestWithHbdConvParams; +class AV1HbdConvolveOptimzTest : public TestWithHbdConvParams { + public: + virtual ~AV1HbdConvolveOptimzTest() {} + virtual void SetUp() { + ConvInit conv_init = GET_PARAM(0); + conv_init(); + conv_horiz_ = GET_PARAM(1); + conv_vert_ = GET_PARAM(2); + BlockDimension block = GET_PARAM(3); + width_ = std::tr1::get<0>(block); + height_ = std::tr1::get<1>(block); + filter_ = GET_PARAM(4); + subpel_ = GET_PARAM(5); + avg_ = GET_PARAM(6); + bit_depth_ = GET_PARAM(7); + + alloc_ = new uint16_t[maxBlockSize * 4]; + src_ = alloc_ + (vertiOffset * maxWidth); + src_ += horizOffset; + src_ref_ = src_ + maxBlockSize; + + dst_ = alloc_ + 2 * maxBlockSize; + dst_ref_ = alloc_ + 3 * maxBlockSize; + } + + virtual void TearDown() { + delete[] alloc_; + libaom_test::ClearSystemState(); + } + + protected: + void RunHorizFilterBitExactCheck(); + void RunVertFilterBitExactCheck(); + + private: + void PrepFilterBuffer(); + void DiffFilterBuffer(); + hbd_conv_filter_t conv_horiz_; + hbd_conv_filter_t conv_vert_; + uint16_t *alloc_; + uint16_t *src_; + uint16_t *dst_; + uint16_t *src_ref_; + uint16_t *dst_ref_; + int width_; + int height_; + InterpFilter filter_; + int subpel_; + int avg_; + int bit_depth_; +}; + +void AV1HbdConvolveOptimzTest::PrepFilterBuffer() { + int r, c; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + + memset(alloc_, 0, 4 * maxBlockSize * sizeof(alloc_[0])); + + uint16_t *src_ptr = src_; + uint16_t *dst_ptr = dst_; + uint16_t *dst_ref_ptr = dst_ref_; + uint16_t hbd_mask = (1 << bit_depth_) - 1; + + for (r = 0; r < height_; ++r) { + for (c = 0; c < width_; ++c) { + src_ptr[c] = rnd.Rand16() & hbd_mask; + dst_ptr[c] = rnd.Rand16() & hbd_mask; + dst_ref_ptr[c] = dst_ptr[c]; + } + src_ptr += stride; + dst_ptr += stride; + dst_ref_ptr += stride; + } +} + +void AV1HbdConvolveOptimzTest::DiffFilterBuffer() { + int r, c; + const uint16_t *dst_ptr = dst_; + const uint16_t *dst_ref_ptr = dst_ref_; + for (r = 0; r < height_; ++r) { + for (c = 0; c < width_; ++c) { + EXPECT_EQ((uint16_t)dst_ref_ptr[c], (uint16_t)dst_ptr[c]) + << "Error at row: " << r << " col: " << c << " " + << "w = " << width_ << " " + << "h = " << height_ << " " + << "filter group index = " << filter_ << " " + << "filter index = " << subpel_ << " " + << "bit depth = " << bit_depth_; + } + dst_ptr += stride; + dst_ref_ptr += stride; + } +} + +void AV1HbdConvolveOptimzTest::RunHorizFilterBitExactCheck() { + PrepFilterBuffer(); + + InterpFilterParams filter_params = av1_get_interp_filter_params(filter_); + + av1_highbd_convolve_horiz_c(src_, stride, dst_ref_, stride, width_, height_, + filter_params, subpel_, x_step_q4, avg_, + bit_depth_); + + conv_horiz_(src_, stride, dst_, stride, width_, height_, filter_params, + subpel_, x_step_q4, avg_, bit_depth_); + + DiffFilterBuffer(); + + // Note: + // Here we need calculate a height which is different from the specified one + // and test again. + int intermediate_height = + (((height_ - 1) * 16 + subpel_) >> SUBPEL_BITS) + filter_params.taps; + PrepFilterBuffer(); + + av1_highbd_convolve_horiz_c(src_, stride, dst_ref_, stride, width_, + intermediate_height, filter_params, subpel_, + x_step_q4, avg_, bit_depth_); + + conv_horiz_(src_, stride, dst_, stride, width_, intermediate_height, + filter_params, subpel_, x_step_q4, avg_, bit_depth_); + + DiffFilterBuffer(); +} + +void AV1HbdConvolveOptimzTest::RunVertFilterBitExactCheck() { + PrepFilterBuffer(); + + InterpFilterParams filter_params = av1_get_interp_filter_params(filter_); + + av1_highbd_convolve_vert_c(src_, stride, dst_ref_, stride, width_, height_, + filter_params, subpel_, x_step_q4, avg_, + bit_depth_); + + conv_vert_(src_, stride, dst_, stride, width_, height_, filter_params, + subpel_, x_step_q4, avg_, bit_depth_); + + DiffFilterBuffer(); +} + +TEST_P(AV1HbdConvolveOptimzTest, HorizBitExactCheck) { + RunHorizFilterBitExactCheck(); +} +TEST_P(AV1HbdConvolveOptimzTest, VertBitExactCheck) { + RunVertFilterBitExactCheck(); +} + +#if HAVE_SSE4_1 && CONFIG_DUAL_FILTER + +const int kBitdepth[] = { 10, 12 }; + +INSTANTIATE_TEST_CASE_P( + SSE4_1, AV1HbdConvolveOptimzTest, + ::testing::Combine(::testing::Values(av1_highbd_convolve_init_sse4_1), + ::testing::Values(av1_highbd_convolve_horiz_sse4_1), + ::testing::Values(av1_highbd_convolve_vert_sse4_1), + ::testing::ValuesIn(kBlockDim), + ::testing::ValuesIn(kFilter), + ::testing::ValuesIn(kSubpelQ4), + ::testing::ValuesIn(kAvg), + ::testing::ValuesIn(kBitdepth))); +#endif // HAVE_SSE4_1 && CONFIG_DUAL_FILTER +#endif // CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/av1_convolve_test.cc b/third_party/aom/test/av1_convolve_test.cc new file mode 100644 index 0000000000..02ac8e7bb9 --- /dev/null +++ b/third_party/aom/test/av1_convolve_test.cc @@ -0,0 +1,522 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" +#include "av1/common/filter.h" +#include "av1/common/convolve.h" +#include "test/acm_random.h" +#include "test/util.h" + +using libaom_test::ACMRandom; + +namespace { +using std::tr1::tuple; +static void filter_block1d_horiz_c(const uint8_t *src_ptr, int src_stride, + const int16_t *filter, int tap, + uint8_t *dst_ptr, int dst_stride, int w, + int h) { + src_ptr -= tap / 2 - 1; + for (int r = 0; r < h; ++r) { + for (int c = 0; c < w; ++c) { + int sum = 0; + for (int i = 0; i < tap; ++i) { + sum += src_ptr[c + i] * filter[i]; + } + dst_ptr[c] = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); + } + src_ptr += src_stride; + dst_ptr += dst_stride; + } +} + +static void filter_block1d_vert_c(const uint8_t *src_ptr, int src_stride, + const int16_t *filter, int tap, + uint8_t *dst_ptr, int dst_stride, int w, + int h) { + src_ptr -= (tap / 2 - 1) * src_stride; + for (int r = 0; r < h; ++r) { + for (int c = 0; c < w; ++c) { + int sum = 0; + for (int i = 0; i < tap; ++i) { + sum += src_ptr[c + i * src_stride] * filter[i]; + } + dst_ptr[c] = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); + } + src_ptr += src_stride; + dst_ptr += dst_stride; + } +} + +static int match(const uint8_t *out, int out_stride, const uint8_t *ref_out, + int ref_out_stride, int w, int h) { + for (int r = 0; r < h; ++r) { + for (int c = 0; c < w; ++c) { + if (out[r * out_stride + c] != ref_out[r * ref_out_stride + c]) return 0; + } + } + return 1; +} + +typedef void (*ConvolveFunc)(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, int w, int h, + const InterpFilterParams filter_params, + const int subpel_q4, int step_q4, + ConvolveParams *conv_params); + +struct ConvolveFunctions { + ConvolveFunctions(ConvolveFunc hf, ConvolveFunc vf) : hf_(hf), vf_(vf) {} + ConvolveFunc hf_; + ConvolveFunc vf_; +}; + +typedef tuple + ConvolveParam; + +class Av1ConvolveTest : public ::testing::TestWithParam { + public: + virtual void SetUp() { + rnd_(ACMRandom::DeterministicSeed()); + cfs_ = GET_PARAM(0); + interp_filter_ls_[0] = GET_PARAM(2); + interp_filter_ls_[2] = interp_filter_ls_[0]; + interp_filter_ls_[1] = GET_PARAM(1); + interp_filter_ls_[3] = interp_filter_ls_[1]; + } + virtual void TearDown() { + while (buf_ls_.size() > 0) { + uint8_t *buf = buf_ls_.back(); + aom_free(buf); + buf_ls_.pop_back(); + } + } + virtual uint8_t *add_input(int w, int h, int *stride) { + uint8_t *buf = + reinterpret_cast(aom_memalign(kDataAlignment, kBufferSize)); + buf_ls_.push_back(buf); + *stride = w + MAX_FILTER_TAP - 1; + int offset = MAX_FILTER_TAP / 2 - 1; + for (int r = 0; r < h + MAX_FILTER_TAP - 1; ++r) { + for (int c = 0; c < w + MAX_FILTER_TAP - 1; ++c) { + buf[r * (*stride) + c] = rnd_.Rand8(); + } + } + return buf + offset * (*stride) + offset; + } + virtual uint8_t *add_output(int w, int /*h*/, int *stride) { + uint8_t *buf = + reinterpret_cast(aom_memalign(kDataAlignment, kBufferSize)); + buf_ls_.push_back(buf); + *stride = w; + return buf; + } + virtual void random_init_buf(uint8_t *buf, int w, int h, int stride) { + for (int r = 0; r < h; ++r) { + for (int c = 0; c < w; ++c) { + buf[r * stride + c] = rnd_.Rand8(); + } + } + } + + protected: + static const int kDataAlignment = 16; + static const int kOuterBlockSize = MAX_SB_SIZE + MAX_FILTER_TAP - 1; + static const int kBufferSize = kOuterBlockSize * kOuterBlockSize; + std::vector buf_ls_; + InterpFilter interp_filter_ls_[4]; + ConvolveFunctions *cfs_; + ACMRandom rnd_; +}; + +int bsize_ls[] = { 1, 2, 4, 8, 16, 32, 64, 3, 7, 15, 31, 63 }; +int bsize_num = sizeof(bsize_ls) / sizeof(bsize_ls[0]); + +TEST_P(Av1ConvolveTest, av1_convolve_vert) { + const int y_step_q4 = 16; + ConvolveParams conv_params = get_conv_params(0, 0); + + int in_stride, out_stride, ref_out_stride, avg_out_stride, ref_avg_out_stride; + uint8_t *in = add_input(MAX_SB_SIZE, MAX_SB_SIZE, &in_stride); + uint8_t *out = add_output(MAX_SB_SIZE, MAX_SB_SIZE, &out_stride); + uint8_t *ref_out = add_output(MAX_SB_SIZE, MAX_SB_SIZE, &ref_out_stride); + uint8_t *avg_out = add_output(MAX_SB_SIZE, MAX_SB_SIZE, &avg_out_stride); + uint8_t *ref_avg_out = + add_output(MAX_SB_SIZE, MAX_SB_SIZE, &ref_avg_out_stride); + for (int hb_idx = 0; hb_idx < bsize_num; ++hb_idx) { + for (int vb_idx = 0; vb_idx < bsize_num; ++vb_idx) { + int w = bsize_ls[hb_idx]; + int h = bsize_ls[vb_idx]; + for (int subpel_y_q4 = 0; subpel_y_q4 < SUBPEL_SHIFTS; ++subpel_y_q4) { + InterpFilter filter_y = interp_filter_ls_[0]; + InterpFilterParams param_vert = av1_get_interp_filter_params(filter_y); + const int16_t *filter_vert = + av1_get_interp_filter_subpel_kernel(param_vert, subpel_y_q4); + + filter_block1d_vert_c(in, in_stride, filter_vert, param_vert.taps, + ref_out, ref_out_stride, w, h); + + conv_params.ref = 0; + cfs_->vf_(in, in_stride, out, out_stride, w, h, param_vert, subpel_y_q4, + y_step_q4, &conv_params); + EXPECT_EQ(match(out, out_stride, ref_out, ref_out_stride, w, h), 1) + << " hb_idx " << hb_idx << " vb_idx " << vb_idx << " filter_y " + << filter_y << " subpel_y_q4 " << subpel_y_q4; + + random_init_buf(avg_out, w, h, avg_out_stride); + for (int r = 0; r < h; ++r) { + for (int c = 0; c < w; ++c) { + ref_avg_out[r * ref_avg_out_stride + c] = ROUND_POWER_OF_TWO( + avg_out[r * avg_out_stride + c] + out[r * out_stride + c], 1); + } + } + conv_params.ref = 1; + cfs_->vf_(in, in_stride, avg_out, avg_out_stride, w, h, param_vert, + subpel_y_q4, y_step_q4, &conv_params); + EXPECT_EQ(match(avg_out, avg_out_stride, ref_avg_out, + ref_avg_out_stride, w, h), + 1) + << " hb_idx " << hb_idx << " vb_idx " << vb_idx << " filter_y " + << filter_y << " subpel_y_q4 " << subpel_y_q4; + } + } + } +}; + +TEST_P(Av1ConvolveTest, av1_convolve_horiz) { + const int x_step_q4 = 16; + ConvolveParams conv_params = get_conv_params(0, 0); + + int in_stride, out_stride, ref_out_stride, avg_out_stride, ref_avg_out_stride; + uint8_t *in = add_input(MAX_SB_SIZE, MAX_SB_SIZE, &in_stride); + uint8_t *out = add_output(MAX_SB_SIZE, MAX_SB_SIZE, &out_stride); + uint8_t *ref_out = add_output(MAX_SB_SIZE, MAX_SB_SIZE, &ref_out_stride); + uint8_t *avg_out = add_output(MAX_SB_SIZE, MAX_SB_SIZE, &avg_out_stride); + uint8_t *ref_avg_out = + add_output(MAX_SB_SIZE, MAX_SB_SIZE, &ref_avg_out_stride); + for (int hb_idx = 0; hb_idx < bsize_num; ++hb_idx) { + for (int vb_idx = 0; vb_idx < bsize_num; ++vb_idx) { + int w = bsize_ls[hb_idx]; + int h = bsize_ls[vb_idx]; + for (int subpel_x_q4 = 0; subpel_x_q4 < SUBPEL_SHIFTS; ++subpel_x_q4) { + InterpFilter filter_x = interp_filter_ls_[1]; + InterpFilterParams param_horiz = av1_get_interp_filter_params(filter_x); + const int16_t *filter_horiz = + av1_get_interp_filter_subpel_kernel(param_horiz, subpel_x_q4); + + filter_block1d_horiz_c(in, in_stride, filter_horiz, param_horiz.taps, + ref_out, ref_out_stride, w, h); + + conv_params.ref = 0; + cfs_->hf_(in, in_stride, out, out_stride, w, h, param_horiz, + subpel_x_q4, x_step_q4, &conv_params); + EXPECT_EQ(match(out, out_stride, ref_out, ref_out_stride, w, h), 1) + << " hb_idx " << hb_idx << " vb_idx " << vb_idx << " filter_x " + << filter_x << " subpel_x_q4 " << subpel_x_q4; + + random_init_buf(avg_out, w, h, avg_out_stride); + for (int r = 0; r < h; ++r) { + for (int c = 0; c < w; ++c) { + ref_avg_out[r * ref_avg_out_stride + c] = ROUND_POWER_OF_TWO( + avg_out[r * avg_out_stride + c] + out[r * out_stride + c], 1); + } + } + conv_params.ref = 1; + cfs_->hf_(in, in_stride, avg_out, avg_out_stride, w, h, param_horiz, + subpel_x_q4, x_step_q4, &conv_params); + EXPECT_EQ(match(avg_out, avg_out_stride, ref_avg_out, + ref_avg_out_stride, w, h), + 1) + << "hb_idx " << hb_idx << "vb_idx" << vb_idx << " filter_x " + << filter_x << "subpel_x_q4 " << subpel_x_q4; + } + } + } +}; + +ConvolveFunctions convolve_functions_c(av1_convolve_horiz_c, + av1_convolve_vert_c); + +InterpFilter filter_ls[] = { EIGHTTAP_REGULAR, EIGHTTAP_SMOOTH, + MULTITAP_SHARP }; + +INSTANTIATE_TEST_CASE_P( + C, Av1ConvolveTest, + ::testing::Combine(::testing::Values(&convolve_functions_c), + ::testing::ValuesIn(filter_ls), + ::testing::ValuesIn(filter_ls))); + +#if CONFIG_HIGHBITDEPTH +TEST(AV1ConvolveTest, av1_highbd_convolve) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); +#if CONFIG_DUAL_FILTER + InterpFilter interp_filter[4] = { EIGHTTAP_REGULAR, EIGHTTAP_REGULAR, + EIGHTTAP_REGULAR, EIGHTTAP_REGULAR }; + InterpFilterParams filter_params = + av1_get_interp_filter_params(interp_filter[0]); +#else + InterpFilter interp_filter = EIGHTTAP_REGULAR; + InterpFilterParams filter_params = + av1_get_interp_filter_params(interp_filter); +#endif + int filter_size = filter_params.taps; + int filter_center = filter_size / 2 - 1; + uint16_t src[12 * 12]; + int src_stride = filter_size; + uint16_t dst[1] = { 0 }; + int dst_stride = 1; + int x_step_q4 = 16; + int y_step_q4 = 16; + int avg = 0; + int bd = 10; + int w = 1; + int h = 1; + + int subpel_x_q4; + int subpel_y_q4; + + for (int i = 0; i < filter_size * filter_size; i++) { + src[i] = rnd.Rand16() % (1 << bd); + } + + for (subpel_x_q4 = 0; subpel_x_q4 < SUBPEL_SHIFTS; subpel_x_q4++) { + for (subpel_y_q4 = 0; subpel_y_q4 < SUBPEL_SHIFTS; subpel_y_q4++) { + av1_highbd_convolve( + CONVERT_TO_BYTEPTR(src + src_stride * filter_center + filter_center), + src_stride, CONVERT_TO_BYTEPTR(dst), dst_stride, w, h, interp_filter, + subpel_x_q4, x_step_q4, subpel_y_q4, y_step_q4, avg, bd); + + const int16_t *x_filter = + av1_get_interp_filter_subpel_kernel(filter_params, subpel_x_q4); + const int16_t *y_filter = + av1_get_interp_filter_subpel_kernel(filter_params, subpel_y_q4); + + int temp[12]; + int dst_ref = 0; + for (int r = 0; r < filter_size; r++) { + temp[r] = 0; + for (int c = 0; c < filter_size; c++) { + temp[r] += x_filter[c] * src[r * filter_size + c]; + } + temp[r] = + clip_pixel_highbd(ROUND_POWER_OF_TWO(temp[r], FILTER_BITS), bd); + dst_ref += temp[r] * y_filter[r]; + } + dst_ref = clip_pixel_highbd(ROUND_POWER_OF_TWO(dst_ref, FILTER_BITS), bd); + EXPECT_EQ(dst[0], dst_ref); + } + } +} + +TEST(AV1ConvolveTest, av1_highbd_convolve_avg) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); +#if CONFIG_DUAL_FILTER + InterpFilter interp_filter[4] = { EIGHTTAP_REGULAR, EIGHTTAP_REGULAR, + EIGHTTAP_REGULAR, EIGHTTAP_REGULAR }; + InterpFilterParams filter_params = + av1_get_interp_filter_params(interp_filter[0]); +#else + InterpFilter interp_filter = EIGHTTAP_REGULAR; + InterpFilterParams filter_params = + av1_get_interp_filter_params(interp_filter); +#endif + int filter_size = filter_params.taps; + int filter_center = filter_size / 2 - 1; + uint16_t src0[12 * 12]; + uint16_t src1[12 * 12]; + int src_stride = filter_size; + uint16_t dst0[1] = { 0 }; + uint16_t dst1[1] = { 0 }; + uint16_t dst[1] = { 0 }; + int dst_stride = 1; + int x_step_q4 = 16; + int y_step_q4 = 16; + int avg = 0; + int bd = 10; + + int w = 1; + int h = 1; + + int subpel_x_q4; + int subpel_y_q4; + + for (int i = 0; i < filter_size * filter_size; i++) { + src0[i] = rnd.Rand16() % (1 << bd); + src1[i] = rnd.Rand16() % (1 << bd); + } + + for (subpel_x_q4 = 0; subpel_x_q4 < SUBPEL_SHIFTS; subpel_x_q4++) { + for (subpel_y_q4 = 0; subpel_y_q4 < SUBPEL_SHIFTS; subpel_y_q4++) { + int offset = filter_size * filter_center + filter_center; + + avg = 0; + av1_highbd_convolve(CONVERT_TO_BYTEPTR(src0 + offset), src_stride, + CONVERT_TO_BYTEPTR(dst0), dst_stride, w, h, + interp_filter, subpel_x_q4, x_step_q4, subpel_y_q4, + y_step_q4, avg, bd); + avg = 0; + av1_highbd_convolve(CONVERT_TO_BYTEPTR(src1 + offset), src_stride, + CONVERT_TO_BYTEPTR(dst1), dst_stride, w, h, + interp_filter, subpel_x_q4, x_step_q4, subpel_y_q4, + y_step_q4, avg, bd); + + avg = 0; + av1_highbd_convolve(CONVERT_TO_BYTEPTR(src0 + offset), src_stride, + CONVERT_TO_BYTEPTR(dst), dst_stride, w, h, + interp_filter, subpel_x_q4, x_step_q4, subpel_y_q4, + y_step_q4, avg, bd); + avg = 1; + av1_highbd_convolve(CONVERT_TO_BYTEPTR(src1 + offset), src_stride, + CONVERT_TO_BYTEPTR(dst), dst_stride, w, h, + interp_filter, subpel_x_q4, x_step_q4, subpel_y_q4, + y_step_q4, avg, bd); + + EXPECT_EQ(dst[0], ROUND_POWER_OF_TWO(dst0[0] + dst1[0], 1)); + } + } +} +#endif // CONFIG_HIGHBITDEPTH + +#define CONVOLVE_SPEED_TEST 0 +#if CONVOLVE_SPEED_TEST +#define highbd_convolve_speed(func, block_size, frame_size) \ + TEST(AV1ConvolveTest, func##_speed_##block_size##_##frame_size) { \ + ACMRandom rnd(ACMRandom::DeterministicSeed()); \ + InterpFilter interp_filter = EIGHTTAP; \ + InterpFilterParams filter_params = \ + av1_get_interp_filter_params(interp_filter); \ + int filter_size = filter_params.tap; \ + int filter_center = filter_size / 2 - 1; \ + DECLARE_ALIGNED(16, uint16_t, \ + src[(frame_size + 7) * (frame_size + 7)]) = { 0 }; \ + int src_stride = frame_size + 7; \ + DECLARE_ALIGNED(16, uint16_t, dst[frame_size * frame_size]) = { 0 }; \ + int dst_stride = frame_size; \ + int x_step_q4 = 16; \ + int y_step_q4 = 16; \ + int subpel_x_q4 = 8; \ + int subpel_y_q4 = 6; \ + int bd = 10; \ + \ + int w = block_size; \ + int h = block_size; \ + \ + const int16_t *filter_x = \ + av1_get_interp_filter_kernel(filter_params, subpel_x_q4); \ + const int16_t *filter_y = \ + av1_get_interp_filter_kernel(filter_params, subpel_y_q4); \ + \ + for (int i = 0; i < src_stride * src_stride; i++) { \ + src[i] = rnd.Rand16() % (1 << bd); \ + } \ + \ + int offset = filter_center * src_stride + filter_center; \ + int row_offset = 0; \ + int col_offset = 0; \ + for (int i = 0; i < 100000; i++) { \ + int src_total_offset = offset + col_offset * src_stride + row_offset; \ + int dst_total_offset = col_offset * dst_stride + row_offset; \ + func(CONVERT_TO_BYTEPTR(src + src_total_offset), src_stride, \ + CONVERT_TO_BYTEPTR(dst + dst_total_offset), dst_stride, filter_x, \ + x_step_q4, filter_y, y_step_q4, w, h, bd); \ + if (offset + w + w < frame_size) { \ + row_offset += w; \ + } else { \ + row_offset = 0; \ + col_offset += h; \ + } \ + if (col_offset + h >= frame_size) { \ + col_offset = 0; \ + } \ + } \ + } + +#define lowbd_convolve_speed(func, block_size, frame_size) \ + TEST(AV1ConvolveTest, func##_speed_l_##block_size##_##frame_size) { \ + ACMRandom rnd(ACMRandom::DeterministicSeed()); \ + InterpFilter interp_filter = EIGHTTAP; \ + InterpFilterParams filter_params = \ + av1_get_interp_filter_params(interp_filter); \ + int filter_size = filter_params.tap; \ + int filter_center = filter_size / 2 - 1; \ + DECLARE_ALIGNED(16, uint8_t, src[(frame_size + 7) * (frame_size + 7)]); \ + int src_stride = frame_size + 7; \ + DECLARE_ALIGNED(16, uint8_t, dst[frame_size * frame_size]); \ + int dst_stride = frame_size; \ + int x_step_q4 = 16; \ + int y_step_q4 = 16; \ + int subpel_x_q4 = 8; \ + int subpel_y_q4 = 6; \ + int bd = 8; \ + \ + int w = block_size; \ + int h = block_size; \ + \ + const int16_t *filter_x = \ + av1_get_interp_filter_kernel(filter_params, subpel_x_q4); \ + const int16_t *filter_y = \ + av1_get_interp_filter_kernel(filter_params, subpel_y_q4); \ + \ + for (int i = 0; i < src_stride * src_stride; i++) { \ + src[i] = rnd.Rand16() % (1 << bd); \ + } \ + \ + int offset = filter_center * src_stride + filter_center; \ + int row_offset = 0; \ + int col_offset = 0; \ + for (int i = 0; i < 100000; i++) { \ + func(src + offset, src_stride, dst, dst_stride, filter_x, x_step_q4, \ + filter_y, y_step_q4, w, h); \ + if (offset + w + w < frame_size) { \ + row_offset += w; \ + } else { \ + row_offset = 0; \ + col_offset += h; \ + } \ + if (col_offset + h >= frame_size) { \ + col_offset = 0; \ + } \ + } \ + } + +// This experiment shows that when frame size is 64x64 +// aom_highbd_convolve8_sse2 and aom_convolve8_sse2's speed are similar. +// However when frame size becomes 1024x1024 +// aom_highbd_convolve8_sse2 is around 50% slower than aom_convolve8_sse2 +// we think the bottleneck is from memory IO +highbd_convolve_speed(aom_highbd_convolve8_sse2, 8, 64); +highbd_convolve_speed(aom_highbd_convolve8_sse2, 16, 64); +highbd_convolve_speed(aom_highbd_convolve8_sse2, 32, 64); +highbd_convolve_speed(aom_highbd_convolve8_sse2, 64, 64); + +lowbd_convolve_speed(aom_convolve8_sse2, 8, 64); +lowbd_convolve_speed(aom_convolve8_sse2, 16, 64); +lowbd_convolve_speed(aom_convolve8_sse2, 32, 64); +lowbd_convolve_speed(aom_convolve8_sse2, 64, 64); + +highbd_convolve_speed(aom_highbd_convolve8_sse2, 8, 1024); +highbd_convolve_speed(aom_highbd_convolve8_sse2, 16, 1024); +highbd_convolve_speed(aom_highbd_convolve8_sse2, 32, 1024); +highbd_convolve_speed(aom_highbd_convolve8_sse2, 64, 1024); + +lowbd_convolve_speed(aom_convolve8_sse2, 8, 1024); +lowbd_convolve_speed(aom_convolve8_sse2, 16, 1024); +lowbd_convolve_speed(aom_convolve8_sse2, 32, 1024); +lowbd_convolve_speed(aom_convolve8_sse2, 64, 1024); +#endif // CONVOLVE_SPEED_TEST +} // namespace diff --git a/third_party/aom/test/av1_dct_test.cc b/third_party/aom/test/av1_dct_test.cc new file mode 100644 index 0000000000..691cc8b794 --- /dev/null +++ b/third_party/aom/test/av1_dct_test.cc @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/acm_random.h" +#include "test/util.h" +#include "./aom_config.h" +#include "aom_ports/msvc.h" + +#undef CONFIG_COEFFICIENT_RANGE_CHECKING +#define CONFIG_COEFFICIENT_RANGE_CHECKING 1 +#define AV1_DCT_GTEST +#include "av1/encoder/dct.c" + +using libaom_test::ACMRandom; + +namespace { +void reference_dct_1d(const double *in, double *out, int size) { + const double kInvSqrt2 = 0.707106781186547524400844362104; + for (int k = 0; k < size; ++k) { + out[k] = 0; + for (int n = 0; n < size; ++n) { + out[k] += in[n] * cos(PI * (2 * n + 1) * k / (2 * size)); + } + if (k == 0) out[k] = out[k] * kInvSqrt2; + } +} + +typedef void (*FdctFuncRef)(const double *in, double *out, int size); +typedef void (*IdctFuncRef)(const double *in, double *out, int size); +typedef void (*FdctFunc)(const tran_low_t *in, tran_low_t *out); +typedef void (*IdctFunc)(const tran_low_t *in, tran_low_t *out); + +class TransTestBase { + public: + virtual ~TransTestBase() {} + + protected: + void RunFwdAccuracyCheck() { + tran_low_t *input = new tran_low_t[txfm_size_]; + tran_low_t *output = new tran_low_t[txfm_size_]; + double *ref_input = new double[txfm_size_]; + double *ref_output = new double[txfm_size_]; + + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 5000; + for (int ti = 0; ti < count_test_block; ++ti) { + for (int ni = 0; ni < txfm_size_; ++ni) { + input[ni] = rnd.Rand8() - rnd.Rand8(); + ref_input[ni] = static_cast(input[ni]); + } + + fwd_txfm_(input, output); + fwd_txfm_ref_(ref_input, ref_output, txfm_size_); + + for (int ni = 0; ni < txfm_size_; ++ni) { + EXPECT_LE( + abs(output[ni] - static_cast(round(ref_output[ni]))), + max_error_); + } + } + + delete[] input; + delete[] output; + delete[] ref_input; + delete[] ref_output; + } + + double max_error_; + int txfm_size_; + FdctFunc fwd_txfm_; + FdctFuncRef fwd_txfm_ref_; +}; + +typedef std::tr1::tuple FdctParam; +class AV1FwdTxfm : public TransTestBase, + public ::testing::TestWithParam { + public: + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + fwd_txfm_ref_ = GET_PARAM(1); + txfm_size_ = GET_PARAM(2); + max_error_ = GET_PARAM(3); + } + virtual void TearDown() {} +}; + +TEST_P(AV1FwdTxfm, RunFwdAccuracyCheck) { RunFwdAccuracyCheck(); } + +INSTANTIATE_TEST_CASE_P( + C, AV1FwdTxfm, + ::testing::Values(FdctParam(&fdct4, &reference_dct_1d, 4, 1), + FdctParam(&fdct8, &reference_dct_1d, 8, 1), + FdctParam(&fdct16, &reference_dct_1d, 16, 2), + FdctParam(&fdct32, &reference_dct_1d, 32, 3))); +} // namespace diff --git a/third_party/aom/test/av1_ext_tile_test.cc b/third_party/aom/test/av1_ext_tile_test.cc new file mode 100644 index 0000000000..f96447965d --- /dev/null +++ b/third_party/aom/test/av1_ext_tile_test.cc @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/md5_helper.h" +#include "test/util.h" + +namespace { +// The number of frames to be encoded/decoded +const int kLimit = 8; +// Skip 1 frame to check the frame decoding independency. +const int kSkip = 5; +const int kTileSize = 1; +const int kTIleSizeInPixels = (kTileSize << 6); +// Fake width and height so that they can be multiples of the tile size. +const int kImgWidth = 704; +const int kImgHeight = 576; + +// This test tests "tile_encoding_mode = TILE_VR" case. The TILE_NORMAL case is +// tested by the tile_independence test. +class AV1ExtTileTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params { + protected: + AV1ExtTileTest() + : EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)), + set_cpu_used_(GET_PARAM(2)) { + init_flags_ = AOM_CODEC_USE_PSNR; + aom_codec_dec_cfg_t cfg = aom_codec_dec_cfg_t(); + cfg.w = kImgWidth; + cfg.h = kImgHeight; + + decoder_ = codec_->CreateDecoder(cfg, 0); + decoder_->Control(AV1_SET_DECODE_TILE_ROW, -1); + decoder_->Control(AV1_SET_DECODE_TILE_COL, -1); + + // Allocate buffer to store tile image. + aom_img_alloc(&tile_img_, AOM_IMG_FMT_I420, kImgWidth, kImgHeight, 32); + + md5_.clear(); + tile_md5_.clear(); + } + + virtual ~AV1ExtTileTest() { + aom_img_free(&tile_img_); + delete decoder_; + } + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + + cfg_.g_lag_in_frames = 0; + cfg_.rc_end_usage = AOM_VBR; + cfg_.g_error_resilient = 1; + + cfg_.rc_max_quantizer = 56; + cfg_.rc_min_quantizer = 0; + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 0) { + // Encode setting + encoder->Control(AOME_SET_CPUUSED, set_cpu_used_); + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 0); + encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1); + + // The tile size is 64x64. + encoder->Control(AV1E_SET_TILE_COLUMNS, kTileSize); + encoder->Control(AV1E_SET_TILE_ROWS, kTileSize); + encoder->Control(AV1E_SET_TILE_ENCODING_MODE, 1); // TILE_VR +#if CONFIG_EXT_PARTITION + // Always use 64x64 max partition. + encoder->Control(AV1E_SET_SUPERBLOCK_SIZE, AOM_SUPERBLOCK_SIZE_64X64); +#endif + } + + if (video->frame() == 1) { + frame_flags_ = + AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | AOM_EFLAG_NO_UPD_ARF; + } + } + + virtual void DecompressedFrameHook(const aom_image_t &img, + aom_codec_pts_t pts) { + // Skip 1 already decoded frame to be consistent with the decoder in this + // test. + if (pts == (aom_codec_pts_t)kSkip) return; + + // Calculate MD5 as the reference. + ::libaom_test::MD5 md5_res; + md5_res.Add(&img); + md5_.push_back(md5_res.Get()); + } + + virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) { + // Skip decoding 1 frame. + if (pkt->data.frame.pts == (aom_codec_pts_t)kSkip) return; + + bool IsLastFrame = (pkt->data.frame.pts == (aom_codec_pts_t)(kLimit - 1)); + + // Decode the first (kLimit - 1) frames as whole frame, and decode the last + // frame in single tiles. + for (int r = 0; r < kImgHeight / kTIleSizeInPixels; ++r) { + for (int c = 0; c < kImgWidth / kTIleSizeInPixels; ++c) { + if (!IsLastFrame) { + decoder_->Control(AV1_SET_DECODE_TILE_ROW, -1); + decoder_->Control(AV1_SET_DECODE_TILE_COL, -1); + } else { + decoder_->Control(AV1_SET_DECODE_TILE_ROW, r); + decoder_->Control(AV1_SET_DECODE_TILE_COL, c); + } + + const aom_codec_err_t res = decoder_->DecodeFrame( + reinterpret_cast(pkt->data.frame.buf), + pkt->data.frame.sz); + if (res != AOM_CODEC_OK) { + abort_ = true; + ASSERT_EQ(AOM_CODEC_OK, res); + } + const aom_image_t *img = decoder_->GetDxData().Next(); + + if (!IsLastFrame) { + if (img) { + ::libaom_test::MD5 md5_res; + md5_res.Add(img); + tile_md5_.push_back(md5_res.Get()); + } + break; + } + + const int kMaxMBPlane = 3; + for (int plane = 0; plane < kMaxMBPlane; ++plane) { + const int shift = (plane == 0) ? 0 : 1; + int tile_height = kTIleSizeInPixels >> shift; + int tile_width = kTIleSizeInPixels >> shift; + + for (int tr = 0; tr < tile_height; ++tr) { + memcpy(tile_img_.planes[plane] + + tile_img_.stride[plane] * (r * tile_height + tr) + + c * tile_width, + img->planes[plane] + img->stride[plane] * tr, tile_width); + } + } + } + + if (!IsLastFrame) break; + } + + if (IsLastFrame) { + ::libaom_test::MD5 md5_res; + md5_res.Add(&tile_img_); + tile_md5_.push_back(md5_res.Get()); + } + } + + ::libaom_test::TestMode encoding_mode_; + int set_cpu_used_; + ::libaom_test::Decoder *decoder_; + aom_image_t tile_img_; + std::vector md5_; + std::vector tile_md5_; +}; + +TEST_P(AV1ExtTileTest, DecoderResultTest) { + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", kImgWidth, + kImgHeight, 30, 1, 0, kLimit); + cfg_.rc_target_bitrate = 500; + cfg_.g_error_resilient = AOM_ERROR_RESILIENT_DEFAULT; + cfg_.g_lag_in_frames = 0; + cfg_.g_threads = 1; + + // Tile encoding + init_flags_ = AOM_CODEC_USE_PSNR; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + + // Compare to check if two vectors are equal. + ASSERT_EQ(md5_, tile_md5_); +} + +AV1_INSTANTIATE_TEST_CASE( + // Now only test 2-pass mode. + AV1ExtTileTest, ::testing::Values(::libaom_test::kTwoPassGood), + ::testing::Range(0, 4)); +} // namespace diff --git a/third_party/aom/test/av1_fht16x16_test.cc b/third_party/aom/test/av1_fht16x16_test.cc new file mode 100644 index 0000000000..e1032ef241 --- /dev/null +++ b/third_party/aom/test/av1_fht16x16_test.cc @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" + +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/transform_test_base.h" +#include "test/util.h" +#include "aom_ports/mem.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); +using std::tr1::tuple; +using libaom_test::FhtFunc; +typedef tuple Ht16x16Param; + +void fht16x16_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht16x16_c(in, out, stride, tx_type); +} + +void iht16x16_ref(const tran_low_t *in, uint8_t *dest, int stride, + int tx_type) { + av1_iht16x16_256_add_c(in, dest, stride, tx_type); +} + +#if CONFIG_HIGHBITDEPTH +typedef void (*IHbdHtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type, int bd); +typedef void (*HbdHtFunc)(const int16_t *input, int32_t *output, int stride, + int tx_type, int bd); + +// Target optimized function, tx_type, bit depth +typedef tuple HighbdHt16x16Param; + +void highbd_fht16x16_ref(const int16_t *in, int32_t *out, int stride, + int tx_type, int bd) { + av1_fwd_txfm2d_16x16_c(in, out, stride, tx_type, bd); +} +#endif // CONFIG_HIGHBITDEPTH + +class AV1Trans16x16HT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~AV1Trans16x16HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 16; + height_ = 16; + fwd_txfm_ref = fht16x16_ref; + inv_txfm_ref = iht16x16_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(AV1Trans16x16HT, MemCheck) { RunMemCheck(); } +TEST_P(AV1Trans16x16HT, AccuracyCheck) { RunAccuracyCheck(1, 0.001); } +TEST_P(AV1Trans16x16HT, InvAccuracyCheck) { RunInvAccuracyCheck(1); } +TEST_P(AV1Trans16x16HT, CoeffCheck) { RunCoeffCheck(); } +TEST_P(AV1Trans16x16HT, InvCoeffCheck) { RunInvCoeffCheck(); } + +#if CONFIG_HIGHBITDEPTH +class AV1HighbdTrans16x16HT + : public ::testing::TestWithParam { + public: + virtual ~AV1HighbdTrans16x16HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + fwd_txfm_ref_ = highbd_fht16x16_ref; + tx_type_ = GET_PARAM(1); + bit_depth_ = GET_PARAM(2); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = 256; + + input_ = reinterpret_cast( + aom_memalign(32, sizeof(int16_t) * num_coeffs_)); + output_ = reinterpret_cast( + aom_memalign(32, sizeof(int32_t) * num_coeffs_)); + output_ref_ = reinterpret_cast( + aom_memalign(32, sizeof(int32_t) * num_coeffs_)); + } + + virtual void TearDown() { + aom_free(input_); + aom_free(output_); + aom_free(output_ref_); + libaom_test::ClearSystemState(); + } + + protected: + void RunBitexactCheck(); + + private: + HbdHtFunc fwd_txfm_; + HbdHtFunc fwd_txfm_ref_; + int tx_type_; + int bit_depth_; + int mask_; + int num_coeffs_; + int16_t *input_; + int32_t *output_; + int32_t *output_ref_; +}; + +void AV1HighbdTrans16x16HT::RunBitexactCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + int i, j; + const int stride = 16; + const int num_tests = 1000; + + for (i = 0; i < num_tests; ++i) { + for (j = 0; j < num_coeffs_; ++j) { + input_[j] = (rnd.Rand16() & mask_) - (rnd.Rand16() & mask_); + } + + fwd_txfm_ref_(input_, output_ref_, stride, tx_type_, bit_depth_); + ASM_REGISTER_STATE_CHECK( + fwd_txfm_(input_, output_, stride, tx_type_, bit_depth_)); + + for (j = 0; j < num_coeffs_; ++j) { + EXPECT_EQ(output_ref_[j], output_[j]) + << "Not bit-exact result at index: " << j << " at test block: " << i; + } + } +} + +TEST_P(AV1HighbdTrans16x16HT, HighbdCoeffCheck) { RunBitexactCheck(); } +#endif // CONFIG_HIGHBITDEPTH + +using std::tr1::make_tuple; + +#if HAVE_SSE2 +const Ht16x16Param kArrayHt16x16Param_sse2[] = { + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 0, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 1, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 2, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 3, AOM_BITS_8, + 256), +#if CONFIG_EXT_TX + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 4, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 5, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 6, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 7, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 8, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 9, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 10, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 11, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 12, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 13, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 14, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, 15, AOM_BITS_8, + 256) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(SSE2, AV1Trans16x16HT, + ::testing::ValuesIn(kArrayHt16x16Param_sse2)); +#endif // HAVE_SSE2 + +#if HAVE_AVX2 +const Ht16x16Param kArrayHt16x16Param_avx2[] = { + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 0, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 1, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 2, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 3, AOM_BITS_8, + 256), +#if CONFIG_EXT_TX + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 4, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 5, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 6, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 7, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 8, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 9, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 10, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 11, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 12, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 13, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 14, AOM_BITS_8, + 256), + make_tuple(&av1_fht16x16_avx2, &av1_iht16x16_256_add_avx2, 15, AOM_BITS_8, + 256) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(AVX2, AV1Trans16x16HT, + ::testing::ValuesIn(kArrayHt16x16Param_avx2)); +#endif // HAVE_AVX2 + +#if HAVE_SSE4_1 && CONFIG_HIGHBITDEPTH +const HighbdHt16x16Param kArrayHBDHt16x16Param_sse4_1[] = { + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 0, 10), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 0, 12), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 1, 10), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 1, 12), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 2, 10), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 2, 12), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 3, 10), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 3, 12), +#if CONFIG_EXT_TX + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 4, 10), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 4, 12), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 5, 10), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 5, 12), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 6, 10), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 6, 12), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 7, 10), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 7, 12), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 8, 10), + make_tuple(&av1_fwd_txfm2d_16x16_sse4_1, 8, 12), +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(SSE4_1, AV1HighbdTrans16x16HT, + ::testing::ValuesIn(kArrayHBDHt16x16Param_sse4_1)); +#endif // HAVE_SSE4_1 && CONFIG_HIGHBITDEPTH + +} // namespace diff --git a/third_party/aom/test/av1_fht16x32_test.cc b/third_party/aom/test/av1_fht16x32_test.cc new file mode 100644 index 0000000000..43d0253273 --- /dev/null +++ b/third_party/aom/test/av1_fht16x32_test.cc @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" + +#include "aom_ports/mem.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/transform_test_base.h" +#include "test/util.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); +using std::tr1::tuple; +using libaom_test::FhtFunc; +typedef tuple Ht16x32Param; + +void fht16x32_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht16x32_c(in, out, stride, tx_type); +} + +void iht16x32_ref(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_iht16x32_512_add_c(in, out, stride, tx_type); +} + +class AV1Trans16x32HT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~AV1Trans16x32HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 16; + height_ = 32; + fwd_txfm_ref = fht16x32_ref; + inv_txfm_ref = iht16x32_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(AV1Trans16x32HT, AccuracyCheck) { RunAccuracyCheck(4, 0.2); } +TEST_P(AV1Trans16x32HT, CoeffCheck) { RunCoeffCheck(); } +TEST_P(AV1Trans16x32HT, MemCheck) { RunMemCheck(); } +TEST_P(AV1Trans16x32HT, InvCoeffCheck) { RunInvCoeffCheck(); } +TEST_P(AV1Trans16x32HT, InvAccuracyCheck) { RunInvAccuracyCheck(4); } + +using std::tr1::make_tuple; +const Ht16x32Param kArrayHt16x32Param_c[] = { + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 0, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 1, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 2, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 3, AOM_BITS_8, 512), +#if CONFIG_EXT_TX + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 4, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 5, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 6, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 7, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 8, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 9, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 10, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 11, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 12, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 13, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 14, AOM_BITS_8, 512), + make_tuple(&av1_fht16x32_c, &av1_iht16x32_512_add_c, 15, AOM_BITS_8, 512) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(C, AV1Trans16x32HT, + ::testing::ValuesIn(kArrayHt16x32Param_c)); + +#if HAVE_SSE2 +const Ht16x32Param kArrayHt16x32Param_sse2[] = { + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 0, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 1, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 2, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 3, AOM_BITS_8, + 512), +#if CONFIG_EXT_TX + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 4, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 5, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 6, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 7, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 8, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 9, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 10, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 11, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 12, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 13, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 14, AOM_BITS_8, + 512), + make_tuple(&av1_fht16x32_sse2, &av1_iht16x32_512_add_sse2, 15, AOM_BITS_8, + 512) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(SSE2, AV1Trans16x32HT, + ::testing::ValuesIn(kArrayHt16x32Param_sse2)); +#endif // HAVE_SSE2 + +} // namespace diff --git a/third_party/aom/test/av1_fht16x8_test.cc b/third_party/aom/test/av1_fht16x8_test.cc new file mode 100644 index 0000000000..d99bec5eb3 --- /dev/null +++ b/third_party/aom/test/av1_fht16x8_test.cc @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" + +#include "aom_ports/mem.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/transform_test_base.h" +#include "test/util.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); +using std::tr1::tuple; +using libaom_test::FhtFunc; +typedef tuple Ht16x8Param; + +void fht16x8_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht16x8_c(in, out, stride, tx_type); +} + +void iht16x8_ref(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_iht16x8_128_add_c(in, out, stride, tx_type); +} + +class AV1Trans16x8HT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~AV1Trans16x8HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 16; + height_ = 8; + inv_txfm_ref = iht16x8_ref; + fwd_txfm_ref = fht16x8_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(AV1Trans16x8HT, AccuracyCheck) { RunAccuracyCheck(1, 0.001); } +TEST_P(AV1Trans16x8HT, CoeffCheck) { RunCoeffCheck(); } +TEST_P(AV1Trans16x8HT, MemCheck) { RunMemCheck(); } +TEST_P(AV1Trans16x8HT, InvCoeffCheck) { RunInvCoeffCheck(); } +TEST_P(AV1Trans16x8HT, InvAccuracyCheck) { RunInvAccuracyCheck(1); } + +using std::tr1::make_tuple; + +const Ht16x8Param kArrayHt16x8Param_c[] = { + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 0, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 1, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 2, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 3, AOM_BITS_8, 128), +#if CONFIG_EXT_TX + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 4, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 5, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 6, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 7, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 8, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 9, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 10, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 11, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 12, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 13, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 14, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_c, &av1_iht16x8_128_add_c, 15, AOM_BITS_8, 128) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(C, AV1Trans16x8HT, + ::testing::ValuesIn(kArrayHt16x8Param_c)); + +#if HAVE_SSE2 +const Ht16x8Param kArrayHt16x8Param_sse2[] = { + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 0, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 1, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 2, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 3, AOM_BITS_8, 128), +#if CONFIG_EXT_TX + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 4, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 5, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 6, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 7, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 8, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 9, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 10, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 11, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 12, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 13, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 14, AOM_BITS_8, 128), + make_tuple(&av1_fht16x8_sse2, &av1_iht16x8_128_add_sse2, 15, AOM_BITS_8, 128) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(SSE2, AV1Trans16x8HT, + ::testing::ValuesIn(kArrayHt16x8Param_sse2)); +#endif // HAVE_SSE2 + +} // namespace diff --git a/third_party/aom/test/av1_fht32x16_test.cc b/third_party/aom/test/av1_fht32x16_test.cc new file mode 100644 index 0000000000..e38283f865 --- /dev/null +++ b/third_party/aom/test/av1_fht32x16_test.cc @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" + +#include "aom_ports/mem.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/transform_test_base.h" +#include "test/util.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); +using std::tr1::tuple; +using libaom_test::FhtFunc; +typedef tuple Ht32x16Param; + +void fht32x16_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht32x16_c(in, out, stride, tx_type); +} + +void iht32x16_ref(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_iht32x16_512_add_c(in, out, stride, tx_type); +} + +class AV1Trans32x16HT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~AV1Trans32x16HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 32; + height_ = 16; + fwd_txfm_ref = fht32x16_ref; + inv_txfm_ref = iht32x16_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(AV1Trans32x16HT, MemCheck) { RunMemCheck(); } +TEST_P(AV1Trans32x16HT, AccuracyCheck) { RunAccuracyCheck(4, 0.2); } +TEST_P(AV1Trans32x16HT, CoeffCheck) { RunCoeffCheck(); } +TEST_P(AV1Trans32x16HT, InvCoeffCheck) { RunInvCoeffCheck(); } +TEST_P(AV1Trans32x16HT, InvAccuracyCheck) { RunInvAccuracyCheck(4); } + +using std::tr1::make_tuple; +const Ht32x16Param kArrayHt32x16Param_c[] = { + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 0, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 1, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 2, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 3, AOM_BITS_8, 512), +#if CONFIG_EXT_TX + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 4, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 5, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 6, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 7, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 8, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 9, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 10, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 11, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 12, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 13, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 14, AOM_BITS_8, 512), + make_tuple(&av1_fht32x16_c, &av1_iht32x16_512_add_c, 15, AOM_BITS_8, 512) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(C, AV1Trans32x16HT, + ::testing::ValuesIn(kArrayHt32x16Param_c)); + +#if HAVE_SSE2 +const Ht32x16Param kArrayHt32x16Param_sse2[] = { + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 0, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 1, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 2, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 3, AOM_BITS_8, + 512), +#if CONFIG_EXT_TX + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 4, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 5, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 6, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 7, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 8, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 9, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 10, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 11, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 12, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 13, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 14, AOM_BITS_8, + 512), + make_tuple(&av1_fht32x16_sse2, &av1_iht32x16_512_add_sse2, 15, AOM_BITS_8, + 512) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(SSE2, AV1Trans32x16HT, + ::testing::ValuesIn(kArrayHt32x16Param_sse2)); +#endif // HAVE_SSE2 + +} // namespace diff --git a/third_party/aom/test/av1_fht4x4_test.cc b/third_party/aom/test/av1_fht4x4_test.cc new file mode 100644 index 0000000000..42837d3a4c --- /dev/null +++ b/third_party/aom/test/av1_fht4x4_test.cc @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" + +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/transform_test_base.h" +#include "test/util.h" +#include "aom_ports/mem.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); +using std::tr1::tuple; +using libaom_test::FhtFunc; +typedef tuple Ht4x4Param; + +void fht4x4_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht4x4_c(in, out, stride, tx_type); +} + +void iht4x4_ref(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_iht4x4_16_add_c(in, out, stride, tx_type); +} + +#if CONFIG_HIGHBITDEPTH +typedef void (*IhighbdHtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type, int bd); +typedef void (*HBDFhtFunc)(const int16_t *input, int32_t *output, int stride, + int tx_type, int bd); + +// HighbdHt4x4Param argument list: +// +typedef tuple HighbdHt4x4Param; + +void highbe_fht4x4_ref(const int16_t *in, int32_t *out, int stride, int tx_type, + int bd) { + av1_fwd_txfm2d_4x4_c(in, out, stride, tx_type, bd); +} +#endif // CONFIG_HIGHBITDEPTH + +class AV1Trans4x4HT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~AV1Trans4x4HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 4; + height_ = 4; + fwd_txfm_ref = fht4x4_ref; + inv_txfm_ref = iht4x4_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(AV1Trans4x4HT, MemCheck) { RunMemCheck(); } +TEST_P(AV1Trans4x4HT, CoeffCheck) { RunCoeffCheck(); } +// Note: +// TODO(luoyi): Add tx_type, 9-15 for inverse transform. +// Need cleanup since same tests may be done in fdct4x4_test.cc +// TEST_P(AV1Trans4x4HT, AccuracyCheck) { RunAccuracyCheck(0); } +// TEST_P(AV1Trans4x4HT, InvAccuracyCheck) { RunInvAccuracyCheck(0); } +// TEST_P(AV1Trans4x4HT, InvCoeffCheck) { RunInvCoeffCheck(); } + +#if CONFIG_HIGHBITDEPTH +class AV1HighbdTrans4x4HT : public ::testing::TestWithParam { + public: + virtual ~AV1HighbdTrans4x4HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + fwd_txfm_ref_ = highbe_fht4x4_ref; + tx_type_ = GET_PARAM(1); + bit_depth_ = GET_PARAM(2); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = 16; + + input_ = reinterpret_cast( + aom_memalign(16, sizeof(int16_t) * num_coeffs_)); + output_ = reinterpret_cast( + aom_memalign(16, sizeof(int32_t) * num_coeffs_)); + output_ref_ = reinterpret_cast( + aom_memalign(16, sizeof(int32_t) * num_coeffs_)); + } + + virtual void TearDown() { + aom_free(input_); + aom_free(output_); + aom_free(output_ref_); + libaom_test::ClearSystemState(); + } + + protected: + void RunBitexactCheck(); + + private: + HBDFhtFunc fwd_txfm_; + HBDFhtFunc fwd_txfm_ref_; + int tx_type_; + int bit_depth_; + int mask_; + int num_coeffs_; + int16_t *input_; + int32_t *output_; + int32_t *output_ref_; +}; + +void AV1HighbdTrans4x4HT::RunBitexactCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + int i, j; + const int stride = 4; + const int num_tests = 1000; + const int num_coeffs = 16; + + for (i = 0; i < num_tests; ++i) { + for (j = 0; j < num_coeffs; ++j) { + input_[j] = (rnd.Rand16() & mask_) - (rnd.Rand16() & mask_); + } + + fwd_txfm_ref_(input_, output_ref_, stride, tx_type_, bit_depth_); + fwd_txfm_(input_, output_, stride, tx_type_, bit_depth_); + + for (j = 0; j < num_coeffs; ++j) { + EXPECT_EQ(output_[j], output_ref_[j]) + << "Not bit-exact result at index: " << j << " at test block: " << i; + } + } +} + +TEST_P(AV1HighbdTrans4x4HT, HighbdCoeffCheck) { RunBitexactCheck(); } +#endif // CONFIG_HIGHBITDEPTH + +using std::tr1::make_tuple; + +#if HAVE_SSE2 +const Ht4x4Param kArrayHt4x4Param_sse2[] = { + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 0, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 1, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 2, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 3, AOM_BITS_8, 16), +#if CONFIG_EXT_TX + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 4, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 5, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 6, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 7, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 8, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 9, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 10, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 11, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 12, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 13, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 14, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 15, AOM_BITS_8, 16) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(SSE2, AV1Trans4x4HT, + ::testing::ValuesIn(kArrayHt4x4Param_sse2)); +#endif // HAVE_SSE2 + +#if HAVE_SSE4_1 && CONFIG_HIGHBITDEPTH +const HighbdHt4x4Param kArrayHighbdHt4x4Param[] = { + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 0, 10), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 0, 12), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 1, 10), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 1, 12), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 2, 10), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 2, 12), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 3, 10), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 3, 12), +#if CONFIG_EXT_TX + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 4, 10), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 4, 12), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 5, 10), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 5, 12), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 6, 10), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 6, 12), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 7, 10), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 7, 12), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 8, 10), + make_tuple(&av1_fwd_txfm2d_4x4_sse4_1, 8, 12), +#endif // CONFIG_EXT_TX +}; + +INSTANTIATE_TEST_CASE_P(SSE4_1, AV1HighbdTrans4x4HT, + ::testing::ValuesIn(kArrayHighbdHt4x4Param)); + +#endif // HAVE_SSE4_1 && CONFIG_HIGHBITDEPTH + +} // namespace diff --git a/third_party/aom/test/av1_fht4x8_test.cc b/third_party/aom/test/av1_fht4x8_test.cc new file mode 100644 index 0000000000..a899c87391 --- /dev/null +++ b/third_party/aom/test/av1_fht4x8_test.cc @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" + +#include "aom_ports/mem.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/transform_test_base.h" +#include "test/util.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); +using std::tr1::tuple; +using libaom_test::FhtFunc; +typedef tuple Ht4x8Param; + +void fht4x8_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht4x8_c(in, out, stride, tx_type); +} + +void iht4x8_ref(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_iht4x8_32_add_c(in, out, stride, tx_type); +} + +class AV1Trans4x8HT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~AV1Trans4x8HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 4; + height_ = 8; + fwd_txfm_ref = fht4x8_ref; + inv_txfm_ref = iht4x8_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(AV1Trans4x8HT, AccuracyCheck) { RunAccuracyCheck(0, 0.00001); } +TEST_P(AV1Trans4x8HT, CoeffCheck) { RunCoeffCheck(); } +TEST_P(AV1Trans4x8HT, MemCheck) { RunMemCheck(); } +TEST_P(AV1Trans4x8HT, InvCoeffCheck) { RunInvCoeffCheck(); } +TEST_P(AV1Trans4x8HT, InvAccuracyCheck) { RunInvAccuracyCheck(0); } + +using std::tr1::make_tuple; + +const Ht4x8Param kArrayHt4x8Param_c[] = { + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 0, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 1, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 2, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 3, AOM_BITS_8, 32), +#if CONFIG_EXT_TX + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 4, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 5, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 6, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 7, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 8, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 9, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 10, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 11, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 12, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 13, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 14, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_c, &av1_iht4x8_32_add_c, 15, AOM_BITS_8, 32) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(C, AV1Trans4x8HT, + ::testing::ValuesIn(kArrayHt4x8Param_c)); + +#if HAVE_SSE2 +const Ht4x8Param kArrayHt4x8Param_sse2[] = { + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 0, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 1, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 2, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 3, AOM_BITS_8, 32), +#if CONFIG_EXT_TX + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 4, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 5, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 6, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 7, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 8, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 9, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 10, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 11, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 12, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 13, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 14, AOM_BITS_8, 32), + make_tuple(&av1_fht4x8_sse2, &av1_iht4x8_32_add_sse2, 15, AOM_BITS_8, 32) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(SSE2, AV1Trans4x8HT, + ::testing::ValuesIn(kArrayHt4x8Param_sse2)); +#endif // HAVE_SSE2 + +} // namespace diff --git a/third_party/aom/test/av1_fht8x16_test.cc b/third_party/aom/test/av1_fht8x16_test.cc new file mode 100644 index 0000000000..ace9a8f47d --- /dev/null +++ b/third_party/aom/test/av1_fht8x16_test.cc @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" + +#include "aom_ports/mem.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/transform_test_base.h" +#include "test/util.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); +using std::tr1::tuple; +using libaom_test::FhtFunc; +typedef tuple Ht8x16Param; + +void fht8x16_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht8x16_c(in, out, stride, tx_type); +} + +void iht8x16_ref(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_iht8x16_128_add_c(in, out, stride, tx_type); +} + +class AV1Trans8x16HT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~AV1Trans8x16HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 8; + height_ = 16; + inv_txfm_ref = iht8x16_ref; + fwd_txfm_ref = fht8x16_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(AV1Trans8x16HT, AccuracyCheck) { RunAccuracyCheck(1, 0.001); } +TEST_P(AV1Trans8x16HT, MemCheck) { RunMemCheck(); } +TEST_P(AV1Trans8x16HT, CoeffCheck) { RunCoeffCheck(); } +TEST_P(AV1Trans8x16HT, InvCoeffCheck) { RunInvCoeffCheck(); } +TEST_P(AV1Trans8x16HT, InvAccuracyCheck) { RunInvAccuracyCheck(1); } + +using std::tr1::make_tuple; + +const Ht8x16Param kArrayHt8x16Param_c[] = { + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 0, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 1, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 2, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 3, AOM_BITS_8, 128), +#if CONFIG_EXT_TX + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 4, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 5, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 6, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 7, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 8, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 9, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 10, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 11, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 12, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 13, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 14, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_c, &av1_iht8x16_128_add_c, 15, AOM_BITS_8, 128) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(C, AV1Trans8x16HT, + ::testing::ValuesIn(kArrayHt8x16Param_c)); + +#if HAVE_SSE2 +const Ht8x16Param kArrayHt8x16Param_sse2[] = { + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 0, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 1, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 2, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 3, AOM_BITS_8, 128), +#if CONFIG_EXT_TX + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 4, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 5, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 6, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 7, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 8, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 9, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 10, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 11, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 12, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 13, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 14, AOM_BITS_8, 128), + make_tuple(&av1_fht8x16_sse2, &av1_iht8x16_128_add_sse2, 15, AOM_BITS_8, 128) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(SSE2, AV1Trans8x16HT, + ::testing::ValuesIn(kArrayHt8x16Param_sse2)); +#endif // HAVE_SSE2 + +} // namespace diff --git a/third_party/aom/test/av1_fht8x4_test.cc b/third_party/aom/test/av1_fht8x4_test.cc new file mode 100644 index 0000000000..9bf4ff6475 --- /dev/null +++ b/third_party/aom/test/av1_fht8x4_test.cc @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" + +#include "aom_ports/mem.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/transform_test_base.h" +#include "test/util.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); +using std::tr1::tuple; +using libaom_test::FhtFunc; +typedef tuple Ht8x4Param; + +void fht8x4_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht8x4_c(in, out, stride, tx_type); +} + +void iht8x4_ref(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_iht8x4_32_add_c(in, out, stride, tx_type); +} + +class AV1Trans8x4HT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~AV1Trans8x4HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 8; + height_ = 4; + fwd_txfm_ref = fht8x4_ref; + inv_txfm_ref = iht8x4_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(AV1Trans8x4HT, AccuracyCheck) { RunAccuracyCheck(0, 0.00001); } +TEST_P(AV1Trans8x4HT, CoeffCheck) { RunCoeffCheck(); } +TEST_P(AV1Trans8x4HT, MemCheck) { RunMemCheck(); } +TEST_P(AV1Trans8x4HT, InvCoeffCheck) { RunInvCoeffCheck(); } +TEST_P(AV1Trans8x4HT, InvAccuracyCheck) { RunInvAccuracyCheck(0); } + +using std::tr1::make_tuple; + +const Ht8x4Param kArrayHt8x4Param_c[] = { + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 0, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 1, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 2, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 3, AOM_BITS_8, 32), +#if CONFIG_EXT_TX + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 4, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 5, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 6, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 7, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 8, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 9, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 10, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 11, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 12, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 13, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 14, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_c, &av1_iht8x4_32_add_c, 15, AOM_BITS_8, 32) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(C, AV1Trans8x4HT, + ::testing::ValuesIn(kArrayHt8x4Param_c)); + +#if HAVE_SSE2 +const Ht8x4Param kArrayHt8x4Param_sse2[] = { + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 0, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 1, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 2, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 3, AOM_BITS_8, 32), +#if CONFIG_EXT_TX + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 4, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 5, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 6, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 7, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 8, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 9, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 10, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 11, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 12, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 13, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 14, AOM_BITS_8, 32), + make_tuple(&av1_fht8x4_sse2, &av1_iht8x4_32_add_sse2, 15, AOM_BITS_8, 32) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(SSE2, AV1Trans8x4HT, + ::testing::ValuesIn(kArrayHt8x4Param_sse2)); +#endif // HAVE_SSE2 + +} // namespace diff --git a/third_party/aom/test/av1_fht8x8_test.cc b/third_party/aom/test/av1_fht8x8_test.cc new file mode 100644 index 0000000000..99cff1014f --- /dev/null +++ b/third_party/aom/test/av1_fht8x8_test.cc @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" + +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/transform_test_base.h" +#include "test/util.h" +#include "aom_ports/mem.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); + +using libaom_test::FhtFunc; +using std::tr1::tuple; +typedef tuple Ht8x8Param; + +void fht8x8_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht8x8_c(in, out, stride, tx_type); +} + +void iht8x8_ref(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_iht8x8_64_add_c(in, out, stride, tx_type); +} + +#if CONFIG_HIGHBITDEPTH +typedef void (*IHbdHtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type, int bd); +typedef void (*HbdHtFunc)(const int16_t *input, int32_t *output, int stride, + int tx_type, int bd); +// Target optimized function, tx_type, bit depth +typedef tuple HighbdHt8x8Param; + +void highbd_fht8x8_ref(const int16_t *in, int32_t *out, int stride, int tx_type, + int bd) { + av1_fwd_txfm2d_8x8_c(in, out, stride, tx_type, bd); +} +#endif // CONFIG_HIGHBITDEPTH + +class AV1Trans8x8HT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~AV1Trans8x8HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 8; + height_ = 8; + fwd_txfm_ref = fht8x8_ref; + inv_txfm_ref = iht8x8_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(AV1Trans8x8HT, MemCheck) { RunMemCheck(); } +TEST_P(AV1Trans8x8HT, CoeffCheck) { RunCoeffCheck(); } +// Note: +// TODO(luoyi): Add tx_type, 9-15 for inverse transform. +// Need cleanup since same tests may be done in fdct8x8_test.cc +// TEST_P(AV1Trans8x8HT, AccuracyCheck) { RunAccuracyCheck(0); } +// TEST_P(AV1Trans8x8HT, InvAccuracyCheck) { RunInvAccuracyCheck(0); } +// TEST_P(AV1Trans8x8HT, InvCoeffCheck) { RunInvCoeffCheck(); } + +#if CONFIG_HIGHBITDEPTH +class AV1HighbdTrans8x8HT : public ::testing::TestWithParam { + public: + virtual ~AV1HighbdTrans8x8HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + fwd_txfm_ref_ = highbd_fht8x8_ref; + tx_type_ = GET_PARAM(1); + bit_depth_ = GET_PARAM(2); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = 64; + + input_ = reinterpret_cast( + aom_memalign(16, sizeof(int16_t) * num_coeffs_)); + output_ = reinterpret_cast( + aom_memalign(16, sizeof(int32_t) * num_coeffs_)); + output_ref_ = reinterpret_cast( + aom_memalign(16, sizeof(int32_t) * num_coeffs_)); + } + + virtual void TearDown() { + aom_free(input_); + aom_free(output_); + aom_free(output_ref_); + libaom_test::ClearSystemState(); + } + + protected: + void RunBitexactCheck(); + + private: + HbdHtFunc fwd_txfm_; + HbdHtFunc fwd_txfm_ref_; + int tx_type_; + int bit_depth_; + int mask_; + int num_coeffs_; + int16_t *input_; + int32_t *output_; + int32_t *output_ref_; +}; + +void AV1HighbdTrans8x8HT::RunBitexactCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + int i, j; + const int stride = 8; + const int num_tests = 1000; + const int num_coeffs = 64; + + for (i = 0; i < num_tests; ++i) { + for (j = 0; j < num_coeffs; ++j) { + input_[j] = (rnd.Rand16() & mask_) - (rnd.Rand16() & mask_); + } + + fwd_txfm_ref_(input_, output_ref_, stride, tx_type_, bit_depth_); + ASM_REGISTER_STATE_CHECK( + fwd_txfm_(input_, output_, stride, tx_type_, bit_depth_)); + + for (j = 0; j < num_coeffs; ++j) { + EXPECT_EQ(output_ref_[j], output_[j]) + << "Not bit-exact result at index: " << j << " at test block: " << i; + } + } +} + +TEST_P(AV1HighbdTrans8x8HT, HighbdCoeffCheck) { RunBitexactCheck(); } +#endif // CONFIG_HIGHBITDEPTH + +using std::tr1::make_tuple; + +#if HAVE_SSE2 +const Ht8x8Param kArrayHt8x8Param_sse2[] = { + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 0, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 1, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 2, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 3, AOM_BITS_8, 64), +#if CONFIG_EXT_TX + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 4, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 5, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 6, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 7, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 8, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 9, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 10, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 11, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 12, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 13, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 14, AOM_BITS_8, 64), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 15, AOM_BITS_8, 64) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(SSE2, AV1Trans8x8HT, + ::testing::ValuesIn(kArrayHt8x8Param_sse2)); +#endif // HAVE_SSE2 + +#if HAVE_SSE4_1 && CONFIG_HIGHBITDEPTH +const HighbdHt8x8Param kArrayHBDHt8x8Param_sse4_1[] = { + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 0, 10), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 0, 12), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 1, 10), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 1, 12), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 2, 10), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 2, 12), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 3, 10), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 3, 12), +#if CONFIG_EXT_TX + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 4, 10), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 4, 12), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 5, 10), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 5, 12), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 6, 10), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 6, 12), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 7, 10), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 7, 12), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 8, 10), + make_tuple(&av1_fwd_txfm2d_8x8_sse4_1, 8, 12), +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(SSE4_1, AV1HighbdTrans8x8HT, + ::testing::ValuesIn(kArrayHBDHt8x8Param_sse4_1)); +#endif // HAVE_SSE4_1 && CONFIG_HIGHBITDEPTH + +} // namespace diff --git a/third_party/aom/test/av1_fwd_txfm1d_test.cc b/third_party/aom/test/av1_fwd_txfm1d_test.cc new file mode 100644 index 0000000000..a9b3f8e406 --- /dev/null +++ b/third_party/aom/test/av1_fwd_txfm1d_test.cc @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/common/av1_fwd_txfm1d.h" +#include "test/av1_txfm_test.h" + +using libaom_test::ACMRandom; +using libaom_test::input_base; +using libaom_test::reference_hybrid_1d; +using libaom_test::TYPE_TXFM; +using libaom_test::TYPE_DCT; +using libaom_test::TYPE_ADST; + +namespace { +const int txfm_type_num = 2; +const TYPE_TXFM txfm_type_ls[2] = { TYPE_DCT, TYPE_ADST }; + +const int txfm_size_num = 5; +const int txfm_size_ls[5] = { 4, 8, 16, 32, 64 }; + +const TxfmFunc fwd_txfm_func_ls[2][5] = { +#if CONFIG_TX64X64 + { av1_fdct4_new, av1_fdct8_new, av1_fdct16_new, av1_fdct32_new, + av1_fdct64_new }, +#else + { av1_fdct4_new, av1_fdct8_new, av1_fdct16_new, av1_fdct32_new, NULL }, +#endif + { av1_fadst4_new, av1_fadst8_new, av1_fadst16_new, av1_fadst32_new, NULL } +}; + +// the maximum stage number of fwd/inv 1d dct/adst txfm is 12 +const int8_t cos_bit[12] = { 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 }; +const int8_t range_bit[12] = { 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 }; + +TEST(av1_fwd_txfm1d, round_shift) { + EXPECT_EQ(round_shift(7, 1), 4); + EXPECT_EQ(round_shift(-7, 1), -3); + + EXPECT_EQ(round_shift(7, 2), 2); + EXPECT_EQ(round_shift(-7, 2), -2); + + EXPECT_EQ(round_shift(8, 2), 2); + EXPECT_EQ(round_shift(-8, 2), -2); +} + +TEST(av1_fwd_txfm1d, get_max_bit) { + int max_bit = get_max_bit(8); + EXPECT_EQ(max_bit, 3); +} + +TEST(av1_fwd_txfm1d, cospi_arr) { + for (int i = 0; i < 7; i++) { + for (int j = 0; j < 64; j++) { + EXPECT_EQ(cospi_arr[i][j], + (int32_t)round(cos(M_PI * j / 128) * (1 << (cos_bit_min + i)))); + } + } +} + +TEST(av1_fwd_txfm1d, clamp_block) { + int16_t block[5][5] = { { 7, -5, 6, -3, 9 }, + { 7, -5, 6, -3, 9 }, + { 7, -5, 6, -3, 9 }, + { 7, -5, 6, -3, 9 }, + { 7, -5, 6, -3, 9 } }; + + int16_t ref_block[5][5] = { { 7, -5, 6, -3, 9 }, + { 7, -5, 6, -3, 9 }, + { 7, -4, 2, -3, 9 }, + { 7, -4, 2, -3, 9 }, + { 7, -4, 2, -3, 9 } }; + + int row = 2; + int col = 1; + int block_size = 3; + int stride = 5; + clamp_block(block[row] + col, block_size, stride, -4, 2); + for (int r = 0; r < stride; r++) { + for (int c = 0; c < stride; c++) { + EXPECT_EQ(block[r][c], ref_block[r][c]); + } + } +} + +TEST(av1_fwd_txfm1d, accuracy) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + for (int si = 0; si < txfm_size_num; ++si) { + int txfm_size = txfm_size_ls[si]; + int32_t *input = new int32_t[txfm_size]; + int32_t *output = new int32_t[txfm_size]; + double *ref_input = new double[txfm_size]; + double *ref_output = new double[txfm_size]; + + for (int ti = 0; ti < txfm_type_num; ++ti) { + TYPE_TXFM txfm_type = txfm_type_ls[ti]; + TxfmFunc fwd_txfm_func = fwd_txfm_func_ls[ti][si]; + int max_error = 7; + + const int count_test_block = 5000; + if (fwd_txfm_func != NULL) { + for (int ti = 0; ti < count_test_block; ++ti) { + for (int ni = 0; ni < txfm_size; ++ni) { + input[ni] = rnd.Rand16() % input_base - rnd.Rand16() % input_base; + ref_input[ni] = static_cast(input[ni]); + } + + fwd_txfm_func(input, output, cos_bit, range_bit); + reference_hybrid_1d(ref_input, ref_output, txfm_size, txfm_type); + + for (int ni = 0; ni < txfm_size; ++ni) { + EXPECT_LE( + abs(output[ni] - static_cast(round(ref_output[ni]))), + max_error); + } + } + } + } + + delete[] input; + delete[] output; + delete[] ref_input; + delete[] ref_output; + } +} +} // namespace diff --git a/third_party/aom/test/av1_fwd_txfm2d_test.cc b/third_party/aom/test/av1_fwd_txfm2d_test.cc new file mode 100644 index 0000000000..25cf5ad53b --- /dev/null +++ b/third_party/aom/test/av1_fwd_txfm2d_test.cc @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "test/acm_random.h" +#include "test/util.h" +#include "test/av1_txfm_test.h" +#include "av1/common/av1_txfm.h" +#include "./av1_rtcd.h" + +using libaom_test::ACMRandom; +using libaom_test::input_base; +using libaom_test::bd; +using libaom_test::compute_avg_abs_error; +using libaom_test::Fwd_Txfm2d_Func; +using libaom_test::TYPE_TXFM; + +namespace { +#if CONFIG_HIGHBITDEPTH +// tx_type_, tx_size_, max_error_, max_avg_error_ +typedef std::tr1::tuple AV1FwdTxfm2dParam; + +class AV1FwdTxfm2d : public ::testing::TestWithParam { + public: + virtual void SetUp() { + tx_type_ = GET_PARAM(0); + tx_size_ = GET_PARAM(1); + max_error_ = GET_PARAM(2); + max_avg_error_ = GET_PARAM(3); + count_ = 500; + TXFM_2D_FLIP_CFG fwd_txfm_flip_cfg = + av1_get_fwd_txfm_cfg(tx_type_, tx_size_); + const TXFM_2D_CFG *fwd_txfm_cfg = fwd_txfm_flip_cfg.cfg; + int amplify_bit = fwd_txfm_cfg->shift[0] + fwd_txfm_cfg->shift[1] + + fwd_txfm_cfg->shift[2]; + ud_flip_ = fwd_txfm_flip_cfg.ud_flip; + lr_flip_ = fwd_txfm_flip_cfg.lr_flip; + amplify_factor_ = + amplify_bit >= 0 ? (1 << amplify_bit) : (1.0 / (1 << -amplify_bit)); + + fwd_txfm_ = libaom_test::fwd_txfm_func_ls[tx_size_]; + txfm1d_size_ = libaom_test::get_txfm1d_size(tx_size_); + txfm2d_size_ = txfm1d_size_ * txfm1d_size_; + get_txfm1d_type(tx_type_, &type0_, &type1_); + input_ = reinterpret_cast( + aom_memalign(16, sizeof(input_[0]) * txfm2d_size_)); + output_ = reinterpret_cast( + aom_memalign(16, sizeof(output_[0]) * txfm2d_size_)); + ref_input_ = reinterpret_cast( + aom_memalign(16, sizeof(ref_input_[0]) * txfm2d_size_)); + ref_output_ = reinterpret_cast( + aom_memalign(16, sizeof(ref_output_[0]) * txfm2d_size_)); + } + + void RunFwdAccuracyCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + double avg_abs_error = 0; + for (int ci = 0; ci < count_; ci++) { + for (int ni = 0; ni < txfm2d_size_; ++ni) { + input_[ni] = rnd.Rand16() % input_base; + ref_input_[ni] = static_cast(input_[ni]); + output_[ni] = 0; + ref_output_[ni] = 0; + } + + fwd_txfm_(input_, output_, txfm1d_size_, tx_type_, bd); + + if (lr_flip_ && ud_flip_) + libaom_test::fliplrud(ref_input_, txfm1d_size_, txfm1d_size_); + else if (lr_flip_) + libaom_test::fliplr(ref_input_, txfm1d_size_, txfm1d_size_); + else if (ud_flip_) + libaom_test::flipud(ref_input_, txfm1d_size_, txfm1d_size_); + + reference_hybrid_2d(ref_input_, ref_output_, txfm1d_size_, type0_, + type1_); + + for (int ni = 0; ni < txfm2d_size_; ++ni) { + ref_output_[ni] = round(ref_output_[ni] * amplify_factor_); + EXPECT_GE(max_error_, + fabs(output_[ni] - ref_output_[ni]) / amplify_factor_); + } + avg_abs_error += compute_avg_abs_error( + output_, ref_output_, txfm2d_size_); + } + + avg_abs_error /= amplify_factor_; + avg_abs_error /= count_; + // max_abs_avg_error comes from upper bound of avg_abs_error + // printf("type0: %d type1: %d txfm_size: %d accuracy_avg_abs_error: + // %f\n", type0_, type1_, txfm1d_size_, avg_abs_error); + EXPECT_GE(max_avg_error_, avg_abs_error); + } + + virtual void TearDown() { + aom_free(input_); + aom_free(output_); + aom_free(ref_input_); + aom_free(ref_output_); + } + + private: + double max_error_; + double max_avg_error_; + int count_; + double amplify_factor_; + TX_TYPE tx_type_; + TX_SIZE tx_size_; + int txfm1d_size_; + int txfm2d_size_; + Fwd_Txfm2d_Func fwd_txfm_; + TYPE_TXFM type0_; + TYPE_TXFM type1_; + int16_t *input_; + int32_t *output_; + double *ref_input_; + double *ref_output_; + int ud_flip_; // flip upside down + int lr_flip_; // flip left to right +}; + +TEST_P(AV1FwdTxfm2d, RunFwdAccuracyCheck) { RunFwdAccuracyCheck(); } +const AV1FwdTxfm2dParam av1_fwd_txfm2d_param_c[] = { +#if CONFIG_EXT_TX + AV1FwdTxfm2dParam(FLIPADST_DCT, TX_4X4, 2, 0.2), + AV1FwdTxfm2dParam(DCT_FLIPADST, TX_4X4, 2, 0.2), + AV1FwdTxfm2dParam(FLIPADST_FLIPADST, TX_4X4, 2, 0.2), + AV1FwdTxfm2dParam(ADST_FLIPADST, TX_4X4, 2, 0.2), + AV1FwdTxfm2dParam(FLIPADST_ADST, TX_4X4, 2, 0.2), + AV1FwdTxfm2dParam(FLIPADST_DCT, TX_8X8, 5, 0.6), + AV1FwdTxfm2dParam(DCT_FLIPADST, TX_8X8, 5, 0.6), + AV1FwdTxfm2dParam(FLIPADST_FLIPADST, TX_8X8, 5, 0.6), + AV1FwdTxfm2dParam(ADST_FLIPADST, TX_8X8, 5, 0.6), + AV1FwdTxfm2dParam(FLIPADST_ADST, TX_8X8, 5, 0.6), + AV1FwdTxfm2dParam(FLIPADST_DCT, TX_16X16, 11, 1.5), + AV1FwdTxfm2dParam(DCT_FLIPADST, TX_16X16, 11, 1.5), + AV1FwdTxfm2dParam(FLIPADST_FLIPADST, TX_16X16, 11, 1.5), + AV1FwdTxfm2dParam(ADST_FLIPADST, TX_16X16, 11, 1.5), + AV1FwdTxfm2dParam(FLIPADST_ADST, TX_16X16, 11, 1.5), + AV1FwdTxfm2dParam(FLIPADST_DCT, TX_32X32, 70, 7), + AV1FwdTxfm2dParam(DCT_FLIPADST, TX_32X32, 70, 7), + AV1FwdTxfm2dParam(FLIPADST_FLIPADST, TX_32X32, 70, 7), + AV1FwdTxfm2dParam(ADST_FLIPADST, TX_32X32, 70, 7), + AV1FwdTxfm2dParam(FLIPADST_ADST, TX_32X32, 70, 7), +#endif + AV1FwdTxfm2dParam(DCT_DCT, TX_4X4, 2, 0.2), + AV1FwdTxfm2dParam(ADST_DCT, TX_4X4, 2, 0.2), + AV1FwdTxfm2dParam(DCT_ADST, TX_4X4, 2, 0.2), + AV1FwdTxfm2dParam(ADST_ADST, TX_4X4, 2, 0.2), + AV1FwdTxfm2dParam(DCT_DCT, TX_8X8, 5, 0.6), + AV1FwdTxfm2dParam(ADST_DCT, TX_8X8, 5, 0.6), + AV1FwdTxfm2dParam(DCT_ADST, TX_8X8, 5, 0.6), + AV1FwdTxfm2dParam(ADST_ADST, TX_8X8, 5, 0.6), + AV1FwdTxfm2dParam(DCT_DCT, TX_16X16, 11, 1.5), + AV1FwdTxfm2dParam(ADST_DCT, TX_16X16, 11, 1.5), + AV1FwdTxfm2dParam(DCT_ADST, TX_16X16, 11, 1.5), + AV1FwdTxfm2dParam(ADST_ADST, TX_16X16, 11, 1.5), + AV1FwdTxfm2dParam(DCT_DCT, TX_32X32, 70, 7), + AV1FwdTxfm2dParam(ADST_DCT, TX_32X32, 70, 7), + AV1FwdTxfm2dParam(DCT_ADST, TX_32X32, 70, 7), + AV1FwdTxfm2dParam(ADST_ADST, TX_32X32, 70, 7) +}; + +INSTANTIATE_TEST_CASE_P(C, AV1FwdTxfm2d, + ::testing::ValuesIn(av1_fwd_txfm2d_param_c)); + +#endif // CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/av1_highbd_iht_test.cc b/third_party/aom/test/av1_highbd_iht_test.cc new file mode 100644 index 0000000000..3b263638f8 --- /dev/null +++ b/third_party/aom/test/av1_highbd_iht_test.cc @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "av1/common/enums.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_ports/mem.h" + +namespace { + +using std::tr1::tuple; +using libaom_test::ACMRandom; + +typedef void (*HbdHtFunc)(const int16_t *input, int32_t *output, int stride, + int tx_type, int bd); + +typedef void (*IHbdHtFunc)(const int32_t *coeff, uint16_t *output, int stride, + int tx_type, int bd); + +// Test parameter argument list: +// +typedef tuple IHbdHtParam; + +class AV1HighbdInvHTNxN : public ::testing::TestWithParam { + public: + virtual ~AV1HighbdInvHTNxN() {} + + virtual void SetUp() { + txfm_ref_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + inv_txfm_ref_ = GET_PARAM(2); + num_coeffs_ = GET_PARAM(3); + tx_type_ = GET_PARAM(4); + bit_depth_ = GET_PARAM(5); + + input_ = reinterpret_cast( + aom_memalign(16, sizeof(input_[0]) * num_coeffs_)); + + // Note: + // Inverse transform input buffer is 32-byte aligned + // Refer to /av1/encoder/context_tree.c, function, + // void alloc_mode_context(). + coeffs_ = reinterpret_cast( + aom_memalign(32, sizeof(coeffs_[0]) * num_coeffs_)); + output_ = reinterpret_cast( + aom_memalign(32, sizeof(output_[0]) * num_coeffs_)); + output_ref_ = reinterpret_cast( + aom_memalign(32, sizeof(output_ref_[0]) * num_coeffs_)); + } + + virtual void TearDown() { + aom_free(input_); + aom_free(coeffs_); + aom_free(output_); + aom_free(output_ref_); + libaom_test::ClearSystemState(); + } + + protected: + void RunBitexactCheck(); + + private: + int GetStride() const { + if (16 == num_coeffs_) { + return 4; + } else if (64 == num_coeffs_) { + return 8; + } else if (256 == num_coeffs_) { + return 16; + } else if (1024 == num_coeffs_) { + return 32; + } else { + return 0; + } + } + + HbdHtFunc txfm_ref_; + IHbdHtFunc inv_txfm_; + IHbdHtFunc inv_txfm_ref_; + int num_coeffs_; + int tx_type_; + int bit_depth_; + + int16_t *input_; + int32_t *coeffs_; + uint16_t *output_; + uint16_t *output_ref_; +}; + +void AV1HighbdInvHTNxN::RunBitexactCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int stride = GetStride(); + const int num_tests = 20000; + const uint16_t mask = (1 << bit_depth_) - 1; + + for (int i = 0; i < num_tests; ++i) { + for (int j = 0; j < num_coeffs_; ++j) { + input_[j] = (rnd.Rand16() & mask) - (rnd.Rand16() & mask); + output_ref_[j] = rnd.Rand16() & mask; + output_[j] = output_ref_[j]; + } + + txfm_ref_(input_, coeffs_, stride, tx_type_, bit_depth_); + inv_txfm_ref_(coeffs_, output_ref_, stride, tx_type_, bit_depth_); + ASM_REGISTER_STATE_CHECK( + inv_txfm_(coeffs_, output_, stride, tx_type_, bit_depth_)); + + for (int j = 0; j < num_coeffs_; ++j) { + EXPECT_EQ(output_ref_[j], output_[j]) + << "Not bit-exact result at index: " << j << " At test block: " << i; + } + } +} + +TEST_P(AV1HighbdInvHTNxN, InvTransResultCheck) { RunBitexactCheck(); } + +using std::tr1::make_tuple; + +#if HAVE_SSE4_1 && CONFIG_HIGHBITDEPTH +#define PARAM_LIST_4X4 \ + &av1_fwd_txfm2d_4x4_c, &av1_inv_txfm2d_add_4x4_sse4_1, \ + &av1_inv_txfm2d_add_4x4_c, 16 + +#define PARAM_LIST_8X8 \ + &av1_fwd_txfm2d_8x8_c, &av1_inv_txfm2d_add_8x8_sse4_1, \ + &av1_inv_txfm2d_add_8x8_c, 64 + +#define PARAM_LIST_16X16 \ + &av1_fwd_txfm2d_16x16_c, &av1_inv_txfm2d_add_16x16_sse4_1, \ + &av1_inv_txfm2d_add_16x16_c, 256 + +const IHbdHtParam kArrayIhtParam[] = { + // 16x16 + make_tuple(PARAM_LIST_16X16, DCT_DCT, 10), + make_tuple(PARAM_LIST_16X16, DCT_DCT, 12), + make_tuple(PARAM_LIST_16X16, ADST_DCT, 10), + make_tuple(PARAM_LIST_16X16, ADST_DCT, 12), + make_tuple(PARAM_LIST_16X16, DCT_ADST, 10), + make_tuple(PARAM_LIST_16X16, DCT_ADST, 12), + make_tuple(PARAM_LIST_16X16, ADST_ADST, 10), + make_tuple(PARAM_LIST_16X16, ADST_ADST, 12), +#if CONFIG_EXT_TX + make_tuple(PARAM_LIST_16X16, FLIPADST_DCT, 10), + make_tuple(PARAM_LIST_16X16, FLIPADST_DCT, 12), + make_tuple(PARAM_LIST_16X16, DCT_FLIPADST, 10), + make_tuple(PARAM_LIST_16X16, DCT_FLIPADST, 12), + make_tuple(PARAM_LIST_16X16, FLIPADST_FLIPADST, 10), + make_tuple(PARAM_LIST_16X16, FLIPADST_FLIPADST, 12), + make_tuple(PARAM_LIST_16X16, ADST_FLIPADST, 10), + make_tuple(PARAM_LIST_16X16, ADST_FLIPADST, 12), + make_tuple(PARAM_LIST_16X16, FLIPADST_ADST, 10), + make_tuple(PARAM_LIST_16X16, FLIPADST_ADST, 12), +#endif + // 8x8 + make_tuple(PARAM_LIST_8X8, DCT_DCT, 10), + make_tuple(PARAM_LIST_8X8, DCT_DCT, 12), + make_tuple(PARAM_LIST_8X8, ADST_DCT, 10), + make_tuple(PARAM_LIST_8X8, ADST_DCT, 12), + make_tuple(PARAM_LIST_8X8, DCT_ADST, 10), + make_tuple(PARAM_LIST_8X8, DCT_ADST, 12), + make_tuple(PARAM_LIST_8X8, ADST_ADST, 10), + make_tuple(PARAM_LIST_8X8, ADST_ADST, 12), +#if CONFIG_EXT_TX + make_tuple(PARAM_LIST_8X8, FLIPADST_DCT, 10), + make_tuple(PARAM_LIST_8X8, FLIPADST_DCT, 12), + make_tuple(PARAM_LIST_8X8, DCT_FLIPADST, 10), + make_tuple(PARAM_LIST_8X8, DCT_FLIPADST, 12), + make_tuple(PARAM_LIST_8X8, FLIPADST_FLIPADST, 10), + make_tuple(PARAM_LIST_8X8, FLIPADST_FLIPADST, 12), + make_tuple(PARAM_LIST_8X8, ADST_FLIPADST, 10), + make_tuple(PARAM_LIST_8X8, ADST_FLIPADST, 12), + make_tuple(PARAM_LIST_8X8, FLIPADST_ADST, 10), + make_tuple(PARAM_LIST_8X8, FLIPADST_ADST, 12), +#endif + // 4x4 + make_tuple(PARAM_LIST_4X4, DCT_DCT, 10), + make_tuple(PARAM_LIST_4X4, DCT_DCT, 12), + make_tuple(PARAM_LIST_4X4, ADST_DCT, 10), + make_tuple(PARAM_LIST_4X4, ADST_DCT, 12), + make_tuple(PARAM_LIST_4X4, DCT_ADST, 10), + make_tuple(PARAM_LIST_4X4, DCT_ADST, 12), + make_tuple(PARAM_LIST_4X4, ADST_ADST, 10), + make_tuple(PARAM_LIST_4X4, ADST_ADST, 12), +#if CONFIG_EXT_TX + make_tuple(PARAM_LIST_4X4, FLIPADST_DCT, 10), + make_tuple(PARAM_LIST_4X4, FLIPADST_DCT, 12), + make_tuple(PARAM_LIST_4X4, DCT_FLIPADST, 10), + make_tuple(PARAM_LIST_4X4, DCT_FLIPADST, 12), + make_tuple(PARAM_LIST_4X4, FLIPADST_FLIPADST, 10), + make_tuple(PARAM_LIST_4X4, FLIPADST_FLIPADST, 12), + make_tuple(PARAM_LIST_4X4, ADST_FLIPADST, 10), + make_tuple(PARAM_LIST_4X4, ADST_FLIPADST, 12), + make_tuple(PARAM_LIST_4X4, FLIPADST_ADST, 10), + make_tuple(PARAM_LIST_4X4, FLIPADST_ADST, 12), +#endif +}; + +INSTANTIATE_TEST_CASE_P(SSE4_1, AV1HighbdInvHTNxN, + ::testing::ValuesIn(kArrayIhtParam)); +#endif // HAVE_SSE4_1 && CONFIG_HIGHBITDEPTH + +#if HAVE_AVX2 && CONFIG_HIGHBITDEPTH +#define PARAM_LIST_32X32 \ + &av1_fwd_txfm2d_32x32_c, &av1_inv_txfm2d_add_32x32_avx2, \ + &av1_inv_txfm2d_add_32x32_c, 1024 + +const IHbdHtParam kArrayIhtParam32x32[] = { + // 32x32 + make_tuple(PARAM_LIST_32X32, DCT_DCT, 10), + make_tuple(PARAM_LIST_32X32, DCT_DCT, 12), +}; + +INSTANTIATE_TEST_CASE_P(AVX2, AV1HighbdInvHTNxN, + ::testing::ValuesIn(kArrayIhtParam32x32)); + +#endif // HAVE_AVX2 && CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/av1_inv_txfm1d_test.cc b/third_party/aom/test/av1_inv_txfm1d_test.cc new file mode 100644 index 0000000000..9cf33a2fd4 --- /dev/null +++ b/third_party/aom/test/av1_inv_txfm1d_test.cc @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "test/av1_txfm_test.h" +#include "av1/common/av1_fwd_txfm1d.h" +#include "av1/common/av1_inv_txfm1d.h" + +using libaom_test::ACMRandom; +using libaom_test::input_base; + +namespace { +const int txfm_type_num = 2; +const int txfm_size_ls[5] = { 4, 8, 16, 32, 64 }; + +const TxfmFunc fwd_txfm_func_ls[][2] = { + { av1_fdct4_new, av1_fadst4_new }, + { av1_fdct8_new, av1_fadst8_new }, + { av1_fdct16_new, av1_fadst16_new }, + { av1_fdct32_new, av1_fadst32_new }, +#if CONFIG_TX64X64 + { av1_fdct64_new, NULL }, +#endif +}; + +const TxfmFunc inv_txfm_func_ls[][2] = { + { av1_idct4_new, av1_iadst4_new }, + { av1_idct8_new, av1_iadst8_new }, + { av1_idct16_new, av1_iadst16_new }, + { av1_idct32_new, av1_iadst32_new }, +#if CONFIG_TX64X64 + { av1_idct64_new, NULL }, +#endif +}; + +// the maximum stage number of fwd/inv 1d dct/adst txfm is 12 +const int8_t cos_bit[12] = { 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 }; +const int8_t range_bit[12] = { 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 }; + +#define ARRAY_SIZE(x) (int)(sizeof(x) / sizeof(x[0])) + +TEST(av1_inv_txfm1d, round_trip) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + for (int si = 0; si < ARRAY_SIZE(fwd_txfm_func_ls); ++si) { + int txfm_size = txfm_size_ls[si]; + + for (int ti = 0; ti < txfm_type_num; ++ti) { + TxfmFunc fwd_txfm_func = fwd_txfm_func_ls[si][ti]; + TxfmFunc inv_txfm_func = inv_txfm_func_ls[si][ti]; + int max_error = 2; + + if (!fwd_txfm_func) continue; + + const int count_test_block = 5000; + for (int ci = 0; ci < count_test_block; ++ci) { + int32_t input[64]; + int32_t output[64]; + int32_t round_trip_output[64]; + + assert(txfm_size <= ARRAY_SIZE(input)); + + for (int ni = 0; ni < txfm_size; ++ni) { + input[ni] = rnd.Rand16() % input_base - rnd.Rand16() % input_base; + } + + fwd_txfm_func(input, output, cos_bit, range_bit); + inv_txfm_func(output, round_trip_output, cos_bit, range_bit); + + for (int ni = 0; ni < txfm_size; ++ni) { + int node_err = + abs(input[ni] - round_shift(round_trip_output[ni], + get_max_bit(txfm_size) - 1)); + EXPECT_LE(node_err, max_error); + } + } + } + } +} + +} // namespace diff --git a/third_party/aom/test/av1_inv_txfm2d_test.cc b/third_party/aom/test/av1_inv_txfm2d_test.cc new file mode 100644 index 0000000000..bb2743af1b --- /dev/null +++ b/third_party/aom/test/av1_inv_txfm2d_test.cc @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "./av1_rtcd.h" +#include "test/acm_random.h" +#include "test/util.h" +#include "test/av1_txfm_test.h" +#include "av1/common/av1_inv_txfm2d_cfg.h" + +using libaom_test::ACMRandom; +using libaom_test::input_base; +using libaom_test::bd; +using libaom_test::compute_avg_abs_error; +using libaom_test::Fwd_Txfm2d_Func; +using libaom_test::Inv_Txfm2d_Func; + +namespace { + +#if CONFIG_HIGHBITDEPTH +// AV1InvTxfm2dParam argument list: +// tx_type_, tx_size_, max_error_, max_avg_error_ +typedef std::tr1::tuple AV1InvTxfm2dParam; + +class AV1InvTxfm2d : public ::testing::TestWithParam { + public: + virtual void SetUp() { + tx_type_ = GET_PARAM(0); + tx_size_ = GET_PARAM(1); + max_error_ = GET_PARAM(2); + max_avg_error_ = GET_PARAM(3); + txfm1d_size_ = libaom_test::get_txfm1d_size(tx_size_); + txfm2d_size_ = txfm1d_size_ * txfm1d_size_; + count_ = 500; + + input_ = reinterpret_cast( + aom_memalign(16, sizeof(int16_t) * txfm2d_size_)); + ref_input_ = reinterpret_cast( + aom_memalign(16, sizeof(uint16_t) * txfm2d_size_)); + output_ = reinterpret_cast( + aom_memalign(16, sizeof(int32_t) * txfm2d_size_)); + } + + void RunRoundtripCheck() { + const Fwd_Txfm2d_Func fwd_txfm_func = + libaom_test::fwd_txfm_func_ls[tx_size_]; + const Inv_Txfm2d_Func inv_txfm_func = + libaom_test::inv_txfm_func_ls[tx_size_]; + double avg_abs_error = 0; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + for (int ci = 0; ci < count_; ci++) { + for (int ni = 0; ni < txfm2d_size_; ++ni) { + if (ci == 0) { + int extreme_input = input_base - 1; + input_[ni] = extreme_input; // extreme case + ref_input_[ni] = 0; + } else { + input_[ni] = rnd.Rand16() % input_base; + ref_input_[ni] = 0; + } + } + + fwd_txfm_func(input_, output_, txfm1d_size_, tx_type_, bd); + inv_txfm_func(output_, ref_input_, txfm1d_size_, tx_type_, bd); + + for (int ni = 0; ni < txfm2d_size_; ++ni) { + EXPECT_GE(max_error_, abs(input_[ni] - ref_input_[ni])); + } + avg_abs_error += compute_avg_abs_error( + input_, ref_input_, txfm2d_size_); + } + + avg_abs_error /= count_; + // max_abs_avg_error comes from upper bound of + // printf("txfm1d_size: %d accuracy_avg_abs_error: %f\n", + // txfm1d_size_, avg_abs_error); + EXPECT_GE(max_avg_error_, avg_abs_error); + } + + virtual void TearDown() { + aom_free(input_); + aom_free(output_); + aom_free(ref_input_); + } + + private: + int count_; + int max_error_; + double max_avg_error_; + TX_TYPE tx_type_; + TX_SIZE tx_size_; + int txfm1d_size_; + int txfm2d_size_; + int16_t *input_; + uint16_t *ref_input_; + int32_t *output_; +}; + +TEST_P(AV1InvTxfm2d, RunRoundtripCheck) { RunRoundtripCheck(); } + +const AV1InvTxfm2dParam av1_inv_txfm2d_param[] = { +#if CONFIG_EXT_TX + AV1InvTxfm2dParam(FLIPADST_DCT, TX_4X4, 2, 0.002), + AV1InvTxfm2dParam(DCT_FLIPADST, TX_4X4, 2, 0.002), + AV1InvTxfm2dParam(FLIPADST_FLIPADST, TX_4X4, 2, 0.002), + AV1InvTxfm2dParam(ADST_FLIPADST, TX_4X4, 2, 0.002), + AV1InvTxfm2dParam(FLIPADST_ADST, TX_4X4, 2, 0.002), + AV1InvTxfm2dParam(FLIPADST_DCT, TX_8X8, 2, 0.02), + AV1InvTxfm2dParam(DCT_FLIPADST, TX_8X8, 2, 0.02), + AV1InvTxfm2dParam(FLIPADST_FLIPADST, TX_8X8, 2, 0.02), + AV1InvTxfm2dParam(ADST_FLIPADST, TX_8X8, 2, 0.02), + AV1InvTxfm2dParam(FLIPADST_ADST, TX_8X8, 2, 0.02), + AV1InvTxfm2dParam(FLIPADST_DCT, TX_16X16, 2, 0.04), + AV1InvTxfm2dParam(DCT_FLIPADST, TX_16X16, 2, 0.04), + AV1InvTxfm2dParam(FLIPADST_FLIPADST, TX_16X16, 11, 0.04), + AV1InvTxfm2dParam(ADST_FLIPADST, TX_16X16, 2, 0.04), + AV1InvTxfm2dParam(FLIPADST_ADST, TX_16X16, 2, 0.04), + AV1InvTxfm2dParam(FLIPADST_DCT, TX_32X32, 4, 0.4), + AV1InvTxfm2dParam(DCT_FLIPADST, TX_32X32, 4, 0.4), + AV1InvTxfm2dParam(FLIPADST_FLIPADST, TX_32X32, 4, 0.4), + AV1InvTxfm2dParam(ADST_FLIPADST, TX_32X32, 4, 0.4), + AV1InvTxfm2dParam(FLIPADST_ADST, TX_32X32, 4, 0.4), +#endif + AV1InvTxfm2dParam(DCT_DCT, TX_4X4, 2, 0.002), + AV1InvTxfm2dParam(ADST_DCT, TX_4X4, 2, 0.002), + AV1InvTxfm2dParam(DCT_ADST, TX_4X4, 2, 0.002), + AV1InvTxfm2dParam(ADST_ADST, TX_4X4, 2, 0.002), + AV1InvTxfm2dParam(DCT_DCT, TX_8X8, 2, 0.02), + AV1InvTxfm2dParam(ADST_DCT, TX_8X8, 2, 0.02), + AV1InvTxfm2dParam(DCT_ADST, TX_8X8, 2, 0.02), + AV1InvTxfm2dParam(ADST_ADST, TX_8X8, 2, 0.02), + AV1InvTxfm2dParam(DCT_DCT, TX_16X16, 2, 0.04), + AV1InvTxfm2dParam(ADST_DCT, TX_16X16, 2, 0.04), + AV1InvTxfm2dParam(DCT_ADST, TX_16X16, 2, 0.04), + AV1InvTxfm2dParam(ADST_ADST, TX_16X16, 2, 0.04), + AV1InvTxfm2dParam(DCT_DCT, TX_32X32, 4, 0.4), + AV1InvTxfm2dParam(ADST_DCT, TX_32X32, 4, 0.4), + AV1InvTxfm2dParam(DCT_ADST, TX_32X32, 4, 0.4), + AV1InvTxfm2dParam(ADST_ADST, TX_32X32, 4, 0.4) +}; + +INSTANTIATE_TEST_CASE_P(C, AV1InvTxfm2d, + ::testing::ValuesIn(av1_inv_txfm2d_param)); + +#endif // CONFIG_HIGHBITDEPTH + +} // namespace diff --git a/third_party/aom/test/av1_inv_txfm_test.cc b/third_party/aom/test/av1_inv_txfm_test.cc new file mode 100644 index 0000000000..af3fee8726 --- /dev/null +++ b/third_party/aom/test/av1_inv_txfm_test.cc @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "av1/common/blockd.h" +#include "av1/common/scan.h" +#include "aom/aom_integer.h" +#include "aom_dsp/inv_txfm.h" + +using libaom_test::ACMRandom; + +namespace { +const double kInvSqrt2 = 0.707106781186547524400844362104; + +void reference_idct_1d(const double *in, double *out, int size) { + for (int n = 0; n < size; ++n) { + out[n] = 0; + for (int k = 0; k < size; ++k) { + if (k == 0) + out[n] += kInvSqrt2 * in[k] * cos(PI * (2 * n + 1) * k / (2 * size)); + else + out[n] += in[k] * cos(PI * (2 * n + 1) * k / (2 * size)); + } + } +} + +typedef void (*IdctFuncRef)(const double *in, double *out, int size); +typedef void (*IdctFunc)(const tran_low_t *in, tran_low_t *out); + +class TransTestBase { + public: + virtual ~TransTestBase() {} + + protected: + void RunInvAccuracyCheck() { + tran_low_t *input = new tran_low_t[txfm_size_]; + tran_low_t *output = new tran_low_t[txfm_size_]; + double *ref_input = new double[txfm_size_]; + double *ref_output = new double[txfm_size_]; + + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 5000; + for (int ti = 0; ti < count_test_block; ++ti) { + for (int ni = 0; ni < txfm_size_; ++ni) { + input[ni] = rnd.Rand8() - rnd.Rand8(); + ref_input[ni] = static_cast(input[ni]); + } + + fwd_txfm_(input, output); + fwd_txfm_ref_(ref_input, ref_output, txfm_size_); + + for (int ni = 0; ni < txfm_size_; ++ni) { + EXPECT_LE( + abs(output[ni] - static_cast(round(ref_output[ni]))), + max_error_); + } + } + + delete[] input; + delete[] output; + delete[] ref_input; + delete[] ref_output; + } + + double max_error_; + int txfm_size_; + IdctFunc fwd_txfm_; + IdctFuncRef fwd_txfm_ref_; +}; + +typedef std::tr1::tuple IdctParam; +class AV1InvTxfm : public TransTestBase, + public ::testing::TestWithParam { + public: + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + fwd_txfm_ref_ = GET_PARAM(1); + txfm_size_ = GET_PARAM(2); + max_error_ = GET_PARAM(3); + } + virtual void TearDown() {} +}; + +TEST_P(AV1InvTxfm, RunInvAccuracyCheck) { RunInvAccuracyCheck(); } + +INSTANTIATE_TEST_CASE_P( + C, AV1InvTxfm, + ::testing::Values(IdctParam(&aom_idct4_c, &reference_idct_1d, 4, 1), + IdctParam(&aom_idct8_c, &reference_idct_1d, 8, 2), + IdctParam(&aom_idct16_c, &reference_idct_1d, 16, 4), + IdctParam(&aom_idct32_c, &reference_idct_1d, 32, 6))); + +#if CONFIG_AV1_ENCODER +typedef void (*FwdTxfmFunc)(const int16_t *in, tran_low_t *out, int stride); +typedef void (*InvTxfmFunc)(const tran_low_t *in, uint8_t *out, int stride); +typedef std::tr1::tuple + PartialInvTxfmParam; +#if !CONFIG_ADAPT_SCAN +const int kMaxNumCoeffs = 1024; +#endif +class AV1PartialIDctTest + : public ::testing::TestWithParam { + public: + virtual ~AV1PartialIDctTest() {} + virtual void SetUp() { + ftxfm_ = GET_PARAM(0); + full_itxfm_ = GET_PARAM(1); + partial_itxfm_ = GET_PARAM(2); + tx_size_ = GET_PARAM(3); + last_nonzero_ = GET_PARAM(4); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + int last_nonzero_; + TX_SIZE tx_size_; + FwdTxfmFunc ftxfm_; + InvTxfmFunc full_itxfm_; + InvTxfmFunc partial_itxfm_; +}; + +#if !CONFIG_ADAPT_SCAN +TEST_P(AV1PartialIDctTest, RunQuantCheck) { + int size; + switch (tx_size_) { + case TX_4X4: size = 4; break; + case TX_8X8: size = 8; break; + case TX_16X16: size = 16; break; + case TX_32X32: size = 32; break; + default: FAIL() << "Wrong Size!"; break; + } + DECLARE_ALIGNED(16, tran_low_t, test_coef_block1[kMaxNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, test_coef_block2[kMaxNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, dst1[kMaxNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, dst2[kMaxNumCoeffs]); + + const int count_test_block = 1000; + const int block_size = size * size; + + DECLARE_ALIGNED(16, int16_t, input_extreme_block[kMaxNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output_ref_block[kMaxNumCoeffs]); + + int max_error = 0; + for (int m = 0; m < count_test_block; ++m) { + // clear out destination buffer + memset(dst1, 0, sizeof(*dst1) * block_size); + memset(dst2, 0, sizeof(*dst2) * block_size); + memset(test_coef_block1, 0, sizeof(*test_coef_block1) * block_size); + memset(test_coef_block2, 0, sizeof(*test_coef_block2) * block_size); + + ACMRandom rnd(ACMRandom::DeterministicSeed()); + + for (int n = 0; n < count_test_block; ++n) { + // Initialize a test block with input range [-255, 255]. + if (n == 0) { + for (int j = 0; j < block_size; ++j) input_extreme_block[j] = 255; + } else if (n == 1) { + for (int j = 0; j < block_size; ++j) input_extreme_block[j] = -255; + } else { + for (int j = 0; j < block_size; ++j) { + input_extreme_block[j] = rnd.Rand8() % 2 ? 255 : -255; + } + } + + ftxfm_(input_extreme_block, output_ref_block, size); + + // quantization with maximum allowed step sizes + test_coef_block1[0] = (output_ref_block[0] / 1336) * 1336; + for (int j = 1; j < last_nonzero_; ++j) + test_coef_block1[get_scan((const AV1_COMMON *)NULL, tx_size_, DCT_DCT, + 0) + ->scan[j]] = (output_ref_block[j] / 1828) * 1828; + } + + ASM_REGISTER_STATE_CHECK(full_itxfm_(test_coef_block1, dst1, size)); + ASM_REGISTER_STATE_CHECK(partial_itxfm_(test_coef_block1, dst2, size)); + + for (int j = 0; j < block_size; ++j) { + const int diff = dst1[j] - dst2[j]; + const int error = diff * diff; + if (max_error < error) max_error = error; + } + } + + EXPECT_EQ(0, max_error) + << "Error: partial inverse transform produces different results"; +} + +TEST_P(AV1PartialIDctTest, ResultsMatch) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + int size; + switch (tx_size_) { + case TX_4X4: size = 4; break; + case TX_8X8: size = 8; break; + case TX_16X16: size = 16; break; + case TX_32X32: size = 32; break; + default: FAIL() << "Wrong Size!"; break; + } + DECLARE_ALIGNED(16, tran_low_t, test_coef_block1[kMaxNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, test_coef_block2[kMaxNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, dst1[kMaxNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, dst2[kMaxNumCoeffs]); + const int count_test_block = 1000; + const int max_coeff = 32766 / 4; + const int block_size = size * size; + int max_error = 0; + for (int i = 0; i < count_test_block; ++i) { + // clear out destination buffer + memset(dst1, 0, sizeof(*dst1) * block_size); + memset(dst2, 0, sizeof(*dst2) * block_size); + memset(test_coef_block1, 0, sizeof(*test_coef_block1) * block_size); + memset(test_coef_block2, 0, sizeof(*test_coef_block2) * block_size); + int max_energy_leftover = max_coeff * max_coeff; + for (int j = 0; j < last_nonzero_; ++j) { + int16_t coef = static_cast(sqrt(1.0 * max_energy_leftover) * + (rnd.Rand16() - 32768) / 65536); + max_energy_leftover -= coef * coef; + if (max_energy_leftover < 0) { + max_energy_leftover = 0; + coef = 0; + } + test_coef_block1[get_scan((const AV1_COMMON *)NULL, tx_size_, DCT_DCT, 0) + ->scan[j]] = coef; + } + + memcpy(test_coef_block2, test_coef_block1, + sizeof(*test_coef_block2) * block_size); + + ASM_REGISTER_STATE_CHECK(full_itxfm_(test_coef_block1, dst1, size)); + ASM_REGISTER_STATE_CHECK(partial_itxfm_(test_coef_block2, dst2, size)); + + for (int j = 0; j < block_size; ++j) { + const int diff = dst1[j] - dst2[j]; + const int error = diff * diff; + if (max_error < error) max_error = error; + } + } + + EXPECT_EQ(0, max_error) + << "Error: partial inverse transform produces different results"; +} +#endif +using std::tr1::make_tuple; + +INSTANTIATE_TEST_CASE_P( + C, AV1PartialIDctTest, + ::testing::Values(make_tuple(&aom_fdct32x32_c, &aom_idct32x32_1024_add_c, + &aom_idct32x32_34_add_c, TX_32X32, 34), + make_tuple(&aom_fdct32x32_c, &aom_idct32x32_1024_add_c, + &aom_idct32x32_1_add_c, TX_32X32, 1), + make_tuple(&aom_fdct16x16_c, &aom_idct16x16_256_add_c, + &aom_idct16x16_10_add_c, TX_16X16, 10), + make_tuple(&aom_fdct16x16_c, &aom_idct16x16_256_add_c, + &aom_idct16x16_1_add_c, TX_16X16, 1), + make_tuple(&aom_fdct8x8_c, &aom_idct8x8_64_add_c, + &aom_idct8x8_12_add_c, TX_8X8, 12), + make_tuple(&aom_fdct8x8_c, &aom_idct8x8_64_add_c, + &aom_idct8x8_1_add_c, TX_8X8, 1), + make_tuple(&aom_fdct4x4_c, &aom_idct4x4_16_add_c, + &aom_idct4x4_1_add_c, TX_4X4, 1))); +#endif // CONFIG_AV1_ENCODER +} // namespace diff --git a/third_party/aom/test/av1_quantize_test.cc b/third_party/aom/test/av1_quantize_test.cc new file mode 100644 index 0000000000..b5d1531f5c --- /dev/null +++ b/third_party/aom/test/av1_quantize_test.cc @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "av1/common/scan.h" + +namespace { + +typedef void (*QuantizeFpFunc)( + const tran_low_t *coeff_ptr, intptr_t count, int skip_block, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, int log_scale); + +struct QuantizeFuncParams { + QuantizeFuncParams(QuantizeFpFunc qF = NULL, QuantizeFpFunc qRefF = NULL, + int count = 16) + : qFunc(qF), qFuncRef(qRefF), coeffCount(count) {} + QuantizeFpFunc qFunc; + QuantizeFpFunc qFuncRef; + int coeffCount; +}; + +using libaom_test::ACMRandom; + +const int numTests = 1000; +const int maxSize = 1024; +const int roundFactorRange = 127; +const int dequantRange = 32768; +const int coeffRange = (1 << 20) - 1; + +class AV1QuantizeTest : public ::testing::TestWithParam { + public: + void RunQuantizeTest() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, tran_low_t, coeff_ptr[maxSize]); + DECLARE_ALIGNED(16, int16_t, zbin_ptr[2]); + DECLARE_ALIGNED(16, int16_t, round_ptr[2]); + DECLARE_ALIGNED(16, int16_t, quant_ptr[2]); + DECLARE_ALIGNED(16, int16_t, quant_shift_ptr[2]); + DECLARE_ALIGNED(16, tran_low_t, qcoeff_ptr[maxSize]); + DECLARE_ALIGNED(16, tran_low_t, dqcoeff_ptr[maxSize]); + DECLARE_ALIGNED(16, tran_low_t, ref_qcoeff_ptr[maxSize]); + DECLARE_ALIGNED(16, tran_low_t, ref_dqcoeff_ptr[maxSize]); + DECLARE_ALIGNED(16, int16_t, dequant_ptr[2]); + uint16_t eob; + uint16_t ref_eob; + int err_count_total = 0; + int first_failure = -1; + int skip_block = 0; + int count = params_.coeffCount; + const TX_SIZE txSize = getTxSize(count); + int log_scale = (txSize == TX_32X32); + QuantizeFpFunc quanFunc = params_.qFunc; + QuantizeFpFunc quanFuncRef = params_.qFuncRef; + + const SCAN_ORDER scanOrder = av1_default_scan_orders[txSize]; + for (int i = 0; i < numTests; i++) { + int err_count = 0; + ref_eob = eob = -1; + for (int j = 0; j < count; j++) { + coeff_ptr[j] = rnd(coeffRange); + } + + for (int j = 0; j < 2; j++) { + zbin_ptr[j] = rnd.Rand16(); + quant_shift_ptr[j] = rnd.Rand16(); + // int16_t positive + dequant_ptr[j] = abs(rnd(dequantRange)); + quant_ptr[j] = (1 << 16) / dequant_ptr[j]; + round_ptr[j] = (abs(rnd(roundFactorRange)) * dequant_ptr[j]) >> 7; + } + + quanFuncRef(coeff_ptr, count, skip_block, zbin_ptr, round_ptr, quant_ptr, + quant_shift_ptr, ref_qcoeff_ptr, ref_dqcoeff_ptr, dequant_ptr, + &ref_eob, scanOrder.scan, scanOrder.iscan, log_scale); + + ASM_REGISTER_STATE_CHECK( + quanFunc(coeff_ptr, count, skip_block, zbin_ptr, round_ptr, quant_ptr, + quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr, &eob, + scanOrder.scan, scanOrder.iscan, log_scale)); + + for (int j = 0; j < count; ++j) { + err_count += (ref_qcoeff_ptr[j] != qcoeff_ptr[j]) | + (ref_dqcoeff_ptr[j] != dqcoeff_ptr[j]); + EXPECT_EQ(ref_qcoeff_ptr[j], qcoeff_ptr[j]) << "qcoeff error: i = " << i + << " j = " << j << "\n"; + EXPECT_EQ(ref_dqcoeff_ptr[j], dqcoeff_ptr[j]) + << "dqcoeff error: i = " << i << " j = " << j << "\n"; + } + EXPECT_EQ(ref_eob, eob) << "eob error: " + << "i = " << i << "\n"; + err_count += (ref_eob != eob); + if (err_count && !err_count_total) { + first_failure = i; + } + err_count_total += err_count; + } + EXPECT_EQ(0, err_count_total) + << "Error: Quantization Test, C output doesn't match SSE2 output. " + << "First failed at test case " << first_failure; + } + + void RunEobTest() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, tran_low_t, coeff_ptr[maxSize]); + DECLARE_ALIGNED(16, int16_t, zbin_ptr[2]); + DECLARE_ALIGNED(16, int16_t, round_ptr[2]); + DECLARE_ALIGNED(16, int16_t, quant_ptr[2]); + DECLARE_ALIGNED(16, int16_t, quant_shift_ptr[2]); + DECLARE_ALIGNED(16, tran_low_t, qcoeff_ptr[maxSize]); + DECLARE_ALIGNED(16, tran_low_t, dqcoeff_ptr[maxSize]); + DECLARE_ALIGNED(16, tran_low_t, ref_qcoeff_ptr[maxSize]); + DECLARE_ALIGNED(16, tran_low_t, ref_dqcoeff_ptr[maxSize]); + DECLARE_ALIGNED(16, int16_t, dequant_ptr[2]); + uint16_t eob; + uint16_t ref_eob; + int skip_block = 0; + int count = params_.coeffCount; + const TX_SIZE txSize = getTxSize(count); + int log_scale = (txSize == TX_32X32); + QuantizeFpFunc quanFunc = params_.qFunc; + QuantizeFpFunc quanFuncRef = params_.qFuncRef; + const SCAN_ORDER scanOrder = av1_default_scan_orders[txSize]; + + for (int i = 0; i < numTests; i++) { + ref_eob = eob = -1; + for (int j = 0; j < count; j++) { + coeff_ptr[j] = 0; + } + + coeff_ptr[rnd(count)] = rnd(coeffRange); + coeff_ptr[rnd(count)] = rnd(coeffRange); + coeff_ptr[rnd(count)] = rnd(coeffRange); + + for (int j = 0; j < 2; j++) { + zbin_ptr[j] = rnd.Rand16(); + quant_shift_ptr[j] = rnd.Rand16(); + // int16_t positive + dequant_ptr[j] = abs(rnd(dequantRange)); + quant_ptr[j] = (1 << 16) / dequant_ptr[j]; + round_ptr[j] = (abs(rnd(roundFactorRange)) * dequant_ptr[j]) >> 7; + } + + quanFuncRef(coeff_ptr, count, skip_block, zbin_ptr, round_ptr, quant_ptr, + quant_shift_ptr, ref_qcoeff_ptr, ref_dqcoeff_ptr, dequant_ptr, + &ref_eob, scanOrder.scan, scanOrder.iscan, log_scale); + + ASM_REGISTER_STATE_CHECK( + quanFunc(coeff_ptr, count, skip_block, zbin_ptr, round_ptr, quant_ptr, + quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr, &eob, + scanOrder.scan, scanOrder.iscan, log_scale)); + EXPECT_EQ(ref_eob, eob) << "eob error: " + << "i = " << i << "\n"; + } + } + + virtual void SetUp() { params_ = GetParam(); } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + virtual ~AV1QuantizeTest() {} + + private: + TX_SIZE getTxSize(int count) { + switch (count) { + case 16: return TX_4X4; + case 64: return TX_8X8; + case 256: return TX_16X16; + case 1024: return TX_32X32; + default: return TX_4X4; + } + } + + QuantizeFuncParams params_; +}; + +TEST_P(AV1QuantizeTest, BitExactCheck) { RunQuantizeTest(); } +TEST_P(AV1QuantizeTest, EobVerify) { RunEobTest(); } + +#if HAVE_SSE4_1 +#if !CONFIG_AOM_QM +INSTANTIATE_TEST_CASE_P( + SSE4_1, AV1QuantizeTest, + ::testing::Values(QuantizeFuncParams(&av1_highbd_quantize_fp_sse4_1, + &av1_highbd_quantize_fp_c, 16), + QuantizeFuncParams(&av1_highbd_quantize_fp_sse4_1, + &av1_highbd_quantize_fp_c, 64), + QuantizeFuncParams(&av1_highbd_quantize_fp_sse4_1, + &av1_highbd_quantize_fp_c, 256), + QuantizeFuncParams(&av1_highbd_quantize_fp_sse4_1, + &av1_highbd_quantize_fp_c, 1024))); +#endif // !CONFIG_AOM_QM +#endif // HAVE_SSE4_1 +} // namespace diff --git a/third_party/aom/test/av1_txfm_test.cc b/third_party/aom/test/av1_txfm_test.cc new file mode 100644 index 0000000000..1e473b3044 --- /dev/null +++ b/third_party/aom/test/av1_txfm_test.cc @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "test/av1_txfm_test.h" + +namespace libaom_test { + +int get_txfm1d_size(TX_SIZE tx_size) { return tx_size_wide[tx_size]; } + +void get_txfm1d_type(TX_TYPE txfm2d_type, TYPE_TXFM *type0, TYPE_TXFM *type1) { + switch (txfm2d_type) { + case DCT_DCT: + *type0 = TYPE_DCT; + *type1 = TYPE_DCT; + break; + case ADST_DCT: + *type0 = TYPE_ADST; + *type1 = TYPE_DCT; + break; + case DCT_ADST: + *type0 = TYPE_DCT; + *type1 = TYPE_ADST; + break; + case ADST_ADST: + *type0 = TYPE_ADST; + *type1 = TYPE_ADST; + break; +#if CONFIG_EXT_TX + case FLIPADST_DCT: + *type0 = TYPE_ADST; + *type1 = TYPE_DCT; + break; + case DCT_FLIPADST: + *type0 = TYPE_DCT; + *type1 = TYPE_ADST; + break; + case FLIPADST_FLIPADST: + *type0 = TYPE_ADST; + *type1 = TYPE_ADST; + break; + case ADST_FLIPADST: + *type0 = TYPE_ADST; + *type1 = TYPE_ADST; + break; + case FLIPADST_ADST: + *type0 = TYPE_ADST; + *type1 = TYPE_ADST; + break; +#endif // CONFIG_EXT_TX + default: + *type0 = TYPE_DCT; + *type1 = TYPE_DCT; + assert(0); + break; + } +} + +double invSqrt2 = 1 / pow(2, 0.5); + +void reference_dct_1d(const double *in, double *out, int size) { + for (int k = 0; k < size; ++k) { + out[k] = 0; + for (int n = 0; n < size; ++n) { + out[k] += in[n] * cos(M_PI * (2 * n + 1) * k / (2 * size)); + } + if (k == 0) out[k] = out[k] * invSqrt2; + } +} + +void reference_adst_1d(const double *in, double *out, int size) { + for (int k = 0; k < size; ++k) { + out[k] = 0; + for (int n = 0; n < size; ++n) { + out[k] += in[n] * sin(M_PI * (2 * n + 1) * (2 * k + 1) / (4 * size)); + } + } +} + +void reference_hybrid_1d(double *in, double *out, int size, int type) { + if (type == TYPE_DCT) + reference_dct_1d(in, out, size); + else + reference_adst_1d(in, out, size); +} + +void reference_hybrid_2d(double *in, double *out, int size, int type0, + int type1) { + double *tempOut = new double[size * size]; + + for (int r = 0; r < size; r++) { + // out ->tempOut + for (int c = 0; c < size; c++) { + tempOut[r * size + c] = in[c * size + r]; + } + } + + // dct each row: in -> out + for (int r = 0; r < size; r++) { + reference_hybrid_1d(tempOut + r * size, out + r * size, size, type0); + } + + for (int r = 0; r < size; r++) { + // out ->tempOut + for (int c = 0; c < size; c++) { + tempOut[r * size + c] = out[c * size + r]; + } + } + + for (int r = 0; r < size; r++) { + reference_hybrid_1d(tempOut + r * size, out + r * size, size, type1); + } + delete[] tempOut; +} + +template +void fliplr(Type *dest, int stride, int length) { + int i, j; + for (i = 0; i < length; ++i) { + for (j = 0; j < length / 2; ++j) { + const Type tmp = dest[i * stride + j]; + dest[i * stride + j] = dest[i * stride + length - 1 - j]; + dest[i * stride + length - 1 - j] = tmp; + } + } +} + +template +void flipud(Type *dest, int stride, int length) { + int i, j; + for (j = 0; j < length; ++j) { + for (i = 0; i < length / 2; ++i) { + const Type tmp = dest[i * stride + j]; + dest[i * stride + j] = dest[(length - 1 - i) * stride + j]; + dest[(length - 1 - i) * stride + j] = tmp; + } + } +} + +template +void fliplrud(Type *dest, int stride, int length) { + int i, j; + for (i = 0; i < length / 2; ++i) { + for (j = 0; j < length; ++j) { + const Type tmp = dest[i * stride + j]; + dest[i * stride + j] = dest[(length - 1 - i) * stride + length - 1 - j]; + dest[(length - 1 - i) * stride + length - 1 - j] = tmp; + } + } +} + +template void fliplr(double *dest, int stride, int length); +template void flipud(double *dest, int stride, int length); +template void fliplrud(double *dest, int stride, int length); + +} // namespace libaom_test diff --git a/third_party/aom/test/av1_txfm_test.h b/third_party/aom/test/av1_txfm_test.h new file mode 100644 index 0000000000..70f971d090 --- /dev/null +++ b/third_party/aom/test/av1_txfm_test.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_TXFM_TEST_H_ +#define AV1_TXFM_TEST_H_ + +#include +#include +#ifdef _MSC_VER +#define _USE_MATH_DEFINES +#endif +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/acm_random.h" +#include "av1/common/enums.h" +#include "av1/common/av1_txfm.h" +#include "./av1_rtcd.h" + +namespace libaom_test { +typedef enum { + TYPE_DCT = 0, + TYPE_ADST, + TYPE_IDCT, + TYPE_IADST, + TYPE_LAST +} TYPE_TXFM; + +int get_txfm1d_size(TX_SIZE tx_size); + +void get_txfm1d_type(TX_TYPE txfm2d_type, TYPE_TXFM *type0, TYPE_TXFM *type1); + +void reference_dct_1d(const double *in, double *out, int size); + +void reference_adst_1d(const double *in, double *out, int size); + +void reference_hybrid_1d(double *in, double *out, int size, int type); + +void reference_hybrid_2d(double *in, double *out, int size, int type0, + int type1); +template +static double compute_avg_abs_error(const Type1 *a, const Type2 *b, + const int size) { + double error = 0; + for (int i = 0; i < size; i++) { + error += fabs(static_cast(a[i]) - static_cast(b[i])); + } + error = error / size; + return error; +} + +template +void fliplr(Type *dest, int stride, int length); + +template +void flipud(Type *dest, int stride, int length); + +template +void fliplrud(Type *dest, int stride, int length); + +typedef void (*TxfmFunc)(const int32_t *in, int32_t *out, const int8_t *cos_bit, + const int8_t *range_bit); + +typedef void (*Fwd_Txfm2d_Func)(const int16_t *, int32_t *, int, int, int); +typedef void (*Inv_Txfm2d_Func)(const int32_t *, uint16_t *, int, int, int); + +static const int bd = 10; +static const int input_base = (1 << bd); + +#if CONFIG_HIGHBITDEPTH +#if CONFIG_AV1_ENCODER +static const Fwd_Txfm2d_Func fwd_txfm_func_ls[TX_SIZES] = { +#if CONFIG_CB4X4 + NULL, +#endif + av1_fwd_txfm2d_4x4_c, av1_fwd_txfm2d_8x8_c, av1_fwd_txfm2d_16x16_c, + av1_fwd_txfm2d_32x32_c +}; +#endif + +static const Inv_Txfm2d_Func inv_txfm_func_ls[TX_SIZES] = { +#if CONFIG_CB4X4 + NULL, +#endif + av1_inv_txfm2d_add_4x4_c, av1_inv_txfm2d_add_8x8_c, + av1_inv_txfm2d_add_16x16_c, av1_inv_txfm2d_add_32x32_c +}; +#endif // CONFIG_HIGHBITDEPTH + +} // namespace libaom_test +#endif // AV1_TXFM_TEST_H_ diff --git a/third_party/aom/test/av1_wedge_utils_test.cc b/third_party/aom/test/av1_wedge_utils_test.cc new file mode 100644 index 0000000000..d4b560fc10 --- /dev/null +++ b/third_party/aom/test/av1_wedge_utils_test.cc @@ -0,0 +1,383 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" + +#include "./aom_dsp_rtcd.h" +#include "./av1_rtcd.h" + +#include "aom_dsp/aom_dsp_common.h" + +#include "av1/common/enums.h" + +#include "test/acm_random.h" +#include "test/function_equivalence_test.h" +#include "test/register_state_check.h" + +#define WEDGE_WEIGHT_BITS 6 +#define MAX_MASK_VALUE (1 << (WEDGE_WEIGHT_BITS)) + +using libaom_test::ACMRandom; +using libaom_test::FunctionEquivalenceTest; + +namespace { + +static const int16_t kInt13Max = (1 << 12) - 1; + +////////////////////////////////////////////////////////////////////////////// +// av1_wedge_sse_from_residuals - functionality +////////////////////////////////////////////////////////////////////////////// + +class WedgeUtilsSSEFuncTest : public testing::Test { + protected: + WedgeUtilsSSEFuncTest() : rng_(ACMRandom::DeterministicSeed()) {} + + static const int kIterations = 1000; + + ACMRandom rng_; +}; + +static void equiv_blend_residuals(int16_t *r, const int16_t *r0, + const int16_t *r1, const uint8_t *m, int N) { + for (int i = 0; i < N; i++) { + const int32_t m0 = m[i]; + const int32_t m1 = MAX_MASK_VALUE - m0; + const int16_t R = m0 * r0[i] + m1 * r1[i]; + // Note that this rounding is designed to match the result + // you would get when actually blending the 2 predictors and computing + // the residuals. + r[i] = ROUND_POWER_OF_TWO(R - 1, WEDGE_WEIGHT_BITS); + } +} + +static uint64_t equiv_sse_from_residuals(const int16_t *r0, const int16_t *r1, + const uint8_t *m, int N) { + uint64_t acc = 0; + for (int i = 0; i < N; i++) { + const int32_t m0 = m[i]; + const int32_t m1 = MAX_MASK_VALUE - m0; + const int16_t R = m0 * r0[i] + m1 * r1[i]; + const int32_t r = ROUND_POWER_OF_TWO(R - 1, WEDGE_WEIGHT_BITS); + acc += r * r; + } + return acc; +} + +TEST_F(WedgeUtilsSSEFuncTest, ResidualBlendingEquiv) { + DECLARE_ALIGNED(32, uint8_t, s[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, uint8_t, p0[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, uint8_t, p1[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, uint8_t, p[MAX_SB_SQUARE]); + + DECLARE_ALIGNED(32, int16_t, r0[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, r1[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, r_ref[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, r_tst[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, uint8_t, m[MAX_SB_SQUARE]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + s[i] = rng_.Rand8(); + m[i] = rng_(MAX_MASK_VALUE + 1); + } + + const int w = 1 << (rng_(MAX_SB_SIZE_LOG2 + 1 - 3) + 3); + const int h = 1 << (rng_(MAX_SB_SIZE_LOG2 + 1 - 3) + 3); + const int N = w * h; + + for (int j = 0; j < N; j++) { + p0[j] = clamp(s[j] + rng_(33) - 16, 0, UINT8_MAX); + p1[j] = clamp(s[j] + rng_(33) - 16, 0, UINT8_MAX); + } + + aom_blend_a64_mask(p, w, p0, w, p1, w, m, w, h, w, 0, 0); + + aom_subtract_block(h, w, r0, w, s, w, p0, w); + aom_subtract_block(h, w, r1, w, s, w, p1, w); + + aom_subtract_block(h, w, r_ref, w, s, w, p, w); + equiv_blend_residuals(r_tst, r0, r1, m, N); + + for (int i = 0; i < N; ++i) ASSERT_EQ(r_ref[i], r_tst[i]); + + uint64_t ref_sse = aom_sum_squares_i16(r_ref, N); + uint64_t tst_sse = equiv_sse_from_residuals(r0, r1, m, N); + + ASSERT_EQ(ref_sse, tst_sse); + } +} + +static uint64_t sse_from_residuals(const int16_t *r0, const int16_t *r1, + const uint8_t *m, int N) { + uint64_t acc = 0; + for (int i = 0; i < N; i++) { + const int32_t m0 = m[i]; + const int32_t m1 = MAX_MASK_VALUE - m0; + const int32_t r = m0 * r0[i] + m1 * r1[i]; + acc += r * r; + } + return ROUND_POWER_OF_TWO(acc, 2 * WEDGE_WEIGHT_BITS); +} + +TEST_F(WedgeUtilsSSEFuncTest, ResidualBlendingMethod) { + DECLARE_ALIGNED(32, int16_t, r0[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, r1[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, d[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, uint8_t, m[MAX_SB_SQUARE]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + r1[i] = rng_(2 * INT8_MAX - 2 * INT8_MIN + 1) + 2 * INT8_MIN; + d[i] = rng_(2 * INT8_MAX - 2 * INT8_MIN + 1) + 2 * INT8_MIN; + m[i] = rng_(MAX_MASK_VALUE + 1); + } + + const int N = 64 * (rng_(MAX_SB_SQUARE / 64) + 1); + + for (int i = 0; i < N; i++) r0[i] = r1[i] + d[i]; + + const uint64_t ref_res = sse_from_residuals(r0, r1, m, N); + const uint64_t tst_res = av1_wedge_sse_from_residuals(r1, d, m, N); + + ASSERT_EQ(ref_res, tst_res); + } +} + +////////////////////////////////////////////////////////////////////////////// +// av1_wedge_sse_from_residuals - optimizations +////////////////////////////////////////////////////////////////////////////// + +typedef uint64_t (*FSSE)(const int16_t *r1, const int16_t *d, const uint8_t *m, + int N); +typedef libaom_test::FuncParam TestFuncsFSSE; + +class WedgeUtilsSSEOptTest : public FunctionEquivalenceTest { + protected: + static const int kIterations = 10000; +}; + +TEST_P(WedgeUtilsSSEOptTest, RandomValues) { + DECLARE_ALIGNED(32, int16_t, r1[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, d[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, uint8_t, m[MAX_SB_SQUARE]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + r1[i] = rng_(2 * kInt13Max + 1) - kInt13Max; + d[i] = rng_(2 * kInt13Max + 1) - kInt13Max; + m[i] = rng_(MAX_MASK_VALUE + 1); + } + + const int N = 64 * (rng_(MAX_SB_SQUARE / 64) + 1); + + const uint64_t ref_res = params_.ref_func(r1, d, m, N); + uint64_t tst_res; + ASM_REGISTER_STATE_CHECK(tst_res = params_.tst_func(r1, d, m, N)); + + ASSERT_EQ(ref_res, tst_res); + } +} + +TEST_P(WedgeUtilsSSEOptTest, ExtremeValues) { + DECLARE_ALIGNED(32, int16_t, r1[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, d[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, uint8_t, m[MAX_SB_SQUARE]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + if (rng_(2)) { + for (int i = 0; i < MAX_SB_SQUARE; ++i) r1[i] = kInt13Max; + } else { + for (int i = 0; i < MAX_SB_SQUARE; ++i) r1[i] = -kInt13Max; + } + + if (rng_(2)) { + for (int i = 0; i < MAX_SB_SQUARE; ++i) d[i] = kInt13Max; + } else { + for (int i = 0; i < MAX_SB_SQUARE; ++i) d[i] = -kInt13Max; + } + + for (int i = 0; i < MAX_SB_SQUARE; ++i) m[i] = MAX_MASK_VALUE; + + const int N = 64 * (rng_(MAX_SB_SQUARE / 64) + 1); + + const uint64_t ref_res = params_.ref_func(r1, d, m, N); + uint64_t tst_res; + ASM_REGISTER_STATE_CHECK(tst_res = params_.tst_func(r1, d, m, N)); + + ASSERT_EQ(ref_res, tst_res); + } +} + +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P( + SSE2, WedgeUtilsSSEOptTest, + ::testing::Values(TestFuncsFSSE(av1_wedge_sse_from_residuals_c, + av1_wedge_sse_from_residuals_sse2))); + +#endif // HAVE_SSE2 + +////////////////////////////////////////////////////////////////////////////// +// av1_wedge_sign_from_residuals +////////////////////////////////////////////////////////////////////////////// + +typedef int (*FSign)(const int16_t *ds, const uint8_t *m, int N, int64_t limit); +typedef libaom_test::FuncParam TestFuncsFSign; + +class WedgeUtilsSignOptTest : public FunctionEquivalenceTest { + protected: + static const int kIterations = 10000; + static const int kMaxSize = 8196; // Size limited by SIMD implementation. +}; + +TEST_P(WedgeUtilsSignOptTest, RandomValues) { + DECLARE_ALIGNED(32, int16_t, r0[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, r1[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, ds[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, uint8_t, m[MAX_SB_SQUARE]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + r0[i] = rng_(2 * kInt13Max + 1) - kInt13Max; + r1[i] = rng_(2 * kInt13Max + 1) - kInt13Max; + m[i] = rng_(MAX_MASK_VALUE + 1); + } + + const int maxN = AOMMIN(kMaxSize, MAX_SB_SQUARE); + const int N = 64 * (rng_(maxN / 64 - 1) + 1); + + int64_t limit; + limit = (int64_t)aom_sum_squares_i16(r0, N); + limit -= (int64_t)aom_sum_squares_i16(r1, N); + limit *= (1 << WEDGE_WEIGHT_BITS) / 2; + + for (int i = 0; i < N; i++) + ds[i] = clamp(r0[i] * r0[i] - r1[i] * r1[i], INT16_MIN, INT16_MAX); + + const int ref_res = params_.ref_func(ds, m, N, limit); + int tst_res; + ASM_REGISTER_STATE_CHECK(tst_res = params_.tst_func(ds, m, N, limit)); + + ASSERT_EQ(ref_res, tst_res); + } +} + +TEST_P(WedgeUtilsSignOptTest, ExtremeValues) { + DECLARE_ALIGNED(32, int16_t, r0[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, r1[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, ds[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, uint8_t, m[MAX_SB_SQUARE]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + switch (rng_(4)) { + case 0: + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + r0[i] = 0; + r1[i] = kInt13Max; + } + break; + case 1: + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + r0[i] = kInt13Max; + r1[i] = 0; + } + break; + case 2: + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + r0[i] = 0; + r1[i] = -kInt13Max; + } + break; + default: + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + r0[i] = -kInt13Max; + r1[i] = 0; + } + break; + } + + for (int i = 0; i < MAX_SB_SQUARE; ++i) m[i] = MAX_MASK_VALUE; + + const int maxN = AOMMIN(kMaxSize, MAX_SB_SQUARE); + const int N = 64 * (rng_(maxN / 64 - 1) + 1); + + int64_t limit; + limit = (int64_t)aom_sum_squares_i16(r0, N); + limit -= (int64_t)aom_sum_squares_i16(r1, N); + limit *= (1 << WEDGE_WEIGHT_BITS) / 2; + + for (int i = 0; i < N; i++) + ds[i] = clamp(r0[i] * r0[i] - r1[i] * r1[i], INT16_MIN, INT16_MAX); + + const int ref_res = params_.ref_func(ds, m, N, limit); + int tst_res; + ASM_REGISTER_STATE_CHECK(tst_res = params_.tst_func(ds, m, N, limit)); + + ASSERT_EQ(ref_res, tst_res); + } +} + +#if HAVE_SSE2 + +INSTANTIATE_TEST_CASE_P( + SSE2, WedgeUtilsSignOptTest, + ::testing::Values(TestFuncsFSign(av1_wedge_sign_from_residuals_c, + av1_wedge_sign_from_residuals_sse2))); + +#endif // HAVE_SSE2 + +////////////////////////////////////////////////////////////////////////////// +// av1_wedge_compute_delta_squares +////////////////////////////////////////////////////////////////////////////// + +typedef void (*FDS)(int16_t *d, const int16_t *a, const int16_t *b, int N); +typedef libaom_test::FuncParam TestFuncsFDS; + +class WedgeUtilsDeltaSquaresOptTest : public FunctionEquivalenceTest { + protected: + static const int kIterations = 10000; +}; + +TEST_P(WedgeUtilsDeltaSquaresOptTest, RandomValues) { + DECLARE_ALIGNED(32, int16_t, a[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, b[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, d_ref[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int16_t, d_tst[MAX_SB_SQUARE]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + a[i] = rng_.Rand16(); + b[i] = rng_(2 * INT16_MAX + 1) - INT16_MAX; + } + + const int N = 64 * (rng_(MAX_SB_SQUARE / 64) + 1); + + memset(&d_ref, INT16_MAX, sizeof(d_ref)); + memset(&d_tst, INT16_MAX, sizeof(d_tst)); + + params_.ref_func(d_ref, a, b, N); + ASM_REGISTER_STATE_CHECK(params_.tst_func(d_tst, a, b, N)); + + for (int i = 0; i < MAX_SB_SQUARE; ++i) ASSERT_EQ(d_ref[i], d_tst[i]); + } +} + +#if HAVE_SSE2 + +INSTANTIATE_TEST_CASE_P( + SSE2, WedgeUtilsDeltaSquaresOptTest, + ::testing::Values(TestFuncsFDS(av1_wedge_compute_delta_squares_c, + av1_wedge_compute_delta_squares_sse2))); + +#endif // HAVE_SSE2 + +} // namespace diff --git a/third_party/aom/test/avg_test.cc b/third_party/aom/test/avg_test.cc new file mode 100644 index 0000000000..b040f6a34c --- /dev/null +++ b/third_party/aom/test/avg_test.cc @@ -0,0 +1,396 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" + +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "aom_mem/aom_mem.h" + +using libaom_test::ACMRandom; + +namespace { +class AverageTestBase : public ::testing::Test { + public: + AverageTestBase(int width, int height) : width_(width), height_(height) {} + + static void SetUpTestCase() { + source_data_ = reinterpret_cast( + aom_memalign(kDataAlignment, kDataBlockSize)); + } + + static void TearDownTestCase() { + aom_free(source_data_); + source_data_ = NULL; + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + // Handle blocks up to 4 blocks 64x64 with stride up to 128 + static const int kDataAlignment = 16; + static const int kDataBlockSize = 64 * 128; + + virtual void SetUp() { + source_stride_ = (width_ + 31) & ~31; + rnd_.Reset(ACMRandom::DeterministicSeed()); + } + + // Sum Pixels + static unsigned int ReferenceAverage8x8(const uint8_t *source, int pitch) { + unsigned int average = 0; + for (int h = 0; h < 8; ++h) + for (int w = 0; w < 8; ++w) average += source[h * pitch + w]; + return ((average + 32) >> 6); + } + + static unsigned int ReferenceAverage4x4(const uint8_t *source, int pitch) { + unsigned int average = 0; + for (int h = 0; h < 4; ++h) + for (int w = 0; w < 4; ++w) average += source[h * pitch + w]; + return ((average + 8) >> 4); + } + + void FillConstant(uint8_t fill_constant) { + for (int i = 0; i < width_ * height_; ++i) { + source_data_[i] = fill_constant; + } + } + + void FillRandom() { + for (int i = 0; i < width_ * height_; ++i) { + source_data_[i] = rnd_.Rand8(); + } + } + + int width_, height_; + static uint8_t *source_data_; + int source_stride_; + + ACMRandom rnd_; +}; +typedef unsigned int (*AverageFunction)(const uint8_t *s, int pitch); + +typedef std::tr1::tuple AvgFunc; + +class AverageTest : public AverageTestBase, + public ::testing::WithParamInterface { + public: + AverageTest() : AverageTestBase(GET_PARAM(0), GET_PARAM(1)) {} + + protected: + void CheckAverages() { + const int block_size = GET_PARAM(3); + unsigned int expected = 0; + if (block_size == 8) { + expected = + ReferenceAverage8x8(source_data_ + GET_PARAM(2), source_stride_); + } else if (block_size == 4) { + expected = + ReferenceAverage4x4(source_data_ + GET_PARAM(2), source_stride_); + } + + ASM_REGISTER_STATE_CHECK( + GET_PARAM(4)(source_data_ + GET_PARAM(2), source_stride_)); + unsigned int actual = + GET_PARAM(4)(source_data_ + GET_PARAM(2), source_stride_); + + EXPECT_EQ(expected, actual); + } +}; + +typedef void (*IntProRowFunc)(int16_t hbuf[16], uint8_t const *ref, + const int ref_stride, const int height); + +typedef std::tr1::tuple IntProRowParam; + +class IntProRowTest : public AverageTestBase, + public ::testing::WithParamInterface { + public: + IntProRowTest() + : AverageTestBase(16, GET_PARAM(0)), hbuf_asm_(NULL), hbuf_c_(NULL) { + asm_func_ = GET_PARAM(1); + c_func_ = GET_PARAM(2); + } + + protected: + virtual void SetUp() { + hbuf_asm_ = reinterpret_cast( + aom_memalign(kDataAlignment, sizeof(*hbuf_asm_) * 16)); + hbuf_c_ = reinterpret_cast( + aom_memalign(kDataAlignment, sizeof(*hbuf_c_) * 16)); + } + + virtual void TearDown() { + aom_free(hbuf_c_); + hbuf_c_ = NULL; + aom_free(hbuf_asm_); + hbuf_asm_ = NULL; + } + + void RunComparison() { + ASM_REGISTER_STATE_CHECK(c_func_(hbuf_c_, source_data_, 0, height_)); + ASM_REGISTER_STATE_CHECK(asm_func_(hbuf_asm_, source_data_, 0, height_)); + EXPECT_EQ(0, memcmp(hbuf_c_, hbuf_asm_, sizeof(*hbuf_c_) * 16)) + << "Output mismatch"; + } + + private: + IntProRowFunc asm_func_; + IntProRowFunc c_func_; + int16_t *hbuf_asm_; + int16_t *hbuf_c_; +}; + +typedef int16_t (*IntProColFunc)(uint8_t const *ref, const int width); + +typedef std::tr1::tuple IntProColParam; + +class IntProColTest : public AverageTestBase, + public ::testing::WithParamInterface { + public: + IntProColTest() : AverageTestBase(GET_PARAM(0), 1), sum_asm_(0), sum_c_(0) { + asm_func_ = GET_PARAM(1); + c_func_ = GET_PARAM(2); + } + + protected: + void RunComparison() { + ASM_REGISTER_STATE_CHECK(sum_c_ = c_func_(source_data_, width_)); + ASM_REGISTER_STATE_CHECK(sum_asm_ = asm_func_(source_data_, width_)); + EXPECT_EQ(sum_c_, sum_asm_) << "Output mismatch"; + } + + private: + IntProColFunc asm_func_; + IntProColFunc c_func_; + int16_t sum_asm_; + int16_t sum_c_; +}; + +typedef int (*SatdFunc)(const int16_t *coeffs, int length); +typedef std::tr1::tuple SatdTestParam; + +class SatdTest : public ::testing::Test, + public ::testing::WithParamInterface { + protected: + virtual void SetUp() { + satd_size_ = GET_PARAM(0); + satd_func_ = GET_PARAM(1); + rnd_.Reset(ACMRandom::DeterministicSeed()); + src_ = reinterpret_cast( + aom_memalign(16, sizeof(*src_) * satd_size_)); + ASSERT_TRUE(src_ != NULL); + } + + virtual void TearDown() { + libaom_test::ClearSystemState(); + aom_free(src_); + } + + void FillConstant(const int16_t val) { + for (int i = 0; i < satd_size_; ++i) src_[i] = val; + } + + void FillRandom() { + for (int i = 0; i < satd_size_; ++i) src_[i] = rnd_.Rand16(); + } + + void Check(int expected) { + int total; + ASM_REGISTER_STATE_CHECK(total = satd_func_(src_, satd_size_)); + EXPECT_EQ(expected, total); + } + + int satd_size_; + + private: + int16_t *src_; + SatdFunc satd_func_; + ACMRandom rnd_; +}; + +uint8_t *AverageTestBase::source_data_ = NULL; + +TEST_P(AverageTest, MinValue) { + FillConstant(0); + CheckAverages(); +} + +TEST_P(AverageTest, MaxValue) { + FillConstant(255); + CheckAverages(); +} + +TEST_P(AverageTest, Random) { + // The reference frame, but not the source frame, may be unaligned for + // certain types of searches. + for (int i = 0; i < 1000; i++) { + FillRandom(); + CheckAverages(); + } +} + +TEST_P(IntProRowTest, MinValue) { + FillConstant(0); + RunComparison(); +} + +TEST_P(IntProRowTest, MaxValue) { + FillConstant(255); + RunComparison(); +} + +TEST_P(IntProRowTest, Random) { + FillRandom(); + RunComparison(); +} + +TEST_P(IntProColTest, MinValue) { + FillConstant(0); + RunComparison(); +} + +TEST_P(IntProColTest, MaxValue) { + FillConstant(255); + RunComparison(); +} + +TEST_P(IntProColTest, Random) { + FillRandom(); + RunComparison(); +} + +TEST_P(SatdTest, MinValue) { + const int kMin = -32640; + const int expected = -kMin * satd_size_; + FillConstant(kMin); + Check(expected); +} + +TEST_P(SatdTest, MaxValue) { + const int kMax = 32640; + const int expected = kMax * satd_size_; + FillConstant(kMax); + Check(expected); +} + +TEST_P(SatdTest, Random) { + int expected; + switch (satd_size_) { + case 16: expected = 205298; break; + case 64: expected = 1113950; break; + case 256: expected = 4268415; break; + case 1024: expected = 16954082; break; + default: + FAIL() << "Invalid satd size (" << satd_size_ + << ") valid: 16/64/256/1024"; + } + FillRandom(); + Check(expected); +} + +using std::tr1::make_tuple; + +INSTANTIATE_TEST_CASE_P( + C, AverageTest, + ::testing::Values(make_tuple(16, 16, 1, 8, &aom_avg_8x8_c), + make_tuple(16, 16, 1, 4, &aom_avg_4x4_c))); + +INSTANTIATE_TEST_CASE_P(C, SatdTest, + ::testing::Values(make_tuple(16, &aom_satd_c), + make_tuple(64, &aom_satd_c), + make_tuple(256, &aom_satd_c), + make_tuple(1024, &aom_satd_c))); + +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P( + SSE2, AverageTest, + ::testing::Values(make_tuple(16, 16, 0, 8, &aom_avg_8x8_sse2), + make_tuple(16, 16, 5, 8, &aom_avg_8x8_sse2), + make_tuple(32, 32, 15, 8, &aom_avg_8x8_sse2), + make_tuple(16, 16, 0, 4, &aom_avg_4x4_sse2), + make_tuple(16, 16, 5, 4, &aom_avg_4x4_sse2), + make_tuple(32, 32, 15, 4, &aom_avg_4x4_sse2))); + +INSTANTIATE_TEST_CASE_P( + SSE2, IntProRowTest, + ::testing::Values(make_tuple(16, &aom_int_pro_row_sse2, &aom_int_pro_row_c), + make_tuple(32, &aom_int_pro_row_sse2, &aom_int_pro_row_c), + make_tuple(64, &aom_int_pro_row_sse2, + &aom_int_pro_row_c))); + +INSTANTIATE_TEST_CASE_P( + SSE2, IntProColTest, + ::testing::Values(make_tuple(16, &aom_int_pro_col_sse2, &aom_int_pro_col_c), + make_tuple(32, &aom_int_pro_col_sse2, &aom_int_pro_col_c), + make_tuple(64, &aom_int_pro_col_sse2, + &aom_int_pro_col_c))); + +INSTANTIATE_TEST_CASE_P(SSE2, SatdTest, + ::testing::Values(make_tuple(16, &aom_satd_sse2), + make_tuple(64, &aom_satd_sse2), + make_tuple(256, &aom_satd_sse2), + make_tuple(1024, &aom_satd_sse2))); +#endif + +#if HAVE_NEON +INSTANTIATE_TEST_CASE_P( + NEON, AverageTest, + ::testing::Values(make_tuple(16, 16, 0, 8, &aom_avg_8x8_neon), + make_tuple(16, 16, 5, 8, &aom_avg_8x8_neon), + make_tuple(32, 32, 15, 8, &aom_avg_8x8_neon), + make_tuple(16, 16, 0, 4, &aom_avg_4x4_neon), + make_tuple(16, 16, 5, 4, &aom_avg_4x4_neon), + make_tuple(32, 32, 15, 4, &aom_avg_4x4_neon))); + +INSTANTIATE_TEST_CASE_P( + NEON, IntProRowTest, + ::testing::Values(make_tuple(16, &aom_int_pro_row_neon, &aom_int_pro_row_c), + make_tuple(32, &aom_int_pro_row_neon, &aom_int_pro_row_c), + make_tuple(64, &aom_int_pro_row_neon, + &aom_int_pro_row_c))); + +INSTANTIATE_TEST_CASE_P( + NEON, IntProColTest, + ::testing::Values(make_tuple(16, &aom_int_pro_col_neon, &aom_int_pro_col_c), + make_tuple(32, &aom_int_pro_col_neon, &aom_int_pro_col_c), + make_tuple(64, &aom_int_pro_col_neon, + &aom_int_pro_col_c))); + +INSTANTIATE_TEST_CASE_P(NEON, SatdTest, + ::testing::Values(make_tuple(16, &aom_satd_neon), + make_tuple(64, &aom_satd_neon), + make_tuple(256, &aom_satd_neon), + make_tuple(1024, &aom_satd_neon))); +#endif + +#if HAVE_MSA +INSTANTIATE_TEST_CASE_P( + MSA, AverageTest, + ::testing::Values(make_tuple(16, 16, 0, 8, &aom_avg_8x8_msa), + make_tuple(16, 16, 5, 8, &aom_avg_8x8_msa), + make_tuple(32, 32, 15, 8, &aom_avg_8x8_msa), + make_tuple(16, 16, 0, 4, &aom_avg_4x4_msa), + make_tuple(16, 16, 5, 4, &aom_avg_4x4_msa), + make_tuple(32, 32, 15, 4, &aom_avg_4x4_msa))); +#endif + +} // namespace diff --git a/third_party/aom/test/binary_codes_test.cc b/third_party/aom/test/binary_codes_test.cc new file mode 100644 index 0000000000..385ec76871 --- /dev/null +++ b/third_party/aom/test/binary_codes_test.cc @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/acm_random.h" +#include "aom/aom_integer.h" +#include "aom_dsp/bitreader.h" +#include "aom_dsp/bitwriter.h" +#include "aom_dsp/binary_codes_reader.h" +#include "aom_dsp/binary_codes_writer.h" + +using libaom_test::ACMRandom; + +namespace { + +// Test for Bilevel code with reference +TEST(AV1, TestPrimitiveRefbilivel) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int kBufferSize = 65536; + aom_writer bw; + uint8_t bw_buffer[kBufferSize]; + const uint16_t kRanges = 8; + const uint16_t kNearRanges = 8; + const uint16_t kReferences = 8; + const uint16_t kValues = 16; + const uint16_t range_vals[kRanges] = { 1, 13, 64, 120, 230, 420, 1100, 8000 }; + uint16_t enc_values[kRanges][kNearRanges][kReferences][kValues][4]; + aom_start_encode(&bw, bw_buffer); + for (int n = 0; n < kRanges; ++n) { + const uint16_t range = range_vals[n]; + for (int p = 0; p < kNearRanges; ++p) { + const uint16_t near_range = 1 + rnd(range); + for (int r = 0; r < kReferences; ++r) { + const uint16_t ref = rnd(range); + for (int v = 0; v < kValues; ++v) { + const uint16_t value = rnd(range); + enc_values[n][p][r][v][0] = range; + enc_values[n][p][r][v][1] = near_range; + enc_values[n][p][r][v][2] = ref; + enc_values[n][p][r][v][3] = value; + aom_write_primitive_refbilevel(&bw, range, near_range, ref, value); + } + } + } + } + aom_stop_encode(&bw); + aom_reader br; + aom_reader_init(&br, bw_buffer, bw.pos, NULL, NULL); + GTEST_ASSERT_GE(aom_reader_tell(&br), 0u); + GTEST_ASSERT_LE(aom_reader_tell(&br), 1u); + for (int n = 0; n < kRanges; ++n) { + for (int p = 0; p < kNearRanges; ++p) { + for (int r = 0; r < kReferences; ++r) { + for (int v = 0; v < kValues; ++v) { + const uint16_t range = enc_values[n][p][r][v][0]; + const uint16_t near_range = enc_values[n][p][r][v][1]; + const uint16_t ref = enc_values[n][p][r][v][2]; + const uint16_t value = + aom_read_primitive_refbilevel(&br, range, near_range, ref); + GTEST_ASSERT_EQ(value, enc_values[n][p][r][v][3]); + } + } + } + } +} + +// Test for Finite subexponential code with reference +TEST(AV1, TestPrimitiveRefsubexpfin) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int kBufferSize = 65536; + aom_writer bw; + uint8_t bw_buffer[kBufferSize]; + const uint16_t kRanges = 8; + const uint16_t kSubexpParams = 6; + const uint16_t kReferences = 8; + const uint16_t kValues = 16; + uint16_t enc_values[kRanges][kSubexpParams][kReferences][kValues][4]; + const uint16_t range_vals[kRanges] = { 1, 13, 64, 120, 230, 420, 1100, 8000 }; + aom_start_encode(&bw, bw_buffer); + for (int n = 0; n < kRanges; ++n) { + const uint16_t range = range_vals[n]; + for (int k = 0; k < kSubexpParams; ++k) { + for (int r = 0; r < kReferences; ++r) { + const uint16_t ref = rnd(range); + for (int v = 0; v < kValues; ++v) { + const uint16_t value = rnd(range); + enc_values[n][k][r][v][0] = range; + enc_values[n][k][r][v][1] = k; + enc_values[n][k][r][v][2] = ref; + enc_values[n][k][r][v][3] = value; + aom_write_primitive_refsubexpfin(&bw, range, k, ref, value); + } + } + } + } + aom_stop_encode(&bw); + aom_reader br; + aom_reader_init(&br, bw_buffer, bw.pos, NULL, NULL); + GTEST_ASSERT_GE(aom_reader_tell(&br), 0u); + GTEST_ASSERT_LE(aom_reader_tell(&br), 1u); + for (int n = 0; n < kRanges; ++n) { + for (int k = 0; k < kSubexpParams; ++k) { + for (int r = 0; r < kReferences; ++r) { + for (int v = 0; v < kValues; ++v) { + const uint16_t range = enc_values[n][k][r][v][0]; + assert(k == enc_values[n][k][r][v][1]); + const uint16_t ref = enc_values[n][k][r][v][2]; + const uint16_t value = + aom_read_primitive_refsubexpfin(&br, range, k, ref); + GTEST_ASSERT_EQ(value, enc_values[n][k][r][v][3]); + } + } + } + } +} +// TODO(debargha): Adds tests for other primitives +} // namespace diff --git a/third_party/aom/test/blend_a64_mask_1d_test.cc b/third_party/aom/test/blend_a64_mask_1d_test.cc new file mode 100644 index 0000000000..66e741a74d --- /dev/null +++ b/third_party/aom/test/blend_a64_mask_1d_test.cc @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/register_state_check.h" +#include "test/function_equivalence_test.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +#include "./av1_rtcd.h" + +#include "av1/common/enums.h" + +#include "aom_dsp/blend.h" + +using libaom_test::FunctionEquivalenceTest; + +namespace { + +template +class BlendA64Mask1DTest : public FunctionEquivalenceTest { + public: + static const int kIterations = 10000; + static const int kMaxWidth = MAX_SB_SIZE * 5; // * 5 to cover longer strides + static const int kMaxHeight = MAX_SB_SIZE; + static const int kBufSize = kMaxWidth * kMaxHeight; + static const int kMaxMaskWidth = 2 * MAX_SB_SIZE; + static const int kMaxMaskSize = kMaxMaskWidth; + + virtual ~BlendA64Mask1DTest() {} + + virtual void Execute(const T *p_src0, const T *p_src1) = 0; + + void Common() { + w_ = 1 << this->rng_(MAX_SB_SIZE_LOG2 + 1); + h_ = 1 << this->rng_(MAX_SB_SIZE_LOG2 + 1); + + dst_offset_ = this->rng_(33); + dst_stride_ = this->rng_(kMaxWidth + 1 - w_) + w_; + + src0_offset_ = this->rng_(33); + src0_stride_ = this->rng_(kMaxWidth + 1 - w_) + w_; + + src1_offset_ = this->rng_(33); + src1_stride_ = this->rng_(kMaxWidth + 1 - w_) + w_; + + T *p_src0; + T *p_src1; + + switch (this->rng_(3)) { + case 0: // Separate sources + p_src0 = src0_; + p_src1 = src1_; + break; + case 1: // src0 == dst + p_src0 = dst_tst_; + src0_stride_ = dst_stride_; + src0_offset_ = dst_offset_; + p_src1 = src1_; + break; + case 2: // src1 == dst + p_src0 = src0_; + p_src1 = dst_tst_; + src1_stride_ = dst_stride_; + src1_offset_ = dst_offset_; + break; + default: FAIL(); + } + + Execute(p_src0, p_src1); + + for (int r = 0; r < h_; ++r) { + for (int c = 0; c < w_; ++c) { + ASSERT_EQ(dst_ref_[dst_offset_ + r * dst_stride_ + c], + dst_tst_[dst_offset_ + r * dst_stride_ + c]); + } + } + } + + T dst_ref_[kBufSize]; + T dst_tst_[kBufSize]; + uint32_t dst_stride_; + uint32_t dst_offset_; + + T src0_[kBufSize]; + uint32_t src0_stride_; + uint32_t src0_offset_; + + T src1_[kBufSize]; + uint32_t src1_stride_; + uint32_t src1_offset_; + + uint8_t mask_[kMaxMaskSize]; + + int w_; + int h_; +}; + +////////////////////////////////////////////////////////////////////////////// +// 8 bit version +////////////////////////////////////////////////////////////////////////////// + +typedef void (*F8B)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, + uint32_t src1_stride, const uint8_t *mask, int h, int w); +typedef libaom_test::FuncParam TestFuncs; + +class BlendA64Mask1DTest8B : public BlendA64Mask1DTest { + protected: + void Execute(const uint8_t *p_src0, const uint8_t *p_src1) { + params_.ref_func(dst_ref_ + dst_offset_, dst_stride_, p_src0 + src0_offset_, + src0_stride_, p_src1 + src1_offset_, src1_stride_, mask_, + h_, w_); + ASM_REGISTER_STATE_CHECK(params_.tst_func( + dst_tst_ + dst_offset_, dst_stride_, p_src0 + src0_offset_, + src0_stride_, p_src1 + src1_offset_, src1_stride_, mask_, h_, w_)); + } +}; + +TEST_P(BlendA64Mask1DTest8B, RandomValues) { + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + for (int i = 0; i < kBufSize; ++i) { + dst_ref_[i] = rng_.Rand8(); + dst_tst_[i] = rng_.Rand8(); + + src0_[i] = rng_.Rand8(); + src1_[i] = rng_.Rand8(); + } + + for (int i = 0; i < kMaxMaskSize; ++i) + mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1); + + Common(); + } +} + +TEST_P(BlendA64Mask1DTest8B, ExtremeValues) { + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + for (int i = 0; i < kBufSize; ++i) { + dst_ref_[i] = rng_(2) + 254; + dst_tst_[i] = rng_(2) + 254; + src0_[i] = rng_(2) + 254; + src1_[i] = rng_(2) + 254; + } + + for (int i = 0; i < kMaxMaskSize; ++i) + mask_[i] = rng_(2) + AOM_BLEND_A64_MAX_ALPHA - 1; + + Common(); + } +} + +static void blend_a64_hmask_ref(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + uint8_t mask2d[BlendA64Mask1DTest8B::kMaxMaskSize] + [BlendA64Mask1DTest8B::kMaxMaskSize]; + + for (int row = 0; row < h; ++row) + for (int col = 0; col < w; ++col) mask2d[row][col] = mask[col]; + + aom_blend_a64_mask_c(dst, dst_stride, src0, src0_stride, src1, src1_stride, + &mask2d[0][0], BlendA64Mask1DTest8B::kMaxMaskSize, h, w, + 0, 0); +} + +static void blend_a64_vmask_ref(uint8_t *dst, uint32_t dst_stride, + const uint8_t *src0, uint32_t src0_stride, + const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w) { + uint8_t mask2d[BlendA64Mask1DTest8B::kMaxMaskSize] + [BlendA64Mask1DTest8B::kMaxMaskSize]; + + for (int row = 0; row < h; ++row) + for (int col = 0; col < w; ++col) mask2d[row][col] = mask[row]; + + aom_blend_a64_mask_c(dst, dst_stride, src0, src0_stride, src1, src1_stride, + &mask2d[0][0], BlendA64Mask1DTest8B::kMaxMaskSize, h, w, + 0, 0); +} + +INSTANTIATE_TEST_CASE_P( + C, BlendA64Mask1DTest8B, + ::testing::Values(TestFuncs(blend_a64_hmask_ref, aom_blend_a64_hmask_c), + TestFuncs(blend_a64_vmask_ref, aom_blend_a64_vmask_c))); + +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P( + SSE4_1, BlendA64Mask1DTest8B, + ::testing::Values( + TestFuncs(blend_a64_hmask_ref, aom_blend_a64_hmask_sse4_1), + TestFuncs(blend_a64_vmask_ref, aom_blend_a64_vmask_sse4_1))); +#endif // HAVE_SSE4_1 + +#if CONFIG_HIGHBITDEPTH +////////////////////////////////////////////////////////////////////////////// +// High bit-depth version +////////////////////////////////////////////////////////////////////////////// + +typedef void (*FHBD)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, + uint32_t src1_stride, const uint8_t *mask, int h, int w, + int bd); +typedef libaom_test::FuncParam TestFuncsHBD; + +class BlendA64Mask1DTestHBD : public BlendA64Mask1DTest { + protected: + void Execute(const uint16_t *p_src0, const uint16_t *p_src1) { + params_.ref_func(CONVERT_TO_BYTEPTR(dst_ref_ + dst_offset_), dst_stride_, + CONVERT_TO_BYTEPTR(p_src0 + src0_offset_), src0_stride_, + CONVERT_TO_BYTEPTR(p_src1 + src1_offset_), src1_stride_, + mask_, h_, w_, bit_depth_); + ASM_REGISTER_STATE_CHECK(params_.tst_func( + CONVERT_TO_BYTEPTR(dst_tst_ + dst_offset_), dst_stride_, + CONVERT_TO_BYTEPTR(p_src0 + src0_offset_), src0_stride_, + CONVERT_TO_BYTEPTR(p_src1 + src1_offset_), src1_stride_, mask_, h_, w_, + bit_depth_)); + } + + int bit_depth_; +}; + +TEST_P(BlendA64Mask1DTestHBD, RandomValues) { + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + switch (rng_(3)) { + case 0: bit_depth_ = 8; break; + case 1: bit_depth_ = 10; break; + default: bit_depth_ = 12; break; + } + + const int hi = 1 << bit_depth_; + + for (int i = 0; i < kBufSize; ++i) { + dst_ref_[i] = rng_(hi); + dst_tst_[i] = rng_(hi); + src0_[i] = rng_(hi); + src1_[i] = rng_(hi); + } + + for (int i = 0; i < kMaxMaskSize; ++i) + mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1); + + Common(); + } +} + +TEST_P(BlendA64Mask1DTestHBD, ExtremeValues) { + for (int iter = 0; iter < 1000 && !HasFatalFailure(); ++iter) { + switch (rng_(3)) { + case 0: bit_depth_ = 8; break; + case 1: bit_depth_ = 10; break; + default: bit_depth_ = 12; break; + } + + const int hi = 1 << bit_depth_; + const int lo = hi - 2; + + for (int i = 0; i < kBufSize; ++i) { + dst_ref_[i] = rng_(hi - lo) + lo; + dst_tst_[i] = rng_(hi - lo) + lo; + src0_[i] = rng_(hi - lo) + lo; + src1_[i] = rng_(hi - lo) + lo; + } + + for (int i = 0; i < kMaxMaskSize; ++i) + mask_[i] = rng_(2) + AOM_BLEND_A64_MAX_ALPHA - 1; + + Common(); + } +} + +static void highbd_blend_a64_hmask_ref( + uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w, int bd) { + uint8_t mask2d[BlendA64Mask1DTestHBD::kMaxMaskSize] + [BlendA64Mask1DTestHBD::kMaxMaskSize]; + + for (int row = 0; row < h; ++row) + for (int col = 0; col < w; ++col) mask2d[row][col] = mask[col]; + + aom_highbd_blend_a64_mask_c( + dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask2d[0][0], + BlendA64Mask1DTestHBD::kMaxMaskSize, h, w, 0, 0, bd); +} + +static void highbd_blend_a64_vmask_ref( + uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, + const uint8_t *mask, int h, int w, int bd) { + uint8_t mask2d[BlendA64Mask1DTestHBD::kMaxMaskSize] + [BlendA64Mask1DTestHBD::kMaxMaskSize]; + + for (int row = 0; row < h; ++row) + for (int col = 0; col < w; ++col) mask2d[row][col] = mask[row]; + + aom_highbd_blend_a64_mask_c( + dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask2d[0][0], + BlendA64Mask1DTestHBD::kMaxMaskSize, h, w, 0, 0, bd); +} + +INSTANTIATE_TEST_CASE_P( + C, BlendA64Mask1DTestHBD, + ::testing::Values(TestFuncsHBD(highbd_blend_a64_hmask_ref, + aom_highbd_blend_a64_hmask_c), + TestFuncsHBD(highbd_blend_a64_vmask_ref, + aom_highbd_blend_a64_vmask_c))); + +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P( + SSE4_1, BlendA64Mask1DTestHBD, + ::testing::Values(TestFuncsHBD(highbd_blend_a64_hmask_ref, + aom_highbd_blend_a64_hmask_sse4_1), + TestFuncsHBD(highbd_blend_a64_vmask_ref, + aom_highbd_blend_a64_vmask_sse4_1))); +#endif // HAVE_SSE4_1 + +#endif // CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/blend_a64_mask_test.cc b/third_party/aom/test/blend_a64_mask_test.cc new file mode 100644 index 0000000000..fef124d347 --- /dev/null +++ b/third_party/aom/test/blend_a64_mask_test.cc @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/register_state_check.h" +#include "test/function_equivalence_test.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +#include "./av1_rtcd.h" + +#include "av1/common/enums.h" + +#include "aom_dsp/blend.h" + +using libaom_test::FunctionEquivalenceTest; + +namespace { + +template +class BlendA64MaskTest : public FunctionEquivalenceTest { + protected: + static const int kIterations = 10000; + static const int kMaxWidth = MAX_SB_SIZE * 5; // * 5 to cover longer strides + static const int kMaxHeight = MAX_SB_SIZE; + static const int kBufSize = kMaxWidth * kMaxHeight; + static const int kMaxMaskWidth = 2 * MAX_SB_SIZE; + static const int kMaxMaskSize = kMaxMaskWidth * kMaxMaskWidth; + + virtual ~BlendA64MaskTest() {} + + virtual void Execute(const T *p_src0, const T *p_src1) = 0; + + void Common() { + w_ = 1 << this->rng_(MAX_SB_SIZE_LOG2 + 1); + h_ = 1 << this->rng_(MAX_SB_SIZE_LOG2 + 1); + + subx_ = this->rng_(2); + suby_ = this->rng_(2); + + dst_offset_ = this->rng_(33); + dst_stride_ = this->rng_(kMaxWidth + 1 - w_) + w_; + + src0_offset_ = this->rng_(33); + src0_stride_ = this->rng_(kMaxWidth + 1 - w_) + w_; + + src1_offset_ = this->rng_(33); + src1_stride_ = this->rng_(kMaxWidth + 1 - w_) + w_; + + mask_stride_ = + this->rng_(kMaxWidth + 1 - w_ * (subx_ ? 2 : 1)) + w_ * (subx_ ? 2 : 1); + + T *p_src0; + T *p_src1; + + switch (this->rng_(3)) { + case 0: // Separate sources + p_src0 = src0_; + p_src1 = src1_; + break; + case 1: // src0 == dst + p_src0 = dst_tst_; + src0_stride_ = dst_stride_; + src0_offset_ = dst_offset_; + p_src1 = src1_; + break; + case 2: // src1 == dst + p_src0 = src0_; + p_src1 = dst_tst_; + src1_stride_ = dst_stride_; + src1_offset_ = dst_offset_; + break; + default: FAIL(); + } + + Execute(p_src0, p_src1); + + for (int r = 0; r < h_; ++r) { + for (int c = 0; c < w_; ++c) { + ASSERT_EQ(dst_ref_[dst_offset_ + r * dst_stride_ + c], + dst_tst_[dst_offset_ + r * dst_stride_ + c]); + } + } + } + + T dst_ref_[kBufSize]; + T dst_tst_[kBufSize]; + uint32_t dst_stride_; + uint32_t dst_offset_; + + T src0_[kBufSize]; + uint32_t src0_stride_; + uint32_t src0_offset_; + + T src1_[kBufSize]; + uint32_t src1_stride_; + uint32_t src1_offset_; + + uint8_t mask_[kMaxMaskSize]; + size_t mask_stride_; + + int w_; + int h_; + + int suby_; + int subx_; +}; + +////////////////////////////////////////////////////////////////////////////// +// 8 bit version +////////////////////////////////////////////////////////////////////////////// + +typedef void (*F8B)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, + uint32_t src1_stride, const uint8_t *mask, + uint32_t mask_stride, int h, int w, int suby, int subx); +typedef libaom_test::FuncParam TestFuncs; + +class BlendA64MaskTest8B : public BlendA64MaskTest { + protected: + void Execute(const uint8_t *p_src0, const uint8_t *p_src1) { + params_.ref_func(dst_ref_ + dst_offset_, dst_stride_, p_src0 + src0_offset_, + src0_stride_, p_src1 + src1_offset_, src1_stride_, mask_, + kMaxMaskWidth, h_, w_, suby_, subx_); + ASM_REGISTER_STATE_CHECK(params_.tst_func( + dst_tst_ + dst_offset_, dst_stride_, p_src0 + src0_offset_, + src0_stride_, p_src1 + src1_offset_, src1_stride_, mask_, kMaxMaskWidth, + h_, w_, suby_, subx_)); + } +}; + +TEST_P(BlendA64MaskTest8B, RandomValues) { + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + for (int i = 0; i < kBufSize; ++i) { + dst_ref_[i] = rng_.Rand8(); + dst_tst_[i] = rng_.Rand8(); + + src0_[i] = rng_.Rand8(); + src1_[i] = rng_.Rand8(); + } + + for (int i = 0; i < kMaxMaskSize; ++i) + mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1); + + Common(); + } +} + +TEST_P(BlendA64MaskTest8B, ExtremeValues) { + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + for (int i = 0; i < kBufSize; ++i) { + dst_ref_[i] = rng_(2) + 254; + dst_tst_[i] = rng_(2) + 254; + src0_[i] = rng_(2) + 254; + src1_[i] = rng_(2) + 254; + } + + for (int i = 0; i < kMaxMaskSize; ++i) + mask_[i] = rng_(2) + AOM_BLEND_A64_MAX_ALPHA - 1; + + Common(); + } +} + +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P(SSE4_1, BlendA64MaskTest8B, + ::testing::Values(TestFuncs( + aom_blend_a64_mask_c, aom_blend_a64_mask_sse4_1))); +#endif // HAVE_SSE4_1 + +#if CONFIG_HIGHBITDEPTH +////////////////////////////////////////////////////////////////////////////// +// High bit-depth version +////////////////////////////////////////////////////////////////////////////// + +typedef void (*FHBD)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, + uint32_t src1_stride, const uint8_t *mask, + uint32_t mask_stride, int h, int w, int suby, int subx, + int bd); +typedef libaom_test::FuncParam TestFuncsHBD; + +class BlendA64MaskTestHBD : public BlendA64MaskTest { + protected: + void Execute(const uint16_t *p_src0, const uint16_t *p_src1) { + params_.ref_func(CONVERT_TO_BYTEPTR(dst_ref_ + dst_offset_), dst_stride_, + CONVERT_TO_BYTEPTR(p_src0 + src0_offset_), src0_stride_, + CONVERT_TO_BYTEPTR(p_src1 + src1_offset_), src1_stride_, + mask_, kMaxMaskWidth, h_, w_, suby_, subx_, bit_depth_); + ASM_REGISTER_STATE_CHECK(params_.tst_func( + CONVERT_TO_BYTEPTR(dst_tst_ + dst_offset_), dst_stride_, + CONVERT_TO_BYTEPTR(p_src0 + src0_offset_), src0_stride_, + CONVERT_TO_BYTEPTR(p_src1 + src1_offset_), src1_stride_, mask_, + kMaxMaskWidth, h_, w_, suby_, subx_, bit_depth_)); + } + + int bit_depth_; +}; + +TEST_P(BlendA64MaskTestHBD, RandomValues) { + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + switch (rng_(3)) { + case 0: bit_depth_ = 8; break; + case 1: bit_depth_ = 10; break; + default: bit_depth_ = 12; break; + } + + const int hi = 1 << bit_depth_; + + for (int i = 0; i < kBufSize; ++i) { + dst_ref_[i] = rng_(hi); + dst_tst_[i] = rng_(hi); + src0_[i] = rng_(hi); + src1_[i] = rng_(hi); + } + + for (int i = 0; i < kMaxMaskSize; ++i) + mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1); + + Common(); + } +} + +TEST_P(BlendA64MaskTestHBD, ExtremeValues) { + for (int iter = 0; iter < 1000 && !HasFatalFailure(); ++iter) { + switch (rng_(3)) { + case 0: bit_depth_ = 8; break; + case 1: bit_depth_ = 10; break; + default: bit_depth_ = 12; break; + } + + const int hi = 1 << bit_depth_; + const int lo = hi - 2; + + for (int i = 0; i < kBufSize; ++i) { + dst_ref_[i] = rng_(hi - lo) + lo; + dst_tst_[i] = rng_(hi - lo) + lo; + src0_[i] = rng_(hi - lo) + lo; + src1_[i] = rng_(hi - lo) + lo; + } + + for (int i = 0; i < kMaxMaskSize; ++i) + mask_[i] = rng_(2) + AOM_BLEND_A64_MAX_ALPHA - 1; + + Common(); + } +} + +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P( + SSE4_1, BlendA64MaskTestHBD, + ::testing::Values(TestFuncsHBD(aom_highbd_blend_a64_mask_c, + aom_highbd_blend_a64_mask_sse4_1))); +#endif // HAVE_SSE4_1 +#endif // CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/boolcoder_test.cc b/third_party/aom/test/boolcoder_test.cc new file mode 100644 index 0000000000..4d9d7aaf44 --- /dev/null +++ b/third_party/aom/test/boolcoder_test.cc @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/acm_random.h" +#include "aom/aom_integer.h" +#include "aom_dsp/bitreader.h" +#include "aom_dsp/bitwriter.h" + +using libaom_test::ACMRandom; + +namespace { +const int num_tests = 10; +} // namespace + +TEST(AV1, TestBitIO) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + for (int n = 0; n < num_tests; ++n) { + for (int method = 0; method <= 7; ++method) { // we generate various proba + const int kBitsToTest = 1000; + uint8_t probas[kBitsToTest]; + + for (int i = 0; i < kBitsToTest; ++i) { + const int parity = i & 1; + /* clang-format off */ + probas[i] = + (method == 0) ? 0 : (method == 1) ? 255 : + (method == 2) ? 128 : + (method == 3) ? rnd.Rand8() : + (method == 4) ? (parity ? 0 : 255) : + // alternate between low and high proba: + (method == 5) ? (parity ? rnd(128) : 255 - rnd(128)) : + (method == 6) ? + (parity ? rnd(64) : 255 - rnd(64)) : + (parity ? rnd(32) : 255 - rnd(32)); + /* clang-format on */ + } + for (int bit_method = 0; bit_method <= 3; ++bit_method) { + const int random_seed = 6432; + const int kBufferSize = 10000; + ACMRandom bit_rnd(random_seed); + aom_writer bw; + uint8_t bw_buffer[kBufferSize]; + aom_start_encode(&bw, bw_buffer); + + int bit = (bit_method == 0) ? 0 : (bit_method == 1) ? 1 : 0; + for (int i = 0; i < kBitsToTest; ++i) { + if (bit_method == 2) { + bit = (i & 1); + } else if (bit_method == 3) { + bit = bit_rnd(2); + } + aom_write(&bw, bit, static_cast(probas[i])); + } + + aom_stop_encode(&bw); + +#if !CONFIG_DAALA_EC + // First bit should be zero + GTEST_ASSERT_EQ(bw_buffer[0] & 0x80, 0); +#endif + + aom_reader br; + aom_reader_init(&br, bw_buffer, bw.pos, NULL, NULL); + bit_rnd.Reset(random_seed); + for (int i = 0; i < kBitsToTest; ++i) { + if (bit_method == 2) { + bit = (i & 1); + } else if (bit_method == 3) { + bit = bit_rnd(2); + } + GTEST_ASSERT_EQ(aom_read(&br, probas[i], NULL), bit) + << "pos: " << i << " / " << kBitsToTest + << " bit_method: " << bit_method << " method: " << method; + } + } + } + } +} + +#if CONFIG_DAALA_EC +#define FRAC_DIFF_TOTAL_ERROR 0.07 +#else +#define FRAC_DIFF_TOTAL_ERROR 0.2 +#endif + +TEST(AV1, TestTell) { + const int kBufferSize = 10000; + aom_writer bw; + uint8_t bw_buffer[kBufferSize]; + const int kSymbols = 1024; + // Coders are noisier at low probabilities, so we start at p = 4. + for (int p = 4; p < 256; p++) { + double probability = p / 256.; + aom_start_encode(&bw, bw_buffer); + for (int i = 0; i < kSymbols; i++) { + aom_write(&bw, 0, p); + } + aom_stop_encode(&bw); + aom_reader br; + aom_reader_init(&br, bw_buffer, bw.pos, NULL, NULL); + uint32_t last_tell = aom_reader_tell(&br); + uint32_t last_tell_frac = aom_reader_tell_frac(&br); + double frac_diff_total = 0; + GTEST_ASSERT_GE(aom_reader_tell(&br), 0u); + GTEST_ASSERT_LE(aom_reader_tell(&br), 1u); + for (int i = 0; i < kSymbols; i++) { + aom_read(&br, p, NULL); + uint32_t tell = aom_reader_tell(&br); + uint32_t tell_frac = aom_reader_tell_frac(&br); + GTEST_ASSERT_GE(tell, last_tell) << "tell: " << tell + << ", last_tell: " << last_tell; + GTEST_ASSERT_GE(tell_frac, last_tell_frac) + << "tell_frac: " << tell_frac + << ", last_tell_frac: " << last_tell_frac; + // Frac tell should round up to tell. + GTEST_ASSERT_EQ(tell, (tell_frac + 7) >> 3); + last_tell = tell; + frac_diff_total += + fabs(((tell_frac - last_tell_frac) / 8.0) + log2(probability)); + last_tell_frac = tell_frac; + } + const uint32_t expected = (uint32_t)(-kSymbols * log2(probability)); + // Last tell should be close to the expected value. + GTEST_ASSERT_LE(last_tell, expected + 20) << " last_tell: " << last_tell; + // The average frac_diff error should be pretty small. + GTEST_ASSERT_LE(frac_diff_total / kSymbols, FRAC_DIFF_TOTAL_ERROR) + << " frac_diff_total: " << frac_diff_total; + } +} diff --git a/third_party/aom/test/borders_test.cc b/third_party/aom/test/borders_test.cc new file mode 100644 index 0000000000..076f914044 --- /dev/null +++ b/third_party/aom/test/borders_test.cc @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" + +namespace { + +class BordersTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWithParam { + protected: + BordersTest() : EncoderTest(GET_PARAM(0)) {} + virtual ~BordersTest() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(GET_PARAM(1)); + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 1) { + encoder->Control(AOME_SET_CPUUSED, 1); + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); + encoder->Control(AOME_SET_ARNR_MAXFRAMES, 7); + encoder->Control(AOME_SET_ARNR_STRENGTH, 5); + } + } + + virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) { + if (pkt->data.frame.flags & AOM_FRAME_IS_KEY) { + } + } +}; + +TEST_P(BordersTest, TestEncodeHighBitrate) { + // Validate that this non multiple of 64 wide clip encodes and decodes + // without a mismatch when passing in a very low max q. This pushes + // the encoder to producing lots of big partitions which will likely + // extend into the border and test the border condition. + cfg_.g_lag_in_frames = 25; + cfg_.rc_2pass_vbr_minsection_pct = 5; + cfg_.rc_2pass_vbr_maxsection_pct = 2000; + cfg_.rc_target_bitrate = 2000; + cfg_.rc_max_quantizer = 10; + + ::libaom_test::I420VideoSource video("hantro_odd.yuv", 208, 144, 30, 1, 0, + 40); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} +TEST_P(BordersTest, TestLowBitrate) { + // Validate that this clip encodes and decodes without a mismatch + // when passing in a very high min q. This pushes the encoder to producing + // lots of small partitions which might will test the other condition. + + cfg_.g_lag_in_frames = 25; + cfg_.rc_2pass_vbr_minsection_pct = 5; + cfg_.rc_2pass_vbr_maxsection_pct = 2000; + cfg_.rc_target_bitrate = 200; + cfg_.rc_min_quantizer = 40; + + ::libaom_test::I420VideoSource video("hantro_odd.yuv", 208, 144, 30, 1, 0, + 40); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} + +AV1_INSTANTIATE_TEST_CASE(BordersTest, + ::testing::Values(::libaom_test::kTwoPassGood)); +} // namespace diff --git a/third_party/aom/test/clear_system_state.h b/third_party/aom/test/clear_system_state.h new file mode 100644 index 0000000000..4f3c1eed07 --- /dev/null +++ b/third_party/aom/test/clear_system_state.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef TEST_CLEAR_SYSTEM_STATE_H_ +#define TEST_CLEAR_SYSTEM_STATE_H_ + +#include "./aom_config.h" +#if ARCH_X86 || ARCH_X86_64 +#include "aom_ports/x86.h" +#endif + +namespace libaom_test { + +// Reset system to a known state. This function should be used for all non-API +// test cases. +inline void ClearSystemState() { +#if ARCH_X86 || ARCH_X86_64 + aom_reset_mmx_state(); +#endif +} + +} // namespace libaom_test +#endif // TEST_CLEAR_SYSTEM_STATE_H_ diff --git a/third_party/aom/test/clpf_test.cc b/third_party/aom/test/clpf_test.cc new file mode 100644 index 0000000000..2c0f8cf7f3 --- /dev/null +++ b/third_party/aom/test/clpf_test.cc @@ -0,0 +1,437 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "aom_ports/aom_timer.h" +#include "av1/common/od_dering.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" + +using libaom_test::ACMRandom; + +namespace { + +typedef void (*clpf_block_t)(uint8_t *dst, const uint16_t *src, int dstride, + int sstride, int sizex, int sizey, + unsigned int strength, unsigned int bitdepth); + +typedef std::tr1::tuple + clpf_block_param_t; + +class CDEFClpfBlockTest : public ::testing::TestWithParam { + public: + virtual ~CDEFClpfBlockTest() {} + virtual void SetUp() { + clpf = GET_PARAM(0); + ref_clpf = GET_PARAM(1); + sizex = GET_PARAM(2); + sizey = GET_PARAM(3); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + int sizex; + int sizey; + clpf_block_t clpf; + clpf_block_t ref_clpf; +}; + +typedef CDEFClpfBlockTest CDEFClpfSpeedTest; + +#if CONFIG_HIGHBITDEPTH +typedef void (*clpf_block_hbd_t)(uint16_t *dst, const uint16_t *src, + int dstride, int sstride, int sizex, int sizey, + unsigned int strength, unsigned int bitdepth); + +typedef std::tr1::tuple + clpf_block_hbd_param_t; + +class CDEFClpfBlockHbdTest + : public ::testing::TestWithParam { + public: + virtual ~CDEFClpfBlockHbdTest() {} + virtual void SetUp() { + clpf = GET_PARAM(0); + ref_clpf = GET_PARAM(1); + sizex = GET_PARAM(2); + sizey = GET_PARAM(3); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + int sizex; + int sizey; + clpf_block_hbd_t clpf; + clpf_block_hbd_t ref_clpf; +}; + +typedef CDEFClpfBlockHbdTest ClpfHbdSpeedTest; +#endif + +template +void test_clpf(int w, int h, unsigned int depth, unsigned int iterations, + void (*clpf)(pixel *dst, const uint16_t *src, int dstride, + int sstride, int sizex, int sizey, + unsigned int strength, unsigned int bitdepth), + void (*ref_clpf)(pixel *dst, const uint16_t *src, int dstride, + int sstride, int sizex, int sizey, + unsigned int strength, unsigned int bitdepth)) { + const int size = 24; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint16_t, s[size * size]); + DECLARE_ALIGNED(16, pixel, d[size * size]); + DECLARE_ALIGNED(16, pixel, ref_d[size * size]); + memset(ref_d, 0, size * size * sizeof(*ref_d)); + memset(d, 0, size * size * sizeof(*d)); + + int error = 0, pos = 0, xpos = 8, ypos = 8; + unsigned int strength = 0, bits, level, count, damp = 0, boundary = 0; + + assert(size >= w + 16 && size >= h + 16); + assert(depth >= 8); + + // Test every combination of: + // * Input with up to bits of noise + // * Noise level around every value from 0 to (1<Params classes for a variable number of parameters + * to avoid having to include a pointer to the CodecFactory in every test + * definition. + */ +template +class CodecTestWithParam + : public ::testing::TestWithParam< + std::tr1::tuple > {}; + +template +class CodecTestWith2Params + : public ::testing::TestWithParam< + std::tr1::tuple > {}; + +template +class CodecTestWith3Params + : public ::testing::TestWithParam< + std::tr1::tuple > {}; + +/* + * AV1 Codec Definitions + */ +#if CONFIG_AV1 +class AV1Decoder : public Decoder { + public: + explicit AV1Decoder(aom_codec_dec_cfg_t cfg) : Decoder(cfg) {} + + AV1Decoder(aom_codec_dec_cfg_t cfg, const aom_codec_flags_t flag) + : Decoder(cfg, flag) {} + + protected: + virtual aom_codec_iface_t *CodecInterface() const { +#if CONFIG_AV1_DECODER + return &aom_codec_av1_dx_algo; +#else + return NULL; +#endif + } +}; + +class AV1Encoder : public Encoder { + public: + AV1Encoder(aom_codec_enc_cfg_t cfg, unsigned long deadline, + const unsigned long init_flags, TwopassStatsStore *stats) + : Encoder(cfg, deadline, init_flags, stats) {} + + protected: + virtual aom_codec_iface_t *CodecInterface() const { +#if CONFIG_AV1_ENCODER + return &aom_codec_av1_cx_algo; +#else + return NULL; +#endif + } +}; + +class AV1CodecFactory : public CodecFactory { + public: + AV1CodecFactory() : CodecFactory() {} + + virtual Decoder *CreateDecoder(aom_codec_dec_cfg_t cfg) const { + return CreateDecoder(cfg, 0); + } + + virtual Decoder *CreateDecoder(aom_codec_dec_cfg_t cfg, + const aom_codec_flags_t flags) const { +#if CONFIG_AV1_DECODER + return new AV1Decoder(cfg, flags); +#else + (void)cfg; + (void)flags; + return NULL; +#endif + } + + virtual Encoder *CreateEncoder(aom_codec_enc_cfg_t cfg, + unsigned long deadline, + const unsigned long init_flags, + TwopassStatsStore *stats) const { +#if CONFIG_AV1_ENCODER + return new AV1Encoder(cfg, deadline, init_flags, stats); +#else + (void)cfg; + (void)deadline; + (void)init_flags; + (void)stats; + return NULL; +#endif + } + + virtual aom_codec_err_t DefaultEncoderConfig(aom_codec_enc_cfg_t *cfg, + int usage) const { +#if CONFIG_AV1_ENCODER + return aom_codec_enc_config_default(&aom_codec_av1_cx_algo, cfg, usage); +#else + (void)cfg; + (void)usage; + return AOM_CODEC_INCAPABLE; +#endif + } +}; + +const libaom_test::AV1CodecFactory kAV1; + +#define AV1_INSTANTIATE_TEST_CASE(test, ...) \ + INSTANTIATE_TEST_CASE_P( \ + AV1, test, \ + ::testing::Combine( \ + ::testing::Values(static_cast( \ + &libaom_test::kAV1)), \ + __VA_ARGS__)) +#else +#define AV1_INSTANTIATE_TEST_CASE(test, ...) +#endif // CONFIG_AV1 + +} // namespace libaom_test +#endif // TEST_CODEC_FACTORY_H_ diff --git a/third_party/aom/test/convolve_test.cc b/third_party/aom/test/convolve_test.cc new file mode 100644 index 0000000000..a84ef4ec8e --- /dev/null +++ b/third_party/aom/test/convolve_test.cc @@ -0,0 +1,1345 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/aom_filter.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" +#include "aom_ports/aom_timer.h" +#include "av1/common/filter.h" + +namespace { + +static const unsigned int kMaxDimension = MAX_SB_SIZE; + +typedef void (*ConvolveFunc)(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int filter_x_stride, + const int16_t *filter_y, int filter_y_stride, + int w, int h); + +struct ConvolveFunctions { + ConvolveFunctions(ConvolveFunc copy, ConvolveFunc avg, ConvolveFunc h8, + ConvolveFunc h8_avg, ConvolveFunc v8, ConvolveFunc v8_avg, + ConvolveFunc hv8, ConvolveFunc hv8_avg, ConvolveFunc sh8, + ConvolveFunc sh8_avg, ConvolveFunc sv8, + ConvolveFunc sv8_avg, ConvolveFunc shv8, + ConvolveFunc shv8_avg, int bd) + : copy_(copy), avg_(avg), h8_(h8), v8_(v8), hv8_(hv8), h8_avg_(h8_avg), + v8_avg_(v8_avg), hv8_avg_(hv8_avg), sh8_(sh8), sv8_(sv8), shv8_(shv8), + sh8_avg_(sh8_avg), sv8_avg_(sv8_avg), shv8_avg_(shv8_avg), + use_highbd_(bd) {} + + ConvolveFunc copy_; + ConvolveFunc avg_; + ConvolveFunc h8_; + ConvolveFunc v8_; + ConvolveFunc hv8_; + ConvolveFunc h8_avg_; + ConvolveFunc v8_avg_; + ConvolveFunc hv8_avg_; + ConvolveFunc sh8_; // scaled horiz + ConvolveFunc sv8_; // scaled vert + ConvolveFunc shv8_; // scaled horiz/vert + ConvolveFunc sh8_avg_; // scaled avg horiz + ConvolveFunc sv8_avg_; // scaled avg vert + ConvolveFunc shv8_avg_; // scaled avg horiz/vert + int use_highbd_; // 0 if high bitdepth not used, else the actual bit depth. +}; + +typedef std::tr1::tuple ConvolveParam; + +#if CONFIG_AV1 && CONFIG_EXT_PARTITION +#define ALL_SIZES(convolve_fn) \ + make_tuple(128, 64, &convolve_fn), make_tuple(64, 128, &convolve_fn), \ + make_tuple(128, 128, &convolve_fn), make_tuple(4, 4, &convolve_fn), \ + make_tuple(8, 4, &convolve_fn), make_tuple(4, 8, &convolve_fn), \ + make_tuple(8, 8, &convolve_fn), make_tuple(16, 8, &convolve_fn), \ + make_tuple(8, 16, &convolve_fn), make_tuple(16, 16, &convolve_fn), \ + make_tuple(32, 16, &convolve_fn), make_tuple(16, 32, &convolve_fn), \ + make_tuple(32, 32, &convolve_fn), make_tuple(64, 32, &convolve_fn), \ + make_tuple(32, 64, &convolve_fn), make_tuple(64, 64, &convolve_fn) +#else +#define ALL_SIZES(convolve_fn) \ + make_tuple(4, 4, &convolve_fn), make_tuple(8, 4, &convolve_fn), \ + make_tuple(4, 8, &convolve_fn), make_tuple(8, 8, &convolve_fn), \ + make_tuple(16, 8, &convolve_fn), make_tuple(8, 16, &convolve_fn), \ + make_tuple(16, 16, &convolve_fn), make_tuple(32, 16, &convolve_fn), \ + make_tuple(16, 32, &convolve_fn), make_tuple(32, 32, &convolve_fn), \ + make_tuple(64, 32, &convolve_fn), make_tuple(32, 64, &convolve_fn), \ + make_tuple(64, 64, &convolve_fn) +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + +// Reference 8-tap subpixel filter, slightly modified to fit into this test. +#define AV1_FILTER_WEIGHT 128 +#define AV1_FILTER_SHIFT 7 +uint8_t clip_pixel(int x) { return x < 0 ? 0 : x > 255 ? 255 : x; } + +void filter_block2d_8_c(const uint8_t *src_ptr, unsigned int src_stride, + const int16_t *HFilter, const int16_t *VFilter, + uint8_t *dst_ptr, unsigned int dst_stride, + unsigned int output_width, unsigned int output_height) { + // Between passes, we use an intermediate buffer whose height is extended to + // have enough horizontally filtered values as input for the vertical pass. + // This buffer is allocated to be big enough for the largest block type we + // support. + const int kInterp_Extend = 4; + const unsigned int intermediate_height = + (kInterp_Extend - 1) + output_height + kInterp_Extend; + unsigned int i, j; + + assert(intermediate_height > 7); + + // Size of intermediate_buffer is max_intermediate_height * filter_max_width, + // where max_intermediate_height = (kInterp_Extend - 1) + filter_max_height + // + kInterp_Extend + // = 3 + 16 + 4 + // = 23 + // and filter_max_width = 16 + // + uint8_t intermediate_buffer[(kMaxDimension + 8) * kMaxDimension]; + const int intermediate_next_stride = + 1 - static_cast(intermediate_height * output_width); + + // Horizontal pass (src -> transposed intermediate). + uint8_t *output_ptr = intermediate_buffer; + const int src_next_row_stride = src_stride - output_width; + src_ptr -= (kInterp_Extend - 1) * src_stride + (kInterp_Extend - 1); + for (i = 0; i < intermediate_height; ++i) { + for (j = 0; j < output_width; ++j) { + // Apply filter... + const int temp = (src_ptr[0] * HFilter[0]) + (src_ptr[1] * HFilter[1]) + + (src_ptr[2] * HFilter[2]) + (src_ptr[3] * HFilter[3]) + + (src_ptr[4] * HFilter[4]) + (src_ptr[5] * HFilter[5]) + + (src_ptr[6] * HFilter[6]) + (src_ptr[7] * HFilter[7]) + + (AV1_FILTER_WEIGHT >> 1); // Rounding + + // Normalize back to 0-255... + *output_ptr = clip_pixel(temp >> AV1_FILTER_SHIFT); + ++src_ptr; + output_ptr += intermediate_height; + } + src_ptr += src_next_row_stride; + output_ptr += intermediate_next_stride; + } + + // Vertical pass (transposed intermediate -> dst). + src_ptr = intermediate_buffer; + const int dst_next_row_stride = dst_stride - output_width; + for (i = 0; i < output_height; ++i) { + for (j = 0; j < output_width; ++j) { + // Apply filter... + const int temp = (src_ptr[0] * VFilter[0]) + (src_ptr[1] * VFilter[1]) + + (src_ptr[2] * VFilter[2]) + (src_ptr[3] * VFilter[3]) + + (src_ptr[4] * VFilter[4]) + (src_ptr[5] * VFilter[5]) + + (src_ptr[6] * VFilter[6]) + (src_ptr[7] * VFilter[7]) + + (AV1_FILTER_WEIGHT >> 1); // Rounding + + // Normalize back to 0-255... + *dst_ptr++ = clip_pixel(temp >> AV1_FILTER_SHIFT); + src_ptr += intermediate_height; + } + src_ptr += intermediate_next_stride; + dst_ptr += dst_next_row_stride; + } +} + +void block2d_average_c(uint8_t *src, unsigned int src_stride, + uint8_t *output_ptr, unsigned int output_stride, + unsigned int output_width, unsigned int output_height) { + unsigned int i, j; + for (i = 0; i < output_height; ++i) { + for (j = 0; j < output_width; ++j) { + output_ptr[j] = (output_ptr[j] + src[i * src_stride + j] + 1) >> 1; + } + output_ptr += output_stride; + } +} + +void filter_average_block2d_8_c(const uint8_t *src_ptr, + const unsigned int src_stride, + const int16_t *HFilter, const int16_t *VFilter, + uint8_t *dst_ptr, unsigned int dst_stride, + unsigned int output_width, + unsigned int output_height) { + uint8_t tmp[kMaxDimension * kMaxDimension]; + + assert(output_width <= kMaxDimension); + assert(output_height <= kMaxDimension); + filter_block2d_8_c(src_ptr, src_stride, HFilter, VFilter, tmp, kMaxDimension, + output_width, output_height); + block2d_average_c(tmp, kMaxDimension, dst_ptr, dst_stride, output_width, + output_height); +} + +#if CONFIG_HIGHBITDEPTH +void highbd_filter_block2d_8_c(const uint16_t *src_ptr, + const unsigned int src_stride, + const int16_t *HFilter, const int16_t *VFilter, + uint16_t *dst_ptr, unsigned int dst_stride, + unsigned int output_width, + unsigned int output_height, int bd) { + // Between passes, we use an intermediate buffer whose height is extended to + // have enough horizontally filtered values as input for the vertical pass. + // This buffer is allocated to be big enough for the largest block type we + // support. + const int kInterp_Extend = 4; + const unsigned int intermediate_height = + (kInterp_Extend - 1) + output_height + kInterp_Extend; + + /* Size of intermediate_buffer is max_intermediate_height * filter_max_width, + * where max_intermediate_height = (kInterp_Extend - 1) + filter_max_height + * + kInterp_Extend + * = 3 + 16 + 4 + * = 23 + * and filter_max_width = 16 + */ + uint16_t intermediate_buffer[(kMaxDimension + 8) * kMaxDimension]; + const int intermediate_next_stride = + 1 - static_cast(intermediate_height * output_width); + + // Horizontal pass (src -> transposed intermediate). + { + uint16_t *output_ptr = intermediate_buffer; + const int src_next_row_stride = src_stride - output_width; + unsigned int i, j; + src_ptr -= (kInterp_Extend - 1) * src_stride + (kInterp_Extend - 1); + for (i = 0; i < intermediate_height; ++i) { + for (j = 0; j < output_width; ++j) { + // Apply filter... + const int temp = (src_ptr[0] * HFilter[0]) + (src_ptr[1] * HFilter[1]) + + (src_ptr[2] * HFilter[2]) + (src_ptr[3] * HFilter[3]) + + (src_ptr[4] * HFilter[4]) + (src_ptr[5] * HFilter[5]) + + (src_ptr[6] * HFilter[6]) + (src_ptr[7] * HFilter[7]) + + (AV1_FILTER_WEIGHT >> 1); // Rounding + + // Normalize back to 0-255... + *output_ptr = clip_pixel_highbd(temp >> AV1_FILTER_SHIFT, bd); + ++src_ptr; + output_ptr += intermediate_height; + } + src_ptr += src_next_row_stride; + output_ptr += intermediate_next_stride; + } + } + + // Vertical pass (transposed intermediate -> dst). + { + const uint16_t *interm_ptr = intermediate_buffer; + const int dst_next_row_stride = dst_stride - output_width; + unsigned int i, j; + for (i = 0; i < output_height; ++i) { + for (j = 0; j < output_width; ++j) { + // Apply filter... + const int temp = + (interm_ptr[0] * VFilter[0]) + (interm_ptr[1] * VFilter[1]) + + (interm_ptr[2] * VFilter[2]) + (interm_ptr[3] * VFilter[3]) + + (interm_ptr[4] * VFilter[4]) + (interm_ptr[5] * VFilter[5]) + + (interm_ptr[6] * VFilter[6]) + (interm_ptr[7] * VFilter[7]) + + (AV1_FILTER_WEIGHT >> 1); // Rounding + + // Normalize back to 0-255... + *dst_ptr++ = clip_pixel_highbd(temp >> AV1_FILTER_SHIFT, bd); + interm_ptr += intermediate_height; + } + interm_ptr += intermediate_next_stride; + dst_ptr += dst_next_row_stride; + } + } +} + +void highbd_block2d_average_c(uint16_t *src, unsigned int src_stride, + uint16_t *output_ptr, unsigned int output_stride, + unsigned int output_width, + unsigned int output_height) { + unsigned int i, j; + for (i = 0; i < output_height; ++i) { + for (j = 0; j < output_width; ++j) { + output_ptr[j] = (output_ptr[j] + src[i * src_stride + j] + 1) >> 1; + } + output_ptr += output_stride; + } +} + +void highbd_filter_average_block2d_8_c( + const uint16_t *src_ptr, unsigned int src_stride, const int16_t *HFilter, + const int16_t *VFilter, uint16_t *dst_ptr, unsigned int dst_stride, + unsigned int output_width, unsigned int output_height, int bd) { + uint16_t tmp[kMaxDimension * kMaxDimension]; + + assert(output_width <= kMaxDimension); + assert(output_height <= kMaxDimension); + highbd_filter_block2d_8_c(src_ptr, src_stride, HFilter, VFilter, tmp, + kMaxDimension, output_width, output_height, bd); + highbd_block2d_average_c(tmp, kMaxDimension, dst_ptr, dst_stride, + output_width, output_height); +} +#endif // CONFIG_HIGHBITDEPTH + +class ConvolveTest : public ::testing::TestWithParam { + public: + static void SetUpTestCase() { + // Force input_ to be unaligned, output to be 16 byte aligned. + input_ = reinterpret_cast( + aom_memalign(kDataAlignment, kInputBufferSize + 1)) + + 1; + output_ = reinterpret_cast( + aom_memalign(kDataAlignment, kOutputBufferSize)); + output_ref_ = reinterpret_cast( + aom_memalign(kDataAlignment, kOutputBufferSize)); +#if CONFIG_HIGHBITDEPTH + input16_ = reinterpret_cast(aom_memalign( + kDataAlignment, (kInputBufferSize + 1) * sizeof(uint16_t))) + + 1; + output16_ = reinterpret_cast( + aom_memalign(kDataAlignment, (kOutputBufferSize) * sizeof(uint16_t))); + output16_ref_ = reinterpret_cast( + aom_memalign(kDataAlignment, (kOutputBufferSize) * sizeof(uint16_t))); +#endif + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + static void TearDownTestCase() { + aom_free(input_ - 1); + input_ = NULL; + aom_free(output_); + output_ = NULL; + aom_free(output_ref_); + output_ref_ = NULL; +#if CONFIG_HIGHBITDEPTH + aom_free(input16_ - 1); + input16_ = NULL; + aom_free(output16_); + output16_ = NULL; + aom_free(output16_ref_); + output16_ref_ = NULL; +#endif + } + + protected: + static const int kDataAlignment = 16; + static const int kOuterBlockSize = 4 * kMaxDimension; + static const int kInputStride = kOuterBlockSize; + static const int kOutputStride = kOuterBlockSize; + static const int kInputBufferSize = kOuterBlockSize * kOuterBlockSize; + static const int kOutputBufferSize = kOuterBlockSize * kOuterBlockSize; + + int Width() const { return GET_PARAM(0); } + int Height() const { return GET_PARAM(1); } + int BorderLeft() const { + const int center = (kOuterBlockSize - Width()) / 2; + return (center + (kDataAlignment - 1)) & ~(kDataAlignment - 1); + } + int BorderTop() const { return (kOuterBlockSize - Height()) / 2; } + + bool IsIndexInBorder(int i) { + return (i < BorderTop() * kOuterBlockSize || + i >= (BorderTop() + Height()) * kOuterBlockSize || + i % kOuterBlockSize < BorderLeft() || + i % kOuterBlockSize >= (BorderLeft() + Width())); + } + + virtual void SetUp() { + UUT_ = GET_PARAM(2); +#if CONFIG_HIGHBITDEPTH + if (UUT_->use_highbd_ != 0) + mask_ = (1 << UUT_->use_highbd_) - 1; + else + mask_ = 255; +#endif + /* Set up guard blocks for an inner block centered in the outer block */ + for (int i = 0; i < kOutputBufferSize; ++i) { + if (IsIndexInBorder(i)) + output_[i] = 255; + else + output_[i] = 0; + } + + ::libaom_test::ACMRandom prng; + for (int i = 0; i < kInputBufferSize; ++i) { + if (i & 1) { + input_[i] = 255; +#if CONFIG_HIGHBITDEPTH + input16_[i] = mask_; +#endif + } else { + input_[i] = prng.Rand8Extremes(); +#if CONFIG_HIGHBITDEPTH + input16_[i] = prng.Rand16() & mask_; +#endif + } + } + } + + void SetConstantInput(int value) { + memset(input_, value, kInputBufferSize); +#if CONFIG_HIGHBITDEPTH + aom_memset16(input16_, value, kInputBufferSize); +#endif + } + + void CopyOutputToRef() { + memcpy(output_ref_, output_, kOutputBufferSize); +#if CONFIG_HIGHBITDEPTH + // Copy 16-bit pixels values. The effective number of bytes is double. + memcpy(output16_ref_, output16_, sizeof(output16_[0]) * kOutputBufferSize); +#endif + } + + void CheckGuardBlocks() { + for (int i = 0; i < kOutputBufferSize; ++i) { + if (IsIndexInBorder(i)) EXPECT_EQ(255, output_[i]); + } + } + + uint8_t *input() const { + const int offset = BorderTop() * kOuterBlockSize + BorderLeft(); +#if CONFIG_HIGHBITDEPTH + if (UUT_->use_highbd_ == 0) { + return input_ + offset; + } else { + return CONVERT_TO_BYTEPTR(input16_) + offset; + } +#else + return input_ + offset; +#endif + } + + uint8_t *output() const { + const int offset = BorderTop() * kOuterBlockSize + BorderLeft(); +#if CONFIG_HIGHBITDEPTH + if (UUT_->use_highbd_ == 0) { + return output_ + offset; + } else { + return CONVERT_TO_BYTEPTR(output16_) + offset; + } +#else + return output_ + offset; +#endif + } + + uint8_t *output_ref() const { + const int offset = BorderTop() * kOuterBlockSize + BorderLeft(); +#if CONFIG_HIGHBITDEPTH + if (UUT_->use_highbd_ == 0) { + return output_ref_ + offset; + } else { + return CONVERT_TO_BYTEPTR(output16_ref_) + offset; + } +#else + return output_ref_ + offset; +#endif + } + + uint16_t lookup(uint8_t *list, int index) const { +#if CONFIG_HIGHBITDEPTH + if (UUT_->use_highbd_ == 0) { + return list[index]; + } else { + return CONVERT_TO_SHORTPTR(list)[index]; + } +#else + return list[index]; +#endif + } + + void assign_val(uint8_t *list, int index, uint16_t val) const { +#if CONFIG_HIGHBITDEPTH + if (UUT_->use_highbd_ == 0) { + list[index] = (uint8_t)val; + } else { + CONVERT_TO_SHORTPTR(list)[index] = val; + } +#else + list[index] = (uint8_t)val; +#endif + } + + void wrapper_filter_average_block2d_8_c( + const uint8_t *src_ptr, unsigned int src_stride, const int16_t *HFilter, + const int16_t *VFilter, uint8_t *dst_ptr, unsigned int dst_stride, + unsigned int output_width, unsigned int output_height) { +#if CONFIG_HIGHBITDEPTH + if (UUT_->use_highbd_ == 0) { + filter_average_block2d_8_c(src_ptr, src_stride, HFilter, VFilter, dst_ptr, + dst_stride, output_width, output_height); + } else { + highbd_filter_average_block2d_8_c( + CONVERT_TO_SHORTPTR(src_ptr), src_stride, HFilter, VFilter, + CONVERT_TO_SHORTPTR(dst_ptr), dst_stride, output_width, output_height, + UUT_->use_highbd_); + } +#else + filter_average_block2d_8_c(src_ptr, src_stride, HFilter, VFilter, dst_ptr, + dst_stride, output_width, output_height); +#endif + } + + void wrapper_filter_block2d_8_c( + const uint8_t *src_ptr, unsigned int src_stride, const int16_t *HFilter, + const int16_t *VFilter, uint8_t *dst_ptr, unsigned int dst_stride, + unsigned int output_width, unsigned int output_height) { +#if CONFIG_HIGHBITDEPTH + if (UUT_->use_highbd_ == 0) { + filter_block2d_8_c(src_ptr, src_stride, HFilter, VFilter, dst_ptr, + dst_stride, output_width, output_height); + } else { + highbd_filter_block2d_8_c(CONVERT_TO_SHORTPTR(src_ptr), src_stride, + HFilter, VFilter, CONVERT_TO_SHORTPTR(dst_ptr), + dst_stride, output_width, output_height, + UUT_->use_highbd_); + } +#else + filter_block2d_8_c(src_ptr, src_stride, HFilter, VFilter, dst_ptr, + dst_stride, output_width, output_height); +#endif + } + + const ConvolveFunctions *UUT_; + static uint8_t *input_; + static uint8_t *output_; + static uint8_t *output_ref_; +#if CONFIG_HIGHBITDEPTH + static uint16_t *input16_; + static uint16_t *output16_; + static uint16_t *output16_ref_; + int mask_; +#endif +}; + +uint8_t *ConvolveTest::input_ = NULL; +uint8_t *ConvolveTest::output_ = NULL; +uint8_t *ConvolveTest::output_ref_ = NULL; +#if CONFIG_HIGHBITDEPTH +uint16_t *ConvolveTest::input16_ = NULL; +uint16_t *ConvolveTest::output16_ = NULL; +uint16_t *ConvolveTest::output16_ref_ = NULL; +#endif + +TEST_P(ConvolveTest, GuardBlocks) { CheckGuardBlocks(); } + +TEST_P(ConvolveTest, Copy) { + uint8_t *const in = input(); + uint8_t *const out = output(); + + ASM_REGISTER_STATE_CHECK(UUT_->copy_(in, kInputStride, out, kOutputStride, + NULL, 0, NULL, 0, Width(), Height())); + + CheckGuardBlocks(); + + for (int y = 0; y < Height(); ++y) + for (int x = 0; x < Width(); ++x) + ASSERT_EQ(lookup(out, y * kOutputStride + x), + lookup(in, y * kInputStride + x)) + << "(" << x << "," << y << ")"; +} + +TEST_P(ConvolveTest, Avg) { + uint8_t *const in = input(); + uint8_t *const out = output(); + uint8_t *const out_ref = output_ref(); + CopyOutputToRef(); + + ASM_REGISTER_STATE_CHECK(UUT_->avg_(in, kInputStride, out, kOutputStride, + NULL, 0, NULL, 0, Width(), Height())); + + CheckGuardBlocks(); + + for (int y = 0; y < Height(); ++y) + for (int x = 0; x < Width(); ++x) + ASSERT_EQ(lookup(out, y * kOutputStride + x), + ROUND_POWER_OF_TWO(lookup(in, y * kInputStride + x) + + lookup(out_ref, y * kOutputStride + x), + 1)) + << "(" << x << "," << y << ")"; +} + +TEST_P(ConvolveTest, CopyHoriz) { + uint8_t *const in = input(); + uint8_t *const out = output(); + DECLARE_ALIGNED(256, const int16_t, + filter8[8]) = { 0, 0, 0, 128, 0, 0, 0, 0 }; + + ASM_REGISTER_STATE_CHECK(UUT_->sh8_(in, kInputStride, out, kOutputStride, + filter8, 16, filter8, 16, Width(), + Height())); + + CheckGuardBlocks(); + + for (int y = 0; y < Height(); ++y) + for (int x = 0; x < Width(); ++x) + ASSERT_EQ(lookup(out, y * kOutputStride + x), + lookup(in, y * kInputStride + x)) + << "(" << x << "," << y << ")"; +} + +TEST_P(ConvolveTest, CopyVert) { + uint8_t *const in = input(); + uint8_t *const out = output(); + DECLARE_ALIGNED(256, const int16_t, + filter8[8]) = { 0, 0, 0, 128, 0, 0, 0, 0 }; + + ASM_REGISTER_STATE_CHECK(UUT_->sv8_(in, kInputStride, out, kOutputStride, + filter8, 16, filter8, 16, Width(), + Height())); + + CheckGuardBlocks(); + + for (int y = 0; y < Height(); ++y) + for (int x = 0; x < Width(); ++x) + ASSERT_EQ(lookup(out, y * kOutputStride + x), + lookup(in, y * kInputStride + x)) + << "(" << x << "," << y << ")"; +} + +TEST_P(ConvolveTest, Copy2D) { + uint8_t *const in = input(); + uint8_t *const out = output(); + DECLARE_ALIGNED(256, const int16_t, + filter8[8]) = { 0, 0, 0, 128, 0, 0, 0, 0 }; + + ASM_REGISTER_STATE_CHECK(UUT_->shv8_(in, kInputStride, out, kOutputStride, + filter8, 16, filter8, 16, Width(), + Height())); + + CheckGuardBlocks(); + + for (int y = 0; y < Height(); ++y) + for (int x = 0; x < Width(); ++x) + ASSERT_EQ(lookup(out, y * kOutputStride + x), + lookup(in, y * kInputStride + x)) + << "(" << x << "," << y << ")"; +} + +const int kNumFilterBanks = SWITCHABLE_FILTERS; +const int kNumFilters = 16; + +TEST(ConvolveTest, FiltersWontSaturateWhenAddedPairwise) { + for (int filter_bank = 0; filter_bank < kNumFilterBanks; ++filter_bank) { + const InterpFilter filter = (InterpFilter)filter_bank; + const InterpKernel *filters = + (const InterpKernel *)av1_get_interp_filter_kernel(filter); +#if CONFIG_DUAL_FILTER + const InterpFilterParams filter_params = + av1_get_interp_filter_params(filter); + if (filter_params.taps != SUBPEL_TAPS) continue; +#endif + for (int i = 0; i < kNumFilters; i++) { + const int p0 = filters[i][0] + filters[i][1]; + const int p1 = filters[i][2] + filters[i][3]; + const int p2 = filters[i][4] + filters[i][5]; + const int p3 = filters[i][6] + filters[i][7]; + EXPECT_LE(p0, 128); + EXPECT_LE(p1, 128); + EXPECT_LE(p2, 128); + EXPECT_LE(p3, 128); + EXPECT_LE(p0 + p3, 128); + EXPECT_LE(p0 + p3 + p1, 128); + EXPECT_LE(p0 + p3 + p1 + p2, 128); + EXPECT_EQ(p0 + p1 + p2 + p3, 128); + } + } +} + +const int16_t kInvalidFilter[8] = { 0 }; + +TEST_P(ConvolveTest, MatchesReferenceSubpixelFilter) { + uint8_t *const in = input(); + uint8_t *const out = output(); +#if CONFIG_HIGHBITDEPTH + uint8_t ref8[kOutputStride * kMaxDimension]; + uint16_t ref16[kOutputStride * kMaxDimension]; + uint8_t *ref; + if (UUT_->use_highbd_ == 0) { + ref = ref8; + } else { + ref = CONVERT_TO_BYTEPTR(ref16); + } +#else + uint8_t ref[kOutputStride * kMaxDimension]; +#endif + + for (int filter_bank = 0; filter_bank < kNumFilterBanks; ++filter_bank) { + const InterpFilter filter = (InterpFilter)filter_bank; + const InterpKernel *filters = + (const InterpKernel *)av1_get_interp_filter_kernel(filter); +#if CONFIG_DUAL_FILTER + const InterpFilterParams filter_params = + av1_get_interp_filter_params(filter); + if (filter_params.taps != SUBPEL_TAPS) continue; +#endif + + for (int filter_x = 0; filter_x < kNumFilters; ++filter_x) { + for (int filter_y = 0; filter_y < kNumFilters; ++filter_y) { + wrapper_filter_block2d_8_c(in, kInputStride, filters[filter_x], + filters[filter_y], ref, kOutputStride, + Width(), Height()); + + if (filter_x && filter_y) + ASM_REGISTER_STATE_CHECK(UUT_->hv8_( + in, kInputStride, out, kOutputStride, filters[filter_x], 16, + filters[filter_y], 16, Width(), Height())); + else if (filter_y) + ASM_REGISTER_STATE_CHECK( + UUT_->v8_(in, kInputStride, out, kOutputStride, kInvalidFilter, + 16, filters[filter_y], 16, Width(), Height())); + else if (filter_x) + ASM_REGISTER_STATE_CHECK( + UUT_->h8_(in, kInputStride, out, kOutputStride, filters[filter_x], + 16, kInvalidFilter, 16, Width(), Height())); + else + ASM_REGISTER_STATE_CHECK( + UUT_->copy_(in, kInputStride, out, kOutputStride, kInvalidFilter, + 0, kInvalidFilter, 0, Width(), Height())); + + CheckGuardBlocks(); + + for (int y = 0; y < Height(); ++y) + for (int x = 0; x < Width(); ++x) + ASSERT_EQ(lookup(ref, y * kOutputStride + x), + lookup(out, y * kOutputStride + x)) + << "mismatch at (" << x << "," << y << "), " + << "filters (" << filter_bank << "," << filter_x << "," + << filter_y << ")"; + } + } + } +} + +TEST_P(ConvolveTest, MatchesReferenceAveragingSubpixelFilter) { + uint8_t *const in = input(); + uint8_t *const out = output(); +#if CONFIG_HIGHBITDEPTH + uint8_t ref8[kOutputStride * kMaxDimension]; + uint16_t ref16[kOutputStride * kMaxDimension]; + uint8_t *ref; + if (UUT_->use_highbd_ == 0) { + ref = ref8; + } else { + ref = CONVERT_TO_BYTEPTR(ref16); + } +#else + uint8_t ref[kOutputStride * kMaxDimension]; +#endif + + // Populate ref and out with some random data + ::libaom_test::ACMRandom prng; + for (int y = 0; y < Height(); ++y) { + for (int x = 0; x < Width(); ++x) { + uint16_t r; +#if CONFIG_HIGHBITDEPTH + if (UUT_->use_highbd_ == 0 || UUT_->use_highbd_ == 8) { + r = prng.Rand8Extremes(); + } else { + r = prng.Rand16() & mask_; + } +#else + r = prng.Rand8Extremes(); +#endif + + assign_val(out, y * kOutputStride + x, r); + assign_val(ref, y * kOutputStride + x, r); + } + } + + for (int filter_bank = 0; filter_bank < kNumFilterBanks; ++filter_bank) { + const InterpFilter filter = (InterpFilter)filter_bank; + const InterpKernel *filters = + (const InterpKernel *)av1_get_interp_filter_kernel(filter); +#if CONFIG_DUAL_FILTER + const InterpFilterParams filter_params = + av1_get_interp_filter_params(filter); + if (filter_params.taps != SUBPEL_TAPS) continue; +#endif + + for (int filter_x = 0; filter_x < kNumFilters; ++filter_x) { + for (int filter_y = 0; filter_y < kNumFilters; ++filter_y) { + wrapper_filter_average_block2d_8_c(in, kInputStride, filters[filter_x], + filters[filter_y], ref, + kOutputStride, Width(), Height()); + + if (filter_x && filter_y) + ASM_REGISTER_STATE_CHECK(UUT_->hv8_avg_( + in, kInputStride, out, kOutputStride, filters[filter_x], 16, + filters[filter_y], 16, Width(), Height())); + else if (filter_y) + ASM_REGISTER_STATE_CHECK(UUT_->v8_avg_( + in, kInputStride, out, kOutputStride, kInvalidFilter, 16, + filters[filter_y], 16, Width(), Height())); + else if (filter_x) + ASM_REGISTER_STATE_CHECK(UUT_->h8_avg_( + in, kInputStride, out, kOutputStride, filters[filter_x], 16, + kInvalidFilter, 16, Width(), Height())); + else + ASM_REGISTER_STATE_CHECK( + UUT_->avg_(in, kInputStride, out, kOutputStride, kInvalidFilter, + 0, kInvalidFilter, 0, Width(), Height())); + + CheckGuardBlocks(); + + for (int y = 0; y < Height(); ++y) + for (int x = 0; x < Width(); ++x) + ASSERT_EQ(lookup(ref, y * kOutputStride + x), + lookup(out, y * kOutputStride + x)) + << "mismatch at (" << x << "," << y << "), " + << "filters (" << filter_bank << "," << filter_x << "," + << filter_y << ")"; + } + } + } +} + +TEST_P(ConvolveTest, FilterExtremes) { + uint8_t *const in = input(); + uint8_t *const out = output(); +#if CONFIG_HIGHBITDEPTH + uint8_t ref8[kOutputStride * kMaxDimension]; + uint16_t ref16[kOutputStride * kMaxDimension]; + uint8_t *ref; + if (UUT_->use_highbd_ == 0) { + ref = ref8; + } else { + ref = CONVERT_TO_BYTEPTR(ref16); + } +#else + uint8_t ref[kOutputStride * kMaxDimension]; +#endif + + // Populate ref and out with some random data + ::libaom_test::ACMRandom prng; + for (int y = 0; y < Height(); ++y) { + for (int x = 0; x < Width(); ++x) { + uint16_t r; +#if CONFIG_HIGHBITDEPTH + if (UUT_->use_highbd_ == 0 || UUT_->use_highbd_ == 8) { + r = prng.Rand8Extremes(); + } else { + r = prng.Rand16() & mask_; + } +#else + r = prng.Rand8Extremes(); +#endif + assign_val(out, y * kOutputStride + x, r); + assign_val(ref, y * kOutputStride + x, r); + } + } + + for (int axis = 0; axis < 2; axis++) { + int seed_val = 0; + while (seed_val < 256) { + for (int y = 0; y < 8; ++y) { + for (int x = 0; x < 8; ++x) { +#if CONFIG_HIGHBITDEPTH + assign_val(in, y * kOutputStride + x - SUBPEL_TAPS / 2 + 1, + ((seed_val >> (axis ? y : x)) & 1) * mask_); +#else + assign_val(in, y * kOutputStride + x - SUBPEL_TAPS / 2 + 1, + ((seed_val >> (axis ? y : x)) & 1) * 255); +#endif + if (axis) seed_val++; + } + if (axis) + seed_val -= 8; + else + seed_val++; + } + if (axis) seed_val += 8; + + for (int filter_bank = 0; filter_bank < kNumFilterBanks; ++filter_bank) { + const InterpFilter filter = (InterpFilter)filter_bank; + const InterpKernel *filters = + (const InterpKernel *)av1_get_interp_filter_kernel(filter); +#if CONFIG_DUAL_FILTER + const InterpFilterParams filter_params = + av1_get_interp_filter_params(filter); + if (filter_params.taps != SUBPEL_TAPS) continue; +#endif + for (int filter_x = 0; filter_x < kNumFilters; ++filter_x) { + for (int filter_y = 0; filter_y < kNumFilters; ++filter_y) { + wrapper_filter_block2d_8_c(in, kInputStride, filters[filter_x], + filters[filter_y], ref, kOutputStride, + Width(), Height()); + if (filter_x && filter_y) + ASM_REGISTER_STATE_CHECK(UUT_->hv8_( + in, kInputStride, out, kOutputStride, filters[filter_x], 16, + filters[filter_y], 16, Width(), Height())); + else if (filter_y) + ASM_REGISTER_STATE_CHECK(UUT_->v8_( + in, kInputStride, out, kOutputStride, kInvalidFilter, 16, + filters[filter_y], 16, Width(), Height())); + else if (filter_x) + ASM_REGISTER_STATE_CHECK(UUT_->h8_( + in, kInputStride, out, kOutputStride, filters[filter_x], 16, + kInvalidFilter, 16, Width(), Height())); + else + ASM_REGISTER_STATE_CHECK(UUT_->copy_( + in, kInputStride, out, kOutputStride, kInvalidFilter, 0, + kInvalidFilter, 0, Width(), Height())); + + for (int y = 0; y < Height(); ++y) + for (int x = 0; x < Width(); ++x) + ASSERT_EQ(lookup(ref, y * kOutputStride + x), + lookup(out, y * kOutputStride + x)) + << "mismatch at (" << x << "," << y << "), " + << "filters (" << filter_bank << "," << filter_x << "," + << filter_y << ")"; + } + } + } + } + } +} + +/* This test exercises that enough rows and columns are filtered with every + possible initial fractional positions and scaling steps. */ +TEST_P(ConvolveTest, CheckScalingFiltering) { + uint8_t *const in = input(); + uint8_t *const out = output(); + const InterpKernel *const eighttap = + (const InterpKernel *)av1_get_interp_filter_kernel(EIGHTTAP_REGULAR); + + SetConstantInput(127); + + for (int frac = 0; frac < 16; ++frac) { + for (int step = 1; step <= 32; ++step) { + /* Test the horizontal and vertical filters in combination. */ + ASM_REGISTER_STATE_CHECK(UUT_->shv8_(in, kInputStride, out, kOutputStride, + eighttap[frac], step, eighttap[frac], + step, Width(), Height())); + + CheckGuardBlocks(); + + for (int y = 0; y < Height(); ++y) { + for (int x = 0; x < Width(); ++x) { + ASSERT_EQ(lookup(in, y * kInputStride + x), + lookup(out, y * kOutputStride + x)) + << "x == " << x << ", y == " << y << ", frac == " << frac + << ", step == " << step; + } + } + } + } +} + +TEST_P(ConvolveTest, DISABLED_Copy_Speed) { + const uint8_t *const in = input(); + uint8_t *const out = output(); + const int kNumTests = 5000000; + const int width = Width(); + const int height = Height(); + aom_usec_timer timer; + + aom_usec_timer_start(&timer); + for (int n = 0; n < kNumTests; ++n) { + UUT_->copy_(in, kInputStride, out, kOutputStride, NULL, 0, NULL, 0, width, + height); + } + aom_usec_timer_mark(&timer); + + const int elapsed_time = static_cast(aom_usec_timer_elapsed(&timer)); + printf("convolve_copy_%dx%d_%d: %d us\n", width, height, + UUT_->use_highbd_ ? UUT_->use_highbd_ : 8, elapsed_time); +} + +TEST_P(ConvolveTest, DISABLED_Avg_Speed) { + const uint8_t *const in = input(); + uint8_t *const out = output(); + const int kNumTests = 5000000; + const int width = Width(); + const int height = Height(); + aom_usec_timer timer; + + aom_usec_timer_start(&timer); + for (int n = 0; n < kNumTests; ++n) { + UUT_->avg_(in, kInputStride, out, kOutputStride, NULL, 0, NULL, 0, width, + height); + } + aom_usec_timer_mark(&timer); + + const int elapsed_time = static_cast(aom_usec_timer_elapsed(&timer)); + printf("convolve_avg_%dx%d_%d: %d us\n", width, height, + UUT_->use_highbd_ ? UUT_->use_highbd_ : 8, elapsed_time); +} + +TEST_P(ConvolveTest, DISABLED_Speed) { + uint8_t *const in = input(); + uint8_t *const out = output(); +#if CONFIG_HIGHBITDEPTH + uint8_t ref8[kOutputStride * kMaxDimension]; + uint16_t ref16[kOutputStride * kMaxDimension]; + uint8_t *ref; + if (UUT_->use_highbd_ == 0) { + ref = ref8; + } else { + ref = CONVERT_TO_BYTEPTR(ref16); + } +#else + uint8_t ref[kOutputStride * kMaxDimension]; +#endif + + // Populate ref and out with some random data + ::libaom_test::ACMRandom prng; + for (int y = 0; y < Height(); ++y) { + for (int x = 0; x < Width(); ++x) { + uint16_t r; +#if CONFIG_HIGHBITDEPTH + if (UUT_->use_highbd_ == 0 || UUT_->use_highbd_ == 8) { + r = prng.Rand8Extremes(); + } else { + r = prng.Rand16() & mask_; + } +#else + r = prng.Rand8Extremes(); +#endif + + assign_val(out, y * kOutputStride + x, r); + assign_val(ref, y * kOutputStride + x, r); + } + } + + const InterpFilter filter = (InterpFilter)1; + const InterpKernel *filters = + (const InterpKernel *)av1_get_interp_filter_kernel(filter); + wrapper_filter_average_block2d_8_c(in, kInputStride, filters[1], filters[1], + out, kOutputStride, Width(), Height()); + + aom_usec_timer timer; + int tests_num = 1000; + + aom_usec_timer_start(&timer); + while (tests_num > 0) { + for (int filter_bank = 0; filter_bank < kNumFilterBanks; ++filter_bank) { + const InterpFilter filter = (InterpFilter)filter_bank; + const InterpKernel *filters = + (const InterpKernel *)av1_get_interp_filter_kernel(filter); +#if CONFIG_DUAL_FILTER + const InterpFilterParams filter_params = + av1_get_interp_filter_params(filter); + if (filter_params.taps != SUBPEL_TAPS) continue; +#endif + + for (int filter_x = 0; filter_x < kNumFilters; ++filter_x) { + for (int filter_y = 0; filter_y < kNumFilters; ++filter_y) { + if (filter_x && filter_y) + ASM_REGISTER_STATE_CHECK(UUT_->hv8_( + in, kInputStride, out, kOutputStride, filters[filter_x], 16, + filters[filter_y], 16, Width(), Height())); + if (filter_y) + ASM_REGISTER_STATE_CHECK( + UUT_->v8_(in, kInputStride, out, kOutputStride, kInvalidFilter, + 16, filters[filter_y], 16, Width(), Height())); + else if (filter_x) + ASM_REGISTER_STATE_CHECK(UUT_->h8_( + in, kInputStride, out, kOutputStride, filters[filter_x], 16, + kInvalidFilter, 16, Width(), Height())); + } + } + } + tests_num--; + } + aom_usec_timer_mark(&timer); + + const int elapsed_time = + static_cast(aom_usec_timer_elapsed(&timer) / 1000); + printf("%dx%d (bitdepth %d) time: %5d ms\n", Width(), Height(), + UUT_->use_highbd_, elapsed_time); +} + +using std::tr1::make_tuple; + +#if CONFIG_HIGHBITDEPTH +#define WRAP(func, bd) \ + void wrap_##func##_##bd( \ + const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \ + ptrdiff_t dst_stride, const int16_t *filter_x, int filter_x_stride, \ + const int16_t *filter_y, int filter_y_stride, int w, int h) { \ + aom_highbd_##func(src, src_stride, dst, dst_stride, filter_x, \ + filter_x_stride, filter_y, filter_y_stride, w, h, bd); \ + } +#if HAVE_SSE2 && ARCH_X86_64 +WRAP(convolve_copy_sse2, 8) +WRAP(convolve_avg_sse2, 8) +WRAP(convolve_copy_sse2, 10) +WRAP(convolve_avg_sse2, 10) +WRAP(convolve_copy_sse2, 12) +WRAP(convolve_avg_sse2, 12) +WRAP(convolve8_horiz_sse2, 8) +WRAP(convolve8_avg_horiz_sse2, 8) +WRAP(convolve8_vert_sse2, 8) +WRAP(convolve8_avg_vert_sse2, 8) +WRAP(convolve8_sse2, 8) +WRAP(convolve8_avg_sse2, 8) +WRAP(convolve8_horiz_sse2, 10) +WRAP(convolve8_avg_horiz_sse2, 10) +WRAP(convolve8_vert_sse2, 10) +WRAP(convolve8_avg_vert_sse2, 10) +WRAP(convolve8_sse2, 10) +WRAP(convolve8_avg_sse2, 10) +WRAP(convolve8_horiz_sse2, 12) +WRAP(convolve8_avg_horiz_sse2, 12) +WRAP(convolve8_vert_sse2, 12) +WRAP(convolve8_avg_vert_sse2, 12) +WRAP(convolve8_sse2, 12) +WRAP(convolve8_avg_sse2, 12) +#endif // HAVE_SSE2 && ARCH_X86_64 + +WRAP(convolve_copy_c, 8) +WRAP(convolve_avg_c, 8) +WRAP(convolve8_horiz_c, 8) +WRAP(convolve8_avg_horiz_c, 8) +WRAP(convolve8_vert_c, 8) +WRAP(convolve8_avg_vert_c, 8) +WRAP(convolve8_c, 8) +WRAP(convolve8_avg_c, 8) +WRAP(convolve_copy_c, 10) +WRAP(convolve_avg_c, 10) +WRAP(convolve8_horiz_c, 10) +WRAP(convolve8_avg_horiz_c, 10) +WRAP(convolve8_vert_c, 10) +WRAP(convolve8_avg_vert_c, 10) +WRAP(convolve8_c, 10) +WRAP(convolve8_avg_c, 10) +WRAP(convolve_copy_c, 12) +WRAP(convolve_avg_c, 12) +WRAP(convolve8_horiz_c, 12) +WRAP(convolve8_avg_horiz_c, 12) +WRAP(convolve8_vert_c, 12) +WRAP(convolve8_avg_vert_c, 12) +WRAP(convolve8_c, 12) +WRAP(convolve8_avg_c, 12) + +#if HAVE_AVX2 +WRAP(convolve_copy_avx2, 8) +WRAP(convolve_avg_avx2, 8) +WRAP(convolve8_horiz_avx2, 8) +WRAP(convolve8_avg_horiz_avx2, 8) +WRAP(convolve8_vert_avx2, 8) +WRAP(convolve8_avg_vert_avx2, 8) +WRAP(convolve8_avx2, 8) +WRAP(convolve8_avg_avx2, 8) + +WRAP(convolve_copy_avx2, 10) +WRAP(convolve_avg_avx2, 10) +WRAP(convolve8_avx2, 10) +WRAP(convolve8_horiz_avx2, 10) +WRAP(convolve8_vert_avx2, 10) +WRAP(convolve8_avg_avx2, 10) +WRAP(convolve8_avg_horiz_avx2, 10) +WRAP(convolve8_avg_vert_avx2, 10) + +WRAP(convolve_copy_avx2, 12) +WRAP(convolve_avg_avx2, 12) +WRAP(convolve8_avx2, 12) +WRAP(convolve8_horiz_avx2, 12) +WRAP(convolve8_vert_avx2, 12) +WRAP(convolve8_avg_avx2, 12) +WRAP(convolve8_avg_horiz_avx2, 12) +WRAP(convolve8_avg_vert_avx2, 12) +#endif // HAVE_AVX2 + +#undef WRAP + +const ConvolveFunctions convolve8_c( + wrap_convolve_copy_c_8, wrap_convolve_avg_c_8, wrap_convolve8_horiz_c_8, + wrap_convolve8_avg_horiz_c_8, wrap_convolve8_vert_c_8, + wrap_convolve8_avg_vert_c_8, wrap_convolve8_c_8, wrap_convolve8_avg_c_8, + wrap_convolve8_horiz_c_8, wrap_convolve8_avg_horiz_c_8, + wrap_convolve8_vert_c_8, wrap_convolve8_avg_vert_c_8, wrap_convolve8_c_8, + wrap_convolve8_avg_c_8, 8); +const ConvolveFunctions convolve10_c( + wrap_convolve_copy_c_10, wrap_convolve_avg_c_10, wrap_convolve8_horiz_c_10, + wrap_convolve8_avg_horiz_c_10, wrap_convolve8_vert_c_10, + wrap_convolve8_avg_vert_c_10, wrap_convolve8_c_10, wrap_convolve8_avg_c_10, + wrap_convolve8_horiz_c_10, wrap_convolve8_avg_horiz_c_10, + wrap_convolve8_vert_c_10, wrap_convolve8_avg_vert_c_10, wrap_convolve8_c_10, + wrap_convolve8_avg_c_10, 10); +const ConvolveFunctions convolve12_c( + wrap_convolve_copy_c_12, wrap_convolve_avg_c_12, wrap_convolve8_horiz_c_12, + wrap_convolve8_avg_horiz_c_12, wrap_convolve8_vert_c_12, + wrap_convolve8_avg_vert_c_12, wrap_convolve8_c_12, wrap_convolve8_avg_c_12, + wrap_convolve8_horiz_c_12, wrap_convolve8_avg_horiz_c_12, + wrap_convolve8_vert_c_12, wrap_convolve8_avg_vert_c_12, wrap_convolve8_c_12, + wrap_convolve8_avg_c_12, 12); +const ConvolveParam kArrayConvolve_c[] = { + ALL_SIZES(convolve8_c), ALL_SIZES(convolve10_c), ALL_SIZES(convolve12_c) +}; + +#else +const ConvolveFunctions convolve8_c( + aom_convolve_copy_c, aom_convolve_avg_c, aom_convolve8_horiz_c, + aom_convolve8_avg_horiz_c, aom_convolve8_vert_c, aom_convolve8_avg_vert_c, + aom_convolve8_c, aom_convolve8_avg_c, aom_scaled_horiz_c, + aom_scaled_avg_horiz_c, aom_scaled_vert_c, aom_scaled_avg_vert_c, + aom_scaled_2d_c, aom_scaled_avg_2d_c, 0); +const ConvolveParam kArrayConvolve_c[] = { ALL_SIZES(convolve8_c) }; +#endif +INSTANTIATE_TEST_CASE_P(C, ConvolveTest, ::testing::ValuesIn(kArrayConvolve_c)); + +#if HAVE_SSE2 && ARCH_X86_64 +#if CONFIG_HIGHBITDEPTH +const ConvolveFunctions convolve8_sse2( + wrap_convolve_copy_sse2_8, wrap_convolve_avg_sse2_8, + wrap_convolve8_horiz_sse2_8, wrap_convolve8_avg_horiz_sse2_8, + wrap_convolve8_vert_sse2_8, wrap_convolve8_avg_vert_sse2_8, + wrap_convolve8_sse2_8, wrap_convolve8_avg_sse2_8, + wrap_convolve8_horiz_sse2_8, wrap_convolve8_avg_horiz_sse2_8, + wrap_convolve8_vert_sse2_8, wrap_convolve8_avg_vert_sse2_8, + wrap_convolve8_sse2_8, wrap_convolve8_avg_sse2_8, 8); +const ConvolveFunctions convolve10_sse2( + wrap_convolve_copy_sse2_10, wrap_convolve_avg_sse2_10, + wrap_convolve8_horiz_sse2_10, wrap_convolve8_avg_horiz_sse2_10, + wrap_convolve8_vert_sse2_10, wrap_convolve8_avg_vert_sse2_10, + wrap_convolve8_sse2_10, wrap_convolve8_avg_sse2_10, + wrap_convolve8_horiz_sse2_10, wrap_convolve8_avg_horiz_sse2_10, + wrap_convolve8_vert_sse2_10, wrap_convolve8_avg_vert_sse2_10, + wrap_convolve8_sse2_10, wrap_convolve8_avg_sse2_10, 10); +const ConvolveFunctions convolve12_sse2( + wrap_convolve_copy_sse2_12, wrap_convolve_avg_sse2_12, + wrap_convolve8_horiz_sse2_12, wrap_convolve8_avg_horiz_sse2_12, + wrap_convolve8_vert_sse2_12, wrap_convolve8_avg_vert_sse2_12, + wrap_convolve8_sse2_12, wrap_convolve8_avg_sse2_12, + wrap_convolve8_horiz_sse2_12, wrap_convolve8_avg_horiz_sse2_12, + wrap_convolve8_vert_sse2_12, wrap_convolve8_avg_vert_sse2_12, + wrap_convolve8_sse2_12, wrap_convolve8_avg_sse2_12, 12); +const ConvolveParam kArrayConvolve_sse2[] = { ALL_SIZES(convolve8_sse2), + ALL_SIZES(convolve10_sse2), + ALL_SIZES(convolve12_sse2) }; +#else +const ConvolveFunctions convolve8_sse2( + aom_convolve_copy_sse2, aom_convolve_avg_sse2, aom_convolve8_horiz_sse2, + aom_convolve8_avg_horiz_sse2, aom_convolve8_vert_sse2, + aom_convolve8_avg_vert_sse2, aom_convolve8_sse2, aom_convolve8_avg_sse2, + aom_scaled_horiz_c, aom_scaled_avg_horiz_c, aom_scaled_vert_c, + aom_scaled_avg_vert_c, aom_scaled_2d_c, aom_scaled_avg_2d_c, 0); + +const ConvolveParam kArrayConvolve_sse2[] = { ALL_SIZES(convolve8_sse2) }; +#endif // CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(SSE2, ConvolveTest, + ::testing::ValuesIn(kArrayConvolve_sse2)); +#endif + +#if HAVE_SSSE3 +const ConvolveFunctions convolve8_ssse3( + aom_convolve_copy_c, aom_convolve_avg_c, aom_convolve8_horiz_ssse3, + aom_convolve8_avg_horiz_ssse3, aom_convolve8_vert_ssse3, + aom_convolve8_avg_vert_ssse3, aom_convolve8_ssse3, aom_convolve8_avg_ssse3, + aom_scaled_horiz_c, aom_scaled_avg_horiz_c, aom_scaled_vert_c, + aom_scaled_avg_vert_c, aom_scaled_2d_ssse3, aom_scaled_avg_2d_c, 0); + +const ConvolveParam kArrayConvolve8_ssse3[] = { ALL_SIZES(convolve8_ssse3) }; +INSTANTIATE_TEST_CASE_P(SSSE3, ConvolveTest, + ::testing::ValuesIn(kArrayConvolve8_ssse3)); +#endif + +#if HAVE_AVX2 +#if CONFIG_HIGHBITDEPTH +const ConvolveFunctions convolve8_avx2( + wrap_convolve_copy_avx2_8, wrap_convolve_avg_avx2_8, + wrap_convolve8_horiz_avx2_8, wrap_convolve8_avg_horiz_avx2_8, + wrap_convolve8_vert_avx2_8, wrap_convolve8_avg_vert_avx2_8, + wrap_convolve8_avx2_8, wrap_convolve8_avg_avx2_8, wrap_convolve8_horiz_c_8, + wrap_convolve8_avg_horiz_c_8, wrap_convolve8_vert_c_8, + wrap_convolve8_avg_vert_c_8, wrap_convolve8_c_8, wrap_convolve8_avg_c_8, 8); +const ConvolveFunctions convolve10_avx2( + wrap_convolve_copy_avx2_10, wrap_convolve_avg_avx2_10, + wrap_convolve8_horiz_avx2_10, wrap_convolve8_avg_horiz_avx2_10, + wrap_convolve8_vert_avx2_10, wrap_convolve8_avg_vert_avx2_10, + wrap_convolve8_avx2_10, wrap_convolve8_avg_avx2_10, + wrap_convolve8_horiz_c_10, wrap_convolve8_avg_horiz_c_10, + wrap_convolve8_vert_c_10, wrap_convolve8_avg_vert_c_10, wrap_convolve8_c_10, + wrap_convolve8_avg_c_10, 10); +const ConvolveFunctions convolve12_avx2( + wrap_convolve_copy_avx2_12, wrap_convolve_avg_avx2_12, + wrap_convolve8_horiz_avx2_12, wrap_convolve8_avg_horiz_avx2_12, + wrap_convolve8_vert_avx2_12, wrap_convolve8_avg_vert_avx2_12, + wrap_convolve8_avx2_12, wrap_convolve8_avg_avx2_12, + wrap_convolve8_horiz_c_12, wrap_convolve8_avg_horiz_c_12, + wrap_convolve8_vert_c_12, wrap_convolve8_avg_vert_c_12, wrap_convolve8_c_12, + wrap_convolve8_avg_c_12, 12); +const ConvolveParam kArrayConvolve8_avx2[] = { ALL_SIZES(convolve8_avx2), + ALL_SIZES(convolve10_avx2), + ALL_SIZES(convolve12_avx2) }; +#else +const ConvolveFunctions convolve8_avx2( + aom_convolve_copy_c, aom_convolve_avg_c, aom_convolve8_horiz_avx2, + aom_convolve8_avg_horiz_ssse3, aom_convolve8_vert_avx2, + aom_convolve8_avg_vert_ssse3, aom_convolve8_avx2, aom_convolve8_avg_ssse3, + aom_scaled_horiz_c, aom_scaled_avg_horiz_c, aom_scaled_vert_c, + aom_scaled_avg_vert_c, aom_scaled_2d_c, aom_scaled_avg_2d_c, 0); + +const ConvolveParam kArrayConvolve8_avx2[] = { ALL_SIZES(convolve8_avx2) }; +#endif // CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(AVX2, ConvolveTest, + ::testing::ValuesIn(kArrayConvolve8_avx2)); +#endif // HAVE_AVX2 + +// TODO(any): Make NEON versions support 128x128 128x64 64x128 block sizes +#if HAVE_NEON && !(CONFIG_AV1 && CONFIG_EXT_PARTITION) +#if HAVE_NEON_ASM +const ConvolveFunctions convolve8_neon( + aom_convolve_copy_neon, aom_convolve_avg_neon, aom_convolve8_horiz_neon, + aom_convolve8_avg_horiz_neon, aom_convolve8_vert_neon, + aom_convolve8_avg_vert_neon, aom_convolve8_neon, aom_convolve8_avg_neon, + aom_scaled_horiz_c, aom_scaled_avg_horiz_c, aom_scaled_vert_c, + aom_scaled_avg_vert_c, aom_scaled_2d_c, aom_scaled_avg_2d_c, 0); +#else // HAVE_NEON +const ConvolveFunctions convolve8_neon( + aom_convolve_copy_neon, aom_convolve_avg_neon, aom_convolve8_horiz_neon, + aom_convolve8_avg_horiz_neon, aom_convolve8_vert_neon, + aom_convolve8_avg_vert_neon, aom_convolve8_neon, aom_convolve8_avg_neon, + aom_scaled_horiz_c, aom_scaled_avg_horiz_c, aom_scaled_vert_c, + aom_scaled_avg_vert_c, aom_scaled_2d_c, aom_scaled_avg_2d_c, 0); +#endif // HAVE_NEON_ASM + +const ConvolveParam kArrayConvolve8_neon[] = { ALL_SIZES(convolve8_neon) }; +INSTANTIATE_TEST_CASE_P(NEON, ConvolveTest, + ::testing::ValuesIn(kArrayConvolve8_neon)); +#endif // HAVE_NEON + +// TODO(any): Make DSPR2 versions support 128x128 128x64 64x128 block sizes +#if HAVE_DSPR2 && !(CONFIG_AV1 && CONFIG_EXT_PARTITION) +const ConvolveFunctions convolve8_dspr2( + aom_convolve_copy_dspr2, aom_convolve_avg_dspr2, aom_convolve8_horiz_dspr2, + aom_convolve8_avg_horiz_dspr2, aom_convolve8_vert_dspr2, + aom_convolve8_avg_vert_dspr2, aom_convolve8_dspr2, aom_convolve8_avg_dspr2, + aom_scaled_horiz_c, aom_scaled_avg_horiz_c, aom_scaled_vert_c, + aom_scaled_avg_vert_c, aom_scaled_2d_c, aom_scaled_avg_2d_c, 0); + +const ConvolveParam kArrayConvolve8_dspr2[] = { ALL_SIZES(convolve8_dspr2) }; +INSTANTIATE_TEST_CASE_P(DSPR2, ConvolveTest, + ::testing::ValuesIn(kArrayConvolve8_dspr2)); +#endif // HAVE_DSPR2 + +// TODO(any): Make MSA versions support 128x128 128x64 64x128 block sizes +#if HAVE_MSA && !(CONFIG_AV1 && CONFIG_EXT_PARTITION) +const ConvolveFunctions convolve8_msa( + aom_convolve_copy_msa, aom_convolve_avg_msa, aom_convolve8_horiz_msa, + aom_convolve8_avg_horiz_msa, aom_convolve8_vert_msa, + aom_convolve8_avg_vert_msa, aom_convolve8_msa, aom_convolve8_avg_msa, + aom_scaled_horiz_c, aom_scaled_avg_horiz_c, aom_scaled_vert_c, + aom_scaled_avg_vert_c, aom_scaled_2d_c, aom_scaled_avg_2d_c, 0); + +const ConvolveParam kArrayConvolve8_msa[] = { ALL_SIZES(convolve8_msa) }; +INSTANTIATE_TEST_CASE_P(MSA, ConvolveTest, + ::testing::ValuesIn(kArrayConvolve8_msa)); +#endif // HAVE_MSA +} // namespace diff --git a/third_party/aom/test/cpu_speed_test.cc b/third_party/aom/test/cpu_speed_test.cc new file mode 100644 index 0000000000..9b79664627 --- /dev/null +++ b/third_party/aom/test/cpu_speed_test.cc @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" +#include "test/y4m_video_source.h" + +namespace { + +const int kMaxPSNR = 100; + +class CpuSpeedTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params { + protected: + CpuSpeedTest() + : EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)), + set_cpu_used_(GET_PARAM(2)), min_psnr_(kMaxPSNR), + tune_content_(AOM_CONTENT_DEFAULT) {} + virtual ~CpuSpeedTest() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + if (encoding_mode_ != ::libaom_test::kRealTime) { + cfg_.g_lag_in_frames = 25; + cfg_.rc_end_usage = AOM_VBR; + } else { + cfg_.g_lag_in_frames = 0; + cfg_.rc_end_usage = AOM_CBR; + } + } + + virtual void BeginPassHook(unsigned int /*pass*/) { min_psnr_ = kMaxPSNR; } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 1) { + encoder->Control(AOME_SET_CPUUSED, set_cpu_used_); + encoder->Control(AV1E_SET_TUNE_CONTENT, tune_content_); + if (encoding_mode_ != ::libaom_test::kRealTime) { + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); + encoder->Control(AOME_SET_ARNR_MAXFRAMES, 7); + encoder->Control(AOME_SET_ARNR_STRENGTH, 5); + } + } + } + + virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) { + if (pkt->data.psnr.psnr[0] < min_psnr_) min_psnr_ = pkt->data.psnr.psnr[0]; + } + + void TestQ0(); + void TestScreencastQ0(); + void TestTuneScreen(); + void TestEncodeHighBitrate(); + void TestLowBitrate(); + + ::libaom_test::TestMode encoding_mode_; + int set_cpu_used_; + double min_psnr_; + int tune_content_; +}; + +void CpuSpeedTest::TestQ0() { + // Validate that this non multiple of 64 wide clip encodes and decodes + // without a mismatch when passing in a very low max q. This pushes + // the encoder to producing lots of big partitions which will likely + // extend into the border and test the border condition. + cfg_.rc_2pass_vbr_minsection_pct = 5; + cfg_.rc_2pass_vbr_maxsection_pct = 2000; + cfg_.rc_target_bitrate = 400; + cfg_.rc_max_quantizer = 0; + cfg_.rc_min_quantizer = 0; + + ::libaom_test::I420VideoSource video("hantro_odd.yuv", 208, 144, 30, 1, 0, + 10); + + init_flags_ = AOM_CODEC_USE_PSNR; + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + EXPECT_GE(min_psnr_, kMaxPSNR); +} + +void CpuSpeedTest::TestScreencastQ0() { + ::libaom_test::Y4mVideoSource video("screendata.y4m", 0, 10); + cfg_.g_timebase = video.timebase(); + cfg_.rc_2pass_vbr_minsection_pct = 5; + cfg_.rc_2pass_vbr_maxsection_pct = 2000; + cfg_.rc_target_bitrate = 400; + cfg_.rc_max_quantizer = 0; + cfg_.rc_min_quantizer = 0; + + init_flags_ = AOM_CODEC_USE_PSNR; + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + EXPECT_GE(min_psnr_, kMaxPSNR); +} + +void CpuSpeedTest::TestTuneScreen() { + ::libaom_test::Y4mVideoSource video("screendata.y4m", 0, 10); + cfg_.g_timebase = video.timebase(); + cfg_.rc_2pass_vbr_minsection_pct = 5; + cfg_.rc_2pass_vbr_minsection_pct = 2000; + cfg_.rc_target_bitrate = 2000; + cfg_.rc_max_quantizer = 63; + cfg_.rc_min_quantizer = 0; + tune_content_ = AOM_CONTENT_SCREEN; + + init_flags_ = AOM_CODEC_USE_PSNR; + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} + +void CpuSpeedTest::TestEncodeHighBitrate() { + // Validate that this non multiple of 64 wide clip encodes and decodes + // without a mismatch when passing in a very low max q. This pushes + // the encoder to producing lots of big partitions which will likely + // extend into the border and test the border condition. + cfg_.rc_2pass_vbr_minsection_pct = 5; + cfg_.rc_2pass_vbr_maxsection_pct = 2000; + cfg_.rc_target_bitrate = 12000; + cfg_.rc_max_quantizer = 10; + cfg_.rc_min_quantizer = 0; + + ::libaom_test::I420VideoSource video("hantro_odd.yuv", 208, 144, 30, 1, 0, + 10); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} + +void CpuSpeedTest::TestLowBitrate() { + // Validate that this clip encodes and decodes without a mismatch + // when passing in a very high min q. This pushes the encoder to producing + // lots of small partitions which might will test the other condition. + cfg_.rc_2pass_vbr_minsection_pct = 5; + cfg_.rc_2pass_vbr_maxsection_pct = 2000; + cfg_.rc_target_bitrate = 200; + cfg_.rc_min_quantizer = 40; + + ::libaom_test::I420VideoSource video("hantro_odd.yuv", 208, 144, 30, 1, 0, + 10); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} + +TEST_P(CpuSpeedTest, TestQ0) { TestQ0(); } +TEST_P(CpuSpeedTest, TestScreencastQ0) { TestScreencastQ0(); } +TEST_P(CpuSpeedTest, TestTuneScreen) { TestTuneScreen(); } +TEST_P(CpuSpeedTest, TestEncodeHighBitrate) { TestEncodeHighBitrate(); } +TEST_P(CpuSpeedTest, TestLowBitrate) { TestLowBitrate(); } + +class CpuSpeedTestLarge : public CpuSpeedTest {}; + +TEST_P(CpuSpeedTestLarge, TestQ0) { TestQ0(); } +TEST_P(CpuSpeedTestLarge, TestScreencastQ0) { TestScreencastQ0(); } +TEST_P(CpuSpeedTestLarge, TestTuneScreen) { TestTuneScreen(); } +TEST_P(CpuSpeedTestLarge, TestEncodeHighBitrate) { TestEncodeHighBitrate(); } +TEST_P(CpuSpeedTestLarge, TestLowBitrate) { TestLowBitrate(); } + +AV1_INSTANTIATE_TEST_CASE(CpuSpeedTest, + ::testing::Values(::libaom_test::kTwoPassGood, + ::libaom_test::kOnePassGood), + ::testing::Range(1, 3)); +AV1_INSTANTIATE_TEST_CASE(CpuSpeedTestLarge, + ::testing::Values(::libaom_test::kTwoPassGood, + ::libaom_test::kOnePassGood), + ::testing::Range(0, 1)); +} // namespace diff --git a/third_party/aom/test/datarate_test.cc b/third_party/aom/test/datarate_test.cc new file mode 100644 index 0000000000..48be4a46df --- /dev/null +++ b/third_party/aom/test/datarate_test.cc @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "./aom_config.h" +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" +#include "test/y4m_video_source.h" +#include "aom/aom_codec.h" + +namespace { + +class DatarateTestLarge + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params { + public: + DatarateTestLarge() : EncoderTest(GET_PARAM(0)) {} + + protected: + virtual ~DatarateTestLarge() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(GET_PARAM(1)); + set_cpu_used_ = GET_PARAM(2); + ResetModel(); + } + + virtual void ResetModel() { + last_pts_ = 0; + bits_in_buffer_model_ = cfg_.rc_target_bitrate * cfg_.rc_buf_initial_sz; + frame_number_ = 0; + tot_frame_number_ = 0; + first_drop_ = 0; + num_drops_ = 0; + // Denoiser is off by default. + denoiser_on_ = 0; + bits_total_ = 0; + denoiser_offon_test_ = 0; + denoiser_offon_period_ = -1; + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 0) encoder->Control(AOME_SET_CPUUSED, set_cpu_used_); + + if (denoiser_offon_test_) { + ASSERT_GT(denoiser_offon_period_, 0) + << "denoiser_offon_period_ is not positive."; + if ((video->frame() + 1) % denoiser_offon_period_ == 0) { + // Flip denoiser_on_ periodically + denoiser_on_ ^= 1; + } + } + + encoder->Control(AV1E_SET_NOISE_SENSITIVITY, denoiser_on_); + + const aom_rational_t tb = video->timebase(); + timebase_ = static_cast(tb.num) / tb.den; + duration_ = 0; + } + + virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) { + // Time since last timestamp = duration. + aom_codec_pts_t duration = pkt->data.frame.pts - last_pts_; + + if (duration > 1) { + // If first drop not set and we have a drop set it to this time. + if (!first_drop_) first_drop_ = last_pts_ + 1; + // Update the number of frame drops. + num_drops_ += static_cast(duration - 1); + // Update counter for total number of frames (#frames input to encoder). + // Needed for setting the proper layer_id below. + tot_frame_number_ += static_cast(duration - 1); + } + + // Add to the buffer the bits we'd expect from a constant bitrate server. + bits_in_buffer_model_ += static_cast( + duration * timebase_ * cfg_.rc_target_bitrate * 1000); + + // Buffer should not go negative. + ASSERT_GE(bits_in_buffer_model_, 0) << "Buffer Underrun at frame " + << pkt->data.frame.pts; + + const size_t frame_size_in_bits = pkt->data.frame.sz * 8; + + // Update the total encoded bits. + bits_total_ += frame_size_in_bits; + + // Update the most recent pts. + last_pts_ = pkt->data.frame.pts; + ++frame_number_; + ++tot_frame_number_; + } + + virtual void EndPassHook(void) { + duration_ = (last_pts_ + 1) * timebase_; + // Effective file datarate: + effective_datarate_ = (bits_total_ / 1000.0) / duration_; + } + + aom_codec_pts_t last_pts_; + double timebase_; + int frame_number_; // Counter for number of non-dropped/encoded frames. + int tot_frame_number_; // Counter for total number of input frames. + int64_t bits_total_; + double duration_; + double effective_datarate_; + int set_cpu_used_; + int64_t bits_in_buffer_model_; + aom_codec_pts_t first_drop_; + int num_drops_; + int denoiser_on_; + int denoiser_offon_test_; + int denoiser_offon_period_; +}; + +// Check basic rate targeting for VBR mode. +TEST_P(DatarateTestLarge, BasicRateTargetingVBR) { + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 63; + cfg_.g_error_resilient = 0; + cfg_.rc_end_usage = AOM_VBR; + cfg_.g_lag_in_frames = 0; + + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 140); + for (int i = 400; i <= 800; i += 400) { + cfg_.rc_target_bitrate = i; + ResetModel(); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + ASSERT_GE(effective_datarate_, cfg_.rc_target_bitrate * 0.75) + << " The datarate for the file is lower than target by too much!"; + ASSERT_LE(effective_datarate_, cfg_.rc_target_bitrate * 1.25) + << " The datarate for the file is greater than target by too much!"; + } +} + +// Check basic rate targeting for CBR, +TEST_P(DatarateTestLarge, BasicRateTargeting) { + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 500; + cfg_.rc_buf_sz = 1000; + cfg_.rc_dropframe_thresh = 1; + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 63; + cfg_.rc_end_usage = AOM_CBR; + cfg_.g_lag_in_frames = 0; + + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 140); + for (int i = 150; i < 800; i += 400) { + cfg_.rc_target_bitrate = i; + ResetModel(); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + ASSERT_GE(effective_datarate_, cfg_.rc_target_bitrate * 0.85) + << " The datarate for the file is lower than target by too much!"; + ASSERT_LE(effective_datarate_, cfg_.rc_target_bitrate * 1.15) + << " The datarate for the file is greater than target by too much!"; + } +} + +// Check basic rate targeting for CBR. +TEST_P(DatarateTestLarge, BasicRateTargeting444) { + ::libaom_test::Y4mVideoSource video("rush_hour_444.y4m", 0, 140); + + cfg_.g_profile = 1; + cfg_.g_timebase = video.timebase(); + + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 500; + cfg_.rc_buf_sz = 1000; + cfg_.rc_dropframe_thresh = 1; + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 63; + cfg_.rc_end_usage = AOM_CBR; + + for (int i = 250; i < 900; i += 400) { + cfg_.rc_target_bitrate = i; + ResetModel(); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + ASSERT_GE(static_cast(cfg_.rc_target_bitrate), + effective_datarate_ * 0.85) + << " The datarate for the file exceeds the target by too much!"; + ASSERT_LE(static_cast(cfg_.rc_target_bitrate), + effective_datarate_ * 1.15) + << " The datarate for the file missed the target!" + << cfg_.rc_target_bitrate << " " << effective_datarate_; + } +} + +// Check that (1) the first dropped frame gets earlier and earlier +// as the drop frame threshold is increased, and (2) that the total number of +// frame drops does not decrease as we increase frame drop threshold. +// Use a lower qp-max to force some frame drops. +TEST_P(DatarateTestLarge, ChangingDropFrameThresh) { + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 500; + cfg_.rc_buf_sz = 1000; + cfg_.rc_undershoot_pct = 20; + cfg_.rc_undershoot_pct = 20; + cfg_.rc_dropframe_thresh = 10; + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 50; + cfg_.rc_end_usage = AOM_CBR; + cfg_.rc_target_bitrate = 200; + cfg_.g_lag_in_frames = 0; + // TODO(marpan): Investigate datarate target failures with a smaller keyframe + // interval (128). + cfg_.kf_max_dist = 9999; + + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 100); + + const int kDropFrameThreshTestStep = 30; + aom_codec_pts_t last_drop = 140; + int last_num_drops = 0; + for (int i = 40; i < 100; i += kDropFrameThreshTestStep) { + cfg_.rc_dropframe_thresh = i; + ResetModel(); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + ASSERT_GE(effective_datarate_, cfg_.rc_target_bitrate * 0.85) + << " The datarate for the file is lower than target by too much!"; + ASSERT_LE(effective_datarate_, cfg_.rc_target_bitrate * 1.15) + << " The datarate for the file is greater than target by too much!"; + ASSERT_LE(first_drop_, last_drop) + << " The first dropped frame for drop_thresh " << i + << " > first dropped frame for drop_thresh " + << i - kDropFrameThreshTestStep; + ASSERT_GE(num_drops_, last_num_drops * 0.85) + << " The number of dropped frames for drop_thresh " << i + << " < number of dropped frames for drop_thresh " + << i - kDropFrameThreshTestStep; + last_drop = first_drop_; + last_num_drops = num_drops_; + } +} + +AV1_INSTANTIATE_TEST_CASE(DatarateTestLarge, + ::testing::Values(::libaom_test::kOnePassGood, + ::libaom_test::kRealTime), + ::testing::Range(2, 9, 2)); +} // namespace diff --git a/third_party/aom/test/dct16x16_test.cc b/third_party/aom/test/dct16x16_test.cc new file mode 100644 index 0000000000..89263ce89e --- /dev/null +++ b/third_party/aom/test/dct16x16_test.cc @@ -0,0 +1,876 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "av1/common/entropy.h" +#include "av1/common/scan.h" +#include "aom/aom_codec.h" +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" +#include "aom_ports/msvc.h" // for round() + +using libaom_test::ACMRandom; + +namespace { + +const int kNumCoeffs = 256; +const double C1 = 0.995184726672197; +const double C2 = 0.98078528040323; +const double C3 = 0.956940335732209; +const double C4 = 0.923879532511287; +const double C5 = 0.881921264348355; +const double C6 = 0.831469612302545; +const double C7 = 0.773010453362737; +const double C8 = 0.707106781186548; +const double C9 = 0.634393284163646; +const double C10 = 0.555570233019602; +const double C11 = 0.471396736825998; +const double C12 = 0.38268343236509; +const double C13 = 0.290284677254462; +const double C14 = 0.195090322016128; +const double C15 = 0.098017140329561; + +void butterfly_16x16_dct_1d(double input[16], double output[16]) { + double step[16]; + double intermediate[16]; + double temp1, temp2; + + // step 1 + step[0] = input[0] + input[15]; + step[1] = input[1] + input[14]; + step[2] = input[2] + input[13]; + step[3] = input[3] + input[12]; + step[4] = input[4] + input[11]; + step[5] = input[5] + input[10]; + step[6] = input[6] + input[9]; + step[7] = input[7] + input[8]; + step[8] = input[7] - input[8]; + step[9] = input[6] - input[9]; + step[10] = input[5] - input[10]; + step[11] = input[4] - input[11]; + step[12] = input[3] - input[12]; + step[13] = input[2] - input[13]; + step[14] = input[1] - input[14]; + step[15] = input[0] - input[15]; + + // step 2 + output[0] = step[0] + step[7]; + output[1] = step[1] + step[6]; + output[2] = step[2] + step[5]; + output[3] = step[3] + step[4]; + output[4] = step[3] - step[4]; + output[5] = step[2] - step[5]; + output[6] = step[1] - step[6]; + output[7] = step[0] - step[7]; + + temp1 = step[8] * C7; + temp2 = step[15] * C9; + output[8] = temp1 + temp2; + + temp1 = step[9] * C11; + temp2 = step[14] * C5; + output[9] = temp1 - temp2; + + temp1 = step[10] * C3; + temp2 = step[13] * C13; + output[10] = temp1 + temp2; + + temp1 = step[11] * C15; + temp2 = step[12] * C1; + output[11] = temp1 - temp2; + + temp1 = step[11] * C1; + temp2 = step[12] * C15; + output[12] = temp2 + temp1; + + temp1 = step[10] * C13; + temp2 = step[13] * C3; + output[13] = temp2 - temp1; + + temp1 = step[9] * C5; + temp2 = step[14] * C11; + output[14] = temp2 + temp1; + + temp1 = step[8] * C9; + temp2 = step[15] * C7; + output[15] = temp2 - temp1; + + // step 3 + step[0] = output[0] + output[3]; + step[1] = output[1] + output[2]; + step[2] = output[1] - output[2]; + step[3] = output[0] - output[3]; + + temp1 = output[4] * C14; + temp2 = output[7] * C2; + step[4] = temp1 + temp2; + + temp1 = output[5] * C10; + temp2 = output[6] * C6; + step[5] = temp1 + temp2; + + temp1 = output[5] * C6; + temp2 = output[6] * C10; + step[6] = temp2 - temp1; + + temp1 = output[4] * C2; + temp2 = output[7] * C14; + step[7] = temp2 - temp1; + + step[8] = output[8] + output[11]; + step[9] = output[9] + output[10]; + step[10] = output[9] - output[10]; + step[11] = output[8] - output[11]; + + step[12] = output[12] + output[15]; + step[13] = output[13] + output[14]; + step[14] = output[13] - output[14]; + step[15] = output[12] - output[15]; + + // step 4 + output[0] = (step[0] + step[1]); + output[8] = (step[0] - step[1]); + + temp1 = step[2] * C12; + temp2 = step[3] * C4; + temp1 = temp1 + temp2; + output[4] = 2 * (temp1 * C8); + + temp1 = step[2] * C4; + temp2 = step[3] * C12; + temp1 = temp2 - temp1; + output[12] = 2 * (temp1 * C8); + + output[2] = 2 * ((step[4] + step[5]) * C8); + output[14] = 2 * ((step[7] - step[6]) * C8); + + temp1 = step[4] - step[5]; + temp2 = step[6] + step[7]; + output[6] = (temp1 + temp2); + output[10] = (temp1 - temp2); + + intermediate[8] = step[8] + step[14]; + intermediate[9] = step[9] + step[15]; + + temp1 = intermediate[8] * C12; + temp2 = intermediate[9] * C4; + temp1 = temp1 - temp2; + output[3] = 2 * (temp1 * C8); + + temp1 = intermediate[8] * C4; + temp2 = intermediate[9] * C12; + temp1 = temp2 + temp1; + output[13] = 2 * (temp1 * C8); + + output[9] = 2 * ((step[10] + step[11]) * C8); + + intermediate[11] = step[10] - step[11]; + intermediate[12] = step[12] + step[13]; + intermediate[13] = step[12] - step[13]; + intermediate[14] = step[8] - step[14]; + intermediate[15] = step[9] - step[15]; + + output[15] = (intermediate[11] + intermediate[12]); + output[1] = -(intermediate[11] - intermediate[12]); + + output[7] = 2 * (intermediate[13] * C8); + + temp1 = intermediate[14] * C12; + temp2 = intermediate[15] * C4; + temp1 = temp1 - temp2; + output[11] = -2 * (temp1 * C8); + + temp1 = intermediate[14] * C4; + temp2 = intermediate[15] * C12; + temp1 = temp2 + temp1; + output[5] = 2 * (temp1 * C8); +} + +void reference_16x16_dct_2d(int16_t input[256], double output[256]) { + // First transform columns + for (int i = 0; i < 16; ++i) { + double temp_in[16], temp_out[16]; + for (int j = 0; j < 16; ++j) temp_in[j] = input[j * 16 + i]; + butterfly_16x16_dct_1d(temp_in, temp_out); + for (int j = 0; j < 16; ++j) output[j * 16 + i] = temp_out[j]; + } + // Then transform rows + for (int i = 0; i < 16; ++i) { + double temp_in[16], temp_out[16]; + for (int j = 0; j < 16; ++j) temp_in[j] = output[j + i * 16]; + butterfly_16x16_dct_1d(temp_in, temp_out); + // Scale by some magic number + for (int j = 0; j < 16; ++j) output[j + i * 16] = temp_out[j] / 2; + } +} + +typedef void (*FdctFunc)(const int16_t *in, tran_low_t *out, int stride); +typedef void (*IdctFunc)(const tran_low_t *in, uint8_t *out, int stride); +typedef void (*FhtFunc)(const int16_t *in, tran_low_t *out, int stride, + int tx_type); +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); + +typedef std::tr1::tuple Dct16x16Param; +typedef std::tr1::tuple Ht16x16Param; +typedef std::tr1::tuple + Idct16x16Param; + +void fdct16x16_ref(const int16_t *in, tran_low_t *out, int stride, + int /*tx_type*/) { + aom_fdct16x16_c(in, out, stride); +} + +void idct16x16_ref(const tran_low_t *in, uint8_t *dest, int stride, + int /*tx_type*/) { + aom_idct16x16_256_add_c(in, dest, stride); +} + +void fht16x16_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht16x16_c(in, out, stride, tx_type); +} + +void iht16x16_ref(const tran_low_t *in, uint8_t *dest, int stride, + int tx_type) { + av1_iht16x16_256_add_c(in, dest, stride, tx_type); +} + +#if CONFIG_HIGHBITDEPTH +void iht16x16_10(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_highbd_iht16x16_256_add_c(in, out, stride, tx_type, 10); +} + +void iht16x16_12(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_highbd_iht16x16_256_add_c(in, out, stride, tx_type, 12); +} +#endif // CONFIG_HIGHBITDEPTH + +class Trans16x16TestBase { + public: + virtual ~Trans16x16TestBase() {} + + protected: + virtual void RunFwdTxfm(int16_t *in, tran_low_t *out, int stride) = 0; + + virtual void RunInvTxfm(tran_low_t *out, uint8_t *dst, int stride) = 0; + + void RunAccuracyCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + uint32_t max_error = 0; + int64_t total_error = 0; + const int count_test_block = 10000; + for (int i = 0; i < count_test_block; ++i) { + DECLARE_ALIGNED(16, int16_t, test_input_block[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, test_temp_block[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, dst[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, src[kNumCoeffs]); +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, dst16[kNumCoeffs]); + DECLARE_ALIGNED(16, uint16_t, src16[kNumCoeffs]); +#endif + + // Initialize a test block with input range [-mask_, mask_]. + for (int j = 0; j < kNumCoeffs; ++j) { + if (bit_depth_ == AOM_BITS_8) { + src[j] = rnd.Rand8(); + dst[j] = rnd.Rand8(); + test_input_block[j] = src[j] - dst[j]; +#if CONFIG_HIGHBITDEPTH + } else { + src16[j] = rnd.Rand16() & mask_; + dst16[j] = rnd.Rand16() & mask_; + test_input_block[j] = src16[j] - dst16[j]; +#endif + } + } + + ASM_REGISTER_STATE_CHECK( + RunFwdTxfm(test_input_block, test_temp_block, pitch_)); + if (bit_depth_ == AOM_BITS_8) { + ASM_REGISTER_STATE_CHECK(RunInvTxfm(test_temp_block, dst, pitch_)); +#if CONFIG_HIGHBITDEPTH + } else { + ASM_REGISTER_STATE_CHECK( + RunInvTxfm(test_temp_block, CONVERT_TO_BYTEPTR(dst16), pitch_)); +#endif + } + + for (int j = 0; j < kNumCoeffs; ++j) { +#if CONFIG_HIGHBITDEPTH + const int32_t diff = + bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; +#else + const int32_t diff = dst[j] - src[j]; +#endif + const uint32_t error = diff * diff; + if (max_error < error) max_error = error; + total_error += error; + } + } + + EXPECT_GE(1u << 2 * (bit_depth_ - 8), max_error) + << "Error: 16x16 FHT/IHT has an individual round trip error > 1"; + + EXPECT_GE(count_test_block << 2 * (bit_depth_ - 8), total_error) + << "Error: 16x16 FHT/IHT has average round trip error > 1 per block"; + } + + void RunCoeffCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 1000; + DECLARE_ALIGNED(16, int16_t, input_block[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output_ref_block[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output_block[kNumCoeffs]); + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-mask_, mask_]. + for (int j = 0; j < kNumCoeffs; ++j) + input_block[j] = (rnd.Rand16() & mask_) - (rnd.Rand16() & mask_); + + fwd_txfm_ref(input_block, output_ref_block, pitch_, tx_type_); + ASM_REGISTER_STATE_CHECK(RunFwdTxfm(input_block, output_block, pitch_)); + + // The minimum quant value is 4. + for (int j = 0; j < kNumCoeffs; ++j) + EXPECT_EQ(output_block[j], output_ref_block[j]); + } + } + + void RunMemCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 1000; + DECLARE_ALIGNED(16, int16_t, input_extreme_block[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output_ref_block[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output_block[kNumCoeffs]); + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-mask_, mask_]. + for (int j = 0; j < kNumCoeffs; ++j) { + input_extreme_block[j] = rnd.Rand8() % 2 ? mask_ : -mask_; + } + if (i == 0) { + for (int j = 0; j < kNumCoeffs; ++j) input_extreme_block[j] = mask_; + } else if (i == 1) { + for (int j = 0; j < kNumCoeffs; ++j) input_extreme_block[j] = -mask_; + } + + fwd_txfm_ref(input_extreme_block, output_ref_block, pitch_, tx_type_); + ASM_REGISTER_STATE_CHECK( + RunFwdTxfm(input_extreme_block, output_block, pitch_)); + + // The minimum quant value is 4. + for (int j = 0; j < kNumCoeffs; ++j) { + EXPECT_EQ(output_block[j], output_ref_block[j]); + EXPECT_GE(4 * DCT_MAX_VALUE << (bit_depth_ - 8), abs(output_block[j])) + << "Error: 16x16 FDCT has coefficient larger than 4*DCT_MAX_VALUE"; + } + } + } + + void RunQuantCheck(int dc_thred, int ac_thred) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 100000; + DECLARE_ALIGNED(16, int16_t, input_extreme_block[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output_ref_block[kNumCoeffs]); + + DECLARE_ALIGNED(16, uint8_t, dst[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, ref[kNumCoeffs]); +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, dst16[kNumCoeffs]); + DECLARE_ALIGNED(16, uint16_t, ref16[kNumCoeffs]); +#endif + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-mask_, mask_]. + for (int j = 0; j < kNumCoeffs; ++j) { + input_extreme_block[j] = rnd.Rand8() % 2 ? mask_ : -mask_; + } + if (i == 0) + for (int j = 0; j < kNumCoeffs; ++j) input_extreme_block[j] = mask_; + if (i == 1) + for (int j = 0; j < kNumCoeffs; ++j) input_extreme_block[j] = -mask_; + + fwd_txfm_ref(input_extreme_block, output_ref_block, pitch_, tx_type_); + + // clear reconstructed pixel buffers + memset(dst, 0, kNumCoeffs * sizeof(uint8_t)); + memset(ref, 0, kNumCoeffs * sizeof(uint8_t)); +#if CONFIG_HIGHBITDEPTH + memset(dst16, 0, kNumCoeffs * sizeof(uint16_t)); + memset(ref16, 0, kNumCoeffs * sizeof(uint16_t)); +#endif + + // quantization with maximum allowed step sizes + output_ref_block[0] = (output_ref_block[0] / dc_thred) * dc_thred; + for (int j = 1; j < kNumCoeffs; ++j) + output_ref_block[j] = (output_ref_block[j] / ac_thred) * ac_thred; + if (bit_depth_ == AOM_BITS_8) { + inv_txfm_ref(output_ref_block, ref, pitch_, tx_type_); + ASM_REGISTER_STATE_CHECK(RunInvTxfm(output_ref_block, dst, pitch_)); +#if CONFIG_HIGHBITDEPTH + } else { + inv_txfm_ref(output_ref_block, CONVERT_TO_BYTEPTR(ref16), pitch_, + tx_type_); + ASM_REGISTER_STATE_CHECK( + RunInvTxfm(output_ref_block, CONVERT_TO_BYTEPTR(dst16), pitch_)); +#endif + } + if (bit_depth_ == AOM_BITS_8) { + for (int j = 0; j < kNumCoeffs; ++j) EXPECT_EQ(ref[j], dst[j]); +#if CONFIG_HIGHBITDEPTH + } else { + for (int j = 0; j < kNumCoeffs; ++j) EXPECT_EQ(ref16[j], dst16[j]); +#endif + } + } + } + + void RunInvAccuracyCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 1000; + DECLARE_ALIGNED(16, int16_t, in[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, coeff[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, dst[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, src[kNumCoeffs]); +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, dst16[kNumCoeffs]); + DECLARE_ALIGNED(16, uint16_t, src16[kNumCoeffs]); +#endif // CONFIG_HIGHBITDEPTH + + for (int i = 0; i < count_test_block; ++i) { + double out_r[kNumCoeffs]; + + // Initialize a test block with input range [-255, 255]. + for (int j = 0; j < kNumCoeffs; ++j) { + if (bit_depth_ == AOM_BITS_8) { + src[j] = rnd.Rand8(); + dst[j] = rnd.Rand8(); + in[j] = src[j] - dst[j]; +#if CONFIG_HIGHBITDEPTH + } else { + src16[j] = rnd.Rand16() & mask_; + dst16[j] = rnd.Rand16() & mask_; + in[j] = src16[j] - dst16[j]; +#endif // CONFIG_HIGHBITDEPTH + } + } + + reference_16x16_dct_2d(in, out_r); + for (int j = 0; j < kNumCoeffs; ++j) + coeff[j] = static_cast(round(out_r[j])); + + if (bit_depth_ == AOM_BITS_8) { + ASM_REGISTER_STATE_CHECK(RunInvTxfm(coeff, dst, 16)); +#if CONFIG_HIGHBITDEPTH + } else { + ASM_REGISTER_STATE_CHECK( + RunInvTxfm(coeff, CONVERT_TO_BYTEPTR(dst16), 16)); +#endif // CONFIG_HIGHBITDEPTH + } + + for (int j = 0; j < kNumCoeffs; ++j) { +#if CONFIG_HIGHBITDEPTH + const int diff = + bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; +#else + const int diff = dst[j] - src[j]; +#endif // CONFIG_HIGHBITDEPTH + const uint32_t error = diff * diff; + EXPECT_GE(1u, error) << "Error: 16x16 IDCT has error " << error + << " at index " << j; + } + } + } + + void CompareInvReference(IdctFunc ref_txfm, int thresh) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 10000; + const int eob = 10; + const int16_t *scan = av1_default_scan_orders[TX_16X16].scan; + DECLARE_ALIGNED(16, tran_low_t, coeff[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, dst[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, ref[kNumCoeffs]); +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, dst16[kNumCoeffs]); + DECLARE_ALIGNED(16, uint16_t, ref16[kNumCoeffs]); +#endif // CONFIG_HIGHBITDEPTH + + for (int i = 0; i < count_test_block; ++i) { + for (int j = 0; j < kNumCoeffs; ++j) { + if (j < eob) { + // Random values less than the threshold, either positive or negative + coeff[scan[j]] = rnd(thresh) * (1 - 2 * (i % 2)); + } else { + coeff[scan[j]] = 0; + } + if (bit_depth_ == AOM_BITS_8) { + dst[j] = 0; + ref[j] = 0; +#if CONFIG_HIGHBITDEPTH + } else { + dst16[j] = 0; + ref16[j] = 0; +#endif // CONFIG_HIGHBITDEPTH + } + } + if (bit_depth_ == AOM_BITS_8) { + ref_txfm(coeff, ref, pitch_); + ASM_REGISTER_STATE_CHECK(RunInvTxfm(coeff, dst, pitch_)); + } else { +#if CONFIG_HIGHBITDEPTH + ref_txfm(coeff, CONVERT_TO_BYTEPTR(ref16), pitch_); + ASM_REGISTER_STATE_CHECK( + RunInvTxfm(coeff, CONVERT_TO_BYTEPTR(dst16), pitch_)); +#endif // CONFIG_HIGHBITDEPTH + } + + for (int j = 0; j < kNumCoeffs; ++j) { +#if CONFIG_HIGHBITDEPTH + const int diff = + bit_depth_ == AOM_BITS_8 ? dst[j] - ref[j] : dst16[j] - ref16[j]; +#else + const int diff = dst[j] - ref[j]; +#endif // CONFIG_HIGHBITDEPTH + const uint32_t error = diff * diff; + EXPECT_EQ(0u, error) << "Error: 16x16 IDCT Comparison has error " + << error << " at index " << j; + } + } + } + + int pitch_; + int tx_type_; + aom_bit_depth_t bit_depth_; + int mask_; + FhtFunc fwd_txfm_ref; + IhtFunc inv_txfm_ref; +}; + +class Trans16x16DCT : public Trans16x16TestBase, + public ::testing::TestWithParam { + public: + virtual ~Trans16x16DCT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + bit_depth_ = GET_PARAM(3); + pitch_ = 16; + fwd_txfm_ref = fdct16x16_ref; + inv_txfm_ref = idct16x16_ref; + mask_ = (1 << bit_depth_) - 1; + inv_txfm_ref = idct16x16_ref; + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride); + } + void RunInvTxfm(tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride); + } + + FdctFunc fwd_txfm_; + IdctFunc inv_txfm_; +}; + +TEST_P(Trans16x16DCT, AccuracyCheck) { RunAccuracyCheck(); } + +TEST_P(Trans16x16DCT, CoeffCheck) { RunCoeffCheck(); } + +TEST_P(Trans16x16DCT, MemCheck) { RunMemCheck(); } + +TEST_P(Trans16x16DCT, QuantCheck) { + // Use maximally allowed quantization step sizes for DC and AC + // coefficients respectively. + RunQuantCheck(1336, 1828); +} + +TEST_P(Trans16x16DCT, InvAccuracyCheck) { RunInvAccuracyCheck(); } + +class Trans16x16HT : public Trans16x16TestBase, + public ::testing::TestWithParam { + public: + virtual ~Trans16x16HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + bit_depth_ = GET_PARAM(3); + pitch_ = 16; + fwd_txfm_ref = fht16x16_ref; + inv_txfm_ref = iht16x16_ref; + mask_ = (1 << bit_depth_) - 1; +#if CONFIG_HIGHBITDEPTH + switch (bit_depth_) { + case AOM_BITS_10: inv_txfm_ref = iht16x16_10; break; + case AOM_BITS_12: inv_txfm_ref = iht16x16_12; break; + default: inv_txfm_ref = iht16x16_ref; break; + } +#else + inv_txfm_ref = iht16x16_ref; +#endif + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + void RunInvTxfm(tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(Trans16x16HT, AccuracyCheck) { RunAccuracyCheck(); } + +TEST_P(Trans16x16HT, CoeffCheck) { RunCoeffCheck(); } + +TEST_P(Trans16x16HT, MemCheck) { RunMemCheck(); } + +TEST_P(Trans16x16HT, QuantCheck) { + // The encoder skips any non-DC intra prediction modes, + // when the quantization step size goes beyond 988. + RunQuantCheck(429, 729); +} + +class InvTrans16x16DCT : public Trans16x16TestBase, + public ::testing::TestWithParam { + public: + virtual ~InvTrans16x16DCT() {} + + virtual void SetUp() { + ref_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + thresh_ = GET_PARAM(2); + bit_depth_ = GET_PARAM(3); + pitch_ = 16; + mask_ = (1 << bit_depth_) - 1; + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(int16_t * /*in*/, tran_low_t * /*out*/, int /*stride*/) {} + void RunInvTxfm(tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride); + } + + IdctFunc ref_txfm_; + IdctFunc inv_txfm_; + int thresh_; +}; + +TEST_P(InvTrans16x16DCT, CompareReference) { + CompareInvReference(ref_txfm_, thresh_); +} + +class PartialTrans16x16Test : public ::testing::TestWithParam< + std::tr1::tuple > { + public: + virtual ~PartialTrans16x16Test() {} + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + bit_depth_ = GET_PARAM(1); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + aom_bit_depth_t bit_depth_; + FdctFunc fwd_txfm_; +}; + +TEST_P(PartialTrans16x16Test, Extremes) { +#if CONFIG_HIGHBITDEPTH + const int16_t maxval = + static_cast(clip_pixel_highbd(1 << 30, bit_depth_)); +#else + const int16_t maxval = 255; +#endif + const int minval = -maxval; + DECLARE_ALIGNED(16, int16_t, input[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output[kNumCoeffs]); + + for (int i = 0; i < kNumCoeffs; ++i) input[i] = maxval; + output[0] = 0; + ASM_REGISTER_STATE_CHECK(fwd_txfm_(input, output, 16)); + EXPECT_EQ((maxval * kNumCoeffs) >> 1, output[0]); + + for (int i = 0; i < kNumCoeffs; ++i) input[i] = minval; + output[0] = 0; + ASM_REGISTER_STATE_CHECK(fwd_txfm_(input, output, 16)); + EXPECT_EQ((minval * kNumCoeffs) >> 1, output[0]); +} + +TEST_P(PartialTrans16x16Test, Random) { +#if CONFIG_HIGHBITDEPTH + const int16_t maxval = + static_cast(clip_pixel_highbd(1 << 30, bit_depth_)); +#else + const int16_t maxval = 255; +#endif + DECLARE_ALIGNED(16, int16_t, input[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output[kNumCoeffs]); + ACMRandom rnd(ACMRandom::DeterministicSeed()); + + int sum = 0; + for (int i = 0; i < kNumCoeffs; ++i) { + const int val = (i & 1) ? -rnd(maxval + 1) : rnd(maxval + 1); + input[i] = val; + sum += val; + } + output[0] = 0; + ASM_REGISTER_STATE_CHECK(fwd_txfm_(input, output, 16)); + EXPECT_EQ(sum >> 1, output[0]); +} + +using std::tr1::make_tuple; + +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(C, Trans16x16DCT, + ::testing::Values(make_tuple(&aom_fdct16x16_c, + &aom_idct16x16_256_add_c, + 0, AOM_BITS_8))); +#else +INSTANTIATE_TEST_CASE_P(C, Trans16x16DCT, + ::testing::Values(make_tuple(&aom_fdct16x16_c, + &aom_idct16x16_256_add_c, + 0, AOM_BITS_8))); +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + C, Trans16x16HT, + ::testing::Values( + make_tuple(&av1_highbd_fht16x16_c, &iht16x16_10, 0, AOM_BITS_10), + make_tuple(&av1_highbd_fht16x16_c, &iht16x16_10, 1, AOM_BITS_10), + make_tuple(&av1_highbd_fht16x16_c, &iht16x16_10, 2, AOM_BITS_10), + make_tuple(&av1_highbd_fht16x16_c, &iht16x16_10, 3, AOM_BITS_10), + make_tuple(&av1_highbd_fht16x16_c, &iht16x16_12, 0, AOM_BITS_12), + make_tuple(&av1_highbd_fht16x16_c, &iht16x16_12, 1, AOM_BITS_12), + make_tuple(&av1_highbd_fht16x16_c, &iht16x16_12, 2, AOM_BITS_12), + make_tuple(&av1_highbd_fht16x16_c, &iht16x16_12, 3, AOM_BITS_12), + make_tuple(&av1_fht16x16_c, &av1_iht16x16_256_add_c, 0, AOM_BITS_8), + make_tuple(&av1_fht16x16_c, &av1_iht16x16_256_add_c, 1, AOM_BITS_8), + make_tuple(&av1_fht16x16_c, &av1_iht16x16_256_add_c, 2, AOM_BITS_8), + make_tuple(&av1_fht16x16_c, &av1_iht16x16_256_add_c, 3, AOM_BITS_8))); +INSTANTIATE_TEST_CASE_P( + C, PartialTrans16x16Test, + ::testing::Values(make_tuple(&aom_highbd_fdct16x16_1_c, AOM_BITS_8), + make_tuple(&aom_highbd_fdct16x16_1_c, AOM_BITS_10), + make_tuple(&aom_highbd_fdct16x16_1_c, AOM_BITS_12))); +#else +INSTANTIATE_TEST_CASE_P( + C, Trans16x16HT, + ::testing::Values( + make_tuple(&av1_fht16x16_c, &av1_iht16x16_256_add_c, 0, AOM_BITS_8), + make_tuple(&av1_fht16x16_c, &av1_iht16x16_256_add_c, 1, AOM_BITS_8), + make_tuple(&av1_fht16x16_c, &av1_iht16x16_256_add_c, 2, AOM_BITS_8), + make_tuple(&av1_fht16x16_c, &av1_iht16x16_256_add_c, 3, AOM_BITS_8))); +INSTANTIATE_TEST_CASE_P(C, PartialTrans16x16Test, + ::testing::Values(make_tuple(&aom_fdct16x16_1_c, + AOM_BITS_8))); +#endif // CONFIG_HIGHBITDEPTH + +#if HAVE_NEON_ASM && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + NEON, Trans16x16DCT, + ::testing::Values(make_tuple(&aom_fdct16x16_c, &aom_idct16x16_256_add_neon, + 0, AOM_BITS_8))); +#endif + +#if HAVE_SSE2 && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSE2, Trans16x16DCT, + ::testing::Values(make_tuple(&aom_fdct16x16_sse2, + &aom_idct16x16_256_add_sse2, 0, AOM_BITS_8))); +INSTANTIATE_TEST_CASE_P( + SSE2, Trans16x16HT, + ::testing::Values(make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, + 0, AOM_BITS_8), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, + 1, AOM_BITS_8), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, + 2, AOM_BITS_8), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_sse2, + 3, AOM_BITS_8))); +INSTANTIATE_TEST_CASE_P(SSE2, PartialTrans16x16Test, + ::testing::Values(make_tuple(&aom_fdct16x16_1_sse2, + AOM_BITS_8))); +#endif // HAVE_SSE2 && !CONFIG_HIGHBITDEPTH + +#if HAVE_AVX2 && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(AVX2, PartialTrans16x16Test, + ::testing::Values(make_tuple(&aom_fdct16x16_1_avx2, + AOM_BITS_8))); +#endif // HAVE_AVX2 && !CONFIG_HIGHBITDEPTH + +#if HAVE_SSE2 && CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(SSE2, Trans16x16DCT, + ::testing::Values(make_tuple(&aom_fdct16x16_sse2, + &aom_idct16x16_256_add_c, + 0, AOM_BITS_8))); +INSTANTIATE_TEST_CASE_P( + SSE2, Trans16x16HT, + ::testing::Values( + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_c, 0, AOM_BITS_8), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_c, 1, AOM_BITS_8), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_c, 2, AOM_BITS_8), + make_tuple(&av1_fht16x16_sse2, &av1_iht16x16_256_add_c, 3, + AOM_BITS_8))); +// TODO(luoyi): +// For this test case, we should test function: aom_highbd_fdct16x16_1_sse2. +// However this function is not available yet. if we mistakely test +// aom_fdct16x16_1_sse2, it could only pass AOM_BITS_8/AOM_BITS_10 but not +// AOM_BITS_12. +INSTANTIATE_TEST_CASE_P(SSE2, PartialTrans16x16Test, + ::testing::Values(make_tuple(&aom_fdct16x16_1_sse2, + AOM_BITS_8))); +#endif // HAVE_SSE2 && CONFIG_HIGHBITDEPTH + +#if HAVE_MSA && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(MSA, Trans16x16DCT, + ::testing::Values(make_tuple(&aom_fdct16x16_msa, + &aom_idct16x16_256_add_msa, + 0, AOM_BITS_8))); +#if !CONFIG_EXT_TX +// TODO(yaowu): re-enable this after msa versions are updated to match C. +INSTANTIATE_TEST_CASE_P( + DISABLED_MSA, Trans16x16HT, + ::testing::Values( + make_tuple(&av1_fht16x16_msa, &av1_iht16x16_256_add_msa, 0, AOM_BITS_8), + make_tuple(&av1_fht16x16_msa, &av1_iht16x16_256_add_msa, 1, AOM_BITS_8), + make_tuple(&av1_fht16x16_msa, &av1_iht16x16_256_add_msa, 2, AOM_BITS_8), + make_tuple(&av1_fht16x16_msa, &av1_iht16x16_256_add_msa, 3, + AOM_BITS_8))); +#endif // !CONFIG_EXT_TX +INSTANTIATE_TEST_CASE_P(MSA, PartialTrans16x16Test, + ::testing::Values(make_tuple(&aom_fdct16x16_1_msa, + AOM_BITS_8))); +#endif // HAVE_MSA && !CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/dct32x32_test.cc b/third_party/aom/test/dct32x32_test.cc new file mode 100644 index 0000000000..7c1db6501b --- /dev/null +++ b/third_party/aom/test/dct32x32_test.cc @@ -0,0 +1,438 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "av1/common/entropy.h" +#include "aom/aom_codec.h" +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" +#include "aom_ports/msvc.h" // for round() + +using libaom_test::ACMRandom; + +namespace { + +const int kNumCoeffs = 1024; +const double kPi = 3.141592653589793238462643383279502884; +void reference_32x32_dct_1d(const double in[32], double out[32]) { + const double kInvSqrt2 = 0.707106781186547524400844362104; + for (int k = 0; k < 32; k++) { + out[k] = 0.0; + for (int n = 0; n < 32; n++) + out[k] += in[n] * cos(kPi * (2 * n + 1) * k / 64.0); + if (k == 0) out[k] = out[k] * kInvSqrt2; + } +} + +void reference_32x32_dct_2d(const int16_t input[kNumCoeffs], + double output[kNumCoeffs]) { + // First transform columns + for (int i = 0; i < 32; ++i) { + double temp_in[32], temp_out[32]; + for (int j = 0; j < 32; ++j) temp_in[j] = input[j * 32 + i]; + reference_32x32_dct_1d(temp_in, temp_out); + for (int j = 0; j < 32; ++j) output[j * 32 + i] = temp_out[j]; + } + // Then transform rows + for (int i = 0; i < 32; ++i) { + double temp_in[32], temp_out[32]; + for (int j = 0; j < 32; ++j) temp_in[j] = output[j + i * 32]; + reference_32x32_dct_1d(temp_in, temp_out); + // Scale by some magic number + for (int j = 0; j < 32; ++j) output[j + i * 32] = temp_out[j] / 4; + } +} + +typedef void (*FwdTxfmFunc)(const int16_t *in, tran_low_t *out, int stride); +typedef void (*InvTxfmFunc)(const tran_low_t *in, uint8_t *out, int stride); + +typedef std::tr1::tuple + Trans32x32Param; + +class Trans32x32Test : public ::testing::TestWithParam { + public: + virtual ~Trans32x32Test() {} + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + version_ = GET_PARAM(2); // 0: high precision forward transform + // 1: low precision version for rd loop + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + int version_; + aom_bit_depth_t bit_depth_; + int mask_; + FwdTxfmFunc fwd_txfm_; + InvTxfmFunc inv_txfm_; +}; + +TEST_P(Trans32x32Test, AccuracyCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + uint32_t max_error = 0; + int64_t total_error = 0; + const int count_test_block = 10000; + DECLARE_ALIGNED(16, int16_t, test_input_block[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, test_temp_block[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, dst[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, src[kNumCoeffs]); +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, dst16[kNumCoeffs]); + DECLARE_ALIGNED(16, uint16_t, src16[kNumCoeffs]); +#endif + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-mask_, mask_]. + for (int j = 0; j < kNumCoeffs; ++j) { + if (bit_depth_ == AOM_BITS_8) { + src[j] = rnd.Rand8(); + dst[j] = rnd.Rand8(); + test_input_block[j] = src[j] - dst[j]; +#if CONFIG_HIGHBITDEPTH + } else { + src16[j] = rnd.Rand16() & mask_; + dst16[j] = rnd.Rand16() & mask_; + test_input_block[j] = src16[j] - dst16[j]; +#endif + } + } + + ASM_REGISTER_STATE_CHECK(fwd_txfm_(test_input_block, test_temp_block, 32)); + if (bit_depth_ == AOM_BITS_8) { + ASM_REGISTER_STATE_CHECK(inv_txfm_(test_temp_block, dst, 32)); +#if CONFIG_HIGHBITDEPTH + } else { + ASM_REGISTER_STATE_CHECK( + inv_txfm_(test_temp_block, CONVERT_TO_BYTEPTR(dst16), 32)); +#endif + } + + for (int j = 0; j < kNumCoeffs; ++j) { +#if CONFIG_HIGHBITDEPTH + const int32_t diff = + bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; +#else + const int32_t diff = dst[j] - src[j]; +#endif + const uint32_t error = diff * diff; + if (max_error < error) max_error = error; + total_error += error; + } + } + + if (version_ == 1) { + max_error /= 2; + total_error /= 45; + } + + EXPECT_GE(1u << 2 * (bit_depth_ - 8), max_error) + << "Error: 32x32 FDCT/IDCT has an individual round-trip error > 1"; + + EXPECT_GE(count_test_block << 2 * (bit_depth_ - 8), total_error) + << "Error: 32x32 FDCT/IDCT has average round-trip error > 1 per block"; +} + +TEST_P(Trans32x32Test, CoeffCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 1000; + + DECLARE_ALIGNED(16, int16_t, input_block[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output_ref_block[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output_block[kNumCoeffs]); + + for (int i = 0; i < count_test_block; ++i) { + for (int j = 0; j < kNumCoeffs; ++j) + input_block[j] = (rnd.Rand16() & mask_) - (rnd.Rand16() & mask_); + + const int stride = 32; + aom_fdct32x32_c(input_block, output_ref_block, stride); + ASM_REGISTER_STATE_CHECK(fwd_txfm_(input_block, output_block, stride)); + + if (version_ == 0) { + for (int j = 0; j < kNumCoeffs; ++j) + EXPECT_EQ(output_block[j], output_ref_block[j]) + << "Error: 32x32 FDCT versions have mismatched coefficients"; + } else { + for (int j = 0; j < kNumCoeffs; ++j) + EXPECT_GE(6, abs(output_block[j] - output_ref_block[j])) + << "Error: 32x32 FDCT rd has mismatched coefficients"; + } + } +} + +TEST_P(Trans32x32Test, MemCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 2000; + + DECLARE_ALIGNED(16, int16_t, input_extreme_block[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output_ref_block[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output_block[kNumCoeffs]); + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-mask_, mask_]. + for (int j = 0; j < kNumCoeffs; ++j) { + input_extreme_block[j] = rnd.Rand8() & 1 ? mask_ : -mask_; + } + if (i == 0) { + for (int j = 0; j < kNumCoeffs; ++j) input_extreme_block[j] = mask_; + } else if (i == 1) { + for (int j = 0; j < kNumCoeffs; ++j) input_extreme_block[j] = -mask_; + } + + const int stride = 32; + aom_fdct32x32_c(input_extreme_block, output_ref_block, stride); + ASM_REGISTER_STATE_CHECK( + fwd_txfm_(input_extreme_block, output_block, stride)); + + // The minimum quant value is 4. + for (int j = 0; j < kNumCoeffs; ++j) { + if (version_ == 0) { + EXPECT_EQ(output_block[j], output_ref_block[j]) + << "Error: 32x32 FDCT versions have mismatched coefficients"; + } else { + EXPECT_GE(6, abs(output_block[j] - output_ref_block[j])) + << "Error: 32x32 FDCT rd has mismatched coefficients"; + } + EXPECT_GE(4 * DCT_MAX_VALUE << (bit_depth_ - 8), abs(output_ref_block[j])) + << "Error: 32x32 FDCT C has coefficient larger than 4*DCT_MAX_VALUE"; + EXPECT_GE(4 * DCT_MAX_VALUE << (bit_depth_ - 8), abs(output_block[j])) + << "Error: 32x32 FDCT has coefficient larger than " + << "4*DCT_MAX_VALUE"; + } + } +} + +TEST_P(Trans32x32Test, InverseAccuracy) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 1000; + DECLARE_ALIGNED(16, int16_t, in[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, coeff[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, dst[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, src[kNumCoeffs]); +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, dst16[kNumCoeffs]); + DECLARE_ALIGNED(16, uint16_t, src16[kNumCoeffs]); +#endif + + for (int i = 0; i < count_test_block; ++i) { + double out_r[kNumCoeffs]; + + // Initialize a test block with input range [-255, 255] + for (int j = 0; j < kNumCoeffs; ++j) { + if (bit_depth_ == AOM_BITS_8) { + src[j] = rnd.Rand8(); + dst[j] = rnd.Rand8(); + in[j] = src[j] - dst[j]; +#if CONFIG_HIGHBITDEPTH + } else { + src16[j] = rnd.Rand16() & mask_; + dst16[j] = rnd.Rand16() & mask_; + in[j] = src16[j] - dst16[j]; +#endif + } + } + + reference_32x32_dct_2d(in, out_r); + for (int j = 0; j < kNumCoeffs; ++j) + coeff[j] = static_cast(round(out_r[j])); + if (bit_depth_ == AOM_BITS_8) { + ASM_REGISTER_STATE_CHECK(inv_txfm_(coeff, dst, 32)); +#if CONFIG_HIGHBITDEPTH + } else { + ASM_REGISTER_STATE_CHECK(inv_txfm_(coeff, CONVERT_TO_BYTEPTR(dst16), 32)); +#endif + } + for (int j = 0; j < kNumCoeffs; ++j) { +#if CONFIG_HIGHBITDEPTH + const int diff = + bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; +#else + const int diff = dst[j] - src[j]; +#endif + const int error = diff * diff; + EXPECT_GE(1, error) << "Error: 32x32 IDCT has error " << error + << " at index " << j; + } + } +} + +class PartialTrans32x32Test + : public ::testing::TestWithParam< + std::tr1::tuple > { + public: + virtual ~PartialTrans32x32Test() {} + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + bit_depth_ = GET_PARAM(1); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + aom_bit_depth_t bit_depth_; + FwdTxfmFunc fwd_txfm_; +}; + +TEST_P(PartialTrans32x32Test, Extremes) { +#if CONFIG_HIGHBITDEPTH + const int16_t maxval = + static_cast(clip_pixel_highbd(1 << 30, bit_depth_)); +#else + const int16_t maxval = 255; +#endif + const int minval = -maxval; + DECLARE_ALIGNED(16, int16_t, input[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output[kNumCoeffs]); + + for (int i = 0; i < kNumCoeffs; ++i) input[i] = maxval; + output[0] = 0; + ASM_REGISTER_STATE_CHECK(fwd_txfm_(input, output, 32)); + EXPECT_EQ((maxval * kNumCoeffs) >> 3, output[0]); + + for (int i = 0; i < kNumCoeffs; ++i) input[i] = minval; + output[0] = 0; + ASM_REGISTER_STATE_CHECK(fwd_txfm_(input, output, 32)); + EXPECT_EQ((minval * kNumCoeffs) >> 3, output[0]); +} + +TEST_P(PartialTrans32x32Test, Random) { +#if CONFIG_HIGHBITDEPTH + const int16_t maxval = + static_cast(clip_pixel_highbd(1 << 30, bit_depth_)); +#else + const int16_t maxval = 255; +#endif + DECLARE_ALIGNED(16, int16_t, input[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output[kNumCoeffs]); + ACMRandom rnd(ACMRandom::DeterministicSeed()); + + int sum = 0; + for (int i = 0; i < kNumCoeffs; ++i) { + const int val = (i & 1) ? -rnd(maxval + 1) : rnd(maxval + 1); + input[i] = val; + sum += val; + } + output[0] = 0; + ASM_REGISTER_STATE_CHECK(fwd_txfm_(input, output, 32)); + EXPECT_EQ(sum >> 3, output[0]); +} + +using std::tr1::make_tuple; + +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + C, Trans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_c, &aom_idct32x32_1024_add_c, 0, + AOM_BITS_8), + make_tuple(&aom_fdct32x32_rd_c, &aom_idct32x32_1024_add_c, + 1, AOM_BITS_8))); +INSTANTIATE_TEST_CASE_P( + C, PartialTrans32x32Test, + ::testing::Values(make_tuple(&aom_highbd_fdct32x32_1_c, AOM_BITS_8), + make_tuple(&aom_highbd_fdct32x32_1_c, AOM_BITS_10), + make_tuple(&aom_highbd_fdct32x32_1_c, AOM_BITS_12))); +#else +INSTANTIATE_TEST_CASE_P( + C, Trans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_c, &aom_idct32x32_1024_add_c, 0, + AOM_BITS_8), + make_tuple(&aom_fdct32x32_rd_c, &aom_idct32x32_1024_add_c, + 1, AOM_BITS_8))); +INSTANTIATE_TEST_CASE_P(C, PartialTrans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_1_c, + AOM_BITS_8))); +#endif // CONFIG_HIGHBITDEPTH + +#if HAVE_NEON && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + NEON, Trans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_c, &aom_idct32x32_1024_add_neon, + 0, AOM_BITS_8), + make_tuple(&aom_fdct32x32_rd_c, + &aom_idct32x32_1024_add_neon, 1, AOM_BITS_8))); +#endif // HAVE_NEON && !CONFIG_HIGHBITDEPTH + +#if HAVE_SSE2 && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSE2, Trans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_sse2, + &aom_idct32x32_1024_add_sse2, 0, AOM_BITS_8), + make_tuple(&aom_fdct32x32_rd_sse2, + &aom_idct32x32_1024_add_sse2, 1, AOM_BITS_8))); +INSTANTIATE_TEST_CASE_P(SSE2, PartialTrans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_1_sse2, + AOM_BITS_8))); +#endif // HAVE_SSE2 && !CONFIG_HIGHBITDEPTH + +#if HAVE_AVX2 && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(AVX2, PartialTrans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_1_avx2, + AOM_BITS_8))); +#endif // HAVE_AVX2 && !CONFIG_HIGHBITDEPTH + +#if HAVE_SSE2 && CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSE2, Trans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_sse2, &aom_idct32x32_1024_add_c, + 0, AOM_BITS_8), + make_tuple(&aom_fdct32x32_rd_sse2, + &aom_idct32x32_1024_add_c, 1, AOM_BITS_8))); +INSTANTIATE_TEST_CASE_P(SSE2, PartialTrans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_1_sse2, + AOM_BITS_8))); +#endif // HAVE_SSE2 && CONFIG_HIGHBITDEPTH + +#if HAVE_AVX2 && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + AVX2, Trans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_avx2, + &aom_idct32x32_1024_add_sse2, 0, AOM_BITS_8), + make_tuple(&aom_fdct32x32_rd_avx2, + &aom_idct32x32_1024_add_sse2, 1, AOM_BITS_8))); +#endif // HAVE_AVX2 && !CONFIG_HIGHBITDEPTH + +#if HAVE_AVX2 && CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + AVX2, Trans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_avx2, + &aom_idct32x32_1024_add_sse2, 0, AOM_BITS_8), + make_tuple(&aom_fdct32x32_rd_avx2, + &aom_idct32x32_1024_add_sse2, 1, AOM_BITS_8))); +#endif // HAVE_AVX2 && CONFIG_HIGHBITDEPTH + +#if HAVE_MSA && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + MSA, Trans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_msa, + &aom_idct32x32_1024_add_msa, 0, AOM_BITS_8), + make_tuple(&aom_fdct32x32_rd_msa, + &aom_idct32x32_1024_add_msa, 1, AOM_BITS_8))); +INSTANTIATE_TEST_CASE_P(MSA, PartialTrans32x32Test, + ::testing::Values(make_tuple(&aom_fdct32x32_1_msa, + AOM_BITS_8))); +#endif // HAVE_MSA && !CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/decode_api_test.cc b/third_party/aom/test/decode_api_test.cc new file mode 100644 index 0000000000..6bd72a45d4 --- /dev/null +++ b/third_party/aom/test/decode_api_test.cc @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "test/ivf_video_source.h" +#include "aom/aomdx.h" +#include "aom/aom_decoder.h" + +namespace { + +#define NELEMENTS(x) static_cast(sizeof(x) / sizeof(x[0])) + +TEST(DecodeAPI, InvalidParams) { + static const aom_codec_iface_t *kCodecs[] = { +#if CONFIG_AV1_DECODER + &aom_codec_av1_dx_algo, +#endif + }; + uint8_t buf[1] = { 0 }; + aom_codec_ctx_t dec; + + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_dec_init(NULL, NULL, NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_dec_init(&dec, NULL, NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_decode(NULL, NULL, 0, NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_decode(NULL, buf, 0, NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, + aom_codec_decode(NULL, buf, NELEMENTS(buf), NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, + aom_codec_decode(NULL, NULL, NELEMENTS(buf), NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_destroy(NULL)); + EXPECT_TRUE(aom_codec_error(NULL) != NULL); + + for (int i = 0; i < NELEMENTS(kCodecs); ++i) { + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, + aom_codec_dec_init(NULL, kCodecs[i], NULL, 0)); + + EXPECT_EQ(AOM_CODEC_OK, aom_codec_dec_init(&dec, kCodecs[i], NULL, 0)); + EXPECT_EQ(AOM_CODEC_UNSUP_BITSTREAM, + aom_codec_decode(&dec, buf, NELEMENTS(buf), NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, + aom_codec_decode(&dec, NULL, NELEMENTS(buf), NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_decode(&dec, buf, 0, NULL, 0)); + + EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&dec)); + } +} + +} // namespace diff --git a/third_party/aom/test/decode_perf_test.cc b/third_party/aom/test/decode_perf_test.cc new file mode 100644 index 0000000000..ede4f8849d --- /dev/null +++ b/third_party/aom/test/decode_perf_test.cc @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include "test/codec_factory.h" +#include "test/decode_test_driver.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/ivf_video_source.h" +#include "test/md5_helper.h" +#include "test/util.h" +#include "test/webm_video_source.h" +#include "aom_ports/aom_timer.h" +#include "./ivfenc.h" +#include "./aom_version.h" + +using std::tr1::make_tuple; + +namespace { + +#define VIDEO_NAME 0 +#define THREADS 1 + +const int kMaxPsnr = 100; +const double kUsecsInSec = 1000000.0; +const char kNewEncodeOutputFile[] = "new_encode.ivf"; + +/* + DecodePerfTest takes a tuple of filename + number of threads to decode with + */ +typedef std::tr1::tuple DecodePerfParam; + +// TODO(jimbankoski): Add actual test vectors here when available. +// const DecodePerfParam kAV1DecodePerfVectors[] = {}; + +/* + In order to reflect real world performance as much as possible, Perf tests + *DO NOT* do any correctness checks. Please run them alongside correctness + tests to ensure proper codec integrity. Furthermore, in this test we + deliberately limit the amount of system calls we make to avoid OS + preemption. + + TODO(joshualitt) create a more detailed perf measurement test to collect + power/temp/min max frame decode times/etc + */ + +class DecodePerfTest : public ::testing::TestWithParam {}; + +TEST_P(DecodePerfTest, PerfTest) { + const char *const video_name = GET_PARAM(VIDEO_NAME); + const unsigned threads = GET_PARAM(THREADS); + + libaom_test::WebMVideoSource video(video_name); + video.Init(); + + aom_codec_dec_cfg_t cfg = aom_codec_dec_cfg_t(); + cfg.threads = threads; + libaom_test::AV1Decoder decoder(cfg, 0); + + aom_usec_timer t; + aom_usec_timer_start(&t); + + for (video.Begin(); video.cxdata() != NULL; video.Next()) { + decoder.DecodeFrame(video.cxdata(), video.frame_size()); + } + + aom_usec_timer_mark(&t); + const double elapsed_secs = double(aom_usec_timer_elapsed(&t)) / kUsecsInSec; + const unsigned frames = video.frame_number(); + const double fps = double(frames) / elapsed_secs; + + printf("{\n"); + printf("\t\"type\" : \"decode_perf_test\",\n"); + printf("\t\"version\" : \"%s\",\n", VERSION_STRING_NOSP); + printf("\t\"videoName\" : \"%s\",\n", video_name); + printf("\t\"threadCount\" : %u,\n", threads); + printf("\t\"decodeTimeSecs\" : %f,\n", elapsed_secs); + printf("\t\"totalFrames\" : %u,\n", frames); + printf("\t\"framesPerSecond\" : %f\n", fps); + printf("}\n"); +} + +// TODO(jimbankoski): Enabled when we have actual AV1 Decode vectors. +// INSTANTIATE_TEST_CASE_P(AV1, DecodePerfTest, +// ::testing::ValuesIn(kAV1DecodePerfVectors)); + +class AV1NewEncodeDecodePerfTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWithParam { + protected: + AV1NewEncodeDecodePerfTest() + : EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)), speed_(0), + outfile_(0), out_frames_(0) {} + + virtual ~AV1NewEncodeDecodePerfTest() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + + cfg_.g_lag_in_frames = 25; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 56; + cfg_.rc_dropframe_thresh = 0; + cfg_.rc_undershoot_pct = 50; + cfg_.rc_overshoot_pct = 50; + cfg_.rc_buf_sz = 1000; + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 600; + cfg_.rc_resize_allowed = 0; + cfg_.rc_end_usage = AOM_VBR; + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 1) { + encoder->Control(AOME_SET_CPUUSED, speed_); + encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1); + encoder->Control(AV1E_SET_TILE_COLUMNS, 2); + } + } + + virtual void BeginPassHook(unsigned int /*pass*/) { + const std::string data_path = getenv("LIBAOM_TEST_DATA_PATH"); + const std::string path_to_source = data_path + "/" + kNewEncodeOutputFile; + outfile_ = fopen(path_to_source.c_str(), "wb"); + ASSERT_TRUE(outfile_ != NULL); + } + + virtual void EndPassHook() { + if (outfile_ != NULL) { + if (!fseek(outfile_, 0, SEEK_SET)) + ivf_write_file_header(outfile_, &cfg_, AV1_FOURCC, out_frames_); + fclose(outfile_); + outfile_ = NULL; + } + } + + virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) { + ++out_frames_; + + // Write initial file header if first frame. + if (pkt->data.frame.pts == 0) + ivf_write_file_header(outfile_, &cfg_, AV1_FOURCC, out_frames_); + + // Write frame header and data. + ivf_write_frame_header(outfile_, out_frames_, pkt->data.frame.sz); + ASSERT_EQ(fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_), + pkt->data.frame.sz); + } + + virtual bool DoDecode() { return false; } + + void set_speed(unsigned int speed) { speed_ = speed; } + + private: + libaom_test::TestMode encoding_mode_; + uint32_t speed_; + FILE *outfile_; + uint32_t out_frames_; +}; + +struct EncodePerfTestVideo { + EncodePerfTestVideo(const char *name_, uint32_t width_, uint32_t height_, + uint32_t bitrate_, int frames_) + : name(name_), width(width_), height(height_), bitrate(bitrate_), + frames(frames_) {} + const char *name; + uint32_t width; + uint32_t height; + uint32_t bitrate; + int frames; +}; + +const EncodePerfTestVideo kAV1EncodePerfTestVectors[] = { + EncodePerfTestVideo("niklas_1280_720_30.yuv", 1280, 720, 600, 470), +}; + +TEST_P(AV1NewEncodeDecodePerfTest, PerfTest) { + SetUp(); + + // TODO(JBB): Make this work by going through the set of given files. + const int i = 0; + const aom_rational timebase = { 33333333, 1000000000 }; + cfg_.g_timebase = timebase; + cfg_.rc_target_bitrate = kAV1EncodePerfTestVectors[i].bitrate; + + init_flags_ = AOM_CODEC_USE_PSNR; + + const char *video_name = kAV1EncodePerfTestVectors[i].name; + libaom_test::I420VideoSource video( + video_name, kAV1EncodePerfTestVectors[i].width, + kAV1EncodePerfTestVectors[i].height, timebase.den, timebase.num, 0, + kAV1EncodePerfTestVectors[i].frames); + set_speed(2); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + + const uint32_t threads = 4; + + libaom_test::IVFVideoSource decode_video(kNewEncodeOutputFile); + decode_video.Init(); + + aom_codec_dec_cfg_t cfg = aom_codec_dec_cfg_t(); + cfg.threads = threads; + libaom_test::AV1Decoder decoder(cfg, 0); + + aom_usec_timer t; + aom_usec_timer_start(&t); + + for (decode_video.Begin(); decode_video.cxdata() != NULL; + decode_video.Next()) { + decoder.DecodeFrame(decode_video.cxdata(), decode_video.frame_size()); + } + + aom_usec_timer_mark(&t); + const double elapsed_secs = + static_cast(aom_usec_timer_elapsed(&t)) / kUsecsInSec; + const unsigned decode_frames = decode_video.frame_number(); + const double fps = static_cast(decode_frames) / elapsed_secs; + + printf("{\n"); + printf("\t\"type\" : \"decode_perf_test\",\n"); + printf("\t\"version\" : \"%s\",\n", VERSION_STRING_NOSP); + printf("\t\"videoName\" : \"%s\",\n", kNewEncodeOutputFile); + printf("\t\"threadCount\" : %u,\n", threads); + printf("\t\"decodeTimeSecs\" : %f,\n", elapsed_secs); + printf("\t\"totalFrames\" : %u,\n", decode_frames); + printf("\t\"framesPerSecond\" : %f\n", fps); + printf("}\n"); +} + +AV1_INSTANTIATE_TEST_CASE(AV1NewEncodeDecodePerfTest, + ::testing::Values(::libaom_test::kTwoPassGood)); +} // namespace diff --git a/third_party/aom/test/decode_test_driver.cc b/third_party/aom/test/decode_test_driver.cc new file mode 100644 index 0000000000..35c28eafd5 --- /dev/null +++ b/third_party/aom/test/decode_test_driver.cc @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/codec_factory.h" +#include "test/decode_test_driver.h" +#include "test/register_state_check.h" +#include "test/video_source.h" + +namespace libaom_test { + +const char kVP8Name[] = "WebM Project VP8"; +const char kAV1Name[] = "AOMedia Project AV1 Decoder"; + +aom_codec_err_t Decoder::PeekStream(const uint8_t *cxdata, size_t size, + aom_codec_stream_info_t *stream_info) { + return aom_codec_peek_stream_info( + CodecInterface(), cxdata, static_cast(size), stream_info); +} + +aom_codec_err_t Decoder::DecodeFrame(const uint8_t *cxdata, size_t size) { + return DecodeFrame(cxdata, size, NULL); +} + +aom_codec_err_t Decoder::DecodeFrame(const uint8_t *cxdata, size_t size, + void *user_priv) { + aom_codec_err_t res_dec; + InitOnce(); + API_REGISTER_STATE_CHECK( + res_dec = aom_codec_decode( + &decoder_, cxdata, static_cast(size), user_priv, 0)); + return res_dec; +} + +bool Decoder::IsVP8() const { + const char *codec_name = GetDecoderName(); + return strncmp(kVP8Name, codec_name, sizeof(kVP8Name) - 1) == 0; +} + +bool Decoder::IsAV1() const { + const char *codec_name = GetDecoderName(); + return strncmp(kAV1Name, codec_name, sizeof(kAV1Name) - 1) == 0; +} + +void DecoderTest::HandlePeekResult(Decoder *const decoder, + CompressedVideoSource *video, + const aom_codec_err_t res_peek) { + const bool is_vp8 = decoder->IsVP8(); + if (is_vp8) { + /* Vp8's implementation of PeekStream returns an error if the frame you + * pass it is not a keyframe, so we only expect AOM_CODEC_OK on the first + * frame, which must be a keyframe. */ + if (video->frame_number() == 0) + ASSERT_EQ(AOM_CODEC_OK, res_peek) << "Peek return failed: " + << aom_codec_err_to_string(res_peek); + } else { + /* The Av1 implementation of PeekStream returns an error only if the + * data passed to it isn't a valid Av1 chunk. */ + ASSERT_EQ(AOM_CODEC_OK, res_peek) << "Peek return failed: " + << aom_codec_err_to_string(res_peek); + } +} + +void DecoderTest::RunLoop(CompressedVideoSource *video, + const aom_codec_dec_cfg_t &dec_cfg) { + Decoder *const decoder = codec_->CreateDecoder(dec_cfg, flags_); + ASSERT_TRUE(decoder != NULL); + bool end_of_file = false; + + // Decode frames. + for (video->Begin(); !::testing::Test::HasFailure() && !end_of_file; + video->Next()) { + PreDecodeFrameHook(*video, decoder); + + aom_codec_stream_info_t stream_info; + stream_info.sz = sizeof(stream_info); + + if (video->cxdata() != NULL) { + const aom_codec_err_t res_peek = decoder->PeekStream( + video->cxdata(), video->frame_size(), &stream_info); + HandlePeekResult(decoder, video, res_peek); + ASSERT_FALSE(::testing::Test::HasFailure()); + + aom_codec_err_t res_dec = + decoder->DecodeFrame(video->cxdata(), video->frame_size()); + if (!HandleDecodeResult(res_dec, decoder)) break; + } else { + // Signal end of the file to the decoder. + const aom_codec_err_t res_dec = decoder->DecodeFrame(NULL, 0); + ASSERT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError(); + end_of_file = true; + } + + DxDataIterator dec_iter = decoder->GetDxData(); + const aom_image_t *img = NULL; + + // Get decompressed data + while ((img = dec_iter.Next())) + DecompressedFrameHook(*img, video->frame_number()); + } + delete decoder; +} + +void DecoderTest::RunLoop(CompressedVideoSource *video) { + aom_codec_dec_cfg_t dec_cfg = aom_codec_dec_cfg_t(); + RunLoop(video, dec_cfg); +} + +void DecoderTest::set_cfg(const aom_codec_dec_cfg_t &dec_cfg) { + memcpy(&cfg_, &dec_cfg, sizeof(cfg_)); +} + +void DecoderTest::set_flags(const aom_codec_flags_t flags) { flags_ = flags; } + +} // namespace libaom_test diff --git a/third_party/aom/test/decode_test_driver.h b/third_party/aom/test/decode_test_driver.h new file mode 100644 index 0000000000..e7deb389c9 --- /dev/null +++ b/third_party/aom/test/decode_test_driver.h @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef TEST_DECODE_TEST_DRIVER_H_ +#define TEST_DECODE_TEST_DRIVER_H_ +#include +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "./aom_config.h" +#include "aom/aom_decoder.h" + +namespace libaom_test { + +class CodecFactory; +class CompressedVideoSource; + +// Provides an object to handle decoding output +class DxDataIterator { + public: + explicit DxDataIterator(aom_codec_ctx_t *decoder) + : decoder_(decoder), iter_(NULL) {} + + const aom_image_t *Next() { return aom_codec_get_frame(decoder_, &iter_); } + + private: + aom_codec_ctx_t *decoder_; + aom_codec_iter_t iter_; +}; + +// Provides a simplified interface to manage one video decoding. +// Similar to Encoder class, the exact services should be added +// as more tests are added. +class Decoder { + public: + explicit Decoder(aom_codec_dec_cfg_t cfg) + : cfg_(cfg), flags_(0), init_done_(false) { + memset(&decoder_, 0, sizeof(decoder_)); + } + + Decoder(aom_codec_dec_cfg_t cfg, const aom_codec_flags_t flag) + : cfg_(cfg), flags_(flag), init_done_(false) { + memset(&decoder_, 0, sizeof(decoder_)); + } + + virtual ~Decoder() { aom_codec_destroy(&decoder_); } + + aom_codec_err_t PeekStream(const uint8_t *cxdata, size_t size, + aom_codec_stream_info_t *stream_info); + + aom_codec_err_t DecodeFrame(const uint8_t *cxdata, size_t size); + + aom_codec_err_t DecodeFrame(const uint8_t *cxdata, size_t size, + void *user_priv); + + DxDataIterator GetDxData() { return DxDataIterator(&decoder_); } + + void Control(int ctrl_id, int arg) { Control(ctrl_id, arg, AOM_CODEC_OK); } + + void Control(int ctrl_id, const void *arg) { + InitOnce(); + const aom_codec_err_t res = aom_codec_control_(&decoder_, ctrl_id, arg); + ASSERT_EQ(AOM_CODEC_OK, res) << DecodeError(); + } + + void Control(int ctrl_id, int arg, aom_codec_err_t expected_value) { + InitOnce(); + const aom_codec_err_t res = aom_codec_control_(&decoder_, ctrl_id, arg); + ASSERT_EQ(expected_value, res) << DecodeError(); + } + + const char *DecodeError() { + const char *detail = aom_codec_error_detail(&decoder_); + return detail ? detail : aom_codec_error(&decoder_); + } + + // Passes the external frame buffer information to libaom. + aom_codec_err_t SetFrameBufferFunctions( + aom_get_frame_buffer_cb_fn_t cb_get, + aom_release_frame_buffer_cb_fn_t cb_release, void *user_priv) { + InitOnce(); + return aom_codec_set_frame_buffer_functions(&decoder_, cb_get, cb_release, + user_priv); + } + + const char *GetDecoderName() const { + return aom_codec_iface_name(CodecInterface()); + } + + bool IsVP8() const; + + bool IsAV1() const; + + aom_codec_ctx_t *GetDecoder() { return &decoder_; } + + protected: + virtual aom_codec_iface_t *CodecInterface() const = 0; + + void InitOnce() { + if (!init_done_) { + const aom_codec_err_t res = + aom_codec_dec_init(&decoder_, CodecInterface(), &cfg_, flags_); + ASSERT_EQ(AOM_CODEC_OK, res) << DecodeError(); + init_done_ = true; + } + } + + aom_codec_ctx_t decoder_; + aom_codec_dec_cfg_t cfg_; + aom_codec_flags_t flags_; + bool init_done_; +}; + +// Common test functionality for all Decoder tests. +class DecoderTest { + public: + // Main decoding loop + virtual void RunLoop(CompressedVideoSource *video); + virtual void RunLoop(CompressedVideoSource *video, + const aom_codec_dec_cfg_t &dec_cfg); + + virtual void set_cfg(const aom_codec_dec_cfg_t &dec_cfg); + virtual void set_flags(const aom_codec_flags_t flags); + + // Hook to be called before decompressing every frame. + virtual void PreDecodeFrameHook(const CompressedVideoSource & /*video*/, + Decoder * /*decoder*/) {} + + // Hook to be called to handle decode result. Return true to continue. + virtual bool HandleDecodeResult(const aom_codec_err_t res_dec, + Decoder *decoder) { + EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError(); + return AOM_CODEC_OK == res_dec; + } + + // Hook to be called on every decompressed frame. + virtual void DecompressedFrameHook(const aom_image_t & /*img*/, + const unsigned int /*frame_number*/) {} + + // Hook to be called on peek result + virtual void HandlePeekResult(Decoder *const decoder, + CompressedVideoSource *video, + const aom_codec_err_t res_peek); + + protected: + explicit DecoderTest(const CodecFactory *codec) + : codec_(codec), cfg_(), flags_(0) {} + + virtual ~DecoderTest() {} + + const CodecFactory *codec_; + aom_codec_dec_cfg_t cfg_; + aom_codec_flags_t flags_; +}; + +} // namespace libaom_test + +#endif // TEST_DECODE_TEST_DRIVER_H_ diff --git a/third_party/aom/test/decode_to_md5.sh b/third_party/aom/test/decode_to_md5.sh new file mode 100755 index 0000000000..44c9f5f052 --- /dev/null +++ b/third_party/aom/test/decode_to_md5.sh @@ -0,0 +1,67 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This file tests the libaom decode_to_md5 example. To add new tests to this +## file, do the following: +## 1. Write a shell function (this is your test). +## 2. Add the function to decode_to_md5_tests (on a new line). +## +. $(dirname $0)/tools_common.sh + +# Environment check: Make sure input is available: +# $AOM_IVF_FILE and $AV1_IVF_FILE are required. +decode_to_md5_verify_environment() { + if [ "$(av1_encode_available)" != "yes" ] && [ ! -e "${AV1_IVF_FILE}" ]; then + return 1 + fi +} + +# Runs decode_to_md5 on $1 and captures the md5 sum for the final frame. $2 is +# interpreted as codec name and used solely to name the output file. $3 is the +# expected md5 sum: It must match that of the final frame. +decode_to_md5() { + local decoder="${LIBAOM_BIN_PATH}/decode_to_md5${AOM_TEST_EXE_SUFFIX}" + local input_file="$1" + local codec="$2" + local expected_md5="$3" + local output_file="${AOM_TEST_OUTPUT_DIR}/decode_to_md5_${codec}" + + if [ ! -x "${decoder}" ]; then + elog "${decoder} does not exist or is not executable." + return 1 + fi + + eval "${AOM_TEST_PREFIX}" "${decoder}" "${input_file}" "${output_file}" \ + ${devnull} + + [ -e "${output_file}" ] || return 1 + + local md5_last_frame="$(tail -n1 "${output_file}" | awk '{print $1}')" + local actual_md5="$(echo "${md5_last_frame}" | awk '{print $1}')" + [ "${actual_md5}" = "${expected_md5}" ] || return 1 +} + +decode_to_md5_av1() { + # expected MD5 sum for the last frame. + local expected_md5="26d3ef1d60754a1f6acb603c3763efbe" + local file="${AV1_IVF_FILE}" + + if [ "$(av1_decode_available)" = "yes" ]; then + if [ ! -e "${AV1_IVF_FILE}" ]; then + file="${AOM_TEST_OUTPUT_DIR}/test_encode.ivf" + encode_yuv_raw_input_av1 "${file}" --ivf + fi + decode_to_md5 "${file}" "av1" "${expected_md5}" + fi +} + +decode_to_md5_tests="decode_to_md5_av1" + +run_tests decode_to_md5_verify_environment "${decode_to_md5_tests}" diff --git a/third_party/aom/test/decode_with_drops.sh b/third_party/aom/test/decode_with_drops.sh new file mode 100755 index 0000000000..5978312f20 --- /dev/null +++ b/third_party/aom/test/decode_with_drops.sh @@ -0,0 +1,67 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This file tests the libaom decode_with_drops example. To add new tests to +## this file, do the following: +## 1. Write a shell function (this is your test). +## 2. Add the function to decode_with_drops_tests (on a new line). +## +. $(dirname $0)/tools_common.sh + +# Environment check: Make sure input is available: +# $AOM_IVF_FILE and $AV1_IVF_FILE are required. +decode_with_drops_verify_environment() { + if [ "$(av1_encode_available)" != "yes" ] && [ ! -e "${AV1_IVF_FILE}" ]; then + return 1 + fi +} + +# Runs decode_with_drops on $1, $2 is interpreted as codec name and used solely +# to name the output file. $3 is the drop mode, and is passed directly to +# decode_with_drops. +decode_with_drops() { + local decoder="${LIBAOM_BIN_PATH}/decode_with_drops${AOM_TEST_EXE_SUFFIX}" + local input_file="$1" + local codec="$2" + local output_file="${AOM_TEST_OUTPUT_DIR}/decode_with_drops_${codec}" + local drop_mode="$3" + + if [ ! -x "${decoder}" ]; then + elog "${decoder} does not exist or is not executable." + return 1 + fi + + eval "${AOM_TEST_PREFIX}" "${decoder}" "${input_file}" "${output_file}" \ + "${drop_mode}" ${devnull} + + [ -e "${output_file}" ] || return 1 +} + + +# Decodes $AV1_IVF_FILE while dropping frames, twice: once in sequence mode, +# and once in pattern mode. +decode_with_drops_av1() { + if [ "$(av1_decode_available)" = "yes" ]; then + local file="${AV1_IVF_FILE}" + if [ ! -e "${AV1_IVF_FILE}" ]; then + file="${AOM_TEST_OUTPUT_DIR}/test_encode.ivf" + encode_yuv_raw_input_av1 "${file}" --ivf + fi + # Drop frames 2 and 3. + decode_with_drops "${file}" "av1" "2-3" + + # Test pattern mode: Drop 3 of every 4 frames. + decode_with_drops "${file}" "av1" "3/4" + fi +} + +decode_with_drops_tests="decode_with_drops_av1" + +run_tests decode_with_drops_verify_environment "${decode_with_drops_tests}" diff --git a/third_party/aom/test/dering_test.cc b/third_party/aom/test/dering_test.cc new file mode 100644 index 0000000000..195a60ff81 --- /dev/null +++ b/third_party/aom/test/dering_test.cc @@ -0,0 +1,388 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "aom_ports/aom_timer.h" +#include "av1/common/od_dering.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" + +using libaom_test::ACMRandom; + +namespace { + +typedef std::tr1::tuple + dering_dir_param_t; + +class CDEFDeringDirTest : public ::testing::TestWithParam { + public: + virtual ~CDEFDeringDirTest() {} + virtual void SetUp() { + dering = GET_PARAM(0); + ref_dering = GET_PARAM(1); + bsize = GET_PARAM(2); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + int bsize; + od_filter_dering_direction_func dering; + od_filter_dering_direction_func ref_dering; +}; + +typedef CDEFDeringDirTest CDEFDeringSpeedTest; + +void test_dering(int bsize, int iterations, + od_filter_dering_direction_func dering, + od_filter_dering_direction_func ref_dering) { + const int size = 8; + const int ysize = size + 2 * OD_FILT_VBORDER; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint16_t, s[ysize * OD_FILT_BSTRIDE]); + DECLARE_ALIGNED(16, static uint16_t, d[size * size]); + DECLARE_ALIGNED(16, static uint16_t, ref_d[size * size]); + memset(ref_d, 0, sizeof(ref_d)); + memset(d, 0, sizeof(d)); + + int error = 0, threshold = 0, dir; + int boundary, damping, depth, bits, level, count, + errdepth = 0, errthreshold = 0, errboundary = 0, errdamping = 0; + unsigned int pos = 0; + + for (boundary = 0; boundary < 16; boundary++) { + for (depth = 8; depth <= 12; depth += 2) { + for (damping = 5 + depth - 8; damping < 7 + depth - 8; damping++) { + for (count = 0; count < iterations; count++) { + for (level = 0; level < (1 << depth) && !error; + level += (1 + 4 * !!boundary) << (depth - 8)) { + for (bits = 1; bits <= depth && !error; bits++) { + for (unsigned int i = 0; i < sizeof(s) / sizeof(*s); i++) + s[i] = clamp((rnd.Rand16() & ((1 << bits) - 1)) + level, 0, + (1 << depth) - 1); + if (boundary) { + if (boundary & 1) { // Left + for (int i = 0; i < ysize; i++) + for (int j = 0; j < OD_FILT_HBORDER; j++) + s[i * OD_FILT_BSTRIDE + j] = OD_DERING_VERY_LARGE; + } + if (boundary & 2) { // Right + for (int i = 0; i < ysize; i++) + for (int j = OD_FILT_HBORDER + size; j < OD_FILT_BSTRIDE; + j++) + s[i * OD_FILT_BSTRIDE + j] = OD_DERING_VERY_LARGE; + } + if (boundary & 4) { // Above + for (int i = 0; i < OD_FILT_VBORDER; i++) + for (int j = 0; j < OD_FILT_BSTRIDE; j++) + s[i * OD_FILT_BSTRIDE + j] = OD_DERING_VERY_LARGE; + } + if (boundary & 8) { // Below + for (int i = OD_FILT_VBORDER + size; i < ysize; i++) + for (int j = 0; j < OD_FILT_BSTRIDE; j++) + s[i * OD_FILT_BSTRIDE + j] = OD_DERING_VERY_LARGE; + } + } + for (dir = 0; dir < 8; dir++) { + for (threshold = 0; threshold < 64 << (depth - 8) && !error; + threshold += (1 + 4 * !!boundary) << (depth - 8)) { + ref_dering(ref_d, size, s + OD_FILT_HBORDER + + OD_FILT_VBORDER * OD_FILT_BSTRIDE, + threshold, dir, damping); + // If dering and ref_dering are the same, we're just testing + // speed + if (dering != ref_dering) + ASM_REGISTER_STATE_CHECK(dering( + d, size, + s + OD_FILT_HBORDER + OD_FILT_VBORDER * OD_FILT_BSTRIDE, + threshold, dir, damping)); + if (ref_dering != dering) { + for (pos = 0; pos < sizeof(d) / sizeof(*d) && !error; + pos++) { + error = ref_d[pos] != d[pos]; + errdepth = depth; + errthreshold = threshold; + errboundary = boundary; + errdamping = damping; + } + } + } + } + } + } + } + } + } + } + + pos--; + EXPECT_EQ(0, error) << "Error: CDEFDeringDirTest, SIMD and C mismatch." + << std::endl + << "First error at " << pos % size << "," << pos / size + << " (" << (int16_t)ref_d[pos] << " : " << (int16_t)d[pos] + << ") " << std::endl + << "threshold: " << errthreshold << std::endl + << "damping: " << errdamping << std::endl + << "depth: " << errdepth << std::endl + << "size: " << bsize << std::endl + << "boundary: " << errboundary << std::endl + << std::endl; +} + +void test_dering_speed(int bsize, int iterations, + od_filter_dering_direction_func dering, + od_filter_dering_direction_func ref_dering) { + aom_usec_timer ref_timer; + aom_usec_timer timer; + + aom_usec_timer_start(&ref_timer); + test_dering(bsize, iterations, ref_dering, ref_dering); + aom_usec_timer_mark(&ref_timer); + int ref_elapsed_time = (int)aom_usec_timer_elapsed(&ref_timer); + + aom_usec_timer_start(&timer); + test_dering(bsize, iterations, dering, dering); + aom_usec_timer_mark(&timer); + int elapsed_time = (int)aom_usec_timer_elapsed(&timer); + +#if 0 + std::cout << "[ ] C time = " << ref_elapsed_time / 1000 + << " ms, SIMD time = " << elapsed_time / 1000 << " ms" << std::endl; +#endif + + EXPECT_GT(ref_elapsed_time, elapsed_time) + << "Error: CDEFDeringSpeedTest, SIMD slower than C." << std::endl + << "C time: " << ref_elapsed_time << " us" << std::endl + << "SIMD time: " << elapsed_time << " us" << std::endl; +} + +typedef int (*find_dir_t)(const od_dering_in *img, int stride, int32_t *var, + int coeff_shift); + +typedef std::tr1::tuple find_dir_param_t; + +class CDEFDeringFindDirTest + : public ::testing::TestWithParam { + public: + virtual ~CDEFDeringFindDirTest() {} + virtual void SetUp() { + finddir = GET_PARAM(0); + ref_finddir = GET_PARAM(1); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + find_dir_t finddir; + find_dir_t ref_finddir; +}; + +typedef CDEFDeringFindDirTest CDEFDeringFindDirSpeedTest; + +void test_finddir(int (*finddir)(const od_dering_in *img, int stride, + int32_t *var, int coeff_shift), + int (*ref_finddir)(const od_dering_in *img, int stride, + int32_t *var, int coeff_shift)) { + const int size = 8; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint16_t, s[size * size]); + + int error = 0; + int depth, bits, level, count, errdepth = 0; + int ref_res = 0, res = 0; + int32_t ref_var = 0, var = 0; + + for (depth = 8; depth <= 12 && !error; depth += 2) { + for (count = 0; count < 512 && !error; count++) { + for (level = 0; level < (1 << depth) && !error; + level += 1 << (depth - 8)) { + for (bits = 1; bits <= depth && !error; bits++) { + for (unsigned int i = 0; i < sizeof(s) / sizeof(*s); i++) + s[i] = clamp((rnd.Rand16() & ((1 << bits) - 1)) + level, 0, + (1 << depth) - 1); + for (int c = 0; c < 1 + 9 * (finddir == ref_finddir); c++) + ref_res = ref_finddir(s, size, &ref_var, depth - 8); + if (finddir != ref_finddir) + ASM_REGISTER_STATE_CHECK(res = finddir(s, size, &var, depth - 8)); + if (ref_finddir != finddir) { + if (res != ref_res || var != ref_var) error = 1; + errdepth = depth; + } + } + } + } + } + + EXPECT_EQ(0, error) << "Error: CDEFDeringFindDirTest, SIMD and C mismatch." + << std::endl + << "return: " << res << " : " << ref_res << std::endl + << "var: " << var << " : " << ref_var << std::endl + << "depth: " << errdepth << std::endl + << std::endl; +} + +void test_finddir_speed(int (*finddir)(const od_dering_in *img, int stride, + int32_t *var, int coeff_shift), + int (*ref_finddir)(const od_dering_in *img, int stride, + int32_t *var, int coeff_shift)) { + aom_usec_timer ref_timer; + aom_usec_timer timer; + + aom_usec_timer_start(&ref_timer); + test_finddir(ref_finddir, ref_finddir); + aom_usec_timer_mark(&ref_timer); + int ref_elapsed_time = (int)aom_usec_timer_elapsed(&ref_timer); + + aom_usec_timer_start(&timer); + test_finddir(finddir, finddir); + aom_usec_timer_mark(&timer); + int elapsed_time = (int)aom_usec_timer_elapsed(&timer); + +#if 0 + std::cout << "[ ] C time = " << ref_elapsed_time / 1000 + << " ms, SIMD time = " << elapsed_time / 1000 << " ms" << std::endl; +#endif + + EXPECT_GT(ref_elapsed_time, elapsed_time) + << "Error: CDEFDeringFindDirSpeedTest, SIMD slower than C." << std::endl + << "C time: " << ref_elapsed_time << " us" << std::endl + << "SIMD time: " << elapsed_time << " us" << std::endl; +} + +TEST_P(CDEFDeringDirTest, TestSIMDNoMismatch) { + test_dering(bsize, 1, dering, ref_dering); +} + +TEST_P(CDEFDeringSpeedTest, DISABLED_TestSpeed) { + test_dering_speed(bsize, 4, dering, ref_dering); +} + +TEST_P(CDEFDeringFindDirTest, TestSIMDNoMismatch) { + test_finddir(finddir, ref_finddir); +} + +TEST_P(CDEFDeringFindDirSpeedTest, DISABLED_TestSpeed) { + test_finddir_speed(finddir, ref_finddir); +} + +using std::tr1::make_tuple; + +// VS compiling for 32 bit targets does not support vector types in +// structs as arguments, which makes the v256 type of the intrinsics +// hard to support, so optimizations for this target are disabled. +#if defined(_WIN64) || !defined(_MSC_VER) || defined(__clang__) +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P( + SSE2, CDEFDeringDirTest, + ::testing::Values(make_tuple(&od_filter_dering_direction_4x4_sse2, + &od_filter_dering_direction_4x4_c, 4), + make_tuple(&od_filter_dering_direction_8x8_sse2, + &od_filter_dering_direction_8x8_c, 8))); +INSTANTIATE_TEST_CASE_P(SSE2, CDEFDeringFindDirTest, + ::testing::Values(make_tuple(&od_dir_find8_sse2, + &od_dir_find8_c))); +#endif +#if HAVE_SSSE3 +INSTANTIATE_TEST_CASE_P( + SSSE3, CDEFDeringDirTest, + ::testing::Values(make_tuple(&od_filter_dering_direction_4x4_ssse3, + &od_filter_dering_direction_4x4_c, 4), + make_tuple(&od_filter_dering_direction_8x8_ssse3, + &od_filter_dering_direction_8x8_c, 8))); +INSTANTIATE_TEST_CASE_P(SSSE3, CDEFDeringFindDirTest, + ::testing::Values(make_tuple(&od_dir_find8_ssse3, + &od_dir_find8_c))); +#endif + +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P( + SSE4_1, CDEFDeringDirTest, + ::testing::Values(make_tuple(&od_filter_dering_direction_4x4_sse4_1, + &od_filter_dering_direction_4x4_c, 4), + make_tuple(&od_filter_dering_direction_8x8_sse4_1, + &od_filter_dering_direction_8x8_c, 8))); +INSTANTIATE_TEST_CASE_P(SSE4_1, CDEFDeringFindDirTest, + ::testing::Values(make_tuple(&od_dir_find8_sse4_1, + &od_dir_find8_c))); +#endif + +#if HAVE_NEON +INSTANTIATE_TEST_CASE_P( + NEON, CDEFDeringDirTest, + ::testing::Values(make_tuple(&od_filter_dering_direction_4x4_neon, + &od_filter_dering_direction_4x4_c, 4), + make_tuple(&od_filter_dering_direction_8x8_neon, + &od_filter_dering_direction_8x8_c, 8))); +INSTANTIATE_TEST_CASE_P(NEON, CDEFDeringFindDirTest, + ::testing::Values(make_tuple(&od_dir_find8_neon, + &od_dir_find8_c))); +#endif + +// Test speed for all supported architectures +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P( + SSE2, CDEFDeringSpeedTest, + ::testing::Values(make_tuple(&od_filter_dering_direction_4x4_sse2, + &od_filter_dering_direction_4x4_c, 4), + make_tuple(&od_filter_dering_direction_8x8_sse2, + &od_filter_dering_direction_8x8_c, 8))); +INSTANTIATE_TEST_CASE_P(SSE2, CDEFDeringFindDirSpeedTest, + ::testing::Values(make_tuple(&od_dir_find8_sse2, + &od_dir_find8_c))); +#endif + +#if HAVE_SSSE3 +INSTANTIATE_TEST_CASE_P( + SSSE3, CDEFDeringSpeedTest, + ::testing::Values(make_tuple(&od_filter_dering_direction_4x4_ssse3, + &od_filter_dering_direction_4x4_c, 4), + make_tuple(&od_filter_dering_direction_8x8_ssse3, + &od_filter_dering_direction_8x8_c, 8))); +INSTANTIATE_TEST_CASE_P(SSSE3, CDEFDeringFindDirSpeedTest, + ::testing::Values(make_tuple(&od_dir_find8_ssse3, + &od_dir_find8_c))); +#endif + +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P( + SSE4_1, CDEFDeringSpeedTest, + ::testing::Values(make_tuple(&od_filter_dering_direction_4x4_sse4_1, + &od_filter_dering_direction_4x4_c, 4), + make_tuple(&od_filter_dering_direction_8x8_sse4_1, + &od_filter_dering_direction_8x8_c, 8))); +INSTANTIATE_TEST_CASE_P(SSE4_1, CDEFDeringFindDirSpeedTest, + ::testing::Values(make_tuple(&od_dir_find8_sse4_1, + &od_dir_find8_c))); +#endif + +#if HAVE_NEON +INSTANTIATE_TEST_CASE_P( + NEON, CDEFDeringSpeedTest, + ::testing::Values(make_tuple(&od_filter_dering_direction_4x4_neon, + &od_filter_dering_direction_4x4_c, 4), + make_tuple(&od_filter_dering_direction_8x8_neon, + &od_filter_dering_direction_8x8_c, 8))); +INSTANTIATE_TEST_CASE_P(NEON, CDEFDeringFindDirSpeedTest, + ::testing::Values(make_tuple(&od_dir_find8_neon, + &od_dir_find8_c))); +#endif + +#endif // defined(_WIN64) || !defined(_MSC_VER) +} // namespace diff --git a/third_party/aom/test/divu_small_test.cc b/third_party/aom/test/divu_small_test.cc new file mode 100644 index 0000000000..064f8ee454 --- /dev/null +++ b/third_party/aom/test/divu_small_test.cc @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/acm_random.h" +#include "av1/common/odintrin.h" + +using libaom_test::ACMRandom; + +TEST(Daala, TestDIVUuptoMAX) { + for (int d = 1; d <= OD_DIVU_DMAX; d++) { + for (uint32_t x = 1; x <= 1000000; x++) { + GTEST_ASSERT_EQ(x / d, OD_DIVU_SMALL(x, d)) + << "x=" << x << " d=" << d << " x/d=" << (x / d) + << " != " << OD_DIVU_SMALL(x, d); + } + } +} + +TEST(Daala, TestDIVUrandI31) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + for (int d = 1; d < OD_DIVU_DMAX; d++) { + for (int i = 0; i < 1000000; i++) { + uint32_t x = rnd.Rand31(); + GTEST_ASSERT_EQ(x / d, OD_DIVU_SMALL(x, d)) + << "x=" << x << " d=" << d << " x/d=" << (x / d) + << " != " << OD_DIVU_SMALL(x, d); + } + } +} diff --git a/third_party/aom/test/encode_api_test.cc b/third_party/aom/test/encode_api_test.cc new file mode 100644 index 0000000000..14e43c847f --- /dev/null +++ b/third_party/aom/test/encode_api_test.cc @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "aom/aomcx.h" +#include "aom/aom_encoder.h" + +namespace { + +#define NELEMENTS(x) static_cast(sizeof(x) / sizeof(x[0])) + +TEST(EncodeAPI, InvalidParams) { + static const aom_codec_iface_t *kCodecs[] = { +#if CONFIG_AV1_ENCODER + &aom_codec_av1_cx_algo, +#endif + }; + uint8_t buf[1] = { 0 }; + aom_image_t img; + aom_codec_ctx_t enc; + aom_codec_enc_cfg_t cfg; + + EXPECT_EQ(&img, aom_img_wrap(&img, AOM_IMG_FMT_I420, 1, 1, 1, buf)); + + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_enc_init(NULL, NULL, NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_enc_init(&enc, NULL, NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_encode(NULL, NULL, 0, 0, 0, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_encode(NULL, &img, 0, 0, 0, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_destroy(NULL)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, + aom_codec_enc_config_default(NULL, NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, + aom_codec_enc_config_default(NULL, &cfg, 0)); + EXPECT_TRUE(aom_codec_error(NULL) != NULL); + + for (int i = 0; i < NELEMENTS(kCodecs); ++i) { + SCOPED_TRACE(aom_codec_iface_name(kCodecs[i])); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, + aom_codec_enc_init(NULL, kCodecs[i], NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, + aom_codec_enc_init(&enc, kCodecs[i], NULL, 0)); + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, + aom_codec_enc_config_default(kCodecs[i], &cfg, 1)); + + EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(kCodecs[i], &cfg, 0)); + EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, kCodecs[i], &cfg, 0)); + EXPECT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, NULL, 0, 0, 0, 0)); + + EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc)); + } +} + +} // namespace diff --git a/third_party/aom/test/encode_perf_test.cc b/third_party/aom/test/encode_perf_test.cc new file mode 100644 index 0000000000..e2a4f2b71c --- /dev/null +++ b/third_party/aom/test/encode_perf_test.cc @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "./aom_config.h" +#include "./aom_version.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" +#include "test/y4m_video_source.h" +#include "aom_ports/aom_timer.h" + +namespace { + +const int kMaxPsnr = 100; +const double kUsecsInSec = 1000000.0; + +struct EncodePerfTestVideo { + EncodePerfTestVideo(const char *name_, uint32_t width_, uint32_t height_, + uint32_t bitrate_, int frames_) + : name(name_), width(width_), height(height_), bitrate(bitrate_), + frames(frames_) {} + const char *name; + uint32_t width; + uint32_t height; + uint32_t bitrate; + int frames; +}; + +const EncodePerfTestVideo kAV1EncodePerfTestVectors[] = { + EncodePerfTestVideo("desktop_640_360_30.yuv", 640, 360, 200, 2484), + EncodePerfTestVideo("kirland_640_480_30.yuv", 640, 480, 200, 300), + EncodePerfTestVideo("macmarcomoving_640_480_30.yuv", 640, 480, 200, 987), + EncodePerfTestVideo("macmarcostationary_640_480_30.yuv", 640, 480, 200, 718), + EncodePerfTestVideo("niklas_640_480_30.yuv", 640, 480, 200, 471), + EncodePerfTestVideo("tacomanarrows_640_480_30.yuv", 640, 480, 200, 300), + EncodePerfTestVideo("tacomasmallcameramovement_640_480_30.yuv", 640, 480, 200, + 300), + EncodePerfTestVideo("thaloundeskmtg_640_480_30.yuv", 640, 480, 200, 300), + EncodePerfTestVideo("niklas_1280_720_30.yuv", 1280, 720, 600, 470), +}; + +const int kEncodePerfTestSpeeds[] = { 5, 6, 7, 8 }; +const int kEncodePerfTestThreads[] = { 1, 2, 4 }; + +#define NELEMENTS(x) (sizeof((x)) / sizeof((x)[0])) + +class AV1EncodePerfTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWithParam { + protected: + AV1EncodePerfTest() + : EncoderTest(GET_PARAM(0)), min_psnr_(kMaxPsnr), nframes_(0), + encoding_mode_(GET_PARAM(1)), speed_(0), threads_(1) {} + + virtual ~AV1EncodePerfTest() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + + cfg_.g_lag_in_frames = 0; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 56; + cfg_.rc_dropframe_thresh = 0; + cfg_.rc_undershoot_pct = 50; + cfg_.rc_overshoot_pct = 50; + cfg_.rc_buf_sz = 1000; + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 600; + cfg_.rc_resize_allowed = 0; + cfg_.rc_end_usage = AOM_CBR; + cfg_.g_error_resilient = 1; + cfg_.g_threads = threads_; + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 0) { + const int log2_tile_columns = 3; + encoder->Control(AOME_SET_CPUUSED, speed_); + encoder->Control(AV1E_SET_TILE_COLUMNS, log2_tile_columns); + encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1); + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 0); + } + } + + virtual void BeginPassHook(unsigned int /*pass*/) { + min_psnr_ = kMaxPsnr; + nframes_ = 0; + } + + virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) { + if (pkt->data.psnr.psnr[0] < min_psnr_) { + min_psnr_ = pkt->data.psnr.psnr[0]; + } + } + + // for performance reasons don't decode + virtual bool DoDecode() { return 0; } + + double min_psnr() const { return min_psnr_; } + + void set_speed(unsigned int speed) { speed_ = speed; } + + void set_threads(unsigned int threads) { threads_ = threads; } + + private: + double min_psnr_; + unsigned int nframes_; + libaom_test::TestMode encoding_mode_; + unsigned speed_; + unsigned int threads_; +}; + +TEST_P(AV1EncodePerfTest, PerfTest) { + for (size_t i = 0; i < NELEMENTS(kAV1EncodePerfTestVectors); ++i) { + for (size_t j = 0; j < NELEMENTS(kEncodePerfTestSpeeds); ++j) { + for (size_t k = 0; k < NELEMENTS(kEncodePerfTestThreads); ++k) { + if (kAV1EncodePerfTestVectors[i].width < 512 && + kEncodePerfTestThreads[k] > 1) + continue; + else if (kAV1EncodePerfTestVectors[i].width < 1024 && + kEncodePerfTestThreads[k] > 2) + continue; + + set_threads(kEncodePerfTestThreads[k]); + SetUp(); + + const aom_rational timebase = { 33333333, 1000000000 }; + cfg_.g_timebase = timebase; + cfg_.rc_target_bitrate = kAV1EncodePerfTestVectors[i].bitrate; + + init_flags_ = AOM_CODEC_USE_PSNR; + + const unsigned frames = kAV1EncodePerfTestVectors[i].frames; + const char *video_name = kAV1EncodePerfTestVectors[i].name; + libaom_test::I420VideoSource video( + video_name, kAV1EncodePerfTestVectors[i].width, + kAV1EncodePerfTestVectors[i].height, timebase.den, timebase.num, 0, + kAV1EncodePerfTestVectors[i].frames); + set_speed(kEncodePerfTestSpeeds[j]); + + aom_usec_timer t; + aom_usec_timer_start(&t); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + + aom_usec_timer_mark(&t); + const double elapsed_secs = aom_usec_timer_elapsed(&t) / kUsecsInSec; + const double fps = frames / elapsed_secs; + const double minimum_psnr = min_psnr(); + std::string display_name(video_name); + if (kEncodePerfTestThreads[k] > 1) { + char thread_count[32]; + snprintf(thread_count, sizeof(thread_count), "_t-%d", + kEncodePerfTestThreads[k]); + display_name += thread_count; + } + + printf("{\n"); + printf("\t\"type\" : \"encode_perf_test\",\n"); + printf("\t\"version\" : \"%s\",\n", VERSION_STRING_NOSP); + printf("\t\"videoName\" : \"%s\",\n", display_name.c_str()); + printf("\t\"encodeTimeSecs\" : %f,\n", elapsed_secs); + printf("\t\"totalFrames\" : %u,\n", frames); + printf("\t\"framesPerSecond\" : %f,\n", fps); + printf("\t\"minPsnr\" : %f,\n", minimum_psnr); + printf("\t\"speed\" : %d,\n", kEncodePerfTestSpeeds[j]); + printf("\t\"threads\" : %d\n", kEncodePerfTestThreads[k]); + printf("}\n"); + } + } + } +} + +AV1_INSTANTIATE_TEST_CASE(AV1EncodePerfTest, + ::testing::Values(::libaom_test::kRealTime)); +} // namespace diff --git a/third_party/aom/test/encode_test_driver.cc b/third_party/aom/test/encode_test_driver.cc new file mode 100644 index 0000000000..80f155ab26 --- /dev/null +++ b/third_party/aom/test/encode_test_driver.cc @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "aom_ports/mem.h" +#include "test/codec_factory.h" +#include "test/decode_test_driver.h" +#include "test/encode_test_driver.h" +#include "test/register_state_check.h" +#include "test/video_source.h" + +namespace libaom_test { +void Encoder::InitEncoder(VideoSource *video) { + aom_codec_err_t res; + const aom_image_t *img = video->img(); + + if (video->img() && !encoder_.priv) { + cfg_.g_w = img->d_w; + cfg_.g_h = img->d_h; + cfg_.g_timebase = video->timebase(); + cfg_.rc_twopass_stats_in = stats_->buf(); + + res = aom_codec_enc_init(&encoder_, CodecInterface(), &cfg_, init_flags_); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); + +#if CONFIG_AV1_ENCODER + if (CodecInterface() == &aom_codec_av1_cx_algo) { +// Default to 1 tile column for AV1. With CONFIG_EXT_TILE, the +// default is already the largest possible tile size +#if !CONFIG_EXT_TILE + const int log2_tile_columns = 0; + res = aom_codec_control_(&encoder_, AV1E_SET_TILE_COLUMNS, + log2_tile_columns); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); +#endif // !CONFIG_EXT_TILE + } else +#endif + { + } + } +} + +void Encoder::EncodeFrame(VideoSource *video, const unsigned long frame_flags) { + if (video->img()) + EncodeFrameInternal(*video, frame_flags); + else + Flush(); + + // Handle twopass stats + CxDataIterator iter = GetCxData(); + + while (const aom_codec_cx_pkt_t *pkt = iter.Next()) { + if (pkt->kind != AOM_CODEC_STATS_PKT) continue; + + stats_->Append(*pkt); + } +} + +void Encoder::EncodeFrameInternal(const VideoSource &video, + const unsigned long frame_flags) { + aom_codec_err_t res; + const aom_image_t *img = video.img(); + + // Handle frame resizing + if (cfg_.g_w != img->d_w || cfg_.g_h != img->d_h) { + cfg_.g_w = img->d_w; + cfg_.g_h = img->d_h; + res = aom_codec_enc_config_set(&encoder_, &cfg_); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); + } + + // Encode the frame + API_REGISTER_STATE_CHECK(res = aom_codec_encode(&encoder_, img, video.pts(), + video.duration(), frame_flags, + deadline_)); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); +} + +void Encoder::Flush() { + const aom_codec_err_t res = + aom_codec_encode(&encoder_, NULL, 0, 0, 0, deadline_); + if (!encoder_.priv) + ASSERT_EQ(AOM_CODEC_ERROR, res) << EncoderError(); + else + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); +} + +void EncoderTest::InitializeConfig() { + const aom_codec_err_t res = codec_->DefaultEncoderConfig(&cfg_, 0); + dec_cfg_ = aom_codec_dec_cfg_t(); + ASSERT_EQ(AOM_CODEC_OK, res); +} + +void EncoderTest::SetMode(TestMode mode) { + switch (mode) { + case kOnePassGood: + case kTwoPassGood: deadline_ = AOM_DL_GOOD_QUALITY; break; + case kRealTime: + deadline_ = AOM_DL_GOOD_QUALITY; + cfg_.g_lag_in_frames = 0; + break; + default: ASSERT_TRUE(false) << "Unexpected mode " << mode; + } + mode_ = mode; + if (mode == kTwoPassGood) + passes_ = 2; + else + passes_ = 1; +} + +static bool compare_plane(const uint8_t *const buf1, int stride1, + const uint8_t *const buf2, int stride2, int w, int h, + int *const mismatch_row, int *const mismatch_col, + int *const mismatch_pix1, int *const mismatch_pix2) { + int r, c; + + for (r = 0; r < h; ++r) { + for (c = 0; c < w; ++c) { + const int pix1 = buf1[r * stride1 + c]; + const int pix2 = buf2[r * stride2 + c]; + + if (pix1 != pix2) { + if (mismatch_row != NULL) *mismatch_row = r; + if (mismatch_col != NULL) *mismatch_col = c; + if (mismatch_pix1 != NULL) *mismatch_pix1 = pix1; + if (mismatch_pix2 != NULL) *mismatch_pix2 = pix2; + return false; + } + } + } + + return true; +} + +// The function should return "true" most of the time, therefore no early +// break-out is implemented within the match checking process. +static bool compare_img(const aom_image_t *img1, const aom_image_t *img2, + int *const mismatch_row, int *const mismatch_col, + int *const mismatch_plane, int *const mismatch_pix1, + int *const mismatch_pix2) { + const unsigned int w_y = img1->d_w; + const unsigned int h_y = img1->d_h; + const unsigned int w_uv = ROUND_POWER_OF_TWO(w_y, img1->x_chroma_shift); + const unsigned int h_uv = ROUND_POWER_OF_TWO(h_y, img1->y_chroma_shift); + + if (img1->fmt != img2->fmt || img1->cs != img2->cs || + img1->d_w != img2->d_w || img1->d_h != img2->d_h) { + if (mismatch_row != NULL) *mismatch_row = -1; + if (mismatch_col != NULL) *mismatch_col = -1; + return false; + } + + if (!compare_plane(img1->planes[AOM_PLANE_Y], img1->stride[AOM_PLANE_Y], + img2->planes[AOM_PLANE_Y], img2->stride[AOM_PLANE_Y], w_y, + h_y, mismatch_row, mismatch_col, mismatch_pix1, + mismatch_pix2)) { + if (mismatch_plane != NULL) *mismatch_plane = AOM_PLANE_Y; + return false; + } + + if (!compare_plane(img1->planes[AOM_PLANE_U], img1->stride[AOM_PLANE_U], + img2->planes[AOM_PLANE_U], img2->stride[AOM_PLANE_U], w_uv, + h_uv, mismatch_row, mismatch_col, mismatch_pix1, + mismatch_pix2)) { + if (mismatch_plane != NULL) *mismatch_plane = AOM_PLANE_U; + return false; + } + + if (!compare_plane(img1->planes[AOM_PLANE_V], img1->stride[AOM_PLANE_V], + img2->planes[AOM_PLANE_V], img2->stride[AOM_PLANE_V], w_uv, + h_uv, mismatch_row, mismatch_col, mismatch_pix1, + mismatch_pix2)) { + if (mismatch_plane != NULL) *mismatch_plane = AOM_PLANE_U; + return false; + } + + return true; +} + +void EncoderTest::MismatchHook(const aom_image_t *img_enc, + const aom_image_t *img_dec) { + int mismatch_row = 0; + int mismatch_col = 0; + int mismatch_plane = 0; + int mismatch_pix_enc = 0; + int mismatch_pix_dec = 0; + + ASSERT_FALSE(compare_img(img_enc, img_dec, &mismatch_row, &mismatch_col, + &mismatch_plane, &mismatch_pix_enc, + &mismatch_pix_dec)); + + GTEST_FAIL() << "Encode/Decode mismatch found:" << std::endl + << " pixel value enc/dec: " << mismatch_pix_enc << "/" + << mismatch_pix_dec << std::endl + << " plane: " << mismatch_plane << std::endl + << " row/col: " << mismatch_row << "/" + << mismatch_col << std::endl; +} + +void EncoderTest::RunLoop(VideoSource *video) { + aom_codec_dec_cfg_t dec_cfg = aom_codec_dec_cfg_t(); + + stats_.Reset(); + + ASSERT_TRUE(passes_ == 1 || passes_ == 2); + for (unsigned int pass = 0; pass < passes_; pass++) { + last_pts_ = 0; + + if (passes_ == 1) + cfg_.g_pass = AOM_RC_ONE_PASS; + else if (pass == 0) + cfg_.g_pass = AOM_RC_FIRST_PASS; + else + cfg_.g_pass = AOM_RC_LAST_PASS; + + BeginPassHook(pass); + testing::internal::scoped_ptr encoder( + codec_->CreateEncoder(cfg_, deadline_, init_flags_, &stats_)); + ASSERT_TRUE(encoder.get() != NULL); + + ASSERT_NO_FATAL_FAILURE(video->Begin()); + encoder->InitEncoder(video); + + if (mode_ == kRealTime) { + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 0); + } + + ASSERT_FALSE(::testing::Test::HasFatalFailure()); + + unsigned long dec_init_flags = 0; // NOLINT + // Use fragment decoder if encoder outputs partitions. + // NOTE: fragment decoder and partition encoder are only supported by VP8. + if (init_flags_ & AOM_CODEC_USE_OUTPUT_PARTITION) + dec_init_flags |= AOM_CODEC_USE_INPUT_FRAGMENTS; + testing::internal::scoped_ptr decoder( + codec_->CreateDecoder(dec_cfg, dec_init_flags)); +#if CONFIG_AV1 && CONFIG_EXT_TILE + if (decoder->IsAV1()) { + // Set dec_cfg.tile_row = -1 and dec_cfg.tile_col = -1 so that the whole + // frame is decoded. + decoder->Control(AV1_SET_DECODE_TILE_ROW, -1); + decoder->Control(AV1_SET_DECODE_TILE_COL, -1); + } +#endif + + bool again; + for (again = true; again; video->Next()) { + again = (video->img() != NULL); + + PreEncodeFrameHook(video); + PreEncodeFrameHook(video, encoder.get()); + encoder->EncodeFrame(video, frame_flags_); + + CxDataIterator iter = encoder->GetCxData(); + + bool has_cxdata = false; + bool has_dxdata = false; + while (const aom_codec_cx_pkt_t *pkt = iter.Next()) { + pkt = MutateEncoderOutputHook(pkt); + again = true; + switch (pkt->kind) { + case AOM_CODEC_CX_FRAME_PKT: + has_cxdata = true; + if (decoder.get() != NULL && DoDecode()) { + aom_codec_err_t res_dec = decoder->DecodeFrame( + (const uint8_t *)pkt->data.frame.buf, pkt->data.frame.sz); + + if (!HandleDecodeResult(res_dec, decoder.get())) break; + + has_dxdata = true; + } + ASSERT_GE(pkt->data.frame.pts, last_pts_); + last_pts_ = pkt->data.frame.pts; + FramePktHook(pkt); + break; + + case AOM_CODEC_PSNR_PKT: PSNRPktHook(pkt); break; + + default: break; + } + } + + // Flush the decoder when there are no more fragments. + if ((init_flags_ & AOM_CODEC_USE_OUTPUT_PARTITION) && has_dxdata) { + const aom_codec_err_t res_dec = decoder->DecodeFrame(NULL, 0); + if (!HandleDecodeResult(res_dec, decoder.get())) break; + } + + if (has_dxdata && has_cxdata) { + const aom_image_t *img_enc = encoder->GetPreviewFrame(); + DxDataIterator dec_iter = decoder->GetDxData(); + const aom_image_t *img_dec = dec_iter.Next(); + if (img_enc && img_dec) { + const bool res = + compare_img(img_enc, img_dec, NULL, NULL, NULL, NULL, NULL); + if (!res) { // Mismatch + MismatchHook(img_enc, img_dec); + } + } + if (img_dec) DecompressedFrameHook(*img_dec, video->pts()); + } + if (!Continue()) break; + } + + EndPassHook(); + + if (!Continue()) break; + } +} + +} // namespace libaom_test diff --git a/third_party/aom/test/encode_test_driver.h b/third_party/aom/test/encode_test_driver.h new file mode 100644 index 0000000000..91027b4f6d --- /dev/null +++ b/third_party/aom/test/encode_test_driver.h @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef TEST_ENCODE_TEST_DRIVER_H_ +#define TEST_ENCODE_TEST_DRIVER_H_ + +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#if CONFIG_AV1_ENCODER +#include "aom/aomcx.h" +#endif +#include "aom/aom_encoder.h" + +namespace libaom_test { + +class CodecFactory; +class VideoSource; + +enum TestMode { kRealTime, kOnePassGood, kTwoPassGood }; +#define ALL_TEST_MODES \ + ::testing::Values(::libaom_test::kRealTime, ::libaom_test::kOnePassGood, \ + ::libaom_test::kTwoPassGood) + +#define ONE_PASS_TEST_MODES \ + ::testing::Values(::libaom_test::kRealTime, ::libaom_test::kOnePassGood) + +#define TWO_PASS_TEST_MODES ::testing::Values(::libaom_test::kTwoPassGood) + +// Provides an object to handle the libaom get_cx_data() iteration pattern +class CxDataIterator { + public: + explicit CxDataIterator(aom_codec_ctx_t *encoder) + : encoder_(encoder), iter_(NULL) {} + + const aom_codec_cx_pkt_t *Next() { + return aom_codec_get_cx_data(encoder_, &iter_); + } + + private: + aom_codec_ctx_t *encoder_; + aom_codec_iter_t iter_; +}; + +// Implements an in-memory store for libaom twopass statistics +class TwopassStatsStore { + public: + void Append(const aom_codec_cx_pkt_t &pkt) { + buffer_.append(reinterpret_cast(pkt.data.twopass_stats.buf), + pkt.data.twopass_stats.sz); + } + + aom_fixed_buf_t buf() { + const aom_fixed_buf_t buf = { &buffer_[0], buffer_.size() }; + return buf; + } + + void Reset() { buffer_.clear(); } + + protected: + std::string buffer_; +}; + +// Provides a simplified interface to manage one video encoding pass, given +// a configuration and video source. +// +// TODO(jkoleszar): The exact services it provides and the appropriate +// level of abstraction will be fleshed out as more tests are written. +class Encoder { + public: + Encoder(aom_codec_enc_cfg_t cfg, unsigned long deadline, + const unsigned long init_flags, TwopassStatsStore *stats) + : cfg_(cfg), deadline_(deadline), init_flags_(init_flags), stats_(stats) { + memset(&encoder_, 0, sizeof(encoder_)); + } + + virtual ~Encoder() { aom_codec_destroy(&encoder_); } + + CxDataIterator GetCxData() { return CxDataIterator(&encoder_); } + + void InitEncoder(VideoSource *video); + + const aom_image_t *GetPreviewFrame() { + return aom_codec_get_preview_frame(&encoder_); + } + // This is a thin wrapper around aom_codec_encode(), so refer to + // aom_encoder.h for its semantics. + void EncodeFrame(VideoSource *video, const unsigned long frame_flags); + + // Convenience wrapper for EncodeFrame() + void EncodeFrame(VideoSource *video) { EncodeFrame(video, 0); } + + void Control(int ctrl_id, int arg) { + const aom_codec_err_t res = aom_codec_control_(&encoder_, ctrl_id, arg); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); + } + + void Control(int ctrl_id, int *arg) { + const aom_codec_err_t res = aom_codec_control_(&encoder_, ctrl_id, arg); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); + } + + void Control(int ctrl_id, struct aom_scaling_mode *arg) { + const aom_codec_err_t res = aom_codec_control_(&encoder_, ctrl_id, arg); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); + } + +#if CONFIG_AV1_ENCODER + void Control(int ctrl_id, aom_active_map_t *arg) { + const aom_codec_err_t res = aom_codec_control_(&encoder_, ctrl_id, arg); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); + } +#endif + + void Config(const aom_codec_enc_cfg_t *cfg) { + const aom_codec_err_t res = aom_codec_enc_config_set(&encoder_, cfg); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); + cfg_ = *cfg; + } + + void set_deadline(unsigned long deadline) { deadline_ = deadline; } + + protected: + virtual aom_codec_iface_t *CodecInterface() const = 0; + + const char *EncoderError() { + const char *detail = aom_codec_error_detail(&encoder_); + return detail ? detail : aom_codec_error(&encoder_); + } + + // Encode an image + void EncodeFrameInternal(const VideoSource &video, + const unsigned long frame_flags); + + // Flush the encoder on EOS + void Flush(); + + aom_codec_ctx_t encoder_; + aom_codec_enc_cfg_t cfg_; + unsigned long deadline_; + unsigned long init_flags_; + TwopassStatsStore *stats_; +}; + +// Common test functionality for all Encoder tests. +// +// This class is a mixin which provides the main loop common to all +// encoder tests. It provides hooks which can be overridden by subclasses +// to implement each test's specific behavior, while centralizing the bulk +// of the boilerplate. Note that it doesn't inherit the gtest testing +// classes directly, so that tests can be parameterized differently. +class EncoderTest { + protected: + explicit EncoderTest(const CodecFactory *codec) + : codec_(codec), abort_(false), init_flags_(0), frame_flags_(0), + last_pts_(0), mode_(kRealTime) { + // Default to 1 thread. + cfg_.g_threads = 1; + } + + virtual ~EncoderTest() {} + + // Initialize the cfg_ member with the default configuration. + void InitializeConfig(); + + // Map the TestMode enum to the deadline_ and passes_ variables. + void SetMode(TestMode mode); + + // Set encoder flag. + void set_init_flags(unsigned long flag) { // NOLINT(runtime/int) + init_flags_ = flag; + } + + // Main loop + virtual void RunLoop(VideoSource *video); + + // Hook to be called at the beginning of a pass. + virtual void BeginPassHook(unsigned int /*pass*/) {} + + // Hook to be called at the end of a pass. + virtual void EndPassHook() {} + + // Hook to be called before encoding a frame. + virtual void PreEncodeFrameHook(VideoSource * /*video*/) {} + virtual void PreEncodeFrameHook(VideoSource * /*video*/, + Encoder * /*encoder*/) {} + + // Hook to be called on every compressed data packet. + virtual void FramePktHook(const aom_codec_cx_pkt_t * /*pkt*/) {} + + // Hook to be called on every PSNR packet. + virtual void PSNRPktHook(const aom_codec_cx_pkt_t * /*pkt*/) {} + + // Hook to determine whether the encode loop should continue. + virtual bool Continue() const { + return !(::testing::Test::HasFatalFailure() || abort_); + } + + const CodecFactory *codec_; + // Hook to determine whether to decode frame after encoding + virtual bool DoDecode() const { return 1; } + + // Hook to handle encode/decode mismatch + virtual void MismatchHook(const aom_image_t *img1, const aom_image_t *img2); + + // Hook to be called on every decompressed frame. + virtual void DecompressedFrameHook(const aom_image_t & /*img*/, + aom_codec_pts_t /*pts*/) {} + + // Hook to be called to handle decode result. Return true to continue. + virtual bool HandleDecodeResult(const aom_codec_err_t res_dec, + Decoder *decoder) { + EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError(); + return AOM_CODEC_OK == res_dec; + } + + // Hook that can modify the encoder's output data + virtual const aom_codec_cx_pkt_t *MutateEncoderOutputHook( + const aom_codec_cx_pkt_t *pkt) { + return pkt; + } + + bool abort_; + aom_codec_enc_cfg_t cfg_; + aom_codec_dec_cfg_t dec_cfg_; + unsigned int passes_; + unsigned long deadline_; + TwopassStatsStore stats_; + unsigned long init_flags_; + unsigned long frame_flags_; + aom_codec_pts_t last_pts_; + TestMode mode_; +}; + +} // namespace libaom_test + +#endif // TEST_ENCODE_TEST_DRIVER_H_ diff --git a/third_party/aom/test/encoder_parms_get_to_decoder.cc b/third_party/aom/test/encoder_parms_get_to_decoder.cc new file mode 100644 index 0000000000..ca6a24ebea --- /dev/null +++ b/third_party/aom/test/encoder_parms_get_to_decoder.cc @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/util.h" +#include "test/y4m_video_source.h" +#include "av1/av1_dx_iface.c" + +namespace { + +const int kCpuUsed = 2; + +struct EncodePerfTestVideo { + const char *name; + uint32_t width; + uint32_t height; + uint32_t bitrate; + int frames; +}; + +const EncodePerfTestVideo kAV1EncodePerfTestVectors[] = { + { "niklas_1280_720_30.y4m", 1280, 720, 600, 10 }, +}; + +struct EncodeParameters { + int32_t tile_rows; + int32_t tile_cols; + int32_t lossless; + int32_t error_resilient; + int32_t frame_parallel; + aom_color_range_t color_range; + aom_color_space_t cs; + int render_size[2]; + // TODO(JBB): quantizers / bitrate +}; + +const EncodeParameters kAV1EncodeParameterSet[] = { + { 0, 0, 0, 1, 0, AOM_CR_STUDIO_RANGE, AOM_CS_BT_601, { 0, 0 } }, + { 0, 0, 0, 0, 0, AOM_CR_FULL_RANGE, AOM_CS_BT_709, { 0, 0 } }, + { 0, 0, 1, 0, 0, AOM_CR_FULL_RANGE, AOM_CS_BT_2020, { 0, 0 } }, + { 0, 2, 0, 0, 1, AOM_CR_STUDIO_RANGE, AOM_CS_UNKNOWN, { 640, 480 } }, + // TODO(JBB): Test profiles (requires more work). +}; + +class AvxEncoderParmsGetToDecoder + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params { + protected: + AvxEncoderParmsGetToDecoder() + : EncoderTest(GET_PARAM(0)), encode_parms(GET_PARAM(1)) {} + + virtual ~AvxEncoderParmsGetToDecoder() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(::libaom_test::kTwoPassGood); + cfg_.g_lag_in_frames = 25; + cfg_.g_error_resilient = encode_parms.error_resilient; + dec_cfg_.threads = 4; + test_video_ = GET_PARAM(2); + cfg_.rc_target_bitrate = test_video_.bitrate; + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 1) { + encoder->Control(AV1E_SET_COLOR_SPACE, encode_parms.cs); + encoder->Control(AV1E_SET_COLOR_RANGE, encode_parms.color_range); + encoder->Control(AV1E_SET_LOSSLESS, encode_parms.lossless); + encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, + encode_parms.frame_parallel); + encoder->Control(AV1E_SET_TILE_ROWS, encode_parms.tile_rows); + encoder->Control(AV1E_SET_TILE_COLUMNS, encode_parms.tile_cols); + encoder->Control(AOME_SET_CPUUSED, kCpuUsed); + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); + encoder->Control(AOME_SET_ARNR_MAXFRAMES, 7); + encoder->Control(AOME_SET_ARNR_STRENGTH, 5); + if (encode_parms.render_size[0] > 0 && encode_parms.render_size[1] > 0) + encoder->Control(AV1E_SET_RENDER_SIZE, encode_parms.render_size); + } + } + + virtual bool HandleDecodeResult(const aom_codec_err_t res_dec, + libaom_test::Decoder *decoder) { + aom_codec_ctx_t *const av1_decoder = decoder->GetDecoder(); + aom_codec_alg_priv_t *const priv = + reinterpret_cast(av1_decoder->priv); + FrameWorkerData *const worker_data = + reinterpret_cast(priv->frame_workers[0].data1); + AV1_COMMON *const common = &worker_data->pbi->common; + + if (encode_parms.lossless) { + EXPECT_EQ(0, common->base_qindex); + EXPECT_EQ(0, common->y_dc_delta_q); + EXPECT_EQ(0, common->uv_dc_delta_q); + EXPECT_EQ(0, common->uv_ac_delta_q); + EXPECT_EQ(ONLY_4X4, common->tx_mode); + } + EXPECT_EQ(encode_parms.error_resilient, common->error_resilient_mode); + if (encode_parms.error_resilient) { + EXPECT_EQ(0, common->use_prev_frame_mvs); + } + EXPECT_EQ(encode_parms.color_range, common->color_range); + EXPECT_EQ(encode_parms.cs, common->color_space); + if (encode_parms.render_size[0] > 0 && encode_parms.render_size[1] > 0) { + EXPECT_EQ(encode_parms.render_size[0], common->render_width); + EXPECT_EQ(encode_parms.render_size[1], common->render_height); + } + EXPECT_EQ(encode_parms.tile_cols, common->log2_tile_cols); + EXPECT_EQ(encode_parms.tile_rows, common->log2_tile_rows); + + EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError(); + return AOM_CODEC_OK == res_dec; + } + + EncodePerfTestVideo test_video_; + + private: + EncodeParameters encode_parms; +}; + +TEST_P(AvxEncoderParmsGetToDecoder, BitstreamParms) { + init_flags_ = AOM_CODEC_USE_PSNR; + + testing::internal::scoped_ptr video( + new libaom_test::Y4mVideoSource(test_video_.name, 0, test_video_.frames)); + ASSERT_TRUE(video.get() != NULL); + + ASSERT_NO_FATAL_FAILURE(RunLoop(video.get())); +} + +AV1_INSTANTIATE_TEST_CASE(AvxEncoderParmsGetToDecoder, + ::testing::ValuesIn(kAV1EncodeParameterSet), + ::testing::ValuesIn(kAV1EncodePerfTestVectors)); +} // namespace diff --git a/third_party/aom/test/end_to_end_test.cc b/third_party/aom/test/end_to_end_test.cc new file mode 100644 index 0000000000..0c8cbe274c --- /dev/null +++ b/third_party/aom/test/end_to_end_test.cc @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/util.h" +#include "test/y4m_video_source.h" +#include "test/yuv_video_source.h" + +namespace { + +const unsigned int kWidth = 160; +const unsigned int kHeight = 90; +const unsigned int kFramerate = 50; +const unsigned int kFrames = 10; +const int kBitrate = 500; +// List of psnr thresholds for speed settings 0-7 and 5 encoding modes +const double kPsnrThreshold[][5] = { +// Note: +// AV1 HBD average PSNR is slightly lower than AV1. +// We make two cases here to enable the testing and +// guard picture quality. +#if CONFIG_AV1_ENCODER && CONFIG_HIGHBITDEPTH + { 36.0, 37.0, 37.0, 37.0, 37.0 }, { 31.0, 36.0, 36.0, 36.0, 36.0 }, + { 31.0, 35.0, 35.0, 35.0, 35.0 }, { 31.0, 34.0, 34.0, 34.0, 34.0 }, + { 31.0, 33.0, 33.0, 33.0, 33.0 }, { 31.0, 32.0, 32.0, 32.0, 32.0 }, + { 30.0, 31.0, 31.0, 31.0, 31.0 }, { 29.0, 30.0, 30.0, 30.0, 30.0 }, +#else + { 36.0, 37.0, 37.0, 37.0, 37.0 }, { 35.0, 36.0, 36.0, 36.0, 36.0 }, + { 34.0, 35.0, 35.0, 35.0, 35.0 }, { 33.0, 34.0, 34.0, 34.0, 34.0 }, + { 32.0, 33.0, 33.0, 33.0, 33.0 }, { 31.0, 32.0, 32.0, 32.0, 32.0 }, + { 30.0, 31.0, 31.0, 31.0, 31.0 }, { 29.0, 30.0, 30.0, 30.0, 30.0 }, +#endif // CONFIG_HIGHBITDEPTH && CONFIG_AV1_ENCODER +}; + +typedef struct { + const char *filename; + unsigned int input_bit_depth; + aom_img_fmt fmt; + aom_bit_depth_t bit_depth; + unsigned int profile; +} TestVideoParam; + +const TestVideoParam kTestVectors[] = { + { "park_joy_90p_8_420.y4m", 8, AOM_IMG_FMT_I420, AOM_BITS_8, 0 }, + { "park_joy_90p_8_422.y4m", 8, AOM_IMG_FMT_I422, AOM_BITS_8, 1 }, + { "park_joy_90p_8_444.y4m", 8, AOM_IMG_FMT_I444, AOM_BITS_8, 1 }, + { "park_joy_90p_8_440.yuv", 8, AOM_IMG_FMT_I440, AOM_BITS_8, 1 }, +#if CONFIG_HIGHBITDEPTH + { "park_joy_90p_10_420.y4m", 10, AOM_IMG_FMT_I42016, AOM_BITS_10, 2 }, + { "park_joy_90p_10_422.y4m", 10, AOM_IMG_FMT_I42216, AOM_BITS_10, 3 }, + { "park_joy_90p_10_444.y4m", 10, AOM_IMG_FMT_I44416, AOM_BITS_10, 3 }, + { "park_joy_90p_10_440.yuv", 10, AOM_IMG_FMT_I44016, AOM_BITS_10, 3 }, + { "park_joy_90p_12_420.y4m", 12, AOM_IMG_FMT_I42016, AOM_BITS_12, 2 }, + { "park_joy_90p_12_422.y4m", 12, AOM_IMG_FMT_I42216, AOM_BITS_12, 3 }, + { "park_joy_90p_12_444.y4m", 12, AOM_IMG_FMT_I44416, AOM_BITS_12, 3 }, + { "park_joy_90p_12_440.yuv", 12, AOM_IMG_FMT_I44016, AOM_BITS_12, 3 }, +#endif // CONFIG_HIGHBITDEPTH +}; + +// Encoding modes tested +const libaom_test::TestMode kEncodingModeVectors[] = { + ::libaom_test::kTwoPassGood, ::libaom_test::kOnePassGood, + ::libaom_test::kRealTime, +}; + +// Speed settings tested +const int kCpuUsedVectors[] = { 1, 2, 3, 5, 6 }; + +int is_extension_y4m(const char *filename) { + const char *dot = strrchr(filename, '.'); + if (!dot || dot == filename) + return 0; + else + return !strcmp(dot, ".y4m"); +} + +class EndToEndTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith3Params { + protected: + EndToEndTest() + : EncoderTest(GET_PARAM(0)), test_video_param_(GET_PARAM(2)), + cpu_used_(GET_PARAM(3)), psnr_(0.0), nframes_(0), + encoding_mode_(GET_PARAM(1)) {} + + virtual ~EndToEndTest() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + if (encoding_mode_ != ::libaom_test::kRealTime) { + cfg_.g_lag_in_frames = 5; + cfg_.rc_end_usage = AOM_VBR; + } else { + cfg_.g_lag_in_frames = 0; + cfg_.rc_end_usage = AOM_CBR; + cfg_.rc_buf_sz = 1000; + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 600; + } + dec_cfg_.threads = 4; + } + + virtual void BeginPassHook(unsigned int) { + psnr_ = 0.0; + nframes_ = 0; + } + + virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) { + psnr_ += pkt->data.psnr.psnr[0]; + nframes_++; + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 1) { + encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1); + encoder->Control(AV1E_SET_TILE_COLUMNS, 4); + encoder->Control(AOME_SET_CPUUSED, cpu_used_); +#if CONFIG_PALETTE + // Test screen coding tools at cpu_used = 1 && encoding mode is two-pass. + if (cpu_used_ == 1 && encoding_mode_ == ::libaom_test::kTwoPassGood) + encoder->Control(AV1E_SET_TUNE_CONTENT, AOM_CONTENT_SCREEN); + else + encoder->Control(AV1E_SET_TUNE_CONTENT, AOM_CONTENT_DEFAULT); +#endif // CONFIG_PALETTE + if (encoding_mode_ != ::libaom_test::kRealTime) { + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); + encoder->Control(AOME_SET_ARNR_MAXFRAMES, 7); + encoder->Control(AOME_SET_ARNR_STRENGTH, 5); + } + } + } + + double GetAveragePsnr() const { + if (nframes_) return psnr_ / nframes_; + return 0.0; + } + + double GetPsnrThreshold() { + return kPsnrThreshold[cpu_used_][encoding_mode_]; + } + + TestVideoParam test_video_param_; + int cpu_used_; + + private: + double psnr_; + unsigned int nframes_; + libaom_test::TestMode encoding_mode_; +}; + +class EndToEndTestLarge : public EndToEndTest {}; + +TEST_P(EndToEndTestLarge, EndtoEndPSNRTest) { + cfg_.rc_target_bitrate = kBitrate; + cfg_.g_error_resilient = 0; + cfg_.g_profile = test_video_param_.profile; + cfg_.g_input_bit_depth = test_video_param_.input_bit_depth; + cfg_.g_bit_depth = test_video_param_.bit_depth; + init_flags_ = AOM_CODEC_USE_PSNR; + if (cfg_.g_bit_depth > 8) init_flags_ |= AOM_CODEC_USE_HIGHBITDEPTH; + + testing::internal::scoped_ptr video; + if (is_extension_y4m(test_video_param_.filename)) { + video.reset(new libaom_test::Y4mVideoSource(test_video_param_.filename, 0, + kFrames)); + } else { + video.reset(new libaom_test::YUVVideoSource( + test_video_param_.filename, test_video_param_.fmt, kWidth, kHeight, + kFramerate, 1, 0, kFrames)); + } + ASSERT_TRUE(video.get() != NULL); + + ASSERT_NO_FATAL_FAILURE(RunLoop(video.get())); + const double psnr = GetAveragePsnr(); + EXPECT_GT(psnr, GetPsnrThreshold()); +} + +TEST_P(EndToEndTest, EndtoEndPSNRTest) { + cfg_.rc_target_bitrate = kBitrate; + cfg_.g_error_resilient = 0; + cfg_.g_profile = test_video_param_.profile; + cfg_.g_input_bit_depth = test_video_param_.input_bit_depth; + cfg_.g_bit_depth = test_video_param_.bit_depth; + init_flags_ = AOM_CODEC_USE_PSNR; + if (cfg_.g_bit_depth > 8) init_flags_ |= AOM_CODEC_USE_HIGHBITDEPTH; + + testing::internal::scoped_ptr video; + if (is_extension_y4m(test_video_param_.filename)) { + video.reset(new libaom_test::Y4mVideoSource(test_video_param_.filename, 0, + kFrames)); + } else { + video.reset(new libaom_test::YUVVideoSource( + test_video_param_.filename, test_video_param_.fmt, kWidth, kHeight, + kFramerate, 1, 0, kFrames)); + } + ASSERT_TRUE(video.get() != NULL); + + ASSERT_NO_FATAL_FAILURE(RunLoop(video.get())); + const double psnr = GetAveragePsnr(); + EXPECT_GT(psnr, GetPsnrThreshold()); +} + +AV1_INSTANTIATE_TEST_CASE(EndToEndTestLarge, + ::testing::ValuesIn(kEncodingModeVectors), + ::testing::ValuesIn(kTestVectors), + ::testing::ValuesIn(kCpuUsedVectors)); + +AV1_INSTANTIATE_TEST_CASE(EndToEndTest, + ::testing::Values(kEncodingModeVectors[0]), + ::testing::Values(kTestVectors[2]), // 444 + ::testing::Values(kCpuUsedVectors[2])); +} // namespace diff --git a/third_party/aom/test/error_block_test.cc b/third_party/aom/test/error_block_test.cc new file mode 100644 index 0000000000..227065fa90 --- /dev/null +++ b/third_party/aom/test/error_block_test.cc @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "av1/common/entropy.h" +#include "aom/aom_codec.h" +#include "aom/aom_integer.h" + +using libaom_test::ACMRandom; + +namespace { +#if CONFIG_HIGHBITDEPTH +const int kNumIterations = 1000; + +typedef int64_t (*ErrorBlockFunc)(const tran_low_t *coeff, + const tran_low_t *dqcoeff, + intptr_t block_size, int64_t *ssz, int bps); + +typedef std::tr1::tuple + ErrorBlockParam; + +class ErrorBlockTest : public ::testing::TestWithParam { + public: + virtual ~ErrorBlockTest() {} + virtual void SetUp() { + error_block_op_ = GET_PARAM(0); + ref_error_block_op_ = GET_PARAM(1); + bit_depth_ = GET_PARAM(2); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + aom_bit_depth_t bit_depth_; + ErrorBlockFunc error_block_op_; + ErrorBlockFunc ref_error_block_op_; +}; + +TEST_P(ErrorBlockTest, OperationCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, tran_low_t, coeff[4096]); + DECLARE_ALIGNED(16, tran_low_t, dqcoeff[4096]); + int err_count_total = 0; + int first_failure = -1; + intptr_t block_size; + int64_t ssz; + int64_t ret; + int64_t ref_ssz; + int64_t ref_ret; + const int msb = bit_depth_ + 8 - 1; + for (int i = 0; i < kNumIterations; ++i) { + int err_count = 0; + block_size = 16 << (i % 9); // All block sizes from 4x4, 8x4 ..64x64 + for (int j = 0; j < block_size; j++) { + // coeff and dqcoeff will always have at least the same sign, and this + // can be used for optimization, so generate test input precisely. + if (rnd(2)) { + // Positive number + coeff[j] = rnd(1 << msb); + dqcoeff[j] = rnd(1 << msb); + } else { + // Negative number + coeff[j] = -rnd(1 << msb); + dqcoeff[j] = -rnd(1 << msb); + } + } + ref_ret = + ref_error_block_op_(coeff, dqcoeff, block_size, &ref_ssz, bit_depth_); + ASM_REGISTER_STATE_CHECK( + ret = error_block_op_(coeff, dqcoeff, block_size, &ssz, bit_depth_)); + err_count += (ref_ret != ret) | (ref_ssz != ssz); + if (err_count && !err_count_total) { + first_failure = i; + } + err_count_total += err_count; + } + EXPECT_EQ(0, err_count_total) + << "Error: Error Block Test, C output doesn't match optimized output. " + << "First failed at test case " << first_failure; +} + +TEST_P(ErrorBlockTest, ExtremeValues) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, tran_low_t, coeff[4096]); + DECLARE_ALIGNED(16, tran_low_t, dqcoeff[4096]); + int err_count_total = 0; + int first_failure = -1; + intptr_t block_size; + int64_t ssz; + int64_t ret; + int64_t ref_ssz; + int64_t ref_ret; + const int msb = bit_depth_ + 8 - 1; + int max_val = ((1 << msb) - 1); + for (int i = 0; i < kNumIterations; ++i) { + int err_count = 0; + int k = (i / 9) % 9; + + // Change the maximum coeff value, to test different bit boundaries + if (k == 8 && (i % 9) == 0) { + max_val >>= 1; + } + block_size = 16 << (i % 9); // All block sizes from 4x4, 8x4 ..64x64 + for (int j = 0; j < block_size; j++) { + if (k < 4) { + // Test at positive maximum values + coeff[j] = k % 2 ? max_val : 0; + dqcoeff[j] = (k >> 1) % 2 ? max_val : 0; + } else if (k < 8) { + // Test at negative maximum values + coeff[j] = k % 2 ? -max_val : 0; + dqcoeff[j] = (k >> 1) % 2 ? -max_val : 0; + } else { + if (rnd(2)) { + // Positive number + coeff[j] = rnd(1 << 14); + dqcoeff[j] = rnd(1 << 14); + } else { + // Negative number + coeff[j] = -rnd(1 << 14); + dqcoeff[j] = -rnd(1 << 14); + } + } + } + ref_ret = + ref_error_block_op_(coeff, dqcoeff, block_size, &ref_ssz, bit_depth_); + ASM_REGISTER_STATE_CHECK( + ret = error_block_op_(coeff, dqcoeff, block_size, &ssz, bit_depth_)); + err_count += (ref_ret != ret) | (ref_ssz != ssz); + if (err_count && !err_count_total) { + first_failure = i; + } + err_count_total += err_count; + } + EXPECT_EQ(0, err_count_total) + << "Error: Error Block Test, C output doesn't match optimized output. " + << "First failed at test case " << first_failure; +} + +#if HAVE_SSE2 || HAVE_AVX +using std::tr1::make_tuple; + +INSTANTIATE_TEST_CASE_P( + SSE2, ErrorBlockTest, + ::testing::Values(make_tuple(&av1_highbd_block_error_sse2, + &av1_highbd_block_error_c, AOM_BITS_10), + make_tuple(&av1_highbd_block_error_sse2, + &av1_highbd_block_error_c, AOM_BITS_12), + make_tuple(&av1_highbd_block_error_sse2, + &av1_highbd_block_error_c, AOM_BITS_8))); +#endif // HAVE_SSE2 + +#endif // CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/error_resilience_test.cc b/third_party/aom/test/error_resilience_test.cc new file mode 100644 index 0000000000..63f10012f8 --- /dev/null +++ b/third_party/aom/test/error_resilience_test.cc @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" + +namespace { + +const int kMaxErrorFrames = 12; +const int kMaxDroppableFrames = 12; + +class ErrorResilienceTestLarge + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWithParam { + protected: + ErrorResilienceTestLarge() + : EncoderTest(GET_PARAM(0)), psnr_(0.0), nframes_(0), mismatch_psnr_(0.0), + mismatch_nframes_(0), encoding_mode_(GET_PARAM(1)) { + Reset(); + } + + virtual ~ErrorResilienceTestLarge() {} + + void Reset() { + error_nframes_ = 0; + droppable_nframes_ = 0; + pattern_switch_ = 0; + } + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + } + + virtual void BeginPassHook(unsigned int /*pass*/) { + psnr_ = 0.0; + nframes_ = 0; + mismatch_psnr_ = 0.0; + mismatch_nframes_ = 0; + } + + virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) { + psnr_ += pkt->data.psnr.psnr[0]; + nframes_++; + } + + virtual void PreEncodeFrameHook(libaom_test::VideoSource *video) { + frame_flags_ &= + ~(AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | AOM_EFLAG_NO_UPD_ARF); + if (droppable_nframes_ > 0 && + (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { + for (unsigned int i = 0; i < droppable_nframes_; ++i) { + if (droppable_frames_[i] == video->frame()) { + std::cout << "Encoding droppable frame: " << droppable_frames_[i] + << "\n"; + frame_flags_ |= (AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | + AOM_EFLAG_NO_UPD_ARF); + return; + } + } + } + } + + double GetAveragePsnr() const { + if (nframes_) return psnr_ / nframes_; + return 0.0; + } + + double GetAverageMismatchPsnr() const { + if (mismatch_nframes_) return mismatch_psnr_ / mismatch_nframes_; + return 0.0; + } + + virtual bool DoDecode() const { + if (error_nframes_ > 0 && + (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { + for (unsigned int i = 0; i < error_nframes_; ++i) { + if (error_frames_[i] == nframes_ - 1) { + std::cout << " Skipping decoding frame: " + << error_frames_[i] << "\n"; + return 0; + } + } + } + return 1; + } + + virtual void MismatchHook(const aom_image_t *img1, const aom_image_t *img2) { + double mismatch_psnr = compute_psnr(img1, img2); + mismatch_psnr_ += mismatch_psnr; + ++mismatch_nframes_; + // std::cout << "Mismatch frame psnr: " << mismatch_psnr << "\n"; + ::libaom_test::EncoderTest::MismatchHook(img1, img2); + } + + void SetErrorFrames(int num, unsigned int *list) { + if (num > kMaxErrorFrames) + num = kMaxErrorFrames; + else if (num < 0) + num = 0; + error_nframes_ = num; + for (unsigned int i = 0; i < error_nframes_; ++i) + error_frames_[i] = list[i]; + } + + void SetDroppableFrames(int num, unsigned int *list) { + if (num > kMaxDroppableFrames) + num = kMaxDroppableFrames; + else if (num < 0) + num = 0; + droppable_nframes_ = num; + for (unsigned int i = 0; i < droppable_nframes_; ++i) + droppable_frames_[i] = list[i]; + } + + unsigned int GetMismatchFrames() { return mismatch_nframes_; } + + void SetPatternSwitch(int frame_switch) { pattern_switch_ = frame_switch; } + + private: + double psnr_; + unsigned int nframes_; + unsigned int error_nframes_; + unsigned int droppable_nframes_; + unsigned int pattern_switch_; + double mismatch_psnr_; + unsigned int mismatch_nframes_; + unsigned int error_frames_[kMaxErrorFrames]; + unsigned int droppable_frames_[kMaxDroppableFrames]; + libaom_test::TestMode encoding_mode_; +}; + +TEST_P(ErrorResilienceTestLarge, OnVersusOff) { + const aom_rational timebase = { 33333333, 1000000000 }; + cfg_.g_timebase = timebase; + cfg_.rc_target_bitrate = 2000; + cfg_.g_lag_in_frames = 10; + + init_flags_ = AOM_CODEC_USE_PSNR; + + libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + timebase.den, timebase.num, 0, 30); + + // Error resilient mode OFF. + cfg_.g_error_resilient = 0; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + const double psnr_resilience_off = GetAveragePsnr(); + EXPECT_GT(psnr_resilience_off, 25.0); + + // Error resilient mode ON. + cfg_.g_error_resilient = 1; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + const double psnr_resilience_on = GetAveragePsnr(); + EXPECT_GT(psnr_resilience_on, 25.0); + + // Test that turning on error resilient mode hurts by 10% at most. + if (psnr_resilience_off > 0.0) { + const double psnr_ratio = psnr_resilience_on / psnr_resilience_off; + EXPECT_GE(psnr_ratio, 0.9); + EXPECT_LE(psnr_ratio, 1.1); + } +} + +// Check for successful decoding and no encoder/decoder mismatch +// if we lose (i.e., drop before decoding) a set of droppable +// frames (i.e., frames that don't update any reference buffers). +// Check both isolated and consecutive loss. +TEST_P(ErrorResilienceTestLarge, DropFramesWithoutRecovery) { + const aom_rational timebase = { 33333333, 1000000000 }; + cfg_.g_timebase = timebase; + cfg_.rc_target_bitrate = 500; + // FIXME(debargha): Fix this to work for any lag. + // Currently this test only works for lag = 0 + cfg_.g_lag_in_frames = 0; + + init_flags_ = AOM_CODEC_USE_PSNR; + + libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + timebase.den, timebase.num, 0, 40); + + // Error resilient mode ON. + cfg_.g_error_resilient = 1; + cfg_.kf_mode = AOM_KF_DISABLED; + + // Set an arbitrary set of error frames same as droppable frames. + // In addition to isolated loss/drop, add a long consecutive series + // (of size 9) of dropped frames. + unsigned int num_droppable_frames = 11; + unsigned int droppable_frame_list[] = { 5, 16, 22, 23, 24, 25, + 26, 27, 28, 29, 30 }; + SetDroppableFrames(num_droppable_frames, droppable_frame_list); + SetErrorFrames(num_droppable_frames, droppable_frame_list); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + // Test that no mismatches have been found + std::cout << " Mismatch frames: " << GetMismatchFrames() << "\n"; + EXPECT_EQ(GetMismatchFrames(), (unsigned int)0); + + // Reset previously set of error/droppable frames. + Reset(); + +#if 0 + // TODO(jkoleszar): This test is disabled for the time being as too + // sensitive. It's not clear how to set a reasonable threshold for + // this behavior. + + // Now set an arbitrary set of error frames that are non-droppable + unsigned int num_error_frames = 3; + unsigned int error_frame_list[] = {3, 10, 20}; + SetErrorFrames(num_error_frames, error_frame_list); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + + // Test that dropping an arbitrary set of inter frames does not hurt too much + // Note the Average Mismatch PSNR is the average of the PSNR between + // decoded frame and encoder's version of the same frame for all frames + // with mismatch. + const double psnr_resilience_mismatch = GetAverageMismatchPsnr(); + std::cout << " Mismatch PSNR: " + << psnr_resilience_mismatch << "\n"; + EXPECT_GT(psnr_resilience_mismatch, 20.0); +#endif +} + +AV1_INSTANTIATE_TEST_CASE(ErrorResilienceTestLarge, ONE_PASS_TEST_MODES); +} // namespace diff --git a/third_party/aom/test/ethread_test.cc b/third_party/aom/test/ethread_test.cc new file mode 100644 index 0000000000..5b519f8feb --- /dev/null +++ b/third_party/aom/test/ethread_test.cc @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/md5_helper.h" +#include "test/util.h" +#include "test/y4m_video_source.h" + +namespace { +class AVxEncoderThreadTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params { + protected: + AVxEncoderThreadTest() + : EncoderTest(GET_PARAM(0)), encoder_initialized_(false), + encoding_mode_(GET_PARAM(1)), set_cpu_used_(GET_PARAM(2)) { + init_flags_ = AOM_CODEC_USE_PSNR; + aom_codec_dec_cfg_t cfg = aom_codec_dec_cfg_t(); + cfg.w = 1280; + cfg.h = 720; + decoder_ = codec_->CreateDecoder(cfg, 0); +#if CONFIG_AV1 && CONFIG_EXT_TILE + if (decoder_->IsAV1()) { + decoder_->Control(AV1_SET_DECODE_TILE_ROW, -1); + decoder_->Control(AV1_SET_DECODE_TILE_COL, -1); + } +#endif + + size_enc_.clear(); + md5_dec_.clear(); + md5_enc_.clear(); + } + virtual ~AVxEncoderThreadTest() { delete decoder_; } + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + + if (encoding_mode_ != ::libaom_test::kRealTime) { + cfg_.g_lag_in_frames = 3; + cfg_.rc_end_usage = AOM_VBR; + cfg_.rc_2pass_vbr_minsection_pct = 5; + cfg_.rc_2pass_vbr_maxsection_pct = 2000; + } else { + cfg_.g_lag_in_frames = 0; + cfg_.rc_end_usage = AOM_CBR; + cfg_.g_error_resilient = 1; + } + cfg_.rc_max_quantizer = 56; + cfg_.rc_min_quantizer = 0; + } + + virtual void BeginPassHook(unsigned int /*pass*/) { + encoder_initialized_ = false; + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource * /*video*/, + ::libaom_test::Encoder *encoder) { + if (!encoder_initialized_) { +#if CONFIG_AV1 && CONFIG_EXT_TILE + encoder->Control(AV1E_SET_TILE_COLUMNS, 1); + if (codec_ == &libaom_test::kAV1) { + // TODO(geza): Start using multiple tile rows when the multi-threaded + // encoder can handle them + encoder->Control(AV1E_SET_TILE_ROWS, 32); + } else { + encoder->Control(AV1E_SET_TILE_ROWS, 0); + } +#else + // Encode 4 tile columns. + encoder->Control(AV1E_SET_TILE_COLUMNS, 2); + encoder->Control(AV1E_SET_TILE_ROWS, 0); +#endif // CONFIG_AV1 && CONFIG_EXT_TILE +#if CONFIG_LOOPFILTERING_ACROSS_TILES + encoder->Control(AV1E_SET_TILE_LOOPFILTER, 0); +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + encoder->Control(AOME_SET_CPUUSED, set_cpu_used_); + if (encoding_mode_ != ::libaom_test::kRealTime) { + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); + encoder->Control(AOME_SET_ARNR_MAXFRAMES, 7); + encoder->Control(AOME_SET_ARNR_STRENGTH, 5); + encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 0); + } else { + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 0); + encoder->Control(AV1E_SET_AQ_MODE, 3); + } + encoder_initialized_ = true; + } + } + + virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) { + size_enc_.push_back(pkt->data.frame.sz); + + ::libaom_test::MD5 md5_enc; + md5_enc.Add(reinterpret_cast(pkt->data.frame.buf), + pkt->data.frame.sz); + md5_enc_.push_back(md5_enc.Get()); + + const aom_codec_err_t res = decoder_->DecodeFrame( + reinterpret_cast(pkt->data.frame.buf), pkt->data.frame.sz); + if (res != AOM_CODEC_OK) { + abort_ = true; + ASSERT_EQ(AOM_CODEC_OK, res); + } + const aom_image_t *img = decoder_->GetDxData().Next(); + + if (img) { + ::libaom_test::MD5 md5_res; + md5_res.Add(img); + md5_dec_.push_back(md5_res.Get()); + } + } + + void DoTest() { + ::libaom_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 15, 18); + cfg_.rc_target_bitrate = 1000; + + // Encode using single thread. + cfg_.g_threads = 1; + init_flags_ = AOM_CODEC_USE_PSNR; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + std::vector single_thr_size_enc; + std::vector single_thr_md5_enc; + std::vector single_thr_md5_dec; + single_thr_size_enc = size_enc_; + single_thr_md5_enc = md5_enc_; + single_thr_md5_dec = md5_dec_; + size_enc_.clear(); + md5_enc_.clear(); + md5_dec_.clear(); + + // Encode using multiple threads. + cfg_.g_threads = 4; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + std::vector multi_thr_size_enc; + std::vector multi_thr_md5_enc; + std::vector multi_thr_md5_dec; + multi_thr_size_enc = size_enc_; + multi_thr_md5_enc = md5_enc_; + multi_thr_md5_dec = md5_dec_; + size_enc_.clear(); + md5_enc_.clear(); + md5_dec_.clear(); + + // Check that the vectors are equal. + ASSERT_EQ(single_thr_size_enc, multi_thr_size_enc); + ASSERT_EQ(single_thr_md5_enc, multi_thr_md5_enc); + ASSERT_EQ(single_thr_md5_dec, multi_thr_md5_dec); + } + + bool encoder_initialized_; + ::libaom_test::TestMode encoding_mode_; + int set_cpu_used_; + ::libaom_test::Decoder *decoder_; + std::vector size_enc_; + std::vector md5_enc_; + std::vector md5_dec_; +}; + +TEST_P(AVxEncoderThreadTest, EncoderResultTest) { DoTest(); } + +class AVxEncoderThreadTestLarge : public AVxEncoderThreadTest {}; + +TEST_P(AVxEncoderThreadTestLarge, EncoderResultTest) { DoTest(); } + +// For AV1, only test speed 0 to 3. +AV1_INSTANTIATE_TEST_CASE(AVxEncoderThreadTest, + ::testing::Values(::libaom_test::kTwoPassGood, + ::libaom_test::kOnePassGood), + ::testing::Range(2, 4)); + +AV1_INSTANTIATE_TEST_CASE(AVxEncoderThreadTestLarge, + ::testing::Values(::libaom_test::kTwoPassGood, + ::libaom_test::kOnePassGood), + ::testing::Range(0, 2)); +} // namespace diff --git a/third_party/aom/test/examples.sh b/third_party/aom/test/examples.sh new file mode 100755 index 0000000000..d3152be7db --- /dev/null +++ b/third_party/aom/test/examples.sh @@ -0,0 +1,29 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This file runs all of the tests for the libaom examples. +## +. $(dirname $0)/tools_common.sh + +example_tests=$(ls $(dirname $0)/*.sh) + +# List of script names to exclude. +exclude_list="examples tools_common decode_to_md5" + +# Filter out the scripts in $exclude_list. +for word in ${exclude_list}; do + example_tests=$(filter_strings "${example_tests}" "${word}" exclude) +done + +for test in ${example_tests}; do + # Source each test script so that exporting variables can be avoided. + AOM_TEST_NAME="$(basename ${test%.*})" + . "${test}" +done diff --git a/third_party/aom/test/fdct4x4_test.cc b/third_party/aom/test/fdct4x4_test.cc new file mode 100644 index 0000000000..ed265e84f2 --- /dev/null +++ b/third_party/aom/test/fdct4x4_test.cc @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/transform_test_base.h" +#include "test/util.h" +#include "av1/common/entropy.h" +#include "aom/aom_codec.h" +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*FdctFunc)(const int16_t *in, tran_low_t *out, int stride); +typedef void (*IdctFunc)(const tran_low_t *in, uint8_t *out, int stride); +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); +using libaom_test::FhtFunc; + +typedef std::tr1::tuple + Dct4x4Param; +typedef std::tr1::tuple Ht4x4Param; + +void fdct4x4_ref(const int16_t *in, tran_low_t *out, int stride, + int /*tx_type*/) { + aom_fdct4x4_c(in, out, stride); +} + +void fht4x4_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht4x4_c(in, out, stride, tx_type); +} + +void fwht4x4_ref(const int16_t *in, tran_low_t *out, int stride, + int /*tx_type*/) { + av1_fwht4x4_c(in, out, stride); +} + +#if CONFIG_HIGHBITDEPTH +void idct4x4_10(const tran_low_t *in, uint8_t *out, int stride) { + aom_highbd_idct4x4_16_add_c(in, out, stride, 10); +} + +void idct4x4_12(const tran_low_t *in, uint8_t *out, int stride) { + aom_highbd_idct4x4_16_add_c(in, out, stride, 12); +} + +void iht4x4_10(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_highbd_iht4x4_16_add_c(in, out, stride, tx_type, 10); +} + +void iht4x4_12(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_highbd_iht4x4_16_add_c(in, out, stride, tx_type, 12); +} + +void iwht4x4_10(const tran_low_t *in, uint8_t *out, int stride) { + aom_highbd_iwht4x4_16_add_c(in, out, stride, 10); +} + +void iwht4x4_12(const tran_low_t *in, uint8_t *out, int stride) { + aom_highbd_iwht4x4_16_add_c(in, out, stride, 12); +} + +#if HAVE_SSE2 +void idct4x4_10_sse2(const tran_low_t *in, uint8_t *out, int stride) { + aom_highbd_idct4x4_16_add_sse2(in, out, stride, 10); +} + +void idct4x4_12_sse2(const tran_low_t *in, uint8_t *out, int stride) { + aom_highbd_idct4x4_16_add_sse2(in, out, stride, 12); +} +#endif // HAVE_SSE2 +#endif // CONFIG_HIGHBITDEPTH + +class Trans4x4DCT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~Trans4x4DCT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 4; + height_ = 4; + fwd_txfm_ref = fdct4x4_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride); + } + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride); + } + + FdctFunc fwd_txfm_; + IdctFunc inv_txfm_; +}; + +TEST_P(Trans4x4DCT, AccuracyCheck) { RunAccuracyCheck(0, 0.00001); } + +TEST_P(Trans4x4DCT, CoeffCheck) { RunCoeffCheck(); } + +TEST_P(Trans4x4DCT, MemCheck) { RunMemCheck(); } + +TEST_P(Trans4x4DCT, InvAccuracyCheck) { RunInvAccuracyCheck(1); } + +class Trans4x4HT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~Trans4x4HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 4; + height_ = 4; + fwd_txfm_ref = fht4x4_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(Trans4x4HT, AccuracyCheck) { RunAccuracyCheck(1, 0.005); } + +TEST_P(Trans4x4HT, CoeffCheck) { RunCoeffCheck(); } + +TEST_P(Trans4x4HT, MemCheck) { RunMemCheck(); } + +TEST_P(Trans4x4HT, InvAccuracyCheck) { RunInvAccuracyCheck(1); } + +class Trans4x4WHT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~Trans4x4WHT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 4; + height_ = 4; + fwd_txfm_ref = fwht4x4_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride); + } + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride); + } + + FdctFunc fwd_txfm_; + IdctFunc inv_txfm_; +}; + +TEST_P(Trans4x4WHT, AccuracyCheck) { RunAccuracyCheck(0, 0.00001); } + +TEST_P(Trans4x4WHT, CoeffCheck) { RunCoeffCheck(); } + +TEST_P(Trans4x4WHT, MemCheck) { RunMemCheck(); } + +TEST_P(Trans4x4WHT, InvAccuracyCheck) { RunInvAccuracyCheck(0); } +using std::tr1::make_tuple; + +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + C, Trans4x4DCT, + ::testing::Values( + make_tuple(&aom_highbd_fdct4x4_c, &idct4x4_10, 0, AOM_BITS_10, 16), + make_tuple(&aom_highbd_fdct4x4_c, &idct4x4_12, 0, AOM_BITS_12, 16), + make_tuple(&aom_fdct4x4_c, &aom_idct4x4_16_add_c, 0, AOM_BITS_8, 16))); +#else +INSTANTIATE_TEST_CASE_P(C, Trans4x4DCT, + ::testing::Values(make_tuple(&aom_fdct4x4_c, + &aom_idct4x4_16_add_c, 0, + AOM_BITS_8, 16))); +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + C, Trans4x4HT, + ::testing::Values( + make_tuple(&av1_highbd_fht4x4_c, &iht4x4_10, 0, AOM_BITS_10, 16), + make_tuple(&av1_highbd_fht4x4_c, &iht4x4_10, 1, AOM_BITS_10, 16), + make_tuple(&av1_highbd_fht4x4_c, &iht4x4_10, 2, AOM_BITS_10, 16), + make_tuple(&av1_highbd_fht4x4_c, &iht4x4_10, 3, AOM_BITS_10, 16), + make_tuple(&av1_highbd_fht4x4_c, &iht4x4_12, 0, AOM_BITS_12, 16), + make_tuple(&av1_highbd_fht4x4_c, &iht4x4_12, 1, AOM_BITS_12, 16), + make_tuple(&av1_highbd_fht4x4_c, &iht4x4_12, 2, AOM_BITS_12, 16), + make_tuple(&av1_highbd_fht4x4_c, &iht4x4_12, 3, AOM_BITS_12, 16), + make_tuple(&av1_fht4x4_c, &av1_iht4x4_16_add_c, 0, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_c, &av1_iht4x4_16_add_c, 1, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_c, &av1_iht4x4_16_add_c, 2, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_c, &av1_iht4x4_16_add_c, 3, AOM_BITS_8, 16))); +#else +INSTANTIATE_TEST_CASE_P( + C, Trans4x4HT, + ::testing::Values( + make_tuple(&av1_fht4x4_c, &av1_iht4x4_16_add_c, 0, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_c, &av1_iht4x4_16_add_c, 1, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_c, &av1_iht4x4_16_add_c, 2, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_c, &av1_iht4x4_16_add_c, 3, AOM_BITS_8, 16))); +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + C, Trans4x4WHT, + ::testing::Values( + make_tuple(&av1_highbd_fwht4x4_c, &iwht4x4_10, 0, AOM_BITS_10, 16), + make_tuple(&av1_highbd_fwht4x4_c, &iwht4x4_12, 0, AOM_BITS_12, 16), + make_tuple(&av1_fwht4x4_c, &aom_iwht4x4_16_add_c, 0, AOM_BITS_8, 16))); +#else +INSTANTIATE_TEST_CASE_P(C, Trans4x4WHT, + ::testing::Values(make_tuple(&av1_fwht4x4_c, + &aom_iwht4x4_16_add_c, 0, + AOM_BITS_8, 16))); +#endif // CONFIG_HIGHBITDEPTH + +#if HAVE_NEON_ASM && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(NEON, Trans4x4DCT, + ::testing::Values(make_tuple(&aom_fdct4x4_c, + &aom_idct4x4_16_add_neon, + 0, AOM_BITS_8, 16))); +#endif // HAVE_NEON_ASM && !CONFIG_HIGHBITDEPTH + +#if HAVE_NEON && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + NEON, Trans4x4HT, + ::testing::Values( + make_tuple(&av1_fht4x4_c, &av1_iht4x4_16_add_neon, 0, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_c, &av1_iht4x4_16_add_neon, 1, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_c, &av1_iht4x4_16_add_neon, 2, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_c, &av1_iht4x4_16_add_neon, 3, AOM_BITS_8, 16))); +#endif // HAVE_NEON && !CONFIG_HIGHBITDEPTH + +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P( + SSE2, Trans4x4WHT, + ::testing::Values(make_tuple(&av1_fwht4x4_c, &aom_iwht4x4_16_add_c, 0, + AOM_BITS_8, 16), + make_tuple(&av1_fwht4x4_c, &aom_iwht4x4_16_add_sse2, 0, + AOM_BITS_8, 16))); +#endif + +#if HAVE_SSE2 && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(SSE2, Trans4x4DCT, + ::testing::Values(make_tuple(&aom_fdct4x4_sse2, + &aom_idct4x4_16_add_sse2, + 0, AOM_BITS_8, 16))); +INSTANTIATE_TEST_CASE_P( + SSE2, Trans4x4HT, + ::testing::Values(make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 0, + AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 1, + AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 2, + AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_sse2, 3, + AOM_BITS_8, 16))); +#endif // HAVE_SSE2 && !CONFIG_HIGHBITDEPTH + +#if HAVE_SSE2 && CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSE2, Trans4x4DCT, + ::testing::Values( + make_tuple(&aom_highbd_fdct4x4_c, &idct4x4_10_sse2, 0, AOM_BITS_10, 16), + make_tuple(&aom_highbd_fdct4x4_sse2, &idct4x4_10_sse2, 0, AOM_BITS_10, + 16), + make_tuple(&aom_highbd_fdct4x4_c, &idct4x4_12_sse2, 0, AOM_BITS_12, 16), + make_tuple(&aom_highbd_fdct4x4_sse2, &idct4x4_12_sse2, 0, AOM_BITS_12, + 16), + make_tuple(&aom_fdct4x4_sse2, &aom_idct4x4_16_add_c, 0, AOM_BITS_8, + 16))); + +INSTANTIATE_TEST_CASE_P( + SSE2, Trans4x4HT, + ::testing::Values( + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_c, 0, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_c, 1, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_c, 2, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_sse2, &av1_iht4x4_16_add_c, 3, AOM_BITS_8, 16))); +#endif // HAVE_SSE2 && CONFIG_HIGHBITDEPTH + +#if HAVE_MSA && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(MSA, Trans4x4DCT, + ::testing::Values(make_tuple(&aom_fdct4x4_msa, + &aom_idct4x4_16_add_msa, 0, + AOM_BITS_8, 16))); +#if !CONFIG_EXT_TX +INSTANTIATE_TEST_CASE_P( + MSA, Trans4x4HT, + ::testing::Values( + make_tuple(&av1_fht4x4_msa, &av1_iht4x4_16_add_msa, 0, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_msa, &av1_iht4x4_16_add_msa, 1, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_msa, &av1_iht4x4_16_add_msa, 2, AOM_BITS_8, 16), + make_tuple(&av1_fht4x4_msa, &av1_iht4x4_16_add_msa, 3, AOM_BITS_8, + 16))); +#endif // !CONFIG_EXT_TX +#endif // HAVE_MSA && !CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/fdct8x8_test.cc b/third_party/aom/test/fdct8x8_test.cc new file mode 100644 index 0000000000..0e86c70aae --- /dev/null +++ b/third_party/aom/test/fdct8x8_test.cc @@ -0,0 +1,699 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "av1/common/entropy.h" +#include "av1/common/scan.h" +#include "aom/aom_codec.h" +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" + +using libaom_test::ACMRandom; + +namespace { + +const int kNumCoeffs = 64; +const double kPi = 3.141592653589793238462643383279502884; + +const int kSignBiasMaxDiff255 = 1500; +const int kSignBiasMaxDiff15 = 10000; + +typedef void (*FdctFunc)(const int16_t *in, tran_low_t *out, int stride); +typedef void (*IdctFunc)(const tran_low_t *in, uint8_t *out, int stride); +typedef void (*FhtFunc)(const int16_t *in, tran_low_t *out, int stride, + int tx_type); +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); + +typedef std::tr1::tuple Dct8x8Param; +typedef std::tr1::tuple Ht8x8Param; +typedef std::tr1::tuple Idct8x8Param; + +void reference_8x8_dct_1d(const double in[8], double out[8]) { + const double kInvSqrt2 = 0.707106781186547524400844362104; + for (int k = 0; k < 8; k++) { + out[k] = 0.0; + for (int n = 0; n < 8; n++) + out[k] += in[n] * cos(kPi * (2 * n + 1) * k / 16.0); + if (k == 0) out[k] = out[k] * kInvSqrt2; + } +} + +void reference_8x8_dct_2d(const int16_t input[kNumCoeffs], + double output[kNumCoeffs]) { + // First transform columns + for (int i = 0; i < 8; ++i) { + double temp_in[8], temp_out[8]; + for (int j = 0; j < 8; ++j) temp_in[j] = input[j * 8 + i]; + reference_8x8_dct_1d(temp_in, temp_out); + for (int j = 0; j < 8; ++j) output[j * 8 + i] = temp_out[j]; + } + // Then transform rows + for (int i = 0; i < 8; ++i) { + double temp_in[8], temp_out[8]; + for (int j = 0; j < 8; ++j) temp_in[j] = output[j + i * 8]; + reference_8x8_dct_1d(temp_in, temp_out); + // Scale by some magic number + for (int j = 0; j < 8; ++j) output[j + i * 8] = temp_out[j] * 2; + } +} + +void fdct8x8_ref(const int16_t *in, tran_low_t *out, int stride, + int /*tx_type*/) { + aom_fdct8x8_c(in, out, stride); +} + +void fht8x8_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht8x8_c(in, out, stride, tx_type); +} + +#if CONFIG_HIGHBITDEPTH +void iht8x8_10(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_highbd_iht8x8_64_add_c(in, out, stride, tx_type, 10); +} + +void iht8x8_12(const tran_low_t *in, uint8_t *out, int stride, int tx_type) { + av1_highbd_iht8x8_64_add_c(in, out, stride, tx_type, 12); +} + +#endif // CONFIG_HIGHBITDEPTH + +class FwdTrans8x8TestBase { + public: + virtual ~FwdTrans8x8TestBase() {} + + protected: + virtual void RunFwdTxfm(int16_t *in, tran_low_t *out, int stride) = 0; + virtual void RunInvTxfm(tran_low_t *out, uint8_t *dst, int stride) = 0; + + void RunSignBiasCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, int16_t, test_input_block[64]); + DECLARE_ALIGNED(16, tran_low_t, test_output_block[64]); + int count_sign_block[64][2]; + const int count_test_block = 100000; + + memset(count_sign_block, 0, sizeof(count_sign_block)); + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-255, 255]. + for (int j = 0; j < 64; ++j) + test_input_block[j] = ((rnd.Rand16() >> (16 - bit_depth_)) & mask_) - + ((rnd.Rand16() >> (16 - bit_depth_)) & mask_); + ASM_REGISTER_STATE_CHECK( + RunFwdTxfm(test_input_block, test_output_block, pitch_)); + + for (int j = 0; j < 64; ++j) { + if (test_output_block[j] < 0) + ++count_sign_block[j][0]; + else if (test_output_block[j] > 0) + ++count_sign_block[j][1]; + } + } + + for (int j = 0; j < 64; ++j) { + const int diff = abs(count_sign_block[j][0] - count_sign_block[j][1]); + const int max_diff = kSignBiasMaxDiff255; + EXPECT_LT(diff, max_diff << (bit_depth_ - 8)) + << "Error: 8x8 FDCT/FHT has a sign bias > " + << 1. * max_diff / count_test_block * 100 << "%" + << " for input range [-255, 255] at index " << j + << " count0: " << count_sign_block[j][0] + << " count1: " << count_sign_block[j][1] << " diff: " << diff; + } + + memset(count_sign_block, 0, sizeof(count_sign_block)); + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-mask_ / 16, mask_ / 16]. + for (int j = 0; j < 64; ++j) + test_input_block[j] = + ((rnd.Rand16() & mask_) >> 4) - ((rnd.Rand16() & mask_) >> 4); + ASM_REGISTER_STATE_CHECK( + RunFwdTxfm(test_input_block, test_output_block, pitch_)); + + for (int j = 0; j < 64; ++j) { + if (test_output_block[j] < 0) + ++count_sign_block[j][0]; + else if (test_output_block[j] > 0) + ++count_sign_block[j][1]; + } + } + + for (int j = 0; j < 64; ++j) { + const int diff = abs(count_sign_block[j][0] - count_sign_block[j][1]); + const int max_diff = kSignBiasMaxDiff15; + EXPECT_LT(diff, max_diff << (bit_depth_ - 8)) + << "Error: 8x8 FDCT/FHT has a sign bias > " + << 1. * max_diff / count_test_block * 100 << "%" + << " for input range [-15, 15] at index " << j + << " count0: " << count_sign_block[j][0] + << " count1: " << count_sign_block[j][1] << " diff: " << diff; + } + } + + void RunRoundTripErrorCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + int max_error = 0; + int total_error = 0; + const int count_test_block = 100000; + DECLARE_ALIGNED(16, int16_t, test_input_block[64]); + DECLARE_ALIGNED(16, tran_low_t, test_temp_block[64]); + DECLARE_ALIGNED(16, uint8_t, dst[64]); + DECLARE_ALIGNED(16, uint8_t, src[64]); +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, dst16[64]); + DECLARE_ALIGNED(16, uint16_t, src16[64]); +#endif + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-mask_, mask_]. + for (int j = 0; j < 64; ++j) { + if (bit_depth_ == AOM_BITS_8) { + src[j] = rnd.Rand8(); + dst[j] = rnd.Rand8(); + test_input_block[j] = src[j] - dst[j]; +#if CONFIG_HIGHBITDEPTH + } else { + src16[j] = rnd.Rand16() & mask_; + dst16[j] = rnd.Rand16() & mask_; + test_input_block[j] = src16[j] - dst16[j]; +#endif + } + } + + ASM_REGISTER_STATE_CHECK( + RunFwdTxfm(test_input_block, test_temp_block, pitch_)); + for (int j = 0; j < 64; ++j) { + if (test_temp_block[j] > 0) { + test_temp_block[j] += 2; + test_temp_block[j] /= 4; + test_temp_block[j] *= 4; + } else { + test_temp_block[j] -= 2; + test_temp_block[j] /= 4; + test_temp_block[j] *= 4; + } + } + if (bit_depth_ == AOM_BITS_8) { + ASM_REGISTER_STATE_CHECK(RunInvTxfm(test_temp_block, dst, pitch_)); +#if CONFIG_HIGHBITDEPTH + } else { + ASM_REGISTER_STATE_CHECK( + RunInvTxfm(test_temp_block, CONVERT_TO_BYTEPTR(dst16), pitch_)); +#endif + } + + for (int j = 0; j < 64; ++j) { +#if CONFIG_HIGHBITDEPTH + const int diff = + bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; +#else + const int diff = dst[j] - src[j]; +#endif + const int error = diff * diff; + if (max_error < error) max_error = error; + total_error += error; + } + } + + EXPECT_GE(1 << 2 * (bit_depth_ - 8), max_error) + << "Error: 8x8 FDCT/IDCT or FHT/IHT has an individual" + << " roundtrip error > 1"; + + EXPECT_GE((count_test_block << 2 * (bit_depth_ - 8)) / 5, total_error) + << "Error: 8x8 FDCT/IDCT or FHT/IHT has average roundtrip " + << "error > 1/5 per block"; + } + + void RunExtremalCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + int max_error = 0; + int total_error = 0; + int total_coeff_error = 0; + const int count_test_block = 100000; + DECLARE_ALIGNED(16, int16_t, test_input_block[64]); + DECLARE_ALIGNED(16, tran_low_t, test_temp_block[64]); + DECLARE_ALIGNED(16, tran_low_t, ref_temp_block[64]); + DECLARE_ALIGNED(16, uint8_t, dst[64]); + DECLARE_ALIGNED(16, uint8_t, src[64]); +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, dst16[64]); + DECLARE_ALIGNED(16, uint16_t, src16[64]); +#endif + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-mask_, mask_]. + for (int j = 0; j < 64; ++j) { + if (bit_depth_ == AOM_BITS_8) { + if (i == 0) { + src[j] = 255; + dst[j] = 0; + } else if (i == 1) { + src[j] = 0; + dst[j] = 255; + } else { + src[j] = rnd.Rand8() % 2 ? 255 : 0; + dst[j] = rnd.Rand8() % 2 ? 255 : 0; + } + test_input_block[j] = src[j] - dst[j]; +#if CONFIG_HIGHBITDEPTH + } else { + if (i == 0) { + src16[j] = mask_; + dst16[j] = 0; + } else if (i == 1) { + src16[j] = 0; + dst16[j] = mask_; + } else { + src16[j] = rnd.Rand8() % 2 ? mask_ : 0; + dst16[j] = rnd.Rand8() % 2 ? mask_ : 0; + } + test_input_block[j] = src16[j] - dst16[j]; +#endif + } + } + + ASM_REGISTER_STATE_CHECK( + RunFwdTxfm(test_input_block, test_temp_block, pitch_)); + ASM_REGISTER_STATE_CHECK( + fwd_txfm_ref(test_input_block, ref_temp_block, pitch_, tx_type_)); + if (bit_depth_ == AOM_BITS_8) { + ASM_REGISTER_STATE_CHECK(RunInvTxfm(test_temp_block, dst, pitch_)); +#if CONFIG_HIGHBITDEPTH + } else { + ASM_REGISTER_STATE_CHECK( + RunInvTxfm(test_temp_block, CONVERT_TO_BYTEPTR(dst16), pitch_)); +#endif + } + + for (int j = 0; j < 64; ++j) { +#if CONFIG_HIGHBITDEPTH + const int diff = + bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; +#else + const int diff = dst[j] - src[j]; +#endif + const int error = diff * diff; + if (max_error < error) max_error = error; + total_error += error; + + const int coeff_diff = test_temp_block[j] - ref_temp_block[j]; + total_coeff_error += abs(coeff_diff); + } + + EXPECT_GE(1 << 2 * (bit_depth_ - 8), max_error) + << "Error: Extremal 8x8 FDCT/IDCT or FHT/IHT has" + << "an individual roundtrip error > 1"; + + EXPECT_GE((count_test_block << 2 * (bit_depth_ - 8)) / 5, total_error) + << "Error: Extremal 8x8 FDCT/IDCT or FHT/IHT has average" + << " roundtrip error > 1/5 per block"; + + EXPECT_EQ(0, total_coeff_error) + << "Error: Extremal 8x8 FDCT/FHT has" + << "overflow issues in the intermediate steps > 1"; + } + } + + void RunInvAccuracyCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 1000; + DECLARE_ALIGNED(16, int16_t, in[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, coeff[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, dst[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, src[kNumCoeffs]); +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, src16[kNumCoeffs]); + DECLARE_ALIGNED(16, uint16_t, dst16[kNumCoeffs]); +#endif + + for (int i = 0; i < count_test_block; ++i) { + double out_r[kNumCoeffs]; + + // Initialize a test block with input range [-255, 255]. + for (int j = 0; j < kNumCoeffs; ++j) { + if (bit_depth_ == AOM_BITS_8) { + src[j] = rnd.Rand8() % 2 ? 255 : 0; + dst[j] = src[j] > 0 ? 0 : 255; + in[j] = src[j] - dst[j]; +#if CONFIG_HIGHBITDEPTH + } else { + src16[j] = rnd.Rand8() % 2 ? mask_ : 0; + dst16[j] = src16[j] > 0 ? 0 : mask_; + in[j] = src16[j] - dst16[j]; +#endif + } + } + + reference_8x8_dct_2d(in, out_r); + for (int j = 0; j < kNumCoeffs; ++j) + coeff[j] = static_cast(round(out_r[j])); + + if (bit_depth_ == AOM_BITS_8) { + ASM_REGISTER_STATE_CHECK(RunInvTxfm(coeff, dst, pitch_)); +#if CONFIG_HIGHBITDEPTH + } else { + ASM_REGISTER_STATE_CHECK( + RunInvTxfm(coeff, CONVERT_TO_BYTEPTR(dst16), pitch_)); +#endif + } + + for (int j = 0; j < kNumCoeffs; ++j) { +#if CONFIG_HIGHBITDEPTH + const int diff = + bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; +#else + const int diff = dst[j] - src[j]; +#endif + const uint32_t error = diff * diff; + EXPECT_GE(1u << 2 * (bit_depth_ - 8), error) + << "Error: 8x8 IDCT has error " << error << " at index " << j; + } + } + } + + void RunFwdAccuracyCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 1000; + DECLARE_ALIGNED(16, int16_t, in[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, coeff_r[kNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, coeff[kNumCoeffs]); + + for (int i = 0; i < count_test_block; ++i) { + double out_r[kNumCoeffs]; + + // Initialize a test block with input range [-mask_, mask_]. + for (int j = 0; j < kNumCoeffs; ++j) + in[j] = rnd.Rand8() % 2 == 0 ? mask_ : -mask_; + + RunFwdTxfm(in, coeff, pitch_); + reference_8x8_dct_2d(in, out_r); + for (int j = 0; j < kNumCoeffs; ++j) + coeff_r[j] = static_cast(round(out_r[j])); + + for (int j = 0; j < kNumCoeffs; ++j) { + const int32_t diff = coeff[j] - coeff_r[j]; + const uint32_t error = diff * diff; + EXPECT_GE(9u << 2 * (bit_depth_ - 8), error) + << "Error: 8x8 DCT has error " << error << " at index " << j; + } + } + } + + void CompareInvReference(IdctFunc ref_txfm, int thresh) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 10000; + const int eob = 12; + DECLARE_ALIGNED(16, tran_low_t, coeff[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, dst[kNumCoeffs]); + DECLARE_ALIGNED(16, uint8_t, ref[kNumCoeffs]); +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, dst16[kNumCoeffs]); + DECLARE_ALIGNED(16, uint16_t, ref16[kNumCoeffs]); +#endif + const int16_t *scan = av1_default_scan_orders[TX_8X8].scan; + + for (int i = 0; i < count_test_block; ++i) { + for (int j = 0; j < kNumCoeffs; ++j) { + if (j < eob) { + // Random values less than the threshold, either positive or negative + coeff[scan[j]] = rnd(thresh) * (1 - 2 * (i % 2)); + } else { + coeff[scan[j]] = 0; + } + if (bit_depth_ == AOM_BITS_8) { + dst[j] = 0; + ref[j] = 0; +#if CONFIG_HIGHBITDEPTH + } else { + dst16[j] = 0; + ref16[j] = 0; +#endif + } + } + if (bit_depth_ == AOM_BITS_8) { + ref_txfm(coeff, ref, pitch_); + ASM_REGISTER_STATE_CHECK(RunInvTxfm(coeff, dst, pitch_)); +#if CONFIG_HIGHBITDEPTH + } else { + ref_txfm(coeff, CONVERT_TO_BYTEPTR(ref16), pitch_); + ASM_REGISTER_STATE_CHECK( + RunInvTxfm(coeff, CONVERT_TO_BYTEPTR(dst16), pitch_)); +#endif + } + + for (int j = 0; j < kNumCoeffs; ++j) { +#if CONFIG_HIGHBITDEPTH + const int diff = + bit_depth_ == AOM_BITS_8 ? dst[j] - ref[j] : dst16[j] - ref16[j]; +#else + const int diff = dst[j] - ref[j]; +#endif + const uint32_t error = diff * diff; + EXPECT_EQ(0u, error) << "Error: 8x8 IDCT has error " << error + << " at index " << j; + } + } + } + int pitch_; + int tx_type_; + FhtFunc fwd_txfm_ref; + aom_bit_depth_t bit_depth_; + int mask_; +}; + +class FwdTrans8x8DCT : public FwdTrans8x8TestBase, + public ::testing::TestWithParam { + public: + virtual ~FwdTrans8x8DCT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 8; + fwd_txfm_ref = fdct8x8_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride); + } + void RunInvTxfm(tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride); + } + + FdctFunc fwd_txfm_; + IdctFunc inv_txfm_; +}; + +TEST_P(FwdTrans8x8DCT, SignBiasCheck) { RunSignBiasCheck(); } + +TEST_P(FwdTrans8x8DCT, RoundTripErrorCheck) { RunRoundTripErrorCheck(); } + +TEST_P(FwdTrans8x8DCT, ExtremalCheck) { RunExtremalCheck(); } + +TEST_P(FwdTrans8x8DCT, FwdAccuracyCheck) { RunFwdAccuracyCheck(); } + +TEST_P(FwdTrans8x8DCT, InvAccuracyCheck) { RunInvAccuracyCheck(); } + +class FwdTrans8x8HT : public FwdTrans8x8TestBase, + public ::testing::TestWithParam { + public: + virtual ~FwdTrans8x8HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 8; + fwd_txfm_ref = fht8x8_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + void RunInvTxfm(tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(FwdTrans8x8HT, SignBiasCheck) { RunSignBiasCheck(); } + +TEST_P(FwdTrans8x8HT, RoundTripErrorCheck) { RunRoundTripErrorCheck(); } + +TEST_P(FwdTrans8x8HT, ExtremalCheck) { RunExtremalCheck(); } + +class InvTrans8x8DCT : public FwdTrans8x8TestBase, + public ::testing::TestWithParam { + public: + virtual ~InvTrans8x8DCT() {} + + virtual void SetUp() { + ref_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + thresh_ = GET_PARAM(2); + pitch_ = 8; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunInvTxfm(tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride); + } + void RunFwdTxfm(int16_t * /*out*/, tran_low_t * /*dst*/, int /*stride*/) {} + + IdctFunc ref_txfm_; + IdctFunc inv_txfm_; + int thresh_; +}; + +TEST_P(InvTrans8x8DCT, CompareReference) { + CompareInvReference(ref_txfm_, thresh_); +} + +using std::tr1::make_tuple; + +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(C, FwdTrans8x8DCT, + ::testing::Values(make_tuple(&aom_fdct8x8_c, + &aom_idct8x8_64_add_c, 0, + AOM_BITS_8))); +#else +INSTANTIATE_TEST_CASE_P(C, FwdTrans8x8DCT, + ::testing::Values(make_tuple(&aom_fdct8x8_c, + &aom_idct8x8_64_add_c, 0, + AOM_BITS_8))); +#endif // CONFIG_HIGHBITDEPTH + +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + C, FwdTrans8x8HT, + ::testing::Values( + make_tuple(&av1_fht8x8_c, &av1_iht8x8_64_add_c, 0, AOM_BITS_8), + make_tuple(&av1_highbd_fht8x8_c, &iht8x8_10, 0, AOM_BITS_10), + make_tuple(&av1_highbd_fht8x8_c, &iht8x8_10, 1, AOM_BITS_10), + make_tuple(&av1_highbd_fht8x8_c, &iht8x8_10, 2, AOM_BITS_10), + make_tuple(&av1_highbd_fht8x8_c, &iht8x8_10, 3, AOM_BITS_10), + make_tuple(&av1_highbd_fht8x8_c, &iht8x8_12, 0, AOM_BITS_12), + make_tuple(&av1_highbd_fht8x8_c, &iht8x8_12, 1, AOM_BITS_12), + make_tuple(&av1_highbd_fht8x8_c, &iht8x8_12, 2, AOM_BITS_12), + make_tuple(&av1_highbd_fht8x8_c, &iht8x8_12, 3, AOM_BITS_12), + make_tuple(&av1_fht8x8_c, &av1_iht8x8_64_add_c, 1, AOM_BITS_8), + make_tuple(&av1_fht8x8_c, &av1_iht8x8_64_add_c, 2, AOM_BITS_8), + make_tuple(&av1_fht8x8_c, &av1_iht8x8_64_add_c, 3, AOM_BITS_8))); +#else +INSTANTIATE_TEST_CASE_P( + C, FwdTrans8x8HT, + ::testing::Values( + make_tuple(&av1_fht8x8_c, &av1_iht8x8_64_add_c, 0, AOM_BITS_8), + make_tuple(&av1_fht8x8_c, &av1_iht8x8_64_add_c, 1, AOM_BITS_8), + make_tuple(&av1_fht8x8_c, &av1_iht8x8_64_add_c, 2, AOM_BITS_8), + make_tuple(&av1_fht8x8_c, &av1_iht8x8_64_add_c, 3, AOM_BITS_8))); +#endif // CONFIG_HIGHBITDEPTH + +#if HAVE_NEON_ASM && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(NEON, FwdTrans8x8DCT, + ::testing::Values(make_tuple(&aom_fdct8x8_neon, + &aom_idct8x8_64_add_neon, + 0, AOM_BITS_8))); +#endif // HAVE_NEON_ASM && !CONFIG_HIGHBITDEPTH + +#if HAVE_NEON && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + NEON, FwdTrans8x8HT, + ::testing::Values( + make_tuple(&av1_fht8x8_c, &av1_iht8x8_64_add_neon, 0, AOM_BITS_8), + make_tuple(&av1_fht8x8_c, &av1_iht8x8_64_add_neon, 1, AOM_BITS_8), + make_tuple(&av1_fht8x8_c, &av1_iht8x8_64_add_neon, 2, AOM_BITS_8), + make_tuple(&av1_fht8x8_c, &av1_iht8x8_64_add_neon, 3, AOM_BITS_8))); +#endif // HAVE_NEON && !CONFIG_HIGHBITDEPTH + +#if HAVE_SSE2 && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(SSE2, FwdTrans8x8DCT, + ::testing::Values(make_tuple(&aom_fdct8x8_sse2, + &aom_idct8x8_64_add_sse2, + 0, AOM_BITS_8))); +INSTANTIATE_TEST_CASE_P( + SSE2, FwdTrans8x8HT, + ::testing::Values( + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 0, AOM_BITS_8), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 1, AOM_BITS_8), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 2, AOM_BITS_8), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_sse2, 3, AOM_BITS_8))); +#endif // HAVE_SSE2 && !CONFIG_HIGHBITDEPTH + +#if HAVE_SSE2 && CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(SSE2, FwdTrans8x8DCT, + ::testing::Values(make_tuple(&aom_fdct8x8_sse2, + &aom_idct8x8_64_add_c, 0, + AOM_BITS_8))); + +INSTANTIATE_TEST_CASE_P( + SSE2, FwdTrans8x8HT, + ::testing::Values( + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_c, 0, AOM_BITS_8), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_c, 1, AOM_BITS_8), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_c, 2, AOM_BITS_8), + make_tuple(&av1_fht8x8_sse2, &av1_iht8x8_64_add_c, 3, AOM_BITS_8))); + +#endif // HAVE_SSE2 && CONFIG_HIGHBITDEPTH + +#if HAVE_SSSE3 && ARCH_X86_64 +INSTANTIATE_TEST_CASE_P(SSSE3, FwdTrans8x8DCT, + ::testing::Values(make_tuple(&aom_fdct8x8_ssse3, + &aom_idct8x8_64_add_ssse3, + 0, AOM_BITS_8))); +#endif + +#if HAVE_MSA && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(MSA, FwdTrans8x8DCT, + ::testing::Values(make_tuple(&aom_fdct8x8_msa, + &aom_idct8x8_64_add_msa, 0, + AOM_BITS_8))); +#if !CONFIG_EXT_TX +INSTANTIATE_TEST_CASE_P( + MSA, FwdTrans8x8HT, + ::testing::Values( + make_tuple(&av1_fht8x8_msa, &av1_iht8x8_64_add_msa, 0, AOM_BITS_8), + make_tuple(&av1_fht8x8_msa, &av1_iht8x8_64_add_msa, 1, AOM_BITS_8), + make_tuple(&av1_fht8x8_msa, &av1_iht8x8_64_add_msa, 2, AOM_BITS_8), + make_tuple(&av1_fht8x8_msa, &av1_iht8x8_64_add_msa, 3, AOM_BITS_8))); +#endif // !CONFIG_EXT_TX +#endif // HAVE_MSA && !CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/fht32x32_test.cc b/third_party/aom/test/fht32x32_test.cc new file mode 100644 index 0000000000..56ac597c03 --- /dev/null +++ b/third_party/aom/test/fht32x32_test.cc @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" + +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/transform_test_base.h" +#include "test/util.h" +#include "aom_ports/mem.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); +using std::tr1::tuple; +using libaom_test::FhtFunc; +typedef tuple Ht32x32Param; + +void fht32x32_ref(const int16_t *in, tran_low_t *out, int stride, int tx_type) { + av1_fht32x32_c(in, out, stride, tx_type); +} + +#if CONFIG_HIGHBITDEPTH +typedef void (*IHbdHtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type, int bd); +typedef void (*HbdHtFunc)(const int16_t *input, int32_t *output, int stride, + int tx_type, int bd); + +// Target optimized function, tx_type, bit depth +typedef tuple HighbdHt32x32Param; + +void highbd_fht32x32_ref(const int16_t *in, int32_t *out, int stride, + int tx_type, int bd) { + av1_fwd_txfm2d_32x32_c(in, out, stride, tx_type, bd); +} +#endif // CONFIG_HIGHBITDEPTH + +#if HAVE_AVX2 +void dummy_inv_txfm(const tran_low_t *in, uint8_t *out, int stride, + int tx_type) { + (void)in; + (void)out; + (void)stride; + (void)tx_type; +} +#endif + +class AV1Trans32x32HT : public libaom_test::TransformTestBase, + public ::testing::TestWithParam { + public: + virtual ~AV1Trans32x32HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + inv_txfm_ = GET_PARAM(1); + tx_type_ = GET_PARAM(2); + pitch_ = 32; + height_ = 32; + fwd_txfm_ref = fht32x32_ref; + bit_depth_ = GET_PARAM(3); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = GET_PARAM(4); + } + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) { + fwd_txfm_(in, out, stride, tx_type_); + } + + void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) { + inv_txfm_(out, dst, stride, tx_type_); + } + + FhtFunc fwd_txfm_; + IhtFunc inv_txfm_; +}; + +TEST_P(AV1Trans32x32HT, CoeffCheck) { RunCoeffCheck(); } +TEST_P(AV1Trans32x32HT, MemCheck) { RunMemCheck(); } + +#if CONFIG_HIGHBITDEPTH +class AV1HighbdTrans32x32HT + : public ::testing::TestWithParam { + public: + virtual ~AV1HighbdTrans32x32HT() {} + + virtual void SetUp() { + fwd_txfm_ = GET_PARAM(0); + fwd_txfm_ref_ = highbd_fht32x32_ref; + tx_type_ = GET_PARAM(1); + bit_depth_ = GET_PARAM(2); + mask_ = (1 << bit_depth_) - 1; + num_coeffs_ = 1024; + + input_ = reinterpret_cast( + aom_memalign(32, sizeof(int16_t) * num_coeffs_)); + output_ = reinterpret_cast( + aom_memalign(32, sizeof(int32_t) * num_coeffs_)); + output_ref_ = reinterpret_cast( + aom_memalign(32, sizeof(int32_t) * num_coeffs_)); + } + + virtual void TearDown() { + aom_free(input_); + aom_free(output_); + aom_free(output_ref_); + libaom_test::ClearSystemState(); + } + + protected: + void RunBitexactCheck(); + + private: + HbdHtFunc fwd_txfm_; + HbdHtFunc fwd_txfm_ref_; + int tx_type_; + int bit_depth_; + int mask_; + int num_coeffs_; + int16_t *input_; + int32_t *output_; + int32_t *output_ref_; +}; + +void AV1HighbdTrans32x32HT::RunBitexactCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + int i, j; + const int stride = 32; + const int num_tests = 1000; + + for (i = 0; i < num_tests; ++i) { + for (j = 0; j < num_coeffs_; ++j) { + input_[j] = (rnd.Rand16() & mask_) - (rnd.Rand16() & mask_); + } + + fwd_txfm_ref_(input_, output_ref_, stride, tx_type_, bit_depth_); + ASM_REGISTER_STATE_CHECK( + fwd_txfm_(input_, output_, stride, tx_type_, bit_depth_)); + + for (j = 0; j < num_coeffs_; ++j) { + EXPECT_EQ(output_ref_[j], output_[j]) + << "Not bit-exact result at index: " << j << " at test block: " << i; + } + } +} + +TEST_P(AV1HighbdTrans32x32HT, HighbdCoeffCheck) { RunBitexactCheck(); } +#endif // CONFIG_HIGHBITDEPTH + +using std::tr1::make_tuple; + +#if HAVE_SSE2 +const Ht32x32Param kArrayHt32x32Param_sse2[] = { + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 0, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 1, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 2, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 3, AOM_BITS_8, 1024), +#if CONFIG_EXT_TX + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 4, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 5, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 6, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 7, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 8, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 9, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 10, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 11, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 12, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 13, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 14, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_sse2, &dummy_inv_txfm, 15, AOM_BITS_8, 1024) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(SSE2, AV1Trans32x32HT, + ::testing::ValuesIn(kArrayHt32x32Param_sse2)); +#endif // HAVE_SSE2 + +#if HAVE_AVX2 +const Ht32x32Param kArrayHt32x32Param_avx2[] = { + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 0, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 1, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 2, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 3, AOM_BITS_8, 1024), +#if CONFIG_EXT_TX + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 4, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 5, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 6, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 7, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 8, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 9, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 10, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 11, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 12, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 13, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 14, AOM_BITS_8, 1024), + make_tuple(&av1_fht32x32_avx2, &dummy_inv_txfm, 15, AOM_BITS_8, 1024) +#endif // CONFIG_EXT_TX +}; +INSTANTIATE_TEST_CASE_P(AVX2, AV1Trans32x32HT, + ::testing::ValuesIn(kArrayHt32x32Param_avx2)); +#endif // HAVE_AVX2 +} // namespace diff --git a/third_party/aom/test/filterintra_predictors_test.cc b/third_party/aom/test/filterintra_predictors_test.cc new file mode 100644 index 0000000000..5c6b56d144 --- /dev/null +++ b/third_party/aom/test/filterintra_predictors_test.cc @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "av1/common/enums.h" + +namespace { + +using std::tr1::tuple; +using libaom_test::ACMRandom; + +typedef void (*Predictor)(uint8_t *dst, ptrdiff_t stride, int bs, + const uint8_t *above, const uint8_t *left); + +// Note: +// Test parameter list: +// Reference predictor, optimized predictor, prediction mode, block size +// +typedef tuple PredFuncMode; +typedef tuple PredParams; + +#if CONFIG_HIGHBITDEPTH +typedef void (*HbdPredictor)(uint16_t *dst, ptrdiff_t stride, int bs, + const uint16_t *above, const uint16_t *left, + int bd); + +// Note: +// Test parameter list: +// Reference predictor, optimized predictor, prediction mode, block size, +// bit depth +// +typedef tuple HbdPredFuncMode; +typedef tuple HbdPredParams; +#endif + +const int MaxBlkSize = 32; + +// By default, disable speed test +#define PREDICTORS_SPEED_TEST (0) + +#if PREDICTORS_SPEED_TEST +const int MaxTestNum = 100000; +#else +const int MaxTestNum = 100; +#endif + +class AV1FilterIntraPredOptimzTest + : public ::testing::TestWithParam { + public: + virtual ~AV1FilterIntraPredOptimzTest() {} + virtual void SetUp() { + PredFuncMode funcMode = GET_PARAM(0); + predFuncRef_ = std::tr1::get<0>(funcMode); + predFunc_ = std::tr1::get<1>(funcMode); + mode_ = std::tr1::get<2>(funcMode); + blockSize_ = GET_PARAM(1); + + alloc_ = new uint8_t[3 * MaxBlkSize + 2]; + predRef_ = new uint8_t[MaxBlkSize * MaxBlkSize]; + pred_ = new uint8_t[MaxBlkSize * MaxBlkSize]; + } + + virtual void TearDown() { + delete[] alloc_; + delete[] predRef_; + delete[] pred_; + libaom_test::ClearSystemState(); + } + + protected: + void RunTest() const { + int tstIndex = 0; + int stride = blockSize_; + uint8_t *left = alloc_; + uint8_t *above = alloc_ + MaxBlkSize + 1; + while (tstIndex < MaxTestNum) { + PrepareBuffer(); + predFuncRef_(predRef_, stride, blockSize_, &above[1], left); + ASM_REGISTER_STATE_CHECK( + predFunc_(pred_, stride, blockSize_, &above[1], left)); + DiffPred(tstIndex); + tstIndex += 1; + } + } + + void RunSpeedTestC() const { + int tstIndex = 0; + int stride = blockSize_; + uint8_t *left = alloc_; + uint8_t *above = alloc_ + MaxBlkSize + 1; + PrepareBuffer(); + while (tstIndex < MaxTestNum) { + predFuncRef_(predRef_, stride, blockSize_, &above[1], left); + tstIndex += 1; + } + } + + void RunSpeedTestSSE() const { + int tstIndex = 0; + int stride = blockSize_; + uint8_t *left = alloc_; + uint8_t *above = alloc_ + MaxBlkSize + 1; + PrepareBuffer(); + while (tstIndex < MaxTestNum) { + predFunc_(predRef_, stride, blockSize_, &above[1], left); + tstIndex += 1; + } + } + + private: + void PrepareBuffer() const { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + int i = 0; + while (i < (3 * MaxBlkSize + 2)) { + alloc_[i] = rnd.Rand8(); + i += 1; + } + } + + void DiffPred(int testNum) const { + int i = 0; + while (i < blockSize_ * blockSize_) { + EXPECT_EQ(predRef_[i], pred_[i]) << "Error at position: " << i << " " + << "Block size: " << blockSize_ << " " + << "Test number: " << testNum; + i += 1; + } + } + + Predictor predFunc_; + Predictor predFuncRef_; + int mode_; + int blockSize_; + uint8_t *alloc_; + uint8_t *pred_; + uint8_t *predRef_; +}; + +#if CONFIG_HIGHBITDEPTH +class AV1HbdFilterIntraPredOptimzTest + : public ::testing::TestWithParam { + public: + virtual ~AV1HbdFilterIntraPredOptimzTest() {} + virtual void SetUp() { + HbdPredFuncMode funcMode = GET_PARAM(0); + predFuncRef_ = std::tr1::get<0>(funcMode); + predFunc_ = std::tr1::get<1>(funcMode); + mode_ = std::tr1::get<2>(funcMode); + blockSize_ = GET_PARAM(1); + bd_ = GET_PARAM(2); + + alloc_ = new uint16_t[3 * MaxBlkSize + 2]; + predRef_ = new uint16_t[MaxBlkSize * MaxBlkSize]; + pred_ = new uint16_t[MaxBlkSize * MaxBlkSize]; + } + + virtual void TearDown() { + delete[] alloc_; + delete[] predRef_; + delete[] pred_; + libaom_test::ClearSystemState(); + } + + protected: + void RunTest() const { + int tstIndex = 0; + int stride = blockSize_; + uint16_t *left = alloc_; + uint16_t *above = alloc_ + MaxBlkSize + 1; + while (tstIndex < MaxTestNum) { + PrepareBuffer(); + predFuncRef_(predRef_, stride, blockSize_, &above[1], left, bd_); + ASM_REGISTER_STATE_CHECK( + predFunc_(pred_, stride, blockSize_, &above[1], left, bd_)); + DiffPred(tstIndex); + tstIndex += 1; + } + } + + void RunSpeedTestC() const { + int tstIndex = 0; + int stride = blockSize_; + uint16_t *left = alloc_; + uint16_t *above = alloc_ + MaxBlkSize + 1; + PrepareBuffer(); + while (tstIndex < MaxTestNum) { + predFuncRef_(predRef_, stride, blockSize_, &above[1], left, bd_); + tstIndex += 1; + } + } + + void RunSpeedTestSSE() const { + int tstIndex = 0; + int stride = blockSize_; + uint16_t *left = alloc_; + uint16_t *above = alloc_ + MaxBlkSize + 1; + PrepareBuffer(); + while (tstIndex < MaxTestNum) { + predFunc_(predRef_, stride, blockSize_, &above[1], left, bd_); + tstIndex += 1; + } + } + + private: + void PrepareBuffer() const { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + int i = 0; + while (i < (3 * MaxBlkSize + 2)) { + alloc_[i] = rnd.Rand16() & ((1 << bd_) - 1); + i += 1; + } + } + + void DiffPred(int testNum) const { + int i = 0; + while (i < blockSize_ * blockSize_) { + EXPECT_EQ(predRef_[i], pred_[i]) << "Error at position: " << i << " " + << "Block size: " << blockSize_ << " " + << "Bit depth: " << bd_ << " " + << "Test number: " << testNum; + i += 1; + } + } + + HbdPredictor predFunc_; + HbdPredictor predFuncRef_; + int mode_; + int blockSize_; + int bd_; + uint16_t *alloc_; + uint16_t *pred_; + uint16_t *predRef_; +}; +#endif // CONFIG_HIGHBITDEPTH + +TEST_P(AV1FilterIntraPredOptimzTest, BitExactCheck) { RunTest(); } + +#if PREDICTORS_SPEED_TEST +TEST_P(AV1FilterIntraPredOptimzTest, SpeedCheckC) { RunSpeedTestC(); } + +TEST_P(AV1FilterIntraPredOptimzTest, SpeedCheckSSE) { RunSpeedTestSSE(); } +#endif + +#if CONFIG_HIGHBITDEPTH +TEST_P(AV1HbdFilterIntraPredOptimzTest, BitExactCheck) { RunTest(); } + +#if PREDICTORS_SPEED_TEST +TEST_P(AV1HbdFilterIntraPredOptimzTest, SpeedCheckC) { RunSpeedTestC(); } + +TEST_P(AV1HbdFilterIntraPredOptimzTest, SpeedCheckSSE) { RunSpeedTestSSE(); } +#endif // PREDICTORS_SPEED_TEST +#endif // CONFIG_HIGHBITDEPTH + +using std::tr1::make_tuple; + +const PredFuncMode kPredFuncMdArray[] = { + make_tuple(av1_dc_filter_predictor_c, av1_dc_filter_predictor_sse4_1, + DC_PRED), + make_tuple(av1_v_filter_predictor_c, av1_v_filter_predictor_sse4_1, V_PRED), + make_tuple(av1_h_filter_predictor_c, av1_h_filter_predictor_sse4_1, H_PRED), + make_tuple(av1_d45_filter_predictor_c, av1_d45_filter_predictor_sse4_1, + D45_PRED), + make_tuple(av1_d135_filter_predictor_c, av1_d135_filter_predictor_sse4_1, + D135_PRED), + make_tuple(av1_d117_filter_predictor_c, av1_d117_filter_predictor_sse4_1, + D117_PRED), + make_tuple(av1_d153_filter_predictor_c, av1_d153_filter_predictor_sse4_1, + D153_PRED), + make_tuple(av1_d207_filter_predictor_c, av1_d207_filter_predictor_sse4_1, + D207_PRED), + make_tuple(av1_d63_filter_predictor_c, av1_d63_filter_predictor_sse4_1, + D63_PRED), + make_tuple(av1_tm_filter_predictor_c, av1_tm_filter_predictor_sse4_1, + TM_PRED), +}; + +const int kBlkSize[] = { 4, 8, 16, 32 }; + +INSTANTIATE_TEST_CASE_P( + SSE4_1, AV1FilterIntraPredOptimzTest, + ::testing::Combine(::testing::ValuesIn(kPredFuncMdArray), + ::testing::ValuesIn(kBlkSize))); + +#if CONFIG_HIGHBITDEPTH +const HbdPredFuncMode kHbdPredFuncMdArray[] = { + make_tuple(av1_highbd_dc_filter_predictor_c, + av1_highbd_dc_filter_predictor_sse4_1, DC_PRED), + make_tuple(av1_highbd_v_filter_predictor_c, + av1_highbd_v_filter_predictor_sse4_1, V_PRED), + make_tuple(av1_highbd_h_filter_predictor_c, + av1_highbd_h_filter_predictor_sse4_1, H_PRED), + make_tuple(av1_highbd_d45_filter_predictor_c, + av1_highbd_d45_filter_predictor_sse4_1, D45_PRED), + make_tuple(av1_highbd_d135_filter_predictor_c, + av1_highbd_d135_filter_predictor_sse4_1, D135_PRED), + make_tuple(av1_highbd_d117_filter_predictor_c, + av1_highbd_d117_filter_predictor_sse4_1, D117_PRED), + make_tuple(av1_highbd_d153_filter_predictor_c, + av1_highbd_d153_filter_predictor_sse4_1, D153_PRED), + make_tuple(av1_highbd_d207_filter_predictor_c, + av1_highbd_d207_filter_predictor_sse4_1, D207_PRED), + make_tuple(av1_highbd_d63_filter_predictor_c, + av1_highbd_d63_filter_predictor_sse4_1, D63_PRED), + make_tuple(av1_highbd_tm_filter_predictor_c, + av1_highbd_tm_filter_predictor_sse4_1, TM_PRED), +}; + +const int kBd[] = { 10, 12 }; + +INSTANTIATE_TEST_CASE_P( + SSE4_1, AV1HbdFilterIntraPredOptimzTest, + ::testing::Combine(::testing::ValuesIn(kHbdPredFuncMdArray), + ::testing::ValuesIn(kBlkSize), + ::testing::ValuesIn(kBd))); +#endif // CONFIG_HIGHBITDEPTH + +} // namespace diff --git a/third_party/aom/test/frame_size_tests.cc b/third_party/aom/test/frame_size_tests.cc new file mode 100644 index 0000000000..73cc9c0755 --- /dev/null +++ b/third_party/aom/test/frame_size_tests.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/video_source.h" + +namespace { + +class AV1FrameSizeTests : public ::libaom_test::EncoderTest, + public ::testing::Test { + protected: + AV1FrameSizeTests() + : EncoderTest(&::libaom_test::kAV1), expected_res_(AOM_CODEC_OK) {} + virtual ~AV1FrameSizeTests() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(::libaom_test::kRealTime); + } + + virtual bool HandleDecodeResult(const aom_codec_err_t res_dec, + libaom_test::Decoder *decoder) { + EXPECT_EQ(expected_res_, res_dec) << decoder->DecodeError(); + return !::testing::Test::HasFailure(); + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 1) { + encoder->Control(AOME_SET_CPUUSED, 7); + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); + encoder->Control(AOME_SET_ARNR_MAXFRAMES, 7); + encoder->Control(AOME_SET_ARNR_STRENGTH, 5); + } + } + + int expected_res_; +}; + +TEST_F(AV1FrameSizeTests, TestInvalidSizes) { + ::libaom_test::RandomVideoSource video; + +#if CONFIG_SIZE_LIMIT + video.SetSize(DECODE_WIDTH_LIMIT + 16, DECODE_HEIGHT_LIMIT + 16); + video.set_limit(2); + expected_res_ = AOM_CODEC_CORRUPT_FRAME; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +#endif +} + +TEST_F(AV1FrameSizeTests, LargeValidSizes) { + ::libaom_test::RandomVideoSource video; + +#if CONFIG_SIZE_LIMIT + video.SetSize(DECODE_WIDTH_LIMIT, DECODE_HEIGHT_LIMIT); + video.set_limit(2); + expected_res_ = AOM_CODEC_OK; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +#else + // This test produces a pretty large single frame allocation, (roughly + // 25 megabits). The encoder allocates a good number of these frames + // one for each lag in frames (for 2 pass), and then one for each possible + // reference buffer (8) - we can end up with up to 30 buffers of roughly this + // size or almost 1 gig of memory. + // In total the allocations will exceed 2GiB which may cause a failure with + // non-64 bit platforms, use a smaller size in that case. + if (sizeof(void *) < 8) + video.SetSize(2560, 1440); + else + video.SetSize(4096, 4096); + + video.set_limit(2); + expected_res_ = AOM_CODEC_OK; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +#endif +} + +TEST_F(AV1FrameSizeTests, OneByOneVideo) { + ::libaom_test::RandomVideoSource video; + + video.SetSize(1, 1); + video.set_limit(2); + expected_res_ = AOM_CODEC_OK; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} +#undef ONE_BY_ONE_VIDEO_NAME +} // namespace diff --git a/third_party/aom/test/function_equivalence_test.h b/third_party/aom/test/function_equivalence_test.h new file mode 100644 index 0000000000..4b22c74a2b --- /dev/null +++ b/third_party/aom/test/function_equivalence_test.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef TEST_FUNCTION_EQUIVALENCE_TEST_H_ +#define TEST_FUNCTION_EQUIVALENCE_TEST_H_ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/util.h" + +using libaom_test::ACMRandom; + +namespace libaom_test { +// Base class for tests that compare 2 implementations of the same function +// for equivalence. The template parameter should be pointer to a function +// that is being tested. +// +// The test takes a 3-parameters encapsulating struct 'FuncParam', containing: +// - Pointer to reference function +// - Pointer to tested function +// - Integer bit depth (default to 0). +// +// These values are then accessible in the tests as member of params_: +// params_.ref_func, params_.tst_func, and params_.bit_depth. +// + +template +struct FuncParam { + FuncParam(T ref = NULL, T tst = NULL, int bit_depth = 0) + : ref_func(ref), tst_func(tst), bit_depth(bit_depth) {} + T ref_func; + T tst_func; + int bit_depth; +}; + +template +std::ostream &operator<<(std::ostream &os, const FuncParam &p) { + return os << "bit_depth:" << p.bit_depth + << " function:" << reinterpret_cast(p.ref_func) + << " function:" << reinterpret_cast(p.tst_func); +} + +template +class FunctionEquivalenceTest : public ::testing::TestWithParam > { + public: + FunctionEquivalenceTest() : rng_(ACMRandom::DeterministicSeed()) {} + + virtual ~FunctionEquivalenceTest() {} + + virtual void SetUp() { params_ = this->GetParam(); } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + ACMRandom rng_; + FuncParam params_; +}; + +} // namespace libaom_test +#endif // TEST_FUNCTION_EQUIVALENCE_TEST_H_ diff --git a/third_party/aom/test/hadamard_test.cc b/third_party/aom/test/hadamard_test.cc new file mode 100644 index 0000000000..db5cb74747 --- /dev/null +++ b/third_party/aom/test/hadamard_test.cc @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_dsp_rtcd.h" + +#include "test/acm_random.h" +#include "test/register_state_check.h" + +namespace { + +using ::libaom_test::ACMRandom; + +typedef void (*HadamardFunc)(const int16_t *a, int a_stride, int16_t *b); + +void hadamard_loop(const int16_t *a, int a_stride, int16_t *out) { + int16_t b[8]; + for (int i = 0; i < 8; i += 2) { + b[i + 0] = a[i * a_stride] + a[(i + 1) * a_stride]; + b[i + 1] = a[i * a_stride] - a[(i + 1) * a_stride]; + } + int16_t c[8]; + for (int i = 0; i < 8; i += 4) { + c[i + 0] = b[i + 0] + b[i + 2]; + c[i + 1] = b[i + 1] + b[i + 3]; + c[i + 2] = b[i + 0] - b[i + 2]; + c[i + 3] = b[i + 1] - b[i + 3]; + } + out[0] = c[0] + c[4]; + out[7] = c[1] + c[5]; + out[3] = c[2] + c[6]; + out[4] = c[3] + c[7]; + out[2] = c[0] - c[4]; + out[6] = c[1] - c[5]; + out[1] = c[2] - c[6]; + out[5] = c[3] - c[7]; +} + +void reference_hadamard8x8(const int16_t *a, int a_stride, int16_t *b) { + int16_t buf[64]; + for (int i = 0; i < 8; ++i) { + hadamard_loop(a + i, a_stride, buf + i * 8); + } + + for (int i = 0; i < 8; ++i) { + hadamard_loop(buf + i, 8, b + i * 8); + } +} + +void reference_hadamard16x16(const int16_t *a, int a_stride, int16_t *b) { + /* The source is a 16x16 block. The destination is rearranged to 8x32. + * Input is 9 bit. */ + reference_hadamard8x8(a + 0 + 0 * a_stride, a_stride, b + 0); + reference_hadamard8x8(a + 8 + 0 * a_stride, a_stride, b + 64); + reference_hadamard8x8(a + 0 + 8 * a_stride, a_stride, b + 128); + reference_hadamard8x8(a + 8 + 8 * a_stride, a_stride, b + 192); + + /* Overlay the 8x8 blocks and combine. */ + for (int i = 0; i < 64; ++i) { + /* 8x8 steps the range up to 15 bits. */ + const int16_t a0 = b[0]; + const int16_t a1 = b[64]; + const int16_t a2 = b[128]; + const int16_t a3 = b[192]; + + /* Prevent the result from escaping int16_t. */ + const int16_t b0 = (a0 + a1) >> 1; + const int16_t b1 = (a0 - a1) >> 1; + const int16_t b2 = (a2 + a3) >> 1; + const int16_t b3 = (a2 - a3) >> 1; + + /* Store a 16 bit value. */ + b[0] = b0 + b2; + b[64] = b1 + b3; + b[128] = b0 - b2; + b[192] = b1 - b3; + + ++b; + } +} + +class HadamardTestBase : public ::testing::TestWithParam { + public: + virtual void SetUp() { + h_func_ = GetParam(); + rnd_.Reset(ACMRandom::DeterministicSeed()); + } + + protected: + HadamardFunc h_func_; + ACMRandom rnd_; +}; + +class Hadamard8x8Test : public HadamardTestBase {}; + +TEST_P(Hadamard8x8Test, CompareReferenceRandom) { + DECLARE_ALIGNED(16, int16_t, a[64]); + DECLARE_ALIGNED(16, int16_t, b[64]); + int16_t b_ref[64]; + for (int i = 0; i < 64; ++i) { + a[i] = rnd_.Rand9Signed(); + } + memset(b, 0, sizeof(b)); + memset(b_ref, 0, sizeof(b_ref)); + + reference_hadamard8x8(a, 8, b_ref); + ASM_REGISTER_STATE_CHECK(h_func_(a, 8, b)); + + // The order of the output is not important. Sort before checking. + std::sort(b, b + 64); + std::sort(b_ref, b_ref + 64); + EXPECT_EQ(0, memcmp(b, b_ref, sizeof(b))); +} + +TEST_P(Hadamard8x8Test, VaryStride) { + DECLARE_ALIGNED(16, int16_t, a[64 * 8]); + DECLARE_ALIGNED(16, int16_t, b[64]); + int16_t b_ref[64]; + for (int i = 0; i < 64 * 8; ++i) { + a[i] = rnd_.Rand9Signed(); + } + + for (int i = 8; i < 64; i += 8) { + memset(b, 0, sizeof(b)); + memset(b_ref, 0, sizeof(b_ref)); + + reference_hadamard8x8(a, i, b_ref); + ASM_REGISTER_STATE_CHECK(h_func_(a, i, b)); + + // The order of the output is not important. Sort before checking. + std::sort(b, b + 64); + std::sort(b_ref, b_ref + 64); + EXPECT_EQ(0, memcmp(b, b_ref, sizeof(b))); + } +} + +INSTANTIATE_TEST_CASE_P(C, Hadamard8x8Test, + ::testing::Values(&aom_hadamard_8x8_c)); + +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P(SSE2, Hadamard8x8Test, + ::testing::Values(&aom_hadamard_8x8_sse2)); +#endif // HAVE_SSE2 + +#if HAVE_SSSE3 && ARCH_X86_64 +INSTANTIATE_TEST_CASE_P(SSSE3, Hadamard8x8Test, + ::testing::Values(&aom_hadamard_8x8_ssse3)); +#endif // HAVE_SSSE3 && ARCH_X86_64 + +#if HAVE_NEON +INSTANTIATE_TEST_CASE_P(NEON, Hadamard8x8Test, + ::testing::Values(&aom_hadamard_8x8_neon)); +#endif // HAVE_NEON + +class Hadamard16x16Test : public HadamardTestBase {}; + +TEST_P(Hadamard16x16Test, CompareReferenceRandom) { + DECLARE_ALIGNED(16, int16_t, a[16 * 16]); + DECLARE_ALIGNED(16, int16_t, b[16 * 16]); + int16_t b_ref[16 * 16]; + for (int i = 0; i < 16 * 16; ++i) { + a[i] = rnd_.Rand9Signed(); + } + memset(b, 0, sizeof(b)); + memset(b_ref, 0, sizeof(b_ref)); + + reference_hadamard16x16(a, 16, b_ref); + ASM_REGISTER_STATE_CHECK(h_func_(a, 16, b)); + + // The order of the output is not important. Sort before checking. + std::sort(b, b + 16 * 16); + std::sort(b_ref, b_ref + 16 * 16); + EXPECT_EQ(0, memcmp(b, b_ref, sizeof(b))); +} + +TEST_P(Hadamard16x16Test, VaryStride) { + DECLARE_ALIGNED(16, int16_t, a[16 * 16 * 8]); + DECLARE_ALIGNED(16, int16_t, b[16 * 16]); + int16_t b_ref[16 * 16]; + for (int i = 0; i < 16 * 16 * 8; ++i) { + a[i] = rnd_.Rand9Signed(); + } + + for (int i = 8; i < 64; i += 8) { + memset(b, 0, sizeof(b)); + memset(b_ref, 0, sizeof(b_ref)); + + reference_hadamard16x16(a, i, b_ref); + ASM_REGISTER_STATE_CHECK(h_func_(a, i, b)); + + // The order of the output is not important. Sort before checking. + std::sort(b, b + 16 * 16); + std::sort(b_ref, b_ref + 16 * 16); + EXPECT_EQ(0, memcmp(b, b_ref, sizeof(b))); + } +} + +INSTANTIATE_TEST_CASE_P(C, Hadamard16x16Test, + ::testing::Values(&aom_hadamard_16x16_c)); + +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P(SSE2, Hadamard16x16Test, + ::testing::Values(&aom_hadamard_16x16_sse2)); +#endif // HAVE_SSE2 + +#if HAVE_NEON +INSTANTIATE_TEST_CASE_P(NEON, Hadamard16x16Test, + ::testing::Values(&aom_hadamard_16x16_neon)); +#endif // HAVE_NEON +} // namespace diff --git a/third_party/aom/test/hbd_metrics_test.cc b/third_party/aom/test/hbd_metrics_test.cc new file mode 100644 index 0000000000..4def53b215 --- /dev/null +++ b/third_party/aom/test/hbd_metrics_test.cc @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/acm_random.h" +#include "test/util.h" +#include "./aom_config.h" +#include "aom_dsp/psnr.h" +#include "aom_dsp/ssim.h" +#include "aom_ports/mem.h" +#include "aom_ports/msvc.h" +#include "aom_scale/yv12config.h" + +using libaom_test::ACMRandom; + +namespace { + +typedef double (*LBDMetricFunc)(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest); +typedef double (*HBDMetricFunc)(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, uint32_t in_bd, + uint32_t bd); + +double compute_hbd_psnr(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, uint32_t in_bd, + uint32_t bd) { + PSNR_STATS psnr; + aom_calc_highbd_psnr(source, dest, &psnr, bd, in_bd); + return psnr.psnr[0]; +} + +double compute_psnr(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest) { + PSNR_STATS psnr; + aom_calc_psnr(source, dest, &psnr); + return psnr.psnr[0]; +} + +double compute_hbd_psnrhvs(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, uint32_t in_bd, + uint32_t bd) { + double tempy, tempu, tempv; + return aom_psnrhvs(source, dest, &tempy, &tempu, &tempv, bd, in_bd); +} + +double compute_psnrhvs(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest) { + double tempy, tempu, tempv; + return aom_psnrhvs(source, dest, &tempy, &tempu, &tempv, 8, 8); +} + +double compute_hbd_fastssim(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, uint32_t in_bd, + uint32_t bd) { + double tempy, tempu, tempv; + return aom_calc_fastssim(source, dest, &tempy, &tempu, &tempv, bd, in_bd); +} + +double compute_fastssim(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest) { + double tempy, tempu, tempv; + return aom_calc_fastssim(source, dest, &tempy, &tempu, &tempv, 8, 8); +} + +double compute_hbd_aomssim(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, uint32_t in_bd, + uint32_t bd) { + double ssim, weight; + ssim = aom_highbd_calc_ssim(source, dest, &weight, bd, in_bd); + return 100 * pow(ssim / weight, 8.0); +} + +double compute_aomssim(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest) { + double ssim, weight; + ssim = aom_calc_ssim(source, dest, &weight); + return 100 * pow(ssim / weight, 8.0); +} + +class HBDMetricsTestBase { + public: + virtual ~HBDMetricsTestBase() {} + + protected: + void RunAccuracyCheck() { + const int width = 1920; + const int height = 1080; + size_t i = 0; + const uint8_t kPixFiller = 128; + YV12_BUFFER_CONFIG lbd_src, lbd_dst; + YV12_BUFFER_CONFIG hbd_src, hbd_dst; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + double lbd_db, hbd_db; + + memset(&lbd_src, 0, sizeof(lbd_src)); + memset(&lbd_dst, 0, sizeof(lbd_dst)); + memset(&hbd_src, 0, sizeof(hbd_src)); + memset(&hbd_dst, 0, sizeof(hbd_dst)); + + aom_alloc_frame_buffer(&lbd_src, width, height, 1, 1, 0, 32, 16); + aom_alloc_frame_buffer(&lbd_dst, width, height, 1, 1, 0, 32, 16); + aom_alloc_frame_buffer(&hbd_src, width, height, 1, 1, 1, 32, 16); + aom_alloc_frame_buffer(&hbd_dst, width, height, 1, 1, 1, 32, 16); + + memset(lbd_src.buffer_alloc, kPixFiller, lbd_src.buffer_alloc_sz); + while (i < lbd_src.buffer_alloc_sz) { + uint16_t spel, dpel; + spel = lbd_src.buffer_alloc[i]; + // Create some distortion for dst buffer. + dpel = rnd.Rand8(); + lbd_dst.buffer_alloc[i] = (uint8_t)dpel; + ((uint16_t *)(hbd_src.buffer_alloc))[i] = spel << (bit_depth_ - 8); + ((uint16_t *)(hbd_dst.buffer_alloc))[i] = dpel << (bit_depth_ - 8); + i++; + } + + lbd_db = lbd_metric_(&lbd_src, &lbd_dst); + hbd_db = hbd_metric_(&hbd_src, &hbd_dst, input_bit_depth_, bit_depth_); + EXPECT_LE(fabs(lbd_db - hbd_db), threshold_); + + i = 0; + while (i < lbd_src.buffer_alloc_sz) { + uint16_t dpel; + // Create some small distortion for dst buffer. + dpel = 120 + (rnd.Rand8() >> 4); + lbd_dst.buffer_alloc[i] = (uint8_t)dpel; + ((uint16_t *)(hbd_dst.buffer_alloc))[i] = dpel << (bit_depth_ - 8); + i++; + } + + lbd_db = lbd_metric_(&lbd_src, &lbd_dst); + hbd_db = hbd_metric_(&hbd_src, &hbd_dst, input_bit_depth_, bit_depth_); + EXPECT_LE(fabs(lbd_db - hbd_db), threshold_); + + i = 0; + while (i < lbd_src.buffer_alloc_sz) { + uint16_t dpel; + // Create some small distortion for dst buffer. + dpel = 126 + (rnd.Rand8() >> 6); + lbd_dst.buffer_alloc[i] = (uint8_t)dpel; + ((uint16_t *)(hbd_dst.buffer_alloc))[i] = dpel << (bit_depth_ - 8); + i++; + } + + lbd_db = lbd_metric_(&lbd_src, &lbd_dst); + hbd_db = hbd_metric_(&hbd_src, &hbd_dst, input_bit_depth_, bit_depth_); + EXPECT_LE(fabs(lbd_db - hbd_db), threshold_); + + aom_free_frame_buffer(&lbd_src); + aom_free_frame_buffer(&lbd_dst); + aom_free_frame_buffer(&hbd_src); + aom_free_frame_buffer(&hbd_dst); + } + + int input_bit_depth_; + int bit_depth_; + double threshold_; + LBDMetricFunc lbd_metric_; + HBDMetricFunc hbd_metric_; +}; + +typedef std::tr1::tuple + MetricTestTParam; +class HBDMetricsTest : public HBDMetricsTestBase, + public ::testing::TestWithParam { + public: + virtual void SetUp() { + lbd_metric_ = GET_PARAM(0); + hbd_metric_ = GET_PARAM(1); + input_bit_depth_ = GET_PARAM(2); + bit_depth_ = GET_PARAM(3); + threshold_ = GET_PARAM(4); + } + virtual void TearDown() {} +}; + +TEST_P(HBDMetricsTest, RunAccuracyCheck) { RunAccuracyCheck(); } + +// Allow small variation due to floating point operations. +static const double kSsim_thresh = 0.001; +// Allow some additional errors accumulated in floating point operations. +static const double kFSsim_thresh = 0.03; +// Allow some extra variation due to rounding error accumulated in dct. +static const double kPhvs_thresh = 0.3; + +INSTANTIATE_TEST_CASE_P( + AOMSSIM, HBDMetricsTest, + ::testing::Values(MetricTestTParam(&compute_aomssim, &compute_hbd_aomssim, + 8, 10, kSsim_thresh), + MetricTestTParam(&compute_aomssim, &compute_hbd_aomssim, + 10, 10, kPhvs_thresh), + MetricTestTParam(&compute_aomssim, &compute_hbd_aomssim, + 8, 12, kSsim_thresh), + MetricTestTParam(&compute_aomssim, &compute_hbd_aomssim, + 12, 12, kPhvs_thresh))); +INSTANTIATE_TEST_CASE_P( + FASTSSIM, HBDMetricsTest, + ::testing::Values(MetricTestTParam(&compute_fastssim, &compute_hbd_fastssim, + 8, 10, kFSsim_thresh), + MetricTestTParam(&compute_fastssim, &compute_hbd_fastssim, + 10, 10, kFSsim_thresh), + MetricTestTParam(&compute_fastssim, &compute_hbd_fastssim, + 8, 12, kFSsim_thresh), + MetricTestTParam(&compute_fastssim, &compute_hbd_fastssim, + 12, 12, kFSsim_thresh))); +INSTANTIATE_TEST_CASE_P( + PSNRHVS, HBDMetricsTest, + ::testing::Values(MetricTestTParam(&compute_psnrhvs, &compute_hbd_psnrhvs, + 8, 10, kPhvs_thresh), + MetricTestTParam(&compute_psnrhvs, &compute_hbd_psnrhvs, + 10, 10, kPhvs_thresh), + MetricTestTParam(&compute_psnrhvs, &compute_hbd_psnrhvs, + 8, 12, kPhvs_thresh), + MetricTestTParam(&compute_psnrhvs, &compute_hbd_psnrhvs, + 12, 12, kPhvs_thresh))); +INSTANTIATE_TEST_CASE_P( + PSNR, HBDMetricsTest, + ::testing::Values( + MetricTestTParam(&compute_psnr, &compute_hbd_psnr, 8, 10, kPhvs_thresh), + MetricTestTParam(&compute_psnr, &compute_hbd_psnr, 10, 10, + kPhvs_thresh), + MetricTestTParam(&compute_psnr, &compute_hbd_psnr, 8, 12, kPhvs_thresh), + MetricTestTParam(&compute_psnr, &compute_hbd_psnr, 12, 12, + kPhvs_thresh))); +} // namespace diff --git a/third_party/aom/test/i420_video_source.h b/third_party/aom/test/i420_video_source.h new file mode 100644 index 0000000000..0825296d73 --- /dev/null +++ b/third_party/aom/test/i420_video_source.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef TEST_I420_VIDEO_SOURCE_H_ +#define TEST_I420_VIDEO_SOURCE_H_ +#include +#include +#include + +#include "test/yuv_video_source.h" + +namespace libaom_test { + +// This class extends VideoSource to allow parsing of raw yv12 +// so that we can do actual file encodes. +class I420VideoSource : public YUVVideoSource { + public: + I420VideoSource(const std::string &file_name, unsigned int width, + unsigned int height, int rate_numerator, int rate_denominator, + unsigned int start, int limit) + : YUVVideoSource(file_name, AOM_IMG_FMT_I420, width, height, + rate_numerator, rate_denominator, start, limit) {} +}; + +} // namespace libaom_test + +#endif // TEST_I420_VIDEO_SOURCE_H_ diff --git a/third_party/aom/test/idct8x8_test.cc b/third_party/aom/test/idct8x8_test.cc new file mode 100644 index 0000000000..f99a4075f8 --- /dev/null +++ b/third_party/aom/test/idct8x8_test.cc @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "aom/aom_integer.h" +#include "aom_ports/msvc.h" // for round() + +using libaom_test::ACMRandom; + +namespace { + +void reference_dct_1d(double input[8], double output[8]) { + const double kPi = 3.141592653589793238462643383279502884; + const double kInvSqrt2 = 0.707106781186547524400844362104; + for (int k = 0; k < 8; k++) { + output[k] = 0.0; + for (int n = 0; n < 8; n++) + output[k] += input[n] * cos(kPi * (2 * n + 1) * k / 16.0); + if (k == 0) output[k] = output[k] * kInvSqrt2; + } +} + +void reference_dct_2d(int16_t input[64], double output[64]) { + // First transform columns + for (int i = 0; i < 8; ++i) { + double temp_in[8], temp_out[8]; + for (int j = 0; j < 8; ++j) temp_in[j] = input[j * 8 + i]; + reference_dct_1d(temp_in, temp_out); + for (int j = 0; j < 8; ++j) output[j * 8 + i] = temp_out[j]; + } + // Then transform rows + for (int i = 0; i < 8; ++i) { + double temp_in[8], temp_out[8]; + for (int j = 0; j < 8; ++j) temp_in[j] = output[j + i * 8]; + reference_dct_1d(temp_in, temp_out); + for (int j = 0; j < 8; ++j) output[j + i * 8] = temp_out[j]; + } + // Scale by some magic number + for (int i = 0; i < 64; ++i) output[i] *= 2; +} + +TEST(AV1Idct8x8Test, AccuracyCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 10000; + for (int i = 0; i < count_test_block; ++i) { + int16_t input[64]; + tran_low_t coeff[64]; + double output_r[64]; + uint8_t dst[64], src[64]; + + for (int j = 0; j < 64; ++j) { + src[j] = rnd.Rand8(); + dst[j] = rnd.Rand8(); + } + // Initialize a test block with input range [-255, 255]. + for (int j = 0; j < 64; ++j) input[j] = src[j] - dst[j]; + + reference_dct_2d(input, output_r); + for (int j = 0; j < 64; ++j) + coeff[j] = static_cast(round(output_r[j])); + aom_idct8x8_64_add_c(coeff, dst, 8); + for (int j = 0; j < 64; ++j) { + const int diff = dst[j] - src[j]; + const int error = diff * diff; + EXPECT_GE(1, error) << "Error: 8x8 FDCT/IDCT has error " << error + << " at index " << j; + } + } +} + +} // namespace diff --git a/third_party/aom/test/idct_test.cc b/third_party/aom/test/idct_test.cc new file mode 100644 index 0000000000..a880a91825 --- /dev/null +++ b/third_party/aom/test/idct_test.cc @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "./aom_config.h" +#include "./aom_rtcd.h" + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "aom/aom_integer.h" + +typedef void (*IdctFunc)(int16_t *input, unsigned char *pred_ptr, + int pred_stride, unsigned char *dst_ptr, + int dst_stride); +namespace { +class IDCTTest : public ::testing::TestWithParam { + protected: + virtual void SetUp() { + int i; + + UUT = GetParam(); + memset(input, 0, sizeof(input)); + /* Set up guard blocks */ + for (i = 0; i < 256; i++) output[i] = ((i & 0xF) < 4 && (i < 64)) ? 0 : -1; + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + IdctFunc UUT; + int16_t input[16]; + unsigned char output[256]; + unsigned char predict[256]; +}; + +TEST_P(IDCTTest, TestGuardBlocks) { + int i; + + for (i = 0; i < 256; i++) + if ((i & 0xF) < 4 && i < 64) + EXPECT_EQ(0, output[i]) << i; + else + EXPECT_EQ(255, output[i]); +} + +TEST_P(IDCTTest, TestAllZeros) { + int i; + + ASM_REGISTER_STATE_CHECK(UUT(input, output, 16, output, 16)); + + for (i = 0; i < 256; i++) + if ((i & 0xF) < 4 && i < 64) + EXPECT_EQ(0, output[i]) << "i==" << i; + else + EXPECT_EQ(255, output[i]) << "i==" << i; +} + +TEST_P(IDCTTest, TestAllOnes) { + int i; + + input[0] = 4; + ASM_REGISTER_STATE_CHECK(UUT(input, output, 16, output, 16)); + + for (i = 0; i < 256; i++) + if ((i & 0xF) < 4 && i < 64) + EXPECT_EQ(1, output[i]) << "i==" << i; + else + EXPECT_EQ(255, output[i]) << "i==" << i; +} + +TEST_P(IDCTTest, TestAddOne) { + int i; + + for (i = 0; i < 256; i++) predict[i] = i; + input[0] = 4; + ASM_REGISTER_STATE_CHECK(UUT(input, predict, 16, output, 16)); + + for (i = 0; i < 256; i++) + if ((i & 0xF) < 4 && i < 64) + EXPECT_EQ(i + 1, output[i]) << "i==" << i; + else + EXPECT_EQ(255, output[i]) << "i==" << i; +} + +TEST_P(IDCTTest, TestWithData) { + int i; + + for (i = 0; i < 16; i++) input[i] = i; + + ASM_REGISTER_STATE_CHECK(UUT(input, output, 16, output, 16)); + + for (i = 0; i < 256; i++) + if ((i & 0xF) > 3 || i > 63) + EXPECT_EQ(255, output[i]) << "i==" << i; + else if (i == 0) + EXPECT_EQ(11, output[i]) << "i==" << i; + else if (i == 34) + EXPECT_EQ(1, output[i]) << "i==" << i; + else if (i == 2 || i == 17 || i == 32) + EXPECT_EQ(3, output[i]) << "i==" << i; + else + EXPECT_EQ(0, output[i]) << "i==" << i; +} + +INSTANTIATE_TEST_CASE_P(C, IDCTTest, ::testing::Values(aom_short_idct4x4llm_c)); +#if HAVE_MMX +INSTANTIATE_TEST_CASE_P(MMX, IDCTTest, + ::testing::Values(aom_short_idct4x4llm_mmx)); +#endif +#if HAVE_MSA +INSTANTIATE_TEST_CASE_P(MSA, IDCTTest, + ::testing::Values(aom_short_idct4x4llm_msa)); +#endif +} diff --git a/third_party/aom/test/intrabc_test.cc b/third_party/aom/test/intrabc_test.cc new file mode 100644 index 0000000000..84cfa5c485 --- /dev/null +++ b/third_party/aom/test/intrabc_test.cc @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "av1/common/enums.h" +#include "av1/common/mv.h" +#include "av1/common/mvref_common.h" +#include "av1/common/tile_common.h" + +namespace { +TEST(IntrabcTest, DvValidation) { + struct DvTestCase { + MV dv; + int mi_row_offset; + int mi_col_offset; + BLOCK_SIZE bsize; + bool valid; + }; + const int kSubPelScale = 8; + const int kTileMaxMibWidth = 8; + const DvTestCase kDvCases[] = { +#if CONFIG_EXT_PARTITION + { { 0, 0 }, 0, 0, BLOCK_128X128, false }, +#endif + { { 0, 0 }, 0, 0, BLOCK_64X64, false }, + { { 0, 0 }, 0, 0, BLOCK_32X32, false }, + { { 0, 0 }, 0, 0, BLOCK_16X16, false }, + { { 0, 0 }, 0, 0, BLOCK_8X8, false }, + { { 0, 0 }, 0, 0, BLOCK_4X4, false }, + { { -MAX_SB_SIZE * kSubPelScale, -MAX_SB_SIZE * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_16X16, + true }, + { { 0, -MAX_SB_SIZE * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_16X16, + true }, + { { -MAX_SB_SIZE * kSubPelScale, 0 }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_16X16, + true }, + { { MAX_SB_SIZE * kSubPelScale, 0 }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_16X16, + false }, + { { 0, MAX_SB_SIZE * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_16X16, + false }, + { { -32 * kSubPelScale, -32 * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_32X32, + true }, + { { -32 * kSubPelScale, -32 * kSubPelScale }, + 32 / MI_SIZE, + 32 / MI_SIZE, + BLOCK_32X32, + false }, + { { -32 * kSubPelScale - kSubPelScale / 2, -32 * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_32X32, + false }, + { { -33 * kSubPelScale, -32 * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_32X32, + true }, + { { -32 * kSubPelScale, -32 * kSubPelScale - kSubPelScale / 2 }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_32X32, + false }, + { { -32 * kSubPelScale, -33 * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_32X32, + true }, + { { -MAX_SB_SIZE * kSubPelScale, -MAX_SB_SIZE * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_LARGEST, + true }, + { { -(MAX_SB_SIZE + 1) * kSubPelScale, -MAX_SB_SIZE * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_LARGEST, + false }, + { { -MAX_SB_SIZE * kSubPelScale, -(MAX_SB_SIZE + 1) * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_LARGEST, + false }, + { { -(MAX_SB_SIZE - 1) * kSubPelScale, -MAX_SB_SIZE * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_LARGEST, + true }, + { { -MAX_SB_SIZE * kSubPelScale, -(MAX_SB_SIZE - 1) * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_LARGEST, + true }, + { { -(MAX_SB_SIZE - 1) * kSubPelScale, -(MAX_SB_SIZE - 1) * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_LARGEST, + false }, + { { -MAX_SB_SIZE * kSubPelScale, MAX_SB_SIZE * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_LARGEST, + true }, + { { -MAX_SB_SIZE * kSubPelScale, + (kTileMaxMibWidth - 2) * MAX_SB_SIZE * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_LARGEST, + true }, + { { -MAX_SB_SIZE * kSubPelScale, + ((kTileMaxMibWidth - 2) * MAX_SB_SIZE + 1) * kSubPelScale }, + MAX_SB_SIZE / MI_SIZE, + MAX_SB_SIZE / MI_SIZE, + BLOCK_LARGEST, + false }, + }; + TileInfo tile; + tile.mi_row_start = 8 * MAX_MIB_SIZE; + tile.mi_row_end = 16 * MAX_MIB_SIZE; + tile.mi_col_start = 24 * MAX_MIB_SIZE; + tile.mi_col_end = tile.mi_col_start + kTileMaxMibWidth * MAX_MIB_SIZE; + for (int i = 0; i < static_cast(GTEST_ARRAY_SIZE_(kDvCases)); ++i) { + EXPECT_EQ(kDvCases[i].valid, + is_dv_valid(kDvCases[i].dv, &tile, + tile.mi_row_start + kDvCases[i].mi_row_offset, + tile.mi_col_start + kDvCases[i].mi_col_offset, + kDvCases[i].bsize)) + << "DvCases[" << i << "]"; + } +} +} // namespace diff --git a/third_party/aom/test/intrapred_test.cc b/third_party/aom/test/intrapred_test.cc new file mode 100644 index 0000000000..4efed57b6b --- /dev/null +++ b/third_party/aom/test/intrapred_test.cc @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "av1/common/blockd.h" +#include "av1/common/pred_common.h" +#include "aom_mem/aom_mem.h" + +namespace { + +using libaom_test::ACMRandom; + +const int count_test_block = 100000; + +typedef void (*IntraPred)(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, const uint16_t *left, int bps); + +struct IntraPredFunc { + IntraPredFunc(IntraPred pred = NULL, IntraPred ref = NULL, + int block_size_value = 0, int bit_depth_value = 0) + : pred_fn(pred), ref_fn(ref), block_size(block_size_value), + bit_depth(bit_depth_value) {} + + IntraPred pred_fn; + IntraPred ref_fn; + int block_size; + int bit_depth; +}; + +class AV1IntraPredTest : public ::testing::TestWithParam { + public: + void RunTest(uint16_t *left_col, uint16_t *above_data, uint16_t *dst, + uint16_t *ref_dst) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int block_size = params_.block_size; + above_row_ = above_data + 16; + left_col_ = left_col; + dst_ = dst; + ref_dst_ = ref_dst; + int error_count = 0; + for (int i = 0; i < count_test_block; ++i) { + // Fill edges with random data, try first with saturated values. + for (int x = -1; x <= block_size * 2; x++) { + if (i == 0) { + above_row_[x] = mask_; + } else { + above_row_[x] = rnd.Rand16() & mask_; + } + } + for (int y = 0; y < block_size; y++) { + if (i == 0) { + left_col_[y] = mask_; + } else { + left_col_[y] = rnd.Rand16() & mask_; + } + } + Predict(); + CheckPrediction(i, &error_count); + } + ASSERT_EQ(0, error_count); + } + + protected: + virtual void SetUp() { + params_ = GetParam(); + stride_ = params_.block_size * 3; + mask_ = (1 << params_.bit_depth) - 1; + } + + void Predict() { + const int bit_depth = params_.bit_depth; + params_.ref_fn(ref_dst_, stride_, above_row_, left_col_, bit_depth); + ASM_REGISTER_STATE_CHECK( + params_.pred_fn(dst_, stride_, above_row_, left_col_, bit_depth)); + } + + void CheckPrediction(int test_case_number, int *error_count) const { + // For each pixel ensure that the calculated value is the same as reference. + const int block_size = params_.block_size; + for (int y = 0; y < block_size; y++) { + for (int x = 0; x < block_size; x++) { + *error_count += ref_dst_[x + y * stride_] != dst_[x + y * stride_]; + if (*error_count == 1) { + ASSERT_EQ(ref_dst_[x + y * stride_], dst_[x + y * stride_]) + << " Failed on Test Case Number " << test_case_number; + } + } + } + } + + uint16_t *above_row_; + uint16_t *left_col_; + uint16_t *dst_; + uint16_t *ref_dst_; + ptrdiff_t stride_; + int mask_; + + IntraPredFunc params_; +}; + +TEST_P(AV1IntraPredTest, IntraPredTests) { + // max block size is 32 + DECLARE_ALIGNED(16, uint16_t, left_col[2 * 32]); + DECLARE_ALIGNED(16, uint16_t, above_data[2 * 32 + 32]); + DECLARE_ALIGNED(16, uint16_t, dst[3 * 32 * 32]); + DECLARE_ALIGNED(16, uint16_t, ref_dst[3 * 32 * 32]); + RunTest(left_col, above_data, dst, ref_dst); +} + +#if HAVE_SSE2 +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSE2_TO_C_8, AV1IntraPredTest, + ::testing::Values(IntraPredFunc(&aom_highbd_dc_predictor_32x32_sse2, + &aom_highbd_dc_predictor_32x32_c, 32, 8), +#if !CONFIG_ALT_INTRA + IntraPredFunc(&aom_highbd_tm_predictor_16x16_sse2, + &aom_highbd_tm_predictor_16x16_c, 16, 8), + IntraPredFunc(&aom_highbd_tm_predictor_32x32_sse2, + &aom_highbd_tm_predictor_32x32_c, 32, 8), +#endif // !CONFIG_ALT_INTRA + + IntraPredFunc(&aom_highbd_dc_predictor_4x4_sse2, + &aom_highbd_dc_predictor_4x4_c, 4, 8), + IntraPredFunc(&aom_highbd_dc_predictor_8x8_sse2, + &aom_highbd_dc_predictor_8x8_c, 8, 8), + IntraPredFunc(&aom_highbd_dc_predictor_16x16_sse2, + &aom_highbd_dc_predictor_16x16_c, 16, 8), + IntraPredFunc(&aom_highbd_v_predictor_4x4_sse2, + &aom_highbd_v_predictor_4x4_c, 4, 8), + IntraPredFunc(&aom_highbd_v_predictor_8x8_sse2, + &aom_highbd_v_predictor_8x8_c, 8, 8), + IntraPredFunc(&aom_highbd_v_predictor_16x16_sse2, + &aom_highbd_v_predictor_16x16_c, 16, 8), + IntraPredFunc(&aom_highbd_v_predictor_32x32_sse2, + &aom_highbd_v_predictor_32x32_c, 32, 8) +#if !CONFIG_ALT_INTRA + , + IntraPredFunc(&aom_highbd_tm_predictor_4x4_sse2, + &aom_highbd_tm_predictor_4x4_c, 4, 8), + IntraPredFunc(&aom_highbd_tm_predictor_8x8_sse2, + &aom_highbd_tm_predictor_8x8_c, 8, 8) +#endif // !CONFIG_ALT_INTRA + )); + +INSTANTIATE_TEST_CASE_P( + SSE2_TO_C_10, AV1IntraPredTest, + ::testing::Values(IntraPredFunc(&aom_highbd_dc_predictor_32x32_sse2, + &aom_highbd_dc_predictor_32x32_c, 32, 10), +#if !CONFIG_ALT_INTRA + IntraPredFunc(&aom_highbd_tm_predictor_16x16_sse2, + &aom_highbd_tm_predictor_16x16_c, 16, 10), + IntraPredFunc(&aom_highbd_tm_predictor_32x32_sse2, + &aom_highbd_tm_predictor_32x32_c, 32, 10), +#endif // !CONFIG_ALT_INTRA + IntraPredFunc(&aom_highbd_dc_predictor_4x4_sse2, + &aom_highbd_dc_predictor_4x4_c, 4, 10), + IntraPredFunc(&aom_highbd_dc_predictor_8x8_sse2, + &aom_highbd_dc_predictor_8x8_c, 8, 10), + IntraPredFunc(&aom_highbd_dc_predictor_16x16_sse2, + &aom_highbd_dc_predictor_16x16_c, 16, 10), + IntraPredFunc(&aom_highbd_v_predictor_4x4_sse2, + &aom_highbd_v_predictor_4x4_c, 4, 10), + IntraPredFunc(&aom_highbd_v_predictor_8x8_sse2, + &aom_highbd_v_predictor_8x8_c, 8, 10), + IntraPredFunc(&aom_highbd_v_predictor_16x16_sse2, + &aom_highbd_v_predictor_16x16_c, 16, 10), + IntraPredFunc(&aom_highbd_v_predictor_32x32_sse2, + &aom_highbd_v_predictor_32x32_c, 32, 10) +#if !CONFIG_ALT_INTRA + , + IntraPredFunc(&aom_highbd_tm_predictor_4x4_sse2, + &aom_highbd_tm_predictor_4x4_c, 4, 10), + IntraPredFunc(&aom_highbd_tm_predictor_8x8_sse2, + &aom_highbd_tm_predictor_8x8_c, 8, 10) +#endif // !CONFIG_ALT_INTRA + )); + +INSTANTIATE_TEST_CASE_P( + SSE2_TO_C_12, AV1IntraPredTest, + ::testing::Values(IntraPredFunc(&aom_highbd_dc_predictor_32x32_sse2, + &aom_highbd_dc_predictor_32x32_c, 32, 12), +#if !CONFIG_ALT_INTRA + IntraPredFunc(&aom_highbd_tm_predictor_16x16_sse2, + &aom_highbd_tm_predictor_16x16_c, 16, 12), + IntraPredFunc(&aom_highbd_tm_predictor_32x32_sse2, + &aom_highbd_tm_predictor_32x32_c, 32, 12), +#endif // !CONFIG_ALT_INTRA + IntraPredFunc(&aom_highbd_dc_predictor_4x4_sse2, + &aom_highbd_dc_predictor_4x4_c, 4, 12), + IntraPredFunc(&aom_highbd_dc_predictor_8x8_sse2, + &aom_highbd_dc_predictor_8x8_c, 8, 12), + IntraPredFunc(&aom_highbd_dc_predictor_16x16_sse2, + &aom_highbd_dc_predictor_16x16_c, 16, 12), + IntraPredFunc(&aom_highbd_v_predictor_4x4_sse2, + &aom_highbd_v_predictor_4x4_c, 4, 12), + IntraPredFunc(&aom_highbd_v_predictor_8x8_sse2, + &aom_highbd_v_predictor_8x8_c, 8, 12), + IntraPredFunc(&aom_highbd_v_predictor_16x16_sse2, + &aom_highbd_v_predictor_16x16_c, 16, 12), + IntraPredFunc(&aom_highbd_v_predictor_32x32_sse2, + &aom_highbd_v_predictor_32x32_c, 32, 12) +#if !CONFIG_ALT_INTRA + , + IntraPredFunc(&aom_highbd_tm_predictor_4x4_sse2, + &aom_highbd_tm_predictor_4x4_c, 4, 12), + IntraPredFunc(&aom_highbd_tm_predictor_8x8_sse2, + &aom_highbd_tm_predictor_8x8_c, 8, 12) +#endif // !CONFIG_ALT_INTRA + )); + +#endif // CONFIG_HIGHBITDEPTH +#endif // HAVE_SSE2 +} // namespace diff --git a/third_party/aom/test/ivf_video_source.h b/third_party/aom/test/ivf_video_source.h new file mode 100644 index 0000000000..0d3e9f9cb4 --- /dev/null +++ b/third_party/aom/test/ivf_video_source.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef TEST_IVF_VIDEO_SOURCE_H_ +#define TEST_IVF_VIDEO_SOURCE_H_ +#include +#include +#include +#include +#include "test/video_source.h" + +namespace libaom_test { +const unsigned int kCodeBufferSize = 256 * 1024; +const unsigned int kIvfFileHdrSize = 32; +const unsigned int kIvfFrameHdrSize = 12; + +static unsigned int MemGetLe32(const uint8_t *mem) { + return (mem[3] << 24) | (mem[2] << 16) | (mem[1] << 8) | (mem[0]); +} + +// This class extends VideoSource to allow parsing of ivf files, +// so that we can do actual file decodes. +class IVFVideoSource : public CompressedVideoSource { + public: + explicit IVFVideoSource(const std::string &file_name) + : file_name_(file_name), input_file_(NULL), compressed_frame_buf_(NULL), + frame_sz_(0), frame_(0), end_of_file_(false) {} + + virtual ~IVFVideoSource() { + delete[] compressed_frame_buf_; + + if (input_file_) fclose(input_file_); + } + + virtual void Init() { + // Allocate a buffer for read in the compressed video frame. + compressed_frame_buf_ = new uint8_t[libaom_test::kCodeBufferSize]; + ASSERT_TRUE(compressed_frame_buf_ != NULL) + << "Allocate frame buffer failed"; + } + + virtual void Begin() { + input_file_ = OpenTestDataFile(file_name_); + ASSERT_TRUE(input_file_ != NULL) << "Input file open failed. Filename: " + << file_name_; + + // Read file header + uint8_t file_hdr[kIvfFileHdrSize]; + ASSERT_EQ(kIvfFileHdrSize, fread(file_hdr, 1, kIvfFileHdrSize, input_file_)) + << "File header read failed."; + // Check file header + ASSERT_TRUE(file_hdr[0] == 'D' && file_hdr[1] == 'K' && + file_hdr[2] == 'I' && file_hdr[3] == 'F') + << "Input is not an IVF file."; + + FillFrame(); + } + + virtual void Next() { + ++frame_; + FillFrame(); + } + + void FillFrame() { + ASSERT_TRUE(input_file_ != NULL); + uint8_t frame_hdr[kIvfFrameHdrSize]; + // Check frame header and read a frame from input_file. + if (fread(frame_hdr, 1, kIvfFrameHdrSize, input_file_) != + kIvfFrameHdrSize) { + end_of_file_ = true; + } else { + end_of_file_ = false; + + frame_sz_ = MemGetLe32(frame_hdr); + ASSERT_LE(frame_sz_, kCodeBufferSize) + << "Frame is too big for allocated code buffer"; + ASSERT_EQ(frame_sz_, + fread(compressed_frame_buf_, 1, frame_sz_, input_file_)) + << "Failed to read complete frame"; + } + } + + virtual const uint8_t *cxdata() const { + return end_of_file_ ? NULL : compressed_frame_buf_; + } + virtual size_t frame_size() const { return frame_sz_; } + virtual unsigned int frame_number() const { return frame_; } + + protected: + std::string file_name_; + FILE *input_file_; + uint8_t *compressed_frame_buf_; + size_t frame_sz_; + unsigned int frame_; + bool end_of_file_; +}; + +} // namespace libaom_test + +#endif // TEST_IVF_VIDEO_SOURCE_H_ diff --git a/third_party/aom/test/level_test.cc b/third_party/aom/test/level_test.cc new file mode 100644 index 0000000000..1049d4901e --- /dev/null +++ b/third_party/aom/test/level_test.cc @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" + +namespace { +class LevelTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params { + protected: + LevelTest() + : EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)), + cpu_used_(GET_PARAM(2)), min_gf_internal_(24), target_level_(0), + level_(0) {} + virtual ~LevelTest() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + if (encoding_mode_ != ::libaom_test::kRealTime) { + cfg_.g_lag_in_frames = 25; + cfg_.rc_end_usage = AOM_VBR; + } else { + cfg_.g_lag_in_frames = 0; + cfg_.rc_end_usage = AOM_CBR; + } + cfg_.rc_2pass_vbr_minsection_pct = 5; + cfg_.rc_2pass_vbr_maxsection_pct = 2000; + cfg_.rc_target_bitrate = 400; + cfg_.rc_max_quantizer = 63; + cfg_.rc_min_quantizer = 0; + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 0) { + encoder->Control(AOME_SET_CPUUSED, cpu_used_); + encoder->Control(AV1E_SET_TARGET_LEVEL, target_level_); + encoder->Control(AV1E_SET_MIN_GF_INTERVAL, min_gf_internal_); + if (encoding_mode_ != ::libaom_test::kRealTime) { + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); + encoder->Control(AOME_SET_ARNR_MAXFRAMES, 7); + encoder->Control(AOME_SET_ARNR_STRENGTH, 5); + } + } + encoder->Control(AV1E_GET_LEVEL, &level_); + ASSERT_LE(level_, 51); + ASSERT_GE(level_, 0); + } + + ::libaom_test::TestMode encoding_mode_; + int cpu_used_; + int min_gf_internal_; + int target_level_; + int level_; +}; + +// Test for keeping level stats only +TEST_P(LevelTest, TestTargetLevel0) { + ::libaom_test::I420VideoSource video("hantro_odd.yuv", 208, 144, 30, 1, 0, + 40); + target_level_ = 0; + min_gf_internal_ = 4; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + ASSERT_EQ(11, level_); + + cfg_.rc_target_bitrate = 1600; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + ASSERT_EQ(20, level_); +} + +// Test for level control being turned off +TEST_P(LevelTest, TestTargetLevel255) { + ::libaom_test::I420VideoSource video("hantro_odd.yuv", 208, 144, 30, 1, 0, + 30); + target_level_ = 255; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} + +TEST_P(LevelTest, TestTargetLevelApi) { + ::libaom_test::I420VideoSource video("hantro_odd.yuv", 208, 144, 30, 1, 0, 1); + static const aom_codec_iface_t *codec = &aom_codec_av1_cx_algo; + aom_codec_ctx_t enc; + aom_codec_enc_cfg_t cfg; + EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(codec, &cfg, 0)); + EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, codec, &cfg, 0)); + for (int level = 0; level <= 256; ++level) { + if (level == 10 || level == 11 || level == 20 || level == 21 || + level == 30 || level == 31 || level == 40 || level == 41 || + level == 50 || level == 51 || level == 52 || level == 60 || + level == 61 || level == 62 || level == 0 || level == 255) + EXPECT_EQ(AOM_CODEC_OK, + aom_codec_control(&enc, AV1E_SET_TARGET_LEVEL, level)); + else + EXPECT_EQ(AOM_CODEC_INVALID_PARAM, + aom_codec_control(&enc, AV1E_SET_TARGET_LEVEL, level)); + } + EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc)); +} + +AV1_INSTANTIATE_TEST_CASE(LevelTest, + ::testing::Values(::libaom_test::kTwoPassGood, + ::libaom_test::kOnePassGood), + ::testing::Range(0, 9)); +} // namespace diff --git a/third_party/aom/test/lossless_test.cc b/third_party/aom/test/lossless_test.cc new file mode 100644 index 0000000000..5c5b32d932 --- /dev/null +++ b/third_party/aom/test/lossless_test.cc @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" +#include "test/y4m_video_source.h" + +namespace { + +const int kMaxPsnr = 100; + +class LosslessTestLarge + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWithParam { + protected: + LosslessTestLarge() + : EncoderTest(GET_PARAM(0)), psnr_(kMaxPsnr), nframes_(0), + encoding_mode_(GET_PARAM(1)) {} + + virtual ~LosslessTestLarge() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 1) { + // Only call Control if quantizer > 0 to verify that using quantizer + // alone will activate lossless + if (cfg_.rc_max_quantizer > 0 || cfg_.rc_min_quantizer > 0) { + encoder->Control(AV1E_SET_LOSSLESS, 1); + } + } + } + + virtual void BeginPassHook(unsigned int /*pass*/) { + psnr_ = kMaxPsnr; + nframes_ = 0; + } + + virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) { + if (pkt->data.psnr.psnr[0] < psnr_) psnr_ = pkt->data.psnr.psnr[0]; + } + + double GetMinPsnr() const { return psnr_; } + + private: + double psnr_; + unsigned int nframes_; + libaom_test::TestMode encoding_mode_; +}; + +TEST_P(LosslessTestLarge, TestLossLessEncoding) { + const aom_rational timebase = { 33333333, 1000000000 }; + cfg_.g_timebase = timebase; + cfg_.rc_target_bitrate = 2000; + cfg_.g_lag_in_frames = 25; + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 0; + + init_flags_ = AOM_CODEC_USE_PSNR; + + // intentionally changed the dimension for better testing coverage + libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + timebase.den, timebase.num, 0, 5); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + const double psnr_lossless = GetMinPsnr(); + EXPECT_GE(psnr_lossless, kMaxPsnr); +} + +TEST_P(LosslessTestLarge, TestLossLessEncoding444) { + libaom_test::Y4mVideoSource video("rush_hour_444.y4m", 0, 5); + + cfg_.g_profile = 1; + cfg_.g_timebase = video.timebase(); + cfg_.rc_target_bitrate = 2000; + cfg_.g_lag_in_frames = 25; + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 0; + + init_flags_ = AOM_CODEC_USE_PSNR; + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + const double psnr_lossless = GetMinPsnr(); + EXPECT_GE(psnr_lossless, kMaxPsnr); +} + +TEST_P(LosslessTestLarge, TestLossLessEncodingCtrl) { + const aom_rational timebase = { 33333333, 1000000000 }; + cfg_.g_timebase = timebase; + cfg_.rc_target_bitrate = 2000; + cfg_.g_lag_in_frames = 25; + // Intentionally set Q > 0, to make sure control can be used to activate + // lossless + cfg_.rc_min_quantizer = 10; + cfg_.rc_max_quantizer = 20; + + init_flags_ = AOM_CODEC_USE_PSNR; + + libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + timebase.den, timebase.num, 0, 5); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + const double psnr_lossless = GetMinPsnr(); + EXPECT_GE(psnr_lossless, kMaxPsnr); +} + +AV1_INSTANTIATE_TEST_CASE(LosslessTestLarge, + ::testing::Values(::libaom_test::kOnePassGood, + ::libaom_test::kTwoPassGood)); +} // namespace diff --git a/third_party/aom/test/lpf_8_test.cc b/third_party/aom/test/lpf_8_test.cc new file mode 100644 index 0000000000..cee0d3b814 --- /dev/null +++ b/third_party/aom/test/lpf_8_test.cc @@ -0,0 +1,624 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "av1/common/av1_loopfilter.h" +#include "av1/common/entropy.h" +#include "aom/aom_integer.h" + +using libaom_test::ACMRandom; + +namespace { +// Horizontally and Vertically need 32x32: 8 Coeffs preceeding filtered section +// 16 Coefs within filtered section +// 8 Coeffs following filtered section +const int kNumCoeffs = 1024; + +const int number_of_iterations = 10000; + +#if CONFIG_HIGHBITDEPTH +typedef void (*loop_op_t)(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, int bd); +typedef void (*dual_loop_op_t)(uint16_t *s, int p, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, + const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1, int bd); +#else +typedef void (*loop_op_t)(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh); +typedef void (*dual_loop_op_t)(uint8_t *s, int p, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, + const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1); +#endif // CONFIG_HIGHBITDEPTH + +typedef std::tr1::tuple loop8_param_t; +typedef std::tr1::tuple dualloop8_param_t; + +class Loop8Test6Param : public ::testing::TestWithParam { + public: + virtual ~Loop8Test6Param() {} + virtual void SetUp() { + loopfilter_op_ = GET_PARAM(0); + ref_loopfilter_op_ = GET_PARAM(1); + bit_depth_ = GET_PARAM(2); + mask_ = (1 << bit_depth_) - 1; + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + int bit_depth_; + int mask_; + loop_op_t loopfilter_op_; + loop_op_t ref_loopfilter_op_; +}; + +class Loop8Test9Param : public ::testing::TestWithParam { + public: + virtual ~Loop8Test9Param() {} + virtual void SetUp() { + loopfilter_op_ = GET_PARAM(0); + ref_loopfilter_op_ = GET_PARAM(1); + bit_depth_ = GET_PARAM(2); + mask_ = (1 << bit_depth_) - 1; + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + int bit_depth_; + int mask_; + dual_loop_op_t loopfilter_op_; + dual_loop_op_t ref_loopfilter_op_; +}; + +TEST_P(Loop8Test6Param, OperationCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = number_of_iterations; +#if CONFIG_HIGHBITDEPTH + int32_t bd = bit_depth_; + DECLARE_ALIGNED(16, uint16_t, s[kNumCoeffs]); + DECLARE_ALIGNED(16, uint16_t, ref_s[kNumCoeffs]); +#else + DECLARE_ALIGNED(8, uint8_t, s[kNumCoeffs]); + DECLARE_ALIGNED(8, uint8_t, ref_s[kNumCoeffs]); +#endif // CONFIG_HIGHBITDEPTH + int err_count_total = 0; + int first_failure = -1; + for (int i = 0; i < count_test_block; ++i) { + int err_count = 0; + uint8_t tmp = static_cast(rnd(3 * MAX_LOOP_FILTER + 4)); + DECLARE_ALIGNED(16, const uint8_t, + blimit[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = static_cast(rnd(MAX_LOOP_FILTER)); + DECLARE_ALIGNED(16, const uint8_t, + limit[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, + thresh[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + int32_t p = kNumCoeffs / 32; + + uint16_t tmp_s[kNumCoeffs]; + int j = 0; + while (j < kNumCoeffs) { + uint8_t val = rnd.Rand8(); + if (val & 0x80) { // 50% chance to choose a new value. + tmp_s[j] = rnd.Rand16(); + j++; + } else { // 50% chance to repeat previous value in row X times + int k = 0; + while (k++ < ((val & 0x1f) + 1) && j < kNumCoeffs) { + if (j < 1) { + tmp_s[j] = rnd.Rand16(); + } else if (val & 0x20) { // Increment by an value within the limit + tmp_s[j] = (tmp_s[j - 1] + (*limit - 1)); + } else { // Decrement by an value within the limit + tmp_s[j] = (tmp_s[j - 1] - (*limit - 1)); + } + j++; + } + } + } + for (j = 0; j < kNumCoeffs; j++) { + if (i % 2) { + s[j] = tmp_s[j] & mask_; + } else { + s[j] = tmp_s[p * (j % p) + j / p] & mask_; + } + ref_s[j] = s[j]; + } +#if CONFIG_HIGHBITDEPTH + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit, limit, thresh, bd); + ASM_REGISTER_STATE_CHECK( + loopfilter_op_(s + 8 + p * 8, p, blimit, limit, thresh, bd)); +#else + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit, limit, thresh); + ASM_REGISTER_STATE_CHECK( + loopfilter_op_(s + 8 + p * 8, p, blimit, limit, thresh)); +#endif // CONFIG_HIGHBITDEPTH + + for (j = 0; j < kNumCoeffs; ++j) { + err_count += ref_s[j] != s[j]; + } + if (err_count && !err_count_total) { + first_failure = i; + } + err_count_total += err_count; + } + EXPECT_EQ(0, err_count_total) + << "Error: Loop8Test6Param, C output doesn't match SSE2 " + "loopfilter output. " + << "First failed at test case " << first_failure; +} + +TEST_P(Loop8Test6Param, ValueCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = number_of_iterations; +#if CONFIG_HIGHBITDEPTH + const int32_t bd = bit_depth_; + DECLARE_ALIGNED(16, uint16_t, s[kNumCoeffs]); + DECLARE_ALIGNED(16, uint16_t, ref_s[kNumCoeffs]); +#else + DECLARE_ALIGNED(8, uint8_t, s[kNumCoeffs]); + DECLARE_ALIGNED(8, uint8_t, ref_s[kNumCoeffs]); +#endif // CONFIG_HIGHBITDEPTH + int err_count_total = 0; + int first_failure = -1; + + // NOTE: The code in av1_loopfilter.c:update_sharpness computes mblim as a + // function of sharpness_lvl and the loopfilter lvl as: + // block_inside_limit = lvl >> ((sharpness_lvl > 0) + (sharpness_lvl > 4)); + // ... + // memset(lfi->lfthr[lvl].mblim, (2 * (lvl + 2) + block_inside_limit), + // SIMD_WIDTH); + // This means that the largest value for mblim will occur when sharpness_lvl + // is equal to 0, and lvl is equal to its greatest value (MAX_LOOP_FILTER). + // In this case block_inside_limit will be equal to MAX_LOOP_FILTER and + // therefore mblim will be equal to (2 * (lvl + 2) + block_inside_limit) = + // 2 * (MAX_LOOP_FILTER + 2) + MAX_LOOP_FILTER = 3 * MAX_LOOP_FILTER + 4 + + for (int i = 0; i < count_test_block; ++i) { + int err_count = 0; + uint8_t tmp = static_cast(rnd(3 * MAX_LOOP_FILTER + 4)); + DECLARE_ALIGNED(16, const uint8_t, + blimit[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = static_cast(rnd(MAX_LOOP_FILTER)); + DECLARE_ALIGNED(16, const uint8_t, + limit[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, + thresh[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + int32_t p = kNumCoeffs / 32; + for (int j = 0; j < kNumCoeffs; ++j) { + s[j] = rnd.Rand16() & mask_; + ref_s[j] = s[j]; + } +#if CONFIG_HIGHBITDEPTH + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit, limit, thresh, bd); + ASM_REGISTER_STATE_CHECK( + loopfilter_op_(s + 8 + p * 8, p, blimit, limit, thresh, bd)); +#else + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit, limit, thresh); + ASM_REGISTER_STATE_CHECK( + loopfilter_op_(s + 8 + p * 8, p, blimit, limit, thresh)); +#endif // CONFIG_HIGHBITDEPTH + for (int j = 0; j < kNumCoeffs; ++j) { + err_count += ref_s[j] != s[j]; + } + if (err_count && !err_count_total) { + first_failure = i; + } + err_count_total += err_count; + } + EXPECT_EQ(0, err_count_total) + << "Error: Loop8Test6Param, C output doesn't match SSE2 " + "loopfilter output. " + << "First failed at test case " << first_failure; +} + +TEST_P(Loop8Test9Param, OperationCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = number_of_iterations; +#if CONFIG_HIGHBITDEPTH + const int32_t bd = bit_depth_; + DECLARE_ALIGNED(16, uint16_t, s[kNumCoeffs]); + DECLARE_ALIGNED(16, uint16_t, ref_s[kNumCoeffs]); +#else + DECLARE_ALIGNED(8, uint8_t, s[kNumCoeffs]); + DECLARE_ALIGNED(8, uint8_t, ref_s[kNumCoeffs]); +#endif // CONFIG_HIGHBITDEPTH + int err_count_total = 0; + int first_failure = -1; + for (int i = 0; i < count_test_block; ++i) { + int err_count = 0; + uint8_t tmp = static_cast(rnd(3 * MAX_LOOP_FILTER + 4)); + DECLARE_ALIGNED(16, const uint8_t, + blimit0[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = static_cast(rnd(MAX_LOOP_FILTER)); + DECLARE_ALIGNED(16, const uint8_t, + limit0[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, + thresh0[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = static_cast(rnd(3 * MAX_LOOP_FILTER + 4)); + DECLARE_ALIGNED(16, const uint8_t, + blimit1[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = static_cast(rnd(MAX_LOOP_FILTER)); + DECLARE_ALIGNED(16, const uint8_t, + limit1[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, + thresh1[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + int32_t p = kNumCoeffs / 32; + uint16_t tmp_s[kNumCoeffs]; + int j = 0; + const uint8_t limit = *limit0 < *limit1 ? *limit0 : *limit1; + while (j < kNumCoeffs) { + uint8_t val = rnd.Rand8(); + if (val & 0x80) { // 50% chance to choose a new value. + tmp_s[j] = rnd.Rand16(); + j++; + } else { // 50% chance to repeat previous value in row X times. + int k = 0; + while (k++ < ((val & 0x1f) + 1) && j < kNumCoeffs) { + if (j < 1) { + tmp_s[j] = rnd.Rand16(); + } else if (val & 0x20) { // Increment by a value within the limit. + tmp_s[j] = (tmp_s[j - 1] + (limit - 1)); + } else { // Decrement by an value within the limit. + tmp_s[j] = (tmp_s[j - 1] - (limit - 1)); + } + j++; + } + } + } + for (j = 0; j < kNumCoeffs; j++) { + if (i % 2) { + s[j] = tmp_s[j] & mask_; + } else { + s[j] = tmp_s[p * (j % p) + j / p] & mask_; + } + ref_s[j] = s[j]; + } +#if CONFIG_HIGHBITDEPTH + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit0, limit0, thresh0, blimit1, + limit1, thresh1, bd); + ASM_REGISTER_STATE_CHECK(loopfilter_op_(s + 8 + p * 8, p, blimit0, limit0, + thresh0, blimit1, limit1, thresh1, + bd)); +#else + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit0, limit0, thresh0, blimit1, + limit1, thresh1); + ASM_REGISTER_STATE_CHECK(loopfilter_op_(s + 8 + p * 8, p, blimit0, limit0, + thresh0, blimit1, limit1, thresh1)); +#endif // CONFIG_HIGHBITDEPTH + for (j = 0; j < kNumCoeffs; ++j) { + err_count += ref_s[j] != s[j]; + } + if (err_count && !err_count_total) { + first_failure = i; + } + err_count_total += err_count; + } + EXPECT_EQ(0, err_count_total) + << "Error: Loop8Test9Param, C output doesn't match SSE2 " + "loopfilter output. " + << "First failed at test case " << first_failure; +} + +TEST_P(Loop8Test9Param, ValueCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = number_of_iterations; +#if CONFIG_HIGHBITDEPTH + DECLARE_ALIGNED(16, uint16_t, s[kNumCoeffs]); + DECLARE_ALIGNED(16, uint16_t, ref_s[kNumCoeffs]); +#else + DECLARE_ALIGNED(8, uint8_t, s[kNumCoeffs]); + DECLARE_ALIGNED(8, uint8_t, ref_s[kNumCoeffs]); +#endif // CONFIG_HIGHBITDEPTH + int err_count_total = 0; + int first_failure = -1; + for (int i = 0; i < count_test_block; ++i) { + int err_count = 0; + uint8_t tmp = static_cast(rnd(3 * MAX_LOOP_FILTER + 4)); + DECLARE_ALIGNED(16, const uint8_t, + blimit0[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = static_cast(rnd(MAX_LOOP_FILTER)); + DECLARE_ALIGNED(16, const uint8_t, + limit0[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, + thresh0[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = static_cast(rnd(3 * MAX_LOOP_FILTER + 4)); + DECLARE_ALIGNED(16, const uint8_t, + blimit1[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = static_cast(rnd(MAX_LOOP_FILTER)); + DECLARE_ALIGNED(16, const uint8_t, + limit1[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, + thresh1[16]) = { tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp }; + int32_t p = kNumCoeffs / 32; // TODO(pdlf) can we have non-square here? + for (int j = 0; j < kNumCoeffs; ++j) { + s[j] = rnd.Rand16() & mask_; + ref_s[j] = s[j]; + } +#if CONFIG_HIGHBITDEPTH + const int32_t bd = bit_depth_; + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit0, limit0, thresh0, blimit1, + limit1, thresh1, bd); + ASM_REGISTER_STATE_CHECK(loopfilter_op_(s + 8 + p * 8, p, blimit0, limit0, + thresh0, blimit1, limit1, thresh1, + bd)); +#else + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit0, limit0, thresh0, blimit1, + limit1, thresh1); + ASM_REGISTER_STATE_CHECK(loopfilter_op_(s + 8 + p * 8, p, blimit0, limit0, + thresh0, blimit1, limit1, thresh1)); +#endif // CONFIG_HIGHBITDEPTH + for (int j = 0; j < kNumCoeffs; ++j) { + err_count += ref_s[j] != s[j]; + } + if (err_count && !err_count_total) { + first_failure = i; + } + err_count_total += err_count; + } + EXPECT_EQ(0, err_count_total) + << "Error: Loop8Test9Param, C output doesn't match SSE2" + "loopfilter output. " + << "First failed at test case " << first_failure; +} + +using std::tr1::make_tuple; + +#if HAVE_SSE2 +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSE2, Loop8Test6Param, + ::testing::Values(make_tuple(&aom_highbd_lpf_horizontal_4_sse2, + &aom_highbd_lpf_horizontal_4_c, 8), + make_tuple(&aom_highbd_lpf_vertical_4_sse2, + &aom_highbd_lpf_vertical_4_c, 8), + make_tuple(&aom_highbd_lpf_horizontal_8_sse2, + &aom_highbd_lpf_horizontal_8_c, 8), + make_tuple(&aom_highbd_lpf_horizontal_edge_8_sse2, + &aom_highbd_lpf_horizontal_edge_8_c, 8), + make_tuple(&aom_highbd_lpf_horizontal_edge_16_sse2, + &aom_highbd_lpf_horizontal_edge_16_c, 8), + make_tuple(&aom_highbd_lpf_vertical_8_sse2, + &aom_highbd_lpf_vertical_8_c, 8), + make_tuple(&aom_highbd_lpf_vertical_16_sse2, + &aom_highbd_lpf_vertical_16_c, 8), + make_tuple(&aom_highbd_lpf_horizontal_4_sse2, + &aom_highbd_lpf_horizontal_4_c, 10), + make_tuple(&aom_highbd_lpf_vertical_4_sse2, + &aom_highbd_lpf_vertical_4_c, 10), + make_tuple(&aom_highbd_lpf_horizontal_8_sse2, + &aom_highbd_lpf_horizontal_8_c, 10), + make_tuple(&aom_highbd_lpf_horizontal_edge_8_sse2, + &aom_highbd_lpf_horizontal_edge_8_c, 10), + make_tuple(&aom_highbd_lpf_horizontal_edge_16_sse2, + &aom_highbd_lpf_horizontal_edge_16_c, 10), + make_tuple(&aom_highbd_lpf_vertical_8_sse2, + &aom_highbd_lpf_vertical_8_c, 10), + make_tuple(&aom_highbd_lpf_vertical_16_sse2, + &aom_highbd_lpf_vertical_16_c, 10), + make_tuple(&aom_highbd_lpf_horizontal_4_sse2, + &aom_highbd_lpf_horizontal_4_c, 12), + make_tuple(&aom_highbd_lpf_vertical_4_sse2, + &aom_highbd_lpf_vertical_4_c, 12), + make_tuple(&aom_highbd_lpf_horizontal_8_sse2, + &aom_highbd_lpf_horizontal_8_c, 12), + make_tuple(&aom_highbd_lpf_horizontal_edge_8_sse2, + &aom_highbd_lpf_horizontal_edge_8_c, 12), + make_tuple(&aom_highbd_lpf_horizontal_edge_16_sse2, + &aom_highbd_lpf_horizontal_edge_16_c, 12), + make_tuple(&aom_highbd_lpf_vertical_8_sse2, + &aom_highbd_lpf_vertical_8_c, 12), + make_tuple(&aom_highbd_lpf_vertical_16_sse2, + &aom_highbd_lpf_vertical_16_c, 12), + make_tuple(&aom_highbd_lpf_vertical_16_dual_sse2, + &aom_highbd_lpf_vertical_16_dual_c, 8), + make_tuple(&aom_highbd_lpf_vertical_16_dual_sse2, + &aom_highbd_lpf_vertical_16_dual_c, 10), + make_tuple(&aom_highbd_lpf_vertical_16_dual_sse2, + &aom_highbd_lpf_vertical_16_dual_c, 12))); +#else +INSTANTIATE_TEST_CASE_P( + SSE2, Loop8Test6Param, + ::testing::Values( + make_tuple(&aom_lpf_horizontal_4_sse2, &aom_lpf_horizontal_4_c, 8), + make_tuple(&aom_lpf_horizontal_8_sse2, &aom_lpf_horizontal_8_c, 8), + make_tuple(&aom_lpf_horizontal_edge_8_sse2, + &aom_lpf_horizontal_edge_8_c, 8), + make_tuple(&aom_lpf_horizontal_edge_16_sse2, + &aom_lpf_horizontal_edge_16_c, 8), + make_tuple(&aom_lpf_vertical_4_sse2, &aom_lpf_vertical_4_c, 8), + make_tuple(&aom_lpf_vertical_8_sse2, &aom_lpf_vertical_8_c, 8), + make_tuple(&aom_lpf_vertical_16_sse2, &aom_lpf_vertical_16_c, 8), + make_tuple(&aom_lpf_vertical_16_dual_sse2, &aom_lpf_vertical_16_dual_c, + 8))); +#endif // CONFIG_HIGHBITDEPTH +#endif + +#if HAVE_AVX2 && (!CONFIG_HIGHBITDEPTH) +INSTANTIATE_TEST_CASE_P( + AVX2, Loop8Test6Param, + ::testing::Values(make_tuple(&aom_lpf_horizontal_edge_8_avx2, + &aom_lpf_horizontal_edge_8_c, 8), + make_tuple(&aom_lpf_horizontal_edge_16_avx2, + &aom_lpf_horizontal_edge_16_c, 8))); +#endif + +#if HAVE_SSE2 +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSE2, Loop8Test9Param, + ::testing::Values(make_tuple(&aom_highbd_lpf_horizontal_4_dual_sse2, + &aom_highbd_lpf_horizontal_4_dual_c, 8), + make_tuple(&aom_highbd_lpf_horizontal_8_dual_sse2, + &aom_highbd_lpf_horizontal_8_dual_c, 8), + make_tuple(&aom_highbd_lpf_vertical_4_dual_sse2, + &aom_highbd_lpf_vertical_4_dual_c, 8), + make_tuple(&aom_highbd_lpf_vertical_8_dual_sse2, + &aom_highbd_lpf_vertical_8_dual_c, 8), + make_tuple(&aom_highbd_lpf_horizontal_4_dual_sse2, + &aom_highbd_lpf_horizontal_4_dual_c, 10), + make_tuple(&aom_highbd_lpf_horizontal_8_dual_sse2, + &aom_highbd_lpf_horizontal_8_dual_c, 10), + make_tuple(&aom_highbd_lpf_vertical_4_dual_sse2, + &aom_highbd_lpf_vertical_4_dual_c, 10), + make_tuple(&aom_highbd_lpf_vertical_8_dual_sse2, + &aom_highbd_lpf_vertical_8_dual_c, 10), + make_tuple(&aom_highbd_lpf_horizontal_4_dual_sse2, + &aom_highbd_lpf_horizontal_4_dual_c, 12), + make_tuple(&aom_highbd_lpf_horizontal_8_dual_sse2, + &aom_highbd_lpf_horizontal_8_dual_c, 12), + make_tuple(&aom_highbd_lpf_vertical_4_dual_sse2, + &aom_highbd_lpf_vertical_4_dual_c, 12), + make_tuple(&aom_highbd_lpf_vertical_8_dual_sse2, + &aom_highbd_lpf_vertical_8_dual_c, 12))); +#else +INSTANTIATE_TEST_CASE_P( + SSE2, Loop8Test9Param, + ::testing::Values(make_tuple(&aom_lpf_horizontal_4_dual_sse2, + &aom_lpf_horizontal_4_dual_c, 8), + make_tuple(&aom_lpf_horizontal_8_dual_sse2, + &aom_lpf_horizontal_8_dual_c, 8), + make_tuple(&aom_lpf_vertical_4_dual_sse2, + &aom_lpf_vertical_4_dual_c, 8), + make_tuple(&aom_lpf_vertical_8_dual_sse2, + &aom_lpf_vertical_8_dual_c, 8))); +#endif // CONFIG_HIGHBITDEPTH +#endif + +#if HAVE_NEON +#if CONFIG_HIGHBITDEPTH +// No neon high bitdepth functions. +#else +INSTANTIATE_TEST_CASE_P( + NEON, Loop8Test6Param, + ::testing::Values( +#if HAVE_NEON_ASM + // Using #if inside the macro is unsupported on MSVS but the tests are + // not + // currently built for MSVS with ARM and NEON. + make_tuple(&aom_lpf_horizontal_edge_8_neon, + &aom_lpf_horizontal_edge_8_c, 8), + make_tuple(&aom_lpf_horizontal_edge_16_neon, + &aom_lpf_horizontal_edge_16_c, 8), + make_tuple(&aom_lpf_vertical_16_neon, &aom_lpf_vertical_16_c, 8), + make_tuple(&aom_lpf_vertical_16_dual_neon, &aom_lpf_vertical_16_dual_c, + 8), +#endif // HAVE_NEON_ASM + make_tuple(&aom_lpf_horizontal_8_neon, &aom_lpf_horizontal_8_c, 8), + make_tuple(&aom_lpf_vertical_8_neon, &aom_lpf_vertical_8_c, 8), + make_tuple(&aom_lpf_horizontal_4_neon, &aom_lpf_horizontal_4_c, 8), + make_tuple(&aom_lpf_vertical_4_neon, &aom_lpf_vertical_4_c, 8))); +INSTANTIATE_TEST_CASE_P(NEON, Loop8Test9Param, + ::testing::Values( +#if HAVE_NEON_ASM + make_tuple(&aom_lpf_horizontal_8_dual_neon, + &aom_lpf_horizontal_8_dual_c, 8), + make_tuple(&aom_lpf_vertical_8_dual_neon, + &aom_lpf_vertical_8_dual_c, 8), +#endif // HAVE_NEON_ASM + make_tuple(&aom_lpf_horizontal_4_dual_neon, + &aom_lpf_horizontal_4_dual_c, 8), + make_tuple(&aom_lpf_vertical_4_dual_neon, + &aom_lpf_vertical_4_dual_c, 8))); +#endif // CONFIG_HIGHBITDEPTH +#endif // HAVE_NEON + +#if HAVE_DSPR2 && !CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + DSPR2, Loop8Test6Param, + ::testing::Values( + make_tuple(&aom_lpf_horizontal_4_dspr2, &aom_lpf_horizontal_4_c, 8), + make_tuple(&aom_lpf_horizontal_8_dspr2, &aom_lpf_horizontal_8_c, 8), + make_tuple(&aom_lpf_horizontal_edge_8, &aom_lpf_horizontal_edge_8, 8), + make_tuple(&aom_lpf_horizontal_edge_16, &aom_lpf_horizontal_edge_16, 8), + make_tuple(&aom_lpf_vertical_4_dspr2, &aom_lpf_vertical_4_c, 8), + make_tuple(&aom_lpf_vertical_8_dspr2, &aom_lpf_vertical_8_c, 8), + make_tuple(&aom_lpf_vertical_16_dspr2, &aom_lpf_vertical_16_c, 8), + make_tuple(&aom_lpf_vertical_16_dual_dspr2, &aom_lpf_vertical_16_dual_c, + 8))); + +INSTANTIATE_TEST_CASE_P( + DSPR2, Loop8Test9Param, + ::testing::Values(make_tuple(&aom_lpf_horizontal_4_dual_dspr2, + &aom_lpf_horizontal_4_dual_c, 8), + make_tuple(&aom_lpf_horizontal_8_dual_dspr2, + &aom_lpf_horizontal_8_dual_c, 8), + make_tuple(&aom_lpf_vertical_4_dual_dspr2, + &aom_lpf_vertical_4_dual_c, 8), + make_tuple(&aom_lpf_vertical_8_dual_dspr2, + &aom_lpf_vertical_8_dual_c, 8))); +#endif // HAVE_DSPR2 && !CONFIG_HIGHBITDEPTH + +#if HAVE_MSA && (!CONFIG_HIGHBITDEPTH) +INSTANTIATE_TEST_CASE_P( + MSA, Loop8Test6Param, + ::testing::Values( + make_tuple(&aom_lpf_horizontal_4_msa, &aom_lpf_horizontal_4_c, 8), + make_tuple(&aom_lpf_horizontal_8_msa, &aom_lpf_horizontal_8_c, 8), + make_tuple(&aom_lpf_horizontal_edge_8_msa, &aom_lpf_horizontal_edge_8_c, + 8), + make_tuple(&aom_lpf_horizontal_edge_16_msa, + &aom_lpf_horizontal_edge_16_c, 8), + make_tuple(&aom_lpf_vertical_4_msa, &aom_lpf_vertical_4_c, 8), + make_tuple(&aom_lpf_vertical_8_msa, &aom_lpf_vertical_8_c, 8), + make_tuple(&aom_lpf_vertical_16_msa, &aom_lpf_vertical_16_c, 8))); + +INSTANTIATE_TEST_CASE_P( + MSA, Loop8Test9Param, + ::testing::Values(make_tuple(&aom_lpf_horizontal_4_dual_msa, + &aom_lpf_horizontal_4_dual_c, 8), + make_tuple(&aom_lpf_horizontal_8_dual_msa, + &aom_lpf_horizontal_8_dual_c, 8), + make_tuple(&aom_lpf_vertical_4_dual_msa, + &aom_lpf_vertical_4_dual_c, 8), + make_tuple(&aom_lpf_vertical_8_dual_msa, + &aom_lpf_vertical_8_dual_c, 8))); +#endif // HAVE_MSA && (!CONFIG_HIGHBITDEPTH) + +} // namespace diff --git a/third_party/aom/test/masked_sad_test.cc b/third_party/aom/test/masked_sad_test.cc new file mode 100644 index 0000000000..53f85eef7d --- /dev/null +++ b/third_party/aom/test/masked_sad_test.cc @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +using libaom_test::ACMRandom; + +namespace { +const int number_of_iterations = 500; + +typedef unsigned int (*MaskedSADFunc)(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + const uint8_t *m, int m_stride); +typedef std::tr1::tuple MaskedSADParam; + +class MaskedSADTest : public ::testing::TestWithParam { + public: + virtual ~MaskedSADTest() {} + virtual void SetUp() { + maskedSAD_op_ = GET_PARAM(0); + ref_maskedSAD_op_ = GET_PARAM(1); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + MaskedSADFunc maskedSAD_op_; + MaskedSADFunc ref_maskedSAD_op_; +}; + +TEST_P(MaskedSADTest, OperationCheck) { + unsigned int ref_ret, ret; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint8_t, src_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint8_t, ref_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint8_t, msk_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + int err_count = 0; + int first_failure = -1; + int src_stride = MAX_SB_SIZE; + int ref_stride = MAX_SB_SIZE; + int msk_stride = MAX_SB_SIZE; + for (int i = 0; i < number_of_iterations; ++i) { + for (int j = 0; j < MAX_SB_SIZE * MAX_SB_SIZE; j++) { + src_ptr[j] = rnd.Rand8(); + ref_ptr[j] = rnd.Rand8(); + msk_ptr[j] = ((rnd.Rand8() & 0x7f) > 64) ? rnd.Rand8() & 0x3f : 64; + assert(msk_ptr[j] <= 64); + } + + ref_ret = ref_maskedSAD_op_(src_ptr, src_stride, ref_ptr, ref_stride, + msk_ptr, msk_stride); + ASM_REGISTER_STATE_CHECK(ret = maskedSAD_op_(src_ptr, src_stride, ref_ptr, + ref_stride, msk_ptr, + msk_stride)); + if (ret != ref_ret) { + err_count++; + if (first_failure == -1) first_failure = i; + } + } + EXPECT_EQ(0, err_count) + << "Error: Masked SAD Test, C output doesn't match SSSE3 output. " + << "First failed at test case " << first_failure; +} + +#if CONFIG_HIGHBITDEPTH +typedef unsigned int (*HighbdMaskedSADFunc)(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + const uint8_t *m, int m_stride); +typedef std::tr1::tuple + HighbdMaskedSADParam; + +class HighbdMaskedSADTest + : public ::testing::TestWithParam { + public: + virtual ~HighbdMaskedSADTest() {} + virtual void SetUp() { + maskedSAD_op_ = GET_PARAM(0); + ref_maskedSAD_op_ = GET_PARAM(1); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + HighbdMaskedSADFunc maskedSAD_op_; + HighbdMaskedSADFunc ref_maskedSAD_op_; +}; + +TEST_P(HighbdMaskedSADTest, OperationCheck) { + unsigned int ref_ret, ret; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint16_t, src_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint16_t, ref_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint8_t, msk_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + uint8_t *src8_ptr = CONVERT_TO_BYTEPTR(src_ptr); + uint8_t *ref8_ptr = CONVERT_TO_BYTEPTR(ref_ptr); + int err_count = 0; + int first_failure = -1; + int src_stride = MAX_SB_SIZE; + int ref_stride = MAX_SB_SIZE; + int msk_stride = MAX_SB_SIZE; + for (int i = 0; i < number_of_iterations; ++i) { + for (int j = 0; j < MAX_SB_SIZE * MAX_SB_SIZE; j++) { + src_ptr[j] = rnd.Rand16() & 0xfff; + ref_ptr[j] = rnd.Rand16() & 0xfff; + msk_ptr[j] = ((rnd.Rand8() & 0x7f) > 64) ? rnd.Rand8() & 0x3f : 64; + } + + ref_ret = ref_maskedSAD_op_(src8_ptr, src_stride, ref8_ptr, ref_stride, + msk_ptr, msk_stride); + ASM_REGISTER_STATE_CHECK(ret = maskedSAD_op_(src8_ptr, src_stride, ref8_ptr, + ref_stride, msk_ptr, + msk_stride)); + if (ret != ref_ret) { + err_count++; + if (first_failure == -1) first_failure = i; + } + } + EXPECT_EQ(0, err_count) + << "Error: High BD Masked SAD Test, C output doesn't match SSSE3 output. " + << "First failed at test case " << first_failure; +} +#endif // CONFIG_HIGHBITDEPTH + +using std::tr1::make_tuple; + +#if HAVE_SSSE3 +INSTANTIATE_TEST_CASE_P( + SSSE3_C_COMPARE, MaskedSADTest, + ::testing::Values( +#if CONFIG_EXT_PARTITION + make_tuple(&aom_masked_sad128x128_ssse3, &aom_masked_sad128x128_c), + make_tuple(&aom_masked_sad128x64_ssse3, &aom_masked_sad128x64_c), + make_tuple(&aom_masked_sad64x128_ssse3, &aom_masked_sad64x128_c), +#endif // CONFIG_EXT_PARTITION + make_tuple(&aom_masked_sad64x64_ssse3, &aom_masked_sad64x64_c), + make_tuple(&aom_masked_sad64x32_ssse3, &aom_masked_sad64x32_c), + make_tuple(&aom_masked_sad32x64_ssse3, &aom_masked_sad32x64_c), + make_tuple(&aom_masked_sad32x32_ssse3, &aom_masked_sad32x32_c), + make_tuple(&aom_masked_sad32x16_ssse3, &aom_masked_sad32x16_c), + make_tuple(&aom_masked_sad16x32_ssse3, &aom_masked_sad16x32_c), + make_tuple(&aom_masked_sad16x16_ssse3, &aom_masked_sad16x16_c), + make_tuple(&aom_masked_sad16x8_ssse3, &aom_masked_sad16x8_c), + make_tuple(&aom_masked_sad8x16_ssse3, &aom_masked_sad8x16_c), + make_tuple(&aom_masked_sad8x8_ssse3, &aom_masked_sad8x8_c), + make_tuple(&aom_masked_sad8x4_ssse3, &aom_masked_sad8x4_c), + make_tuple(&aom_masked_sad4x8_ssse3, &aom_masked_sad4x8_c), + make_tuple(&aom_masked_sad4x4_ssse3, &aom_masked_sad4x4_c))); +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(SSSE3_C_COMPARE, HighbdMaskedSADTest, + ::testing::Values( +#if CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_masked_sad128x128_ssse3, + &aom_highbd_masked_sad128x128_c), + make_tuple(&aom_highbd_masked_sad128x64_ssse3, + &aom_highbd_masked_sad128x64_c), + make_tuple(&aom_highbd_masked_sad64x128_ssse3, + &aom_highbd_masked_sad64x128_c), +#endif // CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_masked_sad64x64_ssse3, + &aom_highbd_masked_sad64x64_c), + make_tuple(&aom_highbd_masked_sad64x32_ssse3, + &aom_highbd_masked_sad64x32_c), + make_tuple(&aom_highbd_masked_sad32x64_ssse3, + &aom_highbd_masked_sad32x64_c), + make_tuple(&aom_highbd_masked_sad32x32_ssse3, + &aom_highbd_masked_sad32x32_c), + make_tuple(&aom_highbd_masked_sad32x16_ssse3, + &aom_highbd_masked_sad32x16_c), + make_tuple(&aom_highbd_masked_sad16x32_ssse3, + &aom_highbd_masked_sad16x32_c), + make_tuple(&aom_highbd_masked_sad16x16_ssse3, + &aom_highbd_masked_sad16x16_c), + make_tuple(&aom_highbd_masked_sad16x8_ssse3, + &aom_highbd_masked_sad16x8_c), + make_tuple(&aom_highbd_masked_sad8x16_ssse3, + &aom_highbd_masked_sad8x16_c), + make_tuple(&aom_highbd_masked_sad8x8_ssse3, + &aom_highbd_masked_sad8x8_c), + make_tuple(&aom_highbd_masked_sad8x4_ssse3, + &aom_highbd_masked_sad8x4_c), + make_tuple(&aom_highbd_masked_sad4x8_ssse3, + &aom_highbd_masked_sad4x8_c), + make_tuple(&aom_highbd_masked_sad4x4_ssse3, + &aom_highbd_masked_sad4x4_c))); +#endif // CONFIG_HIGHBITDEPTH +#endif // HAVE_SSSE3 +} // namespace diff --git a/third_party/aom/test/masked_variance_test.cc b/third_party/aom/test/masked_variance_test.cc new file mode 100644 index 0000000000..65e852aeab --- /dev/null +++ b/third_party/aom/test/masked_variance_test.cc @@ -0,0 +1,790 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_codec.h" +#include "aom/aom_integer.h" +#include "aom_dsp/aom_filter.h" +#include "aom_mem/aom_mem.h" + +using libaom_test::ACMRandom; + +namespace { +const int number_of_iterations = 500; + +typedef unsigned int (*MaskedVarianceFunc)(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + const uint8_t *m, int m_stride, + unsigned int *sse); + +typedef std::tr1::tuple + MaskedVarianceParam; + +class MaskedVarianceTest + : public ::testing::TestWithParam { + public: + virtual ~MaskedVarianceTest() {} + virtual void SetUp() { + opt_func_ = GET_PARAM(0); + ref_func_ = GET_PARAM(1); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + MaskedVarianceFunc opt_func_; + MaskedVarianceFunc ref_func_; +}; + +TEST_P(MaskedVarianceTest, OperationCheck) { + unsigned int ref_ret, opt_ret; + unsigned int ref_sse, opt_sse; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint8_t, src_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint8_t, ref_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint8_t, msk_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + int err_count = 0; + int first_failure = -1; + int src_stride = MAX_SB_SIZE; + int ref_stride = MAX_SB_SIZE; + int msk_stride = MAX_SB_SIZE; + + for (int i = 0; i < number_of_iterations; ++i) { + for (int j = 0; j < MAX_SB_SIZE * MAX_SB_SIZE; j++) { + src_ptr[j] = rnd.Rand8(); + ref_ptr[j] = rnd.Rand8(); + msk_ptr[j] = rnd(65); + } + + ref_ret = ref_func_(src_ptr, src_stride, ref_ptr, ref_stride, msk_ptr, + msk_stride, &ref_sse); + ASM_REGISTER_STATE_CHECK(opt_ret = opt_func_(src_ptr, src_stride, ref_ptr, + ref_stride, msk_ptr, + msk_stride, &opt_sse)); + + if (opt_ret != ref_ret || opt_sse != ref_sse) { + err_count++; + if (first_failure == -1) first_failure = i; + } + } + + EXPECT_EQ(0, err_count) << "Error: Masked Variance Test OperationCheck," + << "C output doesn't match SSSE3 output. " + << "First failed at test case " << first_failure; +} + +TEST_P(MaskedVarianceTest, ExtremeValues) { + unsigned int ref_ret, opt_ret; + unsigned int ref_sse, opt_sse; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint8_t, src_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint8_t, ref_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint8_t, msk_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + int err_count = 0; + int first_failure = -1; + int src_stride = MAX_SB_SIZE; + int ref_stride = MAX_SB_SIZE; + int msk_stride = MAX_SB_SIZE; + + for (int i = 0; i < 8; ++i) { + memset(src_ptr, (i & 0x1) ? 255 : 0, MAX_SB_SIZE * MAX_SB_SIZE); + memset(ref_ptr, (i & 0x2) ? 255 : 0, MAX_SB_SIZE * MAX_SB_SIZE); + memset(msk_ptr, (i & 0x4) ? 64 : 0, MAX_SB_SIZE * MAX_SB_SIZE); + + ref_ret = ref_func_(src_ptr, src_stride, ref_ptr, ref_stride, msk_ptr, + msk_stride, &ref_sse); + ASM_REGISTER_STATE_CHECK(opt_ret = opt_func_(src_ptr, src_stride, ref_ptr, + ref_stride, msk_ptr, + msk_stride, &opt_sse)); + + if (opt_ret != ref_ret || opt_sse != ref_sse) { + err_count++; + if (first_failure == -1) first_failure = i; + } + } + + EXPECT_EQ(0, err_count) << "Error: Masked Variance Test ExtremeValues," + << "C output doesn't match SSSE3 output. " + << "First failed at test case " << first_failure; +} + +typedef unsigned int (*MaskedSubPixelVarianceFunc)( + const uint8_t *a, int a_stride, int xoffset, int yoffset, const uint8_t *b, + int b_stride, const uint8_t *m, int m_stride, unsigned int *sse); + +typedef std::tr1::tuple + MaskedSubPixelVarianceParam; + +class MaskedSubPixelVarianceTest + : public ::testing::TestWithParam { + public: + virtual ~MaskedSubPixelVarianceTest() {} + virtual void SetUp() { + opt_func_ = GET_PARAM(0); + ref_func_ = GET_PARAM(1); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + MaskedSubPixelVarianceFunc opt_func_; + MaskedSubPixelVarianceFunc ref_func_; +}; + +TEST_P(MaskedSubPixelVarianceTest, OperationCheck) { + unsigned int ref_ret, opt_ret; + unsigned int ref_sse, opt_sse; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint8_t, src_ptr[(MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)]); + DECLARE_ALIGNED(16, uint8_t, ref_ptr[(MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)]); + DECLARE_ALIGNED(16, uint8_t, msk_ptr[(MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)]); + int err_count = 0; + int first_failure = -1; + int src_stride = (MAX_SB_SIZE + 1); + int ref_stride = (MAX_SB_SIZE + 1); + int msk_stride = (MAX_SB_SIZE + 1); + int xoffset; + int yoffset; + + for (int i = 0; i < number_of_iterations; ++i) { + int xoffsets[] = { 0, 4, rnd(BIL_SUBPEL_SHIFTS) }; + int yoffsets[] = { 0, 4, rnd(BIL_SUBPEL_SHIFTS) }; + for (int j = 0; j < (MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1); j++) { + src_ptr[j] = rnd.Rand8(); + ref_ptr[j] = rnd.Rand8(); + msk_ptr[j] = rnd(65); + } + for (int k = 0; k < 3; k++) { + xoffset = xoffsets[k]; + for (int l = 0; l < 3; l++) { + xoffset = xoffsets[k]; + yoffset = yoffsets[l]; + + ref_ret = ref_func_(src_ptr, src_stride, xoffset, yoffset, ref_ptr, + ref_stride, msk_ptr, msk_stride, &ref_sse); + ASM_REGISTER_STATE_CHECK( + opt_ret = opt_func_(src_ptr, src_stride, xoffset, yoffset, ref_ptr, + ref_stride, msk_ptr, msk_stride, &opt_sse)); + + if (opt_ret != ref_ret || opt_sse != ref_sse) { + err_count++; + if (first_failure == -1) first_failure = i; + } + } + } + } + + EXPECT_EQ(0, err_count) + << "Error: Masked Sub Pixel Variance Test OperationCheck," + << "C output doesn't match SSSE3 output. " + << "First failed at test case " << first_failure; +} + +TEST_P(MaskedSubPixelVarianceTest, ExtremeValues) { + unsigned int ref_ret, opt_ret; + unsigned int ref_sse, opt_sse; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint8_t, src_ptr[(MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)]); + DECLARE_ALIGNED(16, uint8_t, ref_ptr[(MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)]); + DECLARE_ALIGNED(16, uint8_t, msk_ptr[(MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)]); + int first_failure_x = -1; + int first_failure_y = -1; + int err_count = 0; + int first_failure = -1; + int src_stride = (MAX_SB_SIZE + 1); + int ref_stride = (MAX_SB_SIZE + 1); + int msk_stride = (MAX_SB_SIZE + 1); + + for (int xoffset = 0; xoffset < BIL_SUBPEL_SHIFTS; xoffset++) { + for (int yoffset = 0; yoffset < BIL_SUBPEL_SHIFTS; yoffset++) { + for (int i = 0; i < 8; ++i) { + memset(src_ptr, (i & 0x1) ? 255 : 0, + (MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)); + memset(ref_ptr, (i & 0x2) ? 255 : 0, + (MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)); + memset(msk_ptr, (i & 0x4) ? 64 : 0, + (MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)); + + ref_ret = ref_func_(src_ptr, src_stride, xoffset, yoffset, ref_ptr, + ref_stride, msk_ptr, msk_stride, &ref_sse); + ASM_REGISTER_STATE_CHECK( + opt_ret = opt_func_(src_ptr, src_stride, xoffset, yoffset, ref_ptr, + ref_stride, msk_ptr, msk_stride, &opt_sse)); + + if (opt_ret != ref_ret || opt_sse != ref_sse) { + err_count++; + if (first_failure == -1) { + first_failure = i; + first_failure_x = xoffset; + first_failure_y = yoffset; + } + } + } + } + } + + EXPECT_EQ(0, err_count) << "Error: Masked Variance Test ExtremeValues," + << "C output doesn't match SSSE3 output. " + << "First failed at test case " << first_failure + << " x_offset = " << first_failure_x + << " y_offset = " << first_failure_y; +} + +#if CONFIG_HIGHBITDEPTH +typedef std::tr1::tuple + HighbdMaskedVarianceParam; + +class HighbdMaskedVarianceTest + : public ::testing::TestWithParam { + public: + virtual ~HighbdMaskedVarianceTest() {} + virtual void SetUp() { + opt_func_ = GET_PARAM(0); + ref_func_ = GET_PARAM(1); + bit_depth_ = GET_PARAM(2); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + MaskedVarianceFunc opt_func_; + MaskedVarianceFunc ref_func_; + aom_bit_depth_t bit_depth_; +}; + +TEST_P(HighbdMaskedVarianceTest, OperationCheck) { + unsigned int ref_ret, opt_ret; + unsigned int ref_sse, opt_sse; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint16_t, src_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint16_t, ref_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint8_t, msk_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + uint8_t *src8_ptr = CONVERT_TO_BYTEPTR(src_ptr); + uint8_t *ref8_ptr = CONVERT_TO_BYTEPTR(ref_ptr); + int err_count = 0; + int first_failure = -1; + int src_stride = MAX_SB_SIZE; + int ref_stride = MAX_SB_SIZE; + int msk_stride = MAX_SB_SIZE; + + for (int i = 0; i < number_of_iterations; ++i) { + for (int j = 0; j < MAX_SB_SIZE * MAX_SB_SIZE; j++) { + src_ptr[j] = rnd.Rand16() & ((1 << bit_depth_) - 1); + ref_ptr[j] = rnd.Rand16() & ((1 << bit_depth_) - 1); + msk_ptr[j] = rnd(65); + } + + ref_ret = ref_func_(src8_ptr, src_stride, ref8_ptr, ref_stride, msk_ptr, + msk_stride, &ref_sse); + ASM_REGISTER_STATE_CHECK(opt_ret = opt_func_(src8_ptr, src_stride, ref8_ptr, + ref_stride, msk_ptr, + msk_stride, &opt_sse)); + + if (opt_ret != ref_ret || opt_sse != ref_sse) { + err_count++; + if (first_failure == -1) first_failure = i; + } + } + + EXPECT_EQ(0, err_count) << "Error: Masked Variance Test OperationCheck," + << "C output doesn't match SSSE3 output. " + << "First failed at test case " << first_failure; +} + +TEST_P(HighbdMaskedVarianceTest, ExtremeValues) { + unsigned int ref_ret, opt_ret; + unsigned int ref_sse, opt_sse; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint16_t, src_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint16_t, ref_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint8_t, msk_ptr[MAX_SB_SIZE * MAX_SB_SIZE]); + uint8_t *src8_ptr = CONVERT_TO_BYTEPTR(src_ptr); + uint8_t *ref8_ptr = CONVERT_TO_BYTEPTR(ref_ptr); + int err_count = 0; + int first_failure = -1; + int src_stride = MAX_SB_SIZE; + int ref_stride = MAX_SB_SIZE; + int msk_stride = MAX_SB_SIZE; + + for (int i = 0; i < 8; ++i) { + aom_memset16(src_ptr, (i & 0x1) ? ((1 << bit_depth_) - 1) : 0, + MAX_SB_SIZE * MAX_SB_SIZE); + aom_memset16(ref_ptr, (i & 0x2) ? ((1 << bit_depth_) - 1) : 0, + MAX_SB_SIZE * MAX_SB_SIZE); + memset(msk_ptr, (i & 0x4) ? 64 : 0, MAX_SB_SIZE * MAX_SB_SIZE); + + ref_ret = ref_func_(src8_ptr, src_stride, ref8_ptr, ref_stride, msk_ptr, + msk_stride, &ref_sse); + ASM_REGISTER_STATE_CHECK(opt_ret = opt_func_(src8_ptr, src_stride, ref8_ptr, + ref_stride, msk_ptr, + msk_stride, &opt_sse)); + + if (opt_ret != ref_ret || opt_sse != ref_sse) { + err_count++; + if (first_failure == -1) first_failure = i; + } + } + + EXPECT_EQ(0, err_count) << "Error: Masked Variance Test ExtremeValues," + << "C output doesn't match SSSE3 output. " + << "First failed at test case " << first_failure; +} + +typedef std::tr1::tuple + HighbdMaskedSubPixelVarianceParam; + +class HighbdMaskedSubPixelVarianceTest + : public ::testing::TestWithParam { + public: + virtual ~HighbdMaskedSubPixelVarianceTest() {} + virtual void SetUp() { + opt_func_ = GET_PARAM(0); + ref_func_ = GET_PARAM(1); + bit_depth_ = GET_PARAM(2); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + MaskedSubPixelVarianceFunc opt_func_; + MaskedSubPixelVarianceFunc ref_func_; + aom_bit_depth_t bit_depth_; +}; + +TEST_P(HighbdMaskedSubPixelVarianceTest, OperationCheck) { + unsigned int ref_ret, opt_ret; + unsigned int ref_sse, opt_sse; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint16_t, src_ptr[(MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)]); + DECLARE_ALIGNED(16, uint16_t, ref_ptr[(MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)]); + DECLARE_ALIGNED(16, uint8_t, msk_ptr[(MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)]); + uint8_t *src8_ptr = CONVERT_TO_BYTEPTR(src_ptr); + uint8_t *ref8_ptr = CONVERT_TO_BYTEPTR(ref_ptr); + int err_count = 0; + int first_failure = -1; + int first_failure_x = -1; + int first_failure_y = -1; + int src_stride = (MAX_SB_SIZE + 1); + int ref_stride = (MAX_SB_SIZE + 1); + int msk_stride = (MAX_SB_SIZE + 1); + int xoffset, yoffset; + + for (int i = 0; i < number_of_iterations; ++i) { + for (xoffset = 0; xoffset < BIL_SUBPEL_SHIFTS; xoffset++) { + for (yoffset = 0; yoffset < BIL_SUBPEL_SHIFTS; yoffset++) { + for (int j = 0; j < (MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1); j++) { + src_ptr[j] = rnd.Rand16() & ((1 << bit_depth_) - 1); + ref_ptr[j] = rnd.Rand16() & ((1 << bit_depth_) - 1); + msk_ptr[j] = rnd(65); + } + + ref_ret = ref_func_(src8_ptr, src_stride, xoffset, yoffset, ref8_ptr, + ref_stride, msk_ptr, msk_stride, &ref_sse); + ASM_REGISTER_STATE_CHECK(opt_ret = + opt_func_(src8_ptr, src_stride, xoffset, + yoffset, ref8_ptr, ref_stride, + msk_ptr, msk_stride, &opt_sse)); + + if (opt_ret != ref_ret || opt_sse != ref_sse) { + err_count++; + if (first_failure == -1) { + first_failure = i; + first_failure_x = xoffset; + first_failure_y = yoffset; + } + } + } + } + } + + EXPECT_EQ(0, err_count) + << "Error: Masked Sub Pixel Variance Test OperationCheck," + << "C output doesn't match SSSE3 output. " + << "First failed at test case " << first_failure + << " x_offset = " << first_failure_x << " y_offset = " << first_failure_y; +} + +TEST_P(HighbdMaskedSubPixelVarianceTest, ExtremeValues) { + unsigned int ref_ret, opt_ret; + unsigned int ref_sse, opt_sse; + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, uint16_t, src_ptr[(MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)]); + DECLARE_ALIGNED(16, uint16_t, ref_ptr[(MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)]); + DECLARE_ALIGNED(16, uint8_t, msk_ptr[(MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)]); + uint8_t *src8_ptr = CONVERT_TO_BYTEPTR(src_ptr); + uint8_t *ref8_ptr = CONVERT_TO_BYTEPTR(ref_ptr); + int first_failure_x = -1; + int first_failure_y = -1; + int err_count = 0; + int first_failure = -1; + int src_stride = (MAX_SB_SIZE + 1); + int ref_stride = (MAX_SB_SIZE + 1); + int msk_stride = (MAX_SB_SIZE + 1); + + for (int xoffset = 0; xoffset < BIL_SUBPEL_SHIFTS; xoffset++) { + for (int yoffset = 0; yoffset < BIL_SUBPEL_SHIFTS; yoffset++) { + for (int i = 0; i < 8; ++i) { + aom_memset16(src_ptr, (i & 0x1) ? ((1 << bit_depth_) - 1) : 0, + (MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)); + aom_memset16(ref_ptr, (i & 0x2) ? ((1 << bit_depth_) - 1) : 0, + (MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)); + memset(msk_ptr, (i & 0x4) ? 64 : 0, + (MAX_SB_SIZE + 1) * (MAX_SB_SIZE + 1)); + + ref_ret = ref_func_(src8_ptr, src_stride, xoffset, yoffset, ref8_ptr, + ref_stride, msk_ptr, msk_stride, &ref_sse); + ASM_REGISTER_STATE_CHECK(opt_ret = + opt_func_(src8_ptr, src_stride, xoffset, + yoffset, ref8_ptr, ref_stride, + msk_ptr, msk_stride, &opt_sse)); + + if (opt_ret != ref_ret || opt_sse != ref_sse) { + err_count++; + if (first_failure == -1) { + first_failure = i; + first_failure_x = xoffset; + first_failure_y = yoffset; + } + } + } + } + } + + EXPECT_EQ(0, err_count) << "Error: Masked Variance Test ExtremeValues," + << "C output doesn't match SSSE3 output. " + << "First failed at test case " << first_failure + << " x_offset = " << first_failure_x + << " y_offset = " << first_failure_y; +} +#endif // CONFIG_HIGHBITDEPTH + +using std::tr1::make_tuple; + +#if HAVE_SSSE3 +INSTANTIATE_TEST_CASE_P( + SSSE3_C_COMPARE, MaskedVarianceTest, + ::testing::Values( +#if CONFIG_EXT_PARTITION + make_tuple(&aom_masked_variance128x128_ssse3, + &aom_masked_variance128x128_c), + make_tuple(&aom_masked_variance128x64_ssse3, + &aom_masked_variance128x64_c), + make_tuple(&aom_masked_variance64x128_ssse3, + &aom_masked_variance64x128_c), +#endif // CONFIG_EXT_PARTITION + make_tuple(&aom_masked_variance64x64_ssse3, + &aom_masked_variance64x64_c), + make_tuple(&aom_masked_variance64x32_ssse3, + &aom_masked_variance64x32_c), + make_tuple(&aom_masked_variance32x64_ssse3, + &aom_masked_variance32x64_c), + make_tuple(&aom_masked_variance32x32_ssse3, + &aom_masked_variance32x32_c), + make_tuple(&aom_masked_variance32x16_ssse3, + &aom_masked_variance32x16_c), + make_tuple(&aom_masked_variance16x32_ssse3, + &aom_masked_variance16x32_c), + make_tuple(&aom_masked_variance16x16_ssse3, + &aom_masked_variance16x16_c), + make_tuple(&aom_masked_variance16x8_ssse3, &aom_masked_variance16x8_c), + make_tuple(&aom_masked_variance8x16_ssse3, &aom_masked_variance8x16_c), + make_tuple(&aom_masked_variance8x8_ssse3, &aom_masked_variance8x8_c), + make_tuple(&aom_masked_variance8x4_ssse3, &aom_masked_variance8x4_c), + make_tuple(&aom_masked_variance4x8_ssse3, &aom_masked_variance4x8_c), + make_tuple(&aom_masked_variance4x4_ssse3, &aom_masked_variance4x4_c))); + +INSTANTIATE_TEST_CASE_P( + SSSE3_C_COMPARE, MaskedSubPixelVarianceTest, + ::testing::Values( +#if CONFIG_EXT_PARTITION + make_tuple(&aom_masked_sub_pixel_variance128x128_ssse3, + &aom_masked_sub_pixel_variance128x128_c), + make_tuple(&aom_masked_sub_pixel_variance128x64_ssse3, + &aom_masked_sub_pixel_variance128x64_c), + make_tuple(&aom_masked_sub_pixel_variance64x128_ssse3, + &aom_masked_sub_pixel_variance64x128_c), +#endif // CONFIG_EXT_PARTITION + make_tuple(&aom_masked_sub_pixel_variance64x64_ssse3, + &aom_masked_sub_pixel_variance64x64_c), + make_tuple(&aom_masked_sub_pixel_variance64x32_ssse3, + &aom_masked_sub_pixel_variance64x32_c), + make_tuple(&aom_masked_sub_pixel_variance32x64_ssse3, + &aom_masked_sub_pixel_variance32x64_c), + make_tuple(&aom_masked_sub_pixel_variance32x32_ssse3, + &aom_masked_sub_pixel_variance32x32_c), + make_tuple(&aom_masked_sub_pixel_variance32x16_ssse3, + &aom_masked_sub_pixel_variance32x16_c), + make_tuple(&aom_masked_sub_pixel_variance16x32_ssse3, + &aom_masked_sub_pixel_variance16x32_c), + make_tuple(&aom_masked_sub_pixel_variance16x16_ssse3, + &aom_masked_sub_pixel_variance16x16_c), + make_tuple(&aom_masked_sub_pixel_variance16x8_ssse3, + &aom_masked_sub_pixel_variance16x8_c), + make_tuple(&aom_masked_sub_pixel_variance8x16_ssse3, + &aom_masked_sub_pixel_variance8x16_c), + make_tuple(&aom_masked_sub_pixel_variance8x8_ssse3, + &aom_masked_sub_pixel_variance8x8_c), + make_tuple(&aom_masked_sub_pixel_variance8x4_ssse3, + &aom_masked_sub_pixel_variance8x4_c), + make_tuple(&aom_masked_sub_pixel_variance4x8_ssse3, + &aom_masked_sub_pixel_variance4x8_c), + make_tuple(&aom_masked_sub_pixel_variance4x4_ssse3, + &aom_masked_sub_pixel_variance4x4_c))); + +#if CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSSE3_C_COMPARE, HighbdMaskedVarianceTest, + ::testing::Values( +#if CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_masked_variance128x128_ssse3, + &aom_highbd_masked_variance128x128_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance128x64_ssse3, + &aom_highbd_masked_variance128x64_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance64x128_ssse3, + &aom_highbd_masked_variance64x128_c, AOM_BITS_8), +#endif // CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_masked_variance64x64_ssse3, + &aom_highbd_masked_variance64x64_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance64x32_ssse3, + &aom_highbd_masked_variance64x32_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance32x64_ssse3, + &aom_highbd_masked_variance32x64_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance32x32_ssse3, + &aom_highbd_masked_variance32x32_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance32x16_ssse3, + &aom_highbd_masked_variance32x16_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance16x32_ssse3, + &aom_highbd_masked_variance16x32_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance16x16_ssse3, + &aom_highbd_masked_variance16x16_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance16x8_ssse3, + &aom_highbd_masked_variance16x8_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance8x16_ssse3, + &aom_highbd_masked_variance8x16_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance8x8_ssse3, + &aom_highbd_masked_variance8x8_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance8x4_ssse3, + &aom_highbd_masked_variance8x4_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance4x8_ssse3, + &aom_highbd_masked_variance4x8_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_variance4x4_ssse3, + &aom_highbd_masked_variance4x4_c, AOM_BITS_8), +#if CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_10_masked_variance128x128_ssse3, + &aom_highbd_10_masked_variance128x128_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance128x64_ssse3, + &aom_highbd_10_masked_variance128x64_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance64x128_ssse3, + &aom_highbd_10_masked_variance64x128_c, AOM_BITS_10), +#endif // CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_10_masked_variance64x64_ssse3, + &aom_highbd_10_masked_variance64x64_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance64x32_ssse3, + &aom_highbd_10_masked_variance64x32_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance32x64_ssse3, + &aom_highbd_10_masked_variance32x64_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance32x32_ssse3, + &aom_highbd_10_masked_variance32x32_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance32x16_ssse3, + &aom_highbd_10_masked_variance32x16_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance16x32_ssse3, + &aom_highbd_10_masked_variance16x32_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance16x16_ssse3, + &aom_highbd_10_masked_variance16x16_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance16x8_ssse3, + &aom_highbd_10_masked_variance16x8_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance8x16_ssse3, + &aom_highbd_10_masked_variance8x16_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance8x8_ssse3, + &aom_highbd_10_masked_variance8x8_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance8x4_ssse3, + &aom_highbd_10_masked_variance8x4_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance4x8_ssse3, + &aom_highbd_10_masked_variance4x8_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_variance4x4_ssse3, + &aom_highbd_10_masked_variance4x4_c, AOM_BITS_10), +#if CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_12_masked_variance128x128_ssse3, + &aom_highbd_12_masked_variance128x128_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance128x64_ssse3, + &aom_highbd_12_masked_variance128x64_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance64x128_ssse3, + &aom_highbd_12_masked_variance64x128_c, AOM_BITS_12), +#endif // CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_12_masked_variance64x64_ssse3, + &aom_highbd_12_masked_variance64x64_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance64x32_ssse3, + &aom_highbd_12_masked_variance64x32_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance32x64_ssse3, + &aom_highbd_12_masked_variance32x64_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance32x32_ssse3, + &aom_highbd_12_masked_variance32x32_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance32x16_ssse3, + &aom_highbd_12_masked_variance32x16_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance16x32_ssse3, + &aom_highbd_12_masked_variance16x32_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance16x16_ssse3, + &aom_highbd_12_masked_variance16x16_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance16x8_ssse3, + &aom_highbd_12_masked_variance16x8_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance8x16_ssse3, + &aom_highbd_12_masked_variance8x16_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance8x8_ssse3, + &aom_highbd_12_masked_variance8x8_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance8x4_ssse3, + &aom_highbd_12_masked_variance8x4_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance4x8_ssse3, + &aom_highbd_12_masked_variance4x8_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_variance4x4_ssse3, + &aom_highbd_12_masked_variance4x4_c, AOM_BITS_12))); + +INSTANTIATE_TEST_CASE_P( + SSSE3_C_COMPARE, HighbdMaskedSubPixelVarianceTest, + ::testing::Values( +#if CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_masked_sub_pixel_variance128x128_ssse3, + &aom_highbd_masked_sub_pixel_variance128x128_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance128x64_ssse3, + &aom_highbd_masked_sub_pixel_variance128x64_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance64x128_ssse3, + &aom_highbd_masked_sub_pixel_variance64x128_c, AOM_BITS_8), +#endif // CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_masked_sub_pixel_variance64x64_ssse3, + &aom_highbd_masked_sub_pixel_variance64x64_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance64x32_ssse3, + &aom_highbd_masked_sub_pixel_variance64x32_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance32x64_ssse3, + &aom_highbd_masked_sub_pixel_variance32x64_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance32x32_ssse3, + &aom_highbd_masked_sub_pixel_variance32x32_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance32x16_ssse3, + &aom_highbd_masked_sub_pixel_variance32x16_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance16x32_ssse3, + &aom_highbd_masked_sub_pixel_variance16x32_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance16x16_ssse3, + &aom_highbd_masked_sub_pixel_variance16x16_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance16x8_ssse3, + &aom_highbd_masked_sub_pixel_variance16x8_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance8x16_ssse3, + &aom_highbd_masked_sub_pixel_variance8x16_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance8x8_ssse3, + &aom_highbd_masked_sub_pixel_variance8x8_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance8x4_ssse3, + &aom_highbd_masked_sub_pixel_variance8x4_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance4x8_ssse3, + &aom_highbd_masked_sub_pixel_variance4x8_c, AOM_BITS_8), + make_tuple(&aom_highbd_masked_sub_pixel_variance4x4_ssse3, + &aom_highbd_masked_sub_pixel_variance4x4_c, AOM_BITS_8), +#if CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_10_masked_sub_pixel_variance128x128_ssse3, + &aom_highbd_10_masked_sub_pixel_variance128x128_c, + AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance128x64_ssse3, + &aom_highbd_10_masked_sub_pixel_variance128x64_c, + AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance64x128_ssse3, + &aom_highbd_10_masked_sub_pixel_variance64x128_c, + AOM_BITS_10), +#endif // CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_10_masked_sub_pixel_variance64x64_ssse3, + &aom_highbd_10_masked_sub_pixel_variance64x64_c, + AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance64x32_ssse3, + &aom_highbd_10_masked_sub_pixel_variance64x32_c, + AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance32x64_ssse3, + &aom_highbd_10_masked_sub_pixel_variance32x64_c, + AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance32x32_ssse3, + &aom_highbd_10_masked_sub_pixel_variance32x32_c, + AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance32x16_ssse3, + &aom_highbd_10_masked_sub_pixel_variance32x16_c, + AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance16x32_ssse3, + &aom_highbd_10_masked_sub_pixel_variance16x32_c, + AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance16x16_ssse3, + &aom_highbd_10_masked_sub_pixel_variance16x16_c, + AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance16x8_ssse3, + &aom_highbd_10_masked_sub_pixel_variance16x8_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance8x16_ssse3, + &aom_highbd_10_masked_sub_pixel_variance8x16_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance8x8_ssse3, + &aom_highbd_10_masked_sub_pixel_variance8x8_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance8x4_ssse3, + &aom_highbd_10_masked_sub_pixel_variance8x4_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance4x8_ssse3, + &aom_highbd_10_masked_sub_pixel_variance4x8_c, AOM_BITS_10), + make_tuple(&aom_highbd_10_masked_sub_pixel_variance4x4_ssse3, + &aom_highbd_10_masked_sub_pixel_variance4x4_c, AOM_BITS_10), +#if CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_12_masked_sub_pixel_variance128x128_ssse3, + &aom_highbd_12_masked_sub_pixel_variance128x128_c, + AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance128x64_ssse3, + &aom_highbd_12_masked_sub_pixel_variance128x64_c, + AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance64x128_ssse3, + &aom_highbd_12_masked_sub_pixel_variance64x128_c, + AOM_BITS_12), +#endif // CONFIG_EXT_PARTITION + make_tuple(&aom_highbd_12_masked_sub_pixel_variance64x64_ssse3, + &aom_highbd_12_masked_sub_pixel_variance64x64_c, + AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance64x32_ssse3, + &aom_highbd_12_masked_sub_pixel_variance64x32_c, + AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance32x64_ssse3, + &aom_highbd_12_masked_sub_pixel_variance32x64_c, + AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance32x32_ssse3, + &aom_highbd_12_masked_sub_pixel_variance32x32_c, + AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance32x16_ssse3, + &aom_highbd_12_masked_sub_pixel_variance32x16_c, + AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance16x32_ssse3, + &aom_highbd_12_masked_sub_pixel_variance16x32_c, + AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance16x16_ssse3, + &aom_highbd_12_masked_sub_pixel_variance16x16_c, + AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance16x8_ssse3, + &aom_highbd_12_masked_sub_pixel_variance16x8_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance8x16_ssse3, + &aom_highbd_12_masked_sub_pixel_variance8x16_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance8x8_ssse3, + &aom_highbd_12_masked_sub_pixel_variance8x8_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance8x4_ssse3, + &aom_highbd_12_masked_sub_pixel_variance8x4_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance4x8_ssse3, + &aom_highbd_12_masked_sub_pixel_variance4x8_c, AOM_BITS_12), + make_tuple(&aom_highbd_12_masked_sub_pixel_variance4x4_ssse3, + &aom_highbd_12_masked_sub_pixel_variance4x4_c, + AOM_BITS_12))); +#endif // CONFIG_HIGHBITDEPTH + +#endif // HAVE_SSSE3 +} // namespace diff --git a/third_party/aom/test/md5_helper.h b/third_party/aom/test/md5_helper.h new file mode 100644 index 0000000000..8c9d4f706f --- /dev/null +++ b/third_party/aom/test/md5_helper.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef TEST_MD5_HELPER_H_ +#define TEST_MD5_HELPER_H_ + +#include "./md5_utils.h" +#include "aom/aom_decoder.h" + +namespace libaom_test { +class MD5 { + public: + MD5() { MD5Init(&md5_); } + + void Add(const aom_image_t *img) { + for (int plane = 0; plane < 3; ++plane) { + const uint8_t *buf = img->planes[plane]; + // Calculate the width and height to do the md5 check. For the chroma + // plane, we never want to round down and thus skip a pixel so if + // we are shifting by 1 (chroma_shift) we add 1 before doing the shift. + // This works only for chroma_shift of 0 and 1. + const int bytes_per_sample = + (img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1; + const int h = + plane ? (img->d_h + img->y_chroma_shift) >> img->y_chroma_shift + : img->d_h; + const int w = + (plane ? (img->d_w + img->x_chroma_shift) >> img->x_chroma_shift + : img->d_w) * + bytes_per_sample; + + for (int y = 0; y < h; ++y) { + MD5Update(&md5_, buf, w); + buf += img->stride[plane]; + } + } + } + + void Add(const uint8_t *data, size_t size) { + MD5Update(&md5_, data, static_cast(size)); + } + + const char *Get(void) { + static const char hex[16] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', + }; + uint8_t tmp[16]; + MD5Context ctx_tmp = md5_; + + MD5Final(tmp, &ctx_tmp); + for (int i = 0; i < 16; i++) { + res_[i * 2 + 0] = hex[tmp[i] >> 4]; + res_[i * 2 + 1] = hex[tmp[i] & 0xf]; + } + res_[32] = 0; + + return res_; + } + + protected: + char res_[33]; + MD5Context md5_; +}; + +} // namespace libaom_test + +#endif // TEST_MD5_HELPER_H_ diff --git a/third_party/aom/test/minmax_test.cc b/third_party/aom/test/minmax_test.cc new file mode 100644 index 0000000000..f825291927 --- /dev/null +++ b/third_party/aom/test/minmax_test.cc @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +#include "test/acm_random.h" +#include "test/register_state_check.h" + +namespace { + +using ::libaom_test::ACMRandom; + +typedef void (*MinMaxFunc)(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int *min, int *max); + +class MinMaxTest : public ::testing::TestWithParam { + public: + virtual void SetUp() { + mm_func_ = GetParam(); + rnd_.Reset(ACMRandom::DeterministicSeed()); + } + + protected: + MinMaxFunc mm_func_; + ACMRandom rnd_; +}; + +void reference_minmax(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int *min_ret, int *max_ret) { + int min = 255; + int max = 0; + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + const int diff = abs(a[i * a_stride + j] - b[i * b_stride + j]); + if (min > diff) min = diff; + if (max < diff) max = diff; + } + } + + *min_ret = min; + *max_ret = max; +} + +TEST_P(MinMaxTest, MinValue) { + for (int i = 0; i < 64; i++) { + uint8_t a[64], b[64]; + memset(a, 0, sizeof(a)); + memset(b, 255, sizeof(b)); + b[i] = i; // Set a minimum difference of i. + + int min, max; + ASM_REGISTER_STATE_CHECK(mm_func_(a, 8, b, 8, &min, &max)); + EXPECT_EQ(255, max); + EXPECT_EQ(i, min); + } +} + +TEST_P(MinMaxTest, MaxValue) { + for (int i = 0; i < 64; i++) { + uint8_t a[64], b[64]; + memset(a, 0, sizeof(a)); + memset(b, 0, sizeof(b)); + b[i] = i; // Set a maximum difference of i. + + int min, max; + ASM_REGISTER_STATE_CHECK(mm_func_(a, 8, b, 8, &min, &max)); + EXPECT_EQ(i, max); + EXPECT_EQ(0, min); + } +} + +TEST_P(MinMaxTest, CompareReference) { + uint8_t a[64], b[64]; + for (int j = 0; j < 64; j++) { + a[j] = rnd_.Rand8(); + b[j] = rnd_.Rand8(); + } + + int min_ref, max_ref, min, max; + reference_minmax(a, 8, b, 8, &min_ref, &max_ref); + ASM_REGISTER_STATE_CHECK(mm_func_(a, 8, b, 8, &min, &max)); + EXPECT_EQ(max_ref, max); + EXPECT_EQ(min_ref, min); +} + +TEST_P(MinMaxTest, CompareReferenceAndVaryStride) { + uint8_t a[8 * 64], b[8 * 64]; + for (int i = 0; i < 8 * 64; i++) { + a[i] = rnd_.Rand8(); + b[i] = rnd_.Rand8(); + } + for (int a_stride = 8; a_stride <= 64; a_stride += 8) { + for (int b_stride = 8; b_stride <= 64; b_stride += 8) { + int min_ref, max_ref, min, max; + reference_minmax(a, a_stride, b, b_stride, &min_ref, &max_ref); + ASM_REGISTER_STATE_CHECK(mm_func_(a, a_stride, b, b_stride, &min, &max)); + EXPECT_EQ(max_ref, max) << "when a_stride = " << a_stride + << " and b_stride = " << b_stride; + EXPECT_EQ(min_ref, min) << "when a_stride = " << a_stride + << " and b_stride = " << b_stride; + } + } +} + +INSTANTIATE_TEST_CASE_P(C, MinMaxTest, ::testing::Values(&aom_minmax_8x8_c)); + +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P(SSE2, MinMaxTest, + ::testing::Values(&aom_minmax_8x8_sse2)); +#endif + +#if HAVE_NEON +INSTANTIATE_TEST_CASE_P(NEON, MinMaxTest, + ::testing::Values(&aom_minmax_8x8_neon)); +#endif + +} // namespace diff --git a/third_party/aom/test/motion_vector_test.cc b/third_party/aom/test/motion_vector_test.cc new file mode 100644 index 0000000000..403a8f1a79 --- /dev/null +++ b/third_party/aom/test/motion_vector_test.cc @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/util.h" +#include "test/yuv_video_source.h" + +namespace { +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define BUILDING_WITH_ASAN +#endif +#endif + +#define MAX_EXTREME_MV 1 +#define MIN_EXTREME_MV 2 + +// Encoding modes +const libaom_test::TestMode kEncodingModeVectors[] = { + ::libaom_test::kTwoPassGood, ::libaom_test::kOnePassGood, +}; + +// Encoding speeds +const int kCpuUsedVectors[] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + +// MV test modes: 1 - always use maximum MV; 2 - always use minimum MV. +const int kMVTestModes[] = { MAX_EXTREME_MV, MIN_EXTREME_MV }; + +class MotionVectorTestLarge + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith3Params { + protected: + MotionVectorTestLarge() + : EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)), + cpu_used_(GET_PARAM(2)), mv_test_mode_(GET_PARAM(3)) {} + + virtual ~MotionVectorTestLarge() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + if (encoding_mode_ != ::libaom_test::kRealTime) { + cfg_.g_lag_in_frames = 3; + cfg_.rc_end_usage = AOM_VBR; + } else { + cfg_.g_lag_in_frames = 0; + cfg_.rc_end_usage = AOM_CBR; + cfg_.rc_buf_sz = 1000; + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 600; + } + } + + virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) { + if (video->frame() == 1) { + encoder->Control(AOME_SET_CPUUSED, cpu_used_); + encoder->Control(AV1E_ENABLE_MOTION_VECTOR_UNIT_TEST, mv_test_mode_); + if (encoding_mode_ != ::libaom_test::kRealTime) { + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); + encoder->Control(AOME_SET_ARNR_MAXFRAMES, 7); + encoder->Control(AOME_SET_ARNR_STRENGTH, 5); + } + } + } + + libaom_test::TestMode encoding_mode_; + int cpu_used_; + int mv_test_mode_; +}; + +TEST_P(MotionVectorTestLarge, OverallTest) { + int width = 3840; + int height = 2160; + +#ifdef BUILDING_WITH_ASAN + // On the 32-bit system, if using 4k test clip, an "out of memory" error + // occurs because of the AddressSanitizer instrumentation memory overhead. + // Here, reduce the test clip's resolution while testing on 32-bit system + // and AddressSanitizer is enabled. + if (sizeof(void *) == 4) { + width = 2048; + height = 1080; + } +#endif + + cfg_.rc_target_bitrate = 24000; + cfg_.g_profile = 0; + init_flags_ = AOM_CODEC_USE_PSNR; + + testing::internal::scoped_ptr video; + video.reset(new libaom_test::YUVVideoSource( + "niklas_640_480_30.yuv", AOM_IMG_FMT_I420, width, height, 30, 1, 0, 5)); + + ASSERT_TRUE(video.get() != NULL); + ASSERT_NO_FATAL_FAILURE(RunLoop(video.get())); +} + +AV1_INSTANTIATE_TEST_CASE(MotionVectorTestLarge, + ::testing::ValuesIn(kEncodingModeVectors), + ::testing::ValuesIn(kCpuUsedVectors), + ::testing::ValuesIn(kMVTestModes)); +} // namespace diff --git a/third_party/aom/test/obmc_sad_test.cc b/third_party/aom/test/obmc_sad_test.cc new file mode 100644 index 0000000000..219c5d8109 --- /dev/null +++ b/third_party/aom/test/obmc_sad_test.cc @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/function_equivalence_test.h" +#include "test/register_state_check.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +#define MAX_SB_SQUARE (MAX_SB_SIZE * MAX_SB_SIZE) + +using libaom_test::FunctionEquivalenceTest; + +namespace { + +static const int kIterations = 1000; +static const int kMaskMax = 64; + +typedef unsigned int (*ObmcSadF)(const uint8_t *pre, int pre_stride, + const int32_t *wsrc, const int32_t *mask); +typedef libaom_test::FuncParam TestFuncs; + +//////////////////////////////////////////////////////////////////////////////// +// 8 bit +//////////////////////////////////////////////////////////////////////////////// + +class ObmcSadTest : public FunctionEquivalenceTest {}; + +TEST_P(ObmcSadTest, RandomValues) { + DECLARE_ALIGNED(32, uint8_t, pre[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + const int pre_stride = rng_(MAX_SB_SIZE + 1); + + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + pre[i] = rng_.Rand8(); + wsrc[i] = rng_.Rand8() * rng_(kMaskMax * kMaskMax + 1); + mask[i] = rng_(kMaskMax * kMaskMax + 1); + } + + const unsigned int ref_res = params_.ref_func(pre, pre_stride, wsrc, mask); + unsigned int tst_res; + ASM_REGISTER_STATE_CHECK(tst_res = + params_.tst_func(pre, pre_stride, wsrc, mask)); + + ASSERT_EQ(ref_res, tst_res); + } +} + +TEST_P(ObmcSadTest, ExtremeValues) { + DECLARE_ALIGNED(32, uint8_t, pre[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0; iter < MAX_SB_SIZE && !HasFatalFailure(); ++iter) { + const int pre_stride = iter; + + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + pre[i] = UINT8_MAX; + wsrc[i] = UINT8_MAX * kMaskMax * kMaskMax; + mask[i] = kMaskMax * kMaskMax; + } + + const unsigned int ref_res = params_.ref_func(pre, pre_stride, wsrc, mask); + unsigned int tst_res; + ASM_REGISTER_STATE_CHECK(tst_res = + params_.tst_func(pre, pre_stride, wsrc, mask)); + + ASSERT_EQ(ref_res, tst_res); + } +} + +#if HAVE_SSE4_1 +#if CONFIG_MOTION_VAR +const ObmcSadTest::ParamType sse4_functions[] = { +#if CONFIG_EXT_PARTITION + TestFuncs(aom_obmc_sad128x128_c, aom_obmc_sad128x128_sse4_1), + TestFuncs(aom_obmc_sad128x64_c, aom_obmc_sad128x64_sse4_1), + TestFuncs(aom_obmc_sad64x128_c, aom_obmc_sad64x128_sse4_1), +#endif // CONFIG_EXT_PARTITION + TestFuncs(aom_obmc_sad64x64_c, aom_obmc_sad64x64_sse4_1), + TestFuncs(aom_obmc_sad64x32_c, aom_obmc_sad64x32_sse4_1), + TestFuncs(aom_obmc_sad32x64_c, aom_obmc_sad32x64_sse4_1), + TestFuncs(aom_obmc_sad32x32_c, aom_obmc_sad32x32_sse4_1), + TestFuncs(aom_obmc_sad32x16_c, aom_obmc_sad32x16_sse4_1), + TestFuncs(aom_obmc_sad16x32_c, aom_obmc_sad16x32_sse4_1), + TestFuncs(aom_obmc_sad16x16_c, aom_obmc_sad16x16_sse4_1), + TestFuncs(aom_obmc_sad16x8_c, aom_obmc_sad16x8_sse4_1), + TestFuncs(aom_obmc_sad8x16_c, aom_obmc_sad8x16_sse4_1), + TestFuncs(aom_obmc_sad8x8_c, aom_obmc_sad8x8_sse4_1), + TestFuncs(aom_obmc_sad8x4_c, aom_obmc_sad8x4_sse4_1), + TestFuncs(aom_obmc_sad4x8_c, aom_obmc_sad4x8_sse4_1), + TestFuncs(aom_obmc_sad4x4_c, aom_obmc_sad4x4_sse4_1) +}; + +INSTANTIATE_TEST_CASE_P(SSE4_1, ObmcSadTest, + ::testing::ValuesIn(sse4_functions)); +#endif // CONFIG_MOTION_VAR +#endif // HAVE_SSE4_1 + +//////////////////////////////////////////////////////////////////////////////// +// High bit-depth +//////////////////////////////////////////////////////////////////////////////// + +#if CONFIG_HIGHBITDEPTH +class ObmcSadHBDTest : public FunctionEquivalenceTest {}; + +TEST_P(ObmcSadHBDTest, RandomValues) { + DECLARE_ALIGNED(32, uint16_t, pre[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + const int pre_stride = rng_(MAX_SB_SIZE + 1); + + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + pre[i] = rng_(1 << 12); + wsrc[i] = rng_(1 << 12) * rng_(kMaskMax * kMaskMax + 1); + mask[i] = rng_(kMaskMax * kMaskMax + 1); + } + + const unsigned int ref_res = + params_.ref_func(CONVERT_TO_BYTEPTR(pre), pre_stride, wsrc, mask); + unsigned int tst_res; + ASM_REGISTER_STATE_CHECK( + tst_res = + params_.tst_func(CONVERT_TO_BYTEPTR(pre), pre_stride, wsrc, mask)); + + ASSERT_EQ(ref_res, tst_res); + } +} + +TEST_P(ObmcSadHBDTest, ExtremeValues) { + DECLARE_ALIGNED(32, uint16_t, pre[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0; iter < MAX_SB_SIZE && !HasFatalFailure(); ++iter) { + const int pre_stride = iter; + + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + pre[i] = (1 << 12) - 1; + wsrc[i] = ((1 << 12) - 1) * kMaskMax * kMaskMax; + mask[i] = kMaskMax * kMaskMax; + } + + const unsigned int ref_res = + params_.ref_func(CONVERT_TO_BYTEPTR(pre), pre_stride, wsrc, mask); + unsigned int tst_res; + ASM_REGISTER_STATE_CHECK( + tst_res = + params_.tst_func(CONVERT_TO_BYTEPTR(pre), pre_stride, wsrc, mask)); + + ASSERT_EQ(ref_res, tst_res); + } +} + +#if HAVE_SSE4_1 +#if CONFIG_MOTION_VAR +ObmcSadHBDTest::ParamType sse4_functions_hbd[] = { +#if CONFIG_EXT_PARTITION + TestFuncs(aom_highbd_obmc_sad128x128_c, aom_highbd_obmc_sad128x128_sse4_1), + TestFuncs(aom_highbd_obmc_sad128x64_c, aom_highbd_obmc_sad128x64_sse4_1), + TestFuncs(aom_highbd_obmc_sad64x128_c, aom_highbd_obmc_sad64x128_sse4_1), +#endif // CONFIG_EXT_PARTITION + TestFuncs(aom_highbd_obmc_sad64x64_c, aom_highbd_obmc_sad64x64_sse4_1), + TestFuncs(aom_highbd_obmc_sad64x32_c, aom_highbd_obmc_sad64x32_sse4_1), + TestFuncs(aom_highbd_obmc_sad32x64_c, aom_highbd_obmc_sad32x64_sse4_1), + TestFuncs(aom_highbd_obmc_sad32x32_c, aom_highbd_obmc_sad32x32_sse4_1), + TestFuncs(aom_highbd_obmc_sad32x16_c, aom_highbd_obmc_sad32x16_sse4_1), + TestFuncs(aom_highbd_obmc_sad16x32_c, aom_highbd_obmc_sad16x32_sse4_1), + TestFuncs(aom_highbd_obmc_sad16x16_c, aom_highbd_obmc_sad16x16_sse4_1), + TestFuncs(aom_highbd_obmc_sad16x8_c, aom_highbd_obmc_sad16x8_sse4_1), + TestFuncs(aom_highbd_obmc_sad8x16_c, aom_highbd_obmc_sad8x16_sse4_1), + TestFuncs(aom_highbd_obmc_sad8x8_c, aom_highbd_obmc_sad8x8_sse4_1), + TestFuncs(aom_highbd_obmc_sad8x4_c, aom_highbd_obmc_sad8x4_sse4_1), + TestFuncs(aom_highbd_obmc_sad4x8_c, aom_highbd_obmc_sad4x8_sse4_1), + TestFuncs(aom_highbd_obmc_sad4x4_c, aom_highbd_obmc_sad4x4_sse4_1) +}; + +INSTANTIATE_TEST_CASE_P(SSE4_1, ObmcSadHBDTest, + ::testing::ValuesIn(sse4_functions_hbd)); +#endif // CONFIG_MOTION_VAR +#endif // HAVE_SSE4_1 +#endif // CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/obmc_variance_test.cc b/third_party/aom/test/obmc_variance_test.cc new file mode 100644 index 0000000000..1b30645a52 --- /dev/null +++ b/third_party/aom/test/obmc_variance_test.cc @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/acm_random.h" + +#include "test/function_equivalence_test.h" +#include "test/register_state_check.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +#define MAX_SB_SQUARE (MAX_SB_SIZE * MAX_SB_SIZE) + +using libaom_test::ACMRandom; +using libaom_test::FunctionEquivalenceTest; + +namespace { + +static const int kIterations = 1000; +static const int kMaskMax = 64; + +typedef unsigned int (*ObmcVarF)(const uint8_t *pre, int pre_stride, + const int32_t *wsrc, const int32_t *mask, + unsigned int *sse); +typedef libaom_test::FuncParam TestFuncs; + +//////////////////////////////////////////////////////////////////////////////// +// 8 bit +//////////////////////////////////////////////////////////////////////////////// + +class ObmcVarianceTest : public FunctionEquivalenceTest {}; + +TEST_P(ObmcVarianceTest, RandomValues) { + DECLARE_ALIGNED(32, uint8_t, pre[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + const int pre_stride = this->rng_(MAX_SB_SIZE + 1); + + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + pre[i] = this->rng_.Rand8(); + wsrc[i] = this->rng_.Rand8() * this->rng_(kMaskMax * kMaskMax + 1); + mask[i] = this->rng_(kMaskMax * kMaskMax + 1); + } + + unsigned int ref_sse, tst_sse; + const unsigned int ref_res = + params_.ref_func(pre, pre_stride, wsrc, mask, &ref_sse); + unsigned int tst_res; + ASM_REGISTER_STATE_CHECK( + tst_res = params_.tst_func(pre, pre_stride, wsrc, mask, &tst_sse)); + + ASSERT_EQ(ref_res, tst_res); + ASSERT_EQ(ref_sse, tst_sse); + } +} + +TEST_P(ObmcVarianceTest, ExtremeValues) { + DECLARE_ALIGNED(32, uint8_t, pre[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0; iter < MAX_SB_SIZE && !HasFatalFailure(); ++iter) { + const int pre_stride = iter; + + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + pre[i] = UINT8_MAX; + wsrc[i] = UINT8_MAX * kMaskMax * kMaskMax; + mask[i] = kMaskMax * kMaskMax; + } + + unsigned int ref_sse, tst_sse; + const unsigned int ref_res = + params_.ref_func(pre, pre_stride, wsrc, mask, &ref_sse); + unsigned int tst_res; + ASM_REGISTER_STATE_CHECK( + tst_res = params_.tst_func(pre, pre_stride, wsrc, mask, &tst_sse)); + + ASSERT_EQ(ref_res, tst_res); + ASSERT_EQ(ref_sse, tst_sse); + } +} + +#if HAVE_SSE4_1 +#if CONFIG_MOTION_VAR +const ObmcVarianceTest::ParamType sse4_functions[] = { +#if CONFIG_EXT_PARTITION + TestFuncs(aom_obmc_variance128x128_c, aom_obmc_variance128x128_sse4_1), + TestFuncs(aom_obmc_variance128x64_c, aom_obmc_variance128x64_sse4_1), + TestFuncs(aom_obmc_variance64x128_c, aom_obmc_variance64x128_sse4_1), +#endif // CONFIG_EXT_PARTITION + TestFuncs(aom_obmc_variance64x64_c, aom_obmc_variance64x64_sse4_1), + TestFuncs(aom_obmc_variance64x32_c, aom_obmc_variance64x32_sse4_1), + TestFuncs(aom_obmc_variance32x64_c, aom_obmc_variance32x64_sse4_1), + TestFuncs(aom_obmc_variance32x32_c, aom_obmc_variance32x32_sse4_1), + TestFuncs(aom_obmc_variance32x16_c, aom_obmc_variance32x16_sse4_1), + TestFuncs(aom_obmc_variance16x32_c, aom_obmc_variance16x32_sse4_1), + TestFuncs(aom_obmc_variance16x16_c, aom_obmc_variance16x16_sse4_1), + TestFuncs(aom_obmc_variance16x8_c, aom_obmc_variance16x8_sse4_1), + TestFuncs(aom_obmc_variance8x16_c, aom_obmc_variance8x16_sse4_1), + TestFuncs(aom_obmc_variance8x8_c, aom_obmc_variance8x8_sse4_1), + TestFuncs(aom_obmc_variance8x4_c, aom_obmc_variance8x4_sse4_1), + TestFuncs(aom_obmc_variance4x8_c, aom_obmc_variance4x8_sse4_1), + TestFuncs(aom_obmc_variance4x4_c, aom_obmc_variance4x4_sse4_1) +}; + +INSTANTIATE_TEST_CASE_P(SSE4_1, ObmcVarianceTest, + ::testing::ValuesIn(sse4_functions)); +#endif // CONFIG_MOTION_VAR +#endif // HAVE_SSE4_1 + +//////////////////////////////////////////////////////////////////////////////// +// High bit-depth +//////////////////////////////////////////////////////////////////////////////// + +#if CONFIG_HIGHBITDEPTH +class ObmcVarianceHBDTest : public FunctionEquivalenceTest {}; + +TEST_P(ObmcVarianceHBDTest, RandomValues) { + DECLARE_ALIGNED(32, uint16_t, pre[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + const int pre_stride = this->rng_(MAX_SB_SIZE + 1); + + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + pre[i] = this->rng_(1 << params_.bit_depth); + wsrc[i] = this->rng_(1 << params_.bit_depth) * + this->rng_(kMaskMax * kMaskMax + 1); + mask[i] = this->rng_(kMaskMax * kMaskMax + 1); + } + + unsigned int ref_sse, tst_sse; + const unsigned int ref_res = params_.ref_func( + CONVERT_TO_BYTEPTR(pre), pre_stride, wsrc, mask, &ref_sse); + unsigned int tst_res; + ASM_REGISTER_STATE_CHECK(tst_res = params_.tst_func(CONVERT_TO_BYTEPTR(pre), + pre_stride, wsrc, mask, + &tst_sse)); + + ASSERT_EQ(ref_res, tst_res); + ASSERT_EQ(ref_sse, tst_sse); + } +} + +TEST_P(ObmcVarianceHBDTest, ExtremeValues) { + DECLARE_ALIGNED(32, uint16_t, pre[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0; iter < MAX_SB_SIZE && !HasFatalFailure(); ++iter) { + const int pre_stride = iter; + + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + pre[i] = (1 << params_.bit_depth) - 1; + wsrc[i] = ((1 << params_.bit_depth) - 1) * kMaskMax * kMaskMax; + mask[i] = kMaskMax * kMaskMax; + } + + unsigned int ref_sse, tst_sse; + const unsigned int ref_res = params_.ref_func( + CONVERT_TO_BYTEPTR(pre), pre_stride, wsrc, mask, &ref_sse); + unsigned int tst_res; + ASM_REGISTER_STATE_CHECK(tst_res = params_.tst_func(CONVERT_TO_BYTEPTR(pre), + pre_stride, wsrc, mask, + &tst_sse)); + + ASSERT_EQ(ref_res, tst_res); + ASSERT_EQ(ref_sse, tst_sse); + } +} + +#if HAVE_SSE4_1 +#if CONFIG_MOTION_VAR +ObmcVarianceHBDTest::ParamType sse4_functions_hbd[] = { +#if CONFIG_EXT_PARTITION + TestFuncs(aom_highbd_obmc_variance128x128_c, + aom_highbd_obmc_variance128x128_sse4_1, 8), + TestFuncs(aom_highbd_obmc_variance128x64_c, + aom_highbd_obmc_variance128x64_sse4_1, 8), + TestFuncs(aom_highbd_obmc_variance64x128_c, + aom_highbd_obmc_variance64x128_sse4_1, 8), +#endif // CONFIG_EXT_PARTITION + TestFuncs(aom_highbd_obmc_variance64x64_c, + aom_highbd_obmc_variance64x64_sse4_1, 8), + TestFuncs(aom_highbd_obmc_variance64x32_c, + aom_highbd_obmc_variance64x32_sse4_1, 8), + TestFuncs(aom_highbd_obmc_variance32x64_c, + aom_highbd_obmc_variance32x64_sse4_1, 8), + TestFuncs(aom_highbd_obmc_variance32x32_c, + aom_highbd_obmc_variance32x32_sse4_1, 8), + TestFuncs(aom_highbd_obmc_variance32x16_c, + aom_highbd_obmc_variance32x16_sse4_1, 8), + TestFuncs(aom_highbd_obmc_variance16x32_c, + aom_highbd_obmc_variance16x32_sse4_1, 8), + TestFuncs(aom_highbd_obmc_variance16x16_c, + aom_highbd_obmc_variance16x16_sse4_1, 8), + TestFuncs(aom_highbd_obmc_variance16x8_c, aom_highbd_obmc_variance16x8_sse4_1, + 8), + TestFuncs(aom_highbd_obmc_variance8x16_c, aom_highbd_obmc_variance8x16_sse4_1, + 8), + TestFuncs(aom_highbd_obmc_variance8x8_c, aom_highbd_obmc_variance8x8_sse4_1, + 8), + TestFuncs(aom_highbd_obmc_variance8x4_c, aom_highbd_obmc_variance8x4_sse4_1, + 8), + TestFuncs(aom_highbd_obmc_variance4x8_c, aom_highbd_obmc_variance4x8_sse4_1, + 8), + TestFuncs(aom_highbd_obmc_variance4x4_c, aom_highbd_obmc_variance4x4_sse4_1, + 8), +#if CONFIG_EXT_PARTITION + TestFuncs(aom_highbd_10_obmc_variance128x128_c, + aom_highbd_10_obmc_variance128x128_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance128x64_c, + aom_highbd_10_obmc_variance128x64_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance64x128_c, + aom_highbd_10_obmc_variance64x128_sse4_1, 10), +#endif // CONFIG_EXT_PARTITION + TestFuncs(aom_highbd_10_obmc_variance64x64_c, + aom_highbd_10_obmc_variance64x64_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance64x32_c, + aom_highbd_10_obmc_variance64x32_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance32x64_c, + aom_highbd_10_obmc_variance32x64_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance32x32_c, + aom_highbd_10_obmc_variance32x32_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance32x16_c, + aom_highbd_10_obmc_variance32x16_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance16x32_c, + aom_highbd_10_obmc_variance16x32_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance16x16_c, + aom_highbd_10_obmc_variance16x16_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance16x8_c, + aom_highbd_10_obmc_variance16x8_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance8x16_c, + aom_highbd_10_obmc_variance8x16_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance8x8_c, + aom_highbd_10_obmc_variance8x8_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance8x4_c, + aom_highbd_10_obmc_variance8x4_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance4x8_c, + aom_highbd_10_obmc_variance4x8_sse4_1, 10), + TestFuncs(aom_highbd_10_obmc_variance4x4_c, + aom_highbd_10_obmc_variance4x4_sse4_1, 10), +#if CONFIG_EXT_PARTITION + TestFuncs(aom_highbd_12_obmc_variance128x128_c, + aom_highbd_12_obmc_variance128x128_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance128x64_c, + aom_highbd_12_obmc_variance128x64_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance64x128_c, + aom_highbd_12_obmc_variance64x128_sse4_1, 12), +#endif // CONFIG_EXT_PARTITION + TestFuncs(aom_highbd_12_obmc_variance64x64_c, + aom_highbd_12_obmc_variance64x64_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance64x32_c, + aom_highbd_12_obmc_variance64x32_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance32x64_c, + aom_highbd_12_obmc_variance32x64_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance32x32_c, + aom_highbd_12_obmc_variance32x32_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance32x16_c, + aom_highbd_12_obmc_variance32x16_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance16x32_c, + aom_highbd_12_obmc_variance16x32_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance16x16_c, + aom_highbd_12_obmc_variance16x16_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance16x8_c, + aom_highbd_12_obmc_variance16x8_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance8x16_c, + aom_highbd_12_obmc_variance8x16_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance8x8_c, + aom_highbd_12_obmc_variance8x8_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance8x4_c, + aom_highbd_12_obmc_variance8x4_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance4x8_c, + aom_highbd_12_obmc_variance4x8_sse4_1, 12), + TestFuncs(aom_highbd_12_obmc_variance4x4_c, + aom_highbd_12_obmc_variance4x4_sse4_1, 12) +}; + +INSTANTIATE_TEST_CASE_P(SSE4_1, ObmcVarianceHBDTest, + ::testing::ValuesIn(sse4_functions_hbd)); +#endif // CONFIG_MOTION_VAR +#endif // HAVE_SSE4_1 +#endif // CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/partial_idct_test.cc b/third_party/aom/test/partial_idct_test.cc new file mode 100644 index 0000000000..0899b60c32 --- /dev/null +++ b/third_party/aom/test/partial_idct_test.cc @@ -0,0 +1,485 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "av1/common/blockd.h" +#include "av1/common/scan.h" +#include "aom/aom_integer.h" +#include "aom_ports/aom_timer.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*FwdTxfmFunc)(const int16_t *in, tran_low_t *out, int stride); +typedef void (*InvTxfmFunc)(const tran_low_t *in, uint8_t *out, int stride); +typedef void (*InvTxfmWithBdFunc)(const tran_low_t *in, uint8_t *out, + int stride, int bd); + +template +void wrapper(const tran_low_t *in, uint8_t *out, int stride, int bd) { + (void)bd; + fn(in, out, stride); +} + +#if CONFIG_HIGHBITDEPTH +template +void highbd_wrapper(const tran_low_t *in, uint8_t *out, int stride, int bd) { + fn(in, CONVERT_TO_BYTEPTR(out), stride, bd); +} +#endif + +typedef std::tr1::tuple + PartialInvTxfmParam; +const int kMaxNumCoeffs = 1024; +const int kCountTestBlock = 1000; + +class PartialIDctTest : public ::testing::TestWithParam { + public: + virtual ~PartialIDctTest() {} + virtual void SetUp() { + rnd_.Reset(ACMRandom::DeterministicSeed()); + ftxfm_ = GET_PARAM(0); + full_itxfm_ = GET_PARAM(1); + partial_itxfm_ = GET_PARAM(2); + tx_size_ = GET_PARAM(3); + last_nonzero_ = GET_PARAM(4); + bit_depth_ = GET_PARAM(5); + pixel_size_ = GET_PARAM(6); + mask_ = (1 << bit_depth_) - 1; + + switch (tx_size_) { + case TX_4X4: size_ = 4; break; + case TX_8X8: size_ = 8; break; + case TX_16X16: size_ = 16; break; + case TX_32X32: size_ = 32; break; + default: FAIL() << "Wrong Size!"; break; + } + + // Randomize stride_ to a value less than or equal to 1024 + stride_ = rnd_(1024) + 1; + if (stride_ < size_) { + stride_ = size_; + } + // Align stride_ to 16 if it's bigger than 16. + if (stride_ > 16) { + stride_ &= ~15; + } + + input_block_size_ = size_ * size_; + output_block_size_ = size_ * stride_; + + input_block_ = reinterpret_cast( + aom_memalign(16, sizeof(*input_block_) * input_block_size_)); + output_block_ = reinterpret_cast( + aom_memalign(16, pixel_size_ * output_block_size_)); + output_block_ref_ = reinterpret_cast( + aom_memalign(16, pixel_size_ * output_block_size_)); + } + + virtual void TearDown() { + aom_free(input_block_); + input_block_ = NULL; + aom_free(output_block_); + output_block_ = NULL; + aom_free(output_block_ref_); + output_block_ref_ = NULL; + libaom_test::ClearSystemState(); + } + + void InitMem() { + memset(input_block_, 0, sizeof(*input_block_) * input_block_size_); + if (pixel_size_ == 1) { + for (int j = 0; j < output_block_size_; ++j) { + output_block_[j] = output_block_ref_[j] = rnd_.Rand16() & mask_; + } + } else { + ASSERT_EQ(2, pixel_size_); + uint16_t *const output = reinterpret_cast(output_block_); + uint16_t *const output_ref = + reinterpret_cast(output_block_ref_); + for (int j = 0; j < output_block_size_; ++j) { + output[j] = output_ref[j] = rnd_.Rand16() & mask_; + } + } + } + + void InitInput() { + const int max_coeff = 32766 / 4; + int max_energy_leftover = max_coeff * max_coeff; + for (int j = 0; j < last_nonzero_; ++j) { + int16_t coeff = static_cast(sqrt(1.0 * max_energy_leftover) * + (rnd_.Rand16() - 32768) / 65536); + max_energy_leftover -= coeff * coeff; + if (max_energy_leftover < 0) { + max_energy_leftover = 0; + coeff = 0; + } + input_block_[av1_default_scan_orders[tx_size_].scan[j]] = coeff; + } + } + + protected: + int last_nonzero_; + TX_SIZE tx_size_; + tran_low_t *input_block_; + uint8_t *output_block_; + uint8_t *output_block_ref_; + int size_; + int stride_; + int pixel_size_; + int input_block_size_; + int output_block_size_; + int bit_depth_; + int mask_; + FwdTxfmFunc ftxfm_; + InvTxfmWithBdFunc full_itxfm_; + InvTxfmWithBdFunc partial_itxfm_; + ACMRandom rnd_; +}; + +TEST_P(PartialIDctTest, RunQuantCheck) { + DECLARE_ALIGNED(16, int16_t, input_extreme_block[kMaxNumCoeffs]); + DECLARE_ALIGNED(16, tran_low_t, output_ref_block[kMaxNumCoeffs]); + + InitMem(); + for (int i = 0; i < kCountTestBlock; ++i) { + // Initialize a test block with input range [-mask_, mask_]. + if (i == 0) { + for (int k = 0; k < input_block_size_; ++k) { + input_extreme_block[k] = mask_; + } + } else if (i == 1) { + for (int k = 0; k < input_block_size_; ++k) { + input_extreme_block[k] = -mask_; + } + } else { + for (int k = 0; k < input_block_size_; ++k) { + input_extreme_block[k] = rnd_.Rand8() % 2 ? mask_ : -mask_; + } + } + + ftxfm_(input_extreme_block, output_ref_block, size_); + + // quantization with minimum allowed step sizes + input_block_[0] = (output_ref_block[0] / 4) * 4; + for (int k = 1; k < last_nonzero_; ++k) { + const int pos = av1_default_scan_orders[tx_size_].scan[k]; + input_block_[pos] = (output_ref_block[pos] / 4) * 4; + } + + ASM_REGISTER_STATE_CHECK( + full_itxfm_(input_block_, output_block_ref_, stride_, bit_depth_)); + ASM_REGISTER_STATE_CHECK( + partial_itxfm_(input_block_, output_block_, stride_, bit_depth_)); + ASSERT_EQ(0, memcmp(output_block_ref_, output_block_, + pixel_size_ * output_block_size_)) + << "Error: partial inverse transform produces different results"; + } +} + +TEST_P(PartialIDctTest, ResultsMatch) { + for (int i = 0; i < kCountTestBlock; ++i) { + InitMem(); + InitInput(); + + ASM_REGISTER_STATE_CHECK( + full_itxfm_(input_block_, output_block_ref_, stride_, bit_depth_)); + ASM_REGISTER_STATE_CHECK( + partial_itxfm_(input_block_, output_block_, stride_, bit_depth_)); + ASSERT_EQ(0, memcmp(output_block_ref_, output_block_, + pixel_size_ * output_block_size_)) + << "Error: partial inverse transform produces different results"; + } +} + +TEST_P(PartialIDctTest, AddOutputBlock) { + for (int i = 0; i < kCountTestBlock; ++i) { + InitMem(); + for (int j = 0; j < last_nonzero_; ++j) { + input_block_[av1_default_scan_orders[tx_size_].scan[j]] = 10; + } + + ASM_REGISTER_STATE_CHECK( + full_itxfm_(input_block_, output_block_ref_, stride_, bit_depth_)); + ASM_REGISTER_STATE_CHECK( + partial_itxfm_(input_block_, output_block_, stride_, bit_depth_)); + ASSERT_EQ(0, memcmp(output_block_ref_, output_block_, + pixel_size_ * output_block_size_)) + << "Error: Transform results are not correctly added to output."; + } +} + +TEST_P(PartialIDctTest, SingleExtremeCoeff) { + const int16_t max_coeff = std::numeric_limits::max(); + const int16_t min_coeff = std::numeric_limits::min(); + for (int i = 0; i < last_nonzero_; ++i) { + memset(input_block_, 0, sizeof(*input_block_) * input_block_size_); + // Run once for min and once for max. + for (int j = 0; j < 2; ++j) { + const int coeff = j ? min_coeff : max_coeff; + + memset(output_block_, 0, pixel_size_ * output_block_size_); + memset(output_block_ref_, 0, pixel_size_ * output_block_size_); + input_block_[av1_default_scan_orders[tx_size_].scan[i]] = coeff; + + ASM_REGISTER_STATE_CHECK( + full_itxfm_(input_block_, output_block_ref_, stride_, bit_depth_)); + ASM_REGISTER_STATE_CHECK( + partial_itxfm_(input_block_, output_block_, stride_, bit_depth_)); + ASSERT_EQ(0, memcmp(output_block_ref_, output_block_, + pixel_size_ * output_block_size_)) + << "Error: Fails with single coeff of " << coeff << " at " << i + << "."; + } + } +} + +TEST_P(PartialIDctTest, DISABLED_Speed) { + // Keep runtime stable with transform size. + const int kCountSpeedTestBlock = 500000000 / input_block_size_; + InitMem(); + InitInput(); + + for (int i = 0; i < kCountSpeedTestBlock; ++i) { + ASM_REGISTER_STATE_CHECK( + full_itxfm_(input_block_, output_block_ref_, stride_, bit_depth_)); + } + aom_usec_timer timer; + aom_usec_timer_start(&timer); + for (int i = 0; i < kCountSpeedTestBlock; ++i) { + partial_itxfm_(input_block_, output_block_, stride_, bit_depth_); + } + libaom_test::ClearSystemState(); + aom_usec_timer_mark(&timer); + const int elapsed_time = + static_cast(aom_usec_timer_elapsed(&timer) / 1000); + printf("idct%dx%d_%d (bitdepth %d) time: %5d ms\n", size_, size_, + last_nonzero_, bit_depth_, elapsed_time); + + ASSERT_EQ(0, memcmp(output_block_ref_, output_block_, + pixel_size_ * output_block_size_)) + << "Error: partial inverse transform produces different results"; +} + +using std::tr1::make_tuple; + +const PartialInvTxfmParam c_partial_idct_tests[] = { +#if CONFIG_HIGHBITDEPTH + make_tuple(&aom_highbd_fdct4x4_c, + &highbd_wrapper, + &highbd_wrapper, TX_4X4, 16, 8, 2), + make_tuple(&aom_highbd_fdct4x4_c, + &highbd_wrapper, + &highbd_wrapper, TX_4X4, 16, 10, 2), + make_tuple(&aom_highbd_fdct4x4_c, + &highbd_wrapper, + &highbd_wrapper, TX_4X4, 16, 12, 2), + make_tuple(&aom_highbd_fdct4x4_c, + &highbd_wrapper, + &highbd_wrapper, TX_4X4, 1, 8, 2), + make_tuple(&aom_highbd_fdct4x4_c, + &highbd_wrapper, + &highbd_wrapper, TX_4X4, 1, 10, 2), + make_tuple(&aom_highbd_fdct4x4_c, + &highbd_wrapper, + &highbd_wrapper, TX_4X4, 1, 12, 2), +#endif // CONFIG_HIGHBITDEPTH + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1024, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 135, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 34, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 256, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 38, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 10, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 1, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 64, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 12, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 1, 8, 1), + make_tuple(&aom_fdct4x4_c, &wrapper, + &wrapper, TX_4X4, 16, 8, 1), + make_tuple(&aom_fdct4x4_c, &wrapper, + &wrapper, TX_4X4, 1, 8, 1) +}; + +INSTANTIATE_TEST_CASE_P(C, PartialIDctTest, + ::testing::ValuesIn(c_partial_idct_tests)); + +#if HAVE_NEON && !CONFIG_HIGHBITDEPTH +const PartialInvTxfmParam neon_partial_idct_tests[] = { + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 10, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 1, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 12, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 1, 8, 1), + make_tuple(&aom_fdct4x4_c, &wrapper, + &wrapper, TX_4X4, 1, 8, 1) +}; + +INSTANTIATE_TEST_CASE_P(NEON, PartialIDctTest, + ::testing::ValuesIn(neon_partial_idct_tests)); +#endif // HAVE_NEON && !CONFIG_HIGHBITDEPTH + +#if HAVE_SSE2 +const PartialInvTxfmParam sse2_partial_idct_tests[] = { +#if CONFIG_HIGHBITDEPTH + make_tuple(&aom_highbd_fdct4x4_c, + &highbd_wrapper, + &highbd_wrapper, TX_4X4, 16, 8, 2), + make_tuple( + &aom_highbd_fdct4x4_c, &highbd_wrapper, + &highbd_wrapper, TX_4X4, 16, 10, 2), + make_tuple( + &aom_highbd_fdct4x4_c, &highbd_wrapper, + &highbd_wrapper, TX_4X4, 16, 12, 2), +#endif // CONFIG_HIGHBITDEPTH + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1024, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 135, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 34, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 256, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 10, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 1, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 64, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 12, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 1, 8, 1), + make_tuple(&aom_fdct4x4_c, &wrapper, + &wrapper, TX_4X4, 16, 8, 1), + make_tuple(&aom_fdct4x4_c, &wrapper, + &wrapper, TX_4X4, 1, 8, 1) +}; + +INSTANTIATE_TEST_CASE_P(SSE2, PartialIDctTest, + ::testing::ValuesIn(sse2_partial_idct_tests)); + +#endif // HAVE_SSE2 + +#if HAVE_SSSE3 +const PartialInvTxfmParam ssse3_partial_idct_tests[] = { + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1024, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 135, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 34, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 64, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 12, 8, 1) +}; + +INSTANTIATE_TEST_CASE_P(SSSE3, PartialIDctTest, + ::testing::ValuesIn(ssse3_partial_idct_tests)); +#endif // HAVE_SSSE3 + +#if HAVE_DSPR2 && !CONFIG_HIGHBITDEPTH +const PartialInvTxfmParam dspr2_partial_idct_tests[] = { + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1024, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 135, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 34, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 256, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 10, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 1, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 64, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 12, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 1, 8, 1), + make_tuple(&aom_fdct4x4_c, &wrapper, + &wrapper, TX_4X4, 16, 8, 1), + make_tuple(&aom_fdct4x4_c, &wrapper, + &wrapper, TX_4X4, 1, 8, 1) +}; + +INSTANTIATE_TEST_CASE_P(DSPR2, PartialIDctTest, + ::testing::ValuesIn(dspr2_partial_idct_tests)); +#endif // HAVE_DSPR2 && !CONFIG_HIGHBITDEPTH + +#if HAVE_MSA && !CONFIG_HIGHBITDEPTH +const PartialInvTxfmParam msa_partial_idct_tests[] = { + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1024, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 135, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 34, 8, 1), + make_tuple(&aom_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 256, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 10, 8, 1), + make_tuple(&aom_fdct16x16_c, &wrapper, + &wrapper, TX_16X16, 1, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 64, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 12, 8, 1), + make_tuple(&aom_fdct8x8_c, &wrapper, + &wrapper, TX_8X8, 1, 8, 1), + make_tuple(&aom_fdct4x4_c, &wrapper, + &wrapper, TX_4X4, 16, 8, 1), + make_tuple(&aom_fdct4x4_c, &wrapper, + &wrapper, TX_4X4, 1, 8, 1) +}; + +INSTANTIATE_TEST_CASE_P(MSA, PartialIDctTest, + ::testing::ValuesIn(msa_partial_idct_tests)); +#endif // HAVE_MSA && !CONFIG_HIGHBITDEPTH + +} // namespace diff --git a/third_party/aom/test/quantize_test.cc b/third_party/aom/test/quantize_test.cc new file mode 100644 index 0000000000..4f61484a26 --- /dev/null +++ b/third_party/aom/test/quantize_test.cc @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "vp8/common/blockd.h" +#include "vp8/common/onyx.h" +#include "vp8/encoder/block.h" +#include "vp8/encoder/onyx_int.h" +#include "vp8/encoder/quantize.h" +#include "aom/aom_integer.h" +#include "aom_mem/aom_mem.h" + +namespace { +#if !CONFIG_AOM_QM + +const int kNumBlocks = 25; +const int kNumBlockEntries = 16; + +typedef void (*VP8Quantize)(BLOCK *b, BLOCKD *d); + +typedef std::tr1::tuple VP8QuantizeParam; + +using libaom_test::ACMRandom; +using std::tr1::make_tuple; + +// Create and populate a VP8_COMP instance which has a complete set of +// quantization inputs as well as a second MACROBLOCKD for output. +class QuantizeTestBase { + public: + virtual ~QuantizeTestBase() { + vp8_remove_compressor(&vp8_comp_); + vp8_comp_ = NULL; + aom_free(macroblockd_dst_); + macroblockd_dst_ = NULL; + libaom_test::ClearSystemState(); + } + + protected: + void SetupCompressor() { + rnd_.Reset(ACMRandom::DeterministicSeed()); + + // The full configuration is necessary to generate the quantization tables. + VP8_CONFIG vp8_config; + memset(&vp8_config, 0, sizeof(vp8_config)); + + vp8_comp_ = vp8_create_compressor(&vp8_config); + + // Set the tables based on a quantizer of 0. + vp8_set_quantizer(vp8_comp_, 0); + + // Set up all the block/blockd pointers for the mb in vp8_comp_. + vp8cx_frame_init_quantizer(vp8_comp_); + + // Copy macroblockd from the reference to get pre-set-up dequant values. + macroblockd_dst_ = reinterpret_cast( + aom_memalign(32, sizeof(*macroblockd_dst_))); + memcpy(macroblockd_dst_, &vp8_comp_->mb.e_mbd, sizeof(*macroblockd_dst_)); + // Fix block pointers - currently they point to the blocks in the reference + // structure. + vp8_setup_block_dptrs(macroblockd_dst_); + } + + void UpdateQuantizer(int q) { + vp8_set_quantizer(vp8_comp_, q); + + memcpy(macroblockd_dst_, &vp8_comp_->mb.e_mbd, sizeof(*macroblockd_dst_)); + vp8_setup_block_dptrs(macroblockd_dst_); + } + + void FillCoeffConstant(int16_t c) { + for (int i = 0; i < kNumBlocks * kNumBlockEntries; ++i) { + vp8_comp_->mb.coeff[i] = c; + } + } + + void FillCoeffRandom() { + for (int i = 0; i < kNumBlocks * kNumBlockEntries; ++i) { + vp8_comp_->mb.coeff[i] = rnd_.Rand8(); + } + } + + void CheckOutput() { + EXPECT_EQ(0, memcmp(vp8_comp_->mb.e_mbd.qcoeff, macroblockd_dst_->qcoeff, + sizeof(*macroblockd_dst_->qcoeff) * kNumBlocks * + kNumBlockEntries)) + << "qcoeff mismatch"; + EXPECT_EQ(0, memcmp(vp8_comp_->mb.e_mbd.dqcoeff, macroblockd_dst_->dqcoeff, + sizeof(*macroblockd_dst_->dqcoeff) * kNumBlocks * + kNumBlockEntries)) + << "dqcoeff mismatch"; + EXPECT_EQ(0, memcmp(vp8_comp_->mb.e_mbd.eobs, macroblockd_dst_->eobs, + sizeof(*macroblockd_dst_->eobs) * kNumBlocks)) + << "eobs mismatch"; + } + + VP8_COMP *vp8_comp_; + MACROBLOCKD *macroblockd_dst_; + + private: + ACMRandom rnd_; +}; + +class QuantizeTest : public QuantizeTestBase, + public ::testing::TestWithParam { + protected: + virtual void SetUp() { + SetupCompressor(); + asm_quant_ = GET_PARAM(0); + c_quant_ = GET_PARAM(1); + } + + void RunComparison() { + for (int i = 0; i < kNumBlocks; ++i) { + ASM_REGISTER_STATE_CHECK( + c_quant_(&vp8_comp_->mb.block[i], &vp8_comp_->mb.e_mbd.block[i])); + ASM_REGISTER_STATE_CHECK( + asm_quant_(&vp8_comp_->mb.block[i], ¯oblockd_dst_->block[i])); + } + + CheckOutput(); + } + + private: + VP8Quantize asm_quant_; + VP8Quantize c_quant_; +}; + +TEST_P(QuantizeTest, TestZeroInput) { + FillCoeffConstant(0); + RunComparison(); +} + +TEST_P(QuantizeTest, TestLargeNegativeInput) { + FillCoeffConstant(0); + // Generate a qcoeff which contains 512/-512 (0x0100/0xFE00) to catch issues + // like BUG=883 where the constant being compared was incorrectly initialized. + vp8_comp_->mb.coeff[0] = -8191; + RunComparison(); +} + +TEST_P(QuantizeTest, TestRandomInput) { + FillCoeffRandom(); + RunComparison(); +} + +TEST_P(QuantizeTest, TestMultipleQ) { + for (int q = 0; q < QINDEX_RANGE; ++q) { + UpdateQuantizer(q); + FillCoeffRandom(); + RunComparison(); + } +} + +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P( + SSE2, QuantizeTest, + ::testing::Values( + make_tuple(&vp8_fast_quantize_b_sse2, &vp8_fast_quantize_b_c), + make_tuple(&vp8_regular_quantize_b_sse2, &vp8_regular_quantize_b_c))); +#endif // HAVE_SSE2 + +#if HAVE_SSSE3 +INSTANTIATE_TEST_CASE_P(SSSE3, QuantizeTest, + ::testing::Values(make_tuple(&vp8_fast_quantize_b_ssse3, + &vp8_fast_quantize_b_c))); +#endif // HAVE_SSSE3 + +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P( + SSE4_1, QuantizeTest, + ::testing::Values(make_tuple(&vp8_regular_quantize_b_sse4_1, + &vp8_regular_quantize_b_c))); +#endif // HAVE_SSE4_1 + +#if HAVE_NEON +INSTANTIATE_TEST_CASE_P(NEON, QuantizeTest, + ::testing::Values(make_tuple(&vp8_fast_quantize_b_neon, + &vp8_fast_quantize_b_c))); +#endif // HAVE_NEON + +#if HAVE_MSA +INSTANTIATE_TEST_CASE_P( + MSA, QuantizeTest, + ::testing::Values( + make_tuple(&vp8_fast_quantize_b_msa, &vp8_fast_quantize_b_c), + make_tuple(&vp8_regular_quantize_b_msa, &vp8_regular_quantize_b_c))); +#endif // HAVE_MSA +#endif // CONFIG_AOM_QM +} // namespace diff --git a/third_party/aom/test/realtime_test.cc b/third_party/aom/test/realtime_test.cc new file mode 100644 index 0000000000..ffe4a3146b --- /dev/null +++ b/third_party/aom/test/realtime_test.cc @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/util.h" +#include "test/video_source.h" +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +namespace { + +const int kVideoSourceWidth = 320; +const int kVideoSourceHeight = 240; +const int kFramesToEncode = 2; + +class RealtimeTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWithParam { + protected: + RealtimeTest() : EncoderTest(GET_PARAM(0)), frame_packets_(0) {} + virtual ~RealtimeTest() {} + + virtual void SetUp() { + InitializeConfig(); + cfg_.g_lag_in_frames = 0; + SetMode(::libaom_test::kRealTime); + } + + virtual void BeginPassHook(unsigned int /*pass*/) { + // TODO(tomfinegan): We're changing the pass value here to make sure + // we get frames when real time mode is combined with |g_pass| set to + // AOM_RC_FIRST_PASS. This is necessary because EncoderTest::RunLoop() sets + // the pass value based on the mode passed into EncoderTest::SetMode(), + // which overrides the one specified in SetUp() above. + cfg_.g_pass = AOM_RC_FIRST_PASS; + } + virtual void FramePktHook(const aom_codec_cx_pkt_t * /*pkt*/) { + frame_packets_++; + } + + int frame_packets_; +}; + +TEST_P(RealtimeTest, RealtimeFirstPassProducesFrames) { + ::libaom_test::RandomVideoSource video; + video.SetSize(kVideoSourceWidth, kVideoSourceHeight); + video.set_limit(kFramesToEncode); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + EXPECT_EQ(kFramesToEncode, frame_packets_); +} + +AV1_INSTANTIATE_TEST_CASE(RealtimeTest, + ::testing::Values(::libaom_test::kRealTime)); + +} // namespace diff --git a/third_party/aom/test/register_state_check.h b/third_party/aom/test/register_state_check.h new file mode 100644 index 0000000000..3308201739 --- /dev/null +++ b/third_party/aom/test/register_state_check.h @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef TEST_REGISTER_STATE_CHECK_H_ +#define TEST_REGISTER_STATE_CHECK_H_ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "./aom_config.h" +#include "aom/aom_integer.h" + +// ASM_REGISTER_STATE_CHECK(asm_function) +// Minimally validates the environment pre & post function execution. This +// variant should be used with assembly functions which are not expected to +// fully restore the system state. See platform implementations of +// RegisterStateCheck for details. +// +// API_REGISTER_STATE_CHECK(api_function) +// Performs all the checks done by ASM_REGISTER_STATE_CHECK() and any +// additional checks to ensure the environment is in a consistent state pre & +// post function execution. This variant should be used with API functions. +// See platform implementations of RegisterStateCheckXXX for details. +// + +#if defined(_WIN64) + +#undef NOMINMAX +#define NOMINMAX +#define WIN32_LEAN_AND_MEAN +#include +#include + +inline bool operator==(const M128A &lhs, const M128A &rhs) { + return (lhs.Low == rhs.Low && lhs.High == rhs.High); +} + +namespace libaom_test { + +// Compares the state of xmm[6-15] at construction with their state at +// destruction. These registers should be preserved by the callee on +// Windows x64. +class RegisterStateCheck { + public: + RegisterStateCheck() { initialized_ = StoreRegisters(&pre_context_); } + ~RegisterStateCheck() { EXPECT_TRUE(Check()); } + + private: + static bool StoreRegisters(CONTEXT *const context) { + const HANDLE this_thread = GetCurrentThread(); + EXPECT_TRUE(this_thread != NULL); + context->ContextFlags = CONTEXT_FLOATING_POINT; + const bool context_saved = GetThreadContext(this_thread, context) == TRUE; + EXPECT_TRUE(context_saved) << "GetLastError: " << GetLastError(); + return context_saved; + } + + // Compares the register state. Returns true if the states match. + bool Check() const { + if (!initialized_) return false; + CONTEXT post_context; + if (!StoreRegisters(&post_context)) return false; + + const M128A *xmm_pre = &pre_context_.Xmm6; + const M128A *xmm_post = &post_context.Xmm6; + for (int i = 6; i <= 15; ++i) { + EXPECT_EQ(*xmm_pre, *xmm_post) << "xmm" << i << " has been modified!"; + ++xmm_pre; + ++xmm_post; + } + return !testing::Test::HasNonfatalFailure(); + } + + bool initialized_; + CONTEXT pre_context_; +}; + +#define ASM_REGISTER_STATE_CHECK(statement) \ + do { \ + libaom_test::RegisterStateCheck reg_check; \ + statement; \ + } while (false) + +} // namespace libaom_test + +#elif defined(CONFIG_SHARED) && defined(HAVE_NEON_ASM) && !CONFIG_SHARED && \ + HAVE_NEON_ASM && CONFIG_AV1 + +extern "C" { +// Save the d8-d15 registers into store. +void aom_push_neon(int64_t *store); +} + +namespace libaom_test { + +// Compares the state of d8-d15 at construction with their state at +// destruction. These registers should be preserved by the callee on +// arm platform. +class RegisterStateCheck { + public: + RegisterStateCheck() { initialized_ = StoreRegisters(pre_store_); } + ~RegisterStateCheck() { EXPECT_TRUE(Check()); } + + private: + static bool StoreRegisters(int64_t store[8]) { + aom_push_neon(store); + return true; + } + + // Compares the register state. Returns true if the states match. + bool Check() const { + if (!initialized_) return false; + int64_t post_store[8]; + aom_push_neon(post_store); + for (int i = 0; i < 8; ++i) { + EXPECT_EQ(pre_store_[i], post_store[i]) << "d" << i + 8 + << " has been modified"; + } + return !testing::Test::HasNonfatalFailure(); + } + + bool initialized_; + int64_t pre_store_[8]; +}; + +#define ASM_REGISTER_STATE_CHECK(statement) \ + do { \ + libaom_test::RegisterStateCheck reg_check; \ + statement; \ + } while (false) + +} // namespace libaom_test + +#else + +namespace libaom_test { + +class RegisterStateCheck {}; +#define ASM_REGISTER_STATE_CHECK(statement) statement + +} // namespace libaom_test + +#endif // _WIN64 + +#if ARCH_X86 || ARCH_X86_64 +#if defined(__GNUC__) + +namespace libaom_test { + +// Checks the FPU tag word pre/post execution to ensure emms has been called. +class RegisterStateCheckMMX { + public: + RegisterStateCheckMMX() { + __asm__ volatile("fstenv %0" : "=rm"(pre_fpu_env_)); + } + ~RegisterStateCheckMMX() { EXPECT_TRUE(Check()); } + + private: + // Checks the FPU tag word pre/post execution, returning false if not cleared + // to 0xffff. + bool Check() const { + EXPECT_EQ(0xffff, pre_fpu_env_[4]) + << "FPU was in an inconsistent state prior to call"; + + uint16_t post_fpu_env[14]; + __asm__ volatile("fstenv %0" : "=rm"(post_fpu_env)); + EXPECT_EQ(0xffff, post_fpu_env[4]) + << "FPU was left in an inconsistent state after call"; + return !testing::Test::HasNonfatalFailure(); + } + + uint16_t pre_fpu_env_[14]; +}; + +#define API_REGISTER_STATE_CHECK(statement) \ + do { \ + libaom_test::RegisterStateCheckMMX reg_check; \ + ASM_REGISTER_STATE_CHECK(statement); \ + } while (false) + +} // namespace libaom_test + +#endif // __GNUC__ +#endif // ARCH_X86 || ARCH_X86_64 + +#ifndef API_REGISTER_STATE_CHECK +#define API_REGISTER_STATE_CHECK ASM_REGISTER_STATE_CHECK +#endif + +#endif // TEST_REGISTER_STATE_CHECK_H_ diff --git a/third_party/aom/test/resize_test.cc b/third_party/aom/test/resize_test.cc new file mode 100644 index 0000000000..994b30117b --- /dev/null +++ b/third_party/aom/test/resize_test.cc @@ -0,0 +1,717 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/video_source.h" +#include "test/util.h" + +// Enable(1) or Disable(0) writing of the compressed bitstream. +#define WRITE_COMPRESSED_STREAM 0 + +namespace { + +#if WRITE_COMPRESSED_STREAM +static void mem_put_le16(char *const mem, unsigned int val) { + mem[0] = val; + mem[1] = val >> 8; +} + +static void mem_put_le32(char *const mem, unsigned int val) { + mem[0] = val; + mem[1] = val >> 8; + mem[2] = val >> 16; + mem[3] = val >> 24; +} + +static void write_ivf_file_header(const aom_codec_enc_cfg_t *const cfg, + int frame_cnt, FILE *const outfile) { + char header[32]; + + header[0] = 'D'; + header[1] = 'K'; + header[2] = 'I'; + header[3] = 'F'; + mem_put_le16(header + 4, 0); /* version */ + mem_put_le16(header + 6, 32); /* headersize */ + mem_put_le32(header + 8, 0x30395056); /* fourcc (av1) */ + mem_put_le16(header + 12, cfg->g_w); /* width */ + mem_put_le16(header + 14, cfg->g_h); /* height */ + mem_put_le32(header + 16, cfg->g_timebase.den); /* rate */ + mem_put_le32(header + 20, cfg->g_timebase.num); /* scale */ + mem_put_le32(header + 24, frame_cnt); /* length */ + mem_put_le32(header + 28, 0); /* unused */ + + (void)fwrite(header, 1, 32, outfile); +} + +static void write_ivf_frame_size(FILE *const outfile, const size_t size) { + char header[4]; + mem_put_le32(header, static_cast(size)); + (void)fwrite(header, 1, 4, outfile); +} + +static void write_ivf_frame_header(const aom_codec_cx_pkt_t *const pkt, + FILE *const outfile) { + char header[12]; + aom_codec_pts_t pts; + + if (pkt->kind != AOM_CODEC_CX_FRAME_PKT) return; + + pts = pkt->data.frame.pts; + mem_put_le32(header, static_cast(pkt->data.frame.sz)); + mem_put_le32(header + 4, pts & 0xFFFFFFFF); + mem_put_le32(header + 8, pts >> 32); + + (void)fwrite(header, 1, 12, outfile); +} +#endif // WRITE_COMPRESSED_STREAM + +const unsigned int kInitialWidth = 320; +const unsigned int kInitialHeight = 240; + +struct FrameInfo { + FrameInfo(aom_codec_pts_t _pts, unsigned int _w, unsigned int _h) + : pts(_pts), w(_w), h(_h) {} + + aom_codec_pts_t pts; + unsigned int w; + unsigned int h; +}; + +void ScaleForFrameNumber(unsigned int frame, unsigned int initial_w, + unsigned int initial_h, unsigned int *w, + unsigned int *h, int flag_codec) { + if (frame < 10) { + *w = initial_w; + *h = initial_h; + return; + } + if (frame < 20) { + *w = initial_w * 3 / 4; + *h = initial_h * 3 / 4; + return; + } + if (frame < 30) { + *w = initial_w / 2; + *h = initial_h / 2; + return; + } + if (frame < 40) { + *w = initial_w; + *h = initial_h; + return; + } + if (frame < 50) { + *w = initial_w * 3 / 4; + *h = initial_h * 3 / 4; + return; + } + if (frame < 60) { + *w = initial_w / 2; + *h = initial_h / 2; + return; + } + if (frame < 70) { + *w = initial_w; + *h = initial_h; + return; + } + if (frame < 80) { + *w = initial_w * 3 / 4; + *h = initial_h * 3 / 4; + return; + } + if (frame < 90) { + *w = initial_w / 2; + *h = initial_h / 2; + return; + } + if (frame < 100) { + *w = initial_w * 3 / 4; + *h = initial_h * 3 / 4; + return; + } + if (frame < 110) { + *w = initial_w; + *h = initial_h; + return; + } + if (frame < 120) { + *w = initial_w * 3 / 4; + *h = initial_h * 3 / 4; + return; + } + if (frame < 130) { + *w = initial_w / 2; + *h = initial_h / 2; + return; + } + if (frame < 140) { + *w = initial_w * 3 / 4; + *h = initial_h * 3 / 4; + return; + } + if (frame < 150) { + *w = initial_w; + *h = initial_h; + return; + } + if (frame < 160) { + *w = initial_w * 3 / 4; + *h = initial_h * 3 / 4; + return; + } + if (frame < 170) { + *w = initial_w / 2; + *h = initial_h / 2; + return; + } + if (frame < 180) { + *w = initial_w * 3 / 4; + *h = initial_h * 3 / 4; + return; + } + if (frame < 190) { + *w = initial_w; + *h = initial_h; + return; + } + if (frame < 200) { + *w = initial_w * 3 / 4; + *h = initial_h * 3 / 4; + return; + } + if (frame < 210) { + *w = initial_w / 2; + *h = initial_h / 2; + return; + } + if (frame < 220) { + *w = initial_w * 3 / 4; + *h = initial_h * 3 / 4; + return; + } + if (frame < 230) { + *w = initial_w; + *h = initial_h; + return; + } + if (frame < 240) { + *w = initial_w * 3 / 4; + *h = initial_h * 3 / 4; + return; + } + if (frame < 250) { + *w = initial_w / 2; + *h = initial_h / 2; + return; + } + if (frame < 260) { + *w = initial_w; + *h = initial_h; + return; + } + // Go down very low. + if (frame < 270) { + *w = initial_w / 4; + *h = initial_h / 4; + return; + } + if (flag_codec == 1) { + // Cases that only works for AV1. + // For AV1: Swap width and height of original. + if (frame < 320) { + *w = initial_h; + *h = initial_w; + return; + } + } + *w = initial_w; + *h = initial_h; +} + +class ResizingVideoSource : public ::libaom_test::DummyVideoSource { + public: + ResizingVideoSource() { + SetSize(kInitialWidth, kInitialHeight); + limit_ = 350; + } + int flag_codec_; + virtual ~ResizingVideoSource() {} + + protected: + virtual void Next() { + ++frame_; + unsigned int width; + unsigned int height; + ScaleForFrameNumber(frame_, kInitialWidth, kInitialHeight, &width, &height, + flag_codec_); + SetSize(width, height); + FillFrame(); + } +}; + +class ResizeTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWithParam { + protected: + ResizeTest() : EncoderTest(GET_PARAM(0)) {} + + virtual ~ResizeTest() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(GET_PARAM(1)); + } + + virtual void DecompressedFrameHook(const aom_image_t &img, + aom_codec_pts_t pts) { + frame_info_list_.push_back(FrameInfo(pts, img.d_w, img.d_h)); + } + + std::vector frame_info_list_; +}; + +TEST_P(ResizeTest, TestExternalResizeWorks) { + ResizingVideoSource video; + video.flag_codec_ = 0; + cfg_.g_lag_in_frames = 0; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + + for (std::vector::const_iterator info = frame_info_list_.begin(); + info != frame_info_list_.end(); ++info) { + const unsigned int frame = static_cast(info->pts); + unsigned int expected_w; + unsigned int expected_h; + ScaleForFrameNumber(frame, kInitialWidth, kInitialHeight, &expected_w, + &expected_h, 0); + EXPECT_EQ(expected_w, info->w) << "Frame " << frame + << " had unexpected width"; + EXPECT_EQ(expected_h, info->h) << "Frame " << frame + << " had unexpected height"; + } +} + +const unsigned int kStepDownFrame = 3; +const unsigned int kStepUpFrame = 6; + +class ResizeInternalTest : public ResizeTest { + protected: +#if WRITE_COMPRESSED_STREAM + ResizeInternalTest() + : ResizeTest(), frame0_psnr_(0.0), outfile_(NULL), out_frames_(0) {} +#else + ResizeInternalTest() : ResizeTest(), frame0_psnr_(0.0) {} +#endif + + virtual ~ResizeInternalTest() {} + + virtual void BeginPassHook(unsigned int /*pass*/) { +#if WRITE_COMPRESSED_STREAM + outfile_ = fopen("av10-2-05-resize.ivf", "wb"); +#endif + } + + virtual void EndPassHook() { +#if WRITE_COMPRESSED_STREAM + if (outfile_) { + if (!fseek(outfile_, 0, SEEK_SET)) + write_ivf_file_header(&cfg_, out_frames_, outfile_); + fclose(outfile_); + outfile_ = NULL; + } +#endif + } + + virtual void PreEncodeFrameHook(libaom_test::VideoSource *video, + libaom_test::Encoder *encoder) { + if (change_config_) { + int new_q = 60; + if (video->frame() == 0) { + struct aom_scaling_mode mode = { AOME_ONETWO, AOME_ONETWO }; + encoder->Control(AOME_SET_SCALEMODE, &mode); + } + if (video->frame() == 1) { + struct aom_scaling_mode mode = { AOME_NORMAL, AOME_NORMAL }; + encoder->Control(AOME_SET_SCALEMODE, &mode); + cfg_.rc_min_quantizer = cfg_.rc_max_quantizer = new_q; + encoder->Config(&cfg_); + } + } else { + if (video->frame() == kStepDownFrame) { + struct aom_scaling_mode mode = { AOME_FOURFIVE, AOME_THREEFIVE }; + encoder->Control(AOME_SET_SCALEMODE, &mode); + } + if (video->frame() == kStepUpFrame) { + struct aom_scaling_mode mode = { AOME_NORMAL, AOME_NORMAL }; + encoder->Control(AOME_SET_SCALEMODE, &mode); + } + } + } + + virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) { + if (frame0_psnr_ == 0.) frame0_psnr_ = pkt->data.psnr.psnr[0]; + EXPECT_NEAR(pkt->data.psnr.psnr[0], frame0_psnr_, 2.0); + } + +#if WRITE_COMPRESSED_STREAM + virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) { + ++out_frames_; + + // Write initial file header if first frame. + if (pkt->data.frame.pts == 0) write_ivf_file_header(&cfg_, 0, outfile_); + + // Write frame header and data. + write_ivf_frame_header(pkt, outfile_); + (void)fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_); + } +#endif + + double frame0_psnr_; + bool change_config_; +#if WRITE_COMPRESSED_STREAM + FILE *outfile_; + unsigned int out_frames_; +#endif +}; + +TEST_P(ResizeInternalTest, TestInternalResizeWorks) { + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 10); + init_flags_ = AOM_CODEC_USE_PSNR; + change_config_ = false; + + // q picked such that initial keyframe on this clip is ~30dB PSNR + cfg_.rc_min_quantizer = cfg_.rc_max_quantizer = 48; + + // If the number of frames being encoded is smaller than g_lag_in_frames + // the encoded frame is unavailable using the current API. Comparing + // frames to detect mismatch would then not be possible. Set + // g_lag_in_frames = 0 to get around this. + cfg_.g_lag_in_frames = 0; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + + for (std::vector::const_iterator info = frame_info_list_.begin(); + info != frame_info_list_.end(); ++info) { + const aom_codec_pts_t pts = info->pts; + if (pts >= kStepDownFrame && pts < kStepUpFrame) { + ASSERT_EQ(282U, info->w) << "Frame " << pts << " had unexpected width"; + ASSERT_EQ(173U, info->h) << "Frame " << pts << " had unexpected height"; + } else { + EXPECT_EQ(352U, info->w) << "Frame " << pts << " had unexpected width"; + EXPECT_EQ(288U, info->h) << "Frame " << pts << " had unexpected height"; + } + } +} + +TEST_P(ResizeInternalTest, TestInternalResizeChangeConfig) { + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 10); + cfg_.g_w = 352; + cfg_.g_h = 288; + change_config_ = true; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} + +class ResizeRealtimeTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params { + protected: + ResizeRealtimeTest() : EncoderTest(GET_PARAM(0)) {} + virtual ~ResizeRealtimeTest() {} + + virtual void PreEncodeFrameHook(libaom_test::VideoSource *video, + libaom_test::Encoder *encoder) { + if (video->frame() == 0) { + encoder->Control(AV1E_SET_AQ_MODE, 3); + encoder->Control(AOME_SET_CPUUSED, set_cpu_used_); + } + + if (change_bitrate_ && video->frame() == 120) { + change_bitrate_ = false; + cfg_.rc_target_bitrate = 500; + encoder->Config(&cfg_); + } + } + + virtual void SetUp() { + InitializeConfig(); + SetMode(GET_PARAM(1)); + set_cpu_used_ = GET_PARAM(2); + } + + virtual void DecompressedFrameHook(const aom_image_t &img, + aom_codec_pts_t pts) { + frame_info_list_.push_back(FrameInfo(pts, img.d_w, img.d_h)); + } + + virtual void MismatchHook(const aom_image_t *img1, const aom_image_t *img2) { + double mismatch_psnr = compute_psnr(img1, img2); + mismatch_psnr_ += mismatch_psnr; + ++mismatch_nframes_; + } + + unsigned int GetMismatchFrames() { return mismatch_nframes_; } + + void DefaultConfig() { + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 600; + cfg_.rc_buf_sz = 1000; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 56; + cfg_.rc_undershoot_pct = 50; + cfg_.rc_overshoot_pct = 50; + cfg_.rc_end_usage = AOM_CBR; + cfg_.kf_mode = AOM_KF_AUTO; + cfg_.g_lag_in_frames = 0; + cfg_.kf_min_dist = cfg_.kf_max_dist = 3000; + // Enable dropped frames. + cfg_.rc_dropframe_thresh = 1; + // Enable error_resilience mode. + cfg_.g_error_resilient = 1; + // Enable dynamic resizing. + cfg_.rc_resize_allowed = 1; + // Run at low bitrate. + cfg_.rc_target_bitrate = 200; + } + + std::vector frame_info_list_; + int set_cpu_used_; + bool change_bitrate_; + double mismatch_psnr_; + int mismatch_nframes_; +}; + +TEST_P(ResizeRealtimeTest, TestExternalResizeWorks) { + ResizingVideoSource video; + video.flag_codec_ = 1; + DefaultConfig(); + // Disable internal resize for this test. + cfg_.rc_resize_allowed = 0; + change_bitrate_ = false; + mismatch_psnr_ = 0.0; + mismatch_nframes_ = 0; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + + for (std::vector::const_iterator info = frame_info_list_.begin(); + info != frame_info_list_.end(); ++info) { + const unsigned int frame = static_cast(info->pts); + unsigned int expected_w; + unsigned int expected_h; + ScaleForFrameNumber(frame, kInitialWidth, kInitialHeight, &expected_w, + &expected_h, 1); + EXPECT_EQ(expected_w, info->w) << "Frame " << frame + << " had unexpected width"; + EXPECT_EQ(expected_h, info->h) << "Frame " << frame + << " had unexpected height"; + EXPECT_EQ(static_cast(0), GetMismatchFrames()); + } +} + +// Verify the dynamic resizer behavior for real time, 1 pass CBR mode. +// Run at low bitrate, with resize_allowed = 1, and verify that we get +// one resize down event. +TEST_P(ResizeRealtimeTest, TestInternalResizeDown) { + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 299); + DefaultConfig(); + cfg_.g_w = 352; + cfg_.g_h = 288; + change_bitrate_ = false; + mismatch_psnr_ = 0.0; + mismatch_nframes_ = 0; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + + unsigned int last_w = cfg_.g_w; + unsigned int last_h = cfg_.g_h; + int resize_count = 0; + for (std::vector::const_iterator info = frame_info_list_.begin(); + info != frame_info_list_.end(); ++info) { + if (info->w != last_w || info->h != last_h) { + // Verify that resize down occurs. + ASSERT_LT(info->w, last_w); + ASSERT_LT(info->h, last_h); + last_w = info->w; + last_h = info->h; + resize_count++; + } + } + +#if CONFIG_AV1_DECODER + // Verify that we get 1 resize down event in this test. + ASSERT_EQ(1, resize_count) << "Resizing should occur."; + EXPECT_EQ(static_cast(0), GetMismatchFrames()); +#else + printf("Warning: AV1 decoder unavailable, unable to check resize count!\n"); +#endif +} + +// Verify the dynamic resizer behavior for real time, 1 pass CBR mode. +// Start at low target bitrate, raise the bitrate in the middle of the clip, +// scaling-up should occur after bitrate changed. +TEST_P(ResizeRealtimeTest, TestInternalResizeDownUpChangeBitRate) { + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 359); + DefaultConfig(); + cfg_.g_w = 352; + cfg_.g_h = 288; + change_bitrate_ = true; + mismatch_psnr_ = 0.0; + mismatch_nframes_ = 0; + // Disable dropped frames. + cfg_.rc_dropframe_thresh = 0; + // Starting bitrate low. + cfg_.rc_target_bitrate = 80; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + + unsigned int last_w = cfg_.g_w; + unsigned int last_h = cfg_.g_h; + int resize_count = 0; + for (std::vector::const_iterator info = frame_info_list_.begin(); + info != frame_info_list_.end(); ++info) { + if (info->w != last_w || info->h != last_h) { + resize_count++; + if (resize_count == 1) { + // Verify that resize down occurs. + ASSERT_LT(info->w, last_w); + ASSERT_LT(info->h, last_h); + } else if (resize_count == 2) { + // Verify that resize up occurs. + ASSERT_GT(info->w, last_w); + ASSERT_GT(info->h, last_h); + } + last_w = info->w; + last_h = info->h; + } + } + +#if CONFIG_AV1_DECODER + // Verify that we get 2 resize events in this test. + ASSERT_EQ(resize_count, 2) << "Resizing should occur twice."; + EXPECT_EQ(static_cast(0), GetMismatchFrames()); +#else + printf("Warning: AV1 decoder unavailable, unable to check resize count!\n"); +#endif +} + +aom_img_fmt_t CspForFrameNumber(int frame) { + if (frame < 10) return AOM_IMG_FMT_I420; + if (frame < 20) return AOM_IMG_FMT_I444; + return AOM_IMG_FMT_I420; +} + +class ResizeCspTest : public ResizeTest { + protected: +#if WRITE_COMPRESSED_STREAM + ResizeCspTest() + : ResizeTest(), frame0_psnr_(0.0), outfile_(NULL), out_frames_(0) {} +#else + ResizeCspTest() : ResizeTest(), frame0_psnr_(0.0) {} +#endif + + virtual ~ResizeCspTest() {} + + virtual void BeginPassHook(unsigned int /*pass*/) { +#if WRITE_COMPRESSED_STREAM + outfile_ = fopen("av11-2-05-cspchape.ivf", "wb"); +#endif + } + + virtual void EndPassHook() { +#if WRITE_COMPRESSED_STREAM + if (outfile_) { + if (!fseek(outfile_, 0, SEEK_SET)) + write_ivf_file_header(&cfg_, out_frames_, outfile_); + fclose(outfile_); + outfile_ = NULL; + } +#endif + } + + virtual void PreEncodeFrameHook(libaom_test::VideoSource *video, + libaom_test::Encoder *encoder) { + if (CspForFrameNumber(video->frame()) != AOM_IMG_FMT_I420 && + cfg_.g_profile != 1) { + cfg_.g_profile = 1; + encoder->Config(&cfg_); + } + if (CspForFrameNumber(video->frame()) == AOM_IMG_FMT_I420 && + cfg_.g_profile != 0) { + cfg_.g_profile = 0; + encoder->Config(&cfg_); + } + } + + virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) { + if (frame0_psnr_ == 0.) frame0_psnr_ = pkt->data.psnr.psnr[0]; + EXPECT_NEAR(pkt->data.psnr.psnr[0], frame0_psnr_, 2.0); + } + +#if WRITE_COMPRESSED_STREAM + virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) { + ++out_frames_; + + // Write initial file header if first frame. + if (pkt->data.frame.pts == 0) write_ivf_file_header(&cfg_, 0, outfile_); + + // Write frame header and data. + write_ivf_frame_header(pkt, outfile_); + (void)fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_); + } +#endif + + double frame0_psnr_; +#if WRITE_COMPRESSED_STREAM + FILE *outfile_; + unsigned int out_frames_; +#endif +}; + +class ResizingCspVideoSource : public ::libaom_test::DummyVideoSource { + public: + ResizingCspVideoSource() { + SetSize(kInitialWidth, kInitialHeight); + limit_ = 30; + } + + virtual ~ResizingCspVideoSource() {} + + protected: + virtual void Next() { + ++frame_; + SetImageFormat(CspForFrameNumber(frame_)); + FillFrame(); + } +}; + +TEST_P(ResizeCspTest, TestResizeCspWorks) { + ResizingCspVideoSource video; + init_flags_ = AOM_CODEC_USE_PSNR; + cfg_.rc_min_quantizer = cfg_.rc_max_quantizer = 48; + cfg_.g_lag_in_frames = 0; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} + +AV1_INSTANTIATE_TEST_CASE(ResizeTest, + ::testing::Values(::libaom_test::kRealTime)); +AV1_INSTANTIATE_TEST_CASE(ResizeInternalTest, + ::testing::Values(::libaom_test::kOnePassBest)); +AV1_INSTANTIATE_TEST_CASE(ResizeRealtimeTest, + ::testing::Values(::libaom_test::kRealTime), + ::testing::Range(5, 9)); +AV1_INSTANTIATE_TEST_CASE(ResizeCspTest, + ::testing::Values(::libaom_test::kRealTime)); +} // namespace diff --git a/third_party/aom/test/sad_test.cc b/third_party/aom/test/sad_test.cc new file mode 100644 index 0000000000..c3b5dac422 --- /dev/null +++ b/third_party/aom/test/sad_test.cc @@ -0,0 +1,1172 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "aom/aom_codec.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +typedef unsigned int (*SadMxNFunc)(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride); +typedef std::tr1::tuple SadMxNParam; + +typedef uint32_t (*SadMxNAvgFunc)(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + const uint8_t *second_pred); +typedef std::tr1::tuple SadMxNAvgParam; + +typedef void (*SadMxNx4Func)(const uint8_t *src_ptr, int src_stride, + const uint8_t *const ref_ptr[], int ref_stride, + uint32_t *sad_array); +typedef std::tr1::tuple SadMxNx4Param; + +using libaom_test::ACMRandom; + +namespace { +class SADTestBase : public ::testing::Test { + public: + SADTestBase(int width, int height, int bit_depth) + : width_(width), height_(height), bd_(bit_depth) {} + + static void SetUpTestCase() { + source_data8_ = reinterpret_cast( + aom_memalign(kDataAlignment, kDataBlockSize)); + reference_data8_ = reinterpret_cast( + aom_memalign(kDataAlignment, kDataBufferSize)); + second_pred8_ = + reinterpret_cast(aom_memalign(kDataAlignment, 128 * 128)); + source_data16_ = reinterpret_cast( + aom_memalign(kDataAlignment, kDataBlockSize * sizeof(uint16_t))); + reference_data16_ = reinterpret_cast( + aom_memalign(kDataAlignment, kDataBufferSize * sizeof(uint16_t))); + second_pred16_ = reinterpret_cast( + aom_memalign(kDataAlignment, 128 * 128 * sizeof(uint16_t))); + } + + static void TearDownTestCase() { + aom_free(source_data8_); + source_data8_ = NULL; + aom_free(reference_data8_); + reference_data8_ = NULL; + aom_free(second_pred8_); + second_pred8_ = NULL; + aom_free(source_data16_); + source_data16_ = NULL; + aom_free(reference_data16_); + reference_data16_ = NULL; + aom_free(second_pred16_); + second_pred16_ = NULL; + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + // Handle up to 4 128x128 blocks, with stride up to 256 + static const int kDataAlignment = 16; + static const int kDataBlockSize = 128 * 256; + static const int kDataBufferSize = 4 * kDataBlockSize; + + virtual void SetUp() { + if (bd_ == -1) { + use_high_bit_depth_ = false; + bit_depth_ = AOM_BITS_8; + source_data_ = source_data8_; + reference_data_ = reference_data8_; + second_pred_ = second_pred8_; +#if CONFIG_HIGHBITDEPTH + } else { + use_high_bit_depth_ = true; + bit_depth_ = static_cast(bd_); + source_data_ = CONVERT_TO_BYTEPTR(source_data16_); + reference_data_ = CONVERT_TO_BYTEPTR(reference_data16_); + second_pred_ = CONVERT_TO_BYTEPTR(second_pred16_); +#endif // CONFIG_HIGHBITDEPTH + } + mask_ = (1 << bit_depth_) - 1; + source_stride_ = (width_ + 31) & ~31; + reference_stride_ = width_ * 2; + rnd_.Reset(ACMRandom::DeterministicSeed()); + } + + virtual uint8_t *GetReference(int block_idx) { +#if CONFIG_HIGHBITDEPTH + if (use_high_bit_depth_) + return CONVERT_TO_BYTEPTR(CONVERT_TO_SHORTPTR(reference_data_) + + block_idx * kDataBlockSize); +#endif // CONFIG_HIGHBITDEPTH + return reference_data_ + block_idx * kDataBlockSize; + } + + // Sum of Absolute Differences. Given two blocks, calculate the absolute + // difference between two pixels in the same relative location; accumulate. + unsigned int ReferenceSAD(int block_idx) { + unsigned int sad = 0; + const uint8_t *const reference8 = GetReference(block_idx); + const uint8_t *const source8 = source_data_; +#if CONFIG_HIGHBITDEPTH + const uint16_t *const reference16 = + CONVERT_TO_SHORTPTR(GetReference(block_idx)); + const uint16_t *const source16 = CONVERT_TO_SHORTPTR(source_data_); +#endif // CONFIG_HIGHBITDEPTH + for (int h = 0; h < height_; ++h) { + for (int w = 0; w < width_; ++w) { + if (!use_high_bit_depth_) { + sad += abs(source8[h * source_stride_ + w] - + reference8[h * reference_stride_ + w]); +#if CONFIG_HIGHBITDEPTH + } else { + sad += abs(source16[h * source_stride_ + w] - + reference16[h * reference_stride_ + w]); +#endif // CONFIG_HIGHBITDEPTH + } + } + } + return sad; + } + + // Sum of Absolute Differences Average. Given two blocks, and a prediction + // calculate the absolute difference between one pixel and average of the + // corresponding and predicted pixels; accumulate. + unsigned int ReferenceSADavg(int block_idx) { + unsigned int sad = 0; + const uint8_t *const reference8 = GetReference(block_idx); + const uint8_t *const source8 = source_data_; + const uint8_t *const second_pred8 = second_pred_; +#if CONFIG_HIGHBITDEPTH + const uint16_t *const reference16 = + CONVERT_TO_SHORTPTR(GetReference(block_idx)); + const uint16_t *const source16 = CONVERT_TO_SHORTPTR(source_data_); + const uint16_t *const second_pred16 = CONVERT_TO_SHORTPTR(second_pred_); +#endif // CONFIG_HIGHBITDEPTH + for (int h = 0; h < height_; ++h) { + for (int w = 0; w < width_; ++w) { + if (!use_high_bit_depth_) { + const int tmp = second_pred8[h * width_ + w] + + reference8[h * reference_stride_ + w]; + const uint8_t comp_pred = ROUND_POWER_OF_TWO(tmp, 1); + sad += abs(source8[h * source_stride_ + w] - comp_pred); +#if CONFIG_HIGHBITDEPTH + } else { + const int tmp = second_pred16[h * width_ + w] + + reference16[h * reference_stride_ + w]; + const uint16_t comp_pred = ROUND_POWER_OF_TWO(tmp, 1); + sad += abs(source16[h * source_stride_ + w] - comp_pred); +#endif // CONFIG_HIGHBITDEPTH + } + } + } + return sad; + } + + void FillConstant(uint8_t *data, int stride, uint16_t fill_constant) { + uint8_t *data8 = data; +#if CONFIG_HIGHBITDEPTH + uint16_t *data16 = CONVERT_TO_SHORTPTR(data); +#endif // CONFIG_HIGHBITDEPTH + for (int h = 0; h < height_; ++h) { + for (int w = 0; w < width_; ++w) { + if (!use_high_bit_depth_) { + data8[h * stride + w] = static_cast(fill_constant); +#if CONFIG_HIGHBITDEPTH + } else { + data16[h * stride + w] = fill_constant; +#endif // CONFIG_HIGHBITDEPTH + } + } + } + } + + void FillRandom(uint8_t *data, int stride) { + uint8_t *data8 = data; +#if CONFIG_HIGHBITDEPTH + uint16_t *data16 = CONVERT_TO_SHORTPTR(data); +#endif // CONFIG_HIGHBITDEPTH + for (int h = 0; h < height_; ++h) { + for (int w = 0; w < width_; ++w) { + if (!use_high_bit_depth_) { + data8[h * stride + w] = rnd_.Rand8(); +#if CONFIG_HIGHBITDEPTH + } else { + data16[h * stride + w] = rnd_.Rand16() & mask_; +#endif // CONFIG_HIGHBITDEPTH + } + } + } + } + + int width_, height_, mask_, bd_; + aom_bit_depth_t bit_depth_; + static uint8_t *source_data_; + static uint8_t *reference_data_; + static uint8_t *second_pred_; + int source_stride_; + bool use_high_bit_depth_; + static uint8_t *source_data8_; + static uint8_t *reference_data8_; + static uint8_t *second_pred8_; + static uint16_t *source_data16_; + static uint16_t *reference_data16_; + static uint16_t *second_pred16_; + int reference_stride_; + + ACMRandom rnd_; +}; + +class SADx4Test : public SADTestBase, + public ::testing::WithParamInterface { + public: + SADx4Test() : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {} + + protected: + void SADs(unsigned int *results) { + const uint8_t *references[] = { GetReference(0), GetReference(1), + GetReference(2), GetReference(3) }; + + ASM_REGISTER_STATE_CHECK(GET_PARAM(2)( + source_data_, source_stride_, references, reference_stride_, results)); + } + + void CheckSADs() { + unsigned int reference_sad, exp_sad[4]; + + SADs(exp_sad); + for (int block = 0; block < 4; ++block) { + reference_sad = ReferenceSAD(block); + + EXPECT_EQ(reference_sad, exp_sad[block]) << "block " << block; + } + } +}; + +class SADTest : public SADTestBase, + public ::testing::WithParamInterface { + public: + SADTest() : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {} + + protected: + unsigned int SAD(int block_idx) { + unsigned int ret; + const uint8_t *const reference = GetReference(block_idx); + + ASM_REGISTER_STATE_CHECK(ret = GET_PARAM(2)(source_data_, source_stride_, + reference, reference_stride_)); + return ret; + } + + void CheckSAD() { + const unsigned int reference_sad = ReferenceSAD(0); + const unsigned int exp_sad = SAD(0); + + ASSERT_EQ(reference_sad, exp_sad); + } + + void SpeedSAD() { + int test_count = 20000000; + while (test_count > 0) { + SAD(0); + test_count -= 1; + } + } +}; + +class SADavgTest : public SADTestBase, + public ::testing::WithParamInterface { + public: + SADavgTest() : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {} + + protected: + unsigned int SAD_avg(int block_idx) { + unsigned int ret; + const uint8_t *const reference = GetReference(block_idx); + + ASM_REGISTER_STATE_CHECK(ret = GET_PARAM(2)(source_data_, source_stride_, + reference, reference_stride_, + second_pred_)); + return ret; + } + + void CheckSAD() { + const unsigned int reference_sad = ReferenceSADavg(0); + const unsigned int exp_sad = SAD_avg(0); + + ASSERT_EQ(reference_sad, exp_sad); + } +}; + +uint8_t *SADTestBase::source_data_ = NULL; +uint8_t *SADTestBase::reference_data_ = NULL; +uint8_t *SADTestBase::second_pred_ = NULL; +uint8_t *SADTestBase::source_data8_ = NULL; +uint8_t *SADTestBase::reference_data8_ = NULL; +uint8_t *SADTestBase::second_pred8_ = NULL; +uint16_t *SADTestBase::source_data16_ = NULL; +uint16_t *SADTestBase::reference_data16_ = NULL; +uint16_t *SADTestBase::second_pred16_ = NULL; + +TEST_P(SADTest, MaxRef) { + FillConstant(source_data_, source_stride_, 0); + FillConstant(reference_data_, reference_stride_, mask_); + CheckSAD(); +} + +TEST_P(SADTest, MaxSrc) { + FillConstant(source_data_, source_stride_, mask_); + FillConstant(reference_data_, reference_stride_, 0); + CheckSAD(); +} + +TEST_P(SADTest, ShortRef) { + const int tmp_stride = reference_stride_; + reference_stride_ >>= 1; + FillRandom(source_data_, source_stride_); + FillRandom(reference_data_, reference_stride_); + CheckSAD(); + reference_stride_ = tmp_stride; +} + +TEST_P(SADTest, UnalignedRef) { + // The reference frame, but not the source frame, may be unaligned for + // certain types of searches. + const int tmp_stride = reference_stride_; + reference_stride_ -= 1; + FillRandom(source_data_, source_stride_); + FillRandom(reference_data_, reference_stride_); + CheckSAD(); + reference_stride_ = tmp_stride; +} + +TEST_P(SADTest, ShortSrc) { + const int tmp_stride = source_stride_; + source_stride_ >>= 1; + int test_count = 2000; + while (test_count > 0) { + FillRandom(source_data_, source_stride_); + FillRandom(reference_data_, reference_stride_); + CheckSAD(); + test_count -= 1; + } + source_stride_ = tmp_stride; +} + +#define SPEED_TEST (0) +#if SPEED_TEST +TEST_P(SADTest, Speed) { + const int tmp_stride = source_stride_; + source_stride_ >>= 1; + FillRandom(source_data_, source_stride_); + FillRandom(reference_data_, reference_stride_); + SpeedSAD(); + source_stride_ = tmp_stride; +} +#endif + +TEST_P(SADavgTest, MaxRef) { + FillConstant(source_data_, source_stride_, 0); + FillConstant(reference_data_, reference_stride_, mask_); + FillConstant(second_pred_, width_, 0); + CheckSAD(); +} +TEST_P(SADavgTest, MaxSrc) { + FillConstant(source_data_, source_stride_, mask_); + FillConstant(reference_data_, reference_stride_, 0); + FillConstant(second_pred_, width_, 0); + CheckSAD(); +} + +TEST_P(SADavgTest, ShortRef) { + const int tmp_stride = reference_stride_; + reference_stride_ >>= 1; + FillRandom(source_data_, source_stride_); + FillRandom(reference_data_, reference_stride_); + FillRandom(second_pred_, width_); + CheckSAD(); + reference_stride_ = tmp_stride; +} + +TEST_P(SADavgTest, UnalignedRef) { + // The reference frame, but not the source frame, may be unaligned for + // certain types of searches. + const int tmp_stride = reference_stride_; + reference_stride_ -= 1; + FillRandom(source_data_, source_stride_); + FillRandom(reference_data_, reference_stride_); + FillRandom(second_pred_, width_); + CheckSAD(); + reference_stride_ = tmp_stride; +} + +TEST_P(SADavgTest, ShortSrc) { + const int tmp_stride = source_stride_; + source_stride_ >>= 1; + int test_count = 2000; + while (test_count > 0) { + FillRandom(source_data_, source_stride_); + FillRandom(reference_data_, reference_stride_); + FillRandom(second_pred_, width_); + CheckSAD(); + test_count -= 1; + } + source_stride_ = tmp_stride; +} + +TEST_P(SADx4Test, MaxRef) { + FillConstant(source_data_, source_stride_, 0); + FillConstant(GetReference(0), reference_stride_, mask_); + FillConstant(GetReference(1), reference_stride_, mask_); + FillConstant(GetReference(2), reference_stride_, mask_); + FillConstant(GetReference(3), reference_stride_, mask_); + CheckSADs(); +} + +TEST_P(SADx4Test, MaxSrc) { + FillConstant(source_data_, source_stride_, mask_); + FillConstant(GetReference(0), reference_stride_, 0); + FillConstant(GetReference(1), reference_stride_, 0); + FillConstant(GetReference(2), reference_stride_, 0); + FillConstant(GetReference(3), reference_stride_, 0); + CheckSADs(); +} + +TEST_P(SADx4Test, ShortRef) { + int tmp_stride = reference_stride_; + reference_stride_ >>= 1; + FillRandom(source_data_, source_stride_); + FillRandom(GetReference(0), reference_stride_); + FillRandom(GetReference(1), reference_stride_); + FillRandom(GetReference(2), reference_stride_); + FillRandom(GetReference(3), reference_stride_); + CheckSADs(); + reference_stride_ = tmp_stride; +} + +TEST_P(SADx4Test, UnalignedRef) { + // The reference frame, but not the source frame, may be unaligned for + // certain types of searches. + int tmp_stride = reference_stride_; + reference_stride_ -= 1; + FillRandom(source_data_, source_stride_); + FillRandom(GetReference(0), reference_stride_); + FillRandom(GetReference(1), reference_stride_); + FillRandom(GetReference(2), reference_stride_); + FillRandom(GetReference(3), reference_stride_); + CheckSADs(); + reference_stride_ = tmp_stride; +} + +TEST_P(SADx4Test, ShortSrc) { + int tmp_stride = source_stride_; + source_stride_ >>= 1; + int test_count = 1000; + while (test_count > 0) { + FillRandom(source_data_, source_stride_); + FillRandom(GetReference(0), reference_stride_); + FillRandom(GetReference(1), reference_stride_); + FillRandom(GetReference(2), reference_stride_); + FillRandom(GetReference(3), reference_stride_); + CheckSADs(); + test_count -= 1; + } + source_stride_ = tmp_stride; +} + +TEST_P(SADx4Test, SrcAlignedByWidth) { + uint8_t *tmp_source_data = source_data_; + source_data_ += width_; + FillRandom(source_data_, source_stride_); + FillRandom(GetReference(0), reference_stride_); + FillRandom(GetReference(1), reference_stride_); + FillRandom(GetReference(2), reference_stride_); + FillRandom(GetReference(3), reference_stride_); + CheckSADs(); + source_data_ = tmp_source_data; +} + +using std::tr1::make_tuple; + +//------------------------------------------------------------------------------ +// C functions +const SadMxNParam c_tests[] = { +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_sad128x128_c, -1), + make_tuple(128, 64, &aom_sad128x64_c, -1), + make_tuple(64, 128, &aom_sad64x128_c, -1), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_sad64x64_c, -1), + make_tuple(64, 32, &aom_sad64x32_c, -1), + make_tuple(32, 64, &aom_sad32x64_c, -1), + make_tuple(32, 32, &aom_sad32x32_c, -1), + make_tuple(32, 16, &aom_sad32x16_c, -1), + make_tuple(16, 32, &aom_sad16x32_c, -1), + make_tuple(16, 16, &aom_sad16x16_c, -1), + make_tuple(16, 8, &aom_sad16x8_c, -1), + make_tuple(8, 16, &aom_sad8x16_c, -1), + make_tuple(8, 8, &aom_sad8x8_c, -1), + make_tuple(8, 4, &aom_sad8x4_c, -1), + make_tuple(4, 8, &aom_sad4x8_c, -1), + make_tuple(4, 4, &aom_sad4x4_c, -1), +#if CONFIG_HIGHBITDEPTH +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_highbd_sad128x128_c, 8), + make_tuple(128, 64, &aom_highbd_sad128x64_c, 8), + make_tuple(64, 128, &aom_highbd_sad64x128_c, 8), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_highbd_sad64x64_c, 8), + make_tuple(64, 32, &aom_highbd_sad64x32_c, 8), + make_tuple(32, 64, &aom_highbd_sad32x64_c, 8), + make_tuple(32, 32, &aom_highbd_sad32x32_c, 8), + make_tuple(32, 16, &aom_highbd_sad32x16_c, 8), + make_tuple(16, 32, &aom_highbd_sad16x32_c, 8), + make_tuple(16, 16, &aom_highbd_sad16x16_c, 8), + make_tuple(16, 8, &aom_highbd_sad16x8_c, 8), + make_tuple(8, 16, &aom_highbd_sad8x16_c, 8), + make_tuple(8, 8, &aom_highbd_sad8x8_c, 8), + make_tuple(8, 4, &aom_highbd_sad8x4_c, 8), + make_tuple(4, 8, &aom_highbd_sad4x8_c, 8), + make_tuple(4, 4, &aom_highbd_sad4x4_c, 8), +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_highbd_sad128x128_c, 10), + make_tuple(128, 64, &aom_highbd_sad128x64_c, 10), + make_tuple(64, 128, &aom_highbd_sad64x128_c, 10), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_highbd_sad64x64_c, 10), + make_tuple(64, 32, &aom_highbd_sad64x32_c, 10), + make_tuple(32, 64, &aom_highbd_sad32x64_c, 10), + make_tuple(32, 32, &aom_highbd_sad32x32_c, 10), + make_tuple(32, 16, &aom_highbd_sad32x16_c, 10), + make_tuple(16, 32, &aom_highbd_sad16x32_c, 10), + make_tuple(16, 16, &aom_highbd_sad16x16_c, 10), + make_tuple(16, 8, &aom_highbd_sad16x8_c, 10), + make_tuple(8, 16, &aom_highbd_sad8x16_c, 10), + make_tuple(8, 8, &aom_highbd_sad8x8_c, 10), + make_tuple(8, 4, &aom_highbd_sad8x4_c, 10), + make_tuple(4, 8, &aom_highbd_sad4x8_c, 10), + make_tuple(4, 4, &aom_highbd_sad4x4_c, 10), +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_highbd_sad128x128_c, 12), + make_tuple(128, 64, &aom_highbd_sad128x64_c, 12), + make_tuple(64, 128, &aom_highbd_sad64x128_c, 12), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_highbd_sad64x64_c, 12), + make_tuple(64, 32, &aom_highbd_sad64x32_c, 12), + make_tuple(32, 64, &aom_highbd_sad32x64_c, 12), + make_tuple(32, 32, &aom_highbd_sad32x32_c, 12), + make_tuple(32, 16, &aom_highbd_sad32x16_c, 12), + make_tuple(16, 32, &aom_highbd_sad16x32_c, 12), + make_tuple(16, 16, &aom_highbd_sad16x16_c, 12), + make_tuple(16, 8, &aom_highbd_sad16x8_c, 12), + make_tuple(8, 16, &aom_highbd_sad8x16_c, 12), + make_tuple(8, 8, &aom_highbd_sad8x8_c, 12), + make_tuple(8, 4, &aom_highbd_sad8x4_c, 12), + make_tuple(4, 8, &aom_highbd_sad4x8_c, 12), + make_tuple(4, 4, &aom_highbd_sad4x4_c, 12), +#endif // CONFIG_HIGHBITDEPTH +}; +INSTANTIATE_TEST_CASE_P(C, SADTest, ::testing::ValuesIn(c_tests)); + +const SadMxNAvgParam avg_c_tests[] = { +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_sad128x128_avg_c, -1), + make_tuple(128, 64, &aom_sad128x64_avg_c, -1), + make_tuple(64, 128, &aom_sad64x128_avg_c, -1), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_sad64x64_avg_c, -1), + make_tuple(64, 32, &aom_sad64x32_avg_c, -1), + make_tuple(32, 64, &aom_sad32x64_avg_c, -1), + make_tuple(32, 32, &aom_sad32x32_avg_c, -1), + make_tuple(32, 16, &aom_sad32x16_avg_c, -1), + make_tuple(16, 32, &aom_sad16x32_avg_c, -1), + make_tuple(16, 16, &aom_sad16x16_avg_c, -1), + make_tuple(16, 8, &aom_sad16x8_avg_c, -1), + make_tuple(8, 16, &aom_sad8x16_avg_c, -1), + make_tuple(8, 8, &aom_sad8x8_avg_c, -1), + make_tuple(8, 4, &aom_sad8x4_avg_c, -1), + make_tuple(4, 8, &aom_sad4x8_avg_c, -1), + make_tuple(4, 4, &aom_sad4x4_avg_c, -1), +#if CONFIG_HIGHBITDEPTH +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_highbd_sad128x128_avg_c, 8), + make_tuple(128, 64, &aom_highbd_sad128x64_avg_c, 8), + make_tuple(64, 128, &aom_highbd_sad64x128_avg_c, 8), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_highbd_sad64x64_avg_c, 8), + make_tuple(64, 32, &aom_highbd_sad64x32_avg_c, 8), + make_tuple(32, 64, &aom_highbd_sad32x64_avg_c, 8), + make_tuple(32, 32, &aom_highbd_sad32x32_avg_c, 8), + make_tuple(32, 16, &aom_highbd_sad32x16_avg_c, 8), + make_tuple(16, 32, &aom_highbd_sad16x32_avg_c, 8), + make_tuple(16, 16, &aom_highbd_sad16x16_avg_c, 8), + make_tuple(16, 8, &aom_highbd_sad16x8_avg_c, 8), + make_tuple(8, 16, &aom_highbd_sad8x16_avg_c, 8), + make_tuple(8, 8, &aom_highbd_sad8x8_avg_c, 8), + make_tuple(8, 4, &aom_highbd_sad8x4_avg_c, 8), + make_tuple(4, 8, &aom_highbd_sad4x8_avg_c, 8), + make_tuple(4, 4, &aom_highbd_sad4x4_avg_c, 8), +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_highbd_sad128x128_avg_c, 10), + make_tuple(128, 64, &aom_highbd_sad128x64_avg_c, 10), + make_tuple(64, 128, &aom_highbd_sad64x128_avg_c, 10), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_highbd_sad64x64_avg_c, 10), + make_tuple(64, 32, &aom_highbd_sad64x32_avg_c, 10), + make_tuple(32, 64, &aom_highbd_sad32x64_avg_c, 10), + make_tuple(32, 32, &aom_highbd_sad32x32_avg_c, 10), + make_tuple(32, 16, &aom_highbd_sad32x16_avg_c, 10), + make_tuple(16, 32, &aom_highbd_sad16x32_avg_c, 10), + make_tuple(16, 16, &aom_highbd_sad16x16_avg_c, 10), + make_tuple(16, 8, &aom_highbd_sad16x8_avg_c, 10), + make_tuple(8, 16, &aom_highbd_sad8x16_avg_c, 10), + make_tuple(8, 8, &aom_highbd_sad8x8_avg_c, 10), + make_tuple(8, 4, &aom_highbd_sad8x4_avg_c, 10), + make_tuple(4, 8, &aom_highbd_sad4x8_avg_c, 10), + make_tuple(4, 4, &aom_highbd_sad4x4_avg_c, 10), +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_highbd_sad128x128_avg_c, 12), + make_tuple(128, 64, &aom_highbd_sad128x64_avg_c, 12), + make_tuple(64, 128, &aom_highbd_sad64x128_avg_c, 12), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_highbd_sad64x64_avg_c, 12), + make_tuple(64, 32, &aom_highbd_sad64x32_avg_c, 12), + make_tuple(32, 64, &aom_highbd_sad32x64_avg_c, 12), + make_tuple(32, 32, &aom_highbd_sad32x32_avg_c, 12), + make_tuple(32, 16, &aom_highbd_sad32x16_avg_c, 12), + make_tuple(16, 32, &aom_highbd_sad16x32_avg_c, 12), + make_tuple(16, 16, &aom_highbd_sad16x16_avg_c, 12), + make_tuple(16, 8, &aom_highbd_sad16x8_avg_c, 12), + make_tuple(8, 16, &aom_highbd_sad8x16_avg_c, 12), + make_tuple(8, 8, &aom_highbd_sad8x8_avg_c, 12), + make_tuple(8, 4, &aom_highbd_sad8x4_avg_c, 12), + make_tuple(4, 8, &aom_highbd_sad4x8_avg_c, 12), + make_tuple(4, 4, &aom_highbd_sad4x4_avg_c, 12), +#endif // CONFIG_HIGHBITDEPTH +}; +INSTANTIATE_TEST_CASE_P(C, SADavgTest, ::testing::ValuesIn(avg_c_tests)); + +const SadMxNx4Param x4d_c_tests[] = { +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_sad128x128x4d_c, -1), + make_tuple(128, 64, &aom_sad128x64x4d_c, -1), + make_tuple(64, 128, &aom_sad64x128x4d_c, -1), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_sad64x64x4d_c, -1), + make_tuple(64, 32, &aom_sad64x32x4d_c, -1), + make_tuple(32, 64, &aom_sad32x64x4d_c, -1), + make_tuple(32, 32, &aom_sad32x32x4d_c, -1), + make_tuple(32, 16, &aom_sad32x16x4d_c, -1), + make_tuple(16, 32, &aom_sad16x32x4d_c, -1), + make_tuple(16, 16, &aom_sad16x16x4d_c, -1), + make_tuple(16, 8, &aom_sad16x8x4d_c, -1), + make_tuple(8, 16, &aom_sad8x16x4d_c, -1), + make_tuple(8, 8, &aom_sad8x8x4d_c, -1), + make_tuple(8, 4, &aom_sad8x4x4d_c, -1), + make_tuple(4, 8, &aom_sad4x8x4d_c, -1), + make_tuple(4, 4, &aom_sad4x4x4d_c, -1), +#if CONFIG_HIGHBITDEPTH +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_highbd_sad128x128x4d_c, 8), + make_tuple(128, 64, &aom_highbd_sad128x64x4d_c, 8), + make_tuple(64, 128, &aom_highbd_sad64x128x4d_c, 8), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_highbd_sad64x64x4d_c, 8), + make_tuple(64, 32, &aom_highbd_sad64x32x4d_c, 8), + make_tuple(32, 64, &aom_highbd_sad32x64x4d_c, 8), + make_tuple(32, 32, &aom_highbd_sad32x32x4d_c, 8), + make_tuple(32, 16, &aom_highbd_sad32x16x4d_c, 8), + make_tuple(16, 32, &aom_highbd_sad16x32x4d_c, 8), + make_tuple(16, 16, &aom_highbd_sad16x16x4d_c, 8), + make_tuple(16, 8, &aom_highbd_sad16x8x4d_c, 8), + make_tuple(8, 16, &aom_highbd_sad8x16x4d_c, 8), + make_tuple(8, 8, &aom_highbd_sad8x8x4d_c, 8), + make_tuple(8, 4, &aom_highbd_sad8x4x4d_c, 8), + make_tuple(4, 8, &aom_highbd_sad4x8x4d_c, 8), + make_tuple(4, 4, &aom_highbd_sad4x4x4d_c, 8), +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_highbd_sad128x128x4d_c, 10), + make_tuple(128, 64, &aom_highbd_sad128x64x4d_c, 10), + make_tuple(64, 128, &aom_highbd_sad64x128x4d_c, 10), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_highbd_sad64x64x4d_c, 10), + make_tuple(64, 32, &aom_highbd_sad64x32x4d_c, 10), + make_tuple(32, 64, &aom_highbd_sad32x64x4d_c, 10), + make_tuple(32, 32, &aom_highbd_sad32x32x4d_c, 10), + make_tuple(32, 16, &aom_highbd_sad32x16x4d_c, 10), + make_tuple(16, 32, &aom_highbd_sad16x32x4d_c, 10), + make_tuple(16, 16, &aom_highbd_sad16x16x4d_c, 10), + make_tuple(16, 8, &aom_highbd_sad16x8x4d_c, 10), + make_tuple(8, 16, &aom_highbd_sad8x16x4d_c, 10), + make_tuple(8, 8, &aom_highbd_sad8x8x4d_c, 10), + make_tuple(8, 4, &aom_highbd_sad8x4x4d_c, 10), + make_tuple(4, 8, &aom_highbd_sad4x8x4d_c, 10), + make_tuple(4, 4, &aom_highbd_sad4x4x4d_c, 10), +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_highbd_sad128x128x4d_c, 12), + make_tuple(128, 64, &aom_highbd_sad128x64x4d_c, 12), + make_tuple(64, 128, &aom_highbd_sad64x128x4d_c, 12), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_highbd_sad64x64x4d_c, 12), + make_tuple(64, 32, &aom_highbd_sad64x32x4d_c, 12), + make_tuple(32, 64, &aom_highbd_sad32x64x4d_c, 12), + make_tuple(32, 32, &aom_highbd_sad32x32x4d_c, 12), + make_tuple(32, 16, &aom_highbd_sad32x16x4d_c, 12), + make_tuple(16, 32, &aom_highbd_sad16x32x4d_c, 12), + make_tuple(16, 16, &aom_highbd_sad16x16x4d_c, 12), + make_tuple(16, 8, &aom_highbd_sad16x8x4d_c, 12), + make_tuple(8, 16, &aom_highbd_sad8x16x4d_c, 12), + make_tuple(8, 8, &aom_highbd_sad8x8x4d_c, 12), + make_tuple(8, 4, &aom_highbd_sad8x4x4d_c, 12), + make_tuple(4, 8, &aom_highbd_sad4x8x4d_c, 12), + make_tuple(4, 4, &aom_highbd_sad4x4x4d_c, 12), +#endif // CONFIG_HIGHBITDEPTH +}; +INSTANTIATE_TEST_CASE_P(C, SADx4Test, ::testing::ValuesIn(x4d_c_tests)); + +//------------------------------------------------------------------------------ +// ARM functions +#if HAVE_MEDIA +const SadMxNParam media_tests[] = { + make_tuple(16, 16, &aom_sad16x16_media, -1), +}; +INSTANTIATE_TEST_CASE_P(MEDIA, SADTest, ::testing::ValuesIn(media_tests)); +#endif // HAVE_MEDIA + +#if HAVE_NEON +const SadMxNParam neon_tests[] = { + make_tuple(64, 64, &aom_sad64x64_neon, -1), + make_tuple(32, 32, &aom_sad32x32_neon, -1), + make_tuple(16, 16, &aom_sad16x16_neon, -1), + make_tuple(16, 8, &aom_sad16x8_neon, -1), + make_tuple(8, 16, &aom_sad8x16_neon, -1), + make_tuple(8, 8, &aom_sad8x8_neon, -1), + make_tuple(4, 4, &aom_sad4x4_neon, -1), +}; +INSTANTIATE_TEST_CASE_P(NEON, SADTest, ::testing::ValuesIn(neon_tests)); + +const SadMxNx4Param x4d_neon_tests[] = { + make_tuple(64, 64, &aom_sad64x64x4d_neon, -1), + make_tuple(32, 32, &aom_sad32x32x4d_neon, -1), + make_tuple(16, 16, &aom_sad16x16x4d_neon, -1), +}; +INSTANTIATE_TEST_CASE_P(NEON, SADx4Test, ::testing::ValuesIn(x4d_neon_tests)); +#endif // HAVE_NEON + +//------------------------------------------------------------------------------ +// x86 functions +#if HAVE_SSE2 +const SadMxNParam sse2_tests[] = { +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_sad128x128_sse2, -1), + make_tuple(128, 64, &aom_sad128x64_sse2, -1), + make_tuple(64, 128, &aom_sad64x128_sse2, -1), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_sad64x64_sse2, -1), + make_tuple(64, 32, &aom_sad64x32_sse2, -1), + make_tuple(32, 64, &aom_sad32x64_sse2, -1), + make_tuple(32, 32, &aom_sad32x32_sse2, -1), + make_tuple(32, 16, &aom_sad32x16_sse2, -1), + make_tuple(16, 32, &aom_sad16x32_sse2, -1), + make_tuple(16, 16, &aom_sad16x16_sse2, -1), + make_tuple(16, 8, &aom_sad16x8_sse2, -1), + make_tuple(8, 16, &aom_sad8x16_sse2, -1), + make_tuple(8, 8, &aom_sad8x8_sse2, -1), + make_tuple(8, 4, &aom_sad8x4_sse2, -1), + make_tuple(4, 8, &aom_sad4x8_sse2, -1), + make_tuple(4, 4, &aom_sad4x4_sse2, -1), +#if CONFIG_HIGHBITDEPTH + make_tuple(64, 64, &aom_highbd_sad64x64_sse2, 8), + make_tuple(64, 32, &aom_highbd_sad64x32_sse2, 8), + make_tuple(32, 64, &aom_highbd_sad32x64_sse2, 8), + make_tuple(32, 32, &aom_highbd_sad32x32_sse2, 8), + make_tuple(32, 16, &aom_highbd_sad32x16_sse2, 8), + make_tuple(16, 32, &aom_highbd_sad16x32_sse2, 8), + make_tuple(16, 16, &aom_highbd_sad16x16_sse2, 8), + make_tuple(16, 8, &aom_highbd_sad16x8_sse2, 8), + make_tuple(8, 16, &aom_highbd_sad8x16_sse2, 8), + make_tuple(8, 8, &aom_highbd_sad8x8_sse2, 8), + make_tuple(8, 4, &aom_highbd_sad8x4_sse2, 8), + make_tuple(64, 64, &aom_highbd_sad64x64_sse2, 10), + make_tuple(64, 32, &aom_highbd_sad64x32_sse2, 10), + make_tuple(32, 64, &aom_highbd_sad32x64_sse2, 10), + make_tuple(32, 32, &aom_highbd_sad32x32_sse2, 10), + make_tuple(32, 16, &aom_highbd_sad32x16_sse2, 10), + make_tuple(16, 32, &aom_highbd_sad16x32_sse2, 10), + make_tuple(16, 16, &aom_highbd_sad16x16_sse2, 10), + make_tuple(16, 8, &aom_highbd_sad16x8_sse2, 10), + make_tuple(8, 16, &aom_highbd_sad8x16_sse2, 10), + make_tuple(8, 8, &aom_highbd_sad8x8_sse2, 10), + make_tuple(8, 4, &aom_highbd_sad8x4_sse2, 10), + make_tuple(64, 64, &aom_highbd_sad64x64_sse2, 12), + make_tuple(64, 32, &aom_highbd_sad64x32_sse2, 12), + make_tuple(32, 64, &aom_highbd_sad32x64_sse2, 12), + make_tuple(32, 32, &aom_highbd_sad32x32_sse2, 12), + make_tuple(32, 16, &aom_highbd_sad32x16_sse2, 12), + make_tuple(16, 32, &aom_highbd_sad16x32_sse2, 12), + make_tuple(16, 16, &aom_highbd_sad16x16_sse2, 12), + make_tuple(16, 8, &aom_highbd_sad16x8_sse2, 12), + make_tuple(8, 16, &aom_highbd_sad8x16_sse2, 12), + make_tuple(8, 8, &aom_highbd_sad8x8_sse2, 12), + make_tuple(8, 4, &aom_highbd_sad8x4_sse2, 12), +#endif // CONFIG_HIGHBITDEPTH +}; +INSTANTIATE_TEST_CASE_P(SSE2, SADTest, ::testing::ValuesIn(sse2_tests)); + +const SadMxNAvgParam avg_sse2_tests[] = { +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_sad128x128_avg_sse2, -1), + make_tuple(128, 64, &aom_sad128x64_avg_sse2, -1), + make_tuple(64, 128, &aom_sad64x128_avg_sse2, -1), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_sad64x64_avg_sse2, -1), + make_tuple(64, 32, &aom_sad64x32_avg_sse2, -1), + make_tuple(32, 64, &aom_sad32x64_avg_sse2, -1), + make_tuple(32, 32, &aom_sad32x32_avg_sse2, -1), + make_tuple(32, 16, &aom_sad32x16_avg_sse2, -1), + make_tuple(16, 32, &aom_sad16x32_avg_sse2, -1), + make_tuple(16, 16, &aom_sad16x16_avg_sse2, -1), + make_tuple(16, 8, &aom_sad16x8_avg_sse2, -1), + make_tuple(8, 16, &aom_sad8x16_avg_sse2, -1), + make_tuple(8, 8, &aom_sad8x8_avg_sse2, -1), + make_tuple(8, 4, &aom_sad8x4_avg_sse2, -1), + make_tuple(4, 8, &aom_sad4x8_avg_sse2, -1), + make_tuple(4, 4, &aom_sad4x4_avg_sse2, -1), +#if CONFIG_HIGHBITDEPTH + make_tuple(64, 64, &aom_highbd_sad64x64_avg_sse2, 8), + make_tuple(64, 32, &aom_highbd_sad64x32_avg_sse2, 8), + make_tuple(32, 64, &aom_highbd_sad32x64_avg_sse2, 8), + make_tuple(32, 32, &aom_highbd_sad32x32_avg_sse2, 8), + make_tuple(32, 16, &aom_highbd_sad32x16_avg_sse2, 8), + make_tuple(16, 32, &aom_highbd_sad16x32_avg_sse2, 8), + make_tuple(16, 16, &aom_highbd_sad16x16_avg_sse2, 8), + make_tuple(16, 8, &aom_highbd_sad16x8_avg_sse2, 8), + make_tuple(8, 16, &aom_highbd_sad8x16_avg_sse2, 8), + make_tuple(8, 8, &aom_highbd_sad8x8_avg_sse2, 8), + make_tuple(8, 4, &aom_highbd_sad8x4_avg_sse2, 8), + make_tuple(64, 64, &aom_highbd_sad64x64_avg_sse2, 10), + make_tuple(64, 32, &aom_highbd_sad64x32_avg_sse2, 10), + make_tuple(32, 64, &aom_highbd_sad32x64_avg_sse2, 10), + make_tuple(32, 32, &aom_highbd_sad32x32_avg_sse2, 10), + make_tuple(32, 16, &aom_highbd_sad32x16_avg_sse2, 10), + make_tuple(16, 32, &aom_highbd_sad16x32_avg_sse2, 10), + make_tuple(16, 16, &aom_highbd_sad16x16_avg_sse2, 10), + make_tuple(16, 8, &aom_highbd_sad16x8_avg_sse2, 10), + make_tuple(8, 16, &aom_highbd_sad8x16_avg_sse2, 10), + make_tuple(8, 8, &aom_highbd_sad8x8_avg_sse2, 10), + make_tuple(8, 4, &aom_highbd_sad8x4_avg_sse2, 10), + make_tuple(64, 64, &aom_highbd_sad64x64_avg_sse2, 12), + make_tuple(64, 32, &aom_highbd_sad64x32_avg_sse2, 12), + make_tuple(32, 64, &aom_highbd_sad32x64_avg_sse2, 12), + make_tuple(32, 32, &aom_highbd_sad32x32_avg_sse2, 12), + make_tuple(32, 16, &aom_highbd_sad32x16_avg_sse2, 12), + make_tuple(16, 32, &aom_highbd_sad16x32_avg_sse2, 12), + make_tuple(16, 16, &aom_highbd_sad16x16_avg_sse2, 12), + make_tuple(16, 8, &aom_highbd_sad16x8_avg_sse2, 12), + make_tuple(8, 16, &aom_highbd_sad8x16_avg_sse2, 12), + make_tuple(8, 8, &aom_highbd_sad8x8_avg_sse2, 12), + make_tuple(8, 4, &aom_highbd_sad8x4_avg_sse2, 12), +#endif // CONFIG_HIGHBITDEPTH +}; +INSTANTIATE_TEST_CASE_P(SSE2, SADavgTest, ::testing::ValuesIn(avg_sse2_tests)); + +const SadMxNx4Param x4d_sse2_tests[] = { +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_sad128x128x4d_sse2, -1), + make_tuple(128, 64, &aom_sad128x64x4d_sse2, -1), + make_tuple(64, 128, &aom_sad64x128x4d_sse2, -1), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(64, 64, &aom_sad64x64x4d_sse2, -1), + make_tuple(64, 32, &aom_sad64x32x4d_sse2, -1), + make_tuple(32, 64, &aom_sad32x64x4d_sse2, -1), + make_tuple(32, 32, &aom_sad32x32x4d_sse2, -1), + make_tuple(32, 16, &aom_sad32x16x4d_sse2, -1), + make_tuple(16, 32, &aom_sad16x32x4d_sse2, -1), + make_tuple(16, 16, &aom_sad16x16x4d_sse2, -1), + make_tuple(16, 8, &aom_sad16x8x4d_sse2, -1), + make_tuple(8, 16, &aom_sad8x16x4d_sse2, -1), + make_tuple(8, 8, &aom_sad8x8x4d_sse2, -1), + make_tuple(8, 4, &aom_sad8x4x4d_sse2, -1), + make_tuple(4, 8, &aom_sad4x8x4d_sse2, -1), + make_tuple(4, 4, &aom_sad4x4x4d_sse2, -1), +#if CONFIG_HIGHBITDEPTH + make_tuple(64, 64, &aom_highbd_sad64x64x4d_sse2, 8), + make_tuple(64, 32, &aom_highbd_sad64x32x4d_sse2, 8), + make_tuple(32, 64, &aom_highbd_sad32x64x4d_sse2, 8), + make_tuple(32, 32, &aom_highbd_sad32x32x4d_sse2, 8), + make_tuple(32, 16, &aom_highbd_sad32x16x4d_sse2, 8), + make_tuple(16, 32, &aom_highbd_sad16x32x4d_sse2, 8), + make_tuple(16, 16, &aom_highbd_sad16x16x4d_sse2, 8), + make_tuple(16, 8, &aom_highbd_sad16x8x4d_sse2, 8), + make_tuple(8, 16, &aom_highbd_sad8x16x4d_sse2, 8), + make_tuple(8, 8, &aom_highbd_sad8x8x4d_sse2, 8), + make_tuple(8, 4, &aom_highbd_sad8x4x4d_sse2, 8), + make_tuple(4, 8, &aom_highbd_sad4x8x4d_sse2, 8), + make_tuple(4, 4, &aom_highbd_sad4x4x4d_sse2, 8), + make_tuple(64, 64, &aom_highbd_sad64x64x4d_sse2, 10), + make_tuple(64, 32, &aom_highbd_sad64x32x4d_sse2, 10), + make_tuple(32, 64, &aom_highbd_sad32x64x4d_sse2, 10), + make_tuple(32, 32, &aom_highbd_sad32x32x4d_sse2, 10), + make_tuple(32, 16, &aom_highbd_sad32x16x4d_sse2, 10), + make_tuple(16, 32, &aom_highbd_sad16x32x4d_sse2, 10), + make_tuple(16, 16, &aom_highbd_sad16x16x4d_sse2, 10), + make_tuple(16, 8, &aom_highbd_sad16x8x4d_sse2, 10), + make_tuple(8, 16, &aom_highbd_sad8x16x4d_sse2, 10), + make_tuple(8, 8, &aom_highbd_sad8x8x4d_sse2, 10), + make_tuple(8, 4, &aom_highbd_sad8x4x4d_sse2, 10), + make_tuple(4, 8, &aom_highbd_sad4x8x4d_sse2, 10), + make_tuple(4, 4, &aom_highbd_sad4x4x4d_sse2, 10), + make_tuple(64, 64, &aom_highbd_sad64x64x4d_sse2, 12), + make_tuple(64, 32, &aom_highbd_sad64x32x4d_sse2, 12), + make_tuple(32, 64, &aom_highbd_sad32x64x4d_sse2, 12), + make_tuple(32, 32, &aom_highbd_sad32x32x4d_sse2, 12), + make_tuple(32, 16, &aom_highbd_sad32x16x4d_sse2, 12), + make_tuple(16, 32, &aom_highbd_sad16x32x4d_sse2, 12), + make_tuple(16, 16, &aom_highbd_sad16x16x4d_sse2, 12), + make_tuple(16, 8, &aom_highbd_sad16x8x4d_sse2, 12), + make_tuple(8, 16, &aom_highbd_sad8x16x4d_sse2, 12), + make_tuple(8, 8, &aom_highbd_sad8x8x4d_sse2, 12), + make_tuple(8, 4, &aom_highbd_sad8x4x4d_sse2, 12), + make_tuple(4, 8, &aom_highbd_sad4x8x4d_sse2, 12), + make_tuple(4, 4, &aom_highbd_sad4x4x4d_sse2, 12), +#endif // CONFIG_HIGHBITDEPTH +}; +INSTANTIATE_TEST_CASE_P(SSE2, SADx4Test, ::testing::ValuesIn(x4d_sse2_tests)); +#endif // HAVE_SSE2 + +#if HAVE_SSE3 +// Only functions are x3, which do not have tests. +#endif // HAVE_SSE3 + +#if HAVE_SSSE3 +// Only functions are x3, which do not have tests. +#endif // HAVE_SSSE3 + +#if HAVE_SSE4_1 +// Only functions are x8, which do not have tests. +#endif // HAVE_SSE4_1 + +#if HAVE_AVX2 +const SadMxNParam avx2_tests[] = { +#if CONFIG_EXT_PARTITION + make_tuple(64, 128, &aom_sad64x128_avx2, -1), + make_tuple(128, 64, &aom_sad128x64_avx2, -1), + make_tuple(128, 128, &aom_sad128x128_avx2, -1), +#endif + make_tuple(64, 64, &aom_sad64x64_avx2, -1), + make_tuple(64, 32, &aom_sad64x32_avx2, -1), + make_tuple(32, 64, &aom_sad32x64_avx2, -1), + make_tuple(32, 32, &aom_sad32x32_avx2, -1), + make_tuple(32, 16, &aom_sad32x16_avx2, -1), +#if CONFIG_HIGHBITDEPTH +#if CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_highbd_sad128x128_avx2, 8), + make_tuple(128, 128, &aom_highbd_sad128x128_avx2, 10), + make_tuple(128, 128, &aom_highbd_sad128x128_avx2, 12), + make_tuple(128, 64, &aom_highbd_sad128x64_avx2, 8), + make_tuple(128, 64, &aom_highbd_sad128x64_avx2, 10), + make_tuple(128, 64, &aom_highbd_sad128x64_avx2, 12), + make_tuple(64, 128, &aom_highbd_sad64x128_avx2, 8), + make_tuple(64, 128, &aom_highbd_sad64x128_avx2, 10), + make_tuple(64, 128, &aom_highbd_sad64x128_avx2, 12), +#endif + make_tuple(64, 64, &aom_highbd_sad64x64_avx2, 8), + make_tuple(64, 64, &aom_highbd_sad64x64_avx2, 10), + make_tuple(64, 64, &aom_highbd_sad64x64_avx2, 12), + make_tuple(64, 32, &aom_highbd_sad64x32_avx2, 8), + make_tuple(64, 32, &aom_highbd_sad64x32_avx2, 10), + make_tuple(64, 32, &aom_highbd_sad64x32_avx2, 12), + make_tuple(32, 64, &aom_highbd_sad32x64_avx2, 8), + make_tuple(32, 64, &aom_highbd_sad32x64_avx2, 10), + make_tuple(32, 64, &aom_highbd_sad32x64_avx2, 12), + make_tuple(32, 32, &aom_highbd_sad32x32_avx2, 8), + make_tuple(32, 32, &aom_highbd_sad32x32_avx2, 10), + make_tuple(32, 32, &aom_highbd_sad32x32_avx2, 12), + make_tuple(32, 16, &aom_highbd_sad32x16_avx2, 8), + make_tuple(32, 16, &aom_highbd_sad32x16_avx2, 10), + make_tuple(32, 16, &aom_highbd_sad32x16_avx2, 12), + make_tuple(16, 32, &aom_highbd_sad16x32_avx2, 8), + make_tuple(16, 32, &aom_highbd_sad16x32_avx2, 10), + make_tuple(16, 32, &aom_highbd_sad16x32_avx2, 12), + make_tuple(16, 16, &aom_highbd_sad16x16_avx2, 8), + make_tuple(16, 16, &aom_highbd_sad16x16_avx2, 10), + make_tuple(16, 16, &aom_highbd_sad16x16_avx2, 12), + make_tuple(16, 8, &aom_highbd_sad16x8_avx2, 8), + make_tuple(16, 8, &aom_highbd_sad16x8_avx2, 10), + make_tuple(16, 8, &aom_highbd_sad16x8_avx2, 12), +#endif +}; +INSTANTIATE_TEST_CASE_P(AVX2, SADTest, ::testing::ValuesIn(avx2_tests)); + +const SadMxNAvgParam avg_avx2_tests[] = { +#if CONFIG_EXT_PARTITION + make_tuple(64, 128, &aom_sad64x128_avg_avx2, -1), + make_tuple(128, 64, &aom_sad128x64_avg_avx2, -1), + make_tuple(128, 128, &aom_sad128x128_avg_avx2, -1), +#endif + make_tuple(64, 64, &aom_sad64x64_avg_avx2, -1), + make_tuple(64, 32, &aom_sad64x32_avg_avx2, -1), + make_tuple(32, 64, &aom_sad32x64_avg_avx2, -1), + make_tuple(32, 32, &aom_sad32x32_avg_avx2, -1), + make_tuple(32, 16, &aom_sad32x16_avg_avx2, -1), +#if CONFIG_HIGHBITDEPTH +#if CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_highbd_sad128x128_avg_avx2, 8), + make_tuple(128, 128, &aom_highbd_sad128x128_avg_avx2, 10), + make_tuple(128, 128, &aom_highbd_sad128x128_avg_avx2, 12), + make_tuple(128, 64, &aom_highbd_sad128x64_avg_avx2, 8), + make_tuple(128, 64, &aom_highbd_sad128x64_avg_avx2, 10), + make_tuple(128, 64, &aom_highbd_sad128x64_avg_avx2, 12), + make_tuple(64, 128, &aom_highbd_sad64x128_avg_avx2, 8), + make_tuple(64, 128, &aom_highbd_sad64x128_avg_avx2, 10), + make_tuple(64, 128, &aom_highbd_sad64x128_avg_avx2, 12), +#endif + make_tuple(64, 64, &aom_highbd_sad64x64_avg_avx2, 8), + make_tuple(64, 64, &aom_highbd_sad64x64_avg_avx2, 10), + make_tuple(64, 64, &aom_highbd_sad64x64_avg_avx2, 12), + make_tuple(64, 32, &aom_highbd_sad64x32_avg_avx2, 8), + make_tuple(64, 32, &aom_highbd_sad64x32_avg_avx2, 10), + make_tuple(64, 32, &aom_highbd_sad64x32_avg_avx2, 12), + make_tuple(32, 64, &aom_highbd_sad32x64_avg_avx2, 8), + make_tuple(32, 64, &aom_highbd_sad32x64_avg_avx2, 10), + make_tuple(32, 64, &aom_highbd_sad32x64_avg_avx2, 12), + make_tuple(32, 32, &aom_highbd_sad32x32_avg_avx2, 8), + make_tuple(32, 32, &aom_highbd_sad32x32_avg_avx2, 10), + make_tuple(32, 32, &aom_highbd_sad32x32_avg_avx2, 12), + make_tuple(32, 16, &aom_highbd_sad32x16_avg_avx2, 8), + make_tuple(32, 16, &aom_highbd_sad32x16_avg_avx2, 10), + make_tuple(32, 16, &aom_highbd_sad32x16_avg_avx2, 12), + make_tuple(16, 32, &aom_highbd_sad16x32_avg_avx2, 8), + make_tuple(16, 32, &aom_highbd_sad16x32_avg_avx2, 10), + make_tuple(16, 32, &aom_highbd_sad16x32_avg_avx2, 12), + make_tuple(16, 16, &aom_highbd_sad16x16_avg_avx2, 8), + make_tuple(16, 16, &aom_highbd_sad16x16_avg_avx2, 10), + make_tuple(16, 16, &aom_highbd_sad16x16_avg_avx2, 12), + make_tuple(16, 8, &aom_highbd_sad16x8_avg_avx2, 8), + make_tuple(16, 8, &aom_highbd_sad16x8_avg_avx2, 10), + make_tuple(16, 8, &aom_highbd_sad16x8_avg_avx2, 12), +#endif +}; +INSTANTIATE_TEST_CASE_P(AVX2, SADavgTest, ::testing::ValuesIn(avg_avx2_tests)); + +const SadMxNx4Param x4d_avx2_tests[] = { +#if CONFIG_EXT_PARTITION + make_tuple(64, 128, &aom_sad64x128x4d_avx2, -1), + make_tuple(128, 64, &aom_sad128x64x4d_avx2, -1), + make_tuple(128, 128, &aom_sad128x128x4d_avx2, -1), +#endif + make_tuple(64, 64, &aom_sad64x64x4d_avx2, -1), + make_tuple(32, 64, &aom_sad32x64x4d_avx2, -1), + make_tuple(64, 32, &aom_sad64x32x4d_avx2, -1), + make_tuple(32, 32, &aom_sad32x32x4d_avx2, -1), +#if CONFIG_HIGHBITDEPTH +#if CONFIG_EXT_PARTITION + make_tuple(128, 128, &aom_highbd_sad128x128x4d_avx2, 8), + make_tuple(128, 128, &aom_highbd_sad128x128x4d_avx2, 10), + make_tuple(128, 128, &aom_highbd_sad128x128x4d_avx2, 12), + make_tuple(128, 64, &aom_highbd_sad128x64x4d_avx2, 8), + make_tuple(128, 64, &aom_highbd_sad128x64x4d_avx2, 10), + make_tuple(128, 64, &aom_highbd_sad128x64x4d_avx2, 12), + make_tuple(64, 128, &aom_highbd_sad64x128x4d_avx2, 8), + make_tuple(64, 128, &aom_highbd_sad64x128x4d_avx2, 10), + make_tuple(64, 128, &aom_highbd_sad64x128x4d_avx2, 12), +#endif + make_tuple(64, 64, &aom_highbd_sad64x64x4d_avx2, 8), + make_tuple(64, 64, &aom_highbd_sad64x64x4d_avx2, 10), + make_tuple(64, 64, &aom_highbd_sad64x64x4d_avx2, 12), + make_tuple(64, 32, &aom_highbd_sad64x32x4d_avx2, 8), + make_tuple(64, 32, &aom_highbd_sad64x32x4d_avx2, 10), + make_tuple(64, 32, &aom_highbd_sad64x32x4d_avx2, 12), + make_tuple(32, 64, &aom_highbd_sad32x64x4d_avx2, 8), + make_tuple(32, 64, &aom_highbd_sad32x64x4d_avx2, 10), + make_tuple(32, 64, &aom_highbd_sad32x64x4d_avx2, 12), + make_tuple(32, 32, &aom_highbd_sad32x32x4d_avx2, 8), + make_tuple(32, 32, &aom_highbd_sad32x32x4d_avx2, 10), + make_tuple(32, 32, &aom_highbd_sad32x32x4d_avx2, 12), + make_tuple(32, 16, &aom_highbd_sad32x16x4d_avx2, 8), + make_tuple(32, 16, &aom_highbd_sad32x16x4d_avx2, 10), + make_tuple(32, 16, &aom_highbd_sad32x16x4d_avx2, 12), + make_tuple(16, 32, &aom_highbd_sad16x32x4d_avx2, 8), + make_tuple(16, 32, &aom_highbd_sad16x32x4d_avx2, 10), + make_tuple(16, 32, &aom_highbd_sad16x32x4d_avx2, 12), + make_tuple(16, 16, &aom_highbd_sad16x16x4d_avx2, 8), + make_tuple(16, 16, &aom_highbd_sad16x16x4d_avx2, 10), + make_tuple(16, 16, &aom_highbd_sad16x16x4d_avx2, 12), + make_tuple(16, 8, &aom_highbd_sad16x8x4d_avx2, 8), + make_tuple(16, 8, &aom_highbd_sad16x8x4d_avx2, 10), + make_tuple(16, 8, &aom_highbd_sad16x8x4d_avx2, 12), +#endif +}; +INSTANTIATE_TEST_CASE_P(AVX2, SADx4Test, ::testing::ValuesIn(x4d_avx2_tests)); +#endif // HAVE_AVX2 + +//------------------------------------------------------------------------------ +// MIPS functions +#if HAVE_MSA +const SadMxNParam msa_tests[] = { + make_tuple(64, 64, &aom_sad64x64_msa, -1), + make_tuple(64, 32, &aom_sad64x32_msa, -1), + make_tuple(32, 64, &aom_sad32x64_msa, -1), + make_tuple(32, 32, &aom_sad32x32_msa, -1), + make_tuple(32, 16, &aom_sad32x16_msa, -1), + make_tuple(16, 32, &aom_sad16x32_msa, -1), + make_tuple(16, 16, &aom_sad16x16_msa, -1), + make_tuple(16, 8, &aom_sad16x8_msa, -1), + make_tuple(8, 16, &aom_sad8x16_msa, -1), + make_tuple(8, 8, &aom_sad8x8_msa, -1), + make_tuple(8, 4, &aom_sad8x4_msa, -1), + make_tuple(4, 8, &aom_sad4x8_msa, -1), + make_tuple(4, 4, &aom_sad4x4_msa, -1), +}; +INSTANTIATE_TEST_CASE_P(MSA, SADTest, ::testing::ValuesIn(msa_tests)); + +const SadMxNAvgParam avg_msa_tests[] = { + make_tuple(64, 64, &aom_sad64x64_avg_msa, -1), + make_tuple(64, 32, &aom_sad64x32_avg_msa, -1), + make_tuple(32, 64, &aom_sad32x64_avg_msa, -1), + make_tuple(32, 32, &aom_sad32x32_avg_msa, -1), + make_tuple(32, 16, &aom_sad32x16_avg_msa, -1), + make_tuple(16, 32, &aom_sad16x32_avg_msa, -1), + make_tuple(16, 16, &aom_sad16x16_avg_msa, -1), + make_tuple(16, 8, &aom_sad16x8_avg_msa, -1), + make_tuple(8, 16, &aom_sad8x16_avg_msa, -1), + make_tuple(8, 8, &aom_sad8x8_avg_msa, -1), + make_tuple(8, 4, &aom_sad8x4_avg_msa, -1), + make_tuple(4, 8, &aom_sad4x8_avg_msa, -1), + make_tuple(4, 4, &aom_sad4x4_avg_msa, -1), +}; +INSTANTIATE_TEST_CASE_P(MSA, SADavgTest, ::testing::ValuesIn(avg_msa_tests)); + +const SadMxNx4Param x4d_msa_tests[] = { + make_tuple(64, 64, &aom_sad64x64x4d_msa, -1), + make_tuple(64, 32, &aom_sad64x32x4d_msa, -1), + make_tuple(32, 64, &aom_sad32x64x4d_msa, -1), + make_tuple(32, 32, &aom_sad32x32x4d_msa, -1), + make_tuple(32, 16, &aom_sad32x16x4d_msa, -1), + make_tuple(16, 32, &aom_sad16x32x4d_msa, -1), + make_tuple(16, 16, &aom_sad16x16x4d_msa, -1), + make_tuple(16, 8, &aom_sad16x8x4d_msa, -1), + make_tuple(8, 16, &aom_sad8x16x4d_msa, -1), + make_tuple(8, 8, &aom_sad8x8x4d_msa, -1), + make_tuple(8, 4, &aom_sad8x4x4d_msa, -1), + make_tuple(4, 8, &aom_sad4x8x4d_msa, -1), + make_tuple(4, 4, &aom_sad4x4x4d_msa, -1), +}; +INSTANTIATE_TEST_CASE_P(MSA, SADx4Test, ::testing::ValuesIn(x4d_msa_tests)); +#endif // HAVE_MSA + +} // namespace diff --git a/third_party/aom/test/scan_test.cc b/third_party/aom/test/scan_test.cc new file mode 100644 index 0000000000..16c831c8ea --- /dev/null +++ b/third_party/aom/test/scan_test.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/common/common_data.h" +#include "av1/common/scan.h" +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +namespace { + +TEST(ScanTest, av1_augment_prob) { + const TX_SIZE tx_size = TX_4X4; + const TX_TYPE tx_type = DCT_DCT; + const int tx1d_size = tx_size_wide[tx_size]; + uint32_t prob[16] = { 8, 8, 7, 7, 8, 8, 4, 2, 3, 3, 2, 2, 2, 2, 2, 2 }; + const uint32_t ref_prob[16] = { + 8, 8, 7, 7, 8, 8, 4, 2, 3, 3, 2, 2, 2, 2, 2, 2 + }; + av1_augment_prob(tx_size, tx_type, prob); + for (int r = 0; r < tx1d_size; ++r) { + for (int c = 0; c < tx1d_size; ++c) { + const uint32_t idx = r * tx1d_size + c; + EXPECT_EQ(ref_prob[idx], prob[idx] >> 16); + } + } + + const SCAN_ORDER *sc = get_default_scan(tx_size, tx_type, 0); + const uint32_t mask = (1 << 16) - 1; + for (int r = 0; r < tx1d_size; ++r) { + for (int c = 0; c < tx1d_size; ++c) { + const uint32_t ref_idx = r * tx1d_size + c; + const uint32_t scan_idx = mask ^ (prob[r * tx1d_size + c] & mask); + const uint32_t idx = sc->scan[scan_idx]; + EXPECT_EQ(ref_idx, idx); + } + } +} + +TEST(ScanTest, av1_update_sort_order) { + const TX_SIZE tx_size = TX_4X4; + const TX_TYPE tx_type = DCT_DCT; + const uint32_t prob[16] = { 15, 14, 11, 10, 13, 12, 9, 5, + 8, 7, 4, 2, 6, 3, 1, 0 }; + const int16_t ref_sort_order[16] = { 0, 1, 4, 5, 2, 3, 6, 8, + 9, 12, 7, 10, 13, 11, 14, 15 }; + int16_t sort_order[16]; + av1_update_sort_order(tx_size, tx_type, prob, sort_order); + for (int i = 0; i < 16; ++i) EXPECT_EQ(ref_sort_order[i], sort_order[i]); +} + +TEST(ScanTest, av1_update_scan_order) { + TX_SIZE tx_size = TX_4X4; + const TX_TYPE tx_type = DCT_DCT; + const uint32_t prob[16] = { 10, 12, 14, 9, 11, 13, 15, 5, + 8, 7, 4, 2, 6, 3, 1, 0 }; + int16_t sort_order[16]; + int16_t scan[16]; + int16_t iscan[16]; + const int16_t ref_iscan[16] = { 0, 1, 2, 6, 3, 4, 5, 10, + 7, 8, 11, 13, 9, 12, 14, 15 }; + + av1_update_sort_order(tx_size, tx_type, prob, sort_order); + av1_update_scan_order(tx_size, sort_order, scan, iscan); + + for (int i = 0; i < 16; ++i) { + EXPECT_EQ(ref_iscan[i], iscan[i]); + EXPECT_EQ(i, scan[ref_iscan[i]]); + } +} + +TEST(ScanTest, av1_update_neighbors) { + TX_SIZE tx_size = TX_4X4; + // raster order + const int16_t scan[16] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 }; + int16_t nb[(16 + 1) * 2]; + const int16_t ref_nb[(16 + 1) * 2] = { 0, 0, 0, 0, 1, 1, 2, 2, 0, + 1, 1, 4, 2, 5, 3, 6, 4, 5, + 5, 8, 6, 9, 7, 10, 8, 9, 9, + 12, 10, 13, 11, 14, 0, 0 }; + + // raster order's scan and iscan are the same + av1_update_neighbors(tx_size, scan, scan, nb); + + for (int i = 0; i < (16 + 1) * 2; ++i) { + EXPECT_EQ(ref_nb[i], nb[i]); + } +} + +} // namespace diff --git a/third_party/aom/test/selfguided_filter_test.cc b/third_party/aom/test/selfguided_filter_test.cc new file mode 100644 index 0000000000..e87fe339a3 --- /dev/null +++ b/third_party/aom/test/selfguided_filter_test.cc @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./av1_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" + +#include "av1/common/mv.h" +#include "av1/common/restoration.h" + +namespace { + +using std::tr1::tuple; +using std::tr1::make_tuple; +using libaom_test::ACMRandom; + +typedef tuple<> FilterTestParam; + +class AV1SelfguidedFilterTest + : public ::testing::TestWithParam { + public: + virtual ~AV1SelfguidedFilterTest() {} + virtual void SetUp() {} + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunSpeedTest() { + const int w = 256, h = 256; + const int NUM_ITERS = 2000; + int i, j; + + uint8_t *input = (uint8_t *)aom_memalign(16, w * h * sizeof(uint8_t)); + uint8_t *output = (uint8_t *)aom_memalign(16, w * h * sizeof(uint8_t)); + int32_t *tmpbuf = (int32_t *)aom_memalign(16, RESTORATION_TMPBUF_SIZE); + memset(tmpbuf, 0, RESTORATION_TMPBUF_SIZE); + + ACMRandom rnd(ACMRandom::DeterministicSeed()); + + for (i = 0; i < h; ++i) + for (j = 0; j < w; ++j) input[i * w + j] = rnd.Rand16() & 0xFF; + + int xqd[2] = { + SGRPROJ_PRJ_MIN0 + + rnd.PseudoUniform(SGRPROJ_PRJ_MAX0 + 1 - SGRPROJ_PRJ_MIN0), + SGRPROJ_PRJ_MIN1 + + rnd.PseudoUniform(SGRPROJ_PRJ_MAX1 + 1 - SGRPROJ_PRJ_MIN1) + }; + // Fix a parameter set, since the speed depends slightly on r. + // Change this to test different combinations of values of r. + int eps = 15; + + av1_loop_restoration_precal(); + + std::clock_t start = std::clock(); + for (i = 0; i < NUM_ITERS; ++i) { + apply_selfguided_restoration(input, w, h, w, eps, xqd, output, w, tmpbuf); + } + std::clock_t end = std::clock(); + double elapsed = ((end - start) / (double)CLOCKS_PER_SEC); + + printf("%5d %dx%d blocks in %7.3fs = %7.3fus/block\n", NUM_ITERS, w, h, + elapsed, elapsed * 1000000. / NUM_ITERS); + + aom_free(input); + aom_free(output); + aom_free(tmpbuf); + } + + void RunCorrectnessTest() { + // Set the maximum width/height to test here. We actually test a small + // range of sizes *up to* this size, so that we can check, eg., + // the behaviour on tiles which are not a multiple of 4 wide. + const int max_w = 260, max_h = 260, stride = 672, out_stride = 672; + const int NUM_ITERS = 81; + int i, j, k; + + uint8_t *input = + (uint8_t *)aom_memalign(16, stride * max_h * sizeof(uint8_t)); + uint8_t *output = + (uint8_t *)aom_memalign(16, out_stride * max_h * sizeof(uint8_t)); + uint8_t *output2 = + (uint8_t *)aom_memalign(16, out_stride * max_h * sizeof(uint8_t)); + int32_t *tmpbuf = (int32_t *)aom_memalign(16, RESTORATION_TMPBUF_SIZE); + memset(tmpbuf, 0, RESTORATION_TMPBUF_SIZE); + + ACMRandom rnd(ACMRandom::DeterministicSeed()); + + av1_loop_restoration_precal(); + + for (i = 0; i < NUM_ITERS; ++i) { + for (j = 0; j < max_h; ++j) + for (k = 0; k < max_w; ++k) input[j * stride + k] = rnd.Rand16() & 0xFF; + + int xqd[2] = { + SGRPROJ_PRJ_MIN0 + + rnd.PseudoUniform(SGRPROJ_PRJ_MAX0 + 1 - SGRPROJ_PRJ_MIN0), + SGRPROJ_PRJ_MIN1 + + rnd.PseudoUniform(SGRPROJ_PRJ_MAX1 + 1 - SGRPROJ_PRJ_MIN1) + }; + int eps = rnd.PseudoUniform(1 << SGRPROJ_PARAMS_BITS); + + // Test various tile sizes around 256x256 + int test_w = max_w - (i / 9); + int test_h = max_h - (i % 9); + + apply_selfguided_restoration(input, test_w, test_h, stride, eps, xqd, + output, out_stride, tmpbuf); + apply_selfguided_restoration_c(input, test_w, test_h, stride, eps, xqd, + output2, out_stride, tmpbuf); + for (j = 0; j < test_h; ++j) + for (k = 0; k < test_w; ++k) + ASSERT_EQ(output[j * out_stride + k], output2[j * out_stride + k]); + } + + aom_free(input); + aom_free(output); + aom_free(output2); + aom_free(tmpbuf); + } +}; + +TEST_P(AV1SelfguidedFilterTest, SpeedTest) { RunSpeedTest(); } +TEST_P(AV1SelfguidedFilterTest, CorrectnessTest) { RunCorrectnessTest(); } + +const FilterTestParam params[] = { make_tuple() }; + +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P(SSE4_1, AV1SelfguidedFilterTest, + ::testing::ValuesIn(params)); +#endif + +#if CONFIG_HIGHBITDEPTH + +typedef tuple HighbdFilterTestParam; + +class AV1HighbdSelfguidedFilterTest + : public ::testing::TestWithParam { + public: + virtual ~AV1HighbdSelfguidedFilterTest() {} + virtual void SetUp() {} + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + void RunSpeedTest() { + const int w = 256, h = 256; + const int NUM_ITERS = 2000; + int i, j; + int bit_depth = GET_PARAM(0); + int mask = (1 << bit_depth) - 1; + + uint16_t *input = (uint16_t *)aom_memalign(16, w * h * sizeof(uint16_t)); + uint16_t *output = (uint16_t *)aom_memalign(16, w * h * sizeof(uint16_t)); + int32_t *tmpbuf = (int32_t *)aom_memalign(16, RESTORATION_TMPBUF_SIZE); + memset(tmpbuf, 0, RESTORATION_TMPBUF_SIZE); + + ACMRandom rnd(ACMRandom::DeterministicSeed()); + + for (i = 0; i < h; ++i) + for (j = 0; j < w; ++j) input[i * w + j] = rnd.Rand16() & mask; + + int xqd[2] = { + SGRPROJ_PRJ_MIN0 + + rnd.PseudoUniform(SGRPROJ_PRJ_MAX0 + 1 - SGRPROJ_PRJ_MIN0), + SGRPROJ_PRJ_MIN1 + + rnd.PseudoUniform(SGRPROJ_PRJ_MAX1 + 1 - SGRPROJ_PRJ_MIN1) + }; + // Fix a parameter set, since the speed depends slightly on r. + // Change this to test different combinations of values of r. + int eps = 15; + + av1_loop_restoration_precal(); + + std::clock_t start = std::clock(); + for (i = 0; i < NUM_ITERS; ++i) { + apply_selfguided_restoration_highbd(input, w, h, w, bit_depth, eps, xqd, + output, w, tmpbuf); + } + std::clock_t end = std::clock(); + double elapsed = ((end - start) / (double)CLOCKS_PER_SEC); + + printf("%5d %dx%d blocks in %7.3fs = %7.3fus/block\n", NUM_ITERS, w, h, + elapsed, elapsed * 1000000. / NUM_ITERS); + + aom_free(input); + aom_free(output); + aom_free(tmpbuf); + } + + void RunCorrectnessTest() { + // Set the maximum width/height to test here. We actually test a small + // range of sizes *up to* this size, so that we can check, eg., + // the behaviour on tiles which are not a multiple of 4 wide. + const int max_w = 260, max_h = 260, stride = 672, out_stride = 672; + const int NUM_ITERS = 81; + int i, j, k; + int bit_depth = GET_PARAM(0); + int mask = (1 << bit_depth) - 1; + + uint16_t *input = + (uint16_t *)aom_memalign(16, stride * max_h * sizeof(uint16_t)); + uint16_t *output = + (uint16_t *)aom_memalign(16, out_stride * max_h * sizeof(uint16_t)); + uint16_t *output2 = + (uint16_t *)aom_memalign(16, out_stride * max_h * sizeof(uint16_t)); + int32_t *tmpbuf = (int32_t *)aom_memalign(16, RESTORATION_TMPBUF_SIZE); + memset(tmpbuf, 0, RESTORATION_TMPBUF_SIZE); + + ACMRandom rnd(ACMRandom::DeterministicSeed()); + + av1_loop_restoration_precal(); + + for (i = 0; i < NUM_ITERS; ++i) { + for (j = 0; j < max_h; ++j) + for (k = 0; k < max_w; ++k) input[j * stride + k] = rnd.Rand16() & mask; + + int xqd[2] = { + SGRPROJ_PRJ_MIN0 + + rnd.PseudoUniform(SGRPROJ_PRJ_MAX0 + 1 - SGRPROJ_PRJ_MIN0), + SGRPROJ_PRJ_MIN1 + + rnd.PseudoUniform(SGRPROJ_PRJ_MAX1 + 1 - SGRPROJ_PRJ_MIN1) + }; + int eps = rnd.PseudoUniform(1 << SGRPROJ_PARAMS_BITS); + + // Test various tile sizes around 256x256 + int test_w = max_w - (i / 9); + int test_h = max_h - (i % 9); + + apply_selfguided_restoration_highbd(input, test_w, test_h, stride, + bit_depth, eps, xqd, output, + out_stride, tmpbuf); + apply_selfguided_restoration_highbd_c(input, test_w, test_h, stride, + bit_depth, eps, xqd, output2, + out_stride, tmpbuf); + for (j = 0; j < test_h; ++j) + for (k = 0; k < test_w; ++k) + ASSERT_EQ(output[j * out_stride + k], output2[j * out_stride + k]); + } + + aom_free(input); + aom_free(output); + aom_free(output2); + aom_free(tmpbuf); + } +}; + +TEST_P(AV1HighbdSelfguidedFilterTest, SpeedTest) { RunSpeedTest(); } +TEST_P(AV1HighbdSelfguidedFilterTest, CorrectnessTest) { RunCorrectnessTest(); } + +const HighbdFilterTestParam highbd_params[] = { make_tuple(8), make_tuple(10), + make_tuple(12) }; + +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P(SSE4_1, AV1HighbdSelfguidedFilterTest, + ::testing::ValuesIn(highbd_params)); +#endif +#endif + +} // namespace diff --git a/third_party/aom/test/set_maps.sh b/third_party/aom/test/set_maps.sh new file mode 100755 index 0000000000..4f59b06d69 --- /dev/null +++ b/third_party/aom/test/set_maps.sh @@ -0,0 +1,52 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This file tests the libaom set_maps example. To add new tests to this file, +## do the following: +## 1. Write a shell function (this is your test). +## 2. Add the function to set_maps_tests (on a new line). +## +. $(dirname $0)/tools_common.sh + +# Environment check: $YUV_RAW_INPUT is required, and set_maps must exist in +# $LIBAOM_BIN_PATH. +set_maps_verify_environment() { + if [ ! -e "${YUV_RAW_INPUT}" ]; then + echo "Libaom test data must exist in LIBAOM_TEST_DATA_PATH." + return 1 + fi + if [ -z "$(aom_tool_path set_maps)" ]; then + elog "set_maps not found. It must exist in LIBAOM_BIN_PATH or its parent." + return 1 + fi +} + +# Runs set_maps using the codec specified by $1. +set_maps() { + local encoder="$(aom_tool_path set_maps)" + local codec="$1" + local output_file="${AOM_TEST_OUTPUT_DIR}/set_maps_${codec}.ivf" + + eval "${AOM_TEST_PREFIX}" "${encoder}" "${codec}" "${YUV_RAW_INPUT_WIDTH}" \ + "${YUV_RAW_INPUT_HEIGHT}" "${YUV_RAW_INPUT}" "${output_file}" \ + ${devnull} + + [ -e "${output_file}" ] || return 1 +} + +set_maps_av1() { + if [ "$(av1_encode_available)" = "yes" ]; then + set_maps av1 || return 1 + fi +} + +set_maps_tests="set_maps_av1" + +run_tests set_maps_verify_environment "${set_maps_tests}" diff --git a/third_party/aom/test/simd_cmp_impl.h b/third_party/aom/test/simd_cmp_impl.h new file mode 100644 index 0000000000..28bd64a5bc --- /dev/null +++ b/third_party/aom/test/simd_cmp_impl.h @@ -0,0 +1,1212 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "aom_dsp/aom_simd.h" +#undef SIMD_INLINE +#define SIMD_INLINE static // Don't enforce inlining +#include "aom_dsp/simd/v128_intrinsics_c.h" + +// Machine tuned code goes into this file. This file is included from +// simd_cmp_sse2.cc, simd_cmp_ssse3.cc etc which define the macros +// ARCH (=neon, sse2, ssse3, etc), SIMD_NAMESPACE and ARCH_POSTFIX(). + +using libaom_test::ACMRandom; + +namespace SIMD_NAMESPACE { + +// Wrap templates around intrinsics using immediate values +template +v64 imm_v64_shl_n_byte(v64 a) { + return v64_shl_n_byte(a, shift); +} +template +v64 imm_v64_shr_n_byte(v64 a) { + return v64_shr_n_byte(a, shift); +} +template +v64 imm_v64_shl_n_8(v64 a) { + return v64_shl_n_8(a, shift); +} +template +v64 imm_v64_shr_n_u8(v64 a) { + return v64_shr_n_u8(a, shift); +} +template +v64 imm_v64_shr_n_s8(v64 a) { + return v64_shr_n_s8(a, shift); +} +template +v64 imm_v64_shl_n_16(v64 a) { + return v64_shl_n_16(a, shift); +} +template +v64 imm_v64_shr_n_u16(v64 a) { + return v64_shr_n_u16(a, shift); +} +template +v64 imm_v64_shr_n_s16(v64 a) { + return v64_shr_n_s16(a, shift); +} +template +v64 imm_v64_shl_n_32(v64 a) { + return v64_shl_n_32(a, shift); +} +template +v64 imm_v64_shr_n_u32(v64 a) { + return v64_shr_n_u32(a, shift); +} +template +v64 imm_v64_shr_n_s32(v64 a) { + return v64_shr_n_s32(a, shift); +} +template +v64 imm_v64_align(v64 a, v64 b) { + return v64_align(a, b, shift); +} + +// Wrap templates around corresponding C implementations of the above +template +c_v64 c_imm_v64_shl_n_byte(c_v64 a) { + return c_v64_shl_n_byte(a, shift); +} +template +c_v64 c_imm_v64_shr_n_byte(c_v64 a) { + return c_v64_shr_n_byte(a, shift); +} +template +c_v64 c_imm_v64_shl_n_8(c_v64 a) { + return c_v64_shl_n_8(a, shift); +} +template +c_v64 c_imm_v64_shr_n_u8(c_v64 a) { + return c_v64_shr_n_u8(a, shift); +} +template +c_v64 c_imm_v64_shr_n_s8(c_v64 a) { + return c_v64_shr_n_s8(a, shift); +} +template +c_v64 c_imm_v64_shl_n_16(c_v64 a) { + return c_v64_shl_n_16(a, shift); +} +template +c_v64 c_imm_v64_shr_n_u16(c_v64 a) { + return c_v64_shr_n_u16(a, shift); +} +template +c_v64 c_imm_v64_shr_n_s16(c_v64 a) { + return c_v64_shr_n_s16(a, shift); +} +template +c_v64 c_imm_v64_shl_n_32(c_v64 a) { + return c_v64_shl_n_32(a, shift); +} +template +c_v64 c_imm_v64_shr_n_u32(c_v64 a) { + return c_v64_shr_n_u32(a, shift); +} +template +c_v64 c_imm_v64_shr_n_s32(c_v64 a) { + return c_v64_shr_n_s32(a, shift); +} +template +c_v64 c_imm_v64_align(c_v64 a, c_v64 b) { + return c_v64_align(a, b, shift); +} + +template +v128 imm_v128_shl_n_byte(v128 a) { + return v128_shl_n_byte(a, shift); +} +template +v128 imm_v128_shr_n_byte(v128 a) { + return v128_shr_n_byte(a, shift); +} +template +v128 imm_v128_shl_n_8(v128 a) { + return v128_shl_n_8(a, shift); +} +template +v128 imm_v128_shr_n_u8(v128 a) { + return v128_shr_n_u8(a, shift); +} +template +v128 imm_v128_shr_n_s8(v128 a) { + return v128_shr_n_s8(a, shift); +} +template +v128 imm_v128_shl_n_16(v128 a) { + return v128_shl_n_16(a, shift); +} +template +v128 imm_v128_shr_n_u16(v128 a) { + return v128_shr_n_u16(a, shift); +} +template +v128 imm_v128_shr_n_s16(v128 a) { + return v128_shr_n_s16(a, shift); +} +template +v128 imm_v128_shl_n_32(v128 a) { + return v128_shl_n_32(a, shift); +} +template +v128 imm_v128_shr_n_u32(v128 a) { + return v128_shr_n_u32(a, shift); +} +template +v128 imm_v128_shr_n_s32(v128 a) { + return v128_shr_n_s32(a, shift); +} +template +v128 imm_v128_align(v128 a, v128 b) { + return v128_align(a, b, shift); +} + +template +c_v128 c_imm_v128_shl_n_byte(c_v128 a) { + return c_v128_shl_n_byte(a, shift); +} +template +c_v128 c_imm_v128_shr_n_byte(c_v128 a) { + return c_v128_shr_n_byte(a, shift); +} +template +c_v128 c_imm_v128_shl_n_8(c_v128 a) { + return c_v128_shl_n_8(a, shift); +} +template +c_v128 c_imm_v128_shr_n_u8(c_v128 a) { + return c_v128_shr_n_u8(a, shift); +} +template +c_v128 c_imm_v128_shr_n_s8(c_v128 a) { + return c_v128_shr_n_s8(a, shift); +} +template +c_v128 c_imm_v128_shl_n_16(c_v128 a) { + return c_v128_shl_n_16(a, shift); +} +template +c_v128 c_imm_v128_shr_n_u16(c_v128 a) { + return c_v128_shr_n_u16(a, shift); +} +template +c_v128 c_imm_v128_shr_n_s16(c_v128 a) { + return c_v128_shr_n_s16(a, shift); +} +template +c_v128 c_imm_v128_shl_n_32(c_v128 a) { + return c_v128_shl_n_32(a, shift); +} +template +c_v128 c_imm_v128_shr_n_u32(c_v128 a) { + return c_v128_shr_n_u32(a, shift); +} +template +c_v128 c_imm_v128_shr_n_s32(c_v128 a) { + return c_v128_shr_n_s32(a, shift); +} +template +c_v128 c_imm_v128_align(c_v128 a, c_v128 b) { + return c_v128_align(a, b, shift); +} + +// Wrappers around the the SAD and SSD functions +uint32_t v64_sad_u8(v64 a, v64 b) { + return v64_sad_u8_sum(::v64_sad_u8(v64_sad_u8_init(), a, b)); +} +uint32_t v64_ssd_u8(v64 a, v64 b) { + return v64_ssd_u8_sum(::v64_ssd_u8(v64_ssd_u8_init(), a, b)); +} + +uint32_t c_v64_sad_u8(c_v64 a, c_v64 b) { + return c_v64_sad_u8_sum(::c_v64_sad_u8(c_v64_sad_u8_init(), a, b)); +} +uint32_t c_v64_ssd_u8(c_v64 a, c_v64 b) { + return c_v64_ssd_u8_sum(::c_v64_ssd_u8(c_v64_ssd_u8_init(), a, b)); +} +uint32_t v128_sad_u8(v128 a, v128 b) { + return v128_sad_u8_sum(::v128_sad_u8(v128_sad_u8_init(), a, b)); +} +uint32_t v128_ssd_u8(v128 a, v128 b) { + return v128_ssd_u8_sum(::v128_ssd_u8(v128_ssd_u8_init(), a, b)); +} +uint32_t c_v128_sad_u8(c_v128 a, c_v128 b) { + return c_v128_sad_u8_sum(::c_v128_sad_u8(c_v128_sad_u8_init(), a, b)); +} +uint32_t c_v128_ssd_u8(c_v128 a, c_v128 b) { + return c_v128_ssd_u8_sum(::c_v128_ssd_u8(c_v128_ssd_u8_init(), a, b)); +} + +namespace { + +typedef void (*fptr)(); + +typedef struct { + const char *name; + fptr ref; + fptr simd; +} mapping; + +#define MAP(name) \ + { \ + #name, reinterpret_cast < fptr > (c_##name), \ + reinterpret_cast < fptr > (name) \ + } + +const mapping m[] = { MAP(v64_sad_u8), + MAP(v64_ssd_u8), + MAP(v64_add_8), + MAP(v64_add_16), + MAP(v64_sadd_s16), + MAP(v64_add_32), + MAP(v64_sub_8), + MAP(v64_ssub_u8), + MAP(v64_ssub_s8), + MAP(v64_sub_16), + MAP(v64_ssub_s16), + MAP(v64_ssub_u16), + MAP(v64_sub_32), + MAP(v64_ziplo_8), + MAP(v64_ziphi_8), + MAP(v64_ziplo_16), + MAP(v64_ziphi_16), + MAP(v64_ziplo_32), + MAP(v64_ziphi_32), + MAP(v64_pack_s32_s16), + MAP(v64_pack_s16_u8), + MAP(v64_pack_s16_s8), + MAP(v64_unziphi_8), + MAP(v64_unziplo_8), + MAP(v64_unziphi_16), + MAP(v64_unziplo_16), + MAP(v64_or), + MAP(v64_xor), + MAP(v64_and), + MAP(v64_andn), + MAP(v64_mullo_s16), + MAP(v64_mulhi_s16), + MAP(v64_mullo_s32), + MAP(v64_madd_s16), + MAP(v64_madd_us8), + MAP(v64_avg_u8), + MAP(v64_rdavg_u8), + MAP(v64_avg_u16), + MAP(v64_min_u8), + MAP(v64_max_u8), + MAP(v64_min_s8), + MAP(v64_max_s8), + MAP(v64_min_s16), + MAP(v64_max_s16), + MAP(v64_cmpgt_s8), + MAP(v64_cmplt_s8), + MAP(v64_cmpeq_8), + MAP(v64_cmpgt_s16), + MAP(v64_cmplt_s16), + MAP(v64_cmpeq_16), + MAP(v64_shuffle_8), + MAP(imm_v64_align<1>), + MAP(imm_v64_align<2>), + MAP(imm_v64_align<3>), + MAP(imm_v64_align<4>), + MAP(imm_v64_align<5>), + MAP(imm_v64_align<6>), + MAP(imm_v64_align<7>), + MAP(v64_abs_s8), + MAP(v64_abs_s16), + MAP(v64_unpacklo_u8_s16), + MAP(v64_unpackhi_u8_s16), + MAP(v64_unpacklo_s8_s16), + MAP(v64_unpackhi_s8_s16), + MAP(v64_unpacklo_u16_s32), + MAP(v64_unpacklo_s16_s32), + MAP(v64_unpackhi_u16_s32), + MAP(v64_unpackhi_s16_s32), + MAP(imm_v64_shr_n_byte<1>), + MAP(imm_v64_shr_n_byte<2>), + MAP(imm_v64_shr_n_byte<3>), + MAP(imm_v64_shr_n_byte<4>), + MAP(imm_v64_shr_n_byte<5>), + MAP(imm_v64_shr_n_byte<6>), + MAP(imm_v64_shr_n_byte<7>), + MAP(imm_v64_shl_n_byte<1>), + MAP(imm_v64_shl_n_byte<2>), + MAP(imm_v64_shl_n_byte<3>), + MAP(imm_v64_shl_n_byte<4>), + MAP(imm_v64_shl_n_byte<5>), + MAP(imm_v64_shl_n_byte<6>), + MAP(imm_v64_shl_n_byte<7>), + MAP(imm_v64_shl_n_8<1>), + MAP(imm_v64_shl_n_8<2>), + MAP(imm_v64_shl_n_8<3>), + MAP(imm_v64_shl_n_8<4>), + MAP(imm_v64_shl_n_8<5>), + MAP(imm_v64_shl_n_8<6>), + MAP(imm_v64_shl_n_8<7>), + MAP(imm_v64_shr_n_u8<1>), + MAP(imm_v64_shr_n_u8<2>), + MAP(imm_v64_shr_n_u8<3>), + MAP(imm_v64_shr_n_u8<4>), + MAP(imm_v64_shr_n_u8<5>), + MAP(imm_v64_shr_n_u8<6>), + MAP(imm_v64_shr_n_u8<7>), + MAP(imm_v64_shr_n_s8<1>), + MAP(imm_v64_shr_n_s8<2>), + MAP(imm_v64_shr_n_s8<3>), + MAP(imm_v64_shr_n_s8<4>), + MAP(imm_v64_shr_n_s8<5>), + MAP(imm_v64_shr_n_s8<6>), + MAP(imm_v64_shr_n_s8<7>), + MAP(imm_v64_shl_n_16<1>), + MAP(imm_v64_shl_n_16<2>), + MAP(imm_v64_shl_n_16<4>), + MAP(imm_v64_shl_n_16<6>), + MAP(imm_v64_shl_n_16<8>), + MAP(imm_v64_shl_n_16<10>), + MAP(imm_v64_shl_n_16<12>), + MAP(imm_v64_shl_n_16<14>), + MAP(imm_v64_shr_n_u16<1>), + MAP(imm_v64_shr_n_u16<2>), + MAP(imm_v64_shr_n_u16<4>), + MAP(imm_v64_shr_n_u16<6>), + MAP(imm_v64_shr_n_u16<8>), + MAP(imm_v64_shr_n_u16<10>), + MAP(imm_v64_shr_n_u16<12>), + MAP(imm_v64_shr_n_u16<14>), + MAP(imm_v64_shr_n_s16<1>), + MAP(imm_v64_shr_n_s16<2>), + MAP(imm_v64_shr_n_s16<4>), + MAP(imm_v64_shr_n_s16<6>), + MAP(imm_v64_shr_n_s16<8>), + MAP(imm_v64_shr_n_s16<10>), + MAP(imm_v64_shr_n_s16<12>), + MAP(imm_v64_shr_n_s16<14>), + MAP(imm_v64_shl_n_32<1>), + MAP(imm_v64_shl_n_32<4>), + MAP(imm_v64_shl_n_32<8>), + MAP(imm_v64_shl_n_32<12>), + MAP(imm_v64_shl_n_32<16>), + MAP(imm_v64_shl_n_32<20>), + MAP(imm_v64_shl_n_32<24>), + MAP(imm_v64_shl_n_32<28>), + MAP(imm_v64_shr_n_u32<1>), + MAP(imm_v64_shr_n_u32<4>), + MAP(imm_v64_shr_n_u32<8>), + MAP(imm_v64_shr_n_u32<12>), + MAP(imm_v64_shr_n_u32<16>), + MAP(imm_v64_shr_n_u32<20>), + MAP(imm_v64_shr_n_u32<24>), + MAP(imm_v64_shr_n_u32<28>), + MAP(imm_v64_shr_n_s32<1>), + MAP(imm_v64_shr_n_s32<4>), + MAP(imm_v64_shr_n_s32<8>), + MAP(imm_v64_shr_n_s32<12>), + MAP(imm_v64_shr_n_s32<16>), + MAP(imm_v64_shr_n_s32<20>), + MAP(imm_v64_shr_n_s32<24>), + MAP(imm_v64_shr_n_s32<28>), + MAP(v64_shl_8), + MAP(v64_shr_u8), + MAP(v64_shr_s8), + MAP(v64_shl_16), + MAP(v64_shr_u16), + MAP(v64_shr_s16), + MAP(v64_shl_32), + MAP(v64_shr_u32), + MAP(v64_shr_s32), + MAP(v64_hadd_u8), + MAP(v64_hadd_s16), + MAP(v64_dotp_s16), + MAP(v64_dotp_su8), + MAP(v64_u64), + MAP(v64_low_u32), + MAP(v64_high_u32), + MAP(v64_low_s32), + MAP(v64_high_s32), + MAP(v64_dup_8), + MAP(v64_dup_16), + MAP(v64_dup_32), + MAP(v64_from_32), + MAP(v64_zero), + MAP(v64_from_16), + MAP(v128_sad_u8), + MAP(v128_ssd_u8), + MAP(v128_add_8), + MAP(v128_add_16), + MAP(v128_sadd_s16), + MAP(v128_add_32), + MAP(v128_sub_8), + MAP(v128_ssub_u8), + MAP(v128_ssub_s8), + MAP(v128_sub_16), + MAP(v128_ssub_s16), + MAP(v128_ssub_u16), + MAP(v128_sub_32), + MAP(v128_ziplo_8), + MAP(v128_ziphi_8), + MAP(v128_ziplo_16), + MAP(v128_ziphi_16), + MAP(v128_ziplo_32), + MAP(v128_ziphi_32), + MAP(v128_ziplo_64), + MAP(v128_ziphi_64), + MAP(v128_unziphi_8), + MAP(v128_unziplo_8), + MAP(v128_unziphi_16), + MAP(v128_unziplo_16), + MAP(v128_unziphi_32), + MAP(v128_unziplo_32), + MAP(v128_pack_s32_s16), + MAP(v128_pack_s16_u8), + MAP(v128_pack_s16_s8), + MAP(v128_or), + MAP(v128_xor), + MAP(v128_and), + MAP(v128_andn), + MAP(v128_mullo_s16), + MAP(v128_mulhi_s16), + MAP(v128_mullo_s32), + MAP(v128_madd_s16), + MAP(v128_madd_us8), + MAP(v128_avg_u8), + MAP(v128_rdavg_u8), + MAP(v128_avg_u16), + MAP(v128_min_u8), + MAP(v128_max_u8), + MAP(v128_min_s8), + MAP(v128_max_s8), + MAP(v128_min_s16), + MAP(v128_max_s16), + MAP(v128_cmpgt_s8), + MAP(v128_cmplt_s8), + MAP(v128_cmpeq_8), + MAP(v128_cmpgt_s16), + MAP(v128_cmpeq_16), + MAP(v128_cmplt_s16), + MAP(v128_shuffle_8), + MAP(imm_v128_align<1>), + MAP(imm_v128_align<2>), + MAP(imm_v128_align<3>), + MAP(imm_v128_align<4>), + MAP(imm_v128_align<5>), + MAP(imm_v128_align<6>), + MAP(imm_v128_align<7>), + MAP(imm_v128_align<8>), + MAP(imm_v128_align<9>), + MAP(imm_v128_align<10>), + MAP(imm_v128_align<11>), + MAP(imm_v128_align<12>), + MAP(imm_v128_align<13>), + MAP(imm_v128_align<14>), + MAP(imm_v128_align<15>), + MAP(v128_abs_s8), + MAP(v128_abs_s16), + MAP(v128_padd_s16), + MAP(v128_unpacklo_u16_s32), + MAP(v128_unpacklo_s16_s32), + MAP(v128_unpackhi_u16_s32), + MAP(v128_unpackhi_s16_s32), + MAP(imm_v128_shr_n_byte<1>), + MAP(imm_v128_shr_n_byte<2>), + MAP(imm_v128_shr_n_byte<3>), + MAP(imm_v128_shr_n_byte<4>), + MAP(imm_v128_shr_n_byte<5>), + MAP(imm_v128_shr_n_byte<6>), + MAP(imm_v128_shr_n_byte<7>), + MAP(imm_v128_shr_n_byte<8>), + MAP(imm_v128_shr_n_byte<9>), + MAP(imm_v128_shr_n_byte<10>), + MAP(imm_v128_shr_n_byte<11>), + MAP(imm_v128_shr_n_byte<12>), + MAP(imm_v128_shr_n_byte<13>), + MAP(imm_v128_shr_n_byte<14>), + MAP(imm_v128_shr_n_byte<15>), + MAP(imm_v128_shl_n_byte<1>), + MAP(imm_v128_shl_n_byte<2>), + MAP(imm_v128_shl_n_byte<3>), + MAP(imm_v128_shl_n_byte<4>), + MAP(imm_v128_shl_n_byte<5>), + MAP(imm_v128_shl_n_byte<6>), + MAP(imm_v128_shl_n_byte<7>), + MAP(imm_v128_shl_n_byte<8>), + MAP(imm_v128_shl_n_byte<9>), + MAP(imm_v128_shl_n_byte<10>), + MAP(imm_v128_shl_n_byte<11>), + MAP(imm_v128_shl_n_byte<12>), + MAP(imm_v128_shl_n_byte<13>), + MAP(imm_v128_shl_n_byte<14>), + MAP(imm_v128_shl_n_byte<15>), + MAP(imm_v128_shl_n_8<1>), + MAP(imm_v128_shl_n_8<2>), + MAP(imm_v128_shl_n_8<3>), + MAP(imm_v128_shl_n_8<4>), + MAP(imm_v128_shl_n_8<5>), + MAP(imm_v128_shl_n_8<6>), + MAP(imm_v128_shl_n_8<7>), + MAP(imm_v128_shr_n_u8<1>), + MAP(imm_v128_shr_n_u8<2>), + MAP(imm_v128_shr_n_u8<3>), + MAP(imm_v128_shr_n_u8<4>), + MAP(imm_v128_shr_n_u8<5>), + MAP(imm_v128_shr_n_u8<6>), + MAP(imm_v128_shr_n_u8<7>), + MAP(imm_v128_shr_n_s8<1>), + MAP(imm_v128_shr_n_s8<2>), + MAP(imm_v128_shr_n_s8<3>), + MAP(imm_v128_shr_n_s8<4>), + MAP(imm_v128_shr_n_s8<5>), + MAP(imm_v128_shr_n_s8<6>), + MAP(imm_v128_shr_n_s8<7>), + MAP(imm_v128_shl_n_16<1>), + MAP(imm_v128_shl_n_16<2>), + MAP(imm_v128_shl_n_16<4>), + MAP(imm_v128_shl_n_16<6>), + MAP(imm_v128_shl_n_16<8>), + MAP(imm_v128_shl_n_16<10>), + MAP(imm_v128_shl_n_16<12>), + MAP(imm_v128_shl_n_16<14>), + MAP(imm_v128_shr_n_u16<1>), + MAP(imm_v128_shr_n_u16<2>), + MAP(imm_v128_shr_n_u16<4>), + MAP(imm_v128_shr_n_u16<6>), + MAP(imm_v128_shr_n_u16<8>), + MAP(imm_v128_shr_n_u16<10>), + MAP(imm_v128_shr_n_u16<12>), + MAP(imm_v128_shr_n_u16<14>), + MAP(imm_v128_shr_n_s16<1>), + MAP(imm_v128_shr_n_s16<2>), + MAP(imm_v128_shr_n_s16<4>), + MAP(imm_v128_shr_n_s16<6>), + MAP(imm_v128_shr_n_s16<8>), + MAP(imm_v128_shr_n_s16<10>), + MAP(imm_v128_shr_n_s16<12>), + MAP(imm_v128_shr_n_s16<14>), + MAP(imm_v128_shl_n_32<1>), + MAP(imm_v128_shl_n_32<4>), + MAP(imm_v128_shl_n_32<8>), + MAP(imm_v128_shl_n_32<12>), + MAP(imm_v128_shl_n_32<16>), + MAP(imm_v128_shl_n_32<20>), + MAP(imm_v128_shl_n_32<24>), + MAP(imm_v128_shl_n_32<28>), + MAP(imm_v128_shr_n_u32<1>), + MAP(imm_v128_shr_n_u32<4>), + MAP(imm_v128_shr_n_u32<8>), + MAP(imm_v128_shr_n_u32<12>), + MAP(imm_v128_shr_n_u32<16>), + MAP(imm_v128_shr_n_u32<20>), + MAP(imm_v128_shr_n_u32<24>), + MAP(imm_v128_shr_n_u32<28>), + MAP(imm_v128_shr_n_s32<1>), + MAP(imm_v128_shr_n_s32<4>), + MAP(imm_v128_shr_n_s32<8>), + MAP(imm_v128_shr_n_s32<12>), + MAP(imm_v128_shr_n_s32<16>), + MAP(imm_v128_shr_n_s32<20>), + MAP(imm_v128_shr_n_s32<24>), + MAP(imm_v128_shr_n_s32<28>), + MAP(v128_from_v64), + MAP(v128_zip_8), + MAP(v128_zip_16), + MAP(v128_zip_32), + MAP(v128_mul_s16), + MAP(v128_unpack_u8_s16), + MAP(v128_unpack_s8_s16), + MAP(v128_unpack_u16_s32), + MAP(v128_unpack_s16_s32), + MAP(v128_shl_8), + MAP(v128_shr_u8), + MAP(v128_shr_s8), + MAP(v128_shl_16), + MAP(v128_shr_u16), + MAP(v128_shr_s16), + MAP(v128_shl_32), + MAP(v128_shr_u32), + MAP(v128_shr_s32), + MAP(v128_hadd_u8), + MAP(v128_dotp_s16), + MAP(v128_low_u32), + MAP(v128_low_v64), + MAP(v128_high_v64), + MAP(v128_from_64), + MAP(v128_from_32), + MAP(v128_zero), + MAP(v128_dup_8), + MAP(v128_dup_16), + MAP(v128_dup_32), + MAP(v128_unpacklo_u8_s16), + MAP(v128_unpackhi_u8_s16), + MAP(v128_unpacklo_s8_s16), + MAP(v128_unpackhi_s8_s16), + MAP(u32_load_unaligned), + MAP(u32_store_unaligned), + MAP(v64_load_unaligned), + MAP(v64_store_unaligned), + MAP(v128_load_unaligned), + MAP(v128_store_unaligned), + { NULL, NULL, NULL } }; +#undef MAP + +// Map reference functions to machine tuned functions. Since the +// functions depend on machine tuned types, the non-machine tuned +// instantiations of the test can't refer to these functions directly, +// so we refer to them by name and do the mapping here. +void Map(const char *name, fptr *ref, fptr *simd) { + unsigned int i; + for (i = 0; m[i].name && strcmp(name, m[i].name); i++) { + } + + *ref = m[i].ref; + *simd = m[i].simd; +} + +// Used for printing errors in TestSimd1Arg and TestSimd2Args +std::string Print(const uint8_t *a, int size) { + std::string text = "0x"; + for (int i = 0; i < size; i++) { + const uint8_t c = a[!CONFIG_BIG_ENDIAN ? size - 1 - i : i]; + // Same as snprintf(..., ..., "%02x", c) + text += (c >> 4) + '0' + ((c >> 4) > 9) * ('a' - '0' - 10); + text += (c & 15) + '0' + ((c & 15) > 9) * ('a' - '0' - 10); + } + + return text; +} + +// Used in TestSimd1Arg and TestSimd2Args to restrict argument ranges +void SetMask(uint8_t *s, int size, uint32_t mask, uint32_t maskwidth) { + switch (maskwidth) { + case 0: { + break; + } + case 8: { + for (int i = 0; i < size; i++) s[i] &= mask; + break; + } + case 16: { + uint16_t *t = reinterpret_cast(s); + assert(!(reinterpret_cast(s) & 1)); + for (int i = 0; i < size / 2; i++) t[i] &= mask; + break; + } + case 32: { + uint32_t *t = reinterpret_cast(s); + assert(!(reinterpret_cast(s) & 3)); + for (int i = 0; i < size / 4; i++) t[i] &= mask; + break; + } + case 64: { + uint64_t *t = reinterpret_cast(s); + assert(!(reinterpret_cast(s) & 7)); + for (int i = 0; i < size / 8; i++) t[i] &= mask; + break; + } + default: { + FAIL() << "Unsupported mask width"; + break; + } + } +} + +// We need some extra load/store functions +void u64_store_aligned(void *p, uint64_t a) { + v64_store_aligned(p, v64_from_64(a)); +} +void s32_store_aligned(void *p, int32_t a) { + u32_store_aligned(p, static_cast(a)); +} +void s64_store_aligned(void *p, int64_t a) { + v64_store_aligned(p, v64_from_64(static_cast(a))); +} + +void c_u64_store_aligned(void *p, uint64_t a) { + c_v64_store_aligned(p, c_v64_from_64(a)); +} + +void c_s32_store_aligned(void *p, int32_t a) { + c_u32_store_aligned(p, static_cast(a)); +} + +void c_s64_store_aligned(void *p, int64_t a) { + c_v64_store_aligned(p, c_v64_from_64(static_cast(a))); +} + +uint64_t u64_load_aligned(const void *p) { + return v64_u64(v64_load_aligned(p)); +} +uint16_t u16_load_aligned(const void *p) { + return *(reinterpret_cast(p)); +} +uint8_t u8_load_aligned(const void *p) { + return *(reinterpret_cast(p)); +} + +uint64_t c_u64_load_aligned(const void *p) { + return c_v64_u64(c_v64_load_aligned(p)); +} +uint16_t c_u16_load_aligned(const void *p) { + return *(reinterpret_cast(p)); +} +uint8_t c_u8_load_aligned(const void *p) { + return *(reinterpret_cast(p)); +} + +// CompareSimd1Arg and CompareSimd2Args compare intrinsics taking 1 or +// 2 arguments respectively with their corresponding C reference. +// Ideally, the loads and stores should have gone into the template +// parameter list, but v64 and v128 could be typedef'ed to the same +// type (which is the case on x86) and then we can't instantiate both +// v64 and v128, so the function return and argument types, including +// the always differing types in the C equivalent are used instead. +// The function arguments must be void pointers and then go through a +// cast to avoid matching errors in the branches eliminated by the +// typeid tests in the calling function. +template +int CompareSimd1Arg(fptr store, fptr load, fptr simd, void *d, fptr c_store, + fptr c_load, fptr c_simd, void *ref_d, const void *a) { + void (*const my_store)(void *, Ret) = (void (*const)(void *, Ret))store; + Arg (*const my_load)(const void *) = (Arg(*const)(const void *))load; + Ret (*const my_simd)(Arg) = (Ret(*const)(Arg))simd; + void (*const my_c_store)(void *, CRet) = (void (*const)(void *, CRet))c_store; + CArg (*const my_c_load)(const void *) = (CArg(*const)(const void *))c_load; + CRet (*const my_c_simd)(CArg) = (CRet(*const)(CArg))c_simd; + + // Call reference and intrinsic + my_c_store(ref_d, my_c_simd(my_c_load(a))); + my_store(d, my_simd(my_load(a))); + + // Compare results + return memcmp(ref_d, d, sizeof(CRet)); +} + +template +int CompareSimd2Args(fptr store, fptr load1, fptr load2, fptr simd, void *d, + fptr c_store, fptr c_load1, fptr c_load2, fptr c_simd, + void *ref_d, const void *a, const void *b) { + void (*const my_store)(void *, Ret) = (void (*const)(void *, Ret))store; + Arg1 (*const my_load1)(const void *) = (Arg1(*const)(const void *))load1; + Arg2 (*const my_load2)(const void *) = (Arg2(*const)(const void *))load2; + Ret (*const my_simd)(Arg1, Arg2) = (Ret(*const)(Arg1, Arg2))simd; + void (*const my_c_store)(void *, CRet) = (void (*const)(void *, CRet))c_store; + CArg1 (*const my_c_load1)(const void *) = + (CArg1(*const)(const void *))c_load1; + CArg2 (*const my_c_load2)(const void *) = + (CArg2(*const)(const void *))c_load2; + CRet (*const my_c_simd)(CArg1, CArg2) = (CRet(*const)(CArg1, CArg2))c_simd; + + // Call reference and intrinsic + my_c_store(ref_d, my_c_simd(my_c_load1(a), my_c_load2(b))); + my_store(d, my_simd(my_load1(a), my_load2(b))); + + // Compare results + return memcmp(ref_d, d, sizeof(CRet)); +} + +} // namespace + +template +void TestSimd1Arg(uint32_t iterations, uint32_t mask, uint32_t maskwidth, + const char *name) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + fptr ref_simd; + fptr simd; + int error = 0; + DECLARE_ALIGNED(32, uint8_t, s[sizeof(CArg)]); + DECLARE_ALIGNED(32, uint8_t, d[sizeof(CRet)]); + DECLARE_ALIGNED(32, uint8_t, ref_d[sizeof(CRet)]); + memset(ref_d, 0, sizeof(ref_d)); + memset(d, 0, sizeof(d)); + + Map(name, &ref_simd, &simd); + if (simd == NULL || ref_simd == NULL) { + FAIL() << "Internal error: Unknown intrinsic function " << name; + } + for (unsigned int count = 0; + count < iterations && !error && !testing::Test::HasFailure(); count++) { + for (unsigned int c = 0; c < sizeof(CArg); c++) s[c] = rnd.Rand8(); + + if (maskwidth) { + SetMask(s, sizeof(CArg), mask, maskwidth); + } + + if (typeid(CRet) == typeid(c_v64) && typeid(CArg) == typeid(c_v64)) { + // V64_V64 + error = CompareSimd1Arg( + reinterpret_cast(v64_store_aligned), + reinterpret_cast(v64_load_aligned), simd, d, + reinterpret_cast(c_v64_store_aligned), + reinterpret_cast(c_v64_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(c_v64) && + typeid(CArg) == typeid(uint8_t)) { + // V64_U8 + error = CompareSimd1Arg( + reinterpret_cast(v64_store_aligned), + reinterpret_cast(u8_load_aligned), simd, d, + reinterpret_cast(c_v64_store_aligned), + reinterpret_cast(c_u8_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(c_v64) && + typeid(CArg) == typeid(uint16_t)) { + // V64_U16 + error = CompareSimd1Arg( + reinterpret_cast(v64_store_aligned), + reinterpret_cast(u16_load_aligned), simd, d, + reinterpret_cast(c_v64_store_aligned), + reinterpret_cast(c_u16_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(c_v64) && + typeid(CArg) == typeid(uint32_t)) { + // V64_U32 + error = CompareSimd1Arg( + reinterpret_cast(v64_store_aligned), + reinterpret_cast(u32_load_aligned), simd, d, + reinterpret_cast(c_v64_store_aligned), + reinterpret_cast(c_u32_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(uint64_t) && + typeid(CArg) == typeid(c_v64)) { + // U64_V64 + error = CompareSimd1Arg( + reinterpret_cast(u64_store_aligned), + reinterpret_cast(v64_load_aligned), simd, d, + reinterpret_cast(c_u64_store_aligned), + reinterpret_cast(c_v64_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(int64_t) && + typeid(CArg) == typeid(c_v64)) { + // S64_V64 + error = CompareSimd1Arg( + reinterpret_cast(s64_store_aligned), + reinterpret_cast(v64_load_aligned), simd, d, + reinterpret_cast(c_s64_store_aligned), + reinterpret_cast(c_v64_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(uint32_t) && + typeid(CArg) == typeid(c_v64)) { + // U32_V64 + error = CompareSimd1Arg( + reinterpret_cast(u32_store_aligned), + reinterpret_cast(v64_load_aligned), simd, d, + reinterpret_cast(c_u32_store_aligned), + reinterpret_cast(c_v64_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(int32_t) && + typeid(CArg) == typeid(c_v64)) { + // S32_V64 + error = CompareSimd1Arg( + reinterpret_cast(s32_store_aligned), + reinterpret_cast(v64_load_aligned), simd, d, + reinterpret_cast(c_s32_store_aligned), + reinterpret_cast(c_v64_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(uint32_t) && + typeid(CArg) == typeid(c_v128)) { + // U32_V128 + error = CompareSimd1Arg( + reinterpret_cast(u32_store_aligned), + reinterpret_cast(v128_load_aligned), simd, d, + reinterpret_cast(c_u32_store_aligned), + reinterpret_cast(c_v128_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(uint64_t) && + typeid(CArg) == typeid(c_v128)) { + // U64_V128 + error = CompareSimd1Arg( + reinterpret_cast(u64_store_aligned), + reinterpret_cast(v128_load_aligned), simd, d, + reinterpret_cast(c_u64_store_aligned), + reinterpret_cast(c_v128_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(c_v64) && + typeid(CArg) == typeid(c_v128)) { + // V64_V128 + error = CompareSimd1Arg( + reinterpret_cast(v64_store_aligned), + reinterpret_cast(v128_load_aligned), simd, d, + reinterpret_cast(c_v64_store_aligned), + reinterpret_cast(c_v128_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(c_v128) && + typeid(CArg) == typeid(c_v128)) { + // V128_V128 + error = CompareSimd1Arg( + reinterpret_cast(v128_store_aligned), + reinterpret_cast(v128_load_aligned), simd, d, + reinterpret_cast(c_v128_store_aligned), + reinterpret_cast(c_v128_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(c_v128) && + typeid(CArg) == typeid(c_v64)) { + // V128_V64 + error = CompareSimd1Arg( + reinterpret_cast(v128_store_aligned), + reinterpret_cast(v64_load_aligned), simd, d, + reinterpret_cast(c_v128_store_aligned), + reinterpret_cast(c_v64_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(c_v128) && + typeid(CArg) == typeid(uint8_t)) { + // V128_U8 + error = CompareSimd1Arg( + reinterpret_cast(v128_store_aligned), + reinterpret_cast(u8_load_aligned), simd, d, + reinterpret_cast(c_v128_store_aligned), + reinterpret_cast(c_u8_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(c_v128) && + typeid(CArg) == typeid(uint16_t)) { + // V128_U16 + error = CompareSimd1Arg( + reinterpret_cast(v128_store_aligned), + reinterpret_cast(u16_load_aligned), simd, d, + reinterpret_cast(c_v128_store_aligned), + reinterpret_cast(c_u16_load_aligned), ref_simd, ref_d, s); + } else if (typeid(CRet) == typeid(c_v128) && + typeid(CArg) == typeid(uint32_t)) { + // V128_U32 + error = CompareSimd1Arg( + reinterpret_cast(v128_store_aligned), + reinterpret_cast(u32_load_aligned), simd, d, + reinterpret_cast(c_v128_store_aligned), + reinterpret_cast(c_u32_load_aligned), ref_simd, ref_d, s); + } else { + FAIL() << "Internal error: Unknown intrinsic function " + << typeid(CRet).name() << " " << name << "(" << typeid(CArg).name() + << ")"; + } + } + + EXPECT_EQ(0, error) << "Error: mismatch for " << name << "(" + << Print(s, sizeof(s)) << ") -> " << Print(d, sizeof(d)) + << " (simd), " << Print(ref_d, sizeof(ref_d)) << " (ref)"; +} + +template +void TestSimd2Args(uint32_t iterations, uint32_t mask, uint32_t maskwidth, + const char *name) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + fptr ref_simd; + fptr simd; + int error = 0; + DECLARE_ALIGNED(32, uint8_t, s1[sizeof(CArg1)]); + DECLARE_ALIGNED(32, uint8_t, s2[sizeof(CArg2)]); + DECLARE_ALIGNED(32, uint8_t, d[sizeof(CRet)]); + DECLARE_ALIGNED(32, uint8_t, ref_d[sizeof(CRet)]); + memset(ref_d, 0, sizeof(ref_d)); + memset(d, 0, sizeof(d)); + + Map(name, &ref_simd, &simd); + if (simd == NULL || ref_simd == NULL) { + FAIL() << "Internal error: Unknown intrinsic function " << name; + } + + for (unsigned int count = 0; + count < iterations && !error && !testing::Test::HasFailure(); count++) { + for (unsigned int c = 0; c < sizeof(CArg1); c++) s1[c] = rnd.Rand8(); + + for (unsigned int c = 0; c < sizeof(CArg2); c++) s2[c] = rnd.Rand8(); + + if (maskwidth) SetMask(s2, sizeof(CArg2), mask, maskwidth); + + if (typeid(CRet) == typeid(c_v64) && typeid(CArg1) == typeid(c_v64) && + typeid(CArg2) == typeid(c_v64)) { + // V64_V64V64 + error = CompareSimd2Args( + reinterpret_cast(v64_store_aligned), + reinterpret_cast(v64_load_aligned), + reinterpret_cast(v64_load_aligned), simd, d, + reinterpret_cast(c_v64_store_aligned), + reinterpret_cast(c_v64_load_aligned), + reinterpret_cast(c_v64_load_aligned), + reinterpret_cast(ref_simd), ref_d, s1, s2); + } else if (typeid(CRet) == typeid(c_v64) && + typeid(CArg1) == typeid(uint32_t) && + typeid(CArg2) == typeid(uint32_t)) { + // V64_U32U32 + error = CompareSimd2Args( + reinterpret_cast(v64_store_aligned), + reinterpret_cast(u32_load_aligned), + reinterpret_cast(u32_load_aligned), simd, d, + reinterpret_cast(c_v64_store_aligned), + reinterpret_cast(c_u32_load_aligned), + reinterpret_cast(c_u32_load_aligned), + reinterpret_cast(ref_simd), ref_d, s1, s2); + } else if (typeid(CRet) == typeid(uint32_t) && + typeid(CArg1) == typeid(c_v64) && + typeid(CArg2) == typeid(c_v64)) { + // U32_V64V64 + error = CompareSimd2Args( + reinterpret_cast(u32_store_aligned), + reinterpret_cast(v64_load_aligned), + reinterpret_cast(v64_load_aligned), simd, d, + reinterpret_cast(c_u32_store_aligned), + reinterpret_cast(c_v64_load_aligned), + reinterpret_cast(c_v64_load_aligned), + reinterpret_cast(ref_simd), ref_d, s1, s2); + } else if (typeid(CRet) == typeid(int64_t) && + typeid(CArg1) == typeid(c_v64) && + typeid(CArg2) == typeid(c_v64)) { + // S64_V64V64 + error = CompareSimd2Args( + reinterpret_cast(s64_store_aligned), + reinterpret_cast(v64_load_aligned), + reinterpret_cast(v64_load_aligned), simd, d, + reinterpret_cast(c_s64_store_aligned), + reinterpret_cast(c_v64_load_aligned), + reinterpret_cast(c_v64_load_aligned), + reinterpret_cast(ref_simd), ref_d, s1, s2); + } else if (typeid(CRet) == typeid(c_v64) && + typeid(CArg1) == typeid(c_v64) && + typeid(CArg2) == typeid(uint32_t)) { + // V64_V64U32 + error = CompareSimd2Args( + reinterpret_cast(v64_store_aligned), + reinterpret_cast(v64_load_aligned), + reinterpret_cast(u32_load_aligned), simd, d, + reinterpret_cast(c_v64_store_aligned), + reinterpret_cast(c_v64_load_aligned), + reinterpret_cast(c_u32_load_aligned), + reinterpret_cast(ref_simd), ref_d, s1, s2); + } else if (typeid(CRet) == typeid(c_v128) && + typeid(CArg1) == typeid(c_v128) && + typeid(CArg2) == typeid(c_v128)) { + // V128_V128V128 + error = CompareSimd2Args( + reinterpret_cast(v128_store_aligned), + reinterpret_cast(v128_load_aligned), + reinterpret_cast(v128_load_aligned), simd, d, + reinterpret_cast(c_v128_store_aligned), + reinterpret_cast(c_v128_load_aligned), + reinterpret_cast(c_v128_load_aligned), + reinterpret_cast(ref_simd), ref_d, s1, s2); + } else if (typeid(CRet) == typeid(uint32_t) && + typeid(CArg1) == typeid(c_v128) && + typeid(CArg2) == typeid(c_v128)) { + // U32_V128V128 + error = CompareSimd2Args( + reinterpret_cast(u32_store_aligned), + reinterpret_cast(v128_load_aligned), + reinterpret_cast(v128_load_aligned), simd, d, + reinterpret_cast(c_u32_store_aligned), + reinterpret_cast(c_v128_load_aligned), + reinterpret_cast(c_v128_load_aligned), + reinterpret_cast(ref_simd), ref_d, s1, s2); + } else if (typeid(CRet) == typeid(int64_t) && + typeid(CArg1) == typeid(c_v128) && + typeid(CArg2) == typeid(c_v128)) { + // S64_V128V128 + error = CompareSimd2Args( + reinterpret_cast(s64_store_aligned), + reinterpret_cast(v128_load_aligned), + reinterpret_cast(v128_load_aligned), simd, d, + reinterpret_cast(c_s64_store_aligned), + reinterpret_cast(c_v128_load_aligned), + reinterpret_cast(c_v128_load_aligned), + reinterpret_cast(ref_simd), ref_d, s1, s2); + } else if (typeid(CRet) == typeid(c_v128) && + typeid(CArg1) == typeid(uint64_t) && + typeid(CArg2) == typeid(uint64_t)) { + // V128_U64U64 + error = CompareSimd2Args( + reinterpret_cast(v128_store_aligned), + reinterpret_cast(u64_load_aligned), + reinterpret_cast(u64_load_aligned), simd, d, + reinterpret_cast(c_v128_store_aligned), + reinterpret_cast(c_u64_load_aligned), + reinterpret_cast(c_u64_load_aligned), + reinterpret_cast(ref_simd), ref_d, s1, s2); + } else if (typeid(CRet) == typeid(c_v128) && + typeid(CArg1) == typeid(c_v64) && + typeid(CArg2) == typeid(c_v64)) { + // V128_V64V64 + error = CompareSimd2Args( + reinterpret_cast(v128_store_aligned), + reinterpret_cast(v64_load_aligned), + reinterpret_cast(v64_load_aligned), simd, d, + reinterpret_cast(c_v128_store_aligned), + reinterpret_cast(c_v64_load_aligned), + reinterpret_cast(c_v64_load_aligned), + reinterpret_cast(ref_simd), ref_d, s1, s2); + } else if (typeid(CRet) == typeid(c_v128) && + typeid(CArg1) == typeid(c_v128) && + typeid(CArg2) == typeid(uint32_t)) { + // V128_V128U32 + error = CompareSimd2Args( + reinterpret_cast(v128_store_aligned), + reinterpret_cast(v128_load_aligned), + reinterpret_cast(u32_load_aligned), simd, d, + reinterpret_cast(c_v128_store_aligned), + reinterpret_cast(c_v128_load_aligned), + reinterpret_cast(c_u32_load_aligned), + reinterpret_cast(ref_simd), ref_d, s1, s2); + } else { + FAIL() << "Internal error: Unknown intrinsic function " + << typeid(CRet).name() << " " << name << "(" + << typeid(CArg1).name() << ", " << typeid(CArg2).name() << ")"; + } + } + + EXPECT_EQ(0, error) << "Error: mismatch for " << name << "(" + << Print(s1, sizeof(s1)) << ", " << Print(s2, sizeof(s2)) + << ") -> " << Print(d, sizeof(d)) << " (simd), " + << Print(ref_d, sizeof(ref_d)) << " (ref)"; +} + +// Instantiations to make the functions callable from another files +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd2Args(uint32_t, uint32_t, + uint32_t, const char *); +template void TestSimd2Args(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd2Args(uint32_t, uint32_t, + uint32_t, const char *); +template void TestSimd2Args(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd2Args(uint32_t, uint32_t, + uint32_t, const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd1Arg(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd2Args(uint32_t, uint32_t, + uint32_t, const char *); +template void TestSimd2Args(uint32_t, uint32_t, + uint32_t, const char *); +template void TestSimd2Args(uint32_t, uint32_t, + uint32_t, const char *); +template void TestSimd2Args(uint32_t, uint32_t, uint32_t, + const char *); +template void TestSimd2Args(uint32_t, uint32_t, + uint32_t, const char *); +template void TestSimd2Args(uint32_t, uint32_t, + uint32_t, const char *); + +} // namespace SIMD_NAMESPACE diff --git a/third_party/aom/test/simd_cmp_neon.cc b/third_party/aom/test/simd_cmp_neon.cc new file mode 100644 index 0000000000..c8004cc8b1 --- /dev/null +++ b/third_party/aom/test/simd_cmp_neon.cc @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if defined(__OPTIMIZE__) && __OPTIMIZE__ +#define ARCH NEON +#define ARCH_POSTFIX(name) name##_neon +#define SIMD_NAMESPACE simd_test_neon +#include "./simd_cmp_impl.h" +#endif diff --git a/third_party/aom/test/simd_cmp_sse2.cc b/third_party/aom/test/simd_cmp_sse2.cc new file mode 100644 index 0000000000..67cb43c101 --- /dev/null +++ b/third_party/aom/test/simd_cmp_sse2.cc @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if (defined(__OPTIMIZE__) && __OPTIMIZE__) || \ + (!defined(__GNUC__) && !defined(_DEBUG)) +#define ARCH SSE2 +#define ARCH_POSTFIX(name) name##_sse2 +#define SIMD_NAMESPACE simd_test_sse2 +#include "./simd_cmp_impl.h" +#endif diff --git a/third_party/aom/test/simd_cmp_sse4.cc b/third_party/aom/test/simd_cmp_sse4.cc new file mode 100644 index 0000000000..ba826d8983 --- /dev/null +++ b/third_party/aom/test/simd_cmp_sse4.cc @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if (defined(__OPTIMIZE__) && __OPTIMIZE__) || \ + (!defined(__GNUC__) && !defined(_DEBUG)) +#define ARCH SSE4_1 +#define ARCH_POSTFIX(name) name##_sse4_1 +#define SIMD_NAMESPACE simd_test_sse4_1 +#include "./simd_cmp_impl.h" +#endif diff --git a/third_party/aom/test/simd_cmp_ssse3.cc b/third_party/aom/test/simd_cmp_ssse3.cc new file mode 100644 index 0000000000..a6c7000fd3 --- /dev/null +++ b/third_party/aom/test/simd_cmp_ssse3.cc @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if (defined(__OPTIMIZE__) && __OPTIMIZE__) || \ + (!defined(__GNUC__) && !defined(_DEBUG)) +#define ARCH SSSE3 +#define ARCH_POSTFIX(name) name##_ssse3 +#define SIMD_NAMESPACE simd_test_ssse3 +#include "./simd_cmp_impl.h" +#endif diff --git a/third_party/aom/test/simd_impl.h b/third_party/aom/test/simd_impl.h new file mode 100644 index 0000000000..5cfda675d3 --- /dev/null +++ b/third_party/aom/test/simd_impl.h @@ -0,0 +1,594 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#define SIMD_CHECK 1 +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "aom_dsp/aom_simd_inline.h" +#include "aom_dsp/simd/v128_intrinsics_c.h" + +namespace SIMD_NAMESPACE { + +template +class TestIntrinsic : public ::testing::TestWithParam { + public: + virtual ~TestIntrinsic() {} + virtual void SetUp() { + mask = std::tr1::get<0>(this->GetParam()); + maskwidth = std::tr1::get<1>(this->GetParam()); + name = std::tr1::get<2>(this->GetParam()); + } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + uint32_t mask, maskwidth; + const char *name; +}; + +// Create one typedef for each function signature +#define TYPEDEF_SIMD(name) \ + typedef TestIntrinsic > \ + ARCH_POSTFIX(name) + +TYPEDEF_SIMD(V64_U8); +TYPEDEF_SIMD(V64_U16); +TYPEDEF_SIMD(V64_U32); +TYPEDEF_SIMD(V64_V64); +TYPEDEF_SIMD(U32_V64); +TYPEDEF_SIMD(S32_V64); +TYPEDEF_SIMD(U64_V64); +TYPEDEF_SIMD(S64_V64); +TYPEDEF_SIMD(V64_U32U32); +TYPEDEF_SIMD(V64_V64V64); +TYPEDEF_SIMD(S64_V64V64); +TYPEDEF_SIMD(V64_V64U32); +TYPEDEF_SIMD(U32_V64V64); +TYPEDEF_SIMD(V128_V64); +TYPEDEF_SIMD(V128_V128); +TYPEDEF_SIMD(U32_V128); +TYPEDEF_SIMD(U64_V128); +TYPEDEF_SIMD(V64_V128); +TYPEDEF_SIMD(V128_U8); +TYPEDEF_SIMD(V128_U16); +TYPEDEF_SIMD(V128_U32); +TYPEDEF_SIMD(V128_U64U64); +TYPEDEF_SIMD(V128_V64V64); +TYPEDEF_SIMD(V128_V128V128); +TYPEDEF_SIMD(S64_V128V128); +TYPEDEF_SIMD(V128_V128U32); +TYPEDEF_SIMD(U32_V128V128); + +// Google Test allows up to 50 tests per case, so split the largest +typedef ARCH_POSTFIX(V64_V64) ARCH_POSTFIX(V64_V64_Part2); +typedef ARCH_POSTFIX(V64_V64V64) ARCH_POSTFIX(V64_V64V64_Part2); +typedef ARCH_POSTFIX(V128_V128) ARCH_POSTFIX(V128_V128_Part2); +typedef ARCH_POSTFIX(V128_V128) ARCH_POSTFIX(V128_V128_Part3); +typedef ARCH_POSTFIX(V128_V128V128) ARCH_POSTFIX(V128_V128V128_Part2); + +// These functions are machine tuned located elsewhere +template +void TestSimd1Arg(uint32_t iterations, uint32_t mask, uint32_t maskwidth, + const char *name); + +template +void TestSimd2Args(uint32_t iterations, uint32_t mask, uint32_t maskwidth, + const char *name); + +const int kIterations = 65536; + +// Add a macro layer since TEST_P will quote the name so we need to +// expand it first with the prefix. +#define MY_TEST_P(name, test) TEST_P(name, test) + +MY_TEST_P(ARCH_POSTFIX(V64_U8), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V64_U16), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V64_U32), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V64_V64), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(U64_V64), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(S64_V64), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(U32_V64), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(S32_V64), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V64_U32U32), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V64_V64V64), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(S64_V64V64), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(U32_V64V64), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V64_V64U32), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +// Google Test allows up to 50 tests per case, so split the largest +MY_TEST_P(ARCH_POSTFIX(V64_V64_Part2), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V64_V64V64_Part2), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(U32_V128), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(U64_V128), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V64_V128), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V128_V128), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V128_U8), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V128_U16), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V128_U32), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V128_V64), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V128_V128V128), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(U32_V128V128), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(S64_V128V128), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V128_U64U64), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V128_V64V64), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V128_V128U32), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V128_V128V128_Part2), TestIntrinsics) { + TestSimd2Args(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V128_V128_Part2), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +MY_TEST_P(ARCH_POSTFIX(V128_V128_Part3), TestIntrinsics) { + TestSimd1Arg(kIterations, mask, maskwidth, name); +} + +// Add a macro layer since INSTANTIATE_TEST_CASE_P will quote the name +// so we need to expand it first with the prefix +#define INSTANTIATE(name, type, ...) \ + INSTANTIATE_TEST_CASE_P(name, type, ::testing::Values(__VA_ARGS__)) + +#define SIMD_TUPLE(name, mask, maskwidth) \ + std::tr1::make_tuple(mask, maskwidth, static_cast(#name)) + +INSTANTIATE(ARCH, ARCH_POSTFIX(U32_V64V64), + (SIMD_TUPLE(v64_sad_u8, 0U, 0U), SIMD_TUPLE(v64_ssd_u8, 0U, 0U))); + +INSTANTIATE( + ARCH, ARCH_POSTFIX(V64_V64V64), SIMD_TUPLE(v64_add_8, 0U, 0U), + SIMD_TUPLE(v64_add_16, 0U, 0U), SIMD_TUPLE(v64_sadd_s16, 0U, 0U), + SIMD_TUPLE(v64_add_32, 0U, 0U), SIMD_TUPLE(v64_sub_8, 0U, 0U), + SIMD_TUPLE(v64_ssub_u8, 0U, 0U), SIMD_TUPLE(v64_ssub_s8, 0U, 0U), + SIMD_TUPLE(v64_sub_16, 0U, 0U), SIMD_TUPLE(v64_ssub_s16, 0U, 0U), + SIMD_TUPLE(v64_ssub_u16, 0U, 0U), SIMD_TUPLE(v64_sub_32, 0U, 0U), + SIMD_TUPLE(v64_ziplo_8, 0U, 0U), SIMD_TUPLE(v64_ziphi_8, 0U, 0U), + SIMD_TUPLE(v64_ziplo_16, 0U, 0U), SIMD_TUPLE(v64_ziphi_16, 0U, 0U), + SIMD_TUPLE(v64_ziplo_32, 0U, 0U), SIMD_TUPLE(v64_ziphi_32, 0U, 0U), + SIMD_TUPLE(v64_pack_s32_s16, 0U, 0U), SIMD_TUPLE(v64_pack_s16_u8, 0U, 0U), + SIMD_TUPLE(v64_pack_s16_s8, 0U, 0U), SIMD_TUPLE(v64_unziphi_8, 0U, 0U), + SIMD_TUPLE(v64_unziplo_8, 0U, 0U), SIMD_TUPLE(v64_unziphi_16, 0U, 0U), + SIMD_TUPLE(v64_unziplo_16, 0U, 0U), SIMD_TUPLE(v64_or, 0U, 0U), + SIMD_TUPLE(v64_xor, 0U, 0U), SIMD_TUPLE(v64_and, 0U, 0U), + SIMD_TUPLE(v64_andn, 0U, 0U), SIMD_TUPLE(v64_mullo_s16, 0U, 0U), + SIMD_TUPLE(v64_mulhi_s16, 0U, 0U), SIMD_TUPLE(v64_mullo_s32, 0U, 0U), + SIMD_TUPLE(v64_madd_s16, 0U, 0U), SIMD_TUPLE(v64_madd_us8, 0U, 0U), + SIMD_TUPLE(v64_avg_u8, 0U, 0U), SIMD_TUPLE(v64_rdavg_u8, 0U, 0U), + SIMD_TUPLE(v64_avg_u16, 0U, 0U), SIMD_TUPLE(v64_min_u8, 0U, 0U), + SIMD_TUPLE(v64_max_u8, 0U, 0U), SIMD_TUPLE(v64_min_s8, 0U, 0U), + SIMD_TUPLE(v64_max_s8, 0U, 0U), SIMD_TUPLE(v64_min_s16, 0U, 0U), + SIMD_TUPLE(v64_max_s16, 0U, 0U), SIMD_TUPLE(v64_cmpgt_s8, 0U, 0U), + SIMD_TUPLE(v64_cmplt_s8, 0U, 0U), SIMD_TUPLE(v64_cmpeq_8, 0U, 0U), + SIMD_TUPLE(v64_cmpgt_s16, 0U, 0U), SIMD_TUPLE(v64_cmplt_s16, 0U, 0U), + SIMD_TUPLE(v64_cmpeq_16, 0U, 0U)); + +INSTANTIATE( + ARCH, ARCH_POSTFIX(V64_V64V64_Part2), SIMD_TUPLE(v64_shuffle_8, 7U, 8U), + SIMD_TUPLE(imm_v64_align<1>, 0U, 0U), SIMD_TUPLE(imm_v64_align<2>, 0U, 0U), + SIMD_TUPLE(imm_v64_align<3>, 0U, 0U), SIMD_TUPLE(imm_v64_align<4>, 0U, 0U), + SIMD_TUPLE(imm_v64_align<5>, 0U, 0U), SIMD_TUPLE(imm_v64_align<6>, 0U, 0U), + SIMD_TUPLE(imm_v64_align<7>, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V64_V64), SIMD_TUPLE(v64_abs_s8, 0U, 0U), + SIMD_TUPLE(v64_abs_s16, 0U, 0U), + SIMD_TUPLE(v64_unpacklo_u8_s16, 0U, 0U), + SIMD_TUPLE(v64_unpackhi_u8_s16, 0U, 0U), + SIMD_TUPLE(v64_unpacklo_s8_s16, 0U, 0U), + SIMD_TUPLE(v64_unpackhi_s8_s16, 0U, 0U), + SIMD_TUPLE(v64_unpacklo_u16_s32, 0U, 0U), + SIMD_TUPLE(v64_unpacklo_s16_s32, 0U, 0U), + SIMD_TUPLE(v64_unpackhi_u16_s32, 0U, 0U), + SIMD_TUPLE(v64_unpackhi_s16_s32, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_byte<1>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_byte<2>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_byte<3>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_byte<4>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_byte<5>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_byte<6>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_byte<7>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_byte<1>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_byte<2>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_byte<3>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_byte<4>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_byte<5>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_byte<6>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_byte<7>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_8<1>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_8<2>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_8<3>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_8<4>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_8<5>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_8<6>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_8<7>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u8<1>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u8<2>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u8<3>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u8<4>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u8<5>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u8<6>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u8<7>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s8<1>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s8<2>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s8<3>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s8<4>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s8<5>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s8<6>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s8<7>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_16<1>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_16<2>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_16<4>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_16<6>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_16<8>, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V64_V64_Part2), + SIMD_TUPLE(imm_v64_shl_n_16<10>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_16<12>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_16<14>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u16<1>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u16<2>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u16<4>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u16<6>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u16<8>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u16<10>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u16<12>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u16<14>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s16<1>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s16<2>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s16<4>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s16<6>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s16<8>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s16<10>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s16<12>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s16<14>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_32<1>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_32<4>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_32<8>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_32<12>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_32<16>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_32<20>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_32<24>, 0U, 0U), + SIMD_TUPLE(imm_v64_shl_n_32<28>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u32<1>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u32<4>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u32<8>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u32<12>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u32<16>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u32<20>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u32<24>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_u32<28>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s32<1>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s32<4>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s32<8>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s32<12>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s32<16>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s32<20>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s32<24>, 0U, 0U), + SIMD_TUPLE(imm_v64_shr_n_s32<28>, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V64_V64U32), SIMD_TUPLE(v64_shl_8, 7U, 32U), + SIMD_TUPLE(v64_shr_u8, 7U, 32U), SIMD_TUPLE(v64_shr_s8, 7U, 32U), + SIMD_TUPLE(v64_shl_16, 15U, 32U), SIMD_TUPLE(v64_shr_u16, 15U, 32U), + SIMD_TUPLE(v64_shr_s16, 15U, 32U), SIMD_TUPLE(v64_shl_32, 31U, 32U), + SIMD_TUPLE(v64_shr_u32, 31U, 32U), + SIMD_TUPLE(v64_shr_s32, 31U, 32U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(U64_V64), SIMD_TUPLE(v64_hadd_u8, 0U, 0U), + SIMD_TUPLE(v64_u64, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(S64_V64), SIMD_TUPLE(v64_hadd_s16, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(U32_V64), SIMD_TUPLE(v64_low_u32, 0U, 0U), + SIMD_TUPLE(v64_high_u32, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(S32_V64), SIMD_TUPLE(v64_low_s32, 0U, 0U), + SIMD_TUPLE(v64_high_s32, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(S64_V64V64), SIMD_TUPLE(v64_dotp_s16, 0U, 0U), + SIMD_TUPLE(v64_dotp_su8, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V64_U8), SIMD_TUPLE(v64_dup_8, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V64_U16), SIMD_TUPLE(v64_dup_16, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V64_U32), SIMD_TUPLE(v64_dup_32, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V64_U32U32), SIMD_TUPLE(v64_from_32, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(U32_V128V128), SIMD_TUPLE(v128_sad_u8, 0U, 0U), + SIMD_TUPLE(v128_ssd_u8, 0U, 0U)); + +INSTANTIATE( + ARCH, ARCH_POSTFIX(V128_V128V128), SIMD_TUPLE(v128_add_8, 0U, 0U), + SIMD_TUPLE(v128_add_16, 0U, 0U), SIMD_TUPLE(v128_sadd_s16, 0U, 0U), + SIMD_TUPLE(v128_add_32, 0U, 0U), SIMD_TUPLE(v128_sub_8, 0U, 0U), + SIMD_TUPLE(v128_ssub_u8, 0U, 0U), SIMD_TUPLE(v128_ssub_s8, 0U, 0U), + SIMD_TUPLE(v128_sub_16, 0U, 0U), SIMD_TUPLE(v128_ssub_s16, 0U, 0U), + SIMD_TUPLE(v128_ssub_u16, 0U, 0U), SIMD_TUPLE(v128_sub_32, 0U, 0U), + SIMD_TUPLE(v128_ziplo_8, 0U, 0U), SIMD_TUPLE(v128_ziphi_8, 0U, 0U), + SIMD_TUPLE(v128_ziplo_16, 0U, 0U), SIMD_TUPLE(v128_ziphi_16, 0U, 0U), + SIMD_TUPLE(v128_ziplo_32, 0U, 0U), SIMD_TUPLE(v128_ziphi_32, 0U, 0U), + SIMD_TUPLE(v128_ziplo_64, 0U, 0U), SIMD_TUPLE(v128_ziphi_64, 0U, 0U), + SIMD_TUPLE(v128_unziphi_8, 0U, 0U), SIMD_TUPLE(v128_unziplo_8, 0U, 0U), + SIMD_TUPLE(v128_unziphi_16, 0U, 0U), SIMD_TUPLE(v128_unziplo_16, 0U, 0U), + SIMD_TUPLE(v128_unziphi_32, 0U, 0U), SIMD_TUPLE(v128_unziplo_32, 0U, 0U), + SIMD_TUPLE(v128_pack_s32_s16, 0U, 0U), SIMD_TUPLE(v128_pack_s16_u8, 0U, 0U), + SIMD_TUPLE(v128_pack_s16_s8, 0U, 0U), SIMD_TUPLE(v128_or, 0U, 0U), + SIMD_TUPLE(v128_xor, 0U, 0U), SIMD_TUPLE(v128_and, 0U, 0U), + SIMD_TUPLE(v128_andn, 0U, 0U), SIMD_TUPLE(v128_mullo_s16, 0U, 0U), + SIMD_TUPLE(v128_mulhi_s16, 0U, 0U), SIMD_TUPLE(v128_mullo_s32, 0U, 0U), + SIMD_TUPLE(v128_madd_s16, 0U, 0U), SIMD_TUPLE(v128_madd_us8, 0U, 0U), + SIMD_TUPLE(v128_avg_u8, 0U, 0U), SIMD_TUPLE(v128_rdavg_u8, 0U, 0U), + SIMD_TUPLE(v128_avg_u16, 0U, 0U), SIMD_TUPLE(v128_min_u8, 0U, 0U), + SIMD_TUPLE(v128_max_u8, 0U, 0U), SIMD_TUPLE(v128_min_s8, 0U, 0U), + SIMD_TUPLE(v128_max_s8, 0U, 0U), SIMD_TUPLE(v128_min_s16, 0U, 0U), + SIMD_TUPLE(v128_max_s16, 0U, 0U), SIMD_TUPLE(v128_cmpgt_s8, 0U, 0U), + SIMD_TUPLE(v128_cmplt_s8, 0U, 0U), SIMD_TUPLE(v128_cmpeq_8, 0U, 0U), + SIMD_TUPLE(v128_cmpgt_s16, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V128_V128V128_Part2), + SIMD_TUPLE(v128_cmpeq_16, 0U, 0U), + SIMD_TUPLE(v128_cmplt_s16, 0U, 0U), + SIMD_TUPLE(v128_shuffle_8, 15U, 8U), + SIMD_TUPLE(imm_v128_align<1>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<2>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<3>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<4>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<5>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<6>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<7>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<8>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<9>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<10>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<11>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<12>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<13>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<14>, 0U, 0U), + SIMD_TUPLE(imm_v128_align<15>, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V128_V128), SIMD_TUPLE(v128_abs_s8, 0U, 0U), + SIMD_TUPLE(v128_abs_s16, 0U, 0U), SIMD_TUPLE(v128_padd_s16, 0U, 0U), + SIMD_TUPLE(v128_unpacklo_u8_s16, 0U, 0U), + SIMD_TUPLE(v128_unpacklo_s8_s16, 0U, 0U), + SIMD_TUPLE(v128_unpacklo_u16_s32, 0U, 0U), + SIMD_TUPLE(v128_unpacklo_s16_s32, 0U, 0U), + SIMD_TUPLE(v128_unpackhi_u8_s16, 0U, 0U), + SIMD_TUPLE(v128_unpackhi_s8_s16, 0U, 0U), + SIMD_TUPLE(v128_unpackhi_u16_s32, 0U, 0U), + SIMD_TUPLE(v128_unpackhi_s16_s32, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<1>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<2>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<3>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<4>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<5>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<6>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<7>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<8>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<9>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<10>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<11>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<12>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<13>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<14>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_byte<15>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<1>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<2>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<3>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<4>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<5>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<6>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<7>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<8>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<9>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<10>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<11>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<12>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<13>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<14>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_byte<15>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_8<1>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_8<2>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_8<3>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_8<4>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_8<5>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_8<6>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_8<7>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u8<1>, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V128_V128_Part2), + SIMD_TUPLE(imm_v128_shr_n_u8<2>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u8<3>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u8<4>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u8<5>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u8<6>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u8<7>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s8<1>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s8<2>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s8<3>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s8<4>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s8<5>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s8<6>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s8<7>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_16<1>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_16<2>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_16<4>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_16<6>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_16<8>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_16<10>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_16<12>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_16<14>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u16<1>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u16<2>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u16<4>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u16<6>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u16<8>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u16<10>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u16<12>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u16<14>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s16<1>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s16<2>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s16<4>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s16<6>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s16<8>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s16<10>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s16<12>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s16<14>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_32<1>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_32<4>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_32<8>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_32<12>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_32<16>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_32<20>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_32<24>, 0U, 0U), + SIMD_TUPLE(imm_v128_shl_n_32<28>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u32<1>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u32<4>, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V128_V128_Part3), + SIMD_TUPLE(imm_v128_shr_n_u32<8>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u32<12>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u32<16>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u32<20>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u32<24>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_u32<28>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s32<1>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s32<4>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s32<8>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s32<12>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s32<16>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s32<20>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s32<24>, 0U, 0U), + SIMD_TUPLE(imm_v128_shr_n_s32<28>, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V128_V64V64), SIMD_TUPLE(v128_from_v64, 0U, 0U), + SIMD_TUPLE(v128_zip_8, 0U, 0U), SIMD_TUPLE(v128_zip_16, 0U, 0U), + SIMD_TUPLE(v128_zip_32, 0U, 0U), SIMD_TUPLE(v128_mul_s16, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V128_U64U64), SIMD_TUPLE(v128_from_64, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V128_V64), + SIMD_TUPLE(v128_unpack_u8_s16, 0U, 0U), + SIMD_TUPLE(v128_unpack_s8_s16, 0U, 0U), + SIMD_TUPLE(v128_unpack_u16_s32, 0U, 0U), + SIMD_TUPLE(v128_unpack_s16_s32, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V128_V128U32), SIMD_TUPLE(v128_shl_8, 7U, 32U), + SIMD_TUPLE(v128_shr_u8, 7U, 32U), SIMD_TUPLE(v128_shr_s8, 7U, 32U), + SIMD_TUPLE(v128_shl_16, 15U, 32U), + SIMD_TUPLE(v128_shr_u16, 15U, 32U), + SIMD_TUPLE(v128_shr_s16, 15U, 32U), + SIMD_TUPLE(v128_shl_32, 31U, 32U), + SIMD_TUPLE(v128_shr_u32, 31U, 32U), + SIMD_TUPLE(v128_shr_s32, 31U, 32U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(U32_V128), SIMD_TUPLE(v128_low_u32, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(U64_V128), SIMD_TUPLE(v128_hadd_u8, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V64_V128), SIMD_TUPLE(v128_low_v64, 0U, 0U), + SIMD_TUPLE(v128_high_v64, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V128_U8), SIMD_TUPLE(v128_dup_8, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V128_U16), SIMD_TUPLE(v128_dup_16, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(V128_U32), SIMD_TUPLE(v128_dup_32, 0U, 0U)); + +INSTANTIATE(ARCH, ARCH_POSTFIX(S64_V128V128), + SIMD_TUPLE(v128_dotp_s16, 0U, 0U)); + +} // namespace SIMD_NAMESPACE diff --git a/third_party/aom/test/simd_neon_test.cc b/third_party/aom/test/simd_neon_test.cc new file mode 100644 index 0000000000..0565fb4e2a --- /dev/null +++ b/third_party/aom/test/simd_neon_test.cc @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if defined(__OPTIMIZE__) && __OPTIMIZE__ +#define ARCH NEON +#define ARCH_POSTFIX(name) name##_neon +#define SIMD_NAMESPACE simd_test_neon +#include "./simd_impl.h" +#endif diff --git a/third_party/aom/test/simd_sse2_test.cc b/third_party/aom/test/simd_sse2_test.cc new file mode 100644 index 0000000000..a0b49d77e6 --- /dev/null +++ b/third_party/aom/test/simd_sse2_test.cc @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if (defined(__OPTIMIZE__) && __OPTIMIZE__) || \ + (!defined(__GNUC__) && !defined(_DEBUG)) +#define ARCH SSE2 +#define ARCH_POSTFIX(name) name##_sse2 +#define SIMD_NAMESPACE simd_test_sse2 +#include "./simd_impl.h" +#endif diff --git a/third_party/aom/test/simd_sse4_test.cc b/third_party/aom/test/simd_sse4_test.cc new file mode 100644 index 0000000000..73c96427f2 --- /dev/null +++ b/third_party/aom/test/simd_sse4_test.cc @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if (defined(__OPTIMIZE__) && __OPTIMIZE__) || \ + (!defined(__GNUC__) && !defined(_DEBUG)) +#define ARCH SSE4_1 +#define ARCH_POSTFIX(name) name##_sse4_1 +#define SIMD_NAMESPACE simd_test_sse4_1 +#include "./simd_impl.h" +#endif diff --git a/third_party/aom/test/simd_ssse3_test.cc b/third_party/aom/test/simd_ssse3_test.cc new file mode 100644 index 0000000000..9ebeeef1b5 --- /dev/null +++ b/third_party/aom/test/simd_ssse3_test.cc @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#if (defined(__OPTIMIZE__) && __OPTIMIZE__) || \ + (!defined(__GNUC__) && !defined(_DEBUG)) +#define ARCH SSSE3 +#define ARCH_POSTFIX(name) name##_ssse3 +#define SIMD_NAMESPACE simd_test_ssse3 +#include "./simd_impl.h" +#endif diff --git a/third_party/aom/test/simple_decoder.sh b/third_party/aom/test/simple_decoder.sh new file mode 100755 index 0000000000..ac3a07b189 --- /dev/null +++ b/third_party/aom/test/simple_decoder.sh @@ -0,0 +1,58 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This file tests the libaom simple_decoder example code. To add new tests to +## this file, do the following: +## 1. Write a shell function (this is your test). +## 2. Add the function to simple_decoder_tests (on a new line). +## +. $(dirname $0)/tools_common.sh + +# Environment check: Make sure input is available: +simple_decoder_verify_environment() { + if [ ! "$(av1_encode_available)" = "yes" ] && [ ! -e "${AV1_IVF_FILE}" ]; then + return 1 + fi +} + +# Runs simple_decoder using $1 as input file. $2 is the codec name, and is used +# solely to name the output file. +simple_decoder() { + local decoder="${LIBAOM_BIN_PATH}/simple_decoder${AOM_TEST_EXE_SUFFIX}" + local input_file="$1" + local codec="$2" + local output_file="${AOM_TEST_OUTPUT_DIR}/simple_decoder_${codec}.raw" + + if [ ! -x "${decoder}" ]; then + elog "${decoder} does not exist or is not executable." + return 1 + fi + + eval "${AOM_TEST_PREFIX}" "${decoder}" "${input_file}" "${output_file}" \ + ${devnull} + + [ -e "${output_file}" ] || return 1 +} + +simple_decoder_av1() { + if [ "$(av1_decode_available)" = "yes" ]; then + if [ ! -e "${AV1_IVF_FILE}" ]; then + local file="${AOM_TEST_OUTPUT_DIR}/test_encode.ivf" + encode_yuv_raw_input_av1 "${file}" --ivf + simple_decoder "${file}" av1 || return 1 + else + simple_decoder "${AV1_IVF_FILE}" av1 || return 1 + fi + fi +} + +simple_decoder_tests="simple_decoder_av1" + +run_tests simple_decoder_verify_environment "${simple_decoder_tests}" diff --git a/third_party/aom/test/simple_encoder.sh b/third_party/aom/test/simple_encoder.sh new file mode 100755 index 0000000000..5cd6b46a10 --- /dev/null +++ b/third_party/aom/test/simple_encoder.sh @@ -0,0 +1,53 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This file tests the libaom simple_encoder example. To add new tests to this +## file, do the following: +## 1. Write a shell function (this is your test). +## 2. Add the function to simple_encoder_tests (on a new line). +## +. $(dirname $0)/tools_common.sh + +# Environment check: $YUV_RAW_INPUT is required. +simple_encoder_verify_environment() { + if [ ! -e "${YUV_RAW_INPUT}" ]; then + echo "Libaom test data must exist in LIBAOM_TEST_DATA_PATH." + return 1 + fi +} + +# Runs simple_encoder using the codec specified by $1 with a frame limit of 100. +simple_encoder() { + local encoder="${LIBAOM_BIN_PATH}/simple_encoder${AOM_TEST_EXE_SUFFIX}" + local codec="$1" + local output_file="${AOM_TEST_OUTPUT_DIR}/simple_encoder_${codec}.ivf" + + if [ ! -x "${encoder}" ]; then + elog "${encoder} does not exist or is not executable." + return 1 + fi + + eval "${AOM_TEST_PREFIX}" "${encoder}" "${codec}" "${YUV_RAW_INPUT_WIDTH}" \ + "${YUV_RAW_INPUT_HEIGHT}" "${YUV_RAW_INPUT}" "${output_file}" 9999 0 5 \ + ${devnull} + + [ -e "${output_file}" ] || return 1 +} + + +simple_encoder_av1() { + if [ "$(av1_encode_available)" = "yes" ]; then + simple_encoder av1 || return 1 + fi +} + +simple_encoder_tests="simple_encoder_av1" + +run_tests simple_encoder_verify_environment "${simple_encoder_tests}" diff --git a/third_party/aom/test/subtract_test.cc b/third_party/aom/test/subtract_test.cc new file mode 100644 index 0000000000..c90ca8d56c --- /dev/null +++ b/third_party/aom/test/subtract_test.cc @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#if CONFIG_AV1 +#include "av1/common/blockd.h" +#endif +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +#define USE_SPEED_TEST (0) + +typedef void (*SubtractFunc)(int rows, int cols, int16_t *diff_ptr, + ptrdiff_t diff_stride, const uint8_t *src_ptr, + ptrdiff_t src_stride, const uint8_t *pred_ptr, + ptrdiff_t pred_stride); + +namespace { + +class AV1SubtractBlockTest : public ::testing::TestWithParam { + public: + virtual void TearDown() { libaom_test::ClearSystemState(); } +}; + +using libaom_test::ACMRandom; + +TEST_P(AV1SubtractBlockTest, SimpleSubtract) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + + // FIXME(rbultje) split in its own file + for (BLOCK_SIZE bsize = BLOCK_4X4; bsize < BLOCK_SIZES; + bsize = static_cast(static_cast(bsize) + 1)) { + const int block_width = block_size_wide[bsize]; + const int block_height = block_size_high[bsize]; + int16_t *diff = reinterpret_cast( + aom_memalign(16, sizeof(*diff) * block_width * block_height * 2)); + uint8_t *pred = reinterpret_cast( + aom_memalign(16, block_width * block_height * 2)); + uint8_t *src = reinterpret_cast( + aom_memalign(16, block_width * block_height * 2)); + + for (int n = 0; n < 100; n++) { + for (int r = 0; r < block_height; ++r) { + for (int c = 0; c < block_width * 2; ++c) { + src[r * block_width * 2 + c] = rnd.Rand8(); + pred[r * block_width * 2 + c] = rnd.Rand8(); + } + } + + GetParam()(block_height, block_width, diff, block_width, src, block_width, + pred, block_width); + + for (int r = 0; r < block_height; ++r) { + for (int c = 0; c < block_width; ++c) { + EXPECT_EQ(diff[r * block_width + c], + (src[r * block_width + c] - pred[r * block_width + c])) + << "r = " << r << ", c = " << c << ", bs = " << bsize; + } + } + + GetParam()(block_height, block_width, diff, block_width * 2, src, + block_width * 2, pred, block_width * 2); + + for (int r = 0; r < block_height; ++r) { + for (int c = 0; c < block_width; ++c) { + EXPECT_EQ( + diff[r * block_width * 2 + c], + (src[r * block_width * 2 + c] - pred[r * block_width * 2 + c])) + << "r = " << r << ", c = " << c << ", bs = " << bsize; + } + } + } + aom_free(diff); + aom_free(pred); + aom_free(src); + } +} + +INSTANTIATE_TEST_CASE_P(C, AV1SubtractBlockTest, + ::testing::Values(aom_subtract_block_c)); + +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P(SSE2, AV1SubtractBlockTest, + ::testing::Values(aom_subtract_block_sse2)); +#endif +#if HAVE_NEON +INSTANTIATE_TEST_CASE_P(NEON, AV1SubtractBlockTest, + ::testing::Values(aom_subtract_block_neon)); +#endif +#if HAVE_MSA +INSTANTIATE_TEST_CASE_P(MSA, AV1SubtractBlockTest, + ::testing::Values(aom_subtract_block_msa)); +#endif + +typedef void (*HBDSubtractFunc)(int rows, int cols, int16_t *diff_ptr, + ptrdiff_t diff_stride, const uint8_t *src_ptr, + ptrdiff_t src_stride, const uint8_t *pred_ptr, + ptrdiff_t pred_stride, int bd); + +using ::std::tr1::get; +using ::std::tr1::make_tuple; +using ::std::tr1::tuple; + +// +typedef tuple Params; + +#if CONFIG_HIGHBITDEPTH +class AV1HBDSubtractBlockTest : public ::testing::TestWithParam { + public: + virtual void SetUp() { + block_width_ = GET_PARAM(0); + block_height_ = GET_PARAM(1); + bit_depth_ = static_cast(GET_PARAM(2)); + func_ = GET_PARAM(3); + + rnd_.Reset(ACMRandom::DeterministicSeed()); + + const size_t max_width = 128; + const size_t max_block_size = max_width * max_width; + src_ = CONVERT_TO_BYTEPTR(reinterpret_cast( + aom_memalign(16, max_block_size * sizeof(uint16_t)))); + pred_ = CONVERT_TO_BYTEPTR(reinterpret_cast( + aom_memalign(16, max_block_size * sizeof(uint16_t)))); + diff_ = reinterpret_cast( + aom_memalign(16, max_block_size * sizeof(int16_t))); + } + + virtual void TearDown() { + aom_free(CONVERT_TO_SHORTPTR(src_)); + aom_free(CONVERT_TO_SHORTPTR(pred_)); + aom_free(diff_); + } + + protected: + void RunForSpeed(); + void CheckResult(); + + private: + ACMRandom rnd_; + int block_height_; + int block_width_; + aom_bit_depth_t bit_depth_; + HBDSubtractFunc func_; + uint8_t *src_; + uint8_t *pred_; + int16_t *diff_; +}; + +void AV1HBDSubtractBlockTest::RunForSpeed() { + const int test_num = 200000; + const int max_width = 128; + const int max_block_size = max_width * max_width; + const int mask = (1 << bit_depth_) - 1; + int i, j; + + for (j = 0; j < max_block_size; ++j) { + CONVERT_TO_SHORTPTR(src_)[j] = rnd_.Rand16() & mask; + CONVERT_TO_SHORTPTR(pred_)[j] = rnd_.Rand16() & mask; + } + + for (i = 0; i < test_num; ++i) { + func_(block_height_, block_width_, diff_, block_width_, src_, block_width_, + pred_, block_width_, bit_depth_); + } +} + +void AV1HBDSubtractBlockTest::CheckResult() { + const int test_num = 100; + const int max_width = 128; + const int max_block_size = max_width * max_width; + const int mask = (1 << bit_depth_) - 1; + int i, j; + + for (i = 0; i < test_num; ++i) { + for (j = 0; j < max_block_size; ++j) { + CONVERT_TO_SHORTPTR(src_)[j] = rnd_.Rand16() & mask; + CONVERT_TO_SHORTPTR(pred_)[j] = rnd_.Rand16() & mask; + } + + func_(block_height_, block_width_, diff_, block_width_, src_, block_width_, + pred_, block_width_, bit_depth_); + + for (int r = 0; r < block_height_; ++r) { + for (int c = 0; c < block_width_; ++c) { + EXPECT_EQ(diff_[r * block_width_ + c], + (CONVERT_TO_SHORTPTR(src_)[r * block_width_ + c] - + CONVERT_TO_SHORTPTR(pred_)[r * block_width_ + c])) + << "r = " << r << ", c = " << c << ", test: " << i; + } + } + } +} + +TEST_P(AV1HBDSubtractBlockTest, CheckResult) { CheckResult(); } + +#if USE_SPEED_TEST +TEST_P(AV1HBDSubtractBlockTest, CheckSpeed) { RunForSpeed(); } +#endif // USE_SPEED_TEST + +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P( + SSE2, AV1HBDSubtractBlockTest, + ::testing::Values(make_tuple(4, 4, 12, &aom_highbd_subtract_block_sse2), + make_tuple(4, 4, 12, &aom_highbd_subtract_block_c), + make_tuple(4, 8, 12, &aom_highbd_subtract_block_sse2), + make_tuple(4, 8, 12, &aom_highbd_subtract_block_c), + make_tuple(8, 4, 12, &aom_highbd_subtract_block_sse2), + make_tuple(8, 4, 12, &aom_highbd_subtract_block_c), + make_tuple(8, 8, 12, &aom_highbd_subtract_block_sse2), + make_tuple(8, 8, 12, &aom_highbd_subtract_block_c), + make_tuple(8, 16, 12, &aom_highbd_subtract_block_sse2), + make_tuple(8, 16, 12, &aom_highbd_subtract_block_c), + make_tuple(16, 8, 12, &aom_highbd_subtract_block_sse2), + make_tuple(16, 8, 12, &aom_highbd_subtract_block_c), + make_tuple(16, 16, 12, &aom_highbd_subtract_block_sse2), + make_tuple(16, 16, 12, &aom_highbd_subtract_block_c), + make_tuple(16, 32, 12, &aom_highbd_subtract_block_sse2), + make_tuple(16, 32, 12, &aom_highbd_subtract_block_c), + make_tuple(32, 16, 12, &aom_highbd_subtract_block_sse2), + make_tuple(32, 16, 12, &aom_highbd_subtract_block_c), + make_tuple(32, 32, 12, &aom_highbd_subtract_block_sse2), + make_tuple(32, 32, 12, &aom_highbd_subtract_block_c), + make_tuple(32, 64, 12, &aom_highbd_subtract_block_sse2), + make_tuple(32, 64, 12, &aom_highbd_subtract_block_c), + make_tuple(64, 32, 12, &aom_highbd_subtract_block_sse2), + make_tuple(64, 32, 12, &aom_highbd_subtract_block_c), + make_tuple(64, 64, 12, &aom_highbd_subtract_block_sse2), + make_tuple(64, 64, 12, &aom_highbd_subtract_block_c), + make_tuple(64, 128, 12, &aom_highbd_subtract_block_sse2), + make_tuple(64, 128, 12, &aom_highbd_subtract_block_c), + make_tuple(128, 64, 12, &aom_highbd_subtract_block_sse2), + make_tuple(128, 64, 12, &aom_highbd_subtract_block_c), + make_tuple(128, 128, 12, &aom_highbd_subtract_block_sse2), + make_tuple(128, 128, 12, &aom_highbd_subtract_block_c))); +#endif // HAVE_SSE2 +#endif // CONFIG_HIGHBITDEPTH +} // namespace diff --git a/third_party/aom/test/sum_squares_test.cc b/third_party/aom/test/sum_squares_test.cc new file mode 100644 index 0000000000..b8701c1964 --- /dev/null +++ b/third_party/aom/test/sum_squares_test.cc @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "aom_ports/mem.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" +#include "test/function_equivalence_test.h" + +using libaom_test::ACMRandom; +using libaom_test::FunctionEquivalenceTest; + +namespace { +const int kNumIterations = 10000; + +static const int16_t kInt13Max = (1 << 12) - 1; + +typedef uint64_t (*SSI16Func)(const int16_t *src, int stride, int width, + int height); +typedef libaom_test::FuncParam TestFuncs; + +class SumSquaresTest : public ::testing::TestWithParam { + public: + virtual ~SumSquaresTest() {} + virtual void SetUp() { params_ = this->GetParam(); } + + virtual void TearDown() { libaom_test::ClearSystemState(); } + + protected: + TestFuncs params_; +}; + +TEST_P(SumSquaresTest, OperationCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, int16_t, src[256 * 256]); + + int failed = 0; + + const int msb = 11; // Up to 12 bit input + const int limit = 1 << (msb + 1); + + for (int k = 0; k < kNumIterations; k++) { + int width = 4 * rnd(32); // Up to 128x128 + int height = 4 * rnd(32); // Up to 128x128 + int stride = 4 << rnd(7); // Up to 256 stride + while (stride < width) { // Make sure it's valid + stride = 4 << rnd(7); + } + + for (int ii = 0; ii < height; ii++) { + for (int jj = 0; jj < width; jj++) { + src[ii * stride + jj] = rnd(2) ? rnd(limit) : -rnd(limit); + } + } + + const uint64_t res_ref = params_.ref_func(src, stride, width, height); + uint64_t res_tst; + ASM_REGISTER_STATE_CHECK(res_tst = + params_.tst_func(src, stride, width, height)); + + if (!failed) { + failed = res_ref != res_tst; + EXPECT_EQ(res_ref, res_tst) + << "Error: Sum Squares Test" + << " C output does not match optimized output."; + } + } +} + +TEST_P(SumSquaresTest, ExtremeValues) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + DECLARE_ALIGNED(16, int16_t, src[256 * 256]); + + int failed = 0; + + const int msb = 11; // Up to 12 bit input + const int limit = 1 << (msb + 1); + + for (int k = 0; k < kNumIterations; k++) { + int width = 4 * rnd(32); // Up to 128x128 + int height = 4 * rnd(32); // Up to 128x128 + int stride = 4 << rnd(7); // Up to 256 stride + while (stride < width) { // Make sure it's valid + stride = 4 << rnd(7); + } + + int val = rnd(2) ? limit - 1 : -(limit - 1); + for (int ii = 0; ii < height; ii++) { + for (int jj = 0; jj < width; jj++) { + src[ii * stride + jj] = val; + } + } + + const uint64_t res_ref = params_.ref_func(src, stride, width, height); + uint64_t res_tst; + ASM_REGISTER_STATE_CHECK(res_tst = + params_.tst_func(src, stride, width, height)); + + if (!failed) { + failed = res_ref != res_tst; + EXPECT_EQ(res_ref, res_tst) + << "Error: Sum Squares Test" + << " C output does not match optimized output."; + } + } +} + +#if HAVE_SSE2 + +INSTANTIATE_TEST_CASE_P( + SSE2, SumSquaresTest, + ::testing::Values(TestFuncs(&aom_sum_squares_2d_i16_c, + &aom_sum_squares_2d_i16_sse2))); + +#endif // HAVE_SSE2 + +////////////////////////////////////////////////////////////////////////////// +// 1D version +////////////////////////////////////////////////////////////////////////////// + +typedef uint64_t (*F1D)(const int16_t *src, uint32_t N); +typedef libaom_test::FuncParam TestFuncs1D; + +class SumSquares1DTest : public FunctionEquivalenceTest { + protected: + static const int kIterations = 1000; + static const int kMaxSize = 256; +}; + +TEST_P(SumSquares1DTest, RandomValues) { + DECLARE_ALIGNED(16, int16_t, src[kMaxSize * kMaxSize]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + for (int i = 0; i < kMaxSize * kMaxSize; ++i) + src[i] = rng_(kInt13Max * 2 + 1) - kInt13Max; + + const int N = rng_(2) ? rng_(kMaxSize * kMaxSize + 1 - kMaxSize) + kMaxSize + : rng_(kMaxSize) + 1; + + const uint64_t ref_res = params_.ref_func(src, N); + uint64_t tst_res; + ASM_REGISTER_STATE_CHECK(tst_res = params_.tst_func(src, N)); + + ASSERT_EQ(ref_res, tst_res); + } +} + +TEST_P(SumSquares1DTest, ExtremeValues) { + DECLARE_ALIGNED(16, int16_t, src[kMaxSize * kMaxSize]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + if (rng_(2)) { + for (int i = 0; i < kMaxSize * kMaxSize; ++i) src[i] = kInt13Max; + } else { + for (int i = 0; i < kMaxSize * kMaxSize; ++i) src[i] = -kInt13Max; + } + + const int N = rng_(2) ? rng_(kMaxSize * kMaxSize + 1 - kMaxSize) + kMaxSize + : rng_(kMaxSize) + 1; + + const uint64_t ref_res = params_.ref_func(src, N); + uint64_t tst_res; + ASM_REGISTER_STATE_CHECK(tst_res = params_.tst_func(src, N)); + + ASSERT_EQ(ref_res, tst_res); + } +} + +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P(SSE2, SumSquares1DTest, + ::testing::Values(TestFuncs1D( + aom_sum_squares_i16_c, aom_sum_squares_i16_sse2))); + +#endif // HAVE_SSE2 +} // namespace diff --git a/third_party/aom/test/superframe_test.cc b/third_party/aom/test/superframe_test.cc new file mode 100644 index 0000000000..0f54baeaf4 --- /dev/null +++ b/third_party/aom/test/superframe_test.cc @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" + +namespace { + +const int kTestMode = 0; +const int kTileCols = 1; +const int kTileRows = 2; + +typedef std::tr1::tuple SuperframeTestParam; + +class SuperframeTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWithParam { + protected: + SuperframeTest() + : EncoderTest(GET_PARAM(0)), modified_buf_(NULL), last_sf_pts_(0) {} + virtual ~SuperframeTest() {} + + virtual void SetUp() { + InitializeConfig(); + const SuperframeTestParam input = GET_PARAM(1); + const libaom_test::TestMode mode = std::tr1::get(input); + SetMode(mode); + sf_count_ = 0; + sf_count_max_ = INT_MAX; + n_tile_cols_ = std::tr1::get(input); + n_tile_rows_ = std::tr1::get(input); + } + + virtual void TearDown() { delete[] modified_buf_; } + + virtual void PreEncodeFrameHook(libaom_test::VideoSource *video, + libaom_test::Encoder *encoder) { + if (video->frame() == 1) { + encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); + encoder->Control(AOME_SET_CPUUSED, 2); + encoder->Control(AV1E_SET_TILE_COLUMNS, n_tile_cols_); + encoder->Control(AV1E_SET_TILE_ROWS, n_tile_rows_); +#if CONFIG_LOOPFILTERING_ACROSS_TILES + encoder->Control(AV1E_SET_TILE_LOOPFILTER, 0); +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + } + } + + virtual const aom_codec_cx_pkt_t *MutateEncoderOutputHook( + const aom_codec_cx_pkt_t *pkt) { + if (pkt->kind != AOM_CODEC_CX_FRAME_PKT) return pkt; + + const uint8_t *buffer = reinterpret_cast(pkt->data.frame.buf); + const uint8_t marker = buffer[pkt->data.frame.sz - 1]; + const int frames = (marker & 0x7) + 1; + const int mag = ((marker >> 3) & 3) + 1; + const unsigned int index_sz = 2 + mag * (frames - 1); + if ((marker & 0xe0) == 0xc0 && pkt->data.frame.sz >= index_sz && + buffer[pkt->data.frame.sz - index_sz] == marker) { + // frame is a superframe. strip off the index. + if (modified_buf_) delete[] modified_buf_; + modified_buf_ = new uint8_t[pkt->data.frame.sz - index_sz]; + memcpy(modified_buf_, pkt->data.frame.buf, pkt->data.frame.sz - index_sz); + modified_pkt_ = *pkt; + modified_pkt_.data.frame.buf = modified_buf_; + modified_pkt_.data.frame.sz -= index_sz; + + sf_count_++; + last_sf_pts_ = pkt->data.frame.pts; + return &modified_pkt_; + } + + // Make sure we do a few frames after the last SF + abort_ |= + sf_count_ > sf_count_max_ && pkt->data.frame.pts - last_sf_pts_ >= 5; + return pkt; + } + + int sf_count_; + int sf_count_max_; + aom_codec_cx_pkt_t modified_pkt_; + uint8_t *modified_buf_; + aom_codec_pts_t last_sf_pts_; + + private: + int n_tile_cols_; + int n_tile_rows_; +}; + +TEST_P(SuperframeTest, TestSuperframeIndexIsOptional) { + sf_count_max_ = 0; // early exit on successful test. + cfg_.g_lag_in_frames = 25; + + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 40); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +#if CONFIG_EXT_REFS + // NOTE: The use of BWDREF_FRAME will enable the coding of more non-show + // frames besides ALTREF_FRAME. + EXPECT_GE(sf_count_, 1); +#else + EXPECT_EQ(sf_count_, 1); +#endif // CONFIG_EXT_REFS +} + +// The superframe index is currently mandatory with both ANS and DAALA_EC due +// to the decoder starting at the end of the buffer. +#if CONFIG_EXT_TILE +// Single tile does not work with ANS (see comment above). +#if CONFIG_ANS || CONFIG_DAALA_EC +const int tile_col_values[] = { 1, 2 }; +#else +const int tile_col_values[] = { 1, 2, 32 }; +#endif +const int tile_row_values[] = { 1, 2, 32 }; +AV1_INSTANTIATE_TEST_CASE( + SuperframeTest, + ::testing::Combine(::testing::Values(::libaom_test::kTwoPassGood), + ::testing::ValuesIn(tile_col_values), + ::testing::ValuesIn(tile_row_values))); +#else +#if !CONFIG_ANS && !CONFIG_DAALA_EC +AV1_INSTANTIATE_TEST_CASE( + SuperframeTest, + ::testing::Combine(::testing::Values(::libaom_test::kTwoPassGood), + ::testing::Values(0), ::testing::Values(0))); +#endif // !CONFIG_ANS +#endif // CONFIG_EXT_TILE +} // namespace diff --git a/third_party/aom/test/test-data.mk b/third_party/aom/test/test-data.mk new file mode 100644 index 0000000000..168144a00f --- /dev/null +++ b/third_party/aom/test/test-data.mk @@ -0,0 +1,45 @@ +LIBAOM_TEST_SRCS-yes += test-data.mk + +# Encoder test source +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += hantro_collage_w352h288.yuv +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += hantro_odd.yuv + +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_10_420.y4m +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_10_422.y4m +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_10_444.y4m +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_10_440.yuv +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_12_420.y4m +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_12_422.y4m +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_12_444.y4m +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_12_440.yuv +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_8_420_a10-1.y4m +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_8_420.y4m +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_8_422.y4m +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_8_444.y4m +LIBAOM_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_8_440.yuv + +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += desktop_credits.y4m +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += niklas_1280_720_30.y4m +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += rush_hour_444.y4m +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += screendata.y4m +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += niklas_640_480_30.yuv + +ifeq ($(CONFIG_DECODE_PERF_TESTS),yes) +# Encode / Decode test +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += niklas_1280_720_30.yuv +endif # CONFIG_DECODE_PERF_TESTS + +ifeq ($(CONFIG_ENCODE_PERF_TESTS),yes) +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += desktop_640_360_30.yuv +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += kirland_640_480_30.yuv +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += macmarcomoving_640_480_30.yuv +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += macmarcostationary_640_480_30.yuv +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += niklas_1280_720_30.yuv +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += tacomanarrows_640_480_30.yuv +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += tacomasmallcameramovement_640_480_30.yuv +LIBAOM_TEST_DATA-$(CONFIG_AV1_ENCODER) += thaloundeskmtg_640_480_30.yuv +endif # CONFIG_ENCODE_PERF_TESTS + +# sort and remove duplicates +LIBAOM_TEST_DATA-yes := $(sort $(LIBAOM_TEST_DATA-yes)) + diff --git a/third_party/aom/test/test-data.sha1 b/third_party/aom/test/test-data.sha1 new file mode 100644 index 0000000000..3d9bfc7c4a --- /dev/null +++ b/third_party/aom/test/test-data.sha1 @@ -0,0 +1,28 @@ +d5dfb0151c9051f8c85999255645d7a23916d3c0 *hantro_collage_w352h288.yuv +b87815bf86020c592ccc7a846ba2e28ec8043902 *hantro_odd.yuv +a432f96ff0a787268e2f94a8092ab161a18d1b06 *park_joy_90p_10_420.y4m +0b194cc312c3a2e84d156a221b0a5eb615dfddc5 *park_joy_90p_10_422.y4m +ff0e0a21dc2adc95b8c1b37902713700655ced17 *park_joy_90p_10_444.y4m +c934da6fb8cc54ee2a8c17c54cf6076dac37ead0 *park_joy_90p_10_440.yuv +614c32ae1eca391e867c70d19974f0d62664dd99 *park_joy_90p_12_420.y4m +c92825f1ea25c5c37855083a69faac6ac4641a9e *park_joy_90p_12_422.y4m +b592189b885b6cc85db55cc98512a197d73d3b34 *park_joy_90p_12_444.y4m +82c1bfcca368c2f22bad7d693d690d5499ecdd11 *park_joy_90p_12_440.yuv +b9e1e90aece2be6e2c90d89e6ab2372d5f8c792d *park_joy_90p_8_420_a10-1.y4m +4e0eb61e76f0684188d9bc9f3ce61f6b6b77bb2c *park_joy_90p_8_420.y4m +7a193ff7dfeb96ba5f82b2afd7afa9e1fe83d947 *park_joy_90p_8_422.y4m +bdb7856e6bc93599bdda05c2e773a9f22b6c6d03 *park_joy_90p_8_444.y4m +81e1f3843748438b8f2e71db484eb22daf72e939 *park_joy_90p_8_440.yuv +b1f1c3ec79114b9a0651af24ce634afb44a9a419 *rush_hour_444.y4m +eb438c6540eb429f74404eedfa3228d409c57874 *desktop_640_360_30.yuv +89e70ebd22c27d275fe14dc2f1a41841a6d8b9ab *kirland_640_480_30.yuv +33c533192759e5bb4f07abfbac389dc259db4686 *macmarcomoving_640_480_30.yuv +8bfaab121080821b8f03b23467911e59ec59b8fe *macmarcostationary_640_480_30.yuv +70894878d916a599842d9ad0dcd24e10c13e5467 *niklas_640_480_30.yuv +8784b6df2d8cc946195a90ac00540500d2e522e4 *tacomanarrows_640_480_30.yuv +edd86a1f5e62fd9da9a9d46078247759c2638009 *tacomasmallcameramovement_640_480_30.yuv +9a70e8b7d14fba9234d0e51dce876635413ce444 *thaloundeskmtg_640_480_30.yuv +e7d315dbf4f3928779e0dc624311196d44491d32 *niklas_1280_720_30.yuv +717da707afcaa1f692ff1946f291054eb75a4f06 *screendata.y4m +9cfc855459e7549fd015c79e8eca512b2f2cb7e3 *niklas_1280_720_30.y4m +5b5763b388b1b52a81bb82b39f7ec25c4bd3d0e1 *desktop_credits.y4m diff --git a/third_party/aom/test/test.cmake b/third_party/aom/test/test.cmake new file mode 100644 index 0000000000..8d3ab7059c --- /dev/null +++ b/third_party/aom/test/test.cmake @@ -0,0 +1,315 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +include("${AOM_ROOT}/test/test_data_util.cmake") + +set(AOM_UNIT_TEST_WRAPPER_SOURCES + "${AOM_CONFIG_DIR}/usage_exit.c" + "${AOM_ROOT}/test/test_libaom.cc") + +set(AOM_UNIT_TEST_COMMON_SOURCES + "${AOM_ROOT}/test/acm_random.h" + "${AOM_ROOT}/test/clear_system_state.h" + "${AOM_ROOT}/test/codec_factory.h" + "${AOM_ROOT}/test/convolve_test.cc" + "${AOM_ROOT}/test/function_equivalence_test.h" + "${AOM_ROOT}/test/md5_helper.h" + "${AOM_ROOT}/test/register_state_check.h" + "${AOM_ROOT}/test/transform_test_base.h" + "${AOM_ROOT}/test/util.h" + "${AOM_ROOT}/test/video_source.h") + +if (CONFIG_ACCOUNTING) + set(AOM_UNIT_TEST_COMMON_SOURCES + ${AOM_UNIT_TEST_COMMON_SOURCES} + "${AOM_ROOT}/test/accounting_test.cc") +endif () + +if (CONFIG_ADAPT_SCAN) + set(AOM_UNIT_TEST_COMMON_SOURCES + ${AOM_UNIT_TEST_COMMON_SOURCES} + "${AOM_ROOT}/test/scan_test.cc") +endif () + +if (CONFIG_GLOBAL_MOTION OR CONFIG_WARPED_MOTION) + if (HAVE_SSE2) + set(AOM_UNIT_TEST_COMMON_SOURCES + ${AOM_UNIT_TEST_COMMON_SOURCES} + "${AOM_ROOT}/test/warp_filter_test.cc" + "${AOM_ROOT}/test/warp_filter_test_util.cc" + "${AOM_ROOT}/test/warp_filter_test_util.h") + endif () +endif () + +set(AOM_UNIT_TEST_DECODER_SOURCES + "${AOM_ROOT}/test/decode_api_test.cc" + "${AOM_ROOT}/test/decode_test_driver.cc" + "${AOM_ROOT}/test/decode_test_driver.h" + "${AOM_ROOT}/test/ivf_video_source.h") + +set(AOM_UNIT_TEST_ENCODER_SOURCES + "${AOM_ROOT}/test/altref_test.cc" + "${AOM_ROOT}/test/aq_segment_test.cc" + "${AOM_ROOT}/test/datarate_test.cc" + "${AOM_ROOT}/test/dct16x16_test.cc" + "${AOM_ROOT}/test/dct32x32_test.cc" + "${AOM_ROOT}/test/encode_api_test.cc" + "${AOM_ROOT}/test/encode_test_driver.cc" + "${AOM_ROOT}/test/encode_test_driver.h" + "${AOM_ROOT}/test/error_resilience_test.cc" + "${AOM_ROOT}/test/i420_video_source.h" + "${AOM_ROOT}/test/sad_test.cc" + "${AOM_ROOT}/test/y4m_test.cc" + "${AOM_ROOT}/test/y4m_video_source.h" + "${AOM_ROOT}/test/yuv_video_source.h") + +set(AOM_DECODE_PERF_TEST_SOURCES "${AOM_ROOT}/test/decode_perf_test.cc") +set(AOM_ENCODE_PERF_TEST_SOURCES "${AOM_ROOT}/test/encode_perf_test.cc") +set(AOM_UNIT_TEST_WEBM_SOURCES "${AOM_ROOT}/test/webm_video_source.h") + +set(AOM_TEST_INTRA_PRED_SPEED_SOURCES + "${AOM_CONFIG_DIR}/usage_exit.c" + "${AOM_ROOT}/test/test_intra_pred_speed.cc") + +if (CONFIG_AV1) + set(AOM_UNIT_TEST_COMMON_SOURCES + ${AOM_UNIT_TEST_COMMON_SOURCES} + "${AOM_ROOT}/test/av1_convolve_optimz_test.cc" + "${AOM_ROOT}/test/av1_convolve_test.cc" + "${AOM_ROOT}/test/av1_fwd_txfm1d_test.cc" + "${AOM_ROOT}/test/av1_fwd_txfm2d_test.cc" + "${AOM_ROOT}/test/av1_inv_txfm1d_test.cc" + "${AOM_ROOT}/test/av1_inv_txfm2d_test.cc" + "${AOM_ROOT}/test/av1_txfm_test.cc" + "${AOM_ROOT}/test/av1_txfm_test.h" + "${AOM_ROOT}/test/intrapred_test.cc" + "${AOM_ROOT}/test/lpf_8_test.cc" + "${AOM_ROOT}/test/simd_cmp_impl.h") + + if (CONFIG_CDEF) + set(AOM_UNIT_TEST_COMMON_SOURCES + ${AOM_UNIT_TEST_COMMON_SOURCES} + "${AOM_ROOT}/test/clpf_test.cc") + endif () + + if (CONFIG_FILTER_INTRA) + if (HAVE_SSE4_1) + set(AOM_UNIT_TEST_COMMON_SOURCES + ${AOM_UNIT_TEST_COMMON_SOURCES} + # TODO: not sure if this intrinsics or a wrapper calling intrin/asm. + #"${AOM_ROOT}/test/filterintra_predictors_test.cc") + ) + endif () + endif () + + set(AOM_UNIT_TEST_COMMON_INTRIN_NEON + ${AOM_UNIT_TEST_COMMON_INTRIN_NEON} + "${AOM_ROOT}/test/simd_cmp_neon.cc" + "${AOM_ROOT}/test/simd_neon_test.cc") + set(AOM_UNIT_TEST_COMMON_INTRIN_SSE2 + ${AOM_UNIT_TEST_COMMON_INTRIN_SSE2} + "${AOM_ROOT}/test/simd_cmp_sse2.cc") + set(AOM_UNIT_TEST_COMMON_INTRIN_SSSE3 + ${AOM_UNIT_TEST_COMMON_INTRIN_SSSE3} + "${AOM_ROOT}/test/simd_cmp_ssse3.cc") + set(AOM_UNIT_TEST_COMMON_INTRIN_SSE4_1 + ${AOM_UNIT_TEST_COMMON_INTRIN_SSE4_1} + "${AOM_ROOT}/test/simd_cmp_sse4.cc") +endif () + +if (CONFIG_AV1_ENCODER) + set(AOM_UNIT_TEST_ENCODER_SOURCES + ${AOM_UNIT_TEST_ENCODER_SOURCES} + "${AOM_ROOT}/test/active_map_test.cc" + "${AOM_ROOT}/test/arf_freq_test.cc" + "${AOM_ROOT}/test/av1_dct_test.cc" + "${AOM_ROOT}/test/av1_fht16x16_test.cc" + "${AOM_ROOT}/test/av1_fht8x8_test.cc" + "${AOM_ROOT}/test/av1_inv_txfm_test.cc" + "${AOM_ROOT}/test/avg_test.cc" + "${AOM_ROOT}/test/blend_a64_mask_1d_test.cc" + "${AOM_ROOT}/test/blend_a64_mask_test.cc" + "${AOM_ROOT}/test/borders_test.cc" + "${AOM_ROOT}/test/cpu_speed_test.cc" + "${AOM_ROOT}/test/end_to_end_test.cc" + "${AOM_ROOT}/test/error_block_test.cc" + "${AOM_ROOT}/test/fdct4x4_test.cc" + "${AOM_ROOT}/test/fdct8x8_test.cc" + "${AOM_ROOT}/test/frame_size_tests.cc" + "${AOM_ROOT}/test/hadamard_test.cc" + "${AOM_ROOT}/test/lossless_test.cc" + "${AOM_ROOT}/test/minmax_test.cc" + "${AOM_ROOT}/test/subtract_test.cc" + "${AOM_ROOT}/test/sum_squares_test.cc" + "${AOM_ROOT}/test/variance_test.cc") + + if (CONFIG_EXT_INTER) + set(AOM_UNIT_TEST_ENCODER_SOURCES + ${AOM_UNIT_TEST_ENCODER_SOURCES} + "${AOM_ROOT}/test/av1_wedge_utils_test.cc" + "${AOM_ROOT}/test/masked_sad_test.cc" + "${AOM_ROOT}/test/masked_variance_test.cc") + endif () + + if (CONFIG_EXT_TX) + set(AOM_UNIT_TEST_ENCODER_SOURCES + ${AOM_UNIT_TEST_ENCODER_SOURCES} + "${AOM_ROOT}/test/av1_fht16x32_test.cc" + "${AOM_ROOT}/test/av1_fht16x8_test.cc" + "${AOM_ROOT}/test/av1_fht32x16_test.cc" + "${AOM_ROOT}/test/av1_fht4x4_test.cc" + "${AOM_ROOT}/test/av1_fht4x8_test.cc" + "${AOM_ROOT}/test/av1_fht8x16_test.cc" + "${AOM_ROOT}/test/av1_fht8x4_test.cc" + "${AOM_ROOT}/test/fht32x32_test.cc") + endif () + + if (CONFIG_MOTION_VAR) + set(AOM_UNIT_TEST_ENCODER_SOURCES + ${AOM_UNIT_TEST_ENCODER_SOURCES} + "${AOM_ROOT}/test/obmc_sad_test.cc" + "${AOM_ROOT}/test/obmc_variance_test.cc") + endif () +endif () + +if (CONFIG_AV1_DECODER AND CONFIG_AV1_ENCODER) + set(AOM_UNIT_TEST_COMMON_SOURCES + ${AOM_UNIT_TEST_COMMON_SOURCES} + "${AOM_ROOT}/test/divu_small_test.cc" + "${AOM_ROOT}/test/ethread_test.cc" + "${AOM_ROOT}/test/idct8x8_test.cc" + "${AOM_ROOT}/test/partial_idct_test.cc" + "${AOM_ROOT}/test/superframe_test.cc" + "${AOM_ROOT}/test/binary_codes_test.cc" + "${AOM_ROOT}/test/tile_independence_test.cc") + + if (CONFIG_ANS) + set(AOM_UNIT_TEST_COMMON_SOURCES + ${AOM_UNIT_TEST_COMMON_SOURCES} + "${AOM_ROOT}/test/ans_codec_test.cc" + "${AOM_ROOT}/test/ans_test.cc") + else () + set(AOM_UNIT_TEST_COMMON_SOURCES + ${AOM_UNIT_TEST_COMMON_SOURCES} + "${AOM_ROOT}/test/boolcoder_test.cc") + endif () + + if (CONFIG_EXT_TILE) + set(AOM_UNIT_TEST_COMMON_SOURCES + ${AOM_UNIT_TEST_COMMON_SOURCES} + "${AOM_ROOT}/test/av1_ext_tile_test.cc") + endif () +endif () + +if (CONFIG_HIGHBITDEPTH) + if (CONFIG_AV1) + set(AOM_UNIT_TEST_COMMON_INTRIN_SSE4_1 + ${AOM_UNIT_TEST_COMMON_INTRIN_SSE4_1} + "${AOM_ROOT}/test/av1_highbd_iht_test.cc" + "${AOM_ROOT}/test/av1_quantize_test.cc") + endif () + + if (CONFIG_INTERNAL_STATS) + set(AOM_UNIT_TEST_COMMON_SOURCES + ${AOM_UNIT_TEST_COMMON_SOURCES} + "${AOM_ROOT}/test/hbd_metrics_test.cc") + endif () +endif () + +if (CONFIG_UNIT_TESTS) + if (MSVC) + # Force static run time to avoid collisions with googletest. + include("${AOM_ROOT}/build/cmake/msvc_runtime.cmake") + endif () + include_directories( + "${AOM_ROOT}/third_party/googletest/src/googletest/src" + "${AOM_ROOT}/third_party/googletest/src/googletest/include") + add_subdirectory("${AOM_ROOT}/third_party/googletest/src/googletest" + EXCLUDE_FROM_ALL) + + # Generate a stub file containing the C function usage_exit(); this is + # required because of the test dependency on aom_common_app_util. + # Specifically, the function die() in tools_common.c calls usage_exit() to + # terminate the program on the caller's behalf. + file(WRITE "${AOM_CONFIG_DIR}/usage_exit.c" "void usage_exit(void) {}") +endif () + +# Setup the targets for CONFIG_UNIT_TESTS. The libaom and app util targets must +# exist before this function is called. +function (setup_aom_test_targets) + add_library(test_aom_common OBJECT ${AOM_UNIT_TEST_COMMON_SOURCES}) + add_library(test_aom_decoder OBJECT ${AOM_UNIT_TEST_DECODER_SOURCES}) + add_library(test_aom_encoder OBJECT ${AOM_UNIT_TEST_ENCODER_SOURCES}) + + set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} test_aom_common test_aom_decoder + test_aom_encoder PARENT_SCOPE) + + add_executable(test_libaom ${AOM_UNIT_TEST_WRAPPER_SOURCES} + $ + $) + + if (CONFIG_DECODERS) + target_sources(test_libaom PUBLIC + $ + $) + + if (CONFIG_DECODE_PERF_TESTS AND CONFIG_WEBM_IO) + target_sources(test_libaom PUBLIC ${AOM_DECODE_PERF_TEST_SOURCES}) + endif () + endif () + + if (CONFIG_ENCODERS) + target_sources(test_libaom PUBLIC + $ + $) + + if (CONFIG_ENCODE_PERF_TESTS) + target_sources(test_libaom PUBLIC ${AOM_ENCODE_PERF_TEST_SOURCES}) + endif () + endif () + + target_link_libraries(test_libaom PUBLIC aom gtest) + + add_executable(test_intra_pred_speed + ${AOM_TEST_INTRA_PRED_SPEED_SOURCES} + $) + target_link_libraries(test_intra_pred_speed PUBLIC aom gtest) + + if (CONFIG_LIBYUV) + target_sources(test_libaom PUBLIC $) + endif () + if (CONFIG_WEBM_IO) + target_sources(test_libaom PUBLIC ${AOM_UNIT_TEST_WEBM_SOURCES} + $) + endif () + if (HAVE_SSE2) + add_intrinsics_source_to_target("-msse2" "test_libaom" + "AOM_UNIT_TEST_COMMON_INTRIN_SSE2") + endif () + if (HAVE_SSSE3) + add_intrinsics_source_to_target("-mssse3" "test_libaom" + "AOM_UNIT_TEST_COMMON_INTRIN_SSSE3") + endif () + if (HAVE_SSE4_1) + add_intrinsics_source_to_target("-msse4.1" "test_libaom" + "AOM_UNIT_TEST_COMMON_INTRIN_SSE4_1") + endif () + if (HAVE_NEON) + add_intrinsics_source_to_target("${AOM_NEON_INTRIN_FLAG}" "test_libaom" + "AOM_UNIT_TEST_COMMON_INTRIN_NEON") + endif () + + add_custom_target(testdata + COMMAND ${CMAKE_COMMAND} + -DAOM_CONFIG_DIR="${AOM_CONFIG_DIR}" + -DAOM_ROOT="${AOM_ROOT}" + -P "${AOM_ROOT}/test/test_worker.cmake" + SOURCES ${AOM_TEST_DATA_LIST}) +endfunction () diff --git a/third_party/aom/test/test.mk b/third_party/aom/test/test.mk new file mode 100644 index 0000000000..fb0ab371e3 --- /dev/null +++ b/third_party/aom/test/test.mk @@ -0,0 +1,241 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +LIBAOM_TEST_SRCS-yes += acm_random.h +LIBAOM_TEST_SRCS-yes += clear_system_state.h +LIBAOM_TEST_SRCS-yes += codec_factory.h +LIBAOM_TEST_SRCS-yes += md5_helper.h +LIBAOM_TEST_SRCS-yes += register_state_check.h +LIBAOM_TEST_SRCS-yes += test.mk +LIBAOM_TEST_SRCS-yes += test_libaom.cc +LIBAOM_TEST_SRCS-yes += util.h +LIBAOM_TEST_SRCS-yes += video_source.h +LIBAOM_TEST_SRCS-yes += transform_test_base.h +LIBAOM_TEST_SRCS-yes += function_equivalence_test.h +LIBAOM_TEST_SRCS-yes += warp_filter_test_util.h + +## +## BLACK BOX TESTS +## +## Black box tests only use the public API. +## +LIBAOM_TEST_SRCS-yes += ../md5_utils.h ../md5_utils.c +LIBAOM_TEST_SRCS-$(CONFIG_DECODERS) += ivf_video_source.h +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += ../y4minput.h ../y4minput.c +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += altref_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += aq_segment_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += datarate_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += encode_api_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += error_resilience_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += i420_video_source.h +#LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += realtime_test.cc +#LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += resize_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += y4m_video_source.h +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += yuv_video_source.h + +#LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += level_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += active_map_refresh_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += active_map_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += borders_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += cpu_speed_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += frame_size_tests.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += lossless_test.cc + +LIBAOM_TEST_SRCS-yes += decode_test_driver.cc +LIBAOM_TEST_SRCS-yes += decode_test_driver.h +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += encode_test_driver.cc +LIBAOM_TEST_SRCS-yes += encode_test_driver.h + +## IVF writing. +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += ../ivfenc.c ../ivfenc.h + +## Y4m parsing. +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += y4m_test.cc ../y4menc.c ../y4menc.h + +## WebM Parsing +ifeq ($(CONFIG_WEBM_IO), yes) +LIBWEBM_PARSER_SRCS += ../third_party/libwebm/mkvparser/mkvparser.cc +LIBWEBM_PARSER_SRCS += ../third_party/libwebm/mkvparser/mkvreader.cc +LIBWEBM_PARSER_SRCS += ../third_party/libwebm/mkvparser/mkvparser.h +LIBWEBM_PARSER_SRCS += ../third_party/libwebm/mkvparser/mkvreader.h +LIBAOM_TEST_SRCS-$(CONFIG_DECODERS) += $(LIBWEBM_PARSER_SRCS) +LIBAOM_TEST_SRCS-$(CONFIG_DECODERS) += ../tools_common.h +LIBAOM_TEST_SRCS-$(CONFIG_DECODERS) += ../webmdec.cc +LIBAOM_TEST_SRCS-$(CONFIG_DECODERS) += ../webmdec.h +LIBAOM_TEST_SRCS-$(CONFIG_DECODERS) += webm_video_source.h +endif + +LIBAOM_TEST_SRCS-$(CONFIG_DECODERS) += decode_api_test.cc + +# Currently we only support decoder perf tests for av1. Also they read from WebM +# files, so WebM IO is required. +ifeq ($(CONFIG_DECODE_PERF_TESTS)$(CONFIG_AV1_DECODER)$(CONFIG_WEBM_IO), \ + yesyesyes) +LIBAOM_TEST_SRCS-yes += decode_perf_test.cc +endif + +# encode perf tests are av1 only +ifeq ($(CONFIG_ENCODE_PERF_TESTS)$(CONFIG_AV1_ENCODER), yesyes) +LIBAOM_TEST_SRCS-yes += encode_perf_test.cc +endif + +## Multi-codec / unconditional black box tests. +ifeq ($(findstring yes,$(CONFIG_AV1_ENCODER)),yes) +LIBAOM_TEST_SRCS-yes += active_map_refresh_test.cc +LIBAOM_TEST_SRCS-yes += active_map_test.cc +LIBAOM_TEST_SRCS-yes += end_to_end_test.cc +endif + +## +## WHITE BOX TESTS +## +## Whitebox tests invoke functions not exposed via the public API. Certain +## shared library builds don't make these functions accessible. +## +ifeq ($(CONFIG_SHARED),) + +## AV1 +ifeq ($(CONFIG_AV1),yes) + +# These tests require both the encoder and decoder to be built. +ifeq ($(CONFIG_AV1_ENCODER)$(CONFIG_AV1_DECODER),yesyes) +# IDCT test currently depends on FDCT function +LIBAOM_TEST_SRCS-yes += idct8x8_test.cc +LIBAOM_TEST_SRCS-yes += partial_idct_test.cc +LIBAOM_TEST_SRCS-yes += superframe_test.cc +LIBAOM_TEST_SRCS-yes += tile_independence_test.cc +LIBAOM_TEST_SRCS-yes += ethread_test.cc +LIBAOM_TEST_SRCS-yes += motion_vector_test.cc +ifneq ($(CONFIG_ANS),yes) +LIBAOM_TEST_SRCS-yes += binary_codes_test.cc +endif +ifeq ($(CONFIG_EXT_TILE),yes) +LIBAOM_TEST_SRCS-yes += av1_ext_tile_test.cc +endif +ifeq ($(CONFIG_ANS),yes) +LIBAOM_TEST_SRCS-yes += ans_test.cc +LIBAOM_TEST_SRCS-yes += ans_codec_test.cc +else +LIBAOM_TEST_SRCS-yes += boolcoder_test.cc +ifeq ($(CONFIG_ACCOUNTING),yes) +LIBAOM_TEST_SRCS-yes += accounting_test.cc +endif +endif +LIBAOM_TEST_SRCS-yes += divu_small_test.cc +#LIBAOM_TEST_SRCS-yes += encoder_parms_get_to_decoder.cc +endif + +LIBAOM_TEST_SRCS-$(CONFIG_ADAPT_SCAN) += scan_test.cc +LIBAOM_TEST_SRCS-yes += convolve_test.cc +LIBAOM_TEST_SRCS-yes += lpf_8_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_CDEF) += dering_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_CDEF) += clpf_test.cc +LIBAOM_TEST_SRCS-yes += simd_cmp_impl.h +LIBAOM_TEST_SRCS-$(HAVE_SSE2) += simd_cmp_sse2.cc +LIBAOM_TEST_SRCS-$(HAVE_SSSE3) += simd_cmp_ssse3.cc +LIBAOM_TEST_SRCS-$(HAVE_SSE4_1) += simd_cmp_sse4.cc +LIBAOM_TEST_SRCS-$(HAVE_NEON) += simd_cmp_neon.cc +LIBAOM_TEST_SRCS-yes += simd_impl.h +LIBAOM_TEST_SRCS-$(HAVE_SSE2) += simd_sse2_test.cc +LIBAOM_TEST_SRCS-$(HAVE_SSSE3) += simd_ssse3_test.cc +LIBAOM_TEST_SRCS-$(HAVE_SSE4_1) += simd_sse4_test.cc +LIBAOM_TEST_SRCS-$(HAVE_NEON) += simd_neon_test.cc +LIBAOM_TEST_SRCS-yes += intrapred_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_INTRABC) += intrabc_test.cc +#LIBAOM_TEST_SRCS-$(CONFIG_AV1_DECODER) += av1_thread_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += dct16x16_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += dct32x32_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += fdct4x4_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += fdct8x8_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += hadamard_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += minmax_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += variance_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += error_block_test.cc +#LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_quantize_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += subtract_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += arf_freq_test.cc + + +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_inv_txfm_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_dct_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_fht4x4_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_fht8x8_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_fht16x16_test.cc +ifeq ($(CONFIG_EXT_TX),yes) +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_fht4x8_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_fht8x4_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_fht8x16_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_fht16x8_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_fht16x32_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_fht32x16_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += fht32x32_test.cc +endif + +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += sum_squares_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += subtract_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += blend_a64_mask_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += blend_a64_mask_1d_test.cc + +ifeq ($(CONFIG_EXT_INTER),yes) +LIBAOM_TEST_SRCS-$(HAVE_SSSE3) += masked_variance_test.cc +LIBAOM_TEST_SRCS-$(HAVE_SSSE3) += masked_sad_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_wedge_utils_test.cc +endif + +## Skip the unit test written for 4-tap filter intra predictor, because we +## revert to 3-tap filter. +## ifeq ($(CONFIG_FILTER_INTRA),yes) +## LIBAOM_TEST_SRCS-$(HAVE_SSE4_1) += filterintra_predictors_test.cc +## endif + +ifeq ($(CONFIG_MOTION_VAR),yes) +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += obmc_sad_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += obmc_variance_test.cc +endif + +ifeq ($(CONFIG_HIGHBITDEPTH),yes) +ifeq ($(CONFIG_AV1_ENCODER),yes) +LIBAOM_TEST_SRCS-$(HAVE_SSE4_1) += av1_quantize_test.cc +LIBAOM_TEST_SRCS-$(HAVE_SSE4_1) += av1_highbd_iht_test.cc +endif +endif # CONFIG_HIGHBITDEPTH +endif # AV1 + +## Multi-codec / unconditional whitebox tests. + +ifeq ($(CONFIG_AV1_ENCODER),yes) +LIBAOM_TEST_SRCS-yes += avg_test.cc +endif +ifeq ($(CONFIG_INTERNAL_STATS),yes) +LIBAOM_TEST_SRCS-$(CONFIG_HIGHBITDEPTH) += hbd_metrics_test.cc +endif +LIBAOM_TEST_SRCS-$(CONFIG_ENCODERS) += sad_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1) += av1_txfm_test.h +LIBAOM_TEST_SRCS-$(CONFIG_AV1) += av1_txfm_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_fwd_txfm1d_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_inv_txfm1d_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_fwd_txfm2d_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1_ENCODER) += av1_inv_txfm2d_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1) += av1_convolve_test.cc +LIBAOM_TEST_SRCS-$(CONFIG_AV1) += av1_convolve_optimz_test.cc +ifneq ($(findstring yes,$(CONFIG_GLOBAL_MOTION) $(CONFIG_WARPED_MOTION)),) +LIBAOM_TEST_SRCS-$(HAVE_SSE2) += warp_filter_test.cc warp_filter_test_util.cc +endif +ifeq ($(CONFIG_LOOP_RESTORATION),yes) +LIBAOM_TEST_SRCS-$(HAVE_SSE4_1) += selfguided_filter_test.cc +endif + +TEST_INTRA_PRED_SPEED_SRCS-yes := test_intra_pred_speed.cc +TEST_INTRA_PRED_SPEED_SRCS-yes += ../md5_utils.h ../md5_utils.c + +endif # CONFIG_SHARED + +include $(SRC_PATH_BARE)/test/test-data.mk diff --git a/third_party/aom/test/test_data_util.cmake b/third_party/aom/test/test_data_util.cmake new file mode 100644 index 0000000000..f096e4e12d --- /dev/null +++ b/third_party/aom/test/test_data_util.cmake @@ -0,0 +1,76 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## + +# Parses test/test-data.sha1 and writes captured file names and checksums to +# $out_files and $out_checksums as lists. +function (make_test_data_lists out_files out_checksums) + if (NOT AOM_TEST_DATA_LIST OR NOT EXISTS "${AOM_TEST_DATA_LIST}") + message(FATAL_ERROR "AOM_TEST_DATA_LIST (${AOM_TEST_DATA_LIST}) missing or " + "variable empty.") + endif () + + # Read test-data.sha1 into $files_and_checksums. $files_and_checksums becomes + # a list with an entry for each line from $AOM_TEST_DATA_LIST. + file(STRINGS "${AOM_TEST_DATA_LIST}" files_and_checksums) + + # Iterate over the list of lines and split it into $checksums and $filenames. + foreach (line ${files_and_checksums}) + string(FIND "${line}" " *" delim_pos) + + math(EXPR filename_pos "${delim_pos} + 2") + string(SUBSTRING "${line}" 0 ${delim_pos} checksum) + string(SUBSTRING "${line}" ${filename_pos} -1 filename) + + set(checksums ${checksums} ${checksum}) + set(filenames ${filenames} ${filename}) + endforeach () + + if (NOT checksums OR NOT filenames) + message(FATAL_ERROR "Parsing of ${AOM_TEST_DATA_LIST} failed.") + endif () + + set(${out_checksums} ${checksums} PARENT_SCOPE) + set(${out_files} ${filenames} PARENT_SCOPE) +endfunction () + +# Appends each file name in $test_files to $test_dir and adds the result path to +# $out_path_list. +function (expand_test_file_paths test_files test_dir out_path_list) + foreach (filename ${${test_files}}) + set(path_list ${path_list} "${test_dir}/${filename}") + endforeach () + set(${out_path_list} ${path_list} PARENT_SCOPE) +endfunction () + +function (check_file local_path expected_checksum out_needs_update) + if (EXISTS "${local_path}") + file(SHA1 "${local_path}" file_checksum) + else () + set(${out_needs_update} 1 PARENT_SCOPE) + return () + endif () + + if ("${file_checksum}" STREQUAL "${expected_checksum}") + unset(${out_needs_update} PARENT_SCOPE) + else () + set(${out_needs_update} 1 PARENT_SCOPE) + endif () +endfunction () + +# Downloads data from $file_url, confirms that $file_checksum matches, and +# writes it to $local_path. +function (download_test_file file_url file_checksum local_path) + message("Downloading ${file_url} ...") + file(DOWNLOAD "${file_url}" "${local_path}" + SHOW_PROGRESS + EXPECTED_HASH SHA1=${file_checksum}) + message("Download of ${file_url} complete.") +endfunction () diff --git a/third_party/aom/test/test_intra_pred_speed.cc b/third_party/aom/test/test_intra_pred_speed.cc new file mode 100644 index 0000000000..c4253628e7 --- /dev/null +++ b/third_party/aom/test/test_intra_pred_speed.cc @@ -0,0 +1,515 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +// Test and time AOM intra-predictor functions + +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/md5_helper.h" +#include "aom/aom_integer.h" +#include "aom_ports/mem.h" +#include "aom_ports/aom_timer.h" + +// ----------------------------------------------------------------------------- + +namespace { + +typedef void (*AvxPredFunc)(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left); + +#if CONFIG_ALT_INTRA +const int kNumAv1IntraFuncs = 14; +#else +const int kNumAv1IntraFuncs = 13; +#endif // CONFIG_ALT_INTRA +const char *kAv1IntraPredNames[kNumAv1IntraFuncs] = { + "DC_PRED", "DC_LEFT_PRED", "DC_TOP_PRED", "DC_128_PRED", "V_PRED", + "H_PRED", "D45_PRED", "D135_PRED", "D117_PRED", "D153_PRED", + "D207_PRED", "D63_PRED", "TM_PRED", +#if CONFIG_ALT_INTRA + "SMOOTH_PRED" +#endif // CONFIG_ALT_INTRA +}; + +void TestIntraPred(const char name[], AvxPredFunc const *pred_funcs, + const char *const pred_func_names[], int num_funcs, + const char *const signatures[], int /*block_size*/, + int num_pixels_per_test) { + libaom_test::ACMRandom rnd(libaom_test::ACMRandom::DeterministicSeed()); + const int kBPS = 32; + const int kTotalPixels = 32 * kBPS; + DECLARE_ALIGNED(16, uint8_t, src[kTotalPixels]); + DECLARE_ALIGNED(16, uint8_t, ref_src[kTotalPixels]); + DECLARE_ALIGNED(16, uint8_t, left[2 * kBPS]); + DECLARE_ALIGNED(16, uint8_t, above_mem[2 * kBPS + 16]); + uint8_t *const above = above_mem + 16; + for (int i = 0; i < kTotalPixels; ++i) ref_src[i] = rnd.Rand8(); + for (int i = 0; i < kBPS; ++i) left[i] = rnd.Rand8(); + for (int i = -1; i < kBPS; ++i) above[i] = rnd.Rand8(); + const int kNumTests = static_cast(2.e10 / num_pixels_per_test); + + // Fill up bottom-left and top-right pixels. + for (int i = kBPS; i < 2 * kBPS; ++i) { + left[i] = rnd.Rand8(); + above[i] = rnd.Rand8(); + } + + for (int k = 0; k < num_funcs; ++k) { + if (pred_funcs[k] == NULL) continue; + memcpy(src, ref_src, sizeof(src)); + aom_usec_timer timer; + aom_usec_timer_start(&timer); + for (int num_tests = 0; num_tests < kNumTests; ++num_tests) { + pred_funcs[k](src, kBPS, above, left); + } + libaom_test::ClearSystemState(); + aom_usec_timer_mark(&timer); + const int elapsed_time = + static_cast(aom_usec_timer_elapsed(&timer) / 1000); + libaom_test::MD5 md5; + md5.Add(src, sizeof(src)); + printf("Mode %s[%12s]: %5d ms MD5: %s\n", name, pred_func_names[k], + elapsed_time, md5.Get()); + EXPECT_STREQ(signatures[k], md5.Get()); + } +} + +void TestIntraPred4(AvxPredFunc const *pred_funcs) { + static const char *const kSignatures[kNumAv1IntraFuncs] = { + "4334156168b34ab599d9b5b30f522fe9", + "bc4649d5ba47c7ff178d92e475960fb0", + "8d316e5933326dcac24e1064794b5d12", + "a27270fed024eafd762c95de85f4da51", + "c33dff000d4256c2b8f3bf9e9bab14d2", + "44d8cddc2ad8f79b8ed3306051722b4f", + "df62e96dfcb25d8a435482756a6fa990", + "ecb0d56ae5f677ea45127ce9d5c058e4", + "0b7936841f6813da818275944895b574", + "9117972ef64f91a58ff73e1731c81db2", + "46d493dccf6e5356c6f3c0c73b7dd141", + "b852f42e6c4991d415400332d567872f", +#if CONFIG_ALT_INTRA + "828c49a4248993cce4876fa26eab697f", + "718c8cee9011f92ef31f77a9a7560010" +#else + "309a618577b27c648f9c5ee45252bc8f", +#endif // CONFIG_ALT_INTRA + }; + TestIntraPred("Intra4", pred_funcs, kAv1IntraPredNames, kNumAv1IntraFuncs, + kSignatures, 4, 4 * 4 * kNumAv1IntraFuncs); +} + +void TestIntraPred8(AvxPredFunc const *pred_funcs) { + static const char *const kSignatures[kNumAv1IntraFuncs] = { + "7694ddeeefed887faf9d339d18850928", + "7d726b1213591b99f736be6dec65065b", + "19c5711281357a485591aaf9c96c0a67", + "ba6b66877a089e71cd938e3b8c40caac", + "802440c93317e0f8ba93fab02ef74265", + "9e09a47a15deb0b9d8372824f9805080", + "a2fd4b66e1a667a3e582588934a7e4bd", + "78339c1c60bb1d67d248ab8c4da08b7f", + "5c97d70f7d47de1882a6cd86c165c8a9", + "8182bf60688b42205acd95e59e967157", + "9d69fcaf12398e67242d3fcf5cf2267e", + "7a09adb0fa6c2bf889a99dd816622feb", +#if CONFIG_ALT_INTRA + "f6ade499c626d38eb70661184b79bc57", + "1ad5b106c79b792e514ba25e87139b5e" +#else + "815b75c8e0d91cc1ae766dc5d3e445a3", +#endif // CONFIG_ALT_INTRA + }; + TestIntraPred("Intra8", pred_funcs, kAv1IntraPredNames, kNumAv1IntraFuncs, + kSignatures, 8, 8 * 8 * kNumAv1IntraFuncs); +} + +void TestIntraPred16(AvxPredFunc const *pred_funcs) { + static const char *const kSignatures[kNumAv1IntraFuncs] = { + "b40dbb555d5d16a043dc361e6694fe53", + "fb08118cee3b6405d64c1fd68be878c6", + "6c190f341475c837cc38c2e566b64875", + "db5c34ccbe2c7f595d9b08b0dc2c698c", + "a62cbfd153a1f0b9fed13e62b8408a7a", + "143df5b4c89335e281103f610f5052e4", + "404944b521d16f6edd160feeeb31ff35", + "7841fae7d4d47b519322e6a03eeed9dc", + "f6ebed3f71cbcf8d6d0516ce87e11093", + "3cc480297dbfeed01a1c2d78dd03d0c5", + "fbd607f15da218c5390a5b183b634a10", + "f7063ccbc29f87303d5c3d0555b08944", +#if CONFIG_ALT_INTRA + "7adcaaa3554eb71a81fc48cb9043984b", + "c0acea4397c1b4d54a21bbcec5731dff" +#else + "b8a41aa968ec108af447af4217cba91b", +#endif // CONFIG_ALT_INTRA + }; + TestIntraPred("Intra16", pred_funcs, kAv1IntraPredNames, kNumAv1IntraFuncs, + kSignatures, 16, 16 * 16 * kNumAv1IntraFuncs); +} + +void TestIntraPred32(AvxPredFunc const *pred_funcs) { + static const char *const kSignatures[kNumAv1IntraFuncs] = { + "558541656d84f9ae7896db655826febe", + "b3587a1f9a01495fa38c8cd3c8e2a1bf", + "4c6501e64f25aacc55a2a16c7e8f0255", + "b3b01379ba08916ef6b1b35f7d9ad51c", + "0f1eb38b6cbddb3d496199ef9f329071", + "911c06efb9ed1c3b4c104b232b55812f", + "b4f9f177a8a259514f039cfb403a61e3", + "0a6d584a44f8db9aa7ade2e2fdb9fc9e", + "b01c9076525216925f3456f034fb6eee", + "d267e20ad9e5cd2915d1a47254d3d149", + "3c45418137114cb6cef4c7a7baf4855c", + "d520125ebd512c63c301bf67fea8e059", +#if CONFIG_ALT_INTRA + "297e8fbb5d33c29b12b228fa9d7c40a4", + "31b9296d70dd82238c87173e6d5e65fd" +#else + "9e1370c6d42e08d357d9612c93a71cfc", +#endif // CONFIG_ALT_INTRA + }; + TestIntraPred("Intra32", pred_funcs, kAv1IntraPredNames, kNumAv1IntraFuncs, + kSignatures, 32, 32 * 32 * kNumAv1IntraFuncs); +} + +} // namespace + +// Defines a test case for |arch| (e.g., C, SSE2, ...) passing the predictors +// to |test_func|. The test name is 'arch.test_func', e.g., C.TestIntraPred4. +#define INTRA_PRED_TEST(arch, test_func, dc, dc_left, dc_top, dc_128, v, h, \ + d45e, d135, d117, d153, d207e, d63e, tm, smooth) \ + TEST(arch, test_func) { \ + static const AvxPredFunc aom_intra_pred[] = { \ + dc, dc_left, dc_top, dc_128, v, h, d45e, \ + d135, d117, d153, d207e, d63e, tm, smooth \ + }; \ + test_func(aom_intra_pred); \ + } + +// ----------------------------------------------------------------------------- +// 4x4 + +#if CONFIG_ALT_INTRA +#define tm_pred_func aom_paeth_predictor_4x4_c +#define smooth_pred_func aom_smooth_predictor_4x4_c +#else +#define tm_pred_func aom_tm_predictor_4x4_c +#define smooth_pred_func NULL +#endif // CONFIG_ALT_INTRA + +INTRA_PRED_TEST(C, TestIntraPred4, aom_dc_predictor_4x4_c, + aom_dc_left_predictor_4x4_c, aom_dc_top_predictor_4x4_c, + aom_dc_128_predictor_4x4_c, aom_v_predictor_4x4_c, + aom_h_predictor_4x4_c, aom_d45e_predictor_4x4_c, + aom_d135_predictor_4x4_c, aom_d117_predictor_4x4_c, + aom_d153_predictor_4x4_c, aom_d207e_predictor_4x4_c, + aom_d63e_predictor_4x4_c, tm_pred_func, smooth_pred_func) + +#undef tm_pred_func +#undef smooth_pred_func + +#if HAVE_SSE2 +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_4x4_sse2 +#endif // CONFIG_ALT_INTRA + +INTRA_PRED_TEST(SSE2, TestIntraPred4, aom_dc_predictor_4x4_sse2, + aom_dc_left_predictor_4x4_sse2, aom_dc_top_predictor_4x4_sse2, + aom_dc_128_predictor_4x4_sse2, aom_v_predictor_4x4_sse2, + aom_h_predictor_4x4_sse2, NULL, NULL, NULL, NULL, NULL, NULL, + tm_pred_func, NULL) + +#undef tm_pred_func +#endif // HAVE_SSE2 + +#if HAVE_SSSE3 +INTRA_PRED_TEST(SSSE3, TestIntraPred4, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, aom_d153_predictor_4x4_ssse3, NULL, + aom_d63e_predictor_4x4_ssse3, NULL, NULL) +#endif // HAVE_SSSE3 + +#if HAVE_DSPR2 +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_4x4_dspr2 +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(DSPR2, TestIntraPred4, aom_dc_predictor_4x4_dspr2, NULL, NULL, + NULL, NULL, aom_h_predictor_4x4_dspr2, NULL, NULL, NULL, NULL, + NULL, NULL, tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_DSPR2 + +#if HAVE_NEON +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_4x4_neon +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(NEON, TestIntraPred4, aom_dc_predictor_4x4_neon, + aom_dc_left_predictor_4x4_neon, aom_dc_top_predictor_4x4_neon, + aom_dc_128_predictor_4x4_neon, aom_v_predictor_4x4_neon, + aom_h_predictor_4x4_neon, NULL, aom_d135_predictor_4x4_neon, + NULL, NULL, NULL, NULL, tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_NEON + +#if HAVE_MSA +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_4x4_msa +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(MSA, TestIntraPred4, aom_dc_predictor_4x4_msa, + aom_dc_left_predictor_4x4_msa, aom_dc_top_predictor_4x4_msa, + aom_dc_128_predictor_4x4_msa, aom_v_predictor_4x4_msa, + aom_h_predictor_4x4_msa, NULL, NULL, NULL, NULL, NULL, NULL, + tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_MSA + +// ----------------------------------------------------------------------------- +// 8x8 + +#if CONFIG_ALT_INTRA +#define tm_pred_func aom_paeth_predictor_8x8_c +#define smooth_pred_func aom_smooth_predictor_8x8_c +#else +#define tm_pred_func aom_tm_predictor_8x8_c +#define smooth_pred_func NULL +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(C, TestIntraPred8, aom_dc_predictor_8x8_c, + aom_dc_left_predictor_8x8_c, aom_dc_top_predictor_8x8_c, + aom_dc_128_predictor_8x8_c, aom_v_predictor_8x8_c, + aom_h_predictor_8x8_c, aom_d45e_predictor_8x8_c, + aom_d135_predictor_8x8_c, aom_d117_predictor_8x8_c, + aom_d153_predictor_8x8_c, aom_d207e_predictor_8x8_c, + aom_d63e_predictor_8x8_c, tm_pred_func, smooth_pred_func) +#undef tm_pred_func +#undef smooth_pred_func + +#if HAVE_SSE2 +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_8x8_sse2 +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(SSE2, TestIntraPred8, aom_dc_predictor_8x8_sse2, + aom_dc_left_predictor_8x8_sse2, aom_dc_top_predictor_8x8_sse2, + aom_dc_128_predictor_8x8_sse2, aom_v_predictor_8x8_sse2, + aom_h_predictor_8x8_sse2, NULL, NULL, NULL, NULL, NULL, NULL, + tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_SSE2 + +#if HAVE_SSSE3 +INTRA_PRED_TEST(SSSE3, TestIntraPred8, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, aom_d153_predictor_8x8_ssse3, NULL, NULL, NULL, + NULL) +#endif // HAVE_SSSE3 + +#if HAVE_DSPR2 +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_8x8_dspr2 +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(DSPR2, TestIntraPred8, aom_dc_predictor_8x8_dspr2, NULL, NULL, + NULL, NULL, aom_h_predictor_8x8_dspr2, NULL, NULL, NULL, NULL, + NULL, NULL, tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_DSPR2 + +#if HAVE_NEON +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_8x8_neon +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(NEON, TestIntraPred8, aom_dc_predictor_8x8_neon, + aom_dc_left_predictor_8x8_neon, aom_dc_top_predictor_8x8_neon, + aom_dc_128_predictor_8x8_neon, aom_v_predictor_8x8_neon, + aom_h_predictor_8x8_neon, NULL, NULL, NULL, NULL, NULL, NULL, + tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_NEON + +#if HAVE_MSA +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_8x8_msa +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(MSA, TestIntraPred8, aom_dc_predictor_8x8_msa, + aom_dc_left_predictor_8x8_msa, aom_dc_top_predictor_8x8_msa, + aom_dc_128_predictor_8x8_msa, aom_v_predictor_8x8_msa, + aom_h_predictor_8x8_msa, NULL, NULL, NULL, NULL, NULL, NULL, + tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_MSA + +// ----------------------------------------------------------------------------- +// 16x16 + +#if CONFIG_ALT_INTRA +#define tm_pred_func aom_paeth_predictor_16x16_c +#define smooth_pred_func aom_smooth_predictor_16x16_c +#else +#define tm_pred_func aom_tm_predictor_16x16_c +#define smooth_pred_func NULL +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(C, TestIntraPred16, aom_dc_predictor_16x16_c, + aom_dc_left_predictor_16x16_c, aom_dc_top_predictor_16x16_c, + aom_dc_128_predictor_16x16_c, aom_v_predictor_16x16_c, + aom_h_predictor_16x16_c, aom_d45e_predictor_16x16_c, + aom_d135_predictor_16x16_c, aom_d117_predictor_16x16_c, + aom_d153_predictor_16x16_c, aom_d207e_predictor_16x16_c, + aom_d63e_predictor_16x16_c, tm_pred_func, smooth_pred_func) +#undef tm_pred_func +#undef smooth_pred_func + +#if HAVE_SSE2 +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_16x16_sse2 +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(SSE2, TestIntraPred16, aom_dc_predictor_16x16_sse2, + aom_dc_left_predictor_16x16_sse2, + aom_dc_top_predictor_16x16_sse2, + aom_dc_128_predictor_16x16_sse2, aom_v_predictor_16x16_sse2, + aom_h_predictor_16x16_sse2, NULL, NULL, NULL, NULL, NULL, NULL, + tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_SSE2 + +#if HAVE_SSSE3 +INTRA_PRED_TEST(SSSE3, TestIntraPred16, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, aom_d153_predictor_16x16_ssse3, NULL, NULL, + NULL, NULL) +#endif // HAVE_SSSE3 + +#if HAVE_DSPR2 +INTRA_PRED_TEST(DSPR2, TestIntraPred16, aom_dc_predictor_16x16_dspr2, NULL, + NULL, NULL, NULL, aom_h_predictor_16x16_dspr2, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL) +#endif // HAVE_DSPR2 + +#if HAVE_NEON +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_16x16_neon +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(NEON, TestIntraPred16, aom_dc_predictor_16x16_neon, + aom_dc_left_predictor_16x16_neon, + aom_dc_top_predictor_16x16_neon, + aom_dc_128_predictor_16x16_neon, aom_v_predictor_16x16_neon, + aom_h_predictor_16x16_neon, NULL, NULL, NULL, NULL, NULL, NULL, + tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_NEON + +#if HAVE_MSA +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_16x16_msa +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(MSA, TestIntraPred16, aom_dc_predictor_16x16_msa, + aom_dc_left_predictor_16x16_msa, aom_dc_top_predictor_16x16_msa, + aom_dc_128_predictor_16x16_msa, aom_v_predictor_16x16_msa, + aom_h_predictor_16x16_msa, NULL, NULL, NULL, NULL, NULL, NULL, + tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_MSA + +// ----------------------------------------------------------------------------- +// 32x32 + +#if CONFIG_ALT_INTRA +#define tm_pred_func aom_paeth_predictor_32x32_c +#define smooth_pred_func aom_smooth_predictor_32x32_c +#else +#define tm_pred_func aom_tm_predictor_32x32_c +#define smooth_pred_func NULL +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(C, TestIntraPred32, aom_dc_predictor_32x32_c, + aom_dc_left_predictor_32x32_c, aom_dc_top_predictor_32x32_c, + aom_dc_128_predictor_32x32_c, aom_v_predictor_32x32_c, + aom_h_predictor_32x32_c, aom_d45e_predictor_32x32_c, + aom_d135_predictor_32x32_c, aom_d117_predictor_32x32_c, + aom_d153_predictor_32x32_c, aom_d207e_predictor_32x32_c, + aom_d63e_predictor_32x32_c, tm_pred_func, smooth_pred_func) +#undef tm_pred_func +#undef smooth_pred_func + +#if HAVE_SSE2 +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_32x32_sse2 +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(SSE2, TestIntraPred32, aom_dc_predictor_32x32_sse2, + aom_dc_left_predictor_32x32_sse2, + aom_dc_top_predictor_32x32_sse2, + aom_dc_128_predictor_32x32_sse2, aom_v_predictor_32x32_sse2, + aom_h_predictor_32x32_sse2, NULL, NULL, NULL, NULL, NULL, NULL, + tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_SSE2 + +#if HAVE_SSSE3 +INTRA_PRED_TEST(SSSE3, TestIntraPred32, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, aom_d153_predictor_32x32_ssse3, NULL, NULL, + NULL, NULL) +#endif // HAVE_SSSE3 + +#if HAVE_NEON +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_32x32_neon +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(NEON, TestIntraPred32, aom_dc_predictor_32x32_neon, + aom_dc_left_predictor_32x32_neon, + aom_dc_top_predictor_32x32_neon, + aom_dc_128_predictor_32x32_neon, aom_v_predictor_32x32_neon, + aom_h_predictor_32x32_neon, NULL, NULL, NULL, NULL, NULL, NULL, + tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_NEON + +#if HAVE_MSA +#if CONFIG_ALT_INTRA +#define tm_pred_func NULL +#else +#define tm_pred_func aom_tm_predictor_32x32_msa +#endif // CONFIG_ALT_INTRA +INTRA_PRED_TEST(MSA, TestIntraPred32, aom_dc_predictor_32x32_msa, + aom_dc_left_predictor_32x32_msa, aom_dc_top_predictor_32x32_msa, + aom_dc_128_predictor_32x32_msa, aom_v_predictor_32x32_msa, + aom_h_predictor_32x32_msa, NULL, NULL, NULL, NULL, NULL, NULL, + tm_pred_func, NULL) +#undef tm_pred_func +#endif // HAVE_MSA + +#include "test/test_libaom.cc" diff --git a/third_party/aom/test/test_libaom.cc b/third_party/aom/test/test_libaom.cc new file mode 100644 index 0000000000..6d83ce66e7 --- /dev/null +++ b/third_party/aom/test/test_libaom.cc @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#if ARCH_X86 || ARCH_X86_64 +#include "aom_ports/x86.h" +#endif +extern "C" { +#if CONFIG_AV1 +extern void av1_rtcd(); +#endif // CONFIG_AV1 +extern void aom_dsp_rtcd(); +extern void aom_scale_rtcd(); +} + +#if ARCH_X86 || ARCH_X86_64 +static void append_negative_gtest_filter(const char *str) { + std::string filter = ::testing::FLAGS_gtest_filter; + // Negative patterns begin with one '-' followed by a ':' separated list. + if (filter.find('-') == std::string::npos) filter += '-'; + filter += str; + ::testing::FLAGS_gtest_filter = filter; +} +#endif // ARCH_X86 || ARCH_X86_64 + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + +#if ARCH_X86 || ARCH_X86_64 + const int simd_caps = x86_simd_caps(); + if (!(simd_caps & HAS_MMX)) append_negative_gtest_filter(":MMX.*:MMX/*"); + if (!(simd_caps & HAS_SSE)) append_negative_gtest_filter(":SSE.*:SSE/*"); + if (!(simd_caps & HAS_SSE2)) append_negative_gtest_filter(":SSE2.*:SSE2/*"); + if (!(simd_caps & HAS_SSE3)) append_negative_gtest_filter(":SSE3.*:SSE3/*"); + if (!(simd_caps & HAS_SSSE3)) + append_negative_gtest_filter(":SSSE3.*:SSSE3/*"); + if (!(simd_caps & HAS_SSE4_1)) + append_negative_gtest_filter(":SSE4_1.*:SSE4_1/*"); + if (!(simd_caps & HAS_AVX)) append_negative_gtest_filter(":AVX.*:AVX/*"); + if (!(simd_caps & HAS_AVX2)) append_negative_gtest_filter(":AVX2.*:AVX2/*"); +#endif // ARCH_X86 || ARCH_X86_64 + +#if !CONFIG_SHARED +// Shared library builds don't support whitebox tests +// that exercise internal symbols. + +#if CONFIG_AV1 + av1_rtcd(); +#endif // CONFIG_AV1 + aom_dsp_rtcd(); + aom_scale_rtcd(); +#endif // !CONFIG_SHARED + + return RUN_ALL_TESTS(); +} diff --git a/third_party/aom/test/test_worker.cmake b/third_party/aom/test/test_worker.cmake new file mode 100644 index 0000000000..fa1d581306 --- /dev/null +++ b/third_party/aom/test/test_worker.cmake @@ -0,0 +1,49 @@ +## +## Copyright (c) 2017, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +if (NOT AOM_ROOT OR NOT AOM_CONFIG_DIR) + message(FATAL_ERROR "AOM_ROOT AND AOM_CONFIG_DIR must be defined.") +endif () + +set(AOM_TEST_DATA_LIST "${AOM_ROOT}/test/test-data.sha1") +set(AOM_TEST_DATA_URL "http://downloads.webmproject.org/test_data/libvpx") +set(AOM_TEST_DATA_PATH "$ENV{LIBAOM_TEST_DATA_PATH}") + +include("${AOM_ROOT}/test/test_data_util.cmake") + +if (${AOM_TEST_DATA_PATH} STREQUAL "") + message(WARNING "Writing test data to ${AOM_CONFIG_DIR}, set " + "$LIBAOM_TEST_DATA_PATH in your environment to avoid this warning.") + set(AOM_TEST_DATA_PATH "${AOM_CONFIG_DIR}") +endif () + +if (NOT EXISTS "${AOM_TEST_DATA_PATH}") + file(MAKE_DIRECTORY "${AOM_TEST_DATA_PATH}") +endif () + +make_test_data_lists("AOM_TEST_DATA_FILES" "AOM_TEST_DATA_CHECKSUMS") +expand_test_file_paths("AOM_TEST_DATA_FILES" "${AOM_TEST_DATA_PATH}" + "AOM_TEST_DATA_FILE_PATHS") +expand_test_file_paths("AOM_TEST_DATA_FILES" "${AOM_TEST_DATA_URL}" + "AOM_TEST_DATA_URLS") +list(LENGTH AOM_TEST_DATA_FILES num_files) +math(EXPR num_files "${num_files} - 1") + +foreach (file_num RANGE ${num_files}) + list(GET AOM_TEST_DATA_FILES ${file_num} filename) + list(GET AOM_TEST_DATA_CHECKSUMS ${file_num} checksum) + list(GET AOM_TEST_DATA_FILE_PATHS ${file_num} filepath) + list(GET AOM_TEST_DATA_URLS ${file_num} url) + + check_file("${filepath}" "${checksum}" "needs_download") + if (needs_download) + download_test_file("${url}" "${checksum}" "${filepath}") + endif () +endforeach () diff --git a/third_party/aom/test/tile_independence_test.cc b/third_party/aom/test/tile_independence_test.cc new file mode 100644 index 0000000000..a29051f2f7 --- /dev/null +++ b/third_party/aom/test/tile_independence_test.cc @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include +#include +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" +#include "test/md5_helper.h" +#include "aom_mem/aom_mem.h" + +namespace { +class TileIndependenceTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params { + protected: + TileIndependenceTest() + : EncoderTest(GET_PARAM(0)), md5_fw_order_(), md5_inv_order_(), + n_tile_cols_(GET_PARAM(1)), n_tile_rows_(GET_PARAM(2)) { + init_flags_ = AOM_CODEC_USE_PSNR; + aom_codec_dec_cfg_t cfg = aom_codec_dec_cfg_t(); + cfg.w = 704; + cfg.h = 144; + cfg.threads = 1; + fw_dec_ = codec_->CreateDecoder(cfg, 0); + inv_dec_ = codec_->CreateDecoder(cfg, 0); + inv_dec_->Control(AV1_INVERT_TILE_DECODE_ORDER, 1); + +#if CONFIG_AV1 && CONFIG_EXT_TILE + if (fw_dec_->IsAV1() && inv_dec_->IsAV1()) { + fw_dec_->Control(AV1_SET_DECODE_TILE_ROW, -1); + fw_dec_->Control(AV1_SET_DECODE_TILE_COL, -1); + inv_dec_->Control(AV1_SET_DECODE_TILE_ROW, -1); + inv_dec_->Control(AV1_SET_DECODE_TILE_COL, -1); + } +#endif + } + + virtual ~TileIndependenceTest() { + delete fw_dec_; + delete inv_dec_; + } + + virtual void SetUp() { + InitializeConfig(); + SetMode(libaom_test::kTwoPassGood); + } + + virtual void PreEncodeFrameHook(libaom_test::VideoSource *video, + libaom_test::Encoder *encoder) { + if (video->frame() == 1) { + encoder->Control(AV1E_SET_TILE_COLUMNS, n_tile_cols_); + encoder->Control(AV1E_SET_TILE_ROWS, n_tile_rows_); +#if CONFIG_EXT_TILE + encoder->Control(AV1E_SET_TILE_ENCODING_MODE, 0); // TILE_NORMAL +#endif // CONFIG_EXT_TILE +#if CONFIG_LOOPFILTERING_ACROSS_TILES + encoder->Control(AV1E_SET_TILE_LOOPFILTER, 0); +#endif // CONFIG_LOOPFILTERING_ACROSS_TILES + SetCpuUsed(encoder); + } + } + + virtual void SetCpuUsed(libaom_test::Encoder *encoder) { + static const int kCpuUsed = 3; + encoder->Control(AOME_SET_CPUUSED, kCpuUsed); + } + + void UpdateMD5(::libaom_test::Decoder *dec, const aom_codec_cx_pkt_t *pkt, + ::libaom_test::MD5 *md5) { + const aom_codec_err_t res = dec->DecodeFrame( + reinterpret_cast(pkt->data.frame.buf), pkt->data.frame.sz); + if (res != AOM_CODEC_OK) { + abort_ = true; + ASSERT_EQ(AOM_CODEC_OK, res); + } + const aom_image_t *img = dec->GetDxData().Next(); + md5->Add(img); + } + + virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) { + UpdateMD5(fw_dec_, pkt, &md5_fw_order_); + UpdateMD5(inv_dec_, pkt, &md5_inv_order_); + } + + void DoTest() { + const aom_rational timebase = { 33333333, 1000000000 }; + cfg_.g_timebase = timebase; + cfg_.rc_target_bitrate = 500; + cfg_.g_lag_in_frames = 12; + cfg_.rc_end_usage = AOM_VBR; + + libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 704, 576, + timebase.den, timebase.num, 0, 5); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + + const char *md5_fw_str = md5_fw_order_.Get(); + const char *md5_inv_str = md5_inv_order_.Get(); + ASSERT_STREQ(md5_fw_str, md5_inv_str); + } + + ::libaom_test::MD5 md5_fw_order_, md5_inv_order_; + ::libaom_test::Decoder *fw_dec_, *inv_dec_; + + private: + int n_tile_cols_; + int n_tile_rows_; +}; + +// run an encode with 2 or 4 tiles, and do the decode both in normal and +// inverted tile ordering. Ensure that the MD5 of the output in both cases +// is identical. If so, tiles are considered independent and the test passes. +TEST_P(TileIndependenceTest, MD5Match) { DoTest(); } + +class TileIndependenceTestLarge : public TileIndependenceTest { + virtual void SetCpuUsed(libaom_test::Encoder *encoder) { + static const int kCpuUsed = 0; + encoder->Control(AOME_SET_CPUUSED, kCpuUsed); + } +}; + +TEST_P(TileIndependenceTestLarge, MD5Match) { DoTest(); } + +#if CONFIG_EXT_TILE +AV1_INSTANTIATE_TEST_CASE(TileIndependenceTest, ::testing::Values(1, 2, 32), + ::testing::Values(1, 2, 32)); +AV1_INSTANTIATE_TEST_CASE(TileIndependenceTestLarge, + ::testing::Values(1, 2, 32), + ::testing::Values(1, 2, 32)); +#else +AV1_INSTANTIATE_TEST_CASE(TileIndependenceTest, ::testing::Values(0, 1), + ::testing::Values(0, 1)); +AV1_INSTANTIATE_TEST_CASE(TileIndependenceTestLarge, ::testing::Values(0, 1), + ::testing::Values(0, 1)); +#endif // CONFIG_EXT_TILE +} // namespace diff --git a/third_party/aom/test/tools_common.sh b/third_party/aom/test/tools_common.sh new file mode 100755 index 0000000000..254e6b2960 --- /dev/null +++ b/third_party/aom/test/tools_common.sh @@ -0,0 +1,454 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This file contains shell code shared by test scripts for libaom tools. + +# Use $AOM_TEST_TOOLS_COMMON_SH as a pseudo include guard. +if [ -z "${AOM_TEST_TOOLS_COMMON_SH}" ]; then +AOM_TEST_TOOLS_COMMON_SH=included + +set -e +devnull='> /dev/null 2>&1' +AOM_TEST_PREFIX="" + +elog() { + echo "$@" 1>&2 +} + +vlog() { + if [ "${AOM_TEST_VERBOSE_OUTPUT}" = "yes" ]; then + echo "$@" + fi +} + +# Sets $AOM_TOOL_TEST to the name specified by positional parameter one. +test_begin() { + AOM_TOOL_TEST="${1}" +} + +# Clears the AOM_TOOL_TEST variable after confirming that $AOM_TOOL_TEST matches +# positional parameter one. +test_end() { + if [ "$1" != "${AOM_TOOL_TEST}" ]; then + echo "FAIL completed test mismatch!." + echo " completed test: ${1}" + echo " active test: ${AOM_TOOL_TEST}." + return 1 + fi + AOM_TOOL_TEST='' +} + +# Echoes the target configuration being tested. +test_configuration_target() { + aom_config_mk="${LIBAOM_CONFIG_PATH}/config.mk" + # Find the TOOLCHAIN line, split it using ':=' as the field separator, and + # print the last field to get the value. Then pipe the value to tr to consume + # any leading/trailing spaces while allowing tr to echo the output to stdout. + awk -F ':=' '/TOOLCHAIN/ { print $NF }' "${aom_config_mk}" | tr -d ' ' +} + +# Trap function used for failure reports and tool output directory removal. +# When the contents of $AOM_TOOL_TEST do not match the string '', reports +# failure of test stored in $AOM_TOOL_TEST. +cleanup() { + if [ -n "${AOM_TOOL_TEST}" ] && [ "${AOM_TOOL_TEST}" != '' ]; then + echo "FAIL: $AOM_TOOL_TEST" + fi + if [ -n "${AOM_TEST_OUTPUT_DIR}" ] && [ -d "${AOM_TEST_OUTPUT_DIR}" ]; then + rm -rf "${AOM_TEST_OUTPUT_DIR}" + fi +} + +# Echoes the git hash portion of the VERSION_STRING variable defined in +# $LIBAOM_CONFIG_PATH/config.mk to stdout, or the version number string when +# no git hash is contained in VERSION_STRING. +config_hash() { + aom_config_mk="${LIBAOM_CONFIG_PATH}/config.mk" + # Find VERSION_STRING line, split it with "-g" and print the last field to + # output the git hash to stdout. + aom_version=$(awk -F -g '/VERSION_STRING/ {print $NF}' "${aom_config_mk}") + # Handle two situations here: + # 1. The default case: $aom_version is a git hash, so echo it unchanged. + # 2. When being run a non-dev tree, the -g portion is not present in the + # version string: It's only the version number. + # In this case $aom_version is something like 'VERSION_STRING=v1.3.0', so + # we echo only what is after the '='. + echo "${aom_version##*=}" +} + +# Echoes the short form of the current git hash. +current_hash() { + if git --version > /dev/null 2>&1; then + (cd "$(dirname "${0}")" + git rev-parse --short HEAD) + else + # Return the config hash if git is unavailable: Fail silently, git hashes + # are used only for warnings. + config_hash + fi +} + +# Echoes warnings to stdout when git hash in aom_config.h does not match the +# current git hash. +check_git_hashes() { + hash_at_configure_time=$(config_hash) + hash_now=$(current_hash) + + if [ "${hash_at_configure_time}" != "${hash_now}" ]; then + echo "Warning: git hash has changed since last configure." + fi +} + +# $1 is the name of an environment variable containing a directory name to +# test. +test_env_var_dir() { + local dir=$(eval echo "\${$1}") + if [ ! -d "${dir}" ]; then + elog "'${dir}': No such directory" + elog "The $1 environment variable must be set to a valid directory." + return 1 + fi +} + +# This script requires that the LIBAOM_BIN_PATH, LIBAOM_CONFIG_PATH, and +# LIBAOM_TEST_DATA_PATH variables are in the environment: Confirm that +# the variables are set and that they all evaluate to directory paths. +verify_aom_test_environment() { + test_env_var_dir "LIBAOM_BIN_PATH" \ + && test_env_var_dir "LIBAOM_CONFIG_PATH" \ + && test_env_var_dir "LIBAOM_TEST_DATA_PATH" +} + +# Greps aom_config.h in LIBAOM_CONFIG_PATH for positional parameter one, which +# should be a LIBAOM preprocessor flag. Echoes yes to stdout when the feature +# is available. +aom_config_option_enabled() { + aom_config_option="${1}" + aom_config_file="${LIBAOM_CONFIG_PATH}/aom_config.h" + config_line=$(grep "${aom_config_option}" "${aom_config_file}") + if echo "${config_line}" | egrep -q '1$'; then + echo yes + fi +} + +# Echoes yes when output of test_configuration_target() contains win32 or win64. +is_windows_target() { + if test_configuration_target \ + | grep -q -e win32 -e win64 > /dev/null 2>&1; then + echo yes + fi +} + +# Echoes path to $1 when it's executable and exists in ${LIBAOM_BIN_PATH}, or an +# empty string. Caller is responsible for testing the string once the function +# returns. +aom_tool_path() { + local readonly tool_name="$1" + local tool_path="${LIBAOM_BIN_PATH}/${tool_name}${AOM_TEST_EXE_SUFFIX}" + if [ ! -x "${tool_path}" ]; then + # Try one directory up: when running via examples.sh the tool could be in + # the parent directory of $LIBAOM_BIN_PATH. + tool_path="${LIBAOM_BIN_PATH}/../${tool_name}${AOM_TEST_EXE_SUFFIX}" + fi + + if [ ! -x "${tool_path}" ]; then + tool_path="" + fi + echo "${tool_path}" +} + +# Echoes yes to stdout when the file named by positional parameter one exists +# in LIBAOM_BIN_PATH, and is executable. +aom_tool_available() { + local tool_name="$1" + local tool="${LIBAOM_BIN_PATH}/${tool_name}${AOM_TEST_EXE_SUFFIX}" + [ -x "${tool}" ] && echo yes +} + +# Echoes yes to stdout when aom_config_option_enabled() reports yes for +# CONFIG_AV1_DECODER. +av1_decode_available() { + [ "$(aom_config_option_enabled CONFIG_AV1_DECODER)" = "yes" ] && echo yes +} + +# Echoes yes to stdout when aom_config_option_enabled() reports yes for +# CONFIG_AV1_ENCODER. +av1_encode_available() { + [ "$(aom_config_option_enabled CONFIG_AV1_ENCODER)" = "yes" ] && echo yes +} + +# Echoes yes to stdout when aom_config_option_enabled() reports yes for +# CONFIG_WEBM_IO. +webm_io_available() { + [ "$(aom_config_option_enabled CONFIG_WEBM_IO)" = "yes" ] && echo yes +} + +# Filters strings from $1 using the filter specified by $2. Filter behavior +# depends on the presence of $3. When $3 is present, strings that match the +# filter are excluded. When $3 is omitted, strings matching the filter are +# included. +# The filtered result is echoed to stdout. +filter_strings() { + strings=${1} + filter=${2} + exclude=${3} + + if [ -n "${exclude}" ]; then + # When positional parameter three exists the caller wants to remove strings. + # Tell grep to invert matches using the -v argument. + exclude='-v' + else + unset exclude + fi + + if [ -n "${filter}" ]; then + for s in ${strings}; do + if echo "${s}" | egrep -q ${exclude} "${filter}" > /dev/null 2>&1; then + filtered_strings="${filtered_strings} ${s}" + fi + done + else + filtered_strings="${strings}" + fi + echo "${filtered_strings}" +} + +# Runs user test functions passed via positional parameters one and two. +# Functions in positional parameter one are treated as environment verification +# functions and are run unconditionally. Functions in positional parameter two +# are run according to the rules specified in aom_test_usage(). +run_tests() { + local env_tests="verify_aom_test_environment $1" + local tests_to_filter="$2" + local test_name="${AOM_TEST_NAME}" + + if [ -z "${test_name}" ]; then + test_name="$(basename "${0%.*}")" + fi + + if [ "${AOM_TEST_RUN_DISABLED_TESTS}" != "yes" ]; then + # Filter out DISABLED tests. + tests_to_filter=$(filter_strings "${tests_to_filter}" ^DISABLED exclude) + fi + + if [ -n "${AOM_TEST_FILTER}" ]; then + # Remove tests not matching the user's filter. + tests_to_filter=$(filter_strings "${tests_to_filter}" ${AOM_TEST_FILTER}) + fi + + # User requested test listing: Dump test names and return. + if [ "${AOM_TEST_LIST_TESTS}" = "yes" ]; then + for test_name in $tests_to_filter; do + echo ${test_name} + done + return + fi + + # Don't bother with the environment tests if everything else was disabled. + [ -z "${tests_to_filter}" ] && return + + # Combine environment and actual tests. + local tests_to_run="${env_tests} ${tests_to_filter}" + + check_git_hashes + + # Run tests. + for test in ${tests_to_run}; do + test_begin "${test}" + vlog " RUN ${test}" + "${test}" + vlog " PASS ${test}" + test_end "${test}" + done + + local tested_config="$(test_configuration_target) @ $(current_hash)" + echo "${test_name}: Done, all tests pass for ${tested_config}." +} + +aom_test_usage() { +cat << EOF + Usage: ${0##*/} [arguments] + --bin-path + --config-path + --filter : User test filter. Only tests matching filter are run. + --run-disabled-tests: Run disabled tests. + --help: Display this message and exit. + --test-data-path + --show-program-output: Shows output from all programs being tested. + --prefix: Allows for a user specified prefix to be inserted before all test + programs. Grants the ability, for example, to run test programs + within valgrind. + --list-tests: List all test names and exit without actually running tests. + --verbose: Verbose output. + + When the --bin-path option is not specified the script attempts to use + \$LIBAOM_BIN_PATH and then the current directory. + + When the --config-path option is not specified the script attempts to use + \$LIBAOM_CONFIG_PATH and then the current directory. + + When the -test-data-path option is not specified the script attempts to use + \$LIBAOM_TEST_DATA_PATH and then the current directory. +EOF +} + +# Returns non-zero (failure) when required environment variables are empty +# strings. +aom_test_check_environment() { + if [ -z "${LIBAOM_BIN_PATH}" ] || \ + [ -z "${LIBAOM_CONFIG_PATH}" ] || \ + [ -z "${LIBAOM_TEST_DATA_PATH}" ]; then + return 1 + fi +} + +# Echo aomenc command line parameters allowing use of a raw yuv file as +# input to aomenc. +yuv_raw_input() { + echo ""${YUV_RAW_INPUT}" + --width="${YUV_RAW_INPUT_WIDTH}" + --height="${YUV_RAW_INPUT_HEIGHT}"" +} + +# Do a small encode for testing decoders. +encode_yuv_raw_input_av1() { + if [ "$(av1_encode_available)" = "yes" ]; then + local readonly output="$1" + local readonly encoder="$(aom_tool_path aomenc)" + shift + eval "${encoder}" $(yuv_raw_input) \ + --codec=av1 \ + $@ \ + --limit=5 \ + --output="${output}" \ + ${devnull} + + if [ ! -e "${output}" ]; then + elog "Output file does not exist." + return 1 + fi + fi +} + +# Parse the command line. +while [ -n "$1" ]; do + case "$1" in + --bin-path) + LIBAOM_BIN_PATH="$2" + shift + ;; + --config-path) + LIBAOM_CONFIG_PATH="$2" + shift + ;; + --filter) + AOM_TEST_FILTER="$2" + shift + ;; + --run-disabled-tests) + AOM_TEST_RUN_DISABLED_TESTS=yes + ;; + --help) + aom_test_usage + exit + ;; + --test-data-path) + LIBAOM_TEST_DATA_PATH="$2" + shift + ;; + --prefix) + AOM_TEST_PREFIX="$2" + shift + ;; + --verbose) + AOM_TEST_VERBOSE_OUTPUT=yes + ;; + --show-program-output) + devnull= + ;; + --list-tests) + AOM_TEST_LIST_TESTS=yes + ;; + *) + aom_test_usage + exit 1 + ;; + esac + shift +done + +# Handle running the tests from a build directory without arguments when running +# the tests on *nix/macosx. +LIBAOM_BIN_PATH="${LIBAOM_BIN_PATH:-.}" +LIBAOM_CONFIG_PATH="${LIBAOM_CONFIG_PATH:-.}" +LIBAOM_TEST_DATA_PATH="${LIBAOM_TEST_DATA_PATH:-.}" + +# Create a temporary directory for output files, and a trap to clean it up. +if [ -n "${TMPDIR}" ]; then + AOM_TEST_TEMP_ROOT="${TMPDIR}" +elif [ -n "${TEMPDIR}" ]; then + AOM_TEST_TEMP_ROOT="${TEMPDIR}" +else + AOM_TEST_TEMP_ROOT=/tmp +fi + +AOM_TEST_OUTPUT_DIR="${AOM_TEST_TEMP_ROOT}/aom_test_$$" + +if ! mkdir -p "${AOM_TEST_OUTPUT_DIR}" || \ + [ ! -d "${AOM_TEST_OUTPUT_DIR}" ]; then + echo "${0##*/}: Cannot create output directory, giving up." + echo "${0##*/}: AOM_TEST_OUTPUT_DIR=${AOM_TEST_OUTPUT_DIR}" + exit 1 +fi + +if [ "$(is_windows_target)" = "yes" ]; then + AOM_TEST_EXE_SUFFIX=".exe" +fi + +# Variables shared by tests. +VP8_IVF_FILE="${LIBAOM_TEST_DATA_PATH}/vp80-00-comprehensive-001.ivf" +AV1_IVF_FILE="${LIBAOM_TEST_DATA_PATH}/vp90-2-09-subpixel-00.ivf" + +AV1_WEBM_FILE="${LIBAOM_TEST_DATA_PATH}/vp90-2-00-quantizer-00.webm" +AV1_FPM_WEBM_FILE="${LIBAOM_TEST_DATA_PATH}/vp90-2-07-frame_parallel-1.webm" +AV1_LT_50_FRAMES_WEBM_FILE="${LIBAOM_TEST_DATA_PATH}/vp90-2-02-size-32x08.webm" + +YUV_RAW_INPUT="${LIBAOM_TEST_DATA_PATH}/hantro_collage_w352h288.yuv" +YUV_RAW_INPUT_WIDTH=352 +YUV_RAW_INPUT_HEIGHT=288 + +Y4M_NOSQ_PAR_INPUT="${LIBAOM_TEST_DATA_PATH}/park_joy_90p_8_420_a10-1.y4m" +Y4M_720P_INPUT="${LIBAOM_TEST_DATA_PATH}/niklas_1280_720_30.y4m" + +# Setup a trap function to clean up after tests complete. +trap cleanup EXIT + +vlog "$(basename "${0%.*}") test configuration: + LIBAOM_BIN_PATH=${LIBAOM_BIN_PATH} + LIBAOM_CONFIG_PATH=${LIBAOM_CONFIG_PATH} + LIBAOM_TEST_DATA_PATH=${LIBAOM_TEST_DATA_PATH} + AOM_IVF_FILE=${AOM_IVF_FILE} + AV1_IVF_FILE=${AV1_IVF_FILE} + AV1_WEBM_FILE=${AV1_WEBM_FILE} + AOM_TEST_EXE_SUFFIX=${AOM_TEST_EXE_SUFFIX} + AOM_TEST_FILTER=${AOM_TEST_FILTER} + AOM_TEST_LIST_TESTS=${AOM_TEST_LIST_TESTS} + AOM_TEST_OUTPUT_DIR=${AOM_TEST_OUTPUT_DIR} + AOM_TEST_PREFIX=${AOM_TEST_PREFIX} + AOM_TEST_RUN_DISABLED_TESTS=${AOM_TEST_RUN_DISABLED_TESTS} + AOM_TEST_SHOW_PROGRAM_OUTPUT=${AOM_TEST_SHOW_PROGRAM_OUTPUT} + AOM_TEST_TEMP_ROOT=${AOM_TEST_TEMP_ROOT} + AOM_TEST_VERBOSE_OUTPUT=${AOM_TEST_VERBOSE_OUTPUT} + YUV_RAW_INPUT=${YUV_RAW_INPUT} + YUV_RAW_INPUT_WIDTH=${YUV_RAW_INPUT_WIDTH} + YUV_RAW_INPUT_HEIGHT=${YUV_RAW_INPUT_HEIGHT} + Y4M_NOSQ_PAR_INPUT=${Y4M_NOSQ_PAR_INPUT}" + +fi # End $AOM_TEST_TOOLS_COMMON_SH pseudo include guard. diff --git a/third_party/aom/test/transform_test_base.h b/third_party/aom/test/transform_test_base.h new file mode 100644 index 0000000000..4c1a554968 --- /dev/null +++ b/third_party/aom/test/transform_test_base.h @@ -0,0 +1,367 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef TEST_TRANSFORM_TEST_BASE_H_ +#define TEST_TRANSFORM_TEST_BASE_H_ + +#include "./aom_config.h" +#include "aom_mem/aom_mem.h" +#include "aom/aom_codec.h" + +namespace libaom_test { + +// Note: +// Same constant are defined in av1/common/av1_entropy.h and +// av1/common/entropy.h. Goal is to make this base class +// to use for future codec transform testing. But including +// either of them would lead to compiling error when we do +// unit test for another codec. Suggest to move the definition +// to a aom header file. +const int kDctMaxValue = 16384; + +typedef void (*FhtFunc)(const int16_t *in, tran_low_t *out, int stride, + int tx_type); + +typedef void (*IhtFunc)(const tran_low_t *in, uint8_t *out, int stride, + int tx_type); + +class TransformTestBase { + public: + virtual ~TransformTestBase() {} + + protected: + virtual void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) = 0; + + virtual void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) = 0; + + void RunAccuracyCheck(uint32_t ref_max_error, double ref_avg_error) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + uint32_t max_error = 0; + int64_t total_error = 0; + const int count_test_block = 10000; + + int16_t *test_input_block = reinterpret_cast( + aom_memalign(16, sizeof(int16_t) * num_coeffs_)); + tran_low_t *test_temp_block = reinterpret_cast( + aom_memalign(16, sizeof(tran_low_t) * num_coeffs_)); + uint8_t *dst = reinterpret_cast( + aom_memalign(16, sizeof(uint8_t) * num_coeffs_)); + uint8_t *src = reinterpret_cast( + aom_memalign(16, sizeof(uint8_t) * num_coeffs_)); +#if CONFIG_HIGHBITDEPTH + uint16_t *dst16 = reinterpret_cast( + aom_memalign(16, sizeof(uint16_t) * num_coeffs_)); + uint16_t *src16 = reinterpret_cast( + aom_memalign(16, sizeof(uint16_t) * num_coeffs_)); +#endif + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-255, 255]. + for (int j = 0; j < num_coeffs_; ++j) { + if (bit_depth_ == AOM_BITS_8) { + src[j] = rnd.Rand8(); + dst[j] = rnd.Rand8(); + test_input_block[j] = src[j] - dst[j]; +#if CONFIG_HIGHBITDEPTH + } else { + src16[j] = rnd.Rand16() & mask_; + dst16[j] = rnd.Rand16() & mask_; + test_input_block[j] = src16[j] - dst16[j]; +#endif + } + } + + ASM_REGISTER_STATE_CHECK( + RunFwdTxfm(test_input_block, test_temp_block, pitch_)); + if (bit_depth_ == AOM_BITS_8) { + ASM_REGISTER_STATE_CHECK(RunInvTxfm(test_temp_block, dst, pitch_)); +#if CONFIG_HIGHBITDEPTH + } else { + ASM_REGISTER_STATE_CHECK( + RunInvTxfm(test_temp_block, CONVERT_TO_BYTEPTR(dst16), pitch_)); +#endif + } + + for (int j = 0; j < num_coeffs_; ++j) { +#if CONFIG_HIGHBITDEPTH + const int diff = + bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; +#else + ASSERT_EQ(AOM_BITS_8, bit_depth_); + const int diff = dst[j] - src[j]; +#endif + const uint32_t error = diff * diff; + if (max_error < error) max_error = error; + total_error += error; + } + } + + double avg_error = total_error * 1. / count_test_block / num_coeffs_; + + EXPECT_GE(ref_max_error, max_error) + << "Error: FHT/IHT has an individual round trip error > " + << ref_max_error; + + EXPECT_GE(ref_avg_error, avg_error) + << "Error: FHT/IHT has average round trip error > " << ref_avg_error + << " per block"; + + aom_free(test_input_block); + aom_free(test_temp_block); + aom_free(dst); + aom_free(src); +#if CONFIG_HIGHBITDEPTH + aom_free(dst16); + aom_free(src16); +#endif + } + + void RunCoeffCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 5000; + + // Use a stride value which is not the width of any transform, to catch + // cases where the transforms use the stride incorrectly. + int stride = 96; + + int16_t *input_block = reinterpret_cast( + aom_memalign(16, sizeof(int16_t) * stride * height_)); + tran_low_t *output_ref_block = reinterpret_cast( + aom_memalign(16, sizeof(tran_low_t) * num_coeffs_)); + tran_low_t *output_block = reinterpret_cast( + aom_memalign(16, sizeof(tran_low_t) * num_coeffs_)); + + for (int i = 0; i < count_test_block; ++i) { + int j, k; + for (j = 0; j < height_; ++j) { + for (k = 0; k < pitch_; ++k) { + int in_idx = j * stride + k; + int out_idx = j * pitch_ + k; + input_block[in_idx] = (rnd.Rand16() & mask_) - (rnd.Rand16() & mask_); + if (bit_depth_ == AOM_BITS_8) { + output_block[out_idx] = output_ref_block[out_idx] = rnd.Rand8(); +#if CONFIG_HIGHBITDEPTH + } else { + output_block[out_idx] = output_ref_block[out_idx] = + rnd.Rand16() & mask_; +#endif + } + } + } + + fwd_txfm_ref(input_block, output_ref_block, stride, tx_type_); + ASM_REGISTER_STATE_CHECK(RunFwdTxfm(input_block, output_block, stride)); + + // The minimum quant value is 4. + for (j = 0; j < height_; ++j) { + for (k = 0; k < pitch_; ++k) { + int out_idx = j * pitch_ + k; + ASSERT_EQ(output_block[out_idx], output_ref_block[out_idx]) + << "Error: not bit-exact result at index: " << out_idx + << " at test block: " << i; + } + } + } + aom_free(input_block); + aom_free(output_ref_block); + aom_free(output_block); + } + + void RunInvCoeffCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 5000; + + // Use a stride value which is not the width of any transform, to catch + // cases where the transforms use the stride incorrectly. + int stride = 96; + + int16_t *input_block = reinterpret_cast( + aom_memalign(16, sizeof(int16_t) * num_coeffs_)); + tran_low_t *trans_block = reinterpret_cast( + aom_memalign(16, sizeof(tran_low_t) * num_coeffs_)); + uint8_t *output_block = reinterpret_cast( + aom_memalign(16, sizeof(uint8_t) * stride * height_)); + uint8_t *output_ref_block = reinterpret_cast( + aom_memalign(16, sizeof(uint8_t) * stride * height_)); + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-mask_, mask_]. + int j, k; + for (j = 0; j < height_; ++j) { + for (k = 0; k < pitch_; ++k) { + int in_idx = j * pitch_ + k; + int out_idx = j * stride + k; + input_block[in_idx] = (rnd.Rand16() & mask_) - (rnd.Rand16() & mask_); + output_ref_block[out_idx] = rnd.Rand16() & mask_; + output_block[out_idx] = output_ref_block[out_idx]; + } + } + + fwd_txfm_ref(input_block, trans_block, pitch_, tx_type_); + + inv_txfm_ref(trans_block, output_ref_block, stride, tx_type_); + ASM_REGISTER_STATE_CHECK(RunInvTxfm(trans_block, output_block, stride)); + + for (j = 0; j < height_; ++j) { + for (k = 0; k < pitch_; ++k) { + int out_idx = j * stride + k; + ASSERT_EQ(output_block[out_idx], output_ref_block[out_idx]) + << "Error: not bit-exact result at index: " << out_idx + << " j = " << j << " k = " << k << " at test block: " << i; + } + } + } + aom_free(input_block); + aom_free(trans_block); + aom_free(output_ref_block); + aom_free(output_block); + } + + void RunMemCheck() { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 5000; + + int16_t *input_extreme_block = reinterpret_cast( + aom_memalign(16, sizeof(int16_t) * num_coeffs_)); + tran_low_t *output_ref_block = reinterpret_cast( + aom_memalign(16, sizeof(tran_low_t) * num_coeffs_)); + tran_low_t *output_block = reinterpret_cast( + aom_memalign(16, sizeof(tran_low_t) * num_coeffs_)); + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-mask_, mask_]. + for (int j = 0; j < num_coeffs_; ++j) { + input_extreme_block[j] = rnd.Rand8() % 2 ? mask_ : -mask_; + } + if (i == 0) { + for (int j = 0; j < num_coeffs_; ++j) input_extreme_block[j] = mask_; + } else if (i == 1) { + for (int j = 0; j < num_coeffs_; ++j) input_extreme_block[j] = -mask_; + } + + fwd_txfm_ref(input_extreme_block, output_ref_block, pitch_, tx_type_); + ASM_REGISTER_STATE_CHECK( + RunFwdTxfm(input_extreme_block, output_block, pitch_)); + + int row_length = FindRowLength(); + // The minimum quant value is 4. + for (int j = 0; j < num_coeffs_; ++j) { + EXPECT_EQ(output_block[j], output_ref_block[j]) + << "Not bit-exact at test index: " << i << ", " + << "j = " << j << std::endl; + EXPECT_GE(row_length * kDctMaxValue << (bit_depth_ - 8), + abs(output_block[j])) + << "Error: NxN FDCT has coefficient larger than N*DCT_MAX_VALUE"; + } + } + aom_free(input_extreme_block); + aom_free(output_ref_block); + aom_free(output_block); + } + + void RunInvAccuracyCheck(int limit) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = 1000; + + int16_t *in = reinterpret_cast( + aom_memalign(16, sizeof(int16_t) * num_coeffs_)); + tran_low_t *coeff = reinterpret_cast( + aom_memalign(16, sizeof(tran_low_t) * num_coeffs_)); + uint8_t *dst = reinterpret_cast( + aom_memalign(16, sizeof(uint8_t) * num_coeffs_)); + uint8_t *src = reinterpret_cast( + aom_memalign(16, sizeof(uint8_t) * num_coeffs_)); + +#if CONFIG_HIGHBITDEPTH + uint16_t *dst16 = reinterpret_cast( + aom_memalign(16, sizeof(uint16_t) * num_coeffs_)); + uint16_t *src16 = reinterpret_cast( + aom_memalign(16, sizeof(uint16_t) * num_coeffs_)); +#endif + + for (int i = 0; i < count_test_block; ++i) { + // Initialize a test block with input range [-mask_, mask_]. + for (int j = 0; j < num_coeffs_; ++j) { + if (bit_depth_ == AOM_BITS_8) { + src[j] = rnd.Rand8(); + dst[j] = rnd.Rand8(); + in[j] = src[j] - dst[j]; +#if CONFIG_HIGHBITDEPTH + } else { + src16[j] = rnd.Rand16() & mask_; + dst16[j] = rnd.Rand16() & mask_; + in[j] = src16[j] - dst16[j]; +#endif + } + } + + fwd_txfm_ref(in, coeff, pitch_, tx_type_); + + if (bit_depth_ == AOM_BITS_8) { + ASM_REGISTER_STATE_CHECK(RunInvTxfm(coeff, dst, pitch_)); +#if CONFIG_HIGHBITDEPTH + } else { + ASM_REGISTER_STATE_CHECK( + RunInvTxfm(coeff, CONVERT_TO_BYTEPTR(dst16), pitch_)); +#endif + } + + for (int j = 0; j < num_coeffs_; ++j) { +#if CONFIG_HIGHBITDEPTH + const int diff = + bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; +#else + const int diff = dst[j] - src[j]; +#endif + const uint32_t error = diff * diff; + EXPECT_GE(static_cast(limit), error) + << "Error: 4x4 IDCT has error " << error << " at index " << j; + } + } + aom_free(in); + aom_free(coeff); + aom_free(dst); + aom_free(src); +#if CONFIG_HIGHBITDEPTH + aom_free(src16); + aom_free(dst16); +#endif + } + + int pitch_; + int height_; + int tx_type_; + FhtFunc fwd_txfm_ref; + IhtFunc inv_txfm_ref; + aom_bit_depth_t bit_depth_; + int mask_; + int num_coeffs_; + + private: + // Assume transform size is 4x4, 8x8, 16x16,... + int FindRowLength() const { + int row = 4; + if (16 == num_coeffs_) { + row = 4; + } else if (64 == num_coeffs_) { + row = 8; + } else if (256 == num_coeffs_) { + row = 16; + } else if (1024 == num_coeffs_) { + row = 32; + } + return row; + } +}; + +} // namespace libaom_test + +#endif // TEST_TRANSFORM_TEST_BASE_H_ diff --git a/third_party/aom/test/twopass_encoder.sh b/third_party/aom/test/twopass_encoder.sh new file mode 100755 index 0000000000..3abb7628b3 --- /dev/null +++ b/third_party/aom/test/twopass_encoder.sh @@ -0,0 +1,54 @@ +#!/bin/sh +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This file tests the libaom twopass_encoder example. To add new tests to this +## file, do the following: +## 1. Write a shell function (this is your test). +## 2. Add the function to twopass_encoder_tests (on a new line). +## +. $(dirname $0)/tools_common.sh + +# Environment check: $YUV_RAW_INPUT is required. +twopass_encoder_verify_environment() { + if [ ! -e "${YUV_RAW_INPUT}" ]; then + echo "Libaom test data must exist in LIBAOM_TEST_DATA_PATH." + return 1 + fi +} + +# Runs twopass_encoder using the codec specified by $1 with a frame limit of +# 100. +twopass_encoder() { + local encoder="${LIBAOM_BIN_PATH}/twopass_encoder${AOM_TEST_EXE_SUFFIX}" + local codec="$1" + local output_file="${AOM_TEST_OUTPUT_DIR}/twopass_encoder_${codec}.ivf" + local limit=7 + + if [ ! -x "${encoder}" ]; then + elog "${encoder} does not exist or is not executable." + return 1 + fi + + eval "${AOM_TEST_PREFIX}" "${encoder}" "${codec}" "${YUV_RAW_INPUT_WIDTH}" \ + "${YUV_RAW_INPUT_HEIGHT}" "${YUV_RAW_INPUT}" "${output_file}" "${limit}" \ + ${devnull} + + [ -e "${output_file}" ] || return 1 +} + +twopass_encoder_av1() { + if [ "$(av1_encode_available)" = "yes" ]; then + twopass_encoder av1 || return 1 + fi +} + +twopass_encoder_tests="twopass_encoder_av1" + +run_tests twopass_encoder_verify_environment "${twopass_encoder_tests}" diff --git a/third_party/aom/test/user_priv_test.cc b/third_party/aom/test/user_priv_test.cc new file mode 100644 index 0000000000..3052b27b1c --- /dev/null +++ b/third_party/aom/test/user_priv_test.cc @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "./aom_config.h" +#include "test/acm_random.h" +#include "test/codec_factory.h" +#include "test/decode_test_driver.h" +#include "test/ivf_video_source.h" +#include "test/md5_helper.h" +#include "test/util.h" +#if CONFIG_WEBM_IO +#include "test/webm_video_source.h" +#endif +#include "aom_mem/aom_mem.h" +#include "aom/aom.h" + +namespace { + +using std::string; +using libaom_test::ACMRandom; + +#if CONFIG_WEBM_IO + +void CheckUserPrivateData(void *user_priv, int *target) { + // actual pointer value should be the same as expected. + EXPECT_EQ(reinterpret_cast(target), user_priv) + << "user_priv pointer value does not match."; +} + +// Decodes |filename|. Passes in user_priv data when calling DecodeFrame and +// compares the user_priv from return img with the original user_priv to see if +// they match. Both the pointer values and the values inside the addresses +// should match. +string DecodeFile(const string &filename) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + libaom_test::WebMVideoSource video(filename); + video.Init(); + + aom_codec_dec_cfg_t cfg = aom_codec_dec_cfg_t(); + libaom_test::AV1Decoder decoder(cfg, 0); + + libaom_test::MD5 md5; + int frame_num = 0; + for (video.Begin(); !::testing::Test::HasFailure() && video.cxdata(); + video.Next()) { + void *user_priv = reinterpret_cast(&frame_num); + const aom_codec_err_t res = + decoder.DecodeFrame(video.cxdata(), video.frame_size(), + (frame_num == 0) ? NULL : user_priv); + if (res != AOM_CODEC_OK) { + EXPECT_EQ(AOM_CODEC_OK, res) << decoder.DecodeError(); + break; + } + libaom_test::DxDataIterator dec_iter = decoder.GetDxData(); + const aom_image_t *img = NULL; + + // Get decompressed data. + while ((img = dec_iter.Next())) { + if (frame_num == 0) { + CheckUserPrivateData(img->user_priv, NULL); + } else { + CheckUserPrivateData(img->user_priv, &frame_num); + + // Also test ctrl_get_reference api. + struct av1_ref_frame ref; + // Randomly fetch a reference frame. + ref.idx = rnd.Rand8() % 3; + decoder.Control(AV1_GET_REFERENCE, &ref); + + CheckUserPrivateData(ref.img.user_priv, NULL); + } + md5.Add(img); + } + + frame_num++; + } + return string(md5.Get()); +} + +TEST(UserPrivTest, VideoDecode) { + // no tiles or frame parallel; this exercises the decoding to test the + // user_priv. + EXPECT_STREQ("b35a1b707b28e82be025d960aba039bc", + DecodeFile("av10-2-03-size-226x226.webm").c_str()); +} + +#endif // CONFIG_WEBM_IO + +} // namespace diff --git a/third_party/aom/test/util.h b/third_party/aom/test/util.h new file mode 100644 index 0000000000..a20fab65cc --- /dev/null +++ b/third_party/aom/test/util.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef TEST_UTIL_H_ +#define TEST_UTIL_H_ + +#include +#include +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "aom/aom_image.h" + +// Macros +#define GET_PARAM(k) std::tr1::get(GetParam()) + +inline double compute_psnr(const aom_image_t *img1, const aom_image_t *img2) { + assert((img1->fmt == img2->fmt) && (img1->d_w == img2->d_w) && + (img1->d_h == img2->d_h)); + + const unsigned int width_y = img1->d_w; + const unsigned int height_y = img1->d_h; + unsigned int i, j; + + int64_t sqrerr = 0; + for (i = 0; i < height_y; ++i) + for (j = 0; j < width_y; ++j) { + int64_t d = img1->planes[AOM_PLANE_Y][i * img1->stride[AOM_PLANE_Y] + j] - + img2->planes[AOM_PLANE_Y][i * img2->stride[AOM_PLANE_Y] + j]; + sqrerr += d * d; + } + double mse = static_cast(sqrerr) / (width_y * height_y); + double psnr = 100.0; + if (mse > 0.0) { + psnr = 10 * log10(255.0 * 255.0 / mse); + } + return psnr; +} + +#endif // TEST_UTIL_H_ diff --git a/third_party/aom/test/variance_test.cc b/third_party/aom/test/variance_test.cc new file mode 100644 index 0000000000..5b1003ca7b --- /dev/null +++ b/third_party/aom/test/variance_test.cc @@ -0,0 +1,1385 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./aom_dsp_rtcd.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "aom/aom_codec.h" +#include "aom/aom_integer.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/mem.h" + +namespace { + +typedef unsigned int (*VarianceMxNFunc)(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + unsigned int *sse); +typedef unsigned int (*SubpixVarMxNFunc)(const uint8_t *a, int a_stride, + int xoffset, int yoffset, + const uint8_t *b, int b_stride, + unsigned int *sse); +typedef unsigned int (*SubpixAvgVarMxNFunc)(const uint8_t *a, int a_stride, + int xoffset, int yoffset, + const uint8_t *b, int b_stride, + uint32_t *sse, + const uint8_t *second_pred); +typedef unsigned int (*Get4x4SseFunc)(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride); +typedef unsigned int (*SumOfSquaresFunction)(const int16_t *src); + +using libaom_test::ACMRandom; + +// Truncate high bit depth results by downshifting (with rounding) by: +// 2 * (bit_depth - 8) for sse +// (bit_depth - 8) for se +static void RoundHighBitDepth(int bit_depth, int64_t *se, uint64_t *sse) { + switch (bit_depth) { + case AOM_BITS_12: + *sse = (*sse + 128) >> 8; + *se = (*se + 8) >> 4; + break; + case AOM_BITS_10: + *sse = (*sse + 8) >> 4; + *se = (*se + 2) >> 2; + break; + case AOM_BITS_8: + default: break; + } +} + +static unsigned int mb_ss_ref(const int16_t *src) { + unsigned int res = 0; + for (int i = 0; i < 256; ++i) { + res += src[i] * src[i]; + } + return res; +} + +/* Note: + * Our codebase calculates the "diff" value in the variance algorithm by + * (src - ref). + */ +static uint32_t variance_ref(const uint8_t *src, const uint8_t *ref, int l2w, + int l2h, int src_stride, int ref_stride, + uint32_t *sse_ptr, bool use_high_bit_depth_, + aom_bit_depth_t bit_depth) { + int64_t se = 0; + uint64_t sse = 0; + const int w = 1 << l2w; + const int h = 1 << l2h; + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + int diff; + if (!use_high_bit_depth_) { + diff = src[y * src_stride + x] - ref[y * ref_stride + x]; + se += diff; + sse += diff * diff; +#if CONFIG_HIGHBITDEPTH + } else { + diff = CONVERT_TO_SHORTPTR(src)[y * src_stride + x] - + CONVERT_TO_SHORTPTR(ref)[y * ref_stride + x]; + se += diff; + sse += diff * diff; +#endif // CONFIG_HIGHBITDEPTH + } + } + } + RoundHighBitDepth(bit_depth, &se, &sse); + *sse_ptr = static_cast(sse); + return static_cast(sse - ((se * se) >> (l2w + l2h))); +} + +/* The subpel reference functions differ from the codec version in one aspect: + * they calculate the bilinear factors directly instead of using a lookup table + * and therefore upshift xoff and yoff by 1. Only every other calculated value + * is used so the codec version shrinks the table to save space and maintain + * compatibility with vp8. + */ +static uint32_t subpel_variance_ref(const uint8_t *ref, const uint8_t *src, + int l2w, int l2h, int xoff, int yoff, + uint32_t *sse_ptr, bool use_high_bit_depth_, + aom_bit_depth_t bit_depth) { + int64_t se = 0; + uint64_t sse = 0; + const int w = 1 << l2w; + const int h = 1 << l2h; + + xoff <<= 1; + yoff <<= 1; + + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + // Bilinear interpolation at a 16th pel step. + if (!use_high_bit_depth_) { + const int a1 = ref[(w + 1) * (y + 0) + x + 0]; + const int a2 = ref[(w + 1) * (y + 0) + x + 1]; + const int b1 = ref[(w + 1) * (y + 1) + x + 0]; + const int b2 = ref[(w + 1) * (y + 1) + x + 1]; + const int a = a1 + (((a2 - a1) * xoff + 8) >> 4); + const int b = b1 + (((b2 - b1) * xoff + 8) >> 4); + const int r = a + (((b - a) * yoff + 8) >> 4); + const int diff = r - src[w * y + x]; + se += diff; + sse += diff * diff; +#if CONFIG_HIGHBITDEPTH + } else { + uint16_t *ref16 = CONVERT_TO_SHORTPTR(ref); + uint16_t *src16 = CONVERT_TO_SHORTPTR(src); + const int a1 = ref16[(w + 1) * (y + 0) + x + 0]; + const int a2 = ref16[(w + 1) * (y + 0) + x + 1]; + const int b1 = ref16[(w + 1) * (y + 1) + x + 0]; + const int b2 = ref16[(w + 1) * (y + 1) + x + 1]; + const int a = a1 + (((a2 - a1) * xoff + 8) >> 4); + const int b = b1 + (((b2 - b1) * xoff + 8) >> 4); + const int r = a + (((b - a) * yoff + 8) >> 4); + const int diff = r - src16[w * y + x]; + se += diff; + sse += diff * diff; +#endif // CONFIG_HIGHBITDEPTH + } + } + } + RoundHighBitDepth(bit_depth, &se, &sse); + *sse_ptr = static_cast(sse); + return static_cast(sse - ((se * se) >> (l2w + l2h))); +} + +static uint32_t subpel_avg_variance_ref(const uint8_t *ref, const uint8_t *src, + const uint8_t *second_pred, int l2w, + int l2h, int xoff, int yoff, + uint32_t *sse_ptr, + bool use_high_bit_depth, + aom_bit_depth_t bit_depth) { + int64_t se = 0; + uint64_t sse = 0; + const int w = 1 << l2w; + const int h = 1 << l2h; + + xoff <<= 1; + yoff <<= 1; + + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + // bilinear interpolation at a 16th pel step + if (!use_high_bit_depth) { + const int a1 = ref[(w + 1) * (y + 0) + x + 0]; + const int a2 = ref[(w + 1) * (y + 0) + x + 1]; + const int b1 = ref[(w + 1) * (y + 1) + x + 0]; + const int b2 = ref[(w + 1) * (y + 1) + x + 1]; + const int a = a1 + (((a2 - a1) * xoff + 8) >> 4); + const int b = b1 + (((b2 - b1) * xoff + 8) >> 4); + const int r = a + (((b - a) * yoff + 8) >> 4); + const int diff = + ((r + second_pred[w * y + x] + 1) >> 1) - src[w * y + x]; + se += diff; + sse += diff * diff; +#if CONFIG_HIGHBITDEPTH + } else { + const uint16_t *ref16 = CONVERT_TO_SHORTPTR(ref); + const uint16_t *src16 = CONVERT_TO_SHORTPTR(src); + const uint16_t *sec16 = CONVERT_TO_SHORTPTR(second_pred); + const int a1 = ref16[(w + 1) * (y + 0) + x + 0]; + const int a2 = ref16[(w + 1) * (y + 0) + x + 1]; + const int b1 = ref16[(w + 1) * (y + 1) + x + 0]; + const int b2 = ref16[(w + 1) * (y + 1) + x + 1]; + const int a = a1 + (((a2 - a1) * xoff + 8) >> 4); + const int b = b1 + (((b2 - b1) * xoff + 8) >> 4); + const int r = a + (((b - a) * yoff + 8) >> 4); + const int diff = ((r + sec16[w * y + x] + 1) >> 1) - src16[w * y + x]; + se += diff; + sse += diff * diff; +#endif // CONFIG_HIGHBITDEPTH + } + } + } + RoundHighBitDepth(bit_depth, &se, &sse); + *sse_ptr = static_cast(sse); + return static_cast(sse - ((se * se) >> (l2w + l2h))); +} + +//////////////////////////////////////////////////////////////////////////////// + +class SumOfSquaresTest : public ::testing::TestWithParam { + public: + SumOfSquaresTest() : func_(GetParam()) {} + + virtual ~SumOfSquaresTest() { libaom_test::ClearSystemState(); } + + protected: + void ConstTest(); + void RefTest(); + + SumOfSquaresFunction func_; + ACMRandom rnd_; +}; + +void SumOfSquaresTest::ConstTest() { + int16_t mem[256]; + unsigned int res; + for (int v = 0; v < 256; ++v) { + for (int i = 0; i < 256; ++i) { + mem[i] = v; + } + ASM_REGISTER_STATE_CHECK(res = func_(mem)); + EXPECT_EQ(256u * (v * v), res); + } +} + +void SumOfSquaresTest::RefTest() { + int16_t mem[256]; + for (int i = 0; i < 100; ++i) { + for (int j = 0; j < 256; ++j) { + mem[j] = rnd_.Rand8() - rnd_.Rand8(); + } + + const unsigned int expected = mb_ss_ref(mem); + unsigned int res; + ASM_REGISTER_STATE_CHECK(res = func_(mem)); + EXPECT_EQ(expected, res); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Encapsulating struct to store the function to test along with +// some testing context. +// Can be used for MSE, SSE, Variance, etc. + +template +struct TestParams { + TestParams(int log2w = 0, int log2h = 0, Func function = NULL, + int bit_depth_value = 0) + : log2width(log2w), log2height(log2h), func(function) { + use_high_bit_depth = (bit_depth_value > 0); + if (use_high_bit_depth) { + bit_depth = static_cast(bit_depth_value); + } else { + bit_depth = AOM_BITS_8; + } + width = 1 << log2width; + height = 1 << log2height; + block_size = width * height; + mask = (1u << bit_depth) - 1; + } + + int log2width, log2height; + int width, height; + int block_size; + Func func; + aom_bit_depth_t bit_depth; + bool use_high_bit_depth; + uint32_t mask; +}; + +template +std::ostream &operator<<(std::ostream &os, const TestParams &p) { + return os << "log2width/height:" << p.log2width << "/" << p.log2height + << " function:" << reinterpret_cast(p.func) + << " bit-depth:" << p.bit_depth; +} + +// Main class for testing a function type +template +class MainTestClass + : public ::testing::TestWithParam > { + public: + virtual void SetUp() { + params_ = this->GetParam(); + + rnd_.Reset(ACMRandom::DeterministicSeed()); + const size_t unit = + use_high_bit_depth() ? sizeof(uint16_t) : sizeof(uint8_t); + src_ = reinterpret_cast(aom_memalign(16, block_size() * unit)); + ref_ = new uint8_t[block_size() * unit]; + ASSERT_TRUE(src_ != NULL); + ASSERT_TRUE(ref_ != NULL); +#if CONFIG_HIGHBITDEPTH + if (use_high_bit_depth()) { + // TODO(skal): remove! + src_ = CONVERT_TO_BYTEPTR(src_); + ref_ = CONVERT_TO_BYTEPTR(ref_); + } +#endif + } + + virtual void TearDown() { +#if CONFIG_HIGHBITDEPTH + if (use_high_bit_depth()) { + // TODO(skal): remove! + src_ = reinterpret_cast(CONVERT_TO_SHORTPTR(src_)); + ref_ = reinterpret_cast(CONVERT_TO_SHORTPTR(ref_)); + } +#endif + + aom_free(src_); + delete[] ref_; + src_ = NULL; + ref_ = NULL; + libaom_test::ClearSystemState(); + } + + protected: + // We could sub-class MainTestClass into dedicated class for Variance + // and MSE/SSE, but it involves a lot of 'this->xxx' dereferencing + // to access top class fields xxx. That's cumbersome, so for now we'll just + // implement the testing methods here: + + // Variance tests + void ZeroTest(); + void RefTest(); + void RefStrideTest(); + void OneQuarterTest(); + + // MSE/SSE tests + void RefTestMse(); + void RefTestSse(); + void MaxTestMse(); + void MaxTestSse(); + + protected: + ACMRandom rnd_; + uint8_t *src_; + uint8_t *ref_; + TestParams params_; + + // some relay helpers + bool use_high_bit_depth() const { return params_.use_high_bit_depth; } + int byte_shift() const { return params_.bit_depth - 8; } + int block_size() const { return params_.block_size; } + int width() const { return params_.width; } + uint32_t mask() const { return params_.mask; } +}; + +//////////////////////////////////////////////////////////////////////////////// +// Tests related to variance. + +template +void MainTestClass::ZeroTest() { + for (int i = 0; i <= 255; ++i) { + if (!use_high_bit_depth()) { + memset(src_, i, block_size()); + } else { + uint16_t *const src16 = CONVERT_TO_SHORTPTR(src_); + for (int k = 0; k < block_size(); ++k) src16[k] = i << byte_shift(); + } + for (int j = 0; j <= 255; ++j) { + if (!use_high_bit_depth()) { + memset(ref_, j, block_size()); + } else { + uint16_t *const ref16 = CONVERT_TO_SHORTPTR(ref_); + for (int k = 0; k < block_size(); ++k) ref16[k] = j << byte_shift(); + } + unsigned int sse, var; + ASM_REGISTER_STATE_CHECK( + var = params_.func(src_, width(), ref_, width(), &sse)); + EXPECT_EQ(0u, var) << "src values: " << i << " ref values: " << j; + } + } +} + +template +void MainTestClass::RefTest() { + for (int i = 0; i < 10; ++i) { + for (int j = 0; j < block_size(); j++) { + if (!use_high_bit_depth()) { + src_[j] = rnd_.Rand8(); + ref_[j] = rnd_.Rand8(); +#if CONFIG_HIGHBITDEPTH + } else { + CONVERT_TO_SHORTPTR(src_)[j] = rnd_.Rand16() & mask(); + CONVERT_TO_SHORTPTR(ref_)[j] = rnd_.Rand16() & mask(); +#endif // CONFIG_HIGHBITDEPTH + } + } + unsigned int sse1, sse2, var1, var2; + const int stride = width(); + ASM_REGISTER_STATE_CHECK( + var1 = params_.func(src_, stride, ref_, stride, &sse1)); + var2 = + variance_ref(src_, ref_, params_.log2width, params_.log2height, stride, + stride, &sse2, use_high_bit_depth(), params_.bit_depth); + EXPECT_EQ(sse1, sse2) << "Error at test index: " << i; + EXPECT_EQ(var1, var2) << "Error at test index: " << i; + } +} + +template +void MainTestClass::RefStrideTest() { + for (int i = 0; i < 10; ++i) { + const int ref_stride = (i & 1) * width(); + const int src_stride = ((i >> 1) & 1) * width(); + for (int j = 0; j < block_size(); j++) { + const int ref_ind = (j / width()) * ref_stride + j % width(); + const int src_ind = (j / width()) * src_stride + j % width(); + if (!use_high_bit_depth()) { + src_[src_ind] = rnd_.Rand8(); + ref_[ref_ind] = rnd_.Rand8(); +#if CONFIG_HIGHBITDEPTH + } else { + CONVERT_TO_SHORTPTR(src_)[src_ind] = rnd_.Rand16() & mask(); + CONVERT_TO_SHORTPTR(ref_)[ref_ind] = rnd_.Rand16() & mask(); +#endif // CONFIG_HIGHBITDEPTH + } + } + unsigned int sse1, sse2; + unsigned int var1, var2; + + ASM_REGISTER_STATE_CHECK( + var1 = params_.func(src_, src_stride, ref_, ref_stride, &sse1)); + var2 = variance_ref(src_, ref_, params_.log2width, params_.log2height, + src_stride, ref_stride, &sse2, use_high_bit_depth(), + params_.bit_depth); + EXPECT_EQ(sse1, sse2) << "Error at test index: " << i; + EXPECT_EQ(var1, var2) << "Error at test index: " << i; + } +} + +template +void MainTestClass::OneQuarterTest() { + const int half = block_size() / 2; + if (!use_high_bit_depth()) { + memset(src_, 255, block_size()); + memset(ref_, 255, half); + memset(ref_ + half, 0, half); +#if CONFIG_HIGHBITDEPTH + } else { + aom_memset16(CONVERT_TO_SHORTPTR(src_), 255 << byte_shift(), block_size()); + aom_memset16(CONVERT_TO_SHORTPTR(ref_), 255 << byte_shift(), half); + aom_memset16(CONVERT_TO_SHORTPTR(ref_) + half, 0, half); +#endif // CONFIG_HIGHBITDEPTH + } + unsigned int sse, var, expected; + ASM_REGISTER_STATE_CHECK( + var = params_.func(src_, width(), ref_, width(), &sse)); + expected = block_size() * 255 * 255 / 4; + EXPECT_EQ(expected, var); +} + +//////////////////////////////////////////////////////////////////////////////// +// Tests related to MSE / SSE. + +template +void MainTestClass::RefTestMse() { + for (int i = 0; i < 10; ++i) { + for (int j = 0; j < block_size(); ++j) { + src_[j] = rnd_.Rand8(); + ref_[j] = rnd_.Rand8(); + } + unsigned int sse1, sse2; + const int stride = width(); + ASM_REGISTER_STATE_CHECK(params_.func(src_, stride, ref_, stride, &sse1)); + variance_ref(src_, ref_, params_.log2width, params_.log2height, stride, + stride, &sse2, false, AOM_BITS_8); + EXPECT_EQ(sse1, sse2); + } +} + +template +void MainTestClass::RefTestSse() { + for (int i = 0; i < 10; ++i) { + for (int j = 0; j < block_size(); ++j) { + src_[j] = rnd_.Rand8(); + ref_[j] = rnd_.Rand8(); + } + unsigned int sse2; + unsigned int var1; + const int stride = width(); + ASM_REGISTER_STATE_CHECK(var1 = params_.func(src_, stride, ref_, stride)); + variance_ref(src_, ref_, params_.log2width, params_.log2height, stride, + stride, &sse2, false, AOM_BITS_8); + EXPECT_EQ(var1, sse2); + } +} + +template +void MainTestClass::MaxTestMse() { + memset(src_, 255, block_size()); + memset(ref_, 0, block_size()); + unsigned int sse; + ASM_REGISTER_STATE_CHECK(params_.func(src_, width(), ref_, width(), &sse)); + const unsigned int expected = block_size() * 255 * 255; + EXPECT_EQ(expected, sse); +} + +template +void MainTestClass::MaxTestSse() { + memset(src_, 255, block_size()); + memset(ref_, 0, block_size()); + unsigned int var; + ASM_REGISTER_STATE_CHECK(var = params_.func(src_, width(), ref_, width())); + const unsigned int expected = block_size() * 255 * 255; + EXPECT_EQ(expected, var); +} + +//////////////////////////////////////////////////////////////////////////////// + +using ::std::tr1::get; +using ::std::tr1::make_tuple; +using ::std::tr1::tuple; + +template +class SubpelVarianceTest + : public ::testing::TestWithParam< + tuple > { + public: + virtual void SetUp() { + const tuple ¶ms = + this->GetParam(); + log2width_ = get<0>(params); + width_ = 1 << log2width_; + log2height_ = get<1>(params); + height_ = 1 << log2height_; + subpel_variance_ = get<2>(params); + if (get<3>(params)) { + bit_depth_ = (aom_bit_depth_t)get<3>(params); + use_high_bit_depth_ = true; + } else { + bit_depth_ = AOM_BITS_8; + use_high_bit_depth_ = false; + } + mask_ = (1 << bit_depth_) - 1; + + rnd_.Reset(ACMRandom::DeterministicSeed()); + block_size_ = width_ * height_; + if (!use_high_bit_depth_) { + src_ = reinterpret_cast(aom_memalign(16, block_size_)); + sec_ = reinterpret_cast(aom_memalign(16, block_size_)); + ref_ = new uint8_t[block_size_ + width_ + height_ + 1]; +#if CONFIG_HIGHBITDEPTH + } else { + src_ = CONVERT_TO_BYTEPTR(reinterpret_cast( + aom_memalign(16, block_size_ * sizeof(uint16_t)))); + sec_ = CONVERT_TO_BYTEPTR(reinterpret_cast( + aom_memalign(16, block_size_ * sizeof(uint16_t)))); + ref_ = + CONVERT_TO_BYTEPTR(new uint16_t[block_size_ + width_ + height_ + 1]); +#endif // CONFIG_HIGHBITDEPTH + } + ASSERT_TRUE(src_ != NULL); + ASSERT_TRUE(sec_ != NULL); + ASSERT_TRUE(ref_ != NULL); + } + + virtual void TearDown() { + if (!use_high_bit_depth_) { + aom_free(src_); + delete[] ref_; + aom_free(sec_); +#if CONFIG_HIGHBITDEPTH + } else { + aom_free(CONVERT_TO_SHORTPTR(src_)); + delete[] CONVERT_TO_SHORTPTR(ref_); + aom_free(CONVERT_TO_SHORTPTR(sec_)); +#endif // CONFIG_HIGHBITDEPTH + } + libaom_test::ClearSystemState(); + } + + protected: + void RefTest(); + void ExtremeRefTest(); + + ACMRandom rnd_; + uint8_t *src_; + uint8_t *ref_; + uint8_t *sec_; + bool use_high_bit_depth_; + aom_bit_depth_t bit_depth_; + int width_, log2width_; + int height_, log2height_; + int block_size_, mask_; + SubpelVarianceFunctionType subpel_variance_; +}; + +template +void SubpelVarianceTest::RefTest() { + for (int x = 0; x < 8; ++x) { + for (int y = 0; y < 8; ++y) { + if (!use_high_bit_depth_) { + for (int j = 0; j < block_size_; j++) { + src_[j] = rnd_.Rand8(); + } + for (int j = 0; j < block_size_ + width_ + height_ + 1; j++) { + ref_[j] = rnd_.Rand8(); + } +#if CONFIG_HIGHBITDEPTH + } else { + for (int j = 0; j < block_size_; j++) { + CONVERT_TO_SHORTPTR(src_)[j] = rnd_.Rand16() & mask_; + } + for (int j = 0; j < block_size_ + width_ + height_ + 1; j++) { + CONVERT_TO_SHORTPTR(ref_)[j] = rnd_.Rand16() & mask_; + } +#endif // CONFIG_HIGHBITDEPTH + } + unsigned int sse1, sse2; + unsigned int var1; + ASM_REGISTER_STATE_CHECK( + var1 = subpel_variance_(ref_, width_ + 1, x, y, src_, width_, &sse1)); + const unsigned int var2 = + subpel_variance_ref(ref_, src_, log2width_, log2height_, x, y, &sse2, + use_high_bit_depth_, bit_depth_); + EXPECT_EQ(sse1, sse2) << "at position " << x << ", " << y; + EXPECT_EQ(var1, var2) << "at position " << x << ", " << y; + } + } +} + +template +void SubpelVarianceTest::ExtremeRefTest() { + // Compare against reference. + // Src: Set the first half of values to 0, the second half to the maximum. + // Ref: Set the first half of values to the maximum, the second half to 0. + for (int x = 0; x < 8; ++x) { + for (int y = 0; y < 8; ++y) { + const int half = block_size_ / 2; + if (!use_high_bit_depth_) { + memset(src_, 0, half); + memset(src_ + half, 255, half); + memset(ref_, 255, half); + memset(ref_ + half, 0, half + width_ + height_ + 1); +#if CONFIG_HIGHBITDEPTH + } else { + aom_memset16(CONVERT_TO_SHORTPTR(src_), mask_, half); + aom_memset16(CONVERT_TO_SHORTPTR(src_) + half, 0, half); + aom_memset16(CONVERT_TO_SHORTPTR(ref_), 0, half); + aom_memset16(CONVERT_TO_SHORTPTR(ref_) + half, mask_, + half + width_ + height_ + 1); +#endif // CONFIG_HIGHBITDEPTH + } + unsigned int sse1, sse2; + unsigned int var1; + ASM_REGISTER_STATE_CHECK( + var1 = subpel_variance_(ref_, width_ + 1, x, y, src_, width_, &sse1)); + const unsigned int var2 = + subpel_variance_ref(ref_, src_, log2width_, log2height_, x, y, &sse2, + use_high_bit_depth_, bit_depth_); + EXPECT_EQ(sse1, sse2) << "for xoffset " << x << " and yoffset " << y; + EXPECT_EQ(var1, var2) << "for xoffset " << x << " and yoffset " << y; + } + } +} + +template <> +void SubpelVarianceTest::RefTest() { + for (int x = 0; x < 8; ++x) { + for (int y = 0; y < 8; ++y) { + if (!use_high_bit_depth_) { + for (int j = 0; j < block_size_; j++) { + src_[j] = rnd_.Rand8(); + sec_[j] = rnd_.Rand8(); + } + for (int j = 0; j < block_size_ + width_ + height_ + 1; j++) { + ref_[j] = rnd_.Rand8(); + } +#if CONFIG_HIGHBITDEPTH + } else { + for (int j = 0; j < block_size_; j++) { + CONVERT_TO_SHORTPTR(src_)[j] = rnd_.Rand16() & mask_; + CONVERT_TO_SHORTPTR(sec_)[j] = rnd_.Rand16() & mask_; + } + for (int j = 0; j < block_size_ + width_ + height_ + 1; j++) { + CONVERT_TO_SHORTPTR(ref_)[j] = rnd_.Rand16() & mask_; + } +#endif // CONFIG_HIGHBITDEPTH + } + uint32_t sse1, sse2; + uint32_t var1, var2; + ASM_REGISTER_STATE_CHECK(var1 = + subpel_variance_(ref_, width_ + 1, x, y, + src_, width_, &sse1, sec_)); + var2 = subpel_avg_variance_ref(ref_, src_, sec_, log2width_, log2height_, + x, y, &sse2, use_high_bit_depth_, + static_cast(bit_depth_)); + EXPECT_EQ(sse1, sse2) << "at position " << x << ", " << y; + EXPECT_EQ(var1, var2) << "at position " << x << ", " << y; + } + } +} + +typedef MainTestClass AvxSseTest; +typedef MainTestClass AvxMseTest; +typedef MainTestClass AvxVarianceTest; +typedef SubpelVarianceTest AvxSubpelVarianceTest; +typedef SubpelVarianceTest AvxSubpelAvgVarianceTest; + +TEST_P(AvxSseTest, RefSse) { RefTestSse(); } +TEST_P(AvxSseTest, MaxSse) { MaxTestSse(); } +TEST_P(AvxMseTest, RefMse) { RefTestMse(); } +TEST_P(AvxMseTest, MaxMse) { MaxTestMse(); } +TEST_P(AvxVarianceTest, Zero) { ZeroTest(); } +TEST_P(AvxVarianceTest, Ref) { RefTest(); } +TEST_P(AvxVarianceTest, RefStride) { RefStrideTest(); } +TEST_P(AvxVarianceTest, OneQuarter) { OneQuarterTest(); } +TEST_P(SumOfSquaresTest, Const) { ConstTest(); } +TEST_P(SumOfSquaresTest, Ref) { RefTest(); } +TEST_P(AvxSubpelVarianceTest, Ref) { RefTest(); } +TEST_P(AvxSubpelVarianceTest, ExtremeRef) { ExtremeRefTest(); } +TEST_P(AvxSubpelAvgVarianceTest, Ref) { RefTest(); } + +INSTANTIATE_TEST_CASE_P(C, SumOfSquaresTest, + ::testing::Values(aom_get_mb_ss_c)); + +typedef TestParams SseParams; +INSTANTIATE_TEST_CASE_P(C, AvxSseTest, + ::testing::Values(SseParams(2, 2, + &aom_get4x4sse_cs_c))); + +typedef TestParams MseParams; +INSTANTIATE_TEST_CASE_P(C, AvxMseTest, + ::testing::Values(MseParams(4, 4, &aom_mse16x16_c), + MseParams(4, 3, &aom_mse16x8_c), + MseParams(3, 4, &aom_mse8x16_c), + MseParams(3, 3, &aom_mse8x8_c))); + +typedef TestParams VarianceParams; +INSTANTIATE_TEST_CASE_P( + C, AvxVarianceTest, + ::testing::Values(VarianceParams(6, 6, &aom_variance64x64_c), + VarianceParams(6, 5, &aom_variance64x32_c), + VarianceParams(5, 6, &aom_variance32x64_c), + VarianceParams(5, 5, &aom_variance32x32_c), + VarianceParams(5, 4, &aom_variance32x16_c), + VarianceParams(4, 5, &aom_variance16x32_c), + VarianceParams(4, 4, &aom_variance16x16_c), + VarianceParams(4, 3, &aom_variance16x8_c), + VarianceParams(3, 4, &aom_variance8x16_c), + VarianceParams(3, 3, &aom_variance8x8_c), + VarianceParams(3, 2, &aom_variance8x4_c), + VarianceParams(2, 3, &aom_variance4x8_c), + VarianceParams(2, 2, &aom_variance4x4_c))); + +INSTANTIATE_TEST_CASE_P( + C, AvxSubpelVarianceTest, + ::testing::Values(make_tuple(6, 6, &aom_sub_pixel_variance64x64_c, 0), + make_tuple(6, 5, &aom_sub_pixel_variance64x32_c, 0), + make_tuple(5, 6, &aom_sub_pixel_variance32x64_c, 0), + make_tuple(5, 5, &aom_sub_pixel_variance32x32_c, 0), + make_tuple(5, 4, &aom_sub_pixel_variance32x16_c, 0), + make_tuple(4, 5, &aom_sub_pixel_variance16x32_c, 0), + make_tuple(4, 4, &aom_sub_pixel_variance16x16_c, 0), + make_tuple(4, 3, &aom_sub_pixel_variance16x8_c, 0), + make_tuple(3, 4, &aom_sub_pixel_variance8x16_c, 0), + make_tuple(3, 3, &aom_sub_pixel_variance8x8_c, 0), + make_tuple(3, 2, &aom_sub_pixel_variance8x4_c, 0), + make_tuple(2, 3, &aom_sub_pixel_variance4x8_c, 0), + make_tuple(2, 2, &aom_sub_pixel_variance4x4_c, 0))); + +INSTANTIATE_TEST_CASE_P( + C, AvxSubpelAvgVarianceTest, + ::testing::Values(make_tuple(6, 6, &aom_sub_pixel_avg_variance64x64_c, 0), + make_tuple(6, 5, &aom_sub_pixel_avg_variance64x32_c, 0), + make_tuple(5, 6, &aom_sub_pixel_avg_variance32x64_c, 0), + make_tuple(5, 5, &aom_sub_pixel_avg_variance32x32_c, 0), + make_tuple(5, 4, &aom_sub_pixel_avg_variance32x16_c, 0), + make_tuple(4, 5, &aom_sub_pixel_avg_variance16x32_c, 0), + make_tuple(4, 4, &aom_sub_pixel_avg_variance16x16_c, 0), + make_tuple(4, 3, &aom_sub_pixel_avg_variance16x8_c, 0), + make_tuple(3, 4, &aom_sub_pixel_avg_variance8x16_c, 0), + make_tuple(3, 3, &aom_sub_pixel_avg_variance8x8_c, 0), + make_tuple(3, 2, &aom_sub_pixel_avg_variance8x4_c, 0), + make_tuple(2, 3, &aom_sub_pixel_avg_variance4x8_c, 0), + make_tuple(2, 2, &aom_sub_pixel_avg_variance4x4_c, 0))); + +#if CONFIG_HIGHBITDEPTH +typedef MainTestClass AvxHBDMseTest; +typedef MainTestClass AvxHBDVarianceTest; +typedef SubpelVarianceTest AvxHBDSubpelVarianceTest; +typedef SubpelVarianceTest AvxHBDSubpelAvgVarianceTest; + +TEST_P(AvxHBDMseTest, RefMse) { RefTestMse(); } +TEST_P(AvxHBDMseTest, MaxMse) { MaxTestMse(); } +TEST_P(AvxHBDVarianceTest, Zero) { ZeroTest(); } +TEST_P(AvxHBDVarianceTest, Ref) { RefTest(); } +TEST_P(AvxHBDVarianceTest, RefStride) { RefStrideTest(); } +TEST_P(AvxHBDVarianceTest, OneQuarter) { OneQuarterTest(); } +TEST_P(AvxHBDSubpelVarianceTest, Ref) { RefTest(); } +TEST_P(AvxHBDSubpelVarianceTest, ExtremeRef) { ExtremeRefTest(); } +TEST_P(AvxHBDSubpelAvgVarianceTest, Ref) { RefTest(); } + +/* TODO(debargha): This test does not support the highbd version +INSTANTIATE_TEST_CASE_P( + C, AvxHBDMseTest, + ::testing::Values(make_tuple(4, 4, &aom_highbd_12_mse16x16_c), + make_tuple(4, 4, &aom_highbd_12_mse16x8_c), + make_tuple(4, 4, &aom_highbd_12_mse8x16_c), + make_tuple(4, 4, &aom_highbd_12_mse8x8_c), + make_tuple(4, 4, &aom_highbd_10_mse16x16_c), + make_tuple(4, 4, &aom_highbd_10_mse16x8_c), + make_tuple(4, 4, &aom_highbd_10_mse8x16_c), + make_tuple(4, 4, &aom_highbd_10_mse8x8_c), + make_tuple(4, 4, &aom_highbd_8_mse16x16_c), + make_tuple(4, 4, &aom_highbd_8_mse16x8_c), + make_tuple(4, 4, &aom_highbd_8_mse8x16_c), + make_tuple(4, 4, &aom_highbd_8_mse8x8_c))); +*/ + +const VarianceParams kArrayHBDVariance_c[] = { +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + VarianceParams(7, 7, &aom_highbd_12_variance128x128_c, 12), + VarianceParams(7, 6, &aom_highbd_12_variance128x64_c, 12), + VarianceParams(6, 7, &aom_highbd_12_variance64x128_c, 12), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + VarianceParams(6, 6, &aom_highbd_12_variance64x64_c, 12), + VarianceParams(6, 5, &aom_highbd_12_variance64x32_c, 12), + VarianceParams(5, 6, &aom_highbd_12_variance32x64_c, 12), + VarianceParams(5, 5, &aom_highbd_12_variance32x32_c, 12), + VarianceParams(5, 4, &aom_highbd_12_variance32x16_c, 12), + VarianceParams(4, 5, &aom_highbd_12_variance16x32_c, 12), + VarianceParams(4, 4, &aom_highbd_12_variance16x16_c, 12), + VarianceParams(4, 3, &aom_highbd_12_variance16x8_c, 12), + VarianceParams(3, 4, &aom_highbd_12_variance8x16_c, 12), + VarianceParams(3, 3, &aom_highbd_12_variance8x8_c, 12), + VarianceParams(3, 2, &aom_highbd_12_variance8x4_c, 12), + VarianceParams(2, 3, &aom_highbd_12_variance4x8_c, 12), + VarianceParams(2, 2, &aom_highbd_12_variance4x4_c, 12), +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + VarianceParams(7, 7, &aom_highbd_10_variance128x128_c, 10), + VarianceParams(7, 6, &aom_highbd_10_variance128x64_c, 10), + VarianceParams(6, 7, &aom_highbd_10_variance64x128_c, 10), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + VarianceParams(6, 6, &aom_highbd_10_variance64x64_c, 10), + VarianceParams(6, 5, &aom_highbd_10_variance64x32_c, 10), + VarianceParams(5, 6, &aom_highbd_10_variance32x64_c, 10), + VarianceParams(5, 5, &aom_highbd_10_variance32x32_c, 10), + VarianceParams(5, 4, &aom_highbd_10_variance32x16_c, 10), + VarianceParams(4, 5, &aom_highbd_10_variance16x32_c, 10), + VarianceParams(4, 4, &aom_highbd_10_variance16x16_c, 10), + VarianceParams(4, 3, &aom_highbd_10_variance16x8_c, 10), + VarianceParams(3, 4, &aom_highbd_10_variance8x16_c, 10), + VarianceParams(3, 3, &aom_highbd_10_variance8x8_c, 10), + VarianceParams(3, 2, &aom_highbd_10_variance8x4_c, 10), + VarianceParams(2, 3, &aom_highbd_10_variance4x8_c, 10), + VarianceParams(2, 2, &aom_highbd_10_variance4x4_c, 10), +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + VarianceParams(7, 7, &aom_highbd_8_variance128x128_c, 8), + VarianceParams(7, 6, &aom_highbd_8_variance128x64_c, 8), + VarianceParams(6, 7, &aom_highbd_8_variance64x128_c, 8), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + VarianceParams(6, 6, &aom_highbd_8_variance64x64_c, 8), + VarianceParams(6, 5, &aom_highbd_8_variance64x32_c, 8), + VarianceParams(5, 6, &aom_highbd_8_variance32x64_c, 8), + VarianceParams(5, 5, &aom_highbd_8_variance32x32_c, 8), + VarianceParams(5, 4, &aom_highbd_8_variance32x16_c, 8), + VarianceParams(4, 5, &aom_highbd_8_variance16x32_c, 8), + VarianceParams(4, 4, &aom_highbd_8_variance16x16_c, 8), + VarianceParams(4, 3, &aom_highbd_8_variance16x8_c, 8), + VarianceParams(3, 4, &aom_highbd_8_variance8x16_c, 8), + VarianceParams(3, 3, &aom_highbd_8_variance8x8_c, 8), + VarianceParams(3, 2, &aom_highbd_8_variance8x4_c, 8), + VarianceParams(2, 3, &aom_highbd_8_variance4x8_c, 8), + VarianceParams(2, 2, &aom_highbd_8_variance4x4_c, 8) +}; +INSTANTIATE_TEST_CASE_P(C, AvxHBDVarianceTest, + ::testing::ValuesIn(kArrayHBDVariance_c)); + +#if HAVE_SSE4_1 && CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSE4_1, AvxHBDVarianceTest, + ::testing::Values( + VarianceParams(2, 2, &aom_highbd_8_variance4x4_sse4_1, 8), + VarianceParams(2, 2, &aom_highbd_10_variance4x4_sse4_1, 10), + VarianceParams(2, 2, &aom_highbd_12_variance4x4_sse4_1, 12))); +#endif // HAVE_SSE4_1 && CONFIG_HIGHBITDEPTH + +const AvxHBDSubpelVarianceTest::ParamType kArrayHBDSubpelVariance_c[] = { +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(7, 7, &aom_highbd_8_sub_pixel_variance128x128_c, 8), + make_tuple(7, 6, &aom_highbd_8_sub_pixel_variance128x64_c, 8), + make_tuple(6, 7, &aom_highbd_8_sub_pixel_variance64x128_c, 8), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(6, 6, &aom_highbd_8_sub_pixel_variance64x64_c, 8), + make_tuple(6, 5, &aom_highbd_8_sub_pixel_variance64x32_c, 8), + make_tuple(5, 6, &aom_highbd_8_sub_pixel_variance32x64_c, 8), + make_tuple(5, 5, &aom_highbd_8_sub_pixel_variance32x32_c, 8), + make_tuple(5, 4, &aom_highbd_8_sub_pixel_variance32x16_c, 8), + make_tuple(4, 5, &aom_highbd_8_sub_pixel_variance16x32_c, 8), + make_tuple(4, 4, &aom_highbd_8_sub_pixel_variance16x16_c, 8), + make_tuple(4, 3, &aom_highbd_8_sub_pixel_variance16x8_c, 8), + make_tuple(3, 4, &aom_highbd_8_sub_pixel_variance8x16_c, 8), + make_tuple(3, 3, &aom_highbd_8_sub_pixel_variance8x8_c, 8), + make_tuple(3, 2, &aom_highbd_8_sub_pixel_variance8x4_c, 8), + make_tuple(2, 3, &aom_highbd_8_sub_pixel_variance4x8_c, 8), + make_tuple(2, 2, &aom_highbd_8_sub_pixel_variance4x4_c, 8), +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(7, 7, &aom_highbd_10_sub_pixel_variance128x128_c, 10), + make_tuple(7, 6, &aom_highbd_10_sub_pixel_variance128x64_c, 10), + make_tuple(6, 7, &aom_highbd_10_sub_pixel_variance64x128_c, 10), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(6, 6, &aom_highbd_10_sub_pixel_variance64x64_c, 10), + make_tuple(6, 5, &aom_highbd_10_sub_pixel_variance64x32_c, 10), + make_tuple(5, 6, &aom_highbd_10_sub_pixel_variance32x64_c, 10), + make_tuple(5, 5, &aom_highbd_10_sub_pixel_variance32x32_c, 10), + make_tuple(5, 4, &aom_highbd_10_sub_pixel_variance32x16_c, 10), + make_tuple(4, 5, &aom_highbd_10_sub_pixel_variance16x32_c, 10), + make_tuple(4, 4, &aom_highbd_10_sub_pixel_variance16x16_c, 10), + make_tuple(4, 3, &aom_highbd_10_sub_pixel_variance16x8_c, 10), + make_tuple(3, 4, &aom_highbd_10_sub_pixel_variance8x16_c, 10), + make_tuple(3, 3, &aom_highbd_10_sub_pixel_variance8x8_c, 10), + make_tuple(3, 2, &aom_highbd_10_sub_pixel_variance8x4_c, 10), + make_tuple(2, 3, &aom_highbd_10_sub_pixel_variance4x8_c, 10), + make_tuple(2, 2, &aom_highbd_10_sub_pixel_variance4x4_c, 10), +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(7, 7, &aom_highbd_12_sub_pixel_variance128x128_c, 12), + make_tuple(7, 6, &aom_highbd_12_sub_pixel_variance128x64_c, 12), + make_tuple(6, 7, &aom_highbd_12_sub_pixel_variance64x128_c, 12), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(6, 6, &aom_highbd_12_sub_pixel_variance64x64_c, 12), + make_tuple(6, 5, &aom_highbd_12_sub_pixel_variance64x32_c, 12), + make_tuple(5, 6, &aom_highbd_12_sub_pixel_variance32x64_c, 12), + make_tuple(5, 5, &aom_highbd_12_sub_pixel_variance32x32_c, 12), + make_tuple(5, 4, &aom_highbd_12_sub_pixel_variance32x16_c, 12), + make_tuple(4, 5, &aom_highbd_12_sub_pixel_variance16x32_c, 12), + make_tuple(4, 4, &aom_highbd_12_sub_pixel_variance16x16_c, 12), + make_tuple(4, 3, &aom_highbd_12_sub_pixel_variance16x8_c, 12), + make_tuple(3, 4, &aom_highbd_12_sub_pixel_variance8x16_c, 12), + make_tuple(3, 3, &aom_highbd_12_sub_pixel_variance8x8_c, 12), + make_tuple(3, 2, &aom_highbd_12_sub_pixel_variance8x4_c, 12), + make_tuple(2, 3, &aom_highbd_12_sub_pixel_variance4x8_c, 12), + make_tuple(2, 2, &aom_highbd_12_sub_pixel_variance4x4_c, 12), +}; +INSTANTIATE_TEST_CASE_P(C, AvxHBDSubpelVarianceTest, + ::testing::ValuesIn(kArrayHBDSubpelVariance_c)); + +const AvxHBDSubpelAvgVarianceTest::ParamType kArrayHBDSubpelAvgVariance_c[] = { +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(7, 7, &aom_highbd_8_sub_pixel_avg_variance128x128_c, 8), + make_tuple(7, 6, &aom_highbd_8_sub_pixel_avg_variance128x64_c, 8), + make_tuple(6, 7, &aom_highbd_8_sub_pixel_avg_variance64x128_c, 8), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(6, 6, &aom_highbd_8_sub_pixel_avg_variance64x64_c, 8), + make_tuple(6, 5, &aom_highbd_8_sub_pixel_avg_variance64x32_c, 8), + make_tuple(5, 6, &aom_highbd_8_sub_pixel_avg_variance32x64_c, 8), + make_tuple(5, 5, &aom_highbd_8_sub_pixel_avg_variance32x32_c, 8), + make_tuple(5, 4, &aom_highbd_8_sub_pixel_avg_variance32x16_c, 8), + make_tuple(4, 5, &aom_highbd_8_sub_pixel_avg_variance16x32_c, 8), + make_tuple(4, 4, &aom_highbd_8_sub_pixel_avg_variance16x16_c, 8), + make_tuple(4, 3, &aom_highbd_8_sub_pixel_avg_variance16x8_c, 8), + make_tuple(3, 4, &aom_highbd_8_sub_pixel_avg_variance8x16_c, 8), + make_tuple(3, 3, &aom_highbd_8_sub_pixel_avg_variance8x8_c, 8), + make_tuple(3, 2, &aom_highbd_8_sub_pixel_avg_variance8x4_c, 8), + make_tuple(2, 3, &aom_highbd_8_sub_pixel_avg_variance4x8_c, 8), + make_tuple(2, 2, &aom_highbd_8_sub_pixel_avg_variance4x4_c, 8), +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(7, 7, &aom_highbd_10_sub_pixel_avg_variance128x128_c, 10), + make_tuple(7, 6, &aom_highbd_10_sub_pixel_avg_variance128x64_c, 10), + make_tuple(6, 7, &aom_highbd_10_sub_pixel_avg_variance64x128_c, 10), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(6, 6, &aom_highbd_10_sub_pixel_avg_variance64x64_c, 10), + make_tuple(6, 5, &aom_highbd_10_sub_pixel_avg_variance64x32_c, 10), + make_tuple(5, 6, &aom_highbd_10_sub_pixel_avg_variance32x64_c, 10), + make_tuple(5, 5, &aom_highbd_10_sub_pixel_avg_variance32x32_c, 10), + make_tuple(5, 4, &aom_highbd_10_sub_pixel_avg_variance32x16_c, 10), + make_tuple(4, 5, &aom_highbd_10_sub_pixel_avg_variance16x32_c, 10), + make_tuple(4, 4, &aom_highbd_10_sub_pixel_avg_variance16x16_c, 10), + make_tuple(4, 3, &aom_highbd_10_sub_pixel_avg_variance16x8_c, 10), + make_tuple(3, 4, &aom_highbd_10_sub_pixel_avg_variance8x16_c, 10), + make_tuple(3, 3, &aom_highbd_10_sub_pixel_avg_variance8x8_c, 10), + make_tuple(3, 2, &aom_highbd_10_sub_pixel_avg_variance8x4_c, 10), + make_tuple(2, 3, &aom_highbd_10_sub_pixel_avg_variance4x8_c, 10), + make_tuple(2, 2, &aom_highbd_10_sub_pixel_avg_variance4x4_c, 10), +#if CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(7, 7, &aom_highbd_12_sub_pixel_avg_variance128x128_c, 12), + make_tuple(7, 6, &aom_highbd_12_sub_pixel_avg_variance128x64_c, 12), + make_tuple(6, 7, &aom_highbd_12_sub_pixel_avg_variance64x128_c, 12), +#endif // CONFIG_AV1 && CONFIG_EXT_PARTITION + make_tuple(6, 6, &aom_highbd_12_sub_pixel_avg_variance64x64_c, 12), + make_tuple(6, 5, &aom_highbd_12_sub_pixel_avg_variance64x32_c, 12), + make_tuple(5, 6, &aom_highbd_12_sub_pixel_avg_variance32x64_c, 12), + make_tuple(5, 5, &aom_highbd_12_sub_pixel_avg_variance32x32_c, 12), + make_tuple(5, 4, &aom_highbd_12_sub_pixel_avg_variance32x16_c, 12), + make_tuple(4, 5, &aom_highbd_12_sub_pixel_avg_variance16x32_c, 12), + make_tuple(4, 4, &aom_highbd_12_sub_pixel_avg_variance16x16_c, 12), + make_tuple(4, 3, &aom_highbd_12_sub_pixel_avg_variance16x8_c, 12), + make_tuple(3, 4, &aom_highbd_12_sub_pixel_avg_variance8x16_c, 12), + make_tuple(3, 3, &aom_highbd_12_sub_pixel_avg_variance8x8_c, 12), + make_tuple(3, 2, &aom_highbd_12_sub_pixel_avg_variance8x4_c, 12), + make_tuple(2, 3, &aom_highbd_12_sub_pixel_avg_variance4x8_c, 12), + make_tuple(2, 2, &aom_highbd_12_sub_pixel_avg_variance4x4_c, 12) +}; +INSTANTIATE_TEST_CASE_P(C, AvxHBDSubpelAvgVarianceTest, + ::testing::ValuesIn(kArrayHBDSubpelAvgVariance_c)); +#endif // CONFIG_HIGHBITDEPTH + +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P(SSE2, SumOfSquaresTest, + ::testing::Values(aom_get_mb_ss_sse2)); + +INSTANTIATE_TEST_CASE_P(SSE2, AvxMseTest, + ::testing::Values(MseParams(4, 4, &aom_mse16x16_sse2), + MseParams(4, 3, &aom_mse16x8_sse2), + MseParams(3, 4, &aom_mse8x16_sse2), + MseParams(3, 3, &aom_mse8x8_sse2))); + +INSTANTIATE_TEST_CASE_P( + SSE2, AvxVarianceTest, + ::testing::Values(VarianceParams(6, 6, &aom_variance64x64_sse2), + VarianceParams(6, 5, &aom_variance64x32_sse2), + VarianceParams(5, 6, &aom_variance32x64_sse2), + VarianceParams(5, 5, &aom_variance32x32_sse2), + VarianceParams(5, 4, &aom_variance32x16_sse2), + VarianceParams(4, 5, &aom_variance16x32_sse2), + VarianceParams(4, 4, &aom_variance16x16_sse2), + VarianceParams(4, 3, &aom_variance16x8_sse2), + VarianceParams(3, 4, &aom_variance8x16_sse2), + VarianceParams(3, 3, &aom_variance8x8_sse2), + VarianceParams(3, 2, &aom_variance8x4_sse2), + VarianceParams(2, 3, &aom_variance4x8_sse2), + VarianceParams(2, 2, &aom_variance4x4_sse2))); + +INSTANTIATE_TEST_CASE_P( + SSE2, AvxSubpelVarianceTest, + ::testing::Values(make_tuple(6, 6, &aom_sub_pixel_variance64x64_sse2, 0), + make_tuple(6, 5, &aom_sub_pixel_variance64x32_sse2, 0), + make_tuple(5, 6, &aom_sub_pixel_variance32x64_sse2, 0), + make_tuple(5, 5, &aom_sub_pixel_variance32x32_sse2, 0), + make_tuple(5, 4, &aom_sub_pixel_variance32x16_sse2, 0), + make_tuple(4, 5, &aom_sub_pixel_variance16x32_sse2, 0), + make_tuple(4, 4, &aom_sub_pixel_variance16x16_sse2, 0), + make_tuple(4, 3, &aom_sub_pixel_variance16x8_sse2, 0), + make_tuple(3, 4, &aom_sub_pixel_variance8x16_sse2, 0), + make_tuple(3, 3, &aom_sub_pixel_variance8x8_sse2, 0), + make_tuple(3, 2, &aom_sub_pixel_variance8x4_sse2, 0), + make_tuple(2, 3, &aom_sub_pixel_variance4x8_sse2, 0), + make_tuple(2, 2, &aom_sub_pixel_variance4x4_sse2, 0))); + +INSTANTIATE_TEST_CASE_P( + SSE2, AvxSubpelAvgVarianceTest, + ::testing::Values( + make_tuple(6, 6, &aom_sub_pixel_avg_variance64x64_sse2, 0), + make_tuple(6, 5, &aom_sub_pixel_avg_variance64x32_sse2, 0), + make_tuple(5, 6, &aom_sub_pixel_avg_variance32x64_sse2, 0), + make_tuple(5, 5, &aom_sub_pixel_avg_variance32x32_sse2, 0), + make_tuple(5, 4, &aom_sub_pixel_avg_variance32x16_sse2, 0), + make_tuple(4, 5, &aom_sub_pixel_avg_variance16x32_sse2, 0), + make_tuple(4, 4, &aom_sub_pixel_avg_variance16x16_sse2, 0), + make_tuple(4, 3, &aom_sub_pixel_avg_variance16x8_sse2, 0), + make_tuple(3, 4, &aom_sub_pixel_avg_variance8x16_sse2, 0), + make_tuple(3, 3, &aom_sub_pixel_avg_variance8x8_sse2, 0), + make_tuple(3, 2, &aom_sub_pixel_avg_variance8x4_sse2, 0), + make_tuple(2, 3, &aom_sub_pixel_avg_variance4x8_sse2, 0), + make_tuple(2, 2, &aom_sub_pixel_avg_variance4x4_sse2, 0))); + +#if HAVE_SSE4_1 && CONFIG_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSE4_1, AvxSubpelVarianceTest, + ::testing::Values( + make_tuple(2, 2, &aom_highbd_8_sub_pixel_variance4x4_sse4_1, 8), + make_tuple(2, 2, &aom_highbd_10_sub_pixel_variance4x4_sse4_1, 10), + make_tuple(2, 2, &aom_highbd_12_sub_pixel_variance4x4_sse4_1, 12))); + +INSTANTIATE_TEST_CASE_P( + SSE4_1, AvxSubpelAvgVarianceTest, + ::testing::Values( + make_tuple(2, 2, &aom_highbd_8_sub_pixel_avg_variance4x4_sse4_1, 8), + make_tuple(2, 2, &aom_highbd_10_sub_pixel_avg_variance4x4_sse4_1, 10), + make_tuple(2, 2, &aom_highbd_12_sub_pixel_avg_variance4x4_sse4_1, 12))); +#endif // HAVE_SSE4_1 && CONFIG_HIGHBITDEPTH + +#if CONFIG_HIGHBITDEPTH +/* TODO(debargha): This test does not support the highbd version +INSTANTIATE_TEST_CASE_P( + SSE2, AvxHBDMseTest, + ::testing::Values(MseParams(4, 4, &aom_highbd_12_mse16x16_sse2), + MseParams(4, 3, &aom_highbd_12_mse16x8_sse2), + MseParams(3, 4, &aom_highbd_12_mse8x16_sse2), + MseParams(3, 3, &aom_highbd_12_mse8x8_sse2), + MseParams(4, 4, &aom_highbd_10_mse16x16_sse2), + MseParams(4, 3, &aom_highbd_10_mse16x8_sse2), + MseParams(3, 4, &aom_highbd_10_mse8x16_sse2), + MseParams(3, 3, &aom_highbd_10_mse8x8_sse2), + MseParams(4, 4, &aom_highbd_8_mse16x16_sse2), + MseParams(4, 3, &aom_highbd_8_mse16x8_sse2), + MseParams(3, 4, &aom_highbd_8_mse8x16_sse2), + MseParams(3, 3, &aom_highbd_8_mse8x8_sse2))); +*/ + +INSTANTIATE_TEST_CASE_P( + SSE2, AvxHBDVarianceTest, + ::testing::Values( + VarianceParams(6, 6, &aom_highbd_12_variance64x64_sse2, 12), + VarianceParams(6, 5, &aom_highbd_12_variance64x32_sse2, 12), + VarianceParams(5, 6, &aom_highbd_12_variance32x64_sse2, 12), + VarianceParams(5, 5, &aom_highbd_12_variance32x32_sse2, 12), + VarianceParams(5, 4, &aom_highbd_12_variance32x16_sse2, 12), + VarianceParams(4, 5, &aom_highbd_12_variance16x32_sse2, 12), + VarianceParams(4, 4, &aom_highbd_12_variance16x16_sse2, 12), + VarianceParams(4, 3, &aom_highbd_12_variance16x8_sse2, 12), + VarianceParams(3, 4, &aom_highbd_12_variance8x16_sse2, 12), + VarianceParams(3, 3, &aom_highbd_12_variance8x8_sse2, 12), + VarianceParams(6, 6, &aom_highbd_10_variance64x64_sse2, 10), + VarianceParams(6, 5, &aom_highbd_10_variance64x32_sse2, 10), + VarianceParams(5, 6, &aom_highbd_10_variance32x64_sse2, 10), + VarianceParams(5, 5, &aom_highbd_10_variance32x32_sse2, 10), + VarianceParams(5, 4, &aom_highbd_10_variance32x16_sse2, 10), + VarianceParams(4, 5, &aom_highbd_10_variance16x32_sse2, 10), + VarianceParams(4, 4, &aom_highbd_10_variance16x16_sse2, 10), + VarianceParams(4, 3, &aom_highbd_10_variance16x8_sse2, 10), + VarianceParams(3, 4, &aom_highbd_10_variance8x16_sse2, 10), + VarianceParams(3, 3, &aom_highbd_10_variance8x8_sse2, 10), + VarianceParams(6, 6, &aom_highbd_8_variance64x64_sse2, 8), + VarianceParams(6, 5, &aom_highbd_8_variance64x32_sse2, 8), + VarianceParams(5, 6, &aom_highbd_8_variance32x64_sse2, 8), + VarianceParams(5, 5, &aom_highbd_8_variance32x32_sse2, 8), + VarianceParams(5, 4, &aom_highbd_8_variance32x16_sse2, 8), + VarianceParams(4, 5, &aom_highbd_8_variance16x32_sse2, 8), + VarianceParams(4, 4, &aom_highbd_8_variance16x16_sse2, 8), + VarianceParams(4, 3, &aom_highbd_8_variance16x8_sse2, 8), + VarianceParams(3, 4, &aom_highbd_8_variance8x16_sse2, 8), + VarianceParams(3, 3, &aom_highbd_8_variance8x8_sse2, 8))); + +INSTANTIATE_TEST_CASE_P( + SSE2, AvxHBDSubpelVarianceTest, + ::testing::Values( + make_tuple(6, 6, &aom_highbd_12_sub_pixel_variance64x64_sse2, 12), + make_tuple(6, 5, &aom_highbd_12_sub_pixel_variance64x32_sse2, 12), + make_tuple(5, 6, &aom_highbd_12_sub_pixel_variance32x64_sse2, 12), + make_tuple(5, 5, &aom_highbd_12_sub_pixel_variance32x32_sse2, 12), + make_tuple(5, 4, &aom_highbd_12_sub_pixel_variance32x16_sse2, 12), + make_tuple(4, 5, &aom_highbd_12_sub_pixel_variance16x32_sse2, 12), + make_tuple(4, 4, &aom_highbd_12_sub_pixel_variance16x16_sse2, 12), + make_tuple(4, 3, &aom_highbd_12_sub_pixel_variance16x8_sse2, 12), + make_tuple(3, 4, &aom_highbd_12_sub_pixel_variance8x16_sse2, 12), + make_tuple(3, 3, &aom_highbd_12_sub_pixel_variance8x8_sse2, 12), + make_tuple(3, 2, &aom_highbd_12_sub_pixel_variance8x4_sse2, 12), + make_tuple(6, 6, &aom_highbd_10_sub_pixel_variance64x64_sse2, 10), + make_tuple(6, 5, &aom_highbd_10_sub_pixel_variance64x32_sse2, 10), + make_tuple(5, 6, &aom_highbd_10_sub_pixel_variance32x64_sse2, 10), + make_tuple(5, 5, &aom_highbd_10_sub_pixel_variance32x32_sse2, 10), + make_tuple(5, 4, &aom_highbd_10_sub_pixel_variance32x16_sse2, 10), + make_tuple(4, 5, &aom_highbd_10_sub_pixel_variance16x32_sse2, 10), + make_tuple(4, 4, &aom_highbd_10_sub_pixel_variance16x16_sse2, 10), + make_tuple(4, 3, &aom_highbd_10_sub_pixel_variance16x8_sse2, 10), + make_tuple(3, 4, &aom_highbd_10_sub_pixel_variance8x16_sse2, 10), + make_tuple(3, 3, &aom_highbd_10_sub_pixel_variance8x8_sse2, 10), + make_tuple(3, 2, &aom_highbd_10_sub_pixel_variance8x4_sse2, 10), + make_tuple(6, 6, &aom_highbd_8_sub_pixel_variance64x64_sse2, 8), + make_tuple(6, 5, &aom_highbd_8_sub_pixel_variance64x32_sse2, 8), + make_tuple(5, 6, &aom_highbd_8_sub_pixel_variance32x64_sse2, 8), + make_tuple(5, 5, &aom_highbd_8_sub_pixel_variance32x32_sse2, 8), + make_tuple(5, 4, &aom_highbd_8_sub_pixel_variance32x16_sse2, 8), + make_tuple(4, 5, &aom_highbd_8_sub_pixel_variance16x32_sse2, 8), + make_tuple(4, 4, &aom_highbd_8_sub_pixel_variance16x16_sse2, 8), + make_tuple(4, 3, &aom_highbd_8_sub_pixel_variance16x8_sse2, 8), + make_tuple(3, 4, &aom_highbd_8_sub_pixel_variance8x16_sse2, 8), + make_tuple(3, 3, &aom_highbd_8_sub_pixel_variance8x8_sse2, 8), + make_tuple(3, 2, &aom_highbd_8_sub_pixel_variance8x4_sse2, 8))); + +INSTANTIATE_TEST_CASE_P( + SSE2, AvxHBDSubpelAvgVarianceTest, + ::testing::Values( + make_tuple(6, 6, &aom_highbd_12_sub_pixel_avg_variance64x64_sse2, 12), + make_tuple(6, 5, &aom_highbd_12_sub_pixel_avg_variance64x32_sse2, 12), + make_tuple(5, 6, &aom_highbd_12_sub_pixel_avg_variance32x64_sse2, 12), + make_tuple(5, 5, &aom_highbd_12_sub_pixel_avg_variance32x32_sse2, 12), + make_tuple(5, 4, &aom_highbd_12_sub_pixel_avg_variance32x16_sse2, 12), + make_tuple(4, 5, &aom_highbd_12_sub_pixel_avg_variance16x32_sse2, 12), + make_tuple(4, 4, &aom_highbd_12_sub_pixel_avg_variance16x16_sse2, 12), + make_tuple(4, 3, &aom_highbd_12_sub_pixel_avg_variance16x8_sse2, 12), + make_tuple(3, 4, &aom_highbd_12_sub_pixel_avg_variance8x16_sse2, 12), + make_tuple(3, 3, &aom_highbd_12_sub_pixel_avg_variance8x8_sse2, 12), + make_tuple(3, 2, &aom_highbd_12_sub_pixel_avg_variance8x4_sse2, 12), + make_tuple(6, 6, &aom_highbd_10_sub_pixel_avg_variance64x64_sse2, 10), + make_tuple(6, 5, &aom_highbd_10_sub_pixel_avg_variance64x32_sse2, 10), + make_tuple(5, 6, &aom_highbd_10_sub_pixel_avg_variance32x64_sse2, 10), + make_tuple(5, 5, &aom_highbd_10_sub_pixel_avg_variance32x32_sse2, 10), + make_tuple(5, 4, &aom_highbd_10_sub_pixel_avg_variance32x16_sse2, 10), + make_tuple(4, 5, &aom_highbd_10_sub_pixel_avg_variance16x32_sse2, 10), + make_tuple(4, 4, &aom_highbd_10_sub_pixel_avg_variance16x16_sse2, 10), + make_tuple(4, 3, &aom_highbd_10_sub_pixel_avg_variance16x8_sse2, 10), + make_tuple(3, 4, &aom_highbd_10_sub_pixel_avg_variance8x16_sse2, 10), + make_tuple(3, 3, &aom_highbd_10_sub_pixel_avg_variance8x8_sse2, 10), + make_tuple(3, 2, &aom_highbd_10_sub_pixel_avg_variance8x4_sse2, 10), + make_tuple(6, 6, &aom_highbd_8_sub_pixel_avg_variance64x64_sse2, 8), + make_tuple(6, 5, &aom_highbd_8_sub_pixel_avg_variance64x32_sse2, 8), + make_tuple(5, 6, &aom_highbd_8_sub_pixel_avg_variance32x64_sse2, 8), + make_tuple(5, 5, &aom_highbd_8_sub_pixel_avg_variance32x32_sse2, 8), + make_tuple(5, 4, &aom_highbd_8_sub_pixel_avg_variance32x16_sse2, 8), + make_tuple(4, 5, &aom_highbd_8_sub_pixel_avg_variance16x32_sse2, 8), + make_tuple(4, 4, &aom_highbd_8_sub_pixel_avg_variance16x16_sse2, 8), + make_tuple(4, 3, &aom_highbd_8_sub_pixel_avg_variance16x8_sse2, 8), + make_tuple(3, 4, &aom_highbd_8_sub_pixel_avg_variance8x16_sse2, 8), + make_tuple(3, 3, &aom_highbd_8_sub_pixel_avg_variance8x8_sse2, 8), + make_tuple(3, 2, &aom_highbd_8_sub_pixel_avg_variance8x4_sse2, 8))); +#endif // CONFIG_HIGHBITDEPTH +#endif // HAVE_SSE2 + +#if HAVE_SSSE3 +INSTANTIATE_TEST_CASE_P( + SSSE3, AvxSubpelVarianceTest, + ::testing::Values(make_tuple(6, 6, &aom_sub_pixel_variance64x64_ssse3, 0), + make_tuple(6, 5, &aom_sub_pixel_variance64x32_ssse3, 0), + make_tuple(5, 6, &aom_sub_pixel_variance32x64_ssse3, 0), + make_tuple(5, 5, &aom_sub_pixel_variance32x32_ssse3, 0), + make_tuple(5, 4, &aom_sub_pixel_variance32x16_ssse3, 0), + make_tuple(4, 5, &aom_sub_pixel_variance16x32_ssse3, 0), + make_tuple(4, 4, &aom_sub_pixel_variance16x16_ssse3, 0), + make_tuple(4, 3, &aom_sub_pixel_variance16x8_ssse3, 0), + make_tuple(3, 4, &aom_sub_pixel_variance8x16_ssse3, 0), + make_tuple(3, 3, &aom_sub_pixel_variance8x8_ssse3, 0), + make_tuple(3, 2, &aom_sub_pixel_variance8x4_ssse3, 0), + make_tuple(2, 3, &aom_sub_pixel_variance4x8_ssse3, 0), + make_tuple(2, 2, &aom_sub_pixel_variance4x4_ssse3, 0))); + +INSTANTIATE_TEST_CASE_P( + SSSE3, AvxSubpelAvgVarianceTest, + ::testing::Values( + make_tuple(6, 6, &aom_sub_pixel_avg_variance64x64_ssse3, 0), + make_tuple(6, 5, &aom_sub_pixel_avg_variance64x32_ssse3, 0), + make_tuple(5, 6, &aom_sub_pixel_avg_variance32x64_ssse3, 0), + make_tuple(5, 5, &aom_sub_pixel_avg_variance32x32_ssse3, 0), + make_tuple(5, 4, &aom_sub_pixel_avg_variance32x16_ssse3, 0), + make_tuple(4, 5, &aom_sub_pixel_avg_variance16x32_ssse3, 0), + make_tuple(4, 4, &aom_sub_pixel_avg_variance16x16_ssse3, 0), + make_tuple(4, 3, &aom_sub_pixel_avg_variance16x8_ssse3, 0), + make_tuple(3, 4, &aom_sub_pixel_avg_variance8x16_ssse3, 0), + make_tuple(3, 3, &aom_sub_pixel_avg_variance8x8_ssse3, 0), + make_tuple(3, 2, &aom_sub_pixel_avg_variance8x4_ssse3, 0), + make_tuple(2, 3, &aom_sub_pixel_avg_variance4x8_ssse3, 0), + make_tuple(2, 2, &aom_sub_pixel_avg_variance4x4_ssse3, 0))); +#endif // HAVE_SSSE3 + +#if HAVE_AVX2 +INSTANTIATE_TEST_CASE_P(AVX2, AvxMseTest, + ::testing::Values(MseParams(4, 4, &aom_mse16x16_avx2))); + +INSTANTIATE_TEST_CASE_P( + AVX2, AvxVarianceTest, + ::testing::Values(VarianceParams(6, 6, &aom_variance64x64_avx2), + VarianceParams(6, 5, &aom_variance64x32_avx2), + VarianceParams(5, 5, &aom_variance32x32_avx2), + VarianceParams(5, 4, &aom_variance32x16_avx2), + VarianceParams(4, 4, &aom_variance16x16_avx2))); + +INSTANTIATE_TEST_CASE_P( + AVX2, AvxSubpelVarianceTest, + ::testing::Values(make_tuple(6, 6, &aom_sub_pixel_variance64x64_avx2, 0), + make_tuple(5, 5, &aom_sub_pixel_variance32x32_avx2, 0))); + +INSTANTIATE_TEST_CASE_P( + AVX2, AvxSubpelAvgVarianceTest, + ::testing::Values( + make_tuple(6, 6, &aom_sub_pixel_avg_variance64x64_avx2, 0), + make_tuple(5, 5, &aom_sub_pixel_avg_variance32x32_avx2, 0))); +#endif // HAVE_AVX2 + +#if HAVE_MEDIA +INSTANTIATE_TEST_CASE_P(MEDIA, AvxMseTest, + ::testing::Values(MseParams(4, 4, + &aom_mse16x16_media))); + +INSTANTIATE_TEST_CASE_P( + MEDIA, AvxVarianceTest, + ::testing::Values(VarianceParams(4, 4, &aom_variance16x16_media), + VarianceParams(3, 3, &aom_variance8x8_media))); + +INSTANTIATE_TEST_CASE_P( + MEDIA, AvxSubpelVarianceTest, + ::testing::Values(make_tuple(4, 4, &aom_sub_pixel_variance16x16_media, 0), + make_tuple(3, 3, &aom_sub_pixel_variance8x8_media, 0))); +#endif // HAVE_MEDIA + +#if HAVE_NEON +INSTANTIATE_TEST_CASE_P(NEON, AvxSseTest, + ::testing::Values(SseParams(2, 2, + &aom_get4x4sse_cs_neon))); + +INSTANTIATE_TEST_CASE_P(NEON, AvxMseTest, + ::testing::Values(MseParams(4, 4, &aom_mse16x16_neon))); + +INSTANTIATE_TEST_CASE_P( + NEON, AvxVarianceTest, + ::testing::Values(VarianceParams(6, 6, &aom_variance64x64_neon), + VarianceParams(6, 5, &aom_variance64x32_neon), + VarianceParams(5, 6, &aom_variance32x64_neon), + VarianceParams(5, 5, &aom_variance32x32_neon), + VarianceParams(4, 4, &aom_variance16x16_neon), + VarianceParams(4, 3, &aom_variance16x8_neon), + VarianceParams(3, 4, &aom_variance8x16_neon), + VarianceParams(3, 3, &aom_variance8x8_neon))); + +INSTANTIATE_TEST_CASE_P( + NEON, AvxSubpelVarianceTest, + ::testing::Values(make_tuple(6, 6, &aom_sub_pixel_variance64x64_neon, 0), + make_tuple(5, 5, &aom_sub_pixel_variance32x32_neon, 0), + make_tuple(4, 4, &aom_sub_pixel_variance16x16_neon, 0), + make_tuple(3, 3, &aom_sub_pixel_variance8x8_neon, 0))); +#endif // HAVE_NEON + +#if HAVE_MSA +INSTANTIATE_TEST_CASE_P(MSA, SumOfSquaresTest, + ::testing::Values(aom_get_mb_ss_msa)); + +INSTANTIATE_TEST_CASE_P(MSA, AvxSseTest, + ::testing::Values(SseParams(2, 2, + &aom_get4x4sse_cs_msa))); + +INSTANTIATE_TEST_CASE_P(MSA, AvxMseTest, + ::testing::Values(MseParams(4, 4, &aom_mse16x16_msa), + MseParams(4, 3, &aom_mse16x8_msa), + MseParams(3, 4, &aom_mse8x16_msa), + MseParams(3, 3, &aom_mse8x8_msa))); + +INSTANTIATE_TEST_CASE_P( + MSA, AvxVarianceTest, + ::testing::Values(VarianceParams(6, 6, &aom_variance64x64_msa), + VarianceParams(6, 5, &aom_variance64x32_msa), + VarianceParams(5, 6, &aom_variance32x64_msa), + VarianceParams(5, 5, &aom_variance32x32_msa), + VarianceParams(5, 4, &aom_variance32x16_msa), + VarianceParams(4, 5, &aom_variance16x32_msa), + VarianceParams(4, 4, &aom_variance16x16_msa), + VarianceParams(4, 3, &aom_variance16x8_msa), + VarianceParams(3, 4, &aom_variance8x16_msa), + VarianceParams(3, 3, &aom_variance8x8_msa), + VarianceParams(3, 2, &aom_variance8x4_msa), + VarianceParams(2, 3, &aom_variance4x8_msa), + VarianceParams(2, 2, &aom_variance4x4_msa))); + +INSTANTIATE_TEST_CASE_P( + MSA, AvxSubpelVarianceTest, + ::testing::Values(make_tuple(2, 2, &aom_sub_pixel_variance4x4_msa, 0), + make_tuple(2, 3, &aom_sub_pixel_variance4x8_msa, 0), + make_tuple(3, 2, &aom_sub_pixel_variance8x4_msa, 0), + make_tuple(3, 3, &aom_sub_pixel_variance8x8_msa, 0), + make_tuple(3, 4, &aom_sub_pixel_variance8x16_msa, 0), + make_tuple(4, 3, &aom_sub_pixel_variance16x8_msa, 0), + make_tuple(4, 4, &aom_sub_pixel_variance16x16_msa, 0), + make_tuple(4, 5, &aom_sub_pixel_variance16x32_msa, 0), + make_tuple(5, 4, &aom_sub_pixel_variance32x16_msa, 0), + make_tuple(5, 5, &aom_sub_pixel_variance32x32_msa, 0), + make_tuple(5, 6, &aom_sub_pixel_variance32x64_msa, 0), + make_tuple(6, 5, &aom_sub_pixel_variance64x32_msa, 0), + make_tuple(6, 6, &aom_sub_pixel_variance64x64_msa, 0))); + +INSTANTIATE_TEST_CASE_P( + MSA, AvxSubpelAvgVarianceTest, + ::testing::Values(make_tuple(6, 6, &aom_sub_pixel_avg_variance64x64_msa, 0), + make_tuple(6, 5, &aom_sub_pixel_avg_variance64x32_msa, 0), + make_tuple(5, 6, &aom_sub_pixel_avg_variance32x64_msa, 0), + make_tuple(5, 5, &aom_sub_pixel_avg_variance32x32_msa, 0), + make_tuple(5, 4, &aom_sub_pixel_avg_variance32x16_msa, 0), + make_tuple(4, 5, &aom_sub_pixel_avg_variance16x32_msa, 0), + make_tuple(4, 4, &aom_sub_pixel_avg_variance16x16_msa, 0), + make_tuple(4, 3, &aom_sub_pixel_avg_variance16x8_msa, 0), + make_tuple(3, 4, &aom_sub_pixel_avg_variance8x16_msa, 0), + make_tuple(3, 3, &aom_sub_pixel_avg_variance8x8_msa, 0), + make_tuple(3, 2, &aom_sub_pixel_avg_variance8x4_msa, 0), + make_tuple(2, 3, &aom_sub_pixel_avg_variance4x8_msa, 0), + make_tuple(2, 2, &aom_sub_pixel_avg_variance4x4_msa, 0))); +#endif // HAVE_MSA +} // namespace diff --git a/third_party/aom/test/video_source.h b/third_party/aom/test/video_source.h new file mode 100644 index 0000000000..e986ffb373 --- /dev/null +++ b/third_party/aom/test/video_source.h @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef TEST_VIDEO_SOURCE_H_ +#define TEST_VIDEO_SOURCE_H_ + +#if defined(_WIN32) +#undef NOMINMAX +#define NOMINMAX +#define WIN32_LEAN_AND_MEAN +#include +#endif +#include +#include +#include +#include "test/acm_random.h" +#include "aom/aom_encoder.h" + +namespace libaom_test { + +// Helper macros to ensure LIBAOM_TEST_DATA_PATH is a quoted string. +// These are undefined right below GetDataPath +// NOTE: LIBAOM_TEST_DATA_PATH MUST NOT be a quoted string before +// Stringification or the GetDataPath will fail at runtime +#define TO_STRING(S) #S +#define STRINGIFY(S) TO_STRING(S) + +// A simple function to encapsulate cross platform retrieval of test data path +static std::string GetDataPath() { + const char *const data_path = getenv("LIBAOM_TEST_DATA_PATH"); + if (data_path == NULL) { +#ifdef LIBAOM_TEST_DATA_PATH + // In some environments, we cannot set environment variables + // Instead, we set the data path by using a preprocessor symbol + // which can be set from make files + return STRINGIFY(LIBAOM_TEST_DATA_PATH); +#else + return "."; +#endif + } + return data_path; +} + +// Undefining stringification macros because they are not used elsewhere +#undef TO_STRING +#undef STRINGIFY + +inline FILE *OpenTestDataFile(const std::string &file_name) { + const std::string path_to_source = GetDataPath() + "/" + file_name; + return fopen(path_to_source.c_str(), "rb"); +} + +static FILE *GetTempOutFile(std::string *file_name) { + file_name->clear(); +#if defined(_WIN32) + char fname[MAX_PATH]; + char tmppath[MAX_PATH]; + if (GetTempPathA(MAX_PATH, tmppath)) { + // Assume for now that the filename generated is unique per process + if (GetTempFileNameA(tmppath, "lvx", 0, fname)) { + file_name->assign(fname); + return fopen(fname, "wb+"); + } + } + return NULL; +#else + return tmpfile(); +#endif +} + +class TempOutFile { + public: + TempOutFile() { file_ = GetTempOutFile(&file_name_); } + ~TempOutFile() { + CloseFile(); + if (!file_name_.empty()) { + EXPECT_EQ(0, remove(file_name_.c_str())); + } + } + FILE *file() { return file_; } + const std::string &file_name() { return file_name_; } + + protected: + void CloseFile() { + if (file_) { + fclose(file_); + file_ = NULL; + } + } + FILE *file_; + std::string file_name_; +}; + +// Abstract base class for test video sources, which provide a stream of +// aom_image_t images with associated timestamps and duration. +class VideoSource { + public: + virtual ~VideoSource() {} + + // Prepare the stream for reading, rewind/open as necessary. + virtual void Begin() = 0; + + // Advance the cursor to the next frame + virtual void Next() = 0; + + // Get the current video frame, or NULL on End-Of-Stream. + virtual aom_image_t *img() const = 0; + + // Get the presentation timestamp of the current frame. + virtual aom_codec_pts_t pts() const = 0; + + // Get the current frame's duration + virtual unsigned long duration() const = 0; + + // Get the timebase for the stream + virtual aom_rational_t timebase() const = 0; + + // Get the current frame counter, starting at 0. + virtual unsigned int frame() const = 0; + + // Get the current file limit. + virtual unsigned int limit() const = 0; +}; + +class DummyVideoSource : public VideoSource { + public: + DummyVideoSource() + : img_(NULL), limit_(100), width_(80), height_(64), + format_(AOM_IMG_FMT_I420) { + ReallocImage(); + } + + virtual ~DummyVideoSource() { aom_img_free(img_); } + + virtual void Begin() { + frame_ = 0; + FillFrame(); + } + + virtual void Next() { + ++frame_; + FillFrame(); + } + + virtual aom_image_t *img() const { return (frame_ < limit_) ? img_ : NULL; } + + // Models a stream where Timebase = 1/FPS, so pts == frame. + virtual aom_codec_pts_t pts() const { return frame_; } + + virtual unsigned long duration() const { return 1; } + + virtual aom_rational_t timebase() const { + const aom_rational_t t = { 1, 30 }; + return t; + } + + virtual unsigned int frame() const { return frame_; } + + virtual unsigned int limit() const { return limit_; } + + void set_limit(unsigned int limit) { limit_ = limit; } + + void SetSize(unsigned int width, unsigned int height) { + if (width != width_ || height != height_) { + width_ = width; + height_ = height; + ReallocImage(); + } + } + + void SetImageFormat(aom_img_fmt_t format) { + if (format_ != format) { + format_ = format; + ReallocImage(); + } + } + + protected: + virtual void FillFrame() { + if (img_) memset(img_->img_data, 0, raw_sz_); + } + + void ReallocImage() { + aom_img_free(img_); + img_ = aom_img_alloc(NULL, format_, width_, height_, 32); + raw_sz_ = ((img_->w + 31) & ~31) * img_->h * img_->bps / 8; + } + + aom_image_t *img_; + size_t raw_sz_; + unsigned int limit_; + unsigned int frame_; + unsigned int width_; + unsigned int height_; + aom_img_fmt_t format_; +}; + +class RandomVideoSource : public DummyVideoSource { + public: + RandomVideoSource(int seed = ACMRandom::DeterministicSeed()) + : rnd_(seed), seed_(seed) {} + + protected: + // Reset the RNG to get a matching stream for the second pass + virtual void Begin() { + frame_ = 0; + rnd_.Reset(seed_); + FillFrame(); + } + + // 15 frames of noise, followed by 15 static frames. Reset to 0 rather + // than holding previous frames to encourage keyframes to be thrown. + virtual void FillFrame() { + if (img_) { + if (frame_ % 30 < 15) + for (size_t i = 0; i < raw_sz_; ++i) img_->img_data[i] = rnd_.Rand8(); + else + memset(img_->img_data, 0, raw_sz_); + } + } + + ACMRandom rnd_; + int seed_; +}; + +// Abstract base class for test video sources, which provide a stream of +// decompressed images to the decoder. +class CompressedVideoSource { + public: + virtual ~CompressedVideoSource() {} + + virtual void Init() = 0; + + // Prepare the stream for reading, rewind/open as necessary. + virtual void Begin() = 0; + + // Advance the cursor to the next frame + virtual void Next() = 0; + + virtual const uint8_t *cxdata() const = 0; + + virtual size_t frame_size() const = 0; + + virtual unsigned int frame_number() const = 0; +}; + +} // namespace libaom_test + +#endif // TEST_VIDEO_SOURCE_H_ diff --git a/third_party/aom/test/warp_filter_test.cc b/third_party/aom/test/warp_filter_test.cc new file mode 100644 index 0000000000..fd6608bfc1 --- /dev/null +++ b/third_party/aom/test/warp_filter_test.cc @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/warp_filter_test_util.h" + +using std::tr1::tuple; +using std::tr1::make_tuple; +using libaom_test::ACMRandom; +using libaom_test::AV1WarpFilter::AV1WarpFilterTest; +#if CONFIG_HIGHBITDEPTH +using libaom_test::AV1HighbdWarpFilter::AV1HighbdWarpFilterTest; +#endif + +namespace { + +TEST_P(AV1WarpFilterTest, CheckOutput) { RunCheckOutput(av1_warp_affine_sse2); } + +INSTANTIATE_TEST_CASE_P(SSE2, AV1WarpFilterTest, + libaom_test::AV1WarpFilter::GetDefaultParams()); + +#if CONFIG_HIGHBITDEPTH +TEST_P(AV1HighbdWarpFilterTest, CheckOutput) { + RunCheckOutput(av1_highbd_warp_affine_ssse3); +} + +INSTANTIATE_TEST_CASE_P(SSSE3, AV1HighbdWarpFilterTest, + libaom_test::AV1HighbdWarpFilter::GetDefaultParams()); +#endif + +} // namespace diff --git a/third_party/aom/test/warp_filter_test_util.cc b/third_party/aom/test/warp_filter_test_util.cc new file mode 100644 index 0000000000..1ce265b609 --- /dev/null +++ b/third_party/aom/test/warp_filter_test_util.cc @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "test/warp_filter_test_util.h" + +using std::tr1::tuple; +using std::tr1::make_tuple; +using std::vector; +using libaom_test::ACMRandom; +using libaom_test::AV1WarpFilter::AV1WarpFilterTest; +using libaom_test::AV1WarpFilter::WarpTestParam; +#if CONFIG_HIGHBITDEPTH +using libaom_test::AV1HighbdWarpFilter::AV1HighbdWarpFilterTest; +using libaom_test::AV1HighbdWarpFilter::HighbdWarpTestParam; +#endif + +::testing::internal::ParamGenerator +libaom_test::AV1WarpFilter::GetDefaultParams() { + const WarpTestParam defaultParams[] = { + make_tuple(4, 4, 50000), make_tuple(8, 8, 50000), + make_tuple(64, 64, 1000), make_tuple(4, 16, 20000), + make_tuple(32, 8, 10000), + }; + return ::testing::ValuesIn(defaultParams); +} + +AV1WarpFilterTest::~AV1WarpFilterTest() {} +void AV1WarpFilterTest::SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); } + +void AV1WarpFilterTest::TearDown() { libaom_test::ClearSystemState(); } + +int32_t AV1WarpFilterTest::random_param(int bits) { + // 1 in 8 chance of generating zero (arbitrarily chosen) + if (((rnd_.Rand8()) & 7) == 0) return 0; + // Otherwise, enerate uniform values in the range + // [-(1 << bits), 1] U [1, 1<= (1 << WARPEDMODEL_PREC_BITS)) || + (4 * abs(*gamma) + 4 * abs(*delta) >= (1 << WARPEDMODEL_PREC_BITS))) + continue; + + // We have a valid model, so finish + return; + } +} + +void AV1WarpFilterTest::RunCheckOutput(warp_affine_func test_impl) { + const int w = 128, h = 128; + const int border = 16; + const int stride = w + 2 * border; + const int out_w = GET_PARAM(0), out_h = GET_PARAM(1); + const int num_iters = GET_PARAM(2); + int i, j, sub_x, sub_y; + + uint8_t *input_ = new uint8_t[h * stride]; + uint8_t *input = input_ + border; + + // The warp functions always write rows with widths that are multiples of 8. + // So to avoid a buffer overflow, we may need to pad rows to a multiple of 8. + int output_n = ((out_w + 7) & ~7) * out_h; + uint8_t *output = new uint8_t[output_n]; + uint8_t *output2 = new uint8_t[output_n]; + int32_t mat[8]; + int16_t alpha, beta, gamma, delta; + + // Generate an input block and extend its borders horizontally + for (i = 0; i < h; ++i) + for (j = 0; j < w; ++j) input[i * stride + j] = rnd_.Rand8(); + for (i = 0; i < h; ++i) { + memset(input + i * stride - border, input[i * stride], border); + memset(input + i * stride + w, input[i * stride + (w - 1)], border); + } + + for (i = 0; i < num_iters; ++i) { + for (sub_x = 0; sub_x < 2; ++sub_x) + for (sub_y = 0; sub_y < 2; ++sub_y) { + generate_model(mat, &alpha, &beta, &gamma, &delta); + av1_warp_affine_c(mat, input, w, h, stride, output, 32, 32, out_w, + out_h, out_w, sub_x, sub_y, 0, alpha, beta, gamma, + delta); + test_impl(mat, input, w, h, stride, output2, 32, 32, out_w, out_h, + out_w, sub_x, sub_y, 0, alpha, beta, gamma, delta); + + for (j = 0; j < out_w * out_h; ++j) + ASSERT_EQ(output[j], output2[j]) + << "Pixel mismatch at index " << j << " = (" << (j % out_w) + << ", " << (j / out_w) << ") on iteration " << i; + } + } + delete[] input_; + delete[] output; + delete[] output2; +} + +#if CONFIG_HIGHBITDEPTH +::testing::internal::ParamGenerator +libaom_test::AV1HighbdWarpFilter::GetDefaultParams() { + const HighbdWarpTestParam defaultParams[] = { + make_tuple(4, 4, 50000, 8), make_tuple(8, 8, 50000, 8), + make_tuple(64, 64, 1000, 8), make_tuple(4, 16, 20000, 8), + make_tuple(32, 8, 10000, 8), make_tuple(4, 4, 50000, 10), + make_tuple(8, 8, 50000, 10), make_tuple(64, 64, 1000, 10), + make_tuple(4, 16, 20000, 10), make_tuple(32, 8, 10000, 10), + make_tuple(4, 4, 50000, 12), make_tuple(8, 8, 50000, 12), + make_tuple(64, 64, 1000, 12), make_tuple(4, 16, 20000, 12), + make_tuple(32, 8, 10000, 12), + }; + return ::testing::ValuesIn(defaultParams); +} + +AV1HighbdWarpFilterTest::~AV1HighbdWarpFilterTest() {} +void AV1HighbdWarpFilterTest::SetUp() { + rnd_.Reset(ACMRandom::DeterministicSeed()); +} + +void AV1HighbdWarpFilterTest::TearDown() { libaom_test::ClearSystemState(); } + +int32_t AV1HighbdWarpFilterTest::random_param(int bits) { + // 1 in 8 chance of generating zero (arbitrarily chosen) + if (((rnd_.Rand8()) & 7) == 0) return 0; + // Otherwise, enerate uniform values in the range + // [-(1 << bits), 1] U [1, 1<= (1 << WARPEDMODEL_PREC_BITS)) || + (4 * abs(*gamma) + 4 * abs(*delta) >= (1 << WARPEDMODEL_PREC_BITS))) + continue; + + // We have a valid model, so finish + return; + } +} + +void AV1HighbdWarpFilterTest::RunCheckOutput( + highbd_warp_affine_func test_impl) { + const int w = 128, h = 128; + const int border = 16; + const int stride = w + 2 * border; + const int out_w = GET_PARAM(0), out_h = GET_PARAM(1); + const int num_iters = GET_PARAM(2); + const int bd = GET_PARAM(3); + const int mask = (1 << bd) - 1; + int i, j, sub_x, sub_y; + + // The warp functions always write rows with widths that are multiples of 8. + // So to avoid a buffer overflow, we may need to pad rows to a multiple of 8. + int output_n = ((out_w + 7) & ~7) * out_h; + uint16_t *input_ = new uint16_t[h * stride]; + uint16_t *input = input_ + border; + uint16_t *output = new uint16_t[output_n]; + uint16_t *output2 = new uint16_t[output_n]; + int32_t mat[8]; + int16_t alpha, beta, gamma, delta; + + // Generate an input block and extend its borders horizontally + for (i = 0; i < h; ++i) + for (j = 0; j < w; ++j) input[i * stride + j] = rnd_.Rand16() & mask; + for (i = 0; i < h; ++i) { + for (j = 0; j < border; ++j) { + input[i * stride - border + j] = input[i * stride]; + input[i * stride + w + j] = input[i * stride + (w - 1)]; + } + } + + for (i = 0; i < num_iters; ++i) { + for (sub_x = 0; sub_x < 2; ++sub_x) + for (sub_y = 0; sub_y < 2; ++sub_y) { + generate_model(mat, &alpha, &beta, &gamma, &delta); + + av1_highbd_warp_affine_c(mat, input, w, h, stride, output, 32, 32, + out_w, out_h, out_w, sub_x, sub_y, bd, 0, + alpha, beta, gamma, delta); + test_impl(mat, input, w, h, stride, output2, 32, 32, out_w, out_h, + out_w, sub_x, sub_y, bd, 0, alpha, beta, gamma, delta); + + for (j = 0; j < out_w * out_h; ++j) + ASSERT_EQ(output[j], output2[j]) + << "Pixel mismatch at index " << j << " = (" << (j % out_w) + << ", " << (j / out_w) << ") on iteration " << i; + } + } + + delete[] input_; + delete[] output; + delete[] output2; +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/test/warp_filter_test_util.h b/third_party/aom/test/warp_filter_test_util.h new file mode 100644 index 0000000000..6a87e46d05 --- /dev/null +++ b/third_party/aom/test/warp_filter_test_util.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef TEST_WARP_FILTER_TEST_UTIL_H_ +#define TEST_WARP_FILTER_TEST_UTIL_H_ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/acm_random.h" +#include "test/util.h" +#include "./av1_rtcd.h" +#include "./aom_dsp_rtcd.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" + +#include "av1/common/mv.h" + +namespace libaom_test { + +namespace AV1WarpFilter { + +typedef void (*warp_affine_func)(int32_t *mat, uint8_t *ref, int width, + int height, int stride, uint8_t *pred, + int p_col, int p_row, int p_width, + int p_height, int p_stride, int subsampling_x, + int subsampling_y, int ref_frm, int16_t alpha, + int16_t beta, int16_t gamma, int16_t delta); + +typedef std::tr1::tuple WarpTestParam; + +::testing::internal::ParamGenerator GetDefaultParams(); + +class AV1WarpFilterTest : public ::testing::TestWithParam { + public: + virtual ~AV1WarpFilterTest(); + virtual void SetUp(); + + virtual void TearDown(); + + protected: + int32_t random_param(int bits); + void generate_model(int32_t *mat, int16_t *alpha, int16_t *beta, + int16_t *gamma, int16_t *delta); + + void RunCheckOutput(warp_affine_func test_impl); + + libaom_test::ACMRandom rnd_; +}; + +} // namespace AV1WarpFilter + +#if CONFIG_HIGHBITDEPTH +namespace AV1HighbdWarpFilter { +typedef void (*highbd_warp_affine_func)( + int32_t *mat, uint16_t *ref, int width, int height, int stride, + uint16_t *pred, int p_col, int p_row, int p_width, int p_height, + int p_stride, int subsampling_x, int subsampling_y, int bd, int ref_frm, + int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); + +typedef std::tr1::tuple HighbdWarpTestParam; + +::testing::internal::ParamGenerator GetDefaultParams(); + +class AV1HighbdWarpFilterTest + : public ::testing::TestWithParam { + public: + virtual ~AV1HighbdWarpFilterTest(); + virtual void SetUp(); + + virtual void TearDown(); + + protected: + int32_t random_param(int bits); + void generate_model(int32_t *mat, int16_t *alpha, int16_t *beta, + int16_t *gamma, int16_t *delta); + + void RunCheckOutput(highbd_warp_affine_func test_impl); + + libaom_test::ACMRandom rnd_; +}; + +} // namespace AV1HighbdWarpFilter +#endif // CONFIG_HIGHBITDEPTH + +} // namespace libaom_test + +#endif // TEST_WARP_FILTER_TEST_UTIL_H_ diff --git a/third_party/aom/test/webm_video_source.h b/third_party/aom/test/webm_video_source.h new file mode 100644 index 0000000000..286f69cbf1 --- /dev/null +++ b/third_party/aom/test/webm_video_source.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef TEST_WEBM_VIDEO_SOURCE_H_ +#define TEST_WEBM_VIDEO_SOURCE_H_ +#include +#include +#include +#include +#include +#include "../tools_common.h" +#include "../webmdec.h" +#include "test/video_source.h" + +namespace libaom_test { + +// This class extends VideoSource to allow parsing of WebM files, +// so that we can do actual file decodes. +class WebMVideoSource : public CompressedVideoSource { + public: + explicit WebMVideoSource(const std::string &file_name) + : file_name_(file_name), aom_ctx_(new AvxInputContext()), + webm_ctx_(new WebmInputContext()), buf_(NULL), buf_sz_(0), frame_(0), + end_of_file_(false) {} + + virtual ~WebMVideoSource() { + if (aom_ctx_->file != NULL) fclose(aom_ctx_->file); + webm_free(webm_ctx_); + delete aom_ctx_; + delete webm_ctx_; + } + + virtual void Init() {} + + virtual void Begin() { + aom_ctx_->file = OpenTestDataFile(file_name_); + ASSERT_TRUE(aom_ctx_->file != NULL) << "Input file open failed. Filename: " + << file_name_; + + ASSERT_EQ(file_is_webm(webm_ctx_, aom_ctx_), 1) << "file is not WebM"; + + FillFrame(); + } + + virtual void Next() { + ++frame_; + FillFrame(); + } + + void FillFrame() { + ASSERT_TRUE(aom_ctx_->file != NULL); + const int status = webm_read_frame(webm_ctx_, &buf_, &buf_sz_); + ASSERT_GE(status, 0) << "webm_read_frame failed"; + if (status == 1) { + end_of_file_ = true; + } + } + + void SeekToNextKeyFrame() { + ASSERT_TRUE(aom_ctx_->file != NULL); + do { + const int status = webm_read_frame(webm_ctx_, &buf_, &buf_sz_); + ASSERT_GE(status, 0) << "webm_read_frame failed"; + ++frame_; + if (status == 1) { + end_of_file_ = true; + } + } while (!webm_ctx_->is_key_frame && !end_of_file_); + } + + virtual const uint8_t *cxdata() const { return end_of_file_ ? NULL : buf_; } + virtual size_t frame_size() const { return buf_sz_; } + virtual unsigned int frame_number() const { return frame_; } + + protected: + std::string file_name_; + AvxInputContext *aom_ctx_; + WebmInputContext *webm_ctx_; + uint8_t *buf_; + size_t buf_sz_; + unsigned int frame_; + bool end_of_file_; +}; + +} // namespace libaom_test + +#endif // TEST_WEBM_VIDEO_SOURCE_H_ diff --git a/third_party/aom/test/y4m_test.cc b/third_party/aom/test/y4m_test.cc new file mode 100644 index 0000000000..fc9fff5142 --- /dev/null +++ b/third_party/aom/test/y4m_test.cc @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "./aom_config.h" +#include "./y4menc.h" +#include "test/md5_helper.h" +#include "test/util.h" +#include "test/y4m_video_source.h" + +namespace { + +using std::string; + +static const unsigned int kWidth = 160; +static const unsigned int kHeight = 90; +static const unsigned int kFrames = 10; + +struct Y4mTestParam { + const char *filename; + unsigned int bit_depth; + aom_img_fmt format; + const char *md5raw; +}; + +const Y4mTestParam kY4mTestVectors[] = { + { "park_joy_90p_8_420.y4m", 8, AOM_IMG_FMT_I420, + "e5406275b9fc6bb3436c31d4a05c1cab" }, + { "park_joy_90p_8_422.y4m", 8, AOM_IMG_FMT_I422, + "284a47a47133b12884ec3a14e959a0b6" }, + { "park_joy_90p_8_444.y4m", 8, AOM_IMG_FMT_I444, + "90517ff33843d85de712fd4fe60dbed0" }, + { "park_joy_90p_10_420.y4m", 10, AOM_IMG_FMT_I42016, + "63f21f9f717d8b8631bd2288ee87137b" }, + { "park_joy_90p_10_422.y4m", 10, AOM_IMG_FMT_I42216, + "48ab51fb540aed07f7ff5af130c9b605" }, + { "park_joy_90p_10_444.y4m", 10, AOM_IMG_FMT_I44416, + "067bfd75aa85ff9bae91fa3e0edd1e3e" }, + { "park_joy_90p_12_420.y4m", 12, AOM_IMG_FMT_I42016, + "9e6d8f6508c6e55625f6b697bc461cef" }, + { "park_joy_90p_12_422.y4m", 12, AOM_IMG_FMT_I42216, + "b239c6b301c0b835485be349ca83a7e3" }, + { "park_joy_90p_12_444.y4m", 12, AOM_IMG_FMT_I44416, + "5a6481a550821dab6d0192f5c63845e9" }, +}; + +static void write_image_file(const aom_image_t *img, FILE *file) { + int plane, y; + for (plane = 0; plane < 3; ++plane) { + const unsigned char *buf = img->planes[plane]; + const int stride = img->stride[plane]; + const int bytes_per_sample = (img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1; + const int h = + (plane ? (img->d_h + img->y_chroma_shift) >> img->y_chroma_shift + : img->d_h); + const int w = + (plane ? (img->d_w + img->x_chroma_shift) >> img->x_chroma_shift + : img->d_w); + for (y = 0; y < h; ++y) { + fwrite(buf, bytes_per_sample, w, file); + buf += stride; + } + } +} + +class Y4mVideoSourceTest : public ::testing::TestWithParam, + public ::libaom_test::Y4mVideoSource { + protected: + Y4mVideoSourceTest() : Y4mVideoSource("", 0, 0) {} + + virtual ~Y4mVideoSourceTest() { CloseSource(); } + + virtual void Init(const std::string &file_name, int limit) { + file_name_ = file_name; + start_ = 0; + limit_ = limit; + frame_ = 0; + Begin(); + } + + // Checks y4m header information + void HeaderChecks(unsigned int bit_depth, aom_img_fmt_t fmt) { + ASSERT_TRUE(input_file_ != NULL); + ASSERT_EQ(y4m_.pic_w, (int)kWidth); + ASSERT_EQ(y4m_.pic_h, (int)kHeight); + ASSERT_EQ(img()->d_w, kWidth); + ASSERT_EQ(img()->d_h, kHeight); + ASSERT_EQ(y4m_.bit_depth, bit_depth); + ASSERT_EQ(y4m_.aom_fmt, fmt); + if (fmt == AOM_IMG_FMT_I420 || fmt == AOM_IMG_FMT_I42016) { + ASSERT_EQ(y4m_.bps, (int)y4m_.bit_depth * 3 / 2); + ASSERT_EQ(img()->x_chroma_shift, 1U); + ASSERT_EQ(img()->y_chroma_shift, 1U); + } + if (fmt == AOM_IMG_FMT_I422 || fmt == AOM_IMG_FMT_I42216) { + ASSERT_EQ(y4m_.bps, (int)y4m_.bit_depth * 2); + ASSERT_EQ(img()->x_chroma_shift, 1U); + ASSERT_EQ(img()->y_chroma_shift, 0U); + } + if (fmt == AOM_IMG_FMT_I444 || fmt == AOM_IMG_FMT_I44416) { + ASSERT_EQ(y4m_.bps, (int)y4m_.bit_depth * 3); + ASSERT_EQ(img()->x_chroma_shift, 0U); + ASSERT_EQ(img()->y_chroma_shift, 0U); + } + } + + // Checks MD5 of the raw frame data + void Md5Check(const string &expected_md5) { + ASSERT_TRUE(input_file_ != NULL); + libaom_test::MD5 md5; + for (unsigned int i = start_; i < limit_; i++) { + md5.Add(img()); + Next(); + } + ASSERT_EQ(string(md5.Get()), expected_md5); + } +}; + +TEST_P(Y4mVideoSourceTest, SourceTest) { + const Y4mTestParam t = GetParam(); + Init(t.filename, kFrames); + HeaderChecks(t.bit_depth, t.format); + Md5Check(t.md5raw); +} + +INSTANTIATE_TEST_CASE_P(C, Y4mVideoSourceTest, + ::testing::ValuesIn(kY4mTestVectors)); + +class Y4mVideoWriteTest : public Y4mVideoSourceTest { + protected: + Y4mVideoWriteTest() : tmpfile_(NULL) {} + + virtual ~Y4mVideoWriteTest() { + delete tmpfile_; + input_file_ = NULL; + } + + void ReplaceInputFile(FILE *input_file) { + CloseSource(); + frame_ = 0; + input_file_ = input_file; + rewind(input_file_); + ReadSourceToStart(); + } + + // Writes out a y4m file and then reads it back + void WriteY4mAndReadBack() { + ASSERT_TRUE(input_file_ != NULL); + char buf[Y4M_BUFFER_SIZE] = { 0 }; + const struct AvxRational framerate = { y4m_.fps_n, y4m_.fps_d }; + tmpfile_ = new libaom_test::TempOutFile; + ASSERT_TRUE(tmpfile_->file() != NULL); + y4m_write_file_header(buf, sizeof(buf), kWidth, kHeight, &framerate, + y4m_.aom_fmt, y4m_.bit_depth); + fputs(buf, tmpfile_->file()); + for (unsigned int i = start_; i < limit_; i++) { + y4m_write_frame_header(buf, sizeof(buf)); + fputs(buf, tmpfile_->file()); + write_image_file(img(), tmpfile_->file()); + Next(); + } + ReplaceInputFile(tmpfile_->file()); + } + + virtual void Init(const std::string &file_name, int limit) { + Y4mVideoSourceTest::Init(file_name, limit); + WriteY4mAndReadBack(); + } + libaom_test::TempOutFile *tmpfile_; +}; + +TEST_P(Y4mVideoWriteTest, WriteTest) { + const Y4mTestParam t = GetParam(); + Init(t.filename, kFrames); + HeaderChecks(t.bit_depth, t.format); + Md5Check(t.md5raw); +} + +INSTANTIATE_TEST_CASE_P(C, Y4mVideoWriteTest, + ::testing::ValuesIn(kY4mTestVectors)); +} // namespace diff --git a/third_party/aom/test/y4m_video_source.h b/third_party/aom/test/y4m_video_source.h new file mode 100644 index 0000000000..2279d7970e --- /dev/null +++ b/third_party/aom/test/y4m_video_source.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef TEST_Y4M_VIDEO_SOURCE_H_ +#define TEST_Y4M_VIDEO_SOURCE_H_ +#include +#include + +#include "test/video_source.h" +#include "./y4minput.h" + +namespace libaom_test { + +// This class extends VideoSource to allow parsing of raw yv12 +// so that we can do actual file encodes. +class Y4mVideoSource : public VideoSource { + public: + Y4mVideoSource(const std::string &file_name, unsigned int start, int limit) + : file_name_(file_name), input_file_(NULL), img_(new aom_image_t()), + start_(start), limit_(limit), frame_(0), framerate_numerator_(0), + framerate_denominator_(0), y4m_() {} + + virtual ~Y4mVideoSource() { + aom_img_free(img_.get()); + CloseSource(); + } + + virtual void OpenSource() { + CloseSource(); + input_file_ = OpenTestDataFile(file_name_); + ASSERT_TRUE(input_file_ != NULL) << "Input file open failed. Filename: " + << file_name_; + } + + virtual void ReadSourceToStart() { + ASSERT_TRUE(input_file_ != NULL); + ASSERT_FALSE(y4m_input_open(&y4m_, input_file_, NULL, 0, 0)); + framerate_numerator_ = y4m_.fps_n; + framerate_denominator_ = y4m_.fps_d; + frame_ = 0; + for (unsigned int i = 0; i < start_; i++) { + Next(); + } + FillFrame(); + } + + virtual void Begin() { + OpenSource(); + ReadSourceToStart(); + } + + virtual void Next() { + ++frame_; + FillFrame(); + } + + virtual aom_image_t *img() const { + return (frame_ < limit_) ? img_.get() : NULL; + } + + // Models a stream where Timebase = 1/FPS, so pts == frame. + virtual aom_codec_pts_t pts() const { return frame_; } + + virtual unsigned long duration() const { return 1; } + + virtual aom_rational_t timebase() const { + const aom_rational_t t = { framerate_denominator_, framerate_numerator_ }; + return t; + } + + virtual unsigned int frame() const { return frame_; } + + virtual unsigned int limit() const { return limit_; } + + virtual void FillFrame() { + ASSERT_TRUE(input_file_ != NULL); + // Read a frame from input_file. + y4m_input_fetch_frame(&y4m_, input_file_, img_.get()); + } + + // Swap buffers with another y4m source. This allows reading a new frame + // while keeping the old frame around. A whole Y4mSource is required and + // not just a aom_image_t because of how the y4m reader manipulates + // aom_image_t internals, + void SwapBuffers(Y4mVideoSource *other) { + std::swap(other->y4m_.dst_buf, y4m_.dst_buf); + aom_image_t *tmp; + tmp = other->img_.release(); + other->img_.reset(img_.release()); + img_.reset(tmp); + } + + protected: + void CloseSource() { + y4m_input_close(&y4m_); + y4m_ = y4m_input(); + if (input_file_ != NULL) { + fclose(input_file_); + input_file_ = NULL; + } + } + + std::string file_name_; + FILE *input_file_; + testing::internal::scoped_ptr img_; + unsigned int start_; + unsigned int limit_; + unsigned int frame_; + int framerate_numerator_; + int framerate_denominator_; + y4m_input y4m_; +}; + +} // namespace libaom_test + +#endif // TEST_Y4M_VIDEO_SOURCE_H_ diff --git a/third_party/aom/test/yuv_video_source.h b/third_party/aom/test/yuv_video_source.h new file mode 100644 index 0000000000..9ff76a8d8f --- /dev/null +++ b/third_party/aom/test/yuv_video_source.h @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef TEST_YUV_VIDEO_SOURCE_H_ +#define TEST_YUV_VIDEO_SOURCE_H_ + +#include +#include +#include + +#include "test/video_source.h" +#include "aom/aom_image.h" + +namespace libaom_test { + +// This class extends VideoSource to allow parsing of raw YUV +// formats of various color sampling and bit-depths so that we can +// do actual file encodes. +class YUVVideoSource : public VideoSource { + public: + YUVVideoSource(const std::string &file_name, aom_img_fmt format, + unsigned int width, unsigned int height, int rate_numerator, + int rate_denominator, unsigned int start, int limit) + : file_name_(file_name), input_file_(NULL), img_(NULL), start_(start), + limit_(limit), frame_(0), width_(0), height_(0), + format_(AOM_IMG_FMT_NONE), framerate_numerator_(rate_numerator), + framerate_denominator_(rate_denominator) { + // This initializes format_, raw_size_, width_, height_ and allocates img. + SetSize(width, height, format); + } + + virtual ~YUVVideoSource() { + aom_img_free(img_); + if (input_file_) fclose(input_file_); + } + + virtual void Begin() { + if (input_file_) fclose(input_file_); + input_file_ = OpenTestDataFile(file_name_); + ASSERT_TRUE(input_file_ != NULL) << "Input file open failed. Filename: " + << file_name_; + if (start_) + fseek(input_file_, static_cast(raw_size_) * start_, SEEK_SET); + + frame_ = start_; + FillFrame(); + } + + virtual void Next() { + ++frame_; + FillFrame(); + } + + virtual aom_image_t *img() const { return (frame_ < limit_) ? img_ : NULL; } + + // Models a stream where Timebase = 1/FPS, so pts == frame. + virtual aom_codec_pts_t pts() const { return frame_; } + + virtual unsigned long duration() const { return 1; } + + virtual aom_rational_t timebase() const { + const aom_rational_t t = { framerate_denominator_, framerate_numerator_ }; + return t; + } + + virtual unsigned int frame() const { return frame_; } + + virtual unsigned int limit() const { return limit_; } + + virtual void SetSize(unsigned int width, unsigned int height, + aom_img_fmt format) { + if (width != width_ || height != height_ || format != format_) { + aom_img_free(img_); + img_ = aom_img_alloc(NULL, format, width, height, 1); + ASSERT_TRUE(img_ != NULL); + width_ = width; + height_ = height; + format_ = format; + switch (format) { + case AOM_IMG_FMT_I420: raw_size_ = width * height * 3 / 2; break; + case AOM_IMG_FMT_I422: raw_size_ = width * height * 2; break; + case AOM_IMG_FMT_I440: raw_size_ = width * height * 2; break; + case AOM_IMG_FMT_I444: raw_size_ = width * height * 3; break; + case AOM_IMG_FMT_I42016: raw_size_ = width * height * 3; break; + case AOM_IMG_FMT_I42216: raw_size_ = width * height * 4; break; + case AOM_IMG_FMT_I44016: raw_size_ = width * height * 4; break; + case AOM_IMG_FMT_I44416: raw_size_ = width * height * 6; break; + default: ASSERT_TRUE(0); + } + } + } + + virtual void FillFrame() { + ASSERT_TRUE(input_file_ != NULL); + // Read a frame from input_file. + if (fread(img_->img_data, raw_size_, 1, input_file_) == 0) { + limit_ = frame_; + } + } + + protected: + std::string file_name_; + FILE *input_file_; + aom_image_t *img_; + size_t raw_size_; + unsigned int start_; + unsigned int limit_; + unsigned int frame_; + unsigned int width_; + unsigned int height_; + aom_img_fmt format_; + int framerate_numerator_; + int framerate_denominator_; +}; + +} // namespace libaom_test + +#endif // TEST_YUV_VIDEO_SOURCE_H_ diff --git a/third_party/aom/third_party/fastfeat/LICENSE b/third_party/aom/third_party/fastfeat/LICENSE new file mode 100644 index 0000000000..f347008d6e --- /dev/null +++ b/third_party/aom/third_party/fastfeat/LICENSE @@ -0,0 +1,30 @@ +Copyright (c) 2006, 2008 Edward Rosten +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + + *Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + *Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + *Neither the name of the University of Cambridge nor the names of + its contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/aom/third_party/fastfeat/README.libvpx b/third_party/aom/third_party/fastfeat/README.libvpx new file mode 100644 index 0000000000..2edd6e7bfb --- /dev/null +++ b/third_party/aom/third_party/fastfeat/README.libvpx @@ -0,0 +1,38 @@ +URL: https://github.com/edrosten/fast-C-src +Version: 391d5e939eb1545d24c10533d7de424db8d9c191 +License: BSD +License File: LICENSE + +Description: +Library to compute FAST features with non-maximum suppression. + +The files are valid C and C++ code, and have no special requirements for +compiling, and they do not depend on any libraries. Just compile them along with +the rest of your project. + +To use the functions, #include "fast.h" + +The corner detectors have the following prototype (where X is 9, 10, 11 or 12): + +xy* fastX_detect_nonmax(const unsigned char * data, int xsize, int ysize, int stride, int threshold, int* numcorners) + +Where xy is the following simple struct typedef: + +typedef struct +{ + int x, y; +} xy; + +The image is passed in as a block of data and dimensions, and the list of +corners is returned as an array of xy structs, and an integer (numcorners) +with the number of corners returned. The data can be deallocated with free(). +Nonmaximal suppression is performed on the corners. Note that the stride +is the number of bytes between rows. If your image has no padding, then this +is the same as xsize. + +The detection, scoring and nonmaximal suppression are available as individual +functions. To see how to use the individual functions, see fast.c + +Local Modifications: +Add lines to turn off clang formatting for these files +Remove Fast 10, 11 and 12 diff --git a/third_party/aom/third_party/fastfeat/fast.c b/third_party/aom/third_party/fastfeat/fast.c new file mode 100644 index 0000000000..0d7efc154d --- /dev/null +++ b/third_party/aom/third_party/fastfeat/fast.c @@ -0,0 +1,22 @@ +// clang-format off +#include +#include "fast.h" + + +xy* fast9_detect_nonmax(const byte* im, int xsize, int ysize, int stride, int b, int* ret_num_corners) +{ + xy* corners; + int num_corners; + int* scores; + xy* nonmax; + + corners = fast9_detect(im, xsize, ysize, stride, b, &num_corners); + scores = fast9_score(im, stride, corners, num_corners, b); + nonmax = nonmax_suppression(corners, scores, num_corners, ret_num_corners); + + free(corners); + free(scores); + + return nonmax; +} +// clang-format on diff --git a/third_party/aom/third_party/fastfeat/fast.h b/third_party/aom/third_party/fastfeat/fast.h new file mode 100644 index 0000000000..a00730e3d3 --- /dev/null +++ b/third_party/aom/third_party/fastfeat/fast.h @@ -0,0 +1,20 @@ +// clang-format off +#ifndef FAST_H +#define FAST_H + +typedef struct { int x, y; } xy; +typedef unsigned char byte; + +int fast9_corner_score(const byte* p, const int pixel[], int bstart); + +xy* fast9_detect(const byte* im, int xsize, int ysize, int stride, int b, int* ret_num_corners); + +int* fast9_score(const byte* i, int stride, xy* corners, int num_corners, int b); + +xy* fast9_detect_nonmax(const byte* im, int xsize, int ysize, int stride, int b, int* ret_num_corners); + +xy* nonmax_suppression(const xy* corners, const int* scores, int num_corners, int* ret_num_nonmax); + + +#endif +// clang-format on diff --git a/third_party/aom/third_party/fastfeat/fast_9.c b/third_party/aom/third_party/fastfeat/fast_9.c new file mode 100644 index 0000000000..36aee19fed --- /dev/null +++ b/third_party/aom/third_party/fastfeat/fast_9.c @@ -0,0 +1,5911 @@ +// clang-format off +/*This is mechanically generated code*/ +#include + +typedef struct { int x, y; } xy; +typedef unsigned char byte; + +int fast9_corner_score(const byte* p, const int pixel[], int bstart) +{ + int bmin = bstart; + int bmax = 255; + int b = (bmax + bmin)/2; + + /*Compute the score using binary search*/ + for(;;) + { + int cb = *p + b; + int c_b= *p - b; + + + if( p[pixel[0]] > cb) + if( p[pixel[1]] > cb) + if( p[pixel[2]] > cb) + if( p[pixel[3]] > cb) + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + goto is_a_corner; + else + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[7]] < c_b) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[14]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[6]] < c_b) + if( p[pixel[15]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[13]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[14]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[13]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[5]] < c_b) + if( p[pixel[14]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[12]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[13]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[14]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[6]] < c_b) + goto is_a_corner; + else + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[12]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[6]] < c_b) + goto is_a_corner; + else + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[4]] < c_b) + if( p[pixel[13]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[11]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[12]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[13]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[5]] < c_b) + goto is_a_corner; + else + if( p[pixel[14]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[11]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[5]] < c_b) + goto is_a_corner; + else + if( p[pixel[14]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[3]] < c_b) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[10]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[4]] < c_b) + goto is_a_corner; + else + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[10]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[4]] < c_b) + goto is_a_corner; + else + if( p[pixel[13]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[2]] < c_b) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[3]] > cb) + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[9]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[4]] < c_b) + if( p[pixel[3]] < c_b) + goto is_a_corner; + else + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[3]] > cb) + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[9]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[4]] < c_b) + if( p[pixel[3]] < c_b) + goto is_a_corner; + else + if( p[pixel[12]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[1]] < c_b) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[3]] > cb) + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[2]] > cb) + if( p[pixel[3]] > cb) + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[8]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[4]] < c_b) + if( p[pixel[3]] < c_b) + if( p[pixel[2]] < c_b) + goto is_a_corner; + else + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[3]] > cb) + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[2]] > cb) + if( p[pixel[3]] > cb) + if( p[pixel[4]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[8]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[4]] < c_b) + if( p[pixel[3]] < c_b) + if( p[pixel[2]] < c_b) + goto is_a_corner; + else + if( p[pixel[11]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[0]] < c_b) + if( p[pixel[1]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[4]] > cb) + if( p[pixel[3]] > cb) + if( p[pixel[2]] > cb) + goto is_a_corner; + else + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[4]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[3]] < c_b) + if( p[pixel[4]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[2]] < c_b) + if( p[pixel[3]] < c_b) + if( p[pixel[4]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[1]] < c_b) + if( p[pixel[2]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[4]] > cb) + if( p[pixel[3]] > cb) + goto is_a_corner; + else + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[4]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[3]] < c_b) + if( p[pixel[4]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[2]] < c_b) + if( p[pixel[3]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[4]] > cb) + goto is_a_corner; + else + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[4]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[3]] < c_b) + if( p[pixel[4]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[5]] > cb) + goto is_a_corner; + else + if( p[pixel[14]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[13]] < c_b) + if( p[pixel[11]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[12]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[4]] < c_b) + if( p[pixel[5]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[6]] > cb) + goto is_a_corner; + else + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[14]] < c_b) + if( p[pixel[12]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[13]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[6]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[5]] < c_b) + if( p[pixel[6]] > cb) + if( p[pixel[15]] < c_b) + if( p[pixel[13]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[14]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[6]] < c_b) + if( p[pixel[7]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + goto is_a_corner; + else + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[13]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[12]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[6]] > cb) + goto is_a_corner; + else + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[5]] > cb) + goto is_a_corner; + else + if( p[pixel[14]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[4]] > cb) + goto is_a_corner; + else + if( p[pixel[13]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[4]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[9]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[4]] > cb) + if( p[pixel[3]] > cb) + goto is_a_corner; + else + if( p[pixel[12]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[4]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[3]] < c_b) + if( p[pixel[4]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[8]] > cb) + if( p[pixel[7]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[10]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[4]] > cb) + if( p[pixel[3]] > cb) + if( p[pixel[2]] > cb) + goto is_a_corner; + else + if( p[pixel[11]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[4]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[3]] < c_b) + if( p[pixel[4]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[2]] < c_b) + if( p[pixel[3]] < c_b) + if( p[pixel[4]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[7]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[7]] > cb) + if( p[pixel[8]] > cb) + if( p[pixel[9]] > cb) + if( p[pixel[6]] > cb) + if( p[pixel[5]] > cb) + if( p[pixel[4]] > cb) + if( p[pixel[3]] > cb) + if( p[pixel[2]] > cb) + if( p[pixel[1]] > cb) + goto is_a_corner; + else + if( p[pixel[10]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] > cb) + if( p[pixel[11]] > cb) + if( p[pixel[12]] > cb) + if( p[pixel[13]] > cb) + if( p[pixel[14]] > cb) + if( p[pixel[15]] > cb) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else if( p[pixel[7]] < c_b) + if( p[pixel[8]] < c_b) + if( p[pixel[9]] < c_b) + if( p[pixel[6]] < c_b) + if( p[pixel[5]] < c_b) + if( p[pixel[4]] < c_b) + if( p[pixel[3]] < c_b) + if( p[pixel[2]] < c_b) + if( p[pixel[1]] < c_b) + goto is_a_corner; + else + if( p[pixel[10]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + if( p[pixel[10]] < c_b) + if( p[pixel[11]] < c_b) + if( p[pixel[12]] < c_b) + if( p[pixel[13]] < c_b) + if( p[pixel[14]] < c_b) + if( p[pixel[15]] < c_b) + goto is_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + else + goto is_not_a_corner; + + is_a_corner: + bmin=b; + goto end_if; + + is_not_a_corner: + bmax=b; + goto end_if; + + end_if: + + if(bmin == bmax - 1 || bmin == bmax) + return bmin; + b = (bmin + bmax) / 2; + } +} + +static void make_offsets(int pixel[], int row_stride) +{ + pixel[0] = 0 + row_stride * 3; + pixel[1] = 1 + row_stride * 3; + pixel[2] = 2 + row_stride * 2; + pixel[3] = 3 + row_stride * 1; + pixel[4] = 3 + row_stride * 0; + pixel[5] = 3 + row_stride * -1; + pixel[6] = 2 + row_stride * -2; + pixel[7] = 1 + row_stride * -3; + pixel[8] = 0 + row_stride * -3; + pixel[9] = -1 + row_stride * -3; + pixel[10] = -2 + row_stride * -2; + pixel[11] = -3 + row_stride * -1; + pixel[12] = -3 + row_stride * 0; + pixel[13] = -3 + row_stride * 1; + pixel[14] = -2 + row_stride * 2; + pixel[15] = -1 + row_stride * 3; +} + + + +int* fast9_score(const byte* i, int stride, xy* corners, int num_corners, int b) +{ + int* scores = (int*)malloc(sizeof(int)* num_corners); + int n; + + int pixel[16]; + make_offsets(pixel, stride); + + for(n=0; n < num_corners; n++) + scores[n] = fast9_corner_score(i + corners[n].y*stride + corners[n].x, pixel, b); + + return scores; +} + + +xy* fast9_detect(const byte* im, int xsize, int ysize, int stride, int b, int* ret_num_corners) +{ + int num_corners=0; + xy* ret_corners; + int rsize=512; + int pixel[16]; + int x, y; + + ret_corners = (xy*)malloc(sizeof(xy)*rsize); + make_offsets(pixel, stride); + + for(y=3; y < ysize - 3; y++) + for(x=3; x < xsize - 3; x++) + { + const byte* p = im + y*stride + x; + + int cb = *p + b; + int c_b= *p - b; + if(p[pixel[0]] > cb) + if(p[pixel[1]] > cb) + if(p[pixel[2]] > cb) + if(p[pixel[3]] > cb) + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + {} + else + if(p[pixel[15]] > cb) + {} + else + continue; + else if(p[pixel[7]] < c_b) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else if(p[pixel[14]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else if(p[pixel[6]] < c_b) + if(p[pixel[15]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + {} + else + continue; + else if(p[pixel[13]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[14]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else if(p[pixel[13]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[5]] < c_b) + if(p[pixel[14]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[15]] > cb) + {} + else + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[12]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[13]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[14]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[6]] < c_b) + {} + else + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[12]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[6]] < c_b) + {} + else + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[4]] < c_b) + if(p[pixel[13]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[11]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[12]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[13]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[5]] < c_b) + {} + else + if(p[pixel[14]] < c_b) + {} + else + continue; + else + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[11]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[5]] < c_b) + {} + else + if(p[pixel[14]] < c_b) + {} + else + continue; + else + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[3]] < c_b) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[10]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[4]] < c_b) + {} + else + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + {} + else + continue; + else + continue; + else + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[10]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[4]] < c_b) + {} + else + if(p[pixel[13]] < c_b) + {} + else + continue; + else + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + {} + else + continue; + else + continue; + else + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[2]] < c_b) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[3]] > cb) + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[9]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[4]] < c_b) + if(p[pixel[3]] < c_b) + {} + else + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + {} + else + continue; + else + continue; + else + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[3]] > cb) + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[9]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[4]] < c_b) + if(p[pixel[3]] < c_b) + {} + else + if(p[pixel[12]] < c_b) + {} + else + continue; + else + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + {} + else + continue; + else + continue; + else + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[1]] < c_b) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + {} + else + continue; + else + continue; + else + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[3]] > cb) + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[2]] > cb) + if(p[pixel[3]] > cb) + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[8]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[4]] < c_b) + if(p[pixel[3]] < c_b) + if(p[pixel[2]] < c_b) + {} + else + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + {} + else + continue; + else + continue; + else + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + {} + else + continue; + else + continue; + else + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[3]] > cb) + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[2]] > cb) + if(p[pixel[3]] > cb) + if(p[pixel[4]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[8]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[4]] < c_b) + if(p[pixel[3]] < c_b) + if(p[pixel[2]] < c_b) + {} + else + if(p[pixel[11]] < c_b) + {} + else + continue; + else + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + {} + else + continue; + else + continue; + else + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[0]] < c_b) + if(p[pixel[1]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[4]] > cb) + if(p[pixel[3]] > cb) + if(p[pixel[2]] > cb) + {} + else + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + {} + else + continue; + else + continue; + else + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + {} + else + continue; + else + continue; + else + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[4]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[3]] < c_b) + if(p[pixel[4]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[2]] < c_b) + if(p[pixel[3]] < c_b) + if(p[pixel[4]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[1]] < c_b) + if(p[pixel[2]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[4]] > cb) + if(p[pixel[3]] > cb) + {} + else + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + {} + else + continue; + else + continue; + else + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[4]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[3]] < c_b) + if(p[pixel[4]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[2]] < c_b) + if(p[pixel[3]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[4]] > cb) + {} + else + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + {} + else + continue; + else + continue; + else + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[4]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[3]] < c_b) + if(p[pixel[4]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[5]] > cb) + {} + else + if(p[pixel[14]] > cb) + {} + else + continue; + else + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[13]] < c_b) + if(p[pixel[11]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[12]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[4]] < c_b) + if(p[pixel[5]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[6]] > cb) + {} + else + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[14]] < c_b) + if(p[pixel[12]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[13]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[6]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[5]] < c_b) + if(p[pixel[6]] > cb) + if(p[pixel[15]] < c_b) + if(p[pixel[13]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[14]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + {} + else + continue; + else + continue; + else + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[6]] < c_b) + if(p[pixel[7]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + {} + else + if(p[pixel[15]] < c_b) + {} + else + continue; + else + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + if(p[pixel[13]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[12]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[6]] > cb) + {} + else + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[11]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[5]] > cb) + {} + else + if(p[pixel[14]] > cb) + {} + else + continue; + else + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[4]] > cb) + {} + else + if(p[pixel[13]] > cb) + {} + else + continue; + else + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + {} + else + continue; + else + continue; + else + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[4]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[9]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[4]] > cb) + if(p[pixel[3]] > cb) + {} + else + if(p[pixel[12]] > cb) + {} + else + continue; + else + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + {} + else + continue; + else + continue; + else + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[4]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[3]] < c_b) + if(p[pixel[4]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[8]] > cb) + if(p[pixel[7]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[10]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[4]] > cb) + if(p[pixel[3]] > cb) + if(p[pixel[2]] > cb) + {} + else + if(p[pixel[11]] > cb) + {} + else + continue; + else + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + {} + else + continue; + else + continue; + else + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + {} + else + continue; + else + continue; + else + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[4]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[3]] < c_b) + if(p[pixel[4]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[2]] < c_b) + if(p[pixel[3]] < c_b) + if(p[pixel[4]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[7]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[7]] > cb) + if(p[pixel[8]] > cb) + if(p[pixel[9]] > cb) + if(p[pixel[6]] > cb) + if(p[pixel[5]] > cb) + if(p[pixel[4]] > cb) + if(p[pixel[3]] > cb) + if(p[pixel[2]] > cb) + if(p[pixel[1]] > cb) + {} + else + if(p[pixel[10]] > cb) + {} + else + continue; + else + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + {} + else + continue; + else + continue; + else + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] > cb) + if(p[pixel[11]] > cb) + if(p[pixel[12]] > cb) + if(p[pixel[13]] > cb) + if(p[pixel[14]] > cb) + if(p[pixel[15]] > cb) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else if(p[pixel[7]] < c_b) + if(p[pixel[8]] < c_b) + if(p[pixel[9]] < c_b) + if(p[pixel[6]] < c_b) + if(p[pixel[5]] < c_b) + if(p[pixel[4]] < c_b) + if(p[pixel[3]] < c_b) + if(p[pixel[2]] < c_b) + if(p[pixel[1]] < c_b) + {} + else + if(p[pixel[10]] < c_b) + {} + else + continue; + else + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + {} + else + continue; + else + continue; + else + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + if(p[pixel[10]] < c_b) + if(p[pixel[11]] < c_b) + if(p[pixel[12]] < c_b) + if(p[pixel[13]] < c_b) + if(p[pixel[14]] < c_b) + if(p[pixel[15]] < c_b) + {} + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + else + continue; + if(num_corners == rsize) + { + rsize*=2; + ret_corners = (xy*)realloc(ret_corners, sizeof(xy)*rsize); + } + ret_corners[num_corners].x = x; + ret_corners[num_corners].y = y; + num_corners++; + + } + + *ret_num_corners = num_corners; + return ret_corners; + +} + +// clang-format on diff --git a/third_party/aom/third_party/fastfeat/nonmax.c b/third_party/aom/third_party/fastfeat/nonmax.c new file mode 100644 index 0000000000..0438c4dc18 --- /dev/null +++ b/third_party/aom/third_party/fastfeat/nonmax.c @@ -0,0 +1,121 @@ +// clang-format off +#include +#include "fast.h" + + +#define Compare(X, Y) ((X)>=(Y)) + +xy* nonmax_suppression(const xy* corners, const int* scores, int num_corners, int* ret_num_nonmax) +{ + int num_nonmax=0; + int last_row; + int* row_start; + int i, j; + xy* ret_nonmax; + const int sz = (int)num_corners; + + /*Point above points (roughly) to the pixel above the one of interest, if there + is a feature there.*/ + int point_above = 0; + int point_below = 0; + + + if(num_corners < 1) + { + *ret_num_nonmax = 0; + return 0; + } + + ret_nonmax = (xy*)malloc(num_corners * sizeof(xy)); + + /* Find where each row begins + (the corners are output in raster scan order). A beginning of -1 signifies + that there are no corners on that row. */ + last_row = corners[num_corners-1].y; + row_start = (int*)malloc((last_row+1)*sizeof(int)); + + for(i=0; i < last_row+1; i++) + row_start[i] = -1; + + { + int prev_row = -1; + for(i=0; i< num_corners; i++) + if(corners[i].y != prev_row) + { + row_start[corners[i].y] = i; + prev_row = corners[i].y; + } + } + + + + for(i=0; i < sz; i++) + { + int score = scores[i]; + xy pos = corners[i]; + + /*Check left */ + if(i > 0) + if(corners[i-1].x == pos.x-1 && corners[i-1].y == pos.y && Compare(scores[i-1], score)) + continue; + + /*Check right*/ + if(i < (sz - 1)) + if(corners[i+1].x == pos.x+1 && corners[i+1].y == pos.y && Compare(scores[i+1], score)) + continue; + + /*Check above (if there is a valid row above)*/ + if(pos.y > 0) + if (row_start[pos.y - 1] != -1) + { + /*Make sure that current point_above is one + row above.*/ + if(corners[point_above].y < pos.y - 1) + point_above = row_start[pos.y-1]; + + /*Make point_above point to the first of the pixels above the current point, + if it exists.*/ + for(; corners[point_above].y < pos.y && corners[point_above].x < pos.x - 1; point_above++) + {} + + + for(j=point_above; corners[j].y < pos.y && corners[j].x <= pos.x + 1; j++) + { + int x = corners[j].x; + if( (x == pos.x - 1 || x ==pos.x || x == pos.x+1) && Compare(scores[j], score)) + goto cont; + } + + } + + /*Check below (if there is anything below)*/ + if(pos.y >= 0) + if (pos.y != last_row && row_start[pos.y + 1] != -1 && point_below < sz) /*Nothing below*/ + { + if(corners[point_below].y < pos.y + 1) + point_below = row_start[pos.y+1]; + + /* Make point below point to one of the pixels belowthe current point, if it + exists.*/ + for(; point_below < sz && corners[point_below].y == pos.y+1 && corners[point_below].x < pos.x - 1; point_below++) + {} + + for(j=point_below; j < sz && corners[j].y == pos.y+1 && corners[j].x <= pos.x + 1; j++) + { + int x = corners[j].x; + if( (x == pos.x - 1 || x ==pos.x || x == pos.x+1) && Compare(scores[j],score)) + goto cont; + } + } + + ret_nonmax[num_nonmax++] = corners[i]; +cont: + ; + } + + free(row_start); + *ret_num_nonmax = num_nonmax; + return ret_nonmax; +} + +// clang-format on diff --git a/third_party/aom/third_party/googletest/README.libaom b/third_party/aom/third_party/googletest/README.libaom new file mode 100644 index 0000000000..a53d7e0085 --- /dev/null +++ b/third_party/aom/third_party/googletest/README.libaom @@ -0,0 +1,24 @@ +URL: https://github.com/google/googletest +Version: 1.8.0 +License: BSD +License File: LICENSE + +Description: +Google's framework for writing C++ tests on a variety of platforms +(Linux, Mac OS X, Windows, Windows CE, Symbian, etc). Based on the +xUnit architecture. Supports automatic test discovery, a rich set of +assertions, user-defined assertions, death tests, fatal and non-fatal +failures, various options for running the tests, and XML test report +generation. + +Local Modifications: +- Remove everything but: + googletest-release-1.8.0/googletest/ + cmake/ + include/ + src/ + CHANGES + CMakelists.txt + CONTRIBUTORS + LICENSE + README.md diff --git a/third_party/aom/third_party/googletest/gtest.mk b/third_party/aom/third_party/googletest/gtest.mk new file mode 100644 index 0000000000..fc4dbdc240 --- /dev/null +++ b/third_party/aom/third_party/googletest/gtest.mk @@ -0,0 +1 @@ +GTEST_SRCS-yes += googletest/src/googletest/src/gtest-all.cc diff --git a/third_party/aom/third_party/googletest/src/googletest/CHANGES b/third_party/aom/third_party/googletest/src/googletest/CHANGES new file mode 100644 index 0000000000..0552132421 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/CHANGES @@ -0,0 +1,157 @@ +Changes for 1.7.0: + +* New feature: death tests are supported on OpenBSD and in iOS + simulator now. +* New feature: Google Test now implements a protocol to allow + a test runner to detect that a test program has exited + prematurely and report it as a failure (before it would be + falsely reported as a success if the exit code is 0). +* New feature: Test::RecordProperty() can now be used outside of the + lifespan of a test method, in which case it will be attributed to + the current test case or the test program in the XML report. +* New feature (potentially breaking): --gtest_list_tests now prints + the type parameters and value parameters for each test. +* Improvement: char pointers and char arrays are now escaped properly + in failure messages. +* Improvement: failure summary in XML reports now includes file and + line information. +* Improvement: the XML element now has a timestamp attribute. +* Improvement: When --gtest_filter is specified, XML report now doesn't + contain information about tests that are filtered out. +* Fixed the bug where long --gtest_filter flag values are truncated in + death tests. +* Potentially breaking change: RUN_ALL_TESTS() is now implemented as a + function instead of a macro in order to work better with Clang. +* Compatibility fixes with C++ 11 and various platforms. +* Bug/warning fixes. + +Changes for 1.6.0: + +* New feature: ADD_FAILURE_AT() for reporting a test failure at the + given source location -- useful for writing testing utilities. +* New feature: the universal value printer is moved from Google Mock + to Google Test. +* New feature: type parameters and value parameters are reported in + the XML report now. +* A gtest_disable_pthreads CMake option. +* Colored output works in GNU Screen sessions now. +* Parameters of value-parameterized tests are now printed in the + textual output. +* Failures from ad hoc test assertions run before RUN_ALL_TESTS() are + now correctly reported. +* Arguments of ASSERT_XY and EXPECT_XY no longer need to support << to + ostream. +* More complete handling of exceptions. +* GTEST_ASSERT_XY can be used instead of ASSERT_XY in case the latter + name is already used by another library. +* --gtest_catch_exceptions is now true by default, allowing a test + program to continue after an exception is thrown. +* Value-parameterized test fixtures can now derive from Test and + WithParamInterface separately, easing conversion of legacy tests. +* Death test messages are clearly marked to make them more + distinguishable from other messages. +* Compatibility fixes for Android, Google Native Client, MinGW, HP UX, + PowerPC, Lucid autotools, libCStd, Sun C++, Borland C++ Builder (Code Gear), + IBM XL C++ (Visual Age C++), and C++0x. +* Bug fixes and implementation clean-ups. +* Potentially incompatible changes: disables the harmful 'make install' + command in autotools. + +Changes for 1.5.0: + + * New feature: assertions can be safely called in multiple threads + where the pthreads library is available. + * New feature: predicates used inside EXPECT_TRUE() and friends + can now generate custom failure messages. + * New feature: Google Test can now be compiled as a DLL. + * New feature: fused source files are included. + * New feature: prints help when encountering unrecognized Google Test flags. + * Experimental feature: CMake build script (requires CMake 2.6.4+). + * Experimental feature: the Pump script for meta programming. + * double values streamed to an assertion are printed with enough precision + to differentiate any two different values. + * Google Test now works on Solaris and AIX. + * Build and test script improvements. + * Bug fixes and implementation clean-ups. + + Potentially breaking changes: + + * Stopped supporting VC++ 7.1 with exceptions disabled. + * Dropped support for 'make install'. + +Changes for 1.4.0: + + * New feature: the event listener API + * New feature: test shuffling + * New feature: the XML report format is closer to junitreport and can + be parsed by Hudson now. + * New feature: when a test runs under Visual Studio, its failures are + integrated in the IDE. + * New feature: /MD(d) versions of VC++ projects. + * New feature: elapsed time for the tests is printed by default. + * New feature: comes with a TR1 tuple implementation such that Boost + is no longer needed for Combine(). + * New feature: EXPECT_DEATH_IF_SUPPORTED macro and friends. + * New feature: the Xcode project can now produce static gtest + libraries in addition to a framework. + * Compatibility fixes for Solaris, Cygwin, minGW, Windows Mobile, + Symbian, gcc, and C++Builder. + * Bug fixes and implementation clean-ups. + +Changes for 1.3.0: + + * New feature: death tests on Windows, Cygwin, and Mac. + * New feature: ability to use Google Test assertions in other testing + frameworks. + * New feature: ability to run disabled test via + --gtest_also_run_disabled_tests. + * New feature: the --help flag for printing the usage. + * New feature: access to Google Test flag values in user code. + * New feature: a script that packs Google Test into one .h and one + .cc file for easy deployment. + * New feature: support for distributing test functions to multiple + machines (requires support from the test runner). + * Bug fixes and implementation clean-ups. + +Changes for 1.2.1: + + * Compatibility fixes for Linux IA-64 and IBM z/OS. + * Added support for using Boost and other TR1 implementations. + * Changes to the build scripts to support upcoming release of Google C++ + Mocking Framework. + * Added Makefile to the distribution package. + * Improved build instructions in README. + +Changes for 1.2.0: + + * New feature: value-parameterized tests. + * New feature: the ASSERT/EXPECT_(NON)FATAL_FAILURE(_ON_ALL_THREADS) + macros. + * Changed the XML report format to match JUnit/Ant's. + * Added tests to the Xcode project. + * Added scons/SConscript for building with SCons. + * Added src/gtest-all.cc for building Google Test from a single file. + * Fixed compatibility with Solaris and z/OS. + * Enabled running Python tests on systems with python 2.3 installed, + e.g. Mac OS X 10.4. + * Bug fixes. + +Changes for 1.1.0: + + * New feature: type-parameterized tests. + * New feature: exception assertions. + * New feature: printing elapsed time of tests. + * Improved the robustness of death tests. + * Added an Xcode project and samples. + * Adjusted the output format on Windows to be understandable by Visual Studio. + * Minor bug fixes. + +Changes for 1.0.1: + + * Added project files for Visual Studio 7.1. + * Fixed issues with compiling on Mac OS X. + * Fixed issues with compiling on Cygwin. + +Changes for 1.0.0: + + * Initial Open Source release of Google Test diff --git a/third_party/aom/third_party/googletest/src/googletest/CMakeLists.txt b/third_party/aom/third_party/googletest/src/googletest/CMakeLists.txt new file mode 100644 index 0000000000..621d0f0421 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/CMakeLists.txt @@ -0,0 +1,286 @@ +######################################################################## +# CMake build script for Google Test. +# +# To run the tests for Google Test itself on Linux, use 'make test' or +# ctest. You can select which tests to run using 'ctest -R regex'. +# For more options, run 'ctest --help'. + +# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to +# make it prominent in the GUI. +option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)." OFF) + +# When other libraries are using a shared version of runtime libraries, +# Google Test also has to use one. +option( + gtest_force_shared_crt + "Use shared (DLL) run-time lib even when Google Test is built as static lib." + OFF) + +option(gtest_build_tests "Build all of gtest's own tests." OFF) + +option(gtest_build_samples "Build gtest's sample programs." OFF) + +option(gtest_disable_pthreads "Disable uses of pthreads in gtest." OFF) + +option( + gtest_hide_internal_symbols + "Build gtest with internal symbols hidden in shared libraries." + OFF) + +# Defines pre_project_set_up_hermetic_build() and set_up_hermetic_build(). +include(cmake/hermetic_build.cmake OPTIONAL) + +if (COMMAND pre_project_set_up_hermetic_build) + pre_project_set_up_hermetic_build() +endif() + +######################################################################## +# +# Project-wide settings + +# Name of the project. +# +# CMake files in this project can refer to the root source directory +# as ${gtest_SOURCE_DIR} and to the root binary directory as +# ${gtest_BINARY_DIR}. +# Language "C" is required for find_package(Threads). +project(gtest CXX C) +cmake_minimum_required(VERSION 2.6.2) + +if (COMMAND set_up_hermetic_build) + set_up_hermetic_build() +endif() + +if (gtest_hide_internal_symbols) + set(CMAKE_CXX_VISIBILITY_PRESET hidden) + set(CMAKE_VISIBILITY_INLINES_HIDDEN 1) +endif() + +# Define helper functions and macros used by Google Test. +include(cmake/internal_utils.cmake) + +config_compiler_and_linker() # Defined in internal_utils.cmake. + +# Where Google Test's .h files can be found. +include_directories( + ${gtest_SOURCE_DIR}/include + ${gtest_SOURCE_DIR}) + +# Where Google Test's libraries can be found. +link_directories(${gtest_BINARY_DIR}/src) + +# Summary of tuple support for Microsoft Visual Studio: +# Compiler version(MS) version(cmake) Support +# ---------- ----------- -------------- ----------------------------- +# <= VS 2010 <= 10 <= 1600 Use Google Tests's own tuple. +# VS 2012 11 1700 std::tr1::tuple + _VARIADIC_MAX=10 +# VS 2013 12 1800 std::tr1::tuple +if (MSVC AND MSVC_VERSION EQUAL 1700) + add_definitions(/D _VARIADIC_MAX=10) +endif() + +######################################################################## +# +# Defines the gtest & gtest_main libraries. User tests should link +# with one of them. + +# Google Test libraries. We build them using more strict warnings than what +# are used for other targets, to ensure that gtest can be compiled by a user +# aggressive about warnings. +cxx_library(gtest "${cxx_strict}" src/gtest-all.cc) +cxx_library(gtest_main "${cxx_strict}" src/gtest_main.cc) +target_link_libraries(gtest_main gtest) + +# If the CMake version supports it, attach header directory information +# to the targets for when we are part of a parent build (ie being pulled +# in via add_subdirectory() rather than being a standalone build). +if (DEFINED CMAKE_VERSION AND NOT "${CMAKE_VERSION}" VERSION_LESS "2.8.11") + target_include_directories(gtest INTERFACE "${gtest_SOURCE_DIR}/include") + target_include_directories(gtest_main INTERFACE "${gtest_SOURCE_DIR}/include") +endif() + +######################################################################## +# +# Install rules +install(TARGETS gtest gtest_main + DESTINATION lib) +install(DIRECTORY ${gtest_SOURCE_DIR}/include/gtest + DESTINATION include) + +######################################################################## +# +# Samples on how to link user tests with gtest or gtest_main. +# +# They are not built by default. To build them, set the +# gtest_build_samples option to ON. You can do it by running ccmake +# or specifying the -Dgtest_build_samples=ON flag when running cmake. + +if (gtest_build_samples) + cxx_executable(sample1_unittest samples gtest_main samples/sample1.cc) + cxx_executable(sample2_unittest samples gtest_main samples/sample2.cc) + cxx_executable(sample3_unittest samples gtest_main) + cxx_executable(sample4_unittest samples gtest_main samples/sample4.cc) + cxx_executable(sample5_unittest samples gtest_main samples/sample1.cc) + cxx_executable(sample6_unittest samples gtest_main) + cxx_executable(sample7_unittest samples gtest_main) + cxx_executable(sample8_unittest samples gtest_main) + cxx_executable(sample9_unittest samples gtest) + cxx_executable(sample10_unittest samples gtest) +endif() + +######################################################################## +# +# Google Test's own tests. +# +# You can skip this section if you aren't interested in testing +# Google Test itself. +# +# The tests are not built by default. To build them, set the +# gtest_build_tests option to ON. You can do it by running ccmake +# or specifying the -Dgtest_build_tests=ON flag when running cmake. + +if (gtest_build_tests) + # This must be set in the root directory for the tests to be run by + # 'make test' or ctest. + enable_testing() + + ############################################################ + # C++ tests built with standard compiler flags. + + cxx_test(gtest-death-test_test gtest_main) + cxx_test(gtest_environment_test gtest) + cxx_test(gtest-filepath_test gtest_main) + cxx_test(gtest-linked_ptr_test gtest_main) + cxx_test(gtest-listener_test gtest_main) + cxx_test(gtest_main_unittest gtest_main) + cxx_test(gtest-message_test gtest_main) + cxx_test(gtest_no_test_unittest gtest) + cxx_test(gtest-options_test gtest_main) + cxx_test(gtest-param-test_test gtest + test/gtest-param-test2_test.cc) + cxx_test(gtest-port_test gtest_main) + cxx_test(gtest_pred_impl_unittest gtest_main) + cxx_test(gtest_premature_exit_test gtest + test/gtest_premature_exit_test.cc) + cxx_test(gtest-printers_test gtest_main) + cxx_test(gtest_prod_test gtest_main + test/production.cc) + cxx_test(gtest_repeat_test gtest) + cxx_test(gtest_sole_header_test gtest_main) + cxx_test(gtest_stress_test gtest) + cxx_test(gtest-test-part_test gtest_main) + cxx_test(gtest_throw_on_failure_ex_test gtest) + cxx_test(gtest-typed-test_test gtest_main + test/gtest-typed-test2_test.cc) + cxx_test(gtest_unittest gtest_main) + cxx_test(gtest-unittest-api_test gtest) + + ############################################################ + # C++ tests built with non-standard compiler flags. + + # MSVC 7.1 does not support STL with exceptions disabled. + if (NOT MSVC OR MSVC_VERSION GREATER 1310) + cxx_library(gtest_no_exception "${cxx_no_exception}" + src/gtest-all.cc) + cxx_library(gtest_main_no_exception "${cxx_no_exception}" + src/gtest-all.cc src/gtest_main.cc) + endif() + cxx_library(gtest_main_no_rtti "${cxx_no_rtti}" + src/gtest-all.cc src/gtest_main.cc) + + cxx_test_with_flags(gtest-death-test_ex_nocatch_test + "${cxx_exception} -DGTEST_ENABLE_CATCH_EXCEPTIONS_=0" + gtest test/gtest-death-test_ex_test.cc) + cxx_test_with_flags(gtest-death-test_ex_catch_test + "${cxx_exception} -DGTEST_ENABLE_CATCH_EXCEPTIONS_=1" + gtest test/gtest-death-test_ex_test.cc) + + cxx_test_with_flags(gtest_no_rtti_unittest "${cxx_no_rtti}" + gtest_main_no_rtti test/gtest_unittest.cc) + + cxx_shared_library(gtest_dll "${cxx_default}" + src/gtest-all.cc src/gtest_main.cc) + + cxx_executable_with_flags(gtest_dll_test_ "${cxx_default}" + gtest_dll test/gtest_all_test.cc) + set_target_properties(gtest_dll_test_ + PROPERTIES + COMPILE_DEFINITIONS "GTEST_LINKED_AS_SHARED_LIBRARY=1") + + if (NOT MSVC OR MSVC_VERSION LESS 1600) # 1600 is Visual Studio 2010. + # Visual Studio 2010, 2012, and 2013 define symbols in std::tr1 that + # conflict with our own definitions. Therefore using our own tuple does not + # work on those compilers. + cxx_library(gtest_main_use_own_tuple "${cxx_use_own_tuple}" + src/gtest-all.cc src/gtest_main.cc) + + cxx_test_with_flags(gtest-tuple_test "${cxx_use_own_tuple}" + gtest_main_use_own_tuple test/gtest-tuple_test.cc) + + cxx_test_with_flags(gtest_use_own_tuple_test "${cxx_use_own_tuple}" + gtest_main_use_own_tuple + test/gtest-param-test_test.cc test/gtest-param-test2_test.cc) + endif() + + ############################################################ + # Python tests. + + cxx_executable(gtest_break_on_failure_unittest_ test gtest) + py_test(gtest_break_on_failure_unittest) + + # Visual Studio .NET 2003 does not support STL with exceptions disabled. + if (NOT MSVC OR MSVC_VERSION GREATER 1310) # 1310 is Visual Studio .NET 2003 + cxx_executable_with_flags( + gtest_catch_exceptions_no_ex_test_ + "${cxx_no_exception}" + gtest_main_no_exception + test/gtest_catch_exceptions_test_.cc) + endif() + + cxx_executable_with_flags( + gtest_catch_exceptions_ex_test_ + "${cxx_exception}" + gtest_main + test/gtest_catch_exceptions_test_.cc) + py_test(gtest_catch_exceptions_test) + + cxx_executable(gtest_color_test_ test gtest) + py_test(gtest_color_test) + + cxx_executable(gtest_env_var_test_ test gtest) + py_test(gtest_env_var_test) + + cxx_executable(gtest_filter_unittest_ test gtest) + py_test(gtest_filter_unittest) + + cxx_executable(gtest_help_test_ test gtest_main) + py_test(gtest_help_test) + + cxx_executable(gtest_list_tests_unittest_ test gtest) + py_test(gtest_list_tests_unittest) + + cxx_executable(gtest_output_test_ test gtest) + py_test(gtest_output_test) + + cxx_executable(gtest_shuffle_test_ test gtest) + py_test(gtest_shuffle_test) + + # MSVC 7.1 does not support STL with exceptions disabled. + if (NOT MSVC OR MSVC_VERSION GREATER 1310) + cxx_executable(gtest_throw_on_failure_test_ test gtest_no_exception) + set_target_properties(gtest_throw_on_failure_test_ + PROPERTIES + COMPILE_FLAGS "${cxx_no_exception}") + py_test(gtest_throw_on_failure_test) + endif() + + cxx_executable(gtest_uninitialized_test_ test gtest) + py_test(gtest_uninitialized_test) + + cxx_executable(gtest_xml_outfile1_test_ test gtest_main) + cxx_executable(gtest_xml_outfile2_test_ test gtest_main) + py_test(gtest_xml_outfiles_test) + + cxx_executable(gtest_xml_output_unittest_ test gtest) + py_test(gtest_xml_output_unittest) +endif() diff --git a/third_party/aom/third_party/googletest/src/googletest/CONTRIBUTORS b/third_party/aom/third_party/googletest/src/googletest/CONTRIBUTORS new file mode 100644 index 0000000000..feae2fc044 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/CONTRIBUTORS @@ -0,0 +1,37 @@ +# This file contains a list of people who've made non-trivial +# contribution to the Google C++ Testing Framework project. People +# who commit code to the project are encouraged to add their names +# here. Please keep the list sorted by first names. + +Ajay Joshi +Balázs Dán +Bharat Mediratta +Chandler Carruth +Chris Prince +Chris Taylor +Dan Egnor +Eric Roman +Hady Zalek +Jeffrey Yasskin +Jói Sigurðsson +Keir Mierle +Keith Ray +Kenton Varda +Manuel Klimek +Markus Heule +Mika Raento +Miklós Fazekas +Pasi Valminen +Patrick Hanna +Patrick Riley +Peter Kaminski +Preston Jackson +Rainer Klaffenboeck +Russ Cox +Russ Rufer +Sean Mcafee +Sigurður Ásgeirsson +Tracy Bialik +Vadim Berman +Vlad Losev +Zhanyong Wan diff --git a/third_party/aom/third_party/googletest/src/googletest/LICENSE b/third_party/aom/third_party/googletest/src/googletest/LICENSE new file mode 100644 index 0000000000..1941a11f8c --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/LICENSE @@ -0,0 +1,28 @@ +Copyright 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/aom/third_party/googletest/src/googletest/README.md b/third_party/aom/third_party/googletest/src/googletest/README.md new file mode 100644 index 0000000000..edd4408054 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/README.md @@ -0,0 +1,280 @@ + +### Generic Build Instructions ### + +#### Setup #### + +To build Google Test and your tests that use it, you need to tell your +build system where to find its headers and source files. The exact +way to do it depends on which build system you use, and is usually +straightforward. + +#### Build #### + +Suppose you put Google Test in directory `${GTEST_DIR}`. To build it, +create a library build target (or a project as called by Visual Studio +and Xcode) to compile + + ${GTEST_DIR}/src/gtest-all.cc + +with `${GTEST_DIR}/include` in the system header search path and `${GTEST_DIR}` +in the normal header search path. Assuming a Linux-like system and gcc, +something like the following will do: + + g++ -isystem ${GTEST_DIR}/include -I${GTEST_DIR} \ + -pthread -c ${GTEST_DIR}/src/gtest-all.cc + ar -rv libgtest.a gtest-all.o + +(We need `-pthread` as Google Test uses threads.) + +Next, you should compile your test source file with +`${GTEST_DIR}/include` in the system header search path, and link it +with gtest and any other necessary libraries: + + g++ -isystem ${GTEST_DIR}/include -pthread path/to/your_test.cc libgtest.a \ + -o your_test + +As an example, the make/ directory contains a Makefile that you can +use to build Google Test on systems where GNU make is available +(e.g. Linux, Mac OS X, and Cygwin). It doesn't try to build Google +Test's own tests. Instead, it just builds the Google Test library and +a sample test. You can use it as a starting point for your own build +script. + +If the default settings are correct for your environment, the +following commands should succeed: + + cd ${GTEST_DIR}/make + make + ./sample1_unittest + +If you see errors, try to tweak the contents of `make/Makefile` to make +them go away. There are instructions in `make/Makefile` on how to do +it. + +### Using CMake ### + +Google Test comes with a CMake build script ( +[CMakeLists.txt](CMakeLists.txt)) that can be used on a wide range of platforms ("C" stands for +cross-platform.). If you don't have CMake installed already, you can +download it for free from . + +CMake works by generating native makefiles or build projects that can +be used in the compiler environment of your choice. The typical +workflow starts with: + + mkdir mybuild # Create a directory to hold the build output. + cd mybuild + cmake ${GTEST_DIR} # Generate native build scripts. + +If you want to build Google Test's samples, you should replace the +last command with + + cmake -Dgtest_build_samples=ON ${GTEST_DIR} + +If you are on a \*nix system, you should now see a Makefile in the +current directory. Just type 'make' to build gtest. + +If you use Windows and have Visual Studio installed, a `gtest.sln` file +and several `.vcproj` files will be created. You can then build them +using Visual Studio. + +On Mac OS X with Xcode installed, a `.xcodeproj` file will be generated. + +### Legacy Build Scripts ### + +Before settling on CMake, we have been providing hand-maintained build +projects/scripts for Visual Studio, Xcode, and Autotools. While we +continue to provide them for convenience, they are not actively +maintained any more. We highly recommend that you follow the +instructions in the previous two sections to integrate Google Test +with your existing build system. + +If you still need to use the legacy build scripts, here's how: + +The msvc\ folder contains two solutions with Visual C++ projects. +Open the `gtest.sln` or `gtest-md.sln` file using Visual Studio, and you +are ready to build Google Test the same way you build any Visual +Studio project. Files that have names ending with -md use DLL +versions of Microsoft runtime libraries (the /MD or the /MDd compiler +option). Files without that suffix use static versions of the runtime +libraries (the /MT or the /MTd option). Please note that one must use +the same option to compile both gtest and the test code. If you use +Visual Studio 2005 or above, we recommend the -md version as /MD is +the default for new projects in these versions of Visual Studio. + +On Mac OS X, open the `gtest.xcodeproj` in the `xcode/` folder using +Xcode. Build the "gtest" target. The universal binary framework will +end up in your selected build directory (selected in the Xcode +"Preferences..." -> "Building" pane and defaults to xcode/build). +Alternatively, at the command line, enter: + + xcodebuild + +This will build the "Release" configuration of gtest.framework in your +default build location. See the "xcodebuild" man page for more +information about building different configurations and building in +different locations. + +If you wish to use the Google Test Xcode project with Xcode 4.x and +above, you need to either: + + * update the SDK configuration options in xcode/Config/General.xconfig. + Comment options `SDKROOT`, `MACOS_DEPLOYMENT_TARGET`, and `GCC_VERSION`. If + you choose this route you lose the ability to target earlier versions + of MacOS X. + * Install an SDK for an earlier version. This doesn't appear to be + supported by Apple, but has been reported to work + (http://stackoverflow.com/questions/5378518). + +### Tweaking Google Test ### + +Google Test can be used in diverse environments. The default +configuration may not work (or may not work well) out of the box in +some environments. However, you can easily tweak Google Test by +defining control macros on the compiler command line. Generally, +these macros are named like `GTEST_XYZ` and you define them to either 1 +or 0 to enable or disable a certain feature. + +We list the most frequently used macros below. For a complete list, +see file [include/gtest/internal/gtest-port.h](include/gtest/internal/gtest-port.h). + +### Choosing a TR1 Tuple Library ### + +Some Google Test features require the C++ Technical Report 1 (TR1) +tuple library, which is not yet available with all compilers. The +good news is that Google Test implements a subset of TR1 tuple that's +enough for its own need, and will automatically use this when the +compiler doesn't provide TR1 tuple. + +Usually you don't need to care about which tuple library Google Test +uses. However, if your project already uses TR1 tuple, you need to +tell Google Test to use the same TR1 tuple library the rest of your +project uses, or the two tuple implementations will clash. To do +that, add + + -DGTEST_USE_OWN_TR1_TUPLE=0 + +to the compiler flags while compiling Google Test and your tests. If +you want to force Google Test to use its own tuple library, just add + + -DGTEST_USE_OWN_TR1_TUPLE=1 + +to the compiler flags instead. + +If you don't want Google Test to use tuple at all, add + + -DGTEST_HAS_TR1_TUPLE=0 + +and all features using tuple will be disabled. + +### Multi-threaded Tests ### + +Google Test is thread-safe where the pthread library is available. +After `#include "gtest/gtest.h"`, you can check the `GTEST_IS_THREADSAFE` +macro to see whether this is the case (yes if the macro is `#defined` to +1, no if it's undefined.). + +If Google Test doesn't correctly detect whether pthread is available +in your environment, you can force it with + + -DGTEST_HAS_PTHREAD=1 + +or + + -DGTEST_HAS_PTHREAD=0 + +When Google Test uses pthread, you may need to add flags to your +compiler and/or linker to select the pthread library, or you'll get +link errors. If you use the CMake script or the deprecated Autotools +script, this is taken care of for you. If you use your own build +script, you'll need to read your compiler and linker's manual to +figure out what flags to add. + +### As a Shared Library (DLL) ### + +Google Test is compact, so most users can build and link it as a +static library for the simplicity. You can choose to use Google Test +as a shared library (known as a DLL on Windows) if you prefer. + +To compile *gtest* as a shared library, add + + -DGTEST_CREATE_SHARED_LIBRARY=1 + +to the compiler flags. You'll also need to tell the linker to produce +a shared library instead - consult your linker's manual for how to do +it. + +To compile your *tests* that use the gtest shared library, add + + -DGTEST_LINKED_AS_SHARED_LIBRARY=1 + +to the compiler flags. + +Note: while the above steps aren't technically necessary today when +using some compilers (e.g. GCC), they may become necessary in the +future, if we decide to improve the speed of loading the library (see + for details). Therefore you are +recommended to always add the above flags when using Google Test as a +shared library. Otherwise a future release of Google Test may break +your build script. + +### Avoiding Macro Name Clashes ### + +In C++, macros don't obey namespaces. Therefore two libraries that +both define a macro of the same name will clash if you `#include` both +definitions. In case a Google Test macro clashes with another +library, you can force Google Test to rename its macro to avoid the +conflict. + +Specifically, if both Google Test and some other code define macro +FOO, you can add + + -DGTEST_DONT_DEFINE_FOO=1 + +to the compiler flags to tell Google Test to change the macro's name +from `FOO` to `GTEST_FOO`. Currently `FOO` can be `FAIL`, `SUCCEED`, +or `TEST`. For example, with `-DGTEST_DONT_DEFINE_TEST=1`, you'll +need to write + + GTEST_TEST(SomeTest, DoesThis) { ... } + +instead of + + TEST(SomeTest, DoesThis) { ... } + +in order to define a test. + +## Developing Google Test ## + +This section discusses how to make your own changes to Google Test. + +### Testing Google Test Itself ### + +To make sure your changes work as intended and don't break existing +functionality, you'll want to compile and run Google Test's own tests. +For that you can use CMake: + + mkdir mybuild + cd mybuild + cmake -Dgtest_build_tests=ON ${GTEST_DIR} + +Make sure you have Python installed, as some of Google Test's tests +are written in Python. If the cmake command complains about not being +able to find Python (`Could NOT find PythonInterp (missing: +PYTHON_EXECUTABLE)`), try telling it explicitly where your Python +executable can be found: + + cmake -DPYTHON_EXECUTABLE=path/to/python -Dgtest_build_tests=ON ${GTEST_DIR} + +Next, you can build Google Test and all of its own tests. On \*nix, +this is usually done by 'make'. To run the tests, do + + make test + +All tests should pass. + +Normally you don't need to worry about regenerating the source files, +unless you need to modify them. In that case, you should modify the +corresponding .pump files instead and run the pump.py Python script to +regenerate them. You can find pump.py in the [scripts/](scripts/) directory. +Read the [Pump manual](docs/PumpManual.md) for how to use it. diff --git a/third_party/aom/third_party/googletest/src/googletest/cmake/internal_utils.cmake b/third_party/aom/third_party/googletest/src/googletest/cmake/internal_utils.cmake new file mode 100644 index 0000000000..777b91ed4b --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/cmake/internal_utils.cmake @@ -0,0 +1,254 @@ +# Defines functions and macros useful for building Google Test and +# Google Mock. +# +# Note: +# +# - This file will be run twice when building Google Mock (once via +# Google Test's CMakeLists.txt, and once via Google Mock's). +# Therefore it shouldn't have any side effects other than defining +# the functions and macros. +# +# - The functions/macros defined in this file may depend on Google +# Test and Google Mock's option() definitions, and thus must be +# called *after* the options have been defined. + +# Tweaks CMake's default compiler/linker settings to suit Google Test's needs. +# +# This must be a macro(), as inside a function string() can only +# update variables in the function scope. +macro(fix_default_compiler_settings_) + if (MSVC) + # For MSVC, CMake sets certain flags to defaults we want to override. + # This replacement code is taken from sample in the CMake Wiki at + # http://www.cmake.org/Wiki/CMake_FAQ#Dynamic_Replace. + foreach (flag_var + CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE + CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO) + if (NOT BUILD_SHARED_LIBS AND NOT gtest_force_shared_crt) + # When Google Test is built as a shared library, it should also use + # shared runtime libraries. Otherwise, it may end up with multiple + # copies of runtime library data in different modules, resulting in + # hard-to-find crashes. When it is built as a static library, it is + # preferable to use CRT as static libraries, as we don't have to rely + # on CRT DLLs being available. CMake always defaults to using shared + # CRT libraries, so we override that default here. + string(REPLACE "/MD" "-MT" ${flag_var} "${${flag_var}}") + endif() + + # We prefer more strict warning checking for building Google Test. + # Replaces /W3 with /W4 in defaults. + string(REPLACE "/W3" "/W4" ${flag_var} "${${flag_var}}") + endforeach() + endif() +endmacro() + +# Defines the compiler/linker flags used to build Google Test and +# Google Mock. You can tweak these definitions to suit your need. A +# variable's value is empty before it's explicitly assigned to. +macro(config_compiler_and_linker) + if (NOT gtest_disable_pthreads) + # Defines CMAKE_USE_PTHREADS_INIT and CMAKE_THREAD_LIBS_INIT. + find_package(Threads) + endif() + + fix_default_compiler_settings_() + if (MSVC) + # Newlines inside flags variables break CMake's NMake generator. + # TODO(vladl@google.com): Add -RTCs and -RTCu to debug builds. + set(cxx_base_flags "-GS -W4 -WX -wd4251 -wd4275 -nologo -J -Zi") + if (MSVC_VERSION LESS 1400) # 1400 is Visual Studio 2005 + # Suppress spurious warnings MSVC 7.1 sometimes issues. + # Forcing value to bool. + set(cxx_base_flags "${cxx_base_flags} -wd4800") + # Copy constructor and assignment operator could not be generated. + set(cxx_base_flags "${cxx_base_flags} -wd4511 -wd4512") + # Compatibility warnings not applicable to Google Test. + # Resolved overload was found by argument-dependent lookup. + set(cxx_base_flags "${cxx_base_flags} -wd4675") + endif() + if (MSVC_VERSION LESS 1500) # 1500 is Visual Studio 2008 + # Conditional expression is constant. + # When compiling with /W4, we get several instances of C4127 + # (Conditional expression is constant). In our code, we disable that + # warning on a case-by-case basis. However, on Visual Studio 2005, + # the warning fires on std::list. Therefore on that compiler and earlier, + # we disable the warning project-wide. + set(cxx_base_flags "${cxx_base_flags} -wd4127") + endif() + if (NOT (MSVC_VERSION LESS 1700)) # 1700 is Visual Studio 2012. + # Suppress "unreachable code" warning on VS 2012 and later. + # http://stackoverflow.com/questions/3232669 explains the issue. + set(cxx_base_flags "${cxx_base_flags} -wd4702") + endif() + if (NOT (MSVC_VERSION GREATER 1900)) # 1900 is Visual Studio 2015 + # BigObj required for tests. + set(cxx_base_flags "${cxx_base_flags} -bigobj") + endif() + + set(cxx_base_flags "${cxx_base_flags} -D_UNICODE -DUNICODE -DWIN32 -D_WIN32") + set(cxx_base_flags "${cxx_base_flags} -DSTRICT -DWIN32_LEAN_AND_MEAN") + set(cxx_exception_flags "-EHsc -D_HAS_EXCEPTIONS=1") + set(cxx_no_exception_flags "-D_HAS_EXCEPTIONS=0") + set(cxx_no_rtti_flags "-GR-") + elseif (CMAKE_COMPILER_IS_GNUCXX) + set(cxx_base_flags "-Wall -Wshadow") + set(cxx_exception_flags "-fexceptions") + set(cxx_no_exception_flags "-fno-exceptions") + # Until version 4.3.2, GCC doesn't define a macro to indicate + # whether RTTI is enabled. Therefore we define GTEST_HAS_RTTI + # explicitly. + set(cxx_no_rtti_flags "-fno-rtti -DGTEST_HAS_RTTI=0") + set(cxx_strict_flags + "-Wextra -Wno-unused-parameter -Wno-missing-field-initializers") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "SunPro") + set(cxx_exception_flags "-features=except") + # Sun Pro doesn't provide macros to indicate whether exceptions and + # RTTI are enabled, so we define GTEST_HAS_* explicitly. + set(cxx_no_exception_flags "-features=no%except -DGTEST_HAS_EXCEPTIONS=0") + set(cxx_no_rtti_flags "-features=no%rtti -DGTEST_HAS_RTTI=0") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "VisualAge" OR + CMAKE_CXX_COMPILER_ID STREQUAL "XL") + # CMake 2.8 changes Visual Age's compiler ID to "XL". + set(cxx_exception_flags "-qeh") + set(cxx_no_exception_flags "-qnoeh") + # Until version 9.0, Visual Age doesn't define a macro to indicate + # whether RTTI is enabled. Therefore we define GTEST_HAS_RTTI + # explicitly. + set(cxx_no_rtti_flags "-qnortti -DGTEST_HAS_RTTI=0") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "HP") + set(cxx_base_flags "-AA -mt") + set(cxx_exception_flags "-DGTEST_HAS_EXCEPTIONS=1") + set(cxx_no_exception_flags "+noeh -DGTEST_HAS_EXCEPTIONS=0") + # RTTI can not be disabled in HP aCC compiler. + set(cxx_no_rtti_flags "") + endif() + + if (CMAKE_USE_PTHREADS_INIT) # The pthreads library is available and allowed. + set(cxx_base_flags "${cxx_base_flags} -DGTEST_HAS_PTHREAD=1") + else() + set(cxx_base_flags "${cxx_base_flags} -DGTEST_HAS_PTHREAD=0") + endif() + + # For building gtest's own tests and samples. + set(cxx_exception "${CMAKE_CXX_FLAGS} ${cxx_base_flags} ${cxx_exception_flags}") + set(cxx_no_exception + "${CMAKE_CXX_FLAGS} ${cxx_base_flags} ${cxx_no_exception_flags}") + set(cxx_default "${cxx_exception}") + set(cxx_no_rtti "${cxx_default} ${cxx_no_rtti_flags}") + set(cxx_use_own_tuple "${cxx_default} -DGTEST_USE_OWN_TR1_TUPLE=1") + + # For building the gtest libraries. + set(cxx_strict "${cxx_default} ${cxx_strict_flags}") +endmacro() + +# Defines the gtest & gtest_main libraries. User tests should link +# with one of them. +function(cxx_library_with_type name type cxx_flags) + # type can be either STATIC or SHARED to denote a static or shared library. + # ARGN refers to additional arguments after 'cxx_flags'. + add_library(${name} ${type} ${ARGN}) + set_target_properties(${name} + PROPERTIES + COMPILE_FLAGS "${cxx_flags}") + if (BUILD_SHARED_LIBS OR type STREQUAL "SHARED") + set_target_properties(${name} + PROPERTIES + COMPILE_DEFINITIONS "GTEST_CREATE_SHARED_LIBRARY=1") + endif() + if (CMAKE_USE_PTHREADS_INIT) + target_link_libraries(${name} ${CMAKE_THREAD_LIBS_INIT}) + endif() +endfunction() + +######################################################################## +# +# Helper functions for creating build targets. + +function(cxx_shared_library name cxx_flags) + cxx_library_with_type(${name} SHARED "${cxx_flags}" ${ARGN}) +endfunction() + +function(cxx_library name cxx_flags) + cxx_library_with_type(${name} "" "${cxx_flags}" ${ARGN}) +endfunction() + +# cxx_executable_with_flags(name cxx_flags libs srcs...) +# +# creates a named C++ executable that depends on the given libraries and +# is built from the given source files with the given compiler flags. +function(cxx_executable_with_flags name cxx_flags libs) + add_executable(${name} ${ARGN}) + if (cxx_flags) + set_target_properties(${name} + PROPERTIES + COMPILE_FLAGS "${cxx_flags}") + endif() + if (BUILD_SHARED_LIBS) + set_target_properties(${name} + PROPERTIES + COMPILE_DEFINITIONS "GTEST_LINKED_AS_SHARED_LIBRARY=1") + endif() + # To support mixing linking in static and dynamic libraries, link each + # library in with an extra call to target_link_libraries. + foreach (lib "${libs}") + target_link_libraries(${name} ${lib}) + endforeach() +endfunction() + +# cxx_executable(name dir lib srcs...) +# +# creates a named target that depends on the given libs and is built +# from the given source files. dir/name.cc is implicitly included in +# the source file list. +function(cxx_executable name dir libs) + cxx_executable_with_flags( + ${name} "${cxx_default}" "${libs}" "${dir}/${name}.cc" ${ARGN}) +endfunction() + +# Sets PYTHONINTERP_FOUND and PYTHON_EXECUTABLE. +find_package(PythonInterp) + +# cxx_test_with_flags(name cxx_flags libs srcs...) +# +# creates a named C++ test that depends on the given libs and is built +# from the given source files with the given compiler flags. +function(cxx_test_with_flags name cxx_flags libs) + cxx_executable_with_flags(${name} "${cxx_flags}" "${libs}" ${ARGN}) + add_test(${name} ${name}) +endfunction() + +# cxx_test(name libs srcs...) +# +# creates a named test target that depends on the given libs and is +# built from the given source files. Unlike cxx_test_with_flags, +# test/name.cc is already implicitly included in the source file list. +function(cxx_test name libs) + cxx_test_with_flags("${name}" "${cxx_default}" "${libs}" + "test/${name}.cc" ${ARGN}) +endfunction() + +# py_test(name) +# +# creates a Python test with the given name whose main module is in +# test/name.py. It does nothing if Python is not installed. +function(py_test name) + # We are not supporting Python tests on Linux yet as they consider + # all Linux environments to be google3 and try to use google3 features. + if (PYTHONINTERP_FOUND) + # ${CMAKE_BINARY_DIR} is known at configuration time, so we can + # directly bind it from cmake. ${CTEST_CONFIGURATION_TYPE} is known + # only at ctest runtime (by calling ctest -c ), so + # we have to escape $ to delay variable substitution here. + if (${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION} GREATER 3.1) + add_test( + NAME ${name} + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/${name}.py + --build_dir=${CMAKE_CURRENT_BINARY_DIR}/$) + else (${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION} GREATER 3.1) + add_test( + ${name} + ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/${name}.py + --build_dir=${CMAKE_CURRENT_BINARY_DIR}/\${CTEST_CONFIGURATION_TYPE}) + endif (${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION} GREATER 3.1) + endif() +endfunction() diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-death-test.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-death-test.h new file mode 100644 index 0000000000..957a69c6a9 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-death-test.h @@ -0,0 +1,294 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines the public API for death tests. It is +// #included by gtest.h so a user doesn't need to include this +// directly. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ + +#include "gtest/internal/gtest-death-test-internal.h" + +namespace testing { + +// This flag controls the style of death tests. Valid values are "threadsafe", +// meaning that the death test child process will re-execute the test binary +// from the start, running only a single death test, or "fast", +// meaning that the child process will execute the test logic immediately +// after forking. +GTEST_DECLARE_string_(death_test_style); + +#if GTEST_HAS_DEATH_TEST + +namespace internal { + +// Returns a Boolean value indicating whether the caller is currently +// executing in the context of the death test child process. Tools such as +// Valgrind heap checkers may need this to modify their behavior in death +// tests. IMPORTANT: This is an internal utility. Using it may break the +// implementation of death tests. User code MUST NOT use it. +GTEST_API_ bool InDeathTestChild(); + +} // namespace internal + +// The following macros are useful for writing death tests. + +// Here's what happens when an ASSERT_DEATH* or EXPECT_DEATH* is +// executed: +// +// 1. It generates a warning if there is more than one active +// thread. This is because it's safe to fork() or clone() only +// when there is a single thread. +// +// 2. The parent process clone()s a sub-process and runs the death +// test in it; the sub-process exits with code 0 at the end of the +// death test, if it hasn't exited already. +// +// 3. The parent process waits for the sub-process to terminate. +// +// 4. The parent process checks the exit code and error message of +// the sub-process. +// +// Examples: +// +// ASSERT_DEATH(server.SendMessage(56, "Hello"), "Invalid port number"); +// for (int i = 0; i < 5; i++) { +// EXPECT_DEATH(server.ProcessRequest(i), +// "Invalid request .* in ProcessRequest()") +// << "Failed to die on request " << i; +// } +// +// ASSERT_EXIT(server.ExitNow(), ::testing::ExitedWithCode(0), "Exiting"); +// +// bool KilledBySIGHUP(int exit_code) { +// return WIFSIGNALED(exit_code) && WTERMSIG(exit_code) == SIGHUP; +// } +// +// ASSERT_EXIT(client.HangUpServer(), KilledBySIGHUP, "Hanging up!"); +// +// On the regular expressions used in death tests: +// +// On POSIX-compliant systems (*nix), we use the library, +// which uses the POSIX extended regex syntax. +// +// On other platforms (e.g. Windows), we only support a simple regex +// syntax implemented as part of Google Test. This limited +// implementation should be enough most of the time when writing +// death tests; though it lacks many features you can find in PCRE +// or POSIX extended regex syntax. For example, we don't support +// union ("x|y"), grouping ("(xy)"), brackets ("[xy]"), and +// repetition count ("x{5,7}"), among others. +// +// Below is the syntax that we do support. We chose it to be a +// subset of both PCRE and POSIX extended regex, so it's easy to +// learn wherever you come from. In the following: 'A' denotes a +// literal character, period (.), or a single \\ escape sequence; +// 'x' and 'y' denote regular expressions; 'm' and 'n' are for +// natural numbers. +// +// c matches any literal character c +// \\d matches any decimal digit +// \\D matches any character that's not a decimal digit +// \\f matches \f +// \\n matches \n +// \\r matches \r +// \\s matches any ASCII whitespace, including \n +// \\S matches any character that's not a whitespace +// \\t matches \t +// \\v matches \v +// \\w matches any letter, _, or decimal digit +// \\W matches any character that \\w doesn't match +// \\c matches any literal character c, which must be a punctuation +// . matches any single character except \n +// A? matches 0 or 1 occurrences of A +// A* matches 0 or many occurrences of A +// A+ matches 1 or many occurrences of A +// ^ matches the beginning of a string (not that of each line) +// $ matches the end of a string (not that of each line) +// xy matches x followed by y +// +// If you accidentally use PCRE or POSIX extended regex features +// not implemented by us, you will get a run-time failure. In that +// case, please try to rewrite your regular expression within the +// above syntax. +// +// This implementation is *not* meant to be as highly tuned or robust +// as a compiled regex library, but should perform well enough for a +// death test, which already incurs significant overhead by launching +// a child process. +// +// Known caveats: +// +// A "threadsafe" style death test obtains the path to the test +// program from argv[0] and re-executes it in the sub-process. For +// simplicity, the current implementation doesn't search the PATH +// when launching the sub-process. This means that the user must +// invoke the test program via a path that contains at least one +// path separator (e.g. path/to/foo_test and +// /absolute/path/to/bar_test are fine, but foo_test is not). This +// is rarely a problem as people usually don't put the test binary +// directory in PATH. +// +// TODO(wan@google.com): make thread-safe death tests search the PATH. + +// Asserts that a given statement causes the program to exit, with an +// integer exit status that satisfies predicate, and emitting error output +// that matches regex. +# define ASSERT_EXIT(statement, predicate, regex) \ + GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_FATAL_FAILURE_) + +// Like ASSERT_EXIT, but continues on to successive tests in the +// test case, if any: +# define EXPECT_EXIT(statement, predicate, regex) \ + GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_NONFATAL_FAILURE_) + +// Asserts that a given statement causes the program to exit, either by +// explicitly exiting with a nonzero exit code or being killed by a +// signal, and emitting error output that matches regex. +# define ASSERT_DEATH(statement, regex) \ + ASSERT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) + +// Like ASSERT_DEATH, but continues on to successive tests in the +// test case, if any: +# define EXPECT_DEATH(statement, regex) \ + EXPECT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) + +// Two predicate classes that can be used in {ASSERT,EXPECT}_EXIT*: + +// Tests that an exit code describes a normal exit with a given exit code. +class GTEST_API_ ExitedWithCode { + public: + explicit ExitedWithCode(int exit_code); + bool operator()(int exit_status) const; + private: + // No implementation - assignment is unsupported. + void operator=(const ExitedWithCode& other); + + const int exit_code_; +}; + +# if !GTEST_OS_WINDOWS +// Tests that an exit code describes an exit due to termination by a +// given signal. +class GTEST_API_ KilledBySignal { + public: + explicit KilledBySignal(int signum); + bool operator()(int exit_status) const; + private: + const int signum_; +}; +# endif // !GTEST_OS_WINDOWS + +// EXPECT_DEBUG_DEATH asserts that the given statements die in debug mode. +// The death testing framework causes this to have interesting semantics, +// since the sideeffects of the call are only visible in opt mode, and not +// in debug mode. +// +// In practice, this can be used to test functions that utilize the +// LOG(DFATAL) macro using the following style: +// +// int DieInDebugOr12(int* sideeffect) { +// if (sideeffect) { +// *sideeffect = 12; +// } +// LOG(DFATAL) << "death"; +// return 12; +// } +// +// TEST(TestCase, TestDieOr12WorksInDgbAndOpt) { +// int sideeffect = 0; +// // Only asserts in dbg. +// EXPECT_DEBUG_DEATH(DieInDebugOr12(&sideeffect), "death"); +// +// #ifdef NDEBUG +// // opt-mode has sideeffect visible. +// EXPECT_EQ(12, sideeffect); +// #else +// // dbg-mode no visible sideeffect. +// EXPECT_EQ(0, sideeffect); +// #endif +// } +// +// This will assert that DieInDebugReturn12InOpt() crashes in debug +// mode, usually due to a DCHECK or LOG(DFATAL), but returns the +// appropriate fallback value (12 in this case) in opt mode. If you +// need to test that a function has appropriate side-effects in opt +// mode, include assertions against the side-effects. A general +// pattern for this is: +// +// EXPECT_DEBUG_DEATH({ +// // Side-effects here will have an effect after this statement in +// // opt mode, but none in debug mode. +// EXPECT_EQ(12, DieInDebugOr12(&sideeffect)); +// }, "death"); +// +# ifdef NDEBUG + +# define EXPECT_DEBUG_DEATH(statement, regex) \ + GTEST_EXECUTE_STATEMENT_(statement, regex) + +# define ASSERT_DEBUG_DEATH(statement, regex) \ + GTEST_EXECUTE_STATEMENT_(statement, regex) + +# else + +# define EXPECT_DEBUG_DEATH(statement, regex) \ + EXPECT_DEATH(statement, regex) + +# define ASSERT_DEBUG_DEATH(statement, regex) \ + ASSERT_DEATH(statement, regex) + +# endif // NDEBUG for EXPECT_DEBUG_DEATH +#endif // GTEST_HAS_DEATH_TEST + +// EXPECT_DEATH_IF_SUPPORTED(statement, regex) and +// ASSERT_DEATH_IF_SUPPORTED(statement, regex) expand to real death tests if +// death tests are supported; otherwise they just issue a warning. This is +// useful when you are combining death test assertions with normal test +// assertions in one test. +#if GTEST_HAS_DEATH_TEST +# define EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ + EXPECT_DEATH(statement, regex) +# define ASSERT_DEATH_IF_SUPPORTED(statement, regex) \ + ASSERT_DEATH(statement, regex) +#else +# define EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ + GTEST_UNSUPPORTED_DEATH_TEST_(statement, regex, ) +# define ASSERT_DEATH_IF_SUPPORTED(statement, regex) \ + GTEST_UNSUPPORTED_DEATH_TEST_(statement, regex, return) +#endif + +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-message.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-message.h new file mode 100644 index 0000000000..fe879bca79 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-message.h @@ -0,0 +1,250 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines the Message class. +// +// IMPORTANT NOTE: Due to limitation of the C++ language, we have to +// leave some internal implementation details in this header file. +// They are clearly marked by comments like this: +// +// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +// +// Such code is NOT meant to be used by a user directly, and is subject +// to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user +// program! + +#ifndef GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ +#define GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ + +#include + +#include "gtest/internal/gtest-port.h" + +// Ensures that there is at least one operator<< in the global namespace. +// See Message& operator<<(...) below for why. +void operator<<(const testing::internal::Secret&, int); + +namespace testing { + +// The Message class works like an ostream repeater. +// +// Typical usage: +// +// 1. You stream a bunch of values to a Message object. +// It will remember the text in a stringstream. +// 2. Then you stream the Message object to an ostream. +// This causes the text in the Message to be streamed +// to the ostream. +// +// For example; +// +// testing::Message foo; +// foo << 1 << " != " << 2; +// std::cout << foo; +// +// will print "1 != 2". +// +// Message is not intended to be inherited from. In particular, its +// destructor is not virtual. +// +// Note that stringstream behaves differently in gcc and in MSVC. You +// can stream a NULL char pointer to it in the former, but not in the +// latter (it causes an access violation if you do). The Message +// class hides this difference by treating a NULL char pointer as +// "(null)". +class GTEST_API_ Message { + private: + // The type of basic IO manipulators (endl, ends, and flush) for + // narrow streams. + typedef std::ostream& (*BasicNarrowIoManip)(std::ostream&); + + public: + // Constructs an empty Message. + Message(); + + // Copy constructor. + Message(const Message& msg) : ss_(new ::std::stringstream) { // NOLINT + *ss_ << msg.GetString(); + } + + // Constructs a Message from a C-string. + explicit Message(const char* str) : ss_(new ::std::stringstream) { + *ss_ << str; + } + +#if GTEST_OS_SYMBIAN + // Streams a value (either a pointer or not) to this object. + template + inline Message& operator <<(const T& value) { + StreamHelper(typename internal::is_pointer::type(), value); + return *this; + } +#else + // Streams a non-pointer value to this object. + template + inline Message& operator <<(const T& val) { + // Some libraries overload << for STL containers. These + // overloads are defined in the global namespace instead of ::std. + // + // C++'s symbol lookup rule (i.e. Koenig lookup) says that these + // overloads are visible in either the std namespace or the global + // namespace, but not other namespaces, including the testing + // namespace which Google Test's Message class is in. + // + // To allow STL containers (and other types that has a << operator + // defined in the global namespace) to be used in Google Test + // assertions, testing::Message must access the custom << operator + // from the global namespace. With this using declaration, + // overloads of << defined in the global namespace and those + // visible via Koenig lookup are both exposed in this function. + using ::operator <<; + *ss_ << val; + return *this; + } + + // Streams a pointer value to this object. + // + // This function is an overload of the previous one. When you + // stream a pointer to a Message, this definition will be used as it + // is more specialized. (The C++ Standard, section + // [temp.func.order].) If you stream a non-pointer, then the + // previous definition will be used. + // + // The reason for this overload is that streaming a NULL pointer to + // ostream is undefined behavior. Depending on the compiler, you + // may get "0", "(nil)", "(null)", or an access violation. To + // ensure consistent result across compilers, we always treat NULL + // as "(null)". + template + inline Message& operator <<(T* const& pointer) { // NOLINT + if (pointer == NULL) { + *ss_ << "(null)"; + } else { + *ss_ << pointer; + } + return *this; + } +#endif // GTEST_OS_SYMBIAN + + // Since the basic IO manipulators are overloaded for both narrow + // and wide streams, we have to provide this specialized definition + // of operator <<, even though its body is the same as the + // templatized version above. Without this definition, streaming + // endl or other basic IO manipulators to Message will confuse the + // compiler. + Message& operator <<(BasicNarrowIoManip val) { + *ss_ << val; + return *this; + } + + // Instead of 1/0, we want to see true/false for bool values. + Message& operator <<(bool b) { + return *this << (b ? "true" : "false"); + } + + // These two overloads allow streaming a wide C string to a Message + // using the UTF-8 encoding. + Message& operator <<(const wchar_t* wide_c_str); + Message& operator <<(wchar_t* wide_c_str); + +#if GTEST_HAS_STD_WSTRING + // Converts the given wide string to a narrow string using the UTF-8 + // encoding, and streams the result to this Message object. + Message& operator <<(const ::std::wstring& wstr); +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_GLOBAL_WSTRING + // Converts the given wide string to a narrow string using the UTF-8 + // encoding, and streams the result to this Message object. + Message& operator <<(const ::wstring& wstr); +#endif // GTEST_HAS_GLOBAL_WSTRING + + // Gets the text streamed to this object so far as an std::string. + // Each '\0' character in the buffer is replaced with "\\0". + // + // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + std::string GetString() const; + + private: + +#if GTEST_OS_SYMBIAN + // These are needed as the Nokia Symbian Compiler cannot decide between + // const T& and const T* in a function template. The Nokia compiler _can_ + // decide between class template specializations for T and T*, so a + // tr1::type_traits-like is_pointer works, and we can overload on that. + template + inline void StreamHelper(internal::true_type /*is_pointer*/, T* pointer) { + if (pointer == NULL) { + *ss_ << "(null)"; + } else { + *ss_ << pointer; + } + } + template + inline void StreamHelper(internal::false_type /*is_pointer*/, + const T& value) { + // See the comments in Message& operator <<(const T&) above for why + // we need this using statement. + using ::operator <<; + *ss_ << value; + } +#endif // GTEST_OS_SYMBIAN + + // We'll hold the text streamed to this object here. + const internal::scoped_ptr< ::std::stringstream> ss_; + + // We declare (but don't implement) this to prevent the compiler + // from implementing the assignment operator. + void operator=(const Message&); +}; + +// Streams a Message to an ostream. +inline std::ostream& operator <<(std::ostream& os, const Message& sb) { + return os << sb.GetString(); +} + +namespace internal { + +// Converts a streamable value to an std::string. A NULL pointer is +// converted to "(null)". When the input value is a ::string, +// ::std::string, ::wstring, or ::std::wstring object, each NUL +// character in it is replaced with "\\0". +template +std::string StreamableToString(const T& streamable) { + return (Message() << streamable).GetString(); +} + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-param-test.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-param-test.h new file mode 100644 index 0000000000..038f9ba79e --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-param-test.h @@ -0,0 +1,1444 @@ +// This file was GENERATED by command: +// pump.py gtest-param-test.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: vladl@google.com (Vlad Losev) +// +// Macros and functions for implementing parameterized tests +// in Google C++ Testing Framework (Google Test) +// +// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +#ifndef GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ + + +// Value-parameterized tests allow you to test your code with different +// parameters without writing multiple copies of the same test. +// +// Here is how you use value-parameterized tests: + +#if 0 + +// To write value-parameterized tests, first you should define a fixture +// class. It is usually derived from testing::TestWithParam (see below for +// another inheritance scheme that's sometimes useful in more complicated +// class hierarchies), where the type of your parameter values. +// TestWithParam is itself derived from testing::Test. T can be any +// copyable type. If it's a raw pointer, you are responsible for managing the +// lifespan of the pointed values. + +class FooTest : public ::testing::TestWithParam { + // You can implement all the usual class fixture members here. +}; + +// Then, use the TEST_P macro to define as many parameterized tests +// for this fixture as you want. The _P suffix is for "parameterized" +// or "pattern", whichever you prefer to think. + +TEST_P(FooTest, DoesBlah) { + // Inside a test, access the test parameter with the GetParam() method + // of the TestWithParam class: + EXPECT_TRUE(foo.Blah(GetParam())); + ... +} + +TEST_P(FooTest, HasBlahBlah) { + ... +} + +// Finally, you can use INSTANTIATE_TEST_CASE_P to instantiate the test +// case with any set of parameters you want. Google Test defines a number +// of functions for generating test parameters. They return what we call +// (surprise!) parameter generators. Here is a summary of them, which +// are all in the testing namespace: +// +// +// Range(begin, end [, step]) - Yields values {begin, begin+step, +// begin+step+step, ...}. The values do not +// include end. step defaults to 1. +// Values(v1, v2, ..., vN) - Yields values {v1, v2, ..., vN}. +// ValuesIn(container) - Yields values from a C-style array, an STL +// ValuesIn(begin,end) container, or an iterator range [begin, end). +// Bool() - Yields sequence {false, true}. +// Combine(g1, g2, ..., gN) - Yields all combinations (the Cartesian product +// for the math savvy) of the values generated +// by the N generators. +// +// For more details, see comments at the definitions of these functions below +// in this file. +// +// The following statement will instantiate tests from the FooTest test case +// each with parameter values "meeny", "miny", and "moe". + +INSTANTIATE_TEST_CASE_P(InstantiationName, + FooTest, + Values("meeny", "miny", "moe")); + +// To distinguish different instances of the pattern, (yes, you +// can instantiate it more then once) the first argument to the +// INSTANTIATE_TEST_CASE_P macro is a prefix that will be added to the +// actual test case name. Remember to pick unique prefixes for different +// instantiations. The tests from the instantiation above will have +// these names: +// +// * InstantiationName/FooTest.DoesBlah/0 for "meeny" +// * InstantiationName/FooTest.DoesBlah/1 for "miny" +// * InstantiationName/FooTest.DoesBlah/2 for "moe" +// * InstantiationName/FooTest.HasBlahBlah/0 for "meeny" +// * InstantiationName/FooTest.HasBlahBlah/1 for "miny" +// * InstantiationName/FooTest.HasBlahBlah/2 for "moe" +// +// You can use these names in --gtest_filter. +// +// This statement will instantiate all tests from FooTest again, each +// with parameter values "cat" and "dog": + +const char* pets[] = {"cat", "dog"}; +INSTANTIATE_TEST_CASE_P(AnotherInstantiationName, FooTest, ValuesIn(pets)); + +// The tests from the instantiation above will have these names: +// +// * AnotherInstantiationName/FooTest.DoesBlah/0 for "cat" +// * AnotherInstantiationName/FooTest.DoesBlah/1 for "dog" +// * AnotherInstantiationName/FooTest.HasBlahBlah/0 for "cat" +// * AnotherInstantiationName/FooTest.HasBlahBlah/1 for "dog" +// +// Please note that INSTANTIATE_TEST_CASE_P will instantiate all tests +// in the given test case, whether their definitions come before or +// AFTER the INSTANTIATE_TEST_CASE_P statement. +// +// Please also note that generator expressions (including parameters to the +// generators) are evaluated in InitGoogleTest(), after main() has started. +// This allows the user on one hand, to adjust generator parameters in order +// to dynamically determine a set of tests to run and on the other hand, +// give the user a chance to inspect the generated tests with Google Test +// reflection API before RUN_ALL_TESTS() is executed. +// +// You can see samples/sample7_unittest.cc and samples/sample8_unittest.cc +// for more examples. +// +// In the future, we plan to publish the API for defining new parameter +// generators. But for now this interface remains part of the internal +// implementation and is subject to change. +// +// +// A parameterized test fixture must be derived from testing::Test and from +// testing::WithParamInterface, where T is the type of the parameter +// values. Inheriting from TestWithParam satisfies that requirement because +// TestWithParam inherits from both Test and WithParamInterface. In more +// complicated hierarchies, however, it is occasionally useful to inherit +// separately from Test and WithParamInterface. For example: + +class BaseTest : public ::testing::Test { + // You can inherit all the usual members for a non-parameterized test + // fixture here. +}; + +class DerivedTest : public BaseTest, public ::testing::WithParamInterface { + // The usual test fixture members go here too. +}; + +TEST_F(BaseTest, HasFoo) { + // This is an ordinary non-parameterized test. +} + +TEST_P(DerivedTest, DoesBlah) { + // GetParam works just the same here as if you inherit from TestWithParam. + EXPECT_TRUE(foo.Blah(GetParam())); +} + +#endif // 0 + +#include "gtest/internal/gtest-port.h" + +#if !GTEST_OS_SYMBIAN +# include +#endif + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. +#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-param-util.h" +#include "gtest/internal/gtest-param-util-generated.h" + +#if GTEST_HAS_PARAM_TEST + +namespace testing { + +// Functions producing parameter generators. +// +// Google Test uses these generators to produce parameters for value- +// parameterized tests. When a parameterized test case is instantiated +// with a particular generator, Google Test creates and runs tests +// for each element in the sequence produced by the generator. +// +// In the following sample, tests from test case FooTest are instantiated +// each three times with parameter values 3, 5, and 8: +// +// class FooTest : public TestWithParam { ... }; +// +// TEST_P(FooTest, TestThis) { +// } +// TEST_P(FooTest, TestThat) { +// } +// INSTANTIATE_TEST_CASE_P(TestSequence, FooTest, Values(3, 5, 8)); +// + +// Range() returns generators providing sequences of values in a range. +// +// Synopsis: +// Range(start, end) +// - returns a generator producing a sequence of values {start, start+1, +// start+2, ..., }. +// Range(start, end, step) +// - returns a generator producing a sequence of values {start, start+step, +// start+step+step, ..., }. +// Notes: +// * The generated sequences never include end. For example, Range(1, 5) +// returns a generator producing a sequence {1, 2, 3, 4}. Range(1, 9, 2) +// returns a generator producing {1, 3, 5, 7}. +// * start and end must have the same type. That type may be any integral or +// floating-point type or a user defined type satisfying these conditions: +// * It must be assignable (have operator=() defined). +// * It must have operator+() (operator+(int-compatible type) for +// two-operand version). +// * It must have operator<() defined. +// Elements in the resulting sequences will also have that type. +// * Condition start < end must be satisfied in order for resulting sequences +// to contain any elements. +// +template +internal::ParamGenerator Range(T start, T end, IncrementT step) { + return internal::ParamGenerator( + new internal::RangeGenerator(start, end, step)); +} + +template +internal::ParamGenerator Range(T start, T end) { + return Range(start, end, 1); +} + +// ValuesIn() function allows generation of tests with parameters coming from +// a container. +// +// Synopsis: +// ValuesIn(const T (&array)[N]) +// - returns a generator producing sequences with elements from +// a C-style array. +// ValuesIn(const Container& container) +// - returns a generator producing sequences with elements from +// an STL-style container. +// ValuesIn(Iterator begin, Iterator end) +// - returns a generator producing sequences with elements from +// a range [begin, end) defined by a pair of STL-style iterators. These +// iterators can also be plain C pointers. +// +// Please note that ValuesIn copies the values from the containers +// passed in and keeps them to generate tests in RUN_ALL_TESTS(). +// +// Examples: +// +// This instantiates tests from test case StringTest +// each with C-string values of "foo", "bar", and "baz": +// +// const char* strings[] = {"foo", "bar", "baz"}; +// INSTANTIATE_TEST_CASE_P(StringSequence, SrtingTest, ValuesIn(strings)); +// +// This instantiates tests from test case StlStringTest +// each with STL strings with values "a" and "b": +// +// ::std::vector< ::std::string> GetParameterStrings() { +// ::std::vector< ::std::string> v; +// v.push_back("a"); +// v.push_back("b"); +// return v; +// } +// +// INSTANTIATE_TEST_CASE_P(CharSequence, +// StlStringTest, +// ValuesIn(GetParameterStrings())); +// +// +// This will also instantiate tests from CharTest +// each with parameter values 'a' and 'b': +// +// ::std::list GetParameterChars() { +// ::std::list list; +// list.push_back('a'); +// list.push_back('b'); +// return list; +// } +// ::std::list l = GetParameterChars(); +// INSTANTIATE_TEST_CASE_P(CharSequence2, +// CharTest, +// ValuesIn(l.begin(), l.end())); +// +template +internal::ParamGenerator< + typename ::testing::internal::IteratorTraits::value_type> +ValuesIn(ForwardIterator begin, ForwardIterator end) { + typedef typename ::testing::internal::IteratorTraits + ::value_type ParamType; + return internal::ParamGenerator( + new internal::ValuesInIteratorRangeGenerator(begin, end)); +} + +template +internal::ParamGenerator ValuesIn(const T (&array)[N]) { + return ValuesIn(array, array + N); +} + +template +internal::ParamGenerator ValuesIn( + const Container& container) { + return ValuesIn(container.begin(), container.end()); +} + +// Values() allows generating tests from explicitly specified list of +// parameters. +// +// Synopsis: +// Values(T v1, T v2, ..., T vN) +// - returns a generator producing sequences with elements v1, v2, ..., vN. +// +// For example, this instantiates tests from test case BarTest each +// with values "one", "two", and "three": +// +// INSTANTIATE_TEST_CASE_P(NumSequence, BarTest, Values("one", "two", "three")); +// +// This instantiates tests from test case BazTest each with values 1, 2, 3.5. +// The exact type of values will depend on the type of parameter in BazTest. +// +// INSTANTIATE_TEST_CASE_P(FloatingNumbers, BazTest, Values(1, 2, 3.5)); +// +// Currently, Values() supports from 1 to 50 parameters. +// +template +internal::ValueArray1 Values(T1 v1) { + return internal::ValueArray1(v1); +} + +template +internal::ValueArray2 Values(T1 v1, T2 v2) { + return internal::ValueArray2(v1, v2); +} + +template +internal::ValueArray3 Values(T1 v1, T2 v2, T3 v3) { + return internal::ValueArray3(v1, v2, v3); +} + +template +internal::ValueArray4 Values(T1 v1, T2 v2, T3 v3, T4 v4) { + return internal::ValueArray4(v1, v2, v3, v4); +} + +template +internal::ValueArray5 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5) { + return internal::ValueArray5(v1, v2, v3, v4, v5); +} + +template +internal::ValueArray6 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6) { + return internal::ValueArray6(v1, v2, v3, v4, v5, v6); +} + +template +internal::ValueArray7 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6, T7 v7) { + return internal::ValueArray7(v1, v2, v3, v4, v5, + v6, v7); +} + +template +internal::ValueArray8 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8) { + return internal::ValueArray8(v1, v2, v3, v4, + v5, v6, v7, v8); +} + +template +internal::ValueArray9 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9) { + return internal::ValueArray9(v1, v2, v3, + v4, v5, v6, v7, v8, v9); +} + +template +internal::ValueArray10 Values(T1 v1, + T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10) { + return internal::ValueArray10(v1, + v2, v3, v4, v5, v6, v7, v8, v9, v10); +} + +template +internal::ValueArray11 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11) { + return internal::ValueArray11(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11); +} + +template +internal::ValueArray12 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12) { + return internal::ValueArray12(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12); +} + +template +internal::ValueArray13 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13) { + return internal::ValueArray13(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13); +} + +template +internal::ValueArray14 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14) { + return internal::ValueArray14(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14); +} + +template +internal::ValueArray15 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15) { + return internal::ValueArray15(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); +} + +template +internal::ValueArray16 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16) { + return internal::ValueArray16(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15, v16); +} + +template +internal::ValueArray17 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17) { + return internal::ValueArray17(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, + v11, v12, v13, v14, v15, v16, v17); +} + +template +internal::ValueArray18 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, + T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18) { + return internal::ValueArray18(v1, v2, v3, v4, v5, v6, v7, v8, v9, + v10, v11, v12, v13, v14, v15, v16, v17, v18); +} + +template +internal::ValueArray19 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, + T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, + T15 v15, T16 v16, T17 v17, T18 v18, T19 v19) { + return internal::ValueArray19(v1, v2, v3, v4, v5, v6, v7, v8, + v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19); +} + +template +internal::ValueArray20 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20) { + return internal::ValueArray20(v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20); +} + +template +internal::ValueArray21 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21) { + return internal::ValueArray21(v1, v2, v3, v4, v5, v6, + v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21); +} + +template +internal::ValueArray22 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22) { + return internal::ValueArray22(v1, v2, v3, v4, + v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22); +} + +template +internal::ValueArray23 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23) { + return internal::ValueArray23(v1, v2, v3, + v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23); +} + +template +internal::ValueArray24 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24) { + return internal::ValueArray24(v1, v2, + v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, + v19, v20, v21, v22, v23, v24); +} + +template +internal::ValueArray25 Values(T1 v1, + T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, + T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, + T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25) { + return internal::ValueArray25(v1, + v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, + v18, v19, v20, v21, v22, v23, v24, v25); +} + +template +internal::ValueArray26 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26) { + return internal::ValueArray26(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26); +} + +template +internal::ValueArray27 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27) { + return internal::ValueArray27(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, + v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27); +} + +template +internal::ValueArray28 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28) { + return internal::ValueArray28(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, + v28); +} + +template +internal::ValueArray29 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29) { + return internal::ValueArray29(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, + v27, v28, v29); +} + +template +internal::ValueArray30 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, + T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, + T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30) { + return internal::ValueArray30(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, + v26, v27, v28, v29, v30); +} + +template +internal::ValueArray31 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31) { + return internal::ValueArray31(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, + v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, + v25, v26, v27, v28, v29, v30, v31); +} + +template +internal::ValueArray32 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32) { + return internal::ValueArray32(v1, v2, v3, v4, v5, v6, v7, v8, v9, + v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32); +} + +template +internal::ValueArray33 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, + T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33) { + return internal::ValueArray33(v1, v2, v3, v4, v5, v6, v7, v8, + v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32, v33); +} + +template +internal::ValueArray34 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, + T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, + T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, + T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, + T31 v31, T32 v32, T33 v33, T34 v34) { + return internal::ValueArray34(v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, + v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34); +} + +template +internal::ValueArray35 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, + T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, + T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35) { + return internal::ValueArray35(v1, v2, v3, v4, v5, v6, + v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, + v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35); +} + +template +internal::ValueArray36 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, + T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, + T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36) { + return internal::ValueArray36(v1, v2, v3, v4, + v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, + v34, v35, v36); +} + +template +internal::ValueArray37 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, + T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, + T37 v37) { + return internal::ValueArray37(v1, v2, v3, + v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, + v34, v35, v36, v37); +} + +template +internal::ValueArray38 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, + T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, + T37 v37, T38 v38) { + return internal::ValueArray38(v1, v2, + v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, + v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, + v33, v34, v35, v36, v37, v38); +} + +template +internal::ValueArray39 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, + T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, + T37 v37, T38 v38, T39 v39) { + return internal::ValueArray39(v1, + v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, + v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, + v32, v33, v34, v35, v36, v37, v38, v39); +} + +template +internal::ValueArray40 Values(T1 v1, + T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, + T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, + T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, + T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, + T36 v36, T37 v37, T38 v38, T39 v39, T40 v40) { + return internal::ValueArray40(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, + v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40); +} + +template +internal::ValueArray41 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41) { + return internal::ValueArray41(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, + v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, + v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41); +} + +template +internal::ValueArray42 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42) { + return internal::ValueArray42(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, + v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, + v42); +} + +template +internal::ValueArray43 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43) { + return internal::ValueArray43(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, + v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, + v41, v42, v43); +} + +template +internal::ValueArray44 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44) { + return internal::ValueArray44(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, + v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, + v40, v41, v42, v43, v44); +} + +template +internal::ValueArray45 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, + T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, + T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, + T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, + T41 v41, T42 v42, T43 v43, T44 v44, T45 v45) { + return internal::ValueArray45(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, + v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, + v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, + v39, v40, v41, v42, v43, v44, v45); +} + +template +internal::ValueArray46 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, + T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46) { + return internal::ValueArray46(v1, v2, v3, v4, v5, v6, v7, v8, v9, + v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, + v38, v39, v40, v41, v42, v43, v44, v45, v46); +} + +template +internal::ValueArray47 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, + T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47) { + return internal::ValueArray47(v1, v2, v3, v4, v5, v6, v7, v8, + v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, + v38, v39, v40, v41, v42, v43, v44, v45, v46, v47); +} + +template +internal::ValueArray48 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, + T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, + T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, + T48 v48) { + return internal::ValueArray48(v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, + v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, + v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48); +} + +template +internal::ValueArray49 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, + T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, + T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, + T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, + T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, + T39 v39, T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, + T47 v47, T48 v48, T49 v49) { + return internal::ValueArray49(v1, v2, v3, v4, v5, v6, + v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, + v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, + v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49); +} + +template +internal::ValueArray50 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, + T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, + T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, + T38 v38, T39 v39, T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, + T46 v46, T47 v47, T48 v48, T49 v49, T50 v50) { + return internal::ValueArray50(v1, v2, v3, v4, + v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, + v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, + v48, v49, v50); +} + +// Bool() allows generating tests with parameters in a set of (false, true). +// +// Synopsis: +// Bool() +// - returns a generator producing sequences with elements {false, true}. +// +// It is useful when testing code that depends on Boolean flags. Combinations +// of multiple flags can be tested when several Bool()'s are combined using +// Combine() function. +// +// In the following example all tests in the test case FlagDependentTest +// will be instantiated twice with parameters false and true. +// +// class FlagDependentTest : public testing::TestWithParam { +// virtual void SetUp() { +// external_flag = GetParam(); +// } +// } +// INSTANTIATE_TEST_CASE_P(BoolSequence, FlagDependentTest, Bool()); +// +inline internal::ParamGenerator Bool() { + return Values(false, true); +} + +# if GTEST_HAS_COMBINE +// Combine() allows the user to combine two or more sequences to produce +// values of a Cartesian product of those sequences' elements. +// +// Synopsis: +// Combine(gen1, gen2, ..., genN) +// - returns a generator producing sequences with elements coming from +// the Cartesian product of elements from the sequences generated by +// gen1, gen2, ..., genN. The sequence elements will have a type of +// tuple where T1, T2, ..., TN are the types +// of elements from sequences produces by gen1, gen2, ..., genN. +// +// Combine can have up to 10 arguments. This number is currently limited +// by the maximum number of elements in the tuple implementation used by Google +// Test. +// +// Example: +// +// This will instantiate tests in test case AnimalTest each one with +// the parameter values tuple("cat", BLACK), tuple("cat", WHITE), +// tuple("dog", BLACK), and tuple("dog", WHITE): +// +// enum Color { BLACK, GRAY, WHITE }; +// class AnimalTest +// : public testing::TestWithParam > {...}; +// +// TEST_P(AnimalTest, AnimalLooksNice) {...} +// +// INSTANTIATE_TEST_CASE_P(AnimalVariations, AnimalTest, +// Combine(Values("cat", "dog"), +// Values(BLACK, WHITE))); +// +// This will instantiate tests in FlagDependentTest with all variations of two +// Boolean flags: +// +// class FlagDependentTest +// : public testing::TestWithParam > { +// virtual void SetUp() { +// // Assigns external_flag_1 and external_flag_2 values from the tuple. +// tie(external_flag_1, external_flag_2) = GetParam(); +// } +// }; +// +// TEST_P(FlagDependentTest, TestFeature1) { +// // Test your code using external_flag_1 and external_flag_2 here. +// } +// INSTANTIATE_TEST_CASE_P(TwoBoolSequence, FlagDependentTest, +// Combine(Bool(), Bool())); +// +template +internal::CartesianProductHolder2 Combine( + const Generator1& g1, const Generator2& g2) { + return internal::CartesianProductHolder2( + g1, g2); +} + +template +internal::CartesianProductHolder3 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3) { + return internal::CartesianProductHolder3( + g1, g2, g3); +} + +template +internal::CartesianProductHolder4 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4) { + return internal::CartesianProductHolder4( + g1, g2, g3, g4); +} + +template +internal::CartesianProductHolder5 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5) { + return internal::CartesianProductHolder5( + g1, g2, g3, g4, g5); +} + +template +internal::CartesianProductHolder6 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6) { + return internal::CartesianProductHolder6( + g1, g2, g3, g4, g5, g6); +} + +template +internal::CartesianProductHolder7 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7) { + return internal::CartesianProductHolder7( + g1, g2, g3, g4, g5, g6, g7); +} + +template +internal::CartesianProductHolder8 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7, const Generator8& g8) { + return internal::CartesianProductHolder8( + g1, g2, g3, g4, g5, g6, g7, g8); +} + +template +internal::CartesianProductHolder9 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7, const Generator8& g8, const Generator9& g9) { + return internal::CartesianProductHolder9( + g1, g2, g3, g4, g5, g6, g7, g8, g9); +} + +template +internal::CartesianProductHolder10 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7, const Generator8& g8, const Generator9& g9, + const Generator10& g10) { + return internal::CartesianProductHolder10( + g1, g2, g3, g4, g5, g6, g7, g8, g9, g10); +} +# endif // GTEST_HAS_COMBINE + + + +# define TEST_P(test_case_name, test_name) \ + class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) \ + : public test_case_name { \ + public: \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {} \ + virtual void TestBody(); \ + private: \ + static int AddToRegistry() { \ + ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ + GetTestCasePatternHolder(\ + #test_case_name, \ + ::testing::internal::CodeLocation(\ + __FILE__, __LINE__))->AddTestPattern(\ + #test_case_name, \ + #test_name, \ + new ::testing::internal::TestMetaFactory< \ + GTEST_TEST_CLASS_NAME_(\ + test_case_name, test_name)>()); \ + return 0; \ + } \ + static int gtest_registering_dummy_ GTEST_ATTRIBUTE_UNUSED_; \ + GTEST_DISALLOW_COPY_AND_ASSIGN_(\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)); \ + }; \ + int GTEST_TEST_CLASS_NAME_(test_case_name, \ + test_name)::gtest_registering_dummy_ = \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::AddToRegistry(); \ + void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() + +// The optional last argument to INSTANTIATE_TEST_CASE_P allows the user +// to specify a function or functor that generates custom test name suffixes +// based on the test parameters. The function should accept one argument of +// type testing::TestParamInfo, and return std::string. +// +// testing::PrintToStringParamName is a builtin test suffix generator that +// returns the value of testing::PrintToString(GetParam()). It does not work +// for std::string or C strings. +// +// Note: test names must be non-empty, unique, and may only contain ASCII +// alphanumeric characters or underscore. + +# define INSTANTIATE_TEST_CASE_P(prefix, test_case_name, generator, ...) \ + ::testing::internal::ParamGenerator \ + gtest_##prefix##test_case_name##_EvalGenerator_() { return generator; } \ + ::std::string gtest_##prefix##test_case_name##_EvalGenerateName_( \ + const ::testing::TestParamInfo& info) { \ + return ::testing::internal::GetParamNameGen \ + (__VA_ARGS__)(info); \ + } \ + int gtest_##prefix##test_case_name##_dummy_ GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ + GetTestCasePatternHolder(\ + #test_case_name, \ + ::testing::internal::CodeLocation(\ + __FILE__, __LINE__))->AddTestCaseInstantiation(\ + #prefix, \ + >est_##prefix##test_case_name##_EvalGenerator_, \ + >est_##prefix##test_case_name##_EvalGenerateName_, \ + __FILE__, __LINE__) + +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-param-test.h.pump b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-param-test.h.pump new file mode 100644 index 0000000000..3078d6d2a1 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-param-test.h.pump @@ -0,0 +1,510 @@ +$$ -*- mode: c++; -*- +$var n = 50 $$ Maximum length of Values arguments we want to support. +$var maxtuple = 10 $$ Maximum number of Combine arguments we want to support. +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: vladl@google.com (Vlad Losev) +// +// Macros and functions for implementing parameterized tests +// in Google C++ Testing Framework (Google Test) +// +// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +#ifndef GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ + + +// Value-parameterized tests allow you to test your code with different +// parameters without writing multiple copies of the same test. +// +// Here is how you use value-parameterized tests: + +#if 0 + +// To write value-parameterized tests, first you should define a fixture +// class. It is usually derived from testing::TestWithParam (see below for +// another inheritance scheme that's sometimes useful in more complicated +// class hierarchies), where the type of your parameter values. +// TestWithParam is itself derived from testing::Test. T can be any +// copyable type. If it's a raw pointer, you are responsible for managing the +// lifespan of the pointed values. + +class FooTest : public ::testing::TestWithParam { + // You can implement all the usual class fixture members here. +}; + +// Then, use the TEST_P macro to define as many parameterized tests +// for this fixture as you want. The _P suffix is for "parameterized" +// or "pattern", whichever you prefer to think. + +TEST_P(FooTest, DoesBlah) { + // Inside a test, access the test parameter with the GetParam() method + // of the TestWithParam class: + EXPECT_TRUE(foo.Blah(GetParam())); + ... +} + +TEST_P(FooTest, HasBlahBlah) { + ... +} + +// Finally, you can use INSTANTIATE_TEST_CASE_P to instantiate the test +// case with any set of parameters you want. Google Test defines a number +// of functions for generating test parameters. They return what we call +// (surprise!) parameter generators. Here is a summary of them, which +// are all in the testing namespace: +// +// +// Range(begin, end [, step]) - Yields values {begin, begin+step, +// begin+step+step, ...}. The values do not +// include end. step defaults to 1. +// Values(v1, v2, ..., vN) - Yields values {v1, v2, ..., vN}. +// ValuesIn(container) - Yields values from a C-style array, an STL +// ValuesIn(begin,end) container, or an iterator range [begin, end). +// Bool() - Yields sequence {false, true}. +// Combine(g1, g2, ..., gN) - Yields all combinations (the Cartesian product +// for the math savvy) of the values generated +// by the N generators. +// +// For more details, see comments at the definitions of these functions below +// in this file. +// +// The following statement will instantiate tests from the FooTest test case +// each with parameter values "meeny", "miny", and "moe". + +INSTANTIATE_TEST_CASE_P(InstantiationName, + FooTest, + Values("meeny", "miny", "moe")); + +// To distinguish different instances of the pattern, (yes, you +// can instantiate it more then once) the first argument to the +// INSTANTIATE_TEST_CASE_P macro is a prefix that will be added to the +// actual test case name. Remember to pick unique prefixes for different +// instantiations. The tests from the instantiation above will have +// these names: +// +// * InstantiationName/FooTest.DoesBlah/0 for "meeny" +// * InstantiationName/FooTest.DoesBlah/1 for "miny" +// * InstantiationName/FooTest.DoesBlah/2 for "moe" +// * InstantiationName/FooTest.HasBlahBlah/0 for "meeny" +// * InstantiationName/FooTest.HasBlahBlah/1 for "miny" +// * InstantiationName/FooTest.HasBlahBlah/2 for "moe" +// +// You can use these names in --gtest_filter. +// +// This statement will instantiate all tests from FooTest again, each +// with parameter values "cat" and "dog": + +const char* pets[] = {"cat", "dog"}; +INSTANTIATE_TEST_CASE_P(AnotherInstantiationName, FooTest, ValuesIn(pets)); + +// The tests from the instantiation above will have these names: +// +// * AnotherInstantiationName/FooTest.DoesBlah/0 for "cat" +// * AnotherInstantiationName/FooTest.DoesBlah/1 for "dog" +// * AnotherInstantiationName/FooTest.HasBlahBlah/0 for "cat" +// * AnotherInstantiationName/FooTest.HasBlahBlah/1 for "dog" +// +// Please note that INSTANTIATE_TEST_CASE_P will instantiate all tests +// in the given test case, whether their definitions come before or +// AFTER the INSTANTIATE_TEST_CASE_P statement. +// +// Please also note that generator expressions (including parameters to the +// generators) are evaluated in InitGoogleTest(), after main() has started. +// This allows the user on one hand, to adjust generator parameters in order +// to dynamically determine a set of tests to run and on the other hand, +// give the user a chance to inspect the generated tests with Google Test +// reflection API before RUN_ALL_TESTS() is executed. +// +// You can see samples/sample7_unittest.cc and samples/sample8_unittest.cc +// for more examples. +// +// In the future, we plan to publish the API for defining new parameter +// generators. But for now this interface remains part of the internal +// implementation and is subject to change. +// +// +// A parameterized test fixture must be derived from testing::Test and from +// testing::WithParamInterface, where T is the type of the parameter +// values. Inheriting from TestWithParam satisfies that requirement because +// TestWithParam inherits from both Test and WithParamInterface. In more +// complicated hierarchies, however, it is occasionally useful to inherit +// separately from Test and WithParamInterface. For example: + +class BaseTest : public ::testing::Test { + // You can inherit all the usual members for a non-parameterized test + // fixture here. +}; + +class DerivedTest : public BaseTest, public ::testing::WithParamInterface { + // The usual test fixture members go here too. +}; + +TEST_F(BaseTest, HasFoo) { + // This is an ordinary non-parameterized test. +} + +TEST_P(DerivedTest, DoesBlah) { + // GetParam works just the same here as if you inherit from TestWithParam. + EXPECT_TRUE(foo.Blah(GetParam())); +} + +#endif // 0 + +#include "gtest/internal/gtest-port.h" + +#if !GTEST_OS_SYMBIAN +# include +#endif + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. +#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-param-util.h" +#include "gtest/internal/gtest-param-util-generated.h" + +#if GTEST_HAS_PARAM_TEST + +namespace testing { + +// Functions producing parameter generators. +// +// Google Test uses these generators to produce parameters for value- +// parameterized tests. When a parameterized test case is instantiated +// with a particular generator, Google Test creates and runs tests +// for each element in the sequence produced by the generator. +// +// In the following sample, tests from test case FooTest are instantiated +// each three times with parameter values 3, 5, and 8: +// +// class FooTest : public TestWithParam { ... }; +// +// TEST_P(FooTest, TestThis) { +// } +// TEST_P(FooTest, TestThat) { +// } +// INSTANTIATE_TEST_CASE_P(TestSequence, FooTest, Values(3, 5, 8)); +// + +// Range() returns generators providing sequences of values in a range. +// +// Synopsis: +// Range(start, end) +// - returns a generator producing a sequence of values {start, start+1, +// start+2, ..., }. +// Range(start, end, step) +// - returns a generator producing a sequence of values {start, start+step, +// start+step+step, ..., }. +// Notes: +// * The generated sequences never include end. For example, Range(1, 5) +// returns a generator producing a sequence {1, 2, 3, 4}. Range(1, 9, 2) +// returns a generator producing {1, 3, 5, 7}. +// * start and end must have the same type. That type may be any integral or +// floating-point type or a user defined type satisfying these conditions: +// * It must be assignable (have operator=() defined). +// * It must have operator+() (operator+(int-compatible type) for +// two-operand version). +// * It must have operator<() defined. +// Elements in the resulting sequences will also have that type. +// * Condition start < end must be satisfied in order for resulting sequences +// to contain any elements. +// +template +internal::ParamGenerator Range(T start, T end, IncrementT step) { + return internal::ParamGenerator( + new internal::RangeGenerator(start, end, step)); +} + +template +internal::ParamGenerator Range(T start, T end) { + return Range(start, end, 1); +} + +// ValuesIn() function allows generation of tests with parameters coming from +// a container. +// +// Synopsis: +// ValuesIn(const T (&array)[N]) +// - returns a generator producing sequences with elements from +// a C-style array. +// ValuesIn(const Container& container) +// - returns a generator producing sequences with elements from +// an STL-style container. +// ValuesIn(Iterator begin, Iterator end) +// - returns a generator producing sequences with elements from +// a range [begin, end) defined by a pair of STL-style iterators. These +// iterators can also be plain C pointers. +// +// Please note that ValuesIn copies the values from the containers +// passed in and keeps them to generate tests in RUN_ALL_TESTS(). +// +// Examples: +// +// This instantiates tests from test case StringTest +// each with C-string values of "foo", "bar", and "baz": +// +// const char* strings[] = {"foo", "bar", "baz"}; +// INSTANTIATE_TEST_CASE_P(StringSequence, SrtingTest, ValuesIn(strings)); +// +// This instantiates tests from test case StlStringTest +// each with STL strings with values "a" and "b": +// +// ::std::vector< ::std::string> GetParameterStrings() { +// ::std::vector< ::std::string> v; +// v.push_back("a"); +// v.push_back("b"); +// return v; +// } +// +// INSTANTIATE_TEST_CASE_P(CharSequence, +// StlStringTest, +// ValuesIn(GetParameterStrings())); +// +// +// This will also instantiate tests from CharTest +// each with parameter values 'a' and 'b': +// +// ::std::list GetParameterChars() { +// ::std::list list; +// list.push_back('a'); +// list.push_back('b'); +// return list; +// } +// ::std::list l = GetParameterChars(); +// INSTANTIATE_TEST_CASE_P(CharSequence2, +// CharTest, +// ValuesIn(l.begin(), l.end())); +// +template +internal::ParamGenerator< + typename ::testing::internal::IteratorTraits::value_type> +ValuesIn(ForwardIterator begin, ForwardIterator end) { + typedef typename ::testing::internal::IteratorTraits + ::value_type ParamType; + return internal::ParamGenerator( + new internal::ValuesInIteratorRangeGenerator(begin, end)); +} + +template +internal::ParamGenerator ValuesIn(const T (&array)[N]) { + return ValuesIn(array, array + N); +} + +template +internal::ParamGenerator ValuesIn( + const Container& container) { + return ValuesIn(container.begin(), container.end()); +} + +// Values() allows generating tests from explicitly specified list of +// parameters. +// +// Synopsis: +// Values(T v1, T v2, ..., T vN) +// - returns a generator producing sequences with elements v1, v2, ..., vN. +// +// For example, this instantiates tests from test case BarTest each +// with values "one", "two", and "three": +// +// INSTANTIATE_TEST_CASE_P(NumSequence, BarTest, Values("one", "two", "three")); +// +// This instantiates tests from test case BazTest each with values 1, 2, 3.5. +// The exact type of values will depend on the type of parameter in BazTest. +// +// INSTANTIATE_TEST_CASE_P(FloatingNumbers, BazTest, Values(1, 2, 3.5)); +// +// Currently, Values() supports from 1 to $n parameters. +// +$range i 1..n +$for i [[ +$range j 1..i + +template <$for j, [[typename T$j]]> +internal::ValueArray$i<$for j, [[T$j]]> Values($for j, [[T$j v$j]]) { + return internal::ValueArray$i<$for j, [[T$j]]>($for j, [[v$j]]); +} + +]] + +// Bool() allows generating tests with parameters in a set of (false, true). +// +// Synopsis: +// Bool() +// - returns a generator producing sequences with elements {false, true}. +// +// It is useful when testing code that depends on Boolean flags. Combinations +// of multiple flags can be tested when several Bool()'s are combined using +// Combine() function. +// +// In the following example all tests in the test case FlagDependentTest +// will be instantiated twice with parameters false and true. +// +// class FlagDependentTest : public testing::TestWithParam { +// virtual void SetUp() { +// external_flag = GetParam(); +// } +// } +// INSTANTIATE_TEST_CASE_P(BoolSequence, FlagDependentTest, Bool()); +// +inline internal::ParamGenerator Bool() { + return Values(false, true); +} + +# if GTEST_HAS_COMBINE +// Combine() allows the user to combine two or more sequences to produce +// values of a Cartesian product of those sequences' elements. +// +// Synopsis: +// Combine(gen1, gen2, ..., genN) +// - returns a generator producing sequences with elements coming from +// the Cartesian product of elements from the sequences generated by +// gen1, gen2, ..., genN. The sequence elements will have a type of +// tuple where T1, T2, ..., TN are the types +// of elements from sequences produces by gen1, gen2, ..., genN. +// +// Combine can have up to $maxtuple arguments. This number is currently limited +// by the maximum number of elements in the tuple implementation used by Google +// Test. +// +// Example: +// +// This will instantiate tests in test case AnimalTest each one with +// the parameter values tuple("cat", BLACK), tuple("cat", WHITE), +// tuple("dog", BLACK), and tuple("dog", WHITE): +// +// enum Color { BLACK, GRAY, WHITE }; +// class AnimalTest +// : public testing::TestWithParam > {...}; +// +// TEST_P(AnimalTest, AnimalLooksNice) {...} +// +// INSTANTIATE_TEST_CASE_P(AnimalVariations, AnimalTest, +// Combine(Values("cat", "dog"), +// Values(BLACK, WHITE))); +// +// This will instantiate tests in FlagDependentTest with all variations of two +// Boolean flags: +// +// class FlagDependentTest +// : public testing::TestWithParam > { +// virtual void SetUp() { +// // Assigns external_flag_1 and external_flag_2 values from the tuple. +// tie(external_flag_1, external_flag_2) = GetParam(); +// } +// }; +// +// TEST_P(FlagDependentTest, TestFeature1) { +// // Test your code using external_flag_1 and external_flag_2 here. +// } +// INSTANTIATE_TEST_CASE_P(TwoBoolSequence, FlagDependentTest, +// Combine(Bool(), Bool())); +// +$range i 2..maxtuple +$for i [[ +$range j 1..i + +template <$for j, [[typename Generator$j]]> +internal::CartesianProductHolder$i<$for j, [[Generator$j]]> Combine( + $for j, [[const Generator$j& g$j]]) { + return internal::CartesianProductHolder$i<$for j, [[Generator$j]]>( + $for j, [[g$j]]); +} + +]] +# endif // GTEST_HAS_COMBINE + + + +# define TEST_P(test_case_name, test_name) \ + class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) \ + : public test_case_name { \ + public: \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {} \ + virtual void TestBody(); \ + private: \ + static int AddToRegistry() { \ + ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ + GetTestCasePatternHolder(\ + #test_case_name, \ + ::testing::internal::CodeLocation(\ + __FILE__, __LINE__))->AddTestPattern(\ + #test_case_name, \ + #test_name, \ + new ::testing::internal::TestMetaFactory< \ + GTEST_TEST_CLASS_NAME_(\ + test_case_name, test_name)>()); \ + return 0; \ + } \ + static int gtest_registering_dummy_ GTEST_ATTRIBUTE_UNUSED_; \ + GTEST_DISALLOW_COPY_AND_ASSIGN_(\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)); \ + }; \ + int GTEST_TEST_CLASS_NAME_(test_case_name, \ + test_name)::gtest_registering_dummy_ = \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::AddToRegistry(); \ + void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() + +// The optional last argument to INSTANTIATE_TEST_CASE_P allows the user +// to specify a function or functor that generates custom test name suffixes +// based on the test parameters. The function should accept one argument of +// type testing::TestParamInfo, and return std::string. +// +// testing::PrintToStringParamName is a builtin test suffix generator that +// returns the value of testing::PrintToString(GetParam()). +// +// Note: test names must be non-empty, unique, and may only contain ASCII +// alphanumeric characters or underscore. Because PrintToString adds quotes +// to std::string and C strings, it won't work for these types. + +# define INSTANTIATE_TEST_CASE_P(prefix, test_case_name, generator, ...) \ + ::testing::internal::ParamGenerator \ + gtest_##prefix##test_case_name##_EvalGenerator_() { return generator; } \ + ::std::string gtest_##prefix##test_case_name##_EvalGenerateName_( \ + const ::testing::TestParamInfo& info) { \ + return ::testing::internal::GetParamNameGen \ + (__VA_ARGS__)(info); \ + } \ + int gtest_##prefix##test_case_name##_dummy_ GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ + GetTestCasePatternHolder(\ + #test_case_name, \ + ::testing::internal::CodeLocation(\ + __FILE__, __LINE__))->AddTestCaseInstantiation(\ + #prefix, \ + >est_##prefix##test_case_name##_EvalGenerator_, \ + >est_##prefix##test_case_name##_EvalGenerateName_, \ + __FILE__, __LINE__) + +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-printers.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-printers.h new file mode 100644 index 0000000000..8a33164cb3 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-printers.h @@ -0,0 +1,993 @@ +// Copyright 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Google Test - The Google C++ Testing Framework +// +// This file implements a universal value printer that can print a +// value of any type T: +// +// void ::testing::internal::UniversalPrinter::Print(value, ostream_ptr); +// +// A user can teach this function how to print a class type T by +// defining either operator<<() or PrintTo() in the namespace that +// defines T. More specifically, the FIRST defined function in the +// following list will be used (assuming T is defined in namespace +// foo): +// +// 1. foo::PrintTo(const T&, ostream*) +// 2. operator<<(ostream&, const T&) defined in either foo or the +// global namespace. +// +// If none of the above is defined, it will print the debug string of +// the value if it is a protocol buffer, or print the raw bytes in the +// value otherwise. +// +// To aid debugging: when T is a reference type, the address of the +// value is also printed; when T is a (const) char pointer, both the +// pointer value and the NUL-terminated string it points to are +// printed. +// +// We also provide some convenient wrappers: +// +// // Prints a value to a string. For a (const or not) char +// // pointer, the NUL-terminated string (but not the pointer) is +// // printed. +// std::string ::testing::PrintToString(const T& value); +// +// // Prints a value tersely: for a reference type, the referenced +// // value (but not the address) is printed; for a (const or not) char +// // pointer, the NUL-terminated string (but not the pointer) is +// // printed. +// void ::testing::internal::UniversalTersePrint(const T& value, ostream*); +// +// // Prints value using the type inferred by the compiler. The difference +// // from UniversalTersePrint() is that this function prints both the +// // pointer and the NUL-terminated string for a (const or not) char pointer. +// void ::testing::internal::UniversalPrint(const T& value, ostream*); +// +// // Prints the fields of a tuple tersely to a string vector, one +// // element for each field. Tuple support must be enabled in +// // gtest-port.h. +// std::vector UniversalTersePrintTupleFieldsToStrings( +// const Tuple& value); +// +// Known limitation: +// +// The print primitives print the elements of an STL-style container +// using the compiler-inferred type of *iter where iter is a +// const_iterator of the container. When const_iterator is an input +// iterator but not a forward iterator, this inferred type may not +// match value_type, and the print output may be incorrect. In +// practice, this is rarely a problem as for most containers +// const_iterator is a forward iterator. We'll fix this if there's an +// actual need for it. Note that this fix cannot rely on value_type +// being defined as many user-defined container types don't have +// value_type. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ + +#include // NOLINT +#include +#include +#include +#include +#include "gtest/internal/gtest-port.h" +#include "gtest/internal/gtest-internal.h" + +#if GTEST_HAS_STD_TUPLE_ +# include +#endif + +namespace testing { + +// Definitions in the 'internal' and 'internal2' name spaces are +// subject to change without notice. DO NOT USE THEM IN USER CODE! +namespace internal2 { + +// Prints the given number of bytes in the given object to the given +// ostream. +GTEST_API_ void PrintBytesInObjectTo(const unsigned char* obj_bytes, + size_t count, + ::std::ostream* os); + +// For selecting which printer to use when a given type has neither << +// nor PrintTo(). +enum TypeKind { + kProtobuf, // a protobuf type + kConvertibleToInteger, // a type implicitly convertible to BiggestInt + // (e.g. a named or unnamed enum type) + kOtherType // anything else +}; + +// TypeWithoutFormatter::PrintValue(value, os) is called +// by the universal printer to print a value of type T when neither +// operator<< nor PrintTo() is defined for T, where kTypeKind is the +// "kind" of T as defined by enum TypeKind. +template +class TypeWithoutFormatter { + public: + // This default version is called when kTypeKind is kOtherType. + static void PrintValue(const T& value, ::std::ostream* os) { + PrintBytesInObjectTo(reinterpret_cast(&value), + sizeof(value), os); + } +}; + +// We print a protobuf using its ShortDebugString() when the string +// doesn't exceed this many characters; otherwise we print it using +// DebugString() for better readability. +const size_t kProtobufOneLinerMaxLength = 50; + +template +class TypeWithoutFormatter { + public: + static void PrintValue(const T& value, ::std::ostream* os) { + const ::testing::internal::string short_str = value.ShortDebugString(); + const ::testing::internal::string pretty_str = + short_str.length() <= kProtobufOneLinerMaxLength ? + short_str : ("\n" + value.DebugString()); + *os << ("<" + pretty_str + ">"); + } +}; + +template +class TypeWithoutFormatter { + public: + // Since T has no << operator or PrintTo() but can be implicitly + // converted to BiggestInt, we print it as a BiggestInt. + // + // Most likely T is an enum type (either named or unnamed), in which + // case printing it as an integer is the desired behavior. In case + // T is not an enum, printing it as an integer is the best we can do + // given that it has no user-defined printer. + static void PrintValue(const T& value, ::std::ostream* os) { + const internal::BiggestInt kBigInt = value; + *os << kBigInt; + } +}; + +// Prints the given value to the given ostream. If the value is a +// protocol message, its debug string is printed; if it's an enum or +// of a type implicitly convertible to BiggestInt, it's printed as an +// integer; otherwise the bytes in the value are printed. This is +// what UniversalPrinter::Print() does when it knows nothing about +// type T and T has neither << operator nor PrintTo(). +// +// A user can override this behavior for a class type Foo by defining +// a << operator in the namespace where Foo is defined. +// +// We put this operator in namespace 'internal2' instead of 'internal' +// to simplify the implementation, as much code in 'internal' needs to +// use << in STL, which would conflict with our own << were it defined +// in 'internal'. +// +// Note that this operator<< takes a generic std::basic_ostream type instead of the more restricted std::ostream. If +// we define it to take an std::ostream instead, we'll get an +// "ambiguous overloads" compiler error when trying to print a type +// Foo that supports streaming to std::basic_ostream, as the compiler cannot tell whether +// operator<<(std::ostream&, const T&) or +// operator<<(std::basic_stream, const Foo&) is more +// specific. +template +::std::basic_ostream& operator<<( + ::std::basic_ostream& os, const T& x) { + TypeWithoutFormatter::value ? kProtobuf : + internal::ImplicitlyConvertible::value ? + kConvertibleToInteger : kOtherType)>::PrintValue(x, &os); + return os; +} + +} // namespace internal2 +} // namespace testing + +// This namespace MUST NOT BE NESTED IN ::testing, or the name look-up +// magic needed for implementing UniversalPrinter won't work. +namespace testing_internal { + +// Used to print a value that is not an STL-style container when the +// user doesn't define PrintTo() for it. +template +void DefaultPrintNonContainerTo(const T& value, ::std::ostream* os) { + // With the following statement, during unqualified name lookup, + // testing::internal2::operator<< appears as if it was declared in + // the nearest enclosing namespace that contains both + // ::testing_internal and ::testing::internal2, i.e. the global + // namespace. For more details, refer to the C++ Standard section + // 7.3.4-1 [namespace.udir]. This allows us to fall back onto + // testing::internal2::operator<< in case T doesn't come with a << + // operator. + // + // We cannot write 'using ::testing::internal2::operator<<;', which + // gcc 3.3 fails to compile due to a compiler bug. + using namespace ::testing::internal2; // NOLINT + + // Assuming T is defined in namespace foo, in the next statement, + // the compiler will consider all of: + // + // 1. foo::operator<< (thanks to Koenig look-up), + // 2. ::operator<< (as the current namespace is enclosed in ::), + // 3. testing::internal2::operator<< (thanks to the using statement above). + // + // The operator<< whose type matches T best will be picked. + // + // We deliberately allow #2 to be a candidate, as sometimes it's + // impossible to define #1 (e.g. when foo is ::std, defining + // anything in it is undefined behavior unless you are a compiler + // vendor.). + *os << value; +} + +} // namespace testing_internal + +namespace testing { +namespace internal { + +// FormatForComparison::Format(value) formats a +// value of type ToPrint that is an operand of a comparison assertion +// (e.g. ASSERT_EQ). OtherOperand is the type of the other operand in +// the comparison, and is used to help determine the best way to +// format the value. In particular, when the value is a C string +// (char pointer) and the other operand is an STL string object, we +// want to format the C string as a string, since we know it is +// compared by value with the string object. If the value is a char +// pointer but the other operand is not an STL string object, we don't +// know whether the pointer is supposed to point to a NUL-terminated +// string, and thus want to print it as a pointer to be safe. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + +// The default case. +template +class FormatForComparison { + public: + static ::std::string Format(const ToPrint& value) { + return ::testing::PrintToString(value); + } +}; + +// Array. +template +class FormatForComparison { + public: + static ::std::string Format(const ToPrint* value) { + return FormatForComparison::Format(value); + } +}; + +// By default, print C string as pointers to be safe, as we don't know +// whether they actually point to a NUL-terminated string. + +#define GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(CharType) \ + template \ + class FormatForComparison { \ + public: \ + static ::std::string Format(CharType* value) { \ + return ::testing::PrintToString(static_cast(value)); \ + } \ + } + +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(wchar_t); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const wchar_t); + +#undef GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_ + +// If a C string is compared with an STL string object, we know it's meant +// to point to a NUL-terminated string, and thus can print it as a string. + +#define GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(CharType, OtherStringType) \ + template <> \ + class FormatForComparison { \ + public: \ + static ::std::string Format(CharType* value) { \ + return ::testing::PrintToString(value); \ + } \ + } + +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::std::string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::std::string); + +#if GTEST_HAS_GLOBAL_STRING +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::string); +#endif + +#if GTEST_HAS_GLOBAL_WSTRING +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::wstring); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::wstring); +#endif + +#if GTEST_HAS_STD_WSTRING +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::std::wstring); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::std::wstring); +#endif + +#undef GTEST_IMPL_FORMAT_C_STRING_AS_STRING_ + +// Formats a comparison assertion (e.g. ASSERT_EQ, EXPECT_LT, and etc) +// operand to be used in a failure message. The type (but not value) +// of the other operand may affect the format. This allows us to +// print a char* as a raw pointer when it is compared against another +// char* or void*, and print it as a C string when it is compared +// against an std::string object, for example. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +template +std::string FormatForComparisonFailureMessage( + const T1& value, const T2& /* other_operand */) { + return FormatForComparison::Format(value); +} + +// UniversalPrinter::Print(value, ostream_ptr) prints the given +// value to the given ostream. The caller must ensure that +// 'ostream_ptr' is not NULL, or the behavior is undefined. +// +// We define UniversalPrinter as a class template (as opposed to a +// function template), as we need to partially specialize it for +// reference types, which cannot be done with function templates. +template +class UniversalPrinter; + +template +void UniversalPrint(const T& value, ::std::ostream* os); + +// Used to print an STL-style container when the user doesn't define +// a PrintTo() for it. +template +void DefaultPrintTo(IsContainer /* dummy */, + false_type /* is not a pointer */, + const C& container, ::std::ostream* os) { + const size_t kMaxCount = 32; // The maximum number of elements to print. + *os << '{'; + size_t count = 0; + for (typename C::const_iterator it = container.begin(); + it != container.end(); ++it, ++count) { + if (count > 0) { + *os << ','; + if (count == kMaxCount) { // Enough has been printed. + *os << " ..."; + break; + } + } + *os << ' '; + // We cannot call PrintTo(*it, os) here as PrintTo() doesn't + // handle *it being a native array. + internal::UniversalPrint(*it, os); + } + + if (count > 0) { + *os << ' '; + } + *os << '}'; +} + +// Used to print a pointer that is neither a char pointer nor a member +// pointer, when the user doesn't define PrintTo() for it. (A member +// variable pointer or member function pointer doesn't really point to +// a location in the address space. Their representation is +// implementation-defined. Therefore they will be printed as raw +// bytes.) +template +void DefaultPrintTo(IsNotContainer /* dummy */, + true_type /* is a pointer */, + T* p, ::std::ostream* os) { + if (p == NULL) { + *os << "NULL"; + } else { + // C++ doesn't allow casting from a function pointer to any object + // pointer. + // + // IsTrue() silences warnings: "Condition is always true", + // "unreachable code". + if (IsTrue(ImplicitlyConvertible::value)) { + // T is not a function type. We just call << to print p, + // relying on ADL to pick up user-defined << for their pointer + // types, if any. + *os << p; + } else { + // T is a function type, so '*os << p' doesn't do what we want + // (it just prints p as bool). We want to print p as a const + // void*. However, we cannot cast it to const void* directly, + // even using reinterpret_cast, as earlier versions of gcc + // (e.g. 3.4.5) cannot compile the cast when p is a function + // pointer. Casting to UInt64 first solves the problem. + *os << reinterpret_cast( + reinterpret_cast(p)); + } + } +} + +// Used to print a non-container, non-pointer value when the user +// doesn't define PrintTo() for it. +template +void DefaultPrintTo(IsNotContainer /* dummy */, + false_type /* is not a pointer */, + const T& value, ::std::ostream* os) { + ::testing_internal::DefaultPrintNonContainerTo(value, os); +} + +// Prints the given value using the << operator if it has one; +// otherwise prints the bytes in it. This is what +// UniversalPrinter::Print() does when PrintTo() is not specialized +// or overloaded for type T. +// +// A user can override this behavior for a class type Foo by defining +// an overload of PrintTo() in the namespace where Foo is defined. We +// give the user this option as sometimes defining a << operator for +// Foo is not desirable (e.g. the coding style may prevent doing it, +// or there is already a << operator but it doesn't do what the user +// wants). +template +void PrintTo(const T& value, ::std::ostream* os) { + // DefaultPrintTo() is overloaded. The type of its first two + // arguments determine which version will be picked. If T is an + // STL-style container, the version for container will be called; if + // T is a pointer, the pointer version will be called; otherwise the + // generic version will be called. + // + // Note that we check for container types here, prior to we check + // for protocol message types in our operator<<. The rationale is: + // + // For protocol messages, we want to give people a chance to + // override Google Mock's format by defining a PrintTo() or + // operator<<. For STL containers, other formats can be + // incompatible with Google Mock's format for the container + // elements; therefore we check for container types here to ensure + // that our format is used. + // + // The second argument of DefaultPrintTo() is needed to bypass a bug + // in Symbian's C++ compiler that prevents it from picking the right + // overload between: + // + // PrintTo(const T& x, ...); + // PrintTo(T* x, ...); + DefaultPrintTo(IsContainerTest(0), is_pointer(), value, os); +} + +// The following list of PrintTo() overloads tells +// UniversalPrinter::Print() how to print standard types (built-in +// types, strings, plain arrays, and pointers). + +// Overloads for various char types. +GTEST_API_ void PrintTo(unsigned char c, ::std::ostream* os); +GTEST_API_ void PrintTo(signed char c, ::std::ostream* os); +inline void PrintTo(char c, ::std::ostream* os) { + // When printing a plain char, we always treat it as unsigned. This + // way, the output won't be affected by whether the compiler thinks + // char is signed or not. + PrintTo(static_cast(c), os); +} + +// Overloads for other simple built-in types. +inline void PrintTo(bool x, ::std::ostream* os) { + *os << (x ? "true" : "false"); +} + +// Overload for wchar_t type. +// Prints a wchar_t as a symbol if it is printable or as its internal +// code otherwise and also as its decimal code (except for L'\0'). +// The L'\0' char is printed as "L'\\0'". The decimal code is printed +// as signed integer when wchar_t is implemented by the compiler +// as a signed type and is printed as an unsigned integer when wchar_t +// is implemented as an unsigned type. +GTEST_API_ void PrintTo(wchar_t wc, ::std::ostream* os); + +// Overloads for C strings. +GTEST_API_ void PrintTo(const char* s, ::std::ostream* os); +inline void PrintTo(char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} + +// signed/unsigned char is often used for representing binary data, so +// we print pointers to it as void* to be safe. +inline void PrintTo(const signed char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +inline void PrintTo(signed char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +inline void PrintTo(const unsigned char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +inline void PrintTo(unsigned char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} + +// MSVC can be configured to define wchar_t as a typedef of unsigned +// short. It defines _NATIVE_WCHAR_T_DEFINED when wchar_t is a native +// type. When wchar_t is a typedef, defining an overload for const +// wchar_t* would cause unsigned short* be printed as a wide string, +// possibly causing invalid memory accesses. +#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) +// Overloads for wide C strings +GTEST_API_ void PrintTo(const wchar_t* s, ::std::ostream* os); +inline void PrintTo(wchar_t* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +#endif + +// Overload for C arrays. Multi-dimensional arrays are printed +// properly. + +// Prints the given number of elements in an array, without printing +// the curly braces. +template +void PrintRawArrayTo(const T a[], size_t count, ::std::ostream* os) { + UniversalPrint(a[0], os); + for (size_t i = 1; i != count; i++) { + *os << ", "; + UniversalPrint(a[i], os); + } +} + +// Overloads for ::string and ::std::string. +#if GTEST_HAS_GLOBAL_STRING +GTEST_API_ void PrintStringTo(const ::string&s, ::std::ostream* os); +inline void PrintTo(const ::string& s, ::std::ostream* os) { + PrintStringTo(s, os); +} +#endif // GTEST_HAS_GLOBAL_STRING + +GTEST_API_ void PrintStringTo(const ::std::string&s, ::std::ostream* os); +inline void PrintTo(const ::std::string& s, ::std::ostream* os) { + PrintStringTo(s, os); +} + +// Overloads for ::wstring and ::std::wstring. +#if GTEST_HAS_GLOBAL_WSTRING +GTEST_API_ void PrintWideStringTo(const ::wstring&s, ::std::ostream* os); +inline void PrintTo(const ::wstring& s, ::std::ostream* os) { + PrintWideStringTo(s, os); +} +#endif // GTEST_HAS_GLOBAL_WSTRING + +#if GTEST_HAS_STD_WSTRING +GTEST_API_ void PrintWideStringTo(const ::std::wstring&s, ::std::ostream* os); +inline void PrintTo(const ::std::wstring& s, ::std::ostream* os) { + PrintWideStringTo(s, os); +} +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ +// Helper function for printing a tuple. T must be instantiated with +// a tuple type. +template +void PrintTupleTo(const T& t, ::std::ostream* os); +#endif // GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ + +#if GTEST_HAS_TR1_TUPLE +// Overload for ::std::tr1::tuple. Needed for printing function arguments, +// which are packed as tuples. + +// Overloaded PrintTo() for tuples of various arities. We support +// tuples of up-to 10 fields. The following implementation works +// regardless of whether tr1::tuple is implemented using the +// non-standard variadic template feature or not. + +inline void PrintTo(const ::std::tr1::tuple<>& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo( + const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} +#endif // GTEST_HAS_TR1_TUPLE + +#if GTEST_HAS_STD_TUPLE_ +template +void PrintTo(const ::std::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} +#endif // GTEST_HAS_STD_TUPLE_ + +// Overload for std::pair. +template +void PrintTo(const ::std::pair& value, ::std::ostream* os) { + *os << '('; + // We cannot use UniversalPrint(value.first, os) here, as T1 may be + // a reference type. The same for printing value.second. + UniversalPrinter::Print(value.first, os); + *os << ", "; + UniversalPrinter::Print(value.second, os); + *os << ')'; +} + +// Implements printing a non-reference type T by letting the compiler +// pick the right overload of PrintTo() for T. +template +class UniversalPrinter { + public: + // MSVC warns about adding const to a function type, so we want to + // disable the warning. + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4180) + + // Note: we deliberately don't call this PrintTo(), as that name + // conflicts with ::testing::internal::PrintTo in the body of the + // function. + static void Print(const T& value, ::std::ostream* os) { + // By default, ::testing::internal::PrintTo() is used for printing + // the value. + // + // Thanks to Koenig look-up, if T is a class and has its own + // PrintTo() function defined in its namespace, that function will + // be visible here. Since it is more specific than the generic ones + // in ::testing::internal, it will be picked by the compiler in the + // following statement - exactly what we want. + PrintTo(value, os); + } + + GTEST_DISABLE_MSC_WARNINGS_POP_() +}; + +// UniversalPrintArray(begin, len, os) prints an array of 'len' +// elements, starting at address 'begin'. +template +void UniversalPrintArray(const T* begin, size_t len, ::std::ostream* os) { + if (len == 0) { + *os << "{}"; + } else { + *os << "{ "; + const size_t kThreshold = 18; + const size_t kChunkSize = 8; + // If the array has more than kThreshold elements, we'll have to + // omit some details by printing only the first and the last + // kChunkSize elements. + // TODO(wan@google.com): let the user control the threshold using a flag. + if (len <= kThreshold) { + PrintRawArrayTo(begin, len, os); + } else { + PrintRawArrayTo(begin, kChunkSize, os); + *os << ", ..., "; + PrintRawArrayTo(begin + len - kChunkSize, kChunkSize, os); + } + *os << " }"; + } +} +// This overload prints a (const) char array compactly. +GTEST_API_ void UniversalPrintArray( + const char* begin, size_t len, ::std::ostream* os); + +// This overload prints a (const) wchar_t array compactly. +GTEST_API_ void UniversalPrintArray( + const wchar_t* begin, size_t len, ::std::ostream* os); + +// Implements printing an array type T[N]. +template +class UniversalPrinter { + public: + // Prints the given array, omitting some elements when there are too + // many. + static void Print(const T (&a)[N], ::std::ostream* os) { + UniversalPrintArray(a, N, os); + } +}; + +// Implements printing a reference type T&. +template +class UniversalPrinter { + public: + // MSVC warns about adding const to a function type, so we want to + // disable the warning. + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4180) + + static void Print(const T& value, ::std::ostream* os) { + // Prints the address of the value. We use reinterpret_cast here + // as static_cast doesn't compile when T is a function type. + *os << "@" << reinterpret_cast(&value) << " "; + + // Then prints the value itself. + UniversalPrint(value, os); + } + + GTEST_DISABLE_MSC_WARNINGS_POP_() +}; + +// Prints a value tersely: for a reference type, the referenced value +// (but not the address) is printed; for a (const) char pointer, the +// NUL-terminated string (but not the pointer) is printed. + +template +class UniversalTersePrinter { + public: + static void Print(const T& value, ::std::ostream* os) { + UniversalPrint(value, os); + } +}; +template +class UniversalTersePrinter { + public: + static void Print(const T& value, ::std::ostream* os) { + UniversalPrint(value, os); + } +}; +template +class UniversalTersePrinter { + public: + static void Print(const T (&value)[N], ::std::ostream* os) { + UniversalPrinter::Print(value, os); + } +}; +template <> +class UniversalTersePrinter { + public: + static void Print(const char* str, ::std::ostream* os) { + if (str == NULL) { + *os << "NULL"; + } else { + UniversalPrint(string(str), os); + } + } +}; +template <> +class UniversalTersePrinter { + public: + static void Print(char* str, ::std::ostream* os) { + UniversalTersePrinter::Print(str, os); + } +}; + +#if GTEST_HAS_STD_WSTRING +template <> +class UniversalTersePrinter { + public: + static void Print(const wchar_t* str, ::std::ostream* os) { + if (str == NULL) { + *os << "NULL"; + } else { + UniversalPrint(::std::wstring(str), os); + } + } +}; +#endif + +template <> +class UniversalTersePrinter { + public: + static void Print(wchar_t* str, ::std::ostream* os) { + UniversalTersePrinter::Print(str, os); + } +}; + +template +void UniversalTersePrint(const T& value, ::std::ostream* os) { + UniversalTersePrinter::Print(value, os); +} + +// Prints a value using the type inferred by the compiler. The +// difference between this and UniversalTersePrint() is that for a +// (const) char pointer, this prints both the pointer and the +// NUL-terminated string. +template +void UniversalPrint(const T& value, ::std::ostream* os) { + // A workarond for the bug in VC++ 7.1 that prevents us from instantiating + // UniversalPrinter with T directly. + typedef T T1; + UniversalPrinter::Print(value, os); +} + +typedef ::std::vector Strings; + +// TuplePolicy must provide: +// - tuple_size +// size of tuple TupleT. +// - get(const TupleT& t) +// static function extracting element I of tuple TupleT. +// - tuple_element::type +// type of element I of tuple TupleT. +template +struct TuplePolicy; + +#if GTEST_HAS_TR1_TUPLE +template +struct TuplePolicy { + typedef TupleT Tuple; + static const size_t tuple_size = ::std::tr1::tuple_size::value; + + template + struct tuple_element : ::std::tr1::tuple_element {}; + + template + static typename AddReference< + const typename ::std::tr1::tuple_element::type>::type get( + const Tuple& tuple) { + return ::std::tr1::get(tuple); + } +}; +template +const size_t TuplePolicy::tuple_size; +#endif // GTEST_HAS_TR1_TUPLE + +#if GTEST_HAS_STD_TUPLE_ +template +struct TuplePolicy< ::std::tuple > { + typedef ::std::tuple Tuple; + static const size_t tuple_size = ::std::tuple_size::value; + + template + struct tuple_element : ::std::tuple_element {}; + + template + static const typename ::std::tuple_element::type& get( + const Tuple& tuple) { + return ::std::get(tuple); + } +}; +template +const size_t TuplePolicy< ::std::tuple >::tuple_size; +#endif // GTEST_HAS_STD_TUPLE_ + +#if GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ +// This helper template allows PrintTo() for tuples and +// UniversalTersePrintTupleFieldsToStrings() to be defined by +// induction on the number of tuple fields. The idea is that +// TuplePrefixPrinter::PrintPrefixTo(t, os) prints the first N +// fields in tuple t, and can be defined in terms of +// TuplePrefixPrinter. +// +// The inductive case. +template +struct TuplePrefixPrinter { + // Prints the first N fields of a tuple. + template + static void PrintPrefixTo(const Tuple& t, ::std::ostream* os) { + TuplePrefixPrinter::PrintPrefixTo(t, os); + GTEST_INTENTIONAL_CONST_COND_PUSH_() + if (N > 1) { + GTEST_INTENTIONAL_CONST_COND_POP_() + *os << ", "; + } + UniversalPrinter< + typename TuplePolicy::template tuple_element::type> + ::Print(TuplePolicy::template get(t), os); + } + + // Tersely prints the first N fields of a tuple to a string vector, + // one element for each field. + template + static void TersePrintPrefixToStrings(const Tuple& t, Strings* strings) { + TuplePrefixPrinter::TersePrintPrefixToStrings(t, strings); + ::std::stringstream ss; + UniversalTersePrint(TuplePolicy::template get(t), &ss); + strings->push_back(ss.str()); + } +}; + +// Base case. +template <> +struct TuplePrefixPrinter<0> { + template + static void PrintPrefixTo(const Tuple&, ::std::ostream*) {} + + template + static void TersePrintPrefixToStrings(const Tuple&, Strings*) {} +}; + +// Helper function for printing a tuple. +// Tuple must be either std::tr1::tuple or std::tuple type. +template +void PrintTupleTo(const Tuple& t, ::std::ostream* os) { + *os << "("; + TuplePrefixPrinter::tuple_size>::PrintPrefixTo(t, os); + *os << ")"; +} + +// Prints the fields of a tuple tersely to a string vector, one +// element for each field. See the comment before +// UniversalTersePrint() for how we define "tersely". +template +Strings UniversalTersePrintTupleFieldsToStrings(const Tuple& value) { + Strings result; + TuplePrefixPrinter::tuple_size>:: + TersePrintPrefixToStrings(value, &result); + return result; +} +#endif // GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ + +} // namespace internal + +template +::std::string PrintToString(const T& value) { + ::std::stringstream ss; + internal::UniversalTersePrinter::Print(value, &ss); + return ss.str(); +} + +} // namespace testing + +// Include any custom printer added by the local installation. +// We must include this header at the end to make sure it can use the +// declarations from this file. +#include "gtest/internal/custom/gtest-printers.h" + +#endif // GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-spi.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-spi.h new file mode 100644 index 0000000000..f63fa9a1b2 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-spi.h @@ -0,0 +1,232 @@ +// Copyright 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// Utilities for testing Google Test itself and code that uses Google Test +// (e.g. frameworks built on top of Google Test). + +#ifndef GTEST_INCLUDE_GTEST_GTEST_SPI_H_ +#define GTEST_INCLUDE_GTEST_GTEST_SPI_H_ + +#include "gtest/gtest.h" + +namespace testing { + +// This helper class can be used to mock out Google Test failure reporting +// so that we can test Google Test or code that builds on Google Test. +// +// An object of this class appends a TestPartResult object to the +// TestPartResultArray object given in the constructor whenever a Google Test +// failure is reported. It can either intercept only failures that are +// generated in the same thread that created this object or it can intercept +// all generated failures. The scope of this mock object can be controlled with +// the second argument to the two arguments constructor. +class GTEST_API_ ScopedFakeTestPartResultReporter + : public TestPartResultReporterInterface { + public: + // The two possible mocking modes of this object. + enum InterceptMode { + INTERCEPT_ONLY_CURRENT_THREAD, // Intercepts only thread local failures. + INTERCEPT_ALL_THREADS // Intercepts all failures. + }; + + // The c'tor sets this object as the test part result reporter used + // by Google Test. The 'result' parameter specifies where to report the + // results. This reporter will only catch failures generated in the current + // thread. DEPRECATED + explicit ScopedFakeTestPartResultReporter(TestPartResultArray* result); + + // Same as above, but you can choose the interception scope of this object. + ScopedFakeTestPartResultReporter(InterceptMode intercept_mode, + TestPartResultArray* result); + + // The d'tor restores the previous test part result reporter. + virtual ~ScopedFakeTestPartResultReporter(); + + // Appends the TestPartResult object to the TestPartResultArray + // received in the constructor. + // + // This method is from the TestPartResultReporterInterface + // interface. + virtual void ReportTestPartResult(const TestPartResult& result); + private: + void Init(); + + const InterceptMode intercept_mode_; + TestPartResultReporterInterface* old_reporter_; + TestPartResultArray* const result_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedFakeTestPartResultReporter); +}; + +namespace internal { + +// A helper class for implementing EXPECT_FATAL_FAILURE() and +// EXPECT_NONFATAL_FAILURE(). Its destructor verifies that the given +// TestPartResultArray contains exactly one failure that has the given +// type and contains the given substring. If that's not the case, a +// non-fatal failure will be generated. +class GTEST_API_ SingleFailureChecker { + public: + // The constructor remembers the arguments. + SingleFailureChecker(const TestPartResultArray* results, + TestPartResult::Type type, + const string& substr); + ~SingleFailureChecker(); + private: + const TestPartResultArray* const results_; + const TestPartResult::Type type_; + const string substr_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(SingleFailureChecker); +}; + +} // namespace internal + +} // namespace testing + +// A set of macros for testing Google Test assertions or code that's expected +// to generate Google Test fatal failures. It verifies that the given +// statement will cause exactly one fatal Google Test failure with 'substr' +// being part of the failure message. +// +// There are two different versions of this macro. EXPECT_FATAL_FAILURE only +// affects and considers failures generated in the current thread and +// EXPECT_FATAL_FAILURE_ON_ALL_THREADS does the same but for all threads. +// +// The verification of the assertion is done correctly even when the statement +// throws an exception or aborts the current function. +// +// Known restrictions: +// - 'statement' cannot reference local non-static variables or +// non-static members of the current object. +// - 'statement' cannot return a value. +// - You cannot stream a failure message to this macro. +// +// Note that even though the implementations of the following two +// macros are much alike, we cannot refactor them to use a common +// helper macro, due to some peculiarity in how the preprocessor +// works. The AcceptsMacroThatExpandsToUnprotectedComma test in +// gtest_unittest.cc will fail to compile if we do that. +#define EXPECT_FATAL_FAILURE(statement, substr) \ + do { \ + class GTestExpectFatalFailureHelper {\ + public:\ + static void Execute() { statement; }\ + };\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kFatalFailure, (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ONLY_CURRENT_THREAD, >est_failures);\ + GTestExpectFatalFailureHelper::Execute();\ + }\ + } while (::testing::internal::AlwaysFalse()) + +#define EXPECT_FATAL_FAILURE_ON_ALL_THREADS(statement, substr) \ + do { \ + class GTestExpectFatalFailureHelper {\ + public:\ + static void Execute() { statement; }\ + };\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kFatalFailure, (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ALL_THREADS, >est_failures);\ + GTestExpectFatalFailureHelper::Execute();\ + }\ + } while (::testing::internal::AlwaysFalse()) + +// A macro for testing Google Test assertions or code that's expected to +// generate Google Test non-fatal failures. It asserts that the given +// statement will cause exactly one non-fatal Google Test failure with 'substr' +// being part of the failure message. +// +// There are two different versions of this macro. EXPECT_NONFATAL_FAILURE only +// affects and considers failures generated in the current thread and +// EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS does the same but for all threads. +// +// 'statement' is allowed to reference local variables and members of +// the current object. +// +// The verification of the assertion is done correctly even when the statement +// throws an exception or aborts the current function. +// +// Known restrictions: +// - You cannot stream a failure message to this macro. +// +// Note that even though the implementations of the following two +// macros are much alike, we cannot refactor them to use a common +// helper macro, due to some peculiarity in how the preprocessor +// works. If we do that, the code won't compile when the user gives +// EXPECT_NONFATAL_FAILURE() a statement that contains a macro that +// expands to code containing an unprotected comma. The +// AcceptsMacroThatExpandsToUnprotectedComma test in gtest_unittest.cc +// catches that. +// +// For the same reason, we have to write +// if (::testing::internal::AlwaysTrue()) { statement; } +// instead of +// GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) +// to avoid an MSVC warning on unreachable code. +#define EXPECT_NONFATAL_FAILURE(statement, substr) \ + do {\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kNonFatalFailure, \ + (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ONLY_CURRENT_THREAD, >est_failures);\ + if (::testing::internal::AlwaysTrue()) { statement; }\ + }\ + } while (::testing::internal::AlwaysFalse()) + +#define EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS(statement, substr) \ + do {\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kNonFatalFailure, \ + (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter::INTERCEPT_ALL_THREADS, \ + >est_failures);\ + if (::testing::internal::AlwaysTrue()) { statement; }\ + }\ + } while (::testing::internal::AlwaysFalse()) + +#endif // GTEST_INCLUDE_GTEST_GTEST_SPI_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-test-part.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-test-part.h new file mode 100644 index 0000000000..77eb844839 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-test-part.h @@ -0,0 +1,179 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: mheule@google.com (Markus Heule) +// + +#ifndef GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ +#define GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ + +#include +#include +#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-string.h" + +namespace testing { + +// A copyable object representing the result of a test part (i.e. an +// assertion or an explicit FAIL(), ADD_FAILURE(), or SUCCESS()). +// +// Don't inherit from TestPartResult as its destructor is not virtual. +class GTEST_API_ TestPartResult { + public: + // The possible outcomes of a test part (i.e. an assertion or an + // explicit SUCCEED(), FAIL(), or ADD_FAILURE()). + enum Type { + kSuccess, // Succeeded. + kNonFatalFailure, // Failed but the test can continue. + kFatalFailure // Failed and the test should be terminated. + }; + + // C'tor. TestPartResult does NOT have a default constructor. + // Always use this constructor (with parameters) to create a + // TestPartResult object. + TestPartResult(Type a_type, + const char* a_file_name, + int a_line_number, + const char* a_message) + : type_(a_type), + file_name_(a_file_name == NULL ? "" : a_file_name), + line_number_(a_line_number), + summary_(ExtractSummary(a_message)), + message_(a_message) { + } + + // Gets the outcome of the test part. + Type type() const { return type_; } + + // Gets the name of the source file where the test part took place, or + // NULL if it's unknown. + const char* file_name() const { + return file_name_.empty() ? NULL : file_name_.c_str(); + } + + // Gets the line in the source file where the test part took place, + // or -1 if it's unknown. + int line_number() const { return line_number_; } + + // Gets the summary of the failure message. + const char* summary() const { return summary_.c_str(); } + + // Gets the message associated with the test part. + const char* message() const { return message_.c_str(); } + + // Returns true iff the test part passed. + bool passed() const { return type_ == kSuccess; } + + // Returns true iff the test part failed. + bool failed() const { return type_ != kSuccess; } + + // Returns true iff the test part non-fatally failed. + bool nonfatally_failed() const { return type_ == kNonFatalFailure; } + + // Returns true iff the test part fatally failed. + bool fatally_failed() const { return type_ == kFatalFailure; } + + private: + Type type_; + + // Gets the summary of the failure message by omitting the stack + // trace in it. + static std::string ExtractSummary(const char* message); + + // The name of the source file where the test part took place, or + // "" if the source file is unknown. + std::string file_name_; + // The line in the source file where the test part took place, or -1 + // if the line number is unknown. + int line_number_; + std::string summary_; // The test failure summary. + std::string message_; // The test failure message. +}; + +// Prints a TestPartResult object. +std::ostream& operator<<(std::ostream& os, const TestPartResult& result); + +// An array of TestPartResult objects. +// +// Don't inherit from TestPartResultArray as its destructor is not +// virtual. +class GTEST_API_ TestPartResultArray { + public: + TestPartResultArray() {} + + // Appends the given TestPartResult to the array. + void Append(const TestPartResult& result); + + // Returns the TestPartResult at the given index (0-based). + const TestPartResult& GetTestPartResult(int index) const; + + // Returns the number of TestPartResult objects in the array. + int size() const; + + private: + std::vector array_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestPartResultArray); +}; + +// This interface knows how to report a test part result. +class TestPartResultReporterInterface { + public: + virtual ~TestPartResultReporterInterface() {} + + virtual void ReportTestPartResult(const TestPartResult& result) = 0; +}; + +namespace internal { + +// This helper class is used by {ASSERT|EXPECT}_NO_FATAL_FAILURE to check if a +// statement generates new fatal failures. To do so it registers itself as the +// current test part result reporter. Besides checking if fatal failures were +// reported, it only delegates the reporting to the former result reporter. +// The original result reporter is restored in the destructor. +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +class GTEST_API_ HasNewFatalFailureHelper + : public TestPartResultReporterInterface { + public: + HasNewFatalFailureHelper(); + virtual ~HasNewFatalFailureHelper(); + virtual void ReportTestPartResult(const TestPartResult& result); + bool has_new_fatal_failure() const { return has_new_fatal_failure_; } + private: + bool has_new_fatal_failure_; + TestPartResultReporterInterface* original_reporter_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(HasNewFatalFailureHelper); +}; + +} // namespace internal + +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-typed-test.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-typed-test.h new file mode 100644 index 0000000000..5f69d5678e --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest-typed-test.h @@ -0,0 +1,263 @@ +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +#ifndef GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ + +// This header implements typed tests and type-parameterized tests. + +// Typed (aka type-driven) tests repeat the same test for types in a +// list. You must know which types you want to test with when writing +// typed tests. Here's how you do it: + +#if 0 + +// First, define a fixture class template. It should be parameterized +// by a type. Remember to derive it from testing::Test. +template +class FooTest : public testing::Test { + public: + ... + typedef std::list List; + static T shared_; + T value_; +}; + +// Next, associate a list of types with the test case, which will be +// repeated for each type in the list. The typedef is necessary for +// the macro to parse correctly. +typedef testing::Types MyTypes; +TYPED_TEST_CASE(FooTest, MyTypes); + +// If the type list contains only one type, you can write that type +// directly without Types<...>: +// TYPED_TEST_CASE(FooTest, int); + +// Then, use TYPED_TEST() instead of TEST_F() to define as many typed +// tests for this test case as you want. +TYPED_TEST(FooTest, DoesBlah) { + // Inside a test, refer to TypeParam to get the type parameter. + // Since we are inside a derived class template, C++ requires use to + // visit the members of FooTest via 'this'. + TypeParam n = this->value_; + + // To visit static members of the fixture, add the TestFixture:: + // prefix. + n += TestFixture::shared_; + + // To refer to typedefs in the fixture, add the "typename + // TestFixture::" prefix. + typename TestFixture::List values; + values.push_back(n); + ... +} + +TYPED_TEST(FooTest, HasPropertyA) { ... } + +#endif // 0 + +// Type-parameterized tests are abstract test patterns parameterized +// by a type. Compared with typed tests, type-parameterized tests +// allow you to define the test pattern without knowing what the type +// parameters are. The defined pattern can be instantiated with +// different types any number of times, in any number of translation +// units. +// +// If you are designing an interface or concept, you can define a +// suite of type-parameterized tests to verify properties that any +// valid implementation of the interface/concept should have. Then, +// each implementation can easily instantiate the test suite to verify +// that it conforms to the requirements, without having to write +// similar tests repeatedly. Here's an example: + +#if 0 + +// First, define a fixture class template. It should be parameterized +// by a type. Remember to derive it from testing::Test. +template +class FooTest : public testing::Test { + ... +}; + +// Next, declare that you will define a type-parameterized test case +// (the _P suffix is for "parameterized" or "pattern", whichever you +// prefer): +TYPED_TEST_CASE_P(FooTest); + +// Then, use TYPED_TEST_P() to define as many type-parameterized tests +// for this type-parameterized test case as you want. +TYPED_TEST_P(FooTest, DoesBlah) { + // Inside a test, refer to TypeParam to get the type parameter. + TypeParam n = 0; + ... +} + +TYPED_TEST_P(FooTest, HasPropertyA) { ... } + +// Now the tricky part: you need to register all test patterns before +// you can instantiate them. The first argument of the macro is the +// test case name; the rest are the names of the tests in this test +// case. +REGISTER_TYPED_TEST_CASE_P(FooTest, + DoesBlah, HasPropertyA); + +// Finally, you are free to instantiate the pattern with the types you +// want. If you put the above code in a header file, you can #include +// it in multiple C++ source files and instantiate it multiple times. +// +// To distinguish different instances of the pattern, the first +// argument to the INSTANTIATE_* macro is a prefix that will be added +// to the actual test case name. Remember to pick unique prefixes for +// different instances. +typedef testing::Types MyTypes; +INSTANTIATE_TYPED_TEST_CASE_P(My, FooTest, MyTypes); + +// If the type list contains only one type, you can write that type +// directly without Types<...>: +// INSTANTIATE_TYPED_TEST_CASE_P(My, FooTest, int); + +#endif // 0 + +#include "gtest/internal/gtest-port.h" +#include "gtest/internal/gtest-type-util.h" + +// Implements typed tests. + +#if GTEST_HAS_TYPED_TEST + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Expands to the name of the typedef for the type parameters of the +// given test case. +# define GTEST_TYPE_PARAMS_(TestCaseName) gtest_type_params_##TestCaseName##_ + +// The 'Types' template argument below must have spaces around it +// since some compilers may choke on '>>' when passing a template +// instance (e.g. Types) +# define TYPED_TEST_CASE(CaseName, Types) \ + typedef ::testing::internal::TypeList< Types >::type \ + GTEST_TYPE_PARAMS_(CaseName) + +# define TYPED_TEST(CaseName, TestName) \ + template \ + class GTEST_TEST_CLASS_NAME_(CaseName, TestName) \ + : public CaseName { \ + private: \ + typedef CaseName TestFixture; \ + typedef gtest_TypeParam_ TypeParam; \ + virtual void TestBody(); \ + }; \ + bool gtest_##CaseName##_##TestName##_registered_ GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::internal::TypeParameterizedTest< \ + CaseName, \ + ::testing::internal::TemplateSel< \ + GTEST_TEST_CLASS_NAME_(CaseName, TestName)>, \ + GTEST_TYPE_PARAMS_(CaseName)>::Register(\ + "", ::testing::internal::CodeLocation(__FILE__, __LINE__), \ + #CaseName, #TestName, 0); \ + template \ + void GTEST_TEST_CLASS_NAME_(CaseName, TestName)::TestBody() + +#endif // GTEST_HAS_TYPED_TEST + +// Implements type-parameterized tests. + +#if GTEST_HAS_TYPED_TEST_P + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Expands to the namespace name that the type-parameterized tests for +// the given type-parameterized test case are defined in. The exact +// name of the namespace is subject to change without notice. +# define GTEST_CASE_NAMESPACE_(TestCaseName) \ + gtest_case_##TestCaseName##_ + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Expands to the name of the variable used to remember the names of +// the defined tests in the given test case. +# define GTEST_TYPED_TEST_CASE_P_STATE_(TestCaseName) \ + gtest_typed_test_case_p_state_##TestCaseName##_ + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE DIRECTLY. +// +// Expands to the name of the variable used to remember the names of +// the registered tests in the given test case. +# define GTEST_REGISTERED_TEST_NAMES_(TestCaseName) \ + gtest_registered_test_names_##TestCaseName##_ + +// The variables defined in the type-parameterized test macros are +// static as typically these macros are used in a .h file that can be +// #included in multiple translation units linked together. +# define TYPED_TEST_CASE_P(CaseName) \ + static ::testing::internal::TypedTestCasePState \ + GTEST_TYPED_TEST_CASE_P_STATE_(CaseName) + +# define TYPED_TEST_P(CaseName, TestName) \ + namespace GTEST_CASE_NAMESPACE_(CaseName) { \ + template \ + class TestName : public CaseName { \ + private: \ + typedef CaseName TestFixture; \ + typedef gtest_TypeParam_ TypeParam; \ + virtual void TestBody(); \ + }; \ + static bool gtest_##TestName##_defined_ GTEST_ATTRIBUTE_UNUSED_ = \ + GTEST_TYPED_TEST_CASE_P_STATE_(CaseName).AddTestName(\ + __FILE__, __LINE__, #CaseName, #TestName); \ + } \ + template \ + void GTEST_CASE_NAMESPACE_(CaseName)::TestName::TestBody() + +# define REGISTER_TYPED_TEST_CASE_P(CaseName, ...) \ + namespace GTEST_CASE_NAMESPACE_(CaseName) { \ + typedef ::testing::internal::Templates<__VA_ARGS__>::type gtest_AllTests_; \ + } \ + static const char* const GTEST_REGISTERED_TEST_NAMES_(CaseName) = \ + GTEST_TYPED_TEST_CASE_P_STATE_(CaseName).VerifyRegisteredTestNames(\ + __FILE__, __LINE__, #__VA_ARGS__) + +// The 'Types' template argument below must have spaces around it +// since some compilers may choke on '>>' when passing a template +// instance (e.g. Types) +# define INSTANTIATE_TYPED_TEST_CASE_P(Prefix, CaseName, Types) \ + bool gtest_##Prefix##_##CaseName GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::internal::TypeParameterizedTestCase::type>::Register(\ + #Prefix, \ + ::testing::internal::CodeLocation(__FILE__, __LINE__), \ + >EST_TYPED_TEST_CASE_P_STATE_(CaseName), \ + #CaseName, GTEST_REGISTERED_TEST_NAMES_(CaseName)) + +#endif // GTEST_HAS_TYPED_TEST_P + +#endif // GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest.h new file mode 100644 index 0000000000..f846c5bd66 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest.h @@ -0,0 +1,2236 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines the public API for Google Test. It should be +// included by any test program that uses Google Test. +// +// IMPORTANT NOTE: Due to limitation of the C++ language, we have to +// leave some internal implementation details in this header file. +// They are clearly marked by comments like this: +// +// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +// +// Such code is NOT meant to be used by a user directly, and is subject +// to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user +// program! +// +// Acknowledgment: Google Test borrowed the idea of automatic test +// registration from Barthelemy Dagenais' (barthelemy@prologique.com) +// easyUnit framework. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_H_ + +#include +#include +#include + +#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-string.h" +#include "gtest/gtest-death-test.h" +#include "gtest/gtest-message.h" +#include "gtest/gtest-param-test.h" +#include "gtest/gtest-printers.h" +#include "gtest/gtest_prod.h" +#include "gtest/gtest-test-part.h" +#include "gtest/gtest-typed-test.h" + +// Depending on the platform, different string classes are available. +// On Linux, in addition to ::std::string, Google also makes use of +// class ::string, which has the same interface as ::std::string, but +// has a different implementation. +// +// You can define GTEST_HAS_GLOBAL_STRING to 1 to indicate that +// ::string is available AND is a distinct type to ::std::string, or +// define it to 0 to indicate otherwise. +// +// If ::std::string and ::string are the same class on your platform +// due to aliasing, you should define GTEST_HAS_GLOBAL_STRING to 0. +// +// If you do not define GTEST_HAS_GLOBAL_STRING, it is defined +// heuristically. + +namespace testing { + +// Declares the flags. + +// This flag temporary enables the disabled tests. +GTEST_DECLARE_bool_(also_run_disabled_tests); + +// This flag brings the debugger on an assertion failure. +GTEST_DECLARE_bool_(break_on_failure); + +// This flag controls whether Google Test catches all test-thrown exceptions +// and logs them as failures. +GTEST_DECLARE_bool_(catch_exceptions); + +// This flag enables using colors in terminal output. Available values are +// "yes" to enable colors, "no" (disable colors), or "auto" (the default) +// to let Google Test decide. +GTEST_DECLARE_string_(color); + +// This flag sets up the filter to select by name using a glob pattern +// the tests to run. If the filter is not given all tests are executed. +GTEST_DECLARE_string_(filter); + +// This flag causes the Google Test to list tests. None of the tests listed +// are actually run if the flag is provided. +GTEST_DECLARE_bool_(list_tests); + +// This flag controls whether Google Test emits a detailed XML report to a file +// in addition to its normal textual output. +GTEST_DECLARE_string_(output); + +// This flags control whether Google Test prints the elapsed time for each +// test. +GTEST_DECLARE_bool_(print_time); + +// This flag specifies the random number seed. +GTEST_DECLARE_int32_(random_seed); + +// This flag sets how many times the tests are repeated. The default value +// is 1. If the value is -1 the tests are repeating forever. +GTEST_DECLARE_int32_(repeat); + +// This flag controls whether Google Test includes Google Test internal +// stack frames in failure stack traces. +GTEST_DECLARE_bool_(show_internal_stack_frames); + +// When this flag is specified, tests' order is randomized on every iteration. +GTEST_DECLARE_bool_(shuffle); + +// This flag specifies the maximum number of stack frames to be +// printed in a failure message. +GTEST_DECLARE_int32_(stack_trace_depth); + +// When this flag is specified, a failed assertion will throw an +// exception if exceptions are enabled, or exit the program with a +// non-zero code otherwise. +GTEST_DECLARE_bool_(throw_on_failure); + +// When this flag is set with a "host:port" string, on supported +// platforms test results are streamed to the specified port on +// the specified host machine. +GTEST_DECLARE_string_(stream_result_to); + +// The upper limit for valid stack trace depths. +const int kMaxStackTraceDepth = 100; + +namespace internal { + +class AssertHelper; +class DefaultGlobalTestPartResultReporter; +class ExecDeathTest; +class NoExecDeathTest; +class FinalSuccessChecker; +class GTestFlagSaver; +class StreamingListenerTest; +class TestResultAccessor; +class TestEventListenersAccessor; +class TestEventRepeater; +class UnitTestRecordPropertyTestHelper; +class WindowsDeathTest; +class UnitTestImpl* GetUnitTestImpl(); +void ReportFailureInUnknownLocation(TestPartResult::Type result_type, + const std::string& message); + +} // namespace internal + +// The friend relationship of some of these classes is cyclic. +// If we don't forward declare them the compiler might confuse the classes +// in friendship clauses with same named classes on the scope. +class Test; +class TestCase; +class TestInfo; +class UnitTest; + +// A class for indicating whether an assertion was successful. When +// the assertion wasn't successful, the AssertionResult object +// remembers a non-empty message that describes how it failed. +// +// To create an instance of this class, use one of the factory functions +// (AssertionSuccess() and AssertionFailure()). +// +// This class is useful for two purposes: +// 1. Defining predicate functions to be used with Boolean test assertions +// EXPECT_TRUE/EXPECT_FALSE and their ASSERT_ counterparts +// 2. Defining predicate-format functions to be +// used with predicate assertions (ASSERT_PRED_FORMAT*, etc). +// +// For example, if you define IsEven predicate: +// +// testing::AssertionResult IsEven(int n) { +// if ((n % 2) == 0) +// return testing::AssertionSuccess(); +// else +// return testing::AssertionFailure() << n << " is odd"; +// } +// +// Then the failed expectation EXPECT_TRUE(IsEven(Fib(5))) +// will print the message +// +// Value of: IsEven(Fib(5)) +// Actual: false (5 is odd) +// Expected: true +// +// instead of a more opaque +// +// Value of: IsEven(Fib(5)) +// Actual: false +// Expected: true +// +// in case IsEven is a simple Boolean predicate. +// +// If you expect your predicate to be reused and want to support informative +// messages in EXPECT_FALSE and ASSERT_FALSE (negative assertions show up +// about half as often as positive ones in our tests), supply messages for +// both success and failure cases: +// +// testing::AssertionResult IsEven(int n) { +// if ((n % 2) == 0) +// return testing::AssertionSuccess() << n << " is even"; +// else +// return testing::AssertionFailure() << n << " is odd"; +// } +// +// Then a statement EXPECT_FALSE(IsEven(Fib(6))) will print +// +// Value of: IsEven(Fib(6)) +// Actual: true (8 is even) +// Expected: false +// +// NB: Predicates that support negative Boolean assertions have reduced +// performance in positive ones so be careful not to use them in tests +// that have lots (tens of thousands) of positive Boolean assertions. +// +// To use this class with EXPECT_PRED_FORMAT assertions such as: +// +// // Verifies that Foo() returns an even number. +// EXPECT_PRED_FORMAT1(IsEven, Foo()); +// +// you need to define: +// +// testing::AssertionResult IsEven(const char* expr, int n) { +// if ((n % 2) == 0) +// return testing::AssertionSuccess(); +// else +// return testing::AssertionFailure() +// << "Expected: " << expr << " is even\n Actual: it's " << n; +// } +// +// If Foo() returns 5, you will see the following message: +// +// Expected: Foo() is even +// Actual: it's 5 +// +class GTEST_API_ AssertionResult { + public: + // Copy constructor. + // Used in EXPECT_TRUE/FALSE(assertion_result). + AssertionResult(const AssertionResult& other); + + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4800 /* forcing value to bool */) + + // Used in the EXPECT_TRUE/FALSE(bool_expression). + // + // T must be contextually convertible to bool. + // + // The second parameter prevents this overload from being considered if + // the argument is implicitly convertible to AssertionResult. In that case + // we want AssertionResult's copy constructor to be used. + template + explicit AssertionResult( + const T& success, + typename internal::EnableIf< + !internal::ImplicitlyConvertible::value>::type* + /*enabler*/ = NULL) + : success_(success) {} + + GTEST_DISABLE_MSC_WARNINGS_POP_() + + // Assignment operator. + AssertionResult& operator=(AssertionResult other) { + swap(other); + return *this; + } + + // Returns true iff the assertion succeeded. + operator bool() const { return success_; } // NOLINT + + // Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE. + AssertionResult operator!() const; + + // Returns the text streamed into this AssertionResult. Test assertions + // use it when they fail (i.e., the predicate's outcome doesn't match the + // assertion's expectation). When nothing has been streamed into the + // object, returns an empty string. + const char* message() const { + return message_.get() != NULL ? message_->c_str() : ""; + } + // TODO(vladl@google.com): Remove this after making sure no clients use it. + // Deprecated; please use message() instead. + const char* failure_message() const { return message(); } + + // Streams a custom failure message into this object. + template AssertionResult& operator<<(const T& value) { + AppendMessage(Message() << value); + return *this; + } + + // Allows streaming basic output manipulators such as endl or flush into + // this object. + AssertionResult& operator<<( + ::std::ostream& (*basic_manipulator)(::std::ostream& stream)) { + AppendMessage(Message() << basic_manipulator); + return *this; + } + + private: + // Appends the contents of message to message_. + void AppendMessage(const Message& a_message) { + if (message_.get() == NULL) + message_.reset(new ::std::string); + message_->append(a_message.GetString().c_str()); + } + + // Swap the contents of this AssertionResult with other. + void swap(AssertionResult& other); + + // Stores result of the assertion predicate. + bool success_; + // Stores the message describing the condition in case the expectation + // construct is not satisfied with the predicate's outcome. + // Referenced via a pointer to avoid taking too much stack frame space + // with test assertions. + internal::scoped_ptr< ::std::string> message_; +}; + +// Makes a successful assertion result. +GTEST_API_ AssertionResult AssertionSuccess(); + +// Makes a failed assertion result. +GTEST_API_ AssertionResult AssertionFailure(); + +// Makes a failed assertion result with the given failure message. +// Deprecated; use AssertionFailure() << msg. +GTEST_API_ AssertionResult AssertionFailure(const Message& msg); + +// The abstract class that all tests inherit from. +// +// In Google Test, a unit test program contains one or many TestCases, and +// each TestCase contains one or many Tests. +// +// When you define a test using the TEST macro, you don't need to +// explicitly derive from Test - the TEST macro automatically does +// this for you. +// +// The only time you derive from Test is when defining a test fixture +// to be used a TEST_F. For example: +// +// class FooTest : public testing::Test { +// protected: +// void SetUp() override { ... } +// void TearDown() override { ... } +// ... +// }; +// +// TEST_F(FooTest, Bar) { ... } +// TEST_F(FooTest, Baz) { ... } +// +// Test is not copyable. +class GTEST_API_ Test { + public: + friend class TestInfo; + + // Defines types for pointers to functions that set up and tear down + // a test case. + typedef internal::SetUpTestCaseFunc SetUpTestCaseFunc; + typedef internal::TearDownTestCaseFunc TearDownTestCaseFunc; + + // The d'tor is virtual as we intend to inherit from Test. + virtual ~Test(); + + // Sets up the stuff shared by all tests in this test case. + // + // Google Test will call Foo::SetUpTestCase() before running the first + // test in test case Foo. Hence a sub-class can define its own + // SetUpTestCase() method to shadow the one defined in the super + // class. + static void SetUpTestCase() {} + + // Tears down the stuff shared by all tests in this test case. + // + // Google Test will call Foo::TearDownTestCase() after running the last + // test in test case Foo. Hence a sub-class can define its own + // TearDownTestCase() method to shadow the one defined in the super + // class. + static void TearDownTestCase() {} + + // Returns true iff the current test has a fatal failure. + static bool HasFatalFailure(); + + // Returns true iff the current test has a non-fatal failure. + static bool HasNonfatalFailure(); + + // Returns true iff the current test has a (either fatal or + // non-fatal) failure. + static bool HasFailure() { return HasFatalFailure() || HasNonfatalFailure(); } + + // Logs a property for the current test, test case, or for the entire + // invocation of the test program when used outside of the context of a + // test case. Only the last value for a given key is remembered. These + // are public static so they can be called from utility functions that are + // not members of the test fixture. Calls to RecordProperty made during + // lifespan of the test (from the moment its constructor starts to the + // moment its destructor finishes) will be output in XML as attributes of + // the element. Properties recorded from fixture's + // SetUpTestCase or TearDownTestCase are logged as attributes of the + // corresponding element. Calls to RecordProperty made in the + // global context (before or after invocation of RUN_ALL_TESTS and from + // SetUp/TearDown method of Environment objects registered with Google + // Test) will be output as attributes of the element. + static void RecordProperty(const std::string& key, const std::string& value); + static void RecordProperty(const std::string& key, int value); + + protected: + // Creates a Test object. + Test(); + + // Sets up the test fixture. + virtual void SetUp(); + + // Tears down the test fixture. + virtual void TearDown(); + + private: + // Returns true iff the current test has the same fixture class as + // the first test in the current test case. + static bool HasSameFixtureClass(); + + // Runs the test after the test fixture has been set up. + // + // A sub-class must implement this to define the test logic. + // + // DO NOT OVERRIDE THIS FUNCTION DIRECTLY IN A USER PROGRAM. + // Instead, use the TEST or TEST_F macro. + virtual void TestBody() = 0; + + // Sets up, executes, and tears down the test. + void Run(); + + // Deletes self. We deliberately pick an unusual name for this + // internal method to avoid clashing with names used in user TESTs. + void DeleteSelf_() { delete this; } + + const internal::scoped_ptr< GTEST_FLAG_SAVER_ > gtest_flag_saver_; + + // Often a user misspells SetUp() as Setup() and spends a long time + // wondering why it is never called by Google Test. The declaration of + // the following method is solely for catching such an error at + // compile time: + // + // - The return type is deliberately chosen to be not void, so it + // will be a conflict if void Setup() is declared in the user's + // test fixture. + // + // - This method is private, so it will be another compiler error + // if the method is called from the user's test fixture. + // + // DO NOT OVERRIDE THIS FUNCTION. + // + // If you see an error about overriding the following function or + // about it being private, you have mis-spelled SetUp() as Setup(). + struct Setup_should_be_spelled_SetUp {}; + virtual Setup_should_be_spelled_SetUp* Setup() { return NULL; } + + // We disallow copying Tests. + GTEST_DISALLOW_COPY_AND_ASSIGN_(Test); +}; + +typedef internal::TimeInMillis TimeInMillis; + +// A copyable object representing a user specified test property which can be +// output as a key/value string pair. +// +// Don't inherit from TestProperty as its destructor is not virtual. +class TestProperty { + public: + // C'tor. TestProperty does NOT have a default constructor. + // Always use this constructor (with parameters) to create a + // TestProperty object. + TestProperty(const std::string& a_key, const std::string& a_value) : + key_(a_key), value_(a_value) { + } + + // Gets the user supplied key. + const char* key() const { + return key_.c_str(); + } + + // Gets the user supplied value. + const char* value() const { + return value_.c_str(); + } + + // Sets a new value, overriding the one supplied in the constructor. + void SetValue(const std::string& new_value) { + value_ = new_value; + } + + private: + // The key supplied by the user. + std::string key_; + // The value supplied by the user. + std::string value_; +}; + +// The result of a single Test. This includes a list of +// TestPartResults, a list of TestProperties, a count of how many +// death tests there are in the Test, and how much time it took to run +// the Test. +// +// TestResult is not copyable. +class GTEST_API_ TestResult { + public: + // Creates an empty TestResult. + TestResult(); + + // D'tor. Do not inherit from TestResult. + ~TestResult(); + + // Gets the number of all test parts. This is the sum of the number + // of successful test parts and the number of failed test parts. + int total_part_count() const; + + // Returns the number of the test properties. + int test_property_count() const; + + // Returns true iff the test passed (i.e. no test part failed). + bool Passed() const { return !Failed(); } + + // Returns true iff the test failed. + bool Failed() const; + + // Returns true iff the test fatally failed. + bool HasFatalFailure() const; + + // Returns true iff the test has a non-fatal failure. + bool HasNonfatalFailure() const; + + // Returns the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const { return elapsed_time_; } + + // Returns the i-th test part result among all the results. i can range + // from 0 to test_property_count() - 1. If i is not in that range, aborts + // the program. + const TestPartResult& GetTestPartResult(int i) const; + + // Returns the i-th test property. i can range from 0 to + // test_property_count() - 1. If i is not in that range, aborts the + // program. + const TestProperty& GetTestProperty(int i) const; + + private: + friend class TestInfo; + friend class TestCase; + friend class UnitTest; + friend class internal::DefaultGlobalTestPartResultReporter; + friend class internal::ExecDeathTest; + friend class internal::TestResultAccessor; + friend class internal::UnitTestImpl; + friend class internal::WindowsDeathTest; + + // Gets the vector of TestPartResults. + const std::vector& test_part_results() const { + return test_part_results_; + } + + // Gets the vector of TestProperties. + const std::vector& test_properties() const { + return test_properties_; + } + + // Sets the elapsed time. + void set_elapsed_time(TimeInMillis elapsed) { elapsed_time_ = elapsed; } + + // Adds a test property to the list. The property is validated and may add + // a non-fatal failure if invalid (e.g., if it conflicts with reserved + // key names). If a property is already recorded for the same key, the + // value will be updated, rather than storing multiple values for the same + // key. xml_element specifies the element for which the property is being + // recorded and is used for validation. + void RecordProperty(const std::string& xml_element, + const TestProperty& test_property); + + // Adds a failure if the key is a reserved attribute of Google Test + // testcase tags. Returns true if the property is valid. + // TODO(russr): Validate attribute names are legal and human readable. + static bool ValidateTestProperty(const std::string& xml_element, + const TestProperty& test_property); + + // Adds a test part result to the list. + void AddTestPartResult(const TestPartResult& test_part_result); + + // Returns the death test count. + int death_test_count() const { return death_test_count_; } + + // Increments the death test count, returning the new count. + int increment_death_test_count() { return ++death_test_count_; } + + // Clears the test part results. + void ClearTestPartResults(); + + // Clears the object. + void Clear(); + + // Protects mutable state of the property vector and of owned + // properties, whose values may be updated. + internal::Mutex test_properites_mutex_; + + // The vector of TestPartResults + std::vector test_part_results_; + // The vector of TestProperties + std::vector test_properties_; + // Running count of death tests. + int death_test_count_; + // The elapsed time, in milliseconds. + TimeInMillis elapsed_time_; + + // We disallow copying TestResult. + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestResult); +}; // class TestResult + +// A TestInfo object stores the following information about a test: +// +// Test case name +// Test name +// Whether the test should be run +// A function pointer that creates the test object when invoked +// Test result +// +// The constructor of TestInfo registers itself with the UnitTest +// singleton such that the RUN_ALL_TESTS() macro knows which tests to +// run. +class GTEST_API_ TestInfo { + public: + // Destructs a TestInfo object. This function is not virtual, so + // don't inherit from TestInfo. + ~TestInfo(); + + // Returns the test case name. + const char* test_case_name() const { return test_case_name_.c_str(); } + + // Returns the test name. + const char* name() const { return name_.c_str(); } + + // Returns the name of the parameter type, or NULL if this is not a typed + // or a type-parameterized test. + const char* type_param() const { + if (type_param_.get() != NULL) + return type_param_->c_str(); + return NULL; + } + + // Returns the text representation of the value parameter, or NULL if this + // is not a value-parameterized test. + const char* value_param() const { + if (value_param_.get() != NULL) + return value_param_->c_str(); + return NULL; + } + + // Returns the file name where this test is defined. + const char* file() const { return location_.file.c_str(); } + + // Returns the line where this test is defined. + int line() const { return location_.line; } + + // Returns true if this test should run, that is if the test is not + // disabled (or it is disabled but the also_run_disabled_tests flag has + // been specified) and its full name matches the user-specified filter. + // + // Google Test allows the user to filter the tests by their full names. + // The full name of a test Bar in test case Foo is defined as + // "Foo.Bar". Only the tests that match the filter will run. + // + // A filter is a colon-separated list of glob (not regex) patterns, + // optionally followed by a '-' and a colon-separated list of + // negative patterns (tests to exclude). A test is run if it + // matches one of the positive patterns and does not match any of + // the negative patterns. + // + // For example, *A*:Foo.* is a filter that matches any string that + // contains the character 'A' or starts with "Foo.". + bool should_run() const { return should_run_; } + + // Returns true iff this test will appear in the XML report. + bool is_reportable() const { + // For now, the XML report includes all tests matching the filter. + // In the future, we may trim tests that are excluded because of + // sharding. + return matches_filter_; + } + + // Returns the result of the test. + const TestResult* result() const { return &result_; } + + private: +#if GTEST_HAS_DEATH_TEST + friend class internal::DefaultDeathTestFactory; +#endif // GTEST_HAS_DEATH_TEST + friend class Test; + friend class TestCase; + friend class internal::UnitTestImpl; + friend class internal::StreamingListenerTest; + friend TestInfo* internal::MakeAndRegisterTestInfo( + const char* test_case_name, + const char* name, + const char* type_param, + const char* value_param, + internal::CodeLocation code_location, + internal::TypeId fixture_class_id, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc, + internal::TestFactoryBase* factory); + + // Constructs a TestInfo object. The newly constructed instance assumes + // ownership of the factory object. + TestInfo(const std::string& test_case_name, + const std::string& name, + const char* a_type_param, // NULL if not a type-parameterized test + const char* a_value_param, // NULL if not a value-parameterized test + internal::CodeLocation a_code_location, + internal::TypeId fixture_class_id, + internal::TestFactoryBase* factory); + + // Increments the number of death tests encountered in this test so + // far. + int increment_death_test_count() { + return result_.increment_death_test_count(); + } + + // Creates the test object, runs it, records its result, and then + // deletes it. + void Run(); + + static void ClearTestResult(TestInfo* test_info) { + test_info->result_.Clear(); + } + + // These fields are immutable properties of the test. + const std::string test_case_name_; // Test case name + const std::string name_; // Test name + // Name of the parameter type, or NULL if this is not a typed or a + // type-parameterized test. + const internal::scoped_ptr type_param_; + // Text representation of the value parameter, or NULL if this is not a + // value-parameterized test. + const internal::scoped_ptr value_param_; + internal::CodeLocation location_; + const internal::TypeId fixture_class_id_; // ID of the test fixture class + bool should_run_; // True iff this test should run + bool is_disabled_; // True iff this test is disabled + bool matches_filter_; // True if this test matches the + // user-specified filter. + internal::TestFactoryBase* const factory_; // The factory that creates + // the test object + + // This field is mutable and needs to be reset before running the + // test for the second time. + TestResult result_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestInfo); +}; + +// A test case, which consists of a vector of TestInfos. +// +// TestCase is not copyable. +class GTEST_API_ TestCase { + public: + // Creates a TestCase with the given name. + // + // TestCase does NOT have a default constructor. Always use this + // constructor to create a TestCase object. + // + // Arguments: + // + // name: name of the test case + // a_type_param: the name of the test's type parameter, or NULL if + // this is not a type-parameterized test. + // set_up_tc: pointer to the function that sets up the test case + // tear_down_tc: pointer to the function that tears down the test case + TestCase(const char* name, const char* a_type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc); + + // Destructor of TestCase. + virtual ~TestCase(); + + // Gets the name of the TestCase. + const char* name() const { return name_.c_str(); } + + // Returns the name of the parameter type, or NULL if this is not a + // type-parameterized test case. + const char* type_param() const { + if (type_param_.get() != NULL) + return type_param_->c_str(); + return NULL; + } + + // Returns true if any test in this test case should run. + bool should_run() const { return should_run_; } + + // Gets the number of successful tests in this test case. + int successful_test_count() const; + + // Gets the number of failed tests in this test case. + int failed_test_count() const; + + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + + // Gets the number of disabled tests in this test case. + int disabled_test_count() const; + + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + + // Get the number of tests in this test case that should run. + int test_to_run_count() const; + + // Gets the number of all tests in this test case. + int total_test_count() const; + + // Returns true iff the test case passed. + bool Passed() const { return !Failed(); } + + // Returns true iff the test case failed. + bool Failed() const { return failed_test_count() > 0; } + + // Returns the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const { return elapsed_time_; } + + // Returns the i-th test among all the tests. i can range from 0 to + // total_test_count() - 1. If i is not in that range, returns NULL. + const TestInfo* GetTestInfo(int i) const; + + // Returns the TestResult that holds test properties recorded during + // execution of SetUpTestCase and TearDownTestCase. + const TestResult& ad_hoc_test_result() const { return ad_hoc_test_result_; } + + private: + friend class Test; + friend class internal::UnitTestImpl; + + // Gets the (mutable) vector of TestInfos in this TestCase. + std::vector& test_info_list() { return test_info_list_; } + + // Gets the (immutable) vector of TestInfos in this TestCase. + const std::vector& test_info_list() const { + return test_info_list_; + } + + // Returns the i-th test among all the tests. i can range from 0 to + // total_test_count() - 1. If i is not in that range, returns NULL. + TestInfo* GetMutableTestInfo(int i); + + // Sets the should_run member. + void set_should_run(bool should) { should_run_ = should; } + + // Adds a TestInfo to this test case. Will delete the TestInfo upon + // destruction of the TestCase object. + void AddTestInfo(TestInfo * test_info); + + // Clears the results of all tests in this test case. + void ClearResult(); + + // Clears the results of all tests in the given test case. + static void ClearTestCaseResult(TestCase* test_case) { + test_case->ClearResult(); + } + + // Runs every test in this TestCase. + void Run(); + + // Runs SetUpTestCase() for this TestCase. This wrapper is needed + // for catching exceptions thrown from SetUpTestCase(). + void RunSetUpTestCase() { (*set_up_tc_)(); } + + // Runs TearDownTestCase() for this TestCase. This wrapper is + // needed for catching exceptions thrown from TearDownTestCase(). + void RunTearDownTestCase() { (*tear_down_tc_)(); } + + // Returns true iff test passed. + static bool TestPassed(const TestInfo* test_info) { + return test_info->should_run() && test_info->result()->Passed(); + } + + // Returns true iff test failed. + static bool TestFailed(const TestInfo* test_info) { + return test_info->should_run() && test_info->result()->Failed(); + } + + // Returns true iff the test is disabled and will be reported in the XML + // report. + static bool TestReportableDisabled(const TestInfo* test_info) { + return test_info->is_reportable() && test_info->is_disabled_; + } + + // Returns true iff test is disabled. + static bool TestDisabled(const TestInfo* test_info) { + return test_info->is_disabled_; + } + + // Returns true iff this test will appear in the XML report. + static bool TestReportable(const TestInfo* test_info) { + return test_info->is_reportable(); + } + + // Returns true if the given test should run. + static bool ShouldRunTest(const TestInfo* test_info) { + return test_info->should_run(); + } + + // Shuffles the tests in this test case. + void ShuffleTests(internal::Random* random); + + // Restores the test order to before the first shuffle. + void UnshuffleTests(); + + // Name of the test case. + std::string name_; + // Name of the parameter type, or NULL if this is not a typed or a + // type-parameterized test. + const internal::scoped_ptr type_param_; + // The vector of TestInfos in their original order. It owns the + // elements in the vector. + std::vector test_info_list_; + // Provides a level of indirection for the test list to allow easy + // shuffling and restoring the test order. The i-th element in this + // vector is the index of the i-th test in the shuffled test list. + std::vector test_indices_; + // Pointer to the function that sets up the test case. + Test::SetUpTestCaseFunc set_up_tc_; + // Pointer to the function that tears down the test case. + Test::TearDownTestCaseFunc tear_down_tc_; + // True iff any test in this test case should run. + bool should_run_; + // Elapsed time, in milliseconds. + TimeInMillis elapsed_time_; + // Holds test properties recorded during execution of SetUpTestCase and + // TearDownTestCase. + TestResult ad_hoc_test_result_; + + // We disallow copying TestCases. + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestCase); +}; + +// An Environment object is capable of setting up and tearing down an +// environment. You should subclass this to define your own +// environment(s). +// +// An Environment object does the set-up and tear-down in virtual +// methods SetUp() and TearDown() instead of the constructor and the +// destructor, as: +// +// 1. You cannot safely throw from a destructor. This is a problem +// as in some cases Google Test is used where exceptions are enabled, and +// we may want to implement ASSERT_* using exceptions where they are +// available. +// 2. You cannot use ASSERT_* directly in a constructor or +// destructor. +class Environment { + public: + // The d'tor is virtual as we need to subclass Environment. + virtual ~Environment() {} + + // Override this to define how to set up the environment. + virtual void SetUp() {} + + // Override this to define how to tear down the environment. + virtual void TearDown() {} + private: + // If you see an error about overriding the following function or + // about it being private, you have mis-spelled SetUp() as Setup(). + struct Setup_should_be_spelled_SetUp {}; + virtual Setup_should_be_spelled_SetUp* Setup() { return NULL; } +}; + +// The interface for tracing execution of tests. The methods are organized in +// the order the corresponding events are fired. +class TestEventListener { + public: + virtual ~TestEventListener() {} + + // Fired before any test activity starts. + virtual void OnTestProgramStart(const UnitTest& unit_test) = 0; + + // Fired before each iteration of tests starts. There may be more than + // one iteration if GTEST_FLAG(repeat) is set. iteration is the iteration + // index, starting from 0. + virtual void OnTestIterationStart(const UnitTest& unit_test, + int iteration) = 0; + + // Fired before environment set-up for each iteration of tests starts. + virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test) = 0; + + // Fired after environment set-up for each iteration of tests ends. + virtual void OnEnvironmentsSetUpEnd(const UnitTest& unit_test) = 0; + + // Fired before the test case starts. + virtual void OnTestCaseStart(const TestCase& test_case) = 0; + + // Fired before the test starts. + virtual void OnTestStart(const TestInfo& test_info) = 0; + + // Fired after a failed assertion or a SUCCEED() invocation. + virtual void OnTestPartResult(const TestPartResult& test_part_result) = 0; + + // Fired after the test ends. + virtual void OnTestEnd(const TestInfo& test_info) = 0; + + // Fired after the test case ends. + virtual void OnTestCaseEnd(const TestCase& test_case) = 0; + + // Fired before environment tear-down for each iteration of tests starts. + virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test) = 0; + + // Fired after environment tear-down for each iteration of tests ends. + virtual void OnEnvironmentsTearDownEnd(const UnitTest& unit_test) = 0; + + // Fired after each iteration of tests finishes. + virtual void OnTestIterationEnd(const UnitTest& unit_test, + int iteration) = 0; + + // Fired after all test activities have ended. + virtual void OnTestProgramEnd(const UnitTest& unit_test) = 0; +}; + +// The convenience class for users who need to override just one or two +// methods and are not concerned that a possible change to a signature of +// the methods they override will not be caught during the build. For +// comments about each method please see the definition of TestEventListener +// above. +class EmptyTestEventListener : public TestEventListener { + public: + virtual void OnTestProgramStart(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationStart(const UnitTest& /*unit_test*/, + int /*iteration*/) {} + virtual void OnEnvironmentsSetUpStart(const UnitTest& /*unit_test*/) {} + virtual void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestCaseStart(const TestCase& /*test_case*/) {} + virtual void OnTestStart(const TestInfo& /*test_info*/) {} + virtual void OnTestPartResult(const TestPartResult& /*test_part_result*/) {} + virtual void OnTestEnd(const TestInfo& /*test_info*/) {} + virtual void OnTestCaseEnd(const TestCase& /*test_case*/) {} + virtual void OnEnvironmentsTearDownStart(const UnitTest& /*unit_test*/) {} + virtual void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationEnd(const UnitTest& /*unit_test*/, + int /*iteration*/) {} + virtual void OnTestProgramEnd(const UnitTest& /*unit_test*/) {} +}; + +// TestEventListeners lets users add listeners to track events in Google Test. +class GTEST_API_ TestEventListeners { + public: + TestEventListeners(); + ~TestEventListeners(); + + // Appends an event listener to the end of the list. Google Test assumes + // the ownership of the listener (i.e. it will delete the listener when + // the test program finishes). + void Append(TestEventListener* listener); + + // Removes the given event listener from the list and returns it. It then + // becomes the caller's responsibility to delete the listener. Returns + // NULL if the listener is not found in the list. + TestEventListener* Release(TestEventListener* listener); + + // Returns the standard listener responsible for the default console + // output. Can be removed from the listeners list to shut down default + // console output. Note that removing this object from the listener list + // with Release transfers its ownership to the caller and makes this + // function return NULL the next time. + TestEventListener* default_result_printer() const { + return default_result_printer_; + } + + // Returns the standard listener responsible for the default XML output + // controlled by the --gtest_output=xml flag. Can be removed from the + // listeners list by users who want to shut down the default XML output + // controlled by this flag and substitute it with custom one. Note that + // removing this object from the listener list with Release transfers its + // ownership to the caller and makes this function return NULL the next + // time. + TestEventListener* default_xml_generator() const { + return default_xml_generator_; + } + + private: + friend class TestCase; + friend class TestInfo; + friend class internal::DefaultGlobalTestPartResultReporter; + friend class internal::NoExecDeathTest; + friend class internal::TestEventListenersAccessor; + friend class internal::UnitTestImpl; + + // Returns repeater that broadcasts the TestEventListener events to all + // subscribers. + TestEventListener* repeater(); + + // Sets the default_result_printer attribute to the provided listener. + // The listener is also added to the listener list and previous + // default_result_printer is removed from it and deleted. The listener can + // also be NULL in which case it will not be added to the list. Does + // nothing if the previous and the current listener objects are the same. + void SetDefaultResultPrinter(TestEventListener* listener); + + // Sets the default_xml_generator attribute to the provided listener. The + // listener is also added to the listener list and previous + // default_xml_generator is removed from it and deleted. The listener can + // also be NULL in which case it will not be added to the list. Does + // nothing if the previous and the current listener objects are the same. + void SetDefaultXmlGenerator(TestEventListener* listener); + + // Controls whether events will be forwarded by the repeater to the + // listeners in the list. + bool EventForwardingEnabled() const; + void SuppressEventForwarding(); + + // The actual list of listeners. + internal::TestEventRepeater* repeater_; + // Listener responsible for the standard result output. + TestEventListener* default_result_printer_; + // Listener responsible for the creation of the XML output file. + TestEventListener* default_xml_generator_; + + // We disallow copying TestEventListeners. + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestEventListeners); +}; + +// A UnitTest consists of a vector of TestCases. +// +// This is a singleton class. The only instance of UnitTest is +// created when UnitTest::GetInstance() is first called. This +// instance is never deleted. +// +// UnitTest is not copyable. +// +// This class is thread-safe as long as the methods are called +// according to their specification. +class GTEST_API_ UnitTest { + public: + // Gets the singleton UnitTest object. The first time this method + // is called, a UnitTest object is constructed and returned. + // Consecutive calls will return the same object. + static UnitTest* GetInstance(); + + // Runs all tests in this UnitTest object and prints the result. + // Returns 0 if successful, or 1 otherwise. + // + // This method can only be called from the main thread. + // + // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + int Run() GTEST_MUST_USE_RESULT_; + + // Returns the working directory when the first TEST() or TEST_F() + // was executed. The UnitTest object owns the string. + const char* original_working_dir() const; + + // Returns the TestCase object for the test that's currently running, + // or NULL if no test is running. + const TestCase* current_test_case() const + GTEST_LOCK_EXCLUDED_(mutex_); + + // Returns the TestInfo object for the test that's currently running, + // or NULL if no test is running. + const TestInfo* current_test_info() const + GTEST_LOCK_EXCLUDED_(mutex_); + + // Returns the random seed used at the start of the current test run. + int random_seed() const; + +#if GTEST_HAS_PARAM_TEST + // Returns the ParameterizedTestCaseRegistry object used to keep track of + // value-parameterized tests and instantiate and register them. + // + // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + internal::ParameterizedTestCaseRegistry& parameterized_test_registry() + GTEST_LOCK_EXCLUDED_(mutex_); +#endif // GTEST_HAS_PARAM_TEST + + // Gets the number of successful test cases. + int successful_test_case_count() const; + + // Gets the number of failed test cases. + int failed_test_case_count() const; + + // Gets the number of all test cases. + int total_test_case_count() const; + + // Gets the number of all test cases that contain at least one test + // that should run. + int test_case_to_run_count() const; + + // Gets the number of successful tests. + int successful_test_count() const; + + // Gets the number of failed tests. + int failed_test_count() const; + + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + + // Gets the number of disabled tests. + int disabled_test_count() const; + + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + + // Gets the number of all tests. + int total_test_count() const; + + // Gets the number of tests that should run. + int test_to_run_count() const; + + // Gets the time of the test program start, in ms from the start of the + // UNIX epoch. + TimeInMillis start_timestamp() const; + + // Gets the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const; + + // Returns true iff the unit test passed (i.e. all test cases passed). + bool Passed() const; + + // Returns true iff the unit test failed (i.e. some test case failed + // or something outside of all tests failed). + bool Failed() const; + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + const TestCase* GetTestCase(int i) const; + + // Returns the TestResult containing information on test failures and + // properties logged outside of individual test cases. + const TestResult& ad_hoc_test_result() const; + + // Returns the list of event listeners that can be used to track events + // inside Google Test. + TestEventListeners& listeners(); + + private: + // Registers and returns a global test environment. When a test + // program is run, all global test environments will be set-up in + // the order they were registered. After all tests in the program + // have finished, all global test environments will be torn-down in + // the *reverse* order they were registered. + // + // The UnitTest object takes ownership of the given environment. + // + // This method can only be called from the main thread. + Environment* AddEnvironment(Environment* env); + + // Adds a TestPartResult to the current TestResult object. All + // Google Test assertion macros (e.g. ASSERT_TRUE, EXPECT_EQ, etc) + // eventually call this to report their results. The user code + // should use the assertion macros instead of calling this directly. + void AddTestPartResult(TestPartResult::Type result_type, + const char* file_name, + int line_number, + const std::string& message, + const std::string& os_stack_trace) + GTEST_LOCK_EXCLUDED_(mutex_); + + // Adds a TestProperty to the current TestResult object when invoked from + // inside a test, to current TestCase's ad_hoc_test_result_ when invoked + // from SetUpTestCase or TearDownTestCase, or to the global property set + // when invoked elsewhere. If the result already contains a property with + // the same key, the value will be updated. + void RecordProperty(const std::string& key, const std::string& value); + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + TestCase* GetMutableTestCase(int i); + + // Accessors for the implementation object. + internal::UnitTestImpl* impl() { return impl_; } + const internal::UnitTestImpl* impl() const { return impl_; } + + // These classes and funcions are friends as they need to access private + // members of UnitTest. + friend class Test; + friend class internal::AssertHelper; + friend class internal::ScopedTrace; + friend class internal::StreamingListenerTest; + friend class internal::UnitTestRecordPropertyTestHelper; + friend Environment* AddGlobalTestEnvironment(Environment* env); + friend internal::UnitTestImpl* internal::GetUnitTestImpl(); + friend void internal::ReportFailureInUnknownLocation( + TestPartResult::Type result_type, + const std::string& message); + + // Creates an empty UnitTest. + UnitTest(); + + // D'tor + virtual ~UnitTest(); + + // Pushes a trace defined by SCOPED_TRACE() on to the per-thread + // Google Test trace stack. + void PushGTestTrace(const internal::TraceInfo& trace) + GTEST_LOCK_EXCLUDED_(mutex_); + + // Pops a trace from the per-thread Google Test trace stack. + void PopGTestTrace() + GTEST_LOCK_EXCLUDED_(mutex_); + + // Protects mutable state in *impl_. This is mutable as some const + // methods need to lock it too. + mutable internal::Mutex mutex_; + + // Opaque implementation object. This field is never changed once + // the object is constructed. We don't mark it as const here, as + // doing so will cause a warning in the constructor of UnitTest. + // Mutable state in *impl_ is protected by mutex_. + internal::UnitTestImpl* impl_; + + // We disallow copying UnitTest. + GTEST_DISALLOW_COPY_AND_ASSIGN_(UnitTest); +}; + +// A convenient wrapper for adding an environment for the test +// program. +// +// You should call this before RUN_ALL_TESTS() is called, probably in +// main(). If you use gtest_main, you need to call this before main() +// starts for it to take effect. For example, you can define a global +// variable like this: +// +// testing::Environment* const foo_env = +// testing::AddGlobalTestEnvironment(new FooEnvironment); +// +// However, we strongly recommend you to write your own main() and +// call AddGlobalTestEnvironment() there, as relying on initialization +// of global variables makes the code harder to read and may cause +// problems when you register multiple environments from different +// translation units and the environments have dependencies among them +// (remember that the compiler doesn't guarantee the order in which +// global variables from different translation units are initialized). +inline Environment* AddGlobalTestEnvironment(Environment* env) { + return UnitTest::GetInstance()->AddEnvironment(env); +} + +// Initializes Google Test. This must be called before calling +// RUN_ALL_TESTS(). In particular, it parses a command line for the +// flags that Google Test recognizes. Whenever a Google Test flag is +// seen, it is removed from argv, and *argc is decremented. +// +// No value is returned. Instead, the Google Test flag variables are +// updated. +// +// Calling the function for the second time has no user-visible effect. +GTEST_API_ void InitGoogleTest(int* argc, char** argv); + +// This overloaded version can be used in Windows programs compiled in +// UNICODE mode. +GTEST_API_ void InitGoogleTest(int* argc, wchar_t** argv); + +namespace internal { + +// Separate the error generating code from the code path to reduce the stack +// frame size of CmpHelperEQ. This helps reduce the overhead of some sanitizers +// when calling EXPECT_* in a tight loop. +template +AssertionResult CmpHelperEQFailure(const char* lhs_expression, + const char* rhs_expression, + const T1& lhs, const T2& rhs) { + return EqFailure(lhs_expression, + rhs_expression, + FormatForComparisonFailureMessage(lhs, rhs), + FormatForComparisonFailureMessage(rhs, lhs), + false); +} + +// The helper function for {ASSERT|EXPECT}_EQ. +template +AssertionResult CmpHelperEQ(const char* lhs_expression, + const char* rhs_expression, + const T1& lhs, + const T2& rhs) { +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4389 /* signed/unsigned mismatch */) + if (lhs == rhs) { + return AssertionSuccess(); + } +GTEST_DISABLE_MSC_WARNINGS_POP_() + + return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs); +} + +// With this overloaded version, we allow anonymous enums to be used +// in {ASSERT|EXPECT}_EQ when compiled with gcc 4, as anonymous enums +// can be implicitly cast to BiggestInt. +GTEST_API_ AssertionResult CmpHelperEQ(const char* lhs_expression, + const char* rhs_expression, + BiggestInt lhs, + BiggestInt rhs); + +// The helper class for {ASSERT|EXPECT}_EQ. The template argument +// lhs_is_null_literal is true iff the first argument to ASSERT_EQ() +// is a null pointer literal. The following default implementation is +// for lhs_is_null_literal being false. +template +class EqHelper { + public: + // This templatized version is for the general case. + template + static AssertionResult Compare(const char* lhs_expression, + const char* rhs_expression, + const T1& lhs, + const T2& rhs) { + return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs); + } + + // With this overloaded version, we allow anonymous enums to be used + // in {ASSERT|EXPECT}_EQ when compiled with gcc 4, as anonymous + // enums can be implicitly cast to BiggestInt. + // + // Even though its body looks the same as the above version, we + // cannot merge the two, as it will make anonymous enums unhappy. + static AssertionResult Compare(const char* lhs_expression, + const char* rhs_expression, + BiggestInt lhs, + BiggestInt rhs) { + return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs); + } +}; + +// This specialization is used when the first argument to ASSERT_EQ() +// is a null pointer literal, like NULL, false, or 0. +template <> +class EqHelper { + public: + // We define two overloaded versions of Compare(). The first + // version will be picked when the second argument to ASSERT_EQ() is + // NOT a pointer, e.g. ASSERT_EQ(0, AnIntFunction()) or + // EXPECT_EQ(false, a_bool). + template + static AssertionResult Compare( + const char* lhs_expression, + const char* rhs_expression, + const T1& lhs, + const T2& rhs, + // The following line prevents this overload from being considered if T2 + // is not a pointer type. We need this because ASSERT_EQ(NULL, my_ptr) + // expands to Compare("", "", NULL, my_ptr), which requires a conversion + // to match the Secret* in the other overload, which would otherwise make + // this template match better. + typename EnableIf::value>::type* = 0) { + return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs); + } + + // This version will be picked when the second argument to ASSERT_EQ() is a + // pointer, e.g. ASSERT_EQ(NULL, a_pointer). + template + static AssertionResult Compare( + const char* lhs_expression, + const char* rhs_expression, + // We used to have a second template parameter instead of Secret*. That + // template parameter would deduce to 'long', making this a better match + // than the first overload even without the first overload's EnableIf. + // Unfortunately, gcc with -Wconversion-null warns when "passing NULL to + // non-pointer argument" (even a deduced integral argument), so the old + // implementation caused warnings in user code. + Secret* /* lhs (NULL) */, + T* rhs) { + // We already know that 'lhs' is a null pointer. + return CmpHelperEQ(lhs_expression, rhs_expression, + static_cast(NULL), rhs); + } +}; + +// Separate the error generating code from the code path to reduce the stack +// frame size of CmpHelperOP. This helps reduce the overhead of some sanitizers +// when calling EXPECT_OP in a tight loop. +template +AssertionResult CmpHelperOpFailure(const char* expr1, const char* expr2, + const T1& val1, const T2& val2, + const char* op) { + return AssertionFailure() + << "Expected: (" << expr1 << ") " << op << " (" << expr2 + << "), actual: " << FormatForComparisonFailureMessage(val1, val2) + << " vs " << FormatForComparisonFailureMessage(val2, val1); +} + +// A macro for implementing the helper functions needed to implement +// ASSERT_?? and EXPECT_??. It is here just to avoid copy-and-paste +// of similar code. +// +// For each templatized helper function, we also define an overloaded +// version for BiggestInt in order to reduce code bloat and allow +// anonymous enums to be used with {ASSERT|EXPECT}_?? when compiled +// with gcc 4. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + +#define GTEST_IMPL_CMP_HELPER_(op_name, op)\ +template \ +AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ + const T1& val1, const T2& val2) {\ + if (val1 op val2) {\ + return AssertionSuccess();\ + } else {\ + return CmpHelperOpFailure(expr1, expr2, val1, val2, #op);\ + }\ +}\ +GTEST_API_ AssertionResult CmpHelper##op_name(\ + const char* expr1, const char* expr2, BiggestInt val1, BiggestInt val2) + +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + +// Implements the helper function for {ASSERT|EXPECT}_NE +GTEST_IMPL_CMP_HELPER_(NE, !=); +// Implements the helper function for {ASSERT|EXPECT}_LE +GTEST_IMPL_CMP_HELPER_(LE, <=); +// Implements the helper function for {ASSERT|EXPECT}_LT +GTEST_IMPL_CMP_HELPER_(LT, <); +// Implements the helper function for {ASSERT|EXPECT}_GE +GTEST_IMPL_CMP_HELPER_(GE, >=); +// Implements the helper function for {ASSERT|EXPECT}_GT +GTEST_IMPL_CMP_HELPER_(GT, >); + +#undef GTEST_IMPL_CMP_HELPER_ + +// The helper function for {ASSERT|EXPECT}_STREQ. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTREQ(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + +// The helper function for {ASSERT|EXPECT}_STRCASEEQ. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRCASEEQ(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + +// The helper function for {ASSERT|EXPECT}_STRNE. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + +// The helper function for {ASSERT|EXPECT}_STRCASENE. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRCASENE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + + +// Helper function for *_STREQ on wide strings. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTREQ(const char* s1_expression, + const char* s2_expression, + const wchar_t* s1, + const wchar_t* s2); + +// Helper function for *_STRNE on wide strings. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const wchar_t* s1, + const wchar_t* s2); + +} // namespace internal + +// IsSubstring() and IsNotSubstring() are intended to be used as the +// first argument to {EXPECT,ASSERT}_PRED_FORMAT2(), not by +// themselves. They check whether needle is a substring of haystack +// (NULL is considered a substring of itself only), and return an +// appropriate error message when they fail. +// +// The {needle,haystack}_expr arguments are the stringified +// expressions that generated the two real arguments. +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack); +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack); +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack); + +#if GTEST_HAS_STD_WSTRING +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack); +#endif // GTEST_HAS_STD_WSTRING + +namespace internal { + +// Helper template function for comparing floating-points. +// +// Template parameter: +// +// RawType: the raw floating-point type (either float or double) +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +template +AssertionResult CmpHelperFloatingPointEQ(const char* lhs_expression, + const char* rhs_expression, + RawType lhs_value, + RawType rhs_value) { + const FloatingPoint lhs(lhs_value), rhs(rhs_value); + + if (lhs.AlmostEquals(rhs)) { + return AssertionSuccess(); + } + + ::std::stringstream lhs_ss; + lhs_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << lhs_value; + + ::std::stringstream rhs_ss; + rhs_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << rhs_value; + + return EqFailure(lhs_expression, + rhs_expression, + StringStreamToString(&lhs_ss), + StringStreamToString(&rhs_ss), + false); +} + +// Helper function for implementing ASSERT_NEAR. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult DoubleNearPredFormat(const char* expr1, + const char* expr2, + const char* abs_error_expr, + double val1, + double val2, + double abs_error); + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// A class that enables one to stream messages to assertion macros +class GTEST_API_ AssertHelper { + public: + // Constructor. + AssertHelper(TestPartResult::Type type, + const char* file, + int line, + const char* message); + ~AssertHelper(); + + // Message assignment is a semantic trick to enable assertion + // streaming; see the GTEST_MESSAGE_ macro below. + void operator=(const Message& message) const; + + private: + // We put our data in a struct so that the size of the AssertHelper class can + // be as small as possible. This is important because gcc is incapable of + // re-using stack space even for temporary variables, so every EXPECT_EQ + // reserves stack space for another AssertHelper. + struct AssertHelperData { + AssertHelperData(TestPartResult::Type t, + const char* srcfile, + int line_num, + const char* msg) + : type(t), file(srcfile), line(line_num), message(msg) { } + + TestPartResult::Type const type; + const char* const file; + int const line; + std::string const message; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelperData); + }; + + AssertHelperData* const data_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelper); +}; + +} // namespace internal + +#if GTEST_HAS_PARAM_TEST +// The pure interface class that all value-parameterized tests inherit from. +// A value-parameterized class must inherit from both ::testing::Test and +// ::testing::WithParamInterface. In most cases that just means inheriting +// from ::testing::TestWithParam, but more complicated test hierarchies +// may need to inherit from Test and WithParamInterface at different levels. +// +// This interface has support for accessing the test parameter value via +// the GetParam() method. +// +// Use it with one of the parameter generator defining functions, like Range(), +// Values(), ValuesIn(), Bool(), and Combine(). +// +// class FooTest : public ::testing::TestWithParam { +// protected: +// FooTest() { +// // Can use GetParam() here. +// } +// virtual ~FooTest() { +// // Can use GetParam() here. +// } +// virtual void SetUp() { +// // Can use GetParam() here. +// } +// virtual void TearDown { +// // Can use GetParam() here. +// } +// }; +// TEST_P(FooTest, DoesBar) { +// // Can use GetParam() method here. +// Foo foo; +// ASSERT_TRUE(foo.DoesBar(GetParam())); +// } +// INSTANTIATE_TEST_CASE_P(OneToTenRange, FooTest, ::testing::Range(1, 10)); + +template +class WithParamInterface { + public: + typedef T ParamType; + virtual ~WithParamInterface() {} + + // The current parameter value. Is also available in the test fixture's + // constructor. This member function is non-static, even though it only + // references static data, to reduce the opportunity for incorrect uses + // like writing 'WithParamInterface::GetParam()' for a test that + // uses a fixture whose parameter type is int. + const ParamType& GetParam() const { + GTEST_CHECK_(parameter_ != NULL) + << "GetParam() can only be called inside a value-parameterized test " + << "-- did you intend to write TEST_P instead of TEST_F?"; + return *parameter_; + } + + private: + // Sets parameter value. The caller is responsible for making sure the value + // remains alive and unchanged throughout the current test. + static void SetParam(const ParamType* parameter) { + parameter_ = parameter; + } + + // Static value used for accessing parameter during a test lifetime. + static const ParamType* parameter_; + + // TestClass must be a subclass of WithParamInterface and Test. + template friend class internal::ParameterizedTestFactory; +}; + +template +const T* WithParamInterface::parameter_ = NULL; + +// Most value-parameterized classes can ignore the existence of +// WithParamInterface, and can just inherit from ::testing::TestWithParam. + +template +class TestWithParam : public Test, public WithParamInterface { +}; + +#endif // GTEST_HAS_PARAM_TEST + +// Macros for indicating success/failure in test code. + +// ADD_FAILURE unconditionally adds a failure to the current test. +// SUCCEED generates a success - it doesn't automatically make the +// current test successful, as a test is only successful when it has +// no failure. +// +// EXPECT_* verifies that a certain condition is satisfied. If not, +// it behaves like ADD_FAILURE. In particular: +// +// EXPECT_TRUE verifies that a Boolean condition is true. +// EXPECT_FALSE verifies that a Boolean condition is false. +// +// FAIL and ASSERT_* are similar to ADD_FAILURE and EXPECT_*, except +// that they will also abort the current function on failure. People +// usually want the fail-fast behavior of FAIL and ASSERT_*, but those +// writing data-driven tests often find themselves using ADD_FAILURE +// and EXPECT_* more. + +// Generates a nonfatal failure with a generic message. +#define ADD_FAILURE() GTEST_NONFATAL_FAILURE_("Failed") + +// Generates a nonfatal failure at the given source file location with +// a generic message. +#define ADD_FAILURE_AT(file, line) \ + GTEST_MESSAGE_AT_(file, line, "Failed", \ + ::testing::TestPartResult::kNonFatalFailure) + +// Generates a fatal failure with a generic message. +#define GTEST_FAIL() GTEST_FATAL_FAILURE_("Failed") + +// Define this macro to 1 to omit the definition of FAIL(), which is a +// generic name and clashes with some other libraries. +#if !GTEST_DONT_DEFINE_FAIL +# define FAIL() GTEST_FAIL() +#endif + +// Generates a success with a generic message. +#define GTEST_SUCCEED() GTEST_SUCCESS_("Succeeded") + +// Define this macro to 1 to omit the definition of SUCCEED(), which +// is a generic name and clashes with some other libraries. +#if !GTEST_DONT_DEFINE_SUCCEED +# define SUCCEED() GTEST_SUCCEED() +#endif + +// Macros for testing exceptions. +// +// * {ASSERT|EXPECT}_THROW(statement, expected_exception): +// Tests that the statement throws the expected exception. +// * {ASSERT|EXPECT}_NO_THROW(statement): +// Tests that the statement doesn't throw any exception. +// * {ASSERT|EXPECT}_ANY_THROW(statement): +// Tests that the statement throws an exception. + +#define EXPECT_THROW(statement, expected_exception) \ + GTEST_TEST_THROW_(statement, expected_exception, GTEST_NONFATAL_FAILURE_) +#define EXPECT_NO_THROW(statement) \ + GTEST_TEST_NO_THROW_(statement, GTEST_NONFATAL_FAILURE_) +#define EXPECT_ANY_THROW(statement) \ + GTEST_TEST_ANY_THROW_(statement, GTEST_NONFATAL_FAILURE_) +#define ASSERT_THROW(statement, expected_exception) \ + GTEST_TEST_THROW_(statement, expected_exception, GTEST_FATAL_FAILURE_) +#define ASSERT_NO_THROW(statement) \ + GTEST_TEST_NO_THROW_(statement, GTEST_FATAL_FAILURE_) +#define ASSERT_ANY_THROW(statement) \ + GTEST_TEST_ANY_THROW_(statement, GTEST_FATAL_FAILURE_) + +// Boolean assertions. Condition can be either a Boolean expression or an +// AssertionResult. For more information on how to use AssertionResult with +// these macros see comments on that class. +#define EXPECT_TRUE(condition) \ + GTEST_TEST_BOOLEAN_((condition), #condition, false, true, \ + GTEST_NONFATAL_FAILURE_) +#define EXPECT_FALSE(condition) \ + GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ + GTEST_NONFATAL_FAILURE_) +#define ASSERT_TRUE(condition) \ + GTEST_TEST_BOOLEAN_((condition), #condition, false, true, \ + GTEST_FATAL_FAILURE_) +#define ASSERT_FALSE(condition) \ + GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ + GTEST_FATAL_FAILURE_) + +// Includes the auto-generated header that implements a family of +// generic predicate assertion macros. +#include "gtest/gtest_pred_impl.h" + +// Macros for testing equalities and inequalities. +// +// * {ASSERT|EXPECT}_EQ(v1, v2): Tests that v1 == v2 +// * {ASSERT|EXPECT}_NE(v1, v2): Tests that v1 != v2 +// * {ASSERT|EXPECT}_LT(v1, v2): Tests that v1 < v2 +// * {ASSERT|EXPECT}_LE(v1, v2): Tests that v1 <= v2 +// * {ASSERT|EXPECT}_GT(v1, v2): Tests that v1 > v2 +// * {ASSERT|EXPECT}_GE(v1, v2): Tests that v1 >= v2 +// +// When they are not, Google Test prints both the tested expressions and +// their actual values. The values must be compatible built-in types, +// or you will get a compiler error. By "compatible" we mean that the +// values can be compared by the respective operator. +// +// Note: +// +// 1. It is possible to make a user-defined type work with +// {ASSERT|EXPECT}_??(), but that requires overloading the +// comparison operators and is thus discouraged by the Google C++ +// Usage Guide. Therefore, you are advised to use the +// {ASSERT|EXPECT}_TRUE() macro to assert that two objects are +// equal. +// +// 2. The {ASSERT|EXPECT}_??() macros do pointer comparisons on +// pointers (in particular, C strings). Therefore, if you use it +// with two C strings, you are testing how their locations in memory +// are related, not how their content is related. To compare two C +// strings by content, use {ASSERT|EXPECT}_STR*(). +// +// 3. {ASSERT|EXPECT}_EQ(v1, v2) is preferred to +// {ASSERT|EXPECT}_TRUE(v1 == v2), as the former tells you +// what the actual value is when it fails, and similarly for the +// other comparisons. +// +// 4. Do not depend on the order in which {ASSERT|EXPECT}_??() +// evaluate their arguments, which is undefined. +// +// 5. These macros evaluate their arguments exactly once. +// +// Examples: +// +// EXPECT_NE(5, Foo()); +// EXPECT_EQ(NULL, a_pointer); +// ASSERT_LT(i, array_size); +// ASSERT_GT(records.size(), 0) << "There is no record left."; + +#define EXPECT_EQ(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal:: \ + EqHelper::Compare, \ + val1, val2) +#define EXPECT_NE(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperNE, val1, val2) +#define EXPECT_LE(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperLE, val1, val2) +#define EXPECT_LT(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperLT, val1, val2) +#define EXPECT_GE(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperGE, val1, val2) +#define EXPECT_GT(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperGT, val1, val2) + +#define GTEST_ASSERT_EQ(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal:: \ + EqHelper::Compare, \ + val1, val2) +#define GTEST_ASSERT_NE(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperNE, val1, val2) +#define GTEST_ASSERT_LE(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperLE, val1, val2) +#define GTEST_ASSERT_LT(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperLT, val1, val2) +#define GTEST_ASSERT_GE(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperGE, val1, val2) +#define GTEST_ASSERT_GT(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperGT, val1, val2) + +// Define macro GTEST_DONT_DEFINE_ASSERT_XY to 1 to omit the definition of +// ASSERT_XY(), which clashes with some users' own code. + +#if !GTEST_DONT_DEFINE_ASSERT_EQ +# define ASSERT_EQ(val1, val2) GTEST_ASSERT_EQ(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_NE +# define ASSERT_NE(val1, val2) GTEST_ASSERT_NE(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_LE +# define ASSERT_LE(val1, val2) GTEST_ASSERT_LE(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_LT +# define ASSERT_LT(val1, val2) GTEST_ASSERT_LT(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_GE +# define ASSERT_GE(val1, val2) GTEST_ASSERT_GE(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_GT +# define ASSERT_GT(val1, val2) GTEST_ASSERT_GT(val1, val2) +#endif + +// C-string Comparisons. All tests treat NULL and any non-NULL string +// as different. Two NULLs are equal. +// +// * {ASSERT|EXPECT}_STREQ(s1, s2): Tests that s1 == s2 +// * {ASSERT|EXPECT}_STRNE(s1, s2): Tests that s1 != s2 +// * {ASSERT|EXPECT}_STRCASEEQ(s1, s2): Tests that s1 == s2, ignoring case +// * {ASSERT|EXPECT}_STRCASENE(s1, s2): Tests that s1 != s2, ignoring case +// +// For wide or narrow string objects, you can use the +// {ASSERT|EXPECT}_??() macros. +// +// Don't depend on the order in which the arguments are evaluated, +// which is undefined. +// +// These macros evaluate their arguments exactly once. + +#define EXPECT_STREQ(s1, s2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTREQ, s1, s2) +#define EXPECT_STRNE(s1, s2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRNE, s1, s2) +#define EXPECT_STRCASEEQ(s1, s2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASEEQ, s1, s2) +#define EXPECT_STRCASENE(s1, s2)\ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASENE, s1, s2) + +#define ASSERT_STREQ(s1, s2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTREQ, s1, s2) +#define ASSERT_STRNE(s1, s2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRNE, s1, s2) +#define ASSERT_STRCASEEQ(s1, s2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASEEQ, s1, s2) +#define ASSERT_STRCASENE(s1, s2)\ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASENE, s1, s2) + +// Macros for comparing floating-point numbers. +// +// * {ASSERT|EXPECT}_FLOAT_EQ(val1, val2): +// Tests that two float values are almost equal. +// * {ASSERT|EXPECT}_DOUBLE_EQ(val1, val2): +// Tests that two double values are almost equal. +// * {ASSERT|EXPECT}_NEAR(v1, v2, abs_error): +// Tests that v1 and v2 are within the given distance to each other. +// +// Google Test uses ULP-based comparison to automatically pick a default +// error bound that is appropriate for the operands. See the +// FloatingPoint template class in gtest-internal.h if you are +// interested in the implementation details. + +#define EXPECT_FLOAT_EQ(val1, val2)\ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + val1, val2) + +#define EXPECT_DOUBLE_EQ(val1, val2)\ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + val1, val2) + +#define ASSERT_FLOAT_EQ(val1, val2)\ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + val1, val2) + +#define ASSERT_DOUBLE_EQ(val1, val2)\ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + val1, val2) + +#define EXPECT_NEAR(val1, val2, abs_error)\ + EXPECT_PRED_FORMAT3(::testing::internal::DoubleNearPredFormat, \ + val1, val2, abs_error) + +#define ASSERT_NEAR(val1, val2, abs_error)\ + ASSERT_PRED_FORMAT3(::testing::internal::DoubleNearPredFormat, \ + val1, val2, abs_error) + +// These predicate format functions work on floating-point values, and +// can be used in {ASSERT|EXPECT}_PRED_FORMAT2*(), e.g. +// +// EXPECT_PRED_FORMAT2(testing::DoubleLE, Foo(), 5.0); + +// Asserts that val1 is less than, or almost equal to, val2. Fails +// otherwise. In particular, it fails if either val1 or val2 is NaN. +GTEST_API_ AssertionResult FloatLE(const char* expr1, const char* expr2, + float val1, float val2); +GTEST_API_ AssertionResult DoubleLE(const char* expr1, const char* expr2, + double val1, double val2); + + +#if GTEST_OS_WINDOWS + +// Macros that test for HRESULT failure and success, these are only useful +// on Windows, and rely on Windows SDK macros and APIs to compile. +// +// * {ASSERT|EXPECT}_HRESULT_{SUCCEEDED|FAILED}(expr) +// +// When expr unexpectedly fails or succeeds, Google Test prints the +// expected result and the actual result with both a human-readable +// string representation of the error, if available, as well as the +// hex result code. +# define EXPECT_HRESULT_SUCCEEDED(expr) \ + EXPECT_PRED_FORMAT1(::testing::internal::IsHRESULTSuccess, (expr)) + +# define ASSERT_HRESULT_SUCCEEDED(expr) \ + ASSERT_PRED_FORMAT1(::testing::internal::IsHRESULTSuccess, (expr)) + +# define EXPECT_HRESULT_FAILED(expr) \ + EXPECT_PRED_FORMAT1(::testing::internal::IsHRESULTFailure, (expr)) + +# define ASSERT_HRESULT_FAILED(expr) \ + ASSERT_PRED_FORMAT1(::testing::internal::IsHRESULTFailure, (expr)) + +#endif // GTEST_OS_WINDOWS + +// Macros that execute statement and check that it doesn't generate new fatal +// failures in the current thread. +// +// * {ASSERT|EXPECT}_NO_FATAL_FAILURE(statement); +// +// Examples: +// +// EXPECT_NO_FATAL_FAILURE(Process()); +// ASSERT_NO_FATAL_FAILURE(Process()) << "Process() failed"; +// +#define ASSERT_NO_FATAL_FAILURE(statement) \ + GTEST_TEST_NO_FATAL_FAILURE_(statement, GTEST_FATAL_FAILURE_) +#define EXPECT_NO_FATAL_FAILURE(statement) \ + GTEST_TEST_NO_FATAL_FAILURE_(statement, GTEST_NONFATAL_FAILURE_) + +// Causes a trace (including the source file path, the current line +// number, and the given message) to be included in every test failure +// message generated by code in the current scope. The effect is +// undone when the control leaves the current scope. +// +// The message argument can be anything streamable to std::ostream. +// +// In the implementation, we include the current line number as part +// of the dummy variable name, thus allowing multiple SCOPED_TRACE()s +// to appear in the same block - as long as they are on different +// lines. +#define SCOPED_TRACE(message) \ + ::testing::internal::ScopedTrace GTEST_CONCAT_TOKEN_(gtest_trace_, __LINE__)(\ + __FILE__, __LINE__, ::testing::Message() << (message)) + +// Compile-time assertion for type equality. +// StaticAssertTypeEq() compiles iff type1 and type2 are +// the same type. The value it returns is not interesting. +// +// Instead of making StaticAssertTypeEq a class template, we make it a +// function template that invokes a helper class template. This +// prevents a user from misusing StaticAssertTypeEq by +// defining objects of that type. +// +// CAVEAT: +// +// When used inside a method of a class template, +// StaticAssertTypeEq() is effective ONLY IF the method is +// instantiated. For example, given: +// +// template class Foo { +// public: +// void Bar() { testing::StaticAssertTypeEq(); } +// }; +// +// the code: +// +// void Test1() { Foo foo; } +// +// will NOT generate a compiler error, as Foo::Bar() is never +// actually instantiated. Instead, you need: +// +// void Test2() { Foo foo; foo.Bar(); } +// +// to cause a compiler error. +template +bool StaticAssertTypeEq() { + (void)internal::StaticAssertTypeEqHelper(); + return true; +} + +// Defines a test. +// +// The first parameter is the name of the test case, and the second +// parameter is the name of the test within the test case. +// +// The convention is to end the test case name with "Test". For +// example, a test case for the Foo class can be named FooTest. +// +// Test code should appear between braces after an invocation of +// this macro. Example: +// +// TEST(FooTest, InitializesCorrectly) { +// Foo foo; +// EXPECT_TRUE(foo.StatusIsOK()); +// } + +// Note that we call GetTestTypeId() instead of GetTypeId< +// ::testing::Test>() here to get the type ID of testing::Test. This +// is to work around a suspected linker bug when using Google Test as +// a framework on Mac OS X. The bug causes GetTypeId< +// ::testing::Test>() to return different values depending on whether +// the call is from the Google Test framework itself or from user test +// code. GetTestTypeId() is guaranteed to always return the same +// value, as it always calls GetTypeId<>() from the Google Test +// framework. +#define GTEST_TEST(test_case_name, test_name)\ + GTEST_TEST_(test_case_name, test_name, \ + ::testing::Test, ::testing::internal::GetTestTypeId()) + +// Define this macro to 1 to omit the definition of TEST(), which +// is a generic name and clashes with some other libraries. +#if !GTEST_DONT_DEFINE_TEST +# define TEST(test_case_name, test_name) GTEST_TEST(test_case_name, test_name) +#endif + +// Defines a test that uses a test fixture. +// +// The first parameter is the name of the test fixture class, which +// also doubles as the test case name. The second parameter is the +// name of the test within the test case. +// +// A test fixture class must be declared earlier. The user should put +// his test code between braces after using this macro. Example: +// +// class FooTest : public testing::Test { +// protected: +// virtual void SetUp() { b_.AddElement(3); } +// +// Foo a_; +// Foo b_; +// }; +// +// TEST_F(FooTest, InitializesCorrectly) { +// EXPECT_TRUE(a_.StatusIsOK()); +// } +// +// TEST_F(FooTest, ReturnsElementCountCorrectly) { +// EXPECT_EQ(0, a_.size()); +// EXPECT_EQ(1, b_.size()); +// } + +#define TEST_F(test_fixture, test_name)\ + GTEST_TEST_(test_fixture, test_name, test_fixture, \ + ::testing::internal::GetTypeId()) + +} // namespace testing + +// Use this function in main() to run all tests. It returns 0 if all +// tests are successful, or 1 otherwise. +// +// RUN_ALL_TESTS() should be invoked after the command line has been +// parsed by InitGoogleTest(). +// +// This function was formerly a macro; thus, it is in the global +// namespace and has an all-caps name. +int RUN_ALL_TESTS() GTEST_MUST_USE_RESULT_; + +inline int RUN_ALL_TESTS() { + return ::testing::UnitTest::GetInstance()->Run(); +} + +#endif // GTEST_INCLUDE_GTEST_GTEST_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest_pred_impl.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest_pred_impl.h new file mode 100644 index 0000000000..30ae712f50 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest_pred_impl.h @@ -0,0 +1,358 @@ +// Copyright 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file is AUTOMATICALLY GENERATED on 10/31/2011 by command +// 'gen_gtest_pred_impl.py 5'. DO NOT EDIT BY HAND! +// +// Implements a family of generic predicate assertion macros. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ + +// Makes sure this header is not included before gtest.h. +#ifndef GTEST_INCLUDE_GTEST_GTEST_H_ +# error Do not include gtest_pred_impl.h directly. Include gtest.h instead. +#endif // GTEST_INCLUDE_GTEST_GTEST_H_ + +// This header implements a family of generic predicate assertion +// macros: +// +// ASSERT_PRED_FORMAT1(pred_format, v1) +// ASSERT_PRED_FORMAT2(pred_format, v1, v2) +// ... +// +// where pred_format is a function or functor that takes n (in the +// case of ASSERT_PRED_FORMATn) values and their source expression +// text, and returns a testing::AssertionResult. See the definition +// of ASSERT_EQ in gtest.h for an example. +// +// If you don't care about formatting, you can use the more +// restrictive version: +// +// ASSERT_PRED1(pred, v1) +// ASSERT_PRED2(pred, v1, v2) +// ... +// +// where pred is an n-ary function or functor that returns bool, +// and the values v1, v2, ..., must support the << operator for +// streaming to std::ostream. +// +// We also define the EXPECT_* variations. +// +// For now we only support predicates whose arity is at most 5. +// Please email googletestframework@googlegroups.com if you need +// support for higher arities. + +// GTEST_ASSERT_ is the basic statement to which all of the assertions +// in this file reduce. Don't use this in your code. + +#define GTEST_ASSERT_(expression, on_failure) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (const ::testing::AssertionResult gtest_ar = (expression)) \ + ; \ + else \ + on_failure(gtest_ar.failure_message()) + + +// Helper function for implementing {EXPECT|ASSERT}_PRED1. Don't use +// this in your code. +template +AssertionResult AssertPred1Helper(const char* pred_text, + const char* e1, + Pred pred, + const T1& v1) { + if (pred(v1)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT1. +// Don't use this in your code. +#define GTEST_PRED_FORMAT1_(pred_format, v1, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, v1), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED1. Don't use +// this in your code. +#define GTEST_PRED1_(pred, v1, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred1Helper(#pred, \ + #v1, \ + pred, \ + v1), on_failure) + +// Unary predicate assertion macros. +#define EXPECT_PRED_FORMAT1(pred_format, v1) \ + GTEST_PRED_FORMAT1_(pred_format, v1, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED1(pred, v1) \ + GTEST_PRED1_(pred, v1, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT1(pred_format, v1) \ + GTEST_PRED_FORMAT1_(pred_format, v1, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED1(pred, v1) \ + GTEST_PRED1_(pred, v1, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED2. Don't use +// this in your code. +template +AssertionResult AssertPred2Helper(const char* pred_text, + const char* e1, + const char* e2, + Pred pred, + const T1& v1, + const T2& v2) { + if (pred(v1, v2)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT2. +// Don't use this in your code. +#define GTEST_PRED_FORMAT2_(pred_format, v1, v2, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, v1, v2), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED2. Don't use +// this in your code. +#define GTEST_PRED2_(pred, v1, v2, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred2Helper(#pred, \ + #v1, \ + #v2, \ + pred, \ + v1, \ + v2), on_failure) + +// Binary predicate assertion macros. +#define EXPECT_PRED_FORMAT2(pred_format, v1, v2) \ + GTEST_PRED_FORMAT2_(pred_format, v1, v2, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED2(pred, v1, v2) \ + GTEST_PRED2_(pred, v1, v2, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT2(pred_format, v1, v2) \ + GTEST_PRED_FORMAT2_(pred_format, v1, v2, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED2(pred, v1, v2) \ + GTEST_PRED2_(pred, v1, v2, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED3. Don't use +// this in your code. +template +AssertionResult AssertPred3Helper(const char* pred_text, + const char* e1, + const char* e2, + const char* e3, + Pred pred, + const T1& v1, + const T2& v2, + const T3& v3) { + if (pred(v1, v2, v3)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ", " + << e3 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2 + << "\n" << e3 << " evaluates to " << v3; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT3. +// Don't use this in your code. +#define GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, v1, v2, v3), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED3. Don't use +// this in your code. +#define GTEST_PRED3_(pred, v1, v2, v3, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred3Helper(#pred, \ + #v1, \ + #v2, \ + #v3, \ + pred, \ + v1, \ + v2, \ + v3), on_failure) + +// Ternary predicate assertion macros. +#define EXPECT_PRED_FORMAT3(pred_format, v1, v2, v3) \ + GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED3(pred, v1, v2, v3) \ + GTEST_PRED3_(pred, v1, v2, v3, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT3(pred_format, v1, v2, v3) \ + GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED3(pred, v1, v2, v3) \ + GTEST_PRED3_(pred, v1, v2, v3, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED4. Don't use +// this in your code. +template +AssertionResult AssertPred4Helper(const char* pred_text, + const char* e1, + const char* e2, + const char* e3, + const char* e4, + Pred pred, + const T1& v1, + const T2& v2, + const T3& v3, + const T4& v4) { + if (pred(v1, v2, v3, v4)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ", " + << e3 << ", " + << e4 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2 + << "\n" << e3 << " evaluates to " << v3 + << "\n" << e4 << " evaluates to " << v4; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT4. +// Don't use this in your code. +#define GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, v1, v2, v3, v4), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED4. Don't use +// this in your code. +#define GTEST_PRED4_(pred, v1, v2, v3, v4, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred4Helper(#pred, \ + #v1, \ + #v2, \ + #v3, \ + #v4, \ + pred, \ + v1, \ + v2, \ + v3, \ + v4), on_failure) + +// 4-ary predicate assertion macros. +#define EXPECT_PRED_FORMAT4(pred_format, v1, v2, v3, v4) \ + GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED4(pred, v1, v2, v3, v4) \ + GTEST_PRED4_(pred, v1, v2, v3, v4, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT4(pred_format, v1, v2, v3, v4) \ + GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED4(pred, v1, v2, v3, v4) \ + GTEST_PRED4_(pred, v1, v2, v3, v4, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED5. Don't use +// this in your code. +template +AssertionResult AssertPred5Helper(const char* pred_text, + const char* e1, + const char* e2, + const char* e3, + const char* e4, + const char* e5, + Pred pred, + const T1& v1, + const T2& v2, + const T3& v3, + const T4& v4, + const T5& v5) { + if (pred(v1, v2, v3, v4, v5)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ", " + << e3 << ", " + << e4 << ", " + << e5 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2 + << "\n" << e3 << " evaluates to " << v3 + << "\n" << e4 << " evaluates to " << v4 + << "\n" << e5 << " evaluates to " << v5; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT5. +// Don't use this in your code. +#define GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, #v5, v1, v2, v3, v4, v5), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED5. Don't use +// this in your code. +#define GTEST_PRED5_(pred, v1, v2, v3, v4, v5, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred5Helper(#pred, \ + #v1, \ + #v2, \ + #v3, \ + #v4, \ + #v5, \ + pred, \ + v1, \ + v2, \ + v3, \ + v4, \ + v5), on_failure) + +// 5-ary predicate assertion macros. +#define EXPECT_PRED_FORMAT5(pred_format, v1, v2, v3, v4, v5) \ + GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED5(pred, v1, v2, v3, v4, v5) \ + GTEST_PRED5_(pred, v1, v2, v3, v4, v5, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT5(pred_format, v1, v2, v3, v4, v5) \ + GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED5(pred, v1, v2, v3, v4, v5) \ + GTEST_PRED5_(pred, v1, v2, v3, v4, v5, GTEST_FATAL_FAILURE_) + + + +#endif // GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest_prod.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest_prod.h new file mode 100644 index 0000000000..da80ddc6c7 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/gtest_prod.h @@ -0,0 +1,58 @@ +// Copyright 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// Google C++ Testing Framework definitions useful in production code. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_PROD_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PROD_H_ + +// When you need to test the private or protected members of a class, +// use the FRIEND_TEST macro to declare your tests as friends of the +// class. For example: +// +// class MyClass { +// private: +// void MyMethod(); +// FRIEND_TEST(MyClassTest, MyMethod); +// }; +// +// class MyClassTest : public testing::Test { +// // ... +// }; +// +// TEST_F(MyClassTest, MyMethod) { +// // Can call MyClass::MyMethod() here. +// } + +#define FRIEND_TEST(test_case_name, test_name)\ +friend class test_case_name##_##test_name##_Test + +#endif // GTEST_INCLUDE_GTEST_GTEST_PROD_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/custom/gtest-port.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/custom/gtest-port.h new file mode 100644 index 0000000000..7e744bd3bb --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/custom/gtest-port.h @@ -0,0 +1,69 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Injection point for custom user configurations. +// The following macros can be defined: +// +// Flag related macros: +// GTEST_FLAG(flag_name) +// GTEST_USE_OWN_FLAGFILE_FLAG_ - Define to 0 when the system provides its +// own flagfile flag parsing. +// GTEST_DECLARE_bool_(name) +// GTEST_DECLARE_int32_(name) +// GTEST_DECLARE_string_(name) +// GTEST_DEFINE_bool_(name, default_val, doc) +// GTEST_DEFINE_int32_(name, default_val, doc) +// GTEST_DEFINE_string_(name, default_val, doc) +// +// Test filtering: +// GTEST_TEST_FILTER_ENV_VAR_ - The name of an environment variable that +// will be used if --GTEST_FLAG(test_filter) +// is not provided. +// +// Logging: +// GTEST_LOG_(severity) +// GTEST_CHECK_(condition) +// Functions LogToStderr() and FlushInfoLog() have to be provided too. +// +// Threading: +// GTEST_HAS_NOTIFICATION_ - Enabled if Notification is already provided. +// GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ - Enabled if Mutex and ThreadLocal are +// already provided. +// Must also provide GTEST_DECLARE_STATIC_MUTEX_(mutex) and +// GTEST_DEFINE_STATIC_MUTEX_(mutex) +// +// GTEST_EXCLUSIVE_LOCK_REQUIRED_(locks) +// GTEST_LOCK_EXCLUDED_(locks) +// +// ** Custom implementation starts here ** + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/custom/gtest-printers.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/custom/gtest-printers.h new file mode 100644 index 0000000000..60c1ea050b --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/custom/gtest-printers.h @@ -0,0 +1,42 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// This file provides an injection point for custom printers in a local +// installation of gTest. +// It will be included from gtest-printers.h and the overrides in this file +// will be visible to everyone. +// See documentation at gtest/gtest-printers.h for details on how to define a +// custom printer. +// +// ** Custom implementation starts here ** + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/custom/gtest.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/custom/gtest.h new file mode 100644 index 0000000000..c27412a898 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/custom/gtest.h @@ -0,0 +1,41 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Injection point for custom user configurations. +// The following macros can be defined: +// +// GTEST_OS_STACK_TRACE_GETTER_ - The name of an implementation of +// OsStackTraceGetterInterface. +// +// ** Custom implementation starts here ** + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-death-test-internal.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-death-test-internal.h new file mode 100644 index 0000000000..2b3a78f5bf --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-death-test-internal.h @@ -0,0 +1,319 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: wan@google.com (Zhanyong Wan), eefacm@gmail.com (Sean Mcafee) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines internal utilities needed for implementing +// death tests. They are subject to change without notice. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ + +#include "gtest/internal/gtest-internal.h" + +#include + +namespace testing { +namespace internal { + +GTEST_DECLARE_string_(internal_run_death_test); + +// Names of the flags (needed for parsing Google Test flags). +const char kDeathTestStyleFlag[] = "death_test_style"; +const char kDeathTestUseFork[] = "death_test_use_fork"; +const char kInternalRunDeathTestFlag[] = "internal_run_death_test"; + +#if GTEST_HAS_DEATH_TEST + +// DeathTest is a class that hides much of the complexity of the +// GTEST_DEATH_TEST_ macro. It is abstract; its static Create method +// returns a concrete class that depends on the prevailing death test +// style, as defined by the --gtest_death_test_style and/or +// --gtest_internal_run_death_test flags. + +// In describing the results of death tests, these terms are used with +// the corresponding definitions: +// +// exit status: The integer exit information in the format specified +// by wait(2) +// exit code: The integer code passed to exit(3), _exit(2), or +// returned from main() +class GTEST_API_ DeathTest { + public: + // Create returns false if there was an error determining the + // appropriate action to take for the current death test; for example, + // if the gtest_death_test_style flag is set to an invalid value. + // The LastMessage method will return a more detailed message in that + // case. Otherwise, the DeathTest pointer pointed to by the "test" + // argument is set. If the death test should be skipped, the pointer + // is set to NULL; otherwise, it is set to the address of a new concrete + // DeathTest object that controls the execution of the current test. + static bool Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test); + DeathTest(); + virtual ~DeathTest() { } + + // A helper class that aborts a death test when it's deleted. + class ReturnSentinel { + public: + explicit ReturnSentinel(DeathTest* test) : test_(test) { } + ~ReturnSentinel() { test_->Abort(TEST_ENCOUNTERED_RETURN_STATEMENT); } + private: + DeathTest* const test_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ReturnSentinel); + } GTEST_ATTRIBUTE_UNUSED_; + + // An enumeration of possible roles that may be taken when a death + // test is encountered. EXECUTE means that the death test logic should + // be executed immediately. OVERSEE means that the program should prepare + // the appropriate environment for a child process to execute the death + // test, then wait for it to complete. + enum TestRole { OVERSEE_TEST, EXECUTE_TEST }; + + // An enumeration of the three reasons that a test might be aborted. + enum AbortReason { + TEST_ENCOUNTERED_RETURN_STATEMENT, + TEST_THREW_EXCEPTION, + TEST_DID_NOT_DIE + }; + + // Assumes one of the above roles. + virtual TestRole AssumeRole() = 0; + + // Waits for the death test to finish and returns its status. + virtual int Wait() = 0; + + // Returns true if the death test passed; that is, the test process + // exited during the test, its exit status matches a user-supplied + // predicate, and its stderr output matches a user-supplied regular + // expression. + // The user-supplied predicate may be a macro expression rather + // than a function pointer or functor, or else Wait and Passed could + // be combined. + virtual bool Passed(bool exit_status_ok) = 0; + + // Signals that the death test did not die as expected. + virtual void Abort(AbortReason reason) = 0; + + // Returns a human-readable outcome message regarding the outcome of + // the last death test. + static const char* LastMessage(); + + static void set_last_death_test_message(const std::string& message); + + private: + // A string containing a description of the outcome of the last death test. + static std::string last_death_test_message_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(DeathTest); +}; + +// Factory interface for death tests. May be mocked out for testing. +class DeathTestFactory { + public: + virtual ~DeathTestFactory() { } + virtual bool Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test) = 0; +}; + +// A concrete DeathTestFactory implementation for normal use. +class DefaultDeathTestFactory : public DeathTestFactory { + public: + virtual bool Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test); +}; + +// Returns true if exit_status describes a process that was terminated +// by a signal, or exited normally with a nonzero exit code. +GTEST_API_ bool ExitedUnsuccessfully(int exit_status); + +// Traps C++ exceptions escaping statement and reports them as test +// failures. Note that trapping SEH exceptions is not implemented here. +# if GTEST_HAS_EXCEPTIONS +# define GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, death_test) \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } catch (const ::std::exception& gtest_exception) { \ + fprintf(\ + stderr, \ + "\n%s: Caught std::exception-derived exception escaping the " \ + "death test statement. Exception message: %s\n", \ + ::testing::internal::FormatFileLocation(__FILE__, __LINE__).c_str(), \ + gtest_exception.what()); \ + fflush(stderr); \ + death_test->Abort(::testing::internal::DeathTest::TEST_THREW_EXCEPTION); \ + } catch (...) { \ + death_test->Abort(::testing::internal::DeathTest::TEST_THREW_EXCEPTION); \ + } + +# else +# define GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, death_test) \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) + +# endif + +// This macro is for implementing ASSERT_DEATH*, EXPECT_DEATH*, +// ASSERT_EXIT*, and EXPECT_EXIT*. +# define GTEST_DEATH_TEST_(statement, predicate, regex, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + const ::testing::internal::RE& gtest_regex = (regex); \ + ::testing::internal::DeathTest* gtest_dt; \ + if (!::testing::internal::DeathTest::Create(#statement, >est_regex, \ + __FILE__, __LINE__, >est_dt)) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__); \ + } \ + if (gtest_dt != NULL) { \ + ::testing::internal::scoped_ptr< ::testing::internal::DeathTest> \ + gtest_dt_ptr(gtest_dt); \ + switch (gtest_dt->AssumeRole()) { \ + case ::testing::internal::DeathTest::OVERSEE_TEST: \ + if (!gtest_dt->Passed(predicate(gtest_dt->Wait()))) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__); \ + } \ + break; \ + case ::testing::internal::DeathTest::EXECUTE_TEST: { \ + ::testing::internal::DeathTest::ReturnSentinel \ + gtest_sentinel(gtest_dt); \ + GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, gtest_dt); \ + gtest_dt->Abort(::testing::internal::DeathTest::TEST_DID_NOT_DIE); \ + break; \ + } \ + default: \ + break; \ + } \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__): \ + fail(::testing::internal::DeathTest::LastMessage()) +// The symbol "fail" here expands to something into which a message +// can be streamed. + +// This macro is for implementing ASSERT/EXPECT_DEBUG_DEATH when compiled in +// NDEBUG mode. In this case we need the statements to be executed, the regex is +// ignored, and the macro must accept a streamed message even though the message +// is never printed. +# define GTEST_EXECUTE_STATEMENT_(statement, regex) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } else \ + ::testing::Message() + +// A class representing the parsed contents of the +// --gtest_internal_run_death_test flag, as it existed when +// RUN_ALL_TESTS was called. +class InternalRunDeathTestFlag { + public: + InternalRunDeathTestFlag(const std::string& a_file, + int a_line, + int an_index, + int a_write_fd) + : file_(a_file), line_(a_line), index_(an_index), + write_fd_(a_write_fd) {} + + ~InternalRunDeathTestFlag() { + if (write_fd_ >= 0) + posix::Close(write_fd_); + } + + const std::string& file() const { return file_; } + int line() const { return line_; } + int index() const { return index_; } + int write_fd() const { return write_fd_; } + + private: + std::string file_; + int line_; + int index_; + int write_fd_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(InternalRunDeathTestFlag); +}; + +// Returns a newly created InternalRunDeathTestFlag object with fields +// initialized from the GTEST_FLAG(internal_run_death_test) flag if +// the flag is specified; otherwise returns NULL. +InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag(); + +#else // GTEST_HAS_DEATH_TEST + +// This macro is used for implementing macros such as +// EXPECT_DEATH_IF_SUPPORTED and ASSERT_DEATH_IF_SUPPORTED on systems where +// death tests are not supported. Those macros must compile on such systems +// iff EXPECT_DEATH and ASSERT_DEATH compile with the same parameters on +// systems that support death tests. This allows one to write such a macro +// on a system that does not support death tests and be sure that it will +// compile on a death-test supporting system. +// +// Parameters: +// statement - A statement that a macro such as EXPECT_DEATH would test +// for program termination. This macro has to make sure this +// statement is compiled but not executed, to ensure that +// EXPECT_DEATH_IF_SUPPORTED compiles with a certain +// parameter iff EXPECT_DEATH compiles with it. +// regex - A regex that a macro such as EXPECT_DEATH would use to test +// the output of statement. This parameter has to be +// compiled but not evaluated by this macro, to ensure that +// this macro only accepts expressions that a macro such as +// EXPECT_DEATH would accept. +// terminator - Must be an empty statement for EXPECT_DEATH_IF_SUPPORTED +// and a return statement for ASSERT_DEATH_IF_SUPPORTED. +// This ensures that ASSERT_DEATH_IF_SUPPORTED will not +// compile inside functions where ASSERT_DEATH doesn't +// compile. +// +// The branch that has an always false condition is used to ensure that +// statement and regex are compiled (and thus syntactically correct) but +// never executed. The unreachable code macro protects the terminator +// statement from generating an 'unreachable code' warning in case +// statement unconditionally returns or throws. The Message constructor at +// the end allows the syntax of streaming additional messages into the +// macro, for compilational compatibility with EXPECT_DEATH/ASSERT_DEATH. +# define GTEST_UNSUPPORTED_DEATH_TEST_(statement, regex, terminator) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + GTEST_LOG_(WARNING) \ + << "Death tests are not supported on this platform.\n" \ + << "Statement '" #statement "' cannot be verified."; \ + } else if (::testing::internal::AlwaysFalse()) { \ + ::testing::internal::RE::PartialMatch(".*", (regex)); \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + terminator; \ + } else \ + ::testing::Message() + +#endif // GTEST_HAS_DEATH_TEST + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-filepath.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-filepath.h new file mode 100644 index 0000000000..7a13b4b0de --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-filepath.h @@ -0,0 +1,206 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: keith.ray@gmail.com (Keith Ray) +// +// Google Test filepath utilities +// +// This header file declares classes and functions used internally by +// Google Test. They are subject to change without notice. +// +// This file is #included in . +// Do not include this header file separately! + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ + +#include "gtest/internal/gtest-string.h" + +namespace testing { +namespace internal { + +// FilePath - a class for file and directory pathname manipulation which +// handles platform-specific conventions (like the pathname separator). +// Used for helper functions for naming files in a directory for xml output. +// Except for Set methods, all methods are const or static, which provides an +// "immutable value object" -- useful for peace of mind. +// A FilePath with a value ending in a path separator ("like/this/") represents +// a directory, otherwise it is assumed to represent a file. In either case, +// it may or may not represent an actual file or directory in the file system. +// Names are NOT checked for syntax correctness -- no checking for illegal +// characters, malformed paths, etc. + +class GTEST_API_ FilePath { + public: + FilePath() : pathname_("") { } + FilePath(const FilePath& rhs) : pathname_(rhs.pathname_) { } + + explicit FilePath(const std::string& pathname) : pathname_(pathname) { + Normalize(); + } + + FilePath& operator=(const FilePath& rhs) { + Set(rhs); + return *this; + } + + void Set(const FilePath& rhs) { + pathname_ = rhs.pathname_; + } + + const std::string& string() const { return pathname_; } + const char* c_str() const { return pathname_.c_str(); } + + // Returns the current working directory, or "" if unsuccessful. + static FilePath GetCurrentDir(); + + // Given directory = "dir", base_name = "test", number = 0, + // extension = "xml", returns "dir/test.xml". If number is greater + // than zero (e.g., 12), returns "dir/test_12.xml". + // On Windows platform, uses \ as the separator rather than /. + static FilePath MakeFileName(const FilePath& directory, + const FilePath& base_name, + int number, + const char* extension); + + // Given directory = "dir", relative_path = "test.xml", + // returns "dir/test.xml". + // On Windows, uses \ as the separator rather than /. + static FilePath ConcatPaths(const FilePath& directory, + const FilePath& relative_path); + + // Returns a pathname for a file that does not currently exist. The pathname + // will be directory/base_name.extension or + // directory/base_name_.extension if directory/base_name.extension + // already exists. The number will be incremented until a pathname is found + // that does not already exist. + // Examples: 'dir/foo_test.xml' or 'dir/foo_test_1.xml'. + // There could be a race condition if two or more processes are calling this + // function at the same time -- they could both pick the same filename. + static FilePath GenerateUniqueFileName(const FilePath& directory, + const FilePath& base_name, + const char* extension); + + // Returns true iff the path is "". + bool IsEmpty() const { return pathname_.empty(); } + + // If input name has a trailing separator character, removes it and returns + // the name, otherwise return the name string unmodified. + // On Windows platform, uses \ as the separator, other platforms use /. + FilePath RemoveTrailingPathSeparator() const; + + // Returns a copy of the FilePath with the directory part removed. + // Example: FilePath("path/to/file").RemoveDirectoryName() returns + // FilePath("file"). If there is no directory part ("just_a_file"), it returns + // the FilePath unmodified. If there is no file part ("just_a_dir/") it + // returns an empty FilePath (""). + // On Windows platform, '\' is the path separator, otherwise it is '/'. + FilePath RemoveDirectoryName() const; + + // RemoveFileName returns the directory path with the filename removed. + // Example: FilePath("path/to/file").RemoveFileName() returns "path/to/". + // If the FilePath is "a_file" or "/a_file", RemoveFileName returns + // FilePath("./") or, on Windows, FilePath(".\\"). If the filepath does + // not have a file, like "just/a/dir/", it returns the FilePath unmodified. + // On Windows platform, '\' is the path separator, otherwise it is '/'. + FilePath RemoveFileName() const; + + // Returns a copy of the FilePath with the case-insensitive extension removed. + // Example: FilePath("dir/file.exe").RemoveExtension("EXE") returns + // FilePath("dir/file"). If a case-insensitive extension is not + // found, returns a copy of the original FilePath. + FilePath RemoveExtension(const char* extension) const; + + // Creates directories so that path exists. Returns true if successful or if + // the directories already exist; returns false if unable to create + // directories for any reason. Will also return false if the FilePath does + // not represent a directory (that is, it doesn't end with a path separator). + bool CreateDirectoriesRecursively() const; + + // Create the directory so that path exists. Returns true if successful or + // if the directory already exists; returns false if unable to create the + // directory for any reason, including if the parent directory does not + // exist. Not named "CreateDirectory" because that's a macro on Windows. + bool CreateFolder() const; + + // Returns true if FilePath describes something in the file-system, + // either a file, directory, or whatever, and that something exists. + bool FileOrDirectoryExists() const; + + // Returns true if pathname describes a directory in the file-system + // that exists. + bool DirectoryExists() const; + + // Returns true if FilePath ends with a path separator, which indicates that + // it is intended to represent a directory. Returns false otherwise. + // This does NOT check that a directory (or file) actually exists. + bool IsDirectory() const; + + // Returns true if pathname describes a root directory. (Windows has one + // root directory per disk drive.) + bool IsRootDirectory() const; + + // Returns true if pathname describes an absolute path. + bool IsAbsolutePath() const; + + private: + // Replaces multiple consecutive separators with a single separator. + // For example, "bar///foo" becomes "bar/foo". Does not eliminate other + // redundancies that might be in a pathname involving "." or "..". + // + // A pathname with multiple consecutive separators may occur either through + // user error or as a result of some scripts or APIs that generate a pathname + // with a trailing separator. On other platforms the same API or script + // may NOT generate a pathname with a trailing "/". Then elsewhere that + // pathname may have another "/" and pathname components added to it, + // without checking for the separator already being there. + // The script language and operating system may allow paths like "foo//bar" + // but some of the functions in FilePath will not handle that correctly. In + // particular, RemoveTrailingPathSeparator() only removes one separator, and + // it is called in CreateDirectoriesRecursively() assuming that it will change + // a pathname from directory syntax (trailing separator) to filename syntax. + // + // On Windows this method also replaces the alternate path separator '/' with + // the primary path separator '\\', so that for example "bar\\/\\foo" becomes + // "bar\\foo". + + void Normalize(); + + // Returns a pointer to the last occurence of a valid path separator in + // the FilePath. On Windows, for example, both '/' and '\' are valid path + // separators. Returns NULL if no path separator was found. + const char* FindLastPathSeparator() const; + + std::string pathname_; +}; // class FilePath + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-internal.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-internal.h new file mode 100644 index 0000000000..ebd1cf615d --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-internal.h @@ -0,0 +1,1238 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: wan@google.com (Zhanyong Wan), eefacm@gmail.com (Sean Mcafee) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file declares functions and macros used internally by +// Google Test. They are subject to change without notice. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ + +#include "gtest/internal/gtest-port.h" + +#if GTEST_OS_LINUX +# include +# include +# include +# include +#endif // GTEST_OS_LINUX + +#if GTEST_HAS_EXCEPTIONS +# include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest-message.h" +#include "gtest/internal/gtest-string.h" +#include "gtest/internal/gtest-filepath.h" +#include "gtest/internal/gtest-type-util.h" + +// Due to C++ preprocessor weirdness, we need double indirection to +// concatenate two tokens when one of them is __LINE__. Writing +// +// foo ## __LINE__ +// +// will result in the token foo__LINE__, instead of foo followed by +// the current line number. For more details, see +// http://www.parashift.com/c++-faq-lite/misc-technical-issues.html#faq-39.6 +#define GTEST_CONCAT_TOKEN_(foo, bar) GTEST_CONCAT_TOKEN_IMPL_(foo, bar) +#define GTEST_CONCAT_TOKEN_IMPL_(foo, bar) foo ## bar + +class ProtocolMessage; +namespace proto2 { class Message; } + +namespace testing { + +// Forward declarations. + +class AssertionResult; // Result of an assertion. +class Message; // Represents a failure message. +class Test; // Represents a test. +class TestInfo; // Information about a test. +class TestPartResult; // Result of a test part. +class UnitTest; // A collection of test cases. + +template +::std::string PrintToString(const T& value); + +namespace internal { + +struct TraceInfo; // Information about a trace point. +class ScopedTrace; // Implements scoped trace. +class TestInfoImpl; // Opaque implementation of TestInfo +class UnitTestImpl; // Opaque implementation of UnitTest + +// The text used in failure messages to indicate the start of the +// stack trace. +GTEST_API_ extern const char kStackTraceMarker[]; + +// Two overloaded helpers for checking at compile time whether an +// expression is a null pointer literal (i.e. NULL or any 0-valued +// compile-time integral constant). Their return values have +// different sizes, so we can use sizeof() to test which version is +// picked by the compiler. These helpers have no implementations, as +// we only need their signatures. +// +// Given IsNullLiteralHelper(x), the compiler will pick the first +// version if x can be implicitly converted to Secret*, and pick the +// second version otherwise. Since Secret is a secret and incomplete +// type, the only expression a user can write that has type Secret* is +// a null pointer literal. Therefore, we know that x is a null +// pointer literal if and only if the first version is picked by the +// compiler. +char IsNullLiteralHelper(Secret* p); +char (&IsNullLiteralHelper(...))[2]; // NOLINT + +// A compile-time bool constant that is true if and only if x is a +// null pointer literal (i.e. NULL or any 0-valued compile-time +// integral constant). +#ifdef GTEST_ELLIPSIS_NEEDS_POD_ +// We lose support for NULL detection where the compiler doesn't like +// passing non-POD classes through ellipsis (...). +# define GTEST_IS_NULL_LITERAL_(x) false +#else +# define GTEST_IS_NULL_LITERAL_(x) \ + (sizeof(::testing::internal::IsNullLiteralHelper(x)) == 1) +#endif // GTEST_ELLIPSIS_NEEDS_POD_ + +// Appends the user-supplied message to the Google-Test-generated message. +GTEST_API_ std::string AppendUserMessage( + const std::string& gtest_msg, const Message& user_msg); + +#if GTEST_HAS_EXCEPTIONS + +// This exception is thrown by (and only by) a failed Google Test +// assertion when GTEST_FLAG(throw_on_failure) is true (if exceptions +// are enabled). We derive it from std::runtime_error, which is for +// errors presumably detectable only at run time. Since +// std::runtime_error inherits from std::exception, many testing +// frameworks know how to extract and print the message inside it. +class GTEST_API_ GoogleTestFailureException : public ::std::runtime_error { + public: + explicit GoogleTestFailureException(const TestPartResult& failure); +}; + +#endif // GTEST_HAS_EXCEPTIONS + +// A helper class for creating scoped traces in user programs. +class GTEST_API_ ScopedTrace { + public: + // The c'tor pushes the given source file location and message onto + // a trace stack maintained by Google Test. + ScopedTrace(const char* file, int line, const Message& message); + + // The d'tor pops the info pushed by the c'tor. + // + // Note that the d'tor is not virtual in order to be efficient. + // Don't inherit from ScopedTrace! + ~ScopedTrace(); + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedTrace); +} GTEST_ATTRIBUTE_UNUSED_; // A ScopedTrace object does its job in its + // c'tor and d'tor. Therefore it doesn't + // need to be used otherwise. + +namespace edit_distance { +// Returns the optimal edits to go from 'left' to 'right'. +// All edits cost the same, with replace having lower priority than +// add/remove. +// Simple implementation of the Wagner–Fischer algorithm. +// See http://en.wikipedia.org/wiki/Wagner-Fischer_algorithm +enum EditType { kMatch, kAdd, kRemove, kReplace }; +GTEST_API_ std::vector CalculateOptimalEdits( + const std::vector& left, const std::vector& right); + +// Same as above, but the input is represented as strings. +GTEST_API_ std::vector CalculateOptimalEdits( + const std::vector& left, + const std::vector& right); + +// Create a diff of the input strings in Unified diff format. +GTEST_API_ std::string CreateUnifiedDiff(const std::vector& left, + const std::vector& right, + size_t context = 2); + +} // namespace edit_distance + +// Calculate the diff between 'left' and 'right' and return it in unified diff +// format. +// If not null, stores in 'total_line_count' the total number of lines found +// in left + right. +GTEST_API_ std::string DiffStrings(const std::string& left, + const std::string& right, + size_t* total_line_count); + +// Constructs and returns the message for an equality assertion +// (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure. +// +// The first four parameters are the expressions used in the assertion +// and their values, as strings. For example, for ASSERT_EQ(foo, bar) +// where foo is 5 and bar is 6, we have: +// +// expected_expression: "foo" +// actual_expression: "bar" +// expected_value: "5" +// actual_value: "6" +// +// The ignoring_case parameter is true iff the assertion is a +// *_STRCASEEQ*. When it's true, the string " (ignoring case)" will +// be inserted into the message. +GTEST_API_ AssertionResult EqFailure(const char* expected_expression, + const char* actual_expression, + const std::string& expected_value, + const std::string& actual_value, + bool ignoring_case); + +// Constructs a failure message for Boolean assertions such as EXPECT_TRUE. +GTEST_API_ std::string GetBoolAssertionFailureMessage( + const AssertionResult& assertion_result, + const char* expression_text, + const char* actual_predicate_value, + const char* expected_predicate_value); + +// This template class represents an IEEE floating-point number +// (either single-precision or double-precision, depending on the +// template parameters). +// +// The purpose of this class is to do more sophisticated number +// comparison. (Due to round-off error, etc, it's very unlikely that +// two floating-points will be equal exactly. Hence a naive +// comparison by the == operation often doesn't work.) +// +// Format of IEEE floating-point: +// +// The most-significant bit being the leftmost, an IEEE +// floating-point looks like +// +// sign_bit exponent_bits fraction_bits +// +// Here, sign_bit is a single bit that designates the sign of the +// number. +// +// For float, there are 8 exponent bits and 23 fraction bits. +// +// For double, there are 11 exponent bits and 52 fraction bits. +// +// More details can be found at +// http://en.wikipedia.org/wiki/IEEE_floating-point_standard. +// +// Template parameter: +// +// RawType: the raw floating-point type (either float or double) +template +class FloatingPoint { + public: + // Defines the unsigned integer type that has the same size as the + // floating point number. + typedef typename TypeWithSize::UInt Bits; + + // Constants. + + // # of bits in a number. + static const size_t kBitCount = 8*sizeof(RawType); + + // # of fraction bits in a number. + static const size_t kFractionBitCount = + std::numeric_limits::digits - 1; + + // # of exponent bits in a number. + static const size_t kExponentBitCount = kBitCount - 1 - kFractionBitCount; + + // The mask for the sign bit. + static const Bits kSignBitMask = static_cast(1) << (kBitCount - 1); + + // The mask for the fraction bits. + static const Bits kFractionBitMask = + ~static_cast(0) >> (kExponentBitCount + 1); + + // The mask for the exponent bits. + static const Bits kExponentBitMask = ~(kSignBitMask | kFractionBitMask); + + // How many ULP's (Units in the Last Place) we want to tolerate when + // comparing two numbers. The larger the value, the more error we + // allow. A 0 value means that two numbers must be exactly the same + // to be considered equal. + // + // The maximum error of a single floating-point operation is 0.5 + // units in the last place. On Intel CPU's, all floating-point + // calculations are done with 80-bit precision, while double has 64 + // bits. Therefore, 4 should be enough for ordinary use. + // + // See the following article for more details on ULP: + // http://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/ + static const size_t kMaxUlps = 4; + + // Constructs a FloatingPoint from a raw floating-point number. + // + // On an Intel CPU, passing a non-normalized NAN (Not a Number) + // around may change its bits, although the new value is guaranteed + // to be also a NAN. Therefore, don't expect this constructor to + // preserve the bits in x when x is a NAN. + explicit FloatingPoint(const RawType& x) { u_.value_ = x; } + + // Static methods + + // Reinterprets a bit pattern as a floating-point number. + // + // This function is needed to test the AlmostEquals() method. + static RawType ReinterpretBits(const Bits bits) { + FloatingPoint fp(0); + fp.u_.bits_ = bits; + return fp.u_.value_; + } + + // Returns the floating-point number that represent positive infinity. + static RawType Infinity() { + return ReinterpretBits(kExponentBitMask); + } + + // Returns the maximum representable finite floating-point number. + static RawType Max(); + + // Non-static methods + + // Returns the bits that represents this number. + const Bits &bits() const { return u_.bits_; } + + // Returns the exponent bits of this number. + Bits exponent_bits() const { return kExponentBitMask & u_.bits_; } + + // Returns the fraction bits of this number. + Bits fraction_bits() const { return kFractionBitMask & u_.bits_; } + + // Returns the sign bit of this number. + Bits sign_bit() const { return kSignBitMask & u_.bits_; } + + // Returns true iff this is NAN (not a number). + bool is_nan() const { + // It's a NAN if the exponent bits are all ones and the fraction + // bits are not entirely zeros. + return (exponent_bits() == kExponentBitMask) && (fraction_bits() != 0); + } + + // Returns true iff this number is at most kMaxUlps ULP's away from + // rhs. In particular, this function: + // + // - returns false if either number is (or both are) NAN. + // - treats really large numbers as almost equal to infinity. + // - thinks +0.0 and -0.0 are 0 DLP's apart. + bool AlmostEquals(const FloatingPoint& rhs) const { + // The IEEE standard says that any comparison operation involving + // a NAN must return false. + if (is_nan() || rhs.is_nan()) return false; + + return DistanceBetweenSignAndMagnitudeNumbers(u_.bits_, rhs.u_.bits_) + <= kMaxUlps; + } + + private: + // The data type used to store the actual floating-point number. + union FloatingPointUnion { + RawType value_; // The raw floating-point number. + Bits bits_; // The bits that represent the number. + }; + + // Converts an integer from the sign-and-magnitude representation to + // the biased representation. More precisely, let N be 2 to the + // power of (kBitCount - 1), an integer x is represented by the + // unsigned number x + N. + // + // For instance, + // + // -N + 1 (the most negative number representable using + // sign-and-magnitude) is represented by 1; + // 0 is represented by N; and + // N - 1 (the biggest number representable using + // sign-and-magnitude) is represented by 2N - 1. + // + // Read http://en.wikipedia.org/wiki/Signed_number_representations + // for more details on signed number representations. + static Bits SignAndMagnitudeToBiased(const Bits &sam) { + if (kSignBitMask & sam) { + // sam represents a negative number. + return ~sam + 1; + } else { + // sam represents a positive number. + return kSignBitMask | sam; + } + } + + // Given two numbers in the sign-and-magnitude representation, + // returns the distance between them as an unsigned number. + static Bits DistanceBetweenSignAndMagnitudeNumbers(const Bits &sam1, + const Bits &sam2) { + const Bits biased1 = SignAndMagnitudeToBiased(sam1); + const Bits biased2 = SignAndMagnitudeToBiased(sam2); + return (biased1 >= biased2) ? (biased1 - biased2) : (biased2 - biased1); + } + + FloatingPointUnion u_; +}; + +// We cannot use std::numeric_limits::max() as it clashes with the max() +// macro defined by . +template <> +inline float FloatingPoint::Max() { return FLT_MAX; } +template <> +inline double FloatingPoint::Max() { return DBL_MAX; } + +// Typedefs the instances of the FloatingPoint template class that we +// care to use. +typedef FloatingPoint Float; +typedef FloatingPoint Double; + +// In order to catch the mistake of putting tests that use different +// test fixture classes in the same test case, we need to assign +// unique IDs to fixture classes and compare them. The TypeId type is +// used to hold such IDs. The user should treat TypeId as an opaque +// type: the only operation allowed on TypeId values is to compare +// them for equality using the == operator. +typedef const void* TypeId; + +template +class TypeIdHelper { + public: + // dummy_ must not have a const type. Otherwise an overly eager + // compiler (e.g. MSVC 7.1 & 8.0) may try to merge + // TypeIdHelper::dummy_ for different Ts as an "optimization". + static bool dummy_; +}; + +template +bool TypeIdHelper::dummy_ = false; + +// GetTypeId() returns the ID of type T. Different values will be +// returned for different types. Calling the function twice with the +// same type argument is guaranteed to return the same ID. +template +TypeId GetTypeId() { + // The compiler is required to allocate a different + // TypeIdHelper::dummy_ variable for each T used to instantiate + // the template. Therefore, the address of dummy_ is guaranteed to + // be unique. + return &(TypeIdHelper::dummy_); +} + +// Returns the type ID of ::testing::Test. Always call this instead +// of GetTypeId< ::testing::Test>() to get the type ID of +// ::testing::Test, as the latter may give the wrong result due to a +// suspected linker bug when compiling Google Test as a Mac OS X +// framework. +GTEST_API_ TypeId GetTestTypeId(); + +// Defines the abstract factory interface that creates instances +// of a Test object. +class TestFactoryBase { + public: + virtual ~TestFactoryBase() {} + + // Creates a test instance to run. The instance is both created and destroyed + // within TestInfoImpl::Run() + virtual Test* CreateTest() = 0; + + protected: + TestFactoryBase() {} + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestFactoryBase); +}; + +// This class provides implementation of TeastFactoryBase interface. +// It is used in TEST and TEST_F macros. +template +class TestFactoryImpl : public TestFactoryBase { + public: + virtual Test* CreateTest() { return new TestClass; } +}; + +#if GTEST_OS_WINDOWS + +// Predicate-formatters for implementing the HRESULT checking macros +// {ASSERT|EXPECT}_HRESULT_{SUCCEEDED|FAILED} +// We pass a long instead of HRESULT to avoid causing an +// include dependency for the HRESULT type. +GTEST_API_ AssertionResult IsHRESULTSuccess(const char* expr, + long hr); // NOLINT +GTEST_API_ AssertionResult IsHRESULTFailure(const char* expr, + long hr); // NOLINT + +#endif // GTEST_OS_WINDOWS + +// Types of SetUpTestCase() and TearDownTestCase() functions. +typedef void (*SetUpTestCaseFunc)(); +typedef void (*TearDownTestCaseFunc)(); + +struct CodeLocation { + CodeLocation(const string& a_file, int a_line) : file(a_file), line(a_line) {} + + string file; + int line; +}; + +// Creates a new TestInfo object and registers it with Google Test; +// returns the created object. +// +// Arguments: +// +// test_case_name: name of the test case +// name: name of the test +// type_param the name of the test's type parameter, or NULL if +// this is not a typed or a type-parameterized test. +// value_param text representation of the test's value parameter, +// or NULL if this is not a type-parameterized test. +// code_location: code location where the test is defined +// fixture_class_id: ID of the test fixture class +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +// factory: pointer to the factory that creates a test object. +// The newly created TestInfo instance will assume +// ownership of the factory object. +GTEST_API_ TestInfo* MakeAndRegisterTestInfo( + const char* test_case_name, + const char* name, + const char* type_param, + const char* value_param, + CodeLocation code_location, + TypeId fixture_class_id, + SetUpTestCaseFunc set_up_tc, + TearDownTestCaseFunc tear_down_tc, + TestFactoryBase* factory); + +// If *pstr starts with the given prefix, modifies *pstr to be right +// past the prefix and returns true; otherwise leaves *pstr unchanged +// and returns false. None of pstr, *pstr, and prefix can be NULL. +GTEST_API_ bool SkipPrefix(const char* prefix, const char** pstr); + +#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// State of the definition of a type-parameterized test case. +class GTEST_API_ TypedTestCasePState { + public: + TypedTestCasePState() : registered_(false) {} + + // Adds the given test name to defined_test_names_ and return true + // if the test case hasn't been registered; otherwise aborts the + // program. + bool AddTestName(const char* file, int line, const char* case_name, + const char* test_name) { + if (registered_) { + fprintf(stderr, "%s Test %s must be defined before " + "REGISTER_TYPED_TEST_CASE_P(%s, ...).\n", + FormatFileLocation(file, line).c_str(), test_name, case_name); + fflush(stderr); + posix::Abort(); + } + registered_tests_.insert( + ::std::make_pair(test_name, CodeLocation(file, line))); + return true; + } + + bool TestExists(const std::string& test_name) const { + return registered_tests_.count(test_name) > 0; + } + + const CodeLocation& GetCodeLocation(const std::string& test_name) const { + RegisteredTestsMap::const_iterator it = registered_tests_.find(test_name); + GTEST_CHECK_(it != registered_tests_.end()); + return it->second; + } + + // Verifies that registered_tests match the test names in + // defined_test_names_; returns registered_tests if successful, or + // aborts the program otherwise. + const char* VerifyRegisteredTestNames( + const char* file, int line, const char* registered_tests); + + private: + typedef ::std::map RegisteredTestsMap; + + bool registered_; + RegisteredTestsMap registered_tests_; +}; + +// Skips to the first non-space char after the first comma in 'str'; +// returns NULL if no comma is found in 'str'. +inline const char* SkipComma(const char* str) { + const char* comma = strchr(str, ','); + if (comma == NULL) { + return NULL; + } + while (IsSpace(*(++comma))) {} + return comma; +} + +// Returns the prefix of 'str' before the first comma in it; returns +// the entire string if it contains no comma. +inline std::string GetPrefixUntilComma(const char* str) { + const char* comma = strchr(str, ','); + return comma == NULL ? str : std::string(str, comma); +} + +// Splits a given string on a given delimiter, populating a given +// vector with the fields. +void SplitString(const ::std::string& str, char delimiter, + ::std::vector< ::std::string>* dest); + +// TypeParameterizedTest::Register() +// registers a list of type-parameterized tests with Google Test. The +// return value is insignificant - we just need to return something +// such that we can call this function in a namespace scope. +// +// Implementation note: The GTEST_TEMPLATE_ macro declares a template +// template parameter. It's defined in gtest-type-util.h. +template +class TypeParameterizedTest { + public: + // 'index' is the index of the test in the type list 'Types' + // specified in INSTANTIATE_TYPED_TEST_CASE_P(Prefix, TestCase, + // Types). Valid values for 'index' are [0, N - 1] where N is the + // length of Types. + static bool Register(const char* prefix, + CodeLocation code_location, + const char* case_name, const char* test_names, + int index) { + typedef typename Types::Head Type; + typedef Fixture FixtureClass; + typedef typename GTEST_BIND_(TestSel, Type) TestClass; + + // First, registers the first type-parameterized test in the type + // list. + MakeAndRegisterTestInfo( + (std::string(prefix) + (prefix[0] == '\0' ? "" : "/") + case_name + "/" + + StreamableToString(index)).c_str(), + StripTrailingSpaces(GetPrefixUntilComma(test_names)).c_str(), + GetTypeName().c_str(), + NULL, // No value parameter. + code_location, + GetTypeId(), + TestClass::SetUpTestCase, + TestClass::TearDownTestCase, + new TestFactoryImpl); + + // Next, recurses (at compile time) with the tail of the type list. + return TypeParameterizedTest + ::Register(prefix, code_location, case_name, test_names, index + 1); + } +}; + +// The base case for the compile time recursion. +template +class TypeParameterizedTest { + public: + static bool Register(const char* /*prefix*/, CodeLocation, + const char* /*case_name*/, const char* /*test_names*/, + int /*index*/) { + return true; + } +}; + +// TypeParameterizedTestCase::Register() +// registers *all combinations* of 'Tests' and 'Types' with Google +// Test. The return value is insignificant - we just need to return +// something such that we can call this function in a namespace scope. +template +class TypeParameterizedTestCase { + public: + static bool Register(const char* prefix, CodeLocation code_location, + const TypedTestCasePState* state, + const char* case_name, const char* test_names) { + std::string test_name = StripTrailingSpaces( + GetPrefixUntilComma(test_names)); + if (!state->TestExists(test_name)) { + fprintf(stderr, "Failed to get code location for test %s.%s at %s.", + case_name, test_name.c_str(), + FormatFileLocation(code_location.file.c_str(), + code_location.line).c_str()); + fflush(stderr); + posix::Abort(); + } + const CodeLocation& test_location = state->GetCodeLocation(test_name); + + typedef typename Tests::Head Head; + + // First, register the first test in 'Test' for each type in 'Types'. + TypeParameterizedTest::Register( + prefix, test_location, case_name, test_names, 0); + + // Next, recurses (at compile time) with the tail of the test list. + return TypeParameterizedTestCase + ::Register(prefix, code_location, state, + case_name, SkipComma(test_names)); + } +}; + +// The base case for the compile time recursion. +template +class TypeParameterizedTestCase { + public: + static bool Register(const char* /*prefix*/, CodeLocation, + const TypedTestCasePState* /*state*/, + const char* /*case_name*/, const char* /*test_names*/) { + return true; + } +}; + +#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// Returns the current OS stack trace as an std::string. +// +// The maximum number of stack frames to be included is specified by +// the gtest_stack_trace_depth flag. The skip_count parameter +// specifies the number of top frames to be skipped, which doesn't +// count against the number of frames to be included. +// +// For example, if Foo() calls Bar(), which in turn calls +// GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in +// the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't. +GTEST_API_ std::string GetCurrentOsStackTraceExceptTop( + UnitTest* unit_test, int skip_count); + +// Helpers for suppressing warnings on unreachable code or constant +// condition. + +// Always returns true. +GTEST_API_ bool AlwaysTrue(); + +// Always returns false. +inline bool AlwaysFalse() { return !AlwaysTrue(); } + +// Helper for suppressing false warning from Clang on a const char* +// variable declared in a conditional expression always being NULL in +// the else branch. +struct GTEST_API_ ConstCharPtr { + ConstCharPtr(const char* str) : value(str) {} + operator bool() const { return true; } + const char* value; +}; + +// A simple Linear Congruential Generator for generating random +// numbers with a uniform distribution. Unlike rand() and srand(), it +// doesn't use global state (and therefore can't interfere with user +// code). Unlike rand_r(), it's portable. An LCG isn't very random, +// but it's good enough for our purposes. +class GTEST_API_ Random { + public: + static const UInt32 kMaxRange = 1u << 31; + + explicit Random(UInt32 seed) : state_(seed) {} + + void Reseed(UInt32 seed) { state_ = seed; } + + // Generates a random number from [0, range). Crashes if 'range' is + // 0 or greater than kMaxRange. + UInt32 Generate(UInt32 range); + + private: + UInt32 state_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(Random); +}; + +// Defining a variable of type CompileAssertTypesEqual will cause a +// compiler error iff T1 and T2 are different types. +template +struct CompileAssertTypesEqual; + +template +struct CompileAssertTypesEqual { +}; + +// Removes the reference from a type if it is a reference type, +// otherwise leaves it unchanged. This is the same as +// tr1::remove_reference, which is not widely available yet. +template +struct RemoveReference { typedef T type; }; // NOLINT +template +struct RemoveReference { typedef T type; }; // NOLINT + +// A handy wrapper around RemoveReference that works when the argument +// T depends on template parameters. +#define GTEST_REMOVE_REFERENCE_(T) \ + typename ::testing::internal::RemoveReference::type + +// Removes const from a type if it is a const type, otherwise leaves +// it unchanged. This is the same as tr1::remove_const, which is not +// widely available yet. +template +struct RemoveConst { typedef T type; }; // NOLINT +template +struct RemoveConst { typedef T type; }; // NOLINT + +// MSVC 8.0, Sun C++, and IBM XL C++ have a bug which causes the above +// definition to fail to remove the const in 'const int[3]' and 'const +// char[3][4]'. The following specialization works around the bug. +template +struct RemoveConst { + typedef typename RemoveConst::type type[N]; +}; + +#if defined(_MSC_VER) && _MSC_VER < 1400 +// This is the only specialization that allows VC++ 7.1 to remove const in +// 'const int[3] and 'const int[3][4]'. However, it causes trouble with GCC +// and thus needs to be conditionally compiled. +template +struct RemoveConst { + typedef typename RemoveConst::type type[N]; +}; +#endif + +// A handy wrapper around RemoveConst that works when the argument +// T depends on template parameters. +#define GTEST_REMOVE_CONST_(T) \ + typename ::testing::internal::RemoveConst::type + +// Turns const U&, U&, const U, and U all into U. +#define GTEST_REMOVE_REFERENCE_AND_CONST_(T) \ + GTEST_REMOVE_CONST_(GTEST_REMOVE_REFERENCE_(T)) + +// Adds reference to a type if it is not a reference type, +// otherwise leaves it unchanged. This is the same as +// tr1::add_reference, which is not widely available yet. +template +struct AddReference { typedef T& type; }; // NOLINT +template +struct AddReference { typedef T& type; }; // NOLINT + +// A handy wrapper around AddReference that works when the argument T +// depends on template parameters. +#define GTEST_ADD_REFERENCE_(T) \ + typename ::testing::internal::AddReference::type + +// Adds a reference to const on top of T as necessary. For example, +// it transforms +// +// char ==> const char& +// const char ==> const char& +// char& ==> const char& +// const char& ==> const char& +// +// The argument T must depend on some template parameters. +#define GTEST_REFERENCE_TO_CONST_(T) \ + GTEST_ADD_REFERENCE_(const GTEST_REMOVE_REFERENCE_(T)) + +// ImplicitlyConvertible::value is a compile-time bool +// constant that's true iff type From can be implicitly converted to +// type To. +template +class ImplicitlyConvertible { + private: + // We need the following helper functions only for their types. + // They have no implementations. + + // MakeFrom() is an expression whose type is From. We cannot simply + // use From(), as the type From may not have a public default + // constructor. + static typename AddReference::type MakeFrom(); + + // These two functions are overloaded. Given an expression + // Helper(x), the compiler will pick the first version if x can be + // implicitly converted to type To; otherwise it will pick the + // second version. + // + // The first version returns a value of size 1, and the second + // version returns a value of size 2. Therefore, by checking the + // size of Helper(x), which can be done at compile time, we can tell + // which version of Helper() is used, and hence whether x can be + // implicitly converted to type To. + static char Helper(To); + static char (&Helper(...))[2]; // NOLINT + + // We have to put the 'public' section after the 'private' section, + // or MSVC refuses to compile the code. + public: +#if defined(__BORLANDC__) + // C++Builder cannot use member overload resolution during template + // instantiation. The simplest workaround is to use its C++0x type traits + // functions (C++Builder 2009 and above only). + static const bool value = __is_convertible(From, To); +#else + // MSVC warns about implicitly converting from double to int for + // possible loss of data, so we need to temporarily disable the + // warning. + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4244) + static const bool value = + sizeof(Helper(ImplicitlyConvertible::MakeFrom())) == 1; + GTEST_DISABLE_MSC_WARNINGS_POP_() +#endif // __BORLANDC__ +}; +template +const bool ImplicitlyConvertible::value; + +// IsAProtocolMessage::value is a compile-time bool constant that's +// true iff T is type ProtocolMessage, proto2::Message, or a subclass +// of those. +template +struct IsAProtocolMessage + : public bool_constant< + ImplicitlyConvertible::value || + ImplicitlyConvertible::value> { +}; + +// When the compiler sees expression IsContainerTest(0), if C is an +// STL-style container class, the first overload of IsContainerTest +// will be viable (since both C::iterator* and C::const_iterator* are +// valid types and NULL can be implicitly converted to them). It will +// be picked over the second overload as 'int' is a perfect match for +// the type of argument 0. If C::iterator or C::const_iterator is not +// a valid type, the first overload is not viable, and the second +// overload will be picked. Therefore, we can determine whether C is +// a container class by checking the type of IsContainerTest(0). +// The value of the expression is insignificant. +// +// Note that we look for both C::iterator and C::const_iterator. The +// reason is that C++ injects the name of a class as a member of the +// class itself (e.g. you can refer to class iterator as either +// 'iterator' or 'iterator::iterator'). If we look for C::iterator +// only, for example, we would mistakenly think that a class named +// iterator is an STL container. +// +// Also note that the simpler approach of overloading +// IsContainerTest(typename C::const_iterator*) and +// IsContainerTest(...) doesn't work with Visual Age C++ and Sun C++. +typedef int IsContainer; +template +IsContainer IsContainerTest(int /* dummy */, + typename C::iterator* /* it */ = NULL, + typename C::const_iterator* /* const_it */ = NULL) { + return 0; +} + +typedef char IsNotContainer; +template +IsNotContainer IsContainerTest(long /* dummy */) { return '\0'; } + +// EnableIf::type is void when 'Cond' is true, and +// undefined when 'Cond' is false. To use SFINAE to make a function +// overload only apply when a particular expression is true, add +// "typename EnableIf::type* = 0" as the last parameter. +template struct EnableIf; +template<> struct EnableIf { typedef void type; }; // NOLINT + +// Utilities for native arrays. + +// ArrayEq() compares two k-dimensional native arrays using the +// elements' operator==, where k can be any integer >= 0. When k is +// 0, ArrayEq() degenerates into comparing a single pair of values. + +template +bool ArrayEq(const T* lhs, size_t size, const U* rhs); + +// This generic version is used when k is 0. +template +inline bool ArrayEq(const T& lhs, const U& rhs) { return lhs == rhs; } + +// This overload is used when k >= 1. +template +inline bool ArrayEq(const T(&lhs)[N], const U(&rhs)[N]) { + return internal::ArrayEq(lhs, N, rhs); +} + +// This helper reduces code bloat. If we instead put its logic inside +// the previous ArrayEq() function, arrays with different sizes would +// lead to different copies of the template code. +template +bool ArrayEq(const T* lhs, size_t size, const U* rhs) { + for (size_t i = 0; i != size; i++) { + if (!internal::ArrayEq(lhs[i], rhs[i])) + return false; + } + return true; +} + +// Finds the first element in the iterator range [begin, end) that +// equals elem. Element may be a native array type itself. +template +Iter ArrayAwareFind(Iter begin, Iter end, const Element& elem) { + for (Iter it = begin; it != end; ++it) { + if (internal::ArrayEq(*it, elem)) + return it; + } + return end; +} + +// CopyArray() copies a k-dimensional native array using the elements' +// operator=, where k can be any integer >= 0. When k is 0, +// CopyArray() degenerates into copying a single value. + +template +void CopyArray(const T* from, size_t size, U* to); + +// This generic version is used when k is 0. +template +inline void CopyArray(const T& from, U* to) { *to = from; } + +// This overload is used when k >= 1. +template +inline void CopyArray(const T(&from)[N], U(*to)[N]) { + internal::CopyArray(from, N, *to); +} + +// This helper reduces code bloat. If we instead put its logic inside +// the previous CopyArray() function, arrays with different sizes +// would lead to different copies of the template code. +template +void CopyArray(const T* from, size_t size, U* to) { + for (size_t i = 0; i != size; i++) { + internal::CopyArray(from[i], to + i); + } +} + +// The relation between an NativeArray object (see below) and the +// native array it represents. +// We use 2 different structs to allow non-copyable types to be used, as long +// as RelationToSourceReference() is passed. +struct RelationToSourceReference {}; +struct RelationToSourceCopy {}; + +// Adapts a native array to a read-only STL-style container. Instead +// of the complete STL container concept, this adaptor only implements +// members useful for Google Mock's container matchers. New members +// should be added as needed. To simplify the implementation, we only +// support Element being a raw type (i.e. having no top-level const or +// reference modifier). It's the client's responsibility to satisfy +// this requirement. Element can be an array type itself (hence +// multi-dimensional arrays are supported). +template +class NativeArray { + public: + // STL-style container typedefs. + typedef Element value_type; + typedef Element* iterator; + typedef const Element* const_iterator; + + // Constructs from a native array. References the source. + NativeArray(const Element* array, size_t count, RelationToSourceReference) { + InitRef(array, count); + } + + // Constructs from a native array. Copies the source. + NativeArray(const Element* array, size_t count, RelationToSourceCopy) { + InitCopy(array, count); + } + + // Copy constructor. + NativeArray(const NativeArray& rhs) { + (this->*rhs.clone_)(rhs.array_, rhs.size_); + } + + ~NativeArray() { + if (clone_ != &NativeArray::InitRef) + delete[] array_; + } + + // STL-style container methods. + size_t size() const { return size_; } + const_iterator begin() const { return array_; } + const_iterator end() const { return array_ + size_; } + bool operator==(const NativeArray& rhs) const { + return size() == rhs.size() && + ArrayEq(begin(), size(), rhs.begin()); + } + + private: + enum { + kCheckTypeIsNotConstOrAReference = StaticAssertTypeEqHelper< + Element, GTEST_REMOVE_REFERENCE_AND_CONST_(Element)>::value, + }; + + // Initializes this object with a copy of the input. + void InitCopy(const Element* array, size_t a_size) { + Element* const copy = new Element[a_size]; + CopyArray(array, a_size, copy); + array_ = copy; + size_ = a_size; + clone_ = &NativeArray::InitCopy; + } + + // Initializes this object with a reference of the input. + void InitRef(const Element* array, size_t a_size) { + array_ = array; + size_ = a_size; + clone_ = &NativeArray::InitRef; + } + + const Element* array_; + size_t size_; + void (NativeArray::*clone_)(const Element*, size_t); + + GTEST_DISALLOW_ASSIGN_(NativeArray); +}; + +} // namespace internal +} // namespace testing + +#define GTEST_MESSAGE_AT_(file, line, message, result_type) \ + ::testing::internal::AssertHelper(result_type, file, line, message) \ + = ::testing::Message() + +#define GTEST_MESSAGE_(message, result_type) \ + GTEST_MESSAGE_AT_(__FILE__, __LINE__, message, result_type) + +#define GTEST_FATAL_FAILURE_(message) \ + return GTEST_MESSAGE_(message, ::testing::TestPartResult::kFatalFailure) + +#define GTEST_NONFATAL_FAILURE_(message) \ + GTEST_MESSAGE_(message, ::testing::TestPartResult::kNonFatalFailure) + +#define GTEST_SUCCESS_(message) \ + GTEST_MESSAGE_(message, ::testing::TestPartResult::kSuccess) + +// Suppresses MSVC warnings 4072 (unreachable code) for the code following +// statement if it returns or throws (or doesn't return or throw in some +// situations). +#define GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) \ + if (::testing::internal::AlwaysTrue()) { statement; } + +#define GTEST_TEST_THROW_(statement, expected_exception, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::ConstCharPtr gtest_msg = "") { \ + bool gtest_caught_expected = false; \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (expected_exception const&) { \ + gtest_caught_expected = true; \ + } \ + catch (...) { \ + gtest_msg.value = \ + "Expected: " #statement " throws an exception of type " \ + #expected_exception ".\n Actual: it throws a different type."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } \ + if (!gtest_caught_expected) { \ + gtest_msg.value = \ + "Expected: " #statement " throws an exception of type " \ + #expected_exception ".\n Actual: it throws nothing."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__): \ + fail(gtest_msg.value) + +#define GTEST_TEST_NO_THROW_(statement, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (...) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testnothrow_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testnothrow_, __LINE__): \ + fail("Expected: " #statement " doesn't throw an exception.\n" \ + " Actual: it throws.") + +#define GTEST_TEST_ANY_THROW_(statement, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + bool gtest_caught_any = false; \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (...) { \ + gtest_caught_any = true; \ + } \ + if (!gtest_caught_any) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testanythrow_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testanythrow_, __LINE__): \ + fail("Expected: " #statement " throws an exception.\n" \ + " Actual: it doesn't.") + + +// Implements Boolean test assertions such as EXPECT_TRUE. expression can be +// either a boolean expression or an AssertionResult. text is a textual +// represenation of expression as it was passed into the EXPECT_TRUE. +#define GTEST_TEST_BOOLEAN_(expression, text, actual, expected, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (const ::testing::AssertionResult gtest_ar_ = \ + ::testing::AssertionResult(expression)) \ + ; \ + else \ + fail(::testing::internal::GetBoolAssertionFailureMessage(\ + gtest_ar_, text, #actual, #expected).c_str()) + +#define GTEST_TEST_NO_FATAL_FAILURE_(statement, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + ::testing::internal::HasNewFatalFailureHelper gtest_fatal_failure_checker; \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + if (gtest_fatal_failure_checker.has_new_fatal_failure()) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testnofatal_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testnofatal_, __LINE__): \ + fail("Expected: " #statement " doesn't generate new fatal " \ + "failures in the current thread.\n" \ + " Actual: it does.") + +// Expands to the name of the class that implements the given test. +#define GTEST_TEST_CLASS_NAME_(test_case_name, test_name) \ + test_case_name##_##test_name##_Test + +// Helper macro for defining tests. +#define GTEST_TEST_(test_case_name, test_name, parent_class, parent_id)\ +class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) : public parent_class {\ + public:\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {}\ + private:\ + virtual void TestBody();\ + static ::testing::TestInfo* const test_info_ GTEST_ATTRIBUTE_UNUSED_;\ + GTEST_DISALLOW_COPY_AND_ASSIGN_(\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name));\ +};\ +\ +::testing::TestInfo* const GTEST_TEST_CLASS_NAME_(test_case_name, test_name)\ + ::test_info_ =\ + ::testing::internal::MakeAndRegisterTestInfo(\ + #test_case_name, #test_name, NULL, NULL, \ + ::testing::internal::CodeLocation(__FILE__, __LINE__), \ + (parent_id), \ + parent_class::SetUpTestCase, \ + parent_class::TearDownTestCase, \ + new ::testing::internal::TestFactoryImpl<\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)>);\ +void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ + diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-linked_ptr.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-linked_ptr.h new file mode 100644 index 0000000000..3602942217 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-linked_ptr.h @@ -0,0 +1,243 @@ +// Copyright 2003 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: Dan Egnor (egnor@google.com) +// +// A "smart" pointer type with reference tracking. Every pointer to a +// particular object is kept on a circular linked list. When the last pointer +// to an object is destroyed or reassigned, the object is deleted. +// +// Used properly, this deletes the object when the last reference goes away. +// There are several caveats: +// - Like all reference counting schemes, cycles lead to leaks. +// - Each smart pointer is actually two pointers (8 bytes instead of 4). +// - Every time a pointer is assigned, the entire list of pointers to that +// object is traversed. This class is therefore NOT SUITABLE when there +// will often be more than two or three pointers to a particular object. +// - References are only tracked as long as linked_ptr<> objects are copied. +// If a linked_ptr<> is converted to a raw pointer and back, BAD THINGS +// will happen (double deletion). +// +// A good use of this class is storing object references in STL containers. +// You can safely put linked_ptr<> in a vector<>. +// Other uses may not be as good. +// +// Note: If you use an incomplete type with linked_ptr<>, the class +// *containing* linked_ptr<> must have a constructor and destructor (even +// if they do nothing!). +// +// Bill Gibbons suggested we use something like this. +// +// Thread Safety: +// Unlike other linked_ptr implementations, in this implementation +// a linked_ptr object is thread-safe in the sense that: +// - it's safe to copy linked_ptr objects concurrently, +// - it's safe to copy *from* a linked_ptr and read its underlying +// raw pointer (e.g. via get()) concurrently, and +// - it's safe to write to two linked_ptrs that point to the same +// shared object concurrently. +// TODO(wan@google.com): rename this to safe_linked_ptr to avoid +// confusion with normal linked_ptr. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_LINKED_PTR_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_LINKED_PTR_H_ + +#include +#include + +#include "gtest/internal/gtest-port.h" + +namespace testing { +namespace internal { + +// Protects copying of all linked_ptr objects. +GTEST_API_ GTEST_DECLARE_STATIC_MUTEX_(g_linked_ptr_mutex); + +// This is used internally by all instances of linked_ptr<>. It needs to be +// a non-template class because different types of linked_ptr<> can refer to +// the same object (linked_ptr(obj) vs linked_ptr(obj)). +// So, it needs to be possible for different types of linked_ptr to participate +// in the same circular linked list, so we need a single class type here. +// +// DO NOT USE THIS CLASS DIRECTLY YOURSELF. Use linked_ptr. +class linked_ptr_internal { + public: + // Create a new circle that includes only this instance. + void join_new() { + next_ = this; + } + + // Many linked_ptr operations may change p.link_ for some linked_ptr + // variable p in the same circle as this object. Therefore we need + // to prevent two such operations from occurring concurrently. + // + // Note that different types of linked_ptr objects can coexist in a + // circle (e.g. linked_ptr, linked_ptr, and + // linked_ptr). Therefore we must use a single mutex to + // protect all linked_ptr objects. This can create serious + // contention in production code, but is acceptable in a testing + // framework. + + // Join an existing circle. + void join(linked_ptr_internal const* ptr) + GTEST_LOCK_EXCLUDED_(g_linked_ptr_mutex) { + MutexLock lock(&g_linked_ptr_mutex); + + linked_ptr_internal const* p = ptr; + while (p->next_ != ptr) { + assert(p->next_ != this && + "Trying to join() a linked ring we are already in. " + "Is GMock thread safety enabled?"); + p = p->next_; + } + p->next_ = this; + next_ = ptr; + } + + // Leave whatever circle we're part of. Returns true if we were the + // last member of the circle. Once this is done, you can join() another. + bool depart() + GTEST_LOCK_EXCLUDED_(g_linked_ptr_mutex) { + MutexLock lock(&g_linked_ptr_mutex); + + if (next_ == this) return true; + linked_ptr_internal const* p = next_; + while (p->next_ != this) { + assert(p->next_ != next_ && + "Trying to depart() a linked ring we are not in. " + "Is GMock thread safety enabled?"); + p = p->next_; + } + p->next_ = next_; + return false; + } + + private: + mutable linked_ptr_internal const* next_; +}; + +template +class linked_ptr { + public: + typedef T element_type; + + // Take over ownership of a raw pointer. This should happen as soon as + // possible after the object is created. + explicit linked_ptr(T* ptr = NULL) { capture(ptr); } + ~linked_ptr() { depart(); } + + // Copy an existing linked_ptr<>, adding ourselves to the list of references. + template linked_ptr(linked_ptr const& ptr) { copy(&ptr); } + linked_ptr(linked_ptr const& ptr) { // NOLINT + assert(&ptr != this); + copy(&ptr); + } + + // Assignment releases the old value and acquires the new. + template linked_ptr& operator=(linked_ptr const& ptr) { + depart(); + copy(&ptr); + return *this; + } + + linked_ptr& operator=(linked_ptr const& ptr) { + if (&ptr != this) { + depart(); + copy(&ptr); + } + return *this; + } + + // Smart pointer members. + void reset(T* ptr = NULL) { + depart(); + capture(ptr); + } + T* get() const { return value_; } + T* operator->() const { return value_; } + T& operator*() const { return *value_; } + + bool operator==(T* p) const { return value_ == p; } + bool operator!=(T* p) const { return value_ != p; } + template + bool operator==(linked_ptr const& ptr) const { + return value_ == ptr.get(); + } + template + bool operator!=(linked_ptr const& ptr) const { + return value_ != ptr.get(); + } + + private: + template + friend class linked_ptr; + + T* value_; + linked_ptr_internal link_; + + void depart() { + if (link_.depart()) delete value_; + } + + void capture(T* ptr) { + value_ = ptr; + link_.join_new(); + } + + template void copy(linked_ptr const* ptr) { + value_ = ptr->get(); + if (value_) + link_.join(&ptr->link_); + else + link_.join_new(); + } +}; + +template inline +bool operator==(T* ptr, const linked_ptr& x) { + return ptr == x.get(); +} + +template inline +bool operator!=(T* ptr, const linked_ptr& x) { + return ptr != x.get(); +} + +// A function to convert T* into linked_ptr +// Doing e.g. make_linked_ptr(new FooBarBaz(arg)) is a shorter notation +// for linked_ptr >(new FooBarBaz(arg)) +template +linked_ptr make_linked_ptr(T* ptr) { + return linked_ptr(ptr); +} + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_LINKED_PTR_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-param-util-generated.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-param-util-generated.h new file mode 100644 index 0000000000..4d1d81d20f --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-param-util-generated.h @@ -0,0 +1,5146 @@ +// This file was GENERATED by command: +// pump.py gtest-param-util-generated.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: vladl@google.com (Vlad Losev) + +// Type and function utilities for implementing parameterized tests. +// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// Currently Google Test supports at most 50 arguments in Values, +// and at most 10 arguments in Combine. Please contact +// googletestframework@googlegroups.com if you need more. +// Please note that the number of arguments to Combine is limited +// by the maximum arity of the implementation of tuple which is +// currently set at 10. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. +#include "gtest/internal/gtest-param-util.h" +#include "gtest/internal/gtest-port.h" + +#if GTEST_HAS_PARAM_TEST + +namespace testing { + +// Forward declarations of ValuesIn(), which is implemented in +// include/gtest/gtest-param-test.h. +template +internal::ParamGenerator< + typename ::testing::internal::IteratorTraits::value_type> +ValuesIn(ForwardIterator begin, ForwardIterator end); + +template +internal::ParamGenerator ValuesIn(const T (&array)[N]); + +template +internal::ParamGenerator ValuesIn( + const Container& container); + +namespace internal { + +// Used in the Values() function to provide polymorphic capabilities. +template +class ValueArray1 { + public: + explicit ValueArray1(T1 v1) : v1_(v1) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray1& other); + + const T1 v1_; +}; + +template +class ValueArray2 { + public: + ValueArray2(T1 v1, T2 v2) : v1_(v1), v2_(v2) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray2& other); + + const T1 v1_; + const T2 v2_; +}; + +template +class ValueArray3 { + public: + ValueArray3(T1 v1, T2 v2, T3 v3) : v1_(v1), v2_(v2), v3_(v3) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray3& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; +}; + +template +class ValueArray4 { + public: + ValueArray4(T1 v1, T2 v2, T3 v3, T4 v4) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray4& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; +}; + +template +class ValueArray5 { + public: + ValueArray5(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray5& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; +}; + +template +class ValueArray6 { + public: + ValueArray6(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray6& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; +}; + +template +class ValueArray7 { + public: + ValueArray7(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray7& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; +}; + +template +class ValueArray8 { + public: + ValueArray8(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray8& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; +}; + +template +class ValueArray9 { + public: + ValueArray9(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray9& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; +}; + +template +class ValueArray10 { + public: + ValueArray10(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray10& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; +}; + +template +class ValueArray11 { + public: + ValueArray11(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray11& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; +}; + +template +class ValueArray12 { + public: + ValueArray12(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray12& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; +}; + +template +class ValueArray13 { + public: + ValueArray13(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray13& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; +}; + +template +class ValueArray14 { + public: + ValueArray14(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray14& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; +}; + +template +class ValueArray15 { + public: + ValueArray15(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray15& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; +}; + +template +class ValueArray16 { + public: + ValueArray16(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray16& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; +}; + +template +class ValueArray17 { + public: + ValueArray17(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, + T17 v17) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray17& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; +}; + +template +class ValueArray18 { + public: + ValueArray18(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray18& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; +}; + +template +class ValueArray19 { + public: + ValueArray19(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray19& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; +}; + +template +class ValueArray20 { + public: + ValueArray20(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray20& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; +}; + +template +class ValueArray21 { + public: + ValueArray21(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray21& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; +}; + +template +class ValueArray22 { + public: + ValueArray22(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray22& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; +}; + +template +class ValueArray23 { + public: + ValueArray23(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray23& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; +}; + +template +class ValueArray24 { + public: + ValueArray24(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray24& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; +}; + +template +class ValueArray25 { + public: + ValueArray25(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, + T25 v25) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray25& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; +}; + +template +class ValueArray26 { + public: + ValueArray26(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray26& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; +}; + +template +class ValueArray27 { + public: + ValueArray27(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), + v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), + v26_(v26), v27_(v27) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray27& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; +}; + +template +class ValueArray28 { + public: + ValueArray28(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), + v25_(v25), v26_(v26), v27_(v27), v28_(v28) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray28& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; +}; + +template +class ValueArray29 { + public: + ValueArray29(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), + v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray29& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; +}; + +template +class ValueArray30 { + public: + ValueArray30(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray30& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; +}; + +template +class ValueArray31 { + public: + ValueArray31(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray31& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; +}; + +template +class ValueArray32 { + public: + ValueArray32(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), + v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray32& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; +}; + +template +class ValueArray33 { + public: + ValueArray33(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, + T33 v33) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray33& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; +}; + +template +class ValueArray34 { + public: + ValueArray34(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray34& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; +}; + +template +class ValueArray35 { + public: + ValueArray35(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), + v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), + v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), + v32_(v32), v33_(v33), v34_(v34), v35_(v35) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray35& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; +}; + +template +class ValueArray36 { + public: + ValueArray36(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), + v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), + v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), v36_(v36) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray36& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; +}; + +template +class ValueArray37 { + public: + ValueArray37(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), + v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), + v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), + v36_(v36), v37_(v37) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray37& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; +}; + +template +class ValueArray38 { + public: + ValueArray38(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray38& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; +}; + +template +class ValueArray39 { + public: + ValueArray39(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray39& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; +}; + +template +class ValueArray40 { + public: + ValueArray40(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), + v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), + v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), + v40_(v40) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray40& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; +}; + +template +class ValueArray41 { + public: + ValueArray41(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, + T41 v41) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray41& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; +}; + +template +class ValueArray42 { + public: + ValueArray42(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41), v42_(v42) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray42& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; +}; + +template +class ValueArray43 { + public: + ValueArray43(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), + v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), + v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), + v32_(v32), v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), + v38_(v38), v39_(v39), v40_(v40), v41_(v41), v42_(v42), v43_(v43) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray43& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; +}; + +template +class ValueArray44 { + public: + ValueArray44(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), + v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), + v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), v36_(v36), + v37_(v37), v38_(v38), v39_(v39), v40_(v40), v41_(v41), v42_(v42), + v43_(v43), v44_(v44) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray44& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; +}; + +template +class ValueArray45 { + public: + ValueArray45(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), + v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), + v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), + v36_(v36), v37_(v37), v38_(v38), v39_(v39), v40_(v40), v41_(v41), + v42_(v42), v43_(v43), v44_(v44), v45_(v45) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray45& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; +}; + +template +class ValueArray46 { + public: + ValueArray46(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), v40_(v40), + v41_(v41), v42_(v42), v43_(v43), v44_(v44), v45_(v45), v46_(v46) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray46& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; +}; + +template +class ValueArray47 { + public: + ValueArray47(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), v40_(v40), + v41_(v41), v42_(v42), v43_(v43), v44_(v44), v45_(v45), v46_(v46), + v47_(v47) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_), static_cast(v47_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray47& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; +}; + +template +class ValueArray48 { + public: + ValueArray48(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, T48 v48) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), + v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), + v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), + v40_(v40), v41_(v41), v42_(v42), v43_(v43), v44_(v44), v45_(v45), + v46_(v46), v47_(v47), v48_(v48) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_), static_cast(v47_), + static_cast(v48_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray48& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; + const T48 v48_; +}; + +template +class ValueArray49 { + public: + ValueArray49(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, T48 v48, + T49 v49) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41), v42_(v42), v43_(v43), v44_(v44), + v45_(v45), v46_(v46), v47_(v47), v48_(v48), v49_(v49) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_), static_cast(v47_), + static_cast(v48_), static_cast(v49_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray49& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; + const T48 v48_; + const T49 v49_; +}; + +template +class ValueArray50 { + public: + ValueArray50(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, T48 v48, T49 v49, + T50 v50) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41), v42_(v42), v43_(v43), v44_(v44), + v45_(v45), v46_(v46), v47_(v47), v48_(v48), v49_(v49), v50_(v50) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_), static_cast(v47_), + static_cast(v48_), static_cast(v49_), static_cast(v50_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray50& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; + const T48 v48_; + const T49 v49_; + const T50 v50_; +}; + +# if GTEST_HAS_COMBINE +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Generates values from the Cartesian product of values produced +// by the argument generators. +// +template +class CartesianProductGenerator2 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator2(const ParamGenerator& g1, + const ParamGenerator& g2) + : g1_(g1), g2_(g2) {} + virtual ~CartesianProductGenerator2() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current2_; + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + ParamType current_value_; + }; // class CartesianProductGenerator2::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator2& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; +}; // class CartesianProductGenerator2 + + +template +class CartesianProductGenerator3 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator3(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3) + : g1_(g1), g2_(g2), g3_(g3) {} + virtual ~CartesianProductGenerator3() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current3_; + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + ParamType current_value_; + }; // class CartesianProductGenerator3::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator3& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; +}; // class CartesianProductGenerator3 + + +template +class CartesianProductGenerator4 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator4(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4) {} + virtual ~CartesianProductGenerator4() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current4_; + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + ParamType current_value_; + }; // class CartesianProductGenerator4::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator4& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; +}; // class CartesianProductGenerator4 + + +template +class CartesianProductGenerator5 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator5(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5) {} + virtual ~CartesianProductGenerator5() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current5_; + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + ParamType current_value_; + }; // class CartesianProductGenerator5::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator5& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; +}; // class CartesianProductGenerator5 + + +template +class CartesianProductGenerator6 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator6(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6) {} + virtual ~CartesianProductGenerator6() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current6_; + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + ParamType current_value_; + }; // class CartesianProductGenerator6::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator6& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; +}; // class CartesianProductGenerator6 + + +template +class CartesianProductGenerator7 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator7(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7) {} + virtual ~CartesianProductGenerator7() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current7_; + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + ParamType current_value_; + }; // class CartesianProductGenerator7::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator7& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; +}; // class CartesianProductGenerator7 + + +template +class CartesianProductGenerator8 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator8(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7, + const ParamGenerator& g8) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), + g8_(g8) {} + virtual ~CartesianProductGenerator8() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin(), g8_, g8_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end(), g8_, + g8_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7, + const ParamGenerator& g8, + const typename ParamGenerator::iterator& current8) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7), + begin8_(g8.begin()), end8_(g8.end()), current8_(current8) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current8_; + if (current8_ == end8_) { + current8_ = begin8_; + ++current7_; + } + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_ && + current8_ == typed_other->current8_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_), + begin8_(other.begin8_), + end8_(other.end8_), + current8_(other.current8_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_, *current8_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_ || + current8_ == end8_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + const typename ParamGenerator::iterator begin8_; + const typename ParamGenerator::iterator end8_; + typename ParamGenerator::iterator current8_; + ParamType current_value_; + }; // class CartesianProductGenerator8::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator8& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; + const ParamGenerator g8_; +}; // class CartesianProductGenerator8 + + +template +class CartesianProductGenerator9 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator9(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7, + const ParamGenerator& g8, const ParamGenerator& g9) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9) {} + virtual ~CartesianProductGenerator9() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin(), g8_, g8_.begin(), g9_, g9_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end(), g8_, + g8_.end(), g9_, g9_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7, + const ParamGenerator& g8, + const typename ParamGenerator::iterator& current8, + const ParamGenerator& g9, + const typename ParamGenerator::iterator& current9) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7), + begin8_(g8.begin()), end8_(g8.end()), current8_(current8), + begin9_(g9.begin()), end9_(g9.end()), current9_(current9) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current9_; + if (current9_ == end9_) { + current9_ = begin9_; + ++current8_; + } + if (current8_ == end8_) { + current8_ = begin8_; + ++current7_; + } + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_ && + current8_ == typed_other->current8_ && + current9_ == typed_other->current9_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_), + begin8_(other.begin8_), + end8_(other.end8_), + current8_(other.current8_), + begin9_(other.begin9_), + end9_(other.end9_), + current9_(other.current9_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_, *current8_, + *current9_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_ || + current8_ == end8_ || + current9_ == end9_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + const typename ParamGenerator::iterator begin8_; + const typename ParamGenerator::iterator end8_; + typename ParamGenerator::iterator current8_; + const typename ParamGenerator::iterator begin9_; + const typename ParamGenerator::iterator end9_; + typename ParamGenerator::iterator current9_; + ParamType current_value_; + }; // class CartesianProductGenerator9::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator9& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; + const ParamGenerator g8_; + const ParamGenerator g9_; +}; // class CartesianProductGenerator9 + + +template +class CartesianProductGenerator10 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator10(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7, + const ParamGenerator& g8, const ParamGenerator& g9, + const ParamGenerator& g10) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9), g10_(g10) {} + virtual ~CartesianProductGenerator10() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin(), g8_, g8_.begin(), g9_, g9_.begin(), g10_, g10_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end(), g8_, + g8_.end(), g9_, g9_.end(), g10_, g10_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7, + const ParamGenerator& g8, + const typename ParamGenerator::iterator& current8, + const ParamGenerator& g9, + const typename ParamGenerator::iterator& current9, + const ParamGenerator& g10, + const typename ParamGenerator::iterator& current10) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7), + begin8_(g8.begin()), end8_(g8.end()), current8_(current8), + begin9_(g9.begin()), end9_(g9.end()), current9_(current9), + begin10_(g10.begin()), end10_(g10.end()), current10_(current10) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current10_; + if (current10_ == end10_) { + current10_ = begin10_; + ++current9_; + } + if (current9_ == end9_) { + current9_ = begin9_; + ++current8_; + } + if (current8_ == end8_) { + current8_ = begin8_; + ++current7_; + } + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_ && + current8_ == typed_other->current8_ && + current9_ == typed_other->current9_ && + current10_ == typed_other->current10_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_), + begin8_(other.begin8_), + end8_(other.end8_), + current8_(other.current8_), + begin9_(other.begin9_), + end9_(other.end9_), + current9_(other.current9_), + begin10_(other.begin10_), + end10_(other.end10_), + current10_(other.current10_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_, *current8_, + *current9_, *current10_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_ || + current8_ == end8_ || + current9_ == end9_ || + current10_ == end10_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + const typename ParamGenerator::iterator begin8_; + const typename ParamGenerator::iterator end8_; + typename ParamGenerator::iterator current8_; + const typename ParamGenerator::iterator begin9_; + const typename ParamGenerator::iterator end9_; + typename ParamGenerator::iterator current9_; + const typename ParamGenerator::iterator begin10_; + const typename ParamGenerator::iterator end10_; + typename ParamGenerator::iterator current10_; + ParamType current_value_; + }; // class CartesianProductGenerator10::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator10& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; + const ParamGenerator g8_; + const ParamGenerator g9_; + const ParamGenerator g10_; +}; // class CartesianProductGenerator10 + + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Helper classes providing Combine() with polymorphic features. They allow +// casting CartesianProductGeneratorN to ParamGenerator if T is +// convertible to U. +// +template +class CartesianProductHolder2 { + public: +CartesianProductHolder2(const Generator1& g1, const Generator2& g2) + : g1_(g1), g2_(g2) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator2( + static_cast >(g1_), + static_cast >(g2_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder2& other); + + const Generator1 g1_; + const Generator2 g2_; +}; // class CartesianProductHolder2 + +template +class CartesianProductHolder3 { + public: +CartesianProductHolder3(const Generator1& g1, const Generator2& g2, + const Generator3& g3) + : g1_(g1), g2_(g2), g3_(g3) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator3( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder3& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; +}; // class CartesianProductHolder3 + +template +class CartesianProductHolder4 { + public: +CartesianProductHolder4(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator4( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder4& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; +}; // class CartesianProductHolder4 + +template +class CartesianProductHolder5 { + public: +CartesianProductHolder5(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator5( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder5& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; +}; // class CartesianProductHolder5 + +template +class CartesianProductHolder6 { + public: +CartesianProductHolder6(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator6( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder6& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; +}; // class CartesianProductHolder6 + +template +class CartesianProductHolder7 { + public: +CartesianProductHolder7(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator7( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder7& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; +}; // class CartesianProductHolder7 + +template +class CartesianProductHolder8 { + public: +CartesianProductHolder8(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7, const Generator8& g8) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), + g8_(g8) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator8( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_), + static_cast >(g8_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder8& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; + const Generator8 g8_; +}; // class CartesianProductHolder8 + +template +class CartesianProductHolder9 { + public: +CartesianProductHolder9(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7, const Generator8& g8, + const Generator9& g9) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator9( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_), + static_cast >(g8_), + static_cast >(g9_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder9& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; + const Generator8 g8_; + const Generator9 g9_; +}; // class CartesianProductHolder9 + +template +class CartesianProductHolder10 { + public: +CartesianProductHolder10(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7, const Generator8& g8, + const Generator9& g9, const Generator10& g10) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9), g10_(g10) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator10( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_), + static_cast >(g8_), + static_cast >(g9_), + static_cast >(g10_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder10& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; + const Generator8 g8_; + const Generator9 g9_; + const Generator10 g10_; +}; // class CartesianProductHolder10 + +# endif // GTEST_HAS_COMBINE + +} // namespace internal +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-param-util-generated.h.pump b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-param-util-generated.h.pump new file mode 100644 index 0000000000..5c7c47af0b --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-param-util-generated.h.pump @@ -0,0 +1,286 @@ +$$ -*- mode: c++; -*- +$var n = 50 $$ Maximum length of Values arguments we want to support. +$var maxtuple = 10 $$ Maximum number of Combine arguments we want to support. +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: vladl@google.com (Vlad Losev) + +// Type and function utilities for implementing parameterized tests. +// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// Currently Google Test supports at most $n arguments in Values, +// and at most $maxtuple arguments in Combine. Please contact +// googletestframework@googlegroups.com if you need more. +// Please note that the number of arguments to Combine is limited +// by the maximum arity of the implementation of tuple which is +// currently set at $maxtuple. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. +#include "gtest/internal/gtest-param-util.h" +#include "gtest/internal/gtest-port.h" + +#if GTEST_HAS_PARAM_TEST + +namespace testing { + +// Forward declarations of ValuesIn(), which is implemented in +// include/gtest/gtest-param-test.h. +template +internal::ParamGenerator< + typename ::testing::internal::IteratorTraits::value_type> +ValuesIn(ForwardIterator begin, ForwardIterator end); + +template +internal::ParamGenerator ValuesIn(const T (&array)[N]); + +template +internal::ParamGenerator ValuesIn( + const Container& container); + +namespace internal { + +// Used in the Values() function to provide polymorphic capabilities. +$range i 1..n +$for i [[ +$range j 1..i + +template <$for j, [[typename T$j]]> +class ValueArray$i { + public: + $if i==1 [[explicit ]]ValueArray$i($for j, [[T$j v$j]]) : $for j, [[v$(j)_(v$j)]] {} + + template + operator ParamGenerator() const { + const T array[] = {$for j, [[static_cast(v$(j)_)]]}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray$i& other); + +$for j [[ + + const T$j v$(j)_; +]] + +}; + +]] + +# if GTEST_HAS_COMBINE +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Generates values from the Cartesian product of values produced +// by the argument generators. +// +$range i 2..maxtuple +$for i [[ +$range j 1..i +$range k 2..i + +template <$for j, [[typename T$j]]> +class CartesianProductGenerator$i + : public ParamGeneratorInterface< ::testing::tuple<$for j, [[T$j]]> > { + public: + typedef ::testing::tuple<$for j, [[T$j]]> ParamType; + + CartesianProductGenerator$i($for j, [[const ParamGenerator& g$j]]) + : $for j, [[g$(j)_(g$j)]] {} + virtual ~CartesianProductGenerator$i() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, $for j, [[g$(j)_, g$(j)_.begin()]]); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, $for j, [[g$(j)_, g$(j)_.end()]]); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, $for j, [[ + + const ParamGenerator& g$j, + const typename ParamGenerator::iterator& current$(j)]]) + : base_(base), +$for j, [[ + + begin$(j)_(g$j.begin()), end$(j)_(g$j.end()), current$(j)_(current$j) +]] { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current$(i)_; + +$for k [[ + if (current$(i+2-k)_ == end$(i+2-k)_) { + current$(i+2-k)_ = begin$(i+2-k)_; + ++current$(i+2-k-1)_; + } + +]] + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ($for j && [[ + + current$(j)_ == typed_other->current$(j)_ +]]); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), $for j, [[ + + begin$(j)_(other.begin$(j)_), + end$(j)_(other.end$(j)_), + current$(j)_(other.current$(j)_) +]] { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType($for j, [[*current$(j)_]]); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return +$for j || [[ + + current$(j)_ == end$(j)_ +]]; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. +$for j [[ + + const typename ParamGenerator::iterator begin$(j)_; + const typename ParamGenerator::iterator end$(j)_; + typename ParamGenerator::iterator current$(j)_; +]] + + ParamType current_value_; + }; // class CartesianProductGenerator$i::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator$i& other); + + +$for j [[ + const ParamGenerator g$(j)_; + +]] +}; // class CartesianProductGenerator$i + + +]] + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Helper classes providing Combine() with polymorphic features. They allow +// casting CartesianProductGeneratorN to ParamGenerator if T is +// convertible to U. +// +$range i 2..maxtuple +$for i [[ +$range j 1..i + +template <$for j, [[class Generator$j]]> +class CartesianProductHolder$i { + public: +CartesianProductHolder$i($for j, [[const Generator$j& g$j]]) + : $for j, [[g$(j)_(g$j)]] {} + template <$for j, [[typename T$j]]> + operator ParamGenerator< ::testing::tuple<$for j, [[T$j]]> >() const { + return ParamGenerator< ::testing::tuple<$for j, [[T$j]]> >( + new CartesianProductGenerator$i<$for j, [[T$j]]>( +$for j,[[ + + static_cast >(g$(j)_) +]])); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder$i& other); + + +$for j [[ + const Generator$j g$(j)_; + +]] +}; // class CartesianProductHolder$i + +]] + +# endif // GTEST_HAS_COMBINE + +} // namespace internal +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-param-util.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-param-util.h new file mode 100644 index 0000000000..82cab9b020 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-param-util.h @@ -0,0 +1,731 @@ +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: vladl@google.com (Vlad Losev) + +// Type and function utilities for implementing parameterized tests. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ + +#include + +#include +#include +#include +#include + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. +#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-linked_ptr.h" +#include "gtest/internal/gtest-port.h" +#include "gtest/gtest-printers.h" + +#if GTEST_HAS_PARAM_TEST + +namespace testing { + +// Input to a parameterized test name generator, describing a test parameter. +// Consists of the parameter value and the integer parameter index. +template +struct TestParamInfo { + TestParamInfo(const ParamType& a_param, size_t an_index) : + param(a_param), + index(an_index) {} + ParamType param; + size_t index; +}; + +// A builtin parameterized test name generator which returns the result of +// testing::PrintToString. +struct PrintToStringParamName { + template + std::string operator()(const TestParamInfo& info) const { + return PrintToString(info.param); + } +}; + +namespace internal { + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Outputs a message explaining invalid registration of different +// fixture class for the same test case. This may happen when +// TEST_P macro is used to define two tests with the same name +// but in different namespaces. +GTEST_API_ void ReportInvalidTestCaseType(const char* test_case_name, + CodeLocation code_location); + +template class ParamGeneratorInterface; +template class ParamGenerator; + +// Interface for iterating over elements provided by an implementation +// of ParamGeneratorInterface. +template +class ParamIteratorInterface { + public: + virtual ~ParamIteratorInterface() {} + // A pointer to the base generator instance. + // Used only for the purposes of iterator comparison + // to make sure that two iterators belong to the same generator. + virtual const ParamGeneratorInterface* BaseGenerator() const = 0; + // Advances iterator to point to the next element + // provided by the generator. The caller is responsible + // for not calling Advance() on an iterator equal to + // BaseGenerator()->End(). + virtual void Advance() = 0; + // Clones the iterator object. Used for implementing copy semantics + // of ParamIterator. + virtual ParamIteratorInterface* Clone() const = 0; + // Dereferences the current iterator and provides (read-only) access + // to the pointed value. It is the caller's responsibility not to call + // Current() on an iterator equal to BaseGenerator()->End(). + // Used for implementing ParamGenerator::operator*(). + virtual const T* Current() const = 0; + // Determines whether the given iterator and other point to the same + // element in the sequence generated by the generator. + // Used for implementing ParamGenerator::operator==(). + virtual bool Equals(const ParamIteratorInterface& other) const = 0; +}; + +// Class iterating over elements provided by an implementation of +// ParamGeneratorInterface. It wraps ParamIteratorInterface +// and implements the const forward iterator concept. +template +class ParamIterator { + public: + typedef T value_type; + typedef const T& reference; + typedef ptrdiff_t difference_type; + + // ParamIterator assumes ownership of the impl_ pointer. + ParamIterator(const ParamIterator& other) : impl_(other.impl_->Clone()) {} + ParamIterator& operator=(const ParamIterator& other) { + if (this != &other) + impl_.reset(other.impl_->Clone()); + return *this; + } + + const T& operator*() const { return *impl_->Current(); } + const T* operator->() const { return impl_->Current(); } + // Prefix version of operator++. + ParamIterator& operator++() { + impl_->Advance(); + return *this; + } + // Postfix version of operator++. + ParamIterator operator++(int /*unused*/) { + ParamIteratorInterface* clone = impl_->Clone(); + impl_->Advance(); + return ParamIterator(clone); + } + bool operator==(const ParamIterator& other) const { + return impl_.get() == other.impl_.get() || impl_->Equals(*other.impl_); + } + bool operator!=(const ParamIterator& other) const { + return !(*this == other); + } + + private: + friend class ParamGenerator; + explicit ParamIterator(ParamIteratorInterface* impl) : impl_(impl) {} + scoped_ptr > impl_; +}; + +// ParamGeneratorInterface is the binary interface to access generators +// defined in other translation units. +template +class ParamGeneratorInterface { + public: + typedef T ParamType; + + virtual ~ParamGeneratorInterface() {} + + // Generator interface definition + virtual ParamIteratorInterface* Begin() const = 0; + virtual ParamIteratorInterface* End() const = 0; +}; + +// Wraps ParamGeneratorInterface and provides general generator syntax +// compatible with the STL Container concept. +// This class implements copy initialization semantics and the contained +// ParamGeneratorInterface instance is shared among all copies +// of the original object. This is possible because that instance is immutable. +template +class ParamGenerator { + public: + typedef ParamIterator iterator; + + explicit ParamGenerator(ParamGeneratorInterface* impl) : impl_(impl) {} + ParamGenerator(const ParamGenerator& other) : impl_(other.impl_) {} + + ParamGenerator& operator=(const ParamGenerator& other) { + impl_ = other.impl_; + return *this; + } + + iterator begin() const { return iterator(impl_->Begin()); } + iterator end() const { return iterator(impl_->End()); } + + private: + linked_ptr > impl_; +}; + +// Generates values from a range of two comparable values. Can be used to +// generate sequences of user-defined types that implement operator+() and +// operator<(). +// This class is used in the Range() function. +template +class RangeGenerator : public ParamGeneratorInterface { + public: + RangeGenerator(T begin, T end, IncrementT step) + : begin_(begin), end_(end), + step_(step), end_index_(CalculateEndIndex(begin, end, step)) {} + virtual ~RangeGenerator() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, begin_, 0, step_); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, end_, end_index_, step_); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, T value, int index, + IncrementT step) + : base_(base), value_(value), index_(index), step_(step) {} + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + virtual void Advance() { + value_ = static_cast(value_ + step_); + index_++; + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const T* Current() const { return &value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const int other_index = + CheckedDowncastToActualType(&other)->index_; + return index_ == other_index; + } + + private: + Iterator(const Iterator& other) + : ParamIteratorInterface(), + base_(other.base_), value_(other.value_), index_(other.index_), + step_(other.step_) {} + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + T value_; + int index_; + const IncrementT step_; + }; // class RangeGenerator::Iterator + + static int CalculateEndIndex(const T& begin, + const T& end, + const IncrementT& step) { + int end_index = 0; + for (T i = begin; i < end; i = static_cast(i + step)) + end_index++; + return end_index; + } + + // No implementation - assignment is unsupported. + void operator=(const RangeGenerator& other); + + const T begin_; + const T end_; + const IncrementT step_; + // The index for the end() iterator. All the elements in the generated + // sequence are indexed (0-based) to aid iterator comparison. + const int end_index_; +}; // class RangeGenerator + + +// Generates values from a pair of STL-style iterators. Used in the +// ValuesIn() function. The elements are copied from the source range +// since the source can be located on the stack, and the generator +// is likely to persist beyond that stack frame. +template +class ValuesInIteratorRangeGenerator : public ParamGeneratorInterface { + public: + template + ValuesInIteratorRangeGenerator(ForwardIterator begin, ForwardIterator end) + : container_(begin, end) {} + virtual ~ValuesInIteratorRangeGenerator() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, container_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, container_.end()); + } + + private: + typedef typename ::std::vector ContainerType; + + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + typename ContainerType::const_iterator iterator) + : base_(base), iterator_(iterator) {} + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + virtual void Advance() { + ++iterator_; + value_.reset(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + // We need to use cached value referenced by iterator_ because *iterator_ + // can return a temporary object (and of type other then T), so just + // having "return &*iterator_;" doesn't work. + // value_ is updated here and not in Advance() because Advance() + // can advance iterator_ beyond the end of the range, and we cannot + // detect that fact. The client code, on the other hand, is + // responsible for not calling Current() on an out-of-range iterator. + virtual const T* Current() const { + if (value_.get() == NULL) + value_.reset(new T(*iterator_)); + return value_.get(); + } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + return iterator_ == + CheckedDowncastToActualType(&other)->iterator_; + } + + private: + Iterator(const Iterator& other) + // The explicit constructor call suppresses a false warning + // emitted by gcc when supplied with the -Wextra option. + : ParamIteratorInterface(), + base_(other.base_), + iterator_(other.iterator_) {} + + const ParamGeneratorInterface* const base_; + typename ContainerType::const_iterator iterator_; + // A cached value of *iterator_. We keep it here to allow access by + // pointer in the wrapping iterator's operator->(). + // value_ needs to be mutable to be accessed in Current(). + // Use of scoped_ptr helps manage cached value's lifetime, + // which is bound by the lifespan of the iterator itself. + mutable scoped_ptr value_; + }; // class ValuesInIteratorRangeGenerator::Iterator + + // No implementation - assignment is unsupported. + void operator=(const ValuesInIteratorRangeGenerator& other); + + const ContainerType container_; +}; // class ValuesInIteratorRangeGenerator + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Default parameterized test name generator, returns a string containing the +// integer test parameter index. +template +std::string DefaultParamName(const TestParamInfo& info) { + Message name_stream; + name_stream << info.index; + return name_stream.GetString(); +} + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Parameterized test name overload helpers, which help the +// INSTANTIATE_TEST_CASE_P macro choose between the default parameterized +// test name generator and user param name generator. +template +ParamNameGenFunctor GetParamNameGen(ParamNameGenFunctor func) { + return func; +} + +template +struct ParamNameGenFunc { + typedef std::string Type(const TestParamInfo&); +}; + +template +typename ParamNameGenFunc::Type *GetParamNameGen() { + return DefaultParamName; +} + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Stores a parameter value and later creates tests parameterized with that +// value. +template +class ParameterizedTestFactory : public TestFactoryBase { + public: + typedef typename TestClass::ParamType ParamType; + explicit ParameterizedTestFactory(ParamType parameter) : + parameter_(parameter) {} + virtual Test* CreateTest() { + TestClass::SetParam(¶meter_); + return new TestClass(); + } + + private: + const ParamType parameter_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestFactory); +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// TestMetaFactoryBase is a base class for meta-factories that create +// test factories for passing into MakeAndRegisterTestInfo function. +template +class TestMetaFactoryBase { + public: + virtual ~TestMetaFactoryBase() {} + + virtual TestFactoryBase* CreateTestFactory(ParamType parameter) = 0; +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// TestMetaFactory creates test factories for passing into +// MakeAndRegisterTestInfo function. Since MakeAndRegisterTestInfo receives +// ownership of test factory pointer, same factory object cannot be passed +// into that method twice. But ParameterizedTestCaseInfo is going to call +// it for each Test/Parameter value combination. Thus it needs meta factory +// creator class. +template +class TestMetaFactory + : public TestMetaFactoryBase { + public: + typedef typename TestCase::ParamType ParamType; + + TestMetaFactory() {} + + virtual TestFactoryBase* CreateTestFactory(ParamType parameter) { + return new ParameterizedTestFactory(parameter); + } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestMetaFactory); +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// ParameterizedTestCaseInfoBase is a generic interface +// to ParameterizedTestCaseInfo classes. ParameterizedTestCaseInfoBase +// accumulates test information provided by TEST_P macro invocations +// and generators provided by INSTANTIATE_TEST_CASE_P macro invocations +// and uses that information to register all resulting test instances +// in RegisterTests method. The ParameterizeTestCaseRegistry class holds +// a collection of pointers to the ParameterizedTestCaseInfo objects +// and calls RegisterTests() on each of them when asked. +class ParameterizedTestCaseInfoBase { + public: + virtual ~ParameterizedTestCaseInfoBase() {} + + // Base part of test case name for display purposes. + virtual const string& GetTestCaseName() const = 0; + // Test case id to verify identity. + virtual TypeId GetTestCaseTypeId() const = 0; + // UnitTest class invokes this method to register tests in this + // test case right before running them in RUN_ALL_TESTS macro. + // This method should not be called more then once on any single + // instance of a ParameterizedTestCaseInfoBase derived class. + virtual void RegisterTests() = 0; + + protected: + ParameterizedTestCaseInfoBase() {} + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestCaseInfoBase); +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// ParameterizedTestCaseInfo accumulates tests obtained from TEST_P +// macro invocations for a particular test case and generators +// obtained from INSTANTIATE_TEST_CASE_P macro invocations for that +// test case. It registers tests with all values generated by all +// generators when asked. +template +class ParameterizedTestCaseInfo : public ParameterizedTestCaseInfoBase { + public: + // ParamType and GeneratorCreationFunc are private types but are required + // for declarations of public methods AddTestPattern() and + // AddTestCaseInstantiation(). + typedef typename TestCase::ParamType ParamType; + // A function that returns an instance of appropriate generator type. + typedef ParamGenerator(GeneratorCreationFunc)(); + typedef typename ParamNameGenFunc::Type ParamNameGeneratorFunc; + + explicit ParameterizedTestCaseInfo( + const char* name, CodeLocation code_location) + : test_case_name_(name), code_location_(code_location) {} + + // Test case base name for display purposes. + virtual const string& GetTestCaseName() const { return test_case_name_; } + // Test case id to verify identity. + virtual TypeId GetTestCaseTypeId() const { return GetTypeId(); } + // TEST_P macro uses AddTestPattern() to record information + // about a single test in a LocalTestInfo structure. + // test_case_name is the base name of the test case (without invocation + // prefix). test_base_name is the name of an individual test without + // parameter index. For the test SequenceA/FooTest.DoBar/1 FooTest is + // test case base name and DoBar is test base name. + void AddTestPattern(const char* test_case_name, + const char* test_base_name, + TestMetaFactoryBase* meta_factory) { + tests_.push_back(linked_ptr(new TestInfo(test_case_name, + test_base_name, + meta_factory))); + } + // INSTANTIATE_TEST_CASE_P macro uses AddGenerator() to record information + // about a generator. + int AddTestCaseInstantiation(const string& instantiation_name, + GeneratorCreationFunc* func, + ParamNameGeneratorFunc* name_func, + const char* file, + int line) { + instantiations_.push_back( + InstantiationInfo(instantiation_name, func, name_func, file, line)); + return 0; // Return value used only to run this method in namespace scope. + } + // UnitTest class invokes this method to register tests in this test case + // test cases right before running tests in RUN_ALL_TESTS macro. + // This method should not be called more then once on any single + // instance of a ParameterizedTestCaseInfoBase derived class. + // UnitTest has a guard to prevent from calling this method more then once. + virtual void RegisterTests() { + for (typename TestInfoContainer::iterator test_it = tests_.begin(); + test_it != tests_.end(); ++test_it) { + linked_ptr test_info = *test_it; + for (typename InstantiationContainer::iterator gen_it = + instantiations_.begin(); gen_it != instantiations_.end(); + ++gen_it) { + const string& instantiation_name = gen_it->name; + ParamGenerator generator((*gen_it->generator)()); + ParamNameGeneratorFunc* name_func = gen_it->name_func; + const char* file = gen_it->file; + int line = gen_it->line; + + string test_case_name; + if ( !instantiation_name.empty() ) + test_case_name = instantiation_name + "/"; + test_case_name += test_info->test_case_base_name; + + size_t i = 0; + std::set test_param_names; + for (typename ParamGenerator::iterator param_it = + generator.begin(); + param_it != generator.end(); ++param_it, ++i) { + Message test_name_stream; + + std::string param_name = name_func( + TestParamInfo(*param_it, i)); + + GTEST_CHECK_(IsValidParamName(param_name)) + << "Parameterized test name '" << param_name + << "' is invalid, in " << file + << " line " << line << std::endl; + + GTEST_CHECK_(test_param_names.count(param_name) == 0) + << "Duplicate parameterized test name '" << param_name + << "', in " << file << " line " << line << std::endl; + + test_param_names.insert(param_name); + + test_name_stream << test_info->test_base_name << "/" << param_name; + MakeAndRegisterTestInfo( + test_case_name.c_str(), + test_name_stream.GetString().c_str(), + NULL, // No type parameter. + PrintToString(*param_it).c_str(), + code_location_, + GetTestCaseTypeId(), + TestCase::SetUpTestCase, + TestCase::TearDownTestCase, + test_info->test_meta_factory->CreateTestFactory(*param_it)); + } // for param_it + } // for gen_it + } // for test_it + } // RegisterTests + + private: + // LocalTestInfo structure keeps information about a single test registered + // with TEST_P macro. + struct TestInfo { + TestInfo(const char* a_test_case_base_name, + const char* a_test_base_name, + TestMetaFactoryBase* a_test_meta_factory) : + test_case_base_name(a_test_case_base_name), + test_base_name(a_test_base_name), + test_meta_factory(a_test_meta_factory) {} + + const string test_case_base_name; + const string test_base_name; + const scoped_ptr > test_meta_factory; + }; + typedef ::std::vector > TestInfoContainer; + // Records data received from INSTANTIATE_TEST_CASE_P macros: + // + struct InstantiationInfo { + InstantiationInfo(const std::string &name_in, + GeneratorCreationFunc* generator_in, + ParamNameGeneratorFunc* name_func_in, + const char* file_in, + int line_in) + : name(name_in), + generator(generator_in), + name_func(name_func_in), + file(file_in), + line(line_in) {} + + std::string name; + GeneratorCreationFunc* generator; + ParamNameGeneratorFunc* name_func; + const char* file; + int line; + }; + typedef ::std::vector InstantiationContainer; + + static bool IsValidParamName(const std::string& name) { + // Check for empty string + if (name.empty()) + return false; + + // Check for invalid characters + for (std::string::size_type index = 0; index < name.size(); ++index) { + if (!isalnum(name[index]) && name[index] != '_') + return false; + } + + return true; + } + + const string test_case_name_; + CodeLocation code_location_; + TestInfoContainer tests_; + InstantiationContainer instantiations_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestCaseInfo); +}; // class ParameterizedTestCaseInfo + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// ParameterizedTestCaseRegistry contains a map of ParameterizedTestCaseInfoBase +// classes accessed by test case names. TEST_P and INSTANTIATE_TEST_CASE_P +// macros use it to locate their corresponding ParameterizedTestCaseInfo +// descriptors. +class ParameterizedTestCaseRegistry { + public: + ParameterizedTestCaseRegistry() {} + ~ParameterizedTestCaseRegistry() { + for (TestCaseInfoContainer::iterator it = test_case_infos_.begin(); + it != test_case_infos_.end(); ++it) { + delete *it; + } + } + + // Looks up or creates and returns a structure containing information about + // tests and instantiations of a particular test case. + template + ParameterizedTestCaseInfo* GetTestCasePatternHolder( + const char* test_case_name, + CodeLocation code_location) { + ParameterizedTestCaseInfo* typed_test_info = NULL; + for (TestCaseInfoContainer::iterator it = test_case_infos_.begin(); + it != test_case_infos_.end(); ++it) { + if ((*it)->GetTestCaseName() == test_case_name) { + if ((*it)->GetTestCaseTypeId() != GetTypeId()) { + // Complain about incorrect usage of Google Test facilities + // and terminate the program since we cannot guaranty correct + // test case setup and tear-down in this case. + ReportInvalidTestCaseType(test_case_name, code_location); + posix::Abort(); + } else { + // At this point we are sure that the object we found is of the same + // type we are looking for, so we downcast it to that type + // without further checks. + typed_test_info = CheckedDowncastToActualType< + ParameterizedTestCaseInfo >(*it); + } + break; + } + } + if (typed_test_info == NULL) { + typed_test_info = new ParameterizedTestCaseInfo( + test_case_name, code_location); + test_case_infos_.push_back(typed_test_info); + } + return typed_test_info; + } + void RegisterTests() { + for (TestCaseInfoContainer::iterator it = test_case_infos_.begin(); + it != test_case_infos_.end(); ++it) { + (*it)->RegisterTests(); + } + } + + private: + typedef ::std::vector TestCaseInfoContainer; + + TestCaseInfoContainer test_case_infos_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestCaseRegistry); +}; + +} // namespace internal +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-port-arch.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-port-arch.h new file mode 100644 index 0000000000..74ab949057 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-port-arch.h @@ -0,0 +1,93 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines the GTEST_OS_* macro. +// It is separate from gtest-port.h so that custom/gtest-port.h can include it. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ + +// Determines the platform on which Google Test is compiled. +#ifdef __CYGWIN__ +# define GTEST_OS_CYGWIN 1 +#elif defined __SYMBIAN32__ +# define GTEST_OS_SYMBIAN 1 +#elif defined _WIN32 +# define GTEST_OS_WINDOWS 1 +# ifdef _WIN32_WCE +# define GTEST_OS_WINDOWS_MOBILE 1 +# elif defined(__MINGW__) || defined(__MINGW32__) +# define GTEST_OS_WINDOWS_MINGW 1 +# elif defined(WINAPI_FAMILY) +# include +# if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +# define GTEST_OS_WINDOWS_DESKTOP 1 +# elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_PHONE_APP) +# define GTEST_OS_WINDOWS_PHONE 1 +# elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) +# define GTEST_OS_WINDOWS_RT 1 +# else + // WINAPI_FAMILY defined but no known partition matched. + // Default to desktop. +# define GTEST_OS_WINDOWS_DESKTOP 1 +# endif +# else +# define GTEST_OS_WINDOWS_DESKTOP 1 +# endif // _WIN32_WCE +#elif defined __APPLE__ +# define GTEST_OS_MAC 1 +# if TARGET_OS_IPHONE +# define GTEST_OS_IOS 1 +# endif +#elif defined __FreeBSD__ +# define GTEST_OS_FREEBSD 1 +#elif defined __linux__ +# define GTEST_OS_LINUX 1 +# if defined __ANDROID__ +# define GTEST_OS_LINUX_ANDROID 1 +# endif +#elif defined __MVS__ +# define GTEST_OS_ZOS 1 +#elif defined(__sun) && defined(__SVR4) +# define GTEST_OS_SOLARIS 1 +#elif defined(_AIX) +# define GTEST_OS_AIX 1 +#elif defined(__hpux) +# define GTEST_OS_HPUX 1 +#elif defined __native_client__ +# define GTEST_OS_NACL 1 +#elif defined __OpenBSD__ +# define GTEST_OS_OPENBSD 1 +#elif defined __QNX__ +# define GTEST_OS_QNX 1 +#endif // __CYGWIN__ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-port.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-port.h new file mode 100644 index 0000000000..0094ed5077 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-port.h @@ -0,0 +1,2554 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: wan@google.com (Zhanyong Wan) +// +// Low-level types and utilities for porting Google Test to various +// platforms. All macros ending with _ and symbols defined in an +// internal namespace are subject to change without notice. Code +// outside Google Test MUST NOT USE THEM DIRECTLY. Macros that don't +// end with _ are part of Google Test's public API and can be used by +// code outside Google Test. +// +// This file is fundamental to Google Test. All other Google Test source +// files are expected to #include this. Therefore, it cannot #include +// any other Google Test header. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ + +// Environment-describing macros +// ----------------------------- +// +// Google Test can be used in many different environments. Macros in +// this section tell Google Test what kind of environment it is being +// used in, such that Google Test can provide environment-specific +// features and implementations. +// +// Google Test tries to automatically detect the properties of its +// environment, so users usually don't need to worry about these +// macros. However, the automatic detection is not perfect. +// Sometimes it's necessary for a user to define some of the following +// macros in the build script to override Google Test's decisions. +// +// If the user doesn't define a macro in the list, Google Test will +// provide a default definition. After this header is #included, all +// macros in this list will be defined to either 1 or 0. +// +// Notes to maintainers: +// - Each macro here is a user-tweakable knob; do not grow the list +// lightly. +// - Use #if to key off these macros. Don't use #ifdef or "#if +// defined(...)", which will not work as these macros are ALWAYS +// defined. +// +// GTEST_HAS_CLONE - Define it to 1/0 to indicate that clone(2) +// is/isn't available. +// GTEST_HAS_EXCEPTIONS - Define it to 1/0 to indicate that exceptions +// are enabled. +// GTEST_HAS_GLOBAL_STRING - Define it to 1/0 to indicate that ::string +// is/isn't available (some systems define +// ::string, which is different to std::string). +// GTEST_HAS_GLOBAL_WSTRING - Define it to 1/0 to indicate that ::string +// is/isn't available (some systems define +// ::wstring, which is different to std::wstring). +// GTEST_HAS_POSIX_RE - Define it to 1/0 to indicate that POSIX regular +// expressions are/aren't available. +// GTEST_HAS_PTHREAD - Define it to 1/0 to indicate that +// is/isn't available. +// GTEST_HAS_RTTI - Define it to 1/0 to indicate that RTTI is/isn't +// enabled. +// GTEST_HAS_STD_WSTRING - Define it to 1/0 to indicate that +// std::wstring does/doesn't work (Google Test can +// be used where std::wstring is unavailable). +// GTEST_HAS_TR1_TUPLE - Define it to 1/0 to indicate tr1::tuple +// is/isn't available. +// GTEST_HAS_SEH - Define it to 1/0 to indicate whether the +// compiler supports Microsoft's "Structured +// Exception Handling". +// GTEST_HAS_STREAM_REDIRECTION +// - Define it to 1/0 to indicate whether the +// platform supports I/O stream redirection using +// dup() and dup2(). +// GTEST_USE_OWN_TR1_TUPLE - Define it to 1/0 to indicate whether Google +// Test's own tr1 tuple implementation should be +// used. Unused when the user sets +// GTEST_HAS_TR1_TUPLE to 0. +// GTEST_LANG_CXX11 - Define it to 1/0 to indicate that Google Test +// is building in C++11/C++98 mode. +// GTEST_LINKED_AS_SHARED_LIBRARY +// - Define to 1 when compiling tests that use +// Google Test as a shared library (known as +// DLL on Windows). +// GTEST_CREATE_SHARED_LIBRARY +// - Define to 1 when compiling Google Test itself +// as a shared library. + +// Platform-indicating macros +// -------------------------- +// +// Macros indicating the platform on which Google Test is being used +// (a macro is defined to 1 if compiled on the given platform; +// otherwise UNDEFINED -- it's never defined to 0.). Google Test +// defines these macros automatically. Code outside Google Test MUST +// NOT define them. +// +// GTEST_OS_AIX - IBM AIX +// GTEST_OS_CYGWIN - Cygwin +// GTEST_OS_FREEBSD - FreeBSD +// GTEST_OS_HPUX - HP-UX +// GTEST_OS_LINUX - Linux +// GTEST_OS_LINUX_ANDROID - Google Android +// GTEST_OS_MAC - Mac OS X +// GTEST_OS_IOS - iOS +// GTEST_OS_NACL - Google Native Client (NaCl) +// GTEST_OS_OPENBSD - OpenBSD +// GTEST_OS_QNX - QNX +// GTEST_OS_SOLARIS - Sun Solaris +// GTEST_OS_SYMBIAN - Symbian +// GTEST_OS_WINDOWS - Windows (Desktop, MinGW, or Mobile) +// GTEST_OS_WINDOWS_DESKTOP - Windows Desktop +// GTEST_OS_WINDOWS_MINGW - MinGW +// GTEST_OS_WINDOWS_MOBILE - Windows Mobile +// GTEST_OS_WINDOWS_PHONE - Windows Phone +// GTEST_OS_WINDOWS_RT - Windows Store App/WinRT +// GTEST_OS_ZOS - z/OS +// +// Among the platforms, Cygwin, Linux, Max OS X, and Windows have the +// most stable support. Since core members of the Google Test project +// don't have access to other platforms, support for them may be less +// stable. If you notice any problems on your platform, please notify +// googletestframework@googlegroups.com (patches for fixing them are +// even more welcome!). +// +// It is possible that none of the GTEST_OS_* macros are defined. + +// Feature-indicating macros +// ------------------------- +// +// Macros indicating which Google Test features are available (a macro +// is defined to 1 if the corresponding feature is supported; +// otherwise UNDEFINED -- it's never defined to 0.). Google Test +// defines these macros automatically. Code outside Google Test MUST +// NOT define them. +// +// These macros are public so that portable tests can be written. +// Such tests typically surround code using a feature with an #if +// which controls that code. For example: +// +// #if GTEST_HAS_DEATH_TEST +// EXPECT_DEATH(DoSomethingDeadly()); +// #endif +// +// GTEST_HAS_COMBINE - the Combine() function (for value-parameterized +// tests) +// GTEST_HAS_DEATH_TEST - death tests +// GTEST_HAS_PARAM_TEST - value-parameterized tests +// GTEST_HAS_TYPED_TEST - typed tests +// GTEST_HAS_TYPED_TEST_P - type-parameterized tests +// GTEST_IS_THREADSAFE - Google Test is thread-safe. +// GTEST_USES_POSIX_RE - enhanced POSIX regex is used. Do not confuse with +// GTEST_HAS_POSIX_RE (see above) which users can +// define themselves. +// GTEST_USES_SIMPLE_RE - our own simple regex is used; +// the above two are mutually exclusive. +// GTEST_CAN_COMPARE_NULL - accepts untyped NULL in EXPECT_EQ(). + +// Misc public macros +// ------------------ +// +// GTEST_FLAG(flag_name) - references the variable corresponding to +// the given Google Test flag. + +// Internal utilities +// ------------------ +// +// The following macros and utilities are for Google Test's INTERNAL +// use only. Code outside Google Test MUST NOT USE THEM DIRECTLY. +// +// Macros for basic C++ coding: +// GTEST_AMBIGUOUS_ELSE_BLOCKER_ - for disabling a gcc warning. +// GTEST_ATTRIBUTE_UNUSED_ - declares that a class' instances or a +// variable don't have to be used. +// GTEST_DISALLOW_ASSIGN_ - disables operator=. +// GTEST_DISALLOW_COPY_AND_ASSIGN_ - disables copy ctor and operator=. +// GTEST_MUST_USE_RESULT_ - declares that a function's result must be used. +// GTEST_INTENTIONAL_CONST_COND_PUSH_ - start code section where MSVC C4127 is +// suppressed (constant conditional). +// GTEST_INTENTIONAL_CONST_COND_POP_ - finish code section where MSVC C4127 +// is suppressed. +// +// C++11 feature wrappers: +// +// testing::internal::move - portability wrapper for std::move. +// +// Synchronization: +// Mutex, MutexLock, ThreadLocal, GetThreadCount() +// - synchronization primitives. +// +// Template meta programming: +// is_pointer - as in TR1; needed on Symbian and IBM XL C/C++ only. +// IteratorTraits - partial implementation of std::iterator_traits, which +// is not available in libCstd when compiled with Sun C++. +// +// Smart pointers: +// scoped_ptr - as in TR2. +// +// Regular expressions: +// RE - a simple regular expression class using the POSIX +// Extended Regular Expression syntax on UNIX-like +// platforms, or a reduced regular exception syntax on +// other platforms, including Windows. +// +// Logging: +// GTEST_LOG_() - logs messages at the specified severity level. +// LogToStderr() - directs all log messages to stderr. +// FlushInfoLog() - flushes informational log messages. +// +// Stdout and stderr capturing: +// CaptureStdout() - starts capturing stdout. +// GetCapturedStdout() - stops capturing stdout and returns the captured +// string. +// CaptureStderr() - starts capturing stderr. +// GetCapturedStderr() - stops capturing stderr and returns the captured +// string. +// +// Integer types: +// TypeWithSize - maps an integer to a int type. +// Int32, UInt32, Int64, UInt64, TimeInMillis +// - integers of known sizes. +// BiggestInt - the biggest signed integer type. +// +// Command-line utilities: +// GTEST_DECLARE_*() - declares a flag. +// GTEST_DEFINE_*() - defines a flag. +// GetInjectableArgvs() - returns the command line as a vector of strings. +// +// Environment variable utilities: +// GetEnv() - gets the value of an environment variable. +// BoolFromGTestEnv() - parses a bool environment variable. +// Int32FromGTestEnv() - parses an Int32 environment variable. +// StringFromGTestEnv() - parses a string environment variable. + +#include // for isspace, etc +#include // for ptrdiff_t +#include +#include +#include +#ifndef _WIN32_WCE +# include +# include +#endif // !_WIN32_WCE + +#if defined __APPLE__ +# include +# include +#endif + +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include +#include // NOLINT + +#include "gtest/internal/gtest-port-arch.h" +#include "gtest/internal/custom/gtest-port.h" + +#if !defined(GTEST_DEV_EMAIL_) +# define GTEST_DEV_EMAIL_ "googletestframework@@googlegroups.com" +# define GTEST_FLAG_PREFIX_ "gtest_" +# define GTEST_FLAG_PREFIX_DASH_ "gtest-" +# define GTEST_FLAG_PREFIX_UPPER_ "GTEST_" +# define GTEST_NAME_ "Google Test" +# define GTEST_PROJECT_URL_ "https://github.com/google/googletest/" +#endif // !defined(GTEST_DEV_EMAIL_) + +#if !defined(GTEST_INIT_GOOGLE_TEST_NAME_) +# define GTEST_INIT_GOOGLE_TEST_NAME_ "testing::InitGoogleTest" +#endif // !defined(GTEST_INIT_GOOGLE_TEST_NAME_) + +// Determines the version of gcc that is used to compile this. +#ifdef __GNUC__ +// 40302 means version 4.3.2. +# define GTEST_GCC_VER_ \ + (__GNUC__*10000 + __GNUC_MINOR__*100 + __GNUC_PATCHLEVEL__) +#endif // __GNUC__ + +// Macros for disabling Microsoft Visual C++ warnings. +// +// GTEST_DISABLE_MSC_WARNINGS_PUSH_(4800 4385) +// /* code that triggers warnings C4800 and C4385 */ +// GTEST_DISABLE_MSC_WARNINGS_POP_() +#if _MSC_VER >= 1500 +# define GTEST_DISABLE_MSC_WARNINGS_PUSH_(warnings) \ + __pragma(warning(push)) \ + __pragma(warning(disable: warnings)) +# define GTEST_DISABLE_MSC_WARNINGS_POP_() \ + __pragma(warning(pop)) +#else +// Older versions of MSVC don't have __pragma. +# define GTEST_DISABLE_MSC_WARNINGS_PUSH_(warnings) +# define GTEST_DISABLE_MSC_WARNINGS_POP_() +#endif + +#ifndef GTEST_LANG_CXX11 +// gcc and clang define __GXX_EXPERIMENTAL_CXX0X__ when +// -std={c,gnu}++{0x,11} is passed. The C++11 standard specifies a +// value for __cplusplus, and recent versions of clang, gcc, and +// probably other compilers set that too in C++11 mode. +# if __GXX_EXPERIMENTAL_CXX0X__ || __cplusplus >= 201103L +// Compiling in at least C++11 mode. +# define GTEST_LANG_CXX11 1 +# else +# define GTEST_LANG_CXX11 0 +# endif +#endif + +// Distinct from C++11 language support, some environments don't provide +// proper C++11 library support. Notably, it's possible to build in +// C++11 mode when targeting Mac OS X 10.6, which has an old libstdc++ +// with no C++11 support. +// +// libstdc++ has sufficient C++11 support as of GCC 4.6.0, __GLIBCXX__ +// 20110325, but maintenance releases in the 4.4 and 4.5 series followed +// this date, so check for those versions by their date stamps. +// https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning +#if GTEST_LANG_CXX11 && \ + (!defined(__GLIBCXX__) || ( \ + __GLIBCXX__ >= 20110325ul && /* GCC >= 4.6.0 */ \ + /* Blacklist of patch releases of older branches: */ \ + __GLIBCXX__ != 20110416ul && /* GCC 4.4.6 */ \ + __GLIBCXX__ != 20120313ul && /* GCC 4.4.7 */ \ + __GLIBCXX__ != 20110428ul && /* GCC 4.5.3 */ \ + __GLIBCXX__ != 20120702ul)) /* GCC 4.5.4 */ +# define GTEST_STDLIB_CXX11 1 +#endif + +// Only use C++11 library features if the library provides them. +#if GTEST_STDLIB_CXX11 +# define GTEST_HAS_STD_BEGIN_AND_END_ 1 +# define GTEST_HAS_STD_FORWARD_LIST_ 1 +# define GTEST_HAS_STD_FUNCTION_ 1 +# define GTEST_HAS_STD_INITIALIZER_LIST_ 1 +# define GTEST_HAS_STD_MOVE_ 1 +# define GTEST_HAS_STD_SHARED_PTR_ 1 +# define GTEST_HAS_STD_TYPE_TRAITS_ 1 +# define GTEST_HAS_STD_UNIQUE_PTR_ 1 +#endif + +// C++11 specifies that provides std::tuple. +// Some platforms still might not have it, however. +#if GTEST_LANG_CXX11 +# define GTEST_HAS_STD_TUPLE_ 1 +# if defined(__clang__) +// Inspired by http://clang.llvm.org/docs/LanguageExtensions.html#__has_include +# if defined(__has_include) && !__has_include() +# undef GTEST_HAS_STD_TUPLE_ +# endif +# elif defined(_MSC_VER) +// Inspired by boost/config/stdlib/dinkumware.hpp +# if defined(_CPPLIB_VER) && _CPPLIB_VER < 520 +# undef GTEST_HAS_STD_TUPLE_ +# endif +# elif defined(__GLIBCXX__) +// Inspired by boost/config/stdlib/libstdcpp3.hpp, +// http://gcc.gnu.org/gcc-4.2/changes.html and +// http://gcc.gnu.org/onlinedocs/libstdc++/manual/bk01pt01ch01.html#manual.intro.status.standard.200x +# if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 2) +# undef GTEST_HAS_STD_TUPLE_ +# endif +# endif +#endif + +// Brings in definitions for functions used in the testing::internal::posix +// namespace (read, write, close, chdir, isatty, stat). We do not currently +// use them on Windows Mobile. +#if GTEST_OS_WINDOWS +# if !GTEST_OS_WINDOWS_MOBILE +# include +# include +# endif +// In order to avoid having to include , use forward declaration +// assuming CRITICAL_SECTION is a typedef of _RTL_CRITICAL_SECTION. +// This assumption is verified by +// WindowsTypesTest.CRITICAL_SECTIONIs_RTL_CRITICAL_SECTION. +struct _RTL_CRITICAL_SECTION; +#else +// This assumes that non-Windows OSes provide unistd.h. For OSes where this +// is not the case, we need to include headers that provide the functions +// mentioned above. +# include +# include +#endif // GTEST_OS_WINDOWS + +#if GTEST_OS_LINUX_ANDROID +// Used to define __ANDROID_API__ matching the target NDK API level. +# include // NOLINT +#endif + +// Defines this to true iff Google Test can use POSIX regular expressions. +#ifndef GTEST_HAS_POSIX_RE +# if GTEST_OS_LINUX_ANDROID +// On Android, is only available starting with Gingerbread. +# define GTEST_HAS_POSIX_RE (__ANDROID_API__ >= 9) +# else +# define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS) +# endif +#endif + +#if GTEST_USES_PCRE +// The appropriate headers have already been included. + +#elif GTEST_HAS_POSIX_RE + +// On some platforms, needs someone to define size_t, and +// won't compile otherwise. We can #include it here as we already +// included , which is guaranteed to define size_t through +// . +# include // NOLINT + +# define GTEST_USES_POSIX_RE 1 + +#elif GTEST_OS_WINDOWS + +// is not available on Windows. Use our own simple regex +// implementation instead. +# define GTEST_USES_SIMPLE_RE 1 + +#else + +// may not be available on this platform. Use our own +// simple regex implementation instead. +# define GTEST_USES_SIMPLE_RE 1 + +#endif // GTEST_USES_PCRE + +#ifndef GTEST_HAS_EXCEPTIONS +// The user didn't tell us whether exceptions are enabled, so we need +// to figure it out. +# if defined(_MSC_VER) || defined(__BORLANDC__) +// MSVC's and C++Builder's implementations of the STL use the _HAS_EXCEPTIONS +// macro to enable exceptions, so we'll do the same. +// Assumes that exceptions are enabled by default. +# ifndef _HAS_EXCEPTIONS +# define _HAS_EXCEPTIONS 1 +# endif // _HAS_EXCEPTIONS +# define GTEST_HAS_EXCEPTIONS _HAS_EXCEPTIONS +# elif defined(__clang__) +// clang defines __EXCEPTIONS iff exceptions are enabled before clang 220714, +// but iff cleanups are enabled after that. In Obj-C++ files, there can be +// cleanups for ObjC exceptions which also need cleanups, even if C++ exceptions +// are disabled. clang has __has_feature(cxx_exceptions) which checks for C++ +// exceptions starting at clang r206352, but which checked for cleanups prior to +// that. To reliably check for C++ exception availability with clang, check for +// __EXCEPTIONS && __has_feature(cxx_exceptions). +# define GTEST_HAS_EXCEPTIONS (__EXCEPTIONS && __has_feature(cxx_exceptions)) +# elif defined(__GNUC__) && __EXCEPTIONS +// gcc defines __EXCEPTIONS to 1 iff exceptions are enabled. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__SUNPRO_CC) +// Sun Pro CC supports exceptions. However, there is no compile-time way of +// detecting whether they are enabled or not. Therefore, we assume that +// they are enabled unless the user tells us otherwise. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__IBMCPP__) && __EXCEPTIONS +// xlC defines __EXCEPTIONS to 1 iff exceptions are enabled. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__HP_aCC) +// Exception handling is in effect by default in HP aCC compiler. It has to +// be turned of by +noeh compiler option if desired. +# define GTEST_HAS_EXCEPTIONS 1 +# else +// For other compilers, we assume exceptions are disabled to be +// conservative. +# define GTEST_HAS_EXCEPTIONS 0 +# endif // defined(_MSC_VER) || defined(__BORLANDC__) +#endif // GTEST_HAS_EXCEPTIONS + +#if !defined(GTEST_HAS_STD_STRING) +// Even though we don't use this macro any longer, we keep it in case +// some clients still depend on it. +# define GTEST_HAS_STD_STRING 1 +#elif !GTEST_HAS_STD_STRING +// The user told us that ::std::string isn't available. +# error "Google Test cannot be used where ::std::string isn't available." +#endif // !defined(GTEST_HAS_STD_STRING) + +#ifndef GTEST_HAS_GLOBAL_STRING +// The user didn't tell us whether ::string is available, so we need +// to figure it out. + +# define GTEST_HAS_GLOBAL_STRING 0 + +#endif // GTEST_HAS_GLOBAL_STRING + +#ifndef GTEST_HAS_STD_WSTRING +// The user didn't tell us whether ::std::wstring is available, so we need +// to figure it out. +// TODO(wan@google.com): uses autoconf to detect whether ::std::wstring +// is available. + +// Cygwin 1.7 and below doesn't support ::std::wstring. +// Solaris' libc++ doesn't support it either. Android has +// no support for it at least as recent as Froyo (2.2). +# define GTEST_HAS_STD_WSTRING \ + (!(GTEST_OS_LINUX_ANDROID || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS)) + +#endif // GTEST_HAS_STD_WSTRING + +#ifndef GTEST_HAS_GLOBAL_WSTRING +// The user didn't tell us whether ::wstring is available, so we need +// to figure it out. +# define GTEST_HAS_GLOBAL_WSTRING \ + (GTEST_HAS_STD_WSTRING && GTEST_HAS_GLOBAL_STRING) +#endif // GTEST_HAS_GLOBAL_WSTRING + +// Determines whether RTTI is available. +#ifndef GTEST_HAS_RTTI +// The user didn't tell us whether RTTI is enabled, so we need to +// figure it out. + +# ifdef _MSC_VER + +# ifdef _CPPRTTI // MSVC defines this macro iff RTTI is enabled. +# define GTEST_HAS_RTTI 1 +# else +# define GTEST_HAS_RTTI 0 +# endif + +// Starting with version 4.3.2, gcc defines __GXX_RTTI iff RTTI is enabled. +# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40302) + +# ifdef __GXX_RTTI +// When building against STLport with the Android NDK and with +// -frtti -fno-exceptions, the build fails at link time with undefined +// references to __cxa_bad_typeid. Note sure if STL or toolchain bug, +// so disable RTTI when detected. +# if GTEST_OS_LINUX_ANDROID && defined(_STLPORT_MAJOR) && \ + !defined(__EXCEPTIONS) +# define GTEST_HAS_RTTI 0 +# else +# define GTEST_HAS_RTTI 1 +# endif // GTEST_OS_LINUX_ANDROID && __STLPORT_MAJOR && !__EXCEPTIONS +# else +# define GTEST_HAS_RTTI 0 +# endif // __GXX_RTTI + +// Clang defines __GXX_RTTI starting with version 3.0, but its manual recommends +// using has_feature instead. has_feature(cxx_rtti) is supported since 2.7, the +// first version with C++ support. +# elif defined(__clang__) + +# define GTEST_HAS_RTTI __has_feature(cxx_rtti) + +// Starting with version 9.0 IBM Visual Age defines __RTTI_ALL__ to 1 if +// both the typeid and dynamic_cast features are present. +# elif defined(__IBMCPP__) && (__IBMCPP__ >= 900) + +# ifdef __RTTI_ALL__ +# define GTEST_HAS_RTTI 1 +# else +# define GTEST_HAS_RTTI 0 +# endif + +# else + +// For all other compilers, we assume RTTI is enabled. +# define GTEST_HAS_RTTI 1 + +# endif // _MSC_VER + +#endif // GTEST_HAS_RTTI + +// It's this header's responsibility to #include when RTTI +// is enabled. +#if GTEST_HAS_RTTI +# include +#endif + +// Determines whether Google Test can use the pthreads library. +#ifndef GTEST_HAS_PTHREAD +// The user didn't tell us explicitly, so we make reasonable assumptions about +// which platforms have pthreads support. +// +// To disable threading support in Google Test, add -DGTEST_HAS_PTHREAD=0 +// to your compiler flags. +# define GTEST_HAS_PTHREAD (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_HPUX \ + || GTEST_OS_QNX || GTEST_OS_FREEBSD || GTEST_OS_NACL) +#endif // GTEST_HAS_PTHREAD + +#if GTEST_HAS_PTHREAD +// gtest-port.h guarantees to #include when GTEST_HAS_PTHREAD is +// true. +# include // NOLINT + +// For timespec and nanosleep, used below. +# include // NOLINT +#endif + +// Determines if hash_map/hash_set are available. +// Only used for testing against those containers. +#if !defined(GTEST_HAS_HASH_MAP_) +# if _MSC_VER +# define GTEST_HAS_HASH_MAP_ 1 // Indicates that hash_map is available. +# define GTEST_HAS_HASH_SET_ 1 // Indicates that hash_set is available. +# endif // _MSC_VER +#endif // !defined(GTEST_HAS_HASH_MAP_) + +// Determines whether Google Test can use tr1/tuple. You can define +// this macro to 0 to prevent Google Test from using tuple (any +// feature depending on tuple with be disabled in this mode). +#ifndef GTEST_HAS_TR1_TUPLE +# if GTEST_OS_LINUX_ANDROID && defined(_STLPORT_MAJOR) +// STLport, provided with the Android NDK, has neither or . +# define GTEST_HAS_TR1_TUPLE 0 +# else +// The user didn't tell us not to do it, so we assume it's OK. +# define GTEST_HAS_TR1_TUPLE 1 +# endif +#endif // GTEST_HAS_TR1_TUPLE + +// Determines whether Google Test's own tr1 tuple implementation +// should be used. +#ifndef GTEST_USE_OWN_TR1_TUPLE +// The user didn't tell us, so we need to figure it out. + +// We use our own TR1 tuple if we aren't sure the user has an +// implementation of it already. At this time, libstdc++ 4.0.0+ and +// MSVC 2010 are the only mainstream standard libraries that come +// with a TR1 tuple implementation. NVIDIA's CUDA NVCC compiler +// pretends to be GCC by defining __GNUC__ and friends, but cannot +// compile GCC's tuple implementation. MSVC 2008 (9.0) provides TR1 +// tuple in a 323 MB Feature Pack download, which we cannot assume the +// user has. QNX's QCC compiler is a modified GCC but it doesn't +// support TR1 tuple. libc++ only provides std::tuple, in C++11 mode, +// and it can be used with some compilers that define __GNUC__. +# if (defined(__GNUC__) && !defined(__CUDACC__) && (GTEST_GCC_VER_ >= 40000) \ + && !GTEST_OS_QNX && !defined(_LIBCPP_VERSION)) || _MSC_VER >= 1600 +# define GTEST_ENV_HAS_TR1_TUPLE_ 1 +# endif + +// C++11 specifies that provides std::tuple. Use that if gtest is used +// in C++11 mode and libstdc++ isn't very old (binaries targeting OS X 10.6 +// can build with clang but need to use gcc4.2's libstdc++). +# if GTEST_LANG_CXX11 && (!defined(__GLIBCXX__) || __GLIBCXX__ > 20110325) +# define GTEST_ENV_HAS_STD_TUPLE_ 1 +# endif + +# if GTEST_ENV_HAS_TR1_TUPLE_ || GTEST_ENV_HAS_STD_TUPLE_ +# define GTEST_USE_OWN_TR1_TUPLE 0 +# else +# define GTEST_USE_OWN_TR1_TUPLE 1 +# endif + +#endif // GTEST_USE_OWN_TR1_TUPLE + +// To avoid conditional compilation everywhere, we make it +// gtest-port.h's responsibility to #include the header implementing +// tuple. +#if GTEST_HAS_STD_TUPLE_ +# include // IWYU pragma: export +# define GTEST_TUPLE_NAMESPACE_ ::std +#endif // GTEST_HAS_STD_TUPLE_ + +// We include tr1::tuple even if std::tuple is available to define printers for +// them. +#if GTEST_HAS_TR1_TUPLE +# ifndef GTEST_TUPLE_NAMESPACE_ +# define GTEST_TUPLE_NAMESPACE_ ::std::tr1 +# endif // GTEST_TUPLE_NAMESPACE_ + +# if GTEST_USE_OWN_TR1_TUPLE +# include "gtest/internal/gtest-tuple.h" // IWYU pragma: export // NOLINT +# elif GTEST_ENV_HAS_STD_TUPLE_ +# include +// C++11 puts its tuple into the ::std namespace rather than +// ::std::tr1. gtest expects tuple to live in ::std::tr1, so put it there. +// This causes undefined behavior, but supported compilers react in +// the way we intend. +namespace std { +namespace tr1 { +using ::std::get; +using ::std::make_tuple; +using ::std::tuple; +using ::std::tuple_element; +using ::std::tuple_size; +} +} + +# elif GTEST_OS_SYMBIAN + +// On Symbian, BOOST_HAS_TR1_TUPLE causes Boost's TR1 tuple library to +// use STLport's tuple implementation, which unfortunately doesn't +// work as the copy of STLport distributed with Symbian is incomplete. +// By making sure BOOST_HAS_TR1_TUPLE is undefined, we force Boost to +// use its own tuple implementation. +# ifdef BOOST_HAS_TR1_TUPLE +# undef BOOST_HAS_TR1_TUPLE +# endif // BOOST_HAS_TR1_TUPLE + +// This prevents , which defines +// BOOST_HAS_TR1_TUPLE, from being #included by Boost's . +# define BOOST_TR1_DETAIL_CONFIG_HPP_INCLUDED +# include // IWYU pragma: export // NOLINT + +# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40000) +// GCC 4.0+ implements tr1/tuple in the header. This does +// not conform to the TR1 spec, which requires the header to be . + +# if !GTEST_HAS_RTTI && GTEST_GCC_VER_ < 40302 +// Until version 4.3.2, gcc has a bug that causes , +// which is #included by , to not compile when RTTI is +// disabled. _TR1_FUNCTIONAL is the header guard for +// . Hence the following #define is a hack to prevent +// from being included. +# define _TR1_FUNCTIONAL 1 +# include +# undef _TR1_FUNCTIONAL // Allows the user to #include + // if he chooses to. +# else +# include // NOLINT +# endif // !GTEST_HAS_RTTI && GTEST_GCC_VER_ < 40302 + +# else +// If the compiler is not GCC 4.0+, we assume the user is using a +// spec-conforming TR1 implementation. +# include // IWYU pragma: export // NOLINT +# endif // GTEST_USE_OWN_TR1_TUPLE + +#endif // GTEST_HAS_TR1_TUPLE + +// Determines whether clone(2) is supported. +// Usually it will only be available on Linux, excluding +// Linux on the Itanium architecture. +// Also see http://linux.die.net/man/2/clone. +#ifndef GTEST_HAS_CLONE +// The user didn't tell us, so we need to figure it out. + +# if GTEST_OS_LINUX && !defined(__ia64__) +# if GTEST_OS_LINUX_ANDROID +// On Android, clone() is only available on ARM starting with Gingerbread. +# if defined(__arm__) && __ANDROID_API__ >= 9 +# define GTEST_HAS_CLONE 1 +# else +# define GTEST_HAS_CLONE 0 +# endif +# else +# define GTEST_HAS_CLONE 1 +# endif +# else +# define GTEST_HAS_CLONE 0 +# endif // GTEST_OS_LINUX && !defined(__ia64__) + +#endif // GTEST_HAS_CLONE + +// Determines whether to support stream redirection. This is used to test +// output correctness and to implement death tests. +#ifndef GTEST_HAS_STREAM_REDIRECTION +// By default, we assume that stream redirection is supported on all +// platforms except known mobile ones. +# if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || \ + GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT +# define GTEST_HAS_STREAM_REDIRECTION 0 +# else +# define GTEST_HAS_STREAM_REDIRECTION 1 +# endif // !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_SYMBIAN +#endif // GTEST_HAS_STREAM_REDIRECTION + +// Determines whether to support death tests. +// Google Test does not support death tests for VC 7.1 and earlier as +// abort() in a VC 7.1 application compiled as GUI in debug config +// pops up a dialog window that cannot be suppressed programmatically. +#if (GTEST_OS_LINUX || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \ + (GTEST_OS_MAC && !GTEST_OS_IOS) || \ + (GTEST_OS_WINDOWS_DESKTOP && _MSC_VER >= 1400) || \ + GTEST_OS_WINDOWS_MINGW || GTEST_OS_AIX || GTEST_OS_HPUX || \ + GTEST_OS_OPENBSD || GTEST_OS_QNX || GTEST_OS_FREEBSD) +# define GTEST_HAS_DEATH_TEST 1 +#endif + +// We don't support MSVC 7.1 with exceptions disabled now. Therefore +// all the compilers we care about are adequate for supporting +// value-parameterized tests. +#define GTEST_HAS_PARAM_TEST 1 + +// Determines whether to support type-driven tests. + +// Typed tests need and variadic macros, which GCC, VC++ 8.0, +// Sun Pro CC, IBM Visual Age, and HP aCC support. +#if defined(__GNUC__) || (_MSC_VER >= 1400) || defined(__SUNPRO_CC) || \ + defined(__IBMCPP__) || defined(__HP_aCC) +# define GTEST_HAS_TYPED_TEST 1 +# define GTEST_HAS_TYPED_TEST_P 1 +#endif + +// Determines whether to support Combine(). This only makes sense when +// value-parameterized tests are enabled. The implementation doesn't +// work on Sun Studio since it doesn't understand templated conversion +// operators. +#if GTEST_HAS_PARAM_TEST && GTEST_HAS_TR1_TUPLE && !defined(__SUNPRO_CC) +# define GTEST_HAS_COMBINE 1 +#endif + +// Determines whether the system compiler uses UTF-16 for encoding wide strings. +#define GTEST_WIDE_STRING_USES_UTF16_ \ + (GTEST_OS_WINDOWS || GTEST_OS_CYGWIN || GTEST_OS_SYMBIAN || GTEST_OS_AIX) + +// Determines whether test results can be streamed to a socket. +#if GTEST_OS_LINUX +# define GTEST_CAN_STREAM_RESULTS_ 1 +#endif + +// Defines some utility macros. + +// The GNU compiler emits a warning if nested "if" statements are followed by +// an "else" statement and braces are not used to explicitly disambiguate the +// "else" binding. This leads to problems with code like: +// +// if (gate) +// ASSERT_*(condition) << "Some message"; +// +// The "switch (0) case 0:" idiom is used to suppress this. +#ifdef __INTEL_COMPILER +# define GTEST_AMBIGUOUS_ELSE_BLOCKER_ +#else +# define GTEST_AMBIGUOUS_ELSE_BLOCKER_ switch (0) case 0: default: // NOLINT +#endif + +// Use this annotation at the end of a struct/class definition to +// prevent the compiler from optimizing away instances that are never +// used. This is useful when all interesting logic happens inside the +// c'tor and / or d'tor. Example: +// +// struct Foo { +// Foo() { ... } +// } GTEST_ATTRIBUTE_UNUSED_; +// +// Also use it after a variable or parameter declaration to tell the +// compiler the variable/parameter does not have to be used. +#if defined(__GNUC__) && !defined(COMPILER_ICC) +# define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused)) +#elif defined(__clang__) +# if __has_attribute(unused) +# define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused)) +# endif +#endif +#ifndef GTEST_ATTRIBUTE_UNUSED_ +# define GTEST_ATTRIBUTE_UNUSED_ +#endif + +// A macro to disallow operator= +// This should be used in the private: declarations for a class. +#define GTEST_DISALLOW_ASSIGN_(type)\ + void operator=(type const &) + +// A macro to disallow copy constructor and operator= +// This should be used in the private: declarations for a class. +#define GTEST_DISALLOW_COPY_AND_ASSIGN_(type)\ + type(type const &);\ + GTEST_DISALLOW_ASSIGN_(type) + +// Tell the compiler to warn about unused return values for functions declared +// with this macro. The macro should be used on function declarations +// following the argument list: +// +// Sprocket* AllocateSprocket() GTEST_MUST_USE_RESULT_; +#if defined(__GNUC__) && (GTEST_GCC_VER_ >= 30400) && !defined(COMPILER_ICC) +# define GTEST_MUST_USE_RESULT_ __attribute__ ((warn_unused_result)) +#else +# define GTEST_MUST_USE_RESULT_ +#endif // __GNUC__ && (GTEST_GCC_VER_ >= 30400) && !COMPILER_ICC + +// MS C++ compiler emits warning when a conditional expression is compile time +// constant. In some contexts this warning is false positive and needs to be +// suppressed. Use the following two macros in such cases: +// +// GTEST_INTENTIONAL_CONST_COND_PUSH_() +// while (true) { +// GTEST_INTENTIONAL_CONST_COND_POP_() +// } +# define GTEST_INTENTIONAL_CONST_COND_PUSH_() \ + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4127) +# define GTEST_INTENTIONAL_CONST_COND_POP_() \ + GTEST_DISABLE_MSC_WARNINGS_POP_() + +// Determine whether the compiler supports Microsoft's Structured Exception +// Handling. This is supported by several Windows compilers but generally +// does not exist on any other system. +#ifndef GTEST_HAS_SEH +// The user didn't tell us, so we need to figure it out. + +# if defined(_MSC_VER) || defined(__BORLANDC__) +// These two compilers are known to support SEH. +# define GTEST_HAS_SEH 1 +# else +// Assume no SEH. +# define GTEST_HAS_SEH 0 +# endif + +#define GTEST_IS_THREADSAFE \ + (GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ \ + || (GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT) \ + || GTEST_HAS_PTHREAD) + +#endif // GTEST_HAS_SEH + +#ifdef _MSC_VER +# if GTEST_LINKED_AS_SHARED_LIBRARY +# define GTEST_API_ __declspec(dllimport) +# elif GTEST_CREATE_SHARED_LIBRARY +# define GTEST_API_ __declspec(dllexport) +# endif +#elif __GNUC__ >= 4 || defined(__clang__) +# define GTEST_API_ __attribute__((visibility ("default"))) +#endif // _MSC_VER + +#ifndef GTEST_API_ +# define GTEST_API_ +#endif + +#ifdef __GNUC__ +// Ask the compiler to never inline a given function. +# define GTEST_NO_INLINE_ __attribute__((noinline)) +#else +# define GTEST_NO_INLINE_ +#endif + +// _LIBCPP_VERSION is defined by the libc++ library from the LLVM project. +#if defined(__GLIBCXX__) || defined(_LIBCPP_VERSION) +# define GTEST_HAS_CXXABI_H_ 1 +#else +# define GTEST_HAS_CXXABI_H_ 0 +#endif + +// A function level attribute to disable checking for use of uninitialized +// memory when built with MemorySanitizer. +#if defined(__clang__) +# if __has_feature(memory_sanitizer) +# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ \ + __attribute__((no_sanitize_memory)) +# else +# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +# endif // __has_feature(memory_sanitizer) +#else +# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +#endif // __clang__ + +// A function level attribute to disable AddressSanitizer instrumentation. +#if defined(__clang__) +# if __has_feature(address_sanitizer) +# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ \ + __attribute__((no_sanitize_address)) +# else +# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +# endif // __has_feature(address_sanitizer) +#else +# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +#endif // __clang__ + +// A function level attribute to disable ThreadSanitizer instrumentation. +#if defined(__clang__) +# if __has_feature(thread_sanitizer) +# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ \ + __attribute__((no_sanitize_thread)) +# else +# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +# endif // __has_feature(thread_sanitizer) +#else +# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +#endif // __clang__ + +namespace testing { + +class Message; + +#if defined(GTEST_TUPLE_NAMESPACE_) +// Import tuple and friends into the ::testing namespace. +// It is part of our interface, having them in ::testing allows us to change +// their types as needed. +using GTEST_TUPLE_NAMESPACE_::get; +using GTEST_TUPLE_NAMESPACE_::make_tuple; +using GTEST_TUPLE_NAMESPACE_::tuple; +using GTEST_TUPLE_NAMESPACE_::tuple_size; +using GTEST_TUPLE_NAMESPACE_::tuple_element; +#endif // defined(GTEST_TUPLE_NAMESPACE_) + +namespace internal { + +// A secret type that Google Test users don't know about. It has no +// definition on purpose. Therefore it's impossible to create a +// Secret object, which is what we want. +class Secret; + +// The GTEST_COMPILE_ASSERT_ macro can be used to verify that a compile time +// expression is true. For example, you could use it to verify the +// size of a static array: +// +// GTEST_COMPILE_ASSERT_(GTEST_ARRAY_SIZE_(names) == NUM_NAMES, +// names_incorrect_size); +// +// or to make sure a struct is smaller than a certain size: +// +// GTEST_COMPILE_ASSERT_(sizeof(foo) < 128, foo_too_large); +// +// The second argument to the macro is the name of the variable. If +// the expression is false, most compilers will issue a warning/error +// containing the name of the variable. + +#if GTEST_LANG_CXX11 +# define GTEST_COMPILE_ASSERT_(expr, msg) static_assert(expr, #msg) +#else // !GTEST_LANG_CXX11 +template + struct CompileAssert { +}; + +# define GTEST_COMPILE_ASSERT_(expr, msg) \ + typedef ::testing::internal::CompileAssert<(static_cast(expr))> \ + msg[static_cast(expr) ? 1 : -1] GTEST_ATTRIBUTE_UNUSED_ +#endif // !GTEST_LANG_CXX11 + +// Implementation details of GTEST_COMPILE_ASSERT_: +// +// (In C++11, we simply use static_assert instead of the following) +// +// - GTEST_COMPILE_ASSERT_ works by defining an array type that has -1 +// elements (and thus is invalid) when the expression is false. +// +// - The simpler definition +// +// #define GTEST_COMPILE_ASSERT_(expr, msg) typedef char msg[(expr) ? 1 : -1] +// +// does not work, as gcc supports variable-length arrays whose sizes +// are determined at run-time (this is gcc's extension and not part +// of the C++ standard). As a result, gcc fails to reject the +// following code with the simple definition: +// +// int foo; +// GTEST_COMPILE_ASSERT_(foo, msg); // not supposed to compile as foo is +// // not a compile-time constant. +// +// - By using the type CompileAssert<(bool(expr))>, we ensures that +// expr is a compile-time constant. (Template arguments must be +// determined at compile-time.) +// +// - The outter parentheses in CompileAssert<(bool(expr))> are necessary +// to work around a bug in gcc 3.4.4 and 4.0.1. If we had written +// +// CompileAssert +// +// instead, these compilers will refuse to compile +// +// GTEST_COMPILE_ASSERT_(5 > 0, some_message); +// +// (They seem to think the ">" in "5 > 0" marks the end of the +// template argument list.) +// +// - The array size is (bool(expr) ? 1 : -1), instead of simply +// +// ((expr) ? 1 : -1). +// +// This is to avoid running into a bug in MS VC 7.1, which +// causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1. + +// StaticAssertTypeEqHelper is used by StaticAssertTypeEq defined in gtest.h. +// +// This template is declared, but intentionally undefined. +template +struct StaticAssertTypeEqHelper; + +template +struct StaticAssertTypeEqHelper { + enum { value = true }; +}; + +// Evaluates to the number of elements in 'array'. +#define GTEST_ARRAY_SIZE_(array) (sizeof(array) / sizeof(array[0])) + +#if GTEST_HAS_GLOBAL_STRING +typedef ::string string; +#else +typedef ::std::string string; +#endif // GTEST_HAS_GLOBAL_STRING + +#if GTEST_HAS_GLOBAL_WSTRING +typedef ::wstring wstring; +#elif GTEST_HAS_STD_WSTRING +typedef ::std::wstring wstring; +#endif // GTEST_HAS_GLOBAL_WSTRING + +// A helper for suppressing warnings on constant condition. It just +// returns 'condition'. +GTEST_API_ bool IsTrue(bool condition); + +// Defines scoped_ptr. + +// This implementation of scoped_ptr is PARTIAL - it only contains +// enough stuff to satisfy Google Test's need. +template +class scoped_ptr { + public: + typedef T element_type; + + explicit scoped_ptr(T* p = NULL) : ptr_(p) {} + ~scoped_ptr() { reset(); } + + T& operator*() const { return *ptr_; } + T* operator->() const { return ptr_; } + T* get() const { return ptr_; } + + T* release() { + T* const ptr = ptr_; + ptr_ = NULL; + return ptr; + } + + void reset(T* p = NULL) { + if (p != ptr_) { + if (IsTrue(sizeof(T) > 0)) { // Makes sure T is a complete type. + delete ptr_; + } + ptr_ = p; + } + } + + friend void swap(scoped_ptr& a, scoped_ptr& b) { + using std::swap; + swap(a.ptr_, b.ptr_); + } + + private: + T* ptr_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(scoped_ptr); +}; + +// Defines RE. + +// A simple C++ wrapper for . It uses the POSIX Extended +// Regular Expression syntax. +class GTEST_API_ RE { + public: + // A copy constructor is required by the Standard to initialize object + // references from r-values. + RE(const RE& other) { Init(other.pattern()); } + + // Constructs an RE from a string. + RE(const ::std::string& regex) { Init(regex.c_str()); } // NOLINT + +#if GTEST_HAS_GLOBAL_STRING + + RE(const ::string& regex) { Init(regex.c_str()); } // NOLINT + +#endif // GTEST_HAS_GLOBAL_STRING + + RE(const char* regex) { Init(regex); } // NOLINT + ~RE(); + + // Returns the string representation of the regex. + const char* pattern() const { return pattern_; } + + // FullMatch(str, re) returns true iff regular expression re matches + // the entire str. + // PartialMatch(str, re) returns true iff regular expression re + // matches a substring of str (including str itself). + // + // TODO(wan@google.com): make FullMatch() and PartialMatch() work + // when str contains NUL characters. + static bool FullMatch(const ::std::string& str, const RE& re) { + return FullMatch(str.c_str(), re); + } + static bool PartialMatch(const ::std::string& str, const RE& re) { + return PartialMatch(str.c_str(), re); + } + +#if GTEST_HAS_GLOBAL_STRING + + static bool FullMatch(const ::string& str, const RE& re) { + return FullMatch(str.c_str(), re); + } + static bool PartialMatch(const ::string& str, const RE& re) { + return PartialMatch(str.c_str(), re); + } + +#endif // GTEST_HAS_GLOBAL_STRING + + static bool FullMatch(const char* str, const RE& re); + static bool PartialMatch(const char* str, const RE& re); + + private: + void Init(const char* regex); + + // We use a const char* instead of an std::string, as Google Test used to be + // used where std::string is not available. TODO(wan@google.com): change to + // std::string. + const char* pattern_; + bool is_valid_; + +#if GTEST_USES_POSIX_RE + + regex_t full_regex_; // For FullMatch(). + regex_t partial_regex_; // For PartialMatch(). + +#else // GTEST_USES_SIMPLE_RE + + const char* full_pattern_; // For FullMatch(); + +#endif + + GTEST_DISALLOW_ASSIGN_(RE); +}; + +// Formats a source file path and a line number as they would appear +// in an error message from the compiler used to compile this code. +GTEST_API_ ::std::string FormatFileLocation(const char* file, int line); + +// Formats a file location for compiler-independent XML output. +// Although this function is not platform dependent, we put it next to +// FormatFileLocation in order to contrast the two functions. +GTEST_API_ ::std::string FormatCompilerIndependentFileLocation(const char* file, + int line); + +// Defines logging utilities: +// GTEST_LOG_(severity) - logs messages at the specified severity level. The +// message itself is streamed into the macro. +// LogToStderr() - directs all log messages to stderr. +// FlushInfoLog() - flushes informational log messages. + +enum GTestLogSeverity { + GTEST_INFO, + GTEST_WARNING, + GTEST_ERROR, + GTEST_FATAL +}; + +// Formats log entry severity, provides a stream object for streaming the +// log message, and terminates the message with a newline when going out of +// scope. +class GTEST_API_ GTestLog { + public: + GTestLog(GTestLogSeverity severity, const char* file, int line); + + // Flushes the buffers and, if severity is GTEST_FATAL, aborts the program. + ~GTestLog(); + + ::std::ostream& GetStream() { return ::std::cerr; } + + private: + const GTestLogSeverity severity_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestLog); +}; + +#if !defined(GTEST_LOG_) + +# define GTEST_LOG_(severity) \ + ::testing::internal::GTestLog(::testing::internal::GTEST_##severity, \ + __FILE__, __LINE__).GetStream() + +inline void LogToStderr() {} +inline void FlushInfoLog() { fflush(NULL); } + +#endif // !defined(GTEST_LOG_) + +#if !defined(GTEST_CHECK_) +// INTERNAL IMPLEMENTATION - DO NOT USE. +// +// GTEST_CHECK_ is an all-mode assert. It aborts the program if the condition +// is not satisfied. +// Synopsys: +// GTEST_CHECK_(boolean_condition); +// or +// GTEST_CHECK_(boolean_condition) << "Additional message"; +// +// This checks the condition and if the condition is not satisfied +// it prints message about the condition violation, including the +// condition itself, plus additional message streamed into it, if any, +// and then it aborts the program. It aborts the program irrespective of +// whether it is built in the debug mode or not. +# define GTEST_CHECK_(condition) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::IsTrue(condition)) \ + ; \ + else \ + GTEST_LOG_(FATAL) << "Condition " #condition " failed. " +#endif // !defined(GTEST_CHECK_) + +// An all-mode assert to verify that the given POSIX-style function +// call returns 0 (indicating success). Known limitation: this +// doesn't expand to a balanced 'if' statement, so enclose the macro +// in {} if you need to use it as the only statement in an 'if' +// branch. +#define GTEST_CHECK_POSIX_SUCCESS_(posix_call) \ + if (const int gtest_error = (posix_call)) \ + GTEST_LOG_(FATAL) << #posix_call << "failed with error " \ + << gtest_error + +#if GTEST_HAS_STD_MOVE_ +using std::move; +#else // GTEST_HAS_STD_MOVE_ +template +const T& move(const T& t) { + return t; +} +#endif // GTEST_HAS_STD_MOVE_ + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Use ImplicitCast_ as a safe version of static_cast for upcasting in +// the type hierarchy (e.g. casting a Foo* to a SuperclassOfFoo* or a +// const Foo*). When you use ImplicitCast_, the compiler checks that +// the cast is safe. Such explicit ImplicitCast_s are necessary in +// surprisingly many situations where C++ demands an exact type match +// instead of an argument type convertable to a target type. +// +// The syntax for using ImplicitCast_ is the same as for static_cast: +// +// ImplicitCast_(expr) +// +// ImplicitCast_ would have been part of the C++ standard library, +// but the proposal was submitted too late. It will probably make +// its way into the language in the future. +// +// This relatively ugly name is intentional. It prevents clashes with +// similar functions users may have (e.g., implicit_cast). The internal +// namespace alone is not enough because the function can be found by ADL. +template +inline To ImplicitCast_(To x) { return x; } + +// When you upcast (that is, cast a pointer from type Foo to type +// SuperclassOfFoo), it's fine to use ImplicitCast_<>, since upcasts +// always succeed. When you downcast (that is, cast a pointer from +// type Foo to type SubclassOfFoo), static_cast<> isn't safe, because +// how do you know the pointer is really of type SubclassOfFoo? It +// could be a bare Foo, or of type DifferentSubclassOfFoo. Thus, +// when you downcast, you should use this macro. In debug mode, we +// use dynamic_cast<> to double-check the downcast is legal (we die +// if it's not). In normal mode, we do the efficient static_cast<> +// instead. Thus, it's important to test in debug mode to make sure +// the cast is legal! +// This is the only place in the code we should use dynamic_cast<>. +// In particular, you SHOULDN'T be using dynamic_cast<> in order to +// do RTTI (eg code like this: +// if (dynamic_cast(foo)) HandleASubclass1Object(foo); +// if (dynamic_cast(foo)) HandleASubclass2Object(foo); +// You should design the code some other way not to need this. +// +// This relatively ugly name is intentional. It prevents clashes with +// similar functions users may have (e.g., down_cast). The internal +// namespace alone is not enough because the function can be found by ADL. +template // use like this: DownCast_(foo); +inline To DownCast_(From* f) { // so we only accept pointers + // Ensures that To is a sub-type of From *. This test is here only + // for compile-time type checking, and has no overhead in an + // optimized build at run-time, as it will be optimized away + // completely. + GTEST_INTENTIONAL_CONST_COND_PUSH_() + if (false) { + GTEST_INTENTIONAL_CONST_COND_POP_() + const To to = NULL; + ::testing::internal::ImplicitCast_(to); + } + +#if GTEST_HAS_RTTI + // RTTI: debug mode only! + GTEST_CHECK_(f == NULL || dynamic_cast(f) != NULL); +#endif + return static_cast(f); +} + +// Downcasts the pointer of type Base to Derived. +// Derived must be a subclass of Base. The parameter MUST +// point to a class of type Derived, not any subclass of it. +// When RTTI is available, the function performs a runtime +// check to enforce this. +template +Derived* CheckedDowncastToActualType(Base* base) { +#if GTEST_HAS_RTTI + GTEST_CHECK_(typeid(*base) == typeid(Derived)); +#endif + +#if GTEST_HAS_DOWNCAST_ + return ::down_cast(base); +#elif GTEST_HAS_RTTI + return dynamic_cast(base); // NOLINT +#else + return static_cast(base); // Poor man's downcast. +#endif +} + +#if GTEST_HAS_STREAM_REDIRECTION + +// Defines the stderr capturer: +// CaptureStdout - starts capturing stdout. +// GetCapturedStdout - stops capturing stdout and returns the captured string. +// CaptureStderr - starts capturing stderr. +// GetCapturedStderr - stops capturing stderr and returns the captured string. +// +GTEST_API_ void CaptureStdout(); +GTEST_API_ std::string GetCapturedStdout(); +GTEST_API_ void CaptureStderr(); +GTEST_API_ std::string GetCapturedStderr(); + +#endif // GTEST_HAS_STREAM_REDIRECTION + +// Returns a path to temporary directory. +GTEST_API_ std::string TempDir(); + +// Returns the size (in bytes) of a file. +GTEST_API_ size_t GetFileSize(FILE* file); + +// Reads the entire content of a file as a string. +GTEST_API_ std::string ReadEntireFile(FILE* file); + +// All command line arguments. +GTEST_API_ const ::std::vector& GetArgvs(); + +#if GTEST_HAS_DEATH_TEST + +const ::std::vector& GetInjectableArgvs(); +void SetInjectableArgvs(const ::std::vector* + new_argvs); + + +#endif // GTEST_HAS_DEATH_TEST + +// Defines synchronization primitives. +#if GTEST_IS_THREADSAFE +# if GTEST_HAS_PTHREAD +// Sleeps for (roughly) n milliseconds. This function is only for testing +// Google Test's own constructs. Don't use it in user tests, either +// directly or indirectly. +inline void SleepMilliseconds(int n) { + const timespec time = { + 0, // 0 seconds. + n * 1000L * 1000L, // And n ms. + }; + nanosleep(&time, NULL); +} +# endif // GTEST_HAS_PTHREAD + +# if GTEST_HAS_NOTIFICATION_ +// Notification has already been imported into the namespace. +// Nothing to do here. + +# elif GTEST_HAS_PTHREAD +// Allows a controller thread to pause execution of newly created +// threads until notified. Instances of this class must be created +// and destroyed in the controller thread. +// +// This class is only for testing Google Test's own constructs. Do not +// use it in user tests, either directly or indirectly. +class Notification { + public: + Notification() : notified_(false) { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, NULL)); + } + ~Notification() { + pthread_mutex_destroy(&mutex_); + } + + // Notifies all threads created with this notification to start. Must + // be called from the controller thread. + void Notify() { + pthread_mutex_lock(&mutex_); + notified_ = true; + pthread_mutex_unlock(&mutex_); + } + + // Blocks until the controller thread notifies. Must be called from a test + // thread. + void WaitForNotification() { + for (;;) { + pthread_mutex_lock(&mutex_); + const bool notified = notified_; + pthread_mutex_unlock(&mutex_); + if (notified) + break; + SleepMilliseconds(10); + } + } + + private: + pthread_mutex_t mutex_; + bool notified_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification); +}; + +# elif GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + +GTEST_API_ void SleepMilliseconds(int n); + +// Provides leak-safe Windows kernel handle ownership. +// Used in death tests and in threading support. +class GTEST_API_ AutoHandle { + public: + // Assume that Win32 HANDLE type is equivalent to void*. Doing so allows us to + // avoid including in this header file. Including is + // undesirable because it defines a lot of symbols and macros that tend to + // conflict with client code. This assumption is verified by + // WindowsTypesTest.HANDLEIsVoidStar. + typedef void* Handle; + AutoHandle(); + explicit AutoHandle(Handle handle); + + ~AutoHandle(); + + Handle Get() const; + void Reset(); + void Reset(Handle handle); + + private: + // Returns true iff the handle is a valid handle object that can be closed. + bool IsCloseable() const; + + Handle handle_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(AutoHandle); +}; + +// Allows a controller thread to pause execution of newly created +// threads until notified. Instances of this class must be created +// and destroyed in the controller thread. +// +// This class is only for testing Google Test's own constructs. Do not +// use it in user tests, either directly or indirectly. +class GTEST_API_ Notification { + public: + Notification(); + void Notify(); + void WaitForNotification(); + + private: + AutoHandle event_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification); +}; +# endif // GTEST_HAS_NOTIFICATION_ + +// On MinGW, we can have both GTEST_OS_WINDOWS and GTEST_HAS_PTHREAD +// defined, but we don't want to use MinGW's pthreads implementation, which +// has conformance problems with some versions of the POSIX standard. +# if GTEST_HAS_PTHREAD && !GTEST_OS_WINDOWS_MINGW + +// As a C-function, ThreadFuncWithCLinkage cannot be templated itself. +// Consequently, it cannot select a correct instantiation of ThreadWithParam +// in order to call its Run(). Introducing ThreadWithParamBase as a +// non-templated base class for ThreadWithParam allows us to bypass this +// problem. +class ThreadWithParamBase { + public: + virtual ~ThreadWithParamBase() {} + virtual void Run() = 0; +}; + +// pthread_create() accepts a pointer to a function type with the C linkage. +// According to the Standard (7.5/1), function types with different linkages +// are different even if they are otherwise identical. Some compilers (for +// example, SunStudio) treat them as different types. Since class methods +// cannot be defined with C-linkage we need to define a free C-function to +// pass into pthread_create(). +extern "C" inline void* ThreadFuncWithCLinkage(void* thread) { + static_cast(thread)->Run(); + return NULL; +} + +// Helper class for testing Google Test's multi-threading constructs. +// To use it, write: +// +// void ThreadFunc(int param) { /* Do things with param */ } +// Notification thread_can_start; +// ... +// // The thread_can_start parameter is optional; you can supply NULL. +// ThreadWithParam thread(&ThreadFunc, 5, &thread_can_start); +// thread_can_start.Notify(); +// +// These classes are only for testing Google Test's own constructs. Do +// not use them in user tests, either directly or indirectly. +template +class ThreadWithParam : public ThreadWithParamBase { + public: + typedef void UserThreadFunc(T); + + ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start) + : func_(func), + param_(param), + thread_can_start_(thread_can_start), + finished_(false) { + ThreadWithParamBase* const base = this; + // The thread can be created only after all fields except thread_ + // have been initialized. + GTEST_CHECK_POSIX_SUCCESS_( + pthread_create(&thread_, 0, &ThreadFuncWithCLinkage, base)); + } + ~ThreadWithParam() { Join(); } + + void Join() { + if (!finished_) { + GTEST_CHECK_POSIX_SUCCESS_(pthread_join(thread_, 0)); + finished_ = true; + } + } + + virtual void Run() { + if (thread_can_start_ != NULL) + thread_can_start_->WaitForNotification(); + func_(param_); + } + + private: + UserThreadFunc* const func_; // User-supplied thread function. + const T param_; // User-supplied parameter to the thread function. + // When non-NULL, used to block execution until the controller thread + // notifies. + Notification* const thread_can_start_; + bool finished_; // true iff we know that the thread function has finished. + pthread_t thread_; // The native thread object. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam); +}; +# endif // !GTEST_OS_WINDOWS && GTEST_HAS_PTHREAD || + // GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ + +# if GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ +// Mutex and ThreadLocal have already been imported into the namespace. +// Nothing to do here. + +# elif GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + +// Mutex implements mutex on Windows platforms. It is used in conjunction +// with class MutexLock: +// +// Mutex mutex; +// ... +// MutexLock lock(&mutex); // Acquires the mutex and releases it at the +// // end of the current scope. +// +// A static Mutex *must* be defined or declared using one of the following +// macros: +// GTEST_DEFINE_STATIC_MUTEX_(g_some_mutex); +// GTEST_DECLARE_STATIC_MUTEX_(g_some_mutex); +// +// (A non-static Mutex is defined/declared in the usual way). +class GTEST_API_ Mutex { + public: + enum MutexType { kStatic = 0, kDynamic = 1 }; + // We rely on kStaticMutex being 0 as it is to what the linker initializes + // type_ in static mutexes. critical_section_ will be initialized lazily + // in ThreadSafeLazyInit(). + enum StaticConstructorSelector { kStaticMutex = 0 }; + + // This constructor intentionally does nothing. It relies on type_ being + // statically initialized to 0 (effectively setting it to kStatic) and on + // ThreadSafeLazyInit() to lazily initialize the rest of the members. + explicit Mutex(StaticConstructorSelector /*dummy*/) {} + + Mutex(); + ~Mutex(); + + void Lock(); + + void Unlock(); + + // Does nothing if the current thread holds the mutex. Otherwise, crashes + // with high probability. + void AssertHeld(); + + private: + // Initializes owner_thread_id_ and critical_section_ in static mutexes. + void ThreadSafeLazyInit(); + + // Per http://blogs.msdn.com/b/oldnewthing/archive/2004/02/23/78395.aspx, + // we assume that 0 is an invalid value for thread IDs. + unsigned int owner_thread_id_; + + // For static mutexes, we rely on these members being initialized to zeros + // by the linker. + MutexType type_; + long critical_section_init_phase_; // NOLINT + _RTL_CRITICAL_SECTION* critical_section_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex); +}; + +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + extern ::testing::internal::Mutex mutex + +# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ + ::testing::internal::Mutex mutex(::testing::internal::Mutex::kStaticMutex) + +// We cannot name this class MutexLock because the ctor declaration would +// conflict with a macro named MutexLock, which is defined on some +// platforms. That macro is used as a defensive measure to prevent against +// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than +// "MutexLock l(&mu)". Hence the typedef trick below. +class GTestMutexLock { + public: + explicit GTestMutexLock(Mutex* mutex) + : mutex_(mutex) { mutex_->Lock(); } + + ~GTestMutexLock() { mutex_->Unlock(); } + + private: + Mutex* const mutex_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock); +}; + +typedef GTestMutexLock MutexLock; + +// Base class for ValueHolder. Allows a caller to hold and delete a value +// without knowing its type. +class ThreadLocalValueHolderBase { + public: + virtual ~ThreadLocalValueHolderBase() {} +}; + +// Provides a way for a thread to send notifications to a ThreadLocal +// regardless of its parameter type. +class ThreadLocalBase { + public: + // Creates a new ValueHolder object holding a default value passed to + // this ThreadLocal's constructor and returns it. It is the caller's + // responsibility not to call this when the ThreadLocal instance already + // has a value on the current thread. + virtual ThreadLocalValueHolderBase* NewValueForCurrentThread() const = 0; + + protected: + ThreadLocalBase() {} + virtual ~ThreadLocalBase() {} + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocalBase); +}; + +// Maps a thread to a set of ThreadLocals that have values instantiated on that +// thread and notifies them when the thread exits. A ThreadLocal instance is +// expected to persist until all threads it has values on have terminated. +class GTEST_API_ ThreadLocalRegistry { + public: + // Registers thread_local_instance as having value on the current thread. + // Returns a value that can be used to identify the thread from other threads. + static ThreadLocalValueHolderBase* GetValueOnCurrentThread( + const ThreadLocalBase* thread_local_instance); + + // Invoked when a ThreadLocal instance is destroyed. + static void OnThreadLocalDestroyed( + const ThreadLocalBase* thread_local_instance); +}; + +class GTEST_API_ ThreadWithParamBase { + public: + void Join(); + + protected: + class Runnable { + public: + virtual ~Runnable() {} + virtual void Run() = 0; + }; + + ThreadWithParamBase(Runnable *runnable, Notification* thread_can_start); + virtual ~ThreadWithParamBase(); + + private: + AutoHandle thread_; +}; + +// Helper class for testing Google Test's multi-threading constructs. +template +class ThreadWithParam : public ThreadWithParamBase { + public: + typedef void UserThreadFunc(T); + + ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start) + : ThreadWithParamBase(new RunnableImpl(func, param), thread_can_start) { + } + virtual ~ThreadWithParam() {} + + private: + class RunnableImpl : public Runnable { + public: + RunnableImpl(UserThreadFunc* func, T param) + : func_(func), + param_(param) { + } + virtual ~RunnableImpl() {} + virtual void Run() { + func_(param_); + } + + private: + UserThreadFunc* const func_; + const T param_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(RunnableImpl); + }; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam); +}; + +// Implements thread-local storage on Windows systems. +// +// // Thread 1 +// ThreadLocal tl(100); // 100 is the default value for each thread. +// +// // Thread 2 +// tl.set(150); // Changes the value for thread 2 only. +// EXPECT_EQ(150, tl.get()); +// +// // Thread 1 +// EXPECT_EQ(100, tl.get()); // In thread 1, tl has the original value. +// tl.set(200); +// EXPECT_EQ(200, tl.get()); +// +// The template type argument T must have a public copy constructor. +// In addition, the default ThreadLocal constructor requires T to have +// a public default constructor. +// +// The users of a TheadLocal instance have to make sure that all but one +// threads (including the main one) using that instance have exited before +// destroying it. Otherwise, the per-thread objects managed for them by the +// ThreadLocal instance are not guaranteed to be destroyed on all platforms. +// +// Google Test only uses global ThreadLocal objects. That means they +// will die after main() has returned. Therefore, no per-thread +// object managed by Google Test will be leaked as long as all threads +// using Google Test have exited when main() returns. +template +class ThreadLocal : public ThreadLocalBase { + public: + ThreadLocal() : default_factory_(new DefaultValueHolderFactory()) {} + explicit ThreadLocal(const T& value) + : default_factory_(new InstanceValueHolderFactory(value)) {} + + ~ThreadLocal() { ThreadLocalRegistry::OnThreadLocalDestroyed(this); } + + T* pointer() { return GetOrCreateValue(); } + const T* pointer() const { return GetOrCreateValue(); } + const T& get() const { return *pointer(); } + void set(const T& value) { *pointer() = value; } + + private: + // Holds a value of T. Can be deleted via its base class without the caller + // knowing the type of T. + class ValueHolder : public ThreadLocalValueHolderBase { + public: + ValueHolder() : value_() {} + explicit ValueHolder(const T& value) : value_(value) {} + + T* pointer() { return &value_; } + + private: + T value_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder); + }; + + + T* GetOrCreateValue() const { + return static_cast( + ThreadLocalRegistry::GetValueOnCurrentThread(this))->pointer(); + } + + virtual ThreadLocalValueHolderBase* NewValueForCurrentThread() const { + return default_factory_->MakeNewHolder(); + } + + class ValueHolderFactory { + public: + ValueHolderFactory() {} + virtual ~ValueHolderFactory() {} + virtual ValueHolder* MakeNewHolder() const = 0; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolderFactory); + }; + + class DefaultValueHolderFactory : public ValueHolderFactory { + public: + DefaultValueHolderFactory() {} + virtual ValueHolder* MakeNewHolder() const { return new ValueHolder(); } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultValueHolderFactory); + }; + + class InstanceValueHolderFactory : public ValueHolderFactory { + public: + explicit InstanceValueHolderFactory(const T& value) : value_(value) {} + virtual ValueHolder* MakeNewHolder() const { + return new ValueHolder(value_); + } + + private: + const T value_; // The value for each thread. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(InstanceValueHolderFactory); + }; + + scoped_ptr default_factory_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal); +}; + +# elif GTEST_HAS_PTHREAD + +// MutexBase and Mutex implement mutex on pthreads-based platforms. +class MutexBase { + public: + // Acquires this mutex. + void Lock() { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_lock(&mutex_)); + owner_ = pthread_self(); + has_owner_ = true; + } + + // Releases this mutex. + void Unlock() { + // Since the lock is being released the owner_ field should no longer be + // considered valid. We don't protect writing to has_owner_ here, as it's + // the caller's responsibility to ensure that the current thread holds the + // mutex when this is called. + has_owner_ = false; + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_unlock(&mutex_)); + } + + // Does nothing if the current thread holds the mutex. Otherwise, crashes + // with high probability. + void AssertHeld() const { + GTEST_CHECK_(has_owner_ && pthread_equal(owner_, pthread_self())) + << "The current thread is not holding the mutex @" << this; + } + + // A static mutex may be used before main() is entered. It may even + // be used before the dynamic initialization stage. Therefore we + // must be able to initialize a static mutex object at link time. + // This means MutexBase has to be a POD and its member variables + // have to be public. + public: + pthread_mutex_t mutex_; // The underlying pthread mutex. + // has_owner_ indicates whether the owner_ field below contains a valid thread + // ID and is therefore safe to inspect (e.g., to use in pthread_equal()). All + // accesses to the owner_ field should be protected by a check of this field. + // An alternative might be to memset() owner_ to all zeros, but there's no + // guarantee that a zero'd pthread_t is necessarily invalid or even different + // from pthread_self(). + bool has_owner_; + pthread_t owner_; // The thread holding the mutex. +}; + +// Forward-declares a static mutex. +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + extern ::testing::internal::MutexBase mutex + +// Defines and statically (i.e. at link time) initializes a static mutex. +# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ + ::testing::internal::MutexBase mutex = { PTHREAD_MUTEX_INITIALIZER, false, pthread_t() } + +// The Mutex class can only be used for mutexes created at runtime. It +// shares its API with MutexBase otherwise. +class Mutex : public MutexBase { + public: + Mutex() { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, NULL)); + has_owner_ = false; + } + ~Mutex() { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_destroy(&mutex_)); + } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex); +}; + +// We cannot name this class MutexLock because the ctor declaration would +// conflict with a macro named MutexLock, which is defined on some +// platforms. That macro is used as a defensive measure to prevent against +// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than +// "MutexLock l(&mu)". Hence the typedef trick below. +class GTestMutexLock { + public: + explicit GTestMutexLock(MutexBase* mutex) + : mutex_(mutex) { mutex_->Lock(); } + + ~GTestMutexLock() { mutex_->Unlock(); } + + private: + MutexBase* const mutex_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock); +}; + +typedef GTestMutexLock MutexLock; + +// Helpers for ThreadLocal. + +// pthread_key_create() requires DeleteThreadLocalValue() to have +// C-linkage. Therefore it cannot be templatized to access +// ThreadLocal. Hence the need for class +// ThreadLocalValueHolderBase. +class ThreadLocalValueHolderBase { + public: + virtual ~ThreadLocalValueHolderBase() {} +}; + +// Called by pthread to delete thread-local data stored by +// pthread_setspecific(). +extern "C" inline void DeleteThreadLocalValue(void* value_holder) { + delete static_cast(value_holder); +} + +// Implements thread-local storage on pthreads-based systems. +template +class ThreadLocal { + public: + ThreadLocal() + : key_(CreateKey()), default_factory_(new DefaultValueHolderFactory()) {} + explicit ThreadLocal(const T& value) + : key_(CreateKey()), + default_factory_(new InstanceValueHolderFactory(value)) {} + + ~ThreadLocal() { + // Destroys the managed object for the current thread, if any. + DeleteThreadLocalValue(pthread_getspecific(key_)); + + // Releases resources associated with the key. This will *not* + // delete managed objects for other threads. + GTEST_CHECK_POSIX_SUCCESS_(pthread_key_delete(key_)); + } + + T* pointer() { return GetOrCreateValue(); } + const T* pointer() const { return GetOrCreateValue(); } + const T& get() const { return *pointer(); } + void set(const T& value) { *pointer() = value; } + + private: + // Holds a value of type T. + class ValueHolder : public ThreadLocalValueHolderBase { + public: + ValueHolder() : value_() {} + explicit ValueHolder(const T& value) : value_(value) {} + + T* pointer() { return &value_; } + + private: + T value_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder); + }; + + static pthread_key_t CreateKey() { + pthread_key_t key; + // When a thread exits, DeleteThreadLocalValue() will be called on + // the object managed for that thread. + GTEST_CHECK_POSIX_SUCCESS_( + pthread_key_create(&key, &DeleteThreadLocalValue)); + return key; + } + + T* GetOrCreateValue() const { + ThreadLocalValueHolderBase* const holder = + static_cast(pthread_getspecific(key_)); + if (holder != NULL) { + return CheckedDowncastToActualType(holder)->pointer(); + } + + ValueHolder* const new_holder = default_factory_->MakeNewHolder(); + ThreadLocalValueHolderBase* const holder_base = new_holder; + GTEST_CHECK_POSIX_SUCCESS_(pthread_setspecific(key_, holder_base)); + return new_holder->pointer(); + } + + class ValueHolderFactory { + public: + ValueHolderFactory() {} + virtual ~ValueHolderFactory() {} + virtual ValueHolder* MakeNewHolder() const = 0; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolderFactory); + }; + + class DefaultValueHolderFactory : public ValueHolderFactory { + public: + DefaultValueHolderFactory() {} + virtual ValueHolder* MakeNewHolder() const { return new ValueHolder(); } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultValueHolderFactory); + }; + + class InstanceValueHolderFactory : public ValueHolderFactory { + public: + explicit InstanceValueHolderFactory(const T& value) : value_(value) {} + virtual ValueHolder* MakeNewHolder() const { + return new ValueHolder(value_); + } + + private: + const T value_; // The value for each thread. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(InstanceValueHolderFactory); + }; + + // A key pthreads uses for looking up per-thread values. + const pthread_key_t key_; + scoped_ptr default_factory_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal); +}; + +# endif // GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ + +#else // GTEST_IS_THREADSAFE + +// A dummy implementation of synchronization primitives (mutex, lock, +// and thread-local variable). Necessary for compiling Google Test where +// mutex is not supported - using Google Test in multiple threads is not +// supported on such platforms. + +class Mutex { + public: + Mutex() {} + void Lock() {} + void Unlock() {} + void AssertHeld() const {} +}; + +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + extern ::testing::internal::Mutex mutex + +# define GTEST_DEFINE_STATIC_MUTEX_(mutex) ::testing::internal::Mutex mutex + +// We cannot name this class MutexLock because the ctor declaration would +// conflict with a macro named MutexLock, which is defined on some +// platforms. That macro is used as a defensive measure to prevent against +// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than +// "MutexLock l(&mu)". Hence the typedef trick below. +class GTestMutexLock { + public: + explicit GTestMutexLock(Mutex*) {} // NOLINT +}; + +typedef GTestMutexLock MutexLock; + +template +class ThreadLocal { + public: + ThreadLocal() : value_() {} + explicit ThreadLocal(const T& value) : value_(value) {} + T* pointer() { return &value_; } + const T* pointer() const { return &value_; } + const T& get() const { return value_; } + void set(const T& value) { value_ = value; } + private: + T value_; +}; + +#endif // GTEST_IS_THREADSAFE + +// Returns the number of threads running in the process, or 0 to indicate that +// we cannot detect it. +GTEST_API_ size_t GetThreadCount(); + +// Passing non-POD classes through ellipsis (...) crashes the ARM +// compiler and generates a warning in Sun Studio. The Nokia Symbian +// and the IBM XL C/C++ compiler try to instantiate a copy constructor +// for objects passed through ellipsis (...), failing for uncopyable +// objects. We define this to ensure that only POD is passed through +// ellipsis on these systems. +#if defined(__SYMBIAN32__) || defined(__IBMCPP__) || defined(__SUNPRO_CC) +// We lose support for NULL detection where the compiler doesn't like +// passing non-POD classes through ellipsis (...). +# define GTEST_ELLIPSIS_NEEDS_POD_ 1 +#else +# define GTEST_CAN_COMPARE_NULL 1 +#endif + +// The Nokia Symbian and IBM XL C/C++ compilers cannot decide between +// const T& and const T* in a function template. These compilers +// _can_ decide between class template specializations for T and T*, +// so a tr1::type_traits-like is_pointer works. +#if defined(__SYMBIAN32__) || defined(__IBMCPP__) +# define GTEST_NEEDS_IS_POINTER_ 1 +#endif + +template +struct bool_constant { + typedef bool_constant type; + static const bool value = bool_value; +}; +template const bool bool_constant::value; + +typedef bool_constant false_type; +typedef bool_constant true_type; + +template +struct is_pointer : public false_type {}; + +template +struct is_pointer : public true_type {}; + +template +struct IteratorTraits { + typedef typename Iterator::value_type value_type; +}; + +template +struct IteratorTraits { + typedef T value_type; +}; + +template +struct IteratorTraits { + typedef T value_type; +}; + +#if GTEST_OS_WINDOWS +# define GTEST_PATH_SEP_ "\\" +# define GTEST_HAS_ALT_PATH_SEP_ 1 +// The biggest signed integer type the compiler supports. +typedef __int64 BiggestInt; +#else +# define GTEST_PATH_SEP_ "/" +# define GTEST_HAS_ALT_PATH_SEP_ 0 +typedef long long BiggestInt; // NOLINT +#endif // GTEST_OS_WINDOWS + +// Utilities for char. + +// isspace(int ch) and friends accept an unsigned char or EOF. char +// may be signed, depending on the compiler (or compiler flags). +// Therefore we need to cast a char to unsigned char before calling +// isspace(), etc. + +inline bool IsAlpha(char ch) { + return isalpha(static_cast(ch)) != 0; +} +inline bool IsAlNum(char ch) { + return isalnum(static_cast(ch)) != 0; +} +inline bool IsDigit(char ch) { + return isdigit(static_cast(ch)) != 0; +} +inline bool IsLower(char ch) { + return islower(static_cast(ch)) != 0; +} +inline bool IsSpace(char ch) { + return isspace(static_cast(ch)) != 0; +} +inline bool IsUpper(char ch) { + return isupper(static_cast(ch)) != 0; +} +inline bool IsXDigit(char ch) { + return isxdigit(static_cast(ch)) != 0; +} +inline bool IsXDigit(wchar_t ch) { + const unsigned char low_byte = static_cast(ch); + return ch == low_byte && isxdigit(low_byte) != 0; +} + +inline char ToLower(char ch) { + return static_cast(tolower(static_cast(ch))); +} +inline char ToUpper(char ch) { + return static_cast(toupper(static_cast(ch))); +} + +inline std::string StripTrailingSpaces(std::string str) { + std::string::iterator it = str.end(); + while (it != str.begin() && IsSpace(*--it)) + it = str.erase(it); + return str; +} + +// The testing::internal::posix namespace holds wrappers for common +// POSIX functions. These wrappers hide the differences between +// Windows/MSVC and POSIX systems. Since some compilers define these +// standard functions as macros, the wrapper cannot have the same name +// as the wrapped function. + +namespace posix { + +// Functions with a different name on Windows. + +#if GTEST_OS_WINDOWS + +typedef struct _stat StatStruct; + +# ifdef __BORLANDC__ +inline int IsATTY(int fd) { return isatty(fd); } +inline int StrCaseCmp(const char* s1, const char* s2) { + return stricmp(s1, s2); +} +inline char* StrDup(const char* src) { return strdup(src); } +# else // !__BORLANDC__ +# if GTEST_OS_WINDOWS_MOBILE +inline int IsATTY(int /* fd */) { return 0; } +# else +inline int IsATTY(int fd) { return _isatty(fd); } +# endif // GTEST_OS_WINDOWS_MOBILE +inline int StrCaseCmp(const char* s1, const char* s2) { + return _stricmp(s1, s2); +} +inline char* StrDup(const char* src) { return _strdup(src); } +# endif // __BORLANDC__ + +# if GTEST_OS_WINDOWS_MOBILE +inline int FileNo(FILE* file) { return reinterpret_cast(_fileno(file)); } +// Stat(), RmDir(), and IsDir() are not needed on Windows CE at this +// time and thus not defined there. +# else +inline int FileNo(FILE* file) { return _fileno(file); } +inline int Stat(const char* path, StatStruct* buf) { return _stat(path, buf); } +inline int RmDir(const char* dir) { return _rmdir(dir); } +inline bool IsDir(const StatStruct& st) { + return (_S_IFDIR & st.st_mode) != 0; +} +# endif // GTEST_OS_WINDOWS_MOBILE + +#else + +typedef struct stat StatStruct; + +inline int FileNo(FILE* file) { return fileno(file); } +inline int IsATTY(int fd) { return isatty(fd); } +inline int Stat(const char* path, StatStruct* buf) { return stat(path, buf); } +inline int StrCaseCmp(const char* s1, const char* s2) { + return strcasecmp(s1, s2); +} +inline char* StrDup(const char* src) { return strdup(src); } +inline int RmDir(const char* dir) { return rmdir(dir); } +inline bool IsDir(const StatStruct& st) { return S_ISDIR(st.st_mode); } + +#endif // GTEST_OS_WINDOWS + +// Functions deprecated by MSVC 8.0. + +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996 /* deprecated function */) + +inline const char* StrNCpy(char* dest, const char* src, size_t n) { + return strncpy(dest, src, n); +} + +// ChDir(), FReopen(), FDOpen(), Read(), Write(), Close(), and +// StrError() aren't needed on Windows CE at this time and thus not +// defined there. + +#if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT +inline int ChDir(const char* dir) { return chdir(dir); } +#endif +inline FILE* FOpen(const char* path, const char* mode) { + return fopen(path, mode); +} +#if !GTEST_OS_WINDOWS_MOBILE +inline FILE *FReopen(const char* path, const char* mode, FILE* stream) { + return freopen(path, mode, stream); +} +inline FILE* FDOpen(int fd, const char* mode) { return fdopen(fd, mode); } +#endif +inline int FClose(FILE* fp) { return fclose(fp); } +#if !GTEST_OS_WINDOWS_MOBILE +inline int Read(int fd, void* buf, unsigned int count) { + return static_cast(read(fd, buf, count)); +} +inline int Write(int fd, const void* buf, unsigned int count) { + return static_cast(write(fd, buf, count)); +} +inline int Close(int fd) { return close(fd); } +inline const char* StrError(int errnum) { return strerror(errnum); } +#endif +inline const char* GetEnv(const char* name) { +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE | GTEST_OS_WINDOWS_RT + // We are on Windows CE, which has no environment variables. + static_cast(name); // To prevent 'unused argument' warning. + return NULL; +#elif defined(__BORLANDC__) || defined(__SunOS_5_8) || defined(__SunOS_5_9) + // Environment variables which we programmatically clear will be set to the + // empty string rather than unset (NULL). Handle that case. + const char* const env = getenv(name); + return (env != NULL && env[0] != '\0') ? env : NULL; +#else + return getenv(name); +#endif +} + +GTEST_DISABLE_MSC_WARNINGS_POP_() + +#if GTEST_OS_WINDOWS_MOBILE +// Windows CE has no C library. The abort() function is used in +// several places in Google Test. This implementation provides a reasonable +// imitation of standard behaviour. +void Abort(); +#else +inline void Abort() { abort(); } +#endif // GTEST_OS_WINDOWS_MOBILE + +} // namespace posix + +// MSVC "deprecates" snprintf and issues warnings wherever it is used. In +// order to avoid these warnings, we need to use _snprintf or _snprintf_s on +// MSVC-based platforms. We map the GTEST_SNPRINTF_ macro to the appropriate +// function in order to achieve that. We use macro definition here because +// snprintf is a variadic function. +#if _MSC_VER >= 1400 && !GTEST_OS_WINDOWS_MOBILE +// MSVC 2005 and above support variadic macros. +# define GTEST_SNPRINTF_(buffer, size, format, ...) \ + _snprintf_s(buffer, size, size, format, __VA_ARGS__) +#elif defined(_MSC_VER) +// Windows CE does not define _snprintf_s and MSVC prior to 2005 doesn't +// complain about _snprintf. +# define GTEST_SNPRINTF_ _snprintf +#else +# define GTEST_SNPRINTF_ snprintf +#endif + +// The maximum number a BiggestInt can represent. This definition +// works no matter BiggestInt is represented in one's complement or +// two's complement. +// +// We cannot rely on numeric_limits in STL, as __int64 and long long +// are not part of standard C++ and numeric_limits doesn't need to be +// defined for them. +const BiggestInt kMaxBiggestInt = + ~(static_cast(1) << (8*sizeof(BiggestInt) - 1)); + +// This template class serves as a compile-time function from size to +// type. It maps a size in bytes to a primitive type with that +// size. e.g. +// +// TypeWithSize<4>::UInt +// +// is typedef-ed to be unsigned int (unsigned integer made up of 4 +// bytes). +// +// Such functionality should belong to STL, but I cannot find it +// there. +// +// Google Test uses this class in the implementation of floating-point +// comparison. +// +// For now it only handles UInt (unsigned int) as that's all Google Test +// needs. Other types can be easily added in the future if need +// arises. +template +class TypeWithSize { + public: + // This prevents the user from using TypeWithSize with incorrect + // values of N. + typedef void UInt; +}; + +// The specialization for size 4. +template <> +class TypeWithSize<4> { + public: + // unsigned int has size 4 in both gcc and MSVC. + // + // As base/basictypes.h doesn't compile on Windows, we cannot use + // uint32, uint64, and etc here. + typedef int Int; + typedef unsigned int UInt; +}; + +// The specialization for size 8. +template <> +class TypeWithSize<8> { + public: +#if GTEST_OS_WINDOWS + typedef __int64 Int; + typedef unsigned __int64 UInt; +#else + typedef long long Int; // NOLINT + typedef unsigned long long UInt; // NOLINT +#endif // GTEST_OS_WINDOWS +}; + +// Integer types of known sizes. +typedef TypeWithSize<4>::Int Int32; +typedef TypeWithSize<4>::UInt UInt32; +typedef TypeWithSize<8>::Int Int64; +typedef TypeWithSize<8>::UInt UInt64; +typedef TypeWithSize<8>::Int TimeInMillis; // Represents time in milliseconds. + +// Utilities for command line flags and environment variables. + +// Macro for referencing flags. +#if !defined(GTEST_FLAG) +# define GTEST_FLAG(name) FLAGS_gtest_##name +#endif // !defined(GTEST_FLAG) + +#if !defined(GTEST_USE_OWN_FLAGFILE_FLAG_) +# define GTEST_USE_OWN_FLAGFILE_FLAG_ 1 +#endif // !defined(GTEST_USE_OWN_FLAGFILE_FLAG_) + +#if !defined(GTEST_DECLARE_bool_) +# define GTEST_FLAG_SAVER_ ::testing::internal::GTestFlagSaver + +// Macros for declaring flags. +# define GTEST_DECLARE_bool_(name) GTEST_API_ extern bool GTEST_FLAG(name) +# define GTEST_DECLARE_int32_(name) \ + GTEST_API_ extern ::testing::internal::Int32 GTEST_FLAG(name) +#define GTEST_DECLARE_string_(name) \ + GTEST_API_ extern ::std::string GTEST_FLAG(name) + +// Macros for defining flags. +#define GTEST_DEFINE_bool_(name, default_val, doc) \ + GTEST_API_ bool GTEST_FLAG(name) = (default_val) +#define GTEST_DEFINE_int32_(name, default_val, doc) \ + GTEST_API_ ::testing::internal::Int32 GTEST_FLAG(name) = (default_val) +#define GTEST_DEFINE_string_(name, default_val, doc) \ + GTEST_API_ ::std::string GTEST_FLAG(name) = (default_val) + +#endif // !defined(GTEST_DECLARE_bool_) + +// Thread annotations +#if !defined(GTEST_EXCLUSIVE_LOCK_REQUIRED_) +# define GTEST_EXCLUSIVE_LOCK_REQUIRED_(locks) +# define GTEST_LOCK_EXCLUDED_(locks) +#endif // !defined(GTEST_EXCLUSIVE_LOCK_REQUIRED_) + +// Parses 'str' for a 32-bit signed integer. If successful, writes the result +// to *value and returns true; otherwise leaves *value unchanged and returns +// false. +// TODO(chandlerc): Find a better way to refactor flag and environment parsing +// out of both gtest-port.cc and gtest.cc to avoid exporting this utility +// function. +bool ParseInt32(const Message& src_text, const char* str, Int32* value); + +// Parses a bool/Int32/string from the environment variable +// corresponding to the given Google Test flag. +bool BoolFromGTestEnv(const char* flag, bool default_val); +GTEST_API_ Int32 Int32FromGTestEnv(const char* flag, Int32 default_val); +std::string StringFromGTestEnv(const char* flag, const char* default_val); + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-string.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-string.h new file mode 100644 index 0000000000..97f1a7fdd2 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-string.h @@ -0,0 +1,167 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: wan@google.com (Zhanyong Wan), eefacm@gmail.com (Sean Mcafee) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file declares the String class and functions used internally by +// Google Test. They are subject to change without notice. They should not used +// by code external to Google Test. +// +// This header file is #included by . +// It should not be #included by other files. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ + +#ifdef __BORLANDC__ +// string.h is not guaranteed to provide strcpy on C++ Builder. +# include +#endif + +#include +#include + +#include "gtest/internal/gtest-port.h" + +namespace testing { +namespace internal { + +// String - an abstract class holding static string utilities. +class GTEST_API_ String { + public: + // Static utility methods + + // Clones a 0-terminated C string, allocating memory using new. The + // caller is responsible for deleting the return value using + // delete[]. Returns the cloned string, or NULL if the input is + // NULL. + // + // This is different from strdup() in string.h, which allocates + // memory using malloc(). + static const char* CloneCString(const char* c_str); + +#if GTEST_OS_WINDOWS_MOBILE + // Windows CE does not have the 'ANSI' versions of Win32 APIs. To be + // able to pass strings to Win32 APIs on CE we need to convert them + // to 'Unicode', UTF-16. + + // Creates a UTF-16 wide string from the given ANSI string, allocating + // memory using new. The caller is responsible for deleting the return + // value using delete[]. Returns the wide string, or NULL if the + // input is NULL. + // + // The wide string is created using the ANSI codepage (CP_ACP) to + // match the behaviour of the ANSI versions of Win32 calls and the + // C runtime. + static LPCWSTR AnsiToUtf16(const char* c_str); + + // Creates an ANSI string from the given wide string, allocating + // memory using new. The caller is responsible for deleting the return + // value using delete[]. Returns the ANSI string, or NULL if the + // input is NULL. + // + // The returned string is created using the ANSI codepage (CP_ACP) to + // match the behaviour of the ANSI versions of Win32 calls and the + // C runtime. + static const char* Utf16ToAnsi(LPCWSTR utf16_str); +#endif + + // Compares two C strings. Returns true iff they have the same content. + // + // Unlike strcmp(), this function can handle NULL argument(s). A + // NULL C string is considered different to any non-NULL C string, + // including the empty string. + static bool CStringEquals(const char* lhs, const char* rhs); + + // Converts a wide C string to a String using the UTF-8 encoding. + // NULL will be converted to "(null)". If an error occurred during + // the conversion, "(failed to convert from wide string)" is + // returned. + static std::string ShowWideCString(const wchar_t* wide_c_str); + + // Compares two wide C strings. Returns true iff they have the same + // content. + // + // Unlike wcscmp(), this function can handle NULL argument(s). A + // NULL C string is considered different to any non-NULL C string, + // including the empty string. + static bool WideCStringEquals(const wchar_t* lhs, const wchar_t* rhs); + + // Compares two C strings, ignoring case. Returns true iff they + // have the same content. + // + // Unlike strcasecmp(), this function can handle NULL argument(s). + // A NULL C string is considered different to any non-NULL C string, + // including the empty string. + static bool CaseInsensitiveCStringEquals(const char* lhs, + const char* rhs); + + // Compares two wide C strings, ignoring case. Returns true iff they + // have the same content. + // + // Unlike wcscasecmp(), this function can handle NULL argument(s). + // A NULL C string is considered different to any non-NULL wide C string, + // including the empty string. + // NB: The implementations on different platforms slightly differ. + // On windows, this method uses _wcsicmp which compares according to LC_CTYPE + // environment variable. On GNU platform this method uses wcscasecmp + // which compares according to LC_CTYPE category of the current locale. + // On MacOS X, it uses towlower, which also uses LC_CTYPE category of the + // current locale. + static bool CaseInsensitiveWideCStringEquals(const wchar_t* lhs, + const wchar_t* rhs); + + // Returns true iff the given string ends with the given suffix, ignoring + // case. Any string is considered to end with an empty suffix. + static bool EndsWithCaseInsensitive( + const std::string& str, const std::string& suffix); + + // Formats an int value as "%02d". + static std::string FormatIntWidth2(int value); // "%02d" for width == 2 + + // Formats an int value as "%X". + static std::string FormatHexInt(int value); + + // Formats a byte as "%02X". + static std::string FormatByte(unsigned char value); + + private: + String(); // Not meant to be instantiated. +}; // class String + +// Gets the content of the stringstream's buffer as an std::string. Each '\0' +// character in the buffer is replaced with "\\0". +GTEST_API_ std::string StringStreamToString(::std::stringstream* stream); + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-tuple.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-tuple.h new file mode 100644 index 0000000000..e9b405340a --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-tuple.h @@ -0,0 +1,1020 @@ +// This file was GENERATED by command: +// pump.py gtest-tuple.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2009 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Implements a subset of TR1 tuple needed by Google Test and Google Mock. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ + +#include // For ::std::pair. + +// The compiler used in Symbian has a bug that prevents us from declaring the +// tuple template as a friend (it complains that tuple is redefined). This +// hack bypasses the bug by declaring the members that should otherwise be +// private as public. +// Sun Studio versions < 12 also have the above bug. +#if defined(__SYMBIAN32__) || (defined(__SUNPRO_CC) && __SUNPRO_CC < 0x590) +# define GTEST_DECLARE_TUPLE_AS_FRIEND_ public: +#else +# define GTEST_DECLARE_TUPLE_AS_FRIEND_ \ + template friend class tuple; \ + private: +#endif + +// Visual Studio 2010, 2012, and 2013 define symbols in std::tr1 that conflict +// with our own definitions. Therefore using our own tuple does not work on +// those compilers. +#if defined(_MSC_VER) && _MSC_VER >= 1600 /* 1600 is Visual Studio 2010 */ +# error "gtest's tuple doesn't compile on Visual Studio 2010 or later. \ +GTEST_USE_OWN_TR1_TUPLE must be set to 0 on those compilers." +#endif + +// GTEST_n_TUPLE_(T) is the type of an n-tuple. +#define GTEST_0_TUPLE_(T) tuple<> +#define GTEST_1_TUPLE_(T) tuple +#define GTEST_2_TUPLE_(T) tuple +#define GTEST_3_TUPLE_(T) tuple +#define GTEST_4_TUPLE_(T) tuple +#define GTEST_5_TUPLE_(T) tuple +#define GTEST_6_TUPLE_(T) tuple +#define GTEST_7_TUPLE_(T) tuple +#define GTEST_8_TUPLE_(T) tuple +#define GTEST_9_TUPLE_(T) tuple +#define GTEST_10_TUPLE_(T) tuple + +// GTEST_n_TYPENAMES_(T) declares a list of n typenames. +#define GTEST_0_TYPENAMES_(T) +#define GTEST_1_TYPENAMES_(T) typename T##0 +#define GTEST_2_TYPENAMES_(T) typename T##0, typename T##1 +#define GTEST_3_TYPENAMES_(T) typename T##0, typename T##1, typename T##2 +#define GTEST_4_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3 +#define GTEST_5_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4 +#define GTEST_6_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5 +#define GTEST_7_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6 +#define GTEST_8_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6, typename T##7 +#define GTEST_9_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6, \ + typename T##7, typename T##8 +#define GTEST_10_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6, \ + typename T##7, typename T##8, typename T##9 + +// In theory, defining stuff in the ::std namespace is undefined +// behavior. We can do this as we are playing the role of a standard +// library vendor. +namespace std { +namespace tr1 { + +template +class tuple; + +// Anything in namespace gtest_internal is Google Test's INTERNAL +// IMPLEMENTATION DETAIL and MUST NOT BE USED DIRECTLY in user code. +namespace gtest_internal { + +// ByRef::type is T if T is a reference; otherwise it's const T&. +template +struct ByRef { typedef const T& type; }; // NOLINT +template +struct ByRef { typedef T& type; }; // NOLINT + +// A handy wrapper for ByRef. +#define GTEST_BY_REF_(T) typename ::std::tr1::gtest_internal::ByRef::type + +// AddRef::type is T if T is a reference; otherwise it's T&. This +// is the same as tr1::add_reference::type. +template +struct AddRef { typedef T& type; }; // NOLINT +template +struct AddRef { typedef T& type; }; // NOLINT + +// A handy wrapper for AddRef. +#define GTEST_ADD_REF_(T) typename ::std::tr1::gtest_internal::AddRef::type + +// A helper for implementing get(). +template class Get; + +// A helper for implementing tuple_element. kIndexValid is true +// iff k < the number of fields in tuple type T. +template +struct TupleElement; + +template +struct TupleElement { + typedef T0 type; +}; + +template +struct TupleElement { + typedef T1 type; +}; + +template +struct TupleElement { + typedef T2 type; +}; + +template +struct TupleElement { + typedef T3 type; +}; + +template +struct TupleElement { + typedef T4 type; +}; + +template +struct TupleElement { + typedef T5 type; +}; + +template +struct TupleElement { + typedef T6 type; +}; + +template +struct TupleElement { + typedef T7 type; +}; + +template +struct TupleElement { + typedef T8 type; +}; + +template +struct TupleElement { + typedef T9 type; +}; + +} // namespace gtest_internal + +template <> +class tuple<> { + public: + tuple() {} + tuple(const tuple& /* t */) {} + tuple& operator=(const tuple& /* t */) { return *this; } +}; + +template +class GTEST_1_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0) : f0_(f0) {} + + tuple(const tuple& t) : f0_(t.f0_) {} + + template + tuple(const GTEST_1_TUPLE_(U)& t) : f0_(t.f0_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_1_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_1_TUPLE_(U)& t) { + f0_ = t.f0_; + return *this; + } + + T0 f0_; +}; + +template +class GTEST_2_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1) : f0_(f0), + f1_(f1) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_) {} + + template + tuple(const GTEST_2_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_) {} + template + tuple(const ::std::pair& p) : f0_(p.first), f1_(p.second) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_2_TUPLE_(U)& t) { + return CopyFrom(t); + } + template + tuple& operator=(const ::std::pair& p) { + f0_ = p.first; + f1_ = p.second; + return *this; + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_2_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + return *this; + } + + T0 f0_; + T1 f1_; +}; + +template +class GTEST_3_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2) : f0_(f0), f1_(f1), f2_(f2) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_) {} + + template + tuple(const GTEST_3_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_3_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_3_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; +}; + +template +class GTEST_4_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3) : f0_(f0), f1_(f1), f2_(f2), + f3_(f3) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_) {} + + template + tuple(const GTEST_4_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_4_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_4_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; +}; + +template +class GTEST_5_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, + GTEST_BY_REF_(T4) f4) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_) {} + + template + tuple(const GTEST_5_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_5_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_5_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; +}; + +template +class GTEST_6_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4), + f5_(f5) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_) {} + + template + tuple(const GTEST_6_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_6_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_6_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; +}; + +template +class GTEST_7_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6) : f0_(f0), f1_(f1), f2_(f2), + f3_(f3), f4_(f4), f5_(f5), f6_(f6) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_) {} + + template + tuple(const GTEST_7_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_7_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_7_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; +}; + +template +class GTEST_8_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, + GTEST_BY_REF_(T7) f7) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4), + f5_(f5), f6_(f6), f7_(f7) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_) {} + + template + tuple(const GTEST_8_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_8_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_8_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + f7_ = t.f7_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; + T7 f7_; +}; + +template +class GTEST_9_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_(), f8_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, GTEST_BY_REF_(T7) f7, + GTEST_BY_REF_(T8) f8) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4), + f5_(f5), f6_(f6), f7_(f7), f8_(f8) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_) {} + + template + tuple(const GTEST_9_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_9_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_9_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + f7_ = t.f7_; + f8_ = t.f8_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; + T7 f7_; + T8 f8_; +}; + +template +class tuple { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_(), f8_(), + f9_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, GTEST_BY_REF_(T7) f7, + GTEST_BY_REF_(T8) f8, GTEST_BY_REF_(T9) f9) : f0_(f0), f1_(f1), f2_(f2), + f3_(f3), f4_(f4), f5_(f5), f6_(f6), f7_(f7), f8_(f8), f9_(f9) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_), f9_(t.f9_) {} + + template + tuple(const GTEST_10_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_), + f9_(t.f9_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_10_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_10_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + f7_ = t.f7_; + f8_ = t.f8_; + f9_ = t.f9_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; + T7 f7_; + T8 f8_; + T9 f9_; +}; + +// 6.1.3.2 Tuple creation functions. + +// Known limitations: we don't support passing an +// std::tr1::reference_wrapper to make_tuple(). And we don't +// implement tie(). + +inline tuple<> make_tuple() { return tuple<>(); } + +template +inline GTEST_1_TUPLE_(T) make_tuple(const T0& f0) { + return GTEST_1_TUPLE_(T)(f0); +} + +template +inline GTEST_2_TUPLE_(T) make_tuple(const T0& f0, const T1& f1) { + return GTEST_2_TUPLE_(T)(f0, f1); +} + +template +inline GTEST_3_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2) { + return GTEST_3_TUPLE_(T)(f0, f1, f2); +} + +template +inline GTEST_4_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3) { + return GTEST_4_TUPLE_(T)(f0, f1, f2, f3); +} + +template +inline GTEST_5_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4) { + return GTEST_5_TUPLE_(T)(f0, f1, f2, f3, f4); +} + +template +inline GTEST_6_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5) { + return GTEST_6_TUPLE_(T)(f0, f1, f2, f3, f4, f5); +} + +template +inline GTEST_7_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6) { + return GTEST_7_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6); +} + +template +inline GTEST_8_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7) { + return GTEST_8_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7); +} + +template +inline GTEST_9_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7, + const T8& f8) { + return GTEST_9_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7, f8); +} + +template +inline GTEST_10_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7, + const T8& f8, const T9& f9) { + return GTEST_10_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7, f8, f9); +} + +// 6.1.3.3 Tuple helper classes. + +template struct tuple_size; + +template +struct tuple_size { + static const int value = 0; +}; + +template +struct tuple_size { + static const int value = 1; +}; + +template +struct tuple_size { + static const int value = 2; +}; + +template +struct tuple_size { + static const int value = 3; +}; + +template +struct tuple_size { + static const int value = 4; +}; + +template +struct tuple_size { + static const int value = 5; +}; + +template +struct tuple_size { + static const int value = 6; +}; + +template +struct tuple_size { + static const int value = 7; +}; + +template +struct tuple_size { + static const int value = 8; +}; + +template +struct tuple_size { + static const int value = 9; +}; + +template +struct tuple_size { + static const int value = 10; +}; + +template +struct tuple_element { + typedef typename gtest_internal::TupleElement< + k < (tuple_size::value), k, Tuple>::type type; +}; + +#define GTEST_TUPLE_ELEMENT_(k, Tuple) typename tuple_element::type + +// 6.1.3.4 Element access. + +namespace gtest_internal { + +template <> +class Get<0> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(0, Tuple)) + Field(Tuple& t) { return t.f0_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(0, Tuple)) + ConstField(const Tuple& t) { return t.f0_; } +}; + +template <> +class Get<1> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(1, Tuple)) + Field(Tuple& t) { return t.f1_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(1, Tuple)) + ConstField(const Tuple& t) { return t.f1_; } +}; + +template <> +class Get<2> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(2, Tuple)) + Field(Tuple& t) { return t.f2_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(2, Tuple)) + ConstField(const Tuple& t) { return t.f2_; } +}; + +template <> +class Get<3> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(3, Tuple)) + Field(Tuple& t) { return t.f3_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(3, Tuple)) + ConstField(const Tuple& t) { return t.f3_; } +}; + +template <> +class Get<4> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(4, Tuple)) + Field(Tuple& t) { return t.f4_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(4, Tuple)) + ConstField(const Tuple& t) { return t.f4_; } +}; + +template <> +class Get<5> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(5, Tuple)) + Field(Tuple& t) { return t.f5_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(5, Tuple)) + ConstField(const Tuple& t) { return t.f5_; } +}; + +template <> +class Get<6> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(6, Tuple)) + Field(Tuple& t) { return t.f6_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(6, Tuple)) + ConstField(const Tuple& t) { return t.f6_; } +}; + +template <> +class Get<7> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(7, Tuple)) + Field(Tuple& t) { return t.f7_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(7, Tuple)) + ConstField(const Tuple& t) { return t.f7_; } +}; + +template <> +class Get<8> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(8, Tuple)) + Field(Tuple& t) { return t.f8_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(8, Tuple)) + ConstField(const Tuple& t) { return t.f8_; } +}; + +template <> +class Get<9> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(9, Tuple)) + Field(Tuple& t) { return t.f9_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(9, Tuple)) + ConstField(const Tuple& t) { return t.f9_; } +}; + +} // namespace gtest_internal + +template +GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_10_TUPLE_(T))) +get(GTEST_10_TUPLE_(T)& t) { + return gtest_internal::Get::Field(t); +} + +template +GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_10_TUPLE_(T))) +get(const GTEST_10_TUPLE_(T)& t) { + return gtest_internal::Get::ConstField(t); +} + +// 6.1.3.5 Relational operators + +// We only implement == and !=, as we don't have a need for the rest yet. + +namespace gtest_internal { + +// SameSizeTuplePrefixComparator::Eq(t1, t2) returns true if the +// first k fields of t1 equals the first k fields of t2. +// SameSizeTuplePrefixComparator(k1, k2) would be a compiler error if +// k1 != k2. +template +struct SameSizeTuplePrefixComparator; + +template <> +struct SameSizeTuplePrefixComparator<0, 0> { + template + static bool Eq(const Tuple1& /* t1 */, const Tuple2& /* t2 */) { + return true; + } +}; + +template +struct SameSizeTuplePrefixComparator { + template + static bool Eq(const Tuple1& t1, const Tuple2& t2) { + return SameSizeTuplePrefixComparator::Eq(t1, t2) && + ::std::tr1::get(t1) == ::std::tr1::get(t2); + } +}; + +} // namespace gtest_internal + +template +inline bool operator==(const GTEST_10_TUPLE_(T)& t, + const GTEST_10_TUPLE_(U)& u) { + return gtest_internal::SameSizeTuplePrefixComparator< + tuple_size::value, + tuple_size::value>::Eq(t, u); +} + +template +inline bool operator!=(const GTEST_10_TUPLE_(T)& t, + const GTEST_10_TUPLE_(U)& u) { return !(t == u); } + +// 6.1.4 Pairs. +// Unimplemented. + +} // namespace tr1 +} // namespace std + +#undef GTEST_0_TUPLE_ +#undef GTEST_1_TUPLE_ +#undef GTEST_2_TUPLE_ +#undef GTEST_3_TUPLE_ +#undef GTEST_4_TUPLE_ +#undef GTEST_5_TUPLE_ +#undef GTEST_6_TUPLE_ +#undef GTEST_7_TUPLE_ +#undef GTEST_8_TUPLE_ +#undef GTEST_9_TUPLE_ +#undef GTEST_10_TUPLE_ + +#undef GTEST_0_TYPENAMES_ +#undef GTEST_1_TYPENAMES_ +#undef GTEST_2_TYPENAMES_ +#undef GTEST_3_TYPENAMES_ +#undef GTEST_4_TYPENAMES_ +#undef GTEST_5_TYPENAMES_ +#undef GTEST_6_TYPENAMES_ +#undef GTEST_7_TYPENAMES_ +#undef GTEST_8_TYPENAMES_ +#undef GTEST_9_TYPENAMES_ +#undef GTEST_10_TYPENAMES_ + +#undef GTEST_DECLARE_TUPLE_AS_FRIEND_ +#undef GTEST_BY_REF_ +#undef GTEST_ADD_REF_ +#undef GTEST_TUPLE_ELEMENT_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-tuple.h.pump b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-tuple.h.pump new file mode 100644 index 0000000000..429ddfeeca --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-tuple.h.pump @@ -0,0 +1,347 @@ +$$ -*- mode: c++; -*- +$var n = 10 $$ Maximum number of tuple fields we want to support. +$$ This meta comment fixes auto-indentation in Emacs. }} +// Copyright 2009 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Implements a subset of TR1 tuple needed by Google Test and Google Mock. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ + +#include // For ::std::pair. + +// The compiler used in Symbian has a bug that prevents us from declaring the +// tuple template as a friend (it complains that tuple is redefined). This +// hack bypasses the bug by declaring the members that should otherwise be +// private as public. +// Sun Studio versions < 12 also have the above bug. +#if defined(__SYMBIAN32__) || (defined(__SUNPRO_CC) && __SUNPRO_CC < 0x590) +# define GTEST_DECLARE_TUPLE_AS_FRIEND_ public: +#else +# define GTEST_DECLARE_TUPLE_AS_FRIEND_ \ + template friend class tuple; \ + private: +#endif + +// Visual Studio 2010, 2012, and 2013 define symbols in std::tr1 that conflict +// with our own definitions. Therefore using our own tuple does not work on +// those compilers. +#if defined(_MSC_VER) && _MSC_VER >= 1600 /* 1600 is Visual Studio 2010 */ +# error "gtest's tuple doesn't compile on Visual Studio 2010 or later. \ +GTEST_USE_OWN_TR1_TUPLE must be set to 0 on those compilers." +#endif + + +$range i 0..n-1 +$range j 0..n +$range k 1..n +// GTEST_n_TUPLE_(T) is the type of an n-tuple. +#define GTEST_0_TUPLE_(T) tuple<> + +$for k [[ +$range m 0..k-1 +$range m2 k..n-1 +#define GTEST_$(k)_TUPLE_(T) tuple<$for m, [[T##$m]]$for m2 [[, void]]> + +]] + +// GTEST_n_TYPENAMES_(T) declares a list of n typenames. + +$for j [[ +$range m 0..j-1 +#define GTEST_$(j)_TYPENAMES_(T) $for m, [[typename T##$m]] + + +]] + +// In theory, defining stuff in the ::std namespace is undefined +// behavior. We can do this as we are playing the role of a standard +// library vendor. +namespace std { +namespace tr1 { + +template <$for i, [[typename T$i = void]]> +class tuple; + +// Anything in namespace gtest_internal is Google Test's INTERNAL +// IMPLEMENTATION DETAIL and MUST NOT BE USED DIRECTLY in user code. +namespace gtest_internal { + +// ByRef::type is T if T is a reference; otherwise it's const T&. +template +struct ByRef { typedef const T& type; }; // NOLINT +template +struct ByRef { typedef T& type; }; // NOLINT + +// A handy wrapper for ByRef. +#define GTEST_BY_REF_(T) typename ::std::tr1::gtest_internal::ByRef::type + +// AddRef::type is T if T is a reference; otherwise it's T&. This +// is the same as tr1::add_reference::type. +template +struct AddRef { typedef T& type; }; // NOLINT +template +struct AddRef { typedef T& type; }; // NOLINT + +// A handy wrapper for AddRef. +#define GTEST_ADD_REF_(T) typename ::std::tr1::gtest_internal::AddRef::type + +// A helper for implementing get(). +template class Get; + +// A helper for implementing tuple_element. kIndexValid is true +// iff k < the number of fields in tuple type T. +template +struct TupleElement; + + +$for i [[ +template +struct TupleElement { + typedef T$i type; +}; + + +]] +} // namespace gtest_internal + +template <> +class tuple<> { + public: + tuple() {} + tuple(const tuple& /* t */) {} + tuple& operator=(const tuple& /* t */) { return *this; } +}; + + +$for k [[ +$range m 0..k-1 +template +class $if k < n [[GTEST_$(k)_TUPLE_(T)]] $else [[tuple]] { + public: + template friend class gtest_internal::Get; + + tuple() : $for m, [[f$(m)_()]] {} + + explicit tuple($for m, [[GTEST_BY_REF_(T$m) f$m]]) : [[]] +$for m, [[f$(m)_(f$m)]] {} + + tuple(const tuple& t) : $for m, [[f$(m)_(t.f$(m)_)]] {} + + template + tuple(const GTEST_$(k)_TUPLE_(U)& t) : $for m, [[f$(m)_(t.f$(m)_)]] {} + +$if k == 2 [[ + template + tuple(const ::std::pair& p) : f0_(p.first), f1_(p.second) {} + +]] + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_$(k)_TUPLE_(U)& t) { + return CopyFrom(t); + } + +$if k == 2 [[ + template + tuple& operator=(const ::std::pair& p) { + f0_ = p.first; + f1_ = p.second; + return *this; + } + +]] + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_$(k)_TUPLE_(U)& t) { + +$for m [[ + f$(m)_ = t.f$(m)_; + +]] + return *this; + } + + +$for m [[ + T$m f$(m)_; + +]] +}; + + +]] +// 6.1.3.2 Tuple creation functions. + +// Known limitations: we don't support passing an +// std::tr1::reference_wrapper to make_tuple(). And we don't +// implement tie(). + +inline tuple<> make_tuple() { return tuple<>(); } + +$for k [[ +$range m 0..k-1 + +template +inline GTEST_$(k)_TUPLE_(T) make_tuple($for m, [[const T$m& f$m]]) { + return GTEST_$(k)_TUPLE_(T)($for m, [[f$m]]); +} + +]] + +// 6.1.3.3 Tuple helper classes. + +template struct tuple_size; + + +$for j [[ +template +struct tuple_size { + static const int value = $j; +}; + + +]] +template +struct tuple_element { + typedef typename gtest_internal::TupleElement< + k < (tuple_size::value), k, Tuple>::type type; +}; + +#define GTEST_TUPLE_ELEMENT_(k, Tuple) typename tuple_element::type + +// 6.1.3.4 Element access. + +namespace gtest_internal { + + +$for i [[ +template <> +class Get<$i> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_($i, Tuple)) + Field(Tuple& t) { return t.f$(i)_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_($i, Tuple)) + ConstField(const Tuple& t) { return t.f$(i)_; } +}; + + +]] +} // namespace gtest_internal + +template +GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_$(n)_TUPLE_(T))) +get(GTEST_$(n)_TUPLE_(T)& t) { + return gtest_internal::Get::Field(t); +} + +template +GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_$(n)_TUPLE_(T))) +get(const GTEST_$(n)_TUPLE_(T)& t) { + return gtest_internal::Get::ConstField(t); +} + +// 6.1.3.5 Relational operators + +// We only implement == and !=, as we don't have a need for the rest yet. + +namespace gtest_internal { + +// SameSizeTuplePrefixComparator::Eq(t1, t2) returns true if the +// first k fields of t1 equals the first k fields of t2. +// SameSizeTuplePrefixComparator(k1, k2) would be a compiler error if +// k1 != k2. +template +struct SameSizeTuplePrefixComparator; + +template <> +struct SameSizeTuplePrefixComparator<0, 0> { + template + static bool Eq(const Tuple1& /* t1 */, const Tuple2& /* t2 */) { + return true; + } +}; + +template +struct SameSizeTuplePrefixComparator { + template + static bool Eq(const Tuple1& t1, const Tuple2& t2) { + return SameSizeTuplePrefixComparator::Eq(t1, t2) && + ::std::tr1::get(t1) == ::std::tr1::get(t2); + } +}; + +} // namespace gtest_internal + +template +inline bool operator==(const GTEST_$(n)_TUPLE_(T)& t, + const GTEST_$(n)_TUPLE_(U)& u) { + return gtest_internal::SameSizeTuplePrefixComparator< + tuple_size::value, + tuple_size::value>::Eq(t, u); +} + +template +inline bool operator!=(const GTEST_$(n)_TUPLE_(T)& t, + const GTEST_$(n)_TUPLE_(U)& u) { return !(t == u); } + +// 6.1.4 Pairs. +// Unimplemented. + +} // namespace tr1 +} // namespace std + + +$for j [[ +#undef GTEST_$(j)_TUPLE_ + +]] + + +$for j [[ +#undef GTEST_$(j)_TYPENAMES_ + +]] + +#undef GTEST_DECLARE_TUPLE_AS_FRIEND_ +#undef GTEST_BY_REF_ +#undef GTEST_ADD_REF_ +#undef GTEST_TUPLE_ELEMENT_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-type-util.h b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-type-util.h new file mode 100644 index 0000000000..e46f7cfcb4 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-type-util.h @@ -0,0 +1,3331 @@ +// This file was GENERATED by command: +// pump.py gtest-type-util.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Type utilities needed for implementing typed and type-parameterized +// tests. This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// Currently we support at most 50 types in a list, and at most 50 +// type-parameterized tests in one type-parameterized test case. +// Please contact googletestframework@googlegroups.com if you need +// more. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ + +#include "gtest/internal/gtest-port.h" + +// #ifdef __GNUC__ is too general here. It is possible to use gcc without using +// libstdc++ (which is where cxxabi.h comes from). +# if GTEST_HAS_CXXABI_H_ +# include +# elif defined(__HP_aCC) +# include +# endif // GTEST_HASH_CXXABI_H_ + +namespace testing { +namespace internal { + +// GetTypeName() returns a human-readable name of type T. +// NB: This function is also used in Google Mock, so don't move it inside of +// the typed-test-only section below. +template +std::string GetTypeName() { +# if GTEST_HAS_RTTI + + const char* const name = typeid(T).name(); +# if GTEST_HAS_CXXABI_H_ || defined(__HP_aCC) + int status = 0; + // gcc's implementation of typeid(T).name() mangles the type name, + // so we have to demangle it. +# if GTEST_HAS_CXXABI_H_ + using abi::__cxa_demangle; +# endif // GTEST_HAS_CXXABI_H_ + char* const readable_name = __cxa_demangle(name, 0, 0, &status); + const std::string name_str(status == 0 ? readable_name : name); + free(readable_name); + return name_str; +# else + return name; +# endif // GTEST_HAS_CXXABI_H_ || __HP_aCC + +# else + + return ""; + +# endif // GTEST_HAS_RTTI +} + +#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// AssertyTypeEq::type is defined iff T1 and T2 are the same +// type. This can be used as a compile-time assertion to ensure that +// two types are equal. + +template +struct AssertTypeEq; + +template +struct AssertTypeEq { + typedef bool type; +}; + +// A unique type used as the default value for the arguments of class +// template Types. This allows us to simulate variadic templates +// (e.g. Types, Type, and etc), which C++ doesn't +// support directly. +struct None {}; + +// The following family of struct and struct templates are used to +// represent type lists. In particular, TypesN +// represents a type list with N types (T1, T2, ..., and TN) in it. +// Except for Types0, every struct in the family has two member types: +// Head for the first type in the list, and Tail for the rest of the +// list. + +// The empty type list. +struct Types0 {}; + +// Type lists of length 1, 2, 3, and so on. + +template +struct Types1 { + typedef T1 Head; + typedef Types0 Tail; +}; +template +struct Types2 { + typedef T1 Head; + typedef Types1 Tail; +}; + +template +struct Types3 { + typedef T1 Head; + typedef Types2 Tail; +}; + +template +struct Types4 { + typedef T1 Head; + typedef Types3 Tail; +}; + +template +struct Types5 { + typedef T1 Head; + typedef Types4 Tail; +}; + +template +struct Types6 { + typedef T1 Head; + typedef Types5 Tail; +}; + +template +struct Types7 { + typedef T1 Head; + typedef Types6 Tail; +}; + +template +struct Types8 { + typedef T1 Head; + typedef Types7 Tail; +}; + +template +struct Types9 { + typedef T1 Head; + typedef Types8 Tail; +}; + +template +struct Types10 { + typedef T1 Head; + typedef Types9 Tail; +}; + +template +struct Types11 { + typedef T1 Head; + typedef Types10 Tail; +}; + +template +struct Types12 { + typedef T1 Head; + typedef Types11 Tail; +}; + +template +struct Types13 { + typedef T1 Head; + typedef Types12 Tail; +}; + +template +struct Types14 { + typedef T1 Head; + typedef Types13 Tail; +}; + +template +struct Types15 { + typedef T1 Head; + typedef Types14 Tail; +}; + +template +struct Types16 { + typedef T1 Head; + typedef Types15 Tail; +}; + +template +struct Types17 { + typedef T1 Head; + typedef Types16 Tail; +}; + +template +struct Types18 { + typedef T1 Head; + typedef Types17 Tail; +}; + +template +struct Types19 { + typedef T1 Head; + typedef Types18 Tail; +}; + +template +struct Types20 { + typedef T1 Head; + typedef Types19 Tail; +}; + +template +struct Types21 { + typedef T1 Head; + typedef Types20 Tail; +}; + +template +struct Types22 { + typedef T1 Head; + typedef Types21 Tail; +}; + +template +struct Types23 { + typedef T1 Head; + typedef Types22 Tail; +}; + +template +struct Types24 { + typedef T1 Head; + typedef Types23 Tail; +}; + +template +struct Types25 { + typedef T1 Head; + typedef Types24 Tail; +}; + +template +struct Types26 { + typedef T1 Head; + typedef Types25 Tail; +}; + +template +struct Types27 { + typedef T1 Head; + typedef Types26 Tail; +}; + +template +struct Types28 { + typedef T1 Head; + typedef Types27 Tail; +}; + +template +struct Types29 { + typedef T1 Head; + typedef Types28 Tail; +}; + +template +struct Types30 { + typedef T1 Head; + typedef Types29 Tail; +}; + +template +struct Types31 { + typedef T1 Head; + typedef Types30 Tail; +}; + +template +struct Types32 { + typedef T1 Head; + typedef Types31 Tail; +}; + +template +struct Types33 { + typedef T1 Head; + typedef Types32 Tail; +}; + +template +struct Types34 { + typedef T1 Head; + typedef Types33 Tail; +}; + +template +struct Types35 { + typedef T1 Head; + typedef Types34 Tail; +}; + +template +struct Types36 { + typedef T1 Head; + typedef Types35 Tail; +}; + +template +struct Types37 { + typedef T1 Head; + typedef Types36 Tail; +}; + +template +struct Types38 { + typedef T1 Head; + typedef Types37 Tail; +}; + +template +struct Types39 { + typedef T1 Head; + typedef Types38 Tail; +}; + +template +struct Types40 { + typedef T1 Head; + typedef Types39 Tail; +}; + +template +struct Types41 { + typedef T1 Head; + typedef Types40 Tail; +}; + +template +struct Types42 { + typedef T1 Head; + typedef Types41 Tail; +}; + +template +struct Types43 { + typedef T1 Head; + typedef Types42 Tail; +}; + +template +struct Types44 { + typedef T1 Head; + typedef Types43 Tail; +}; + +template +struct Types45 { + typedef T1 Head; + typedef Types44 Tail; +}; + +template +struct Types46 { + typedef T1 Head; + typedef Types45 Tail; +}; + +template +struct Types47 { + typedef T1 Head; + typedef Types46 Tail; +}; + +template +struct Types48 { + typedef T1 Head; + typedef Types47 Tail; +}; + +template +struct Types49 { + typedef T1 Head; + typedef Types48 Tail; +}; + +template +struct Types50 { + typedef T1 Head; + typedef Types49 Tail; +}; + + +} // namespace internal + +// We don't want to require the users to write TypesN<...> directly, +// as that would require them to count the length. Types<...> is much +// easier to write, but generates horrible messages when there is a +// compiler error, as gcc insists on printing out each template +// argument, even if it has the default value (this means Types +// will appear as Types in the compiler +// errors). +// +// Our solution is to combine the best part of the two approaches: a +// user would write Types, and Google Test will translate +// that to TypesN internally to make error messages +// readable. The translation is done by the 'type' member of the +// Types template. +template +struct Types { + typedef internal::Types50 type; +}; + +template <> +struct Types { + typedef internal::Types0 type; +}; +template +struct Types { + typedef internal::Types1 type; +}; +template +struct Types { + typedef internal::Types2 type; +}; +template +struct Types { + typedef internal::Types3 type; +}; +template +struct Types { + typedef internal::Types4 type; +}; +template +struct Types { + typedef internal::Types5 type; +}; +template +struct Types { + typedef internal::Types6 type; +}; +template +struct Types { + typedef internal::Types7 type; +}; +template +struct Types { + typedef internal::Types8 type; +}; +template +struct Types { + typedef internal::Types9 type; +}; +template +struct Types { + typedef internal::Types10 type; +}; +template +struct Types { + typedef internal::Types11 type; +}; +template +struct Types { + typedef internal::Types12 type; +}; +template +struct Types { + typedef internal::Types13 type; +}; +template +struct Types { + typedef internal::Types14 type; +}; +template +struct Types { + typedef internal::Types15 type; +}; +template +struct Types { + typedef internal::Types16 type; +}; +template +struct Types { + typedef internal::Types17 type; +}; +template +struct Types { + typedef internal::Types18 type; +}; +template +struct Types { + typedef internal::Types19 type; +}; +template +struct Types { + typedef internal::Types20 type; +}; +template +struct Types { + typedef internal::Types21 type; +}; +template +struct Types { + typedef internal::Types22 type; +}; +template +struct Types { + typedef internal::Types23 type; +}; +template +struct Types { + typedef internal::Types24 type; +}; +template +struct Types { + typedef internal::Types25 type; +}; +template +struct Types { + typedef internal::Types26 type; +}; +template +struct Types { + typedef internal::Types27 type; +}; +template +struct Types { + typedef internal::Types28 type; +}; +template +struct Types { + typedef internal::Types29 type; +}; +template +struct Types { + typedef internal::Types30 type; +}; +template +struct Types { + typedef internal::Types31 type; +}; +template +struct Types { + typedef internal::Types32 type; +}; +template +struct Types { + typedef internal::Types33 type; +}; +template +struct Types { + typedef internal::Types34 type; +}; +template +struct Types { + typedef internal::Types35 type; +}; +template +struct Types { + typedef internal::Types36 type; +}; +template +struct Types { + typedef internal::Types37 type; +}; +template +struct Types { + typedef internal::Types38 type; +}; +template +struct Types { + typedef internal::Types39 type; +}; +template +struct Types { + typedef internal::Types40 type; +}; +template +struct Types { + typedef internal::Types41 type; +}; +template +struct Types { + typedef internal::Types42 type; +}; +template +struct Types { + typedef internal::Types43 type; +}; +template +struct Types { + typedef internal::Types44 type; +}; +template +struct Types { + typedef internal::Types45 type; +}; +template +struct Types { + typedef internal::Types46 type; +}; +template +struct Types { + typedef internal::Types47 type; +}; +template +struct Types { + typedef internal::Types48 type; +}; +template +struct Types { + typedef internal::Types49 type; +}; + +namespace internal { + +# define GTEST_TEMPLATE_ template class + +// The template "selector" struct TemplateSel is used to +// represent Tmpl, which must be a class template with one type +// parameter, as a type. TemplateSel::Bind::type is defined +// as the type Tmpl. This allows us to actually instantiate the +// template "selected" by TemplateSel. +// +// This trick is necessary for simulating typedef for class templates, +// which C++ doesn't support directly. +template +struct TemplateSel { + template + struct Bind { + typedef Tmpl type; + }; +}; + +# define GTEST_BIND_(TmplSel, T) \ + TmplSel::template Bind::type + +// A unique struct template used as the default value for the +// arguments of class template Templates. This allows us to simulate +// variadic templates (e.g. Templates, Templates, +// and etc), which C++ doesn't support directly. +template +struct NoneT {}; + +// The following family of struct and struct templates are used to +// represent template lists. In particular, TemplatesN represents a list of N templates (T1, T2, ..., and TN). Except +// for Templates0, every struct in the family has two member types: +// Head for the selector of the first template in the list, and Tail +// for the rest of the list. + +// The empty template list. +struct Templates0 {}; + +// Template lists of length 1, 2, 3, and so on. + +template +struct Templates1 { + typedef TemplateSel Head; + typedef Templates0 Tail; +}; +template +struct Templates2 { + typedef TemplateSel Head; + typedef Templates1 Tail; +}; + +template +struct Templates3 { + typedef TemplateSel Head; + typedef Templates2 Tail; +}; + +template +struct Templates4 { + typedef TemplateSel Head; + typedef Templates3 Tail; +}; + +template +struct Templates5 { + typedef TemplateSel Head; + typedef Templates4 Tail; +}; + +template +struct Templates6 { + typedef TemplateSel Head; + typedef Templates5 Tail; +}; + +template +struct Templates7 { + typedef TemplateSel Head; + typedef Templates6 Tail; +}; + +template +struct Templates8 { + typedef TemplateSel Head; + typedef Templates7 Tail; +}; + +template +struct Templates9 { + typedef TemplateSel Head; + typedef Templates8 Tail; +}; + +template +struct Templates10 { + typedef TemplateSel Head; + typedef Templates9 Tail; +}; + +template +struct Templates11 { + typedef TemplateSel Head; + typedef Templates10 Tail; +}; + +template +struct Templates12 { + typedef TemplateSel Head; + typedef Templates11 Tail; +}; + +template +struct Templates13 { + typedef TemplateSel Head; + typedef Templates12 Tail; +}; + +template +struct Templates14 { + typedef TemplateSel Head; + typedef Templates13 Tail; +}; + +template +struct Templates15 { + typedef TemplateSel Head; + typedef Templates14 Tail; +}; + +template +struct Templates16 { + typedef TemplateSel Head; + typedef Templates15 Tail; +}; + +template +struct Templates17 { + typedef TemplateSel Head; + typedef Templates16 Tail; +}; + +template +struct Templates18 { + typedef TemplateSel Head; + typedef Templates17 Tail; +}; + +template +struct Templates19 { + typedef TemplateSel Head; + typedef Templates18 Tail; +}; + +template +struct Templates20 { + typedef TemplateSel Head; + typedef Templates19 Tail; +}; + +template +struct Templates21 { + typedef TemplateSel Head; + typedef Templates20 Tail; +}; + +template +struct Templates22 { + typedef TemplateSel Head; + typedef Templates21 Tail; +}; + +template +struct Templates23 { + typedef TemplateSel Head; + typedef Templates22 Tail; +}; + +template +struct Templates24 { + typedef TemplateSel Head; + typedef Templates23 Tail; +}; + +template +struct Templates25 { + typedef TemplateSel Head; + typedef Templates24 Tail; +}; + +template +struct Templates26 { + typedef TemplateSel Head; + typedef Templates25 Tail; +}; + +template +struct Templates27 { + typedef TemplateSel Head; + typedef Templates26 Tail; +}; + +template +struct Templates28 { + typedef TemplateSel Head; + typedef Templates27 Tail; +}; + +template +struct Templates29 { + typedef TemplateSel Head; + typedef Templates28 Tail; +}; + +template +struct Templates30 { + typedef TemplateSel Head; + typedef Templates29 Tail; +}; + +template +struct Templates31 { + typedef TemplateSel Head; + typedef Templates30 Tail; +}; + +template +struct Templates32 { + typedef TemplateSel Head; + typedef Templates31 Tail; +}; + +template +struct Templates33 { + typedef TemplateSel Head; + typedef Templates32 Tail; +}; + +template +struct Templates34 { + typedef TemplateSel Head; + typedef Templates33 Tail; +}; + +template +struct Templates35 { + typedef TemplateSel Head; + typedef Templates34 Tail; +}; + +template +struct Templates36 { + typedef TemplateSel Head; + typedef Templates35 Tail; +}; + +template +struct Templates37 { + typedef TemplateSel Head; + typedef Templates36 Tail; +}; + +template +struct Templates38 { + typedef TemplateSel Head; + typedef Templates37 Tail; +}; + +template +struct Templates39 { + typedef TemplateSel Head; + typedef Templates38 Tail; +}; + +template +struct Templates40 { + typedef TemplateSel Head; + typedef Templates39 Tail; +}; + +template +struct Templates41 { + typedef TemplateSel Head; + typedef Templates40 Tail; +}; + +template +struct Templates42 { + typedef TemplateSel Head; + typedef Templates41 Tail; +}; + +template +struct Templates43 { + typedef TemplateSel Head; + typedef Templates42 Tail; +}; + +template +struct Templates44 { + typedef TemplateSel Head; + typedef Templates43 Tail; +}; + +template +struct Templates45 { + typedef TemplateSel Head; + typedef Templates44 Tail; +}; + +template +struct Templates46 { + typedef TemplateSel Head; + typedef Templates45 Tail; +}; + +template +struct Templates47 { + typedef TemplateSel Head; + typedef Templates46 Tail; +}; + +template +struct Templates48 { + typedef TemplateSel Head; + typedef Templates47 Tail; +}; + +template +struct Templates49 { + typedef TemplateSel Head; + typedef Templates48 Tail; +}; + +template +struct Templates50 { + typedef TemplateSel Head; + typedef Templates49 Tail; +}; + + +// We don't want to require the users to write TemplatesN<...> directly, +// as that would require them to count the length. Templates<...> is much +// easier to write, but generates horrible messages when there is a +// compiler error, as gcc insists on printing out each template +// argument, even if it has the default value (this means Templates +// will appear as Templates in the compiler +// errors). +// +// Our solution is to combine the best part of the two approaches: a +// user would write Templates, and Google Test will translate +// that to TemplatesN internally to make error messages +// readable. The translation is done by the 'type' member of the +// Templates template. +template +struct Templates { + typedef Templates50 type; +}; + +template <> +struct Templates { + typedef Templates0 type; +}; +template +struct Templates { + typedef Templates1 type; +}; +template +struct Templates { + typedef Templates2 type; +}; +template +struct Templates { + typedef Templates3 type; +}; +template +struct Templates { + typedef Templates4 type; +}; +template +struct Templates { + typedef Templates5 type; +}; +template +struct Templates { + typedef Templates6 type; +}; +template +struct Templates { + typedef Templates7 type; +}; +template +struct Templates { + typedef Templates8 type; +}; +template +struct Templates { + typedef Templates9 type; +}; +template +struct Templates { + typedef Templates10 type; +}; +template +struct Templates { + typedef Templates11 type; +}; +template +struct Templates { + typedef Templates12 type; +}; +template +struct Templates { + typedef Templates13 type; +}; +template +struct Templates { + typedef Templates14 type; +}; +template +struct Templates { + typedef Templates15 type; +}; +template +struct Templates { + typedef Templates16 type; +}; +template +struct Templates { + typedef Templates17 type; +}; +template +struct Templates { + typedef Templates18 type; +}; +template +struct Templates { + typedef Templates19 type; +}; +template +struct Templates { + typedef Templates20 type; +}; +template +struct Templates { + typedef Templates21 type; +}; +template +struct Templates { + typedef Templates22 type; +}; +template +struct Templates { + typedef Templates23 type; +}; +template +struct Templates { + typedef Templates24 type; +}; +template +struct Templates { + typedef Templates25 type; +}; +template +struct Templates { + typedef Templates26 type; +}; +template +struct Templates { + typedef Templates27 type; +}; +template +struct Templates { + typedef Templates28 type; +}; +template +struct Templates { + typedef Templates29 type; +}; +template +struct Templates { + typedef Templates30 type; +}; +template +struct Templates { + typedef Templates31 type; +}; +template +struct Templates { + typedef Templates32 type; +}; +template +struct Templates { + typedef Templates33 type; +}; +template +struct Templates { + typedef Templates34 type; +}; +template +struct Templates { + typedef Templates35 type; +}; +template +struct Templates { + typedef Templates36 type; +}; +template +struct Templates { + typedef Templates37 type; +}; +template +struct Templates { + typedef Templates38 type; +}; +template +struct Templates { + typedef Templates39 type; +}; +template +struct Templates { + typedef Templates40 type; +}; +template +struct Templates { + typedef Templates41 type; +}; +template +struct Templates { + typedef Templates42 type; +}; +template +struct Templates { + typedef Templates43 type; +}; +template +struct Templates { + typedef Templates44 type; +}; +template +struct Templates { + typedef Templates45 type; +}; +template +struct Templates { + typedef Templates46 type; +}; +template +struct Templates { + typedef Templates47 type; +}; +template +struct Templates { + typedef Templates48 type; +}; +template +struct Templates { + typedef Templates49 type; +}; + +// The TypeList template makes it possible to use either a single type +// or a Types<...> list in TYPED_TEST_CASE() and +// INSTANTIATE_TYPED_TEST_CASE_P(). + +template +struct TypeList { + typedef Types1 type; +}; + +template +struct TypeList > { + typedef typename Types::type type; +}; + +#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-type-util.h.pump b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-type-util.h.pump new file mode 100644 index 0000000000..251fdf025b --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/include/gtest/internal/gtest-type-util.h.pump @@ -0,0 +1,297 @@ +$$ -*- mode: c++; -*- +$var n = 50 $$ Maximum length of type lists we want to support. +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Type utilities needed for implementing typed and type-parameterized +// tests. This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// Currently we support at most $n types in a list, and at most $n +// type-parameterized tests in one type-parameterized test case. +// Please contact googletestframework@googlegroups.com if you need +// more. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ + +#include "gtest/internal/gtest-port.h" + +// #ifdef __GNUC__ is too general here. It is possible to use gcc without using +// libstdc++ (which is where cxxabi.h comes from). +# if GTEST_HAS_CXXABI_H_ +# include +# elif defined(__HP_aCC) +# include +# endif // GTEST_HASH_CXXABI_H_ + +namespace testing { +namespace internal { + +// GetTypeName() returns a human-readable name of type T. +// NB: This function is also used in Google Mock, so don't move it inside of +// the typed-test-only section below. +template +std::string GetTypeName() { +# if GTEST_HAS_RTTI + + const char* const name = typeid(T).name(); +# if GTEST_HAS_CXXABI_H_ || defined(__HP_aCC) + int status = 0; + // gcc's implementation of typeid(T).name() mangles the type name, + // so we have to demangle it. +# if GTEST_HAS_CXXABI_H_ + using abi::__cxa_demangle; +# endif // GTEST_HAS_CXXABI_H_ + char* const readable_name = __cxa_demangle(name, 0, 0, &status); + const std::string name_str(status == 0 ? readable_name : name); + free(readable_name); + return name_str; +# else + return name; +# endif // GTEST_HAS_CXXABI_H_ || __HP_aCC + +# else + + return ""; + +# endif // GTEST_HAS_RTTI +} + +#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// AssertyTypeEq::type is defined iff T1 and T2 are the same +// type. This can be used as a compile-time assertion to ensure that +// two types are equal. + +template +struct AssertTypeEq; + +template +struct AssertTypeEq { + typedef bool type; +}; + +// A unique type used as the default value for the arguments of class +// template Types. This allows us to simulate variadic templates +// (e.g. Types, Type, and etc), which C++ doesn't +// support directly. +struct None {}; + +// The following family of struct and struct templates are used to +// represent type lists. In particular, TypesN +// represents a type list with N types (T1, T2, ..., and TN) in it. +// Except for Types0, every struct in the family has two member types: +// Head for the first type in the list, and Tail for the rest of the +// list. + +// The empty type list. +struct Types0 {}; + +// Type lists of length 1, 2, 3, and so on. + +template +struct Types1 { + typedef T1 Head; + typedef Types0 Tail; +}; + +$range i 2..n + +$for i [[ +$range j 1..i +$range k 2..i +template <$for j, [[typename T$j]]> +struct Types$i { + typedef T1 Head; + typedef Types$(i-1)<$for k, [[T$k]]> Tail; +}; + + +]] + +} // namespace internal + +// We don't want to require the users to write TypesN<...> directly, +// as that would require them to count the length. Types<...> is much +// easier to write, but generates horrible messages when there is a +// compiler error, as gcc insists on printing out each template +// argument, even if it has the default value (this means Types +// will appear as Types in the compiler +// errors). +// +// Our solution is to combine the best part of the two approaches: a +// user would write Types, and Google Test will translate +// that to TypesN internally to make error messages +// readable. The translation is done by the 'type' member of the +// Types template. + +$range i 1..n +template <$for i, [[typename T$i = internal::None]]> +struct Types { + typedef internal::Types$n<$for i, [[T$i]]> type; +}; + +template <> +struct Types<$for i, [[internal::None]]> { + typedef internal::Types0 type; +}; + +$range i 1..n-1 +$for i [[ +$range j 1..i +$range k i+1..n +template <$for j, [[typename T$j]]> +struct Types<$for j, [[T$j]]$for k[[, internal::None]]> { + typedef internal::Types$i<$for j, [[T$j]]> type; +}; + +]] + +namespace internal { + +# define GTEST_TEMPLATE_ template class + +// The template "selector" struct TemplateSel is used to +// represent Tmpl, which must be a class template with one type +// parameter, as a type. TemplateSel::Bind::type is defined +// as the type Tmpl. This allows us to actually instantiate the +// template "selected" by TemplateSel. +// +// This trick is necessary for simulating typedef for class templates, +// which C++ doesn't support directly. +template +struct TemplateSel { + template + struct Bind { + typedef Tmpl type; + }; +}; + +# define GTEST_BIND_(TmplSel, T) \ + TmplSel::template Bind::type + +// A unique struct template used as the default value for the +// arguments of class template Templates. This allows us to simulate +// variadic templates (e.g. Templates, Templates, +// and etc), which C++ doesn't support directly. +template +struct NoneT {}; + +// The following family of struct and struct templates are used to +// represent template lists. In particular, TemplatesN represents a list of N templates (T1, T2, ..., and TN). Except +// for Templates0, every struct in the family has two member types: +// Head for the selector of the first template in the list, and Tail +// for the rest of the list. + +// The empty template list. +struct Templates0 {}; + +// Template lists of length 1, 2, 3, and so on. + +template +struct Templates1 { + typedef TemplateSel Head; + typedef Templates0 Tail; +}; + +$range i 2..n + +$for i [[ +$range j 1..i +$range k 2..i +template <$for j, [[GTEST_TEMPLATE_ T$j]]> +struct Templates$i { + typedef TemplateSel Head; + typedef Templates$(i-1)<$for k, [[T$k]]> Tail; +}; + + +]] + +// We don't want to require the users to write TemplatesN<...> directly, +// as that would require them to count the length. Templates<...> is much +// easier to write, but generates horrible messages when there is a +// compiler error, as gcc insists on printing out each template +// argument, even if it has the default value (this means Templates +// will appear as Templates in the compiler +// errors). +// +// Our solution is to combine the best part of the two approaches: a +// user would write Templates, and Google Test will translate +// that to TemplatesN internally to make error messages +// readable. The translation is done by the 'type' member of the +// Templates template. + +$range i 1..n +template <$for i, [[GTEST_TEMPLATE_ T$i = NoneT]]> +struct Templates { + typedef Templates$n<$for i, [[T$i]]> type; +}; + +template <> +struct Templates<$for i, [[NoneT]]> { + typedef Templates0 type; +}; + +$range i 1..n-1 +$for i [[ +$range j 1..i +$range k i+1..n +template <$for j, [[GTEST_TEMPLATE_ T$j]]> +struct Templates<$for j, [[T$j]]$for k[[, NoneT]]> { + typedef Templates$i<$for j, [[T$j]]> type; +}; + +]] + +// The TypeList template makes it possible to use either a single type +// or a Types<...> list in TYPED_TEST_CASE() and +// INSTANTIATE_TYPED_TEST_CASE_P(). + +template +struct TypeList { + typedef Types1 type; +}; + + +$range i 1..n +template <$for i, [[typename T$i]]> +struct TypeList > { + typedef typename Types<$for i, [[T$i]]>::type type; +}; + +#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/src/gtest-all.cc b/third_party/aom/third_party/googletest/src/googletest/src/gtest-all.cc new file mode 100644 index 0000000000..0a9cee5223 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/src/gtest-all.cc @@ -0,0 +1,48 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: mheule@google.com (Markus Heule) +// +// Google C++ Testing Framework (Google Test) +// +// Sometimes it's desirable to build Google Test by compiling a single file. +// This file serves this purpose. + +// This line ensures that gtest.h can be compiled on its own, even +// when it's fused. +#include "gtest/gtest.h" + +// The following lines pull in the real gtest *.cc files. +#include "src/gtest.cc" +#include "src/gtest-death-test.cc" +#include "src/gtest-filepath.cc" +#include "src/gtest-port.cc" +#include "src/gtest-printers.cc" +#include "src/gtest-test-part.cc" +#include "src/gtest-typed-test.cc" diff --git a/third_party/aom/third_party/googletest/src/googletest/src/gtest-death-test.cc b/third_party/aom/third_party/googletest/src/googletest/src/gtest-death-test.cc new file mode 100644 index 0000000000..a01a369830 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/src/gtest-death-test.cc @@ -0,0 +1,1342 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan), vladl@google.com (Vlad Losev) +// +// This file implements death tests. + +#include "gtest/gtest-death-test.h" +#include "gtest/internal/gtest-port.h" +#include "gtest/internal/custom/gtest.h" + +#if GTEST_HAS_DEATH_TEST + +# if GTEST_OS_MAC +# include +# endif // GTEST_OS_MAC + +# include +# include +# include + +# if GTEST_OS_LINUX +# include +# endif // GTEST_OS_LINUX + +# include + +# if GTEST_OS_WINDOWS +# include +# else +# include +# include +# endif // GTEST_OS_WINDOWS + +# if GTEST_OS_QNX +# include +# endif // GTEST_OS_QNX + +#endif // GTEST_HAS_DEATH_TEST + +#include "gtest/gtest-message.h" +#include "gtest/internal/gtest-string.h" + +// Indicates that this translation unit is part of Google Test's +// implementation. It must come before gtest-internal-inl.h is +// included, or there will be a compiler error. This trick exists to +// prevent the accidental inclusion of gtest-internal-inl.h in the +// user's code. +#define GTEST_IMPLEMENTATION_ 1 +#include "src/gtest-internal-inl.h" +#undef GTEST_IMPLEMENTATION_ + +namespace testing { + +// Constants. + +// The default death test style. +static const char kDefaultDeathTestStyle[] = "fast"; + +GTEST_DEFINE_string_( + death_test_style, + internal::StringFromGTestEnv("death_test_style", kDefaultDeathTestStyle), + "Indicates how to run a death test in a forked child process: " + "\"threadsafe\" (child process re-executes the test binary " + "from the beginning, running only the specific death test) or " + "\"fast\" (child process runs the death test immediately " + "after forking)."); + +GTEST_DEFINE_bool_( + death_test_use_fork, + internal::BoolFromGTestEnv("death_test_use_fork", false), + "Instructs to use fork()/_exit() instead of clone() in death tests. " + "Ignored and always uses fork() on POSIX systems where clone() is not " + "implemented. Useful when running under valgrind or similar tools if " + "those do not support clone(). Valgrind 3.3.1 will just fail if " + "it sees an unsupported combination of clone() flags. " + "It is not recommended to use this flag w/o valgrind though it will " + "work in 99% of the cases. Once valgrind is fixed, this flag will " + "most likely be removed."); + +namespace internal { +GTEST_DEFINE_string_( + internal_run_death_test, "", + "Indicates the file, line number, temporal index of " + "the single death test to run, and a file descriptor to " + "which a success code may be sent, all separated by " + "the '|' characters. This flag is specified if and only if the current " + "process is a sub-process launched for running a thread-safe " + "death test. FOR INTERNAL USE ONLY."); +} // namespace internal + +#if GTEST_HAS_DEATH_TEST + +namespace internal { + +// Valid only for fast death tests. Indicates the code is running in the +// child process of a fast style death test. +# if !GTEST_OS_WINDOWS +static bool g_in_fast_death_test_child = false; +# endif + +// Returns a Boolean value indicating whether the caller is currently +// executing in the context of the death test child process. Tools such as +// Valgrind heap checkers may need this to modify their behavior in death +// tests. IMPORTANT: This is an internal utility. Using it may break the +// implementation of death tests. User code MUST NOT use it. +bool InDeathTestChild() { +# if GTEST_OS_WINDOWS + + // On Windows, death tests are thread-safe regardless of the value of the + // death_test_style flag. + return !GTEST_FLAG(internal_run_death_test).empty(); + +# else + + if (GTEST_FLAG(death_test_style) == "threadsafe") + return !GTEST_FLAG(internal_run_death_test).empty(); + else + return g_in_fast_death_test_child; +#endif +} + +} // namespace internal + +// ExitedWithCode constructor. +ExitedWithCode::ExitedWithCode(int exit_code) : exit_code_(exit_code) { +} + +// ExitedWithCode function-call operator. +bool ExitedWithCode::operator()(int exit_status) const { +# if GTEST_OS_WINDOWS + + return exit_status == exit_code_; + +# else + + return WIFEXITED(exit_status) && WEXITSTATUS(exit_status) == exit_code_; + +# endif // GTEST_OS_WINDOWS +} + +# if !GTEST_OS_WINDOWS +// KilledBySignal constructor. +KilledBySignal::KilledBySignal(int signum) : signum_(signum) { +} + +// KilledBySignal function-call operator. +bool KilledBySignal::operator()(int exit_status) const { +# if defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_) + { + bool result; + if (GTEST_KILLED_BY_SIGNAL_OVERRIDE_(signum_, exit_status, &result)) { + return result; + } + } +# endif // defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_) + return WIFSIGNALED(exit_status) && WTERMSIG(exit_status) == signum_; +} +# endif // !GTEST_OS_WINDOWS + +namespace internal { + +// Utilities needed for death tests. + +// Generates a textual description of a given exit code, in the format +// specified by wait(2). +static std::string ExitSummary(int exit_code) { + Message m; + +# if GTEST_OS_WINDOWS + + m << "Exited with exit status " << exit_code; + +# else + + if (WIFEXITED(exit_code)) { + m << "Exited with exit status " << WEXITSTATUS(exit_code); + } else if (WIFSIGNALED(exit_code)) { + m << "Terminated by signal " << WTERMSIG(exit_code); + } +# ifdef WCOREDUMP + if (WCOREDUMP(exit_code)) { + m << " (core dumped)"; + } +# endif +# endif // GTEST_OS_WINDOWS + + return m.GetString(); +} + +// Returns true if exit_status describes a process that was terminated +// by a signal, or exited normally with a nonzero exit code. +bool ExitedUnsuccessfully(int exit_status) { + return !ExitedWithCode(0)(exit_status); +} + +# if !GTEST_OS_WINDOWS +// Generates a textual failure message when a death test finds more than +// one thread running, or cannot determine the number of threads, prior +// to executing the given statement. It is the responsibility of the +// caller not to pass a thread_count of 1. +static std::string DeathTestThreadWarning(size_t thread_count) { + Message msg; + msg << "Death tests use fork(), which is unsafe particularly" + << " in a threaded context. For this test, " << GTEST_NAME_ << " "; + if (thread_count == 0) + msg << "couldn't detect the number of threads."; + else + msg << "detected " << thread_count << " threads."; + return msg.GetString(); +} +# endif // !GTEST_OS_WINDOWS + +// Flag characters for reporting a death test that did not die. +static const char kDeathTestLived = 'L'; +static const char kDeathTestReturned = 'R'; +static const char kDeathTestThrew = 'T'; +static const char kDeathTestInternalError = 'I'; + +// An enumeration describing all of the possible ways that a death test can +// conclude. DIED means that the process died while executing the test +// code; LIVED means that process lived beyond the end of the test code; +// RETURNED means that the test statement attempted to execute a return +// statement, which is not allowed; THREW means that the test statement +// returned control by throwing an exception. IN_PROGRESS means the test +// has not yet concluded. +// TODO(vladl@google.com): Unify names and possibly values for +// AbortReason, DeathTestOutcome, and flag characters above. +enum DeathTestOutcome { IN_PROGRESS, DIED, LIVED, RETURNED, THREW }; + +// Routine for aborting the program which is safe to call from an +// exec-style death test child process, in which case the error +// message is propagated back to the parent process. Otherwise, the +// message is simply printed to stderr. In either case, the program +// then exits with status 1. +void DeathTestAbort(const std::string& message) { + // On a POSIX system, this function may be called from a threadsafe-style + // death test child process, which operates on a very small stack. Use + // the heap for any additional non-minuscule memory requirements. + const InternalRunDeathTestFlag* const flag = + GetUnitTestImpl()->internal_run_death_test_flag(); + if (flag != NULL) { + FILE* parent = posix::FDOpen(flag->write_fd(), "w"); + fputc(kDeathTestInternalError, parent); + fprintf(parent, "%s", message.c_str()); + fflush(parent); + _exit(1); + } else { + fprintf(stderr, "%s", message.c_str()); + fflush(stderr); + posix::Abort(); + } +} + +// A replacement for CHECK that calls DeathTestAbort if the assertion +// fails. +# define GTEST_DEATH_TEST_CHECK_(expression) \ + do { \ + if (!::testing::internal::IsTrue(expression)) { \ + DeathTestAbort( \ + ::std::string("CHECK failed: File ") + __FILE__ + ", line " \ + + ::testing::internal::StreamableToString(__LINE__) + ": " \ + + #expression); \ + } \ + } while (::testing::internal::AlwaysFalse()) + +// This macro is similar to GTEST_DEATH_TEST_CHECK_, but it is meant for +// evaluating any system call that fulfills two conditions: it must return +// -1 on failure, and set errno to EINTR when it is interrupted and +// should be tried again. The macro expands to a loop that repeatedly +// evaluates the expression as long as it evaluates to -1 and sets +// errno to EINTR. If the expression evaluates to -1 but errno is +// something other than EINTR, DeathTestAbort is called. +# define GTEST_DEATH_TEST_CHECK_SYSCALL_(expression) \ + do { \ + int gtest_retval; \ + do { \ + gtest_retval = (expression); \ + } while (gtest_retval == -1 && errno == EINTR); \ + if (gtest_retval == -1) { \ + DeathTestAbort( \ + ::std::string("CHECK failed: File ") + __FILE__ + ", line " \ + + ::testing::internal::StreamableToString(__LINE__) + ": " \ + + #expression + " != -1"); \ + } \ + } while (::testing::internal::AlwaysFalse()) + +// Returns the message describing the last system error in errno. +std::string GetLastErrnoDescription() { + return errno == 0 ? "" : posix::StrError(errno); +} + +// This is called from a death test parent process to read a failure +// message from the death test child process and log it with the FATAL +// severity. On Windows, the message is read from a pipe handle. On other +// platforms, it is read from a file descriptor. +static void FailFromInternalError(int fd) { + Message error; + char buffer[256]; + int num_read; + + do { + while ((num_read = posix::Read(fd, buffer, 255)) > 0) { + buffer[num_read] = '\0'; + error << buffer; + } + } while (num_read == -1 && errno == EINTR); + + if (num_read == 0) { + GTEST_LOG_(FATAL) << error.GetString(); + } else { + const int last_error = errno; + GTEST_LOG_(FATAL) << "Error while reading death test internal: " + << GetLastErrnoDescription() << " [" << last_error << "]"; + } +} + +// Death test constructor. Increments the running death test count +// for the current test. +DeathTest::DeathTest() { + TestInfo* const info = GetUnitTestImpl()->current_test_info(); + if (info == NULL) { + DeathTestAbort("Cannot run a death test outside of a TEST or " + "TEST_F construct"); + } +} + +// Creates and returns a death test by dispatching to the current +// death test factory. +bool DeathTest::Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test) { + return GetUnitTestImpl()->death_test_factory()->Create( + statement, regex, file, line, test); +} + +const char* DeathTest::LastMessage() { + return last_death_test_message_.c_str(); +} + +void DeathTest::set_last_death_test_message(const std::string& message) { + last_death_test_message_ = message; +} + +std::string DeathTest::last_death_test_message_; + +// Provides cross platform implementation for some death functionality. +class DeathTestImpl : public DeathTest { + protected: + DeathTestImpl(const char* a_statement, const RE* a_regex) + : statement_(a_statement), + regex_(a_regex), + spawned_(false), + status_(-1), + outcome_(IN_PROGRESS), + read_fd_(-1), + write_fd_(-1) {} + + // read_fd_ is expected to be closed and cleared by a derived class. + ~DeathTestImpl() { GTEST_DEATH_TEST_CHECK_(read_fd_ == -1); } + + void Abort(AbortReason reason); + virtual bool Passed(bool status_ok); + + const char* statement() const { return statement_; } + const RE* regex() const { return regex_; } + bool spawned() const { return spawned_; } + void set_spawned(bool is_spawned) { spawned_ = is_spawned; } + int status() const { return status_; } + void set_status(int a_status) { status_ = a_status; } + DeathTestOutcome outcome() const { return outcome_; } + void set_outcome(DeathTestOutcome an_outcome) { outcome_ = an_outcome; } + int read_fd() const { return read_fd_; } + void set_read_fd(int fd) { read_fd_ = fd; } + int write_fd() const { return write_fd_; } + void set_write_fd(int fd) { write_fd_ = fd; } + + // Called in the parent process only. Reads the result code of the death + // test child process via a pipe, interprets it to set the outcome_ + // member, and closes read_fd_. Outputs diagnostics and terminates in + // case of unexpected codes. + void ReadAndInterpretStatusByte(); + + private: + // The textual content of the code this object is testing. This class + // doesn't own this string and should not attempt to delete it. + const char* const statement_; + // The regular expression which test output must match. DeathTestImpl + // doesn't own this object and should not attempt to delete it. + const RE* const regex_; + // True if the death test child process has been successfully spawned. + bool spawned_; + // The exit status of the child process. + int status_; + // How the death test concluded. + DeathTestOutcome outcome_; + // Descriptor to the read end of the pipe to the child process. It is + // always -1 in the child process. The child keeps its write end of the + // pipe in write_fd_. + int read_fd_; + // Descriptor to the child's write end of the pipe to the parent process. + // It is always -1 in the parent process. The parent keeps its end of the + // pipe in read_fd_. + int write_fd_; +}; + +// Called in the parent process only. Reads the result code of the death +// test child process via a pipe, interprets it to set the outcome_ +// member, and closes read_fd_. Outputs diagnostics and terminates in +// case of unexpected codes. +void DeathTestImpl::ReadAndInterpretStatusByte() { + char flag; + int bytes_read; + + // The read() here blocks until data is available (signifying the + // failure of the death test) or until the pipe is closed (signifying + // its success), so it's okay to call this in the parent before + // the child process has exited. + do { + bytes_read = posix::Read(read_fd(), &flag, 1); + } while (bytes_read == -1 && errno == EINTR); + + if (bytes_read == 0) { + set_outcome(DIED); + } else if (bytes_read == 1) { + switch (flag) { + case kDeathTestReturned: + set_outcome(RETURNED); + break; + case kDeathTestThrew: + set_outcome(THREW); + break; + case kDeathTestLived: + set_outcome(LIVED); + break; + case kDeathTestInternalError: + FailFromInternalError(read_fd()); // Does not return. + break; + default: + GTEST_LOG_(FATAL) << "Death test child process reported " + << "unexpected status byte (" + << static_cast(flag) << ")"; + } + } else { + GTEST_LOG_(FATAL) << "Read from death test child process failed: " + << GetLastErrnoDescription(); + } + GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Close(read_fd())); + set_read_fd(-1); +} + +// Signals that the death test code which should have exited, didn't. +// Should be called only in a death test child process. +// Writes a status byte to the child's status file descriptor, then +// calls _exit(1). +void DeathTestImpl::Abort(AbortReason reason) { + // The parent process considers the death test to be a failure if + // it finds any data in our pipe. So, here we write a single flag byte + // to the pipe, then exit. + const char status_ch = + reason == TEST_DID_NOT_DIE ? kDeathTestLived : + reason == TEST_THREW_EXCEPTION ? kDeathTestThrew : kDeathTestReturned; + + GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Write(write_fd(), &status_ch, 1)); + // We are leaking the descriptor here because on some platforms (i.e., + // when built as Windows DLL), destructors of global objects will still + // run after calling _exit(). On such systems, write_fd_ will be + // indirectly closed from the destructor of UnitTestImpl, causing double + // close if it is also closed here. On debug configurations, double close + // may assert. As there are no in-process buffers to flush here, we are + // relying on the OS to close the descriptor after the process terminates + // when the destructors are not run. + _exit(1); // Exits w/o any normal exit hooks (we were supposed to crash) +} + +// Returns an indented copy of stderr output for a death test. +// This makes distinguishing death test output lines from regular log lines +// much easier. +static ::std::string FormatDeathTestOutput(const ::std::string& output) { + ::std::string ret; + for (size_t at = 0; ; ) { + const size_t line_end = output.find('\n', at); + ret += "[ DEATH ] "; + if (line_end == ::std::string::npos) { + ret += output.substr(at); + break; + } + ret += output.substr(at, line_end + 1 - at); + at = line_end + 1; + } + return ret; +} + +// Assesses the success or failure of a death test, using both private +// members which have previously been set, and one argument: +// +// Private data members: +// outcome: An enumeration describing how the death test +// concluded: DIED, LIVED, THREW, or RETURNED. The death test +// fails in the latter three cases. +// status: The exit status of the child process. On *nix, it is in the +// in the format specified by wait(2). On Windows, this is the +// value supplied to the ExitProcess() API or a numeric code +// of the exception that terminated the program. +// regex: A regular expression object to be applied to +// the test's captured standard error output; the death test +// fails if it does not match. +// +// Argument: +// status_ok: true if exit_status is acceptable in the context of +// this particular death test, which fails if it is false +// +// Returns true iff all of the above conditions are met. Otherwise, the +// first failing condition, in the order given above, is the one that is +// reported. Also sets the last death test message string. +bool DeathTestImpl::Passed(bool status_ok) { + if (!spawned()) + return false; + + const std::string error_message = GetCapturedStderr(); + + bool success = false; + Message buffer; + + buffer << "Death test: " << statement() << "\n"; + switch (outcome()) { + case LIVED: + buffer << " Result: failed to die.\n" + << " Error msg:\n" << FormatDeathTestOutput(error_message); + break; + case THREW: + buffer << " Result: threw an exception.\n" + << " Error msg:\n" << FormatDeathTestOutput(error_message); + break; + case RETURNED: + buffer << " Result: illegal return in test statement.\n" + << " Error msg:\n" << FormatDeathTestOutput(error_message); + break; + case DIED: + if (status_ok) { + const bool matched = RE::PartialMatch(error_message.c_str(), *regex()); + if (matched) { + success = true; + } else { + buffer << " Result: died but not with expected error.\n" + << " Expected: " << regex()->pattern() << "\n" + << "Actual msg:\n" << FormatDeathTestOutput(error_message); + } + } else { + buffer << " Result: died but not with expected exit code:\n" + << " " << ExitSummary(status()) << "\n" + << "Actual msg:\n" << FormatDeathTestOutput(error_message); + } + break; + case IN_PROGRESS: + default: + GTEST_LOG_(FATAL) + << "DeathTest::Passed somehow called before conclusion of test"; + } + + DeathTest::set_last_death_test_message(buffer.GetString()); + return success; +} + +# if GTEST_OS_WINDOWS +// WindowsDeathTest implements death tests on Windows. Due to the +// specifics of starting new processes on Windows, death tests there are +// always threadsafe, and Google Test considers the +// --gtest_death_test_style=fast setting to be equivalent to +// --gtest_death_test_style=threadsafe there. +// +// A few implementation notes: Like the Linux version, the Windows +// implementation uses pipes for child-to-parent communication. But due to +// the specifics of pipes on Windows, some extra steps are required: +// +// 1. The parent creates a communication pipe and stores handles to both +// ends of it. +// 2. The parent starts the child and provides it with the information +// necessary to acquire the handle to the write end of the pipe. +// 3. The child acquires the write end of the pipe and signals the parent +// using a Windows event. +// 4. Now the parent can release the write end of the pipe on its side. If +// this is done before step 3, the object's reference count goes down to +// 0 and it is destroyed, preventing the child from acquiring it. The +// parent now has to release it, or read operations on the read end of +// the pipe will not return when the child terminates. +// 5. The parent reads child's output through the pipe (outcome code and +// any possible error messages) from the pipe, and its stderr and then +// determines whether to fail the test. +// +// Note: to distinguish Win32 API calls from the local method and function +// calls, the former are explicitly resolved in the global namespace. +// +class WindowsDeathTest : public DeathTestImpl { + public: + WindowsDeathTest(const char* a_statement, + const RE* a_regex, + const char* file, + int line) + : DeathTestImpl(a_statement, a_regex), file_(file), line_(line) {} + + // All of these virtual functions are inherited from DeathTest. + virtual int Wait(); + virtual TestRole AssumeRole(); + + private: + // The name of the file in which the death test is located. + const char* const file_; + // The line number on which the death test is located. + const int line_; + // Handle to the write end of the pipe to the child process. + AutoHandle write_handle_; + // Child process handle. + AutoHandle child_handle_; + // Event the child process uses to signal the parent that it has + // acquired the handle to the write end of the pipe. After seeing this + // event the parent can release its own handles to make sure its + // ReadFile() calls return when the child terminates. + AutoHandle event_handle_; +}; + +// Waits for the child in a death test to exit, returning its exit +// status, or 0 if no child process exists. As a side effect, sets the +// outcome data member. +int WindowsDeathTest::Wait() { + if (!spawned()) + return 0; + + // Wait until the child either signals that it has acquired the write end + // of the pipe or it dies. + const HANDLE wait_handles[2] = { child_handle_.Get(), event_handle_.Get() }; + switch (::WaitForMultipleObjects(2, + wait_handles, + FALSE, // Waits for any of the handles. + INFINITE)) { + case WAIT_OBJECT_0: + case WAIT_OBJECT_0 + 1: + break; + default: + GTEST_DEATH_TEST_CHECK_(false); // Should not get here. + } + + // The child has acquired the write end of the pipe or exited. + // We release the handle on our side and continue. + write_handle_.Reset(); + event_handle_.Reset(); + + ReadAndInterpretStatusByte(); + + // Waits for the child process to exit if it haven't already. This + // returns immediately if the child has already exited, regardless of + // whether previous calls to WaitForMultipleObjects synchronized on this + // handle or not. + GTEST_DEATH_TEST_CHECK_( + WAIT_OBJECT_0 == ::WaitForSingleObject(child_handle_.Get(), + INFINITE)); + DWORD status_code; + GTEST_DEATH_TEST_CHECK_( + ::GetExitCodeProcess(child_handle_.Get(), &status_code) != FALSE); + child_handle_.Reset(); + set_status(static_cast(status_code)); + return status(); +} + +// The AssumeRole process for a Windows death test. It creates a child +// process with the same executable as the current process to run the +// death test. The child process is given the --gtest_filter and +// --gtest_internal_run_death_test flags such that it knows to run the +// current death test only. +DeathTest::TestRole WindowsDeathTest::AssumeRole() { + const UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const TestInfo* const info = impl->current_test_info(); + const int death_test_index = info->result()->death_test_count(); + + if (flag != NULL) { + // ParseInternalRunDeathTestFlag() has performed all the necessary + // processing. + set_write_fd(flag->write_fd()); + return EXECUTE_TEST; + } + + // WindowsDeathTest uses an anonymous pipe to communicate results of + // a death test. + SECURITY_ATTRIBUTES handles_are_inheritable = { + sizeof(SECURITY_ATTRIBUTES), NULL, TRUE }; + HANDLE read_handle, write_handle; + GTEST_DEATH_TEST_CHECK_( + ::CreatePipe(&read_handle, &write_handle, &handles_are_inheritable, + 0) // Default buffer size. + != FALSE); + set_read_fd(::_open_osfhandle(reinterpret_cast(read_handle), + O_RDONLY)); + write_handle_.Reset(write_handle); + event_handle_.Reset(::CreateEvent( + &handles_are_inheritable, + TRUE, // The event will automatically reset to non-signaled state. + FALSE, // The initial state is non-signalled. + NULL)); // The even is unnamed. + GTEST_DEATH_TEST_CHECK_(event_handle_.Get() != NULL); + const std::string filter_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" + + info->test_case_name() + "." + info->name(); + const std::string internal_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + + "=" + file_ + "|" + StreamableToString(line_) + "|" + + StreamableToString(death_test_index) + "|" + + StreamableToString(static_cast(::GetCurrentProcessId())) + + // size_t has the same width as pointers on both 32-bit and 64-bit + // Windows platforms. + // See http://msdn.microsoft.com/en-us/library/tcxf1dw6.aspx. + "|" + StreamableToString(reinterpret_cast(write_handle)) + + "|" + StreamableToString(reinterpret_cast(event_handle_.Get())); + + char executable_path[_MAX_PATH + 1]; // NOLINT + GTEST_DEATH_TEST_CHECK_( + _MAX_PATH + 1 != ::GetModuleFileNameA(NULL, + executable_path, + _MAX_PATH)); + + std::string command_line = + std::string(::GetCommandLineA()) + " " + filter_flag + " \"" + + internal_flag + "\""; + + DeathTest::set_last_death_test_message(""); + + CaptureStderr(); + // Flush the log buffers since the log streams are shared with the child. + FlushInfoLog(); + + // The child process will share the standard handles with the parent. + STARTUPINFOA startup_info; + memset(&startup_info, 0, sizeof(STARTUPINFO)); + startup_info.dwFlags = STARTF_USESTDHANDLES; + startup_info.hStdInput = ::GetStdHandle(STD_INPUT_HANDLE); + startup_info.hStdOutput = ::GetStdHandle(STD_OUTPUT_HANDLE); + startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE); + + PROCESS_INFORMATION process_info; + GTEST_DEATH_TEST_CHECK_(::CreateProcessA( + executable_path, + const_cast(command_line.c_str()), + NULL, // Retuned process handle is not inheritable. + NULL, // Retuned thread handle is not inheritable. + TRUE, // Child inherits all inheritable handles (for write_handle_). + 0x0, // Default creation flags. + NULL, // Inherit the parent's environment. + UnitTest::GetInstance()->original_working_dir(), + &startup_info, + &process_info) != FALSE); + child_handle_.Reset(process_info.hProcess); + ::CloseHandle(process_info.hThread); + set_spawned(true); + return OVERSEE_TEST; +} +# else // We are not on Windows. + +// ForkingDeathTest provides implementations for most of the abstract +// methods of the DeathTest interface. Only the AssumeRole method is +// left undefined. +class ForkingDeathTest : public DeathTestImpl { + public: + ForkingDeathTest(const char* statement, const RE* regex); + + // All of these virtual functions are inherited from DeathTest. + virtual int Wait(); + + protected: + void set_child_pid(pid_t child_pid) { child_pid_ = child_pid; } + + private: + // PID of child process during death test; 0 in the child process itself. + pid_t child_pid_; +}; + +// Constructs a ForkingDeathTest. +ForkingDeathTest::ForkingDeathTest(const char* a_statement, const RE* a_regex) + : DeathTestImpl(a_statement, a_regex), + child_pid_(-1) {} + +// Waits for the child in a death test to exit, returning its exit +// status, or 0 if no child process exists. As a side effect, sets the +// outcome data member. +int ForkingDeathTest::Wait() { + if (!spawned()) + return 0; + + ReadAndInterpretStatusByte(); + + int status_value; + GTEST_DEATH_TEST_CHECK_SYSCALL_(waitpid(child_pid_, &status_value, 0)); + set_status(status_value); + return status_value; +} + +// A concrete death test class that forks, then immediately runs the test +// in the child process. +class NoExecDeathTest : public ForkingDeathTest { + public: + NoExecDeathTest(const char* a_statement, const RE* a_regex) : + ForkingDeathTest(a_statement, a_regex) { } + virtual TestRole AssumeRole(); +}; + +// The AssumeRole process for a fork-and-run death test. It implements a +// straightforward fork, with a simple pipe to transmit the status byte. +DeathTest::TestRole NoExecDeathTest::AssumeRole() { + const size_t thread_count = GetThreadCount(); + if (thread_count != 1) { + GTEST_LOG_(WARNING) << DeathTestThreadWarning(thread_count); + } + + int pipe_fd[2]; + GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1); + + DeathTest::set_last_death_test_message(""); + CaptureStderr(); + // When we fork the process below, the log file buffers are copied, but the + // file descriptors are shared. We flush all log files here so that closing + // the file descriptors in the child process doesn't throw off the + // synchronization between descriptors and buffers in the parent process. + // This is as close to the fork as possible to avoid a race condition in case + // there are multiple threads running before the death test, and another + // thread writes to the log file. + FlushInfoLog(); + + const pid_t child_pid = fork(); + GTEST_DEATH_TEST_CHECK_(child_pid != -1); + set_child_pid(child_pid); + if (child_pid == 0) { + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[0])); + set_write_fd(pipe_fd[1]); + // Redirects all logging to stderr in the child process to prevent + // concurrent writes to the log files. We capture stderr in the parent + // process and append the child process' output to a log. + LogToStderr(); + // Event forwarding to the listeners of event listener API mush be shut + // down in death test subprocesses. + GetUnitTestImpl()->listeners()->SuppressEventForwarding(); + g_in_fast_death_test_child = true; + return EXECUTE_TEST; + } else { + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1])); + set_read_fd(pipe_fd[0]); + set_spawned(true); + return OVERSEE_TEST; + } +} + +// A concrete death test class that forks and re-executes the main +// program from the beginning, with command-line flags set that cause +// only this specific death test to be run. +class ExecDeathTest : public ForkingDeathTest { + public: + ExecDeathTest(const char* a_statement, const RE* a_regex, + const char* file, int line) : + ForkingDeathTest(a_statement, a_regex), file_(file), line_(line) { } + virtual TestRole AssumeRole(); + private: + static ::std::vector + GetArgvsForDeathTestChildProcess() { + ::std::vector args = GetInjectableArgvs(); +# if defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_) + ::std::vector extra_args = + GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_(); + args.insert(args.end(), extra_args.begin(), extra_args.end()); +# endif // defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_) + return args; + } + // The name of the file in which the death test is located. + const char* const file_; + // The line number on which the death test is located. + const int line_; +}; + +// Utility class for accumulating command-line arguments. +class Arguments { + public: + Arguments() { + args_.push_back(NULL); + } + + ~Arguments() { + for (std::vector::iterator i = args_.begin(); i != args_.end(); + ++i) { + free(*i); + } + } + void AddArgument(const char* argument) { + args_.insert(args_.end() - 1, posix::StrDup(argument)); + } + + template + void AddArguments(const ::std::vector& arguments) { + for (typename ::std::vector::const_iterator i = arguments.begin(); + i != arguments.end(); + ++i) { + args_.insert(args_.end() - 1, posix::StrDup(i->c_str())); + } + } + char* const* Argv() { + return &args_[0]; + } + + private: + std::vector args_; +}; + +// A struct that encompasses the arguments to the child process of a +// threadsafe-style death test process. +struct ExecDeathTestArgs { + char* const* argv; // Command-line arguments for the child's call to exec + int close_fd; // File descriptor to close; the read end of a pipe +}; + +# if GTEST_OS_MAC +inline char** GetEnviron() { + // When Google Test is built as a framework on MacOS X, the environ variable + // is unavailable. Apple's documentation (man environ) recommends using + // _NSGetEnviron() instead. + return *_NSGetEnviron(); +} +# else +// Some POSIX platforms expect you to declare environ. extern "C" makes +// it reside in the global namespace. +extern "C" char** environ; +inline char** GetEnviron() { return environ; } +# endif // GTEST_OS_MAC + +# if !GTEST_OS_QNX +// The main function for a threadsafe-style death test child process. +// This function is called in a clone()-ed process and thus must avoid +// any potentially unsafe operations like malloc or libc functions. +static int ExecDeathTestChildMain(void* child_arg) { + ExecDeathTestArgs* const args = static_cast(child_arg); + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(args->close_fd)); + + // We need to execute the test program in the same environment where + // it was originally invoked. Therefore we change to the original + // working directory first. + const char* const original_dir = + UnitTest::GetInstance()->original_working_dir(); + // We can safely call chdir() as it's a direct system call. + if (chdir(original_dir) != 0) { + DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " + + GetLastErrnoDescription()); + return EXIT_FAILURE; + } + + // We can safely call execve() as it's a direct system call. We + // cannot use execvp() as it's a libc function and thus potentially + // unsafe. Since execve() doesn't search the PATH, the user must + // invoke the test program via a valid path that contains at least + // one path separator. + execve(args->argv[0], args->argv, GetEnviron()); + DeathTestAbort(std::string("execve(") + args->argv[0] + ", ...) in " + + original_dir + " failed: " + + GetLastErrnoDescription()); + return EXIT_FAILURE; +} +# endif // !GTEST_OS_QNX + +// Two utility routines that together determine the direction the stack +// grows. +// This could be accomplished more elegantly by a single recursive +// function, but we want to guard against the unlikely possibility of +// a smart compiler optimizing the recursion away. +// +// GTEST_NO_INLINE_ is required to prevent GCC 4.6 from inlining +// StackLowerThanAddress into StackGrowsDown, which then doesn't give +// correct answer. +void StackLowerThanAddress(const void* ptr, bool* result) GTEST_NO_INLINE_; +void StackLowerThanAddress(const void* ptr, bool* result) { + int dummy; + *result = (&dummy < ptr); +} + +// Make sure AddressSanitizer does not tamper with the stack here. +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +bool StackGrowsDown() { + int dummy; + bool result; + StackLowerThanAddress(&dummy, &result); + return result; +} + +// Spawns a child process with the same executable as the current process in +// a thread-safe manner and instructs it to run the death test. The +// implementation uses fork(2) + exec. On systems where clone(2) is +// available, it is used instead, being slightly more thread-safe. On QNX, +// fork supports only single-threaded environments, so this function uses +// spawn(2) there instead. The function dies with an error message if +// anything goes wrong. +static pid_t ExecDeathTestSpawnChild(char* const* argv, int close_fd) { + ExecDeathTestArgs args = { argv, close_fd }; + pid_t child_pid = -1; + +# if GTEST_OS_QNX + // Obtains the current directory and sets it to be closed in the child + // process. + const int cwd_fd = open(".", O_RDONLY); + GTEST_DEATH_TEST_CHECK_(cwd_fd != -1); + GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(cwd_fd, F_SETFD, FD_CLOEXEC)); + // We need to execute the test program in the same environment where + // it was originally invoked. Therefore we change to the original + // working directory first. + const char* const original_dir = + UnitTest::GetInstance()->original_working_dir(); + // We can safely call chdir() as it's a direct system call. + if (chdir(original_dir) != 0) { + DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " + + GetLastErrnoDescription()); + return EXIT_FAILURE; + } + + int fd_flags; + // Set close_fd to be closed after spawn. + GTEST_DEATH_TEST_CHECK_SYSCALL_(fd_flags = fcntl(close_fd, F_GETFD)); + GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(close_fd, F_SETFD, + fd_flags | FD_CLOEXEC)); + struct inheritance inherit = {0}; + // spawn is a system call. + child_pid = spawn(args.argv[0], 0, NULL, &inherit, args.argv, GetEnviron()); + // Restores the current working directory. + GTEST_DEATH_TEST_CHECK_(fchdir(cwd_fd) != -1); + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(cwd_fd)); + +# else // GTEST_OS_QNX +# if GTEST_OS_LINUX + // When a SIGPROF signal is received while fork() or clone() are executing, + // the process may hang. To avoid this, we ignore SIGPROF here and re-enable + // it after the call to fork()/clone() is complete. + struct sigaction saved_sigprof_action; + struct sigaction ignore_sigprof_action; + memset(&ignore_sigprof_action, 0, sizeof(ignore_sigprof_action)); + sigemptyset(&ignore_sigprof_action.sa_mask); + ignore_sigprof_action.sa_handler = SIG_IGN; + GTEST_DEATH_TEST_CHECK_SYSCALL_(sigaction( + SIGPROF, &ignore_sigprof_action, &saved_sigprof_action)); +# endif // GTEST_OS_LINUX + +# if GTEST_HAS_CLONE + const bool use_fork = GTEST_FLAG(death_test_use_fork); + + if (!use_fork) { + static const bool stack_grows_down = StackGrowsDown(); + const size_t stack_size = getpagesize(); + // MMAP_ANONYMOUS is not defined on Mac, so we use MAP_ANON instead. + void* const stack = mmap(NULL, stack_size, PROT_READ | PROT_WRITE, + MAP_ANON | MAP_PRIVATE, -1, 0); + GTEST_DEATH_TEST_CHECK_(stack != MAP_FAILED); + + // Maximum stack alignment in bytes: For a downward-growing stack, this + // amount is subtracted from size of the stack space to get an address + // that is within the stack space and is aligned on all systems we care + // about. As far as I know there is no ABI with stack alignment greater + // than 64. We assume stack and stack_size already have alignment of + // kMaxStackAlignment. + const size_t kMaxStackAlignment = 64; + void* const stack_top = + static_cast(stack) + + (stack_grows_down ? stack_size - kMaxStackAlignment : 0); + GTEST_DEATH_TEST_CHECK_(stack_size > kMaxStackAlignment && + reinterpret_cast(stack_top) % kMaxStackAlignment == 0); + + child_pid = clone(&ExecDeathTestChildMain, stack_top, SIGCHLD, &args); + + GTEST_DEATH_TEST_CHECK_(munmap(stack, stack_size) != -1); + } +# else + const bool use_fork = true; +# endif // GTEST_HAS_CLONE + + if (use_fork && (child_pid = fork()) == 0) { + ExecDeathTestChildMain(&args); + _exit(0); + } +# endif // GTEST_OS_QNX +# if GTEST_OS_LINUX + GTEST_DEATH_TEST_CHECK_SYSCALL_( + sigaction(SIGPROF, &saved_sigprof_action, NULL)); +# endif // GTEST_OS_LINUX + + GTEST_DEATH_TEST_CHECK_(child_pid != -1); + return child_pid; +} + +// The AssumeRole process for a fork-and-exec death test. It re-executes the +// main program from the beginning, setting the --gtest_filter +// and --gtest_internal_run_death_test flags to cause only the current +// death test to be re-run. +DeathTest::TestRole ExecDeathTest::AssumeRole() { + const UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const TestInfo* const info = impl->current_test_info(); + const int death_test_index = info->result()->death_test_count(); + + if (flag != NULL) { + set_write_fd(flag->write_fd()); + return EXECUTE_TEST; + } + + int pipe_fd[2]; + GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1); + // Clear the close-on-exec flag on the write end of the pipe, lest + // it be closed when the child process does an exec: + GTEST_DEATH_TEST_CHECK_(fcntl(pipe_fd[1], F_SETFD, 0) != -1); + + const std::string filter_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" + + info->test_case_name() + "." + info->name(); + const std::string internal_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + "=" + + file_ + "|" + StreamableToString(line_) + "|" + + StreamableToString(death_test_index) + "|" + + StreamableToString(pipe_fd[1]); + Arguments args; + args.AddArguments(GetArgvsForDeathTestChildProcess()); + args.AddArgument(filter_flag.c_str()); + args.AddArgument(internal_flag.c_str()); + + DeathTest::set_last_death_test_message(""); + + CaptureStderr(); + // See the comment in NoExecDeathTest::AssumeRole for why the next line + // is necessary. + FlushInfoLog(); + + const pid_t child_pid = ExecDeathTestSpawnChild(args.Argv(), pipe_fd[0]); + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1])); + set_child_pid(child_pid); + set_read_fd(pipe_fd[0]); + set_spawned(true); + return OVERSEE_TEST; +} + +# endif // !GTEST_OS_WINDOWS + +// Creates a concrete DeathTest-derived class that depends on the +// --gtest_death_test_style flag, and sets the pointer pointed to +// by the "test" argument to its address. If the test should be +// skipped, sets that pointer to NULL. Returns true, unless the +// flag is set to an invalid value. +bool DefaultDeathTestFactory::Create(const char* statement, const RE* regex, + const char* file, int line, + DeathTest** test) { + UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const int death_test_index = impl->current_test_info() + ->increment_death_test_count(); + + if (flag != NULL) { + if (death_test_index > flag->index()) { + DeathTest::set_last_death_test_message( + "Death test count (" + StreamableToString(death_test_index) + + ") somehow exceeded expected maximum (" + + StreamableToString(flag->index()) + ")"); + return false; + } + + if (!(flag->file() == file && flag->line() == line && + flag->index() == death_test_index)) { + *test = NULL; + return true; + } + } + +# if GTEST_OS_WINDOWS + + if (GTEST_FLAG(death_test_style) == "threadsafe" || + GTEST_FLAG(death_test_style) == "fast") { + *test = new WindowsDeathTest(statement, regex, file, line); + } + +# else + + if (GTEST_FLAG(death_test_style) == "threadsafe") { + *test = new ExecDeathTest(statement, regex, file, line); + } else if (GTEST_FLAG(death_test_style) == "fast") { + *test = new NoExecDeathTest(statement, regex); + } + +# endif // GTEST_OS_WINDOWS + + else { // NOLINT - this is more readable than unbalanced brackets inside #if. + DeathTest::set_last_death_test_message( + "Unknown death test style \"" + GTEST_FLAG(death_test_style) + + "\" encountered"); + return false; + } + + return true; +} + +# if GTEST_OS_WINDOWS +// Recreates the pipe and event handles from the provided parameters, +// signals the event, and returns a file descriptor wrapped around the pipe +// handle. This function is called in the child process only. +int GetStatusFileDescriptor(unsigned int parent_process_id, + size_t write_handle_as_size_t, + size_t event_handle_as_size_t) { + AutoHandle parent_process_handle(::OpenProcess(PROCESS_DUP_HANDLE, + FALSE, // Non-inheritable. + parent_process_id)); + if (parent_process_handle.Get() == INVALID_HANDLE_VALUE) { + DeathTestAbort("Unable to open parent process " + + StreamableToString(parent_process_id)); + } + + // TODO(vladl@google.com): Replace the following check with a + // compile-time assertion when available. + GTEST_CHECK_(sizeof(HANDLE) <= sizeof(size_t)); + + const HANDLE write_handle = + reinterpret_cast(write_handle_as_size_t); + HANDLE dup_write_handle; + + // The newly initialized handle is accessible only in in the parent + // process. To obtain one accessible within the child, we need to use + // DuplicateHandle. + if (!::DuplicateHandle(parent_process_handle.Get(), write_handle, + ::GetCurrentProcess(), &dup_write_handle, + 0x0, // Requested privileges ignored since + // DUPLICATE_SAME_ACCESS is used. + FALSE, // Request non-inheritable handler. + DUPLICATE_SAME_ACCESS)) { + DeathTestAbort("Unable to duplicate the pipe handle " + + StreamableToString(write_handle_as_size_t) + + " from the parent process " + + StreamableToString(parent_process_id)); + } + + const HANDLE event_handle = reinterpret_cast(event_handle_as_size_t); + HANDLE dup_event_handle; + + if (!::DuplicateHandle(parent_process_handle.Get(), event_handle, + ::GetCurrentProcess(), &dup_event_handle, + 0x0, + FALSE, + DUPLICATE_SAME_ACCESS)) { + DeathTestAbort("Unable to duplicate the event handle " + + StreamableToString(event_handle_as_size_t) + + " from the parent process " + + StreamableToString(parent_process_id)); + } + + const int write_fd = + ::_open_osfhandle(reinterpret_cast(dup_write_handle), O_APPEND); + if (write_fd == -1) { + DeathTestAbort("Unable to convert pipe handle " + + StreamableToString(write_handle_as_size_t) + + " to a file descriptor"); + } + + // Signals the parent that the write end of the pipe has been acquired + // so the parent can release its own write end. + ::SetEvent(dup_event_handle); + + return write_fd; +} +# endif // GTEST_OS_WINDOWS + +// Returns a newly created InternalRunDeathTestFlag object with fields +// initialized from the GTEST_FLAG(internal_run_death_test) flag if +// the flag is specified; otherwise returns NULL. +InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag() { + if (GTEST_FLAG(internal_run_death_test) == "") return NULL; + + // GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we + // can use it here. + int line = -1; + int index = -1; + ::std::vector< ::std::string> fields; + SplitString(GTEST_FLAG(internal_run_death_test).c_str(), '|', &fields); + int write_fd = -1; + +# if GTEST_OS_WINDOWS + + unsigned int parent_process_id = 0; + size_t write_handle_as_size_t = 0; + size_t event_handle_as_size_t = 0; + + if (fields.size() != 6 + || !ParseNaturalNumber(fields[1], &line) + || !ParseNaturalNumber(fields[2], &index) + || !ParseNaturalNumber(fields[3], &parent_process_id) + || !ParseNaturalNumber(fields[4], &write_handle_as_size_t) + || !ParseNaturalNumber(fields[5], &event_handle_as_size_t)) { + DeathTestAbort("Bad --gtest_internal_run_death_test flag: " + + GTEST_FLAG(internal_run_death_test)); + } + write_fd = GetStatusFileDescriptor(parent_process_id, + write_handle_as_size_t, + event_handle_as_size_t); +# else + + if (fields.size() != 4 + || !ParseNaturalNumber(fields[1], &line) + || !ParseNaturalNumber(fields[2], &index) + || !ParseNaturalNumber(fields[3], &write_fd)) { + DeathTestAbort("Bad --gtest_internal_run_death_test flag: " + + GTEST_FLAG(internal_run_death_test)); + } + +# endif // GTEST_OS_WINDOWS + + return new InternalRunDeathTestFlag(fields[0], line, index, write_fd); +} + +} // namespace internal + +#endif // GTEST_HAS_DEATH_TEST + +} // namespace testing diff --git a/third_party/aom/third_party/googletest/src/googletest/src/gtest-filepath.cc b/third_party/aom/third_party/googletest/src/googletest/src/gtest-filepath.cc new file mode 100644 index 0000000000..0292dc1195 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/src/gtest-filepath.cc @@ -0,0 +1,387 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: keith.ray@gmail.com (Keith Ray) + +#include "gtest/gtest-message.h" +#include "gtest/internal/gtest-filepath.h" +#include "gtest/internal/gtest-port.h" + +#include + +#if GTEST_OS_WINDOWS_MOBILE +# include +#elif GTEST_OS_WINDOWS +# include +# include +#elif GTEST_OS_SYMBIAN +// Symbian OpenC has PATH_MAX in sys/syslimits.h +# include +#else +# include +# include // Some Linux distributions define PATH_MAX here. +#endif // GTEST_OS_WINDOWS_MOBILE + +#if GTEST_OS_WINDOWS +# define GTEST_PATH_MAX_ _MAX_PATH +#elif defined(PATH_MAX) +# define GTEST_PATH_MAX_ PATH_MAX +#elif defined(_XOPEN_PATH_MAX) +# define GTEST_PATH_MAX_ _XOPEN_PATH_MAX +#else +# define GTEST_PATH_MAX_ _POSIX_PATH_MAX +#endif // GTEST_OS_WINDOWS + +#include "gtest/internal/gtest-string.h" + +namespace testing { +namespace internal { + +#if GTEST_OS_WINDOWS +// On Windows, '\\' is the standard path separator, but many tools and the +// Windows API also accept '/' as an alternate path separator. Unless otherwise +// noted, a file path can contain either kind of path separators, or a mixture +// of them. +const char kPathSeparator = '\\'; +const char kAlternatePathSeparator = '/'; +const char kAlternatePathSeparatorString[] = "/"; +# if GTEST_OS_WINDOWS_MOBILE +// Windows CE doesn't have a current directory. You should not use +// the current directory in tests on Windows CE, but this at least +// provides a reasonable fallback. +const char kCurrentDirectoryString[] = "\\"; +// Windows CE doesn't define INVALID_FILE_ATTRIBUTES +const DWORD kInvalidFileAttributes = 0xffffffff; +# else +const char kCurrentDirectoryString[] = ".\\"; +# endif // GTEST_OS_WINDOWS_MOBILE +#else +const char kPathSeparator = '/'; +const char kCurrentDirectoryString[] = "./"; +#endif // GTEST_OS_WINDOWS + +// Returns whether the given character is a valid path separator. +static bool IsPathSeparator(char c) { +#if GTEST_HAS_ALT_PATH_SEP_ + return (c == kPathSeparator) || (c == kAlternatePathSeparator); +#else + return c == kPathSeparator; +#endif +} + +// Returns the current working directory, or "" if unsuccessful. +FilePath FilePath::GetCurrentDir() { +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT + // Windows CE doesn't have a current directory, so we just return + // something reasonable. + return FilePath(kCurrentDirectoryString); +#elif GTEST_OS_WINDOWS + char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; + return FilePath(_getcwd(cwd, sizeof(cwd)) == NULL ? "" : cwd); +#else + char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; + char* result = getcwd(cwd, sizeof(cwd)); +# if GTEST_OS_NACL + // getcwd will likely fail in NaCl due to the sandbox, so return something + // reasonable. The user may have provided a shim implementation for getcwd, + // however, so fallback only when failure is detected. + return FilePath(result == NULL ? kCurrentDirectoryString : cwd); +# endif // GTEST_OS_NACL + return FilePath(result == NULL ? "" : cwd); +#endif // GTEST_OS_WINDOWS_MOBILE +} + +// Returns a copy of the FilePath with the case-insensitive extension removed. +// Example: FilePath("dir/file.exe").RemoveExtension("EXE") returns +// FilePath("dir/file"). If a case-insensitive extension is not +// found, returns a copy of the original FilePath. +FilePath FilePath::RemoveExtension(const char* extension) const { + const std::string dot_extension = std::string(".") + extension; + if (String::EndsWithCaseInsensitive(pathname_, dot_extension)) { + return FilePath(pathname_.substr( + 0, pathname_.length() - dot_extension.length())); + } + return *this; +} + +// Returns a pointer to the last occurence of a valid path separator in +// the FilePath. On Windows, for example, both '/' and '\' are valid path +// separators. Returns NULL if no path separator was found. +const char* FilePath::FindLastPathSeparator() const { + const char* const last_sep = strrchr(c_str(), kPathSeparator); +#if GTEST_HAS_ALT_PATH_SEP_ + const char* const last_alt_sep = strrchr(c_str(), kAlternatePathSeparator); + // Comparing two pointers of which only one is NULL is undefined. + if (last_alt_sep != NULL && + (last_sep == NULL || last_alt_sep > last_sep)) { + return last_alt_sep; + } +#endif + return last_sep; +} + +// Returns a copy of the FilePath with the directory part removed. +// Example: FilePath("path/to/file").RemoveDirectoryName() returns +// FilePath("file"). If there is no directory part ("just_a_file"), it returns +// the FilePath unmodified. If there is no file part ("just_a_dir/") it +// returns an empty FilePath (""). +// On Windows platform, '\' is the path separator, otherwise it is '/'. +FilePath FilePath::RemoveDirectoryName() const { + const char* const last_sep = FindLastPathSeparator(); + return last_sep ? FilePath(last_sep + 1) : *this; +} + +// RemoveFileName returns the directory path with the filename removed. +// Example: FilePath("path/to/file").RemoveFileName() returns "path/to/". +// If the FilePath is "a_file" or "/a_file", RemoveFileName returns +// FilePath("./") or, on Windows, FilePath(".\\"). If the filepath does +// not have a file, like "just/a/dir/", it returns the FilePath unmodified. +// On Windows platform, '\' is the path separator, otherwise it is '/'. +FilePath FilePath::RemoveFileName() const { + const char* const last_sep = FindLastPathSeparator(); + std::string dir; + if (last_sep) { + dir = std::string(c_str(), last_sep + 1 - c_str()); + } else { + dir = kCurrentDirectoryString; + } + return FilePath(dir); +} + +// Helper functions for naming files in a directory for xml output. + +// Given directory = "dir", base_name = "test", number = 0, +// extension = "xml", returns "dir/test.xml". If number is greater +// than zero (e.g., 12), returns "dir/test_12.xml". +// On Windows platform, uses \ as the separator rather than /. +FilePath FilePath::MakeFileName(const FilePath& directory, + const FilePath& base_name, + int number, + const char* extension) { + std::string file; + if (number == 0) { + file = base_name.string() + "." + extension; + } else { + file = base_name.string() + "_" + StreamableToString(number) + + "." + extension; + } + return ConcatPaths(directory, FilePath(file)); +} + +// Given directory = "dir", relative_path = "test.xml", returns "dir/test.xml". +// On Windows, uses \ as the separator rather than /. +FilePath FilePath::ConcatPaths(const FilePath& directory, + const FilePath& relative_path) { + if (directory.IsEmpty()) + return relative_path; + const FilePath dir(directory.RemoveTrailingPathSeparator()); + return FilePath(dir.string() + kPathSeparator + relative_path.string()); +} + +// Returns true if pathname describes something findable in the file-system, +// either a file, directory, or whatever. +bool FilePath::FileOrDirectoryExists() const { +#if GTEST_OS_WINDOWS_MOBILE + LPCWSTR unicode = String::AnsiToUtf16(pathname_.c_str()); + const DWORD attributes = GetFileAttributes(unicode); + delete [] unicode; + return attributes != kInvalidFileAttributes; +#else + posix::StatStruct file_stat; + return posix::Stat(pathname_.c_str(), &file_stat) == 0; +#endif // GTEST_OS_WINDOWS_MOBILE +} + +// Returns true if pathname describes a directory in the file-system +// that exists. +bool FilePath::DirectoryExists() const { + bool result = false; +#if GTEST_OS_WINDOWS + // Don't strip off trailing separator if path is a root directory on + // Windows (like "C:\\"). + const FilePath& path(IsRootDirectory() ? *this : + RemoveTrailingPathSeparator()); +#else + const FilePath& path(*this); +#endif + +#if GTEST_OS_WINDOWS_MOBILE + LPCWSTR unicode = String::AnsiToUtf16(path.c_str()); + const DWORD attributes = GetFileAttributes(unicode); + delete [] unicode; + if ((attributes != kInvalidFileAttributes) && + (attributes & FILE_ATTRIBUTE_DIRECTORY)) { + result = true; + } +#else + posix::StatStruct file_stat; + result = posix::Stat(path.c_str(), &file_stat) == 0 && + posix::IsDir(file_stat); +#endif // GTEST_OS_WINDOWS_MOBILE + + return result; +} + +// Returns true if pathname describes a root directory. (Windows has one +// root directory per disk drive.) +bool FilePath::IsRootDirectory() const { +#if GTEST_OS_WINDOWS + // TODO(wan@google.com): on Windows a network share like + // \\server\share can be a root directory, although it cannot be the + // current directory. Handle this properly. + return pathname_.length() == 3 && IsAbsolutePath(); +#else + return pathname_.length() == 1 && IsPathSeparator(pathname_.c_str()[0]); +#endif +} + +// Returns true if pathname describes an absolute path. +bool FilePath::IsAbsolutePath() const { + const char* const name = pathname_.c_str(); +#if GTEST_OS_WINDOWS + return pathname_.length() >= 3 && + ((name[0] >= 'a' && name[0] <= 'z') || + (name[0] >= 'A' && name[0] <= 'Z')) && + name[1] == ':' && + IsPathSeparator(name[2]); +#else + return IsPathSeparator(name[0]); +#endif +} + +// Returns a pathname for a file that does not currently exist. The pathname +// will be directory/base_name.extension or +// directory/base_name_.extension if directory/base_name.extension +// already exists. The number will be incremented until a pathname is found +// that does not already exist. +// Examples: 'dir/foo_test.xml' or 'dir/foo_test_1.xml'. +// There could be a race condition if two or more processes are calling this +// function at the same time -- they could both pick the same filename. +FilePath FilePath::GenerateUniqueFileName(const FilePath& directory, + const FilePath& base_name, + const char* extension) { + FilePath full_pathname; + int number = 0; + do { + full_pathname.Set(MakeFileName(directory, base_name, number++, extension)); + } while (full_pathname.FileOrDirectoryExists()); + return full_pathname; +} + +// Returns true if FilePath ends with a path separator, which indicates that +// it is intended to represent a directory. Returns false otherwise. +// This does NOT check that a directory (or file) actually exists. +bool FilePath::IsDirectory() const { + return !pathname_.empty() && + IsPathSeparator(pathname_.c_str()[pathname_.length() - 1]); +} + +// Create directories so that path exists. Returns true if successful or if +// the directories already exist; returns false if unable to create directories +// for any reason. +bool FilePath::CreateDirectoriesRecursively() const { + if (!this->IsDirectory()) { + return false; + } + + if (pathname_.length() == 0 || this->DirectoryExists()) { + return true; + } + + const FilePath parent(this->RemoveTrailingPathSeparator().RemoveFileName()); + return parent.CreateDirectoriesRecursively() && this->CreateFolder(); +} + +// Create the directory so that path exists. Returns true if successful or +// if the directory already exists; returns false if unable to create the +// directory for any reason, including if the parent directory does not +// exist. Not named "CreateDirectory" because that's a macro on Windows. +bool FilePath::CreateFolder() const { +#if GTEST_OS_WINDOWS_MOBILE + FilePath removed_sep(this->RemoveTrailingPathSeparator()); + LPCWSTR unicode = String::AnsiToUtf16(removed_sep.c_str()); + int result = CreateDirectory(unicode, NULL) ? 0 : -1; + delete [] unicode; +#elif GTEST_OS_WINDOWS + int result = _mkdir(pathname_.c_str()); +#else + int result = mkdir(pathname_.c_str(), 0777); +#endif // GTEST_OS_WINDOWS_MOBILE + + if (result == -1) { + return this->DirectoryExists(); // An error is OK if the directory exists. + } + return true; // No error. +} + +// If input name has a trailing separator character, remove it and return the +// name, otherwise return the name string unmodified. +// On Windows platform, uses \ as the separator, other platforms use /. +FilePath FilePath::RemoveTrailingPathSeparator() const { + return IsDirectory() + ? FilePath(pathname_.substr(0, pathname_.length() - 1)) + : *this; +} + +// Removes any redundant separators that might be in the pathname. +// For example, "bar///foo" becomes "bar/foo". Does not eliminate other +// redundancies that might be in a pathname involving "." or "..". +// TODO(wan@google.com): handle Windows network shares (e.g. \\server\share). +void FilePath::Normalize() { + if (pathname_.c_str() == NULL) { + pathname_ = ""; + return; + } + const char* src = pathname_.c_str(); + char* const dest = new char[pathname_.length() + 1]; + char* dest_ptr = dest; + memset(dest_ptr, 0, pathname_.length() + 1); + + while (*src != '\0') { + *dest_ptr = *src; + if (!IsPathSeparator(*src)) { + src++; + } else { +#if GTEST_HAS_ALT_PATH_SEP_ + if (*dest_ptr == kAlternatePathSeparator) { + *dest_ptr = kPathSeparator; + } +#endif + while (IsPathSeparator(*src)) + src++; + } + dest_ptr++; + } + *dest_ptr = '\0'; + pathname_ = dest; + delete[] dest; +} + +} // namespace internal +} // namespace testing diff --git a/third_party/aom/third_party/googletest/src/googletest/src/gtest-internal-inl.h b/third_party/aom/third_party/googletest/src/googletest/src/gtest-internal-inl.h new file mode 100644 index 0000000000..ed8a682a96 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/src/gtest-internal-inl.h @@ -0,0 +1,1183 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Utility functions and classes used by the Google C++ testing framework. +// +// Author: wan@google.com (Zhanyong Wan) +// +// This file contains purely Google Test's internal implementation. Please +// DO NOT #INCLUDE IT IN A USER PROGRAM. + +#ifndef GTEST_SRC_GTEST_INTERNAL_INL_H_ +#define GTEST_SRC_GTEST_INTERNAL_INL_H_ + +// GTEST_IMPLEMENTATION_ is defined to 1 iff the current translation unit is +// part of Google Test's implementation; otherwise it's undefined. +#if !GTEST_IMPLEMENTATION_ +// If this file is included from the user's code, just say no. +# error "gtest-internal-inl.h is part of Google Test's internal implementation." +# error "It must not be included except by Google Test itself." +#endif // GTEST_IMPLEMENTATION_ + +#ifndef _WIN32_WCE +# include +#endif // !_WIN32_WCE +#include +#include // For strtoll/_strtoul64/malloc/free. +#include // For memmove. + +#include +#include +#include + +#include "gtest/internal/gtest-port.h" + +#if GTEST_CAN_STREAM_RESULTS_ +# include // NOLINT +# include // NOLINT +#endif + +#if GTEST_OS_WINDOWS +# include // NOLINT +#endif // GTEST_OS_WINDOWS + +#include "gtest/gtest.h" // NOLINT +#include "gtest/gtest-spi.h" + +namespace testing { + +// Declares the flags. +// +// We don't want the users to modify this flag in the code, but want +// Google Test's own unit tests to be able to access it. Therefore we +// declare it here as opposed to in gtest.h. +GTEST_DECLARE_bool_(death_test_use_fork); + +namespace internal { + +// The value of GetTestTypeId() as seen from within the Google Test +// library. This is solely for testing GetTestTypeId(). +GTEST_API_ extern const TypeId kTestTypeIdInGoogleTest; + +// Names of the flags (needed for parsing Google Test flags). +const char kAlsoRunDisabledTestsFlag[] = "also_run_disabled_tests"; +const char kBreakOnFailureFlag[] = "break_on_failure"; +const char kCatchExceptionsFlag[] = "catch_exceptions"; +const char kColorFlag[] = "color"; +const char kFilterFlag[] = "filter"; +const char kListTestsFlag[] = "list_tests"; +const char kOutputFlag[] = "output"; +const char kPrintTimeFlag[] = "print_time"; +const char kRandomSeedFlag[] = "random_seed"; +const char kRepeatFlag[] = "repeat"; +const char kShuffleFlag[] = "shuffle"; +const char kStackTraceDepthFlag[] = "stack_trace_depth"; +const char kStreamResultToFlag[] = "stream_result_to"; +const char kThrowOnFailureFlag[] = "throw_on_failure"; +const char kFlagfileFlag[] = "flagfile"; + +// A valid random seed must be in [1, kMaxRandomSeed]. +const int kMaxRandomSeed = 99999; + +// g_help_flag is true iff the --help flag or an equivalent form is +// specified on the command line. +GTEST_API_ extern bool g_help_flag; + +// Returns the current time in milliseconds. +GTEST_API_ TimeInMillis GetTimeInMillis(); + +// Returns true iff Google Test should use colors in the output. +GTEST_API_ bool ShouldUseColor(bool stdout_is_tty); + +// Formats the given time in milliseconds as seconds. +GTEST_API_ std::string FormatTimeInMillisAsSeconds(TimeInMillis ms); + +// Converts the given time in milliseconds to a date string in the ISO 8601 +// format, without the timezone information. N.B.: due to the use the +// non-reentrant localtime() function, this function is not thread safe. Do +// not use it in any code that can be called from multiple threads. +GTEST_API_ std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms); + +// Parses a string for an Int32 flag, in the form of "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +GTEST_API_ bool ParseInt32Flag( + const char* str, const char* flag, Int32* value); + +// Returns a random seed in range [1, kMaxRandomSeed] based on the +// given --gtest_random_seed flag value. +inline int GetRandomSeedFromFlag(Int32 random_seed_flag) { + const unsigned int raw_seed = (random_seed_flag == 0) ? + static_cast(GetTimeInMillis()) : + static_cast(random_seed_flag); + + // Normalizes the actual seed to range [1, kMaxRandomSeed] such that + // it's easy to type. + const int normalized_seed = + static_cast((raw_seed - 1U) % + static_cast(kMaxRandomSeed)) + 1; + return normalized_seed; +} + +// Returns the first valid random seed after 'seed'. The behavior is +// undefined if 'seed' is invalid. The seed after kMaxRandomSeed is +// considered to be 1. +inline int GetNextRandomSeed(int seed) { + GTEST_CHECK_(1 <= seed && seed <= kMaxRandomSeed) + << "Invalid random seed " << seed << " - must be in [1, " + << kMaxRandomSeed << "]."; + const int next_seed = seed + 1; + return (next_seed > kMaxRandomSeed) ? 1 : next_seed; +} + +// This class saves the values of all Google Test flags in its c'tor, and +// restores them in its d'tor. +class GTestFlagSaver { + public: + // The c'tor. + GTestFlagSaver() { + also_run_disabled_tests_ = GTEST_FLAG(also_run_disabled_tests); + break_on_failure_ = GTEST_FLAG(break_on_failure); + catch_exceptions_ = GTEST_FLAG(catch_exceptions); + color_ = GTEST_FLAG(color); + death_test_style_ = GTEST_FLAG(death_test_style); + death_test_use_fork_ = GTEST_FLAG(death_test_use_fork); + filter_ = GTEST_FLAG(filter); + internal_run_death_test_ = GTEST_FLAG(internal_run_death_test); + list_tests_ = GTEST_FLAG(list_tests); + output_ = GTEST_FLAG(output); + print_time_ = GTEST_FLAG(print_time); + random_seed_ = GTEST_FLAG(random_seed); + repeat_ = GTEST_FLAG(repeat); + shuffle_ = GTEST_FLAG(shuffle); + stack_trace_depth_ = GTEST_FLAG(stack_trace_depth); + stream_result_to_ = GTEST_FLAG(stream_result_to); + throw_on_failure_ = GTEST_FLAG(throw_on_failure); + } + + // The d'tor is not virtual. DO NOT INHERIT FROM THIS CLASS. + ~GTestFlagSaver() { + GTEST_FLAG(also_run_disabled_tests) = also_run_disabled_tests_; + GTEST_FLAG(break_on_failure) = break_on_failure_; + GTEST_FLAG(catch_exceptions) = catch_exceptions_; + GTEST_FLAG(color) = color_; + GTEST_FLAG(death_test_style) = death_test_style_; + GTEST_FLAG(death_test_use_fork) = death_test_use_fork_; + GTEST_FLAG(filter) = filter_; + GTEST_FLAG(internal_run_death_test) = internal_run_death_test_; + GTEST_FLAG(list_tests) = list_tests_; + GTEST_FLAG(output) = output_; + GTEST_FLAG(print_time) = print_time_; + GTEST_FLAG(random_seed) = random_seed_; + GTEST_FLAG(repeat) = repeat_; + GTEST_FLAG(shuffle) = shuffle_; + GTEST_FLAG(stack_trace_depth) = stack_trace_depth_; + GTEST_FLAG(stream_result_to) = stream_result_to_; + GTEST_FLAG(throw_on_failure) = throw_on_failure_; + } + + private: + // Fields for saving the original values of flags. + bool also_run_disabled_tests_; + bool break_on_failure_; + bool catch_exceptions_; + std::string color_; + std::string death_test_style_; + bool death_test_use_fork_; + std::string filter_; + std::string internal_run_death_test_; + bool list_tests_; + std::string output_; + bool print_time_; + internal::Int32 random_seed_; + internal::Int32 repeat_; + bool shuffle_; + internal::Int32 stack_trace_depth_; + std::string stream_result_to_; + bool throw_on_failure_; +} GTEST_ATTRIBUTE_UNUSED_; + +// Converts a Unicode code point to a narrow string in UTF-8 encoding. +// code_point parameter is of type UInt32 because wchar_t may not be +// wide enough to contain a code point. +// If the code_point is not a valid Unicode code point +// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be converted +// to "(Invalid Unicode 0xXXXXXXXX)". +GTEST_API_ std::string CodePointToUtf8(UInt32 code_point); + +// Converts a wide string to a narrow string in UTF-8 encoding. +// The wide string is assumed to have the following encoding: +// UTF-16 if sizeof(wchar_t) == 2 (on Windows, Cygwin, Symbian OS) +// UTF-32 if sizeof(wchar_t) == 4 (on Linux) +// Parameter str points to a null-terminated wide string. +// Parameter num_chars may additionally limit the number +// of wchar_t characters processed. -1 is used when the entire string +// should be processed. +// If the string contains code points that are not valid Unicode code points +// (i.e. outside of Unicode range U+0 to U+10FFFF) they will be output +// as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding +// and contains invalid UTF-16 surrogate pairs, values in those pairs +// will be encoded as individual Unicode characters from Basic Normal Plane. +GTEST_API_ std::string WideStringToUtf8(const wchar_t* str, int num_chars); + +// Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file +// if the variable is present. If a file already exists at this location, this +// function will write over it. If the variable is present, but the file cannot +// be created, prints an error and exits. +void WriteToShardStatusFileIfNeeded(); + +// Checks whether sharding is enabled by examining the relevant +// environment variable values. If the variables are present, +// but inconsistent (e.g., shard_index >= total_shards), prints +// an error and exits. If in_subprocess_for_death_test, sharding is +// disabled because it must only be applied to the original test +// process. Otherwise, we could filter out death tests we intended to execute. +GTEST_API_ bool ShouldShard(const char* total_shards_str, + const char* shard_index_str, + bool in_subprocess_for_death_test); + +// Parses the environment variable var as an Int32. If it is unset, +// returns default_val. If it is not an Int32, prints an error and +// and aborts. +GTEST_API_ Int32 Int32FromEnvOrDie(const char* env_var, Int32 default_val); + +// Given the total number of shards, the shard index, and the test id, +// returns true iff the test should be run on this shard. The test id is +// some arbitrary but unique non-negative integer assigned to each test +// method. Assumes that 0 <= shard_index < total_shards. +GTEST_API_ bool ShouldRunTestOnShard( + int total_shards, int shard_index, int test_id); + +// STL container utilities. + +// Returns the number of elements in the given container that satisfy +// the given predicate. +template +inline int CountIf(const Container& c, Predicate predicate) { + // Implemented as an explicit loop since std::count_if() in libCstd on + // Solaris has a non-standard signature. + int count = 0; + for (typename Container::const_iterator it = c.begin(); it != c.end(); ++it) { + if (predicate(*it)) + ++count; + } + return count; +} + +// Applies a function/functor to each element in the container. +template +void ForEach(const Container& c, Functor functor) { + std::for_each(c.begin(), c.end(), functor); +} + +// Returns the i-th element of the vector, or default_value if i is not +// in range [0, v.size()). +template +inline E GetElementOr(const std::vector& v, int i, E default_value) { + return (i < 0 || i >= static_cast(v.size())) ? default_value : v[i]; +} + +// Performs an in-place shuffle of a range of the vector's elements. +// 'begin' and 'end' are element indices as an STL-style range; +// i.e. [begin, end) are shuffled, where 'end' == size() means to +// shuffle to the end of the vector. +template +void ShuffleRange(internal::Random* random, int begin, int end, + std::vector* v) { + const int size = static_cast(v->size()); + GTEST_CHECK_(0 <= begin && begin <= size) + << "Invalid shuffle range start " << begin << ": must be in range [0, " + << size << "]."; + GTEST_CHECK_(begin <= end && end <= size) + << "Invalid shuffle range finish " << end << ": must be in range [" + << begin << ", " << size << "]."; + + // Fisher-Yates shuffle, from + // http://en.wikipedia.org/wiki/Fisher-Yates_shuffle + for (int range_width = end - begin; range_width >= 2; range_width--) { + const int last_in_range = begin + range_width - 1; + const int selected = begin + random->Generate(range_width); + std::swap((*v)[selected], (*v)[last_in_range]); + } +} + +// Performs an in-place shuffle of the vector's elements. +template +inline void Shuffle(internal::Random* random, std::vector* v) { + ShuffleRange(random, 0, static_cast(v->size()), v); +} + +// A function for deleting an object. Handy for being used as a +// functor. +template +static void Delete(T* x) { + delete x; +} + +// A predicate that checks the key of a TestProperty against a known key. +// +// TestPropertyKeyIs is copyable. +class TestPropertyKeyIs { + public: + // Constructor. + // + // TestPropertyKeyIs has NO default constructor. + explicit TestPropertyKeyIs(const std::string& key) : key_(key) {} + + // Returns true iff the test name of test property matches on key_. + bool operator()(const TestProperty& test_property) const { + return test_property.key() == key_; + } + + private: + std::string key_; +}; + +// Class UnitTestOptions. +// +// This class contains functions for processing options the user +// specifies when running the tests. It has only static members. +// +// In most cases, the user can specify an option using either an +// environment variable or a command line flag. E.g. you can set the +// test filter using either GTEST_FILTER or --gtest_filter. If both +// the variable and the flag are present, the latter overrides the +// former. +class GTEST_API_ UnitTestOptions { + public: + // Functions for processing the gtest_output flag. + + // Returns the output format, or "" for normal printed output. + static std::string GetOutputFormat(); + + // Returns the absolute path of the requested output file, or the + // default (test_detail.xml in the original working directory) if + // none was explicitly specified. + static std::string GetAbsolutePathToOutputFile(); + + // Functions for processing the gtest_filter flag. + + // Returns true iff the wildcard pattern matches the string. The + // first ':' or '\0' character in pattern marks the end of it. + // + // This recursive algorithm isn't very efficient, but is clear and + // works well enough for matching test names, which are short. + static bool PatternMatchesString(const char *pattern, const char *str); + + // Returns true iff the user-specified filter matches the test case + // name and the test name. + static bool FilterMatchesTest(const std::string &test_case_name, + const std::string &test_name); + +#if GTEST_OS_WINDOWS + // Function for supporting the gtest_catch_exception flag. + + // Returns EXCEPTION_EXECUTE_HANDLER if Google Test should handle the + // given SEH exception, or EXCEPTION_CONTINUE_SEARCH otherwise. + // This function is useful as an __except condition. + static int GTestShouldProcessSEH(DWORD exception_code); +#endif // GTEST_OS_WINDOWS + + // Returns true if "name" matches the ':' separated list of glob-style + // filters in "filter". + static bool MatchesFilter(const std::string& name, const char* filter); +}; + +// Returns the current application's name, removing directory path if that +// is present. Used by UnitTestOptions::GetOutputFile. +GTEST_API_ FilePath GetCurrentExecutableName(); + +// The role interface for getting the OS stack trace as a string. +class OsStackTraceGetterInterface { + public: + OsStackTraceGetterInterface() {} + virtual ~OsStackTraceGetterInterface() {} + + // Returns the current OS stack trace as an std::string. Parameters: + // + // max_depth - the maximum number of stack frames to be included + // in the trace. + // skip_count - the number of top frames to be skipped; doesn't count + // against max_depth. + virtual string CurrentStackTrace(int max_depth, int skip_count) = 0; + + // UponLeavingGTest() should be called immediately before Google Test calls + // user code. It saves some information about the current stack that + // CurrentStackTrace() will use to find and hide Google Test stack frames. + virtual void UponLeavingGTest() = 0; + + // This string is inserted in place of stack frames that are part of + // Google Test's implementation. + static const char* const kElidedFramesMarker; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(OsStackTraceGetterInterface); +}; + +// A working implementation of the OsStackTraceGetterInterface interface. +class OsStackTraceGetter : public OsStackTraceGetterInterface { + public: + OsStackTraceGetter() {} + + virtual string CurrentStackTrace(int max_depth, int skip_count); + virtual void UponLeavingGTest(); + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(OsStackTraceGetter); +}; + +// Information about a Google Test trace point. +struct TraceInfo { + const char* file; + int line; + std::string message; +}; + +// This is the default global test part result reporter used in UnitTestImpl. +// This class should only be used by UnitTestImpl. +class DefaultGlobalTestPartResultReporter + : public TestPartResultReporterInterface { + public: + explicit DefaultGlobalTestPartResultReporter(UnitTestImpl* unit_test); + // Implements the TestPartResultReporterInterface. Reports the test part + // result in the current test. + virtual void ReportTestPartResult(const TestPartResult& result); + + private: + UnitTestImpl* const unit_test_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultGlobalTestPartResultReporter); +}; + +// This is the default per thread test part result reporter used in +// UnitTestImpl. This class should only be used by UnitTestImpl. +class DefaultPerThreadTestPartResultReporter + : public TestPartResultReporterInterface { + public: + explicit DefaultPerThreadTestPartResultReporter(UnitTestImpl* unit_test); + // Implements the TestPartResultReporterInterface. The implementation just + // delegates to the current global test part result reporter of *unit_test_. + virtual void ReportTestPartResult(const TestPartResult& result); + + private: + UnitTestImpl* const unit_test_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultPerThreadTestPartResultReporter); +}; + +// The private implementation of the UnitTest class. We don't protect +// the methods under a mutex, as this class is not accessible by a +// user and the UnitTest class that delegates work to this class does +// proper locking. +class GTEST_API_ UnitTestImpl { + public: + explicit UnitTestImpl(UnitTest* parent); + virtual ~UnitTestImpl(); + + // There are two different ways to register your own TestPartResultReporter. + // You can register your own repoter to listen either only for test results + // from the current thread or for results from all threads. + // By default, each per-thread test result repoter just passes a new + // TestPartResult to the global test result reporter, which registers the + // test part result for the currently running test. + + // Returns the global test part result reporter. + TestPartResultReporterInterface* GetGlobalTestPartResultReporter(); + + // Sets the global test part result reporter. + void SetGlobalTestPartResultReporter( + TestPartResultReporterInterface* reporter); + + // Returns the test part result reporter for the current thread. + TestPartResultReporterInterface* GetTestPartResultReporterForCurrentThread(); + + // Sets the test part result reporter for the current thread. + void SetTestPartResultReporterForCurrentThread( + TestPartResultReporterInterface* reporter); + + // Gets the number of successful test cases. + int successful_test_case_count() const; + + // Gets the number of failed test cases. + int failed_test_case_count() const; + + // Gets the number of all test cases. + int total_test_case_count() const; + + // Gets the number of all test cases that contain at least one test + // that should run. + int test_case_to_run_count() const; + + // Gets the number of successful tests. + int successful_test_count() const; + + // Gets the number of failed tests. + int failed_test_count() const; + + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + + // Gets the number of disabled tests. + int disabled_test_count() const; + + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + + // Gets the number of all tests. + int total_test_count() const; + + // Gets the number of tests that should run. + int test_to_run_count() const; + + // Gets the time of the test program start, in ms from the start of the + // UNIX epoch. + TimeInMillis start_timestamp() const { return start_timestamp_; } + + // Gets the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const { return elapsed_time_; } + + // Returns true iff the unit test passed (i.e. all test cases passed). + bool Passed() const { return !Failed(); } + + // Returns true iff the unit test failed (i.e. some test case failed + // or something outside of all tests failed). + bool Failed() const { + return failed_test_case_count() > 0 || ad_hoc_test_result()->Failed(); + } + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + const TestCase* GetTestCase(int i) const { + const int index = GetElementOr(test_case_indices_, i, -1); + return index < 0 ? NULL : test_cases_[i]; + } + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + TestCase* GetMutableTestCase(int i) { + const int index = GetElementOr(test_case_indices_, i, -1); + return index < 0 ? NULL : test_cases_[index]; + } + + // Provides access to the event listener list. + TestEventListeners* listeners() { return &listeners_; } + + // Returns the TestResult for the test that's currently running, or + // the TestResult for the ad hoc test if no test is running. + TestResult* current_test_result(); + + // Returns the TestResult for the ad hoc test. + const TestResult* ad_hoc_test_result() const { return &ad_hoc_test_result_; } + + // Sets the OS stack trace getter. + // + // Does nothing if the input and the current OS stack trace getter + // are the same; otherwise, deletes the old getter and makes the + // input the current getter. + void set_os_stack_trace_getter(OsStackTraceGetterInterface* getter); + + // Returns the current OS stack trace getter if it is not NULL; + // otherwise, creates an OsStackTraceGetter, makes it the current + // getter, and returns it. + OsStackTraceGetterInterface* os_stack_trace_getter(); + + // Returns the current OS stack trace as an std::string. + // + // The maximum number of stack frames to be included is specified by + // the gtest_stack_trace_depth flag. The skip_count parameter + // specifies the number of top frames to be skipped, which doesn't + // count against the number of frames to be included. + // + // For example, if Foo() calls Bar(), which in turn calls + // CurrentOsStackTraceExceptTop(1), Foo() will be included in the + // trace but Bar() and CurrentOsStackTraceExceptTop() won't. + std::string CurrentOsStackTraceExceptTop(int skip_count) GTEST_NO_INLINE_; + + // Finds and returns a TestCase with the given name. If one doesn't + // exist, creates one and returns it. + // + // Arguments: + // + // test_case_name: name of the test case + // type_param: the name of the test's type parameter, or NULL if + // this is not a typed or a type-parameterized test. + // set_up_tc: pointer to the function that sets up the test case + // tear_down_tc: pointer to the function that tears down the test case + TestCase* GetTestCase(const char* test_case_name, + const char* type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc); + + // Adds a TestInfo to the unit test. + // + // Arguments: + // + // set_up_tc: pointer to the function that sets up the test case + // tear_down_tc: pointer to the function that tears down the test case + // test_info: the TestInfo object + void AddTestInfo(Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc, + TestInfo* test_info) { + // In order to support thread-safe death tests, we need to + // remember the original working directory when the test program + // was first invoked. We cannot do this in RUN_ALL_TESTS(), as + // the user may have changed the current directory before calling + // RUN_ALL_TESTS(). Therefore we capture the current directory in + // AddTestInfo(), which is called to register a TEST or TEST_F + // before main() is reached. + if (original_working_dir_.IsEmpty()) { + original_working_dir_.Set(FilePath::GetCurrentDir()); + GTEST_CHECK_(!original_working_dir_.IsEmpty()) + << "Failed to get the current working directory."; + } + + GetTestCase(test_info->test_case_name(), + test_info->type_param(), + set_up_tc, + tear_down_tc)->AddTestInfo(test_info); + } + +#if GTEST_HAS_PARAM_TEST + // Returns ParameterizedTestCaseRegistry object used to keep track of + // value-parameterized tests and instantiate and register them. + internal::ParameterizedTestCaseRegistry& parameterized_test_registry() { + return parameterized_test_registry_; + } +#endif // GTEST_HAS_PARAM_TEST + + // Sets the TestCase object for the test that's currently running. + void set_current_test_case(TestCase* a_current_test_case) { + current_test_case_ = a_current_test_case; + } + + // Sets the TestInfo object for the test that's currently running. If + // current_test_info is NULL, the assertion results will be stored in + // ad_hoc_test_result_. + void set_current_test_info(TestInfo* a_current_test_info) { + current_test_info_ = a_current_test_info; + } + + // Registers all parameterized tests defined using TEST_P and + // INSTANTIATE_TEST_CASE_P, creating regular tests for each test/parameter + // combination. This method can be called more then once; it has guards + // protecting from registering the tests more then once. If + // value-parameterized tests are disabled, RegisterParameterizedTests is + // present but does nothing. + void RegisterParameterizedTests(); + + // Runs all tests in this UnitTest object, prints the result, and + // returns true if all tests are successful. If any exception is + // thrown during a test, this test is considered to be failed, but + // the rest of the tests will still be run. + bool RunAllTests(); + + // Clears the results of all tests, except the ad hoc tests. + void ClearNonAdHocTestResult() { + ForEach(test_cases_, TestCase::ClearTestCaseResult); + } + + // Clears the results of ad-hoc test assertions. + void ClearAdHocTestResult() { + ad_hoc_test_result_.Clear(); + } + + // Adds a TestProperty to the current TestResult object when invoked in a + // context of a test or a test case, or to the global property set. If the + // result already contains a property with the same key, the value will be + // updated. + void RecordProperty(const TestProperty& test_property); + + enum ReactionToSharding { + HONOR_SHARDING_PROTOCOL, + IGNORE_SHARDING_PROTOCOL + }; + + // Matches the full name of each test against the user-specified + // filter to decide whether the test should run, then records the + // result in each TestCase and TestInfo object. + // If shard_tests == HONOR_SHARDING_PROTOCOL, further filters tests + // based on sharding variables in the environment. + // Returns the number of tests that should run. + int FilterTests(ReactionToSharding shard_tests); + + // Prints the names of the tests matching the user-specified filter flag. + void ListTestsMatchingFilter(); + + const TestCase* current_test_case() const { return current_test_case_; } + TestInfo* current_test_info() { return current_test_info_; } + const TestInfo* current_test_info() const { return current_test_info_; } + + // Returns the vector of environments that need to be set-up/torn-down + // before/after the tests are run. + std::vector& environments() { return environments_; } + + // Getters for the per-thread Google Test trace stack. + std::vector& gtest_trace_stack() { + return *(gtest_trace_stack_.pointer()); + } + const std::vector& gtest_trace_stack() const { + return gtest_trace_stack_.get(); + } + +#if GTEST_HAS_DEATH_TEST + void InitDeathTestSubprocessControlInfo() { + internal_run_death_test_flag_.reset(ParseInternalRunDeathTestFlag()); + } + // Returns a pointer to the parsed --gtest_internal_run_death_test + // flag, or NULL if that flag was not specified. + // This information is useful only in a death test child process. + // Must not be called before a call to InitGoogleTest. + const InternalRunDeathTestFlag* internal_run_death_test_flag() const { + return internal_run_death_test_flag_.get(); + } + + // Returns a pointer to the current death test factory. + internal::DeathTestFactory* death_test_factory() { + return death_test_factory_.get(); + } + + void SuppressTestEventsIfInSubprocess(); + + friend class ReplaceDeathTestFactory; +#endif // GTEST_HAS_DEATH_TEST + + // Initializes the event listener performing XML output as specified by + // UnitTestOptions. Must not be called before InitGoogleTest. + void ConfigureXmlOutput(); + +#if GTEST_CAN_STREAM_RESULTS_ + // Initializes the event listener for streaming test results to a socket. + // Must not be called before InitGoogleTest. + void ConfigureStreamingOutput(); +#endif + + // Performs initialization dependent upon flag values obtained in + // ParseGoogleTestFlagsOnly. Is called from InitGoogleTest after the call to + // ParseGoogleTestFlagsOnly. In case a user neglects to call InitGoogleTest + // this function is also called from RunAllTests. Since this function can be + // called more than once, it has to be idempotent. + void PostFlagParsingInit(); + + // Gets the random seed used at the start of the current test iteration. + int random_seed() const { return random_seed_; } + + // Gets the random number generator. + internal::Random* random() { return &random_; } + + // Shuffles all test cases, and the tests within each test case, + // making sure that death tests are still run first. + void ShuffleTests(); + + // Restores the test cases and tests to their order before the first shuffle. + void UnshuffleTests(); + + // Returns the value of GTEST_FLAG(catch_exceptions) at the moment + // UnitTest::Run() starts. + bool catch_exceptions() const { return catch_exceptions_; } + + private: + friend class ::testing::UnitTest; + + // Used by UnitTest::Run() to capture the state of + // GTEST_FLAG(catch_exceptions) at the moment it starts. + void set_catch_exceptions(bool value) { catch_exceptions_ = value; } + + // The UnitTest object that owns this implementation object. + UnitTest* const parent_; + + // The working directory when the first TEST() or TEST_F() was + // executed. + internal::FilePath original_working_dir_; + + // The default test part result reporters. + DefaultGlobalTestPartResultReporter default_global_test_part_result_reporter_; + DefaultPerThreadTestPartResultReporter + default_per_thread_test_part_result_reporter_; + + // Points to (but doesn't own) the global test part result reporter. + TestPartResultReporterInterface* global_test_part_result_repoter_; + + // Protects read and write access to global_test_part_result_reporter_. + internal::Mutex global_test_part_result_reporter_mutex_; + + // Points to (but doesn't own) the per-thread test part result reporter. + internal::ThreadLocal + per_thread_test_part_result_reporter_; + + // The vector of environments that need to be set-up/torn-down + // before/after the tests are run. + std::vector environments_; + + // The vector of TestCases in their original order. It owns the + // elements in the vector. + std::vector test_cases_; + + // Provides a level of indirection for the test case list to allow + // easy shuffling and restoring the test case order. The i-th + // element of this vector is the index of the i-th test case in the + // shuffled order. + std::vector test_case_indices_; + +#if GTEST_HAS_PARAM_TEST + // ParameterizedTestRegistry object used to register value-parameterized + // tests. + internal::ParameterizedTestCaseRegistry parameterized_test_registry_; + + // Indicates whether RegisterParameterizedTests() has been called already. + bool parameterized_tests_registered_; +#endif // GTEST_HAS_PARAM_TEST + + // Index of the last death test case registered. Initially -1. + int last_death_test_case_; + + // This points to the TestCase for the currently running test. It + // changes as Google Test goes through one test case after another. + // When no test is running, this is set to NULL and Google Test + // stores assertion results in ad_hoc_test_result_. Initially NULL. + TestCase* current_test_case_; + + // This points to the TestInfo for the currently running test. It + // changes as Google Test goes through one test after another. When + // no test is running, this is set to NULL and Google Test stores + // assertion results in ad_hoc_test_result_. Initially NULL. + TestInfo* current_test_info_; + + // Normally, a user only writes assertions inside a TEST or TEST_F, + // or inside a function called by a TEST or TEST_F. Since Google + // Test keeps track of which test is current running, it can + // associate such an assertion with the test it belongs to. + // + // If an assertion is encountered when no TEST or TEST_F is running, + // Google Test attributes the assertion result to an imaginary "ad hoc" + // test, and records the result in ad_hoc_test_result_. + TestResult ad_hoc_test_result_; + + // The list of event listeners that can be used to track events inside + // Google Test. + TestEventListeners listeners_; + + // The OS stack trace getter. Will be deleted when the UnitTest + // object is destructed. By default, an OsStackTraceGetter is used, + // but the user can set this field to use a custom getter if that is + // desired. + OsStackTraceGetterInterface* os_stack_trace_getter_; + + // True iff PostFlagParsingInit() has been called. + bool post_flag_parse_init_performed_; + + // The random number seed used at the beginning of the test run. + int random_seed_; + + // Our random number generator. + internal::Random random_; + + // The time of the test program start, in ms from the start of the + // UNIX epoch. + TimeInMillis start_timestamp_; + + // How long the test took to run, in milliseconds. + TimeInMillis elapsed_time_; + +#if GTEST_HAS_DEATH_TEST + // The decomposed components of the gtest_internal_run_death_test flag, + // parsed when RUN_ALL_TESTS is called. + internal::scoped_ptr internal_run_death_test_flag_; + internal::scoped_ptr death_test_factory_; +#endif // GTEST_HAS_DEATH_TEST + + // A per-thread stack of traces created by the SCOPED_TRACE() macro. + internal::ThreadLocal > gtest_trace_stack_; + + // The value of GTEST_FLAG(catch_exceptions) at the moment RunAllTests() + // starts. + bool catch_exceptions_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(UnitTestImpl); +}; // class UnitTestImpl + +// Convenience function for accessing the global UnitTest +// implementation object. +inline UnitTestImpl* GetUnitTestImpl() { + return UnitTest::GetInstance()->impl(); +} + +#if GTEST_USES_SIMPLE_RE + +// Internal helper functions for implementing the simple regular +// expression matcher. +GTEST_API_ bool IsInSet(char ch, const char* str); +GTEST_API_ bool IsAsciiDigit(char ch); +GTEST_API_ bool IsAsciiPunct(char ch); +GTEST_API_ bool IsRepeat(char ch); +GTEST_API_ bool IsAsciiWhiteSpace(char ch); +GTEST_API_ bool IsAsciiWordChar(char ch); +GTEST_API_ bool IsValidEscape(char ch); +GTEST_API_ bool AtomMatchesChar(bool escaped, char pattern, char ch); +GTEST_API_ bool ValidateRegex(const char* regex); +GTEST_API_ bool MatchRegexAtHead(const char* regex, const char* str); +GTEST_API_ bool MatchRepetitionAndRegexAtHead( + bool escaped, char ch, char repeat, const char* regex, const char* str); +GTEST_API_ bool MatchRegexAnywhere(const char* regex, const char* str); + +#endif // GTEST_USES_SIMPLE_RE + +// Parses the command line for Google Test flags, without initializing +// other parts of Google Test. +GTEST_API_ void ParseGoogleTestFlagsOnly(int* argc, char** argv); +GTEST_API_ void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv); + +#if GTEST_HAS_DEATH_TEST + +// Returns the message describing the last system error, regardless of the +// platform. +GTEST_API_ std::string GetLastErrnoDescription(); + +// Attempts to parse a string into a positive integer pointed to by the +// number parameter. Returns true if that is possible. +// GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we can use +// it here. +template +bool ParseNaturalNumber(const ::std::string& str, Integer* number) { + // Fail fast if the given string does not begin with a digit; + // this bypasses strtoXXX's "optional leading whitespace and plus + // or minus sign" semantics, which are undesirable here. + if (str.empty() || !IsDigit(str[0])) { + return false; + } + errno = 0; + + char* end; + // BiggestConvertible is the largest integer type that system-provided + // string-to-number conversion routines can return. + +# if GTEST_OS_WINDOWS && !defined(__GNUC__) + + // MSVC and C++ Builder define __int64 instead of the standard long long. + typedef unsigned __int64 BiggestConvertible; + const BiggestConvertible parsed = _strtoui64(str.c_str(), &end, 10); + +# else + + typedef unsigned long long BiggestConvertible; // NOLINT + const BiggestConvertible parsed = strtoull(str.c_str(), &end, 10); + +# endif // GTEST_OS_WINDOWS && !defined(__GNUC__) + + const bool parse_success = *end == '\0' && errno == 0; + + // TODO(vladl@google.com): Convert this to compile time assertion when it is + // available. + GTEST_CHECK_(sizeof(Integer) <= sizeof(parsed)); + + const Integer result = static_cast(parsed); + if (parse_success && static_cast(result) == parsed) { + *number = result; + return true; + } + return false; +} +#endif // GTEST_HAS_DEATH_TEST + +// TestResult contains some private methods that should be hidden from +// Google Test user but are required for testing. This class allow our tests +// to access them. +// +// This class is supplied only for the purpose of testing Google Test's own +// constructs. Do not use it in user tests, either directly or indirectly. +class TestResultAccessor { + public: + static void RecordProperty(TestResult* test_result, + const std::string& xml_element, + const TestProperty& property) { + test_result->RecordProperty(xml_element, property); + } + + static void ClearTestPartResults(TestResult* test_result) { + test_result->ClearTestPartResults(); + } + + static const std::vector& test_part_results( + const TestResult& test_result) { + return test_result.test_part_results(); + } +}; + +#if GTEST_CAN_STREAM_RESULTS_ + +// Streams test results to the given port on the given host machine. +class GTEST_API_ StreamingListener : public EmptyTestEventListener { + public: + // Abstract base class for writing strings to a socket. + class AbstractSocketWriter { + public: + virtual ~AbstractSocketWriter() {} + + // Sends a string to the socket. + virtual void Send(const string& message) = 0; + + // Closes the socket. + virtual void CloseConnection() {} + + // Sends a string and a newline to the socket. + void SendLn(const string& message) { + Send(message + "\n"); + } + }; + + // Concrete class for actually writing strings to a socket. + class SocketWriter : public AbstractSocketWriter { + public: + SocketWriter(const string& host, const string& port) + : sockfd_(-1), host_name_(host), port_num_(port) { + MakeConnection(); + } + + virtual ~SocketWriter() { + if (sockfd_ != -1) + CloseConnection(); + } + + // Sends a string to the socket. + virtual void Send(const string& message) { + GTEST_CHECK_(sockfd_ != -1) + << "Send() can be called only when there is a connection."; + + const int len = static_cast(message.length()); + if (write(sockfd_, message.c_str(), len) != len) { + GTEST_LOG_(WARNING) + << "stream_result_to: failed to stream to " + << host_name_ << ":" << port_num_; + } + } + + private: + // Creates a client socket and connects to the server. + void MakeConnection(); + + // Closes the socket. + void CloseConnection() { + GTEST_CHECK_(sockfd_ != -1) + << "CloseConnection() can be called only when there is a connection."; + + close(sockfd_); + sockfd_ = -1; + } + + int sockfd_; // socket file descriptor + const string host_name_; + const string port_num_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(SocketWriter); + }; // class SocketWriter + + // Escapes '=', '&', '%', and '\n' characters in str as "%xx". + static string UrlEncode(const char* str); + + StreamingListener(const string& host, const string& port) + : socket_writer_(new SocketWriter(host, port)) { Start(); } + + explicit StreamingListener(AbstractSocketWriter* socket_writer) + : socket_writer_(socket_writer) { Start(); } + + void OnTestProgramStart(const UnitTest& /* unit_test */) { + SendLn("event=TestProgramStart"); + } + + void OnTestProgramEnd(const UnitTest& unit_test) { + // Note that Google Test current only report elapsed time for each + // test iteration, not for the entire test program. + SendLn("event=TestProgramEnd&passed=" + FormatBool(unit_test.Passed())); + + // Notify the streaming server to stop. + socket_writer_->CloseConnection(); + } + + void OnTestIterationStart(const UnitTest& /* unit_test */, int iteration) { + SendLn("event=TestIterationStart&iteration=" + + StreamableToString(iteration)); + } + + void OnTestIterationEnd(const UnitTest& unit_test, int /* iteration */) { + SendLn("event=TestIterationEnd&passed=" + + FormatBool(unit_test.Passed()) + "&elapsed_time=" + + StreamableToString(unit_test.elapsed_time()) + "ms"); + } + + void OnTestCaseStart(const TestCase& test_case) { + SendLn(std::string("event=TestCaseStart&name=") + test_case.name()); + } + + void OnTestCaseEnd(const TestCase& test_case) { + SendLn("event=TestCaseEnd&passed=" + FormatBool(test_case.Passed()) + + "&elapsed_time=" + StreamableToString(test_case.elapsed_time()) + + "ms"); + } + + void OnTestStart(const TestInfo& test_info) { + SendLn(std::string("event=TestStart&name=") + test_info.name()); + } + + void OnTestEnd(const TestInfo& test_info) { + SendLn("event=TestEnd&passed=" + + FormatBool((test_info.result())->Passed()) + + "&elapsed_time=" + + StreamableToString((test_info.result())->elapsed_time()) + "ms"); + } + + void OnTestPartResult(const TestPartResult& test_part_result) { + const char* file_name = test_part_result.file_name(); + if (file_name == NULL) + file_name = ""; + SendLn("event=TestPartResult&file=" + UrlEncode(file_name) + + "&line=" + StreamableToString(test_part_result.line_number()) + + "&message=" + UrlEncode(test_part_result.message())); + } + + private: + // Sends the given message and a newline to the socket. + void SendLn(const string& message) { socket_writer_->SendLn(message); } + + // Called at the start of streaming to notify the receiver what + // protocol we are using. + void Start() { SendLn("gtest_streaming_protocol_version=1.0"); } + + string FormatBool(bool value) { return value ? "1" : "0"; } + + const scoped_ptr socket_writer_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(StreamingListener); +}; // class StreamingListener + +#endif // GTEST_CAN_STREAM_RESULTS_ + +} // namespace internal +} // namespace testing + +#endif // GTEST_SRC_GTEST_INTERNAL_INL_H_ diff --git a/third_party/aom/third_party/googletest/src/googletest/src/gtest-port.cc b/third_party/aom/third_party/googletest/src/googletest/src/gtest-port.cc new file mode 100644 index 0000000000..e5bf3dd2be --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/src/gtest-port.cc @@ -0,0 +1,1259 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +#include "gtest/internal/gtest-port.h" + +#include +#include +#include +#include +#include + +#if GTEST_OS_WINDOWS +# include +# include +# include +# include // Used in ThreadLocal. +#else +# include +#endif // GTEST_OS_WINDOWS + +#if GTEST_OS_MAC +# include +# include +# include +#endif // GTEST_OS_MAC + +#if GTEST_OS_QNX +# include +# include +# include +#endif // GTEST_OS_QNX + +#if GTEST_OS_AIX +# include +# include +#endif // GTEST_OS_AIX + +#include "gtest/gtest-spi.h" +#include "gtest/gtest-message.h" +#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-string.h" + +// Indicates that this translation unit is part of Google Test's +// implementation. It must come before gtest-internal-inl.h is +// included, or there will be a compiler error. This trick exists to +// prevent the accidental inclusion of gtest-internal-inl.h in the +// user's code. +#define GTEST_IMPLEMENTATION_ 1 +#include "src/gtest-internal-inl.h" +#undef GTEST_IMPLEMENTATION_ + +namespace testing { +namespace internal { + +#if defined(_MSC_VER) || defined(__BORLANDC__) +// MSVC and C++Builder do not provide a definition of STDERR_FILENO. +const int kStdOutFileno = 1; +const int kStdErrFileno = 2; +#else +const int kStdOutFileno = STDOUT_FILENO; +const int kStdErrFileno = STDERR_FILENO; +#endif // _MSC_VER + +#if GTEST_OS_LINUX + +namespace { +template +T ReadProcFileField(const string& filename, int field) { + std::string dummy; + std::ifstream file(filename.c_str()); + while (field-- > 0) { + file >> dummy; + } + T output = 0; + file >> output; + return output; +} +} // namespace + +// Returns the number of active threads, or 0 when there is an error. +size_t GetThreadCount() { + const string filename = + (Message() << "/proc/" << getpid() << "/stat").GetString(); + return ReadProcFileField(filename, 19); +} + +#elif GTEST_OS_MAC + +size_t GetThreadCount() { + const task_t task = mach_task_self(); + mach_msg_type_number_t thread_count; + thread_act_array_t thread_list; + const kern_return_t status = task_threads(task, &thread_list, &thread_count); + if (status == KERN_SUCCESS) { + // task_threads allocates resources in thread_list and we need to free them + // to avoid leaks. + vm_deallocate(task, + reinterpret_cast(thread_list), + sizeof(thread_t) * thread_count); + return static_cast(thread_count); + } else { + return 0; + } +} + +#elif GTEST_OS_QNX + +// Returns the number of threads running in the process, or 0 to indicate that +// we cannot detect it. +size_t GetThreadCount() { + const int fd = open("/proc/self/as", O_RDONLY); + if (fd < 0) { + return 0; + } + procfs_info process_info; + const int status = + devctl(fd, DCMD_PROC_INFO, &process_info, sizeof(process_info), NULL); + close(fd); + if (status == EOK) { + return static_cast(process_info.num_threads); + } else { + return 0; + } +} + +#elif GTEST_OS_AIX + +size_t GetThreadCount() { + struct procentry64 entry; + pid_t pid = getpid(); + int status = getprocs64(&entry, sizeof(entry), NULL, 0, &pid, 1); + if (status == 1) { + return entry.pi_thcount; + } else { + return 0; + } +} + +#else + +size_t GetThreadCount() { + // There's no portable way to detect the number of threads, so we just + // return 0 to indicate that we cannot detect it. + return 0; +} + +#endif // GTEST_OS_LINUX + +#if GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS + +void SleepMilliseconds(int n) { + ::Sleep(n); +} + +AutoHandle::AutoHandle() + : handle_(INVALID_HANDLE_VALUE) {} + +AutoHandle::AutoHandle(Handle handle) + : handle_(handle) {} + +AutoHandle::~AutoHandle() { + Reset(); +} + +AutoHandle::Handle AutoHandle::Get() const { + return handle_; +} + +void AutoHandle::Reset() { + Reset(INVALID_HANDLE_VALUE); +} + +void AutoHandle::Reset(HANDLE handle) { + // Resetting with the same handle we already own is invalid. + if (handle_ != handle) { + if (IsCloseable()) { + ::CloseHandle(handle_); + } + handle_ = handle; + } else { + GTEST_CHECK_(!IsCloseable()) + << "Resetting a valid handle to itself is likely a programmer error " + "and thus not allowed."; + } +} + +bool AutoHandle::IsCloseable() const { + // Different Windows APIs may use either of these values to represent an + // invalid handle. + return handle_ != NULL && handle_ != INVALID_HANDLE_VALUE; +} + +Notification::Notification() + : event_(::CreateEvent(NULL, // Default security attributes. + TRUE, // Do not reset automatically. + FALSE, // Initially unset. + NULL)) { // Anonymous event. + GTEST_CHECK_(event_.Get() != NULL); +} + +void Notification::Notify() { + GTEST_CHECK_(::SetEvent(event_.Get()) != FALSE); +} + +void Notification::WaitForNotification() { + GTEST_CHECK_( + ::WaitForSingleObject(event_.Get(), INFINITE) == WAIT_OBJECT_0); +} + +Mutex::Mutex() + : owner_thread_id_(0), + type_(kDynamic), + critical_section_init_phase_(0), + critical_section_(new CRITICAL_SECTION) { + ::InitializeCriticalSection(critical_section_); +} + +Mutex::~Mutex() { + // Static mutexes are leaked intentionally. It is not thread-safe to try + // to clean them up. + // TODO(yukawa): Switch to Slim Reader/Writer (SRW) Locks, which requires + // nothing to clean it up but is available only on Vista and later. + // http://msdn.microsoft.com/en-us/library/windows/desktop/aa904937.aspx + if (type_ == kDynamic) { + ::DeleteCriticalSection(critical_section_); + delete critical_section_; + critical_section_ = NULL; + } +} + +void Mutex::Lock() { + ThreadSafeLazyInit(); + ::EnterCriticalSection(critical_section_); + owner_thread_id_ = ::GetCurrentThreadId(); +} + +void Mutex::Unlock() { + ThreadSafeLazyInit(); + // We don't protect writing to owner_thread_id_ here, as it's the + // caller's responsibility to ensure that the current thread holds the + // mutex when this is called. + owner_thread_id_ = 0; + ::LeaveCriticalSection(critical_section_); +} + +// Does nothing if the current thread holds the mutex. Otherwise, crashes +// with high probability. +void Mutex::AssertHeld() { + ThreadSafeLazyInit(); + GTEST_CHECK_(owner_thread_id_ == ::GetCurrentThreadId()) + << "The current thread is not holding the mutex @" << this; +} + +// Initializes owner_thread_id_ and critical_section_ in static mutexes. +void Mutex::ThreadSafeLazyInit() { + // Dynamic mutexes are initialized in the constructor. + if (type_ == kStatic) { + switch ( + ::InterlockedCompareExchange(&critical_section_init_phase_, 1L, 0L)) { + case 0: + // If critical_section_init_phase_ was 0 before the exchange, we + // are the first to test it and need to perform the initialization. + owner_thread_id_ = 0; + critical_section_ = new CRITICAL_SECTION; + ::InitializeCriticalSection(critical_section_); + // Updates the critical_section_init_phase_ to 2 to signal + // initialization complete. + GTEST_CHECK_(::InterlockedCompareExchange( + &critical_section_init_phase_, 2L, 1L) == + 1L); + break; + case 1: + // Somebody else is already initializing the mutex; spin until they + // are done. + while (::InterlockedCompareExchange(&critical_section_init_phase_, + 2L, + 2L) != 2L) { + // Possibly yields the rest of the thread's time slice to other + // threads. + ::Sleep(0); + } + break; + + case 2: + break; // The mutex is already initialized and ready for use. + + default: + GTEST_CHECK_(false) + << "Unexpected value of critical_section_init_phase_ " + << "while initializing a static mutex."; + } + } +} + +namespace { + +class ThreadWithParamSupport : public ThreadWithParamBase { + public: + static HANDLE CreateThread(Runnable* runnable, + Notification* thread_can_start) { + ThreadMainParam* param = new ThreadMainParam(runnable, thread_can_start); + DWORD thread_id; + // TODO(yukawa): Consider to use _beginthreadex instead. + HANDLE thread_handle = ::CreateThread( + NULL, // Default security. + 0, // Default stack size. + &ThreadWithParamSupport::ThreadMain, + param, // Parameter to ThreadMainStatic + 0x0, // Default creation flags. + &thread_id); // Need a valid pointer for the call to work under Win98. + GTEST_CHECK_(thread_handle != NULL) << "CreateThread failed with error " + << ::GetLastError() << "."; + if (thread_handle == NULL) { + delete param; + } + return thread_handle; + } + + private: + struct ThreadMainParam { + ThreadMainParam(Runnable* runnable, Notification* thread_can_start) + : runnable_(runnable), + thread_can_start_(thread_can_start) { + } + scoped_ptr runnable_; + // Does not own. + Notification* thread_can_start_; + }; + + static DWORD WINAPI ThreadMain(void* ptr) { + // Transfers ownership. + scoped_ptr param(static_cast(ptr)); + if (param->thread_can_start_ != NULL) + param->thread_can_start_->WaitForNotification(); + param->runnable_->Run(); + return 0; + } + + // Prohibit instantiation. + ThreadWithParamSupport(); + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParamSupport); +}; + +} // namespace + +ThreadWithParamBase::ThreadWithParamBase(Runnable *runnable, + Notification* thread_can_start) + : thread_(ThreadWithParamSupport::CreateThread(runnable, + thread_can_start)) { +} + +ThreadWithParamBase::~ThreadWithParamBase() { + Join(); +} + +void ThreadWithParamBase::Join() { + GTEST_CHECK_(::WaitForSingleObject(thread_.Get(), INFINITE) == WAIT_OBJECT_0) + << "Failed to join the thread with error " << ::GetLastError() << "."; +} + +// Maps a thread to a set of ThreadIdToThreadLocals that have values +// instantiated on that thread and notifies them when the thread exits. A +// ThreadLocal instance is expected to persist until all threads it has +// values on have terminated. +class ThreadLocalRegistryImpl { + public: + // Registers thread_local_instance as having value on the current thread. + // Returns a value that can be used to identify the thread from other threads. + static ThreadLocalValueHolderBase* GetValueOnCurrentThread( + const ThreadLocalBase* thread_local_instance) { + DWORD current_thread = ::GetCurrentThreadId(); + MutexLock lock(&mutex_); + ThreadIdToThreadLocals* const thread_to_thread_locals = + GetThreadLocalsMapLocked(); + ThreadIdToThreadLocals::iterator thread_local_pos = + thread_to_thread_locals->find(current_thread); + if (thread_local_pos == thread_to_thread_locals->end()) { + thread_local_pos = thread_to_thread_locals->insert( + std::make_pair(current_thread, ThreadLocalValues())).first; + StartWatcherThreadFor(current_thread); + } + ThreadLocalValues& thread_local_values = thread_local_pos->second; + ThreadLocalValues::iterator value_pos = + thread_local_values.find(thread_local_instance); + if (value_pos == thread_local_values.end()) { + value_pos = + thread_local_values + .insert(std::make_pair( + thread_local_instance, + linked_ptr( + thread_local_instance->NewValueForCurrentThread()))) + .first; + } + return value_pos->second.get(); + } + + static void OnThreadLocalDestroyed( + const ThreadLocalBase* thread_local_instance) { + std::vector > value_holders; + // Clean up the ThreadLocalValues data structure while holding the lock, but + // defer the destruction of the ThreadLocalValueHolderBases. + { + MutexLock lock(&mutex_); + ThreadIdToThreadLocals* const thread_to_thread_locals = + GetThreadLocalsMapLocked(); + for (ThreadIdToThreadLocals::iterator it = + thread_to_thread_locals->begin(); + it != thread_to_thread_locals->end(); + ++it) { + ThreadLocalValues& thread_local_values = it->second; + ThreadLocalValues::iterator value_pos = + thread_local_values.find(thread_local_instance); + if (value_pos != thread_local_values.end()) { + value_holders.push_back(value_pos->second); + thread_local_values.erase(value_pos); + // This 'if' can only be successful at most once, so theoretically we + // could break out of the loop here, but we don't bother doing so. + } + } + } + // Outside the lock, let the destructor for 'value_holders' deallocate the + // ThreadLocalValueHolderBases. + } + + static void OnThreadExit(DWORD thread_id) { + GTEST_CHECK_(thread_id != 0) << ::GetLastError(); + std::vector > value_holders; + // Clean up the ThreadIdToThreadLocals data structure while holding the + // lock, but defer the destruction of the ThreadLocalValueHolderBases. + { + MutexLock lock(&mutex_); + ThreadIdToThreadLocals* const thread_to_thread_locals = + GetThreadLocalsMapLocked(); + ThreadIdToThreadLocals::iterator thread_local_pos = + thread_to_thread_locals->find(thread_id); + if (thread_local_pos != thread_to_thread_locals->end()) { + ThreadLocalValues& thread_local_values = thread_local_pos->second; + for (ThreadLocalValues::iterator value_pos = + thread_local_values.begin(); + value_pos != thread_local_values.end(); + ++value_pos) { + value_holders.push_back(value_pos->second); + } + thread_to_thread_locals->erase(thread_local_pos); + } + } + // Outside the lock, let the destructor for 'value_holders' deallocate the + // ThreadLocalValueHolderBases. + } + + private: + // In a particular thread, maps a ThreadLocal object to its value. + typedef std::map > ThreadLocalValues; + // Stores all ThreadIdToThreadLocals having values in a thread, indexed by + // thread's ID. + typedef std::map ThreadIdToThreadLocals; + + // Holds the thread id and thread handle that we pass from + // StartWatcherThreadFor to WatcherThreadFunc. + typedef std::pair ThreadIdAndHandle; + + static void StartWatcherThreadFor(DWORD thread_id) { + // The returned handle will be kept in thread_map and closed by + // watcher_thread in WatcherThreadFunc. + HANDLE thread = ::OpenThread(SYNCHRONIZE | THREAD_QUERY_INFORMATION, + FALSE, + thread_id); + GTEST_CHECK_(thread != NULL); + // We need to to pass a valid thread ID pointer into CreateThread for it + // to work correctly under Win98. + DWORD watcher_thread_id; + HANDLE watcher_thread = ::CreateThread( + NULL, // Default security. + 0, // Default stack size + &ThreadLocalRegistryImpl::WatcherThreadFunc, + reinterpret_cast(new ThreadIdAndHandle(thread_id, thread)), + CREATE_SUSPENDED, + &watcher_thread_id); + GTEST_CHECK_(watcher_thread != NULL); + // Give the watcher thread the same priority as ours to avoid being + // blocked by it. + ::SetThreadPriority(watcher_thread, + ::GetThreadPriority(::GetCurrentThread())); + ::ResumeThread(watcher_thread); + ::CloseHandle(watcher_thread); + } + + // Monitors exit from a given thread and notifies those + // ThreadIdToThreadLocals about thread termination. + static DWORD WINAPI WatcherThreadFunc(LPVOID param) { + const ThreadIdAndHandle* tah = + reinterpret_cast(param); + GTEST_CHECK_( + ::WaitForSingleObject(tah->second, INFINITE) == WAIT_OBJECT_0); + OnThreadExit(tah->first); + ::CloseHandle(tah->second); + delete tah; + return 0; + } + + // Returns map of thread local instances. + static ThreadIdToThreadLocals* GetThreadLocalsMapLocked() { + mutex_.AssertHeld(); + static ThreadIdToThreadLocals* map = new ThreadIdToThreadLocals; + return map; + } + + // Protects access to GetThreadLocalsMapLocked() and its return value. + static Mutex mutex_; + // Protects access to GetThreadMapLocked() and its return value. + static Mutex thread_map_mutex_; +}; + +Mutex ThreadLocalRegistryImpl::mutex_(Mutex::kStaticMutex); +Mutex ThreadLocalRegistryImpl::thread_map_mutex_(Mutex::kStaticMutex); + +ThreadLocalValueHolderBase* ThreadLocalRegistry::GetValueOnCurrentThread( + const ThreadLocalBase* thread_local_instance) { + return ThreadLocalRegistryImpl::GetValueOnCurrentThread( + thread_local_instance); +} + +void ThreadLocalRegistry::OnThreadLocalDestroyed( + const ThreadLocalBase* thread_local_instance) { + ThreadLocalRegistryImpl::OnThreadLocalDestroyed(thread_local_instance); +} + +#endif // GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS + +#if GTEST_USES_POSIX_RE + +// Implements RE. Currently only needed for death tests. + +RE::~RE() { + if (is_valid_) { + // regfree'ing an invalid regex might crash because the content + // of the regex is undefined. Since the regex's are essentially + // the same, one cannot be valid (or invalid) without the other + // being so too. + regfree(&partial_regex_); + regfree(&full_regex_); + } + free(const_cast(pattern_)); +} + +// Returns true iff regular expression re matches the entire str. +bool RE::FullMatch(const char* str, const RE& re) { + if (!re.is_valid_) return false; + + regmatch_t match; + return regexec(&re.full_regex_, str, 1, &match, 0) == 0; +} + +// Returns true iff regular expression re matches a substring of str +// (including str itself). +bool RE::PartialMatch(const char* str, const RE& re) { + if (!re.is_valid_) return false; + + regmatch_t match; + return regexec(&re.partial_regex_, str, 1, &match, 0) == 0; +} + +// Initializes an RE from its string representation. +void RE::Init(const char* regex) { + pattern_ = posix::StrDup(regex); + + // Reserves enough bytes to hold the regular expression used for a + // full match. + const size_t full_regex_len = strlen(regex) + 10; + char* const full_pattern = new char[full_regex_len]; + + snprintf(full_pattern, full_regex_len, "^(%s)$", regex); + is_valid_ = regcomp(&full_regex_, full_pattern, REG_EXTENDED) == 0; + // We want to call regcomp(&partial_regex_, ...) even if the + // previous expression returns false. Otherwise partial_regex_ may + // not be properly initialized can may cause trouble when it's + // freed. + // + // Some implementation of POSIX regex (e.g. on at least some + // versions of Cygwin) doesn't accept the empty string as a valid + // regex. We change it to an equivalent form "()" to be safe. + if (is_valid_) { + const char* const partial_regex = (*regex == '\0') ? "()" : regex; + is_valid_ = regcomp(&partial_regex_, partial_regex, REG_EXTENDED) == 0; + } + EXPECT_TRUE(is_valid_) + << "Regular expression \"" << regex + << "\" is not a valid POSIX Extended regular expression."; + + delete[] full_pattern; +} + +#elif GTEST_USES_SIMPLE_RE + +// Returns true iff ch appears anywhere in str (excluding the +// terminating '\0' character). +bool IsInSet(char ch, const char* str) { + return ch != '\0' && strchr(str, ch) != NULL; +} + +// Returns true iff ch belongs to the given classification. Unlike +// similar functions in , these aren't affected by the +// current locale. +bool IsAsciiDigit(char ch) { return '0' <= ch && ch <= '9'; } +bool IsAsciiPunct(char ch) { + return IsInSet(ch, "^-!\"#$%&'()*+,./:;<=>?@[\\]_`{|}~"); +} +bool IsRepeat(char ch) { return IsInSet(ch, "?*+"); } +bool IsAsciiWhiteSpace(char ch) { return IsInSet(ch, " \f\n\r\t\v"); } +bool IsAsciiWordChar(char ch) { + return ('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z') || + ('0' <= ch && ch <= '9') || ch == '_'; +} + +// Returns true iff "\\c" is a supported escape sequence. +bool IsValidEscape(char c) { + return (IsAsciiPunct(c) || IsInSet(c, "dDfnrsStvwW")); +} + +// Returns true iff the given atom (specified by escaped and pattern) +// matches ch. The result is undefined if the atom is invalid. +bool AtomMatchesChar(bool escaped, char pattern_char, char ch) { + if (escaped) { // "\\p" where p is pattern_char. + switch (pattern_char) { + case 'd': return IsAsciiDigit(ch); + case 'D': return !IsAsciiDigit(ch); + case 'f': return ch == '\f'; + case 'n': return ch == '\n'; + case 'r': return ch == '\r'; + case 's': return IsAsciiWhiteSpace(ch); + case 'S': return !IsAsciiWhiteSpace(ch); + case 't': return ch == '\t'; + case 'v': return ch == '\v'; + case 'w': return IsAsciiWordChar(ch); + case 'W': return !IsAsciiWordChar(ch); + } + return IsAsciiPunct(pattern_char) && pattern_char == ch; + } + + return (pattern_char == '.' && ch != '\n') || pattern_char == ch; +} + +// Helper function used by ValidateRegex() to format error messages. +std::string FormatRegexSyntaxError(const char* regex, int index) { + return (Message() << "Syntax error at index " << index + << " in simple regular expression \"" << regex << "\": ").GetString(); +} + +// Generates non-fatal failures and returns false if regex is invalid; +// otherwise returns true. +bool ValidateRegex(const char* regex) { + if (regex == NULL) { + // TODO(wan@google.com): fix the source file location in the + // assertion failures to match where the regex is used in user + // code. + ADD_FAILURE() << "NULL is not a valid simple regular expression."; + return false; + } + + bool is_valid = true; + + // True iff ?, *, or + can follow the previous atom. + bool prev_repeatable = false; + for (int i = 0; regex[i]; i++) { + if (regex[i] == '\\') { // An escape sequence + i++; + if (regex[i] == '\0') { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1) + << "'\\' cannot appear at the end."; + return false; + } + + if (!IsValidEscape(regex[i])) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1) + << "invalid escape sequence \"\\" << regex[i] << "\"."; + is_valid = false; + } + prev_repeatable = true; + } else { // Not an escape sequence. + const char ch = regex[i]; + + if (ch == '^' && i > 0) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'^' can only appear at the beginning."; + is_valid = false; + } else if (ch == '$' && regex[i + 1] != '\0') { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'$' can only appear at the end."; + is_valid = false; + } else if (IsInSet(ch, "()[]{}|")) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'" << ch << "' is unsupported."; + is_valid = false; + } else if (IsRepeat(ch) && !prev_repeatable) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'" << ch << "' can only follow a repeatable token."; + is_valid = false; + } + + prev_repeatable = !IsInSet(ch, "^$?*+"); + } + } + + return is_valid; +} + +// Matches a repeated regex atom followed by a valid simple regular +// expression. The regex atom is defined as c if escaped is false, +// or \c otherwise. repeat is the repetition meta character (?, *, +// or +). The behavior is undefined if str contains too many +// characters to be indexable by size_t, in which case the test will +// probably time out anyway. We are fine with this limitation as +// std::string has it too. +bool MatchRepetitionAndRegexAtHead( + bool escaped, char c, char repeat, const char* regex, + const char* str) { + const size_t min_count = (repeat == '+') ? 1 : 0; + const size_t max_count = (repeat == '?') ? 1 : + static_cast(-1) - 1; + // We cannot call numeric_limits::max() as it conflicts with the + // max() macro on Windows. + + for (size_t i = 0; i <= max_count; ++i) { + // We know that the atom matches each of the first i characters in str. + if (i >= min_count && MatchRegexAtHead(regex, str + i)) { + // We have enough matches at the head, and the tail matches too. + // Since we only care about *whether* the pattern matches str + // (as opposed to *how* it matches), there is no need to find a + // greedy match. + return true; + } + if (str[i] == '\0' || !AtomMatchesChar(escaped, c, str[i])) + return false; + } + return false; +} + +// Returns true iff regex matches a prefix of str. regex must be a +// valid simple regular expression and not start with "^", or the +// result is undefined. +bool MatchRegexAtHead(const char* regex, const char* str) { + if (*regex == '\0') // An empty regex matches a prefix of anything. + return true; + + // "$" only matches the end of a string. Note that regex being + // valid guarantees that there's nothing after "$" in it. + if (*regex == '$') + return *str == '\0'; + + // Is the first thing in regex an escape sequence? + const bool escaped = *regex == '\\'; + if (escaped) + ++regex; + if (IsRepeat(regex[1])) { + // MatchRepetitionAndRegexAtHead() calls MatchRegexAtHead(), so + // here's an indirect recursion. It terminates as the regex gets + // shorter in each recursion. + return MatchRepetitionAndRegexAtHead( + escaped, regex[0], regex[1], regex + 2, str); + } else { + // regex isn't empty, isn't "$", and doesn't start with a + // repetition. We match the first atom of regex with the first + // character of str and recurse. + return (*str != '\0') && AtomMatchesChar(escaped, *regex, *str) && + MatchRegexAtHead(regex + 1, str + 1); + } +} + +// Returns true iff regex matches any substring of str. regex must be +// a valid simple regular expression, or the result is undefined. +// +// The algorithm is recursive, but the recursion depth doesn't exceed +// the regex length, so we won't need to worry about running out of +// stack space normally. In rare cases the time complexity can be +// exponential with respect to the regex length + the string length, +// but usually it's must faster (often close to linear). +bool MatchRegexAnywhere(const char* regex, const char* str) { + if (regex == NULL || str == NULL) + return false; + + if (*regex == '^') + return MatchRegexAtHead(regex + 1, str); + + // A successful match can be anywhere in str. + do { + if (MatchRegexAtHead(regex, str)) + return true; + } while (*str++ != '\0'); + return false; +} + +// Implements the RE class. + +RE::~RE() { + free(const_cast(pattern_)); + free(const_cast(full_pattern_)); +} + +// Returns true iff regular expression re matches the entire str. +bool RE::FullMatch(const char* str, const RE& re) { + return re.is_valid_ && MatchRegexAnywhere(re.full_pattern_, str); +} + +// Returns true iff regular expression re matches a substring of str +// (including str itself). +bool RE::PartialMatch(const char* str, const RE& re) { + return re.is_valid_ && MatchRegexAnywhere(re.pattern_, str); +} + +// Initializes an RE from its string representation. +void RE::Init(const char* regex) { + pattern_ = full_pattern_ = NULL; + if (regex != NULL) { + pattern_ = posix::StrDup(regex); + } + + is_valid_ = ValidateRegex(regex); + if (!is_valid_) { + // No need to calculate the full pattern when the regex is invalid. + return; + } + + const size_t len = strlen(regex); + // Reserves enough bytes to hold the regular expression used for a + // full match: we need space to prepend a '^', append a '$', and + // terminate the string with '\0'. + char* buffer = static_cast(malloc(len + 3)); + full_pattern_ = buffer; + + if (*regex != '^') + *buffer++ = '^'; // Makes sure full_pattern_ starts with '^'. + + // We don't use snprintf or strncpy, as they trigger a warning when + // compiled with VC++ 8.0. + memcpy(buffer, regex, len); + buffer += len; + + if (len == 0 || regex[len - 1] != '$') + *buffer++ = '$'; // Makes sure full_pattern_ ends with '$'. + + *buffer = '\0'; +} + +#endif // GTEST_USES_POSIX_RE + +const char kUnknownFile[] = "unknown file"; + +// Formats a source file path and a line number as they would appear +// in an error message from the compiler used to compile this code. +GTEST_API_ ::std::string FormatFileLocation(const char* file, int line) { + const std::string file_name(file == NULL ? kUnknownFile : file); + + if (line < 0) { + return file_name + ":"; + } +#ifdef _MSC_VER + return file_name + "(" + StreamableToString(line) + "):"; +#else + return file_name + ":" + StreamableToString(line) + ":"; +#endif // _MSC_VER +} + +// Formats a file location for compiler-independent XML output. +// Although this function is not platform dependent, we put it next to +// FormatFileLocation in order to contrast the two functions. +// Note that FormatCompilerIndependentFileLocation() does NOT append colon +// to the file location it produces, unlike FormatFileLocation(). +GTEST_API_ ::std::string FormatCompilerIndependentFileLocation( + const char* file, int line) { + const std::string file_name(file == NULL ? kUnknownFile : file); + + if (line < 0) + return file_name; + else + return file_name + ":" + StreamableToString(line); +} + +GTestLog::GTestLog(GTestLogSeverity severity, const char* file, int line) + : severity_(severity) { + const char* const marker = + severity == GTEST_INFO ? "[ INFO ]" : + severity == GTEST_WARNING ? "[WARNING]" : + severity == GTEST_ERROR ? "[ ERROR ]" : "[ FATAL ]"; + GetStream() << ::std::endl << marker << " " + << FormatFileLocation(file, line).c_str() << ": "; +} + +// Flushes the buffers and, if severity is GTEST_FATAL, aborts the program. +GTestLog::~GTestLog() { + GetStream() << ::std::endl; + if (severity_ == GTEST_FATAL) { + fflush(stderr); + posix::Abort(); + } +} +// Disable Microsoft deprecation warnings for POSIX functions called from +// this class (creat, dup, dup2, and close) +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996) + +#if GTEST_HAS_STREAM_REDIRECTION + +// Object that captures an output stream (stdout/stderr). +class CapturedStream { + public: + // The ctor redirects the stream to a temporary file. + explicit CapturedStream(int fd) : fd_(fd), uncaptured_fd_(dup(fd)) { +# if GTEST_OS_WINDOWS + char temp_dir_path[MAX_PATH + 1] = { '\0' }; // NOLINT + char temp_file_path[MAX_PATH + 1] = { '\0' }; // NOLINT + + ::GetTempPathA(sizeof(temp_dir_path), temp_dir_path); + const UINT success = ::GetTempFileNameA(temp_dir_path, + "gtest_redir", + 0, // Generate unique file name. + temp_file_path); + GTEST_CHECK_(success != 0) + << "Unable to create a temporary file in " << temp_dir_path; + const int captured_fd = creat(temp_file_path, _S_IREAD | _S_IWRITE); + GTEST_CHECK_(captured_fd != -1) << "Unable to open temporary file " + << temp_file_path; + filename_ = temp_file_path; +# else + // There's no guarantee that a test has write access to the current + // directory, so we create the temporary file in the /tmp directory + // instead. We use /tmp on most systems, and /sdcard on Android. + // That's because Android doesn't have /tmp. +# if GTEST_OS_LINUX_ANDROID + // Note: Android applications are expected to call the framework's + // Context.getExternalStorageDirectory() method through JNI to get + // the location of the world-writable SD Card directory. However, + // this requires a Context handle, which cannot be retrieved + // globally from native code. Doing so also precludes running the + // code as part of a regular standalone executable, which doesn't + // run in a Dalvik process (e.g. when running it through 'adb shell'). + // + // The location /sdcard is directly accessible from native code + // and is the only location (unofficially) supported by the Android + // team. It's generally a symlink to the real SD Card mount point + // which can be /mnt/sdcard, /mnt/sdcard0, /system/media/sdcard, or + // other OEM-customized locations. Never rely on these, and always + // use /sdcard. + char name_template[] = "/sdcard/gtest_captured_stream.XXXXXX"; +# else + char name_template[] = "/tmp/captured_stream.XXXXXX"; +# endif // GTEST_OS_LINUX_ANDROID + const int captured_fd = mkstemp(name_template); + filename_ = name_template; +# endif // GTEST_OS_WINDOWS + fflush(NULL); + dup2(captured_fd, fd_); + close(captured_fd); + } + + ~CapturedStream() { + remove(filename_.c_str()); + } + + std::string GetCapturedString() { + if (uncaptured_fd_ != -1) { + // Restores the original stream. + fflush(NULL); + dup2(uncaptured_fd_, fd_); + close(uncaptured_fd_); + uncaptured_fd_ = -1; + } + + FILE* const file = posix::FOpen(filename_.c_str(), "r"); + const std::string content = ReadEntireFile(file); + posix::FClose(file); + return content; + } + + private: + const int fd_; // A stream to capture. + int uncaptured_fd_; + // Name of the temporary file holding the stderr output. + ::std::string filename_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(CapturedStream); +}; + +GTEST_DISABLE_MSC_WARNINGS_POP_() + +static CapturedStream* g_captured_stderr = NULL; +static CapturedStream* g_captured_stdout = NULL; + +// Starts capturing an output stream (stdout/stderr). +void CaptureStream(int fd, const char* stream_name, CapturedStream** stream) { + if (*stream != NULL) { + GTEST_LOG_(FATAL) << "Only one " << stream_name + << " capturer can exist at a time."; + } + *stream = new CapturedStream(fd); +} + +// Stops capturing the output stream and returns the captured string. +std::string GetCapturedStream(CapturedStream** captured_stream) { + const std::string content = (*captured_stream)->GetCapturedString(); + + delete *captured_stream; + *captured_stream = NULL; + + return content; +} + +// Starts capturing stdout. +void CaptureStdout() { + CaptureStream(kStdOutFileno, "stdout", &g_captured_stdout); +} + +// Starts capturing stderr. +void CaptureStderr() { + CaptureStream(kStdErrFileno, "stderr", &g_captured_stderr); +} + +// Stops capturing stdout and returns the captured string. +std::string GetCapturedStdout() { + return GetCapturedStream(&g_captured_stdout); +} + +// Stops capturing stderr and returns the captured string. +std::string GetCapturedStderr() { + return GetCapturedStream(&g_captured_stderr); +} + +#endif // GTEST_HAS_STREAM_REDIRECTION + +std::string TempDir() { +#if GTEST_OS_WINDOWS_MOBILE + return "\\temp\\"; +#elif GTEST_OS_WINDOWS + const char* temp_dir = posix::GetEnv("TEMP"); + if (temp_dir == NULL || temp_dir[0] == '\0') + return "\\temp\\"; + else if (temp_dir[strlen(temp_dir) - 1] == '\\') + return temp_dir; + else + return std::string(temp_dir) + "\\"; +#elif GTEST_OS_LINUX_ANDROID + return "/sdcard/"; +#else + return "/tmp/"; +#endif // GTEST_OS_WINDOWS_MOBILE +} + +size_t GetFileSize(FILE* file) { + fseek(file, 0, SEEK_END); + return static_cast(ftell(file)); +} + +std::string ReadEntireFile(FILE* file) { + const size_t file_size = GetFileSize(file); + char* const buffer = new char[file_size]; + + size_t bytes_last_read = 0; // # of bytes read in the last fread() + size_t bytes_read = 0; // # of bytes read so far + + fseek(file, 0, SEEK_SET); + + // Keeps reading the file until we cannot read further or the + // pre-determined file size is reached. + do { + bytes_last_read = fread(buffer+bytes_read, 1, file_size-bytes_read, file); + bytes_read += bytes_last_read; + } while (bytes_last_read > 0 && bytes_read < file_size); + + const std::string content(buffer, bytes_read); + delete[] buffer; + + return content; +} + +#if GTEST_HAS_DEATH_TEST + +static const ::std::vector* g_injected_test_argvs = + NULL; // Owned. + +void SetInjectableArgvs(const ::std::vector* argvs) { + if (g_injected_test_argvs != argvs) + delete g_injected_test_argvs; + g_injected_test_argvs = argvs; +} + +const ::std::vector& GetInjectableArgvs() { + if (g_injected_test_argvs != NULL) { + return *g_injected_test_argvs; + } + return GetArgvs(); +} +#endif // GTEST_HAS_DEATH_TEST + +#if GTEST_OS_WINDOWS_MOBILE +namespace posix { +void Abort() { + DebugBreak(); + TerminateProcess(GetCurrentProcess(), 1); +} +} // namespace posix +#endif // GTEST_OS_WINDOWS_MOBILE + +// Returns the name of the environment variable corresponding to the +// given flag. For example, FlagToEnvVar("foo") will return +// "GTEST_FOO" in the open-source version. +static std::string FlagToEnvVar(const char* flag) { + const std::string full_flag = + (Message() << GTEST_FLAG_PREFIX_ << flag).GetString(); + + Message env_var; + for (size_t i = 0; i != full_flag.length(); i++) { + env_var << ToUpper(full_flag.c_str()[i]); + } + + return env_var.GetString(); +} + +// Parses 'str' for a 32-bit signed integer. If successful, writes +// the result to *value and returns true; otherwise leaves *value +// unchanged and returns false. +bool ParseInt32(const Message& src_text, const char* str, Int32* value) { + // Parses the environment variable as a decimal integer. + char* end = NULL; + const long long_value = strtol(str, &end, 10); // NOLINT + + // Has strtol() consumed all characters in the string? + if (*end != '\0') { + // No - an invalid character was encountered. + Message msg; + msg << "WARNING: " << src_text + << " is expected to be a 32-bit integer, but actually" + << " has value \"" << str << "\".\n"; + printf("%s", msg.GetString().c_str()); + fflush(stdout); + return false; + } + + // Is the parsed value in the range of an Int32? + const Int32 result = static_cast(long_value); + if (long_value == LONG_MAX || long_value == LONG_MIN || + // The parsed value overflows as a long. (strtol() returns + // LONG_MAX or LONG_MIN when the input overflows.) + result != long_value + // The parsed value overflows as an Int32. + ) { + Message msg; + msg << "WARNING: " << src_text + << " is expected to be a 32-bit integer, but actually" + << " has value " << str << ", which overflows.\n"; + printf("%s", msg.GetString().c_str()); + fflush(stdout); + return false; + } + + *value = result; + return true; +} + +// Reads and returns the Boolean environment variable corresponding to +// the given flag; if it's not set, returns default_value. +// +// The value is considered true iff it's not "0". +bool BoolFromGTestEnv(const char* flag, bool default_value) { +#if defined(GTEST_GET_BOOL_FROM_ENV_) + return GTEST_GET_BOOL_FROM_ENV_(flag, default_value); +#endif // defined(GTEST_GET_BOOL_FROM_ENV_) + const std::string env_var = FlagToEnvVar(flag); + const char* const string_value = posix::GetEnv(env_var.c_str()); + return string_value == NULL ? + default_value : strcmp(string_value, "0") != 0; +} + +// Reads and returns a 32-bit integer stored in the environment +// variable corresponding to the given flag; if it isn't set or +// doesn't represent a valid 32-bit integer, returns default_value. +Int32 Int32FromGTestEnv(const char* flag, Int32 default_value) { +#if defined(GTEST_GET_INT32_FROM_ENV_) + return GTEST_GET_INT32_FROM_ENV_(flag, default_value); +#endif // defined(GTEST_GET_INT32_FROM_ENV_) + const std::string env_var = FlagToEnvVar(flag); + const char* const string_value = posix::GetEnv(env_var.c_str()); + if (string_value == NULL) { + // The environment variable is not set. + return default_value; + } + + Int32 result = default_value; + if (!ParseInt32(Message() << "Environment variable " << env_var, + string_value, &result)) { + printf("The default value %s is used.\n", + (Message() << default_value).GetString().c_str()); + fflush(stdout); + return default_value; + } + + return result; +} + +// Reads and returns the string environment variable corresponding to +// the given flag; if it's not set, returns default_value. +std::string StringFromGTestEnv(const char* flag, const char* default_value) { +#if defined(GTEST_GET_STRING_FROM_ENV_) + return GTEST_GET_STRING_FROM_ENV_(flag, default_value); +#endif // defined(GTEST_GET_STRING_FROM_ENV_) + const std::string env_var = FlagToEnvVar(flag); + const char* value = posix::GetEnv(env_var.c_str()); + if (value != NULL) { + return value; + } + + // As a special case for the 'output' flag, if GTEST_OUTPUT is not + // set, we look for XML_OUTPUT_FILE, which is set by the Bazel build + // system. The value of XML_OUTPUT_FILE is a filename without the + // "xml:" prefix of GTEST_OUTPUT. + // + // The net priority order after flag processing is thus: + // --gtest_output command line flag + // GTEST_OUTPUT environment variable + // XML_OUTPUT_FILE environment variable + // 'default_value' + if (strcmp(flag, "output") == 0) { + value = posix::GetEnv("XML_OUTPUT_FILE"); + if (value != NULL) { + return std::string("xml:") + value; + } + } + return default_value; +} + +} // namespace internal +} // namespace testing diff --git a/third_party/aom/third_party/googletest/src/googletest/src/gtest-printers.cc b/third_party/aom/third_party/googletest/src/googletest/src/gtest-printers.cc new file mode 100644 index 0000000000..a2df412f8a --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/src/gtest-printers.cc @@ -0,0 +1,373 @@ +// Copyright 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Google Test - The Google C++ Testing Framework +// +// This file implements a universal value printer that can print a +// value of any type T: +// +// void ::testing::internal::UniversalPrinter::Print(value, ostream_ptr); +// +// It uses the << operator when possible, and prints the bytes in the +// object otherwise. A user can override its behavior for a class +// type Foo by defining either operator<<(::std::ostream&, const Foo&) +// or void PrintTo(const Foo&, ::std::ostream*) in the namespace that +// defines Foo. + +#include "gtest/gtest-printers.h" +#include +#include +#include +#include // NOLINT +#include +#include "gtest/internal/gtest-port.h" + +namespace testing { + +namespace { + +using ::std::ostream; + +// Prints a segment of bytes in the given object. +GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +void PrintByteSegmentInObjectTo(const unsigned char* obj_bytes, size_t start, + size_t count, ostream* os) { + char text[5] = ""; + for (size_t i = 0; i != count; i++) { + const size_t j = start + i; + if (i != 0) { + // Organizes the bytes into groups of 2 for easy parsing by + // human. + if ((j % 2) == 0) + *os << ' '; + else + *os << '-'; + } + GTEST_SNPRINTF_(text, sizeof(text), "%02X", obj_bytes[j]); + *os << text; + } +} + +// Prints the bytes in the given value to the given ostream. +void PrintBytesInObjectToImpl(const unsigned char* obj_bytes, size_t count, + ostream* os) { + // Tells the user how big the object is. + *os << count << "-byte object <"; + + const size_t kThreshold = 132; + const size_t kChunkSize = 64; + // If the object size is bigger than kThreshold, we'll have to omit + // some details by printing only the first and the last kChunkSize + // bytes. + // TODO(wan): let the user control the threshold using a flag. + if (count < kThreshold) { + PrintByteSegmentInObjectTo(obj_bytes, 0, count, os); + } else { + PrintByteSegmentInObjectTo(obj_bytes, 0, kChunkSize, os); + *os << " ... "; + // Rounds up to 2-byte boundary. + const size_t resume_pos = (count - kChunkSize + 1)/2*2; + PrintByteSegmentInObjectTo(obj_bytes, resume_pos, count - resume_pos, os); + } + *os << ">"; +} + +} // namespace + +namespace internal2 { + +// Delegates to PrintBytesInObjectToImpl() to print the bytes in the +// given object. The delegation simplifies the implementation, which +// uses the << operator and thus is easier done outside of the +// ::testing::internal namespace, which contains a << operator that +// sometimes conflicts with the one in STL. +void PrintBytesInObjectTo(const unsigned char* obj_bytes, size_t count, + ostream* os) { + PrintBytesInObjectToImpl(obj_bytes, count, os); +} + +} // namespace internal2 + +namespace internal { + +// Depending on the value of a char (or wchar_t), we print it in one +// of three formats: +// - as is if it's a printable ASCII (e.g. 'a', '2', ' '), +// - as a hexidecimal escape sequence (e.g. '\x7F'), or +// - as a special escape sequence (e.g. '\r', '\n'). +enum CharFormat { + kAsIs, + kHexEscape, + kSpecialEscape +}; + +// Returns true if c is a printable ASCII character. We test the +// value of c directly instead of calling isprint(), which is buggy on +// Windows Mobile. +inline bool IsPrintableAscii(wchar_t c) { + return 0x20 <= c && c <= 0x7E; +} + +// Prints a wide or narrow char c as a character literal without the +// quotes, escaping it when necessary; returns how c was formatted. +// The template argument UnsignedChar is the unsigned version of Char, +// which is the type of c. +template +static CharFormat PrintAsCharLiteralTo(Char c, ostream* os) { + switch (static_cast(c)) { + case L'\0': + *os << "\\0"; + break; + case L'\'': + *os << "\\'"; + break; + case L'\\': + *os << "\\\\"; + break; + case L'\a': + *os << "\\a"; + break; + case L'\b': + *os << "\\b"; + break; + case L'\f': + *os << "\\f"; + break; + case L'\n': + *os << "\\n"; + break; + case L'\r': + *os << "\\r"; + break; + case L'\t': + *os << "\\t"; + break; + case L'\v': + *os << "\\v"; + break; + default: + if (IsPrintableAscii(c)) { + *os << static_cast(c); + return kAsIs; + } else { + *os << "\\x" + String::FormatHexInt(static_cast(c)); + return kHexEscape; + } + } + return kSpecialEscape; +} + +// Prints a wchar_t c as if it's part of a string literal, escaping it when +// necessary; returns how c was formatted. +static CharFormat PrintAsStringLiteralTo(wchar_t c, ostream* os) { + switch (c) { + case L'\'': + *os << "'"; + return kAsIs; + case L'"': + *os << "\\\""; + return kSpecialEscape; + default: + return PrintAsCharLiteralTo(c, os); + } +} + +// Prints a char c as if it's part of a string literal, escaping it when +// necessary; returns how c was formatted. +static CharFormat PrintAsStringLiteralTo(char c, ostream* os) { + return PrintAsStringLiteralTo( + static_cast(static_cast(c)), os); +} + +// Prints a wide or narrow character c and its code. '\0' is printed +// as "'\\0'", other unprintable characters are also properly escaped +// using the standard C++ escape sequence. The template argument +// UnsignedChar is the unsigned version of Char, which is the type of c. +template +void PrintCharAndCodeTo(Char c, ostream* os) { + // First, print c as a literal in the most readable form we can find. + *os << ((sizeof(c) > 1) ? "L'" : "'"); + const CharFormat format = PrintAsCharLiteralTo(c, os); + *os << "'"; + + // To aid user debugging, we also print c's code in decimal, unless + // it's 0 (in which case c was printed as '\\0', making the code + // obvious). + if (c == 0) + return; + *os << " (" << static_cast(c); + + // For more convenience, we print c's code again in hexidecimal, + // unless c was already printed in the form '\x##' or the code is in + // [1, 9]. + if (format == kHexEscape || (1 <= c && c <= 9)) { + // Do nothing. + } else { + *os << ", 0x" << String::FormatHexInt(static_cast(c)); + } + *os << ")"; +} + +void PrintTo(unsigned char c, ::std::ostream* os) { + PrintCharAndCodeTo(c, os); +} +void PrintTo(signed char c, ::std::ostream* os) { + PrintCharAndCodeTo(c, os); +} + +// Prints a wchar_t as a symbol if it is printable or as its internal +// code otherwise and also as its code. L'\0' is printed as "L'\\0'". +void PrintTo(wchar_t wc, ostream* os) { + PrintCharAndCodeTo(wc, os); +} + +// Prints the given array of characters to the ostream. CharType must be either +// char or wchar_t. +// The array starts at begin, the length is len, it may include '\0' characters +// and may not be NUL-terminated. +template +GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +static void PrintCharsAsStringTo( + const CharType* begin, size_t len, ostream* os) { + const char* const kQuoteBegin = sizeof(CharType) == 1 ? "\"" : "L\""; + *os << kQuoteBegin; + bool is_previous_hex = false; + for (size_t index = 0; index < len; ++index) { + const CharType cur = begin[index]; + if (is_previous_hex && IsXDigit(cur)) { + // Previous character is of '\x..' form and this character can be + // interpreted as another hexadecimal digit in its number. Break string to + // disambiguate. + *os << "\" " << kQuoteBegin; + } + is_previous_hex = PrintAsStringLiteralTo(cur, os) == kHexEscape; + } + *os << "\""; +} + +// Prints a (const) char/wchar_t array of 'len' elements, starting at address +// 'begin'. CharType must be either char or wchar_t. +template +GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +static void UniversalPrintCharArray( + const CharType* begin, size_t len, ostream* os) { + // The code + // const char kFoo[] = "foo"; + // generates an array of 4, not 3, elements, with the last one being '\0'. + // + // Therefore when printing a char array, we don't print the last element if + // it's '\0', such that the output matches the string literal as it's + // written in the source code. + if (len > 0 && begin[len - 1] == '\0') { + PrintCharsAsStringTo(begin, len - 1, os); + return; + } + + // If, however, the last element in the array is not '\0', e.g. + // const char kFoo[] = { 'f', 'o', 'o' }; + // we must print the entire array. We also print a message to indicate + // that the array is not NUL-terminated. + PrintCharsAsStringTo(begin, len, os); + *os << " (no terminating NUL)"; +} + +// Prints a (const) char array of 'len' elements, starting at address 'begin'. +void UniversalPrintArray(const char* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); +} + +// Prints a (const) wchar_t array of 'len' elements, starting at address +// 'begin'. +void UniversalPrintArray(const wchar_t* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); +} + +// Prints the given C string to the ostream. +void PrintTo(const char* s, ostream* os) { + if (s == NULL) { + *os << "NULL"; + } else { + *os << ImplicitCast_(s) << " pointing to "; + PrintCharsAsStringTo(s, strlen(s), os); + } +} + +// MSVC compiler can be configured to define whar_t as a typedef +// of unsigned short. Defining an overload for const wchar_t* in that case +// would cause pointers to unsigned shorts be printed as wide strings, +// possibly accessing more memory than intended and causing invalid +// memory accesses. MSVC defines _NATIVE_WCHAR_T_DEFINED symbol when +// wchar_t is implemented as a native type. +#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) +// Prints the given wide C string to the ostream. +void PrintTo(const wchar_t* s, ostream* os) { + if (s == NULL) { + *os << "NULL"; + } else { + *os << ImplicitCast_(s) << " pointing to "; + PrintCharsAsStringTo(s, std::wcslen(s), os); + } +} +#endif // wchar_t is native + +// Prints a ::string object. +#if GTEST_HAS_GLOBAL_STRING +void PrintStringTo(const ::string& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} +#endif // GTEST_HAS_GLOBAL_STRING + +void PrintStringTo(const ::std::string& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} + +// Prints a ::wstring object. +#if GTEST_HAS_GLOBAL_WSTRING +void PrintWideStringTo(const ::wstring& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} +#endif // GTEST_HAS_GLOBAL_WSTRING + +#if GTEST_HAS_STD_WSTRING +void PrintWideStringTo(const ::std::wstring& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} +#endif // GTEST_HAS_STD_WSTRING + +} // namespace internal + +} // namespace testing diff --git a/third_party/aom/third_party/googletest/src/googletest/src/gtest-test-part.cc b/third_party/aom/third_party/googletest/src/googletest/src/gtest-test-part.cc new file mode 100644 index 0000000000..fb0e35425e --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/src/gtest-test-part.cc @@ -0,0 +1,110 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: mheule@google.com (Markus Heule) +// +// The Google C++ Testing Framework (Google Test) + +#include "gtest/gtest-test-part.h" + +// Indicates that this translation unit is part of Google Test's +// implementation. It must come before gtest-internal-inl.h is +// included, or there will be a compiler error. This trick exists to +// prevent the accidental inclusion of gtest-internal-inl.h in the +// user's code. +#define GTEST_IMPLEMENTATION_ 1 +#include "src/gtest-internal-inl.h" +#undef GTEST_IMPLEMENTATION_ + +namespace testing { + +using internal::GetUnitTestImpl; + +// Gets the summary of the failure message by omitting the stack trace +// in it. +std::string TestPartResult::ExtractSummary(const char* message) { + const char* const stack_trace = strstr(message, internal::kStackTraceMarker); + return stack_trace == NULL ? message : + std::string(message, stack_trace); +} + +// Prints a TestPartResult object. +std::ostream& operator<<(std::ostream& os, const TestPartResult& result) { + return os + << result.file_name() << ":" << result.line_number() << ": " + << (result.type() == TestPartResult::kSuccess ? "Success" : + result.type() == TestPartResult::kFatalFailure ? "Fatal failure" : + "Non-fatal failure") << ":\n" + << result.message() << std::endl; +} + +// Appends a TestPartResult to the array. +void TestPartResultArray::Append(const TestPartResult& result) { + array_.push_back(result); +} + +// Returns the TestPartResult at the given index (0-based). +const TestPartResult& TestPartResultArray::GetTestPartResult(int index) const { + if (index < 0 || index >= size()) { + printf("\nInvalid index (%d) into TestPartResultArray.\n", index); + internal::posix::Abort(); + } + + return array_[index]; +} + +// Returns the number of TestPartResult objects in the array. +int TestPartResultArray::size() const { + return static_cast(array_.size()); +} + +namespace internal { + +HasNewFatalFailureHelper::HasNewFatalFailureHelper() + : has_new_fatal_failure_(false), + original_reporter_(GetUnitTestImpl()-> + GetTestPartResultReporterForCurrentThread()) { + GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread(this); +} + +HasNewFatalFailureHelper::~HasNewFatalFailureHelper() { + GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread( + original_reporter_); +} + +void HasNewFatalFailureHelper::ReportTestPartResult( + const TestPartResult& result) { + if (result.fatally_failed()) + has_new_fatal_failure_ = true; + original_reporter_->ReportTestPartResult(result); +} + +} // namespace internal + +} // namespace testing diff --git a/third_party/aom/third_party/googletest/src/googletest/src/gtest-typed-test.cc b/third_party/aom/third_party/googletest/src/googletest/src/gtest-typed-test.cc new file mode 100644 index 0000000000..df1eef4754 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/src/gtest-typed-test.cc @@ -0,0 +1,118 @@ +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +#include "gtest/gtest-typed-test.h" +#include "gtest/gtest.h" + +namespace testing { +namespace internal { + +#if GTEST_HAS_TYPED_TEST_P + +// Skips to the first non-space char in str. Returns an empty string if str +// contains only whitespace characters. +static const char* SkipSpaces(const char* str) { + while (IsSpace(*str)) + str++; + return str; +} + +static std::vector SplitIntoTestNames(const char* src) { + std::vector name_vec; + src = SkipSpaces(src); + for (; src != NULL; src = SkipComma(src)) { + name_vec.push_back(StripTrailingSpaces(GetPrefixUntilComma(src))); + } + return name_vec; +} + +// Verifies that registered_tests match the test names in +// registered_tests_; returns registered_tests if successful, or +// aborts the program otherwise. +const char* TypedTestCasePState::VerifyRegisteredTestNames( + const char* file, int line, const char* registered_tests) { + typedef RegisteredTestsMap::const_iterator RegisteredTestIter; + registered_ = true; + + std::vector name_vec = SplitIntoTestNames(registered_tests); + + Message errors; + + std::set tests; + for (std::vector::const_iterator name_it = name_vec.begin(); + name_it != name_vec.end(); ++name_it) { + const std::string& name = *name_it; + if (tests.count(name) != 0) { + errors << "Test " << name << " is listed more than once.\n"; + continue; + } + + bool found = false; + for (RegisteredTestIter it = registered_tests_.begin(); + it != registered_tests_.end(); + ++it) { + if (name == it->first) { + found = true; + break; + } + } + + if (found) { + tests.insert(name); + } else { + errors << "No test named " << name + << " can be found in this test case.\n"; + } + } + + for (RegisteredTestIter it = registered_tests_.begin(); + it != registered_tests_.end(); + ++it) { + if (tests.count(it->first) == 0) { + errors << "You forgot to list test " << it->first << ".\n"; + } + } + + const std::string& errors_str = errors.GetString(); + if (errors_str != "") { + fprintf(stderr, "%s %s", FormatFileLocation(file, line).c_str(), + errors_str.c_str()); + fflush(stderr); + posix::Abort(); + } + + return registered_tests; +} + +#endif // GTEST_HAS_TYPED_TEST_P + +} // namespace internal +} // namespace testing diff --git a/third_party/aom/third_party/googletest/src/googletest/src/gtest.cc b/third_party/aom/third_party/googletest/src/googletest/src/gtest.cc new file mode 100644 index 0000000000..d882ab2e36 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/src/gtest.cc @@ -0,0 +1,5388 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) + +#include "gtest/gtest.h" +#include "gtest/internal/custom/gtest.h" +#include "gtest/gtest-spi.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include // NOLINT +#include +#include + +#if GTEST_OS_LINUX + +// TODO(kenton@google.com): Use autoconf to detect availability of +// gettimeofday(). +# define GTEST_HAS_GETTIMEOFDAY_ 1 + +# include // NOLINT +# include // NOLINT +# include // NOLINT +// Declares vsnprintf(). This header is not available on Windows. +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include + +#elif GTEST_OS_SYMBIAN +# define GTEST_HAS_GETTIMEOFDAY_ 1 +# include // NOLINT + +#elif GTEST_OS_ZOS +# define GTEST_HAS_GETTIMEOFDAY_ 1 +# include // NOLINT + +// On z/OS we additionally need strings.h for strcasecmp. +# include // NOLINT + +#elif GTEST_OS_WINDOWS_MOBILE // We are on Windows CE. + +# include // NOLINT +# undef min + +#elif GTEST_OS_WINDOWS // We are on Windows proper. + +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include // NOLINT + +# if GTEST_OS_WINDOWS_MINGW +// MinGW has gettimeofday() but not _ftime64(). +// TODO(kenton@google.com): Use autoconf to detect availability of +// gettimeofday(). +// TODO(kenton@google.com): There are other ways to get the time on +// Windows, like GetTickCount() or GetSystemTimeAsFileTime(). MinGW +// supports these. consider using them instead. +# define GTEST_HAS_GETTIMEOFDAY_ 1 +# include // NOLINT +# endif // GTEST_OS_WINDOWS_MINGW + +// cpplint thinks that the header is already included, so we want to +// silence it. +# include // NOLINT +# undef min + +#else + +// Assume other platforms have gettimeofday(). +// TODO(kenton@google.com): Use autoconf to detect availability of +// gettimeofday(). +# define GTEST_HAS_GETTIMEOFDAY_ 1 + +// cpplint thinks that the header is already included, so we want to +// silence it. +# include // NOLINT +# include // NOLINT + +#endif // GTEST_OS_LINUX + +#if GTEST_HAS_EXCEPTIONS +# include +#endif + +#if GTEST_CAN_STREAM_RESULTS_ +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include // NOLINT +#endif + +// Indicates that this translation unit is part of Google Test's +// implementation. It must come before gtest-internal-inl.h is +// included, or there will be a compiler error. This trick is to +// prevent a user from accidentally including gtest-internal-inl.h in +// his code. +#define GTEST_IMPLEMENTATION_ 1 +#include "src/gtest-internal-inl.h" +#undef GTEST_IMPLEMENTATION_ + +#if GTEST_OS_WINDOWS +# define vsnprintf _vsnprintf +#endif // GTEST_OS_WINDOWS + +namespace testing { + +using internal::CountIf; +using internal::ForEach; +using internal::GetElementOr; +using internal::Shuffle; + +// Constants. + +// A test whose test case name or test name matches this filter is +// disabled and not run. +static const char kDisableTestFilter[] = "DISABLED_*:*/DISABLED_*"; + +// A test case whose name matches this filter is considered a death +// test case and will be run before test cases whose name doesn't +// match this filter. +static const char kDeathTestCaseFilter[] = "*DeathTest:*DeathTest/*"; + +// A test filter that matches everything. +static const char kUniversalFilter[] = "*"; + +// The default output file for XML output. +static const char kDefaultOutputFile[] = "test_detail.xml"; + +// The environment variable name for the test shard index. +static const char kTestShardIndex[] = "GTEST_SHARD_INDEX"; +// The environment variable name for the total number of test shards. +static const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS"; +// The environment variable name for the test shard status file. +static const char kTestShardStatusFile[] = "GTEST_SHARD_STATUS_FILE"; + +namespace internal { + +// The text used in failure messages to indicate the start of the +// stack trace. +const char kStackTraceMarker[] = "\nStack trace:\n"; + +// g_help_flag is true iff the --help flag or an equivalent form is +// specified on the command line. +bool g_help_flag = false; + +} // namespace internal + +static const char* GetDefaultFilter() { +#ifdef GTEST_TEST_FILTER_ENV_VAR_ + const char* const testbridge_test_only = getenv(GTEST_TEST_FILTER_ENV_VAR_); + if (testbridge_test_only != NULL) { + return testbridge_test_only; + } +#endif // GTEST_TEST_FILTER_ENV_VAR_ + return kUniversalFilter; +} + +GTEST_DEFINE_bool_( + also_run_disabled_tests, + internal::BoolFromGTestEnv("also_run_disabled_tests", false), + "Run disabled tests too, in addition to the tests normally being run."); + +GTEST_DEFINE_bool_( + break_on_failure, + internal::BoolFromGTestEnv("break_on_failure", false), + "True iff a failed assertion should be a debugger break-point."); + +GTEST_DEFINE_bool_( + catch_exceptions, + internal::BoolFromGTestEnv("catch_exceptions", true), + "True iff " GTEST_NAME_ + " should catch exceptions and treat them as test failures."); + +GTEST_DEFINE_string_( + color, + internal::StringFromGTestEnv("color", "auto"), + "Whether to use colors in the output. Valid values: yes, no, " + "and auto. 'auto' means to use colors if the output is " + "being sent to a terminal and the TERM environment variable " + "is set to a terminal type that supports colors."); + +GTEST_DEFINE_string_( + filter, + internal::StringFromGTestEnv("filter", GetDefaultFilter()), + "A colon-separated list of glob (not regex) patterns " + "for filtering the tests to run, optionally followed by a " + "'-' and a : separated list of negative patterns (tests to " + "exclude). A test is run if it matches one of the positive " + "patterns and does not match any of the negative patterns."); + +GTEST_DEFINE_bool_(list_tests, false, + "List all tests without running them."); + +GTEST_DEFINE_string_( + output, + internal::StringFromGTestEnv("output", ""), + "A format (currently must be \"xml\"), optionally followed " + "by a colon and an output file name or directory. A directory " + "is indicated by a trailing pathname separator. " + "Examples: \"xml:filename.xml\", \"xml::directoryname/\". " + "If a directory is specified, output files will be created " + "within that directory, with file-names based on the test " + "executable's name and, if necessary, made unique by adding " + "digits."); + +GTEST_DEFINE_bool_( + print_time, + internal::BoolFromGTestEnv("print_time", true), + "True iff " GTEST_NAME_ + " should display elapsed time in text output."); + +GTEST_DEFINE_int32_( + random_seed, + internal::Int32FromGTestEnv("random_seed", 0), + "Random number seed to use when shuffling test orders. Must be in range " + "[1, 99999], or 0 to use a seed based on the current time."); + +GTEST_DEFINE_int32_( + repeat, + internal::Int32FromGTestEnv("repeat", 1), + "How many times to repeat each test. Specify a negative number " + "for repeating forever. Useful for shaking out flaky tests."); + +GTEST_DEFINE_bool_( + show_internal_stack_frames, false, + "True iff " GTEST_NAME_ " should include internal stack frames when " + "printing test failure stack traces."); + +GTEST_DEFINE_bool_( + shuffle, + internal::BoolFromGTestEnv("shuffle", false), + "True iff " GTEST_NAME_ + " should randomize tests' order on every run."); + +GTEST_DEFINE_int32_( + stack_trace_depth, + internal::Int32FromGTestEnv("stack_trace_depth", kMaxStackTraceDepth), + "The maximum number of stack frames to print when an " + "assertion fails. The valid range is 0 through 100, inclusive."); + +GTEST_DEFINE_string_( + stream_result_to, + internal::StringFromGTestEnv("stream_result_to", ""), + "This flag specifies the host name and the port number on which to stream " + "test results. Example: \"localhost:555\". The flag is effective only on " + "Linux."); + +GTEST_DEFINE_bool_( + throw_on_failure, + internal::BoolFromGTestEnv("throw_on_failure", false), + "When this flag is specified, a failed assertion will throw an exception " + "if exceptions are enabled or exit the program with a non-zero code " + "otherwise."); + +#if GTEST_USE_OWN_FLAGFILE_FLAG_ +GTEST_DEFINE_string_( + flagfile, + internal::StringFromGTestEnv("flagfile", ""), + "This flag specifies the flagfile to read command-line flags from."); +#endif // GTEST_USE_OWN_FLAGFILE_FLAG_ + +namespace internal { + +// Generates a random number from [0, range), using a Linear +// Congruential Generator (LCG). Crashes if 'range' is 0 or greater +// than kMaxRange. +UInt32 Random::Generate(UInt32 range) { + // These constants are the same as are used in glibc's rand(3). + state_ = (1103515245U*state_ + 12345U) % kMaxRange; + + GTEST_CHECK_(range > 0) + << "Cannot generate a number in the range [0, 0)."; + GTEST_CHECK_(range <= kMaxRange) + << "Generation of a number in [0, " << range << ") was requested, " + << "but this can only generate numbers in [0, " << kMaxRange << ")."; + + // Converting via modulus introduces a bit of downward bias, but + // it's simple, and a linear congruential generator isn't too good + // to begin with. + return state_ % range; +} + +// GTestIsInitialized() returns true iff the user has initialized +// Google Test. Useful for catching the user mistake of not initializing +// Google Test before calling RUN_ALL_TESTS(). +static bool GTestIsInitialized() { return GetArgvs().size() > 0; } + +// Iterates over a vector of TestCases, keeping a running sum of the +// results of calling a given int-returning method on each. +// Returns the sum. +static int SumOverTestCaseList(const std::vector& case_list, + int (TestCase::*method)() const) { + int sum = 0; + for (size_t i = 0; i < case_list.size(); i++) { + sum += (case_list[i]->*method)(); + } + return sum; +} + +// Returns true iff the test case passed. +static bool TestCasePassed(const TestCase* test_case) { + return test_case->should_run() && test_case->Passed(); +} + +// Returns true iff the test case failed. +static bool TestCaseFailed(const TestCase* test_case) { + return test_case->should_run() && test_case->Failed(); +} + +// Returns true iff test_case contains at least one test that should +// run. +static bool ShouldRunTestCase(const TestCase* test_case) { + return test_case->should_run(); +} + +// AssertHelper constructor. +AssertHelper::AssertHelper(TestPartResult::Type type, + const char* file, + int line, + const char* message) + : data_(new AssertHelperData(type, file, line, message)) { +} + +AssertHelper::~AssertHelper() { + delete data_; +} + +// Message assignment, for assertion streaming support. +void AssertHelper::operator=(const Message& message) const { + UnitTest::GetInstance()-> + AddTestPartResult(data_->type, data_->file, data_->line, + AppendUserMessage(data_->message, message), + UnitTest::GetInstance()->impl() + ->CurrentOsStackTraceExceptTop(1) + // Skips the stack frame for this function itself. + ); // NOLINT +} + +// Mutex for linked pointers. +GTEST_API_ GTEST_DEFINE_STATIC_MUTEX_(g_linked_ptr_mutex); + +// A copy of all command line arguments. Set by InitGoogleTest(). +::std::vector g_argvs; + +const ::std::vector& GetArgvs() { +#if defined(GTEST_CUSTOM_GET_ARGVS_) + return GTEST_CUSTOM_GET_ARGVS_(); +#else // defined(GTEST_CUSTOM_GET_ARGVS_) + return g_argvs; +#endif // defined(GTEST_CUSTOM_GET_ARGVS_) +} + +// Returns the current application's name, removing directory path if that +// is present. +FilePath GetCurrentExecutableName() { + FilePath result; + +#if GTEST_OS_WINDOWS + result.Set(FilePath(GetArgvs()[0]).RemoveExtension("exe")); +#else + result.Set(FilePath(GetArgvs()[0])); +#endif // GTEST_OS_WINDOWS + + return result.RemoveDirectoryName(); +} + +// Functions for processing the gtest_output flag. + +// Returns the output format, or "" for normal printed output. +std::string UnitTestOptions::GetOutputFormat() { + const char* const gtest_output_flag = GTEST_FLAG(output).c_str(); + if (gtest_output_flag == NULL) return std::string(""); + + const char* const colon = strchr(gtest_output_flag, ':'); + return (colon == NULL) ? + std::string(gtest_output_flag) : + std::string(gtest_output_flag, colon - gtest_output_flag); +} + +// Returns the name of the requested output file, or the default if none +// was explicitly specified. +std::string UnitTestOptions::GetAbsolutePathToOutputFile() { + const char* const gtest_output_flag = GTEST_FLAG(output).c_str(); + if (gtest_output_flag == NULL) + return ""; + + const char* const colon = strchr(gtest_output_flag, ':'); + if (colon == NULL) + return internal::FilePath::ConcatPaths( + internal::FilePath( + UnitTest::GetInstance()->original_working_dir()), + internal::FilePath(kDefaultOutputFile)).string(); + + internal::FilePath output_name(colon + 1); + if (!output_name.IsAbsolutePath()) + // TODO(wan@google.com): on Windows \some\path is not an absolute + // path (as its meaning depends on the current drive), yet the + // following logic for turning it into an absolute path is wrong. + // Fix it. + output_name = internal::FilePath::ConcatPaths( + internal::FilePath(UnitTest::GetInstance()->original_working_dir()), + internal::FilePath(colon + 1)); + + if (!output_name.IsDirectory()) + return output_name.string(); + + internal::FilePath result(internal::FilePath::GenerateUniqueFileName( + output_name, internal::GetCurrentExecutableName(), + GetOutputFormat().c_str())); + return result.string(); +} + +// Returns true iff the wildcard pattern matches the string. The +// first ':' or '\0' character in pattern marks the end of it. +// +// This recursive algorithm isn't very efficient, but is clear and +// works well enough for matching test names, which are short. +bool UnitTestOptions::PatternMatchesString(const char *pattern, + const char *str) { + switch (*pattern) { + case '\0': + case ':': // Either ':' or '\0' marks the end of the pattern. + return *str == '\0'; + case '?': // Matches any single character. + return *str != '\0' && PatternMatchesString(pattern + 1, str + 1); + case '*': // Matches any string (possibly empty) of characters. + return (*str != '\0' && PatternMatchesString(pattern, str + 1)) || + PatternMatchesString(pattern + 1, str); + default: // Non-special character. Matches itself. + return *pattern == *str && + PatternMatchesString(pattern + 1, str + 1); + } +} + +bool UnitTestOptions::MatchesFilter( + const std::string& name, const char* filter) { + const char *cur_pattern = filter; + for (;;) { + if (PatternMatchesString(cur_pattern, name.c_str())) { + return true; + } + + // Finds the next pattern in the filter. + cur_pattern = strchr(cur_pattern, ':'); + + // Returns if no more pattern can be found. + if (cur_pattern == NULL) { + return false; + } + + // Skips the pattern separater (the ':' character). + cur_pattern++; + } +} + +// Returns true iff the user-specified filter matches the test case +// name and the test name. +bool UnitTestOptions::FilterMatchesTest(const std::string &test_case_name, + const std::string &test_name) { + const std::string& full_name = test_case_name + "." + test_name.c_str(); + + // Split --gtest_filter at '-', if there is one, to separate into + // positive filter and negative filter portions + const char* const p = GTEST_FLAG(filter).c_str(); + const char* const dash = strchr(p, '-'); + std::string positive; + std::string negative; + if (dash == NULL) { + positive = GTEST_FLAG(filter).c_str(); // Whole string is a positive filter + negative = ""; + } else { + positive = std::string(p, dash); // Everything up to the dash + negative = std::string(dash + 1); // Everything after the dash + if (positive.empty()) { + // Treat '-test1' as the same as '*-test1' + positive = kUniversalFilter; + } + } + + // A filter is a colon-separated list of patterns. It matches a + // test if any pattern in it matches the test. + return (MatchesFilter(full_name, positive.c_str()) && + !MatchesFilter(full_name, negative.c_str())); +} + +#if GTEST_HAS_SEH +// Returns EXCEPTION_EXECUTE_HANDLER if Google Test should handle the +// given SEH exception, or EXCEPTION_CONTINUE_SEARCH otherwise. +// This function is useful as an __except condition. +int UnitTestOptions::GTestShouldProcessSEH(DWORD exception_code) { + // Google Test should handle a SEH exception if: + // 1. the user wants it to, AND + // 2. this is not a breakpoint exception, AND + // 3. this is not a C++ exception (VC++ implements them via SEH, + // apparently). + // + // SEH exception code for C++ exceptions. + // (see http://support.microsoft.com/kb/185294 for more information). + const DWORD kCxxExceptionCode = 0xe06d7363; + + bool should_handle = true; + + if (!GTEST_FLAG(catch_exceptions)) + should_handle = false; + else if (exception_code == EXCEPTION_BREAKPOINT) + should_handle = false; + else if (exception_code == kCxxExceptionCode) + should_handle = false; + + return should_handle ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH; +} +#endif // GTEST_HAS_SEH + +} // namespace internal + +// The c'tor sets this object as the test part result reporter used by +// Google Test. The 'result' parameter specifies where to report the +// results. Intercepts only failures from the current thread. +ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter( + TestPartResultArray* result) + : intercept_mode_(INTERCEPT_ONLY_CURRENT_THREAD), + result_(result) { + Init(); +} + +// The c'tor sets this object as the test part result reporter used by +// Google Test. The 'result' parameter specifies where to report the +// results. +ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter( + InterceptMode intercept_mode, TestPartResultArray* result) + : intercept_mode_(intercept_mode), + result_(result) { + Init(); +} + +void ScopedFakeTestPartResultReporter::Init() { + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + if (intercept_mode_ == INTERCEPT_ALL_THREADS) { + old_reporter_ = impl->GetGlobalTestPartResultReporter(); + impl->SetGlobalTestPartResultReporter(this); + } else { + old_reporter_ = impl->GetTestPartResultReporterForCurrentThread(); + impl->SetTestPartResultReporterForCurrentThread(this); + } +} + +// The d'tor restores the test part result reporter used by Google Test +// before. +ScopedFakeTestPartResultReporter::~ScopedFakeTestPartResultReporter() { + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + if (intercept_mode_ == INTERCEPT_ALL_THREADS) { + impl->SetGlobalTestPartResultReporter(old_reporter_); + } else { + impl->SetTestPartResultReporterForCurrentThread(old_reporter_); + } +} + +// Increments the test part result count and remembers the result. +// This method is from the TestPartResultReporterInterface interface. +void ScopedFakeTestPartResultReporter::ReportTestPartResult( + const TestPartResult& result) { + result_->Append(result); +} + +namespace internal { + +// Returns the type ID of ::testing::Test. We should always call this +// instead of GetTypeId< ::testing::Test>() to get the type ID of +// testing::Test. This is to work around a suspected linker bug when +// using Google Test as a framework on Mac OS X. The bug causes +// GetTypeId< ::testing::Test>() to return different values depending +// on whether the call is from the Google Test framework itself or +// from user test code. GetTestTypeId() is guaranteed to always +// return the same value, as it always calls GetTypeId<>() from the +// gtest.cc, which is within the Google Test framework. +TypeId GetTestTypeId() { + return GetTypeId(); +} + +// The value of GetTestTypeId() as seen from within the Google Test +// library. This is solely for testing GetTestTypeId(). +extern const TypeId kTestTypeIdInGoogleTest = GetTestTypeId(); + +// This predicate-formatter checks that 'results' contains a test part +// failure of the given type and that the failure message contains the +// given substring. +AssertionResult HasOneFailure(const char* /* results_expr */, + const char* /* type_expr */, + const char* /* substr_expr */, + const TestPartResultArray& results, + TestPartResult::Type type, + const string& substr) { + const std::string expected(type == TestPartResult::kFatalFailure ? + "1 fatal failure" : + "1 non-fatal failure"); + Message msg; + if (results.size() != 1) { + msg << "Expected: " << expected << "\n" + << " Actual: " << results.size() << " failures"; + for (int i = 0; i < results.size(); i++) { + msg << "\n" << results.GetTestPartResult(i); + } + return AssertionFailure() << msg; + } + + const TestPartResult& r = results.GetTestPartResult(0); + if (r.type() != type) { + return AssertionFailure() << "Expected: " << expected << "\n" + << " Actual:\n" + << r; + } + + if (strstr(r.message(), substr.c_str()) == NULL) { + return AssertionFailure() << "Expected: " << expected << " containing \"" + << substr << "\"\n" + << " Actual:\n" + << r; + } + + return AssertionSuccess(); +} + +// The constructor of SingleFailureChecker remembers where to look up +// test part results, what type of failure we expect, and what +// substring the failure message should contain. +SingleFailureChecker:: SingleFailureChecker( + const TestPartResultArray* results, + TestPartResult::Type type, + const string& substr) + : results_(results), + type_(type), + substr_(substr) {} + +// The destructor of SingleFailureChecker verifies that the given +// TestPartResultArray contains exactly one failure that has the given +// type and contains the given substring. If that's not the case, a +// non-fatal failure will be generated. +SingleFailureChecker::~SingleFailureChecker() { + EXPECT_PRED_FORMAT3(HasOneFailure, *results_, type_, substr_); +} + +DefaultGlobalTestPartResultReporter::DefaultGlobalTestPartResultReporter( + UnitTestImpl* unit_test) : unit_test_(unit_test) {} + +void DefaultGlobalTestPartResultReporter::ReportTestPartResult( + const TestPartResult& result) { + unit_test_->current_test_result()->AddTestPartResult(result); + unit_test_->listeners()->repeater()->OnTestPartResult(result); +} + +DefaultPerThreadTestPartResultReporter::DefaultPerThreadTestPartResultReporter( + UnitTestImpl* unit_test) : unit_test_(unit_test) {} + +void DefaultPerThreadTestPartResultReporter::ReportTestPartResult( + const TestPartResult& result) { + unit_test_->GetGlobalTestPartResultReporter()->ReportTestPartResult(result); +} + +// Returns the global test part result reporter. +TestPartResultReporterInterface* +UnitTestImpl::GetGlobalTestPartResultReporter() { + internal::MutexLock lock(&global_test_part_result_reporter_mutex_); + return global_test_part_result_repoter_; +} + +// Sets the global test part result reporter. +void UnitTestImpl::SetGlobalTestPartResultReporter( + TestPartResultReporterInterface* reporter) { + internal::MutexLock lock(&global_test_part_result_reporter_mutex_); + global_test_part_result_repoter_ = reporter; +} + +// Returns the test part result reporter for the current thread. +TestPartResultReporterInterface* +UnitTestImpl::GetTestPartResultReporterForCurrentThread() { + return per_thread_test_part_result_reporter_.get(); +} + +// Sets the test part result reporter for the current thread. +void UnitTestImpl::SetTestPartResultReporterForCurrentThread( + TestPartResultReporterInterface* reporter) { + per_thread_test_part_result_reporter_.set(reporter); +} + +// Gets the number of successful test cases. +int UnitTestImpl::successful_test_case_count() const { + return CountIf(test_cases_, TestCasePassed); +} + +// Gets the number of failed test cases. +int UnitTestImpl::failed_test_case_count() const { + return CountIf(test_cases_, TestCaseFailed); +} + +// Gets the number of all test cases. +int UnitTestImpl::total_test_case_count() const { + return static_cast(test_cases_.size()); +} + +// Gets the number of all test cases that contain at least one test +// that should run. +int UnitTestImpl::test_case_to_run_count() const { + return CountIf(test_cases_, ShouldRunTestCase); +} + +// Gets the number of successful tests. +int UnitTestImpl::successful_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::successful_test_count); +} + +// Gets the number of failed tests. +int UnitTestImpl::failed_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::failed_test_count); +} + +// Gets the number of disabled tests that will be reported in the XML report. +int UnitTestImpl::reportable_disabled_test_count() const { + return SumOverTestCaseList(test_cases_, + &TestCase::reportable_disabled_test_count); +} + +// Gets the number of disabled tests. +int UnitTestImpl::disabled_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::disabled_test_count); +} + +// Gets the number of tests to be printed in the XML report. +int UnitTestImpl::reportable_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::reportable_test_count); +} + +// Gets the number of all tests. +int UnitTestImpl::total_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::total_test_count); +} + +// Gets the number of tests that should run. +int UnitTestImpl::test_to_run_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::test_to_run_count); +} + +// Returns the current OS stack trace as an std::string. +// +// The maximum number of stack frames to be included is specified by +// the gtest_stack_trace_depth flag. The skip_count parameter +// specifies the number of top frames to be skipped, which doesn't +// count against the number of frames to be included. +// +// For example, if Foo() calls Bar(), which in turn calls +// CurrentOsStackTraceExceptTop(1), Foo() will be included in the +// trace but Bar() and CurrentOsStackTraceExceptTop() won't. +std::string UnitTestImpl::CurrentOsStackTraceExceptTop(int skip_count) { + return os_stack_trace_getter()->CurrentStackTrace( + static_cast(GTEST_FLAG(stack_trace_depth)), + skip_count + 1 + // Skips the user-specified number of frames plus this function + // itself. + ); // NOLINT +} + +// Returns the current time in milliseconds. +TimeInMillis GetTimeInMillis() { +#if GTEST_OS_WINDOWS_MOBILE || defined(__BORLANDC__) + // Difference between 1970-01-01 and 1601-01-01 in milliseconds. + // http://analogous.blogspot.com/2005/04/epoch.html + const TimeInMillis kJavaEpochToWinFileTimeDelta = + static_cast(116444736UL) * 100000UL; + const DWORD kTenthMicrosInMilliSecond = 10000; + + SYSTEMTIME now_systime; + FILETIME now_filetime; + ULARGE_INTEGER now_int64; + // TODO(kenton@google.com): Shouldn't this just use + // GetSystemTimeAsFileTime()? + GetSystemTime(&now_systime); + if (SystemTimeToFileTime(&now_systime, &now_filetime)) { + now_int64.LowPart = now_filetime.dwLowDateTime; + now_int64.HighPart = now_filetime.dwHighDateTime; + now_int64.QuadPart = (now_int64.QuadPart / kTenthMicrosInMilliSecond) - + kJavaEpochToWinFileTimeDelta; + return now_int64.QuadPart; + } + return 0; +#elif GTEST_OS_WINDOWS && !GTEST_HAS_GETTIMEOFDAY_ + __timeb64 now; + + // MSVC 8 deprecates _ftime64(), so we want to suppress warning 4996 + // (deprecated function) there. + // TODO(kenton@google.com): Use GetTickCount()? Or use + // SystemTimeToFileTime() + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996) + _ftime64(&now); + GTEST_DISABLE_MSC_WARNINGS_POP_() + + return static_cast(now.time) * 1000 + now.millitm; +#elif GTEST_HAS_GETTIMEOFDAY_ + struct timeval now; + gettimeofday(&now, NULL); + return static_cast(now.tv_sec) * 1000 + now.tv_usec / 1000; +#else +# error "Don't know how to get the current time on your system." +#endif +} + +// Utilities + +// class String. + +#if GTEST_OS_WINDOWS_MOBILE +// Creates a UTF-16 wide string from the given ANSI string, allocating +// memory using new. The caller is responsible for deleting the return +// value using delete[]. Returns the wide string, or NULL if the +// input is NULL. +LPCWSTR String::AnsiToUtf16(const char* ansi) { + if (!ansi) return NULL; + const int length = strlen(ansi); + const int unicode_length = + MultiByteToWideChar(CP_ACP, 0, ansi, length, + NULL, 0); + WCHAR* unicode = new WCHAR[unicode_length + 1]; + MultiByteToWideChar(CP_ACP, 0, ansi, length, + unicode, unicode_length); + unicode[unicode_length] = 0; + return unicode; +} + +// Creates an ANSI string from the given wide string, allocating +// memory using new. The caller is responsible for deleting the return +// value using delete[]. Returns the ANSI string, or NULL if the +// input is NULL. +const char* String::Utf16ToAnsi(LPCWSTR utf16_str) { + if (!utf16_str) return NULL; + const int ansi_length = + WideCharToMultiByte(CP_ACP, 0, utf16_str, -1, + NULL, 0, NULL, NULL); + char* ansi = new char[ansi_length + 1]; + WideCharToMultiByte(CP_ACP, 0, utf16_str, -1, + ansi, ansi_length, NULL, NULL); + ansi[ansi_length] = 0; + return ansi; +} + +#endif // GTEST_OS_WINDOWS_MOBILE + +// Compares two C strings. Returns true iff they have the same content. +// +// Unlike strcmp(), this function can handle NULL argument(s). A NULL +// C string is considered different to any non-NULL C string, +// including the empty string. +bool String::CStringEquals(const char * lhs, const char * rhs) { + if ( lhs == NULL ) return rhs == NULL; + + if ( rhs == NULL ) return false; + + return strcmp(lhs, rhs) == 0; +} + +#if GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING + +// Converts an array of wide chars to a narrow string using the UTF-8 +// encoding, and streams the result to the given Message object. +static void StreamWideCharsToMessage(const wchar_t* wstr, size_t length, + Message* msg) { + for (size_t i = 0; i != length; ) { // NOLINT + if (wstr[i] != L'\0') { + *msg << WideStringToUtf8(wstr + i, static_cast(length - i)); + while (i != length && wstr[i] != L'\0') + i++; + } else { + *msg << '\0'; + i++; + } + } +} + +#endif // GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING + +void SplitString(const ::std::string& str, char delimiter, + ::std::vector< ::std::string>* dest) { + ::std::vector< ::std::string> parsed; + ::std::string::size_type pos = 0; + while (::testing::internal::AlwaysTrue()) { + const ::std::string::size_type colon = str.find(delimiter, pos); + if (colon == ::std::string::npos) { + parsed.push_back(str.substr(pos)); + break; + } else { + parsed.push_back(str.substr(pos, colon - pos)); + pos = colon + 1; + } + } + dest->swap(parsed); +} + +} // namespace internal + +// Constructs an empty Message. +// We allocate the stringstream separately because otherwise each use of +// ASSERT/EXPECT in a procedure adds over 200 bytes to the procedure's +// stack frame leading to huge stack frames in some cases; gcc does not reuse +// the stack space. +Message::Message() : ss_(new ::std::stringstream) { + // By default, we want there to be enough precision when printing + // a double to a Message. + *ss_ << std::setprecision(std::numeric_limits::digits10 + 2); +} + +// These two overloads allow streaming a wide C string to a Message +// using the UTF-8 encoding. +Message& Message::operator <<(const wchar_t* wide_c_str) { + return *this << internal::String::ShowWideCString(wide_c_str); +} +Message& Message::operator <<(wchar_t* wide_c_str) { + return *this << internal::String::ShowWideCString(wide_c_str); +} + +#if GTEST_HAS_STD_WSTRING +// Converts the given wide string to a narrow string using the UTF-8 +// encoding, and streams the result to this Message object. +Message& Message::operator <<(const ::std::wstring& wstr) { + internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this); + return *this; +} +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_GLOBAL_WSTRING +// Converts the given wide string to a narrow string using the UTF-8 +// encoding, and streams the result to this Message object. +Message& Message::operator <<(const ::wstring& wstr) { + internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this); + return *this; +} +#endif // GTEST_HAS_GLOBAL_WSTRING + +// Gets the text streamed to this object so far as an std::string. +// Each '\0' character in the buffer is replaced with "\\0". +std::string Message::GetString() const { + return internal::StringStreamToString(ss_.get()); +} + +// AssertionResult constructors. +// Used in EXPECT_TRUE/FALSE(assertion_result). +AssertionResult::AssertionResult(const AssertionResult& other) + : success_(other.success_), + message_(other.message_.get() != NULL ? + new ::std::string(*other.message_) : + static_cast< ::std::string*>(NULL)) { +} + +// Swaps two AssertionResults. +void AssertionResult::swap(AssertionResult& other) { + using std::swap; + swap(success_, other.success_); + swap(message_, other.message_); +} + +// Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE. +AssertionResult AssertionResult::operator!() const { + AssertionResult negation(!success_); + if (message_.get() != NULL) + negation << *message_; + return negation; +} + +// Makes a successful assertion result. +AssertionResult AssertionSuccess() { + return AssertionResult(true); +} + +// Makes a failed assertion result. +AssertionResult AssertionFailure() { + return AssertionResult(false); +} + +// Makes a failed assertion result with the given failure message. +// Deprecated; use AssertionFailure() << message. +AssertionResult AssertionFailure(const Message& message) { + return AssertionFailure() << message; +} + +namespace internal { + +namespace edit_distance { +std::vector CalculateOptimalEdits(const std::vector& left, + const std::vector& right) { + std::vector > costs( + left.size() + 1, std::vector(right.size() + 1)); + std::vector > best_move( + left.size() + 1, std::vector(right.size() + 1)); + + // Populate for empty right. + for (size_t l_i = 0; l_i < costs.size(); ++l_i) { + costs[l_i][0] = static_cast(l_i); + best_move[l_i][0] = kRemove; + } + // Populate for empty left. + for (size_t r_i = 1; r_i < costs[0].size(); ++r_i) { + costs[0][r_i] = static_cast(r_i); + best_move[0][r_i] = kAdd; + } + + for (size_t l_i = 0; l_i < left.size(); ++l_i) { + for (size_t r_i = 0; r_i < right.size(); ++r_i) { + if (left[l_i] == right[r_i]) { + // Found a match. Consume it. + costs[l_i + 1][r_i + 1] = costs[l_i][r_i]; + best_move[l_i + 1][r_i + 1] = kMatch; + continue; + } + + const double add = costs[l_i + 1][r_i]; + const double remove = costs[l_i][r_i + 1]; + const double replace = costs[l_i][r_i]; + if (add < remove && add < replace) { + costs[l_i + 1][r_i + 1] = add + 1; + best_move[l_i + 1][r_i + 1] = kAdd; + } else if (remove < add && remove < replace) { + costs[l_i + 1][r_i + 1] = remove + 1; + best_move[l_i + 1][r_i + 1] = kRemove; + } else { + // We make replace a little more expensive than add/remove to lower + // their priority. + costs[l_i + 1][r_i + 1] = replace + 1.00001; + best_move[l_i + 1][r_i + 1] = kReplace; + } + } + } + + // Reconstruct the best path. We do it in reverse order. + std::vector best_path; + for (size_t l_i = left.size(), r_i = right.size(); l_i > 0 || r_i > 0;) { + EditType move = best_move[l_i][r_i]; + best_path.push_back(move); + l_i -= move != kAdd; + r_i -= move != kRemove; + } + std::reverse(best_path.begin(), best_path.end()); + return best_path; +} + +namespace { + +// Helper class to convert string into ids with deduplication. +class InternalStrings { + public: + size_t GetId(const std::string& str) { + IdMap::iterator it = ids_.find(str); + if (it != ids_.end()) return it->second; + size_t id = ids_.size(); + return ids_[str] = id; + } + + private: + typedef std::map IdMap; + IdMap ids_; +}; + +} // namespace + +std::vector CalculateOptimalEdits( + const std::vector& left, + const std::vector& right) { + std::vector left_ids, right_ids; + { + InternalStrings intern_table; + for (size_t i = 0; i < left.size(); ++i) { + left_ids.push_back(intern_table.GetId(left[i])); + } + for (size_t i = 0; i < right.size(); ++i) { + right_ids.push_back(intern_table.GetId(right[i])); + } + } + return CalculateOptimalEdits(left_ids, right_ids); +} + +namespace { + +// Helper class that holds the state for one hunk and prints it out to the +// stream. +// It reorders adds/removes when possible to group all removes before all +// adds. It also adds the hunk header before printint into the stream. +class Hunk { + public: + Hunk(size_t left_start, size_t right_start) + : left_start_(left_start), + right_start_(right_start), + adds_(), + removes_(), + common_() {} + + void PushLine(char edit, const char* line) { + switch (edit) { + case ' ': + ++common_; + FlushEdits(); + hunk_.push_back(std::make_pair(' ', line)); + break; + case '-': + ++removes_; + hunk_removes_.push_back(std::make_pair('-', line)); + break; + case '+': + ++adds_; + hunk_adds_.push_back(std::make_pair('+', line)); + break; + } + } + + void PrintTo(std::ostream* os) { + PrintHeader(os); + FlushEdits(); + for (std::list >::const_iterator it = + hunk_.begin(); + it != hunk_.end(); ++it) { + *os << it->first << it->second << "\n"; + } + } + + bool has_edits() const { return adds_ || removes_; } + + private: + void FlushEdits() { + hunk_.splice(hunk_.end(), hunk_removes_); + hunk_.splice(hunk_.end(), hunk_adds_); + } + + // Print a unified diff header for one hunk. + // The format is + // "@@ -, +, @@" + // where the left/right parts are ommitted if unnecessary. + void PrintHeader(std::ostream* ss) const { + *ss << "@@ "; + if (removes_) { + *ss << "-" << left_start_ << "," << (removes_ + common_); + } + if (removes_ && adds_) { + *ss << " "; + } + if (adds_) { + *ss << "+" << right_start_ << "," << (adds_ + common_); + } + *ss << " @@\n"; + } + + size_t left_start_, right_start_; + size_t adds_, removes_, common_; + std::list > hunk_, hunk_adds_, hunk_removes_; +}; + +} // namespace + +// Create a list of diff hunks in Unified diff format. +// Each hunk has a header generated by PrintHeader above plus a body with +// lines prefixed with ' ' for no change, '-' for deletion and '+' for +// addition. +// 'context' represents the desired unchanged prefix/suffix around the diff. +// If two hunks are close enough that their contexts overlap, then they are +// joined into one hunk. +std::string CreateUnifiedDiff(const std::vector& left, + const std::vector& right, + size_t context) { + const std::vector edits = CalculateOptimalEdits(left, right); + + size_t l_i = 0, r_i = 0, edit_i = 0; + std::stringstream ss; + while (edit_i < edits.size()) { + // Find first edit. + while (edit_i < edits.size() && edits[edit_i] == kMatch) { + ++l_i; + ++r_i; + ++edit_i; + } + + // Find the first line to include in the hunk. + const size_t prefix_context = std::min(l_i, context); + Hunk hunk(l_i - prefix_context + 1, r_i - prefix_context + 1); + for (size_t i = prefix_context; i > 0; --i) { + hunk.PushLine(' ', left[l_i - i].c_str()); + } + + // Iterate the edits until we found enough suffix for the hunk or the input + // is over. + size_t n_suffix = 0; + for (; edit_i < edits.size(); ++edit_i) { + if (n_suffix >= context) { + // Continue only if the next hunk is very close. + std::vector::const_iterator it = edits.begin() + edit_i; + while (it != edits.end() && *it == kMatch) ++it; + if (it == edits.end() || (it - edits.begin()) - edit_i >= context) { + // There is no next edit or it is too far away. + break; + } + } + + EditType edit = edits[edit_i]; + // Reset count when a non match is found. + n_suffix = edit == kMatch ? n_suffix + 1 : 0; + + if (edit == kMatch || edit == kRemove || edit == kReplace) { + hunk.PushLine(edit == kMatch ? ' ' : '-', left[l_i].c_str()); + } + if (edit == kAdd || edit == kReplace) { + hunk.PushLine('+', right[r_i].c_str()); + } + + // Advance indices, depending on edit type. + l_i += edit != kAdd; + r_i += edit != kRemove; + } + + if (!hunk.has_edits()) { + // We are done. We don't want this hunk. + break; + } + + hunk.PrintTo(&ss); + } + return ss.str(); +} + +} // namespace edit_distance + +namespace { + +// The string representation of the values received in EqFailure() are already +// escaped. Split them on escaped '\n' boundaries. Leave all other escaped +// characters the same. +std::vector SplitEscapedString(const std::string& str) { + std::vector lines; + size_t start = 0, end = str.size(); + if (end > 2 && str[0] == '"' && str[end - 1] == '"') { + ++start; + --end; + } + bool escaped = false; + for (size_t i = start; i + 1 < end; ++i) { + if (escaped) { + escaped = false; + if (str[i] == 'n') { + lines.push_back(str.substr(start, i - start - 1)); + start = i + 1; + } + } else { + escaped = str[i] == '\\'; + } + } + lines.push_back(str.substr(start, end - start)); + return lines; +} + +} // namespace + +// Constructs and returns the message for an equality assertion +// (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure. +// +// The first four parameters are the expressions used in the assertion +// and their values, as strings. For example, for ASSERT_EQ(foo, bar) +// where foo is 5 and bar is 6, we have: +// +// lhs_expression: "foo" +// rhs_expression: "bar" +// lhs_value: "5" +// rhs_value: "6" +// +// The ignoring_case parameter is true iff the assertion is a +// *_STRCASEEQ*. When it's true, the string "Ignoring case" will +// be inserted into the message. +AssertionResult EqFailure(const char* lhs_expression, + const char* rhs_expression, + const std::string& lhs_value, + const std::string& rhs_value, + bool ignoring_case) { + Message msg; + msg << " Expected: " << lhs_expression; + if (lhs_value != lhs_expression) { + msg << "\n Which is: " << lhs_value; + } + msg << "\nTo be equal to: " << rhs_expression; + if (rhs_value != rhs_expression) { + msg << "\n Which is: " << rhs_value; + } + + if (ignoring_case) { + msg << "\nIgnoring case"; + } + + if (!lhs_value.empty() && !rhs_value.empty()) { + const std::vector lhs_lines = + SplitEscapedString(lhs_value); + const std::vector rhs_lines = + SplitEscapedString(rhs_value); + if (lhs_lines.size() > 1 || rhs_lines.size() > 1) { + msg << "\nWith diff:\n" + << edit_distance::CreateUnifiedDiff(lhs_lines, rhs_lines); + } + } + + return AssertionFailure() << msg; +} + +// Constructs a failure message for Boolean assertions such as EXPECT_TRUE. +std::string GetBoolAssertionFailureMessage( + const AssertionResult& assertion_result, + const char* expression_text, + const char* actual_predicate_value, + const char* expected_predicate_value) { + const char* actual_message = assertion_result.message(); + Message msg; + msg << "Value of: " << expression_text + << "\n Actual: " << actual_predicate_value; + if (actual_message[0] != '\0') + msg << " (" << actual_message << ")"; + msg << "\nExpected: " << expected_predicate_value; + return msg.GetString(); +} + +// Helper function for implementing ASSERT_NEAR. +AssertionResult DoubleNearPredFormat(const char* expr1, + const char* expr2, + const char* abs_error_expr, + double val1, + double val2, + double abs_error) { + const double diff = fabs(val1 - val2); + if (diff <= abs_error) return AssertionSuccess(); + + // TODO(wan): do not print the value of an expression if it's + // already a literal. + return AssertionFailure() + << "The difference between " << expr1 << " and " << expr2 + << " is " << diff << ", which exceeds " << abs_error_expr << ", where\n" + << expr1 << " evaluates to " << val1 << ",\n" + << expr2 << " evaluates to " << val2 << ", and\n" + << abs_error_expr << " evaluates to " << abs_error << "."; +} + + +// Helper template for implementing FloatLE() and DoubleLE(). +template +AssertionResult FloatingPointLE(const char* expr1, + const char* expr2, + RawType val1, + RawType val2) { + // Returns success if val1 is less than val2, + if (val1 < val2) { + return AssertionSuccess(); + } + + // or if val1 is almost equal to val2. + const FloatingPoint lhs(val1), rhs(val2); + if (lhs.AlmostEquals(rhs)) { + return AssertionSuccess(); + } + + // Note that the above two checks will both fail if either val1 or + // val2 is NaN, as the IEEE floating-point standard requires that + // any predicate involving a NaN must return false. + + ::std::stringstream val1_ss; + val1_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << val1; + + ::std::stringstream val2_ss; + val2_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << val2; + + return AssertionFailure() + << "Expected: (" << expr1 << ") <= (" << expr2 << ")\n" + << " Actual: " << StringStreamToString(&val1_ss) << " vs " + << StringStreamToString(&val2_ss); +} + +} // namespace internal + +// Asserts that val1 is less than, or almost equal to, val2. Fails +// otherwise. In particular, it fails if either val1 or val2 is NaN. +AssertionResult FloatLE(const char* expr1, const char* expr2, + float val1, float val2) { + return internal::FloatingPointLE(expr1, expr2, val1, val2); +} + +// Asserts that val1 is less than, or almost equal to, val2. Fails +// otherwise. In particular, it fails if either val1 or val2 is NaN. +AssertionResult DoubleLE(const char* expr1, const char* expr2, + double val1, double val2) { + return internal::FloatingPointLE(expr1, expr2, val1, val2); +} + +namespace internal { + +// The helper function for {ASSERT|EXPECT}_EQ with int or enum +// arguments. +AssertionResult CmpHelperEQ(const char* lhs_expression, + const char* rhs_expression, + BiggestInt lhs, + BiggestInt rhs) { + if (lhs == rhs) { + return AssertionSuccess(); + } + + return EqFailure(lhs_expression, + rhs_expression, + FormatForComparisonFailureMessage(lhs, rhs), + FormatForComparisonFailureMessage(rhs, lhs), + false); +} + +// A macro for implementing the helper functions needed to implement +// ASSERT_?? and EXPECT_?? with integer or enum arguments. It is here +// just to avoid copy-and-paste of similar code. +#define GTEST_IMPL_CMP_HELPER_(op_name, op)\ +AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ + BiggestInt val1, BiggestInt val2) {\ + if (val1 op val2) {\ + return AssertionSuccess();\ + } else {\ + return AssertionFailure() \ + << "Expected: (" << expr1 << ") " #op " (" << expr2\ + << "), actual: " << FormatForComparisonFailureMessage(val1, val2)\ + << " vs " << FormatForComparisonFailureMessage(val2, val1);\ + }\ +} + +// Implements the helper function for {ASSERT|EXPECT}_NE with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(NE, !=) +// Implements the helper function for {ASSERT|EXPECT}_LE with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(LE, <=) +// Implements the helper function for {ASSERT|EXPECT}_LT with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(LT, < ) +// Implements the helper function for {ASSERT|EXPECT}_GE with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(GE, >=) +// Implements the helper function for {ASSERT|EXPECT}_GT with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(GT, > ) + +#undef GTEST_IMPL_CMP_HELPER_ + +// The helper function for {ASSERT|EXPECT}_STREQ. +AssertionResult CmpHelperSTREQ(const char* lhs_expression, + const char* rhs_expression, + const char* lhs, + const char* rhs) { + if (String::CStringEquals(lhs, rhs)) { + return AssertionSuccess(); + } + + return EqFailure(lhs_expression, + rhs_expression, + PrintToString(lhs), + PrintToString(rhs), + false); +} + +// The helper function for {ASSERT|EXPECT}_STRCASEEQ. +AssertionResult CmpHelperSTRCASEEQ(const char* lhs_expression, + const char* rhs_expression, + const char* lhs, + const char* rhs) { + if (String::CaseInsensitiveCStringEquals(lhs, rhs)) { + return AssertionSuccess(); + } + + return EqFailure(lhs_expression, + rhs_expression, + PrintToString(lhs), + PrintToString(rhs), + true); +} + +// The helper function for {ASSERT|EXPECT}_STRNE. +AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2) { + if (!String::CStringEquals(s1, s2)) { + return AssertionSuccess(); + } else { + return AssertionFailure() << "Expected: (" << s1_expression << ") != (" + << s2_expression << "), actual: \"" + << s1 << "\" vs \"" << s2 << "\""; + } +} + +// The helper function for {ASSERT|EXPECT}_STRCASENE. +AssertionResult CmpHelperSTRCASENE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2) { + if (!String::CaseInsensitiveCStringEquals(s1, s2)) { + return AssertionSuccess(); + } else { + return AssertionFailure() + << "Expected: (" << s1_expression << ") != (" + << s2_expression << ") (ignoring case), actual: \"" + << s1 << "\" vs \"" << s2 << "\""; + } +} + +} // namespace internal + +namespace { + +// Helper functions for implementing IsSubString() and IsNotSubstring(). + +// This group of overloaded functions return true iff needle is a +// substring of haystack. NULL is considered a substring of itself +// only. + +bool IsSubstringPred(const char* needle, const char* haystack) { + if (needle == NULL || haystack == NULL) + return needle == haystack; + + return strstr(haystack, needle) != NULL; +} + +bool IsSubstringPred(const wchar_t* needle, const wchar_t* haystack) { + if (needle == NULL || haystack == NULL) + return needle == haystack; + + return wcsstr(haystack, needle) != NULL; +} + +// StringType here can be either ::std::string or ::std::wstring. +template +bool IsSubstringPred(const StringType& needle, + const StringType& haystack) { + return haystack.find(needle) != StringType::npos; +} + +// This function implements either IsSubstring() or IsNotSubstring(), +// depending on the value of the expected_to_be_substring parameter. +// StringType here can be const char*, const wchar_t*, ::std::string, +// or ::std::wstring. +template +AssertionResult IsSubstringImpl( + bool expected_to_be_substring, + const char* needle_expr, const char* haystack_expr, + const StringType& needle, const StringType& haystack) { + if (IsSubstringPred(needle, haystack) == expected_to_be_substring) + return AssertionSuccess(); + + const bool is_wide_string = sizeof(needle[0]) > 1; + const char* const begin_string_quote = is_wide_string ? "L\"" : "\""; + return AssertionFailure() + << "Value of: " << needle_expr << "\n" + << " Actual: " << begin_string_quote << needle << "\"\n" + << "Expected: " << (expected_to_be_substring ? "" : "not ") + << "a substring of " << haystack_expr << "\n" + << "Which is: " << begin_string_quote << haystack << "\""; +} + +} // namespace + +// IsSubstring() and IsNotSubstring() check whether needle is a +// substring of haystack (NULL is considered a substring of itself +// only), and return an appropriate error message when they fail. + +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} + +#if GTEST_HAS_STD_WSTRING +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} +#endif // GTEST_HAS_STD_WSTRING + +namespace internal { + +#if GTEST_OS_WINDOWS + +namespace { + +// Helper function for IsHRESULT{SuccessFailure} predicates +AssertionResult HRESULTFailureHelper(const char* expr, + const char* expected, + long hr) { // NOLINT +# if GTEST_OS_WINDOWS_MOBILE + + // Windows CE doesn't support FormatMessage. + const char error_text[] = ""; + +# else + + // Looks up the human-readable system message for the HRESULT code + // and since we're not passing any params to FormatMessage, we don't + // want inserts expanded. + const DWORD kFlags = FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS; + const DWORD kBufSize = 4096; + // Gets the system's human readable message string for this HRESULT. + char error_text[kBufSize] = { '\0' }; + DWORD message_length = ::FormatMessageA(kFlags, + 0, // no source, we're asking system + hr, // the error + 0, // no line width restrictions + error_text, // output buffer + kBufSize, // buf size + NULL); // no arguments for inserts + // Trims tailing white space (FormatMessage leaves a trailing CR-LF) + for (; message_length && IsSpace(error_text[message_length - 1]); + --message_length) { + error_text[message_length - 1] = '\0'; + } + +# endif // GTEST_OS_WINDOWS_MOBILE + + const std::string error_hex("0x" + String::FormatHexInt(hr)); + return ::testing::AssertionFailure() + << "Expected: " << expr << " " << expected << ".\n" + << " Actual: " << error_hex << " " << error_text << "\n"; +} + +} // namespace + +AssertionResult IsHRESULTSuccess(const char* expr, long hr) { // NOLINT + if (SUCCEEDED(hr)) { + return AssertionSuccess(); + } + return HRESULTFailureHelper(expr, "succeeds", hr); +} + +AssertionResult IsHRESULTFailure(const char* expr, long hr) { // NOLINT + if (FAILED(hr)) { + return AssertionSuccess(); + } + return HRESULTFailureHelper(expr, "fails", hr); +} + +#endif // GTEST_OS_WINDOWS + +// Utility functions for encoding Unicode text (wide strings) in +// UTF-8. + +// A Unicode code-point can have upto 21 bits, and is encoded in UTF-8 +// like this: +// +// Code-point length Encoding +// 0 - 7 bits 0xxxxxxx +// 8 - 11 bits 110xxxxx 10xxxxxx +// 12 - 16 bits 1110xxxx 10xxxxxx 10xxxxxx +// 17 - 21 bits 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + +// The maximum code-point a one-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint1 = (static_cast(1) << 7) - 1; + +// The maximum code-point a two-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint2 = (static_cast(1) << (5 + 6)) - 1; + +// The maximum code-point a three-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint3 = (static_cast(1) << (4 + 2*6)) - 1; + +// The maximum code-point a four-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint4 = (static_cast(1) << (3 + 3*6)) - 1; + +// Chops off the n lowest bits from a bit pattern. Returns the n +// lowest bits. As a side effect, the original bit pattern will be +// shifted to the right by n bits. +inline UInt32 ChopLowBits(UInt32* bits, int n) { + const UInt32 low_bits = *bits & ((static_cast(1) << n) - 1); + *bits >>= n; + return low_bits; +} + +// Converts a Unicode code point to a narrow string in UTF-8 encoding. +// code_point parameter is of type UInt32 because wchar_t may not be +// wide enough to contain a code point. +// If the code_point is not a valid Unicode code point +// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be converted +// to "(Invalid Unicode 0xXXXXXXXX)". +std::string CodePointToUtf8(UInt32 code_point) { + if (code_point > kMaxCodePoint4) { + return "(Invalid Unicode 0x" + String::FormatHexInt(code_point) + ")"; + } + + char str[5]; // Big enough for the largest valid code point. + if (code_point <= kMaxCodePoint1) { + str[1] = '\0'; + str[0] = static_cast(code_point); // 0xxxxxxx + } else if (code_point <= kMaxCodePoint2) { + str[2] = '\0'; + str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[0] = static_cast(0xC0 | code_point); // 110xxxxx + } else if (code_point <= kMaxCodePoint3) { + str[3] = '\0'; + str[2] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[0] = static_cast(0xE0 | code_point); // 1110xxxx + } else { // code_point <= kMaxCodePoint4 + str[4] = '\0'; + str[3] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[2] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[0] = static_cast(0xF0 | code_point); // 11110xxx + } + return str; +} + +// The following two functions only make sense if the the system +// uses UTF-16 for wide string encoding. All supported systems +// with 16 bit wchar_t (Windows, Cygwin, Symbian OS) do use UTF-16. + +// Determines if the arguments constitute UTF-16 surrogate pair +// and thus should be combined into a single Unicode code point +// using CreateCodePointFromUtf16SurrogatePair. +inline bool IsUtf16SurrogatePair(wchar_t first, wchar_t second) { + return sizeof(wchar_t) == 2 && + (first & 0xFC00) == 0xD800 && (second & 0xFC00) == 0xDC00; +} + +// Creates a Unicode code point from UTF16 surrogate pair. +inline UInt32 CreateCodePointFromUtf16SurrogatePair(wchar_t first, + wchar_t second) { + const UInt32 mask = (1 << 10) - 1; + return (sizeof(wchar_t) == 2) ? + (((first & mask) << 10) | (second & mask)) + 0x10000 : + // This function should not be called when the condition is + // false, but we provide a sensible default in case it is. + static_cast(first); +} + +// Converts a wide string to a narrow string in UTF-8 encoding. +// The wide string is assumed to have the following encoding: +// UTF-16 if sizeof(wchar_t) == 2 (on Windows, Cygwin, Symbian OS) +// UTF-32 if sizeof(wchar_t) == 4 (on Linux) +// Parameter str points to a null-terminated wide string. +// Parameter num_chars may additionally limit the number +// of wchar_t characters processed. -1 is used when the entire string +// should be processed. +// If the string contains code points that are not valid Unicode code points +// (i.e. outside of Unicode range U+0 to U+10FFFF) they will be output +// as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding +// and contains invalid UTF-16 surrogate pairs, values in those pairs +// will be encoded as individual Unicode characters from Basic Normal Plane. +std::string WideStringToUtf8(const wchar_t* str, int num_chars) { + if (num_chars == -1) + num_chars = static_cast(wcslen(str)); + + ::std::stringstream stream; + for (int i = 0; i < num_chars; ++i) { + UInt32 unicode_code_point; + + if (str[i] == L'\0') { + break; + } else if (i + 1 < num_chars && IsUtf16SurrogatePair(str[i], str[i + 1])) { + unicode_code_point = CreateCodePointFromUtf16SurrogatePair(str[i], + str[i + 1]); + i++; + } else { + unicode_code_point = static_cast(str[i]); + } + + stream << CodePointToUtf8(unicode_code_point); + } + return StringStreamToString(&stream); +} + +// Converts a wide C string to an std::string using the UTF-8 encoding. +// NULL will be converted to "(null)". +std::string String::ShowWideCString(const wchar_t * wide_c_str) { + if (wide_c_str == NULL) return "(null)"; + + return internal::WideStringToUtf8(wide_c_str, -1); +} + +// Compares two wide C strings. Returns true iff they have the same +// content. +// +// Unlike wcscmp(), this function can handle NULL argument(s). A NULL +// C string is considered different to any non-NULL C string, +// including the empty string. +bool String::WideCStringEquals(const wchar_t * lhs, const wchar_t * rhs) { + if (lhs == NULL) return rhs == NULL; + + if (rhs == NULL) return false; + + return wcscmp(lhs, rhs) == 0; +} + +// Helper function for *_STREQ on wide strings. +AssertionResult CmpHelperSTREQ(const char* lhs_expression, + const char* rhs_expression, + const wchar_t* lhs, + const wchar_t* rhs) { + if (String::WideCStringEquals(lhs, rhs)) { + return AssertionSuccess(); + } + + return EqFailure(lhs_expression, + rhs_expression, + PrintToString(lhs), + PrintToString(rhs), + false); +} + +// Helper function for *_STRNE on wide strings. +AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const wchar_t* s1, + const wchar_t* s2) { + if (!String::WideCStringEquals(s1, s2)) { + return AssertionSuccess(); + } + + return AssertionFailure() << "Expected: (" << s1_expression << ") != (" + << s2_expression << "), actual: " + << PrintToString(s1) + << " vs " << PrintToString(s2); +} + +// Compares two C strings, ignoring case. Returns true iff they have +// the same content. +// +// Unlike strcasecmp(), this function can handle NULL argument(s). A +// NULL C string is considered different to any non-NULL C string, +// including the empty string. +bool String::CaseInsensitiveCStringEquals(const char * lhs, const char * rhs) { + if (lhs == NULL) + return rhs == NULL; + if (rhs == NULL) + return false; + return posix::StrCaseCmp(lhs, rhs) == 0; +} + + // Compares two wide C strings, ignoring case. Returns true iff they + // have the same content. + // + // Unlike wcscasecmp(), this function can handle NULL argument(s). + // A NULL C string is considered different to any non-NULL wide C string, + // including the empty string. + // NB: The implementations on different platforms slightly differ. + // On windows, this method uses _wcsicmp which compares according to LC_CTYPE + // environment variable. On GNU platform this method uses wcscasecmp + // which compares according to LC_CTYPE category of the current locale. + // On MacOS X, it uses towlower, which also uses LC_CTYPE category of the + // current locale. +bool String::CaseInsensitiveWideCStringEquals(const wchar_t* lhs, + const wchar_t* rhs) { + if (lhs == NULL) return rhs == NULL; + + if (rhs == NULL) return false; + +#if GTEST_OS_WINDOWS + return _wcsicmp(lhs, rhs) == 0; +#elif GTEST_OS_LINUX && !GTEST_OS_LINUX_ANDROID + return wcscasecmp(lhs, rhs) == 0; +#else + // Android, Mac OS X and Cygwin don't define wcscasecmp. + // Other unknown OSes may not define it either. + wint_t left, right; + do { + left = towlower(*lhs++); + right = towlower(*rhs++); + } while (left && left == right); + return left == right; +#endif // OS selector +} + +// Returns true iff str ends with the given suffix, ignoring case. +// Any string is considered to end with an empty suffix. +bool String::EndsWithCaseInsensitive( + const std::string& str, const std::string& suffix) { + const size_t str_len = str.length(); + const size_t suffix_len = suffix.length(); + return (str_len >= suffix_len) && + CaseInsensitiveCStringEquals(str.c_str() + str_len - suffix_len, + suffix.c_str()); +} + +// Formats an int value as "%02d". +std::string String::FormatIntWidth2(int value) { + std::stringstream ss; + ss << std::setfill('0') << std::setw(2) << value; + return ss.str(); +} + +// Formats an int value as "%X". +std::string String::FormatHexInt(int value) { + std::stringstream ss; + ss << std::hex << std::uppercase << value; + return ss.str(); +} + +// Formats a byte as "%02X". +std::string String::FormatByte(unsigned char value) { + std::stringstream ss; + ss << std::setfill('0') << std::setw(2) << std::hex << std::uppercase + << static_cast(value); + return ss.str(); +} + +// Converts the buffer in a stringstream to an std::string, converting NUL +// bytes to "\\0" along the way. +std::string StringStreamToString(::std::stringstream* ss) { + const ::std::string& str = ss->str(); + const char* const start = str.c_str(); + const char* const end = start + str.length(); + + std::string result; + result.reserve(2 * (end - start)); + for (const char* ch = start; ch != end; ++ch) { + if (*ch == '\0') { + result += "\\0"; // Replaces NUL with "\\0"; + } else { + result += *ch; + } + } + + return result; +} + +// Appends the user-supplied message to the Google-Test-generated message. +std::string AppendUserMessage(const std::string& gtest_msg, + const Message& user_msg) { + // Appends the user message if it's non-empty. + const std::string user_msg_string = user_msg.GetString(); + if (user_msg_string.empty()) { + return gtest_msg; + } + + return gtest_msg + "\n" + user_msg_string; +} + +} // namespace internal + +// class TestResult + +// Creates an empty TestResult. +TestResult::TestResult() + : death_test_count_(0), + elapsed_time_(0) { +} + +// D'tor. +TestResult::~TestResult() { +} + +// Returns the i-th test part result among all the results. i can +// range from 0 to total_part_count() - 1. If i is not in that range, +// aborts the program. +const TestPartResult& TestResult::GetTestPartResult(int i) const { + if (i < 0 || i >= total_part_count()) + internal::posix::Abort(); + return test_part_results_.at(i); +} + +// Returns the i-th test property. i can range from 0 to +// test_property_count() - 1. If i is not in that range, aborts the +// program. +const TestProperty& TestResult::GetTestProperty(int i) const { + if (i < 0 || i >= test_property_count()) + internal::posix::Abort(); + return test_properties_.at(i); +} + +// Clears the test part results. +void TestResult::ClearTestPartResults() { + test_part_results_.clear(); +} + +// Adds a test part result to the list. +void TestResult::AddTestPartResult(const TestPartResult& test_part_result) { + test_part_results_.push_back(test_part_result); +} + +// Adds a test property to the list. If a property with the same key as the +// supplied property is already represented, the value of this test_property +// replaces the old value for that key. +void TestResult::RecordProperty(const std::string& xml_element, + const TestProperty& test_property) { + if (!ValidateTestProperty(xml_element, test_property)) { + return; + } + internal::MutexLock lock(&test_properites_mutex_); + const std::vector::iterator property_with_matching_key = + std::find_if(test_properties_.begin(), test_properties_.end(), + internal::TestPropertyKeyIs(test_property.key())); + if (property_with_matching_key == test_properties_.end()) { + test_properties_.push_back(test_property); + return; + } + property_with_matching_key->SetValue(test_property.value()); +} + +// The list of reserved attributes used in the element of XML +// output. +static const char* const kReservedTestSuitesAttributes[] = { + "disabled", + "errors", + "failures", + "name", + "random_seed", + "tests", + "time", + "timestamp" +}; + +// The list of reserved attributes used in the element of XML +// output. +static const char* const kReservedTestSuiteAttributes[] = { + "disabled", + "errors", + "failures", + "name", + "tests", + "time" +}; + +// The list of reserved attributes used in the element of XML output. +static const char* const kReservedTestCaseAttributes[] = { + "classname", + "name", + "status", + "time", + "type_param", + "value_param" +}; + +template +std::vector ArrayAsVector(const char* const (&array)[kSize]) { + return std::vector(array, array + kSize); +} + +static std::vector GetReservedAttributesForElement( + const std::string& xml_element) { + if (xml_element == "testsuites") { + return ArrayAsVector(kReservedTestSuitesAttributes); + } else if (xml_element == "testsuite") { + return ArrayAsVector(kReservedTestSuiteAttributes); + } else if (xml_element == "testcase") { + return ArrayAsVector(kReservedTestCaseAttributes); + } else { + GTEST_CHECK_(false) << "Unrecognized xml_element provided: " << xml_element; + } + // This code is unreachable but some compilers may not realizes that. + return std::vector(); +} + +static std::string FormatWordList(const std::vector& words) { + Message word_list; + for (size_t i = 0; i < words.size(); ++i) { + if (i > 0 && words.size() > 2) { + word_list << ", "; + } + if (i == words.size() - 1) { + word_list << "and "; + } + word_list << "'" << words[i] << "'"; + } + return word_list.GetString(); +} + +bool ValidateTestPropertyName(const std::string& property_name, + const std::vector& reserved_names) { + if (std::find(reserved_names.begin(), reserved_names.end(), property_name) != + reserved_names.end()) { + ADD_FAILURE() << "Reserved key used in RecordProperty(): " << property_name + << " (" << FormatWordList(reserved_names) + << " are reserved by " << GTEST_NAME_ << ")"; + return false; + } + return true; +} + +// Adds a failure if the key is a reserved attribute of the element named +// xml_element. Returns true if the property is valid. +bool TestResult::ValidateTestProperty(const std::string& xml_element, + const TestProperty& test_property) { + return ValidateTestPropertyName(test_property.key(), + GetReservedAttributesForElement(xml_element)); +} + +// Clears the object. +void TestResult::Clear() { + test_part_results_.clear(); + test_properties_.clear(); + death_test_count_ = 0; + elapsed_time_ = 0; +} + +// Returns true iff the test failed. +bool TestResult::Failed() const { + for (int i = 0; i < total_part_count(); ++i) { + if (GetTestPartResult(i).failed()) + return true; + } + return false; +} + +// Returns true iff the test part fatally failed. +static bool TestPartFatallyFailed(const TestPartResult& result) { + return result.fatally_failed(); +} + +// Returns true iff the test fatally failed. +bool TestResult::HasFatalFailure() const { + return CountIf(test_part_results_, TestPartFatallyFailed) > 0; +} + +// Returns true iff the test part non-fatally failed. +static bool TestPartNonfatallyFailed(const TestPartResult& result) { + return result.nonfatally_failed(); +} + +// Returns true iff the test has a non-fatal failure. +bool TestResult::HasNonfatalFailure() const { + return CountIf(test_part_results_, TestPartNonfatallyFailed) > 0; +} + +// Gets the number of all test parts. This is the sum of the number +// of successful test parts and the number of failed test parts. +int TestResult::total_part_count() const { + return static_cast(test_part_results_.size()); +} + +// Returns the number of the test properties. +int TestResult::test_property_count() const { + return static_cast(test_properties_.size()); +} + +// class Test + +// Creates a Test object. + +// The c'tor saves the states of all flags. +Test::Test() + : gtest_flag_saver_(new GTEST_FLAG_SAVER_) { +} + +// The d'tor restores the states of all flags. The actual work is +// done by the d'tor of the gtest_flag_saver_ field, and thus not +// visible here. +Test::~Test() { +} + +// Sets up the test fixture. +// +// A sub-class may override this. +void Test::SetUp() { +} + +// Tears down the test fixture. +// +// A sub-class may override this. +void Test::TearDown() { +} + +// Allows user supplied key value pairs to be recorded for later output. +void Test::RecordProperty(const std::string& key, const std::string& value) { + UnitTest::GetInstance()->RecordProperty(key, value); +} + +// Allows user supplied key value pairs to be recorded for later output. +void Test::RecordProperty(const std::string& key, int value) { + Message value_message; + value_message << value; + RecordProperty(key, value_message.GetString().c_str()); +} + +namespace internal { + +void ReportFailureInUnknownLocation(TestPartResult::Type result_type, + const std::string& message) { + // This function is a friend of UnitTest and as such has access to + // AddTestPartResult. + UnitTest::GetInstance()->AddTestPartResult( + result_type, + NULL, // No info about the source file where the exception occurred. + -1, // We have no info on which line caused the exception. + message, + ""); // No stack trace, either. +} + +} // namespace internal + +// Google Test requires all tests in the same test case to use the same test +// fixture class. This function checks if the current test has the +// same fixture class as the first test in the current test case. If +// yes, it returns true; otherwise it generates a Google Test failure and +// returns false. +bool Test::HasSameFixtureClass() { + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + const TestCase* const test_case = impl->current_test_case(); + + // Info about the first test in the current test case. + const TestInfo* const first_test_info = test_case->test_info_list()[0]; + const internal::TypeId first_fixture_id = first_test_info->fixture_class_id_; + const char* const first_test_name = first_test_info->name(); + + // Info about the current test. + const TestInfo* const this_test_info = impl->current_test_info(); + const internal::TypeId this_fixture_id = this_test_info->fixture_class_id_; + const char* const this_test_name = this_test_info->name(); + + if (this_fixture_id != first_fixture_id) { + // Is the first test defined using TEST? + const bool first_is_TEST = first_fixture_id == internal::GetTestTypeId(); + // Is this test defined using TEST? + const bool this_is_TEST = this_fixture_id == internal::GetTestTypeId(); + + if (first_is_TEST || this_is_TEST) { + // Both TEST and TEST_F appear in same test case, which is incorrect. + // Tell the user how to fix this. + + // Gets the name of the TEST and the name of the TEST_F. Note + // that first_is_TEST and this_is_TEST cannot both be true, as + // the fixture IDs are different for the two tests. + const char* const TEST_name = + first_is_TEST ? first_test_name : this_test_name; + const char* const TEST_F_name = + first_is_TEST ? this_test_name : first_test_name; + + ADD_FAILURE() + << "All tests in the same test case must use the same test fixture\n" + << "class, so mixing TEST_F and TEST in the same test case is\n" + << "illegal. In test case " << this_test_info->test_case_name() + << ",\n" + << "test " << TEST_F_name << " is defined using TEST_F but\n" + << "test " << TEST_name << " is defined using TEST. You probably\n" + << "want to change the TEST to TEST_F or move it to another test\n" + << "case."; + } else { + // Two fixture classes with the same name appear in two different + // namespaces, which is not allowed. Tell the user how to fix this. + ADD_FAILURE() + << "All tests in the same test case must use the same test fixture\n" + << "class. However, in test case " + << this_test_info->test_case_name() << ",\n" + << "you defined test " << first_test_name + << " and test " << this_test_name << "\n" + << "using two different test fixture classes. This can happen if\n" + << "the two classes are from different namespaces or translation\n" + << "units and have the same name. You should probably rename one\n" + << "of the classes to put the tests into different test cases."; + } + return false; + } + + return true; +} + +#if GTEST_HAS_SEH + +// Adds an "exception thrown" fatal failure to the current test. This +// function returns its result via an output parameter pointer because VC++ +// prohibits creation of objects with destructors on stack in functions +// using __try (see error C2712). +static std::string* FormatSehExceptionMessage(DWORD exception_code, + const char* location) { + Message message; + message << "SEH exception with code 0x" << std::setbase(16) << + exception_code << std::setbase(10) << " thrown in " << location << "."; + + return new std::string(message.GetString()); +} + +#endif // GTEST_HAS_SEH + +namespace internal { + +#if GTEST_HAS_EXCEPTIONS + +// Adds an "exception thrown" fatal failure to the current test. +static std::string FormatCxxExceptionMessage(const char* description, + const char* location) { + Message message; + if (description != NULL) { + message << "C++ exception with description \"" << description << "\""; + } else { + message << "Unknown C++ exception"; + } + message << " thrown in " << location << "."; + + return message.GetString(); +} + +static std::string PrintTestPartResultToString( + const TestPartResult& test_part_result); + +GoogleTestFailureException::GoogleTestFailureException( + const TestPartResult& failure) + : ::std::runtime_error(PrintTestPartResultToString(failure).c_str()) {} + +#endif // GTEST_HAS_EXCEPTIONS + +// We put these helper functions in the internal namespace as IBM's xlC +// compiler rejects the code if they were declared static. + +// Runs the given method and handles SEH exceptions it throws, when +// SEH is supported; returns the 0-value for type Result in case of an +// SEH exception. (Microsoft compilers cannot handle SEH and C++ +// exceptions in the same function. Therefore, we provide a separate +// wrapper function for handling SEH exceptions.) +template +Result HandleSehExceptionsInMethodIfSupported( + T* object, Result (T::*method)(), const char* location) { +#if GTEST_HAS_SEH + __try { + return (object->*method)(); + } __except (internal::UnitTestOptions::GTestShouldProcessSEH( // NOLINT + GetExceptionCode())) { + // We create the exception message on the heap because VC++ prohibits + // creation of objects with destructors on stack in functions using __try + // (see error C2712). + std::string* exception_message = FormatSehExceptionMessage( + GetExceptionCode(), location); + internal::ReportFailureInUnknownLocation(TestPartResult::kFatalFailure, + *exception_message); + delete exception_message; + return static_cast(0); + } +#else + (void)location; + return (object->*method)(); +#endif // GTEST_HAS_SEH +} + +// Runs the given method and catches and reports C++ and/or SEH-style +// exceptions, if they are supported; returns the 0-value for type +// Result in case of an SEH exception. +template +Result HandleExceptionsInMethodIfSupported( + T* object, Result (T::*method)(), const char* location) { + // NOTE: The user code can affect the way in which Google Test handles + // exceptions by setting GTEST_FLAG(catch_exceptions), but only before + // RUN_ALL_TESTS() starts. It is technically possible to check the flag + // after the exception is caught and either report or re-throw the + // exception based on the flag's value: + // + // try { + // // Perform the test method. + // } catch (...) { + // if (GTEST_FLAG(catch_exceptions)) + // // Report the exception as failure. + // else + // throw; // Re-throws the original exception. + // } + // + // However, the purpose of this flag is to allow the program to drop into + // the debugger when the exception is thrown. On most platforms, once the + // control enters the catch block, the exception origin information is + // lost and the debugger will stop the program at the point of the + // re-throw in this function -- instead of at the point of the original + // throw statement in the code under test. For this reason, we perform + // the check early, sacrificing the ability to affect Google Test's + // exception handling in the method where the exception is thrown. + if (internal::GetUnitTestImpl()->catch_exceptions()) { +#if GTEST_HAS_EXCEPTIONS + try { + return HandleSehExceptionsInMethodIfSupported(object, method, location); + } catch (const internal::GoogleTestFailureException&) { // NOLINT + // This exception type can only be thrown by a failed Google + // Test assertion with the intention of letting another testing + // framework catch it. Therefore we just re-throw it. + throw; + } catch (const std::exception& e) { // NOLINT + internal::ReportFailureInUnknownLocation( + TestPartResult::kFatalFailure, + FormatCxxExceptionMessage(e.what(), location)); + } catch (...) { // NOLINT + internal::ReportFailureInUnknownLocation( + TestPartResult::kFatalFailure, + FormatCxxExceptionMessage(NULL, location)); + } + return static_cast(0); +#else + return HandleSehExceptionsInMethodIfSupported(object, method, location); +#endif // GTEST_HAS_EXCEPTIONS + } else { + return (object->*method)(); + } +} + +} // namespace internal + +// Runs the test and updates the test result. +void Test::Run() { + if (!HasSameFixtureClass()) return; + + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported(this, &Test::SetUp, "SetUp()"); + // We will run the test only if SetUp() was successful. + if (!HasFatalFailure()) { + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &Test::TestBody, "the test body"); + } + + // However, we want to clean up as much as possible. Hence we will + // always call TearDown(), even if SetUp() or the test body has + // failed. + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &Test::TearDown, "TearDown()"); +} + +// Returns true iff the current test has a fatal failure. +bool Test::HasFatalFailure() { + return internal::GetUnitTestImpl()->current_test_result()->HasFatalFailure(); +} + +// Returns true iff the current test has a non-fatal failure. +bool Test::HasNonfatalFailure() { + return internal::GetUnitTestImpl()->current_test_result()-> + HasNonfatalFailure(); +} + +// class TestInfo + +// Constructs a TestInfo object. It assumes ownership of the test factory +// object. +TestInfo::TestInfo(const std::string& a_test_case_name, + const std::string& a_name, + const char* a_type_param, + const char* a_value_param, + internal::CodeLocation a_code_location, + internal::TypeId fixture_class_id, + internal::TestFactoryBase* factory) + : test_case_name_(a_test_case_name), + name_(a_name), + type_param_(a_type_param ? new std::string(a_type_param) : NULL), + value_param_(a_value_param ? new std::string(a_value_param) : NULL), + location_(a_code_location), + fixture_class_id_(fixture_class_id), + should_run_(false), + is_disabled_(false), + matches_filter_(false), + factory_(factory), + result_() {} + +// Destructs a TestInfo object. +TestInfo::~TestInfo() { delete factory_; } + +namespace internal { + +// Creates a new TestInfo object and registers it with Google Test; +// returns the created object. +// +// Arguments: +// +// test_case_name: name of the test case +// name: name of the test +// type_param: the name of the test's type parameter, or NULL if +// this is not a typed or a type-parameterized test. +// value_param: text representation of the test's value parameter, +// or NULL if this is not a value-parameterized test. +// code_location: code location where the test is defined +// fixture_class_id: ID of the test fixture class +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +// factory: pointer to the factory that creates a test object. +// The newly created TestInfo instance will assume +// ownership of the factory object. +TestInfo* MakeAndRegisterTestInfo( + const char* test_case_name, + const char* name, + const char* type_param, + const char* value_param, + CodeLocation code_location, + TypeId fixture_class_id, + SetUpTestCaseFunc set_up_tc, + TearDownTestCaseFunc tear_down_tc, + TestFactoryBase* factory) { + TestInfo* const test_info = + new TestInfo(test_case_name, name, type_param, value_param, + code_location, fixture_class_id, factory); + GetUnitTestImpl()->AddTestInfo(set_up_tc, tear_down_tc, test_info); + return test_info; +} + +#if GTEST_HAS_PARAM_TEST +void ReportInvalidTestCaseType(const char* test_case_name, + CodeLocation code_location) { + Message errors; + errors + << "Attempted redefinition of test case " << test_case_name << ".\n" + << "All tests in the same test case must use the same test fixture\n" + << "class. However, in test case " << test_case_name << ", you tried\n" + << "to define a test using a fixture class different from the one\n" + << "used earlier. This can happen if the two fixture classes are\n" + << "from different namespaces and have the same name. You should\n" + << "probably rename one of the classes to put the tests into different\n" + << "test cases."; + + fprintf(stderr, "%s %s", + FormatFileLocation(code_location.file.c_str(), + code_location.line).c_str(), + errors.GetString().c_str()); +} +#endif // GTEST_HAS_PARAM_TEST + +} // namespace internal + +namespace { + +// A predicate that checks the test name of a TestInfo against a known +// value. +// +// This is used for implementation of the TestCase class only. We put +// it in the anonymous namespace to prevent polluting the outer +// namespace. +// +// TestNameIs is copyable. +class TestNameIs { + public: + // Constructor. + // + // TestNameIs has NO default constructor. + explicit TestNameIs(const char* name) + : name_(name) {} + + // Returns true iff the test name of test_info matches name_. + bool operator()(const TestInfo * test_info) const { + return test_info && test_info->name() == name_; + } + + private: + std::string name_; +}; + +} // namespace + +namespace internal { + +// This method expands all parameterized tests registered with macros TEST_P +// and INSTANTIATE_TEST_CASE_P into regular tests and registers those. +// This will be done just once during the program runtime. +void UnitTestImpl::RegisterParameterizedTests() { +#if GTEST_HAS_PARAM_TEST + if (!parameterized_tests_registered_) { + parameterized_test_registry_.RegisterTests(); + parameterized_tests_registered_ = true; + } +#endif +} + +} // namespace internal + +// Creates the test object, runs it, records its result, and then +// deletes it. +void TestInfo::Run() { + if (!should_run_) return; + + // Tells UnitTest where to store test result. + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->set_current_test_info(this); + + TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); + + // Notifies the unit test event listeners that a test is about to start. + repeater->OnTestStart(*this); + + const TimeInMillis start = internal::GetTimeInMillis(); + + impl->os_stack_trace_getter()->UponLeavingGTest(); + + // Creates the test object. + Test* const test = internal::HandleExceptionsInMethodIfSupported( + factory_, &internal::TestFactoryBase::CreateTest, + "the test fixture's constructor"); + + // Runs the test only if the test object was created and its + // constructor didn't generate a fatal failure. + if ((test != NULL) && !Test::HasFatalFailure()) { + // This doesn't throw as all user code that can throw are wrapped into + // exception handling code. + test->Run(); + } + + // Deletes the test object. + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + test, &Test::DeleteSelf_, "the test fixture's destructor"); + + result_.set_elapsed_time(internal::GetTimeInMillis() - start); + + // Notifies the unit test event listener that a test has just finished. + repeater->OnTestEnd(*this); + + // Tells UnitTest to stop associating assertion results to this + // test. + impl->set_current_test_info(NULL); +} + +// class TestCase + +// Gets the number of successful tests in this test case. +int TestCase::successful_test_count() const { + return CountIf(test_info_list_, TestPassed); +} + +// Gets the number of failed tests in this test case. +int TestCase::failed_test_count() const { + return CountIf(test_info_list_, TestFailed); +} + +// Gets the number of disabled tests that will be reported in the XML report. +int TestCase::reportable_disabled_test_count() const { + return CountIf(test_info_list_, TestReportableDisabled); +} + +// Gets the number of disabled tests in this test case. +int TestCase::disabled_test_count() const { + return CountIf(test_info_list_, TestDisabled); +} + +// Gets the number of tests to be printed in the XML report. +int TestCase::reportable_test_count() const { + return CountIf(test_info_list_, TestReportable); +} + +// Get the number of tests in this test case that should run. +int TestCase::test_to_run_count() const { + return CountIf(test_info_list_, ShouldRunTest); +} + +// Gets the number of all tests. +int TestCase::total_test_count() const { + return static_cast(test_info_list_.size()); +} + +// Creates a TestCase with the given name. +// +// Arguments: +// +// name: name of the test case +// a_type_param: the name of the test case's type parameter, or NULL if +// this is not a typed or a type-parameterized test case. +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +TestCase::TestCase(const char* a_name, const char* a_type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc) + : name_(a_name), + type_param_(a_type_param ? new std::string(a_type_param) : NULL), + set_up_tc_(set_up_tc), + tear_down_tc_(tear_down_tc), + should_run_(false), + elapsed_time_(0) { +} + +// Destructor of TestCase. +TestCase::~TestCase() { + // Deletes every Test in the collection. + ForEach(test_info_list_, internal::Delete); +} + +// Returns the i-th test among all the tests. i can range from 0 to +// total_test_count() - 1. If i is not in that range, returns NULL. +const TestInfo* TestCase::GetTestInfo(int i) const { + const int index = GetElementOr(test_indices_, i, -1); + return index < 0 ? NULL : test_info_list_[index]; +} + +// Returns the i-th test among all the tests. i can range from 0 to +// total_test_count() - 1. If i is not in that range, returns NULL. +TestInfo* TestCase::GetMutableTestInfo(int i) { + const int index = GetElementOr(test_indices_, i, -1); + return index < 0 ? NULL : test_info_list_[index]; +} + +// Adds a test to this test case. Will delete the test upon +// destruction of the TestCase object. +void TestCase::AddTestInfo(TestInfo * test_info) { + test_info_list_.push_back(test_info); + test_indices_.push_back(static_cast(test_indices_.size())); +} + +// Runs every test in this TestCase. +void TestCase::Run() { + if (!should_run_) return; + + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->set_current_test_case(this); + + TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); + + repeater->OnTestCaseStart(*this); + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &TestCase::RunSetUpTestCase, "SetUpTestCase()"); + + const internal::TimeInMillis start = internal::GetTimeInMillis(); + for (int i = 0; i < total_test_count(); i++) { + GetMutableTestInfo(i)->Run(); + } + elapsed_time_ = internal::GetTimeInMillis() - start; + + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &TestCase::RunTearDownTestCase, "TearDownTestCase()"); + + repeater->OnTestCaseEnd(*this); + impl->set_current_test_case(NULL); +} + +// Clears the results of all tests in this test case. +void TestCase::ClearResult() { + ad_hoc_test_result_.Clear(); + ForEach(test_info_list_, TestInfo::ClearTestResult); +} + +// Shuffles the tests in this test case. +void TestCase::ShuffleTests(internal::Random* random) { + Shuffle(random, &test_indices_); +} + +// Restores the test order to before the first shuffle. +void TestCase::UnshuffleTests() { + for (size_t i = 0; i < test_indices_.size(); i++) { + test_indices_[i] = static_cast(i); + } +} + +// Formats a countable noun. Depending on its quantity, either the +// singular form or the plural form is used. e.g. +// +// FormatCountableNoun(1, "formula", "formuli") returns "1 formula". +// FormatCountableNoun(5, "book", "books") returns "5 books". +static std::string FormatCountableNoun(int count, + const char * singular_form, + const char * plural_form) { + return internal::StreamableToString(count) + " " + + (count == 1 ? singular_form : plural_form); +} + +// Formats the count of tests. +static std::string FormatTestCount(int test_count) { + return FormatCountableNoun(test_count, "test", "tests"); +} + +// Formats the count of test cases. +static std::string FormatTestCaseCount(int test_case_count) { + return FormatCountableNoun(test_case_count, "test case", "test cases"); +} + +// Converts a TestPartResult::Type enum to human-friendly string +// representation. Both kNonFatalFailure and kFatalFailure are translated +// to "Failure", as the user usually doesn't care about the difference +// between the two when viewing the test result. +static const char * TestPartResultTypeToString(TestPartResult::Type type) { + switch (type) { + case TestPartResult::kSuccess: + return "Success"; + + case TestPartResult::kNonFatalFailure: + case TestPartResult::kFatalFailure: +#ifdef _MSC_VER + return "error: "; +#else + return "Failure\n"; +#endif + default: + return "Unknown result type"; + } +} + +namespace internal { + +// Prints a TestPartResult to an std::string. +static std::string PrintTestPartResultToString( + const TestPartResult& test_part_result) { + return (Message() + << internal::FormatFileLocation(test_part_result.file_name(), + test_part_result.line_number()) + << " " << TestPartResultTypeToString(test_part_result.type()) + << test_part_result.message()).GetString(); +} + +// Prints a TestPartResult. +static void PrintTestPartResult(const TestPartResult& test_part_result) { + const std::string& result = + PrintTestPartResultToString(test_part_result); + printf("%s\n", result.c_str()); + fflush(stdout); + // If the test program runs in Visual Studio or a debugger, the + // following statements add the test part result message to the Output + // window such that the user can double-click on it to jump to the + // corresponding source code location; otherwise they do nothing. +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + // We don't call OutputDebugString*() on Windows Mobile, as printing + // to stdout is done by OutputDebugString() there already - we don't + // want the same message printed twice. + ::OutputDebugStringA(result.c_str()); + ::OutputDebugStringA("\n"); +#endif +} + +// class PrettyUnitTestResultPrinter + +enum GTestColor { + COLOR_DEFAULT, + COLOR_RED, + COLOR_GREEN, + COLOR_YELLOW +}; + +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \ + !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + +// Returns the character attribute for the given color. +WORD GetColorAttribute(GTestColor color) { + switch (color) { + case COLOR_RED: return FOREGROUND_RED; + case COLOR_GREEN: return FOREGROUND_GREEN; + case COLOR_YELLOW: return FOREGROUND_RED | FOREGROUND_GREEN; + default: return 0; + } +} + +#else + +// Returns the ANSI color code for the given color. COLOR_DEFAULT is +// an invalid input. +const char* GetAnsiColorCode(GTestColor color) { + switch (color) { + case COLOR_RED: return "1"; + case COLOR_GREEN: return "2"; + case COLOR_YELLOW: return "3"; + default: return NULL; + }; +} + +#endif // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + +// Returns true iff Google Test should use colors in the output. +bool ShouldUseColor(bool stdout_is_tty) { + const char* const gtest_color = GTEST_FLAG(color).c_str(); + + if (String::CaseInsensitiveCStringEquals(gtest_color, "auto")) { +#if GTEST_OS_WINDOWS + // On Windows the TERM variable is usually not set, but the + // console there does support colors. + return stdout_is_tty; +#else + // On non-Windows platforms, we rely on the TERM variable. + const char* const term = posix::GetEnv("TERM"); + const bool term_supports_color = + String::CStringEquals(term, "xterm") || + String::CStringEquals(term, "xterm-color") || + String::CStringEquals(term, "xterm-256color") || + String::CStringEquals(term, "screen") || + String::CStringEquals(term, "screen-256color") || + String::CStringEquals(term, "tmux") || + String::CStringEquals(term, "tmux-256color") || + String::CStringEquals(term, "rxvt-unicode") || + String::CStringEquals(term, "rxvt-unicode-256color") || + String::CStringEquals(term, "linux") || + String::CStringEquals(term, "cygwin"); + return stdout_is_tty && term_supports_color; +#endif // GTEST_OS_WINDOWS + } + + return String::CaseInsensitiveCStringEquals(gtest_color, "yes") || + String::CaseInsensitiveCStringEquals(gtest_color, "true") || + String::CaseInsensitiveCStringEquals(gtest_color, "t") || + String::CStringEquals(gtest_color, "1"); + // We take "yes", "true", "t", and "1" as meaning "yes". If the + // value is neither one of these nor "auto", we treat it as "no" to + // be conservative. +} + +// Helpers for printing colored strings to stdout. Note that on Windows, we +// cannot simply emit special characters and have the terminal change colors. +// This routine must actually emit the characters rather than return a string +// that would be colored when printed, as can be done on Linux. +void ColoredPrintf(GTestColor color, const char* fmt, ...) { + va_list args; + va_start(args, fmt); + +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS || \ + GTEST_OS_IOS || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT + const bool use_color = AlwaysFalse(); +#else + static const bool in_color_mode = + ShouldUseColor(posix::IsATTY(posix::FileNo(stdout)) != 0); + const bool use_color = in_color_mode && (color != COLOR_DEFAULT); +#endif // GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS + // The '!= 0' comparison is necessary to satisfy MSVC 7.1. + + if (!use_color) { + vprintf(fmt, args); + va_end(args); + return; + } + +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \ + !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + const HANDLE stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE); + + // Gets the current text color. + CONSOLE_SCREEN_BUFFER_INFO buffer_info; + GetConsoleScreenBufferInfo(stdout_handle, &buffer_info); + const WORD old_color_attrs = buffer_info.wAttributes; + + // We need to flush the stream buffers into the console before each + // SetConsoleTextAttribute call lest it affect the text that is already + // printed but has not yet reached the console. + fflush(stdout); + SetConsoleTextAttribute(stdout_handle, + GetColorAttribute(color) | FOREGROUND_INTENSITY); + vprintf(fmt, args); + + fflush(stdout); + // Restores the text color. + SetConsoleTextAttribute(stdout_handle, old_color_attrs); +#else + printf("\033[0;3%sm", GetAnsiColorCode(color)); + vprintf(fmt, args); + printf("\033[m"); // Resets the terminal to default. +#endif // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + va_end(args); +} + +// Text printed in Google Test's text output and --gunit_list_tests +// output to label the type parameter and value parameter for a test. +static const char kTypeParamLabel[] = "TypeParam"; +static const char kValueParamLabel[] = "GetParam()"; + +void PrintFullTestCommentIfPresent(const TestInfo& test_info) { + const char* const type_param = test_info.type_param(); + const char* const value_param = test_info.value_param(); + + if (type_param != NULL || value_param != NULL) { + printf(", where "); + if (type_param != NULL) { + printf("%s = %s", kTypeParamLabel, type_param); + if (value_param != NULL) + printf(" and "); + } + if (value_param != NULL) { + printf("%s = %s", kValueParamLabel, value_param); + } + } +} + +// This class implements the TestEventListener interface. +// +// Class PrettyUnitTestResultPrinter is copyable. +class PrettyUnitTestResultPrinter : public TestEventListener { + public: + PrettyUnitTestResultPrinter() {} + static void PrintTestName(const char * test_case, const char * test) { + printf("%s.%s", test_case, test); + } + + // The following methods override what's in the TestEventListener class. + virtual void OnTestProgramStart(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationStart(const UnitTest& unit_test, int iteration); + virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test); + virtual void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestCaseStart(const TestCase& test_case); + virtual void OnTestStart(const TestInfo& test_info); + virtual void OnTestPartResult(const TestPartResult& result); + virtual void OnTestEnd(const TestInfo& test_info); + virtual void OnTestCaseEnd(const TestCase& test_case); + virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test); + virtual void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + virtual void OnTestProgramEnd(const UnitTest& /*unit_test*/) {} + + private: + static void PrintFailedTests(const UnitTest& unit_test); +}; + + // Fired before each iteration of tests starts. +void PrettyUnitTestResultPrinter::OnTestIterationStart( + const UnitTest& unit_test, int iteration) { + if (GTEST_FLAG(repeat) != 1) + printf("\nRepeating all tests (iteration %d) . . .\n\n", iteration + 1); + + const char* const filter = GTEST_FLAG(filter).c_str(); + + // Prints the filter if it's not *. This reminds the user that some + // tests may be skipped. + if (!String::CStringEquals(filter, kUniversalFilter)) { + ColoredPrintf(COLOR_YELLOW, + "Note: %s filter = %s\n", GTEST_NAME_, filter); + } + + if (internal::ShouldShard(kTestTotalShards, kTestShardIndex, false)) { + const Int32 shard_index = Int32FromEnvOrDie(kTestShardIndex, -1); + ColoredPrintf(COLOR_YELLOW, + "Note: This is test shard %d of %s.\n", + static_cast(shard_index) + 1, + internal::posix::GetEnv(kTestTotalShards)); + } + + if (GTEST_FLAG(shuffle)) { + ColoredPrintf(COLOR_YELLOW, + "Note: Randomizing tests' orders with a seed of %d .\n", + unit_test.random_seed()); + } + + ColoredPrintf(COLOR_GREEN, "[==========] "); + printf("Running %s from %s.\n", + FormatTestCount(unit_test.test_to_run_count()).c_str(), + FormatTestCaseCount(unit_test.test_case_to_run_count()).c_str()); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnEnvironmentsSetUpStart( + const UnitTest& /*unit_test*/) { + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("Global test environment set-up.\n"); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestCaseStart(const TestCase& test_case) { + const std::string counts = + FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("%s from %s", counts.c_str(), test_case.name()); + if (test_case.type_param() == NULL) { + printf("\n"); + } else { + printf(", where %s = %s\n", kTypeParamLabel, test_case.type_param()); + } + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestStart(const TestInfo& test_info) { + ColoredPrintf(COLOR_GREEN, "[ RUN ] "); + PrintTestName(test_info.test_case_name(), test_info.name()); + printf("\n"); + fflush(stdout); +} + +// Called after an assertion failure. +void PrettyUnitTestResultPrinter::OnTestPartResult( + const TestPartResult& result) { + // If the test part succeeded, we don't need to do anything. + if (result.type() == TestPartResult::kSuccess) + return; + + // Print failure message from the assertion (e.g. expected this and got that). + PrintTestPartResult(result); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestEnd(const TestInfo& test_info) { + if (test_info.result()->Passed()) { + ColoredPrintf(COLOR_GREEN, "[ OK ] "); + } else { + ColoredPrintf(COLOR_RED, "[ FAILED ] "); + } + PrintTestName(test_info.test_case_name(), test_info.name()); + if (test_info.result()->Failed()) + PrintFullTestCommentIfPresent(test_info); + + if (GTEST_FLAG(print_time)) { + printf(" (%s ms)\n", internal::StreamableToString( + test_info.result()->elapsed_time()).c_str()); + } else { + printf("\n"); + } + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestCaseEnd(const TestCase& test_case) { + if (!GTEST_FLAG(print_time)) return; + + const std::string counts = + FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("%s from %s (%s ms total)\n\n", + counts.c_str(), test_case.name(), + internal::StreamableToString(test_case.elapsed_time()).c_str()); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnEnvironmentsTearDownStart( + const UnitTest& /*unit_test*/) { + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("Global test environment tear-down\n"); + fflush(stdout); +} + +// Internal helper for printing the list of failed tests. +void PrettyUnitTestResultPrinter::PrintFailedTests(const UnitTest& unit_test) { + const int failed_test_count = unit_test.failed_test_count(); + if (failed_test_count == 0) { + return; + } + + for (int i = 0; i < unit_test.total_test_case_count(); ++i) { + const TestCase& test_case = *unit_test.GetTestCase(i); + if (!test_case.should_run() || (test_case.failed_test_count() == 0)) { + continue; + } + for (int j = 0; j < test_case.total_test_count(); ++j) { + const TestInfo& test_info = *test_case.GetTestInfo(j); + if (!test_info.should_run() || test_info.result()->Passed()) { + continue; + } + ColoredPrintf(COLOR_RED, "[ FAILED ] "); + printf("%s.%s", test_case.name(), test_info.name()); + PrintFullTestCommentIfPresent(test_info); + printf("\n"); + } + } +} + +void PrettyUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + ColoredPrintf(COLOR_GREEN, "[==========] "); + printf("%s from %s ran.", + FormatTestCount(unit_test.test_to_run_count()).c_str(), + FormatTestCaseCount(unit_test.test_case_to_run_count()).c_str()); + if (GTEST_FLAG(print_time)) { + printf(" (%s ms total)", + internal::StreamableToString(unit_test.elapsed_time()).c_str()); + } + printf("\n"); + ColoredPrintf(COLOR_GREEN, "[ PASSED ] "); + printf("%s.\n", FormatTestCount(unit_test.successful_test_count()).c_str()); + + int num_failures = unit_test.failed_test_count(); + if (!unit_test.Passed()) { + const int failed_test_count = unit_test.failed_test_count(); + ColoredPrintf(COLOR_RED, "[ FAILED ] "); + printf("%s, listed below:\n", FormatTestCount(failed_test_count).c_str()); + PrintFailedTests(unit_test); + printf("\n%2d FAILED %s\n", num_failures, + num_failures == 1 ? "TEST" : "TESTS"); + } + + int num_disabled = unit_test.reportable_disabled_test_count(); + if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) { + if (!num_failures) { + printf("\n"); // Add a spacer if no FAILURE banner is displayed. + } + ColoredPrintf(COLOR_YELLOW, + " YOU HAVE %d DISABLED %s\n\n", + num_disabled, + num_disabled == 1 ? "TEST" : "TESTS"); + } + // Ensure that Google Test output is printed before, e.g., heapchecker output. + fflush(stdout); +} + +// End PrettyUnitTestResultPrinter + +// class TestEventRepeater +// +// This class forwards events to other event listeners. +class TestEventRepeater : public TestEventListener { + public: + TestEventRepeater() : forwarding_enabled_(true) {} + virtual ~TestEventRepeater(); + void Append(TestEventListener *listener); + TestEventListener* Release(TestEventListener* listener); + + // Controls whether events will be forwarded to listeners_. Set to false + // in death test child processes. + bool forwarding_enabled() const { return forwarding_enabled_; } + void set_forwarding_enabled(bool enable) { forwarding_enabled_ = enable; } + + virtual void OnTestProgramStart(const UnitTest& unit_test); + virtual void OnTestIterationStart(const UnitTest& unit_test, int iteration); + virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test); + virtual void OnEnvironmentsSetUpEnd(const UnitTest& unit_test); + virtual void OnTestCaseStart(const TestCase& test_case); + virtual void OnTestStart(const TestInfo& test_info); + virtual void OnTestPartResult(const TestPartResult& result); + virtual void OnTestEnd(const TestInfo& test_info); + virtual void OnTestCaseEnd(const TestCase& test_case); + virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test); + virtual void OnEnvironmentsTearDownEnd(const UnitTest& unit_test); + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + virtual void OnTestProgramEnd(const UnitTest& unit_test); + + private: + // Controls whether events will be forwarded to listeners_. Set to false + // in death test child processes. + bool forwarding_enabled_; + // The list of listeners that receive events. + std::vector listeners_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestEventRepeater); +}; + +TestEventRepeater::~TestEventRepeater() { + ForEach(listeners_, Delete); +} + +void TestEventRepeater::Append(TestEventListener *listener) { + listeners_.push_back(listener); +} + +// TODO(vladl@google.com): Factor the search functionality into Vector::Find. +TestEventListener* TestEventRepeater::Release(TestEventListener *listener) { + for (size_t i = 0; i < listeners_.size(); ++i) { + if (listeners_[i] == listener) { + listeners_.erase(listeners_.begin() + i); + return listener; + } + } + + return NULL; +} + +// Since most methods are very similar, use macros to reduce boilerplate. +// This defines a member that forwards the call to all listeners. +#define GTEST_REPEATER_METHOD_(Name, Type) \ +void TestEventRepeater::Name(const Type& parameter) { \ + if (forwarding_enabled_) { \ + for (size_t i = 0; i < listeners_.size(); i++) { \ + listeners_[i]->Name(parameter); \ + } \ + } \ +} +// This defines a member that forwards the call to all listeners in reverse +// order. +#define GTEST_REVERSE_REPEATER_METHOD_(Name, Type) \ +void TestEventRepeater::Name(const Type& parameter) { \ + if (forwarding_enabled_) { \ + for (int i = static_cast(listeners_.size()) - 1; i >= 0; i--) { \ + listeners_[i]->Name(parameter); \ + } \ + } \ +} + +GTEST_REPEATER_METHOD_(OnTestProgramStart, UnitTest) +GTEST_REPEATER_METHOD_(OnEnvironmentsSetUpStart, UnitTest) +GTEST_REPEATER_METHOD_(OnTestCaseStart, TestCase) +GTEST_REPEATER_METHOD_(OnTestStart, TestInfo) +GTEST_REPEATER_METHOD_(OnTestPartResult, TestPartResult) +GTEST_REPEATER_METHOD_(OnEnvironmentsTearDownStart, UnitTest) +GTEST_REVERSE_REPEATER_METHOD_(OnEnvironmentsSetUpEnd, UnitTest) +GTEST_REVERSE_REPEATER_METHOD_(OnEnvironmentsTearDownEnd, UnitTest) +GTEST_REVERSE_REPEATER_METHOD_(OnTestEnd, TestInfo) +GTEST_REVERSE_REPEATER_METHOD_(OnTestCaseEnd, TestCase) +GTEST_REVERSE_REPEATER_METHOD_(OnTestProgramEnd, UnitTest) + +#undef GTEST_REPEATER_METHOD_ +#undef GTEST_REVERSE_REPEATER_METHOD_ + +void TestEventRepeater::OnTestIterationStart(const UnitTest& unit_test, + int iteration) { + if (forwarding_enabled_) { + for (size_t i = 0; i < listeners_.size(); i++) { + listeners_[i]->OnTestIterationStart(unit_test, iteration); + } + } +} + +void TestEventRepeater::OnTestIterationEnd(const UnitTest& unit_test, + int iteration) { + if (forwarding_enabled_) { + for (int i = static_cast(listeners_.size()) - 1; i >= 0; i--) { + listeners_[i]->OnTestIterationEnd(unit_test, iteration); + } + } +} + +// End TestEventRepeater + +// This class generates an XML output file. +class XmlUnitTestResultPrinter : public EmptyTestEventListener { + public: + explicit XmlUnitTestResultPrinter(const char* output_file); + + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + + private: + // Is c a whitespace character that is normalized to a space character + // when it appears in an XML attribute value? + static bool IsNormalizableWhitespace(char c) { + return c == 0x9 || c == 0xA || c == 0xD; + } + + // May c appear in a well-formed XML document? + static bool IsValidXmlCharacter(char c) { + return IsNormalizableWhitespace(c) || c >= 0x20; + } + + // Returns an XML-escaped copy of the input string str. If + // is_attribute is true, the text is meant to appear as an attribute + // value, and normalizable whitespace is preserved by replacing it + // with character references. + static std::string EscapeXml(const std::string& str, bool is_attribute); + + // Returns the given string with all characters invalid in XML removed. + static std::string RemoveInvalidXmlCharacters(const std::string& str); + + // Convenience wrapper around EscapeXml when str is an attribute value. + static std::string EscapeXmlAttribute(const std::string& str) { + return EscapeXml(str, true); + } + + // Convenience wrapper around EscapeXml when str is not an attribute value. + static std::string EscapeXmlText(const char* str) { + return EscapeXml(str, false); + } + + // Verifies that the given attribute belongs to the given element and + // streams the attribute as XML. + static void OutputXmlAttribute(std::ostream* stream, + const std::string& element_name, + const std::string& name, + const std::string& value); + + // Streams an XML CDATA section, escaping invalid CDATA sequences as needed. + static void OutputXmlCDataSection(::std::ostream* stream, const char* data); + + // Streams an XML representation of a TestInfo object. + static void OutputXmlTestInfo(::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info); + + // Prints an XML representation of a TestCase object + static void PrintXmlTestCase(::std::ostream* stream, + const TestCase& test_case); + + // Prints an XML summary of unit_test to output stream out. + static void PrintXmlUnitTest(::std::ostream* stream, + const UnitTest& unit_test); + + // Produces a string representing the test properties in a result as space + // delimited XML attributes based on the property key="value" pairs. + // When the std::string is not empty, it includes a space at the beginning, + // to delimit this attribute from prior attributes. + static std::string TestPropertiesAsXmlAttributes(const TestResult& result); + + // The output file. + const std::string output_file_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(XmlUnitTestResultPrinter); +}; + +// Creates a new XmlUnitTestResultPrinter. +XmlUnitTestResultPrinter::XmlUnitTestResultPrinter(const char* output_file) + : output_file_(output_file) { + if (output_file_.c_str() == NULL || output_file_.empty()) { + fprintf(stderr, "XML output file may not be null\n"); + fflush(stderr); + exit(EXIT_FAILURE); + } +} + +// Called after the unit test ends. +void XmlUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + FILE* xmlout = NULL; + FilePath output_file(output_file_); + FilePath output_dir(output_file.RemoveFileName()); + + if (output_dir.CreateDirectoriesRecursively()) { + xmlout = posix::FOpen(output_file_.c_str(), "w"); + } + if (xmlout == NULL) { + // TODO(wan): report the reason of the failure. + // + // We don't do it for now as: + // + // 1. There is no urgent need for it. + // 2. It's a bit involved to make the errno variable thread-safe on + // all three operating systems (Linux, Windows, and Mac OS). + // 3. To interpret the meaning of errno in a thread-safe way, + // we need the strerror_r() function, which is not available on + // Windows. + fprintf(stderr, + "Unable to open file \"%s\"\n", + output_file_.c_str()); + fflush(stderr); + exit(EXIT_FAILURE); + } + std::stringstream stream; + PrintXmlUnitTest(&stream, unit_test); + fprintf(xmlout, "%s", StringStreamToString(&stream).c_str()); + fclose(xmlout); +} + +// Returns an XML-escaped copy of the input string str. If is_attribute +// is true, the text is meant to appear as an attribute value, and +// normalizable whitespace is preserved by replacing it with character +// references. +// +// Invalid XML characters in str, if any, are stripped from the output. +// It is expected that most, if not all, of the text processed by this +// module will consist of ordinary English text. +// If this module is ever modified to produce version 1.1 XML output, +// most invalid characters can be retained using character references. +// TODO(wan): It might be nice to have a minimally invasive, human-readable +// escaping scheme for invalid characters, rather than dropping them. +std::string XmlUnitTestResultPrinter::EscapeXml( + const std::string& str, bool is_attribute) { + Message m; + + for (size_t i = 0; i < str.size(); ++i) { + const char ch = str[i]; + switch (ch) { + case '<': + m << "<"; + break; + case '>': + m << ">"; + break; + case '&': + m << "&"; + break; + case '\'': + if (is_attribute) + m << "'"; + else + m << '\''; + break; + case '"': + if (is_attribute) + m << """; + else + m << '"'; + break; + default: + if (IsValidXmlCharacter(ch)) { + if (is_attribute && IsNormalizableWhitespace(ch)) + m << "&#x" << String::FormatByte(static_cast(ch)) + << ";"; + else + m << ch; + } + break; + } + } + + return m.GetString(); +} + +// Returns the given string with all characters invalid in XML removed. +// Currently invalid characters are dropped from the string. An +// alternative is to replace them with certain characters such as . or ?. +std::string XmlUnitTestResultPrinter::RemoveInvalidXmlCharacters( + const std::string& str) { + std::string output; + output.reserve(str.size()); + for (std::string::const_iterator it = str.begin(); it != str.end(); ++it) + if (IsValidXmlCharacter(*it)) + output.push_back(*it); + + return output; +} + +// The following routines generate an XML representation of a UnitTest +// object. +// +// This is how Google Test concepts map to the DTD: +// +// <-- corresponds to a UnitTest object +// <-- corresponds to a TestCase object +// <-- corresponds to a TestInfo object +// ... +// ... +// ... +// <-- individual assertion failures +// +// +// + +// Formats the given time in milliseconds as seconds. +std::string FormatTimeInMillisAsSeconds(TimeInMillis ms) { + ::std::stringstream ss; + ss << (static_cast(ms) * 1e-3); + return ss.str(); +} + +static bool PortableLocaltime(time_t seconds, struct tm* out) { +#if defined(_MSC_VER) + return localtime_s(out, &seconds) == 0; +#elif defined(__MINGW32__) || defined(__MINGW64__) + // MINGW provides neither localtime_r nor localtime_s, but uses + // Windows' localtime(), which has a thread-local tm buffer. + struct tm* tm_ptr = localtime(&seconds); // NOLINT + if (tm_ptr == NULL) + return false; + *out = *tm_ptr; + return true; +#else + return localtime_r(&seconds, out) != NULL; +#endif +} + +// Converts the given epoch time in milliseconds to a date string in the ISO +// 8601 format, without the timezone information. +std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms) { + struct tm time_struct; + if (!PortableLocaltime(static_cast(ms / 1000), &time_struct)) + return ""; + // YYYY-MM-DDThh:mm:ss + return StreamableToString(time_struct.tm_year + 1900) + "-" + + String::FormatIntWidth2(time_struct.tm_mon + 1) + "-" + + String::FormatIntWidth2(time_struct.tm_mday) + "T" + + String::FormatIntWidth2(time_struct.tm_hour) + ":" + + String::FormatIntWidth2(time_struct.tm_min) + ":" + + String::FormatIntWidth2(time_struct.tm_sec); +} + +// Streams an XML CDATA section, escaping invalid CDATA sequences as needed. +void XmlUnitTestResultPrinter::OutputXmlCDataSection(::std::ostream* stream, + const char* data) { + const char* segment = data; + *stream << ""); + if (next_segment != NULL) { + stream->write( + segment, static_cast(next_segment - segment)); + *stream << "]]>]]>"); + } else { + *stream << segment; + break; + } + } + *stream << "]]>"; +} + +void XmlUnitTestResultPrinter::OutputXmlAttribute( + std::ostream* stream, + const std::string& element_name, + const std::string& name, + const std::string& value) { + const std::vector& allowed_names = + GetReservedAttributesForElement(element_name); + + GTEST_CHECK_(std::find(allowed_names.begin(), allowed_names.end(), name) != + allowed_names.end()) + << "Attribute " << name << " is not allowed for element <" << element_name + << ">."; + + *stream << " " << name << "=\"" << EscapeXmlAttribute(value) << "\""; +} + +// Prints an XML representation of a TestInfo object. +// TODO(wan): There is also value in printing properties with the plain printer. +void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info) { + const TestResult& result = *test_info.result(); + const std::string kTestcase = "testcase"; + + *stream << " \n"; + } + const string location = internal::FormatCompilerIndependentFileLocation( + part.file_name(), part.line_number()); + const string summary = location + "\n" + part.summary(); + *stream << " "; + const string detail = location + "\n" + part.message(); + OutputXmlCDataSection(stream, RemoveInvalidXmlCharacters(detail).c_str()); + *stream << "\n"; + } + } + + if (failures == 0) + *stream << " />\n"; + else + *stream << " \n"; +} + +// Prints an XML representation of a TestCase object +void XmlUnitTestResultPrinter::PrintXmlTestCase(std::ostream* stream, + const TestCase& test_case) { + const std::string kTestsuite = "testsuite"; + *stream << " <" << kTestsuite; + OutputXmlAttribute(stream, kTestsuite, "name", test_case.name()); + OutputXmlAttribute(stream, kTestsuite, "tests", + StreamableToString(test_case.reportable_test_count())); + OutputXmlAttribute(stream, kTestsuite, "failures", + StreamableToString(test_case.failed_test_count())); + OutputXmlAttribute( + stream, kTestsuite, "disabled", + StreamableToString(test_case.reportable_disabled_test_count())); + OutputXmlAttribute(stream, kTestsuite, "errors", "0"); + OutputXmlAttribute(stream, kTestsuite, "time", + FormatTimeInMillisAsSeconds(test_case.elapsed_time())); + *stream << TestPropertiesAsXmlAttributes(test_case.ad_hoc_test_result()) + << ">\n"; + + for (int i = 0; i < test_case.total_test_count(); ++i) { + if (test_case.GetTestInfo(i)->is_reportable()) + OutputXmlTestInfo(stream, test_case.name(), *test_case.GetTestInfo(i)); + } + *stream << " \n"; +} + +// Prints an XML summary of unit_test to output stream out. +void XmlUnitTestResultPrinter::PrintXmlUnitTest(std::ostream* stream, + const UnitTest& unit_test) { + const std::string kTestsuites = "testsuites"; + + *stream << "\n"; + *stream << "<" << kTestsuites; + + OutputXmlAttribute(stream, kTestsuites, "tests", + StreamableToString(unit_test.reportable_test_count())); + OutputXmlAttribute(stream, kTestsuites, "failures", + StreamableToString(unit_test.failed_test_count())); + OutputXmlAttribute( + stream, kTestsuites, "disabled", + StreamableToString(unit_test.reportable_disabled_test_count())); + OutputXmlAttribute(stream, kTestsuites, "errors", "0"); + OutputXmlAttribute( + stream, kTestsuites, "timestamp", + FormatEpochTimeInMillisAsIso8601(unit_test.start_timestamp())); + OutputXmlAttribute(stream, kTestsuites, "time", + FormatTimeInMillisAsSeconds(unit_test.elapsed_time())); + + if (GTEST_FLAG(shuffle)) { + OutputXmlAttribute(stream, kTestsuites, "random_seed", + StreamableToString(unit_test.random_seed())); + } + + *stream << TestPropertiesAsXmlAttributes(unit_test.ad_hoc_test_result()); + + OutputXmlAttribute(stream, kTestsuites, "name", "AllTests"); + *stream << ">\n"; + + for (int i = 0; i < unit_test.total_test_case_count(); ++i) { + if (unit_test.GetTestCase(i)->reportable_test_count() > 0) + PrintXmlTestCase(stream, *unit_test.GetTestCase(i)); + } + *stream << "\n"; +} + +// Produces a string representing the test properties in a result as space +// delimited XML attributes based on the property key="value" pairs. +std::string XmlUnitTestResultPrinter::TestPropertiesAsXmlAttributes( + const TestResult& result) { + Message attributes; + for (int i = 0; i < result.test_property_count(); ++i) { + const TestProperty& property = result.GetTestProperty(i); + attributes << " " << property.key() << "=" + << "\"" << EscapeXmlAttribute(property.value()) << "\""; + } + return attributes.GetString(); +} + +// End XmlUnitTestResultPrinter + +#if GTEST_CAN_STREAM_RESULTS_ + +// Checks if str contains '=', '&', '%' or '\n' characters. If yes, +// replaces them by "%xx" where xx is their hexadecimal value. For +// example, replaces "=" with "%3D". This algorithm is O(strlen(str)) +// in both time and space -- important as the input str may contain an +// arbitrarily long test failure message and stack trace. +string StreamingListener::UrlEncode(const char* str) { + string result; + result.reserve(strlen(str) + 1); + for (char ch = *str; ch != '\0'; ch = *++str) { + switch (ch) { + case '%': + case '=': + case '&': + case '\n': + result.append("%" + String::FormatByte(static_cast(ch))); + break; + default: + result.push_back(ch); + break; + } + } + return result; +} + +void StreamingListener::SocketWriter::MakeConnection() { + GTEST_CHECK_(sockfd_ == -1) + << "MakeConnection() can't be called when there is already a connection."; + + addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; // To allow both IPv4 and IPv6 addresses. + hints.ai_socktype = SOCK_STREAM; + addrinfo* servinfo = NULL; + + // Use the getaddrinfo() to get a linked list of IP addresses for + // the given host name. + const int error_num = getaddrinfo( + host_name_.c_str(), port_num_.c_str(), &hints, &servinfo); + if (error_num != 0) { + GTEST_LOG_(WARNING) << "stream_result_to: getaddrinfo() failed: " + << gai_strerror(error_num); + } + + // Loop through all the results and connect to the first we can. + for (addrinfo* cur_addr = servinfo; sockfd_ == -1 && cur_addr != NULL; + cur_addr = cur_addr->ai_next) { + sockfd_ = socket( + cur_addr->ai_family, cur_addr->ai_socktype, cur_addr->ai_protocol); + if (sockfd_ != -1) { + // Connect the client socket to the server socket. + if (connect(sockfd_, cur_addr->ai_addr, cur_addr->ai_addrlen) == -1) { + close(sockfd_); + sockfd_ = -1; + } + } + } + + freeaddrinfo(servinfo); // all done with this structure + + if (sockfd_ == -1) { + GTEST_LOG_(WARNING) << "stream_result_to: failed to connect to " + << host_name_ << ":" << port_num_; + } +} + +// End of class Streaming Listener +#endif // GTEST_CAN_STREAM_RESULTS__ + +// Class ScopedTrace + +// Pushes the given source file location and message onto a per-thread +// trace stack maintained by Google Test. +ScopedTrace::ScopedTrace(const char* file, int line, const Message& message) + GTEST_LOCK_EXCLUDED_(&UnitTest::mutex_) { + TraceInfo trace; + trace.file = file; + trace.line = line; + trace.message = message.GetString(); + + UnitTest::GetInstance()->PushGTestTrace(trace); +} + +// Pops the info pushed by the c'tor. +ScopedTrace::~ScopedTrace() + GTEST_LOCK_EXCLUDED_(&UnitTest::mutex_) { + UnitTest::GetInstance()->PopGTestTrace(); +} + + +// class OsStackTraceGetter + +const char* const OsStackTraceGetterInterface::kElidedFramesMarker = + "... " GTEST_NAME_ " internal frames ..."; + +string OsStackTraceGetter::CurrentStackTrace(int /*max_depth*/, + int /*skip_count*/) { + return ""; +} + +void OsStackTraceGetter::UponLeavingGTest() {} + +// A helper class that creates the premature-exit file in its +// constructor and deletes the file in its destructor. +class ScopedPrematureExitFile { + public: + explicit ScopedPrematureExitFile(const char* premature_exit_filepath) + : premature_exit_filepath_(premature_exit_filepath) { + // If a path to the premature-exit file is specified... + if (premature_exit_filepath != NULL && *premature_exit_filepath != '\0') { + // create the file with a single "0" character in it. I/O + // errors are ignored as there's nothing better we can do and we + // don't want to fail the test because of this. + FILE* pfile = posix::FOpen(premature_exit_filepath, "w"); + fwrite("0", 1, 1, pfile); + fclose(pfile); + } + } + + ~ScopedPrematureExitFile() { + if (premature_exit_filepath_ != NULL && *premature_exit_filepath_ != '\0') { + remove(premature_exit_filepath_); + } + } + + private: + const char* const premature_exit_filepath_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedPrematureExitFile); +}; + +} // namespace internal + +// class TestEventListeners + +TestEventListeners::TestEventListeners() + : repeater_(new internal::TestEventRepeater()), + default_result_printer_(NULL), + default_xml_generator_(NULL) { +} + +TestEventListeners::~TestEventListeners() { delete repeater_; } + +// Returns the standard listener responsible for the default console +// output. Can be removed from the listeners list to shut down default +// console output. Note that removing this object from the listener list +// with Release transfers its ownership to the user. +void TestEventListeners::Append(TestEventListener* listener) { + repeater_->Append(listener); +} + +// Removes the given event listener from the list and returns it. It then +// becomes the caller's responsibility to delete the listener. Returns +// NULL if the listener is not found in the list. +TestEventListener* TestEventListeners::Release(TestEventListener* listener) { + if (listener == default_result_printer_) + default_result_printer_ = NULL; + else if (listener == default_xml_generator_) + default_xml_generator_ = NULL; + return repeater_->Release(listener); +} + +// Returns repeater that broadcasts the TestEventListener events to all +// subscribers. +TestEventListener* TestEventListeners::repeater() { return repeater_; } + +// Sets the default_result_printer attribute to the provided listener. +// The listener is also added to the listener list and previous +// default_result_printer is removed from it and deleted. The listener can +// also be NULL in which case it will not be added to the list. Does +// nothing if the previous and the current listener objects are the same. +void TestEventListeners::SetDefaultResultPrinter(TestEventListener* listener) { + if (default_result_printer_ != listener) { + // It is an error to pass this method a listener that is already in the + // list. + delete Release(default_result_printer_); + default_result_printer_ = listener; + if (listener != NULL) + Append(listener); + } +} + +// Sets the default_xml_generator attribute to the provided listener. The +// listener is also added to the listener list and previous +// default_xml_generator is removed from it and deleted. The listener can +// also be NULL in which case it will not be added to the list. Does +// nothing if the previous and the current listener objects are the same. +void TestEventListeners::SetDefaultXmlGenerator(TestEventListener* listener) { + if (default_xml_generator_ != listener) { + // It is an error to pass this method a listener that is already in the + // list. + delete Release(default_xml_generator_); + default_xml_generator_ = listener; + if (listener != NULL) + Append(listener); + } +} + +// Controls whether events will be forwarded by the repeater to the +// listeners in the list. +bool TestEventListeners::EventForwardingEnabled() const { + return repeater_->forwarding_enabled(); +} + +void TestEventListeners::SuppressEventForwarding() { + repeater_->set_forwarding_enabled(false); +} + +// class UnitTest + +// Gets the singleton UnitTest object. The first time this method is +// called, a UnitTest object is constructed and returned. Consecutive +// calls will return the same object. +// +// We don't protect this under mutex_ as a user is not supposed to +// call this before main() starts, from which point on the return +// value will never change. +UnitTest* UnitTest::GetInstance() { + // When compiled with MSVC 7.1 in optimized mode, destroying the + // UnitTest object upon exiting the program messes up the exit code, + // causing successful tests to appear failed. We have to use a + // different implementation in this case to bypass the compiler bug. + // This implementation makes the compiler happy, at the cost of + // leaking the UnitTest object. + + // CodeGear C++Builder insists on a public destructor for the + // default implementation. Use this implementation to keep good OO + // design with private destructor. + +#if (_MSC_VER == 1310 && !defined(_DEBUG)) || defined(__BORLANDC__) + static UnitTest* const instance = new UnitTest; + return instance; +#else + static UnitTest instance; + return &instance; +#endif // (_MSC_VER == 1310 && !defined(_DEBUG)) || defined(__BORLANDC__) +} + +// Gets the number of successful test cases. +int UnitTest::successful_test_case_count() const { + return impl()->successful_test_case_count(); +} + +// Gets the number of failed test cases. +int UnitTest::failed_test_case_count() const { + return impl()->failed_test_case_count(); +} + +// Gets the number of all test cases. +int UnitTest::total_test_case_count() const { + return impl()->total_test_case_count(); +} + +// Gets the number of all test cases that contain at least one test +// that should run. +int UnitTest::test_case_to_run_count() const { + return impl()->test_case_to_run_count(); +} + +// Gets the number of successful tests. +int UnitTest::successful_test_count() const { + return impl()->successful_test_count(); +} + +// Gets the number of failed tests. +int UnitTest::failed_test_count() const { return impl()->failed_test_count(); } + +// Gets the number of disabled tests that will be reported in the XML report. +int UnitTest::reportable_disabled_test_count() const { + return impl()->reportable_disabled_test_count(); +} + +// Gets the number of disabled tests. +int UnitTest::disabled_test_count() const { + return impl()->disabled_test_count(); +} + +// Gets the number of tests to be printed in the XML report. +int UnitTest::reportable_test_count() const { + return impl()->reportable_test_count(); +} + +// Gets the number of all tests. +int UnitTest::total_test_count() const { return impl()->total_test_count(); } + +// Gets the number of tests that should run. +int UnitTest::test_to_run_count() const { return impl()->test_to_run_count(); } + +// Gets the time of the test program start, in ms from the start of the +// UNIX epoch. +internal::TimeInMillis UnitTest::start_timestamp() const { + return impl()->start_timestamp(); +} + +// Gets the elapsed time, in milliseconds. +internal::TimeInMillis UnitTest::elapsed_time() const { + return impl()->elapsed_time(); +} + +// Returns true iff the unit test passed (i.e. all test cases passed). +bool UnitTest::Passed() const { return impl()->Passed(); } + +// Returns true iff the unit test failed (i.e. some test case failed +// or something outside of all tests failed). +bool UnitTest::Failed() const { return impl()->Failed(); } + +// Gets the i-th test case among all the test cases. i can range from 0 to +// total_test_case_count() - 1. If i is not in that range, returns NULL. +const TestCase* UnitTest::GetTestCase(int i) const { + return impl()->GetTestCase(i); +} + +// Returns the TestResult containing information on test failures and +// properties logged outside of individual test cases. +const TestResult& UnitTest::ad_hoc_test_result() const { + return *impl()->ad_hoc_test_result(); +} + +// Gets the i-th test case among all the test cases. i can range from 0 to +// total_test_case_count() - 1. If i is not in that range, returns NULL. +TestCase* UnitTest::GetMutableTestCase(int i) { + return impl()->GetMutableTestCase(i); +} + +// Returns the list of event listeners that can be used to track events +// inside Google Test. +TestEventListeners& UnitTest::listeners() { + return *impl()->listeners(); +} + +// Registers and returns a global test environment. When a test +// program is run, all global test environments will be set-up in the +// order they were registered. After all tests in the program have +// finished, all global test environments will be torn-down in the +// *reverse* order they were registered. +// +// The UnitTest object takes ownership of the given environment. +// +// We don't protect this under mutex_, as we only support calling it +// from the main thread. +Environment* UnitTest::AddEnvironment(Environment* env) { + if (env == NULL) { + return NULL; + } + + impl_->environments().push_back(env); + return env; +} + +// Adds a TestPartResult to the current TestResult object. All Google Test +// assertion macros (e.g. ASSERT_TRUE, EXPECT_EQ, etc) eventually call +// this to report their results. The user code should use the +// assertion macros instead of calling this directly. +void UnitTest::AddTestPartResult( + TestPartResult::Type result_type, + const char* file_name, + int line_number, + const std::string& message, + const std::string& os_stack_trace) GTEST_LOCK_EXCLUDED_(mutex_) { + Message msg; + msg << message; + + internal::MutexLock lock(&mutex_); + if (impl_->gtest_trace_stack().size() > 0) { + msg << "\n" << GTEST_NAME_ << " trace:"; + + for (int i = static_cast(impl_->gtest_trace_stack().size()); + i > 0; --i) { + const internal::TraceInfo& trace = impl_->gtest_trace_stack()[i - 1]; + msg << "\n" << internal::FormatFileLocation(trace.file, trace.line) + << " " << trace.message; + } + } + + if (os_stack_trace.c_str() != NULL && !os_stack_trace.empty()) { + msg << internal::kStackTraceMarker << os_stack_trace; + } + + const TestPartResult result = + TestPartResult(result_type, file_name, line_number, + msg.GetString().c_str()); + impl_->GetTestPartResultReporterForCurrentThread()-> + ReportTestPartResult(result); + + if (result_type != TestPartResult::kSuccess) { + // gtest_break_on_failure takes precedence over + // gtest_throw_on_failure. This allows a user to set the latter + // in the code (perhaps in order to use Google Test assertions + // with another testing framework) and specify the former on the + // command line for debugging. + if (GTEST_FLAG(break_on_failure)) { +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + // Using DebugBreak on Windows allows gtest to still break into a debugger + // when a failure happens and both the --gtest_break_on_failure and + // the --gtest_catch_exceptions flags are specified. + DebugBreak(); +#else + // Dereference NULL through a volatile pointer to prevent the compiler + // from removing. We use this rather than abort() or __builtin_trap() for + // portability: Symbian doesn't implement abort() well, and some debuggers + // don't correctly trap abort(). + *static_cast(NULL) = 1; +#endif // GTEST_OS_WINDOWS + } else if (GTEST_FLAG(throw_on_failure)) { +#if GTEST_HAS_EXCEPTIONS + throw internal::GoogleTestFailureException(result); +#else + // We cannot call abort() as it generates a pop-up in debug mode + // that cannot be suppressed in VC 7.1 or below. + exit(1); +#endif + } + } +} + +// Adds a TestProperty to the current TestResult object when invoked from +// inside a test, to current TestCase's ad_hoc_test_result_ when invoked +// from SetUpTestCase or TearDownTestCase, or to the global property set +// when invoked elsewhere. If the result already contains a property with +// the same key, the value will be updated. +void UnitTest::RecordProperty(const std::string& key, + const std::string& value) { + impl_->RecordProperty(TestProperty(key, value)); +} + +// Runs all tests in this UnitTest object and prints the result. +// Returns 0 if successful, or 1 otherwise. +// +// We don't protect this under mutex_, as we only support calling it +// from the main thread. +int UnitTest::Run() { + const bool in_death_test_child_process = + internal::GTEST_FLAG(internal_run_death_test).length() > 0; + + // Google Test implements this protocol for catching that a test + // program exits before returning control to Google Test: + // + // 1. Upon start, Google Test creates a file whose absolute path + // is specified by the environment variable + // TEST_PREMATURE_EXIT_FILE. + // 2. When Google Test has finished its work, it deletes the file. + // + // This allows a test runner to set TEST_PREMATURE_EXIT_FILE before + // running a Google-Test-based test program and check the existence + // of the file at the end of the test execution to see if it has + // exited prematurely. + + // If we are in the child process of a death test, don't + // create/delete the premature exit file, as doing so is unnecessary + // and will confuse the parent process. Otherwise, create/delete + // the file upon entering/leaving this function. If the program + // somehow exits before this function has a chance to return, the + // premature-exit file will be left undeleted, causing a test runner + // that understands the premature-exit-file protocol to report the + // test as having failed. + const internal::ScopedPrematureExitFile premature_exit_file( + in_death_test_child_process ? + NULL : internal::posix::GetEnv("TEST_PREMATURE_EXIT_FILE")); + + // Captures the value of GTEST_FLAG(catch_exceptions). This value will be + // used for the duration of the program. + impl()->set_catch_exceptions(GTEST_FLAG(catch_exceptions)); + +#if GTEST_HAS_SEH + // Either the user wants Google Test to catch exceptions thrown by the + // tests or this is executing in the context of death test child + // process. In either case the user does not want to see pop-up dialogs + // about crashes - they are expected. + if (impl()->catch_exceptions() || in_death_test_child_process) { +# if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + // SetErrorMode doesn't exist on CE. + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT | + SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); +# endif // !GTEST_OS_WINDOWS_MOBILE + +# if (defined(_MSC_VER) || GTEST_OS_WINDOWS_MINGW) && !GTEST_OS_WINDOWS_MOBILE + // Death test children can be terminated with _abort(). On Windows, + // _abort() can show a dialog with a warning message. This forces the + // abort message to go to stderr instead. + _set_error_mode(_OUT_TO_STDERR); +# endif + +# if _MSC_VER >= 1400 && !GTEST_OS_WINDOWS_MOBILE + // In the debug version, Visual Studio pops up a separate dialog + // offering a choice to debug the aborted program. We need to suppress + // this dialog or it will pop up for every EXPECT/ASSERT_DEATH statement + // executed. Google Test will notify the user of any unexpected + // failure via stderr. + // + // VC++ doesn't define _set_abort_behavior() prior to the version 8.0. + // Users of prior VC versions shall suffer the agony and pain of + // clicking through the countless debug dialogs. + // TODO(vladl@google.com): find a way to suppress the abort dialog() in the + // debug mode when compiled with VC 7.1 or lower. + if (!GTEST_FLAG(break_on_failure)) + _set_abort_behavior( + 0x0, // Clear the following flags: + _WRITE_ABORT_MSG | _CALL_REPORTFAULT); // pop-up window, core dump. +# endif + } +#endif // GTEST_HAS_SEH + + return internal::HandleExceptionsInMethodIfSupported( + impl(), + &internal::UnitTestImpl::RunAllTests, + "auxiliary test code (environments or event listeners)") ? 0 : 1; +} + +// Returns the working directory when the first TEST() or TEST_F() was +// executed. +const char* UnitTest::original_working_dir() const { + return impl_->original_working_dir_.c_str(); +} + +// Returns the TestCase object for the test that's currently running, +// or NULL if no test is running. +const TestCase* UnitTest::current_test_case() const + GTEST_LOCK_EXCLUDED_(mutex_) { + internal::MutexLock lock(&mutex_); + return impl_->current_test_case(); +} + +// Returns the TestInfo object for the test that's currently running, +// or NULL if no test is running. +const TestInfo* UnitTest::current_test_info() const + GTEST_LOCK_EXCLUDED_(mutex_) { + internal::MutexLock lock(&mutex_); + return impl_->current_test_info(); +} + +// Returns the random seed used at the start of the current test run. +int UnitTest::random_seed() const { return impl_->random_seed(); } + +#if GTEST_HAS_PARAM_TEST +// Returns ParameterizedTestCaseRegistry object used to keep track of +// value-parameterized tests and instantiate and register them. +internal::ParameterizedTestCaseRegistry& + UnitTest::parameterized_test_registry() + GTEST_LOCK_EXCLUDED_(mutex_) { + return impl_->parameterized_test_registry(); +} +#endif // GTEST_HAS_PARAM_TEST + +// Creates an empty UnitTest. +UnitTest::UnitTest() { + impl_ = new internal::UnitTestImpl(this); +} + +// Destructor of UnitTest. +UnitTest::~UnitTest() { + delete impl_; +} + +// Pushes a trace defined by SCOPED_TRACE() on to the per-thread +// Google Test trace stack. +void UnitTest::PushGTestTrace(const internal::TraceInfo& trace) + GTEST_LOCK_EXCLUDED_(mutex_) { + internal::MutexLock lock(&mutex_); + impl_->gtest_trace_stack().push_back(trace); +} + +// Pops a trace from the per-thread Google Test trace stack. +void UnitTest::PopGTestTrace() + GTEST_LOCK_EXCLUDED_(mutex_) { + internal::MutexLock lock(&mutex_); + impl_->gtest_trace_stack().pop_back(); +} + +namespace internal { + +UnitTestImpl::UnitTestImpl(UnitTest* parent) + : parent_(parent), + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4355 /* using this in initializer */) + default_global_test_part_result_reporter_(this), + default_per_thread_test_part_result_reporter_(this), + GTEST_DISABLE_MSC_WARNINGS_POP_() + global_test_part_result_repoter_( + &default_global_test_part_result_reporter_), + per_thread_test_part_result_reporter_( + &default_per_thread_test_part_result_reporter_), +#if GTEST_HAS_PARAM_TEST + parameterized_test_registry_(), + parameterized_tests_registered_(false), +#endif // GTEST_HAS_PARAM_TEST + last_death_test_case_(-1), + current_test_case_(NULL), + current_test_info_(NULL), + ad_hoc_test_result_(), + os_stack_trace_getter_(NULL), + post_flag_parse_init_performed_(false), + random_seed_(0), // Will be overridden by the flag before first use. + random_(0), // Will be reseeded before first use. + start_timestamp_(0), + elapsed_time_(0), +#if GTEST_HAS_DEATH_TEST + death_test_factory_(new DefaultDeathTestFactory), +#endif + // Will be overridden by the flag before first use. + catch_exceptions_(false) { + listeners()->SetDefaultResultPrinter(new PrettyUnitTestResultPrinter); +} + +UnitTestImpl::~UnitTestImpl() { + // Deletes every TestCase. + ForEach(test_cases_, internal::Delete); + + // Deletes every Environment. + ForEach(environments_, internal::Delete); + + delete os_stack_trace_getter_; +} + +// Adds a TestProperty to the current TestResult object when invoked in a +// context of a test, to current test case's ad_hoc_test_result when invoke +// from SetUpTestCase/TearDownTestCase, or to the global property set +// otherwise. If the result already contains a property with the same key, +// the value will be updated. +void UnitTestImpl::RecordProperty(const TestProperty& test_property) { + std::string xml_element; + TestResult* test_result; // TestResult appropriate for property recording. + + if (current_test_info_ != NULL) { + xml_element = "testcase"; + test_result = &(current_test_info_->result_); + } else if (current_test_case_ != NULL) { + xml_element = "testsuite"; + test_result = &(current_test_case_->ad_hoc_test_result_); + } else { + xml_element = "testsuites"; + test_result = &ad_hoc_test_result_; + } + test_result->RecordProperty(xml_element, test_property); +} + +#if GTEST_HAS_DEATH_TEST +// Disables event forwarding if the control is currently in a death test +// subprocess. Must not be called before InitGoogleTest. +void UnitTestImpl::SuppressTestEventsIfInSubprocess() { + if (internal_run_death_test_flag_.get() != NULL) + listeners()->SuppressEventForwarding(); +} +#endif // GTEST_HAS_DEATH_TEST + +// Initializes event listeners performing XML output as specified by +// UnitTestOptions. Must not be called before InitGoogleTest. +void UnitTestImpl::ConfigureXmlOutput() { + const std::string& output_format = UnitTestOptions::GetOutputFormat(); + if (output_format == "xml") { + listeners()->SetDefaultXmlGenerator(new XmlUnitTestResultPrinter( + UnitTestOptions::GetAbsolutePathToOutputFile().c_str())); + } else if (output_format != "") { + printf("WARNING: unrecognized output format \"%s\" ignored.\n", + output_format.c_str()); + fflush(stdout); + } +} + +#if GTEST_CAN_STREAM_RESULTS_ +// Initializes event listeners for streaming test results in string form. +// Must not be called before InitGoogleTest. +void UnitTestImpl::ConfigureStreamingOutput() { + const std::string& target = GTEST_FLAG(stream_result_to); + if (!target.empty()) { + const size_t pos = target.find(':'); + if (pos != std::string::npos) { + listeners()->Append(new StreamingListener(target.substr(0, pos), + target.substr(pos+1))); + } else { + printf("WARNING: unrecognized streaming target \"%s\" ignored.\n", + target.c_str()); + fflush(stdout); + } + } +} +#endif // GTEST_CAN_STREAM_RESULTS_ + +// Performs initialization dependent upon flag values obtained in +// ParseGoogleTestFlagsOnly. Is called from InitGoogleTest after the call to +// ParseGoogleTestFlagsOnly. In case a user neglects to call InitGoogleTest +// this function is also called from RunAllTests. Since this function can be +// called more than once, it has to be idempotent. +void UnitTestImpl::PostFlagParsingInit() { + // Ensures that this function does not execute more than once. + if (!post_flag_parse_init_performed_) { + post_flag_parse_init_performed_ = true; + +#if defined(GTEST_CUSTOM_TEST_EVENT_LISTENER_) + // Register to send notifications about key process state changes. + listeners()->Append(new GTEST_CUSTOM_TEST_EVENT_LISTENER_()); +#endif // defined(GTEST_CUSTOM_TEST_EVENT_LISTENER_) + +#if GTEST_HAS_DEATH_TEST + InitDeathTestSubprocessControlInfo(); + SuppressTestEventsIfInSubprocess(); +#endif // GTEST_HAS_DEATH_TEST + + // Registers parameterized tests. This makes parameterized tests + // available to the UnitTest reflection API without running + // RUN_ALL_TESTS. + RegisterParameterizedTests(); + + // Configures listeners for XML output. This makes it possible for users + // to shut down the default XML output before invoking RUN_ALL_TESTS. + ConfigureXmlOutput(); + +#if GTEST_CAN_STREAM_RESULTS_ + // Configures listeners for streaming test results to the specified server. + ConfigureStreamingOutput(); +#endif // GTEST_CAN_STREAM_RESULTS_ + } +} + +// A predicate that checks the name of a TestCase against a known +// value. +// +// This is used for implementation of the UnitTest class only. We put +// it in the anonymous namespace to prevent polluting the outer +// namespace. +// +// TestCaseNameIs is copyable. +class TestCaseNameIs { + public: + // Constructor. + explicit TestCaseNameIs(const std::string& name) + : name_(name) {} + + // Returns true iff the name of test_case matches name_. + bool operator()(const TestCase* test_case) const { + return test_case != NULL && strcmp(test_case->name(), name_.c_str()) == 0; + } + + private: + std::string name_; +}; + +// Finds and returns a TestCase with the given name. If one doesn't +// exist, creates one and returns it. It's the CALLER'S +// RESPONSIBILITY to ensure that this function is only called WHEN THE +// TESTS ARE NOT SHUFFLED. +// +// Arguments: +// +// test_case_name: name of the test case +// type_param: the name of the test case's type parameter, or NULL if +// this is not a typed or a type-parameterized test case. +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +TestCase* UnitTestImpl::GetTestCase(const char* test_case_name, + const char* type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc) { + // Can we find a TestCase with the given name? + const std::vector::const_iterator test_case = + std::find_if(test_cases_.begin(), test_cases_.end(), + TestCaseNameIs(test_case_name)); + + if (test_case != test_cases_.end()) + return *test_case; + + // No. Let's create one. + TestCase* const new_test_case = + new TestCase(test_case_name, type_param, set_up_tc, tear_down_tc); + + // Is this a death test case? + if (internal::UnitTestOptions::MatchesFilter(test_case_name, + kDeathTestCaseFilter)) { + // Yes. Inserts the test case after the last death test case + // defined so far. This only works when the test cases haven't + // been shuffled. Otherwise we may end up running a death test + // after a non-death test. + ++last_death_test_case_; + test_cases_.insert(test_cases_.begin() + last_death_test_case_, + new_test_case); + } else { + // No. Appends to the end of the list. + test_cases_.push_back(new_test_case); + } + + test_case_indices_.push_back(static_cast(test_case_indices_.size())); + return new_test_case; +} + +// Helpers for setting up / tearing down the given environment. They +// are for use in the ForEach() function. +static void SetUpEnvironment(Environment* env) { env->SetUp(); } +static void TearDownEnvironment(Environment* env) { env->TearDown(); } + +// Runs all tests in this UnitTest object, prints the result, and +// returns true if all tests are successful. If any exception is +// thrown during a test, the test is considered to be failed, but the +// rest of the tests will still be run. +// +// When parameterized tests are enabled, it expands and registers +// parameterized tests first in RegisterParameterizedTests(). +// All other functions called from RunAllTests() may safely assume that +// parameterized tests are ready to be counted and run. +bool UnitTestImpl::RunAllTests() { + // Makes sure InitGoogleTest() was called. + if (!GTestIsInitialized()) { + printf("%s", + "\nThis test program did NOT call ::testing::InitGoogleTest " + "before calling RUN_ALL_TESTS(). Please fix it.\n"); + return false; + } + + // Do not run any test if the --help flag was specified. + if (g_help_flag) + return true; + + // Repeats the call to the post-flag parsing initialization in case the + // user didn't call InitGoogleTest. + PostFlagParsingInit(); + + // Even if sharding is not on, test runners may want to use the + // GTEST_SHARD_STATUS_FILE to query whether the test supports the sharding + // protocol. + internal::WriteToShardStatusFileIfNeeded(); + + // True iff we are in a subprocess for running a thread-safe-style + // death test. + bool in_subprocess_for_death_test = false; + +#if GTEST_HAS_DEATH_TEST + in_subprocess_for_death_test = (internal_run_death_test_flag_.get() != NULL); +# if defined(GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_) + if (in_subprocess_for_death_test) { + GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_(); + } +# endif // defined(GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_) +#endif // GTEST_HAS_DEATH_TEST + + const bool should_shard = ShouldShard(kTestTotalShards, kTestShardIndex, + in_subprocess_for_death_test); + + // Compares the full test names with the filter to decide which + // tests to run. + const bool has_tests_to_run = FilterTests(should_shard + ? HONOR_SHARDING_PROTOCOL + : IGNORE_SHARDING_PROTOCOL) > 0; + + // Lists the tests and exits if the --gtest_list_tests flag was specified. + if (GTEST_FLAG(list_tests)) { + // This must be called *after* FilterTests() has been called. + ListTestsMatchingFilter(); + return true; + } + + random_seed_ = GTEST_FLAG(shuffle) ? + GetRandomSeedFromFlag(GTEST_FLAG(random_seed)) : 0; + + // True iff at least one test has failed. + bool failed = false; + + TestEventListener* repeater = listeners()->repeater(); + + start_timestamp_ = GetTimeInMillis(); + repeater->OnTestProgramStart(*parent_); + + // How many times to repeat the tests? We don't want to repeat them + // when we are inside the subprocess of a death test. + const int repeat = in_subprocess_for_death_test ? 1 : GTEST_FLAG(repeat); + // Repeats forever if the repeat count is negative. + const bool forever = repeat < 0; + for (int i = 0; forever || i != repeat; i++) { + // We want to preserve failures generated by ad-hoc test + // assertions executed before RUN_ALL_TESTS(). + ClearNonAdHocTestResult(); + + const TimeInMillis start = GetTimeInMillis(); + + // Shuffles test cases and tests if requested. + if (has_tests_to_run && GTEST_FLAG(shuffle)) { + random()->Reseed(random_seed_); + // This should be done before calling OnTestIterationStart(), + // such that a test event listener can see the actual test order + // in the event. + ShuffleTests(); + } + + // Tells the unit test event listeners that the tests are about to start. + repeater->OnTestIterationStart(*parent_, i); + + // Runs each test case if there is at least one test to run. + if (has_tests_to_run) { + // Sets up all environments beforehand. + repeater->OnEnvironmentsSetUpStart(*parent_); + ForEach(environments_, SetUpEnvironment); + repeater->OnEnvironmentsSetUpEnd(*parent_); + + // Runs the tests only if there was no fatal failure during global + // set-up. + if (!Test::HasFatalFailure()) { + for (int test_index = 0; test_index < total_test_case_count(); + test_index++) { + GetMutableTestCase(test_index)->Run(); + } + } + + // Tears down all environments in reverse order afterwards. + repeater->OnEnvironmentsTearDownStart(*parent_); + std::for_each(environments_.rbegin(), environments_.rend(), + TearDownEnvironment); + repeater->OnEnvironmentsTearDownEnd(*parent_); + } + + elapsed_time_ = GetTimeInMillis() - start; + + // Tells the unit test event listener that the tests have just finished. + repeater->OnTestIterationEnd(*parent_, i); + + // Gets the result and clears it. + if (!Passed()) { + failed = true; + } + + // Restores the original test order after the iteration. This + // allows the user to quickly repro a failure that happens in the + // N-th iteration without repeating the first (N - 1) iterations. + // This is not enclosed in "if (GTEST_FLAG(shuffle)) { ... }", in + // case the user somehow changes the value of the flag somewhere + // (it's always safe to unshuffle the tests). + UnshuffleTests(); + + if (GTEST_FLAG(shuffle)) { + // Picks a new random seed for each iteration. + random_seed_ = GetNextRandomSeed(random_seed_); + } + } + + repeater->OnTestProgramEnd(*parent_); + + return !failed; +} + +// Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file +// if the variable is present. If a file already exists at this location, this +// function will write over it. If the variable is present, but the file cannot +// be created, prints an error and exits. +void WriteToShardStatusFileIfNeeded() { + const char* const test_shard_file = posix::GetEnv(kTestShardStatusFile); + if (test_shard_file != NULL) { + FILE* const file = posix::FOpen(test_shard_file, "w"); + if (file == NULL) { + ColoredPrintf(COLOR_RED, + "Could not write to the test shard status file \"%s\" " + "specified by the %s environment variable.\n", + test_shard_file, kTestShardStatusFile); + fflush(stdout); + exit(EXIT_FAILURE); + } + fclose(file); + } +} + +// Checks whether sharding is enabled by examining the relevant +// environment variable values. If the variables are present, +// but inconsistent (i.e., shard_index >= total_shards), prints +// an error and exits. If in_subprocess_for_death_test, sharding is +// disabled because it must only be applied to the original test +// process. Otherwise, we could filter out death tests we intended to execute. +bool ShouldShard(const char* total_shards_env, + const char* shard_index_env, + bool in_subprocess_for_death_test) { + if (in_subprocess_for_death_test) { + return false; + } + + const Int32 total_shards = Int32FromEnvOrDie(total_shards_env, -1); + const Int32 shard_index = Int32FromEnvOrDie(shard_index_env, -1); + + if (total_shards == -1 && shard_index == -1) { + return false; + } else if (total_shards == -1 && shard_index != -1) { + const Message msg = Message() + << "Invalid environment variables: you have " + << kTestShardIndex << " = " << shard_index + << ", but have left " << kTestTotalShards << " unset.\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } else if (total_shards != -1 && shard_index == -1) { + const Message msg = Message() + << "Invalid environment variables: you have " + << kTestTotalShards << " = " << total_shards + << ", but have left " << kTestShardIndex << " unset.\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } else if (shard_index < 0 || shard_index >= total_shards) { + const Message msg = Message() + << "Invalid environment variables: we require 0 <= " + << kTestShardIndex << " < " << kTestTotalShards + << ", but you have " << kTestShardIndex << "=" << shard_index + << ", " << kTestTotalShards << "=" << total_shards << ".\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } + + return total_shards > 1; +} + +// Parses the environment variable var as an Int32. If it is unset, +// returns default_val. If it is not an Int32, prints an error +// and aborts. +Int32 Int32FromEnvOrDie(const char* var, Int32 default_val) { + const char* str_val = posix::GetEnv(var); + if (str_val == NULL) { + return default_val; + } + + Int32 result; + if (!ParseInt32(Message() << "The value of environment variable " << var, + str_val, &result)) { + exit(EXIT_FAILURE); + } + return result; +} + +// Given the total number of shards, the shard index, and the test id, +// returns true iff the test should be run on this shard. The test id is +// some arbitrary but unique non-negative integer assigned to each test +// method. Assumes that 0 <= shard_index < total_shards. +bool ShouldRunTestOnShard(int total_shards, int shard_index, int test_id) { + return (test_id % total_shards) == shard_index; +} + +// Compares the name of each test with the user-specified filter to +// decide whether the test should be run, then records the result in +// each TestCase and TestInfo object. +// If shard_tests == true, further filters tests based on sharding +// variables in the environment - see +// http://code.google.com/p/googletest/wiki/GoogleTestAdvancedGuide. +// Returns the number of tests that should run. +int UnitTestImpl::FilterTests(ReactionToSharding shard_tests) { + const Int32 total_shards = shard_tests == HONOR_SHARDING_PROTOCOL ? + Int32FromEnvOrDie(kTestTotalShards, -1) : -1; + const Int32 shard_index = shard_tests == HONOR_SHARDING_PROTOCOL ? + Int32FromEnvOrDie(kTestShardIndex, -1) : -1; + + // num_runnable_tests are the number of tests that will + // run across all shards (i.e., match filter and are not disabled). + // num_selected_tests are the number of tests to be run on + // this shard. + int num_runnable_tests = 0; + int num_selected_tests = 0; + for (size_t i = 0; i < test_cases_.size(); i++) { + TestCase* const test_case = test_cases_[i]; + const std::string &test_case_name = test_case->name(); + test_case->set_should_run(false); + + for (size_t j = 0; j < test_case->test_info_list().size(); j++) { + TestInfo* const test_info = test_case->test_info_list()[j]; + const std::string test_name(test_info->name()); + // A test is disabled if test case name or test name matches + // kDisableTestFilter. + const bool is_disabled = + internal::UnitTestOptions::MatchesFilter(test_case_name, + kDisableTestFilter) || + internal::UnitTestOptions::MatchesFilter(test_name, + kDisableTestFilter); + test_info->is_disabled_ = is_disabled; + + const bool matches_filter = + internal::UnitTestOptions::FilterMatchesTest(test_case_name, + test_name); + test_info->matches_filter_ = matches_filter; + + const bool is_runnable = + (GTEST_FLAG(also_run_disabled_tests) || !is_disabled) && + matches_filter; + + const bool is_selected = is_runnable && + (shard_tests == IGNORE_SHARDING_PROTOCOL || + ShouldRunTestOnShard(total_shards, shard_index, + num_runnable_tests)); + + num_runnable_tests += is_runnable; + num_selected_tests += is_selected; + + test_info->should_run_ = is_selected; + test_case->set_should_run(test_case->should_run() || is_selected); + } + } + return num_selected_tests; +} + +// Prints the given C-string on a single line by replacing all '\n' +// characters with string "\\n". If the output takes more than +// max_length characters, only prints the first max_length characters +// and "...". +static void PrintOnOneLine(const char* str, int max_length) { + if (str != NULL) { + for (int i = 0; *str != '\0'; ++str) { + if (i >= max_length) { + printf("..."); + break; + } + if (*str == '\n') { + printf("\\n"); + i += 2; + } else { + printf("%c", *str); + ++i; + } + } + } +} + +// Prints the names of the tests matching the user-specified filter flag. +void UnitTestImpl::ListTestsMatchingFilter() { + // Print at most this many characters for each type/value parameter. + const int kMaxParamLength = 250; + + for (size_t i = 0; i < test_cases_.size(); i++) { + const TestCase* const test_case = test_cases_[i]; + bool printed_test_case_name = false; + + for (size_t j = 0; j < test_case->test_info_list().size(); j++) { + const TestInfo* const test_info = + test_case->test_info_list()[j]; + if (test_info->matches_filter_) { + if (!printed_test_case_name) { + printed_test_case_name = true; + printf("%s.", test_case->name()); + if (test_case->type_param() != NULL) { + printf(" # %s = ", kTypeParamLabel); + // We print the type parameter on a single line to make + // the output easy to parse by a program. + PrintOnOneLine(test_case->type_param(), kMaxParamLength); + } + printf("\n"); + } + printf(" %s", test_info->name()); + if (test_info->value_param() != NULL) { + printf(" # %s = ", kValueParamLabel); + // We print the value parameter on a single line to make the + // output easy to parse by a program. + PrintOnOneLine(test_info->value_param(), kMaxParamLength); + } + printf("\n"); + } + } + } + fflush(stdout); +} + +// Sets the OS stack trace getter. +// +// Does nothing if the input and the current OS stack trace getter are +// the same; otherwise, deletes the old getter and makes the input the +// current getter. +void UnitTestImpl::set_os_stack_trace_getter( + OsStackTraceGetterInterface* getter) { + if (os_stack_trace_getter_ != getter) { + delete os_stack_trace_getter_; + os_stack_trace_getter_ = getter; + } +} + +// Returns the current OS stack trace getter if it is not NULL; +// otherwise, creates an OsStackTraceGetter, makes it the current +// getter, and returns it. +OsStackTraceGetterInterface* UnitTestImpl::os_stack_trace_getter() { + if (os_stack_trace_getter_ == NULL) { +#ifdef GTEST_OS_STACK_TRACE_GETTER_ + os_stack_trace_getter_ = new GTEST_OS_STACK_TRACE_GETTER_; +#else + os_stack_trace_getter_ = new OsStackTraceGetter; +#endif // GTEST_OS_STACK_TRACE_GETTER_ + } + + return os_stack_trace_getter_; +} + +// Returns the TestResult for the test that's currently running, or +// the TestResult for the ad hoc test if no test is running. +TestResult* UnitTestImpl::current_test_result() { + return current_test_info_ ? + &(current_test_info_->result_) : &ad_hoc_test_result_; +} + +// Shuffles all test cases, and the tests within each test case, +// making sure that death tests are still run first. +void UnitTestImpl::ShuffleTests() { + // Shuffles the death test cases. + ShuffleRange(random(), 0, last_death_test_case_ + 1, &test_case_indices_); + + // Shuffles the non-death test cases. + ShuffleRange(random(), last_death_test_case_ + 1, + static_cast(test_cases_.size()), &test_case_indices_); + + // Shuffles the tests inside each test case. + for (size_t i = 0; i < test_cases_.size(); i++) { + test_cases_[i]->ShuffleTests(random()); + } +} + +// Restores the test cases and tests to their order before the first shuffle. +void UnitTestImpl::UnshuffleTests() { + for (size_t i = 0; i < test_cases_.size(); i++) { + // Unshuffles the tests in each test case. + test_cases_[i]->UnshuffleTests(); + // Resets the index of each test case. + test_case_indices_[i] = static_cast(i); + } +} + +// Returns the current OS stack trace as an std::string. +// +// The maximum number of stack frames to be included is specified by +// the gtest_stack_trace_depth flag. The skip_count parameter +// specifies the number of top frames to be skipped, which doesn't +// count against the number of frames to be included. +// +// For example, if Foo() calls Bar(), which in turn calls +// GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in +// the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't. +std::string GetCurrentOsStackTraceExceptTop(UnitTest* /*unit_test*/, + int skip_count) { + // We pass skip_count + 1 to skip this wrapper function in addition + // to what the user really wants to skip. + return GetUnitTestImpl()->CurrentOsStackTraceExceptTop(skip_count + 1); +} + +// Used by the GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_ macro to +// suppress unreachable code warnings. +namespace { +class ClassUniqueToAlwaysTrue {}; +} + +bool IsTrue(bool condition) { return condition; } + +bool AlwaysTrue() { +#if GTEST_HAS_EXCEPTIONS + // This condition is always false so AlwaysTrue() never actually throws, + // but it makes the compiler think that it may throw. + if (IsTrue(false)) + throw ClassUniqueToAlwaysTrue(); +#endif // GTEST_HAS_EXCEPTIONS + return true; +} + +// If *pstr starts with the given prefix, modifies *pstr to be right +// past the prefix and returns true; otherwise leaves *pstr unchanged +// and returns false. None of pstr, *pstr, and prefix can be NULL. +bool SkipPrefix(const char* prefix, const char** pstr) { + const size_t prefix_len = strlen(prefix); + if (strncmp(*pstr, prefix, prefix_len) == 0) { + *pstr += prefix_len; + return true; + } + return false; +} + +// Parses a string as a command line flag. The string should have +// the format "--flag=value". When def_optional is true, the "=value" +// part can be omitted. +// +// Returns the value of the flag, or NULL if the parsing failed. +const char* ParseFlagValue(const char* str, + const char* flag, + bool def_optional) { + // str and flag must not be NULL. + if (str == NULL || flag == NULL) return NULL; + + // The flag must start with "--" followed by GTEST_FLAG_PREFIX_. + const std::string flag_str = std::string("--") + GTEST_FLAG_PREFIX_ + flag; + const size_t flag_len = flag_str.length(); + if (strncmp(str, flag_str.c_str(), flag_len) != 0) return NULL; + + // Skips the flag name. + const char* flag_end = str + flag_len; + + // When def_optional is true, it's OK to not have a "=value" part. + if (def_optional && (flag_end[0] == '\0')) { + return flag_end; + } + + // If def_optional is true and there are more characters after the + // flag name, or if def_optional is false, there must be a '=' after + // the flag name. + if (flag_end[0] != '=') return NULL; + + // Returns the string after "=". + return flag_end + 1; +} + +// Parses a string for a bool flag, in the form of either +// "--flag=value" or "--flag". +// +// In the former case, the value is taken as true as long as it does +// not start with '0', 'f', or 'F'. +// +// In the latter case, the value is taken as true. +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseBoolFlag(const char* str, const char* flag, bool* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, true); + + // Aborts if the parsing failed. + if (value_str == NULL) return false; + + // Converts the string value to a bool. + *value = !(*value_str == '0' || *value_str == 'f' || *value_str == 'F'); + return true; +} + +// Parses a string for an Int32 flag, in the form of +// "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseInt32Flag(const char* str, const char* flag, Int32* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, false); + + // Aborts if the parsing failed. + if (value_str == NULL) return false; + + // Sets *value to the value of the flag. + return ParseInt32(Message() << "The value of flag --" << flag, + value_str, value); +} + +// Parses a string for a string flag, in the form of +// "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseStringFlag(const char* str, const char* flag, std::string* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, false); + + // Aborts if the parsing failed. + if (value_str == NULL) return false; + + // Sets *value to the value of the flag. + *value = value_str; + return true; +} + +// Determines whether a string has a prefix that Google Test uses for its +// flags, i.e., starts with GTEST_FLAG_PREFIX_ or GTEST_FLAG_PREFIX_DASH_. +// If Google Test detects that a command line flag has its prefix but is not +// recognized, it will print its help message. Flags starting with +// GTEST_INTERNAL_PREFIX_ followed by "internal_" are considered Google Test +// internal flags and do not trigger the help message. +static bool HasGoogleTestFlagPrefix(const char* str) { + return (SkipPrefix("--", &str) || + SkipPrefix("-", &str) || + SkipPrefix("/", &str)) && + !SkipPrefix(GTEST_FLAG_PREFIX_ "internal_", &str) && + (SkipPrefix(GTEST_FLAG_PREFIX_, &str) || + SkipPrefix(GTEST_FLAG_PREFIX_DASH_, &str)); +} + +// Prints a string containing code-encoded text. The following escape +// sequences can be used in the string to control the text color: +// +// @@ prints a single '@' character. +// @R changes the color to red. +// @G changes the color to green. +// @Y changes the color to yellow. +// @D changes to the default terminal text color. +// +// TODO(wan@google.com): Write tests for this once we add stdout +// capturing to Google Test. +static void PrintColorEncoded(const char* str) { + GTestColor color = COLOR_DEFAULT; // The current color. + + // Conceptually, we split the string into segments divided by escape + // sequences. Then we print one segment at a time. At the end of + // each iteration, the str pointer advances to the beginning of the + // next segment. + for (;;) { + const char* p = strchr(str, '@'); + if (p == NULL) { + ColoredPrintf(color, "%s", str); + return; + } + + ColoredPrintf(color, "%s", std::string(str, p).c_str()); + + const char ch = p[1]; + str = p + 2; + if (ch == '@') { + ColoredPrintf(color, "@"); + } else if (ch == 'D') { + color = COLOR_DEFAULT; + } else if (ch == 'R') { + color = COLOR_RED; + } else if (ch == 'G') { + color = COLOR_GREEN; + } else if (ch == 'Y') { + color = COLOR_YELLOW; + } else { + --str; + } + } +} + +static const char kColorEncodedHelpMessage[] = +"This program contains tests written using " GTEST_NAME_ ". You can use the\n" +"following command line flags to control its behavior:\n" +"\n" +"Test Selection:\n" +" @G--" GTEST_FLAG_PREFIX_ "list_tests@D\n" +" List the names of all tests instead of running them. The name of\n" +" TEST(Foo, Bar) is \"Foo.Bar\".\n" +" @G--" GTEST_FLAG_PREFIX_ "filter=@YPOSTIVE_PATTERNS" + "[@G-@YNEGATIVE_PATTERNS]@D\n" +" Run only the tests whose name matches one of the positive patterns but\n" +" none of the negative patterns. '?' matches any single character; '*'\n" +" matches any substring; ':' separates two patterns.\n" +" @G--" GTEST_FLAG_PREFIX_ "also_run_disabled_tests@D\n" +" Run all disabled tests too.\n" +"\n" +"Test Execution:\n" +" @G--" GTEST_FLAG_PREFIX_ "repeat=@Y[COUNT]@D\n" +" Run the tests repeatedly; use a negative count to repeat forever.\n" +" @G--" GTEST_FLAG_PREFIX_ "shuffle@D\n" +" Randomize tests' orders on every iteration.\n" +" @G--" GTEST_FLAG_PREFIX_ "random_seed=@Y[NUMBER]@D\n" +" Random number seed to use for shuffling test orders (between 1 and\n" +" 99999, or 0 to use a seed based on the current time).\n" +"\n" +"Test Output:\n" +" @G--" GTEST_FLAG_PREFIX_ "color=@Y(@Gyes@Y|@Gno@Y|@Gauto@Y)@D\n" +" Enable/disable colored output. The default is @Gauto@D.\n" +" -@G-" GTEST_FLAG_PREFIX_ "print_time=0@D\n" +" Don't print the elapsed time of each test.\n" +" @G--" GTEST_FLAG_PREFIX_ "output=xml@Y[@G:@YDIRECTORY_PATH@G" + GTEST_PATH_SEP_ "@Y|@G:@YFILE_PATH]@D\n" +" Generate an XML report in the given directory or with the given file\n" +" name. @YFILE_PATH@D defaults to @Gtest_details.xml@D.\n" +#if GTEST_CAN_STREAM_RESULTS_ +" @G--" GTEST_FLAG_PREFIX_ "stream_result_to=@YHOST@G:@YPORT@D\n" +" Stream test results to the given server.\n" +#endif // GTEST_CAN_STREAM_RESULTS_ +"\n" +"Assertion Behavior:\n" +#if GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS +" @G--" GTEST_FLAG_PREFIX_ "death_test_style=@Y(@Gfast@Y|@Gthreadsafe@Y)@D\n" +" Set the default death test style.\n" +#endif // GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS +" @G--" GTEST_FLAG_PREFIX_ "break_on_failure@D\n" +" Turn assertion failures into debugger break-points.\n" +" @G--" GTEST_FLAG_PREFIX_ "throw_on_failure@D\n" +" Turn assertion failures into C++ exceptions.\n" +" @G--" GTEST_FLAG_PREFIX_ "catch_exceptions=0@D\n" +" Do not report exceptions as test failures. Instead, allow them\n" +" to crash the program or throw a pop-up (on Windows).\n" +"\n" +"Except for @G--" GTEST_FLAG_PREFIX_ "list_tests@D, you can alternatively set " + "the corresponding\n" +"environment variable of a flag (all letters in upper-case). For example, to\n" +"disable colored text output, you can either specify @G--" GTEST_FLAG_PREFIX_ + "color=no@D or set\n" +"the @G" GTEST_FLAG_PREFIX_UPPER_ "COLOR@D environment variable to @Gno@D.\n" +"\n" +"For more information, please read the " GTEST_NAME_ " documentation at\n" +"@G" GTEST_PROJECT_URL_ "@D. If you find a bug in " GTEST_NAME_ "\n" +"(not one in your own code or tests), please report it to\n" +"@G<" GTEST_DEV_EMAIL_ ">@D.\n"; + +bool ParseGoogleTestFlag(const char* const arg) { + return ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag, + >EST_FLAG(also_run_disabled_tests)) || + ParseBoolFlag(arg, kBreakOnFailureFlag, + >EST_FLAG(break_on_failure)) || + ParseBoolFlag(arg, kCatchExceptionsFlag, + >EST_FLAG(catch_exceptions)) || + ParseStringFlag(arg, kColorFlag, >EST_FLAG(color)) || + ParseStringFlag(arg, kDeathTestStyleFlag, + >EST_FLAG(death_test_style)) || + ParseBoolFlag(arg, kDeathTestUseFork, + >EST_FLAG(death_test_use_fork)) || + ParseStringFlag(arg, kFilterFlag, >EST_FLAG(filter)) || + ParseStringFlag(arg, kInternalRunDeathTestFlag, + >EST_FLAG(internal_run_death_test)) || + ParseBoolFlag(arg, kListTestsFlag, >EST_FLAG(list_tests)) || + ParseStringFlag(arg, kOutputFlag, >EST_FLAG(output)) || + ParseBoolFlag(arg, kPrintTimeFlag, >EST_FLAG(print_time)) || + ParseInt32Flag(arg, kRandomSeedFlag, >EST_FLAG(random_seed)) || + ParseInt32Flag(arg, kRepeatFlag, >EST_FLAG(repeat)) || + ParseBoolFlag(arg, kShuffleFlag, >EST_FLAG(shuffle)) || + ParseInt32Flag(arg, kStackTraceDepthFlag, + >EST_FLAG(stack_trace_depth)) || + ParseStringFlag(arg, kStreamResultToFlag, + >EST_FLAG(stream_result_to)) || + ParseBoolFlag(arg, kThrowOnFailureFlag, + >EST_FLAG(throw_on_failure)); +} + +#if GTEST_USE_OWN_FLAGFILE_FLAG_ +void LoadFlagsFromFile(const std::string& path) { + FILE* flagfile = posix::FOpen(path.c_str(), "r"); + if (!flagfile) { + fprintf(stderr, + "Unable to open file \"%s\"\n", + GTEST_FLAG(flagfile).c_str()); + fflush(stderr); + exit(EXIT_FAILURE); + } + std::string contents(ReadEntireFile(flagfile)); + posix::FClose(flagfile); + std::vector lines; + SplitString(contents, '\n', &lines); + for (size_t i = 0; i < lines.size(); ++i) { + if (lines[i].empty()) + continue; + if (!ParseGoogleTestFlag(lines[i].c_str())) + g_help_flag = true; + } +} +#endif // GTEST_USE_OWN_FLAGFILE_FLAG_ + +// Parses the command line for Google Test flags, without initializing +// other parts of Google Test. The type parameter CharType can be +// instantiated to either char or wchar_t. +template +void ParseGoogleTestFlagsOnlyImpl(int* argc, CharType** argv) { + for (int i = 1; i < *argc; i++) { + const std::string arg_string = StreamableToString(argv[i]); + const char* const arg = arg_string.c_str(); + + using internal::ParseBoolFlag; + using internal::ParseInt32Flag; + using internal::ParseStringFlag; + + bool remove_flag = false; + if (ParseGoogleTestFlag(arg)) { + remove_flag = true; +#if GTEST_USE_OWN_FLAGFILE_FLAG_ + } else if (ParseStringFlag(arg, kFlagfileFlag, >EST_FLAG(flagfile))) { + LoadFlagsFromFile(GTEST_FLAG(flagfile)); + remove_flag = true; +#endif // GTEST_USE_OWN_FLAGFILE_FLAG_ + } else if (arg_string == "--help" || arg_string == "-h" || + arg_string == "-?" || arg_string == "/?" || + HasGoogleTestFlagPrefix(arg)) { + // Both help flag and unrecognized Google Test flags (excluding + // internal ones) trigger help display. + g_help_flag = true; + } + + if (remove_flag) { + // Shift the remainder of the argv list left by one. Note + // that argv has (*argc + 1) elements, the last one always being + // NULL. The following loop moves the trailing NULL element as + // well. + for (int j = i; j != *argc; j++) { + argv[j] = argv[j + 1]; + } + + // Decrements the argument count. + (*argc)--; + + // We also need to decrement the iterator as we just removed + // an element. + i--; + } + } + + if (g_help_flag) { + // We print the help here instead of in RUN_ALL_TESTS(), as the + // latter may not be called at all if the user is using Google + // Test with another testing framework. + PrintColorEncoded(kColorEncodedHelpMessage); + } +} + +// Parses the command line for Google Test flags, without initializing +// other parts of Google Test. +void ParseGoogleTestFlagsOnly(int* argc, char** argv) { + ParseGoogleTestFlagsOnlyImpl(argc, argv); +} +void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv) { + ParseGoogleTestFlagsOnlyImpl(argc, argv); +} + +// The internal implementation of InitGoogleTest(). +// +// The type parameter CharType can be instantiated to either char or +// wchar_t. +template +void InitGoogleTestImpl(int* argc, CharType** argv) { + // We don't want to run the initialization code twice. + if (GTestIsInitialized()) return; + + if (*argc <= 0) return; + + g_argvs.clear(); + for (int i = 0; i != *argc; i++) { + g_argvs.push_back(StreamableToString(argv[i])); + } + + ParseGoogleTestFlagsOnly(argc, argv); + GetUnitTestImpl()->PostFlagParsingInit(); +} + +} // namespace internal + +// Initializes Google Test. This must be called before calling +// RUN_ALL_TESTS(). In particular, it parses a command line for the +// flags that Google Test recognizes. Whenever a Google Test flag is +// seen, it is removed from argv, and *argc is decremented. +// +// No value is returned. Instead, the Google Test flag variables are +// updated. +// +// Calling the function for the second time has no user-visible effect. +void InitGoogleTest(int* argc, char** argv) { +#if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) + GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(argc, argv); +#else // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) + internal::InitGoogleTestImpl(argc, argv); +#endif // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) +} + +// This overloaded version can be used in Windows programs compiled in +// UNICODE mode. +void InitGoogleTest(int* argc, wchar_t** argv) { +#if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) + GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(argc, argv); +#else // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) + internal::InitGoogleTestImpl(argc, argv); +#endif // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) +} + +} // namespace testing diff --git a/third_party/aom/third_party/googletest/src/googletest/src/gtest_main.cc b/third_party/aom/third_party/googletest/src/googletest/src/gtest_main.cc new file mode 100644 index 0000000000..f302822552 --- /dev/null +++ b/third_party/aom/third_party/googletest/src/googletest/src/gtest_main.cc @@ -0,0 +1,38 @@ +// Copyright 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include "gtest/gtest.h" + +GTEST_API_ int main(int argc, char **argv) { + printf("Running main() from gtest_main.cc\n"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/third_party/aom/third_party/libwebm/AUTHORS.TXT b/third_party/aom/third_party/libwebm/AUTHORS.TXT new file mode 100644 index 0000000000..9686ac13eb --- /dev/null +++ b/third_party/aom/third_party/libwebm/AUTHORS.TXT @@ -0,0 +1,4 @@ +# Names should be added to this file like so: +# Name or Organization + +Google Inc. diff --git a/third_party/aom/third_party/libwebm/Android.mk b/third_party/aom/third_party/libwebm/Android.mk new file mode 100644 index 0000000000..8149a083f4 --- /dev/null +++ b/third_party/aom/third_party/libwebm/Android.mk @@ -0,0 +1,17 @@ +LOCAL_PATH:= $(call my-dir) + +include $(CLEAR_VARS) +LOCAL_MODULE:= libwebm +LOCAL_CPPFLAGS:=-D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS +LOCAL_CPPFLAGS+=-D__STDC_LIMIT_MACROS -Wno-extern-c-compat +LOCAL_C_INCLUDES:= $(LOCAL_PATH) +LOCAL_EXPORT_C_INCLUDES:= $(LOCAL_PATH) + +LOCAL_SRC_FILES:= common/file_util.cc \ + common/hdr_util.cc \ + mkvparser/mkvparser.cc \ + mkvparser/mkvreader.cc \ + mkvmuxer/mkvmuxer.cc \ + mkvmuxer/mkvmuxerutil.cc \ + mkvmuxer/mkvwriter.cc +include $(BUILD_STATIC_LIBRARY) diff --git a/third_party/aom/third_party/libwebm/LICENSE.TXT b/third_party/aom/third_party/libwebm/LICENSE.TXT new file mode 100644 index 0000000000..7a6f99547d --- /dev/null +++ b/third_party/aom/third_party/libwebm/LICENSE.TXT @@ -0,0 +1,30 @@ +Copyright (c) 2010, Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/third_party/aom/third_party/libwebm/PATENTS.TXT b/third_party/aom/third_party/libwebm/PATENTS.TXT new file mode 100644 index 0000000000..caedf607e9 --- /dev/null +++ b/third_party/aom/third_party/libwebm/PATENTS.TXT @@ -0,0 +1,23 @@ +Additional IP Rights Grant (Patents) +------------------------------------ + +"These implementations" means the copyrightable works that implement the WebM +codecs distributed by Google as part of the WebM Project. + +Google hereby grants to you a perpetual, worldwide, non-exclusive, no-charge, +royalty-free, irrevocable (except as stated in this section) patent license to +make, have made, use, offer to sell, sell, import, transfer, and otherwise +run, modify and propagate the contents of these implementations of WebM, where +such license applies only to those patent claims, both currently owned by +Google and acquired in the future, licensable by Google that are necessarily +infringed by these implementations of WebM. This grant does not include claims +that would be infringed only as a consequence of further modification of these +implementations. If you or your agent or exclusive licensee institute or order +or agree to the institution of patent litigation or any other patent +enforcement activity against any entity (including a cross-claim or +counterclaim in a lawsuit) alleging that any of these implementations of WebM +or any code incorporated within any of these implementations of WebM +constitute direct or contributory patent infringement, or inducement of +patent infringement, then any patent rights granted to you under this License +for these implementations of WebM shall terminate as of the date such +litigation is filed. diff --git a/third_party/aom/third_party/libwebm/README.libaom b/third_party/aom/third_party/libwebm/README.libaom new file mode 100644 index 0000000000..73f8303222 --- /dev/null +++ b/third_party/aom/third_party/libwebm/README.libaom @@ -0,0 +1,10 @@ +URL: https://chromium.googlesource.com/webm/libwebm +Version: 32d5ac49414a8914ec1e1f285f3f927c6e8ec29d +License: BSD +License File: LICENSE.txt + +Description: +libwebm is used to handle WebM container I/O. + +Local Changes: +* diff --git a/third_party/aom/third_party/libwebm/common/file_util.cc b/third_party/aom/third_party/libwebm/common/file_util.cc new file mode 100644 index 0000000000..4f91318f3e --- /dev/null +++ b/third_party/aom/third_party/libwebm/common/file_util.cc @@ -0,0 +1,67 @@ +// Copyright (c) 2016 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +#include "common/file_util.h" + +#include +#ifndef _MSC_VER +#include // close() +#endif + +#include +#include +#include +#include + +namespace libwebm { + +std::string GetTempFileName() { +#if !defined _MSC_VER && !defined __MINGW32__ + char temp_file_name_template[] = "libwebm_temp.XXXXXX"; + int fd = mkstemp(temp_file_name_template); + if (fd != -1) { + close(fd); + return std::string(temp_file_name_template); + } + return std::string(); +#else + char tmp_file_name[_MAX_PATH]; + errno_t err = tmpnam_s(tmp_file_name); + if (err == 0) { + return std::string(tmp_file_name); + } + return std::string(); +#endif +} + +uint64_t GetFileSize(const std::string& file_name) { + uint64_t file_size = 0; +#ifndef _MSC_VER + struct stat st; + st.st_size = 0; + if (stat(file_name.c_str(), &st) == 0) { +#else + struct _stat st; + st.st_size = 0; + if (_stat(file_name.c_str(), &st) == 0) { +#endif + file_size = st.st_size; + } + return file_size; +} + +TempFileDeleter::TempFileDeleter() { file_name_ = GetTempFileName(); } + +TempFileDeleter::~TempFileDeleter() { + std::ifstream file(file_name_.c_str()); + if (file.good()) { + file.close(); + std::remove(file_name_.c_str()); + } +} + +} // namespace libwebm diff --git a/third_party/aom/third_party/libwebm/common/file_util.h b/third_party/aom/third_party/libwebm/common/file_util.h new file mode 100644 index 0000000000..0e71eac11e --- /dev/null +++ b/third_party/aom/third_party/libwebm/common/file_util.h @@ -0,0 +1,41 @@ +// Copyright (c) 2016 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +#ifndef LIBWEBM_COMMON_FILE_UTIL_H_ +#define LIBWEBM_COMMON_FILE_UTIL_H_ + +#include + +#include + +#include "mkvmuxer/mkvmuxertypes.h" // LIBWEBM_DISALLOW_COPY_AND_ASSIGN() + +namespace libwebm { + +// Returns a temporary file name. +std::string GetTempFileName(); + +// Returns size of file specified by |file_name|, or 0 upon failure. +uint64_t GetFileSize(const std::string& file_name); + +// Manages life of temporary file specified at time of construction. Deletes +// file upon destruction. +class TempFileDeleter { + public: + TempFileDeleter(); + explicit TempFileDeleter(std::string file_name) : file_name_(file_name) {} + ~TempFileDeleter(); + const std::string& name() const { return file_name_; } + + private: + std::string file_name_; + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(TempFileDeleter); +}; + +} // namespace libwebm + +#endif // LIBWEBM_COMMON_FILE_UTIL_H_ \ No newline at end of file diff --git a/third_party/aom/third_party/libwebm/common/hdr_util.cc b/third_party/aom/third_party/libwebm/common/hdr_util.cc new file mode 100644 index 0000000000..e1a9842fb6 --- /dev/null +++ b/third_party/aom/third_party/libwebm/common/hdr_util.cc @@ -0,0 +1,182 @@ +// Copyright (c) 2016 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +#include "hdr_util.h" + +#include +#include + +#include "mkvparser/mkvparser.h" + +namespace libwebm { +bool CopyPrimaryChromaticity(const mkvparser::PrimaryChromaticity& parser_pc, + PrimaryChromaticityPtr* muxer_pc) { + muxer_pc->reset(new (std::nothrow) + mkvmuxer::PrimaryChromaticity(parser_pc.x, parser_pc.y)); + if (!muxer_pc->get()) + return false; + return true; +} + +bool MasteringMetadataValuePresent(double value) { + return value != mkvparser::MasteringMetadata::kValueNotPresent; +} + +bool CopyMasteringMetadata(const mkvparser::MasteringMetadata& parser_mm, + mkvmuxer::MasteringMetadata* muxer_mm) { + if (MasteringMetadataValuePresent(parser_mm.luminance_max)) + muxer_mm->luminance_max = parser_mm.luminance_max; + if (MasteringMetadataValuePresent(parser_mm.luminance_min)) + muxer_mm->luminance_min = parser_mm.luminance_min; + + PrimaryChromaticityPtr r_ptr(NULL); + PrimaryChromaticityPtr g_ptr(NULL); + PrimaryChromaticityPtr b_ptr(NULL); + PrimaryChromaticityPtr wp_ptr(NULL); + + if (parser_mm.r) { + if (!CopyPrimaryChromaticity(*parser_mm.r, &r_ptr)) + return false; + } + if (parser_mm.g) { + if (!CopyPrimaryChromaticity(*parser_mm.g, &g_ptr)) + return false; + } + if (parser_mm.b) { + if (!CopyPrimaryChromaticity(*parser_mm.b, &b_ptr)) + return false; + } + if (parser_mm.white_point) { + if (!CopyPrimaryChromaticity(*parser_mm.white_point, &wp_ptr)) + return false; + } + + if (!muxer_mm->SetChromaticity(r_ptr.get(), g_ptr.get(), b_ptr.get(), + wp_ptr.get())) { + return false; + } + + return true; +} + +bool ColourValuePresent(long long value) { + return value != mkvparser::Colour::kValueNotPresent; +} + +bool CopyColour(const mkvparser::Colour& parser_colour, + mkvmuxer::Colour* muxer_colour) { + if (!muxer_colour) + return false; + + if (ColourValuePresent(parser_colour.matrix_coefficients)) + muxer_colour->matrix_coefficients = parser_colour.matrix_coefficients; + if (ColourValuePresent(parser_colour.bits_per_channel)) + muxer_colour->bits_per_channel = parser_colour.bits_per_channel; + if (ColourValuePresent(parser_colour.chroma_subsampling_horz)) + muxer_colour->chroma_subsampling_horz = + parser_colour.chroma_subsampling_horz; + if (ColourValuePresent(parser_colour.chroma_subsampling_vert)) + muxer_colour->chroma_subsampling_vert = + parser_colour.chroma_subsampling_vert; + if (ColourValuePresent(parser_colour.cb_subsampling_horz)) + muxer_colour->cb_subsampling_horz = parser_colour.cb_subsampling_horz; + if (ColourValuePresent(parser_colour.cb_subsampling_vert)) + muxer_colour->cb_subsampling_vert = parser_colour.cb_subsampling_vert; + if (ColourValuePresent(parser_colour.chroma_siting_horz)) + muxer_colour->chroma_siting_horz = parser_colour.chroma_siting_horz; + if (ColourValuePresent(parser_colour.chroma_siting_vert)) + muxer_colour->chroma_siting_vert = parser_colour.chroma_siting_vert; + if (ColourValuePresent(parser_colour.range)) + muxer_colour->range = parser_colour.range; + if (ColourValuePresent(parser_colour.transfer_characteristics)) + muxer_colour->transfer_characteristics = + parser_colour.transfer_characteristics; + if (ColourValuePresent(parser_colour.primaries)) + muxer_colour->primaries = parser_colour.primaries; + if (ColourValuePresent(parser_colour.max_cll)) + muxer_colour->max_cll = parser_colour.max_cll; + if (ColourValuePresent(parser_colour.max_fall)) + muxer_colour->max_fall = parser_colour.max_fall; + + if (parser_colour.mastering_metadata) { + mkvmuxer::MasteringMetadata muxer_mm; + if (!CopyMasteringMetadata(*parser_colour.mastering_metadata, &muxer_mm)) + return false; + if (!muxer_colour->SetMasteringMetadata(muxer_mm)) + return false; + } + return true; +} + +// Format of VPx private data: +// +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ID Byte | Length | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | +// | | +// : Bytes 1..Length of Codec Feature : +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// ID Byte Format +// ID byte is an unsigned byte. +// 0 1 2 3 4 5 6 7 +// +-+-+-+-+-+-+-+-+ +// |X| ID | +// +-+-+-+-+-+-+-+-+ +// +// The X bit is reserved. +// +// Currently only profile level is supported. ID byte must be set to 1, and +// length must be 1. Supported values are: +// +// 10: Level 1 +// 11: Level 1.1 +// 20: Level 2 +// 21: Level 2.1 +// 30: Level 3 +// 31: Level 3.1 +// 40: Level 4 +// 41: Level 4.1 +// 50: Level 5 +// 51: Level 5.1 +// 52: Level 5.2 +// 60: Level 6 +// 61: Level 6.1 +// 62: Level 6.2 +// +// See the following link for more information: +// http://www.webmproject.org/vp9/profiles/ +int ParseVpxCodecPrivate(const uint8_t* private_data, int32_t length) { + const int kVpxCodecPrivateLength = 3; + if (!private_data || length != kVpxCodecPrivateLength) + return 0; + + const uint8_t id_byte = *private_data; + if (id_byte != 1) + return 0; + + const int kVpxProfileLength = 1; + const uint8_t length_byte = private_data[1]; + if (length_byte != kVpxProfileLength) + return 0; + + const int level = static_cast(private_data[2]); + + const int kNumLevels = 14; + const int levels[kNumLevels] = {10, 11, 20, 21, 30, 31, 40, + 41, 50, 51, 52, 60, 61, 62}; + + for (int i = 0; i < kNumLevels; ++i) { + if (level == levels[i]) + return level; + } + + return 0; +} +} // namespace libwebm diff --git a/third_party/aom/third_party/libwebm/common/hdr_util.h b/third_party/aom/third_party/libwebm/common/hdr_util.h new file mode 100644 index 0000000000..d30c2b9f2a --- /dev/null +++ b/third_party/aom/third_party/libwebm/common/hdr_util.h @@ -0,0 +1,51 @@ +// Copyright (c) 2016 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +#ifndef LIBWEBM_COMMON_HDR_UTIL_H_ +#define LIBWEBM_COMMON_HDR_UTIL_H_ + +#include + +#include + +#include "mkvmuxer/mkvmuxer.h" + +namespace mkvparser { +struct Colour; +struct MasteringMetadata; +struct PrimaryChromaticity; +} // namespace mkvparser + +namespace libwebm { +// Utility types and functions for working with the Colour element and its +// children. Copiers return true upon success. Presence functions return true +// when the specified element is present. + +// TODO(tomfinegan): These should be moved to libwebm_utils once c++11 is +// required by libwebm. + +typedef std::auto_ptr PrimaryChromaticityPtr; + +bool CopyPrimaryChromaticity(const mkvparser::PrimaryChromaticity& parser_pc, + PrimaryChromaticityPtr* muxer_pc); + +bool MasteringMetadataValuePresent(double value); + +bool CopyMasteringMetadata(const mkvparser::MasteringMetadata& parser_mm, + mkvmuxer::MasteringMetadata* muxer_mm); + +bool ColourValuePresent(long long value); + +bool CopyColour(const mkvparser::Colour& parser_colour, + mkvmuxer::Colour* muxer_colour); + +// Returns VP9 profile upon success or 0 upon failure. +int ParseVpxCodecPrivate(const uint8_t* private_data, int32_t length); + +} // namespace libwebm + +#endif // LIBWEBM_COMMON_HDR_UTIL_H_ diff --git a/third_party/aom/third_party/libwebm/common/webmids.h b/third_party/aom/third_party/libwebm/common/webmids.h new file mode 100644 index 0000000000..32a0c5fb91 --- /dev/null +++ b/third_party/aom/third_party/libwebm/common/webmids.h @@ -0,0 +1,184 @@ +// Copyright (c) 2012 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. + +#ifndef COMMON_WEBMIDS_H_ +#define COMMON_WEBMIDS_H_ + +namespace libwebm { + +enum MkvId { + kMkvEBML = 0x1A45DFA3, + kMkvEBMLVersion = 0x4286, + kMkvEBMLReadVersion = 0x42F7, + kMkvEBMLMaxIDLength = 0x42F2, + kMkvEBMLMaxSizeLength = 0x42F3, + kMkvDocType = 0x4282, + kMkvDocTypeVersion = 0x4287, + kMkvDocTypeReadVersion = 0x4285, + kMkvVoid = 0xEC, + kMkvSignatureSlot = 0x1B538667, + kMkvSignatureAlgo = 0x7E8A, + kMkvSignatureHash = 0x7E9A, + kMkvSignaturePublicKey = 0x7EA5, + kMkvSignature = 0x7EB5, + kMkvSignatureElements = 0x7E5B, + kMkvSignatureElementList = 0x7E7B, + kMkvSignedElement = 0x6532, + // segment + kMkvSegment = 0x18538067, + // Meta Seek Information + kMkvSeekHead = 0x114D9B74, + kMkvSeek = 0x4DBB, + kMkvSeekID = 0x53AB, + kMkvSeekPosition = 0x53AC, + // Segment Information + kMkvInfo = 0x1549A966, + kMkvTimecodeScale = 0x2AD7B1, + kMkvDuration = 0x4489, + kMkvDateUTC = 0x4461, + kMkvTitle = 0x7BA9, + kMkvMuxingApp = 0x4D80, + kMkvWritingApp = 0x5741, + // Cluster + kMkvCluster = 0x1F43B675, + kMkvTimecode = 0xE7, + kMkvPrevSize = 0xAB, + kMkvBlockGroup = 0xA0, + kMkvBlock = 0xA1, + kMkvBlockDuration = 0x9B, + kMkvReferenceBlock = 0xFB, + kMkvLaceNumber = 0xCC, + kMkvSimpleBlock = 0xA3, + kMkvBlockAdditions = 0x75A1, + kMkvBlockMore = 0xA6, + kMkvBlockAddID = 0xEE, + kMkvBlockAdditional = 0xA5, + kMkvDiscardPadding = 0x75A2, + // Track + kMkvTracks = 0x1654AE6B, + kMkvTrackEntry = 0xAE, + kMkvTrackNumber = 0xD7, + kMkvTrackUID = 0x73C5, + kMkvTrackType = 0x83, + kMkvFlagEnabled = 0xB9, + kMkvFlagDefault = 0x88, + kMkvFlagForced = 0x55AA, + kMkvFlagLacing = 0x9C, + kMkvDefaultDuration = 0x23E383, + kMkvMaxBlockAdditionID = 0x55EE, + kMkvName = 0x536E, + kMkvLanguage = 0x22B59C, + kMkvCodecID = 0x86, + kMkvCodecPrivate = 0x63A2, + kMkvCodecName = 0x258688, + kMkvCodecDelay = 0x56AA, + kMkvSeekPreRoll = 0x56BB, + // video + kMkvVideo = 0xE0, + kMkvFlagInterlaced = 0x9A, + kMkvStereoMode = 0x53B8, + kMkvAlphaMode = 0x53C0, + kMkvPixelWidth = 0xB0, + kMkvPixelHeight = 0xBA, + kMkvPixelCropBottom = 0x54AA, + kMkvPixelCropTop = 0x54BB, + kMkvPixelCropLeft = 0x54CC, + kMkvPixelCropRight = 0x54DD, + kMkvDisplayWidth = 0x54B0, + kMkvDisplayHeight = 0x54BA, + kMkvDisplayUnit = 0x54B2, + kMkvAspectRatioType = 0x54B3, + kMkvFrameRate = 0x2383E3, + // end video + // colour + kMkvColour = 0x55B0, + kMkvMatrixCoefficients = 0x55B1, + kMkvBitsPerChannel = 0x55B2, + kMkvChromaSubsamplingHorz = 0x55B3, + kMkvChromaSubsamplingVert = 0x55B4, + kMkvCbSubsamplingHorz = 0x55B5, + kMkvCbSubsamplingVert = 0x55B6, + kMkvChromaSitingHorz = 0x55B7, + kMkvChromaSitingVert = 0x55B8, + kMkvRange = 0x55B9, + kMkvTransferCharacteristics = 0x55BA, + kMkvPrimaries = 0x55BB, + kMkvMaxCLL = 0x55BC, + kMkvMaxFALL = 0x55BD, + // mastering metadata + kMkvMasteringMetadata = 0x55D0, + kMkvPrimaryRChromaticityX = 0x55D1, + kMkvPrimaryRChromaticityY = 0x55D2, + kMkvPrimaryGChromaticityX = 0x55D3, + kMkvPrimaryGChromaticityY = 0x55D4, + kMkvPrimaryBChromaticityX = 0x55D5, + kMkvPrimaryBChromaticityY = 0x55D6, + kMkvWhitePointChromaticityX = 0x55D7, + kMkvWhitePointChromaticityY = 0x55D8, + kMkvLuminanceMax = 0x55D9, + kMkvLuminanceMin = 0x55DA, + // end mastering metadata + // end colour + // audio + kMkvAudio = 0xE1, + kMkvSamplingFrequency = 0xB5, + kMkvOutputSamplingFrequency = 0x78B5, + kMkvChannels = 0x9F, + kMkvBitDepth = 0x6264, + // end audio + // ContentEncodings + kMkvContentEncodings = 0x6D80, + kMkvContentEncoding = 0x6240, + kMkvContentEncodingOrder = 0x5031, + kMkvContentEncodingScope = 0x5032, + kMkvContentEncodingType = 0x5033, + kMkvContentCompression = 0x5034, + kMkvContentCompAlgo = 0x4254, + kMkvContentCompSettings = 0x4255, + kMkvContentEncryption = 0x5035, + kMkvContentEncAlgo = 0x47E1, + kMkvContentEncKeyID = 0x47E2, + kMkvContentSignature = 0x47E3, + kMkvContentSigKeyID = 0x47E4, + kMkvContentSigAlgo = 0x47E5, + kMkvContentSigHashAlgo = 0x47E6, + kMkvContentEncAESSettings = 0x47E7, + kMkvAESSettingsCipherMode = 0x47E8, + kMkvAESSettingsCipherInitData = 0x47E9, + // end ContentEncodings + // Cueing Data + kMkvCues = 0x1C53BB6B, + kMkvCuePoint = 0xBB, + kMkvCueTime = 0xB3, + kMkvCueTrackPositions = 0xB7, + kMkvCueTrack = 0xF7, + kMkvCueClusterPosition = 0xF1, + kMkvCueBlockNumber = 0x5378, + // Chapters + kMkvChapters = 0x1043A770, + kMkvEditionEntry = 0x45B9, + kMkvChapterAtom = 0xB6, + kMkvChapterUID = 0x73C4, + kMkvChapterStringUID = 0x5654, + kMkvChapterTimeStart = 0x91, + kMkvChapterTimeEnd = 0x92, + kMkvChapterDisplay = 0x80, + kMkvChapString = 0x85, + kMkvChapLanguage = 0x437C, + kMkvChapCountry = 0x437E, + // Tags + kMkvTags = 0x1254C367, + kMkvTag = 0x7373, + kMkvSimpleTag = 0x67C8, + kMkvTagName = 0x45A3, + kMkvTagString = 0x4487 +}; + +} // namespace libwebm + +#endif // COMMON_WEBMIDS_H_ diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.cc b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.cc new file mode 100644 index 0000000000..689b5ae427 --- /dev/null +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.cc @@ -0,0 +1,3769 @@ +// Copyright (c) 2012 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. + +#include "mkvmuxer/mkvmuxer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/webmids.h" +#include "mkvmuxer/mkvmuxerutil.h" +#include "mkvmuxer/mkvwriter.h" +#include "mkvparser/mkvparser.h" + +namespace mkvmuxer { + +const float MasteringMetadata::kValueNotPresent = FLT_MAX; +const uint64_t Colour::kValueNotPresent = UINT64_MAX; + +namespace { +// Deallocate the string designated by |dst|, and then copy the |src| +// string to |dst|. The caller owns both the |src| string and the +// |dst| copy (hence the caller is responsible for eventually +// deallocating the strings, either directly, or indirectly via +// StrCpy). Returns true if the source string was successfully copied +// to the destination. +bool StrCpy(const char* src, char** dst_ptr) { + if (dst_ptr == NULL) + return false; + + char*& dst = *dst_ptr; + + delete[] dst; + dst = NULL; + + if (src == NULL) + return true; + + const size_t size = strlen(src) + 1; + + dst = new (std::nothrow) char[size]; // NOLINT + if (dst == NULL) + return false; + + strcpy(dst, src); // NOLINT + return true; +} + +typedef std::auto_ptr PrimaryChromaticityPtr; +bool CopyChromaticity(const PrimaryChromaticity* src, + PrimaryChromaticityPtr* dst) { + if (!dst) + return false; + + dst->reset(new (std::nothrow) PrimaryChromaticity(src->x, src->y)); + if (!dst->get()) + return false; + + return true; +} + +} // namespace + +/////////////////////////////////////////////////////////////// +// +// IMkvWriter Class + +IMkvWriter::IMkvWriter() {} + +IMkvWriter::~IMkvWriter() {} + +bool WriteEbmlHeader(IMkvWriter* writer, uint64_t doc_type_version) { + // Level 0 + uint64_t size = EbmlElementSize(libwebm::kMkvEBMLVersion, UINT64_C(1)); + size += EbmlElementSize(libwebm::kMkvEBMLReadVersion, UINT64_C(1)); + size += EbmlElementSize(libwebm::kMkvEBMLMaxIDLength, UINT64_C(4)); + size += EbmlElementSize(libwebm::kMkvEBMLMaxSizeLength, UINT64_C(8)); + size += EbmlElementSize(libwebm::kMkvDocType, "webm"); + size += EbmlElementSize(libwebm::kMkvDocTypeVersion, doc_type_version); + size += EbmlElementSize(libwebm::kMkvDocTypeReadVersion, UINT64_C(2)); + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvEBML, size)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvEBMLVersion, UINT64_C(1))) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvEBMLReadVersion, UINT64_C(1))) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvEBMLMaxIDLength, UINT64_C(4))) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvEBMLMaxSizeLength, UINT64_C(8))) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvDocType, "webm")) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvDocTypeVersion, doc_type_version)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvDocTypeReadVersion, UINT64_C(2))) + return false; + + return true; +} + +bool WriteEbmlHeader(IMkvWriter* writer) { + return WriteEbmlHeader(writer, mkvmuxer::Segment::kDefaultDocTypeVersion); +} + +bool ChunkedCopy(mkvparser::IMkvReader* source, mkvmuxer::IMkvWriter* dst, + int64_t start, int64_t size) { + // TODO(vigneshv): Check if this is a reasonable value. + const uint32_t kBufSize = 2048; + uint8_t* buf = new uint8_t[kBufSize]; + int64_t offset = start; + while (size > 0) { + const int64_t read_len = (size > kBufSize) ? kBufSize : size; + if (source->Read(offset, static_cast(read_len), buf)) + return false; + dst->Write(buf, static_cast(read_len)); + offset += read_len; + size -= read_len; + } + delete[] buf; + return true; +} + +/////////////////////////////////////////////////////////////// +// +// Frame Class + +Frame::Frame() + : add_id_(0), + additional_(NULL), + additional_length_(0), + duration_(0), + duration_set_(false), + frame_(NULL), + is_key_(false), + length_(0), + track_number_(0), + timestamp_(0), + discard_padding_(0), + reference_block_timestamp_(0), + reference_block_timestamp_set_(false) {} + +Frame::~Frame() { + delete[] frame_; + delete[] additional_; +} + +bool Frame::CopyFrom(const Frame& frame) { + delete[] frame_; + frame_ = NULL; + length_ = 0; + if (frame.length() > 0 && frame.frame() != NULL && + !Init(frame.frame(), frame.length())) { + return false; + } + add_id_ = 0; + delete[] additional_; + additional_ = NULL; + additional_length_ = 0; + if (frame.additional_length() > 0 && frame.additional() != NULL && + !AddAdditionalData(frame.additional(), frame.additional_length(), + frame.add_id())) { + return false; + } + duration_ = frame.duration(); + duration_set_ = frame.duration_set(); + is_key_ = frame.is_key(); + track_number_ = frame.track_number(); + timestamp_ = frame.timestamp(); + discard_padding_ = frame.discard_padding(); + reference_block_timestamp_ = frame.reference_block_timestamp(); + reference_block_timestamp_set_ = frame.reference_block_timestamp_set(); + return true; +} + +bool Frame::Init(const uint8_t* frame, uint64_t length) { + uint8_t* const data = + new (std::nothrow) uint8_t[static_cast(length)]; // NOLINT + if (!data) + return false; + + delete[] frame_; + frame_ = data; + length_ = length; + + memcpy(frame_, frame, static_cast(length_)); + return true; +} + +bool Frame::AddAdditionalData(const uint8_t* additional, uint64_t length, + uint64_t add_id) { + uint8_t* const data = + new (std::nothrow) uint8_t[static_cast(length)]; // NOLINT + if (!data) + return false; + + delete[] additional_; + additional_ = data; + additional_length_ = length; + add_id_ = add_id; + + memcpy(additional_, additional, static_cast(additional_length_)); + return true; +} + +bool Frame::IsValid() const { + if (length_ == 0 || !frame_) { + return false; + } + if ((additional_length_ != 0 && !additional_) || + (additional_ != NULL && additional_length_ == 0)) { + return false; + } + if (track_number_ == 0 || track_number_ > kMaxTrackNumber) { + return false; + } + if (!CanBeSimpleBlock() && !is_key_ && !reference_block_timestamp_set_) { + return false; + } + return true; +} + +bool Frame::CanBeSimpleBlock() const { + return additional_ == NULL && discard_padding_ == 0 && duration_ == 0; +} + +void Frame::set_duration(uint64_t duration) { + duration_ = duration; + duration_set_ = true; +} + +void Frame::set_reference_block_timestamp(int64_t reference_block_timestamp) { + reference_block_timestamp_ = reference_block_timestamp; + reference_block_timestamp_set_ = true; +} + +/////////////////////////////////////////////////////////////// +// +// CuePoint Class + +CuePoint::CuePoint() + : time_(0), + track_(0), + cluster_pos_(0), + block_number_(1), + output_block_number_(true) {} + +CuePoint::~CuePoint() {} + +bool CuePoint::Write(IMkvWriter* writer) const { + if (!writer || track_ < 1 || cluster_pos_ < 1) + return false; + + uint64_t size = + EbmlElementSize(libwebm::kMkvCueClusterPosition, cluster_pos_); + size += EbmlElementSize(libwebm::kMkvCueTrack, track_); + if (output_block_number_ && block_number_ > 1) + size += EbmlElementSize(libwebm::kMkvCueBlockNumber, block_number_); + const uint64_t track_pos_size = + EbmlMasterElementSize(libwebm::kMkvCueTrackPositions, size) + size; + const uint64_t payload_size = + EbmlElementSize(libwebm::kMkvCueTime, time_) + track_pos_size; + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvCuePoint, payload_size)) + return false; + + const int64_t payload_position = writer->Position(); + if (payload_position < 0) + return false; + + if (!WriteEbmlElement(writer, libwebm::kMkvCueTime, time_)) + return false; + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvCueTrackPositions, size)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvCueTrack, track_)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvCueClusterPosition, cluster_pos_)) + return false; + if (output_block_number_ && block_number_ > 1) + if (!WriteEbmlElement(writer, libwebm::kMkvCueBlockNumber, block_number_)) + return false; + + const int64_t stop_position = writer->Position(); + if (stop_position < 0) + return false; + + if (stop_position - payload_position != static_cast(payload_size)) + return false; + + return true; +} + +uint64_t CuePoint::PayloadSize() const { + uint64_t size = + EbmlElementSize(libwebm::kMkvCueClusterPosition, cluster_pos_); + size += EbmlElementSize(libwebm::kMkvCueTrack, track_); + if (output_block_number_ && block_number_ > 1) + size += EbmlElementSize(libwebm::kMkvCueBlockNumber, block_number_); + const uint64_t track_pos_size = + EbmlMasterElementSize(libwebm::kMkvCueTrackPositions, size) + size; + const uint64_t payload_size = + EbmlElementSize(libwebm::kMkvCueTime, time_) + track_pos_size; + + return payload_size; +} + +uint64_t CuePoint::Size() const { + const uint64_t payload_size = PayloadSize(); + return EbmlMasterElementSize(libwebm::kMkvCuePoint, payload_size) + + payload_size; +} + +/////////////////////////////////////////////////////////////// +// +// Cues Class + +Cues::Cues() + : cue_entries_capacity_(0), + cue_entries_size_(0), + cue_entries_(NULL), + output_block_number_(true) {} + +Cues::~Cues() { + if (cue_entries_) { + for (int32_t i = 0; i < cue_entries_size_; ++i) { + CuePoint* const cue = cue_entries_[i]; + delete cue; + } + delete[] cue_entries_; + } +} + +bool Cues::AddCue(CuePoint* cue) { + if (!cue) + return false; + + if ((cue_entries_size_ + 1) > cue_entries_capacity_) { + // Add more CuePoints. + const int32_t new_capacity = + (!cue_entries_capacity_) ? 2 : cue_entries_capacity_ * 2; + + if (new_capacity < 1) + return false; + + CuePoint** const cues = + new (std::nothrow) CuePoint*[new_capacity]; // NOLINT + if (!cues) + return false; + + for (int32_t i = 0; i < cue_entries_size_; ++i) { + cues[i] = cue_entries_[i]; + } + + delete[] cue_entries_; + + cue_entries_ = cues; + cue_entries_capacity_ = new_capacity; + } + + cue->set_output_block_number(output_block_number_); + cue_entries_[cue_entries_size_++] = cue; + return true; +} + +CuePoint* Cues::GetCueByIndex(int32_t index) const { + if (cue_entries_ == NULL) + return NULL; + + if (index >= cue_entries_size_) + return NULL; + + return cue_entries_[index]; +} + +uint64_t Cues::Size() { + uint64_t size = 0; + for (int32_t i = 0; i < cue_entries_size_; ++i) + size += GetCueByIndex(i)->Size(); + size += EbmlMasterElementSize(libwebm::kMkvCues, size); + return size; +} + +bool Cues::Write(IMkvWriter* writer) const { + if (!writer) + return false; + + uint64_t size = 0; + for (int32_t i = 0; i < cue_entries_size_; ++i) { + const CuePoint* const cue = GetCueByIndex(i); + + if (!cue) + return false; + + size += cue->Size(); + } + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvCues, size)) + return false; + + const int64_t payload_position = writer->Position(); + if (payload_position < 0) + return false; + + for (int32_t i = 0; i < cue_entries_size_; ++i) { + const CuePoint* const cue = GetCueByIndex(i); + + if (!cue->Write(writer)) + return false; + } + + const int64_t stop_position = writer->Position(); + if (stop_position < 0) + return false; + + if (stop_position - payload_position != static_cast(size)) + return false; + + return true; +} + +/////////////////////////////////////////////////////////////// +// +// ContentEncAESSettings Class + +ContentEncAESSettings::ContentEncAESSettings() : cipher_mode_(kCTR) {} + +uint64_t ContentEncAESSettings::Size() const { + const uint64_t payload = PayloadSize(); + const uint64_t size = + EbmlMasterElementSize(libwebm::kMkvContentEncAESSettings, payload) + + payload; + return size; +} + +bool ContentEncAESSettings::Write(IMkvWriter* writer) const { + const uint64_t payload = PayloadSize(); + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvContentEncAESSettings, + payload)) + return false; + + const int64_t payload_position = writer->Position(); + if (payload_position < 0) + return false; + + if (!WriteEbmlElement(writer, libwebm::kMkvAESSettingsCipherMode, + cipher_mode_)) + return false; + + const int64_t stop_position = writer->Position(); + if (stop_position < 0 || + stop_position - payload_position != static_cast(payload)) + return false; + + return true; +} + +uint64_t ContentEncAESSettings::PayloadSize() const { + uint64_t size = + EbmlElementSize(libwebm::kMkvAESSettingsCipherMode, cipher_mode_); + return size; +} + +/////////////////////////////////////////////////////////////// +// +// ContentEncoding Class + +ContentEncoding::ContentEncoding() + : enc_algo_(5), + enc_key_id_(NULL), + encoding_order_(0), + encoding_scope_(1), + encoding_type_(1), + enc_key_id_length_(0) {} + +ContentEncoding::~ContentEncoding() { delete[] enc_key_id_; } + +bool ContentEncoding::SetEncryptionID(const uint8_t* id, uint64_t length) { + if (!id || length < 1) + return false; + + delete[] enc_key_id_; + + enc_key_id_ = + new (std::nothrow) uint8_t[static_cast(length)]; // NOLINT + if (!enc_key_id_) + return false; + + memcpy(enc_key_id_, id, static_cast(length)); + enc_key_id_length_ = length; + + return true; +} + +uint64_t ContentEncoding::Size() const { + const uint64_t encryption_size = EncryptionSize(); + const uint64_t encoding_size = EncodingSize(0, encryption_size); + const uint64_t encodings_size = + EbmlMasterElementSize(libwebm::kMkvContentEncoding, encoding_size) + + encoding_size; + + return encodings_size; +} + +bool ContentEncoding::Write(IMkvWriter* writer) const { + const uint64_t encryption_size = EncryptionSize(); + const uint64_t encoding_size = EncodingSize(0, encryption_size); + const uint64_t size = + EbmlMasterElementSize(libwebm::kMkvContentEncoding, encoding_size) + + encoding_size; + + const int64_t payload_position = writer->Position(); + if (payload_position < 0) + return false; + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvContentEncoding, + encoding_size)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvContentEncodingOrder, + encoding_order_)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvContentEncodingScope, + encoding_scope_)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvContentEncodingType, + encoding_type_)) + return false; + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvContentEncryption, + encryption_size)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvContentEncAlgo, enc_algo_)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvContentEncKeyID, enc_key_id_, + enc_key_id_length_)) + return false; + + if (!enc_aes_settings_.Write(writer)) + return false; + + const int64_t stop_position = writer->Position(); + if (stop_position < 0 || + stop_position - payload_position != static_cast(size)) + return false; + + return true; +} + +uint64_t ContentEncoding::EncodingSize(uint64_t compresion_size, + uint64_t encryption_size) const { + // TODO(fgalligan): Add support for compression settings. + if (compresion_size != 0) + return 0; + + uint64_t encoding_size = 0; + + if (encryption_size > 0) { + encoding_size += + EbmlMasterElementSize(libwebm::kMkvContentEncryption, encryption_size) + + encryption_size; + } + encoding_size += + EbmlElementSize(libwebm::kMkvContentEncodingType, encoding_type_); + encoding_size += + EbmlElementSize(libwebm::kMkvContentEncodingScope, encoding_scope_); + encoding_size += + EbmlElementSize(libwebm::kMkvContentEncodingOrder, encoding_order_); + + return encoding_size; +} + +uint64_t ContentEncoding::EncryptionSize() const { + const uint64_t aes_size = enc_aes_settings_.Size(); + + uint64_t encryption_size = EbmlElementSize(libwebm::kMkvContentEncKeyID, + enc_key_id_, enc_key_id_length_); + encryption_size += EbmlElementSize(libwebm::kMkvContentEncAlgo, enc_algo_); + + return encryption_size + aes_size; +} + +/////////////////////////////////////////////////////////////// +// +// Track Class + +Track::Track(unsigned int* seed) + : codec_id_(NULL), + codec_private_(NULL), + language_(NULL), + max_block_additional_id_(0), + name_(NULL), + number_(0), + type_(0), + uid_(MakeUID(seed)), + codec_delay_(0), + seek_pre_roll_(0), + default_duration_(0), + codec_private_length_(0), + content_encoding_entries_(NULL), + content_encoding_entries_size_(0) {} + +Track::~Track() { + delete[] codec_id_; + delete[] codec_private_; + delete[] language_; + delete[] name_; + + if (content_encoding_entries_) { + for (uint32_t i = 0; i < content_encoding_entries_size_; ++i) { + ContentEncoding* const encoding = content_encoding_entries_[i]; + delete encoding; + } + delete[] content_encoding_entries_; + } +} + +bool Track::AddContentEncoding() { + const uint32_t count = content_encoding_entries_size_ + 1; + + ContentEncoding** const content_encoding_entries = + new (std::nothrow) ContentEncoding*[count]; // NOLINT + if (!content_encoding_entries) + return false; + + ContentEncoding* const content_encoding = + new (std::nothrow) ContentEncoding(); // NOLINT + if (!content_encoding) { + delete[] content_encoding_entries; + return false; + } + + for (uint32_t i = 0; i < content_encoding_entries_size_; ++i) { + content_encoding_entries[i] = content_encoding_entries_[i]; + } + + delete[] content_encoding_entries_; + + content_encoding_entries_ = content_encoding_entries; + content_encoding_entries_[content_encoding_entries_size_] = content_encoding; + content_encoding_entries_size_ = count; + return true; +} + +ContentEncoding* Track::GetContentEncodingByIndex(uint32_t index) const { + if (content_encoding_entries_ == NULL) + return NULL; + + if (index >= content_encoding_entries_size_) + return NULL; + + return content_encoding_entries_[index]; +} + +uint64_t Track::PayloadSize() const { + uint64_t size = EbmlElementSize(libwebm::kMkvTrackNumber, number_); + size += EbmlElementSize(libwebm::kMkvTrackUID, uid_); + size += EbmlElementSize(libwebm::kMkvTrackType, type_); + if (codec_id_) + size += EbmlElementSize(libwebm::kMkvCodecID, codec_id_); + if (codec_private_) + size += EbmlElementSize(libwebm::kMkvCodecPrivate, codec_private_, + codec_private_length_); + if (language_) + size += EbmlElementSize(libwebm::kMkvLanguage, language_); + if (name_) + size += EbmlElementSize(libwebm::kMkvName, name_); + if (max_block_additional_id_) + size += EbmlElementSize(libwebm::kMkvMaxBlockAdditionID, + max_block_additional_id_); + if (codec_delay_) + size += EbmlElementSize(libwebm::kMkvCodecDelay, codec_delay_); + if (seek_pre_roll_) + size += EbmlElementSize(libwebm::kMkvSeekPreRoll, seek_pre_roll_); + if (default_duration_) + size += EbmlElementSize(libwebm::kMkvDefaultDuration, default_duration_); + + if (content_encoding_entries_size_ > 0) { + uint64_t content_encodings_size = 0; + for (uint32_t i = 0; i < content_encoding_entries_size_; ++i) { + ContentEncoding* const encoding = content_encoding_entries_[i]; + content_encodings_size += encoding->Size(); + } + + size += EbmlMasterElementSize(libwebm::kMkvContentEncodings, + content_encodings_size) + + content_encodings_size; + } + + return size; +} + +uint64_t Track::Size() const { + uint64_t size = PayloadSize(); + size += EbmlMasterElementSize(libwebm::kMkvTrackEntry, size); + return size; +} + +bool Track::Write(IMkvWriter* writer) const { + if (!writer) + return false; + + // mandatory elements without a default value. + if (!type_ || !codec_id_) + return false; + + // |size| may be bigger than what is written out in this function because + // derived classes may write out more data in the Track element. + const uint64_t payload_size = PayloadSize(); + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvTrackEntry, payload_size)) + return false; + + uint64_t size = EbmlElementSize(libwebm::kMkvTrackNumber, number_); + size += EbmlElementSize(libwebm::kMkvTrackUID, uid_); + size += EbmlElementSize(libwebm::kMkvTrackType, type_); + if (codec_id_) + size += EbmlElementSize(libwebm::kMkvCodecID, codec_id_); + if (codec_private_) + size += EbmlElementSize(libwebm::kMkvCodecPrivate, codec_private_, + codec_private_length_); + if (language_) + size += EbmlElementSize(libwebm::kMkvLanguage, language_); + if (name_) + size += EbmlElementSize(libwebm::kMkvName, name_); + if (max_block_additional_id_) + size += EbmlElementSize(libwebm::kMkvMaxBlockAdditionID, + max_block_additional_id_); + if (codec_delay_) + size += EbmlElementSize(libwebm::kMkvCodecDelay, codec_delay_); + if (seek_pre_roll_) + size += EbmlElementSize(libwebm::kMkvSeekPreRoll, seek_pre_roll_); + if (default_duration_) + size += EbmlElementSize(libwebm::kMkvDefaultDuration, default_duration_); + + const int64_t payload_position = writer->Position(); + if (payload_position < 0) + return false; + + if (!WriteEbmlElement(writer, libwebm::kMkvTrackNumber, number_)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvTrackUID, uid_)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvTrackType, type_)) + return false; + if (max_block_additional_id_) { + if (!WriteEbmlElement(writer, libwebm::kMkvMaxBlockAdditionID, + max_block_additional_id_)) { + return false; + } + } + if (codec_delay_) { + if (!WriteEbmlElement(writer, libwebm::kMkvCodecDelay, codec_delay_)) + return false; + } + if (seek_pre_roll_) { + if (!WriteEbmlElement(writer, libwebm::kMkvSeekPreRoll, seek_pre_roll_)) + return false; + } + if (default_duration_) { + if (!WriteEbmlElement(writer, libwebm::kMkvDefaultDuration, + default_duration_)) + return false; + } + if (codec_id_) { + if (!WriteEbmlElement(writer, libwebm::kMkvCodecID, codec_id_)) + return false; + } + if (codec_private_) { + if (!WriteEbmlElement(writer, libwebm::kMkvCodecPrivate, codec_private_, + codec_private_length_)) + return false; + } + if (language_) { + if (!WriteEbmlElement(writer, libwebm::kMkvLanguage, language_)) + return false; + } + if (name_) { + if (!WriteEbmlElement(writer, libwebm::kMkvName, name_)) + return false; + } + + int64_t stop_position = writer->Position(); + if (stop_position < 0 || + stop_position - payload_position != static_cast(size)) + return false; + + if (content_encoding_entries_size_ > 0) { + uint64_t content_encodings_size = 0; + for (uint32_t i = 0; i < content_encoding_entries_size_; ++i) { + ContentEncoding* const encoding = content_encoding_entries_[i]; + content_encodings_size += encoding->Size(); + } + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvContentEncodings, + content_encodings_size)) + return false; + + for (uint32_t i = 0; i < content_encoding_entries_size_; ++i) { + ContentEncoding* const encoding = content_encoding_entries_[i]; + if (!encoding->Write(writer)) + return false; + } + } + + stop_position = writer->Position(); + if (stop_position < 0) + return false; + return true; +} + +bool Track::SetCodecPrivate(const uint8_t* codec_private, uint64_t length) { + if (!codec_private || length < 1) + return false; + + delete[] codec_private_; + + codec_private_ = + new (std::nothrow) uint8_t[static_cast(length)]; // NOLINT + if (!codec_private_) + return false; + + memcpy(codec_private_, codec_private, static_cast(length)); + codec_private_length_ = length; + + return true; +} + +void Track::set_codec_id(const char* codec_id) { + if (codec_id) { + delete[] codec_id_; + + const size_t length = strlen(codec_id) + 1; + codec_id_ = new (std::nothrow) char[length]; // NOLINT + if (codec_id_) { +#ifdef _MSC_VER + strcpy_s(codec_id_, length, codec_id); +#else + strcpy(codec_id_, codec_id); +#endif + } + } +} + +// TODO(fgalligan): Vet the language parameter. +void Track::set_language(const char* language) { + if (language) { + delete[] language_; + + const size_t length = strlen(language) + 1; + language_ = new (std::nothrow) char[length]; // NOLINT + if (language_) { +#ifdef _MSC_VER + strcpy_s(language_, length, language); +#else + strcpy(language_, language); +#endif + } + } +} + +void Track::set_name(const char* name) { + if (name) { + delete[] name_; + + const size_t length = strlen(name) + 1; + name_ = new (std::nothrow) char[length]; // NOLINT + if (name_) { +#ifdef _MSC_VER + strcpy_s(name_, length, name); +#else + strcpy(name_, name); +#endif + } + } +} + +/////////////////////////////////////////////////////////////// +// +// Colour and its child elements + +uint64_t PrimaryChromaticity::PrimaryChromaticityPayloadSize( + libwebm::MkvId x_id, libwebm::MkvId y_id) const { + return EbmlElementSize(x_id, x) + EbmlElementSize(y_id, y); +} + +bool PrimaryChromaticity::Write(IMkvWriter* writer, libwebm::MkvId x_id, + libwebm::MkvId y_id) const { + return WriteEbmlElement(writer, x_id, x) && WriteEbmlElement(writer, y_id, y); +} + +uint64_t MasteringMetadata::MasteringMetadataSize() const { + uint64_t size = PayloadSize(); + + if (size > 0) + size += EbmlMasterElementSize(libwebm::kMkvMasteringMetadata, size); + + return size; +} + +bool MasteringMetadata::Write(IMkvWriter* writer) const { + const uint64_t size = PayloadSize(); + + // Don't write an empty element. + if (size == 0) + return true; + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvMasteringMetadata, size)) + return false; + if (luminance_max != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvLuminanceMax, luminance_max)) { + return false; + } + if (luminance_min != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvLuminanceMin, luminance_min)) { + return false; + } + if (r_ && + !r_->Write(writer, libwebm::kMkvPrimaryRChromaticityX, + libwebm::kMkvPrimaryRChromaticityY)) { + return false; + } + if (g_ && + !g_->Write(writer, libwebm::kMkvPrimaryGChromaticityX, + libwebm::kMkvPrimaryGChromaticityY)) { + return false; + } + if (b_ && + !b_->Write(writer, libwebm::kMkvPrimaryBChromaticityX, + libwebm::kMkvPrimaryBChromaticityY)) { + return false; + } + if (white_point_ && + !white_point_->Write(writer, libwebm::kMkvWhitePointChromaticityX, + libwebm::kMkvWhitePointChromaticityY)) { + return false; + } + + return true; +} + +bool MasteringMetadata::SetChromaticity( + const PrimaryChromaticity* r, const PrimaryChromaticity* g, + const PrimaryChromaticity* b, const PrimaryChromaticity* white_point) { + PrimaryChromaticityPtr r_ptr(NULL); + if (r) { + if (!CopyChromaticity(r, &r_ptr)) + return false; + } + PrimaryChromaticityPtr g_ptr(NULL); + if (g) { + if (!CopyChromaticity(g, &g_ptr)) + return false; + } + PrimaryChromaticityPtr b_ptr(NULL); + if (b) { + if (!CopyChromaticity(b, &b_ptr)) + return false; + } + PrimaryChromaticityPtr wp_ptr(NULL); + if (white_point) { + if (!CopyChromaticity(white_point, &wp_ptr)) + return false; + } + + r_ = r_ptr.release(); + g_ = g_ptr.release(); + b_ = b_ptr.release(); + white_point_ = wp_ptr.release(); + return true; +} + +uint64_t MasteringMetadata::PayloadSize() const { + uint64_t size = 0; + + if (luminance_max != kValueNotPresent) + size += EbmlElementSize(libwebm::kMkvLuminanceMax, luminance_max); + if (luminance_min != kValueNotPresent) + size += EbmlElementSize(libwebm::kMkvLuminanceMin, luminance_min); + + if (r_) { + size += r_->PrimaryChromaticityPayloadSize( + libwebm::kMkvPrimaryRChromaticityX, libwebm::kMkvPrimaryRChromaticityY); + } + if (g_) { + size += g_->PrimaryChromaticityPayloadSize( + libwebm::kMkvPrimaryGChromaticityX, libwebm::kMkvPrimaryGChromaticityY); + } + if (b_) { + size += b_->PrimaryChromaticityPayloadSize( + libwebm::kMkvPrimaryBChromaticityX, libwebm::kMkvPrimaryBChromaticityY); + } + if (white_point_) { + size += white_point_->PrimaryChromaticityPayloadSize( + libwebm::kMkvWhitePointChromaticityX, + libwebm::kMkvWhitePointChromaticityY); + } + + return size; +} + +uint64_t Colour::ColourSize() const { + uint64_t size = PayloadSize(); + + if (size > 0) + size += EbmlMasterElementSize(libwebm::kMkvColour, size); + + return size; +} + +bool Colour::Write(IMkvWriter* writer) const { + const uint64_t size = PayloadSize(); + + // Don't write an empty element. + if (size == 0) + return true; + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvColour, size)) + return false; + + if (matrix_coefficients != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvMatrixCoefficients, + matrix_coefficients)) { + return false; + } + if (bits_per_channel != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvBitsPerChannel, + bits_per_channel)) { + return false; + } + if (chroma_subsampling_horz != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvChromaSubsamplingHorz, + chroma_subsampling_horz)) { + return false; + } + if (chroma_subsampling_vert != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvChromaSubsamplingVert, + chroma_subsampling_vert)) { + return false; + } + + if (cb_subsampling_horz != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvCbSubsamplingHorz, + cb_subsampling_horz)) { + return false; + } + if (cb_subsampling_vert != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvCbSubsamplingVert, + cb_subsampling_vert)) { + return false; + } + if (chroma_siting_horz != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvChromaSitingHorz, + chroma_siting_horz)) { + return false; + } + if (chroma_siting_vert != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvChromaSitingVert, + chroma_siting_vert)) { + return false; + } + if (range != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvRange, range)) { + return false; + } + if (transfer_characteristics != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvTransferCharacteristics, + transfer_characteristics)) { + return false; + } + if (primaries != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvPrimaries, primaries)) { + return false; + } + if (max_cll != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvMaxCLL, max_cll)) { + return false; + } + if (max_fall != kValueNotPresent && + !WriteEbmlElement(writer, libwebm::kMkvMaxFALL, max_fall)) { + return false; + } + + if (mastering_metadata_ && !mastering_metadata_->Write(writer)) + return false; + + return true; +} + +bool Colour::SetMasteringMetadata(const MasteringMetadata& mastering_metadata) { + std::auto_ptr mm_ptr(new MasteringMetadata()); + if (!mm_ptr.get()) + return false; + + mm_ptr->luminance_max = mastering_metadata.luminance_max; + mm_ptr->luminance_min = mastering_metadata.luminance_min; + + if (!mm_ptr->SetChromaticity(mastering_metadata.r(), mastering_metadata.g(), + mastering_metadata.b(), + mastering_metadata.white_point())) { + return false; + } + + delete mastering_metadata_; + mastering_metadata_ = mm_ptr.release(); + return true; +} + +uint64_t Colour::PayloadSize() const { + uint64_t size = 0; + + if (matrix_coefficients != kValueNotPresent) + size += + EbmlElementSize(libwebm::kMkvMatrixCoefficients, matrix_coefficients); + if (bits_per_channel != kValueNotPresent) + size += EbmlElementSize(libwebm::kMkvBitsPerChannel, bits_per_channel); + if (chroma_subsampling_horz != kValueNotPresent) + size += EbmlElementSize(libwebm::kMkvChromaSubsamplingHorz, + chroma_subsampling_horz); + if (chroma_subsampling_vert != kValueNotPresent) + size += EbmlElementSize(libwebm::kMkvChromaSubsamplingVert, + chroma_subsampling_vert); + if (cb_subsampling_horz != kValueNotPresent) + size += + EbmlElementSize(libwebm::kMkvCbSubsamplingHorz, cb_subsampling_horz); + if (cb_subsampling_vert != kValueNotPresent) + size += + EbmlElementSize(libwebm::kMkvCbSubsamplingVert, cb_subsampling_vert); + if (chroma_siting_horz != kValueNotPresent) + size += EbmlElementSize(libwebm::kMkvChromaSitingHorz, chroma_siting_horz); + if (chroma_siting_vert != kValueNotPresent) + size += EbmlElementSize(libwebm::kMkvChromaSitingVert, chroma_siting_vert); + if (range != kValueNotPresent) + size += EbmlElementSize(libwebm::kMkvRange, range); + if (transfer_characteristics != kValueNotPresent) + size += EbmlElementSize(libwebm::kMkvTransferCharacteristics, + transfer_characteristics); + if (primaries != kValueNotPresent) + size += EbmlElementSize(libwebm::kMkvPrimaries, primaries); + if (max_cll != kValueNotPresent) + size += EbmlElementSize(libwebm::kMkvMaxCLL, max_cll); + if (max_fall != kValueNotPresent) + size += EbmlElementSize(libwebm::kMkvMaxFALL, max_fall); + + if (mastering_metadata_) + size += mastering_metadata_->MasteringMetadataSize(); + + return size; +} + +/////////////////////////////////////////////////////////////// +// +// VideoTrack Class + +VideoTrack::VideoTrack(unsigned int* seed) + : Track(seed), + display_height_(0), + display_width_(0), + crop_left_(0), + crop_right_(0), + crop_top_(0), + crop_bottom_(0), + frame_rate_(0.0), + height_(0), + stereo_mode_(0), + alpha_mode_(0), + width_(0), + colour_(NULL) {} + +VideoTrack::~VideoTrack() { delete colour_; } + +bool VideoTrack::SetStereoMode(uint64_t stereo_mode) { + if (stereo_mode != kMono && stereo_mode != kSideBySideLeftIsFirst && + stereo_mode != kTopBottomRightIsFirst && + stereo_mode != kTopBottomLeftIsFirst && + stereo_mode != kSideBySideRightIsFirst) + return false; + + stereo_mode_ = stereo_mode; + return true; +} + +bool VideoTrack::SetAlphaMode(uint64_t alpha_mode) { + if (alpha_mode != kNoAlpha && alpha_mode != kAlpha) + return false; + + alpha_mode_ = alpha_mode; + return true; +} + +uint64_t VideoTrack::PayloadSize() const { + const uint64_t parent_size = Track::PayloadSize(); + + uint64_t size = VideoPayloadSize(); + size += EbmlMasterElementSize(libwebm::kMkvVideo, size); + + return parent_size + size; +} + +bool VideoTrack::Write(IMkvWriter* writer) const { + if (!Track::Write(writer)) + return false; + + const uint64_t size = VideoPayloadSize(); + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvVideo, size)) + return false; + + const int64_t payload_position = writer->Position(); + if (payload_position < 0) + return false; + + if (!WriteEbmlElement(writer, libwebm::kMkvPixelWidth, width_)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvPixelHeight, height_)) + return false; + if (display_width_ > 0) { + if (!WriteEbmlElement(writer, libwebm::kMkvDisplayWidth, display_width_)) + return false; + } + if (display_height_ > 0) { + if (!WriteEbmlElement(writer, libwebm::kMkvDisplayHeight, display_height_)) + return false; + } + if (crop_left_ > 0) { + if (!WriteEbmlElement(writer, libwebm::kMkvPixelCropLeft, crop_left_)) + return false; + } + if (crop_right_ > 0) { + if (!WriteEbmlElement(writer, libwebm::kMkvPixelCropRight, crop_right_)) + return false; + } + if (crop_top_ > 0) { + if (!WriteEbmlElement(writer, libwebm::kMkvPixelCropTop, crop_top_)) + return false; + } + if (crop_bottom_ > 0) { + if (!WriteEbmlElement(writer, libwebm::kMkvPixelCropBottom, crop_bottom_)) + return false; + } + if (stereo_mode_ > kMono) { + if (!WriteEbmlElement(writer, libwebm::kMkvStereoMode, stereo_mode_)) + return false; + } + if (alpha_mode_ > kNoAlpha) { + if (!WriteEbmlElement(writer, libwebm::kMkvAlphaMode, alpha_mode_)) + return false; + } + if (frame_rate_ > 0.0) { + if (!WriteEbmlElement(writer, libwebm::kMkvFrameRate, + static_cast(frame_rate_))) { + return false; + } + } + if (colour_) { + if (!colour_->Write(writer)) + return false; + } + + const int64_t stop_position = writer->Position(); + if (stop_position < 0 || + stop_position - payload_position != static_cast(size)) { + return false; + } + + return true; +} + +bool VideoTrack::SetColour(const Colour& colour) { + std::auto_ptr colour_ptr(new Colour()); + if (!colour_ptr.get()) + return false; + + if (colour.mastering_metadata()) { + if (!colour_ptr->SetMasteringMetadata(*colour.mastering_metadata())) + return false; + } + + colour_ptr->matrix_coefficients = colour.matrix_coefficients; + colour_ptr->bits_per_channel = colour.bits_per_channel; + colour_ptr->chroma_subsampling_horz = colour.chroma_subsampling_horz; + colour_ptr->chroma_subsampling_vert = colour.chroma_subsampling_vert; + colour_ptr->cb_subsampling_horz = colour.cb_subsampling_horz; + colour_ptr->cb_subsampling_vert = colour.cb_subsampling_vert; + colour_ptr->chroma_siting_horz = colour.chroma_siting_horz; + colour_ptr->chroma_siting_vert = colour.chroma_siting_vert; + colour_ptr->range = colour.range; + colour_ptr->transfer_characteristics = colour.transfer_characteristics; + colour_ptr->primaries = colour.primaries; + colour_ptr->max_cll = colour.max_cll; + colour_ptr->max_fall = colour.max_fall; + colour_ = colour_ptr.release(); + return true; +} + +uint64_t VideoTrack::VideoPayloadSize() const { + uint64_t size = EbmlElementSize(libwebm::kMkvPixelWidth, width_); + size += EbmlElementSize(libwebm::kMkvPixelHeight, height_); + if (display_width_ > 0) + size += EbmlElementSize(libwebm::kMkvDisplayWidth, display_width_); + if (display_height_ > 0) + size += EbmlElementSize(libwebm::kMkvDisplayHeight, display_height_); + if (crop_left_ > 0) + size += EbmlElementSize(libwebm::kMkvPixelCropLeft, crop_left_); + if (crop_right_ > 0) + size += EbmlElementSize(libwebm::kMkvPixelCropRight, crop_right_); + if (crop_top_ > 0) + size += EbmlElementSize(libwebm::kMkvPixelCropTop, crop_top_); + if (crop_bottom_ > 0) + size += EbmlElementSize(libwebm::kMkvPixelCropBottom, crop_bottom_); + if (stereo_mode_ > kMono) + size += EbmlElementSize(libwebm::kMkvStereoMode, stereo_mode_); + if (alpha_mode_ > kNoAlpha) + size += EbmlElementSize(libwebm::kMkvAlphaMode, alpha_mode_); + if (frame_rate_ > 0.0) + size += EbmlElementSize(libwebm::kMkvFrameRate, + static_cast(frame_rate_)); + if (colour_) + size += colour_->ColourSize(); + + return size; +} + +/////////////////////////////////////////////////////////////// +// +// AudioTrack Class + +AudioTrack::AudioTrack(unsigned int* seed) + : Track(seed), bit_depth_(0), channels_(1), sample_rate_(0.0) {} + +AudioTrack::~AudioTrack() {} + +uint64_t AudioTrack::PayloadSize() const { + const uint64_t parent_size = Track::PayloadSize(); + + uint64_t size = EbmlElementSize(libwebm::kMkvSamplingFrequency, + static_cast(sample_rate_)); + size += EbmlElementSize(libwebm::kMkvChannels, channels_); + if (bit_depth_ > 0) + size += EbmlElementSize(libwebm::kMkvBitDepth, bit_depth_); + size += EbmlMasterElementSize(libwebm::kMkvAudio, size); + + return parent_size + size; +} + +bool AudioTrack::Write(IMkvWriter* writer) const { + if (!Track::Write(writer)) + return false; + + // Calculate AudioSettings size. + uint64_t size = EbmlElementSize(libwebm::kMkvSamplingFrequency, + static_cast(sample_rate_)); + size += EbmlElementSize(libwebm::kMkvChannels, channels_); + if (bit_depth_ > 0) + size += EbmlElementSize(libwebm::kMkvBitDepth, bit_depth_); + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvAudio, size)) + return false; + + const int64_t payload_position = writer->Position(); + if (payload_position < 0) + return false; + + if (!WriteEbmlElement(writer, libwebm::kMkvSamplingFrequency, + static_cast(sample_rate_))) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvChannels, channels_)) + return false; + if (bit_depth_ > 0) + if (!WriteEbmlElement(writer, libwebm::kMkvBitDepth, bit_depth_)) + return false; + + const int64_t stop_position = writer->Position(); + if (stop_position < 0 || + stop_position - payload_position != static_cast(size)) + return false; + + return true; +} + +/////////////////////////////////////////////////////////////// +// +// Tracks Class + +const char Tracks::kOpusCodecId[] = "A_OPUS"; +const char Tracks::kVorbisCodecId[] = "A_VORBIS"; +const char Tracks::kVp8CodecId[] = "V_VP8"; +const char Tracks::kVp9CodecId[] = "V_VP9"; +const char Tracks::kAv1CodecId[] = "V_AV1"; + +Tracks::Tracks() + : track_entries_(NULL), track_entries_size_(0), wrote_tracks_(false) {} + +Tracks::~Tracks() { + if (track_entries_) { + for (uint32_t i = 0; i < track_entries_size_; ++i) { + Track* const track = track_entries_[i]; + delete track; + } + delete[] track_entries_; + } +} + +bool Tracks::AddTrack(Track* track, int32_t number) { + if (number < 0 || wrote_tracks_) + return false; + + // This muxer only supports track numbers in the range [1, 126], in + // order to be able (to use Matroska integer representation) to + // serialize the block header (of which the track number is a part) + // for a frame using exactly 4 bytes. + + if (number > 0x7E) + return false; + + uint32_t track_num = number; + + if (track_num > 0) { + // Check to make sure a track does not already have |track_num|. + for (uint32_t i = 0; i < track_entries_size_; ++i) { + if (track_entries_[i]->number() == track_num) + return false; + } + } + + const uint32_t count = track_entries_size_ + 1; + + Track** const track_entries = new (std::nothrow) Track*[count]; // NOLINT + if (!track_entries) + return false; + + for (uint32_t i = 0; i < track_entries_size_; ++i) { + track_entries[i] = track_entries_[i]; + } + + delete[] track_entries_; + + // Find the lowest availible track number > 0. + if (track_num == 0) { + track_num = count; + + // Check to make sure a track does not already have |track_num|. + bool exit = false; + do { + exit = true; + for (uint32_t i = 0; i < track_entries_size_; ++i) { + if (track_entries[i]->number() == track_num) { + track_num++; + exit = false; + break; + } + } + } while (!exit); + } + track->set_number(track_num); + + track_entries_ = track_entries; + track_entries_[track_entries_size_] = track; + track_entries_size_ = count; + return true; +} + +const Track* Tracks::GetTrackByIndex(uint32_t index) const { + if (track_entries_ == NULL) + return NULL; + + if (index >= track_entries_size_) + return NULL; + + return track_entries_[index]; +} + +Track* Tracks::GetTrackByNumber(uint64_t track_number) const { + const int32_t count = track_entries_size(); + for (int32_t i = 0; i < count; ++i) { + if (track_entries_[i]->number() == track_number) + return track_entries_[i]; + } + + return NULL; +} + +bool Tracks::TrackIsAudio(uint64_t track_number) const { + const Track* const track = GetTrackByNumber(track_number); + + if (track->type() == kAudio) + return true; + + return false; +} + +bool Tracks::TrackIsVideo(uint64_t track_number) const { + const Track* const track = GetTrackByNumber(track_number); + + if (track->type() == kVideo) + return true; + + return false; +} + +bool Tracks::Write(IMkvWriter* writer) const { + uint64_t size = 0; + const int32_t count = track_entries_size(); + for (int32_t i = 0; i < count; ++i) { + const Track* const track = GetTrackByIndex(i); + + if (!track) + return false; + + size += track->Size(); + } + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvTracks, size)) + return false; + + const int64_t payload_position = writer->Position(); + if (payload_position < 0) + return false; + + for (int32_t i = 0; i < count; ++i) { + const Track* const track = GetTrackByIndex(i); + if (!track->Write(writer)) + return false; + } + + const int64_t stop_position = writer->Position(); + if (stop_position < 0 || + stop_position - payload_position != static_cast(size)) + return false; + + wrote_tracks_ = true; + return true; +} + +/////////////////////////////////////////////////////////////// +// +// Chapter Class + +bool Chapter::set_id(const char* id) { return StrCpy(id, &id_); } + +void Chapter::set_time(const Segment& segment, uint64_t start_ns, + uint64_t end_ns) { + const SegmentInfo* const info = segment.GetSegmentInfo(); + const uint64_t timecode_scale = info->timecode_scale(); + start_timecode_ = start_ns / timecode_scale; + end_timecode_ = end_ns / timecode_scale; +} + +bool Chapter::add_string(const char* title, const char* language, + const char* country) { + if (!ExpandDisplaysArray()) + return false; + + Display& d = displays_[displays_count_++]; + d.Init(); + + if (!d.set_title(title)) + return false; + + if (!d.set_language(language)) + return false; + + if (!d.set_country(country)) + return false; + + return true; +} + +Chapter::Chapter() { + // This ctor only constructs the object. Proper initialization is + // done in Init() (called in Chapters::AddChapter()). The only + // reason we bother implementing this ctor is because we had to + // declare it as private (along with the dtor), in order to prevent + // clients from creating Chapter instances (a privelege we grant + // only to the Chapters class). Doing no initialization here also + // means that creating arrays of chapter objects is more efficient, + // because we only initialize each new chapter object as it becomes + // active on the array. +} + +Chapter::~Chapter() {} + +void Chapter::Init(unsigned int* seed) { + id_ = NULL; + start_timecode_ = 0; + end_timecode_ = 0; + displays_ = NULL; + displays_size_ = 0; + displays_count_ = 0; + uid_ = MakeUID(seed); +} + +void Chapter::ShallowCopy(Chapter* dst) const { + dst->id_ = id_; + dst->start_timecode_ = start_timecode_; + dst->end_timecode_ = end_timecode_; + dst->uid_ = uid_; + dst->displays_ = displays_; + dst->displays_size_ = displays_size_; + dst->displays_count_ = displays_count_; +} + +void Chapter::Clear() { + StrCpy(NULL, &id_); + + while (displays_count_ > 0) { + Display& d = displays_[--displays_count_]; + d.Clear(); + } + + delete[] displays_; + displays_ = NULL; + + displays_size_ = 0; +} + +bool Chapter::ExpandDisplaysArray() { + if (displays_size_ > displays_count_) + return true; // nothing to do yet + + const int size = (displays_size_ == 0) ? 1 : 2 * displays_size_; + + Display* const displays = new (std::nothrow) Display[size]; // NOLINT + if (displays == NULL) + return false; + + for (int idx = 0; idx < displays_count_; ++idx) { + displays[idx] = displays_[idx]; // shallow copy + } + + delete[] displays_; + + displays_ = displays; + displays_size_ = size; + + return true; +} + +uint64_t Chapter::WriteAtom(IMkvWriter* writer) const { + uint64_t payload_size = + EbmlElementSize(libwebm::kMkvChapterStringUID, id_) + + EbmlElementSize(libwebm::kMkvChapterUID, uid_) + + EbmlElementSize(libwebm::kMkvChapterTimeStart, start_timecode_) + + EbmlElementSize(libwebm::kMkvChapterTimeEnd, end_timecode_); + + for (int idx = 0; idx < displays_count_; ++idx) { + const Display& d = displays_[idx]; + payload_size += d.WriteDisplay(NULL); + } + + const uint64_t atom_size = + EbmlMasterElementSize(libwebm::kMkvChapterAtom, payload_size) + + payload_size; + + if (writer == NULL) + return atom_size; + + const int64_t start = writer->Position(); + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvChapterAtom, payload_size)) + return 0; + + if (!WriteEbmlElement(writer, libwebm::kMkvChapterStringUID, id_)) + return 0; + + if (!WriteEbmlElement(writer, libwebm::kMkvChapterUID, uid_)) + return 0; + + if (!WriteEbmlElement(writer, libwebm::kMkvChapterTimeStart, start_timecode_)) + return 0; + + if (!WriteEbmlElement(writer, libwebm::kMkvChapterTimeEnd, end_timecode_)) + return 0; + + for (int idx = 0; idx < displays_count_; ++idx) { + const Display& d = displays_[idx]; + + if (!d.WriteDisplay(writer)) + return 0; + } + + const int64_t stop = writer->Position(); + + if (stop >= start && uint64_t(stop - start) != atom_size) + return 0; + + return atom_size; +} + +void Chapter::Display::Init() { + title_ = NULL; + language_ = NULL; + country_ = NULL; +} + +void Chapter::Display::Clear() { + StrCpy(NULL, &title_); + StrCpy(NULL, &language_); + StrCpy(NULL, &country_); +} + +bool Chapter::Display::set_title(const char* title) { + return StrCpy(title, &title_); +} + +bool Chapter::Display::set_language(const char* language) { + return StrCpy(language, &language_); +} + +bool Chapter::Display::set_country(const char* country) { + return StrCpy(country, &country_); +} + +uint64_t Chapter::Display::WriteDisplay(IMkvWriter* writer) const { + uint64_t payload_size = EbmlElementSize(libwebm::kMkvChapString, title_); + + if (language_) + payload_size += EbmlElementSize(libwebm::kMkvChapLanguage, language_); + + if (country_) + payload_size += EbmlElementSize(libwebm::kMkvChapCountry, country_); + + const uint64_t display_size = + EbmlMasterElementSize(libwebm::kMkvChapterDisplay, payload_size) + + payload_size; + + if (writer == NULL) + return display_size; + + const int64_t start = writer->Position(); + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvChapterDisplay, + payload_size)) + return 0; + + if (!WriteEbmlElement(writer, libwebm::kMkvChapString, title_)) + return 0; + + if (language_) { + if (!WriteEbmlElement(writer, libwebm::kMkvChapLanguage, language_)) + return 0; + } + + if (country_) { + if (!WriteEbmlElement(writer, libwebm::kMkvChapCountry, country_)) + return 0; + } + + const int64_t stop = writer->Position(); + + if (stop >= start && uint64_t(stop - start) != display_size) + return 0; + + return display_size; +} + +/////////////////////////////////////////////////////////////// +// +// Chapters Class + +Chapters::Chapters() : chapters_size_(0), chapters_count_(0), chapters_(NULL) {} + +Chapters::~Chapters() { + while (chapters_count_ > 0) { + Chapter& chapter = chapters_[--chapters_count_]; + chapter.Clear(); + } + + delete[] chapters_; + chapters_ = NULL; +} + +int Chapters::Count() const { return chapters_count_; } + +Chapter* Chapters::AddChapter(unsigned int* seed) { + if (!ExpandChaptersArray()) + return NULL; + + Chapter& chapter = chapters_[chapters_count_++]; + chapter.Init(seed); + + return &chapter; +} + +bool Chapters::Write(IMkvWriter* writer) const { + if (writer == NULL) + return false; + + const uint64_t payload_size = WriteEdition(NULL); // return size only + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvChapters, payload_size)) + return false; + + const int64_t start = writer->Position(); + + if (WriteEdition(writer) == 0) // error + return false; + + const int64_t stop = writer->Position(); + + if (stop >= start && uint64_t(stop - start) != payload_size) + return false; + + return true; +} + +bool Chapters::ExpandChaptersArray() { + if (chapters_size_ > chapters_count_) + return true; // nothing to do yet + + const int size = (chapters_size_ == 0) ? 1 : 2 * chapters_size_; + + Chapter* const chapters = new (std::nothrow) Chapter[size]; // NOLINT + if (chapters == NULL) + return false; + + for (int idx = 0; idx < chapters_count_; ++idx) { + const Chapter& src = chapters_[idx]; + Chapter* const dst = chapters + idx; + src.ShallowCopy(dst); + } + + delete[] chapters_; + + chapters_ = chapters; + chapters_size_ = size; + + return true; +} + +uint64_t Chapters::WriteEdition(IMkvWriter* writer) const { + uint64_t payload_size = 0; + + for (int idx = 0; idx < chapters_count_; ++idx) { + const Chapter& chapter = chapters_[idx]; + payload_size += chapter.WriteAtom(NULL); + } + + const uint64_t edition_size = + EbmlMasterElementSize(libwebm::kMkvEditionEntry, payload_size) + + payload_size; + + if (writer == NULL) // return size only + return edition_size; + + const int64_t start = writer->Position(); + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvEditionEntry, payload_size)) + return 0; // error + + for (int idx = 0; idx < chapters_count_; ++idx) { + const Chapter& chapter = chapters_[idx]; + + const uint64_t chapter_size = chapter.WriteAtom(writer); + if (chapter_size == 0) // error + return 0; + } + + const int64_t stop = writer->Position(); + + if (stop >= start && uint64_t(stop - start) != edition_size) + return 0; + + return edition_size; +} + +// Tag Class + +bool Tag::add_simple_tag(const char* tag_name, const char* tag_string) { + if (!ExpandSimpleTagsArray()) + return false; + + SimpleTag& st = simple_tags_[simple_tags_count_++]; + st.Init(); + + if (!st.set_tag_name(tag_name)) + return false; + + if (!st.set_tag_string(tag_string)) + return false; + + return true; +} + +Tag::Tag() { + simple_tags_ = NULL; + simple_tags_size_ = 0; + simple_tags_count_ = 0; +} + +Tag::~Tag() {} + +void Tag::ShallowCopy(Tag* dst) const { + dst->simple_tags_ = simple_tags_; + dst->simple_tags_size_ = simple_tags_size_; + dst->simple_tags_count_ = simple_tags_count_; +} + +void Tag::Clear() { + while (simple_tags_count_ > 0) { + SimpleTag& st = simple_tags_[--simple_tags_count_]; + st.Clear(); + } + + delete[] simple_tags_; + simple_tags_ = NULL; + + simple_tags_size_ = 0; +} + +bool Tag::ExpandSimpleTagsArray() { + if (simple_tags_size_ > simple_tags_count_) + return true; // nothing to do yet + + const int size = (simple_tags_size_ == 0) ? 1 : 2 * simple_tags_size_; + + SimpleTag* const simple_tags = new (std::nothrow) SimpleTag[size]; // NOLINT + if (simple_tags == NULL) + return false; + + for (int idx = 0; idx < simple_tags_count_; ++idx) { + simple_tags[idx] = simple_tags_[idx]; // shallow copy + } + + delete[] simple_tags_; + + simple_tags_ = simple_tags; + simple_tags_size_ = size; + + return true; +} + +uint64_t Tag::Write(IMkvWriter* writer) const { + uint64_t payload_size = 0; + + for (int idx = 0; idx < simple_tags_count_; ++idx) { + const SimpleTag& st = simple_tags_[idx]; + payload_size += st.Write(NULL); + } + + const uint64_t tag_size = + EbmlMasterElementSize(libwebm::kMkvTag, payload_size) + payload_size; + + if (writer == NULL) + return tag_size; + + const int64_t start = writer->Position(); + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvTag, payload_size)) + return 0; + + for (int idx = 0; idx < simple_tags_count_; ++idx) { + const SimpleTag& st = simple_tags_[idx]; + + if (!st.Write(writer)) + return 0; + } + + const int64_t stop = writer->Position(); + + if (stop >= start && uint64_t(stop - start) != tag_size) + return 0; + + return tag_size; +} + +// Tag::SimpleTag + +void Tag::SimpleTag::Init() { + tag_name_ = NULL; + tag_string_ = NULL; +} + +void Tag::SimpleTag::Clear() { + StrCpy(NULL, &tag_name_); + StrCpy(NULL, &tag_string_); +} + +bool Tag::SimpleTag::set_tag_name(const char* tag_name) { + return StrCpy(tag_name, &tag_name_); +} + +bool Tag::SimpleTag::set_tag_string(const char* tag_string) { + return StrCpy(tag_string, &tag_string_); +} + +uint64_t Tag::SimpleTag::Write(IMkvWriter* writer) const { + uint64_t payload_size = EbmlElementSize(libwebm::kMkvTagName, tag_name_); + + payload_size += EbmlElementSize(libwebm::kMkvTagString, tag_string_); + + const uint64_t simple_tag_size = + EbmlMasterElementSize(libwebm::kMkvSimpleTag, payload_size) + + payload_size; + + if (writer == NULL) + return simple_tag_size; + + const int64_t start = writer->Position(); + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvSimpleTag, payload_size)) + return 0; + + if (!WriteEbmlElement(writer, libwebm::kMkvTagName, tag_name_)) + return 0; + + if (!WriteEbmlElement(writer, libwebm::kMkvTagString, tag_string_)) + return 0; + + const int64_t stop = writer->Position(); + + if (stop >= start && uint64_t(stop - start) != simple_tag_size) + return 0; + + return simple_tag_size; +} + +// Tags Class + +Tags::Tags() : tags_size_(0), tags_count_(0), tags_(NULL) {} + +Tags::~Tags() { + while (tags_count_ > 0) { + Tag& tag = tags_[--tags_count_]; + tag.Clear(); + } + + delete[] tags_; + tags_ = NULL; +} + +int Tags::Count() const { return tags_count_; } + +Tag* Tags::AddTag() { + if (!ExpandTagsArray()) + return NULL; + + Tag& tag = tags_[tags_count_++]; + + return &tag; +} + +bool Tags::Write(IMkvWriter* writer) const { + if (writer == NULL) + return false; + + uint64_t payload_size = 0; + + for (int idx = 0; idx < tags_count_; ++idx) { + const Tag& tag = tags_[idx]; + payload_size += tag.Write(NULL); + } + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvTags, payload_size)) + return false; + + const int64_t start = writer->Position(); + + for (int idx = 0; idx < tags_count_; ++idx) { + const Tag& tag = tags_[idx]; + + const uint64_t tag_size = tag.Write(writer); + if (tag_size == 0) // error + return 0; + } + + const int64_t stop = writer->Position(); + + if (stop >= start && uint64_t(stop - start) != payload_size) + return false; + + return true; +} + +bool Tags::ExpandTagsArray() { + if (tags_size_ > tags_count_) + return true; // nothing to do yet + + const int size = (tags_size_ == 0) ? 1 : 2 * tags_size_; + + Tag* const tags = new (std::nothrow) Tag[size]; // NOLINT + if (tags == NULL) + return false; + + for (int idx = 0; idx < tags_count_; ++idx) { + const Tag& src = tags_[idx]; + Tag* const dst = tags + idx; + src.ShallowCopy(dst); + } + + delete[] tags_; + + tags_ = tags; + tags_size_ = size; + + return true; +} + +/////////////////////////////////////////////////////////////// +// +// Cluster class + +Cluster::Cluster(uint64_t timecode, int64_t cues_pos, uint64_t timecode_scale, + bool write_last_frame_with_duration, bool fixed_size_timecode) + : blocks_added_(0), + finalized_(false), + fixed_size_timecode_(fixed_size_timecode), + header_written_(false), + payload_size_(0), + position_for_cues_(cues_pos), + size_position_(-1), + timecode_(timecode), + timecode_scale_(timecode_scale), + write_last_frame_with_duration_(write_last_frame_with_duration), + writer_(NULL) {} + +Cluster::~Cluster() {} + +bool Cluster::Init(IMkvWriter* ptr_writer) { + if (!ptr_writer) { + return false; + } + writer_ = ptr_writer; + return true; +} + +bool Cluster::AddFrame(const Frame* const frame) { + return QueueOrWriteFrame(frame); +} + +bool Cluster::AddFrame(const uint8_t* data, uint64_t length, + uint64_t track_number, uint64_t abs_timecode, + bool is_key) { + Frame frame; + if (!frame.Init(data, length)) + return false; + frame.set_track_number(track_number); + frame.set_timestamp(abs_timecode); + frame.set_is_key(is_key); + return QueueOrWriteFrame(&frame); +} + +bool Cluster::AddFrameWithAdditional(const uint8_t* data, uint64_t length, + const uint8_t* additional, + uint64_t additional_length, + uint64_t add_id, uint64_t track_number, + uint64_t abs_timecode, bool is_key) { + if (!additional || additional_length == 0) { + return false; + } + Frame frame; + if (!frame.Init(data, length) || + !frame.AddAdditionalData(additional, additional_length, add_id)) { + return false; + } + frame.set_track_number(track_number); + frame.set_timestamp(abs_timecode); + frame.set_is_key(is_key); + return QueueOrWriteFrame(&frame); +} + +bool Cluster::AddFrameWithDiscardPadding(const uint8_t* data, uint64_t length, + int64_t discard_padding, + uint64_t track_number, + uint64_t abs_timecode, bool is_key) { + Frame frame; + if (!frame.Init(data, length)) + return false; + frame.set_discard_padding(discard_padding); + frame.set_track_number(track_number); + frame.set_timestamp(abs_timecode); + frame.set_is_key(is_key); + return QueueOrWriteFrame(&frame); +} + +bool Cluster::AddMetadata(const uint8_t* data, uint64_t length, + uint64_t track_number, uint64_t abs_timecode, + uint64_t duration_timecode) { + Frame frame; + if (!frame.Init(data, length)) + return false; + frame.set_track_number(track_number); + frame.set_timestamp(abs_timecode); + frame.set_duration(duration_timecode); + frame.set_is_key(true); // All metadata blocks are keyframes. + return QueueOrWriteFrame(&frame); +} + +void Cluster::AddPayloadSize(uint64_t size) { payload_size_ += size; } + +bool Cluster::Finalize() { + return !write_last_frame_with_duration_ && Finalize(false, 0); +} + +bool Cluster::Finalize(bool set_last_frame_duration, uint64_t duration) { + if (!writer_ || finalized_) + return false; + + if (write_last_frame_with_duration_) { + // Write out held back Frames. This essentially performs a k-way merge + // across all tracks in the increasing order of timestamps. + while (!stored_frames_.empty()) { + Frame* frame = stored_frames_.begin()->second.front(); + + // Get the next frame to write (frame with least timestamp across all + // tracks). + for (FrameMapIterator frames_iterator = ++stored_frames_.begin(); + frames_iterator != stored_frames_.end(); ++frames_iterator) { + if (frames_iterator->second.front()->timestamp() < frame->timestamp()) { + frame = frames_iterator->second.front(); + } + } + + // Set the duration if it's the last frame for the track. + if (set_last_frame_duration && + stored_frames_[frame->track_number()].size() == 1 && + !frame->duration_set()) { + frame->set_duration(duration - frame->timestamp()); + if (!frame->is_key() && !frame->reference_block_timestamp_set()) { + frame->set_reference_block_timestamp( + last_block_timestamp_[frame->track_number()]); + } + } + + // Write the frame and remove it from |stored_frames_|. + const bool wrote_frame = DoWriteFrame(frame); + stored_frames_[frame->track_number()].pop_front(); + if (stored_frames_[frame->track_number()].empty()) { + stored_frames_.erase(frame->track_number()); + } + delete frame; + if (!wrote_frame) + return false; + } + } + + if (size_position_ == -1) + return false; + + if (writer_->Seekable()) { + const int64_t pos = writer_->Position(); + + if (writer_->Position(size_position_)) + return false; + + if (WriteUIntSize(writer_, payload_size(), 8)) + return false; + + if (writer_->Position(pos)) + return false; + } + + finalized_ = true; + + return true; +} + +uint64_t Cluster::Size() const { + const uint64_t element_size = + EbmlMasterElementSize(libwebm::kMkvCluster, 0xFFFFFFFFFFFFFFFFULL) + + payload_size_; + return element_size; +} + +bool Cluster::PreWriteBlock() { + if (finalized_) + return false; + + if (!header_written_) { + if (!WriteClusterHeader()) + return false; + } + + return true; +} + +void Cluster::PostWriteBlock(uint64_t element_size) { + AddPayloadSize(element_size); + ++blocks_added_; +} + +int64_t Cluster::GetRelativeTimecode(int64_t abs_timecode) const { + const int64_t cluster_timecode = this->Cluster::timecode(); + const int64_t rel_timecode = + static_cast(abs_timecode) - cluster_timecode; + + if (rel_timecode < 0 || rel_timecode > kMaxBlockTimecode) + return -1; + + return rel_timecode; +} + +bool Cluster::DoWriteFrame(const Frame* const frame) { + if (!frame || !frame->IsValid()) + return false; + + if (!PreWriteBlock()) + return false; + + const uint64_t element_size = WriteFrame(writer_, frame, this); + if (element_size == 0) + return false; + + PostWriteBlock(element_size); + last_block_timestamp_[frame->track_number()] = frame->timestamp(); + return true; +} + +bool Cluster::QueueOrWriteFrame(const Frame* const frame) { + if (!frame || !frame->IsValid()) + return false; + + // If |write_last_frame_with_duration_| is not set, then write the frame right + // away. + if (!write_last_frame_with_duration_) { + return DoWriteFrame(frame); + } + + // Queue the current frame. + uint64_t track_number = frame->track_number(); + Frame* const frame_to_store = new Frame(); + frame_to_store->CopyFrom(*frame); + stored_frames_[track_number].push_back(frame_to_store); + + // Iterate through all queued frames in the current track except the last one + // and write it if it is okay to do so (i.e.) no other track has an held back + // frame with timestamp <= the timestamp of the frame in question. + std::vector::iterator> frames_to_erase; + for (std::list::iterator + current_track_iterator = stored_frames_[track_number].begin(), + end = --stored_frames_[track_number].end(); + current_track_iterator != end; ++current_track_iterator) { + const Frame* const frame_to_write = *current_track_iterator; + bool okay_to_write = true; + for (FrameMapIterator track_iterator = stored_frames_.begin(); + track_iterator != stored_frames_.end(); ++track_iterator) { + if (track_iterator->first == track_number) { + continue; + } + if (track_iterator->second.front()->timestamp() < + frame_to_write->timestamp()) { + okay_to_write = false; + break; + } + } + if (okay_to_write) { + const bool wrote_frame = DoWriteFrame(frame_to_write); + delete frame_to_write; + if (!wrote_frame) + return false; + frames_to_erase.push_back(current_track_iterator); + } else { + break; + } + } + for (std::vector::iterator>::iterator iterator = + frames_to_erase.begin(); + iterator != frames_to_erase.end(); ++iterator) { + stored_frames_[track_number].erase(*iterator); + } + return true; +} + +bool Cluster::WriteClusterHeader() { + if (finalized_) + return false; + + if (WriteID(writer_, libwebm::kMkvCluster)) + return false; + + // Save for later. + size_position_ = writer_->Position(); + + // Write "unknown" (EBML coded -1) as cluster size value. We need to write 8 + // bytes because we do not know how big our cluster will be. + if (SerializeInt(writer_, kEbmlUnknownValue, 8)) + return false; + + if (!WriteEbmlElement(writer_, libwebm::kMkvTimecode, timecode(), + fixed_size_timecode_ ? 8 : 0)) { + return false; + } + AddPayloadSize(EbmlElementSize(libwebm::kMkvTimecode, timecode(), + fixed_size_timecode_ ? 8 : 0)); + header_written_ = true; + + return true; +} + +/////////////////////////////////////////////////////////////// +// +// SeekHead Class + +SeekHead::SeekHead() : start_pos_(0ULL) { + for (int32_t i = 0; i < kSeekEntryCount; ++i) { + seek_entry_id_[i] = 0; + seek_entry_pos_[i] = 0; + } +} + +SeekHead::~SeekHead() {} + +bool SeekHead::Finalize(IMkvWriter* writer) const { + if (writer->Seekable()) { + if (start_pos_ == -1) + return false; + + uint64_t payload_size = 0; + uint64_t entry_size[kSeekEntryCount]; + + for (int32_t i = 0; i < kSeekEntryCount; ++i) { + if (seek_entry_id_[i] != 0) { + entry_size[i] = EbmlElementSize( + libwebm::kMkvSeekID, static_cast(seek_entry_id_[i])); + entry_size[i] += + EbmlElementSize(libwebm::kMkvSeekPosition, seek_entry_pos_[i]); + + payload_size += + EbmlMasterElementSize(libwebm::kMkvSeek, entry_size[i]) + + entry_size[i]; + } + } + + // No SeekHead elements + if (payload_size == 0) + return true; + + const int64_t pos = writer->Position(); + if (writer->Position(start_pos_)) + return false; + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvSeekHead, payload_size)) + return false; + + for (int32_t i = 0; i < kSeekEntryCount; ++i) { + if (seek_entry_id_[i] != 0) { + if (!WriteEbmlMasterElement(writer, libwebm::kMkvSeek, entry_size[i])) + return false; + + if (!WriteEbmlElement(writer, libwebm::kMkvSeekID, + static_cast(seek_entry_id_[i]))) + return false; + + if (!WriteEbmlElement(writer, libwebm::kMkvSeekPosition, + seek_entry_pos_[i])) + return false; + } + } + + const uint64_t total_entry_size = kSeekEntryCount * MaxEntrySize(); + const uint64_t total_size = + EbmlMasterElementSize(libwebm::kMkvSeekHead, total_entry_size) + + total_entry_size; + const int64_t size_left = total_size - (writer->Position() - start_pos_); + + const uint64_t bytes_written = WriteVoidElement(writer, size_left); + if (!bytes_written) + return false; + + if (writer->Position(pos)) + return false; + } + + return true; +} + +bool SeekHead::Write(IMkvWriter* writer) { + const uint64_t entry_size = kSeekEntryCount * MaxEntrySize(); + const uint64_t size = + EbmlMasterElementSize(libwebm::kMkvSeekHead, entry_size); + + start_pos_ = writer->Position(); + + const uint64_t bytes_written = WriteVoidElement(writer, size + entry_size); + if (!bytes_written) + return false; + + return true; +} + +bool SeekHead::AddSeekEntry(uint32_t id, uint64_t pos) { + for (int32_t i = 0; i < kSeekEntryCount; ++i) { + if (seek_entry_id_[i] == 0) { + seek_entry_id_[i] = id; + seek_entry_pos_[i] = pos; + return true; + } + } + return false; +} + +uint32_t SeekHead::GetId(int index) const { + if (index < 0 || index >= kSeekEntryCount) + return UINT_MAX; + return seek_entry_id_[index]; +} + +uint64_t SeekHead::GetPosition(int index) const { + if (index < 0 || index >= kSeekEntryCount) + return ULLONG_MAX; + return seek_entry_pos_[index]; +} + +bool SeekHead::SetSeekEntry(int index, uint32_t id, uint64_t position) { + if (index < 0 || index >= kSeekEntryCount) + return false; + seek_entry_id_[index] = id; + seek_entry_pos_[index] = position; + return true; +} + +uint64_t SeekHead::MaxEntrySize() const { + const uint64_t max_entry_payload_size = + EbmlElementSize(libwebm::kMkvSeekID, UINT64_C(0xffffffff)) + + EbmlElementSize(libwebm::kMkvSeekPosition, UINT64_C(0xffffffffffffffff)); + const uint64_t max_entry_size = + EbmlMasterElementSize(libwebm::kMkvSeek, max_entry_payload_size) + + max_entry_payload_size; + + return max_entry_size; +} + +/////////////////////////////////////////////////////////////// +// +// SegmentInfo Class + +SegmentInfo::SegmentInfo() + : duration_(-1.0), + muxing_app_(NULL), + timecode_scale_(1000000ULL), + writing_app_(NULL), + date_utc_(LLONG_MIN), + duration_pos_(-1) {} + +SegmentInfo::~SegmentInfo() { + delete[] muxing_app_; + delete[] writing_app_; +} + +bool SegmentInfo::Init() { + int32_t major; + int32_t minor; + int32_t build; + int32_t revision; + GetVersion(&major, &minor, &build, &revision); + char temp[256]; +#ifdef _MSC_VER + sprintf_s(temp, sizeof(temp) / sizeof(temp[0]), "libwebm-%d.%d.%d.%d", major, + minor, build, revision); +#else + snprintf(temp, sizeof(temp) / sizeof(temp[0]), "libwebm-%d.%d.%d.%d", major, + minor, build, revision); +#endif + + const size_t app_len = strlen(temp) + 1; + + delete[] muxing_app_; + + muxing_app_ = new (std::nothrow) char[app_len]; // NOLINT + if (!muxing_app_) + return false; + +#ifdef _MSC_VER + strcpy_s(muxing_app_, app_len, temp); +#else + strcpy(muxing_app_, temp); +#endif + + set_writing_app(temp); + if (!writing_app_) + return false; + return true; +} + +bool SegmentInfo::Finalize(IMkvWriter* writer) const { + if (!writer) + return false; + + if (duration_ > 0.0) { + if (writer->Seekable()) { + if (duration_pos_ == -1) + return false; + + const int64_t pos = writer->Position(); + + if (writer->Position(duration_pos_)) + return false; + + if (!WriteEbmlElement(writer, libwebm::kMkvDuration, + static_cast(duration_))) + return false; + + if (writer->Position(pos)) + return false; + } + } + + return true; +} + +bool SegmentInfo::Write(IMkvWriter* writer) { + if (!writer || !muxing_app_ || !writing_app_) + return false; + + uint64_t size = EbmlElementSize(libwebm::kMkvTimecodeScale, timecode_scale_); + if (duration_ > 0.0) + size += + EbmlElementSize(libwebm::kMkvDuration, static_cast(duration_)); + if (date_utc_ != LLONG_MIN) + size += EbmlDateElementSize(libwebm::kMkvDateUTC); + size += EbmlElementSize(libwebm::kMkvMuxingApp, muxing_app_); + size += EbmlElementSize(libwebm::kMkvWritingApp, writing_app_); + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvInfo, size)) + return false; + + const int64_t payload_position = writer->Position(); + if (payload_position < 0) + return false; + + if (!WriteEbmlElement(writer, libwebm::kMkvTimecodeScale, timecode_scale_)) + return false; + + if (duration_ > 0.0) { + // Save for later + duration_pos_ = writer->Position(); + + if (!WriteEbmlElement(writer, libwebm::kMkvDuration, + static_cast(duration_))) + return false; + } + + if (date_utc_ != LLONG_MIN) + WriteEbmlDateElement(writer, libwebm::kMkvDateUTC, date_utc_); + + if (!WriteEbmlElement(writer, libwebm::kMkvMuxingApp, muxing_app_)) + return false; + if (!WriteEbmlElement(writer, libwebm::kMkvWritingApp, writing_app_)) + return false; + + const int64_t stop_position = writer->Position(); + if (stop_position < 0 || + stop_position - payload_position != static_cast(size)) + return false; + + return true; +} + +void SegmentInfo::set_muxing_app(const char* app) { + if (app) { + const size_t length = strlen(app) + 1; + char* temp_str = new (std::nothrow) char[length]; // NOLINT + if (!temp_str) + return; + +#ifdef _MSC_VER + strcpy_s(temp_str, length, app); +#else + strcpy(temp_str, app); +#endif + + delete[] muxing_app_; + muxing_app_ = temp_str; + } +} + +void SegmentInfo::set_writing_app(const char* app) { + if (app) { + const size_t length = strlen(app) + 1; + char* temp_str = new (std::nothrow) char[length]; // NOLINT + if (!temp_str) + return; + +#ifdef _MSC_VER + strcpy_s(temp_str, length, app); +#else + strcpy(temp_str, app); +#endif + + delete[] writing_app_; + writing_app_ = temp_str; + } +} + +/////////////////////////////////////////////////////////////// +// +// Segment Class + +Segment::Segment() + : chunk_count_(0), + chunk_name_(NULL), + chunk_writer_cluster_(NULL), + chunk_writer_cues_(NULL), + chunk_writer_header_(NULL), + chunking_(false), + chunking_base_name_(NULL), + cluster_list_(NULL), + cluster_list_capacity_(0), + cluster_list_size_(0), + cues_position_(kAfterClusters), + cues_track_(0), + force_new_cluster_(false), + frames_(NULL), + frames_capacity_(0), + frames_size_(0), + has_video_(false), + header_written_(false), + last_block_duration_(0), + last_timestamp_(0), + max_cluster_duration_(kDefaultMaxClusterDuration), + max_cluster_size_(0), + mode_(kFile), + new_cuepoint_(false), + output_cues_(true), + accurate_cluster_duration_(false), + fixed_size_cluster_timecode_(false), + payload_pos_(0), + size_position_(0), + doc_type_version_(kDefaultDocTypeVersion), + doc_type_version_written_(0), + writer_cluster_(NULL), + writer_cues_(NULL), + writer_header_(NULL) { + const time_t curr_time = time(NULL); + seed_ = static_cast(curr_time); +#ifdef _WIN32 + srand(seed_); +#endif +} + +Segment::~Segment() { + if (cluster_list_) { + for (int32_t i = 0; i < cluster_list_size_; ++i) { + Cluster* const cluster = cluster_list_[i]; + delete cluster; + } + delete[] cluster_list_; + } + + if (frames_) { + for (int32_t i = 0; i < frames_size_; ++i) { + Frame* const frame = frames_[i]; + delete frame; + } + delete[] frames_; + } + + delete[] chunk_name_; + delete[] chunking_base_name_; + + if (chunk_writer_cluster_) { + chunk_writer_cluster_->Close(); + delete chunk_writer_cluster_; + } + if (chunk_writer_cues_) { + chunk_writer_cues_->Close(); + delete chunk_writer_cues_; + } + if (chunk_writer_header_) { + chunk_writer_header_->Close(); + delete chunk_writer_header_; + } +} + +void Segment::MoveCuesBeforeClustersHelper(uint64_t diff, int32_t index, + uint64_t* cues_size) { + CuePoint* const cue_point = cues_.GetCueByIndex(index); + if (cue_point == NULL) + return; + const uint64_t old_cue_point_size = cue_point->Size(); + const uint64_t cluster_pos = cue_point->cluster_pos() + diff; + cue_point->set_cluster_pos(cluster_pos); // update the new cluster position + // New size of the cue is computed as follows + // Let a = current sum of size of all CuePoints + // Let b = Increase in Cue Point's size due to this iteration + // Let c = Increase in size of Cues Element's length due to this iteration + // (This is computed as CodedSize(a + b) - CodedSize(a)) + // Let d = b + c. Now d is the |diff| passed to the next recursive call. + // Let e = a + b. Now e is the |cues_size| passed to the next recursive + // call. + const uint64_t cue_point_size_diff = cue_point->Size() - old_cue_point_size; + const uint64_t cue_size_diff = + GetCodedUIntSize(*cues_size + cue_point_size_diff) - + GetCodedUIntSize(*cues_size); + *cues_size += cue_point_size_diff; + diff = cue_size_diff + cue_point_size_diff; + if (diff > 0) { + for (int32_t i = 0; i < cues_.cue_entries_size(); ++i) { + MoveCuesBeforeClustersHelper(diff, i, cues_size); + } + } +} + +void Segment::MoveCuesBeforeClusters() { + const uint64_t current_cue_size = cues_.Size(); + uint64_t cue_size = 0; + for (int32_t i = 0; i < cues_.cue_entries_size(); ++i) + cue_size += cues_.GetCueByIndex(i)->Size(); + for (int32_t i = 0; i < cues_.cue_entries_size(); ++i) + MoveCuesBeforeClustersHelper(current_cue_size, i, &cue_size); + + // Adjust the Seek Entry to reflect the change in position + // of Cluster and Cues + int32_t cluster_index = 0; + int32_t cues_index = 0; + for (int32_t i = 0; i < SeekHead::kSeekEntryCount; ++i) { + if (seek_head_.GetId(i) == libwebm::kMkvCluster) + cluster_index = i; + if (seek_head_.GetId(i) == libwebm::kMkvCues) + cues_index = i; + } + seek_head_.SetSeekEntry(cues_index, libwebm::kMkvCues, + seek_head_.GetPosition(cluster_index)); + seek_head_.SetSeekEntry(cluster_index, libwebm::kMkvCluster, + cues_.Size() + seek_head_.GetPosition(cues_index)); +} + +bool Segment::Init(IMkvWriter* ptr_writer) { + if (!ptr_writer) { + return false; + } + writer_cluster_ = ptr_writer; + writer_cues_ = ptr_writer; + writer_header_ = ptr_writer; + return segment_info_.Init(); +} + +bool Segment::CopyAndMoveCuesBeforeClusters(mkvparser::IMkvReader* reader, + IMkvWriter* writer) { + if (!writer->Seekable() || chunking_) + return false; + const int64_t cluster_offset = + cluster_list_[0]->size_position() - GetUIntSize(libwebm::kMkvCluster); + + // Copy the headers. + if (!ChunkedCopy(reader, writer, 0, cluster_offset)) + return false; + + // Recompute cue positions and seek entries. + MoveCuesBeforeClusters(); + + // Write cues and seek entries. + // TODO(vigneshv): As of now, it's safe to call seek_head_.Finalize() for the + // second time with a different writer object. But the name Finalize() doesn't + // indicate something we want to call more than once. So consider renaming it + // to write() or some such. + if (!cues_.Write(writer) || !seek_head_.Finalize(writer)) + return false; + + // Copy the Clusters. + if (!ChunkedCopy(reader, writer, cluster_offset, + cluster_end_offset_ - cluster_offset)) + return false; + + // Update the Segment size in case the Cues size has changed. + const int64_t pos = writer->Position(); + const int64_t segment_size = writer->Position() - payload_pos_; + if (writer->Position(size_position_) || + WriteUIntSize(writer, segment_size, 8) || writer->Position(pos)) + return false; + return true; +} + +bool Segment::Finalize() { + if (WriteFramesAll() < 0) + return false; + + if (cluster_list_size_ > 0) { + // Update last cluster's size + Cluster* const old_cluster = cluster_list_[cluster_list_size_ - 1]; + + // For the last frame of the last Cluster, we don't write it as a BlockGroup + // with Duration unless the frame itself has duration set explicitly. + if (!old_cluster || !old_cluster->Finalize(false, 0)) + return false; + } + + if (mode_ == kFile) { + if (chunking_ && chunk_writer_cluster_) { + chunk_writer_cluster_->Close(); + chunk_count_++; + } + + const double duration = + (static_cast(last_timestamp_) + last_block_duration_) / + segment_info_.timecode_scale(); + segment_info_.set_duration(duration); + if (!segment_info_.Finalize(writer_header_)) + return false; + + if (output_cues_) + if (!seek_head_.AddSeekEntry(libwebm::kMkvCues, MaxOffset())) + return false; + + if (chunking_) { + if (!chunk_writer_cues_) + return false; + + char* name = NULL; + if (!UpdateChunkName("cues", &name)) + return false; + + const bool cues_open = chunk_writer_cues_->Open(name); + delete[] name; + if (!cues_open) + return false; + } + + cluster_end_offset_ = writer_cluster_->Position(); + + // Write the seek headers and cues + if (output_cues_) + if (!cues_.Write(writer_cues_)) + return false; + + if (!seek_head_.Finalize(writer_header_)) + return false; + + if (writer_header_->Seekable()) { + if (size_position_ == -1) + return false; + + const int64_t segment_size = MaxOffset(); + if (segment_size < 1) + return false; + + const int64_t pos = writer_header_->Position(); + UpdateDocTypeVersion(); + if (doc_type_version_ != doc_type_version_written_) { + if (writer_header_->Position(0)) + return false; + + if (!WriteEbmlHeader(writer_header_, doc_type_version_)) + return false; + if (writer_header_->Position() != ebml_header_size_) + return false; + + doc_type_version_written_ = doc_type_version_; + } + + if (writer_header_->Position(size_position_)) + return false; + + if (WriteUIntSize(writer_header_, segment_size, 8)) + return false; + + if (writer_header_->Position(pos)) + return false; + } + + if (chunking_) { + // Do not close any writers until the segment size has been written, + // otherwise the size may be off. + if (!chunk_writer_cues_ || !chunk_writer_header_) + return false; + + chunk_writer_cues_->Close(); + chunk_writer_header_->Close(); + } + } + + return true; +} + +Track* Segment::AddTrack(int32_t number) { + Track* const track = new (std::nothrow) Track(&seed_); // NOLINT + + if (!track) + return NULL; + + if (!tracks_.AddTrack(track, number)) { + delete track; + return NULL; + } + + return track; +} + +Chapter* Segment::AddChapter() { return chapters_.AddChapter(&seed_); } + +Tag* Segment::AddTag() { return tags_.AddTag(); } + +uint64_t Segment::AddVideoTrack(int32_t width, int32_t height, int32_t number) { + VideoTrack* const track = new (std::nothrow) VideoTrack(&seed_); // NOLINT + if (!track) + return 0; + + track->set_type(Tracks::kVideo); + track->set_codec_id(Tracks::kVp8CodecId); + track->set_width(width); + track->set_height(height); + + tracks_.AddTrack(track, number); + has_video_ = true; + + return track->number(); +} + +bool Segment::AddCuePoint(uint64_t timestamp, uint64_t track) { + if (cluster_list_size_ < 1) + return false; + + const Cluster* const cluster = cluster_list_[cluster_list_size_ - 1]; + if (!cluster) + return false; + + CuePoint* const cue = new (std::nothrow) CuePoint(); // NOLINT + if (!cue) + return false; + + cue->set_time(timestamp / segment_info_.timecode_scale()); + cue->set_block_number(cluster->blocks_added()); + cue->set_cluster_pos(cluster->position_for_cues()); + cue->set_track(track); + if (!cues_.AddCue(cue)) + return false; + + new_cuepoint_ = false; + return true; +} + +uint64_t Segment::AddAudioTrack(int32_t sample_rate, int32_t channels, + int32_t number) { + AudioTrack* const track = new (std::nothrow) AudioTrack(&seed_); // NOLINT + if (!track) + return 0; + + track->set_type(Tracks::kAudio); + track->set_codec_id(Tracks::kVorbisCodecId); + track->set_sample_rate(sample_rate); + track->set_channels(channels); + + tracks_.AddTrack(track, number); + + return track->number(); +} + +bool Segment::AddFrame(const uint8_t* data, uint64_t length, + uint64_t track_number, uint64_t timestamp, bool is_key) { + if (!data) + return false; + + Frame frame; + if (!frame.Init(data, length)) + return false; + frame.set_track_number(track_number); + frame.set_timestamp(timestamp); + frame.set_is_key(is_key); + return AddGenericFrame(&frame); +} + +bool Segment::AddFrameWithAdditional(const uint8_t* data, uint64_t length, + const uint8_t* additional, + uint64_t additional_length, + uint64_t add_id, uint64_t track_number, + uint64_t timestamp, bool is_key) { + if (!data || !additional) + return false; + + Frame frame; + if (!frame.Init(data, length) || + !frame.AddAdditionalData(additional, additional_length, add_id)) { + return false; + } + frame.set_track_number(track_number); + frame.set_timestamp(timestamp); + frame.set_is_key(is_key); + return AddGenericFrame(&frame); +} + +bool Segment::AddFrameWithDiscardPadding(const uint8_t* data, uint64_t length, + int64_t discard_padding, + uint64_t track_number, + uint64_t timestamp, bool is_key) { + if (!data) + return false; + + Frame frame; + if (!frame.Init(data, length)) + return false; + frame.set_discard_padding(discard_padding); + frame.set_track_number(track_number); + frame.set_timestamp(timestamp); + frame.set_is_key(is_key); + return AddGenericFrame(&frame); +} + +bool Segment::AddMetadata(const uint8_t* data, uint64_t length, + uint64_t track_number, uint64_t timestamp_ns, + uint64_t duration_ns) { + if (!data) + return false; + + Frame frame; + if (!frame.Init(data, length)) + return false; + frame.set_track_number(track_number); + frame.set_timestamp(timestamp_ns); + frame.set_duration(duration_ns); + frame.set_is_key(true); // All metadata blocks are keyframes. + return AddGenericFrame(&frame); +} + +bool Segment::AddGenericFrame(const Frame* frame) { + if (!frame) + return false; + + if (!CheckHeaderInfo()) + return false; + + // Check for non-monotonically increasing timestamps. + if (frame->timestamp() < last_timestamp_) + return false; + + // Check if the track number is valid. + if (!tracks_.GetTrackByNumber(frame->track_number())) + return false; + + if (frame->discard_padding() != 0) + doc_type_version_ = 4; + + // If the segment has a video track hold onto audio frames to make sure the + // audio that is associated with the start time of a video key-frame is + // muxed into the same cluster. + if (has_video_ && tracks_.TrackIsAudio(frame->track_number()) && + !force_new_cluster_) { + Frame* const new_frame = new (std::nothrow) Frame(); + if (!new_frame || !new_frame->CopyFrom(*frame)) + return false; + return QueueFrame(new_frame); + } + + if (!DoNewClusterProcessing(frame->track_number(), frame->timestamp(), + frame->is_key())) { + return false; + } + + if (cluster_list_size_ < 1) + return false; + + Cluster* const cluster = cluster_list_[cluster_list_size_ - 1]; + if (!cluster) + return false; + + // If the Frame is not a SimpleBlock, then set the reference_block_timestamp + // if it is not set already. + bool frame_created = false; + if (!frame->CanBeSimpleBlock() && !frame->is_key() && + !frame->reference_block_timestamp_set()) { + Frame* const new_frame = new (std::nothrow) Frame(); + if (!new_frame->CopyFrom(*frame)) + return false; + new_frame->set_reference_block_timestamp( + last_track_timestamp_[frame->track_number() - 1]); + frame = new_frame; + frame_created = true; + } + + if (!cluster->AddFrame(frame)) + return false; + + if (new_cuepoint_ && cues_track_ == frame->track_number()) { + if (!AddCuePoint(frame->timestamp(), cues_track_)) + return false; + } + + last_timestamp_ = frame->timestamp(); + last_track_timestamp_[frame->track_number() - 1] = frame->timestamp(); + last_block_duration_ = frame->duration(); + + if (frame_created) + delete frame; + + return true; +} + +void Segment::OutputCues(bool output_cues) { output_cues_ = output_cues; } + +void Segment::AccurateClusterDuration(bool accurate_cluster_duration) { + accurate_cluster_duration_ = accurate_cluster_duration; +} + +void Segment::UseFixedSizeClusterTimecode(bool fixed_size_cluster_timecode) { + fixed_size_cluster_timecode_ = fixed_size_cluster_timecode; +} + +bool Segment::SetChunking(bool chunking, const char* filename) { + if (chunk_count_ > 0) + return false; + + if (chunking) { + if (!filename) + return false; + + // Check if we are being set to what is already set. + if (chunking_ && !strcmp(filename, chunking_base_name_)) + return true; + + const size_t name_length = strlen(filename) + 1; + char* const temp = new (std::nothrow) char[name_length]; // NOLINT + if (!temp) + return false; + +#ifdef _MSC_VER + strcpy_s(temp, name_length, filename); +#else + strcpy(temp, filename); +#endif + + delete[] chunking_base_name_; + chunking_base_name_ = temp; + + if (!UpdateChunkName("chk", &chunk_name_)) + return false; + + if (!chunk_writer_cluster_) { + chunk_writer_cluster_ = new (std::nothrow) MkvWriter(); // NOLINT + if (!chunk_writer_cluster_) + return false; + } + + if (!chunk_writer_cues_) { + chunk_writer_cues_ = new (std::nothrow) MkvWriter(); // NOLINT + if (!chunk_writer_cues_) + return false; + } + + if (!chunk_writer_header_) { + chunk_writer_header_ = new (std::nothrow) MkvWriter(); // NOLINT + if (!chunk_writer_header_) + return false; + } + + if (!chunk_writer_cluster_->Open(chunk_name_)) + return false; + + const size_t header_length = strlen(filename) + strlen(".hdr") + 1; + char* const header = new (std::nothrow) char[header_length]; // NOLINT + if (!header) + return false; + +#ifdef _MSC_VER + strcpy_s(header, header_length - strlen(".hdr"), chunking_base_name_); + strcat_s(header, header_length, ".hdr"); +#else + strcpy(header, chunking_base_name_); + strcat(header, ".hdr"); +#endif + if (!chunk_writer_header_->Open(header)) { + delete[] header; + return false; + } + + writer_cluster_ = chunk_writer_cluster_; + writer_cues_ = chunk_writer_cues_; + writer_header_ = chunk_writer_header_; + + delete[] header; + } + + chunking_ = chunking; + + return true; +} + +bool Segment::CuesTrack(uint64_t track_number) { + const Track* const track = GetTrackByNumber(track_number); + if (!track) + return false; + + cues_track_ = track_number; + return true; +} + +void Segment::ForceNewClusterOnNextFrame() { force_new_cluster_ = true; } + +Track* Segment::GetTrackByNumber(uint64_t track_number) const { + return tracks_.GetTrackByNumber(track_number); +} + +bool Segment::WriteSegmentHeader() { + UpdateDocTypeVersion(); + + // TODO(fgalligan): Support more than one segment. + if (!WriteEbmlHeader(writer_header_, doc_type_version_)) + return false; + doc_type_version_written_ = doc_type_version_; + ebml_header_size_ = static_cast(writer_header_->Position()); + + // Write "unknown" (-1) as segment size value. If mode is kFile, Segment + // will write over duration when the file is finalized. + if (WriteID(writer_header_, libwebm::kMkvSegment)) + return false; + + // Save for later. + size_position_ = writer_header_->Position(); + + // Write "unknown" (EBML coded -1) as segment size value. We need to write 8 + // bytes because if we are going to overwrite the segment size later we do + // not know how big our segment will be. + if (SerializeInt(writer_header_, kEbmlUnknownValue, 8)) + return false; + + payload_pos_ = writer_header_->Position(); + + if (mode_ == kFile && writer_header_->Seekable()) { + // Set the duration > 0.0 so SegmentInfo will write out the duration. When + // the muxer is done writing we will set the correct duration and have + // SegmentInfo upadte it. + segment_info_.set_duration(1.0); + + if (!seek_head_.Write(writer_header_)) + return false; + } + + if (!seek_head_.AddSeekEntry(libwebm::kMkvInfo, MaxOffset())) + return false; + if (!segment_info_.Write(writer_header_)) + return false; + + if (!seek_head_.AddSeekEntry(libwebm::kMkvTracks, MaxOffset())) + return false; + if (!tracks_.Write(writer_header_)) + return false; + + if (chapters_.Count() > 0) { + if (!seek_head_.AddSeekEntry(libwebm::kMkvChapters, MaxOffset())) + return false; + if (!chapters_.Write(writer_header_)) + return false; + } + + if (tags_.Count() > 0) { + if (!seek_head_.AddSeekEntry(libwebm::kMkvTags, MaxOffset())) + return false; + if (!tags_.Write(writer_header_)) + return false; + } + + if (chunking_ && (mode_ == kLive || !writer_header_->Seekable())) { + if (!chunk_writer_header_) + return false; + + chunk_writer_header_->Close(); + } + + header_written_ = true; + + return true; +} + +// Here we are testing whether to create a new cluster, given a frame +// having time frame_timestamp_ns. +// +int Segment::TestFrame(uint64_t track_number, uint64_t frame_timestamp_ns, + bool is_key) const { + if (force_new_cluster_) + return 1; + + // If no clusters have been created yet, then create a new cluster + // and write this frame immediately, in the new cluster. This path + // should only be followed once, the first time we attempt to write + // a frame. + + if (cluster_list_size_ <= 0) + return 1; + + // There exists at least one cluster. We must compare the frame to + // the last cluster, in order to determine whether the frame is + // written to the existing cluster, or that a new cluster should be + // created. + + const uint64_t timecode_scale = segment_info_.timecode_scale(); + const uint64_t frame_timecode = frame_timestamp_ns / timecode_scale; + + const Cluster* const last_cluster = cluster_list_[cluster_list_size_ - 1]; + const uint64_t last_cluster_timecode = last_cluster->timecode(); + + // For completeness we test for the case when the frame's timecode + // is less than the cluster's timecode. Although in principle that + // is allowed, this muxer doesn't actually write clusters like that, + // so this indicates a bug somewhere in our algorithm. + + if (frame_timecode < last_cluster_timecode) // should never happen + return -1; + + // If the frame has a timestamp significantly larger than the last + // cluster (in Matroska, cluster-relative timestamps are serialized + // using a 16-bit signed integer), then we cannot write this frame + // to that cluster, and so we must create a new cluster. + + const int64_t delta_timecode = frame_timecode - last_cluster_timecode; + + if (delta_timecode > kMaxBlockTimecode) + return 2; + + // We decide to create a new cluster when we have a video keyframe. + // This will flush queued (audio) frames, and write the keyframe + // immediately, in the newly-created cluster. + + if (is_key && tracks_.TrackIsVideo(track_number)) + return 1; + + // Create a new cluster if we have accumulated too many frames + // already, where "too many" is defined as "the total time of frames + // in the cluster exceeds a threshold". + + const uint64_t delta_ns = delta_timecode * timecode_scale; + + if (max_cluster_duration_ > 0 && delta_ns >= max_cluster_duration_) + return 1; + + // This is similar to the case above, with the difference that a new + // cluster is created when the size of the current cluster exceeds a + // threshold. + + const uint64_t cluster_size = last_cluster->payload_size(); + + if (max_cluster_size_ > 0 && cluster_size >= max_cluster_size_) + return 1; + + // There's no need to create a new cluster, so emit this frame now. + + return 0; +} + +bool Segment::MakeNewCluster(uint64_t frame_timestamp_ns) { + const int32_t new_size = cluster_list_size_ + 1; + + if (new_size > cluster_list_capacity_) { + // Add more clusters. + const int32_t new_capacity = + (cluster_list_capacity_ <= 0) ? 1 : cluster_list_capacity_ * 2; + Cluster** const clusters = + new (std::nothrow) Cluster*[new_capacity]; // NOLINT + if (!clusters) + return false; + + for (int32_t i = 0; i < cluster_list_size_; ++i) { + clusters[i] = cluster_list_[i]; + } + + delete[] cluster_list_; + + cluster_list_ = clusters; + cluster_list_capacity_ = new_capacity; + } + + if (!WriteFramesLessThan(frame_timestamp_ns)) + return false; + + if (cluster_list_size_ > 0) { + // Update old cluster's size + Cluster* const old_cluster = cluster_list_[cluster_list_size_ - 1]; + + if (!old_cluster || !old_cluster->Finalize(true, frame_timestamp_ns)) + return false; + } + + if (output_cues_) + new_cuepoint_ = true; + + if (chunking_ && cluster_list_size_ > 0) { + chunk_writer_cluster_->Close(); + chunk_count_++; + + if (!UpdateChunkName("chk", &chunk_name_)) + return false; + if (!chunk_writer_cluster_->Open(chunk_name_)) + return false; + } + + const uint64_t timecode_scale = segment_info_.timecode_scale(); + const uint64_t frame_timecode = frame_timestamp_ns / timecode_scale; + + uint64_t cluster_timecode = frame_timecode; + + if (frames_size_ > 0) { + const Frame* const f = frames_[0]; // earliest queued frame + const uint64_t ns = f->timestamp(); + const uint64_t tc = ns / timecode_scale; + + if (tc < cluster_timecode) + cluster_timecode = tc; + } + + Cluster*& cluster = cluster_list_[cluster_list_size_]; + const int64_t offset = MaxOffset(); + cluster = new (std::nothrow) + Cluster(cluster_timecode, offset, segment_info_.timecode_scale(), + accurate_cluster_duration_, fixed_size_cluster_timecode_); + if (!cluster) + return false; + + if (!cluster->Init(writer_cluster_)) + return false; + + cluster_list_size_ = new_size; + return true; +} + +bool Segment::DoNewClusterProcessing(uint64_t track_number, + uint64_t frame_timestamp_ns, bool is_key) { + for (;;) { + // Based on the characteristics of the current frame and current + // cluster, decide whether to create a new cluster. + const int result = TestFrame(track_number, frame_timestamp_ns, is_key); + if (result < 0) // error + return false; + + // Always set force_new_cluster_ to false after TestFrame. + force_new_cluster_ = false; + + // A non-zero result means create a new cluster. + if (result > 0 && !MakeNewCluster(frame_timestamp_ns)) + return false; + + // Write queued (audio) frames. + const int frame_count = WriteFramesAll(); + if (frame_count < 0) // error + return false; + + // Write the current frame to the current cluster (if TestFrame + // returns 0) or to a newly created cluster (TestFrame returns 1). + if (result <= 1) + return true; + + // TestFrame returned 2, which means there was a large time + // difference between the cluster and the frame itself. Do the + // test again, comparing the frame to the new cluster. + } +} + +bool Segment::CheckHeaderInfo() { + if (!header_written_) { + if (!WriteSegmentHeader()) + return false; + + if (!seek_head_.AddSeekEntry(libwebm::kMkvCluster, MaxOffset())) + return false; + + if (output_cues_ && cues_track_ == 0) { + // Check for a video track + for (uint32_t i = 0; i < tracks_.track_entries_size(); ++i) { + const Track* const track = tracks_.GetTrackByIndex(i); + if (!track) + return false; + + if (tracks_.TrackIsVideo(track->number())) { + cues_track_ = track->number(); + break; + } + } + + // Set first track found + if (cues_track_ == 0) { + const Track* const track = tracks_.GetTrackByIndex(0); + if (!track) + return false; + + cues_track_ = track->number(); + } + } + } + return true; +} + +void Segment::UpdateDocTypeVersion() { + for (uint32_t index = 0; index < tracks_.track_entries_size(); ++index) { + const Track* track = tracks_.GetTrackByIndex(index); + if (track == NULL) + break; + if ((track->codec_delay() || track->seek_pre_roll()) && + doc_type_version_ < 4) { + doc_type_version_ = 4; + break; + } + } +} + +bool Segment::UpdateChunkName(const char* ext, char** name) const { + if (!name || !ext) + return false; + + char ext_chk[64]; +#ifdef _MSC_VER + sprintf_s(ext_chk, sizeof(ext_chk), "_%06d.%s", chunk_count_, ext); +#else + snprintf(ext_chk, sizeof(ext_chk), "_%06d.%s", chunk_count_, ext); +#endif + + const size_t length = strlen(chunking_base_name_) + strlen(ext_chk) + 1; + char* const str = new (std::nothrow) char[length]; // NOLINT + if (!str) + return false; + +#ifdef _MSC_VER + strcpy_s(str, length - strlen(ext_chk), chunking_base_name_); + strcat_s(str, length, ext_chk); +#else + strcpy(str, chunking_base_name_); + strcat(str, ext_chk); +#endif + + delete[] * name; + *name = str; + + return true; +} + +int64_t Segment::MaxOffset() { + if (!writer_header_) + return -1; + + int64_t offset = writer_header_->Position() - payload_pos_; + + if (chunking_) { + for (int32_t i = 0; i < cluster_list_size_; ++i) { + Cluster* const cluster = cluster_list_[i]; + offset += cluster->Size(); + } + + if (writer_cues_) + offset += writer_cues_->Position(); + } + + return offset; +} + +bool Segment::QueueFrame(Frame* frame) { + const int32_t new_size = frames_size_ + 1; + + if (new_size > frames_capacity_) { + // Add more frames. + const int32_t new_capacity = (!frames_capacity_) ? 2 : frames_capacity_ * 2; + + if (new_capacity < 1) + return false; + + Frame** const frames = new (std::nothrow) Frame*[new_capacity]; // NOLINT + if (!frames) + return false; + + for (int32_t i = 0; i < frames_size_; ++i) { + frames[i] = frames_[i]; + } + + delete[] frames_; + frames_ = frames; + frames_capacity_ = new_capacity; + } + + frames_[frames_size_++] = frame; + + return true; +} + +int Segment::WriteFramesAll() { + if (frames_ == NULL) + return 0; + + if (cluster_list_size_ < 1) + return -1; + + Cluster* const cluster = cluster_list_[cluster_list_size_ - 1]; + + if (!cluster) + return -1; + + for (int32_t i = 0; i < frames_size_; ++i) { + Frame*& frame = frames_[i]; + // TODO(jzern/vigneshv): using Segment::AddGenericFrame here would limit the + // places where |doc_type_version_| needs to be updated. + if (frame->discard_padding() != 0) + doc_type_version_ = 4; + if (!cluster->AddFrame(frame)) + return -1; + + if (new_cuepoint_ && cues_track_ == frame->track_number()) { + if (!AddCuePoint(frame->timestamp(), cues_track_)) + return -1; + } + + if (frame->timestamp() > last_timestamp_) { + last_timestamp_ = frame->timestamp(); + last_track_timestamp_[frame->track_number() - 1] = frame->timestamp(); + } + + delete frame; + frame = NULL; + } + + const int result = frames_size_; + frames_size_ = 0; + + return result; +} + +bool Segment::WriteFramesLessThan(uint64_t timestamp) { + // Check |cluster_list_size_| to see if this is the first cluster. If it is + // the first cluster the audio frames that are less than the first video + // timesatmp will be written in a later step. + if (frames_size_ > 0 && cluster_list_size_ > 0) { + if (!frames_) + return false; + + Cluster* const cluster = cluster_list_[cluster_list_size_ - 1]; + if (!cluster) + return false; + + int32_t shift_left = 0; + + // TODO(fgalligan): Change this to use the durations of frames instead of + // the next frame's start time if the duration is accurate. + for (int32_t i = 1; i < frames_size_; ++i) { + const Frame* const frame_curr = frames_[i]; + + if (frame_curr->timestamp() > timestamp) + break; + + const Frame* const frame_prev = frames_[i - 1]; + if (frame_prev->discard_padding() != 0) + doc_type_version_ = 4; + if (!cluster->AddFrame(frame_prev)) + return false; + + if (new_cuepoint_ && cues_track_ == frame_prev->track_number()) { + if (!AddCuePoint(frame_prev->timestamp(), cues_track_)) + return false; + } + + ++shift_left; + if (frame_prev->timestamp() > last_timestamp_) { + last_timestamp_ = frame_prev->timestamp(); + last_track_timestamp_[frame_prev->track_number() - 1] = + frame_prev->timestamp(); + } + + delete frame_prev; + } + + if (shift_left > 0) { + if (shift_left >= frames_size_) + return false; + + const int32_t new_frames_size = frames_size_ - shift_left; + for (int32_t i = 0; i < new_frames_size; ++i) { + frames_[i] = frames_[i + shift_left]; + } + + frames_size_ = new_frames_size; + } + } + + return true; +} + +} // namespace mkvmuxer diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.h b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.h new file mode 100644 index 0000000000..01b26a212b --- /dev/null +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.h @@ -0,0 +1,1695 @@ +// Copyright (c) 2012 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. + +#ifndef MKVMUXER_MKVMUXER_H_ +#define MKVMUXER_MKVMUXER_H_ + +#include + +#include +#include +#include + +#include "common/webmids.h" +#include "mkvmuxer/mkvmuxertypes.h" + +// For a description of the WebM elements see +// http://www.webmproject.org/code/specs/container/. + +namespace mkvparser { +class IMkvReader; +} // namespace mkvparser + +namespace mkvmuxer { + +class MkvWriter; +class Segment; + +const uint64_t kMaxTrackNumber = 126; + +/////////////////////////////////////////////////////////////// +// Interface used by the mkvmuxer to write out the Mkv data. +class IMkvWriter { + public: + // Writes out |len| bytes of |buf|. Returns 0 on success. + virtual int32 Write(const void* buf, uint32 len) = 0; + + // Returns the offset of the output position from the beginning of the + // output. + virtual int64 Position() const = 0; + + // Set the current File position. Returns 0 on success. + virtual int32 Position(int64 position) = 0; + + // Returns true if the writer is seekable. + virtual bool Seekable() const = 0; + + // Element start notification. Called whenever an element identifier is about + // to be written to the stream. |element_id| is the element identifier, and + // |position| is the location in the WebM stream where the first octet of the + // element identifier will be written. + // Note: the |MkvId| enumeration in webmids.hpp defines element values. + virtual void ElementStartNotify(uint64 element_id, int64 position) = 0; + + protected: + IMkvWriter(); + virtual ~IMkvWriter(); + + private: + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(IMkvWriter); +}; + +// Writes out the EBML header for a WebM file. This function must be called +// before any other libwebm writing functions are called. +bool WriteEbmlHeader(IMkvWriter* writer, uint64_t doc_type_version); + +// Deprecated. Writes out EBML header with doc_type_version as +// kDefaultDocTypeVersion. Exists for backward compatibility. +bool WriteEbmlHeader(IMkvWriter* writer); + +// Copies in Chunk from source to destination between the given byte positions +bool ChunkedCopy(mkvparser::IMkvReader* source, IMkvWriter* dst, int64_t start, + int64_t size); + +/////////////////////////////////////////////////////////////// +// Class to hold data the will be written to a block. +class Frame { + public: + Frame(); + ~Frame(); + + // Sets this frame's contents based on |frame|. Returns true on success. On + // failure, this frame's existing contents may be lost. + bool CopyFrom(const Frame& frame); + + // Copies |frame| data into |frame_|. Returns true on success. + bool Init(const uint8_t* frame, uint64_t length); + + // Copies |additional| data into |additional_|. Returns true on success. + bool AddAdditionalData(const uint8_t* additional, uint64_t length, + uint64_t add_id); + + // Returns true if the frame has valid parameters. + bool IsValid() const; + + // Returns true if the frame can be written as a SimpleBlock based on current + // parameters. + bool CanBeSimpleBlock() const; + + uint64_t add_id() const { return add_id_; } + const uint8_t* additional() const { return additional_; } + uint64_t additional_length() const { return additional_length_; } + void set_duration(uint64_t duration); + uint64_t duration() const { return duration_; } + bool duration_set() const { return duration_set_; } + const uint8_t* frame() const { return frame_; } + void set_is_key(bool key) { is_key_ = key; } + bool is_key() const { return is_key_; } + uint64_t length() const { return length_; } + void set_track_number(uint64_t track_number) { track_number_ = track_number; } + uint64_t track_number() const { return track_number_; } + void set_timestamp(uint64_t timestamp) { timestamp_ = timestamp; } + uint64_t timestamp() const { return timestamp_; } + void set_discard_padding(int64_t discard_padding) { + discard_padding_ = discard_padding; + } + int64_t discard_padding() const { return discard_padding_; } + void set_reference_block_timestamp(int64_t reference_block_timestamp); + int64_t reference_block_timestamp() const { + return reference_block_timestamp_; + } + bool reference_block_timestamp_set() const { + return reference_block_timestamp_set_; + } + + private: + // Id of the Additional data. + uint64_t add_id_; + + // Pointer to additional data. Owned by this class. + uint8_t* additional_; + + // Length of the additional data. + uint64_t additional_length_; + + // Duration of the frame in nanoseconds. + uint64_t duration_; + + // Flag indicating that |duration_| has been set. Setting duration causes the + // frame to be written out as a Block with BlockDuration instead of as a + // SimpleBlock. + bool duration_set_; + + // Pointer to the data. Owned by this class. + uint8_t* frame_; + + // Flag telling if the data should set the key flag of a block. + bool is_key_; + + // Length of the data. + uint64_t length_; + + // Mkv track number the data is associated with. + uint64_t track_number_; + + // Timestamp of the data in nanoseconds. + uint64_t timestamp_; + + // Discard padding for the frame. + int64_t discard_padding_; + + // Reference block timestamp. + int64_t reference_block_timestamp_; + + // Flag indicating if |reference_block_timestamp_| has been set. + bool reference_block_timestamp_set_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(Frame); +}; + +/////////////////////////////////////////////////////////////// +// Class to hold one cue point in a Cues element. +class CuePoint { + public: + CuePoint(); + ~CuePoint(); + + // Returns the size in bytes for the entire CuePoint element. + uint64_t Size() const; + + // Output the CuePoint element to the writer. Returns true on success. + bool Write(IMkvWriter* writer) const; + + void set_time(uint64_t time) { time_ = time; } + uint64_t time() const { return time_; } + void set_track(uint64_t track) { track_ = track; } + uint64_t track() const { return track_; } + void set_cluster_pos(uint64_t cluster_pos) { cluster_pos_ = cluster_pos; } + uint64_t cluster_pos() const { return cluster_pos_; } + void set_block_number(uint64_t block_number) { block_number_ = block_number; } + uint64_t block_number() const { return block_number_; } + void set_output_block_number(bool output_block_number) { + output_block_number_ = output_block_number; + } + bool output_block_number() const { return output_block_number_; } + + private: + // Returns the size in bytes for the payload of the CuePoint element. + uint64_t PayloadSize() const; + + // Absolute timecode according to the segment time base. + uint64_t time_; + + // The Track element associated with the CuePoint. + uint64_t track_; + + // The position of the Cluster containing the Block. + uint64_t cluster_pos_; + + // Number of the Block within the Cluster, starting from 1. + uint64_t block_number_; + + // If true the muxer will write out the block number for the cue if the + // block number is different than the default of 1. Default is set to true. + bool output_block_number_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(CuePoint); +}; + +/////////////////////////////////////////////////////////////// +// Cues element. +class Cues { + public: + Cues(); + ~Cues(); + + // Adds a cue point to the Cues element. Returns true on success. + bool AddCue(CuePoint* cue); + + // Returns the cue point by index. Returns NULL if there is no cue point + // match. + CuePoint* GetCueByIndex(int32_t index) const; + + // Returns the total size of the Cues element + uint64_t Size(); + + // Output the Cues element to the writer. Returns true on success. + bool Write(IMkvWriter* writer) const; + + int32_t cue_entries_size() const { return cue_entries_size_; } + void set_output_block_number(bool output_block_number) { + output_block_number_ = output_block_number; + } + bool output_block_number() const { return output_block_number_; } + + private: + // Number of allocated elements in |cue_entries_|. + int32_t cue_entries_capacity_; + + // Number of CuePoints in |cue_entries_|. + int32_t cue_entries_size_; + + // CuePoint list. + CuePoint** cue_entries_; + + // If true the muxer will write out the block number for the cue if the + // block number is different than the default of 1. Default is set to true. + bool output_block_number_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(Cues); +}; + +/////////////////////////////////////////////////////////////// +// ContentEncAESSettings element +class ContentEncAESSettings { + public: + enum { kCTR = 1 }; + + ContentEncAESSettings(); + ~ContentEncAESSettings() {} + + // Returns the size in bytes for the ContentEncAESSettings element. + uint64_t Size() const; + + // Writes out the ContentEncAESSettings element to |writer|. Returns true on + // success. + bool Write(IMkvWriter* writer) const; + + uint64_t cipher_mode() const { return cipher_mode_; } + + private: + // Returns the size in bytes for the payload of the ContentEncAESSettings + // element. + uint64_t PayloadSize() const; + + // Sub elements + uint64_t cipher_mode_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(ContentEncAESSettings); +}; + +/////////////////////////////////////////////////////////////// +// ContentEncoding element +// Elements used to describe if the track data has been encrypted or +// compressed with zlib or header stripping. +// Currently only whole frames can be encrypted with AES. This dictates that +// ContentEncodingOrder will be 0, ContentEncodingScope will be 1, +// ContentEncodingType will be 1, and ContentEncAlgo will be 5. +class ContentEncoding { + public: + ContentEncoding(); + ~ContentEncoding(); + + // Sets the content encryption id. Copies |length| bytes from |id| to + // |enc_key_id_|. Returns true on success. + bool SetEncryptionID(const uint8_t* id, uint64_t length); + + // Returns the size in bytes for the ContentEncoding element. + uint64_t Size() const; + + // Writes out the ContentEncoding element to |writer|. Returns true on + // success. + bool Write(IMkvWriter* writer) const; + + uint64_t enc_algo() const { return enc_algo_; } + uint64_t encoding_order() const { return encoding_order_; } + uint64_t encoding_scope() const { return encoding_scope_; } + uint64_t encoding_type() const { return encoding_type_; } + ContentEncAESSettings* enc_aes_settings() { return &enc_aes_settings_; } + + private: + // Returns the size in bytes for the encoding elements. + uint64_t EncodingSize(uint64_t compresion_size, + uint64_t encryption_size) const; + + // Returns the size in bytes for the encryption elements. + uint64_t EncryptionSize() const; + + // Track element names + uint64_t enc_algo_; + uint8_t* enc_key_id_; + uint64_t encoding_order_; + uint64_t encoding_scope_; + uint64_t encoding_type_; + + // ContentEncAESSettings element. + ContentEncAESSettings enc_aes_settings_; + + // Size of the ContentEncKeyID data in bytes. + uint64_t enc_key_id_length_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(ContentEncoding); +}; + +/////////////////////////////////////////////////////////////// +// Colour element. +struct PrimaryChromaticity { + PrimaryChromaticity(float x_val, float y_val) : x(x_val), y(y_val) {} + PrimaryChromaticity() : x(0), y(0) {} + ~PrimaryChromaticity() {} + uint64_t PrimaryChromaticityPayloadSize(libwebm::MkvId x_id, + libwebm::MkvId y_id) const; + bool Write(IMkvWriter* writer, libwebm::MkvId x_id, + libwebm::MkvId y_id) const; + + float x; + float y; +}; + +class MasteringMetadata { + public: + static const float kValueNotPresent; + + MasteringMetadata() + : luminance_max(kValueNotPresent), + luminance_min(kValueNotPresent), + r_(NULL), + g_(NULL), + b_(NULL), + white_point_(NULL) {} + ~MasteringMetadata() { + delete r_; + delete g_; + delete b_; + delete white_point_; + } + + // Returns total size of the MasteringMetadata element. + uint64_t MasteringMetadataSize() const; + bool Write(IMkvWriter* writer) const; + + // Copies non-null chromaticity. + bool SetChromaticity(const PrimaryChromaticity* r, + const PrimaryChromaticity* g, + const PrimaryChromaticity* b, + const PrimaryChromaticity* white_point); + const PrimaryChromaticity* r() const { return r_; } + const PrimaryChromaticity* g() const { return g_; } + const PrimaryChromaticity* b() const { return b_; } + const PrimaryChromaticity* white_point() const { return white_point_; } + + float luminance_max; + float luminance_min; + + private: + // Returns size of MasteringMetadata child elements. + uint64_t PayloadSize() const; + + PrimaryChromaticity* r_; + PrimaryChromaticity* g_; + PrimaryChromaticity* b_; + PrimaryChromaticity* white_point_; +}; + +class Colour { + public: + static const uint64_t kValueNotPresent; + Colour() + : matrix_coefficients(kValueNotPresent), + bits_per_channel(kValueNotPresent), + chroma_subsampling_horz(kValueNotPresent), + chroma_subsampling_vert(kValueNotPresent), + cb_subsampling_horz(kValueNotPresent), + cb_subsampling_vert(kValueNotPresent), + chroma_siting_horz(kValueNotPresent), + chroma_siting_vert(kValueNotPresent), + range(kValueNotPresent), + transfer_characteristics(kValueNotPresent), + primaries(kValueNotPresent), + max_cll(kValueNotPresent), + max_fall(kValueNotPresent), + mastering_metadata_(NULL) {} + ~Colour() { delete mastering_metadata_; } + + // Returns total size of the Colour element. + uint64_t ColourSize() const; + bool Write(IMkvWriter* writer) const; + + // Deep copies |mastering_metadata|. + bool SetMasteringMetadata(const MasteringMetadata& mastering_metadata); + + const MasteringMetadata* mastering_metadata() const { + return mastering_metadata_; + } + + uint64_t matrix_coefficients; + uint64_t bits_per_channel; + uint64_t chroma_subsampling_horz; + uint64_t chroma_subsampling_vert; + uint64_t cb_subsampling_horz; + uint64_t cb_subsampling_vert; + uint64_t chroma_siting_horz; + uint64_t chroma_siting_vert; + uint64_t range; + uint64_t transfer_characteristics; + uint64_t primaries; + uint64_t max_cll; + uint64_t max_fall; + + private: + // Returns size of Colour child elements. + uint64_t PayloadSize() const; + + MasteringMetadata* mastering_metadata_; +}; + +/////////////////////////////////////////////////////////////// +// Track element. +class Track { + public: + // The |seed| parameter is used to synthesize a UID for the track. + explicit Track(unsigned int* seed); + virtual ~Track(); + + // Adds a ContentEncoding element to the Track. Returns true on success. + virtual bool AddContentEncoding(); + + // Returns the ContentEncoding by index. Returns NULL if there is no + // ContentEncoding match. + ContentEncoding* GetContentEncodingByIndex(uint32_t index) const; + + // Returns the size in bytes for the payload of the Track element. + virtual uint64_t PayloadSize() const; + + // Returns the size in bytes of the Track element. + virtual uint64_t Size() const; + + // Output the Track element to the writer. Returns true on success. + virtual bool Write(IMkvWriter* writer) const; + + // Sets the CodecPrivate element of the Track element. Copies |length| + // bytes from |codec_private| to |codec_private_|. Returns true on success. + bool SetCodecPrivate(const uint8_t* codec_private, uint64_t length); + + void set_codec_id(const char* codec_id); + const char* codec_id() const { return codec_id_; } + const uint8_t* codec_private() const { return codec_private_; } + void set_language(const char* language); + const char* language() const { return language_; } + void set_max_block_additional_id(uint64_t max_block_additional_id) { + max_block_additional_id_ = max_block_additional_id; + } + uint64_t max_block_additional_id() const { return max_block_additional_id_; } + void set_name(const char* name); + const char* name() const { return name_; } + void set_number(uint64_t number) { number_ = number; } + uint64_t number() const { return number_; } + void set_type(uint64_t type) { type_ = type; } + uint64_t type() const { return type_; } + void set_uid(uint64_t uid) { uid_ = uid; } + uint64_t uid() const { return uid_; } + void set_codec_delay(uint64_t codec_delay) { codec_delay_ = codec_delay; } + uint64_t codec_delay() const { return codec_delay_; } + void set_seek_pre_roll(uint64_t seek_pre_roll) { + seek_pre_roll_ = seek_pre_roll; + } + uint64_t seek_pre_roll() const { return seek_pre_roll_; } + void set_default_duration(uint64_t default_duration) { + default_duration_ = default_duration; + } + uint64_t default_duration() const { return default_duration_; } + + uint64_t codec_private_length() const { return codec_private_length_; } + uint32_t content_encoding_entries_size() const { + return content_encoding_entries_size_; + } + + private: + // Track element names. + char* codec_id_; + uint8_t* codec_private_; + char* language_; + uint64_t max_block_additional_id_; + char* name_; + uint64_t number_; + uint64_t type_; + uint64_t uid_; + uint64_t codec_delay_; + uint64_t seek_pre_roll_; + uint64_t default_duration_; + + // Size of the CodecPrivate data in bytes. + uint64_t codec_private_length_; + + // ContentEncoding element list. + ContentEncoding** content_encoding_entries_; + + // Number of ContentEncoding elements added. + uint32_t content_encoding_entries_size_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(Track); +}; + +/////////////////////////////////////////////////////////////// +// Track that has video specific elements. +class VideoTrack : public Track { + public: + // Supported modes for stereo 3D. + enum StereoMode { + kMono = 0, + kSideBySideLeftIsFirst = 1, + kTopBottomRightIsFirst = 2, + kTopBottomLeftIsFirst = 3, + kSideBySideRightIsFirst = 11 + }; + + enum AlphaMode { kNoAlpha = 0, kAlpha = 1 }; + + // The |seed| parameter is used to synthesize a UID for the track. + explicit VideoTrack(unsigned int* seed); + virtual ~VideoTrack(); + + // Returns the size in bytes for the payload of the Track element plus the + // video specific elements. + virtual uint64_t PayloadSize() const; + + // Output the VideoTrack element to the writer. Returns true on success. + virtual bool Write(IMkvWriter* writer) const; + + // Sets the video's stereo mode. Returns true on success. + bool SetStereoMode(uint64_t stereo_mode); + + // Sets the video's alpha mode. Returns true on success. + bool SetAlphaMode(uint64_t alpha_mode); + + void set_display_height(uint64_t height) { display_height_ = height; } + uint64_t display_height() const { return display_height_; } + void set_display_width(uint64_t width) { display_width_ = width; } + uint64_t display_width() const { return display_width_; } + + void set_crop_left(uint64_t crop_left) { crop_left_ = crop_left; } + uint64_t crop_left() const { return crop_left_; } + void set_crop_right(uint64_t crop_right) { crop_right_ = crop_right; } + uint64_t crop_right() const { return crop_right_; } + void set_crop_top(uint64_t crop_top) { crop_top_ = crop_top; } + uint64_t crop_top() const { return crop_top_; } + void set_crop_bottom(uint64_t crop_bottom) { crop_bottom_ = crop_bottom; } + uint64_t crop_bottom() const { return crop_bottom_; } + + void set_frame_rate(double frame_rate) { frame_rate_ = frame_rate; } + double frame_rate() const { return frame_rate_; } + void set_height(uint64_t height) { height_ = height; } + uint64_t height() const { return height_; } + uint64_t stereo_mode() { return stereo_mode_; } + uint64_t alpha_mode() { return alpha_mode_; } + void set_width(uint64_t width) { width_ = width; } + uint64_t width() const { return width_; } + + Colour* colour() { return colour_; } + + // Deep copies |colour|. + bool SetColour(const Colour& colour); + + private: + // Returns the size in bytes of the Video element. + uint64_t VideoPayloadSize() const; + + // Video track element names. + uint64_t display_height_; + uint64_t display_width_; + uint64_t crop_left_; + uint64_t crop_right_; + uint64_t crop_top_; + uint64_t crop_bottom_; + double frame_rate_; + uint64_t height_; + uint64_t stereo_mode_; + uint64_t alpha_mode_; + uint64_t width_; + + Colour* colour_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(VideoTrack); +}; + +/////////////////////////////////////////////////////////////// +// Track that has audio specific elements. +class AudioTrack : public Track { + public: + // The |seed| parameter is used to synthesize a UID for the track. + explicit AudioTrack(unsigned int* seed); + virtual ~AudioTrack(); + + // Returns the size in bytes for the payload of the Track element plus the + // audio specific elements. + virtual uint64_t PayloadSize() const; + + // Output the AudioTrack element to the writer. Returns true on success. + virtual bool Write(IMkvWriter* writer) const; + + void set_bit_depth(uint64_t bit_depth) { bit_depth_ = bit_depth; } + uint64_t bit_depth() const { return bit_depth_; } + void set_channels(uint64_t channels) { channels_ = channels; } + uint64_t channels() const { return channels_; } + void set_sample_rate(double sample_rate) { sample_rate_ = sample_rate; } + double sample_rate() const { return sample_rate_; } + + private: + // Audio track element names. + uint64_t bit_depth_; + uint64_t channels_; + double sample_rate_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(AudioTrack); +}; + +/////////////////////////////////////////////////////////////// +// Tracks element +class Tracks { + public: + // Audio and video type defined by the Matroska specs. + enum { kVideo = 0x1, kAudio = 0x2 }; + + static const char kOpusCodecId[]; + static const char kVorbisCodecId[]; + static const char kVp8CodecId[]; + static const char kVp9CodecId[]; + static const char kAv1CodecId[]; + + Tracks(); + ~Tracks(); + + // Adds a Track element to the Tracks object. |track| will be owned and + // deleted by the Tracks object. Returns true on success. |number| is the + // number to use for the track. |number| must be >= 0. If |number| == 0 + // then the muxer will decide on the track number. + bool AddTrack(Track* track, int32_t number); + + // Returns the track by index. Returns NULL if there is no track match. + const Track* GetTrackByIndex(uint32_t idx) const; + + // Search the Tracks and return the track that matches |tn|. Returns NULL + // if there is no track match. + Track* GetTrackByNumber(uint64_t track_number) const; + + // Returns true if the track number is an audio track. + bool TrackIsAudio(uint64_t track_number) const; + + // Returns true if the track number is a video track. + bool TrackIsVideo(uint64_t track_number) const; + + // Output the Tracks element to the writer. Returns true on success. + bool Write(IMkvWriter* writer) const; + + uint32_t track_entries_size() const { return track_entries_size_; } + + private: + // Track element list. + Track** track_entries_; + + // Number of Track elements added. + uint32_t track_entries_size_; + + // Whether or not Tracks element has already been written via IMkvWriter. + mutable bool wrote_tracks_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(Tracks); +}; + +/////////////////////////////////////////////////////////////// +// Chapter element +// +class Chapter { + public: + // Set the identifier for this chapter. (This corresponds to the + // Cue Identifier line in WebVTT.) + // TODO(matthewjheaney): the actual serialization of this item in + // MKV is pending. + bool set_id(const char* id); + + // Converts the nanosecond start and stop times of this chapter to + // their corresponding timecode values, and stores them that way. + void set_time(const Segment& segment, uint64_t start_time_ns, + uint64_t end_time_ns); + + // Sets the uid for this chapter. Primarily used to enable + // deterministic output from the muxer. + void set_uid(const uint64_t uid) { uid_ = uid; } + + // Add a title string to this chapter, per the semantics described + // here: + // http://www.matroska.org/technical/specs/index.html + // + // The title ("chapter string") is a UTF-8 string. + // + // The language has ISO 639-2 representation, described here: + // http://www.loc.gov/standards/iso639-2/englangn.html + // http://www.loc.gov/standards/iso639-2/php/English_list.php + // If you specify NULL as the language value, this implies + // English ("eng"). + // + // The country value corresponds to the codes listed here: + // http://www.iana.org/domains/root/db/ + // + // The function returns false if the string could not be allocated. + bool add_string(const char* title, const char* language, const char* country); + + private: + friend class Chapters; + + // For storage of chapter titles that differ by language. + class Display { + public: + // Establish representation invariant for new Display object. + void Init(); + + // Reclaim resources, in anticipation of destruction. + void Clear(); + + // Copies the title to the |title_| member. Returns false on + // error. + bool set_title(const char* title); + + // Copies the language to the |language_| member. Returns false + // on error. + bool set_language(const char* language); + + // Copies the country to the |country_| member. Returns false on + // error. + bool set_country(const char* country); + + // If |writer| is non-NULL, serialize the Display sub-element of + // the Atom into the stream. Returns the Display element size on + // success, 0 if error. + uint64_t WriteDisplay(IMkvWriter* writer) const; + + private: + char* title_; + char* language_; + char* country_; + }; + + Chapter(); + ~Chapter(); + + // Establish the representation invariant for a newly-created + // Chapter object. The |seed| parameter is used to create the UID + // for this chapter atom. + void Init(unsigned int* seed); + + // Copies this Chapter object to a different one. This is used when + // expanding a plain array of Chapter objects (see Chapters). + void ShallowCopy(Chapter* dst) const; + + // Reclaim resources used by this Chapter object, pending its + // destruction. + void Clear(); + + // If there is no storage remaining on the |displays_| array for a + // new display object, creates a new, longer array and copies the + // existing Display objects to the new array. Returns false if the + // array cannot be expanded. + bool ExpandDisplaysArray(); + + // If |writer| is non-NULL, serialize the Atom sub-element into the + // stream. Returns the total size of the element on success, 0 if + // error. + uint64_t WriteAtom(IMkvWriter* writer) const; + + // The string identifier for this chapter (corresponds to WebVTT cue + // identifier). + char* id_; + + // Start timecode of the chapter. + uint64_t start_timecode_; + + // Stop timecode of the chapter. + uint64_t end_timecode_; + + // The binary identifier for this chapter. + uint64_t uid_; + + // The Atom element can contain multiple Display sub-elements, as + // the same logical title can be rendered in different languages. + Display* displays_; + + // The physical length (total size) of the |displays_| array. + int displays_size_; + + // The logical length (number of active elements) on the |displays_| + // array. + int displays_count_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(Chapter); +}; + +/////////////////////////////////////////////////////////////// +// Chapters element +// +class Chapters { + public: + Chapters(); + ~Chapters(); + + Chapter* AddChapter(unsigned int* seed); + + // Returns the number of chapters that have been added. + int Count() const; + + // Output the Chapters element to the writer. Returns true on success. + bool Write(IMkvWriter* writer) const; + + private: + // Expands the chapters_ array if there is not enough space to contain + // another chapter object. Returns true on success. + bool ExpandChaptersArray(); + + // If |writer| is non-NULL, serialize the Edition sub-element of the + // Chapters element into the stream. Returns the Edition element + // size on success, 0 if error. + uint64_t WriteEdition(IMkvWriter* writer) const; + + // Total length of the chapters_ array. + int chapters_size_; + + // Number of active chapters on the chapters_ array. + int chapters_count_; + + // Array for storage of chapter objects. + Chapter* chapters_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(Chapters); +}; + +/////////////////////////////////////////////////////////////// +// Tag element +// +class Tag { + public: + bool add_simple_tag(const char* tag_name, const char* tag_string); + + private: + // Tags calls Clear and the destructor of Tag + friend class Tags; + + // For storage of simple tags + class SimpleTag { + public: + // Establish representation invariant for new SimpleTag object. + void Init(); + + // Reclaim resources, in anticipation of destruction. + void Clear(); + + // Copies the title to the |tag_name_| member. Returns false on + // error. + bool set_tag_name(const char* tag_name); + + // Copies the language to the |tag_string_| member. Returns false + // on error. + bool set_tag_string(const char* tag_string); + + // If |writer| is non-NULL, serialize the SimpleTag sub-element of + // the Atom into the stream. Returns the SimpleTag element size on + // success, 0 if error. + uint64_t Write(IMkvWriter* writer) const; + + private: + char* tag_name_; + char* tag_string_; + }; + + Tag(); + ~Tag(); + + // Copies this Tag object to a different one. This is used when + // expanding a plain array of Tag objects (see Tags). + void ShallowCopy(Tag* dst) const; + + // Reclaim resources used by this Tag object, pending its + // destruction. + void Clear(); + + // If there is no storage remaining on the |simple_tags_| array for a + // new display object, creates a new, longer array and copies the + // existing SimpleTag objects to the new array. Returns false if the + // array cannot be expanded. + bool ExpandSimpleTagsArray(); + + // If |writer| is non-NULL, serialize the Tag sub-element into the + // stream. Returns the total size of the element on success, 0 if + // error. + uint64_t Write(IMkvWriter* writer) const; + + // The Atom element can contain multiple SimpleTag sub-elements + SimpleTag* simple_tags_; + + // The physical length (total size) of the |simple_tags_| array. + int simple_tags_size_; + + // The logical length (number of active elements) on the |simple_tags_| + // array. + int simple_tags_count_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(Tag); +}; + +/////////////////////////////////////////////////////////////// +// Tags element +// +class Tags { + public: + Tags(); + ~Tags(); + + Tag* AddTag(); + + // Returns the number of tags that have been added. + int Count() const; + + // Output the Tags element to the writer. Returns true on success. + bool Write(IMkvWriter* writer) const; + + private: + // Expands the tags_ array if there is not enough space to contain + // another tag object. Returns true on success. + bool ExpandTagsArray(); + + // Total length of the tags_ array. + int tags_size_; + + // Number of active tags on the tags_ array. + int tags_count_; + + // Array for storage of tag objects. + Tag* tags_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(Tags); +}; + +/////////////////////////////////////////////////////////////// +// Cluster element +// +// Notes: +// |Init| must be called before any other method in this class. +class Cluster { + public: + // |timecode| is the absolute timecode of the cluster. |cues_pos| is the + // position for the cluster within the segment that should be written in + // the cues element. |timecode_scale| is the timecode scale of the segment. + Cluster(uint64_t timecode, int64_t cues_pos, uint64_t timecode_scale, + bool write_last_frame_with_duration = false, + bool fixed_size_timecode = false); + ~Cluster(); + + bool Init(IMkvWriter* ptr_writer); + + // Adds a frame to be output in the file. The frame is written out through + // |writer_| if successful. Returns true on success. + bool AddFrame(const Frame* frame); + + // Adds a frame to be output in the file. The frame is written out through + // |writer_| if successful. Returns true on success. + // Inputs: + // data: Pointer to the data + // length: Length of the data + // track_number: Track to add the data to. Value returned by Add track + // functions. The range of allowed values is [1, 126]. + // timecode: Absolute (not relative to cluster) timestamp of the + // frame, expressed in timecode units. + // is_key: Flag telling whether or not this frame is a key frame. + bool AddFrame(const uint8_t* data, uint64_t length, uint64_t track_number, + uint64_t timecode, // timecode units (absolute) + bool is_key); + + // Adds a frame to be output in the file. The frame is written out through + // |writer_| if successful. Returns true on success. + // Inputs: + // data: Pointer to the data + // length: Length of the data + // additional: Pointer to the additional data + // additional_length: Length of the additional data + // add_id: Value of BlockAddID element + // track_number: Track to add the data to. Value returned by Add track + // functions. The range of allowed values is [1, 126]. + // abs_timecode: Absolute (not relative to cluster) timestamp of the + // frame, expressed in timecode units. + // is_key: Flag telling whether or not this frame is a key frame. + bool AddFrameWithAdditional(const uint8_t* data, uint64_t length, + const uint8_t* additional, + uint64_t additional_length, uint64_t add_id, + uint64_t track_number, uint64_t abs_timecode, + bool is_key); + + // Adds a frame to be output in the file. The frame is written out through + // |writer_| if successful. Returns true on success. + // Inputs: + // data: Pointer to the data. + // length: Length of the data. + // discard_padding: DiscardPadding element value. + // track_number: Track to add the data to. Value returned by Add track + // functions. The range of allowed values is [1, 126]. + // abs_timecode: Absolute (not relative to cluster) timestamp of the + // frame, expressed in timecode units. + // is_key: Flag telling whether or not this frame is a key frame. + bool AddFrameWithDiscardPadding(const uint8_t* data, uint64_t length, + int64_t discard_padding, + uint64_t track_number, uint64_t abs_timecode, + bool is_key); + + // Writes a frame of metadata to the output medium; returns true on + // success. + // Inputs: + // data: Pointer to the data + // length: Length of the data + // track_number: Track to add the data to. Value returned by Add track + // functions. The range of allowed values is [1, 126]. + // timecode: Absolute (not relative to cluster) timestamp of the + // metadata frame, expressed in timecode units. + // duration: Duration of metadata frame, in timecode units. + // + // The metadata frame is written as a block group, with a duration + // sub-element but no reference time sub-elements (indicating that + // it is considered a keyframe, per Matroska semantics). + bool AddMetadata(const uint8_t* data, uint64_t length, uint64_t track_number, + uint64_t timecode, uint64_t duration); + + // Increments the size of the cluster's data in bytes. + void AddPayloadSize(uint64_t size); + + // Closes the cluster so no more data can be written to it. Will update the + // cluster's size if |writer_| is seekable. Returns true on success. This + // variant of Finalize() fails when |write_last_frame_with_duration_| is set + // to true. + bool Finalize(); + + // Closes the cluster so no more data can be written to it. Will update the + // cluster's size if |writer_| is seekable. Returns true on success. + // Inputs: + // set_last_frame_duration: Boolean indicating whether or not the duration + // of the last frame should be set. If set to + // false, the |duration| value is ignored and + // |write_last_frame_with_duration_| will not be + // honored. + // duration: Duration of the Cluster in timecode scale. + bool Finalize(bool set_last_frame_duration, uint64_t duration); + + // Returns the size in bytes for the entire Cluster element. + uint64_t Size() const; + + // Given |abs_timecode|, calculates timecode relative to most recent timecode. + // Returns -1 on failure, or a relative timecode. + int64_t GetRelativeTimecode(int64_t abs_timecode) const; + + int64_t size_position() const { return size_position_; } + int32_t blocks_added() const { return blocks_added_; } + uint64_t payload_size() const { return payload_size_; } + int64_t position_for_cues() const { return position_for_cues_; } + uint64_t timecode() const { return timecode_; } + uint64_t timecode_scale() const { return timecode_scale_; } + void set_write_last_frame_with_duration(bool write_last_frame_with_duration) { + write_last_frame_with_duration_ = write_last_frame_with_duration; + } + bool write_last_frame_with_duration() const { + return write_last_frame_with_duration_; + } + + private: + // Iterator type for the |stored_frames_| map. + typedef std::map >::iterator FrameMapIterator; + + // Utility method that confirms that blocks can still be added, and that the + // cluster header has been written. Used by |DoWriteFrame*|. Returns true + // when successful. + bool PreWriteBlock(); + + // Utility method used by the |DoWriteFrame*| methods that handles the book + // keeping required after each block is written. + void PostWriteBlock(uint64_t element_size); + + // Does some verification and calls WriteFrame. + bool DoWriteFrame(const Frame* const frame); + + // Either holds back the given frame, or writes it out depending on whether or + // not |write_last_frame_with_duration_| is set. + bool QueueOrWriteFrame(const Frame* const frame); + + // Outputs the Cluster header to |writer_|. Returns true on success. + bool WriteClusterHeader(); + + // Number of blocks added to the cluster. + int32_t blocks_added_; + + // Flag telling if the cluster has been closed. + bool finalized_; + + // Flag indicating whether the cluster's timecode will always be written out + // using 8 bytes. + bool fixed_size_timecode_; + + // Flag telling if the cluster's header has been written. + bool header_written_; + + // The size of the cluster elements in bytes. + uint64_t payload_size_; + + // The file position used for cue points. + const int64_t position_for_cues_; + + // The file position of the cluster's size element. + int64_t size_position_; + + // The absolute timecode of the cluster. + const uint64_t timecode_; + + // The timecode scale of the Segment containing the cluster. + const uint64_t timecode_scale_; + + // Flag indicating whether the last frame of the cluster should be written as + // a Block with Duration. If set to true, then it will result in holding back + // of frames and the parameterized version of Finalize() must be called to + // finish writing the Cluster. + bool write_last_frame_with_duration_; + + // Map used to hold back frames, if required. Track number is the key. + std::map > stored_frames_; + + // Map from track number to the timestamp of the last block written for that + // track. + std::map last_block_timestamp_; + + // Pointer to the writer object. Not owned by this class. + IMkvWriter* writer_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(Cluster); +}; + +/////////////////////////////////////////////////////////////// +// SeekHead element +class SeekHead { + public: + SeekHead(); + ~SeekHead(); + + // TODO(fgalligan): Change this to reserve a certain size. Then check how + // big the seek entry to be added is as not every seek entry will be the + // maximum size it could be. + // Adds a seek entry to be written out when the element is finalized. |id| + // must be the coded mkv element id. |pos| is the file position of the + // element. Returns true on success. + bool AddSeekEntry(uint32_t id, uint64_t pos); + + // Writes out SeekHead and SeekEntry elements. Returns true on success. + bool Finalize(IMkvWriter* writer) const; + + // Returns the id of the Seek Entry at the given index. Returns -1 if index is + // out of range. + uint32_t GetId(int index) const; + + // Returns the position of the Seek Entry at the given index. Returns -1 if + // index is out of range. + uint64_t GetPosition(int index) const; + + // Sets the Seek Entry id and position at given index. + // Returns true on success. + bool SetSeekEntry(int index, uint32_t id, uint64_t position); + + // Reserves space by writing out a Void element which will be updated with + // a SeekHead element later. Returns true on success. + bool Write(IMkvWriter* writer); + + // We are going to put a cap on the number of Seek Entries. + const static int32_t kSeekEntryCount = 5; + + private: + // Returns the maximum size in bytes of one seek entry. + uint64_t MaxEntrySize() const; + + // Seek entry id element list. + uint32_t seek_entry_id_[kSeekEntryCount]; + + // Seek entry pos element list. + uint64_t seek_entry_pos_[kSeekEntryCount]; + + // The file position of SeekHead element. + int64_t start_pos_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(SeekHead); +}; + +/////////////////////////////////////////////////////////////// +// Segment Information element +class SegmentInfo { + public: + SegmentInfo(); + ~SegmentInfo(); + + // Will update the duration if |duration_| is > 0.0. Returns true on success. + bool Finalize(IMkvWriter* writer) const; + + // Sets |muxing_app_| and |writing_app_|. + bool Init(); + + // Output the Segment Information element to the writer. Returns true on + // success. + bool Write(IMkvWriter* writer); + + void set_duration(double duration) { duration_ = duration; } + double duration() const { return duration_; } + void set_muxing_app(const char* app); + const char* muxing_app() const { return muxing_app_; } + void set_timecode_scale(uint64_t scale) { timecode_scale_ = scale; } + uint64_t timecode_scale() const { return timecode_scale_; } + void set_writing_app(const char* app); + const char* writing_app() const { return writing_app_; } + void set_date_utc(int64_t date_utc) { date_utc_ = date_utc; } + int64_t date_utc() const { return date_utc_; } + + private: + // Segment Information element names. + // Initially set to -1 to signify that a duration has not been set and should + // not be written out. + double duration_; + // Set to libwebm-%d.%d.%d.%d, major, minor, build, revision. + char* muxing_app_; + uint64_t timecode_scale_; + // Initially set to libwebm-%d.%d.%d.%d, major, minor, build, revision. + char* writing_app_; + // LLONG_MIN when DateUTC is not set. + int64_t date_utc_; + + // The file position of the duration element. + int64_t duration_pos_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(SegmentInfo); +}; + +/////////////////////////////////////////////////////////////// +// This class represents the main segment in a WebM file. Currently only +// supports one Segment element. +// +// Notes: +// |Init| must be called before any other method in this class. +class Segment { + public: + enum Mode { kLive = 0x1, kFile = 0x2 }; + + enum CuesPosition { + kAfterClusters = 0x0, // Position Cues after Clusters - Default + kBeforeClusters = 0x1 // Position Cues before Clusters + }; + + const static uint32_t kDefaultDocTypeVersion = 2; + const static uint64_t kDefaultMaxClusterDuration = 30000000000ULL; + + Segment(); + ~Segment(); + + // Initializes |SegmentInfo| and returns result. Always returns false when + // |ptr_writer| is NULL. + bool Init(IMkvWriter* ptr_writer); + + // Adds a generic track to the segment. Returns the newly-allocated + // track object (which is owned by the segment) on success, NULL on + // error. |number| is the number to use for the track. |number| + // must be >= 0. If |number| == 0 then the muxer will decide on the + // track number. + Track* AddTrack(int32_t number); + + // Adds a Vorbis audio track to the segment. Returns the number of the track + // on success, 0 on error. |number| is the number to use for the audio track. + // |number| must be >= 0. If |number| == 0 then the muxer will decide on + // the track number. + uint64_t AddAudioTrack(int32_t sample_rate, int32_t channels, int32_t number); + + // Adds an empty chapter to the chapters of this segment. Returns + // non-NULL on success. After adding the chapter, the caller should + // populate its fields via the Chapter member functions. + Chapter* AddChapter(); + + // Adds an empty tag to the tags of this segment. Returns + // non-NULL on success. After adding the tag, the caller should + // populate its fields via the Tag member functions. + Tag* AddTag(); + + // Adds a cue point to the Cues element. |timestamp| is the time in + // nanoseconds of the cue's time. |track| is the Track of the Cue. This + // function must be called after AddFrame to calculate the correct + // BlockNumber for the CuePoint. Returns true on success. + bool AddCuePoint(uint64_t timestamp, uint64_t track); + + // Adds a frame to be output in the file. Returns true on success. + // Inputs: + // data: Pointer to the data + // length: Length of the data + // track_number: Track to add the data to. Value returned by Add track + // functions. + // timestamp: Timestamp of the frame in nanoseconds from 0. + // is_key: Flag telling whether or not this frame is a key frame. + bool AddFrame(const uint8_t* data, uint64_t length, uint64_t track_number, + uint64_t timestamp_ns, bool is_key); + + // Writes a frame of metadata to the output medium; returns true on + // success. + // Inputs: + // data: Pointer to the data + // length: Length of the data + // track_number: Track to add the data to. Value returned by Add track + // functions. + // timecode: Absolute timestamp of the metadata frame, expressed + // in nanosecond units. + // duration: Duration of metadata frame, in nanosecond units. + // + // The metadata frame is written as a block group, with a duration + // sub-element but no reference time sub-elements (indicating that + // it is considered a keyframe, per Matroska semantics). + bool AddMetadata(const uint8_t* data, uint64_t length, uint64_t track_number, + uint64_t timestamp_ns, uint64_t duration_ns); + + // Writes a frame with additional data to the output medium; returns true on + // success. + // Inputs: + // data: Pointer to the data. + // length: Length of the data. + // additional: Pointer to additional data. + // additional_length: Length of additional data. + // add_id: Additional ID which identifies the type of additional data. + // track_number: Track to add the data to. Value returned by Add track + // functions. + // timestamp: Absolute timestamp of the frame, expressed in nanosecond + // units. + // is_key: Flag telling whether or not this frame is a key frame. + bool AddFrameWithAdditional(const uint8_t* data, uint64_t length, + const uint8_t* additional, + uint64_t additional_length, uint64_t add_id, + uint64_t track_number, uint64_t timestamp, + bool is_key); + + // Writes a frame with DiscardPadding to the output medium; returns true on + // success. + // Inputs: + // data: Pointer to the data. + // length: Length of the data. + // discard_padding: DiscardPadding element value. + // track_number: Track to add the data to. Value returned by Add track + // functions. + // timestamp: Absolute timestamp of the frame, expressed in nanosecond + // units. + // is_key: Flag telling whether or not this frame is a key frame. + bool AddFrameWithDiscardPadding(const uint8_t* data, uint64_t length, + int64_t discard_padding, + uint64_t track_number, uint64_t timestamp, + bool is_key); + + // Writes a Frame to the output medium. Chooses the correct way of writing + // the frame (Block vs SimpleBlock) based on the parameters passed. + // Inputs: + // frame: frame object + bool AddGenericFrame(const Frame* frame); + + // Adds a VP8 video track to the segment. Returns the number of the track on + // success, 0 on error. |number| is the number to use for the video track. + // |number| must be >= 0. If |number| == 0 then the muxer will decide on + // the track number. + uint64_t AddVideoTrack(int32_t width, int32_t height, int32_t number); + + // This function must be called after Finalize() if you need a copy of the + // output with Cues written before the Clusters. It will return false if the + // writer is not seekable of if chunking is set to true. + // Input parameters: + // reader - an IMkvReader object created with the same underlying file of the + // current writer object. Make sure to close the existing writer + // object before creating this so that all the data is properly + // flushed and available for reading. + // writer - an IMkvWriter object pointing to a *different* file than the one + // pointed by the current writer object. This file will contain the + // Cues element before the Clusters. + bool CopyAndMoveCuesBeforeClusters(mkvparser::IMkvReader* reader, + IMkvWriter* writer); + + // Sets which track to use for the Cues element. Must have added the track + // before calling this function. Returns true on success. |track_number| is + // returned by the Add track functions. + bool CuesTrack(uint64_t track_number); + + // This will force the muxer to create a new Cluster when the next frame is + // added. + void ForceNewClusterOnNextFrame(); + + // Writes out any frames that have not been written out. Finalizes the last + // cluster. May update the size and duration of the segment. May output the + // Cues element. May finalize the SeekHead element. Returns true on success. + bool Finalize(); + + // Returns the Cues object. + Cues* GetCues() { return &cues_; } + + // Returns the Segment Information object. + const SegmentInfo* GetSegmentInfo() const { return &segment_info_; } + SegmentInfo* GetSegmentInfo() { return &segment_info_; } + + // Search the Tracks and return the track that matches |track_number|. + // Returns NULL if there is no track match. + Track* GetTrackByNumber(uint64_t track_number) const; + + // Toggles whether to output a cues element. + void OutputCues(bool output_cues); + + // Toggles whether to write the last frame in each Cluster with Duration. + void AccurateClusterDuration(bool accurate_cluster_duration); + + // Toggles whether to write the Cluster Timecode using exactly 8 bytes. + void UseFixedSizeClusterTimecode(bool fixed_size_cluster_timecode); + + // Sets if the muxer will output files in chunks or not. |chunking| is a + // flag telling whether or not to turn on chunking. |filename| is the base + // filename for the chunk files. The header chunk file will be named + // |filename|.hdr and the data chunks will be named + // |filename|_XXXXXX.chk. Chunking implies that the muxer will be writing + // to files so the muxer will use the default MkvWriter class to control + // what data is written to what files. Returns true on success. + // TODO: Should we change the IMkvWriter Interface to add Open and Close? + // That will force the interface to be dependent on files. + bool SetChunking(bool chunking, const char* filename); + + bool chunking() const { return chunking_; } + uint64_t cues_track() const { return cues_track_; } + void set_max_cluster_duration(uint64_t max_cluster_duration) { + max_cluster_duration_ = max_cluster_duration; + } + uint64_t max_cluster_duration() const { return max_cluster_duration_; } + void set_max_cluster_size(uint64_t max_cluster_size) { + max_cluster_size_ = max_cluster_size; + } + uint64_t max_cluster_size() const { return max_cluster_size_; } + void set_mode(Mode mode) { mode_ = mode; } + Mode mode() const { return mode_; } + CuesPosition cues_position() const { return cues_position_; } + bool output_cues() const { return output_cues_; } + const SegmentInfo* segment_info() const { return &segment_info_; } + + private: + // Checks if header information has been output and initialized. If not it + // will output the Segment element and initialize the SeekHead elment and + // Cues elements. + bool CheckHeaderInfo(); + + // Sets |doc_type_version_| based on the current element requirements. + void UpdateDocTypeVersion(); + + // Sets |name| according to how many chunks have been written. |ext| is the + // file extension. |name| must be deleted by the calling app. Returns true + // on success. + bool UpdateChunkName(const char* ext, char** name) const; + + // Returns the maximum offset within the segment's payload. When chunking + // this function is needed to determine offsets of elements within the + // chunked files. Returns -1 on error. + int64_t MaxOffset(); + + // Adds the frame to our frame array. + bool QueueFrame(Frame* frame); + + // Output all frames that are queued. Returns -1 on error, otherwise + // it returns the number of frames written. + int WriteFramesAll(); + + // Output all frames that are queued that have an end time that is less + // then |timestamp|. Returns true on success and if there are no frames + // queued. + bool WriteFramesLessThan(uint64_t timestamp); + + // Outputs the segment header, Segment Information element, SeekHead element, + // and Tracks element to |writer_|. + bool WriteSegmentHeader(); + + // Given a frame with the specified timestamp (nanosecond units) and + // keyframe status, determine whether a new cluster should be + // created, before writing enqueued frames and the frame itself. The + // function returns one of the following values: + // -1 = error: an out-of-order frame was detected + // 0 = do not create a new cluster, and write frame to the existing cluster + // 1 = create a new cluster, and write frame to that new cluster + // 2 = create a new cluster, and re-run test + int TestFrame(uint64_t track_num, uint64_t timestamp_ns, bool key) const; + + // Create a new cluster, using the earlier of the first enqueued + // frame, or the indicated time. Returns true on success. + bool MakeNewCluster(uint64_t timestamp_ns); + + // Checks whether a new cluster needs to be created, and if so + // creates a new cluster. Returns false if creation of a new cluster + // was necessary but creation was not successful. + bool DoNewClusterProcessing(uint64_t track_num, uint64_t timestamp_ns, + bool key); + + // Adjusts Cue Point values (to place Cues before Clusters) so that they + // reflect the correct offsets. + void MoveCuesBeforeClusters(); + + // This function recursively computes the correct cluster offsets (this is + // done to move the Cues before Clusters). It recursively updates the change + // in size (which indicates a change in cluster offset) until no sizes change. + // Parameters: + // diff - indicates the difference in size of the Cues element that needs to + // accounted for. + // index - index in the list of Cues which is currently being adjusted. + // cue_size - sum of size of all the CuePoint elements. + void MoveCuesBeforeClustersHelper(uint64_t diff, int index, + uint64_t* cue_size); + + // Seeds the random number generator used to make UIDs. + unsigned int seed_; + + // WebM elements + Cues cues_; + SeekHead seek_head_; + SegmentInfo segment_info_; + Tracks tracks_; + Chapters chapters_; + Tags tags_; + + // Number of chunks written. + int chunk_count_; + + // Current chunk filename. + char* chunk_name_; + + // Default MkvWriter object created by this class used for writing clusters + // out in separate files. + MkvWriter* chunk_writer_cluster_; + + // Default MkvWriter object created by this class used for writing Cues + // element out to a file. + MkvWriter* chunk_writer_cues_; + + // Default MkvWriter object created by this class used for writing the + // Matroska header out to a file. + MkvWriter* chunk_writer_header_; + + // Flag telling whether or not the muxer is chunking output to multiple + // files. + bool chunking_; + + // Base filename for the chunked files. + char* chunking_base_name_; + + // File position offset where the Clusters end. + int64_t cluster_end_offset_; + + // List of clusters. + Cluster** cluster_list_; + + // Number of cluster pointers allocated in the cluster list. + int32_t cluster_list_capacity_; + + // Number of clusters in the cluster list. + int32_t cluster_list_size_; + + // Indicates whether Cues should be written before or after Clusters + CuesPosition cues_position_; + + // Track number that is associated with the cues element for this segment. + uint64_t cues_track_; + + // Tells the muxer to force a new cluster on the next Block. + bool force_new_cluster_; + + // List of stored audio frames. These variables are used to store frames so + // the muxer can follow the guideline "Audio blocks that contain the video + // key frame's timecode should be in the same cluster as the video key frame + // block." + Frame** frames_; + + // Number of frame pointers allocated in the frame list. + int32_t frames_capacity_; + + // Number of frames in the frame list. + int32_t frames_size_; + + // Flag telling if a video track has been added to the segment. + bool has_video_; + + // Flag telling if the segment's header has been written. + bool header_written_; + + // Duration of the last block in nanoseconds. + uint64_t last_block_duration_; + + // Last timestamp in nanoseconds added to a cluster. + uint64_t last_timestamp_; + + // Last timestamp in nanoseconds by track number added to a cluster. + uint64_t last_track_timestamp_[kMaxTrackNumber]; + + // Maximum time in nanoseconds for a cluster duration. This variable is a + // guideline and some clusters may have a longer duration. Default is 30 + // seconds. + uint64_t max_cluster_duration_; + + // Maximum size in bytes for a cluster. This variable is a guideline and + // some clusters may have a larger size. Default is 0 which signifies that + // the muxer will decide the size. + uint64_t max_cluster_size_; + + // The mode that segment is in. If set to |kLive| the writer must not + // seek backwards. + Mode mode_; + + // Flag telling the muxer that a new cue point should be added. + bool new_cuepoint_; + + // TODO(fgalligan): Should we add support for more than one Cues element? + // Flag whether or not the muxer should output a Cues element. + bool output_cues_; + + // Flag whether or not the last frame in each Cluster will have a Duration + // element in it. + bool accurate_cluster_duration_; + + // Flag whether or not to write the Cluster Timecode using exactly 8 bytes. + bool fixed_size_cluster_timecode_; + + // The size of the EBML header, used to validate the header if + // WriteEbmlHeader() is called more than once. + int32_t ebml_header_size_; + + // The file position of the segment's payload. + int64_t payload_pos_; + + // The file position of the element's size. + int64_t size_position_; + + // Current DocTypeVersion (|doc_type_version_|) and that written in + // WriteSegmentHeader(). + // WriteEbmlHeader() will be called from Finalize() if |doc_type_version_| + // differs from |doc_type_version_written_|. + uint32_t doc_type_version_; + uint32_t doc_type_version_written_; + + // Pointer to the writer objects. Not owned by this class. + IMkvWriter* writer_cluster_; + IMkvWriter* writer_cues_; + IMkvWriter* writer_header_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(Segment); +}; + +} // namespace mkvmuxer + +#endif // MKVMUXER_MKVMUXER_H_ diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxertypes.h b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxertypes.h new file mode 100644 index 0000000000..e5db121605 --- /dev/null +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxertypes.h @@ -0,0 +1,28 @@ +// Copyright (c) 2012 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. + +#ifndef MKVMUXER_MKVMUXERTYPES_H_ +#define MKVMUXER_MKVMUXERTYPES_H_ + +namespace mkvmuxer { +typedef unsigned char uint8; +typedef short int16; +typedef int int32; +typedef unsigned int uint32; +typedef long long int64; +typedef unsigned long long uint64; +} // namespace mkvmuxer + +// Copied from Chromium basictypes.h +// A macro to disallow the copy constructor and operator= functions +// This should be used in the private: declarations for a class +#define LIBWEBM_DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&); \ + void operator=(const TypeName&) + +#endif // MKVMUXER_MKVMUXERTYPES_HPP_ diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc new file mode 100644 index 0000000000..3562b8ab82 --- /dev/null +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc @@ -0,0 +1,650 @@ +// Copyright (c) 2012 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. + +#include "mkvmuxer/mkvmuxerutil.h" + +#ifdef __ANDROID__ +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "common/webmids.h" +#include "mkvmuxer/mkvmuxer.h" +#include "mkvmuxer/mkvwriter.h" + +namespace mkvmuxer { + +namespace { + +// Date elements are always 8 octets in size. +const int kDateElementSize = 8; + +uint64_t WriteBlock(IMkvWriter* writer, const Frame* const frame, + int64_t timecode, uint64_t timecode_scale) { + uint64_t block_additional_elem_size = 0; + uint64_t block_addid_elem_size = 0; + uint64_t block_more_payload_size = 0; + uint64_t block_more_elem_size = 0; + uint64_t block_additions_payload_size = 0; + uint64_t block_additions_elem_size = 0; + if (frame->additional()) { + block_additional_elem_size = + EbmlElementSize(libwebm::kMkvBlockAdditional, frame->additional(), + frame->additional_length()); + block_addid_elem_size = + EbmlElementSize(libwebm::kMkvBlockAddID, frame->add_id()); + + block_more_payload_size = + block_addid_elem_size + block_additional_elem_size; + block_more_elem_size = + EbmlMasterElementSize(libwebm::kMkvBlockMore, block_more_payload_size) + + block_more_payload_size; + block_additions_payload_size = block_more_elem_size; + block_additions_elem_size = + EbmlMasterElementSize(libwebm::kMkvBlockAdditions, + block_additions_payload_size) + + block_additions_payload_size; + } + + uint64_t discard_padding_elem_size = 0; + if (frame->discard_padding() != 0) { + discard_padding_elem_size = + EbmlElementSize(libwebm::kMkvDiscardPadding, frame->discard_padding()); + } + + const uint64_t reference_block_timestamp = + frame->reference_block_timestamp() / timecode_scale; + uint64_t reference_block_elem_size = 0; + if (!frame->is_key()) { + reference_block_elem_size = + EbmlElementSize(libwebm::kMkvReferenceBlock, reference_block_timestamp); + } + + const uint64_t duration = frame->duration() / timecode_scale; + uint64_t block_duration_elem_size = 0; + if (duration > 0) + block_duration_elem_size = + EbmlElementSize(libwebm::kMkvBlockDuration, duration); + + const uint64_t block_payload_size = 4 + frame->length(); + const uint64_t block_elem_size = + EbmlMasterElementSize(libwebm::kMkvBlock, block_payload_size) + + block_payload_size; + + const uint64_t block_group_payload_size = + block_elem_size + block_additions_elem_size + block_duration_elem_size + + discard_padding_elem_size + reference_block_elem_size; + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvBlockGroup, + block_group_payload_size)) { + return 0; + } + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvBlock, block_payload_size)) + return 0; + + if (WriteUInt(writer, frame->track_number())) + return 0; + + if (SerializeInt(writer, timecode, 2)) + return 0; + + // For a Block, flags is always 0. + if (SerializeInt(writer, 0, 1)) + return 0; + + if (writer->Write(frame->frame(), static_cast(frame->length()))) + return 0; + + if (frame->additional()) { + if (!WriteEbmlMasterElement(writer, libwebm::kMkvBlockAdditions, + block_additions_payload_size)) { + return 0; + } + + if (!WriteEbmlMasterElement(writer, libwebm::kMkvBlockMore, + block_more_payload_size)) + return 0; + + if (!WriteEbmlElement(writer, libwebm::kMkvBlockAddID, frame->add_id())) + return 0; + + if (!WriteEbmlElement(writer, libwebm::kMkvBlockAdditional, + frame->additional(), frame->additional_length())) { + return 0; + } + } + + if (frame->discard_padding() != 0 && + !WriteEbmlElement(writer, libwebm::kMkvDiscardPadding, + frame->discard_padding())) { + return false; + } + + if (!frame->is_key() && + !WriteEbmlElement(writer, libwebm::kMkvReferenceBlock, + reference_block_timestamp)) { + return false; + } + + if (duration > 0 && + !WriteEbmlElement(writer, libwebm::kMkvBlockDuration, duration)) { + return false; + } + return EbmlMasterElementSize(libwebm::kMkvBlockGroup, + block_group_payload_size) + + block_group_payload_size; +} + +uint64_t WriteSimpleBlock(IMkvWriter* writer, const Frame* const frame, + int64_t timecode) { + if (WriteID(writer, libwebm::kMkvSimpleBlock)) + return 0; + + const int32_t size = static_cast(frame->length()) + 4; + if (WriteUInt(writer, size)) + return 0; + + if (WriteUInt(writer, static_cast(frame->track_number()))) + return 0; + + if (SerializeInt(writer, timecode, 2)) + return 0; + + uint64_t flags = 0; + if (frame->is_key()) + flags |= 0x80; + + if (SerializeInt(writer, flags, 1)) + return 0; + + if (writer->Write(frame->frame(), static_cast(frame->length()))) + return 0; + + return static_cast(GetUIntSize(libwebm::kMkvSimpleBlock) + + GetCodedUIntSize(size) + 4 + frame->length()); +} + +} // namespace + +int32_t GetCodedUIntSize(uint64_t value) { + if (value < 0x000000000000007FULL) + return 1; + else if (value < 0x0000000000003FFFULL) + return 2; + else if (value < 0x00000000001FFFFFULL) + return 3; + else if (value < 0x000000000FFFFFFFULL) + return 4; + else if (value < 0x00000007FFFFFFFFULL) + return 5; + else if (value < 0x000003FFFFFFFFFFULL) + return 6; + else if (value < 0x0001FFFFFFFFFFFFULL) + return 7; + return 8; +} + +int32_t GetUIntSize(uint64_t value) { + if (value < 0x0000000000000100ULL) + return 1; + else if (value < 0x0000000000010000ULL) + return 2; + else if (value < 0x0000000001000000ULL) + return 3; + else if (value < 0x0000000100000000ULL) + return 4; + else if (value < 0x0000010000000000ULL) + return 5; + else if (value < 0x0001000000000000ULL) + return 6; + else if (value < 0x0100000000000000ULL) + return 7; + return 8; +} + +int32_t GetIntSize(int64_t value) { + // Doubling the requested value ensures positive values with their high bit + // set are written with 0-padding to avoid flipping the signedness. + const uint64_t v = (value < 0) ? value ^ -1LL : value; + return GetUIntSize(2 * v); +} + +uint64_t EbmlMasterElementSize(uint64_t type, uint64_t value) { + // Size of EBML ID + int32_t ebml_size = GetUIntSize(type); + + // Datasize + ebml_size += GetCodedUIntSize(value); + + return static_cast(ebml_size); +} + +uint64_t EbmlElementSize(uint64_t type, int64_t value) { + // Size of EBML ID + int32_t ebml_size = GetUIntSize(type); + + // Datasize + ebml_size += GetIntSize(value); + + // Size of Datasize + ebml_size++; + + return static_cast(ebml_size); +} + +uint64_t EbmlElementSize(uint64_t type, uint64_t value) { + return EbmlElementSize(type, value, 0); +} + +uint64_t EbmlElementSize(uint64_t type, uint64_t value, uint64_t fixed_size) { + // Size of EBML ID + uint64_t ebml_size = static_cast(GetUIntSize(type)); + + // Datasize + ebml_size += + (fixed_size > 0) ? fixed_size : static_cast(GetUIntSize(value)); + + // Size of Datasize + ebml_size++; + + return ebml_size; +} + +uint64_t EbmlElementSize(uint64_t type, float /* value */) { + // Size of EBML ID + uint64_t ebml_size = static_cast(GetUIntSize(type)); + + // Datasize + ebml_size += sizeof(float); + + // Size of Datasize + ebml_size++; + + return ebml_size; +} + +uint64_t EbmlElementSize(uint64_t type, const char* value) { + if (!value) + return 0; + + // Size of EBML ID + uint64_t ebml_size = static_cast(GetUIntSize(type)); + + // Datasize + ebml_size += strlen(value); + + // Size of Datasize + ebml_size++; + + return ebml_size; +} + +uint64_t EbmlElementSize(uint64_t type, const uint8_t* value, uint64_t size) { + if (!value) + return 0; + + // Size of EBML ID + uint64_t ebml_size = static_cast(GetUIntSize(type)); + + // Datasize + ebml_size += size; + + // Size of Datasize + ebml_size += GetCodedUIntSize(size); + + return ebml_size; +} + +uint64_t EbmlDateElementSize(uint64_t type) { + // Size of EBML ID + uint64_t ebml_size = static_cast(GetUIntSize(type)); + + // Datasize + ebml_size += kDateElementSize; + + // Size of Datasize + ebml_size++; + + return ebml_size; +} + +int32_t SerializeInt(IMkvWriter* writer, int64_t value, int32_t size) { + if (!writer || size < 1 || size > 8) + return -1; + + for (int32_t i = 1; i <= size; ++i) { + const int32_t byte_count = size - i; + const int32_t bit_count = byte_count * 8; + + const int64_t bb = value >> bit_count; + const uint8_t b = static_cast(bb); + + const int32_t status = writer->Write(&b, 1); + + if (status < 0) + return status; + } + + return 0; +} + +int32_t SerializeFloat(IMkvWriter* writer, float f) { + if (!writer) + return -1; + + assert(sizeof(uint32_t) == sizeof(float)); + // This union is merely used to avoid a reinterpret_cast from float& to + // uint32& which will result in violation of strict aliasing. + union U32 { + uint32_t u32; + float f; + } value; + value.f = f; + + for (int32_t i = 1; i <= 4; ++i) { + const int32_t byte_count = 4 - i; + const int32_t bit_count = byte_count * 8; + + const uint8_t byte = static_cast(value.u32 >> bit_count); + + const int32_t status = writer->Write(&byte, 1); + + if (status < 0) + return status; + } + + return 0; +} + +int32_t WriteUInt(IMkvWriter* writer, uint64_t value) { + if (!writer) + return -1; + + int32_t size = GetCodedUIntSize(value); + + return WriteUIntSize(writer, value, size); +} + +int32_t WriteUIntSize(IMkvWriter* writer, uint64_t value, int32_t size) { + if (!writer || size < 0 || size > 8) + return -1; + + if (size > 0) { + const uint64_t bit = 1LL << (size * 7); + + if (value > (bit - 2)) + return -1; + + value |= bit; + } else { + size = 1; + int64_t bit; + + for (;;) { + bit = 1LL << (size * 7); + const uint64_t max = bit - 2; + + if (value <= max) + break; + + ++size; + } + + if (size > 8) + return false; + + value |= bit; + } + + return SerializeInt(writer, value, size); +} + +int32_t WriteID(IMkvWriter* writer, uint64_t type) { + if (!writer) + return -1; + + writer->ElementStartNotify(type, writer->Position()); + + const int32_t size = GetUIntSize(type); + + return SerializeInt(writer, type, size); +} + +bool WriteEbmlMasterElement(IMkvWriter* writer, uint64_t type, uint64_t size) { + if (!writer) + return false; + + if (WriteID(writer, type)) + return false; + + if (WriteUInt(writer, size)) + return false; + + return true; +} + +bool WriteEbmlElement(IMkvWriter* writer, uint64_t type, uint64_t value) { + return WriteEbmlElement(writer, type, value, 0); +} + +bool WriteEbmlElement(IMkvWriter* writer, uint64_t type, uint64_t value, + uint64_t fixed_size) { + if (!writer) + return false; + + if (WriteID(writer, type)) + return false; + + uint64_t size = static_cast(GetUIntSize(value)); + if (fixed_size > 0) { + if (size > fixed_size) + return false; + size = fixed_size; + } + if (WriteUInt(writer, size)) + return false; + + if (SerializeInt(writer, value, static_cast(size))) + return false; + + return true; +} + +bool WriteEbmlElement(IMkvWriter* writer, uint64_t type, int64_t value) { + if (!writer) + return false; + + if (WriteID(writer, type)) + return 0; + + const uint64_t size = GetIntSize(value); + if (WriteUInt(writer, size)) + return false; + + if (SerializeInt(writer, value, static_cast(size))) + return false; + + return true; +} + +bool WriteEbmlElement(IMkvWriter* writer, uint64_t type, float value) { + if (!writer) + return false; + + if (WriteID(writer, type)) + return false; + + if (WriteUInt(writer, 4)) + return false; + + if (SerializeFloat(writer, value)) + return false; + + return true; +} + +bool WriteEbmlElement(IMkvWriter* writer, uint64_t type, const char* value) { + if (!writer || !value) + return false; + + if (WriteID(writer, type)) + return false; + + const uint64_t length = strlen(value); + if (WriteUInt(writer, length)) + return false; + + if (writer->Write(value, static_cast(length))) + return false; + + return true; +} + +bool WriteEbmlElement(IMkvWriter* writer, uint64_t type, const uint8_t* value, + uint64_t size) { + if (!writer || !value || size < 1) + return false; + + if (WriteID(writer, type)) + return false; + + if (WriteUInt(writer, size)) + return false; + + if (writer->Write(value, static_cast(size))) + return false; + + return true; +} + +bool WriteEbmlDateElement(IMkvWriter* writer, uint64_t type, int64_t value) { + if (!writer) + return false; + + if (WriteID(writer, type)) + return false; + + if (WriteUInt(writer, kDateElementSize)) + return false; + + if (SerializeInt(writer, value, kDateElementSize)) + return false; + + return true; +} + +uint64_t WriteFrame(IMkvWriter* writer, const Frame* const frame, + Cluster* cluster) { + if (!writer || !frame || !frame->IsValid() || !cluster || + !cluster->timecode_scale()) + return 0; + + // Technically the timecode for a block can be less than the + // timecode for the cluster itself (remember that block timecode + // is a signed, 16-bit integer). However, as a simplification we + // only permit non-negative cluster-relative timecodes for blocks. + const int64_t relative_timecode = cluster->GetRelativeTimecode( + frame->timestamp() / cluster->timecode_scale()); + if (relative_timecode < 0 || relative_timecode > kMaxBlockTimecode) + return 0; + + return frame->CanBeSimpleBlock() ? + WriteSimpleBlock(writer, frame, relative_timecode) : + WriteBlock(writer, frame, relative_timecode, + cluster->timecode_scale()); +} + +uint64_t WriteVoidElement(IMkvWriter* writer, uint64_t size) { + if (!writer) + return false; + + // Subtract one for the void ID and the coded size. + uint64_t void_entry_size = size - 1 - GetCodedUIntSize(size - 1); + uint64_t void_size = + EbmlMasterElementSize(libwebm::kMkvVoid, void_entry_size) + + void_entry_size; + + if (void_size != size) + return 0; + + const int64_t payload_position = writer->Position(); + if (payload_position < 0) + return 0; + + if (WriteID(writer, libwebm::kMkvVoid)) + return 0; + + if (WriteUInt(writer, void_entry_size)) + return 0; + + const uint8_t value = 0; + for (int32_t i = 0; i < static_cast(void_entry_size); ++i) { + if (writer->Write(&value, 1)) + return 0; + } + + const int64_t stop_position = writer->Position(); + if (stop_position < 0 || + stop_position - payload_position != static_cast(void_size)) + return 0; + + return void_size; +} + +void GetVersion(int32_t* major, int32_t* minor, int32_t* build, + int32_t* revision) { + *major = 0; + *minor = 2; + *build = 1; + *revision = 0; +} + +uint64_t MakeUID(unsigned int* seed) { + uint64_t uid = 0; + +#ifdef __MINGW32__ + srand(*seed); +#endif + + for (int i = 0; i < 7; ++i) { // avoid problems with 8-byte values + uid <<= 8; + +// TODO(fgalligan): Move random number generation to platform specific code. +#ifdef _MSC_VER + (void)seed; + const int32_t nn = rand(); +#elif __ANDROID__ + int32_t temp_num = 1; + int fd = open("/dev/urandom", O_RDONLY); + if (fd != -1) { + read(fd, &temp_num, sizeof(temp_num)); + close(fd); + } + const int32_t nn = temp_num; +#elif defined __MINGW32__ + const int32_t nn = rand(); +#else + const int32_t nn = rand_r(seed); +#endif + const int32_t n = 0xFF & (nn >> 4); // throw away low-order bits + + uid |= n; + } + + return uid; +} + +} // namespace mkvmuxer diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.h b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.h new file mode 100644 index 0000000000..0e21a2dcbe --- /dev/null +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.h @@ -0,0 +1,95 @@ +// Copyright (c) 2012 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +#ifndef MKVMUXER_MKVMUXERUTIL_H_ +#define MKVMUXER_MKVMUXERUTIL_H_ + +#include + +namespace mkvmuxer { +class Cluster; +class Frame; +class IMkvWriter; + +const uint64_t kEbmlUnknownValue = 0x01FFFFFFFFFFFFFFULL; +const int64_t kMaxBlockTimecode = 0x07FFFLL; + +// Writes out |value| in Big Endian order. Returns 0 on success. +int32_t SerializeInt(IMkvWriter* writer, int64_t value, int32_t size); + +// Returns the size in bytes of the element. +int32_t GetUIntSize(uint64_t value); +int32_t GetIntSize(int64_t value); +int32_t GetCodedUIntSize(uint64_t value); +uint64_t EbmlMasterElementSize(uint64_t type, uint64_t value); +uint64_t EbmlElementSize(uint64_t type, int64_t value); +uint64_t EbmlElementSize(uint64_t type, uint64_t value); +uint64_t EbmlElementSize(uint64_t type, float value); +uint64_t EbmlElementSize(uint64_t type, const char* value); +uint64_t EbmlElementSize(uint64_t type, const uint8_t* value, uint64_t size); +uint64_t EbmlDateElementSize(uint64_t type); + +// Returns the size in bytes of the element assuming that the element was +// written using |fixed_size| bytes. If |fixed_size| is set to zero, then it +// computes the necessary number of bytes based on |value|. +uint64_t EbmlElementSize(uint64_t type, uint64_t value, uint64_t fixed_size); + +// Creates an EBML coded number from |value| and writes it out. The size of +// the coded number is determined by the value of |value|. |value| must not +// be in a coded form. Returns 0 on success. +int32_t WriteUInt(IMkvWriter* writer, uint64_t value); + +// Creates an EBML coded number from |value| and writes it out. The size of +// the coded number is determined by the value of |size|. |value| must not +// be in a coded form. Returns 0 on success. +int32_t WriteUIntSize(IMkvWriter* writer, uint64_t value, int32_t size); + +// Output an Mkv master element. Returns true if the element was written. +bool WriteEbmlMasterElement(IMkvWriter* writer, uint64_t value, uint64_t size); + +// Outputs an Mkv ID, calls |IMkvWriter::ElementStartNotify|, and passes the +// ID to |SerializeInt|. Returns 0 on success. +int32_t WriteID(IMkvWriter* writer, uint64_t type); + +// Output an Mkv non-master element. Returns true if the element was written. +bool WriteEbmlElement(IMkvWriter* writer, uint64_t type, uint64_t value); +bool WriteEbmlElement(IMkvWriter* writer, uint64_t type, int64_t value); +bool WriteEbmlElement(IMkvWriter* writer, uint64_t type, float value); +bool WriteEbmlElement(IMkvWriter* writer, uint64_t type, const char* value); +bool WriteEbmlElement(IMkvWriter* writer, uint64_t type, const uint8_t* value, + uint64_t size); +bool WriteEbmlDateElement(IMkvWriter* writer, uint64_t type, int64_t value); + +// Output an Mkv non-master element using fixed size. The element will be +// written out using exactly |fixed_size| bytes. If |fixed_size| is set to zero +// then it computes the necessary number of bytes based on |value|. Returns true +// if the element was written. +bool WriteEbmlElement(IMkvWriter* writer, uint64_t type, uint64_t value, + uint64_t fixed_size); + +// Output a Mkv Frame. It decides the correct element to write (Block vs +// SimpleBlock) based on the parameters of the Frame. +uint64_t WriteFrame(IMkvWriter* writer, const Frame* const frame, + Cluster* cluster); + +// Output a void element. |size| must be the entire size in bytes that will be +// void. The function will calculate the size of the void header and subtract +// it from |size|. +uint64_t WriteVoidElement(IMkvWriter* writer, uint64_t size); + +// Returns the version number of the muxer in |major|, |minor|, |build|, +// and |revision|. +void GetVersion(int32_t* major, int32_t* minor, int32_t* build, + int32_t* revision); + +// Returns a random number to be used for UID, using |seed| to seed +// the random-number generator (see POSIX rand_r() for semantics). +uint64_t MakeUID(unsigned int* seed); + +} // namespace mkvmuxer + +#endif // MKVMUXER_MKVMUXERUTIL_H_ diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc b/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc new file mode 100644 index 0000000000..ca48e149c6 --- /dev/null +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc @@ -0,0 +1,88 @@ +// Copyright (c) 2012 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. + +#include "mkvmuxer/mkvwriter.h" + +#ifdef _MSC_VER +#include // for _SH_DENYWR +#endif + +namespace mkvmuxer { + +MkvWriter::MkvWriter() : file_(NULL), writer_owns_file_(true) {} + +MkvWriter::MkvWriter(FILE* fp) : file_(fp), writer_owns_file_(false) {} + +MkvWriter::~MkvWriter() { Close(); } + +int32 MkvWriter::Write(const void* buffer, uint32 length) { + if (!file_) + return -1; + + if (length == 0) + return 0; + + if (buffer == NULL) + return -1; + + const size_t bytes_written = fwrite(buffer, 1, length, file_); + + return (bytes_written == length) ? 0 : -1; +} + +bool MkvWriter::Open(const char* filename) { + if (filename == NULL) + return false; + + if (file_) + return false; + +#ifdef _MSC_VER + file_ = _fsopen(filename, "wb", _SH_DENYWR); +#else + file_ = fopen(filename, "wb"); +#endif + if (file_ == NULL) + return false; + return true; +} + +void MkvWriter::Close() { + if (file_ && writer_owns_file_) { + fclose(file_); + } + file_ = NULL; +} + +int64 MkvWriter::Position() const { + if (!file_) + return 0; + +#ifdef _MSC_VER + return _ftelli64(file_); +#else + return ftell(file_); +#endif +} + +int32 MkvWriter::Position(int64 position) { + if (!file_) + return -1; + +#ifdef _MSC_VER + return _fseeki64(file_, position, SEEK_SET); +#else + return fseek(file_, position, SEEK_SET); +#endif +} + +bool MkvWriter::Seekable() const { return true; } + +void MkvWriter::ElementStartNotify(uint64, int64) {} + +} // namespace mkvmuxer diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.h b/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.h new file mode 100644 index 0000000000..4227c63748 --- /dev/null +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.h @@ -0,0 +1,51 @@ +// Copyright (c) 2012 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. + +#ifndef MKVMUXER_MKVWRITER_H_ +#define MKVMUXER_MKVWRITER_H_ + +#include + +#include "mkvmuxer/mkvmuxer.h" +#include "mkvmuxer/mkvmuxertypes.h" + +namespace mkvmuxer { + +// Default implementation of the IMkvWriter interface on Windows. +class MkvWriter : public IMkvWriter { + public: + MkvWriter(); + explicit MkvWriter(FILE* fp); + virtual ~MkvWriter(); + + // IMkvWriter interface + virtual int64 Position() const; + virtual int32 Position(int64 position); + virtual bool Seekable() const; + virtual int32 Write(const void* buffer, uint32 length); + virtual void ElementStartNotify(uint64 element_id, int64 position); + + // Creates and opens a file for writing. |filename| is the name of the file + // to open. This function will overwrite the contents of |filename|. Returns + // true on success. + bool Open(const char* filename); + + // Closes an opened file. + void Close(); + + private: + // File handle to output file. + FILE* file_; + bool writer_owns_file_; + + LIBWEBM_DISALLOW_COPY_AND_ASSIGN(MkvWriter); +}; + +} // namespace mkvmuxer + +#endif // MKVMUXER_MKVWRITER_H_ diff --git a/third_party/aom/third_party/libwebm/mkvparser/mkvparser.cc b/third_party/aom/third_party/libwebm/mkvparser/mkvparser.cc new file mode 100644 index 0000000000..21801154d9 --- /dev/null +++ b/third_party/aom/third_party/libwebm/mkvparser/mkvparser.cc @@ -0,0 +1,7940 @@ +// Copyright (c) 2012 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +#include "mkvparser/mkvparser.h" + +#if defined(_MSC_VER) && _MSC_VER < 1800 +#include // _isnan() / _finite() +#define MSC_COMPAT +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "common/webmids.h" + +namespace mkvparser { +const float MasteringMetadata::kValueNotPresent = FLT_MAX; +const long long Colour::kValueNotPresent = LLONG_MAX; + +#ifdef MSC_COMPAT +inline bool isnan(double val) { return !!_isnan(val); } +inline bool isinf(double val) { return !_finite(val); } +#else +inline bool isnan(double val) { return std::isnan(val); } +inline bool isinf(double val) { return std::isinf(val); } +#endif // MSC_COMPAT + +IMkvReader::~IMkvReader() {} + +template +Type* SafeArrayAlloc(unsigned long long num_elements, + unsigned long long element_size) { + if (num_elements == 0 || element_size == 0) + return NULL; + + const size_t kMaxAllocSize = 0x80000000; // 2GiB + const unsigned long long num_bytes = num_elements * element_size; + if (element_size > (kMaxAllocSize / num_elements)) + return NULL; + if (num_bytes != static_cast(num_bytes)) + return NULL; + + return new (std::nothrow) Type[static_cast(num_bytes)]; +} + +void GetVersion(int& major, int& minor, int& build, int& revision) { + major = 1; + minor = 0; + build = 0; + revision = 30; +} + +long long ReadUInt(IMkvReader* pReader, long long pos, long& len) { + if (!pReader || pos < 0) + return E_FILE_FORMAT_INVALID; + + len = 1; + unsigned char b; + int status = pReader->Read(pos, 1, &b); + + if (status < 0) // error or underflow + return status; + + if (status > 0) // interpreted as "underflow" + return E_BUFFER_NOT_FULL; + + if (b == 0) // we can't handle u-int values larger than 8 bytes + return E_FILE_FORMAT_INVALID; + + unsigned char m = 0x80; + + while (!(b & m)) { + m >>= 1; + ++len; + } + + long long result = b & (~m); + ++pos; + + for (int i = 1; i < len; ++i) { + status = pReader->Read(pos, 1, &b); + + if (status < 0) { + len = 1; + return status; + } + + if (status > 0) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result <<= 8; + result |= b; + + ++pos; + } + + return result; +} + +// Reads an EBML ID and returns it. +// An ID must at least 1 byte long, cannot exceed 4, and its value must be +// greater than 0. +// See known EBML values and EBMLMaxIDLength: +// http://www.matroska.org/technical/specs/index.html +// Returns the ID, or a value less than 0 to report an error while reading the +// ID. +long long ReadID(IMkvReader* pReader, long long pos, long& len) { + if (pReader == NULL || pos < 0) + return E_FILE_FORMAT_INVALID; + + // Read the first byte. The length in bytes of the ID is determined by + // finding the first set bit in the first byte of the ID. + unsigned char temp_byte = 0; + int read_status = pReader->Read(pos, 1, &temp_byte); + + if (read_status < 0) + return E_FILE_FORMAT_INVALID; + else if (read_status > 0) // No data to read. + return E_BUFFER_NOT_FULL; + + if (temp_byte == 0) // ID length > 8 bytes; invalid file. + return E_FILE_FORMAT_INVALID; + + int bit_pos = 0; + const int kMaxIdLengthInBytes = 4; + const int kCheckByte = 0x80; + + // Find the first bit that's set. + bool found_bit = false; + for (; bit_pos < kMaxIdLengthInBytes; ++bit_pos) { + if ((kCheckByte >> bit_pos) & temp_byte) { + found_bit = true; + break; + } + } + + if (!found_bit) { + // The value is too large to be a valid ID. + return E_FILE_FORMAT_INVALID; + } + + // Read the remaining bytes of the ID (if any). + const int id_length = bit_pos + 1; + long long ebml_id = temp_byte; + for (int i = 1; i < id_length; ++i) { + ebml_id <<= 8; + read_status = pReader->Read(pos + i, 1, &temp_byte); + + if (read_status < 0) + return E_FILE_FORMAT_INVALID; + else if (read_status > 0) + return E_BUFFER_NOT_FULL; + + ebml_id |= temp_byte; + } + + len = id_length; + return ebml_id; +} + +long long GetUIntLength(IMkvReader* pReader, long long pos, long& len) { + if (!pReader || pos < 0) + return E_FILE_FORMAT_INVALID; + + long long total, available; + + int status = pReader->Length(&total, &available); + if (status < 0 || (total >= 0 && available > total)) + return E_FILE_FORMAT_INVALID; + + len = 1; + + if (pos >= available) + return pos; // too few bytes available + + unsigned char b; + + status = pReader->Read(pos, 1, &b); + + if (status != 0) + return status; + + if (b == 0) // we can't handle u-int values larger than 8 bytes + return E_FILE_FORMAT_INVALID; + + unsigned char m = 0x80; + + while (!(b & m)) { + m >>= 1; + ++len; + } + + return 0; // success +} + +// TODO(vigneshv): This function assumes that unsigned values never have their +// high bit set. +long long UnserializeUInt(IMkvReader* pReader, long long pos, long long size) { + if (!pReader || pos < 0 || (size <= 0) || (size > 8)) + return E_FILE_FORMAT_INVALID; + + long long result = 0; + + for (long long i = 0; i < size; ++i) { + unsigned char b; + + const long status = pReader->Read(pos, 1, &b); + + if (status < 0) + return status; + + result <<= 8; + result |= b; + + ++pos; + } + + return result; +} + +long UnserializeFloat(IMkvReader* pReader, long long pos, long long size_, + double& result) { + if (!pReader || pos < 0 || ((size_ != 4) && (size_ != 8))) + return E_FILE_FORMAT_INVALID; + + const long size = static_cast(size_); + + unsigned char buf[8]; + + const int status = pReader->Read(pos, size, buf); + + if (status < 0) // error + return status; + + if (size == 4) { + union { + float f; + unsigned long ff; + }; + + ff = 0; + + for (int i = 0;;) { + ff |= buf[i]; + + if (++i >= 4) + break; + + ff <<= 8; + } + + result = f; + } else { + union { + double d; + unsigned long long dd; + }; + + dd = 0; + + for (int i = 0;;) { + dd |= buf[i]; + + if (++i >= 8) + break; + + dd <<= 8; + } + + result = d; + } + + if (mkvparser::isinf(result) || mkvparser::isnan(result)) + return E_FILE_FORMAT_INVALID; + + return 0; +} + +long UnserializeInt(IMkvReader* pReader, long long pos, long long size, + long long& result_ref) { + if (!pReader || pos < 0 || size < 1 || size > 8) + return E_FILE_FORMAT_INVALID; + + signed char first_byte = 0; + const long status = pReader->Read(pos, 1, (unsigned char*)&first_byte); + + if (status < 0) + return status; + + unsigned long long result = first_byte; + ++pos; + + for (long i = 1; i < size; ++i) { + unsigned char b; + + const long status = pReader->Read(pos, 1, &b); + + if (status < 0) + return status; + + result <<= 8; + result |= b; + + ++pos; + } + + result_ref = static_cast(result); + return 0; +} + +long UnserializeString(IMkvReader* pReader, long long pos, long long size, + char*& str) { + delete[] str; + str = NULL; + + if (size >= LONG_MAX || size < 0) + return E_FILE_FORMAT_INVALID; + + // +1 for '\0' terminator + const long required_size = static_cast(size) + 1; + + str = SafeArrayAlloc(1, required_size); + if (str == NULL) + return E_FILE_FORMAT_INVALID; + + unsigned char* const buf = reinterpret_cast(str); + + const long status = pReader->Read(pos, static_cast(size), buf); + + if (status) { + delete[] str; + str = NULL; + + return status; + } + + str[required_size - 1] = '\0'; + return 0; +} + +long ParseElementHeader(IMkvReader* pReader, long long& pos, long long stop, + long long& id, long long& size) { + if (stop >= 0 && pos >= stop) + return E_FILE_FORMAT_INVALID; + + long len; + + id = ReadID(pReader, pos, len); + + if (id < 0) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume id + + if (stop >= 0 && pos >= stop) + return E_FILE_FORMAT_INVALID; + + size = ReadUInt(pReader, pos, len); + + if (size < 0 || len < 1 || len > 8) { + // Invalid: Negative payload size, negative or 0 length integer, or integer + // larger than 64 bits (libwebm cannot handle them). + return E_FILE_FORMAT_INVALID; + } + + // Avoid rolling over pos when very close to LLONG_MAX. + const unsigned long long rollover_check = + static_cast(pos) + len; + if (rollover_check > LLONG_MAX) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume length of size + + // pos now designates payload + + if (stop >= 0 && pos > stop) + return E_FILE_FORMAT_INVALID; + + return 0; // success +} + +bool Match(IMkvReader* pReader, long long& pos, unsigned long expected_id, + long long& val) { + if (!pReader || pos < 0) + return false; + + long long total = 0; + long long available = 0; + + const long status = pReader->Length(&total, &available); + if (status < 0 || (total >= 0 && available > total)) + return false; + + long len = 0; + + const long long id = ReadID(pReader, pos, len); + if (id < 0 || (available - pos) > len) + return false; + + if (static_cast(id) != expected_id) + return false; + + pos += len; // consume id + + const long long size = ReadUInt(pReader, pos, len); + if (size < 0 || size > 8 || len < 1 || len > 8 || (available - pos) > len) + return false; + + pos += len; // consume length of size of payload + + val = UnserializeUInt(pReader, pos, size); + if (val < 0) + return false; + + pos += size; // consume size of payload + + return true; +} + +bool Match(IMkvReader* pReader, long long& pos, unsigned long expected_id, + unsigned char*& buf, size_t& buflen) { + if (!pReader || pos < 0) + return false; + + long long total = 0; + long long available = 0; + + long status = pReader->Length(&total, &available); + if (status < 0 || (total >= 0 && available > total)) + return false; + + long len = 0; + const long long id = ReadID(pReader, pos, len); + if (id < 0 || (available - pos) > len) + return false; + + if (static_cast(id) != expected_id) + return false; + + pos += len; // consume id + + const long long size = ReadUInt(pReader, pos, len); + if (size < 0 || len <= 0 || len > 8 || (available - pos) > len) + return false; + + unsigned long long rollover_check = + static_cast(pos) + len; + if (rollover_check > LLONG_MAX) + return false; + + pos += len; // consume length of size of payload + + rollover_check = static_cast(pos) + size; + if (rollover_check > LLONG_MAX) + return false; + + if ((pos + size) > available) + return false; + + if (size >= LONG_MAX) + return false; + + const long buflen_ = static_cast(size); + + buf = SafeArrayAlloc(1, buflen_); + if (!buf) + return false; + + status = pReader->Read(pos, buflen_, buf); + if (status != 0) + return false; + + buflen = buflen_; + + pos += size; // consume size of payload + return true; +} + +EBMLHeader::EBMLHeader() : m_docType(NULL) { Init(); } + +EBMLHeader::~EBMLHeader() { delete[] m_docType; } + +void EBMLHeader::Init() { + m_version = 1; + m_readVersion = 1; + m_maxIdLength = 4; + m_maxSizeLength = 8; + + if (m_docType) { + delete[] m_docType; + m_docType = NULL; + } + + m_docTypeVersion = 1; + m_docTypeReadVersion = 1; +} + +long long EBMLHeader::Parse(IMkvReader* pReader, long long& pos) { + if (!pReader) + return E_FILE_FORMAT_INVALID; + + long long total, available; + + long status = pReader->Length(&total, &available); + + if (status < 0) // error + return status; + + pos = 0; + + // Scan until we find what looks like the first byte of the EBML header. + const long long kMaxScanBytes = (available >= 1024) ? 1024 : available; + const unsigned char kEbmlByte0 = 0x1A; + unsigned char scan_byte = 0; + + while (pos < kMaxScanBytes) { + status = pReader->Read(pos, 1, &scan_byte); + + if (status < 0) // error + return status; + else if (status > 0) + return E_BUFFER_NOT_FULL; + + if (scan_byte == kEbmlByte0) + break; + + ++pos; + } + + long len = 0; + const long long ebml_id = ReadID(pReader, pos, len); + + if (ebml_id == E_BUFFER_NOT_FULL) + return E_BUFFER_NOT_FULL; + + if (len != 4 || ebml_id != libwebm::kMkvEBML) + return E_FILE_FORMAT_INVALID; + + // Move read pos forward to the EBML header size field. + pos += 4; + + // Read length of size field. + long long result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return E_FILE_FORMAT_INVALID; + else if (result > 0) // need more data + return E_BUFFER_NOT_FULL; + + if (len < 1 || len > 8) + return E_FILE_FORMAT_INVALID; + + if ((total >= 0) && ((total - pos) < len)) + return E_FILE_FORMAT_INVALID; + + if ((available - pos) < len) + return pos + len; // try again later + + // Read the EBML header size. + result = ReadUInt(pReader, pos, len); + + if (result < 0) // error + return result; + + pos += len; // consume size field + + // pos now designates start of payload + + if ((total >= 0) && ((total - pos) < result)) + return E_FILE_FORMAT_INVALID; + + if ((available - pos) < result) + return pos + result; + + const long long end = pos + result; + + Init(); + + while (pos < end) { + long long id, size; + + status = ParseElementHeader(pReader, pos, end, id, size); + + if (status < 0) // error + return status; + + if (size == 0) + return E_FILE_FORMAT_INVALID; + + if (id == libwebm::kMkvEBMLVersion) { + m_version = UnserializeUInt(pReader, pos, size); + + if (m_version <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvEBMLReadVersion) { + m_readVersion = UnserializeUInt(pReader, pos, size); + + if (m_readVersion <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvEBMLMaxIDLength) { + m_maxIdLength = UnserializeUInt(pReader, pos, size); + + if (m_maxIdLength <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvEBMLMaxSizeLength) { + m_maxSizeLength = UnserializeUInt(pReader, pos, size); + + if (m_maxSizeLength <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvDocType) { + if (m_docType) + return E_FILE_FORMAT_INVALID; + + status = UnserializeString(pReader, pos, size, m_docType); + + if (status) // error + return status; + } else if (id == libwebm::kMkvDocTypeVersion) { + m_docTypeVersion = UnserializeUInt(pReader, pos, size); + + if (m_docTypeVersion <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvDocTypeReadVersion) { + m_docTypeReadVersion = UnserializeUInt(pReader, pos, size); + + if (m_docTypeReadVersion <= 0) + return E_FILE_FORMAT_INVALID; + } + + pos += size; + } + + if (pos != end) + return E_FILE_FORMAT_INVALID; + + // Make sure DocType, DocTypeReadVersion, and DocTypeVersion are valid. + if (m_docType == NULL || m_docTypeReadVersion <= 0 || m_docTypeVersion <= 0) + return E_FILE_FORMAT_INVALID; + + // Make sure EBMLMaxIDLength and EBMLMaxSizeLength are valid. + if (m_maxIdLength <= 0 || m_maxIdLength > 4 || m_maxSizeLength <= 0 || + m_maxSizeLength > 8) + return E_FILE_FORMAT_INVALID; + + return 0; +} + +Segment::Segment(IMkvReader* pReader, long long elem_start, + // long long elem_size, + long long start, long long size) + : m_pReader(pReader), + m_element_start(elem_start), + // m_element_size(elem_size), + m_start(start), + m_size(size), + m_pos(start), + m_pUnknownSize(0), + m_pSeekHead(NULL), + m_pInfo(NULL), + m_pTracks(NULL), + m_pCues(NULL), + m_pChapters(NULL), + m_pTags(NULL), + m_clusters(NULL), + m_clusterCount(0), + m_clusterPreloadCount(0), + m_clusterSize(0) {} + +Segment::~Segment() { + const long count = m_clusterCount + m_clusterPreloadCount; + + Cluster** i = m_clusters; + Cluster** j = m_clusters + count; + + while (i != j) { + Cluster* const p = *i++; + delete p; + } + + delete[] m_clusters; + + delete m_pTracks; + delete m_pInfo; + delete m_pCues; + delete m_pChapters; + delete m_pTags; + delete m_pSeekHead; +} + +long long Segment::CreateInstance(IMkvReader* pReader, long long pos, + Segment*& pSegment) { + if (pReader == NULL || pos < 0) + return E_PARSE_FAILED; + + pSegment = NULL; + + long long total, available; + + const long status = pReader->Length(&total, &available); + + if (status < 0) // error + return status; + + if (available < 0) + return -1; + + if ((total >= 0) && (available > total)) + return -1; + + // I would assume that in practice this loop would execute + // exactly once, but we allow for other elements (e.g. Void) + // to immediately follow the EBML header. This is fine for + // the source filter case (since the entire file is available), + // but in the splitter case over a network we should probably + // just give up early. We could for example decide only to + // execute this loop a maximum of, say, 10 times. + // TODO: + // There is an implied "give up early" by only parsing up + // to the available limit. We do do that, but only if the + // total file size is unknown. We could decide to always + // use what's available as our limit (irrespective of whether + // we happen to know the total file length). This would have + // as its sense "parse this much of the file before giving up", + // which a slightly different sense from "try to parse up to + // 10 EMBL elements before giving up". + + for (;;) { + if ((total >= 0) && (pos >= total)) + return E_FILE_FORMAT_INVALID; + + // Read ID + long len; + long long result = GetUIntLength(pReader, pos, len); + + if (result) // error, or too few available bytes + return result; + + if ((total >= 0) && ((pos + len) > total)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > available) + return pos + len; + + const long long idpos = pos; + const long long id = ReadID(pReader, pos, len); + + if (id < 0) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume ID + + // Read Size + + result = GetUIntLength(pReader, pos, len); + + if (result) // error, or too few available bytes + return result; + + if ((total >= 0) && ((pos + len) > total)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > available) + return pos + len; + + long long size = ReadUInt(pReader, pos, len); + + if (size < 0) // error + return size; + + pos += len; // consume length of size of element + + // Pos now points to start of payload + + // Handle "unknown size" for live streaming of webm files. + const long long unknown_size = (1LL << (7 * len)) - 1; + + if (id == libwebm::kMkvSegment) { + if (size == unknown_size) + size = -1; + + else if (total < 0) + size = -1; + + else if ((pos + size) > total) + size = -1; + + pSegment = new (std::nothrow) Segment(pReader, idpos, pos, size); + if (pSegment == NULL) + return E_PARSE_FAILED; + + return 0; // success + } + + if (size == unknown_size) + return E_FILE_FORMAT_INVALID; + + if ((total >= 0) && ((pos + size) > total)) + return E_FILE_FORMAT_INVALID; + + if ((pos + size) > available) + return pos + size; + + pos += size; // consume payload + } +} + +long long Segment::ParseHeaders() { + // Outermost (level 0) segment object has been constructed, + // and pos designates start of payload. We need to find the + // inner (level 1) elements. + long long total, available; + + const int status = m_pReader->Length(&total, &available); + + if (status < 0) // error + return status; + + if (total > 0 && available > total) + return E_FILE_FORMAT_INVALID; + + const long long segment_stop = (m_size < 0) ? -1 : m_start + m_size; + + if ((segment_stop >= 0 && total >= 0 && segment_stop > total) || + (segment_stop >= 0 && m_pos > segment_stop)) { + return E_FILE_FORMAT_INVALID; + } + + for (;;) { + if ((total >= 0) && (m_pos >= total)) + break; + + if ((segment_stop >= 0) && (m_pos >= segment_stop)) + break; + + long long pos = m_pos; + const long long element_start = pos; + + // Avoid rolling over pos when very close to LLONG_MAX. + unsigned long long rollover_check = pos + 1ULL; + if (rollover_check > LLONG_MAX) + return E_FILE_FORMAT_INVALID; + + if ((pos + 1) > available) + return (pos + 1); + + long len; + long long result = GetUIntLength(m_pReader, pos, len); + + if (result < 0) // error + return result; + + if (result > 0) { + // MkvReader doesn't have enough data to satisfy this read attempt. + return (pos + 1); + } + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > available) + return pos + len; + + const long long idpos = pos; + const long long id = ReadID(m_pReader, idpos, len); + + if (id < 0) + return E_FILE_FORMAT_INVALID; + + if (id == libwebm::kMkvCluster) + break; + + pos += len; // consume ID + + if ((pos + 1) > available) + return (pos + 1); + + // Read Size + result = GetUIntLength(m_pReader, pos, len); + + if (result < 0) // error + return result; + + if (result > 0) { + // MkvReader doesn't have enough data to satisfy this read attempt. + return (pos + 1); + } + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > available) + return pos + len; + + const long long size = ReadUInt(m_pReader, pos, len); + + if (size < 0 || len < 1 || len > 8) { + // TODO(tomfinegan): ReadUInt should return an error when len is < 1 or + // len > 8 is true instead of checking this _everywhere_. + return size; + } + + pos += len; // consume length of size of element + + // Avoid rolling over pos when very close to LLONG_MAX. + rollover_check = static_cast(pos) + size; + if (rollover_check > LLONG_MAX) + return E_FILE_FORMAT_INVALID; + + const long long element_size = size + pos - element_start; + + // Pos now points to start of payload + + if ((segment_stop >= 0) && ((pos + size) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + // We read EBML elements either in total or nothing at all. + + if ((pos + size) > available) + return pos + size; + + if (id == libwebm::kMkvInfo) { + if (m_pInfo) + return E_FILE_FORMAT_INVALID; + + m_pInfo = new (std::nothrow) + SegmentInfo(this, pos, size, element_start, element_size); + + if (m_pInfo == NULL) + return -1; + + const long status = m_pInfo->Parse(); + + if (status) + return status; + } else if (id == libwebm::kMkvTracks) { + if (m_pTracks) + return E_FILE_FORMAT_INVALID; + + m_pTracks = new (std::nothrow) + Tracks(this, pos, size, element_start, element_size); + + if (m_pTracks == NULL) + return -1; + + const long status = m_pTracks->Parse(); + + if (status) + return status; + } else if (id == libwebm::kMkvCues) { + if (m_pCues == NULL) { + m_pCues = new (std::nothrow) + Cues(this, pos, size, element_start, element_size); + + if (m_pCues == NULL) + return -1; + } + } else if (id == libwebm::kMkvSeekHead) { + if (m_pSeekHead == NULL) { + m_pSeekHead = new (std::nothrow) + SeekHead(this, pos, size, element_start, element_size); + + if (m_pSeekHead == NULL) + return -1; + + const long status = m_pSeekHead->Parse(); + + if (status) + return status; + } + } else if (id == libwebm::kMkvChapters) { + if (m_pChapters == NULL) { + m_pChapters = new (std::nothrow) + Chapters(this, pos, size, element_start, element_size); + + if (m_pChapters == NULL) + return -1; + + const long status = m_pChapters->Parse(); + + if (status) + return status; + } + } else if (id == libwebm::kMkvTags) { + if (m_pTags == NULL) { + m_pTags = new (std::nothrow) + Tags(this, pos, size, element_start, element_size); + + if (m_pTags == NULL) + return -1; + + const long status = m_pTags->Parse(); + + if (status) + return status; + } + } + + m_pos = pos + size; // consume payload + } + + if (segment_stop >= 0 && m_pos > segment_stop) + return E_FILE_FORMAT_INVALID; + + if (m_pInfo == NULL) // TODO: liberalize this behavior + return E_FILE_FORMAT_INVALID; + + if (m_pTracks == NULL) + return E_FILE_FORMAT_INVALID; + + return 0; // success +} + +long Segment::LoadCluster(long long& pos, long& len) { + for (;;) { + const long result = DoLoadCluster(pos, len); + + if (result <= 1) + return result; + } +} + +long Segment::DoLoadCluster(long long& pos, long& len) { + if (m_pos < 0) + return DoLoadClusterUnknownSize(pos, len); + + long long total, avail; + + long status = m_pReader->Length(&total, &avail); + + if (status < 0) // error + return status; + + if (total >= 0 && avail > total) + return E_FILE_FORMAT_INVALID; + + const long long segment_stop = (m_size < 0) ? -1 : m_start + m_size; + + long long cluster_off = -1; // offset relative to start of segment + long long cluster_size = -1; // size of cluster payload + + for (;;) { + if ((total >= 0) && (m_pos >= total)) + return 1; // no more clusters + + if ((segment_stop >= 0) && (m_pos >= segment_stop)) + return 1; // no more clusters + + pos = m_pos; + + // Read ID + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + long long result = GetUIntLength(m_pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) + return E_BUFFER_NOT_FULL; + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long idpos = pos; + const long long id = ReadID(m_pReader, idpos, len); + + if (id < 0) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume ID + + // Read Size + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result = GetUIntLength(m_pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) + return E_BUFFER_NOT_FULL; + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long size = ReadUInt(m_pReader, pos, len); + + if (size < 0) // error + return static_cast(size); + + pos += len; // consume length of size of element + + // pos now points to start of payload + + if (size == 0) { + // Missing element payload: move on. + m_pos = pos; + continue; + } + + const long long unknown_size = (1LL << (7 * len)) - 1; + + if ((segment_stop >= 0) && (size != unknown_size) && + ((pos + size) > segment_stop)) { + return E_FILE_FORMAT_INVALID; + } + + if (id == libwebm::kMkvCues) { + if (size == unknown_size) { + // Cues element of unknown size: Not supported. + return E_FILE_FORMAT_INVALID; + } + + if (m_pCues == NULL) { + const long long element_size = (pos - idpos) + size; + + m_pCues = new (std::nothrow) Cues(this, pos, size, idpos, element_size); + if (m_pCues == NULL) + return -1; + } + + m_pos = pos + size; // consume payload + continue; + } + + if (id != libwebm::kMkvCluster) { + // Besides the Segment, Libwebm allows only cluster elements of unknown + // size. Fail the parse upon encountering a non-cluster element reporting + // unknown size. + if (size == unknown_size) + return E_FILE_FORMAT_INVALID; + + m_pos = pos + size; // consume payload + continue; + } + + // We have a cluster. + + cluster_off = idpos - m_start; // relative pos + + if (size != unknown_size) + cluster_size = size; + + break; + } + + if (cluster_off < 0) { + // No cluster, die. + return E_FILE_FORMAT_INVALID; + } + + long long pos_; + long len_; + + status = Cluster::HasBlockEntries(this, cluster_off, pos_, len_); + + if (status < 0) { // error, or underflow + pos = pos_; + len = len_; + + return status; + } + + // status == 0 means "no block entries found" + // status > 0 means "found at least one block entry" + + // TODO: + // The issue here is that the segment increments its own + // pos ptr past the most recent cluster parsed, and then + // starts from there to parse the next cluster. If we + // don't know the size of the current cluster, then we + // must either parse its payload (as we do below), looking + // for the cluster (or cues) ID to terminate the parse. + // This isn't really what we want: rather, we really need + // a way to create the curr cluster object immediately. + // The pity is that cluster::parse can determine its own + // boundary, and we largely duplicate that same logic here. + // + // Maybe we need to get rid of our look-ahead preloading + // in source::parse??? + // + // As we're parsing the blocks in the curr cluster + //(in cluster::parse), we should have some way to signal + // to the segment that we have determined the boundary, + // so it can adjust its own segment::m_pos member. + // + // The problem is that we're asserting in asyncreadinit, + // because we adjust the pos down to the curr seek pos, + // and the resulting adjusted len is > 2GB. I'm suspicious + // that this is even correct, but even if it is, we can't + // be loading that much data in the cache anyway. + + const long idx = m_clusterCount; + + if (m_clusterPreloadCount > 0) { + if (idx >= m_clusterSize) + return E_FILE_FORMAT_INVALID; + + Cluster* const pCluster = m_clusters[idx]; + if (pCluster == NULL || pCluster->m_index >= 0) + return E_FILE_FORMAT_INVALID; + + const long long off = pCluster->GetPosition(); + if (off < 0) + return E_FILE_FORMAT_INVALID; + + if (off == cluster_off) { // preloaded already + if (status == 0) // no entries found + return E_FILE_FORMAT_INVALID; + + if (cluster_size >= 0) + pos += cluster_size; + else { + const long long element_size = pCluster->GetElementSize(); + + if (element_size <= 0) + return E_FILE_FORMAT_INVALID; // TODO: handle this case + + pos = pCluster->m_element_start + element_size; + } + + pCluster->m_index = idx; // move from preloaded to loaded + ++m_clusterCount; + --m_clusterPreloadCount; + + m_pos = pos; // consume payload + if (segment_stop >= 0 && m_pos > segment_stop) + return E_FILE_FORMAT_INVALID; + + return 0; // success + } + } + + if (status == 0) { // no entries found + if (cluster_size >= 0) + pos += cluster_size; + + if ((total >= 0) && (pos >= total)) { + m_pos = total; + return 1; // no more clusters + } + + if ((segment_stop >= 0) && (pos >= segment_stop)) { + m_pos = segment_stop; + return 1; // no more clusters + } + + m_pos = pos; + return 2; // try again + } + + // status > 0 means we have an entry + + Cluster* const pCluster = Cluster::Create(this, idx, cluster_off); + if (pCluster == NULL) + return -1; + + if (!AppendCluster(pCluster)) { + delete pCluster; + return -1; + } + + if (cluster_size >= 0) { + pos += cluster_size; + + m_pos = pos; + + if (segment_stop > 0 && m_pos > segment_stop) + return E_FILE_FORMAT_INVALID; + + return 0; + } + + m_pUnknownSize = pCluster; + m_pos = -pos; + + return 0; // partial success, since we have a new cluster + + // status == 0 means "no block entries found" + // pos designates start of payload + // m_pos has NOT been adjusted yet (in case we need to come back here) +} + +long Segment::DoLoadClusterUnknownSize(long long& pos, long& len) { + if (m_pos >= 0 || m_pUnknownSize == NULL) + return E_PARSE_FAILED; + + const long status = m_pUnknownSize->Parse(pos, len); + + if (status < 0) // error or underflow + return status; + + if (status == 0) // parsed a block + return 2; // continue parsing + + const long long start = m_pUnknownSize->m_element_start; + const long long size = m_pUnknownSize->GetElementSize(); + + if (size < 0) + return E_FILE_FORMAT_INVALID; + + pos = start + size; + m_pos = pos; + + m_pUnknownSize = 0; + + return 2; // continue parsing +} + +bool Segment::AppendCluster(Cluster* pCluster) { + if (pCluster == NULL || pCluster->m_index < 0) + return false; + + const long count = m_clusterCount + m_clusterPreloadCount; + + long& size = m_clusterSize; + const long idx = pCluster->m_index; + + if (size < count || idx != m_clusterCount) + return false; + + if (count >= size) { + const long n = (size <= 0) ? 2048 : 2 * size; + + Cluster** const qq = new (std::nothrow) Cluster*[n]; + if (qq == NULL) + return false; + + Cluster** q = qq; + Cluster** p = m_clusters; + Cluster** const pp = p + count; + + while (p != pp) + *q++ = *p++; + + delete[] m_clusters; + + m_clusters = qq; + size = n; + } + + if (m_clusterPreloadCount > 0) { + Cluster** const p = m_clusters + m_clusterCount; + if (*p == NULL || (*p)->m_index >= 0) + return false; + + Cluster** q = p + m_clusterPreloadCount; + if (q >= (m_clusters + size)) + return false; + + for (;;) { + Cluster** const qq = q - 1; + if ((*qq)->m_index >= 0) + return false; + + *q = *qq; + q = qq; + + if (q == p) + break; + } + } + + m_clusters[idx] = pCluster; + ++m_clusterCount; + return true; +} + +bool Segment::PreloadCluster(Cluster* pCluster, ptrdiff_t idx) { + if (pCluster == NULL || pCluster->m_index >= 0 || idx < m_clusterCount) + return false; + + const long count = m_clusterCount + m_clusterPreloadCount; + + long& size = m_clusterSize; + if (size < count) + return false; + + if (count >= size) { + const long n = (size <= 0) ? 2048 : 2 * size; + + Cluster** const qq = new (std::nothrow) Cluster*[n]; + if (qq == NULL) + return false; + Cluster** q = qq; + + Cluster** p = m_clusters; + Cluster** const pp = p + count; + + while (p != pp) + *q++ = *p++; + + delete[] m_clusters; + + m_clusters = qq; + size = n; + } + + if (m_clusters == NULL) + return false; + + Cluster** const p = m_clusters + idx; + + Cluster** q = m_clusters + count; + if (q < p || q >= (m_clusters + size)) + return false; + + while (q > p) { + Cluster** const qq = q - 1; + + if ((*qq)->m_index >= 0) + return false; + + *q = *qq; + q = qq; + } + + m_clusters[idx] = pCluster; + ++m_clusterPreloadCount; + return true; +} + +long Segment::Load() { + if (m_clusters != NULL || m_clusterSize != 0 || m_clusterCount != 0) + return E_PARSE_FAILED; + + // Outermost (level 0) segment object has been constructed, + // and pos designates start of payload. We need to find the + // inner (level 1) elements. + + const long long header_status = ParseHeaders(); + + if (header_status < 0) // error + return static_cast(header_status); + + if (header_status > 0) // underflow + return E_BUFFER_NOT_FULL; + + if (m_pInfo == NULL || m_pTracks == NULL) + return E_FILE_FORMAT_INVALID; + + for (;;) { + const long status = LoadCluster(); + + if (status < 0) // error + return status; + + if (status >= 1) // no more clusters + return 0; + } +} + +SeekHead::SeekHead(Segment* pSegment, long long start, long long size_, + long long element_start, long long element_size) + : m_pSegment(pSegment), + m_start(start), + m_size(size_), + m_element_start(element_start), + m_element_size(element_size), + m_entries(0), + m_entry_count(0), + m_void_elements(0), + m_void_element_count(0) {} + +SeekHead::~SeekHead() { + delete[] m_entries; + delete[] m_void_elements; +} + +long SeekHead::Parse() { + IMkvReader* const pReader = m_pSegment->m_pReader; + + long long pos = m_start; + const long long stop = m_start + m_size; + + // first count the seek head entries + + int entry_count = 0; + int void_element_count = 0; + + while (pos < stop) { + long long id, size; + + const long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) // error + return status; + + if (id == libwebm::kMkvSeek) + ++entry_count; + else if (id == libwebm::kMkvVoid) + ++void_element_count; + + pos += size; // consume payload + + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + + m_entries = new (std::nothrow) Entry[entry_count]; + + if (m_entries == NULL) + return -1; + + m_void_elements = new (std::nothrow) VoidElement[void_element_count]; + + if (m_void_elements == NULL) + return -1; + + // now parse the entries and void elements + + Entry* pEntry = m_entries; + VoidElement* pVoidElement = m_void_elements; + + pos = m_start; + + while (pos < stop) { + const long long idpos = pos; + + long long id, size; + + const long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) // error + return status; + + if (id == libwebm::kMkvSeek) { + if (ParseEntry(pReader, pos, size, pEntry)) { + Entry& e = *pEntry++; + + e.element_start = idpos; + e.element_size = (pos + size) - idpos; + } + } else if (id == libwebm::kMkvVoid) { + VoidElement& e = *pVoidElement++; + + e.element_start = idpos; + e.element_size = (pos + size) - idpos; + } + + pos += size; // consume payload + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + + ptrdiff_t count_ = ptrdiff_t(pEntry - m_entries); + assert(count_ >= 0); + assert(count_ <= entry_count); + + m_entry_count = static_cast(count_); + + count_ = ptrdiff_t(pVoidElement - m_void_elements); + assert(count_ >= 0); + assert(count_ <= void_element_count); + + m_void_element_count = static_cast(count_); + + return 0; +} + +int SeekHead::GetCount() const { return m_entry_count; } + +const SeekHead::Entry* SeekHead::GetEntry(int idx) const { + if (idx < 0) + return 0; + + if (idx >= m_entry_count) + return 0; + + return m_entries + idx; +} + +int SeekHead::GetVoidElementCount() const { return m_void_element_count; } + +const SeekHead::VoidElement* SeekHead::GetVoidElement(int idx) const { + if (idx < 0) + return 0; + + if (idx >= m_void_element_count) + return 0; + + return m_void_elements + idx; +} + +long Segment::ParseCues(long long off, long long& pos, long& len) { + if (m_pCues) + return 0; // success + + if (off < 0) + return -1; + + long long total, avail; + + const int status = m_pReader->Length(&total, &avail); + + if (status < 0) // error + return status; + + assert((total < 0) || (avail <= total)); + + pos = m_start + off; + + if ((total < 0) || (pos >= total)) + return 1; // don't bother parsing cues + + const long long element_start = pos; + const long long segment_stop = (m_size < 0) ? -1 : m_start + m_size; + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + long long result = GetUIntLength(m_pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // underflow (weird) + { + len = 1; + return E_BUFFER_NOT_FULL; + } + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long idpos = pos; + + const long long id = ReadID(m_pReader, idpos, len); + + if (id != libwebm::kMkvCues) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume ID + assert((segment_stop < 0) || (pos <= segment_stop)); + + // Read Size + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result = GetUIntLength(m_pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // underflow (weird) + { + len = 1; + return E_BUFFER_NOT_FULL; + } + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long size = ReadUInt(m_pReader, pos, len); + + if (size < 0) // error + return static_cast(size); + + if (size == 0) // weird, although technically not illegal + return 1; // done + + pos += len; // consume length of size of element + assert((segment_stop < 0) || (pos <= segment_stop)); + + // Pos now points to start of payload + + const long long element_stop = pos + size; + + if ((segment_stop >= 0) && (element_stop > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((total >= 0) && (element_stop > total)) + return 1; // don't bother parsing anymore + + len = static_cast(size); + + if (element_stop > avail) + return E_BUFFER_NOT_FULL; + + const long long element_size = element_stop - element_start; + + m_pCues = + new (std::nothrow) Cues(this, pos, size, element_start, element_size); + if (m_pCues == NULL) + return -1; + + return 0; // success +} + +bool SeekHead::ParseEntry(IMkvReader* pReader, long long start, long long size_, + Entry* pEntry) { + if (size_ <= 0) + return false; + + long long pos = start; + const long long stop = start + size_; + + long len; + + // parse the container for the level-1 element ID + + const long long seekIdId = ReadID(pReader, pos, len); + if (seekIdId < 0) + return false; + + if (seekIdId != libwebm::kMkvSeekID) + return false; + + if ((pos + len) > stop) + return false; + + pos += len; // consume SeekID id + + const long long seekIdSize = ReadUInt(pReader, pos, len); + + if (seekIdSize <= 0) + return false; + + if ((pos + len) > stop) + return false; + + pos += len; // consume size of field + + if ((pos + seekIdSize) > stop) + return false; + + // Note that the SeekId payload really is serialized + // as a "Matroska integer", not as a plain binary value. + // In fact, Matroska requires that ID values in the + // stream exactly match the binary representation as listed + // in the Matroska specification. + // + // This parser is more liberal, and permits IDs to have + // any width. (This could make the representation in the stream + // different from what's in the spec, but it doesn't matter here, + // since we always normalize "Matroska integer" values.) + + pEntry->id = ReadUInt(pReader, pos, len); // payload + + if (pEntry->id <= 0) + return false; + + if (len != seekIdSize) + return false; + + pos += seekIdSize; // consume SeekID payload + + const long long seekPosId = ReadID(pReader, pos, len); + + if (seekPosId != libwebm::kMkvSeekPosition) + return false; + + if ((pos + len) > stop) + return false; + + pos += len; // consume id + + const long long seekPosSize = ReadUInt(pReader, pos, len); + + if (seekPosSize <= 0) + return false; + + if ((pos + len) > stop) + return false; + + pos += len; // consume size + + if ((pos + seekPosSize) > stop) + return false; + + pEntry->pos = UnserializeUInt(pReader, pos, seekPosSize); + + if (pEntry->pos < 0) + return false; + + pos += seekPosSize; // consume payload + + if (pos != stop) + return false; + + return true; +} + +Cues::Cues(Segment* pSegment, long long start_, long long size_, + long long element_start, long long element_size) + : m_pSegment(pSegment), + m_start(start_), + m_size(size_), + m_element_start(element_start), + m_element_size(element_size), + m_cue_points(NULL), + m_count(0), + m_preload_count(0), + m_pos(start_) {} + +Cues::~Cues() { + const long n = m_count + m_preload_count; + + CuePoint** p = m_cue_points; + CuePoint** const q = p + n; + + while (p != q) { + CuePoint* const pCP = *p++; + assert(pCP); + + delete pCP; + } + + delete[] m_cue_points; +} + +long Cues::GetCount() const { + if (m_cue_points == NULL) + return -1; + + return m_count; // TODO: really ignore preload count? +} + +bool Cues::DoneParsing() const { + const long long stop = m_start + m_size; + return (m_pos >= stop); +} + +bool Cues::Init() const { + if (m_cue_points) + return true; + + if (m_count != 0 || m_preload_count != 0) + return false; + + IMkvReader* const pReader = m_pSegment->m_pReader; + + const long long stop = m_start + m_size; + long long pos = m_start; + + long cue_points_size = 0; + + while (pos < stop) { + const long long idpos = pos; + + long len; + + const long long id = ReadID(pReader, pos, len); + if (id < 0 || (pos + len) > stop) { + return false; + } + + pos += len; // consume ID + + const long long size = ReadUInt(pReader, pos, len); + if (size < 0 || (pos + len > stop)) { + return false; + } + + pos += len; // consume Size field + if (pos + size > stop) { + return false; + } + + if (id == libwebm::kMkvCuePoint) { + if (!PreloadCuePoint(cue_points_size, idpos)) + return false; + } + + pos += size; // skip payload + } + return true; +} + +bool Cues::PreloadCuePoint(long& cue_points_size, long long pos) const { + if (m_count != 0) + return false; + + if (m_preload_count >= cue_points_size) { + const long n = (cue_points_size <= 0) ? 2048 : 2 * cue_points_size; + + CuePoint** const qq = new (std::nothrow) CuePoint*[n]; + if (qq == NULL) + return false; + + CuePoint** q = qq; // beginning of target + + CuePoint** p = m_cue_points; // beginning of source + CuePoint** const pp = p + m_preload_count; // end of source + + while (p != pp) + *q++ = *p++; + + delete[] m_cue_points; + + m_cue_points = qq; + cue_points_size = n; + } + + CuePoint* const pCP = new (std::nothrow) CuePoint(m_preload_count, pos); + if (pCP == NULL) + return false; + + m_cue_points[m_preload_count++] = pCP; + return true; +} + +bool Cues::LoadCuePoint() const { + const long long stop = m_start + m_size; + + if (m_pos >= stop) + return false; // nothing else to do + + if (!Init()) { + m_pos = stop; + return false; + } + + IMkvReader* const pReader = m_pSegment->m_pReader; + + while (m_pos < stop) { + const long long idpos = m_pos; + + long len; + + const long long id = ReadID(pReader, m_pos, len); + if (id < 0 || (m_pos + len) > stop) + return false; + + m_pos += len; // consume ID + + const long long size = ReadUInt(pReader, m_pos, len); + if (size < 0 || (m_pos + len) > stop) + return false; + + m_pos += len; // consume Size field + if ((m_pos + size) > stop) + return false; + + if (id != libwebm::kMkvCuePoint) { + m_pos += size; // consume payload + if (m_pos > stop) + return false; + + continue; + } + + if (m_preload_count < 1) + return false; + + CuePoint* const pCP = m_cue_points[m_count]; + if (!pCP || (pCP->GetTimeCode() < 0 && (-pCP->GetTimeCode() != idpos))) + return false; + + if (!pCP->Load(pReader)) { + m_pos = stop; + return false; + } + ++m_count; + --m_preload_count; + + m_pos += size; // consume payload + if (m_pos > stop) + return false; + + return true; // yes, we loaded a cue point + } + + return false; // no, we did not load a cue point +} + +bool Cues::Find(long long time_ns, const Track* pTrack, const CuePoint*& pCP, + const CuePoint::TrackPosition*& pTP) const { + if (time_ns < 0 || pTrack == NULL || m_cue_points == NULL || m_count == 0) + return false; + + CuePoint** const ii = m_cue_points; + CuePoint** i = ii; + + CuePoint** const jj = ii + m_count; + CuePoint** j = jj; + + pCP = *i; + if (pCP == NULL) + return false; + + if (time_ns <= pCP->GetTime(m_pSegment)) { + pTP = pCP->Find(pTrack); + return (pTP != NULL); + } + + while (i < j) { + // INVARIANT: + //[ii, i) <= time_ns + //[i, j) ? + //[j, jj) > time_ns + + CuePoint** const k = i + (j - i) / 2; + if (k >= jj) + return false; + + CuePoint* const pCP = *k; + if (pCP == NULL) + return false; + + const long long t = pCP->GetTime(m_pSegment); + + if (t <= time_ns) + i = k + 1; + else + j = k; + + if (i > j) + return false; + } + + if (i != j || i > jj || i <= ii) + return false; + + pCP = *--i; + + if (pCP == NULL || pCP->GetTime(m_pSegment) > time_ns) + return false; + + // TODO: here and elsewhere, it's probably not correct to search + // for the cue point with this time, and then search for a matching + // track. In principle, the matching track could be on some earlier + // cue point, and with our current algorithm, we'd miss it. To make + // this bullet-proof, we'd need to create a secondary structure, + // with a list of cue points that apply to a track, and then search + // that track-based structure for a matching cue point. + + pTP = pCP->Find(pTrack); + return (pTP != NULL); +} + +const CuePoint* Cues::GetFirst() const { + if (m_cue_points == NULL || m_count == 0) + return NULL; + + CuePoint* const* const pp = m_cue_points; + if (pp == NULL) + return NULL; + + CuePoint* const pCP = pp[0]; + if (pCP == NULL || pCP->GetTimeCode() < 0) + return NULL; + + return pCP; +} + +const CuePoint* Cues::GetLast() const { + if (m_cue_points == NULL || m_count <= 0) + return NULL; + + const long index = m_count - 1; + + CuePoint* const* const pp = m_cue_points; + if (pp == NULL) + return NULL; + + CuePoint* const pCP = pp[index]; + if (pCP == NULL || pCP->GetTimeCode() < 0) + return NULL; + + return pCP; +} + +const CuePoint* Cues::GetNext(const CuePoint* pCurr) const { + if (pCurr == NULL || pCurr->GetTimeCode() < 0 || m_cue_points == NULL || + m_count < 1) { + return NULL; + } + + long index = pCurr->m_index; + if (index >= m_count) + return NULL; + + CuePoint* const* const pp = m_cue_points; + if (pp == NULL || pp[index] != pCurr) + return NULL; + + ++index; + + if (index >= m_count) + return NULL; + + CuePoint* const pNext = pp[index]; + + if (pNext == NULL || pNext->GetTimeCode() < 0) + return NULL; + + return pNext; +} + +const BlockEntry* Cues::GetBlock(const CuePoint* pCP, + const CuePoint::TrackPosition* pTP) const { + if (pCP == NULL || pTP == NULL) + return NULL; + + return m_pSegment->GetBlock(*pCP, *pTP); +} + +const BlockEntry* Segment::GetBlock(const CuePoint& cp, + const CuePoint::TrackPosition& tp) { + Cluster** const ii = m_clusters; + Cluster** i = ii; + + const long count = m_clusterCount + m_clusterPreloadCount; + + Cluster** const jj = ii + count; + Cluster** j = jj; + + while (i < j) { + // INVARIANT: + //[ii, i) < pTP->m_pos + //[i, j) ? + //[j, jj) > pTP->m_pos + + Cluster** const k = i + (j - i) / 2; + assert(k < jj); + + Cluster* const pCluster = *k; + assert(pCluster); + + // const long long pos_ = pCluster->m_pos; + // assert(pos_); + // const long long pos = pos_ * ((pos_ < 0) ? -1 : 1); + + const long long pos = pCluster->GetPosition(); + assert(pos >= 0); + + if (pos < tp.m_pos) + i = k + 1; + else if (pos > tp.m_pos) + j = k; + else + return pCluster->GetEntry(cp, tp); + } + + assert(i == j); + // assert(Cluster::HasBlockEntries(this, tp.m_pos)); + + Cluster* const pCluster = Cluster::Create(this, -1, tp.m_pos); //, -1); + if (pCluster == NULL) + return NULL; + + const ptrdiff_t idx = i - m_clusters; + + if (!PreloadCluster(pCluster, idx)) { + delete pCluster; + return NULL; + } + assert(m_clusters); + assert(m_clusterPreloadCount > 0); + assert(m_clusters[idx] == pCluster); + + return pCluster->GetEntry(cp, tp); +} + +const Cluster* Segment::FindOrPreloadCluster(long long requested_pos) { + if (requested_pos < 0) + return 0; + + Cluster** const ii = m_clusters; + Cluster** i = ii; + + const long count = m_clusterCount + m_clusterPreloadCount; + + Cluster** const jj = ii + count; + Cluster** j = jj; + + while (i < j) { + // INVARIANT: + //[ii, i) < pTP->m_pos + //[i, j) ? + //[j, jj) > pTP->m_pos + + Cluster** const k = i + (j - i) / 2; + assert(k < jj); + + Cluster* const pCluster = *k; + assert(pCluster); + + // const long long pos_ = pCluster->m_pos; + // assert(pos_); + // const long long pos = pos_ * ((pos_ < 0) ? -1 : 1); + + const long long pos = pCluster->GetPosition(); + assert(pos >= 0); + + if (pos < requested_pos) + i = k + 1; + else if (pos > requested_pos) + j = k; + else + return pCluster; + } + + assert(i == j); + // assert(Cluster::HasBlockEntries(this, tp.m_pos)); + + Cluster* const pCluster = Cluster::Create(this, -1, requested_pos); + if (pCluster == NULL) + return NULL; + + const ptrdiff_t idx = i - m_clusters; + + if (!PreloadCluster(pCluster, idx)) { + delete pCluster; + return NULL; + } + assert(m_clusters); + assert(m_clusterPreloadCount > 0); + assert(m_clusters[idx] == pCluster); + + return pCluster; +} + +CuePoint::CuePoint(long idx, long long pos) + : m_element_start(0), + m_element_size(0), + m_index(idx), + m_timecode(-1 * pos), + m_track_positions(NULL), + m_track_positions_count(0) { + assert(pos > 0); +} + +CuePoint::~CuePoint() { delete[] m_track_positions; } + +bool CuePoint::Load(IMkvReader* pReader) { + // odbgstream os; + // os << "CuePoint::Load(begin): timecode=" << m_timecode << endl; + + if (m_timecode >= 0) // already loaded + return true; + + assert(m_track_positions == NULL); + assert(m_track_positions_count == 0); + + long long pos_ = -m_timecode; + const long long element_start = pos_; + + long long stop; + + { + long len; + + const long long id = ReadID(pReader, pos_, len); + if (id != libwebm::kMkvCuePoint) + return false; + + pos_ += len; // consume ID + + const long long size = ReadUInt(pReader, pos_, len); + assert(size >= 0); + + pos_ += len; // consume Size field + // pos_ now points to start of payload + + stop = pos_ + size; + } + + const long long element_size = stop - element_start; + + long long pos = pos_; + + // First count number of track positions + + while (pos < stop) { + long len; + + const long long id = ReadID(pReader, pos, len); + if ((id < 0) || (pos + len > stop)) { + return false; + } + + pos += len; // consume ID + + const long long size = ReadUInt(pReader, pos, len); + if ((size < 0) || (pos + len > stop)) { + return false; + } + + pos += len; // consume Size field + if ((pos + size) > stop) { + return false; + } + + if (id == libwebm::kMkvCueTime) + m_timecode = UnserializeUInt(pReader, pos, size); + + else if (id == libwebm::kMkvCueTrackPositions) + ++m_track_positions_count; + + pos += size; // consume payload + } + + if (m_timecode < 0 || m_track_positions_count <= 0) { + return false; + } + + // os << "CuePoint::Load(cont'd): idpos=" << idpos + // << " timecode=" << m_timecode + // << endl; + + m_track_positions = new (std::nothrow) TrackPosition[m_track_positions_count]; + if (m_track_positions == NULL) + return false; + + // Now parse track positions + + TrackPosition* p = m_track_positions; + pos = pos_; + + while (pos < stop) { + long len; + + const long long id = ReadID(pReader, pos, len); + if (id < 0 || (pos + len) > stop) + return false; + + pos += len; // consume ID + + const long long size = ReadUInt(pReader, pos, len); + assert(size >= 0); + assert((pos + len) <= stop); + + pos += len; // consume Size field + assert((pos + size) <= stop); + + if (id == libwebm::kMkvCueTrackPositions) { + TrackPosition& tp = *p++; + if (!tp.Parse(pReader, pos, size)) { + return false; + } + } + + pos += size; // consume payload + if (pos > stop) + return false; + } + + assert(size_t(p - m_track_positions) == m_track_positions_count); + + m_element_start = element_start; + m_element_size = element_size; + + return true; +} + +bool CuePoint::TrackPosition::Parse(IMkvReader* pReader, long long start_, + long long size_) { + const long long stop = start_ + size_; + long long pos = start_; + + m_track = -1; + m_pos = -1; + m_block = 1; // default + + while (pos < stop) { + long len; + + const long long id = ReadID(pReader, pos, len); + if ((id < 0) || ((pos + len) > stop)) { + return false; + } + + pos += len; // consume ID + + const long long size = ReadUInt(pReader, pos, len); + if ((size < 0) || ((pos + len) > stop)) { + return false; + } + + pos += len; // consume Size field + if ((pos + size) > stop) { + return false; + } + + if (id == libwebm::kMkvCueTrack) + m_track = UnserializeUInt(pReader, pos, size); + else if (id == libwebm::kMkvCueClusterPosition) + m_pos = UnserializeUInt(pReader, pos, size); + else if (id == libwebm::kMkvCueBlockNumber) + m_block = UnserializeUInt(pReader, pos, size); + + pos += size; // consume payload + } + + if ((m_pos < 0) || (m_track <= 0)) { + return false; + } + + return true; +} + +const CuePoint::TrackPosition* CuePoint::Find(const Track* pTrack) const { + assert(pTrack); + + const long long n = pTrack->GetNumber(); + + const TrackPosition* i = m_track_positions; + const TrackPosition* const j = i + m_track_positions_count; + + while (i != j) { + const TrackPosition& p = *i++; + + if (p.m_track == n) + return &p; + } + + return NULL; // no matching track number found +} + +long long CuePoint::GetTimeCode() const { return m_timecode; } + +long long CuePoint::GetTime(const Segment* pSegment) const { + assert(pSegment); + assert(m_timecode >= 0); + + const SegmentInfo* const pInfo = pSegment->GetInfo(); + assert(pInfo); + + const long long scale = pInfo->GetTimeCodeScale(); + assert(scale >= 1); + + const long long time = scale * m_timecode; + + return time; +} + +bool Segment::DoneParsing() const { + if (m_size < 0) { + long long total, avail; + + const int status = m_pReader->Length(&total, &avail); + + if (status < 0) // error + return true; // must assume done + + if (total < 0) + return false; // assume live stream + + return (m_pos >= total); + } + + const long long stop = m_start + m_size; + + return (m_pos >= stop); +} + +const Cluster* Segment::GetFirst() const { + if ((m_clusters == NULL) || (m_clusterCount <= 0)) + return &m_eos; + + Cluster* const pCluster = m_clusters[0]; + assert(pCluster); + + return pCluster; +} + +const Cluster* Segment::GetLast() const { + if ((m_clusters == NULL) || (m_clusterCount <= 0)) + return &m_eos; + + const long idx = m_clusterCount - 1; + + Cluster* const pCluster = m_clusters[idx]; + assert(pCluster); + + return pCluster; +} + +unsigned long Segment::GetCount() const { return m_clusterCount; } + +const Cluster* Segment::GetNext(const Cluster* pCurr) { + assert(pCurr); + assert(pCurr != &m_eos); + assert(m_clusters); + + long idx = pCurr->m_index; + + if (idx >= 0) { + assert(m_clusterCount > 0); + assert(idx < m_clusterCount); + assert(pCurr == m_clusters[idx]); + + ++idx; + + if (idx >= m_clusterCount) + return &m_eos; // caller will LoadCluster as desired + + Cluster* const pNext = m_clusters[idx]; + assert(pNext); + assert(pNext->m_index >= 0); + assert(pNext->m_index == idx); + + return pNext; + } + + assert(m_clusterPreloadCount > 0); + + long long pos = pCurr->m_element_start; + + assert(m_size >= 0); // TODO + const long long stop = m_start + m_size; // end of segment + + { + long len; + + long long result = GetUIntLength(m_pReader, pos, len); + assert(result == 0); + assert((pos + len) <= stop); // TODO + if (result != 0) + return NULL; + + const long long id = ReadID(m_pReader, pos, len); + if (id != libwebm::kMkvCluster) + return NULL; + + pos += len; // consume ID + + // Read Size + result = GetUIntLength(m_pReader, pos, len); + assert(result == 0); // TODO + assert((pos + len) <= stop); // TODO + + const long long size = ReadUInt(m_pReader, pos, len); + assert(size > 0); // TODO + // assert((pCurr->m_size <= 0) || (pCurr->m_size == size)); + + pos += len; // consume length of size of element + assert((pos + size) <= stop); // TODO + + // Pos now points to start of payload + + pos += size; // consume payload + } + + long long off_next = 0; + + while (pos < stop) { + long len; + + long long result = GetUIntLength(m_pReader, pos, len); + assert(result == 0); + assert((pos + len) <= stop); // TODO + if (result != 0) + return NULL; + + const long long idpos = pos; // pos of next (potential) cluster + + const long long id = ReadID(m_pReader, idpos, len); + if (id < 0) + return NULL; + + pos += len; // consume ID + + // Read Size + result = GetUIntLength(m_pReader, pos, len); + assert(result == 0); // TODO + assert((pos + len) <= stop); // TODO + + const long long size = ReadUInt(m_pReader, pos, len); + assert(size >= 0); // TODO + + pos += len; // consume length of size of element + assert((pos + size) <= stop); // TODO + + // Pos now points to start of payload + + if (size == 0) // weird + continue; + + if (id == libwebm::kMkvCluster) { + const long long off_next_ = idpos - m_start; + + long long pos_; + long len_; + + const long status = Cluster::HasBlockEntries(this, off_next_, pos_, len_); + + assert(status >= 0); + + if (status > 0) { + off_next = off_next_; + break; + } + } + + pos += size; // consume payload + } + + if (off_next <= 0) + return 0; + + Cluster** const ii = m_clusters + m_clusterCount; + Cluster** i = ii; + + Cluster** const jj = ii + m_clusterPreloadCount; + Cluster** j = jj; + + while (i < j) { + // INVARIANT: + //[0, i) < pos_next + //[i, j) ? + //[j, jj) > pos_next + + Cluster** const k = i + (j - i) / 2; + assert(k < jj); + + Cluster* const pNext = *k; + assert(pNext); + assert(pNext->m_index < 0); + + // const long long pos_ = pNext->m_pos; + // assert(pos_); + // pos = pos_ * ((pos_ < 0) ? -1 : 1); + + pos = pNext->GetPosition(); + + if (pos < off_next) + i = k + 1; + else if (pos > off_next) + j = k; + else + return pNext; + } + + assert(i == j); + + Cluster* const pNext = Cluster::Create(this, -1, off_next); + if (pNext == NULL) + return NULL; + + const ptrdiff_t idx_next = i - m_clusters; // insertion position + + if (!PreloadCluster(pNext, idx_next)) { + delete pNext; + return NULL; + } + assert(m_clusters); + assert(idx_next < m_clusterSize); + assert(m_clusters[idx_next] == pNext); + + return pNext; +} + +long Segment::ParseNext(const Cluster* pCurr, const Cluster*& pResult, + long long& pos, long& len) { + assert(pCurr); + assert(!pCurr->EOS()); + assert(m_clusters); + + pResult = 0; + + if (pCurr->m_index >= 0) { // loaded (not merely preloaded) + assert(m_clusters[pCurr->m_index] == pCurr); + + const long next_idx = pCurr->m_index + 1; + + if (next_idx < m_clusterCount) { + pResult = m_clusters[next_idx]; + return 0; // success + } + + // curr cluster is last among loaded + + const long result = LoadCluster(pos, len); + + if (result < 0) // error or underflow + return result; + + if (result > 0) // no more clusters + { + // pResult = &m_eos; + return 1; + } + + pResult = GetLast(); + return 0; // success + } + + assert(m_pos > 0); + + long long total, avail; + + long status = m_pReader->Length(&total, &avail); + + if (status < 0) // error + return status; + + assert((total < 0) || (avail <= total)); + + const long long segment_stop = (m_size < 0) ? -1 : m_start + m_size; + + // interrogate curr cluster + + pos = pCurr->m_element_start; + + if (pCurr->m_element_size >= 0) + pos += pCurr->m_element_size; + else { + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + long long result = GetUIntLength(m_pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // weird + return E_BUFFER_NOT_FULL; + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long id = ReadUInt(m_pReader, pos, len); + + if (id != libwebm::kMkvCluster) + return -1; + + pos += len; // consume ID + + // Read Size + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result = GetUIntLength(m_pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // weird + return E_BUFFER_NOT_FULL; + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long size = ReadUInt(m_pReader, pos, len); + + if (size < 0) // error + return static_cast(size); + + pos += len; // consume size field + + const long long unknown_size = (1LL << (7 * len)) - 1; + + if (size == unknown_size) // TODO: should never happen + return E_FILE_FORMAT_INVALID; // TODO: resolve this + + // assert((pCurr->m_size <= 0) || (pCurr->m_size == size)); + + if ((segment_stop >= 0) && ((pos + size) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + // Pos now points to start of payload + + pos += size; // consume payload (that is, the current cluster) + if (segment_stop >= 0 && pos > segment_stop) + return E_FILE_FORMAT_INVALID; + + // By consuming the payload, we are assuming that the curr + // cluster isn't interesting. That is, we don't bother checking + // whether the payload of the curr cluster is less than what + // happens to be available (obtained via IMkvReader::Length). + // Presumably the caller has already dispensed with the current + // cluster, and really does want the next cluster. + } + + // pos now points to just beyond the last fully-loaded cluster + + for (;;) { + const long status = DoParseNext(pResult, pos, len); + + if (status <= 1) + return status; + } +} + +long Segment::DoParseNext(const Cluster*& pResult, long long& pos, long& len) { + long long total, avail; + + long status = m_pReader->Length(&total, &avail); + + if (status < 0) // error + return status; + + assert((total < 0) || (avail <= total)); + + const long long segment_stop = (m_size < 0) ? -1 : m_start + m_size; + + // Parse next cluster. This is strictly a parsing activity. + // Creation of a new cluster object happens later, after the + // parsing is done. + + long long off_next = 0; + long long cluster_size = -1; + + for (;;) { + if ((total >= 0) && (pos >= total)) + return 1; // EOF + + if ((segment_stop >= 0) && (pos >= segment_stop)) + return 1; // EOF + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + long long result = GetUIntLength(m_pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // weird + return E_BUFFER_NOT_FULL; + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long idpos = pos; // absolute + const long long idoff = pos - m_start; // relative + + const long long id = ReadID(m_pReader, idpos, len); // absolute + + if (id < 0) // error + return static_cast(id); + + if (id == 0) // weird + return -1; // generic error + + pos += len; // consume ID + + // Read Size + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result = GetUIntLength(m_pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // weird + return E_BUFFER_NOT_FULL; + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long size = ReadUInt(m_pReader, pos, len); + + if (size < 0) // error + return static_cast(size); + + pos += len; // consume length of size of element + + // Pos now points to start of payload + + if (size == 0) // weird + continue; + + const long long unknown_size = (1LL << (7 * len)) - 1; + + if ((segment_stop >= 0) && (size != unknown_size) && + ((pos + size) > segment_stop)) { + return E_FILE_FORMAT_INVALID; + } + + if (id == libwebm::kMkvCues) { + if (size == unknown_size) + return E_FILE_FORMAT_INVALID; + + const long long element_stop = pos + size; + + if ((segment_stop >= 0) && (element_stop > segment_stop)) + return E_FILE_FORMAT_INVALID; + + const long long element_start = idpos; + const long long element_size = element_stop - element_start; + + if (m_pCues == NULL) { + m_pCues = new (std::nothrow) + Cues(this, pos, size, element_start, element_size); + if (m_pCues == NULL) + return false; + } + + pos += size; // consume payload + if (segment_stop >= 0 && pos > segment_stop) + return E_FILE_FORMAT_INVALID; + + continue; + } + + if (id != libwebm::kMkvCluster) { // not a Cluster ID + if (size == unknown_size) + return E_FILE_FORMAT_INVALID; + + pos += size; // consume payload + if (segment_stop >= 0 && pos > segment_stop) + return E_FILE_FORMAT_INVALID; + + continue; + } + + // We have a cluster. + off_next = idoff; + + if (size != unknown_size) + cluster_size = size; + + break; + } + + assert(off_next > 0); // have cluster + + // We have parsed the next cluster. + // We have not created a cluster object yet. What we need + // to do now is determine whether it has already be preloaded + //(in which case, an object for this cluster has already been + // created), and if not, create a new cluster object. + + Cluster** const ii = m_clusters + m_clusterCount; + Cluster** i = ii; + + Cluster** const jj = ii + m_clusterPreloadCount; + Cluster** j = jj; + + while (i < j) { + // INVARIANT: + //[0, i) < pos_next + //[i, j) ? + //[j, jj) > pos_next + + Cluster** const k = i + (j - i) / 2; + assert(k < jj); + + const Cluster* const pNext = *k; + assert(pNext); + assert(pNext->m_index < 0); + + pos = pNext->GetPosition(); + assert(pos >= 0); + + if (pos < off_next) + i = k + 1; + else if (pos > off_next) + j = k; + else { + pResult = pNext; + return 0; // success + } + } + + assert(i == j); + + long long pos_; + long len_; + + status = Cluster::HasBlockEntries(this, off_next, pos_, len_); + + if (status < 0) { // error or underflow + pos = pos_; + len = len_; + + return status; + } + + if (status > 0) { // means "found at least one block entry" + Cluster* const pNext = Cluster::Create(this, + -1, // preloaded + off_next); + if (pNext == NULL) + return -1; + + const ptrdiff_t idx_next = i - m_clusters; // insertion position + + if (!PreloadCluster(pNext, idx_next)) { + delete pNext; + return -1; + } + assert(m_clusters); + assert(idx_next < m_clusterSize); + assert(m_clusters[idx_next] == pNext); + + pResult = pNext; + return 0; // success + } + + // status == 0 means "no block entries found" + + if (cluster_size < 0) { // unknown size + const long long payload_pos = pos; // absolute pos of cluster payload + + for (;;) { // determine cluster size + if ((total >= 0) && (pos >= total)) + break; + + if ((segment_stop >= 0) && (pos >= segment_stop)) + break; // no more clusters + + // Read ID + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + long long result = GetUIntLength(m_pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // weird + return E_BUFFER_NOT_FULL; + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long idpos = pos; + const long long id = ReadID(m_pReader, idpos, len); + + if (id < 0) // error (or underflow) + return static_cast(id); + + // This is the distinguished set of ID's we use to determine + // that we have exhausted the sub-element's inside the cluster + // whose ID we parsed earlier. + + if (id == libwebm::kMkvCluster || id == libwebm::kMkvCues) + break; + + pos += len; // consume ID (of sub-element) + + // Read Size + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result = GetUIntLength(m_pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // weird + return E_BUFFER_NOT_FULL; + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long size = ReadUInt(m_pReader, pos, len); + + if (size < 0) // error + return static_cast(size); + + pos += len; // consume size field of element + + // pos now points to start of sub-element's payload + + if (size == 0) // weird + continue; + + const long long unknown_size = (1LL << (7 * len)) - 1; + + if (size == unknown_size) + return E_FILE_FORMAT_INVALID; // not allowed for sub-elements + + if ((segment_stop >= 0) && ((pos + size) > segment_stop)) // weird + return E_FILE_FORMAT_INVALID; + + pos += size; // consume payload of sub-element + if (segment_stop >= 0 && pos > segment_stop) + return E_FILE_FORMAT_INVALID; + } // determine cluster size + + cluster_size = pos - payload_pos; + assert(cluster_size >= 0); // TODO: handle cluster_size = 0 + + pos = payload_pos; // reset and re-parse original cluster + } + + pos += cluster_size; // consume payload + if (segment_stop >= 0 && pos > segment_stop) + return E_FILE_FORMAT_INVALID; + + return 2; // try to find a cluster that follows next +} + +const Cluster* Segment::FindCluster(long long time_ns) const { + if ((m_clusters == NULL) || (m_clusterCount <= 0)) + return &m_eos; + + { + Cluster* const pCluster = m_clusters[0]; + assert(pCluster); + assert(pCluster->m_index == 0); + + if (time_ns <= pCluster->GetTime()) + return pCluster; + } + + // Binary search of cluster array + + long i = 0; + long j = m_clusterCount; + + while (i < j) { + // INVARIANT: + //[0, i) <= time_ns + //[i, j) ? + //[j, m_clusterCount) > time_ns + + const long k = i + (j - i) / 2; + assert(k < m_clusterCount); + + Cluster* const pCluster = m_clusters[k]; + assert(pCluster); + assert(pCluster->m_index == k); + + const long long t = pCluster->GetTime(); + + if (t <= time_ns) + i = k + 1; + else + j = k; + + assert(i <= j); + } + + assert(i == j); + assert(i > 0); + assert(i <= m_clusterCount); + + const long k = i - 1; + + Cluster* const pCluster = m_clusters[k]; + assert(pCluster); + assert(pCluster->m_index == k); + assert(pCluster->GetTime() <= time_ns); + + return pCluster; +} + +const Tracks* Segment::GetTracks() const { return m_pTracks; } +const SegmentInfo* Segment::GetInfo() const { return m_pInfo; } +const Cues* Segment::GetCues() const { return m_pCues; } +const Chapters* Segment::GetChapters() const { return m_pChapters; } +const Tags* Segment::GetTags() const { return m_pTags; } +const SeekHead* Segment::GetSeekHead() const { return m_pSeekHead; } + +long long Segment::GetDuration() const { + assert(m_pInfo); + return m_pInfo->GetDuration(); +} + +Chapters::Chapters(Segment* pSegment, long long payload_start, + long long payload_size, long long element_start, + long long element_size) + : m_pSegment(pSegment), + m_start(payload_start), + m_size(payload_size), + m_element_start(element_start), + m_element_size(element_size), + m_editions(NULL), + m_editions_size(0), + m_editions_count(0) {} + +Chapters::~Chapters() { + while (m_editions_count > 0) { + Edition& e = m_editions[--m_editions_count]; + e.Clear(); + } + delete[] m_editions; +} + +long Chapters::Parse() { + IMkvReader* const pReader = m_pSegment->m_pReader; + + long long pos = m_start; // payload start + const long long stop = pos + m_size; // payload stop + + while (pos < stop) { + long long id, size; + + long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) // error + return status; + + if (size == 0) // weird + continue; + + if (id == libwebm::kMkvEditionEntry) { + status = ParseEdition(pos, size); + + if (status < 0) // error + return status; + } + + pos += size; + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + return 0; +} + +int Chapters::GetEditionCount() const { return m_editions_count; } + +const Chapters::Edition* Chapters::GetEdition(int idx) const { + if (idx < 0) + return NULL; + + if (idx >= m_editions_count) + return NULL; + + return m_editions + idx; +} + +bool Chapters::ExpandEditionsArray() { + if (m_editions_size > m_editions_count) + return true; // nothing else to do + + const int size = (m_editions_size == 0) ? 1 : 2 * m_editions_size; + + Edition* const editions = new (std::nothrow) Edition[size]; + + if (editions == NULL) + return false; + + for (int idx = 0; idx < m_editions_count; ++idx) { + m_editions[idx].ShallowCopy(editions[idx]); + } + + delete[] m_editions; + m_editions = editions; + + m_editions_size = size; + return true; +} + +long Chapters::ParseEdition(long long pos, long long size) { + if (!ExpandEditionsArray()) + return -1; + + Edition& e = m_editions[m_editions_count++]; + e.Init(); + + return e.Parse(m_pSegment->m_pReader, pos, size); +} + +Chapters::Edition::Edition() {} + +Chapters::Edition::~Edition() {} + +int Chapters::Edition::GetAtomCount() const { return m_atoms_count; } + +const Chapters::Atom* Chapters::Edition::GetAtom(int index) const { + if (index < 0) + return NULL; + + if (index >= m_atoms_count) + return NULL; + + return m_atoms + index; +} + +void Chapters::Edition::Init() { + m_atoms = NULL; + m_atoms_size = 0; + m_atoms_count = 0; +} + +void Chapters::Edition::ShallowCopy(Edition& rhs) const { + rhs.m_atoms = m_atoms; + rhs.m_atoms_size = m_atoms_size; + rhs.m_atoms_count = m_atoms_count; +} + +void Chapters::Edition::Clear() { + while (m_atoms_count > 0) { + Atom& a = m_atoms[--m_atoms_count]; + a.Clear(); + } + + delete[] m_atoms; + m_atoms = NULL; + + m_atoms_size = 0; +} + +long Chapters::Edition::Parse(IMkvReader* pReader, long long pos, + long long size) { + const long long stop = pos + size; + + while (pos < stop) { + long long id, size; + + long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) // error + return status; + + if (size == 0) + continue; + + if (id == libwebm::kMkvChapterAtom) { + status = ParseAtom(pReader, pos, size); + + if (status < 0) // error + return status; + } + + pos += size; + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + return 0; +} + +long Chapters::Edition::ParseAtom(IMkvReader* pReader, long long pos, + long long size) { + if (!ExpandAtomsArray()) + return -1; + + Atom& a = m_atoms[m_atoms_count++]; + a.Init(); + + return a.Parse(pReader, pos, size); +} + +bool Chapters::Edition::ExpandAtomsArray() { + if (m_atoms_size > m_atoms_count) + return true; // nothing else to do + + const int size = (m_atoms_size == 0) ? 1 : 2 * m_atoms_size; + + Atom* const atoms = new (std::nothrow) Atom[size]; + + if (atoms == NULL) + return false; + + for (int idx = 0; idx < m_atoms_count; ++idx) { + m_atoms[idx].ShallowCopy(atoms[idx]); + } + + delete[] m_atoms; + m_atoms = atoms; + + m_atoms_size = size; + return true; +} + +Chapters::Atom::Atom() {} + +Chapters::Atom::~Atom() {} + +unsigned long long Chapters::Atom::GetUID() const { return m_uid; } + +const char* Chapters::Atom::GetStringUID() const { return m_string_uid; } + +long long Chapters::Atom::GetStartTimecode() const { return m_start_timecode; } + +long long Chapters::Atom::GetStopTimecode() const { return m_stop_timecode; } + +long long Chapters::Atom::GetStartTime(const Chapters* pChapters) const { + return GetTime(pChapters, m_start_timecode); +} + +long long Chapters::Atom::GetStopTime(const Chapters* pChapters) const { + return GetTime(pChapters, m_stop_timecode); +} + +int Chapters::Atom::GetDisplayCount() const { return m_displays_count; } + +const Chapters::Display* Chapters::Atom::GetDisplay(int index) const { + if (index < 0) + return NULL; + + if (index >= m_displays_count) + return NULL; + + return m_displays + index; +} + +void Chapters::Atom::Init() { + m_string_uid = NULL; + m_uid = 0; + m_start_timecode = -1; + m_stop_timecode = -1; + + m_displays = NULL; + m_displays_size = 0; + m_displays_count = 0; +} + +void Chapters::Atom::ShallowCopy(Atom& rhs) const { + rhs.m_string_uid = m_string_uid; + rhs.m_uid = m_uid; + rhs.m_start_timecode = m_start_timecode; + rhs.m_stop_timecode = m_stop_timecode; + + rhs.m_displays = m_displays; + rhs.m_displays_size = m_displays_size; + rhs.m_displays_count = m_displays_count; +} + +void Chapters::Atom::Clear() { + delete[] m_string_uid; + m_string_uid = NULL; + + while (m_displays_count > 0) { + Display& d = m_displays[--m_displays_count]; + d.Clear(); + } + + delete[] m_displays; + m_displays = NULL; + + m_displays_size = 0; +} + +long Chapters::Atom::Parse(IMkvReader* pReader, long long pos, long long size) { + const long long stop = pos + size; + + while (pos < stop) { + long long id, size; + + long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) // error + return status; + + if (size == 0) // 0 length payload, skip. + continue; + + if (id == libwebm::kMkvChapterDisplay) { + status = ParseDisplay(pReader, pos, size); + + if (status < 0) // error + return status; + } else if (id == libwebm::kMkvChapterStringUID) { + status = UnserializeString(pReader, pos, size, m_string_uid); + + if (status < 0) // error + return status; + } else if (id == libwebm::kMkvChapterUID) { + long long val; + status = UnserializeInt(pReader, pos, size, val); + + if (status < 0) // error + return status; + + m_uid = static_cast(val); + } else if (id == libwebm::kMkvChapterTimeStart) { + const long long val = UnserializeUInt(pReader, pos, size); + + if (val < 0) // error + return static_cast(val); + + m_start_timecode = val; + } else if (id == libwebm::kMkvChapterTimeEnd) { + const long long val = UnserializeUInt(pReader, pos, size); + + if (val < 0) // error + return static_cast(val); + + m_stop_timecode = val; + } + + pos += size; + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + return 0; +} + +long long Chapters::Atom::GetTime(const Chapters* pChapters, + long long timecode) { + if (pChapters == NULL) + return -1; + + Segment* const pSegment = pChapters->m_pSegment; + + if (pSegment == NULL) // weird + return -1; + + const SegmentInfo* const pInfo = pSegment->GetInfo(); + + if (pInfo == NULL) + return -1; + + const long long timecode_scale = pInfo->GetTimeCodeScale(); + + if (timecode_scale < 1) // weird + return -1; + + if (timecode < 0) + return -1; + + const long long result = timecode_scale * timecode; + + return result; +} + +long Chapters::Atom::ParseDisplay(IMkvReader* pReader, long long pos, + long long size) { + if (!ExpandDisplaysArray()) + return -1; + + Display& d = m_displays[m_displays_count++]; + d.Init(); + + return d.Parse(pReader, pos, size); +} + +bool Chapters::Atom::ExpandDisplaysArray() { + if (m_displays_size > m_displays_count) + return true; // nothing else to do + + const int size = (m_displays_size == 0) ? 1 : 2 * m_displays_size; + + Display* const displays = new (std::nothrow) Display[size]; + + if (displays == NULL) + return false; + + for (int idx = 0; idx < m_displays_count; ++idx) { + m_displays[idx].ShallowCopy(displays[idx]); + } + + delete[] m_displays; + m_displays = displays; + + m_displays_size = size; + return true; +} + +Chapters::Display::Display() {} + +Chapters::Display::~Display() {} + +const char* Chapters::Display::GetString() const { return m_string; } + +const char* Chapters::Display::GetLanguage() const { return m_language; } + +const char* Chapters::Display::GetCountry() const { return m_country; } + +void Chapters::Display::Init() { + m_string = NULL; + m_language = NULL; + m_country = NULL; +} + +void Chapters::Display::ShallowCopy(Display& rhs) const { + rhs.m_string = m_string; + rhs.m_language = m_language; + rhs.m_country = m_country; +} + +void Chapters::Display::Clear() { + delete[] m_string; + m_string = NULL; + + delete[] m_language; + m_language = NULL; + + delete[] m_country; + m_country = NULL; +} + +long Chapters::Display::Parse(IMkvReader* pReader, long long pos, + long long size) { + const long long stop = pos + size; + + while (pos < stop) { + long long id, size; + + long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) // error + return status; + + if (size == 0) // No payload. + continue; + + if (id == libwebm::kMkvChapString) { + status = UnserializeString(pReader, pos, size, m_string); + + if (status) + return status; + } else if (id == libwebm::kMkvChapLanguage) { + status = UnserializeString(pReader, pos, size, m_language); + + if (status) + return status; + } else if (id == libwebm::kMkvChapCountry) { + status = UnserializeString(pReader, pos, size, m_country); + + if (status) + return status; + } + + pos += size; + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + return 0; +} + +Tags::Tags(Segment* pSegment, long long payload_start, long long payload_size, + long long element_start, long long element_size) + : m_pSegment(pSegment), + m_start(payload_start), + m_size(payload_size), + m_element_start(element_start), + m_element_size(element_size), + m_tags(NULL), + m_tags_size(0), + m_tags_count(0) {} + +Tags::~Tags() { + while (m_tags_count > 0) { + Tag& t = m_tags[--m_tags_count]; + t.Clear(); + } + delete[] m_tags; +} + +long Tags::Parse() { + IMkvReader* const pReader = m_pSegment->m_pReader; + + long long pos = m_start; // payload start + const long long stop = pos + m_size; // payload stop + + while (pos < stop) { + long long id, size; + + long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) + return status; + + if (size == 0) // 0 length tag, read another + continue; + + if (id == libwebm::kMkvTag) { + status = ParseTag(pos, size); + + if (status < 0) + return status; + } + + pos += size; + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + + return 0; +} + +int Tags::GetTagCount() const { return m_tags_count; } + +const Tags::Tag* Tags::GetTag(int idx) const { + if (idx < 0) + return NULL; + + if (idx >= m_tags_count) + return NULL; + + return m_tags + idx; +} + +bool Tags::ExpandTagsArray() { + if (m_tags_size > m_tags_count) + return true; // nothing else to do + + const int size = (m_tags_size == 0) ? 1 : 2 * m_tags_size; + + Tag* const tags = new (std::nothrow) Tag[size]; + + if (tags == NULL) + return false; + + for (int idx = 0; idx < m_tags_count; ++idx) { + m_tags[idx].ShallowCopy(tags[idx]); + } + + delete[] m_tags; + m_tags = tags; + + m_tags_size = size; + return true; +} + +long Tags::ParseTag(long long pos, long long size) { + if (!ExpandTagsArray()) + return -1; + + Tag& t = m_tags[m_tags_count++]; + t.Init(); + + return t.Parse(m_pSegment->m_pReader, pos, size); +} + +Tags::Tag::Tag() {} + +Tags::Tag::~Tag() {} + +int Tags::Tag::GetSimpleTagCount() const { return m_simple_tags_count; } + +const Tags::SimpleTag* Tags::Tag::GetSimpleTag(int index) const { + if (index < 0) + return NULL; + + if (index >= m_simple_tags_count) + return NULL; + + return m_simple_tags + index; +} + +void Tags::Tag::Init() { + m_simple_tags = NULL; + m_simple_tags_size = 0; + m_simple_tags_count = 0; +} + +void Tags::Tag::ShallowCopy(Tag& rhs) const { + rhs.m_simple_tags = m_simple_tags; + rhs.m_simple_tags_size = m_simple_tags_size; + rhs.m_simple_tags_count = m_simple_tags_count; +} + +void Tags::Tag::Clear() { + while (m_simple_tags_count > 0) { + SimpleTag& d = m_simple_tags[--m_simple_tags_count]; + d.Clear(); + } + + delete[] m_simple_tags; + m_simple_tags = NULL; + + m_simple_tags_size = 0; +} + +long Tags::Tag::Parse(IMkvReader* pReader, long long pos, long long size) { + const long long stop = pos + size; + + while (pos < stop) { + long long id, size; + + long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) + return status; + + if (size == 0) // 0 length tag, read another + continue; + + if (id == libwebm::kMkvSimpleTag) { + status = ParseSimpleTag(pReader, pos, size); + + if (status < 0) + return status; + } + + pos += size; + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + return 0; +} + +long Tags::Tag::ParseSimpleTag(IMkvReader* pReader, long long pos, + long long size) { + if (!ExpandSimpleTagsArray()) + return -1; + + SimpleTag& st = m_simple_tags[m_simple_tags_count++]; + st.Init(); + + return st.Parse(pReader, pos, size); +} + +bool Tags::Tag::ExpandSimpleTagsArray() { + if (m_simple_tags_size > m_simple_tags_count) + return true; // nothing else to do + + const int size = (m_simple_tags_size == 0) ? 1 : 2 * m_simple_tags_size; + + SimpleTag* const displays = new (std::nothrow) SimpleTag[size]; + + if (displays == NULL) + return false; + + for (int idx = 0; idx < m_simple_tags_count; ++idx) { + m_simple_tags[idx].ShallowCopy(displays[idx]); + } + + delete[] m_simple_tags; + m_simple_tags = displays; + + m_simple_tags_size = size; + return true; +} + +Tags::SimpleTag::SimpleTag() {} + +Tags::SimpleTag::~SimpleTag() {} + +const char* Tags::SimpleTag::GetTagName() const { return m_tag_name; } + +const char* Tags::SimpleTag::GetTagString() const { return m_tag_string; } + +void Tags::SimpleTag::Init() { + m_tag_name = NULL; + m_tag_string = NULL; +} + +void Tags::SimpleTag::ShallowCopy(SimpleTag& rhs) const { + rhs.m_tag_name = m_tag_name; + rhs.m_tag_string = m_tag_string; +} + +void Tags::SimpleTag::Clear() { + delete[] m_tag_name; + m_tag_name = NULL; + + delete[] m_tag_string; + m_tag_string = NULL; +} + +long Tags::SimpleTag::Parse(IMkvReader* pReader, long long pos, + long long size) { + const long long stop = pos + size; + + while (pos < stop) { + long long id, size; + + long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) // error + return status; + + if (size == 0) // weird + continue; + + if (id == libwebm::kMkvTagName) { + status = UnserializeString(pReader, pos, size, m_tag_name); + + if (status) + return status; + } else if (id == libwebm::kMkvTagString) { + status = UnserializeString(pReader, pos, size, m_tag_string); + + if (status) + return status; + } + + pos += size; + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + return 0; +} + +SegmentInfo::SegmentInfo(Segment* pSegment, long long start, long long size_, + long long element_start, long long element_size) + : m_pSegment(pSegment), + m_start(start), + m_size(size_), + m_element_start(element_start), + m_element_size(element_size), + m_pMuxingAppAsUTF8(NULL), + m_pWritingAppAsUTF8(NULL), + m_pTitleAsUTF8(NULL) {} + +SegmentInfo::~SegmentInfo() { + delete[] m_pMuxingAppAsUTF8; + m_pMuxingAppAsUTF8 = NULL; + + delete[] m_pWritingAppAsUTF8; + m_pWritingAppAsUTF8 = NULL; + + delete[] m_pTitleAsUTF8; + m_pTitleAsUTF8 = NULL; +} + +long SegmentInfo::Parse() { + assert(m_pMuxingAppAsUTF8 == NULL); + assert(m_pWritingAppAsUTF8 == NULL); + assert(m_pTitleAsUTF8 == NULL); + + IMkvReader* const pReader = m_pSegment->m_pReader; + + long long pos = m_start; + const long long stop = m_start + m_size; + + m_timecodeScale = 1000000; + m_duration = -1; + + while (pos < stop) { + long long id, size; + + const long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) // error + return status; + + if (id == libwebm::kMkvTimecodeScale) { + m_timecodeScale = UnserializeUInt(pReader, pos, size); + + if (m_timecodeScale <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvDuration) { + const long status = UnserializeFloat(pReader, pos, size, m_duration); + + if (status < 0) + return status; + + if (m_duration < 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvMuxingApp) { + const long status = + UnserializeString(pReader, pos, size, m_pMuxingAppAsUTF8); + + if (status) + return status; + } else if (id == libwebm::kMkvWritingApp) { + const long status = + UnserializeString(pReader, pos, size, m_pWritingAppAsUTF8); + + if (status) + return status; + } else if (id == libwebm::kMkvTitle) { + const long status = UnserializeString(pReader, pos, size, m_pTitleAsUTF8); + + if (status) + return status; + } + + pos += size; + + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + const double rollover_check = m_duration * m_timecodeScale; + if (rollover_check > LLONG_MAX) + return E_FILE_FORMAT_INVALID; + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + + return 0; +} + +long long SegmentInfo::GetTimeCodeScale() const { return m_timecodeScale; } + +long long SegmentInfo::GetDuration() const { + if (m_duration < 0) + return -1; + + assert(m_timecodeScale >= 1); + + const double dd = double(m_duration) * double(m_timecodeScale); + const long long d = static_cast(dd); + + return d; +} + +const char* SegmentInfo::GetMuxingAppAsUTF8() const { + return m_pMuxingAppAsUTF8; +} + +const char* SegmentInfo::GetWritingAppAsUTF8() const { + return m_pWritingAppAsUTF8; +} + +const char* SegmentInfo::GetTitleAsUTF8() const { return m_pTitleAsUTF8; } + +/////////////////////////////////////////////////////////////// +// ContentEncoding element +ContentEncoding::ContentCompression::ContentCompression() + : algo(0), settings(NULL), settings_len(0) {} + +ContentEncoding::ContentCompression::~ContentCompression() { + delete[] settings; +} + +ContentEncoding::ContentEncryption::ContentEncryption() + : algo(0), + key_id(NULL), + key_id_len(0), + signature(NULL), + signature_len(0), + sig_key_id(NULL), + sig_key_id_len(0), + sig_algo(0), + sig_hash_algo(0) {} + +ContentEncoding::ContentEncryption::~ContentEncryption() { + delete[] key_id; + delete[] signature; + delete[] sig_key_id; +} + +ContentEncoding::ContentEncoding() + : compression_entries_(NULL), + compression_entries_end_(NULL), + encryption_entries_(NULL), + encryption_entries_end_(NULL), + encoding_order_(0), + encoding_scope_(1), + encoding_type_(0) {} + +ContentEncoding::~ContentEncoding() { + ContentCompression** comp_i = compression_entries_; + ContentCompression** const comp_j = compression_entries_end_; + + while (comp_i != comp_j) { + ContentCompression* const comp = *comp_i++; + delete comp; + } + + delete[] compression_entries_; + + ContentEncryption** enc_i = encryption_entries_; + ContentEncryption** const enc_j = encryption_entries_end_; + + while (enc_i != enc_j) { + ContentEncryption* const enc = *enc_i++; + delete enc; + } + + delete[] encryption_entries_; +} + +const ContentEncoding::ContentCompression* + ContentEncoding::GetCompressionByIndex(unsigned long idx) const { + const ptrdiff_t count = compression_entries_end_ - compression_entries_; + assert(count >= 0); + + if (idx >= static_cast(count)) + return NULL; + + return compression_entries_[idx]; +} + +unsigned long ContentEncoding::GetCompressionCount() const { + const ptrdiff_t count = compression_entries_end_ - compression_entries_; + assert(count >= 0); + + return static_cast(count); +} + +const ContentEncoding::ContentEncryption* ContentEncoding::GetEncryptionByIndex( + unsigned long idx) const { + const ptrdiff_t count = encryption_entries_end_ - encryption_entries_; + assert(count >= 0); + + if (idx >= static_cast(count)) + return NULL; + + return encryption_entries_[idx]; +} + +unsigned long ContentEncoding::GetEncryptionCount() const { + const ptrdiff_t count = encryption_entries_end_ - encryption_entries_; + assert(count >= 0); + + return static_cast(count); +} + +long ContentEncoding::ParseContentEncAESSettingsEntry( + long long start, long long size, IMkvReader* pReader, + ContentEncAESSettings* aes) { + assert(pReader); + assert(aes); + + long long pos = start; + const long long stop = start + size; + + while (pos < stop) { + long long id, size; + const long status = ParseElementHeader(pReader, pos, stop, id, size); + if (status < 0) // error + return status; + + if (id == libwebm::kMkvAESSettingsCipherMode) { + aes->cipher_mode = UnserializeUInt(pReader, pos, size); + if (aes->cipher_mode != 1) + return E_FILE_FORMAT_INVALID; + } + + pos += size; // consume payload + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + return 0; +} + +long ContentEncoding::ParseContentEncodingEntry(long long start, long long size, + IMkvReader* pReader) { + assert(pReader); + + long long pos = start; + const long long stop = start + size; + + // Count ContentCompression and ContentEncryption elements. + int compression_count = 0; + int encryption_count = 0; + + while (pos < stop) { + long long id, size; + const long status = ParseElementHeader(pReader, pos, stop, id, size); + if (status < 0) // error + return status; + + if (id == libwebm::kMkvContentCompression) + ++compression_count; + + if (id == libwebm::kMkvContentEncryption) + ++encryption_count; + + pos += size; // consume payload + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (compression_count <= 0 && encryption_count <= 0) + return -1; + + if (compression_count > 0) { + compression_entries_ = + new (std::nothrow) ContentCompression*[compression_count]; + if (!compression_entries_) + return -1; + compression_entries_end_ = compression_entries_; + } + + if (encryption_count > 0) { + encryption_entries_ = + new (std::nothrow) ContentEncryption*[encryption_count]; + if (!encryption_entries_) { + delete[] compression_entries_; + return -1; + } + encryption_entries_end_ = encryption_entries_; + } + + pos = start; + while (pos < stop) { + long long id, size; + long status = ParseElementHeader(pReader, pos, stop, id, size); + if (status < 0) // error + return status; + + if (id == libwebm::kMkvContentEncodingOrder) { + encoding_order_ = UnserializeUInt(pReader, pos, size); + } else if (id == libwebm::kMkvContentEncodingScope) { + encoding_scope_ = UnserializeUInt(pReader, pos, size); + if (encoding_scope_ < 1) + return -1; + } else if (id == libwebm::kMkvContentEncodingType) { + encoding_type_ = UnserializeUInt(pReader, pos, size); + } else if (id == libwebm::kMkvContentCompression) { + ContentCompression* const compression = + new (std::nothrow) ContentCompression(); + if (!compression) + return -1; + + status = ParseCompressionEntry(pos, size, pReader, compression); + if (status) { + delete compression; + return status; + } + *compression_entries_end_++ = compression; + } else if (id == libwebm::kMkvContentEncryption) { + ContentEncryption* const encryption = + new (std::nothrow) ContentEncryption(); + if (!encryption) + return -1; + + status = ParseEncryptionEntry(pos, size, pReader, encryption); + if (status) { + delete encryption; + return status; + } + *encryption_entries_end_++ = encryption; + } + + pos += size; // consume payload + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + return 0; +} + +long ContentEncoding::ParseCompressionEntry(long long start, long long size, + IMkvReader* pReader, + ContentCompression* compression) { + assert(pReader); + assert(compression); + + long long pos = start; + const long long stop = start + size; + + bool valid = false; + + while (pos < stop) { + long long id, size; + const long status = ParseElementHeader(pReader, pos, stop, id, size); + if (status < 0) // error + return status; + + if (id == libwebm::kMkvContentCompAlgo) { + long long algo = UnserializeUInt(pReader, pos, size); + if (algo < 0) + return E_FILE_FORMAT_INVALID; + compression->algo = algo; + valid = true; + } else if (id == libwebm::kMkvContentCompSettings) { + if (size <= 0) + return E_FILE_FORMAT_INVALID; + + const size_t buflen = static_cast(size); + unsigned char* buf = SafeArrayAlloc(1, buflen); + if (buf == NULL) + return -1; + + const int read_status = + pReader->Read(pos, static_cast(buflen), buf); + if (read_status) { + delete[] buf; + return status; + } + + compression->settings = buf; + compression->settings_len = buflen; + } + + pos += size; // consume payload + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + // ContentCompAlgo is mandatory + if (!valid) + return E_FILE_FORMAT_INVALID; + + return 0; +} + +long ContentEncoding::ParseEncryptionEntry(long long start, long long size, + IMkvReader* pReader, + ContentEncryption* encryption) { + assert(pReader); + assert(encryption); + + long long pos = start; + const long long stop = start + size; + + while (pos < stop) { + long long id, size; + const long status = ParseElementHeader(pReader, pos, stop, id, size); + if (status < 0) // error + return status; + + if (id == libwebm::kMkvContentEncAlgo) { + encryption->algo = UnserializeUInt(pReader, pos, size); + if (encryption->algo != 5) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvContentEncKeyID) { + delete[] encryption->key_id; + encryption->key_id = NULL; + encryption->key_id_len = 0; + + if (size <= 0) + return E_FILE_FORMAT_INVALID; + + const size_t buflen = static_cast(size); + unsigned char* buf = SafeArrayAlloc(1, buflen); + if (buf == NULL) + return -1; + + const int read_status = + pReader->Read(pos, static_cast(buflen), buf); + if (read_status) { + delete[] buf; + return status; + } + + encryption->key_id = buf; + encryption->key_id_len = buflen; + } else if (id == libwebm::kMkvContentSignature) { + delete[] encryption->signature; + encryption->signature = NULL; + encryption->signature_len = 0; + + if (size <= 0) + return E_FILE_FORMAT_INVALID; + + const size_t buflen = static_cast(size); + unsigned char* buf = SafeArrayAlloc(1, buflen); + if (buf == NULL) + return -1; + + const int read_status = + pReader->Read(pos, static_cast(buflen), buf); + if (read_status) { + delete[] buf; + return status; + } + + encryption->signature = buf; + encryption->signature_len = buflen; + } else if (id == libwebm::kMkvContentSigKeyID) { + delete[] encryption->sig_key_id; + encryption->sig_key_id = NULL; + encryption->sig_key_id_len = 0; + + if (size <= 0) + return E_FILE_FORMAT_INVALID; + + const size_t buflen = static_cast(size); + unsigned char* buf = SafeArrayAlloc(1, buflen); + if (buf == NULL) + return -1; + + const int read_status = + pReader->Read(pos, static_cast(buflen), buf); + if (read_status) { + delete[] buf; + return status; + } + + encryption->sig_key_id = buf; + encryption->sig_key_id_len = buflen; + } else if (id == libwebm::kMkvContentSigAlgo) { + encryption->sig_algo = UnserializeUInt(pReader, pos, size); + } else if (id == libwebm::kMkvContentSigHashAlgo) { + encryption->sig_hash_algo = UnserializeUInt(pReader, pos, size); + } else if (id == libwebm::kMkvContentEncAESSettings) { + const long status = ParseContentEncAESSettingsEntry( + pos, size, pReader, &encryption->aes_settings); + if (status) + return status; + } + + pos += size; // consume payload + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + return 0; +} + +Track::Track(Segment* pSegment, long long element_start, long long element_size) + : m_pSegment(pSegment), + m_element_start(element_start), + m_element_size(element_size), + content_encoding_entries_(NULL), + content_encoding_entries_end_(NULL) {} + +Track::~Track() { + Info& info = const_cast(m_info); + info.Clear(); + + ContentEncoding** i = content_encoding_entries_; + ContentEncoding** const j = content_encoding_entries_end_; + + while (i != j) { + ContentEncoding* const encoding = *i++; + delete encoding; + } + + delete[] content_encoding_entries_; +} + +long Track::Create(Segment* pSegment, const Info& info, long long element_start, + long long element_size, Track*& pResult) { + if (pResult) + return -1; + + Track* const pTrack = + new (std::nothrow) Track(pSegment, element_start, element_size); + + if (pTrack == NULL) + return -1; // generic error + + const int status = info.Copy(pTrack->m_info); + + if (status) { // error + delete pTrack; + return status; + } + + pResult = pTrack; + return 0; // success +} + +Track::Info::Info() + : uid(0), + defaultDuration(0), + codecDelay(0), + seekPreRoll(0), + nameAsUTF8(NULL), + language(NULL), + codecId(NULL), + codecNameAsUTF8(NULL), + codecPrivate(NULL), + codecPrivateSize(0), + lacing(false) {} + +Track::Info::~Info() { Clear(); } + +void Track::Info::Clear() { + delete[] nameAsUTF8; + nameAsUTF8 = NULL; + + delete[] language; + language = NULL; + + delete[] codecId; + codecId = NULL; + + delete[] codecPrivate; + codecPrivate = NULL; + codecPrivateSize = 0; + + delete[] codecNameAsUTF8; + codecNameAsUTF8 = NULL; +} + +int Track::Info::CopyStr(char* Info::*str, Info& dst_) const { + if (str == static_cast(NULL)) + return -1; + + char*& dst = dst_.*str; + + if (dst) // should be NULL already + return -1; + + const char* const src = this->*str; + + if (src == NULL) + return 0; + + const size_t len = strlen(src); + + dst = SafeArrayAlloc(1, len + 1); + + if (dst == NULL) + return -1; + + strcpy(dst, src); + + return 0; +} + +int Track::Info::Copy(Info& dst) const { + if (&dst == this) + return 0; + + dst.type = type; + dst.number = number; + dst.defaultDuration = defaultDuration; + dst.codecDelay = codecDelay; + dst.seekPreRoll = seekPreRoll; + dst.uid = uid; + dst.lacing = lacing; + dst.settings = settings; + + // We now copy the string member variables from src to dst. + // This involves memory allocation so in principle the operation + // can fail (indeed, that's why we have Info::Copy), so we must + // report this to the caller. An error return from this function + // therefore implies that the copy was only partially successful. + + if (int status = CopyStr(&Info::nameAsUTF8, dst)) + return status; + + if (int status = CopyStr(&Info::language, dst)) + return status; + + if (int status = CopyStr(&Info::codecId, dst)) + return status; + + if (int status = CopyStr(&Info::codecNameAsUTF8, dst)) + return status; + + if (codecPrivateSize > 0) { + if (codecPrivate == NULL) + return -1; + + if (dst.codecPrivate) + return -1; + + if (dst.codecPrivateSize != 0) + return -1; + + dst.codecPrivate = SafeArrayAlloc(1, codecPrivateSize); + + if (dst.codecPrivate == NULL) + return -1; + + memcpy(dst.codecPrivate, codecPrivate, codecPrivateSize); + dst.codecPrivateSize = codecPrivateSize; + } + + return 0; +} + +const BlockEntry* Track::GetEOS() const { return &m_eos; } + +long Track::GetType() const { return m_info.type; } + +long Track::GetNumber() const { return m_info.number; } + +unsigned long long Track::GetUid() const { return m_info.uid; } + +const char* Track::GetNameAsUTF8() const { return m_info.nameAsUTF8; } + +const char* Track::GetLanguage() const { return m_info.language; } + +const char* Track::GetCodecNameAsUTF8() const { return m_info.codecNameAsUTF8; } + +const char* Track::GetCodecId() const { return m_info.codecId; } + +const unsigned char* Track::GetCodecPrivate(size_t& size) const { + size = m_info.codecPrivateSize; + return m_info.codecPrivate; +} + +bool Track::GetLacing() const { return m_info.lacing; } + +unsigned long long Track::GetDefaultDuration() const { + return m_info.defaultDuration; +} + +unsigned long long Track::GetCodecDelay() const { return m_info.codecDelay; } + +unsigned long long Track::GetSeekPreRoll() const { return m_info.seekPreRoll; } + +long Track::GetFirst(const BlockEntry*& pBlockEntry) const { + const Cluster* pCluster = m_pSegment->GetFirst(); + + for (int i = 0;;) { + if (pCluster == NULL) { + pBlockEntry = GetEOS(); + return 1; + } + + if (pCluster->EOS()) { + if (m_pSegment->DoneParsing()) { + pBlockEntry = GetEOS(); + return 1; + } + + pBlockEntry = 0; + return E_BUFFER_NOT_FULL; + } + + long status = pCluster->GetFirst(pBlockEntry); + + if (status < 0) // error + return status; + + if (pBlockEntry == 0) { // empty cluster + pCluster = m_pSegment->GetNext(pCluster); + continue; + } + + for (;;) { + const Block* const pBlock = pBlockEntry->GetBlock(); + assert(pBlock); + + const long long tn = pBlock->GetTrackNumber(); + + if ((tn == m_info.number) && VetEntry(pBlockEntry)) + return 0; + + const BlockEntry* pNextEntry; + + status = pCluster->GetNext(pBlockEntry, pNextEntry); + + if (status < 0) // error + return status; + + if (pNextEntry == 0) + break; + + pBlockEntry = pNextEntry; + } + + ++i; + + if (i >= 100) + break; + + pCluster = m_pSegment->GetNext(pCluster); + } + + // NOTE: if we get here, it means that we didn't find a block with + // a matching track number. We interpret that as an error (which + // might be too conservative). + + pBlockEntry = GetEOS(); // so we can return a non-NULL value + return 1; +} + +long Track::GetNext(const BlockEntry* pCurrEntry, + const BlockEntry*& pNextEntry) const { + assert(pCurrEntry); + assert(!pCurrEntry->EOS()); //? + + const Block* const pCurrBlock = pCurrEntry->GetBlock(); + assert(pCurrBlock && pCurrBlock->GetTrackNumber() == m_info.number); + if (!pCurrBlock || pCurrBlock->GetTrackNumber() != m_info.number) + return -1; + + const Cluster* pCluster = pCurrEntry->GetCluster(); + assert(pCluster); + assert(!pCluster->EOS()); + + long status = pCluster->GetNext(pCurrEntry, pNextEntry); + + if (status < 0) // error + return status; + + for (int i = 0;;) { + while (pNextEntry) { + const Block* const pNextBlock = pNextEntry->GetBlock(); + assert(pNextBlock); + + if (pNextBlock->GetTrackNumber() == m_info.number) + return 0; + + pCurrEntry = pNextEntry; + + status = pCluster->GetNext(pCurrEntry, pNextEntry); + + if (status < 0) // error + return status; + } + + pCluster = m_pSegment->GetNext(pCluster); + + if (pCluster == NULL) { + pNextEntry = GetEOS(); + return 1; + } + + if (pCluster->EOS()) { + if (m_pSegment->DoneParsing()) { + pNextEntry = GetEOS(); + return 1; + } + + // TODO: there is a potential O(n^2) problem here: we tell the + // caller to (pre)load another cluster, which he does, but then he + // calls GetNext again, which repeats the same search. This is + // a pathological case, since the only way it can happen is if + // there exists a long sequence of clusters none of which contain a + // block from this track. One way around this problem is for the + // caller to be smarter when he loads another cluster: don't call + // us back until you have a cluster that contains a block from this + // track. (Of course, that's not cheap either, since our caller + // would have to scan the each cluster as it's loaded, so that + // would just push back the problem.) + + pNextEntry = NULL; + return E_BUFFER_NOT_FULL; + } + + status = pCluster->GetFirst(pNextEntry); + + if (status < 0) // error + return status; + + if (pNextEntry == NULL) // empty cluster + continue; + + ++i; + + if (i >= 100) + break; + } + + // NOTE: if we get here, it means that we didn't find a block with + // a matching track number after lots of searching, so we give + // up trying. + + pNextEntry = GetEOS(); // so we can return a non-NULL value + return 1; +} + +bool Track::VetEntry(const BlockEntry* pBlockEntry) const { + assert(pBlockEntry); + const Block* const pBlock = pBlockEntry->GetBlock(); + assert(pBlock); + assert(pBlock->GetTrackNumber() == m_info.number); + if (!pBlock || pBlock->GetTrackNumber() != m_info.number) + return false; + + // This function is used during a seek to determine whether the + // frame is a valid seek target. This default function simply + // returns true, which means all frames are valid seek targets. + // It gets overridden by the VideoTrack class, because only video + // keyframes can be used as seek target. + + return true; +} + +long Track::Seek(long long time_ns, const BlockEntry*& pResult) const { + const long status = GetFirst(pResult); + + if (status < 0) // buffer underflow, etc + return status; + + assert(pResult); + + if (pResult->EOS()) + return 0; + + const Cluster* pCluster = pResult->GetCluster(); + assert(pCluster); + assert(pCluster->GetIndex() >= 0); + + if (time_ns <= pResult->GetBlock()->GetTime(pCluster)) + return 0; + + Cluster** const clusters = m_pSegment->m_clusters; + assert(clusters); + + const long count = m_pSegment->GetCount(); // loaded only, not preloaded + assert(count > 0); + + Cluster** const i = clusters + pCluster->GetIndex(); + assert(i); + assert(*i == pCluster); + assert(pCluster->GetTime() <= time_ns); + + Cluster** const j = clusters + count; + + Cluster** lo = i; + Cluster** hi = j; + + while (lo < hi) { + // INVARIANT: + //[i, lo) <= time_ns + //[lo, hi) ? + //[hi, j) > time_ns + + Cluster** const mid = lo + (hi - lo) / 2; + assert(mid < hi); + + pCluster = *mid; + assert(pCluster); + assert(pCluster->GetIndex() >= 0); + assert(pCluster->GetIndex() == long(mid - m_pSegment->m_clusters)); + + const long long t = pCluster->GetTime(); + + if (t <= time_ns) + lo = mid + 1; + else + hi = mid; + + assert(lo <= hi); + } + + assert(lo == hi); + assert(lo > i); + assert(lo <= j); + + while (lo > i) { + pCluster = *--lo; + assert(pCluster); + assert(pCluster->GetTime() <= time_ns); + + pResult = pCluster->GetEntry(this); + + if ((pResult != 0) && !pResult->EOS()) + return 0; + + // landed on empty cluster (no entries) + } + + pResult = GetEOS(); // weird + return 0; +} + +const ContentEncoding* Track::GetContentEncodingByIndex( + unsigned long idx) const { + const ptrdiff_t count = + content_encoding_entries_end_ - content_encoding_entries_; + assert(count >= 0); + + if (idx >= static_cast(count)) + return NULL; + + return content_encoding_entries_[idx]; +} + +unsigned long Track::GetContentEncodingCount() const { + const ptrdiff_t count = + content_encoding_entries_end_ - content_encoding_entries_; + assert(count >= 0); + + return static_cast(count); +} + +long Track::ParseContentEncodingsEntry(long long start, long long size) { + IMkvReader* const pReader = m_pSegment->m_pReader; + assert(pReader); + + long long pos = start; + const long long stop = start + size; + + // Count ContentEncoding elements. + int count = 0; + while (pos < stop) { + long long id, size; + const long status = ParseElementHeader(pReader, pos, stop, id, size); + if (status < 0) // error + return status; + + // pos now designates start of element + if (id == libwebm::kMkvContentEncoding) + ++count; + + pos += size; // consume payload + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (count <= 0) + return -1; + + content_encoding_entries_ = new (std::nothrow) ContentEncoding*[count]; + if (!content_encoding_entries_) + return -1; + + content_encoding_entries_end_ = content_encoding_entries_; + + pos = start; + while (pos < stop) { + long long id, size; + long status = ParseElementHeader(pReader, pos, stop, id, size); + if (status < 0) // error + return status; + + // pos now designates start of element + if (id == libwebm::kMkvContentEncoding) { + ContentEncoding* const content_encoding = + new (std::nothrow) ContentEncoding(); + if (!content_encoding) + return -1; + + status = content_encoding->ParseContentEncodingEntry(pos, size, pReader); + if (status) { + delete content_encoding; + return status; + } + + *content_encoding_entries_end_++ = content_encoding; + } + + pos += size; // consume payload + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + + return 0; +} + +Track::EOSBlock::EOSBlock() : BlockEntry(NULL, LONG_MIN) {} + +BlockEntry::Kind Track::EOSBlock::GetKind() const { return kBlockEOS; } + +const Block* Track::EOSBlock::GetBlock() const { return NULL; } + +bool PrimaryChromaticity::Parse(IMkvReader* reader, long long read_pos, + long long value_size, bool is_x, + PrimaryChromaticity** chromaticity) { + if (!reader) + return false; + + std::auto_ptr chromaticity_ptr; + + if (!*chromaticity) { + chromaticity_ptr.reset(new PrimaryChromaticity()); + } else { + chromaticity_ptr.reset(*chromaticity); + } + + if (!chromaticity_ptr.get()) + return false; + + float* value = is_x ? &chromaticity_ptr->x : &chromaticity_ptr->y; + + double parser_value = 0; + const long long value_parse_status = + UnserializeFloat(reader, read_pos, value_size, parser_value); + + *value = static_cast(parser_value); + + if (value_parse_status < 0 || *value < 0.0 || *value > 1.0) + return false; + + *chromaticity = chromaticity_ptr.release(); + return true; +} + +bool MasteringMetadata::Parse(IMkvReader* reader, long long mm_start, + long long mm_size, MasteringMetadata** mm) { + if (!reader || *mm) + return false; + + std::auto_ptr mm_ptr(new MasteringMetadata()); + if (!mm_ptr.get()) + return false; + + const long long mm_end = mm_start + mm_size; + long long read_pos = mm_start; + + while (read_pos < mm_end) { + long long child_id = 0; + long long child_size = 0; + + const long long status = + ParseElementHeader(reader, read_pos, mm_end, child_id, child_size); + if (status < 0) + return false; + + if (child_id == libwebm::kMkvLuminanceMax) { + double value = 0; + const long long value_parse_status = + UnserializeFloat(reader, read_pos, child_size, value); + mm_ptr->luminance_max = static_cast(value); + if (value_parse_status < 0 || mm_ptr->luminance_max < 0.0 || + mm_ptr->luminance_max > 9999.99) { + return false; + } + } else if (child_id == libwebm::kMkvLuminanceMin) { + double value = 0; + const long long value_parse_status = + UnserializeFloat(reader, read_pos, child_size, value); + mm_ptr->luminance_min = static_cast(value); + if (value_parse_status < 0 || mm_ptr->luminance_min < 0.0 || + mm_ptr->luminance_min > 999.9999) { + return false; + } + } else { + bool is_x = false; + PrimaryChromaticity** chromaticity; + switch (child_id) { + case libwebm::kMkvPrimaryRChromaticityX: + case libwebm::kMkvPrimaryRChromaticityY: + is_x = child_id == libwebm::kMkvPrimaryRChromaticityX; + chromaticity = &mm_ptr->r; + break; + case libwebm::kMkvPrimaryGChromaticityX: + case libwebm::kMkvPrimaryGChromaticityY: + is_x = child_id == libwebm::kMkvPrimaryGChromaticityX; + chromaticity = &mm_ptr->g; + break; + case libwebm::kMkvPrimaryBChromaticityX: + case libwebm::kMkvPrimaryBChromaticityY: + is_x = child_id == libwebm::kMkvPrimaryBChromaticityX; + chromaticity = &mm_ptr->b; + break; + case libwebm::kMkvWhitePointChromaticityX: + case libwebm::kMkvWhitePointChromaticityY: + is_x = child_id == libwebm::kMkvWhitePointChromaticityX; + chromaticity = &mm_ptr->white_point; + break; + default: + return false; + } + const bool value_parse_status = PrimaryChromaticity::Parse( + reader, read_pos, child_size, is_x, chromaticity); + if (!value_parse_status) + return false; + } + + read_pos += child_size; + if (read_pos > mm_end) + return false; + } + + *mm = mm_ptr.release(); + return true; +} + +bool Colour::Parse(IMkvReader* reader, long long colour_start, + long long colour_size, Colour** colour) { + if (!reader || *colour) + return false; + + std::auto_ptr colour_ptr(new Colour()); + if (!colour_ptr.get()) + return false; + + const long long colour_end = colour_start + colour_size; + long long read_pos = colour_start; + + while (read_pos < colour_end) { + long long child_id = 0; + long long child_size = 0; + + const long status = + ParseElementHeader(reader, read_pos, colour_end, child_id, child_size); + if (status < 0) + return false; + + if (child_id == libwebm::kMkvMatrixCoefficients) { + colour_ptr->matrix_coefficients = + UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->matrix_coefficients < 0) + return false; + } else if (child_id == libwebm::kMkvBitsPerChannel) { + colour_ptr->bits_per_channel = + UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->bits_per_channel < 0) + return false; + } else if (child_id == libwebm::kMkvChromaSubsamplingHorz) { + colour_ptr->chroma_subsampling_horz = + UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->chroma_subsampling_horz < 0) + return false; + } else if (child_id == libwebm::kMkvChromaSubsamplingVert) { + colour_ptr->chroma_subsampling_vert = + UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->chroma_subsampling_vert < 0) + return false; + } else if (child_id == libwebm::kMkvCbSubsamplingHorz) { + colour_ptr->cb_subsampling_horz = + UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->cb_subsampling_horz < 0) + return false; + } else if (child_id == libwebm::kMkvCbSubsamplingVert) { + colour_ptr->cb_subsampling_vert = + UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->cb_subsampling_vert < 0) + return false; + } else if (child_id == libwebm::kMkvChromaSitingHorz) { + colour_ptr->chroma_siting_horz = + UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->chroma_siting_horz < 0) + return false; + } else if (child_id == libwebm::kMkvChromaSitingVert) { + colour_ptr->chroma_siting_vert = + UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->chroma_siting_vert < 0) + return false; + } else if (child_id == libwebm::kMkvRange) { + colour_ptr->range = UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->range < 0) + return false; + } else if (child_id == libwebm::kMkvTransferCharacteristics) { + colour_ptr->transfer_characteristics = + UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->transfer_characteristics < 0) + return false; + } else if (child_id == libwebm::kMkvPrimaries) { + colour_ptr->primaries = UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->primaries < 0) + return false; + } else if (child_id == libwebm::kMkvMaxCLL) { + colour_ptr->max_cll = UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->max_cll < 0) + return false; + } else if (child_id == libwebm::kMkvMaxFALL) { + colour_ptr->max_fall = UnserializeUInt(reader, read_pos, child_size); + if (colour_ptr->max_fall < 0) + return false; + } else if (child_id == libwebm::kMkvMasteringMetadata) { + if (!MasteringMetadata::Parse(reader, read_pos, child_size, + &colour_ptr->mastering_metadata)) + return false; + } else { + return false; + } + + read_pos += child_size; + if (read_pos > colour_end) + return false; + } + *colour = colour_ptr.release(); + return true; +} + +VideoTrack::VideoTrack(Segment* pSegment, long long element_start, + long long element_size) + : Track(pSegment, element_start, element_size), m_colour(NULL) {} + +VideoTrack::~VideoTrack() { delete m_colour; } + +long VideoTrack::Parse(Segment* pSegment, const Info& info, + long long element_start, long long element_size, + VideoTrack*& pResult) { + if (pResult) + return -1; + + if (info.type != Track::kVideo) + return -1; + + long long width = 0; + long long height = 0; + long long display_width = 0; + long long display_height = 0; + long long display_unit = 0; + long long stereo_mode = 0; + + double rate = 0.0; + + IMkvReader* const pReader = pSegment->m_pReader; + + const Settings& s = info.settings; + assert(s.start >= 0); + assert(s.size >= 0); + + long long pos = s.start; + assert(pos >= 0); + + const long long stop = pos + s.size; + + Colour* colour = NULL; + + while (pos < stop) { + long long id, size; + + const long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) // error + return status; + + if (id == libwebm::kMkvPixelWidth) { + width = UnserializeUInt(pReader, pos, size); + + if (width <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvPixelHeight) { + height = UnserializeUInt(pReader, pos, size); + + if (height <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvDisplayWidth) { + display_width = UnserializeUInt(pReader, pos, size); + + if (display_width <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvDisplayHeight) { + display_height = UnserializeUInt(pReader, pos, size); + + if (display_height <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvDisplayUnit) { + display_unit = UnserializeUInt(pReader, pos, size); + + if (display_unit < 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvStereoMode) { + stereo_mode = UnserializeUInt(pReader, pos, size); + + if (stereo_mode < 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvFrameRate) { + const long status = UnserializeFloat(pReader, pos, size, rate); + + if (status < 0) + return status; + + if (rate <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvColour) { + if (!Colour::Parse(pReader, pos, size, &colour)) + return E_FILE_FORMAT_INVALID; + } + + pos += size; // consume payload + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + + VideoTrack* const pTrack = + new (std::nothrow) VideoTrack(pSegment, element_start, element_size); + + if (pTrack == NULL) + return -1; // generic error + + const int status = info.Copy(pTrack->m_info); + + if (status) { // error + delete pTrack; + return status; + } + + pTrack->m_width = width; + pTrack->m_height = height; + pTrack->m_display_width = display_width; + pTrack->m_display_height = display_height; + pTrack->m_display_unit = display_unit; + pTrack->m_stereo_mode = stereo_mode; + pTrack->m_rate = rate; + pTrack->m_colour = colour; + + pResult = pTrack; + return 0; // success +} + +bool VideoTrack::VetEntry(const BlockEntry* pBlockEntry) const { + return Track::VetEntry(pBlockEntry) && pBlockEntry->GetBlock()->IsKey(); +} + +long VideoTrack::Seek(long long time_ns, const BlockEntry*& pResult) const { + const long status = GetFirst(pResult); + + if (status < 0) // buffer underflow, etc + return status; + + assert(pResult); + + if (pResult->EOS()) + return 0; + + const Cluster* pCluster = pResult->GetCluster(); + assert(pCluster); + assert(pCluster->GetIndex() >= 0); + + if (time_ns <= pResult->GetBlock()->GetTime(pCluster)) + return 0; + + Cluster** const clusters = m_pSegment->m_clusters; + assert(clusters); + + const long count = m_pSegment->GetCount(); // loaded only, not pre-loaded + assert(count > 0); + + Cluster** const i = clusters + pCluster->GetIndex(); + assert(i); + assert(*i == pCluster); + assert(pCluster->GetTime() <= time_ns); + + Cluster** const j = clusters + count; + + Cluster** lo = i; + Cluster** hi = j; + + while (lo < hi) { + // INVARIANT: + //[i, lo) <= time_ns + //[lo, hi) ? + //[hi, j) > time_ns + + Cluster** const mid = lo + (hi - lo) / 2; + assert(mid < hi); + + pCluster = *mid; + assert(pCluster); + assert(pCluster->GetIndex() >= 0); + assert(pCluster->GetIndex() == long(mid - m_pSegment->m_clusters)); + + const long long t = pCluster->GetTime(); + + if (t <= time_ns) + lo = mid + 1; + else + hi = mid; + + assert(lo <= hi); + } + + assert(lo == hi); + assert(lo > i); + assert(lo <= j); + + pCluster = *--lo; + assert(pCluster); + assert(pCluster->GetTime() <= time_ns); + + pResult = pCluster->GetEntry(this, time_ns); + + if ((pResult != 0) && !pResult->EOS()) // found a keyframe + return 0; + + while (lo != i) { + pCluster = *--lo; + assert(pCluster); + assert(pCluster->GetTime() <= time_ns); + + pResult = pCluster->GetEntry(this, time_ns); + + if ((pResult != 0) && !pResult->EOS()) + return 0; + } + + // weird: we're on the first cluster, but no keyframe found + // should never happen but we must return something anyway + + pResult = GetEOS(); + return 0; +} + +Colour* VideoTrack::GetColour() const { return m_colour; } + +long long VideoTrack::GetWidth() const { return m_width; } + +long long VideoTrack::GetHeight() const { return m_height; } + +long long VideoTrack::GetDisplayWidth() const { + return m_display_width > 0 ? m_display_width : GetWidth(); +} + +long long VideoTrack::GetDisplayHeight() const { + return m_display_height > 0 ? m_display_height : GetHeight(); +} + +long long VideoTrack::GetDisplayUnit() const { return m_display_unit; } + +long long VideoTrack::GetStereoMode() const { return m_stereo_mode; } + +double VideoTrack::GetFrameRate() const { return m_rate; } + +AudioTrack::AudioTrack(Segment* pSegment, long long element_start, + long long element_size) + : Track(pSegment, element_start, element_size) {} + +long AudioTrack::Parse(Segment* pSegment, const Info& info, + long long element_start, long long element_size, + AudioTrack*& pResult) { + if (pResult) + return -1; + + if (info.type != Track::kAudio) + return -1; + + IMkvReader* const pReader = pSegment->m_pReader; + + const Settings& s = info.settings; + assert(s.start >= 0); + assert(s.size >= 0); + + long long pos = s.start; + assert(pos >= 0); + + const long long stop = pos + s.size; + + double rate = 8000.0; // MKV default + long long channels = 1; + long long bit_depth = 0; + + while (pos < stop) { + long long id, size; + + long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) // error + return status; + + if (id == libwebm::kMkvSamplingFrequency) { + status = UnserializeFloat(pReader, pos, size, rate); + + if (status < 0) + return status; + + if (rate <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvChannels) { + channels = UnserializeUInt(pReader, pos, size); + + if (channels <= 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvBitDepth) { + bit_depth = UnserializeUInt(pReader, pos, size); + + if (bit_depth <= 0) + return E_FILE_FORMAT_INVALID; + } + + pos += size; // consume payload + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + + AudioTrack* const pTrack = + new (std::nothrow) AudioTrack(pSegment, element_start, element_size); + + if (pTrack == NULL) + return -1; // generic error + + const int status = info.Copy(pTrack->m_info); + + if (status) { + delete pTrack; + return status; + } + + pTrack->m_rate = rate; + pTrack->m_channels = channels; + pTrack->m_bitDepth = bit_depth; + + pResult = pTrack; + return 0; // success +} + +double AudioTrack::GetSamplingRate() const { return m_rate; } + +long long AudioTrack::GetChannels() const { return m_channels; } + +long long AudioTrack::GetBitDepth() const { return m_bitDepth; } + +Tracks::Tracks(Segment* pSegment, long long start, long long size_, + long long element_start, long long element_size) + : m_pSegment(pSegment), + m_start(start), + m_size(size_), + m_element_start(element_start), + m_element_size(element_size), + m_trackEntries(NULL), + m_trackEntriesEnd(NULL) {} + +long Tracks::Parse() { + assert(m_trackEntries == NULL); + assert(m_trackEntriesEnd == NULL); + + const long long stop = m_start + m_size; + IMkvReader* const pReader = m_pSegment->m_pReader; + + int count = 0; + long long pos = m_start; + + while (pos < stop) { + long long id, size; + + const long status = ParseElementHeader(pReader, pos, stop, id, size); + + if (status < 0) // error + return status; + + if (size == 0) // weird + continue; + + if (id == libwebm::kMkvTrackEntry) + ++count; + + pos += size; // consume payload + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + + if (count <= 0) + return 0; // success + + m_trackEntries = new (std::nothrow) Track*[count]; + + if (m_trackEntries == NULL) + return -1; + + m_trackEntriesEnd = m_trackEntries; + + pos = m_start; + + while (pos < stop) { + const long long element_start = pos; + + long long id, payload_size; + + const long status = + ParseElementHeader(pReader, pos, stop, id, payload_size); + + if (status < 0) // error + return status; + + if (payload_size == 0) // weird + continue; + + const long long payload_stop = pos + payload_size; + assert(payload_stop <= stop); // checked in ParseElement + + const long long element_size = payload_stop - element_start; + + if (id == libwebm::kMkvTrackEntry) { + Track*& pTrack = *m_trackEntriesEnd; + pTrack = NULL; + + const long status = ParseTrackEntry(pos, payload_size, element_start, + element_size, pTrack); + if (status) + return status; + + if (pTrack) + ++m_trackEntriesEnd; + } + + pos = payload_stop; + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + + return 0; // success +} + +unsigned long Tracks::GetTracksCount() const { + const ptrdiff_t result = m_trackEntriesEnd - m_trackEntries; + assert(result >= 0); + + return static_cast(result); +} + +long Tracks::ParseTrackEntry(long long track_start, long long track_size, + long long element_start, long long element_size, + Track*& pResult) const { + if (pResult) + return -1; + + IMkvReader* const pReader = m_pSegment->m_pReader; + + long long pos = track_start; + const long long track_stop = track_start + track_size; + + Track::Info info; + + info.type = 0; + info.number = 0; + info.uid = 0; + info.defaultDuration = 0; + + Track::Settings v; + v.start = -1; + v.size = -1; + + Track::Settings a; + a.start = -1; + a.size = -1; + + Track::Settings e; // content_encodings_settings; + e.start = -1; + e.size = -1; + + long long lacing = 1; // default is true + + while (pos < track_stop) { + long long id, size; + + const long status = ParseElementHeader(pReader, pos, track_stop, id, size); + + if (status < 0) // error + return status; + + if (size < 0) + return E_FILE_FORMAT_INVALID; + + const long long start = pos; + + if (id == libwebm::kMkvVideo) { + v.start = start; + v.size = size; + } else if (id == libwebm::kMkvAudio) { + a.start = start; + a.size = size; + } else if (id == libwebm::kMkvContentEncodings) { + e.start = start; + e.size = size; + } else if (id == libwebm::kMkvTrackUID) { + if (size > 8) + return E_FILE_FORMAT_INVALID; + + info.uid = 0; + + long long pos_ = start; + const long long pos_end = start + size; + + while (pos_ != pos_end) { + unsigned char b; + + const int status = pReader->Read(pos_, 1, &b); + + if (status) + return status; + + info.uid <<= 8; + info.uid |= b; + + ++pos_; + } + } else if (id == libwebm::kMkvTrackNumber) { + const long long num = UnserializeUInt(pReader, pos, size); + + if ((num <= 0) || (num > 127)) + return E_FILE_FORMAT_INVALID; + + info.number = static_cast(num); + } else if (id == libwebm::kMkvTrackType) { + const long long type = UnserializeUInt(pReader, pos, size); + + if ((type <= 0) || (type > 254)) + return E_FILE_FORMAT_INVALID; + + info.type = static_cast(type); + } else if (id == libwebm::kMkvName) { + const long status = + UnserializeString(pReader, pos, size, info.nameAsUTF8); + + if (status) + return status; + } else if (id == libwebm::kMkvLanguage) { + const long status = UnserializeString(pReader, pos, size, info.language); + + if (status) + return status; + } else if (id == libwebm::kMkvDefaultDuration) { + const long long duration = UnserializeUInt(pReader, pos, size); + + if (duration < 0) + return E_FILE_FORMAT_INVALID; + + info.defaultDuration = static_cast(duration); + } else if (id == libwebm::kMkvCodecID) { + const long status = UnserializeString(pReader, pos, size, info.codecId); + + if (status) + return status; + } else if (id == libwebm::kMkvFlagLacing) { + lacing = UnserializeUInt(pReader, pos, size); + + if ((lacing < 0) || (lacing > 1)) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvCodecPrivate) { + delete[] info.codecPrivate; + info.codecPrivate = NULL; + info.codecPrivateSize = 0; + + const size_t buflen = static_cast(size); + + if (buflen) { + unsigned char* buf = SafeArrayAlloc(1, buflen); + + if (buf == NULL) + return -1; + + const int status = pReader->Read(pos, static_cast(buflen), buf); + + if (status) { + delete[] buf; + return status; + } + + info.codecPrivate = buf; + info.codecPrivateSize = buflen; + } + } else if (id == libwebm::kMkvCodecName) { + const long status = + UnserializeString(pReader, pos, size, info.codecNameAsUTF8); + + if (status) + return status; + } else if (id == libwebm::kMkvCodecDelay) { + info.codecDelay = UnserializeUInt(pReader, pos, size); + } else if (id == libwebm::kMkvSeekPreRoll) { + info.seekPreRoll = UnserializeUInt(pReader, pos, size); + } + + pos += size; // consume payload + if (pos > track_stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != track_stop) + return E_FILE_FORMAT_INVALID; + + if (info.number <= 0) // not specified + return E_FILE_FORMAT_INVALID; + + if (GetTrackByNumber(info.number)) + return E_FILE_FORMAT_INVALID; + + if (info.type <= 0) // not specified + return E_FILE_FORMAT_INVALID; + + info.lacing = (lacing > 0) ? true : false; + + if (info.type == Track::kVideo) { + if (v.start < 0) + return E_FILE_FORMAT_INVALID; + + if (a.start >= 0) + return E_FILE_FORMAT_INVALID; + + info.settings = v; + + VideoTrack* pTrack = NULL; + + const long status = VideoTrack::Parse(m_pSegment, info, element_start, + element_size, pTrack); + + if (status) + return status; + + pResult = pTrack; + assert(pResult); + + if (e.start >= 0) + pResult->ParseContentEncodingsEntry(e.start, e.size); + } else if (info.type == Track::kAudio) { + if (a.start < 0) + return E_FILE_FORMAT_INVALID; + + if (v.start >= 0) + return E_FILE_FORMAT_INVALID; + + info.settings = a; + + AudioTrack* pTrack = NULL; + + const long status = AudioTrack::Parse(m_pSegment, info, element_start, + element_size, pTrack); + + if (status) + return status; + + pResult = pTrack; + assert(pResult); + + if (e.start >= 0) + pResult->ParseContentEncodingsEntry(e.start, e.size); + } else { + // neither video nor audio - probably metadata or subtitles + + if (a.start >= 0) + return E_FILE_FORMAT_INVALID; + + if (v.start >= 0) + return E_FILE_FORMAT_INVALID; + + if (info.type == Track::kMetadata && e.start >= 0) + return E_FILE_FORMAT_INVALID; + + info.settings.start = -1; + info.settings.size = 0; + + Track* pTrack = NULL; + + const long status = + Track::Create(m_pSegment, info, element_start, element_size, pTrack); + + if (status) + return status; + + pResult = pTrack; + assert(pResult); + } + + return 0; // success +} + +Tracks::~Tracks() { + Track** i = m_trackEntries; + Track** const j = m_trackEntriesEnd; + + while (i != j) { + Track* const pTrack = *i++; + delete pTrack; + } + + delete[] m_trackEntries; +} + +const Track* Tracks::GetTrackByNumber(long tn) const { + if (tn < 0) + return NULL; + + Track** i = m_trackEntries; + Track** const j = m_trackEntriesEnd; + + while (i != j) { + Track* const pTrack = *i++; + + if (pTrack == NULL) + continue; + + if (tn == pTrack->GetNumber()) + return pTrack; + } + + return NULL; // not found +} + +const Track* Tracks::GetTrackByIndex(unsigned long idx) const { + const ptrdiff_t count = m_trackEntriesEnd - m_trackEntries; + + if (idx >= static_cast(count)) + return NULL; + + return m_trackEntries[idx]; +} + +long Cluster::Load(long long& pos, long& len) const { + if (m_pSegment == NULL) + return E_PARSE_FAILED; + + if (m_timecode >= 0) // at least partially loaded + return 0; + + if (m_pos != m_element_start || m_element_size >= 0) + return E_PARSE_FAILED; + + IMkvReader* const pReader = m_pSegment->m_pReader; + long long total, avail; + const int status = pReader->Length(&total, &avail); + + if (status < 0) // error + return status; + + if (total >= 0 && (avail > total || m_pos > total)) + return E_FILE_FORMAT_INVALID; + + pos = m_pos; + + long long cluster_size = -1; + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + long long result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error or underflow + return static_cast(result); + + if (result > 0) + return E_BUFFER_NOT_FULL; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long id_ = ReadID(pReader, pos, len); + + if (id_ < 0) // error + return static_cast(id_); + + if (id_ != libwebm::kMkvCluster) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume id + + // read cluster size + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) + return E_BUFFER_NOT_FULL; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long size = ReadUInt(pReader, pos, len); + + if (size < 0) // error + return static_cast(cluster_size); + + if (size == 0) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume length of size of element + + const long long unknown_size = (1LL << (7 * len)) - 1; + + if (size != unknown_size) + cluster_size = size; + + // pos points to start of payload + long long timecode = -1; + long long new_pos = -1; + bool bBlock = false; + + long long cluster_stop = (cluster_size < 0) ? -1 : pos + cluster_size; + + for (;;) { + if ((cluster_stop >= 0) && (pos >= cluster_stop)) + break; + + // Parse ID + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + long long result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) + return E_BUFFER_NOT_FULL; + + if ((cluster_stop >= 0) && ((pos + len) > cluster_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long id = ReadID(pReader, pos, len); + + if (id < 0) // error + return static_cast(id); + + if (id == 0) + return E_FILE_FORMAT_INVALID; + + // This is the distinguished set of ID's we use to determine + // that we have exhausted the sub-element's inside the cluster + // whose ID we parsed earlier. + + if (id == libwebm::kMkvCluster) + break; + + if (id == libwebm::kMkvCues) + break; + + pos += len; // consume ID field + + // Parse Size + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) + return E_BUFFER_NOT_FULL; + + if ((cluster_stop >= 0) && ((pos + len) > cluster_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long size = ReadUInt(pReader, pos, len); + + if (size < 0) // error + return static_cast(size); + + const long long unknown_size = (1LL << (7 * len)) - 1; + + if (size == unknown_size) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume size field + + if ((cluster_stop >= 0) && (pos > cluster_stop)) + return E_FILE_FORMAT_INVALID; + + // pos now points to start of payload + + if (size == 0) + continue; + + if ((cluster_stop >= 0) && ((pos + size) > cluster_stop)) + return E_FILE_FORMAT_INVALID; + + if (id == libwebm::kMkvTimecode) { + len = static_cast(size); + + if ((pos + size) > avail) + return E_BUFFER_NOT_FULL; + + timecode = UnserializeUInt(pReader, pos, size); + + if (timecode < 0) // error (or underflow) + return static_cast(timecode); + + new_pos = pos + size; + + if (bBlock) + break; + } else if (id == libwebm::kMkvBlockGroup) { + bBlock = true; + break; + } else if (id == libwebm::kMkvSimpleBlock) { + bBlock = true; + break; + } + + pos += size; // consume payload + if (cluster_stop >= 0 && pos > cluster_stop) + return E_FILE_FORMAT_INVALID; + } + + if (cluster_stop >= 0 && pos > cluster_stop) + return E_FILE_FORMAT_INVALID; + + if (timecode < 0) // no timecode found + return E_FILE_FORMAT_INVALID; + + if (!bBlock) + return E_FILE_FORMAT_INVALID; + + m_pos = new_pos; // designates position just beyond timecode payload + m_timecode = timecode; // m_timecode >= 0 means we're partially loaded + + if (cluster_size >= 0) + m_element_size = cluster_stop - m_element_start; + + return 0; +} + +long Cluster::Parse(long long& pos, long& len) const { + long status = Load(pos, len); + + if (status < 0) + return status; + + if (m_pos < m_element_start || m_timecode < 0) + return E_PARSE_FAILED; + + const long long cluster_stop = + (m_element_size < 0) ? -1 : m_element_start + m_element_size; + + if ((cluster_stop >= 0) && (m_pos >= cluster_stop)) + return 1; // nothing else to do + + IMkvReader* const pReader = m_pSegment->m_pReader; + + long long total, avail; + + status = pReader->Length(&total, &avail); + + if (status < 0) // error + return status; + + if (total >= 0 && avail > total) + return E_FILE_FORMAT_INVALID; + + pos = m_pos; + + for (;;) { + if ((cluster_stop >= 0) && (pos >= cluster_stop)) + break; + + if ((total >= 0) && (pos >= total)) { + if (m_element_size < 0) + m_element_size = pos - m_element_start; + + break; + } + + // Parse ID + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + long long result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) + return E_BUFFER_NOT_FULL; + + if ((cluster_stop >= 0) && ((pos + len) > cluster_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long id = ReadID(pReader, pos, len); + + if (id < 0) + return E_FILE_FORMAT_INVALID; + + // This is the distinguished set of ID's we use to determine + // that we have exhausted the sub-element's inside the cluster + // whose ID we parsed earlier. + + if ((id == libwebm::kMkvCluster) || (id == libwebm::kMkvCues)) { + if (m_element_size < 0) + m_element_size = pos - m_element_start; + + break; + } + + pos += len; // consume ID field + + // Parse Size + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) + return E_BUFFER_NOT_FULL; + + if ((cluster_stop >= 0) && ((pos + len) > cluster_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long size = ReadUInt(pReader, pos, len); + + if (size < 0) // error + return static_cast(size); + + const long long unknown_size = (1LL << (7 * len)) - 1; + + if (size == unknown_size) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume size field + + if ((cluster_stop >= 0) && (pos > cluster_stop)) + return E_FILE_FORMAT_INVALID; + + // pos now points to start of payload + + if (size == 0) + continue; + + // const long long block_start = pos; + const long long block_stop = pos + size; + + if (cluster_stop >= 0) { + if (block_stop > cluster_stop) { + if (id == libwebm::kMkvBlockGroup || id == libwebm::kMkvSimpleBlock) { + return E_FILE_FORMAT_INVALID; + } + + pos = cluster_stop; + break; + } + } else if ((total >= 0) && (block_stop > total)) { + m_element_size = total - m_element_start; + pos = total; + break; + } else if (block_stop > avail) { + len = static_cast(size); + return E_BUFFER_NOT_FULL; + } + + Cluster* const this_ = const_cast(this); + + if (id == libwebm::kMkvBlockGroup) + return this_->ParseBlockGroup(size, pos, len); + + if (id == libwebm::kMkvSimpleBlock) + return this_->ParseSimpleBlock(size, pos, len); + + pos += size; // consume payload + if (cluster_stop >= 0 && pos > cluster_stop) + return E_FILE_FORMAT_INVALID; + } + + if (m_element_size < 1) + return E_FILE_FORMAT_INVALID; + + m_pos = pos; + if (cluster_stop >= 0 && m_pos > cluster_stop) + return E_FILE_FORMAT_INVALID; + + if (m_entries_count > 0) { + const long idx = m_entries_count - 1; + + const BlockEntry* const pLast = m_entries[idx]; + if (pLast == NULL) + return E_PARSE_FAILED; + + const Block* const pBlock = pLast->GetBlock(); + if (pBlock == NULL) + return E_PARSE_FAILED; + + const long long start = pBlock->m_start; + + if ((total >= 0) && (start > total)) + return E_PARSE_FAILED; // defend against trucated stream + + const long long size = pBlock->m_size; + + const long long stop = start + size; + if (cluster_stop >= 0 && stop > cluster_stop) + return E_FILE_FORMAT_INVALID; + + if ((total >= 0) && (stop > total)) + return E_PARSE_FAILED; // defend against trucated stream + } + + return 1; // no more entries +} + +long Cluster::ParseSimpleBlock(long long block_size, long long& pos, + long& len) { + const long long block_start = pos; + const long long block_stop = pos + block_size; + + IMkvReader* const pReader = m_pSegment->m_pReader; + + long long total, avail; + + long status = pReader->Length(&total, &avail); + + if (status < 0) // error + return status; + + assert((total < 0) || (avail <= total)); + + // parse track number + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + long long result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // weird + return E_BUFFER_NOT_FULL; + + if ((pos + len) > block_stop) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long track = ReadUInt(pReader, pos, len); + + if (track < 0) // error + return static_cast(track); + + if (track == 0) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume track number + + if ((pos + 2) > block_stop) + return E_FILE_FORMAT_INVALID; + + if ((pos + 2) > avail) { + len = 2; + return E_BUFFER_NOT_FULL; + } + + pos += 2; // consume timecode + + if ((pos + 1) > block_stop) + return E_FILE_FORMAT_INVALID; + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + unsigned char flags; + + status = pReader->Read(pos, 1, &flags); + + if (status < 0) { // error or underflow + len = 1; + return status; + } + + ++pos; // consume flags byte + assert(pos <= avail); + + if (pos >= block_stop) + return E_FILE_FORMAT_INVALID; + + const int lacing = int(flags & 0x06) >> 1; + + if ((lacing != 0) && (block_stop > avail)) { + len = static_cast(block_stop - pos); + return E_BUFFER_NOT_FULL; + } + + status = CreateBlock(libwebm::kMkvSimpleBlock, block_start, block_size, + 0); // DiscardPadding + + if (status != 0) + return status; + + m_pos = block_stop; + + return 0; // success +} + +long Cluster::ParseBlockGroup(long long payload_size, long long& pos, + long& len) { + const long long payload_start = pos; + const long long payload_stop = pos + payload_size; + + IMkvReader* const pReader = m_pSegment->m_pReader; + + long long total, avail; + + long status = pReader->Length(&total, &avail); + + if (status < 0) // error + return status; + + assert((total < 0) || (avail <= total)); + + if ((total >= 0) && (payload_stop > total)) + return E_FILE_FORMAT_INVALID; + + if (payload_stop > avail) { + len = static_cast(payload_size); + return E_BUFFER_NOT_FULL; + } + + long long discard_padding = 0; + + while (pos < payload_stop) { + // parse sub-block element ID + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + long long result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // weird + return E_BUFFER_NOT_FULL; + + if ((pos + len) > payload_stop) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long id = ReadID(pReader, pos, len); + + if (id < 0) // error + return static_cast(id); + + if (id == 0) // not a valid ID + return E_FILE_FORMAT_INVALID; + + pos += len; // consume ID field + + // Parse Size + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // weird + return E_BUFFER_NOT_FULL; + + if ((pos + len) > payload_stop) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long size = ReadUInt(pReader, pos, len); + + if (size < 0) // error + return static_cast(size); + + pos += len; // consume size field + + // pos now points to start of sub-block group payload + + if (pos > payload_stop) + return E_FILE_FORMAT_INVALID; + + if (size == 0) // weird + continue; + + const long long unknown_size = (1LL << (7 * len)) - 1; + + if (size == unknown_size) + return E_FILE_FORMAT_INVALID; + + if (id == libwebm::kMkvDiscardPadding) { + status = UnserializeInt(pReader, pos, size, discard_padding); + + if (status < 0) // error + return status; + } + + if (id != libwebm::kMkvBlock) { + pos += size; // consume sub-part of block group + + if (pos > payload_stop) + return E_FILE_FORMAT_INVALID; + + continue; + } + + const long long block_stop = pos + size; + + if (block_stop > payload_stop) + return E_FILE_FORMAT_INVALID; + + // parse track number + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // weird + return E_BUFFER_NOT_FULL; + + if ((pos + len) > block_stop) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long track = ReadUInt(pReader, pos, len); + + if (track < 0) // error + return static_cast(track); + + if (track == 0) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume track number + + if ((pos + 2) > block_stop) + return E_FILE_FORMAT_INVALID; + + if ((pos + 2) > avail) { + len = 2; + return E_BUFFER_NOT_FULL; + } + + pos += 2; // consume timecode + + if ((pos + 1) > block_stop) + return E_FILE_FORMAT_INVALID; + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + unsigned char flags; + + status = pReader->Read(pos, 1, &flags); + + if (status < 0) { // error or underflow + len = 1; + return status; + } + + ++pos; // consume flags byte + assert(pos <= avail); + + if (pos >= block_stop) + return E_FILE_FORMAT_INVALID; + + const int lacing = int(flags & 0x06) >> 1; + + if ((lacing != 0) && (block_stop > avail)) { + len = static_cast(block_stop - pos); + return E_BUFFER_NOT_FULL; + } + + pos = block_stop; // consume block-part of block group + if (pos > payload_stop) + return E_FILE_FORMAT_INVALID; + } + + if (pos != payload_stop) + return E_FILE_FORMAT_INVALID; + + status = CreateBlock(libwebm::kMkvBlockGroup, payload_start, payload_size, + discard_padding); + if (status != 0) + return status; + + m_pos = payload_stop; + + return 0; // success +} + +long Cluster::GetEntry(long index, const mkvparser::BlockEntry*& pEntry) const { + assert(m_pos >= m_element_start); + + pEntry = NULL; + + if (index < 0) + return -1; // generic error + + if (m_entries_count < 0) + return E_BUFFER_NOT_FULL; + + assert(m_entries); + assert(m_entries_size > 0); + assert(m_entries_count <= m_entries_size); + + if (index < m_entries_count) { + pEntry = m_entries[index]; + assert(pEntry); + + return 1; // found entry + } + + if (m_element_size < 0) // we don't know cluster end yet + return E_BUFFER_NOT_FULL; // underflow + + const long long element_stop = m_element_start + m_element_size; + + if (m_pos >= element_stop) + return 0; // nothing left to parse + + return E_BUFFER_NOT_FULL; // underflow, since more remains to be parsed +} + +Cluster* Cluster::Create(Segment* pSegment, long idx, long long off) { + if (!pSegment || off < 0) + return NULL; + + const long long element_start = pSegment->m_start + off; + + Cluster* const pCluster = + new (std::nothrow) Cluster(pSegment, idx, element_start); + + return pCluster; +} + +Cluster::Cluster() + : m_pSegment(NULL), + m_element_start(0), + m_index(0), + m_pos(0), + m_element_size(0), + m_timecode(0), + m_entries(NULL), + m_entries_size(0), + m_entries_count(0) // means "no entries" +{} + +Cluster::Cluster(Segment* pSegment, long idx, long long element_start + /* long long element_size */) + : m_pSegment(pSegment), + m_element_start(element_start), + m_index(idx), + m_pos(element_start), + m_element_size(-1 /* element_size */), + m_timecode(-1), + m_entries(NULL), + m_entries_size(0), + m_entries_count(-1) // means "has not been parsed yet" +{} + +Cluster::~Cluster() { + if (m_entries_count <= 0) + return; + + BlockEntry** i = m_entries; + BlockEntry** const j = m_entries + m_entries_count; + + while (i != j) { + BlockEntry* p = *i++; + assert(p); + + delete p; + } + + delete[] m_entries; +} + +bool Cluster::EOS() const { return (m_pSegment == NULL); } + +long Cluster::GetIndex() const { return m_index; } + +long long Cluster::GetPosition() const { + const long long pos = m_element_start - m_pSegment->m_start; + assert(pos >= 0); + + return pos; +} + +long long Cluster::GetElementSize() const { return m_element_size; } + +long Cluster::HasBlockEntries( + const Segment* pSegment, + long long off, // relative to start of segment payload + long long& pos, long& len) { + assert(pSegment); + assert(off >= 0); // relative to segment + + IMkvReader* const pReader = pSegment->m_pReader; + + long long total, avail; + + long status = pReader->Length(&total, &avail); + + if (status < 0) // error + return status; + + assert((total < 0) || (avail <= total)); + + pos = pSegment->m_start + off; // absolute + + if ((total >= 0) && (pos >= total)) + return 0; // we don't even have a complete cluster + + const long long segment_stop = + (pSegment->m_size < 0) ? -1 : pSegment->m_start + pSegment->m_size; + + long long cluster_stop = -1; // interpreted later to mean "unknown size" + + { + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + long long result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // need more data + return E_BUFFER_NOT_FULL; + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((total >= 0) && ((pos + len) > total)) + return 0; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long id = ReadID(pReader, pos, len); + + if (id < 0) // error + return static_cast(id); + + if (id != libwebm::kMkvCluster) + return E_PARSE_FAILED; + + pos += len; // consume Cluster ID field + + // read size field + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // weird + return E_BUFFER_NOT_FULL; + + if ((segment_stop >= 0) && ((pos + len) > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((total >= 0) && ((pos + len) > total)) + return 0; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long size = ReadUInt(pReader, pos, len); + + if (size < 0) // error + return static_cast(size); + + if (size == 0) + return 0; // cluster does not have entries + + pos += len; // consume size field + + // pos now points to start of payload + + const long long unknown_size = (1LL << (7 * len)) - 1; + + if (size != unknown_size) { + cluster_stop = pos + size; + assert(cluster_stop >= 0); + + if ((segment_stop >= 0) && (cluster_stop > segment_stop)) + return E_FILE_FORMAT_INVALID; + + if ((total >= 0) && (cluster_stop > total)) + // return E_FILE_FORMAT_INVALID; //too conservative + return 0; // cluster does not have any entries + } + } + + for (;;) { + if ((cluster_stop >= 0) && (pos >= cluster_stop)) + return 0; // no entries detected + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + long long result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // need more data + return E_BUFFER_NOT_FULL; + + if ((cluster_stop >= 0) && ((pos + len) > cluster_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long id = ReadID(pReader, pos, len); + + if (id < 0) // error + return static_cast(id); + + // This is the distinguished set of ID's we use to determine + // that we have exhausted the sub-element's inside the cluster + // whose ID we parsed earlier. + + if (id == libwebm::kMkvCluster) + return 0; // no entries found + + if (id == libwebm::kMkvCues) + return 0; // no entries found + + pos += len; // consume id field + + if ((cluster_stop >= 0) && (pos >= cluster_stop)) + return E_FILE_FORMAT_INVALID; + + // read size field + + if ((pos + 1) > avail) { + len = 1; + return E_BUFFER_NOT_FULL; + } + + result = GetUIntLength(pReader, pos, len); + + if (result < 0) // error + return static_cast(result); + + if (result > 0) // underflow + return E_BUFFER_NOT_FULL; + + if ((cluster_stop >= 0) && ((pos + len) > cluster_stop)) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > avail) + return E_BUFFER_NOT_FULL; + + const long long size = ReadUInt(pReader, pos, len); + + if (size < 0) // error + return static_cast(size); + + pos += len; // consume size field + + // pos now points to start of payload + + if ((cluster_stop >= 0) && (pos > cluster_stop)) + return E_FILE_FORMAT_INVALID; + + if (size == 0) // weird + continue; + + const long long unknown_size = (1LL << (7 * len)) - 1; + + if (size == unknown_size) + return E_FILE_FORMAT_INVALID; // not supported inside cluster + + if ((cluster_stop >= 0) && ((pos + size) > cluster_stop)) + return E_FILE_FORMAT_INVALID; + + if (id == libwebm::kMkvBlockGroup) + return 1; // have at least one entry + + if (id == libwebm::kMkvSimpleBlock) + return 1; // have at least one entry + + pos += size; // consume payload + if (cluster_stop >= 0 && pos > cluster_stop) + return E_FILE_FORMAT_INVALID; + } +} + +long long Cluster::GetTimeCode() const { + long long pos; + long len; + + const long status = Load(pos, len); + + if (status < 0) // error + return status; + + return m_timecode; +} + +long long Cluster::GetTime() const { + const long long tc = GetTimeCode(); + + if (tc < 0) + return tc; + + const SegmentInfo* const pInfo = m_pSegment->GetInfo(); + assert(pInfo); + + const long long scale = pInfo->GetTimeCodeScale(); + assert(scale >= 1); + + const long long t = m_timecode * scale; + + return t; +} + +long long Cluster::GetFirstTime() const { + const BlockEntry* pEntry; + + const long status = GetFirst(pEntry); + + if (status < 0) // error + return status; + + if (pEntry == NULL) // empty cluster + return GetTime(); + + const Block* const pBlock = pEntry->GetBlock(); + assert(pBlock); + + return pBlock->GetTime(this); +} + +long long Cluster::GetLastTime() const { + const BlockEntry* pEntry; + + const long status = GetLast(pEntry); + + if (status < 0) // error + return status; + + if (pEntry == NULL) // empty cluster + return GetTime(); + + const Block* const pBlock = pEntry->GetBlock(); + assert(pBlock); + + return pBlock->GetTime(this); +} + +long Cluster::CreateBlock(long long id, + long long pos, // absolute pos of payload + long long size, long long discard_padding) { + if (id != libwebm::kMkvBlockGroup && id != libwebm::kMkvSimpleBlock) + return E_PARSE_FAILED; + + if (m_entries_count < 0) { // haven't parsed anything yet + assert(m_entries == NULL); + assert(m_entries_size == 0); + + m_entries_size = 1024; + m_entries = new (std::nothrow) BlockEntry*[m_entries_size]; + if (m_entries == NULL) + return -1; + + m_entries_count = 0; + } else { + assert(m_entries); + assert(m_entries_size > 0); + assert(m_entries_count <= m_entries_size); + + if (m_entries_count >= m_entries_size) { + const long entries_size = 2 * m_entries_size; + + BlockEntry** const entries = new (std::nothrow) BlockEntry*[entries_size]; + if (entries == NULL) + return -1; + + BlockEntry** src = m_entries; + BlockEntry** const src_end = src + m_entries_count; + + BlockEntry** dst = entries; + + while (src != src_end) + *dst++ = *src++; + + delete[] m_entries; + + m_entries = entries; + m_entries_size = entries_size; + } + } + + if (id == libwebm::kMkvBlockGroup) + return CreateBlockGroup(pos, size, discard_padding); + else + return CreateSimpleBlock(pos, size); +} + +long Cluster::CreateBlockGroup(long long start_offset, long long size, + long long discard_padding) { + assert(m_entries); + assert(m_entries_size > 0); + assert(m_entries_count >= 0); + assert(m_entries_count < m_entries_size); + + IMkvReader* const pReader = m_pSegment->m_pReader; + + long long pos = start_offset; + const long long stop = start_offset + size; + + // For WebM files, there is a bias towards previous reference times + //(in order to support alt-ref frames, which refer back to the previous + // keyframe). Normally a 0 value is not possible, but here we tenatively + // allow 0 as the value of a reference frame, with the interpretation + // that this is a "previous" reference time. + + long long prev = 1; // nonce + long long next = 0; // nonce + long long duration = -1; // really, this is unsigned + + long long bpos = -1; + long long bsize = -1; + + while (pos < stop) { + long len; + const long long id = ReadID(pReader, pos, len); + if (id < 0 || (pos + len) > stop) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume ID + + const long long size = ReadUInt(pReader, pos, len); + assert(size >= 0); // TODO + assert((pos + len) <= stop); + + pos += len; // consume size + + if (id == libwebm::kMkvBlock) { + if (bpos < 0) { // Block ID + bpos = pos; + bsize = size; + } + } else if (id == libwebm::kMkvBlockDuration) { + if (size > 8) + return E_FILE_FORMAT_INVALID; + + duration = UnserializeUInt(pReader, pos, size); + + if (duration < 0) + return E_FILE_FORMAT_INVALID; + } else if (id == libwebm::kMkvReferenceBlock) { + if (size > 8 || size <= 0) + return E_FILE_FORMAT_INVALID; + const long size_ = static_cast(size); + + long long time; + + long status = UnserializeInt(pReader, pos, size_, time); + assert(status == 0); + if (status != 0) + return -1; + + if (time <= 0) // see note above + prev = time; + else + next = time; + } + + pos += size; // consume payload + if (pos > stop) + return E_FILE_FORMAT_INVALID; + } + if (bpos < 0) + return E_FILE_FORMAT_INVALID; + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + assert(bsize >= 0); + + const long idx = m_entries_count; + + BlockEntry** const ppEntry = m_entries + idx; + BlockEntry*& pEntry = *ppEntry; + + pEntry = new (std::nothrow) + BlockGroup(this, idx, bpos, bsize, prev, next, duration, discard_padding); + + if (pEntry == NULL) + return -1; // generic error + + BlockGroup* const p = static_cast(pEntry); + + const long status = p->Parse(); + + if (status == 0) { // success + ++m_entries_count; + return 0; + } + + delete pEntry; + pEntry = 0; + + return status; +} + +long Cluster::CreateSimpleBlock(long long st, long long sz) { + assert(m_entries); + assert(m_entries_size > 0); + assert(m_entries_count >= 0); + assert(m_entries_count < m_entries_size); + + const long idx = m_entries_count; + + BlockEntry** const ppEntry = m_entries + idx; + BlockEntry*& pEntry = *ppEntry; + + pEntry = new (std::nothrow) SimpleBlock(this, idx, st, sz); + + if (pEntry == NULL) + return -1; // generic error + + SimpleBlock* const p = static_cast(pEntry); + + const long status = p->Parse(); + + if (status == 0) { + ++m_entries_count; + return 0; + } + + delete pEntry; + pEntry = 0; + + return status; +} + +long Cluster::GetFirst(const BlockEntry*& pFirst) const { + if (m_entries_count <= 0) { + long long pos; + long len; + + const long status = Parse(pos, len); + + if (status < 0) { // error + pFirst = NULL; + return status; + } + + if (m_entries_count <= 0) { // empty cluster + pFirst = NULL; + return 0; + } + } + + assert(m_entries); + + pFirst = m_entries[0]; + assert(pFirst); + + return 0; // success +} + +long Cluster::GetLast(const BlockEntry*& pLast) const { + for (;;) { + long long pos; + long len; + + const long status = Parse(pos, len); + + if (status < 0) { // error + pLast = NULL; + return status; + } + + if (status > 0) // no new block + break; + } + + if (m_entries_count <= 0) { + pLast = NULL; + return 0; + } + + assert(m_entries); + + const long idx = m_entries_count - 1; + + pLast = m_entries[idx]; + assert(pLast); + + return 0; +} + +long Cluster::GetNext(const BlockEntry* pCurr, const BlockEntry*& pNext) const { + assert(pCurr); + assert(m_entries); + assert(m_entries_count > 0); + + size_t idx = pCurr->GetIndex(); + assert(idx < size_t(m_entries_count)); + assert(m_entries[idx] == pCurr); + + ++idx; + + if (idx >= size_t(m_entries_count)) { + long long pos; + long len; + + const long status = Parse(pos, len); + + if (status < 0) { // error + pNext = NULL; + return status; + } + + if (status > 0) { + pNext = NULL; + return 0; + } + + assert(m_entries); + assert(m_entries_count > 0); + assert(idx < size_t(m_entries_count)); + } + + pNext = m_entries[idx]; + assert(pNext); + + return 0; +} + +long Cluster::GetEntryCount() const { return m_entries_count; } + +const BlockEntry* Cluster::GetEntry(const Track* pTrack, + long long time_ns) const { + assert(pTrack); + + if (m_pSegment == NULL) // this is the special EOS cluster + return pTrack->GetEOS(); + + const BlockEntry* pResult = pTrack->GetEOS(); + + long index = 0; + + for (;;) { + if (index >= m_entries_count) { + long long pos; + long len; + + const long status = Parse(pos, len); + assert(status >= 0); + + if (status > 0) // completely parsed, and no more entries + return pResult; + + if (status < 0) // should never happen + return 0; + + assert(m_entries); + assert(index < m_entries_count); + } + + const BlockEntry* const pEntry = m_entries[index]; + assert(pEntry); + assert(!pEntry->EOS()); + + const Block* const pBlock = pEntry->GetBlock(); + assert(pBlock); + + if (pBlock->GetTrackNumber() != pTrack->GetNumber()) { + ++index; + continue; + } + + if (pTrack->VetEntry(pEntry)) { + if (time_ns < 0) // just want first candidate block + return pEntry; + + const long long ns = pBlock->GetTime(this); + + if (ns > time_ns) + return pResult; + + pResult = pEntry; // have a candidate + } else if (time_ns >= 0) { + const long long ns = pBlock->GetTime(this); + + if (ns > time_ns) + return pResult; + } + + ++index; + } +} + +const BlockEntry* Cluster::GetEntry(const CuePoint& cp, + const CuePoint::TrackPosition& tp) const { + assert(m_pSegment); + const long long tc = cp.GetTimeCode(); + + if (tp.m_block > 0) { + const long block = static_cast(tp.m_block); + const long index = block - 1; + + while (index >= m_entries_count) { + long long pos; + long len; + + const long status = Parse(pos, len); + + if (status < 0) // TODO: can this happen? + return NULL; + + if (status > 0) // nothing remains to be parsed + return NULL; + } + + const BlockEntry* const pEntry = m_entries[index]; + assert(pEntry); + assert(!pEntry->EOS()); + + const Block* const pBlock = pEntry->GetBlock(); + assert(pBlock); + + if ((pBlock->GetTrackNumber() == tp.m_track) && + (pBlock->GetTimeCode(this) == tc)) { + return pEntry; + } + } + + long index = 0; + + for (;;) { + if (index >= m_entries_count) { + long long pos; + long len; + + const long status = Parse(pos, len); + + if (status < 0) // TODO: can this happen? + return NULL; + + if (status > 0) // nothing remains to be parsed + return NULL; + + assert(m_entries); + assert(index < m_entries_count); + } + + const BlockEntry* const pEntry = m_entries[index]; + assert(pEntry); + assert(!pEntry->EOS()); + + const Block* const pBlock = pEntry->GetBlock(); + assert(pBlock); + + if (pBlock->GetTrackNumber() != tp.m_track) { + ++index; + continue; + } + + const long long tc_ = pBlock->GetTimeCode(this); + + if (tc_ < tc) { + ++index; + continue; + } + + if (tc_ > tc) + return NULL; + + const Tracks* const pTracks = m_pSegment->GetTracks(); + assert(pTracks); + + const long tn = static_cast(tp.m_track); + const Track* const pTrack = pTracks->GetTrackByNumber(tn); + + if (pTrack == NULL) + return NULL; + + const long long type = pTrack->GetType(); + + if (type == 2) // audio + return pEntry; + + if (type != 1) // not video + return NULL; + + if (!pBlock->IsKey()) + return NULL; + + return pEntry; + } +} + +BlockEntry::BlockEntry(Cluster* p, long idx) : m_pCluster(p), m_index(idx) {} +BlockEntry::~BlockEntry() {} +const Cluster* BlockEntry::GetCluster() const { return m_pCluster; } +long BlockEntry::GetIndex() const { return m_index; } + +SimpleBlock::SimpleBlock(Cluster* pCluster, long idx, long long start, + long long size) + : BlockEntry(pCluster, idx), m_block(start, size, 0) {} + +long SimpleBlock::Parse() { return m_block.Parse(m_pCluster); } +BlockEntry::Kind SimpleBlock::GetKind() const { return kBlockSimple; } +const Block* SimpleBlock::GetBlock() const { return &m_block; } + +BlockGroup::BlockGroup(Cluster* pCluster, long idx, long long block_start, + long long block_size, long long prev, long long next, + long long duration, long long discard_padding) + : BlockEntry(pCluster, idx), + m_block(block_start, block_size, discard_padding), + m_prev(prev), + m_next(next), + m_duration(duration) {} + +long BlockGroup::Parse() { + const long status = m_block.Parse(m_pCluster); + + if (status) + return status; + + m_block.SetKey((m_prev > 0) && (m_next <= 0)); + + return 0; +} + +BlockEntry::Kind BlockGroup::GetKind() const { return kBlockGroup; } +const Block* BlockGroup::GetBlock() const { return &m_block; } +long long BlockGroup::GetPrevTimeCode() const { return m_prev; } +long long BlockGroup::GetNextTimeCode() const { return m_next; } +long long BlockGroup::GetDurationTimeCode() const { return m_duration; } + +Block::Block(long long start, long long size_, long long discard_padding) + : m_start(start), + m_size(size_), + m_track(0), + m_timecode(-1), + m_flags(0), + m_frames(NULL), + m_frame_count(-1), + m_discard_padding(discard_padding) {} + +Block::~Block() { delete[] m_frames; } + +long Block::Parse(const Cluster* pCluster) { + if (pCluster == NULL) + return -1; + + if (pCluster->m_pSegment == NULL) + return -1; + + assert(m_start >= 0); + assert(m_size >= 0); + assert(m_track <= 0); + assert(m_frames == NULL); + assert(m_frame_count <= 0); + + long long pos = m_start; + const long long stop = m_start + m_size; + + long len; + + IMkvReader* const pReader = pCluster->m_pSegment->m_pReader; + + m_track = ReadUInt(pReader, pos, len); + + if (m_track <= 0) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > stop) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume track number + + if ((stop - pos) < 2) + return E_FILE_FORMAT_INVALID; + + long status; + long long value; + + status = UnserializeInt(pReader, pos, 2, value); + + if (status) + return E_FILE_FORMAT_INVALID; + + if (value < SHRT_MIN) + return E_FILE_FORMAT_INVALID; + + if (value > SHRT_MAX) + return E_FILE_FORMAT_INVALID; + + m_timecode = static_cast(value); + + pos += 2; + + if ((stop - pos) <= 0) + return E_FILE_FORMAT_INVALID; + + status = pReader->Read(pos, 1, &m_flags); + + if (status) + return E_FILE_FORMAT_INVALID; + + const int lacing = int(m_flags & 0x06) >> 1; + + ++pos; // consume flags byte + + if (lacing == 0) { // no lacing + if (pos > stop) + return E_FILE_FORMAT_INVALID; + + m_frame_count = 1; + m_frames = new (std::nothrow) Frame[m_frame_count]; + if (m_frames == NULL) + return -1; + + Frame& f = m_frames[0]; + f.pos = pos; + + const long long frame_size = stop - pos; + + if (frame_size > LONG_MAX || frame_size <= 0) + return E_FILE_FORMAT_INVALID; + + f.len = static_cast(frame_size); + + return 0; // success + } + + if (pos >= stop) + return E_FILE_FORMAT_INVALID; + + unsigned char biased_count; + + status = pReader->Read(pos, 1, &biased_count); + + if (status) + return E_FILE_FORMAT_INVALID; + + ++pos; // consume frame count + if (pos > stop) + return E_FILE_FORMAT_INVALID; + + m_frame_count = int(biased_count) + 1; + + m_frames = new (std::nothrow) Frame[m_frame_count]; + if (m_frames == NULL) + return -1; + + if (!m_frames) + return E_FILE_FORMAT_INVALID; + + if (lacing == 1) { // Xiph + Frame* pf = m_frames; + Frame* const pf_end = pf + m_frame_count; + + long long size = 0; + int frame_count = m_frame_count; + + while (frame_count > 1) { + long frame_size = 0; + + for (;;) { + unsigned char val; + + if (pos >= stop) + return E_FILE_FORMAT_INVALID; + + status = pReader->Read(pos, 1, &val); + + if (status) + return E_FILE_FORMAT_INVALID; + + ++pos; // consume xiph size byte + + frame_size += val; + + if (val < 255) + break; + } + + Frame& f = *pf++; + assert(pf < pf_end); + if (pf >= pf_end) + return E_FILE_FORMAT_INVALID; + + f.pos = 0; // patch later + + if (frame_size <= 0) + return E_FILE_FORMAT_INVALID; + + f.len = frame_size; + size += frame_size; // contribution of this frame + + --frame_count; + } + + if (pf >= pf_end || pos > stop) + return E_FILE_FORMAT_INVALID; + + { + Frame& f = *pf++; + + if (pf != pf_end) + return E_FILE_FORMAT_INVALID; + + f.pos = 0; // patch later + + const long long total_size = stop - pos; + + if (total_size < size) + return E_FILE_FORMAT_INVALID; + + const long long frame_size = total_size - size; + + if (frame_size > LONG_MAX || frame_size <= 0) + return E_FILE_FORMAT_INVALID; + + f.len = static_cast(frame_size); + } + + pf = m_frames; + while (pf != pf_end) { + Frame& f = *pf++; + assert((pos + f.len) <= stop); + + if ((pos + f.len) > stop) + return E_FILE_FORMAT_INVALID; + + f.pos = pos; + pos += f.len; + } + + assert(pos == stop); + if (pos != stop) + return E_FILE_FORMAT_INVALID; + + } else if (lacing == 2) { // fixed-size lacing + if (pos >= stop) + return E_FILE_FORMAT_INVALID; + + const long long total_size = stop - pos; + + if ((total_size % m_frame_count) != 0) + return E_FILE_FORMAT_INVALID; + + const long long frame_size = total_size / m_frame_count; + + if (frame_size > LONG_MAX || frame_size <= 0) + return E_FILE_FORMAT_INVALID; + + Frame* pf = m_frames; + Frame* const pf_end = pf + m_frame_count; + + while (pf != pf_end) { + assert((pos + frame_size) <= stop); + if ((pos + frame_size) > stop) + return E_FILE_FORMAT_INVALID; + + Frame& f = *pf++; + + f.pos = pos; + f.len = static_cast(frame_size); + + pos += frame_size; + } + + assert(pos == stop); + if (pos != stop) + return E_FILE_FORMAT_INVALID; + + } else { + assert(lacing == 3); // EBML lacing + + if (pos >= stop) + return E_FILE_FORMAT_INVALID; + + long long size = 0; + int frame_count = m_frame_count; + + long long frame_size = ReadUInt(pReader, pos, len); + + if (frame_size <= 0) + return E_FILE_FORMAT_INVALID; + + if (frame_size > LONG_MAX) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > stop) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume length of size of first frame + + if ((pos + frame_size) > stop) + return E_FILE_FORMAT_INVALID; + + Frame* pf = m_frames; + Frame* const pf_end = pf + m_frame_count; + + { + Frame& curr = *pf; + + curr.pos = 0; // patch later + + curr.len = static_cast(frame_size); + size += curr.len; // contribution of this frame + } + + --frame_count; + + while (frame_count > 1) { + if (pos >= stop) + return E_FILE_FORMAT_INVALID; + + assert(pf < pf_end); + if (pf >= pf_end) + return E_FILE_FORMAT_INVALID; + + const Frame& prev = *pf++; + assert(prev.len == frame_size); + if (prev.len != frame_size) + return E_FILE_FORMAT_INVALID; + + assert(pf < pf_end); + if (pf >= pf_end) + return E_FILE_FORMAT_INVALID; + + Frame& curr = *pf; + + curr.pos = 0; // patch later + + const long long delta_size_ = ReadUInt(pReader, pos, len); + + if (delta_size_ < 0) + return E_FILE_FORMAT_INVALID; + + if ((pos + len) > stop) + return E_FILE_FORMAT_INVALID; + + pos += len; // consume length of (delta) size + if (pos > stop) + return E_FILE_FORMAT_INVALID; + + const long exp = 7 * len - 1; + const long long bias = (1LL << exp) - 1LL; + const long long delta_size = delta_size_ - bias; + + frame_size += delta_size; + + if (frame_size <= 0) + return E_FILE_FORMAT_INVALID; + + if (frame_size > LONG_MAX) + return E_FILE_FORMAT_INVALID; + + curr.len = static_cast(frame_size); + size += curr.len; // contribution of this frame + + --frame_count; + } + + // parse last frame + if (frame_count > 0) { + if (pos > stop || pf >= pf_end) + return E_FILE_FORMAT_INVALID; + + const Frame& prev = *pf++; + assert(prev.len == frame_size); + if (prev.len != frame_size) + return E_FILE_FORMAT_INVALID; + + if (pf >= pf_end) + return E_FILE_FORMAT_INVALID; + + Frame& curr = *pf++; + if (pf != pf_end) + return E_FILE_FORMAT_INVALID; + + curr.pos = 0; // patch later + + const long long total_size = stop - pos; + + if (total_size < size) + return E_FILE_FORMAT_INVALID; + + frame_size = total_size - size; + + if (frame_size > LONG_MAX || frame_size <= 0) + return E_FILE_FORMAT_INVALID; + + curr.len = static_cast(frame_size); + } + + pf = m_frames; + while (pf != pf_end) { + Frame& f = *pf++; + assert((pos + f.len) <= stop); + if ((pos + f.len) > stop) + return E_FILE_FORMAT_INVALID; + + f.pos = pos; + pos += f.len; + } + + if (pos != stop) + return E_FILE_FORMAT_INVALID; + } + + return 0; // success +} + +long long Block::GetTimeCode(const Cluster* pCluster) const { + if (pCluster == 0) + return m_timecode; + + const long long tc0 = pCluster->GetTimeCode(); + assert(tc0 >= 0); + + const long long tc = tc0 + m_timecode; + + return tc; // unscaled timecode units +} + +long long Block::GetTime(const Cluster* pCluster) const { + assert(pCluster); + + const long long tc = GetTimeCode(pCluster); + + const Segment* const pSegment = pCluster->m_pSegment; + const SegmentInfo* const pInfo = pSegment->GetInfo(); + assert(pInfo); + + const long long scale = pInfo->GetTimeCodeScale(); + assert(scale >= 1); + + const long long ns = tc * scale; + + return ns; +} + +long long Block::GetTrackNumber() const { return m_track; } + +bool Block::IsKey() const { + return ((m_flags & static_cast(1 << 7)) != 0); +} + +void Block::SetKey(bool bKey) { + if (bKey) + m_flags |= static_cast(1 << 7); + else + m_flags &= 0x7F; +} + +bool Block::IsInvisible() const { return bool(int(m_flags & 0x08) != 0); } + +Block::Lacing Block::GetLacing() const { + const int value = int(m_flags & 0x06) >> 1; + return static_cast(value); +} + +int Block::GetFrameCount() const { return m_frame_count; } + +const Block::Frame& Block::GetFrame(int idx) const { + assert(idx >= 0); + assert(idx < m_frame_count); + + const Frame& f = m_frames[idx]; + assert(f.pos > 0); + assert(f.len > 0); + + return f; +} + +long Block::Frame::Read(IMkvReader* pReader, unsigned char* buf) const { + assert(pReader); + assert(buf); + + const long status = pReader->Read(pos, len, buf); + return status; +} + +long long Block::GetDiscardPadding() const { return m_discard_padding; } + +} // namespace mkvparser diff --git a/third_party/aom/third_party/libwebm/mkvparser/mkvparser.h b/third_party/aom/third_party/libwebm/mkvparser/mkvparser.h new file mode 100644 index 0000000000..42e6e88ab4 --- /dev/null +++ b/third_party/aom/third_party/libwebm/mkvparser/mkvparser.h @@ -0,0 +1,1112 @@ +// Copyright (c) 2012 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +#ifndef MKVPARSER_MKVPARSER_H_ +#define MKVPARSER_MKVPARSER_H_ + +#include + +namespace mkvparser { + +const int E_PARSE_FAILED = -1; +const int E_FILE_FORMAT_INVALID = -2; +const int E_BUFFER_NOT_FULL = -3; + +class IMkvReader { + public: + virtual int Read(long long pos, long len, unsigned char* buf) = 0; + virtual int Length(long long* total, long long* available) = 0; + + protected: + virtual ~IMkvReader(); +}; + +template +Type* SafeArrayAlloc(unsigned long long num_elements, + unsigned long long element_size); +long long GetUIntLength(IMkvReader*, long long, long&); +long long ReadUInt(IMkvReader*, long long, long&); +long long ReadID(IMkvReader* pReader, long long pos, long& len); +long long UnserializeUInt(IMkvReader*, long long pos, long long size); + +long UnserializeFloat(IMkvReader*, long long pos, long long size, double&); +long UnserializeInt(IMkvReader*, long long pos, long long size, + long long& result); + +long UnserializeString(IMkvReader*, long long pos, long long size, char*& str); + +long ParseElementHeader(IMkvReader* pReader, + long long& pos, // consume id and size fields + long long stop, // if you know size of element's parent + long long& id, long long& size); + +bool Match(IMkvReader*, long long&, unsigned long, long long&); +bool Match(IMkvReader*, long long&, unsigned long, unsigned char*&, size_t&); + +void GetVersion(int& major, int& minor, int& build, int& revision); + +struct EBMLHeader { + EBMLHeader(); + ~EBMLHeader(); + long long m_version; + long long m_readVersion; + long long m_maxIdLength; + long long m_maxSizeLength; + char* m_docType; + long long m_docTypeVersion; + long long m_docTypeReadVersion; + + long long Parse(IMkvReader*, long long&); + void Init(); +}; + +class Segment; +class Track; +class Cluster; + +class Block { + Block(const Block&); + Block& operator=(const Block&); + + public: + const long long m_start; + const long long m_size; + + Block(long long start, long long size, long long discard_padding); + ~Block(); + + long Parse(const Cluster*); + + long long GetTrackNumber() const; + long long GetTimeCode(const Cluster*) const; // absolute, but not scaled + long long GetTime(const Cluster*) const; // absolute, and scaled (ns) + bool IsKey() const; + void SetKey(bool); + bool IsInvisible() const; + + enum Lacing { kLacingNone, kLacingXiph, kLacingFixed, kLacingEbml }; + Lacing GetLacing() const; + + int GetFrameCount() const; // to index frames: [0, count) + + struct Frame { + long long pos; // absolute offset + long len; + + long Read(IMkvReader*, unsigned char*) const; + }; + + const Frame& GetFrame(int frame_index) const; + + long long GetDiscardPadding() const; + + private: + long long m_track; // Track::Number() + short m_timecode; // relative to cluster + unsigned char m_flags; + + Frame* m_frames; + int m_frame_count; + + protected: + const long long m_discard_padding; +}; + +class BlockEntry { + BlockEntry(const BlockEntry&); + BlockEntry& operator=(const BlockEntry&); + + protected: + BlockEntry(Cluster*, long index); + + public: + virtual ~BlockEntry(); + + bool EOS() const { return (GetKind() == kBlockEOS); } + const Cluster* GetCluster() const; + long GetIndex() const; + virtual const Block* GetBlock() const = 0; + + enum Kind { kBlockEOS, kBlockSimple, kBlockGroup }; + virtual Kind GetKind() const = 0; + + protected: + Cluster* const m_pCluster; + const long m_index; +}; + +class SimpleBlock : public BlockEntry { + SimpleBlock(const SimpleBlock&); + SimpleBlock& operator=(const SimpleBlock&); + + public: + SimpleBlock(Cluster*, long index, long long start, long long size); + long Parse(); + + Kind GetKind() const; + const Block* GetBlock() const; + + protected: + Block m_block; +}; + +class BlockGroup : public BlockEntry { + BlockGroup(const BlockGroup&); + BlockGroup& operator=(const BlockGroup&); + + public: + BlockGroup(Cluster*, long index, + long long block_start, // absolute pos of block's payload + long long block_size, // size of block's payload + long long prev, long long next, long long duration, + long long discard_padding); + + long Parse(); + + Kind GetKind() const; + const Block* GetBlock() const; + + long long GetPrevTimeCode() const; // relative to block's time + long long GetNextTimeCode() const; // as above + long long GetDurationTimeCode() const; + + private: + Block m_block; + const long long m_prev; + const long long m_next; + const long long m_duration; +}; + +/////////////////////////////////////////////////////////////// +// ContentEncoding element +// Elements used to describe if the track data has been encrypted or +// compressed with zlib or header stripping. +class ContentEncoding { + public: + enum { kCTR = 1 }; + + ContentEncoding(); + ~ContentEncoding(); + + // ContentCompression element names + struct ContentCompression { + ContentCompression(); + ~ContentCompression(); + + unsigned long long algo; + unsigned char* settings; + long long settings_len; + }; + + // ContentEncAESSettings element names + struct ContentEncAESSettings { + ContentEncAESSettings() : cipher_mode(kCTR) {} + ~ContentEncAESSettings() {} + + unsigned long long cipher_mode; + }; + + // ContentEncryption element names + struct ContentEncryption { + ContentEncryption(); + ~ContentEncryption(); + + unsigned long long algo; + unsigned char* key_id; + long long key_id_len; + unsigned char* signature; + long long signature_len; + unsigned char* sig_key_id; + long long sig_key_id_len; + unsigned long long sig_algo; + unsigned long long sig_hash_algo; + + ContentEncAESSettings aes_settings; + }; + + // Returns ContentCompression represented by |idx|. Returns NULL if |idx| + // is out of bounds. + const ContentCompression* GetCompressionByIndex(unsigned long idx) const; + + // Returns number of ContentCompression elements in this ContentEncoding + // element. + unsigned long GetCompressionCount() const; + + // Parses the ContentCompression element from |pReader|. |start| is the + // starting offset of the ContentCompression payload. |size| is the size in + // bytes of the ContentCompression payload. |compression| is where the parsed + // values will be stored. + long ParseCompressionEntry(long long start, long long size, + IMkvReader* pReader, + ContentCompression* compression); + + // Returns ContentEncryption represented by |idx|. Returns NULL if |idx| + // is out of bounds. + const ContentEncryption* GetEncryptionByIndex(unsigned long idx) const; + + // Returns number of ContentEncryption elements in this ContentEncoding + // element. + unsigned long GetEncryptionCount() const; + + // Parses the ContentEncAESSettings element from |pReader|. |start| is the + // starting offset of the ContentEncAESSettings payload. |size| is the + // size in bytes of the ContentEncAESSettings payload. |encryption| is + // where the parsed values will be stored. + long ParseContentEncAESSettingsEntry(long long start, long long size, + IMkvReader* pReader, + ContentEncAESSettings* aes); + + // Parses the ContentEncoding element from |pReader|. |start| is the + // starting offset of the ContentEncoding payload. |size| is the size in + // bytes of the ContentEncoding payload. Returns true on success. + long ParseContentEncodingEntry(long long start, long long size, + IMkvReader* pReader); + + // Parses the ContentEncryption element from |pReader|. |start| is the + // starting offset of the ContentEncryption payload. |size| is the size in + // bytes of the ContentEncryption payload. |encryption| is where the parsed + // values will be stored. + long ParseEncryptionEntry(long long start, long long size, + IMkvReader* pReader, ContentEncryption* encryption); + + unsigned long long encoding_order() const { return encoding_order_; } + unsigned long long encoding_scope() const { return encoding_scope_; } + unsigned long long encoding_type() const { return encoding_type_; } + + private: + // Member variables for list of ContentCompression elements. + ContentCompression** compression_entries_; + ContentCompression** compression_entries_end_; + + // Member variables for list of ContentEncryption elements. + ContentEncryption** encryption_entries_; + ContentEncryption** encryption_entries_end_; + + // ContentEncoding element names + unsigned long long encoding_order_; + unsigned long long encoding_scope_; + unsigned long long encoding_type_; + + // LIBWEBM_DISALLOW_COPY_AND_ASSIGN(ContentEncoding); + ContentEncoding(const ContentEncoding&); + ContentEncoding& operator=(const ContentEncoding&); +}; + +class Track { + Track(const Track&); + Track& operator=(const Track&); + + public: + class Info; + static long Create(Segment*, const Info&, long long element_start, + long long element_size, Track*&); + + enum Type { kVideo = 1, kAudio = 2, kSubtitle = 0x11, kMetadata = 0x21 }; + + Segment* const m_pSegment; + const long long m_element_start; + const long long m_element_size; + virtual ~Track(); + + long GetType() const; + long GetNumber() const; + unsigned long long GetUid() const; + const char* GetNameAsUTF8() const; + const char* GetLanguage() const; + const char* GetCodecNameAsUTF8() const; + const char* GetCodecId() const; + const unsigned char* GetCodecPrivate(size_t&) const; + bool GetLacing() const; + unsigned long long GetDefaultDuration() const; + unsigned long long GetCodecDelay() const; + unsigned long long GetSeekPreRoll() const; + + const BlockEntry* GetEOS() const; + + struct Settings { + long long start; + long long size; + }; + + class Info { + public: + Info(); + ~Info(); + int Copy(Info&) const; + void Clear(); + long type; + long number; + unsigned long long uid; + unsigned long long defaultDuration; + unsigned long long codecDelay; + unsigned long long seekPreRoll; + char* nameAsUTF8; + char* language; + char* codecId; + char* codecNameAsUTF8; + unsigned char* codecPrivate; + size_t codecPrivateSize; + bool lacing; + Settings settings; + + private: + Info(const Info&); + Info& operator=(const Info&); + int CopyStr(char* Info::*str, Info&) const; + }; + + long GetFirst(const BlockEntry*&) const; + long GetNext(const BlockEntry* pCurr, const BlockEntry*& pNext) const; + virtual bool VetEntry(const BlockEntry*) const; + virtual long Seek(long long time_ns, const BlockEntry*&) const; + + const ContentEncoding* GetContentEncodingByIndex(unsigned long idx) const; + unsigned long GetContentEncodingCount() const; + + long ParseContentEncodingsEntry(long long start, long long size); + + protected: + Track(Segment*, long long element_start, long long element_size); + + Info m_info; + + class EOSBlock : public BlockEntry { + public: + EOSBlock(); + + Kind GetKind() const; + const Block* GetBlock() const; + }; + + EOSBlock m_eos; + + private: + ContentEncoding** content_encoding_entries_; + ContentEncoding** content_encoding_entries_end_; +}; + +struct PrimaryChromaticity { + PrimaryChromaticity() : x(0), y(0) {} + ~PrimaryChromaticity() {} + static bool Parse(IMkvReader* reader, long long read_pos, + long long value_size, bool is_x, + PrimaryChromaticity** chromaticity); + float x; + float y; +}; + +struct MasteringMetadata { + static const float kValueNotPresent; + + MasteringMetadata() + : r(NULL), + g(NULL), + b(NULL), + white_point(NULL), + luminance_max(kValueNotPresent), + luminance_min(kValueNotPresent) {} + ~MasteringMetadata() { + delete r; + delete g; + delete b; + delete white_point; + } + + static bool Parse(IMkvReader* reader, long long element_start, + long long element_size, + MasteringMetadata** mastering_metadata); + + PrimaryChromaticity* r; + PrimaryChromaticity* g; + PrimaryChromaticity* b; + PrimaryChromaticity* white_point; + float luminance_max; + float luminance_min; +}; + +struct Colour { + static const long long kValueNotPresent; + + // Unless otherwise noted all values assigned upon construction are the + // equivalent of unspecified/default. + Colour() + : matrix_coefficients(kValueNotPresent), + bits_per_channel(kValueNotPresent), + chroma_subsampling_horz(kValueNotPresent), + chroma_subsampling_vert(kValueNotPresent), + cb_subsampling_horz(kValueNotPresent), + cb_subsampling_vert(kValueNotPresent), + chroma_siting_horz(kValueNotPresent), + chroma_siting_vert(kValueNotPresent), + range(kValueNotPresent), + transfer_characteristics(kValueNotPresent), + primaries(kValueNotPresent), + max_cll(kValueNotPresent), + max_fall(kValueNotPresent), + mastering_metadata(NULL) {} + ~Colour() { + delete mastering_metadata; + mastering_metadata = NULL; + } + + static bool Parse(IMkvReader* reader, long long element_start, + long long element_size, Colour** colour); + + long long matrix_coefficients; + long long bits_per_channel; + long long chroma_subsampling_horz; + long long chroma_subsampling_vert; + long long cb_subsampling_horz; + long long cb_subsampling_vert; + long long chroma_siting_horz; + long long chroma_siting_vert; + long long range; + long long transfer_characteristics; + long long primaries; + long long max_cll; + long long max_fall; + + MasteringMetadata* mastering_metadata; +}; + +class VideoTrack : public Track { + VideoTrack(const VideoTrack&); + VideoTrack& operator=(const VideoTrack&); + + VideoTrack(Segment*, long long element_start, long long element_size); + + public: + virtual ~VideoTrack(); + static long Parse(Segment*, const Info&, long long element_start, + long long element_size, VideoTrack*&); + + long long GetWidth() const; + long long GetHeight() const; + long long GetDisplayWidth() const; + long long GetDisplayHeight() const; + long long GetDisplayUnit() const; + long long GetStereoMode() const; + double GetFrameRate() const; + + bool VetEntry(const BlockEntry*) const; + long Seek(long long time_ns, const BlockEntry*&) const; + + Colour* GetColour() const; + + private: + long long m_width; + long long m_height; + long long m_display_width; + long long m_display_height; + long long m_display_unit; + long long m_stereo_mode; + + double m_rate; + + Colour* m_colour; +}; + +class AudioTrack : public Track { + AudioTrack(const AudioTrack&); + AudioTrack& operator=(const AudioTrack&); + + AudioTrack(Segment*, long long element_start, long long element_size); + + public: + static long Parse(Segment*, const Info&, long long element_start, + long long element_size, AudioTrack*&); + + double GetSamplingRate() const; + long long GetChannels() const; + long long GetBitDepth() const; + + private: + double m_rate; + long long m_channels; + long long m_bitDepth; +}; + +class Tracks { + Tracks(const Tracks&); + Tracks& operator=(const Tracks&); + + public: + Segment* const m_pSegment; + const long long m_start; + const long long m_size; + const long long m_element_start; + const long long m_element_size; + + Tracks(Segment*, long long start, long long size, long long element_start, + long long element_size); + + ~Tracks(); + + long Parse(); + + unsigned long GetTracksCount() const; + + const Track* GetTrackByNumber(long tn) const; + const Track* GetTrackByIndex(unsigned long idx) const; + + private: + Track** m_trackEntries; + Track** m_trackEntriesEnd; + + long ParseTrackEntry(long long payload_start, long long payload_size, + long long element_start, long long element_size, + Track*&) const; +}; + +class Chapters { + Chapters(const Chapters&); + Chapters& operator=(const Chapters&); + + public: + Segment* const m_pSegment; + const long long m_start; + const long long m_size; + const long long m_element_start; + const long long m_element_size; + + Chapters(Segment*, long long payload_start, long long payload_size, + long long element_start, long long element_size); + + ~Chapters(); + + long Parse(); + + class Atom; + class Edition; + + class Display { + friend class Atom; + Display(); + Display(const Display&); + ~Display(); + Display& operator=(const Display&); + + public: + const char* GetString() const; + const char* GetLanguage() const; + const char* GetCountry() const; + + private: + void Init(); + void ShallowCopy(Display&) const; + void Clear(); + long Parse(IMkvReader*, long long pos, long long size); + + char* m_string; + char* m_language; + char* m_country; + }; + + class Atom { + friend class Edition; + Atom(); + Atom(const Atom&); + ~Atom(); + Atom& operator=(const Atom&); + + public: + unsigned long long GetUID() const; + const char* GetStringUID() const; + + long long GetStartTimecode() const; + long long GetStopTimecode() const; + + long long GetStartTime(const Chapters*) const; + long long GetStopTime(const Chapters*) const; + + int GetDisplayCount() const; + const Display* GetDisplay(int index) const; + + private: + void Init(); + void ShallowCopy(Atom&) const; + void Clear(); + long Parse(IMkvReader*, long long pos, long long size); + static long long GetTime(const Chapters*, long long timecode); + + long ParseDisplay(IMkvReader*, long long pos, long long size); + bool ExpandDisplaysArray(); + + char* m_string_uid; + unsigned long long m_uid; + long long m_start_timecode; + long long m_stop_timecode; + + Display* m_displays; + int m_displays_size; + int m_displays_count; + }; + + class Edition { + friend class Chapters; + Edition(); + Edition(const Edition&); + ~Edition(); + Edition& operator=(const Edition&); + + public: + int GetAtomCount() const; + const Atom* GetAtom(int index) const; + + private: + void Init(); + void ShallowCopy(Edition&) const; + void Clear(); + long Parse(IMkvReader*, long long pos, long long size); + + long ParseAtom(IMkvReader*, long long pos, long long size); + bool ExpandAtomsArray(); + + Atom* m_atoms; + int m_atoms_size; + int m_atoms_count; + }; + + int GetEditionCount() const; + const Edition* GetEdition(int index) const; + + private: + long ParseEdition(long long pos, long long size); + bool ExpandEditionsArray(); + + Edition* m_editions; + int m_editions_size; + int m_editions_count; +}; + +class Tags { + Tags(const Tags&); + Tags& operator=(const Tags&); + + public: + Segment* const m_pSegment; + const long long m_start; + const long long m_size; + const long long m_element_start; + const long long m_element_size; + + Tags(Segment*, long long payload_start, long long payload_size, + long long element_start, long long element_size); + + ~Tags(); + + long Parse(); + + class Tag; + class SimpleTag; + + class SimpleTag { + friend class Tag; + SimpleTag(); + SimpleTag(const SimpleTag&); + ~SimpleTag(); + SimpleTag& operator=(const SimpleTag&); + + public: + const char* GetTagName() const; + const char* GetTagString() const; + + private: + void Init(); + void ShallowCopy(SimpleTag&) const; + void Clear(); + long Parse(IMkvReader*, long long pos, long long size); + + char* m_tag_name; + char* m_tag_string; + }; + + class Tag { + friend class Tags; + Tag(); + Tag(const Tag&); + ~Tag(); + Tag& operator=(const Tag&); + + public: + int GetSimpleTagCount() const; + const SimpleTag* GetSimpleTag(int index) const; + + private: + void Init(); + void ShallowCopy(Tag&) const; + void Clear(); + long Parse(IMkvReader*, long long pos, long long size); + + long ParseSimpleTag(IMkvReader*, long long pos, long long size); + bool ExpandSimpleTagsArray(); + + SimpleTag* m_simple_tags; + int m_simple_tags_size; + int m_simple_tags_count; + }; + + int GetTagCount() const; + const Tag* GetTag(int index) const; + + private: + long ParseTag(long long pos, long long size); + bool ExpandTagsArray(); + + Tag* m_tags; + int m_tags_size; + int m_tags_count; +}; + +class SegmentInfo { + SegmentInfo(const SegmentInfo&); + SegmentInfo& operator=(const SegmentInfo&); + + public: + Segment* const m_pSegment; + const long long m_start; + const long long m_size; + const long long m_element_start; + const long long m_element_size; + + SegmentInfo(Segment*, long long start, long long size, + long long element_start, long long element_size); + + ~SegmentInfo(); + + long Parse(); + + long long GetTimeCodeScale() const; + long long GetDuration() const; // scaled + const char* GetMuxingAppAsUTF8() const; + const char* GetWritingAppAsUTF8() const; + const char* GetTitleAsUTF8() const; + + private: + long long m_timecodeScale; + double m_duration; + char* m_pMuxingAppAsUTF8; + char* m_pWritingAppAsUTF8; + char* m_pTitleAsUTF8; +}; + +class SeekHead { + SeekHead(const SeekHead&); + SeekHead& operator=(const SeekHead&); + + public: + Segment* const m_pSegment; + const long long m_start; + const long long m_size; + const long long m_element_start; + const long long m_element_size; + + SeekHead(Segment*, long long start, long long size, long long element_start, + long long element_size); + + ~SeekHead(); + + long Parse(); + + struct Entry { + // the SeekHead entry payload + long long id; + long long pos; + + // absolute pos of SeekEntry ID + long long element_start; + + // SeekEntry ID size + size size + payload + long long element_size; + }; + + int GetCount() const; + const Entry* GetEntry(int idx) const; + + struct VoidElement { + // absolute pos of Void ID + long long element_start; + + // ID size + size size + payload size + long long element_size; + }; + + int GetVoidElementCount() const; + const VoidElement* GetVoidElement(int idx) const; + + private: + Entry* m_entries; + int m_entry_count; + + VoidElement* m_void_elements; + int m_void_element_count; + + static bool ParseEntry(IMkvReader*, + long long pos, // payload + long long size, Entry*); +}; + +class Cues; +class CuePoint { + friend class Cues; + + CuePoint(long, long long); + ~CuePoint(); + + CuePoint(const CuePoint&); + CuePoint& operator=(const CuePoint&); + + public: + long long m_element_start; + long long m_element_size; + + bool Load(IMkvReader*); + + long long GetTimeCode() const; // absolute but unscaled + long long GetTime(const Segment*) const; // absolute and scaled (ns units) + + struct TrackPosition { + long long m_track; + long long m_pos; // of cluster + long long m_block; + // codec_state //defaults to 0 + // reference = clusters containing req'd referenced blocks + // reftime = timecode of the referenced block + + bool Parse(IMkvReader*, long long, long long); + }; + + const TrackPosition* Find(const Track*) const; + + private: + const long m_index; + long long m_timecode; + TrackPosition* m_track_positions; + size_t m_track_positions_count; +}; + +class Cues { + friend class Segment; + + Cues(Segment*, long long start, long long size, long long element_start, + long long element_size); + ~Cues(); + + Cues(const Cues&); + Cues& operator=(const Cues&); + + public: + Segment* const m_pSegment; + const long long m_start; + const long long m_size; + const long long m_element_start; + const long long m_element_size; + + bool Find( // lower bound of time_ns + long long time_ns, const Track*, const CuePoint*&, + const CuePoint::TrackPosition*&) const; + + const CuePoint* GetFirst() const; + const CuePoint* GetLast() const; + const CuePoint* GetNext(const CuePoint*) const; + + const BlockEntry* GetBlock(const CuePoint*, + const CuePoint::TrackPosition*) const; + + bool LoadCuePoint() const; + long GetCount() const; // loaded only + // long GetTotal() const; //loaded + preloaded + bool DoneParsing() const; + + private: + bool Init() const; + bool PreloadCuePoint(long&, long long) const; + + mutable CuePoint** m_cue_points; + mutable long m_count; + mutable long m_preload_count; + mutable long long m_pos; +}; + +class Cluster { + friend class Segment; + + Cluster(const Cluster&); + Cluster& operator=(const Cluster&); + + public: + Segment* const m_pSegment; + + public: + static Cluster* Create(Segment*, + long index, // index in segment + long long off); // offset relative to segment + // long long element_size); + + Cluster(); // EndOfStream + ~Cluster(); + + bool EOS() const; + + long long GetTimeCode() const; // absolute, but not scaled + long long GetTime() const; // absolute, and scaled (nanosecond units) + long long GetFirstTime() const; // time (ns) of first (earliest) block + long long GetLastTime() const; // time (ns) of last (latest) block + + long GetFirst(const BlockEntry*&) const; + long GetLast(const BlockEntry*&) const; + long GetNext(const BlockEntry* curr, const BlockEntry*& next) const; + + const BlockEntry* GetEntry(const Track*, long long ns = -1) const; + const BlockEntry* GetEntry(const CuePoint&, + const CuePoint::TrackPosition&) const; + // const BlockEntry* GetMaxKey(const VideoTrack*) const; + + // static bool HasBlockEntries(const Segment*, long long); + + static long HasBlockEntries(const Segment*, long long idoff, long long& pos, + long& size); + + long GetEntryCount() const; + + long Load(long long& pos, long& size) const; + + long Parse(long long& pos, long& size) const; + long GetEntry(long index, const mkvparser::BlockEntry*&) const; + + protected: + Cluster(Segment*, long index, long long element_start); + // long long element_size); + + public: + const long long m_element_start; + long long GetPosition() const; // offset relative to segment + + long GetIndex() const; + long long GetElementSize() const; + // long long GetPayloadSize() const; + + // long long Unparsed() const; + + private: + long m_index; + mutable long long m_pos; + // mutable long long m_size; + mutable long long m_element_size; + mutable long long m_timecode; + mutable BlockEntry** m_entries; + mutable long m_entries_size; + mutable long m_entries_count; + + long ParseSimpleBlock(long long, long long&, long&); + long ParseBlockGroup(long long, long long&, long&); + + long CreateBlock(long long id, long long pos, long long size, + long long discard_padding); + long CreateBlockGroup(long long start_offset, long long size, + long long discard_padding); + long CreateSimpleBlock(long long, long long); +}; + +class Segment { + friend class Cues; + friend class Track; + friend class VideoTrack; + + Segment(const Segment&); + Segment& operator=(const Segment&); + + private: + Segment(IMkvReader*, long long elem_start, + // long long elem_size, + long long pos, long long size); + + public: + IMkvReader* const m_pReader; + const long long m_element_start; + // const long long m_element_size; + const long long m_start; // posn of segment payload + const long long m_size; // size of segment payload + Cluster m_eos; // TODO: make private? + + static long long CreateInstance(IMkvReader*, long long, Segment*&); + ~Segment(); + + long Load(); // loads headers and all clusters + + // for incremental loading + // long long Unparsed() const; + bool DoneParsing() const; + long long ParseHeaders(); // stops when first cluster is found + // long FindNextCluster(long long& pos, long& size) const; + long LoadCluster(long long& pos, long& size); // load one cluster + long LoadCluster(); + + long ParseNext(const Cluster* pCurr, const Cluster*& pNext, long long& pos, + long& size); + + const SeekHead* GetSeekHead() const; + const Tracks* GetTracks() const; + const SegmentInfo* GetInfo() const; + const Cues* GetCues() const; + const Chapters* GetChapters() const; + const Tags* GetTags() const; + + long long GetDuration() const; + + unsigned long GetCount() const; + const Cluster* GetFirst() const; + const Cluster* GetLast() const; + const Cluster* GetNext(const Cluster*); + + const Cluster* FindCluster(long long time_nanoseconds) const; + // const BlockEntry* Seek(long long time_nanoseconds, const Track*) const; + + const Cluster* FindOrPreloadCluster(long long pos); + + long ParseCues(long long cues_off, // offset relative to start of segment + long long& parse_pos, long& parse_len); + + private: + long long m_pos; // absolute file posn; what has been consumed so far + Cluster* m_pUnknownSize; + + SeekHead* m_pSeekHead; + SegmentInfo* m_pInfo; + Tracks* m_pTracks; + Cues* m_pCues; + Chapters* m_pChapters; + Tags* m_pTags; + Cluster** m_clusters; + long m_clusterCount; // number of entries for which m_index >= 0 + long m_clusterPreloadCount; // number of entries for which m_index < 0 + long m_clusterSize; // array size + + long DoLoadCluster(long long&, long&); + long DoLoadClusterUnknownSize(long long&, long&); + long DoParseNext(const Cluster*&, long long&, long&); + + bool AppendCluster(Cluster*); + bool PreloadCluster(Cluster*, ptrdiff_t); + + // void ParseSeekHead(long long pos, long long size); + // void ParseSeekEntry(long long pos, long long size); + // void ParseCues(long long); + + const BlockEntry* GetBlock(const CuePoint&, const CuePoint::TrackPosition&); +}; + +} // namespace mkvparser + +inline long mkvparser::Segment::LoadCluster() { + long long pos; + long size; + + return LoadCluster(pos, size); +} + +#endif // MKVPARSER_MKVPARSER_H_ diff --git a/third_party/aom/third_party/libwebm/mkvparser/mkvreader.cc b/third_party/aom/third_party/libwebm/mkvparser/mkvreader.cc new file mode 100644 index 0000000000..9f90d8c4f8 --- /dev/null +++ b/third_party/aom/third_party/libwebm/mkvparser/mkvreader.cc @@ -0,0 +1,131 @@ +// Copyright (c) 2010 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +#include "mkvparser/mkvreader.h" + +#include + +namespace mkvparser { + +MkvReader::MkvReader() : m_file(NULL), reader_owns_file_(true) {} + +MkvReader::MkvReader(FILE* fp) : m_file(fp), reader_owns_file_(false) { + GetFileSize(); +} + +MkvReader::~MkvReader() { + if (reader_owns_file_) + Close(); + m_file = NULL; +} + +int MkvReader::Open(const char* fileName) { + if (fileName == NULL) + return -1; + + if (m_file) + return -1; + +#ifdef _MSC_VER + const errno_t e = fopen_s(&m_file, fileName, "rb"); + + if (e) + return -1; // error +#else + m_file = fopen(fileName, "rb"); + + if (m_file == NULL) + return -1; +#endif + return !GetFileSize(); +} + +bool MkvReader::GetFileSize() { + if (m_file == NULL) + return false; +#ifdef _MSC_VER + int status = _fseeki64(m_file, 0L, SEEK_END); + + if (status) + return false; // error + + m_length = _ftelli64(m_file); +#else + fseek(m_file, 0L, SEEK_END); + m_length = ftell(m_file); +#endif + assert(m_length >= 0); + + if (m_length < 0) + return false; + +#ifdef _MSC_VER + status = _fseeki64(m_file, 0L, SEEK_SET); + + if (status) + return false; // error +#else + fseek(m_file, 0L, SEEK_SET); +#endif + + return true; +} + +void MkvReader::Close() { + if (m_file != NULL) { + fclose(m_file); + m_file = NULL; + } +} + +int MkvReader::Length(long long* total, long long* available) { + if (m_file == NULL) + return -1; + + if (total) + *total = m_length; + + if (available) + *available = m_length; + + return 0; +} + +int MkvReader::Read(long long offset, long len, unsigned char* buffer) { + if (m_file == NULL) + return -1; + + if (offset < 0) + return -1; + + if (len < 0) + return -1; + + if (len == 0) + return 0; + + if (offset >= m_length) + return -1; + +#ifdef _MSC_VER + const int status = _fseeki64(m_file, offset, SEEK_SET); + + if (status) + return -1; // error +#else + fseek(m_file, offset, SEEK_SET); +#endif + + const size_t size = fread(buffer, 1, len, m_file); + + if (size < size_t(len)) + return -1; // error + + return 0; // success +} + +} // namespace mkvparser \ No newline at end of file diff --git a/third_party/aom/third_party/libwebm/mkvparser/mkvreader.h b/third_party/aom/third_party/libwebm/mkvparser/mkvreader.h new file mode 100644 index 0000000000..9831ecf645 --- /dev/null +++ b/third_party/aom/third_party/libwebm/mkvparser/mkvreader.h @@ -0,0 +1,45 @@ +// Copyright (c) 2010 The WebM project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +#ifndef MKVPARSER_MKVREADER_H_ +#define MKVPARSER_MKVREADER_H_ + +#include + +#include "mkvparser/mkvparser.h" + +namespace mkvparser { + +class MkvReader : public IMkvReader { + public: + MkvReader(); + explicit MkvReader(FILE* fp); + virtual ~MkvReader(); + + int Open(const char*); + void Close(); + + virtual int Read(long long position, long length, unsigned char* buffer); + virtual int Length(long long* total, long long* available); + + private: + MkvReader(const MkvReader&); + MkvReader& operator=(const MkvReader&); + + // Determines the size of the file. This is called either by the constructor + // or by the Open function depending on file ownership. Returns true on + // success. + bool GetFileSize(); + + long long m_length; + FILE* m_file; + bool reader_owns_file_; +}; + +} // namespace mkvparser + +#endif // MKVPARSER_MKVREADER_H_ diff --git a/third_party/aom/third_party/libyuv/README.libaom b/third_party/aom/third_party/libyuv/README.libaom new file mode 100644 index 0000000000..09693c1f2c --- /dev/null +++ b/third_party/aom/third_party/libyuv/README.libaom @@ -0,0 +1,15 @@ +Name: libyuv +URL: http://code.google.com/p/libyuv/ +Version: 1456 +License: BSD +License File: LICENSE + +Description: +libyuv is an open source project that includes YUV conversion and scaling +functionality. + +The optimized scaler in libyuv is used in multiple resolution encoder example, +which down-samples the original input video (f.g. 1280x720) a number of times +in order to encode multiple resolution bit streams. + +Local Modifications: diff --git a/third_party/aom/third_party/libyuv/include/libyuv/basic_types.h b/third_party/aom/third_party/libyuv/include/libyuv/basic_types.h new file mode 100644 index 0000000000..66e68536cb --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/basic_types.h @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_BASIC_TYPES_H_ // NOLINT +#define INCLUDE_LIBYUV_BASIC_TYPES_H_ + +#include // for NULL, size_t + +#if defined(__ANDROID__) || (defined(_MSC_VER) && (_MSC_VER < 1600)) +#include // for uintptr_t on x86 +#else +#include // for uintptr_t +#endif + +#ifndef GG_LONGLONG +#ifndef INT_TYPES_DEFINED +#define INT_TYPES_DEFINED +#ifdef COMPILER_MSVC +typedef unsigned __int64 uint64; +typedef __int64 int64; +#ifndef INT64_C +#define INT64_C(x) x ## I64 +#endif +#ifndef UINT64_C +#define UINT64_C(x) x ## UI64 +#endif +#define INT64_F "I64" +#else // COMPILER_MSVC +#if defined(__LP64__) && !defined(__OpenBSD__) && !defined(__APPLE__) +typedef unsigned long uint64; // NOLINT +typedef long int64; // NOLINT +#ifndef INT64_C +#define INT64_C(x) x ## L +#endif +#ifndef UINT64_C +#define UINT64_C(x) x ## UL +#endif +#define INT64_F "l" +#else // defined(__LP64__) && !defined(__OpenBSD__) && !defined(__APPLE__) +typedef unsigned long long uint64; // NOLINT +typedef long long int64; // NOLINT +#ifndef INT64_C +#define INT64_C(x) x ## LL +#endif +#ifndef UINT64_C +#define UINT64_C(x) x ## ULL +#endif +#define INT64_F "ll" +#endif // __LP64__ +#endif // COMPILER_MSVC +typedef unsigned int uint32; +typedef int int32; +typedef unsigned short uint16; // NOLINT +typedef short int16; // NOLINT +typedef unsigned char uint8; +typedef signed char int8; +#endif // INT_TYPES_DEFINED +#endif // GG_LONGLONG + +// Detect compiler is for x86 or x64. +#if defined(__x86_64__) || defined(_M_X64) || \ + defined(__i386__) || defined(_M_IX86) +#define CPU_X86 1 +#endif +// Detect compiler is for ARM. +#if defined(__arm__) || defined(_M_ARM) +#define CPU_ARM 1 +#endif + +#ifndef ALIGNP +#ifdef __cplusplus +#define ALIGNP(p, t) \ + (reinterpret_cast(((reinterpret_cast(p) + \ + ((t) - 1)) & ~((t) - 1)))) +#else +#define ALIGNP(p, t) \ + ((uint8*)((((uintptr_t)(p) + ((t) - 1)) & ~((t) - 1)))) /* NOLINT */ +#endif +#endif + +#if !defined(LIBYUV_API) +#if defined(_WIN32) || defined(__CYGWIN__) +#if defined(LIBYUV_BUILDING_SHARED_LIBRARY) +#define LIBYUV_API __declspec(dllexport) +#elif defined(LIBYUV_USING_SHARED_LIBRARY) +#define LIBYUV_API __declspec(dllimport) +#else +#define LIBYUV_API +#endif // LIBYUV_BUILDING_SHARED_LIBRARY +#elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__APPLE__) && \ + (defined(LIBYUV_BUILDING_SHARED_LIBRARY) || \ + defined(LIBYUV_USING_SHARED_LIBRARY)) +#define LIBYUV_API __attribute__ ((visibility ("default"))) +#else +#define LIBYUV_API +#endif // __GNUC__ +#endif // LIBYUV_API + +#define LIBYUV_BOOL int +#define LIBYUV_FALSE 0 +#define LIBYUV_TRUE 1 + +// Visual C x86 or GCC little endian. +#if defined(__x86_64__) || defined(_M_X64) || \ + defined(__i386__) || defined(_M_IX86) || \ + defined(__arm__) || defined(_M_ARM) || \ + (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +#define LIBYUV_LITTLE_ENDIAN +#endif + +#endif // INCLUDE_LIBYUV_BASIC_TYPES_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/compare.h b/third_party/aom/third_party/libyuv/include/libyuv/compare.h new file mode 100644 index 0000000000..2a9f1560ce --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/compare.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_COMPARE_H_ // NOLINT +#define INCLUDE_LIBYUV_COMPARE_H_ + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Compute a hash for specified memory. Seed of 5381 recommended. +LIBYUV_API +uint32 HashDjb2(const uint8* src, uint64 count, uint32 seed); + +// Scan an opaque argb image and return fourcc based on alpha offset. +// Returns FOURCC_ARGB, FOURCC_BGRA, or 0 if unknown. +LIBYUV_API +uint32 ARGBDetect(const uint8* argb, int stride_argb, int width, int height); + +// Sum Square Error - used to compute Mean Square Error or PSNR. +LIBYUV_API +uint64 ComputeSumSquareError(const uint8* src_a, + const uint8* src_b, int count); + +LIBYUV_API +uint64 ComputeSumSquareErrorPlane(const uint8* src_a, int stride_a, + const uint8* src_b, int stride_b, + int width, int height); + +static const int kMaxPsnr = 128; + +LIBYUV_API +double SumSquareErrorToPsnr(uint64 sse, uint64 count); + +LIBYUV_API +double CalcFramePsnr(const uint8* src_a, int stride_a, + const uint8* src_b, int stride_b, + int width, int height); + +LIBYUV_API +double I420Psnr(const uint8* src_y_a, int stride_y_a, + const uint8* src_u_a, int stride_u_a, + const uint8* src_v_a, int stride_v_a, + const uint8* src_y_b, int stride_y_b, + const uint8* src_u_b, int stride_u_b, + const uint8* src_v_b, int stride_v_b, + int width, int height); + +LIBYUV_API +double CalcFrameSsim(const uint8* src_a, int stride_a, + const uint8* src_b, int stride_b, + int width, int height); + +LIBYUV_API +double I420Ssim(const uint8* src_y_a, int stride_y_a, + const uint8* src_u_a, int stride_u_a, + const uint8* src_v_a, int stride_v_a, + const uint8* src_y_b, int stride_y_b, + const uint8* src_u_b, int stride_u_b, + const uint8* src_v_b, int stride_v_b, + int width, int height); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_COMPARE_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/convert.h b/third_party/aom/third_party/libyuv/include/libyuv/convert.h new file mode 100644 index 0000000000..d6f206c10f --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/convert.h @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_CONVERT_H_ // NOLINT +#define INCLUDE_LIBYUV_CONVERT_H_ + +#include "libyuv/basic_types.h" +// TODO(fbarchard): Remove the following headers includes. +#include "libyuv/convert_from.h" +#include "libyuv/planar_functions.h" +#include "libyuv/rotate.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Convert I444 to I420. +LIBYUV_API +int I444ToI420(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert I422 to I420. +LIBYUV_API +int I422ToI420(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert I411 to I420. +LIBYUV_API +int I411ToI420(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Copy I420 to I420. +#define I420ToI420 I420Copy +LIBYUV_API +int I420Copy(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert I400 (grey) to I420. +LIBYUV_API +int I400ToI420(const uint8* src_y, int src_stride_y, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +#define J400ToJ420 I400ToI420 + +// Convert NV12 to I420. +LIBYUV_API +int NV12ToI420(const uint8* src_y, int src_stride_y, + const uint8* src_uv, int src_stride_uv, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert NV21 to I420. +LIBYUV_API +int NV21ToI420(const uint8* src_y, int src_stride_y, + const uint8* src_vu, int src_stride_vu, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert YUY2 to I420. +LIBYUV_API +int YUY2ToI420(const uint8* src_yuy2, int src_stride_yuy2, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert UYVY to I420. +LIBYUV_API +int UYVYToI420(const uint8* src_uyvy, int src_stride_uyvy, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert M420 to I420. +LIBYUV_API +int M420ToI420(const uint8* src_m420, int src_stride_m420, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// ARGB little endian (bgra in memory) to I420. +LIBYUV_API +int ARGBToI420(const uint8* src_frame, int src_stride_frame, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// BGRA little endian (argb in memory) to I420. +LIBYUV_API +int BGRAToI420(const uint8* src_frame, int src_stride_frame, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// ABGR little endian (rgba in memory) to I420. +LIBYUV_API +int ABGRToI420(const uint8* src_frame, int src_stride_frame, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// RGBA little endian (abgr in memory) to I420. +LIBYUV_API +int RGBAToI420(const uint8* src_frame, int src_stride_frame, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// RGB little endian (bgr in memory) to I420. +LIBYUV_API +int RGB24ToI420(const uint8* src_frame, int src_stride_frame, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// RGB big endian (rgb in memory) to I420. +LIBYUV_API +int RAWToI420(const uint8* src_frame, int src_stride_frame, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// RGB16 (RGBP fourcc) little endian to I420. +LIBYUV_API +int RGB565ToI420(const uint8* src_frame, int src_stride_frame, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// RGB15 (RGBO fourcc) little endian to I420. +LIBYUV_API +int ARGB1555ToI420(const uint8* src_frame, int src_stride_frame, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// RGB12 (R444 fourcc) little endian to I420. +LIBYUV_API +int ARGB4444ToI420(const uint8* src_frame, int src_stride_frame, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +#ifdef HAVE_JPEG +// src_width/height provided by capture. +// dst_width/height for clipping determine final size. +LIBYUV_API +int MJPGToI420(const uint8* sample, size_t sample_size, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int src_width, int src_height, + int dst_width, int dst_height); + +// Query size of MJPG in pixels. +LIBYUV_API +int MJPGSize(const uint8* sample, size_t sample_size, + int* width, int* height); +#endif + +// Convert camera sample to I420 with cropping, rotation and vertical flip. +// "src_size" is needed to parse MJPG. +// "dst_stride_y" number of bytes in a row of the dst_y plane. +// Normally this would be the same as dst_width, with recommended alignment +// to 16 bytes for better efficiency. +// If rotation of 90 or 270 is used, stride is affected. The caller should +// allocate the I420 buffer according to rotation. +// "dst_stride_u" number of bytes in a row of the dst_u plane. +// Normally this would be the same as (dst_width + 1) / 2, with +// recommended alignment to 16 bytes for better efficiency. +// If rotation of 90 or 270 is used, stride is affected. +// "crop_x" and "crop_y" are starting position for cropping. +// To center, crop_x = (src_width - dst_width) / 2 +// crop_y = (src_height - dst_height) / 2 +// "src_width" / "src_height" is size of src_frame in pixels. +// "src_height" can be negative indicating a vertically flipped image source. +// "crop_width" / "crop_height" is the size to crop the src to. +// Must be less than or equal to src_width/src_height +// Cropping parameters are pre-rotation. +// "rotation" can be 0, 90, 180 or 270. +// "format" is a fourcc. ie 'I420', 'YUY2' +// Returns 0 for successful; -1 for invalid parameter. Non-zero for failure. +LIBYUV_API +int ConvertToI420(const uint8* src_frame, size_t src_size, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int crop_x, int crop_y, + int src_width, int src_height, + int crop_width, int crop_height, + enum RotationMode rotation, + uint32 format); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_CONVERT_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/convert_argb.h b/third_party/aom/third_party/libyuv/include/libyuv/convert_argb.h new file mode 100644 index 0000000000..ea75c0b26a --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/convert_argb.h @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_CONVERT_ARGB_H_ // NOLINT +#define INCLUDE_LIBYUV_CONVERT_ARGB_H_ + +#include "libyuv/basic_types.h" +// TODO(fbarchard): Remove the following headers includes +#include "libyuv/convert_from.h" +#include "libyuv/planar_functions.h" +#include "libyuv/rotate.h" + +// TODO(fbarchard): This set of functions should exactly match convert.h +// TODO(fbarchard): Add tests. Create random content of right size and convert +// with C vs Opt and or to I420 and compare. +// TODO(fbarchard): Some of these functions lack parameter setting. + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Alias. +#define ARGBToARGB ARGBCopy + +// Copy ARGB to ARGB. +LIBYUV_API +int ARGBCopy(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert I420 to ARGB. +LIBYUV_API +int I420ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert I422 to ARGB. +LIBYUV_API +int I422ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert I444 to ARGB. +LIBYUV_API +int I444ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert I411 to ARGB. +LIBYUV_API +int I411ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert I400 (grey) to ARGB. Reverse of ARGBToI400. +LIBYUV_API +int I400ToARGB(const uint8* src_y, int src_stride_y, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert J400 (jpeg grey) to ARGB. +LIBYUV_API +int J400ToARGB(const uint8* src_y, int src_stride_y, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Alias. +#define YToARGB I400ToARGB + +// Convert NV12 to ARGB. +LIBYUV_API +int NV12ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_uv, int src_stride_uv, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert NV21 to ARGB. +LIBYUV_API +int NV21ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_vu, int src_stride_vu, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert M420 to ARGB. +LIBYUV_API +int M420ToARGB(const uint8* src_m420, int src_stride_m420, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert YUY2 to ARGB. +LIBYUV_API +int YUY2ToARGB(const uint8* src_yuy2, int src_stride_yuy2, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert UYVY to ARGB. +LIBYUV_API +int UYVYToARGB(const uint8* src_uyvy, int src_stride_uyvy, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert J420 to ARGB. +LIBYUV_API +int J420ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert J422 to ARGB. +LIBYUV_API +int J422ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// BGRA little endian (argb in memory) to ARGB. +LIBYUV_API +int BGRAToARGB(const uint8* src_frame, int src_stride_frame, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// ABGR little endian (rgba in memory) to ARGB. +LIBYUV_API +int ABGRToARGB(const uint8* src_frame, int src_stride_frame, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// RGBA little endian (abgr in memory) to ARGB. +LIBYUV_API +int RGBAToARGB(const uint8* src_frame, int src_stride_frame, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Deprecated function name. +#define BG24ToARGB RGB24ToARGB + +// RGB little endian (bgr in memory) to ARGB. +LIBYUV_API +int RGB24ToARGB(const uint8* src_frame, int src_stride_frame, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// RGB big endian (rgb in memory) to ARGB. +LIBYUV_API +int RAWToARGB(const uint8* src_frame, int src_stride_frame, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// RGB16 (RGBP fourcc) little endian to ARGB. +LIBYUV_API +int RGB565ToARGB(const uint8* src_frame, int src_stride_frame, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// RGB15 (RGBO fourcc) little endian to ARGB. +LIBYUV_API +int ARGB1555ToARGB(const uint8* src_frame, int src_stride_frame, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// RGB12 (R444 fourcc) little endian to ARGB. +LIBYUV_API +int ARGB4444ToARGB(const uint8* src_frame, int src_stride_frame, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +#ifdef HAVE_JPEG +// src_width/height provided by capture +// dst_width/height for clipping determine final size. +LIBYUV_API +int MJPGToARGB(const uint8* sample, size_t sample_size, + uint8* dst_argb, int dst_stride_argb, + int src_width, int src_height, + int dst_width, int dst_height); +#endif + +// Convert camera sample to ARGB with cropping, rotation and vertical flip. +// "src_size" is needed to parse MJPG. +// "dst_stride_argb" number of bytes in a row of the dst_argb plane. +// Normally this would be the same as dst_width, with recommended alignment +// to 16 bytes for better efficiency. +// If rotation of 90 or 270 is used, stride is affected. The caller should +// allocate the I420 buffer according to rotation. +// "dst_stride_u" number of bytes in a row of the dst_u plane. +// Normally this would be the same as (dst_width + 1) / 2, with +// recommended alignment to 16 bytes for better efficiency. +// If rotation of 90 or 270 is used, stride is affected. +// "crop_x" and "crop_y" are starting position for cropping. +// To center, crop_x = (src_width - dst_width) / 2 +// crop_y = (src_height - dst_height) / 2 +// "src_width" / "src_height" is size of src_frame in pixels. +// "src_height" can be negative indicating a vertically flipped image source. +// "crop_width" / "crop_height" is the size to crop the src to. +// Must be less than or equal to src_width/src_height +// Cropping parameters are pre-rotation. +// "rotation" can be 0, 90, 180 or 270. +// "format" is a fourcc. ie 'I420', 'YUY2' +// Returns 0 for successful; -1 for invalid parameter. Non-zero for failure. +LIBYUV_API +int ConvertToARGB(const uint8* src_frame, size_t src_size, + uint8* dst_argb, int dst_stride_argb, + int crop_x, int crop_y, + int src_width, int src_height, + int crop_width, int crop_height, + enum RotationMode rotation, + uint32 format); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_CONVERT_ARGB_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/convert_from.h b/third_party/aom/third_party/libyuv/include/libyuv/convert_from.h new file mode 100644 index 0000000000..3591b4fd6a --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/convert_from.h @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_CONVERT_FROM_H_ // NOLINT +#define INCLUDE_LIBYUV_CONVERT_FROM_H_ + +#include "libyuv/basic_types.h" +#include "libyuv/rotate.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// See Also convert.h for conversions from formats to I420. + +// I420Copy in convert to I420ToI420. + +LIBYUV_API +int I420ToI422(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +LIBYUV_API +int I420ToI444(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +LIBYUV_API +int I420ToI411(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Copy to I400. Source can be I420, I422, I444, I400, NV12 or NV21. +LIBYUV_API +int I400Copy(const uint8* src_y, int src_stride_y, + uint8* dst_y, int dst_stride_y, + int width, int height); + +// TODO(fbarchard): I420ToM420 + +LIBYUV_API +int I420ToNV12(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_uv, int dst_stride_uv, + int width, int height); + +LIBYUV_API +int I420ToNV21(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_vu, int dst_stride_vu, + int width, int height); + +LIBYUV_API +int I420ToYUY2(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_frame, int dst_stride_frame, + int width, int height); + +LIBYUV_API +int I420ToUYVY(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_frame, int dst_stride_frame, + int width, int height); + +LIBYUV_API +int I420ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +LIBYUV_API +int I420ToBGRA(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +LIBYUV_API +int I420ToABGR(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +LIBYUV_API +int I420ToRGBA(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_rgba, int dst_stride_rgba, + int width, int height); + +LIBYUV_API +int I420ToRGB24(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_frame, int dst_stride_frame, + int width, int height); + +LIBYUV_API +int I420ToRAW(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_frame, int dst_stride_frame, + int width, int height); + +LIBYUV_API +int I420ToRGB565(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_frame, int dst_stride_frame, + int width, int height); + +// Convert I420 To RGB565 with 4x4 dither matrix (16 bytes). +// Values in dither matrix from 0 to 7 recommended. +// The order of the dither matrix is first byte is upper left. + +LIBYUV_API +int I420ToRGB565Dither(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_frame, int dst_stride_frame, + const uint8* dither4x4, int width, int height); + +LIBYUV_API +int I420ToARGB1555(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_frame, int dst_stride_frame, + int width, int height); + +LIBYUV_API +int I420ToARGB4444(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_frame, int dst_stride_frame, + int width, int height); + +// Convert I420 to specified format. +// "dst_sample_stride" is bytes in a row for the destination. Pass 0 if the +// buffer has contiguous rows. Can be negative. A multiple of 16 is optimal. +LIBYUV_API +int ConvertFromI420(const uint8* y, int y_stride, + const uint8* u, int u_stride, + const uint8* v, int v_stride, + uint8* dst_sample, int dst_sample_stride, + int width, int height, + uint32 format); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_CONVERT_FROM_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/convert_from_argb.h b/third_party/aom/third_party/libyuv/include/libyuv/convert_from_argb.h new file mode 100644 index 0000000000..4a62268138 --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/convert_from_argb.h @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_CONVERT_FROM_ARGB_H_ // NOLINT +#define INCLUDE_LIBYUV_CONVERT_FROM_ARGB_H_ + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Copy ARGB to ARGB. +#define ARGBToARGB ARGBCopy +LIBYUV_API +int ARGBCopy(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert ARGB To BGRA. +LIBYUV_API +int ARGBToBGRA(const uint8* src_argb, int src_stride_argb, + uint8* dst_bgra, int dst_stride_bgra, + int width, int height); + +// Convert ARGB To ABGR. +LIBYUV_API +int ARGBToABGR(const uint8* src_argb, int src_stride_argb, + uint8* dst_abgr, int dst_stride_abgr, + int width, int height); + +// Convert ARGB To RGBA. +LIBYUV_API +int ARGBToRGBA(const uint8* src_argb, int src_stride_argb, + uint8* dst_rgba, int dst_stride_rgba, + int width, int height); + +// Convert ARGB To RGB24. +LIBYUV_API +int ARGBToRGB24(const uint8* src_argb, int src_stride_argb, + uint8* dst_rgb24, int dst_stride_rgb24, + int width, int height); + +// Convert ARGB To RAW. +LIBYUV_API +int ARGBToRAW(const uint8* src_argb, int src_stride_argb, + uint8* dst_rgb, int dst_stride_rgb, + int width, int height); + +// Convert ARGB To RGB565. +LIBYUV_API +int ARGBToRGB565(const uint8* src_argb, int src_stride_argb, + uint8* dst_rgb565, int dst_stride_rgb565, + int width, int height); + +// Convert ARGB To RGB565 with 4x4 dither matrix (16 bytes). +// Values in dither matrix from 0 to 7 recommended. +// The order of the dither matrix is first byte is upper left. +// TODO(fbarchard): Consider pointer to 2d array for dither4x4. +// const uint8(*dither)[4][4]; +LIBYUV_API +int ARGBToRGB565Dither(const uint8* src_argb, int src_stride_argb, + uint8* dst_rgb565, int dst_stride_rgb565, + const uint8* dither4x4, int width, int height); + +// Convert ARGB To ARGB1555. +LIBYUV_API +int ARGBToARGB1555(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb1555, int dst_stride_argb1555, + int width, int height); + +// Convert ARGB To ARGB4444. +LIBYUV_API +int ARGBToARGB4444(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb4444, int dst_stride_argb4444, + int width, int height); + +// Convert ARGB To I444. +LIBYUV_API +int ARGBToI444(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert ARGB To I422. +LIBYUV_API +int ARGBToI422(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert ARGB To I420. (also in convert.h) +LIBYUV_API +int ARGBToI420(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert ARGB to J420. (JPeg full range I420). +LIBYUV_API +int ARGBToJ420(const uint8* src_argb, int src_stride_argb, + uint8* dst_yj, int dst_stride_yj, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert ARGB to J422. +LIBYUV_API +int ARGBToJ422(const uint8* src_argb, int src_stride_argb, + uint8* dst_yj, int dst_stride_yj, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert ARGB To I411. +LIBYUV_API +int ARGBToI411(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert ARGB to J400. (JPeg full range). +LIBYUV_API +int ARGBToJ400(const uint8* src_argb, int src_stride_argb, + uint8* dst_yj, int dst_stride_yj, + int width, int height); + +// Convert ARGB to I400. +LIBYUV_API +int ARGBToI400(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + int width, int height); + +// Convert ARGB to G. (Reverse of J400toARGB, which replicates G back to ARGB) +LIBYUV_API +int ARGBToG(const uint8* src_argb, int src_stride_argb, + uint8* dst_g, int dst_stride_g, + int width, int height); + +// Convert ARGB To NV12. +LIBYUV_API +int ARGBToNV12(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_uv, int dst_stride_uv, + int width, int height); + +// Convert ARGB To NV21. +LIBYUV_API +int ARGBToNV21(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_vu, int dst_stride_vu, + int width, int height); + +// Convert ARGB To NV21. +LIBYUV_API +int ARGBToNV21(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_vu, int dst_stride_vu, + int width, int height); + +// Convert ARGB To YUY2. +LIBYUV_API +int ARGBToYUY2(const uint8* src_argb, int src_stride_argb, + uint8* dst_yuy2, int dst_stride_yuy2, + int width, int height); + +// Convert ARGB To UYVY. +LIBYUV_API +int ARGBToUYVY(const uint8* src_argb, int src_stride_argb, + uint8* dst_uyvy, int dst_stride_uyvy, + int width, int height); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_CONVERT_FROM_ARGB_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/cpu_id.h b/third_party/aom/third_party/libyuv/include/libyuv/cpu_id.h new file mode 100644 index 0000000000..870e94e8cd --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/cpu_id.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_CPU_ID_H_ // NOLINT +#define INCLUDE_LIBYUV_CPU_ID_H_ + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// TODO(fbarchard): Consider overlapping bits for different architectures. +// Internal flag to indicate cpuid requires initialization. +#define kCpuInit 0x1 + +// These flags are only valid on ARM processors. +static const int kCpuHasARM = 0x2; +static const int kCpuHasNEON = 0x4; +// 0x8 reserved for future ARM flag. + +// These flags are only valid on x86 processors. +static const int kCpuHasX86 = 0x10; +static const int kCpuHasSSE2 = 0x20; +static const int kCpuHasSSSE3 = 0x40; +static const int kCpuHasSSE41 = 0x80; +static const int kCpuHasSSE42 = 0x100; +static const int kCpuHasAVX = 0x200; +static const int kCpuHasAVX2 = 0x400; +static const int kCpuHasERMS = 0x800; +static const int kCpuHasFMA3 = 0x1000; +// 0x2000, 0x4000, 0x8000 reserved for future X86 flags. + +// These flags are only valid on MIPS processors. +static const int kCpuHasMIPS = 0x10000; +static const int kCpuHasMIPS_DSP = 0x20000; +static const int kCpuHasMIPS_DSPR2 = 0x40000; + +// Internal function used to auto-init. +LIBYUV_API +int InitCpuFlags(void); + +// Internal function for parsing /proc/cpuinfo. +LIBYUV_API +int ArmCpuCaps(const char* cpuinfo_name); + +// Detect CPU has SSE2 etc. +// Test_flag parameter should be one of kCpuHas constants above. +// returns non-zero if instruction set is detected +static __inline int TestCpuFlag(int test_flag) { + LIBYUV_API extern int cpu_info_; + return (cpu_info_ == kCpuInit ? InitCpuFlags() : cpu_info_) & test_flag; +} + +// For testing, allow CPU flags to be disabled. +// ie MaskCpuFlags(~kCpuHasSSSE3) to disable SSSE3. +// MaskCpuFlags(-1) to enable all cpu specific optimizations. +// MaskCpuFlags(0) to disable all cpu specific optimizations. +LIBYUV_API +void MaskCpuFlags(int enable_flags); + +// Low level cpuid for X86. Returns zeros on other CPUs. +// eax is the info type that you want. +// ecx is typically the cpu number, and should normally be zero. +LIBYUV_API +void CpuId(uint32 eax, uint32 ecx, uint32* cpu_info); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_CPU_ID_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/mjpeg_decoder.h b/third_party/aom/third_party/libyuv/include/libyuv/mjpeg_decoder.h new file mode 100644 index 0000000000..fa1e51f9ac --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/mjpeg_decoder.h @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_MJPEG_DECODER_H_ // NOLINT +#define INCLUDE_LIBYUV_MJPEG_DECODER_H_ + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +// NOTE: For a simplified public API use convert.h MJPGToI420(). + +struct jpeg_common_struct; +struct jpeg_decompress_struct; +struct jpeg_source_mgr; + +namespace libyuv { + +#ifdef __cplusplus +extern "C" { +#endif + +LIBYUV_BOOL ValidateJpeg(const uint8* sample, size_t sample_size); + +#ifdef __cplusplus +} // extern "C" +#endif + +static const uint32 kUnknownDataSize = 0xFFFFFFFF; + +enum JpegSubsamplingType { + kJpegYuv420, + kJpegYuv422, + kJpegYuv411, + kJpegYuv444, + kJpegYuv400, + kJpegUnknown +}; + +struct Buffer { + const uint8* data; + int len; +}; + +struct BufferVector { + Buffer* buffers; + int len; + int pos; +}; + +struct SetJmpErrorMgr; + +// MJPEG ("Motion JPEG") is a pseudo-standard video codec where the frames are +// simply independent JPEG images with a fixed huffman table (which is omitted). +// It is rarely used in video transmission, but is common as a camera capture +// format, especially in Logitech devices. This class implements a decoder for +// MJPEG frames. +// +// See http://tools.ietf.org/html/rfc2435 +class LIBYUV_API MJpegDecoder { + public: + typedef void (*CallbackFunction)(void* opaque, + const uint8* const* data, + const int* strides, + int rows); + + static const int kColorSpaceUnknown; + static const int kColorSpaceGrayscale; + static const int kColorSpaceRgb; + static const int kColorSpaceYCbCr; + static const int kColorSpaceCMYK; + static const int kColorSpaceYCCK; + + MJpegDecoder(); + ~MJpegDecoder(); + + // Loads a new frame, reads its headers, and determines the uncompressed + // image format. + // Returns LIBYUV_TRUE if image looks valid and format is supported. + // If return value is LIBYUV_TRUE, then the values for all the following + // getters are populated. + // src_len is the size of the compressed mjpeg frame in bytes. + LIBYUV_BOOL LoadFrame(const uint8* src, size_t src_len); + + // Returns width of the last loaded frame in pixels. + int GetWidth(); + + // Returns height of the last loaded frame in pixels. + int GetHeight(); + + // Returns format of the last loaded frame. The return value is one of the + // kColorSpace* constants. + int GetColorSpace(); + + // Number of color components in the color space. + int GetNumComponents(); + + // Sample factors of the n-th component. + int GetHorizSampFactor(int component); + + int GetVertSampFactor(int component); + + int GetHorizSubSampFactor(int component); + + int GetVertSubSampFactor(int component); + + // Public for testability. + int GetImageScanlinesPerImcuRow(); + + // Public for testability. + int GetComponentScanlinesPerImcuRow(int component); + + // Width of a component in bytes. + int GetComponentWidth(int component); + + // Height of a component. + int GetComponentHeight(int component); + + // Width of a component in bytes with padding for DCTSIZE. Public for testing. + int GetComponentStride(int component); + + // Size of a component in bytes. + int GetComponentSize(int component); + + // Call this after LoadFrame() if you decide you don't want to decode it + // after all. + LIBYUV_BOOL UnloadFrame(); + + // Decodes the entire image into a one-buffer-per-color-component format. + // dst_width must match exactly. dst_height must be <= to image height; if + // less, the image is cropped. "planes" must have size equal to at least + // GetNumComponents() and they must point to non-overlapping buffers of size + // at least GetComponentSize(i). The pointers in planes are incremented + // to point to after the end of the written data. + // TODO(fbarchard): Add dst_x, dst_y to allow specific rect to be decoded. + LIBYUV_BOOL DecodeToBuffers(uint8** planes, int dst_width, int dst_height); + + // Decodes the entire image and passes the data via repeated calls to a + // callback function. Each call will get the data for a whole number of + // image scanlines. + // TODO(fbarchard): Add dst_x, dst_y to allow specific rect to be decoded. + LIBYUV_BOOL DecodeToCallback(CallbackFunction fn, void* opaque, + int dst_width, int dst_height); + + // The helper function which recognizes the jpeg sub-sampling type. + static JpegSubsamplingType JpegSubsamplingTypeHelper( + int* subsample_x, int* subsample_y, int number_of_components); + + private: + void AllocOutputBuffers(int num_outbufs); + void DestroyOutputBuffers(); + + LIBYUV_BOOL StartDecode(); + LIBYUV_BOOL FinishDecode(); + + void SetScanlinePointers(uint8** data); + LIBYUV_BOOL DecodeImcuRow(); + + int GetComponentScanlinePadding(int component); + + // A buffer holding the input data for a frame. + Buffer buf_; + BufferVector buf_vec_; + + jpeg_decompress_struct* decompress_struct_; + jpeg_source_mgr* source_mgr_; + SetJmpErrorMgr* error_mgr_; + + // LIBYUV_TRUE iff at least one component has scanline padding. (i.e., + // GetComponentScanlinePadding() != 0.) + LIBYUV_BOOL has_scanline_padding_; + + // Temporaries used to point to scanline outputs. + int num_outbufs_; // Outermost size of all arrays below. + uint8*** scanlines_; + int* scanlines_sizes_; + // Temporary buffer used for decoding when we can't decode directly to the + // output buffers. Large enough for just one iMCU row. + uint8** databuf_; + int* databuf_strides_; +}; + +} // namespace libyuv + +#endif // __cplusplus +#endif // INCLUDE_LIBYUV_MJPEG_DECODER_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/planar_functions.h b/third_party/aom/third_party/libyuv/include/libyuv/planar_functions.h new file mode 100644 index 0000000000..7fe4d8eedd --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/planar_functions.h @@ -0,0 +1,454 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_PLANAR_FUNCTIONS_H_ // NOLINT +#define INCLUDE_LIBYUV_PLANAR_FUNCTIONS_H_ + +#include "libyuv/basic_types.h" + +// TODO(fbarchard): Remove the following headers includes. +#include "libyuv/convert.h" +#include "libyuv/convert_argb.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Copy a plane of data. +LIBYUV_API +void CopyPlane(const uint8* src_y, int src_stride_y, + uint8* dst_y, int dst_stride_y, + int width, int height); + +LIBYUV_API +void CopyPlane_16(const uint16* src_y, int src_stride_y, + uint16* dst_y, int dst_stride_y, + int width, int height); + +// Set a plane of data to a 32 bit value. +LIBYUV_API +void SetPlane(uint8* dst_y, int dst_stride_y, + int width, int height, + uint32 value); + +// Copy I400. Supports inverting. +LIBYUV_API +int I400ToI400(const uint8* src_y, int src_stride_y, + uint8* dst_y, int dst_stride_y, + int width, int height); + +#define J400ToJ400 I400ToI400 + +// Copy I422 to I422. +#define I422ToI422 I422Copy +LIBYUV_API +int I422Copy(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Copy I444 to I444. +#define I444ToI444 I444Copy +LIBYUV_API +int I444Copy(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert YUY2 to I422. +LIBYUV_API +int YUY2ToI422(const uint8* src_yuy2, int src_stride_yuy2, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Convert UYVY to I422. +LIBYUV_API +int UYVYToI422(const uint8* src_uyvy, int src_stride_uyvy, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +LIBYUV_API +int YUY2ToNV12(const uint8* src_yuy2, int src_stride_yuy2, + uint8* dst_y, int dst_stride_y, + uint8* dst_uv, int dst_stride_uv, + int width, int height); + +LIBYUV_API +int UYVYToNV12(const uint8* src_uyvy, int src_stride_uyvy, + uint8* dst_y, int dst_stride_y, + uint8* dst_uv, int dst_stride_uv, + int width, int height); + +// Convert I420 to I400. (calls CopyPlane ignoring u/v). +LIBYUV_API +int I420ToI400(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + int width, int height); + +// Alias +#define J420ToJ400 I420ToI400 +#define I420ToI420Mirror I420Mirror + +// I420 mirror. +LIBYUV_API +int I420Mirror(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height); + +// Alias +#define I400ToI400Mirror I400Mirror + +// I400 mirror. A single plane is mirrored horizontally. +// Pass negative height to achieve 180 degree rotation. +LIBYUV_API +int I400Mirror(const uint8* src_y, int src_stride_y, + uint8* dst_y, int dst_stride_y, + int width, int height); + +// Alias +#define ARGBToARGBMirror ARGBMirror + +// ARGB mirror. +LIBYUV_API +int ARGBMirror(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert NV12 to RGB565. +LIBYUV_API +int NV12ToRGB565(const uint8* src_y, int src_stride_y, + const uint8* src_uv, int src_stride_uv, + uint8* dst_rgb565, int dst_stride_rgb565, + int width, int height); + +// Convert NV21 to RGB565. +LIBYUV_API +int NV21ToRGB565(const uint8* src_y, int src_stride_y, + const uint8* src_uv, int src_stride_uv, + uint8* dst_rgb565, int dst_stride_rgb565, + int width, int height); + +// I422ToARGB is in convert_argb.h +// Convert I422 to BGRA. +LIBYUV_API +int I422ToBGRA(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_bgra, int dst_stride_bgra, + int width, int height); + +// Convert I422 to ABGR. +LIBYUV_API +int I422ToABGR(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_abgr, int dst_stride_abgr, + int width, int height); + +// Convert I422 to RGBA. +LIBYUV_API +int I422ToRGBA(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_rgba, int dst_stride_rgba, + int width, int height); + +// Draw a rectangle into I420. +LIBYUV_API +int I420Rect(uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int x, int y, int width, int height, + int value_y, int value_u, int value_v); + +// Draw a rectangle into ARGB. +LIBYUV_API +int ARGBRect(uint8* dst_argb, int dst_stride_argb, + int x, int y, int width, int height, uint32 value); + +// Convert ARGB to gray scale ARGB. +LIBYUV_API +int ARGBGrayTo(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Make a rectangle of ARGB gray scale. +LIBYUV_API +int ARGBGray(uint8* dst_argb, int dst_stride_argb, + int x, int y, int width, int height); + +// Make a rectangle of ARGB Sepia tone. +LIBYUV_API +int ARGBSepia(uint8* dst_argb, int dst_stride_argb, + int x, int y, int width, int height); + +// Apply a matrix rotation to each ARGB pixel. +// matrix_argb is 4 signed ARGB values. -128 to 127 representing -2 to 2. +// The first 4 coefficients apply to B, G, R, A and produce B of the output. +// The next 4 coefficients apply to B, G, R, A and produce G of the output. +// The next 4 coefficients apply to B, G, R, A and produce R of the output. +// The last 4 coefficients apply to B, G, R, A and produce A of the output. +LIBYUV_API +int ARGBColorMatrix(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + const int8* matrix_argb, + int width, int height); + +// Deprecated. Use ARGBColorMatrix instead. +// Apply a matrix rotation to each ARGB pixel. +// matrix_argb is 3 signed ARGB values. -128 to 127 representing -1 to 1. +// The first 4 coefficients apply to B, G, R, A and produce B of the output. +// The next 4 coefficients apply to B, G, R, A and produce G of the output. +// The last 4 coefficients apply to B, G, R, A and produce R of the output. +LIBYUV_API +int RGBColorMatrix(uint8* dst_argb, int dst_stride_argb, + const int8* matrix_rgb, + int x, int y, int width, int height); + +// Apply a color table each ARGB pixel. +// Table contains 256 ARGB values. +LIBYUV_API +int ARGBColorTable(uint8* dst_argb, int dst_stride_argb, + const uint8* table_argb, + int x, int y, int width, int height); + +// Apply a color table each ARGB pixel but preserve destination alpha. +// Table contains 256 ARGB values. +LIBYUV_API +int RGBColorTable(uint8* dst_argb, int dst_stride_argb, + const uint8* table_argb, + int x, int y, int width, int height); + +// Apply a luma/color table each ARGB pixel but preserve destination alpha. +// Table contains 32768 values indexed by [Y][C] where 7 it 7 bit luma from +// RGB (YJ style) and C is an 8 bit color component (R, G or B). +LIBYUV_API +int ARGBLumaColorTable(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + const uint8* luma_rgb_table, + int width, int height); + +// Apply a 3 term polynomial to ARGB values. +// poly points to a 4x4 matrix. The first row is constants. The 2nd row is +// coefficients for b, g, r and a. The 3rd row is coefficients for b squared, +// g squared, r squared and a squared. The 4rd row is coefficients for b to +// the 3, g to the 3, r to the 3 and a to the 3. The values are summed and +// result clamped to 0 to 255. +// A polynomial approximation can be dirived using software such as 'R'. + +LIBYUV_API +int ARGBPolynomial(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + const float* poly, + int width, int height); + +// Quantize a rectangle of ARGB. Alpha unaffected. +// scale is a 16 bit fractional fixed point scaler between 0 and 65535. +// interval_size should be a value between 1 and 255. +// interval_offset should be a value between 0 and 255. +LIBYUV_API +int ARGBQuantize(uint8* dst_argb, int dst_stride_argb, + int scale, int interval_size, int interval_offset, + int x, int y, int width, int height); + +// Copy ARGB to ARGB. +LIBYUV_API +int ARGBCopy(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Copy ARGB to ARGB. +LIBYUV_API +int ARGBCopyAlpha(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Copy ARGB to ARGB. +LIBYUV_API +int ARGBCopyYToAlpha(const uint8* src_y, int src_stride_y, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +typedef void (*ARGBBlendRow)(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width); + +// Get function to Alpha Blend ARGB pixels and store to destination. +LIBYUV_API +ARGBBlendRow GetARGBBlend(); + +// Alpha Blend ARGB images and store to destination. +// Alpha of destination is set to 255. +LIBYUV_API +int ARGBBlend(const uint8* src_argb0, int src_stride_argb0, + const uint8* src_argb1, int src_stride_argb1, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Multiply ARGB image by ARGB image. Shifted down by 8. Saturates to 255. +LIBYUV_API +int ARGBMultiply(const uint8* src_argb0, int src_stride_argb0, + const uint8* src_argb1, int src_stride_argb1, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Add ARGB image with ARGB image. Saturates to 255. +LIBYUV_API +int ARGBAdd(const uint8* src_argb0, int src_stride_argb0, + const uint8* src_argb1, int src_stride_argb1, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Subtract ARGB image (argb1) from ARGB image (argb0). Saturates to 0. +LIBYUV_API +int ARGBSubtract(const uint8* src_argb0, int src_stride_argb0, + const uint8* src_argb1, int src_stride_argb1, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert I422 to YUY2. +LIBYUV_API +int I422ToYUY2(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_frame, int dst_stride_frame, + int width, int height); + +// Convert I422 to UYVY. +LIBYUV_API +int I422ToUYVY(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_frame, int dst_stride_frame, + int width, int height); + +// Convert unattentuated ARGB to preattenuated ARGB. +LIBYUV_API +int ARGBAttenuate(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert preattentuated ARGB to unattenuated ARGB. +LIBYUV_API +int ARGBUnattenuate(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Convert MJPG to ARGB. +LIBYUV_API +int MJPGToARGB(const uint8* sample, size_t sample_size, + uint8* argb, int argb_stride, + int w, int h, int dw, int dh); + +// Internal function - do not call directly. +// Computes table of cumulative sum for image where the value is the sum +// of all values above and to the left of the entry. Used by ARGBBlur. +LIBYUV_API +int ARGBComputeCumulativeSum(const uint8* src_argb, int src_stride_argb, + int32* dst_cumsum, int dst_stride32_cumsum, + int width, int height); + +// Blur ARGB image. +// dst_cumsum table of width * (height + 1) * 16 bytes aligned to +// 16 byte boundary. +// dst_stride32_cumsum is number of ints in a row (width * 4). +// radius is number of pixels around the center. e.g. 1 = 3x3. 2=5x5. +// Blur is optimized for radius of 5 (11x11) or less. +LIBYUV_API +int ARGBBlur(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int32* dst_cumsum, int dst_stride32_cumsum, + int width, int height, int radius); + +// Multiply ARGB image by ARGB value. +LIBYUV_API +int ARGBShade(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height, uint32 value); + +// Interpolate between two ARGB images using specified amount of interpolation +// (0 to 255) and store to destination. +// 'interpolation' is specified as 8 bit fraction where 0 means 100% src_argb0 +// and 255 means 1% src_argb0 and 99% src_argb1. +// Internally uses ARGBScale bilinear filtering. +// Caveat: This function will write up to 16 bytes beyond the end of dst_argb. +LIBYUV_API +int ARGBInterpolate(const uint8* src_argb0, int src_stride_argb0, + const uint8* src_argb1, int src_stride_argb1, + uint8* dst_argb, int dst_stride_argb, + int width, int height, int interpolation); + +#if defined(__pnacl__) || defined(__CLR_VER) || \ + (defined(__i386__) && !defined(__SSE2__)) +#define LIBYUV_DISABLE_X86 +#endif +// The following are available on all x86 platforms: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)) +#define HAS_ARGBAFFINEROW_SSE2 +#endif + +// Row function for copying pixels from a source with a slope to a row +// of destination. Useful for scaling, rotation, mirror, texture mapping. +LIBYUV_API +void ARGBAffineRow_C(const uint8* src_argb, int src_argb_stride, + uint8* dst_argb, const float* uv_dudv, int width); +LIBYUV_API +void ARGBAffineRow_SSE2(const uint8* src_argb, int src_argb_stride, + uint8* dst_argb, const float* uv_dudv, int width); + +// Shuffle ARGB channel order. e.g. BGRA to ARGB. +// shuffler is 16 bytes and must be aligned. +LIBYUV_API +int ARGBShuffle(const uint8* src_bgra, int src_stride_bgra, + uint8* dst_argb, int dst_stride_argb, + const uint8* shuffler, int width, int height); + +// Sobel ARGB effect with planar output. +LIBYUV_API +int ARGBSobelToPlane(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + int width, int height); + +// Sobel ARGB effect. +LIBYUV_API +int ARGBSobel(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +// Sobel ARGB effect w/ Sobel X, Sobel, Sobel Y in ARGB. +LIBYUV_API +int ARGBSobelXY(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_PLANAR_FUNCTIONS_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/rotate.h b/third_party/aom/third_party/libyuv/include/libyuv/rotate.h new file mode 100644 index 0000000000..8a9673f280 --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/rotate.h @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_ROTATE_H_ // NOLINT +#define INCLUDE_LIBYUV_ROTATE_H_ + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Supported rotation. +typedef enum RotationMode { + kRotate0 = 0, // No rotation. + kRotate90 = 90, // Rotate 90 degrees clockwise. + kRotate180 = 180, // Rotate 180 degrees. + kRotate270 = 270, // Rotate 270 degrees clockwise. + + // Deprecated. + kRotateNone = 0, + kRotateClockwise = 90, + kRotateCounterClockwise = 270, +} RotationModeEnum; + +// Rotate I420 frame. +LIBYUV_API +int I420Rotate(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int src_width, int src_height, enum RotationMode mode); + +// Rotate NV12 input and store in I420. +LIBYUV_API +int NV12ToI420Rotate(const uint8* src_y, int src_stride_y, + const uint8* src_uv, int src_stride_uv, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int src_width, int src_height, enum RotationMode mode); + +// Rotate a plane by 0, 90, 180, or 270. +LIBYUV_API +int RotatePlane(const uint8* src, int src_stride, + uint8* dst, int dst_stride, + int src_width, int src_height, enum RotationMode mode); + +// Rotate planes by 90, 180, 270. Deprecated. +LIBYUV_API +void RotatePlane90(const uint8* src, int src_stride, + uint8* dst, int dst_stride, + int width, int height); + +LIBYUV_API +void RotatePlane180(const uint8* src, int src_stride, + uint8* dst, int dst_stride, + int width, int height); + +LIBYUV_API +void RotatePlane270(const uint8* src, int src_stride, + uint8* dst, int dst_stride, + int width, int height); + +LIBYUV_API +void RotateUV90(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width, int height); + +// Rotations for when U and V are interleaved. +// These functions take one input pointer and +// split the data into two buffers while +// rotating them. Deprecated. +LIBYUV_API +void RotateUV180(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width, int height); + +LIBYUV_API +void RotateUV270(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width, int height); + +// The 90 and 270 functions are based on transposes. +// Doing a transpose with reversing the read/write +// order will result in a rotation by +- 90 degrees. +// Deprecated. +LIBYUV_API +void TransposePlane(const uint8* src, int src_stride, + uint8* dst, int dst_stride, + int width, int height); + +LIBYUV_API +void TransposeUV(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width, int height); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_ROTATE_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/rotate_argb.h b/third_party/aom/third_party/libyuv/include/libyuv/rotate_argb.h new file mode 100644 index 0000000000..2bdc8ec6b4 --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/rotate_argb.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_ROTATE_ARGB_H_ // NOLINT +#define INCLUDE_LIBYUV_ROTATE_ARGB_H_ + +#include "libyuv/basic_types.h" +#include "libyuv/rotate.h" // For RotationMode. + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Rotate ARGB frame +LIBYUV_API +int ARGBRotate(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int src_width, int src_height, enum RotationMode mode); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_ROTATE_ARGB_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/rotate_row.h b/third_party/aom/third_party/libyuv/include/libyuv/rotate_row.h new file mode 100644 index 0000000000..d0bfbdd2b0 --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/rotate_row.h @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_ROTATE_ROW_H_ // NOLINT +#define INCLUDE_LIBYUV_ROTATE_ROW_H_ + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#if defined(__pnacl__) || defined(__CLR_VER) || \ + (defined(__i386__) && !defined(__SSE2__)) +#define LIBYUV_DISABLE_X86 +#endif + +// Visual C 2012 required for AVX2. +#if defined(_M_IX86) && !defined(__clang__) && \ + defined(_MSC_VER) && _MSC_VER >= 1700 +#define VISUALC_HAS_AVX2 1 +#endif // VisualStudio >= 2012 + +// TODO(fbarchard): switch to standard form of inline; fails on clangcl. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)) +#if defined(__APPLE__) && defined(__i386__) +#define DECLARE_FUNCTION(name) \ + ".text \n" \ + ".private_extern _" #name " \n" \ + ".align 4,0x90 \n" \ +"_" #name ": \n" +#elif defined(__MINGW32__) || defined(__CYGWIN__) && defined(__i386__) +#define DECLARE_FUNCTION(name) \ + ".text \n" \ + ".align 4,0x90 \n" \ +"_" #name ": \n" +#else +#define DECLARE_FUNCTION(name) \ + ".text \n" \ + ".align 4,0x90 \n" \ +#name ": \n" +#endif +#endif + +// The following are available for Visual C: +#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && \ + defined(_MSC_VER) && !defined(__clang__) +#define HAS_TRANSPOSEWX8_SSSE3 +#define HAS_TRANSPOSEUVWX8_SSE2 +#endif + +// The following are available for GCC but not NaCL: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__i386__) || (defined(__x86_64__) && !defined(__native_client__))) +#define HAS_TRANSPOSEWX8_SSSE3 +#endif + +// The following are available for 32 bit GCC: +#if !defined(LIBYUV_DISABLE_X86) && defined(__i386__) && !defined(__clang__) +#define HAS_TRANSPOSEUVWX8_SSE2 +#endif + +// The following are available for 64 bit GCC but not NaCL: +#if !defined(LIBYUV_DISABLE_X86) && !defined(__native_client__) && \ + defined(__x86_64__) +#define HAS_TRANSPOSEWX8_FAST_SSSE3 +#define HAS_TRANSPOSEUVWX8_SSE2 +#endif + +#if !defined(LIBYUV_DISABLE_NEON) && !defined(__native_client__) && \ + (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) +#define HAS_TRANSPOSEWX8_NEON +#define HAS_TRANSPOSEUVWX8_NEON +#endif + +#if !defined(LIBYUV_DISABLE_MIPS) && !defined(__native_client__) && \ + defined(__mips__) && \ + defined(__mips_dsp) && (__mips_dsp_rev >= 2) +#define HAS_TRANSPOSEWX8_MIPS_DSPR2 +#define HAS_TRANSPOSEUVWx8_MIPS_DSPR2 +#endif // defined(__mips__) + +void TransposeWxH_C(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width, int height); + +void TransposeWx8_C(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width); +void TransposeWx8_NEON(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width); +void TransposeWx8_SSSE3(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width); +void TransposeWx8_Fast_SSSE3(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width); +void TransposeWx8_MIPS_DSPR2(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width); + +void TransposeWx8_Any_NEON(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width); +void TransposeWx8_Any_SSSE3(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width); +void TransposeWx8_Fast_Any_SSSE3(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width); +void TransposeWx8_Any_MIPS_DSPR2(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width); + +void TransposeUVWxH_C(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width, int height); + +void TransposeUVWx8_C(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, int width); +void TransposeUVWx8_SSE2(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, int width); +void TransposeUVWx8_NEON(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, int width); +void TransposeUVWx8_MIPS_DSPR2(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, int width); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_ROTATE_ROW_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/row.h b/third_party/aom/third_party/libyuv/include/libyuv/row.h new file mode 100644 index 0000000000..5c3187ef79 --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/row.h @@ -0,0 +1,1857 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_ROW_H_ // NOLINT +#define INCLUDE_LIBYUV_ROW_H_ + +#include // For malloc. + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#define IS_ALIGNED(p, a) (!((uintptr_t)(p) & ((a) - 1))) + +#ifdef __cplusplus +#define align_buffer_64(var, size) \ + uint8* var##_mem = reinterpret_cast(malloc((size) + 63)); \ + uint8* var = reinterpret_cast \ + ((reinterpret_cast(var##_mem) + 63) & ~63) +#else +#define align_buffer_64(var, size) \ + uint8* var##_mem = (uint8*)(malloc((size) + 63)); /* NOLINT */ \ + uint8* var = (uint8*)(((intptr_t)(var##_mem) + 63) & ~63) /* NOLINT */ +#endif + +#define free_aligned_buffer_64(var) \ + free(var##_mem); \ + var = 0 + +#if defined(__pnacl__) || defined(__CLR_VER) || \ + (defined(__i386__) && !defined(__SSE2__)) +#define LIBYUV_DISABLE_X86 +#endif +// True if compiling for SSSE3 as a requirement. +#if defined(__SSSE3__) || (defined(_M_IX86_FP) && (_M_IX86_FP >= 3)) +#define LIBYUV_SSSE3_ONLY +#endif + +#if defined(__native_client__) +#define LIBYUV_DISABLE_NEON +#endif +// clang >= 3.5.0 required for Arm64. +#if defined(__clang__) && defined(__aarch64__) && !defined(LIBYUV_DISABLE_NEON) +#if (__clang_major__ < 3) || (__clang_major__ == 3 && (__clang_minor__ < 5)) +#define LIBYUV_DISABLE_NEON +#endif // clang >= 3.5 +#endif // __clang__ + +// The following are available on all x86 platforms: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)) +// Conversions: +#define HAS_ABGRTOUVROW_SSSE3 +#define HAS_ABGRTOYROW_SSSE3 +#define HAS_ARGB1555TOARGBROW_SSE2 +#define HAS_ARGB4444TOARGBROW_SSE2 +#define HAS_ARGBSETROW_X86 +#define HAS_ARGBSHUFFLEROW_SSE2 +#define HAS_ARGBSHUFFLEROW_SSSE3 +#define HAS_ARGBTOARGB1555ROW_SSE2 +#define HAS_ARGBTOARGB4444ROW_SSE2 +#define HAS_ARGBTORAWROW_SSSE3 +#define HAS_ARGBTORGB24ROW_SSSE3 +#define HAS_ARGBTORGB565ROW_SSE2 +#define HAS_ARGBTOUV422ROW_SSSE3 +#define HAS_ARGBTOUV444ROW_SSSE3 +#define HAS_ARGBTOUVJROW_SSSE3 +#define HAS_ARGBTOUVROW_SSSE3 +#define HAS_ARGBTOYJROW_SSSE3 +#define HAS_ARGBTOYROW_SSSE3 +#define HAS_BGRATOUVROW_SSSE3 +#define HAS_BGRATOYROW_SSSE3 +#define HAS_COPYROW_ERMS +#define HAS_COPYROW_SSE2 +#define HAS_I400TOARGBROW_SSE2 +#define HAS_I411TOARGBROW_SSSE3 +#define HAS_I422TOABGRROW_SSSE3 +#define HAS_I422TOARGB1555ROW_SSSE3 +#define HAS_I422TOARGB4444ROW_SSSE3 +#define HAS_I422TOARGBROW_SSSE3 +#define HAS_I422TOBGRAROW_SSSE3 +#define HAS_I422TORAWROW_SSSE3 +#define HAS_I422TORGB24ROW_SSSE3 +#define HAS_I422TORGB565ROW_SSSE3 +#define HAS_I422TORGBAROW_SSSE3 +#define HAS_I422TOUYVYROW_SSE2 +#define HAS_I422TOYUY2ROW_SSE2 +#define HAS_I444TOARGBROW_SSSE3 +#define HAS_J400TOARGBROW_SSE2 +#define HAS_J422TOARGBROW_SSSE3 +#define HAS_MERGEUVROW_SSE2 +#define HAS_MIRRORROW_SSE2 +#define HAS_MIRRORROW_SSSE3 +#define HAS_MIRRORROW_UV_SSSE3 +#define HAS_MIRRORUVROW_SSSE3 +#define HAS_NV12TOARGBROW_SSSE3 +#define HAS_NV12TORGB565ROW_SSSE3 +#define HAS_NV21TOARGBROW_SSSE3 +#define HAS_NV21TORGB565ROW_SSSE3 +#define HAS_RAWTOARGBROW_SSSE3 +#define HAS_RAWTOYROW_SSSE3 +#define HAS_RGB24TOARGBROW_SSSE3 +#define HAS_RGB24TOYROW_SSSE3 +#define HAS_RGB565TOARGBROW_SSE2 +#define HAS_RGBATOUVROW_SSSE3 +#define HAS_RGBATOYROW_SSSE3 +#define HAS_SETROW_ERMS +#define HAS_SETROW_X86 +#define HAS_SPLITUVROW_SSE2 +#define HAS_UYVYTOARGBROW_SSSE3 +#define HAS_UYVYTOUV422ROW_SSE2 +#define HAS_UYVYTOUVROW_SSE2 +#define HAS_UYVYTOYROW_SSE2 +#define HAS_YUY2TOARGBROW_SSSE3 +#define HAS_YUY2TOUV422ROW_SSE2 +#define HAS_YUY2TOUVROW_SSE2 +#define HAS_YUY2TOYROW_SSE2 + +// Effects: +#define HAS_ARGBADDROW_SSE2 +#define HAS_ARGBAFFINEROW_SSE2 +#define HAS_ARGBATTENUATEROW_SSSE3 +#define HAS_ARGBBLENDROW_SSSE3 +#define HAS_ARGBCOLORMATRIXROW_SSSE3 +#define HAS_ARGBCOLORTABLEROW_X86 +#define HAS_ARGBCOPYALPHAROW_SSE2 +#define HAS_ARGBCOPYYTOALPHAROW_SSE2 +#define HAS_ARGBGRAYROW_SSSE3 +#define HAS_ARGBLUMACOLORTABLEROW_SSSE3 +#define HAS_ARGBMIRRORROW_SSE2 +#define HAS_ARGBMULTIPLYROW_SSE2 +#define HAS_ARGBPOLYNOMIALROW_SSE2 +#define HAS_ARGBQUANTIZEROW_SSE2 +#define HAS_ARGBSEPIAROW_SSSE3 +#define HAS_ARGBSHADEROW_SSE2 +#define HAS_ARGBSUBTRACTROW_SSE2 +#define HAS_ARGBUNATTENUATEROW_SSE2 +#define HAS_COMPUTECUMULATIVESUMROW_SSE2 +#define HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 +#define HAS_INTERPOLATEROW_SSE2 +#define HAS_INTERPOLATEROW_SSSE3 +#define HAS_RGBCOLORTABLEROW_X86 +#define HAS_SOBELROW_SSE2 +#define HAS_SOBELTOPLANEROW_SSE2 +#define HAS_SOBELXROW_SSE2 +#define HAS_SOBELXYROW_SSE2 +#define HAS_SOBELYROW_SSE2 +#endif + +// The following are available on x64 Visual C and clangcl. +#if !defined(LIBYUV_DISABLE_X86) && defined (_M_X64) && \ + (!defined(__clang__) || defined(__SSSE3__)) +#define HAS_I422TOARGBROW_SSSE3 +#endif + +// GCC >= 4.7.0 required for AVX2. +#if defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) +#if (__GNUC__ > 4) || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 7)) +#define GCC_HAS_AVX2 1 +#endif // GNUC >= 4.7 +#endif // __GNUC__ + +// clang >= 3.4.0 required for AVX2. +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) +#if (__clang_major__ > 3) || (__clang_major__ == 3 && (__clang_minor__ >= 4)) +#define CLANG_HAS_AVX2 1 +#endif // clang >= 3.4 +#endif // __clang__ + +// Visual C 2012 required for AVX2. +#if defined(_M_IX86) && !defined(__clang__) && \ + defined(_MSC_VER) && _MSC_VER >= 1700 +#define VISUALC_HAS_AVX2 1 +#endif // VisualStudio >= 2012 + +// The following are available require VS2012. Port to GCC. +#if !defined(LIBYUV_DISABLE_X86) && defined(VISUALC_HAS_AVX2) +#define HAS_ARGB1555TOARGBROW_AVX2 +#define HAS_ARGB4444TOARGBROW_AVX2 +#define HAS_ARGBTOARGB1555ROW_AVX2 +#define HAS_ARGBTOARGB4444ROW_AVX2 +#define HAS_ARGBTORGB565DITHERROW_AVX2 +#define HAS_ARGBTORGB565DITHERROW_SSE2 +#define HAS_ARGBTORGB565ROW_AVX2 +#define HAS_I411TOARGBROW_AVX2 +#define HAS_I422TOARGB1555ROW_AVX2 +#define HAS_I422TOARGB4444ROW_AVX2 +#define HAS_I422TORGB565ROW_AVX2 +#define HAS_I444TOARGBROW_AVX2 +#define HAS_J400TOARGBROW_AVX2 +#define HAS_NV12TOARGBROW_AVX2 +#define HAS_NV12TORGB565ROW_AVX2 +#define HAS_NV21TOARGBROW_AVX2 +#define HAS_NV21TORGB565ROW_AVX2 +#define HAS_RGB565TOARGBROW_AVX2 +#endif + +// The following are available on all x86 platforms, but +// require VS2012, clang 3.4 or gcc 4.7. +// The code supports NaCL but requires a new compiler and validator. +#if !defined(LIBYUV_DISABLE_X86) && (defined(VISUALC_HAS_AVX2) || \ + defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) +#define HAS_ARGBCOPYALPHAROW_AVX2 +#define HAS_ARGBCOPYYTOALPHAROW_AVX2 +#define HAS_ARGBMIRRORROW_AVX2 +#define HAS_ARGBPOLYNOMIALROW_AVX2 +#define HAS_ARGBSHUFFLEROW_AVX2 +#define HAS_ARGBTOUVROW_AVX2 +#define HAS_ARGBTOYJROW_AVX2 +#define HAS_ARGBTOYROW_AVX2 +#define HAS_COPYROW_AVX +#define HAS_I400TOARGBROW_AVX2 +#define HAS_I422TOABGRROW_AVX2 +#define HAS_I422TOARGBROW_AVX2 +#define HAS_I422TOBGRAROW_AVX2 +#define HAS_I422TORAWROW_AVX2 +#define HAS_I422TORGB24ROW_AVX2 +#define HAS_I422TORGBAROW_AVX2 +#define HAS_INTERPOLATEROW_AVX2 +#define HAS_J422TOARGBROW_AVX2 +#define HAS_MERGEUVROW_AVX2 +#define HAS_MIRRORROW_AVX2 +#define HAS_SPLITUVROW_AVX2 +#define HAS_UYVYTOARGBROW_AVX2 +#define HAS_UYVYTOUV422ROW_AVX2 +#define HAS_UYVYTOUVROW_AVX2 +#define HAS_UYVYTOYROW_AVX2 +#define HAS_YUY2TOARGBROW_AVX2 +#define HAS_YUY2TOUV422ROW_AVX2 +#define HAS_YUY2TOUVROW_AVX2 +#define HAS_YUY2TOYROW_AVX2 + +// Effects: +#define HAS_ARGBADDROW_AVX2 +#define HAS_ARGBATTENUATEROW_AVX2 +#define HAS_ARGBMULTIPLYROW_AVX2 +#define HAS_ARGBSUBTRACTROW_AVX2 +#define HAS_ARGBUNATTENUATEROW_AVX2 +#endif + +// The following are disabled when SSSE3 is available: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)) && \ + !defined(LIBYUV_SSSE3_ONLY) +#define HAS_ARGBATTENUATEROW_SSE2 +#define HAS_ARGBBLENDROW_SSE2 +#define HAS_MIRRORROW_SSE2 +#endif + +// The following are available on Neon platforms: +#if !defined(LIBYUV_DISABLE_NEON) && \ + (defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON)) +#define HAS_ABGRTOUVROW_NEON +#define HAS_ABGRTOYROW_NEON +#define HAS_ARGB1555TOARGBROW_NEON +#define HAS_ARGB1555TOUVROW_NEON +#define HAS_ARGB1555TOYROW_NEON +#define HAS_ARGB4444TOARGBROW_NEON +#define HAS_ARGB4444TOUVROW_NEON +#define HAS_ARGB4444TOYROW_NEON +#define HAS_ARGBTOARGB1555ROW_NEON +#define HAS_ARGBTOARGB4444ROW_NEON +#define HAS_ARGBTORAWROW_NEON +#define HAS_ARGBTORGB24ROW_NEON +#define HAS_ARGBTORGB565ROW_NEON +#define HAS_ARGBTOUV411ROW_NEON +#define HAS_ARGBTOUV422ROW_NEON +#define HAS_ARGBTOUV444ROW_NEON +#define HAS_ARGBTOUVJROW_NEON +#define HAS_ARGBTOUVROW_NEON +#define HAS_ARGBTOYJROW_NEON +#define HAS_ARGBTOYROW_NEON +#define HAS_BGRATOUVROW_NEON +#define HAS_BGRATOYROW_NEON +#define HAS_COPYROW_NEON +#define HAS_J400TOARGBROW_NEON +#define HAS_I411TOARGBROW_NEON +#define HAS_I422TOABGRROW_NEON +#define HAS_I422TOARGB1555ROW_NEON +#define HAS_I422TOARGB4444ROW_NEON +#define HAS_I422TOARGBROW_NEON +#define HAS_I422TOBGRAROW_NEON +#define HAS_I422TORAWROW_NEON +#define HAS_I422TORGB24ROW_NEON +#define HAS_I422TORGB565ROW_NEON +#define HAS_I422TORGBAROW_NEON +#define HAS_I422TOUYVYROW_NEON +#define HAS_I422TOYUY2ROW_NEON +#define HAS_I444TOARGBROW_NEON +#define HAS_MERGEUVROW_NEON +#define HAS_MIRRORROW_NEON +#define HAS_MIRRORUVROW_NEON +#define HAS_NV12TOARGBROW_NEON +#define HAS_NV12TORGB565ROW_NEON +#define HAS_NV21TOARGBROW_NEON +#define HAS_NV21TORGB565ROW_NEON +#define HAS_RAWTOARGBROW_NEON +#define HAS_RAWTOUVROW_NEON +#define HAS_RAWTOYROW_NEON +#define HAS_RGB24TOARGBROW_NEON +#define HAS_RGB24TOUVROW_NEON +#define HAS_RGB24TOYROW_NEON +#define HAS_RGB565TOARGBROW_NEON +#define HAS_RGB565TOUVROW_NEON +#define HAS_RGB565TOYROW_NEON +#define HAS_RGBATOUVROW_NEON +#define HAS_RGBATOYROW_NEON +#define HAS_SETROW_NEON +#define HAS_ARGBSETROW_NEON +#define HAS_SPLITUVROW_NEON +#define HAS_UYVYTOARGBROW_NEON +#define HAS_UYVYTOUV422ROW_NEON +#define HAS_UYVYTOUVROW_NEON +#define HAS_UYVYTOYROW_NEON +#define HAS_I400TOARGBROW_NEON +#define HAS_YUY2TOARGBROW_NEON +#define HAS_YUY2TOUV422ROW_NEON +#define HAS_YUY2TOUVROW_NEON +#define HAS_YUY2TOYROW_NEON +#define HAS_ARGBTORGB565DITHERROW_NEON + +// Effects: +#define HAS_ARGBADDROW_NEON +#define HAS_ARGBATTENUATEROW_NEON +#define HAS_ARGBBLENDROW_NEON +#define HAS_ARGBGRAYROW_NEON +#define HAS_ARGBMIRRORROW_NEON +#define HAS_ARGBMULTIPLYROW_NEON +#define HAS_ARGBQUANTIZEROW_NEON +#define HAS_ARGBSEPIAROW_NEON +#define HAS_ARGBSHADEROW_NEON +#define HAS_ARGBSUBTRACTROW_NEON +#define HAS_INTERPOLATEROW_NEON +#define HAS_SOBELROW_NEON +#define HAS_SOBELTOPLANEROW_NEON +#define HAS_SOBELXROW_NEON +#define HAS_SOBELXYROW_NEON +#define HAS_SOBELYROW_NEON +#define HAS_ARGBCOLORMATRIXROW_NEON +#define HAS_ARGBSHUFFLEROW_NEON +#endif + +// The following are available on Mips platforms: +#if !defined(LIBYUV_DISABLE_MIPS) && defined(__mips__) && \ + (_MIPS_SIM == _MIPS_SIM_ABI32) && (__mips_isa_rev < 6) +#define HAS_COPYROW_MIPS +#if defined(__mips_dsp) && (__mips_dsp_rev >= 2) +#define HAS_I422TOABGRROW_MIPS_DSPR2 +#define HAS_I422TOARGBROW_MIPS_DSPR2 +#define HAS_I422TOBGRAROW_MIPS_DSPR2 +#define HAS_INTERPOLATEROW_MIPS_DSPR2 +#define HAS_MIRRORROW_MIPS_DSPR2 +#define HAS_MIRRORUVROW_MIPS_DSPR2 +#define HAS_SPLITUVROW_MIPS_DSPR2 +#endif +#endif + +#if defined(_MSC_VER) && !defined(__CLR_VER) +#define SIMD_ALIGNED(var) __declspec(align(16)) var +#define SIMD_ALIGNED32(var) __declspec(align(64)) var +typedef __declspec(align(16)) int16 vec16[8]; +typedef __declspec(align(16)) int32 vec32[4]; +typedef __declspec(align(16)) int8 vec8[16]; +typedef __declspec(align(16)) uint16 uvec16[8]; +typedef __declspec(align(16)) uint32 uvec32[4]; +typedef __declspec(align(16)) uint8 uvec8[16]; +typedef __declspec(align(32)) int16 lvec16[16]; +typedef __declspec(align(32)) int32 lvec32[8]; +typedef __declspec(align(32)) int8 lvec8[32]; +typedef __declspec(align(32)) uint16 ulvec16[16]; +typedef __declspec(align(32)) uint32 ulvec32[8]; +typedef __declspec(align(32)) uint8 ulvec8[32]; +#elif defined(__GNUC__) +// Caveat GCC 4.2 to 4.7 have a known issue using vectors with const. +#define SIMD_ALIGNED(var) var __attribute__((aligned(16))) +#define SIMD_ALIGNED32(var) var __attribute__((aligned(64))) +typedef int16 __attribute__((vector_size(16))) vec16; +typedef int32 __attribute__((vector_size(16))) vec32; +typedef int8 __attribute__((vector_size(16))) vec8; +typedef uint16 __attribute__((vector_size(16))) uvec16; +typedef uint32 __attribute__((vector_size(16))) uvec32; +typedef uint8 __attribute__((vector_size(16))) uvec8; +typedef int16 __attribute__((vector_size(32))) lvec16; +typedef int32 __attribute__((vector_size(32))) lvec32; +typedef int8 __attribute__((vector_size(32))) lvec8; +typedef uint16 __attribute__((vector_size(32))) ulvec16; +typedef uint32 __attribute__((vector_size(32))) ulvec32; +typedef uint8 __attribute__((vector_size(32))) ulvec8; +#else +#define SIMD_ALIGNED(var) var +#define SIMD_ALIGNED32(var) var +typedef int16 vec16[8]; +typedef int32 vec32[4]; +typedef int8 vec8[16]; +typedef uint16 uvec16[8]; +typedef uint32 uvec32[4]; +typedef uint8 uvec8[16]; +typedef int16 lvec16[16]; +typedef int32 lvec32[8]; +typedef int8 lvec8[32]; +typedef uint16 ulvec16[16]; +typedef uint32 ulvec32[8]; +typedef uint8 ulvec8[32]; +#endif + +#if defined(__APPLE__) || defined(__x86_64__) || defined(__llvm__) +#define OMITFP +#else +#define OMITFP __attribute__((optimize("omit-frame-pointer"))) +#endif + +// NaCL macros for GCC x86 and x64. +#if defined(__native_client__) +#define LABELALIGN ".p2align 5\n" +#else +#define LABELALIGN +#endif +#if defined(__native_client__) && defined(__x86_64__) +// r14 is used for MEMOP macros. +#define NACL_R14 "r14", +#define BUNDLELOCK ".bundle_lock\n" +#define BUNDLEUNLOCK ".bundle_unlock\n" +#define MEMACCESS(base) "%%nacl:(%%r15,%q" #base ")" +#define MEMACCESS2(offset, base) "%%nacl:" #offset "(%%r15,%q" #base ")" +#define MEMLEA(offset, base) #offset "(%q" #base ")" +#define MEMLEA3(offset, index, scale) \ + #offset "(,%q" #index "," #scale ")" +#define MEMLEA4(offset, base, index, scale) \ + #offset "(%q" #base ",%q" #index "," #scale ")" +#define MEMMOVESTRING(s, d) "%%nacl:(%q" #s "),%%nacl:(%q" #d "), %%r15" +#define MEMSTORESTRING(reg, d) "%%" #reg ",%%nacl:(%q" #d "), %%r15" +#define MEMOPREG(opcode, offset, base, index, scale, reg) \ + BUNDLELOCK \ + "lea " #offset "(%q" #base ",%q" #index "," #scale "),%%r14d\n" \ + #opcode " (%%r15,%%r14),%%" #reg "\n" \ + BUNDLEUNLOCK +#define MEMOPMEM(opcode, reg, offset, base, index, scale) \ + BUNDLELOCK \ + "lea " #offset "(%q" #base ",%q" #index "," #scale "),%%r14d\n" \ + #opcode " %%" #reg ",(%%r15,%%r14)\n" \ + BUNDLEUNLOCK +#define MEMOPARG(opcode, offset, base, index, scale, arg) \ + BUNDLELOCK \ + "lea " #offset "(%q" #base ",%q" #index "," #scale "),%%r14d\n" \ + #opcode " (%%r15,%%r14),%" #arg "\n" \ + BUNDLEUNLOCK +#define VMEMOPREG(opcode, offset, base, index, scale, reg1, reg2) \ + BUNDLELOCK \ + "lea " #offset "(%q" #base ",%q" #index "," #scale "),%%r14d\n" \ + #opcode " (%%r15,%%r14),%%" #reg1 ",%%" #reg2 "\n" \ + BUNDLEUNLOCK +#define VEXTOPMEM(op, sel, reg, offset, base, index, scale) \ + BUNDLELOCK \ + "lea " #offset "(%q" #base ",%q" #index "," #scale "),%%r14d\n" \ + #op " $" #sel ",%%" #reg ",(%%r15,%%r14)\n" \ + BUNDLEUNLOCK +#else // defined(__native_client__) && defined(__x86_64__) +#define NACL_R14 +#define BUNDLEALIGN +#define MEMACCESS(base) "(%" #base ")" +#define MEMACCESS2(offset, base) #offset "(%" #base ")" +#define MEMLEA(offset, base) #offset "(%" #base ")" +#define MEMLEA3(offset, index, scale) \ + #offset "(,%" #index "," #scale ")" +#define MEMLEA4(offset, base, index, scale) \ + #offset "(%" #base ",%" #index "," #scale ")" +#define MEMMOVESTRING(s, d) +#define MEMSTORESTRING(reg, d) +#define MEMOPREG(opcode, offset, base, index, scale, reg) \ + #opcode " " #offset "(%" #base ",%" #index "," #scale "),%%" #reg "\n" +#define MEMOPMEM(opcode, reg, offset, base, index, scale) \ + #opcode " %%" #reg ","#offset "(%" #base ",%" #index "," #scale ")\n" +#define MEMOPARG(opcode, offset, base, index, scale, arg) \ + #opcode " " #offset "(%" #base ",%" #index "," #scale "),%" #arg "\n" +#define VMEMOPREG(opcode, offset, base, index, scale, reg1, reg2) \ + #opcode " " #offset "(%" #base ",%" #index "," #scale "),%%" #reg1 ",%%" \ + #reg2 "\n" +#define VEXTOPMEM(op, sel, reg, offset, base, index, scale) \ + #op " $" #sel ",%%" #reg ","#offset "(%" #base ",%" #index "," #scale ")\n" +#endif // defined(__native_client__) && defined(__x86_64__) + +#if defined(__arm__) || defined(__aarch64__) +#undef MEMACCESS +#if defined(__native_client__) +#define MEMACCESS(base) ".p2align 3\nbic %" #base ", #0xc0000000\n" +#else +#define MEMACCESS(base) +#endif +#endif + +void I444ToARGBRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToARGBRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I411ToARGBRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToBGRARow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_bgra, + int width); +void I422ToABGRRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_abgr, + int width); +void I422ToRGBARow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgba, + int width); +void I422ToRGB24Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb24, + int width); +void I422ToRAWRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_raw, + int width); +void I422ToRGB565Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb565, + int width); +void I422ToARGB1555Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb1555, + int width); +void I422ToARGB4444Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb4444, + int width); +void NV12ToARGBRow_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToARGBRow_NEON(const uint8* src_y, + const uint8* src_vu, + uint8* dst_argb, + int width); +void NV12ToRGB565Row_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_rgb565, + int width); +void NV21ToRGB565Row_NEON(const uint8* src_y, + const uint8* src_vu, + uint8* dst_rgb565, + int width); +void YUY2ToARGBRow_NEON(const uint8* src_yuy2, + uint8* dst_argb, + int width); +void UYVYToARGBRow_NEON(const uint8* src_uyvy, + uint8* dst_argb, + int width); + +void ARGBToYRow_AVX2(const uint8* src_argb, uint8* dst_y, int pix); +void ARGBToYRow_Any_AVX2(const uint8* src_argb, uint8* dst_y, int pix); +void ARGBToYRow_SSSE3(const uint8* src_argb, uint8* dst_y, int pix); +void ARGBToYJRow_AVX2(const uint8* src_argb, uint8* dst_y, int pix); +void ARGBToYJRow_Any_AVX2(const uint8* src_argb, uint8* dst_y, int pix); +void ARGBToYJRow_SSSE3(const uint8* src_argb, uint8* dst_y, int pix); +void BGRAToYRow_SSSE3(const uint8* src_bgra, uint8* dst_y, int pix); +void ABGRToYRow_SSSE3(const uint8* src_abgr, uint8* dst_y, int pix); +void RGBAToYRow_SSSE3(const uint8* src_rgba, uint8* dst_y, int pix); +void RGB24ToYRow_SSSE3(const uint8* src_rgb24, uint8* dst_y, int pix); +void RAWToYRow_SSSE3(const uint8* src_raw, uint8* dst_y, int pix); +void ARGBToYRow_NEON(const uint8* src_argb, uint8* dst_y, int pix); +void ARGBToYJRow_NEON(const uint8* src_argb, uint8* dst_y, int pix); +void ARGBToUV444Row_NEON(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix); +void ARGBToUV422Row_NEON(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix); +void ARGBToUV411Row_NEON(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix); +void ARGBToUVRow_NEON(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int pix); +void ARGBToUVJRow_NEON(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int pix); +void BGRAToUVRow_NEON(const uint8* src_bgra, int src_stride_bgra, + uint8* dst_u, uint8* dst_v, int pix); +void ABGRToUVRow_NEON(const uint8* src_abgr, int src_stride_abgr, + uint8* dst_u, uint8* dst_v, int pix); +void RGBAToUVRow_NEON(const uint8* src_rgba, int src_stride_rgba, + uint8* dst_u, uint8* dst_v, int pix); +void RGB24ToUVRow_NEON(const uint8* src_rgb24, int src_stride_rgb24, + uint8* dst_u, uint8* dst_v, int pix); +void RAWToUVRow_NEON(const uint8* src_raw, int src_stride_raw, + uint8* dst_u, uint8* dst_v, int pix); +void RGB565ToUVRow_NEON(const uint8* src_rgb565, int src_stride_rgb565, + uint8* dst_u, uint8* dst_v, int pix); +void ARGB1555ToUVRow_NEON(const uint8* src_argb1555, int src_stride_argb1555, + uint8* dst_u, uint8* dst_v, int pix); +void ARGB4444ToUVRow_NEON(const uint8* src_argb4444, int src_stride_argb4444, + uint8* dst_u, uint8* dst_v, int pix); +void BGRAToYRow_NEON(const uint8* src_bgra, uint8* dst_y, int pix); +void ABGRToYRow_NEON(const uint8* src_abgr, uint8* dst_y, int pix); +void RGBAToYRow_NEON(const uint8* src_rgba, uint8* dst_y, int pix); +void RGB24ToYRow_NEON(const uint8* src_rgb24, uint8* dst_y, int pix); +void RAWToYRow_NEON(const uint8* src_raw, uint8* dst_y, int pix); +void RGB565ToYRow_NEON(const uint8* src_rgb565, uint8* dst_y, int pix); +void ARGB1555ToYRow_NEON(const uint8* src_argb1555, uint8* dst_y, int pix); +void ARGB4444ToYRow_NEON(const uint8* src_argb4444, uint8* dst_y, int pix); +void ARGBToYRow_C(const uint8* src_argb, uint8* dst_y, int pix); +void ARGBToYJRow_C(const uint8* src_argb, uint8* dst_y, int pix); +void BGRAToYRow_C(const uint8* src_bgra, uint8* dst_y, int pix); +void ABGRToYRow_C(const uint8* src_abgr, uint8* dst_y, int pix); +void RGBAToYRow_C(const uint8* src_rgba, uint8* dst_y, int pix); +void RGB24ToYRow_C(const uint8* src_rgb24, uint8* dst_y, int pix); +void RAWToYRow_C(const uint8* src_raw, uint8* dst_y, int pix); +void RGB565ToYRow_C(const uint8* src_rgb565, uint8* dst_y, int pix); +void ARGB1555ToYRow_C(const uint8* src_argb1555, uint8* dst_y, int pix); +void ARGB4444ToYRow_C(const uint8* src_argb4444, uint8* dst_y, int pix); +void ARGBToYRow_Any_SSSE3(const uint8* src_argb, uint8* dst_y, int pix); +void ARGBToYJRow_Any_SSSE3(const uint8* src_argb, uint8* dst_y, int pix); +void BGRAToYRow_Any_SSSE3(const uint8* src_bgra, uint8* dst_y, int pix); +void ABGRToYRow_Any_SSSE3(const uint8* src_abgr, uint8* dst_y, int pix); +void RGBAToYRow_Any_SSSE3(const uint8* src_rgba, uint8* dst_y, int pix); +void RGB24ToYRow_Any_SSSE3(const uint8* src_rgb24, uint8* dst_y, int pix); +void RAWToYRow_Any_SSSE3(const uint8* src_raw, uint8* dst_y, int pix); +void ARGBToYRow_Any_NEON(const uint8* src_argb, uint8* dst_y, int pix); +void ARGBToYJRow_Any_NEON(const uint8* src_argb, uint8* dst_y, int pix); +void BGRAToYRow_Any_NEON(const uint8* src_bgra, uint8* dst_y, int pix); +void ABGRToYRow_Any_NEON(const uint8* src_abgr, uint8* dst_y, int pix); +void RGBAToYRow_Any_NEON(const uint8* src_rgba, uint8* dst_y, int pix); +void RGB24ToYRow_Any_NEON(const uint8* src_rgb24, uint8* dst_y, int pix); +void RAWToYRow_Any_NEON(const uint8* src_raw, uint8* dst_y, int pix); +void RGB565ToYRow_Any_NEON(const uint8* src_rgb565, uint8* dst_y, int pix); +void ARGB1555ToYRow_Any_NEON(const uint8* src_argb1555, uint8* dst_y, int pix); +void ARGB4444ToYRow_Any_NEON(const uint8* src_argb4444, uint8* dst_y, int pix); + +void ARGBToUVRow_AVX2(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width); +void ARGBToUVRow_Any_AVX2(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width); +void ARGBToUVRow_SSSE3(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width); +void ARGBToUVJRow_SSSE3(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width); +void BGRAToUVRow_SSSE3(const uint8* src_bgra, int src_stride_bgra, + uint8* dst_u, uint8* dst_v, int width); +void ABGRToUVRow_SSSE3(const uint8* src_abgr, int src_stride_abgr, + uint8* dst_u, uint8* dst_v, int width); +void RGBAToUVRow_SSSE3(const uint8* src_rgba, int src_stride_rgba, + uint8* dst_u, uint8* dst_v, int width); +void ARGBToUVRow_Any_SSSE3(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width); +void ARGBToUVJRow_Any_SSSE3(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width); +void BGRAToUVRow_Any_SSSE3(const uint8* src_bgra, int src_stride_bgra, + uint8* dst_u, uint8* dst_v, int width); +void ABGRToUVRow_Any_SSSE3(const uint8* src_abgr, int src_stride_abgr, + uint8* dst_u, uint8* dst_v, int width); +void RGBAToUVRow_Any_SSSE3(const uint8* src_rgba, int src_stride_rgba, + uint8* dst_u, uint8* dst_v, int width); +void ARGBToUV444Row_Any_NEON(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix); +void ARGBToUV422Row_Any_NEON(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix); +void ARGBToUV411Row_Any_NEON(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix); +void ARGBToUVRow_Any_NEON(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int pix); +void ARGBToUVJRow_Any_NEON(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int pix); +void BGRAToUVRow_Any_NEON(const uint8* src_bgra, int src_stride_bgra, + uint8* dst_u, uint8* dst_v, int pix); +void ABGRToUVRow_Any_NEON(const uint8* src_abgr, int src_stride_abgr, + uint8* dst_u, uint8* dst_v, int pix); +void RGBAToUVRow_Any_NEON(const uint8* src_rgba, int src_stride_rgba, + uint8* dst_u, uint8* dst_v, int pix); +void RGB24ToUVRow_Any_NEON(const uint8* src_rgb24, int src_stride_rgb24, + uint8* dst_u, uint8* dst_v, int pix); +void RAWToUVRow_Any_NEON(const uint8* src_raw, int src_stride_raw, + uint8* dst_u, uint8* dst_v, int pix); +void RGB565ToUVRow_Any_NEON(const uint8* src_rgb565, int src_stride_rgb565, + uint8* dst_u, uint8* dst_v, int pix); +void ARGB1555ToUVRow_Any_NEON(const uint8* src_argb1555, + int src_stride_argb1555, + uint8* dst_u, uint8* dst_v, int pix); +void ARGB4444ToUVRow_Any_NEON(const uint8* src_argb4444, + int src_stride_argb4444, + uint8* dst_u, uint8* dst_v, int pix); +void ARGBToUVRow_C(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width); +void ARGBToUVJRow_C(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width); +void BGRAToUVRow_C(const uint8* src_bgra, int src_stride_bgra, + uint8* dst_u, uint8* dst_v, int width); +void ABGRToUVRow_C(const uint8* src_abgr, int src_stride_abgr, + uint8* dst_u, uint8* dst_v, int width); +void RGBAToUVRow_C(const uint8* src_rgba, int src_stride_rgba, + uint8* dst_u, uint8* dst_v, int width); +void RGB24ToUVRow_C(const uint8* src_rgb24, int src_stride_rgb24, + uint8* dst_u, uint8* dst_v, int width); +void RAWToUVRow_C(const uint8* src_raw, int src_stride_raw, + uint8* dst_u, uint8* dst_v, int width); +void RGB565ToUVRow_C(const uint8* src_rgb565, int src_stride_rgb565, + uint8* dst_u, uint8* dst_v, int width); +void ARGB1555ToUVRow_C(const uint8* src_argb1555, int src_stride_argb1555, + uint8* dst_u, uint8* dst_v, int width); +void ARGB4444ToUVRow_C(const uint8* src_argb4444, int src_stride_argb4444, + uint8* dst_u, uint8* dst_v, int width); + +void ARGBToUV444Row_SSSE3(const uint8* src_argb, + uint8* dst_u, uint8* dst_v, int width); +void ARGBToUV444Row_Any_SSSE3(const uint8* src_argb, + uint8* dst_u, uint8* dst_v, int width); + +void ARGBToUV422Row_SSSE3(const uint8* src_argb, + uint8* dst_u, uint8* dst_v, int width); +void ARGBToUV422Row_Any_SSSE3(const uint8* src_argb, + uint8* dst_u, uint8* dst_v, int width); + +void ARGBToUV444Row_C(const uint8* src_argb, + uint8* dst_u, uint8* dst_v, int width); +void ARGBToUV422Row_C(const uint8* src_argb, + uint8* dst_u, uint8* dst_v, int width); +void ARGBToUV411Row_C(const uint8* src_argb, + uint8* dst_u, uint8* dst_v, int width); +void ARGBToUVJ422Row_C(const uint8* src_argb, + uint8* dst_u, uint8* dst_v, int width); + +void MirrorRow_AVX2(const uint8* src, uint8* dst, int width); +void MirrorRow_SSSE3(const uint8* src, uint8* dst, int width); +void MirrorRow_SSE2(const uint8* src, uint8* dst, int width); +void MirrorRow_NEON(const uint8* src, uint8* dst, int width); +void MirrorRow_MIPS_DSPR2(const uint8* src, uint8* dst, int width); +void MirrorRow_C(const uint8* src, uint8* dst, int width); +void MirrorRow_Any_AVX2(const uint8* src, uint8* dst, int width); +void MirrorRow_Any_SSSE3(const uint8* src, uint8* dst, int width); +void MirrorRow_Any_SSE2(const uint8* src, uint8* dst, int width); +void MirrorRow_Any_NEON(const uint8* src, uint8* dst, int width); + +void MirrorUVRow_SSSE3(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int width); +void MirrorUVRow_NEON(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int width); +void MirrorUVRow_MIPS_DSPR2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int width); +void MirrorUVRow_C(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int width); + +void ARGBMirrorRow_AVX2(const uint8* src, uint8* dst, int width); +void ARGBMirrorRow_SSE2(const uint8* src, uint8* dst, int width); +void ARGBMirrorRow_NEON(const uint8* src, uint8* dst, int width); +void ARGBMirrorRow_C(const uint8* src, uint8* dst, int width); +void ARGBMirrorRow_Any_AVX2(const uint8* src, uint8* dst, int width); +void ARGBMirrorRow_Any_SSE2(const uint8* src, uint8* dst, int width); +void ARGBMirrorRow_Any_NEON(const uint8* src, uint8* dst, int width); + +void SplitUVRow_C(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix); +void SplitUVRow_SSE2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix); +void SplitUVRow_AVX2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix); +void SplitUVRow_NEON(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix); +void SplitUVRow_MIPS_DSPR2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int pix); +void SplitUVRow_Any_SSE2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int pix); +void SplitUVRow_Any_AVX2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int pix); +void SplitUVRow_Any_NEON(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int pix); +void SplitUVRow_Any_MIPS_DSPR2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int pix); + +void MergeUVRow_C(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width); +void MergeUVRow_SSE2(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width); +void MergeUVRow_AVX2(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width); +void MergeUVRow_NEON(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width); +void MergeUVRow_Any_SSE2(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width); +void MergeUVRow_Any_AVX2(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width); +void MergeUVRow_Any_NEON(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width); + +void CopyRow_SSE2(const uint8* src, uint8* dst, int count); +void CopyRow_AVX(const uint8* src, uint8* dst, int count); +void CopyRow_ERMS(const uint8* src, uint8* dst, int count); +void CopyRow_NEON(const uint8* src, uint8* dst, int count); +void CopyRow_MIPS(const uint8* src, uint8* dst, int count); +void CopyRow_C(const uint8* src, uint8* dst, int count); +void CopyRow_Any_SSE2(const uint8* src, uint8* dst, int count); +void CopyRow_Any_AVX(const uint8* src, uint8* dst, int count); +void CopyRow_Any_NEON(const uint8* src, uint8* dst, int count); + +void CopyRow_16_C(const uint16* src, uint16* dst, int count); + +void ARGBCopyAlphaRow_C(const uint8* src_argb, uint8* dst_argb, int width); +void ARGBCopyAlphaRow_SSE2(const uint8* src_argb, uint8* dst_argb, int width); +void ARGBCopyAlphaRow_AVX2(const uint8* src_argb, uint8* dst_argb, int width); + +void ARGBCopyYToAlphaRow_C(const uint8* src_y, uint8* dst_argb, int width); +void ARGBCopyYToAlphaRow_SSE2(const uint8* src_y, uint8* dst_argb, int width); +void ARGBCopyYToAlphaRow_AVX2(const uint8* src_y, uint8* dst_argb, int width); + +void SetRow_C(uint8* dst, uint8 v8, int count); +void SetRow_X86(uint8* dst, uint8 v8, int count); +void SetRow_ERMS(uint8* dst, uint8 v8, int count); +void SetRow_NEON(uint8* dst, uint8 v8, int count); +void SetRow_Any_X86(uint8* dst, uint8 v8, int count); +void SetRow_Any_NEON(uint8* dst, uint8 v8, int count); + +void ARGBSetRow_C(uint8* dst_argb, uint32 v32, int count); +void ARGBSetRow_X86(uint8* dst_argb, uint32 v32, int count); +void ARGBSetRow_NEON(uint8* dst_argb, uint32 v32, int count); +void ARGBSetRow_Any_NEON(uint8* dst_argb, uint32 v32, int count); + +// ARGBShufflers for BGRAToARGB etc. +void ARGBShuffleRow_C(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix); +void ARGBShuffleRow_SSE2(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix); +void ARGBShuffleRow_SSSE3(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix); +void ARGBShuffleRow_AVX2(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix); +void ARGBShuffleRow_NEON(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix); +void ARGBShuffleRow_Any_SSE2(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix); +void ARGBShuffleRow_Any_SSSE3(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix); +void ARGBShuffleRow_Any_AVX2(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix); +void ARGBShuffleRow_Any_NEON(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix); + +void RGB24ToARGBRow_SSSE3(const uint8* src_rgb24, uint8* dst_argb, int pix); +void RAWToARGBRow_SSSE3(const uint8* src_raw, uint8* dst_argb, int pix); +void RGB565ToARGBRow_SSE2(const uint8* src_rgb565, uint8* dst_argb, int pix); +void ARGB1555ToARGBRow_SSE2(const uint8* src_argb1555, uint8* dst_argb, + int pix); +void ARGB4444ToARGBRow_SSE2(const uint8* src_argb4444, uint8* dst_argb, + int pix); +void RGB565ToARGBRow_AVX2(const uint8* src_rgb565, uint8* dst_argb, int pix); +void ARGB1555ToARGBRow_AVX2(const uint8* src_argb1555, uint8* dst_argb, + int pix); +void ARGB4444ToARGBRow_AVX2(const uint8* src_argb4444, uint8* dst_argb, + int pix); + +void RGB24ToARGBRow_NEON(const uint8* src_rgb24, uint8* dst_argb, int pix); +void RAWToARGBRow_NEON(const uint8* src_raw, uint8* dst_argb, int pix); +void RGB565ToARGBRow_NEON(const uint8* src_rgb565, uint8* dst_argb, int pix); +void ARGB1555ToARGBRow_NEON(const uint8* src_argb1555, uint8* dst_argb, + int pix); +void ARGB4444ToARGBRow_NEON(const uint8* src_argb4444, uint8* dst_argb, + int pix); +void RGB24ToARGBRow_C(const uint8* src_rgb24, uint8* dst_argb, int pix); +void RAWToARGBRow_C(const uint8* src_raw, uint8* dst_argb, int pix); +void RGB565ToARGBRow_C(const uint8* src_rgb, uint8* dst_argb, int pix); +void ARGB1555ToARGBRow_C(const uint8* src_argb, uint8* dst_argb, int pix); +void ARGB4444ToARGBRow_C(const uint8* src_argb, uint8* dst_argb, int pix); +void RGB24ToARGBRow_Any_SSSE3(const uint8* src_rgb24, uint8* dst_argb, int pix); +void RAWToARGBRow_Any_SSSE3(const uint8* src_raw, uint8* dst_argb, int pix); + +void RGB565ToARGBRow_Any_SSE2(const uint8* src_rgb565, uint8* dst_argb, + int pix); +void ARGB1555ToARGBRow_Any_SSE2(const uint8* src_argb1555, uint8* dst_argb, + int pix); +void ARGB4444ToARGBRow_Any_SSE2(const uint8* src_argb4444, uint8* dst_argb, + int pix); +void RGB565ToARGBRow_Any_AVX2(const uint8* src_rgb565, uint8* dst_argb, + int pix); +void ARGB1555ToARGBRow_Any_AVX2(const uint8* src_argb1555, uint8* dst_argb, + int pix); +void ARGB4444ToARGBRow_Any_AVX2(const uint8* src_argb4444, uint8* dst_argb, + int pix); + +void RGB24ToARGBRow_Any_NEON(const uint8* src_rgb24, uint8* dst_argb, int pix); +void RAWToARGBRow_Any_NEON(const uint8* src_raw, uint8* dst_argb, int pix); +void RGB565ToARGBRow_Any_NEON(const uint8* src_rgb565, uint8* dst_argb, + int pix); +void ARGB1555ToARGBRow_Any_NEON(const uint8* src_argb1555, uint8* dst_argb, + int pix); +void ARGB4444ToARGBRow_Any_NEON(const uint8* src_argb4444, uint8* dst_argb, + int pix); + +void ARGBToRGB24Row_SSSE3(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRAWRow_SSSE3(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRGB565Row_SSE2(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB1555Row_SSE2(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB4444Row_SSE2(const uint8* src_argb, uint8* dst_rgb, int pix); + +void ARGBToRGB565DitherRow_C(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int pix); +void ARGBToRGB565DitherRow_SSE2(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int pix); +void ARGBToRGB565DitherRow_AVX2(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int pix); + +void ARGBToRGB565Row_AVX2(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB1555Row_AVX2(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB4444Row_AVX2(const uint8* src_argb, uint8* dst_rgb, int pix); + +void ARGBToRGB24Row_NEON(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRAWRow_NEON(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRGB565Row_NEON(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB1555Row_NEON(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB4444Row_NEON(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRGB565DitherRow_NEON(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int width); + +void ARGBToRGBARow_C(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRGB24Row_C(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRAWRow_C(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRGB565Row_C(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB1555Row_C(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB4444Row_C(const uint8* src_argb, uint8* dst_rgb, int pix); + +void J400ToARGBRow_SSE2(const uint8* src_y, uint8* dst_argb, int pix); +void J400ToARGBRow_AVX2(const uint8* src_y, uint8* dst_argb, int pix); +void J400ToARGBRow_NEON(const uint8* src_y, uint8* dst_argb, int pix); +void J400ToARGBRow_C(const uint8* src_y, uint8* dst_argb, int pix); +void J400ToARGBRow_Any_SSE2(const uint8* src_y, uint8* dst_argb, int pix); +void J400ToARGBRow_Any_AVX2(const uint8* src_y, uint8* dst_argb, int pix); +void J400ToARGBRow_Any_NEON(const uint8* src_y, uint8* dst_argb, int pix); + +void I444ToARGBRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToARGBRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I411ToARGBRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void NV12ToARGBRow_C(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToRGB565Row_C(const uint8* src_y, + const uint8* src_vu, + uint8* dst_argb, + int width); +void NV12ToRGB565Row_C(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToARGBRow_C(const uint8* src_y, + const uint8* src_vu, + uint8* dst_argb, + int width); +void YUY2ToARGBRow_C(const uint8* src_yuy2, + uint8* dst_argb, + int width); +void UYVYToARGBRow_C(const uint8* src_uyvy, + uint8* dst_argb, + int width); +void J422ToARGBRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToBGRARow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_bgra, + int width); +void I422ToABGRRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_abgr, + int width); +void I422ToRGBARow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgba, + int width); +void I422ToRGB24Row_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb24, + int width); +void I422ToRAWRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_raw, + int width); +void I422ToARGB4444Row_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb4444, + int width); +void I422ToARGB1555Row_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb4444, + int width); +void I422ToRGB565Row_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb565, + int width); +void I422ToARGBRow_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToBGRARow_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToRGBARow_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToABGRRow_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I444ToARGBRow_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I444ToARGBRow_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToARGBRow_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I411ToARGBRow_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I411ToARGBRow_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void NV12ToARGBRow_SSSE3(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToARGBRow_SSSE3(const uint8* src_y, + const uint8* src_vu, + uint8* dst_argb, + int width); +void NV12ToARGBRow_AVX2(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToARGBRow_AVX2(const uint8* src_y, + const uint8* src_vu, + uint8* dst_argb, + int width); +void NV12ToRGB565Row_SSSE3(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToRGB565Row_SSSE3(const uint8* src_y, + const uint8* src_vu, + uint8* dst_argb, + int width); +void NV12ToRGB565Row_AVX2(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToRGB565Row_AVX2(const uint8* src_y, + const uint8* src_vu, + uint8* dst_argb, + int width); +void YUY2ToARGBRow_SSSE3(const uint8* src_yuy2, + uint8* dst_argb, + int width); +void UYVYToARGBRow_SSSE3(const uint8* src_uyvy, + uint8* dst_argb, + int width); +void YUY2ToARGBRow_AVX2(const uint8* src_yuy2, + uint8* dst_argb, + int width); +void UYVYToARGBRow_AVX2(const uint8* src_uyvy, + uint8* dst_argb, + int width); +void J422ToARGBRow_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void J422ToARGBRow_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToBGRARow_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_bgra, + int width); +void I422ToABGRRow_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_abgr, + int width); +void I422ToRGBARow_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgba, + int width); +void I422ToARGB4444Row_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToARGB4444Row_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToARGB1555Row_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToARGB1555Row_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToRGB565Row_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToRGB565Row_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToRGB24Row_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb24, + int width); +void I422ToRGB24Row_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb24, + int width); +void I422ToRAWRow_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_raw, + int width); +void I422ToRAWRow_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_raw, + int width); +void I422ToARGBRow_Any_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToBGRARow_Any_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToRGBARow_Any_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToABGRRow_Any_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I444ToARGBRow_Any_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I444ToARGBRow_Any_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToARGBRow_Any_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I411ToARGBRow_Any_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I411ToARGBRow_Any_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void NV12ToARGBRow_Any_SSSE3(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToARGBRow_Any_SSSE3(const uint8* src_y, + const uint8* src_vu, + uint8* dst_argb, + int width); +void NV12ToARGBRow_Any_AVX2(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToARGBRow_Any_AVX2(const uint8* src_y, + const uint8* src_vu, + uint8* dst_argb, + int width); +void NV12ToRGB565Row_Any_SSSE3(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToRGB565Row_Any_SSSE3(const uint8* src_y, + const uint8* src_vu, + uint8* dst_argb, + int width); +void NV12ToRGB565Row_Any_AVX2(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToRGB565Row_Any_AVX2(const uint8* src_y, + const uint8* src_vu, + uint8* dst_argb, + int width); +void YUY2ToARGBRow_Any_SSSE3(const uint8* src_yuy2, + uint8* dst_argb, + int width); +void UYVYToARGBRow_Any_SSSE3(const uint8* src_uyvy, + uint8* dst_argb, + int width); +void YUY2ToARGBRow_Any_AVX2(const uint8* src_yuy2, + uint8* dst_argb, + int width); +void UYVYToARGBRow_Any_AVX2(const uint8* src_uyvy, + uint8* dst_argb, + int width); +void J422ToARGBRow_Any_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void J422ToARGBRow_Any_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToBGRARow_Any_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_bgra, + int width); +void I422ToABGRRow_Any_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_abgr, + int width); +void I422ToRGBARow_Any_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgba, + int width); +void I422ToARGB4444Row_Any_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgba, + int width); +void I422ToARGB4444Row_Any_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgba, + int width); +void I422ToARGB1555Row_Any_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgba, + int width); +void I422ToARGB1555Row_Any_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgba, + int width); +void I422ToRGB565Row_Any_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgba, + int width); +void I422ToRGB565Row_Any_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgba, + int width); +void I422ToRGB24Row_Any_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToRGB24Row_Any_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToRAWRow_Any_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToRAWRow_Any_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); + +void I400ToARGBRow_C(const uint8* src_y, uint8* dst_argb, int width); +void I400ToARGBRow_SSE2(const uint8* src_y, uint8* dst_argb, int width); +void I400ToARGBRow_AVX2(const uint8* src_y, uint8* dst_argb, int width); +void I400ToARGBRow_NEON(const uint8* src_y, uint8* dst_argb, int width); +void I400ToARGBRow_Any_SSE2(const uint8* src_y, uint8* dst_argb, int width); +void I400ToARGBRow_Any_AVX2(const uint8* src_y, uint8* dst_argb, int width); +void I400ToARGBRow_Any_NEON(const uint8* src_y, uint8* dst_argb, int width); + +// ARGB preattenuated alpha blend. +void ARGBBlendRow_SSSE3(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBBlendRow_SSE2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBBlendRow_NEON(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBBlendRow_C(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); + +// ARGB multiply images. Same API as Blend, but these require +// pointer and width alignment for SSE2. +void ARGBMultiplyRow_C(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBMultiplyRow_SSE2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBMultiplyRow_Any_SSE2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBMultiplyRow_AVX2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBMultiplyRow_Any_AVX2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBMultiplyRow_NEON(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBMultiplyRow_Any_NEON(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); + +// ARGB add images. +void ARGBAddRow_C(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBAddRow_SSE2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBAddRow_Any_SSE2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBAddRow_AVX2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBAddRow_Any_AVX2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBAddRow_NEON(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBAddRow_Any_NEON(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); + +// ARGB subtract images. Same API as Blend, but these require +// pointer and width alignment for SSE2. +void ARGBSubtractRow_C(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBSubtractRow_SSE2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBSubtractRow_Any_SSE2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBSubtractRow_AVX2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBSubtractRow_Any_AVX2(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBSubtractRow_NEON(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); +void ARGBSubtractRow_Any_NEON(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width); + +void ARGBToRGB24Row_Any_SSSE3(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRAWRow_Any_SSSE3(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRGB565Row_Any_SSE2(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB1555Row_Any_SSE2(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB4444Row_Any_SSE2(const uint8* src_argb, uint8* dst_rgb, int pix); + +void ARGBToRGB565DitherRow_Any_SSE2(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int pix); +void ARGBToRGB565DitherRow_Any_AVX2(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int pix); + +void ARGBToRGB565Row_Any_AVX2(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB1555Row_Any_AVX2(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB4444Row_Any_AVX2(const uint8* src_argb, uint8* dst_rgb, int pix); + +void ARGBToRGB24Row_Any_NEON(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRAWRow_Any_NEON(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRGB565Row_Any_NEON(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB1555Row_Any_NEON(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToARGB4444Row_Any_NEON(const uint8* src_argb, uint8* dst_rgb, int pix); +void ARGBToRGB565DitherRow_Any_NEON(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int width); + +void I444ToARGBRow_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToARGBRow_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I411ToARGBRow_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToBGRARow_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToABGRRow_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToRGBARow_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToRGB24Row_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToRAWRow_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToARGB4444Row_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToARGB1555Row_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToRGB565Row_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void NV12ToARGBRow_Any_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToARGBRow_Any_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV12ToRGB565Row_Any_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void NV21ToRGB565Row_Any_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width); +void YUY2ToARGBRow_Any_NEON(const uint8* src_yuy2, + uint8* dst_argb, + int width); +void UYVYToARGBRow_Any_NEON(const uint8* src_uyvy, + uint8* dst_argb, + int width); +void I422ToARGBRow_MIPS_DSPR2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToBGRARow_MIPS_DSPR2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToABGRRow_MIPS_DSPR2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToARGBRow_MIPS_DSPR2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToBGRARow_MIPS_DSPR2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); +void I422ToABGRRow_MIPS_DSPR2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width); + +void YUY2ToYRow_AVX2(const uint8* src_yuy2, uint8* dst_y, int pix); +void YUY2ToUVRow_AVX2(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToUV422Row_AVX2(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToYRow_SSE2(const uint8* src_yuy2, uint8* dst_y, int pix); +void YUY2ToUVRow_SSE2(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToUV422Row_SSE2(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToYRow_NEON(const uint8* src_yuy2, uint8* dst_y, int pix); +void YUY2ToUVRow_NEON(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToUV422Row_NEON(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToYRow_C(const uint8* src_yuy2, uint8* dst_y, int pix); +void YUY2ToUVRow_C(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToUV422Row_C(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToYRow_Any_AVX2(const uint8* src_yuy2, uint8* dst_y, int pix); +void YUY2ToUVRow_Any_AVX2(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToUV422Row_Any_AVX2(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToYRow_Any_SSE2(const uint8* src_yuy2, uint8* dst_y, int pix); +void YUY2ToUVRow_Any_SSE2(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToUV422Row_Any_SSE2(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToYRow_Any_NEON(const uint8* src_yuy2, uint8* dst_y, int pix); +void YUY2ToUVRow_Any_NEON(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void YUY2ToUV422Row_Any_NEON(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToYRow_AVX2(const uint8* src_uyvy, uint8* dst_y, int pix); +void UYVYToUVRow_AVX2(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToUV422Row_AVX2(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToYRow_SSE2(const uint8* src_uyvy, uint8* dst_y, int pix); +void UYVYToUVRow_SSE2(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToUV422Row_SSE2(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToYRow_AVX2(const uint8* src_uyvy, uint8* dst_y, int pix); +void UYVYToUVRow_AVX2(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToUV422Row_AVX2(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToYRow_NEON(const uint8* src_uyvy, uint8* dst_y, int pix); +void UYVYToUVRow_NEON(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToUV422Row_NEON(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix); + +void UYVYToYRow_C(const uint8* src_uyvy, uint8* dst_y, int pix); +void UYVYToUVRow_C(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToUV422Row_C(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToYRow_Any_AVX2(const uint8* src_uyvy, uint8* dst_y, int pix); +void UYVYToUVRow_Any_AVX2(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToUV422Row_Any_AVX2(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToYRow_Any_SSE2(const uint8* src_uyvy, uint8* dst_y, int pix); +void UYVYToUVRow_Any_SSE2(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToUV422Row_Any_SSE2(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToYRow_Any_NEON(const uint8* src_uyvy, uint8* dst_y, int pix); +void UYVYToUVRow_Any_NEON(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix); +void UYVYToUV422Row_Any_NEON(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix); + +void I422ToYUY2Row_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_yuy2, int width); +void I422ToUYVYRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_uyvy, int width); +void I422ToYUY2Row_SSE2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_yuy2, int width); +void I422ToUYVYRow_SSE2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_uyvy, int width); +void I422ToYUY2Row_Any_SSE2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_yuy2, int width); +void I422ToUYVYRow_Any_SSE2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_uyvy, int width); +void I422ToYUY2Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_yuy2, int width); +void I422ToUYVYRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_uyvy, int width); +void I422ToYUY2Row_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_yuy2, int width); +void I422ToUYVYRow_Any_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_uyvy, int width); + +// Effects related row functions. +void ARGBAttenuateRow_C(const uint8* src_argb, uint8* dst_argb, int width); +void ARGBAttenuateRow_SSE2(const uint8* src_argb, uint8* dst_argb, int width); +void ARGBAttenuateRow_SSSE3(const uint8* src_argb, uint8* dst_argb, int width); +void ARGBAttenuateRow_AVX2(const uint8* src_argb, uint8* dst_argb, int width); +void ARGBAttenuateRow_NEON(const uint8* src_argb, uint8* dst_argb, int width); +void ARGBAttenuateRow_Any_SSE2(const uint8* src_argb, uint8* dst_argb, + int width); +void ARGBAttenuateRow_Any_SSSE3(const uint8* src_argb, uint8* dst_argb, + int width); +void ARGBAttenuateRow_Any_AVX2(const uint8* src_argb, uint8* dst_argb, + int width); +void ARGBAttenuateRow_Any_NEON(const uint8* src_argb, uint8* dst_argb, + int width); + +// Inverse table for unattenuate, shared by C and SSE2. +extern const uint32 fixed_invtbl8[256]; +void ARGBUnattenuateRow_C(const uint8* src_argb, uint8* dst_argb, int width); +void ARGBUnattenuateRow_SSE2(const uint8* src_argb, uint8* dst_argb, int width); +void ARGBUnattenuateRow_AVX2(const uint8* src_argb, uint8* dst_argb, int width); +void ARGBUnattenuateRow_Any_SSE2(const uint8* src_argb, uint8* dst_argb, + int width); +void ARGBUnattenuateRow_Any_AVX2(const uint8* src_argb, uint8* dst_argb, + int width); + +void ARGBGrayRow_C(const uint8* src_argb, uint8* dst_argb, int width); +void ARGBGrayRow_SSSE3(const uint8* src_argb, uint8* dst_argb, int width); +void ARGBGrayRow_NEON(const uint8* src_argb, uint8* dst_argb, int width); + +void ARGBSepiaRow_C(uint8* dst_argb, int width); +void ARGBSepiaRow_SSSE3(uint8* dst_argb, int width); +void ARGBSepiaRow_NEON(uint8* dst_argb, int width); + +void ARGBColorMatrixRow_C(const uint8* src_argb, uint8* dst_argb, + const int8* matrix_argb, int width); +void ARGBColorMatrixRow_SSSE3(const uint8* src_argb, uint8* dst_argb, + const int8* matrix_argb, int width); +void ARGBColorMatrixRow_NEON(const uint8* src_argb, uint8* dst_argb, + const int8* matrix_argb, int width); + +void ARGBColorTableRow_C(uint8* dst_argb, const uint8* table_argb, int width); +void ARGBColorTableRow_X86(uint8* dst_argb, const uint8* table_argb, int width); + +void RGBColorTableRow_C(uint8* dst_argb, const uint8* table_argb, int width); +void RGBColorTableRow_X86(uint8* dst_argb, const uint8* table_argb, int width); + +void ARGBQuantizeRow_C(uint8* dst_argb, int scale, int interval_size, + int interval_offset, int width); +void ARGBQuantizeRow_SSE2(uint8* dst_argb, int scale, int interval_size, + int interval_offset, int width); +void ARGBQuantizeRow_NEON(uint8* dst_argb, int scale, int interval_size, + int interval_offset, int width); + +void ARGBShadeRow_C(const uint8* src_argb, uint8* dst_argb, int width, + uint32 value); +void ARGBShadeRow_SSE2(const uint8* src_argb, uint8* dst_argb, int width, + uint32 value); +void ARGBShadeRow_NEON(const uint8* src_argb, uint8* dst_argb, int width, + uint32 value); + +// Used for blur. +void CumulativeSumToAverageRow_SSE2(const int32* topleft, const int32* botleft, + int width, int area, uint8* dst, int count); +void ComputeCumulativeSumRow_SSE2(const uint8* row, int32* cumsum, + const int32* previous_cumsum, int width); + +void CumulativeSumToAverageRow_C(const int32* topleft, const int32* botleft, + int width, int area, uint8* dst, int count); +void ComputeCumulativeSumRow_C(const uint8* row, int32* cumsum, + const int32* previous_cumsum, int width); + +LIBYUV_API +void ARGBAffineRow_C(const uint8* src_argb, int src_argb_stride, + uint8* dst_argb, const float* uv_dudv, int width); +LIBYUV_API +void ARGBAffineRow_SSE2(const uint8* src_argb, int src_argb_stride, + uint8* dst_argb, const float* uv_dudv, int width); + +// Used for I420Scale, ARGBScale, and ARGBInterpolate. +void InterpolateRow_C(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, + int width, int source_y_fraction); +void InterpolateRow_SSE2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, int width, + int source_y_fraction); +void InterpolateRow_SSSE3(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, int width, + int source_y_fraction); +void InterpolateRow_AVX2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, int width, + int source_y_fraction); +void InterpolateRow_NEON(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, int width, + int source_y_fraction); +void InterpolateRow_MIPS_DSPR2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, int width, + int source_y_fraction); +void InterpolateRow_Any_NEON(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, int width, + int source_y_fraction); +void InterpolateRow_Any_SSE2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, int width, + int source_y_fraction); +void InterpolateRow_Any_SSSE3(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, int width, + int source_y_fraction); +void InterpolateRow_Any_AVX2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, int width, + int source_y_fraction); +void InterpolateRow_Any_MIPS_DSPR2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, int width, + int source_y_fraction); + +void InterpolateRow_16_C(uint16* dst_ptr, const uint16* src_ptr, + ptrdiff_t src_stride_ptr, + int width, int source_y_fraction); + +// Sobel images. +void SobelXRow_C(const uint8* src_y0, const uint8* src_y1, const uint8* src_y2, + uint8* dst_sobelx, int width); +void SobelXRow_SSE2(const uint8* src_y0, const uint8* src_y1, + const uint8* src_y2, uint8* dst_sobelx, int width); +void SobelXRow_NEON(const uint8* src_y0, const uint8* src_y1, + const uint8* src_y2, uint8* dst_sobelx, int width); +void SobelYRow_C(const uint8* src_y0, const uint8* src_y1, + uint8* dst_sobely, int width); +void SobelYRow_SSE2(const uint8* src_y0, const uint8* src_y1, + uint8* dst_sobely, int width); +void SobelYRow_NEON(const uint8* src_y0, const uint8* src_y1, + uint8* dst_sobely, int width); +void SobelRow_C(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width); +void SobelRow_SSE2(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width); +void SobelRow_NEON(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width); +void SobelToPlaneRow_C(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_y, int width); +void SobelToPlaneRow_SSE2(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_y, int width); +void SobelToPlaneRow_NEON(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_y, int width); +void SobelXYRow_C(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width); +void SobelXYRow_SSE2(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width); +void SobelXYRow_NEON(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width); +void SobelRow_Any_SSE2(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width); +void SobelRow_Any_NEON(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width); +void SobelToPlaneRow_Any_SSE2(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_y, int width); +void SobelToPlaneRow_Any_NEON(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_y, int width); +void SobelXYRow_Any_SSE2(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width); +void SobelXYRow_Any_NEON(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width); + +void ARGBPolynomialRow_C(const uint8* src_argb, + uint8* dst_argb, const float* poly, + int width); +void ARGBPolynomialRow_SSE2(const uint8* src_argb, + uint8* dst_argb, const float* poly, + int width); +void ARGBPolynomialRow_AVX2(const uint8* src_argb, + uint8* dst_argb, const float* poly, + int width); + +void ARGBLumaColorTableRow_C(const uint8* src_argb, uint8* dst_argb, int width, + const uint8* luma, uint32 lumacoeff); +void ARGBLumaColorTableRow_SSSE3(const uint8* src_argb, uint8* dst_argb, + int width, + const uint8* luma, uint32 lumacoeff); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_ROW_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/scale.h b/third_party/aom/third_party/libyuv/include/libyuv/scale.h new file mode 100644 index 0000000000..3974aba34e --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/scale.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_SCALE_H_ // NOLINT +#define INCLUDE_LIBYUV_SCALE_H_ + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Supported filtering. +typedef enum FilterMode { + kFilterNone = 0, // Point sample; Fastest. + kFilterLinear = 1, // Filter horizontally only. + kFilterBilinear = 2, // Faster than box, but lower quality scaling down. + kFilterBox = 3 // Highest quality. +} FilterModeEnum; + +// Scale a YUV plane. +LIBYUV_API +void ScalePlane(const uint8* src, int src_stride, + int src_width, int src_height, + uint8* dst, int dst_stride, + int dst_width, int dst_height, + enum FilterMode filtering); + +LIBYUV_API +void ScalePlane_16(const uint16* src, int src_stride, + int src_width, int src_height, + uint16* dst, int dst_stride, + int dst_width, int dst_height, + enum FilterMode filtering); + +// Scales a YUV 4:2:0 image from the src width and height to the +// dst width and height. +// If filtering is kFilterNone, a simple nearest-neighbor algorithm is +// used. This produces basic (blocky) quality at the fastest speed. +// If filtering is kFilterBilinear, interpolation is used to produce a better +// quality image, at the expense of speed. +// If filtering is kFilterBox, averaging is used to produce ever better +// quality image, at further expense of speed. +// Returns 0 if successful. + +LIBYUV_API +int I420Scale(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + int src_width, int src_height, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int dst_width, int dst_height, + enum FilterMode filtering); + +LIBYUV_API +int I420Scale_16(const uint16* src_y, int src_stride_y, + const uint16* src_u, int src_stride_u, + const uint16* src_v, int src_stride_v, + int src_width, int src_height, + uint16* dst_y, int dst_stride_y, + uint16* dst_u, int dst_stride_u, + uint16* dst_v, int dst_stride_v, + int dst_width, int dst_height, + enum FilterMode filtering); + +#ifdef __cplusplus +// Legacy API. Deprecated. +LIBYUV_API +int Scale(const uint8* src_y, const uint8* src_u, const uint8* src_v, + int src_stride_y, int src_stride_u, int src_stride_v, + int src_width, int src_height, + uint8* dst_y, uint8* dst_u, uint8* dst_v, + int dst_stride_y, int dst_stride_u, int dst_stride_v, + int dst_width, int dst_height, + LIBYUV_BOOL interpolate); + +// Legacy API. Deprecated. +LIBYUV_API +int ScaleOffset(const uint8* src_i420, int src_width, int src_height, + uint8* dst_i420, int dst_width, int dst_height, int dst_yoffset, + LIBYUV_BOOL interpolate); + +// For testing, allow disabling of specialized scalers. +LIBYUV_API +void SetUseReferenceImpl(LIBYUV_BOOL use); +#endif // __cplusplus + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_SCALE_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/scale_argb.h b/third_party/aom/third_party/libyuv/include/libyuv/scale_argb.h new file mode 100644 index 0000000000..22563837dd --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/scale_argb.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_SCALE_ARGB_H_ // NOLINT +#define INCLUDE_LIBYUV_SCALE_ARGB_H_ + +#include "libyuv/basic_types.h" +#include "libyuv/scale.h" // For FilterMode + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +LIBYUV_API +int ARGBScale(const uint8* src_argb, int src_stride_argb, + int src_width, int src_height, + uint8* dst_argb, int dst_stride_argb, + int dst_width, int dst_height, + enum FilterMode filtering); + +// Clipped scale takes destination rectangle coordinates for clip values. +LIBYUV_API +int ARGBScaleClip(const uint8* src_argb, int src_stride_argb, + int src_width, int src_height, + uint8* dst_argb, int dst_stride_argb, + int dst_width, int dst_height, + int clip_x, int clip_y, int clip_width, int clip_height, + enum FilterMode filtering); + +// TODO(fbarchard): Implement this. +// Scale with YUV conversion to ARGB and clipping. +LIBYUV_API +int YUVToARGBScaleClip(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint32 src_fourcc, + int src_width, int src_height, + uint8* dst_argb, int dst_stride_argb, + uint32 dst_fourcc, + int dst_width, int dst_height, + int clip_x, int clip_y, int clip_width, int clip_height, + enum FilterMode filtering); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_SCALE_ARGB_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/scale_row.h b/third_party/aom/third_party/libyuv/include/libyuv/scale_row.h new file mode 100644 index 0000000000..a46b5ce692 --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/scale_row.h @@ -0,0 +1,479 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_SCALE_ROW_H_ // NOLINT +#define INCLUDE_LIBYUV_SCALE_ROW_H_ + +#include "libyuv/basic_types.h" +#include "libyuv/scale.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#if defined(__pnacl__) || defined(__CLR_VER) || \ + (defined(__i386__) && !defined(__SSE2__)) +#define LIBYUV_DISABLE_X86 +#endif + +// Visual C 2012 required for AVX2. +#if defined(_M_IX86) && !defined(__clang__) && \ + defined(_MSC_VER) && _MSC_VER >= 1700 +#define VISUALC_HAS_AVX2 1 +#endif // VisualStudio >= 2012 + +// The following are available on all x86 platforms: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)) +#define HAS_FIXEDDIV1_X86 +#define HAS_FIXEDDIV_X86 +#define HAS_SCALEARGBCOLS_SSE2 +#define HAS_SCALEARGBCOLSUP2_SSE2 +#define HAS_SCALEARGBFILTERCOLS_SSSE3 +#define HAS_SCALEARGBROWDOWN2_SSE2 +#define HAS_SCALEARGBROWDOWNEVEN_SSE2 +#define HAS_SCALECOLSUP2_SSE2 +#define HAS_SCALEFILTERCOLS_SSSE3 +#define HAS_SCALEROWDOWN2_SSE2 +#define HAS_SCALEROWDOWN34_SSSE3 +#define HAS_SCALEROWDOWN38_SSSE3 +#define HAS_SCALEROWDOWN4_SSE2 +#endif + +// The following are available on VS2012: +#if !defined(LIBYUV_DISABLE_X86) && defined(VISUALC_HAS_AVX2) +#define HAS_SCALEADDROW_AVX2 +#define HAS_SCALEROWDOWN2_AVX2 +#define HAS_SCALEROWDOWN4_AVX2 +#endif + +// The following are available on Visual C: +#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && !defined(__clang__) +#define HAS_SCALEADDROW_SSE2 +#endif + +// The following are available on Neon platforms: +#if !defined(LIBYUV_DISABLE_NEON) && !defined(__native_client__) && \ + (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) +#define HAS_SCALEARGBCOLS_NEON +#define HAS_SCALEARGBROWDOWN2_NEON +#define HAS_SCALEARGBROWDOWNEVEN_NEON +#define HAS_SCALEFILTERCOLS_NEON +#define HAS_SCALEROWDOWN2_NEON +#define HAS_SCALEROWDOWN34_NEON +#define HAS_SCALEROWDOWN38_NEON +#define HAS_SCALEROWDOWN4_NEON +#define HAS_SCALEARGBFILTERCOLS_NEON +#endif + +// The following are available on Mips platforms: +#if !defined(LIBYUV_DISABLE_MIPS) && !defined(__native_client__) && \ + defined(__mips__) && defined(__mips_dsp) && (__mips_dsp_rev >= 2) +#define HAS_SCALEROWDOWN2_MIPS_DSPR2 +#define HAS_SCALEROWDOWN4_MIPS_DSPR2 +#define HAS_SCALEROWDOWN34_MIPS_DSPR2 +#define HAS_SCALEROWDOWN38_MIPS_DSPR2 +#endif + +// Scale ARGB vertically with bilinear interpolation. +void ScalePlaneVertical(int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_argb, uint8* dst_argb, + int x, int y, int dy, + int bpp, enum FilterMode filtering); + +void ScalePlaneVertical_16(int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint16* src_argb, uint16* dst_argb, + int x, int y, int dy, + int wpp, enum FilterMode filtering); + +// Simplify the filtering based on scale factors. +enum FilterMode ScaleFilterReduce(int src_width, int src_height, + int dst_width, int dst_height, + enum FilterMode filtering); + +// Divide num by div and return as 16.16 fixed point result. +int FixedDiv_C(int num, int div); +int FixedDiv_X86(int num, int div); +// Divide num - 1 by div - 1 and return as 16.16 fixed point result. +int FixedDiv1_C(int num, int div); +int FixedDiv1_X86(int num, int div); +#ifdef HAS_FIXEDDIV_X86 +#define FixedDiv FixedDiv_X86 +#define FixedDiv1 FixedDiv1_X86 +#else +#define FixedDiv FixedDiv_C +#define FixedDiv1 FixedDiv1_C +#endif + +// Compute slope values for stepping. +void ScaleSlope(int src_width, int src_height, + int dst_width, int dst_height, + enum FilterMode filtering, + int* x, int* y, int* dx, int* dy); + +void ScaleRowDown2_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown2_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width); +void ScaleRowDown2Linear_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown2Linear_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width); +void ScaleRowDown2Box_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown2Box_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width); +void ScaleRowDown4_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown4_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width); +void ScaleRowDown4Box_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown4Box_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width); +void ScaleRowDown34_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown34_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width); +void ScaleRowDown34_0_Box_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* d, int dst_width); +void ScaleRowDown34_0_Box_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* d, int dst_width); +void ScaleRowDown34_1_Box_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* d, int dst_width); +void ScaleRowDown34_1_Box_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* d, int dst_width); +void ScaleCols_C(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx); +void ScaleCols_16_C(uint16* dst_ptr, const uint16* src_ptr, + int dst_width, int x, int dx); +void ScaleColsUp2_C(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int, int); +void ScaleColsUp2_16_C(uint16* dst_ptr, const uint16* src_ptr, + int dst_width, int, int); +void ScaleFilterCols_C(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx); +void ScaleFilterCols_16_C(uint16* dst_ptr, const uint16* src_ptr, + int dst_width, int x, int dx); +void ScaleFilterCols64_C(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx); +void ScaleFilterCols64_16_C(uint16* dst_ptr, const uint16* src_ptr, + int dst_width, int x, int dx); +void ScaleRowDown38_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown38_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width); +void ScaleRowDown38_3_Box_C(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown38_3_Box_16_C(const uint16* src_ptr, + ptrdiff_t src_stride, + uint16* dst_ptr, int dst_width); +void ScaleRowDown38_2_Box_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown38_2_Box_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst_ptr, int dst_width); +void ScaleAddRow_C(const uint8* src_ptr, uint16* dst_ptr, int src_width); +void ScaleAddRow_16_C(const uint16* src_ptr, uint32* dst_ptr, int src_width); +void ScaleARGBRowDown2_C(const uint8* src_argb, + ptrdiff_t src_stride, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDown2Linear_C(const uint8* src_argb, + ptrdiff_t src_stride, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDown2Box_C(const uint8* src_argb, ptrdiff_t src_stride, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDownEven_C(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDownEvenBox_C(const uint8* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width); +void ScaleARGBCols_C(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx); +void ScaleARGBCols64_C(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx); +void ScaleARGBColsUp2_C(uint8* dst_argb, const uint8* src_argb, + int dst_width, int, int); +void ScaleARGBFilterCols_C(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx); +void ScaleARGBFilterCols64_C(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx); + +// Specialized scalers for x86. +void ScaleRowDown2_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown2Linear_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown2Box_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown2_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown2Linear_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown2Box_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown4_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown4Box_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown4_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown4Box_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); + +void ScaleRowDown34_SSSE3(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown34_1_Box_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown34_0_Box_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown38_SSSE3(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown38_3_Box_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown38_2_Box_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown2_Any_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown2Linear_Any_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown2Box_Any_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown2_Any_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown2Linear_Any_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown2Box_Any_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown4_Any_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown4Box_Any_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown4_Any_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown4Box_Any_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); + +void ScaleRowDown34_Any_SSSE3(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown34_1_Box_Any_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown34_0_Box_Any_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown38_Any_SSSE3(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown38_3_Box_Any_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown38_2_Box_Any_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); + +void ScaleAddRow_SSE2(const uint8* src_ptr, uint16* dst_ptr, int src_width); +void ScaleAddRow_AVX2(const uint8* src_ptr, uint16* dst_ptr, int src_width); +void ScaleAddRow_Any_SSE2(const uint8* src_ptr, uint16* dst_ptr, int src_width); +void ScaleAddRow_Any_AVX2(const uint8* src_ptr, uint16* dst_ptr, int src_width); + +void ScaleFilterCols_SSSE3(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx); +void ScaleColsUp2_SSE2(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx); + + +// ARGB Column functions +void ScaleARGBCols_SSE2(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx); +void ScaleARGBFilterCols_SSSE3(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx); +void ScaleARGBColsUp2_SSE2(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx); +void ScaleARGBFilterCols_NEON(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx); +void ScaleARGBCols_NEON(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx); +void ScaleARGBFilterCols_Any_NEON(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx); +void ScaleARGBCols_Any_NEON(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx); + +// ARGB Row functions +void ScaleARGBRowDown2_SSE2(const uint8* src_argb, ptrdiff_t src_stride, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDown2Linear_SSE2(const uint8* src_argb, ptrdiff_t src_stride, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDown2Box_SSE2(const uint8* src_argb, ptrdiff_t src_stride, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDown2_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleARGBRowDown2Linear_NEON(const uint8* src_argb, ptrdiff_t src_stride, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDown2Box_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleARGBRowDown2_Any_SSE2(const uint8* src_argb, ptrdiff_t src_stride, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDown2Linear_Any_SSE2(const uint8* src_argb, + ptrdiff_t src_stride, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDown2Box_Any_SSE2(const uint8* src_argb, ptrdiff_t src_stride, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDown2_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleARGBRowDown2Linear_Any_NEON(const uint8* src_argb, + ptrdiff_t src_stride, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDown2Box_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); + +void ScaleARGBRowDownEven_SSE2(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, uint8* dst_argb, int dst_width); +void ScaleARGBRowDownEvenBox_SSE2(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDownEven_NEON(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDownEvenBox_NEON(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDownEven_Any_SSE2(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDownEvenBox_Any_SSE2(const uint8* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDownEven_Any_NEON(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width); +void ScaleARGBRowDownEvenBox_Any_NEON(const uint8* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width); + +// ScaleRowDown2Box also used by planar functions +// NEON downscalers with interpolation. + +// Note - not static due to reuse in convert for 444 to 420. +void ScaleRowDown2_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown2Linear_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown2Box_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); + +void ScaleRowDown4_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown4Box_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); + +// Down scale from 4 to 3 pixels. Use the neon multilane read/write +// to load up the every 4th pixel into a 4 different registers. +// Point samples 32 pixels to 24 pixels. +void ScaleRowDown34_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown34_0_Box_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown34_1_Box_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); + +// 32 -> 12 +void ScaleRowDown38_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +// 32x3 -> 12x1 +void ScaleRowDown38_3_Box_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +// 32x2 -> 12x1 +void ScaleRowDown38_2_Box_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); + +void ScaleRowDown2_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown2Linear_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown2Box_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown4_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown4Box_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown34_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown34_0_Box_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown34_1_Box_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +// 32 -> 12 +void ScaleRowDown38_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +// 32x3 -> 12x1 +void ScaleRowDown38_3_Box_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +// 32x2 -> 12x1 +void ScaleRowDown38_2_Box_Any_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); + +void ScaleAddRow_NEON(const uint8* src_ptr, uint16* dst_ptr, int src_width); +void ScaleAddRow_Any_NEON(const uint8* src_ptr, uint16* dst_ptr, int src_width); + +void ScaleFilterCols_NEON(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx); + +void ScaleFilterCols_Any_NEON(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx); + + +void ScaleRowDown2_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown2Box_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown4_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown4Box_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown34_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown34_0_Box_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* d, int dst_width); +void ScaleRowDown34_1_Box_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* d, int dst_width); +void ScaleRowDown38_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width); +void ScaleRowDown38_2_Box_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); +void ScaleRowDown38_3_Box_MIPS_DSPR2(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_SCALE_ROW_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/version.h b/third_party/aom/third_party/libyuv/include/libyuv/version.h new file mode 100644 index 0000000000..287b98ebf2 --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/version.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef INCLUDE_LIBYUV_VERSION_H_ // NOLINT +#define INCLUDE_LIBYUV_VERSION_H_ + +#define LIBYUV_VERSION 1456 + +#endif // INCLUDE_LIBYUV_VERSION_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/include/libyuv/video_common.h b/third_party/aom/third_party/libyuv/include/libyuv/video_common.h new file mode 100644 index 0000000000..7b0a19cc90 --- /dev/null +++ b/third_party/aom/third_party/libyuv/include/libyuv/video_common.h @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// Common definitions for video, including fourcc and VideoFormat. + +#ifndef INCLUDE_LIBYUV_VIDEO_COMMON_H_ // NOLINT +#define INCLUDE_LIBYUV_VIDEO_COMMON_H_ + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +////////////////////////////////////////////////////////////////////////////// +// Definition of FourCC codes +////////////////////////////////////////////////////////////////////////////// + +// Convert four characters to a FourCC code. +// Needs to be a macro otherwise the OS X compiler complains when the kFormat* +// constants are used in a switch. +#ifdef __cplusplus +#define FOURCC(a, b, c, d) ( \ + (static_cast(a)) | (static_cast(b) << 8) | \ + (static_cast(c) << 16) | (static_cast(d) << 24)) +#else +#define FOURCC(a, b, c, d) ( \ + ((uint32)(a)) | ((uint32)(b) << 8) | /* NOLINT */ \ + ((uint32)(c) << 16) | ((uint32)(d) << 24)) /* NOLINT */ +#endif + +// Some pages discussing FourCC codes: +// http://www.fourcc.org/yuv.php +// http://v4l2spec.bytesex.org/spec/book1.htm +// http://developer.apple.com/quicktime/icefloe/dispatch020.html +// http://msdn.microsoft.com/library/windows/desktop/dd206750.aspx#nv12 +// http://people.xiph.org/~xiphmont/containers/nut/nut4cc.txt + +// FourCC codes grouped according to implementation efficiency. +// Primary formats should convert in 1 efficient step. +// Secondary formats are converted in 2 steps. +// Auxilliary formats call primary converters. +enum FourCC { + // 9 Primary YUV formats: 5 planar, 2 biplanar, 2 packed. + FOURCC_I420 = FOURCC('I', '4', '2', '0'), + FOURCC_I422 = FOURCC('I', '4', '2', '2'), + FOURCC_I444 = FOURCC('I', '4', '4', '4'), + FOURCC_I411 = FOURCC('I', '4', '1', '1'), + FOURCC_I400 = FOURCC('I', '4', '0', '0'), + FOURCC_NV21 = FOURCC('N', 'V', '2', '1'), + FOURCC_NV12 = FOURCC('N', 'V', '1', '2'), + FOURCC_YUY2 = FOURCC('Y', 'U', 'Y', '2'), + FOURCC_UYVY = FOURCC('U', 'Y', 'V', 'Y'), + + // 2 Secondary YUV formats: row biplanar. + FOURCC_M420 = FOURCC('M', '4', '2', '0'), + FOURCC_Q420 = FOURCC('Q', '4', '2', '0'), // deprecated. + + // 9 Primary RGB formats: 4 32 bpp, 2 24 bpp, 3 16 bpp. + FOURCC_ARGB = FOURCC('A', 'R', 'G', 'B'), + FOURCC_BGRA = FOURCC('B', 'G', 'R', 'A'), + FOURCC_ABGR = FOURCC('A', 'B', 'G', 'R'), + FOURCC_24BG = FOURCC('2', '4', 'B', 'G'), + FOURCC_RAW = FOURCC('r', 'a', 'w', ' '), + FOURCC_RGBA = FOURCC('R', 'G', 'B', 'A'), + FOURCC_RGBP = FOURCC('R', 'G', 'B', 'P'), // rgb565 LE. + FOURCC_RGBO = FOURCC('R', 'G', 'B', 'O'), // argb1555 LE. + FOURCC_R444 = FOURCC('R', '4', '4', '4'), // argb4444 LE. + + // 4 Secondary RGB formats: 4 Bayer Patterns. deprecated. + FOURCC_RGGB = FOURCC('R', 'G', 'G', 'B'), + FOURCC_BGGR = FOURCC('B', 'G', 'G', 'R'), + FOURCC_GRBG = FOURCC('G', 'R', 'B', 'G'), + FOURCC_GBRG = FOURCC('G', 'B', 'R', 'G'), + + // 1 Primary Compressed YUV format. + FOURCC_MJPG = FOURCC('M', 'J', 'P', 'G'), + + // 5 Auxiliary YUV variations: 3 with U and V planes are swapped, 1 Alias. + FOURCC_YV12 = FOURCC('Y', 'V', '1', '2'), + FOURCC_YV16 = FOURCC('Y', 'V', '1', '6'), + FOURCC_YV24 = FOURCC('Y', 'V', '2', '4'), + FOURCC_YU12 = FOURCC('Y', 'U', '1', '2'), // Linux version of I420. + FOURCC_J420 = FOURCC('J', '4', '2', '0'), + FOURCC_J400 = FOURCC('J', '4', '0', '0'), + + // 14 Auxiliary aliases. CanonicalFourCC() maps these to canonical fourcc. + FOURCC_IYUV = FOURCC('I', 'Y', 'U', 'V'), // Alias for I420. + FOURCC_YU16 = FOURCC('Y', 'U', '1', '6'), // Alias for I422. + FOURCC_YU24 = FOURCC('Y', 'U', '2', '4'), // Alias for I444. + FOURCC_YUYV = FOURCC('Y', 'U', 'Y', 'V'), // Alias for YUY2. + FOURCC_YUVS = FOURCC('y', 'u', 'v', 's'), // Alias for YUY2 on Mac. + FOURCC_HDYC = FOURCC('H', 'D', 'Y', 'C'), // Alias for UYVY. + FOURCC_2VUY = FOURCC('2', 'v', 'u', 'y'), // Alias for UYVY on Mac. + FOURCC_JPEG = FOURCC('J', 'P', 'E', 'G'), // Alias for MJPG. + FOURCC_DMB1 = FOURCC('d', 'm', 'b', '1'), // Alias for MJPG on Mac. + FOURCC_BA81 = FOURCC('B', 'A', '8', '1'), // Alias for BGGR. + FOURCC_RGB3 = FOURCC('R', 'G', 'B', '3'), // Alias for RAW. + FOURCC_BGR3 = FOURCC('B', 'G', 'R', '3'), // Alias for 24BG. + FOURCC_CM32 = FOURCC(0, 0, 0, 32), // Alias for BGRA kCMPixelFormat_32ARGB + FOURCC_CM24 = FOURCC(0, 0, 0, 24), // Alias for RAW kCMPixelFormat_24RGB + FOURCC_L555 = FOURCC('L', '5', '5', '5'), // Alias for RGBO. + FOURCC_L565 = FOURCC('L', '5', '6', '5'), // Alias for RGBP. + FOURCC_5551 = FOURCC('5', '5', '5', '1'), // Alias for RGBO. + + // 1 Auxiliary compressed YUV format set aside for capturer. + FOURCC_H264 = FOURCC('H', '2', '6', '4'), + + // Match any fourcc. + FOURCC_ANY = -1, +}; + +enum FourCCBpp { + // Canonical fourcc codes used in our code. + FOURCC_BPP_I420 = 12, + FOURCC_BPP_I422 = 16, + FOURCC_BPP_I444 = 24, + FOURCC_BPP_I411 = 12, + FOURCC_BPP_I400 = 8, + FOURCC_BPP_NV21 = 12, + FOURCC_BPP_NV12 = 12, + FOURCC_BPP_YUY2 = 16, + FOURCC_BPP_UYVY = 16, + FOURCC_BPP_M420 = 12, + FOURCC_BPP_Q420 = 12, + FOURCC_BPP_ARGB = 32, + FOURCC_BPP_BGRA = 32, + FOURCC_BPP_ABGR = 32, + FOURCC_BPP_RGBA = 32, + FOURCC_BPP_24BG = 24, + FOURCC_BPP_RAW = 24, + FOURCC_BPP_RGBP = 16, + FOURCC_BPP_RGBO = 16, + FOURCC_BPP_R444 = 16, + FOURCC_BPP_RGGB = 8, + FOURCC_BPP_BGGR = 8, + FOURCC_BPP_GRBG = 8, + FOURCC_BPP_GBRG = 8, + FOURCC_BPP_YV12 = 12, + FOURCC_BPP_YV16 = 16, + FOURCC_BPP_YV24 = 24, + FOURCC_BPP_YU12 = 12, + FOURCC_BPP_J420 = 12, + FOURCC_BPP_J400 = 8, + FOURCC_BPP_MJPG = 0, // 0 means unknown. + FOURCC_BPP_H264 = 0, + FOURCC_BPP_IYUV = 12, + FOURCC_BPP_YU16 = 16, + FOURCC_BPP_YU24 = 24, + FOURCC_BPP_YUYV = 16, + FOURCC_BPP_YUVS = 16, + FOURCC_BPP_HDYC = 16, + FOURCC_BPP_2VUY = 16, + FOURCC_BPP_JPEG = 1, + FOURCC_BPP_DMB1 = 1, + FOURCC_BPP_BA81 = 8, + FOURCC_BPP_RGB3 = 24, + FOURCC_BPP_BGR3 = 24, + FOURCC_BPP_CM32 = 32, + FOURCC_BPP_CM24 = 24, + + // Match any fourcc. + FOURCC_BPP_ANY = 0, // 0 means unknown. +}; + +// Converts fourcc aliases into canonical ones. +LIBYUV_API uint32 CanonicalFourCC(uint32 fourcc); + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // INCLUDE_LIBYUV_VIDEO_COMMON_H_ NOLINT diff --git a/third_party/aom/third_party/libyuv/source/compare.cc b/third_party/aom/third_party/libyuv/source/compare.cc new file mode 100644 index 0000000000..46aa8473d2 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/compare.cc @@ -0,0 +1,373 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/compare.h" + +#include +#include +#ifdef _OPENMP +#include +#endif + +#include "libyuv/basic_types.h" +#include "libyuv/cpu_id.h" +#include "libyuv/row.h" +#include "libyuv/video_common.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// hash seed of 5381 recommended. +// Internal C version of HashDjb2 with int sized count for efficiency. +uint32 HashDjb2_C(const uint8* src, int count, uint32 seed); + +// This module is for Visual C x86 +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || \ + (defined(__x86_64__) || (defined(__i386__) && !defined(__pic__)))) +#define HAS_HASHDJB2_SSE41 +uint32 HashDjb2_SSE41(const uint8* src, int count, uint32 seed); + +#ifdef VISUALC_HAS_AVX2 +#define HAS_HASHDJB2_AVX2 +uint32 HashDjb2_AVX2(const uint8* src, int count, uint32 seed); +#endif + +#endif // HAS_HASHDJB2_SSE41 + +// hash seed of 5381 recommended. +LIBYUV_API +uint32 HashDjb2(const uint8* src, uint64 count, uint32 seed) { + const int kBlockSize = 1 << 15; // 32768; + int remainder; + uint32 (*HashDjb2_SSE)(const uint8* src, int count, uint32 seed) = HashDjb2_C; +#if defined(HAS_HASHDJB2_SSE41) + if (TestCpuFlag(kCpuHasSSE41)) { + HashDjb2_SSE = HashDjb2_SSE41; + } +#endif +#if defined(HAS_HASHDJB2_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + HashDjb2_SSE = HashDjb2_AVX2; + } +#endif + + while (count >= (uint64)(kBlockSize)) { + seed = HashDjb2_SSE(src, kBlockSize, seed); + src += kBlockSize; + count -= kBlockSize; + } + remainder = (int)(count) & ~15; + if (remainder) { + seed = HashDjb2_SSE(src, remainder, seed); + src += remainder; + count -= remainder; + } + remainder = (int)(count) & 15; + if (remainder) { + seed = HashDjb2_C(src, remainder, seed); + } + return seed; +} + +static uint32 ARGBDetectRow_C(const uint8* argb, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB. + return FOURCC_BGRA; + } + if (argb[3] != 255) { // 4th byte is not Alpha of 255, so not BGRA. + return FOURCC_ARGB; + } + if (argb[4] != 255) { // Second pixel first byte is not Alpha of 255. + return FOURCC_BGRA; + } + if (argb[7] != 255) { // Second pixel 4th byte is not Alpha of 255. + return FOURCC_ARGB; + } + argb += 8; + } + if (width & 1) { + if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB. + return FOURCC_BGRA; + } + if (argb[3] != 255) { // 4th byte is not Alpha of 255, so not BGRA. + return FOURCC_ARGB; + } + } + return 0; +} + +// Scan an opaque argb image and return fourcc based on alpha offset. +// Returns FOURCC_ARGB, FOURCC_BGRA, or 0 if unknown. +LIBYUV_API +uint32 ARGBDetect(const uint8* argb, int stride_argb, int width, int height) { + uint32 fourcc = 0; + int h; + + // Coalesce rows. + if (stride_argb == width * 4) { + width *= height; + height = 1; + stride_argb = 0; + } + for (h = 0; h < height && fourcc == 0; ++h) { + fourcc = ARGBDetectRow_C(argb, width); + argb += stride_argb; + } + return fourcc; +} + +uint32 SumSquareError_C(const uint8* src_a, const uint8* src_b, int count); +#if !defined(LIBYUV_DISABLE_NEON) && \ + (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) +#define HAS_SUMSQUAREERROR_NEON +uint32 SumSquareError_NEON(const uint8* src_a, const uint8* src_b, int count); +#endif +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)) +#define HAS_SUMSQUAREERROR_SSE2 +uint32 SumSquareError_SSE2(const uint8* src_a, const uint8* src_b, int count); +#endif + +#ifdef VISUALC_HAS_AVX2 +#define HAS_SUMSQUAREERROR_AVX2 +uint32 SumSquareError_AVX2(const uint8* src_a, const uint8* src_b, int count); +#endif + +// TODO(fbarchard): Refactor into row function. +LIBYUV_API +uint64 ComputeSumSquareError(const uint8* src_a, const uint8* src_b, + int count) { + // SumSquareError returns values 0 to 65535 for each squared difference. + // Up to 65536 of those can be summed and remain within a uint32. + // After each block of 65536 pixels, accumulate into a uint64. + const int kBlockSize = 65536; + int remainder = count & (kBlockSize - 1) & ~31; + uint64 sse = 0; + int i; + uint32 (*SumSquareError)(const uint8* src_a, const uint8* src_b, int count) = + SumSquareError_C; +#if defined(HAS_SUMSQUAREERROR_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + SumSquareError = SumSquareError_NEON; + } +#endif +#if defined(HAS_SUMSQUAREERROR_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + // Note only used for multiples of 16 so count is not checked. + SumSquareError = SumSquareError_SSE2; + } +#endif +#if defined(HAS_SUMSQUAREERROR_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + // Note only used for multiples of 32 so count is not checked. + SumSquareError = SumSquareError_AVX2; + } +#endif +#ifdef _OPENMP +#pragma omp parallel for reduction(+: sse) +#endif + for (i = 0; i < (count - (kBlockSize - 1)); i += kBlockSize) { + sse += SumSquareError(src_a + i, src_b + i, kBlockSize); + } + src_a += count & ~(kBlockSize - 1); + src_b += count & ~(kBlockSize - 1); + if (remainder) { + sse += SumSquareError(src_a, src_b, remainder); + src_a += remainder; + src_b += remainder; + } + remainder = count & 31; + if (remainder) { + sse += SumSquareError_C(src_a, src_b, remainder); + } + return sse; +} + +LIBYUV_API +uint64 ComputeSumSquareErrorPlane(const uint8* src_a, int stride_a, + const uint8* src_b, int stride_b, + int width, int height) { + uint64 sse = 0; + int h; + // Coalesce rows. + if (stride_a == width && + stride_b == width) { + width *= height; + height = 1; + stride_a = stride_b = 0; + } + for (h = 0; h < height; ++h) { + sse += ComputeSumSquareError(src_a, src_b, width); + src_a += stride_a; + src_b += stride_b; + } + return sse; +} + +LIBYUV_API +double SumSquareErrorToPsnr(uint64 sse, uint64 count) { + double psnr; + if (sse > 0) { + double mse = (double)(count) / (double)(sse); + psnr = 10.0 * log10(255.0 * 255.0 * mse); + } else { + psnr = kMaxPsnr; // Limit to prevent divide by 0 + } + + if (psnr > kMaxPsnr) + psnr = kMaxPsnr; + + return psnr; +} + +LIBYUV_API +double CalcFramePsnr(const uint8* src_a, int stride_a, + const uint8* src_b, int stride_b, + int width, int height) { + const uint64 samples = width * height; + const uint64 sse = ComputeSumSquareErrorPlane(src_a, stride_a, + src_b, stride_b, + width, height); + return SumSquareErrorToPsnr(sse, samples); +} + +LIBYUV_API +double I420Psnr(const uint8* src_y_a, int stride_y_a, + const uint8* src_u_a, int stride_u_a, + const uint8* src_v_a, int stride_v_a, + const uint8* src_y_b, int stride_y_b, + const uint8* src_u_b, int stride_u_b, + const uint8* src_v_b, int stride_v_b, + int width, int height) { + const uint64 sse_y = ComputeSumSquareErrorPlane(src_y_a, stride_y_a, + src_y_b, stride_y_b, + width, height); + const int width_uv = (width + 1) >> 1; + const int height_uv = (height + 1) >> 1; + const uint64 sse_u = ComputeSumSquareErrorPlane(src_u_a, stride_u_a, + src_u_b, stride_u_b, + width_uv, height_uv); + const uint64 sse_v = ComputeSumSquareErrorPlane(src_v_a, stride_v_a, + src_v_b, stride_v_b, + width_uv, height_uv); + const uint64 samples = width * height + 2 * (width_uv * height_uv); + const uint64 sse = sse_y + sse_u + sse_v; + return SumSquareErrorToPsnr(sse, samples); +} + +static const int64 cc1 = 26634; // (64^2*(.01*255)^2 +static const int64 cc2 = 239708; // (64^2*(.03*255)^2 + +static double Ssim8x8_C(const uint8* src_a, int stride_a, + const uint8* src_b, int stride_b) { + int64 sum_a = 0; + int64 sum_b = 0; + int64 sum_sq_a = 0; + int64 sum_sq_b = 0; + int64 sum_axb = 0; + + int i; + for (i = 0; i < 8; ++i) { + int j; + for (j = 0; j < 8; ++j) { + sum_a += src_a[j]; + sum_b += src_b[j]; + sum_sq_a += src_a[j] * src_a[j]; + sum_sq_b += src_b[j] * src_b[j]; + sum_axb += src_a[j] * src_b[j]; + } + + src_a += stride_a; + src_b += stride_b; + } + + { + const int64 count = 64; + // scale the constants by number of pixels + const int64 c1 = (cc1 * count * count) >> 12; + const int64 c2 = (cc2 * count * count) >> 12; + + const int64 sum_a_x_sum_b = sum_a * sum_b; + + const int64 ssim_n = (2 * sum_a_x_sum_b + c1) * + (2 * count * sum_axb - 2 * sum_a_x_sum_b + c2); + + const int64 sum_a_sq = sum_a*sum_a; + const int64 sum_b_sq = sum_b*sum_b; + + const int64 ssim_d = (sum_a_sq + sum_b_sq + c1) * + (count * sum_sq_a - sum_a_sq + + count * sum_sq_b - sum_b_sq + c2); + + if (ssim_d == 0.0) { + return DBL_MAX; + } + return ssim_n * 1.0 / ssim_d; + } +} + +// We are using a 8x8 moving window with starting location of each 8x8 window +// on the 4x4 pixel grid. Such arrangement allows the windows to overlap +// block boundaries to penalize blocking artifacts. +LIBYUV_API +double CalcFrameSsim(const uint8* src_a, int stride_a, + const uint8* src_b, int stride_b, + int width, int height) { + int samples = 0; + double ssim_total = 0; + double (*Ssim8x8)(const uint8* src_a, int stride_a, + const uint8* src_b, int stride_b) = Ssim8x8_C; + + // sample point start with each 4x4 location + int i; + for (i = 0; i < height - 8; i += 4) { + int j; + for (j = 0; j < width - 8; j += 4) { + ssim_total += Ssim8x8(src_a + j, stride_a, src_b + j, stride_b); + samples++; + } + + src_a += stride_a * 4; + src_b += stride_b * 4; + } + + ssim_total /= samples; + return ssim_total; +} + +LIBYUV_API +double I420Ssim(const uint8* src_y_a, int stride_y_a, + const uint8* src_u_a, int stride_u_a, + const uint8* src_v_a, int stride_v_a, + const uint8* src_y_b, int stride_y_b, + const uint8* src_u_b, int stride_u_b, + const uint8* src_v_b, int stride_v_b, + int width, int height) { + const double ssim_y = CalcFrameSsim(src_y_a, stride_y_a, + src_y_b, stride_y_b, width, height); + const int width_uv = (width + 1) >> 1; + const int height_uv = (height + 1) >> 1; + const double ssim_u = CalcFrameSsim(src_u_a, stride_u_a, + src_u_b, stride_u_b, + width_uv, height_uv); + const double ssim_v = CalcFrameSsim(src_v_a, stride_v_a, + src_v_b, stride_v_b, + width_uv, height_uv); + return ssim_y * 0.8 + 0.1 * (ssim_u + ssim_v); +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/compare_common.cc b/third_party/aom/third_party/libyuv/source/compare_common.cc new file mode 100644 index 0000000000..c546b51829 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/compare_common.cc @@ -0,0 +1,42 @@ +/* + * Copyright 2012 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +uint32 SumSquareError_C(const uint8* src_a, const uint8* src_b, int count) { + uint32 sse = 0u; + int i; + for (i = 0; i < count; ++i) { + int diff = src_a[i] - src_b[i]; + sse += (uint32)(diff * diff); + } + return sse; +} + +// hash seed of 5381 recommended. +// Internal C version of HashDjb2 with int sized count for efficiency. +uint32 HashDjb2_C(const uint8* src, int count, uint32 seed) { + uint32 hash = seed; + int i; + for (i = 0; i < count; ++i) { + hash += (hash << 5) + src[i]; + } + return hash; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/compare_gcc.cc b/third_party/aom/third_party/libyuv/source/compare_gcc.cc new file mode 100644 index 0000000000..247cb33bba --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/compare_gcc.cc @@ -0,0 +1,152 @@ +/* + * Copyright 2012 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/basic_types.h" +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#if !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__)) + +uint32 SumSquareError_SSE2(const uint8* src_a, const uint8* src_b, int count) { + uint32 sse; + asm volatile ( // NOLINT + "pxor %%xmm0,%%xmm0 \n" + "pxor %%xmm5,%%xmm5 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm1 \n" + "lea " MEMLEA(0x10, 0) ",%0 \n" + "movdqu " MEMACCESS(1) ",%%xmm2 \n" + "lea " MEMLEA(0x10, 1) ",%1 \n" + "movdqa %%xmm1,%%xmm3 \n" + "psubusb %%xmm2,%%xmm1 \n" + "psubusb %%xmm3,%%xmm2 \n" + "por %%xmm2,%%xmm1 \n" + "movdqa %%xmm1,%%xmm2 \n" + "punpcklbw %%xmm5,%%xmm1 \n" + "punpckhbw %%xmm5,%%xmm2 \n" + "pmaddwd %%xmm1,%%xmm1 \n" + "pmaddwd %%xmm2,%%xmm2 \n" + "paddd %%xmm1,%%xmm0 \n" + "paddd %%xmm2,%%xmm0 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + + "pshufd $0xee,%%xmm0,%%xmm1 \n" + "paddd %%xmm1,%%xmm0 \n" + "pshufd $0x1,%%xmm0,%%xmm1 \n" + "paddd %%xmm1,%%xmm0 \n" + "movd %%xmm0,%3 \n" + + : "+r"(src_a), // %0 + "+r"(src_b), // %1 + "+r"(count), // %2 + "=g"(sse) // %3 + :: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); // NOLINT + return sse; +} + +#endif // defined(__x86_64__) || defined(__i386__) + +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || (defined(__i386__) && !defined(__pic__))) +#define HAS_HASHDJB2_SSE41 +static uvec32 kHash16x33 = { 0x92d9e201, 0, 0, 0 }; // 33 ^ 16 +static uvec32 kHashMul0 = { + 0x0c3525e1, // 33 ^ 15 + 0xa3476dc1, // 33 ^ 14 + 0x3b4039a1, // 33 ^ 13 + 0x4f5f0981, // 33 ^ 12 +}; +static uvec32 kHashMul1 = { + 0x30f35d61, // 33 ^ 11 + 0x855cb541, // 33 ^ 10 + 0x040a9121, // 33 ^ 9 + 0x747c7101, // 33 ^ 8 +}; +static uvec32 kHashMul2 = { + 0xec41d4e1, // 33 ^ 7 + 0x4cfa3cc1, // 33 ^ 6 + 0x025528a1, // 33 ^ 5 + 0x00121881, // 33 ^ 4 +}; +static uvec32 kHashMul3 = { + 0x00008c61, // 33 ^ 3 + 0x00000441, // 33 ^ 2 + 0x00000021, // 33 ^ 1 + 0x00000001, // 33 ^ 0 +}; + +uint32 HashDjb2_SSE41(const uint8* src, int count, uint32 seed) { + uint32 hash; + asm volatile ( // NOLINT + "movd %2,%%xmm0 \n" + "pxor %%xmm7,%%xmm7 \n" + "movdqa %4,%%xmm6 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm1 \n" + "lea " MEMLEA(0x10, 0) ",%0 \n" + "pmulld %%xmm6,%%xmm0 \n" + "movdqa %5,%%xmm5 \n" + "movdqa %%xmm1,%%xmm2 \n" + "punpcklbw %%xmm7,%%xmm2 \n" + "movdqa %%xmm2,%%xmm3 \n" + "punpcklwd %%xmm7,%%xmm3 \n" + "pmulld %%xmm5,%%xmm3 \n" + "movdqa %6,%%xmm5 \n" + "movdqa %%xmm2,%%xmm4 \n" + "punpckhwd %%xmm7,%%xmm4 \n" + "pmulld %%xmm5,%%xmm4 \n" + "movdqa %7,%%xmm5 \n" + "punpckhbw %%xmm7,%%xmm1 \n" + "movdqa %%xmm1,%%xmm2 \n" + "punpcklwd %%xmm7,%%xmm2 \n" + "pmulld %%xmm5,%%xmm2 \n" + "movdqa %8,%%xmm5 \n" + "punpckhwd %%xmm7,%%xmm1 \n" + "pmulld %%xmm5,%%xmm1 \n" + "paddd %%xmm4,%%xmm3 \n" + "paddd %%xmm2,%%xmm1 \n" + "paddd %%xmm3,%%xmm1 \n" + "pshufd $0xe,%%xmm1,%%xmm2 \n" + "paddd %%xmm2,%%xmm1 \n" + "pshufd $0x1,%%xmm1,%%xmm2 \n" + "paddd %%xmm2,%%xmm1 \n" + "paddd %%xmm1,%%xmm0 \n" + "sub $0x10,%1 \n" + "jg 1b \n" + "movd %%xmm0,%3 \n" + : "+r"(src), // %0 + "+r"(count), // %1 + "+rm"(seed), // %2 + "=g"(hash) // %3 + : "m"(kHash16x33), // %4 + "m"(kHashMul0), // %5 + "m"(kHashMul1), // %6 + "m"(kHashMul2), // %7 + "m"(kHashMul3) // %8 + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); // NOLINT + return hash; +} +#endif // defined(__x86_64__) || (defined(__i386__) && !defined(__pic__))) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + diff --git a/third_party/aom/third_party/libyuv/source/compare_neon.cc b/third_party/aom/third_party/libyuv/source/compare_neon.cc new file mode 100644 index 0000000000..ef006ec41c --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/compare_neon.cc @@ -0,0 +1,65 @@ +/* + * Copyright 2012 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/basic_types.h" +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \ + !defined(__aarch64__) + +uint32 SumSquareError_NEON(const uint8* src_a, const uint8* src_b, int count) { + volatile uint32 sse; + asm volatile ( + "vmov.u8 q8, #0 \n" + "vmov.u8 q10, #0 \n" + "vmov.u8 q9, #0 \n" + "vmov.u8 q11, #0 \n" + + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" + MEMACCESS(1) + "vld1.8 {q1}, [%1]! \n" + "subs %2, %2, #16 \n" + "vsubl.u8 q2, d0, d2 \n" + "vsubl.u8 q3, d1, d3 \n" + "vmlal.s16 q8, d4, d4 \n" + "vmlal.s16 q9, d6, d6 \n" + "vmlal.s16 q10, d5, d5 \n" + "vmlal.s16 q11, d7, d7 \n" + "bgt 1b \n" + + "vadd.u32 q8, q8, q9 \n" + "vadd.u32 q10, q10, q11 \n" + "vadd.u32 q11, q8, q10 \n" + "vpaddl.u32 q1, q11 \n" + "vadd.u64 d0, d2, d3 \n" + "vmov.32 %3, d0[0] \n" + : "+r"(src_a), + "+r"(src_b), + "+r"(count), + "=r"(sse) + : + : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11"); + return sse; +} + +#endif // defined(__ARM_NEON__) && !defined(__aarch64__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/compare_neon64.cc b/third_party/aom/third_party/libyuv/source/compare_neon64.cc new file mode 100644 index 0000000000..6d1e5e1bc9 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/compare_neon64.cc @@ -0,0 +1,63 @@ +/* + * Copyright 2012 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/basic_types.h" +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) + +uint32 SumSquareError_NEON(const uint8* src_a, const uint8* src_b, int count) { + volatile uint32 sse; + asm volatile ( + "eor v16.16b, v16.16b, v16.16b \n" + "eor v18.16b, v18.16b, v18.16b \n" + "eor v17.16b, v17.16b, v17.16b \n" + "eor v19.16b, v19.16b, v19.16b \n" + + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" + MEMACCESS(1) + "ld1 {v1.16b}, [%1], #16 \n" + "subs %w2, %w2, #16 \n" + "usubl v2.8h, v0.8b, v1.8b \n" + "usubl2 v3.8h, v0.16b, v1.16b \n" + "smlal v16.4s, v2.4h, v2.4h \n" + "smlal v17.4s, v3.4h, v3.4h \n" + "smlal2 v18.4s, v2.8h, v2.8h \n" + "smlal2 v19.4s, v3.8h, v3.8h \n" + "b.gt 1b \n" + + "add v16.4s, v16.4s, v17.4s \n" + "add v18.4s, v18.4s, v19.4s \n" + "add v19.4s, v16.4s, v18.4s \n" + "addv s0, v19.4s \n" + "fmov %w3, s0 \n" + : "+r"(src_a), + "+r"(src_b), + "+r"(count), + "=r"(sse) + : + : "cc", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19"); + return sse; +} + +#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/compare_win.cc b/third_party/aom/third_party/libyuv/source/compare_win.cc new file mode 100644 index 0000000000..19806f2750 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/compare_win.cc @@ -0,0 +1,229 @@ +/* + * Copyright 2012 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/basic_types.h" +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for Visual C x86. +#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && \ + defined(_MSC_VER) && !defined(__clang__) + +__declspec(naked) +uint32 SumSquareError_SSE2(const uint8* src_a, const uint8* src_b, int count) { + __asm { + mov eax, [esp + 4] // src_a + mov edx, [esp + 8] // src_b + mov ecx, [esp + 12] // count + pxor xmm0, xmm0 + pxor xmm5, xmm5 + + wloop: + movdqu xmm1, [eax] + lea eax, [eax + 16] + movdqu xmm2, [edx] + lea edx, [edx + 16] + movdqa xmm3, xmm1 // abs trick + psubusb xmm1, xmm2 + psubusb xmm2, xmm3 + por xmm1, xmm2 + movdqa xmm2, xmm1 + punpcklbw xmm1, xmm5 + punpckhbw xmm2, xmm5 + pmaddwd xmm1, xmm1 + pmaddwd xmm2, xmm2 + paddd xmm0, xmm1 + paddd xmm0, xmm2 + sub ecx, 16 + jg wloop + + pshufd xmm1, xmm0, 0xee + paddd xmm0, xmm1 + pshufd xmm1, xmm0, 0x01 + paddd xmm0, xmm1 + movd eax, xmm0 + ret + } +} + +// Visual C 2012 required for AVX2. +#if _MSC_VER >= 1700 +// C4752: found Intel(R) Advanced Vector Extensions; consider using /arch:AVX. +#pragma warning(disable: 4752) +__declspec(naked) +uint32 SumSquareError_AVX2(const uint8* src_a, const uint8* src_b, int count) { + __asm { + mov eax, [esp + 4] // src_a + mov edx, [esp + 8] // src_b + mov ecx, [esp + 12] // count + vpxor ymm0, ymm0, ymm0 // sum + vpxor ymm5, ymm5, ymm5 // constant 0 for unpck + sub edx, eax + + wloop: + vmovdqu ymm1, [eax] + vmovdqu ymm2, [eax + edx] + lea eax, [eax + 32] + vpsubusb ymm3, ymm1, ymm2 // abs difference trick + vpsubusb ymm2, ymm2, ymm1 + vpor ymm1, ymm2, ymm3 + vpunpcklbw ymm2, ymm1, ymm5 // u16. mutates order. + vpunpckhbw ymm1, ymm1, ymm5 + vpmaddwd ymm2, ymm2, ymm2 // square + hadd to u32. + vpmaddwd ymm1, ymm1, ymm1 + vpaddd ymm0, ymm0, ymm1 + vpaddd ymm0, ymm0, ymm2 + sub ecx, 32 + jg wloop + + vpshufd ymm1, ymm0, 0xee // 3, 2 + 1, 0 both lanes. + vpaddd ymm0, ymm0, ymm1 + vpshufd ymm1, ymm0, 0x01 // 1 + 0 both lanes. + vpaddd ymm0, ymm0, ymm1 + vpermq ymm1, ymm0, 0x02 // high + low lane. + vpaddd ymm0, ymm0, ymm1 + vmovd eax, xmm0 + vzeroupper + ret + } +} +#endif // _MSC_VER >= 1700 + +#define HAS_HASHDJB2_SSE41 +static uvec32 kHash16x33 = { 0x92d9e201, 0, 0, 0 }; // 33 ^ 16 +static uvec32 kHashMul0 = { + 0x0c3525e1, // 33 ^ 15 + 0xa3476dc1, // 33 ^ 14 + 0x3b4039a1, // 33 ^ 13 + 0x4f5f0981, // 33 ^ 12 +}; +static uvec32 kHashMul1 = { + 0x30f35d61, // 33 ^ 11 + 0x855cb541, // 33 ^ 10 + 0x040a9121, // 33 ^ 9 + 0x747c7101, // 33 ^ 8 +}; +static uvec32 kHashMul2 = { + 0xec41d4e1, // 33 ^ 7 + 0x4cfa3cc1, // 33 ^ 6 + 0x025528a1, // 33 ^ 5 + 0x00121881, // 33 ^ 4 +}; +static uvec32 kHashMul3 = { + 0x00008c61, // 33 ^ 3 + 0x00000441, // 33 ^ 2 + 0x00000021, // 33 ^ 1 + 0x00000001, // 33 ^ 0 +}; + +// 27: 66 0F 38 40 C6 pmulld xmm0,xmm6 +// 44: 66 0F 38 40 DD pmulld xmm3,xmm5 +// 59: 66 0F 38 40 E5 pmulld xmm4,xmm5 +// 72: 66 0F 38 40 D5 pmulld xmm2,xmm5 +// 83: 66 0F 38 40 CD pmulld xmm1,xmm5 +#define pmulld(reg) _asm _emit 0x66 _asm _emit 0x0F _asm _emit 0x38 \ + _asm _emit 0x40 _asm _emit reg + +__declspec(naked) +uint32 HashDjb2_SSE41(const uint8* src, int count, uint32 seed) { + __asm { + mov eax, [esp + 4] // src + mov ecx, [esp + 8] // count + movd xmm0, [esp + 12] // seed + + pxor xmm7, xmm7 // constant 0 for unpck + movdqa xmm6, kHash16x33 + + wloop: + movdqu xmm1, [eax] // src[0-15] + lea eax, [eax + 16] + pmulld(0xc6) // pmulld xmm0,xmm6 hash *= 33 ^ 16 + movdqa xmm5, kHashMul0 + movdqa xmm2, xmm1 + punpcklbw xmm2, xmm7 // src[0-7] + movdqa xmm3, xmm2 + punpcklwd xmm3, xmm7 // src[0-3] + pmulld(0xdd) // pmulld xmm3, xmm5 + movdqa xmm5, kHashMul1 + movdqa xmm4, xmm2 + punpckhwd xmm4, xmm7 // src[4-7] + pmulld(0xe5) // pmulld xmm4, xmm5 + movdqa xmm5, kHashMul2 + punpckhbw xmm1, xmm7 // src[8-15] + movdqa xmm2, xmm1 + punpcklwd xmm2, xmm7 // src[8-11] + pmulld(0xd5) // pmulld xmm2, xmm5 + movdqa xmm5, kHashMul3 + punpckhwd xmm1, xmm7 // src[12-15] + pmulld(0xcd) // pmulld xmm1, xmm5 + paddd xmm3, xmm4 // add 16 results + paddd xmm1, xmm2 + paddd xmm1, xmm3 + + pshufd xmm2, xmm1, 0x0e // upper 2 dwords + paddd xmm1, xmm2 + pshufd xmm2, xmm1, 0x01 + paddd xmm1, xmm2 + paddd xmm0, xmm1 + sub ecx, 16 + jg wloop + + movd eax, xmm0 // return hash + ret + } +} + +// Visual C 2012 required for AVX2. +#if _MSC_VER >= 1700 +__declspec(naked) +uint32 HashDjb2_AVX2(const uint8* src, int count, uint32 seed) { + __asm { + mov eax, [esp + 4] // src + mov ecx, [esp + 8] // count + movd xmm0, [esp + 12] // seed + movdqa xmm6, kHash16x33 + + wloop: + vpmovzxbd xmm3, dword ptr [eax] // src[0-3] + pmulld xmm0, xmm6 // hash *= 33 ^ 16 + vpmovzxbd xmm4, dword ptr [eax + 4] // src[4-7] + pmulld xmm3, kHashMul0 + vpmovzxbd xmm2, dword ptr [eax + 8] // src[8-11] + pmulld xmm4, kHashMul1 + vpmovzxbd xmm1, dword ptr [eax + 12] // src[12-15] + pmulld xmm2, kHashMul2 + lea eax, [eax + 16] + pmulld xmm1, kHashMul3 + paddd xmm3, xmm4 // add 16 results + paddd xmm1, xmm2 + paddd xmm1, xmm3 + pshufd xmm2, xmm1, 0x0e // upper 2 dwords + paddd xmm1, xmm2 + pshufd xmm2, xmm1, 0x01 + paddd xmm1, xmm2 + paddd xmm0, xmm1 + sub ecx, 16 + jg wloop + + movd eax, xmm0 // return hash + ret + } +} +#endif // _MSC_VER >= 1700 +#endif // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/convert.cc b/third_party/aom/third_party/libyuv/source/convert.cc new file mode 100644 index 0000000000..3ad6bd7a4b --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/convert.cc @@ -0,0 +1,1389 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/convert.h" + +#include "libyuv/basic_types.h" +#include "libyuv/cpu_id.h" +#include "libyuv/planar_functions.h" +#include "libyuv/rotate.h" +#include "libyuv/scale.h" // For ScalePlane() +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#define SUBSAMPLE(v, a, s) (v < 0) ? (-((-v + a) >> s)) : ((v + a) >> s) +static __inline int Abs(int v) { + return v >= 0 ? v : -v; +} + +// Any I4xx To I420 format with mirroring. +static int I4xxToI420(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int src_y_width, int src_y_height, + int src_uv_width, int src_uv_height) { + const int dst_y_width = Abs(src_y_width); + const int dst_y_height = Abs(src_y_height); + const int dst_uv_width = SUBSAMPLE(dst_y_width, 1, 1); + const int dst_uv_height = SUBSAMPLE(dst_y_height, 1, 1); + if (src_y_width == 0 || src_y_height == 0 || + src_uv_width == 0 || src_uv_height == 0) { + return -1; + } + ScalePlane(src_y, src_stride_y, src_y_width, src_y_height, + dst_y, dst_stride_y, dst_y_width, dst_y_height, + kFilterBilinear); + ScalePlane(src_u, src_stride_u, src_uv_width, src_uv_height, + dst_u, dst_stride_u, dst_uv_width, dst_uv_height, + kFilterBilinear); + ScalePlane(src_v, src_stride_v, src_uv_width, src_uv_height, + dst_v, dst_stride_v, dst_uv_width, dst_uv_height, + kFilterBilinear); + return 0; +} + +// Copy I420 with optional flipping +// TODO(fbarchard): Use Scale plane which supports mirroring, but ensure +// is does row coalescing. +LIBYUV_API +int I420Copy(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int halfwidth = (width + 1) >> 1; + int halfheight = (height + 1) >> 1; + if (!src_y || !src_u || !src_v || + !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + halfheight = (height + 1) >> 1; + src_y = src_y + (height - 1) * src_stride_y; + src_u = src_u + (halfheight - 1) * src_stride_u; + src_v = src_v + (halfheight - 1) * src_stride_v; + src_stride_y = -src_stride_y; + src_stride_u = -src_stride_u; + src_stride_v = -src_stride_v; + } + + if (dst_y) { + CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height); + } + // Copy UV planes. + CopyPlane(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth, halfheight); + CopyPlane(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth, halfheight); + return 0; +} + +// 422 chroma is 1/2 width, 1x height +// 420 chroma is 1/2 width, 1/2 height +LIBYUV_API +int I422ToI420(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + const int src_uv_width = SUBSAMPLE(width, 1, 1); + return I4xxToI420(src_y, src_stride_y, + src_u, src_stride_u, + src_v, src_stride_v, + dst_y, dst_stride_y, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + width, height, + src_uv_width, height); +} + +// 444 chroma is 1x width, 1x height +// 420 chroma is 1/2 width, 1/2 height +LIBYUV_API +int I444ToI420(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + return I4xxToI420(src_y, src_stride_y, + src_u, src_stride_u, + src_v, src_stride_v, + dst_y, dst_stride_y, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + width, height, + width, height); +} + +// 411 chroma is 1/4 width, 1x height +// 420 chroma is 1/2 width, 1/2 height +LIBYUV_API +int I411ToI420(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + const int src_uv_width = SUBSAMPLE(width, 3, 2); + return I4xxToI420(src_y, src_stride_y, + src_u, src_stride_u, + src_v, src_stride_v, + dst_y, dst_stride_y, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + width, height, + src_uv_width, height); +} + +// I400 is greyscale typically used in MJPG +LIBYUV_API +int I400ToI420(const uint8* src_y, int src_stride_y, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int halfwidth = (width + 1) >> 1; + int halfheight = (height + 1) >> 1; + if (!src_y || !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + halfheight = (height + 1) >> 1; + src_y = src_y + (height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } + CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height); + SetPlane(dst_u, dst_stride_u, halfwidth, halfheight, 128); + SetPlane(dst_v, dst_stride_v, halfwidth, halfheight, 128); + return 0; +} + +static void CopyPlane2(const uint8* src, int src_stride_0, int src_stride_1, + uint8* dst, int dst_stride, + int width, int height) { + int y; + void (*CopyRow)(const uint8* src, uint8* dst, int width) = CopyRow_C; +#if defined(HAS_COPYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2; + } +#endif +#if defined(HAS_COPYROW_AVX) + if (TestCpuFlag(kCpuHasAVX)) { + CopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX; + } +#endif +#if defined(HAS_COPYROW_ERMS) + if (TestCpuFlag(kCpuHasERMS)) { + CopyRow = CopyRow_ERMS; + } +#endif +#if defined(HAS_COPYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON; + } +#endif +#if defined(HAS_COPYROW_MIPS) + if (TestCpuFlag(kCpuHasMIPS)) { + CopyRow = CopyRow_MIPS; + } +#endif + + // Copy plane + for (y = 0; y < height - 1; y += 2) { + CopyRow(src, dst, width); + CopyRow(src + src_stride_0, dst + dst_stride, width); + src += src_stride_0 + src_stride_1; + dst += dst_stride * 2; + } + if (height & 1) { + CopyRow(src, dst, width); + } +} + +// Support converting from FOURCC_M420 +// Useful for bandwidth constrained transports like USB 1.0 and 2.0 and for +// easy conversion to I420. +// M420 format description: +// M420 is row biplanar 420: 2 rows of Y and 1 row of UV. +// Chroma is half width / half height. (420) +// src_stride_m420 is row planar. Normally this will be the width in pixels. +// The UV plane is half width, but 2 values, so src_stride_m420 applies to +// this as well as the two Y planes. +static int X420ToI420(const uint8* src_y, + int src_stride_y0, int src_stride_y1, + const uint8* src_uv, int src_stride_uv, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + int halfwidth = (width + 1) >> 1; + int halfheight = (height + 1) >> 1; + void (*SplitUVRow)(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix) = + SplitUVRow_C; + if (!src_y || !src_uv || + !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + halfheight = (height + 1) >> 1; + dst_y = dst_y + (height - 1) * dst_stride_y; + dst_u = dst_u + (halfheight - 1) * dst_stride_u; + dst_v = dst_v + (halfheight - 1) * dst_stride_v; + dst_stride_y = -dst_stride_y; + dst_stride_u = -dst_stride_u; + dst_stride_v = -dst_stride_v; + } + // Coalesce rows. + if (src_stride_y0 == width && + src_stride_y1 == width && + dst_stride_y == width) { + width *= height; + height = 1; + src_stride_y0 = src_stride_y1 = dst_stride_y = 0; + } + // Coalesce rows. + if (src_stride_uv == halfwidth * 2 && + dst_stride_u == halfwidth && + dst_stride_v == halfwidth) { + halfwidth *= halfheight; + halfheight = 1; + src_stride_uv = dst_stride_u = dst_stride_v = 0; + } +#if defined(HAS_SPLITUVROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + SplitUVRow = SplitUVRow_Any_SSE2; + if (IS_ALIGNED(halfwidth, 16)) { + SplitUVRow = SplitUVRow_SSE2; + } + } +#endif +#if defined(HAS_SPLITUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SplitUVRow = SplitUVRow_Any_AVX2; + if (IS_ALIGNED(halfwidth, 32)) { + SplitUVRow = SplitUVRow_AVX2; + } + } +#endif +#if defined(HAS_SPLITUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + SplitUVRow = SplitUVRow_Any_NEON; + if (IS_ALIGNED(halfwidth, 16)) { + SplitUVRow = SplitUVRow_NEON; + } + } +#endif +#if defined(HAS_SPLITUVROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && + IS_ALIGNED(src_uv, 4) && IS_ALIGNED(src_stride_uv, 4) && + IS_ALIGNED(dst_u, 4) && IS_ALIGNED(dst_stride_u, 4) && + IS_ALIGNED(dst_v, 4) && IS_ALIGNED(dst_stride_v, 4)) { + SplitUVRow = SplitUVRow_Any_MIPS_DSPR2; + if (IS_ALIGNED(halfwidth, 16)) { + SplitUVRow = SplitUVRow_MIPS_DSPR2; + } + } +#endif + + if (dst_y) { + if (src_stride_y0 == src_stride_y1) { + CopyPlane(src_y, src_stride_y0, dst_y, dst_stride_y, width, height); + } else { + CopyPlane2(src_y, src_stride_y0, src_stride_y1, dst_y, dst_stride_y, + width, height); + } + } + + for (y = 0; y < halfheight; ++y) { + // Copy a row of UV. + SplitUVRow(src_uv, dst_u, dst_v, halfwidth); + dst_u += dst_stride_u; + dst_v += dst_stride_v; + src_uv += src_stride_uv; + } + return 0; +} + +// Convert NV12 to I420. +LIBYUV_API +int NV12ToI420(const uint8* src_y, int src_stride_y, + const uint8* src_uv, int src_stride_uv, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + return X420ToI420(src_y, src_stride_y, src_stride_y, + src_uv, src_stride_uv, + dst_y, dst_stride_y, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + width, height); +} + +// Convert NV21 to I420. Same as NV12 but u and v pointers swapped. +LIBYUV_API +int NV21ToI420(const uint8* src_y, int src_stride_y, + const uint8* src_vu, int src_stride_vu, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + return X420ToI420(src_y, src_stride_y, src_stride_y, + src_vu, src_stride_vu, + dst_y, dst_stride_y, + dst_v, dst_stride_v, + dst_u, dst_stride_u, + width, height); +} + +// Convert M420 to I420. +LIBYUV_API +int M420ToI420(const uint8* src_m420, int src_stride_m420, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + return X420ToI420(src_m420, src_stride_m420, src_stride_m420 * 2, + src_m420 + src_stride_m420 * 2, src_stride_m420 * 3, + dst_y, dst_stride_y, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + width, height); +} + +// Convert YUY2 to I420. +LIBYUV_API +int YUY2ToI420(const uint8* src_yuy2, int src_stride_yuy2, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*YUY2ToUVRow)(const uint8* src_yuy2, int src_stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix) = YUY2ToUVRow_C; + void (*YUY2ToYRow)(const uint8* src_yuy2, + uint8* dst_y, int pix) = YUY2ToYRow_C; + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_yuy2 = src_yuy2 + (height - 1) * src_stride_yuy2; + src_stride_yuy2 = -src_stride_yuy2; + } +#if defined(HAS_YUY2TOYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + YUY2ToUVRow = YUY2ToUVRow_Any_SSE2; + YUY2ToYRow = YUY2ToYRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + YUY2ToUVRow = YUY2ToUVRow_SSE2; + YUY2ToYRow = YUY2ToYRow_SSE2; + } + } +#endif +#if defined(HAS_YUY2TOYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + YUY2ToUVRow = YUY2ToUVRow_Any_AVX2; + YUY2ToYRow = YUY2ToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + YUY2ToUVRow = YUY2ToUVRow_AVX2; + YUY2ToYRow = YUY2ToYRow_AVX2; + } + } +#endif +#if defined(HAS_YUY2TOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + YUY2ToYRow = YUY2ToYRow_Any_NEON; + YUY2ToUVRow = YUY2ToUVRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + YUY2ToYRow = YUY2ToYRow_NEON; + YUY2ToUVRow = YUY2ToUVRow_NEON; + } + } +#endif + + for (y = 0; y < height - 1; y += 2) { + YUY2ToUVRow(src_yuy2, src_stride_yuy2, dst_u, dst_v, width); + YUY2ToYRow(src_yuy2, dst_y, width); + YUY2ToYRow(src_yuy2 + src_stride_yuy2, dst_y + dst_stride_y, width); + src_yuy2 += src_stride_yuy2 * 2; + dst_y += dst_stride_y * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { + YUY2ToUVRow(src_yuy2, 0, dst_u, dst_v, width); + YUY2ToYRow(src_yuy2, dst_y, width); + } + return 0; +} + +// Convert UYVY to I420. +LIBYUV_API +int UYVYToI420(const uint8* src_uyvy, int src_stride_uyvy, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*UYVYToUVRow)(const uint8* src_uyvy, int src_stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix) = UYVYToUVRow_C; + void (*UYVYToYRow)(const uint8* src_uyvy, + uint8* dst_y, int pix) = UYVYToYRow_C; + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_uyvy = src_uyvy + (height - 1) * src_stride_uyvy; + src_stride_uyvy = -src_stride_uyvy; + } +#if defined(HAS_UYVYTOYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + UYVYToUVRow = UYVYToUVRow_Any_SSE2; + UYVYToYRow = UYVYToYRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + UYVYToUVRow = UYVYToUVRow_SSE2; + UYVYToYRow = UYVYToYRow_SSE2; + } + } +#endif +#if defined(HAS_UYVYTOYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + UYVYToUVRow = UYVYToUVRow_Any_AVX2; + UYVYToYRow = UYVYToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + UYVYToUVRow = UYVYToUVRow_AVX2; + UYVYToYRow = UYVYToYRow_AVX2; + } + } +#endif +#if defined(HAS_UYVYTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + UYVYToYRow = UYVYToYRow_Any_NEON; + UYVYToUVRow = UYVYToUVRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + UYVYToYRow = UYVYToYRow_NEON; + UYVYToUVRow = UYVYToUVRow_NEON; + } + } +#endif + + for (y = 0; y < height - 1; y += 2) { + UYVYToUVRow(src_uyvy, src_stride_uyvy, dst_u, dst_v, width); + UYVYToYRow(src_uyvy, dst_y, width); + UYVYToYRow(src_uyvy + src_stride_uyvy, dst_y + dst_stride_y, width); + src_uyvy += src_stride_uyvy * 2; + dst_y += dst_stride_y * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { + UYVYToUVRow(src_uyvy, 0, dst_u, dst_v, width); + UYVYToYRow(src_uyvy, dst_y, width); + } + return 0; +} + +// Convert ARGB to I420. +LIBYUV_API +int ARGBToI420(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*ARGBToUVRow)(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) = ARGBToUVRow_C; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; + if (!src_argb || + !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } +#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVRow = ARGBToUVRow_Any_SSSE3; + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUVRow = ARGBToUVRow_SSSE3; + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToUVRow = ARGBToUVRow_Any_AVX2; + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToUVRow = ARGBToUVRow_AVX2; + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYRow = ARGBToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYRow = ARGBToYRow_NEON; + } + } +#endif +#if defined(HAS_ARGBTOUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToUVRow = ARGBToUVRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + ARGBToUVRow = ARGBToUVRow_NEON; + } + } +#endif + + for (y = 0; y < height - 1; y += 2) { + ARGBToUVRow(src_argb, src_stride_argb, dst_u, dst_v, width); + ARGBToYRow(src_argb, dst_y, width); + ARGBToYRow(src_argb + src_stride_argb, dst_y + dst_stride_y, width); + src_argb += src_stride_argb * 2; + dst_y += dst_stride_y * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { + ARGBToUVRow(src_argb, 0, dst_u, dst_v, width); + ARGBToYRow(src_argb, dst_y, width); + } + return 0; +} + +// Convert BGRA to I420. +LIBYUV_API +int BGRAToI420(const uint8* src_bgra, int src_stride_bgra, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*BGRAToUVRow)(const uint8* src_bgra0, int src_stride_bgra, + uint8* dst_u, uint8* dst_v, int width) = BGRAToUVRow_C; + void (*BGRAToYRow)(const uint8* src_bgra, uint8* dst_y, int pix) = + BGRAToYRow_C; + if (!src_bgra || + !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_bgra = src_bgra + (height - 1) * src_stride_bgra; + src_stride_bgra = -src_stride_bgra; + } +#if defined(HAS_BGRATOYROW_SSSE3) && defined(HAS_BGRATOUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + BGRAToUVRow = BGRAToUVRow_Any_SSSE3; + BGRAToYRow = BGRAToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + BGRAToUVRow = BGRAToUVRow_SSSE3; + BGRAToYRow = BGRAToYRow_SSSE3; + } + } +#endif +#if defined(HAS_BGRATOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + BGRAToYRow = BGRAToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + BGRAToYRow = BGRAToYRow_NEON; + } + } +#endif +#if defined(HAS_BGRATOUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + BGRAToUVRow = BGRAToUVRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + BGRAToUVRow = BGRAToUVRow_NEON; + } + } +#endif + + for (y = 0; y < height - 1; y += 2) { + BGRAToUVRow(src_bgra, src_stride_bgra, dst_u, dst_v, width); + BGRAToYRow(src_bgra, dst_y, width); + BGRAToYRow(src_bgra + src_stride_bgra, dst_y + dst_stride_y, width); + src_bgra += src_stride_bgra * 2; + dst_y += dst_stride_y * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { + BGRAToUVRow(src_bgra, 0, dst_u, dst_v, width); + BGRAToYRow(src_bgra, dst_y, width); + } + return 0; +} + +// Convert ABGR to I420. +LIBYUV_API +int ABGRToI420(const uint8* src_abgr, int src_stride_abgr, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*ABGRToUVRow)(const uint8* src_abgr0, int src_stride_abgr, + uint8* dst_u, uint8* dst_v, int width) = ABGRToUVRow_C; + void (*ABGRToYRow)(const uint8* src_abgr, uint8* dst_y, int pix) = + ABGRToYRow_C; + if (!src_abgr || + !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_abgr = src_abgr + (height - 1) * src_stride_abgr; + src_stride_abgr = -src_stride_abgr; + } +#if defined(HAS_ABGRTOYROW_SSSE3) && defined(HAS_ABGRTOUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ABGRToUVRow = ABGRToUVRow_Any_SSSE3; + ABGRToYRow = ABGRToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ABGRToUVRow = ABGRToUVRow_SSSE3; + ABGRToYRow = ABGRToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ABGRTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ABGRToYRow = ABGRToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ABGRToYRow = ABGRToYRow_NEON; + } + } +#endif +#if defined(HAS_ABGRTOUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ABGRToUVRow = ABGRToUVRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + ABGRToUVRow = ABGRToUVRow_NEON; + } + } +#endif + + for (y = 0; y < height - 1; y += 2) { + ABGRToUVRow(src_abgr, src_stride_abgr, dst_u, dst_v, width); + ABGRToYRow(src_abgr, dst_y, width); + ABGRToYRow(src_abgr + src_stride_abgr, dst_y + dst_stride_y, width); + src_abgr += src_stride_abgr * 2; + dst_y += dst_stride_y * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { + ABGRToUVRow(src_abgr, 0, dst_u, dst_v, width); + ABGRToYRow(src_abgr, dst_y, width); + } + return 0; +} + +// Convert RGBA to I420. +LIBYUV_API +int RGBAToI420(const uint8* src_rgba, int src_stride_rgba, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*RGBAToUVRow)(const uint8* src_rgba0, int src_stride_rgba, + uint8* dst_u, uint8* dst_v, int width) = RGBAToUVRow_C; + void (*RGBAToYRow)(const uint8* src_rgba, uint8* dst_y, int pix) = + RGBAToYRow_C; + if (!src_rgba || + !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_rgba = src_rgba + (height - 1) * src_stride_rgba; + src_stride_rgba = -src_stride_rgba; + } +#if defined(HAS_RGBATOYROW_SSSE3) && defined(HAS_RGBATOUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGBAToUVRow = RGBAToUVRow_Any_SSSE3; + RGBAToYRow = RGBAToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBAToUVRow = RGBAToUVRow_SSSE3; + RGBAToYRow = RGBAToYRow_SSSE3; + } + } +#endif +#if defined(HAS_RGBATOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RGBAToYRow = RGBAToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + RGBAToYRow = RGBAToYRow_NEON; + } + } +#endif +#if defined(HAS_RGBATOUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RGBAToUVRow = RGBAToUVRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + RGBAToUVRow = RGBAToUVRow_NEON; + } + } +#endif + + for (y = 0; y < height - 1; y += 2) { + RGBAToUVRow(src_rgba, src_stride_rgba, dst_u, dst_v, width); + RGBAToYRow(src_rgba, dst_y, width); + RGBAToYRow(src_rgba + src_stride_rgba, dst_y + dst_stride_y, width); + src_rgba += src_stride_rgba * 2; + dst_y += dst_stride_y * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { + RGBAToUVRow(src_rgba, 0, dst_u, dst_v, width); + RGBAToYRow(src_rgba, dst_y, width); + } + return 0; +} + +// Convert RGB24 to I420. +LIBYUV_API +int RGB24ToI420(const uint8* src_rgb24, int src_stride_rgb24, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; +#if defined(HAS_RGB24TOYROW_NEON) + void (*RGB24ToUVRow)(const uint8* src_rgb24, int src_stride_rgb24, + uint8* dst_u, uint8* dst_v, int width) = RGB24ToUVRow_C; + void (*RGB24ToYRow)(const uint8* src_rgb24, uint8* dst_y, int pix) = + RGB24ToYRow_C; +#else + void (*RGB24ToARGBRow)(const uint8* src_rgb, uint8* dst_argb, int pix) = + RGB24ToARGBRow_C; + void (*ARGBToUVRow)(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) = ARGBToUVRow_C; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; +#endif + if (!src_rgb24 || !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_rgb24 = src_rgb24 + (height - 1) * src_stride_rgb24; + src_stride_rgb24 = -src_stride_rgb24; + } + +// Neon version does direct RGB24 to YUV. +#if defined(HAS_RGB24TOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RGB24ToUVRow = RGB24ToUVRow_Any_NEON; + RGB24ToYRow = RGB24ToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + RGB24ToYRow = RGB24ToYRow_NEON; + if (IS_ALIGNED(width, 16)) { + RGB24ToUVRow = RGB24ToUVRow_NEON; + } + } + } +// Other platforms do intermediate conversion from RGB24 to ARGB. +#else +#if defined(HAS_RGB24TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGB24ToARGBRow = RGB24ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGB24ToARGBRow = RGB24ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVRow = ARGBToUVRow_Any_SSSE3; + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUVRow = ARGBToUVRow_SSSE3; + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToUVRow = ARGBToUVRow_Any_AVX2; + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToUVRow = ARGBToUVRow_AVX2; + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif + { + // Allocate 2 rows of ARGB. + const int kRowSize = (width * 4 + 31) & ~31; + align_buffer_64(row, kRowSize * 2); +#endif + + for (y = 0; y < height - 1; y += 2) { +#if defined(HAS_RGB24TOYROW_NEON) + RGB24ToUVRow(src_rgb24, src_stride_rgb24, dst_u, dst_v, width); + RGB24ToYRow(src_rgb24, dst_y, width); + RGB24ToYRow(src_rgb24 + src_stride_rgb24, dst_y + dst_stride_y, width); +#else + RGB24ToARGBRow(src_rgb24, row, width); + RGB24ToARGBRow(src_rgb24 + src_stride_rgb24, row + kRowSize, width); + ARGBToUVRow(row, kRowSize, dst_u, dst_v, width); + ARGBToYRow(row, dst_y, width); + ARGBToYRow(row + kRowSize, dst_y + dst_stride_y, width); +#endif + src_rgb24 += src_stride_rgb24 * 2; + dst_y += dst_stride_y * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { +#if defined(HAS_RGB24TOYROW_NEON) + RGB24ToUVRow(src_rgb24, 0, dst_u, dst_v, width); + RGB24ToYRow(src_rgb24, dst_y, width); +#else + RGB24ToARGBRow(src_rgb24, row, width); + ARGBToUVRow(row, 0, dst_u, dst_v, width); + ARGBToYRow(row, dst_y, width); +#endif + } +#if !defined(HAS_RGB24TOYROW_NEON) + free_aligned_buffer_64(row); + } +#endif + return 0; +} + +// Convert RAW to I420. +LIBYUV_API +int RAWToI420(const uint8* src_raw, int src_stride_raw, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; +#if defined(HAS_RAWTOYROW_NEON) + void (*RAWToUVRow)(const uint8* src_raw, int src_stride_raw, + uint8* dst_u, uint8* dst_v, int width) = RAWToUVRow_C; + void (*RAWToYRow)(const uint8* src_raw, uint8* dst_y, int pix) = + RAWToYRow_C; +#else + void (*RAWToARGBRow)(const uint8* src_rgb, uint8* dst_argb, int pix) = + RAWToARGBRow_C; + void (*ARGBToUVRow)(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) = ARGBToUVRow_C; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; +#endif + if (!src_raw || !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_raw = src_raw + (height - 1) * src_stride_raw; + src_stride_raw = -src_stride_raw; + } + +// Neon version does direct RAW to YUV. +#if defined(HAS_RAWTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RAWToUVRow = RAWToUVRow_Any_NEON; + RAWToYRow = RAWToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + RAWToYRow = RAWToYRow_NEON; + if (IS_ALIGNED(width, 16)) { + RAWToUVRow = RAWToUVRow_NEON; + } + } + } +// Other platforms do intermediate conversion from RAW to ARGB. +#else +#if defined(HAS_RAWTOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RAWToARGBRow = RAWToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RAWToARGBRow = RAWToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVRow = ARGBToUVRow_Any_SSSE3; + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUVRow = ARGBToUVRow_SSSE3; + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToUVRow = ARGBToUVRow_Any_AVX2; + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToUVRow = ARGBToUVRow_AVX2; + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif + { + // Allocate 2 rows of ARGB. + const int kRowSize = (width * 4 + 31) & ~31; + align_buffer_64(row, kRowSize * 2); +#endif + + for (y = 0; y < height - 1; y += 2) { +#if defined(HAS_RAWTOYROW_NEON) + RAWToUVRow(src_raw, src_stride_raw, dst_u, dst_v, width); + RAWToYRow(src_raw, dst_y, width); + RAWToYRow(src_raw + src_stride_raw, dst_y + dst_stride_y, width); +#else + RAWToARGBRow(src_raw, row, width); + RAWToARGBRow(src_raw + src_stride_raw, row + kRowSize, width); + ARGBToUVRow(row, kRowSize, dst_u, dst_v, width); + ARGBToYRow(row, dst_y, width); + ARGBToYRow(row + kRowSize, dst_y + dst_stride_y, width); +#endif + src_raw += src_stride_raw * 2; + dst_y += dst_stride_y * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { +#if defined(HAS_RAWTOYROW_NEON) + RAWToUVRow(src_raw, 0, dst_u, dst_v, width); + RAWToYRow(src_raw, dst_y, width); +#else + RAWToARGBRow(src_raw, row, width); + ARGBToUVRow(row, 0, dst_u, dst_v, width); + ARGBToYRow(row, dst_y, width); +#endif + } +#if !defined(HAS_RAWTOYROW_NEON) + free_aligned_buffer_64(row); + } +#endif + return 0; +} + +// Convert RGB565 to I420. +LIBYUV_API +int RGB565ToI420(const uint8* src_rgb565, int src_stride_rgb565, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; +#if defined(HAS_RGB565TOYROW_NEON) + void (*RGB565ToUVRow)(const uint8* src_rgb565, int src_stride_rgb565, + uint8* dst_u, uint8* dst_v, int width) = RGB565ToUVRow_C; + void (*RGB565ToYRow)(const uint8* src_rgb565, uint8* dst_y, int pix) = + RGB565ToYRow_C; +#else + void (*RGB565ToARGBRow)(const uint8* src_rgb, uint8* dst_argb, int pix) = + RGB565ToARGBRow_C; + void (*ARGBToUVRow)(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) = ARGBToUVRow_C; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; +#endif + if (!src_rgb565 || !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_rgb565 = src_rgb565 + (height - 1) * src_stride_rgb565; + src_stride_rgb565 = -src_stride_rgb565; + } + +// Neon version does direct RGB565 to YUV. +#if defined(HAS_RGB565TOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RGB565ToUVRow = RGB565ToUVRow_Any_NEON; + RGB565ToYRow = RGB565ToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + RGB565ToYRow = RGB565ToYRow_NEON; + if (IS_ALIGNED(width, 16)) { + RGB565ToUVRow = RGB565ToUVRow_NEON; + } + } + } +// Other platforms do intermediate conversion from RGB565 to ARGB. +#else +#if defined(HAS_RGB565TOARGBROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + RGB565ToARGBRow = RGB565ToARGBRow_Any_SSE2; + if (IS_ALIGNED(width, 8)) { + RGB565ToARGBRow = RGB565ToARGBRow_SSE2; + } + } +#endif +#if defined(HAS_RGB565TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + RGB565ToARGBRow = RGB565ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + RGB565ToARGBRow = RGB565ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVRow = ARGBToUVRow_Any_SSSE3; + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUVRow = ARGBToUVRow_SSSE3; + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToUVRow = ARGBToUVRow_Any_AVX2; + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToUVRow = ARGBToUVRow_AVX2; + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif + { + // Allocate 2 rows of ARGB. + const int kRowSize = (width * 4 + 31) & ~31; + align_buffer_64(row, kRowSize * 2); +#endif + + for (y = 0; y < height - 1; y += 2) { +#if defined(HAS_RGB565TOYROW_NEON) + RGB565ToUVRow(src_rgb565, src_stride_rgb565, dst_u, dst_v, width); + RGB565ToYRow(src_rgb565, dst_y, width); + RGB565ToYRow(src_rgb565 + src_stride_rgb565, dst_y + dst_stride_y, width); +#else + RGB565ToARGBRow(src_rgb565, row, width); + RGB565ToARGBRow(src_rgb565 + src_stride_rgb565, row + kRowSize, width); + ARGBToUVRow(row, kRowSize, dst_u, dst_v, width); + ARGBToYRow(row, dst_y, width); + ARGBToYRow(row + kRowSize, dst_y + dst_stride_y, width); +#endif + src_rgb565 += src_stride_rgb565 * 2; + dst_y += dst_stride_y * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { +#if defined(HAS_RGB565TOYROW_NEON) + RGB565ToUVRow(src_rgb565, 0, dst_u, dst_v, width); + RGB565ToYRow(src_rgb565, dst_y, width); +#else + RGB565ToARGBRow(src_rgb565, row, width); + ARGBToUVRow(row, 0, dst_u, dst_v, width); + ARGBToYRow(row, dst_y, width); +#endif + } +#if !defined(HAS_RGB565TOYROW_NEON) + free_aligned_buffer_64(row); + } +#endif + return 0; +} + +// Convert ARGB1555 to I420. +LIBYUV_API +int ARGB1555ToI420(const uint8* src_argb1555, int src_stride_argb1555, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; +#if defined(HAS_ARGB1555TOYROW_NEON) + void (*ARGB1555ToUVRow)(const uint8* src_argb1555, int src_stride_argb1555, + uint8* dst_u, uint8* dst_v, int width) = ARGB1555ToUVRow_C; + void (*ARGB1555ToYRow)(const uint8* src_argb1555, uint8* dst_y, int pix) = + ARGB1555ToYRow_C; +#else + void (*ARGB1555ToARGBRow)(const uint8* src_rgb, uint8* dst_argb, int pix) = + ARGB1555ToARGBRow_C; + void (*ARGBToUVRow)(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) = ARGBToUVRow_C; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; +#endif + if (!src_argb1555 || !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb1555 = src_argb1555 + (height - 1) * src_stride_argb1555; + src_stride_argb1555 = -src_stride_argb1555; + } + +// Neon version does direct ARGB1555 to YUV. +#if defined(HAS_ARGB1555TOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGB1555ToUVRow = ARGB1555ToUVRow_Any_NEON; + ARGB1555ToYRow = ARGB1555ToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGB1555ToYRow = ARGB1555ToYRow_NEON; + if (IS_ALIGNED(width, 16)) { + ARGB1555ToUVRow = ARGB1555ToUVRow_NEON; + } + } + } +// Other platforms do intermediate conversion from ARGB1555 to ARGB. +#else +#if defined(HAS_ARGB1555TOARGBROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_SSE2; + if (IS_ALIGNED(width, 8)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_SSE2; + } + } +#endif +#if defined(HAS_ARGB1555TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVRow = ARGBToUVRow_Any_SSSE3; + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUVRow = ARGBToUVRow_SSSE3; + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToUVRow = ARGBToUVRow_Any_AVX2; + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToUVRow = ARGBToUVRow_AVX2; + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif + { + // Allocate 2 rows of ARGB. + const int kRowSize = (width * 4 + 31) & ~31; + align_buffer_64(row, kRowSize * 2); +#endif + + for (y = 0; y < height - 1; y += 2) { +#if defined(HAS_ARGB1555TOYROW_NEON) + ARGB1555ToUVRow(src_argb1555, src_stride_argb1555, dst_u, dst_v, width); + ARGB1555ToYRow(src_argb1555, dst_y, width); + ARGB1555ToYRow(src_argb1555 + src_stride_argb1555, dst_y + dst_stride_y, + width); +#else + ARGB1555ToARGBRow(src_argb1555, row, width); + ARGB1555ToARGBRow(src_argb1555 + src_stride_argb1555, row + kRowSize, + width); + ARGBToUVRow(row, kRowSize, dst_u, dst_v, width); + ARGBToYRow(row, dst_y, width); + ARGBToYRow(row + kRowSize, dst_y + dst_stride_y, width); +#endif + src_argb1555 += src_stride_argb1555 * 2; + dst_y += dst_stride_y * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { +#if defined(HAS_ARGB1555TOYROW_NEON) + ARGB1555ToUVRow(src_argb1555, 0, dst_u, dst_v, width); + ARGB1555ToYRow(src_argb1555, dst_y, width); +#else + ARGB1555ToARGBRow(src_argb1555, row, width); + ARGBToUVRow(row, 0, dst_u, dst_v, width); + ARGBToYRow(row, dst_y, width); +#endif + } +#if !defined(HAS_ARGB1555TOYROW_NEON) + free_aligned_buffer_64(row); + } +#endif + return 0; +} + +// Convert ARGB4444 to I420. +LIBYUV_API +int ARGB4444ToI420(const uint8* src_argb4444, int src_stride_argb4444, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; +#if defined(HAS_ARGB4444TOYROW_NEON) + void (*ARGB4444ToUVRow)(const uint8* src_argb4444, int src_stride_argb4444, + uint8* dst_u, uint8* dst_v, int width) = ARGB4444ToUVRow_C; + void (*ARGB4444ToYRow)(const uint8* src_argb4444, uint8* dst_y, int pix) = + ARGB4444ToYRow_C; +#else + void (*ARGB4444ToARGBRow)(const uint8* src_rgb, uint8* dst_argb, int pix) = + ARGB4444ToARGBRow_C; + void (*ARGBToUVRow)(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) = ARGBToUVRow_C; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; +#endif + if (!src_argb4444 || !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb4444 = src_argb4444 + (height - 1) * src_stride_argb4444; + src_stride_argb4444 = -src_stride_argb4444; + } + +// Neon version does direct ARGB4444 to YUV. +#if defined(HAS_ARGB4444TOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGB4444ToUVRow = ARGB4444ToUVRow_Any_NEON; + ARGB4444ToYRow = ARGB4444ToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGB4444ToYRow = ARGB4444ToYRow_NEON; + if (IS_ALIGNED(width, 16)) { + ARGB4444ToUVRow = ARGB4444ToUVRow_NEON; + } + } + } +// Other platforms do intermediate conversion from ARGB4444 to ARGB. +#else +#if defined(HAS_ARGB4444TOARGBROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_SSE2; + if (IS_ALIGNED(width, 8)) { + ARGB4444ToARGBRow = ARGB4444ToARGBRow_SSE2; + } + } +#endif +#if defined(HAS_ARGB4444TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + ARGB4444ToARGBRow = ARGB4444ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVRow = ARGBToUVRow_Any_SSSE3; + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUVRow = ARGBToUVRow_SSSE3; + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToUVRow = ARGBToUVRow_Any_AVX2; + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToUVRow = ARGBToUVRow_AVX2; + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif + { + // Allocate 2 rows of ARGB. + const int kRowSize = (width * 4 + 31) & ~31; + align_buffer_64(row, kRowSize * 2); +#endif + + for (y = 0; y < height - 1; y += 2) { +#if defined(HAS_ARGB4444TOYROW_NEON) + ARGB4444ToUVRow(src_argb4444, src_stride_argb4444, dst_u, dst_v, width); + ARGB4444ToYRow(src_argb4444, dst_y, width); + ARGB4444ToYRow(src_argb4444 + src_stride_argb4444, dst_y + dst_stride_y, + width); +#else + ARGB4444ToARGBRow(src_argb4444, row, width); + ARGB4444ToARGBRow(src_argb4444 + src_stride_argb4444, row + kRowSize, + width); + ARGBToUVRow(row, kRowSize, dst_u, dst_v, width); + ARGBToYRow(row, dst_y, width); + ARGBToYRow(row + kRowSize, dst_y + dst_stride_y, width); +#endif + src_argb4444 += src_stride_argb4444 * 2; + dst_y += dst_stride_y * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { +#if defined(HAS_ARGB4444TOYROW_NEON) + ARGB4444ToUVRow(src_argb4444, 0, dst_u, dst_v, width); + ARGB4444ToYRow(src_argb4444, dst_y, width); +#else + ARGB4444ToARGBRow(src_argb4444, row, width); + ARGBToUVRow(row, 0, dst_u, dst_v, width); + ARGBToYRow(row, dst_y, width); +#endif + } +#if !defined(HAS_ARGB4444TOYROW_NEON) + free_aligned_buffer_64(row); + } +#endif + return 0; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/convert_argb.cc b/third_party/aom/third_party/libyuv/source/convert_argb.cc new file mode 100644 index 0000000000..44756bc41c --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/convert_argb.cc @@ -0,0 +1,1155 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/convert_argb.h" + +#include "libyuv/cpu_id.h" +#ifdef HAVE_JPEG +#include "libyuv/mjpeg_decoder.h" +#endif +#include "libyuv/rotate_argb.h" +#include "libyuv/row.h" +#include "libyuv/video_common.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Copy ARGB with optional flipping +LIBYUV_API +int ARGBCopy(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + if (!src_argb || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + + CopyPlane(src_argb, src_stride_argb, dst_argb, dst_stride_argb, + width * 4, height); + return 0; +} + +// Convert I444 to ARGB. +LIBYUV_API +int I444ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*I444ToARGBRow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I444ToARGBRow_C; + if (!src_y || !src_u || !src_v || + !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + // Coalesce rows. + if (src_stride_y == width && + src_stride_u == width && + src_stride_v == width && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0; + } +#if defined(HAS_I444TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I444ToARGBRow = I444ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I444ToARGBRow = I444ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_I444TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I444ToARGBRow = I444ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I444ToARGBRow = I444ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_I444TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I444ToARGBRow = I444ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I444ToARGBRow = I444ToARGBRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + I444ToARGBRow(src_y, src_u, src_v, dst_argb, width); + dst_argb += dst_stride_argb; + src_y += src_stride_y; + src_u += src_stride_u; + src_v += src_stride_v; + } + return 0; +} + +// Convert I422 to ARGB. +LIBYUV_API +int I422ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*I422ToARGBRow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToARGBRow_C; + if (!src_y || !src_u || !src_v || + !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + // Coalesce rows. + if (src_stride_y == width && + src_stride_u * 2 == width && + src_stride_v * 2 == width && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0; + } +#if defined(HAS_I422TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToARGBRow = I422ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToARGBRow = I422ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_I422TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToARGBRow = I422ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToARGBRow = I422ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_I422TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToARGBRow = I422ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToARGBRow = I422ToARGBRow_NEON; + } + } +#endif +#if defined(HAS_I422TOARGBROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(width, 4) && + IS_ALIGNED(src_y, 4) && IS_ALIGNED(src_stride_y, 4) && + IS_ALIGNED(src_u, 2) && IS_ALIGNED(src_stride_u, 2) && + IS_ALIGNED(src_v, 2) && IS_ALIGNED(src_stride_v, 2) && + IS_ALIGNED(dst_argb, 4) && IS_ALIGNED(dst_stride_argb, 4)) { + I422ToARGBRow = I422ToARGBRow_MIPS_DSPR2; + } +#endif + + for (y = 0; y < height; ++y) { + I422ToARGBRow(src_y, src_u, src_v, dst_argb, width); + dst_argb += dst_stride_argb; + src_y += src_stride_y; + src_u += src_stride_u; + src_v += src_stride_v; + } + return 0; +} + +// Convert I411 to ARGB. +LIBYUV_API +int I411ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*I411ToARGBRow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I411ToARGBRow_C; + if (!src_y || !src_u || !src_v || + !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + // Coalesce rows. + if (src_stride_y == width && + src_stride_u * 4 == width && + src_stride_v * 4 == width && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0; + } +#if defined(HAS_I411TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I411ToARGBRow = I411ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I411ToARGBRow = I411ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_I411TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I411ToARGBRow = I411ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I411ToARGBRow = I411ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_I411TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I411ToARGBRow = I411ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I411ToARGBRow = I411ToARGBRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + I411ToARGBRow(src_y, src_u, src_v, dst_argb, width); + dst_argb += dst_stride_argb; + src_y += src_stride_y; + src_u += src_stride_u; + src_v += src_stride_v; + } + return 0; +} + +// Convert I400 to ARGB. +LIBYUV_API +int I400ToARGB(const uint8* src_y, int src_stride_y, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*I400ToARGBRow)(const uint8* y_buf, + uint8* rgb_buf, + int width) = I400ToARGBRow_C; + if (!src_y || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + // Coalesce rows. + if (src_stride_y == width && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_y = dst_stride_argb = 0; + } +#if defined(HAS_I400TOARGBROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + I400ToARGBRow = I400ToARGBRow_Any_SSE2; + if (IS_ALIGNED(width, 8)) { + I400ToARGBRow = I400ToARGBRow_SSE2; + } + } +#endif +#if defined(HAS_I400TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I400ToARGBRow = I400ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I400ToARGBRow = I400ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_I400TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I400ToARGBRow = I400ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I400ToARGBRow = I400ToARGBRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + I400ToARGBRow(src_y, dst_argb, width); + dst_argb += dst_stride_argb; + src_y += src_stride_y; + } + return 0; +} + +// Convert J400 to ARGB. +LIBYUV_API +int J400ToARGB(const uint8* src_y, int src_stride_y, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*J400ToARGBRow)(const uint8* src_y, uint8* dst_argb, int pix) = + J400ToARGBRow_C; + if (!src_y || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_y = src_y + (height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } + // Coalesce rows. + if (src_stride_y == width && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_y = dst_stride_argb = 0; + } +#if defined(HAS_J400TOARGBROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + J400ToARGBRow = J400ToARGBRow_Any_SSE2; + if (IS_ALIGNED(width, 8)) { + J400ToARGBRow = J400ToARGBRow_SSE2; + } + } +#endif +#if defined(HAS_J400TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + J400ToARGBRow = J400ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + J400ToARGBRow = J400ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_J400TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + J400ToARGBRow = J400ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + J400ToARGBRow = J400ToARGBRow_NEON; + } + } +#endif + for (y = 0; y < height; ++y) { + J400ToARGBRow(src_y, dst_argb, width); + src_y += src_stride_y; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Shuffle table for converting BGRA to ARGB. +static uvec8 kShuffleMaskBGRAToARGB = { + 3u, 2u, 1u, 0u, 7u, 6u, 5u, 4u, 11u, 10u, 9u, 8u, 15u, 14u, 13u, 12u +}; + +// Shuffle table for converting ABGR to ARGB. +static uvec8 kShuffleMaskABGRToARGB = { + 2u, 1u, 0u, 3u, 6u, 5u, 4u, 7u, 10u, 9u, 8u, 11u, 14u, 13u, 12u, 15u +}; + +// Shuffle table for converting RGBA to ARGB. +static uvec8 kShuffleMaskRGBAToARGB = { + 1u, 2u, 3u, 0u, 5u, 6u, 7u, 4u, 9u, 10u, 11u, 8u, 13u, 14u, 15u, 12u +}; + +// Convert BGRA to ARGB. +LIBYUV_API +int BGRAToARGB(const uint8* src_bgra, int src_stride_bgra, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + return ARGBShuffle(src_bgra, src_stride_bgra, + dst_argb, dst_stride_argb, + (const uint8*)(&kShuffleMaskBGRAToARGB), + width, height); +} + +// Convert ARGB to BGRA (same as BGRAToARGB). +LIBYUV_API +int ARGBToBGRA(const uint8* src_bgra, int src_stride_bgra, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + return ARGBShuffle(src_bgra, src_stride_bgra, + dst_argb, dst_stride_argb, + (const uint8*)(&kShuffleMaskBGRAToARGB), + width, height); +} + +// Convert ABGR to ARGB. +LIBYUV_API +int ABGRToARGB(const uint8* src_abgr, int src_stride_abgr, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + return ARGBShuffle(src_abgr, src_stride_abgr, + dst_argb, dst_stride_argb, + (const uint8*)(&kShuffleMaskABGRToARGB), + width, height); +} + +// Convert ARGB to ABGR to (same as ABGRToARGB). +LIBYUV_API +int ARGBToABGR(const uint8* src_abgr, int src_stride_abgr, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + return ARGBShuffle(src_abgr, src_stride_abgr, + dst_argb, dst_stride_argb, + (const uint8*)(&kShuffleMaskABGRToARGB), + width, height); +} + +// Convert RGBA to ARGB. +LIBYUV_API +int RGBAToARGB(const uint8* src_rgba, int src_stride_rgba, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + return ARGBShuffle(src_rgba, src_stride_rgba, + dst_argb, dst_stride_argb, + (const uint8*)(&kShuffleMaskRGBAToARGB), + width, height); +} + +// Convert RGB24 to ARGB. +LIBYUV_API +int RGB24ToARGB(const uint8* src_rgb24, int src_stride_rgb24, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*RGB24ToARGBRow)(const uint8* src_rgb, uint8* dst_argb, int pix) = + RGB24ToARGBRow_C; + if (!src_rgb24 || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_rgb24 = src_rgb24 + (height - 1) * src_stride_rgb24; + src_stride_rgb24 = -src_stride_rgb24; + } + // Coalesce rows. + if (src_stride_rgb24 == width * 3 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_rgb24 = dst_stride_argb = 0; + } +#if defined(HAS_RGB24TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGB24ToARGBRow = RGB24ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGB24ToARGBRow = RGB24ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_RGB24TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RGB24ToARGBRow = RGB24ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + RGB24ToARGBRow = RGB24ToARGBRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + RGB24ToARGBRow(src_rgb24, dst_argb, width); + src_rgb24 += src_stride_rgb24; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Convert RAW to ARGB. +LIBYUV_API +int RAWToARGB(const uint8* src_raw, int src_stride_raw, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*RAWToARGBRow)(const uint8* src_rgb, uint8* dst_argb, int pix) = + RAWToARGBRow_C; + if (!src_raw || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_raw = src_raw + (height - 1) * src_stride_raw; + src_stride_raw = -src_stride_raw; + } + // Coalesce rows. + if (src_stride_raw == width * 3 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_raw = dst_stride_argb = 0; + } +#if defined(HAS_RAWTOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RAWToARGBRow = RAWToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RAWToARGBRow = RAWToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_RAWTOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RAWToARGBRow = RAWToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + RAWToARGBRow = RAWToARGBRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + RAWToARGBRow(src_raw, dst_argb, width); + src_raw += src_stride_raw; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Convert RGB565 to ARGB. +LIBYUV_API +int RGB565ToARGB(const uint8* src_rgb565, int src_stride_rgb565, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*RGB565ToARGBRow)(const uint8* src_rgb565, uint8* dst_argb, int pix) = + RGB565ToARGBRow_C; + if (!src_rgb565 || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_rgb565 = src_rgb565 + (height - 1) * src_stride_rgb565; + src_stride_rgb565 = -src_stride_rgb565; + } + // Coalesce rows. + if (src_stride_rgb565 == width * 2 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_rgb565 = dst_stride_argb = 0; + } +#if defined(HAS_RGB565TOARGBROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + RGB565ToARGBRow = RGB565ToARGBRow_Any_SSE2; + if (IS_ALIGNED(width, 8)) { + RGB565ToARGBRow = RGB565ToARGBRow_SSE2; + } + } +#endif +#if defined(HAS_RGB565TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + RGB565ToARGBRow = RGB565ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + RGB565ToARGBRow = RGB565ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_RGB565TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RGB565ToARGBRow = RGB565ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + RGB565ToARGBRow = RGB565ToARGBRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + RGB565ToARGBRow(src_rgb565, dst_argb, width); + src_rgb565 += src_stride_rgb565; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Convert ARGB1555 to ARGB. +LIBYUV_API +int ARGB1555ToARGB(const uint8* src_argb1555, int src_stride_argb1555, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*ARGB1555ToARGBRow)(const uint8* src_argb1555, uint8* dst_argb, + int pix) = ARGB1555ToARGBRow_C; + if (!src_argb1555 || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb1555 = src_argb1555 + (height - 1) * src_stride_argb1555; + src_stride_argb1555 = -src_stride_argb1555; + } + // Coalesce rows. + if (src_stride_argb1555 == width * 2 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb1555 = dst_stride_argb = 0; + } +#if defined(HAS_ARGB1555TOARGBROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_SSE2; + if (IS_ALIGNED(width, 8)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_SSE2; + } + } +#endif +#if defined(HAS_ARGB1555TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_ARGB1555TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGB1555ToARGBRow(src_argb1555, dst_argb, width); + src_argb1555 += src_stride_argb1555; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Convert ARGB4444 to ARGB. +LIBYUV_API +int ARGB4444ToARGB(const uint8* src_argb4444, int src_stride_argb4444, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*ARGB4444ToARGBRow)(const uint8* src_argb4444, uint8* dst_argb, + int pix) = ARGB4444ToARGBRow_C; + if (!src_argb4444 || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb4444 = src_argb4444 + (height - 1) * src_stride_argb4444; + src_stride_argb4444 = -src_stride_argb4444; + } + // Coalesce rows. + if (src_stride_argb4444 == width * 2 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb4444 = dst_stride_argb = 0; + } +#if defined(HAS_ARGB4444TOARGBROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_SSE2; + if (IS_ALIGNED(width, 8)) { + ARGB4444ToARGBRow = ARGB4444ToARGBRow_SSE2; + } + } +#endif +#if defined(HAS_ARGB4444TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + ARGB4444ToARGBRow = ARGB4444ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_ARGB4444TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGB4444ToARGBRow = ARGB4444ToARGBRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGB4444ToARGBRow(src_argb4444, dst_argb, width); + src_argb4444 += src_stride_argb4444; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Convert NV12 to ARGB. +LIBYUV_API +int NV12ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_uv, int src_stride_uv, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*NV12ToARGBRow)(const uint8* y_buf, + const uint8* uv_buf, + uint8* rgb_buf, + int width) = NV12ToARGBRow_C; + if (!src_y || !src_uv || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } +#if defined(HAS_NV12TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + NV12ToARGBRow = NV12ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + NV12ToARGBRow = NV12ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_NV12TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + NV12ToARGBRow = NV12ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + NV12ToARGBRow = NV12ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_NV12TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + NV12ToARGBRow = NV12ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + NV12ToARGBRow = NV12ToARGBRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + NV12ToARGBRow(src_y, src_uv, dst_argb, width); + dst_argb += dst_stride_argb; + src_y += src_stride_y; + if (y & 1) { + src_uv += src_stride_uv; + } + } + return 0; +} + +// Convert NV21 to ARGB. +LIBYUV_API +int NV21ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_uv, int src_stride_uv, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*NV21ToARGBRow)(const uint8* y_buf, + const uint8* uv_buf, + uint8* rgb_buf, + int width) = NV21ToARGBRow_C; + if (!src_y || !src_uv || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } +#if defined(HAS_NV21TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + NV21ToARGBRow = NV21ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + NV21ToARGBRow = NV21ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_NV21TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + NV21ToARGBRow = NV21ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + NV21ToARGBRow = NV21ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_NV21TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + NV21ToARGBRow = NV21ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + NV21ToARGBRow = NV21ToARGBRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + NV21ToARGBRow(src_y, src_uv, dst_argb, width); + dst_argb += dst_stride_argb; + src_y += src_stride_y; + if (y & 1) { + src_uv += src_stride_uv; + } + } + return 0; +} + +// Convert M420 to ARGB. +LIBYUV_API +int M420ToARGB(const uint8* src_m420, int src_stride_m420, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*NV12ToARGBRow)(const uint8* y_buf, + const uint8* uv_buf, + uint8* rgb_buf, + int width) = NV12ToARGBRow_C; + if (!src_m420 || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } +#if defined(HAS_NV12TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + NV12ToARGBRow = NV12ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + NV12ToARGBRow = NV12ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_NV12TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + NV12ToARGBRow = NV12ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + NV12ToARGBRow = NV12ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_NV12TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + NV12ToARGBRow = NV12ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + NV12ToARGBRow = NV12ToARGBRow_NEON; + } + } +#endif + + for (y = 0; y < height - 1; y += 2) { + NV12ToARGBRow(src_m420, src_m420 + src_stride_m420 * 2, dst_argb, width); + NV12ToARGBRow(src_m420 + src_stride_m420, src_m420 + src_stride_m420 * 2, + dst_argb + dst_stride_argb, width); + dst_argb += dst_stride_argb * 2; + src_m420 += src_stride_m420 * 3; + } + if (height & 1) { + NV12ToARGBRow(src_m420, src_m420 + src_stride_m420 * 2, dst_argb, width); + } + return 0; +} + +// Convert YUY2 to ARGB. +LIBYUV_API +int YUY2ToARGB(const uint8* src_yuy2, int src_stride_yuy2, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*YUY2ToARGBRow)(const uint8* src_yuy2, uint8* dst_argb, int pix) = + YUY2ToARGBRow_C; + if (!src_yuy2 || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_yuy2 = src_yuy2 + (height - 1) * src_stride_yuy2; + src_stride_yuy2 = -src_stride_yuy2; + } + // Coalesce rows. + if (src_stride_yuy2 == width * 2 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_yuy2 = dst_stride_argb = 0; + } +#if defined(HAS_YUY2TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + YUY2ToARGBRow = YUY2ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + YUY2ToARGBRow = YUY2ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_YUY2TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + YUY2ToARGBRow = YUY2ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + YUY2ToARGBRow = YUY2ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_YUY2TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + YUY2ToARGBRow = YUY2ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + YUY2ToARGBRow = YUY2ToARGBRow_NEON; + } + } +#endif + for (y = 0; y < height; ++y) { + YUY2ToARGBRow(src_yuy2, dst_argb, width); + src_yuy2 += src_stride_yuy2; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Convert UYVY to ARGB. +LIBYUV_API +int UYVYToARGB(const uint8* src_uyvy, int src_stride_uyvy, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*UYVYToARGBRow)(const uint8* src_uyvy, uint8* dst_argb, int pix) = + UYVYToARGBRow_C; + if (!src_uyvy || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_uyvy = src_uyvy + (height - 1) * src_stride_uyvy; + src_stride_uyvy = -src_stride_uyvy; + } + // Coalesce rows. + if (src_stride_uyvy == width * 2 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_uyvy = dst_stride_argb = 0; + } +#if defined(HAS_UYVYTOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + UYVYToARGBRow = UYVYToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + UYVYToARGBRow = UYVYToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_UYVYTOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + UYVYToARGBRow = UYVYToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + UYVYToARGBRow = UYVYToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_UYVYTOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + UYVYToARGBRow = UYVYToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + UYVYToARGBRow = UYVYToARGBRow_NEON; + } + } +#endif + for (y = 0; y < height; ++y) { + UYVYToARGBRow(src_uyvy, dst_argb, width); + src_uyvy += src_stride_uyvy; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Convert J420 to ARGB. +LIBYUV_API +int J420ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*J422ToARGBRow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = J422ToARGBRow_C; + if (!src_y || !src_u || !src_v || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } +#if defined(HAS_J422TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + J422ToARGBRow = J422ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + J422ToARGBRow = J422ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_J422TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + J422ToARGBRow = J422ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + J422ToARGBRow = J422ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_J422TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + J422ToARGBRow = J422ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + J422ToARGBRow = J422ToARGBRow_NEON; + } + } +#endif +#if defined(HAS_J422TOARGBROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(width, 4) && + IS_ALIGNED(src_y, 4) && IS_ALIGNED(src_stride_y, 4) && + IS_ALIGNED(src_u, 2) && IS_ALIGNED(src_stride_u, 2) && + IS_ALIGNED(src_v, 2) && IS_ALIGNED(src_stride_v, 2) && + IS_ALIGNED(dst_argb, 4) && IS_ALIGNED(dst_stride_argb, 4)) { + J422ToARGBRow = J422ToARGBRow_MIPS_DSPR2; + } +#endif + + for (y = 0; y < height; ++y) { + J422ToARGBRow(src_y, src_u, src_v, dst_argb, width); + dst_argb += dst_stride_argb; + src_y += src_stride_y; + if (y & 1) { + src_u += src_stride_u; + src_v += src_stride_v; + } + } + return 0; +} + +// Convert J422 to ARGB. +LIBYUV_API +int J422ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*J422ToARGBRow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = J422ToARGBRow_C; + if (!src_y || !src_u || !src_v || + !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + // Coalesce rows. + if (src_stride_y == width && + src_stride_u * 2 == width && + src_stride_v * 2 == width && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0; + } +#if defined(HAS_J422TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + J422ToARGBRow = J422ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + J422ToARGBRow = J422ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_J422TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + J422ToARGBRow = J422ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + J422ToARGBRow = J422ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_J422TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + J422ToARGBRow = J422ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + J422ToARGBRow = J422ToARGBRow_NEON; + } + } +#endif +#if defined(HAS_J422TOARGBROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(width, 4) && + IS_ALIGNED(src_y, 4) && IS_ALIGNED(src_stride_y, 4) && + IS_ALIGNED(src_u, 2) && IS_ALIGNED(src_stride_u, 2) && + IS_ALIGNED(src_v, 2) && IS_ALIGNED(src_stride_v, 2) && + IS_ALIGNED(dst_argb, 4) && IS_ALIGNED(dst_stride_argb, 4)) { + J422ToARGBRow = J422ToARGBRow_MIPS_DSPR2; + } +#endif + + for (y = 0; y < height; ++y) { + J422ToARGBRow(src_y, src_u, src_v, dst_argb, width); + dst_argb += dst_stride_argb; + src_y += src_stride_y; + src_u += src_stride_u; + src_v += src_stride_v; + } + return 0; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/convert_from.cc b/third_party/aom/third_party/libyuv/source/convert_from.cc new file mode 100644 index 0000000000..31f1ac992a --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/convert_from.cc @@ -0,0 +1,1348 @@ +/* + * Copyright 2012 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/convert_from.h" + +#include "libyuv/basic_types.h" +#include "libyuv/convert.h" // For I420Copy +#include "libyuv/cpu_id.h" +#include "libyuv/planar_functions.h" +#include "libyuv/rotate.h" +#include "libyuv/scale.h" // For ScalePlane() +#include "libyuv/video_common.h" +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#define SUBSAMPLE(v, a, s) (v < 0) ? (-((-v + a) >> s)) : ((v + a) >> s) +static __inline int Abs(int v) { + return v >= 0 ? v : -v; +} + +// I420 To any I4xx YUV format with mirroring. +static int I420ToI4xx(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int src_y_width, int src_y_height, + int dst_uv_width, int dst_uv_height) { + const int dst_y_width = Abs(src_y_width); + const int dst_y_height = Abs(src_y_height); + const int src_uv_width = SUBSAMPLE(src_y_width, 1, 1); + const int src_uv_height = SUBSAMPLE(src_y_height, 1, 1); + if (src_y_width == 0 || src_y_height == 0 || + dst_uv_width <= 0 || dst_uv_height <= 0) { + return -1; + } + ScalePlane(src_y, src_stride_y, src_y_width, src_y_height, + dst_y, dst_stride_y, dst_y_width, dst_y_height, + kFilterBilinear); + ScalePlane(src_u, src_stride_u, src_uv_width, src_uv_height, + dst_u, dst_stride_u, dst_uv_width, dst_uv_height, + kFilterBilinear); + ScalePlane(src_v, src_stride_v, src_uv_width, src_uv_height, + dst_v, dst_stride_v, dst_uv_width, dst_uv_height, + kFilterBilinear); + return 0; +} + +// 420 chroma is 1/2 width, 1/2 height +// 422 chroma is 1/2 width, 1x height +LIBYUV_API +int I420ToI422(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + const int dst_uv_width = (Abs(width) + 1) >> 1; + const int dst_uv_height = Abs(height); + return I420ToI4xx(src_y, src_stride_y, + src_u, src_stride_u, + src_v, src_stride_v, + dst_y, dst_stride_y, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + width, height, + dst_uv_width, dst_uv_height); +} + +// 420 chroma is 1/2 width, 1/2 height +// 444 chroma is 1x width, 1x height +LIBYUV_API +int I420ToI444(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + const int dst_uv_width = Abs(width); + const int dst_uv_height = Abs(height); + return I420ToI4xx(src_y, src_stride_y, + src_u, src_stride_u, + src_v, src_stride_v, + dst_y, dst_stride_y, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + width, height, + dst_uv_width, dst_uv_height); +} + +// 420 chroma is 1/2 width, 1/2 height +// 411 chroma is 1/4 width, 1x height +LIBYUV_API +int I420ToI411(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + const int dst_uv_width = (Abs(width) + 3) >> 2; + const int dst_uv_height = Abs(height); + return I420ToI4xx(src_y, src_stride_y, + src_u, src_stride_u, + src_v, src_stride_v, + dst_y, dst_stride_y, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + width, height, + dst_uv_width, dst_uv_height); +} + +// Copy to I400. Source can be I420,422,444,400,NV12,NV21 +LIBYUV_API +int I400Copy(const uint8* src_y, int src_stride_y, + uint8* dst_y, int dst_stride_y, + int width, int height) { + if (!src_y || !dst_y || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_y = src_y + (height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } + CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height); + return 0; +} + +LIBYUV_API +int I422ToYUY2(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_yuy2, int dst_stride_yuy2, + int width, int height) { + int y; + void (*I422ToYUY2Row)(const uint8* src_y, const uint8* src_u, + const uint8* src_v, uint8* dst_yuy2, int width) = + I422ToYUY2Row_C; + if (!src_y || !src_u || !src_v || !dst_yuy2 || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_yuy2 = dst_yuy2 + (height - 1) * dst_stride_yuy2; + dst_stride_yuy2 = -dst_stride_yuy2; + } + // Coalesce rows. + if (src_stride_y == width && + src_stride_u * 2 == width && + src_stride_v * 2 == width && + dst_stride_yuy2 == width * 2) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_yuy2 = 0; + } +#if defined(HAS_I422TOYUY2ROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + I422ToYUY2Row = I422ToYUY2Row_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + I422ToYUY2Row = I422ToYUY2Row_SSE2; + } + } +#endif +#if defined(HAS_I422TOYUY2ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToYUY2Row = I422ToYUY2Row_Any_NEON; + if (IS_ALIGNED(width, 16)) { + I422ToYUY2Row = I422ToYUY2Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + I422ToYUY2Row(src_y, src_u, src_v, dst_yuy2, width); + src_y += src_stride_y; + src_u += src_stride_u; + src_v += src_stride_v; + dst_yuy2 += dst_stride_yuy2; + } + return 0; +} + +LIBYUV_API +int I420ToYUY2(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_yuy2, int dst_stride_yuy2, + int width, int height) { + int y; + void (*I422ToYUY2Row)(const uint8* src_y, const uint8* src_u, + const uint8* src_v, uint8* dst_yuy2, int width) = + I422ToYUY2Row_C; + if (!src_y || !src_u || !src_v || !dst_yuy2 || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_yuy2 = dst_yuy2 + (height - 1) * dst_stride_yuy2; + dst_stride_yuy2 = -dst_stride_yuy2; + } +#if defined(HAS_I422TOYUY2ROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + I422ToYUY2Row = I422ToYUY2Row_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + I422ToYUY2Row = I422ToYUY2Row_SSE2; + } + } +#endif +#if defined(HAS_I422TOYUY2ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToYUY2Row = I422ToYUY2Row_Any_NEON; + if (IS_ALIGNED(width, 16)) { + I422ToYUY2Row = I422ToYUY2Row_NEON; + } + } +#endif + + for (y = 0; y < height - 1; y += 2) { + I422ToYUY2Row(src_y, src_u, src_v, dst_yuy2, width); + I422ToYUY2Row(src_y + src_stride_y, src_u, src_v, + dst_yuy2 + dst_stride_yuy2, width); + src_y += src_stride_y * 2; + src_u += src_stride_u; + src_v += src_stride_v; + dst_yuy2 += dst_stride_yuy2 * 2; + } + if (height & 1) { + I422ToYUY2Row(src_y, src_u, src_v, dst_yuy2, width); + } + return 0; +} + +LIBYUV_API +int I422ToUYVY(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_uyvy, int dst_stride_uyvy, + int width, int height) { + int y; + void (*I422ToUYVYRow)(const uint8* src_y, const uint8* src_u, + const uint8* src_v, uint8* dst_uyvy, int width) = + I422ToUYVYRow_C; + if (!src_y || !src_u || !src_v || !dst_uyvy || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_uyvy = dst_uyvy + (height - 1) * dst_stride_uyvy; + dst_stride_uyvy = -dst_stride_uyvy; + } + // Coalesce rows. + if (src_stride_y == width && + src_stride_u * 2 == width && + src_stride_v * 2 == width && + dst_stride_uyvy == width * 2) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_uyvy = 0; + } +#if defined(HAS_I422TOUYVYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + I422ToUYVYRow = I422ToUYVYRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + I422ToUYVYRow = I422ToUYVYRow_SSE2; + } + } +#endif +#if defined(HAS_I422TOUYVYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToUYVYRow = I422ToUYVYRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + I422ToUYVYRow = I422ToUYVYRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + I422ToUYVYRow(src_y, src_u, src_v, dst_uyvy, width); + src_y += src_stride_y; + src_u += src_stride_u; + src_v += src_stride_v; + dst_uyvy += dst_stride_uyvy; + } + return 0; +} + +LIBYUV_API +int I420ToUYVY(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_uyvy, int dst_stride_uyvy, + int width, int height) { + int y; + void (*I422ToUYVYRow)(const uint8* src_y, const uint8* src_u, + const uint8* src_v, uint8* dst_uyvy, int width) = + I422ToUYVYRow_C; + if (!src_y || !src_u || !src_v || !dst_uyvy || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_uyvy = dst_uyvy + (height - 1) * dst_stride_uyvy; + dst_stride_uyvy = -dst_stride_uyvy; + } +#if defined(HAS_I422TOUYVYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + I422ToUYVYRow = I422ToUYVYRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + I422ToUYVYRow = I422ToUYVYRow_SSE2; + } + } +#endif +#if defined(HAS_I422TOUYVYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToUYVYRow = I422ToUYVYRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + I422ToUYVYRow = I422ToUYVYRow_NEON; + } + } +#endif + + for (y = 0; y < height - 1; y += 2) { + I422ToUYVYRow(src_y, src_u, src_v, dst_uyvy, width); + I422ToUYVYRow(src_y + src_stride_y, src_u, src_v, + dst_uyvy + dst_stride_uyvy, width); + src_y += src_stride_y * 2; + src_u += src_stride_u; + src_v += src_stride_v; + dst_uyvy += dst_stride_uyvy * 2; + } + if (height & 1) { + I422ToUYVYRow(src_y, src_u, src_v, dst_uyvy, width); + } + return 0; +} + +LIBYUV_API +int I420ToNV12(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_uv, int dst_stride_uv, + int width, int height) { + int y; + void (*MergeUVRow_)(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width) = MergeUVRow_C; + // Coalesce rows. + int halfwidth = (width + 1) >> 1; + int halfheight = (height + 1) >> 1; + if (!src_y || !src_u || !src_v || !dst_y || !dst_uv || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + halfheight = (height + 1) >> 1; + dst_y = dst_y + (height - 1) * dst_stride_y; + dst_uv = dst_uv + (halfheight - 1) * dst_stride_uv; + dst_stride_y = -dst_stride_y; + dst_stride_uv = -dst_stride_uv; + } + if (src_stride_y == width && + dst_stride_y == width) { + width *= height; + height = 1; + src_stride_y = dst_stride_y = 0; + } + // Coalesce rows. + if (src_stride_u == halfwidth && + src_stride_v == halfwidth && + dst_stride_uv == halfwidth * 2) { + halfwidth *= halfheight; + halfheight = 1; + src_stride_u = src_stride_v = dst_stride_uv = 0; + } +#if defined(HAS_MERGEUVROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + MergeUVRow_ = MergeUVRow_Any_SSE2; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow_ = MergeUVRow_SSE2; + } + } +#endif +#if defined(HAS_MERGEUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + MergeUVRow_ = MergeUVRow_Any_AVX2; + if (IS_ALIGNED(halfwidth, 32)) { + MergeUVRow_ = MergeUVRow_AVX2; + } + } +#endif +#if defined(HAS_MERGEUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + MergeUVRow_ = MergeUVRow_Any_NEON; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow_ = MergeUVRow_NEON; + } + } +#endif + + CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height); + for (y = 0; y < halfheight; ++y) { + // Merge a row of U and V into a row of UV. + MergeUVRow_(src_u, src_v, dst_uv, halfwidth); + src_u += src_stride_u; + src_v += src_stride_v; + dst_uv += dst_stride_uv; + } + return 0; +} + +LIBYUV_API +int I420ToNV21(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_vu, int dst_stride_vu, + int width, int height) { + return I420ToNV12(src_y, src_stride_y, + src_v, src_stride_v, + src_u, src_stride_u, + dst_y, src_stride_y, + dst_vu, dst_stride_vu, + width, height); +} + +// Convert I420 to ARGB. +LIBYUV_API +int I420ToARGB(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*I422ToARGBRow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToARGBRow_C; + if (!src_y || !src_u || !src_v || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } +#if defined(HAS_I422TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToARGBRow = I422ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToARGBRow = I422ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_I422TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToARGBRow = I422ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToARGBRow = I422ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_I422TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToARGBRow = I422ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToARGBRow = I422ToARGBRow_NEON; + } + } +#endif +#if defined(HAS_I422TOARGBROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(width, 4) && + IS_ALIGNED(src_y, 4) && IS_ALIGNED(src_stride_y, 4) && + IS_ALIGNED(src_u, 2) && IS_ALIGNED(src_stride_u, 2) && + IS_ALIGNED(src_v, 2) && IS_ALIGNED(src_stride_v, 2) && + IS_ALIGNED(dst_argb, 4) && IS_ALIGNED(dst_stride_argb, 4)) { + I422ToARGBRow = I422ToARGBRow_MIPS_DSPR2; + } +#endif + + for (y = 0; y < height; ++y) { + I422ToARGBRow(src_y, src_u, src_v, dst_argb, width); + dst_argb += dst_stride_argb; + src_y += src_stride_y; + if (y & 1) { + src_u += src_stride_u; + src_v += src_stride_v; + } + } + return 0; +} + +// Convert I420 to BGRA. +LIBYUV_API +int I420ToBGRA(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_bgra, int dst_stride_bgra, + int width, int height) { + int y; + void (*I422ToBGRARow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToBGRARow_C; + if (!src_y || !src_u || !src_v || !dst_bgra || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_bgra = dst_bgra + (height - 1) * dst_stride_bgra; + dst_stride_bgra = -dst_stride_bgra; + } +#if defined(HAS_I422TOBGRAROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToBGRARow = I422ToBGRARow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToBGRARow = I422ToBGRARow_SSSE3; + } + } +#endif +#if defined(HAS_I422TOBGRAROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToBGRARow = I422ToBGRARow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToBGRARow = I422ToBGRARow_AVX2; + } + } +#endif +#if defined(HAS_I422TOBGRAROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToBGRARow = I422ToBGRARow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToBGRARow = I422ToBGRARow_NEON; + } + } +#endif +#if defined(HAS_I422TOBGRAROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(width, 4) && + IS_ALIGNED(src_y, 4) && IS_ALIGNED(src_stride_y, 4) && + IS_ALIGNED(src_u, 2) && IS_ALIGNED(src_stride_u, 2) && + IS_ALIGNED(src_v, 2) && IS_ALIGNED(src_stride_v, 2) && + IS_ALIGNED(dst_bgra, 4) && IS_ALIGNED(dst_stride_bgra, 4)) { + I422ToBGRARow = I422ToBGRARow_MIPS_DSPR2; + } +#endif + + for (y = 0; y < height; ++y) { + I422ToBGRARow(src_y, src_u, src_v, dst_bgra, width); + dst_bgra += dst_stride_bgra; + src_y += src_stride_y; + if (y & 1) { + src_u += src_stride_u; + src_v += src_stride_v; + } + } + return 0; +} + +// Convert I420 to ABGR. +LIBYUV_API +int I420ToABGR(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_abgr, int dst_stride_abgr, + int width, int height) { + int y; + void (*I422ToABGRRow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToABGRRow_C; + if (!src_y || !src_u || !src_v || !dst_abgr || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_abgr = dst_abgr + (height - 1) * dst_stride_abgr; + dst_stride_abgr = -dst_stride_abgr; + } +#if defined(HAS_I422TOABGRROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToABGRRow = I422ToABGRRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToABGRRow = I422ToABGRRow_SSSE3; + } + } +#endif +#if defined(HAS_I422TOABGRROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToABGRRow = I422ToABGRRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToABGRRow = I422ToABGRRow_AVX2; + } + } +#endif +#if defined(HAS_I422TOABGRROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToABGRRow = I422ToABGRRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToABGRRow = I422ToABGRRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + I422ToABGRRow(src_y, src_u, src_v, dst_abgr, width); + dst_abgr += dst_stride_abgr; + src_y += src_stride_y; + if (y & 1) { + src_u += src_stride_u; + src_v += src_stride_v; + } + } + return 0; +} + +// Convert I420 to RGBA. +LIBYUV_API +int I420ToRGBA(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_rgba, int dst_stride_rgba, + int width, int height) { + int y; + void (*I422ToRGBARow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToRGBARow_C; + if (!src_y || !src_u || !src_v || !dst_rgba || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_rgba = dst_rgba + (height - 1) * dst_stride_rgba; + dst_stride_rgba = -dst_stride_rgba; + } +#if defined(HAS_I422TORGBAROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToRGBARow = I422ToRGBARow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToRGBARow = I422ToRGBARow_SSSE3; + } + } +#endif +#if defined(HAS_I422TORGBAROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToRGBARow = I422ToRGBARow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToRGBARow = I422ToRGBARow_AVX2; + } + } +#endif +#if defined(HAS_I422TORGBAROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToRGBARow = I422ToRGBARow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToRGBARow = I422ToRGBARow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + I422ToRGBARow(src_y, src_u, src_v, dst_rgba, width); + dst_rgba += dst_stride_rgba; + src_y += src_stride_y; + if (y & 1) { + src_u += src_stride_u; + src_v += src_stride_v; + } + } + return 0; +} + +// Convert I420 to RGB24. +LIBYUV_API +int I420ToRGB24(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_rgb24, int dst_stride_rgb24, + int width, int height) { + int y; + void (*I422ToRGB24Row)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToRGB24Row_C; + if (!src_y || !src_u || !src_v || !dst_rgb24 || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_rgb24 = dst_rgb24 + (height - 1) * dst_stride_rgb24; + dst_stride_rgb24 = -dst_stride_rgb24; + } +#if defined(HAS_I422TORGB24ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToRGB24Row = I422ToRGB24Row_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToRGB24Row = I422ToRGB24Row_SSSE3; + } + } +#endif +#if defined(HAS_I422TORGB24ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToRGB24Row = I422ToRGB24Row_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToRGB24Row = I422ToRGB24Row_AVX2; + } + } +#endif +#if defined(HAS_I422TORGB24ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToRGB24Row = I422ToRGB24Row_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToRGB24Row = I422ToRGB24Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + I422ToRGB24Row(src_y, src_u, src_v, dst_rgb24, width); + dst_rgb24 += dst_stride_rgb24; + src_y += src_stride_y; + if (y & 1) { + src_u += src_stride_u; + src_v += src_stride_v; + } + } + return 0; +} + +// Convert I420 to RAW. +LIBYUV_API +int I420ToRAW(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_raw, int dst_stride_raw, + int width, int height) { + int y; + void (*I422ToRAWRow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToRAWRow_C; + if (!src_y || !src_u || !src_v || !dst_raw || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_raw = dst_raw + (height - 1) * dst_stride_raw; + dst_stride_raw = -dst_stride_raw; + } +#if defined(HAS_I422TORAWROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToRAWRow = I422ToRAWRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToRAWRow = I422ToRAWRow_SSSE3; + } + } +#endif +#if defined(HAS_I422TORAWROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToRAWRow = I422ToRAWRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToRAWRow = I422ToRAWRow_AVX2; + } + } +#endif +#if defined(HAS_I422TORAWROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToRAWRow = I422ToRAWRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToRAWRow = I422ToRAWRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + I422ToRAWRow(src_y, src_u, src_v, dst_raw, width); + dst_raw += dst_stride_raw; + src_y += src_stride_y; + if (y & 1) { + src_u += src_stride_u; + src_v += src_stride_v; + } + } + return 0; +} + +// Convert I420 to ARGB1555. +LIBYUV_API +int I420ToARGB1555(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb1555, int dst_stride_argb1555, + int width, int height) { + int y; + void (*I422ToARGB1555Row)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToARGB1555Row_C; + if (!src_y || !src_u || !src_v || !dst_argb1555 || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb1555 = dst_argb1555 + (height - 1) * dst_stride_argb1555; + dst_stride_argb1555 = -dst_stride_argb1555; + } +#if defined(HAS_I422TOARGB1555ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToARGB1555Row = I422ToARGB1555Row_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToARGB1555Row = I422ToARGB1555Row_SSSE3; + } + } +#endif +#if defined(HAS_I422TOARGB1555ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToARGB1555Row = I422ToARGB1555Row_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToARGB1555Row = I422ToARGB1555Row_AVX2; + } + } +#endif +#if defined(HAS_I422TOARGB1555ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToARGB1555Row = I422ToARGB1555Row_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToARGB1555Row = I422ToARGB1555Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + I422ToARGB1555Row(src_y, src_u, src_v, dst_argb1555, width); + dst_argb1555 += dst_stride_argb1555; + src_y += src_stride_y; + if (y & 1) { + src_u += src_stride_u; + src_v += src_stride_v; + } + } + return 0; +} + + +// Convert I420 to ARGB4444. +LIBYUV_API +int I420ToARGB4444(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_argb4444, int dst_stride_argb4444, + int width, int height) { + int y; + void (*I422ToARGB4444Row)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToARGB4444Row_C; + if (!src_y || !src_u || !src_v || !dst_argb4444 || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb4444 = dst_argb4444 + (height - 1) * dst_stride_argb4444; + dst_stride_argb4444 = -dst_stride_argb4444; + } +#if defined(HAS_I422TOARGB4444ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToARGB4444Row = I422ToARGB4444Row_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToARGB4444Row = I422ToARGB4444Row_SSSE3; + } + } +#endif +#if defined(HAS_I422TOARGB4444ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToARGB4444Row = I422ToARGB4444Row_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToARGB4444Row = I422ToARGB4444Row_AVX2; + } + } +#endif +#if defined(HAS_I422TOARGB4444ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToARGB4444Row = I422ToARGB4444Row_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToARGB4444Row = I422ToARGB4444Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + I422ToARGB4444Row(src_y, src_u, src_v, dst_argb4444, width); + dst_argb4444 += dst_stride_argb4444; + src_y += src_stride_y; + if (y & 1) { + src_u += src_stride_u; + src_v += src_stride_v; + } + } + return 0; +} + +// Convert I420 to RGB565. +LIBYUV_API +int I420ToRGB565(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_rgb565, int dst_stride_rgb565, + int width, int height) { + int y; + void (*I422ToRGB565Row)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToRGB565Row_C; + if (!src_y || !src_u || !src_v || !dst_rgb565 || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_rgb565 = dst_rgb565 + (height - 1) * dst_stride_rgb565; + dst_stride_rgb565 = -dst_stride_rgb565; + } +#if defined(HAS_I422TORGB565ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToRGB565Row = I422ToRGB565Row_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToRGB565Row = I422ToRGB565Row_SSSE3; + } + } +#endif +#if defined(HAS_I422TORGB565ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToRGB565Row = I422ToRGB565Row_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToRGB565Row = I422ToRGB565Row_AVX2; + } + } +#endif +#if defined(HAS_I422TORGB565ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToRGB565Row = I422ToRGB565Row_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToRGB565Row = I422ToRGB565Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + I422ToRGB565Row(src_y, src_u, src_v, dst_rgb565, width); + dst_rgb565 += dst_stride_rgb565; + src_y += src_stride_y; + if (y & 1) { + src_u += src_stride_u; + src_v += src_stride_v; + } + } + return 0; +} + +// Ordered 8x8 dither for 888 to 565. Values from 0 to 7. +static const uint8 kDither565_4x4[16] = { + 0, 4, 1, 5, + 6, 2, 7, 3, + 1, 5, 0, 4, + 7, 3, 6, 2, +}; + +// Convert I420 to RGB565 with dithering. +LIBYUV_API +int I420ToRGB565Dither(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_rgb565, int dst_stride_rgb565, + const uint8* dither4x4, int width, int height) { + int y; + void (*I422ToARGBRow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToARGBRow_C; + void (*ARGBToRGB565DitherRow)(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int pix) = ARGBToRGB565DitherRow_C; + if (!src_y || !src_u || !src_v || !dst_rgb565 || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_rgb565 = dst_rgb565 + (height - 1) * dst_stride_rgb565; + dst_stride_rgb565 = -dst_stride_rgb565; + } + if (!dither4x4) { + dither4x4 = kDither565_4x4; + } +#if defined(HAS_I422TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToARGBRow = I422ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToARGBRow = I422ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_I422TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToARGBRow = I422ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToARGBRow = I422ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_I422TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToARGBRow = I422ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToARGBRow = I422ToARGBRow_NEON; + } + } +#endif +#if defined(HAS_I422TOARGBROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(width, 4) && + IS_ALIGNED(src_y, 4) && IS_ALIGNED(src_stride_y, 4) && + IS_ALIGNED(src_u, 2) && IS_ALIGNED(src_stride_u, 2) && + IS_ALIGNED(src_v, 2) && IS_ALIGNED(src_stride_v, 2)) { + I422ToARGBRow = I422ToARGBRow_MIPS_DSPR2; + } +#endif +#if defined(HAS_ARGBTORGB565DITHERROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SSE2; + } + } +#endif +#if defined(HAS_ARGBTORGB565DITHERROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTORGB565DITHERROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_NEON; + } + } +#endif + { + // Allocate a row of argb. + align_buffer_64(row_argb, width * 4); + for (y = 0; y < height; ++y) { + I422ToARGBRow(src_y, src_u, src_v, row_argb, width); + ARGBToRGB565DitherRow(row_argb, dst_rgb565, + *(uint32*)(dither4x4 + ((y & 3) << 2)), width); + dst_rgb565 += dst_stride_rgb565; + src_y += src_stride_y; + if (y & 1) { + src_u += src_stride_u; + src_v += src_stride_v; + } + } + free_aligned_buffer_64(row_argb); + } + return 0; +} + +// Convert I420 to specified format +LIBYUV_API +int ConvertFromI420(const uint8* y, int y_stride, + const uint8* u, int u_stride, + const uint8* v, int v_stride, + uint8* dst_sample, int dst_sample_stride, + int width, int height, + uint32 fourcc) { + uint32 format = CanonicalFourCC(fourcc); + int r = 0; + if (!y || !u|| !v || !dst_sample || + width <= 0 || height == 0) { + return -1; + } + switch (format) { + // Single plane formats + case FOURCC_YUY2: + r = I420ToYUY2(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width * 2, + width, height); + break; + case FOURCC_UYVY: + r = I420ToUYVY(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width * 2, + width, height); + break; + case FOURCC_RGBP: + r = I420ToRGB565(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width * 2, + width, height); + break; + case FOURCC_RGBO: + r = I420ToARGB1555(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width * 2, + width, height); + break; + case FOURCC_R444: + r = I420ToARGB4444(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width * 2, + width, height); + break; + case FOURCC_24BG: + r = I420ToRGB24(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width * 3, + width, height); + break; + case FOURCC_RAW: + r = I420ToRAW(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width * 3, + width, height); + break; + case FOURCC_ARGB: + r = I420ToARGB(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width * 4, + width, height); + break; + case FOURCC_BGRA: + r = I420ToBGRA(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width * 4, + width, height); + break; + case FOURCC_ABGR: + r = I420ToABGR(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width * 4, + width, height); + break; + case FOURCC_RGBA: + r = I420ToRGBA(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width * 4, + width, height); + break; + case FOURCC_I400: + r = I400Copy(y, y_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width, + width, height); + break; + case FOURCC_NV12: { + uint8* dst_uv = dst_sample + width * height; + r = I420ToNV12(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width, + dst_uv, + dst_sample_stride ? dst_sample_stride : width, + width, height); + break; + } + case FOURCC_NV21: { + uint8* dst_vu = dst_sample + width * height; + r = I420ToNV21(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, + dst_sample_stride ? dst_sample_stride : width, + dst_vu, + dst_sample_stride ? dst_sample_stride : width, + width, height); + break; + } + // TODO(fbarchard): Add M420. + // Triplanar formats + // TODO(fbarchard): halfstride instead of halfwidth + case FOURCC_I420: + case FOURCC_YU12: + case FOURCC_YV12: { + int halfwidth = (width + 1) / 2; + int halfheight = (height + 1) / 2; + uint8* dst_u; + uint8* dst_v; + if (format == FOURCC_YV12) { + dst_v = dst_sample + width * height; + dst_u = dst_v + halfwidth * halfheight; + } else { + dst_u = dst_sample + width * height; + dst_v = dst_u + halfwidth * halfheight; + } + r = I420Copy(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, width, + dst_u, halfwidth, + dst_v, halfwidth, + width, height); + break; + } + case FOURCC_I422: + case FOURCC_YV16: { + int halfwidth = (width + 1) / 2; + uint8* dst_u; + uint8* dst_v; + if (format == FOURCC_YV16) { + dst_v = dst_sample + width * height; + dst_u = dst_v + halfwidth * height; + } else { + dst_u = dst_sample + width * height; + dst_v = dst_u + halfwidth * height; + } + r = I420ToI422(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, width, + dst_u, halfwidth, + dst_v, halfwidth, + width, height); + break; + } + case FOURCC_I444: + case FOURCC_YV24: { + uint8* dst_u; + uint8* dst_v; + if (format == FOURCC_YV24) { + dst_v = dst_sample + width * height; + dst_u = dst_v + width * height; + } else { + dst_u = dst_sample + width * height; + dst_v = dst_u + width * height; + } + r = I420ToI444(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, width, + dst_u, width, + dst_v, width, + width, height); + break; + } + case FOURCC_I411: { + int quarterwidth = (width + 3) / 4; + uint8* dst_u = dst_sample + width * height; + uint8* dst_v = dst_u + quarterwidth * height; + r = I420ToI411(y, y_stride, + u, u_stride, + v, v_stride, + dst_sample, width, + dst_u, quarterwidth, + dst_v, quarterwidth, + width, height); + break; + } + + // Formats not supported - MJPG, biplanar, some rgb formats. + default: + return -1; // unknown fourcc - return failure code. + } + return r; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/convert_from_argb.cc b/third_party/aom/third_party/libyuv/source/convert_from_argb.cc new file mode 100644 index 0000000000..8d1e97aec2 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/convert_from_argb.cc @@ -0,0 +1,1301 @@ +/* + * Copyright 2012 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/convert_from_argb.h" + +#include "libyuv/basic_types.h" +#include "libyuv/cpu_id.h" +#include "libyuv/planar_functions.h" +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// ARGB little endian (bgra in memory) to I444 +LIBYUV_API +int ARGBToI444(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; + void (*ARGBToUV444Row)(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix) = ARGBToUV444Row_C; + if (!src_argb || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_y == width && + dst_stride_u == width && + dst_stride_v == width) { + width *= height; + height = 1; + src_stride_argb = dst_stride_y = dst_stride_u = dst_stride_v = 0; + } +#if defined(HAS_ARGBTOUV444ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUV444Row = ARGBToUV444Row_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUV444Row = ARGBToUV444Row_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOUV444ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToUV444Row = ARGBToUV444Row_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToUV444Row = ARGBToUV444Row_NEON; + } + } +#endif +#if defined(HAS_ARGBTOYROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYRow = ARGBToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYRow = ARGBToYRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBToUV444Row(src_argb, dst_u, dst_v, width); + ARGBToYRow(src_argb, dst_y, width); + src_argb += src_stride_argb; + dst_y += dst_stride_y; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + return 0; +} + +// ARGB little endian (bgra in memory) to I422 +LIBYUV_API +int ARGBToI422(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*ARGBToUV422Row)(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix) = ARGBToUV422Row_C; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; + if (!src_argb || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_y == width && + dst_stride_u * 2 == width && + dst_stride_v * 2 == width) { + width *= height; + height = 1; + src_stride_argb = dst_stride_y = dst_stride_u = dst_stride_v = 0; + } +#if defined(HAS_ARGBTOUV422ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUV422Row = ARGBToUV422Row_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUV422Row = ARGBToUV422Row_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOUV422ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToUV422Row = ARGBToUV422Row_Any_NEON; + if (IS_ALIGNED(width, 16)) { + ARGBToUV422Row = ARGBToUV422Row_NEON; + } + } +#endif +#if defined(HAS_ARGBTOYROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYRow = ARGBToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYRow = ARGBToYRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBToUV422Row(src_argb, dst_u, dst_v, width); + ARGBToYRow(src_argb, dst_y, width); + src_argb += src_stride_argb; + dst_y += dst_stride_y; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + return 0; +} + +// ARGB little endian (bgra in memory) to I411 +LIBYUV_API +int ARGBToI411(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*ARGBToUV411Row)(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix) = ARGBToUV411Row_C; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; + if (!src_argb || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_y == width && + dst_stride_u * 4 == width && + dst_stride_v * 4 == width) { + width *= height; + height = 1; + src_stride_argb = dst_stride_y = dst_stride_u = dst_stride_v = 0; + } +#if defined(HAS_ARGBTOYROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYRow = ARGBToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYRow = ARGBToYRow_NEON; + } + } +#endif +#if defined(HAS_ARGBTOUV411ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToUV411Row = ARGBToUV411Row_Any_NEON; + if (IS_ALIGNED(width, 32)) { + ARGBToUV411Row = ARGBToUV411Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBToUV411Row(src_argb, dst_u, dst_v, width); + ARGBToYRow(src_argb, dst_y, width); + src_argb += src_stride_argb; + dst_y += dst_stride_y; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + return 0; +} + +LIBYUV_API +int ARGBToNV12(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_uv, int dst_stride_uv, + int width, int height) { + int y; + int halfwidth = (width + 1) >> 1; + void (*ARGBToUVRow)(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) = ARGBToUVRow_C; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; + void (*MergeUVRow_)(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width) = MergeUVRow_C; + if (!src_argb || + !dst_y || !dst_uv || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } +#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVRow = ARGBToUVRow_Any_SSSE3; + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUVRow = ARGBToUVRow_SSSE3; + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToUVRow = ARGBToUVRow_Any_AVX2; + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToUVRow = ARGBToUVRow_AVX2; + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYRow = ARGBToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYRow = ARGBToYRow_NEON; + } + } +#endif +#if defined(HAS_ARGBTOUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToUVRow = ARGBToUVRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + ARGBToUVRow = ARGBToUVRow_NEON; + } + } +#endif +#if defined(HAS_MERGEUVROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + MergeUVRow_ = MergeUVRow_Any_SSE2; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow_ = MergeUVRow_SSE2; + } + } +#endif +#if defined(HAS_MERGEUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + MergeUVRow_ = MergeUVRow_Any_AVX2; + if (IS_ALIGNED(halfwidth, 32)) { + MergeUVRow_ = MergeUVRow_AVX2; + } + } +#endif +#if defined(HAS_MERGEUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + MergeUVRow_ = MergeUVRow_Any_NEON; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow_ = MergeUVRow_NEON; + } + } +#endif + { + // Allocate a rows of uv. + align_buffer_64(row_u, ((halfwidth + 31) & ~31) * 2); + uint8* row_v = row_u + ((halfwidth + 31) & ~31); + + for (y = 0; y < height - 1; y += 2) { + ARGBToUVRow(src_argb, src_stride_argb, row_u, row_v, width); + MergeUVRow_(row_u, row_v, dst_uv, halfwidth); + ARGBToYRow(src_argb, dst_y, width); + ARGBToYRow(src_argb + src_stride_argb, dst_y + dst_stride_y, width); + src_argb += src_stride_argb * 2; + dst_y += dst_stride_y * 2; + dst_uv += dst_stride_uv; + } + if (height & 1) { + ARGBToUVRow(src_argb, 0, row_u, row_v, width); + MergeUVRow_(row_u, row_v, dst_uv, halfwidth); + ARGBToYRow(src_argb, dst_y, width); + } + free_aligned_buffer_64(row_u); + } + return 0; +} + +// Same as NV12 but U and V swapped. +LIBYUV_API +int ARGBToNV21(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_uv, int dst_stride_uv, + int width, int height) { + int y; + int halfwidth = (width + 1) >> 1; + void (*ARGBToUVRow)(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) = ARGBToUVRow_C; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; + void (*MergeUVRow_)(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width) = MergeUVRow_C; + if (!src_argb || + !dst_y || !dst_uv || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } +#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVRow = ARGBToUVRow_Any_SSSE3; + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUVRow = ARGBToUVRow_SSSE3; + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToUVRow = ARGBToUVRow_Any_AVX2; + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToUVRow = ARGBToUVRow_AVX2; + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYRow = ARGBToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYRow = ARGBToYRow_NEON; + } + } +#endif +#if defined(HAS_ARGBTOUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToUVRow = ARGBToUVRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + ARGBToUVRow = ARGBToUVRow_NEON; + } + } +#endif +#if defined(HAS_MERGEUVROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + MergeUVRow_ = MergeUVRow_Any_SSE2; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow_ = MergeUVRow_SSE2; + } + } +#endif +#if defined(HAS_MERGEUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + MergeUVRow_ = MergeUVRow_Any_AVX2; + if (IS_ALIGNED(halfwidth, 32)) { + MergeUVRow_ = MergeUVRow_AVX2; + } + } +#endif +#if defined(HAS_MERGEUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + MergeUVRow_ = MergeUVRow_Any_NEON; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow_ = MergeUVRow_NEON; + } + } +#endif + { + // Allocate a rows of uv. + align_buffer_64(row_u, ((halfwidth + 31) & ~31) * 2); + uint8* row_v = row_u + ((halfwidth + 31) & ~31); + + for (y = 0; y < height - 1; y += 2) { + ARGBToUVRow(src_argb, src_stride_argb, row_u, row_v, width); + MergeUVRow_(row_v, row_u, dst_uv, halfwidth); + ARGBToYRow(src_argb, dst_y, width); + ARGBToYRow(src_argb + src_stride_argb, dst_y + dst_stride_y, width); + src_argb += src_stride_argb * 2; + dst_y += dst_stride_y * 2; + dst_uv += dst_stride_uv; + } + if (height & 1) { + ARGBToUVRow(src_argb, 0, row_u, row_v, width); + MergeUVRow_(row_v, row_u, dst_uv, halfwidth); + ARGBToYRow(src_argb, dst_y, width); + } + free_aligned_buffer_64(row_u); + } + return 0; +} + +// Convert ARGB to YUY2. +LIBYUV_API +int ARGBToYUY2(const uint8* src_argb, int src_stride_argb, + uint8* dst_yuy2, int dst_stride_yuy2, + int width, int height) { + int y; + void (*ARGBToUV422Row)(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix) = ARGBToUV422Row_C; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; + void (*I422ToYUY2Row)(const uint8* src_y, const uint8* src_u, + const uint8* src_v, uint8* dst_yuy2, int width) = I422ToYUY2Row_C; + + if (!src_argb || !dst_yuy2 || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_yuy2 = dst_yuy2 + (height - 1) * dst_stride_yuy2; + dst_stride_yuy2 = -dst_stride_yuy2; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_yuy2 == width * 2) { + width *= height; + height = 1; + src_stride_argb = dst_stride_yuy2 = 0; + } +#if defined(HAS_ARGBTOUV422ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUV422Row = ARGBToUV422Row_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUV422Row = ARGBToUV422Row_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOUV422ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToUV422Row = ARGBToUV422Row_Any_NEON; + if (IS_ALIGNED(width, 16)) { + ARGBToUV422Row = ARGBToUV422Row_NEON; + } + } +#endif +#if defined(HAS_ARGBTOYROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYRow = ARGBToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYRow = ARGBToYRow_NEON; + } + } +#endif + +#if defined(HAS_I422TOYUY2ROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + I422ToYUY2Row = I422ToYUY2Row_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + I422ToYUY2Row = I422ToYUY2Row_SSE2; + } + } +#endif +#if defined(HAS_I422TOYUY2ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToYUY2Row = I422ToYUY2Row_Any_NEON; + if (IS_ALIGNED(width, 16)) { + I422ToYUY2Row = I422ToYUY2Row_NEON; + } + } +#endif + + { + // Allocate a rows of yuv. + align_buffer_64(row_y, ((width + 63) & ~63) * 2); + uint8* row_u = row_y + ((width + 63) & ~63); + uint8* row_v = row_u + ((width + 63) & ~63) / 2; + + for (y = 0; y < height; ++y) { + ARGBToUV422Row(src_argb, row_u, row_v, width); + ARGBToYRow(src_argb, row_y, width); + I422ToYUY2Row(row_y, row_u, row_v, dst_yuy2, width); + src_argb += src_stride_argb; + dst_yuy2 += dst_stride_yuy2; + } + + free_aligned_buffer_64(row_y); + } + return 0; +} + +// Convert ARGB to UYVY. +LIBYUV_API +int ARGBToUYVY(const uint8* src_argb, int src_stride_argb, + uint8* dst_uyvy, int dst_stride_uyvy, + int width, int height) { + int y; + void (*ARGBToUV422Row)(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix) = ARGBToUV422Row_C; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; + void (*I422ToUYVYRow)(const uint8* src_y, const uint8* src_u, + const uint8* src_v, uint8* dst_uyvy, int width) = I422ToUYVYRow_C; + + if (!src_argb || !dst_uyvy || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_uyvy = dst_uyvy + (height - 1) * dst_stride_uyvy; + dst_stride_uyvy = -dst_stride_uyvy; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_uyvy == width * 2) { + width *= height; + height = 1; + src_stride_argb = dst_stride_uyvy = 0; + } +#if defined(HAS_ARGBTOUV422ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUV422Row = ARGBToUV422Row_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUV422Row = ARGBToUV422Row_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOUV422ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToUV422Row = ARGBToUV422Row_Any_NEON; + if (IS_ALIGNED(width, 16)) { + ARGBToUV422Row = ARGBToUV422Row_NEON; + } + } +#endif +#if defined(HAS_ARGBTOYROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYRow = ARGBToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYRow = ARGBToYRow_NEON; + } + } +#endif + +#if defined(HAS_I422TOUYVYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + I422ToUYVYRow = I422ToUYVYRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + I422ToUYVYRow = I422ToUYVYRow_SSE2; + } + } +#endif +#if defined(HAS_I422TOUYVYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToUYVYRow = I422ToUYVYRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + I422ToUYVYRow = I422ToUYVYRow_NEON; + } + } +#endif + + { + // Allocate a rows of yuv. + align_buffer_64(row_y, ((width + 63) & ~63) * 2); + uint8* row_u = row_y + ((width + 63) & ~63); + uint8* row_v = row_u + ((width + 63) & ~63) / 2; + + for (y = 0; y < height; ++y) { + ARGBToUV422Row(src_argb, row_u, row_v, width); + ARGBToYRow(src_argb, row_y, width); + I422ToUYVYRow(row_y, row_u, row_v, dst_uyvy, width); + src_argb += src_stride_argb; + dst_uyvy += dst_stride_uyvy; + } + + free_aligned_buffer_64(row_y); + } + return 0; +} + +// Convert ARGB to I400. +LIBYUV_API +int ARGBToI400(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + int width, int height) { + int y; + void (*ARGBToYRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYRow_C; + if (!src_argb || !dst_y || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_y == width) { + width *= height; + height = 1; + src_stride_argb = dst_stride_y = 0; + } +#if defined(HAS_ARGBTOYROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYRow = ARGBToYRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYRow = ARGBToYRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToYRow = ARGBToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToYRow = ARGBToYRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYRow = ARGBToYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYRow = ARGBToYRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBToYRow(src_argb, dst_y, width); + src_argb += src_stride_argb; + dst_y += dst_stride_y; + } + return 0; +} + +// Shuffle table for converting ARGB to RGBA. +static uvec8 kShuffleMaskARGBToRGBA = { + 3u, 0u, 1u, 2u, 7u, 4u, 5u, 6u, 11u, 8u, 9u, 10u, 15u, 12u, 13u, 14u +}; + +// Convert ARGB to RGBA. +LIBYUV_API +int ARGBToRGBA(const uint8* src_argb, int src_stride_argb, + uint8* dst_rgba, int dst_stride_rgba, + int width, int height) { + return ARGBShuffle(src_argb, src_stride_argb, + dst_rgba, dst_stride_rgba, + (const uint8*)(&kShuffleMaskARGBToRGBA), + width, height); +} + +// Convert ARGB To RGB24. +LIBYUV_API +int ARGBToRGB24(const uint8* src_argb, int src_stride_argb, + uint8* dst_rgb24, int dst_stride_rgb24, + int width, int height) { + int y; + void (*ARGBToRGB24Row)(const uint8* src_argb, uint8* dst_rgb, int pix) = + ARGBToRGB24Row_C; + if (!src_argb || !dst_rgb24 || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_rgb24 == width * 3) { + width *= height; + height = 1; + src_stride_argb = dst_stride_rgb24 = 0; + } +#if defined(HAS_ARGBTORGB24ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToRGB24Row = ARGBToRGB24Row_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToRGB24Row = ARGBToRGB24Row_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTORGB24ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToRGB24Row = ARGBToRGB24Row_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToRGB24Row = ARGBToRGB24Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBToRGB24Row(src_argb, dst_rgb24, width); + src_argb += src_stride_argb; + dst_rgb24 += dst_stride_rgb24; + } + return 0; +} + +// Convert ARGB To RAW. +LIBYUV_API +int ARGBToRAW(const uint8* src_argb, int src_stride_argb, + uint8* dst_raw, int dst_stride_raw, + int width, int height) { + int y; + void (*ARGBToRAWRow)(const uint8* src_argb, uint8* dst_rgb, int pix) = + ARGBToRAWRow_C; + if (!src_argb || !dst_raw || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_raw == width * 3) { + width *= height; + height = 1; + src_stride_argb = dst_stride_raw = 0; + } +#if defined(HAS_ARGBTORAWROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToRAWRow = ARGBToRAWRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToRAWRow = ARGBToRAWRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTORAWROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToRAWRow = ARGBToRAWRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToRAWRow = ARGBToRAWRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBToRAWRow(src_argb, dst_raw, width); + src_argb += src_stride_argb; + dst_raw += dst_stride_raw; + } + return 0; +} + +// Ordered 8x8 dither for 888 to 565. Values from 0 to 7. +static const uint8 kDither565_4x4[16] = { + 0, 4, 1, 5, + 6, 2, 7, 3, + 1, 5, 0, 4, + 7, 3, 6, 2, +}; + +// Convert ARGB To RGB565 with 4x4 dither matrix (16 bytes). +LIBYUV_API +int ARGBToRGB565Dither(const uint8* src_argb, int src_stride_argb, + uint8* dst_rgb565, int dst_stride_rgb565, + const uint8* dither4x4, int width, int height) { + int y; + void (*ARGBToRGB565DitherRow)(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int pix) = ARGBToRGB565DitherRow_C; + if (!src_argb || !dst_rgb565 || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + if (!dither4x4) { + dither4x4 = kDither565_4x4; + } +#if defined(HAS_ARGBTORGB565DITHERROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SSE2; + } + } +#endif +#if defined(HAS_ARGBTORGB565DITHERROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTORGB565DITHERROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_NEON; + } + } +#endif + for (y = 0; y < height; ++y) { + ARGBToRGB565DitherRow(src_argb, dst_rgb565, + *(uint32*)(dither4x4 + ((y & 3) << 2)), width); + src_argb += src_stride_argb; + dst_rgb565 += dst_stride_rgb565; + } + return 0; +} + +// Convert ARGB To RGB565. +// TODO(fbarchard): Consider using dither function low level with zeros. +LIBYUV_API +int ARGBToRGB565(const uint8* src_argb, int src_stride_argb, + uint8* dst_rgb565, int dst_stride_rgb565, + int width, int height) { + int y; + void (*ARGBToRGB565Row)(const uint8* src_argb, uint8* dst_rgb, int pix) = + ARGBToRGB565Row_C; + if (!src_argb || !dst_rgb565 || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_rgb565 == width * 2) { + width *= height; + height = 1; + src_stride_argb = dst_stride_rgb565 = 0; + } +#if defined(HAS_ARGBTORGB565ROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBToRGB565Row = ARGBToRGB565Row_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBToRGB565Row = ARGBToRGB565Row_SSE2; + } + } +#endif +#if defined(HAS_ARGBTORGB565ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToRGB565Row = ARGBToRGB565Row_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBToRGB565Row = ARGBToRGB565Row_AVX2; + } + } +#endif +#if defined(HAS_ARGBTORGB565ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToRGB565Row = ARGBToRGB565Row_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToRGB565Row = ARGBToRGB565Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBToRGB565Row(src_argb, dst_rgb565, width); + src_argb += src_stride_argb; + dst_rgb565 += dst_stride_rgb565; + } + return 0; +} + +// Convert ARGB To ARGB1555. +LIBYUV_API +int ARGBToARGB1555(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb1555, int dst_stride_argb1555, + int width, int height) { + int y; + void (*ARGBToARGB1555Row)(const uint8* src_argb, uint8* dst_rgb, int pix) = + ARGBToARGB1555Row_C; + if (!src_argb || !dst_argb1555 || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_argb1555 == width * 2) { + width *= height; + height = 1; + src_stride_argb = dst_stride_argb1555 = 0; + } +#if defined(HAS_ARGBTOARGB1555ROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBToARGB1555Row = ARGBToARGB1555Row_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBToARGB1555Row = ARGBToARGB1555Row_SSE2; + } + } +#endif +#if defined(HAS_ARGBTOARGB1555ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToARGB1555Row = ARGBToARGB1555Row_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBToARGB1555Row = ARGBToARGB1555Row_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOARGB1555ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToARGB1555Row = ARGBToARGB1555Row_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToARGB1555Row = ARGBToARGB1555Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBToARGB1555Row(src_argb, dst_argb1555, width); + src_argb += src_stride_argb; + dst_argb1555 += dst_stride_argb1555; + } + return 0; +} + +// Convert ARGB To ARGB4444. +LIBYUV_API +int ARGBToARGB4444(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb4444, int dst_stride_argb4444, + int width, int height) { + int y; + void (*ARGBToARGB4444Row)(const uint8* src_argb, uint8* dst_rgb, int pix) = + ARGBToARGB4444Row_C; + if (!src_argb || !dst_argb4444 || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_argb4444 == width * 2) { + width *= height; + height = 1; + src_stride_argb = dst_stride_argb4444 = 0; + } +#if defined(HAS_ARGBTOARGB4444ROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBToARGB4444Row = ARGBToARGB4444Row_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBToARGB4444Row = ARGBToARGB4444Row_SSE2; + } + } +#endif +#if defined(HAS_ARGBTOARGB4444ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToARGB4444Row = ARGBToARGB4444Row_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBToARGB4444Row = ARGBToARGB4444Row_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOARGB4444ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToARGB4444Row = ARGBToARGB4444Row_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToARGB4444Row = ARGBToARGB4444Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBToARGB4444Row(src_argb, dst_argb4444, width); + src_argb += src_stride_argb; + dst_argb4444 += dst_stride_argb4444; + } + return 0; +} + +// Convert ARGB to J420. (JPeg full range I420). +LIBYUV_API +int ARGBToJ420(const uint8* src_argb, int src_stride_argb, + uint8* dst_yj, int dst_stride_yj, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*ARGBToUVJRow)(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) = ARGBToUVJRow_C; + void (*ARGBToYJRow)(const uint8* src_argb, uint8* dst_yj, int pix) = + ARGBToYJRow_C; + if (!src_argb || + !dst_yj || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } +#if defined(HAS_ARGBTOYJROW_SSSE3) && defined(HAS_ARGBTOUVJROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVJRow = ARGBToUVJRow_Any_SSSE3; + ARGBToYJRow = ARGBToYJRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUVJRow = ARGBToUVJRow_SSSE3; + ARGBToYJRow = ARGBToYJRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYJROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToYJRow = ARGBToYJRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToYJRow = ARGBToYJRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYJROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYJRow = ARGBToYJRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYJRow = ARGBToYJRow_NEON; + } + } +#endif +#if defined(HAS_ARGBTOUVJROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToUVJRow = ARGBToUVJRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + ARGBToUVJRow = ARGBToUVJRow_NEON; + } + } +#endif + + for (y = 0; y < height - 1; y += 2) { + ARGBToUVJRow(src_argb, src_stride_argb, dst_u, dst_v, width); + ARGBToYJRow(src_argb, dst_yj, width); + ARGBToYJRow(src_argb + src_stride_argb, dst_yj + dst_stride_yj, width); + src_argb += src_stride_argb * 2; + dst_yj += dst_stride_yj * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { + ARGBToUVJRow(src_argb, 0, dst_u, dst_v, width); + ARGBToYJRow(src_argb, dst_yj, width); + } + return 0; +} + +// ARGB little endian (bgra in memory) to J422 +LIBYUV_API +int ARGBToJ422(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*ARGBToUVJ422Row)(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix) = ARGBToUVJ422Row_C; + void (*ARGBToYJRow)(const uint8* src_argb, uint8* dst_y, int pix) = + ARGBToYJRow_C; + if (!src_argb || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_y == width && + dst_stride_u * 2 == width && + dst_stride_v * 2 == width) { + width *= height; + height = 1; + src_stride_argb = dst_stride_y = dst_stride_u = dst_stride_v = 0; + } +#if defined(HAS_ARGBTOUVJ422ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVJ422Row = ARGBToUVJ422Row_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUVJ422Row = ARGBToUVJ422Row_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOUVJ422ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToUVJ422Row = ARGBToUVJ422Row_Any_NEON; + if (IS_ALIGNED(width, 16)) { + ARGBToUVJ422Row = ARGBToUVJ422Row_NEON; + } + } +#endif + +#if defined(HAS_ARGBTOYJROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYJRow = ARGBToYJRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYJRow = ARGBToYJRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYJROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToYJRow = ARGBToYJRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToYJRow = ARGBToYJRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYJROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYJRow = ARGBToYJRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYJRow = ARGBToYJRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBToUVJ422Row(src_argb, dst_u, dst_v, width); + ARGBToYJRow(src_argb, dst_y, width); + src_argb += src_stride_argb; + dst_y += dst_stride_y; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + return 0; +} + +// Convert ARGB to J400. +LIBYUV_API +int ARGBToJ400(const uint8* src_argb, int src_stride_argb, + uint8* dst_yj, int dst_stride_yj, + int width, int height) { + int y; + void (*ARGBToYJRow)(const uint8* src_argb, uint8* dst_yj, int pix) = + ARGBToYJRow_C; + if (!src_argb || !dst_yj || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_yj == width) { + width *= height; + height = 1; + src_stride_argb = dst_stride_yj = 0; + } +#if defined(HAS_ARGBTOYJROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYJRow = ARGBToYJRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYJRow = ARGBToYJRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYJROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToYJRow = ARGBToYJRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToYJRow = ARGBToYJRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYJROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYJRow = ARGBToYJRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYJRow = ARGBToYJRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBToYJRow(src_argb, dst_yj, width); + src_argb += src_stride_argb; + dst_yj += dst_stride_yj; + } + return 0; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/convert_jpeg.cc b/third_party/aom/third_party/libyuv/source/convert_jpeg.cc new file mode 100644 index 0000000000..bcb980f7f1 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/convert_jpeg.cc @@ -0,0 +1,392 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/convert.h" + +#ifdef HAVE_JPEG +#include "libyuv/mjpeg_decoder.h" +#endif + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#ifdef HAVE_JPEG +struct I420Buffers { + uint8* y; + int y_stride; + uint8* u; + int u_stride; + uint8* v; + int v_stride; + int w; + int h; +}; + +static void JpegCopyI420(void* opaque, + const uint8* const* data, + const int* strides, + int rows) { + I420Buffers* dest = (I420Buffers*)(opaque); + I420Copy(data[0], strides[0], + data[1], strides[1], + data[2], strides[2], + dest->y, dest->y_stride, + dest->u, dest->u_stride, + dest->v, dest->v_stride, + dest->w, rows); + dest->y += rows * dest->y_stride; + dest->u += ((rows + 1) >> 1) * dest->u_stride; + dest->v += ((rows + 1) >> 1) * dest->v_stride; + dest->h -= rows; +} + +static void JpegI422ToI420(void* opaque, + const uint8* const* data, + const int* strides, + int rows) { + I420Buffers* dest = (I420Buffers*)(opaque); + I422ToI420(data[0], strides[0], + data[1], strides[1], + data[2], strides[2], + dest->y, dest->y_stride, + dest->u, dest->u_stride, + dest->v, dest->v_stride, + dest->w, rows); + dest->y += rows * dest->y_stride; + dest->u += ((rows + 1) >> 1) * dest->u_stride; + dest->v += ((rows + 1) >> 1) * dest->v_stride; + dest->h -= rows; +} + +static void JpegI444ToI420(void* opaque, + const uint8* const* data, + const int* strides, + int rows) { + I420Buffers* dest = (I420Buffers*)(opaque); + I444ToI420(data[0], strides[0], + data[1], strides[1], + data[2], strides[2], + dest->y, dest->y_stride, + dest->u, dest->u_stride, + dest->v, dest->v_stride, + dest->w, rows); + dest->y += rows * dest->y_stride; + dest->u += ((rows + 1) >> 1) * dest->u_stride; + dest->v += ((rows + 1) >> 1) * dest->v_stride; + dest->h -= rows; +} + +static void JpegI411ToI420(void* opaque, + const uint8* const* data, + const int* strides, + int rows) { + I420Buffers* dest = (I420Buffers*)(opaque); + I411ToI420(data[0], strides[0], + data[1], strides[1], + data[2], strides[2], + dest->y, dest->y_stride, + dest->u, dest->u_stride, + dest->v, dest->v_stride, + dest->w, rows); + dest->y += rows * dest->y_stride; + dest->u += ((rows + 1) >> 1) * dest->u_stride; + dest->v += ((rows + 1) >> 1) * dest->v_stride; + dest->h -= rows; +} + +static void JpegI400ToI420(void* opaque, + const uint8* const* data, + const int* strides, + int rows) { + I420Buffers* dest = (I420Buffers*)(opaque); + I400ToI420(data[0], strides[0], + dest->y, dest->y_stride, + dest->u, dest->u_stride, + dest->v, dest->v_stride, + dest->w, rows); + dest->y += rows * dest->y_stride; + dest->u += ((rows + 1) >> 1) * dest->u_stride; + dest->v += ((rows + 1) >> 1) * dest->v_stride; + dest->h -= rows; +} + +// Query size of MJPG in pixels. +LIBYUV_API +int MJPGSize(const uint8* sample, size_t sample_size, + int* width, int* height) { + MJpegDecoder mjpeg_decoder; + LIBYUV_BOOL ret = mjpeg_decoder.LoadFrame(sample, sample_size); + if (ret) { + *width = mjpeg_decoder.GetWidth(); + *height = mjpeg_decoder.GetHeight(); + } + mjpeg_decoder.UnloadFrame(); + return ret ? 0 : -1; // -1 for runtime failure. +} + +// MJPG (Motion JPeg) to I420 +// TODO(fbarchard): review w and h requirement. dw and dh may be enough. +LIBYUV_API +int MJPGToI420(const uint8* sample, + size_t sample_size, + uint8* y, int y_stride, + uint8* u, int u_stride, + uint8* v, int v_stride, + int w, int h, + int dw, int dh) { + if (sample_size == kUnknownDataSize) { + // ERROR: MJPEG frame size unknown + return -1; + } + + // TODO(fbarchard): Port MJpeg to C. + MJpegDecoder mjpeg_decoder; + LIBYUV_BOOL ret = mjpeg_decoder.LoadFrame(sample, sample_size); + if (ret && (mjpeg_decoder.GetWidth() != w || + mjpeg_decoder.GetHeight() != h)) { + // ERROR: MJPEG frame has unexpected dimensions + mjpeg_decoder.UnloadFrame(); + return 1; // runtime failure + } + if (ret) { + I420Buffers bufs = { y, y_stride, u, u_stride, v, v_stride, dw, dh }; + // YUV420 + if (mjpeg_decoder.GetColorSpace() == + MJpegDecoder::kColorSpaceYCbCr && + mjpeg_decoder.GetNumComponents() == 3 && + mjpeg_decoder.GetVertSampFactor(0) == 2 && + mjpeg_decoder.GetHorizSampFactor(0) == 2 && + mjpeg_decoder.GetVertSampFactor(1) == 1 && + mjpeg_decoder.GetHorizSampFactor(1) == 1 && + mjpeg_decoder.GetVertSampFactor(2) == 1 && + mjpeg_decoder.GetHorizSampFactor(2) == 1) { + ret = mjpeg_decoder.DecodeToCallback(&JpegCopyI420, &bufs, dw, dh); + // YUV422 + } else if (mjpeg_decoder.GetColorSpace() == + MJpegDecoder::kColorSpaceYCbCr && + mjpeg_decoder.GetNumComponents() == 3 && + mjpeg_decoder.GetVertSampFactor(0) == 1 && + mjpeg_decoder.GetHorizSampFactor(0) == 2 && + mjpeg_decoder.GetVertSampFactor(1) == 1 && + mjpeg_decoder.GetHorizSampFactor(1) == 1 && + mjpeg_decoder.GetVertSampFactor(2) == 1 && + mjpeg_decoder.GetHorizSampFactor(2) == 1) { + ret = mjpeg_decoder.DecodeToCallback(&JpegI422ToI420, &bufs, dw, dh); + // YUV444 + } else if (mjpeg_decoder.GetColorSpace() == + MJpegDecoder::kColorSpaceYCbCr && + mjpeg_decoder.GetNumComponents() == 3 && + mjpeg_decoder.GetVertSampFactor(0) == 1 && + mjpeg_decoder.GetHorizSampFactor(0) == 1 && + mjpeg_decoder.GetVertSampFactor(1) == 1 && + mjpeg_decoder.GetHorizSampFactor(1) == 1 && + mjpeg_decoder.GetVertSampFactor(2) == 1 && + mjpeg_decoder.GetHorizSampFactor(2) == 1) { + ret = mjpeg_decoder.DecodeToCallback(&JpegI444ToI420, &bufs, dw, dh); + // YUV411 + } else if (mjpeg_decoder.GetColorSpace() == + MJpegDecoder::kColorSpaceYCbCr && + mjpeg_decoder.GetNumComponents() == 3 && + mjpeg_decoder.GetVertSampFactor(0) == 1 && + mjpeg_decoder.GetHorizSampFactor(0) == 4 && + mjpeg_decoder.GetVertSampFactor(1) == 1 && + mjpeg_decoder.GetHorizSampFactor(1) == 1 && + mjpeg_decoder.GetVertSampFactor(2) == 1 && + mjpeg_decoder.GetHorizSampFactor(2) == 1) { + ret = mjpeg_decoder.DecodeToCallback(&JpegI411ToI420, &bufs, dw, dh); + // YUV400 + } else if (mjpeg_decoder.GetColorSpace() == + MJpegDecoder::kColorSpaceGrayscale && + mjpeg_decoder.GetNumComponents() == 1 && + mjpeg_decoder.GetVertSampFactor(0) == 1 && + mjpeg_decoder.GetHorizSampFactor(0) == 1) { + ret = mjpeg_decoder.DecodeToCallback(&JpegI400ToI420, &bufs, dw, dh); + } else { + // TODO(fbarchard): Implement conversion for any other colorspace/sample + // factors that occur in practice. 411 is supported by libjpeg + // ERROR: Unable to convert MJPEG frame because format is not supported + mjpeg_decoder.UnloadFrame(); + return 1; + } + } + return ret ? 0 : 1; +} + +#ifdef HAVE_JPEG +struct ARGBBuffers { + uint8* argb; + int argb_stride; + int w; + int h; +}; + +static void JpegI420ToARGB(void* opaque, + const uint8* const* data, + const int* strides, + int rows) { + ARGBBuffers* dest = (ARGBBuffers*)(opaque); + I420ToARGB(data[0], strides[0], + data[1], strides[1], + data[2], strides[2], + dest->argb, dest->argb_stride, + dest->w, rows); + dest->argb += rows * dest->argb_stride; + dest->h -= rows; +} + +static void JpegI422ToARGB(void* opaque, + const uint8* const* data, + const int* strides, + int rows) { + ARGBBuffers* dest = (ARGBBuffers*)(opaque); + I422ToARGB(data[0], strides[0], + data[1], strides[1], + data[2], strides[2], + dest->argb, dest->argb_stride, + dest->w, rows); + dest->argb += rows * dest->argb_stride; + dest->h -= rows; +} + +static void JpegI444ToARGB(void* opaque, + const uint8* const* data, + const int* strides, + int rows) { + ARGBBuffers* dest = (ARGBBuffers*)(opaque); + I444ToARGB(data[0], strides[0], + data[1], strides[1], + data[2], strides[2], + dest->argb, dest->argb_stride, + dest->w, rows); + dest->argb += rows * dest->argb_stride; + dest->h -= rows; +} + +static void JpegI411ToARGB(void* opaque, + const uint8* const* data, + const int* strides, + int rows) { + ARGBBuffers* dest = (ARGBBuffers*)(opaque); + I411ToARGB(data[0], strides[0], + data[1], strides[1], + data[2], strides[2], + dest->argb, dest->argb_stride, + dest->w, rows); + dest->argb += rows * dest->argb_stride; + dest->h -= rows; +} + +static void JpegI400ToARGB(void* opaque, + const uint8* const* data, + const int* strides, + int rows) { + ARGBBuffers* dest = (ARGBBuffers*)(opaque); + I400ToARGB(data[0], strides[0], + dest->argb, dest->argb_stride, + dest->w, rows); + dest->argb += rows * dest->argb_stride; + dest->h -= rows; +} + +// MJPG (Motion JPeg) to ARGB +// TODO(fbarchard): review w and h requirement. dw and dh may be enough. +LIBYUV_API +int MJPGToARGB(const uint8* sample, + size_t sample_size, + uint8* argb, int argb_stride, + int w, int h, + int dw, int dh) { + if (sample_size == kUnknownDataSize) { + // ERROR: MJPEG frame size unknown + return -1; + } + + // TODO(fbarchard): Port MJpeg to C. + MJpegDecoder mjpeg_decoder; + LIBYUV_BOOL ret = mjpeg_decoder.LoadFrame(sample, sample_size); + if (ret && (mjpeg_decoder.GetWidth() != w || + mjpeg_decoder.GetHeight() != h)) { + // ERROR: MJPEG frame has unexpected dimensions + mjpeg_decoder.UnloadFrame(); + return 1; // runtime failure + } + if (ret) { + ARGBBuffers bufs = { argb, argb_stride, dw, dh }; + // YUV420 + if (mjpeg_decoder.GetColorSpace() == + MJpegDecoder::kColorSpaceYCbCr && + mjpeg_decoder.GetNumComponents() == 3 && + mjpeg_decoder.GetVertSampFactor(0) == 2 && + mjpeg_decoder.GetHorizSampFactor(0) == 2 && + mjpeg_decoder.GetVertSampFactor(1) == 1 && + mjpeg_decoder.GetHorizSampFactor(1) == 1 && + mjpeg_decoder.GetVertSampFactor(2) == 1 && + mjpeg_decoder.GetHorizSampFactor(2) == 1) { + ret = mjpeg_decoder.DecodeToCallback(&JpegI420ToARGB, &bufs, dw, dh); + // YUV422 + } else if (mjpeg_decoder.GetColorSpace() == + MJpegDecoder::kColorSpaceYCbCr && + mjpeg_decoder.GetNumComponents() == 3 && + mjpeg_decoder.GetVertSampFactor(0) == 1 && + mjpeg_decoder.GetHorizSampFactor(0) == 2 && + mjpeg_decoder.GetVertSampFactor(1) == 1 && + mjpeg_decoder.GetHorizSampFactor(1) == 1 && + mjpeg_decoder.GetVertSampFactor(2) == 1 && + mjpeg_decoder.GetHorizSampFactor(2) == 1) { + ret = mjpeg_decoder.DecodeToCallback(&JpegI422ToARGB, &bufs, dw, dh); + // YUV444 + } else if (mjpeg_decoder.GetColorSpace() == + MJpegDecoder::kColorSpaceYCbCr && + mjpeg_decoder.GetNumComponents() == 3 && + mjpeg_decoder.GetVertSampFactor(0) == 1 && + mjpeg_decoder.GetHorizSampFactor(0) == 1 && + mjpeg_decoder.GetVertSampFactor(1) == 1 && + mjpeg_decoder.GetHorizSampFactor(1) == 1 && + mjpeg_decoder.GetVertSampFactor(2) == 1 && + mjpeg_decoder.GetHorizSampFactor(2) == 1) { + ret = mjpeg_decoder.DecodeToCallback(&JpegI444ToARGB, &bufs, dw, dh); + // YUV411 + } else if (mjpeg_decoder.GetColorSpace() == + MJpegDecoder::kColorSpaceYCbCr && + mjpeg_decoder.GetNumComponents() == 3 && + mjpeg_decoder.GetVertSampFactor(0) == 1 && + mjpeg_decoder.GetHorizSampFactor(0) == 4 && + mjpeg_decoder.GetVertSampFactor(1) == 1 && + mjpeg_decoder.GetHorizSampFactor(1) == 1 && + mjpeg_decoder.GetVertSampFactor(2) == 1 && + mjpeg_decoder.GetHorizSampFactor(2) == 1) { + ret = mjpeg_decoder.DecodeToCallback(&JpegI411ToARGB, &bufs, dw, dh); + // YUV400 + } else if (mjpeg_decoder.GetColorSpace() == + MJpegDecoder::kColorSpaceGrayscale && + mjpeg_decoder.GetNumComponents() == 1 && + mjpeg_decoder.GetVertSampFactor(0) == 1 && + mjpeg_decoder.GetHorizSampFactor(0) == 1) { + ret = mjpeg_decoder.DecodeToCallback(&JpegI400ToARGB, &bufs, dw, dh); + } else { + // TODO(fbarchard): Implement conversion for any other colorspace/sample + // factors that occur in practice. 411 is supported by libjpeg + // ERROR: Unable to convert MJPEG frame because format is not supported + mjpeg_decoder.UnloadFrame(); + return 1; + } + } + return ret ? 0 : 1; +} +#endif + +#endif + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/convert_to_argb.cc b/third_party/aom/third_party/libyuv/source/convert_to_argb.cc new file mode 100644 index 0000000000..af829fbd32 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/convert_to_argb.cc @@ -0,0 +1,306 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/convert_argb.h" + +#include "libyuv/cpu_id.h" +#ifdef HAVE_JPEG +#include "libyuv/mjpeg_decoder.h" +#endif +#include "libyuv/rotate_argb.h" +#include "libyuv/row.h" +#include "libyuv/video_common.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Convert camera sample to I420 with cropping, rotation and vertical flip. +// src_width is used for source stride computation +// src_height is used to compute location of planes, and indicate inversion +// sample_size is measured in bytes and is the size of the frame. +// With MJPEG it is the compressed size of the frame. +LIBYUV_API +int ConvertToARGB(const uint8* sample, size_t sample_size, + uint8* crop_argb, int argb_stride, + int crop_x, int crop_y, + int src_width, int src_height, + int crop_width, int crop_height, + enum RotationMode rotation, + uint32 fourcc) { + uint32 format = CanonicalFourCC(fourcc); + int aligned_src_width = (src_width + 1) & ~1; + const uint8* src; + const uint8* src_uv; + int abs_src_height = (src_height < 0) ? -src_height : src_height; + int inv_crop_height = (crop_height < 0) ? -crop_height : crop_height; + int r = 0; + + // One pass rotation is available for some formats. For the rest, convert + // to I420 (with optional vertical flipping) into a temporary I420 buffer, + // and then rotate the I420 to the final destination buffer. + // For in-place conversion, if destination crop_argb is same as source sample, + // also enable temporary buffer. + LIBYUV_BOOL need_buf = (rotation && format != FOURCC_ARGB) || + crop_argb == sample; + uint8* tmp_argb = crop_argb; + int tmp_argb_stride = argb_stride; + uint8* rotate_buffer = NULL; + int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height; + + if (crop_argb == NULL || sample == NULL || + src_width <= 0 || crop_width <= 0 || + src_height == 0 || crop_height == 0) { + return -1; + } + if (src_height < 0) { + inv_crop_height = -inv_crop_height; + } + + if (need_buf) { + int argb_size = crop_width * abs_crop_height * 4; + rotate_buffer = (uint8*)malloc(argb_size); + if (!rotate_buffer) { + return 1; // Out of memory runtime error. + } + crop_argb = rotate_buffer; + argb_stride = crop_width; + } + + switch (format) { + // Single plane formats + case FOURCC_YUY2: + src = sample + (aligned_src_width * crop_y + crop_x) * 2; + r = YUY2ToARGB(src, aligned_src_width * 2, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_UYVY: + src = sample + (aligned_src_width * crop_y + crop_x) * 2; + r = UYVYToARGB(src, aligned_src_width * 2, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_24BG: + src = sample + (src_width * crop_y + crop_x) * 3; + r = RGB24ToARGB(src, src_width * 3, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_RAW: + src = sample + (src_width * crop_y + crop_x) * 3; + r = RAWToARGB(src, src_width * 3, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_ARGB: + src = sample + (src_width * crop_y + crop_x) * 4; + r = ARGBToARGB(src, src_width * 4, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_BGRA: + src = sample + (src_width * crop_y + crop_x) * 4; + r = BGRAToARGB(src, src_width * 4, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_ABGR: + src = sample + (src_width * crop_y + crop_x) * 4; + r = ABGRToARGB(src, src_width * 4, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_RGBA: + src = sample + (src_width * crop_y + crop_x) * 4; + r = RGBAToARGB(src, src_width * 4, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_RGBP: + src = sample + (src_width * crop_y + crop_x) * 2; + r = RGB565ToARGB(src, src_width * 2, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_RGBO: + src = sample + (src_width * crop_y + crop_x) * 2; + r = ARGB1555ToARGB(src, src_width * 2, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_R444: + src = sample + (src_width * crop_y + crop_x) * 2; + r = ARGB4444ToARGB(src, src_width * 2, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_I400: + src = sample + src_width * crop_y + crop_x; + r = I400ToARGB(src, src_width, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + + // Biplanar formats + case FOURCC_NV12: + src = sample + (src_width * crop_y + crop_x); + src_uv = sample + aligned_src_width * (src_height + crop_y / 2) + crop_x; + r = NV12ToARGB(src, src_width, + src_uv, aligned_src_width, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_NV21: + src = sample + (src_width * crop_y + crop_x); + src_uv = sample + aligned_src_width * (src_height + crop_y / 2) + crop_x; + // Call NV12 but with u and v parameters swapped. + r = NV21ToARGB(src, src_width, + src_uv, aligned_src_width, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + case FOURCC_M420: + src = sample + (src_width * crop_y) * 12 / 8 + crop_x; + r = M420ToARGB(src, src_width, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + // Triplanar formats + case FOURCC_I420: + case FOURCC_YU12: + case FOURCC_YV12: { + const uint8* src_y = sample + (src_width * crop_y + crop_x); + const uint8* src_u; + const uint8* src_v; + int halfwidth = (src_width + 1) / 2; + int halfheight = (abs_src_height + 1) / 2; + if (format == FOURCC_YV12) { + src_v = sample + src_width * abs_src_height + + (halfwidth * crop_y + crop_x) / 2; + src_u = sample + src_width * abs_src_height + + halfwidth * (halfheight + crop_y / 2) + crop_x / 2; + } else { + src_u = sample + src_width * abs_src_height + + (halfwidth * crop_y + crop_x) / 2; + src_v = sample + src_width * abs_src_height + + halfwidth * (halfheight + crop_y / 2) + crop_x / 2; + } + r = I420ToARGB(src_y, src_width, + src_u, halfwidth, + src_v, halfwidth, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + } + + case FOURCC_J420: { + const uint8* src_y = sample + (src_width * crop_y + crop_x); + const uint8* src_u; + const uint8* src_v; + int halfwidth = (src_width + 1) / 2; + int halfheight = (abs_src_height + 1) / 2; + src_u = sample + src_width * abs_src_height + + (halfwidth * crop_y + crop_x) / 2; + src_v = sample + src_width * abs_src_height + + halfwidth * (halfheight + crop_y / 2) + crop_x / 2; + r = J420ToARGB(src_y, src_width, + src_u, halfwidth, + src_v, halfwidth, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + } + + case FOURCC_I422: + case FOURCC_YV16: { + const uint8* src_y = sample + src_width * crop_y + crop_x; + const uint8* src_u; + const uint8* src_v; + int halfwidth = (src_width + 1) / 2; + if (format == FOURCC_YV16) { + src_v = sample + src_width * abs_src_height + + halfwidth * crop_y + crop_x / 2; + src_u = sample + src_width * abs_src_height + + halfwidth * (abs_src_height + crop_y) + crop_x / 2; + } else { + src_u = sample + src_width * abs_src_height + + halfwidth * crop_y + crop_x / 2; + src_v = sample + src_width * abs_src_height + + halfwidth * (abs_src_height + crop_y) + crop_x / 2; + } + r = I422ToARGB(src_y, src_width, + src_u, halfwidth, + src_v, halfwidth, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + } + case FOURCC_I444: + case FOURCC_YV24: { + const uint8* src_y = sample + src_width * crop_y + crop_x; + const uint8* src_u; + const uint8* src_v; + if (format == FOURCC_YV24) { + src_v = sample + src_width * (abs_src_height + crop_y) + crop_x; + src_u = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x; + } else { + src_u = sample + src_width * (abs_src_height + crop_y) + crop_x; + src_v = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x; + } + r = I444ToARGB(src_y, src_width, + src_u, src_width, + src_v, src_width, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + } + case FOURCC_I411: { + int quarterwidth = (src_width + 3) / 4; + const uint8* src_y = sample + src_width * crop_y + crop_x; + const uint8* src_u = sample + src_width * abs_src_height + + quarterwidth * crop_y + crop_x / 4; + const uint8* src_v = sample + src_width * abs_src_height + + quarterwidth * (abs_src_height + crop_y) + crop_x / 4; + r = I411ToARGB(src_y, src_width, + src_u, quarterwidth, + src_v, quarterwidth, + crop_argb, argb_stride, + crop_width, inv_crop_height); + break; + } +#ifdef HAVE_JPEG + case FOURCC_MJPG: + r = MJPGToARGB(sample, sample_size, + crop_argb, argb_stride, + src_width, abs_src_height, crop_width, inv_crop_height); + break; +#endif + default: + r = -1; // unknown fourcc - return failure code. + } + + if (need_buf) { + if (!r) { + r = ARGBRotate(crop_argb, argb_stride, + tmp_argb, tmp_argb_stride, + crop_width, abs_crop_height, rotation); + } + free(rotate_buffer); + } + + return r; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/convert_to_i420.cc b/third_party/aom/third_party/libyuv/source/convert_to_i420.cc new file mode 100644 index 0000000000..5e75369b55 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/convert_to_i420.cc @@ -0,0 +1,339 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "libyuv/convert.h" + +#include "libyuv/video_common.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Convert camera sample to I420 with cropping, rotation and vertical flip. +// src_width is used for source stride computation +// src_height is used to compute location of planes, and indicate inversion +// sample_size is measured in bytes and is the size of the frame. +// With MJPEG it is the compressed size of the frame. +LIBYUV_API +int ConvertToI420(const uint8* sample, + size_t sample_size, + uint8* y, int y_stride, + uint8* u, int u_stride, + uint8* v, int v_stride, + int crop_x, int crop_y, + int src_width, int src_height, + int crop_width, int crop_height, + enum RotationMode rotation, + uint32 fourcc) { + uint32 format = CanonicalFourCC(fourcc); + int aligned_src_width = (src_width + 1) & ~1; + const uint8* src; + const uint8* src_uv; + int abs_src_height = (src_height < 0) ? -src_height : src_height; + int inv_crop_height = (crop_height < 0) ? -crop_height : crop_height; + int r = 0; + LIBYUV_BOOL need_buf = (rotation && format != FOURCC_I420 && + format != FOURCC_NV12 && format != FOURCC_NV21 && + format != FOURCC_YU12 && format != FOURCC_YV12) || y == sample; + uint8* tmp_y = y; + uint8* tmp_u = u; + uint8* tmp_v = v; + int tmp_y_stride = y_stride; + int tmp_u_stride = u_stride; + int tmp_v_stride = v_stride; + uint8* rotate_buffer = NULL; + int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height; + + if (!y || !u || !v || !sample || + src_width <= 0 || crop_width <= 0 || + src_height == 0 || crop_height == 0) { + return -1; + } + if (src_height < 0) { + inv_crop_height = -inv_crop_height; + } + + // One pass rotation is available for some formats. For the rest, convert + // to I420 (with optional vertical flipping) into a temporary I420 buffer, + // and then rotate the I420 to the final destination buffer. + // For in-place conversion, if destination y is same as source sample, + // also enable temporary buffer. + if (need_buf) { + int y_size = crop_width * abs_crop_height; + int uv_size = ((crop_width + 1) / 2) * ((abs_crop_height + 1) / 2); + rotate_buffer = (uint8*)malloc(y_size + uv_size * 2); + if (!rotate_buffer) { + return 1; // Out of memory runtime error. + } + y = rotate_buffer; + u = y + y_size; + v = u + uv_size; + y_stride = crop_width; + u_stride = v_stride = ((crop_width + 1) / 2); + } + + switch (format) { + // Single plane formats + case FOURCC_YUY2: + src = sample + (aligned_src_width * crop_y + crop_x) * 2; + r = YUY2ToI420(src, aligned_src_width * 2, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + case FOURCC_UYVY: + src = sample + (aligned_src_width * crop_y + crop_x) * 2; + r = UYVYToI420(src, aligned_src_width * 2, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + case FOURCC_RGBP: + src = sample + (src_width * crop_y + crop_x) * 2; + r = RGB565ToI420(src, src_width * 2, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + case FOURCC_RGBO: + src = sample + (src_width * crop_y + crop_x) * 2; + r = ARGB1555ToI420(src, src_width * 2, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + case FOURCC_R444: + src = sample + (src_width * crop_y + crop_x) * 2; + r = ARGB4444ToI420(src, src_width * 2, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + case FOURCC_24BG: + src = sample + (src_width * crop_y + crop_x) * 3; + r = RGB24ToI420(src, src_width * 3, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + case FOURCC_RAW: + src = sample + (src_width * crop_y + crop_x) * 3; + r = RAWToI420(src, src_width * 3, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + case FOURCC_ARGB: + src = sample + (src_width * crop_y + crop_x) * 4; + r = ARGBToI420(src, src_width * 4, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + case FOURCC_BGRA: + src = sample + (src_width * crop_y + crop_x) * 4; + r = BGRAToI420(src, src_width * 4, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + case FOURCC_ABGR: + src = sample + (src_width * crop_y + crop_x) * 4; + r = ABGRToI420(src, src_width * 4, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + case FOURCC_RGBA: + src = sample + (src_width * crop_y + crop_x) * 4; + r = RGBAToI420(src, src_width * 4, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + case FOURCC_I400: + src = sample + src_width * crop_y + crop_x; + r = I400ToI420(src, src_width, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + // Biplanar formats + case FOURCC_NV12: + src = sample + (src_width * crop_y + crop_x); + src_uv = sample + (src_width * src_height) + + ((crop_y / 2) * aligned_src_width) + ((crop_x / 2) * 2); + r = NV12ToI420Rotate(src, src_width, + src_uv, aligned_src_width, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height, rotation); + break; + case FOURCC_NV21: + src = sample + (src_width * crop_y + crop_x); + src_uv = sample + (src_width * src_height) + + ((crop_y / 2) * aligned_src_width) + ((crop_x / 2) * 2); + // Call NV12 but with u and v parameters swapped. + r = NV12ToI420Rotate(src, src_width, + src_uv, aligned_src_width, + y, y_stride, + v, v_stride, + u, u_stride, + crop_width, inv_crop_height, rotation); + break; + case FOURCC_M420: + src = sample + (src_width * crop_y) * 12 / 8 + crop_x; + r = M420ToI420(src, src_width, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + // Triplanar formats + case FOURCC_I420: + case FOURCC_YU12: + case FOURCC_YV12: { + const uint8* src_y = sample + (src_width * crop_y + crop_x); + const uint8* src_u; + const uint8* src_v; + int halfwidth = (src_width + 1) / 2; + int halfheight = (abs_src_height + 1) / 2; + if (format == FOURCC_YV12) { + src_v = sample + src_width * abs_src_height + + (halfwidth * crop_y + crop_x) / 2; + src_u = sample + src_width * abs_src_height + + halfwidth * (halfheight + crop_y / 2) + crop_x / 2; + } else { + src_u = sample + src_width * abs_src_height + + (halfwidth * crop_y + crop_x) / 2; + src_v = sample + src_width * abs_src_height + + halfwidth * (halfheight + crop_y / 2) + crop_x / 2; + } + r = I420Rotate(src_y, src_width, + src_u, halfwidth, + src_v, halfwidth, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height, rotation); + break; + } + case FOURCC_I422: + case FOURCC_YV16: { + const uint8* src_y = sample + src_width * crop_y + crop_x; + const uint8* src_u; + const uint8* src_v; + int halfwidth = (src_width + 1) / 2; + if (format == FOURCC_YV16) { + src_v = sample + src_width * abs_src_height + + halfwidth * crop_y + crop_x / 2; + src_u = sample + src_width * abs_src_height + + halfwidth * (abs_src_height + crop_y) + crop_x / 2; + } else { + src_u = sample + src_width * abs_src_height + + halfwidth * crop_y + crop_x / 2; + src_v = sample + src_width * abs_src_height + + halfwidth * (abs_src_height + crop_y) + crop_x / 2; + } + r = I422ToI420(src_y, src_width, + src_u, halfwidth, + src_v, halfwidth, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + } + case FOURCC_I444: + case FOURCC_YV24: { + const uint8* src_y = sample + src_width * crop_y + crop_x; + const uint8* src_u; + const uint8* src_v; + if (format == FOURCC_YV24) { + src_v = sample + src_width * (abs_src_height + crop_y) + crop_x; + src_u = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x; + } else { + src_u = sample + src_width * (abs_src_height + crop_y) + crop_x; + src_v = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x; + } + r = I444ToI420(src_y, src_width, + src_u, src_width, + src_v, src_width, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + } + case FOURCC_I411: { + int quarterwidth = (src_width + 3) / 4; + const uint8* src_y = sample + src_width * crop_y + crop_x; + const uint8* src_u = sample + src_width * abs_src_height + + quarterwidth * crop_y + crop_x / 4; + const uint8* src_v = sample + src_width * abs_src_height + + quarterwidth * (abs_src_height + crop_y) + crop_x / 4; + r = I411ToI420(src_y, src_width, + src_u, quarterwidth, + src_v, quarterwidth, + y, y_stride, + u, u_stride, + v, v_stride, + crop_width, inv_crop_height); + break; + } +#ifdef HAVE_JPEG + case FOURCC_MJPG: + r = MJPGToI420(sample, sample_size, + y, y_stride, + u, u_stride, + v, v_stride, + src_width, abs_src_height, crop_width, inv_crop_height); + break; +#endif + default: + r = -1; // unknown fourcc - return failure code. + } + + if (need_buf) { + if (!r) { + r = I420Rotate(y, y_stride, + u, u_stride, + v, v_stride, + tmp_y, tmp_y_stride, + tmp_u, tmp_u_stride, + tmp_v, tmp_v_stride, + crop_width, abs_crop_height, rotation); + } + free(rotate_buffer); + } + + return r; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/cpu_id.cc b/third_party/aom/third_party/libyuv/source/cpu_id.cc new file mode 100644 index 0000000000..72f686e3b3 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/cpu_id.cc @@ -0,0 +1,307 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/cpu_id.h" + +#if (defined(_MSC_VER) && !defined(__clang__)) && !defined(__clang__) +#include // For __cpuidex() +#endif +#if !defined(__pnacl__) && !defined(__CLR_VER) && \ + !defined(__native_client__) && (defined(_M_IX86) || defined(_M_X64)) && \ + defined(_MSC_VER) && !defined(__clang__) && (_MSC_FULL_VER >= 160040219) +#include // For _xgetbv() +#endif + +#if !defined(__native_client__) +#include // For getenv() +#endif + +// For ArmCpuCaps() but unittested on all platforms +#include +#include + +#include "libyuv/basic_types.h" // For CPU_X86 + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// For functions that use the stack and have runtime checks for overflow, +// use SAFEBUFFERS to avoid additional check. +#if (defined(_MSC_VER) && !defined(__clang__)) && (_MSC_FULL_VER >= 160040219) +#define SAFEBUFFERS __declspec(safebuffers) +#else +#define SAFEBUFFERS +#endif + +// Low level cpuid for X86. +#if (defined(_M_IX86) || defined(_M_X64) || \ + defined(__i386__) || defined(__x86_64__)) && \ + !defined(__pnacl__) && !defined(__CLR_VER) +LIBYUV_API +void CpuId(uint32 info_eax, uint32 info_ecx, uint32* cpu_info) { +#if (defined(_MSC_VER) && !defined(__clang__)) && !defined(__clang__) +// Visual C version uses intrinsic or inline x86 assembly. +#if (_MSC_FULL_VER >= 160040219) + __cpuidex((int*)(cpu_info), info_eax, info_ecx); +#elif defined(_M_IX86) + __asm { + mov eax, info_eax + mov ecx, info_ecx + mov edi, cpu_info + cpuid + mov [edi], eax + mov [edi + 4], ebx + mov [edi + 8], ecx + mov [edi + 12], edx + } +#else + if (info_ecx == 0) { + __cpuid((int*)(cpu_info), info_eax); + } else { + cpu_info[3] = cpu_info[2] = cpu_info[1] = cpu_info[0] = 0; + } +#endif +// GCC version uses inline x86 assembly. +#else // (defined(_MSC_VER) && !defined(__clang__)) && !defined(__clang__) + uint32 info_ebx, info_edx; + asm volatile ( // NOLINT +#if defined( __i386__) && defined(__PIC__) + // Preserve ebx for fpic 32 bit. + "mov %%ebx, %%edi \n" + "cpuid \n" + "xchg %%edi, %%ebx \n" + : "=D" (info_ebx), +#else + "cpuid \n" + : "=b" (info_ebx), +#endif // defined( __i386__) && defined(__PIC__) + "+a" (info_eax), "+c" (info_ecx), "=d" (info_edx)); + cpu_info[0] = info_eax; + cpu_info[1] = info_ebx; + cpu_info[2] = info_ecx; + cpu_info[3] = info_edx; +#endif // (defined(_MSC_VER) && !defined(__clang__)) && !defined(__clang__) +} +#else // (defined(_M_IX86) || defined(_M_X64) ... +LIBYUV_API +void CpuId(uint32 eax, uint32 ecx, uint32* cpu_info) { + cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0; +} +#endif + +// TODO(fbarchard): Enable xgetbv when validator supports it. +#if (defined(_M_IX86) || defined(_M_X64) || \ + defined(__i386__) || defined(__x86_64__)) && \ + !defined(__pnacl__) && !defined(__CLR_VER) && !defined(__native_client__) +#define HAS_XGETBV +// X86 CPUs have xgetbv to detect OS saves high parts of ymm registers. +int TestOsSaveYmm() { + uint32 xcr0 = 0u; +#if (defined(_MSC_VER) && !defined(__clang__)) && (_MSC_FULL_VER >= 160040219) + xcr0 = (uint32)(_xgetbv(0)); // VS2010 SP1 required. +#elif defined(_M_IX86) && defined(_MSC_VER) && !defined(__clang__) + __asm { + xor ecx, ecx // xcr 0 + _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0 // For VS2010 and earlier. + mov xcr0, eax + } +#elif defined(__i386__) || defined(__x86_64__) + asm(".byte 0x0f, 0x01, 0xd0" : "=a" (xcr0) : "c" (0) : "%edx"); +#endif // defined(__i386__) || defined(__x86_64__) + return((xcr0 & 6) == 6); // Is ymm saved? +} +#endif // defined(_M_IX86) || defined(_M_X64) .. + +// based on libaom arm_cpudetect.c +// For Arm, but public to allow testing on any CPU +LIBYUV_API SAFEBUFFERS +int ArmCpuCaps(const char* cpuinfo_name) { + char cpuinfo_line[512]; + FILE* f = fopen(cpuinfo_name, "r"); + if (!f) { + // Assume Neon if /proc/cpuinfo is unavailable. + // This will occur for Chrome sandbox for Pepper or Render process. + return kCpuHasNEON; + } + while (fgets(cpuinfo_line, sizeof(cpuinfo_line) - 1, f)) { + if (memcmp(cpuinfo_line, "Features", 8) == 0) { + char* p = strstr(cpuinfo_line, " neon"); + if (p && (p[5] == ' ' || p[5] == '\n')) { + fclose(f); + return kCpuHasNEON; + } + // aarch64 uses asimd for Neon. + p = strstr(cpuinfo_line, " asimd"); + if (p && (p[6] == ' ' || p[6] == '\n')) { + fclose(f); + return kCpuHasNEON; + } + } + } + fclose(f); + return 0; +} + +#if defined(__mips__) && defined(__linux__) +static int MipsCpuCaps(const char* search_string) { + char cpuinfo_line[512]; + const char* file_name = "/proc/cpuinfo"; + FILE* f = fopen(file_name, "r"); + if (!f) { + // Assume DSP if /proc/cpuinfo is unavailable. + // This will occur for Chrome sandbox for Pepper or Render process. + return kCpuHasMIPS_DSP; + } + while (fgets(cpuinfo_line, sizeof(cpuinfo_line) - 1, f) != NULL) { + if (strstr(cpuinfo_line, search_string) != NULL) { + fclose(f); + return kCpuHasMIPS_DSP; + } + } + fclose(f); + return 0; +} +#endif + +// CPU detect function for SIMD instruction sets. +LIBYUV_API +int cpu_info_ = kCpuInit; // cpu_info is not initialized yet. + +// Test environment variable for disabling CPU features. Any non-zero value +// to disable. Zero ignored to make it easy to set the variable on/off. +#if !defined(__native_client__) && !defined(_M_ARM) + +static LIBYUV_BOOL TestEnv(const char* name) { + const char* var = getenv(name); + if (var) { + if (var[0] != '0') { + return LIBYUV_TRUE; + } + } + return LIBYUV_FALSE; +} +#else // nacl does not support getenv(). +static LIBYUV_BOOL TestEnv(const char*) { + return LIBYUV_FALSE; +} +#endif + +LIBYUV_API SAFEBUFFERS +int InitCpuFlags(void) { +#if !defined(__pnacl__) && !defined(__CLR_VER) && defined(CPU_X86) + + uint32 cpu_info0[4] = { 0, 0, 0, 0 }; + uint32 cpu_info1[4] = { 0, 0, 0, 0 }; + uint32 cpu_info7[4] = { 0, 0, 0, 0 }; + CpuId(0, 0, cpu_info0); + CpuId(1, 0, cpu_info1); + if (cpu_info0[0] >= 7) { + CpuId(7, 0, cpu_info7); + } + cpu_info_ = ((cpu_info1[3] & 0x04000000) ? kCpuHasSSE2 : 0) | + ((cpu_info1[2] & 0x00000200) ? kCpuHasSSSE3 : 0) | + ((cpu_info1[2] & 0x00080000) ? kCpuHasSSE41 : 0) | + ((cpu_info1[2] & 0x00100000) ? kCpuHasSSE42 : 0) | + ((cpu_info7[1] & 0x00000200) ? kCpuHasERMS : 0) | + ((cpu_info1[2] & 0x00001000) ? kCpuHasFMA3 : 0) | + kCpuHasX86; + +#ifdef HAS_XGETBV + if ((cpu_info1[2] & 0x18000000) == 0x18000000 && // AVX and OSSave + TestOsSaveYmm()) { // Saves YMM. + cpu_info_ |= ((cpu_info7[1] & 0x00000020) ? kCpuHasAVX2 : 0) | + kCpuHasAVX; + } +#endif + // Environment variable overrides for testing. + if (TestEnv("LIBYUV_DISABLE_X86")) { + cpu_info_ &= ~kCpuHasX86; + } + if (TestEnv("LIBYUV_DISABLE_SSE2")) { + cpu_info_ &= ~kCpuHasSSE2; + } + if (TestEnv("LIBYUV_DISABLE_SSSE3")) { + cpu_info_ &= ~kCpuHasSSSE3; + } + if (TestEnv("LIBYUV_DISABLE_SSE41")) { + cpu_info_ &= ~kCpuHasSSE41; + } + if (TestEnv("LIBYUV_DISABLE_SSE42")) { + cpu_info_ &= ~kCpuHasSSE42; + } + if (TestEnv("LIBYUV_DISABLE_AVX")) { + cpu_info_ &= ~kCpuHasAVX; + } + if (TestEnv("LIBYUV_DISABLE_AVX2")) { + cpu_info_ &= ~kCpuHasAVX2; + } + if (TestEnv("LIBYUV_DISABLE_ERMS")) { + cpu_info_ &= ~kCpuHasERMS; + } + if (TestEnv("LIBYUV_DISABLE_FMA3")) { + cpu_info_ &= ~kCpuHasFMA3; + } +#endif +#if defined(__mips__) && defined(__linux__) + // Linux mips parse text file for dsp detect. + cpu_info_ = MipsCpuCaps("dsp"); // set kCpuHasMIPS_DSP. +#if defined(__mips_dspr2) + cpu_info_ |= kCpuHasMIPS_DSPR2; +#endif + cpu_info_ |= kCpuHasMIPS; + + if (getenv("LIBYUV_DISABLE_MIPS")) { + cpu_info_ &= ~kCpuHasMIPS; + } + if (getenv("LIBYUV_DISABLE_MIPS_DSP")) { + cpu_info_ &= ~kCpuHasMIPS_DSP; + } + if (getenv("LIBYUV_DISABLE_MIPS_DSPR2")) { + cpu_info_ &= ~kCpuHasMIPS_DSPR2; + } +#endif +#if defined(__arm__) || defined(__aarch64__) +// gcc -mfpu=neon defines __ARM_NEON__ +// __ARM_NEON__ generates code that requires Neon. NaCL also requires Neon. +// For Linux, /proc/cpuinfo can be tested but without that assume Neon. +#if defined(__ARM_NEON__) || defined(__native_client__) || !defined(__linux__) + cpu_info_ = kCpuHasNEON; +// For aarch64(arm64), /proc/cpuinfo's feature is not complete, e.g. no neon +// flag in it. +// So for aarch64, neon enabling is hard coded here. +#endif +#if defined(__aarch64__) + cpu_info_ = kCpuHasNEON; +#else + // Linux arm parse text file for neon detect. + cpu_info_ = ArmCpuCaps("/proc/cpuinfo"); +#endif + cpu_info_ |= kCpuHasARM; + if (TestEnv("LIBYUV_DISABLE_NEON")) { + cpu_info_ &= ~kCpuHasNEON; + } +#endif // __arm__ + if (TestEnv("LIBYUV_DISABLE_ASM")) { + cpu_info_ = 0; + } + return cpu_info_; +} + +LIBYUV_API +void MaskCpuFlags(int enable_flags) { + cpu_info_ = InitCpuFlags() & enable_flags; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/mjpeg_decoder.cc b/third_party/aom/third_party/libyuv/source/mjpeg_decoder.cc new file mode 100644 index 0000000000..75f8a610e3 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/mjpeg_decoder.cc @@ -0,0 +1,572 @@ +/* + * Copyright 2012 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/mjpeg_decoder.h" + +#ifdef HAVE_JPEG +#include + +#if !defined(__pnacl__) && !defined(__CLR_VER) && \ + !defined(COVERAGE_ENABLED) && !defined(TARGET_IPHONE_SIMULATOR) +// Must be included before jpeglib. +#include +#define HAVE_SETJMP + +#if defined(_MSC_VER) +// disable warning 4324: structure was padded due to __declspec(align()) +#pragma warning(disable:4324) +#endif + +#endif +struct FILE; // For jpeglib.h. + +// C++ build requires extern C for jpeg internals. +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#ifdef __cplusplus +} // extern "C" +#endif + +#include "libyuv/planar_functions.h" // For CopyPlane(). + +namespace libyuv { + +#ifdef HAVE_SETJMP +struct SetJmpErrorMgr { + jpeg_error_mgr base; // Must be at the top + jmp_buf setjmp_buffer; +}; +#endif + +const int MJpegDecoder::kColorSpaceUnknown = JCS_UNKNOWN; +const int MJpegDecoder::kColorSpaceGrayscale = JCS_GRAYSCALE; +const int MJpegDecoder::kColorSpaceRgb = JCS_RGB; +const int MJpegDecoder::kColorSpaceYCbCr = JCS_YCbCr; +const int MJpegDecoder::kColorSpaceCMYK = JCS_CMYK; +const int MJpegDecoder::kColorSpaceYCCK = JCS_YCCK; + +// Methods that are passed to jpeglib. +boolean fill_input_buffer(jpeg_decompress_struct* cinfo); +void init_source(jpeg_decompress_struct* cinfo); +void skip_input_data(jpeg_decompress_struct* cinfo, + long num_bytes); // NOLINT +void term_source(jpeg_decompress_struct* cinfo); +void ErrorHandler(jpeg_common_struct* cinfo); + +MJpegDecoder::MJpegDecoder() + : has_scanline_padding_(LIBYUV_FALSE), + num_outbufs_(0), + scanlines_(NULL), + scanlines_sizes_(NULL), + databuf_(NULL), + databuf_strides_(NULL) { + decompress_struct_ = new jpeg_decompress_struct; + source_mgr_ = new jpeg_source_mgr; +#ifdef HAVE_SETJMP + error_mgr_ = new SetJmpErrorMgr; + decompress_struct_->err = jpeg_std_error(&error_mgr_->base); + // Override standard exit()-based error handler. + error_mgr_->base.error_exit = &ErrorHandler; +#endif + decompress_struct_->client_data = NULL; + source_mgr_->init_source = &init_source; + source_mgr_->fill_input_buffer = &fill_input_buffer; + source_mgr_->skip_input_data = &skip_input_data; + source_mgr_->resync_to_restart = &jpeg_resync_to_restart; + source_mgr_->term_source = &term_source; + jpeg_create_decompress(decompress_struct_); + decompress_struct_->src = source_mgr_; + buf_vec_.buffers = &buf_; + buf_vec_.len = 1; +} + +MJpegDecoder::~MJpegDecoder() { + jpeg_destroy_decompress(decompress_struct_); + delete decompress_struct_; + delete source_mgr_; +#ifdef HAVE_SETJMP + delete error_mgr_; +#endif + DestroyOutputBuffers(); +} + +LIBYUV_BOOL MJpegDecoder::LoadFrame(const uint8* src, size_t src_len) { + if (!ValidateJpeg(src, src_len)) { + return LIBYUV_FALSE; + } + + buf_.data = src; + buf_.len = static_cast(src_len); + buf_vec_.pos = 0; + decompress_struct_->client_data = &buf_vec_; +#ifdef HAVE_SETJMP + if (setjmp(error_mgr_->setjmp_buffer)) { + // We called jpeg_read_header, it experienced an error, and we called + // longjmp() and rewound the stack to here. Return error. + return LIBYUV_FALSE; + } +#endif + if (jpeg_read_header(decompress_struct_, TRUE) != JPEG_HEADER_OK) { + // ERROR: Bad MJPEG header + return LIBYUV_FALSE; + } + AllocOutputBuffers(GetNumComponents()); + for (int i = 0; i < num_outbufs_; ++i) { + int scanlines_size = GetComponentScanlinesPerImcuRow(i); + if (scanlines_sizes_[i] != scanlines_size) { + if (scanlines_[i]) { + delete scanlines_[i]; + } + scanlines_[i] = new uint8* [scanlines_size]; + scanlines_sizes_[i] = scanlines_size; + } + + // We allocate padding for the final scanline to pad it up to DCTSIZE bytes + // to avoid memory errors, since jpeglib only reads full MCUs blocks. For + // the preceding scanlines, the padding is not needed/wanted because the + // following addresses will already be valid (they are the initial bytes of + // the next scanline) and will be overwritten when jpeglib writes out that + // next scanline. + int databuf_stride = GetComponentStride(i); + int databuf_size = scanlines_size * databuf_stride; + if (databuf_strides_[i] != databuf_stride) { + if (databuf_[i]) { + delete databuf_[i]; + } + databuf_[i] = new uint8[databuf_size]; + databuf_strides_[i] = databuf_stride; + } + + if (GetComponentStride(i) != GetComponentWidth(i)) { + has_scanline_padding_ = LIBYUV_TRUE; + } + } + return LIBYUV_TRUE; +} + +static int DivideAndRoundUp(int numerator, int denominator) { + return (numerator + denominator - 1) / denominator; +} + +static int DivideAndRoundDown(int numerator, int denominator) { + return numerator / denominator; +} + +// Returns width of the last loaded frame. +int MJpegDecoder::GetWidth() { + return decompress_struct_->image_width; +} + +// Returns height of the last loaded frame. +int MJpegDecoder::GetHeight() { + return decompress_struct_->image_height; +} + +// Returns format of the last loaded frame. The return value is one of the +// kColorSpace* constants. +int MJpegDecoder::GetColorSpace() { + return decompress_struct_->jpeg_color_space; +} + +// Number of color components in the color space. +int MJpegDecoder::GetNumComponents() { + return decompress_struct_->num_components; +} + +// Sample factors of the n-th component. +int MJpegDecoder::GetHorizSampFactor(int component) { + return decompress_struct_->comp_info[component].h_samp_factor; +} + +int MJpegDecoder::GetVertSampFactor(int component) { + return decompress_struct_->comp_info[component].v_samp_factor; +} + +int MJpegDecoder::GetHorizSubSampFactor(int component) { + return decompress_struct_->max_h_samp_factor / + GetHorizSampFactor(component); +} + +int MJpegDecoder::GetVertSubSampFactor(int component) { + return decompress_struct_->max_v_samp_factor / + GetVertSampFactor(component); +} + +int MJpegDecoder::GetImageScanlinesPerImcuRow() { + return decompress_struct_->max_v_samp_factor * DCTSIZE; +} + +int MJpegDecoder::GetComponentScanlinesPerImcuRow(int component) { + int vs = GetVertSubSampFactor(component); + return DivideAndRoundUp(GetImageScanlinesPerImcuRow(), vs); +} + +int MJpegDecoder::GetComponentWidth(int component) { + int hs = GetHorizSubSampFactor(component); + return DivideAndRoundUp(GetWidth(), hs); +} + +int MJpegDecoder::GetComponentHeight(int component) { + int vs = GetVertSubSampFactor(component); + return DivideAndRoundUp(GetHeight(), vs); +} + +// Get width in bytes padded out to a multiple of DCTSIZE +int MJpegDecoder::GetComponentStride(int component) { + return (GetComponentWidth(component) + DCTSIZE - 1) & ~(DCTSIZE - 1); +} + +int MJpegDecoder::GetComponentSize(int component) { + return GetComponentWidth(component) * GetComponentHeight(component); +} + +LIBYUV_BOOL MJpegDecoder::UnloadFrame() { +#ifdef HAVE_SETJMP + if (setjmp(error_mgr_->setjmp_buffer)) { + // We called jpeg_abort_decompress, it experienced an error, and we called + // longjmp() and rewound the stack to here. Return error. + return LIBYUV_FALSE; + } +#endif + jpeg_abort_decompress(decompress_struct_); + return LIBYUV_TRUE; +} + +// TODO(fbarchard): Allow rectangle to be specified: x, y, width, height. +LIBYUV_BOOL MJpegDecoder::DecodeToBuffers( + uint8** planes, int dst_width, int dst_height) { + if (dst_width != GetWidth() || + dst_height > GetHeight()) { + // ERROR: Bad dimensions + return LIBYUV_FALSE; + } +#ifdef HAVE_SETJMP + if (setjmp(error_mgr_->setjmp_buffer)) { + // We called into jpeglib, it experienced an error sometime during this + // function call, and we called longjmp() and rewound the stack to here. + // Return error. + return LIBYUV_FALSE; + } +#endif + if (!StartDecode()) { + return LIBYUV_FALSE; + } + SetScanlinePointers(databuf_); + int lines_left = dst_height; + // Compute amount of lines to skip to implement vertical crop. + // TODO(fbarchard): Ensure skip is a multiple of maximum component + // subsample. ie 2 + int skip = (GetHeight() - dst_height) / 2; + if (skip > 0) { + // There is no API to skip lines in the output data, so we read them + // into the temp buffer. + while (skip >= GetImageScanlinesPerImcuRow()) { + if (!DecodeImcuRow()) { + FinishDecode(); + return LIBYUV_FALSE; + } + skip -= GetImageScanlinesPerImcuRow(); + } + if (skip > 0) { + // Have a partial iMCU row left over to skip. Must read it and then + // copy the parts we want into the destination. + if (!DecodeImcuRow()) { + FinishDecode(); + return LIBYUV_FALSE; + } + for (int i = 0; i < num_outbufs_; ++i) { + // TODO(fbarchard): Compute skip to avoid this + assert(skip % GetVertSubSampFactor(i) == 0); + int rows_to_skip = + DivideAndRoundDown(skip, GetVertSubSampFactor(i)); + int scanlines_to_copy = GetComponentScanlinesPerImcuRow(i) - + rows_to_skip; + int data_to_skip = rows_to_skip * GetComponentStride(i); + CopyPlane(databuf_[i] + data_to_skip, GetComponentStride(i), + planes[i], GetComponentWidth(i), + GetComponentWidth(i), scanlines_to_copy); + planes[i] += scanlines_to_copy * GetComponentWidth(i); + } + lines_left -= (GetImageScanlinesPerImcuRow() - skip); + } + } + + // Read full MCUs but cropped horizontally + for (; lines_left > GetImageScanlinesPerImcuRow(); + lines_left -= GetImageScanlinesPerImcuRow()) { + if (!DecodeImcuRow()) { + FinishDecode(); + return LIBYUV_FALSE; + } + for (int i = 0; i < num_outbufs_; ++i) { + int scanlines_to_copy = GetComponentScanlinesPerImcuRow(i); + CopyPlane(databuf_[i], GetComponentStride(i), + planes[i], GetComponentWidth(i), + GetComponentWidth(i), scanlines_to_copy); + planes[i] += scanlines_to_copy * GetComponentWidth(i); + } + } + + if (lines_left > 0) { + // Have a partial iMCU row left over to decode. + if (!DecodeImcuRow()) { + FinishDecode(); + return LIBYUV_FALSE; + } + for (int i = 0; i < num_outbufs_; ++i) { + int scanlines_to_copy = + DivideAndRoundUp(lines_left, GetVertSubSampFactor(i)); + CopyPlane(databuf_[i], GetComponentStride(i), + planes[i], GetComponentWidth(i), + GetComponentWidth(i), scanlines_to_copy); + planes[i] += scanlines_to_copy * GetComponentWidth(i); + } + } + return FinishDecode(); +} + +LIBYUV_BOOL MJpegDecoder::DecodeToCallback(CallbackFunction fn, void* opaque, + int dst_width, int dst_height) { + if (dst_width != GetWidth() || + dst_height > GetHeight()) { + // ERROR: Bad dimensions + return LIBYUV_FALSE; + } +#ifdef HAVE_SETJMP + if (setjmp(error_mgr_->setjmp_buffer)) { + // We called into jpeglib, it experienced an error sometime during this + // function call, and we called longjmp() and rewound the stack to here. + // Return error. + return LIBYUV_FALSE; + } +#endif + if (!StartDecode()) { + return LIBYUV_FALSE; + } + SetScanlinePointers(databuf_); + int lines_left = dst_height; + // TODO(fbarchard): Compute amount of lines to skip to implement vertical crop + int skip = (GetHeight() - dst_height) / 2; + if (skip > 0) { + while (skip >= GetImageScanlinesPerImcuRow()) { + if (!DecodeImcuRow()) { + FinishDecode(); + return LIBYUV_FALSE; + } + skip -= GetImageScanlinesPerImcuRow(); + } + if (skip > 0) { + // Have a partial iMCU row left over to skip. + if (!DecodeImcuRow()) { + FinishDecode(); + return LIBYUV_FALSE; + } + for (int i = 0; i < num_outbufs_; ++i) { + // TODO(fbarchard): Compute skip to avoid this + assert(skip % GetVertSubSampFactor(i) == 0); + int rows_to_skip = DivideAndRoundDown(skip, GetVertSubSampFactor(i)); + int data_to_skip = rows_to_skip * GetComponentStride(i); + // Change our own data buffer pointers so we can pass them to the + // callback. + databuf_[i] += data_to_skip; + } + int scanlines_to_copy = GetImageScanlinesPerImcuRow() - skip; + (*fn)(opaque, databuf_, databuf_strides_, scanlines_to_copy); + // Now change them back. + for (int i = 0; i < num_outbufs_; ++i) { + int rows_to_skip = DivideAndRoundDown(skip, GetVertSubSampFactor(i)); + int data_to_skip = rows_to_skip * GetComponentStride(i); + databuf_[i] -= data_to_skip; + } + lines_left -= scanlines_to_copy; + } + } + // Read full MCUs until we get to the crop point. + for (; lines_left >= GetImageScanlinesPerImcuRow(); + lines_left -= GetImageScanlinesPerImcuRow()) { + if (!DecodeImcuRow()) { + FinishDecode(); + return LIBYUV_FALSE; + } + (*fn)(opaque, databuf_, databuf_strides_, GetImageScanlinesPerImcuRow()); + } + if (lines_left > 0) { + // Have a partial iMCU row left over to decode. + if (!DecodeImcuRow()) { + FinishDecode(); + return LIBYUV_FALSE; + } + (*fn)(opaque, databuf_, databuf_strides_, lines_left); + } + return FinishDecode(); +} + +void init_source(j_decompress_ptr cinfo) { + fill_input_buffer(cinfo); +} + +boolean fill_input_buffer(j_decompress_ptr cinfo) { + BufferVector* buf_vec = reinterpret_cast(cinfo->client_data); + if (buf_vec->pos >= buf_vec->len) { + assert(0 && "No more data"); + // ERROR: No more data + return FALSE; + } + cinfo->src->next_input_byte = buf_vec->buffers[buf_vec->pos].data; + cinfo->src->bytes_in_buffer = buf_vec->buffers[buf_vec->pos].len; + ++buf_vec->pos; + return TRUE; +} + +void skip_input_data(j_decompress_ptr cinfo, + long num_bytes) { // NOLINT + cinfo->src->next_input_byte += num_bytes; +} + +void term_source(j_decompress_ptr cinfo) { + // Nothing to do. +} + +#ifdef HAVE_SETJMP +void ErrorHandler(j_common_ptr cinfo) { + // This is called when a jpeglib command experiences an error. Unfortunately + // jpeglib's error handling model is not very flexible, because it expects the + // error handler to not return--i.e., it wants the program to terminate. To + // recover from errors we use setjmp() as shown in their example. setjmp() is + // C's implementation for the "call with current continuation" functionality + // seen in some functional programming languages. + // A formatted message can be output, but is unsafe for release. +#ifdef DEBUG + char buf[JMSG_LENGTH_MAX]; + (*cinfo->err->format_message)(cinfo, buf); + // ERROR: Error in jpeglib: buf +#endif + + SetJmpErrorMgr* mgr = reinterpret_cast(cinfo->err); + // This rewinds the call stack to the point of the corresponding setjmp() + // and causes it to return (for a second time) with value 1. + longjmp(mgr->setjmp_buffer, 1); +} +#endif + +void MJpegDecoder::AllocOutputBuffers(int num_outbufs) { + if (num_outbufs != num_outbufs_) { + // We could perhaps optimize this case to resize the output buffers without + // necessarily having to delete and recreate each one, but it's not worth + // it. + DestroyOutputBuffers(); + + scanlines_ = new uint8** [num_outbufs]; + scanlines_sizes_ = new int[num_outbufs]; + databuf_ = new uint8* [num_outbufs]; + databuf_strides_ = new int[num_outbufs]; + + for (int i = 0; i < num_outbufs; ++i) { + scanlines_[i] = NULL; + scanlines_sizes_[i] = 0; + databuf_[i] = NULL; + databuf_strides_[i] = 0; + } + + num_outbufs_ = num_outbufs; + } +} + +void MJpegDecoder::DestroyOutputBuffers() { + for (int i = 0; i < num_outbufs_; ++i) { + delete [] scanlines_[i]; + delete [] databuf_[i]; + } + delete [] scanlines_; + delete [] databuf_; + delete [] scanlines_sizes_; + delete [] databuf_strides_; + scanlines_ = NULL; + databuf_ = NULL; + scanlines_sizes_ = NULL; + databuf_strides_ = NULL; + num_outbufs_ = 0; +} + +// JDCT_IFAST and do_block_smoothing improve performance substantially. +LIBYUV_BOOL MJpegDecoder::StartDecode() { + decompress_struct_->raw_data_out = TRUE; + decompress_struct_->dct_method = JDCT_IFAST; // JDCT_ISLOW is default + decompress_struct_->dither_mode = JDITHER_NONE; + // Not applicable to 'raw': + decompress_struct_->do_fancy_upsampling = (boolean)(LIBYUV_FALSE); + // Only for buffered mode: + decompress_struct_->enable_2pass_quant = (boolean)(LIBYUV_FALSE); + // Blocky but fast: + decompress_struct_->do_block_smoothing = (boolean)(LIBYUV_FALSE); + + if (!jpeg_start_decompress(decompress_struct_)) { + // ERROR: Couldn't start JPEG decompressor"; + return LIBYUV_FALSE; + } + return LIBYUV_TRUE; +} + +LIBYUV_BOOL MJpegDecoder::FinishDecode() { + // jpeglib considers it an error if we finish without decoding the whole + // image, so we call "abort" rather than "finish". + jpeg_abort_decompress(decompress_struct_); + return LIBYUV_TRUE; +} + +void MJpegDecoder::SetScanlinePointers(uint8** data) { + for (int i = 0; i < num_outbufs_; ++i) { + uint8* data_i = data[i]; + for (int j = 0; j < scanlines_sizes_[i]; ++j) { + scanlines_[i][j] = data_i; + data_i += GetComponentStride(i); + } + } +} + +inline LIBYUV_BOOL MJpegDecoder::DecodeImcuRow() { + return (unsigned int)(GetImageScanlinesPerImcuRow()) == + jpeg_read_raw_data(decompress_struct_, + scanlines_, + GetImageScanlinesPerImcuRow()); +} + +// The helper function which recognizes the jpeg sub-sampling type. +JpegSubsamplingType MJpegDecoder::JpegSubsamplingTypeHelper( + int* subsample_x, int* subsample_y, int number_of_components) { + if (number_of_components == 3) { // Color images. + if (subsample_x[0] == 1 && subsample_y[0] == 1 && + subsample_x[1] == 2 && subsample_y[1] == 2 && + subsample_x[2] == 2 && subsample_y[2] == 2) { + return kJpegYuv420; + } else if (subsample_x[0] == 1 && subsample_y[0] == 1 && + subsample_x[1] == 2 && subsample_y[1] == 1 && + subsample_x[2] == 2 && subsample_y[2] == 1) { + return kJpegYuv422; + } else if (subsample_x[0] == 1 && subsample_y[0] == 1 && + subsample_x[1] == 1 && subsample_y[1] == 1 && + subsample_x[2] == 1 && subsample_y[2] == 1) { + return kJpegYuv444; + } + } else if (number_of_components == 1) { // Grey-scale images. + if (subsample_x[0] == 1 && subsample_y[0] == 1) { + return kJpegYuv400; + } + } + return kJpegUnknown; +} + +} // namespace libyuv +#endif // HAVE_JPEG + diff --git a/third_party/aom/third_party/libyuv/source/mjpeg_validate.cc b/third_party/aom/third_party/libyuv/source/mjpeg_validate.cc new file mode 100644 index 0000000000..8edfbe1e74 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/mjpeg_validate.cc @@ -0,0 +1,101 @@ +/* + * Copyright 2012 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/mjpeg_decoder.h" + +#include // For memchr. + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Enable this to try scasb implementation. +// #define ENABLE_SCASB 1 + +#ifdef ENABLE_SCASB + +// Multiple of 1. +__declspec(naked) +const uint8* ScanRow_ERMS(const uint8* src, uint32 val, int count) { + __asm { + mov edx, edi + mov edi, [esp + 4] // src + mov eax, [esp + 8] // val + mov ecx, [esp + 12] // count + repne scasb + jne sr99 + mov eax, edi + sub eax, 1 + mov edi, edx + ret + + sr99: + mov eax, 0 + mov edi, edx + ret + } +} +#endif + +// Helper function to scan for EOI marker. +static LIBYUV_BOOL ScanEOI(const uint8* sample, size_t sample_size) { + const uint8* end = sample + sample_size - 1; + const uint8* it = sample; + for (;;) { +#ifdef ENABLE_SCASB + it = ScanRow_ERMS(it, 0xff, end - it); +#else + it = static_cast(memchr(it, 0xff, end - it)); +#endif + if (it == NULL) { + break; + } + if (it[1] == 0xd9) { + return LIBYUV_TRUE; // Success: Valid jpeg. + } + ++it; // Skip over current 0xff. + } + // ERROR: Invalid jpeg end code not found. Size sample_size + return LIBYUV_FALSE; +} + +// Helper function to validate the jpeg appears intact. +LIBYUV_BOOL ValidateJpeg(const uint8* sample, size_t sample_size) { + const size_t kBackSearchSize = 1024; + if (sample_size < 64) { + // ERROR: Invalid jpeg size: sample_size + return LIBYUV_FALSE; + } + if (sample[0] != 0xff || sample[1] != 0xd8) { // Start Of Image + // ERROR: Invalid jpeg initial start code + return LIBYUV_FALSE; + } + // Step over SOI marker. + sample += 2; + sample_size -= 2; + + // Look for the End Of Image (EOI) marker in the end kilobyte of the buffer. + if (sample_size > kBackSearchSize) { + if (ScanEOI(sample + sample_size - kBackSearchSize, kBackSearchSize)) { + return LIBYUV_TRUE; // Success: Valid jpeg. + } + // Reduce search size for forward search. + sample_size = sample_size - kBackSearchSize + 1; + } + return ScanEOI(sample, sample_size); + +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + diff --git a/third_party/aom/third_party/libyuv/source/planar_functions.cc b/third_party/aom/third_party/libyuv/source/planar_functions.cc new file mode 100644 index 0000000000..b96bd50206 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/planar_functions.cc @@ -0,0 +1,2555 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/planar_functions.h" + +#include // for memset() + +#include "libyuv/cpu_id.h" +#ifdef HAVE_JPEG +#include "libyuv/mjpeg_decoder.h" +#endif +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Copy a plane of data +LIBYUV_API +void CopyPlane(const uint8* src_y, int src_stride_y, + uint8* dst_y, int dst_stride_y, + int width, int height) { + int y; + void (*CopyRow)(const uint8* src, uint8* dst, int width) = CopyRow_C; + // Coalesce rows. + if (src_stride_y == width && + dst_stride_y == width) { + width *= height; + height = 1; + src_stride_y = dst_stride_y = 0; + } + // Nothing to do. + if (src_y == dst_y && src_stride_y == dst_stride_y) { + return; + } +#if defined(HAS_COPYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2; + } +#endif +#if defined(HAS_COPYROW_AVX) + if (TestCpuFlag(kCpuHasAVX)) { + CopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX; + } +#endif +#if defined(HAS_COPYROW_ERMS) + if (TestCpuFlag(kCpuHasERMS)) { + CopyRow = CopyRow_ERMS; + } +#endif +#if defined(HAS_COPYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON; + } +#endif +#if defined(HAS_COPYROW_MIPS) + if (TestCpuFlag(kCpuHasMIPS)) { + CopyRow = CopyRow_MIPS; + } +#endif + + // Copy plane + for (y = 0; y < height; ++y) { + CopyRow(src_y, dst_y, width); + src_y += src_stride_y; + dst_y += dst_stride_y; + } +} + +LIBYUV_API +void CopyPlane_16(const uint16* src_y, int src_stride_y, + uint16* dst_y, int dst_stride_y, + int width, int height) { + int y; + void (*CopyRow)(const uint16* src, uint16* dst, int width) = CopyRow_16_C; + // Coalesce rows. + if (src_stride_y == width && + dst_stride_y == width) { + width *= height; + height = 1; + src_stride_y = dst_stride_y = 0; + } +#if defined(HAS_COPYROW_16_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 32)) { + CopyRow = CopyRow_16_SSE2; + } +#endif +#if defined(HAS_COPYROW_16_ERMS) + if (TestCpuFlag(kCpuHasERMS)) { + CopyRow = CopyRow_16_ERMS; + } +#endif +#if defined(HAS_COPYROW_16_NEON) + if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 32)) { + CopyRow = CopyRow_16_NEON; + } +#endif +#if defined(HAS_COPYROW_16_MIPS) + if (TestCpuFlag(kCpuHasMIPS)) { + CopyRow = CopyRow_16_MIPS; + } +#endif + + // Copy plane + for (y = 0; y < height; ++y) { + CopyRow(src_y, dst_y, width); + src_y += src_stride_y; + dst_y += dst_stride_y; + } +} + +// Copy I422. +LIBYUV_API +int I422Copy(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int halfwidth = (width + 1) >> 1; + if (!src_y || !src_u || !src_v || + !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_y = src_y + (height - 1) * src_stride_y; + src_u = src_u + (height - 1) * src_stride_u; + src_v = src_v + (height - 1) * src_stride_v; + src_stride_y = -src_stride_y; + src_stride_u = -src_stride_u; + src_stride_v = -src_stride_v; + } + CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height); + CopyPlane(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth, height); + CopyPlane(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth, height); + return 0; +} + +// Copy I444. +LIBYUV_API +int I444Copy(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + if (!src_y || !src_u || !src_v || + !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_y = src_y + (height - 1) * src_stride_y; + src_u = src_u + (height - 1) * src_stride_u; + src_v = src_v + (height - 1) * src_stride_v; + src_stride_y = -src_stride_y; + src_stride_u = -src_stride_u; + src_stride_v = -src_stride_v; + } + + CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height); + CopyPlane(src_u, src_stride_u, dst_u, dst_stride_u, width, height); + CopyPlane(src_v, src_stride_v, dst_v, dst_stride_v, width, height); + return 0; +} + +// Copy I400. +LIBYUV_API +int I400ToI400(const uint8* src_y, int src_stride_y, + uint8* dst_y, int dst_stride_y, + int width, int height) { + if (!src_y || !dst_y || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_y = src_y + (height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } + CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height); + return 0; +} + +// Convert I420 to I400. +LIBYUV_API +int I420ToI400(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + int width, int height) { + if (!src_y || !dst_y || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_y = src_y + (height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } + CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height); + return 0; +} + +// Mirror a plane of data. +void MirrorPlane(const uint8* src_y, int src_stride_y, + uint8* dst_y, int dst_stride_y, + int width, int height) { + int y; + void (*MirrorRow)(const uint8* src, uint8* dst, int width) = MirrorRow_C; + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_y = src_y + (height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } +#if defined(HAS_MIRRORROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + MirrorRow = MirrorRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + MirrorRow = MirrorRow_NEON; + } + } +#endif +#if defined(HAS_MIRRORROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + MirrorRow = MirrorRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + MirrorRow = MirrorRow_SSE2; + } + } +#endif +#if defined(HAS_MIRRORROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + MirrorRow = MirrorRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + MirrorRow = MirrorRow_SSSE3; + } + } +#endif +#if defined(HAS_MIRRORROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + MirrorRow = MirrorRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + MirrorRow = MirrorRow_AVX2; + } + } +#endif +// TODO(fbarchard): Mirror on mips handle unaligned memory. +#if defined(HAS_MIRRORROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && + IS_ALIGNED(src_y, 4) && IS_ALIGNED(src_stride_y, 4) && + IS_ALIGNED(dst_y, 4) && IS_ALIGNED(dst_stride_y, 4)) { + MirrorRow = MirrorRow_MIPS_DSPR2; + } +#endif + + // Mirror plane + for (y = 0; y < height; ++y) { + MirrorRow(src_y, dst_y, width); + src_y += src_stride_y; + dst_y += dst_stride_y; + } +} + +// Convert YUY2 to I422. +LIBYUV_API +int YUY2ToI422(const uint8* src_yuy2, int src_stride_yuy2, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*YUY2ToUV422Row)(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int pix) = + YUY2ToUV422Row_C; + void (*YUY2ToYRow)(const uint8* src_yuy2, uint8* dst_y, int pix) = + YUY2ToYRow_C; + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_yuy2 = src_yuy2 + (height - 1) * src_stride_yuy2; + src_stride_yuy2 = -src_stride_yuy2; + } + // Coalesce rows. + if (src_stride_yuy2 == width * 2 && + dst_stride_y == width && + dst_stride_u * 2 == width && + dst_stride_v * 2 == width) { + width *= height; + height = 1; + src_stride_yuy2 = dst_stride_y = dst_stride_u = dst_stride_v = 0; + } +#if defined(HAS_YUY2TOYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + YUY2ToUV422Row = YUY2ToUV422Row_Any_SSE2; + YUY2ToYRow = YUY2ToYRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + YUY2ToUV422Row = YUY2ToUV422Row_SSE2; + YUY2ToYRow = YUY2ToYRow_SSE2; + } + } +#endif +#if defined(HAS_YUY2TOYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + YUY2ToUV422Row = YUY2ToUV422Row_Any_AVX2; + YUY2ToYRow = YUY2ToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + YUY2ToUV422Row = YUY2ToUV422Row_AVX2; + YUY2ToYRow = YUY2ToYRow_AVX2; + } + } +#endif +#if defined(HAS_YUY2TOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + YUY2ToYRow = YUY2ToYRow_Any_NEON; + if (width >= 16) { + YUY2ToUV422Row = YUY2ToUV422Row_Any_NEON; + } + if (IS_ALIGNED(width, 16)) { + YUY2ToYRow = YUY2ToYRow_NEON; + YUY2ToUV422Row = YUY2ToUV422Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + YUY2ToUV422Row(src_yuy2, dst_u, dst_v, width); + YUY2ToYRow(src_yuy2, dst_y, width); + src_yuy2 += src_stride_yuy2; + dst_y += dst_stride_y; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + return 0; +} + +// Convert UYVY to I422. +LIBYUV_API +int UYVYToI422(const uint8* src_uyvy, int src_stride_uyvy, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int y; + void (*UYVYToUV422Row)(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix) = + UYVYToUV422Row_C; + void (*UYVYToYRow)(const uint8* src_uyvy, + uint8* dst_y, int pix) = UYVYToYRow_C; + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_uyvy = src_uyvy + (height - 1) * src_stride_uyvy; + src_stride_uyvy = -src_stride_uyvy; + } + // Coalesce rows. + if (src_stride_uyvy == width * 2 && + dst_stride_y == width && + dst_stride_u * 2 == width && + dst_stride_v * 2 == width) { + width *= height; + height = 1; + src_stride_uyvy = dst_stride_y = dst_stride_u = dst_stride_v = 0; + } +#if defined(HAS_UYVYTOYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + UYVYToUV422Row = UYVYToUV422Row_Any_SSE2; + UYVYToYRow = UYVYToYRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + UYVYToUV422Row = UYVYToUV422Row_SSE2; + UYVYToYRow = UYVYToYRow_SSE2; + } + } +#endif +#if defined(HAS_UYVYTOYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + UYVYToUV422Row = UYVYToUV422Row_Any_AVX2; + UYVYToYRow = UYVYToYRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + UYVYToUV422Row = UYVYToUV422Row_AVX2; + UYVYToYRow = UYVYToYRow_AVX2; + } + } +#endif +#if defined(HAS_UYVYTOYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + UYVYToYRow = UYVYToYRow_Any_NEON; + if (width >= 16) { + UYVYToUV422Row = UYVYToUV422Row_Any_NEON; + } + if (IS_ALIGNED(width, 16)) { + UYVYToYRow = UYVYToYRow_NEON; + UYVYToUV422Row = UYVYToUV422Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + UYVYToUV422Row(src_uyvy, dst_u, dst_v, width); + UYVYToYRow(src_uyvy, dst_y, width); + src_uyvy += src_stride_uyvy; + dst_y += dst_stride_y; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + return 0; +} + +// Mirror I400 with optional flipping +LIBYUV_API +int I400Mirror(const uint8* src_y, int src_stride_y, + uint8* dst_y, int dst_stride_y, + int width, int height) { + if (!src_y || !dst_y || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_y = src_y + (height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } + + MirrorPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height); + return 0; +} + +// Mirror I420 with optional flipping +LIBYUV_API +int I420Mirror(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height) { + int halfwidth = (width + 1) >> 1; + int halfheight = (height + 1) >> 1; + if (!src_y || !src_u || !src_v || !dst_y || !dst_u || !dst_v || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + halfheight = (height + 1) >> 1; + src_y = src_y + (height - 1) * src_stride_y; + src_u = src_u + (halfheight - 1) * src_stride_u; + src_v = src_v + (halfheight - 1) * src_stride_v; + src_stride_y = -src_stride_y; + src_stride_u = -src_stride_u; + src_stride_v = -src_stride_v; + } + + if (dst_y) { + MirrorPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height); + } + MirrorPlane(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth, halfheight); + MirrorPlane(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth, halfheight); + return 0; +} + +// ARGB mirror. +LIBYUV_API +int ARGBMirror(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*ARGBMirrorRow)(const uint8* src, uint8* dst, int width) = + ARGBMirrorRow_C; + if (!src_argb || !dst_argb || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } +#if defined(HAS_ARGBMIRRORROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBMirrorRow = ARGBMirrorRow_Any_NEON; + if (IS_ALIGNED(width, 4)) { + ARGBMirrorRow = ARGBMirrorRow_NEON; + } + } +#endif +#if defined(HAS_ARGBMIRRORROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBMirrorRow = ARGBMirrorRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBMirrorRow = ARGBMirrorRow_SSE2; + } + } +#endif +#if defined(HAS_ARGBMIRRORROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBMirrorRow = ARGBMirrorRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBMirrorRow = ARGBMirrorRow_AVX2; + } + } +#endif + + // Mirror plane + for (y = 0; y < height; ++y) { + ARGBMirrorRow(src_argb, dst_argb, width); + src_argb += src_stride_argb; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Get a blender that optimized for the CPU and pixel count. +// As there are 6 blenders to choose from, the caller should try to use +// the same blend function for all pixels if possible. +LIBYUV_API +ARGBBlendRow GetARGBBlend() { + void (*ARGBBlendRow)(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width) = ARGBBlendRow_C; +#if defined(HAS_ARGBBLENDROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBBlendRow = ARGBBlendRow_SSSE3; + return ARGBBlendRow; + } +#endif +#if defined(HAS_ARGBBLENDROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBBlendRow = ARGBBlendRow_SSE2; + } +#endif +#if defined(HAS_ARGBBLENDROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBBlendRow = ARGBBlendRow_NEON; + } +#endif + return ARGBBlendRow; +} + +// Alpha Blend 2 ARGB images and store to destination. +LIBYUV_API +int ARGBBlend(const uint8* src_argb0, int src_stride_argb0, + const uint8* src_argb1, int src_stride_argb1, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*ARGBBlendRow)(const uint8* src_argb, const uint8* src_argb1, + uint8* dst_argb, int width) = GetARGBBlend(); + if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + // Coalesce rows. + if (src_stride_argb0 == width * 4 && + src_stride_argb1 == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0; + } + + for (y = 0; y < height; ++y) { + ARGBBlendRow(src_argb0, src_argb1, dst_argb, width); + src_argb0 += src_stride_argb0; + src_argb1 += src_stride_argb1; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Multiply 2 ARGB images and store to destination. +LIBYUV_API +int ARGBMultiply(const uint8* src_argb0, int src_stride_argb0, + const uint8* src_argb1, int src_stride_argb1, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*ARGBMultiplyRow)(const uint8* src0, const uint8* src1, uint8* dst, + int width) = ARGBMultiplyRow_C; + if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + // Coalesce rows. + if (src_stride_argb0 == width * 4 && + src_stride_argb1 == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0; + } +#if defined(HAS_ARGBMULTIPLYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBMultiplyRow = ARGBMultiplyRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBMultiplyRow = ARGBMultiplyRow_SSE2; + } + } +#endif +#if defined(HAS_ARGBMULTIPLYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBMultiplyRow = ARGBMultiplyRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBMultiplyRow = ARGBMultiplyRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBMULTIPLYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBMultiplyRow = ARGBMultiplyRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBMultiplyRow = ARGBMultiplyRow_NEON; + } + } +#endif + + // Multiply plane + for (y = 0; y < height; ++y) { + ARGBMultiplyRow(src_argb0, src_argb1, dst_argb, width); + src_argb0 += src_stride_argb0; + src_argb1 += src_stride_argb1; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Add 2 ARGB images and store to destination. +LIBYUV_API +int ARGBAdd(const uint8* src_argb0, int src_stride_argb0, + const uint8* src_argb1, int src_stride_argb1, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*ARGBAddRow)(const uint8* src0, const uint8* src1, uint8* dst, + int width) = ARGBAddRow_C; + if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + // Coalesce rows. + if (src_stride_argb0 == width * 4 && + src_stride_argb1 == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0; + } +#if defined(HAS_ARGBADDROW_SSE2) && (defined(_MSC_VER) && !defined(__clang__)) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBAddRow = ARGBAddRow_SSE2; + } +#endif +#if defined(HAS_ARGBADDROW_SSE2) && !(defined(_MSC_VER) && !defined(__clang__)) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBAddRow = ARGBAddRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBAddRow = ARGBAddRow_SSE2; + } + } +#endif +#if defined(HAS_ARGBADDROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBAddRow = ARGBAddRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBAddRow = ARGBAddRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBADDROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBAddRow = ARGBAddRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBAddRow = ARGBAddRow_NEON; + } + } +#endif + + // Add plane + for (y = 0; y < height; ++y) { + ARGBAddRow(src_argb0, src_argb1, dst_argb, width); + src_argb0 += src_stride_argb0; + src_argb1 += src_stride_argb1; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Subtract 2 ARGB images and store to destination. +LIBYUV_API +int ARGBSubtract(const uint8* src_argb0, int src_stride_argb0, + const uint8* src_argb1, int src_stride_argb1, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*ARGBSubtractRow)(const uint8* src0, const uint8* src1, uint8* dst, + int width) = ARGBSubtractRow_C; + if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + // Coalesce rows. + if (src_stride_argb0 == width * 4 && + src_stride_argb1 == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0; + } +#if defined(HAS_ARGBSUBTRACTROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBSubtractRow = ARGBSubtractRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBSubtractRow = ARGBSubtractRow_SSE2; + } + } +#endif +#if defined(HAS_ARGBSUBTRACTROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBSubtractRow = ARGBSubtractRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBSubtractRow = ARGBSubtractRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBSUBTRACTROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBSubtractRow = ARGBSubtractRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBSubtractRow = ARGBSubtractRow_NEON; + } + } +#endif + + // Subtract plane + for (y = 0; y < height; ++y) { + ARGBSubtractRow(src_argb0, src_argb1, dst_argb, width); + src_argb0 += src_stride_argb0; + src_argb1 += src_stride_argb1; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Convert I422 to BGRA. +LIBYUV_API +int I422ToBGRA(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_bgra, int dst_stride_bgra, + int width, int height) { + int y; + void (*I422ToBGRARow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToBGRARow_C; + if (!src_y || !src_u || !src_v || + !dst_bgra || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_bgra = dst_bgra + (height - 1) * dst_stride_bgra; + dst_stride_bgra = -dst_stride_bgra; + } + // Coalesce rows. + if (src_stride_y == width && + src_stride_u * 2 == width && + src_stride_v * 2 == width && + dst_stride_bgra == width * 4) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_bgra = 0; + } +#if defined(HAS_I422TOBGRAROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToBGRARow = I422ToBGRARow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToBGRARow = I422ToBGRARow_SSSE3; + } + } +#endif +#if defined(HAS_I422TOBGRAROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToBGRARow = I422ToBGRARow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToBGRARow = I422ToBGRARow_AVX2; + } + } +#endif +#if defined(HAS_I422TOBGRAROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToBGRARow = I422ToBGRARow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToBGRARow = I422ToBGRARow_NEON; + } + } +#endif +#if defined(HAS_I422TOBGRAROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(width, 4) && + IS_ALIGNED(src_y, 4) && IS_ALIGNED(src_stride_y, 4) && + IS_ALIGNED(src_u, 2) && IS_ALIGNED(src_stride_u, 2) && + IS_ALIGNED(src_v, 2) && IS_ALIGNED(src_stride_v, 2) && + IS_ALIGNED(dst_bgra, 4) && IS_ALIGNED(dst_stride_bgra, 4)) { + I422ToBGRARow = I422ToBGRARow_MIPS_DSPR2; + } +#endif + + for (y = 0; y < height; ++y) { + I422ToBGRARow(src_y, src_u, src_v, dst_bgra, width); + dst_bgra += dst_stride_bgra; + src_y += src_stride_y; + src_u += src_stride_u; + src_v += src_stride_v; + } + return 0; +} + +// Convert I422 to ABGR. +LIBYUV_API +int I422ToABGR(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_abgr, int dst_stride_abgr, + int width, int height) { + int y; + void (*I422ToABGRRow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToABGRRow_C; + if (!src_y || !src_u || !src_v || + !dst_abgr || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_abgr = dst_abgr + (height - 1) * dst_stride_abgr; + dst_stride_abgr = -dst_stride_abgr; + } + // Coalesce rows. + if (src_stride_y == width && + src_stride_u * 2 == width && + src_stride_v * 2 == width && + dst_stride_abgr == width * 4) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_abgr = 0; + } +#if defined(HAS_I422TOABGRROW_NEON) + if (TestCpuFlag(kCpuHasNEON) && width >= 8) { + I422ToABGRRow = I422ToABGRRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToABGRRow = I422ToABGRRow_NEON; + } + } +#endif +#if defined(HAS_I422TOABGRROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToABGRRow = I422ToABGRRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToABGRRow = I422ToABGRRow_SSSE3; + } + } +#endif +#if defined(HAS_I422TOABGRROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToABGRRow = I422ToABGRRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToABGRRow = I422ToABGRRow_AVX2; + } + } +#endif + + for (y = 0; y < height; ++y) { + I422ToABGRRow(src_y, src_u, src_v, dst_abgr, width); + dst_abgr += dst_stride_abgr; + src_y += src_stride_y; + src_u += src_stride_u; + src_v += src_stride_v; + } + return 0; +} + +// Convert I422 to RGBA. +LIBYUV_API +int I422ToRGBA(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_rgba, int dst_stride_rgba, + int width, int height) { + int y; + void (*I422ToRGBARow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToRGBARow_C; + if (!src_y || !src_u || !src_v || + !dst_rgba || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_rgba = dst_rgba + (height - 1) * dst_stride_rgba; + dst_stride_rgba = -dst_stride_rgba; + } + // Coalesce rows. + if (src_stride_y == width && + src_stride_u * 2 == width && + src_stride_v * 2 == width && + dst_stride_rgba == width * 4) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_rgba = 0; + } +#if defined(HAS_I422TORGBAROW_NEON) + if (TestCpuFlag(kCpuHasNEON) && width >= 8) { + I422ToRGBARow = I422ToRGBARow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + I422ToRGBARow = I422ToRGBARow_NEON; + } + } +#endif +#if defined(HAS_I422TORGBAROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToRGBARow = I422ToRGBARow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + I422ToRGBARow = I422ToRGBARow_SSSE3; + } + } +#endif +#if defined(HAS_I422TORGBAROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToRGBARow = I422ToRGBARow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + I422ToRGBARow = I422ToRGBARow_AVX2; + } + } +#endif + + for (y = 0; y < height; ++y) { + I422ToRGBARow(src_y, src_u, src_v, dst_rgba, width); + dst_rgba += dst_stride_rgba; + src_y += src_stride_y; + src_u += src_stride_u; + src_v += src_stride_v; + } + return 0; +} + +// Convert NV12 to RGB565. +LIBYUV_API +int NV12ToRGB565(const uint8* src_y, int src_stride_y, + const uint8* src_uv, int src_stride_uv, + uint8* dst_rgb565, int dst_stride_rgb565, + int width, int height) { + int y; + void (*NV12ToRGB565Row)(const uint8* y_buf, + const uint8* uv_buf, + uint8* rgb_buf, + int width) = NV12ToRGB565Row_C; + if (!src_y || !src_uv || !dst_rgb565 || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_rgb565 = dst_rgb565 + (height - 1) * dst_stride_rgb565; + dst_stride_rgb565 = -dst_stride_rgb565; + } +#if defined(HAS_NV12TORGB565ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + NV12ToRGB565Row = NV12ToRGB565Row_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + NV12ToRGB565Row = NV12ToRGB565Row_SSSE3; + } + } +#endif +#if defined(HAS_NV12TORGB565ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + NV12ToRGB565Row = NV12ToRGB565Row_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + NV12ToRGB565Row = NV12ToRGB565Row_AVX2; + } + } +#endif +#if defined(HAS_NV12TORGB565ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + NV12ToRGB565Row = NV12ToRGB565Row_Any_NEON; + if (IS_ALIGNED(width, 8)) { + NV12ToRGB565Row = NV12ToRGB565Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + NV12ToRGB565Row(src_y, src_uv, dst_rgb565, width); + dst_rgb565 += dst_stride_rgb565; + src_y += src_stride_y; + if (y & 1) { + src_uv += src_stride_uv; + } + } + return 0; +} + +// Convert NV21 to RGB565. +LIBYUV_API +int NV21ToRGB565(const uint8* src_y, int src_stride_y, + const uint8* src_vu, int src_stride_vu, + uint8* dst_rgb565, int dst_stride_rgb565, + int width, int height) { + int y; + void (*NV21ToRGB565Row)(const uint8* y_buf, + const uint8* src_vu, + uint8* rgb_buf, + int width) = NV21ToRGB565Row_C; + if (!src_y || !src_vu || !dst_rgb565 || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_rgb565 = dst_rgb565 + (height - 1) * dst_stride_rgb565; + dst_stride_rgb565 = -dst_stride_rgb565; + } +#if defined(HAS_NV21TORGB565ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + NV21ToRGB565Row = NV21ToRGB565Row_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + NV21ToRGB565Row = NV21ToRGB565Row_SSSE3; + } + } +#endif +#if defined(HAS_NV21TORGB565ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + NV21ToRGB565Row = NV21ToRGB565Row_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + NV21ToRGB565Row = NV21ToRGB565Row_AVX2; + } + } +#endif +#if defined(HAS_NV21TORGB565ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + NV21ToRGB565Row = NV21ToRGB565Row_Any_NEON; + if (IS_ALIGNED(width, 8)) { + NV21ToRGB565Row = NV21ToRGB565Row_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + NV21ToRGB565Row(src_y, src_vu, dst_rgb565, width); + dst_rgb565 += dst_stride_rgb565; + src_y += src_stride_y; + if (y & 1) { + src_vu += src_stride_vu; + } + } + return 0; +} + +LIBYUV_API +void SetPlane(uint8* dst_y, int dst_stride_y, + int width, int height, + uint32 value) { + int y; + void (*SetRow)(uint8* dst, uint8 value, int pix) = SetRow_C; + if (height < 0) { + height = -height; + dst_y = dst_y + (height - 1) * dst_stride_y; + dst_stride_y = -dst_stride_y; + } + // Coalesce rows. + if (dst_stride_y == width) { + width *= height; + height = 1; + dst_stride_y = 0; + } +#if defined(HAS_SETROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + SetRow = SetRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + SetRow = SetRow_NEON; + } + } +#endif +#if defined(HAS_SETROW_X86) + if (TestCpuFlag(kCpuHasX86)) { + SetRow = SetRow_Any_X86; + if (IS_ALIGNED(width, 4)) { + SetRow = SetRow_X86; + } + } +#endif +#if defined(HAS_SETROW_ERMS) + if (TestCpuFlag(kCpuHasERMS)) { + SetRow = SetRow_ERMS; + } +#endif + + // Set plane + for (y = 0; y < height; ++y) { + SetRow(dst_y, value, width); + dst_y += dst_stride_y; + } +} + +// Draw a rectangle into I420 +LIBYUV_API +int I420Rect(uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int x, int y, + int width, int height, + int value_y, int value_u, int value_v) { + int halfwidth = (width + 1) >> 1; + int halfheight = (height + 1) >> 1; + uint8* start_y = dst_y + y * dst_stride_y + x; + uint8* start_u = dst_u + (y / 2) * dst_stride_u + (x / 2); + uint8* start_v = dst_v + (y / 2) * dst_stride_v + (x / 2); + if (!dst_y || !dst_u || !dst_v || + width <= 0 || height == 0 || + x < 0 || y < 0 || + value_y < 0 || value_y > 255 || + value_u < 0 || value_u > 255 || + value_v < 0 || value_v > 255) { + return -1; + } + + SetPlane(start_y, dst_stride_y, width, height, value_y); + SetPlane(start_u, dst_stride_u, halfwidth, halfheight, value_u); + SetPlane(start_v, dst_stride_v, halfwidth, halfheight, value_v); + return 0; +} + +// Draw a rectangle into ARGB +LIBYUV_API +int ARGBRect(uint8* dst_argb, int dst_stride_argb, + int dst_x, int dst_y, + int width, int height, + uint32 value) { + int y; + void (*ARGBSetRow)(uint8* dst_argb, uint32 value, int pix) = ARGBSetRow_C; + if (!dst_argb || + width <= 0 || height == 0 || + dst_x < 0 || dst_y < 0) { + return -1; + } + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + dst_argb += dst_y * dst_stride_argb + dst_x * 4; + // Coalesce rows. + if (dst_stride_argb == width * 4) { + width *= height; + height = 1; + dst_stride_argb = 0; + } + +#if defined(HAS_ARGBSETROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBSetRow = ARGBSetRow_Any_NEON; + if (IS_ALIGNED(width, 4)) { + ARGBSetRow = ARGBSetRow_NEON; + } + } +#endif +#if defined(HAS_ARGBSETROW_X86) + if (TestCpuFlag(kCpuHasX86)) { + ARGBSetRow = ARGBSetRow_X86; + } +#endif + + // Set plane + for (y = 0; y < height; ++y) { + ARGBSetRow(dst_argb, value, width); + dst_argb += dst_stride_argb; + } + return 0; +} + +// Convert unattentuated ARGB to preattenuated ARGB. +// An unattenutated ARGB alpha blend uses the formula +// p = a * f + (1 - a) * b +// where +// p is output pixel +// f is foreground pixel +// b is background pixel +// a is alpha value from foreground pixel +// An preattenutated ARGB alpha blend uses the formula +// p = f + (1 - a) * b +// where +// f is foreground pixel premultiplied by alpha + +LIBYUV_API +int ARGBAttenuate(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*ARGBAttenuateRow)(const uint8* src_argb, uint8* dst_argb, + int width) = ARGBAttenuateRow_C; + if (!src_argb || !dst_argb || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb = dst_stride_argb = 0; + } +#if defined(HAS_ARGBATTENUATEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_SSE2; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBAttenuateRow = ARGBAttenuateRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBAttenuateRow = ARGBAttenuateRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBAttenuateRow(src_argb, dst_argb, width); + src_argb += src_stride_argb; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Convert preattentuated ARGB to unattenuated ARGB. +LIBYUV_API +int ARGBUnattenuate(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*ARGBUnattenuateRow)(const uint8* src_argb, uint8* dst_argb, + int width) = ARGBUnattenuateRow_C; + if (!src_argb || !dst_argb || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb = dst_stride_argb = 0; + } +#if defined(HAS_ARGBUNATTENUATEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBUnattenuateRow = ARGBUnattenuateRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBUnattenuateRow = ARGBUnattenuateRow_SSE2; + } + } +#endif +#if defined(HAS_ARGBUNATTENUATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBUnattenuateRow = ARGBUnattenuateRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBUnattenuateRow = ARGBUnattenuateRow_AVX2; + } + } +#endif +// TODO(fbarchard): Neon version. + + for (y = 0; y < height; ++y) { + ARGBUnattenuateRow(src_argb, dst_argb, width); + src_argb += src_stride_argb; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Convert ARGB to Grayed ARGB. +LIBYUV_API +int ARGBGrayTo(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*ARGBGrayRow)(const uint8* src_argb, uint8* dst_argb, + int width) = ARGBGrayRow_C; + if (!src_argb || !dst_argb || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb = dst_stride_argb = 0; + } +#if defined(HAS_ARGBGRAYROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) { + ARGBGrayRow = ARGBGrayRow_SSSE3; + } +#endif +#if defined(HAS_ARGBGRAYROW_NEON) + if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { + ARGBGrayRow = ARGBGrayRow_NEON; + } +#endif + + for (y = 0; y < height; ++y) { + ARGBGrayRow(src_argb, dst_argb, width); + src_argb += src_stride_argb; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Make a rectangle of ARGB gray scale. +LIBYUV_API +int ARGBGray(uint8* dst_argb, int dst_stride_argb, + int dst_x, int dst_y, + int width, int height) { + int y; + void (*ARGBGrayRow)(const uint8* src_argb, uint8* dst_argb, + int width) = ARGBGrayRow_C; + uint8* dst = dst_argb + dst_y * dst_stride_argb + dst_x * 4; + if (!dst_argb || width <= 0 || height <= 0 || dst_x < 0 || dst_y < 0) { + return -1; + } + // Coalesce rows. + if (dst_stride_argb == width * 4) { + width *= height; + height = 1; + dst_stride_argb = 0; + } +#if defined(HAS_ARGBGRAYROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) { + ARGBGrayRow = ARGBGrayRow_SSSE3; + } +#endif +#if defined(HAS_ARGBGRAYROW_NEON) + if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { + ARGBGrayRow = ARGBGrayRow_NEON; + } +#endif + for (y = 0; y < height; ++y) { + ARGBGrayRow(dst, dst, width); + dst += dst_stride_argb; + } + return 0; +} + +// Make a rectangle of ARGB Sepia tone. +LIBYUV_API +int ARGBSepia(uint8* dst_argb, int dst_stride_argb, + int dst_x, int dst_y, int width, int height) { + int y; + void (*ARGBSepiaRow)(uint8* dst_argb, int width) = ARGBSepiaRow_C; + uint8* dst = dst_argb + dst_y * dst_stride_argb + dst_x * 4; + if (!dst_argb || width <= 0 || height <= 0 || dst_x < 0 || dst_y < 0) { + return -1; + } + // Coalesce rows. + if (dst_stride_argb == width * 4) { + width *= height; + height = 1; + dst_stride_argb = 0; + } +#if defined(HAS_ARGBSEPIAROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) { + ARGBSepiaRow = ARGBSepiaRow_SSSE3; + } +#endif +#if defined(HAS_ARGBSEPIAROW_NEON) + if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { + ARGBSepiaRow = ARGBSepiaRow_NEON; + } +#endif + for (y = 0; y < height; ++y) { + ARGBSepiaRow(dst, width); + dst += dst_stride_argb; + } + return 0; +} + +// Apply a 4x4 matrix to each ARGB pixel. +// Note: Normally for shading, but can be used to swizzle or invert. +LIBYUV_API +int ARGBColorMatrix(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + const int8* matrix_argb, + int width, int height) { + int y; + void (*ARGBColorMatrixRow)(const uint8* src_argb, uint8* dst_argb, + const int8* matrix_argb, int width) = ARGBColorMatrixRow_C; + if (!src_argb || !dst_argb || !matrix_argb || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb = dst_stride_argb = 0; + } +#if defined(HAS_ARGBCOLORMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_SSSE3; + } +#endif +#if defined(HAS_ARGBCOLORMATRIXROW_NEON) + if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_NEON; + } +#endif + for (y = 0; y < height; ++y) { + ARGBColorMatrixRow(src_argb, dst_argb, matrix_argb, width); + src_argb += src_stride_argb; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Apply a 4x3 matrix to each ARGB pixel. +// Deprecated. +LIBYUV_API +int RGBColorMatrix(uint8* dst_argb, int dst_stride_argb, + const int8* matrix_rgb, + int dst_x, int dst_y, int width, int height) { + SIMD_ALIGNED(int8 matrix_argb[16]); + uint8* dst = dst_argb + dst_y * dst_stride_argb + dst_x * 4; + if (!dst_argb || !matrix_rgb || width <= 0 || height <= 0 || + dst_x < 0 || dst_y < 0) { + return -1; + } + + // Convert 4x3 7 bit matrix to 4x4 6 bit matrix. + matrix_argb[0] = matrix_rgb[0] / 2; + matrix_argb[1] = matrix_rgb[1] / 2; + matrix_argb[2] = matrix_rgb[2] / 2; + matrix_argb[3] = matrix_rgb[3] / 2; + matrix_argb[4] = matrix_rgb[4] / 2; + matrix_argb[5] = matrix_rgb[5] / 2; + matrix_argb[6] = matrix_rgb[6] / 2; + matrix_argb[7] = matrix_rgb[7] / 2; + matrix_argb[8] = matrix_rgb[8] / 2; + matrix_argb[9] = matrix_rgb[9] / 2; + matrix_argb[10] = matrix_rgb[10] / 2; + matrix_argb[11] = matrix_rgb[11] / 2; + matrix_argb[14] = matrix_argb[13] = matrix_argb[12] = 0; + matrix_argb[15] = 64; // 1.0 + + return ARGBColorMatrix((const uint8*)(dst), dst_stride_argb, + dst, dst_stride_argb, + &matrix_argb[0], width, height); +} + +// Apply a color table each ARGB pixel. +// Table contains 256 ARGB values. +LIBYUV_API +int ARGBColorTable(uint8* dst_argb, int dst_stride_argb, + const uint8* table_argb, + int dst_x, int dst_y, int width, int height) { + int y; + void (*ARGBColorTableRow)(uint8* dst_argb, const uint8* table_argb, + int width) = ARGBColorTableRow_C; + uint8* dst = dst_argb + dst_y * dst_stride_argb + dst_x * 4; + if (!dst_argb || !table_argb || width <= 0 || height <= 0 || + dst_x < 0 || dst_y < 0) { + return -1; + } + // Coalesce rows. + if (dst_stride_argb == width * 4) { + width *= height; + height = 1; + dst_stride_argb = 0; + } +#if defined(HAS_ARGBCOLORTABLEROW_X86) + if (TestCpuFlag(kCpuHasX86)) { + ARGBColorTableRow = ARGBColorTableRow_X86; + } +#endif + for (y = 0; y < height; ++y) { + ARGBColorTableRow(dst, table_argb, width); + dst += dst_stride_argb; + } + return 0; +} + +// Apply a color table each ARGB pixel but preserve destination alpha. +// Table contains 256 ARGB values. +LIBYUV_API +int RGBColorTable(uint8* dst_argb, int dst_stride_argb, + const uint8* table_argb, + int dst_x, int dst_y, int width, int height) { + int y; + void (*RGBColorTableRow)(uint8* dst_argb, const uint8* table_argb, + int width) = RGBColorTableRow_C; + uint8* dst = dst_argb + dst_y * dst_stride_argb + dst_x * 4; + if (!dst_argb || !table_argb || width <= 0 || height <= 0 || + dst_x < 0 || dst_y < 0) { + return -1; + } + // Coalesce rows. + if (dst_stride_argb == width * 4) { + width *= height; + height = 1; + dst_stride_argb = 0; + } +#if defined(HAS_RGBCOLORTABLEROW_X86) + if (TestCpuFlag(kCpuHasX86)) { + RGBColorTableRow = RGBColorTableRow_X86; + } +#endif + for (y = 0; y < height; ++y) { + RGBColorTableRow(dst, table_argb, width); + dst += dst_stride_argb; + } + return 0; +} + +// ARGBQuantize is used to posterize art. +// e.g. rgb / qvalue * qvalue + qvalue / 2 +// But the low levels implement efficiently with 3 parameters, and could be +// used for other high level operations. +// dst_argb[0] = (b * scale >> 16) * interval_size + interval_offset; +// where scale is 1 / interval_size as a fixed point value. +// The divide is replaces with a multiply by reciprocal fixed point multiply. +// Caveat - although SSE2 saturates, the C function does not and should be used +// with care if doing anything but quantization. +LIBYUV_API +int ARGBQuantize(uint8* dst_argb, int dst_stride_argb, + int scale, int interval_size, int interval_offset, + int dst_x, int dst_y, int width, int height) { + int y; + void (*ARGBQuantizeRow)(uint8* dst_argb, int scale, int interval_size, + int interval_offset, int width) = ARGBQuantizeRow_C; + uint8* dst = dst_argb + dst_y * dst_stride_argb + dst_x * 4; + if (!dst_argb || width <= 0 || height <= 0 || dst_x < 0 || dst_y < 0 || + interval_size < 1 || interval_size > 255) { + return -1; + } + // Coalesce rows. + if (dst_stride_argb == width * 4) { + width *= height; + height = 1; + dst_stride_argb = 0; + } +#if defined(HAS_ARGBQUANTIZEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 4)) { + ARGBQuantizeRow = ARGBQuantizeRow_SSE2; + } +#endif +#if defined(HAS_ARGBQUANTIZEROW_NEON) + if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { + ARGBQuantizeRow = ARGBQuantizeRow_NEON; + } +#endif + for (y = 0; y < height; ++y) { + ARGBQuantizeRow(dst, scale, interval_size, interval_offset, width); + dst += dst_stride_argb; + } + return 0; +} + +// Computes table of cumulative sum for image where the value is the sum +// of all values above and to the left of the entry. Used by ARGBBlur. +LIBYUV_API +int ARGBComputeCumulativeSum(const uint8* src_argb, int src_stride_argb, + int32* dst_cumsum, int dst_stride32_cumsum, + int width, int height) { + int y; + void (*ComputeCumulativeSumRow)(const uint8* row, int32* cumsum, + const int32* previous_cumsum, int width) = ComputeCumulativeSumRow_C; + int32* previous_cumsum = dst_cumsum; + if (!dst_cumsum || !src_argb || width <= 0 || height <= 0) { + return -1; + } +#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ComputeCumulativeSumRow = ComputeCumulativeSumRow_SSE2; + } +#endif + memset(dst_cumsum, 0, width * sizeof(dst_cumsum[0]) * 4); // 4 int per pixel. + for (y = 0; y < height; ++y) { + ComputeCumulativeSumRow(src_argb, dst_cumsum, previous_cumsum, width); + previous_cumsum = dst_cumsum; + dst_cumsum += dst_stride32_cumsum; + src_argb += src_stride_argb; + } + return 0; +} + +// Blur ARGB image. +// Caller should allocate CumulativeSum table of width * height * 16 bytes +// aligned to 16 byte boundary. height can be radius * 2 + 2 to save memory +// as the buffer is treated as circular. +LIBYUV_API +int ARGBBlur(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int32* dst_cumsum, int dst_stride32_cumsum, + int width, int height, int radius) { + int y; + void (*ComputeCumulativeSumRow)(const uint8 *row, int32 *cumsum, + const int32* previous_cumsum, int width) = ComputeCumulativeSumRow_C; + void (*CumulativeSumToAverageRow)(const int32* topleft, const int32* botleft, + int width, int area, uint8* dst, int count) = CumulativeSumToAverageRow_C; + int32* cumsum_bot_row; + int32* max_cumsum_bot_row; + int32* cumsum_top_row; + + if (!src_argb || !dst_argb || width <= 0 || height == 0) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + if (radius > height) { + radius = height; + } + if (radius > (width / 2 - 1)) { + radius = width / 2 - 1; + } + if (radius <= 0) { + return -1; + } +#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ComputeCumulativeSumRow = ComputeCumulativeSumRow_SSE2; + CumulativeSumToAverageRow = CumulativeSumToAverageRow_SSE2; + } +#endif + // Compute enough CumulativeSum for first row to be blurred. After this + // one row of CumulativeSum is updated at a time. + ARGBComputeCumulativeSum(src_argb, src_stride_argb, + dst_cumsum, dst_stride32_cumsum, + width, radius); + + src_argb = src_argb + radius * src_stride_argb; + cumsum_bot_row = &dst_cumsum[(radius - 1) * dst_stride32_cumsum]; + + max_cumsum_bot_row = &dst_cumsum[(radius * 2 + 2) * dst_stride32_cumsum]; + cumsum_top_row = &dst_cumsum[0]; + + for (y = 0; y < height; ++y) { + int top_y = ((y - radius - 1) >= 0) ? (y - radius - 1) : 0; + int bot_y = ((y + radius) < height) ? (y + radius) : (height - 1); + int area = radius * (bot_y - top_y); + int boxwidth = radius * 4; + int x; + int n; + + // Increment cumsum_top_row pointer with circular buffer wrap around. + if (top_y) { + cumsum_top_row += dst_stride32_cumsum; + if (cumsum_top_row >= max_cumsum_bot_row) { + cumsum_top_row = dst_cumsum; + } + } + // Increment cumsum_bot_row pointer with circular buffer wrap around and + // then fill in a row of CumulativeSum. + if ((y + radius) < height) { + const int32* prev_cumsum_bot_row = cumsum_bot_row; + cumsum_bot_row += dst_stride32_cumsum; + if (cumsum_bot_row >= max_cumsum_bot_row) { + cumsum_bot_row = dst_cumsum; + } + ComputeCumulativeSumRow(src_argb, cumsum_bot_row, prev_cumsum_bot_row, + width); + src_argb += src_stride_argb; + } + + // Left clipped. + for (x = 0; x < radius + 1; ++x) { + CumulativeSumToAverageRow(cumsum_top_row, cumsum_bot_row, + boxwidth, area, &dst_argb[x * 4], 1); + area += (bot_y - top_y); + boxwidth += 4; + } + + // Middle unclipped. + n = (width - 1) - radius - x + 1; + CumulativeSumToAverageRow(cumsum_top_row, cumsum_bot_row, + boxwidth, area, &dst_argb[x * 4], n); + + // Right clipped. + for (x += n; x <= width - 1; ++x) { + area -= (bot_y - top_y); + boxwidth -= 4; + CumulativeSumToAverageRow(cumsum_top_row + (x - radius - 1) * 4, + cumsum_bot_row + (x - radius - 1) * 4, + boxwidth, area, &dst_argb[x * 4], 1); + } + dst_argb += dst_stride_argb; + } + return 0; +} + +// Multiply ARGB image by a specified ARGB value. +LIBYUV_API +int ARGBShade(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height, uint32 value) { + int y; + void (*ARGBShadeRow)(const uint8* src_argb, uint8* dst_argb, + int width, uint32 value) = ARGBShadeRow_C; + if (!src_argb || !dst_argb || width <= 0 || height == 0 || value == 0u) { + return -1; + } + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb = dst_stride_argb = 0; + } +#if defined(HAS_ARGBSHADEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 4)) { + ARGBShadeRow = ARGBShadeRow_SSE2; + } +#endif +#if defined(HAS_ARGBSHADEROW_NEON) + if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { + ARGBShadeRow = ARGBShadeRow_NEON; + } +#endif + + for (y = 0; y < height; ++y) { + ARGBShadeRow(src_argb, dst_argb, width, value); + src_argb += src_stride_argb; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Interpolate 2 ARGB images by specified amount (0 to 255). +LIBYUV_API +int ARGBInterpolate(const uint8* src_argb0, int src_stride_argb0, + const uint8* src_argb1, int src_stride_argb1, + uint8* dst_argb, int dst_stride_argb, + int width, int height, int interpolation) { + int y; + void (*InterpolateRow)(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) = InterpolateRow_C; + if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + // Coalesce rows. + if (src_stride_argb0 == width * 4 && + src_stride_argb1 == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0; + } +#if defined(HAS_INTERPOLATEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + InterpolateRow = InterpolateRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + InterpolateRow = InterpolateRow_SSE2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + InterpolateRow = InterpolateRow_Any_SSSE3; + if (IS_ALIGNED(width, 4)) { + InterpolateRow = InterpolateRow_SSSE3; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_NEON; + if (IS_ALIGNED(width, 4)) { + InterpolateRow = InterpolateRow_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && + IS_ALIGNED(src_argb0, 4) && IS_ALIGNED(src_stride_argb0, 4) && + IS_ALIGNED(src_argb1, 4) && IS_ALIGNED(src_stride_argb1, 4) && + IS_ALIGNED(dst_argb, 4) && IS_ALIGNED(dst_stride_argb, 4)) { + InterpolateRow = InterpolateRow_MIPS_DSPR2; + } +#endif + + for (y = 0; y < height; ++y) { + InterpolateRow(dst_argb, src_argb0, src_argb1 - src_argb0, + width * 4, interpolation); + src_argb0 += src_stride_argb0; + src_argb1 += src_stride_argb1; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Shuffle ARGB channel order. e.g. BGRA to ARGB. +LIBYUV_API +int ARGBShuffle(const uint8* src_bgra, int src_stride_bgra, + uint8* dst_argb, int dst_stride_argb, + const uint8* shuffler, int width, int height) { + int y; + void (*ARGBShuffleRow)(const uint8* src_bgra, uint8* dst_argb, + const uint8* shuffler, int pix) = ARGBShuffleRow_C; + if (!src_bgra || !dst_argb || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_bgra = src_bgra + (height - 1) * src_stride_bgra; + src_stride_bgra = -src_stride_bgra; + } + // Coalesce rows. + if (src_stride_bgra == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_bgra = dst_stride_argb = 0; + } +#if defined(HAS_ARGBSHUFFLEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBShuffleRow = ARGBShuffleRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBShuffleRow = ARGBShuffleRow_SSE2; + } + } +#endif +#if defined(HAS_ARGBSHUFFLEROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBShuffleRow = ARGBShuffleRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + ARGBShuffleRow = ARGBShuffleRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBSHUFFLEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBShuffleRow = ARGBShuffleRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + ARGBShuffleRow = ARGBShuffleRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBSHUFFLEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBShuffleRow = ARGBShuffleRow_Any_NEON; + if (IS_ALIGNED(width, 4)) { + ARGBShuffleRow = ARGBShuffleRow_NEON; + } + } +#endif + + for (y = 0; y < height; ++y) { + ARGBShuffleRow(src_bgra, dst_argb, shuffler, width); + src_bgra += src_stride_bgra; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Sobel ARGB effect. +static int ARGBSobelize(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height, + void (*SobelRow)(const uint8* src_sobelx, + const uint8* src_sobely, + uint8* dst, int width)) { + int y; + void (*ARGBToYJRow)(const uint8* src_argb, uint8* dst_g, int pix) = + ARGBToYJRow_C; + void (*SobelYRow)(const uint8* src_y0, const uint8* src_y1, + uint8* dst_sobely, int width) = SobelYRow_C; + void (*SobelXRow)(const uint8* src_y0, const uint8* src_y1, + const uint8* src_y2, uint8* dst_sobely, int width) = + SobelXRow_C; + const int kEdge = 16; // Extra pixels at start of row for extrude/align. + if (!src_argb || !dst_argb || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + +#if defined(HAS_ARGBTOYJROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYJRow = ARGBToYJRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYJRow = ARGBToYJRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYJROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToYJRow = ARGBToYJRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToYJRow = ARGBToYJRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYJROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYJRow = ARGBToYJRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBToYJRow = ARGBToYJRow_NEON; + } + } +#endif + +#if defined(HAS_SOBELYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + SobelYRow = SobelYRow_SSE2; + } +#endif +#if defined(HAS_SOBELYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + SobelYRow = SobelYRow_NEON; + } +#endif +#if defined(HAS_SOBELXROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + SobelXRow = SobelXRow_SSE2; + } +#endif +#if defined(HAS_SOBELXROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + SobelXRow = SobelXRow_NEON; + } +#endif + { + // 3 rows with edges before/after. + const int kRowSize = (width + kEdge + 31) & ~31; + align_buffer_64(rows, kRowSize * 2 + (kEdge + kRowSize * 3 + kEdge)); + uint8* row_sobelx = rows; + uint8* row_sobely = rows + kRowSize; + uint8* row_y = rows + kRowSize * 2; + + // Convert first row. + uint8* row_y0 = row_y + kEdge; + uint8* row_y1 = row_y0 + kRowSize; + uint8* row_y2 = row_y1 + kRowSize; + ARGBToYJRow(src_argb, row_y0, width); + row_y0[-1] = row_y0[0]; + memset(row_y0 + width, row_y0[width - 1], 16); // Extrude 16 for valgrind. + ARGBToYJRow(src_argb, row_y1, width); + row_y1[-1] = row_y1[0]; + memset(row_y1 + width, row_y1[width - 1], 16); + memset(row_y2 + width, 0, 16); + + for (y = 0; y < height; ++y) { + // Convert next row of ARGB to G. + if (y < (height - 1)) { + src_argb += src_stride_argb; + } + ARGBToYJRow(src_argb, row_y2, width); + row_y2[-1] = row_y2[0]; + row_y2[width] = row_y2[width - 1]; + + SobelXRow(row_y0 - 1, row_y1 - 1, row_y2 - 1, row_sobelx, width); + SobelYRow(row_y0 - 1, row_y2 - 1, row_sobely, width); + SobelRow(row_sobelx, row_sobely, dst_argb, width); + + // Cycle thru circular queue of 3 row_y buffers. + { + uint8* row_yt = row_y0; + row_y0 = row_y1; + row_y1 = row_y2; + row_y2 = row_yt; + } + + dst_argb += dst_stride_argb; + } + free_aligned_buffer_64(rows); + } + return 0; +} + +// Sobel ARGB effect. +LIBYUV_API +int ARGBSobel(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + void (*SobelRow)(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width) = SobelRow_C; +#if defined(HAS_SOBELROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + SobelRow = SobelRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + SobelRow = SobelRow_SSE2; + } + } +#endif +#if defined(HAS_SOBELROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + SobelRow = SobelRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + SobelRow = SobelRow_NEON; + } + } +#endif + return ARGBSobelize(src_argb, src_stride_argb, dst_argb, dst_stride_argb, + width, height, SobelRow); +} + +// Sobel ARGB effect with planar output. +LIBYUV_API +int ARGBSobelToPlane(const uint8* src_argb, int src_stride_argb, + uint8* dst_y, int dst_stride_y, + int width, int height) { + void (*SobelToPlaneRow)(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_, int width) = SobelToPlaneRow_C; +#if defined(HAS_SOBELTOPLANEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + SobelToPlaneRow = SobelToPlaneRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + SobelToPlaneRow = SobelToPlaneRow_SSE2; + } + } +#endif +#if defined(HAS_SOBELTOPLANEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + SobelToPlaneRow = SobelToPlaneRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + SobelToPlaneRow = SobelToPlaneRow_NEON; + } + } +#endif + return ARGBSobelize(src_argb, src_stride_argb, dst_y, dst_stride_y, + width, height, SobelToPlaneRow); +} + +// SobelXY ARGB effect. +// Similar to Sobel, but also stores Sobel X in R and Sobel Y in B. G = Sobel. +LIBYUV_API +int ARGBSobelXY(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + void (*SobelXYRow)(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width) = SobelXYRow_C; +#if defined(HAS_SOBELXYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + SobelXYRow = SobelXYRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + SobelXYRow = SobelXYRow_SSE2; + } + } +#endif +#if defined(HAS_SOBELXYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + SobelXYRow = SobelXYRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + SobelXYRow = SobelXYRow_NEON; + } + } +#endif + return ARGBSobelize(src_argb, src_stride_argb, dst_argb, dst_stride_argb, + width, height, SobelXYRow); +} + +// Apply a 4x4 polynomial to each ARGB pixel. +LIBYUV_API +int ARGBPolynomial(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + const float* poly, + int width, int height) { + int y; + void (*ARGBPolynomialRow)(const uint8* src_argb, + uint8* dst_argb, const float* poly, + int width) = ARGBPolynomialRow_C; + if (!src_argb || !dst_argb || !poly || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb = dst_stride_argb = 0; + } +#if defined(HAS_ARGBPOLYNOMIALROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 2)) { + ARGBPolynomialRow = ARGBPolynomialRow_SSE2; + } +#endif +#if defined(HAS_ARGBPOLYNOMIALROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && TestCpuFlag(kCpuHasFMA3) && + IS_ALIGNED(width, 2)) { + ARGBPolynomialRow = ARGBPolynomialRow_AVX2; + } +#endif + + for (y = 0; y < height; ++y) { + ARGBPolynomialRow(src_argb, dst_argb, poly, width); + src_argb += src_stride_argb; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Apply a lumacolortable to each ARGB pixel. +LIBYUV_API +int ARGBLumaColorTable(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + const uint8* luma, + int width, int height) { + int y; + void (*ARGBLumaColorTableRow)(const uint8* src_argb, uint8* dst_argb, + int width, const uint8* luma, const uint32 lumacoeff) = + ARGBLumaColorTableRow_C; + if (!src_argb || !dst_argb || !luma || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb = dst_stride_argb = 0; + } +#if defined(HAS_ARGBLUMACOLORTABLEROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 4)) { + ARGBLumaColorTableRow = ARGBLumaColorTableRow_SSSE3; + } +#endif + + for (y = 0; y < height; ++y) { + ARGBLumaColorTableRow(src_argb, dst_argb, width, luma, 0x00264b0f); + src_argb += src_stride_argb; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Copy Alpha from one ARGB image to another. +LIBYUV_API +int ARGBCopyAlpha(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*ARGBCopyAlphaRow)(const uint8* src_argb, uint8* dst_argb, int width) = + ARGBCopyAlphaRow_C; + if (!src_argb || !dst_argb || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_argb = dst_stride_argb = 0; + } +#if defined(HAS_ARGBCOPYALPHAROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 8)) { + ARGBCopyAlphaRow = ARGBCopyAlphaRow_SSE2; + } +#endif +#if defined(HAS_ARGBCOPYALPHAROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(width, 16)) { + ARGBCopyAlphaRow = ARGBCopyAlphaRow_AVX2; + } +#endif + + for (y = 0; y < height; ++y) { + ARGBCopyAlphaRow(src_argb, dst_argb, width); + src_argb += src_stride_argb; + dst_argb += dst_stride_argb; + } + return 0; +} + +// Copy a planar Y channel to the alpha channel of a destination ARGB image. +LIBYUV_API +int ARGBCopyYToAlpha(const uint8* src_y, int src_stride_y, + uint8* dst_argb, int dst_stride_argb, + int width, int height) { + int y; + void (*ARGBCopyYToAlphaRow)(const uint8* src_y, uint8* dst_argb, int width) = + ARGBCopyYToAlphaRow_C; + if (!src_y || !dst_argb || width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_y = src_y + (height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } + // Coalesce rows. + if (src_stride_y == width && + dst_stride_argb == width * 4) { + width *= height; + height = 1; + src_stride_y = dst_stride_argb = 0; + } +#if defined(HAS_ARGBCOPYYTOALPHAROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 8)) { + ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_SSE2; + } +#endif +#if defined(HAS_ARGBCOPYYTOALPHAROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(width, 16)) { + ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_AVX2; + } +#endif + + for (y = 0; y < height; ++y) { + ARGBCopyYToAlphaRow(src_y, dst_argb, width); + src_y += src_stride_y; + dst_argb += dst_stride_argb; + } + return 0; +} + +LIBYUV_API +int YUY2ToNV12(const uint8* src_yuy2, int src_stride_yuy2, + uint8* dst_y, int dst_stride_y, + uint8* dst_uv, int dst_stride_uv, + int width, int height) { + int y; + int halfwidth = (width + 1) >> 1; + void (*SplitUVRow)(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix) = + SplitUVRow_C; + void (*InterpolateRow)(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) = InterpolateRow_C; + if (!src_yuy2 || + !dst_y || !dst_uv || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_yuy2 = src_yuy2 + (height - 1) * src_stride_yuy2; + src_stride_yuy2 = -src_stride_yuy2; + } +#if defined(HAS_SPLITUVROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + SplitUVRow = SplitUVRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + SplitUVRow = SplitUVRow_SSE2; + } + } +#endif +#if defined(HAS_SPLITUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SplitUVRow = SplitUVRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + SplitUVRow = SplitUVRow_AVX2; + } + } +#endif +#if defined(HAS_SPLITUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + SplitUVRow = SplitUVRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + SplitUVRow = SplitUVRow_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + InterpolateRow = InterpolateRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + InterpolateRow = InterpolateRow_SSE2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + InterpolateRow = InterpolateRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + InterpolateRow = InterpolateRow_SSSE3; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + InterpolateRow = InterpolateRow_NEON; + } + } +#endif + + { + int awidth = halfwidth * 2; + // 2 rows of uv + align_buffer_64(rows, awidth * 2); + + for (y = 0; y < height - 1; y += 2) { + // Split Y from UV. + SplitUVRow(src_yuy2, dst_y, rows, awidth); + SplitUVRow(src_yuy2 + src_stride_yuy2, dst_y + dst_stride_y, + rows + awidth, awidth); + InterpolateRow(dst_uv, rows, awidth, awidth, 128); + src_yuy2 += src_stride_yuy2 * 2; + dst_y += dst_stride_y * 2; + dst_uv += dst_stride_uv; + } + if (height & 1) { + // Split Y from UV. + SplitUVRow(src_yuy2, dst_y, dst_uv, width); + } + free_aligned_buffer_64(rows); + } + return 0; +} + +LIBYUV_API +int UYVYToNV12(const uint8* src_uyvy, int src_stride_uyvy, + uint8* dst_y, int dst_stride_y, + uint8* dst_uv, int dst_stride_uv, + int width, int height) { + int y; + int halfwidth = (width + 1) >> 1; + void (*SplitUVRow)(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix) = + SplitUVRow_C; + void (*InterpolateRow)(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) = InterpolateRow_C; + if (!src_uyvy || + !dst_y || !dst_uv || + width <= 0 || height == 0) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_uyvy = src_uyvy + (height - 1) * src_stride_uyvy; + src_stride_uyvy = -src_stride_uyvy; + } +#if defined(HAS_SPLITUVROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + SplitUVRow = SplitUVRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + SplitUVRow = SplitUVRow_SSE2; + } + } +#endif +#if defined(HAS_SPLITUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SplitUVRow = SplitUVRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + SplitUVRow = SplitUVRow_AVX2; + } + } +#endif +#if defined(HAS_SPLITUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + SplitUVRow = SplitUVRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + SplitUVRow = SplitUVRow_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + InterpolateRow = InterpolateRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + InterpolateRow = InterpolateRow_SSE2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + InterpolateRow = InterpolateRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + InterpolateRow = InterpolateRow_SSSE3; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + InterpolateRow = InterpolateRow_NEON; + } + } +#endif + + { + int awidth = halfwidth * 2; + // 2 rows of uv + align_buffer_64(rows, awidth * 2); + + for (y = 0; y < height - 1; y += 2) { + // Split Y from UV. + SplitUVRow(src_uyvy, rows, dst_y, awidth); + SplitUVRow(src_uyvy + src_stride_uyvy, rows + awidth, + dst_y + dst_stride_y, awidth); + InterpolateRow(dst_uv, rows, awidth, awidth, 128); + src_uyvy += src_stride_uyvy * 2; + dst_y += dst_stride_y * 2; + dst_uv += dst_stride_uv; + } + if (height & 1) { + // Split Y from UV. + SplitUVRow(src_uyvy, dst_y, dst_uv, width); + } + free_aligned_buffer_64(rows); + } + return 0; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/rotate.cc b/third_party/aom/third_party/libyuv/source/rotate.cc new file mode 100644 index 0000000000..be3d589207 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/rotate.cc @@ -0,0 +1,496 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/rotate.h" + +#include "libyuv/cpu_id.h" +#include "libyuv/convert.h" +#include "libyuv/planar_functions.h" +#include "libyuv/rotate_row.h" +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +LIBYUV_API +void TransposePlane(const uint8* src, int src_stride, + uint8* dst, int dst_stride, + int width, int height) { + int i = height; + void (*TransposeWx8)(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width) = TransposeWx8_C; +#if defined(HAS_TRANSPOSEWX8_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + TransposeWx8 = TransposeWx8_NEON; + } +#endif +#if defined(HAS_TRANSPOSEWX8_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + TransposeWx8 = TransposeWx8_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + TransposeWx8 = TransposeWx8_SSSE3; + } + } +#endif +#if defined(HAS_TRANSPOSEWX8_FAST_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + TransposeWx8 = TransposeWx8_Fast_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + TransposeWx8 = TransposeWx8_Fast_SSSE3; + } + } +#endif +#if defined(HAS_TRANSPOSEWX8_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2)) { + if (IS_ALIGNED(width, 4) && + IS_ALIGNED(src, 4) && IS_ALIGNED(src_stride, 4)) { + TransposeWx8 = TransposeWx8_Fast_MIPS_DSPR2; + } else { + TransposeWx8 = TransposeWx8_MIPS_DSPR2; + } + } +#endif + + // Work across the source in 8x8 tiles + while (i >= 8) { + TransposeWx8(src, src_stride, dst, dst_stride, width); + src += 8 * src_stride; // Go down 8 rows. + dst += 8; // Move over 8 columns. + i -= 8; + } + + if (i > 0) { + TransposeWxH_C(src, src_stride, dst, dst_stride, width, i); + } +} + +LIBYUV_API +void RotatePlane90(const uint8* src, int src_stride, + uint8* dst, int dst_stride, + int width, int height) { + // Rotate by 90 is a transpose with the source read + // from bottom to top. So set the source pointer to the end + // of the buffer and flip the sign of the source stride. + src += src_stride * (height - 1); + src_stride = -src_stride; + TransposePlane(src, src_stride, dst, dst_stride, width, height); +} + +LIBYUV_API +void RotatePlane270(const uint8* src, int src_stride, + uint8* dst, int dst_stride, + int width, int height) { + // Rotate by 270 is a transpose with the destination written + // from bottom to top. So set the destination pointer to the end + // of the buffer and flip the sign of the destination stride. + dst += dst_stride * (width - 1); + dst_stride = -dst_stride; + TransposePlane(src, src_stride, dst, dst_stride, width, height); +} + +LIBYUV_API +void RotatePlane180(const uint8* src, int src_stride, + uint8* dst, int dst_stride, + int width, int height) { + // Swap first and last row and mirror the content. Uses a temporary row. + align_buffer_64(row, width); + const uint8* src_bot = src + src_stride * (height - 1); + uint8* dst_bot = dst + dst_stride * (height - 1); + int half_height = (height + 1) >> 1; + int y; + void (*MirrorRow)(const uint8* src, uint8* dst, int width) = MirrorRow_C; + void (*CopyRow)(const uint8* src, uint8* dst, int width) = CopyRow_C; +#if defined(HAS_MIRRORROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + MirrorRow = MirrorRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + MirrorRow = MirrorRow_NEON; + } + } +#endif +#if defined(HAS_MIRRORROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + MirrorRow = MirrorRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + MirrorRow = MirrorRow_SSE2; + } + } +#endif +#if defined(HAS_MIRRORROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + MirrorRow = MirrorRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + MirrorRow = MirrorRow_SSSE3; + } + } +#endif +#if defined(HAS_MIRRORROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + MirrorRow = MirrorRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + MirrorRow = MirrorRow_AVX2; + } + } +#endif +// TODO(fbarchard): Mirror on mips handle unaligned memory. +#if defined(HAS_MIRRORROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && + IS_ALIGNED(src, 4) && IS_ALIGNED(src_stride, 4) && + IS_ALIGNED(dst, 4) && IS_ALIGNED(dst_stride, 4)) { + MirrorRow = MirrorRow_MIPS_DSPR2; + } +#endif +#if defined(HAS_COPYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2; + } +#endif +#if defined(HAS_COPYROW_AVX) + if (TestCpuFlag(kCpuHasAVX)) { + CopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX; + } +#endif +#if defined(HAS_COPYROW_ERMS) + if (TestCpuFlag(kCpuHasERMS)) { + CopyRow = CopyRow_ERMS; + } +#endif +#if defined(HAS_COPYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON; + } +#endif +#if defined(HAS_COPYROW_MIPS) + if (TestCpuFlag(kCpuHasMIPS)) { + CopyRow = CopyRow_MIPS; + } +#endif + + // Odd height will harmlessly mirror the middle row twice. + for (y = 0; y < half_height; ++y) { + MirrorRow(src, row, width); // Mirror first row into a buffer + src += src_stride; + MirrorRow(src_bot, dst, width); // Mirror last row into first row + dst += dst_stride; + CopyRow(row, dst_bot, width); // Copy first mirrored row into last + src_bot -= src_stride; + dst_bot -= dst_stride; + } + free_aligned_buffer_64(row); +} + +LIBYUV_API +void TransposeUV(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width, int height) { + int i = height; + void (*TransposeUVWx8)(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width) = TransposeUVWx8_C; +#if defined(HAS_TRANSPOSEUVWX8_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + TransposeUVWx8 = TransposeUVWx8_NEON; + } +#endif +#if defined(HAS_TRANSPOSEUVWX8_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 8)) { + TransposeUVWx8 = TransposeUVWx8_SSE2; + } +#endif +#if defined(HAS_TRANSPOSEUVWx8_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(width, 2) && + IS_ALIGNED(src, 4) && IS_ALIGNED(src_stride, 4)) { + TransposeUVWx8 = TransposeUVWx8_MIPS_DSPR2; + } +#endif + + // Work through the source in 8x8 tiles. + while (i >= 8) { + TransposeUVWx8(src, src_stride, + dst_a, dst_stride_a, + dst_b, dst_stride_b, + width); + src += 8 * src_stride; // Go down 8 rows. + dst_a += 8; // Move over 8 columns. + dst_b += 8; // Move over 8 columns. + i -= 8; + } + + if (i > 0) { + TransposeUVWxH_C(src, src_stride, + dst_a, dst_stride_a, + dst_b, dst_stride_b, + width, i); + } +} + +LIBYUV_API +void RotateUV90(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width, int height) { + src += src_stride * (height - 1); + src_stride = -src_stride; + + TransposeUV(src, src_stride, + dst_a, dst_stride_a, + dst_b, dst_stride_b, + width, height); +} + +LIBYUV_API +void RotateUV270(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width, int height) { + dst_a += dst_stride_a * (width - 1); + dst_b += dst_stride_b * (width - 1); + dst_stride_a = -dst_stride_a; + dst_stride_b = -dst_stride_b; + + TransposeUV(src, src_stride, + dst_a, dst_stride_a, + dst_b, dst_stride_b, + width, height); +} + +// Rotate 180 is a horizontal and vertical flip. +LIBYUV_API +void RotateUV180(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width, int height) { + int i; + void (*MirrorRowUV)(const uint8* src, uint8* dst_u, uint8* dst_v, int width) = + MirrorUVRow_C; +#if defined(HAS_MIRRORUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { + MirrorRowUV = MirrorUVRow_NEON; + } +#endif +#if defined(HAS_MIRRORROW_UV_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 16)) { + MirrorRowUV = MirrorUVRow_SSSE3; + } +#endif +#if defined(HAS_MIRRORUVROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && + IS_ALIGNED(src, 4) && IS_ALIGNED(src_stride, 4)) { + MirrorRowUV = MirrorUVRow_MIPS_DSPR2; + } +#endif + + dst_a += dst_stride_a * (height - 1); + dst_b += dst_stride_b * (height - 1); + + for (i = 0; i < height; ++i) { + MirrorRowUV(src, dst_a, dst_b, width); + src += src_stride; + dst_a -= dst_stride_a; + dst_b -= dst_stride_b; + } +} + +LIBYUV_API +int RotatePlane(const uint8* src, int src_stride, + uint8* dst, int dst_stride, + int width, int height, + enum RotationMode mode) { + if (!src || width <= 0 || height == 0 || !dst) { + return -1; + } + + // Negative height means invert the image. + if (height < 0) { + height = -height; + src = src + (height - 1) * src_stride; + src_stride = -src_stride; + } + + switch (mode) { + case kRotate0: + // copy frame + CopyPlane(src, src_stride, + dst, dst_stride, + width, height); + return 0; + case kRotate90: + RotatePlane90(src, src_stride, + dst, dst_stride, + width, height); + return 0; + case kRotate270: + RotatePlane270(src, src_stride, + dst, dst_stride, + width, height); + return 0; + case kRotate180: + RotatePlane180(src, src_stride, + dst, dst_stride, + width, height); + return 0; + default: + break; + } + return -1; +} + +LIBYUV_API +int I420Rotate(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height, + enum RotationMode mode) { + int halfwidth = (width + 1) >> 1; + int halfheight = (height + 1) >> 1; + if (!src_y || !src_u || !src_v || width <= 0 || height == 0 || + !dst_y || !dst_u || !dst_v) { + return -1; + } + + // Negative height means invert the image. + if (height < 0) { + height = -height; + halfheight = (height + 1) >> 1; + src_y = src_y + (height - 1) * src_stride_y; + src_u = src_u + (halfheight - 1) * src_stride_u; + src_v = src_v + (halfheight - 1) * src_stride_v; + src_stride_y = -src_stride_y; + src_stride_u = -src_stride_u; + src_stride_v = -src_stride_v; + } + + switch (mode) { + case kRotate0: + // copy frame + return I420Copy(src_y, src_stride_y, + src_u, src_stride_u, + src_v, src_stride_v, + dst_y, dst_stride_y, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + width, height); + case kRotate90: + RotatePlane90(src_y, src_stride_y, + dst_y, dst_stride_y, + width, height); + RotatePlane90(src_u, src_stride_u, + dst_u, dst_stride_u, + halfwidth, halfheight); + RotatePlane90(src_v, src_stride_v, + dst_v, dst_stride_v, + halfwidth, halfheight); + return 0; + case kRotate270: + RotatePlane270(src_y, src_stride_y, + dst_y, dst_stride_y, + width, height); + RotatePlane270(src_u, src_stride_u, + dst_u, dst_stride_u, + halfwidth, halfheight); + RotatePlane270(src_v, src_stride_v, + dst_v, dst_stride_v, + halfwidth, halfheight); + return 0; + case kRotate180: + RotatePlane180(src_y, src_stride_y, + dst_y, dst_stride_y, + width, height); + RotatePlane180(src_u, src_stride_u, + dst_u, dst_stride_u, + halfwidth, halfheight); + RotatePlane180(src_v, src_stride_v, + dst_v, dst_stride_v, + halfwidth, halfheight); + return 0; + default: + break; + } + return -1; +} + +LIBYUV_API +int NV12ToI420Rotate(const uint8* src_y, int src_stride_y, + const uint8* src_uv, int src_stride_uv, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int width, int height, + enum RotationMode mode) { + int halfwidth = (width + 1) >> 1; + int halfheight = (height + 1) >> 1; + if (!src_y || !src_uv || width <= 0 || height == 0 || + !dst_y || !dst_u || !dst_v) { + return -1; + } + + // Negative height means invert the image. + if (height < 0) { + height = -height; + halfheight = (height + 1) >> 1; + src_y = src_y + (height - 1) * src_stride_y; + src_uv = src_uv + (halfheight - 1) * src_stride_uv; + src_stride_y = -src_stride_y; + src_stride_uv = -src_stride_uv; + } + + switch (mode) { + case kRotate0: + // copy frame + return NV12ToI420(src_y, src_stride_y, + src_uv, src_stride_uv, + dst_y, dst_stride_y, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + width, height); + case kRotate90: + RotatePlane90(src_y, src_stride_y, + dst_y, dst_stride_y, + width, height); + RotateUV90(src_uv, src_stride_uv, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + halfwidth, halfheight); + return 0; + case kRotate270: + RotatePlane270(src_y, src_stride_y, + dst_y, dst_stride_y, + width, height); + RotateUV270(src_uv, src_stride_uv, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + halfwidth, halfheight); + return 0; + case kRotate180: + RotatePlane180(src_y, src_stride_y, + dst_y, dst_stride_y, + width, height); + RotateUV180(src_uv, src_stride_uv, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + halfwidth, halfheight); + return 0; + default: + break; + } + return -1; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/rotate_any.cc b/third_party/aom/third_party/libyuv/source/rotate_any.cc new file mode 100644 index 0000000000..4d6eb34e18 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/rotate_any.cc @@ -0,0 +1,55 @@ +/* + * Copyright 2015 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/rotate.h" +#include "libyuv/rotate_row.h" + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#define TANY(NAMEANY, TPOS_SIMD, TPOS_C, MASK) \ + void NAMEANY(const uint8* src, int src_stride, \ + uint8* dst, int dst_stride, int width) { \ + int r = width & MASK; \ + int n = width - r; \ + if (n > 0) { \ + TPOS_SIMD(src, src_stride, dst, dst_stride, n); \ + } \ + TPOS_C(src + n, src_stride, dst + n * dst_stride, dst_stride, r); \ + } + +#ifdef HAS_TRANSPOSEWX8_NEON +TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, TransposeWx8_C, 7) +#endif +#ifdef HAS_TRANSPOSEWX8_SSSE3 +TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 7) +#endif +#ifdef HAS_TRANSPOSEWX8_FAST_SSSE3 +TANY(TransposeWx8_Fast_Any_SSSE3, TransposeWx8_Fast_SSSE3, TransposeWx8_C, 15) +#endif +#ifdef HAS_TRANSPOSEWX8_MIPS_DSPR2 +TANY(TransposeWx8_Any_MIPS_DSPR2, TransposeWx8_MIPS_DSPR2, TransposeWx8_C, 7) +#endif + +#undef TANY + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + + + + + diff --git a/third_party/aom/third_party/libyuv/source/rotate_argb.cc b/third_party/aom/third_party/libyuv/source/rotate_argb.cc new file mode 100644 index 0000000000..787c0ad1be --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/rotate_argb.cc @@ -0,0 +1,205 @@ +/* + * Copyright 2012 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/rotate.h" + +#include "libyuv/cpu_id.h" +#include "libyuv/convert.h" +#include "libyuv/planar_functions.h" +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// ARGBScale has a function to copy pixels to a row, striding each source +// pixel by a constant. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || \ + (defined(__x86_64__) && !defined(__native_client__)) || defined(__i386__)) +#define HAS_SCALEARGBROWDOWNEVEN_SSE2 +void ScaleARGBRowDownEven_SSE2(const uint8* src_ptr, int src_stride, + int src_stepx, uint8* dst_ptr, int dst_width); +#endif +#if !defined(LIBYUV_DISABLE_NEON) && !defined(__native_client__) && \ + (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) +#define HAS_SCALEARGBROWDOWNEVEN_NEON +void ScaleARGBRowDownEven_NEON(const uint8* src_ptr, int src_stride, + int src_stepx, uint8* dst_ptr, int dst_width); +#endif + +void ScaleARGBRowDownEven_C(const uint8* src_ptr, int, + int src_stepx, uint8* dst_ptr, int dst_width); + +static void ARGBTranspose(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width, int height) { + int i; + int src_pixel_step = src_stride >> 2; + void (*ScaleARGBRowDownEven)(const uint8* src_ptr, int src_stride, + int src_step, uint8* dst_ptr, int dst_width) = ScaleARGBRowDownEven_C; +#if defined(HAS_SCALEARGBROWDOWNEVEN_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(height, 4)) { // Width of dest. + ScaleARGBRowDownEven = ScaleARGBRowDownEven_SSE2; + } +#endif +#if defined(HAS_SCALEARGBROWDOWNEVEN_NEON) + if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(height, 4)) { // Width of dest. + ScaleARGBRowDownEven = ScaleARGBRowDownEven_NEON; + } +#endif + + for (i = 0; i < width; ++i) { // column of source to row of dest. + ScaleARGBRowDownEven(src, 0, src_pixel_step, dst, height); + dst += dst_stride; + src += 4; + } +} + +void ARGBRotate90(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width, int height) { + // Rotate by 90 is a ARGBTranspose with the source read + // from bottom to top. So set the source pointer to the end + // of the buffer and flip the sign of the source stride. + src += src_stride * (height - 1); + src_stride = -src_stride; + ARGBTranspose(src, src_stride, dst, dst_stride, width, height); +} + +void ARGBRotate270(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width, int height) { + // Rotate by 270 is a ARGBTranspose with the destination written + // from bottom to top. So set the destination pointer to the end + // of the buffer and flip the sign of the destination stride. + dst += dst_stride * (width - 1); + dst_stride = -dst_stride; + ARGBTranspose(src, src_stride, dst, dst_stride, width, height); +} + +void ARGBRotate180(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width, int height) { + // Swap first and last row and mirror the content. Uses a temporary row. + align_buffer_64(row, width * 4); + const uint8* src_bot = src + src_stride * (height - 1); + uint8* dst_bot = dst + dst_stride * (height - 1); + int half_height = (height + 1) >> 1; + int y; + void (*ARGBMirrorRow)(const uint8* src, uint8* dst, int width) = + ARGBMirrorRow_C; + void (*CopyRow)(const uint8* src, uint8* dst, int width) = CopyRow_C; +#if defined(HAS_ARGBMIRRORROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBMirrorRow = ARGBMirrorRow_Any_NEON; + if (IS_ALIGNED(width, 4)) { + ARGBMirrorRow = ARGBMirrorRow_NEON; + } + } +#endif +#if defined(HAS_ARGBMIRRORROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBMirrorRow = ARGBMirrorRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBMirrorRow = ARGBMirrorRow_SSE2; + } + } +#endif +#if defined(HAS_ARGBMIRRORROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBMirrorRow = ARGBMirrorRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBMirrorRow = ARGBMirrorRow_AVX2; + } + } +#endif +#if defined(HAS_COPYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2; + } +#endif +#if defined(HAS_COPYROW_AVX) + if (TestCpuFlag(kCpuHasAVX)) { + CopyRow = IS_ALIGNED(width * 4, 64) ? CopyRow_AVX : CopyRow_Any_AVX; + } +#endif +#if defined(HAS_COPYROW_ERMS) + if (TestCpuFlag(kCpuHasERMS)) { + CopyRow = CopyRow_ERMS; + } +#endif +#if defined(HAS_COPYROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_NEON : CopyRow_Any_NEON; + } +#endif +#if defined(HAS_COPYROW_MIPS) + if (TestCpuFlag(kCpuHasMIPS)) { + CopyRow = CopyRow_MIPS; + } +#endif + + // Odd height will harmlessly mirror the middle row twice. + for (y = 0; y < half_height; ++y) { + ARGBMirrorRow(src, row, width); // Mirror first row into a buffer + ARGBMirrorRow(src_bot, dst, width); // Mirror last row into first row + CopyRow(row, dst_bot, width * 4); // Copy first mirrored row into last + src += src_stride; + dst += dst_stride; + src_bot -= src_stride; + dst_bot -= dst_stride; + } + free_aligned_buffer_64(row); +} + +LIBYUV_API +int ARGBRotate(const uint8* src_argb, int src_stride_argb, + uint8* dst_argb, int dst_stride_argb, int width, int height, + enum RotationMode mode) { + if (!src_argb || width <= 0 || height == 0 || !dst_argb) { + return -1; + } + + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + + switch (mode) { + case kRotate0: + // copy frame + return ARGBCopy(src_argb, src_stride_argb, + dst_argb, dst_stride_argb, + width, height); + case kRotate90: + ARGBRotate90(src_argb, src_stride_argb, + dst_argb, dst_stride_argb, + width, height); + return 0; + case kRotate270: + ARGBRotate270(src_argb, src_stride_argb, + dst_argb, dst_stride_argb, + width, height); + return 0; + case kRotate180: + ARGBRotate180(src_argb, src_stride_argb, + dst_argb, dst_stride_argb, + width, height); + return 0; + default: + break; + } + return -1; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/rotate_common.cc b/third_party/aom/third_party/libyuv/source/rotate_common.cc new file mode 100644 index 0000000000..b33a9a0c6e --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/rotate_common.cc @@ -0,0 +1,92 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" +#include "libyuv/rotate_row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +void TransposeWx8_C(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width) { + int i; + for (i = 0; i < width; ++i) { + dst[0] = src[0 * src_stride]; + dst[1] = src[1 * src_stride]; + dst[2] = src[2 * src_stride]; + dst[3] = src[3 * src_stride]; + dst[4] = src[4 * src_stride]; + dst[5] = src[5 * src_stride]; + dst[6] = src[6 * src_stride]; + dst[7] = src[7 * src_stride]; + ++src; + dst += dst_stride; + } +} + +void TransposeUVWx8_C(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, int width) { + int i; + for (i = 0; i < width; ++i) { + dst_a[0] = src[0 * src_stride + 0]; + dst_b[0] = src[0 * src_stride + 1]; + dst_a[1] = src[1 * src_stride + 0]; + dst_b[1] = src[1 * src_stride + 1]; + dst_a[2] = src[2 * src_stride + 0]; + dst_b[2] = src[2 * src_stride + 1]; + dst_a[3] = src[3 * src_stride + 0]; + dst_b[3] = src[3 * src_stride + 1]; + dst_a[4] = src[4 * src_stride + 0]; + dst_b[4] = src[4 * src_stride + 1]; + dst_a[5] = src[5 * src_stride + 0]; + dst_b[5] = src[5 * src_stride + 1]; + dst_a[6] = src[6 * src_stride + 0]; + dst_b[6] = src[6 * src_stride + 1]; + dst_a[7] = src[7 * src_stride + 0]; + dst_b[7] = src[7 * src_stride + 1]; + src += 2; + dst_a += dst_stride_a; + dst_b += dst_stride_b; + } +} + +void TransposeWxH_C(const uint8* src, int src_stride, + uint8* dst, int dst_stride, + int width, int height) { + int i; + for (i = 0; i < width; ++i) { + int j; + for (j = 0; j < height; ++j) { + dst[i * dst_stride + j] = src[j * src_stride + i]; + } + } +} + +void TransposeUVWxH_C(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width, int height) { + int i; + for (i = 0; i < width * 2; i += 2) { + int j; + for (j = 0; j < height; ++j) { + dst_a[j + ((i >> 1) * dst_stride_a)] = src[i + (j * src_stride)]; + dst_b[j + ((i >> 1) * dst_stride_b)] = src[i + (j * src_stride) + 1]; + } + } +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/rotate_gcc.cc b/third_party/aom/third_party/libyuv/source/rotate_gcc.cc new file mode 100644 index 0000000000..fd385bcd30 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/rotate_gcc.cc @@ -0,0 +1,493 @@ +/* + * Copyright 2015 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" +#include "libyuv/rotate_row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for GCC x86 and x64. +#if !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__)) + +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__i386__) || (defined(__x86_64__) && !defined(__native_client__))) +void TransposeWx8_SSSE3(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width) { + asm volatile ( + // Read in the data from the source pointer. + // First round of bit swap. + ".p2align 2 \n" + "1: \n" + "movq (%0),%%xmm0 \n" + "movq (%0,%3),%%xmm1 \n" + "lea (%0,%3,2),%0 \n" + "punpcklbw %%xmm1,%%xmm0 \n" + "movq (%0),%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "palignr $0x8,%%xmm1,%%xmm1 \n" + "movq (%0,%3),%%xmm3 \n" + "lea (%0,%3,2),%0 \n" + "punpcklbw %%xmm3,%%xmm2 \n" + "movdqa %%xmm2,%%xmm3 \n" + "movq (%0),%%xmm4 \n" + "palignr $0x8,%%xmm3,%%xmm3 \n" + "movq (%0,%3),%%xmm5 \n" + "lea (%0,%3,2),%0 \n" + "punpcklbw %%xmm5,%%xmm4 \n" + "movdqa %%xmm4,%%xmm5 \n" + "movq (%0),%%xmm6 \n" + "palignr $0x8,%%xmm5,%%xmm5 \n" + "movq (%0,%3),%%xmm7 \n" + "lea (%0,%3,2),%0 \n" + "punpcklbw %%xmm7,%%xmm6 \n" + "neg %3 \n" + "movdqa %%xmm6,%%xmm7 \n" + "lea 0x8(%0,%3,8),%0 \n" + "palignr $0x8,%%xmm7,%%xmm7 \n" + "neg %3 \n" + // Second round of bit swap. + "punpcklwd %%xmm2,%%xmm0 \n" + "punpcklwd %%xmm3,%%xmm1 \n" + "movdqa %%xmm0,%%xmm2 \n" + "movdqa %%xmm1,%%xmm3 \n" + "palignr $0x8,%%xmm2,%%xmm2 \n" + "palignr $0x8,%%xmm3,%%xmm3 \n" + "punpcklwd %%xmm6,%%xmm4 \n" + "punpcklwd %%xmm7,%%xmm5 \n" + "movdqa %%xmm4,%%xmm6 \n" + "movdqa %%xmm5,%%xmm7 \n" + "palignr $0x8,%%xmm6,%%xmm6 \n" + "palignr $0x8,%%xmm7,%%xmm7 \n" + // Third round of bit swap. + // Write to the destination pointer. + "punpckldq %%xmm4,%%xmm0 \n" + "movq %%xmm0,(%1) \n" + "movdqa %%xmm0,%%xmm4 \n" + "palignr $0x8,%%xmm4,%%xmm4 \n" + "movq %%xmm4,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "punpckldq %%xmm6,%%xmm2 \n" + "movdqa %%xmm2,%%xmm6 \n" + "movq %%xmm2,(%1) \n" + "palignr $0x8,%%xmm6,%%xmm6 \n" + "punpckldq %%xmm5,%%xmm1 \n" + "movq %%xmm6,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "movdqa %%xmm1,%%xmm5 \n" + "movq %%xmm1,(%1) \n" + "palignr $0x8,%%xmm5,%%xmm5 \n" + "movq %%xmm5,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "punpckldq %%xmm7,%%xmm3 \n" + "movq %%xmm3,(%1) \n" + "movdqa %%xmm3,%%xmm7 \n" + "palignr $0x8,%%xmm7,%%xmm7 \n" + "sub $0x8,%2 \n" + "movq %%xmm7,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : "r"((intptr_t)(src_stride)), // %3 + "r"((intptr_t)(dst_stride)) // %4 + : "memory", "cc", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} + +#if !defined(LIBYUV_DISABLE_X86) && defined(__i386__) && !defined(__clang__) +void TransposeUVWx8_SSE2(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, int width); + asm ( + DECLARE_FUNCTION(TransposeUVWx8_SSE2) + "push %ebx \n" + "push %esi \n" + "push %edi \n" + "push %ebp \n" + "mov 0x14(%esp),%eax \n" + "mov 0x18(%esp),%edi \n" + "mov 0x1c(%esp),%edx \n" + "mov 0x20(%esp),%esi \n" + "mov 0x24(%esp),%ebx \n" + "mov 0x28(%esp),%ebp \n" + "mov %esp,%ecx \n" + "sub $0x14,%esp \n" + "and $0xfffffff0,%esp \n" + "mov %ecx,0x10(%esp) \n" + "mov 0x2c(%ecx),%ecx \n" + +"1: \n" + "movdqu (%eax),%xmm0 \n" + "movdqu (%eax,%edi,1),%xmm1 \n" + "lea (%eax,%edi,2),%eax \n" + "movdqa %xmm0,%xmm7 \n" + "punpcklbw %xmm1,%xmm0 \n" + "punpckhbw %xmm1,%xmm7 \n" + "movdqa %xmm7,%xmm1 \n" + "movdqu (%eax),%xmm2 \n" + "movdqu (%eax,%edi,1),%xmm3 \n" + "lea (%eax,%edi,2),%eax \n" + "movdqa %xmm2,%xmm7 \n" + "punpcklbw %xmm3,%xmm2 \n" + "punpckhbw %xmm3,%xmm7 \n" + "movdqa %xmm7,%xmm3 \n" + "movdqu (%eax),%xmm4 \n" + "movdqu (%eax,%edi,1),%xmm5 \n" + "lea (%eax,%edi,2),%eax \n" + "movdqa %xmm4,%xmm7 \n" + "punpcklbw %xmm5,%xmm4 \n" + "punpckhbw %xmm5,%xmm7 \n" + "movdqa %xmm7,%xmm5 \n" + "movdqu (%eax),%xmm6 \n" + "movdqu (%eax,%edi,1),%xmm7 \n" + "lea (%eax,%edi,2),%eax \n" + "movdqu %xmm5,(%esp) \n" + "neg %edi \n" + "movdqa %xmm6,%xmm5 \n" + "punpcklbw %xmm7,%xmm6 \n" + "punpckhbw %xmm7,%xmm5 \n" + "movdqa %xmm5,%xmm7 \n" + "lea 0x10(%eax,%edi,8),%eax \n" + "neg %edi \n" + "movdqa %xmm0,%xmm5 \n" + "punpcklwd %xmm2,%xmm0 \n" + "punpckhwd %xmm2,%xmm5 \n" + "movdqa %xmm5,%xmm2 \n" + "movdqa %xmm1,%xmm5 \n" + "punpcklwd %xmm3,%xmm1 \n" + "punpckhwd %xmm3,%xmm5 \n" + "movdqa %xmm5,%xmm3 \n" + "movdqa %xmm4,%xmm5 \n" + "punpcklwd %xmm6,%xmm4 \n" + "punpckhwd %xmm6,%xmm5 \n" + "movdqa %xmm5,%xmm6 \n" + "movdqu (%esp),%xmm5 \n" + "movdqu %xmm6,(%esp) \n" + "movdqa %xmm5,%xmm6 \n" + "punpcklwd %xmm7,%xmm5 \n" + "punpckhwd %xmm7,%xmm6 \n" + "movdqa %xmm6,%xmm7 \n" + "movdqa %xmm0,%xmm6 \n" + "punpckldq %xmm4,%xmm0 \n" + "punpckhdq %xmm4,%xmm6 \n" + "movdqa %xmm6,%xmm4 \n" + "movdqu (%esp),%xmm6 \n" + "movlpd %xmm0,(%edx) \n" + "movhpd %xmm0,(%ebx) \n" + "movlpd %xmm4,(%edx,%esi,1) \n" + "lea (%edx,%esi,2),%edx \n" + "movhpd %xmm4,(%ebx,%ebp,1) \n" + "lea (%ebx,%ebp,2),%ebx \n" + "movdqa %xmm2,%xmm0 \n" + "punpckldq %xmm6,%xmm2 \n" + "movlpd %xmm2,(%edx) \n" + "movhpd %xmm2,(%ebx) \n" + "punpckhdq %xmm6,%xmm0 \n" + "movlpd %xmm0,(%edx,%esi,1) \n" + "lea (%edx,%esi,2),%edx \n" + "movhpd %xmm0,(%ebx,%ebp,1) \n" + "lea (%ebx,%ebp,2),%ebx \n" + "movdqa %xmm1,%xmm0 \n" + "punpckldq %xmm5,%xmm1 \n" + "movlpd %xmm1,(%edx) \n" + "movhpd %xmm1,(%ebx) \n" + "punpckhdq %xmm5,%xmm0 \n" + "movlpd %xmm0,(%edx,%esi,1) \n" + "lea (%edx,%esi,2),%edx \n" + "movhpd %xmm0,(%ebx,%ebp,1) \n" + "lea (%ebx,%ebp,2),%ebx \n" + "movdqa %xmm3,%xmm0 \n" + "punpckldq %xmm7,%xmm3 \n" + "movlpd %xmm3,(%edx) \n" + "movhpd %xmm3,(%ebx) \n" + "punpckhdq %xmm7,%xmm0 \n" + "sub $0x8,%ecx \n" + "movlpd %xmm0,(%edx,%esi,1) \n" + "lea (%edx,%esi,2),%edx \n" + "movhpd %xmm0,(%ebx,%ebp,1) \n" + "lea (%ebx,%ebp,2),%ebx \n" + "jg 1b \n" + "mov 0x10(%esp),%esp \n" + "pop %ebp \n" + "pop %edi \n" + "pop %esi \n" + "pop %ebx \n" +#if defined(__native_client__) + "pop %ecx \n" + "and $0xffffffe0,%ecx \n" + "jmp *%ecx \n" +#else + "ret \n" +#endif +); +#endif +#if !defined(LIBYUV_DISABLE_X86) && !defined(__native_client__) && \ + defined(__x86_64__) +// 64 bit version has enough registers to do 16x8 to 8x16 at a time. +void TransposeWx8_Fast_SSSE3(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width) { + asm volatile ( + // Read in the data from the source pointer. + // First round of bit swap. + ".p2align 2 \n" +"1: \n" + "movdqu (%0),%%xmm0 \n" + "movdqu (%0,%3),%%xmm1 \n" + "lea (%0,%3,2),%0 \n" + "movdqa %%xmm0,%%xmm8 \n" + "punpcklbw %%xmm1,%%xmm0 \n" + "punpckhbw %%xmm1,%%xmm8 \n" + "movdqu (%0),%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm8,%%xmm9 \n" + "palignr $0x8,%%xmm1,%%xmm1 \n" + "palignr $0x8,%%xmm9,%%xmm9 \n" + "movdqu (%0,%3),%%xmm3 \n" + "lea (%0,%3,2),%0 \n" + "movdqa %%xmm2,%%xmm10 \n" + "punpcklbw %%xmm3,%%xmm2 \n" + "punpckhbw %%xmm3,%%xmm10 \n" + "movdqa %%xmm2,%%xmm3 \n" + "movdqa %%xmm10,%%xmm11 \n" + "movdqu (%0),%%xmm4 \n" + "palignr $0x8,%%xmm3,%%xmm3 \n" + "palignr $0x8,%%xmm11,%%xmm11 \n" + "movdqu (%0,%3),%%xmm5 \n" + "lea (%0,%3,2),%0 \n" + "movdqa %%xmm4,%%xmm12 \n" + "punpcklbw %%xmm5,%%xmm4 \n" + "punpckhbw %%xmm5,%%xmm12 \n" + "movdqa %%xmm4,%%xmm5 \n" + "movdqa %%xmm12,%%xmm13 \n" + "movdqu (%0),%%xmm6 \n" + "palignr $0x8,%%xmm5,%%xmm5 \n" + "palignr $0x8,%%xmm13,%%xmm13 \n" + "movdqu (%0,%3),%%xmm7 \n" + "lea (%0,%3,2),%0 \n" + "movdqa %%xmm6,%%xmm14 \n" + "punpcklbw %%xmm7,%%xmm6 \n" + "punpckhbw %%xmm7,%%xmm14 \n" + "neg %3 \n" + "movdqa %%xmm6,%%xmm7 \n" + "movdqa %%xmm14,%%xmm15 \n" + "lea 0x10(%0,%3,8),%0 \n" + "palignr $0x8,%%xmm7,%%xmm7 \n" + "palignr $0x8,%%xmm15,%%xmm15 \n" + "neg %3 \n" + // Second round of bit swap. + "punpcklwd %%xmm2,%%xmm0 \n" + "punpcklwd %%xmm3,%%xmm1 \n" + "movdqa %%xmm0,%%xmm2 \n" + "movdqa %%xmm1,%%xmm3 \n" + "palignr $0x8,%%xmm2,%%xmm2 \n" + "palignr $0x8,%%xmm3,%%xmm3 \n" + "punpcklwd %%xmm6,%%xmm4 \n" + "punpcklwd %%xmm7,%%xmm5 \n" + "movdqa %%xmm4,%%xmm6 \n" + "movdqa %%xmm5,%%xmm7 \n" + "palignr $0x8,%%xmm6,%%xmm6 \n" + "palignr $0x8,%%xmm7,%%xmm7 \n" + "punpcklwd %%xmm10,%%xmm8 \n" + "punpcklwd %%xmm11,%%xmm9 \n" + "movdqa %%xmm8,%%xmm10 \n" + "movdqa %%xmm9,%%xmm11 \n" + "palignr $0x8,%%xmm10,%%xmm10 \n" + "palignr $0x8,%%xmm11,%%xmm11 \n" + "punpcklwd %%xmm14,%%xmm12 \n" + "punpcklwd %%xmm15,%%xmm13 \n" + "movdqa %%xmm12,%%xmm14 \n" + "movdqa %%xmm13,%%xmm15 \n" + "palignr $0x8,%%xmm14,%%xmm14 \n" + "palignr $0x8,%%xmm15,%%xmm15 \n" + // Third round of bit swap. + // Write to the destination pointer. + "punpckldq %%xmm4,%%xmm0 \n" + "movq %%xmm0,(%1) \n" + "movdqa %%xmm0,%%xmm4 \n" + "palignr $0x8,%%xmm4,%%xmm4 \n" + "movq %%xmm4,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "punpckldq %%xmm6,%%xmm2 \n" + "movdqa %%xmm2,%%xmm6 \n" + "movq %%xmm2,(%1) \n" + "palignr $0x8,%%xmm6,%%xmm6 \n" + "punpckldq %%xmm5,%%xmm1 \n" + "movq %%xmm6,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "movdqa %%xmm1,%%xmm5 \n" + "movq %%xmm1,(%1) \n" + "palignr $0x8,%%xmm5,%%xmm5 \n" + "movq %%xmm5,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "punpckldq %%xmm7,%%xmm3 \n" + "movq %%xmm3,(%1) \n" + "movdqa %%xmm3,%%xmm7 \n" + "palignr $0x8,%%xmm7,%%xmm7 \n" + "movq %%xmm7,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "punpckldq %%xmm12,%%xmm8 \n" + "movq %%xmm8,(%1) \n" + "movdqa %%xmm8,%%xmm12 \n" + "palignr $0x8,%%xmm12,%%xmm12 \n" + "movq %%xmm12,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "punpckldq %%xmm14,%%xmm10 \n" + "movdqa %%xmm10,%%xmm14 \n" + "movq %%xmm10,(%1) \n" + "palignr $0x8,%%xmm14,%%xmm14 \n" + "punpckldq %%xmm13,%%xmm9 \n" + "movq %%xmm14,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "movdqa %%xmm9,%%xmm13 \n" + "movq %%xmm9,(%1) \n" + "palignr $0x8,%%xmm13,%%xmm13 \n" + "movq %%xmm13,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "punpckldq %%xmm15,%%xmm11 \n" + "movq %%xmm11,(%1) \n" + "movdqa %%xmm11,%%xmm15 \n" + "palignr $0x8,%%xmm15,%%xmm15 \n" + "sub $0x10,%2 \n" + "movq %%xmm15,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : "r"((intptr_t)(src_stride)), // %3 + "r"((intptr_t)(dst_stride)) // %4 + : "memory", "cc", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" +); +} + +void TransposeUVWx8_SSE2(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, int width) { + asm volatile ( + // Read in the data from the source pointer. + // First round of bit swap. + ".p2align 2 \n" +"1: \n" + "movdqu (%0),%%xmm0 \n" + "movdqu (%0,%4),%%xmm1 \n" + "lea (%0,%4,2),%0 \n" + "movdqa %%xmm0,%%xmm8 \n" + "punpcklbw %%xmm1,%%xmm0 \n" + "punpckhbw %%xmm1,%%xmm8 \n" + "movdqa %%xmm8,%%xmm1 \n" + "movdqu (%0),%%xmm2 \n" + "movdqu (%0,%4),%%xmm3 \n" + "lea (%0,%4,2),%0 \n" + "movdqa %%xmm2,%%xmm8 \n" + "punpcklbw %%xmm3,%%xmm2 \n" + "punpckhbw %%xmm3,%%xmm8 \n" + "movdqa %%xmm8,%%xmm3 \n" + "movdqu (%0),%%xmm4 \n" + "movdqu (%0,%4),%%xmm5 \n" + "lea (%0,%4,2),%0 \n" + "movdqa %%xmm4,%%xmm8 \n" + "punpcklbw %%xmm5,%%xmm4 \n" + "punpckhbw %%xmm5,%%xmm8 \n" + "movdqa %%xmm8,%%xmm5 \n" + "movdqu (%0),%%xmm6 \n" + "movdqu (%0,%4),%%xmm7 \n" + "lea (%0,%4,2),%0 \n" + "movdqa %%xmm6,%%xmm8 \n" + "punpcklbw %%xmm7,%%xmm6 \n" + "neg %4 \n" + "lea 0x10(%0,%4,8),%0 \n" + "punpckhbw %%xmm7,%%xmm8 \n" + "movdqa %%xmm8,%%xmm7 \n" + "neg %4 \n" + // Second round of bit swap. + "movdqa %%xmm0,%%xmm8 \n" + "movdqa %%xmm1,%%xmm9 \n" + "punpckhwd %%xmm2,%%xmm8 \n" + "punpckhwd %%xmm3,%%xmm9 \n" + "punpcklwd %%xmm2,%%xmm0 \n" + "punpcklwd %%xmm3,%%xmm1 \n" + "movdqa %%xmm8,%%xmm2 \n" + "movdqa %%xmm9,%%xmm3 \n" + "movdqa %%xmm4,%%xmm8 \n" + "movdqa %%xmm5,%%xmm9 \n" + "punpckhwd %%xmm6,%%xmm8 \n" + "punpckhwd %%xmm7,%%xmm9 \n" + "punpcklwd %%xmm6,%%xmm4 \n" + "punpcklwd %%xmm7,%%xmm5 \n" + "movdqa %%xmm8,%%xmm6 \n" + "movdqa %%xmm9,%%xmm7 \n" + // Third round of bit swap. + // Write to the destination pointer. + "movdqa %%xmm0,%%xmm8 \n" + "punpckldq %%xmm4,%%xmm0 \n" + "movlpd %%xmm0,(%1) \n" // Write back U channel + "movhpd %%xmm0,(%2) \n" // Write back V channel + "punpckhdq %%xmm4,%%xmm8 \n" + "movlpd %%xmm8,(%1,%5) \n" + "lea (%1,%5,2),%1 \n" + "movhpd %%xmm8,(%2,%6) \n" + "lea (%2,%6,2),%2 \n" + "movdqa %%xmm2,%%xmm8 \n" + "punpckldq %%xmm6,%%xmm2 \n" + "movlpd %%xmm2,(%1) \n" + "movhpd %%xmm2,(%2) \n" + "punpckhdq %%xmm6,%%xmm8 \n" + "movlpd %%xmm8,(%1,%5) \n" + "lea (%1,%5,2),%1 \n" + "movhpd %%xmm8,(%2,%6) \n" + "lea (%2,%6,2),%2 \n" + "movdqa %%xmm1,%%xmm8 \n" + "punpckldq %%xmm5,%%xmm1 \n" + "movlpd %%xmm1,(%1) \n" + "movhpd %%xmm1,(%2) \n" + "punpckhdq %%xmm5,%%xmm8 \n" + "movlpd %%xmm8,(%1,%5) \n" + "lea (%1,%5,2),%1 \n" + "movhpd %%xmm8,(%2,%6) \n" + "lea (%2,%6,2),%2 \n" + "movdqa %%xmm3,%%xmm8 \n" + "punpckldq %%xmm7,%%xmm3 \n" + "movlpd %%xmm3,(%1) \n" + "movhpd %%xmm3,(%2) \n" + "punpckhdq %%xmm7,%%xmm8 \n" + "sub $0x8,%3 \n" + "movlpd %%xmm8,(%1,%5) \n" + "lea (%1,%5,2),%1 \n" + "movhpd %%xmm8,(%2,%6) \n" + "lea (%2,%6,2),%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst_a), // %1 + "+r"(dst_b), // %2 + "+r"(width) // %3 + : "r"((intptr_t)(src_stride)), // %4 + "r"((intptr_t)(dst_stride_a)), // %5 + "r"((intptr_t)(dst_stride_b)) // %6 + : "memory", "cc", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9" +); +} +#endif +#endif + +#endif // defined(__x86_64__) || defined(__i386__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/rotate_mips.cc b/third_party/aom/third_party/libyuv/source/rotate_mips.cc new file mode 100644 index 0000000000..efe6bd909e --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/rotate_mips.cc @@ -0,0 +1,484 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" +#include "libyuv/rotate_row.h" + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#if !defined(LIBYUV_DISABLE_MIPS) && \ + defined(__mips_dsp) && (__mips_dsp_rev >= 2) && \ + (_MIPS_SIM == _MIPS_SIM_ABI32) + +void TransposeWx8_MIPS_DSPR2(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width) { + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + "sll $t2, %[src_stride], 0x1 \n" // src_stride x 2 + "sll $t4, %[src_stride], 0x2 \n" // src_stride x 4 + "sll $t9, %[src_stride], 0x3 \n" // src_stride x 8 + "addu $t3, $t2, %[src_stride] \n" + "addu $t5, $t4, %[src_stride] \n" + "addu $t6, $t2, $t4 \n" + "andi $t0, %[dst], 0x3 \n" + "andi $t1, %[dst_stride], 0x3 \n" + "or $t0, $t0, $t1 \n" + "bnez $t0, 11f \n" + " subu $t7, $t9, %[src_stride] \n" +//dst + dst_stride word aligned + "1: \n" + "lbu $t0, 0(%[src]) \n" + "lbux $t1, %[src_stride](%[src]) \n" + "lbux $t8, $t2(%[src]) \n" + "lbux $t9, $t3(%[src]) \n" + "sll $t1, $t1, 16 \n" + "sll $t9, $t9, 16 \n" + "or $t0, $t0, $t1 \n" + "or $t8, $t8, $t9 \n" + "precr.qb.ph $s0, $t8, $t0 \n" + "lbux $t0, $t4(%[src]) \n" + "lbux $t1, $t5(%[src]) \n" + "lbux $t8, $t6(%[src]) \n" + "lbux $t9, $t7(%[src]) \n" + "sll $t1, $t1, 16 \n" + "sll $t9, $t9, 16 \n" + "or $t0, $t0, $t1 \n" + "or $t8, $t8, $t9 \n" + "precr.qb.ph $s1, $t8, $t0 \n" + "sw $s0, 0(%[dst]) \n" + "addiu %[width], -1 \n" + "addiu %[src], 1 \n" + "sw $s1, 4(%[dst]) \n" + "bnez %[width], 1b \n" + " addu %[dst], %[dst], %[dst_stride] \n" + "b 2f \n" +//dst + dst_stride unaligned + "11: \n" + "lbu $t0, 0(%[src]) \n" + "lbux $t1, %[src_stride](%[src]) \n" + "lbux $t8, $t2(%[src]) \n" + "lbux $t9, $t3(%[src]) \n" + "sll $t1, $t1, 16 \n" + "sll $t9, $t9, 16 \n" + "or $t0, $t0, $t1 \n" + "or $t8, $t8, $t9 \n" + "precr.qb.ph $s0, $t8, $t0 \n" + "lbux $t0, $t4(%[src]) \n" + "lbux $t1, $t5(%[src]) \n" + "lbux $t8, $t6(%[src]) \n" + "lbux $t9, $t7(%[src]) \n" + "sll $t1, $t1, 16 \n" + "sll $t9, $t9, 16 \n" + "or $t0, $t0, $t1 \n" + "or $t8, $t8, $t9 \n" + "precr.qb.ph $s1, $t8, $t0 \n" + "swr $s0, 0(%[dst]) \n" + "swl $s0, 3(%[dst]) \n" + "addiu %[width], -1 \n" + "addiu %[src], 1 \n" + "swr $s1, 4(%[dst]) \n" + "swl $s1, 7(%[dst]) \n" + "bnez %[width], 11b \n" + "addu %[dst], %[dst], %[dst_stride] \n" + "2: \n" + ".set pop \n" + :[src] "+r" (src), + [dst] "+r" (dst), + [width] "+r" (width) + :[src_stride] "r" (src_stride), + [dst_stride] "r" (dst_stride) + : "t0", "t1", "t2", "t3", "t4", "t5", + "t6", "t7", "t8", "t9", + "s0", "s1" + ); +} + +void TransposeWx8_Fast_MIPS_DSPR2(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width) { + __asm__ __volatile__ ( + ".set noat \n" + ".set push \n" + ".set noreorder \n" + "beqz %[width], 2f \n" + " sll $t2, %[src_stride], 0x1 \n" // src_stride x 2 + "sll $t4, %[src_stride], 0x2 \n" // src_stride x 4 + "sll $t9, %[src_stride], 0x3 \n" // src_stride x 8 + "addu $t3, $t2, %[src_stride] \n" + "addu $t5, $t4, %[src_stride] \n" + "addu $t6, $t2, $t4 \n" + + "srl $AT, %[width], 0x2 \n" + "andi $t0, %[dst], 0x3 \n" + "andi $t1, %[dst_stride], 0x3 \n" + "or $t0, $t0, $t1 \n" + "bnez $t0, 11f \n" + " subu $t7, $t9, %[src_stride] \n" +//dst + dst_stride word aligned + "1: \n" + "lw $t0, 0(%[src]) \n" + "lwx $t1, %[src_stride](%[src]) \n" + "lwx $t8, $t2(%[src]) \n" + "lwx $t9, $t3(%[src]) \n" + +// t0 = | 30 | 20 | 10 | 00 | +// t1 = | 31 | 21 | 11 | 01 | +// t8 = | 32 | 22 | 12 | 02 | +// t9 = | 33 | 23 | 13 | 03 | + + "precr.qb.ph $s0, $t1, $t0 \n" + "precr.qb.ph $s1, $t9, $t8 \n" + "precrq.qb.ph $s2, $t1, $t0 \n" + "precrq.qb.ph $s3, $t9, $t8 \n" + + // s0 = | 21 | 01 | 20 | 00 | + // s1 = | 23 | 03 | 22 | 02 | + // s2 = | 31 | 11 | 30 | 10 | + // s3 = | 33 | 13 | 32 | 12 | + + "precr.qb.ph $s4, $s1, $s0 \n" + "precrq.qb.ph $s5, $s1, $s0 \n" + "precr.qb.ph $s6, $s3, $s2 \n" + "precrq.qb.ph $s7, $s3, $s2 \n" + + // s4 = | 03 | 02 | 01 | 00 | + // s5 = | 23 | 22 | 21 | 20 | + // s6 = | 13 | 12 | 11 | 10 | + // s7 = | 33 | 32 | 31 | 30 | + + "lwx $t0, $t4(%[src]) \n" + "lwx $t1, $t5(%[src]) \n" + "lwx $t8, $t6(%[src]) \n" + "lwx $t9, $t7(%[src]) \n" + +// t0 = | 34 | 24 | 14 | 04 | +// t1 = | 35 | 25 | 15 | 05 | +// t8 = | 36 | 26 | 16 | 06 | +// t9 = | 37 | 27 | 17 | 07 | + + "precr.qb.ph $s0, $t1, $t0 \n" + "precr.qb.ph $s1, $t9, $t8 \n" + "precrq.qb.ph $s2, $t1, $t0 \n" + "precrq.qb.ph $s3, $t9, $t8 \n" + + // s0 = | 25 | 05 | 24 | 04 | + // s1 = | 27 | 07 | 26 | 06 | + // s2 = | 35 | 15 | 34 | 14 | + // s3 = | 37 | 17 | 36 | 16 | + + "precr.qb.ph $t0, $s1, $s0 \n" + "precrq.qb.ph $t1, $s1, $s0 \n" + "precr.qb.ph $t8, $s3, $s2 \n" + "precrq.qb.ph $t9, $s3, $s2 \n" + + // t0 = | 07 | 06 | 05 | 04 | + // t1 = | 27 | 26 | 25 | 24 | + // t8 = | 17 | 16 | 15 | 14 | + // t9 = | 37 | 36 | 35 | 34 | + + "addu $s0, %[dst], %[dst_stride] \n" + "addu $s1, $s0, %[dst_stride] \n" + "addu $s2, $s1, %[dst_stride] \n" + + "sw $s4, 0(%[dst]) \n" + "sw $t0, 4(%[dst]) \n" + "sw $s6, 0($s0) \n" + "sw $t8, 4($s0) \n" + "sw $s5, 0($s1) \n" + "sw $t1, 4($s1) \n" + "sw $s7, 0($s2) \n" + "sw $t9, 4($s2) \n" + + "addiu $AT, -1 \n" + "addiu %[src], 4 \n" + + "bnez $AT, 1b \n" + " addu %[dst], $s2, %[dst_stride] \n" + "b 2f \n" +//dst + dst_stride unaligned + "11: \n" + "lw $t0, 0(%[src]) \n" + "lwx $t1, %[src_stride](%[src]) \n" + "lwx $t8, $t2(%[src]) \n" + "lwx $t9, $t3(%[src]) \n" + +// t0 = | 30 | 20 | 10 | 00 | +// t1 = | 31 | 21 | 11 | 01 | +// t8 = | 32 | 22 | 12 | 02 | +// t9 = | 33 | 23 | 13 | 03 | + + "precr.qb.ph $s0, $t1, $t0 \n" + "precr.qb.ph $s1, $t9, $t8 \n" + "precrq.qb.ph $s2, $t1, $t0 \n" + "precrq.qb.ph $s3, $t9, $t8 \n" + + // s0 = | 21 | 01 | 20 | 00 | + // s1 = | 23 | 03 | 22 | 02 | + // s2 = | 31 | 11 | 30 | 10 | + // s3 = | 33 | 13 | 32 | 12 | + + "precr.qb.ph $s4, $s1, $s0 \n" + "precrq.qb.ph $s5, $s1, $s0 \n" + "precr.qb.ph $s6, $s3, $s2 \n" + "precrq.qb.ph $s7, $s3, $s2 \n" + + // s4 = | 03 | 02 | 01 | 00 | + // s5 = | 23 | 22 | 21 | 20 | + // s6 = | 13 | 12 | 11 | 10 | + // s7 = | 33 | 32 | 31 | 30 | + + "lwx $t0, $t4(%[src]) \n" + "lwx $t1, $t5(%[src]) \n" + "lwx $t8, $t6(%[src]) \n" + "lwx $t9, $t7(%[src]) \n" + +// t0 = | 34 | 24 | 14 | 04 | +// t1 = | 35 | 25 | 15 | 05 | +// t8 = | 36 | 26 | 16 | 06 | +// t9 = | 37 | 27 | 17 | 07 | + + "precr.qb.ph $s0, $t1, $t0 \n" + "precr.qb.ph $s1, $t9, $t8 \n" + "precrq.qb.ph $s2, $t1, $t0 \n" + "precrq.qb.ph $s3, $t9, $t8 \n" + + // s0 = | 25 | 05 | 24 | 04 | + // s1 = | 27 | 07 | 26 | 06 | + // s2 = | 35 | 15 | 34 | 14 | + // s3 = | 37 | 17 | 36 | 16 | + + "precr.qb.ph $t0, $s1, $s0 \n" + "precrq.qb.ph $t1, $s1, $s0 \n" + "precr.qb.ph $t8, $s3, $s2 \n" + "precrq.qb.ph $t9, $s3, $s2 \n" + + // t0 = | 07 | 06 | 05 | 04 | + // t1 = | 27 | 26 | 25 | 24 | + // t8 = | 17 | 16 | 15 | 14 | + // t9 = | 37 | 36 | 35 | 34 | + + "addu $s0, %[dst], %[dst_stride] \n" + "addu $s1, $s0, %[dst_stride] \n" + "addu $s2, $s1, %[dst_stride] \n" + + "swr $s4, 0(%[dst]) \n" + "swl $s4, 3(%[dst]) \n" + "swr $t0, 4(%[dst]) \n" + "swl $t0, 7(%[dst]) \n" + "swr $s6, 0($s0) \n" + "swl $s6, 3($s0) \n" + "swr $t8, 4($s0) \n" + "swl $t8, 7($s0) \n" + "swr $s5, 0($s1) \n" + "swl $s5, 3($s1) \n" + "swr $t1, 4($s1) \n" + "swl $t1, 7($s1) \n" + "swr $s7, 0($s2) \n" + "swl $s7, 3($s2) \n" + "swr $t9, 4($s2) \n" + "swl $t9, 7($s2) \n" + + "addiu $AT, -1 \n" + "addiu %[src], 4 \n" + + "bnez $AT, 11b \n" + " addu %[dst], $s2, %[dst_stride] \n" + "2: \n" + ".set pop \n" + ".set at \n" + :[src] "+r" (src), + [dst] "+r" (dst), + [width] "+r" (width) + :[src_stride] "r" (src_stride), + [dst_stride] "r" (dst_stride) + : "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9", + "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7" + ); +} + +void TransposeUVWx8_MIPS_DSPR2(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width) { + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + "beqz %[width], 2f \n" + " sll $t2, %[src_stride], 0x1 \n" // src_stride x 2 + "sll $t4, %[src_stride], 0x2 \n" // src_stride x 4 + "sll $t9, %[src_stride], 0x3 \n" // src_stride x 8 + "addu $t3, $t2, %[src_stride] \n" + "addu $t5, $t4, %[src_stride] \n" + "addu $t6, $t2, $t4 \n" + "subu $t7, $t9, %[src_stride] \n" + "srl $t1, %[width], 1 \n" + +// check word aligment for dst_a, dst_b, dst_stride_a and dst_stride_b + "andi $t0, %[dst_a], 0x3 \n" + "andi $t8, %[dst_b], 0x3 \n" + "or $t0, $t0, $t8 \n" + "andi $t8, %[dst_stride_a], 0x3 \n" + "andi $s5, %[dst_stride_b], 0x3 \n" + "or $t8, $t8, $s5 \n" + "or $t0, $t0, $t8 \n" + "bnez $t0, 11f \n" + " nop \n" +// dst + dst_stride word aligned (both, a & b dst addresses) + "1: \n" + "lw $t0, 0(%[src]) \n" // |B0|A0|b0|a0| + "lwx $t8, %[src_stride](%[src]) \n" // |B1|A1|b1|a1| + "addu $s5, %[dst_a], %[dst_stride_a] \n" + "lwx $t9, $t2(%[src]) \n" // |B2|A2|b2|a2| + "lwx $s0, $t3(%[src]) \n" // |B3|A3|b3|a3| + "addu $s6, %[dst_b], %[dst_stride_b] \n" + + "precrq.ph.w $s1, $t8, $t0 \n" // |B1|A1|B0|A0| + "precrq.ph.w $s2, $s0, $t9 \n" // |B3|A3|B2|A2| + "precr.qb.ph $s3, $s2, $s1 \n" // |A3|A2|A1|A0| + "precrq.qb.ph $s4, $s2, $s1 \n" // |B3|B2|B1|B0| + + "sll $t0, $t0, 16 \n" + "packrl.ph $s1, $t8, $t0 \n" // |b1|a1|b0|a0| + "sll $t9, $t9, 16 \n" + "packrl.ph $s2, $s0, $t9 \n" // |b3|a3|b2|a2| + + "sw $s3, 0($s5) \n" + "sw $s4, 0($s6) \n" + + "precr.qb.ph $s3, $s2, $s1 \n" // |a3|a2|a1|a0| + "precrq.qb.ph $s4, $s2, $s1 \n" // |b3|b2|b1|b0| + + "lwx $t0, $t4(%[src]) \n" // |B4|A4|b4|a4| + "lwx $t8, $t5(%[src]) \n" // |B5|A5|b5|a5| + "lwx $t9, $t6(%[src]) \n" // |B6|A6|b6|a6| + "lwx $s0, $t7(%[src]) \n" // |B7|A7|b7|a7| + "sw $s3, 0(%[dst_a]) \n" + "sw $s4, 0(%[dst_b]) \n" + + "precrq.ph.w $s1, $t8, $t0 \n" // |B5|A5|B4|A4| + "precrq.ph.w $s2, $s0, $t9 \n" // |B6|A6|B7|A7| + "precr.qb.ph $s3, $s2, $s1 \n" // |A7|A6|A5|A4| + "precrq.qb.ph $s4, $s2, $s1 \n" // |B7|B6|B5|B4| + + "sll $t0, $t0, 16 \n" + "packrl.ph $s1, $t8, $t0 \n" // |b5|a5|b4|a4| + "sll $t9, $t9, 16 \n" + "packrl.ph $s2, $s0, $t9 \n" // |b7|a7|b6|a6| + "sw $s3, 4($s5) \n" + "sw $s4, 4($s6) \n" + + "precr.qb.ph $s3, $s2, $s1 \n" // |a7|a6|a5|a4| + "precrq.qb.ph $s4, $s2, $s1 \n" // |b7|b6|b5|b4| + + "addiu %[src], 4 \n" + "addiu $t1, -1 \n" + "sll $t0, %[dst_stride_a], 1 \n" + "sll $t8, %[dst_stride_b], 1 \n" + "sw $s3, 4(%[dst_a]) \n" + "sw $s4, 4(%[dst_b]) \n" + "addu %[dst_a], %[dst_a], $t0 \n" + "bnez $t1, 1b \n" + " addu %[dst_b], %[dst_b], $t8 \n" + "b 2f \n" + " nop \n" + +// dst_a or dst_b or dst_stride_a or dst_stride_b not word aligned + "11: \n" + "lw $t0, 0(%[src]) \n" // |B0|A0|b0|a0| + "lwx $t8, %[src_stride](%[src]) \n" // |B1|A1|b1|a1| + "addu $s5, %[dst_a], %[dst_stride_a] \n" + "lwx $t9, $t2(%[src]) \n" // |B2|A2|b2|a2| + "lwx $s0, $t3(%[src]) \n" // |B3|A3|b3|a3| + "addu $s6, %[dst_b], %[dst_stride_b] \n" + + "precrq.ph.w $s1, $t8, $t0 \n" // |B1|A1|B0|A0| + "precrq.ph.w $s2, $s0, $t9 \n" // |B3|A3|B2|A2| + "precr.qb.ph $s3, $s2, $s1 \n" // |A3|A2|A1|A0| + "precrq.qb.ph $s4, $s2, $s1 \n" // |B3|B2|B1|B0| + + "sll $t0, $t0, 16 \n" + "packrl.ph $s1, $t8, $t0 \n" // |b1|a1|b0|a0| + "sll $t9, $t9, 16 \n" + "packrl.ph $s2, $s0, $t9 \n" // |b3|a3|b2|a2| + + "swr $s3, 0($s5) \n" + "swl $s3, 3($s5) \n" + "swr $s4, 0($s6) \n" + "swl $s4, 3($s6) \n" + + "precr.qb.ph $s3, $s2, $s1 \n" // |a3|a2|a1|a0| + "precrq.qb.ph $s4, $s2, $s1 \n" // |b3|b2|b1|b0| + + "lwx $t0, $t4(%[src]) \n" // |B4|A4|b4|a4| + "lwx $t8, $t5(%[src]) \n" // |B5|A5|b5|a5| + "lwx $t9, $t6(%[src]) \n" // |B6|A6|b6|a6| + "lwx $s0, $t7(%[src]) \n" // |B7|A7|b7|a7| + "swr $s3, 0(%[dst_a]) \n" + "swl $s3, 3(%[dst_a]) \n" + "swr $s4, 0(%[dst_b]) \n" + "swl $s4, 3(%[dst_b]) \n" + + "precrq.ph.w $s1, $t8, $t0 \n" // |B5|A5|B4|A4| + "precrq.ph.w $s2, $s0, $t9 \n" // |B6|A6|B7|A7| + "precr.qb.ph $s3, $s2, $s1 \n" // |A7|A6|A5|A4| + "precrq.qb.ph $s4, $s2, $s1 \n" // |B7|B6|B5|B4| + + "sll $t0, $t0, 16 \n" + "packrl.ph $s1, $t8, $t0 \n" // |b5|a5|b4|a4| + "sll $t9, $t9, 16 \n" + "packrl.ph $s2, $s0, $t9 \n" // |b7|a7|b6|a6| + + "swr $s3, 4($s5) \n" + "swl $s3, 7($s5) \n" + "swr $s4, 4($s6) \n" + "swl $s4, 7($s6) \n" + + "precr.qb.ph $s3, $s2, $s1 \n" // |a7|a6|a5|a4| + "precrq.qb.ph $s4, $s2, $s1 \n" // |b7|b6|b5|b4| + + "addiu %[src], 4 \n" + "addiu $t1, -1 \n" + "sll $t0, %[dst_stride_a], 1 \n" + "sll $t8, %[dst_stride_b], 1 \n" + "swr $s3, 4(%[dst_a]) \n" + "swl $s3, 7(%[dst_a]) \n" + "swr $s4, 4(%[dst_b]) \n" + "swl $s4, 7(%[dst_b]) \n" + "addu %[dst_a], %[dst_a], $t0 \n" + "bnez $t1, 11b \n" + " addu %[dst_b], %[dst_b], $t8 \n" + + "2: \n" + ".set pop \n" + : [src] "+r" (src), + [dst_a] "+r" (dst_a), + [dst_b] "+r" (dst_b), + [width] "+r" (width), + [src_stride] "+r" (src_stride) + : [dst_stride_a] "r" (dst_stride_a), + [dst_stride_b] "r" (dst_stride_b) + : "t0", "t1", "t2", "t3", "t4", "t5", + "t6", "t7", "t8", "t9", + "s0", "s1", "s2", "s3", + "s4", "s5", "s6" + ); +} + +#endif // defined(__mips_dsp) && (__mips_dsp_rev >= 2) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/rotate_neon.cc b/third_party/aom/third_party/libyuv/source/rotate_neon.cc new file mode 100644 index 0000000000..76043b3b3c --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/rotate_neon.cc @@ -0,0 +1,535 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" +#include "libyuv/rotate_row.h" + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \ + !defined(__aarch64__) + +static uvec8 kVTbl4x4Transpose = + { 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 }; + +void TransposeWx8_NEON(const uint8* src, int src_stride, + uint8* dst, int dst_stride, + int width) { + const uint8* src_temp = NULL; + asm volatile ( + // loops are on blocks of 8. loop will stop when + // counter gets to or below 0. starting the counter + // at w-8 allow for this + "sub %5, #8 \n" + + // handle 8x8 blocks. this should be the majority of the plane + ".p2align 2 \n" + "1: \n" + "mov %0, %1 \n" + + MEMACCESS(0) + "vld1.8 {d0}, [%0], %2 \n" + MEMACCESS(0) + "vld1.8 {d1}, [%0], %2 \n" + MEMACCESS(0) + "vld1.8 {d2}, [%0], %2 \n" + MEMACCESS(0) + "vld1.8 {d3}, [%0], %2 \n" + MEMACCESS(0) + "vld1.8 {d4}, [%0], %2 \n" + MEMACCESS(0) + "vld1.8 {d5}, [%0], %2 \n" + MEMACCESS(0) + "vld1.8 {d6}, [%0], %2 \n" + MEMACCESS(0) + "vld1.8 {d7}, [%0] \n" + + "vtrn.8 d1, d0 \n" + "vtrn.8 d3, d2 \n" + "vtrn.8 d5, d4 \n" + "vtrn.8 d7, d6 \n" + + "vtrn.16 d1, d3 \n" + "vtrn.16 d0, d2 \n" + "vtrn.16 d5, d7 \n" + "vtrn.16 d4, d6 \n" + + "vtrn.32 d1, d5 \n" + "vtrn.32 d0, d4 \n" + "vtrn.32 d3, d7 \n" + "vtrn.32 d2, d6 \n" + + "vrev16.8 q0, q0 \n" + "vrev16.8 q1, q1 \n" + "vrev16.8 q2, q2 \n" + "vrev16.8 q3, q3 \n" + + "mov %0, %3 \n" + + MEMACCESS(0) + "vst1.8 {d1}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d0}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d3}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d2}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d5}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d4}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d7}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d6}, [%0] \n" + + "add %1, #8 \n" // src += 8 + "add %3, %3, %4, lsl #3 \n" // dst += 8 * dst_stride + "subs %5, #8 \n" // w -= 8 + "bge 1b \n" + + // add 8 back to counter. if the result is 0 there are + // no residuals. + "adds %5, #8 \n" + "beq 4f \n" + + // some residual, so between 1 and 7 lines left to transpose + "cmp %5, #2 \n" + "blt 3f \n" + + "cmp %5, #4 \n" + "blt 2f \n" + + // 4x8 block + "mov %0, %1 \n" + MEMACCESS(0) + "vld1.32 {d0[0]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.32 {d0[1]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.32 {d1[0]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.32 {d1[1]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.32 {d2[0]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.32 {d2[1]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.32 {d3[0]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.32 {d3[1]}, [%0] \n" + + "mov %0, %3 \n" + + MEMACCESS(6) + "vld1.8 {q3}, [%6] \n" + + "vtbl.8 d4, {d0, d1}, d6 \n" + "vtbl.8 d5, {d0, d1}, d7 \n" + "vtbl.8 d0, {d2, d3}, d6 \n" + "vtbl.8 d1, {d2, d3}, d7 \n" + + // TODO(frkoenig): Rework shuffle above to + // write out with 4 instead of 8 writes. + MEMACCESS(0) + "vst1.32 {d4[0]}, [%0], %4 \n" + MEMACCESS(0) + "vst1.32 {d4[1]}, [%0], %4 \n" + MEMACCESS(0) + "vst1.32 {d5[0]}, [%0], %4 \n" + MEMACCESS(0) + "vst1.32 {d5[1]}, [%0] \n" + + "add %0, %3, #4 \n" + MEMACCESS(0) + "vst1.32 {d0[0]}, [%0], %4 \n" + MEMACCESS(0) + "vst1.32 {d0[1]}, [%0], %4 \n" + MEMACCESS(0) + "vst1.32 {d1[0]}, [%0], %4 \n" + MEMACCESS(0) + "vst1.32 {d1[1]}, [%0] \n" + + "add %1, #4 \n" // src += 4 + "add %3, %3, %4, lsl #2 \n" // dst += 4 * dst_stride + "subs %5, #4 \n" // w -= 4 + "beq 4f \n" + + // some residual, check to see if it includes a 2x8 block, + // or less + "cmp %5, #2 \n" + "blt 3f \n" + + // 2x8 block + "2: \n" + "mov %0, %1 \n" + MEMACCESS(0) + "vld1.16 {d0[0]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.16 {d1[0]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.16 {d0[1]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.16 {d1[1]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.16 {d0[2]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.16 {d1[2]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.16 {d0[3]}, [%0], %2 \n" + MEMACCESS(0) + "vld1.16 {d1[3]}, [%0] \n" + + "vtrn.8 d0, d1 \n" + + "mov %0, %3 \n" + + MEMACCESS(0) + "vst1.64 {d0}, [%0], %4 \n" + MEMACCESS(0) + "vst1.64 {d1}, [%0] \n" + + "add %1, #2 \n" // src += 2 + "add %3, %3, %4, lsl #1 \n" // dst += 2 * dst_stride + "subs %5, #2 \n" // w -= 2 + "beq 4f \n" + + // 1x8 block + "3: \n" + MEMACCESS(1) + "vld1.8 {d0[0]}, [%1], %2 \n" + MEMACCESS(1) + "vld1.8 {d0[1]}, [%1], %2 \n" + MEMACCESS(1) + "vld1.8 {d0[2]}, [%1], %2 \n" + MEMACCESS(1) + "vld1.8 {d0[3]}, [%1], %2 \n" + MEMACCESS(1) + "vld1.8 {d0[4]}, [%1], %2 \n" + MEMACCESS(1) + "vld1.8 {d0[5]}, [%1], %2 \n" + MEMACCESS(1) + "vld1.8 {d0[6]}, [%1], %2 \n" + MEMACCESS(1) + "vld1.8 {d0[7]}, [%1] \n" + + MEMACCESS(3) + "vst1.64 {d0}, [%3] \n" + + "4: \n" + + : "+r"(src_temp), // %0 + "+r"(src), // %1 + "+r"(src_stride), // %2 + "+r"(dst), // %3 + "+r"(dst_stride), // %4 + "+r"(width) // %5 + : "r"(&kVTbl4x4Transpose) // %6 + : "memory", "cc", "q0", "q1", "q2", "q3" + ); +} + +static uvec8 kVTbl4x4TransposeDi = + { 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 }; + +void TransposeUVWx8_NEON(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width) { + const uint8* src_temp = NULL; + asm volatile ( + // loops are on blocks of 8. loop will stop when + // counter gets to or below 0. starting the counter + // at w-8 allow for this + "sub %7, #8 \n" + + // handle 8x8 blocks. this should be the majority of the plane + ".p2align 2 \n" + "1: \n" + "mov %0, %1 \n" + + MEMACCESS(0) + "vld2.8 {d0, d1}, [%0], %2 \n" + MEMACCESS(0) + "vld2.8 {d2, d3}, [%0], %2 \n" + MEMACCESS(0) + "vld2.8 {d4, d5}, [%0], %2 \n" + MEMACCESS(0) + "vld2.8 {d6, d7}, [%0], %2 \n" + MEMACCESS(0) + "vld2.8 {d16, d17}, [%0], %2 \n" + MEMACCESS(0) + "vld2.8 {d18, d19}, [%0], %2 \n" + MEMACCESS(0) + "vld2.8 {d20, d21}, [%0], %2 \n" + MEMACCESS(0) + "vld2.8 {d22, d23}, [%0] \n" + + "vtrn.8 q1, q0 \n" + "vtrn.8 q3, q2 \n" + "vtrn.8 q9, q8 \n" + "vtrn.8 q11, q10 \n" + + "vtrn.16 q1, q3 \n" + "vtrn.16 q0, q2 \n" + "vtrn.16 q9, q11 \n" + "vtrn.16 q8, q10 \n" + + "vtrn.32 q1, q9 \n" + "vtrn.32 q0, q8 \n" + "vtrn.32 q3, q11 \n" + "vtrn.32 q2, q10 \n" + + "vrev16.8 q0, q0 \n" + "vrev16.8 q1, q1 \n" + "vrev16.8 q2, q2 \n" + "vrev16.8 q3, q3 \n" + "vrev16.8 q8, q8 \n" + "vrev16.8 q9, q9 \n" + "vrev16.8 q10, q10 \n" + "vrev16.8 q11, q11 \n" + + "mov %0, %3 \n" + + MEMACCESS(0) + "vst1.8 {d2}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d0}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d6}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d4}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d18}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d16}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d22}, [%0], %4 \n" + MEMACCESS(0) + "vst1.8 {d20}, [%0] \n" + + "mov %0, %5 \n" + + MEMACCESS(0) + "vst1.8 {d3}, [%0], %6 \n" + MEMACCESS(0) + "vst1.8 {d1}, [%0], %6 \n" + MEMACCESS(0) + "vst1.8 {d7}, [%0], %6 \n" + MEMACCESS(0) + "vst1.8 {d5}, [%0], %6 \n" + MEMACCESS(0) + "vst1.8 {d19}, [%0], %6 \n" + MEMACCESS(0) + "vst1.8 {d17}, [%0], %6 \n" + MEMACCESS(0) + "vst1.8 {d23}, [%0], %6 \n" + MEMACCESS(0) + "vst1.8 {d21}, [%0] \n" + + "add %1, #8*2 \n" // src += 8*2 + "add %3, %3, %4, lsl #3 \n" // dst_a += 8 * dst_stride_a + "add %5, %5, %6, lsl #3 \n" // dst_b += 8 * dst_stride_b + "subs %7, #8 \n" // w -= 8 + "bge 1b \n" + + // add 8 back to counter. if the result is 0 there are + // no residuals. + "adds %7, #8 \n" + "beq 4f \n" + + // some residual, so between 1 and 7 lines left to transpose + "cmp %7, #2 \n" + "blt 3f \n" + + "cmp %7, #4 \n" + "blt 2f \n" + + // TODO(frkoenig): Clean this up + // 4x8 block + "mov %0, %1 \n" + MEMACCESS(0) + "vld1.64 {d0}, [%0], %2 \n" + MEMACCESS(0) + "vld1.64 {d1}, [%0], %2 \n" + MEMACCESS(0) + "vld1.64 {d2}, [%0], %2 \n" + MEMACCESS(0) + "vld1.64 {d3}, [%0], %2 \n" + MEMACCESS(0) + "vld1.64 {d4}, [%0], %2 \n" + MEMACCESS(0) + "vld1.64 {d5}, [%0], %2 \n" + MEMACCESS(0) + "vld1.64 {d6}, [%0], %2 \n" + MEMACCESS(0) + "vld1.64 {d7}, [%0] \n" + + MEMACCESS(8) + "vld1.8 {q15}, [%8] \n" + + "vtrn.8 q0, q1 \n" + "vtrn.8 q2, q3 \n" + + "vtbl.8 d16, {d0, d1}, d30 \n" + "vtbl.8 d17, {d0, d1}, d31 \n" + "vtbl.8 d18, {d2, d3}, d30 \n" + "vtbl.8 d19, {d2, d3}, d31 \n" + "vtbl.8 d20, {d4, d5}, d30 \n" + "vtbl.8 d21, {d4, d5}, d31 \n" + "vtbl.8 d22, {d6, d7}, d30 \n" + "vtbl.8 d23, {d6, d7}, d31 \n" + + "mov %0, %3 \n" + + MEMACCESS(0) + "vst1.32 {d16[0]}, [%0], %4 \n" + MEMACCESS(0) + "vst1.32 {d16[1]}, [%0], %4 \n" + MEMACCESS(0) + "vst1.32 {d17[0]}, [%0], %4 \n" + MEMACCESS(0) + "vst1.32 {d17[1]}, [%0], %4 \n" + + "add %0, %3, #4 \n" + MEMACCESS(0) + "vst1.32 {d20[0]}, [%0], %4 \n" + MEMACCESS(0) + "vst1.32 {d20[1]}, [%0], %4 \n" + MEMACCESS(0) + "vst1.32 {d21[0]}, [%0], %4 \n" + MEMACCESS(0) + "vst1.32 {d21[1]}, [%0] \n" + + "mov %0, %5 \n" + + MEMACCESS(0) + "vst1.32 {d18[0]}, [%0], %6 \n" + MEMACCESS(0) + "vst1.32 {d18[1]}, [%0], %6 \n" + MEMACCESS(0) + "vst1.32 {d19[0]}, [%0], %6 \n" + MEMACCESS(0) + "vst1.32 {d19[1]}, [%0], %6 \n" + + "add %0, %5, #4 \n" + MEMACCESS(0) + "vst1.32 {d22[0]}, [%0], %6 \n" + MEMACCESS(0) + "vst1.32 {d22[1]}, [%0], %6 \n" + MEMACCESS(0) + "vst1.32 {d23[0]}, [%0], %6 \n" + MEMACCESS(0) + "vst1.32 {d23[1]}, [%0] \n" + + "add %1, #4*2 \n" // src += 4 * 2 + "add %3, %3, %4, lsl #2 \n" // dst_a += 4 * dst_stride_a + "add %5, %5, %6, lsl #2 \n" // dst_b += 4 * dst_stride_b + "subs %7, #4 \n" // w -= 4 + "beq 4f \n" + + // some residual, check to see if it includes a 2x8 block, + // or less + "cmp %7, #2 \n" + "blt 3f \n" + + // 2x8 block + "2: \n" + "mov %0, %1 \n" + MEMACCESS(0) + "vld2.16 {d0[0], d2[0]}, [%0], %2 \n" + MEMACCESS(0) + "vld2.16 {d1[0], d3[0]}, [%0], %2 \n" + MEMACCESS(0) + "vld2.16 {d0[1], d2[1]}, [%0], %2 \n" + MEMACCESS(0) + "vld2.16 {d1[1], d3[1]}, [%0], %2 \n" + MEMACCESS(0) + "vld2.16 {d0[2], d2[2]}, [%0], %2 \n" + MEMACCESS(0) + "vld2.16 {d1[2], d3[2]}, [%0], %2 \n" + MEMACCESS(0) + "vld2.16 {d0[3], d2[3]}, [%0], %2 \n" + MEMACCESS(0) + "vld2.16 {d1[3], d3[3]}, [%0] \n" + + "vtrn.8 d0, d1 \n" + "vtrn.8 d2, d3 \n" + + "mov %0, %3 \n" + + MEMACCESS(0) + "vst1.64 {d0}, [%0], %4 \n" + MEMACCESS(0) + "vst1.64 {d2}, [%0] \n" + + "mov %0, %5 \n" + + MEMACCESS(0) + "vst1.64 {d1}, [%0], %6 \n" + MEMACCESS(0) + "vst1.64 {d3}, [%0] \n" + + "add %1, #2*2 \n" // src += 2 * 2 + "add %3, %3, %4, lsl #1 \n" // dst_a += 2 * dst_stride_a + "add %5, %5, %6, lsl #1 \n" // dst_b += 2 * dst_stride_b + "subs %7, #2 \n" // w -= 2 + "beq 4f \n" + + // 1x8 block + "3: \n" + MEMACCESS(1) + "vld2.8 {d0[0], d1[0]}, [%1], %2 \n" + MEMACCESS(1) + "vld2.8 {d0[1], d1[1]}, [%1], %2 \n" + MEMACCESS(1) + "vld2.8 {d0[2], d1[2]}, [%1], %2 \n" + MEMACCESS(1) + "vld2.8 {d0[3], d1[3]}, [%1], %2 \n" + MEMACCESS(1) + "vld2.8 {d0[4], d1[4]}, [%1], %2 \n" + MEMACCESS(1) + "vld2.8 {d0[5], d1[5]}, [%1], %2 \n" + MEMACCESS(1) + "vld2.8 {d0[6], d1[6]}, [%1], %2 \n" + MEMACCESS(1) + "vld2.8 {d0[7], d1[7]}, [%1] \n" + + MEMACCESS(3) + "vst1.64 {d0}, [%3] \n" + MEMACCESS(5) + "vst1.64 {d1}, [%5] \n" + + "4: \n" + + : "+r"(src_temp), // %0 + "+r"(src), // %1 + "+r"(src_stride), // %2 + "+r"(dst_a), // %3 + "+r"(dst_stride_a), // %4 + "+r"(dst_b), // %5 + "+r"(dst_stride_b), // %6 + "+r"(width) // %7 + : "r"(&kVTbl4x4TransposeDi) // %8 + : "memory", "cc", + "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11" + ); +} +#endif // defined(__ARM_NEON__) && !defined(__aarch64__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/rotate_neon64.cc b/third_party/aom/third_party/libyuv/source/rotate_neon64.cc new file mode 100644 index 0000000000..f52c082b3f --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/rotate_neon64.cc @@ -0,0 +1,543 @@ +/* + * Copyright 2014 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" +#include "libyuv/rotate_row.h" + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for GCC Neon armv8 64 bit. +#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) + +static uvec8 kVTbl4x4Transpose = + { 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 }; + +void TransposeWx8_NEON(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width) { + const uint8* src_temp = NULL; + int64 width64 = (int64) width; // Work around clang 3.4 warning. + asm volatile ( + // loops are on blocks of 8. loop will stop when + // counter gets to or below 0. starting the counter + // at w-8 allow for this + "sub %3, %3, #8 \n" + + // handle 8x8 blocks. this should be the majority of the plane + "1: \n" + "mov %0, %1 \n" + + MEMACCESS(0) + "ld1 {v0.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v1.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v2.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v3.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v4.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v5.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v6.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v7.8b}, [%0] \n" + + "trn2 v16.8b, v0.8b, v1.8b \n" + "trn1 v17.8b, v0.8b, v1.8b \n" + "trn2 v18.8b, v2.8b, v3.8b \n" + "trn1 v19.8b, v2.8b, v3.8b \n" + "trn2 v20.8b, v4.8b, v5.8b \n" + "trn1 v21.8b, v4.8b, v5.8b \n" + "trn2 v22.8b, v6.8b, v7.8b \n" + "trn1 v23.8b, v6.8b, v7.8b \n" + + "trn2 v3.4h, v17.4h, v19.4h \n" + "trn1 v1.4h, v17.4h, v19.4h \n" + "trn2 v2.4h, v16.4h, v18.4h \n" + "trn1 v0.4h, v16.4h, v18.4h \n" + "trn2 v7.4h, v21.4h, v23.4h \n" + "trn1 v5.4h, v21.4h, v23.4h \n" + "trn2 v6.4h, v20.4h, v22.4h \n" + "trn1 v4.4h, v20.4h, v22.4h \n" + + "trn2 v21.2s, v1.2s, v5.2s \n" + "trn1 v17.2s, v1.2s, v5.2s \n" + "trn2 v20.2s, v0.2s, v4.2s \n" + "trn1 v16.2s, v0.2s, v4.2s \n" + "trn2 v23.2s, v3.2s, v7.2s \n" + "trn1 v19.2s, v3.2s, v7.2s \n" + "trn2 v22.2s, v2.2s, v6.2s \n" + "trn1 v18.2s, v2.2s, v6.2s \n" + + "mov %0, %2 \n" + + MEMACCESS(0) + "st1 {v17.8b}, [%0], %6 \n" + MEMACCESS(0) + "st1 {v16.8b}, [%0], %6 \n" + MEMACCESS(0) + "st1 {v19.8b}, [%0], %6 \n" + MEMACCESS(0) + "st1 {v18.8b}, [%0], %6 \n" + MEMACCESS(0) + "st1 {v21.8b}, [%0], %6 \n" + MEMACCESS(0) + "st1 {v20.8b}, [%0], %6 \n" + MEMACCESS(0) + "st1 {v23.8b}, [%0], %6 \n" + MEMACCESS(0) + "st1 {v22.8b}, [%0] \n" + + "add %1, %1, #8 \n" // src += 8 + "add %2, %2, %6, lsl #3 \n" // dst += 8 * dst_stride + "subs %3, %3, #8 \n" // w -= 8 + "b.ge 1b \n" + + // add 8 back to counter. if the result is 0 there are + // no residuals. + "adds %3, %3, #8 \n" + "b.eq 4f \n" + + // some residual, so between 1 and 7 lines left to transpose + "cmp %3, #2 \n" + "b.lt 3f \n" + + "cmp %3, #4 \n" + "b.lt 2f \n" + + // 4x8 block + "mov %0, %1 \n" + MEMACCESS(0) + "ld1 {v0.s}[0], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v0.s}[1], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v0.s}[2], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v0.s}[3], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v1.s}[0], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v1.s}[1], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v1.s}[2], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v1.s}[3], [%0] \n" + + "mov %0, %2 \n" + + MEMACCESS(4) + "ld1 {v2.16b}, [%4] \n" + + "tbl v3.16b, {v0.16b}, v2.16b \n" + "tbl v0.16b, {v1.16b}, v2.16b \n" + + // TODO(frkoenig): Rework shuffle above to + // write out with 4 instead of 8 writes. + MEMACCESS(0) + "st1 {v3.s}[0], [%0], %6 \n" + MEMACCESS(0) + "st1 {v3.s}[1], [%0], %6 \n" + MEMACCESS(0) + "st1 {v3.s}[2], [%0], %6 \n" + MEMACCESS(0) + "st1 {v3.s}[3], [%0] \n" + + "add %0, %2, #4 \n" + MEMACCESS(0) + "st1 {v0.s}[0], [%0], %6 \n" + MEMACCESS(0) + "st1 {v0.s}[1], [%0], %6 \n" + MEMACCESS(0) + "st1 {v0.s}[2], [%0], %6 \n" + MEMACCESS(0) + "st1 {v0.s}[3], [%0] \n" + + "add %1, %1, #4 \n" // src += 4 + "add %2, %2, %6, lsl #2 \n" // dst += 4 * dst_stride + "subs %3, %3, #4 \n" // w -= 4 + "b.eq 4f \n" + + // some residual, check to see if it includes a 2x8 block, + // or less + "cmp %3, #2 \n" + "b.lt 3f \n" + + // 2x8 block + "2: \n" + "mov %0, %1 \n" + MEMACCESS(0) + "ld1 {v0.h}[0], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v1.h}[0], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v0.h}[1], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v1.h}[1], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v0.h}[2], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v1.h}[2], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v0.h}[3], [%0], %5 \n" + MEMACCESS(0) + "ld1 {v1.h}[3], [%0] \n" + + "trn2 v2.8b, v0.8b, v1.8b \n" + "trn1 v3.8b, v0.8b, v1.8b \n" + + "mov %0, %2 \n" + + MEMACCESS(0) + "st1 {v3.8b}, [%0], %6 \n" + MEMACCESS(0) + "st1 {v2.8b}, [%0] \n" + + "add %1, %1, #2 \n" // src += 2 + "add %2, %2, %6, lsl #1 \n" // dst += 2 * dst_stride + "subs %3, %3, #2 \n" // w -= 2 + "b.eq 4f \n" + + // 1x8 block + "3: \n" + MEMACCESS(1) + "ld1 {v0.b}[0], [%1], %5 \n" + MEMACCESS(1) + "ld1 {v0.b}[1], [%1], %5 \n" + MEMACCESS(1) + "ld1 {v0.b}[2], [%1], %5 \n" + MEMACCESS(1) + "ld1 {v0.b}[3], [%1], %5 \n" + MEMACCESS(1) + "ld1 {v0.b}[4], [%1], %5 \n" + MEMACCESS(1) + "ld1 {v0.b}[5], [%1], %5 \n" + MEMACCESS(1) + "ld1 {v0.b}[6], [%1], %5 \n" + MEMACCESS(1) + "ld1 {v0.b}[7], [%1] \n" + + MEMACCESS(2) + "st1 {v0.8b}, [%2] \n" + + "4: \n" + + : "+r"(src_temp), // %0 + "+r"(src), // %1 + "+r"(dst), // %2 + "+r"(width64) // %3 + : "r"(&kVTbl4x4Transpose), // %4 + "r"(static_cast(src_stride)), // %5 + "r"(static_cast(dst_stride)) // %6 + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23" + ); +} + +static uint8 kVTbl4x4TransposeDi[32] = + { 0, 16, 32, 48, 2, 18, 34, 50, 4, 20, 36, 52, 6, 22, 38, 54, + 1, 17, 33, 49, 3, 19, 35, 51, 5, 21, 37, 53, 7, 23, 39, 55}; + +void TransposeUVWx8_NEON(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int width) { + const uint8* src_temp = NULL; + int64 width64 = (int64) width; // Work around clang 3.4 warning. + asm volatile ( + // loops are on blocks of 8. loop will stop when + // counter gets to or below 0. starting the counter + // at w-8 allow for this + "sub %4, %4, #8 \n" + + // handle 8x8 blocks. this should be the majority of the plane + "1: \n" + "mov %0, %1 \n" + + MEMACCESS(0) + "ld1 {v0.16b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v1.16b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v2.16b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v3.16b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v4.16b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v5.16b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v6.16b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v7.16b}, [%0] \n" + + "trn1 v16.16b, v0.16b, v1.16b \n" + "trn2 v17.16b, v0.16b, v1.16b \n" + "trn1 v18.16b, v2.16b, v3.16b \n" + "trn2 v19.16b, v2.16b, v3.16b \n" + "trn1 v20.16b, v4.16b, v5.16b \n" + "trn2 v21.16b, v4.16b, v5.16b \n" + "trn1 v22.16b, v6.16b, v7.16b \n" + "trn2 v23.16b, v6.16b, v7.16b \n" + + "trn1 v0.8h, v16.8h, v18.8h \n" + "trn2 v1.8h, v16.8h, v18.8h \n" + "trn1 v2.8h, v20.8h, v22.8h \n" + "trn2 v3.8h, v20.8h, v22.8h \n" + "trn1 v4.8h, v17.8h, v19.8h \n" + "trn2 v5.8h, v17.8h, v19.8h \n" + "trn1 v6.8h, v21.8h, v23.8h \n" + "trn2 v7.8h, v21.8h, v23.8h \n" + + "trn1 v16.4s, v0.4s, v2.4s \n" + "trn2 v17.4s, v0.4s, v2.4s \n" + "trn1 v18.4s, v1.4s, v3.4s \n" + "trn2 v19.4s, v1.4s, v3.4s \n" + "trn1 v20.4s, v4.4s, v6.4s \n" + "trn2 v21.4s, v4.4s, v6.4s \n" + "trn1 v22.4s, v5.4s, v7.4s \n" + "trn2 v23.4s, v5.4s, v7.4s \n" + + "mov %0, %2 \n" + + MEMACCESS(0) + "st1 {v16.d}[0], [%0], %6 \n" + MEMACCESS(0) + "st1 {v18.d}[0], [%0], %6 \n" + MEMACCESS(0) + "st1 {v17.d}[0], [%0], %6 \n" + MEMACCESS(0) + "st1 {v19.d}[0], [%0], %6 \n" + MEMACCESS(0) + "st1 {v16.d}[1], [%0], %6 \n" + MEMACCESS(0) + "st1 {v18.d}[1], [%0], %6 \n" + MEMACCESS(0) + "st1 {v17.d}[1], [%0], %6 \n" + MEMACCESS(0) + "st1 {v19.d}[1], [%0] \n" + + "mov %0, %3 \n" + + MEMACCESS(0) + "st1 {v20.d}[0], [%0], %7 \n" + MEMACCESS(0) + "st1 {v22.d}[0], [%0], %7 \n" + MEMACCESS(0) + "st1 {v21.d}[0], [%0], %7 \n" + MEMACCESS(0) + "st1 {v23.d}[0], [%0], %7 \n" + MEMACCESS(0) + "st1 {v20.d}[1], [%0], %7 \n" + MEMACCESS(0) + "st1 {v22.d}[1], [%0], %7 \n" + MEMACCESS(0) + "st1 {v21.d}[1], [%0], %7 \n" + MEMACCESS(0) + "st1 {v23.d}[1], [%0] \n" + + "add %1, %1, #16 \n" // src += 8*2 + "add %2, %2, %6, lsl #3 \n" // dst_a += 8 * dst_stride_a + "add %3, %3, %7, lsl #3 \n" // dst_b += 8 * dst_stride_b + "subs %4, %4, #8 \n" // w -= 8 + "b.ge 1b \n" + + // add 8 back to counter. if the result is 0 there are + // no residuals. + "adds %4, %4, #8 \n" + "b.eq 4f \n" + + // some residual, so between 1 and 7 lines left to transpose + "cmp %4, #2 \n" + "b.lt 3f \n" + + "cmp %4, #4 \n" + "b.lt 2f \n" + + // TODO(frkoenig): Clean this up + // 4x8 block + "mov %0, %1 \n" + MEMACCESS(0) + "ld1 {v0.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v1.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v2.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v3.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v4.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v5.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v6.8b}, [%0], %5 \n" + MEMACCESS(0) + "ld1 {v7.8b}, [%0] \n" + + MEMACCESS(8) + "ld1 {v30.16b}, [%8], #16 \n" + "ld1 {v31.16b}, [%8] \n" + + "tbl v16.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v30.16b \n" + "tbl v17.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v31.16b \n" + "tbl v18.16b, {v4.16b, v5.16b, v6.16b, v7.16b}, v30.16b \n" + "tbl v19.16b, {v4.16b, v5.16b, v6.16b, v7.16b}, v31.16b \n" + + "mov %0, %2 \n" + + MEMACCESS(0) + "st1 {v16.s}[0], [%0], %6 \n" + MEMACCESS(0) + "st1 {v16.s}[1], [%0], %6 \n" + MEMACCESS(0) + "st1 {v16.s}[2], [%0], %6 \n" + MEMACCESS(0) + "st1 {v16.s}[3], [%0], %6 \n" + + "add %0, %2, #4 \n" + MEMACCESS(0) + "st1 {v18.s}[0], [%0], %6 \n" + MEMACCESS(0) + "st1 {v18.s}[1], [%0], %6 \n" + MEMACCESS(0) + "st1 {v18.s}[2], [%0], %6 \n" + MEMACCESS(0) + "st1 {v18.s}[3], [%0] \n" + + "mov %0, %3 \n" + + MEMACCESS(0) + "st1 {v17.s}[0], [%0], %7 \n" + MEMACCESS(0) + "st1 {v17.s}[1], [%0], %7 \n" + MEMACCESS(0) + "st1 {v17.s}[2], [%0], %7 \n" + MEMACCESS(0) + "st1 {v17.s}[3], [%0], %7 \n" + + "add %0, %3, #4 \n" + MEMACCESS(0) + "st1 {v19.s}[0], [%0], %7 \n" + MEMACCESS(0) + "st1 {v19.s}[1], [%0], %7 \n" + MEMACCESS(0) + "st1 {v19.s}[2], [%0], %7 \n" + MEMACCESS(0) + "st1 {v19.s}[3], [%0] \n" + + "add %1, %1, #8 \n" // src += 4 * 2 + "add %2, %2, %6, lsl #2 \n" // dst_a += 4 * dst_stride_a + "add %3, %3, %7, lsl #2 \n" // dst_b += 4 * dst_stride_b + "subs %4, %4, #4 \n" // w -= 4 + "b.eq 4f \n" + + // some residual, check to see if it includes a 2x8 block, + // or less + "cmp %4, #2 \n" + "b.lt 3f \n" + + // 2x8 block + "2: \n" + "mov %0, %1 \n" + MEMACCESS(0) + "ld2 {v0.h, v1.h}[0], [%0], %5 \n" + MEMACCESS(0) + "ld2 {v2.h, v3.h}[0], [%0], %5 \n" + MEMACCESS(0) + "ld2 {v0.h, v1.h}[1], [%0], %5 \n" + MEMACCESS(0) + "ld2 {v2.h, v3.h}[1], [%0], %5 \n" + MEMACCESS(0) + "ld2 {v0.h, v1.h}[2], [%0], %5 \n" + MEMACCESS(0) + "ld2 {v2.h, v3.h}[2], [%0], %5 \n" + MEMACCESS(0) + "ld2 {v0.h, v1.h}[3], [%0], %5 \n" + MEMACCESS(0) + "ld2 {v2.h, v3.h}[3], [%0] \n" + + "trn1 v4.8b, v0.8b, v2.8b \n" + "trn2 v5.8b, v0.8b, v2.8b \n" + "trn1 v6.8b, v1.8b, v3.8b \n" + "trn2 v7.8b, v1.8b, v3.8b \n" + + "mov %0, %2 \n" + + MEMACCESS(0) + "st1 {v4.d}[0], [%0], %6 \n" + MEMACCESS(0) + "st1 {v6.d}[0], [%0] \n" + + "mov %0, %3 \n" + + MEMACCESS(0) + "st1 {v5.d}[0], [%0], %7 \n" + MEMACCESS(0) + "st1 {v7.d}[0], [%0] \n" + + "add %1, %1, #4 \n" // src += 2 * 2 + "add %2, %2, %6, lsl #1 \n" // dst_a += 2 * dst_stride_a + "add %3, %3, %7, lsl #1 \n" // dst_b += 2 * dst_stride_b + "subs %4, %4, #2 \n" // w -= 2 + "b.eq 4f \n" + + // 1x8 block + "3: \n" + MEMACCESS(1) + "ld2 {v0.b, v1.b}[0], [%1], %5 \n" + MEMACCESS(1) + "ld2 {v0.b, v1.b}[1], [%1], %5 \n" + MEMACCESS(1) + "ld2 {v0.b, v1.b}[2], [%1], %5 \n" + MEMACCESS(1) + "ld2 {v0.b, v1.b}[3], [%1], %5 \n" + MEMACCESS(1) + "ld2 {v0.b, v1.b}[4], [%1], %5 \n" + MEMACCESS(1) + "ld2 {v0.b, v1.b}[5], [%1], %5 \n" + MEMACCESS(1) + "ld2 {v0.b, v1.b}[6], [%1], %5 \n" + MEMACCESS(1) + "ld2 {v0.b, v1.b}[7], [%1] \n" + + MEMACCESS(2) + "st1 {v0.d}[0], [%2] \n" + MEMACCESS(3) + "st1 {v1.d}[0], [%3] \n" + + "4: \n" + + : "+r"(src_temp), // %0 + "+r"(src), // %1 + "+r"(dst_a), // %2 + "+r"(dst_b), // %3 + "+r"(width64) // %4 + : "r"(static_cast(src_stride)), // %5 + "r"(static_cast(dst_stride_a)), // %6 + "r"(static_cast(dst_stride_b)), // %7 + "r"(&kVTbl4x4TransposeDi) // %8 + : "memory", "cc", + "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v30", "v31" + ); +} +#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/rotate_win.cc b/third_party/aom/third_party/libyuv/source/rotate_win.cc new file mode 100644 index 0000000000..2760066dfd --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/rotate_win.cc @@ -0,0 +1,248 @@ +/* + * Copyright 2013 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" +#include "libyuv/rotate_row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for Visual C x86. +#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && \ + defined(_MSC_VER) && !defined(__clang__) + +__declspec(naked) +void TransposeWx8_SSSE3(const uint8* src, int src_stride, + uint8* dst, int dst_stride, int width) { + __asm { + push edi + push esi + push ebp + mov eax, [esp + 12 + 4] // src + mov edi, [esp + 12 + 8] // src_stride + mov edx, [esp + 12 + 12] // dst + mov esi, [esp + 12 + 16] // dst_stride + mov ecx, [esp + 12 + 20] // width + + // Read in the data from the source pointer. + // First round of bit swap. + align 4 + convertloop: + movq xmm0, qword ptr [eax] + lea ebp, [eax + 8] + movq xmm1, qword ptr [eax + edi] + lea eax, [eax + 2 * edi] + punpcklbw xmm0, xmm1 + movq xmm2, qword ptr [eax] + movdqa xmm1, xmm0 + palignr xmm1, xmm1, 8 + movq xmm3, qword ptr [eax + edi] + lea eax, [eax + 2 * edi] + punpcklbw xmm2, xmm3 + movdqa xmm3, xmm2 + movq xmm4, qword ptr [eax] + palignr xmm3, xmm3, 8 + movq xmm5, qword ptr [eax + edi] + punpcklbw xmm4, xmm5 + lea eax, [eax + 2 * edi] + movdqa xmm5, xmm4 + movq xmm6, qword ptr [eax] + palignr xmm5, xmm5, 8 + movq xmm7, qword ptr [eax + edi] + punpcklbw xmm6, xmm7 + mov eax, ebp + movdqa xmm7, xmm6 + palignr xmm7, xmm7, 8 + // Second round of bit swap. + punpcklwd xmm0, xmm2 + punpcklwd xmm1, xmm3 + movdqa xmm2, xmm0 + movdqa xmm3, xmm1 + palignr xmm2, xmm2, 8 + palignr xmm3, xmm3, 8 + punpcklwd xmm4, xmm6 + punpcklwd xmm5, xmm7 + movdqa xmm6, xmm4 + movdqa xmm7, xmm5 + palignr xmm6, xmm6, 8 + palignr xmm7, xmm7, 8 + // Third round of bit swap. + // Write to the destination pointer. + punpckldq xmm0, xmm4 + movq qword ptr [edx], xmm0 + movdqa xmm4, xmm0 + palignr xmm4, xmm4, 8 + movq qword ptr [edx + esi], xmm4 + lea edx, [edx + 2 * esi] + punpckldq xmm2, xmm6 + movdqa xmm6, xmm2 + palignr xmm6, xmm6, 8 + movq qword ptr [edx], xmm2 + punpckldq xmm1, xmm5 + movq qword ptr [edx + esi], xmm6 + lea edx, [edx + 2 * esi] + movdqa xmm5, xmm1 + movq qword ptr [edx], xmm1 + palignr xmm5, xmm5, 8 + punpckldq xmm3, xmm7 + movq qword ptr [edx + esi], xmm5 + lea edx, [edx + 2 * esi] + movq qword ptr [edx], xmm3 + movdqa xmm7, xmm3 + palignr xmm7, xmm7, 8 + sub ecx, 8 + movq qword ptr [edx + esi], xmm7 + lea edx, [edx + 2 * esi] + jg convertloop + + pop ebp + pop esi + pop edi + ret + } +} + +__declspec(naked) +void TransposeUVWx8_SSE2(const uint8* src, int src_stride, + uint8* dst_a, int dst_stride_a, + uint8* dst_b, int dst_stride_b, + int w) { + __asm { + push ebx + push esi + push edi + push ebp + mov eax, [esp + 16 + 4] // src + mov edi, [esp + 16 + 8] // src_stride + mov edx, [esp + 16 + 12] // dst_a + mov esi, [esp + 16 + 16] // dst_stride_a + mov ebx, [esp + 16 + 20] // dst_b + mov ebp, [esp + 16 + 24] // dst_stride_b + mov ecx, esp + sub esp, 4 + 16 + and esp, ~15 + mov [esp + 16], ecx + mov ecx, [ecx + 16 + 28] // w + + align 4 + convertloop: + // Read in the data from the source pointer. + // First round of bit swap. + movdqu xmm0, [eax] + movdqu xmm1, [eax + edi] + lea eax, [eax + 2 * edi] + movdqa xmm7, xmm0 // use xmm7 as temp register. + punpcklbw xmm0, xmm1 + punpckhbw xmm7, xmm1 + movdqa xmm1, xmm7 + movdqu xmm2, [eax] + movdqu xmm3, [eax + edi] + lea eax, [eax + 2 * edi] + movdqa xmm7, xmm2 + punpcklbw xmm2, xmm3 + punpckhbw xmm7, xmm3 + movdqa xmm3, xmm7 + movdqu xmm4, [eax] + movdqu xmm5, [eax + edi] + lea eax, [eax + 2 * edi] + movdqa xmm7, xmm4 + punpcklbw xmm4, xmm5 + punpckhbw xmm7, xmm5 + movdqa xmm5, xmm7 + movdqu xmm6, [eax] + movdqu xmm7, [eax + edi] + lea eax, [eax + 2 * edi] + movdqu [esp], xmm5 // backup xmm5 + neg edi + movdqa xmm5, xmm6 // use xmm5 as temp register. + punpcklbw xmm6, xmm7 + punpckhbw xmm5, xmm7 + movdqa xmm7, xmm5 + lea eax, [eax + 8 * edi + 16] + neg edi + // Second round of bit swap. + movdqa xmm5, xmm0 + punpcklwd xmm0, xmm2 + punpckhwd xmm5, xmm2 + movdqa xmm2, xmm5 + movdqa xmm5, xmm1 + punpcklwd xmm1, xmm3 + punpckhwd xmm5, xmm3 + movdqa xmm3, xmm5 + movdqa xmm5, xmm4 + punpcklwd xmm4, xmm6 + punpckhwd xmm5, xmm6 + movdqa xmm6, xmm5 + movdqu xmm5, [esp] // restore xmm5 + movdqu [esp], xmm6 // backup xmm6 + movdqa xmm6, xmm5 // use xmm6 as temp register. + punpcklwd xmm5, xmm7 + punpckhwd xmm6, xmm7 + movdqa xmm7, xmm6 + // Third round of bit swap. + // Write to the destination pointer. + movdqa xmm6, xmm0 + punpckldq xmm0, xmm4 + punpckhdq xmm6, xmm4 + movdqa xmm4, xmm6 + movdqu xmm6, [esp] // restore xmm6 + movlpd qword ptr [edx], xmm0 + movhpd qword ptr [ebx], xmm0 + movlpd qword ptr [edx + esi], xmm4 + lea edx, [edx + 2 * esi] + movhpd qword ptr [ebx + ebp], xmm4 + lea ebx, [ebx + 2 * ebp] + movdqa xmm0, xmm2 // use xmm0 as the temp register. + punpckldq xmm2, xmm6 + movlpd qword ptr [edx], xmm2 + movhpd qword ptr [ebx], xmm2 + punpckhdq xmm0, xmm6 + movlpd qword ptr [edx + esi], xmm0 + lea edx, [edx + 2 * esi] + movhpd qword ptr [ebx + ebp], xmm0 + lea ebx, [ebx + 2 * ebp] + movdqa xmm0, xmm1 // use xmm0 as the temp register. + punpckldq xmm1, xmm5 + movlpd qword ptr [edx], xmm1 + movhpd qword ptr [ebx], xmm1 + punpckhdq xmm0, xmm5 + movlpd qword ptr [edx + esi], xmm0 + lea edx, [edx + 2 * esi] + movhpd qword ptr [ebx + ebp], xmm0 + lea ebx, [ebx + 2 * ebp] + movdqa xmm0, xmm3 // use xmm0 as the temp register. + punpckldq xmm3, xmm7 + movlpd qword ptr [edx], xmm3 + movhpd qword ptr [ebx], xmm3 + punpckhdq xmm0, xmm7 + sub ecx, 8 + movlpd qword ptr [edx + esi], xmm0 + lea edx, [edx + 2 * esi] + movhpd qword ptr [ebx + ebp], xmm0 + lea ebx, [ebx + 2 * ebp] + jg convertloop + + mov esp, [esp + 16] + pop ebp + pop edi + pop esi + pop ebx + ret + } +} + +#endif // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/row_any.cc b/third_party/aom/third_party/libyuv/source/row_any.cc new file mode 100644 index 0000000000..1cb1f6b930 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/row_any.cc @@ -0,0 +1,680 @@ +/* + * Copyright 2012 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" + +#include // For memset. + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Subsampled source needs to be increase by 1 of not even. +#define SS(width, shift) (((width) + (1 << (shift)) - 1) >> (shift)) + +// Any 3 planes to 1. +#define ANY31(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, BPP, MASK) \ + void NAMEANY(const uint8* y_buf, const uint8* u_buf, const uint8* v_buf, \ + uint8* dst_ptr, int width) { \ + SIMD_ALIGNED(uint8 temp[64 * 4]); \ + memset(temp, 0, 64 * 3); /* for YUY2 and msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(y_buf, u_buf, v_buf, dst_ptr, n); \ + } \ + memcpy(temp, y_buf + n, r); \ + memcpy(temp + 64, u_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \ + memcpy(temp + 128, v_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \ + ANY_SIMD(temp, temp + 64, temp + 128, temp + 192, MASK + 1); \ + memcpy(dst_ptr + (n >> DUVSHIFT) * BPP, temp + 192, \ + SS(r, DUVSHIFT) * BPP); \ + } + +#ifdef HAS_I422TOARGBROW_SSSE3 +ANY31(I422ToARGBRow_Any_SSSE3, I422ToARGBRow_SSSE3, 1, 0, 4, 7) +#endif +#ifdef HAS_I444TOARGBROW_SSSE3 +ANY31(I444ToARGBRow_Any_SSSE3, I444ToARGBRow_SSSE3, 0, 0, 4, 7) +ANY31(I411ToARGBRow_Any_SSSE3, I411ToARGBRow_SSSE3, 2, 0, 4, 7) +ANY31(I422ToBGRARow_Any_SSSE3, I422ToBGRARow_SSSE3, 1, 0, 4, 7) +ANY31(I422ToABGRRow_Any_SSSE3, I422ToABGRRow_SSSE3, 1, 0, 4, 7) +ANY31(I422ToRGBARow_Any_SSSE3, I422ToRGBARow_SSSE3, 1, 0, 4, 7) +ANY31(I422ToARGB4444Row_Any_SSSE3, I422ToARGB4444Row_SSSE3, 1, 0, 2, 7) +ANY31(I422ToARGB1555Row_Any_SSSE3, I422ToARGB1555Row_SSSE3, 1, 0, 2, 7) +ANY31(I422ToRGB565Row_Any_SSSE3, I422ToRGB565Row_SSSE3, 1, 0, 2, 7) +ANY31(I422ToRGB24Row_Any_SSSE3, I422ToRGB24Row_SSSE3, 1, 0, 3, 7) +ANY31(I422ToRAWRow_Any_SSSE3, I422ToRAWRow_SSSE3, 1, 0, 3, 7) +ANY31(I422ToYUY2Row_Any_SSE2, I422ToYUY2Row_SSE2, 1, 1, 4, 15) +ANY31(I422ToUYVYRow_Any_SSE2, I422ToUYVYRow_SSE2, 1, 1, 4, 15) +#endif // HAS_I444TOARGBROW_SSSE3 +#ifdef HAS_I422TORGB24ROW_AVX2 +ANY31(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15) +#endif +#ifdef HAS_I422TORAWROW_AVX2 +ANY31(I422ToRAWRow_Any_AVX2, I422ToRAWRow_AVX2, 1, 0, 3, 15) +#endif +#ifdef HAS_J422TOARGBROW_SSSE3 +ANY31(J422ToARGBRow_Any_SSSE3, J422ToARGBRow_SSSE3, 1, 0, 4, 7) +#endif +#ifdef HAS_J422TOARGBROW_AVX2 +ANY31(J422ToARGBRow_Any_AVX2, J422ToARGBRow_AVX2, 1, 0, 4, 15) +#endif +#ifdef HAS_I422TOARGBROW_AVX2 +ANY31(I422ToARGBRow_Any_AVX2, I422ToARGBRow_AVX2, 1, 0, 4, 15) +#endif +#ifdef HAS_I422TOBGRAROW_AVX2 +ANY31(I422ToBGRARow_Any_AVX2, I422ToBGRARow_AVX2, 1, 0, 4, 15) +#endif +#ifdef HAS_I422TORGBAROW_AVX2 +ANY31(I422ToRGBARow_Any_AVX2, I422ToRGBARow_AVX2, 1, 0, 4, 15) +#endif +#ifdef HAS_I422TOABGRROW_AVX2 +ANY31(I422ToABGRRow_Any_AVX2, I422ToABGRRow_AVX2, 1, 0, 4, 15) +#endif +#ifdef HAS_I444TOARGBROW_AVX2 +ANY31(I444ToARGBRow_Any_AVX2, I444ToARGBRow_AVX2, 0, 0, 4, 15) +#endif +#ifdef HAS_I411TOARGBROW_AVX2 +ANY31(I411ToARGBRow_Any_AVX2, I411ToARGBRow_AVX2, 2, 0, 4, 15) +#endif +#ifdef HAS_I422TOARGB4444ROW_AVX2 +ANY31(I422ToARGB4444Row_Any_AVX2, I422ToARGB4444Row_AVX2, 1, 0, 2, 7) +#endif +#ifdef HAS_I422TOARGB1555ROW_AVX2 +ANY31(I422ToARGB1555Row_Any_AVX2, I422ToARGB1555Row_AVX2, 1, 0, 2, 7) +#endif +#ifdef HAS_I422TORGB565ROW_AVX2 +ANY31(I422ToRGB565Row_Any_AVX2, I422ToRGB565Row_AVX2, 1, 0, 2, 7) +#endif +#ifdef HAS_I422TOARGBROW_NEON +ANY31(I444ToARGBRow_Any_NEON, I444ToARGBRow_NEON, 0, 0, 4, 7) +ANY31(I422ToARGBRow_Any_NEON, I422ToARGBRow_NEON, 1, 0, 4, 7) +ANY31(I411ToARGBRow_Any_NEON, I411ToARGBRow_NEON, 2, 0, 4, 7) +ANY31(I422ToBGRARow_Any_NEON, I422ToBGRARow_NEON, 1, 0, 4, 7) +ANY31(I422ToABGRRow_Any_NEON, I422ToABGRRow_NEON, 1, 0, 4, 7) +ANY31(I422ToRGBARow_Any_NEON, I422ToRGBARow_NEON, 1, 0, 4, 7) +ANY31(I422ToRGB24Row_Any_NEON, I422ToRGB24Row_NEON, 1, 0, 3, 7) +ANY31(I422ToRAWRow_Any_NEON, I422ToRAWRow_NEON, 1, 0, 3, 7) +ANY31(I422ToARGB4444Row_Any_NEON, I422ToARGB4444Row_NEON, 1, 0, 2, 7) +ANY31(I422ToARGB1555Row_Any_NEON, I422ToARGB1555Row_NEON, 1, 0, 2, 7) +ANY31(I422ToRGB565Row_Any_NEON, I422ToRGB565Row_NEON, 1, 0, 2, 7) +#endif +#ifdef HAS_I422TOYUY2ROW_NEON +ANY31(I422ToYUY2Row_Any_NEON, I422ToYUY2Row_NEON, 1, 1, 4, 15) +#endif +#ifdef HAS_I422TOUYVYROW_NEON +ANY31(I422ToUYVYRow_Any_NEON, I422ToUYVYRow_NEON, 1, 1, 4, 15) +#endif +#undef ANY31 + +// Any 2 planes to 1. +#define ANY21(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, SBPP2, BPP, MASK) \ + void NAMEANY(const uint8* y_buf, const uint8* uv_buf, \ + uint8* dst_ptr, int width) { \ + SIMD_ALIGNED(uint8 temp[64 * 3]); \ + memset(temp, 0, 64 * 2); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(y_buf, uv_buf, dst_ptr, n); \ + } \ + memcpy(temp, y_buf + n * SBPP, r * SBPP); \ + memcpy(temp + 64, uv_buf + (n >> UVSHIFT) * SBPP2, \ + SS(r, UVSHIFT) * SBPP2); \ + ANY_SIMD(temp, temp + 64, temp + 128, MASK + 1); \ + memcpy(dst_ptr + n * BPP, temp + 128, r * BPP); \ + } + +// Biplanar to RGB. +#ifdef HAS_NV12TOARGBROW_SSSE3 +ANY21(NV12ToARGBRow_Any_SSSE3, NV12ToARGBRow_SSSE3, 1, 1, 2, 4, 7) +ANY21(NV21ToARGBRow_Any_SSSE3, NV21ToARGBRow_SSSE3, 1, 1, 2, 4, 7) +#endif +#ifdef HAS_NV12TOARGBROW_AVX2 +ANY21(NV12ToARGBRow_Any_AVX2, NV12ToARGBRow_AVX2, 1, 1, 2, 4, 15) +ANY21(NV21ToARGBRow_Any_AVX2, NV21ToARGBRow_AVX2, 1, 1, 2, 4, 15) +#endif +#ifdef HAS_NV12TOARGBROW_NEON +ANY21(NV12ToARGBRow_Any_NEON, NV12ToARGBRow_NEON, 1, 1, 2, 4, 7) +ANY21(NV21ToARGBRow_Any_NEON, NV21ToARGBRow_NEON, 1, 1, 2, 4, 7) +#endif +#ifdef HAS_NV12TORGB565ROW_SSSE3 +ANY21(NV12ToRGB565Row_Any_SSSE3, NV12ToRGB565Row_SSSE3, 1, 1, 2, 2, 7) +ANY21(NV21ToRGB565Row_Any_SSSE3, NV21ToRGB565Row_SSSE3, 1, 1, 2, 2, 7) +#endif +#ifdef HAS_NV12TORGB565ROW_AVX2 +ANY21(NV12ToRGB565Row_Any_AVX2, NV12ToRGB565Row_AVX2, 1, 1, 2, 2, 15) +ANY21(NV21ToRGB565Row_Any_AVX2, NV21ToRGB565Row_AVX2, 1, 1, 2, 2, 15) +#endif +#ifdef HAS_NV12TORGB565ROW_NEON +ANY21(NV12ToRGB565Row_Any_NEON, NV12ToRGB565Row_NEON, 1, 1, 2, 2, 7) +ANY21(NV21ToRGB565Row_Any_NEON, NV21ToRGB565Row_NEON, 1, 1, 2, 2, 7) +#endif + +// Merge functions. +#ifdef HAS_MERGEUVROW_SSE2 +ANY21(MergeUVRow_Any_SSE2, MergeUVRow_SSE2, 0, 1, 1, 2, 15) +#endif +#ifdef HAS_MERGEUVROW_AVX2 +ANY21(MergeUVRow_Any_AVX2, MergeUVRow_AVX2, 0, 1, 1, 2, 31) +#endif +#ifdef HAS_MERGEUVROW_NEON +ANY21(MergeUVRow_Any_NEON, MergeUVRow_NEON, 0, 1, 1, 2, 15) +#endif + +// Math functions. +#ifdef HAS_ARGBMULTIPLYROW_SSE2 +ANY21(ARGBMultiplyRow_Any_SSE2, ARGBMultiplyRow_SSE2, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_ARGBADDROW_SSE2 +ANY21(ARGBAddRow_Any_SSE2, ARGBAddRow_SSE2, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_ARGBSUBTRACTROW_SSE2 +ANY21(ARGBSubtractRow_Any_SSE2, ARGBSubtractRow_SSE2, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_ARGBMULTIPLYROW_AVX2 +ANY21(ARGBMultiplyRow_Any_AVX2, ARGBMultiplyRow_AVX2, 0, 4, 4, 4, 7) +#endif +#ifdef HAS_ARGBADDROW_AVX2 +ANY21(ARGBAddRow_Any_AVX2, ARGBAddRow_AVX2, 0, 4, 4, 4, 7) +#endif +#ifdef HAS_ARGBSUBTRACTROW_AVX2 +ANY21(ARGBSubtractRow_Any_AVX2, ARGBSubtractRow_AVX2, 0, 4, 4, 4, 7) +#endif +#ifdef HAS_ARGBMULTIPLYROW_NEON +ANY21(ARGBMultiplyRow_Any_NEON, ARGBMultiplyRow_NEON, 0, 4, 4, 4, 7) +#endif +#ifdef HAS_ARGBADDROW_NEON +ANY21(ARGBAddRow_Any_NEON, ARGBAddRow_NEON, 0, 4, 4, 4, 7) +#endif +#ifdef HAS_ARGBSUBTRACTROW_NEON +ANY21(ARGBSubtractRow_Any_NEON, ARGBSubtractRow_NEON, 0, 4, 4, 4, 7) +#endif +#ifdef HAS_SOBELROW_SSE2 +ANY21(SobelRow_Any_SSE2, SobelRow_SSE2, 0, 1, 1, 4, 15) +#endif +#ifdef HAS_SOBELROW_NEON +ANY21(SobelRow_Any_NEON, SobelRow_NEON, 0, 1, 1, 4, 7) +#endif +#ifdef HAS_SOBELTOPLANEROW_SSE2 +ANY21(SobelToPlaneRow_Any_SSE2, SobelToPlaneRow_SSE2, 0, 1, 1, 1, 15) +#endif +#ifdef HAS_SOBELTOPLANEROW_NEON +ANY21(SobelToPlaneRow_Any_NEON, SobelToPlaneRow_NEON, 0, 1, 1, 1, 15) +#endif +#ifdef HAS_SOBELXYROW_SSE2 +ANY21(SobelXYRow_Any_SSE2, SobelXYRow_SSE2, 0, 1, 1, 4, 15) +#endif +#ifdef HAS_SOBELXYROW_NEON +ANY21(SobelXYRow_Any_NEON, SobelXYRow_NEON, 0, 1, 1, 4, 7) +#endif +#undef ANY21 + +// Any 1 to 1. +#define ANY11(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, BPP, MASK) \ + void NAMEANY(const uint8* src_ptr, uint8* dst_ptr, int width) { \ + SIMD_ALIGNED(uint8 temp[128 * 2]); \ + memset(temp, 0, 128); /* for YUY2 and msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_ptr, dst_ptr, n); \ + } \ + memcpy(temp, src_ptr + (n >> UVSHIFT) * SBPP, SS(r, UVSHIFT) * SBPP); \ + ANY_SIMD(temp, temp + 128, MASK + 1); \ + memcpy(dst_ptr + n * BPP, temp + 128, r * BPP); \ + } + +#ifdef HAS_COPYROW_AVX +ANY11(CopyRow_Any_AVX, CopyRow_AVX, 0, 1, 1, 63) +#endif +#ifdef HAS_COPYROW_SSE2 +ANY11(CopyRow_Any_SSE2, CopyRow_SSE2, 0, 1, 1, 31) +#endif +#ifdef HAS_COPYROW_NEON +ANY11(CopyRow_Any_NEON, CopyRow_NEON, 0, 1, 1, 31) +#endif +#if defined(HAS_ARGBTORGB24ROW_SSSE3) +ANY11(ARGBToRGB24Row_Any_SSSE3, ARGBToRGB24Row_SSSE3, 0, 4, 3, 15) +ANY11(ARGBToRAWRow_Any_SSSE3, ARGBToRAWRow_SSSE3, 0, 4, 3, 15) +ANY11(ARGBToRGB565Row_Any_SSE2, ARGBToRGB565Row_SSE2, 0, 4, 2, 3) +ANY11(ARGBToARGB1555Row_Any_SSE2, ARGBToARGB1555Row_SSE2, 0, 4, 2, 3) +ANY11(ARGBToARGB4444Row_Any_SSE2, ARGBToARGB4444Row_SSE2, 0, 4, 2, 3) +#endif +#if defined(HAS_ARGBTOARGB4444ROW_AVX2) +ANY11(ARGBToRGB565Row_Any_AVX2, ARGBToRGB565Row_AVX2, 0, 4, 2, 7) +ANY11(ARGBToARGB1555Row_Any_AVX2, ARGBToARGB1555Row_AVX2, 0, 4, 2, 7) +ANY11(ARGBToARGB4444Row_Any_AVX2, ARGBToARGB4444Row_AVX2, 0, 4, 2, 7) +#endif +#if defined(HAS_J400TOARGBROW_SSE2) +ANY11(J400ToARGBRow_Any_SSE2, J400ToARGBRow_SSE2, 0, 1, 4, 7) +#endif +#if defined(HAS_J400TOARGBROW_AVX2) +ANY11(J400ToARGBRow_Any_AVX2, J400ToARGBRow_AVX2, 0, 1, 4, 15) +#endif +#if defined(HAS_I400TOARGBROW_SSE2) +ANY11(I400ToARGBRow_Any_SSE2, I400ToARGBRow_SSE2, 0, 1, 4, 7) +#endif +#if defined(HAS_I400TOARGBROW_AVX2) +ANY11(I400ToARGBRow_Any_AVX2, I400ToARGBRow_AVX2, 0, 1, 4, 15) +#endif +#if defined(HAS_YUY2TOARGBROW_SSSE3) +ANY11(YUY2ToARGBRow_Any_SSSE3, YUY2ToARGBRow_SSSE3, 1, 4, 4, 15) +ANY11(UYVYToARGBRow_Any_SSSE3, UYVYToARGBRow_SSSE3, 1, 4, 4, 15) +ANY11(RGB24ToARGBRow_Any_SSSE3, RGB24ToARGBRow_SSSE3, 0, 3, 4, 15) +ANY11(RAWToARGBRow_Any_SSSE3, RAWToARGBRow_SSSE3, 0, 3, 4, 15) +ANY11(RGB565ToARGBRow_Any_SSE2, RGB565ToARGBRow_SSE2, 0, 2, 4, 7) +ANY11(ARGB1555ToARGBRow_Any_SSE2, ARGB1555ToARGBRow_SSE2, 0, 2, 4, 7) +ANY11(ARGB4444ToARGBRow_Any_SSE2, ARGB4444ToARGBRow_SSE2, 0, 2, 4, 7) +#endif +#if defined(HAS_RGB565TOARGBROW_AVX2) +ANY11(RGB565ToARGBRow_Any_AVX2, RGB565ToARGBRow_AVX2, 0, 2, 4, 15) +#endif +#if defined(HAS_ARGB1555TOARGBROW_AVX2) +ANY11(ARGB1555ToARGBRow_Any_AVX2, ARGB1555ToARGBRow_AVX2, 0, 2, 4, 15) +#endif +#if defined(HAS_ARGB4444TOARGBROW_AVX2) +ANY11(ARGB4444ToARGBRow_Any_AVX2, ARGB4444ToARGBRow_AVX2, 0, 2, 4, 15) +#endif +#if defined(HAS_YUY2TOARGBROW_AVX2) +ANY11(YUY2ToARGBRow_Any_AVX2, YUY2ToARGBRow_AVX2, 1, 4, 4, 31) +ANY11(UYVYToARGBRow_Any_AVX2, UYVYToARGBRow_AVX2, 1, 4, 4, 31) +#endif +#if defined(HAS_ARGBTORGB24ROW_NEON) +ANY11(ARGBToRGB24Row_Any_NEON, ARGBToRGB24Row_NEON, 0, 4, 3, 7) +ANY11(ARGBToRAWRow_Any_NEON, ARGBToRAWRow_NEON, 0, 4, 3, 7) +ANY11(ARGBToRGB565Row_Any_NEON, ARGBToRGB565Row_NEON, 0, 4, 2, 7) +ANY11(ARGBToARGB1555Row_Any_NEON, ARGBToARGB1555Row_NEON, 0, 4, 2, 7) +ANY11(ARGBToARGB4444Row_Any_NEON, ARGBToARGB4444Row_NEON, 0, 4, 2, 7) +ANY11(J400ToARGBRow_Any_NEON, J400ToARGBRow_NEON, 0, 1, 4, 7) +ANY11(I400ToARGBRow_Any_NEON, I400ToARGBRow_NEON, 0, 1, 4, 7) +ANY11(YUY2ToARGBRow_Any_NEON, YUY2ToARGBRow_NEON, 1, 4, 4, 7) +ANY11(UYVYToARGBRow_Any_NEON, UYVYToARGBRow_NEON, 1, 4, 4, 7) +#endif +#ifdef HAS_ARGBTOYROW_AVX2 +ANY11(ARGBToYRow_Any_AVX2, ARGBToYRow_AVX2, 0, 4, 1, 31) +#endif +#ifdef HAS_ARGBTOYJROW_AVX2 +ANY11(ARGBToYJRow_Any_AVX2, ARGBToYJRow_AVX2, 0, 4, 1, 31) +#endif +#ifdef HAS_UYVYTOYROW_AVX2 +ANY11(UYVYToYRow_Any_AVX2, UYVYToYRow_AVX2, 0, 2, 1, 31) +#endif +#ifdef HAS_YUY2TOYROW_AVX2 +ANY11(YUY2ToYRow_Any_AVX2, YUY2ToYRow_AVX2, 1, 4, 1, 31) +#endif +#ifdef HAS_ARGBTOYROW_SSSE3 +ANY11(ARGBToYRow_Any_SSSE3, ARGBToYRow_SSSE3, 0, 4, 1, 15) +#endif +#ifdef HAS_BGRATOYROW_SSSE3 +ANY11(BGRAToYRow_Any_SSSE3, BGRAToYRow_SSSE3, 0, 4, 1, 15) +ANY11(ABGRToYRow_Any_SSSE3, ABGRToYRow_SSSE3, 0, 4, 1, 15) +ANY11(RGBAToYRow_Any_SSSE3, RGBAToYRow_SSSE3, 0, 4, 1, 15) +ANY11(YUY2ToYRow_Any_SSE2, YUY2ToYRow_SSE2, 1, 4, 1, 15) +ANY11(UYVYToYRow_Any_SSE2, UYVYToYRow_SSE2, 1, 4, 1, 15) +#endif +#ifdef HAS_ARGBTOYJROW_SSSE3 +ANY11(ARGBToYJRow_Any_SSSE3, ARGBToYJRow_SSSE3, 0, 4, 1, 15) +#endif +#ifdef HAS_ARGBTOYROW_NEON +ANY11(ARGBToYRow_Any_NEON, ARGBToYRow_NEON, 0, 4, 1, 7) +#endif +#ifdef HAS_ARGBTOYJROW_NEON +ANY11(ARGBToYJRow_Any_NEON, ARGBToYJRow_NEON, 0, 4, 1, 7) +#endif +#ifdef HAS_BGRATOYROW_NEON +ANY11(BGRAToYRow_Any_NEON, BGRAToYRow_NEON, 0, 4, 1, 7) +#endif +#ifdef HAS_ABGRTOYROW_NEON +ANY11(ABGRToYRow_Any_NEON, ABGRToYRow_NEON, 0, 4, 1, 7) +#endif +#ifdef HAS_RGBATOYROW_NEON +ANY11(RGBAToYRow_Any_NEON, RGBAToYRow_NEON, 0, 4, 1, 7) +#endif +#ifdef HAS_RGB24TOYROW_NEON +ANY11(RGB24ToYRow_Any_NEON, RGB24ToYRow_NEON, 0, 3, 1, 7) +#endif +#ifdef HAS_RAWTOYROW_NEON +ANY11(RAWToYRow_Any_NEON, RAWToYRow_NEON, 0, 3, 1, 7) +#endif +#ifdef HAS_RGB565TOYROW_NEON +ANY11(RGB565ToYRow_Any_NEON, RGB565ToYRow_NEON, 0, 2, 1, 7) +#endif +#ifdef HAS_ARGB1555TOYROW_NEON +ANY11(ARGB1555ToYRow_Any_NEON, ARGB1555ToYRow_NEON, 0, 2, 1, 7) +#endif +#ifdef HAS_ARGB4444TOYROW_NEON +ANY11(ARGB4444ToYRow_Any_NEON, ARGB4444ToYRow_NEON, 0, 2, 1, 7) +#endif +#ifdef HAS_YUY2TOYROW_NEON +ANY11(YUY2ToYRow_Any_NEON, YUY2ToYRow_NEON, 1, 4, 1, 15) +#endif +#ifdef HAS_UYVYTOYROW_NEON +ANY11(UYVYToYRow_Any_NEON, UYVYToYRow_NEON, 0, 2, 1, 15) +#endif +#ifdef HAS_RGB24TOARGBROW_NEON +ANY11(RGB24ToARGBRow_Any_NEON, RGB24ToARGBRow_NEON, 0, 3, 4, 7) +#endif +#ifdef HAS_RAWTOARGBROW_NEON +ANY11(RAWToARGBRow_Any_NEON, RAWToARGBRow_NEON, 0, 3, 4, 7) +#endif +#ifdef HAS_RGB565TOARGBROW_NEON +ANY11(RGB565ToARGBRow_Any_NEON, RGB565ToARGBRow_NEON, 0, 2, 4, 7) +#endif +#ifdef HAS_ARGB1555TOARGBROW_NEON +ANY11(ARGB1555ToARGBRow_Any_NEON, ARGB1555ToARGBRow_NEON, 0, 2, 4, 7) +#endif +#ifdef HAS_ARGB4444TOARGBROW_NEON +ANY11(ARGB4444ToARGBRow_Any_NEON, ARGB4444ToARGBRow_NEON, 0, 2, 4, 7) +#endif +#ifdef HAS_ARGBATTENUATEROW_SSSE3 +ANY11(ARGBAttenuateRow_Any_SSSE3, ARGBAttenuateRow_SSSE3, 0, 4, 4, 3) +#endif +#ifdef HAS_ARGBATTENUATEROW_SSE2 +ANY11(ARGBAttenuateRow_Any_SSE2, ARGBAttenuateRow_SSE2, 0, 4, 4, 3) +#endif +#ifdef HAS_ARGBUNATTENUATEROW_SSE2 +ANY11(ARGBUnattenuateRow_Any_SSE2, ARGBUnattenuateRow_SSE2, 0, 4, 4, 3) +#endif +#ifdef HAS_ARGBATTENUATEROW_AVX2 +ANY11(ARGBAttenuateRow_Any_AVX2, ARGBAttenuateRow_AVX2, 0, 4, 4, 7) +#endif +#ifdef HAS_ARGBUNATTENUATEROW_AVX2 +ANY11(ARGBUnattenuateRow_Any_AVX2, ARGBUnattenuateRow_AVX2, 0, 4, 4, 7) +#endif +#ifdef HAS_ARGBATTENUATEROW_NEON +ANY11(ARGBAttenuateRow_Any_NEON, ARGBAttenuateRow_NEON, 0, 4, 4, 7) +#endif +#undef ANY11 + +// Any 1 to 1 with parameter. +#define ANY11P(NAMEANY, ANY_SIMD, T, SBPP, BPP, MASK) \ + void NAMEANY(const uint8* src_ptr, uint8* dst_ptr, \ + T shuffler, int width) { \ + SIMD_ALIGNED(uint8 temp[64 * 2]); \ + memset(temp, 0, 64); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_ptr, dst_ptr, shuffler, n); \ + } \ + memcpy(temp, src_ptr + n * SBPP, r * SBPP); \ + ANY_SIMD(temp, temp + 64, shuffler, MASK + 1); \ + memcpy(dst_ptr + n * BPP, temp + 64, r * BPP); \ + } + +#if defined(HAS_ARGBTORGB565DITHERROW_SSE2) +ANY11P(ARGBToRGB565DitherRow_Any_SSE2, ARGBToRGB565DitherRow_SSE2, + const uint32, 4, 2, 3) +#endif +#if defined(HAS_ARGBTORGB565DITHERROW_AVX2) +ANY11P(ARGBToRGB565DitherRow_Any_AVX2, ARGBToRGB565DitherRow_AVX2, + const uint32, 4, 2, 7) +#endif +#if defined(HAS_ARGBTORGB565DITHERROW_NEON) +ANY11P(ARGBToRGB565DitherRow_Any_NEON, ARGBToRGB565DitherRow_NEON, + const uint32, 4, 2, 7) +#endif +#ifdef HAS_ARGBSHUFFLEROW_SSE2 +ANY11P(ARGBShuffleRow_Any_SSE2, ARGBShuffleRow_SSE2, const uint8*, 4, 4, 3) +#endif +#ifdef HAS_ARGBSHUFFLEROW_SSSE3 +ANY11P(ARGBShuffleRow_Any_SSSE3, ARGBShuffleRow_SSSE3, const uint8*, 4, 4, 7) +#endif +#ifdef HAS_ARGBSHUFFLEROW_AVX2 +ANY11P(ARGBShuffleRow_Any_AVX2, ARGBShuffleRow_AVX2, const uint8*, 4, 4, 15) +#endif +#ifdef HAS_ARGBSHUFFLEROW_NEON +ANY11P(ARGBShuffleRow_Any_NEON, ARGBShuffleRow_NEON, const uint8*, 4, 4, 3) +#endif +#undef ANY11P + +// Any 1 to 1 interpolate. Takes 2 rows of source via stride. +#define ANY11T(NAMEANY, ANY_SIMD, SBPP, BPP, MASK) \ + void NAMEANY(uint8* dst_ptr, const uint8* src_ptr, \ + ptrdiff_t src_stride_ptr, int width, \ + int source_y_fraction) { \ + SIMD_ALIGNED(uint8 temp[64 * 3]); \ + memset(temp, 0, 64 * 2); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(dst_ptr, src_ptr, src_stride_ptr, n, source_y_fraction); \ + } \ + memcpy(temp, src_ptr + n * SBPP, r * SBPP); \ + memcpy(temp + 64, src_ptr + src_stride_ptr + n * SBPP, r * SBPP); \ + ANY_SIMD(temp + 128, temp, 64, MASK + 1, source_y_fraction); \ + memcpy(dst_ptr + n * BPP, temp + 128, r * BPP); \ + } + +#ifdef HAS_INTERPOLATEROW_AVX2 +ANY11T(InterpolateRow_Any_AVX2, InterpolateRow_AVX2, 1, 1, 31) +#endif +#ifdef HAS_INTERPOLATEROW_SSSE3 +ANY11T(InterpolateRow_Any_SSSE3, InterpolateRow_SSSE3, 1, 1, 15) +#endif +#ifdef HAS_INTERPOLATEROW_SSE2 +ANY11T(InterpolateRow_Any_SSE2, InterpolateRow_SSE2, 1, 1, 15) +#endif +#ifdef HAS_INTERPOLATEROW_NEON +ANY11T(InterpolateRow_Any_NEON, InterpolateRow_NEON, 1, 1, 15) +#endif +#ifdef HAS_INTERPOLATEROW_MIPS_DSPR2 +ANY11T(InterpolateRow_Any_MIPS_DSPR2, InterpolateRow_MIPS_DSPR2, 1, 1, 3) +#endif +#undef ANY11T + +// Any 1 to 1 mirror. +#define ANY11M(NAMEANY, ANY_SIMD, BPP, MASK) \ + void NAMEANY(const uint8* src_ptr, uint8* dst_ptr, int width) { \ + SIMD_ALIGNED(uint8 temp[64 * 2]); \ + memset(temp, 0, 64); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_ptr + r * BPP, dst_ptr, n); \ + } \ + memcpy(temp, src_ptr, r * BPP); \ + ANY_SIMD(temp, temp + 64, MASK + 1); \ + memcpy(dst_ptr + n * BPP, temp + 64 + (MASK + 1 - r) * BPP, r * BPP); \ + } + +#ifdef HAS_MIRRORROW_AVX2 +ANY11M(MirrorRow_Any_AVX2, MirrorRow_AVX2, 1, 31) +#endif +#ifdef HAS_MIRRORROW_SSSE3 +ANY11M(MirrorRow_Any_SSSE3, MirrorRow_SSSE3, 1, 15) +#endif +#ifdef HAS_MIRRORROW_SSE2 +ANY11M(MirrorRow_Any_SSE2, MirrorRow_SSE2, 1, 15) +#endif +#ifdef HAS_MIRRORROW_NEON +ANY11M(MirrorRow_Any_NEON, MirrorRow_NEON, 1, 15) +#endif +#ifdef HAS_ARGBMIRRORROW_AVX2 +ANY11M(ARGBMirrorRow_Any_AVX2, ARGBMirrorRow_AVX2, 4, 7) +#endif +#ifdef HAS_ARGBMIRRORROW_SSE2 +ANY11M(ARGBMirrorRow_Any_SSE2, ARGBMirrorRow_SSE2, 4, 3) +#endif +#ifdef HAS_ARGBMIRRORROW_NEON +ANY11M(ARGBMirrorRow_Any_NEON, ARGBMirrorRow_NEON, 4, 3) +#endif +#undef ANY11M + +// Any 1 plane. (memset) +#define ANY1(NAMEANY, ANY_SIMD, T, BPP, MASK) \ + void NAMEANY(uint8* dst_ptr, T v32, int width) { \ + SIMD_ALIGNED(uint8 temp[64]); \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(dst_ptr, v32, n); \ + } \ + ANY_SIMD(temp, v32, MASK + 1); \ + memcpy(dst_ptr + n * BPP, temp, r * BPP); \ + } + +#ifdef HAS_SETROW_X86 +ANY1(SetRow_Any_X86, SetRow_X86, uint8, 1, 3) +#endif +#ifdef HAS_SETROW_NEON +ANY1(SetRow_Any_NEON, SetRow_NEON, uint8, 1, 15) +#endif +#ifdef HAS_ARGBSETROW_NEON +ANY1(ARGBSetRow_Any_NEON, ARGBSetRow_NEON, uint32, 4, 3) +#endif +#undef ANY1 + +// Any 1 to 2. Outputs UV planes. +#define ANY12(NAMEANY, ANY_SIMD, UVSHIFT, BPP, DUVSHIFT, MASK) \ + void NAMEANY(const uint8* src_ptr, uint8* dst_u, uint8* dst_v, int width) {\ + SIMD_ALIGNED(uint8 temp[128 * 3]); \ + memset(temp, 0, 128); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_ptr, dst_u, dst_v, n); \ + } \ + memcpy(temp, src_ptr + (n >> UVSHIFT) * BPP, SS(r, UVSHIFT) * BPP); \ + if ((width & 1) && BPP == 4) { /* repeat last 4 bytes for subsampler */ \ + memcpy(temp + SS(r, UVSHIFT) * BPP, \ + temp + SS(r, UVSHIFT) * BPP - BPP, 4); \ + } \ + ANY_SIMD(temp, temp + 128, temp + 256, MASK + 1); \ + memcpy(dst_u + (n >> DUVSHIFT), temp + 128, SS(r, DUVSHIFT)); \ + memcpy(dst_v + (n >> DUVSHIFT), temp + 256, SS(r, DUVSHIFT)); \ + } + +#ifdef HAS_SPLITUVROW_SSE2 +ANY12(SplitUVRow_Any_SSE2, SplitUVRow_SSE2, 0, 2, 0, 15) +#endif +#ifdef HAS_SPLITUVROW_AVX2 +ANY12(SplitUVRow_Any_AVX2, SplitUVRow_AVX2, 0, 2, 0, 31) +#endif +#ifdef HAS_SPLITUVROW_NEON +ANY12(SplitUVRow_Any_NEON, SplitUVRow_NEON, 0, 2, 0, 15) +#endif +#ifdef HAS_SPLITUVROW_MIPS_DSPR2 +ANY12(SplitUVRow_Any_MIPS_DSPR2, SplitUVRow_MIPS_DSPR2, 0, 2, 0, 15) +#endif +#ifdef HAS_ARGBTOUV444ROW_SSSE3 +ANY12(ARGBToUV444Row_Any_SSSE3, ARGBToUV444Row_SSSE3, 0, 4, 0, 15) +#endif +#ifdef HAS_YUY2TOUV422ROW_AVX2 +ANY12(YUY2ToUV422Row_Any_AVX2, YUY2ToUV422Row_AVX2, 1, 4, 1, 31) +ANY12(UYVYToUV422Row_Any_AVX2, UYVYToUV422Row_AVX2, 1, 4, 1, 31) +#endif +#ifdef HAS_ARGBTOUV422ROW_SSSE3 +ANY12(ARGBToUV422Row_Any_SSSE3, ARGBToUV422Row_SSSE3, 0, 4, 1, 15) +#endif +#ifdef HAS_YUY2TOUV422ROW_SSE2 +ANY12(YUY2ToUV422Row_Any_SSE2, YUY2ToUV422Row_SSE2, 1, 4, 1, 15) +ANY12(UYVYToUV422Row_Any_SSE2, UYVYToUV422Row_SSE2, 1, 4, 1, 15) +#endif +#ifdef HAS_YUY2TOUV422ROW_NEON +ANY12(ARGBToUV444Row_Any_NEON, ARGBToUV444Row_NEON, 0, 4, 0, 7) +ANY12(ARGBToUV422Row_Any_NEON, ARGBToUV422Row_NEON, 0, 4, 1, 15) +ANY12(ARGBToUV411Row_Any_NEON, ARGBToUV411Row_NEON, 0, 4, 2, 31) +ANY12(YUY2ToUV422Row_Any_NEON, YUY2ToUV422Row_NEON, 1, 4, 1, 15) +ANY12(UYVYToUV422Row_Any_NEON, UYVYToUV422Row_NEON, 1, 4, 1, 15) +#endif +#undef ANY12 + +// Any 1 to 2 with source stride (2 rows of source). Outputs UV planes. +// 128 byte row allows for 32 avx ARGB pixels. +#define ANY12S(NAMEANY, ANY_SIMD, UVSHIFT, BPP, MASK) \ + void NAMEANY(const uint8* src_ptr, int src_stride_ptr, \ + uint8* dst_u, uint8* dst_v, int width) { \ + SIMD_ALIGNED(uint8 temp[128 * 4]); \ + memset(temp, 0, 128 * 2); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_ptr, src_stride_ptr, dst_u, dst_v, n); \ + } \ + memcpy(temp, src_ptr + (n >> UVSHIFT) * BPP, SS(r, UVSHIFT) * BPP); \ + memcpy(temp + 128, src_ptr + src_stride_ptr + (n >> UVSHIFT) * BPP, \ + SS(r, UVSHIFT) * BPP); \ + if ((width & 1) && BPP == 4) { /* repeat last 4 bytes for subsampler */ \ + memcpy(temp + SS(r, UVSHIFT) * BPP, \ + temp + SS(r, UVSHIFT) * BPP - BPP, 4); \ + memcpy(temp + 128 + SS(r, UVSHIFT) * BPP, \ + temp + 128 + SS(r, UVSHIFT) * BPP - BPP, 4); \ + } \ + ANY_SIMD(temp, 128, temp + 256, temp + 384, MASK + 1); \ + memcpy(dst_u + (n >> 1), temp + 256, SS(r, 1)); \ + memcpy(dst_v + (n >> 1), temp + 384, SS(r, 1)); \ + } + +#ifdef HAS_ARGBTOUVROW_AVX2 +ANY12S(ARGBToUVRow_Any_AVX2, ARGBToUVRow_AVX2, 0, 4, 31) +#endif +#ifdef HAS_ARGBTOUVROW_SSSE3 +ANY12S(ARGBToUVRow_Any_SSSE3, ARGBToUVRow_SSSE3, 0, 4, 15) +ANY12S(ARGBToUVJRow_Any_SSSE3, ARGBToUVJRow_SSSE3, 0, 4, 15) +ANY12S(BGRAToUVRow_Any_SSSE3, BGRAToUVRow_SSSE3, 0, 4, 15) +ANY12S(ABGRToUVRow_Any_SSSE3, ABGRToUVRow_SSSE3, 0, 4, 15) +ANY12S(RGBAToUVRow_Any_SSSE3, RGBAToUVRow_SSSE3, 0, 4, 15) +#endif +#ifdef HAS_YUY2TOUVROW_AVX2 +ANY12S(YUY2ToUVRow_Any_AVX2, YUY2ToUVRow_AVX2, 1, 4, 31) +ANY12S(UYVYToUVRow_Any_AVX2, UYVYToUVRow_AVX2, 1, 4, 31) +#endif +#ifdef HAS_YUY2TOUVROW_SSE2 +ANY12S(YUY2ToUVRow_Any_SSE2, YUY2ToUVRow_SSE2, 1, 4, 15) +ANY12S(UYVYToUVRow_Any_SSE2, UYVYToUVRow_SSE2, 1, 4, 15) +#endif +#ifdef HAS_ARGBTOUVROW_NEON +ANY12S(ARGBToUVRow_Any_NEON, ARGBToUVRow_NEON, 0, 4, 15) +#endif +#ifdef HAS_ARGBTOUVJROW_NEON +ANY12S(ARGBToUVJRow_Any_NEON, ARGBToUVJRow_NEON, 0, 4, 15) +#endif +#ifdef HAS_BGRATOUVROW_NEON +ANY12S(BGRAToUVRow_Any_NEON, BGRAToUVRow_NEON, 0, 4, 15) +#endif +#ifdef HAS_ABGRTOUVROW_NEON +ANY12S(ABGRToUVRow_Any_NEON, ABGRToUVRow_NEON, 0, 4, 15) +#endif +#ifdef HAS_RGBATOUVROW_NEON +ANY12S(RGBAToUVRow_Any_NEON, RGBAToUVRow_NEON, 0, 4, 15) +#endif +#ifdef HAS_RGB24TOUVROW_NEON +ANY12S(RGB24ToUVRow_Any_NEON, RGB24ToUVRow_NEON, 0, 3, 15) +#endif +#ifdef HAS_RAWTOUVROW_NEON +ANY12S(RAWToUVRow_Any_NEON, RAWToUVRow_NEON, 0, 3, 15) +#endif +#ifdef HAS_RGB565TOUVROW_NEON +ANY12S(RGB565ToUVRow_Any_NEON, RGB565ToUVRow_NEON, 0, 2, 15) +#endif +#ifdef HAS_ARGB1555TOUVROW_NEON +ANY12S(ARGB1555ToUVRow_Any_NEON, ARGB1555ToUVRow_NEON, 0, 2, 15) +#endif +#ifdef HAS_ARGB4444TOUVROW_NEON +ANY12S(ARGB4444ToUVRow_Any_NEON, ARGB4444ToUVRow_NEON, 0, 2, 15) +#endif +#ifdef HAS_YUY2TOUVROW_NEON +ANY12S(YUY2ToUVRow_Any_NEON, YUY2ToUVRow_NEON, 1, 4, 15) +#endif +#ifdef HAS_UYVYTOUVROW_NEON +ANY12S(UYVYToUVRow_Any_NEON, UYVYToUVRow_NEON, 1, 4, 15) +#endif +#undef ANY12S + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/row_common.cc b/third_party/aom/third_party/libyuv/source/row_common.cc new file mode 100644 index 0000000000..49875894fe --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/row_common.cc @@ -0,0 +1,2576 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" + +#include // For memcpy and memset. + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// llvm x86 is poor at ternary operator, so use branchless min/max. + +#define USE_BRANCHLESS 1 +#if USE_BRANCHLESS +static __inline int32 clamp0(int32 v) { + return ((-(v) >> 31) & (v)); +} + +static __inline int32 clamp255(int32 v) { + return (((255 - (v)) >> 31) | (v)) & 255; +} + +static __inline uint32 Clamp(int32 val) { + int v = clamp0(val); + return (uint32)(clamp255(v)); +} + +static __inline uint32 Abs(int32 v) { + int m = v >> 31; + return (v + m) ^ m; +} +#else // USE_BRANCHLESS +static __inline int32 clamp0(int32 v) { + return (v < 0) ? 0 : v; +} + +static __inline int32 clamp255(int32 v) { + return (v > 255) ? 255 : v; +} + +static __inline uint32 Clamp(int32 val) { + int v = clamp0(val); + return (uint32)(clamp255(v)); +} + +static __inline uint32 Abs(int32 v) { + return (v < 0) ? -v : v; +} +#endif // USE_BRANCHLESS + +#ifdef LIBYUV_LITTLE_ENDIAN +#define WRITEWORD(p, v) *(uint32*)(p) = v +#else +static inline void WRITEWORD(uint8* p, uint32 v) { + p[0] = (uint8)(v & 255); + p[1] = (uint8)((v >> 8) & 255); + p[2] = (uint8)((v >> 16) & 255); + p[3] = (uint8)((v >> 24) & 255); +} +#endif + +void RGB24ToARGBRow_C(const uint8* src_rgb24, uint8* dst_argb, int width) { + int x; + for (x = 0; x < width; ++x) { + uint8 b = src_rgb24[0]; + uint8 g = src_rgb24[1]; + uint8 r = src_rgb24[2]; + dst_argb[0] = b; + dst_argb[1] = g; + dst_argb[2] = r; + dst_argb[3] = 255u; + dst_argb += 4; + src_rgb24 += 3; + } +} + +void RAWToARGBRow_C(const uint8* src_raw, uint8* dst_argb, int width) { + int x; + for (x = 0; x < width; ++x) { + uint8 r = src_raw[0]; + uint8 g = src_raw[1]; + uint8 b = src_raw[2]; + dst_argb[0] = b; + dst_argb[1] = g; + dst_argb[2] = r; + dst_argb[3] = 255u; + dst_argb += 4; + src_raw += 3; + } +} + +void RGB565ToARGBRow_C(const uint8* src_rgb565, uint8* dst_argb, int width) { + int x; + for (x = 0; x < width; ++x) { + uint8 b = src_rgb565[0] & 0x1f; + uint8 g = (src_rgb565[0] >> 5) | ((src_rgb565[1] & 0x07) << 3); + uint8 r = src_rgb565[1] >> 3; + dst_argb[0] = (b << 3) | (b >> 2); + dst_argb[1] = (g << 2) | (g >> 4); + dst_argb[2] = (r << 3) | (r >> 2); + dst_argb[3] = 255u; + dst_argb += 4; + src_rgb565 += 2; + } +} + +void ARGB1555ToARGBRow_C(const uint8* src_argb1555, uint8* dst_argb, + int width) { + int x; + for (x = 0; x < width; ++x) { + uint8 b = src_argb1555[0] & 0x1f; + uint8 g = (src_argb1555[0] >> 5) | ((src_argb1555[1] & 0x03) << 3); + uint8 r = (src_argb1555[1] & 0x7c) >> 2; + uint8 a = src_argb1555[1] >> 7; + dst_argb[0] = (b << 3) | (b >> 2); + dst_argb[1] = (g << 3) | (g >> 2); + dst_argb[2] = (r << 3) | (r >> 2); + dst_argb[3] = -a; + dst_argb += 4; + src_argb1555 += 2; + } +} + +void ARGB4444ToARGBRow_C(const uint8* src_argb4444, uint8* dst_argb, + int width) { + int x; + for (x = 0; x < width; ++x) { + uint8 b = src_argb4444[0] & 0x0f; + uint8 g = src_argb4444[0] >> 4; + uint8 r = src_argb4444[1] & 0x0f; + uint8 a = src_argb4444[1] >> 4; + dst_argb[0] = (b << 4) | b; + dst_argb[1] = (g << 4) | g; + dst_argb[2] = (r << 4) | r; + dst_argb[3] = (a << 4) | a; + dst_argb += 4; + src_argb4444 += 2; + } +} + +void ARGBToRGB24Row_C(const uint8* src_argb, uint8* dst_rgb, int width) { + int x; + for (x = 0; x < width; ++x) { + uint8 b = src_argb[0]; + uint8 g = src_argb[1]; + uint8 r = src_argb[2]; + dst_rgb[0] = b; + dst_rgb[1] = g; + dst_rgb[2] = r; + dst_rgb += 3; + src_argb += 4; + } +} + +void ARGBToRAWRow_C(const uint8* src_argb, uint8* dst_rgb, int width) { + int x; + for (x = 0; x < width; ++x) { + uint8 b = src_argb[0]; + uint8 g = src_argb[1]; + uint8 r = src_argb[2]; + dst_rgb[0] = r; + dst_rgb[1] = g; + dst_rgb[2] = b; + dst_rgb += 3; + src_argb += 4; + } +} + +void ARGBToRGB565Row_C(const uint8* src_argb, uint8* dst_rgb, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + uint8 b0 = src_argb[0] >> 3; + uint8 g0 = src_argb[1] >> 2; + uint8 r0 = src_argb[2] >> 3; + uint8 b1 = src_argb[4] >> 3; + uint8 g1 = src_argb[5] >> 2; + uint8 r1 = src_argb[6] >> 3; + WRITEWORD(dst_rgb, b0 | (g0 << 5) | (r0 << 11) | + (b1 << 16) | (g1 << 21) | (r1 << 27)); + dst_rgb += 4; + src_argb += 8; + } + if (width & 1) { + uint8 b0 = src_argb[0] >> 3; + uint8 g0 = src_argb[1] >> 2; + uint8 r0 = src_argb[2] >> 3; + *(uint16*)(dst_rgb) = b0 | (g0 << 5) | (r0 << 11); + } +} + +// dither4 is a row of 4 values from 4x4 dither matrix. +// The 4x4 matrix contains values to increase RGB. When converting to +// fewer bits (565) this provides an ordered dither. +// The order in the 4x4 matrix in first byte is upper left. +// The 4 values are passed as an int, then referenced as an array, so +// endian will not affect order of the original matrix. But the dither4 +// will containing the first pixel in the lower byte for little endian +// or the upper byte for big endian. +void ARGBToRGB565DitherRow_C(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + int dither0 = ((const unsigned char*)(&dither4))[x & 3]; + int dither1 = ((const unsigned char*)(&dither4))[(x + 1) & 3]; + uint8 b0 = clamp255(src_argb[0] + dither0) >> 3; + uint8 g0 = clamp255(src_argb[1] + dither0) >> 2; + uint8 r0 = clamp255(src_argb[2] + dither0) >> 3; + uint8 b1 = clamp255(src_argb[4] + dither1) >> 3; + uint8 g1 = clamp255(src_argb[5] + dither1) >> 2; + uint8 r1 = clamp255(src_argb[6] + dither1) >> 3; + WRITEWORD(dst_rgb, b0 | (g0 << 5) | (r0 << 11) | + (b1 << 16) | (g1 << 21) | (r1 << 27)); + dst_rgb += 4; + src_argb += 8; + } + if (width & 1) { + int dither0 = ((const unsigned char*)(&dither4))[(width - 1) & 3]; + uint8 b0 = clamp255(src_argb[0] + dither0) >> 3; + uint8 g0 = clamp255(src_argb[1] + dither0) >> 2; + uint8 r0 = clamp255(src_argb[2] + dither0) >> 3; + *(uint16*)(dst_rgb) = b0 | (g0 << 5) | (r0 << 11); + } +} + +void ARGBToARGB1555Row_C(const uint8* src_argb, uint8* dst_rgb, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + uint8 b0 = src_argb[0] >> 3; + uint8 g0 = src_argb[1] >> 3; + uint8 r0 = src_argb[2] >> 3; + uint8 a0 = src_argb[3] >> 7; + uint8 b1 = src_argb[4] >> 3; + uint8 g1 = src_argb[5] >> 3; + uint8 r1 = src_argb[6] >> 3; + uint8 a1 = src_argb[7] >> 7; + *(uint32*)(dst_rgb) = + b0 | (g0 << 5) | (r0 << 10) | (a0 << 15) | + (b1 << 16) | (g1 << 21) | (r1 << 26) | (a1 << 31); + dst_rgb += 4; + src_argb += 8; + } + if (width & 1) { + uint8 b0 = src_argb[0] >> 3; + uint8 g0 = src_argb[1] >> 3; + uint8 r0 = src_argb[2] >> 3; + uint8 a0 = src_argb[3] >> 7; + *(uint16*)(dst_rgb) = + b0 | (g0 << 5) | (r0 << 10) | (a0 << 15); + } +} + +void ARGBToARGB4444Row_C(const uint8* src_argb, uint8* dst_rgb, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + uint8 b0 = src_argb[0] >> 4; + uint8 g0 = src_argb[1] >> 4; + uint8 r0 = src_argb[2] >> 4; + uint8 a0 = src_argb[3] >> 4; + uint8 b1 = src_argb[4] >> 4; + uint8 g1 = src_argb[5] >> 4; + uint8 r1 = src_argb[6] >> 4; + uint8 a1 = src_argb[7] >> 4; + *(uint32*)(dst_rgb) = + b0 | (g0 << 4) | (r0 << 8) | (a0 << 12) | + (b1 << 16) | (g1 << 20) | (r1 << 24) | (a1 << 28); + dst_rgb += 4; + src_argb += 8; + } + if (width & 1) { + uint8 b0 = src_argb[0] >> 4; + uint8 g0 = src_argb[1] >> 4; + uint8 r0 = src_argb[2] >> 4; + uint8 a0 = src_argb[3] >> 4; + *(uint16*)(dst_rgb) = + b0 | (g0 << 4) | (r0 << 8) | (a0 << 12); + } +} + +static __inline int RGBToY(uint8 r, uint8 g, uint8 b) { + return (66 * r + 129 * g + 25 * b + 0x1080) >> 8; +} + +static __inline int RGBToU(uint8 r, uint8 g, uint8 b) { + return (112 * b - 74 * g - 38 * r + 0x8080) >> 8; +} +static __inline int RGBToV(uint8 r, uint8 g, uint8 b) { + return (112 * r - 94 * g - 18 * b + 0x8080) >> 8; +} + +#define MAKEROWY(NAME, R, G, B, BPP) \ +void NAME ## ToYRow_C(const uint8* src_argb0, uint8* dst_y, int width) { \ + int x; \ + for (x = 0; x < width; ++x) { \ + dst_y[0] = RGBToY(src_argb0[R], src_argb0[G], src_argb0[B]); \ + src_argb0 += BPP; \ + dst_y += 1; \ + } \ +} \ +void NAME ## ToUVRow_C(const uint8* src_rgb0, int src_stride_rgb, \ + uint8* dst_u, uint8* dst_v, int width) { \ + const uint8* src_rgb1 = src_rgb0 + src_stride_rgb; \ + int x; \ + for (x = 0; x < width - 1; x += 2) { \ + uint8 ab = (src_rgb0[B] + src_rgb0[B + BPP] + \ + src_rgb1[B] + src_rgb1[B + BPP]) >> 2; \ + uint8 ag = (src_rgb0[G] + src_rgb0[G + BPP] + \ + src_rgb1[G] + src_rgb1[G + BPP]) >> 2; \ + uint8 ar = (src_rgb0[R] + src_rgb0[R + BPP] + \ + src_rgb1[R] + src_rgb1[R + BPP]) >> 2; \ + dst_u[0] = RGBToU(ar, ag, ab); \ + dst_v[0] = RGBToV(ar, ag, ab); \ + src_rgb0 += BPP * 2; \ + src_rgb1 += BPP * 2; \ + dst_u += 1; \ + dst_v += 1; \ + } \ + if (width & 1) { \ + uint8 ab = (src_rgb0[B] + src_rgb1[B]) >> 1; \ + uint8 ag = (src_rgb0[G] + src_rgb1[G]) >> 1; \ + uint8 ar = (src_rgb0[R] + src_rgb1[R]) >> 1; \ + dst_u[0] = RGBToU(ar, ag, ab); \ + dst_v[0] = RGBToV(ar, ag, ab); \ + } \ +} + +MAKEROWY(ARGB, 2, 1, 0, 4) +MAKEROWY(BGRA, 1, 2, 3, 4) +MAKEROWY(ABGR, 0, 1, 2, 4) +MAKEROWY(RGBA, 3, 2, 1, 4) +MAKEROWY(RGB24, 2, 1, 0, 3) +MAKEROWY(RAW, 0, 1, 2, 3) +#undef MAKEROWY + +// JPeg uses a variation on BT.601-1 full range +// y = 0.29900 * r + 0.58700 * g + 0.11400 * b +// u = -0.16874 * r - 0.33126 * g + 0.50000 * b + center +// v = 0.50000 * r - 0.41869 * g - 0.08131 * b + center +// BT.601 Mpeg range uses: +// b 0.1016 * 255 = 25.908 = 25 +// g 0.5078 * 255 = 129.489 = 129 +// r 0.2578 * 255 = 65.739 = 66 +// JPeg 8 bit Y (not used): +// b 0.11400 * 256 = 29.184 = 29 +// g 0.58700 * 256 = 150.272 = 150 +// r 0.29900 * 256 = 76.544 = 77 +// JPeg 7 bit Y: +// b 0.11400 * 128 = 14.592 = 15 +// g 0.58700 * 128 = 75.136 = 75 +// r 0.29900 * 128 = 38.272 = 38 +// JPeg 8 bit U: +// b 0.50000 * 255 = 127.5 = 127 +// g -0.33126 * 255 = -84.4713 = -84 +// r -0.16874 * 255 = -43.0287 = -43 +// JPeg 8 bit V: +// b -0.08131 * 255 = -20.73405 = -20 +// g -0.41869 * 255 = -106.76595 = -107 +// r 0.50000 * 255 = 127.5 = 127 + +static __inline int RGBToYJ(uint8 r, uint8 g, uint8 b) { + return (38 * r + 75 * g + 15 * b + 64) >> 7; +} + +static __inline int RGBToUJ(uint8 r, uint8 g, uint8 b) { + return (127 * b - 84 * g - 43 * r + 0x8080) >> 8; +} +static __inline int RGBToVJ(uint8 r, uint8 g, uint8 b) { + return (127 * r - 107 * g - 20 * b + 0x8080) >> 8; +} + +#define AVGB(a, b) (((a) + (b) + 1) >> 1) + +#define MAKEROWYJ(NAME, R, G, B, BPP) \ +void NAME ## ToYJRow_C(const uint8* src_argb0, uint8* dst_y, int width) { \ + int x; \ + for (x = 0; x < width; ++x) { \ + dst_y[0] = RGBToYJ(src_argb0[R], src_argb0[G], src_argb0[B]); \ + src_argb0 += BPP; \ + dst_y += 1; \ + } \ +} \ +void NAME ## ToUVJRow_C(const uint8* src_rgb0, int src_stride_rgb, \ + uint8* dst_u, uint8* dst_v, int width) { \ + const uint8* src_rgb1 = src_rgb0 + src_stride_rgb; \ + int x; \ + for (x = 0; x < width - 1; x += 2) { \ + uint8 ab = AVGB(AVGB(src_rgb0[B], src_rgb1[B]), \ + AVGB(src_rgb0[B + BPP], src_rgb1[B + BPP])); \ + uint8 ag = AVGB(AVGB(src_rgb0[G], src_rgb1[G]), \ + AVGB(src_rgb0[G + BPP], src_rgb1[G + BPP])); \ + uint8 ar = AVGB(AVGB(src_rgb0[R], src_rgb1[R]), \ + AVGB(src_rgb0[R + BPP], src_rgb1[R + BPP])); \ + dst_u[0] = RGBToUJ(ar, ag, ab); \ + dst_v[0] = RGBToVJ(ar, ag, ab); \ + src_rgb0 += BPP * 2; \ + src_rgb1 += BPP * 2; \ + dst_u += 1; \ + dst_v += 1; \ + } \ + if (width & 1) { \ + uint8 ab = AVGB(src_rgb0[B], src_rgb1[B]); \ + uint8 ag = AVGB(src_rgb0[G], src_rgb1[G]); \ + uint8 ar = AVGB(src_rgb0[R], src_rgb1[R]); \ + dst_u[0] = RGBToUJ(ar, ag, ab); \ + dst_v[0] = RGBToVJ(ar, ag, ab); \ + } \ +} + +MAKEROWYJ(ARGB, 2, 1, 0, 4) +#undef MAKEROWYJ + +void ARGBToUVJ422Row_C(const uint8* src_argb, + uint8* dst_u, uint8* dst_v, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + uint8 ab = (src_argb[0] + src_argb[4]) >> 1; + uint8 ag = (src_argb[1] + src_argb[5]) >> 1; + uint8 ar = (src_argb[2] + src_argb[6]) >> 1; + dst_u[0] = RGBToUJ(ar, ag, ab); + dst_v[0] = RGBToVJ(ar, ag, ab); + src_argb += 8; + dst_u += 1; + dst_v += 1; + } + if (width & 1) { + uint8 ab = src_argb[0]; + uint8 ag = src_argb[1]; + uint8 ar = src_argb[2]; + dst_u[0] = RGBToUJ(ar, ag, ab); + dst_v[0] = RGBToVJ(ar, ag, ab); + } +} + +void RGB565ToYRow_C(const uint8* src_rgb565, uint8* dst_y, int width) { + int x; + for (x = 0; x < width; ++x) { + uint8 b = src_rgb565[0] & 0x1f; + uint8 g = (src_rgb565[0] >> 5) | ((src_rgb565[1] & 0x07) << 3); + uint8 r = src_rgb565[1] >> 3; + b = (b << 3) | (b >> 2); + g = (g << 2) | (g >> 4); + r = (r << 3) | (r >> 2); + dst_y[0] = RGBToY(r, g, b); + src_rgb565 += 2; + dst_y += 1; + } +} + +void ARGB1555ToYRow_C(const uint8* src_argb1555, uint8* dst_y, int width) { + int x; + for (x = 0; x < width; ++x) { + uint8 b = src_argb1555[0] & 0x1f; + uint8 g = (src_argb1555[0] >> 5) | ((src_argb1555[1] & 0x03) << 3); + uint8 r = (src_argb1555[1] & 0x7c) >> 2; + b = (b << 3) | (b >> 2); + g = (g << 3) | (g >> 2); + r = (r << 3) | (r >> 2); + dst_y[0] = RGBToY(r, g, b); + src_argb1555 += 2; + dst_y += 1; + } +} + +void ARGB4444ToYRow_C(const uint8* src_argb4444, uint8* dst_y, int width) { + int x; + for (x = 0; x < width; ++x) { + uint8 b = src_argb4444[0] & 0x0f; + uint8 g = src_argb4444[0] >> 4; + uint8 r = src_argb4444[1] & 0x0f; + b = (b << 4) | b; + g = (g << 4) | g; + r = (r << 4) | r; + dst_y[0] = RGBToY(r, g, b); + src_argb4444 += 2; + dst_y += 1; + } +} + +void RGB565ToUVRow_C(const uint8* src_rgb565, int src_stride_rgb565, + uint8* dst_u, uint8* dst_v, int width) { + const uint8* next_rgb565 = src_rgb565 + src_stride_rgb565; + int x; + for (x = 0; x < width - 1; x += 2) { + uint8 b0 = src_rgb565[0] & 0x1f; + uint8 g0 = (src_rgb565[0] >> 5) | ((src_rgb565[1] & 0x07) << 3); + uint8 r0 = src_rgb565[1] >> 3; + uint8 b1 = src_rgb565[2] & 0x1f; + uint8 g1 = (src_rgb565[2] >> 5) | ((src_rgb565[3] & 0x07) << 3); + uint8 r1 = src_rgb565[3] >> 3; + uint8 b2 = next_rgb565[0] & 0x1f; + uint8 g2 = (next_rgb565[0] >> 5) | ((next_rgb565[1] & 0x07) << 3); + uint8 r2 = next_rgb565[1] >> 3; + uint8 b3 = next_rgb565[2] & 0x1f; + uint8 g3 = (next_rgb565[2] >> 5) | ((next_rgb565[3] & 0x07) << 3); + uint8 r3 = next_rgb565[3] >> 3; + uint8 b = (b0 + b1 + b2 + b3); // 565 * 4 = 787. + uint8 g = (g0 + g1 + g2 + g3); + uint8 r = (r0 + r1 + r2 + r3); + b = (b << 1) | (b >> 6); // 787 -> 888. + r = (r << 1) | (r >> 6); + dst_u[0] = RGBToU(r, g, b); + dst_v[0] = RGBToV(r, g, b); + src_rgb565 += 4; + next_rgb565 += 4; + dst_u += 1; + dst_v += 1; + } + if (width & 1) { + uint8 b0 = src_rgb565[0] & 0x1f; + uint8 g0 = (src_rgb565[0] >> 5) | ((src_rgb565[1] & 0x07) << 3); + uint8 r0 = src_rgb565[1] >> 3; + uint8 b2 = next_rgb565[0] & 0x1f; + uint8 g2 = (next_rgb565[0] >> 5) | ((next_rgb565[1] & 0x07) << 3); + uint8 r2 = next_rgb565[1] >> 3; + uint8 b = (b0 + b2); // 565 * 2 = 676. + uint8 g = (g0 + g2); + uint8 r = (r0 + r2); + b = (b << 2) | (b >> 4); // 676 -> 888 + g = (g << 1) | (g >> 6); + r = (r << 2) | (r >> 4); + dst_u[0] = RGBToU(r, g, b); + dst_v[0] = RGBToV(r, g, b); + } +} + +void ARGB1555ToUVRow_C(const uint8* src_argb1555, int src_stride_argb1555, + uint8* dst_u, uint8* dst_v, int width) { + const uint8* next_argb1555 = src_argb1555 + src_stride_argb1555; + int x; + for (x = 0; x < width - 1; x += 2) { + uint8 b0 = src_argb1555[0] & 0x1f; + uint8 g0 = (src_argb1555[0] >> 5) | ((src_argb1555[1] & 0x03) << 3); + uint8 r0 = (src_argb1555[1] & 0x7c) >> 2; + uint8 b1 = src_argb1555[2] & 0x1f; + uint8 g1 = (src_argb1555[2] >> 5) | ((src_argb1555[3] & 0x03) << 3); + uint8 r1 = (src_argb1555[3] & 0x7c) >> 2; + uint8 b2 = next_argb1555[0] & 0x1f; + uint8 g2 = (next_argb1555[0] >> 5) | ((next_argb1555[1] & 0x03) << 3); + uint8 r2 = (next_argb1555[1] & 0x7c) >> 2; + uint8 b3 = next_argb1555[2] & 0x1f; + uint8 g3 = (next_argb1555[2] >> 5) | ((next_argb1555[3] & 0x03) << 3); + uint8 r3 = (next_argb1555[3] & 0x7c) >> 2; + uint8 b = (b0 + b1 + b2 + b3); // 555 * 4 = 777. + uint8 g = (g0 + g1 + g2 + g3); + uint8 r = (r0 + r1 + r2 + r3); + b = (b << 1) | (b >> 6); // 777 -> 888. + g = (g << 1) | (g >> 6); + r = (r << 1) | (r >> 6); + dst_u[0] = RGBToU(r, g, b); + dst_v[0] = RGBToV(r, g, b); + src_argb1555 += 4; + next_argb1555 += 4; + dst_u += 1; + dst_v += 1; + } + if (width & 1) { + uint8 b0 = src_argb1555[0] & 0x1f; + uint8 g0 = (src_argb1555[0] >> 5) | ((src_argb1555[1] & 0x03) << 3); + uint8 r0 = (src_argb1555[1] & 0x7c) >> 2; + uint8 b2 = next_argb1555[0] & 0x1f; + uint8 g2 = (next_argb1555[0] >> 5) | ((next_argb1555[1] & 0x03) << 3); + uint8 r2 = next_argb1555[1] >> 3; + uint8 b = (b0 + b2); // 555 * 2 = 666. + uint8 g = (g0 + g2); + uint8 r = (r0 + r2); + b = (b << 2) | (b >> 4); // 666 -> 888. + g = (g << 2) | (g >> 4); + r = (r << 2) | (r >> 4); + dst_u[0] = RGBToU(r, g, b); + dst_v[0] = RGBToV(r, g, b); + } +} + +void ARGB4444ToUVRow_C(const uint8* src_argb4444, int src_stride_argb4444, + uint8* dst_u, uint8* dst_v, int width) { + const uint8* next_argb4444 = src_argb4444 + src_stride_argb4444; + int x; + for (x = 0; x < width - 1; x += 2) { + uint8 b0 = src_argb4444[0] & 0x0f; + uint8 g0 = src_argb4444[0] >> 4; + uint8 r0 = src_argb4444[1] & 0x0f; + uint8 b1 = src_argb4444[2] & 0x0f; + uint8 g1 = src_argb4444[2] >> 4; + uint8 r1 = src_argb4444[3] & 0x0f; + uint8 b2 = next_argb4444[0] & 0x0f; + uint8 g2 = next_argb4444[0] >> 4; + uint8 r2 = next_argb4444[1] & 0x0f; + uint8 b3 = next_argb4444[2] & 0x0f; + uint8 g3 = next_argb4444[2] >> 4; + uint8 r3 = next_argb4444[3] & 0x0f; + uint8 b = (b0 + b1 + b2 + b3); // 444 * 4 = 666. + uint8 g = (g0 + g1 + g2 + g3); + uint8 r = (r0 + r1 + r2 + r3); + b = (b << 2) | (b >> 4); // 666 -> 888. + g = (g << 2) | (g >> 4); + r = (r << 2) | (r >> 4); + dst_u[0] = RGBToU(r, g, b); + dst_v[0] = RGBToV(r, g, b); + src_argb4444 += 4; + next_argb4444 += 4; + dst_u += 1; + dst_v += 1; + } + if (width & 1) { + uint8 b0 = src_argb4444[0] & 0x0f; + uint8 g0 = src_argb4444[0] >> 4; + uint8 r0 = src_argb4444[1] & 0x0f; + uint8 b2 = next_argb4444[0] & 0x0f; + uint8 g2 = next_argb4444[0] >> 4; + uint8 r2 = next_argb4444[1] & 0x0f; + uint8 b = (b0 + b2); // 444 * 2 = 555. + uint8 g = (g0 + g2); + uint8 r = (r0 + r2); + b = (b << 3) | (b >> 2); // 555 -> 888. + g = (g << 3) | (g >> 2); + r = (r << 3) | (r >> 2); + dst_u[0] = RGBToU(r, g, b); + dst_v[0] = RGBToV(r, g, b); + } +} + +void ARGBToUV444Row_C(const uint8* src_argb, + uint8* dst_u, uint8* dst_v, int width) { + int x; + for (x = 0; x < width; ++x) { + uint8 ab = src_argb[0]; + uint8 ag = src_argb[1]; + uint8 ar = src_argb[2]; + dst_u[0] = RGBToU(ar, ag, ab); + dst_v[0] = RGBToV(ar, ag, ab); + src_argb += 4; + dst_u += 1; + dst_v += 1; + } +} + +void ARGBToUV422Row_C(const uint8* src_argb, + uint8* dst_u, uint8* dst_v, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + uint8 ab = (src_argb[0] + src_argb[4]) >> 1; + uint8 ag = (src_argb[1] + src_argb[5]) >> 1; + uint8 ar = (src_argb[2] + src_argb[6]) >> 1; + dst_u[0] = RGBToU(ar, ag, ab); + dst_v[0] = RGBToV(ar, ag, ab); + src_argb += 8; + dst_u += 1; + dst_v += 1; + } + if (width & 1) { + uint8 ab = src_argb[0]; + uint8 ag = src_argb[1]; + uint8 ar = src_argb[2]; + dst_u[0] = RGBToU(ar, ag, ab); + dst_v[0] = RGBToV(ar, ag, ab); + } +} + +void ARGBToUV411Row_C(const uint8* src_argb, + uint8* dst_u, uint8* dst_v, int width) { + int x; + for (x = 0; x < width - 3; x += 4) { + uint8 ab = (src_argb[0] + src_argb[4] + src_argb[8] + src_argb[12]) >> 2; + uint8 ag = (src_argb[1] + src_argb[5] + src_argb[9] + src_argb[13]) >> 2; + uint8 ar = (src_argb[2] + src_argb[6] + src_argb[10] + src_argb[14]) >> 2; + dst_u[0] = RGBToU(ar, ag, ab); + dst_v[0] = RGBToV(ar, ag, ab); + src_argb += 16; + dst_u += 1; + dst_v += 1; + } + if ((width & 3) == 3) { + uint8 ab = (src_argb[0] + src_argb[4] + src_argb[8]) / 3; + uint8 ag = (src_argb[1] + src_argb[5] + src_argb[9]) / 3; + uint8 ar = (src_argb[2] + src_argb[6] + src_argb[10]) / 3; + dst_u[0] = RGBToU(ar, ag, ab); + dst_v[0] = RGBToV(ar, ag, ab); + } else if ((width & 3) == 2) { + uint8 ab = (src_argb[0] + src_argb[4]) >> 1; + uint8 ag = (src_argb[1] + src_argb[5]) >> 1; + uint8 ar = (src_argb[2] + src_argb[6]) >> 1; + dst_u[0] = RGBToU(ar, ag, ab); + dst_v[0] = RGBToV(ar, ag, ab); + } else if ((width & 3) == 1) { + uint8 ab = src_argb[0]; + uint8 ag = src_argb[1]; + uint8 ar = src_argb[2]; + dst_u[0] = RGBToU(ar, ag, ab); + dst_v[0] = RGBToV(ar, ag, ab); + } +} + +void ARGBGrayRow_C(const uint8* src_argb, uint8* dst_argb, int width) { + int x; + for (x = 0; x < width; ++x) { + uint8 y = RGBToYJ(src_argb[2], src_argb[1], src_argb[0]); + dst_argb[2] = dst_argb[1] = dst_argb[0] = y; + dst_argb[3] = src_argb[3]; + dst_argb += 4; + src_argb += 4; + } +} + +// Convert a row of image to Sepia tone. +void ARGBSepiaRow_C(uint8* dst_argb, int width) { + int x; + for (x = 0; x < width; ++x) { + int b = dst_argb[0]; + int g = dst_argb[1]; + int r = dst_argb[2]; + int sb = (b * 17 + g * 68 + r * 35) >> 7; + int sg = (b * 22 + g * 88 + r * 45) >> 7; + int sr = (b * 24 + g * 98 + r * 50) >> 7; + // b does not over flow. a is preserved from original. + dst_argb[0] = sb; + dst_argb[1] = clamp255(sg); + dst_argb[2] = clamp255(sr); + dst_argb += 4; + } +} + +// Apply color matrix to a row of image. Matrix is signed. +// TODO(fbarchard): Consider adding rounding (+32). +void ARGBColorMatrixRow_C(const uint8* src_argb, uint8* dst_argb, + const int8* matrix_argb, int width) { + int x; + for (x = 0; x < width; ++x) { + int b = src_argb[0]; + int g = src_argb[1]; + int r = src_argb[2]; + int a = src_argb[3]; + int sb = (b * matrix_argb[0] + g * matrix_argb[1] + + r * matrix_argb[2] + a * matrix_argb[3]) >> 6; + int sg = (b * matrix_argb[4] + g * matrix_argb[5] + + r * matrix_argb[6] + a * matrix_argb[7]) >> 6; + int sr = (b * matrix_argb[8] + g * matrix_argb[9] + + r * matrix_argb[10] + a * matrix_argb[11]) >> 6; + int sa = (b * matrix_argb[12] + g * matrix_argb[13] + + r * matrix_argb[14] + a * matrix_argb[15]) >> 6; + dst_argb[0] = Clamp(sb); + dst_argb[1] = Clamp(sg); + dst_argb[2] = Clamp(sr); + dst_argb[3] = Clamp(sa); + src_argb += 4; + dst_argb += 4; + } +} + +// Apply color table to a row of image. +void ARGBColorTableRow_C(uint8* dst_argb, const uint8* table_argb, int width) { + int x; + for (x = 0; x < width; ++x) { + int b = dst_argb[0]; + int g = dst_argb[1]; + int r = dst_argb[2]; + int a = dst_argb[3]; + dst_argb[0] = table_argb[b * 4 + 0]; + dst_argb[1] = table_argb[g * 4 + 1]; + dst_argb[2] = table_argb[r * 4 + 2]; + dst_argb[3] = table_argb[a * 4 + 3]; + dst_argb += 4; + } +} + +// Apply color table to a row of image. +void RGBColorTableRow_C(uint8* dst_argb, const uint8* table_argb, int width) { + int x; + for (x = 0; x < width; ++x) { + int b = dst_argb[0]; + int g = dst_argb[1]; + int r = dst_argb[2]; + dst_argb[0] = table_argb[b * 4 + 0]; + dst_argb[1] = table_argb[g * 4 + 1]; + dst_argb[2] = table_argb[r * 4 + 2]; + dst_argb += 4; + } +} + +void ARGBQuantizeRow_C(uint8* dst_argb, int scale, int interval_size, + int interval_offset, int width) { + int x; + for (x = 0; x < width; ++x) { + int b = dst_argb[0]; + int g = dst_argb[1]; + int r = dst_argb[2]; + dst_argb[0] = (b * scale >> 16) * interval_size + interval_offset; + dst_argb[1] = (g * scale >> 16) * interval_size + interval_offset; + dst_argb[2] = (r * scale >> 16) * interval_size + interval_offset; + dst_argb += 4; + } +} + +#define REPEAT8(v) (v) | ((v) << 8) +#define SHADE(f, v) v * f >> 24 + +void ARGBShadeRow_C(const uint8* src_argb, uint8* dst_argb, int width, + uint32 value) { + const uint32 b_scale = REPEAT8(value & 0xff); + const uint32 g_scale = REPEAT8((value >> 8) & 0xff); + const uint32 r_scale = REPEAT8((value >> 16) & 0xff); + const uint32 a_scale = REPEAT8(value >> 24); + + int i; + for (i = 0; i < width; ++i) { + const uint32 b = REPEAT8(src_argb[0]); + const uint32 g = REPEAT8(src_argb[1]); + const uint32 r = REPEAT8(src_argb[2]); + const uint32 a = REPEAT8(src_argb[3]); + dst_argb[0] = SHADE(b, b_scale); + dst_argb[1] = SHADE(g, g_scale); + dst_argb[2] = SHADE(r, r_scale); + dst_argb[3] = SHADE(a, a_scale); + src_argb += 4; + dst_argb += 4; + } +} +#undef REPEAT8 +#undef SHADE + +#define REPEAT8(v) (v) | ((v) << 8) +#define SHADE(f, v) v * f >> 16 + +void ARGBMultiplyRow_C(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + int i; + for (i = 0; i < width; ++i) { + const uint32 b = REPEAT8(src_argb0[0]); + const uint32 g = REPEAT8(src_argb0[1]); + const uint32 r = REPEAT8(src_argb0[2]); + const uint32 a = REPEAT8(src_argb0[3]); + const uint32 b_scale = src_argb1[0]; + const uint32 g_scale = src_argb1[1]; + const uint32 r_scale = src_argb1[2]; + const uint32 a_scale = src_argb1[3]; + dst_argb[0] = SHADE(b, b_scale); + dst_argb[1] = SHADE(g, g_scale); + dst_argb[2] = SHADE(r, r_scale); + dst_argb[3] = SHADE(a, a_scale); + src_argb0 += 4; + src_argb1 += 4; + dst_argb += 4; + } +} +#undef REPEAT8 +#undef SHADE + +#define SHADE(f, v) clamp255(v + f) + +void ARGBAddRow_C(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + int i; + for (i = 0; i < width; ++i) { + const int b = src_argb0[0]; + const int g = src_argb0[1]; + const int r = src_argb0[2]; + const int a = src_argb0[3]; + const int b_add = src_argb1[0]; + const int g_add = src_argb1[1]; + const int r_add = src_argb1[2]; + const int a_add = src_argb1[3]; + dst_argb[0] = SHADE(b, b_add); + dst_argb[1] = SHADE(g, g_add); + dst_argb[2] = SHADE(r, r_add); + dst_argb[3] = SHADE(a, a_add); + src_argb0 += 4; + src_argb1 += 4; + dst_argb += 4; + } +} +#undef SHADE + +#define SHADE(f, v) clamp0(f - v) + +void ARGBSubtractRow_C(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + int i; + for (i = 0; i < width; ++i) { + const int b = src_argb0[0]; + const int g = src_argb0[1]; + const int r = src_argb0[2]; + const int a = src_argb0[3]; + const int b_sub = src_argb1[0]; + const int g_sub = src_argb1[1]; + const int r_sub = src_argb1[2]; + const int a_sub = src_argb1[3]; + dst_argb[0] = SHADE(b, b_sub); + dst_argb[1] = SHADE(g, g_sub); + dst_argb[2] = SHADE(r, r_sub); + dst_argb[3] = SHADE(a, a_sub); + src_argb0 += 4; + src_argb1 += 4; + dst_argb += 4; + } +} +#undef SHADE + +// Sobel functions which mimics SSSE3. +void SobelXRow_C(const uint8* src_y0, const uint8* src_y1, const uint8* src_y2, + uint8* dst_sobelx, int width) { + int i; + for (i = 0; i < width; ++i) { + int a = src_y0[i]; + int b = src_y1[i]; + int c = src_y2[i]; + int a_sub = src_y0[i + 2]; + int b_sub = src_y1[i + 2]; + int c_sub = src_y2[i + 2]; + int a_diff = a - a_sub; + int b_diff = b - b_sub; + int c_diff = c - c_sub; + int sobel = Abs(a_diff + b_diff * 2 + c_diff); + dst_sobelx[i] = (uint8)(clamp255(sobel)); + } +} + +void SobelYRow_C(const uint8* src_y0, const uint8* src_y1, + uint8* dst_sobely, int width) { + int i; + for (i = 0; i < width; ++i) { + int a = src_y0[i + 0]; + int b = src_y0[i + 1]; + int c = src_y0[i + 2]; + int a_sub = src_y1[i + 0]; + int b_sub = src_y1[i + 1]; + int c_sub = src_y1[i + 2]; + int a_diff = a - a_sub; + int b_diff = b - b_sub; + int c_diff = c - c_sub; + int sobel = Abs(a_diff + b_diff * 2 + c_diff); + dst_sobely[i] = (uint8)(clamp255(sobel)); + } +} + +void SobelRow_C(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width) { + int i; + for (i = 0; i < width; ++i) { + int r = src_sobelx[i]; + int b = src_sobely[i]; + int s = clamp255(r + b); + dst_argb[0] = (uint8)(s); + dst_argb[1] = (uint8)(s); + dst_argb[2] = (uint8)(s); + dst_argb[3] = (uint8)(255u); + dst_argb += 4; + } +} + +void SobelToPlaneRow_C(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_y, int width) { + int i; + for (i = 0; i < width; ++i) { + int r = src_sobelx[i]; + int b = src_sobely[i]; + int s = clamp255(r + b); + dst_y[i] = (uint8)(s); + } +} + +void SobelXYRow_C(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width) { + int i; + for (i = 0; i < width; ++i) { + int r = src_sobelx[i]; + int b = src_sobely[i]; + int g = clamp255(r + b); + dst_argb[0] = (uint8)(b); + dst_argb[1] = (uint8)(g); + dst_argb[2] = (uint8)(r); + dst_argb[3] = (uint8)(255u); + dst_argb += 4; + } +} + +void J400ToARGBRow_C(const uint8* src_y, uint8* dst_argb, int width) { + // Copy a Y to RGB. + int x; + for (x = 0; x < width; ++x) { + uint8 y = src_y[0]; + dst_argb[2] = dst_argb[1] = dst_argb[0] = y; + dst_argb[3] = 255u; + dst_argb += 4; + ++src_y; + } +} + +// BT.601 YUV to RGB reference +// R = (Y - 16) * 1.164 - V * -1.596 +// G = (Y - 16) * 1.164 - U * 0.391 - V * 0.813 +// B = (Y - 16) * 1.164 - U * -2.018 + +// Y contribution to R,G,B. Scale and bias. +// TODO(fbarchard): Consider moving constants into a common header. +#define YG 18997 /* round(1.164 * 64 * 256 * 256 / 257) */ +#define YGB -1160 /* 1.164 * 64 * -16 + 64 / 2 */ + +// U and V contributions to R,G,B. +#define UB -128 /* max(-128, round(-2.018 * 64)) */ +#define UG 25 /* round(0.391 * 64) */ +#define VG 52 /* round(0.813 * 64) */ +#define VR -102 /* round(-1.596 * 64) */ + +// Bias values to subtract 16 from Y and 128 from U and V. +#define BB (UB * 128 + YGB) +#define BG (UG * 128 + VG * 128 + YGB) +#define BR (VR * 128 + YGB) + +// C reference code that mimics the YUV assembly. +static __inline void YuvPixel(uint8 y, uint8 u, uint8 v, + uint8* b, uint8* g, uint8* r) { + uint32 y1 = (uint32)(y * 0x0101 * YG) >> 16; + *b = Clamp((int32)(-(u * UB) + y1 + BB) >> 6); + *g = Clamp((int32)(-(v * VG + u * UG) + y1 + BG) >> 6); + *r = Clamp((int32)(-(v * VR)+ y1 + BR) >> 6); +} + +// C reference code that mimics the YUV assembly. +static __inline void YPixel(uint8 y, uint8* b, uint8* g, uint8* r) { + uint32 y1 = (uint32)(y * 0x0101 * YG) >> 16; + *b = Clamp((int32)(y1 + YGB) >> 6); + *g = Clamp((int32)(y1 + YGB) >> 6); + *r = Clamp((int32)(y1 + YGB) >> 6); +} + +#undef YG +#undef YGB +#undef UB +#undef UG +#undef VG +#undef VR +#undef BB +#undef BG +#undef BR + +// JPEG YUV to RGB reference +// * R = Y - V * -1.40200 +// * G = Y - U * 0.34414 - V * 0.71414 +// * B = Y - U * -1.77200 + +// Y contribution to R,G,B. Scale and bias. +// TODO(fbarchard): Consider moving constants into a common header. +#define YGJ 16320 /* round(1.000 * 64 * 256 * 256 / 257) */ +#define YGBJ 32 /* 64 / 2 */ + +// U and V contributions to R,G,B. +#define UBJ -113 /* round(-1.77200 * 64) */ +#define UGJ 22 /* round(0.34414 * 64) */ +#define VGJ 46 /* round(0.71414 * 64) */ +#define VRJ -90 /* round(-1.40200 * 64) */ + +// Bias values to subtract 16 from Y and 128 from U and V. +#define BBJ (UBJ * 128 + YGBJ) +#define BGJ (UGJ * 128 + VGJ * 128 + YGBJ) +#define BRJ (VRJ * 128 + YGBJ) + +// C reference code that mimics the YUV assembly. +static __inline void YuvJPixel(uint8 y, uint8 u, uint8 v, + uint8* b, uint8* g, uint8* r) { + uint32 y1 = (uint32)(y * 0x0101 * YGJ) >> 16; + *b = Clamp((int32)(-(u * UBJ) + y1 + BBJ) >> 6); + *g = Clamp((int32)(-(v * VGJ + u * UGJ) + y1 + BGJ) >> 6); + *r = Clamp((int32)(-(v * VRJ) + y1 + BRJ) >> 6); +} + +#undef YGJ +#undef YGBJ +#undef UBJ +#undef UGJ +#undef VGJ +#undef VRJ +#undef BBJ +#undef BGJ +#undef BRJ + +#if !defined(LIBYUV_DISABLE_NEON) && \ + (defined(__ARM_NEON__) || defined(__aarch64__) || defined(LIBYUV_NEON)) +// C mimic assembly. +// TODO(fbarchard): Remove subsampling from Neon. +void I444ToARGBRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + uint8 u = (src_u[0] + src_u[1] + 1) >> 1; + uint8 v = (src_v[0] + src_v[1] + 1) >> 1; + YuvPixel(src_y[0], u, v, rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + YuvPixel(src_y[1], u, v, rgb_buf + 4, rgb_buf + 5, rgb_buf + 6); + rgb_buf[7] = 255; + src_y += 2; + src_u += 2; + src_v += 2; + rgb_buf += 8; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + } +} +#else +void I444ToARGBRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width; ++x) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + src_y += 1; + src_u += 1; + src_v += 1; + rgb_buf += 4; // Advance 1 pixel. + } +} +#endif + +// Also used for 420 +void I422ToARGBRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + YuvPixel(src_y[1], src_u[0], src_v[0], + rgb_buf + 4, rgb_buf + 5, rgb_buf + 6); + rgb_buf[7] = 255; + src_y += 2; + src_u += 1; + src_v += 1; + rgb_buf += 8; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + } +} + +void J422ToARGBRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + YuvJPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + YuvJPixel(src_y[1], src_u[0], src_v[0], + rgb_buf + 4, rgb_buf + 5, rgb_buf + 6); + rgb_buf[7] = 255; + src_y += 2; + src_u += 1; + src_v += 1; + rgb_buf += 8; // Advance 2 pixels. + } + if (width & 1) { + YuvJPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + } +} + +void I422ToRGB24Row_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + YuvPixel(src_y[1], src_u[0], src_v[0], + rgb_buf + 3, rgb_buf + 4, rgb_buf + 5); + src_y += 2; + src_u += 1; + src_v += 1; + rgb_buf += 6; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + } +} + +void I422ToRAWRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 2, rgb_buf + 1, rgb_buf + 0); + YuvPixel(src_y[1], src_u[0], src_v[0], + rgb_buf + 5, rgb_buf + 4, rgb_buf + 3); + src_y += 2; + src_u += 1; + src_v += 1; + rgb_buf += 6; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 2, rgb_buf + 1, rgb_buf + 0); + } +} + +void I422ToARGB4444Row_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb4444, + int width) { + uint8 b0; + uint8 g0; + uint8 r0; + uint8 b1; + uint8 g1; + uint8 r1; + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], src_u[0], src_v[0], &b0, &g0, &r0); + YuvPixel(src_y[1], src_u[0], src_v[0], &b1, &g1, &r1); + b0 = b0 >> 4; + g0 = g0 >> 4; + r0 = r0 >> 4; + b1 = b1 >> 4; + g1 = g1 >> 4; + r1 = r1 >> 4; + *(uint32*)(dst_argb4444) = b0 | (g0 << 4) | (r0 << 8) | + (b1 << 16) | (g1 << 20) | (r1 << 24) | 0xf000f000; + src_y += 2; + src_u += 1; + src_v += 1; + dst_argb4444 += 4; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_u[0], src_v[0], &b0, &g0, &r0); + b0 = b0 >> 4; + g0 = g0 >> 4; + r0 = r0 >> 4; + *(uint16*)(dst_argb4444) = b0 | (g0 << 4) | (r0 << 8) | + 0xf000; + } +} + +void I422ToARGB1555Row_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb1555, + int width) { + uint8 b0; + uint8 g0; + uint8 r0; + uint8 b1; + uint8 g1; + uint8 r1; + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], src_u[0], src_v[0], &b0, &g0, &r0); + YuvPixel(src_y[1], src_u[0], src_v[0], &b1, &g1, &r1); + b0 = b0 >> 3; + g0 = g0 >> 3; + r0 = r0 >> 3; + b1 = b1 >> 3; + g1 = g1 >> 3; + r1 = r1 >> 3; + *(uint32*)(dst_argb1555) = b0 | (g0 << 5) | (r0 << 10) | + (b1 << 16) | (g1 << 21) | (r1 << 26) | 0x80008000; + src_y += 2; + src_u += 1; + src_v += 1; + dst_argb1555 += 4; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_u[0], src_v[0], &b0, &g0, &r0); + b0 = b0 >> 3; + g0 = g0 >> 3; + r0 = r0 >> 3; + *(uint16*)(dst_argb1555) = b0 | (g0 << 5) | (r0 << 10) | + 0x8000; + } +} + +void I422ToRGB565Row_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb565, + int width) { + uint8 b0; + uint8 g0; + uint8 r0; + uint8 b1; + uint8 g1; + uint8 r1; + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], src_u[0], src_v[0], &b0, &g0, &r0); + YuvPixel(src_y[1], src_u[0], src_v[0], &b1, &g1, &r1); + b0 = b0 >> 3; + g0 = g0 >> 2; + r0 = r0 >> 3; + b1 = b1 >> 3; + g1 = g1 >> 2; + r1 = r1 >> 3; + *(uint32*)(dst_rgb565) = b0 | (g0 << 5) | (r0 << 11) | + (b1 << 16) | (g1 << 21) | (r1 << 27); + src_y += 2; + src_u += 1; + src_v += 1; + dst_rgb565 += 4; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_u[0], src_v[0], &b0, &g0, &r0); + b0 = b0 >> 3; + g0 = g0 >> 2; + r0 = r0 >> 3; + *(uint16*)(dst_rgb565) = b0 | (g0 << 5) | (r0 << 11); + } +} + +void I411ToARGBRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 3; x += 4) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + YuvPixel(src_y[1], src_u[0], src_v[0], + rgb_buf + 4, rgb_buf + 5, rgb_buf + 6); + rgb_buf[7] = 255; + YuvPixel(src_y[2], src_u[0], src_v[0], + rgb_buf + 8, rgb_buf + 9, rgb_buf + 10); + rgb_buf[11] = 255; + YuvPixel(src_y[3], src_u[0], src_v[0], + rgb_buf + 12, rgb_buf + 13, rgb_buf + 14); + rgb_buf[15] = 255; + src_y += 4; + src_u += 1; + src_v += 1; + rgb_buf += 16; // Advance 4 pixels. + } + if (width & 2) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + YuvPixel(src_y[1], src_u[0], src_v[0], + rgb_buf + 4, rgb_buf + 5, rgb_buf + 6); + rgb_buf[7] = 255; + src_y += 2; + rgb_buf += 8; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + } +} + +void NV12ToARGBRow_C(const uint8* src_y, + const uint8* src_uv, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], src_uv[0], src_uv[1], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + YuvPixel(src_y[1], src_uv[0], src_uv[1], + rgb_buf + 4, rgb_buf + 5, rgb_buf + 6); + rgb_buf[7] = 255; + src_y += 2; + src_uv += 2; + rgb_buf += 8; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_uv[0], src_uv[1], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + } +} + +void NV21ToARGBRow_C(const uint8* src_y, + const uint8* src_vu, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], src_vu[1], src_vu[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + + YuvPixel(src_y[1], src_vu[1], src_vu[0], + rgb_buf + 4, rgb_buf + 5, rgb_buf + 6); + rgb_buf[7] = 255; + + src_y += 2; + src_vu += 2; + rgb_buf += 8; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_vu[1], src_vu[0], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + } +} + +void NV12ToRGB565Row_C(const uint8* src_y, + const uint8* src_uv, + uint8* dst_rgb565, + int width) { + uint8 b0; + uint8 g0; + uint8 r0; + uint8 b1; + uint8 g1; + uint8 r1; + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], src_uv[0], src_uv[1], &b0, &g0, &r0); + YuvPixel(src_y[1], src_uv[0], src_uv[1], &b1, &g1, &r1); + b0 = b0 >> 3; + g0 = g0 >> 2; + r0 = r0 >> 3; + b1 = b1 >> 3; + g1 = g1 >> 2; + r1 = r1 >> 3; + *(uint32*)(dst_rgb565) = b0 | (g0 << 5) | (r0 << 11) | + (b1 << 16) | (g1 << 21) | (r1 << 27); + src_y += 2; + src_uv += 2; + dst_rgb565 += 4; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_uv[0], src_uv[1], &b0, &g0, &r0); + b0 = b0 >> 3; + g0 = g0 >> 2; + r0 = r0 >> 3; + *(uint16*)(dst_rgb565) = b0 | (g0 << 5) | (r0 << 11); + } +} + +void NV21ToRGB565Row_C(const uint8* src_y, + const uint8* vsrc_u, + uint8* dst_rgb565, + int width) { + uint8 b0; + uint8 g0; + uint8 r0; + uint8 b1; + uint8 g1; + uint8 r1; + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], vsrc_u[1], vsrc_u[0], &b0, &g0, &r0); + YuvPixel(src_y[1], vsrc_u[1], vsrc_u[0], &b1, &g1, &r1); + b0 = b0 >> 3; + g0 = g0 >> 2; + r0 = r0 >> 3; + b1 = b1 >> 3; + g1 = g1 >> 2; + r1 = r1 >> 3; + *(uint32*)(dst_rgb565) = b0 | (g0 << 5) | (r0 << 11) | + (b1 << 16) | (g1 << 21) | (r1 << 27); + src_y += 2; + vsrc_u += 2; + dst_rgb565 += 4; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], vsrc_u[1], vsrc_u[0], &b0, &g0, &r0); + b0 = b0 >> 3; + g0 = g0 >> 2; + r0 = r0 >> 3; + *(uint16*)(dst_rgb565) = b0 | (g0 << 5) | (r0 << 11); + } +} + +void YUY2ToARGBRow_C(const uint8* src_yuy2, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_yuy2[0], src_yuy2[1], src_yuy2[3], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + YuvPixel(src_yuy2[2], src_yuy2[1], src_yuy2[3], + rgb_buf + 4, rgb_buf + 5, rgb_buf + 6); + rgb_buf[7] = 255; + src_yuy2 += 4; + rgb_buf += 8; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_yuy2[0], src_yuy2[1], src_yuy2[3], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + } +} + +void UYVYToARGBRow_C(const uint8* src_uyvy, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_uyvy[1], src_uyvy[0], src_uyvy[2], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + YuvPixel(src_uyvy[3], src_uyvy[0], src_uyvy[2], + rgb_buf + 4, rgb_buf + 5, rgb_buf + 6); + rgb_buf[7] = 255; + src_uyvy += 4; + rgb_buf += 8; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_uyvy[1], src_uyvy[0], src_uyvy[2], + rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + } +} + +void I422ToBGRARow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 3, rgb_buf + 2, rgb_buf + 1); + rgb_buf[0] = 255; + YuvPixel(src_y[1], src_u[0], src_v[0], + rgb_buf + 7, rgb_buf + 6, rgb_buf + 5); + rgb_buf[4] = 255; + src_y += 2; + src_u += 1; + src_v += 1; + rgb_buf += 8; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 3, rgb_buf + 2, rgb_buf + 1); + rgb_buf[0] = 255; + } +} + +void I422ToABGRRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 2, rgb_buf + 1, rgb_buf + 0); + rgb_buf[3] = 255; + YuvPixel(src_y[1], src_u[0], src_v[0], + rgb_buf + 6, rgb_buf + 5, rgb_buf + 4); + rgb_buf[7] = 255; + src_y += 2; + src_u += 1; + src_v += 1; + rgb_buf += 8; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 2, rgb_buf + 1, rgb_buf + 0); + rgb_buf[3] = 255; + } +} + +void I422ToRGBARow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* rgb_buf, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 1, rgb_buf + 2, rgb_buf + 3); + rgb_buf[0] = 255; + YuvPixel(src_y[1], src_u[0], src_v[0], + rgb_buf + 5, rgb_buf + 6, rgb_buf + 7); + rgb_buf[4] = 255; + src_y += 2; + src_u += 1; + src_v += 1; + rgb_buf += 8; // Advance 2 pixels. + } + if (width & 1) { + YuvPixel(src_y[0], src_u[0], src_v[0], + rgb_buf + 1, rgb_buf + 2, rgb_buf + 3); + rgb_buf[0] = 255; + } +} + +void I400ToARGBRow_C(const uint8* src_y, uint8* rgb_buf, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + YPixel(src_y[0], rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + YPixel(src_y[1], rgb_buf + 4, rgb_buf + 5, rgb_buf + 6); + rgb_buf[7] = 255; + src_y += 2; + rgb_buf += 8; // Advance 2 pixels. + } + if (width & 1) { + YPixel(src_y[0], rgb_buf + 0, rgb_buf + 1, rgb_buf + 2); + rgb_buf[3] = 255; + } +} + +void MirrorRow_C(const uint8* src, uint8* dst, int width) { + int x; + src += width - 1; + for (x = 0; x < width - 1; x += 2) { + dst[x] = src[0]; + dst[x + 1] = src[-1]; + src -= 2; + } + if (width & 1) { + dst[width - 1] = src[0]; + } +} + +void MirrorUVRow_C(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int width) { + int x; + src_uv += (width - 1) << 1; + for (x = 0; x < width - 1; x += 2) { + dst_u[x] = src_uv[0]; + dst_u[x + 1] = src_uv[-2]; + dst_v[x] = src_uv[1]; + dst_v[x + 1] = src_uv[-2 + 1]; + src_uv -= 4; + } + if (width & 1) { + dst_u[width - 1] = src_uv[0]; + dst_v[width - 1] = src_uv[1]; + } +} + +void ARGBMirrorRow_C(const uint8* src, uint8* dst, int width) { + int x; + const uint32* src32 = (const uint32*)(src); + uint32* dst32 = (uint32*)(dst); + src32 += width - 1; + for (x = 0; x < width - 1; x += 2) { + dst32[x] = src32[0]; + dst32[x + 1] = src32[-1]; + src32 -= 2; + } + if (width & 1) { + dst32[width - 1] = src32[0]; + } +} + +void SplitUVRow_C(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + dst_u[x] = src_uv[0]; + dst_u[x + 1] = src_uv[2]; + dst_v[x] = src_uv[1]; + dst_v[x + 1] = src_uv[3]; + src_uv += 4; + } + if (width & 1) { + dst_u[width - 1] = src_uv[0]; + dst_v[width - 1] = src_uv[1]; + } +} + +void MergeUVRow_C(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + dst_uv[0] = src_u[x]; + dst_uv[1] = src_v[x]; + dst_uv[2] = src_u[x + 1]; + dst_uv[3] = src_v[x + 1]; + dst_uv += 4; + } + if (width & 1) { + dst_uv[0] = src_u[width - 1]; + dst_uv[1] = src_v[width - 1]; + } +} + +void CopyRow_C(const uint8* src, uint8* dst, int count) { + memcpy(dst, src, count); +} + +void CopyRow_16_C(const uint16* src, uint16* dst, int count) { + memcpy(dst, src, count * 2); +} + +void SetRow_C(uint8* dst, uint8 v8, int width) { + memset(dst, v8, width); +} + +void ARGBSetRow_C(uint8* dst_argb, uint32 v32, int width) { + uint32* d = (uint32*)(dst_argb); + int x; + for (x = 0; x < width; ++x) { + d[x] = v32; + } +} + +// Filter 2 rows of YUY2 UV's (422) into U and V (420). +void YUY2ToUVRow_C(const uint8* src_yuy2, int src_stride_yuy2, + uint8* dst_u, uint8* dst_v, int width) { + // Output a row of UV values, filtering 2 rows of YUY2. + int x; + for (x = 0; x < width; x += 2) { + dst_u[0] = (src_yuy2[1] + src_yuy2[src_stride_yuy2 + 1] + 1) >> 1; + dst_v[0] = (src_yuy2[3] + src_yuy2[src_stride_yuy2 + 3] + 1) >> 1; + src_yuy2 += 4; + dst_u += 1; + dst_v += 1; + } +} + +// Copy row of YUY2 UV's (422) into U and V (422). +void YUY2ToUV422Row_C(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int width) { + // Output a row of UV values. + int x; + for (x = 0; x < width; x += 2) { + dst_u[0] = src_yuy2[1]; + dst_v[0] = src_yuy2[3]; + src_yuy2 += 4; + dst_u += 1; + dst_v += 1; + } +} + +// Copy row of YUY2 Y's (422) into Y (420/422). +void YUY2ToYRow_C(const uint8* src_yuy2, uint8* dst_y, int width) { + // Output a row of Y values. + int x; + for (x = 0; x < width - 1; x += 2) { + dst_y[x] = src_yuy2[0]; + dst_y[x + 1] = src_yuy2[2]; + src_yuy2 += 4; + } + if (width & 1) { + dst_y[width - 1] = src_yuy2[0]; + } +} + +// Filter 2 rows of UYVY UV's (422) into U and V (420). +void UYVYToUVRow_C(const uint8* src_uyvy, int src_stride_uyvy, + uint8* dst_u, uint8* dst_v, int width) { + // Output a row of UV values. + int x; + for (x = 0; x < width; x += 2) { + dst_u[0] = (src_uyvy[0] + src_uyvy[src_stride_uyvy + 0] + 1) >> 1; + dst_v[0] = (src_uyvy[2] + src_uyvy[src_stride_uyvy + 2] + 1) >> 1; + src_uyvy += 4; + dst_u += 1; + dst_v += 1; + } +} + +// Copy row of UYVY UV's (422) into U and V (422). +void UYVYToUV422Row_C(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int width) { + // Output a row of UV values. + int x; + for (x = 0; x < width; x += 2) { + dst_u[0] = src_uyvy[0]; + dst_v[0] = src_uyvy[2]; + src_uyvy += 4; + dst_u += 1; + dst_v += 1; + } +} + +// Copy row of UYVY Y's (422) into Y (420/422). +void UYVYToYRow_C(const uint8* src_uyvy, uint8* dst_y, int width) { + // Output a row of Y values. + int x; + for (x = 0; x < width - 1; x += 2) { + dst_y[x] = src_uyvy[1]; + dst_y[x + 1] = src_uyvy[3]; + src_uyvy += 4; + } + if (width & 1) { + dst_y[width - 1] = src_uyvy[1]; + } +} + +#define BLEND(f, b, a) (((256 - a) * b) >> 8) + f + +// Blend src_argb0 over src_argb1 and store to dst_argb. +// dst_argb may be src_argb0 or src_argb1. +// This code mimics the SSSE3 version for better testability. +void ARGBBlendRow_C(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + uint32 fb = src_argb0[0]; + uint32 fg = src_argb0[1]; + uint32 fr = src_argb0[2]; + uint32 a = src_argb0[3]; + uint32 bb = src_argb1[0]; + uint32 bg = src_argb1[1]; + uint32 br = src_argb1[2]; + dst_argb[0] = BLEND(fb, bb, a); + dst_argb[1] = BLEND(fg, bg, a); + dst_argb[2] = BLEND(fr, br, a); + dst_argb[3] = 255u; + + fb = src_argb0[4 + 0]; + fg = src_argb0[4 + 1]; + fr = src_argb0[4 + 2]; + a = src_argb0[4 + 3]; + bb = src_argb1[4 + 0]; + bg = src_argb1[4 + 1]; + br = src_argb1[4 + 2]; + dst_argb[4 + 0] = BLEND(fb, bb, a); + dst_argb[4 + 1] = BLEND(fg, bg, a); + dst_argb[4 + 2] = BLEND(fr, br, a); + dst_argb[4 + 3] = 255u; + src_argb0 += 8; + src_argb1 += 8; + dst_argb += 8; + } + + if (width & 1) { + uint32 fb = src_argb0[0]; + uint32 fg = src_argb0[1]; + uint32 fr = src_argb0[2]; + uint32 a = src_argb0[3]; + uint32 bb = src_argb1[0]; + uint32 bg = src_argb1[1]; + uint32 br = src_argb1[2]; + dst_argb[0] = BLEND(fb, bb, a); + dst_argb[1] = BLEND(fg, bg, a); + dst_argb[2] = BLEND(fr, br, a); + dst_argb[3] = 255u; + } +} +#undef BLEND +#define ATTENUATE(f, a) (a | (a << 8)) * (f | (f << 8)) >> 24 + +// Multiply source RGB by alpha and store to destination. +// This code mimics the SSSE3 version for better testability. +void ARGBAttenuateRow_C(const uint8* src_argb, uint8* dst_argb, int width) { + int i; + for (i = 0; i < width - 1; i += 2) { + uint32 b = src_argb[0]; + uint32 g = src_argb[1]; + uint32 r = src_argb[2]; + uint32 a = src_argb[3]; + dst_argb[0] = ATTENUATE(b, a); + dst_argb[1] = ATTENUATE(g, a); + dst_argb[2] = ATTENUATE(r, a); + dst_argb[3] = a; + b = src_argb[4]; + g = src_argb[5]; + r = src_argb[6]; + a = src_argb[7]; + dst_argb[4] = ATTENUATE(b, a); + dst_argb[5] = ATTENUATE(g, a); + dst_argb[6] = ATTENUATE(r, a); + dst_argb[7] = a; + src_argb += 8; + dst_argb += 8; + } + + if (width & 1) { + const uint32 b = src_argb[0]; + const uint32 g = src_argb[1]; + const uint32 r = src_argb[2]; + const uint32 a = src_argb[3]; + dst_argb[0] = ATTENUATE(b, a); + dst_argb[1] = ATTENUATE(g, a); + dst_argb[2] = ATTENUATE(r, a); + dst_argb[3] = a; + } +} +#undef ATTENUATE + +// Divide source RGB by alpha and store to destination. +// b = (b * 255 + (a / 2)) / a; +// g = (g * 255 + (a / 2)) / a; +// r = (r * 255 + (a / 2)) / a; +// Reciprocal method is off by 1 on some values. ie 125 +// 8.8 fixed point inverse table with 1.0 in upper short and 1 / a in lower. +#define T(a) 0x01000000 + (0x10000 / a) +const uint32 fixed_invtbl8[256] = { + 0x01000000, 0x0100ffff, T(0x02), T(0x03), T(0x04), T(0x05), T(0x06), T(0x07), + T(0x08), T(0x09), T(0x0a), T(0x0b), T(0x0c), T(0x0d), T(0x0e), T(0x0f), + T(0x10), T(0x11), T(0x12), T(0x13), T(0x14), T(0x15), T(0x16), T(0x17), + T(0x18), T(0x19), T(0x1a), T(0x1b), T(0x1c), T(0x1d), T(0x1e), T(0x1f), + T(0x20), T(0x21), T(0x22), T(0x23), T(0x24), T(0x25), T(0x26), T(0x27), + T(0x28), T(0x29), T(0x2a), T(0x2b), T(0x2c), T(0x2d), T(0x2e), T(0x2f), + T(0x30), T(0x31), T(0x32), T(0x33), T(0x34), T(0x35), T(0x36), T(0x37), + T(0x38), T(0x39), T(0x3a), T(0x3b), T(0x3c), T(0x3d), T(0x3e), T(0x3f), + T(0x40), T(0x41), T(0x42), T(0x43), T(0x44), T(0x45), T(0x46), T(0x47), + T(0x48), T(0x49), T(0x4a), T(0x4b), T(0x4c), T(0x4d), T(0x4e), T(0x4f), + T(0x50), T(0x51), T(0x52), T(0x53), T(0x54), T(0x55), T(0x56), T(0x57), + T(0x58), T(0x59), T(0x5a), T(0x5b), T(0x5c), T(0x5d), T(0x5e), T(0x5f), + T(0x60), T(0x61), T(0x62), T(0x63), T(0x64), T(0x65), T(0x66), T(0x67), + T(0x68), T(0x69), T(0x6a), T(0x6b), T(0x6c), T(0x6d), T(0x6e), T(0x6f), + T(0x70), T(0x71), T(0x72), T(0x73), T(0x74), T(0x75), T(0x76), T(0x77), + T(0x78), T(0x79), T(0x7a), T(0x7b), T(0x7c), T(0x7d), T(0x7e), T(0x7f), + T(0x80), T(0x81), T(0x82), T(0x83), T(0x84), T(0x85), T(0x86), T(0x87), + T(0x88), T(0x89), T(0x8a), T(0x8b), T(0x8c), T(0x8d), T(0x8e), T(0x8f), + T(0x90), T(0x91), T(0x92), T(0x93), T(0x94), T(0x95), T(0x96), T(0x97), + T(0x98), T(0x99), T(0x9a), T(0x9b), T(0x9c), T(0x9d), T(0x9e), T(0x9f), + T(0xa0), T(0xa1), T(0xa2), T(0xa3), T(0xa4), T(0xa5), T(0xa6), T(0xa7), + T(0xa8), T(0xa9), T(0xaa), T(0xab), T(0xac), T(0xad), T(0xae), T(0xaf), + T(0xb0), T(0xb1), T(0xb2), T(0xb3), T(0xb4), T(0xb5), T(0xb6), T(0xb7), + T(0xb8), T(0xb9), T(0xba), T(0xbb), T(0xbc), T(0xbd), T(0xbe), T(0xbf), + T(0xc0), T(0xc1), T(0xc2), T(0xc3), T(0xc4), T(0xc5), T(0xc6), T(0xc7), + T(0xc8), T(0xc9), T(0xca), T(0xcb), T(0xcc), T(0xcd), T(0xce), T(0xcf), + T(0xd0), T(0xd1), T(0xd2), T(0xd3), T(0xd4), T(0xd5), T(0xd6), T(0xd7), + T(0xd8), T(0xd9), T(0xda), T(0xdb), T(0xdc), T(0xdd), T(0xde), T(0xdf), + T(0xe0), T(0xe1), T(0xe2), T(0xe3), T(0xe4), T(0xe5), T(0xe6), T(0xe7), + T(0xe8), T(0xe9), T(0xea), T(0xeb), T(0xec), T(0xed), T(0xee), T(0xef), + T(0xf0), T(0xf1), T(0xf2), T(0xf3), T(0xf4), T(0xf5), T(0xf6), T(0xf7), + T(0xf8), T(0xf9), T(0xfa), T(0xfb), T(0xfc), T(0xfd), T(0xfe), 0x01000100 }; +#undef T + +void ARGBUnattenuateRow_C(const uint8* src_argb, uint8* dst_argb, int width) { + int i; + for (i = 0; i < width; ++i) { + uint32 b = src_argb[0]; + uint32 g = src_argb[1]; + uint32 r = src_argb[2]; + const uint32 a = src_argb[3]; + const uint32 ia = fixed_invtbl8[a] & 0xffff; // 8.8 fixed point + b = (b * ia) >> 8; + g = (g * ia) >> 8; + r = (r * ia) >> 8; + // Clamping should not be necessary but is free in assembly. + dst_argb[0] = clamp255(b); + dst_argb[1] = clamp255(g); + dst_argb[2] = clamp255(r); + dst_argb[3] = a; + src_argb += 4; + dst_argb += 4; + } +} + +void ComputeCumulativeSumRow_C(const uint8* row, int32* cumsum, + const int32* previous_cumsum, int width) { + int32 row_sum[4] = {0, 0, 0, 0}; + int x; + for (x = 0; x < width; ++x) { + row_sum[0] += row[x * 4 + 0]; + row_sum[1] += row[x * 4 + 1]; + row_sum[2] += row[x * 4 + 2]; + row_sum[3] += row[x * 4 + 3]; + cumsum[x * 4 + 0] = row_sum[0] + previous_cumsum[x * 4 + 0]; + cumsum[x * 4 + 1] = row_sum[1] + previous_cumsum[x * 4 + 1]; + cumsum[x * 4 + 2] = row_sum[2] + previous_cumsum[x * 4 + 2]; + cumsum[x * 4 + 3] = row_sum[3] + previous_cumsum[x * 4 + 3]; + } +} + +void CumulativeSumToAverageRow_C(const int32* tl, const int32* bl, + int w, int area, uint8* dst, int count) { + float ooa = 1.0f / area; + int i; + for (i = 0; i < count; ++i) { + dst[0] = (uint8)((bl[w + 0] + tl[0] - bl[0] - tl[w + 0]) * ooa); + dst[1] = (uint8)((bl[w + 1] + tl[1] - bl[1] - tl[w + 1]) * ooa); + dst[2] = (uint8)((bl[w + 2] + tl[2] - bl[2] - tl[w + 2]) * ooa); + dst[3] = (uint8)((bl[w + 3] + tl[3] - bl[3] - tl[w + 3]) * ooa); + dst += 4; + tl += 4; + bl += 4; + } +} + +// Copy pixels from rotated source to destination row with a slope. +LIBYUV_API +void ARGBAffineRow_C(const uint8* src_argb, int src_argb_stride, + uint8* dst_argb, const float* uv_dudv, int width) { + int i; + // Render a row of pixels from source into a buffer. + float uv[2]; + uv[0] = uv_dudv[0]; + uv[1] = uv_dudv[1]; + for (i = 0; i < width; ++i) { + int x = (int)(uv[0]); + int y = (int)(uv[1]); + *(uint32*)(dst_argb) = + *(const uint32*)(src_argb + y * src_argb_stride + + x * 4); + dst_argb += 4; + uv[0] += uv_dudv[2]; + uv[1] += uv_dudv[3]; + } +} + +// Blend 2 rows into 1. +static void HalfRow_C(const uint8* src_uv, int src_uv_stride, + uint8* dst_uv, int pix) { + int x; + for (x = 0; x < pix; ++x) { + dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1; + } +} + +static void HalfRow_16_C(const uint16* src_uv, int src_uv_stride, + uint16* dst_uv, int pix) { + int x; + for (x = 0; x < pix; ++x) { + dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1; + } +} + +// C version 2x2 -> 2x1. +void InterpolateRow_C(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, + int width, int source_y_fraction) { + int y1_fraction = source_y_fraction; + int y0_fraction = 256 - y1_fraction; + const uint8* src_ptr1 = src_ptr + src_stride; + int x; + if (source_y_fraction == 0) { + memcpy(dst_ptr, src_ptr, width); + return; + } + if (source_y_fraction == 128) { + HalfRow_C(src_ptr, (int)(src_stride), dst_ptr, width); + return; + } + for (x = 0; x < width - 1; x += 2) { + dst_ptr[0] = (src_ptr[0] * y0_fraction + src_ptr1[0] * y1_fraction) >> 8; + dst_ptr[1] = (src_ptr[1] * y0_fraction + src_ptr1[1] * y1_fraction) >> 8; + src_ptr += 2; + src_ptr1 += 2; + dst_ptr += 2; + } + if (width & 1) { + dst_ptr[0] = (src_ptr[0] * y0_fraction + src_ptr1[0] * y1_fraction) >> 8; + } +} + +void InterpolateRow_16_C(uint16* dst_ptr, const uint16* src_ptr, + ptrdiff_t src_stride, + int width, int source_y_fraction) { + int y1_fraction = source_y_fraction; + int y0_fraction = 256 - y1_fraction; + const uint16* src_ptr1 = src_ptr + src_stride; + int x; + if (source_y_fraction == 0) { + memcpy(dst_ptr, src_ptr, width * 2); + return; + } + if (source_y_fraction == 128) { + HalfRow_16_C(src_ptr, (int)(src_stride), dst_ptr, width); + return; + } + for (x = 0; x < width - 1; x += 2) { + dst_ptr[0] = (src_ptr[0] * y0_fraction + src_ptr1[0] * y1_fraction) >> 8; + dst_ptr[1] = (src_ptr[1] * y0_fraction + src_ptr1[1] * y1_fraction) >> 8; + src_ptr += 2; + src_ptr1 += 2; + dst_ptr += 2; + } + if (width & 1) { + dst_ptr[0] = (src_ptr[0] * y0_fraction + src_ptr1[0] * y1_fraction) >> 8; + } +} + +// Use first 4 shuffler values to reorder ARGB channels. +void ARGBShuffleRow_C(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix) { + int index0 = shuffler[0]; + int index1 = shuffler[1]; + int index2 = shuffler[2]; + int index3 = shuffler[3]; + // Shuffle a row of ARGB. + int x; + for (x = 0; x < pix; ++x) { + // To support in-place conversion. + uint8 b = src_argb[index0]; + uint8 g = src_argb[index1]; + uint8 r = src_argb[index2]; + uint8 a = src_argb[index3]; + dst_argb[0] = b; + dst_argb[1] = g; + dst_argb[2] = r; + dst_argb[3] = a; + src_argb += 4; + dst_argb += 4; + } +} + +void I422ToYUY2Row_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_frame, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + dst_frame[0] = src_y[0]; + dst_frame[1] = src_u[0]; + dst_frame[2] = src_y[1]; + dst_frame[3] = src_v[0]; + dst_frame += 4; + src_y += 2; + src_u += 1; + src_v += 1; + } + if (width & 1) { + dst_frame[0] = src_y[0]; + dst_frame[1] = src_u[0]; + dst_frame[2] = 0; + dst_frame[3] = src_v[0]; + } +} + +void I422ToUYVYRow_C(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_frame, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + dst_frame[0] = src_u[0]; + dst_frame[1] = src_y[0]; + dst_frame[2] = src_v[0]; + dst_frame[3] = src_y[1]; + dst_frame += 4; + src_y += 2; + src_u += 1; + src_v += 1; + } + if (width & 1) { + dst_frame[0] = src_u[0]; + dst_frame[1] = src_y[0]; + dst_frame[2] = src_v[0]; + dst_frame[3] = 0; + } +} + +// Maximum temporary width for wrappers to process at a time, in pixels. +#define MAXTWIDTH 2048 + +#if !(defined(_MSC_VER) && !defined(__clang__)) && \ + defined(HAS_I422TORGB565ROW_SSSE3) +// row_win.cc has asm version, but GCC uses 2 step wrapper. +void I422ToRGB565Row_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb565, + int width) { + SIMD_ALIGNED(uint8 row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_SSSE3(src_y, src_u, src_v, row, twidth); + ARGBToRGB565Row_SSE2(row, dst_rgb565, twidth); + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_rgb565 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_I422TOARGB1555ROW_SSSE3) +void I422ToARGB1555Row_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb1555, + int width) { + // Row buffer for intermediate ARGB pixels. + SIMD_ALIGNED(uint8 row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_SSSE3(src_y, src_u, src_v, row, twidth); + ARGBToARGB1555Row_SSE2(row, dst_argb1555, twidth); + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_argb1555 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_I422TOARGB4444ROW_SSSE3) +void I422ToARGB4444Row_SSSE3(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb4444, + int width) { + // Row buffer for intermediate ARGB pixels. + SIMD_ALIGNED(uint8 row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_SSSE3(src_y, src_u, src_v, row, twidth); + ARGBToARGB4444Row_SSE2(row, dst_argb4444, twidth); + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_argb4444 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_NV12TORGB565ROW_SSSE3) +void NV12ToRGB565Row_SSSE3(const uint8* src_y, const uint8* src_uv, + uint8* dst_rgb565, int width) { + // Row buffer for intermediate ARGB pixels. + SIMD_ALIGNED(uint8 row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + NV12ToARGBRow_SSSE3(src_y, src_uv, row, twidth); + ARGBToRGB565Row_SSE2(row, dst_rgb565, twidth); + src_y += twidth; + src_uv += twidth; + dst_rgb565 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_NV21TORGB565ROW_SSSE3) +void NV21ToRGB565Row_SSSE3(const uint8* src_y, const uint8* src_vu, + uint8* dst_rgb565, int width) { + // Row buffer for intermediate ARGB pixels. + SIMD_ALIGNED(uint8 row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + NV21ToARGBRow_SSSE3(src_y, src_vu, row, twidth); + ARGBToRGB565Row_SSE2(row, dst_rgb565, twidth); + src_y += twidth; + src_vu += twidth; + dst_rgb565 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_YUY2TOARGBROW_SSSE3) +void YUY2ToARGBRow_SSSE3(const uint8* src_yuy2, uint8* dst_argb, int width) { + // Row buffers for intermediate YUV pixels. + SIMD_ALIGNED(uint8 row_y[MAXTWIDTH]); + SIMD_ALIGNED(uint8 row_u[MAXTWIDTH / 2]); + SIMD_ALIGNED(uint8 row_v[MAXTWIDTH / 2]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + YUY2ToUV422Row_SSE2(src_yuy2, row_u, row_v, twidth); + YUY2ToYRow_SSE2(src_yuy2, row_y, twidth); + I422ToARGBRow_SSSE3(row_y, row_u, row_v, dst_argb, twidth); + src_yuy2 += twidth * 2; + dst_argb += twidth * 4; + width -= twidth; + } +} +#endif + +#if defined(HAS_UYVYTOARGBROW_SSSE3) +void UYVYToARGBRow_SSSE3(const uint8* src_uyvy, uint8* dst_argb, int width) { + // Row buffers for intermediate YUV pixels. + SIMD_ALIGNED(uint8 row_y[MAXTWIDTH]); + SIMD_ALIGNED(uint8 row_u[MAXTWIDTH / 2]); + SIMD_ALIGNED(uint8 row_v[MAXTWIDTH / 2]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + UYVYToUV422Row_SSE2(src_uyvy, row_u, row_v, twidth); + UYVYToYRow_SSE2(src_uyvy, row_y, twidth); + I422ToARGBRow_SSSE3(row_y, row_u, row_v, dst_argb, twidth); + src_uyvy += twidth * 2; + dst_argb += twidth * 4; + width -= twidth; + } +} +#endif // !defined(LIBYUV_DISABLE_X86) + +#if defined(HAS_I422TORGB565ROW_AVX2) +void I422ToRGB565Row_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb565, + int width) { + SIMD_ALIGNED32(uint8 row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_AVX2(src_y, src_u, src_v, row, twidth); + ARGBToRGB565Row_AVX2(row, dst_rgb565, twidth); + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_rgb565 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_I422TOARGB1555ROW_AVX2) +void I422ToARGB1555Row_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb1555, + int width) { + // Row buffer for intermediate ARGB pixels. + SIMD_ALIGNED32(uint8 row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_AVX2(src_y, src_u, src_v, row, twidth); + ARGBToARGB1555Row_AVX2(row, dst_argb1555, twidth); + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_argb1555 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_I422TOARGB4444ROW_AVX2) +void I422ToARGB4444Row_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb4444, + int width) { + // Row buffer for intermediate ARGB pixels. + SIMD_ALIGNED32(uint8 row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_AVX2(src_y, src_u, src_v, row, twidth); + ARGBToARGB4444Row_AVX2(row, dst_argb4444, twidth); + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_argb4444 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_I422TORGB24ROW_AVX2) +void I422ToRGB24Row_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb24, + int width) { + // Row buffer for intermediate ARGB pixels. + SIMD_ALIGNED32(uint8 row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_AVX2(src_y, src_u, src_v, row, twidth); + // TODO(fbarchard): ARGBToRGB24Row_AVX2 + ARGBToRGB24Row_SSSE3(row, dst_rgb24, twidth); + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_rgb24 += twidth * 3; + width -= twidth; + } +} +#endif + +#if defined(HAS_I422TORAWROW_AVX2) +void I422ToRAWRow_AVX2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_raw, + int width) { + // Row buffer for intermediate ARGB pixels. + SIMD_ALIGNED32(uint8 row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_AVX2(src_y, src_u, src_v, row, twidth); + // TODO(fbarchard): ARGBToRAWRow_AVX2 + ARGBToRAWRow_SSSE3(row, dst_raw, twidth); + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_raw += twidth * 3; + width -= twidth; + } +} +#endif + +#if defined(HAS_NV12TORGB565ROW_AVX2) +void NV12ToRGB565Row_AVX2(const uint8* src_y, const uint8* src_uv, + uint8* dst_rgb565, int width) { + // Row buffer for intermediate ARGB pixels. + SIMD_ALIGNED32(uint8 row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + NV12ToARGBRow_AVX2(src_y, src_uv, row, twidth); + ARGBToRGB565Row_AVX2(row, dst_rgb565, twidth); + src_y += twidth; + src_uv += twidth; + dst_rgb565 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_NV21TORGB565ROW_AVX2) +void NV21ToRGB565Row_AVX2(const uint8* src_y, const uint8* src_vu, + uint8* dst_rgb565, int width) { + // Row buffer for intermediate ARGB pixels. + SIMD_ALIGNED32(uint8 row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + NV21ToARGBRow_AVX2(src_y, src_vu, row, twidth); + ARGBToRGB565Row_AVX2(row, dst_rgb565, twidth); + src_y += twidth; + src_vu += twidth; + dst_rgb565 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_YUY2TOARGBROW_AVX2) +void YUY2ToARGBRow_AVX2(const uint8* src_yuy2, uint8* dst_argb, int width) { + // Row buffers for intermediate YUV pixels. + SIMD_ALIGNED32(uint8 row_y[MAXTWIDTH]); + SIMD_ALIGNED32(uint8 row_u[MAXTWIDTH / 2]); + SIMD_ALIGNED32(uint8 row_v[MAXTWIDTH / 2]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + YUY2ToUV422Row_AVX2(src_yuy2, row_u, row_v, twidth); + YUY2ToYRow_AVX2(src_yuy2, row_y, twidth); + I422ToARGBRow_AVX2(row_y, row_u, row_v, dst_argb, twidth); + src_yuy2 += twidth * 2; + dst_argb += twidth * 4; + width -= twidth; + } +} +#endif + +#if defined(HAS_UYVYTOARGBROW_AVX2) +void UYVYToARGBRow_AVX2(const uint8* src_uyvy, uint8* dst_argb, int width) { + // Row buffers for intermediate YUV pixels. + SIMD_ALIGNED32(uint8 row_y[MAXTWIDTH]); + SIMD_ALIGNED32(uint8 row_u[MAXTWIDTH / 2]); + SIMD_ALIGNED32(uint8 row_v[MAXTWIDTH / 2]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + UYVYToUV422Row_AVX2(src_uyvy, row_u, row_v, twidth); + UYVYToYRow_AVX2(src_uyvy, row_y, twidth); + I422ToARGBRow_AVX2(row_y, row_u, row_v, dst_argb, twidth); + src_uyvy += twidth * 2; + dst_argb += twidth * 4; + width -= twidth; + } +} +#endif // !defined(LIBYUV_DISABLE_X86) + +void ARGBPolynomialRow_C(const uint8* src_argb, + uint8* dst_argb, const float* poly, + int width) { + int i; + for (i = 0; i < width; ++i) { + float b = (float)(src_argb[0]); + float g = (float)(src_argb[1]); + float r = (float)(src_argb[2]); + float a = (float)(src_argb[3]); + float b2 = b * b; + float g2 = g * g; + float r2 = r * r; + float a2 = a * a; + float db = poly[0] + poly[4] * b; + float dg = poly[1] + poly[5] * g; + float dr = poly[2] + poly[6] * r; + float da = poly[3] + poly[7] * a; + float b3 = b2 * b; + float g3 = g2 * g; + float r3 = r2 * r; + float a3 = a2 * a; + db += poly[8] * b2; + dg += poly[9] * g2; + dr += poly[10] * r2; + da += poly[11] * a2; + db += poly[12] * b3; + dg += poly[13] * g3; + dr += poly[14] * r3; + da += poly[15] * a3; + + dst_argb[0] = Clamp((int32)(db)); + dst_argb[1] = Clamp((int32)(dg)); + dst_argb[2] = Clamp((int32)(dr)); + dst_argb[3] = Clamp((int32)(da)); + src_argb += 4; + dst_argb += 4; + } +} + +void ARGBLumaColorTableRow_C(const uint8* src_argb, uint8* dst_argb, int width, + const uint8* luma, uint32 lumacoeff) { + uint32 bc = lumacoeff & 0xff; + uint32 gc = (lumacoeff >> 8) & 0xff; + uint32 rc = (lumacoeff >> 16) & 0xff; + + int i; + for (i = 0; i < width - 1; i += 2) { + // Luminance in rows, color values in columns. + const uint8* luma0 = ((src_argb[0] * bc + src_argb[1] * gc + + src_argb[2] * rc) & 0x7F00u) + luma; + const uint8* luma1; + dst_argb[0] = luma0[src_argb[0]]; + dst_argb[1] = luma0[src_argb[1]]; + dst_argb[2] = luma0[src_argb[2]]; + dst_argb[3] = src_argb[3]; + luma1 = ((src_argb[4] * bc + src_argb[5] * gc + + src_argb[6] * rc) & 0x7F00u) + luma; + dst_argb[4] = luma1[src_argb[4]]; + dst_argb[5] = luma1[src_argb[5]]; + dst_argb[6] = luma1[src_argb[6]]; + dst_argb[7] = src_argb[7]; + src_argb += 8; + dst_argb += 8; + } + if (width & 1) { + // Luminance in rows, color values in columns. + const uint8* luma0 = ((src_argb[0] * bc + src_argb[1] * gc + + src_argb[2] * rc) & 0x7F00u) + luma; + dst_argb[0] = luma0[src_argb[0]]; + dst_argb[1] = luma0[src_argb[1]]; + dst_argb[2] = luma0[src_argb[2]]; + dst_argb[3] = src_argb[3]; + } +} + +void ARGBCopyAlphaRow_C(const uint8* src, uint8* dst, int width) { + int i; + for (i = 0; i < width - 1; i += 2) { + dst[3] = src[3]; + dst[7] = src[7]; + dst += 8; + src += 8; + } + if (width & 1) { + dst[3] = src[3]; + } +} + +void ARGBCopyYToAlphaRow_C(const uint8* src, uint8* dst, int width) { + int i; + for (i = 0; i < width - 1; i += 2) { + dst[3] = src[0]; + dst[7] = src[1]; + dst += 8; + src += 2; + } + if (width & 1) { + dst[3] = src[0]; + } +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/row_gcc.cc b/third_party/aom/third_party/libyuv/source/row_gcc.cc new file mode 100644 index 0000000000..820de0a1c6 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/row_gcc.cc @@ -0,0 +1,5475 @@ +// VERSION 2 +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for GCC x86 and x64. +#if !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__)) + +#if defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_ARGBGRAYROW_SSSE3) + +// Constants for ARGB +static vec8 kARGBToY = { + 13, 65, 33, 0, 13, 65, 33, 0, 13, 65, 33, 0, 13, 65, 33, 0 +}; + +// JPeg full range. +static vec8 kARGBToYJ = { + 15, 75, 38, 0, 15, 75, 38, 0, 15, 75, 38, 0, 15, 75, 38, 0 +}; +#endif // defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_ARGBGRAYROW_SSSE3) + +#if defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_I422TOARGBROW_SSSE3) + +static vec8 kARGBToU = { + 112, -74, -38, 0, 112, -74, -38, 0, 112, -74, -38, 0, 112, -74, -38, 0 +}; + +static vec8 kARGBToUJ = { + 127, -84, -43, 0, 127, -84, -43, 0, 127, -84, -43, 0, 127, -84, -43, 0 +}; + +static vec8 kARGBToV = { + -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0, +}; + +static vec8 kARGBToVJ = { + -20, -107, 127, 0, -20, -107, 127, 0, -20, -107, 127, 0, -20, -107, 127, 0 +}; + +// Constants for BGRA +static vec8 kBGRAToY = { + 0, 33, 65, 13, 0, 33, 65, 13, 0, 33, 65, 13, 0, 33, 65, 13 +}; + +static vec8 kBGRAToU = { + 0, -38, -74, 112, 0, -38, -74, 112, 0, -38, -74, 112, 0, -38, -74, 112 +}; + +static vec8 kBGRAToV = { + 0, 112, -94, -18, 0, 112, -94, -18, 0, 112, -94, -18, 0, 112, -94, -18 +}; + +// Constants for ABGR +static vec8 kABGRToY = { + 33, 65, 13, 0, 33, 65, 13, 0, 33, 65, 13, 0, 33, 65, 13, 0 +}; + +static vec8 kABGRToU = { + -38, -74, 112, 0, -38, -74, 112, 0, -38, -74, 112, 0, -38, -74, 112, 0 +}; + +static vec8 kABGRToV = { + 112, -94, -18, 0, 112, -94, -18, 0, 112, -94, -18, 0, 112, -94, -18, 0 +}; + +// Constants for RGBA. +static vec8 kRGBAToY = { + 0, 13, 65, 33, 0, 13, 65, 33, 0, 13, 65, 33, 0, 13, 65, 33 +}; + +static vec8 kRGBAToU = { + 0, 112, -74, -38, 0, 112, -74, -38, 0, 112, -74, -38, 0, 112, -74, -38 +}; + +static vec8 kRGBAToV = { + 0, -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112 +}; + +static uvec8 kAddY16 = { + 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u +}; + +// 7 bit fixed point 0.5. +static vec16 kAddYJ64 = { + 64, 64, 64, 64, 64, 64, 64, 64 +}; + +static uvec8 kAddUV128 = { + 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, + 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u +}; + +static uvec16 kAddUVJ128 = { + 0x8080u, 0x8080u, 0x8080u, 0x8080u, 0x8080u, 0x8080u, 0x8080u, 0x8080u +}; +#endif // defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_I422TOARGBROW_SSSE3) + +#ifdef HAS_RGB24TOARGBROW_SSSE3 + +// Shuffle table for converting RGB24 to ARGB. +static uvec8 kShuffleMaskRGB24ToARGB = { + 0u, 1u, 2u, 12u, 3u, 4u, 5u, 13u, 6u, 7u, 8u, 14u, 9u, 10u, 11u, 15u +}; + +// Shuffle table for converting RAW to ARGB. +static uvec8 kShuffleMaskRAWToARGB = { + 2u, 1u, 0u, 12u, 5u, 4u, 3u, 13u, 8u, 7u, 6u, 14u, 11u, 10u, 9u, 15u +}; + +// Shuffle table for converting ARGB to RGB24. +static uvec8 kShuffleMaskARGBToRGB24 = { + 0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u, 128u +}; + +// Shuffle table for converting ARGB to RAW. +static uvec8 kShuffleMaskARGBToRAW = { + 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u, 128u +}; + +// Shuffle table for converting ARGBToRGB24 for I422ToRGB24. First 8 + next 4 +static uvec8 kShuffleMaskARGBToRGB24_0 = { + 0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u, 14u +}; + +// Shuffle table for converting ARGB to RAW. +static uvec8 kShuffleMaskARGBToRAW_0 = { + 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 128u, 128u, 128u, 128u, 8u, 14u, 13u, 12u +}; +#endif // HAS_RGB24TOARGBROW_SSSE3 + +#if defined(TESTING) && defined(__x86_64__) +void TestRow_SSE2(const uint8* src_y, uint8* dst_argb, int pix) { + asm volatile ( + ".p2align 5 \n" + "mov %%eax,%%eax \n" + "mov %%ebx,%%ebx \n" + "mov %%ecx,%%ecx \n" + "mov %%edx,%%edx \n" + "mov %%esi,%%esi \n" + "mov %%edi,%%edi \n" + "mov %%ebp,%%ebp \n" + "mov %%esp,%%esp \n" + ".p2align 5 \n" + "mov %%r8d,%%r8d \n" + "mov %%r9d,%%r9d \n" + "mov %%r10d,%%r10d \n" + "mov %%r11d,%%r11d \n" + "mov %%r12d,%%r12d \n" + "mov %%r13d,%%r13d \n" + "mov %%r14d,%%r14d \n" + "mov %%r15d,%%r15d \n" + ".p2align 5 \n" + "lea (%%rax),%%eax \n" + "lea (%%rbx),%%ebx \n" + "lea (%%rcx),%%ecx \n" + "lea (%%rdx),%%edx \n" + "lea (%%rsi),%%esi \n" + "lea (%%rdi),%%edi \n" + "lea (%%rbp),%%ebp \n" + "lea (%%rsp),%%esp \n" + ".p2align 5 \n" + "lea (%%r8),%%r8d \n" + "lea (%%r9),%%r9d \n" + "lea (%%r10),%%r10d \n" + "lea (%%r11),%%r11d \n" + "lea (%%r12),%%r12d \n" + "lea (%%r13),%%r13d \n" + "lea (%%r14),%%r14d \n" + "lea (%%r15),%%r15d \n" + + ".p2align 5 \n" + "lea 0x10(%%rax),%%eax \n" + "lea 0x10(%%rbx),%%ebx \n" + "lea 0x10(%%rcx),%%ecx \n" + "lea 0x10(%%rdx),%%edx \n" + "lea 0x10(%%rsi),%%esi \n" + "lea 0x10(%%rdi),%%edi \n" + "lea 0x10(%%rbp),%%ebp \n" + "lea 0x10(%%rsp),%%esp \n" + ".p2align 5 \n" + "lea 0x10(%%r8),%%r8d \n" + "lea 0x10(%%r9),%%r9d \n" + "lea 0x10(%%r10),%%r10d \n" + "lea 0x10(%%r11),%%r11d \n" + "lea 0x10(%%r12),%%r12d \n" + "lea 0x10(%%r13),%%r13d \n" + "lea 0x10(%%r14),%%r14d \n" + "lea 0x10(%%r15),%%r15d \n" + + ".p2align 5 \n" + "add 0x10,%%eax \n" + "add 0x10,%%ebx \n" + "add 0x10,%%ecx \n" + "add 0x10,%%edx \n" + "add 0x10,%%esi \n" + "add 0x10,%%edi \n" + "add 0x10,%%ebp \n" + "add 0x10,%%esp \n" + ".p2align 5 \n" + "add 0x10,%%r8d \n" + "add 0x10,%%r9d \n" + "add 0x10,%%r10d \n" + "add 0x10,%%r11d \n" + "add 0x10,%%r12d \n" + "add 0x10,%%r13d \n" + "add 0x10,%%r14d \n" + "add 0x10,%%r15d \n" + + ".p2align 2 \n" + "1: \n" + "movq " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x8,0) ",%0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(src_y), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : + : "memory", "cc", "xmm0", "xmm1", "xmm5" + ); +} +#endif // TESTING + +#ifdef HAS_J400TOARGBROW_SSE2 +void J400ToARGBRow_SSE2(const uint8* src_y, uint8* dst_argb, int pix) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" + "pslld $0x18,%%xmm5 \n" + LABELALIGN + "1: \n" + "movq " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x8,0) ",%0 \n" + "punpcklbw %%xmm0,%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklwd %%xmm0,%%xmm0 \n" + "punpckhwd %%xmm1,%%xmm1 \n" + "por %%xmm5,%%xmm0 \n" + "por %%xmm5,%%xmm1 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(src_y), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + :: "memory", "cc", "xmm0", "xmm1", "xmm5" + ); +} +#endif // HAS_J400TOARGBROW_SSE2 + +#ifdef HAS_RGB24TOARGBROW_SSSE3 +void RGB24ToARGBRow_SSSE3(const uint8* src_rgb24, uint8* dst_argb, int pix) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" // generate mask 0xff000000 + "pslld $0x18,%%xmm5 \n" + "movdqa %3,%%xmm4 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm3 \n" + "lea " MEMLEA(0x30,0) ",%0 \n" + "movdqa %%xmm3,%%xmm2 \n" + "palignr $0x8,%%xmm1,%%xmm2 \n" + "pshufb %%xmm4,%%xmm2 \n" + "por %%xmm5,%%xmm2 \n" + "palignr $0xc,%%xmm0,%%xmm1 \n" + "pshufb %%xmm4,%%xmm0 \n" + "movdqu %%xmm2," MEMACCESS2(0x20,1) " \n" + "por %%xmm5,%%xmm0 \n" + "pshufb %%xmm4,%%xmm1 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "por %%xmm5,%%xmm1 \n" + "palignr $0x4,%%xmm3,%%xmm3 \n" + "pshufb %%xmm4,%%xmm3 \n" + "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" + "por %%xmm5,%%xmm3 \n" + "movdqu %%xmm3," MEMACCESS2(0x30,1) " \n" + "lea " MEMLEA(0x40,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_rgb24), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : "m"(kShuffleMaskRGB24ToARGB) // %3 + : "memory", "cc" , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} + +void RAWToARGBRow_SSSE3(const uint8* src_raw, uint8* dst_argb, int pix) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" // generate mask 0xff000000 + "pslld $0x18,%%xmm5 \n" + "movdqa %3,%%xmm4 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm3 \n" + "lea " MEMLEA(0x30,0) ",%0 \n" + "movdqa %%xmm3,%%xmm2 \n" + "palignr $0x8,%%xmm1,%%xmm2 \n" + "pshufb %%xmm4,%%xmm2 \n" + "por %%xmm5,%%xmm2 \n" + "palignr $0xc,%%xmm0,%%xmm1 \n" + "pshufb %%xmm4,%%xmm0 \n" + "movdqu %%xmm2," MEMACCESS2(0x20,1) " \n" + "por %%xmm5,%%xmm0 \n" + "pshufb %%xmm4,%%xmm1 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "por %%xmm5,%%xmm1 \n" + "palignr $0x4,%%xmm3,%%xmm3 \n" + "pshufb %%xmm4,%%xmm3 \n" + "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" + "por %%xmm5,%%xmm3 \n" + "movdqu %%xmm3," MEMACCESS2(0x30,1) " \n" + "lea " MEMLEA(0x40,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_raw), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : "m"(kShuffleMaskRAWToARGB) // %3 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} + +void RGB565ToARGBRow_SSE2(const uint8* src, uint8* dst, int pix) { + asm volatile ( + "mov $0x1080108,%%eax \n" + "movd %%eax,%%xmm5 \n" + "pshufd $0x0,%%xmm5,%%xmm5 \n" + "mov $0x20802080,%%eax \n" + "movd %%eax,%%xmm6 \n" + "pshufd $0x0,%%xmm6,%%xmm6 \n" + "pcmpeqb %%xmm3,%%xmm3 \n" + "psllw $0xb,%%xmm3 \n" + "pcmpeqb %%xmm4,%%xmm4 \n" + "psllw $0xa,%%xmm4 \n" + "psrlw $0x5,%%xmm4 \n" + "pcmpeqb %%xmm7,%%xmm7 \n" + "psllw $0x8,%%xmm7 \n" + "sub %0,%1 \n" + "sub %0,%1 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm0,%%xmm2 \n" + "pand %%xmm3,%%xmm1 \n" + "psllw $0xb,%%xmm2 \n" + "pmulhuw %%xmm5,%%xmm1 \n" + "pmulhuw %%xmm5,%%xmm2 \n" + "psllw $0x8,%%xmm1 \n" + "por %%xmm2,%%xmm1 \n" + "pand %%xmm4,%%xmm0 \n" + "pmulhuw %%xmm6,%%xmm0 \n" + "por %%xmm7,%%xmm0 \n" + "movdqa %%xmm1,%%xmm2 \n" + "punpcklbw %%xmm0,%%xmm1 \n" + "punpckhbw %%xmm0,%%xmm2 \n" + MEMOPMEM(movdqu,xmm1,0x00,1,0,2) // movdqu %%xmm1,(%1,%0,2) + MEMOPMEM(movdqu,xmm2,0x10,1,0,2) // movdqu %%xmm2,0x10(%1,%0,2) + "lea " MEMLEA(0x10,0) ",%0 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(pix) // %2 + : + : "memory", "cc", "eax", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} + +void ARGB1555ToARGBRow_SSE2(const uint8* src, uint8* dst, int pix) { + asm volatile ( + "mov $0x1080108,%%eax \n" + "movd %%eax,%%xmm5 \n" + "pshufd $0x0,%%xmm5,%%xmm5 \n" + "mov $0x42004200,%%eax \n" + "movd %%eax,%%xmm6 \n" + "pshufd $0x0,%%xmm6,%%xmm6 \n" + "pcmpeqb %%xmm3,%%xmm3 \n" + "psllw $0xb,%%xmm3 \n" + "movdqa %%xmm3,%%xmm4 \n" + "psrlw $0x6,%%xmm4 \n" + "pcmpeqb %%xmm7,%%xmm7 \n" + "psllw $0x8,%%xmm7 \n" + "sub %0,%1 \n" + "sub %0,%1 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm0,%%xmm2 \n" + "psllw $0x1,%%xmm1 \n" + "psllw $0xb,%%xmm2 \n" + "pand %%xmm3,%%xmm1 \n" + "pmulhuw %%xmm5,%%xmm2 \n" + "pmulhuw %%xmm5,%%xmm1 \n" + "psllw $0x8,%%xmm1 \n" + "por %%xmm2,%%xmm1 \n" + "movdqa %%xmm0,%%xmm2 \n" + "pand %%xmm4,%%xmm0 \n" + "psraw $0x8,%%xmm2 \n" + "pmulhuw %%xmm6,%%xmm0 \n" + "pand %%xmm7,%%xmm2 \n" + "por %%xmm2,%%xmm0 \n" + "movdqa %%xmm1,%%xmm2 \n" + "punpcklbw %%xmm0,%%xmm1 \n" + "punpckhbw %%xmm0,%%xmm2 \n" + MEMOPMEM(movdqu,xmm1,0x00,1,0,2) // movdqu %%xmm1,(%1,%0,2) + MEMOPMEM(movdqu,xmm2,0x10,1,0,2) // movdqu %%xmm2,0x10(%1,%0,2) + "lea " MEMLEA(0x10,0) ",%0 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(pix) // %2 + : + : "memory", "cc", "eax", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} + +void ARGB4444ToARGBRow_SSE2(const uint8* src, uint8* dst, int pix) { + asm volatile ( + "mov $0xf0f0f0f,%%eax \n" + "movd %%eax,%%xmm4 \n" + "pshufd $0x0,%%xmm4,%%xmm4 \n" + "movdqa %%xmm4,%%xmm5 \n" + "pslld $0x4,%%xmm5 \n" + "sub %0,%1 \n" + "sub %0,%1 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqa %%xmm0,%%xmm2 \n" + "pand %%xmm4,%%xmm0 \n" + "pand %%xmm5,%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm2,%%xmm3 \n" + "psllw $0x4,%%xmm1 \n" + "psrlw $0x4,%%xmm3 \n" + "por %%xmm1,%%xmm0 \n" + "por %%xmm3,%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklbw %%xmm2,%%xmm0 \n" + "punpckhbw %%xmm2,%%xmm1 \n" + MEMOPMEM(movdqu,xmm0,0x00,1,0,2) // movdqu %%xmm0,(%1,%0,2) + MEMOPMEM(movdqu,xmm1,0x10,1,0,2) // movdqu %%xmm1,0x10(%1,%0,2) + "lea " MEMLEA(0x10,0) ",%0 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(pix) // %2 + : + : "memory", "cc", "eax", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} + +void ARGBToRGB24Row_SSSE3(const uint8* src, uint8* dst, int pix) { + asm volatile ( + "movdqa %3,%%xmm6 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "pshufb %%xmm6,%%xmm0 \n" + "pshufb %%xmm6,%%xmm1 \n" + "pshufb %%xmm6,%%xmm2 \n" + "pshufb %%xmm6,%%xmm3 \n" + "movdqa %%xmm1,%%xmm4 \n" + "psrldq $0x4,%%xmm1 \n" + "pslldq $0xc,%%xmm4 \n" + "movdqa %%xmm2,%%xmm5 \n" + "por %%xmm4,%%xmm0 \n" + "pslldq $0x8,%%xmm5 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "por %%xmm5,%%xmm1 \n" + "psrldq $0x8,%%xmm2 \n" + "pslldq $0x4,%%xmm3 \n" + "por %%xmm3,%%xmm2 \n" + "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" + "movdqu %%xmm2," MEMACCESS2(0x20,1) " \n" + "lea " MEMLEA(0x30,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(pix) // %2 + : "m"(kShuffleMaskARGBToRGB24) // %3 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" + ); +} + +void ARGBToRAWRow_SSSE3(const uint8* src, uint8* dst, int pix) { + asm volatile ( + "movdqa %3,%%xmm6 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "pshufb %%xmm6,%%xmm0 \n" + "pshufb %%xmm6,%%xmm1 \n" + "pshufb %%xmm6,%%xmm2 \n" + "pshufb %%xmm6,%%xmm3 \n" + "movdqa %%xmm1,%%xmm4 \n" + "psrldq $0x4,%%xmm1 \n" + "pslldq $0xc,%%xmm4 \n" + "movdqa %%xmm2,%%xmm5 \n" + "por %%xmm4,%%xmm0 \n" + "pslldq $0x8,%%xmm5 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "por %%xmm5,%%xmm1 \n" + "psrldq $0x8,%%xmm2 \n" + "pslldq $0x4,%%xmm3 \n" + "por %%xmm3,%%xmm2 \n" + "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" + "movdqu %%xmm2," MEMACCESS2(0x20,1) " \n" + "lea " MEMLEA(0x30,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(pix) // %2 + : "m"(kShuffleMaskARGBToRAW) // %3 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" + ); +} + +void ARGBToRGB565Row_SSE2(const uint8* src, uint8* dst, int pix) { + asm volatile ( + "pcmpeqb %%xmm3,%%xmm3 \n" + "psrld $0x1b,%%xmm3 \n" + "pcmpeqb %%xmm4,%%xmm4 \n" + "psrld $0x1a,%%xmm4 \n" + "pslld $0x5,%%xmm4 \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + "pslld $0xb,%%xmm5 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm0,%%xmm2 \n" + "pslld $0x8,%%xmm0 \n" + "psrld $0x3,%%xmm1 \n" + "psrld $0x5,%%xmm2 \n" + "psrad $0x10,%%xmm0 \n" + "pand %%xmm3,%%xmm1 \n" + "pand %%xmm4,%%xmm2 \n" + "pand %%xmm5,%%xmm0 \n" + "por %%xmm2,%%xmm1 \n" + "por %%xmm1,%%xmm0 \n" + "packssdw %%xmm0,%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movq %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x4,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(pix) // %2 + :: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} + +void ARGBToARGB1555Row_SSE2(const uint8* src, uint8* dst, int pix) { + asm volatile ( + "pcmpeqb %%xmm4,%%xmm4 \n" + "psrld $0x1b,%%xmm4 \n" + "movdqa %%xmm4,%%xmm5 \n" + "pslld $0x5,%%xmm5 \n" + "movdqa %%xmm4,%%xmm6 \n" + "pslld $0xa,%%xmm6 \n" + "pcmpeqb %%xmm7,%%xmm7 \n" + "pslld $0xf,%%xmm7 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm0,%%xmm2 \n" + "movdqa %%xmm0,%%xmm3 \n" + "psrad $0x10,%%xmm0 \n" + "psrld $0x3,%%xmm1 \n" + "psrld $0x6,%%xmm2 \n" + "psrld $0x9,%%xmm3 \n" + "pand %%xmm7,%%xmm0 \n" + "pand %%xmm4,%%xmm1 \n" + "pand %%xmm5,%%xmm2 \n" + "pand %%xmm6,%%xmm3 \n" + "por %%xmm1,%%xmm0 \n" + "por %%xmm3,%%xmm2 \n" + "por %%xmm2,%%xmm0 \n" + "packssdw %%xmm0,%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movq %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x4,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(pix) // %2 + :: "memory", "cc", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} + +void ARGBToARGB4444Row_SSE2(const uint8* src, uint8* dst, int pix) { + asm volatile ( + "pcmpeqb %%xmm4,%%xmm4 \n" + "psllw $0xc,%%xmm4 \n" + "movdqa %%xmm4,%%xmm3 \n" + "psrlw $0x8,%%xmm3 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "pand %%xmm3,%%xmm0 \n" + "pand %%xmm4,%%xmm1 \n" + "psrlq $0x4,%%xmm0 \n" + "psrlq $0x8,%%xmm1 \n" + "por %%xmm1,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movq %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x4,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(pix) // %2 + :: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4" + ); +} +#endif // HAS_RGB24TOARGBROW_SSSE3 + +#ifdef HAS_ARGBTOYROW_SSSE3 +// Convert 16 ARGB pixels (64 bytes) to 16 Y values. +void ARGBToYRow_SSSE3(const uint8* src_argb, uint8* dst_y, int pix) { + asm volatile ( + "movdqa %3,%%xmm4 \n" + "movdqa %4,%%xmm5 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm1 \n" + "pmaddubsw %%xmm4,%%xmm2 \n" + "pmaddubsw %%xmm4,%%xmm3 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "phaddw %%xmm1,%%xmm0 \n" + "phaddw %%xmm3,%%xmm2 \n" + "psrlw $0x7,%%xmm0 \n" + "psrlw $0x7,%%xmm2 \n" + "packuswb %%xmm2,%%xmm0 \n" + "paddb %%xmm5,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : "m"(kARGBToY), // %3 + "m"(kAddY16) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} +#endif // HAS_ARGBTOYROW_SSSE3 + +#ifdef HAS_ARGBTOYJROW_SSSE3 +// Convert 16 ARGB pixels (64 bytes) to 16 YJ values. +// Same as ARGBToYRow but different coefficients, no add 16, but do rounding. +void ARGBToYJRow_SSSE3(const uint8* src_argb, uint8* dst_y, int pix) { + asm volatile ( + "movdqa %3,%%xmm4 \n" + "movdqa %4,%%xmm5 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm1 \n" + "pmaddubsw %%xmm4,%%xmm2 \n" + "pmaddubsw %%xmm4,%%xmm3 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "phaddw %%xmm1,%%xmm0 \n" + "phaddw %%xmm3,%%xmm2 \n" + "paddw %%xmm5,%%xmm0 \n" + "paddw %%xmm5,%%xmm2 \n" + "psrlw $0x7,%%xmm0 \n" + "psrlw $0x7,%%xmm2 \n" + "packuswb %%xmm2,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : "m"(kARGBToYJ), // %3 + "m"(kAddYJ64) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} +#endif // HAS_ARGBTOYJROW_SSSE3 + +#ifdef HAS_ARGBTOYROW_AVX2 +// vpermd for vphaddw + vpackuswb vpermd. +static const lvec32 kPermdARGBToY_AVX = { + 0, 4, 1, 5, 2, 6, 3, 7 +}; + +// Convert 32 ARGB pixels (128 bytes) to 32 Y values. +void ARGBToYRow_AVX2(const uint8* src_argb, uint8* dst_y, int pix) { + asm volatile ( + "vbroadcastf128 %3,%%ymm4 \n" + "vbroadcastf128 %4,%%ymm5 \n" + "vmovdqu %5,%%ymm6 \n" + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" + "vmovdqu " MEMACCESS2(0x40,0) ",%%ymm2 \n" + "vmovdqu " MEMACCESS2(0x60,0) ",%%ymm3 \n" + "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n" + "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n" + "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n" + "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n" + "lea " MEMLEA(0x80,0) ",%0 \n" + "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // mutates. + "vphaddw %%ymm3,%%ymm2,%%ymm2 \n" + "vpsrlw $0x7,%%ymm0,%%ymm0 \n" + "vpsrlw $0x7,%%ymm2,%%ymm2 \n" + "vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" // mutates. + "vpermd %%ymm0,%%ymm6,%%ymm0 \n" // unmutate. + "vpaddb %%ymm5,%%ymm0,%%ymm0 \n" // add 16 for Y + "vmovdqu %%ymm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : "m"(kARGBToY), // %3 + "m"(kAddY16), // %4 + "m"(kPermdARGBToY_AVX) // %5 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" + ); +} +#endif // HAS_ARGBTOYROW_AVX2 + +#ifdef HAS_ARGBTOYJROW_AVX2 +// Convert 32 ARGB pixels (128 bytes) to 32 Y values. +void ARGBToYJRow_AVX2(const uint8* src_argb, uint8* dst_y, int pix) { + asm volatile ( + "vbroadcastf128 %3,%%ymm4 \n" + "vbroadcastf128 %4,%%ymm5 \n" + "vmovdqu %5,%%ymm6 \n" + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" + "vmovdqu " MEMACCESS2(0x40,0) ",%%ymm2 \n" + "vmovdqu " MEMACCESS2(0x60,0) ",%%ymm3 \n" + "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n" + "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n" + "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n" + "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n" + "lea " MEMLEA(0x80,0) ",%0 \n" + "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // mutates. + "vphaddw %%ymm3,%%ymm2,%%ymm2 \n" + "vpaddw %%ymm5,%%ymm0,%%ymm0 \n" // Add .5 for rounding. + "vpaddw %%ymm5,%%ymm2,%%ymm2 \n" + "vpsrlw $0x7,%%ymm0,%%ymm0 \n" + "vpsrlw $0x7,%%ymm2,%%ymm2 \n" + "vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" // mutates. + "vpermd %%ymm0,%%ymm6,%%ymm0 \n" // unmutate. + "vmovdqu %%ymm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : "m"(kARGBToYJ), // %3 + "m"(kAddYJ64), // %4 + "m"(kPermdARGBToY_AVX) // %5 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" + ); +} +#endif // HAS_ARGBTOYJROW_AVX2 + +#ifdef HAS_ARGBTOUVROW_SSSE3 +void ARGBToUVRow_SSSE3(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) { + asm volatile ( + "movdqa %5,%%xmm3 \n" + "movdqa %6,%%xmm4 \n" + "movdqa %7,%%xmm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,0,4,1,xmm7) // movdqu (%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + MEMOPREG(movdqu,0x10,0,4,1,xmm7) // movdqu 0x10(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + MEMOPREG(movdqu,0x20,0,4,1,xmm7) // movdqu 0x20(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" + MEMOPREG(movdqu,0x30,0,4,1,xmm7) // movdqu 0x30(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm6 \n" + + "lea " MEMLEA(0x40,0) ",%0 \n" + "movdqa %%xmm0,%%xmm7 \n" + "shufps $0x88,%%xmm1,%%xmm0 \n" + "shufps $0xdd,%%xmm1,%%xmm7 \n" + "pavgb %%xmm7,%%xmm0 \n" + "movdqa %%xmm2,%%xmm7 \n" + "shufps $0x88,%%xmm6,%%xmm2 \n" + "shufps $0xdd,%%xmm6,%%xmm7 \n" + "pavgb %%xmm7,%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm2,%%xmm6 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm2 \n" + "pmaddubsw %%xmm3,%%xmm1 \n" + "pmaddubsw %%xmm3,%%xmm6 \n" + "phaddw %%xmm2,%%xmm0 \n" + "phaddw %%xmm6,%%xmm1 \n" + "psraw $0x8,%%xmm0 \n" + "psraw $0x8,%%xmm1 \n" + "packsswb %%xmm1,%%xmm0 \n" + "paddb %%xmm5,%%xmm0 \n" + "movlps %%xmm0," MEMACCESS(1) " \n" + MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_argb0), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+rm"(width) // %3 + : "r"((intptr_t)(src_stride_argb)), // %4 + "m"(kARGBToV), // %5 + "m"(kARGBToU), // %6 + "m"(kAddUV128) // %7 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" + ); +} +#endif // HAS_ARGBTOUVROW_SSSE3 + +#ifdef HAS_ARGBTOUVROW_AVX2 +// vpshufb for vphaddw + vpackuswb packed to shorts. +static const lvec8 kShufARGBToUV_AVX = { + 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15, + 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +}; +void ARGBToUVRow_AVX2(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) { + asm volatile ( + "vbroadcastf128 %5,%%ymm5 \n" + "vbroadcastf128 %6,%%ymm6 \n" + "vbroadcastf128 %7,%%ymm7 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" + "vmovdqu " MEMACCESS2(0x40,0) ",%%ymm2 \n" + "vmovdqu " MEMACCESS2(0x60,0) ",%%ymm3 \n" + VMEMOPREG(vpavgb,0x00,0,4,1,ymm0,ymm0) // vpavgb (%0,%4,1),%%ymm0,%%ymm0 + VMEMOPREG(vpavgb,0x20,0,4,1,ymm1,ymm1) + VMEMOPREG(vpavgb,0x40,0,4,1,ymm2,ymm2) + VMEMOPREG(vpavgb,0x60,0,4,1,ymm3,ymm3) + "lea " MEMLEA(0x80,0) ",%0 \n" + "vshufps $0x88,%%ymm1,%%ymm0,%%ymm4 \n" + "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm4,%%ymm0,%%ymm0 \n" + "vshufps $0x88,%%ymm3,%%ymm2,%%ymm4 \n" + "vshufps $0xdd,%%ymm3,%%ymm2,%%ymm2 \n" + "vpavgb %%ymm4,%%ymm2,%%ymm2 \n" + + "vpmaddubsw %%ymm7,%%ymm0,%%ymm1 \n" + "vpmaddubsw %%ymm7,%%ymm2,%%ymm3 \n" + "vpmaddubsw %%ymm6,%%ymm0,%%ymm0 \n" + "vpmaddubsw %%ymm6,%%ymm2,%%ymm2 \n" + "vphaddw %%ymm3,%%ymm1,%%ymm1 \n" + "vphaddw %%ymm2,%%ymm0,%%ymm0 \n" + "vpsraw $0x8,%%ymm1,%%ymm1 \n" + "vpsraw $0x8,%%ymm0,%%ymm0 \n" + "vpacksswb %%ymm0,%%ymm1,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpshufb %8,%%ymm0,%%ymm0 \n" + "vpaddb %%ymm5,%%ymm0,%%ymm0 \n" + + "vextractf128 $0x0,%%ymm0," MEMACCESS(1) " \n" + VEXTOPMEM(vextractf128,1,ymm0,0x0,1,2,1) // vextractf128 $1,%%ymm0,(%1,%2,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x20,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb0), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+rm"(width) // %3 + : "r"((intptr_t)(src_stride_argb)), // %4 + "m"(kAddUV128), // %5 + "m"(kARGBToV), // %6 + "m"(kARGBToU), // %7 + "m"(kShufARGBToUV_AVX) // %8 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} +#endif // HAS_ARGBTOUVROW_AVX2 + +#ifdef HAS_ARGBTOUVJROW_SSSE3 +void ARGBToUVJRow_SSSE3(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) { + asm volatile ( + "movdqa %5,%%xmm3 \n" + "movdqa %6,%%xmm4 \n" + "movdqa %7,%%xmm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,0,4,1,xmm7) // movdqu (%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + MEMOPREG(movdqu,0x10,0,4,1,xmm7) // movdqu 0x10(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + MEMOPREG(movdqu,0x20,0,4,1,xmm7) // movdqu 0x20(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" + MEMOPREG(movdqu,0x30,0,4,1,xmm7) // movdqu 0x30(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm6 \n" + + "lea " MEMLEA(0x40,0) ",%0 \n" + "movdqa %%xmm0,%%xmm7 \n" + "shufps $0x88,%%xmm1,%%xmm0 \n" + "shufps $0xdd,%%xmm1,%%xmm7 \n" + "pavgb %%xmm7,%%xmm0 \n" + "movdqa %%xmm2,%%xmm7 \n" + "shufps $0x88,%%xmm6,%%xmm2 \n" + "shufps $0xdd,%%xmm6,%%xmm7 \n" + "pavgb %%xmm7,%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm2,%%xmm6 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm2 \n" + "pmaddubsw %%xmm3,%%xmm1 \n" + "pmaddubsw %%xmm3,%%xmm6 \n" + "phaddw %%xmm2,%%xmm0 \n" + "phaddw %%xmm6,%%xmm1 \n" + "paddw %%xmm5,%%xmm0 \n" + "paddw %%xmm5,%%xmm1 \n" + "psraw $0x8,%%xmm0 \n" + "psraw $0x8,%%xmm1 \n" + "packsswb %%xmm1,%%xmm0 \n" + "movlps %%xmm0," MEMACCESS(1) " \n" + MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_argb0), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+rm"(width) // %3 + : "r"((intptr_t)(src_stride_argb)), // %4 + "m"(kARGBToVJ), // %5 + "m"(kARGBToUJ), // %6 + "m"(kAddUVJ128) // %7 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" + ); +} +#endif // HAS_ARGBTOUVJROW_SSSE3 + +#ifdef HAS_ARGBTOUV444ROW_SSSE3 +void ARGBToUV444Row_SSSE3(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int width) { + asm volatile ( + "movdqa %4,%%xmm3 \n" + "movdqa %5,%%xmm4 \n" + "movdqa %6,%%xmm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm1 \n" + "pmaddubsw %%xmm4,%%xmm2 \n" + "pmaddubsw %%xmm4,%%xmm6 \n" + "phaddw %%xmm1,%%xmm0 \n" + "phaddw %%xmm6,%%xmm2 \n" + "psraw $0x8,%%xmm0 \n" + "psraw $0x8,%%xmm2 \n" + "packsswb %%xmm2,%%xmm0 \n" + "paddb %%xmm5,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" + "pmaddubsw %%xmm3,%%xmm0 \n" + "pmaddubsw %%xmm3,%%xmm1 \n" + "pmaddubsw %%xmm3,%%xmm2 \n" + "pmaddubsw %%xmm3,%%xmm6 \n" + "phaddw %%xmm1,%%xmm0 \n" + "phaddw %%xmm6,%%xmm2 \n" + "psraw $0x8,%%xmm0 \n" + "psraw $0x8,%%xmm2 \n" + "packsswb %%xmm2,%%xmm0 \n" + "paddb %%xmm5,%%xmm0 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + MEMOPMEM(movdqu,xmm0,0x00,1,2,1) // movdqu %%xmm0,(%1,%2,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+rm"(width) // %3 + : "m"(kARGBToV), // %4 + "m"(kARGBToU), // %5 + "m"(kAddUV128) // %6 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm6" + ); +} +#endif // HAS_ARGBTOUV444ROW_SSSE3 + +#ifdef HAS_ARGBTOUV422ROW_SSSE3 +void ARGBToUV422Row_SSSE3(const uint8* src_argb0, + uint8* dst_u, uint8* dst_v, int width) { + asm volatile ( + "movdqa %4,%%xmm3 \n" + "movdqa %5,%%xmm4 \n" + "movdqa %6,%%xmm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "movdqa %%xmm0,%%xmm7 \n" + "shufps $0x88,%%xmm1,%%xmm0 \n" + "shufps $0xdd,%%xmm1,%%xmm7 \n" + "pavgb %%xmm7,%%xmm0 \n" + "movdqa %%xmm2,%%xmm7 \n" + "shufps $0x88,%%xmm6,%%xmm2 \n" + "shufps $0xdd,%%xmm6,%%xmm7 \n" + "pavgb %%xmm7,%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm2,%%xmm6 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm2 \n" + "pmaddubsw %%xmm3,%%xmm1 \n" + "pmaddubsw %%xmm3,%%xmm6 \n" + "phaddw %%xmm2,%%xmm0 \n" + "phaddw %%xmm6,%%xmm1 \n" + "psraw $0x8,%%xmm0 \n" + "psraw $0x8,%%xmm1 \n" + "packsswb %%xmm1,%%xmm0 \n" + "paddb %%xmm5,%%xmm0 \n" + "movlps %%xmm0," MEMACCESS(1) " \n" + MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_argb0), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+rm"(width) // %3 + : "m"(kARGBToV), // %4 + "m"(kARGBToU), // %5 + "m"(kAddUV128) // %6 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" + ); +} +#endif // HAS_ARGBTOUV422ROW_SSSE3 + +void BGRAToYRow_SSSE3(const uint8* src_bgra, uint8* dst_y, int pix) { + asm volatile ( + "movdqa %4,%%xmm5 \n" + "movdqa %3,%%xmm4 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm1 \n" + "pmaddubsw %%xmm4,%%xmm2 \n" + "pmaddubsw %%xmm4,%%xmm3 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "phaddw %%xmm1,%%xmm0 \n" + "phaddw %%xmm3,%%xmm2 \n" + "psrlw $0x7,%%xmm0 \n" + "psrlw $0x7,%%xmm2 \n" + "packuswb %%xmm2,%%xmm0 \n" + "paddb %%xmm5,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_bgra), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : "m"(kBGRAToY), // %3 + "m"(kAddY16) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} + +void BGRAToUVRow_SSSE3(const uint8* src_bgra0, int src_stride_bgra, + uint8* dst_u, uint8* dst_v, int width) { + asm volatile ( + "movdqa %5,%%xmm3 \n" + "movdqa %6,%%xmm4 \n" + "movdqa %7,%%xmm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,0,4,1,xmm7) // movdqu (%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + MEMOPREG(movdqu,0x10,0,4,1,xmm7) // movdqu 0x10(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + MEMOPREG(movdqu,0x20,0,4,1,xmm7) // movdqu 0x20(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" + MEMOPREG(movdqu,0x30,0,4,1,xmm7) // movdqu 0x30(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm6 \n" + + "lea " MEMLEA(0x40,0) ",%0 \n" + "movdqa %%xmm0,%%xmm7 \n" + "shufps $0x88,%%xmm1,%%xmm0 \n" + "shufps $0xdd,%%xmm1,%%xmm7 \n" + "pavgb %%xmm7,%%xmm0 \n" + "movdqa %%xmm2,%%xmm7 \n" + "shufps $0x88,%%xmm6,%%xmm2 \n" + "shufps $0xdd,%%xmm6,%%xmm7 \n" + "pavgb %%xmm7,%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm2,%%xmm6 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm2 \n" + "pmaddubsw %%xmm3,%%xmm1 \n" + "pmaddubsw %%xmm3,%%xmm6 \n" + "phaddw %%xmm2,%%xmm0 \n" + "phaddw %%xmm6,%%xmm1 \n" + "psraw $0x8,%%xmm0 \n" + "psraw $0x8,%%xmm1 \n" + "packsswb %%xmm1,%%xmm0 \n" + "paddb %%xmm5,%%xmm0 \n" + "movlps %%xmm0," MEMACCESS(1) " \n" + MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_bgra0), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+rm"(width) // %3 + : "r"((intptr_t)(src_stride_bgra)), // %4 + "m"(kBGRAToV), // %5 + "m"(kBGRAToU), // %6 + "m"(kAddUV128) // %7 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" + ); +} + +void ABGRToYRow_SSSE3(const uint8* src_abgr, uint8* dst_y, int pix) { + asm volatile ( + "movdqa %4,%%xmm5 \n" + "movdqa %3,%%xmm4 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm1 \n" + "pmaddubsw %%xmm4,%%xmm2 \n" + "pmaddubsw %%xmm4,%%xmm3 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "phaddw %%xmm1,%%xmm0 \n" + "phaddw %%xmm3,%%xmm2 \n" + "psrlw $0x7,%%xmm0 \n" + "psrlw $0x7,%%xmm2 \n" + "packuswb %%xmm2,%%xmm0 \n" + "paddb %%xmm5,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_abgr), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : "m"(kABGRToY), // %3 + "m"(kAddY16) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} + +void RGBAToYRow_SSSE3(const uint8* src_rgba, uint8* dst_y, int pix) { + asm volatile ( + "movdqa %4,%%xmm5 \n" + "movdqa %3,%%xmm4 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm1 \n" + "pmaddubsw %%xmm4,%%xmm2 \n" + "pmaddubsw %%xmm4,%%xmm3 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "phaddw %%xmm1,%%xmm0 \n" + "phaddw %%xmm3,%%xmm2 \n" + "psrlw $0x7,%%xmm0 \n" + "psrlw $0x7,%%xmm2 \n" + "packuswb %%xmm2,%%xmm0 \n" + "paddb %%xmm5,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_rgba), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : "m"(kRGBAToY), // %3 + "m"(kAddY16) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} + +void ABGRToUVRow_SSSE3(const uint8* src_abgr0, int src_stride_abgr, + uint8* dst_u, uint8* dst_v, int width) { + asm volatile ( + "movdqa %5,%%xmm3 \n" + "movdqa %6,%%xmm4 \n" + "movdqa %7,%%xmm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,0,4,1,xmm7) // movdqu (%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + MEMOPREG(movdqu,0x10,0,4,1,xmm7) // movdqu 0x10(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + MEMOPREG(movdqu,0x20,0,4,1,xmm7) // movdqu 0x20(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" + MEMOPREG(movdqu,0x30,0,4,1,xmm7) // movdqu 0x30(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm6 \n" + + "lea " MEMLEA(0x40,0) ",%0 \n" + "movdqa %%xmm0,%%xmm7 \n" + "shufps $0x88,%%xmm1,%%xmm0 \n" + "shufps $0xdd,%%xmm1,%%xmm7 \n" + "pavgb %%xmm7,%%xmm0 \n" + "movdqa %%xmm2,%%xmm7 \n" + "shufps $0x88,%%xmm6,%%xmm2 \n" + "shufps $0xdd,%%xmm6,%%xmm7 \n" + "pavgb %%xmm7,%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm2,%%xmm6 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm2 \n" + "pmaddubsw %%xmm3,%%xmm1 \n" + "pmaddubsw %%xmm3,%%xmm6 \n" + "phaddw %%xmm2,%%xmm0 \n" + "phaddw %%xmm6,%%xmm1 \n" + "psraw $0x8,%%xmm0 \n" + "psraw $0x8,%%xmm1 \n" + "packsswb %%xmm1,%%xmm0 \n" + "paddb %%xmm5,%%xmm0 \n" + "movlps %%xmm0," MEMACCESS(1) " \n" + MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_abgr0), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+rm"(width) // %3 + : "r"((intptr_t)(src_stride_abgr)), // %4 + "m"(kABGRToV), // %5 + "m"(kABGRToU), // %6 + "m"(kAddUV128) // %7 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" + ); +} + +void RGBAToUVRow_SSSE3(const uint8* src_rgba0, int src_stride_rgba, + uint8* dst_u, uint8* dst_v, int width) { + asm volatile ( + "movdqa %5,%%xmm3 \n" + "movdqa %6,%%xmm4 \n" + "movdqa %7,%%xmm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,0,4,1,xmm7) // movdqu (%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + MEMOPREG(movdqu,0x10,0,4,1,xmm7) // movdqu 0x10(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + MEMOPREG(movdqu,0x20,0,4,1,xmm7) // movdqu 0x20(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" + MEMOPREG(movdqu,0x30,0,4,1,xmm7) // movdqu 0x30(%0,%4,1),%%xmm7 + "pavgb %%xmm7,%%xmm6 \n" + + "lea " MEMLEA(0x40,0) ",%0 \n" + "movdqa %%xmm0,%%xmm7 \n" + "shufps $0x88,%%xmm1,%%xmm0 \n" + "shufps $0xdd,%%xmm1,%%xmm7 \n" + "pavgb %%xmm7,%%xmm0 \n" + "movdqa %%xmm2,%%xmm7 \n" + "shufps $0x88,%%xmm6,%%xmm2 \n" + "shufps $0xdd,%%xmm6,%%xmm7 \n" + "pavgb %%xmm7,%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm2,%%xmm6 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm2 \n" + "pmaddubsw %%xmm3,%%xmm1 \n" + "pmaddubsw %%xmm3,%%xmm6 \n" + "phaddw %%xmm2,%%xmm0 \n" + "phaddw %%xmm6,%%xmm1 \n" + "psraw $0x8,%%xmm0 \n" + "psraw $0x8,%%xmm1 \n" + "packsswb %%xmm1,%%xmm0 \n" + "paddb %%xmm5,%%xmm0 \n" + "movlps %%xmm0," MEMACCESS(1) " \n" + MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_rgba0), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+rm"(width) // %3 + : "r"((intptr_t)(src_stride_rgba)), // %4 + "m"(kRGBAToV), // %5 + "m"(kRGBAToU), // %6 + "m"(kAddUV128) // %7 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" + ); +} + +#if defined(HAS_I422TOARGBROW_SSSE3) || defined(HAS_I422TOARGBROW_AVX2) + +struct YuvConstants { + lvec8 kUVToB; // 0 + lvec8 kUVToG; // 32 + lvec8 kUVToR; // 64 + lvec16 kUVBiasB; // 96 + lvec16 kUVBiasG; // 128 + lvec16 kUVBiasR; // 160 + lvec16 kYToRgb; // 192 +}; + +// BT.601 YUV to RGB reference +// R = (Y - 16) * 1.164 - V * -1.596 +// G = (Y - 16) * 1.164 - U * 0.391 - V * 0.813 +// B = (Y - 16) * 1.164 - U * -2.018 + +// Y contribution to R,G,B. Scale and bias. +// TODO(fbarchard): Consider moving constants into a common header. +#define YG 18997 /* round(1.164 * 64 * 256 * 256 / 257) */ +#define YGB -1160 /* 1.164 * 64 * -16 + 64 / 2 */ + +// U and V contributions to R,G,B. +#define UB -128 /* max(-128, round(-2.018 * 64)) */ +#define UG 25 /* round(0.391 * 64) */ +#define VG 52 /* round(0.813 * 64) */ +#define VR -102 /* round(-1.596 * 64) */ + +// Bias values to subtract 16 from Y and 128 from U and V. +#define BB (UB * 128 + YGB) +#define BG (UG * 128 + VG * 128 + YGB) +#define BR (VR * 128 + YGB) + +// BT601 constants for YUV to RGB. +static YuvConstants SIMD_ALIGNED(kYuvConstants) = { + { UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, + UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0 }, + { UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, + UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG }, + { 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, + 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR }, + { BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB }, + { BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG }, + { BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR }, + { YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG } +}; + +// BT601 constants for NV21 where chroma plane is VU instead of UV. +static YuvConstants SIMD_ALIGNED(kYvuConstants) = { + { 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, + 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB }, + { VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, + VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG }, + { VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, + VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0 }, + { BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB }, + { BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG }, + { BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR }, + { YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG } +}; + +#undef YG +#undef YGB +#undef UB +#undef UG +#undef VG +#undef VR +#undef BB +#undef BG +#undef BR + +// JPEG YUV to RGB reference +// * R = Y - V * -1.40200 +// * G = Y - U * 0.34414 - V * 0.71414 +// * B = Y - U * -1.77200 + +// Y contribution to R,G,B. Scale and bias. +// TODO(fbarchard): Consider moving constants into a common header. +#define YGJ 16320 /* round(1.000 * 64 * 256 * 256 / 257) */ +#define YGBJ 32 /* 64 / 2 */ + +// U and V contributions to R,G,B. +#define UBJ -113 /* round(-1.77200 * 64) */ +#define UGJ 22 /* round(0.34414 * 64) */ +#define VGJ 46 /* round(0.71414 * 64) */ +#define VRJ -90 /* round(-1.40200 * 64) */ + +// Bias values to subtract 16 from Y and 128 from U and V. +#define BBJ (UBJ * 128 + YGBJ) +#define BGJ (UGJ * 128 + VGJ * 128 + YGBJ) +#define BRJ (VRJ * 128 + YGBJ) + +// JPEG constants for YUV to RGB. +YuvConstants SIMD_ALIGNED(kYuvJConstants) = { + { UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, + UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0 }, + { UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, + UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, + UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, + UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, UGJ, VGJ }, + { 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, + 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ }, + { BBJ, BBJ, BBJ, BBJ, BBJ, BBJ, BBJ, BBJ, + BBJ, BBJ, BBJ, BBJ, BBJ, BBJ, BBJ, BBJ }, + { BGJ, BGJ, BGJ, BGJ, BGJ, BGJ, BGJ, BGJ, + BGJ, BGJ, BGJ, BGJ, BGJ, BGJ, BGJ, BGJ }, + { BRJ, BRJ, BRJ, BRJ, BRJ, BRJ, BRJ, BRJ, + BRJ, BRJ, BRJ, BRJ, BRJ, BRJ, BRJ, BRJ }, + { YGJ, YGJ, YGJ, YGJ, YGJ, YGJ, YGJ, YGJ, + YGJ, YGJ, YGJ, YGJ, YGJ, YGJ, YGJ, YGJ } +}; + +#undef YGJ +#undef YGBJ +#undef UBJ +#undef UGJ +#undef VGJ +#undef VRJ +#undef BBJ +#undef BGJ +#undef BRJ + +// Read 8 UV from 411 +#define READYUV444 \ + "movq " MEMACCESS([u_buf]) ",%%xmm0 \n" \ + MEMOPREG(movq, 0x00, [u_buf], [v_buf], 1, xmm1) \ + "lea " MEMLEA(0x8, [u_buf]) ",%[u_buf] \n" \ + "punpcklbw %%xmm1,%%xmm0 \n" + +// Read 4 UV from 422, upsample to 8 UV +#define READYUV422 \ + "movd " MEMACCESS([u_buf]) ",%%xmm0 \n" \ + MEMOPREG(movd, 0x00, [u_buf], [v_buf], 1, xmm1) \ + "lea " MEMLEA(0x4, [u_buf]) ",%[u_buf] \n" \ + "punpcklbw %%xmm1,%%xmm0 \n" \ + "punpcklwd %%xmm0,%%xmm0 \n" + +// Read 2 UV from 411, upsample to 8 UV +#define READYUV411 \ + "movd " MEMACCESS([u_buf]) ",%%xmm0 \n" \ + MEMOPREG(movd, 0x00, [u_buf], [v_buf], 1, xmm1) \ + "lea " MEMLEA(0x2, [u_buf]) ",%[u_buf] \n" \ + "punpcklbw %%xmm1,%%xmm0 \n" \ + "punpcklwd %%xmm0,%%xmm0 \n" \ + "punpckldq %%xmm0,%%xmm0 \n" + +// Read 4 UV from NV12, upsample to 8 UV +#define READNV12 \ + "movq " MEMACCESS([uv_buf]) ",%%xmm0 \n" \ + "lea " MEMLEA(0x8, [uv_buf]) ",%[uv_buf] \n" \ + "punpcklwd %%xmm0,%%xmm0 \n" + +// Convert 8 pixels: 8 UV and 8 Y +#define YUVTORGB(YuvConstants) \ + "movdqa %%xmm0,%%xmm1 \n" \ + "movdqa %%xmm0,%%xmm2 \n" \ + "movdqa %%xmm0,%%xmm3 \n" \ + "movdqa " MEMACCESS2(96, [YuvConstants]) ",%%xmm0 \n" \ + "pmaddubsw " MEMACCESS([YuvConstants]) ",%%xmm1 \n" \ + "psubw %%xmm1,%%xmm0 \n" \ + "movdqa " MEMACCESS2(128, [YuvConstants]) ",%%xmm1 \n" \ + "pmaddubsw " MEMACCESS2(32, [YuvConstants]) ",%%xmm2 \n" \ + "psubw %%xmm2,%%xmm1 \n" \ + "movdqa " MEMACCESS2(160, [YuvConstants]) ",%%xmm2 \n" \ + "pmaddubsw " MEMACCESS2(64, [YuvConstants]) ",%%xmm3 \n" \ + "psubw %%xmm3,%%xmm2 \n" \ + "movq " MEMACCESS([y_buf]) ",%%xmm3 \n" \ + "lea " MEMLEA(0x8, [y_buf]) ",%[y_buf] \n" \ + "punpcklbw %%xmm3,%%xmm3 \n" \ + "pmulhuw " MEMACCESS2(192, [YuvConstants]) ",%%xmm3 \n" \ + "paddsw %%xmm3,%%xmm0 \n" \ + "paddsw %%xmm3,%%xmm1 \n" \ + "paddsw %%xmm3,%%xmm2 \n" \ + "psraw $0x6,%%xmm0 \n" \ + "psraw $0x6,%%xmm1 \n" \ + "psraw $0x6,%%xmm2 \n" \ + "packuswb %%xmm0,%%xmm0 \n" \ + "packuswb %%xmm1,%%xmm1 \n" \ + "packuswb %%xmm2,%%xmm2 \n" + +// Store 8 ARGB values. Assumes XMM5 is zero. +#define STOREARGB \ + "punpcklbw %%xmm1,%%xmm0 \n" \ + "punpcklbw %%xmm5,%%xmm2 \n" \ + "movdqa %%xmm0,%%xmm1 \n" \ + "punpcklwd %%xmm2,%%xmm0 \n" \ + "punpckhwd %%xmm2,%%xmm1 \n" \ + "movdqu %%xmm0," MEMACCESS([dst_argb]) " \n" \ + "movdqu %%xmm1," MEMACCESS2(0x10, [dst_argb]) " \n" \ + "lea " MEMLEA(0x20, [dst_argb]) ", %[dst_argb] \n" + +// Store 8 BGRA values. Assumes XMM5 is zero. +#define STOREBGRA \ + "pcmpeqb %%xmm5,%%xmm5 \n" \ + "punpcklbw %%xmm0,%%xmm1 \n" \ + "punpcklbw %%xmm2,%%xmm5 \n" \ + "movdqa %%xmm5,%%xmm0 \n" \ + "punpcklwd %%xmm1,%%xmm5 \n" \ + "punpckhwd %%xmm1,%%xmm0 \n" \ + "movdqu %%xmm5," MEMACCESS([dst_bgra]) " \n" \ + "movdqu %%xmm0," MEMACCESS2(0x10, [dst_bgra]) " \n" \ + "lea " MEMLEA(0x20, [dst_bgra]) ", %[dst_bgra] \n" + +// Store 8 ABGR values. Assumes XMM5 is zero. +#define STOREABGR \ + "punpcklbw %%xmm1,%%xmm2 \n" \ + "punpcklbw %%xmm5,%%xmm0 \n" \ + "movdqa %%xmm2,%%xmm1 \n" \ + "punpcklwd %%xmm0,%%xmm2 \n" \ + "punpckhwd %%xmm0,%%xmm1 \n" \ + "movdqu %%xmm2," MEMACCESS([dst_abgr]) " \n" \ + "movdqu %%xmm1," MEMACCESS2(0x10, [dst_abgr]) " \n" \ + "lea " MEMLEA(0x20, [dst_abgr]) ", %[dst_abgr] \n" + +// Store 8 RGBA values. Assumes XMM5 is zero. +#define STORERGBA \ + "pcmpeqb %%xmm5,%%xmm5 \n" \ + "punpcklbw %%xmm2,%%xmm1 \n" \ + "punpcklbw %%xmm0,%%xmm5 \n" \ + "movdqa %%xmm5,%%xmm0 \n" \ + "punpcklwd %%xmm1,%%xmm5 \n" \ + "punpckhwd %%xmm1,%%xmm0 \n" \ + "movdqu %%xmm5," MEMACCESS([dst_rgba]) " \n" \ + "movdqu %%xmm0," MEMACCESS2(0x10, [dst_rgba]) " \n" \ + "lea " MEMLEA(0x20, [dst_rgba]) ",%[dst_rgba] \n" + +void OMITFP I444ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + asm volatile ( + "sub %[u_buf],%[v_buf] \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + LABELALIGN + "1: \n" + READYUV444 + YUVTORGB(kYuvConstants) + STOREARGB + "sub $0x8,%[width] \n" + "jg 1b \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_argb]"+r"(dst_argb), // %[dst_argb] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} + +// TODO(fbarchard): Consider putting masks into constants. +void OMITFP I422ToRGB24Row_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_rgb24, + int width) { + asm volatile ( + "movdqa %[kShuffleMaskARGBToRGB24_0],%%xmm5 \n" + "movdqa %[kShuffleMaskARGBToRGB24],%%xmm6 \n" + "sub %[u_buf],%[v_buf] \n" + LABELALIGN + "1: \n" + READYUV422 + YUVTORGB(kYuvConstants) + "punpcklbw %%xmm1,%%xmm0 \n" + "punpcklbw %%xmm2,%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklwd %%xmm2,%%xmm0 \n" + "punpckhwd %%xmm2,%%xmm1 \n" + "pshufb %%xmm5,%%xmm0 \n" + "pshufb %%xmm6,%%xmm1 \n" + "palignr $0xc,%%xmm0,%%xmm1 \n" + "movq %%xmm0," MEMACCESS([dst_rgb24]) "\n" + "movdqu %%xmm1," MEMACCESS2(0x8,[dst_rgb24]) "\n" + "lea " MEMLEA(0x18,[dst_rgb24]) ",%[dst_rgb24] \n" + "subl $0x8,%[width] \n" + "jg 1b \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24] +// TODO(fbarchard): Make width a register for 32 bit. +#if defined(__i386__) && defined(__pic__) + [width]"+m"(width) // %[width] +#else + [width]"+rm"(width) // %[width] +#endif + : [kYuvConstants]"r"(&kYuvConstants.kUVToB), + [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24_0), + [kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24) + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6" + ); +} + +void OMITFP I422ToRAWRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_raw, + int width) { + asm volatile ( + "movdqa %[kShuffleMaskARGBToRAW_0],%%xmm5 \n" + "movdqa %[kShuffleMaskARGBToRAW],%%xmm6 \n" + "sub %[u_buf],%[v_buf] \n" + LABELALIGN + "1: \n" + READYUV422 + YUVTORGB(kYuvConstants) + "punpcklbw %%xmm1,%%xmm0 \n" + "punpcklbw %%xmm2,%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklwd %%xmm2,%%xmm0 \n" + "punpckhwd %%xmm2,%%xmm1 \n" + "pshufb %%xmm5,%%xmm0 \n" + "pshufb %%xmm6,%%xmm1 \n" + "palignr $0xc,%%xmm0,%%xmm1 \n" + "movq %%xmm0," MEMACCESS([dst_raw]) " \n" + "movdqu %%xmm1," MEMACCESS2(0x8,[dst_raw]) "\n" + "lea " MEMLEA(0x18,[dst_raw]) ",%[dst_raw] \n" + "subl $0x8,%[width] \n" + "jg 1b \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_raw]"+r"(dst_raw), // %[dst_raw] +// TODO(fbarchard): Make width a register for 32 bit. +#if defined(__i386__) && defined(__pic__) + [width]"+m"(width) // %[width] +#else + [width]"+rm"(width) // %[width] +#endif + : [kYuvConstants]"r"(&kYuvConstants.kUVToB), + [kShuffleMaskARGBToRAW_0]"m"(kShuffleMaskARGBToRAW_0), + [kShuffleMaskARGBToRAW]"m"(kShuffleMaskARGBToRAW) + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6" + ); +} + +void OMITFP I422ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + asm volatile ( + "sub %[u_buf],%[v_buf] \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + LABELALIGN + "1: \n" + READYUV422 + YUVTORGB(kYuvConstants) + STOREARGB + "sub $0x8,%[width] \n" + "jg 1b \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_argb]"+r"(dst_argb), // %[dst_argb] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} + +void OMITFP J422ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + asm volatile ( + "sub %[u_buf],%[v_buf] \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + LABELALIGN + "1: \n" + READYUV422 + YUVTORGB(kYuvConstants) + STOREARGB + "sub $0x8,%[width] \n" + "jg 1b \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_argb]"+r"(dst_argb), // %[dst_argb] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvJConstants.kUVToB) // %[kYuvConstants] + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} + +void OMITFP I411ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + asm volatile ( + "sub %[u_buf],%[v_buf] \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + LABELALIGN + "1: \n" + READYUV411 + YUVTORGB(kYuvConstants) + STOREARGB + "sub $0x8,%[width] \n" + "jg 1b \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_argb]"+r"(dst_argb), // %[dst_argb] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} + +void OMITFP NV12ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* uv_buf, + uint8* dst_argb, + int width) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" + LABELALIGN + "1: \n" + READNV12 + YUVTORGB(kYuvConstants) + STOREARGB + "sub $0x8,%[width] \n" + "jg 1b \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [uv_buf]"+r"(uv_buf), // %[uv_buf] + [dst_argb]"+r"(dst_argb), // %[dst_argb] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] + // Does not use r14. + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} + +void OMITFP NV21ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* uv_buf, + uint8* dst_argb, + int width) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" + LABELALIGN + "1: \n" + READNV12 + YUVTORGB(kYuvConstants) + STOREARGB + "sub $0x8,%[width] \n" + "jg 1b \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [uv_buf]"+r"(uv_buf), // %[uv_buf] + [dst_argb]"+r"(dst_argb), // %[dst_argb] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYvuConstants.kUVToB) // %[kYuvConstants] + // Does not use r14. + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} + +void OMITFP I422ToBGRARow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_bgra, + int width) { + asm volatile ( + "sub %[u_buf],%[v_buf] \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + LABELALIGN + "1: \n" + READYUV422 + YUVTORGB(kYuvConstants) + STOREBGRA + "sub $0x8,%[width] \n" + "jg 1b \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_bgra]"+r"(dst_bgra), // %[dst_bgra] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} + +void OMITFP I422ToABGRRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_abgr, + int width) { + asm volatile ( + "sub %[u_buf],%[v_buf] \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + LABELALIGN + "1: \n" + READYUV422 + YUVTORGB(kYuvConstants) + STOREABGR + "sub $0x8,%[width] \n" + "jg 1b \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_abgr]"+r"(dst_abgr), // %[dst_abgr] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} + +void OMITFP I422ToRGBARow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_rgba, + int width) { + asm volatile ( + "sub %[u_buf],%[v_buf] \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + LABELALIGN + "1: \n" + READYUV422 + YUVTORGB(kYuvConstants) + STORERGBA + "sub $0x8,%[width] \n" + "jg 1b \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_rgba]"+r"(dst_rgba), // %[dst_rgba] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} + +#endif // HAS_I422TOARGBROW_SSSE3 + +// Read 8 UV from 422, upsample to 16 UV. +#define READYUV422_AVX2 \ + "vmovq " MEMACCESS([u_buf]) ",%%xmm0 \n" \ + MEMOPREG(vmovq, 0x00, [u_buf], [v_buf], 1, xmm1) \ + "lea " MEMLEA(0x8, [u_buf]) ",%[u_buf] \n" \ + "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" \ + "vpermq $0xd8,%%ymm0,%%ymm0 \n" \ + "vpunpcklwd %%ymm0,%%ymm0,%%ymm0 \n" + +// Convert 16 pixels: 16 UV and 16 Y. +#define YUVTORGB_AVX2(YuvConstants) \ + "vpmaddubsw " MEMACCESS2(64, [YuvConstants]) ",%%ymm0,%%ymm2 \n" \ + "vpmaddubsw " MEMACCESS2(32, [YuvConstants]) ",%%ymm0,%%ymm1 \n" \ + "vpmaddubsw " MEMACCESS([YuvConstants]) ",%%ymm0,%%ymm0 \n" \ + "vmovdqu " MEMACCESS2(160, [YuvConstants]) ",%%ymm3 \n" \ + "vpsubw %%ymm2,%%ymm3,%%ymm2 \n" \ + "vmovdqu " MEMACCESS2(128, [YuvConstants]) ",%%ymm3 \n" \ + "vpsubw %%ymm1,%%ymm3,%%ymm1 \n" \ + "vmovdqu " MEMACCESS2(96, [YuvConstants]) ",%%ymm3 \n" \ + "vpsubw %%ymm0,%%ymm3,%%ymm0 \n" \ + "vmovdqu " MEMACCESS([y_buf]) ",%%xmm3 \n" \ + "lea " MEMLEA(0x10, [y_buf]) ",%[y_buf] \n" \ + "vpermq $0xd8,%%ymm3,%%ymm3 \n" \ + "vpunpcklbw %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpmulhuw " MEMACCESS2(192, [YuvConstants]) ",%%ymm3,%%ymm3 \n" \ + "vpaddsw %%ymm3,%%ymm0,%%ymm0 \n" \ + "vpaddsw %%ymm3,%%ymm1,%%ymm1 \n" \ + "vpaddsw %%ymm3,%%ymm2,%%ymm2 \n" \ + "vpsraw $0x6,%%ymm0,%%ymm0 \n" \ + "vpsraw $0x6,%%ymm1,%%ymm1 \n" \ + "vpsraw $0x6,%%ymm2,%%ymm2 \n" \ + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" \ + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpackuswb %%ymm2,%%ymm2,%%ymm2 \n" + +#if defined(HAS_I422TOBGRAROW_AVX2) +// 16 pixels +// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 BGRA (64 bytes). +void OMITFP I422ToBGRARow_AVX2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_bgra, + int width) { + asm volatile ( + "sub %[u_buf],%[v_buf] \n" + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + LABELALIGN + "1: \n" + READYUV422_AVX2 + YUVTORGB_AVX2(kYuvConstants) + + // Step 3: Weave into BGRA + "vpunpcklbw %%ymm0,%%ymm1,%%ymm1 \n" // GB + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpunpcklbw %%ymm2,%%ymm5,%%ymm2 \n" // AR + "vpermq $0xd8,%%ymm2,%%ymm2 \n" + "vpunpcklwd %%ymm1,%%ymm2,%%ymm0 \n" // ARGB first 8 pixels + "vpunpckhwd %%ymm1,%%ymm2,%%ymm2 \n" // ARGB next 8 pixels + + "vmovdqu %%ymm0," MEMACCESS([dst_bgra]) "\n" + "vmovdqu %%ymm2," MEMACCESS2(0x20,[dst_bgra]) "\n" + "lea " MEMLEA(0x40,[dst_bgra]) ",%[dst_bgra] \n" + "sub $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_bgra]"+r"(dst_bgra), // %[dst_bgra] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} +#endif // HAS_I422TOBGRAROW_AVX2 + +#if defined(HAS_I422TOARGBROW_AVX2) +// 16 pixels +// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ARGB (64 bytes). +void OMITFP I422ToARGBRow_AVX2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + asm volatile ( + "sub %[u_buf],%[v_buf] \n" + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + LABELALIGN + "1: \n" + READYUV422_AVX2 + YUVTORGB_AVX2(kYuvConstants) + + // Step 3: Weave into ARGB + "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" // BG + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpunpcklbw %%ymm5,%%ymm2,%%ymm2 \n" // RA + "vpermq $0xd8,%%ymm2,%%ymm2 \n" + "vpunpcklwd %%ymm2,%%ymm0,%%ymm1 \n" // BGRA first 8 pixels + "vpunpckhwd %%ymm2,%%ymm0,%%ymm0 \n" // BGRA next 8 pixels + + "vmovdqu %%ymm1," MEMACCESS([dst_argb]) "\n" + "vmovdqu %%ymm0," MEMACCESS2(0x20,[dst_argb]) "\n" + "lea " MEMLEA(0x40,[dst_argb]) ",%[dst_argb] \n" + "sub $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_argb]"+r"(dst_argb), // %[dst_argb] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} +#endif // HAS_I422TOARGBROW_AVX2 + +#if defined(HAS_J422TOARGBROW_AVX2) +// 16 pixels +// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ARGB (64 bytes). +void OMITFP J422ToARGBRow_AVX2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + asm volatile ( + "sub %[u_buf],%[v_buf] \n" + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + LABELALIGN + "1: \n" + READYUV422_AVX2 + YUVTORGB_AVX2(kYuvConstants) + + // Step 3: Weave into ARGB + "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" // BG + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpunpcklbw %%ymm5,%%ymm2,%%ymm2 \n" // RA + "vpermq $0xd8,%%ymm2,%%ymm2 \n" + "vpunpcklwd %%ymm2,%%ymm0,%%ymm1 \n" // BGRA first 8 pixels + "vpunpckhwd %%ymm2,%%ymm0,%%ymm0 \n" // BGRA next 8 pixels + + "vmovdqu %%ymm1," MEMACCESS([dst_argb]) "\n" + "vmovdqu %%ymm0," MEMACCESS2(0x20,[dst_argb]) "\n" + "lea " MEMLEA(0x40,[dst_argb]) ",%[dst_argb] \n" + "sub $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_argb]"+r"(dst_argb), // %[dst_argb] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvJConstants.kUVToB) // %[kYuvConstants] + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} +#endif // HAS_J422TOARGBROW_AVX2 + +#if defined(HAS_I422TOABGRROW_AVX2) +// 16 pixels +// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ABGR (64 bytes). +void OMITFP I422ToABGRRow_AVX2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + asm volatile ( + "sub %[u_buf],%[v_buf] \n" + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + LABELALIGN + "1: \n" + READYUV422_AVX2 + YUVTORGB_AVX2(kYuvConstants) + + // Step 3: Weave into ABGR + "vpunpcklbw %%ymm1,%%ymm2,%%ymm1 \n" // RG + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpunpcklbw %%ymm5,%%ymm0,%%ymm2 \n" // BA + "vpermq $0xd8,%%ymm2,%%ymm2 \n" + "vpunpcklwd %%ymm2,%%ymm1,%%ymm0 \n" // RGBA first 8 pixels + "vpunpckhwd %%ymm2,%%ymm1,%%ymm1 \n" // RGBA next 8 pixels + "vmovdqu %%ymm0," MEMACCESS([dst_argb]) "\n" + "vmovdqu %%ymm1," MEMACCESS2(0x20,[dst_argb]) "\n" + "lea " MEMLEA(0x40,[dst_argb]) ",%[dst_argb] \n" + "sub $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_argb]"+r"(dst_argb), // %[dst_argb] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} +#endif // HAS_I422TOABGRROW_AVX2 + +#if defined(HAS_I422TORGBAROW_AVX2) +// 16 pixels +// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 RGBA (64 bytes). +void OMITFP I422ToRGBARow_AVX2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + asm volatile ( + "sub %[u_buf],%[v_buf] \n" + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + LABELALIGN + "1: \n" + READYUV422_AVX2 + YUVTORGB_AVX2(kYuvConstants) + + // Step 3: Weave into RGBA + "vpunpcklbw %%ymm2,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpunpcklbw %%ymm0,%%ymm5,%%ymm2 \n" + "vpermq $0xd8,%%ymm2,%%ymm2 \n" + "vpunpcklwd %%ymm1,%%ymm2,%%ymm0 \n" + "vpunpckhwd %%ymm1,%%ymm2,%%ymm1 \n" + "vmovdqu %%ymm0," MEMACCESS([dst_argb]) "\n" + "vmovdqu %%ymm1," MEMACCESS2(0x20,[dst_argb]) "\n" + "lea " MEMLEA(0x40,[dst_argb]) ",%[dst_argb] \n" + "sub $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_argb]"+r"(dst_argb), // %[dst_argb] + [width]"+rm"(width) // %[width] + : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} +#endif // HAS_I422TORGBAROW_AVX2 + +#ifdef HAS_I400TOARGBROW_SSE2 +void I400ToARGBRow_SSE2(const uint8* y_buf, uint8* dst_argb, int width) { + asm volatile ( + "mov $0x4a354a35,%%eax \n" // 4a35 = 18997 = 1.164 + "movd %%eax,%%xmm2 \n" + "pshufd $0x0,%%xmm2,%%xmm2 \n" + "mov $0x04880488,%%eax \n" // 0488 = 1160 = 1.164 * 16 + "movd %%eax,%%xmm3 \n" + "pshufd $0x0,%%xmm3,%%xmm3 \n" + "pcmpeqb %%xmm4,%%xmm4 \n" + "pslld $0x18,%%xmm4 \n" + LABELALIGN + "1: \n" + // Step 1: Scale Y contribution to 8 G values. G = (y - 16) * 1.164 + "movq " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x8,0) ",%0 \n" + "punpcklbw %%xmm0,%%xmm0 \n" + "pmulhuw %%xmm2,%%xmm0 \n" + "psubusw %%xmm3,%%xmm0 \n" + "psrlw $6, %%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + + // Step 2: Weave into ARGB + "punpcklbw %%xmm0,%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklwd %%xmm0,%%xmm0 \n" + "punpckhwd %%xmm1,%%xmm1 \n" + "por %%xmm4,%%xmm0 \n" + "por %%xmm4,%%xmm1 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(y_buf), // %0 + "+r"(dst_argb), // %1 + "+rm"(width) // %2 + : + : "memory", "cc", "eax" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4" + ); +} +#endif // HAS_I400TOARGBROW_SSE2 + +#ifdef HAS_I400TOARGBROW_AVX2 +// 16 pixels of Y converted to 16 pixels of ARGB (64 bytes). +// note: vpunpcklbw mutates and vpackuswb unmutates. +void I400ToARGBRow_AVX2(const uint8* y_buf, uint8* dst_argb, int width) { + asm volatile ( + "mov $0x4a354a35,%%eax \n" // 0488 = 1160 = 1.164 * 16 + "vmovd %%eax,%%xmm2 \n" + "vbroadcastss %%xmm2,%%ymm2 \n" + "mov $0x4880488,%%eax \n" // 4a35 = 18997 = 1.164 + "vmovd %%eax,%%xmm3 \n" + "vbroadcastss %%xmm3,%%ymm3 \n" + "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n" + "vpslld $0x18,%%ymm4,%%ymm4 \n" + + LABELALIGN + "1: \n" + // Step 1: Scale Y contribution to 16 G values. G = (y - 16) * 1.164 + "vmovdqu " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n" + "vpsubusw %%ymm3,%%ymm0,%%ymm0 \n" + "vpsrlw $0x6,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpunpcklbw %%ymm0,%%ymm0,%%ymm1 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpunpcklwd %%ymm1,%%ymm1,%%ymm0 \n" + "vpunpckhwd %%ymm1,%%ymm1,%%ymm1 \n" + "vpor %%ymm4,%%ymm0,%%ymm0 \n" + "vpor %%ymm4,%%ymm1,%%ymm1 \n" + "vmovdqu %%ymm0," MEMACCESS(1) " \n" + "vmovdqu %%ymm1," MEMACCESS2(0x20,1) " \n" + "lea " MEMLEA(0x40,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(y_buf), // %0 + "+r"(dst_argb), // %1 + "+rm"(width) // %2 + : + : "memory", "cc", "eax" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4" + ); +} +#endif // HAS_I400TOARGBROW_AVX2 + +#ifdef HAS_MIRRORROW_SSSE3 +// Shuffle table for reversing the bytes. +static uvec8 kShuffleMirror = { + 15u, 14u, 13u, 12u, 11u, 10u, 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u +}; + +void MirrorRow_SSSE3(const uint8* src, uint8* dst, int width) { + intptr_t temp_width = (intptr_t)(width); + asm volatile ( + "movdqa %3,%%xmm5 \n" + LABELALIGN + "1: \n" + MEMOPREG(movdqu,-0x10,0,2,1,xmm0) // movdqu -0x10(%0,%2),%%xmm0 + "pshufb %%xmm5,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(temp_width) // %2 + : "m"(kShuffleMirror) // %3 + : "memory", "cc", NACL_R14 + "xmm0", "xmm5" + ); +} +#endif // HAS_MIRRORROW_SSSE3 + +#ifdef HAS_MIRRORROW_AVX2 +void MirrorRow_AVX2(const uint8* src, uint8* dst, int width) { + intptr_t temp_width = (intptr_t)(width); + asm volatile ( + "vbroadcastf128 %3,%%ymm5 \n" + LABELALIGN + "1: \n" + MEMOPREG(vmovdqu,-0x20,0,2,1,ymm0) // vmovdqu -0x20(%0,%2),%%ymm0 + "vpshufb %%ymm5,%%ymm0,%%ymm0 \n" + "vpermq $0x4e,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(temp_width) // %2 + : "m"(kShuffleMirror) // %3 + : "memory", "cc", NACL_R14 + "xmm0", "xmm5" + ); +} +#endif // HAS_MIRRORROW_AVX2 + +#ifdef HAS_MIRRORROW_SSE2 +void MirrorRow_SSE2(const uint8* src, uint8* dst, int width) { + intptr_t temp_width = (intptr_t)(width); + asm volatile ( + LABELALIGN + "1: \n" + MEMOPREG(movdqu,-0x10,0,2,1,xmm0) // movdqu -0x10(%0,%2),%%xmm0 + "movdqa %%xmm0,%%xmm1 \n" + "psllw $0x8,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" + "por %%xmm1,%%xmm0 \n" + "pshuflw $0x1b,%%xmm0,%%xmm0 \n" + "pshufhw $0x1b,%%xmm0,%%xmm0 \n" + "pshufd $0x4e,%%xmm0,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1)",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(temp_width) // %2 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1" + ); +} +#endif // HAS_MIRRORROW_SSE2 + +#ifdef HAS_MIRRORROW_UV_SSSE3 +// Shuffle table for reversing the bytes of UV channels. +static uvec8 kShuffleMirrorUV = { + 14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u, 15u, 13u, 11u, 9u, 7u, 5u, 3u, 1u +}; +void MirrorUVRow_SSSE3(const uint8* src, uint8* dst_u, uint8* dst_v, + int width) { + intptr_t temp_width = (intptr_t)(width); + asm volatile ( + "movdqa %4,%%xmm1 \n" + "lea " MEMLEA4(-0x10,0,3,2) ",%0 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(-0x10,0) ",%0 \n" + "pshufb %%xmm1,%%xmm0 \n" + "movlpd %%xmm0," MEMACCESS(1) " \n" + MEMOPMEM(movhpd,xmm0,0x00,1,2,1) // movhpd %%xmm0,(%1,%2) + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $8,%3 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(temp_width) // %3 + : "m"(kShuffleMirrorUV) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1" + ); +} +#endif // HAS_MIRRORROW_UV_SSSE3 + +#ifdef HAS_ARGBMIRRORROW_SSE2 + +void ARGBMirrorRow_SSE2(const uint8* src, uint8* dst, int width) { + intptr_t temp_width = (intptr_t)(width); + asm volatile ( + "lea " MEMLEA4(-0x10,0,2,4) ",%0 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "pshufd $0x1b,%%xmm0,%%xmm0 \n" + "lea " MEMLEA(-0x10,0) ",%0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x4,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(temp_width) // %2 + : + : "memory", "cc" + , "xmm0" + ); +} +#endif // HAS_ARGBMIRRORROW_SSE2 + +#ifdef HAS_ARGBMIRRORROW_AVX2 +// Shuffle table for reversing the bytes. +static const ulvec32 kARGBShuffleMirror_AVX2 = { + 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u +}; +void ARGBMirrorRow_AVX2(const uint8* src, uint8* dst, int width) { + intptr_t temp_width = (intptr_t)(width); + asm volatile ( + "vmovdqu %3,%%ymm5 \n" + LABELALIGN + "1: \n" + VMEMOPREG(vpermd,-0x20,0,2,4,ymm5,ymm0) // vpermd -0x20(%0,%2,4),ymm5,ymm0 + "vmovdqu %%ymm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(temp_width) // %2 + : "m"(kARGBShuffleMirror_AVX2) // %3 + : "memory", "cc", NACL_R14 + "xmm0", "xmm5" + ); +} +#endif // HAS_ARGBMIRRORROW_AVX2 + +#ifdef HAS_SPLITUVROW_AVX2 +void SplitUVRow_AVX2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpsrlw $0x8,%%ymm5,%%ymm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "vpsrlw $0x8,%%ymm0,%%ymm2 \n" + "vpsrlw $0x8,%%ymm1,%%ymm3 \n" + "vpand %%ymm5,%%ymm0,%%ymm0 \n" + "vpand %%ymm5,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm3,%%ymm2,%%ymm2 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm2,%%ymm2 \n" + "vmovdqu %%ymm0," MEMACCESS(1) " \n" + MEMOPMEM(vmovdqu,ymm2,0x00,1,2,1) // vmovdqu %%ymm2,(%1,%2) + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x20,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} +#endif // HAS_SPLITUVROW_AVX2 + +#ifdef HAS_SPLITUVROW_SSE2 +void SplitUVRow_SSE2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" + "psrlw $0x8,%%xmm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "movdqa %%xmm0,%%xmm2 \n" + "movdqa %%xmm1,%%xmm3 \n" + "pand %%xmm5,%%xmm0 \n" + "pand %%xmm5,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "psrlw $0x8,%%xmm2 \n" + "psrlw $0x8,%%xmm3 \n" + "packuswb %%xmm3,%%xmm2 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + MEMOPMEM(movdqu,xmm2,0x00,1,2,1) // movdqu %%xmm2,(%1,%2) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_uv), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} +#endif // HAS_SPLITUVROW_SSE2 + +#ifdef HAS_MERGEUVROW_AVX2 +void MergeUVRow_AVX2(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width) { + asm volatile ( + "sub %0,%1 \n" + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + MEMOPREG(vmovdqu,0x00,0,1,1,ymm1) // vmovdqu (%0,%1,1),%%ymm1 + "lea " MEMLEA(0x20,0) ",%0 \n" + "vpunpcklbw %%ymm1,%%ymm0,%%ymm2 \n" + "vpunpckhbw %%ymm1,%%ymm0,%%ymm0 \n" + "vextractf128 $0x0,%%ymm2," MEMACCESS(2) " \n" + "vextractf128 $0x0,%%ymm0," MEMACCESS2(0x10,2) "\n" + "vextractf128 $0x1,%%ymm2," MEMACCESS2(0x20,2) "\n" + "vextractf128 $0x1,%%ymm0," MEMACCESS2(0x30,2) "\n" + "lea " MEMLEA(0x40,2) ",%2 \n" + "sub $0x20,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_u), // %0 + "+r"(src_v), // %1 + "+r"(dst_uv), // %2 + "+r"(width) // %3 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2" + ); +} +#endif // HAS_MERGEUVROW_AVX2 + +#ifdef HAS_MERGEUVROW_SSE2 +void MergeUVRow_SSE2(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width) { + asm volatile ( + "sub %0,%1 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,0,1,1,xmm1) // movdqu (%0,%1,1),%%xmm1 + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqa %%xmm0,%%xmm2 \n" + "punpcklbw %%xmm1,%%xmm0 \n" + "punpckhbw %%xmm1,%%xmm2 \n" + "movdqu %%xmm0," MEMACCESS(2) " \n" + "movdqu %%xmm2," MEMACCESS2(0x10,2) " \n" + "lea " MEMLEA(0x20,2) ",%2 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_u), // %0 + "+r"(src_v), // %1 + "+r"(dst_uv), // %2 + "+r"(width) // %3 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2" + ); +} +#endif // HAS_MERGEUVROW_SSE2 + +#ifdef HAS_COPYROW_SSE2 +void CopyRow_SSE2(const uint8* src, uint8* dst, int count) { + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(count) // %2 + : + : "memory", "cc" + , "xmm0", "xmm1" + ); +} +#endif // HAS_COPYROW_SSE2 + +#ifdef HAS_COPYROW_AVX +void CopyRow_AVX(const uint8* src, uint8* dst, int count) { + asm volatile ( + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "vmovdqu %%ymm0," MEMACCESS(1) " \n" + "vmovdqu %%ymm1," MEMACCESS2(0x20,1) " \n" + "lea " MEMLEA(0x40,1) ",%1 \n" + "sub $0x40,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(count) // %2 + : + : "memory", "cc" + , "xmm0", "xmm1" + ); +} +#endif // HAS_COPYROW_AVX + +#ifdef HAS_COPYROW_ERMS +// Multiple of 1. +void CopyRow_ERMS(const uint8* src, uint8* dst, int width) { + size_t width_tmp = (size_t)(width); + asm volatile ( + "rep movsb " MEMMOVESTRING(0,1) " \n" + : "+S"(src), // %0 + "+D"(dst), // %1 + "+c"(width_tmp) // %2 + : + : "memory", "cc" + ); +} +#endif // HAS_COPYROW_ERMS + +#ifdef HAS_ARGBCOPYALPHAROW_SSE2 +// width in pixels +void ARGBCopyAlphaRow_SSE2(const uint8* src, uint8* dst, int width) { + asm volatile ( + "pcmpeqb %%xmm0,%%xmm0 \n" + "pslld $0x18,%%xmm0 \n" + "pcmpeqb %%xmm1,%%xmm1 \n" + "psrld $0x8,%%xmm1 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm3 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "movdqu " MEMACCESS(1) ",%%xmm4 \n" + "movdqu " MEMACCESS2(0x10,1) ",%%xmm5 \n" + "pand %%xmm0,%%xmm2 \n" + "pand %%xmm0,%%xmm3 \n" + "pand %%xmm1,%%xmm4 \n" + "pand %%xmm1,%%xmm5 \n" + "por %%xmm4,%%xmm2 \n" + "por %%xmm5,%%xmm3 \n" + "movdqu %%xmm2," MEMACCESS(1) " \n" + "movdqu %%xmm3," MEMACCESS2(0x10,1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} +#endif // HAS_ARGBCOPYALPHAROW_SSE2 + +#ifdef HAS_ARGBCOPYALPHAROW_AVX2 +// width in pixels +void ARGBCopyAlphaRow_AVX2(const uint8* src, uint8* dst, int width) { + asm volatile ( + "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n" + "vpsrld $0x8,%%ymm0,%%ymm0 \n" + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm1 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm2 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "vpblendvb %%ymm0," MEMACCESS(1) ",%%ymm1,%%ymm1 \n" + "vpblendvb %%ymm0," MEMACCESS2(0x20,1) ",%%ymm2,%%ymm2 \n" + "vmovdqu %%ymm1," MEMACCESS(1) " \n" + "vmovdqu %%ymm2," MEMACCESS2(0x20,1) " \n" + "lea " MEMLEA(0x40,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : + : "memory", "cc" + , "xmm0", "xmm1", "xmm2" + ); +} +#endif // HAS_ARGBCOPYALPHAROW_AVX2 + +#ifdef HAS_ARGBCOPYYTOALPHAROW_SSE2 +// width in pixels +void ARGBCopyYToAlphaRow_SSE2(const uint8* src, uint8* dst, int width) { + asm volatile ( + "pcmpeqb %%xmm0,%%xmm0 \n" + "pslld $0x18,%%xmm0 \n" + "pcmpeqb %%xmm1,%%xmm1 \n" + "psrld $0x8,%%xmm1 \n" + LABELALIGN + "1: \n" + "movq " MEMACCESS(0) ",%%xmm2 \n" + "lea " MEMLEA(0x8,0) ",%0 \n" + "punpcklbw %%xmm2,%%xmm2 \n" + "punpckhwd %%xmm2,%%xmm3 \n" + "punpcklwd %%xmm2,%%xmm2 \n" + "movdqu " MEMACCESS(1) ",%%xmm4 \n" + "movdqu " MEMACCESS2(0x10,1) ",%%xmm5 \n" + "pand %%xmm0,%%xmm2 \n" + "pand %%xmm0,%%xmm3 \n" + "pand %%xmm1,%%xmm4 \n" + "pand %%xmm1,%%xmm5 \n" + "por %%xmm4,%%xmm2 \n" + "por %%xmm5,%%xmm3 \n" + "movdqu %%xmm2," MEMACCESS(1) " \n" + "movdqu %%xmm3," MEMACCESS2(0x10,1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} +#endif // HAS_ARGBCOPYYTOALPHAROW_SSE2 + +#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX2 +// width in pixels +void ARGBCopyYToAlphaRow_AVX2(const uint8* src, uint8* dst, int width) { + asm volatile ( + "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n" + "vpsrld $0x8,%%ymm0,%%ymm0 \n" + LABELALIGN + "1: \n" + "vpmovzxbd " MEMACCESS(0) ",%%ymm1 \n" + "vpmovzxbd " MEMACCESS2(0x8,0) ",%%ymm2 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "vpslld $0x18,%%ymm1,%%ymm1 \n" + "vpslld $0x18,%%ymm2,%%ymm2 \n" + "vpblendvb %%ymm0," MEMACCESS(1) ",%%ymm1,%%ymm1 \n" + "vpblendvb %%ymm0," MEMACCESS2(0x20,1) ",%%ymm2,%%ymm2 \n" + "vmovdqu %%ymm1," MEMACCESS(1) " \n" + "vmovdqu %%ymm2," MEMACCESS2(0x20,1) " \n" + "lea " MEMLEA(0x40,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : + : "memory", "cc" + , "xmm0", "xmm1", "xmm2" + ); +} +#endif // HAS_ARGBCOPYYTOALPHAROW_AVX2 + +#ifdef HAS_SETROW_X86 +void SetRow_X86(uint8* dst, uint8 v8, int width) { + size_t width_tmp = (size_t)(width >> 2); + const uint32 v32 = v8 * 0x01010101; // Duplicate byte to all bytes. + asm volatile ( + "rep stosl " MEMSTORESTRING(eax,0) " \n" + : "+D"(dst), // %0 + "+c"(width_tmp) // %1 + : "a"(v32) // %2 + : "memory", "cc"); +} + +void SetRow_ERMS(uint8* dst, uint8 v8, int width) { + size_t width_tmp = (size_t)(width); + asm volatile ( + "rep stosb " MEMSTORESTRING(al,0) " \n" + : "+D"(dst), // %0 + "+c"(width_tmp) // %1 + : "a"(v8) // %2 + : "memory", "cc"); +} + +void ARGBSetRow_X86(uint8* dst_argb, uint32 v32, int width) { + size_t width_tmp = (size_t)(width); + asm volatile ( + "rep stosl " MEMSTORESTRING(eax,0) " \n" + : "+D"(dst_argb), // %0 + "+c"(width_tmp) // %1 + : "a"(v32) // %2 + : "memory", "cc"); +} +#endif // HAS_SETROW_X86 + +#ifdef HAS_YUY2TOYROW_SSE2 +void YUY2ToYRow_SSE2(const uint8* src_yuy2, uint8* dst_y, int pix) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" + "psrlw $0x8,%%xmm5 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "pand %%xmm5,%%xmm0 \n" + "pand %%xmm5,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "memory", "cc" + , "xmm0", "xmm1", "xmm5" + ); +} + +void YUY2ToUVRow_SSE2(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" + "psrlw $0x8,%%xmm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + MEMOPREG(movdqu,0x00,0,4,1,xmm2) // movdqu (%0,%4,1),%%xmm2 + MEMOPREG(movdqu,0x10,0,4,1,xmm3) // movdqu 0x10(%0,%4,1),%%xmm3 + "lea " MEMLEA(0x20,0) ",%0 \n" + "pavgb %%xmm2,%%xmm0 \n" + "pavgb %%xmm3,%%xmm1 \n" + "psrlw $0x8,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "pand %%xmm5,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" + "packuswb %%xmm1,%%xmm1 \n" + "movq %%xmm0," MEMACCESS(1) " \n" + MEMOPMEM(movq,xmm1,0x00,1,2,1) // movq %%xmm1,(%1,%2) + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : "r"((intptr_t)(stride_yuy2)) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} + +void YUY2ToUV422Row_SSE2(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" + "psrlw $0x8,%%xmm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "psrlw $0x8,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "pand %%xmm5,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" + "packuswb %%xmm1,%%xmm1 \n" + "movq %%xmm0," MEMACCESS(1) " \n" + MEMOPMEM(movq,xmm1,0x00,1,2,1) // movq %%xmm1,(%1,%2) + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm5" + ); +} + +void UYVYToYRow_SSE2(const uint8* src_uyvy, uint8* dst_y, int pix) { + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "psrlw $0x8,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "memory", "cc" + , "xmm0", "xmm1" + ); +} + +void UYVYToUVRow_SSE2(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" + "psrlw $0x8,%%xmm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + MEMOPREG(movdqu,0x00,0,4,1,xmm2) // movdqu (%0,%4,1),%%xmm2 + MEMOPREG(movdqu,0x10,0,4,1,xmm3) // movdqu 0x10(%0,%4,1),%%xmm3 + "lea " MEMLEA(0x20,0) ",%0 \n" + "pavgb %%xmm2,%%xmm0 \n" + "pavgb %%xmm3,%%xmm1 \n" + "pand %%xmm5,%%xmm0 \n" + "pand %%xmm5,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "pand %%xmm5,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" + "packuswb %%xmm1,%%xmm1 \n" + "movq %%xmm0," MEMACCESS(1) " \n" + MEMOPMEM(movq,xmm1,0x00,1,2,1) // movq %%xmm1,(%1,%2) + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : "r"((intptr_t)(stride_uyvy)) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} + +void UYVYToUV422Row_SSE2(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" + "psrlw $0x8,%%xmm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "pand %%xmm5,%%xmm0 \n" + "pand %%xmm5,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "pand %%xmm5,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" + "packuswb %%xmm1,%%xmm1 \n" + "movq %%xmm0," MEMACCESS(1) " \n" + MEMOPMEM(movq,xmm1,0x00,1,2,1) // movq %%xmm1,(%1,%2) + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm5" + ); +} +#endif // HAS_YUY2TOYROW_SSE2 + +#ifdef HAS_YUY2TOYROW_AVX2 +void YUY2ToYRow_AVX2(const uint8* src_yuy2, uint8* dst_y, int pix) { + asm volatile ( + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpsrlw $0x8,%%ymm5,%%ymm5 \n" + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "vpand %%ymm5,%%ymm0,%%ymm0 \n" + "vpand %%ymm5,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "memory", "cc" + , "xmm0", "xmm1", "xmm5" + ); +} + +void YUY2ToUVRow_AVX2(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpsrlw $0x8,%%ymm5,%%ymm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" + VMEMOPREG(vpavgb,0x00,0,4,1,ymm0,ymm0) // vpavgb (%0,%4,1),%%ymm0,%%ymm0 + VMEMOPREG(vpavgb,0x20,0,4,1,ymm1,ymm1) + "lea " MEMLEA(0x40,0) ",%0 \n" + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" + "vpsrlw $0x8,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpand %%ymm5,%%ymm0,%%ymm1 \n" + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vextractf128 $0x0,%%ymm1," MEMACCESS(1) " \n" + VEXTOPMEM(vextractf128,0,ymm0,0x00,1,2,1) // vextractf128 $0x0,%%ymm0,(%1,%2,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x20,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : "r"((intptr_t)(stride_yuy2)) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm5" + ); +} + +void YUY2ToUV422Row_AVX2(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpsrlw $0x8,%%ymm5,%%ymm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" + "vpsrlw $0x8,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpand %%ymm5,%%ymm0,%%ymm1 \n" + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vextractf128 $0x0,%%ymm1," MEMACCESS(1) " \n" + VEXTOPMEM(vextractf128,0,ymm0,0x00,1,2,1) // vextractf128 $0x0,%%ymm0,(%1,%2,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x20,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm5" + ); +} + +void UYVYToYRow_AVX2(const uint8* src_uyvy, uint8* dst_y, int pix) { + asm volatile ( + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" + "vpsrlw $0x8,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "memory", "cc" + , "xmm0", "xmm1", "xmm5" + ); +} +void UYVYToUVRow_AVX2(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpsrlw $0x8,%%ymm5,%%ymm5 \n" + "sub %1,%2 \n" + + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" + VMEMOPREG(vpavgb,0x00,0,4,1,ymm0,ymm0) // vpavgb (%0,%4,1),%%ymm0,%%ymm0 + VMEMOPREG(vpavgb,0x20,0,4,1,ymm1,ymm1) + "lea " MEMLEA(0x40,0) ",%0 \n" + "vpand %%ymm5,%%ymm0,%%ymm0 \n" + "vpand %%ymm5,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpand %%ymm5,%%ymm0,%%ymm1 \n" + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vextractf128 $0x0,%%ymm1," MEMACCESS(1) " \n" + VEXTOPMEM(vextractf128,0,ymm0,0x00,1,2,1) // vextractf128 $0x0,%%ymm0,(%1,%2,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x20,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : "r"((intptr_t)(stride_uyvy)) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm5" + ); +} + +void UYVYToUV422Row_AVX2(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpsrlw $0x8,%%ymm5,%%ymm5 \n" + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "vpand %%ymm5,%%ymm0,%%ymm0 \n" + "vpand %%ymm5,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpand %%ymm5,%%ymm0,%%ymm1 \n" + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vextractf128 $0x0,%%ymm1," MEMACCESS(1) " \n" + VEXTOPMEM(vextractf128,0,ymm0,0x00,1,2,1) // vextractf128 $0x0,%%ymm0,(%1,%2,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x20,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm5" + ); +} +#endif // HAS_YUY2TOYROW_AVX2 + +#ifdef HAS_ARGBBLENDROW_SSE2 +// Blend 8 pixels at a time. +void ARGBBlendRow_SSE2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + "pcmpeqb %%xmm7,%%xmm7 \n" + "psrlw $0xf,%%xmm7 \n" + "pcmpeqb %%xmm6,%%xmm6 \n" + "psrlw $0x8,%%xmm6 \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + "psllw $0x8,%%xmm5 \n" + "pcmpeqb %%xmm4,%%xmm4 \n" + "pslld $0x18,%%xmm4 \n" + "sub $0x4,%3 \n" + "jl 49f \n" + + // 4 pixel loop. + LABELALIGN + "41: \n" + "movdqu " MEMACCESS(0) ",%%xmm3 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqa %%xmm3,%%xmm0 \n" + "pxor %%xmm4,%%xmm3 \n" + "movdqu " MEMACCESS(1) ",%%xmm2 \n" + "psrlw $0x8,%%xmm3 \n" + "pshufhw $0xf5,%%xmm3,%%xmm3 \n" + "pshuflw $0xf5,%%xmm3,%%xmm3 \n" + "pand %%xmm6,%%xmm2 \n" + "paddw %%xmm7,%%xmm3 \n" + "pmullw %%xmm3,%%xmm2 \n" + "movdqu " MEMACCESS(1) ",%%xmm1 \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "psrlw $0x8,%%xmm1 \n" + "por %%xmm4,%%xmm0 \n" + "pmullw %%xmm3,%%xmm1 \n" + "psrlw $0x8,%%xmm2 \n" + "paddusb %%xmm2,%%xmm0 \n" + "pand %%xmm5,%%xmm1 \n" + "paddusb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "sub $0x4,%3 \n" + "jge 41b \n" + + "49: \n" + "add $0x3,%3 \n" + "jl 99f \n" + + // 1 pixel loop. + "91: \n" + "movd " MEMACCESS(0) ",%%xmm3 \n" + "lea " MEMLEA(0x4,0) ",%0 \n" + "movdqa %%xmm3,%%xmm0 \n" + "pxor %%xmm4,%%xmm3 \n" + "movd " MEMACCESS(1) ",%%xmm2 \n" + "psrlw $0x8,%%xmm3 \n" + "pshufhw $0xf5,%%xmm3,%%xmm3 \n" + "pshuflw $0xf5,%%xmm3,%%xmm3 \n" + "pand %%xmm6,%%xmm2 \n" + "paddw %%xmm7,%%xmm3 \n" + "pmullw %%xmm3,%%xmm2 \n" + "movd " MEMACCESS(1) ",%%xmm1 \n" + "lea " MEMLEA(0x4,1) ",%1 \n" + "psrlw $0x8,%%xmm1 \n" + "por %%xmm4,%%xmm0 \n" + "pmullw %%xmm3,%%xmm1 \n" + "psrlw $0x8,%%xmm2 \n" + "paddusb %%xmm2,%%xmm0 \n" + "pand %%xmm5,%%xmm1 \n" + "paddusb %%xmm1,%%xmm0 \n" + "movd %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x4,2) ",%2 \n" + "sub $0x1,%3 \n" + "jge 91b \n" + "99: \n" + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} +#endif // HAS_ARGBBLENDROW_SSE2 + +#ifdef HAS_ARGBBLENDROW_SSSE3 +// Shuffle table for isolating alpha. +static uvec8 kShuffleAlpha = { + 3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, + 11u, 0x80, 11u, 0x80, 15u, 0x80, 15u, 0x80 +}; + +// Blend 8 pixels at a time +// Shuffle table for reversing the bytes. + +// Same as SSE2, but replaces +// psrlw xmm3, 8 // alpha +// pshufhw xmm3, xmm3,0F5h // 8 alpha words +// pshuflw xmm3, xmm3,0F5h +// with.. +// pshufb xmm3, kShuffleAlpha // alpha + +void ARGBBlendRow_SSSE3(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + "pcmpeqb %%xmm7,%%xmm7 \n" + "psrlw $0xf,%%xmm7 \n" + "pcmpeqb %%xmm6,%%xmm6 \n" + "psrlw $0x8,%%xmm6 \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + "psllw $0x8,%%xmm5 \n" + "pcmpeqb %%xmm4,%%xmm4 \n" + "pslld $0x18,%%xmm4 \n" + "sub $0x4,%3 \n" + "jl 49f \n" + + // 4 pixel loop. + LABELALIGN + "40: \n" + "movdqu " MEMACCESS(0) ",%%xmm3 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqa %%xmm3,%%xmm0 \n" + "pxor %%xmm4,%%xmm3 \n" + "movdqu " MEMACCESS(1) ",%%xmm2 \n" + "pshufb %4,%%xmm3 \n" + "pand %%xmm6,%%xmm2 \n" + "paddw %%xmm7,%%xmm3 \n" + "pmullw %%xmm3,%%xmm2 \n" + "movdqu " MEMACCESS(1) ",%%xmm1 \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "psrlw $0x8,%%xmm1 \n" + "por %%xmm4,%%xmm0 \n" + "pmullw %%xmm3,%%xmm1 \n" + "psrlw $0x8,%%xmm2 \n" + "paddusb %%xmm2,%%xmm0 \n" + "pand %%xmm5,%%xmm1 \n" + "paddusb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "sub $0x4,%3 \n" + "jge 40b \n" + + "49: \n" + "add $0x3,%3 \n" + "jl 99f \n" + + // 1 pixel loop. + "91: \n" + "movd " MEMACCESS(0) ",%%xmm3 \n" + "lea " MEMLEA(0x4,0) ",%0 \n" + "movdqa %%xmm3,%%xmm0 \n" + "pxor %%xmm4,%%xmm3 \n" + "movd " MEMACCESS(1) ",%%xmm2 \n" + "pshufb %4,%%xmm3 \n" + "pand %%xmm6,%%xmm2 \n" + "paddw %%xmm7,%%xmm3 \n" + "pmullw %%xmm3,%%xmm2 \n" + "movd " MEMACCESS(1) ",%%xmm1 \n" + "lea " MEMLEA(0x4,1) ",%1 \n" + "psrlw $0x8,%%xmm1 \n" + "por %%xmm4,%%xmm0 \n" + "pmullw %%xmm3,%%xmm1 \n" + "psrlw $0x8,%%xmm2 \n" + "paddusb %%xmm2,%%xmm0 \n" + "pand %%xmm5,%%xmm1 \n" + "paddusb %%xmm1,%%xmm0 \n" + "movd %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x4,2) ",%2 \n" + "sub $0x1,%3 \n" + "jge 91b \n" + "99: \n" + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : "m"(kShuffleAlpha) // %4 + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} +#endif // HAS_ARGBBLENDROW_SSSE3 + +#ifdef HAS_ARGBATTENUATEROW_SSE2 +// Attenuate 4 pixels at a time. +void ARGBAttenuateRow_SSE2(const uint8* src_argb, uint8* dst_argb, int width) { + asm volatile ( + "pcmpeqb %%xmm4,%%xmm4 \n" + "pslld $0x18,%%xmm4 \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + "psrld $0x8,%%xmm5 \n" + + // 4 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "punpcklbw %%xmm0,%%xmm0 \n" + "pshufhw $0xff,%%xmm0,%%xmm2 \n" + "pshuflw $0xff,%%xmm2,%%xmm2 \n" + "pmulhuw %%xmm2,%%xmm0 \n" + "movdqu " MEMACCESS(0) ",%%xmm1 \n" + "punpckhbw %%xmm1,%%xmm1 \n" + "pshufhw $0xff,%%xmm1,%%xmm2 \n" + "pshuflw $0xff,%%xmm2,%%xmm2 \n" + "pmulhuw %%xmm2,%%xmm1 \n" + "movdqu " MEMACCESS(0) ",%%xmm2 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "psrlw $0x8,%%xmm0 \n" + "pand %%xmm4,%%xmm2 \n" + "psrlw $0x8,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "pand %%xmm5,%%xmm0 \n" + "por %%xmm2,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x4,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} +#endif // HAS_ARGBATTENUATEROW_SSE2 + +#ifdef HAS_ARGBATTENUATEROW_SSSE3 +// Shuffle table duplicating alpha +static uvec8 kShuffleAlpha0 = { + 3u, 3u, 3u, 3u, 3u, 3u, 128u, 128u, 7u, 7u, 7u, 7u, 7u, 7u, 128u, 128u +}; +static uvec8 kShuffleAlpha1 = { + 11u, 11u, 11u, 11u, 11u, 11u, 128u, 128u, + 15u, 15u, 15u, 15u, 15u, 15u, 128u, 128u +}; +// Attenuate 4 pixels at a time. +void ARGBAttenuateRow_SSSE3(const uint8* src_argb, uint8* dst_argb, int width) { + asm volatile ( + "pcmpeqb %%xmm3,%%xmm3 \n" + "pslld $0x18,%%xmm3 \n" + "movdqa %3,%%xmm4 \n" + "movdqa %4,%%xmm5 \n" + + // 4 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "pshufb %%xmm4,%%xmm0 \n" + "movdqu " MEMACCESS(0) ",%%xmm1 \n" + "punpcklbw %%xmm1,%%xmm1 \n" + "pmulhuw %%xmm1,%%xmm0 \n" + "movdqu " MEMACCESS(0) ",%%xmm1 \n" + "pshufb %%xmm5,%%xmm1 \n" + "movdqu " MEMACCESS(0) ",%%xmm2 \n" + "punpckhbw %%xmm2,%%xmm2 \n" + "pmulhuw %%xmm2,%%xmm1 \n" + "movdqu " MEMACCESS(0) ",%%xmm2 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "pand %%xmm3,%%xmm2 \n" + "psrlw $0x8,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "por %%xmm2,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x4,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "m"(kShuffleAlpha0), // %3 + "m"(kShuffleAlpha1) // %4 + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} +#endif // HAS_ARGBATTENUATEROW_SSSE3 + +#ifdef HAS_ARGBATTENUATEROW_AVX2 +// Shuffle table duplicating alpha. +static const uvec8 kShuffleAlpha_AVX2 = { + 6u, 7u, 6u, 7u, 6u, 7u, 128u, 128u, 14u, 15u, 14u, 15u, 14u, 15u, 128u, 128u +}; +// Attenuate 8 pixels at a time. +void ARGBAttenuateRow_AVX2(const uint8* src_argb, uint8* dst_argb, int width) { + asm volatile ( + "vbroadcastf128 %3,%%ymm4 \n" + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpslld $0x18,%%ymm5,%%ymm5 \n" + "sub %0,%1 \n" + + // 8 pixel loop. + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm6 \n" + "vpunpcklbw %%ymm6,%%ymm6,%%ymm0 \n" + "vpunpckhbw %%ymm6,%%ymm6,%%ymm1 \n" + "vpshufb %%ymm4,%%ymm0,%%ymm2 \n" + "vpshufb %%ymm4,%%ymm1,%%ymm3 \n" + "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n" + "vpand %%ymm5,%%ymm6,%%ymm6 \n" + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" + "vpsrlw $0x8,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vpor %%ymm6,%%ymm0,%%ymm0 \n" + MEMOPMEM(vmovdqu,ymm0,0x00,0,1,1) // vmovdqu %%ymm0,(%0,%1) + "lea " MEMLEA(0x20,0) ",%0 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "m"(kShuffleAlpha_AVX2) // %3 + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" + ); +} +#endif // HAS_ARGBATTENUATEROW_AVX2 + +#ifdef HAS_ARGBUNATTENUATEROW_SSE2 +// Unattenuate 4 pixels at a time. +void ARGBUnattenuateRow_SSE2(const uint8* src_argb, uint8* dst_argb, + int width) { + uintptr_t alpha = 0; + asm volatile ( + // 4 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movzb " MEMACCESS2(0x03,0) ",%3 \n" + "punpcklbw %%xmm0,%%xmm0 \n" + MEMOPREG(movd,0x00,4,3,4,xmm2) // movd 0x0(%4,%3,4),%%xmm2 + "movzb " MEMACCESS2(0x07,0) ",%3 \n" + MEMOPREG(movd,0x00,4,3,4,xmm3) // movd 0x0(%4,%3,4),%%xmm3 + "pshuflw $0x40,%%xmm2,%%xmm2 \n" + "pshuflw $0x40,%%xmm3,%%xmm3 \n" + "movlhps %%xmm3,%%xmm2 \n" + "pmulhuw %%xmm2,%%xmm0 \n" + "movdqu " MEMACCESS(0) ",%%xmm1 \n" + "movzb " MEMACCESS2(0x0b,0) ",%3 \n" + "punpckhbw %%xmm1,%%xmm1 \n" + MEMOPREG(movd,0x00,4,3,4,xmm2) // movd 0x0(%4,%3,4),%%xmm2 + "movzb " MEMACCESS2(0x0f,0) ",%3 \n" + MEMOPREG(movd,0x00,4,3,4,xmm3) // movd 0x0(%4,%3,4),%%xmm3 + "pshuflw $0x40,%%xmm2,%%xmm2 \n" + "pshuflw $0x40,%%xmm3,%%xmm3 \n" + "movlhps %%xmm3,%%xmm2 \n" + "pmulhuw %%xmm2,%%xmm1 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x4,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width), // %2 + "+r"(alpha) // %3 + : "r"(fixed_invtbl8) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} +#endif // HAS_ARGBUNATTENUATEROW_SSE2 + +#ifdef HAS_ARGBUNATTENUATEROW_AVX2 +// Shuffle table duplicating alpha. +static const uvec8 kUnattenShuffleAlpha_AVX2 = { + 0u, 1u, 0u, 1u, 0u, 1u, 6u, 7u, 8u, 9u, 8u, 9u, 8u, 9u, 14u, 15u +}; +// Unattenuate 8 pixels at a time. +void ARGBUnattenuateRow_AVX2(const uint8* src_argb, uint8* dst_argb, + int width) { + uintptr_t alpha = 0; + asm volatile ( + "sub %0,%1 \n" + "vbroadcastf128 %5,%%ymm5 \n" + + // 8 pixel loop. + LABELALIGN + "1: \n" + // replace VPGATHER + "movzb " MEMACCESS2(0x03,0) ",%3 \n" + MEMOPREG(vmovd,0x00,4,3,4,xmm0) // vmovd 0x0(%4,%3,4),%%xmm0 + "movzb " MEMACCESS2(0x07,0) ",%3 \n" + MEMOPREG(vmovd,0x00,4,3,4,xmm1) // vmovd 0x0(%4,%3,4),%%xmm1 + "movzb " MEMACCESS2(0x0b,0) ",%3 \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm6 \n" + MEMOPREG(vmovd,0x00,4,3,4,xmm2) // vmovd 0x0(%4,%3,4),%%xmm2 + "movzb " MEMACCESS2(0x0f,0) ",%3 \n" + MEMOPREG(vmovd,0x00,4,3,4,xmm3) // vmovd 0x0(%4,%3,4),%%xmm3 + "movzb " MEMACCESS2(0x13,0) ",%3 \n" + "vpunpckldq %%xmm3,%%xmm2,%%xmm7 \n" + MEMOPREG(vmovd,0x00,4,3,4,xmm0) // vmovd 0x0(%4,%3,4),%%xmm0 + "movzb " MEMACCESS2(0x17,0) ",%3 \n" + MEMOPREG(vmovd,0x00,4,3,4,xmm1) // vmovd 0x0(%4,%3,4),%%xmm1 + "movzb " MEMACCESS2(0x1b,0) ",%3 \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm0 \n" + MEMOPREG(vmovd,0x00,4,3,4,xmm2) // vmovd 0x0(%4,%3,4),%%xmm2 + "movzb " MEMACCESS2(0x1f,0) ",%3 \n" + MEMOPREG(vmovd,0x00,4,3,4,xmm3) // vmovd 0x0(%4,%3,4),%%xmm3 + "vpunpckldq %%xmm3,%%xmm2,%%xmm2 \n" + "vpunpcklqdq %%xmm7,%%xmm6,%%xmm3 \n" + "vpunpcklqdq %%xmm2,%%xmm0,%%xmm0 \n" + "vinserti128 $0x1,%%xmm0,%%ymm3,%%ymm3 \n" + // end of VPGATHER + + "vmovdqu " MEMACCESS(0) ",%%ymm6 \n" + "vpunpcklbw %%ymm6,%%ymm6,%%ymm0 \n" + "vpunpckhbw %%ymm6,%%ymm6,%%ymm1 \n" + "vpunpcklwd %%ymm3,%%ymm3,%%ymm2 \n" + "vpunpckhwd %%ymm3,%%ymm3,%%ymm3 \n" + "vpshufb %%ymm5,%%ymm2,%%ymm2 \n" + "vpshufb %%ymm5,%%ymm3,%%ymm3 \n" + "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + MEMOPMEM(vmovdqu,ymm0,0x00,0,1,1) // vmovdqu %%ymm0,(%0,%1) + "lea " MEMLEA(0x20,0) ",%0 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width), // %2 + "+r"(alpha) // %3 + : "r"(fixed_invtbl8), // %4 + "m"(kUnattenShuffleAlpha_AVX2) // %5 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} +#endif // HAS_ARGBUNATTENUATEROW_AVX2 + +#ifdef HAS_ARGBGRAYROW_SSSE3 +// Convert 8 ARGB pixels (64 bytes) to 8 Gray ARGB pixels +void ARGBGrayRow_SSSE3(const uint8* src_argb, uint8* dst_argb, int width) { + asm volatile ( + "movdqa %3,%%xmm4 \n" + "movdqa %4,%%xmm5 \n" + + // 8 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "pmaddubsw %%xmm4,%%xmm0 \n" + "pmaddubsw %%xmm4,%%xmm1 \n" + "phaddw %%xmm1,%%xmm0 \n" + "paddw %%xmm5,%%xmm0 \n" + "psrlw $0x7,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "movdqu " MEMACCESS(0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm3 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "psrld $0x18,%%xmm2 \n" + "psrld $0x18,%%xmm3 \n" + "packuswb %%xmm3,%%xmm2 \n" + "packuswb %%xmm2,%%xmm2 \n" + "movdqa %%xmm0,%%xmm3 \n" + "punpcklbw %%xmm0,%%xmm0 \n" + "punpcklbw %%xmm2,%%xmm3 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklwd %%xmm3,%%xmm0 \n" + "punpckhwd %%xmm3,%%xmm1 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "m"(kARGBToYJ), // %3 + "m"(kAddYJ64) // %4 + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} +#endif // HAS_ARGBGRAYROW_SSSE3 + +#ifdef HAS_ARGBSEPIAROW_SSSE3 +// b = (r * 35 + g * 68 + b * 17) >> 7 +// g = (r * 45 + g * 88 + b * 22) >> 7 +// r = (r * 50 + g * 98 + b * 24) >> 7 +// Constant for ARGB color to sepia tone +static vec8 kARGBToSepiaB = { + 17, 68, 35, 0, 17, 68, 35, 0, 17, 68, 35, 0, 17, 68, 35, 0 +}; + +static vec8 kARGBToSepiaG = { + 22, 88, 45, 0, 22, 88, 45, 0, 22, 88, 45, 0, 22, 88, 45, 0 +}; + +static vec8 kARGBToSepiaR = { + 24, 98, 50, 0, 24, 98, 50, 0, 24, 98, 50, 0, 24, 98, 50, 0 +}; + +// Convert 8 ARGB pixels (32 bytes) to 8 Sepia ARGB pixels. +void ARGBSepiaRow_SSSE3(uint8* dst_argb, int width) { + asm volatile ( + "movdqa %2,%%xmm2 \n" + "movdqa %3,%%xmm3 \n" + "movdqa %4,%%xmm4 \n" + + // 8 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm6 \n" + "pmaddubsw %%xmm2,%%xmm0 \n" + "pmaddubsw %%xmm2,%%xmm6 \n" + "phaddw %%xmm6,%%xmm0 \n" + "psrlw $0x7,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "movdqu " MEMACCESS(0) ",%%xmm5 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "pmaddubsw %%xmm3,%%xmm5 \n" + "pmaddubsw %%xmm3,%%xmm1 \n" + "phaddw %%xmm1,%%xmm5 \n" + "psrlw $0x7,%%xmm5 \n" + "packuswb %%xmm5,%%xmm5 \n" + "punpcklbw %%xmm5,%%xmm0 \n" + "movdqu " MEMACCESS(0) ",%%xmm5 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "pmaddubsw %%xmm4,%%xmm5 \n" + "pmaddubsw %%xmm4,%%xmm1 \n" + "phaddw %%xmm1,%%xmm5 \n" + "psrlw $0x7,%%xmm5 \n" + "packuswb %%xmm5,%%xmm5 \n" + "movdqu " MEMACCESS(0) ",%%xmm6 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "psrld $0x18,%%xmm6 \n" + "psrld $0x18,%%xmm1 \n" + "packuswb %%xmm1,%%xmm6 \n" + "packuswb %%xmm6,%%xmm6 \n" + "punpcklbw %%xmm6,%%xmm5 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklwd %%xmm5,%%xmm0 \n" + "punpckhwd %%xmm5,%%xmm1 \n" + "movdqu %%xmm0," MEMACCESS(0) " \n" + "movdqu %%xmm1," MEMACCESS2(0x10,0) " \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "sub $0x8,%1 \n" + "jg 1b \n" + : "+r"(dst_argb), // %0 + "+r"(width) // %1 + : "m"(kARGBToSepiaB), // %2 + "m"(kARGBToSepiaG), // %3 + "m"(kARGBToSepiaR) // %4 + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" + ); +} +#endif // HAS_ARGBSEPIAROW_SSSE3 + +#ifdef HAS_ARGBCOLORMATRIXROW_SSSE3 +// Tranform 8 ARGB pixels (32 bytes) with color matrix. +// Same as Sepia except matrix is provided. +void ARGBColorMatrixRow_SSSE3(const uint8* src_argb, uint8* dst_argb, + const int8* matrix_argb, int width) { + asm volatile ( + "movdqu " MEMACCESS(3) ",%%xmm5 \n" + "pshufd $0x00,%%xmm5,%%xmm2 \n" + "pshufd $0x55,%%xmm5,%%xmm3 \n" + "pshufd $0xaa,%%xmm5,%%xmm4 \n" + "pshufd $0xff,%%xmm5,%%xmm5 \n" + + // 8 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm7 \n" + "pmaddubsw %%xmm2,%%xmm0 \n" + "pmaddubsw %%xmm2,%%xmm7 \n" + "movdqu " MEMACCESS(0) ",%%xmm6 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "pmaddubsw %%xmm3,%%xmm6 \n" + "pmaddubsw %%xmm3,%%xmm1 \n" + "phaddsw %%xmm7,%%xmm0 \n" + "phaddsw %%xmm1,%%xmm6 \n" + "psraw $0x6,%%xmm0 \n" + "psraw $0x6,%%xmm6 \n" + "packuswb %%xmm0,%%xmm0 \n" + "packuswb %%xmm6,%%xmm6 \n" + "punpcklbw %%xmm6,%%xmm0 \n" + "movdqu " MEMACCESS(0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm7 \n" + "pmaddubsw %%xmm4,%%xmm1 \n" + "pmaddubsw %%xmm4,%%xmm7 \n" + "phaddsw %%xmm7,%%xmm1 \n" + "movdqu " MEMACCESS(0) ",%%xmm6 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm7 \n" + "pmaddubsw %%xmm5,%%xmm6 \n" + "pmaddubsw %%xmm5,%%xmm7 \n" + "phaddsw %%xmm7,%%xmm6 \n" + "psraw $0x6,%%xmm1 \n" + "psraw $0x6,%%xmm6 \n" + "packuswb %%xmm1,%%xmm1 \n" + "packuswb %%xmm6,%%xmm6 \n" + "punpcklbw %%xmm6,%%xmm1 \n" + "movdqa %%xmm0,%%xmm6 \n" + "punpcklwd %%xmm1,%%xmm0 \n" + "punpckhwd %%xmm1,%%xmm6 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "movdqu %%xmm6," MEMACCESS2(0x10,1) " \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(matrix_argb) // %3 + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} +#endif // HAS_ARGBCOLORMATRIXROW_SSSE3 + +#ifdef HAS_ARGBQUANTIZEROW_SSE2 +// Quantize 4 ARGB pixels (16 bytes). +void ARGBQuantizeRow_SSE2(uint8* dst_argb, int scale, int interval_size, + int interval_offset, int width) { + asm volatile ( + "movd %2,%%xmm2 \n" + "movd %3,%%xmm3 \n" + "movd %4,%%xmm4 \n" + "pshuflw $0x40,%%xmm2,%%xmm2 \n" + "pshufd $0x44,%%xmm2,%%xmm2 \n" + "pshuflw $0x40,%%xmm3,%%xmm3 \n" + "pshufd $0x44,%%xmm3,%%xmm3 \n" + "pshuflw $0x40,%%xmm4,%%xmm4 \n" + "pshufd $0x44,%%xmm4,%%xmm4 \n" + "pxor %%xmm5,%%xmm5 \n" + "pcmpeqb %%xmm6,%%xmm6 \n" + "pslld $0x18,%%xmm6 \n" + + // 4 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "punpcklbw %%xmm5,%%xmm0 \n" + "pmulhuw %%xmm2,%%xmm0 \n" + "movdqu " MEMACCESS(0) ",%%xmm1 \n" + "punpckhbw %%xmm5,%%xmm1 \n" + "pmulhuw %%xmm2,%%xmm1 \n" + "pmullw %%xmm3,%%xmm0 \n" + "movdqu " MEMACCESS(0) ",%%xmm7 \n" + "pmullw %%xmm3,%%xmm1 \n" + "pand %%xmm6,%%xmm7 \n" + "paddw %%xmm4,%%xmm0 \n" + "paddw %%xmm4,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "por %%xmm7,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(0) " \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "sub $0x4,%1 \n" + "jg 1b \n" + : "+r"(dst_argb), // %0 + "+r"(width) // %1 + : "r"(scale), // %2 + "r"(interval_size), // %3 + "r"(interval_offset) // %4 + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} +#endif // HAS_ARGBQUANTIZEROW_SSE2 + +#ifdef HAS_ARGBSHADEROW_SSE2 +// Shade 4 pixels at a time by specified value. +void ARGBShadeRow_SSE2(const uint8* src_argb, uint8* dst_argb, int width, + uint32 value) { + asm volatile ( + "movd %3,%%xmm2 \n" + "punpcklbw %%xmm2,%%xmm2 \n" + "punpcklqdq %%xmm2,%%xmm2 \n" + + // 4 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklbw %%xmm0,%%xmm0 \n" + "punpckhbw %%xmm1,%%xmm1 \n" + "pmulhuw %%xmm2,%%xmm0 \n" + "pmulhuw %%xmm2,%%xmm1 \n" + "psrlw $0x8,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x4,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(value) // %3 + : "memory", "cc" + , "xmm0", "xmm1", "xmm2" + ); +} +#endif // HAS_ARGBSHADEROW_SSE2 + +#ifdef HAS_ARGBMULTIPLYROW_SSE2 +// Multiply 2 rows of ARGB pixels together, 4 pixels at a time. +void ARGBMultiplyRow_SSE2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + "pxor %%xmm5,%%xmm5 \n" + + // 4 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqu " MEMACCESS(1) ",%%xmm2 \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "movdqu %%xmm0,%%xmm1 \n" + "movdqu %%xmm2,%%xmm3 \n" + "punpcklbw %%xmm0,%%xmm0 \n" + "punpckhbw %%xmm1,%%xmm1 \n" + "punpcklbw %%xmm5,%%xmm2 \n" + "punpckhbw %%xmm5,%%xmm3 \n" + "pmulhuw %%xmm2,%%xmm0 \n" + "pmulhuw %%xmm3,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "sub $0x4,%3 \n" + "jg 1b \n" + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} +#endif // HAS_ARGBMULTIPLYROW_SSE2 + +#ifdef HAS_ARGBMULTIPLYROW_AVX2 +// Multiply 2 rows of ARGB pixels together, 8 pixels at a time. +void ARGBMultiplyRow_AVX2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + "vpxor %%ymm5,%%ymm5,%%ymm5 \n" + + // 4 pixel loop. + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "vmovdqu " MEMACCESS(1) ",%%ymm3 \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "vpunpcklbw %%ymm1,%%ymm1,%%ymm0 \n" + "vpunpckhbw %%ymm1,%%ymm1,%%ymm1 \n" + "vpunpcklbw %%ymm5,%%ymm3,%%ymm2 \n" + "vpunpckhbw %%ymm5,%%ymm3,%%ymm3 \n" + "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x20,2) ",%2 \n" + "sub $0x8,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc" +#if defined(__AVX2__) + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" +#endif + ); +} +#endif // HAS_ARGBMULTIPLYROW_AVX2 + +#ifdef HAS_ARGBADDROW_SSE2 +// Add 2 rows of ARGB pixels together, 4 pixels at a time. +void ARGBAddRow_SSE2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + // 4 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqu " MEMACCESS(1) ",%%xmm1 \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "paddusb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "sub $0x4,%3 \n" + "jg 1b \n" + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc" + , "xmm0", "xmm1" + ); +} +#endif // HAS_ARGBADDROW_SSE2 + +#ifdef HAS_ARGBADDROW_AVX2 +// Add 2 rows of ARGB pixels together, 4 pixels at a time. +void ARGBAddRow_AVX2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + // 4 pixel loop. + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "vpaddusb " MEMACCESS(1) ",%%ymm0,%%ymm0 \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "vmovdqu %%ymm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x20,2) ",%2 \n" + "sub $0x8,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc" + , "xmm0" + ); +} +#endif // HAS_ARGBADDROW_AVX2 + +#ifdef HAS_ARGBSUBTRACTROW_SSE2 +// Subtract 2 rows of ARGB pixels, 4 pixels at a time. +void ARGBSubtractRow_SSE2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + // 4 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqu " MEMACCESS(1) ",%%xmm1 \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "psubusb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "sub $0x4,%3 \n" + "jg 1b \n" + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc" + , "xmm0", "xmm1" + ); +} +#endif // HAS_ARGBSUBTRACTROW_SSE2 + +#ifdef HAS_ARGBSUBTRACTROW_AVX2 +// Subtract 2 rows of ARGB pixels, 8 pixels at a time. +void ARGBSubtractRow_AVX2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + // 4 pixel loop. + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "vpsubusb " MEMACCESS(1) ",%%ymm0,%%ymm0 \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "vmovdqu %%ymm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x20,2) ",%2 \n" + "sub $0x8,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc" + , "xmm0" + ); +} +#endif // HAS_ARGBSUBTRACTROW_AVX2 + +#ifdef HAS_SOBELXROW_SSE2 +// SobelX as a matrix is +// -1 0 1 +// -2 0 2 +// -1 0 1 +void SobelXRow_SSE2(const uint8* src_y0, const uint8* src_y1, + const uint8* src_y2, uint8* dst_sobelx, int width) { + asm volatile ( + "sub %0,%1 \n" + "sub %0,%2 \n" + "sub %0,%3 \n" + "pxor %%xmm5,%%xmm5 \n" + + // 8 pixel loop. + LABELALIGN + "1: \n" + "movq " MEMACCESS(0) ",%%xmm0 \n" + "movq " MEMACCESS2(0x2,0) ",%%xmm1 \n" + "punpcklbw %%xmm5,%%xmm0 \n" + "punpcklbw %%xmm5,%%xmm1 \n" + "psubw %%xmm1,%%xmm0 \n" + MEMOPREG(movq,0x00,0,1,1,xmm1) // movq (%0,%1,1),%%xmm1 + MEMOPREG(movq,0x02,0,1,1,xmm2) // movq 0x2(%0,%1,1),%%xmm2 + "punpcklbw %%xmm5,%%xmm1 \n" + "punpcklbw %%xmm5,%%xmm2 \n" + "psubw %%xmm2,%%xmm1 \n" + MEMOPREG(movq,0x00,0,2,1,xmm2) // movq (%0,%2,1),%%xmm2 + MEMOPREG(movq,0x02,0,2,1,xmm3) // movq 0x2(%0,%2,1),%%xmm3 + "punpcklbw %%xmm5,%%xmm2 \n" + "punpcklbw %%xmm5,%%xmm3 \n" + "psubw %%xmm3,%%xmm2 \n" + "paddw %%xmm2,%%xmm0 \n" + "paddw %%xmm1,%%xmm0 \n" + "paddw %%xmm1,%%xmm0 \n" + "pxor %%xmm1,%%xmm1 \n" + "psubw %%xmm0,%%xmm1 \n" + "pmaxsw %%xmm1,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + MEMOPMEM(movq,xmm0,0x00,0,3,1) // movq %%xmm0,(%0,%3,1) + "lea " MEMLEA(0x8,0) ",%0 \n" + "sub $0x8,%4 \n" + "jg 1b \n" + : "+r"(src_y0), // %0 + "+r"(src_y1), // %1 + "+r"(src_y2), // %2 + "+r"(dst_sobelx), // %3 + "+r"(width) // %4 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} +#endif // HAS_SOBELXROW_SSE2 + +#ifdef HAS_SOBELYROW_SSE2 +// SobelY as a matrix is +// -1 -2 -1 +// 0 0 0 +// 1 2 1 +void SobelYRow_SSE2(const uint8* src_y0, const uint8* src_y1, + uint8* dst_sobely, int width) { + asm volatile ( + "sub %0,%1 \n" + "sub %0,%2 \n" + "pxor %%xmm5,%%xmm5 \n" + + // 8 pixel loop. + LABELALIGN + "1: \n" + "movq " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movq,0x00,0,1,1,xmm1) // movq (%0,%1,1),%%xmm1 + "punpcklbw %%xmm5,%%xmm0 \n" + "punpcklbw %%xmm5,%%xmm1 \n" + "psubw %%xmm1,%%xmm0 \n" + "movq " MEMACCESS2(0x1,0) ",%%xmm1 \n" + MEMOPREG(movq,0x01,0,1,1,xmm2) // movq 0x1(%0,%1,1),%%xmm2 + "punpcklbw %%xmm5,%%xmm1 \n" + "punpcklbw %%xmm5,%%xmm2 \n" + "psubw %%xmm2,%%xmm1 \n" + "movq " MEMACCESS2(0x2,0) ",%%xmm2 \n" + MEMOPREG(movq,0x02,0,1,1,xmm3) // movq 0x2(%0,%1,1),%%xmm3 + "punpcklbw %%xmm5,%%xmm2 \n" + "punpcklbw %%xmm5,%%xmm3 \n" + "psubw %%xmm3,%%xmm2 \n" + "paddw %%xmm2,%%xmm0 \n" + "paddw %%xmm1,%%xmm0 \n" + "paddw %%xmm1,%%xmm0 \n" + "pxor %%xmm1,%%xmm1 \n" + "psubw %%xmm0,%%xmm1 \n" + "pmaxsw %%xmm1,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + MEMOPMEM(movq,xmm0,0x00,0,2,1) // movq %%xmm0,(%0,%2,1) + "lea " MEMLEA(0x8,0) ",%0 \n" + "sub $0x8,%3 \n" + "jg 1b \n" + : "+r"(src_y0), // %0 + "+r"(src_y1), // %1 + "+r"(dst_sobely), // %2 + "+r"(width) // %3 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} +#endif // HAS_SOBELYROW_SSE2 + +#ifdef HAS_SOBELROW_SSE2 +// Adds Sobel X and Sobel Y and stores Sobel into ARGB. +// A = 255 +// R = Sobel +// G = Sobel +// B = Sobel +void SobelRow_SSE2(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width) { + asm volatile ( + "sub %0,%1 \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + "pslld $0x18,%%xmm5 \n" + + // 8 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,0,1,1,xmm1) // movdqu (%0,%1,1),%%xmm1 + "lea " MEMLEA(0x10,0) ",%0 \n" + "paddusb %%xmm1,%%xmm0 \n" + "movdqa %%xmm0,%%xmm2 \n" + "punpcklbw %%xmm0,%%xmm2 \n" + "punpckhbw %%xmm0,%%xmm0 \n" + "movdqa %%xmm2,%%xmm1 \n" + "punpcklwd %%xmm2,%%xmm1 \n" + "punpckhwd %%xmm2,%%xmm2 \n" + "por %%xmm5,%%xmm1 \n" + "por %%xmm5,%%xmm2 \n" + "movdqa %%xmm0,%%xmm3 \n" + "punpcklwd %%xmm0,%%xmm3 \n" + "punpckhwd %%xmm0,%%xmm0 \n" + "por %%xmm5,%%xmm3 \n" + "por %%xmm5,%%xmm0 \n" + "movdqu %%xmm1," MEMACCESS(2) " \n" + "movdqu %%xmm2," MEMACCESS2(0x10,2) " \n" + "movdqu %%xmm3," MEMACCESS2(0x20,2) " \n" + "movdqu %%xmm0," MEMACCESS2(0x30,2) " \n" + "lea " MEMLEA(0x40,2) ",%2 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_sobelx), // %0 + "+r"(src_sobely), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} +#endif // HAS_SOBELROW_SSE2 + +#ifdef HAS_SOBELTOPLANEROW_SSE2 +// Adds Sobel X and Sobel Y and stores Sobel into a plane. +void SobelToPlaneRow_SSE2(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_y, int width) { + asm volatile ( + "sub %0,%1 \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + "pslld $0x18,%%xmm5 \n" + + // 8 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,0,1,1,xmm1) // movdqu (%0,%1,1),%%xmm1 + "lea " MEMLEA(0x10,0) ",%0 \n" + "paddusb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_sobelx), // %0 + "+r"(src_sobely), // %1 + "+r"(dst_y), // %2 + "+r"(width) // %3 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1" + ); +} +#endif // HAS_SOBELTOPLANEROW_SSE2 + +#ifdef HAS_SOBELXYROW_SSE2 +// Mixes Sobel X, Sobel Y and Sobel into ARGB. +// A = 255 +// R = Sobel X +// G = Sobel +// B = Sobel Y +void SobelXYRow_SSE2(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width) { + asm volatile ( + "sub %0,%1 \n" + "pcmpeqb %%xmm5,%%xmm5 \n" + + // 8 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,0,1,1,xmm1) // movdqu (%0,%1,1),%%xmm1 + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqa %%xmm0,%%xmm2 \n" + "paddusb %%xmm1,%%xmm2 \n" + "movdqa %%xmm0,%%xmm3 \n" + "punpcklbw %%xmm5,%%xmm3 \n" + "punpckhbw %%xmm5,%%xmm0 \n" + "movdqa %%xmm1,%%xmm4 \n" + "punpcklbw %%xmm2,%%xmm4 \n" + "punpckhbw %%xmm2,%%xmm1 \n" + "movdqa %%xmm4,%%xmm6 \n" + "punpcklwd %%xmm3,%%xmm6 \n" + "punpckhwd %%xmm3,%%xmm4 \n" + "movdqa %%xmm1,%%xmm7 \n" + "punpcklwd %%xmm0,%%xmm7 \n" + "punpckhwd %%xmm0,%%xmm1 \n" + "movdqu %%xmm6," MEMACCESS(2) " \n" + "movdqu %%xmm4," MEMACCESS2(0x10,2) " \n" + "movdqu %%xmm7," MEMACCESS2(0x20,2) " \n" + "movdqu %%xmm1," MEMACCESS2(0x30,2) " \n" + "lea " MEMLEA(0x40,2) ",%2 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_sobelx), // %0 + "+r"(src_sobely), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} +#endif // HAS_SOBELXYROW_SSE2 + +#ifdef HAS_COMPUTECUMULATIVESUMROW_SSE2 +// Creates a table of cumulative sums where each value is a sum of all values +// above and to the left of the value, inclusive of the value. +void ComputeCumulativeSumRow_SSE2(const uint8* row, int32* cumsum, + const int32* previous_cumsum, int width) { + asm volatile ( + "pxor %%xmm0,%%xmm0 \n" + "pxor %%xmm1,%%xmm1 \n" + "sub $0x4,%3 \n" + "jl 49f \n" + "test $0xf,%1 \n" + "jne 49f \n" + + // 4 pixel loop \n" + LABELALIGN + "40: \n" + "movdqu " MEMACCESS(0) ",%%xmm2 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqa %%xmm2,%%xmm4 \n" + "punpcklbw %%xmm1,%%xmm2 \n" + "movdqa %%xmm2,%%xmm3 \n" + "punpcklwd %%xmm1,%%xmm2 \n" + "punpckhwd %%xmm1,%%xmm3 \n" + "punpckhbw %%xmm1,%%xmm4 \n" + "movdqa %%xmm4,%%xmm5 \n" + "punpcklwd %%xmm1,%%xmm4 \n" + "punpckhwd %%xmm1,%%xmm5 \n" + "paddd %%xmm2,%%xmm0 \n" + "movdqu " MEMACCESS(2) ",%%xmm2 \n" + "paddd %%xmm0,%%xmm2 \n" + "paddd %%xmm3,%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,2) ",%%xmm3 \n" + "paddd %%xmm0,%%xmm3 \n" + "paddd %%xmm4,%%xmm0 \n" + "movdqu " MEMACCESS2(0x20,2) ",%%xmm4 \n" + "paddd %%xmm0,%%xmm4 \n" + "paddd %%xmm5,%%xmm0 \n" + "movdqu " MEMACCESS2(0x30,2) ",%%xmm5 \n" + "lea " MEMLEA(0x40,2) ",%2 \n" + "paddd %%xmm0,%%xmm5 \n" + "movdqu %%xmm2," MEMACCESS(1) " \n" + "movdqu %%xmm3," MEMACCESS2(0x10,1) " \n" + "movdqu %%xmm4," MEMACCESS2(0x20,1) " \n" + "movdqu %%xmm5," MEMACCESS2(0x30,1) " \n" + "lea " MEMLEA(0x40,1) ",%1 \n" + "sub $0x4,%3 \n" + "jge 40b \n" + + "49: \n" + "add $0x3,%3 \n" + "jl 19f \n" + + // 1 pixel loop \n" + LABELALIGN + "10: \n" + "movd " MEMACCESS(0) ",%%xmm2 \n" + "lea " MEMLEA(0x4,0) ",%0 \n" + "punpcklbw %%xmm1,%%xmm2 \n" + "punpcklwd %%xmm1,%%xmm2 \n" + "paddd %%xmm2,%%xmm0 \n" + "movdqu " MEMACCESS(2) ",%%xmm2 \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "paddd %%xmm0,%%xmm2 \n" + "movdqu %%xmm2," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x1,%3 \n" + "jge 10b \n" + + "19: \n" + : "+r"(row), // %0 + "+r"(cumsum), // %1 + "+r"(previous_cumsum), // %2 + "+r"(width) // %3 + : + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} +#endif // HAS_COMPUTECUMULATIVESUMROW_SSE2 + +#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 +void CumulativeSumToAverageRow_SSE2(const int32* topleft, const int32* botleft, + int width, int area, uint8* dst, + int count) { + asm volatile ( + "movd %5,%%xmm5 \n" + "cvtdq2ps %%xmm5,%%xmm5 \n" + "rcpss %%xmm5,%%xmm4 \n" + "pshufd $0x0,%%xmm4,%%xmm4 \n" + "sub $0x4,%3 \n" + "jl 49f \n" + "cmpl $0x80,%5 \n" + "ja 40f \n" + + "pshufd $0x0,%%xmm5,%%xmm5 \n" + "pcmpeqb %%xmm6,%%xmm6 \n" + "psrld $0x10,%%xmm6 \n" + "cvtdq2ps %%xmm6,%%xmm6 \n" + "addps %%xmm6,%%xmm5 \n" + "mulps %%xmm4,%%xmm5 \n" + "cvtps2dq %%xmm5,%%xmm5 \n" + "packssdw %%xmm5,%%xmm5 \n" + + // 4 pixel small loop \n" + LABELALIGN + "4: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" + MEMOPREG(psubd,0x00,0,4,4,xmm0) // psubd 0x00(%0,%4,4),%%xmm0 + MEMOPREG(psubd,0x10,0,4,4,xmm1) // psubd 0x10(%0,%4,4),%%xmm1 + MEMOPREG(psubd,0x20,0,4,4,xmm2) // psubd 0x20(%0,%4,4),%%xmm2 + MEMOPREG(psubd,0x30,0,4,4,xmm3) // psubd 0x30(%0,%4,4),%%xmm3 + "lea " MEMLEA(0x40,0) ",%0 \n" + "psubd " MEMACCESS(1) ",%%xmm0 \n" + "psubd " MEMACCESS2(0x10,1) ",%%xmm1 \n" + "psubd " MEMACCESS2(0x20,1) ",%%xmm2 \n" + "psubd " MEMACCESS2(0x30,1) ",%%xmm3 \n" + MEMOPREG(paddd,0x00,1,4,4,xmm0) // paddd 0x00(%1,%4,4),%%xmm0 + MEMOPREG(paddd,0x10,1,4,4,xmm1) // paddd 0x10(%1,%4,4),%%xmm1 + MEMOPREG(paddd,0x20,1,4,4,xmm2) // paddd 0x20(%1,%4,4),%%xmm2 + MEMOPREG(paddd,0x30,1,4,4,xmm3) // paddd 0x30(%1,%4,4),%%xmm3 + "lea " MEMLEA(0x40,1) ",%1 \n" + "packssdw %%xmm1,%%xmm0 \n" + "packssdw %%xmm3,%%xmm2 \n" + "pmulhuw %%xmm5,%%xmm0 \n" + "pmulhuw %%xmm5,%%xmm2 \n" + "packuswb %%xmm2,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "sub $0x4,%3 \n" + "jge 4b \n" + "jmp 49f \n" + + // 4 pixel loop \n" + LABELALIGN + "40: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" + "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" + MEMOPREG(psubd,0x00,0,4,4,xmm0) // psubd 0x00(%0,%4,4),%%xmm0 + MEMOPREG(psubd,0x10,0,4,4,xmm1) // psubd 0x10(%0,%4,4),%%xmm1 + MEMOPREG(psubd,0x20,0,4,4,xmm2) // psubd 0x20(%0,%4,4),%%xmm2 + MEMOPREG(psubd,0x30,0,4,4,xmm3) // psubd 0x30(%0,%4,4),%%xmm3 + "lea " MEMLEA(0x40,0) ",%0 \n" + "psubd " MEMACCESS(1) ",%%xmm0 \n" + "psubd " MEMACCESS2(0x10,1) ",%%xmm1 \n" + "psubd " MEMACCESS2(0x20,1) ",%%xmm2 \n" + "psubd " MEMACCESS2(0x30,1) ",%%xmm3 \n" + MEMOPREG(paddd,0x00,1,4,4,xmm0) // paddd 0x00(%1,%4,4),%%xmm0 + MEMOPREG(paddd,0x10,1,4,4,xmm1) // paddd 0x10(%1,%4,4),%%xmm1 + MEMOPREG(paddd,0x20,1,4,4,xmm2) // paddd 0x20(%1,%4,4),%%xmm2 + MEMOPREG(paddd,0x30,1,4,4,xmm3) // paddd 0x30(%1,%4,4),%%xmm3 + "lea " MEMLEA(0x40,1) ",%1 \n" + "cvtdq2ps %%xmm0,%%xmm0 \n" + "cvtdq2ps %%xmm1,%%xmm1 \n" + "mulps %%xmm4,%%xmm0 \n" + "mulps %%xmm4,%%xmm1 \n" + "cvtdq2ps %%xmm2,%%xmm2 \n" + "cvtdq2ps %%xmm3,%%xmm3 \n" + "mulps %%xmm4,%%xmm2 \n" + "mulps %%xmm4,%%xmm3 \n" + "cvtps2dq %%xmm0,%%xmm0 \n" + "cvtps2dq %%xmm1,%%xmm1 \n" + "cvtps2dq %%xmm2,%%xmm2 \n" + "cvtps2dq %%xmm3,%%xmm3 \n" + "packssdw %%xmm1,%%xmm0 \n" + "packssdw %%xmm3,%%xmm2 \n" + "packuswb %%xmm2,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "sub $0x4,%3 \n" + "jge 40b \n" + + "49: \n" + "add $0x3,%3 \n" + "jl 19f \n" + + // 1 pixel loop \n" + LABELALIGN + "10: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(psubd,0x00,0,4,4,xmm0) // psubd 0x00(%0,%4,4),%%xmm0 + "lea " MEMLEA(0x10,0) ",%0 \n" + "psubd " MEMACCESS(1) ",%%xmm0 \n" + MEMOPREG(paddd,0x00,1,4,4,xmm0) // paddd 0x00(%1,%4,4),%%xmm0 + "lea " MEMLEA(0x10,1) ",%1 \n" + "cvtdq2ps %%xmm0,%%xmm0 \n" + "mulps %%xmm4,%%xmm0 \n" + "cvtps2dq %%xmm0,%%xmm0 \n" + "packssdw %%xmm0,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "movd %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x4,2) ",%2 \n" + "sub $0x1,%3 \n" + "jge 10b \n" + "19: \n" + : "+r"(topleft), // %0 + "+r"(botleft), // %1 + "+r"(dst), // %2 + "+rm"(count) // %3 + : "r"((intptr_t)(width)), // %4 + "rm"(area) // %5 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" + ); +} +#endif // HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 + +#ifdef HAS_ARGBAFFINEROW_SSE2 +// Copy ARGB pixels from source image with slope to a row of destination. +LIBYUV_API +void ARGBAffineRow_SSE2(const uint8* src_argb, int src_argb_stride, + uint8* dst_argb, const float* src_dudv, int width) { + intptr_t src_argb_stride_temp = src_argb_stride; + intptr_t temp = 0; + asm volatile ( + "movq " MEMACCESS(3) ",%%xmm2 \n" + "movq " MEMACCESS2(0x08,3) ",%%xmm7 \n" + "shl $0x10,%1 \n" + "add $0x4,%1 \n" + "movd %1,%%xmm5 \n" + "sub $0x4,%4 \n" + "jl 49f \n" + + "pshufd $0x44,%%xmm7,%%xmm7 \n" + "pshufd $0x0,%%xmm5,%%xmm5 \n" + "movdqa %%xmm2,%%xmm0 \n" + "addps %%xmm7,%%xmm0 \n" + "movlhps %%xmm0,%%xmm2 \n" + "movdqa %%xmm7,%%xmm4 \n" + "addps %%xmm4,%%xmm4 \n" + "movdqa %%xmm2,%%xmm3 \n" + "addps %%xmm4,%%xmm3 \n" + "addps %%xmm4,%%xmm4 \n" + + // 4 pixel loop \n" + LABELALIGN + "40: \n" + "cvttps2dq %%xmm2,%%xmm0 \n" // x, y float to int first 2 + "cvttps2dq %%xmm3,%%xmm1 \n" // x, y float to int next 2 + "packssdw %%xmm1,%%xmm0 \n" // x, y as 8 shorts + "pmaddwd %%xmm5,%%xmm0 \n" // off = x * 4 + y * stride + "movd %%xmm0,%k1 \n" + "pshufd $0x39,%%xmm0,%%xmm0 \n" + "movd %%xmm0,%k5 \n" + "pshufd $0x39,%%xmm0,%%xmm0 \n" + MEMOPREG(movd,0x00,0,1,1,xmm1) // movd (%0,%1,1),%%xmm1 + MEMOPREG(movd,0x00,0,5,1,xmm6) // movd (%0,%5,1),%%xmm6 + "punpckldq %%xmm6,%%xmm1 \n" + "addps %%xmm4,%%xmm2 \n" + "movq %%xmm1," MEMACCESS(2) " \n" + "movd %%xmm0,%k1 \n" + "pshufd $0x39,%%xmm0,%%xmm0 \n" + "movd %%xmm0,%k5 \n" + MEMOPREG(movd,0x00,0,1,1,xmm0) // movd (%0,%1,1),%%xmm0 + MEMOPREG(movd,0x00,0,5,1,xmm6) // movd (%0,%5,1),%%xmm6 + "punpckldq %%xmm6,%%xmm0 \n" + "addps %%xmm4,%%xmm3 \n" + "movq %%xmm0," MEMACCESS2(0x08,2) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "sub $0x4,%4 \n" + "jge 40b \n" + + "49: \n" + "add $0x3,%4 \n" + "jl 19f \n" + + // 1 pixel loop \n" + LABELALIGN + "10: \n" + "cvttps2dq %%xmm2,%%xmm0 \n" + "packssdw %%xmm0,%%xmm0 \n" + "pmaddwd %%xmm5,%%xmm0 \n" + "addps %%xmm7,%%xmm2 \n" + "movd %%xmm0,%k1 \n" + MEMOPREG(movd,0x00,0,1,1,xmm0) // movd (%0,%1,1),%%xmm0 + "movd %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x04,2) ",%2 \n" + "sub $0x1,%4 \n" + "jge 10b \n" + "19: \n" + : "+r"(src_argb), // %0 + "+r"(src_argb_stride_temp), // %1 + "+r"(dst_argb), // %2 + "+r"(src_dudv), // %3 + "+rm"(width), // %4 + "+r"(temp) // %5 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} +#endif // HAS_ARGBAFFINEROW_SSE2 + +#ifdef HAS_INTERPOLATEROW_SSSE3 +// Bilinear filter 16x2 -> 16x1 +void InterpolateRow_SSSE3(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) { + asm volatile ( + "sub %1,%0 \n" + "shr %3 \n" + "cmp $0x0,%3 \n" + "je 100f \n" + "cmp $0x20,%3 \n" + "je 75f \n" + "cmp $0x40,%3 \n" + "je 50f \n" + "cmp $0x60,%3 \n" + "je 25f \n" + + "movd %3,%%xmm0 \n" + "neg %3 \n" + "add $0x80,%3 \n" + "movd %3,%%xmm5 \n" + "punpcklbw %%xmm0,%%xmm5 \n" + "punpcklwd %%xmm5,%%xmm5 \n" + "pshufd $0x0,%%xmm5,%%xmm5 \n" + + // General purpose row blend. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(1) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,1,4,1,xmm2) + "movdqa %%xmm0,%%xmm1 \n" + "punpcklbw %%xmm2,%%xmm0 \n" + "punpckhbw %%xmm2,%%xmm1 \n" + "pmaddubsw %%xmm5,%%xmm0 \n" + "pmaddubsw %%xmm5,%%xmm1 \n" + "psrlw $0x7,%%xmm0 \n" + "psrlw $0x7,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + MEMOPMEM(movdqu,xmm0,0x00,1,0,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "jmp 99f \n" + + // Blend 25 / 75. + LABELALIGN + "25: \n" + "movdqu " MEMACCESS(1) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,1,4,1,xmm1) + "pavgb %%xmm1,%%xmm0 \n" + "pavgb %%xmm1,%%xmm0 \n" + MEMOPMEM(movdqu,xmm0,0x00,1,0,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 25b \n" + "jmp 99f \n" + + // Blend 50 / 50. + LABELALIGN + "50: \n" + "movdqu " MEMACCESS(1) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,1,4,1,xmm1) + "pavgb %%xmm1,%%xmm0 \n" + MEMOPMEM(movdqu,xmm0,0x00,1,0,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 50b \n" + "jmp 99f \n" + + // Blend 75 / 25. + LABELALIGN + "75: \n" + "movdqu " MEMACCESS(1) ",%%xmm1 \n" + MEMOPREG(movdqu,0x00,1,4,1,xmm0) + "pavgb %%xmm1,%%xmm0 \n" + "pavgb %%xmm1,%%xmm0 \n" + MEMOPMEM(movdqu,xmm0,0x00,1,0,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 75b \n" + "jmp 99f \n" + + // Blend 100 / 0 - Copy row unchanged. + LABELALIGN + "100: \n" + "movdqu " MEMACCESS(1) ",%%xmm0 \n" + MEMOPMEM(movdqu,xmm0,0x00,1,0,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 100b \n" + + "99: \n" + : "+r"(dst_ptr), // %0 + "+r"(src_ptr), // %1 + "+r"(dst_width), // %2 + "+r"(source_y_fraction) // %3 + : "r"((intptr_t)(src_stride)) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm5" + ); +} +#endif // HAS_INTERPOLATEROW_SSSE3 + +#ifdef HAS_INTERPOLATEROW_AVX2 +// Bilinear filter 32x2 -> 32x1 +void InterpolateRow_AVX2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) { + asm volatile ( + "shr %3 \n" + "cmp $0x0,%3 \n" + "je 100f \n" + "sub %1,%0 \n" + "cmp $0x20,%3 \n" + "je 75f \n" + "cmp $0x40,%3 \n" + "je 50f \n" + "cmp $0x60,%3 \n" + "je 25f \n" + + "vmovd %3,%%xmm0 \n" + "neg %3 \n" + "add $0x80,%3 \n" + "vmovd %3,%%xmm5 \n" + "vpunpcklbw %%xmm0,%%xmm5,%%xmm5 \n" + "vpunpcklwd %%xmm5,%%xmm5,%%xmm5 \n" + "vpxor %%ymm0,%%ymm0,%%ymm0 \n" + "vpermd %%ymm5,%%ymm0,%%ymm5 \n" + + // General purpose row blend. + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(1) ",%%ymm0 \n" + MEMOPREG(vmovdqu,0x00,1,4,1,ymm2) + "vpunpckhbw %%ymm2,%%ymm0,%%ymm1 \n" + "vpunpcklbw %%ymm2,%%ymm0,%%ymm0 \n" + "vpmaddubsw %%ymm5,%%ymm0,%%ymm0 \n" + "vpmaddubsw %%ymm5,%%ymm1,%%ymm1 \n" + "vpsrlw $0x7,%%ymm0,%%ymm0 \n" + "vpsrlw $0x7,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + MEMOPMEM(vmovdqu,ymm0,0x00,1,0,1) + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + "jmp 99f \n" + + // Blend 25 / 75. + LABELALIGN + "25: \n" + "vmovdqu " MEMACCESS(1) ",%%ymm0 \n" + MEMOPREG(vmovdqu,0x00,1,4,1,ymm1) + "vpavgb %%ymm1,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm1,%%ymm0,%%ymm0 \n" + MEMOPMEM(vmovdqu,ymm0,0x00,1,0,1) + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x20,%2 \n" + "jg 25b \n" + "jmp 99f \n" + + // Blend 50 / 50. + LABELALIGN + "50: \n" + "vmovdqu " MEMACCESS(1) ",%%ymm0 \n" + VMEMOPREG(vpavgb,0x00,1,4,1,ymm0,ymm0) // vpavgb (%1,%4,1),%%ymm0,%%ymm0 + MEMOPMEM(vmovdqu,ymm0,0x00,1,0,1) + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x20,%2 \n" + "jg 50b \n" + "jmp 99f \n" + + // Blend 75 / 25. + LABELALIGN + "75: \n" + "vmovdqu " MEMACCESS(1) ",%%ymm1 \n" + MEMOPREG(vmovdqu,0x00,1,4,1,ymm0) + "vpavgb %%ymm1,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm1,%%ymm0,%%ymm0 \n" + MEMOPMEM(vmovdqu,ymm0,0x00,1,0,1) + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x20,%2 \n" + "jg 75b \n" + "jmp 99f \n" + + // Blend 100 / 0 - Copy row unchanged. + LABELALIGN + "100: \n" + "rep movsb " MEMMOVESTRING(1,0) " \n" + "jmp 999f \n" + + "99: \n" + "vzeroupper \n" + "999: \n" + : "+D"(dst_ptr), // %0 + "+S"(src_ptr), // %1 + "+c"(dst_width), // %2 + "+r"(source_y_fraction) // %3 + : "r"((intptr_t)(src_stride)) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm5" + ); +} +#endif // HAS_INTERPOLATEROW_AVX2 + +#ifdef HAS_INTERPOLATEROW_SSE2 +// Bilinear filter 16x2 -> 16x1 +void InterpolateRow_SSE2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) { + asm volatile ( + "sub %1,%0 \n" + "shr %3 \n" + "cmp $0x0,%3 \n" + "je 100f \n" + "cmp $0x20,%3 \n" + "je 75f \n" + "cmp $0x40,%3 \n" + "je 50f \n" + "cmp $0x60,%3 \n" + "je 25f \n" + + "movd %3,%%xmm0 \n" + "neg %3 \n" + "add $0x80,%3 \n" + "movd %3,%%xmm5 \n" + "punpcklbw %%xmm0,%%xmm5 \n" + "punpcklwd %%xmm5,%%xmm5 \n" + "pshufd $0x0,%%xmm5,%%xmm5 \n" + "pxor %%xmm4,%%xmm4 \n" + + // General purpose row blend. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(1) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,1,4,1,xmm2) // movdqu (%1,%4,1),%%xmm2 + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm2,%%xmm3 \n" + "punpcklbw %%xmm4,%%xmm2 \n" + "punpckhbw %%xmm4,%%xmm3 \n" + "punpcklbw %%xmm4,%%xmm0 \n" + "punpckhbw %%xmm4,%%xmm1 \n" + "psubw %%xmm0,%%xmm2 \n" + "psubw %%xmm1,%%xmm3 \n" + "paddw %%xmm2,%%xmm2 \n" + "paddw %%xmm3,%%xmm3 \n" + "pmulhw %%xmm5,%%xmm2 \n" + "pmulhw %%xmm5,%%xmm3 \n" + "paddw %%xmm2,%%xmm0 \n" + "paddw %%xmm3,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + MEMOPMEM(movdqu,xmm0,0x00,1,0,1) // movdqu %%xmm0,(%1,%0,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "jmp 99f \n" + + // Blend 25 / 75. + LABELALIGN + "25: \n" + "movdqu " MEMACCESS(1) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,1,4,1,xmm1) // movdqu (%1,%4,1),%%xmm1 + "pavgb %%xmm1,%%xmm0 \n" + "pavgb %%xmm1,%%xmm0 \n" + MEMOPMEM(movdqu,xmm0,0x00,1,0,1) // movdqu %%xmm0,(%1,%0,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 25b \n" + "jmp 99f \n" + + // Blend 50 / 50. + LABELALIGN + "50: \n" + "movdqu " MEMACCESS(1) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,1,4,1,xmm1) // movdqu (%1,%4,1),%%xmm1 + "pavgb %%xmm1,%%xmm0 \n" + MEMOPMEM(movdqu,xmm0,0x00,1,0,1) // movdqu %%xmm0,(%1,%0,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 50b \n" + "jmp 99f \n" + + // Blend 75 / 25. + LABELALIGN + "75: \n" + "movdqu " MEMACCESS(1) ",%%xmm1 \n" + MEMOPREG(movdqu,0x00,1,4,1,xmm0) // movdqu (%1,%4,1),%%xmm0 + "pavgb %%xmm1,%%xmm0 \n" + "pavgb %%xmm1,%%xmm0 \n" + MEMOPMEM(movdqu,xmm0,0x00,1,0,1) // movdqu %%xmm0,(%1,%0,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 75b \n" + "jmp 99f \n" + + // Blend 100 / 0 - Copy row unchanged. + LABELALIGN + "100: \n" + "movdqu " MEMACCESS(1) ",%%xmm0 \n" + MEMOPMEM(movdqu,xmm0,0x00,1,0,1) // movdqu %%xmm0,(%1,%0,1) + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 100b \n" + + "99: \n" + : "+r"(dst_ptr), // %0 + "+r"(src_ptr), // %1 + "+r"(dst_width), // %2 + "+r"(source_y_fraction) // %3 + : "r"((intptr_t)(src_stride)) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} +#endif // HAS_INTERPOLATEROW_SSE2 + +#ifdef HAS_ARGBSHUFFLEROW_SSSE3 +// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA. +void ARGBShuffleRow_SSSE3(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix) { + asm volatile ( + "movdqu " MEMACCESS(3) ",%%xmm5 \n" + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "pshufb %%xmm5,%%xmm0 \n" + "pshufb %%xmm5,%%xmm1 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : "r"(shuffler) // %3 + : "memory", "cc" + , "xmm0", "xmm1", "xmm5" + ); +} +#endif // HAS_ARGBSHUFFLEROW_SSSE3 + +#ifdef HAS_ARGBSHUFFLEROW_AVX2 +// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA. +void ARGBShuffleRow_AVX2(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix) { + asm volatile ( + "vbroadcastf128 " MEMACCESS(3) ",%%ymm5 \n" + LABELALIGN + "1: \n" + "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" + "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" + "lea " MEMLEA(0x40,0) ",%0 \n" + "vpshufb %%ymm5,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm5,%%ymm1,%%ymm1 \n" + "vmovdqu %%ymm0," MEMACCESS(1) " \n" + "vmovdqu %%ymm1," MEMACCESS2(0x20,1) " \n" + "lea " MEMLEA(0x40,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : "r"(shuffler) // %3 + : "memory", "cc" + , "xmm0", "xmm1", "xmm5" + ); +} +#endif // HAS_ARGBSHUFFLEROW_AVX2 + +#ifdef HAS_ARGBSHUFFLEROW_SSE2 +// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA. +void ARGBShuffleRow_SSE2(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix) { + uintptr_t pixel_temp = 0u; + asm volatile ( + "pxor %%xmm5,%%xmm5 \n" + "mov " MEMACCESS(4) ",%k2 \n" + "cmp $0x3000102,%k2 \n" + "je 3012f \n" + "cmp $0x10203,%k2 \n" + "je 123f \n" + "cmp $0x30201,%k2 \n" + "je 321f \n" + "cmp $0x2010003,%k2 \n" + "je 2103f \n" + + LABELALIGN + "1: \n" + "movzb " MEMACCESS(4) ",%2 \n" + MEMOPARG(movzb,0x00,0,2,1,2) " \n" // movzb (%0,%2,1),%2 + "mov %b2," MEMACCESS(1) " \n" + "movzb " MEMACCESS2(0x1,4) ",%2 \n" + MEMOPARG(movzb,0x00,0,2,1,2) " \n" // movzb (%0,%2,1),%2 + "mov %b2," MEMACCESS2(0x1,1) " \n" + "movzb " MEMACCESS2(0x2,4) ",%2 \n" + MEMOPARG(movzb,0x00,0,2,1,2) " \n" // movzb (%0,%2,1),%2 + "mov %b2," MEMACCESS2(0x2,1) " \n" + "movzb " MEMACCESS2(0x3,4) ",%2 \n" + MEMOPARG(movzb,0x00,0,2,1,2) " \n" // movzb (%0,%2,1),%2 + "mov %b2," MEMACCESS2(0x3,1) " \n" + "lea " MEMLEA(0x4,0) ",%0 \n" + "lea " MEMLEA(0x4,1) ",%1 \n" + "sub $0x1,%3 \n" + "jg 1b \n" + "jmp 99f \n" + + LABELALIGN + "123: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklbw %%xmm5,%%xmm0 \n" + "punpckhbw %%xmm5,%%xmm1 \n" + "pshufhw $0x1b,%%xmm0,%%xmm0 \n" + "pshuflw $0x1b,%%xmm0,%%xmm0 \n" + "pshufhw $0x1b,%%xmm1,%%xmm1 \n" + "pshuflw $0x1b,%%xmm1,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x4,%3 \n" + "jg 123b \n" + "jmp 99f \n" + + LABELALIGN + "321: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklbw %%xmm5,%%xmm0 \n" + "punpckhbw %%xmm5,%%xmm1 \n" + "pshufhw $0x39,%%xmm0,%%xmm0 \n" + "pshuflw $0x39,%%xmm0,%%xmm0 \n" + "pshufhw $0x39,%%xmm1,%%xmm1 \n" + "pshuflw $0x39,%%xmm1,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x4,%3 \n" + "jg 321b \n" + "jmp 99f \n" + + LABELALIGN + "2103: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklbw %%xmm5,%%xmm0 \n" + "punpckhbw %%xmm5,%%xmm1 \n" + "pshufhw $0x93,%%xmm0,%%xmm0 \n" + "pshuflw $0x93,%%xmm0,%%xmm0 \n" + "pshufhw $0x93,%%xmm1,%%xmm1 \n" + "pshuflw $0x93,%%xmm1,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x4,%3 \n" + "jg 2103b \n" + "jmp 99f \n" + + LABELALIGN + "3012: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklbw %%xmm5,%%xmm0 \n" + "punpckhbw %%xmm5,%%xmm1 \n" + "pshufhw $0xc6,%%xmm0,%%xmm0 \n" + "pshuflw $0xc6,%%xmm0,%%xmm0 \n" + "pshufhw $0xc6,%%xmm1,%%xmm1 \n" + "pshuflw $0xc6,%%xmm1,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x4,%3 \n" + "jg 3012b \n" + + "99: \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+d"(pixel_temp), // %2 + "+r"(pix) // %3 + : "r"(shuffler) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm5" + ); +} +#endif // HAS_ARGBSHUFFLEROW_SSE2 + +#ifdef HAS_I422TOYUY2ROW_SSE2 +void I422ToYUY2Row_SSE2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_frame, int width) { + asm volatile ( + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movq " MEMACCESS(1) ",%%xmm2 \n" + MEMOPREG(movq,0x00,1,2,1,xmm3) // movq (%1,%2,1),%%xmm3 + "lea " MEMLEA(0x8,1) ",%1 \n" + "punpcklbw %%xmm3,%%xmm2 \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklbw %%xmm2,%%xmm0 \n" + "punpckhbw %%xmm2,%%xmm1 \n" + "movdqu %%xmm0," MEMACCESS(3) " \n" + "movdqu %%xmm1," MEMACCESS2(0x10,3) " \n" + "lea " MEMLEA(0x20,3) ",%3 \n" + "sub $0x10,%4 \n" + "jg 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_frame), // %3 + "+rm"(width) // %4 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3" + ); +} +#endif // HAS_I422TOYUY2ROW_SSE2 + +#ifdef HAS_I422TOUYVYROW_SSE2 +void I422ToUYVYRow_SSE2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_frame, int width) { + asm volatile ( + "sub %1,%2 \n" + LABELALIGN + "1: \n" + "movq " MEMACCESS(1) ",%%xmm2 \n" + MEMOPREG(movq,0x00,1,2,1,xmm3) // movq (%1,%2,1),%%xmm3 + "lea " MEMLEA(0x8,1) ",%1 \n" + "punpcklbw %%xmm3,%%xmm2 \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqa %%xmm2,%%xmm1 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" + "punpcklbw %%xmm0,%%xmm1 \n" + "punpckhbw %%xmm0,%%xmm2 \n" + "movdqu %%xmm1," MEMACCESS(3) " \n" + "movdqu %%xmm2," MEMACCESS2(0x10,3) " \n" + "lea " MEMLEA(0x20,3) ",%3 \n" + "sub $0x10,%4 \n" + "jg 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_frame), // %3 + "+rm"(width) // %4 + : + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3" + ); +} +#endif // HAS_I422TOUYVYROW_SSE2 + +#ifdef HAS_ARGBPOLYNOMIALROW_SSE2 +void ARGBPolynomialRow_SSE2(const uint8* src_argb, + uint8* dst_argb, const float* poly, + int width) { + asm volatile ( + "pxor %%xmm3,%%xmm3 \n" + + // 2 pixel loop. + LABELALIGN + "1: \n" + "movq " MEMACCESS(0) ",%%xmm0 \n" + "lea " MEMLEA(0x8,0) ",%0 \n" + "punpcklbw %%xmm3,%%xmm0 \n" + "movdqa %%xmm0,%%xmm4 \n" + "punpcklwd %%xmm3,%%xmm0 \n" + "punpckhwd %%xmm3,%%xmm4 \n" + "cvtdq2ps %%xmm0,%%xmm0 \n" + "cvtdq2ps %%xmm4,%%xmm4 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm4,%%xmm5 \n" + "mulps " MEMACCESS2(0x10,3) ",%%xmm0 \n" + "mulps " MEMACCESS2(0x10,3) ",%%xmm4 \n" + "addps " MEMACCESS(3) ",%%xmm0 \n" + "addps " MEMACCESS(3) ",%%xmm4 \n" + "movdqa %%xmm1,%%xmm2 \n" + "movdqa %%xmm5,%%xmm6 \n" + "mulps %%xmm1,%%xmm2 \n" + "mulps %%xmm5,%%xmm6 \n" + "mulps %%xmm2,%%xmm1 \n" + "mulps %%xmm6,%%xmm5 \n" + "mulps " MEMACCESS2(0x20,3) ",%%xmm2 \n" + "mulps " MEMACCESS2(0x20,3) ",%%xmm6 \n" + "mulps " MEMACCESS2(0x30,3) ",%%xmm1 \n" + "mulps " MEMACCESS2(0x30,3) ",%%xmm5 \n" + "addps %%xmm2,%%xmm0 \n" + "addps %%xmm6,%%xmm4 \n" + "addps %%xmm1,%%xmm0 \n" + "addps %%xmm5,%%xmm4 \n" + "cvttps2dq %%xmm0,%%xmm0 \n" + "cvttps2dq %%xmm4,%%xmm4 \n" + "packuswb %%xmm4,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "movq %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x2,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(poly) // %3 + : "memory", "cc" + , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" + ); +} +#endif // HAS_ARGBPOLYNOMIALROW_SSE2 + +#ifdef HAS_ARGBPOLYNOMIALROW_AVX2 +void ARGBPolynomialRow_AVX2(const uint8* src_argb, + uint8* dst_argb, const float* poly, + int width) { + asm volatile ( + "vbroadcastf128 " MEMACCESS(3) ",%%ymm4 \n" + "vbroadcastf128 " MEMACCESS2(0x10,3) ",%%ymm5 \n" + "vbroadcastf128 " MEMACCESS2(0x20,3) ",%%ymm6 \n" + "vbroadcastf128 " MEMACCESS2(0x30,3) ",%%ymm7 \n" + + // 2 pixel loop. + LABELALIGN + "1: \n" + "vpmovzxbd " MEMACCESS(0) ",%%ymm0 \n" // 2 ARGB pixels + "lea " MEMLEA(0x8,0) ",%0 \n" + "vcvtdq2ps %%ymm0,%%ymm0 \n" // X 8 floats + "vmulps %%ymm0,%%ymm0,%%ymm2 \n" // X * X + "vmulps %%ymm7,%%ymm0,%%ymm3 \n" // C3 * X + "vfmadd132ps %%ymm5,%%ymm4,%%ymm0 \n" // result = C0 + C1 * X + "vfmadd231ps %%ymm6,%%ymm2,%%ymm0 \n" // result += C2 * X * X + "vfmadd231ps %%ymm3,%%ymm2,%%ymm0 \n" // result += C3 * X * X * X + "vcvttps2dq %%ymm0,%%ymm0 \n" + "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vmovq %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x2,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(poly) // %3 + : "memory", "cc", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} +#endif // HAS_ARGBPOLYNOMIALROW_AVX2 + +#ifdef HAS_ARGBCOLORTABLEROW_X86 +// Tranform ARGB pixels with color table. +void ARGBColorTableRow_X86(uint8* dst_argb, const uint8* table_argb, + int width) { + uintptr_t pixel_temp = 0u; + asm volatile ( + // 1 pixel loop. + LABELALIGN + "1: \n" + "movzb " MEMACCESS(0) ",%1 \n" + "lea " MEMLEA(0x4,0) ",%0 \n" + MEMOPARG(movzb,0x00,3,1,4,1) " \n" // movzb (%3,%1,4),%1 + "mov %b1," MEMACCESS2(-0x4,0) " \n" + "movzb " MEMACCESS2(-0x3,0) ",%1 \n" + MEMOPARG(movzb,0x01,3,1,4,1) " \n" // movzb 0x1(%3,%1,4),%1 + "mov %b1," MEMACCESS2(-0x3,0) " \n" + "movzb " MEMACCESS2(-0x2,0) ",%1 \n" + MEMOPARG(movzb,0x02,3,1,4,1) " \n" // movzb 0x2(%3,%1,4),%1 + "mov %b1," MEMACCESS2(-0x2,0) " \n" + "movzb " MEMACCESS2(-0x1,0) ",%1 \n" + MEMOPARG(movzb,0x03,3,1,4,1) " \n" // movzb 0x3(%3,%1,4),%1 + "mov %b1," MEMACCESS2(-0x1,0) " \n" + "dec %2 \n" + "jg 1b \n" + : "+r"(dst_argb), // %0 + "+d"(pixel_temp), // %1 + "+r"(width) // %2 + : "r"(table_argb) // %3 + : "memory", "cc"); +} +#endif // HAS_ARGBCOLORTABLEROW_X86 + +#ifdef HAS_RGBCOLORTABLEROW_X86 +// Tranform RGB pixels with color table. +void RGBColorTableRow_X86(uint8* dst_argb, const uint8* table_argb, int width) { + uintptr_t pixel_temp = 0u; + asm volatile ( + // 1 pixel loop. + LABELALIGN + "1: \n" + "movzb " MEMACCESS(0) ",%1 \n" + "lea " MEMLEA(0x4,0) ",%0 \n" + MEMOPARG(movzb,0x00,3,1,4,1) " \n" // movzb (%3,%1,4),%1 + "mov %b1," MEMACCESS2(-0x4,0) " \n" + "movzb " MEMACCESS2(-0x3,0) ",%1 \n" + MEMOPARG(movzb,0x01,3,1,4,1) " \n" // movzb 0x1(%3,%1,4),%1 + "mov %b1," MEMACCESS2(-0x3,0) " \n" + "movzb " MEMACCESS2(-0x2,0) ",%1 \n" + MEMOPARG(movzb,0x02,3,1,4,1) " \n" // movzb 0x2(%3,%1,4),%1 + "mov %b1," MEMACCESS2(-0x2,0) " \n" + "dec %2 \n" + "jg 1b \n" + : "+r"(dst_argb), // %0 + "+d"(pixel_temp), // %1 + "+r"(width) // %2 + : "r"(table_argb) // %3 + : "memory", "cc"); +} +#endif // HAS_RGBCOLORTABLEROW_X86 + +#ifdef HAS_ARGBLUMACOLORTABLEROW_SSSE3 +// Tranform RGB pixels with luma table. +void ARGBLumaColorTableRow_SSSE3(const uint8* src_argb, uint8* dst_argb, + int width, + const uint8* luma, uint32 lumacoeff) { + uintptr_t pixel_temp = 0u; + uintptr_t table_temp = 0u; + asm volatile ( + "movd %6,%%xmm3 \n" + "pshufd $0x0,%%xmm3,%%xmm3 \n" + "pcmpeqb %%xmm4,%%xmm4 \n" + "psllw $0x8,%%xmm4 \n" + "pxor %%xmm5,%%xmm5 \n" + + // 4 pixel loop. + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(2) ",%%xmm0 \n" + "pmaddubsw %%xmm3,%%xmm0 \n" + "phaddw %%xmm0,%%xmm0 \n" + "pand %%xmm4,%%xmm0 \n" + "punpcklwd %%xmm5,%%xmm0 \n" + "movd %%xmm0,%k1 \n" // 32 bit offset + "add %5,%1 \n" + "pshufd $0x39,%%xmm0,%%xmm0 \n" + + "movzb " MEMACCESS(2) ",%0 \n" + MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 + "mov %b0," MEMACCESS(3) " \n" + "movzb " MEMACCESS2(0x1,2) ",%0 \n" + MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 + "mov %b0," MEMACCESS2(0x1,3) " \n" + "movzb " MEMACCESS2(0x2,2) ",%0 \n" + MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 + "mov %b0," MEMACCESS2(0x2,3) " \n" + "movzb " MEMACCESS2(0x3,2) ",%0 \n" + "mov %b0," MEMACCESS2(0x3,3) " \n" + + "movd %%xmm0,%k1 \n" // 32 bit offset + "add %5,%1 \n" + "pshufd $0x39,%%xmm0,%%xmm0 \n" + + "movzb " MEMACCESS2(0x4,2) ",%0 \n" + MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 + "mov %b0," MEMACCESS2(0x4,3) " \n" + "movzb " MEMACCESS2(0x5,2) ",%0 \n" + MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 + "mov %b0," MEMACCESS2(0x5,3) " \n" + "movzb " MEMACCESS2(0x6,2) ",%0 \n" + MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 + "mov %b0," MEMACCESS2(0x6,3) " \n" + "movzb " MEMACCESS2(0x7,2) ",%0 \n" + "mov %b0," MEMACCESS2(0x7,3) " \n" + + "movd %%xmm0,%k1 \n" // 32 bit offset + "add %5,%1 \n" + "pshufd $0x39,%%xmm0,%%xmm0 \n" + + "movzb " MEMACCESS2(0x8,2) ",%0 \n" + MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 + "mov %b0," MEMACCESS2(0x8,3) " \n" + "movzb " MEMACCESS2(0x9,2) ",%0 \n" + MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 + "mov %b0," MEMACCESS2(0x9,3) " \n" + "movzb " MEMACCESS2(0xa,2) ",%0 \n" + MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 + "mov %b0," MEMACCESS2(0xa,3) " \n" + "movzb " MEMACCESS2(0xb,2) ",%0 \n" + "mov %b0," MEMACCESS2(0xb,3) " \n" + + "movd %%xmm0,%k1 \n" // 32 bit offset + "add %5,%1 \n" + + "movzb " MEMACCESS2(0xc,2) ",%0 \n" + MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 + "mov %b0," MEMACCESS2(0xc,3) " \n" + "movzb " MEMACCESS2(0xd,2) ",%0 \n" + MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 + "mov %b0," MEMACCESS2(0xd,3) " \n" + "movzb " MEMACCESS2(0xe,2) ",%0 \n" + MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 + "mov %b0," MEMACCESS2(0xe,3) " \n" + "movzb " MEMACCESS2(0xf,2) ",%0 \n" + "mov %b0," MEMACCESS2(0xf,3) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "lea " MEMLEA(0x10,3) ",%3 \n" + "sub $0x4,%4 \n" + "jg 1b \n" + : "+d"(pixel_temp), // %0 + "+a"(table_temp), // %1 + "+r"(src_argb), // %2 + "+r"(dst_argb), // %3 + "+rm"(width) // %4 + : "r"(luma), // %5 + "rm"(lumacoeff) // %6 + : "memory", "cc", "xmm0", "xmm3", "xmm4", "xmm5" + ); +} +#endif // HAS_ARGBLUMACOLORTABLEROW_SSSE3 + +#endif // defined(__x86_64__) || defined(__i386__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/row_mips.cc b/third_party/aom/third_party/libyuv/source/row_mips.cc new file mode 100644 index 0000000000..cfc9ffe036 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/row_mips.cc @@ -0,0 +1,911 @@ +/* + * Copyright (c) 2012 The LibYuv project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// The following are available on Mips platforms: +#if !defined(LIBYUV_DISABLE_MIPS) && defined(__mips__) && \ + (_MIPS_SIM == _MIPS_SIM_ABI32) + +#ifdef HAS_COPYROW_MIPS +void CopyRow_MIPS(const uint8* src, uint8* dst, int count) { + __asm__ __volatile__ ( + ".set noreorder \n" + ".set noat \n" + "slti $at, %[count], 8 \n" + "bne $at ,$zero, $last8 \n" + "xor $t8, %[src], %[dst] \n" + "andi $t8, $t8, 0x3 \n" + + "bne $t8, $zero, unaligned \n" + "negu $a3, %[dst] \n" + // make dst/src aligned + "andi $a3, $a3, 0x3 \n" + "beq $a3, $zero, $chk16w \n" + // word-aligned now count is the remining bytes count + "subu %[count], %[count], $a3 \n" + + "lwr $t8, 0(%[src]) \n" + "addu %[src], %[src], $a3 \n" + "swr $t8, 0(%[dst]) \n" + "addu %[dst], %[dst], $a3 \n" + + // Now the dst/src are mutually word-aligned with word-aligned addresses + "$chk16w: \n" + "andi $t8, %[count], 0x3f \n" // whole 64-B chunks? + // t8 is the byte count after 64-byte chunks + "beq %[count], $t8, chk8w \n" + // There will be at most 1 32-byte chunk after it + "subu $a3, %[count], $t8 \n" // the reminder + // Here a3 counts bytes in 16w chunks + "addu $a3, %[dst], $a3 \n" + // Now a3 is the final dst after 64-byte chunks + "addu $t0, %[dst], %[count] \n" + // t0 is the "past the end" address + + // When in the loop we exercise "pref 30,x(a1)", the a1+x should not be past + // the "t0-32" address + // This means: for x=128 the last "safe" a1 address is "t0-160" + // Alternatively, for x=64 the last "safe" a1 address is "t0-96" + // we will use "pref 30,128(a1)", so "t0-160" is the limit + "subu $t9, $t0, 160 \n" + // t9 is the "last safe pref 30,128(a1)" address + "pref 0, 0(%[src]) \n" // first line of src + "pref 0, 32(%[src]) \n" // second line of src + "pref 0, 64(%[src]) \n" + "pref 30, 32(%[dst]) \n" + // In case the a1 > t9 don't use "pref 30" at all + "sgtu $v1, %[dst], $t9 \n" + "bgtz $v1, $loop16w \n" + "nop \n" + // otherwise, start with using pref30 + "pref 30, 64(%[dst]) \n" + "$loop16w: \n" + "pref 0, 96(%[src]) \n" + "lw $t0, 0(%[src]) \n" + "bgtz $v1, $skip_pref30_96 \n" // skip + "lw $t1, 4(%[src]) \n" + "pref 30, 96(%[dst]) \n" // continue + "$skip_pref30_96: \n" + "lw $t2, 8(%[src]) \n" + "lw $t3, 12(%[src]) \n" + "lw $t4, 16(%[src]) \n" + "lw $t5, 20(%[src]) \n" + "lw $t6, 24(%[src]) \n" + "lw $t7, 28(%[src]) \n" + "pref 0, 128(%[src]) \n" + // bring the next lines of src, addr 128 + "sw $t0, 0(%[dst]) \n" + "sw $t1, 4(%[dst]) \n" + "sw $t2, 8(%[dst]) \n" + "sw $t3, 12(%[dst]) \n" + "sw $t4, 16(%[dst]) \n" + "sw $t5, 20(%[dst]) \n" + "sw $t6, 24(%[dst]) \n" + "sw $t7, 28(%[dst]) \n" + "lw $t0, 32(%[src]) \n" + "bgtz $v1, $skip_pref30_128 \n" // skip pref 30,128(a1) + "lw $t1, 36(%[src]) \n" + "pref 30, 128(%[dst]) \n" // set dest, addr 128 + "$skip_pref30_128: \n" + "lw $t2, 40(%[src]) \n" + "lw $t3, 44(%[src]) \n" + "lw $t4, 48(%[src]) \n" + "lw $t5, 52(%[src]) \n" + "lw $t6, 56(%[src]) \n" + "lw $t7, 60(%[src]) \n" + "pref 0, 160(%[src]) \n" + // bring the next lines of src, addr 160 + "sw $t0, 32(%[dst]) \n" + "sw $t1, 36(%[dst]) \n" + "sw $t2, 40(%[dst]) \n" + "sw $t3, 44(%[dst]) \n" + "sw $t4, 48(%[dst]) \n" + "sw $t5, 52(%[dst]) \n" + "sw $t6, 56(%[dst]) \n" + "sw $t7, 60(%[dst]) \n" + + "addiu %[dst], %[dst], 64 \n" // adding 64 to dest + "sgtu $v1, %[dst], $t9 \n" + "bne %[dst], $a3, $loop16w \n" + " addiu %[src], %[src], 64 \n" // adding 64 to src + "move %[count], $t8 \n" + + // Here we have src and dest word-aligned but less than 64-bytes to go + + "chk8w: \n" + "pref 0, 0x0(%[src]) \n" + "andi $t8, %[count], 0x1f \n" // 32-byte chunk? + // the t8 is the reminder count past 32-bytes + "beq %[count], $t8, chk1w \n" + // count=t8,no 32-byte chunk + " nop \n" + + "lw $t0, 0(%[src]) \n" + "lw $t1, 4(%[src]) \n" + "lw $t2, 8(%[src]) \n" + "lw $t3, 12(%[src]) \n" + "lw $t4, 16(%[src]) \n" + "lw $t5, 20(%[src]) \n" + "lw $t6, 24(%[src]) \n" + "lw $t7, 28(%[src]) \n" + "addiu %[src], %[src], 32 \n" + + "sw $t0, 0(%[dst]) \n" + "sw $t1, 4(%[dst]) \n" + "sw $t2, 8(%[dst]) \n" + "sw $t3, 12(%[dst]) \n" + "sw $t4, 16(%[dst]) \n" + "sw $t5, 20(%[dst]) \n" + "sw $t6, 24(%[dst]) \n" + "sw $t7, 28(%[dst]) \n" + "addiu %[dst], %[dst], 32 \n" + + "chk1w: \n" + "andi %[count], $t8, 0x3 \n" + // now count is the reminder past 1w chunks + "beq %[count], $t8, $last8 \n" + " subu $a3, $t8, %[count] \n" + // a3 is count of bytes in 1w chunks + "addu $a3, %[dst], $a3 \n" + // now a3 is the dst address past the 1w chunks + // copying in words (4-byte chunks) + "$wordCopy_loop: \n" + "lw $t3, 0(%[src]) \n" + // the first t3 may be equal t0 ... optimize? + "addiu %[src], %[src],4 \n" + "addiu %[dst], %[dst],4 \n" + "bne %[dst], $a3,$wordCopy_loop \n" + " sw $t3, -4(%[dst]) \n" + + // For the last (<8) bytes + "$last8: \n" + "blez %[count], leave \n" + " addu $a3, %[dst], %[count] \n" // a3 -last dst address + "$last8loop: \n" + "lb $v1, 0(%[src]) \n" + "addiu %[src], %[src], 1 \n" + "addiu %[dst], %[dst], 1 \n" + "bne %[dst], $a3, $last8loop \n" + " sb $v1, -1(%[dst]) \n" + + "leave: \n" + " j $ra \n" + " nop \n" + + // + // UNALIGNED case + // + + "unaligned: \n" + // got here with a3="negu a1" + "andi $a3, $a3, 0x3 \n" // a1 is word aligned? + "beqz $a3, $ua_chk16w \n" + " subu %[count], %[count], $a3 \n" + // bytes left after initial a3 bytes + "lwr $v1, 0(%[src]) \n" + "lwl $v1, 3(%[src]) \n" + "addu %[src], %[src], $a3 \n" // a3 may be 1, 2 or 3 + "swr $v1, 0(%[dst]) \n" + "addu %[dst], %[dst], $a3 \n" + // below the dst will be word aligned (NOTE1) + "$ua_chk16w: \n" + "andi $t8, %[count], 0x3f \n" // whole 64-B chunks? + // t8 is the byte count after 64-byte chunks + "beq %[count], $t8, ua_chk8w \n" + // if a2==t8, no 64-byte chunks + // There will be at most 1 32-byte chunk after it + "subu $a3, %[count], $t8 \n" // the reminder + // Here a3 counts bytes in 16w chunks + "addu $a3, %[dst], $a3 \n" + // Now a3 is the final dst after 64-byte chunks + "addu $t0, %[dst], %[count] \n" // t0 "past the end" + "subu $t9, $t0, 160 \n" + // t9 is the "last safe pref 30,128(a1)" address + "pref 0, 0(%[src]) \n" // first line of src + "pref 0, 32(%[src]) \n" // second line addr 32 + "pref 0, 64(%[src]) \n" + "pref 30, 32(%[dst]) \n" + // safe, as we have at least 64 bytes ahead + // In case the a1 > t9 don't use "pref 30" at all + "sgtu $v1, %[dst], $t9 \n" + "bgtz $v1, $ua_loop16w \n" + // skip "pref 30,64(a1)" for too short arrays + " nop \n" + // otherwise, start with using pref30 + "pref 30, 64(%[dst]) \n" + "$ua_loop16w: \n" + "pref 0, 96(%[src]) \n" + "lwr $t0, 0(%[src]) \n" + "lwl $t0, 3(%[src]) \n" + "lwr $t1, 4(%[src]) \n" + "bgtz $v1, $ua_skip_pref30_96 \n" + " lwl $t1, 7(%[src]) \n" + "pref 30, 96(%[dst]) \n" + // continue setting up the dest, addr 96 + "$ua_skip_pref30_96: \n" + "lwr $t2, 8(%[src]) \n" + "lwl $t2, 11(%[src]) \n" + "lwr $t3, 12(%[src]) \n" + "lwl $t3, 15(%[src]) \n" + "lwr $t4, 16(%[src]) \n" + "lwl $t4, 19(%[src]) \n" + "lwr $t5, 20(%[src]) \n" + "lwl $t5, 23(%[src]) \n" + "lwr $t6, 24(%[src]) \n" + "lwl $t6, 27(%[src]) \n" + "lwr $t7, 28(%[src]) \n" + "lwl $t7, 31(%[src]) \n" + "pref 0, 128(%[src]) \n" + // bring the next lines of src, addr 128 + "sw $t0, 0(%[dst]) \n" + "sw $t1, 4(%[dst]) \n" + "sw $t2, 8(%[dst]) \n" + "sw $t3, 12(%[dst]) \n" + "sw $t4, 16(%[dst]) \n" + "sw $t5, 20(%[dst]) \n" + "sw $t6, 24(%[dst]) \n" + "sw $t7, 28(%[dst]) \n" + "lwr $t0, 32(%[src]) \n" + "lwl $t0, 35(%[src]) \n" + "lwr $t1, 36(%[src]) \n" + "bgtz $v1, ua_skip_pref30_128 \n" + " lwl $t1, 39(%[src]) \n" + "pref 30, 128(%[dst]) \n" + // continue setting up the dest, addr 128 + "ua_skip_pref30_128: \n" + + "lwr $t2, 40(%[src]) \n" + "lwl $t2, 43(%[src]) \n" + "lwr $t3, 44(%[src]) \n" + "lwl $t3, 47(%[src]) \n" + "lwr $t4, 48(%[src]) \n" + "lwl $t4, 51(%[src]) \n" + "lwr $t5, 52(%[src]) \n" + "lwl $t5, 55(%[src]) \n" + "lwr $t6, 56(%[src]) \n" + "lwl $t6, 59(%[src]) \n" + "lwr $t7, 60(%[src]) \n" + "lwl $t7, 63(%[src]) \n" + "pref 0, 160(%[src]) \n" + // bring the next lines of src, addr 160 + "sw $t0, 32(%[dst]) \n" + "sw $t1, 36(%[dst]) \n" + "sw $t2, 40(%[dst]) \n" + "sw $t3, 44(%[dst]) \n" + "sw $t4, 48(%[dst]) \n" + "sw $t5, 52(%[dst]) \n" + "sw $t6, 56(%[dst]) \n" + "sw $t7, 60(%[dst]) \n" + + "addiu %[dst],%[dst],64 \n" // adding 64 to dest + "sgtu $v1,%[dst],$t9 \n" + "bne %[dst],$a3,$ua_loop16w \n" + " addiu %[src],%[src],64 \n" // adding 64 to src + "move %[count],$t8 \n" + + // Here we have src and dest word-aligned but less than 64-bytes to go + + "ua_chk8w: \n" + "pref 0, 0x0(%[src]) \n" + "andi $t8, %[count], 0x1f \n" // 32-byte chunk? + // the t8 is the reminder count + "beq %[count], $t8, $ua_chk1w \n" + // when count==t8, no 32-byte chunk + + "lwr $t0, 0(%[src]) \n" + "lwl $t0, 3(%[src]) \n" + "lwr $t1, 4(%[src]) \n" + "lwl $t1, 7(%[src]) \n" + "lwr $t2, 8(%[src]) \n" + "lwl $t2, 11(%[src]) \n" + "lwr $t3, 12(%[src]) \n" + "lwl $t3, 15(%[src]) \n" + "lwr $t4, 16(%[src]) \n" + "lwl $t4, 19(%[src]) \n" + "lwr $t5, 20(%[src]) \n" + "lwl $t5, 23(%[src]) \n" + "lwr $t6, 24(%[src]) \n" + "lwl $t6, 27(%[src]) \n" + "lwr $t7, 28(%[src]) \n" + "lwl $t7, 31(%[src]) \n" + "addiu %[src], %[src], 32 \n" + + "sw $t0, 0(%[dst]) \n" + "sw $t1, 4(%[dst]) \n" + "sw $t2, 8(%[dst]) \n" + "sw $t3, 12(%[dst]) \n" + "sw $t4, 16(%[dst]) \n" + "sw $t5, 20(%[dst]) \n" + "sw $t6, 24(%[dst]) \n" + "sw $t7, 28(%[dst]) \n" + "addiu %[dst], %[dst], 32 \n" + + "$ua_chk1w: \n" + "andi %[count], $t8, 0x3 \n" + // now count is the reminder past 1w chunks + "beq %[count], $t8, ua_smallCopy \n" + "subu $a3, $t8, %[count] \n" + // a3 is count of bytes in 1w chunks + "addu $a3, %[dst], $a3 \n" + // now a3 is the dst address past the 1w chunks + + // copying in words (4-byte chunks) + "$ua_wordCopy_loop: \n" + "lwr $v1, 0(%[src]) \n" + "lwl $v1, 3(%[src]) \n" + "addiu %[src], %[src], 4 \n" + "addiu %[dst], %[dst], 4 \n" + // note: dst=a1 is word aligned here, see NOTE1 + "bne %[dst], $a3, $ua_wordCopy_loop \n" + " sw $v1,-4(%[dst]) \n" + + // Now less than 4 bytes (value in count) left to copy + "ua_smallCopy: \n" + "beqz %[count], leave \n" + " addu $a3, %[dst], %[count] \n" // a3 = last dst address + "$ua_smallCopy_loop: \n" + "lb $v1, 0(%[src]) \n" + "addiu %[src], %[src], 1 \n" + "addiu %[dst], %[dst], 1 \n" + "bne %[dst],$a3,$ua_smallCopy_loop \n" + " sb $v1, -1(%[dst]) \n" + + "j $ra \n" + " nop \n" + ".set at \n" + ".set reorder \n" + : [dst] "+r" (dst), [src] "+r" (src) + : [count] "r" (count) + : "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", + "t8", "t9", "a3", "v1", "at" + ); +} +#endif // HAS_COPYROW_MIPS + +// MIPS DSPR2 functions +#if !defined(LIBYUV_DISABLE_MIPS) && defined(__mips_dsp) && \ + (__mips_dsp_rev >= 2) && \ + (_MIPS_SIM == _MIPS_SIM_ABI32) && (__mips_isa_rev < 6) + +void SplitUVRow_MIPS_DSPR2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int width) { + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + "srl $t4, %[width], 4 \n" // multiplies of 16 + "blez $t4, 2f \n" + " andi %[width], %[width], 0xf \n" // residual + + ".p2align 2 \n" + "1: \n" + "addiu $t4, $t4, -1 \n" + "lw $t0, 0(%[src_uv]) \n" // V1 | U1 | V0 | U0 + "lw $t1, 4(%[src_uv]) \n" // V3 | U3 | V2 | U2 + "lw $t2, 8(%[src_uv]) \n" // V5 | U5 | V4 | U4 + "lw $t3, 12(%[src_uv]) \n" // V7 | U7 | V6 | U6 + "lw $t5, 16(%[src_uv]) \n" // V9 | U9 | V8 | U8 + "lw $t6, 20(%[src_uv]) \n" // V11 | U11 | V10 | U10 + "lw $t7, 24(%[src_uv]) \n" // V13 | U13 | V12 | U12 + "lw $t8, 28(%[src_uv]) \n" // V15 | U15 | V14 | U14 + "addiu %[src_uv], %[src_uv], 32 \n" + "precrq.qb.ph $t9, $t1, $t0 \n" // V3 | V2 | V1 | V0 + "precr.qb.ph $t0, $t1, $t0 \n" // U3 | U2 | U1 | U0 + "precrq.qb.ph $t1, $t3, $t2 \n" // V7 | V6 | V5 | V4 + "precr.qb.ph $t2, $t3, $t2 \n" // U7 | U6 | U5 | U4 + "precrq.qb.ph $t3, $t6, $t5 \n" // V11 | V10 | V9 | V8 + "precr.qb.ph $t5, $t6, $t5 \n" // U11 | U10 | U9 | U8 + "precrq.qb.ph $t6, $t8, $t7 \n" // V15 | V14 | V13 | V12 + "precr.qb.ph $t7, $t8, $t7 \n" // U15 | U14 | U13 | U12 + "sw $t9, 0(%[dst_v]) \n" + "sw $t0, 0(%[dst_u]) \n" + "sw $t1, 4(%[dst_v]) \n" + "sw $t2, 4(%[dst_u]) \n" + "sw $t3, 8(%[dst_v]) \n" + "sw $t5, 8(%[dst_u]) \n" + "sw $t6, 12(%[dst_v]) \n" + "sw $t7, 12(%[dst_u]) \n" + "addiu %[dst_v], %[dst_v], 16 \n" + "bgtz $t4, 1b \n" + " addiu %[dst_u], %[dst_u], 16 \n" + + "beqz %[width], 3f \n" + " nop \n" + + "2: \n" + "lbu $t0, 0(%[src_uv]) \n" + "lbu $t1, 1(%[src_uv]) \n" + "addiu %[src_uv], %[src_uv], 2 \n" + "addiu %[width], %[width], -1 \n" + "sb $t0, 0(%[dst_u]) \n" + "sb $t1, 0(%[dst_v]) \n" + "addiu %[dst_u], %[dst_u], 1 \n" + "bgtz %[width], 2b \n" + " addiu %[dst_v], %[dst_v], 1 \n" + + "3: \n" + ".set pop \n" + : [src_uv] "+r" (src_uv), + [width] "+r" (width), + [dst_u] "+r" (dst_u), + [dst_v] "+r" (dst_v) + : + : "t0", "t1", "t2", "t3", + "t4", "t5", "t6", "t7", "t8", "t9" + ); +} + +void MirrorRow_MIPS_DSPR2(const uint8* src, uint8* dst, int width) { + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + + "srl $t4, %[width], 4 \n" // multiplies of 16 + "andi $t5, %[width], 0xf \n" + "blez $t4, 2f \n" + " addu %[src], %[src], %[width] \n" // src += width + + ".p2align 2 \n" + "1: \n" + "lw $t0, -16(%[src]) \n" // |3|2|1|0| + "lw $t1, -12(%[src]) \n" // |7|6|5|4| + "lw $t2, -8(%[src]) \n" // |11|10|9|8| + "lw $t3, -4(%[src]) \n" // |15|14|13|12| + "wsbh $t0, $t0 \n" // |2|3|0|1| + "wsbh $t1, $t1 \n" // |6|7|4|5| + "wsbh $t2, $t2 \n" // |10|11|8|9| + "wsbh $t3, $t3 \n" // |14|15|12|13| + "rotr $t0, $t0, 16 \n" // |0|1|2|3| + "rotr $t1, $t1, 16 \n" // |4|5|6|7| + "rotr $t2, $t2, 16 \n" // |8|9|10|11| + "rotr $t3, $t3, 16 \n" // |12|13|14|15| + "addiu %[src], %[src], -16 \n" + "addiu $t4, $t4, -1 \n" + "sw $t3, 0(%[dst]) \n" // |15|14|13|12| + "sw $t2, 4(%[dst]) \n" // |11|10|9|8| + "sw $t1, 8(%[dst]) \n" // |7|6|5|4| + "sw $t0, 12(%[dst]) \n" // |3|2|1|0| + "bgtz $t4, 1b \n" + " addiu %[dst], %[dst], 16 \n" + "beqz $t5, 3f \n" + " nop \n" + + "2: \n" + "lbu $t0, -1(%[src]) \n" + "addiu $t5, $t5, -1 \n" + "addiu %[src], %[src], -1 \n" + "sb $t0, 0(%[dst]) \n" + "bgez $t5, 2b \n" + " addiu %[dst], %[dst], 1 \n" + + "3: \n" + ".set pop \n" + : [src] "+r" (src), [dst] "+r" (dst) + : [width] "r" (width) + : "t0", "t1", "t2", "t3", "t4", "t5" + ); +} + +void MirrorUVRow_MIPS_DSPR2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int width) { + int x = 0; + int y = 0; + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + + "addu $t4, %[width], %[width] \n" + "srl %[x], %[width], 4 \n" + "andi %[y], %[width], 0xf \n" + "blez %[x], 2f \n" + " addu %[src_uv], %[src_uv], $t4 \n" + + ".p2align 2 \n" + "1: \n" + "lw $t0, -32(%[src_uv]) \n" // |3|2|1|0| + "lw $t1, -28(%[src_uv]) \n" // |7|6|5|4| + "lw $t2, -24(%[src_uv]) \n" // |11|10|9|8| + "lw $t3, -20(%[src_uv]) \n" // |15|14|13|12| + "lw $t4, -16(%[src_uv]) \n" // |19|18|17|16| + "lw $t6, -12(%[src_uv]) \n" // |23|22|21|20| + "lw $t7, -8(%[src_uv]) \n" // |27|26|25|24| + "lw $t8, -4(%[src_uv]) \n" // |31|30|29|28| + + "rotr $t0, $t0, 16 \n" // |1|0|3|2| + "rotr $t1, $t1, 16 \n" // |5|4|7|6| + "rotr $t2, $t2, 16 \n" // |9|8|11|10| + "rotr $t3, $t3, 16 \n" // |13|12|15|14| + "rotr $t4, $t4, 16 \n" // |17|16|19|18| + "rotr $t6, $t6, 16 \n" // |21|20|23|22| + "rotr $t7, $t7, 16 \n" // |25|24|27|26| + "rotr $t8, $t8, 16 \n" // |29|28|31|30| + "precr.qb.ph $t9, $t0, $t1 \n" // |0|2|4|6| + "precrq.qb.ph $t5, $t0, $t1 \n" // |1|3|5|7| + "precr.qb.ph $t0, $t2, $t3 \n" // |8|10|12|14| + "precrq.qb.ph $t1, $t2, $t3 \n" // |9|11|13|15| + "precr.qb.ph $t2, $t4, $t6 \n" // |16|18|20|22| + "precrq.qb.ph $t3, $t4, $t6 \n" // |17|19|21|23| + "precr.qb.ph $t4, $t7, $t8 \n" // |24|26|28|30| + "precrq.qb.ph $t6, $t7, $t8 \n" // |25|27|29|31| + "addiu %[src_uv], %[src_uv], -32 \n" + "addiu %[x], %[x], -1 \n" + "swr $t4, 0(%[dst_u]) \n" + "swl $t4, 3(%[dst_u]) \n" // |30|28|26|24| + "swr $t6, 0(%[dst_v]) \n" + "swl $t6, 3(%[dst_v]) \n" // |31|29|27|25| + "swr $t2, 4(%[dst_u]) \n" + "swl $t2, 7(%[dst_u]) \n" // |22|20|18|16| + "swr $t3, 4(%[dst_v]) \n" + "swl $t3, 7(%[dst_v]) \n" // |23|21|19|17| + "swr $t0, 8(%[dst_u]) \n" + "swl $t0, 11(%[dst_u]) \n" // |14|12|10|8| + "swr $t1, 8(%[dst_v]) \n" + "swl $t1, 11(%[dst_v]) \n" // |15|13|11|9| + "swr $t9, 12(%[dst_u]) \n" + "swl $t9, 15(%[dst_u]) \n" // |6|4|2|0| + "swr $t5, 12(%[dst_v]) \n" + "swl $t5, 15(%[dst_v]) \n" // |7|5|3|1| + "addiu %[dst_v], %[dst_v], 16 \n" + "bgtz %[x], 1b \n" + " addiu %[dst_u], %[dst_u], 16 \n" + "beqz %[y], 3f \n" + " nop \n" + "b 2f \n" + " nop \n" + + "2: \n" + "lbu $t0, -2(%[src_uv]) \n" + "lbu $t1, -1(%[src_uv]) \n" + "addiu %[src_uv], %[src_uv], -2 \n" + "addiu %[y], %[y], -1 \n" + "sb $t0, 0(%[dst_u]) \n" + "sb $t1, 0(%[dst_v]) \n" + "addiu %[dst_u], %[dst_u], 1 \n" + "bgtz %[y], 2b \n" + " addiu %[dst_v], %[dst_v], 1 \n" + + "3: \n" + ".set pop \n" + : [src_uv] "+r" (src_uv), + [dst_u] "+r" (dst_u), + [dst_v] "+r" (dst_v), + [x] "=&r" (x), + [y] "+r" (y) + : [width] "r" (width) + : "t0", "t1", "t2", "t3", "t4", + "t5", "t7", "t8", "t9" + ); +} + +// Convert (4 Y and 2 VU) I422 and arrange RGB values into +// t5 = | 0 | B0 | 0 | b0 | +// t4 = | 0 | B1 | 0 | b1 | +// t9 = | 0 | G0 | 0 | g0 | +// t8 = | 0 | G1 | 0 | g1 | +// t2 = | 0 | R0 | 0 | r0 | +// t1 = | 0 | R1 | 0 | r1 | +#define I422ToTransientMipsRGB \ + "lw $t0, 0(%[y_buf]) \n" \ + "lhu $t1, 0(%[u_buf]) \n" \ + "lhu $t2, 0(%[v_buf]) \n" \ + "preceu.ph.qbr $t1, $t1 \n" \ + "preceu.ph.qbr $t2, $t2 \n" \ + "preceu.ph.qbra $t3, $t0 \n" \ + "preceu.ph.qbla $t0, $t0 \n" \ + "subu.ph $t1, $t1, $s5 \n" \ + "subu.ph $t2, $t2, $s5 \n" \ + "subu.ph $t3, $t3, $s4 \n" \ + "subu.ph $t0, $t0, $s4 \n" \ + "mul.ph $t3, $t3, $s0 \n" \ + "mul.ph $t0, $t0, $s0 \n" \ + "shll.ph $t4, $t1, 0x7 \n" \ + "subu.ph $t4, $t4, $t1 \n" \ + "mul.ph $t6, $t1, $s1 \n" \ + "mul.ph $t1, $t2, $s2 \n" \ + "addq_s.ph $t5, $t4, $t3 \n" \ + "addq_s.ph $t4, $t4, $t0 \n" \ + "shra.ph $t5, $t5, 6 \n" \ + "shra.ph $t4, $t4, 6 \n" \ + "addiu %[u_buf], 2 \n" \ + "addiu %[v_buf], 2 \n" \ + "addu.ph $t6, $t6, $t1 \n" \ + "mul.ph $t1, $t2, $s3 \n" \ + "addu.ph $t9, $t6, $t3 \n" \ + "addu.ph $t8, $t6, $t0 \n" \ + "shra.ph $t9, $t9, 6 \n" \ + "shra.ph $t8, $t8, 6 \n" \ + "addu.ph $t2, $t1, $t3 \n" \ + "addu.ph $t1, $t1, $t0 \n" \ + "shra.ph $t2, $t2, 6 \n" \ + "shra.ph $t1, $t1, 6 \n" \ + "subu.ph $t5, $t5, $s5 \n" \ + "subu.ph $t4, $t4, $s5 \n" \ + "subu.ph $t9, $t9, $s5 \n" \ + "subu.ph $t8, $t8, $s5 \n" \ + "subu.ph $t2, $t2, $s5 \n" \ + "subu.ph $t1, $t1, $s5 \n" \ + "shll_s.ph $t5, $t5, 8 \n" \ + "shll_s.ph $t4, $t4, 8 \n" \ + "shll_s.ph $t9, $t9, 8 \n" \ + "shll_s.ph $t8, $t8, 8 \n" \ + "shll_s.ph $t2, $t2, 8 \n" \ + "shll_s.ph $t1, $t1, 8 \n" \ + "shra.ph $t5, $t5, 8 \n" \ + "shra.ph $t4, $t4, 8 \n" \ + "shra.ph $t9, $t9, 8 \n" \ + "shra.ph $t8, $t8, 8 \n" \ + "shra.ph $t2, $t2, 8 \n" \ + "shra.ph $t1, $t1, 8 \n" \ + "addu.ph $t5, $t5, $s5 \n" \ + "addu.ph $t4, $t4, $s5 \n" \ + "addu.ph $t9, $t9, $s5 \n" \ + "addu.ph $t8, $t8, $s5 \n" \ + "addu.ph $t2, $t2, $s5 \n" \ + "addu.ph $t1, $t1, $s5 \n" + +void I422ToARGBRow_MIPS_DSPR2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) { + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + "beqz %[width], 2f \n" + " repl.ph $s0, 74 \n" // |YG|YG| = |74|74| + "repl.ph $s1, -25 \n" // |UG|UG| = |-25|-25| + "repl.ph $s2, -52 \n" // |VG|VG| = |-52|-52| + "repl.ph $s3, 102 \n" // |VR|VR| = |102|102| + "repl.ph $s4, 16 \n" // |0|16|0|16| + "repl.ph $s5, 128 \n" // |128|128| // clipping + "lui $s6, 0xff00 \n" + "ori $s6, 0xff00 \n" // |ff|00|ff|00|ff| + + ".p2align 2 \n" + "1: \n" + I422ToTransientMipsRGB +// Arranging into argb format + "precr.qb.ph $t4, $t8, $t4 \n" // |G1|g1|B1|b1| + "precr.qb.ph $t5, $t9, $t5 \n" // |G0|g0|B0|b0| + "addiu %[width], -4 \n" + "precrq.qb.ph $t8, $t4, $t5 \n" // |G1|B1|G0|B0| + "precr.qb.ph $t9, $t4, $t5 \n" // |g1|b1|g0|b0| + "precr.qb.ph $t2, $t1, $t2 \n" // |R1|r1|R0|r0| + + "addiu %[y_buf], 4 \n" + "preceu.ph.qbla $t1, $t2 \n" // |0 |R1|0 |R0| + "preceu.ph.qbra $t2, $t2 \n" // |0 |r1|0 |r0| + "or $t1, $t1, $s6 \n" // |ff|R1|ff|R0| + "or $t2, $t2, $s6 \n" // |ff|r1|ff|r0| + "precrq.ph.w $t0, $t2, $t9 \n" // |ff|r1|g1|b1| + "precrq.ph.w $t3, $t1, $t8 \n" // |ff|R1|G1|B1| + "sll $t9, $t9, 16 \n" + "sll $t8, $t8, 16 \n" + "packrl.ph $t2, $t2, $t9 \n" // |ff|r0|g0|b0| + "packrl.ph $t1, $t1, $t8 \n" // |ff|R0|G0|B0| +// Store results. + "sw $t2, 0(%[rgb_buf]) \n" + "sw $t0, 4(%[rgb_buf]) \n" + "sw $t1, 8(%[rgb_buf]) \n" + "sw $t3, 12(%[rgb_buf]) \n" + "bnez %[width], 1b \n" + " addiu %[rgb_buf], 16 \n" + "2: \n" + ".set pop \n" + :[y_buf] "+r" (y_buf), + [u_buf] "+r" (u_buf), + [v_buf] "+r" (v_buf), + [width] "+r" (width), + [rgb_buf] "+r" (rgb_buf) + : + : "t0", "t1", "t2", "t3", "t4", "t5", + "t6", "t7", "t8", "t9", + "s0", "s1", "s2", "s3", + "s4", "s5", "s6" + ); +} + +void I422ToABGRRow_MIPS_DSPR2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) { + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + "beqz %[width], 2f \n" + " repl.ph $s0, 74 \n" // |YG|YG| = |74|74| + "repl.ph $s1, -25 \n" // |UG|UG| = |-25|-25| + "repl.ph $s2, -52 \n" // |VG|VG| = |-52|-52| + "repl.ph $s3, 102 \n" // |VR|VR| = |102|102| + "repl.ph $s4, 16 \n" // |0|16|0|16| + "repl.ph $s5, 128 \n" // |128|128| + "lui $s6, 0xff00 \n" + "ori $s6, 0xff00 \n" // |ff|00|ff|00| + + ".p2align 2 \n" + "1: \n" + I422ToTransientMipsRGB +// Arranging into abgr format + "precr.qb.ph $t0, $t8, $t1 \n" // |G1|g1|R1|r1| + "precr.qb.ph $t3, $t9, $t2 \n" // |G0|g0|R0|r0| + "precrq.qb.ph $t8, $t0, $t3 \n" // |G1|R1|G0|R0| + "precr.qb.ph $t9, $t0, $t3 \n" // |g1|r1|g0|r0| + + "precr.qb.ph $t2, $t4, $t5 \n" // |B1|b1|B0|b0| + "addiu %[width], -4 \n" + "addiu %[y_buf], 4 \n" + "preceu.ph.qbla $t1, $t2 \n" // |0 |B1|0 |B0| + "preceu.ph.qbra $t2, $t2 \n" // |0 |b1|0 |b0| + "or $t1, $t1, $s6 \n" // |ff|B1|ff|B0| + "or $t2, $t2, $s6 \n" // |ff|b1|ff|b0| + "precrq.ph.w $t0, $t2, $t9 \n" // |ff|b1|g1|r1| + "precrq.ph.w $t3, $t1, $t8 \n" // |ff|B1|G1|R1| + "sll $t9, $t9, 16 \n" + "sll $t8, $t8, 16 \n" + "packrl.ph $t2, $t2, $t9 \n" // |ff|b0|g0|r0| + "packrl.ph $t1, $t1, $t8 \n" // |ff|B0|G0|R0| +// Store results. + "sw $t2, 0(%[rgb_buf]) \n" + "sw $t0, 4(%[rgb_buf]) \n" + "sw $t1, 8(%[rgb_buf]) \n" + "sw $t3, 12(%[rgb_buf]) \n" + "bnez %[width], 1b \n" + " addiu %[rgb_buf], 16 \n" + "2: \n" + ".set pop \n" + :[y_buf] "+r" (y_buf), + [u_buf] "+r" (u_buf), + [v_buf] "+r" (v_buf), + [width] "+r" (width), + [rgb_buf] "+r" (rgb_buf) + : + : "t0", "t1", "t2", "t3", "t4", "t5", + "t6", "t7", "t8", "t9", + "s0", "s1", "s2", "s3", + "s4", "s5", "s6" + ); +} + +void I422ToBGRARow_MIPS_DSPR2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) { + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + "beqz %[width], 2f \n" + " repl.ph $s0, 74 \n" // |YG|YG| = |74 |74 | + "repl.ph $s1, -25 \n" // |UG|UG| = |-25|-25| + "repl.ph $s2, -52 \n" // |VG|VG| = |-52|-52| + "repl.ph $s3, 102 \n" // |VR|VR| = |102|102| + "repl.ph $s4, 16 \n" // |0|16|0|16| + "repl.ph $s5, 128 \n" // |128|128| + "lui $s6, 0xff \n" + "ori $s6, 0xff \n" // |00|ff|00|ff| + + ".p2align 2 \n" + "1: \n" + I422ToTransientMipsRGB + // Arranging into bgra format + "precr.qb.ph $t4, $t4, $t8 \n" // |B1|b1|G1|g1| + "precr.qb.ph $t5, $t5, $t9 \n" // |B0|b0|G0|g0| + "precrq.qb.ph $t8, $t4, $t5 \n" // |B1|G1|B0|G0| + "precr.qb.ph $t9, $t4, $t5 \n" // |b1|g1|b0|g0| + + "precr.qb.ph $t2, $t1, $t2 \n" // |R1|r1|R0|r0| + "addiu %[width], -4 \n" + "addiu %[y_buf], 4 \n" + "preceu.ph.qbla $t1, $t2 \n" // |0 |R1|0 |R0| + "preceu.ph.qbra $t2, $t2 \n" // |0 |r1|0 |r0| + "sll $t1, $t1, 8 \n" // |R1|0 |R0|0 | + "sll $t2, $t2, 8 \n" // |r1|0 |r0|0 | + "or $t1, $t1, $s6 \n" // |R1|ff|R0|ff| + "or $t2, $t2, $s6 \n" // |r1|ff|r0|ff| + "precrq.ph.w $t0, $t9, $t2 \n" // |b1|g1|r1|ff| + "precrq.ph.w $t3, $t8, $t1 \n" // |B1|G1|R1|ff| + "sll $t1, $t1, 16 \n" + "sll $t2, $t2, 16 \n" + "packrl.ph $t2, $t9, $t2 \n" // |b0|g0|r0|ff| + "packrl.ph $t1, $t8, $t1 \n" // |B0|G0|R0|ff| +// Store results. + "sw $t2, 0(%[rgb_buf]) \n" + "sw $t0, 4(%[rgb_buf]) \n" + "sw $t1, 8(%[rgb_buf]) \n" + "sw $t3, 12(%[rgb_buf]) \n" + "bnez %[width], 1b \n" + " addiu %[rgb_buf], 16 \n" + "2: \n" + ".set pop \n" + :[y_buf] "+r" (y_buf), + [u_buf] "+r" (u_buf), + [v_buf] "+r" (v_buf), + [width] "+r" (width), + [rgb_buf] "+r" (rgb_buf) + : + : "t0", "t1", "t2", "t3", "t4", "t5", + "t6", "t7", "t8", "t9", + "s0", "s1", "s2", "s3", + "s4", "s5", "s6" + ); +} + +// Bilinear filter 8x2 -> 8x1 +void InterpolateRow_MIPS_DSPR2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) { + int y0_fraction = 256 - source_y_fraction; + const uint8* src_ptr1 = src_ptr + src_stride; + + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + + "replv.ph $t0, %[y0_fraction] \n" + "replv.ph $t1, %[source_y_fraction] \n" + + ".p2align 2 \n" + "1: \n" + "lw $t2, 0(%[src_ptr]) \n" + "lw $t3, 0(%[src_ptr1]) \n" + "lw $t4, 4(%[src_ptr]) \n" + "lw $t5, 4(%[src_ptr1]) \n" + "muleu_s.ph.qbl $t6, $t2, $t0 \n" + "muleu_s.ph.qbr $t7, $t2, $t0 \n" + "muleu_s.ph.qbl $t8, $t3, $t1 \n" + "muleu_s.ph.qbr $t9, $t3, $t1 \n" + "muleu_s.ph.qbl $t2, $t4, $t0 \n" + "muleu_s.ph.qbr $t3, $t4, $t0 \n" + "muleu_s.ph.qbl $t4, $t5, $t1 \n" + "muleu_s.ph.qbr $t5, $t5, $t1 \n" + "addq.ph $t6, $t6, $t8 \n" + "addq.ph $t7, $t7, $t9 \n" + "addq.ph $t2, $t2, $t4 \n" + "addq.ph $t3, $t3, $t5 \n" + "shra.ph $t6, $t6, 8 \n" + "shra.ph $t7, $t7, 8 \n" + "shra.ph $t2, $t2, 8 \n" + "shra.ph $t3, $t3, 8 \n" + "precr.qb.ph $t6, $t6, $t7 \n" + "precr.qb.ph $t2, $t2, $t3 \n" + "addiu %[src_ptr], %[src_ptr], 8 \n" + "addiu %[src_ptr1], %[src_ptr1], 8 \n" + "addiu %[dst_width], %[dst_width], -8 \n" + "sw $t6, 0(%[dst_ptr]) \n" + "sw $t2, 4(%[dst_ptr]) \n" + "bgtz %[dst_width], 1b \n" + " addiu %[dst_ptr], %[dst_ptr], 8 \n" + + ".set pop \n" + : [dst_ptr] "+r" (dst_ptr), + [src_ptr1] "+r" (src_ptr1), + [src_ptr] "+r" (src_ptr), + [dst_width] "+r" (dst_width) + : [source_y_fraction] "r" (source_y_fraction), + [y0_fraction] "r" (y0_fraction), + [src_stride] "r" (src_stride) + : "t0", "t1", "t2", "t3", "t4", "t5", + "t6", "t7", "t8", "t9" + ); +} +#endif // __mips_dsp_rev >= 2 + +#endif // defined(__mips__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/row_neon.cc b/third_party/aom/third_party/libyuv/source/row_neon.cc new file mode 100644 index 0000000000..1a72eb9039 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/row_neon.cc @@ -0,0 +1,3084 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for GCC Neon +#if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \ + !defined(__aarch64__) + +// Read 8 Y, 4 U and 4 V from 422 +#define READYUV422 \ + MEMACCESS(0) \ + "vld1.8 {d0}, [%0]! \n" \ + MEMACCESS(1) \ + "vld1.32 {d2[0]}, [%1]! \n" \ + MEMACCESS(2) \ + "vld1.32 {d2[1]}, [%2]! \n" + +// Read 8 Y, 2 U and 2 V from 422 +#define READYUV411 \ + MEMACCESS(0) \ + "vld1.8 {d0}, [%0]! \n" \ + MEMACCESS(1) \ + "vld1.16 {d2[0]}, [%1]! \n" \ + MEMACCESS(2) \ + "vld1.16 {d2[1]}, [%2]! \n" \ + "vmov.u8 d3, d2 \n" \ + "vzip.u8 d2, d3 \n" + +// Read 8 Y, 8 U and 8 V from 444 +#define READYUV444 \ + MEMACCESS(0) \ + "vld1.8 {d0}, [%0]! \n" \ + MEMACCESS(1) \ + "vld1.8 {d2}, [%1]! \n" \ + MEMACCESS(2) \ + "vld1.8 {d3}, [%2]! \n" \ + "vpaddl.u8 q1, q1 \n" \ + "vrshrn.u16 d2, q1, #1 \n" + +// Read 8 Y, and set 4 U and 4 V to 128 +#define READYUV400 \ + MEMACCESS(0) \ + "vld1.8 {d0}, [%0]! \n" \ + "vmov.u8 d2, #128 \n" + +// Read 8 Y and 4 UV from NV12 +#define READNV12 \ + MEMACCESS(0) \ + "vld1.8 {d0}, [%0]! \n" \ + MEMACCESS(1) \ + "vld1.8 {d2}, [%1]! \n" \ + "vmov.u8 d3, d2 \n"/* split odd/even uv apart */\ + "vuzp.u8 d2, d3 \n" \ + "vtrn.u32 d2, d3 \n" + +// Read 8 Y and 4 VU from NV21 +#define READNV21 \ + MEMACCESS(0) \ + "vld1.8 {d0}, [%0]! \n" \ + MEMACCESS(1) \ + "vld1.8 {d2}, [%1]! \n" \ + "vmov.u8 d3, d2 \n"/* split odd/even uv apart */\ + "vuzp.u8 d3, d2 \n" \ + "vtrn.u32 d2, d3 \n" + +// Read 8 YUY2 +#define READYUY2 \ + MEMACCESS(0) \ + "vld2.8 {d0, d2}, [%0]! \n" \ + "vmov.u8 d3, d2 \n" \ + "vuzp.u8 d2, d3 \n" \ + "vtrn.u32 d2, d3 \n" + +// Read 8 UYVY +#define READUYVY \ + MEMACCESS(0) \ + "vld2.8 {d2, d3}, [%0]! \n" \ + "vmov.u8 d0, d3 \n" \ + "vmov.u8 d3, d2 \n" \ + "vuzp.u8 d2, d3 \n" \ + "vtrn.u32 d2, d3 \n" + +#define YUV422TORGB_SETUP_REG \ + MEMACCESS([kUVToRB]) \ + "vld1.8 {d24}, [%[kUVToRB]] \n" \ + MEMACCESS([kUVToG]) \ + "vld1.8 {d25}, [%[kUVToG]] \n" \ + MEMACCESS([kUVBiasBGR]) \ + "vld1.16 {d26[], d27[]}, [%[kUVBiasBGR]]! \n" \ + MEMACCESS([kUVBiasBGR]) \ + "vld1.16 {d8[], d9[]}, [%[kUVBiasBGR]]! \n" \ + MEMACCESS([kUVBiasBGR]) \ + "vld1.16 {d28[], d29[]}, [%[kUVBiasBGR]] \n" \ + MEMACCESS([kYToRgb]) \ + "vld1.32 {d30[], d31[]}, [%[kYToRgb]] \n" + +#define YUV422TORGB \ + "vmull.u8 q8, d2, d24 \n" /* u/v B/R component */\ + "vmull.u8 q9, d2, d25 \n" /* u/v G component */\ + "vmovl.u8 q0, d0 \n" /* Y */\ + "vmovl.s16 q10, d1 \n" \ + "vmovl.s16 q0, d0 \n" \ + "vmul.s32 q10, q10, q15 \n" \ + "vmul.s32 q0, q0, q15 \n" \ + "vqshrun.s32 d0, q0, #16 \n" \ + "vqshrun.s32 d1, q10, #16 \n" /* Y */\ + "vadd.s16 d18, d19 \n" \ + "vshll.u16 q1, d16, #16 \n" /* Replicate u * UB */\ + "vshll.u16 q10, d17, #16 \n" /* Replicate v * VR */\ + "vshll.u16 q3, d18, #16 \n" /* Replicate (v*VG + u*UG)*/\ + "vaddw.u16 q1, q1, d16 \n" \ + "vaddw.u16 q10, q10, d17 \n" \ + "vaddw.u16 q3, q3, d18 \n" \ + "vqadd.s16 q8, q0, q13 \n" /* B */ \ + "vqadd.s16 q9, q0, q14 \n" /* R */ \ + "vqadd.s16 q0, q0, q4 \n" /* G */ \ + "vqadd.s16 q8, q8, q1 \n" /* B */ \ + "vqadd.s16 q9, q9, q10 \n" /* R */ \ + "vqsub.s16 q0, q0, q3 \n" /* G */ \ + "vqshrun.s16 d20, q8, #6 \n" /* B */ \ + "vqshrun.s16 d22, q9, #6 \n" /* R */ \ + "vqshrun.s16 d21, q0, #6 \n" /* G */ + +// YUV to RGB conversion constants. +// Y contribution to R,G,B. Scale and bias. +#define YG 18997 /* round(1.164 * 64 * 256 * 256 / 257) */ +#define YGB 1160 /* 1.164 * 64 * 16 - adjusted for even error distribution */ + +// U and V contributions to R,G,B. +#define UB -128 /* -min(128, round(2.018 * 64)) */ +#define UG 25 /* -round(-0.391 * 64) */ +#define VG 52 /* -round(-0.813 * 64) */ +#define VR -102 /* -round(1.596 * 64) */ + +// Bias values to subtract 16 from Y and 128 from U and V. +#define BB (UB * 128 - YGB) +#define BG (UG * 128 + VG * 128 - YGB) +#define BR (VR * 128 - YGB) + +static uvec8 kUVToRB = { 128, 128, 128, 128, 102, 102, 102, 102, + 0, 0, 0, 0, 0, 0, 0, 0 }; +static uvec8 kUVToG = { 25, 25, 25, 25, 52, 52, 52, 52, + 0, 0, 0, 0, 0, 0, 0, 0 }; +static vec16 kUVBiasBGR = { BB, BG, BR, 0, 0, 0, 0, 0 }; +static vec32 kYToRgb = { 0x0101 * YG, 0, 0, 0 }; + +#undef YG +#undef YGB +#undef UB +#undef UG +#undef VG +#undef VR +#undef BB +#undef BG +#undef BR + +void I444ToARGBRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READYUV444 + YUV422TORGB + "subs %4, %4, #8 \n" + "vmov.u8 d23, #255 \n" + MEMACCESS(3) + "vst4.8 {d20, d21, d22, d23}, [%3]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_argb), // %3 + "+r"(width) // %4 + : [kUVToRB]"r"(&kUVToRB), // %5 + [kUVToG]"r"(&kUVToG), // %6 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void I422ToARGBRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READYUV422 + YUV422TORGB + "subs %4, %4, #8 \n" + "vmov.u8 d23, #255 \n" + MEMACCESS(3) + "vst4.8 {d20, d21, d22, d23}, [%3]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_argb), // %3 + "+r"(width) // %4 + : [kUVToRB]"r"(&kUVToRB), // %5 + [kUVToG]"r"(&kUVToG), // %6 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void I411ToARGBRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READYUV411 + YUV422TORGB + "subs %4, %4, #8 \n" + "vmov.u8 d23, #255 \n" + MEMACCESS(3) + "vst4.8 {d20, d21, d22, d23}, [%3]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_argb), // %3 + "+r"(width) // %4 + : [kUVToRB]"r"(&kUVToRB), // %5 + [kUVToG]"r"(&kUVToG), // %6 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void I422ToBGRARow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_bgra, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READYUV422 + YUV422TORGB + "subs %4, %4, #8 \n" + "vswp.u8 d20, d22 \n" + "vmov.u8 d19, #255 \n" + MEMACCESS(3) + "vst4.8 {d19, d20, d21, d22}, [%3]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_bgra), // %3 + "+r"(width) // %4 + : [kUVToRB]"r"(&kUVToRB), // %5 + [kUVToG]"r"(&kUVToG), // %6 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void I422ToABGRRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_abgr, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READYUV422 + YUV422TORGB + "subs %4, %4, #8 \n" + "vswp.u8 d20, d22 \n" + "vmov.u8 d23, #255 \n" + MEMACCESS(3) + "vst4.8 {d20, d21, d22, d23}, [%3]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_abgr), // %3 + "+r"(width) // %4 + : [kUVToRB]"r"(&kUVToRB), // %5 + [kUVToG]"r"(&kUVToG), // %6 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void I422ToRGBARow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgba, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READYUV422 + YUV422TORGB + "subs %4, %4, #8 \n" + "vmov.u8 d19, #255 \n" + MEMACCESS(3) + "vst4.8 {d19, d20, d21, d22}, [%3]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_rgba), // %3 + "+r"(width) // %4 + : [kUVToRB]"r"(&kUVToRB), // %5 + [kUVToG]"r"(&kUVToG), // %6 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void I422ToRGB24Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb24, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READYUV422 + YUV422TORGB + "subs %4, %4, #8 \n" + MEMACCESS(3) + "vst3.8 {d20, d21, d22}, [%3]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_rgb24), // %3 + "+r"(width) // %4 + : [kUVToRB]"r"(&kUVToRB), // %5 + [kUVToG]"r"(&kUVToG), // %6 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void I422ToRAWRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_raw, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READYUV422 + YUV422TORGB + "subs %4, %4, #8 \n" + "vswp.u8 d20, d22 \n" + MEMACCESS(3) + "vst3.8 {d20, d21, d22}, [%3]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_raw), // %3 + "+r"(width) // %4 + : [kUVToRB]"r"(&kUVToRB), // %5 + [kUVToG]"r"(&kUVToG), // %6 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +#define ARGBTORGB565 \ + "vshr.u8 d20, d20, #3 \n" /* B */ \ + "vshr.u8 d21, d21, #2 \n" /* G */ \ + "vshr.u8 d22, d22, #3 \n" /* R */ \ + "vmovl.u8 q8, d20 \n" /* B */ \ + "vmovl.u8 q9, d21 \n" /* G */ \ + "vmovl.u8 q10, d22 \n" /* R */ \ + "vshl.u16 q9, q9, #5 \n" /* G */ \ + "vshl.u16 q10, q10, #11 \n" /* R */ \ + "vorr q0, q8, q9 \n" /* BG */ \ + "vorr q0, q0, q10 \n" /* BGR */ + +void I422ToRGB565Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb565, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READYUV422 + YUV422TORGB + "subs %4, %4, #8 \n" + ARGBTORGB565 + MEMACCESS(3) + "vst1.8 {q0}, [%3]! \n" // store 8 pixels RGB565. + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_rgb565), // %3 + "+r"(width) // %4 + : [kUVToRB]"r"(&kUVToRB), // %5 + [kUVToG]"r"(&kUVToG), // %6 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +#define ARGBTOARGB1555 \ + "vshr.u8 q10, q10, #3 \n" /* B */ \ + "vshr.u8 d22, d22, #3 \n" /* R */ \ + "vshr.u8 d23, d23, #7 \n" /* A */ \ + "vmovl.u8 q8, d20 \n" /* B */ \ + "vmovl.u8 q9, d21 \n" /* G */ \ + "vmovl.u8 q10, d22 \n" /* R */ \ + "vmovl.u8 q11, d23 \n" /* A */ \ + "vshl.u16 q9, q9, #5 \n" /* G */ \ + "vshl.u16 q10, q10, #10 \n" /* R */ \ + "vshl.u16 q11, q11, #15 \n" /* A */ \ + "vorr q0, q8, q9 \n" /* BG */ \ + "vorr q1, q10, q11 \n" /* RA */ \ + "vorr q0, q0, q1 \n" /* BGRA */ + +void I422ToARGB1555Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb1555, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READYUV422 + YUV422TORGB + "subs %4, %4, #8 \n" + "vmov.u8 d23, #255 \n" + ARGBTOARGB1555 + MEMACCESS(3) + "vst1.8 {q0}, [%3]! \n" // store 8 pixels ARGB1555. + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_argb1555), // %3 + "+r"(width) // %4 + : [kUVToRB]"r"(&kUVToRB), // %5 + [kUVToG]"r"(&kUVToG), // %6 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +#define ARGBTOARGB4444 \ + "vshr.u8 d20, d20, #4 \n" /* B */ \ + "vbic.32 d21, d21, d4 \n" /* G */ \ + "vshr.u8 d22, d22, #4 \n" /* R */ \ + "vbic.32 d23, d23, d4 \n" /* A */ \ + "vorr d0, d20, d21 \n" /* BG */ \ + "vorr d1, d22, d23 \n" /* RA */ \ + "vzip.u8 d0, d1 \n" /* BGRA */ + +void I422ToARGB4444Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb4444, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "vmov.u8 d4, #0x0f \n" // bits to clear with vbic. + ".p2align 2 \n" + "1: \n" + READYUV422 + YUV422TORGB + "subs %4, %4, #8 \n" + "vmov.u8 d23, #255 \n" + ARGBTOARGB4444 + MEMACCESS(3) + "vst1.8 {q0}, [%3]! \n" // store 8 pixels ARGB4444. + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_argb4444), // %3 + "+r"(width) // %4 + : [kUVToRB]"r"(&kUVToRB), // %5 + [kUVToG]"r"(&kUVToG), // %6 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void I400ToARGBRow_NEON(const uint8* src_y, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READYUV400 + YUV422TORGB + "subs %2, %2, #8 \n" + "vmov.u8 d23, #255 \n" + MEMACCESS(1) + "vst4.8 {d20, d21, d22, d23}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : [kUVToRB]"r"(&kUVToRB), // %3 + [kUVToG]"r"(&kUVToG), // %4 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void J400ToARGBRow_NEON(const uint8* src_y, + uint8* dst_argb, + int width) { + asm volatile ( + "vmov.u8 d23, #255 \n" + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {d20}, [%0]! \n" + "vmov d21, d20 \n" + "vmov d22, d20 \n" + "subs %2, %2, #8 \n" + MEMACCESS(1) + "vst4.8 {d20, d21, d22, d23}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : + : "cc", "memory", "d20", "d21", "d22", "d23" + ); +} + +void NV12ToARGBRow_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READNV12 + YUV422TORGB + "subs %3, %3, #8 \n" + "vmov.u8 d23, #255 \n" + MEMACCESS(2) + "vst4.8 {d20, d21, d22, d23}, [%2]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_uv), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : [kUVToRB]"r"(&kUVToRB), // %4 + [kUVToG]"r"(&kUVToG), // %5 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void NV21ToARGBRow_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READNV21 + YUV422TORGB + "subs %3, %3, #8 \n" + "vmov.u8 d23, #255 \n" + MEMACCESS(2) + "vst4.8 {d20, d21, d22, d23}, [%2]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_uv), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : [kUVToRB]"r"(&kUVToRB), // %4 + [kUVToG]"r"(&kUVToG), // %5 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void NV12ToRGB565Row_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_rgb565, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READNV12 + YUV422TORGB + "subs %3, %3, #8 \n" + ARGBTORGB565 + MEMACCESS(2) + "vst1.8 {q0}, [%2]! \n" // store 8 pixels RGB565. + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_uv), // %1 + "+r"(dst_rgb565), // %2 + "+r"(width) // %3 + : [kUVToRB]"r"(&kUVToRB), // %4 + [kUVToG]"r"(&kUVToG), // %5 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void NV21ToRGB565Row_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_rgb565, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READNV21 + YUV422TORGB + "subs %3, %3, #8 \n" + ARGBTORGB565 + MEMACCESS(2) + "vst1.8 {q0}, [%2]! \n" // store 8 pixels RGB565. + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_uv), // %1 + "+r"(dst_rgb565), // %2 + "+r"(width) // %3 + : [kUVToRB]"r"(&kUVToRB), // %4 + [kUVToG]"r"(&kUVToG), // %5 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void YUY2ToARGBRow_NEON(const uint8* src_yuy2, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READYUY2 + YUV422TORGB + "subs %2, %2, #8 \n" + "vmov.u8 d23, #255 \n" + MEMACCESS(1) + "vst4.8 {d20, d21, d22, d23}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : [kUVToRB]"r"(&kUVToRB), // %3 + [kUVToG]"r"(&kUVToG), // %4 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void UYVYToARGBRow_NEON(const uint8* src_uyvy, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + ".p2align 2 \n" + "1: \n" + READUYVY + YUV422TORGB + "subs %2, %2, #8 \n" + "vmov.u8 d23, #255 \n" + MEMACCESS(1) + "vst4.8 {d20, d21, d22, d23}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : [kUVToRB]"r"(&kUVToRB), // %3 + [kUVToG]"r"(&kUVToG), // %4 + [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +// Reads 16 pairs of UV and write even values to dst_u and odd to dst_v. +void SplitUVRow_NEON(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int width) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld2.8 {q0, q1}, [%0]! \n" // load 16 pairs of UV + "subs %3, %3, #16 \n" // 16 processed per loop + MEMACCESS(1) + "vst1.8 {q0}, [%1]! \n" // store U + MEMACCESS(2) + "vst1.8 {q1}, [%2]! \n" // store V + "bgt 1b \n" + : "+r"(src_uv), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width) // %3 // Output registers + : // Input registers + : "cc", "memory", "q0", "q1" // Clobber List + ); +} + +// Reads 16 U's and V's and writes out 16 pairs of UV. +void MergeUVRow_NEON(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load U + MEMACCESS(1) + "vld1.8 {q1}, [%1]! \n" // load V + "subs %3, %3, #16 \n" // 16 processed per loop + MEMACCESS(2) + "vst2.u8 {q0, q1}, [%2]! \n" // store 16 pairs of UV + "bgt 1b \n" + : + "+r"(src_u), // %0 + "+r"(src_v), // %1 + "+r"(dst_uv), // %2 + "+r"(width) // %3 // Output registers + : // Input registers + : "cc", "memory", "q0", "q1" // Clobber List + ); +} + +// Copy multiple of 32. vld4.8 allow unaligned and is fastest on a15. +void CopyRow_NEON(const uint8* src, uint8* dst, int count) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {d0, d1, d2, d3}, [%0]! \n" // load 32 + "subs %2, %2, #32 \n" // 32 processed per loop + MEMACCESS(1) + "vst1.8 {d0, d1, d2, d3}, [%1]! \n" // store 32 + "bgt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(count) // %2 // Output registers + : // Input registers + : "cc", "memory", "q0", "q1" // Clobber List + ); +} + +// SetRow writes 'count' bytes using an 8 bit value repeated. +void SetRow_NEON(uint8* dst, uint8 v8, int count) { + asm volatile ( + "vdup.8 q0, %2 \n" // duplicate 16 bytes + "1: \n" + "subs %1, %1, #16 \n" // 16 bytes per loop + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" // store + "bgt 1b \n" + : "+r"(dst), // %0 + "+r"(count) // %1 + : "r"(v8) // %2 + : "cc", "memory", "q0" + ); +} + +// ARGBSetRow writes 'count' pixels using an 32 bit value repeated. +void ARGBSetRow_NEON(uint8* dst, uint32 v32, int count) { + asm volatile ( + "vdup.u32 q0, %2 \n" // duplicate 4 ints + "1: \n" + "subs %1, %1, #4 \n" // 4 pixels per loop + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" // store + "bgt 1b \n" + : "+r"(dst), // %0 + "+r"(count) // %1 + : "r"(v32) // %2 + : "cc", "memory", "q0" + ); +} + +void MirrorRow_NEON(const uint8* src, uint8* dst, int width) { + asm volatile ( + // Start at end of source row. + "mov r3, #-16 \n" + "add %0, %0, %2 \n" + "sub %0, #16 \n" + + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0], r3 \n" // src -= 16 + "subs %2, #16 \n" // 16 pixels per loop. + "vrev64.8 q0, q0 \n" + MEMACCESS(1) + "vst1.8 {d1}, [%1]! \n" // dst += 16 + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" + "bgt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : + : "cc", "memory", "r3", "q0" + ); +} + +void MirrorUVRow_NEON(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int width) { + asm volatile ( + // Start at end of source row. + "mov r12, #-16 \n" + "add %0, %0, %3, lsl #1 \n" + "sub %0, #16 \n" + + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld2.8 {d0, d1}, [%0], r12 \n" // src -= 16 + "subs %3, #8 \n" // 8 pixels per loop. + "vrev64.8 q0, q0 \n" + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // dst += 8 + MEMACCESS(2) + "vst1.8 {d1}, [%2]! \n" + "bgt 1b \n" + : "+r"(src_uv), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "r12", "q0" + ); +} + +void ARGBMirrorRow_NEON(const uint8* src, uint8* dst, int width) { + asm volatile ( + // Start at end of source row. + "mov r3, #-16 \n" + "add %0, %0, %2, lsl #2 \n" + "sub %0, #16 \n" + + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0], r3 \n" // src -= 16 + "subs %2, #4 \n" // 4 pixels per loop. + "vrev64.32 q0, q0 \n" + MEMACCESS(1) + "vst1.8 {d1}, [%1]! \n" // dst += 16 + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" + "bgt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : + : "cc", "memory", "r3", "q0" + ); +} + +void RGB24ToARGBRow_NEON(const uint8* src_rgb24, uint8* dst_argb, int pix) { + asm volatile ( + "vmov.u8 d4, #255 \n" // Alpha + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RGB24. + "subs %2, %2, #8 \n" // 8 processed per loop. + MEMACCESS(1) + "vst4.8 {d1, d2, d3, d4}, [%1]! \n" // store 8 pixels of ARGB. + "bgt 1b \n" + : "+r"(src_rgb24), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "d1", "d2", "d3", "d4" // Clobber List + ); +} + +void RAWToARGBRow_NEON(const uint8* src_raw, uint8* dst_argb, int pix) { + asm volatile ( + "vmov.u8 d4, #255 \n" // Alpha + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RAW. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vswp.u8 d1, d3 \n" // swap R, B + MEMACCESS(1) + "vst4.8 {d1, d2, d3, d4}, [%1]! \n" // store 8 pixels of ARGB. + "bgt 1b \n" + : "+r"(src_raw), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "d1", "d2", "d3", "d4" // Clobber List + ); +} + +#define RGB565TOARGB \ + "vshrn.u16 d6, q0, #5 \n" /* G xxGGGGGG */ \ + "vuzp.u8 d0, d1 \n" /* d0 xxxBBBBB RRRRRxxx */ \ + "vshl.u8 d6, d6, #2 \n" /* G GGGGGG00 upper 6 */ \ + "vshr.u8 d1, d1, #3 \n" /* R 000RRRRR lower 5 */ \ + "vshl.u8 q0, q0, #3 \n" /* B,R BBBBB000 upper 5 */ \ + "vshr.u8 q2, q0, #5 \n" /* B,R 00000BBB lower 3 */ \ + "vorr.u8 d0, d0, d4 \n" /* B */ \ + "vshr.u8 d4, d6, #6 \n" /* G 000000GG lower 2 */ \ + "vorr.u8 d2, d1, d5 \n" /* R */ \ + "vorr.u8 d1, d4, d6 \n" /* G */ + +void RGB565ToARGBRow_NEON(const uint8* src_rgb565, uint8* dst_argb, int pix) { + asm volatile ( + "vmov.u8 d3, #255 \n" // Alpha + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load 8 RGB565 pixels. + "subs %2, %2, #8 \n" // 8 processed per loop. + RGB565TOARGB + MEMACCESS(1) + "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of ARGB. + "bgt 1b \n" + : "+r"(src_rgb565), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q1", "q2", "q3" // Clobber List + ); +} + +#define ARGB1555TOARGB \ + "vshrn.u16 d7, q0, #8 \n" /* A Arrrrrxx */ \ + "vshr.u8 d6, d7, #2 \n" /* R xxxRRRRR */ \ + "vshrn.u16 d5, q0, #5 \n" /* G xxxGGGGG */ \ + "vmovn.u16 d4, q0 \n" /* B xxxBBBBB */ \ + "vshr.u8 d7, d7, #7 \n" /* A 0000000A */ \ + "vneg.s8 d7, d7 \n" /* A AAAAAAAA upper 8 */ \ + "vshl.u8 d6, d6, #3 \n" /* R RRRRR000 upper 5 */ \ + "vshr.u8 q1, q3, #5 \n" /* R,A 00000RRR lower 3 */ \ + "vshl.u8 q0, q2, #3 \n" /* B,G BBBBB000 upper 5 */ \ + "vshr.u8 q2, q0, #5 \n" /* B,G 00000BBB lower 3 */ \ + "vorr.u8 q1, q1, q3 \n" /* R,A */ \ + "vorr.u8 q0, q0, q2 \n" /* B,G */ \ + +// RGB555TOARGB is same as ARGB1555TOARGB but ignores alpha. +#define RGB555TOARGB \ + "vshrn.u16 d6, q0, #5 \n" /* G xxxGGGGG */ \ + "vuzp.u8 d0, d1 \n" /* d0 xxxBBBBB xRRRRRxx */ \ + "vshl.u8 d6, d6, #3 \n" /* G GGGGG000 upper 5 */ \ + "vshr.u8 d1, d1, #2 \n" /* R 00xRRRRR lower 5 */ \ + "vshl.u8 q0, q0, #3 \n" /* B,R BBBBB000 upper 5 */ \ + "vshr.u8 q2, q0, #5 \n" /* B,R 00000BBB lower 3 */ \ + "vorr.u8 d0, d0, d4 \n" /* B */ \ + "vshr.u8 d4, d6, #5 \n" /* G 00000GGG lower 3 */ \ + "vorr.u8 d2, d1, d5 \n" /* R */ \ + "vorr.u8 d1, d4, d6 \n" /* G */ + +void ARGB1555ToARGBRow_NEON(const uint8* src_argb1555, uint8* dst_argb, + int pix) { + asm volatile ( + "vmov.u8 d3, #255 \n" // Alpha + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load 8 ARGB1555 pixels. + "subs %2, %2, #8 \n" // 8 processed per loop. + ARGB1555TOARGB + MEMACCESS(1) + "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of ARGB. + "bgt 1b \n" + : "+r"(src_argb1555), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q1", "q2", "q3" // Clobber List + ); +} + +#define ARGB4444TOARGB \ + "vuzp.u8 d0, d1 \n" /* d0 BG, d1 RA */ \ + "vshl.u8 q2, q0, #4 \n" /* B,R BBBB0000 */ \ + "vshr.u8 q1, q0, #4 \n" /* G,A 0000GGGG */ \ + "vshr.u8 q0, q2, #4 \n" /* B,R 0000BBBB */ \ + "vorr.u8 q0, q0, q2 \n" /* B,R BBBBBBBB */ \ + "vshl.u8 q2, q1, #4 \n" /* G,A GGGG0000 */ \ + "vorr.u8 q1, q1, q2 \n" /* G,A GGGGGGGG */ \ + "vswp.u8 d1, d2 \n" /* B,R,G,A -> B,G,R,A */ + +void ARGB4444ToARGBRow_NEON(const uint8* src_argb4444, uint8* dst_argb, + int pix) { + asm volatile ( + "vmov.u8 d3, #255 \n" // Alpha + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load 8 ARGB4444 pixels. + "subs %2, %2, #8 \n" // 8 processed per loop. + ARGB4444TOARGB + MEMACCESS(1) + "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of ARGB. + "bgt 1b \n" + : "+r"(src_argb4444), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q1", "q2" // Clobber List + ); +} + +void ARGBToRGB24Row_NEON(const uint8* src_argb, uint8* dst_rgb24, int pix) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d1, d2, d3, d4}, [%0]! \n" // load 8 pixels of ARGB. + "subs %2, %2, #8 \n" // 8 processed per loop. + MEMACCESS(1) + "vst3.8 {d1, d2, d3}, [%1]! \n" // store 8 pixels of RGB24. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_rgb24), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "d1", "d2", "d3", "d4" // Clobber List + ); +} + +void ARGBToRAWRow_NEON(const uint8* src_argb, uint8* dst_raw, int pix) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d1, d2, d3, d4}, [%0]! \n" // load 8 pixels of ARGB. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vswp.u8 d1, d3 \n" // swap R, B + MEMACCESS(1) + "vst3.8 {d1, d2, d3}, [%1]! \n" // store 8 pixels of RAW. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_raw), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "d1", "d2", "d3", "d4" // Clobber List + ); +} + +void YUY2ToYRow_NEON(const uint8* src_yuy2, uint8* dst_y, int pix) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld2.8 {q0, q1}, [%0]! \n" // load 16 pixels of YUY2. + "subs %2, %2, #16 \n" // 16 processed per loop. + MEMACCESS(1) + "vst1.8 {q0}, [%1]! \n" // store 16 pixels of Y. + "bgt 1b \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q1" // Clobber List + ); +} + +void UYVYToYRow_NEON(const uint8* src_uyvy, uint8* dst_y, int pix) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld2.8 {q0, q1}, [%0]! \n" // load 16 pixels of UYVY. + "subs %2, %2, #16 \n" // 16 processed per loop. + MEMACCESS(1) + "vst1.8 {q1}, [%1]! \n" // store 16 pixels of Y. + "bgt 1b \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q1" // Clobber List + ); +} + +void YUY2ToUV422Row_NEON(const uint8* src_yuy2, uint8* dst_u, uint8* dst_v, + int pix) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of YUY2. + "subs %3, %3, #16 \n" // 16 pixels = 8 UVs. + MEMACCESS(1) + "vst1.8 {d1}, [%1]! \n" // store 8 U. + MEMACCESS(2) + "vst1.8 {d3}, [%2]! \n" // store 8 V. + "bgt 1b \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "cc", "memory", "d0", "d1", "d2", "d3" // Clobber List + ); +} + +void UYVYToUV422Row_NEON(const uint8* src_uyvy, uint8* dst_u, uint8* dst_v, + int pix) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of UYVY. + "subs %3, %3, #16 \n" // 16 pixels = 8 UVs. + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 U. + MEMACCESS(2) + "vst1.8 {d2}, [%2]! \n" // store 8 V. + "bgt 1b \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "cc", "memory", "d0", "d1", "d2", "d3" // Clobber List + ); +} + +void YUY2ToUVRow_NEON(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "add %1, %0, %1 \n" // stride + src_yuy2 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of YUY2. + "subs %4, %4, #16 \n" // 16 pixels = 8 UVs. + MEMACCESS(1) + "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load next row YUY2. + "vrhadd.u8 d1, d1, d5 \n" // average rows of U + "vrhadd.u8 d3, d3, d7 \n" // average rows of V + MEMACCESS(2) + "vst1.8 {d1}, [%2]! \n" // store 8 U. + MEMACCESS(3) + "vst1.8 {d3}, [%3]! \n" // store 8 V. + "bgt 1b \n" + : "+r"(src_yuy2), // %0 + "+r"(stride_yuy2), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7" // Clobber List + ); +} + +void UYVYToUVRow_NEON(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "add %1, %0, %1 \n" // stride + src_uyvy + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of UYVY. + "subs %4, %4, #16 \n" // 16 pixels = 8 UVs. + MEMACCESS(1) + "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load next row UYVY. + "vrhadd.u8 d0, d0, d4 \n" // average rows of U + "vrhadd.u8 d2, d2, d6 \n" // average rows of V + MEMACCESS(2) + "vst1.8 {d0}, [%2]! \n" // store 8 U. + MEMACCESS(3) + "vst1.8 {d2}, [%3]! \n" // store 8 V. + "bgt 1b \n" + : "+r"(src_uyvy), // %0 + "+r"(stride_uyvy), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7" // Clobber List + ); +} + +// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA. +void ARGBShuffleRow_NEON(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix) { + asm volatile ( + MEMACCESS(3) + "vld1.8 {q2}, [%3] \n" // shuffler + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load 4 pixels. + "subs %2, %2, #4 \n" // 4 processed per loop + "vtbl.8 d2, {d0, d1}, d4 \n" // look up 2 first pixels + "vtbl.8 d3, {d0, d1}, d5 \n" // look up 2 next pixels + MEMACCESS(1) + "vst1.8 {q1}, [%1]! \n" // store 4. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : "r"(shuffler) // %3 + : "cc", "memory", "q0", "q1", "q2" // Clobber List + ); +} + +void I422ToYUY2Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_yuy2, int width) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld2.8 {d0, d2}, [%0]! \n" // load 16 Ys + MEMACCESS(1) + "vld1.8 {d1}, [%1]! \n" // load 8 Us + MEMACCESS(2) + "vld1.8 {d3}, [%2]! \n" // load 8 Vs + "subs %4, %4, #16 \n" // 16 pixels + MEMACCESS(3) + "vst4.8 {d0, d1, d2, d3}, [%3]! \n" // Store 8 YUY2/16 pixels. + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_yuy2), // %3 + "+r"(width) // %4 + : + : "cc", "memory", "d0", "d1", "d2", "d3" + ); +} + +void I422ToUYVYRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_uyvy, int width) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld2.8 {d1, d3}, [%0]! \n" // load 16 Ys + MEMACCESS(1) + "vld1.8 {d0}, [%1]! \n" // load 8 Us + MEMACCESS(2) + "vld1.8 {d2}, [%2]! \n" // load 8 Vs + "subs %4, %4, #16 \n" // 16 pixels + MEMACCESS(3) + "vst4.8 {d0, d1, d2, d3}, [%3]! \n" // Store 8 UYVY/16 pixels. + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_uyvy), // %3 + "+r"(width) // %4 + : + : "cc", "memory", "d0", "d1", "d2", "d3" + ); +} + +void ARGBToRGB565Row_NEON(const uint8* src_argb, uint8* dst_rgb565, int pix) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d20, d21, d22, d23}, [%0]! \n" // load 8 pixels of ARGB. + "subs %2, %2, #8 \n" // 8 processed per loop. + ARGBTORGB565 + MEMACCESS(1) + "vst1.8 {q0}, [%1]! \n" // store 8 pixels RGB565. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_rgb565), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q8", "q9", "q10", "q11" + ); +} + +void ARGBToRGB565DitherRow_NEON(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int width) { + asm volatile ( + ".p2align 2 \n" + "vdup.32 d2, %2 \n" // dither4 + "1: \n" + MEMACCESS(1) + "vld4.8 {d20, d21, d22, d23}, [%1]! \n" // load 8 pixels of ARGB. + "subs %3, %3, #8 \n" // 8 processed per loop. + "vqadd.u8 d20, d20, d2 \n" + "vqadd.u8 d21, d21, d2 \n" + "vqadd.u8 d22, d22, d2 \n" + ARGBTORGB565 + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" // store 8 pixels RGB565. + "bgt 1b \n" + : "+r"(dst_rgb) // %0 + : "r"(src_argb), // %1 + "r"(dither4), // %2 + "r"(width) // %3 + : "cc", "memory", "q0", "q1", "q8", "q9", "q10", "q11" + ); +} + +void ARGBToARGB1555Row_NEON(const uint8* src_argb, uint8* dst_argb1555, + int pix) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d20, d21, d22, d23}, [%0]! \n" // load 8 pixels of ARGB. + "subs %2, %2, #8 \n" // 8 processed per loop. + ARGBTOARGB1555 + MEMACCESS(1) + "vst1.8 {q0}, [%1]! \n" // store 8 pixels ARGB1555. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb1555), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q8", "q9", "q10", "q11" + ); +} + +void ARGBToARGB4444Row_NEON(const uint8* src_argb, uint8* dst_argb4444, + int pix) { + asm volatile ( + "vmov.u8 d4, #0x0f \n" // bits to clear with vbic. + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d20, d21, d22, d23}, [%0]! \n" // load 8 pixels of ARGB. + "subs %2, %2, #8 \n" // 8 processed per loop. + ARGBTOARGB4444 + MEMACCESS(1) + "vst1.8 {q0}, [%1]! \n" // store 8 pixels ARGB4444. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb4444), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q8", "q9", "q10", "q11" + ); +} + +void ARGBToYRow_NEON(const uint8* src_argb, uint8* dst_y, int pix) { + asm volatile ( + "vmov.u8 d24, #13 \n" // B * 0.1016 coefficient + "vmov.u8 d25, #65 \n" // G * 0.5078 coefficient + "vmov.u8 d26, #33 \n" // R * 0.2578 coefficient + "vmov.u8 d27, #16 \n" // Add 16 constant + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vmull.u8 q2, d0, d24 \n" // B + "vmlal.u8 q2, d1, d25 \n" // G + "vmlal.u8 q2, d2, d26 \n" // R + "vqrshrun.s16 d0, q2, #7 \n" // 16 bit to 8 bit Y + "vqadd.u8 d0, d27 \n" + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q1", "q2", "q12", "q13" + ); +} + +void ARGBToYJRow_NEON(const uint8* src_argb, uint8* dst_y, int pix) { + asm volatile ( + "vmov.u8 d24, #15 \n" // B * 0.11400 coefficient + "vmov.u8 d25, #75 \n" // G * 0.58700 coefficient + "vmov.u8 d26, #38 \n" // R * 0.29900 coefficient + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vmull.u8 q2, d0, d24 \n" // B + "vmlal.u8 q2, d1, d25 \n" // G + "vmlal.u8 q2, d2, d26 \n" // R + "vqrshrun.s16 d0, q2, #7 \n" // 15 bit to 8 bit Y + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q1", "q2", "q12", "q13" + ); +} + +// 8x1 pixels. +void ARGBToUV444Row_NEON(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix) { + asm volatile ( + "vmov.u8 d24, #112 \n" // UB / VR 0.875 coefficient + "vmov.u8 d25, #74 \n" // UG -0.5781 coefficient + "vmov.u8 d26, #38 \n" // UR -0.2969 coefficient + "vmov.u8 d27, #18 \n" // VB -0.1406 coefficient + "vmov.u8 d28, #94 \n" // VG -0.7344 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels. + "subs %3, %3, #8 \n" // 8 processed per loop. + "vmull.u8 q2, d0, d24 \n" // B + "vmlsl.u8 q2, d1, d25 \n" // G + "vmlsl.u8 q2, d2, d26 \n" // R + "vadd.u16 q2, q2, q15 \n" // +128 -> unsigned + + "vmull.u8 q3, d2, d24 \n" // R + "vmlsl.u8 q3, d1, d28 \n" // G + "vmlsl.u8 q3, d0, d27 \n" // B + "vadd.u16 q3, q3, q15 \n" // +128 -> unsigned + + "vqshrn.u16 d0, q2, #8 \n" // 16 bit to 8 bit U + "vqshrn.u16 d1, q3, #8 \n" // 16 bit to 8 bit V + + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels U. + MEMACCESS(2) + "vst1.8 {d1}, [%2]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q12", "q13", "q14", "q15" + ); +} + +// 16x1 pixels -> 8x1. pix is number of argb pixels. e.g. 16. +void ARGBToUV422Row_NEON(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix) { + asm volatile ( + "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient + "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient + "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient + "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient + "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels. + MEMACCESS(0) + "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels. + + "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts. + + "subs %3, %3, #16 \n" // 16 processed per loop. + "vmul.s16 q8, q0, q10 \n" // B + "vmls.s16 q8, q1, q11 \n" // G + "vmls.s16 q8, q2, q12 \n" // R + "vadd.u16 q8, q8, q15 \n" // +128 -> unsigned + + "vmul.s16 q9, q2, q10 \n" // R + "vmls.s16 q9, q1, q14 \n" // G + "vmls.s16 q9, q0, q13 \n" // B + "vadd.u16 q9, q9, q15 \n" // +128 -> unsigned + + "vqshrn.u16 d0, q8, #8 \n" // 16 bit to 8 bit U + "vqshrn.u16 d1, q9, #8 \n" // 16 bit to 8 bit V + + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels U. + MEMACCESS(2) + "vst1.8 {d1}, [%2]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "cc", "memory", "q0", "q1", "q2", "q3", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +// 32x1 pixels -> 8x1. pix is number of argb pixels. e.g. 32. +void ARGBToUV411Row_NEON(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix) { + asm volatile ( + "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient + "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient + "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient + "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient + "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels. + MEMACCESS(0) + "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels. + "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts. + MEMACCESS(0) + "vld4.8 {d8, d10, d12, d14}, [%0]! \n" // load 8 more ARGB pixels. + MEMACCESS(0) + "vld4.8 {d9, d11, d13, d15}, [%0]! \n" // load last 8 ARGB pixels. + "vpaddl.u8 q4, q4 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q5, q5 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q6, q6 \n" // R 16 bytes -> 8 shorts. + + "vpadd.u16 d0, d0, d1 \n" // B 16 shorts -> 8 shorts. + "vpadd.u16 d1, d8, d9 \n" // B + "vpadd.u16 d2, d2, d3 \n" // G 16 shorts -> 8 shorts. + "vpadd.u16 d3, d10, d11 \n" // G + "vpadd.u16 d4, d4, d5 \n" // R 16 shorts -> 8 shorts. + "vpadd.u16 d5, d12, d13 \n" // R + + "vrshr.u16 q0, q0, #1 \n" // 2x average + "vrshr.u16 q1, q1, #1 \n" + "vrshr.u16 q2, q2, #1 \n" + + "subs %3, %3, #32 \n" // 32 processed per loop. + "vmul.s16 q8, q0, q10 \n" // B + "vmls.s16 q8, q1, q11 \n" // G + "vmls.s16 q8, q2, q12 \n" // R + "vadd.u16 q8, q8, q15 \n" // +128 -> unsigned + "vmul.s16 q9, q2, q10 \n" // R + "vmls.s16 q9, q1, q14 \n" // G + "vmls.s16 q9, q0, q13 \n" // B + "vadd.u16 q9, q9, q15 \n" // +128 -> unsigned + "vqshrn.u16 d0, q8, #8 \n" // 16 bit to 8 bit U + "vqshrn.u16 d1, q9, #8 \n" // 16 bit to 8 bit V + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels U. + MEMACCESS(2) + "vst1.8 {d1}, [%2]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +// 16x2 pixels -> 8x1. pix is number of argb pixels. e.g. 16. +#define RGBTOUV(QB, QG, QR) \ + "vmul.s16 q8, " #QB ", q10 \n" /* B */ \ + "vmls.s16 q8, " #QG ", q11 \n" /* G */ \ + "vmls.s16 q8, " #QR ", q12 \n" /* R */ \ + "vadd.u16 q8, q8, q15 \n" /* +128 -> unsigned */ \ + "vmul.s16 q9, " #QR ", q10 \n" /* R */ \ + "vmls.s16 q9, " #QG ", q14 \n" /* G */ \ + "vmls.s16 q9, " #QB ", q13 \n" /* B */ \ + "vadd.u16 q9, q9, q15 \n" /* +128 -> unsigned */ \ + "vqshrn.u16 d0, q8, #8 \n" /* 16 bit to 8 bit U */ \ + "vqshrn.u16 d1, q9, #8 \n" /* 16 bit to 8 bit V */ + +// TODO(fbarchard): Consider vhadd vertical, then vpaddl horizontal, avoid shr. +void ARGBToUVRow_NEON(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "add %1, %0, %1 \n" // src_stride + src_argb + "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient + "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient + "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient + "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient + "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels. + MEMACCESS(0) + "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels. + "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more ARGB pixels. + MEMACCESS(1) + "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 ARGB pixels. + "vpadal.u8 q0, q4 \n" // B 16 bytes -> 8 shorts. + "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts. + "vpadal.u8 q2, q6 \n" // R 16 bytes -> 8 shorts. + + "vrshr.u16 q0, q0, #1 \n" // 2x average + "vrshr.u16 q1, q1, #1 \n" + "vrshr.u16 q2, q2, #1 \n" + + "subs %4, %4, #16 \n" // 32 processed per loop. + RGBTOUV(q0, q1, q2) + MEMACCESS(2) + "vst1.8 {d0}, [%2]! \n" // store 8 pixels U. + MEMACCESS(3) + "vst1.8 {d1}, [%3]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_stride_argb), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +// TODO(fbarchard): Subsample match C code. +void ARGBToUVJRow_NEON(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "add %1, %0, %1 \n" // src_stride + src_argb + "vmov.s16 q10, #127 / 2 \n" // UB / VR 0.500 coefficient + "vmov.s16 q11, #84 / 2 \n" // UG -0.33126 coefficient + "vmov.s16 q12, #43 / 2 \n" // UR -0.16874 coefficient + "vmov.s16 q13, #20 / 2 \n" // VB -0.08131 coefficient + "vmov.s16 q14, #107 / 2 \n" // VG -0.41869 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels. + MEMACCESS(0) + "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels. + "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more ARGB pixels. + MEMACCESS(1) + "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 ARGB pixels. + "vpadal.u8 q0, q4 \n" // B 16 bytes -> 8 shorts. + "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts. + "vpadal.u8 q2, q6 \n" // R 16 bytes -> 8 shorts. + + "vrshr.u16 q0, q0, #1 \n" // 2x average + "vrshr.u16 q1, q1, #1 \n" + "vrshr.u16 q2, q2, #1 \n" + + "subs %4, %4, #16 \n" // 32 processed per loop. + RGBTOUV(q0, q1, q2) + MEMACCESS(2) + "vst1.8 {d0}, [%2]! \n" // store 8 pixels U. + MEMACCESS(3) + "vst1.8 {d1}, [%3]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_stride_argb), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void BGRAToUVRow_NEON(const uint8* src_bgra, int src_stride_bgra, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "add %1, %0, %1 \n" // src_stride + src_bgra + "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient + "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient + "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient + "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient + "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 BGRA pixels. + MEMACCESS(0) + "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 BGRA pixels. + "vpaddl.u8 q3, q3 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q2, q2 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q1, q1 \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more BGRA pixels. + MEMACCESS(1) + "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 BGRA pixels. + "vpadal.u8 q3, q7 \n" // B 16 bytes -> 8 shorts. + "vpadal.u8 q2, q6 \n" // G 16 bytes -> 8 shorts. + "vpadal.u8 q1, q5 \n" // R 16 bytes -> 8 shorts. + + "vrshr.u16 q1, q1, #1 \n" // 2x average + "vrshr.u16 q2, q2, #1 \n" + "vrshr.u16 q3, q3, #1 \n" + + "subs %4, %4, #16 \n" // 32 processed per loop. + RGBTOUV(q3, q2, q1) + MEMACCESS(2) + "vst1.8 {d0}, [%2]! \n" // store 8 pixels U. + MEMACCESS(3) + "vst1.8 {d1}, [%3]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_bgra), // %0 + "+r"(src_stride_bgra), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void ABGRToUVRow_NEON(const uint8* src_abgr, int src_stride_abgr, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "add %1, %0, %1 \n" // src_stride + src_abgr + "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient + "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient + "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient + "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient + "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ABGR pixels. + MEMACCESS(0) + "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ABGR pixels. + "vpaddl.u8 q2, q2 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q0, q0 \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more ABGR pixels. + MEMACCESS(1) + "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 ABGR pixels. + "vpadal.u8 q2, q6 \n" // B 16 bytes -> 8 shorts. + "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts. + "vpadal.u8 q0, q4 \n" // R 16 bytes -> 8 shorts. + + "vrshr.u16 q0, q0, #1 \n" // 2x average + "vrshr.u16 q1, q1, #1 \n" + "vrshr.u16 q2, q2, #1 \n" + + "subs %4, %4, #16 \n" // 32 processed per loop. + RGBTOUV(q2, q1, q0) + MEMACCESS(2) + "vst1.8 {d0}, [%2]! \n" // store 8 pixels U. + MEMACCESS(3) + "vst1.8 {d1}, [%3]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_abgr), // %0 + "+r"(src_stride_abgr), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void RGBAToUVRow_NEON(const uint8* src_rgba, int src_stride_rgba, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "add %1, %0, %1 \n" // src_stride + src_rgba + "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient + "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient + "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient + "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient + "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 RGBA pixels. + MEMACCESS(0) + "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 RGBA pixels. + "vpaddl.u8 q0, q1 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q1, q2 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q2, q3 \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more RGBA pixels. + MEMACCESS(1) + "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 RGBA pixels. + "vpadal.u8 q0, q5 \n" // B 16 bytes -> 8 shorts. + "vpadal.u8 q1, q6 \n" // G 16 bytes -> 8 shorts. + "vpadal.u8 q2, q7 \n" // R 16 bytes -> 8 shorts. + + "vrshr.u16 q0, q0, #1 \n" // 2x average + "vrshr.u16 q1, q1, #1 \n" + "vrshr.u16 q2, q2, #1 \n" + + "subs %4, %4, #16 \n" // 32 processed per loop. + RGBTOUV(q0, q1, q2) + MEMACCESS(2) + "vst1.8 {d0}, [%2]! \n" // store 8 pixels U. + MEMACCESS(3) + "vst1.8 {d1}, [%3]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_rgba), // %0 + "+r"(src_stride_rgba), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void RGB24ToUVRow_NEON(const uint8* src_rgb24, int src_stride_rgb24, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "add %1, %0, %1 \n" // src_stride + src_rgb24 + "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient + "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient + "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient + "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient + "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RGB24 pixels. + MEMACCESS(0) + "vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RGB24 pixels. + "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "vld3.8 {d8, d10, d12}, [%1]! \n" // load 8 more RGB24 pixels. + MEMACCESS(1) + "vld3.8 {d9, d11, d13}, [%1]! \n" // load last 8 RGB24 pixels. + "vpadal.u8 q0, q4 \n" // B 16 bytes -> 8 shorts. + "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts. + "vpadal.u8 q2, q6 \n" // R 16 bytes -> 8 shorts. + + "vrshr.u16 q0, q0, #1 \n" // 2x average + "vrshr.u16 q1, q1, #1 \n" + "vrshr.u16 q2, q2, #1 \n" + + "subs %4, %4, #16 \n" // 32 processed per loop. + RGBTOUV(q0, q1, q2) + MEMACCESS(2) + "vst1.8 {d0}, [%2]! \n" // store 8 pixels U. + MEMACCESS(3) + "vst1.8 {d1}, [%3]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_rgb24), // %0 + "+r"(src_stride_rgb24), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void RAWToUVRow_NEON(const uint8* src_raw, int src_stride_raw, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "add %1, %0, %1 \n" // src_stride + src_raw + "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient + "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient + "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient + "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient + "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RAW pixels. + MEMACCESS(0) + "vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RAW pixels. + "vpaddl.u8 q2, q2 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q0, q0 \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "vld3.8 {d8, d10, d12}, [%1]! \n" // load 8 more RAW pixels. + MEMACCESS(1) + "vld3.8 {d9, d11, d13}, [%1]! \n" // load last 8 RAW pixels. + "vpadal.u8 q2, q6 \n" // B 16 bytes -> 8 shorts. + "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts. + "vpadal.u8 q0, q4 \n" // R 16 bytes -> 8 shorts. + + "vrshr.u16 q0, q0, #1 \n" // 2x average + "vrshr.u16 q1, q1, #1 \n" + "vrshr.u16 q2, q2, #1 \n" + + "subs %4, %4, #16 \n" // 32 processed per loop. + RGBTOUV(q2, q1, q0) + MEMACCESS(2) + "vst1.8 {d0}, [%2]! \n" // store 8 pixels U. + MEMACCESS(3) + "vst1.8 {d1}, [%3]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_raw), // %0 + "+r"(src_stride_raw), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +// 16x2 pixels -> 8x1. pix is number of argb pixels. e.g. 16. +void RGB565ToUVRow_NEON(const uint8* src_rgb565, int src_stride_rgb565, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "add %1, %0, %1 \n" // src_stride + src_argb + "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient + "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient + "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient + "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient + "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load 8 RGB565 pixels. + RGB565TOARGB + "vpaddl.u8 d8, d0 \n" // B 8 bytes -> 4 shorts. + "vpaddl.u8 d10, d1 \n" // G 8 bytes -> 4 shorts. + "vpaddl.u8 d12, d2 \n" // R 8 bytes -> 4 shorts. + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // next 8 RGB565 pixels. + RGB565TOARGB + "vpaddl.u8 d9, d0 \n" // B 8 bytes -> 4 shorts. + "vpaddl.u8 d11, d1 \n" // G 8 bytes -> 4 shorts. + "vpaddl.u8 d13, d2 \n" // R 8 bytes -> 4 shorts. + + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" // load 8 RGB565 pixels. + RGB565TOARGB + "vpadal.u8 d8, d0 \n" // B 8 bytes -> 4 shorts. + "vpadal.u8 d10, d1 \n" // G 8 bytes -> 4 shorts. + "vpadal.u8 d12, d2 \n" // R 8 bytes -> 4 shorts. + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" // next 8 RGB565 pixels. + RGB565TOARGB + "vpadal.u8 d9, d0 \n" // B 8 bytes -> 4 shorts. + "vpadal.u8 d11, d1 \n" // G 8 bytes -> 4 shorts. + "vpadal.u8 d13, d2 \n" // R 8 bytes -> 4 shorts. + + "vrshr.u16 q4, q4, #1 \n" // 2x average + "vrshr.u16 q5, q5, #1 \n" + "vrshr.u16 q6, q6, #1 \n" + + "subs %4, %4, #16 \n" // 16 processed per loop. + "vmul.s16 q8, q4, q10 \n" // B + "vmls.s16 q8, q5, q11 \n" // G + "vmls.s16 q8, q6, q12 \n" // R + "vadd.u16 q8, q8, q15 \n" // +128 -> unsigned + "vmul.s16 q9, q6, q10 \n" // R + "vmls.s16 q9, q5, q14 \n" // G + "vmls.s16 q9, q4, q13 \n" // B + "vadd.u16 q9, q9, q15 \n" // +128 -> unsigned + "vqshrn.u16 d0, q8, #8 \n" // 16 bit to 8 bit U + "vqshrn.u16 d1, q9, #8 \n" // 16 bit to 8 bit V + MEMACCESS(2) + "vst1.8 {d0}, [%2]! \n" // store 8 pixels U. + MEMACCESS(3) + "vst1.8 {d1}, [%3]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_rgb565), // %0 + "+r"(src_stride_rgb565), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +// 16x2 pixels -> 8x1. pix is number of argb pixels. e.g. 16. +void ARGB1555ToUVRow_NEON(const uint8* src_argb1555, int src_stride_argb1555, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "add %1, %0, %1 \n" // src_stride + src_argb + "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient + "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient + "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient + "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient + "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load 8 ARGB1555 pixels. + RGB555TOARGB + "vpaddl.u8 d8, d0 \n" // B 8 bytes -> 4 shorts. + "vpaddl.u8 d10, d1 \n" // G 8 bytes -> 4 shorts. + "vpaddl.u8 d12, d2 \n" // R 8 bytes -> 4 shorts. + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // next 8 ARGB1555 pixels. + RGB555TOARGB + "vpaddl.u8 d9, d0 \n" // B 8 bytes -> 4 shorts. + "vpaddl.u8 d11, d1 \n" // G 8 bytes -> 4 shorts. + "vpaddl.u8 d13, d2 \n" // R 8 bytes -> 4 shorts. + + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" // load 8 ARGB1555 pixels. + RGB555TOARGB + "vpadal.u8 d8, d0 \n" // B 8 bytes -> 4 shorts. + "vpadal.u8 d10, d1 \n" // G 8 bytes -> 4 shorts. + "vpadal.u8 d12, d2 \n" // R 8 bytes -> 4 shorts. + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" // next 8 ARGB1555 pixels. + RGB555TOARGB + "vpadal.u8 d9, d0 \n" // B 8 bytes -> 4 shorts. + "vpadal.u8 d11, d1 \n" // G 8 bytes -> 4 shorts. + "vpadal.u8 d13, d2 \n" // R 8 bytes -> 4 shorts. + + "vrshr.u16 q4, q4, #1 \n" // 2x average + "vrshr.u16 q5, q5, #1 \n" + "vrshr.u16 q6, q6, #1 \n" + + "subs %4, %4, #16 \n" // 16 processed per loop. + "vmul.s16 q8, q4, q10 \n" // B + "vmls.s16 q8, q5, q11 \n" // G + "vmls.s16 q8, q6, q12 \n" // R + "vadd.u16 q8, q8, q15 \n" // +128 -> unsigned + "vmul.s16 q9, q6, q10 \n" // R + "vmls.s16 q9, q5, q14 \n" // G + "vmls.s16 q9, q4, q13 \n" // B + "vadd.u16 q9, q9, q15 \n" // +128 -> unsigned + "vqshrn.u16 d0, q8, #8 \n" // 16 bit to 8 bit U + "vqshrn.u16 d1, q9, #8 \n" // 16 bit to 8 bit V + MEMACCESS(2) + "vst1.8 {d0}, [%2]! \n" // store 8 pixels U. + MEMACCESS(3) + "vst1.8 {d1}, [%3]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_argb1555), // %0 + "+r"(src_stride_argb1555), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +// 16x2 pixels -> 8x1. pix is number of argb pixels. e.g. 16. +void ARGB4444ToUVRow_NEON(const uint8* src_argb4444, int src_stride_argb4444, + uint8* dst_u, uint8* dst_v, int pix) { + asm volatile ( + "add %1, %0, %1 \n" // src_stride + src_argb + "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient + "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient + "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient + "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient + "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient + "vmov.u16 q15, #0x8080 \n" // 128.5 + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load 8 ARGB4444 pixels. + ARGB4444TOARGB + "vpaddl.u8 d8, d0 \n" // B 8 bytes -> 4 shorts. + "vpaddl.u8 d10, d1 \n" // G 8 bytes -> 4 shorts. + "vpaddl.u8 d12, d2 \n" // R 8 bytes -> 4 shorts. + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // next 8 ARGB4444 pixels. + ARGB4444TOARGB + "vpaddl.u8 d9, d0 \n" // B 8 bytes -> 4 shorts. + "vpaddl.u8 d11, d1 \n" // G 8 bytes -> 4 shorts. + "vpaddl.u8 d13, d2 \n" // R 8 bytes -> 4 shorts. + + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" // load 8 ARGB4444 pixels. + ARGB4444TOARGB + "vpadal.u8 d8, d0 \n" // B 8 bytes -> 4 shorts. + "vpadal.u8 d10, d1 \n" // G 8 bytes -> 4 shorts. + "vpadal.u8 d12, d2 \n" // R 8 bytes -> 4 shorts. + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" // next 8 ARGB4444 pixels. + ARGB4444TOARGB + "vpadal.u8 d9, d0 \n" // B 8 bytes -> 4 shorts. + "vpadal.u8 d11, d1 \n" // G 8 bytes -> 4 shorts. + "vpadal.u8 d13, d2 \n" // R 8 bytes -> 4 shorts. + + "vrshr.u16 q4, q4, #1 \n" // 2x average + "vrshr.u16 q5, q5, #1 \n" + "vrshr.u16 q6, q6, #1 \n" + + "subs %4, %4, #16 \n" // 16 processed per loop. + "vmul.s16 q8, q4, q10 \n" // B + "vmls.s16 q8, q5, q11 \n" // G + "vmls.s16 q8, q6, q12 \n" // R + "vadd.u16 q8, q8, q15 \n" // +128 -> unsigned + "vmul.s16 q9, q6, q10 \n" // R + "vmls.s16 q9, q5, q14 \n" // G + "vmls.s16 q9, q4, q13 \n" // B + "vadd.u16 q9, q9, q15 \n" // +128 -> unsigned + "vqshrn.u16 d0, q8, #8 \n" // 16 bit to 8 bit U + "vqshrn.u16 d1, q9, #8 \n" // 16 bit to 8 bit V + MEMACCESS(2) + "vst1.8 {d0}, [%2]! \n" // store 8 pixels U. + MEMACCESS(3) + "vst1.8 {d1}, [%3]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_argb4444), // %0 + "+r"(src_stride_argb4444), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +void RGB565ToYRow_NEON(const uint8* src_rgb565, uint8* dst_y, int pix) { + asm volatile ( + "vmov.u8 d24, #13 \n" // B * 0.1016 coefficient + "vmov.u8 d25, #65 \n" // G * 0.5078 coefficient + "vmov.u8 d26, #33 \n" // R * 0.2578 coefficient + "vmov.u8 d27, #16 \n" // Add 16 constant + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load 8 RGB565 pixels. + "subs %2, %2, #8 \n" // 8 processed per loop. + RGB565TOARGB + "vmull.u8 q2, d0, d24 \n" // B + "vmlal.u8 q2, d1, d25 \n" // G + "vmlal.u8 q2, d2, d26 \n" // R + "vqrshrun.s16 d0, q2, #7 \n" // 16 bit to 8 bit Y + "vqadd.u8 d0, d27 \n" + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y. + "bgt 1b \n" + : "+r"(src_rgb565), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q12", "q13" + ); +} + +void ARGB1555ToYRow_NEON(const uint8* src_argb1555, uint8* dst_y, int pix) { + asm volatile ( + "vmov.u8 d24, #13 \n" // B * 0.1016 coefficient + "vmov.u8 d25, #65 \n" // G * 0.5078 coefficient + "vmov.u8 d26, #33 \n" // R * 0.2578 coefficient + "vmov.u8 d27, #16 \n" // Add 16 constant + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load 8 ARGB1555 pixels. + "subs %2, %2, #8 \n" // 8 processed per loop. + ARGB1555TOARGB + "vmull.u8 q2, d0, d24 \n" // B + "vmlal.u8 q2, d1, d25 \n" // G + "vmlal.u8 q2, d2, d26 \n" // R + "vqrshrun.s16 d0, q2, #7 \n" // 16 bit to 8 bit Y + "vqadd.u8 d0, d27 \n" + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y. + "bgt 1b \n" + : "+r"(src_argb1555), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q12", "q13" + ); +} + +void ARGB4444ToYRow_NEON(const uint8* src_argb4444, uint8* dst_y, int pix) { + asm volatile ( + "vmov.u8 d24, #13 \n" // B * 0.1016 coefficient + "vmov.u8 d25, #65 \n" // G * 0.5078 coefficient + "vmov.u8 d26, #33 \n" // R * 0.2578 coefficient + "vmov.u8 d27, #16 \n" // Add 16 constant + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load 8 ARGB4444 pixels. + "subs %2, %2, #8 \n" // 8 processed per loop. + ARGB4444TOARGB + "vmull.u8 q2, d0, d24 \n" // B + "vmlal.u8 q2, d1, d25 \n" // G + "vmlal.u8 q2, d2, d26 \n" // R + "vqrshrun.s16 d0, q2, #7 \n" // 16 bit to 8 bit Y + "vqadd.u8 d0, d27 \n" + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y. + "bgt 1b \n" + : "+r"(src_argb4444), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q12", "q13" + ); +} + +void BGRAToYRow_NEON(const uint8* src_bgra, uint8* dst_y, int pix) { + asm volatile ( + "vmov.u8 d4, #33 \n" // R * 0.2578 coefficient + "vmov.u8 d5, #65 \n" // G * 0.5078 coefficient + "vmov.u8 d6, #13 \n" // B * 0.1016 coefficient + "vmov.u8 d7, #16 \n" // Add 16 constant + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of BGRA. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vmull.u8 q8, d1, d4 \n" // R + "vmlal.u8 q8, d2, d5 \n" // G + "vmlal.u8 q8, d3, d6 \n" // B + "vqrshrun.s16 d0, q8, #7 \n" // 16 bit to 8 bit Y + "vqadd.u8 d0, d7 \n" + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y. + "bgt 1b \n" + : "+r"(src_bgra), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "q8" + ); +} + +void ABGRToYRow_NEON(const uint8* src_abgr, uint8* dst_y, int pix) { + asm volatile ( + "vmov.u8 d4, #33 \n" // R * 0.2578 coefficient + "vmov.u8 d5, #65 \n" // G * 0.5078 coefficient + "vmov.u8 d6, #13 \n" // B * 0.1016 coefficient + "vmov.u8 d7, #16 \n" // Add 16 constant + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of ABGR. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vmull.u8 q8, d0, d4 \n" // R + "vmlal.u8 q8, d1, d5 \n" // G + "vmlal.u8 q8, d2, d6 \n" // B + "vqrshrun.s16 d0, q8, #7 \n" // 16 bit to 8 bit Y + "vqadd.u8 d0, d7 \n" + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y. + "bgt 1b \n" + : "+r"(src_abgr), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "q8" + ); +} + +void RGBAToYRow_NEON(const uint8* src_rgba, uint8* dst_y, int pix) { + asm volatile ( + "vmov.u8 d4, #13 \n" // B * 0.1016 coefficient + "vmov.u8 d5, #65 \n" // G * 0.5078 coefficient + "vmov.u8 d6, #33 \n" // R * 0.2578 coefficient + "vmov.u8 d7, #16 \n" // Add 16 constant + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of RGBA. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vmull.u8 q8, d1, d4 \n" // B + "vmlal.u8 q8, d2, d5 \n" // G + "vmlal.u8 q8, d3, d6 \n" // R + "vqrshrun.s16 d0, q8, #7 \n" // 16 bit to 8 bit Y + "vqadd.u8 d0, d7 \n" + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y. + "bgt 1b \n" + : "+r"(src_rgba), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "q8" + ); +} + +void RGB24ToYRow_NEON(const uint8* src_rgb24, uint8* dst_y, int pix) { + asm volatile ( + "vmov.u8 d4, #13 \n" // B * 0.1016 coefficient + "vmov.u8 d5, #65 \n" // G * 0.5078 coefficient + "vmov.u8 d6, #33 \n" // R * 0.2578 coefficient + "vmov.u8 d7, #16 \n" // Add 16 constant + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld3.8 {d0, d1, d2}, [%0]! \n" // load 8 pixels of RGB24. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vmull.u8 q8, d0, d4 \n" // B + "vmlal.u8 q8, d1, d5 \n" // G + "vmlal.u8 q8, d2, d6 \n" // R + "vqrshrun.s16 d0, q8, #7 \n" // 16 bit to 8 bit Y + "vqadd.u8 d0, d7 \n" + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y. + "bgt 1b \n" + : "+r"(src_rgb24), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "q8" + ); +} + +void RAWToYRow_NEON(const uint8* src_raw, uint8* dst_y, int pix) { + asm volatile ( + "vmov.u8 d4, #33 \n" // R * 0.2578 coefficient + "vmov.u8 d5, #65 \n" // G * 0.5078 coefficient + "vmov.u8 d6, #13 \n" // B * 0.1016 coefficient + "vmov.u8 d7, #16 \n" // Add 16 constant + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld3.8 {d0, d1, d2}, [%0]! \n" // load 8 pixels of RAW. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vmull.u8 q8, d0, d4 \n" // B + "vmlal.u8 q8, d1, d5 \n" // G + "vmlal.u8 q8, d2, d6 \n" // R + "vqrshrun.s16 d0, q8, #7 \n" // 16 bit to 8 bit Y + "vqadd.u8 d0, d7 \n" + MEMACCESS(1) + "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y. + "bgt 1b \n" + : "+r"(src_raw), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "q8" + ); +} + +// Bilinear filter 16x2 -> 16x1 +void InterpolateRow_NEON(uint8* dst_ptr, + const uint8* src_ptr, ptrdiff_t src_stride, + int dst_width, int source_y_fraction) { + asm volatile ( + "cmp %4, #0 \n" + "beq 100f \n" + "add %2, %1 \n" + "cmp %4, #64 \n" + "beq 75f \n" + "cmp %4, #128 \n" + "beq 50f \n" + "cmp %4, #192 \n" + "beq 25f \n" + + "vdup.8 d5, %4 \n" + "rsb %4, #256 \n" + "vdup.8 d4, %4 \n" + // General purpose row blend. + "1: \n" + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" + MEMACCESS(2) + "vld1.8 {q1}, [%2]! \n" + "subs %3, %3, #16 \n" + "vmull.u8 q13, d0, d4 \n" + "vmull.u8 q14, d1, d4 \n" + "vmlal.u8 q13, d2, d5 \n" + "vmlal.u8 q14, d3, d5 \n" + "vrshrn.u16 d0, q13, #8 \n" + "vrshrn.u16 d1, q14, #8 \n" + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" + "bgt 1b \n" + "b 99f \n" + + // Blend 25 / 75. + "25: \n" + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" + MEMACCESS(2) + "vld1.8 {q1}, [%2]! \n" + "subs %3, %3, #16 \n" + "vrhadd.u8 q0, q1 \n" + "vrhadd.u8 q0, q1 \n" + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" + "bgt 25b \n" + "b 99f \n" + + // Blend 50 / 50. + "50: \n" + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" + MEMACCESS(2) + "vld1.8 {q1}, [%2]! \n" + "subs %3, %3, #16 \n" + "vrhadd.u8 q0, q1 \n" + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" + "bgt 50b \n" + "b 99f \n" + + // Blend 75 / 25. + "75: \n" + MEMACCESS(1) + "vld1.8 {q1}, [%1]! \n" + MEMACCESS(2) + "vld1.8 {q0}, [%2]! \n" + "subs %3, %3, #16 \n" + "vrhadd.u8 q0, q1 \n" + "vrhadd.u8 q0, q1 \n" + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" + "bgt 75b \n" + "b 99f \n" + + // Blend 100 / 0 - Copy row unchanged. + "100: \n" + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" + "subs %3, %3, #16 \n" + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" + "bgt 100b \n" + + "99: \n" + : "+r"(dst_ptr), // %0 + "+r"(src_ptr), // %1 + "+r"(src_stride), // %2 + "+r"(dst_width), // %3 + "+r"(source_y_fraction) // %4 + : + : "cc", "memory", "q0", "q1", "d4", "d5", "q13", "q14" + ); +} + +// dr * (256 - sa) / 256 + sr = dr - dr * sa / 256 + sr +void ARGBBlendRow_NEON(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + "subs %3, #8 \n" + "blt 89f \n" + // Blend 8 pixels. + "8: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of ARGB0. + MEMACCESS(1) + "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load 8 pixels of ARGB1. + "subs %3, %3, #8 \n" // 8 processed per loop. + "vmull.u8 q10, d4, d3 \n" // db * a + "vmull.u8 q11, d5, d3 \n" // dg * a + "vmull.u8 q12, d6, d3 \n" // dr * a + "vqrshrn.u16 d20, q10, #8 \n" // db >>= 8 + "vqrshrn.u16 d21, q11, #8 \n" // dg >>= 8 + "vqrshrn.u16 d22, q12, #8 \n" // dr >>= 8 + "vqsub.u8 q2, q2, q10 \n" // dbg - dbg * a / 256 + "vqsub.u8 d6, d6, d22 \n" // dr - dr * a / 256 + "vqadd.u8 q0, q0, q2 \n" // + sbg + "vqadd.u8 d2, d2, d6 \n" // + sr + "vmov.u8 d3, #255 \n" // a = 255 + MEMACCESS(2) + "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 pixels of ARGB. + "bge 8b \n" + + "89: \n" + "adds %3, #8-1 \n" + "blt 99f \n" + + // Blend 1 pixels. + "1: \n" + MEMACCESS(0) + "vld4.8 {d0[0],d1[0],d2[0],d3[0]}, [%0]! \n" // load 1 pixel ARGB0. + MEMACCESS(1) + "vld4.8 {d4[0],d5[0],d6[0],d7[0]}, [%1]! \n" // load 1 pixel ARGB1. + "subs %3, %3, #1 \n" // 1 processed per loop. + "vmull.u8 q10, d4, d3 \n" // db * a + "vmull.u8 q11, d5, d3 \n" // dg * a + "vmull.u8 q12, d6, d3 \n" // dr * a + "vqrshrn.u16 d20, q10, #8 \n" // db >>= 8 + "vqrshrn.u16 d21, q11, #8 \n" // dg >>= 8 + "vqrshrn.u16 d22, q12, #8 \n" // dr >>= 8 + "vqsub.u8 q2, q2, q10 \n" // dbg - dbg * a / 256 + "vqsub.u8 d6, d6, d22 \n" // dr - dr * a / 256 + "vqadd.u8 q0, q0, q2 \n" // + sbg + "vqadd.u8 d2, d2, d6 \n" // + sr + "vmov.u8 d3, #255 \n" // a = 255 + MEMACCESS(2) + "vst4.8 {d0[0],d1[0],d2[0],d3[0]}, [%2]! \n" // store 1 pixel. + "bge 1b \n" + + "99: \n" + + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q10", "q11", "q12" + ); +} + +// Attenuate 8 pixels at a time. +void ARGBAttenuateRow_NEON(const uint8* src_argb, uint8* dst_argb, int width) { + asm volatile ( + // Attenuate 8 pixels. + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of ARGB. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vmull.u8 q10, d0, d3 \n" // b * a + "vmull.u8 q11, d1, d3 \n" // g * a + "vmull.u8 q12, d2, d3 \n" // r * a + "vqrshrn.u16 d0, q10, #8 \n" // b >>= 8 + "vqrshrn.u16 d1, q11, #8 \n" // g >>= 8 + "vqrshrn.u16 d2, q12, #8 \n" // r >>= 8 + MEMACCESS(1) + "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of ARGB. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : + : "cc", "memory", "q0", "q1", "q10", "q11", "q12" + ); +} + +// Quantize 8 ARGB pixels (32 bytes). +// dst = (dst * scale >> 16) * interval_size + interval_offset; +void ARGBQuantizeRow_NEON(uint8* dst_argb, int scale, int interval_size, + int interval_offset, int width) { + asm volatile ( + "vdup.u16 q8, %2 \n" + "vshr.u16 q8, q8, #1 \n" // scale >>= 1 + "vdup.u16 q9, %3 \n" // interval multiply. + "vdup.u16 q10, %4 \n" // interval add + + // 8 pixel loop. + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d2, d4, d6}, [%0] \n" // load 8 pixels of ARGB. + "subs %1, %1, #8 \n" // 8 processed per loop. + "vmovl.u8 q0, d0 \n" // b (0 .. 255) + "vmovl.u8 q1, d2 \n" + "vmovl.u8 q2, d4 \n" + "vqdmulh.s16 q0, q0, q8 \n" // b * scale + "vqdmulh.s16 q1, q1, q8 \n" // g + "vqdmulh.s16 q2, q2, q8 \n" // r + "vmul.u16 q0, q0, q9 \n" // b * interval_size + "vmul.u16 q1, q1, q9 \n" // g + "vmul.u16 q2, q2, q9 \n" // r + "vadd.u16 q0, q0, q10 \n" // b + interval_offset + "vadd.u16 q1, q1, q10 \n" // g + "vadd.u16 q2, q2, q10 \n" // r + "vqmovn.u16 d0, q0 \n" + "vqmovn.u16 d2, q1 \n" + "vqmovn.u16 d4, q2 \n" + MEMACCESS(0) + "vst4.8 {d0, d2, d4, d6}, [%0]! \n" // store 8 pixels of ARGB. + "bgt 1b \n" + : "+r"(dst_argb), // %0 + "+r"(width) // %1 + : "r"(scale), // %2 + "r"(interval_size), // %3 + "r"(interval_offset) // %4 + : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10" + ); +} + +// Shade 8 pixels at a time by specified value. +// NOTE vqrdmulh.s16 q10, q10, d0[0] must use a scaler register from 0 to 8. +// Rounding in vqrdmulh does +1 to high if high bit of low s16 is set. +void ARGBShadeRow_NEON(const uint8* src_argb, uint8* dst_argb, int width, + uint32 value) { + asm volatile ( + "vdup.u32 q0, %3 \n" // duplicate scale value. + "vzip.u8 d0, d1 \n" // d0 aarrggbb. + "vshr.u16 q0, q0, #1 \n" // scale / 2. + + // 8 pixel loop. + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d20, d22, d24, d26}, [%0]! \n" // load 8 pixels of ARGB. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vmovl.u8 q10, d20 \n" // b (0 .. 255) + "vmovl.u8 q11, d22 \n" + "vmovl.u8 q12, d24 \n" + "vmovl.u8 q13, d26 \n" + "vqrdmulh.s16 q10, q10, d0[0] \n" // b * scale * 2 + "vqrdmulh.s16 q11, q11, d0[1] \n" // g + "vqrdmulh.s16 q12, q12, d0[2] \n" // r + "vqrdmulh.s16 q13, q13, d0[3] \n" // a + "vqmovn.u16 d20, q10 \n" + "vqmovn.u16 d22, q11 \n" + "vqmovn.u16 d24, q12 \n" + "vqmovn.u16 d26, q13 \n" + MEMACCESS(1) + "vst4.8 {d20, d22, d24, d26}, [%1]! \n" // store 8 pixels of ARGB. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(value) // %3 + : "cc", "memory", "q0", "q10", "q11", "q12", "q13" + ); +} + +// Convert 8 ARGB pixels (64 bytes) to 8 Gray ARGB pixels +// Similar to ARGBToYJ but stores ARGB. +// C code is (15 * b + 75 * g + 38 * r + 64) >> 7; +void ARGBGrayRow_NEON(const uint8* src_argb, uint8* dst_argb, int width) { + asm volatile ( + "vmov.u8 d24, #15 \n" // B * 0.11400 coefficient + "vmov.u8 d25, #75 \n" // G * 0.58700 coefficient + "vmov.u8 d26, #38 \n" // R * 0.29900 coefficient + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vmull.u8 q2, d0, d24 \n" // B + "vmlal.u8 q2, d1, d25 \n" // G + "vmlal.u8 q2, d2, d26 \n" // R + "vqrshrun.s16 d0, q2, #7 \n" // 15 bit to 8 bit B + "vmov d1, d0 \n" // G + "vmov d2, d0 \n" // R + MEMACCESS(1) + "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 ARGB pixels. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : + : "cc", "memory", "q0", "q1", "q2", "q12", "q13" + ); +} + +// Convert 8 ARGB pixels (32 bytes) to 8 Sepia ARGB pixels. +// b = (r * 35 + g * 68 + b * 17) >> 7 +// g = (r * 45 + g * 88 + b * 22) >> 7 +// r = (r * 50 + g * 98 + b * 24) >> 7 +void ARGBSepiaRow_NEON(uint8* dst_argb, int width) { + asm volatile ( + "vmov.u8 d20, #17 \n" // BB coefficient + "vmov.u8 d21, #68 \n" // BG coefficient + "vmov.u8 d22, #35 \n" // BR coefficient + "vmov.u8 d24, #22 \n" // GB coefficient + "vmov.u8 d25, #88 \n" // GG coefficient + "vmov.u8 d26, #45 \n" // GR coefficient + "vmov.u8 d28, #24 \n" // BB coefficient + "vmov.u8 d29, #98 \n" // BG coefficient + "vmov.u8 d30, #50 \n" // BR coefficient + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0] \n" // load 8 ARGB pixels. + "subs %1, %1, #8 \n" // 8 processed per loop. + "vmull.u8 q2, d0, d20 \n" // B to Sepia B + "vmlal.u8 q2, d1, d21 \n" // G + "vmlal.u8 q2, d2, d22 \n" // R + "vmull.u8 q3, d0, d24 \n" // B to Sepia G + "vmlal.u8 q3, d1, d25 \n" // G + "vmlal.u8 q3, d2, d26 \n" // R + "vmull.u8 q8, d0, d28 \n" // B to Sepia R + "vmlal.u8 q8, d1, d29 \n" // G + "vmlal.u8 q8, d2, d30 \n" // R + "vqshrn.u16 d0, q2, #7 \n" // 16 bit to 8 bit B + "vqshrn.u16 d1, q3, #7 \n" // 16 bit to 8 bit G + "vqshrn.u16 d2, q8, #7 \n" // 16 bit to 8 bit R + MEMACCESS(0) + "vst4.8 {d0, d1, d2, d3}, [%0]! \n" // store 8 ARGB pixels. + "bgt 1b \n" + : "+r"(dst_argb), // %0 + "+r"(width) // %1 + : + : "cc", "memory", "q0", "q1", "q2", "q3", + "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +// Tranform 8 ARGB pixels (32 bytes) with color matrix. +// TODO(fbarchard): Was same as Sepia except matrix is provided. This function +// needs to saturate. Consider doing a non-saturating version. +void ARGBColorMatrixRow_NEON(const uint8* src_argb, uint8* dst_argb, + const int8* matrix_argb, int width) { + asm volatile ( + MEMACCESS(3) + "vld1.8 {q2}, [%3] \n" // load 3 ARGB vectors. + "vmovl.s8 q0, d4 \n" // B,G coefficients s16. + "vmovl.s8 q1, d5 \n" // R,A coefficients s16. + + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d16, d18, d20, d22}, [%0]! \n" // load 8 ARGB pixels. + "subs %2, %2, #8 \n" // 8 processed per loop. + "vmovl.u8 q8, d16 \n" // b (0 .. 255) 16 bit + "vmovl.u8 q9, d18 \n" // g + "vmovl.u8 q10, d20 \n" // r + "vmovl.u8 q11, d22 \n" // a + "vmul.s16 q12, q8, d0[0] \n" // B = B * Matrix B + "vmul.s16 q13, q8, d1[0] \n" // G = B * Matrix G + "vmul.s16 q14, q8, d2[0] \n" // R = B * Matrix R + "vmul.s16 q15, q8, d3[0] \n" // A = B * Matrix A + "vmul.s16 q4, q9, d0[1] \n" // B += G * Matrix B + "vmul.s16 q5, q9, d1[1] \n" // G += G * Matrix G + "vmul.s16 q6, q9, d2[1] \n" // R += G * Matrix R + "vmul.s16 q7, q9, d3[1] \n" // A += G * Matrix A + "vqadd.s16 q12, q12, q4 \n" // Accumulate B + "vqadd.s16 q13, q13, q5 \n" // Accumulate G + "vqadd.s16 q14, q14, q6 \n" // Accumulate R + "vqadd.s16 q15, q15, q7 \n" // Accumulate A + "vmul.s16 q4, q10, d0[2] \n" // B += R * Matrix B + "vmul.s16 q5, q10, d1[2] \n" // G += R * Matrix G + "vmul.s16 q6, q10, d2[2] \n" // R += R * Matrix R + "vmul.s16 q7, q10, d3[2] \n" // A += R * Matrix A + "vqadd.s16 q12, q12, q4 \n" // Accumulate B + "vqadd.s16 q13, q13, q5 \n" // Accumulate G + "vqadd.s16 q14, q14, q6 \n" // Accumulate R + "vqadd.s16 q15, q15, q7 \n" // Accumulate A + "vmul.s16 q4, q11, d0[3] \n" // B += A * Matrix B + "vmul.s16 q5, q11, d1[3] \n" // G += A * Matrix G + "vmul.s16 q6, q11, d2[3] \n" // R += A * Matrix R + "vmul.s16 q7, q11, d3[3] \n" // A += A * Matrix A + "vqadd.s16 q12, q12, q4 \n" // Accumulate B + "vqadd.s16 q13, q13, q5 \n" // Accumulate G + "vqadd.s16 q14, q14, q6 \n" // Accumulate R + "vqadd.s16 q15, q15, q7 \n" // Accumulate A + "vqshrun.s16 d16, q12, #6 \n" // 16 bit to 8 bit B + "vqshrun.s16 d18, q13, #6 \n" // 16 bit to 8 bit G + "vqshrun.s16 d20, q14, #6 \n" // 16 bit to 8 bit R + "vqshrun.s16 d22, q15, #6 \n" // 16 bit to 8 bit A + MEMACCESS(1) + "vst4.8 {d16, d18, d20, d22}, [%1]! \n" // store 8 ARGB pixels. + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(matrix_argb) // %3 + : "cc", "memory", "q0", "q1", "q2", "q4", "q5", "q6", "q7", "q8", "q9", + "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +// TODO(fbarchard): fix vqshrun in ARGBMultiplyRow_NEON and reenable. +#ifdef HAS_ARGBMULTIPLYROW_NEON +// Multiply 2 rows of ARGB pixels together, 8 pixels at a time. +void ARGBMultiplyRow_NEON(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + // 8 pixel loop. + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels. + MEMACCESS(1) + "vld4.8 {d1, d3, d5, d7}, [%1]! \n" // load 8 more ARGB pixels. + "subs %3, %3, #8 \n" // 8 processed per loop. + "vmull.u8 q0, d0, d1 \n" // multiply B + "vmull.u8 q1, d2, d3 \n" // multiply G + "vmull.u8 q2, d4, d5 \n" // multiply R + "vmull.u8 q3, d6, d7 \n" // multiply A + "vrshrn.u16 d0, q0, #8 \n" // 16 bit to 8 bit B + "vrshrn.u16 d1, q1, #8 \n" // 16 bit to 8 bit G + "vrshrn.u16 d2, q2, #8 \n" // 16 bit to 8 bit R + "vrshrn.u16 d3, q3, #8 \n" // 16 bit to 8 bit A + MEMACCESS(2) + "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels. + "bgt 1b \n" + + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "q0", "q1", "q2", "q3" + ); +} +#endif // HAS_ARGBMULTIPLYROW_NEON + +// Add 2 rows of ARGB pixels together, 8 pixels at a time. +void ARGBAddRow_NEON(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + // 8 pixel loop. + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels. + MEMACCESS(1) + "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load 8 more ARGB pixels. + "subs %3, %3, #8 \n" // 8 processed per loop. + "vqadd.u8 q0, q0, q2 \n" // add B, G + "vqadd.u8 q1, q1, q3 \n" // add R, A + MEMACCESS(2) + "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels. + "bgt 1b \n" + + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "q0", "q1", "q2", "q3" + ); +} + +// Subtract 2 rows of ARGB pixels, 8 pixels at a time. +void ARGBSubtractRow_NEON(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + // 8 pixel loop. + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels. + MEMACCESS(1) + "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load 8 more ARGB pixels. + "subs %3, %3, #8 \n" // 8 processed per loop. + "vqsub.u8 q0, q0, q2 \n" // subtract B, G + "vqsub.u8 q1, q1, q3 \n" // subtract R, A + MEMACCESS(2) + "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels. + "bgt 1b \n" + + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "q0", "q1", "q2", "q3" + ); +} + +// Adds Sobel X and Sobel Y and stores Sobel into ARGB. +// A = 255 +// R = Sobel +// G = Sobel +// B = Sobel +void SobelRow_NEON(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width) { + asm volatile ( + "vmov.u8 d3, #255 \n" // alpha + // 8 pixel loop. + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {d0}, [%0]! \n" // load 8 sobelx. + MEMACCESS(1) + "vld1.8 {d1}, [%1]! \n" // load 8 sobely. + "subs %3, %3, #8 \n" // 8 processed per loop. + "vqadd.u8 d0, d0, d1 \n" // add + "vmov.u8 d1, d0 \n" + "vmov.u8 d2, d0 \n" + MEMACCESS(2) + "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels. + "bgt 1b \n" + : "+r"(src_sobelx), // %0 + "+r"(src_sobely), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "q0", "q1" + ); +} + +// Adds Sobel X and Sobel Y and stores Sobel into plane. +void SobelToPlaneRow_NEON(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_y, int width) { + asm volatile ( + // 16 pixel loop. + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load 16 sobelx. + MEMACCESS(1) + "vld1.8 {q1}, [%1]! \n" // load 16 sobely. + "subs %3, %3, #16 \n" // 16 processed per loop. + "vqadd.u8 q0, q0, q1 \n" // add + MEMACCESS(2) + "vst1.8 {q0}, [%2]! \n" // store 16 pixels. + "bgt 1b \n" + : "+r"(src_sobelx), // %0 + "+r"(src_sobely), // %1 + "+r"(dst_y), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "q0", "q1" + ); +} + +// Mixes Sobel X, Sobel Y and Sobel into ARGB. +// A = 255 +// R = Sobel X +// G = Sobel +// B = Sobel Y +void SobelXYRow_NEON(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width) { + asm volatile ( + "vmov.u8 d3, #255 \n" // alpha + // 8 pixel loop. + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {d2}, [%0]! \n" // load 8 sobelx. + MEMACCESS(1) + "vld1.8 {d0}, [%1]! \n" // load 8 sobely. + "subs %3, %3, #8 \n" // 8 processed per loop. + "vqadd.u8 d1, d0, d2 \n" // add + MEMACCESS(2) + "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels. + "bgt 1b \n" + : "+r"(src_sobelx), // %0 + "+r"(src_sobely), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "q0", "q1" + ); +} + +// SobelX as a matrix is +// -1 0 1 +// -2 0 2 +// -1 0 1 +void SobelXRow_NEON(const uint8* src_y0, const uint8* src_y1, + const uint8* src_y2, uint8* dst_sobelx, int width) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {d0}, [%0],%5 \n" // top + MEMACCESS(0) + "vld1.8 {d1}, [%0],%6 \n" + "vsubl.u8 q0, d0, d1 \n" + MEMACCESS(1) + "vld1.8 {d2}, [%1],%5 \n" // center * 2 + MEMACCESS(1) + "vld1.8 {d3}, [%1],%6 \n" + "vsubl.u8 q1, d2, d3 \n" + "vadd.s16 q0, q0, q1 \n" + "vadd.s16 q0, q0, q1 \n" + MEMACCESS(2) + "vld1.8 {d2}, [%2],%5 \n" // bottom + MEMACCESS(2) + "vld1.8 {d3}, [%2],%6 \n" + "subs %4, %4, #8 \n" // 8 pixels + "vsubl.u8 q1, d2, d3 \n" + "vadd.s16 q0, q0, q1 \n" + "vabs.s16 q0, q0 \n" + "vqmovn.u16 d0, q0 \n" + MEMACCESS(3) + "vst1.8 {d0}, [%3]! \n" // store 8 sobelx + "bgt 1b \n" + : "+r"(src_y0), // %0 + "+r"(src_y1), // %1 + "+r"(src_y2), // %2 + "+r"(dst_sobelx), // %3 + "+r"(width) // %4 + : "r"(2), // %5 + "r"(6) // %6 + : "cc", "memory", "q0", "q1" // Clobber List + ); +} + +// SobelY as a matrix is +// -1 -2 -1 +// 0 0 0 +// 1 2 1 +void SobelYRow_NEON(const uint8* src_y0, const uint8* src_y1, + uint8* dst_sobely, int width) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {d0}, [%0],%4 \n" // left + MEMACCESS(1) + "vld1.8 {d1}, [%1],%4 \n" + "vsubl.u8 q0, d0, d1 \n" + MEMACCESS(0) + "vld1.8 {d2}, [%0],%4 \n" // center * 2 + MEMACCESS(1) + "vld1.8 {d3}, [%1],%4 \n" + "vsubl.u8 q1, d2, d3 \n" + "vadd.s16 q0, q0, q1 \n" + "vadd.s16 q0, q0, q1 \n" + MEMACCESS(0) + "vld1.8 {d2}, [%0],%5 \n" // right + MEMACCESS(1) + "vld1.8 {d3}, [%1],%5 \n" + "subs %3, %3, #8 \n" // 8 pixels + "vsubl.u8 q1, d2, d3 \n" + "vadd.s16 q0, q0, q1 \n" + "vabs.s16 q0, q0 \n" + "vqmovn.u16 d0, q0 \n" + MEMACCESS(2) + "vst1.8 {d0}, [%2]! \n" // store 8 sobely + "bgt 1b \n" + : "+r"(src_y0), // %0 + "+r"(src_y1), // %1 + "+r"(dst_sobely), // %2 + "+r"(width) // %3 + : "r"(1), // %4 + "r"(6) // %5 + : "cc", "memory", "q0", "q1" // Clobber List + ); +} +#endif // defined(__ARM_NEON__) && !defined(__aarch64__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/row_neon64.cc b/third_party/aom/third_party/libyuv/source/row_neon64.cc new file mode 100644 index 0000000000..5d015454b0 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/row_neon64.cc @@ -0,0 +1,3087 @@ +/* + * Copyright 2014 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for GCC Neon armv8 64 bit. +#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) + +// Read 8 Y, 4 U and 4 V from 422 +#define READYUV422 \ + MEMACCESS(0) \ + "ld1 {v0.8b}, [%0], #8 \n" \ + MEMACCESS(1) \ + "ld1 {v1.s}[0], [%1], #4 \n" \ + MEMACCESS(2) \ + "ld1 {v1.s}[1], [%2], #4 \n" + +// Read 8 Y, 2 U and 2 V from 422 +#define READYUV411 \ + MEMACCESS(0) \ + "ld1 {v0.8b}, [%0], #8 \n" \ + MEMACCESS(1) \ + "ld1 {v2.h}[0], [%1], #2 \n" \ + MEMACCESS(2) \ + "ld1 {v2.h}[1], [%2], #2 \n" \ + "zip1 v1.8b, v2.8b, v2.8b \n" + +// Read 8 Y, 8 U and 8 V from 444 +#define READYUV444 \ + MEMACCESS(0) \ + "ld1 {v0.8b}, [%0], #8 \n" \ + MEMACCESS(1) \ + "ld1 {v1.d}[0], [%1], #8 \n" \ + MEMACCESS(2) \ + "ld1 {v1.d}[1], [%2], #8 \n" \ + "uaddlp v1.8h, v1.16b \n" \ + "rshrn v1.8b, v1.8h, #1 \n" + +// Read 8 Y, and set 4 U and 4 V to 128 +#define READYUV400 \ + MEMACCESS(0) \ + "ld1 {v0.8b}, [%0], #8 \n" \ + "movi v1.8b , #128 \n" + +// Read 8 Y and 4 UV from NV12 +#define READNV12 \ + MEMACCESS(0) \ + "ld1 {v0.8b}, [%0], #8 \n" \ + MEMACCESS(1) \ + "ld1 {v2.8b}, [%1], #8 \n" \ + "uzp1 v1.8b, v2.8b, v2.8b \n" \ + "uzp2 v3.8b, v2.8b, v2.8b \n" \ + "ins v1.s[1], v3.s[0] \n" + +// Read 8 Y and 4 VU from NV21 +#define READNV21 \ + MEMACCESS(0) \ + "ld1 {v0.8b}, [%0], #8 \n" \ + MEMACCESS(1) \ + "ld1 {v2.8b}, [%1], #8 \n" \ + "uzp1 v3.8b, v2.8b, v2.8b \n" \ + "uzp2 v1.8b, v2.8b, v2.8b \n" \ + "ins v1.s[1], v3.s[0] \n" + +// Read 8 YUY2 +#define READYUY2 \ + MEMACCESS(0) \ + "ld2 {v0.8b, v1.8b}, [%0], #16 \n" \ + "uzp2 v3.8b, v1.8b, v1.8b \n" \ + "uzp1 v1.8b, v1.8b, v1.8b \n" \ + "ins v1.s[1], v3.s[0] \n" + +// Read 8 UYVY +#define READUYVY \ + MEMACCESS(0) \ + "ld2 {v2.8b, v3.8b}, [%0], #16 \n" \ + "orr v0.8b, v3.8b, v3.8b \n" \ + "uzp1 v1.8b, v2.8b, v2.8b \n" \ + "uzp2 v3.8b, v2.8b, v2.8b \n" \ + "ins v1.s[1], v3.s[0] \n" + +#define YUV422TORGB_SETUP_REG \ + "ld1r {v24.8h}, [%[kUVBiasBGR]], #2 \n" \ + "ld1r {v25.8h}, [%[kUVBiasBGR]], #2 \n" \ + "ld1r {v26.8h}, [%[kUVBiasBGR]] \n" \ + "ld1r {v31.4s}, [%[kYToRgb]] \n" \ + "movi v27.8h, #128 \n" \ + "movi v28.8h, #102 \n" \ + "movi v29.8h, #25 \n" \ + "movi v30.8h, #52 \n" + +#define YUV422TORGB(vR, vG, vB) \ + "uxtl v0.8h, v0.8b \n" /* Extract Y */ \ + "shll v2.8h, v1.8b, #8 \n" /* Replicate UV */ \ + "ushll2 v3.4s, v0.8h, #0 \n" /* Y */ \ + "ushll v0.4s, v0.4h, #0 \n" \ + "mul v3.4s, v3.4s, v31.4s \n" \ + "mul v0.4s, v0.4s, v31.4s \n" \ + "sqshrun v0.4h, v0.4s, #16 \n" \ + "sqshrun2 v0.8h, v3.4s, #16 \n" /* Y */ \ + "uaddw v1.8h, v2.8h, v1.8b \n" /* Replicate UV */ \ + "mov v2.d[0], v1.d[1] \n" /* Extract V */ \ + "uxtl v2.8h, v2.8b \n" \ + "uxtl v1.8h, v1.8b \n" /* Extract U */ \ + "mul v3.8h, v1.8h, v27.8h \n" \ + "mul v5.8h, v1.8h, v29.8h \n" \ + "mul v6.8h, v2.8h, v30.8h \n" \ + "mul v7.8h, v2.8h, v28.8h \n" \ + "sqadd v6.8h, v6.8h, v5.8h \n" \ + "sqadd " #vB ".8h, v24.8h, v0.8h \n" /* B */ \ + "sqadd " #vG ".8h, v25.8h, v0.8h \n" /* G */ \ + "sqadd " #vR ".8h, v26.8h, v0.8h \n" /* R */ \ + "sqadd " #vB ".8h, " #vB ".8h, v3.8h \n" /* B */ \ + "sqsub " #vG ".8h, " #vG ".8h, v6.8h \n" /* G */ \ + "sqadd " #vR ".8h, " #vR ".8h, v7.8h \n" /* R */ \ + "sqshrun " #vB ".8b, " #vB ".8h, #6 \n" /* B */ \ + "sqshrun " #vG ".8b, " #vG ".8h, #6 \n" /* G */ \ + "sqshrun " #vR ".8b, " #vR ".8h, #6 \n" /* R */ \ + +// YUV to RGB conversion constants. +// Y contribution to R,G,B. Scale and bias. +#define YG 18997 /* round(1.164 * 64 * 256 * 256 / 257) */ +#define YGB 1160 /* 1.164 * 64 * 16 - adjusted for even error distribution */ + +// U and V contributions to R,G,B. +#define UB -128 /* -min(128, round(2.018 * 64)) */ +#define UG 25 /* -round(-0.391 * 64) */ +#define VG 52 /* -round(-0.813 * 64) */ +#define VR -102 /* -round(1.596 * 64) */ + +// Bias values to subtract 16 from Y and 128 from U and V. +#define BB (UB * 128 - YGB) +#define BG (UG * 128 + VG * 128 - YGB) +#define BR (VR * 128 - YGB) + +static vec16 kUVBiasBGR = { BB, BG, BR, 0, 0, 0, 0, 0 }; +static vec32 kYToRgb = { 0x0101 * YG, 0, 0, 0 }; + +#undef YG +#undef YGB +#undef UB +#undef UG +#undef VG +#undef VR +#undef BB +#undef BG +#undef BR + +#define RGBTOUV_SETUP_REG \ + "movi v20.8h, #56, lsl #0 \n" /* UB/VR coefficient (0.875) / 2 */ \ + "movi v21.8h, #37, lsl #0 \n" /* UG coefficient (-0.5781) / 2 */ \ + "movi v22.8h, #19, lsl #0 \n" /* UR coefficient (-0.2969) / 2 */ \ + "movi v23.8h, #9, lsl #0 \n" /* VB coefficient (-0.1406) / 2 */ \ + "movi v24.8h, #47, lsl #0 \n" /* VG coefficient (-0.7344) / 2 */ \ + "movi v25.16b, #0x80 \n" /* 128.5 (0x8080 in 16-bit) */ + + +#ifdef HAS_I444TOARGBROW_NEON +void I444ToARGBRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READYUV444 + YUV422TORGB(v22, v21, v20) + "subs %w4, %w4, #8 \n" + "movi v23.8b, #255 \n" /* A */ + MEMACCESS(3) + "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%3], #32 \n" + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_argb), // %3 + "+r"(width) // %4 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_I444TOARGBROW_NEON + +#ifdef HAS_I422TOARGBROW_NEON +void I422ToARGBRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READYUV422 + YUV422TORGB(v22, v21, v20) + "subs %w4, %w4, #8 \n" + "movi v23.8b, #255 \n" /* A */ + MEMACCESS(3) + "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%3], #32 \n" + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_argb), // %3 + "+r"(width) // %4 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_I422TOARGBROW_NEON + +#ifdef HAS_I411TOARGBROW_NEON +void I411ToARGBRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READYUV411 + YUV422TORGB(v22, v21, v20) + "subs %w4, %w4, #8 \n" + "movi v23.8b, #255 \n" /* A */ + MEMACCESS(3) + "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%3], #32 \n" + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_argb), // %3 + "+r"(width) // %4 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_I411TOARGBROW_NEON + +#ifdef HAS_I422TOBGRAROW_NEON +void I422ToBGRARow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_bgra, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READYUV422 + YUV422TORGB(v21, v22, v23) + "subs %w4, %w4, #8 \n" + "movi v20.8b, #255 \n" /* A */ + MEMACCESS(3) + "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%3], #32 \n" + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_bgra), // %3 + "+r"(width) // %4 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_I422TOBGRAROW_NEON + +#ifdef HAS_I422TOABGRROW_NEON +void I422ToABGRRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_abgr, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READYUV422 + YUV422TORGB(v20, v21, v22) + "subs %w4, %w4, #8 \n" + "movi v23.8b, #255 \n" /* A */ + MEMACCESS(3) + "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%3], #32 \n" + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_abgr), // %3 + "+r"(width) // %4 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_I422TOABGRROW_NEON + +#ifdef HAS_I422TORGBAROW_NEON +void I422ToRGBARow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgba, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READYUV422 + YUV422TORGB(v23, v22, v21) + "subs %w4, %w4, #8 \n" + "movi v20.8b, #255 \n" /* A */ + MEMACCESS(3) + "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%3], #32 \n" + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_rgba), // %3 + "+r"(width) // %4 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_I422TORGBAROW_NEON + +#ifdef HAS_I422TORGB24ROW_NEON +void I422ToRGB24Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb24, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READYUV422 + YUV422TORGB(v22, v21, v20) + "subs %w4, %w4, #8 \n" + MEMACCESS(3) + "st3 {v20.8b,v21.8b,v22.8b}, [%3], #24 \n" + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_rgb24), // %3 + "+r"(width) // %4 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_I422TORGB24ROW_NEON + +#ifdef HAS_I422TORAWROW_NEON +void I422ToRAWRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_raw, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READYUV422 + YUV422TORGB(v20, v21, v22) + "subs %w4, %w4, #8 \n" + MEMACCESS(3) + "st3 {v20.8b,v21.8b,v22.8b}, [%3], #24 \n" + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_raw), // %3 + "+r"(width) // %4 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_I422TORAWROW_NEON + +#define ARGBTORGB565 \ + "shll v0.8h, v22.8b, #8 \n" /* R */ \ + "shll v20.8h, v20.8b, #8 \n" /* B */ \ + "shll v21.8h, v21.8b, #8 \n" /* G */ \ + "sri v0.8h, v21.8h, #5 \n" /* RG */ \ + "sri v0.8h, v20.8h, #11 \n" /* RGB */ + +#ifdef HAS_I422TORGB565ROW_NEON +void I422ToRGB565Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_rgb565, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READYUV422 + YUV422TORGB(v22, v21, v20) + "subs %w4, %w4, #8 \n" + ARGBTORGB565 + MEMACCESS(3) + "st1 {v0.8h}, [%3], #16 \n" // store 8 pixels RGB565. + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_rgb565), // %3 + "+r"(width) // %4 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_I422TORGB565ROW_NEON + +#define ARGBTOARGB1555 \ + "shll v0.8h, v23.8b, #8 \n" /* A */ \ + "shll v22.8h, v22.8b, #8 \n" /* R */ \ + "shll v20.8h, v20.8b, #8 \n" /* B */ \ + "shll v21.8h, v21.8b, #8 \n" /* G */ \ + "sri v0.8h, v22.8h, #1 \n" /* AR */ \ + "sri v0.8h, v21.8h, #6 \n" /* ARG */ \ + "sri v0.8h, v20.8h, #11 \n" /* ARGB */ + +#ifdef HAS_I422TOARGB1555ROW_NEON +void I422ToARGB1555Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb1555, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READYUV422 + YUV422TORGB(v22, v21, v20) + "subs %w4, %w4, #8 \n" + "movi v23.8b, #255 \n" + ARGBTOARGB1555 + MEMACCESS(3) + "st1 {v0.8h}, [%3], #16 \n" // store 8 pixels RGB565. + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_argb1555), // %3 + "+r"(width) // %4 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_I422TOARGB1555ROW_NEON + +#define ARGBTOARGB4444 \ + /* Input v20.8b<=B, v21.8b<=G, v22.8b<=R, v23.8b<=A, v4.8b<=0x0f */ \ + "ushr v20.8b, v20.8b, #4 \n" /* B */ \ + "bic v21.8b, v21.8b, v4.8b \n" /* G */ \ + "ushr v22.8b, v22.8b, #4 \n" /* R */ \ + "bic v23.8b, v23.8b, v4.8b \n" /* A */ \ + "orr v0.8b, v20.8b, v21.8b \n" /* BG */ \ + "orr v1.8b, v22.8b, v23.8b \n" /* RA */ \ + "zip1 v0.16b, v0.16b, v1.16b \n" /* BGRA */ + +#ifdef HAS_I422TOARGB4444ROW_NEON +void I422ToARGB4444Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb4444, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "movi v4.16b, #0x0f \n" // bits to clear with vbic. + "1: \n" + READYUV422 + YUV422TORGB(v22, v21, v20) + "subs %w4, %w4, #8 \n" + "movi v23.8b, #255 \n" + ARGBTOARGB4444 + MEMACCESS(3) + "st1 {v0.8h}, [%3], #16 \n" // store 8 pixels ARGB4444. + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_argb4444), // %3 + "+r"(width) // %4 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_I422TOARGB4444ROW_NEON + +#ifdef HAS_I400TOARGBROW_NEON +void I400ToARGBRow_NEON(const uint8* src_y, + uint8* dst_argb, + int width) { + int64 width64 = (int64)(width); + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READYUV400 + YUV422TORGB(v22, v21, v20) + "subs %w2, %w2, #8 \n" + "movi v23.8b, #255 \n" + MEMACCESS(1) + "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%1], #32 \n" + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(dst_argb), // %1 + "+r"(width64) // %2 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_I400TOARGBROW_NEON + +#ifdef HAS_J400TOARGBROW_NEON +void J400ToARGBRow_NEON(const uint8* src_y, + uint8* dst_argb, + int width) { + asm volatile ( + "movi v23.8b, #255 \n" + "1: \n" + MEMACCESS(0) + "ld1 {v20.8b}, [%0], #8 \n" + "orr v21.8b, v20.8b, v20.8b \n" + "orr v22.8b, v20.8b, v20.8b \n" + "subs %w2, %w2, #8 \n" + MEMACCESS(1) + "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%1], #32 \n" + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : + : "cc", "memory", "v20", "v21", "v22", "v23" + ); +} +#endif // HAS_J400TOARGBROW_NEON + +#ifdef HAS_NV12TOARGBROW_NEON +void NV12ToARGBRow_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READNV12 + YUV422TORGB(v22, v21, v20) + "subs %w3, %w3, #8 \n" + "movi v23.8b, #255 \n" + MEMACCESS(2) + "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%2], #32 \n" + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_uv), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_NV12TOARGBROW_NEON + +#ifdef HAS_NV21TOARGBROW_NEON +void NV21ToARGBRow_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_argb, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READNV21 + YUV422TORGB(v22, v21, v20) + "subs %w3, %w3, #8 \n" + "movi v23.8b, #255 \n" + MEMACCESS(2) + "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%2], #32 \n" + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_uv), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_NV21TOARGBROW_NEON + +#ifdef HAS_NV12TORGB565ROW_NEON +void NV12ToRGB565Row_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_rgb565, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READNV12 + YUV422TORGB(v22, v21, v20) + "subs %w3, %w3, #8 \n" + ARGBTORGB565 + MEMACCESS(2) + "st1 {v0.8h}, [%2], 16 \n" // store 8 pixels RGB565. + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_uv), // %1 + "+r"(dst_rgb565), // %2 + "+r"(width) // %3 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_NV12TORGB565ROW_NEON + +#ifdef HAS_NV21TORGB565ROW_NEON +void NV21ToRGB565Row_NEON(const uint8* src_y, + const uint8* src_uv, + uint8* dst_rgb565, + int width) { + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READNV21 + YUV422TORGB(v22, v21, v20) + "subs %w3, %w3, #8 \n" + ARGBTORGB565 + MEMACCESS(2) + "st1 {v0.8h}, [%2], 16 \n" // store 8 pixels RGB565. + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_uv), // %1 + "+r"(dst_rgb565), // %2 + "+r"(width) // %3 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_NV21TORGB565ROW_NEON + +#ifdef HAS_YUY2TOARGBROW_NEON +void YUY2ToARGBRow_NEON(const uint8* src_yuy2, + uint8* dst_argb, + int width) { + int64 width64 = (int64)(width); + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READYUY2 + YUV422TORGB(v22, v21, v20) + "subs %w2, %w2, #8 \n" + "movi v23.8b, #255 \n" + MEMACCESS(1) + "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%1], #32 \n" + "b.gt 1b \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_argb), // %1 + "+r"(width64) // %2 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_YUY2TOARGBROW_NEON + +#ifdef HAS_UYVYTOARGBROW_NEON +void UYVYToARGBRow_NEON(const uint8* src_uyvy, + uint8* dst_argb, + int width) { + int64 width64 = (int64)(width); + asm volatile ( + YUV422TORGB_SETUP_REG + "1: \n" + READUYVY + YUV422TORGB(v22, v21, v20) + "subs %w2, %w2, #8 \n" + "movi v23.8b, #255 \n" + MEMACCESS(1) + "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%1], 32 \n" + "b.gt 1b \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_argb), // %1 + "+r"(width64) // %2 + : [kUVBiasBGR]"r"(&kUVBiasBGR), + [kYToRgb]"r"(&kYToRgb) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30" + ); +} +#endif // HAS_UYVYTOARGBROW_NEON + +// Reads 16 pairs of UV and write even values to dst_u and odd to dst_v. +#ifdef HAS_SPLITUVROW_NEON +void SplitUVRow_NEON(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int width) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pairs of UV + "subs %w3, %w3, #16 \n" // 16 processed per loop + MEMACCESS(1) + "st1 {v0.16b}, [%1], #16 \n" // store U + MEMACCESS(2) + "st1 {v1.16b}, [%2], #16 \n" // store V + "b.gt 1b \n" + : "+r"(src_uv), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width) // %3 // Output registers + : // Input registers + : "cc", "memory", "v0", "v1" // Clobber List + ); +} +#endif // HAS_SPLITUVROW_NEON + +// Reads 16 U's and V's and writes out 16 pairs of UV. +#ifdef HAS_MERGEUVROW_NEON +void MergeUVRow_NEON(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load U + MEMACCESS(1) + "ld1 {v1.16b}, [%1], #16 \n" // load V + "subs %w3, %w3, #16 \n" // 16 processed per loop + MEMACCESS(2) + "st2 {v0.16b,v1.16b}, [%2], #32 \n" // store 16 pairs of UV + "b.gt 1b \n" + : + "+r"(src_u), // %0 + "+r"(src_v), // %1 + "+r"(dst_uv), // %2 + "+r"(width) // %3 // Output registers + : // Input registers + : "cc", "memory", "v0", "v1" // Clobber List + ); +} +#endif // HAS_MERGEUVROW_NEON + +// Copy multiple of 32. vld4.8 allow unaligned and is fastest on a15. +#ifdef HAS_COPYROW_NEON +void CopyRow_NEON(const uint8* src, uint8* dst, int count) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld1 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 32 + "subs %w2, %w2, #32 \n" // 32 processed per loop + MEMACCESS(1) + "st1 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 32 + "b.gt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(count) // %2 // Output registers + : // Input registers + : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List + ); +} +#endif // HAS_COPYROW_NEON + +// SetRow writes 'count' bytes using an 8 bit value repeated. +void SetRow_NEON(uint8* dst, uint8 v8, int count) { + asm volatile ( + "dup v0.16b, %w2 \n" // duplicate 16 bytes + "1: \n" + "subs %w1, %w1, #16 \n" // 16 bytes per loop + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" // store + "b.gt 1b \n" + : "+r"(dst), // %0 + "+r"(count) // %1 + : "r"(v8) // %2 + : "cc", "memory", "v0" + ); +} + +void ARGBSetRow_NEON(uint8* dst, uint32 v32, int count) { + asm volatile ( + "dup v0.4s, %w2 \n" // duplicate 4 ints + "1: \n" + "subs %w1, %w1, #4 \n" // 4 ints per loop + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" // store + "b.gt 1b \n" + : "+r"(dst), // %0 + "+r"(count) // %1 + : "r"(v32) // %2 + : "cc", "memory", "v0" + ); +} + +#ifdef HAS_MIRRORROW_NEON +void MirrorRow_NEON(const uint8* src, uint8* dst, int width) { + int64 width64 = (int64) width; + asm volatile ( + // Start at end of source row. + "add %0, %0, %2 \n" + "sub %0, %0, #16 \n" + + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], %3 \n" // src -= 16 + "subs %2, %2, #16 \n" // 16 pixels per loop. + "rev64 v0.16b, v0.16b \n" + MEMACCESS(1) + "st1 {v0.D}[1], [%1], #8 \n" // dst += 16 + MEMACCESS(1) + "st1 {v0.D}[0], [%1], #8 \n" + "b.gt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width64) // %2 + : "r"((ptrdiff_t)-16) // %3 + : "cc", "memory", "v0" + ); +} +#endif // HAS_MIRRORROW_NEON + +#ifdef HAS_MIRRORUVROW_NEON +void MirrorUVRow_NEON(const uint8* src_uv, uint8* dst_u, uint8* dst_v, + int width) { + int64 width64 = (int64) width; + asm volatile ( + // Start at end of source row. + "add %0, %0, %3, lsl #1 \n" + "sub %0, %0, #16 \n" + + "1: \n" + MEMACCESS(0) + "ld2 {v0.8b, v1.8b}, [%0], %4 \n" // src -= 16 + "subs %3, %3, #8 \n" // 8 pixels per loop. + "rev64 v0.8b, v0.8b \n" + "rev64 v1.8b, v1.8b \n" + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // dst += 8 + MEMACCESS(2) + "st1 {v1.8b}, [%2], #8 \n" + "b.gt 1b \n" + : "+r"(src_uv), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width64) // %3 + : "r"((ptrdiff_t)-16) // %4 + : "cc", "memory", "v0", "v1" + ); +} +#endif // HAS_MIRRORUVROW_NEON + +#ifdef HAS_ARGBMIRRORROW_NEON +void ARGBMirrorRow_NEON(const uint8* src, uint8* dst, int width) { + int64 width64 = (int64) width; + asm volatile ( + // Start at end of source row. + "add %0, %0, %2, lsl #2 \n" + "sub %0, %0, #16 \n" + + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], %3 \n" // src -= 16 + "subs %2, %2, #4 \n" // 4 pixels per loop. + "rev64 v0.4s, v0.4s \n" + MEMACCESS(1) + "st1 {v0.D}[1], [%1], #8 \n" // dst += 16 + MEMACCESS(1) + "st1 {v0.D}[0], [%1], #8 \n" + "b.gt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width64) // %2 + : "r"((ptrdiff_t)-16) // %3 + : "cc", "memory", "v0" + ); +} +#endif // HAS_ARGBMIRRORROW_NEON + +#ifdef HAS_RGB24TOARGBROW_NEON +void RGB24ToARGBRow_NEON(const uint8* src_rgb24, uint8* dst_argb, int pix) { + asm volatile ( + "movi v4.8b, #255 \n" // Alpha + "1: \n" + MEMACCESS(0) + "ld3 {v1.8b,v2.8b,v3.8b}, [%0], #24 \n" // load 8 pixels of RGB24. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + MEMACCESS(1) + "st4 {v1.8b,v2.8b,v3.8b,v4.8b}, [%1], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + : "+r"(src_rgb24), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v1", "v2", "v3", "v4" // Clobber List + ); +} +#endif // HAS_RGB24TOARGBROW_NEON + +#ifdef HAS_RAWTOARGBROW_NEON +void RAWToARGBRow_NEON(const uint8* src_raw, uint8* dst_argb, int pix) { + asm volatile ( + "movi v5.8b, #255 \n" // Alpha + "1: \n" + MEMACCESS(0) + "ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // read r g b + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "orr v3.8b, v1.8b, v1.8b \n" // move g + "orr v4.8b, v0.8b, v0.8b \n" // move r + MEMACCESS(1) + "st4 {v2.8b,v3.8b,v4.8b,v5.8b}, [%1], #32 \n" // store b g r a + "b.gt 1b \n" + : "+r"(src_raw), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5" // Clobber List + ); +} +#endif // HAS_RAWTOARGBROW_NEON + +#define RGB565TOARGB \ + "shrn v6.8b, v0.8h, #5 \n" /* G xxGGGGGG */ \ + "shl v6.8b, v6.8b, #2 \n" /* G GGGGGG00 upper 6 */ \ + "ushr v4.8b, v6.8b, #6 \n" /* G 000000GG lower 2 */ \ + "orr v1.8b, v4.8b, v6.8b \n" /* G */ \ + "xtn v2.8b, v0.8h \n" /* B xxxBBBBB */ \ + "ushr v0.8h, v0.8h, #11 \n" /* R 000RRRRR */ \ + "xtn2 v2.16b,v0.8h \n" /* R in upper part */ \ + "shl v2.16b, v2.16b, #3 \n" /* R,B BBBBB000 upper 5 */ \ + "ushr v0.16b, v2.16b, #5 \n" /* R,B 00000BBB lower 3 */ \ + "orr v0.16b, v0.16b, v2.16b \n" /* R,B */ \ + "dup v2.2D, v0.D[1] \n" /* R */ + +#ifdef HAS_RGB565TOARGBROW_NEON +void RGB565ToARGBRow_NEON(const uint8* src_rgb565, uint8* dst_argb, int pix) { + asm volatile ( + "movi v3.8b, #255 \n" // Alpha + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load 8 RGB565 pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + RGB565TOARGB + MEMACCESS(1) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + : "+r"(src_rgb565), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v6" // Clobber List + ); +} +#endif // HAS_RGB565TOARGBROW_NEON + +#define ARGB1555TOARGB \ + "ushr v2.8h, v0.8h, #10 \n" /* R xxxRRRRR */ \ + "shl v2.8h, v2.8h, #3 \n" /* R RRRRR000 upper 5 */ \ + "xtn v3.8b, v2.8h \n" /* RRRRR000 AAAAAAAA */ \ + \ + "sshr v2.8h, v0.8h, #15 \n" /* A AAAAAAAA */ \ + "xtn2 v3.16b, v2.8h \n" \ + \ + "xtn v2.8b, v0.8h \n" /* B xxxBBBBB */ \ + "shrn2 v2.16b,v0.8h, #5 \n" /* G xxxGGGGG */ \ + \ + "ushr v1.16b, v3.16b, #5 \n" /* R,A 00000RRR lower 3 */ \ + "shl v0.16b, v2.16b, #3 \n" /* B,G BBBBB000 upper 5 */ \ + "ushr v2.16b, v0.16b, #5 \n" /* B,G 00000BBB lower 3 */ \ + \ + "orr v0.16b, v0.16b, v2.16b \n" /* B,G */ \ + "orr v2.16b, v1.16b, v3.16b \n" /* R,A */ \ + "dup v1.2D, v0.D[1] \n" \ + "dup v3.2D, v2.D[1] \n" + +// RGB555TOARGB is same as ARGB1555TOARGB but ignores alpha. +#define RGB555TOARGB \ + "ushr v2.8h, v0.8h, #10 \n" /* R xxxRRRRR */ \ + "shl v2.8h, v2.8h, #3 \n" /* R RRRRR000 upper 5 */ \ + "xtn v3.8b, v2.8h \n" /* RRRRR000 */ \ + \ + "xtn v2.8b, v0.8h \n" /* B xxxBBBBB */ \ + "shrn2 v2.16b,v0.8h, #5 \n" /* G xxxGGGGG */ \ + \ + "ushr v1.16b, v3.16b, #5 \n" /* R 00000RRR lower 3 */ \ + "shl v0.16b, v2.16b, #3 \n" /* B,G BBBBB000 upper 5 */ \ + "ushr v2.16b, v0.16b, #5 \n" /* B,G 00000BBB lower 3 */ \ + \ + "orr v0.16b, v0.16b, v2.16b \n" /* B,G */ \ + "orr v2.16b, v1.16b, v3.16b \n" /* R */ \ + "dup v1.2D, v0.D[1] \n" /* G */ \ + +#ifdef HAS_ARGB1555TOARGBROW_NEON +void ARGB1555ToARGBRow_NEON(const uint8* src_argb1555, uint8* dst_argb, + int pix) { + asm volatile ( + "movi v3.8b, #255 \n" // Alpha + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB1555 pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + ARGB1555TOARGB + MEMACCESS(1) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + : "+r"(src_argb1555), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List + ); +} +#endif // HAS_ARGB1555TOARGBROW_NEON + +#define ARGB4444TOARGB \ + "shrn v1.8b, v0.8h, #8 \n" /* v1(l) AR */ \ + "xtn2 v1.16b, v0.8h \n" /* v1(h) GB */ \ + "shl v2.16b, v1.16b, #4 \n" /* B,R BBBB0000 */ \ + "ushr v3.16b, v1.16b, #4 \n" /* G,A 0000GGGG */ \ + "ushr v0.16b, v2.16b, #4 \n" /* B,R 0000BBBB */ \ + "shl v1.16b, v3.16b, #4 \n" /* G,A GGGG0000 */ \ + "orr v2.16b, v0.16b, v2.16b \n" /* B,R BBBBBBBB */ \ + "orr v3.16b, v1.16b, v3.16b \n" /* G,A GGGGGGGG */ \ + "dup v0.2D, v2.D[1] \n" \ + "dup v1.2D, v3.D[1] \n" + +#ifdef HAS_ARGB4444TOARGBROW_NEON +void ARGB4444ToARGBRow_NEON(const uint8* src_argb4444, uint8* dst_argb, + int pix) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB4444 pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + ARGB4444TOARGB + MEMACCESS(1) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + : "+r"(src_argb4444), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4" // Clobber List + ); +} +#endif // HAS_ARGB4444TOARGBROW_NEON + +#ifdef HAS_ARGBTORGB24ROW_NEON +void ARGBToRGB24Row_NEON(const uint8* src_argb, uint8* dst_rgb24, int pix) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld4 {v1.8b,v2.8b,v3.8b,v4.8b}, [%0], #32 \n" // load 8 ARGB pixels + "subs %w2, %w2, #8 \n" // 8 processed per loop. + MEMACCESS(1) + "st3 {v1.8b,v2.8b,v3.8b}, [%1], #24 \n" // store 8 pixels of RGB24. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_rgb24), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v1", "v2", "v3", "v4" // Clobber List + ); +} +#endif // HAS_ARGBTORGB24ROW_NEON + +#ifdef HAS_ARGBTORAWROW_NEON +void ARGBToRAWRow_NEON(const uint8* src_argb, uint8* dst_raw, int pix) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld4 {v1.8b,v2.8b,v3.8b,v4.8b}, [%0], #32 \n" // load b g r a + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "orr v4.8b, v2.8b, v2.8b \n" // mov g + "orr v5.8b, v1.8b, v1.8b \n" // mov b + MEMACCESS(1) + "st3 {v3.8b,v4.8b,v5.8b}, [%1], #24 \n" // store r g b + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_raw), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v1", "v2", "v3", "v4", "v5" // Clobber List + ); +} +#endif // HAS_ARGBTORAWROW_NEON + +#ifdef HAS_YUY2TOYROW_NEON +void YUY2ToYRow_NEON(const uint8* src_yuy2, uint8* dst_y, int pix) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pixels of YUY2. + "subs %w2, %w2, #16 \n" // 16 processed per loop. + MEMACCESS(1) + "st1 {v0.16b}, [%1], #16 \n" // store 16 pixels of Y. + "b.gt 1b \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1" // Clobber List + ); +} +#endif // HAS_YUY2TOYROW_NEON + +#ifdef HAS_UYVYTOYROW_NEON +void UYVYToYRow_NEON(const uint8* src_uyvy, uint8* dst_y, int pix) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pixels of UYVY. + "subs %w2, %w2, #16 \n" // 16 processed per loop. + MEMACCESS(1) + "st1 {v1.16b}, [%1], #16 \n" // store 16 pixels of Y. + "b.gt 1b \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1" // Clobber List + ); +} +#endif // HAS_UYVYTOYROW_NEON + +#ifdef HAS_YUY2TOUV422ROW_NEON +void YUY2ToUV422Row_NEON(const uint8* src_yuy2, uint8* dst_u, uint8* dst_v, + int pix) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 YUY2 pixels + "subs %w3, %w3, #16 \n" // 16 pixels = 8 UVs. + MEMACCESS(1) + "st1 {v1.8b}, [%1], #8 \n" // store 8 U. + MEMACCESS(2) + "st1 {v3.8b}, [%2], #8 \n" // store 8 V. + "b.gt 1b \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List + ); +} +#endif // HAS_YUY2TOUV422ROW_NEON + +#ifdef HAS_UYVYTOUV422ROW_NEON +void UYVYToUV422Row_NEON(const uint8* src_uyvy, uint8* dst_u, uint8* dst_v, + int pix) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 UYVY pixels + "subs %w3, %w3, #16 \n" // 16 pixels = 8 UVs. + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 U. + MEMACCESS(2) + "st1 {v2.8b}, [%2], #8 \n" // store 8 V. + "b.gt 1b \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List + ); +} +#endif // HAS_UYVYTOUV422ROW_NEON + +#ifdef HAS_YUY2TOUVROW_NEON +void YUY2ToUVRow_NEON(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix) { + const uint8* src_yuy2b = src_yuy2 + stride_yuy2; + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 pixels + "subs %w4, %w4, #16 \n" // 16 pixels = 8 UVs. + MEMACCESS(1) + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load next row + "urhadd v1.8b, v1.8b, v5.8b \n" // average rows of U + "urhadd v3.8b, v3.8b, v7.8b \n" // average rows of V + MEMACCESS(2) + "st1 {v1.8b}, [%2], #8 \n" // store 8 U. + MEMACCESS(3) + "st1 {v3.8b}, [%3], #8 \n" // store 8 V. + "b.gt 1b \n" + : "+r"(src_yuy2), // %0 + "+r"(src_yuy2b), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", + "v5", "v6", "v7" // Clobber List + ); +} +#endif // HAS_YUY2TOUVROW_NEON + +#ifdef HAS_UYVYTOUVROW_NEON +void UYVYToUVRow_NEON(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix) { + const uint8* src_uyvyb = src_uyvy + stride_uyvy; + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 pixels + "subs %w4, %w4, #16 \n" // 16 pixels = 8 UVs. + MEMACCESS(1) + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load next row + "urhadd v0.8b, v0.8b, v4.8b \n" // average rows of U + "urhadd v2.8b, v2.8b, v6.8b \n" // average rows of V + MEMACCESS(2) + "st1 {v0.8b}, [%2], #8 \n" // store 8 U. + MEMACCESS(3) + "st1 {v2.8b}, [%3], #8 \n" // store 8 V. + "b.gt 1b \n" + : "+r"(src_uyvy), // %0 + "+r"(src_uyvyb), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", + "v5", "v6", "v7" // Clobber List + ); +} +#endif // HAS_UYVYTOUVROW_NEON + +// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA. +#ifdef HAS_ARGBSHUFFLEROW_NEON +void ARGBShuffleRow_NEON(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix) { + asm volatile ( + MEMACCESS(3) + "ld1 {v2.16b}, [%3] \n" // shuffler + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load 4 pixels. + "subs %w2, %w2, #4 \n" // 4 processed per loop + "tbl v1.16b, {v0.16b}, v2.16b \n" // look up 4 pixels + MEMACCESS(1) + "st1 {v1.16b}, [%1], #16 \n" // store 4. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(pix) // %2 + : "r"(shuffler) // %3 + : "cc", "memory", "v0", "v1", "v2" // Clobber List + ); +} +#endif // HAS_ARGBSHUFFLEROW_NEON + +#ifdef HAS_I422TOYUY2ROW_NEON +void I422ToYUY2Row_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_yuy2, int width) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld2 {v0.8b, v1.8b}, [%0], #16 \n" // load 16 Ys + "orr v2.8b, v1.8b, v1.8b \n" + MEMACCESS(1) + "ld1 {v1.8b}, [%1], #8 \n" // load 8 Us + MEMACCESS(2) + "ld1 {v3.8b}, [%2], #8 \n" // load 8 Vs + "subs %w4, %w4, #16 \n" // 16 pixels + MEMACCESS(3) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%3], #32 \n" // Store 16 pixels. + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_yuy2), // %3 + "+r"(width) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3" + ); +} +#endif // HAS_I422TOYUY2ROW_NEON + +#ifdef HAS_I422TOUYVYROW_NEON +void I422ToUYVYRow_NEON(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_uyvy, int width) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld2 {v1.8b,v2.8b}, [%0], #16 \n" // load 16 Ys + "orr v3.8b, v2.8b, v2.8b \n" + MEMACCESS(1) + "ld1 {v0.8b}, [%1], #8 \n" // load 8 Us + MEMACCESS(2) + "ld1 {v2.8b}, [%2], #8 \n" // load 8 Vs + "subs %w4, %w4, #16 \n" // 16 pixels + MEMACCESS(3) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%3], #32 \n" // Store 16 pixels. + "b.gt 1b \n" + : "+r"(src_y), // %0 + "+r"(src_u), // %1 + "+r"(src_v), // %2 + "+r"(dst_uyvy), // %3 + "+r"(width) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3" + ); +} +#endif // HAS_I422TOUYVYROW_NEON + +#ifdef HAS_ARGBTORGB565ROW_NEON +void ARGBToRGB565Row_NEON(const uint8* src_argb, uint8* dst_rgb565, int pix) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%0], #32 \n" // load 8 pixels + "subs %w2, %w2, #8 \n" // 8 processed per loop. + ARGBTORGB565 + MEMACCESS(1) + "st1 {v0.16b}, [%1], #16 \n" // store 8 pixels RGB565. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_rgb565), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v20", "v21", "v22", "v23" + ); +} +#endif // HAS_ARGBTORGB565ROW_NEON + +#ifdef HAS_ARGBTORGB565DITHERROW_NEON +void ARGBToRGB565DitherRow_NEON(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int width) { + asm volatile ( + "dup v1.4s, %w2 \n" // dither4 + "1: \n" + MEMACCESS(1) + "ld4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%1], #32 \n" // load 8 pixels + "subs %w3, %w3, #8 \n" // 8 processed per loop. + "uqadd v20.8b, v20.8b, v1.8b \n" + "uqadd v21.8b, v21.8b, v1.8b \n" + "uqadd v22.8b, v22.8b, v1.8b \n" + ARGBTORGB565 + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" // store 8 pixels RGB565. + "b.gt 1b \n" + : "+r"(dst_rgb) // %0 + : "r"(src_argb), // %1 + "r"(dither4), // %2 + "r"(width) // %3 + : "cc", "memory", "v0", "v1", "v20", "v21", "v22", "v23" + ); +} +#endif // HAS_ARGBTORGB565ROW_NEON + +#ifdef HAS_ARGBTOARGB1555ROW_NEON +void ARGBToARGB1555Row_NEON(const uint8* src_argb, uint8* dst_argb1555, + int pix) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%0], #32 \n" // load 8 pixels + "subs %w2, %w2, #8 \n" // 8 processed per loop. + ARGBTOARGB1555 + MEMACCESS(1) + "st1 {v0.16b}, [%1], #16 \n" // store 8 pixels ARGB1555. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb1555), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v20", "v21", "v22", "v23" + ); +} +#endif // HAS_ARGBTOARGB1555ROW_NEON + +#ifdef HAS_ARGBTOARGB4444ROW_NEON +void ARGBToARGB4444Row_NEON(const uint8* src_argb, uint8* dst_argb4444, + int pix) { + asm volatile ( + "movi v4.16b, #0x0f \n" // bits to clear with vbic. + "1: \n" + MEMACCESS(0) + "ld4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%0], #32 \n" // load 8 pixels + "subs %w2, %w2, #8 \n" // 8 processed per loop. + ARGBTOARGB4444 + MEMACCESS(1) + "st1 {v0.16b}, [%1], #16 \n" // store 8 pixels ARGB4444. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb4444), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v4", "v20", "v21", "v22", "v23" + ); +} +#endif // HAS_ARGBTOARGB4444ROW_NEON + +#ifdef HAS_ARGBTOYROW_NEON +void ARGBToYRow_NEON(const uint8* src_argb, uint8* dst_y, int pix) { + asm volatile ( + "movi v4.8b, #13 \n" // B * 0.1016 coefficient + "movi v5.8b, #65 \n" // G * 0.5078 coefficient + "movi v6.8b, #33 \n" // R * 0.2578 coefficient + "movi v7.8b, #16 \n" // Add 16 constant + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "umull v3.8h, v0.8b, v4.8b \n" // B + "umlal v3.8h, v1.8b, v5.8b \n" // G + "umlal v3.8h, v2.8b, v6.8b \n" // R + "sqrshrun v0.8b, v3.8h, #7 \n" // 16 bit to 8 bit Y + "uqadd v0.8b, v0.8b, v7.8b \n" + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + ); +} +#endif // HAS_ARGBTOYROW_NEON + +#ifdef HAS_ARGBTOYJROW_NEON +void ARGBToYJRow_NEON(const uint8* src_argb, uint8* dst_y, int pix) { + asm volatile ( + "movi v4.8b, #15 \n" // B * 0.11400 coefficient + "movi v5.8b, #75 \n" // G * 0.58700 coefficient + "movi v6.8b, #38 \n" // R * 0.29900 coefficient + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "umull v3.8h, v0.8b, v4.8b \n" // B + "umlal v3.8h, v1.8b, v5.8b \n" // G + "umlal v3.8h, v2.8b, v6.8b \n" // R + "sqrshrun v0.8b, v3.8h, #7 \n" // 15 bit to 8 bit Y + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6" + ); +} +#endif // HAS_ARGBTOYJROW_NEON + +// 8x1 pixels. +#ifdef HAS_ARGBTOUV444ROW_NEON +void ARGBToUV444Row_NEON(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix) { + asm volatile ( + "movi v24.8b, #112 \n" // UB / VR 0.875 coefficient + "movi v25.8b, #74 \n" // UG -0.5781 coefficient + "movi v26.8b, #38 \n" // UR -0.2969 coefficient + "movi v27.8b, #18 \n" // VB -0.1406 coefficient + "movi v28.8b, #94 \n" // VG -0.7344 coefficient + "movi v29.16b,#0x80 \n" // 128.5 + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB pixels. + "subs %w3, %w3, #8 \n" // 8 processed per loop. + "umull v4.8h, v0.8b, v24.8b \n" // B + "umlsl v4.8h, v1.8b, v25.8b \n" // G + "umlsl v4.8h, v2.8b, v26.8b \n" // R + "add v4.8h, v4.8h, v29.8h \n" // +128 -> unsigned + + "umull v3.8h, v2.8b, v24.8b \n" // R + "umlsl v3.8h, v1.8b, v28.8b \n" // G + "umlsl v3.8h, v0.8b, v27.8b \n" // B + "add v3.8h, v3.8h, v29.8h \n" // +128 -> unsigned + + "uqshrn v0.8b, v4.8h, #8 \n" // 16 bit to 8 bit U + "uqshrn v1.8b, v3.8h, #8 \n" // 16 bit to 8 bit V + + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels U. + MEMACCESS(2) + "st1 {v1.8b}, [%2], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", + "v24", "v25", "v26", "v27", "v28", "v29" + ); +} +#endif // HAS_ARGBTOUV444ROW_NEON + +// 16x1 pixels -> 8x1. pix is number of argb pixels. e.g. 16. +#ifdef HAS_ARGBTOUV422ROW_NEON +void ARGBToUV422Row_NEON(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix) { + asm volatile ( + RGBTOUV_SETUP_REG + "1: \n" + MEMACCESS(0) + "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels. + + "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts. + "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts. + + "subs %w3, %w3, #16 \n" // 16 processed per loop. + "mul v3.8h, v0.8h, v20.8h \n" // B + "mls v3.8h, v1.8h, v21.8h \n" // G + "mls v3.8h, v2.8h, v22.8h \n" // R + "add v3.8h, v3.8h, v25.8h \n" // +128 -> unsigned + + "mul v4.8h, v2.8h, v20.8h \n" // R + "mls v4.8h, v1.8h, v24.8h \n" // G + "mls v4.8h, v0.8h, v23.8h \n" // B + "add v4.8h, v4.8h, v25.8h \n" // +128 -> unsigned + + "uqshrn v0.8b, v3.8h, #8 \n" // 16 bit to 8 bit U + "uqshrn v1.8b, v4.8h, #8 \n" // 16 bit to 8 bit V + + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels U. + MEMACCESS(2) + "st1 {v1.8b}, [%2], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v20", "v21", "v22", "v23", "v24", "v25" + ); +} +#endif // HAS_ARGBTOUV422ROW_NEON + +// 32x1 pixels -> 8x1. pix is number of argb pixels. e.g. 32. +#ifdef HAS_ARGBTOUV411ROW_NEON +void ARGBToUV411Row_NEON(const uint8* src_argb, uint8* dst_u, uint8* dst_v, + int pix) { + asm volatile ( + RGBTOUV_SETUP_REG + "1: \n" + MEMACCESS(0) + "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels. + "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts. + "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts. + MEMACCESS(0) + "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%0], #64 \n" // load next 16. + "uaddlp v4.8h, v4.16b \n" // B 16 bytes -> 8 shorts. + "uaddlp v5.8h, v5.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v6.8h, v6.16b \n" // R 16 bytes -> 8 shorts. + + "addp v0.8h, v0.8h, v4.8h \n" // B 16 shorts -> 8 shorts. + "addp v1.8h, v1.8h, v5.8h \n" // G 16 shorts -> 8 shorts. + "addp v2.8h, v2.8h, v6.8h \n" // R 16 shorts -> 8 shorts. + + "urshr v0.8h, v0.8h, #1 \n" // 2x average + "urshr v1.8h, v1.8h, #1 \n" + "urshr v2.8h, v2.8h, #1 \n" + + "subs %w3, %w3, #32 \n" // 32 processed per loop. + "mul v3.8h, v0.8h, v20.8h \n" // B + "mls v3.8h, v1.8h, v21.8h \n" // G + "mls v3.8h, v2.8h, v22.8h \n" // R + "add v3.8h, v3.8h, v25.8h \n" // +128 -> unsigned + "mul v4.8h, v2.8h, v20.8h \n" // R + "mls v4.8h, v1.8h, v24.8h \n" // G + "mls v4.8h, v0.8h, v23.8h \n" // B + "add v4.8h, v4.8h, v25.8h \n" // +128 -> unsigned + "uqshrn v0.8b, v3.8h, #8 \n" // 16 bit to 8 bit U + "uqshrn v1.8b, v4.8h, #8 \n" // 16 bit to 8 bit V + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels U. + MEMACCESS(2) + "st1 {v1.8b}, [%2], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(pix) // %3 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v20", "v21", "v22", "v23", "v24", "v25" + ); +} +#endif // HAS_ARGBTOUV411ROW_NEON + +// 16x2 pixels -> 8x1. pix is number of argb pixels. e.g. 16. +#define RGBTOUV(QB, QG, QR) \ + "mul v3.8h, " #QB ",v20.8h \n" /* B */ \ + "mul v4.8h, " #QR ",v20.8h \n" /* R */ \ + "mls v3.8h, " #QG ",v21.8h \n" /* G */ \ + "mls v4.8h, " #QG ",v24.8h \n" /* G */ \ + "mls v3.8h, " #QR ",v22.8h \n" /* R */ \ + "mls v4.8h, " #QB ",v23.8h \n" /* B */ \ + "add v3.8h, v3.8h, v25.8h \n" /* +128 -> unsigned */ \ + "add v4.8h, v4.8h, v25.8h \n" /* +128 -> unsigned */ \ + "uqshrn v0.8b, v3.8h, #8 \n" /* 16 bit to 8 bit U */ \ + "uqshrn v1.8b, v4.8h, #8 \n" /* 16 bit to 8 bit V */ + +// TODO(fbarchard): Consider vhadd vertical, then vpaddl horizontal, avoid shr. +// TODO(fbarchard): consider ptrdiff_t for all strides. + +#ifdef HAS_ARGBTOUVROW_NEON +void ARGBToUVRow_NEON(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int pix) { + const uint8* src_argb_1 = src_argb + src_stride_argb; + asm volatile ( + RGBTOUV_SETUP_REG + "1: \n" + MEMACCESS(0) + "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels. + "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts. + "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts. + + MEMACCESS(1) + "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load next 16 + "uadalp v0.8h, v4.16b \n" // B 16 bytes -> 8 shorts. + "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts. + "uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts. + + "urshr v0.8h, v0.8h, #1 \n" // 2x average + "urshr v1.8h, v1.8h, #1 \n" + "urshr v2.8h, v2.8h, #1 \n" + + "subs %w4, %w4, #16 \n" // 32 processed per loop. + RGBTOUV(v0.8h, v1.8h, v2.8h) + MEMACCESS(2) + "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U. + MEMACCESS(3) + "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_argb_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v20", "v21", "v22", "v23", "v24", "v25" + ); +} +#endif // HAS_ARGBTOUVROW_NEON + +// TODO(fbarchard): Subsample match C code. +#ifdef HAS_ARGBTOUVJROW_NEON +void ARGBToUVJRow_NEON(const uint8* src_argb, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int pix) { + const uint8* src_argb_1 = src_argb + src_stride_argb; + asm volatile ( + "movi v20.8h, #63, lsl #0 \n" // UB/VR coeff (0.500) / 2 + "movi v21.8h, #42, lsl #0 \n" // UG coeff (-0.33126) / 2 + "movi v22.8h, #21, lsl #0 \n" // UR coeff (-0.16874) / 2 + "movi v23.8h, #10, lsl #0 \n" // VB coeff (-0.08131) / 2 + "movi v24.8h, #53, lsl #0 \n" // VG coeff (-0.41869) / 2 + "movi v25.16b, #0x80 \n" // 128.5 (0x8080 in 16-bit) + "1: \n" + MEMACCESS(0) + "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels. + "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts. + "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load next 16 + "uadalp v0.8h, v4.16b \n" // B 16 bytes -> 8 shorts. + "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts. + "uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts. + + "urshr v0.8h, v0.8h, #1 \n" // 2x average + "urshr v1.8h, v1.8h, #1 \n" + "urshr v2.8h, v2.8h, #1 \n" + + "subs %w4, %w4, #16 \n" // 32 processed per loop. + RGBTOUV(v0.8h, v1.8h, v2.8h) + MEMACCESS(2) + "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U. + MEMACCESS(3) + "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_argb_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v20", "v21", "v22", "v23", "v24", "v25" + ); +} +#endif // HAS_ARGBTOUVJROW_NEON + +#ifdef HAS_BGRATOUVROW_NEON +void BGRAToUVRow_NEON(const uint8* src_bgra, int src_stride_bgra, + uint8* dst_u, uint8* dst_v, int pix) { + const uint8* src_bgra_1 = src_bgra + src_stride_bgra; + asm volatile ( + RGBTOUV_SETUP_REG + "1: \n" + MEMACCESS(0) + "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels. + "uaddlp v0.8h, v3.16b \n" // B 16 bytes -> 8 shorts. + "uaddlp v3.8h, v2.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v2.8h, v1.16b \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load 16 more + "uadalp v0.8h, v7.16b \n" // B 16 bytes -> 8 shorts. + "uadalp v3.8h, v6.16b \n" // G 16 bytes -> 8 shorts. + "uadalp v2.8h, v5.16b \n" // R 16 bytes -> 8 shorts. + + "urshr v0.8h, v0.8h, #1 \n" // 2x average + "urshr v1.8h, v3.8h, #1 \n" + "urshr v2.8h, v2.8h, #1 \n" + + "subs %w4, %w4, #16 \n" // 32 processed per loop. + RGBTOUV(v0.8h, v1.8h, v2.8h) + MEMACCESS(2) + "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U. + MEMACCESS(3) + "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_bgra), // %0 + "+r"(src_bgra_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v20", "v21", "v22", "v23", "v24", "v25" + ); +} +#endif // HAS_BGRATOUVROW_NEON + +#ifdef HAS_ABGRTOUVROW_NEON +void ABGRToUVRow_NEON(const uint8* src_abgr, int src_stride_abgr, + uint8* dst_u, uint8* dst_v, int pix) { + const uint8* src_abgr_1 = src_abgr + src_stride_abgr; + asm volatile ( + RGBTOUV_SETUP_REG + "1: \n" + MEMACCESS(0) + "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels. + "uaddlp v3.8h, v2.16b \n" // B 16 bytes -> 8 shorts. + "uaddlp v2.8h, v1.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v1.8h, v0.16b \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load 16 more. + "uadalp v3.8h, v6.16b \n" // B 16 bytes -> 8 shorts. + "uadalp v2.8h, v5.16b \n" // G 16 bytes -> 8 shorts. + "uadalp v1.8h, v4.16b \n" // R 16 bytes -> 8 shorts. + + "urshr v0.8h, v3.8h, #1 \n" // 2x average + "urshr v2.8h, v2.8h, #1 \n" + "urshr v1.8h, v1.8h, #1 \n" + + "subs %w4, %w4, #16 \n" // 32 processed per loop. + RGBTOUV(v0.8h, v2.8h, v1.8h) + MEMACCESS(2) + "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U. + MEMACCESS(3) + "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_abgr), // %0 + "+r"(src_abgr_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v20", "v21", "v22", "v23", "v24", "v25" + ); +} +#endif // HAS_ABGRTOUVROW_NEON + +#ifdef HAS_RGBATOUVROW_NEON +void RGBAToUVRow_NEON(const uint8* src_rgba, int src_stride_rgba, + uint8* dst_u, uint8* dst_v, int pix) { + const uint8* src_rgba_1 = src_rgba + src_stride_rgba; + asm volatile ( + RGBTOUV_SETUP_REG + "1: \n" + MEMACCESS(0) + "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels. + "uaddlp v0.8h, v1.16b \n" // B 16 bytes -> 8 shorts. + "uaddlp v1.8h, v2.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v2.8h, v3.16b \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load 16 more. + "uadalp v0.8h, v5.16b \n" // B 16 bytes -> 8 shorts. + "uadalp v1.8h, v6.16b \n" // G 16 bytes -> 8 shorts. + "uadalp v2.8h, v7.16b \n" // R 16 bytes -> 8 shorts. + + "urshr v0.8h, v0.8h, #1 \n" // 2x average + "urshr v1.8h, v1.8h, #1 \n" + "urshr v2.8h, v2.8h, #1 \n" + + "subs %w4, %w4, #16 \n" // 32 processed per loop. + RGBTOUV(v0.8h, v1.8h, v2.8h) + MEMACCESS(2) + "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U. + MEMACCESS(3) + "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_rgba), // %0 + "+r"(src_rgba_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v20", "v21", "v22", "v23", "v24", "v25" + ); +} +#endif // HAS_RGBATOUVROW_NEON + +#ifdef HAS_RGB24TOUVROW_NEON +void RGB24ToUVRow_NEON(const uint8* src_rgb24, int src_stride_rgb24, + uint8* dst_u, uint8* dst_v, int pix) { + const uint8* src_rgb24_1 = src_rgb24 + src_stride_rgb24; + asm volatile ( + RGBTOUV_SETUP_REG + "1: \n" + MEMACCESS(0) + "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 pixels. + "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts. + "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "ld3 {v4.16b,v5.16b,v6.16b}, [%1], #48 \n" // load 16 more. + "uadalp v0.8h, v4.16b \n" // B 16 bytes -> 8 shorts. + "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts. + "uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts. + + "urshr v0.8h, v0.8h, #1 \n" // 2x average + "urshr v1.8h, v1.8h, #1 \n" + "urshr v2.8h, v2.8h, #1 \n" + + "subs %w4, %w4, #16 \n" // 32 processed per loop. + RGBTOUV(v0.8h, v1.8h, v2.8h) + MEMACCESS(2) + "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U. + MEMACCESS(3) + "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_rgb24), // %0 + "+r"(src_rgb24_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v20", "v21", "v22", "v23", "v24", "v25" + ); +} +#endif // HAS_RGB24TOUVROW_NEON + +#ifdef HAS_RAWTOUVROW_NEON +void RAWToUVRow_NEON(const uint8* src_raw, int src_stride_raw, + uint8* dst_u, uint8* dst_v, int pix) { + const uint8* src_raw_1 = src_raw + src_stride_raw; + asm volatile ( + RGBTOUV_SETUP_REG + "1: \n" + MEMACCESS(0) + "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 8 RAW pixels. + "uaddlp v2.8h, v2.16b \n" // B 16 bytes -> 8 shorts. + "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v0.8h, v0.16b \n" // R 16 bytes -> 8 shorts. + MEMACCESS(1) + "ld3 {v4.16b,v5.16b,v6.16b}, [%1], #48 \n" // load 8 more RAW pixels + "uadalp v2.8h, v6.16b \n" // B 16 bytes -> 8 shorts. + "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts. + "uadalp v0.8h, v4.16b \n" // R 16 bytes -> 8 shorts. + + "urshr v2.8h, v2.8h, #1 \n" // 2x average + "urshr v1.8h, v1.8h, #1 \n" + "urshr v0.8h, v0.8h, #1 \n" + + "subs %w4, %w4, #16 \n" // 32 processed per loop. + RGBTOUV(v2.8h, v1.8h, v0.8h) + MEMACCESS(2) + "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U. + MEMACCESS(3) + "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_raw), // %0 + "+r"(src_raw_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v20", "v21", "v22", "v23", "v24", "v25" + ); +} +#endif // HAS_RAWTOUVROW_NEON + +// 16x2 pixels -> 8x1. pix is number of argb pixels. e.g. 16. +#ifdef HAS_RGB565TOUVROW_NEON +void RGB565ToUVRow_NEON(const uint8* src_rgb565, int src_stride_rgb565, + uint8* dst_u, uint8* dst_v, int pix) { + const uint8* src_rgb565_1 = src_rgb565 + src_stride_rgb565; + asm volatile ( + "movi v22.8h, #56, lsl #0 \n" // UB / VR coeff (0.875) / 2 + "movi v23.8h, #37, lsl #0 \n" // UG coeff (-0.5781) / 2 + "movi v24.8h, #19, lsl #0 \n" // UR coeff (-0.2969) / 2 + "movi v25.8h, #9 , lsl #0 \n" // VB coeff (-0.1406) / 2 + "movi v26.8h, #47, lsl #0 \n" // VG coeff (-0.7344) / 2 + "movi v27.16b, #0x80 \n" // 128.5 (0x8080 in 16-bit) + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load 8 RGB565 pixels. + RGB565TOARGB + "uaddlp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts. + "uaddlp v18.4h, v1.8b \n" // G 8 bytes -> 4 shorts. + "uaddlp v20.4h, v2.8b \n" // R 8 bytes -> 4 shorts. + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // next 8 RGB565 pixels. + RGB565TOARGB + "uaddlp v17.4h, v0.8b \n" // B 8 bytes -> 4 shorts. + "uaddlp v19.4h, v1.8b \n" // G 8 bytes -> 4 shorts. + "uaddlp v21.4h, v2.8b \n" // R 8 bytes -> 4 shorts. + + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" // load 8 RGB565 pixels. + RGB565TOARGB + "uadalp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts. + "uadalp v18.4h, v1.8b \n" // G 8 bytes -> 4 shorts. + "uadalp v20.4h, v2.8b \n" // R 8 bytes -> 4 shorts. + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" // next 8 RGB565 pixels. + RGB565TOARGB + "uadalp v17.4h, v0.8b \n" // B 8 bytes -> 4 shorts. + "uadalp v19.4h, v1.8b \n" // G 8 bytes -> 4 shorts. + "uadalp v21.4h, v2.8b \n" // R 8 bytes -> 4 shorts. + + "ins v16.D[1], v17.D[0] \n" + "ins v18.D[1], v19.D[0] \n" + "ins v20.D[1], v21.D[0] \n" + + "urshr v4.8h, v16.8h, #1 \n" // 2x average + "urshr v5.8h, v18.8h, #1 \n" + "urshr v6.8h, v20.8h, #1 \n" + + "subs %w4, %w4, #16 \n" // 16 processed per loop. + "mul v16.8h, v4.8h, v22.8h \n" // B + "mls v16.8h, v5.8h, v23.8h \n" // G + "mls v16.8h, v6.8h, v24.8h \n" // R + "add v16.8h, v16.8h, v27.8h \n" // +128 -> unsigned + "mul v17.8h, v6.8h, v22.8h \n" // R + "mls v17.8h, v5.8h, v26.8h \n" // G + "mls v17.8h, v4.8h, v25.8h \n" // B + "add v17.8h, v17.8h, v27.8h \n" // +128 -> unsigned + "uqshrn v0.8b, v16.8h, #8 \n" // 16 bit to 8 bit U + "uqshrn v1.8b, v17.8h, #8 \n" // 16 bit to 8 bit V + MEMACCESS(2) + "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U. + MEMACCESS(3) + "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_rgb565), // %0 + "+r"(src_rgb565_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27" + ); +} +#endif // HAS_RGB565TOUVROW_NEON + +// 16x2 pixels -> 8x1. pix is number of argb pixels. e.g. 16. +#ifdef HAS_ARGB1555TOUVROW_NEON +void ARGB1555ToUVRow_NEON(const uint8* src_argb1555, int src_stride_argb1555, + uint8* dst_u, uint8* dst_v, int pix) { + const uint8* src_argb1555_1 = src_argb1555 + src_stride_argb1555; + asm volatile ( + RGBTOUV_SETUP_REG + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB1555 pixels. + RGB555TOARGB + "uaddlp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts. + "uaddlp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts. + "uaddlp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts. + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // next 8 ARGB1555 pixels. + RGB555TOARGB + "uaddlp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts. + "uaddlp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts. + "uaddlp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts. + + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" // load 8 ARGB1555 pixels. + RGB555TOARGB + "uadalp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts. + "uadalp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts. + "uadalp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts. + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" // next 8 ARGB1555 pixels. + RGB555TOARGB + "uadalp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts. + "uadalp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts. + "uadalp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts. + + "ins v16.D[1], v26.D[0] \n" + "ins v17.D[1], v27.D[0] \n" + "ins v18.D[1], v28.D[0] \n" + + "urshr v4.8h, v16.8h, #1 \n" // 2x average + "urshr v5.8h, v17.8h, #1 \n" + "urshr v6.8h, v18.8h, #1 \n" + + "subs %w4, %w4, #16 \n" // 16 processed per loop. + "mul v2.8h, v4.8h, v20.8h \n" // B + "mls v2.8h, v5.8h, v21.8h \n" // G + "mls v2.8h, v6.8h, v22.8h \n" // R + "add v2.8h, v2.8h, v25.8h \n" // +128 -> unsigned + "mul v3.8h, v6.8h, v20.8h \n" // R + "mls v3.8h, v5.8h, v24.8h \n" // G + "mls v3.8h, v4.8h, v23.8h \n" // B + "add v3.8h, v3.8h, v25.8h \n" // +128 -> unsigned + "uqshrn v0.8b, v2.8h, #8 \n" // 16 bit to 8 bit U + "uqshrn v1.8b, v3.8h, #8 \n" // 16 bit to 8 bit V + MEMACCESS(2) + "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U. + MEMACCESS(3) + "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_argb1555), // %0 + "+r"(src_argb1555_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", + "v26", "v27", "v28" + ); +} +#endif // HAS_ARGB1555TOUVROW_NEON + +// 16x2 pixels -> 8x1. pix is number of argb pixels. e.g. 16. +#ifdef HAS_ARGB4444TOUVROW_NEON +void ARGB4444ToUVRow_NEON(const uint8* src_argb4444, int src_stride_argb4444, + uint8* dst_u, uint8* dst_v, int pix) { + const uint8* src_argb4444_1 = src_argb4444 + src_stride_argb4444; + asm volatile ( + RGBTOUV_SETUP_REG + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB4444 pixels. + ARGB4444TOARGB + "uaddlp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts. + "uaddlp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts. + "uaddlp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts. + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // next 8 ARGB4444 pixels. + ARGB4444TOARGB + "uaddlp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts. + "uaddlp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts. + "uaddlp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts. + + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" // load 8 ARGB4444 pixels. + ARGB4444TOARGB + "uadalp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts. + "uadalp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts. + "uadalp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts. + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" // next 8 ARGB4444 pixels. + ARGB4444TOARGB + "uadalp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts. + "uadalp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts. + "uadalp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts. + + "ins v16.D[1], v26.D[0] \n" + "ins v17.D[1], v27.D[0] \n" + "ins v18.D[1], v28.D[0] \n" + + "urshr v4.8h, v16.8h, #1 \n" // 2x average + "urshr v5.8h, v17.8h, #1 \n" + "urshr v6.8h, v18.8h, #1 \n" + + "subs %w4, %w4, #16 \n" // 16 processed per loop. + "mul v2.8h, v4.8h, v20.8h \n" // B + "mls v2.8h, v5.8h, v21.8h \n" // G + "mls v2.8h, v6.8h, v22.8h \n" // R + "add v2.8h, v2.8h, v25.8h \n" // +128 -> unsigned + "mul v3.8h, v6.8h, v20.8h \n" // R + "mls v3.8h, v5.8h, v24.8h \n" // G + "mls v3.8h, v4.8h, v23.8h \n" // B + "add v3.8h, v3.8h, v25.8h \n" // +128 -> unsigned + "uqshrn v0.8b, v2.8h, #8 \n" // 16 bit to 8 bit U + "uqshrn v1.8b, v3.8h, #8 \n" // 16 bit to 8 bit V + MEMACCESS(2) + "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U. + MEMACCESS(3) + "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_argb4444), // %0 + "+r"(src_argb4444_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(pix) // %4 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", + "v26", "v27", "v28" + + ); +} +#endif // HAS_ARGB4444TOUVROW_NEON + +#ifdef HAS_RGB565TOYROW_NEON +void RGB565ToYRow_NEON(const uint8* src_rgb565, uint8* dst_y, int pix) { + asm volatile ( + "movi v24.8b, #13 \n" // B * 0.1016 coefficient + "movi v25.8b, #65 \n" // G * 0.5078 coefficient + "movi v26.8b, #33 \n" // R * 0.2578 coefficient + "movi v27.8b, #16 \n" // Add 16 constant + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load 8 RGB565 pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + RGB565TOARGB + "umull v3.8h, v0.8b, v24.8b \n" // B + "umlal v3.8h, v1.8b, v25.8b \n" // G + "umlal v3.8h, v2.8b, v26.8b \n" // R + "sqrshrun v0.8b, v3.8h, #7 \n" // 16 bit to 8 bit Y + "uqadd v0.8b, v0.8b, v27.8b \n" + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y. + "b.gt 1b \n" + : "+r"(src_rgb565), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v6", + "v24", "v25", "v26", "v27" + ); +} +#endif // HAS_RGB565TOYROW_NEON + +#ifdef HAS_ARGB1555TOYROW_NEON +void ARGB1555ToYRow_NEON(const uint8* src_argb1555, uint8* dst_y, int pix) { + asm volatile ( + "movi v4.8b, #13 \n" // B * 0.1016 coefficient + "movi v5.8b, #65 \n" // G * 0.5078 coefficient + "movi v6.8b, #33 \n" // R * 0.2578 coefficient + "movi v7.8b, #16 \n" // Add 16 constant + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB1555 pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + ARGB1555TOARGB + "umull v3.8h, v0.8b, v4.8b \n" // B + "umlal v3.8h, v1.8b, v5.8b \n" // G + "umlal v3.8h, v2.8b, v6.8b \n" // R + "sqrshrun v0.8b, v3.8h, #7 \n" // 16 bit to 8 bit Y + "uqadd v0.8b, v0.8b, v7.8b \n" + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y. + "b.gt 1b \n" + : "+r"(src_argb1555), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + ); +} +#endif // HAS_ARGB1555TOYROW_NEON + +#ifdef HAS_ARGB4444TOYROW_NEON +void ARGB4444ToYRow_NEON(const uint8* src_argb4444, uint8* dst_y, int pix) { + asm volatile ( + "movi v24.8b, #13 \n" // B * 0.1016 coefficient + "movi v25.8b, #65 \n" // G * 0.5078 coefficient + "movi v26.8b, #33 \n" // R * 0.2578 coefficient + "movi v27.8b, #16 \n" // Add 16 constant + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB4444 pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + ARGB4444TOARGB + "umull v3.8h, v0.8b, v24.8b \n" // B + "umlal v3.8h, v1.8b, v25.8b \n" // G + "umlal v3.8h, v2.8b, v26.8b \n" // R + "sqrshrun v0.8b, v3.8h, #7 \n" // 16 bit to 8 bit Y + "uqadd v0.8b, v0.8b, v27.8b \n" + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y. + "b.gt 1b \n" + : "+r"(src_argb4444), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v24", "v25", "v26", "v27" + ); +} +#endif // HAS_ARGB4444TOYROW_NEON + +#ifdef HAS_BGRATOYROW_NEON +void BGRAToYRow_NEON(const uint8* src_bgra, uint8* dst_y, int pix) { + asm volatile ( + "movi v4.8b, #33 \n" // R * 0.2578 coefficient + "movi v5.8b, #65 \n" // G * 0.5078 coefficient + "movi v6.8b, #13 \n" // B * 0.1016 coefficient + "movi v7.8b, #16 \n" // Add 16 constant + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "umull v16.8h, v1.8b, v4.8b \n" // R + "umlal v16.8h, v2.8b, v5.8b \n" // G + "umlal v16.8h, v3.8b, v6.8b \n" // B + "sqrshrun v0.8b, v16.8h, #7 \n" // 16 bit to 8 bit Y + "uqadd v0.8b, v0.8b, v7.8b \n" + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y. + "b.gt 1b \n" + : "+r"(src_bgra), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16" + ); +} +#endif // HAS_BGRATOYROW_NEON + +#ifdef HAS_ABGRTOYROW_NEON +void ABGRToYRow_NEON(const uint8* src_abgr, uint8* dst_y, int pix) { + asm volatile ( + "movi v4.8b, #33 \n" // R * 0.2578 coefficient + "movi v5.8b, #65 \n" // G * 0.5078 coefficient + "movi v6.8b, #13 \n" // B * 0.1016 coefficient + "movi v7.8b, #16 \n" // Add 16 constant + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "umull v16.8h, v0.8b, v4.8b \n" // R + "umlal v16.8h, v1.8b, v5.8b \n" // G + "umlal v16.8h, v2.8b, v6.8b \n" // B + "sqrshrun v0.8b, v16.8h, #7 \n" // 16 bit to 8 bit Y + "uqadd v0.8b, v0.8b, v7.8b \n" + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y. + "b.gt 1b \n" + : "+r"(src_abgr), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16" + ); +} +#endif // HAS_ABGRTOYROW_NEON + +#ifdef HAS_RGBATOYROW_NEON +void RGBAToYRow_NEON(const uint8* src_rgba, uint8* dst_y, int pix) { + asm volatile ( + "movi v4.8b, #13 \n" // B * 0.1016 coefficient + "movi v5.8b, #65 \n" // G * 0.5078 coefficient + "movi v6.8b, #33 \n" // R * 0.2578 coefficient + "movi v7.8b, #16 \n" // Add 16 constant + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "umull v16.8h, v1.8b, v4.8b \n" // B + "umlal v16.8h, v2.8b, v5.8b \n" // G + "umlal v16.8h, v3.8b, v6.8b \n" // R + "sqrshrun v0.8b, v16.8h, #7 \n" // 16 bit to 8 bit Y + "uqadd v0.8b, v0.8b, v7.8b \n" + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y. + "b.gt 1b \n" + : "+r"(src_rgba), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16" + ); +} +#endif // HAS_RGBATOYROW_NEON + +#ifdef HAS_RGB24TOYROW_NEON +void RGB24ToYRow_NEON(const uint8* src_rgb24, uint8* dst_y, int pix) { + asm volatile ( + "movi v4.8b, #13 \n" // B * 0.1016 coefficient + "movi v5.8b, #65 \n" // G * 0.5078 coefficient + "movi v6.8b, #33 \n" // R * 0.2578 coefficient + "movi v7.8b, #16 \n" // Add 16 constant + "1: \n" + MEMACCESS(0) + "ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // load 8 pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "umull v16.8h, v0.8b, v4.8b \n" // B + "umlal v16.8h, v1.8b, v5.8b \n" // G + "umlal v16.8h, v2.8b, v6.8b \n" // R + "sqrshrun v0.8b, v16.8h, #7 \n" // 16 bit to 8 bit Y + "uqadd v0.8b, v0.8b, v7.8b \n" + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y. + "b.gt 1b \n" + : "+r"(src_rgb24), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16" + ); +} +#endif // HAS_RGB24TOYROW_NEON + +#ifdef HAS_RAWTOYROW_NEON +void RAWToYRow_NEON(const uint8* src_raw, uint8* dst_y, int pix) { + asm volatile ( + "movi v4.8b, #33 \n" // R * 0.2578 coefficient + "movi v5.8b, #65 \n" // G * 0.5078 coefficient + "movi v6.8b, #13 \n" // B * 0.1016 coefficient + "movi v7.8b, #16 \n" // Add 16 constant + "1: \n" + MEMACCESS(0) + "ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // load 8 pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "umull v16.8h, v0.8b, v4.8b \n" // B + "umlal v16.8h, v1.8b, v5.8b \n" // G + "umlal v16.8h, v2.8b, v6.8b \n" // R + "sqrshrun v0.8b, v16.8h, #7 \n" // 16 bit to 8 bit Y + "uqadd v0.8b, v0.8b, v7.8b \n" + MEMACCESS(1) + "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y. + "b.gt 1b \n" + : "+r"(src_raw), // %0 + "+r"(dst_y), // %1 + "+r"(pix) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16" + ); +} +#endif // HAS_RAWTOYROW_NEON + +// Bilinear filter 16x2 -> 16x1 +#ifdef HAS_INTERPOLATEROW_NEON +void InterpolateRow_NEON(uint8* dst_ptr, + const uint8* src_ptr, ptrdiff_t src_stride, + int dst_width, int source_y_fraction) { + int y1_fraction = source_y_fraction; + int y0_fraction = 256 - y1_fraction; + const uint8* src_ptr1 = src_ptr + src_stride; + asm volatile ( + "cmp %w4, #0 \n" + "b.eq 100f \n" + "cmp %w4, #64 \n" + "b.eq 75f \n" + "cmp %w4, #128 \n" + "b.eq 50f \n" + "cmp %w4, #192 \n" + "b.eq 25f \n" + + "dup v5.16b, %w4 \n" + "dup v4.16b, %w5 \n" + // General purpose row blend. + "1: \n" + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" + MEMACCESS(2) + "ld1 {v1.16b}, [%2], #16 \n" + "subs %w3, %w3, #16 \n" + "umull v2.8h, v0.8b, v4.8b \n" + "umull2 v3.8h, v0.16b, v4.16b \n" + "umlal v2.8h, v1.8b, v5.8b \n" + "umlal2 v3.8h, v1.16b, v5.16b \n" + "rshrn v0.8b, v2.8h, #8 \n" + "rshrn2 v0.16b, v3.8h, #8 \n" + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" + "b.gt 1b \n" + "b 99f \n" + + // Blend 25 / 75. + "25: \n" + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" + MEMACCESS(2) + "ld1 {v1.16b}, [%2], #16 \n" + "subs %w3, %w3, #16 \n" + "urhadd v0.16b, v0.16b, v1.16b \n" + "urhadd v0.16b, v0.16b, v1.16b \n" + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" + "b.gt 25b \n" + "b 99f \n" + + // Blend 50 / 50. + "50: \n" + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" + MEMACCESS(2) + "ld1 {v1.16b}, [%2], #16 \n" + "subs %w3, %w3, #16 \n" + "urhadd v0.16b, v0.16b, v1.16b \n" + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" + "b.gt 50b \n" + "b 99f \n" + + // Blend 75 / 25. + "75: \n" + MEMACCESS(1) + "ld1 {v1.16b}, [%1], #16 \n" + MEMACCESS(2) + "ld1 {v0.16b}, [%2], #16 \n" + "subs %w3, %w3, #16 \n" + "urhadd v0.16b, v0.16b, v1.16b \n" + "urhadd v0.16b, v0.16b, v1.16b \n" + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" + "b.gt 75b \n" + "b 99f \n" + + // Blend 100 / 0 - Copy row unchanged. + "100: \n" + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" + "subs %w3, %w3, #16 \n" + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" + "b.gt 100b \n" + + "99: \n" + : "+r"(dst_ptr), // %0 + "+r"(src_ptr), // %1 + "+r"(src_ptr1), // %2 + "+r"(dst_width), // %3 + "+r"(y1_fraction), // %4 + "+r"(y0_fraction) // %5 + : + : "cc", "memory", "v0", "v1", "v3", "v4", "v5" + ); +} +#endif // HAS_INTERPOLATEROW_NEON + +// dr * (256 - sa) / 256 + sr = dr - dr * sa / 256 + sr +#ifdef HAS_ARGBBLENDROW_NEON +void ARGBBlendRow_NEON(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + "subs %w3, %w3, #8 \n" + "b.lt 89f \n" + // Blend 8 pixels. + "8: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB0 pixels + MEMACCESS(1) + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 ARGB1 pixels + "subs %w3, %w3, #8 \n" // 8 processed per loop. + "umull v16.8h, v4.8b, v3.8b \n" // db * a + "umull v17.8h, v5.8b, v3.8b \n" // dg * a + "umull v18.8h, v6.8b, v3.8b \n" // dr * a + "uqrshrn v16.8b, v16.8h, #8 \n" // db >>= 8 + "uqrshrn v17.8b, v17.8h, #8 \n" // dg >>= 8 + "uqrshrn v18.8b, v18.8h, #8 \n" // dr >>= 8 + "uqsub v4.8b, v4.8b, v16.8b \n" // db - (db * a / 256) + "uqsub v5.8b, v5.8b, v17.8b \n" // dg - (dg * a / 256) + "uqsub v6.8b, v6.8b, v18.8b \n" // dr - (dr * a / 256) + "uqadd v0.8b, v0.8b, v4.8b \n" // + sb + "uqadd v1.8b, v1.8b, v5.8b \n" // + sg + "uqadd v2.8b, v2.8b, v6.8b \n" // + sr + "movi v3.8b, #255 \n" // a = 255 + MEMACCESS(2) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB pixels + "b.ge 8b \n" + + "89: \n" + "adds %w3, %w3, #8-1 \n" + "b.lt 99f \n" + + // Blend 1 pixels. + "1: \n" + MEMACCESS(0) + "ld4 {v0.b,v1.b,v2.b,v3.b}[0], [%0], #4 \n" // load 1 pixel ARGB0. + MEMACCESS(1) + "ld4 {v4.b,v5.b,v6.b,v7.b}[0], [%1], #4 \n" // load 1 pixel ARGB1. + "subs %w3, %w3, #1 \n" // 1 processed per loop. + "umull v16.8h, v4.8b, v3.8b \n" // db * a + "umull v17.8h, v5.8b, v3.8b \n" // dg * a + "umull v18.8h, v6.8b, v3.8b \n" // dr * a + "uqrshrn v16.8b, v16.8h, #8 \n" // db >>= 8 + "uqrshrn v17.8b, v17.8h, #8 \n" // dg >>= 8 + "uqrshrn v18.8b, v18.8h, #8 \n" // dr >>= 8 + "uqsub v4.8b, v4.8b, v16.8b \n" // db - (db * a / 256) + "uqsub v5.8b, v5.8b, v17.8b \n" // dg - (dg * a / 256) + "uqsub v6.8b, v6.8b, v18.8b \n" // dr - (dr * a / 256) + "uqadd v0.8b, v0.8b, v4.8b \n" // + sb + "uqadd v1.8b, v1.8b, v5.8b \n" // + sg + "uqadd v2.8b, v2.8b, v6.8b \n" // + sr + "movi v3.8b, #255 \n" // a = 255 + MEMACCESS(2) + "st4 {v0.b,v1.b,v2.b,v3.b}[0], [%2], #4 \n" // store 1 pixel. + "b.ge 1b \n" + + "99: \n" + + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v16", "v17", "v18" + ); +} +#endif // HAS_ARGBBLENDROW_NEON + +// Attenuate 8 pixels at a time. +#ifdef HAS_ARGBATTENUATEROW_NEON +void ARGBAttenuateRow_NEON(const uint8* src_argb, uint8* dst_argb, int width) { + asm volatile ( + // Attenuate 8 pixels. + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB pixels + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "umull v4.8h, v0.8b, v3.8b \n" // b * a + "umull v5.8h, v1.8b, v3.8b \n" // g * a + "umull v6.8h, v2.8b, v3.8b \n" // r * a + "uqrshrn v0.8b, v4.8h, #8 \n" // b >>= 8 + "uqrshrn v1.8b, v5.8h, #8 \n" // g >>= 8 + "uqrshrn v2.8b, v6.8h, #8 \n" // r >>= 8 + MEMACCESS(1) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6" + ); +} +#endif // HAS_ARGBATTENUATEROW_NEON + +// Quantize 8 ARGB pixels (32 bytes). +// dst = (dst * scale >> 16) * interval_size + interval_offset; +#ifdef HAS_ARGBQUANTIZEROW_NEON +void ARGBQuantizeRow_NEON(uint8* dst_argb, int scale, int interval_size, + int interval_offset, int width) { + asm volatile ( + "dup v4.8h, %w2 \n" + "ushr v4.8h, v4.8h, #1 \n" // scale >>= 1 + "dup v5.8h, %w3 \n" // interval multiply. + "dup v6.8h, %w4 \n" // interval add + + // 8 pixel loop. + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0] \n" // load 8 pixels of ARGB. + "subs %w1, %w1, #8 \n" // 8 processed per loop. + "uxtl v0.8h, v0.8b \n" // b (0 .. 255) + "uxtl v1.8h, v1.8b \n" + "uxtl v2.8h, v2.8b \n" + "sqdmulh v0.8h, v0.8h, v4.8h \n" // b * scale + "sqdmulh v1.8h, v1.8h, v4.8h \n" // g + "sqdmulh v2.8h, v2.8h, v4.8h \n" // r + "mul v0.8h, v0.8h, v5.8h \n" // b * interval_size + "mul v1.8h, v1.8h, v5.8h \n" // g + "mul v2.8h, v2.8h, v5.8h \n" // r + "add v0.8h, v0.8h, v6.8h \n" // b + interval_offset + "add v1.8h, v1.8h, v6.8h \n" // g + "add v2.8h, v2.8h, v6.8h \n" // r + "uqxtn v0.8b, v0.8h \n" + "uqxtn v1.8b, v1.8h \n" + "uqxtn v2.8b, v2.8h \n" + MEMACCESS(0) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + : "+r"(dst_argb), // %0 + "+r"(width) // %1 + : "r"(scale), // %2 + "r"(interval_size), // %3 + "r"(interval_offset) // %4 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6" + ); +} +#endif // HAS_ARGBQUANTIZEROW_NEON + +// Shade 8 pixels at a time by specified value. +// NOTE vqrdmulh.s16 q10, q10, d0[0] must use a scaler register from 0 to 8. +// Rounding in vqrdmulh does +1 to high if high bit of low s16 is set. +#ifdef HAS_ARGBSHADEROW_NEON +void ARGBShadeRow_NEON(const uint8* src_argb, uint8* dst_argb, int width, + uint32 value) { + asm volatile ( + "dup v0.4s, %w3 \n" // duplicate scale value. + "zip1 v0.8b, v0.8b, v0.8b \n" // v0.8b aarrggbb. + "ushr v0.8h, v0.8h, #1 \n" // scale / 2. + + // 8 pixel loop. + "1: \n" + MEMACCESS(0) + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%0], #32 \n" // load 8 ARGB pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "uxtl v4.8h, v4.8b \n" // b (0 .. 255) + "uxtl v5.8h, v5.8b \n" + "uxtl v6.8h, v6.8b \n" + "uxtl v7.8h, v7.8b \n" + "sqrdmulh v4.8h, v4.8h, v0.h[0] \n" // b * scale * 2 + "sqrdmulh v5.8h, v5.8h, v0.h[1] \n" // g + "sqrdmulh v6.8h, v6.8h, v0.h[2] \n" // r + "sqrdmulh v7.8h, v7.8h, v0.h[3] \n" // a + "uqxtn v4.8b, v4.8h \n" + "uqxtn v5.8b, v5.8h \n" + "uqxtn v6.8b, v6.8h \n" + "uqxtn v7.8b, v7.8h \n" + MEMACCESS(1) + "st4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(value) // %3 + : "cc", "memory", "v0", "v4", "v5", "v6", "v7" + ); +} +#endif // HAS_ARGBSHADEROW_NEON + +// Convert 8 ARGB pixels (64 bytes) to 8 Gray ARGB pixels +// Similar to ARGBToYJ but stores ARGB. +// C code is (15 * b + 75 * g + 38 * r + 64) >> 7; +#ifdef HAS_ARGBGRAYROW_NEON +void ARGBGrayRow_NEON(const uint8* src_argb, uint8* dst_argb, int width) { + asm volatile ( + "movi v24.8b, #15 \n" // B * 0.11400 coefficient + "movi v25.8b, #75 \n" // G * 0.58700 coefficient + "movi v26.8b, #38 \n" // R * 0.29900 coefficient + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "umull v4.8h, v0.8b, v24.8b \n" // B + "umlal v4.8h, v1.8b, v25.8b \n" // G + "umlal v4.8h, v2.8b, v26.8b \n" // R + "sqrshrun v0.8b, v4.8h, #7 \n" // 15 bit to 8 bit B + "orr v1.8b, v0.8b, v0.8b \n" // G + "orr v2.8b, v0.8b, v0.8b \n" // R + MEMACCESS(1) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 pixels. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v24", "v25", "v26" + ); +} +#endif // HAS_ARGBGRAYROW_NEON + +// Convert 8 ARGB pixels (32 bytes) to 8 Sepia ARGB pixels. +// b = (r * 35 + g * 68 + b * 17) >> 7 +// g = (r * 45 + g * 88 + b * 22) >> 7 +// r = (r * 50 + g * 98 + b * 24) >> 7 + +#ifdef HAS_ARGBSEPIAROW_NEON +void ARGBSepiaRow_NEON(uint8* dst_argb, int width) { + asm volatile ( + "movi v20.8b, #17 \n" // BB coefficient + "movi v21.8b, #68 \n" // BG coefficient + "movi v22.8b, #35 \n" // BR coefficient + "movi v24.8b, #22 \n" // GB coefficient + "movi v25.8b, #88 \n" // GG coefficient + "movi v26.8b, #45 \n" // GR coefficient + "movi v28.8b, #24 \n" // BB coefficient + "movi v29.8b, #98 \n" // BG coefficient + "movi v30.8b, #50 \n" // BR coefficient + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0] \n" // load 8 ARGB pixels. + "subs %w1, %w1, #8 \n" // 8 processed per loop. + "umull v4.8h, v0.8b, v20.8b \n" // B to Sepia B + "umlal v4.8h, v1.8b, v21.8b \n" // G + "umlal v4.8h, v2.8b, v22.8b \n" // R + "umull v5.8h, v0.8b, v24.8b \n" // B to Sepia G + "umlal v5.8h, v1.8b, v25.8b \n" // G + "umlal v5.8h, v2.8b, v26.8b \n" // R + "umull v6.8h, v0.8b, v28.8b \n" // B to Sepia R + "umlal v6.8h, v1.8b, v29.8b \n" // G + "umlal v6.8h, v2.8b, v30.8b \n" // R + "uqshrn v0.8b, v4.8h, #7 \n" // 16 bit to 8 bit B + "uqshrn v1.8b, v5.8h, #7 \n" // 16 bit to 8 bit G + "uqshrn v2.8b, v6.8h, #7 \n" // 16 bit to 8 bit R + MEMACCESS(0) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // store 8 pixels. + "b.gt 1b \n" + : "+r"(dst_argb), // %0 + "+r"(width) // %1 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v20", "v21", "v22", "v24", "v25", "v26", "v28", "v29", "v30" + ); +} +#endif // HAS_ARGBSEPIAROW_NEON + +// Tranform 8 ARGB pixels (32 bytes) with color matrix. +// TODO(fbarchard): Was same as Sepia except matrix is provided. This function +// needs to saturate. Consider doing a non-saturating version. +#ifdef HAS_ARGBCOLORMATRIXROW_NEON +void ARGBColorMatrixRow_NEON(const uint8* src_argb, uint8* dst_argb, + const int8* matrix_argb, int width) { + asm volatile ( + MEMACCESS(3) + "ld1 {v2.16b}, [%3] \n" // load 3 ARGB vectors. + "sxtl v0.8h, v2.8b \n" // B,G coefficients s16. + "sxtl2 v1.8h, v2.16b \n" // R,A coefficients s16. + + "1: \n" + MEMACCESS(0) + "ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%0], #32 \n" // load 8 pixels. + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "uxtl v16.8h, v16.8b \n" // b (0 .. 255) 16 bit + "uxtl v17.8h, v17.8b \n" // g + "uxtl v18.8h, v18.8b \n" // r + "uxtl v19.8h, v19.8b \n" // a + "mul v22.8h, v16.8h, v0.h[0] \n" // B = B * Matrix B + "mul v23.8h, v16.8h, v0.h[4] \n" // G = B * Matrix G + "mul v24.8h, v16.8h, v1.h[0] \n" // R = B * Matrix R + "mul v25.8h, v16.8h, v1.h[4] \n" // A = B * Matrix A + "mul v4.8h, v17.8h, v0.h[1] \n" // B += G * Matrix B + "mul v5.8h, v17.8h, v0.h[5] \n" // G += G * Matrix G + "mul v6.8h, v17.8h, v1.h[1] \n" // R += G * Matrix R + "mul v7.8h, v17.8h, v1.h[5] \n" // A += G * Matrix A + "sqadd v22.8h, v22.8h, v4.8h \n" // Accumulate B + "sqadd v23.8h, v23.8h, v5.8h \n" // Accumulate G + "sqadd v24.8h, v24.8h, v6.8h \n" // Accumulate R + "sqadd v25.8h, v25.8h, v7.8h \n" // Accumulate A + "mul v4.8h, v18.8h, v0.h[2] \n" // B += R * Matrix B + "mul v5.8h, v18.8h, v0.h[6] \n" // G += R * Matrix G + "mul v6.8h, v18.8h, v1.h[2] \n" // R += R * Matrix R + "mul v7.8h, v18.8h, v1.h[6] \n" // A += R * Matrix A + "sqadd v22.8h, v22.8h, v4.8h \n" // Accumulate B + "sqadd v23.8h, v23.8h, v5.8h \n" // Accumulate G + "sqadd v24.8h, v24.8h, v6.8h \n" // Accumulate R + "sqadd v25.8h, v25.8h, v7.8h \n" // Accumulate A + "mul v4.8h, v19.8h, v0.h[3] \n" // B += A * Matrix B + "mul v5.8h, v19.8h, v0.h[7] \n" // G += A * Matrix G + "mul v6.8h, v19.8h, v1.h[3] \n" // R += A * Matrix R + "mul v7.8h, v19.8h, v1.h[7] \n" // A += A * Matrix A + "sqadd v22.8h, v22.8h, v4.8h \n" // Accumulate B + "sqadd v23.8h, v23.8h, v5.8h \n" // Accumulate G + "sqadd v24.8h, v24.8h, v6.8h \n" // Accumulate R + "sqadd v25.8h, v25.8h, v7.8h \n" // Accumulate A + "sqshrun v16.8b, v22.8h, #6 \n" // 16 bit to 8 bit B + "sqshrun v17.8b, v23.8h, #6 \n" // 16 bit to 8 bit G + "sqshrun v18.8b, v24.8h, #6 \n" // 16 bit to 8 bit R + "sqshrun v19.8b, v25.8h, #6 \n" // 16 bit to 8 bit A + MEMACCESS(1) + "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%1], #32 \n" // store 8 pixels. + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(matrix_argb) // %3 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", + "v18", "v19", "v22", "v23", "v24", "v25" + ); +} +#endif // HAS_ARGBCOLORMATRIXROW_NEON + +// TODO(fbarchard): fix vqshrun in ARGBMultiplyRow_NEON and reenable. +// Multiply 2 rows of ARGB pixels together, 8 pixels at a time. +#ifdef HAS_ARGBMULTIPLYROW_NEON +void ARGBMultiplyRow_NEON(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + // 8 pixel loop. + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB pixels. + MEMACCESS(1) + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 more pixels. + "subs %w3, %w3, #8 \n" // 8 processed per loop. + "umull v0.8h, v0.8b, v4.8b \n" // multiply B + "umull v1.8h, v1.8b, v5.8b \n" // multiply G + "umull v2.8h, v2.8b, v6.8b \n" // multiply R + "umull v3.8h, v3.8b, v7.8b \n" // multiply A + "rshrn v0.8b, v0.8h, #8 \n" // 16 bit to 8 bit B + "rshrn v1.8b, v1.8h, #8 \n" // 16 bit to 8 bit G + "rshrn v2.8b, v2.8h, #8 \n" // 16 bit to 8 bit R + "rshrn v3.8b, v3.8h, #8 \n" // 16 bit to 8 bit A + MEMACCESS(2) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + ); +} +#endif // HAS_ARGBMULTIPLYROW_NEON + +// Add 2 rows of ARGB pixels together, 8 pixels at a time. +#ifdef HAS_ARGBADDROW_NEON +void ARGBAddRow_NEON(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + // 8 pixel loop. + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB pixels. + MEMACCESS(1) + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 more pixels. + "subs %w3, %w3, #8 \n" // 8 processed per loop. + "uqadd v0.8b, v0.8b, v4.8b \n" + "uqadd v1.8b, v1.8b, v5.8b \n" + "uqadd v2.8b, v2.8b, v6.8b \n" + "uqadd v3.8b, v3.8b, v7.8b \n" + MEMACCESS(2) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + ); +} +#endif // HAS_ARGBADDROW_NEON + +// Subtract 2 rows of ARGB pixels, 8 pixels at a time. +#ifdef HAS_ARGBSUBTRACTROW_NEON +void ARGBSubtractRow_NEON(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + asm volatile ( + // 8 pixel loop. + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB pixels. + MEMACCESS(1) + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 more pixels. + "subs %w3, %w3, #8 \n" // 8 processed per loop. + "uqsub v0.8b, v0.8b, v4.8b \n" + "uqsub v1.8b, v1.8b, v5.8b \n" + "uqsub v2.8b, v2.8b, v6.8b \n" + "uqsub v3.8b, v3.8b, v7.8b \n" + MEMACCESS(2) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + + : "+r"(src_argb0), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + ); +} +#endif // HAS_ARGBSUBTRACTROW_NEON + +// Adds Sobel X and Sobel Y and stores Sobel into ARGB. +// A = 255 +// R = Sobel +// G = Sobel +// B = Sobel +#ifdef HAS_SOBELROW_NEON +void SobelRow_NEON(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width) { + asm volatile ( + "movi v3.8b, #255 \n" // alpha + // 8 pixel loop. + "1: \n" + MEMACCESS(0) + "ld1 {v0.8b}, [%0], #8 \n" // load 8 sobelx. + MEMACCESS(1) + "ld1 {v1.8b}, [%1], #8 \n" // load 8 sobely. + "subs %w3, %w3, #8 \n" // 8 processed per loop. + "uqadd v0.8b, v0.8b, v1.8b \n" // add + "orr v1.8b, v0.8b, v0.8b \n" + "orr v2.8b, v0.8b, v0.8b \n" + MEMACCESS(2) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + : "+r"(src_sobelx), // %0 + "+r"(src_sobely), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "v0", "v1", "v2", "v3" + ); +} +#endif // HAS_SOBELROW_NEON + +// Adds Sobel X and Sobel Y and stores Sobel into plane. +#ifdef HAS_SOBELTOPLANEROW_NEON +void SobelToPlaneRow_NEON(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_y, int width) { + asm volatile ( + // 16 pixel loop. + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load 16 sobelx. + MEMACCESS(1) + "ld1 {v1.16b}, [%1], #16 \n" // load 16 sobely. + "subs %w3, %w3, #16 \n" // 16 processed per loop. + "uqadd v0.16b, v0.16b, v1.16b \n" // add + MEMACCESS(2) + "st1 {v0.16b}, [%2], #16 \n" // store 16 pixels. + "b.gt 1b \n" + : "+r"(src_sobelx), // %0 + "+r"(src_sobely), // %1 + "+r"(dst_y), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "v0", "v1" + ); +} +#endif // HAS_SOBELTOPLANEROW_NEON + +// Mixes Sobel X, Sobel Y and Sobel into ARGB. +// A = 255 +// R = Sobel X +// G = Sobel +// B = Sobel Y +#ifdef HAS_SOBELXYROW_NEON +void SobelXYRow_NEON(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width) { + asm volatile ( + "movi v3.8b, #255 \n" // alpha + // 8 pixel loop. + "1: \n" + MEMACCESS(0) + "ld1 {v2.8b}, [%0], #8 \n" // load 8 sobelx. + MEMACCESS(1) + "ld1 {v0.8b}, [%1], #8 \n" // load 8 sobely. + "subs %w3, %w3, #8 \n" // 8 processed per loop. + "uqadd v1.8b, v0.8b, v2.8b \n" // add + MEMACCESS(2) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + : "+r"(src_sobelx), // %0 + "+r"(src_sobely), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "v0", "v1", "v2", "v3" + ); +} +#endif // HAS_SOBELXYROW_NEON + +// SobelX as a matrix is +// -1 0 1 +// -2 0 2 +// -1 0 1 +#ifdef HAS_SOBELXROW_NEON +void SobelXRow_NEON(const uint8* src_y0, const uint8* src_y1, + const uint8* src_y2, uint8* dst_sobelx, int width) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld1 {v0.8b}, [%0],%5 \n" // top + MEMACCESS(0) + "ld1 {v1.8b}, [%0],%6 \n" + "usubl v0.8h, v0.8b, v1.8b \n" + MEMACCESS(1) + "ld1 {v2.8b}, [%1],%5 \n" // center * 2 + MEMACCESS(1) + "ld1 {v3.8b}, [%1],%6 \n" + "usubl v1.8h, v2.8b, v3.8b \n" + "add v0.8h, v0.8h, v1.8h \n" + "add v0.8h, v0.8h, v1.8h \n" + MEMACCESS(2) + "ld1 {v2.8b}, [%2],%5 \n" // bottom + MEMACCESS(2) + "ld1 {v3.8b}, [%2],%6 \n" + "subs %w4, %w4, #8 \n" // 8 pixels + "usubl v1.8h, v2.8b, v3.8b \n" + "add v0.8h, v0.8h, v1.8h \n" + "abs v0.8h, v0.8h \n" + "uqxtn v0.8b, v0.8h \n" + MEMACCESS(3) + "st1 {v0.8b}, [%3], #8 \n" // store 8 sobelx + "b.gt 1b \n" + : "+r"(src_y0), // %0 + "+r"(src_y1), // %1 + "+r"(src_y2), // %2 + "+r"(dst_sobelx), // %3 + "+r"(width) // %4 + : "r"(2LL), // %5 + "r"(6LL) // %6 + : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List + ); +} +#endif // HAS_SOBELXROW_NEON + +// SobelY as a matrix is +// -1 -2 -1 +// 0 0 0 +// 1 2 1 +#ifdef HAS_SOBELYROW_NEON +void SobelYRow_NEON(const uint8* src_y0, const uint8* src_y1, + uint8* dst_sobely, int width) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld1 {v0.8b}, [%0],%4 \n" // left + MEMACCESS(1) + "ld1 {v1.8b}, [%1],%4 \n" + "usubl v0.8h, v0.8b, v1.8b \n" + MEMACCESS(0) + "ld1 {v2.8b}, [%0],%4 \n" // center * 2 + MEMACCESS(1) + "ld1 {v3.8b}, [%1],%4 \n" + "usubl v1.8h, v2.8b, v3.8b \n" + "add v0.8h, v0.8h, v1.8h \n" + "add v0.8h, v0.8h, v1.8h \n" + MEMACCESS(0) + "ld1 {v2.8b}, [%0],%5 \n" // right + MEMACCESS(1) + "ld1 {v3.8b}, [%1],%5 \n" + "subs %w3, %w3, #8 \n" // 8 pixels + "usubl v1.8h, v2.8b, v3.8b \n" + "add v0.8h, v0.8h, v1.8h \n" + "abs v0.8h, v0.8h \n" + "uqxtn v0.8b, v0.8h \n" + MEMACCESS(2) + "st1 {v0.8b}, [%2], #8 \n" // store 8 sobely + "b.gt 1b \n" + : "+r"(src_y0), // %0 + "+r"(src_y1), // %1 + "+r"(dst_sobely), // %2 + "+r"(width) // %3 + : "r"(1LL), // %4 + "r"(6LL) // %5 + : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List + ); +} +#endif // HAS_SOBELYROW_NEON +#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/row_win.cc b/third_party/aom/third_party/libyuv/source/row_win.cc new file mode 100644 index 0000000000..71be268b47 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/row_win.cc @@ -0,0 +1,6331 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" + +#if !defined(LIBYUV_DISABLE_X86) && defined(_M_X64) && \ + defined(_MSC_VER) && !defined(__clang__) +#include +#include // For _mm_maddubs_epi16 +#endif + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for Visual C. +#if !defined(LIBYUV_DISABLE_X86) && (defined(_M_IX86) || defined(_M_X64)) && \ + defined(_MSC_VER) && !defined(__clang__) + +struct YuvConstants { + lvec8 kUVToB; // 0 + lvec8 kUVToG; // 32 + lvec8 kUVToR; // 64 + lvec16 kUVBiasB; // 96 + lvec16 kUVBiasG; // 128 + lvec16 kUVBiasR; // 160 + lvec16 kYToRgb; // 192 +}; + +// BT.601 YUV to RGB reference +// R = (Y - 16) * 1.164 - V * -1.596 +// G = (Y - 16) * 1.164 - U * 0.391 - V * 0.813 +// B = (Y - 16) * 1.164 - U * -2.018 + +// Y contribution to R,G,B. Scale and bias. +// TODO(fbarchard): Consider moving constants into a common header. +#define YG 18997 /* round(1.164 * 64 * 256 * 256 / 257) */ +#define YGB -1160 /* 1.164 * 64 * -16 + 64 / 2 */ + +// U and V contributions to R,G,B. +#define UB -128 /* max(-128, round(-2.018 * 64)) */ +#define UG 25 /* round(0.391 * 64) */ +#define VG 52 /* round(0.813 * 64) */ +#define VR -102 /* round(-1.596 * 64) */ + +// Bias values to subtract 16 from Y and 128 from U and V. +#define BB (UB * 128 + YGB) +#define BG (UG * 128 + VG * 128 + YGB) +#define BR (VR * 128 + YGB) + +// BT601 constants for YUV to RGB. +static YuvConstants SIMD_ALIGNED(kYuvConstants) = { + { UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, + UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0 }, + { UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, + UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG }, + { 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, + 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR }, + { BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB }, + { BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG }, + { BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR }, + { YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG } +}; + +// BT601 constants for NV21 where chroma plane is VU instead of UV. +static YuvConstants SIMD_ALIGNED(kYvuConstants) = { + { 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, + 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB }, + { VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, + VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG }, + { VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, + VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0 }, + { BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB }, + { BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG }, + { BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR }, + { YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG } +}; + +#undef YG +#undef YGB +#undef UB +#undef UG +#undef VG +#undef VR +#undef BB +#undef BG +#undef BR + +// JPEG YUV to RGB reference +// * R = Y - V * -1.40200 +// * G = Y - U * 0.34414 - V * 0.71414 +// * B = Y - U * -1.77200 + +// Y contribution to R,G,B. Scale and bias. +// TODO(fbarchard): Consider moving constants into a common header. +#define YGJ 16320 /* round(1.000 * 64 * 256 * 256 / 257) */ +#define YGBJ 32 /* 64 / 2 */ + +// U and V contributions to R,G,B. +#define UBJ -113 /* round(-1.77200 * 64) */ +#define UGJ 22 /* round(0.34414 * 64) */ +#define VGJ 46 /* round(0.71414 * 64) */ +#define VRJ -90 /* round(-1.40200 * 64) */ + +// Bias values to subtract 16 from Y and 128 from U and V. +#define BBJ (UBJ * 128 + YGBJ) +#define BGJ (UGJ * 128 + VGJ * 128 + YGBJ) +#define BRJ (VRJ * 128 + YGBJ) + +// JPEG constants for YUV to RGB. +static YuvConstants SIMD_ALIGNED(kYuvJConstants) = { + { UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, + UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0, UBJ, 0 }, + { UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, + UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, + UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, + UGJ, VGJ, UGJ, VGJ, UGJ, VGJ, UGJ, VGJ }, + { 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, + 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ, 0, VRJ }, + { BBJ, BBJ, BBJ, BBJ, BBJ, BBJ, BBJ, BBJ, + BBJ, BBJ, BBJ, BBJ, BBJ, BBJ, BBJ, BBJ }, + { BGJ, BGJ, BGJ, BGJ, BGJ, BGJ, BGJ, BGJ, + BGJ, BGJ, BGJ, BGJ, BGJ, BGJ, BGJ, BGJ }, + { BRJ, BRJ, BRJ, BRJ, BRJ, BRJ, BRJ, BRJ, + BRJ, BRJ, BRJ, BRJ, BRJ, BRJ, BRJ, BRJ }, + { YGJ, YGJ, YGJ, YGJ, YGJ, YGJ, YGJ, YGJ, + YGJ, YGJ, YGJ, YGJ, YGJ, YGJ, YGJ, YGJ } +}; + +#undef YGJ +#undef YGBJ +#undef UBJ +#undef UGJ +#undef VGJ +#undef VRJ +#undef BBJ +#undef BGJ +#undef BRJ + +// 64 bit +#if defined(_M_X64) +#if defined(HAS_I422TOARGBROW_SSSE3) +void I422ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + __m128i xmm0, xmm1, xmm2, xmm3; + const __m128i xmm5 = _mm_set1_epi8(-1); + const ptrdiff_t offset = (uint8*)v_buf - (uint8*)u_buf; + + while (width > 0) { + xmm0 = _mm_cvtsi32_si128(*(uint32*)u_buf); + xmm1 = _mm_cvtsi32_si128(*(uint32*)(u_buf + offset)); + xmm0 = _mm_unpacklo_epi8(xmm0, xmm1); + xmm0 = _mm_unpacklo_epi16(xmm0, xmm0); + xmm1 = _mm_loadu_si128(&xmm0); + xmm2 = _mm_loadu_si128(&xmm0); + xmm0 = _mm_maddubs_epi16(xmm0, *(__m128i*)kYuvConstants.kUVToB); + xmm1 = _mm_maddubs_epi16(xmm1, *(__m128i*)kYuvConstants.kUVToG); + xmm2 = _mm_maddubs_epi16(xmm2, *(__m128i*)kYuvConstants.kUVToR); + xmm0 = _mm_sub_epi16(*(__m128i*)kYuvConstants.kUVBiasB, xmm0); + xmm1 = _mm_sub_epi16(*(__m128i*)kYuvConstants.kUVBiasG, xmm1); + xmm2 = _mm_sub_epi16(*(__m128i*)kYuvConstants.kUVBiasR, xmm2); + xmm3 = _mm_loadl_epi64((__m128i*)y_buf); + xmm3 = _mm_unpacklo_epi8(xmm3, xmm3); + xmm3 = _mm_mulhi_epu16(xmm3, *(__m128i*)kYuvConstants.kYToRgb); + xmm0 = _mm_adds_epi16(xmm0, xmm3); + xmm1 = _mm_adds_epi16(xmm1, xmm3); + xmm2 = _mm_adds_epi16(xmm2, xmm3); + xmm0 = _mm_srai_epi16(xmm0, 6); + xmm1 = _mm_srai_epi16(xmm1, 6); + xmm2 = _mm_srai_epi16(xmm2, 6); + xmm0 = _mm_packus_epi16(xmm0, xmm0); + xmm1 = _mm_packus_epi16(xmm1, xmm1); + xmm2 = _mm_packus_epi16(xmm2, xmm2); + xmm0 = _mm_unpacklo_epi8(xmm0, xmm1); + xmm2 = _mm_unpacklo_epi8(xmm2, xmm5); + xmm1 = _mm_loadu_si128(&xmm0); + xmm0 = _mm_unpacklo_epi16(xmm0, xmm2); + xmm1 = _mm_unpackhi_epi16(xmm1, xmm2); + + _mm_storeu_si128((__m128i *)dst_argb, xmm0); + _mm_storeu_si128((__m128i *)(dst_argb + 16), xmm1); + + y_buf += 8; + u_buf += 4; + dst_argb += 32; + width -= 8; + } +} +#endif +// 32 bit +#else // defined(_M_X64) +#ifdef HAS_ARGBTOYROW_SSSE3 + +// Constants for ARGB. +static const vec8 kARGBToY = { + 13, 65, 33, 0, 13, 65, 33, 0, 13, 65, 33, 0, 13, 65, 33, 0 +}; + +// JPeg full range. +static const vec8 kARGBToYJ = { + 15, 75, 38, 0, 15, 75, 38, 0, 15, 75, 38, 0, 15, 75, 38, 0 +}; + +static const vec8 kARGBToU = { + 112, -74, -38, 0, 112, -74, -38, 0, 112, -74, -38, 0, 112, -74, -38, 0 +}; + +static const vec8 kARGBToUJ = { + 127, -84, -43, 0, 127, -84, -43, 0, 127, -84, -43, 0, 127, -84, -43, 0 +}; + +static const vec8 kARGBToV = { + -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0, +}; + +static const vec8 kARGBToVJ = { + -20, -107, 127, 0, -20, -107, 127, 0, -20, -107, 127, 0, -20, -107, 127, 0 +}; + +// vpshufb for vphaddw + vpackuswb packed to shorts. +static const lvec8 kShufARGBToUV_AVX = { + 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15, + 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +}; + +// Constants for BGRA. +static const vec8 kBGRAToY = { + 0, 33, 65, 13, 0, 33, 65, 13, 0, 33, 65, 13, 0, 33, 65, 13 +}; + +static const vec8 kBGRAToU = { + 0, -38, -74, 112, 0, -38, -74, 112, 0, -38, -74, 112, 0, -38, -74, 112 +}; + +static const vec8 kBGRAToV = { + 0, 112, -94, -18, 0, 112, -94, -18, 0, 112, -94, -18, 0, 112, -94, -18 +}; + +// Constants for ABGR. +static const vec8 kABGRToY = { + 33, 65, 13, 0, 33, 65, 13, 0, 33, 65, 13, 0, 33, 65, 13, 0 +}; + +static const vec8 kABGRToU = { + -38, -74, 112, 0, -38, -74, 112, 0, -38, -74, 112, 0, -38, -74, 112, 0 +}; + +static const vec8 kABGRToV = { + 112, -94, -18, 0, 112, -94, -18, 0, 112, -94, -18, 0, 112, -94, -18, 0 +}; + +// Constants for RGBA. +static const vec8 kRGBAToY = { + 0, 13, 65, 33, 0, 13, 65, 33, 0, 13, 65, 33, 0, 13, 65, 33 +}; + +static const vec8 kRGBAToU = { + 0, 112, -74, -38, 0, 112, -74, -38, 0, 112, -74, -38, 0, 112, -74, -38 +}; + +static const vec8 kRGBAToV = { + 0, -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112 +}; + +static const uvec8 kAddY16 = { + 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u +}; + +// 7 bit fixed point 0.5. +static const vec16 kAddYJ64 = { + 64, 64, 64, 64, 64, 64, 64, 64 +}; + +static const uvec8 kAddUV128 = { + 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, + 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u +}; + +static const uvec16 kAddUVJ128 = { + 0x8080u, 0x8080u, 0x8080u, 0x8080u, 0x8080u, 0x8080u, 0x8080u, 0x8080u +}; + +// Shuffle table for converting RGB24 to ARGB. +static const uvec8 kShuffleMaskRGB24ToARGB = { + 0u, 1u, 2u, 12u, 3u, 4u, 5u, 13u, 6u, 7u, 8u, 14u, 9u, 10u, 11u, 15u +}; + +// Shuffle table for converting RAW to ARGB. +static const uvec8 kShuffleMaskRAWToARGB = { + 2u, 1u, 0u, 12u, 5u, 4u, 3u, 13u, 8u, 7u, 6u, 14u, 11u, 10u, 9u, 15u +}; + +// Shuffle table for converting ARGB to RGB24. +static const uvec8 kShuffleMaskARGBToRGB24 = { + 0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u, 128u +}; + +// Shuffle table for converting ARGB to RAW. +static const uvec8 kShuffleMaskARGBToRAW = { + 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u, 128u +}; + +// Shuffle table for converting ARGBToRGB24 for I422ToRGB24. First 8 + next 4 +static const uvec8 kShuffleMaskARGBToRGB24_0 = { + 0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u, 14u +}; + +// Shuffle table for converting ARGB to RAW. +static const uvec8 kShuffleMaskARGBToRAW_0 = { + 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 128u, 128u, 128u, 128u, 8u, 14u, 13u, 12u +}; + +// Duplicates gray value 3 times and fills in alpha opaque. +__declspec(naked) +void J400ToARGBRow_SSE2(const uint8* src_y, uint8* dst_argb, int pix) { + __asm { + mov eax, [esp + 4] // src_y + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // pix + pcmpeqb xmm5, xmm5 // generate mask 0xff000000 + pslld xmm5, 24 + + convertloop: + movq xmm0, qword ptr [eax] + lea eax, [eax + 8] + punpcklbw xmm0, xmm0 + movdqa xmm1, xmm0 + punpcklwd xmm0, xmm0 + punpckhwd xmm1, xmm1 + por xmm0, xmm5 + por xmm1, xmm5 + movdqu [edx], xmm0 + movdqu [edx + 16], xmm1 + lea edx, [edx + 32] + sub ecx, 8 + jg convertloop + ret + } +} + +#ifdef HAS_J400TOARGBROW_AVX2 +// Duplicates gray value 3 times and fills in alpha opaque. +__declspec(naked) +void J400ToARGBRow_AVX2(const uint8* src_y, uint8* dst_argb, int pix) { + __asm { + mov eax, [esp + 4] // src_y + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // pix + vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0xff000000 + vpslld ymm5, ymm5, 24 + + convertloop: + vmovdqu xmm0, [eax] + lea eax, [eax + 16] + vpermq ymm0, ymm0, 0xd8 + vpunpcklbw ymm0, ymm0, ymm0 + vpermq ymm0, ymm0, 0xd8 + vpunpckhwd ymm1, ymm0, ymm0 + vpunpcklwd ymm0, ymm0, ymm0 + vpor ymm0, ymm0, ymm5 + vpor ymm1, ymm1, ymm5 + vmovdqu [edx], ymm0 + vmovdqu [edx + 32], ymm1 + lea edx, [edx + 64] + sub ecx, 16 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_J400TOARGBROW_AVX2 + +__declspec(naked) +void RGB24ToARGBRow_SSSE3(const uint8* src_rgb24, uint8* dst_argb, int pix) { + __asm { + mov eax, [esp + 4] // src_rgb24 + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // pix + pcmpeqb xmm5, xmm5 // generate mask 0xff000000 + pslld xmm5, 24 + movdqa xmm4, kShuffleMaskRGB24ToARGB + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm3, [eax + 32] + lea eax, [eax + 48] + movdqa xmm2, xmm3 + palignr xmm2, xmm1, 8 // xmm2 = { xmm3[0:3] xmm1[8:15]} + pshufb xmm2, xmm4 + por xmm2, xmm5 + palignr xmm1, xmm0, 12 // xmm1 = { xmm3[0:7] xmm0[12:15]} + pshufb xmm0, xmm4 + movdqu [edx + 32], xmm2 + por xmm0, xmm5 + pshufb xmm1, xmm4 + movdqu [edx], xmm0 + por xmm1, xmm5 + palignr xmm3, xmm3, 4 // xmm3 = { xmm3[4:15]} + pshufb xmm3, xmm4 + movdqu [edx + 16], xmm1 + por xmm3, xmm5 + movdqu [edx + 48], xmm3 + lea edx, [edx + 64] + sub ecx, 16 + jg convertloop + ret + } +} + +__declspec(naked) +void RAWToARGBRow_SSSE3(const uint8* src_raw, uint8* dst_argb, + int pix) { + __asm { + mov eax, [esp + 4] // src_raw + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // pix + pcmpeqb xmm5, xmm5 // generate mask 0xff000000 + pslld xmm5, 24 + movdqa xmm4, kShuffleMaskRAWToARGB + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm3, [eax + 32] + lea eax, [eax + 48] + movdqa xmm2, xmm3 + palignr xmm2, xmm1, 8 // xmm2 = { xmm3[0:3] xmm1[8:15]} + pshufb xmm2, xmm4 + por xmm2, xmm5 + palignr xmm1, xmm0, 12 // xmm1 = { xmm3[0:7] xmm0[12:15]} + pshufb xmm0, xmm4 + movdqu [edx + 32], xmm2 + por xmm0, xmm5 + pshufb xmm1, xmm4 + movdqu [edx], xmm0 + por xmm1, xmm5 + palignr xmm3, xmm3, 4 // xmm3 = { xmm3[4:15]} + pshufb xmm3, xmm4 + movdqu [edx + 16], xmm1 + por xmm3, xmm5 + movdqu [edx + 48], xmm3 + lea edx, [edx + 64] + sub ecx, 16 + jg convertloop + ret + } +} + +// pmul method to replicate bits. +// Math to replicate bits: +// (v << 8) | (v << 3) +// v * 256 + v * 8 +// v * (256 + 8) +// G shift of 5 is incorporated, so shift is 5 + 8 and 5 + 3 +// 20 instructions. +__declspec(naked) +void RGB565ToARGBRow_SSE2(const uint8* src_rgb565, uint8* dst_argb, + int pix) { + __asm { + mov eax, 0x01080108 // generate multiplier to repeat 5 bits + movd xmm5, eax + pshufd xmm5, xmm5, 0 + mov eax, 0x20802080 // multiplier shift by 5 and then repeat 6 bits + movd xmm6, eax + pshufd xmm6, xmm6, 0 + pcmpeqb xmm3, xmm3 // generate mask 0xf800f800 for Red + psllw xmm3, 11 + pcmpeqb xmm4, xmm4 // generate mask 0x07e007e0 for Green + psllw xmm4, 10 + psrlw xmm4, 5 + pcmpeqb xmm7, xmm7 // generate mask 0xff00ff00 for Alpha + psllw xmm7, 8 + + mov eax, [esp + 4] // src_rgb565 + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // pix + sub edx, eax + sub edx, eax + + convertloop: + movdqu xmm0, [eax] // fetch 8 pixels of bgr565 + movdqa xmm1, xmm0 + movdqa xmm2, xmm0 + pand xmm1, xmm3 // R in upper 5 bits + psllw xmm2, 11 // B in upper 5 bits + pmulhuw xmm1, xmm5 // * (256 + 8) + pmulhuw xmm2, xmm5 // * (256 + 8) + psllw xmm1, 8 + por xmm1, xmm2 // RB + pand xmm0, xmm4 // G in middle 6 bits + pmulhuw xmm0, xmm6 // << 5 * (256 + 4) + por xmm0, xmm7 // AG + movdqa xmm2, xmm1 + punpcklbw xmm1, xmm0 + punpckhbw xmm2, xmm0 + movdqu [eax * 2 + edx], xmm1 // store 4 pixels of ARGB + movdqu [eax * 2 + edx + 16], xmm2 // store next 4 pixels of ARGB + lea eax, [eax + 16] + sub ecx, 8 + jg convertloop + ret + } +} + +#ifdef HAS_RGB565TOARGBROW_AVX2 +// pmul method to replicate bits. +// Math to replicate bits: +// (v << 8) | (v << 3) +// v * 256 + v * 8 +// v * (256 + 8) +// G shift of 5 is incorporated, so shift is 5 + 8 and 5 + 3 +__declspec(naked) +void RGB565ToARGBRow_AVX2(const uint8* src_rgb565, uint8* dst_argb, + int pix) { + __asm { + mov eax, 0x01080108 // generate multiplier to repeat 5 bits + vmovd xmm5, eax + vbroadcastss ymm5, xmm5 + mov eax, 0x20802080 // multiplier shift by 5 and then repeat 6 bits + movd xmm6, eax + vbroadcastss ymm6, xmm6 + vpcmpeqb ymm3, ymm3, ymm3 // generate mask 0xf800f800 for Red + vpsllw ymm3, ymm3, 11 + vpcmpeqb ymm4, ymm4, ymm4 // generate mask 0x07e007e0 for Green + vpsllw ymm4, ymm4, 10 + vpsrlw ymm4, ymm4, 5 + vpcmpeqb ymm7, ymm7, ymm7 // generate mask 0xff00ff00 for Alpha + vpsllw ymm7, ymm7, 8 + + mov eax, [esp + 4] // src_rgb565 + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // pix + sub edx, eax + sub edx, eax + + convertloop: + vmovdqu ymm0, [eax] // fetch 16 pixels of bgr565 + vpand ymm1, ymm0, ymm3 // R in upper 5 bits + vpsllw ymm2, ymm0, 11 // B in upper 5 bits + vpmulhuw ymm1, ymm1, ymm5 // * (256 + 8) + vpmulhuw ymm2, ymm2, ymm5 // * (256 + 8) + vpsllw ymm1, ymm1, 8 + vpor ymm1, ymm1, ymm2 // RB + vpand ymm0, ymm0, ymm4 // G in middle 6 bits + vpmulhuw ymm0, ymm0, ymm6 // << 5 * (256 + 4) + vpor ymm0, ymm0, ymm7 // AG + vpermq ymm0, ymm0, 0xd8 // mutate for unpack + vpermq ymm1, ymm1, 0xd8 + vpunpckhbw ymm2, ymm1, ymm0 + vpunpcklbw ymm1, ymm1, ymm0 + vmovdqu [eax * 2 + edx], ymm1 // store 4 pixels of ARGB + vmovdqu [eax * 2 + edx + 32], ymm2 // store next 4 pixels of ARGB + lea eax, [eax + 32] + sub ecx, 16 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_RGB565TOARGBROW_AVX2 + +#ifdef HAS_ARGB1555TOARGBROW_AVX2 +__declspec(naked) +void ARGB1555ToARGBRow_AVX2(const uint8* src_argb1555, uint8* dst_argb, + int pix) { + __asm { + mov eax, 0x01080108 // generate multiplier to repeat 5 bits + vmovd xmm5, eax + vbroadcastss ymm5, xmm5 + mov eax, 0x42004200 // multiplier shift by 6 and then repeat 5 bits + movd xmm6, eax + vbroadcastss ymm6, xmm6 + vpcmpeqb ymm3, ymm3, ymm3 // generate mask 0xf800f800 for Red + vpsllw ymm3, ymm3, 11 + vpsrlw ymm4, ymm3, 6 // generate mask 0x03e003e0 for Green + vpcmpeqb ymm7, ymm7, ymm7 // generate mask 0xff00ff00 for Alpha + vpsllw ymm7, ymm7, 8 + + mov eax, [esp + 4] // src_argb1555 + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // pix + sub edx, eax + sub edx, eax + + convertloop: + vmovdqu ymm0, [eax] // fetch 16 pixels of 1555 + vpsllw ymm1, ymm0, 1 // R in upper 5 bits + vpsllw ymm2, ymm0, 11 // B in upper 5 bits + vpand ymm1, ymm1, ymm3 + vpmulhuw ymm2, ymm2, ymm5 // * (256 + 8) + vpmulhuw ymm1, ymm1, ymm5 // * (256 + 8) + vpsllw ymm1, ymm1, 8 + vpor ymm1, ymm1, ymm2 // RB + vpsraw ymm2, ymm0, 8 // A + vpand ymm0, ymm0, ymm4 // G in middle 5 bits + vpmulhuw ymm0, ymm0, ymm6 // << 6 * (256 + 8) + vpand ymm2, ymm2, ymm7 + vpor ymm0, ymm0, ymm2 // AG + vpermq ymm0, ymm0, 0xd8 // mutate for unpack + vpermq ymm1, ymm1, 0xd8 + vpunpckhbw ymm2, ymm1, ymm0 + vpunpcklbw ymm1, ymm1, ymm0 + vmovdqu [eax * 2 + edx], ymm1 // store 8 pixels of ARGB + vmovdqu [eax * 2 + edx + 32], ymm2 // store next 8 pixels of ARGB + lea eax, [eax + 32] + sub ecx, 16 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_ARGB1555TOARGBROW_AVX2 + +#ifdef HAS_ARGB4444TOARGBROW_AVX2 +__declspec(naked) +void ARGB4444ToARGBRow_AVX2(const uint8* src_argb4444, uint8* dst_argb, + int pix) { + __asm { + mov eax, 0x0f0f0f0f // generate mask 0x0f0f0f0f + vmovd xmm4, eax + vbroadcastss ymm4, xmm4 + vpslld ymm5, ymm4, 4 // 0xf0f0f0f0 for high nibbles + mov eax, [esp + 4] // src_argb4444 + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // pix + sub edx, eax + sub edx, eax + + convertloop: + vmovdqu ymm0, [eax] // fetch 16 pixels of bgra4444 + vpand ymm2, ymm0, ymm5 // mask high nibbles + vpand ymm0, ymm0, ymm4 // mask low nibbles + vpsrlw ymm3, ymm2, 4 + vpsllw ymm1, ymm0, 4 + vpor ymm2, ymm2, ymm3 + vpor ymm0, ymm0, ymm1 + vpermq ymm0, ymm0, 0xd8 // mutate for unpack + vpermq ymm2, ymm2, 0xd8 + vpunpckhbw ymm1, ymm0, ymm2 + vpunpcklbw ymm0, ymm0, ymm2 + vmovdqu [eax * 2 + edx], ymm0 // store 8 pixels of ARGB + vmovdqu [eax * 2 + edx + 32], ymm1 // store next 8 pixels of ARGB + lea eax, [eax + 32] + sub ecx, 16 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_ARGB4444TOARGBROW_AVX2 + +// 24 instructions +__declspec(naked) +void ARGB1555ToARGBRow_SSE2(const uint8* src_argb1555, uint8* dst_argb, + int pix) { + __asm { + mov eax, 0x01080108 // generate multiplier to repeat 5 bits + movd xmm5, eax + pshufd xmm5, xmm5, 0 + mov eax, 0x42004200 // multiplier shift by 6 and then repeat 5 bits + movd xmm6, eax + pshufd xmm6, xmm6, 0 + pcmpeqb xmm3, xmm3 // generate mask 0xf800f800 for Red + psllw xmm3, 11 + movdqa xmm4, xmm3 // generate mask 0x03e003e0 for Green + psrlw xmm4, 6 + pcmpeqb xmm7, xmm7 // generate mask 0xff00ff00 for Alpha + psllw xmm7, 8 + + mov eax, [esp + 4] // src_argb1555 + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // pix + sub edx, eax + sub edx, eax + + convertloop: + movdqu xmm0, [eax] // fetch 8 pixels of 1555 + movdqa xmm1, xmm0 + movdqa xmm2, xmm0 + psllw xmm1, 1 // R in upper 5 bits + psllw xmm2, 11 // B in upper 5 bits + pand xmm1, xmm3 + pmulhuw xmm2, xmm5 // * (256 + 8) + pmulhuw xmm1, xmm5 // * (256 + 8) + psllw xmm1, 8 + por xmm1, xmm2 // RB + movdqa xmm2, xmm0 + pand xmm0, xmm4 // G in middle 5 bits + psraw xmm2, 8 // A + pmulhuw xmm0, xmm6 // << 6 * (256 + 8) + pand xmm2, xmm7 + por xmm0, xmm2 // AG + movdqa xmm2, xmm1 + punpcklbw xmm1, xmm0 + punpckhbw xmm2, xmm0 + movdqu [eax * 2 + edx], xmm1 // store 4 pixels of ARGB + movdqu [eax * 2 + edx + 16], xmm2 // store next 4 pixels of ARGB + lea eax, [eax + 16] + sub ecx, 8 + jg convertloop + ret + } +} + +// 18 instructions. +__declspec(naked) +void ARGB4444ToARGBRow_SSE2(const uint8* src_argb4444, uint8* dst_argb, + int pix) { + __asm { + mov eax, 0x0f0f0f0f // generate mask 0x0f0f0f0f + movd xmm4, eax + pshufd xmm4, xmm4, 0 + movdqa xmm5, xmm4 // 0xf0f0f0f0 for high nibbles + pslld xmm5, 4 + mov eax, [esp + 4] // src_argb4444 + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // pix + sub edx, eax + sub edx, eax + + convertloop: + movdqu xmm0, [eax] // fetch 8 pixels of bgra4444 + movdqa xmm2, xmm0 + pand xmm0, xmm4 // mask low nibbles + pand xmm2, xmm5 // mask high nibbles + movdqa xmm1, xmm0 + movdqa xmm3, xmm2 + psllw xmm1, 4 + psrlw xmm3, 4 + por xmm0, xmm1 + por xmm2, xmm3 + movdqa xmm1, xmm0 + punpcklbw xmm0, xmm2 + punpckhbw xmm1, xmm2 + movdqu [eax * 2 + edx], xmm0 // store 4 pixels of ARGB + movdqu [eax * 2 + edx + 16], xmm1 // store next 4 pixels of ARGB + lea eax, [eax + 16] + sub ecx, 8 + jg convertloop + ret + } +} + +__declspec(naked) +void ARGBToRGB24Row_SSSE3(const uint8* src_argb, uint8* dst_rgb, int pix) { + __asm { + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_rgb + mov ecx, [esp + 12] // pix + movdqa xmm6, kShuffleMaskARGBToRGB24 + + convertloop: + movdqu xmm0, [eax] // fetch 16 pixels of argb + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + 32] + movdqu xmm3, [eax + 48] + lea eax, [eax + 64] + pshufb xmm0, xmm6 // pack 16 bytes of ARGB to 12 bytes of RGB + pshufb xmm1, xmm6 + pshufb xmm2, xmm6 + pshufb xmm3, xmm6 + movdqa xmm4, xmm1 // 4 bytes from 1 for 0 + psrldq xmm1, 4 // 8 bytes from 1 + pslldq xmm4, 12 // 4 bytes from 1 for 0 + movdqa xmm5, xmm2 // 8 bytes from 2 for 1 + por xmm0, xmm4 // 4 bytes from 1 for 0 + pslldq xmm5, 8 // 8 bytes from 2 for 1 + movdqu [edx], xmm0 // store 0 + por xmm1, xmm5 // 8 bytes from 2 for 1 + psrldq xmm2, 8 // 4 bytes from 2 + pslldq xmm3, 4 // 12 bytes from 3 for 2 + por xmm2, xmm3 // 12 bytes from 3 for 2 + movdqu [edx + 16], xmm1 // store 1 + movdqu [edx + 32], xmm2 // store 2 + lea edx, [edx + 48] + sub ecx, 16 + jg convertloop + ret + } +} + +__declspec(naked) +void ARGBToRAWRow_SSSE3(const uint8* src_argb, uint8* dst_rgb, int pix) { + __asm { + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_rgb + mov ecx, [esp + 12] // pix + movdqa xmm6, kShuffleMaskARGBToRAW + + convertloop: + movdqu xmm0, [eax] // fetch 16 pixels of argb + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + 32] + movdqu xmm3, [eax + 48] + lea eax, [eax + 64] + pshufb xmm0, xmm6 // pack 16 bytes of ARGB to 12 bytes of RGB + pshufb xmm1, xmm6 + pshufb xmm2, xmm6 + pshufb xmm3, xmm6 + movdqa xmm4, xmm1 // 4 bytes from 1 for 0 + psrldq xmm1, 4 // 8 bytes from 1 + pslldq xmm4, 12 // 4 bytes from 1 for 0 + movdqa xmm5, xmm2 // 8 bytes from 2 for 1 + por xmm0, xmm4 // 4 bytes from 1 for 0 + pslldq xmm5, 8 // 8 bytes from 2 for 1 + movdqu [edx], xmm0 // store 0 + por xmm1, xmm5 // 8 bytes from 2 for 1 + psrldq xmm2, 8 // 4 bytes from 2 + pslldq xmm3, 4 // 12 bytes from 3 for 2 + por xmm2, xmm3 // 12 bytes from 3 for 2 + movdqu [edx + 16], xmm1 // store 1 + movdqu [edx + 32], xmm2 // store 2 + lea edx, [edx + 48] + sub ecx, 16 + jg convertloop + ret + } +} + +// 4 pixels +__declspec(naked) +void ARGBToRGB565Row_SSE2(const uint8* src_argb, uint8* dst_rgb, int pix) { + __asm { + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_rgb + mov ecx, [esp + 12] // pix + pcmpeqb xmm3, xmm3 // generate mask 0x0000001f + psrld xmm3, 27 + pcmpeqb xmm4, xmm4 // generate mask 0x000007e0 + psrld xmm4, 26 + pslld xmm4, 5 + pcmpeqb xmm5, xmm5 // generate mask 0xfffff800 + pslld xmm5, 11 + + convertloop: + movdqu xmm0, [eax] // fetch 4 pixels of argb + movdqa xmm1, xmm0 // B + movdqa xmm2, xmm0 // G + pslld xmm0, 8 // R + psrld xmm1, 3 // B + psrld xmm2, 5 // G + psrad xmm0, 16 // R + pand xmm1, xmm3 // B + pand xmm2, xmm4 // G + pand xmm0, xmm5 // R + por xmm1, xmm2 // BG + por xmm0, xmm1 // BGR + packssdw xmm0, xmm0 + lea eax, [eax + 16] + movq qword ptr [edx], xmm0 // store 4 pixels of RGB565 + lea edx, [edx + 8] + sub ecx, 4 + jg convertloop + ret + } +} + +// 8 pixels +__declspec(naked) +void ARGBToRGB565DitherRow_SSE2(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int pix) { + __asm { + + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_rgb + movd xmm6, [esp + 12] // dither4 + mov ecx, [esp + 16] // pix + punpcklbw xmm6, xmm6 // make dither 16 bytes + movdqa xmm7, xmm6 + punpcklwd xmm6, xmm6 + punpckhwd xmm7, xmm7 + pcmpeqb xmm3, xmm3 // generate mask 0x0000001f + psrld xmm3, 27 + pcmpeqb xmm4, xmm4 // generate mask 0x000007e0 + psrld xmm4, 26 + pslld xmm4, 5 + pcmpeqb xmm5, xmm5 // generate mask 0xfffff800 + pslld xmm5, 11 + + convertloop: + movdqu xmm0, [eax] // fetch 4 pixels of argb + paddusb xmm0, xmm6 // add dither + movdqa xmm1, xmm0 // B + movdqa xmm2, xmm0 // G + pslld xmm0, 8 // R + psrld xmm1, 3 // B + psrld xmm2, 5 // G + psrad xmm0, 16 // R + pand xmm1, xmm3 // B + pand xmm2, xmm4 // G + pand xmm0, xmm5 // R + por xmm1, xmm2 // BG + por xmm0, xmm1 // BGR + packssdw xmm0, xmm0 + lea eax, [eax + 16] + movq qword ptr [edx], xmm0 // store 4 pixels of RGB565 + lea edx, [edx + 8] + sub ecx, 4 + jg convertloop + ret + } +} + +#ifdef HAS_ARGBTORGB565DITHERROW_AVX2 +__declspec(naked) +void ARGBToRGB565DitherRow_AVX2(const uint8* src_argb, uint8* dst_rgb, + const uint32 dither4, int pix) { + __asm { + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_rgb + vbroadcastss xmm6, [esp + 12] // dither4 + mov ecx, [esp + 16] // pix + vpunpcklbw xmm6, xmm6, xmm6 // make dither 32 bytes + vpermq ymm6, ymm6, 0xd8 + vpunpcklwd ymm6, ymm6, ymm6 + vpcmpeqb ymm3, ymm3, ymm3 // generate mask 0x0000001f + vpsrld ymm3, ymm3, 27 + vpcmpeqb ymm4, ymm4, ymm4 // generate mask 0x000007e0 + vpsrld ymm4, ymm4, 26 + vpslld ymm4, ymm4, 5 + vpslld ymm5, ymm3, 11 // generate mask 0x0000f800 + + convertloop: + vmovdqu ymm0, [eax] // fetch 8 pixels of argb + vpaddusb ymm0, ymm0, ymm6 // add dither + vpsrld ymm2, ymm0, 5 // G + vpsrld ymm1, ymm0, 3 // B + vpsrld ymm0, ymm0, 8 // R + vpand ymm2, ymm2, ymm4 // G + vpand ymm1, ymm1, ymm3 // B + vpand ymm0, ymm0, ymm5 // R + vpor ymm1, ymm1, ymm2 // BG + vpor ymm0, ymm0, ymm1 // BGR + vpackusdw ymm0, ymm0, ymm0 + vpermq ymm0, ymm0, 0xd8 + lea eax, [eax + 32] + vmovdqu [edx], xmm0 // store 8 pixels of RGB565 + lea edx, [edx + 16] + sub ecx, 8 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_ARGBTORGB565DITHERROW_AVX2 + +// TODO(fbarchard): Improve sign extension/packing. +__declspec(naked) +void ARGBToARGB1555Row_SSE2(const uint8* src_argb, uint8* dst_rgb, int pix) { + __asm { + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_rgb + mov ecx, [esp + 12] // pix + pcmpeqb xmm4, xmm4 // generate mask 0x0000001f + psrld xmm4, 27 + movdqa xmm5, xmm4 // generate mask 0x000003e0 + pslld xmm5, 5 + movdqa xmm6, xmm4 // generate mask 0x00007c00 + pslld xmm6, 10 + pcmpeqb xmm7, xmm7 // generate mask 0xffff8000 + pslld xmm7, 15 + + convertloop: + movdqu xmm0, [eax] // fetch 4 pixels of argb + movdqa xmm1, xmm0 // B + movdqa xmm2, xmm0 // G + movdqa xmm3, xmm0 // R + psrad xmm0, 16 // A + psrld xmm1, 3 // B + psrld xmm2, 6 // G + psrld xmm3, 9 // R + pand xmm0, xmm7 // A + pand xmm1, xmm4 // B + pand xmm2, xmm5 // G + pand xmm3, xmm6 // R + por xmm0, xmm1 // BA + por xmm2, xmm3 // GR + por xmm0, xmm2 // BGRA + packssdw xmm0, xmm0 + lea eax, [eax + 16] + movq qword ptr [edx], xmm0 // store 4 pixels of ARGB1555 + lea edx, [edx + 8] + sub ecx, 4 + jg convertloop + ret + } +} + +__declspec(naked) +void ARGBToARGB4444Row_SSE2(const uint8* src_argb, uint8* dst_rgb, int pix) { + __asm { + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_rgb + mov ecx, [esp + 12] // pix + pcmpeqb xmm4, xmm4 // generate mask 0xf000f000 + psllw xmm4, 12 + movdqa xmm3, xmm4 // generate mask 0x00f000f0 + psrlw xmm3, 8 + + convertloop: + movdqu xmm0, [eax] // fetch 4 pixels of argb + movdqa xmm1, xmm0 + pand xmm0, xmm3 // low nibble + pand xmm1, xmm4 // high nibble + psrld xmm0, 4 + psrld xmm1, 8 + por xmm0, xmm1 + packuswb xmm0, xmm0 + lea eax, [eax + 16] + movq qword ptr [edx], xmm0 // store 4 pixels of ARGB4444 + lea edx, [edx + 8] + sub ecx, 4 + jg convertloop + ret + } +} + +#ifdef HAS_ARGBTORGB565ROW_AVX2 +__declspec(naked) +void ARGBToRGB565Row_AVX2(const uint8* src_argb, uint8* dst_rgb, int pix) { + __asm { + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_rgb + mov ecx, [esp + 12] // pix + vpcmpeqb ymm3, ymm3, ymm3 // generate mask 0x0000001f + vpsrld ymm3, ymm3, 27 + vpcmpeqb ymm4, ymm4, ymm4 // generate mask 0x000007e0 + vpsrld ymm4, ymm4, 26 + vpslld ymm4, ymm4, 5 + vpslld ymm5, ymm3, 11 // generate mask 0x0000f800 + + convertloop: + vmovdqu ymm0, [eax] // fetch 8 pixels of argb + vpsrld ymm2, ymm0, 5 // G + vpsrld ymm1, ymm0, 3 // B + vpsrld ymm0, ymm0, 8 // R + vpand ymm2, ymm2, ymm4 // G + vpand ymm1, ymm1, ymm3 // B + vpand ymm0, ymm0, ymm5 // R + vpor ymm1, ymm1, ymm2 // BG + vpor ymm0, ymm0, ymm1 // BGR + vpackusdw ymm0, ymm0, ymm0 + vpermq ymm0, ymm0, 0xd8 + lea eax, [eax + 32] + vmovdqu [edx], xmm0 // store 8 pixels of RGB565 + lea edx, [edx + 16] + sub ecx, 8 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_ARGBTORGB565ROW_AVX2 + +#ifdef HAS_ARGBTOARGB1555ROW_AVX2 +__declspec(naked) +void ARGBToARGB1555Row_AVX2(const uint8* src_argb, uint8* dst_rgb, int pix) { + __asm { + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_rgb + mov ecx, [esp + 12] // pix + vpcmpeqb ymm4, ymm4, ymm4 + vpsrld ymm4, ymm4, 27 // generate mask 0x0000001f + vpslld ymm5, ymm4, 5 // generate mask 0x000003e0 + vpslld ymm6, ymm4, 10 // generate mask 0x00007c00 + vpcmpeqb ymm7, ymm7, ymm7 // generate mask 0xffff8000 + vpslld ymm7, ymm7, 15 + + convertloop: + vmovdqu ymm0, [eax] // fetch 8 pixels of argb + vpsrld ymm3, ymm0, 9 // R + vpsrld ymm2, ymm0, 6 // G + vpsrld ymm1, ymm0, 3 // B + vpsrad ymm0, ymm0, 16 // A + vpand ymm3, ymm3, ymm6 // R + vpand ymm2, ymm2, ymm5 // G + vpand ymm1, ymm1, ymm4 // B + vpand ymm0, ymm0, ymm7 // A + vpor ymm0, ymm0, ymm1 // BA + vpor ymm2, ymm2, ymm3 // GR + vpor ymm0, ymm0, ymm2 // BGRA + vpackssdw ymm0, ymm0, ymm0 + vpermq ymm0, ymm0, 0xd8 + lea eax, [eax + 32] + vmovdqu [edx], xmm0 // store 8 pixels of ARGB1555 + lea edx, [edx + 16] + sub ecx, 8 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_ARGBTOARGB1555ROW_AVX2 + +#ifdef HAS_ARGBTOARGB4444ROW_AVX2 +__declspec(naked) +void ARGBToARGB4444Row_AVX2(const uint8* src_argb, uint8* dst_rgb, int pix) { + __asm { + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_rgb + mov ecx, [esp + 12] // pix + vpcmpeqb ymm4, ymm4, ymm4 // generate mask 0xf000f000 + vpsllw ymm4, ymm4, 12 + vpsrlw ymm3, ymm4, 8 // generate mask 0x00f000f0 + + convertloop: + vmovdqu ymm0, [eax] // fetch 8 pixels of argb + vpand ymm1, ymm0, ymm4 // high nibble + vpand ymm0, ymm0, ymm3 // low nibble + vpsrld ymm1, ymm1, 8 + vpsrld ymm0, ymm0, 4 + vpor ymm0, ymm0, ymm1 + vpackuswb ymm0, ymm0, ymm0 + vpermq ymm0, ymm0, 0xd8 + lea eax, [eax + 32] + vmovdqu [edx], xmm0 // store 8 pixels of ARGB4444 + lea edx, [edx + 16] + sub ecx, 8 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_ARGBTOARGB4444ROW_AVX2 + +// Convert 16 ARGB pixels (64 bytes) to 16 Y values. +__declspec(naked) +void ARGBToYRow_SSSE3(const uint8* src_argb, uint8* dst_y, int pix) { + __asm { + mov eax, [esp + 4] /* src_argb */ + mov edx, [esp + 8] /* dst_y */ + mov ecx, [esp + 12] /* pix */ + movdqa xmm4, kARGBToY + movdqa xmm5, kAddY16 + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + 32] + movdqu xmm3, [eax + 48] + pmaddubsw xmm0, xmm4 + pmaddubsw xmm1, xmm4 + pmaddubsw xmm2, xmm4 + pmaddubsw xmm3, xmm4 + lea eax, [eax + 64] + phaddw xmm0, xmm1 + phaddw xmm2, xmm3 + psrlw xmm0, 7 + psrlw xmm2, 7 + packuswb xmm0, xmm2 + paddb xmm0, xmm5 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg convertloop + ret + } +} + +// Convert 16 ARGB pixels (64 bytes) to 16 YJ values. +// Same as ARGBToYRow but different coefficients, no add 16, but do rounding. +__declspec(naked) +void ARGBToYJRow_SSSE3(const uint8* src_argb, uint8* dst_y, int pix) { + __asm { + mov eax, [esp + 4] /* src_argb */ + mov edx, [esp + 8] /* dst_y */ + mov ecx, [esp + 12] /* pix */ + movdqa xmm4, kARGBToYJ + movdqa xmm5, kAddYJ64 + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + 32] + movdqu xmm3, [eax + 48] + pmaddubsw xmm0, xmm4 + pmaddubsw xmm1, xmm4 + pmaddubsw xmm2, xmm4 + pmaddubsw xmm3, xmm4 + lea eax, [eax + 64] + phaddw xmm0, xmm1 + phaddw xmm2, xmm3 + paddw xmm0, xmm5 // Add .5 for rounding. + paddw xmm2, xmm5 + psrlw xmm0, 7 + psrlw xmm2, 7 + packuswb xmm0, xmm2 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg convertloop + ret + } +} + +#ifdef HAS_ARGBTOYROW_AVX2 +// vpermd for vphaddw + vpackuswb vpermd. +static const lvec32 kPermdARGBToY_AVX = { + 0, 4, 1, 5, 2, 6, 3, 7 +}; + +// Convert 32 ARGB pixels (128 bytes) to 32 Y values. +__declspec(naked) +void ARGBToYRow_AVX2(const uint8* src_argb, uint8* dst_y, int pix) { + __asm { + mov eax, [esp + 4] /* src_argb */ + mov edx, [esp + 8] /* dst_y */ + mov ecx, [esp + 12] /* pix */ + vbroadcastf128 ymm4, kARGBToY + vbroadcastf128 ymm5, kAddY16 + vmovdqu ymm6, kPermdARGBToY_AVX + + convertloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + vmovdqu ymm2, [eax + 64] + vmovdqu ymm3, [eax + 96] + vpmaddubsw ymm0, ymm0, ymm4 + vpmaddubsw ymm1, ymm1, ymm4 + vpmaddubsw ymm2, ymm2, ymm4 + vpmaddubsw ymm3, ymm3, ymm4 + lea eax, [eax + 128] + vphaddw ymm0, ymm0, ymm1 // mutates. + vphaddw ymm2, ymm2, ymm3 + vpsrlw ymm0, ymm0, 7 + vpsrlw ymm2, ymm2, 7 + vpackuswb ymm0, ymm0, ymm2 // mutates. + vpermd ymm0, ymm6, ymm0 // For vphaddw + vpackuswb mutation. + vpaddb ymm0, ymm0, ymm5 // add 16 for Y + vmovdqu [edx], ymm0 + lea edx, [edx + 32] + sub ecx, 32 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_ARGBTOYROW_AVX2 + +#ifdef HAS_ARGBTOYJROW_AVX2 +// Convert 32 ARGB pixels (128 bytes) to 32 Y values. +__declspec(naked) +void ARGBToYJRow_AVX2(const uint8* src_argb, uint8* dst_y, int pix) { + __asm { + mov eax, [esp + 4] /* src_argb */ + mov edx, [esp + 8] /* dst_y */ + mov ecx, [esp + 12] /* pix */ + vbroadcastf128 ymm4, kARGBToYJ + vbroadcastf128 ymm5, kAddYJ64 + vmovdqu ymm6, kPermdARGBToY_AVX + + convertloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + vmovdqu ymm2, [eax + 64] + vmovdqu ymm3, [eax + 96] + vpmaddubsw ymm0, ymm0, ymm4 + vpmaddubsw ymm1, ymm1, ymm4 + vpmaddubsw ymm2, ymm2, ymm4 + vpmaddubsw ymm3, ymm3, ymm4 + lea eax, [eax + 128] + vphaddw ymm0, ymm0, ymm1 // mutates. + vphaddw ymm2, ymm2, ymm3 + vpaddw ymm0, ymm0, ymm5 // Add .5 for rounding. + vpaddw ymm2, ymm2, ymm5 + vpsrlw ymm0, ymm0, 7 + vpsrlw ymm2, ymm2, 7 + vpackuswb ymm0, ymm0, ymm2 // mutates. + vpermd ymm0, ymm6, ymm0 // For vphaddw + vpackuswb mutation. + vmovdqu [edx], ymm0 + lea edx, [edx + 32] + sub ecx, 32 + jg convertloop + + vzeroupper + ret + } +} +#endif // HAS_ARGBTOYJROW_AVX2 + +__declspec(naked) +void BGRAToYRow_SSSE3(const uint8* src_argb, uint8* dst_y, int pix) { + __asm { + mov eax, [esp + 4] /* src_argb */ + mov edx, [esp + 8] /* dst_y */ + mov ecx, [esp + 12] /* pix */ + movdqa xmm4, kBGRAToY + movdqa xmm5, kAddY16 + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + 32] + movdqu xmm3, [eax + 48] + pmaddubsw xmm0, xmm4 + pmaddubsw xmm1, xmm4 + pmaddubsw xmm2, xmm4 + pmaddubsw xmm3, xmm4 + lea eax, [eax + 64] + phaddw xmm0, xmm1 + phaddw xmm2, xmm3 + psrlw xmm0, 7 + psrlw xmm2, 7 + packuswb xmm0, xmm2 + paddb xmm0, xmm5 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg convertloop + ret + } +} + +__declspec(naked) +void ABGRToYRow_SSSE3(const uint8* src_argb, uint8* dst_y, int pix) { + __asm { + mov eax, [esp + 4] /* src_argb */ + mov edx, [esp + 8] /* dst_y */ + mov ecx, [esp + 12] /* pix */ + movdqa xmm4, kABGRToY + movdqa xmm5, kAddY16 + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + 32] + movdqu xmm3, [eax + 48] + pmaddubsw xmm0, xmm4 + pmaddubsw xmm1, xmm4 + pmaddubsw xmm2, xmm4 + pmaddubsw xmm3, xmm4 + lea eax, [eax + 64] + phaddw xmm0, xmm1 + phaddw xmm2, xmm3 + psrlw xmm0, 7 + psrlw xmm2, 7 + packuswb xmm0, xmm2 + paddb xmm0, xmm5 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg convertloop + ret + } +} + +__declspec(naked) +void RGBAToYRow_SSSE3(const uint8* src_argb, uint8* dst_y, int pix) { + __asm { + mov eax, [esp + 4] /* src_argb */ + mov edx, [esp + 8] /* dst_y */ + mov ecx, [esp + 12] /* pix */ + movdqa xmm4, kRGBAToY + movdqa xmm5, kAddY16 + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + 32] + movdqu xmm3, [eax + 48] + pmaddubsw xmm0, xmm4 + pmaddubsw xmm1, xmm4 + pmaddubsw xmm2, xmm4 + pmaddubsw xmm3, xmm4 + lea eax, [eax + 64] + phaddw xmm0, xmm1 + phaddw xmm2, xmm3 + psrlw xmm0, 7 + psrlw xmm2, 7 + packuswb xmm0, xmm2 + paddb xmm0, xmm5 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg convertloop + ret + } +} + +__declspec(naked) +void ARGBToUVRow_SSSE3(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_argb + mov esi, [esp + 8 + 8] // src_stride_argb + mov edx, [esp + 8 + 12] // dst_u + mov edi, [esp + 8 + 16] // dst_v + mov ecx, [esp + 8 + 20] // pix + movdqa xmm5, kAddUV128 + movdqa xmm6, kARGBToV + movdqa xmm7, kARGBToU + sub edi, edx // stride from u to v + + convertloop: + /* step 1 - subsample 16x2 argb pixels to 8x1 */ + movdqu xmm0, [eax] + movdqu xmm4, [eax + esi] + pavgb xmm0, xmm4 + movdqu xmm1, [eax + 16] + movdqu xmm4, [eax + esi + 16] + pavgb xmm1, xmm4 + movdqu xmm2, [eax + 32] + movdqu xmm4, [eax + esi + 32] + pavgb xmm2, xmm4 + movdqu xmm3, [eax + 48] + movdqu xmm4, [eax + esi + 48] + pavgb xmm3, xmm4 + + lea eax, [eax + 64] + movdqa xmm4, xmm0 + shufps xmm0, xmm1, 0x88 + shufps xmm4, xmm1, 0xdd + pavgb xmm0, xmm4 + movdqa xmm4, xmm2 + shufps xmm2, xmm3, 0x88 + shufps xmm4, xmm3, 0xdd + pavgb xmm2, xmm4 + + // step 2 - convert to U and V + // from here down is very similar to Y code except + // instead of 16 different pixels, its 8 pixels of U and 8 of V + movdqa xmm1, xmm0 + movdqa xmm3, xmm2 + pmaddubsw xmm0, xmm7 // U + pmaddubsw xmm2, xmm7 + pmaddubsw xmm1, xmm6 // V + pmaddubsw xmm3, xmm6 + phaddw xmm0, xmm2 + phaddw xmm1, xmm3 + psraw xmm0, 8 + psraw xmm1, 8 + packsswb xmm0, xmm1 + paddb xmm0, xmm5 // -> unsigned + + // step 3 - store 8 U and 8 V values + movlps qword ptr [edx], xmm0 // U + movhps qword ptr [edx + edi], xmm0 // V + lea edx, [edx + 8] + sub ecx, 16 + jg convertloop + + pop edi + pop esi + ret + } +} + +__declspec(naked) +void ARGBToUVJRow_SSSE3(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_argb + mov esi, [esp + 8 + 8] // src_stride_argb + mov edx, [esp + 8 + 12] // dst_u + mov edi, [esp + 8 + 16] // dst_v + mov ecx, [esp + 8 + 20] // pix + movdqa xmm5, kAddUVJ128 + movdqa xmm6, kARGBToVJ + movdqa xmm7, kARGBToUJ + sub edi, edx // stride from u to v + + convertloop: + /* step 1 - subsample 16x2 argb pixels to 8x1 */ + movdqu xmm0, [eax] + movdqu xmm4, [eax + esi] + pavgb xmm0, xmm4 + movdqu xmm1, [eax + 16] + movdqu xmm4, [eax + esi + 16] + pavgb xmm1, xmm4 + movdqu xmm2, [eax + 32] + movdqu xmm4, [eax + esi + 32] + pavgb xmm2, xmm4 + movdqu xmm3, [eax + 48] + movdqu xmm4, [eax + esi + 48] + pavgb xmm3, xmm4 + + lea eax, [eax + 64] + movdqa xmm4, xmm0 + shufps xmm0, xmm1, 0x88 + shufps xmm4, xmm1, 0xdd + pavgb xmm0, xmm4 + movdqa xmm4, xmm2 + shufps xmm2, xmm3, 0x88 + shufps xmm4, xmm3, 0xdd + pavgb xmm2, xmm4 + + // step 2 - convert to U and V + // from here down is very similar to Y code except + // instead of 16 different pixels, its 8 pixels of U and 8 of V + movdqa xmm1, xmm0 + movdqa xmm3, xmm2 + pmaddubsw xmm0, xmm7 // U + pmaddubsw xmm2, xmm7 + pmaddubsw xmm1, xmm6 // V + pmaddubsw xmm3, xmm6 + phaddw xmm0, xmm2 + phaddw xmm1, xmm3 + paddw xmm0, xmm5 // +.5 rounding -> unsigned + paddw xmm1, xmm5 + psraw xmm0, 8 + psraw xmm1, 8 + packsswb xmm0, xmm1 + + // step 3 - store 8 U and 8 V values + movlps qword ptr [edx], xmm0 // U + movhps qword ptr [edx + edi], xmm0 // V + lea edx, [edx + 8] + sub ecx, 16 + jg convertloop + + pop edi + pop esi + ret + } +} + +#ifdef HAS_ARGBTOUVROW_AVX2 +__declspec(naked) +void ARGBToUVRow_AVX2(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_argb + mov esi, [esp + 8 + 8] // src_stride_argb + mov edx, [esp + 8 + 12] // dst_u + mov edi, [esp + 8 + 16] // dst_v + mov ecx, [esp + 8 + 20] // pix + vbroadcastf128 ymm5, kAddUV128 + vbroadcastf128 ymm6, kARGBToV + vbroadcastf128 ymm7, kARGBToU + sub edi, edx // stride from u to v + + convertloop: + /* step 1 - subsample 32x2 argb pixels to 16x1 */ + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + vmovdqu ymm2, [eax + 64] + vmovdqu ymm3, [eax + 96] + vpavgb ymm0, ymm0, [eax + esi] + vpavgb ymm1, ymm1, [eax + esi + 32] + vpavgb ymm2, ymm2, [eax + esi + 64] + vpavgb ymm3, ymm3, [eax + esi + 96] + lea eax, [eax + 128] + vshufps ymm4, ymm0, ymm1, 0x88 + vshufps ymm0, ymm0, ymm1, 0xdd + vpavgb ymm0, ymm0, ymm4 // mutated by vshufps + vshufps ymm4, ymm2, ymm3, 0x88 + vshufps ymm2, ymm2, ymm3, 0xdd + vpavgb ymm2, ymm2, ymm4 // mutated by vshufps + + // step 2 - convert to U and V + // from here down is very similar to Y code except + // instead of 32 different pixels, its 16 pixels of U and 16 of V + vpmaddubsw ymm1, ymm0, ymm7 // U + vpmaddubsw ymm3, ymm2, ymm7 + vpmaddubsw ymm0, ymm0, ymm6 // V + vpmaddubsw ymm2, ymm2, ymm6 + vphaddw ymm1, ymm1, ymm3 // mutates + vphaddw ymm0, ymm0, ymm2 + vpsraw ymm1, ymm1, 8 + vpsraw ymm0, ymm0, 8 + vpacksswb ymm0, ymm1, ymm0 // mutates + vpermq ymm0, ymm0, 0xd8 // For vpacksswb + vpshufb ymm0, ymm0, kShufARGBToUV_AVX // For vshufps + vphaddw + vpaddb ymm0, ymm0, ymm5 // -> unsigned + + // step 3 - store 16 U and 16 V values + vextractf128 [edx], ymm0, 0 // U + vextractf128 [edx + edi], ymm0, 1 // V + lea edx, [edx + 16] + sub ecx, 32 + jg convertloop + + pop edi + pop esi + vzeroupper + ret + } +} +#endif // HAS_ARGBTOUVROW_AVX2 + +__declspec(naked) +void ARGBToUV444Row_SSSE3(const uint8* src_argb0, + uint8* dst_u, uint8* dst_v, int width) { + __asm { + push edi + mov eax, [esp + 4 + 4] // src_argb + mov edx, [esp + 4 + 8] // dst_u + mov edi, [esp + 4 + 12] // dst_v + mov ecx, [esp + 4 + 16] // pix + movdqa xmm5, kAddUV128 + movdqa xmm6, kARGBToV + movdqa xmm7, kARGBToU + sub edi, edx // stride from u to v + + convertloop: + /* convert to U and V */ + movdqu xmm0, [eax] // U + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + 32] + movdqu xmm3, [eax + 48] + pmaddubsw xmm0, xmm7 + pmaddubsw xmm1, xmm7 + pmaddubsw xmm2, xmm7 + pmaddubsw xmm3, xmm7 + phaddw xmm0, xmm1 + phaddw xmm2, xmm3 + psraw xmm0, 8 + psraw xmm2, 8 + packsswb xmm0, xmm2 + paddb xmm0, xmm5 + movdqu [edx], xmm0 + + movdqu xmm0, [eax] // V + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + 32] + movdqu xmm3, [eax + 48] + pmaddubsw xmm0, xmm6 + pmaddubsw xmm1, xmm6 + pmaddubsw xmm2, xmm6 + pmaddubsw xmm3, xmm6 + phaddw xmm0, xmm1 + phaddw xmm2, xmm3 + psraw xmm0, 8 + psraw xmm2, 8 + packsswb xmm0, xmm2 + paddb xmm0, xmm5 + lea eax, [eax + 64] + movdqu [edx + edi], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg convertloop + + pop edi + ret + } +} + +__declspec(naked) +void ARGBToUV422Row_SSSE3(const uint8* src_argb0, + uint8* dst_u, uint8* dst_v, int width) { + __asm { + push edi + mov eax, [esp + 4 + 4] // src_argb + mov edx, [esp + 4 + 8] // dst_u + mov edi, [esp + 4 + 12] // dst_v + mov ecx, [esp + 4 + 16] // pix + movdqa xmm5, kAddUV128 + movdqa xmm6, kARGBToV + movdqa xmm7, kARGBToU + sub edi, edx // stride from u to v + + convertloop: + /* step 1 - subsample 16x2 argb pixels to 8x1 */ + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + 32] + movdqu xmm3, [eax + 48] + lea eax, [eax + 64] + movdqa xmm4, xmm0 + shufps xmm0, xmm1, 0x88 + shufps xmm4, xmm1, 0xdd + pavgb xmm0, xmm4 + movdqa xmm4, xmm2 + shufps xmm2, xmm3, 0x88 + shufps xmm4, xmm3, 0xdd + pavgb xmm2, xmm4 + + // step 2 - convert to U and V + // from here down is very similar to Y code except + // instead of 16 different pixels, its 8 pixels of U and 8 of V + movdqa xmm1, xmm0 + movdqa xmm3, xmm2 + pmaddubsw xmm0, xmm7 // U + pmaddubsw xmm2, xmm7 + pmaddubsw xmm1, xmm6 // V + pmaddubsw xmm3, xmm6 + phaddw xmm0, xmm2 + phaddw xmm1, xmm3 + psraw xmm0, 8 + psraw xmm1, 8 + packsswb xmm0, xmm1 + paddb xmm0, xmm5 // -> unsigned + + // step 3 - store 8 U and 8 V values + movlps qword ptr [edx], xmm0 // U + movhps qword ptr [edx + edi], xmm0 // V + lea edx, [edx + 8] + sub ecx, 16 + jg convertloop + + pop edi + ret + } +} + +__declspec(naked) +void BGRAToUVRow_SSSE3(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_argb + mov esi, [esp + 8 + 8] // src_stride_argb + mov edx, [esp + 8 + 12] // dst_u + mov edi, [esp + 8 + 16] // dst_v + mov ecx, [esp + 8 + 20] // pix + movdqa xmm5, kAddUV128 + movdqa xmm6, kBGRAToV + movdqa xmm7, kBGRAToU + sub edi, edx // stride from u to v + + convertloop: + /* step 1 - subsample 16x2 argb pixels to 8x1 */ + movdqu xmm0, [eax] + movdqu xmm4, [eax + esi] + pavgb xmm0, xmm4 + movdqu xmm1, [eax + 16] + movdqu xmm4, [eax + esi + 16] + pavgb xmm1, xmm4 + movdqu xmm2, [eax + 32] + movdqu xmm4, [eax + esi + 32] + pavgb xmm2, xmm4 + movdqu xmm3, [eax + 48] + movdqu xmm4, [eax + esi + 48] + pavgb xmm3, xmm4 + + lea eax, [eax + 64] + movdqa xmm4, xmm0 + shufps xmm0, xmm1, 0x88 + shufps xmm4, xmm1, 0xdd + pavgb xmm0, xmm4 + movdqa xmm4, xmm2 + shufps xmm2, xmm3, 0x88 + shufps xmm4, xmm3, 0xdd + pavgb xmm2, xmm4 + + // step 2 - convert to U and V + // from here down is very similar to Y code except + // instead of 16 different pixels, its 8 pixels of U and 8 of V + movdqa xmm1, xmm0 + movdqa xmm3, xmm2 + pmaddubsw xmm0, xmm7 // U + pmaddubsw xmm2, xmm7 + pmaddubsw xmm1, xmm6 // V + pmaddubsw xmm3, xmm6 + phaddw xmm0, xmm2 + phaddw xmm1, xmm3 + psraw xmm0, 8 + psraw xmm1, 8 + packsswb xmm0, xmm1 + paddb xmm0, xmm5 // -> unsigned + + // step 3 - store 8 U and 8 V values + movlps qword ptr [edx], xmm0 // U + movhps qword ptr [edx + edi], xmm0 // V + lea edx, [edx + 8] + sub ecx, 16 + jg convertloop + + pop edi + pop esi + ret + } +} + +__declspec(naked) +void ABGRToUVRow_SSSE3(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_argb + mov esi, [esp + 8 + 8] // src_stride_argb + mov edx, [esp + 8 + 12] // dst_u + mov edi, [esp + 8 + 16] // dst_v + mov ecx, [esp + 8 + 20] // pix + movdqa xmm5, kAddUV128 + movdqa xmm6, kABGRToV + movdqa xmm7, kABGRToU + sub edi, edx // stride from u to v + + convertloop: + /* step 1 - subsample 16x2 argb pixels to 8x1 */ + movdqu xmm0, [eax] + movdqu xmm4, [eax + esi] + pavgb xmm0, xmm4 + movdqu xmm1, [eax + 16] + movdqu xmm4, [eax + esi + 16] + pavgb xmm1, xmm4 + movdqu xmm2, [eax + 32] + movdqu xmm4, [eax + esi + 32] + pavgb xmm2, xmm4 + movdqu xmm3, [eax + 48] + movdqu xmm4, [eax + esi + 48] + pavgb xmm3, xmm4 + + lea eax, [eax + 64] + movdqa xmm4, xmm0 + shufps xmm0, xmm1, 0x88 + shufps xmm4, xmm1, 0xdd + pavgb xmm0, xmm4 + movdqa xmm4, xmm2 + shufps xmm2, xmm3, 0x88 + shufps xmm4, xmm3, 0xdd + pavgb xmm2, xmm4 + + // step 2 - convert to U and V + // from here down is very similar to Y code except + // instead of 16 different pixels, its 8 pixels of U and 8 of V + movdqa xmm1, xmm0 + movdqa xmm3, xmm2 + pmaddubsw xmm0, xmm7 // U + pmaddubsw xmm2, xmm7 + pmaddubsw xmm1, xmm6 // V + pmaddubsw xmm3, xmm6 + phaddw xmm0, xmm2 + phaddw xmm1, xmm3 + psraw xmm0, 8 + psraw xmm1, 8 + packsswb xmm0, xmm1 + paddb xmm0, xmm5 // -> unsigned + + // step 3 - store 8 U and 8 V values + movlps qword ptr [edx], xmm0 // U + movhps qword ptr [edx + edi], xmm0 // V + lea edx, [edx + 8] + sub ecx, 16 + jg convertloop + + pop edi + pop esi + ret + } +} + +__declspec(naked) +void RGBAToUVRow_SSSE3(const uint8* src_argb0, int src_stride_argb, + uint8* dst_u, uint8* dst_v, int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_argb + mov esi, [esp + 8 + 8] // src_stride_argb + mov edx, [esp + 8 + 12] // dst_u + mov edi, [esp + 8 + 16] // dst_v + mov ecx, [esp + 8 + 20] // pix + movdqa xmm5, kAddUV128 + movdqa xmm6, kRGBAToV + movdqa xmm7, kRGBAToU + sub edi, edx // stride from u to v + + convertloop: + /* step 1 - subsample 16x2 argb pixels to 8x1 */ + movdqu xmm0, [eax] + movdqu xmm4, [eax + esi] + pavgb xmm0, xmm4 + movdqu xmm1, [eax + 16] + movdqu xmm4, [eax + esi + 16] + pavgb xmm1, xmm4 + movdqu xmm2, [eax + 32] + movdqu xmm4, [eax + esi + 32] + pavgb xmm2, xmm4 + movdqu xmm3, [eax + 48] + movdqu xmm4, [eax + esi + 48] + pavgb xmm3, xmm4 + + lea eax, [eax + 64] + movdqa xmm4, xmm0 + shufps xmm0, xmm1, 0x88 + shufps xmm4, xmm1, 0xdd + pavgb xmm0, xmm4 + movdqa xmm4, xmm2 + shufps xmm2, xmm3, 0x88 + shufps xmm4, xmm3, 0xdd + pavgb xmm2, xmm4 + + // step 2 - convert to U and V + // from here down is very similar to Y code except + // instead of 16 different pixels, its 8 pixels of U and 8 of V + movdqa xmm1, xmm0 + movdqa xmm3, xmm2 + pmaddubsw xmm0, xmm7 // U + pmaddubsw xmm2, xmm7 + pmaddubsw xmm1, xmm6 // V + pmaddubsw xmm3, xmm6 + phaddw xmm0, xmm2 + phaddw xmm1, xmm3 + psraw xmm0, 8 + psraw xmm1, 8 + packsswb xmm0, xmm1 + paddb xmm0, xmm5 // -> unsigned + + // step 3 - store 8 U and 8 V values + movlps qword ptr [edx], xmm0 // U + movhps qword ptr [edx + edi], xmm0 // V + lea edx, [edx + 8] + sub ecx, 16 + jg convertloop + + pop edi + pop esi + ret + } +} +#endif // HAS_ARGBTOYROW_SSSE3 + +// Read 16 UV from 444 +#define READYUV444_AVX2 __asm { \ + __asm vmovdqu xmm0, [esi] /* U */ /* NOLINT */ \ + __asm vmovdqu xmm1, [esi + edi] /* V */ /* NOLINT */ \ + __asm lea esi, [esi + 16] \ + __asm vpermq ymm0, ymm0, 0xd8 \ + __asm vpermq ymm1, ymm1, 0xd8 \ + __asm vpunpcklbw ymm0, ymm0, ymm1 /* UV */ \ + } + +// Read 8 UV from 422, upsample to 16 UV. +#define READYUV422_AVX2 __asm { \ + __asm vmovq xmm0, qword ptr [esi] /* U */ /* NOLINT */ \ + __asm vmovq xmm1, qword ptr [esi + edi] /* V */ /* NOLINT */ \ + __asm lea esi, [esi + 8] \ + __asm vpunpcklbw ymm0, ymm0, ymm1 /* UV */ \ + __asm vpermq ymm0, ymm0, 0xd8 \ + __asm vpunpcklwd ymm0, ymm0, ymm0 /* UVUV (upsample) */ \ + } + +// Read 4 UV from 411, upsample to 16 UV. +#define READYUV411_AVX2 __asm { \ + __asm vmovd xmm0, dword ptr [esi] /* U */ /* NOLINT */ \ + __asm vmovd xmm1, dword ptr [esi + edi] /* V */ /* NOLINT */ \ + __asm lea esi, [esi + 4] \ + __asm vpunpcklbw ymm0, ymm0, ymm1 /* UV */ \ + __asm vpunpcklwd ymm0, ymm0, ymm0 /* UVUV (upsample) */ \ + __asm vpermq ymm0, ymm0, 0xd8 \ + __asm vpunpckldq ymm0, ymm0, ymm0 /* UVUVUVUV (upsample) */ \ + } + +// Read 8 UV from NV12, upsample to 16 UV. +#define READNV12_AVX2 __asm { \ + __asm vmovdqu xmm0, [esi] /* UV */ \ + __asm lea esi, [esi + 16] \ + __asm vpermq ymm0, ymm0, 0xd8 \ + __asm vpunpcklwd ymm0, ymm0, ymm0 /* UVUV (upsample) */ \ + } + +// Convert 16 pixels: 16 UV and 16 Y. +#define YUVTORGB_AVX2(YuvConstants) __asm { \ + /* Step 1: Find 8 UV contributions to 16 R,G,B values */ \ + __asm vpmaddubsw ymm2, ymm0, YuvConstants.kUVToR /* scale R UV */ \ + __asm vpmaddubsw ymm1, ymm0, YuvConstants.kUVToG /* scale G UV */ \ + __asm vpmaddubsw ymm0, ymm0, YuvConstants.kUVToB /* scale B UV */ \ + __asm vmovdqu ymm3, YuvConstants.kUVBiasR \ + __asm vpsubw ymm2, ymm3, ymm2 \ + __asm vmovdqu ymm3, YuvConstants.kUVBiasG \ + __asm vpsubw ymm1, ymm3, ymm1 \ + __asm vmovdqu ymm3, YuvConstants.kUVBiasB \ + __asm vpsubw ymm0, ymm3, ymm0 \ + /* Step 2: Find Y contribution to 16 R,G,B values */ \ + __asm vmovdqu xmm3, [eax] /* NOLINT */ \ + __asm lea eax, [eax + 16] \ + __asm vpermq ymm3, ymm3, 0xd8 \ + __asm vpunpcklbw ymm3, ymm3, ymm3 \ + __asm vpmulhuw ymm3, ymm3, YuvConstants.kYToRgb \ + __asm vpaddsw ymm0, ymm0, ymm3 /* B += Y */ \ + __asm vpaddsw ymm1, ymm1, ymm3 /* G += Y */ \ + __asm vpaddsw ymm2, ymm2, ymm3 /* R += Y */ \ + __asm vpsraw ymm0, ymm0, 6 \ + __asm vpsraw ymm1, ymm1, 6 \ + __asm vpsraw ymm2, ymm2, 6 \ + __asm vpackuswb ymm0, ymm0, ymm0 /* B */ \ + __asm vpackuswb ymm1, ymm1, ymm1 /* G */ \ + __asm vpackuswb ymm2, ymm2, ymm2 /* R */ \ + } + +// Store 16 ARGB values. +#define STOREARGB_AVX2 __asm { \ + /* Step 3: Weave into ARGB */ \ + __asm vpunpcklbw ymm0, ymm0, ymm1 /* BG */ \ + __asm vpermq ymm0, ymm0, 0xd8 \ + __asm vpunpcklbw ymm2, ymm2, ymm5 /* RA */ \ + __asm vpermq ymm2, ymm2, 0xd8 \ + __asm vpunpcklwd ymm1, ymm0, ymm2 /* BGRA first 8 pixels */ \ + __asm vpunpckhwd ymm0, ymm0, ymm2 /* BGRA next 8 pixels */ \ + __asm vmovdqu 0[edx], ymm1 \ + __asm vmovdqu 32[edx], ymm0 \ + __asm lea edx, [edx + 64] \ + } + +#ifdef HAS_I422TOARGBROW_AVX2 +// 16 pixels +// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ARGB (64 bytes). +__declspec(naked) +void I422ToARGBRow_AVX2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // argb + mov ecx, [esp + 8 + 20] // width + sub edi, esi + vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha + + convertloop: + READYUV422_AVX2 + YUVTORGB_AVX2(kYuvConstants) + STOREARGB_AVX2 + + sub ecx, 16 + jg convertloop + + pop edi + pop esi + vzeroupper + ret + } +} +#endif // HAS_I422TOARGBROW_AVX2 + +#ifdef HAS_J422TOARGBROW_AVX2 +// 16 pixels +// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ARGB (64 bytes). +__declspec(naked) +void J422ToARGBRow_AVX2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // argb + mov ecx, [esp + 8 + 20] // width + sub edi, esi + vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha + + convertloop: + READYUV422_AVX2 + YUVTORGB_AVX2(kYuvJConstants) + STOREARGB_AVX2 + + sub ecx, 16 + jg convertloop + + pop edi + pop esi + vzeroupper + ret + } +} +#endif // HAS_J422TOARGBROW_AVX2 + +#ifdef HAS_I444TOARGBROW_AVX2 +// 16 pixels +// 16 UV values with 16 Y producing 16 ARGB (64 bytes). +__declspec(naked) +void I444ToARGBRow_AVX2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // argb + mov ecx, [esp + 8 + 20] // width + sub edi, esi + vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha + + convertloop: + READYUV444_AVX2 + YUVTORGB_AVX2(kYuvConstants) + STOREARGB_AVX2 + + sub ecx, 16 + jg convertloop + + pop edi + pop esi + vzeroupper + ret + } +} +#endif // HAS_I444TOARGBROW_AVX2 + +#ifdef HAS_I411TOARGBROW_AVX2 +// 16 pixels +// 4 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ARGB (64 bytes). +__declspec(naked) +void I411ToARGBRow_AVX2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // argb + mov ecx, [esp + 8 + 20] // width + sub edi, esi + vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha + + convertloop: + READYUV411_AVX2 + YUVTORGB_AVX2(kYuvConstants) + STOREARGB_AVX2 + + sub ecx, 16 + jg convertloop + + pop edi + pop esi + vzeroupper + ret + } +} +#endif // HAS_I411TOARGBROW_AVX2 + +#ifdef HAS_NV12TOARGBROW_AVX2 +// 16 pixels. +// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ARGB (64 bytes). +__declspec(naked) +void NV12ToARGBRow_AVX2(const uint8* y_buf, + const uint8* uv_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // Y + mov esi, [esp + 4 + 8] // UV + mov edx, [esp + 4 + 12] // argb + mov ecx, [esp + 4 + 16] // width + vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha + + convertloop: + READNV12_AVX2 + YUVTORGB_AVX2(kYuvConstants) + STOREARGB_AVX2 + + sub ecx, 16 + jg convertloop + + pop esi + vzeroupper + ret + } +} +#endif // HAS_NV12TOARGBROW_AVX2 + +#ifdef HAS_NV21TOARGBROW_AVX2 +// 16 pixels. +// 8 VU values upsampled to 16 VU, mixed with 16 Y producing 16 ARGB (64 bytes). +__declspec(naked) +void NV21ToARGBRow_AVX2(const uint8* y_buf, + const uint8* uv_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // Y + mov esi, [esp + 4 + 8] // UV + mov edx, [esp + 4 + 12] // argb + mov ecx, [esp + 4 + 16] // width + vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha + + convertloop: + READNV12_AVX2 + YUVTORGB_AVX2(kYvuConstants) + STOREARGB_AVX2 + + sub ecx, 16 + jg convertloop + + pop esi + vzeroupper + ret + } +} +#endif // HAS_NV21TOARGBROW_AVX2 + +#ifdef HAS_I422TOBGRAROW_AVX2 +// 16 pixels +// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 BGRA (64 bytes). +// TODO(fbarchard): Use macros to reduce duplicate code. See SSSE3. +__declspec(naked) +void I422ToBGRARow_AVX2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // argb + mov ecx, [esp + 8 + 20] // width + sub edi, esi + vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha + + convertloop: + READYUV422_AVX2 + YUVTORGB_AVX2(kYuvConstants) + + // Step 3: Weave into BGRA + vpunpcklbw ymm1, ymm1, ymm0 // GB + vpermq ymm1, ymm1, 0xd8 + vpunpcklbw ymm2, ymm5, ymm2 // AR + vpermq ymm2, ymm2, 0xd8 + vpunpcklwd ymm0, ymm2, ymm1 // ARGB first 8 pixels + vpunpckhwd ymm2, ymm2, ymm1 // ARGB next 8 pixels + vmovdqu [edx], ymm0 + vmovdqu [edx + 32], ymm2 + lea edx, [edx + 64] + sub ecx, 16 + jg convertloop + + pop edi + pop esi + vzeroupper + ret + } +} +#endif // HAS_I422TOBGRAROW_AVX2 + +#ifdef HAS_I422TORGBAROW_AVX2 +// 16 pixels +// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 RGBA (64 bytes). +// TODO(fbarchard): Use macros to reduce duplicate code. See SSSE3. +__declspec(naked) +void I422ToRGBARow_AVX2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // argb + mov ecx, [esp + 8 + 20] // width + sub edi, esi + vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha + + convertloop: + READYUV422_AVX2 + YUVTORGB_AVX2(kYuvConstants) + + // Step 3: Weave into RGBA + vpunpcklbw ymm1, ymm1, ymm2 // GR + vpermq ymm1, ymm1, 0xd8 + vpunpcklbw ymm2, ymm5, ymm0 // AB + vpermq ymm2, ymm2, 0xd8 + vpunpcklwd ymm0, ymm2, ymm1 // ABGR first 8 pixels + vpunpckhwd ymm1, ymm2, ymm1 // ABGR next 8 pixels + vmovdqu [edx], ymm0 + vmovdqu [edx + 32], ymm1 + lea edx, [edx + 64] + sub ecx, 16 + jg convertloop + + pop edi + pop esi + vzeroupper + ret + } +} +#endif // HAS_I422TORGBAROW_AVX2 + +#ifdef HAS_I422TOABGRROW_AVX2 +// 16 pixels +// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ABGR (64 bytes). +// TODO(fbarchard): Use macros to reduce duplicate code. See SSSE3. +__declspec(naked) +void I422ToABGRRow_AVX2(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // argb + mov ecx, [esp + 8 + 20] // width + sub edi, esi + vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha + + convertloop: + READYUV422_AVX2 + YUVTORGB_AVX2(kYuvConstants) + + // Step 3: Weave into ABGR + vpunpcklbw ymm1, ymm2, ymm1 // RG + vpermq ymm1, ymm1, 0xd8 + vpunpcklbw ymm2, ymm0, ymm5 // BA + vpermq ymm2, ymm2, 0xd8 + vpunpcklwd ymm0, ymm1, ymm2 // RGBA first 8 pixels + vpunpckhwd ymm1, ymm1, ymm2 // RGBA next 8 pixels + vmovdqu [edx], ymm0 + vmovdqu [edx + 32], ymm1 + lea edx, [edx + 64] + sub ecx, 16 + jg convertloop + + pop edi + pop esi + vzeroupper + ret + } +} +#endif // HAS_I422TOABGRROW_AVX2 + +#if defined(HAS_I422TOARGBROW_SSSE3) +// TODO(fbarchard): Read that does half size on Y and treats 420 as 444. + +// Read 8 UV from 444. +#define READYUV444 __asm { \ + __asm movq xmm0, qword ptr [esi] /* U */ /* NOLINT */ \ + __asm movq xmm1, qword ptr [esi + edi] /* V */ /* NOLINT */ \ + __asm lea esi, [esi + 8] \ + __asm punpcklbw xmm0, xmm1 /* UV */ \ + } + +// Read 4 UV from 422, upsample to 8 UV. +#define READYUV422 __asm { \ + __asm movd xmm0, [esi] /* U */ \ + __asm movd xmm1, [esi + edi] /* V */ \ + __asm lea esi, [esi + 4] \ + __asm punpcklbw xmm0, xmm1 /* UV */ \ + __asm punpcklwd xmm0, xmm0 /* UVUV (upsample) */ \ + } + +// Read 2 UV from 411, upsample to 8 UV. +#define READYUV411 __asm { \ + __asm movzx ebx, word ptr [esi] /* U */ /* NOLINT */ \ + __asm movd xmm0, ebx \ + __asm movzx ebx, word ptr [esi + edi] /* V */ /* NOLINT */ \ + __asm movd xmm1, ebx \ + __asm lea esi, [esi + 2] \ + __asm punpcklbw xmm0, xmm1 /* UV */ \ + __asm punpcklwd xmm0, xmm0 /* UVUV (upsample) */ \ + __asm punpckldq xmm0, xmm0 /* UVUVUVUV (upsample) */ \ + } + +// Read 4 UV from NV12, upsample to 8 UV. +#define READNV12 __asm { \ + __asm movq xmm0, qword ptr [esi] /* UV */ /* NOLINT */ \ + __asm lea esi, [esi + 8] \ + __asm punpcklwd xmm0, xmm0 /* UVUV (upsample) */ \ + } + +// Convert 8 pixels: 8 UV and 8 Y. +#define YUVTORGB(YuvConstants) __asm { \ + /* Step 1: Find 4 UV contributions to 8 R,G,B values */ \ + __asm movdqa xmm1, xmm0 \ + __asm movdqa xmm2, xmm0 \ + __asm movdqa xmm3, xmm0 \ + __asm movdqa xmm0, YuvConstants.kUVBiasB /* unbias back to signed */ \ + __asm pmaddubsw xmm1, YuvConstants.kUVToB /* scale B UV */ \ + __asm psubw xmm0, xmm1 \ + __asm movdqa xmm1, YuvConstants.kUVBiasG \ + __asm pmaddubsw xmm2, YuvConstants.kUVToG /* scale G UV */ \ + __asm psubw xmm1, xmm2 \ + __asm movdqa xmm2, YuvConstants.kUVBiasR \ + __asm pmaddubsw xmm3, YuvConstants.kUVToR /* scale R UV */ \ + __asm psubw xmm2, xmm3 \ + /* Step 2: Find Y contribution to 8 R,G,B values */ \ + __asm movq xmm3, qword ptr [eax] /* NOLINT */ \ + __asm lea eax, [eax + 8] \ + __asm punpcklbw xmm3, xmm3 \ + __asm pmulhuw xmm3, YuvConstants.kYToRgb \ + __asm paddsw xmm0, xmm3 /* B += Y */ \ + __asm paddsw xmm1, xmm3 /* G += Y */ \ + __asm paddsw xmm2, xmm3 /* R += Y */ \ + __asm psraw xmm0, 6 \ + __asm psraw xmm1, 6 \ + __asm psraw xmm2, 6 \ + __asm packuswb xmm0, xmm0 /* B */ \ + __asm packuswb xmm1, xmm1 /* G */ \ + __asm packuswb xmm2, xmm2 /* R */ \ + } + +// Store 8 ARGB values. +#define STOREARGB __asm { \ + /* Step 3: Weave into ARGB */ \ + __asm punpcklbw xmm0, xmm1 /* BG */ \ + __asm punpcklbw xmm2, xmm5 /* RA */ \ + __asm movdqa xmm1, xmm0 \ + __asm punpcklwd xmm0, xmm2 /* BGRA first 4 pixels */ \ + __asm punpckhwd xmm1, xmm2 /* BGRA next 4 pixels */ \ + __asm movdqu 0[edx], xmm0 \ + __asm movdqu 16[edx], xmm1 \ + __asm lea edx, [edx + 32] \ + } + +// Store 8 BGRA values. +#define STOREBGRA __asm { \ + /* Step 3: Weave into BGRA */ \ + __asm pcmpeqb xmm5, xmm5 /* generate 0xffffffff for alpha */ \ + __asm punpcklbw xmm1, xmm0 /* GB */ \ + __asm punpcklbw xmm5, xmm2 /* AR */ \ + __asm movdqa xmm0, xmm5 \ + __asm punpcklwd xmm5, xmm1 /* BGRA first 4 pixels */ \ + __asm punpckhwd xmm0, xmm1 /* BGRA next 4 pixels */ \ + __asm movdqu 0[edx], xmm5 \ + __asm movdqu 16[edx], xmm0 \ + __asm lea edx, [edx + 32] \ + } + +// Store 8 ABGR values. +#define STOREABGR __asm { \ + /* Step 3: Weave into ABGR */ \ + __asm punpcklbw xmm2, xmm1 /* RG */ \ + __asm punpcklbw xmm0, xmm5 /* BA */ \ + __asm movdqa xmm1, xmm2 \ + __asm punpcklwd xmm2, xmm0 /* RGBA first 4 pixels */ \ + __asm punpckhwd xmm1, xmm0 /* RGBA next 4 pixels */ \ + __asm movdqu 0[edx], xmm2 \ + __asm movdqu 16[edx], xmm1 \ + __asm lea edx, [edx + 32] \ + } + +// Store 8 RGBA values. +#define STORERGBA __asm { \ + /* Step 3: Weave into RGBA */ \ + __asm pcmpeqb xmm5, xmm5 /* generate 0xffffffff for alpha */ \ + __asm punpcklbw xmm1, xmm2 /* GR */ \ + __asm punpcklbw xmm5, xmm0 /* AB */ \ + __asm movdqa xmm0, xmm5 \ + __asm punpcklwd xmm5, xmm1 /* RGBA first 4 pixels */ \ + __asm punpckhwd xmm0, xmm1 /* RGBA next 4 pixels */ \ + __asm movdqu 0[edx], xmm5 \ + __asm movdqu 16[edx], xmm0 \ + __asm lea edx, [edx + 32] \ + } + +// Store 8 RGB24 values. +#define STORERGB24 __asm { \ + /* Step 3: Weave into RRGB */ \ + __asm punpcklbw xmm0, xmm1 /* BG */ \ + __asm punpcklbw xmm2, xmm2 /* RR */ \ + __asm movdqa xmm1, xmm0 \ + __asm punpcklwd xmm0, xmm2 /* BGRR first 4 pixels */ \ + __asm punpckhwd xmm1, xmm2 /* BGRR next 4 pixels */ \ + /* Step 4: RRGB -> RGB24 */ \ + __asm pshufb xmm0, xmm5 /* Pack first 8 and last 4 bytes. */ \ + __asm pshufb xmm1, xmm6 /* Pack first 12 bytes. */ \ + __asm palignr xmm1, xmm0, 12 /* last 4 bytes of xmm0 + 12 xmm1 */ \ + __asm movq qword ptr 0[edx], xmm0 /* First 8 bytes */ \ + __asm movdqu 8[edx], xmm1 /* Last 16 bytes */ \ + __asm lea edx, [edx + 24] \ + } + +// Store 8 RAW values. +#define STORERAW __asm { \ + /* Step 3: Weave into RRGB */ \ + __asm punpcklbw xmm0, xmm1 /* BG */ \ + __asm punpcklbw xmm2, xmm2 /* RR */ \ + __asm movdqa xmm1, xmm0 \ + __asm punpcklwd xmm0, xmm2 /* BGRR first 4 pixels */ \ + __asm punpckhwd xmm1, xmm2 /* BGRR next 4 pixels */ \ + /* Step 4: RRGB -> RAW */ \ + __asm pshufb xmm0, xmm5 /* Pack first 8 and last 4 bytes. */ \ + __asm pshufb xmm1, xmm6 /* Pack first 12 bytes. */ \ + __asm palignr xmm1, xmm0, 12 /* last 4 bytes of xmm0 + 12 xmm1 */ \ + __asm movq qword ptr 0[edx], xmm0 /* First 8 bytes */ \ + __asm movdqu 8[edx], xmm1 /* Last 16 bytes */ \ + __asm lea edx, [edx + 24] \ + } + +// Store 8 RGB565 values. +#define STORERGB565 __asm { \ + /* Step 3: Weave into RRGB */ \ + __asm punpcklbw xmm0, xmm1 /* BG */ \ + __asm punpcklbw xmm2, xmm2 /* RR */ \ + __asm movdqa xmm1, xmm0 \ + __asm punpcklwd xmm0, xmm2 /* BGRR first 4 pixels */ \ + __asm punpckhwd xmm1, xmm2 /* BGRR next 4 pixels */ \ + /* Step 4: RRGB -> RGB565 */ \ + __asm movdqa xmm3, xmm0 /* B first 4 pixels of argb */ \ + __asm movdqa xmm2, xmm0 /* G */ \ + __asm pslld xmm0, 8 /* R */ \ + __asm psrld xmm3, 3 /* B */ \ + __asm psrld xmm2, 5 /* G */ \ + __asm psrad xmm0, 16 /* R */ \ + __asm pand xmm3, xmm5 /* B */ \ + __asm pand xmm2, xmm6 /* G */ \ + __asm pand xmm0, xmm7 /* R */ \ + __asm por xmm3, xmm2 /* BG */ \ + __asm por xmm0, xmm3 /* BGR */ \ + __asm movdqa xmm3, xmm1 /* B next 4 pixels of argb */ \ + __asm movdqa xmm2, xmm1 /* G */ \ + __asm pslld xmm1, 8 /* R */ \ + __asm psrld xmm3, 3 /* B */ \ + __asm psrld xmm2, 5 /* G */ \ + __asm psrad xmm1, 16 /* R */ \ + __asm pand xmm3, xmm5 /* B */ \ + __asm pand xmm2, xmm6 /* G */ \ + __asm pand xmm1, xmm7 /* R */ \ + __asm por xmm3, xmm2 /* BG */ \ + __asm por xmm1, xmm3 /* BGR */ \ + __asm packssdw xmm0, xmm1 \ + __asm movdqu 0[edx], xmm0 /* store 8 pixels of RGB565 */ \ + __asm lea edx, [edx + 16] \ + } + +// 8 pixels. +// 8 UV values, mixed with 8 Y producing 8 ARGB (32 bytes). +__declspec(naked) +void I444ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // argb + mov ecx, [esp + 8 + 20] // width + sub edi, esi + pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha + + convertloop: + READYUV444 + YUVTORGB(kYuvConstants) + STOREARGB + + sub ecx, 8 + jg convertloop + + pop edi + pop esi + ret + } +} + +// 8 pixels. +// 4 UV values upsampled to 8 UV, mixed with 8 Y producing 8 RGB24 (24 bytes). +__declspec(naked) +void I422ToRGB24Row_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_rgb24, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // rgb24 + mov ecx, [esp + 8 + 20] // width + sub edi, esi + movdqa xmm5, kShuffleMaskARGBToRGB24_0 + movdqa xmm6, kShuffleMaskARGBToRGB24 + + convertloop: + READYUV422 + YUVTORGB(kYuvConstants) + STORERGB24 + + sub ecx, 8 + jg convertloop + + pop edi + pop esi + ret + } +} + +// 8 pixels. +// 4 UV values upsampled to 8 UV, mixed with 8 Y producing 8 RAW (24 bytes). +__declspec(naked) +void I422ToRAWRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_raw, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // raw + mov ecx, [esp + 8 + 20] // width + sub edi, esi + movdqa xmm5, kShuffleMaskARGBToRAW_0 + movdqa xmm6, kShuffleMaskARGBToRAW + + convertloop: + READYUV422 + YUVTORGB(kYuvConstants) + STORERAW + + sub ecx, 8 + jg convertloop + + pop edi + pop esi + ret + } +} + +// 8 pixels +// 4 UV values upsampled to 8 UV, mixed with 8 Y producing 8 RGB565 (16 bytes). +__declspec(naked) +void I422ToRGB565Row_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb565_buf, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // rgb565 + mov ecx, [esp + 8 + 20] // width + sub edi, esi + pcmpeqb xmm5, xmm5 // generate mask 0x0000001f + psrld xmm5, 27 + pcmpeqb xmm6, xmm6 // generate mask 0x000007e0 + psrld xmm6, 26 + pslld xmm6, 5 + pcmpeqb xmm7, xmm7 // generate mask 0xfffff800 + pslld xmm7, 11 + + convertloop: + READYUV422 + YUVTORGB(kYuvConstants) + STORERGB565 + + sub ecx, 8 + jg convertloop + + pop edi + pop esi + ret + } +} + +// 8 pixels. +// 4 UV values upsampled to 8 UV, mixed with 8 Y producing 8 ARGB (32 bytes). +__declspec(naked) +void I422ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // argb + mov ecx, [esp + 8 + 20] // width + sub edi, esi + pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha + + convertloop: + READYUV422 + YUVTORGB(kYuvConstants) + STOREARGB + + sub ecx, 8 + jg convertloop + + pop edi + pop esi + ret + } +} + +// 8 pixels. +// JPeg color space version of I422ToARGB +// 4 UV values upsampled to 8 UV, mixed with 8 Y producing 8 ARGB (32 bytes). +__declspec(naked) +void J422ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // argb + mov ecx, [esp + 8 + 20] // width + sub edi, esi + pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha + + convertloop: + READYUV422 + YUVTORGB(kYuvJConstants) + STOREARGB + + sub ecx, 8 + jg convertloop + + pop edi + pop esi + ret + } +} + +// 8 pixels. +// 2 UV values upsampled to 8 UV, mixed with 8 Y producing 8 ARGB (32 bytes). +// Similar to I420 but duplicate UV once more. +__declspec(naked) +void I411ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_argb, + int width) { + __asm { + push ebx + push esi + push edi + mov eax, [esp + 12 + 4] // Y + mov esi, [esp + 12 + 8] // U + mov edi, [esp + 12 + 12] // V + mov edx, [esp + 12 + 16] // argb + mov ecx, [esp + 12 + 20] // width + sub edi, esi + pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha + + convertloop: + READYUV411 // modifies EBX + YUVTORGB(kYuvConstants) + STOREARGB + + sub ecx, 8 + jg convertloop + + pop edi + pop esi + pop ebx + ret + } +} + +// 8 pixels. +// 4 UV values upsampled to 8 UV, mixed with 8 Y producing 8 ARGB (32 bytes). +__declspec(naked) +void NV12ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* uv_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // Y + mov esi, [esp + 4 + 8] // UV + mov edx, [esp + 4 + 12] // argb + mov ecx, [esp + 4 + 16] // width + pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha + + convertloop: + READNV12 + YUVTORGB(kYuvConstants) + STOREARGB + + sub ecx, 8 + jg convertloop + + pop esi + ret + } +} + +// 8 pixels. +// 4 VU values upsampled to 8 VU, mixed with 8 Y producing 8 ARGB (32 bytes). +__declspec(naked) +void NV21ToARGBRow_SSSE3(const uint8* y_buf, + const uint8* uv_buf, + uint8* dst_argb, + int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // Y + mov esi, [esp + 4 + 8] // UV + mov edx, [esp + 4 + 12] // argb + mov ecx, [esp + 4 + 16] // width + pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha + + convertloop: + READNV12 + YUVTORGB(kYvuConstants) + STOREARGB + + sub ecx, 8 + jg convertloop + + pop esi + ret + } +} + +__declspec(naked) +void I422ToBGRARow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_bgra, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // bgra + mov ecx, [esp + 8 + 20] // width + sub edi, esi + + convertloop: + READYUV422 + YUVTORGB(kYuvConstants) + STOREBGRA + + sub ecx, 8 + jg convertloop + + pop edi + pop esi + ret + } +} + +__declspec(naked) +void I422ToABGRRow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_abgr, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // abgr + mov ecx, [esp + 8 + 20] // width + sub edi, esi + pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha + + convertloop: + READYUV422 + YUVTORGB(kYuvConstants) + STOREABGR + + sub ecx, 8 + jg convertloop + + pop edi + pop esi + ret + } +} + +__declspec(naked) +void I422ToRGBARow_SSSE3(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* dst_rgba, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // Y + mov esi, [esp + 8 + 8] // U + mov edi, [esp + 8 + 12] // V + mov edx, [esp + 8 + 16] // rgba + mov ecx, [esp + 8 + 20] // width + sub edi, esi + + convertloop: + READYUV422 + YUVTORGB(kYuvConstants) + STORERGBA + + sub ecx, 8 + jg convertloop + + pop edi + pop esi + ret + } +} + +#endif // HAS_I422TOARGBROW_SSSE3 + +#ifdef HAS_I400TOARGBROW_SSE2 +// 8 pixels of Y converted to 8 pixels of ARGB (32 bytes). +__declspec(naked) +void I400ToARGBRow_SSE2(const uint8* y_buf, + uint8* rgb_buf, + int width) { + __asm { + mov eax, 0x4a354a35 // 4a35 = 18997 = round(1.164 * 64 * 256) + movd xmm2, eax + pshufd xmm2, xmm2,0 + mov eax, 0x04880488 // 0488 = 1160 = round(1.164 * 64 * 16) + movd xmm3, eax + pshufd xmm3, xmm3, 0 + pcmpeqb xmm4, xmm4 // generate mask 0xff000000 + pslld xmm4, 24 + + mov eax, [esp + 4] // Y + mov edx, [esp + 8] // rgb + mov ecx, [esp + 12] // width + + convertloop: + // Step 1: Scale Y contribution to 8 G values. G = (y - 16) * 1.164 + movq xmm0, qword ptr [eax] + lea eax, [eax + 8] + punpcklbw xmm0, xmm0 // Y.Y + pmulhuw xmm0, xmm2 + psubusw xmm0, xmm3 + psrlw xmm0, 6 + packuswb xmm0, xmm0 // G + + // Step 2: Weave into ARGB + punpcklbw xmm0, xmm0 // GG + movdqa xmm1, xmm0 + punpcklwd xmm0, xmm0 // BGRA first 4 pixels + punpckhwd xmm1, xmm1 // BGRA next 4 pixels + por xmm0, xmm4 + por xmm1, xmm4 + movdqu [edx], xmm0 + movdqu [edx + 16], xmm1 + lea edx, [edx + 32] + sub ecx, 8 + jg convertloop + ret + } +} +#endif // HAS_I400TOARGBROW_SSE2 + +#ifdef HAS_I400TOARGBROW_AVX2 +// 16 pixels of Y converted to 16 pixels of ARGB (64 bytes). +// note: vpunpcklbw mutates and vpackuswb unmutates. +__declspec(naked) +void I400ToARGBRow_AVX2(const uint8* y_buf, + uint8* rgb_buf, + int width) { + __asm { + mov eax, 0x4a354a35 // 4a35 = 18997 = round(1.164 * 64 * 256) + vmovd xmm2, eax + vbroadcastss ymm2, xmm2 + mov eax, 0x04880488 // 0488 = 1160 = round(1.164 * 64 * 16) + vmovd xmm3, eax + vbroadcastss ymm3, xmm3 + vpcmpeqb ymm4, ymm4, ymm4 // generate mask 0xff000000 + vpslld ymm4, ymm4, 24 + + mov eax, [esp + 4] // Y + mov edx, [esp + 8] // rgb + mov ecx, [esp + 12] // width + + convertloop: + // Step 1: Scale Y contriportbution to 16 G values. G = (y - 16) * 1.164 + vmovdqu xmm0, [eax] + lea eax, [eax + 16] + vpermq ymm0, ymm0, 0xd8 // vpunpcklbw mutates + vpunpcklbw ymm0, ymm0, ymm0 // Y.Y + vpmulhuw ymm0, ymm0, ymm2 + vpsubusw ymm0, ymm0, ymm3 + vpsrlw ymm0, ymm0, 6 + vpackuswb ymm0, ymm0, ymm0 // G. still mutated: 3120 + + // TODO(fbarchard): Weave alpha with unpack. + // Step 2: Weave into ARGB + vpunpcklbw ymm1, ymm0, ymm0 // GG - mutates + vpermq ymm1, ymm1, 0xd8 + vpunpcklwd ymm0, ymm1, ymm1 // GGGG first 8 pixels + vpunpckhwd ymm1, ymm1, ymm1 // GGGG next 8 pixels + vpor ymm0, ymm0, ymm4 + vpor ymm1, ymm1, ymm4 + vmovdqu [edx], ymm0 + vmovdqu [edx + 32], ymm1 + lea edx, [edx + 64] + sub ecx, 16 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_I400TOARGBROW_AVX2 + +#ifdef HAS_MIRRORROW_SSSE3 +// Shuffle table for reversing the bytes. +static const uvec8 kShuffleMirror = { + 15u, 14u, 13u, 12u, 11u, 10u, 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u +}; + +// TODO(fbarchard): Replace lea with -16 offset. +__declspec(naked) +void MirrorRow_SSSE3(const uint8* src, uint8* dst, int width) { + __asm { + mov eax, [esp + 4] // src + mov edx, [esp + 8] // dst + mov ecx, [esp + 12] // width + movdqa xmm5, kShuffleMirror + + convertloop: + movdqu xmm0, [eax - 16 + ecx] + pshufb xmm0, xmm5 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg convertloop + ret + } +} +#endif // HAS_MIRRORROW_SSSE3 + +#ifdef HAS_MIRRORROW_AVX2 +__declspec(naked) +void MirrorRow_AVX2(const uint8* src, uint8* dst, int width) { + __asm { + mov eax, [esp + 4] // src + mov edx, [esp + 8] // dst + mov ecx, [esp + 12] // width + vbroadcastf128 ymm5, kShuffleMirror + + convertloop: + vmovdqu ymm0, [eax - 32 + ecx] + vpshufb ymm0, ymm0, ymm5 + vpermq ymm0, ymm0, 0x4e // swap high and low halfs + vmovdqu [edx], ymm0 + lea edx, [edx + 32] + sub ecx, 32 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_MIRRORROW_AVX2 + +#ifdef HAS_MIRRORROW_SSE2 +__declspec(naked) +void MirrorRow_SSE2(const uint8* src, uint8* dst, int width) { + __asm { + mov eax, [esp + 4] // src + mov edx, [esp + 8] // dst + mov ecx, [esp + 12] // width + + convertloop: + movdqu xmm0, [eax - 16 + ecx] + movdqa xmm1, xmm0 // swap bytes + psllw xmm0, 8 + psrlw xmm1, 8 + por xmm0, xmm1 + pshuflw xmm0, xmm0, 0x1b // swap words + pshufhw xmm0, xmm0, 0x1b + pshufd xmm0, xmm0, 0x4e // swap qwords + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg convertloop + ret + } +} +#endif // HAS_MIRRORROW_SSE2 + +#ifdef HAS_MIRRORROW_UV_SSSE3 +// Shuffle table for reversing the bytes of UV channels. +static const uvec8 kShuffleMirrorUV = { + 14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u, 15u, 13u, 11u, 9u, 7u, 5u, 3u, 1u +}; + +__declspec(naked) +void MirrorUVRow_SSSE3(const uint8* src, uint8* dst_u, uint8* dst_v, + int width) { + __asm { + push edi + mov eax, [esp + 4 + 4] // src + mov edx, [esp + 4 + 8] // dst_u + mov edi, [esp + 4 + 12] // dst_v + mov ecx, [esp + 4 + 16] // width + movdqa xmm1, kShuffleMirrorUV + lea eax, [eax + ecx * 2 - 16] + sub edi, edx + + convertloop: + movdqu xmm0, [eax] + lea eax, [eax - 16] + pshufb xmm0, xmm1 + movlpd qword ptr [edx], xmm0 + movhpd qword ptr [edx + edi], xmm0 + lea edx, [edx + 8] + sub ecx, 8 + jg convertloop + + pop edi + ret + } +} +#endif // HAS_MIRRORROW_UV_SSSE3 + +#ifdef HAS_ARGBMIRRORROW_SSE2 +__declspec(naked) +void ARGBMirrorRow_SSE2(const uint8* src, uint8* dst, int width) { + __asm { + mov eax, [esp + 4] // src + mov edx, [esp + 8] // dst + mov ecx, [esp + 12] // width + lea eax, [eax - 16 + ecx * 4] // last 4 pixels. + + convertloop: + movdqu xmm0, [eax] + lea eax, [eax - 16] + pshufd xmm0, xmm0, 0x1b + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg convertloop + ret + } +} +#endif // HAS_ARGBMIRRORROW_SSE2 + +#ifdef HAS_ARGBMIRRORROW_AVX2 +// Shuffle table for reversing the bytes. +static const ulvec32 kARGBShuffleMirror_AVX2 = { + 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u +}; + +__declspec(naked) +void ARGBMirrorRow_AVX2(const uint8* src, uint8* dst, int width) { + __asm { + mov eax, [esp + 4] // src + mov edx, [esp + 8] // dst + mov ecx, [esp + 12] // width + vmovdqu ymm5, kARGBShuffleMirror_AVX2 + + convertloop: + vpermd ymm0, ymm5, [eax - 32 + ecx * 4] // permute dword order + vmovdqu [edx], ymm0 + lea edx, [edx + 32] + sub ecx, 8 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_ARGBMIRRORROW_AVX2 + +#ifdef HAS_SPLITUVROW_SSE2 +__declspec(naked) +void SplitUVRow_SSE2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix) { + __asm { + push edi + mov eax, [esp + 4 + 4] // src_uv + mov edx, [esp + 4 + 8] // dst_u + mov edi, [esp + 4 + 12] // dst_v + mov ecx, [esp + 4 + 16] // pix + pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff + psrlw xmm5, 8 + sub edi, edx + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + movdqa xmm2, xmm0 + movdqa xmm3, xmm1 + pand xmm0, xmm5 // even bytes + pand xmm1, xmm5 + packuswb xmm0, xmm1 + psrlw xmm2, 8 // odd bytes + psrlw xmm3, 8 + packuswb xmm2, xmm3 + movdqu [edx], xmm0 + movdqu [edx + edi], xmm2 + lea edx, [edx + 16] + sub ecx, 16 + jg convertloop + + pop edi + ret + } +} + +#endif // HAS_SPLITUVROW_SSE2 + +#ifdef HAS_SPLITUVROW_AVX2 +__declspec(naked) +void SplitUVRow_AVX2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix) { + __asm { + push edi + mov eax, [esp + 4 + 4] // src_uv + mov edx, [esp + 4 + 8] // dst_u + mov edi, [esp + 4 + 12] // dst_v + mov ecx, [esp + 4 + 16] // pix + vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff00ff + vpsrlw ymm5, ymm5, 8 + sub edi, edx + + convertloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + lea eax, [eax + 64] + vpsrlw ymm2, ymm0, 8 // odd bytes + vpsrlw ymm3, ymm1, 8 + vpand ymm0, ymm0, ymm5 // even bytes + vpand ymm1, ymm1, ymm5 + vpackuswb ymm0, ymm0, ymm1 + vpackuswb ymm2, ymm2, ymm3 + vpermq ymm0, ymm0, 0xd8 + vpermq ymm2, ymm2, 0xd8 + vmovdqu [edx], ymm0 + vmovdqu [edx + edi], ymm2 + lea edx, [edx + 32] + sub ecx, 32 + jg convertloop + + pop edi + vzeroupper + ret + } +} +#endif // HAS_SPLITUVROW_AVX2 + +#ifdef HAS_MERGEUVROW_SSE2 +__declspec(naked) +void MergeUVRow_SSE2(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width) { + __asm { + push edi + mov eax, [esp + 4 + 4] // src_u + mov edx, [esp + 4 + 8] // src_v + mov edi, [esp + 4 + 12] // dst_uv + mov ecx, [esp + 4 + 16] // width + sub edx, eax + + convertloop: + movdqu xmm0, [eax] // read 16 U's + movdqu xmm1, [eax + edx] // and 16 V's + lea eax, [eax + 16] + movdqa xmm2, xmm0 + punpcklbw xmm0, xmm1 // first 8 UV pairs + punpckhbw xmm2, xmm1 // next 8 UV pairs + movdqu [edi], xmm0 + movdqu [edi + 16], xmm2 + lea edi, [edi + 32] + sub ecx, 16 + jg convertloop + + pop edi + ret + } +} +#endif // HAS_MERGEUVROW_SSE2 + +#ifdef HAS_MERGEUVROW_AVX2 +__declspec(naked) +void MergeUVRow_AVX2(const uint8* src_u, const uint8* src_v, uint8* dst_uv, + int width) { + __asm { + push edi + mov eax, [esp + 4 + 4] // src_u + mov edx, [esp + 4 + 8] // src_v + mov edi, [esp + 4 + 12] // dst_uv + mov ecx, [esp + 4 + 16] // width + sub edx, eax + + convertloop: + vmovdqu ymm0, [eax] // read 32 U's + vmovdqu ymm1, [eax + edx] // and 32 V's + lea eax, [eax + 32] + vpunpcklbw ymm2, ymm0, ymm1 // low 16 UV pairs. mutated qqword 0,2 + vpunpckhbw ymm0, ymm0, ymm1 // high 16 UV pairs. mutated qqword 1,3 + vextractf128 [edi], ymm2, 0 // bytes 0..15 + vextractf128 [edi + 16], ymm0, 0 // bytes 16..31 + vextractf128 [edi + 32], ymm2, 1 // bytes 32..47 + vextractf128 [edi + 48], ymm0, 1 // bytes 47..63 + lea edi, [edi + 64] + sub ecx, 32 + jg convertloop + + pop edi + vzeroupper + ret + } +} +#endif // HAS_MERGEUVROW_AVX2 + +#ifdef HAS_COPYROW_SSE2 +// CopyRow copys 'count' bytes using a 16 byte load/store, 32 bytes at time. +__declspec(naked) +void CopyRow_SSE2(const uint8* src, uint8* dst, int count) { + __asm { + mov eax, [esp + 4] // src + mov edx, [esp + 8] // dst + mov ecx, [esp + 12] // count + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + movdqu [edx], xmm0 + movdqu [edx + 16], xmm1 + lea edx, [edx + 32] + sub ecx, 32 + jg convertloop + ret + } +} +#endif // HAS_COPYROW_SSE2 + +#ifdef HAS_COPYROW_AVX +// CopyRow copys 'count' bytes using a 32 byte load/store, 64 bytes at time. +__declspec(naked) +void CopyRow_AVX(const uint8* src, uint8* dst, int count) { + __asm { + mov eax, [esp + 4] // src + mov edx, [esp + 8] // dst + mov ecx, [esp + 12] // count + + convertloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + lea eax, [eax + 64] + vmovdqu [edx], ymm0 + vmovdqu [edx + 32], ymm1 + lea edx, [edx + 64] + sub ecx, 64 + jg convertloop + + vzeroupper + ret + } +} +#endif // HAS_COPYROW_AVX + +// Multiple of 1. +__declspec(naked) +void CopyRow_ERMS(const uint8* src, uint8* dst, int count) { + __asm { + mov eax, esi + mov edx, edi + mov esi, [esp + 4] // src + mov edi, [esp + 8] // dst + mov ecx, [esp + 12] // count + rep movsb + mov edi, edx + mov esi, eax + ret + } +} + +#ifdef HAS_ARGBCOPYALPHAROW_SSE2 +// width in pixels +__declspec(naked) +void ARGBCopyAlphaRow_SSE2(const uint8* src, uint8* dst, int width) { + __asm { + mov eax, [esp + 4] // src + mov edx, [esp + 8] // dst + mov ecx, [esp + 12] // count + pcmpeqb xmm0, xmm0 // generate mask 0xff000000 + pslld xmm0, 24 + pcmpeqb xmm1, xmm1 // generate mask 0x00ffffff + psrld xmm1, 8 + + convertloop: + movdqu xmm2, [eax] + movdqu xmm3, [eax + 16] + lea eax, [eax + 32] + movdqu xmm4, [edx] + movdqu xmm5, [edx + 16] + pand xmm2, xmm0 + pand xmm3, xmm0 + pand xmm4, xmm1 + pand xmm5, xmm1 + por xmm2, xmm4 + por xmm3, xmm5 + movdqu [edx], xmm2 + movdqu [edx + 16], xmm3 + lea edx, [edx + 32] + sub ecx, 8 + jg convertloop + + ret + } +} +#endif // HAS_ARGBCOPYALPHAROW_SSE2 + +#ifdef HAS_ARGBCOPYALPHAROW_AVX2 +// width in pixels +__declspec(naked) +void ARGBCopyAlphaRow_AVX2(const uint8* src, uint8* dst, int width) { + __asm { + mov eax, [esp + 4] // src + mov edx, [esp + 8] // dst + mov ecx, [esp + 12] // count + vpcmpeqb ymm0, ymm0, ymm0 + vpsrld ymm0, ymm0, 8 // generate mask 0x00ffffff + + convertloop: + vmovdqu ymm1, [eax] + vmovdqu ymm2, [eax + 32] + lea eax, [eax + 64] + vpblendvb ymm1, ymm1, [edx], ymm0 + vpblendvb ymm2, ymm2, [edx + 32], ymm0 + vmovdqu [edx], ymm1 + vmovdqu [edx + 32], ymm2 + lea edx, [edx + 64] + sub ecx, 16 + jg convertloop + + vzeroupper + ret + } +} +#endif // HAS_ARGBCOPYALPHAROW_AVX2 + +#ifdef HAS_ARGBCOPYYTOALPHAROW_SSE2 +// width in pixels +__declspec(naked) +void ARGBCopyYToAlphaRow_SSE2(const uint8* src, uint8* dst, int width) { + __asm { + mov eax, [esp + 4] // src + mov edx, [esp + 8] // dst + mov ecx, [esp + 12] // count + pcmpeqb xmm0, xmm0 // generate mask 0xff000000 + pslld xmm0, 24 + pcmpeqb xmm1, xmm1 // generate mask 0x00ffffff + psrld xmm1, 8 + + convertloop: + movq xmm2, qword ptr [eax] // 8 Y's + lea eax, [eax + 8] + punpcklbw xmm2, xmm2 + punpckhwd xmm3, xmm2 + punpcklwd xmm2, xmm2 + movdqu xmm4, [edx] + movdqu xmm5, [edx + 16] + pand xmm2, xmm0 + pand xmm3, xmm0 + pand xmm4, xmm1 + pand xmm5, xmm1 + por xmm2, xmm4 + por xmm3, xmm5 + movdqu [edx], xmm2 + movdqu [edx + 16], xmm3 + lea edx, [edx + 32] + sub ecx, 8 + jg convertloop + + ret + } +} +#endif // HAS_ARGBCOPYYTOALPHAROW_SSE2 + +#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX2 +// width in pixels +__declspec(naked) +void ARGBCopyYToAlphaRow_AVX2(const uint8* src, uint8* dst, int width) { + __asm { + mov eax, [esp + 4] // src + mov edx, [esp + 8] // dst + mov ecx, [esp + 12] // count + vpcmpeqb ymm0, ymm0, ymm0 + vpsrld ymm0, ymm0, 8 // generate mask 0x00ffffff + + convertloop: + vpmovzxbd ymm1, qword ptr [eax] + vpmovzxbd ymm2, qword ptr [eax + 8] + lea eax, [eax + 16] + vpslld ymm1, ymm1, 24 + vpslld ymm2, ymm2, 24 + vpblendvb ymm1, ymm1, [edx], ymm0 + vpblendvb ymm2, ymm2, [edx + 32], ymm0 + vmovdqu [edx], ymm1 + vmovdqu [edx + 32], ymm2 + lea edx, [edx + 64] + sub ecx, 16 + jg convertloop + + vzeroupper + ret + } +} +#endif // HAS_ARGBCOPYYTOALPHAROW_AVX2 + +#ifdef HAS_SETROW_X86 +// Write 'count' bytes using an 8 bit value repeated. +// Count should be multiple of 4. +__declspec(naked) +void SetRow_X86(uint8* dst, uint8 v8, int count) { + __asm { + movzx eax, byte ptr [esp + 8] // v8 + mov edx, 0x01010101 // Duplicate byte to all bytes. + mul edx // overwrites edx with upper part of result. + mov edx, edi + mov edi, [esp + 4] // dst + mov ecx, [esp + 12] // count + shr ecx, 2 + rep stosd + mov edi, edx + ret + } +} + +// Write 'count' bytes using an 8 bit value repeated. +__declspec(naked) +void SetRow_ERMS(uint8* dst, uint8 v8, int count) { + __asm { + mov edx, edi + mov edi, [esp + 4] // dst + mov eax, [esp + 8] // v8 + mov ecx, [esp + 12] // count + rep stosb + mov edi, edx + ret + } +} + +// Write 'count' 32 bit values. +__declspec(naked) +void ARGBSetRow_X86(uint8* dst_argb, uint32 v32, int count) { + __asm { + mov edx, edi + mov edi, [esp + 4] // dst + mov eax, [esp + 8] // v32 + mov ecx, [esp + 12] // count + rep stosd + mov edi, edx + ret + } +} +#endif // HAS_SETROW_X86 + +#ifdef HAS_YUY2TOYROW_AVX2 +__declspec(naked) +void YUY2ToYRow_AVX2(const uint8* src_yuy2, + uint8* dst_y, int pix) { + __asm { + mov eax, [esp + 4] // src_yuy2 + mov edx, [esp + 8] // dst_y + mov ecx, [esp + 12] // pix + vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff00ff + vpsrlw ymm5, ymm5, 8 + + convertloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + lea eax, [eax + 64] + vpand ymm0, ymm0, ymm5 // even bytes are Y + vpand ymm1, ymm1, ymm5 + vpackuswb ymm0, ymm0, ymm1 // mutates. + vpermq ymm0, ymm0, 0xd8 + vmovdqu [edx], ymm0 + lea edx, [edx + 32] + sub ecx, 32 + jg convertloop + vzeroupper + ret + } +} + +__declspec(naked) +void YUY2ToUVRow_AVX2(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_yuy2 + mov esi, [esp + 8 + 8] // stride_yuy2 + mov edx, [esp + 8 + 12] // dst_u + mov edi, [esp + 8 + 16] // dst_v + mov ecx, [esp + 8 + 20] // pix + vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff00ff + vpsrlw ymm5, ymm5, 8 + sub edi, edx + + convertloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + vpavgb ymm0, ymm0, [eax + esi] + vpavgb ymm1, ymm1, [eax + esi + 32] + lea eax, [eax + 64] + vpsrlw ymm0, ymm0, 8 // YUYV -> UVUV + vpsrlw ymm1, ymm1, 8 + vpackuswb ymm0, ymm0, ymm1 // mutates. + vpermq ymm0, ymm0, 0xd8 + vpand ymm1, ymm0, ymm5 // U + vpsrlw ymm0, ymm0, 8 // V + vpackuswb ymm1, ymm1, ymm1 // mutates. + vpackuswb ymm0, ymm0, ymm0 // mutates. + vpermq ymm1, ymm1, 0xd8 + vpermq ymm0, ymm0, 0xd8 + vextractf128 [edx], ymm1, 0 // U + vextractf128 [edx + edi], ymm0, 0 // V + lea edx, [edx + 16] + sub ecx, 32 + jg convertloop + + pop edi + pop esi + vzeroupper + ret + } +} + +__declspec(naked) +void YUY2ToUV422Row_AVX2(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int pix) { + __asm { + push edi + mov eax, [esp + 4 + 4] // src_yuy2 + mov edx, [esp + 4 + 8] // dst_u + mov edi, [esp + 4 + 12] // dst_v + mov ecx, [esp + 4 + 16] // pix + vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff00ff + vpsrlw ymm5, ymm5, 8 + sub edi, edx + + convertloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + lea eax, [eax + 64] + vpsrlw ymm0, ymm0, 8 // YUYV -> UVUV + vpsrlw ymm1, ymm1, 8 + vpackuswb ymm0, ymm0, ymm1 // mutates. + vpermq ymm0, ymm0, 0xd8 + vpand ymm1, ymm0, ymm5 // U + vpsrlw ymm0, ymm0, 8 // V + vpackuswb ymm1, ymm1, ymm1 // mutates. + vpackuswb ymm0, ymm0, ymm0 // mutates. + vpermq ymm1, ymm1, 0xd8 + vpermq ymm0, ymm0, 0xd8 + vextractf128 [edx], ymm1, 0 // U + vextractf128 [edx + edi], ymm0, 0 // V + lea edx, [edx + 16] + sub ecx, 32 + jg convertloop + + pop edi + vzeroupper + ret + } +} + +__declspec(naked) +void UYVYToYRow_AVX2(const uint8* src_uyvy, + uint8* dst_y, int pix) { + __asm { + mov eax, [esp + 4] // src_uyvy + mov edx, [esp + 8] // dst_y + mov ecx, [esp + 12] // pix + + convertloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + lea eax, [eax + 64] + vpsrlw ymm0, ymm0, 8 // odd bytes are Y + vpsrlw ymm1, ymm1, 8 + vpackuswb ymm0, ymm0, ymm1 // mutates. + vpermq ymm0, ymm0, 0xd8 + vmovdqu [edx], ymm0 + lea edx, [edx + 32] + sub ecx, 32 + jg convertloop + vzeroupper + ret + } +} + +__declspec(naked) +void UYVYToUVRow_AVX2(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_yuy2 + mov esi, [esp + 8 + 8] // stride_yuy2 + mov edx, [esp + 8 + 12] // dst_u + mov edi, [esp + 8 + 16] // dst_v + mov ecx, [esp + 8 + 20] // pix + vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff00ff + vpsrlw ymm5, ymm5, 8 + sub edi, edx + + convertloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + vpavgb ymm0, ymm0, [eax + esi] + vpavgb ymm1, ymm1, [eax + esi + 32] + lea eax, [eax + 64] + vpand ymm0, ymm0, ymm5 // UYVY -> UVUV + vpand ymm1, ymm1, ymm5 + vpackuswb ymm0, ymm0, ymm1 // mutates. + vpermq ymm0, ymm0, 0xd8 + vpand ymm1, ymm0, ymm5 // U + vpsrlw ymm0, ymm0, 8 // V + vpackuswb ymm1, ymm1, ymm1 // mutates. + vpackuswb ymm0, ymm0, ymm0 // mutates. + vpermq ymm1, ymm1, 0xd8 + vpermq ymm0, ymm0, 0xd8 + vextractf128 [edx], ymm1, 0 // U + vextractf128 [edx + edi], ymm0, 0 // V + lea edx, [edx + 16] + sub ecx, 32 + jg convertloop + + pop edi + pop esi + vzeroupper + ret + } +} + +__declspec(naked) +void UYVYToUV422Row_AVX2(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix) { + __asm { + push edi + mov eax, [esp + 4 + 4] // src_yuy2 + mov edx, [esp + 4 + 8] // dst_u + mov edi, [esp + 4 + 12] // dst_v + mov ecx, [esp + 4 + 16] // pix + vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff00ff + vpsrlw ymm5, ymm5, 8 + sub edi, edx + + convertloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + lea eax, [eax + 64] + vpand ymm0, ymm0, ymm5 // UYVY -> UVUV + vpand ymm1, ymm1, ymm5 + vpackuswb ymm0, ymm0, ymm1 // mutates. + vpermq ymm0, ymm0, 0xd8 + vpand ymm1, ymm0, ymm5 // U + vpsrlw ymm0, ymm0, 8 // V + vpackuswb ymm1, ymm1, ymm1 // mutates. + vpackuswb ymm0, ymm0, ymm0 // mutates. + vpermq ymm1, ymm1, 0xd8 + vpermq ymm0, ymm0, 0xd8 + vextractf128 [edx], ymm1, 0 // U + vextractf128 [edx + edi], ymm0, 0 // V + lea edx, [edx + 16] + sub ecx, 32 + jg convertloop + + pop edi + vzeroupper + ret + } +} +#endif // HAS_YUY2TOYROW_AVX2 + +#ifdef HAS_YUY2TOYROW_SSE2 +__declspec(naked) +void YUY2ToYRow_SSE2(const uint8* src_yuy2, + uint8* dst_y, int pix) { + __asm { + mov eax, [esp + 4] // src_yuy2 + mov edx, [esp + 8] // dst_y + mov ecx, [esp + 12] // pix + pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff + psrlw xmm5, 8 + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + pand xmm0, xmm5 // even bytes are Y + pand xmm1, xmm5 + packuswb xmm0, xmm1 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg convertloop + ret + } +} + +__declspec(naked) +void YUY2ToUVRow_SSE2(const uint8* src_yuy2, int stride_yuy2, + uint8* dst_u, uint8* dst_v, int pix) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_yuy2 + mov esi, [esp + 8 + 8] // stride_yuy2 + mov edx, [esp + 8 + 12] // dst_u + mov edi, [esp + 8 + 16] // dst_v + mov ecx, [esp + 8 + 20] // pix + pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff + psrlw xmm5, 8 + sub edi, edx + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + esi] + movdqu xmm3, [eax + esi + 16] + lea eax, [eax + 32] + pavgb xmm0, xmm2 + pavgb xmm1, xmm3 + psrlw xmm0, 8 // YUYV -> UVUV + psrlw xmm1, 8 + packuswb xmm0, xmm1 + movdqa xmm1, xmm0 + pand xmm0, xmm5 // U + packuswb xmm0, xmm0 + psrlw xmm1, 8 // V + packuswb xmm1, xmm1 + movq qword ptr [edx], xmm0 + movq qword ptr [edx + edi], xmm1 + lea edx, [edx + 8] + sub ecx, 16 + jg convertloop + + pop edi + pop esi + ret + } +} + +__declspec(naked) +void YUY2ToUV422Row_SSE2(const uint8* src_yuy2, + uint8* dst_u, uint8* dst_v, int pix) { + __asm { + push edi + mov eax, [esp + 4 + 4] // src_yuy2 + mov edx, [esp + 4 + 8] // dst_u + mov edi, [esp + 4 + 12] // dst_v + mov ecx, [esp + 4 + 16] // pix + pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff + psrlw xmm5, 8 + sub edi, edx + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + psrlw xmm0, 8 // YUYV -> UVUV + psrlw xmm1, 8 + packuswb xmm0, xmm1 + movdqa xmm1, xmm0 + pand xmm0, xmm5 // U + packuswb xmm0, xmm0 + psrlw xmm1, 8 // V + packuswb xmm1, xmm1 + movq qword ptr [edx], xmm0 + movq qword ptr [edx + edi], xmm1 + lea edx, [edx + 8] + sub ecx, 16 + jg convertloop + + pop edi + ret + } +} + +__declspec(naked) +void UYVYToYRow_SSE2(const uint8* src_uyvy, + uint8* dst_y, int pix) { + __asm { + mov eax, [esp + 4] // src_uyvy + mov edx, [esp + 8] // dst_y + mov ecx, [esp + 12] // pix + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + psrlw xmm0, 8 // odd bytes are Y + psrlw xmm1, 8 + packuswb xmm0, xmm1 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg convertloop + ret + } +} + +__declspec(naked) +void UYVYToUVRow_SSE2(const uint8* src_uyvy, int stride_uyvy, + uint8* dst_u, uint8* dst_v, int pix) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_yuy2 + mov esi, [esp + 8 + 8] // stride_yuy2 + mov edx, [esp + 8 + 12] // dst_u + mov edi, [esp + 8 + 16] // dst_v + mov ecx, [esp + 8 + 20] // pix + pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff + psrlw xmm5, 8 + sub edi, edx + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + esi] + movdqu xmm3, [eax + esi + 16] + lea eax, [eax + 32] + pavgb xmm0, xmm2 + pavgb xmm1, xmm3 + pand xmm0, xmm5 // UYVY -> UVUV + pand xmm1, xmm5 + packuswb xmm0, xmm1 + movdqa xmm1, xmm0 + pand xmm0, xmm5 // U + packuswb xmm0, xmm0 + psrlw xmm1, 8 // V + packuswb xmm1, xmm1 + movq qword ptr [edx], xmm0 + movq qword ptr [edx + edi], xmm1 + lea edx, [edx + 8] + sub ecx, 16 + jg convertloop + + pop edi + pop esi + ret + } +} + +__declspec(naked) +void UYVYToUV422Row_SSE2(const uint8* src_uyvy, + uint8* dst_u, uint8* dst_v, int pix) { + __asm { + push edi + mov eax, [esp + 4 + 4] // src_yuy2 + mov edx, [esp + 4 + 8] // dst_u + mov edi, [esp + 4 + 12] // dst_v + mov ecx, [esp + 4 + 16] // pix + pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff + psrlw xmm5, 8 + sub edi, edx + + convertloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + pand xmm0, xmm5 // UYVY -> UVUV + pand xmm1, xmm5 + packuswb xmm0, xmm1 + movdqa xmm1, xmm0 + pand xmm0, xmm5 // U + packuswb xmm0, xmm0 + psrlw xmm1, 8 // V + packuswb xmm1, xmm1 + movq qword ptr [edx], xmm0 + movq qword ptr [edx + edi], xmm1 + lea edx, [edx + 8] + sub ecx, 16 + jg convertloop + + pop edi + ret + } +} +#endif // HAS_YUY2TOYROW_SSE2 + +#ifdef HAS_ARGBBLENDROW_SSE2 +// Blend 8 pixels at a time. +__declspec(naked) +void ARGBBlendRow_SSE2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_argb0 + mov esi, [esp + 4 + 8] // src_argb1 + mov edx, [esp + 4 + 12] // dst_argb + mov ecx, [esp + 4 + 16] // width + pcmpeqb xmm7, xmm7 // generate constant 1 + psrlw xmm7, 15 + pcmpeqb xmm6, xmm6 // generate mask 0x00ff00ff + psrlw xmm6, 8 + pcmpeqb xmm5, xmm5 // generate mask 0xff00ff00 + psllw xmm5, 8 + pcmpeqb xmm4, xmm4 // generate mask 0xff000000 + pslld xmm4, 24 + sub ecx, 4 + jl convertloop4b // less than 4 pixels? + + // 4 pixel loop. + convertloop4: + movdqu xmm3, [eax] // src argb + lea eax, [eax + 16] + movdqa xmm0, xmm3 // src argb + pxor xmm3, xmm4 // ~alpha + movdqu xmm2, [esi] // _r_b + psrlw xmm3, 8 // alpha + pshufhw xmm3, xmm3, 0F5h // 8 alpha words + pshuflw xmm3, xmm3, 0F5h + pand xmm2, xmm6 // _r_b + paddw xmm3, xmm7 // 256 - alpha + pmullw xmm2, xmm3 // _r_b * alpha + movdqu xmm1, [esi] // _a_g + lea esi, [esi + 16] + psrlw xmm1, 8 // _a_g + por xmm0, xmm4 // set alpha to 255 + pmullw xmm1, xmm3 // _a_g * alpha + psrlw xmm2, 8 // _r_b convert to 8 bits again + paddusb xmm0, xmm2 // + src argb + pand xmm1, xmm5 // a_g_ convert to 8 bits again + paddusb xmm0, xmm1 // + src argb + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jge convertloop4 + + convertloop4b: + add ecx, 4 - 1 + jl convertloop1b + + // 1 pixel loop. + convertloop1: + movd xmm3, [eax] // src argb + lea eax, [eax + 4] + movdqa xmm0, xmm3 // src argb + pxor xmm3, xmm4 // ~alpha + movd xmm2, [esi] // _r_b + psrlw xmm3, 8 // alpha + pshufhw xmm3, xmm3, 0F5h // 8 alpha words + pshuflw xmm3, xmm3, 0F5h + pand xmm2, xmm6 // _r_b + paddw xmm3, xmm7 // 256 - alpha + pmullw xmm2, xmm3 // _r_b * alpha + movd xmm1, [esi] // _a_g + lea esi, [esi + 4] + psrlw xmm1, 8 // _a_g + por xmm0, xmm4 // set alpha to 255 + pmullw xmm1, xmm3 // _a_g * alpha + psrlw xmm2, 8 // _r_b convert to 8 bits again + paddusb xmm0, xmm2 // + src argb + pand xmm1, xmm5 // a_g_ convert to 8 bits again + paddusb xmm0, xmm1 // + src argb + movd [edx], xmm0 + lea edx, [edx + 4] + sub ecx, 1 + jge convertloop1 + + convertloop1b: + pop esi + ret + } +} +#endif // HAS_ARGBBLENDROW_SSE2 + +#ifdef HAS_ARGBBLENDROW_SSSE3 +// Shuffle table for isolating alpha. +static const uvec8 kShuffleAlpha = { + 3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, + 11u, 0x80, 11u, 0x80, 15u, 0x80, 15u, 0x80 +}; +// Same as SSE2, but replaces: +// psrlw xmm3, 8 // alpha +// pshufhw xmm3, xmm3, 0F5h // 8 alpha words +// pshuflw xmm3, xmm3, 0F5h +// with.. +// pshufb xmm3, kShuffleAlpha // alpha +// Blend 8 pixels at a time. + +__declspec(naked) +void ARGBBlendRow_SSSE3(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_argb0 + mov esi, [esp + 4 + 8] // src_argb1 + mov edx, [esp + 4 + 12] // dst_argb + mov ecx, [esp + 4 + 16] // width + pcmpeqb xmm7, xmm7 // generate constant 0x0001 + psrlw xmm7, 15 + pcmpeqb xmm6, xmm6 // generate mask 0x00ff00ff + psrlw xmm6, 8 + pcmpeqb xmm5, xmm5 // generate mask 0xff00ff00 + psllw xmm5, 8 + pcmpeqb xmm4, xmm4 // generate mask 0xff000000 + pslld xmm4, 24 + sub ecx, 4 + jl convertloop4b // less than 4 pixels? + + // 4 pixel loop. + convertloop4: + movdqu xmm3, [eax] // src argb + lea eax, [eax + 16] + movdqa xmm0, xmm3 // src argb + pxor xmm3, xmm4 // ~alpha + movdqu xmm2, [esi] // _r_b + pshufb xmm3, kShuffleAlpha // alpha + pand xmm2, xmm6 // _r_b + paddw xmm3, xmm7 // 256 - alpha + pmullw xmm2, xmm3 // _r_b * alpha + movdqu xmm1, [esi] // _a_g + lea esi, [esi + 16] + psrlw xmm1, 8 // _a_g + por xmm0, xmm4 // set alpha to 255 + pmullw xmm1, xmm3 // _a_g * alpha + psrlw xmm2, 8 // _r_b convert to 8 bits again + paddusb xmm0, xmm2 // + src argb + pand xmm1, xmm5 // a_g_ convert to 8 bits again + paddusb xmm0, xmm1 // + src argb + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jge convertloop4 + + convertloop4b: + add ecx, 4 - 1 + jl convertloop1b + + // 1 pixel loop. + convertloop1: + movd xmm3, [eax] // src argb + lea eax, [eax + 4] + movdqa xmm0, xmm3 // src argb + pxor xmm3, xmm4 // ~alpha + movd xmm2, [esi] // _r_b + pshufb xmm3, kShuffleAlpha // alpha + pand xmm2, xmm6 // _r_b + paddw xmm3, xmm7 // 256 - alpha + pmullw xmm2, xmm3 // _r_b * alpha + movd xmm1, [esi] // _a_g + lea esi, [esi + 4] + psrlw xmm1, 8 // _a_g + por xmm0, xmm4 // set alpha to 255 + pmullw xmm1, xmm3 // _a_g * alpha + psrlw xmm2, 8 // _r_b convert to 8 bits again + paddusb xmm0, xmm2 // + src argb + pand xmm1, xmm5 // a_g_ convert to 8 bits again + paddusb xmm0, xmm1 // + src argb + movd [edx], xmm0 + lea edx, [edx + 4] + sub ecx, 1 + jge convertloop1 + + convertloop1b: + pop esi + ret + } +} +#endif // HAS_ARGBBLENDROW_SSSE3 + +#ifdef HAS_ARGBATTENUATEROW_SSE2 +// Attenuate 4 pixels at a time. +__declspec(naked) +void ARGBAttenuateRow_SSE2(const uint8* src_argb, uint8* dst_argb, int width) { + __asm { + mov eax, [esp + 4] // src_argb0 + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // width + pcmpeqb xmm4, xmm4 // generate mask 0xff000000 + pslld xmm4, 24 + pcmpeqb xmm5, xmm5 // generate mask 0x00ffffff + psrld xmm5, 8 + + convertloop: + movdqu xmm0, [eax] // read 4 pixels + punpcklbw xmm0, xmm0 // first 2 + pshufhw xmm2, xmm0, 0FFh // 8 alpha words + pshuflw xmm2, xmm2, 0FFh + pmulhuw xmm0, xmm2 // rgb * a + movdqu xmm1, [eax] // read 4 pixels + punpckhbw xmm1, xmm1 // next 2 pixels + pshufhw xmm2, xmm1, 0FFh // 8 alpha words + pshuflw xmm2, xmm2, 0FFh + pmulhuw xmm1, xmm2 // rgb * a + movdqu xmm2, [eax] // alphas + lea eax, [eax + 16] + psrlw xmm0, 8 + pand xmm2, xmm4 + psrlw xmm1, 8 + packuswb xmm0, xmm1 + pand xmm0, xmm5 // keep original alphas + por xmm0, xmm2 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg convertloop + + ret + } +} +#endif // HAS_ARGBATTENUATEROW_SSE2 + +#ifdef HAS_ARGBATTENUATEROW_SSSE3 +// Shuffle table duplicating alpha. +static const uvec8 kShuffleAlpha0 = { + 3u, 3u, 3u, 3u, 3u, 3u, 128u, 128u, 7u, 7u, 7u, 7u, 7u, 7u, 128u, 128u, +}; +static const uvec8 kShuffleAlpha1 = { + 11u, 11u, 11u, 11u, 11u, 11u, 128u, 128u, + 15u, 15u, 15u, 15u, 15u, 15u, 128u, 128u, +}; +__declspec(naked) +void ARGBAttenuateRow_SSSE3(const uint8* src_argb, uint8* dst_argb, int width) { + __asm { + mov eax, [esp + 4] // src_argb0 + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // width + pcmpeqb xmm3, xmm3 // generate mask 0xff000000 + pslld xmm3, 24 + movdqa xmm4, kShuffleAlpha0 + movdqa xmm5, kShuffleAlpha1 + + convertloop: + movdqu xmm0, [eax] // read 4 pixels + pshufb xmm0, xmm4 // isolate first 2 alphas + movdqu xmm1, [eax] // read 4 pixels + punpcklbw xmm1, xmm1 // first 2 pixel rgbs + pmulhuw xmm0, xmm1 // rgb * a + movdqu xmm1, [eax] // read 4 pixels + pshufb xmm1, xmm5 // isolate next 2 alphas + movdqu xmm2, [eax] // read 4 pixels + punpckhbw xmm2, xmm2 // next 2 pixel rgbs + pmulhuw xmm1, xmm2 // rgb * a + movdqu xmm2, [eax] // mask original alpha + lea eax, [eax + 16] + pand xmm2, xmm3 + psrlw xmm0, 8 + psrlw xmm1, 8 + packuswb xmm0, xmm1 + por xmm0, xmm2 // copy original alpha + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg convertloop + + ret + } +} +#endif // HAS_ARGBATTENUATEROW_SSSE3 + +#ifdef HAS_ARGBATTENUATEROW_AVX2 +// Shuffle table duplicating alpha. +static const uvec8 kShuffleAlpha_AVX2 = { + 6u, 7u, 6u, 7u, 6u, 7u, 128u, 128u, 14u, 15u, 14u, 15u, 14u, 15u, 128u, 128u +}; +__declspec(naked) +void ARGBAttenuateRow_AVX2(const uint8* src_argb, uint8* dst_argb, int width) { + __asm { + mov eax, [esp + 4] // src_argb0 + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // width + sub edx, eax + vbroadcastf128 ymm4,kShuffleAlpha_AVX2 + vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0xff000000 + vpslld ymm5, ymm5, 24 + + convertloop: + vmovdqu ymm6, [eax] // read 8 pixels. + vpunpcklbw ymm0, ymm6, ymm6 // low 4 pixels. mutated. + vpunpckhbw ymm1, ymm6, ymm6 // high 4 pixels. mutated. + vpshufb ymm2, ymm0, ymm4 // low 4 alphas + vpshufb ymm3, ymm1, ymm4 // high 4 alphas + vpmulhuw ymm0, ymm0, ymm2 // rgb * a + vpmulhuw ymm1, ymm1, ymm3 // rgb * a + vpand ymm6, ymm6, ymm5 // isolate alpha + vpsrlw ymm0, ymm0, 8 + vpsrlw ymm1, ymm1, 8 + vpackuswb ymm0, ymm0, ymm1 // unmutated. + vpor ymm0, ymm0, ymm6 // copy original alpha + vmovdqu [eax + edx], ymm0 + lea eax, [eax + 32] + sub ecx, 8 + jg convertloop + + vzeroupper + ret + } +} +#endif // HAS_ARGBATTENUATEROW_AVX2 + +#ifdef HAS_ARGBUNATTENUATEROW_SSE2 +// Unattenuate 4 pixels at a time. +__declspec(naked) +void ARGBUnattenuateRow_SSE2(const uint8* src_argb, uint8* dst_argb, + int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_argb0 + mov edx, [esp + 8 + 8] // dst_argb + mov ecx, [esp + 8 + 12] // width + + convertloop: + movdqu xmm0, [eax] // read 4 pixels + movzx esi, byte ptr [eax + 3] // first alpha + movzx edi, byte ptr [eax + 7] // second alpha + punpcklbw xmm0, xmm0 // first 2 + movd xmm2, dword ptr fixed_invtbl8[esi * 4] + movd xmm3, dword ptr fixed_invtbl8[edi * 4] + pshuflw xmm2, xmm2, 040h // first 4 inv_alpha words. 1, a, a, a + pshuflw xmm3, xmm3, 040h // next 4 inv_alpha words + movlhps xmm2, xmm3 + pmulhuw xmm0, xmm2 // rgb * a + + movdqu xmm1, [eax] // read 4 pixels + movzx esi, byte ptr [eax + 11] // third alpha + movzx edi, byte ptr [eax + 15] // forth alpha + punpckhbw xmm1, xmm1 // next 2 + movd xmm2, dword ptr fixed_invtbl8[esi * 4] + movd xmm3, dword ptr fixed_invtbl8[edi * 4] + pshuflw xmm2, xmm2, 040h // first 4 inv_alpha words + pshuflw xmm3, xmm3, 040h // next 4 inv_alpha words + movlhps xmm2, xmm3 + pmulhuw xmm1, xmm2 // rgb * a + lea eax, [eax + 16] + + packuswb xmm0, xmm1 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg convertloop + pop edi + pop esi + ret + } +} +#endif // HAS_ARGBUNATTENUATEROW_SSE2 + +#ifdef HAS_ARGBUNATTENUATEROW_AVX2 +// Shuffle table duplicating alpha. +static const uvec8 kUnattenShuffleAlpha_AVX2 = { + 0u, 1u, 0u, 1u, 0u, 1u, 6u, 7u, 8u, 9u, 8u, 9u, 8u, 9u, 14u, 15u +}; +// TODO(fbarchard): Enable USE_GATHER for future hardware if faster. +// USE_GATHER is not on by default, due to being a slow instruction. +#ifdef USE_GATHER +__declspec(naked) +void ARGBUnattenuateRow_AVX2(const uint8* src_argb, uint8* dst_argb, + int width) { + __asm { + mov eax, [esp + 4] // src_argb0 + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // width + sub edx, eax + vbroadcastf128 ymm4, kUnattenShuffleAlpha_AVX2 + + convertloop: + vmovdqu ymm6, [eax] // read 8 pixels. + vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0xffffffff for gather. + vpsrld ymm2, ymm6, 24 // alpha in low 8 bits. + vpunpcklbw ymm0, ymm6, ymm6 // low 4 pixels. mutated. + vpunpckhbw ymm1, ymm6, ymm6 // high 4 pixels. mutated. + vpgatherdd ymm3, [ymm2 * 4 + fixed_invtbl8], ymm5 // ymm5 cleared. 1, a + vpunpcklwd ymm2, ymm3, ymm3 // low 4 inverted alphas. mutated. 1, 1, a, a + vpunpckhwd ymm3, ymm3, ymm3 // high 4 inverted alphas. mutated. + vpshufb ymm2, ymm2, ymm4 // replicate low 4 alphas. 1, a, a, a + vpshufb ymm3, ymm3, ymm4 // replicate high 4 alphas + vpmulhuw ymm0, ymm0, ymm2 // rgb * ia + vpmulhuw ymm1, ymm1, ymm3 // rgb * ia + vpackuswb ymm0, ymm0, ymm1 // unmutated. + vmovdqu [eax + edx], ymm0 + lea eax, [eax + 32] + sub ecx, 8 + jg convertloop + + vzeroupper + ret + } +} +#else // USE_GATHER +__declspec(naked) +void ARGBUnattenuateRow_AVX2(const uint8* src_argb, uint8* dst_argb, + int width) { + __asm { + + mov eax, [esp + 4] // src_argb0 + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // width + sub edx, eax + vbroadcastf128 ymm5, kUnattenShuffleAlpha_AVX2 + + push esi + push edi + + convertloop: + // replace VPGATHER + movzx esi, byte ptr [eax + 3] // alpha0 + movzx edi, byte ptr [eax + 7] // alpha1 + vmovd xmm0, dword ptr fixed_invtbl8[esi * 4] // [1,a0] + vmovd xmm1, dword ptr fixed_invtbl8[edi * 4] // [1,a1] + movzx esi, byte ptr [eax + 11] // alpha2 + movzx edi, byte ptr [eax + 15] // alpha3 + vpunpckldq xmm6, xmm0, xmm1 // [1,a1,1,a0] + vmovd xmm2, dword ptr fixed_invtbl8[esi * 4] // [1,a2] + vmovd xmm3, dword ptr fixed_invtbl8[edi * 4] // [1,a3] + movzx esi, byte ptr [eax + 19] // alpha4 + movzx edi, byte ptr [eax + 23] // alpha5 + vpunpckldq xmm7, xmm2, xmm3 // [1,a3,1,a2] + vmovd xmm0, dword ptr fixed_invtbl8[esi * 4] // [1,a4] + vmovd xmm1, dword ptr fixed_invtbl8[edi * 4] // [1,a5] + movzx esi, byte ptr [eax + 27] // alpha6 + movzx edi, byte ptr [eax + 31] // alpha7 + vpunpckldq xmm0, xmm0, xmm1 // [1,a5,1,a4] + vmovd xmm2, dword ptr fixed_invtbl8[esi * 4] // [1,a6] + vmovd xmm3, dword ptr fixed_invtbl8[edi * 4] // [1,a7] + vpunpckldq xmm2, xmm2, xmm3 // [1,a7,1,a6] + vpunpcklqdq xmm3, xmm6, xmm7 // [1,a3,1,a2,1,a1,1,a0] + vpunpcklqdq xmm0, xmm0, xmm2 // [1,a7,1,a6,1,a5,1,a4] + vinserti128 ymm3, ymm3, xmm0, 1 // [1,a7,1,a6,1,a5,1,a4,1,a3,1,a2,1,a1,1,a0] + // end of VPGATHER + + vmovdqu ymm6, [eax] // read 8 pixels. + vpunpcklbw ymm0, ymm6, ymm6 // low 4 pixels. mutated. + vpunpckhbw ymm1, ymm6, ymm6 // high 4 pixels. mutated. + vpunpcklwd ymm2, ymm3, ymm3 // low 4 inverted alphas. mutated. 1, 1, a, a + vpunpckhwd ymm3, ymm3, ymm3 // high 4 inverted alphas. mutated. + vpshufb ymm2, ymm2, ymm5 // replicate low 4 alphas. 1, a, a, a + vpshufb ymm3, ymm3, ymm5 // replicate high 4 alphas + vpmulhuw ymm0, ymm0, ymm2 // rgb * ia + vpmulhuw ymm1, ymm1, ymm3 // rgb * ia + vpackuswb ymm0, ymm0, ymm1 // unmutated. + vmovdqu [eax + edx], ymm0 + lea eax, [eax + 32] + sub ecx, 8 + jg convertloop + + pop edi + pop esi + vzeroupper + ret + } +} +#endif // USE_GATHER +#endif // HAS_ARGBATTENUATEROW_AVX2 + +#ifdef HAS_ARGBGRAYROW_SSSE3 +// Convert 8 ARGB pixels (64 bytes) to 8 Gray ARGB pixels. +__declspec(naked) +void ARGBGrayRow_SSSE3(const uint8* src_argb, uint8* dst_argb, int width) { + __asm { + mov eax, [esp + 4] /* src_argb */ + mov edx, [esp + 8] /* dst_argb */ + mov ecx, [esp + 12] /* width */ + movdqa xmm4, kARGBToYJ + movdqa xmm5, kAddYJ64 + + convertloop: + movdqu xmm0, [eax] // G + movdqu xmm1, [eax + 16] + pmaddubsw xmm0, xmm4 + pmaddubsw xmm1, xmm4 + phaddw xmm0, xmm1 + paddw xmm0, xmm5 // Add .5 for rounding. + psrlw xmm0, 7 + packuswb xmm0, xmm0 // 8 G bytes + movdqu xmm2, [eax] // A + movdqu xmm3, [eax + 16] + lea eax, [eax + 32] + psrld xmm2, 24 + psrld xmm3, 24 + packuswb xmm2, xmm3 + packuswb xmm2, xmm2 // 8 A bytes + movdqa xmm3, xmm0 // Weave into GG, GA, then GGGA + punpcklbw xmm0, xmm0 // 8 GG words + punpcklbw xmm3, xmm2 // 8 GA words + movdqa xmm1, xmm0 + punpcklwd xmm0, xmm3 // GGGA first 4 + punpckhwd xmm1, xmm3 // GGGA next 4 + movdqu [edx], xmm0 + movdqu [edx + 16], xmm1 + lea edx, [edx + 32] + sub ecx, 8 + jg convertloop + ret + } +} +#endif // HAS_ARGBGRAYROW_SSSE3 + +#ifdef HAS_ARGBSEPIAROW_SSSE3 +// b = (r * 35 + g * 68 + b * 17) >> 7 +// g = (r * 45 + g * 88 + b * 22) >> 7 +// r = (r * 50 + g * 98 + b * 24) >> 7 +// Constant for ARGB color to sepia tone. +static const vec8 kARGBToSepiaB = { + 17, 68, 35, 0, 17, 68, 35, 0, 17, 68, 35, 0, 17, 68, 35, 0 +}; + +static const vec8 kARGBToSepiaG = { + 22, 88, 45, 0, 22, 88, 45, 0, 22, 88, 45, 0, 22, 88, 45, 0 +}; + +static const vec8 kARGBToSepiaR = { + 24, 98, 50, 0, 24, 98, 50, 0, 24, 98, 50, 0, 24, 98, 50, 0 +}; + +// Convert 8 ARGB pixels (32 bytes) to 8 Sepia ARGB pixels. +__declspec(naked) +void ARGBSepiaRow_SSSE3(uint8* dst_argb, int width) { + __asm { + mov eax, [esp + 4] /* dst_argb */ + mov ecx, [esp + 8] /* width */ + movdqa xmm2, kARGBToSepiaB + movdqa xmm3, kARGBToSepiaG + movdqa xmm4, kARGBToSepiaR + + convertloop: + movdqu xmm0, [eax] // B + movdqu xmm6, [eax + 16] + pmaddubsw xmm0, xmm2 + pmaddubsw xmm6, xmm2 + phaddw xmm0, xmm6 + psrlw xmm0, 7 + packuswb xmm0, xmm0 // 8 B values + movdqu xmm5, [eax] // G + movdqu xmm1, [eax + 16] + pmaddubsw xmm5, xmm3 + pmaddubsw xmm1, xmm3 + phaddw xmm5, xmm1 + psrlw xmm5, 7 + packuswb xmm5, xmm5 // 8 G values + punpcklbw xmm0, xmm5 // 8 BG values + movdqu xmm5, [eax] // R + movdqu xmm1, [eax + 16] + pmaddubsw xmm5, xmm4 + pmaddubsw xmm1, xmm4 + phaddw xmm5, xmm1 + psrlw xmm5, 7 + packuswb xmm5, xmm5 // 8 R values + movdqu xmm6, [eax] // A + movdqu xmm1, [eax + 16] + psrld xmm6, 24 + psrld xmm1, 24 + packuswb xmm6, xmm1 + packuswb xmm6, xmm6 // 8 A values + punpcklbw xmm5, xmm6 // 8 RA values + movdqa xmm1, xmm0 // Weave BG, RA together + punpcklwd xmm0, xmm5 // BGRA first 4 + punpckhwd xmm1, xmm5 // BGRA next 4 + movdqu [eax], xmm0 + movdqu [eax + 16], xmm1 + lea eax, [eax + 32] + sub ecx, 8 + jg convertloop + ret + } +} +#endif // HAS_ARGBSEPIAROW_SSSE3 + +#ifdef HAS_ARGBCOLORMATRIXROW_SSSE3 +// Tranform 8 ARGB pixels (32 bytes) with color matrix. +// Same as Sepia except matrix is provided. +// TODO(fbarchard): packuswbs only use half of the reg. To make RGBA, combine R +// and B into a high and low, then G/A, unpackl/hbw and then unpckl/hwd. +__declspec(naked) +void ARGBColorMatrixRow_SSSE3(const uint8* src_argb, uint8* dst_argb, + const int8* matrix_argb, int width) { + __asm { + mov eax, [esp + 4] /* src_argb */ + mov edx, [esp + 8] /* dst_argb */ + mov ecx, [esp + 12] /* matrix_argb */ + movdqu xmm5, [ecx] + pshufd xmm2, xmm5, 0x00 + pshufd xmm3, xmm5, 0x55 + pshufd xmm4, xmm5, 0xaa + pshufd xmm5, xmm5, 0xff + mov ecx, [esp + 16] /* width */ + + convertloop: + movdqu xmm0, [eax] // B + movdqu xmm7, [eax + 16] + pmaddubsw xmm0, xmm2 + pmaddubsw xmm7, xmm2 + movdqu xmm6, [eax] // G + movdqu xmm1, [eax + 16] + pmaddubsw xmm6, xmm3 + pmaddubsw xmm1, xmm3 + phaddsw xmm0, xmm7 // B + phaddsw xmm6, xmm1 // G + psraw xmm0, 6 // B + psraw xmm6, 6 // G + packuswb xmm0, xmm0 // 8 B values + packuswb xmm6, xmm6 // 8 G values + punpcklbw xmm0, xmm6 // 8 BG values + movdqu xmm1, [eax] // R + movdqu xmm7, [eax + 16] + pmaddubsw xmm1, xmm4 + pmaddubsw xmm7, xmm4 + phaddsw xmm1, xmm7 // R + movdqu xmm6, [eax] // A + movdqu xmm7, [eax + 16] + pmaddubsw xmm6, xmm5 + pmaddubsw xmm7, xmm5 + phaddsw xmm6, xmm7 // A + psraw xmm1, 6 // R + psraw xmm6, 6 // A + packuswb xmm1, xmm1 // 8 R values + packuswb xmm6, xmm6 // 8 A values + punpcklbw xmm1, xmm6 // 8 RA values + movdqa xmm6, xmm0 // Weave BG, RA together + punpcklwd xmm0, xmm1 // BGRA first 4 + punpckhwd xmm6, xmm1 // BGRA next 4 + movdqu [edx], xmm0 + movdqu [edx + 16], xmm6 + lea eax, [eax + 32] + lea edx, [edx + 32] + sub ecx, 8 + jg convertloop + ret + } +} +#endif // HAS_ARGBCOLORMATRIXROW_SSSE3 + +#ifdef HAS_ARGBQUANTIZEROW_SSE2 +// Quantize 4 ARGB pixels (16 bytes). +__declspec(naked) +void ARGBQuantizeRow_SSE2(uint8* dst_argb, int scale, int interval_size, + int interval_offset, int width) { + __asm { + mov eax, [esp + 4] /* dst_argb */ + movd xmm2, [esp + 8] /* scale */ + movd xmm3, [esp + 12] /* interval_size */ + movd xmm4, [esp + 16] /* interval_offset */ + mov ecx, [esp + 20] /* width */ + pshuflw xmm2, xmm2, 040h + pshufd xmm2, xmm2, 044h + pshuflw xmm3, xmm3, 040h + pshufd xmm3, xmm3, 044h + pshuflw xmm4, xmm4, 040h + pshufd xmm4, xmm4, 044h + pxor xmm5, xmm5 // constant 0 + pcmpeqb xmm6, xmm6 // generate mask 0xff000000 + pslld xmm6, 24 + + convertloop: + movdqu xmm0, [eax] // read 4 pixels + punpcklbw xmm0, xmm5 // first 2 pixels + pmulhuw xmm0, xmm2 // pixel * scale >> 16 + movdqu xmm1, [eax] // read 4 pixels + punpckhbw xmm1, xmm5 // next 2 pixels + pmulhuw xmm1, xmm2 + pmullw xmm0, xmm3 // * interval_size + movdqu xmm7, [eax] // read 4 pixels + pmullw xmm1, xmm3 + pand xmm7, xmm6 // mask alpha + paddw xmm0, xmm4 // + interval_size / 2 + paddw xmm1, xmm4 + packuswb xmm0, xmm1 + por xmm0, xmm7 + movdqu [eax], xmm0 + lea eax, [eax + 16] + sub ecx, 4 + jg convertloop + ret + } +} +#endif // HAS_ARGBQUANTIZEROW_SSE2 + +#ifdef HAS_ARGBSHADEROW_SSE2 +// Shade 4 pixels at a time by specified value. +__declspec(naked) +void ARGBShadeRow_SSE2(const uint8* src_argb, uint8* dst_argb, int width, + uint32 value) { + __asm { + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // width + movd xmm2, [esp + 16] // value + punpcklbw xmm2, xmm2 + punpcklqdq xmm2, xmm2 + + convertloop: + movdqu xmm0, [eax] // read 4 pixels + lea eax, [eax + 16] + movdqa xmm1, xmm0 + punpcklbw xmm0, xmm0 // first 2 + punpckhbw xmm1, xmm1 // next 2 + pmulhuw xmm0, xmm2 // argb * value + pmulhuw xmm1, xmm2 // argb * value + psrlw xmm0, 8 + psrlw xmm1, 8 + packuswb xmm0, xmm1 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg convertloop + + ret + } +} +#endif // HAS_ARGBSHADEROW_SSE2 + +#ifdef HAS_ARGBMULTIPLYROW_SSE2 +// Multiply 2 rows of ARGB pixels together, 4 pixels at a time. +__declspec(naked) +void ARGBMultiplyRow_SSE2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_argb0 + mov esi, [esp + 4 + 8] // src_argb1 + mov edx, [esp + 4 + 12] // dst_argb + mov ecx, [esp + 4 + 16] // width + pxor xmm5, xmm5 // constant 0 + + convertloop: + movdqu xmm0, [eax] // read 4 pixels from src_argb0 + movdqu xmm2, [esi] // read 4 pixels from src_argb1 + movdqu xmm1, xmm0 + movdqu xmm3, xmm2 + punpcklbw xmm0, xmm0 // first 2 + punpckhbw xmm1, xmm1 // next 2 + punpcklbw xmm2, xmm5 // first 2 + punpckhbw xmm3, xmm5 // next 2 + pmulhuw xmm0, xmm2 // src_argb0 * src_argb1 first 2 + pmulhuw xmm1, xmm3 // src_argb0 * src_argb1 next 2 + lea eax, [eax + 16] + lea esi, [esi + 16] + packuswb xmm0, xmm1 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg convertloop + + pop esi + ret + } +} +#endif // HAS_ARGBMULTIPLYROW_SSE2 + +#ifdef HAS_ARGBADDROW_SSE2 +// Add 2 rows of ARGB pixels together, 4 pixels at a time. +// TODO(fbarchard): Port this to posix, neon and other math functions. +__declspec(naked) +void ARGBAddRow_SSE2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_argb0 + mov esi, [esp + 4 + 8] // src_argb1 + mov edx, [esp + 4 + 12] // dst_argb + mov ecx, [esp + 4 + 16] // width + + sub ecx, 4 + jl convertloop49 + + convertloop4: + movdqu xmm0, [eax] // read 4 pixels from src_argb0 + lea eax, [eax + 16] + movdqu xmm1, [esi] // read 4 pixels from src_argb1 + lea esi, [esi + 16] + paddusb xmm0, xmm1 // src_argb0 + src_argb1 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jge convertloop4 + + convertloop49: + add ecx, 4 - 1 + jl convertloop19 + + convertloop1: + movd xmm0, [eax] // read 1 pixels from src_argb0 + lea eax, [eax + 4] + movd xmm1, [esi] // read 1 pixels from src_argb1 + lea esi, [esi + 4] + paddusb xmm0, xmm1 // src_argb0 + src_argb1 + movd [edx], xmm0 + lea edx, [edx + 4] + sub ecx, 1 + jge convertloop1 + + convertloop19: + pop esi + ret + } +} +#endif // HAS_ARGBADDROW_SSE2 + +#ifdef HAS_ARGBSUBTRACTROW_SSE2 +// Subtract 2 rows of ARGB pixels together, 4 pixels at a time. +__declspec(naked) +void ARGBSubtractRow_SSE2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_argb0 + mov esi, [esp + 4 + 8] // src_argb1 + mov edx, [esp + 4 + 12] // dst_argb + mov ecx, [esp + 4 + 16] // width + + convertloop: + movdqu xmm0, [eax] // read 4 pixels from src_argb0 + lea eax, [eax + 16] + movdqu xmm1, [esi] // read 4 pixels from src_argb1 + lea esi, [esi + 16] + psubusb xmm0, xmm1 // src_argb0 - src_argb1 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg convertloop + + pop esi + ret + } +} +#endif // HAS_ARGBSUBTRACTROW_SSE2 + +#ifdef HAS_ARGBMULTIPLYROW_AVX2 +// Multiply 2 rows of ARGB pixels together, 8 pixels at a time. +__declspec(naked) +void ARGBMultiplyRow_AVX2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_argb0 + mov esi, [esp + 4 + 8] // src_argb1 + mov edx, [esp + 4 + 12] // dst_argb + mov ecx, [esp + 4 + 16] // width + vpxor ymm5, ymm5, ymm5 // constant 0 + + convertloop: + vmovdqu ymm1, [eax] // read 8 pixels from src_argb0 + lea eax, [eax + 32] + vmovdqu ymm3, [esi] // read 8 pixels from src_argb1 + lea esi, [esi + 32] + vpunpcklbw ymm0, ymm1, ymm1 // low 4 + vpunpckhbw ymm1, ymm1, ymm1 // high 4 + vpunpcklbw ymm2, ymm3, ymm5 // low 4 + vpunpckhbw ymm3, ymm3, ymm5 // high 4 + vpmulhuw ymm0, ymm0, ymm2 // src_argb0 * src_argb1 low 4 + vpmulhuw ymm1, ymm1, ymm3 // src_argb0 * src_argb1 high 4 + vpackuswb ymm0, ymm0, ymm1 + vmovdqu [edx], ymm0 + lea edx, [edx + 32] + sub ecx, 8 + jg convertloop + + pop esi + vzeroupper + ret + } +} +#endif // HAS_ARGBMULTIPLYROW_AVX2 + +#ifdef HAS_ARGBADDROW_AVX2 +// Add 2 rows of ARGB pixels together, 8 pixels at a time. +__declspec(naked) +void ARGBAddRow_AVX2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_argb0 + mov esi, [esp + 4 + 8] // src_argb1 + mov edx, [esp + 4 + 12] // dst_argb + mov ecx, [esp + 4 + 16] // width + + convertloop: + vmovdqu ymm0, [eax] // read 8 pixels from src_argb0 + lea eax, [eax + 32] + vpaddusb ymm0, ymm0, [esi] // add 8 pixels from src_argb1 + lea esi, [esi + 32] + vmovdqu [edx], ymm0 + lea edx, [edx + 32] + sub ecx, 8 + jg convertloop + + pop esi + vzeroupper + ret + } +} +#endif // HAS_ARGBADDROW_AVX2 + +#ifdef HAS_ARGBSUBTRACTROW_AVX2 +// Subtract 2 rows of ARGB pixels together, 8 pixels at a time. +__declspec(naked) +void ARGBSubtractRow_AVX2(const uint8* src_argb0, const uint8* src_argb1, + uint8* dst_argb, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_argb0 + mov esi, [esp + 4 + 8] // src_argb1 + mov edx, [esp + 4 + 12] // dst_argb + mov ecx, [esp + 4 + 16] // width + + convertloop: + vmovdqu ymm0, [eax] // read 8 pixels from src_argb0 + lea eax, [eax + 32] + vpsubusb ymm0, ymm0, [esi] // src_argb0 - src_argb1 + lea esi, [esi + 32] + vmovdqu [edx], ymm0 + lea edx, [edx + 32] + sub ecx, 8 + jg convertloop + + pop esi + vzeroupper + ret + } +} +#endif // HAS_ARGBSUBTRACTROW_AVX2 + +#ifdef HAS_SOBELXROW_SSE2 +// SobelX as a matrix is +// -1 0 1 +// -2 0 2 +// -1 0 1 +__declspec(naked) +void SobelXRow_SSE2(const uint8* src_y0, const uint8* src_y1, + const uint8* src_y2, uint8* dst_sobelx, int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_y0 + mov esi, [esp + 8 + 8] // src_y1 + mov edi, [esp + 8 + 12] // src_y2 + mov edx, [esp + 8 + 16] // dst_sobelx + mov ecx, [esp + 8 + 20] // width + sub esi, eax + sub edi, eax + sub edx, eax + pxor xmm5, xmm5 // constant 0 + + convertloop: + movq xmm0, qword ptr [eax] // read 8 pixels from src_y0[0] + movq xmm1, qword ptr [eax + 2] // read 8 pixels from src_y0[2] + punpcklbw xmm0, xmm5 + punpcklbw xmm1, xmm5 + psubw xmm0, xmm1 + movq xmm1, qword ptr [eax + esi] // read 8 pixels from src_y1[0] + movq xmm2, qword ptr [eax + esi + 2] // read 8 pixels from src_y1[2] + punpcklbw xmm1, xmm5 + punpcklbw xmm2, xmm5 + psubw xmm1, xmm2 + movq xmm2, qword ptr [eax + edi] // read 8 pixels from src_y2[0] + movq xmm3, qword ptr [eax + edi + 2] // read 8 pixels from src_y2[2] + punpcklbw xmm2, xmm5 + punpcklbw xmm3, xmm5 + psubw xmm2, xmm3 + paddw xmm0, xmm2 + paddw xmm0, xmm1 + paddw xmm0, xmm1 + pxor xmm1, xmm1 // abs = max(xmm0, -xmm0). SSSE3 could use pabsw + psubw xmm1, xmm0 + pmaxsw xmm0, xmm1 + packuswb xmm0, xmm0 + movq qword ptr [eax + edx], xmm0 + lea eax, [eax + 8] + sub ecx, 8 + jg convertloop + + pop edi + pop esi + ret + } +} +#endif // HAS_SOBELXROW_SSE2 + +#ifdef HAS_SOBELYROW_SSE2 +// SobelY as a matrix is +// -1 -2 -1 +// 0 0 0 +// 1 2 1 +__declspec(naked) +void SobelYRow_SSE2(const uint8* src_y0, const uint8* src_y1, + uint8* dst_sobely, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_y0 + mov esi, [esp + 4 + 8] // src_y1 + mov edx, [esp + 4 + 12] // dst_sobely + mov ecx, [esp + 4 + 16] // width + sub esi, eax + sub edx, eax + pxor xmm5, xmm5 // constant 0 + + convertloop: + movq xmm0, qword ptr [eax] // read 8 pixels from src_y0[0] + movq xmm1, qword ptr [eax + esi] // read 8 pixels from src_y1[0] + punpcklbw xmm0, xmm5 + punpcklbw xmm1, xmm5 + psubw xmm0, xmm1 + movq xmm1, qword ptr [eax + 1] // read 8 pixels from src_y0[1] + movq xmm2, qword ptr [eax + esi + 1] // read 8 pixels from src_y1[1] + punpcklbw xmm1, xmm5 + punpcklbw xmm2, xmm5 + psubw xmm1, xmm2 + movq xmm2, qword ptr [eax + 2] // read 8 pixels from src_y0[2] + movq xmm3, qword ptr [eax + esi + 2] // read 8 pixels from src_y1[2] + punpcklbw xmm2, xmm5 + punpcklbw xmm3, xmm5 + psubw xmm2, xmm3 + paddw xmm0, xmm2 + paddw xmm0, xmm1 + paddw xmm0, xmm1 + pxor xmm1, xmm1 // abs = max(xmm0, -xmm0). SSSE3 could use pabsw + psubw xmm1, xmm0 + pmaxsw xmm0, xmm1 + packuswb xmm0, xmm0 + movq qword ptr [eax + edx], xmm0 + lea eax, [eax + 8] + sub ecx, 8 + jg convertloop + + pop esi + ret + } +} +#endif // HAS_SOBELYROW_SSE2 + +#ifdef HAS_SOBELROW_SSE2 +// Adds Sobel X and Sobel Y and stores Sobel into ARGB. +// A = 255 +// R = Sobel +// G = Sobel +// B = Sobel +__declspec(naked) +void SobelRow_SSE2(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_sobelx + mov esi, [esp + 4 + 8] // src_sobely + mov edx, [esp + 4 + 12] // dst_argb + mov ecx, [esp + 4 + 16] // width + sub esi, eax + pcmpeqb xmm5, xmm5 // alpha 255 + pslld xmm5, 24 // 0xff000000 + + convertloop: + movdqu xmm0, [eax] // read 16 pixels src_sobelx + movdqu xmm1, [eax + esi] // read 16 pixels src_sobely + lea eax, [eax + 16] + paddusb xmm0, xmm1 // sobel = sobelx + sobely + movdqa xmm2, xmm0 // GG + punpcklbw xmm2, xmm0 // First 8 + punpckhbw xmm0, xmm0 // Next 8 + movdqa xmm1, xmm2 // GGGG + punpcklwd xmm1, xmm2 // First 4 + punpckhwd xmm2, xmm2 // Next 4 + por xmm1, xmm5 // GGGA + por xmm2, xmm5 + movdqa xmm3, xmm0 // GGGG + punpcklwd xmm3, xmm0 // Next 4 + punpckhwd xmm0, xmm0 // Last 4 + por xmm3, xmm5 // GGGA + por xmm0, xmm5 + movdqu [edx], xmm1 + movdqu [edx + 16], xmm2 + movdqu [edx + 32], xmm3 + movdqu [edx + 48], xmm0 + lea edx, [edx + 64] + sub ecx, 16 + jg convertloop + + pop esi + ret + } +} +#endif // HAS_SOBELROW_SSE2 + +#ifdef HAS_SOBELTOPLANEROW_SSE2 +// Adds Sobel X and Sobel Y and stores Sobel into a plane. +__declspec(naked) +void SobelToPlaneRow_SSE2(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_y, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_sobelx + mov esi, [esp + 4 + 8] // src_sobely + mov edx, [esp + 4 + 12] // dst_argb + mov ecx, [esp + 4 + 16] // width + sub esi, eax + + convertloop: + movdqu xmm0, [eax] // read 16 pixels src_sobelx + movdqu xmm1, [eax + esi] // read 16 pixels src_sobely + lea eax, [eax + 16] + paddusb xmm0, xmm1 // sobel = sobelx + sobely + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg convertloop + + pop esi + ret + } +} +#endif // HAS_SOBELTOPLANEROW_SSE2 + +#ifdef HAS_SOBELXYROW_SSE2 +// Mixes Sobel X, Sobel Y and Sobel into ARGB. +// A = 255 +// R = Sobel X +// G = Sobel +// B = Sobel Y +__declspec(naked) +void SobelXYRow_SSE2(const uint8* src_sobelx, const uint8* src_sobely, + uint8* dst_argb, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_sobelx + mov esi, [esp + 4 + 8] // src_sobely + mov edx, [esp + 4 + 12] // dst_argb + mov ecx, [esp + 4 + 16] // width + sub esi, eax + pcmpeqb xmm5, xmm5 // alpha 255 + + convertloop: + movdqu xmm0, [eax] // read 16 pixels src_sobelx + movdqu xmm1, [eax + esi] // read 16 pixels src_sobely + lea eax, [eax + 16] + movdqa xmm2, xmm0 + paddusb xmm2, xmm1 // sobel = sobelx + sobely + movdqa xmm3, xmm0 // XA + punpcklbw xmm3, xmm5 + punpckhbw xmm0, xmm5 + movdqa xmm4, xmm1 // YS + punpcklbw xmm4, xmm2 + punpckhbw xmm1, xmm2 + movdqa xmm6, xmm4 // YSXA + punpcklwd xmm6, xmm3 // First 4 + punpckhwd xmm4, xmm3 // Next 4 + movdqa xmm7, xmm1 // YSXA + punpcklwd xmm7, xmm0 // Next 4 + punpckhwd xmm1, xmm0 // Last 4 + movdqu [edx], xmm6 + movdqu [edx + 16], xmm4 + movdqu [edx + 32], xmm7 + movdqu [edx + 48], xmm1 + lea edx, [edx + 64] + sub ecx, 16 + jg convertloop + + pop esi + ret + } +} +#endif // HAS_SOBELXYROW_SSE2 + +#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 +// Consider float CumulativeSum. +// Consider calling CumulativeSum one row at time as needed. +// Consider circular CumulativeSum buffer of radius * 2 + 1 height. +// Convert cumulative sum for an area to an average for 1 pixel. +// topleft is pointer to top left of CumulativeSum buffer for area. +// botleft is pointer to bottom left of CumulativeSum buffer. +// width is offset from left to right of area in CumulativeSum buffer measured +// in number of ints. +// area is the number of pixels in the area being averaged. +// dst points to pixel to store result to. +// count is number of averaged pixels to produce. +// Does 4 pixels at a time. +void CumulativeSumToAverageRow_SSE2(const int32* topleft, const int32* botleft, + int width, int area, uint8* dst, + int count) { + __asm { + mov eax, topleft // eax topleft + mov esi, botleft // esi botleft + mov edx, width + movd xmm5, area + mov edi, dst + mov ecx, count + cvtdq2ps xmm5, xmm5 + rcpss xmm4, xmm5 // 1.0f / area + pshufd xmm4, xmm4, 0 + sub ecx, 4 + jl l4b + + cmp area, 128 // 128 pixels will not overflow 15 bits. + ja l4 + + pshufd xmm5, xmm5, 0 // area + pcmpeqb xmm6, xmm6 // constant of 65536.0 - 1 = 65535.0 + psrld xmm6, 16 + cvtdq2ps xmm6, xmm6 + addps xmm5, xmm6 // (65536.0 + area - 1) + mulps xmm5, xmm4 // (65536.0 + area - 1) * 1 / area + cvtps2dq xmm5, xmm5 // 0.16 fixed point + packssdw xmm5, xmm5 // 16 bit shorts + + // 4 pixel loop small blocks. + s4: + // top left + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + 32] + movdqu xmm3, [eax + 48] + + // - top right + psubd xmm0, [eax + edx * 4] + psubd xmm1, [eax + edx * 4 + 16] + psubd xmm2, [eax + edx * 4 + 32] + psubd xmm3, [eax + edx * 4 + 48] + lea eax, [eax + 64] + + // - bottom left + psubd xmm0, [esi] + psubd xmm1, [esi + 16] + psubd xmm2, [esi + 32] + psubd xmm3, [esi + 48] + + // + bottom right + paddd xmm0, [esi + edx * 4] + paddd xmm1, [esi + edx * 4 + 16] + paddd xmm2, [esi + edx * 4 + 32] + paddd xmm3, [esi + edx * 4 + 48] + lea esi, [esi + 64] + + packssdw xmm0, xmm1 // pack 4 pixels into 2 registers + packssdw xmm2, xmm3 + + pmulhuw xmm0, xmm5 + pmulhuw xmm2, xmm5 + + packuswb xmm0, xmm2 + movdqu [edi], xmm0 + lea edi, [edi + 16] + sub ecx, 4 + jge s4 + + jmp l4b + + // 4 pixel loop + l4: + // top left + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + 32] + movdqu xmm3, [eax + 48] + + // - top right + psubd xmm0, [eax + edx * 4] + psubd xmm1, [eax + edx * 4 + 16] + psubd xmm2, [eax + edx * 4 + 32] + psubd xmm3, [eax + edx * 4 + 48] + lea eax, [eax + 64] + + // - bottom left + psubd xmm0, [esi] + psubd xmm1, [esi + 16] + psubd xmm2, [esi + 32] + psubd xmm3, [esi + 48] + + // + bottom right + paddd xmm0, [esi + edx * 4] + paddd xmm1, [esi + edx * 4 + 16] + paddd xmm2, [esi + edx * 4 + 32] + paddd xmm3, [esi + edx * 4 + 48] + lea esi, [esi + 64] + + cvtdq2ps xmm0, xmm0 // Average = Sum * 1 / Area + cvtdq2ps xmm1, xmm1 + mulps xmm0, xmm4 + mulps xmm1, xmm4 + cvtdq2ps xmm2, xmm2 + cvtdq2ps xmm3, xmm3 + mulps xmm2, xmm4 + mulps xmm3, xmm4 + cvtps2dq xmm0, xmm0 + cvtps2dq xmm1, xmm1 + cvtps2dq xmm2, xmm2 + cvtps2dq xmm3, xmm3 + packssdw xmm0, xmm1 + packssdw xmm2, xmm3 + packuswb xmm0, xmm2 + movdqu [edi], xmm0 + lea edi, [edi + 16] + sub ecx, 4 + jge l4 + + l4b: + add ecx, 4 - 1 + jl l1b + + // 1 pixel loop + l1: + movdqu xmm0, [eax] + psubd xmm0, [eax + edx * 4] + lea eax, [eax + 16] + psubd xmm0, [esi] + paddd xmm0, [esi + edx * 4] + lea esi, [esi + 16] + cvtdq2ps xmm0, xmm0 + mulps xmm0, xmm4 + cvtps2dq xmm0, xmm0 + packssdw xmm0, xmm0 + packuswb xmm0, xmm0 + movd dword ptr [edi], xmm0 + lea edi, [edi + 4] + sub ecx, 1 + jge l1 + l1b: + } +} +#endif // HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 + +#ifdef HAS_COMPUTECUMULATIVESUMROW_SSE2 +// Creates a table of cumulative sums where each value is a sum of all values +// above and to the left of the value. +void ComputeCumulativeSumRow_SSE2(const uint8* row, int32* cumsum, + const int32* previous_cumsum, int width) { + __asm { + mov eax, row + mov edx, cumsum + mov esi, previous_cumsum + mov ecx, width + pxor xmm0, xmm0 + pxor xmm1, xmm1 + + sub ecx, 4 + jl l4b + test edx, 15 + jne l4b + + // 4 pixel loop + l4: + movdqu xmm2, [eax] // 4 argb pixels 16 bytes. + lea eax, [eax + 16] + movdqa xmm4, xmm2 + + punpcklbw xmm2, xmm1 + movdqa xmm3, xmm2 + punpcklwd xmm2, xmm1 + punpckhwd xmm3, xmm1 + + punpckhbw xmm4, xmm1 + movdqa xmm5, xmm4 + punpcklwd xmm4, xmm1 + punpckhwd xmm5, xmm1 + + paddd xmm0, xmm2 + movdqu xmm2, [esi] // previous row above. + paddd xmm2, xmm0 + + paddd xmm0, xmm3 + movdqu xmm3, [esi + 16] + paddd xmm3, xmm0 + + paddd xmm0, xmm4 + movdqu xmm4, [esi + 32] + paddd xmm4, xmm0 + + paddd xmm0, xmm5 + movdqu xmm5, [esi + 48] + lea esi, [esi + 64] + paddd xmm5, xmm0 + + movdqu [edx], xmm2 + movdqu [edx + 16], xmm3 + movdqu [edx + 32], xmm4 + movdqu [edx + 48], xmm5 + + lea edx, [edx + 64] + sub ecx, 4 + jge l4 + + l4b: + add ecx, 4 - 1 + jl l1b + + // 1 pixel loop + l1: + movd xmm2, dword ptr [eax] // 1 argb pixel 4 bytes. + lea eax, [eax + 4] + punpcklbw xmm2, xmm1 + punpcklwd xmm2, xmm1 + paddd xmm0, xmm2 + movdqu xmm2, [esi] + lea esi, [esi + 16] + paddd xmm2, xmm0 + movdqu [edx], xmm2 + lea edx, [edx + 16] + sub ecx, 1 + jge l1 + + l1b: + } +} +#endif // HAS_COMPUTECUMULATIVESUMROW_SSE2 + +#ifdef HAS_ARGBAFFINEROW_SSE2 +// Copy ARGB pixels from source image with slope to a row of destination. +__declspec(naked) +LIBYUV_API +void ARGBAffineRow_SSE2(const uint8* src_argb, int src_argb_stride, + uint8* dst_argb, const float* uv_dudv, int width) { + __asm { + push esi + push edi + mov eax, [esp + 12] // src_argb + mov esi, [esp + 16] // stride + mov edx, [esp + 20] // dst_argb + mov ecx, [esp + 24] // pointer to uv_dudv + movq xmm2, qword ptr [ecx] // uv + movq xmm7, qword ptr [ecx + 8] // dudv + mov ecx, [esp + 28] // width + shl esi, 16 // 4, stride + add esi, 4 + movd xmm5, esi + sub ecx, 4 + jl l4b + + // setup for 4 pixel loop + pshufd xmm7, xmm7, 0x44 // dup dudv + pshufd xmm5, xmm5, 0 // dup 4, stride + movdqa xmm0, xmm2 // x0, y0, x1, y1 + addps xmm0, xmm7 + movlhps xmm2, xmm0 + movdqa xmm4, xmm7 + addps xmm4, xmm4 // dudv *= 2 + movdqa xmm3, xmm2 // x2, y2, x3, y3 + addps xmm3, xmm4 + addps xmm4, xmm4 // dudv *= 4 + + // 4 pixel loop + l4: + cvttps2dq xmm0, xmm2 // x, y float to int first 2 + cvttps2dq xmm1, xmm3 // x, y float to int next 2 + packssdw xmm0, xmm1 // x, y as 8 shorts + pmaddwd xmm0, xmm5 // offsets = x * 4 + y * stride. + movd esi, xmm0 + pshufd xmm0, xmm0, 0x39 // shift right + movd edi, xmm0 + pshufd xmm0, xmm0, 0x39 // shift right + movd xmm1, [eax + esi] // read pixel 0 + movd xmm6, [eax + edi] // read pixel 1 + punpckldq xmm1, xmm6 // combine pixel 0 and 1 + addps xmm2, xmm4 // x, y += dx, dy first 2 + movq qword ptr [edx], xmm1 + movd esi, xmm0 + pshufd xmm0, xmm0, 0x39 // shift right + movd edi, xmm0 + movd xmm6, [eax + esi] // read pixel 2 + movd xmm0, [eax + edi] // read pixel 3 + punpckldq xmm6, xmm0 // combine pixel 2 and 3 + addps xmm3, xmm4 // x, y += dx, dy next 2 + movq qword ptr 8[edx], xmm6 + lea edx, [edx + 16] + sub ecx, 4 + jge l4 + + l4b: + add ecx, 4 - 1 + jl l1b + + // 1 pixel loop + l1: + cvttps2dq xmm0, xmm2 // x, y float to int + packssdw xmm0, xmm0 // x, y as shorts + pmaddwd xmm0, xmm5 // offset = x * 4 + y * stride + addps xmm2, xmm7 // x, y += dx, dy + movd esi, xmm0 + movd xmm0, [eax + esi] // copy a pixel + movd [edx], xmm0 + lea edx, [edx + 4] + sub ecx, 1 + jge l1 + l1b: + pop edi + pop esi + ret + } +} +#endif // HAS_ARGBAFFINEROW_SSE2 + +#ifdef HAS_INTERPOLATEROW_AVX2 +// Bilinear filter 32x2 -> 32x1 +__declspec(naked) +void InterpolateRow_AVX2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) { + __asm { + push esi + push edi + mov edi, [esp + 8 + 4] // dst_ptr + mov esi, [esp + 8 + 8] // src_ptr + mov edx, [esp + 8 + 12] // src_stride + mov ecx, [esp + 8 + 16] // dst_width + mov eax, [esp + 8 + 20] // source_y_fraction (0..255) + shr eax, 1 + // Dispatch to specialized filters if applicable. + cmp eax, 0 + je xloop100 // 0 / 128. Blend 100 / 0. + sub edi, esi + cmp eax, 32 + je xloop75 // 32 / 128 is 0.25. Blend 75 / 25. + cmp eax, 64 + je xloop50 // 64 / 128 is 0.50. Blend 50 / 50. + cmp eax, 96 + je xloop25 // 96 / 128 is 0.75. Blend 25 / 75. + + vmovd xmm0, eax // high fraction 0..127 + neg eax + add eax, 128 + vmovd xmm5, eax // low fraction 128..1 + vpunpcklbw xmm5, xmm5, xmm0 + vpunpcklwd xmm5, xmm5, xmm5 + vpxor ymm0, ymm0, ymm0 + vpermd ymm5, ymm0, ymm5 + + xloop: + vmovdqu ymm0, [esi] + vmovdqu ymm2, [esi + edx] + vpunpckhbw ymm1, ymm0, ymm2 // mutates + vpunpcklbw ymm0, ymm0, ymm2 // mutates + vpmaddubsw ymm0, ymm0, ymm5 + vpmaddubsw ymm1, ymm1, ymm5 + vpsrlw ymm0, ymm0, 7 + vpsrlw ymm1, ymm1, 7 + vpackuswb ymm0, ymm0, ymm1 // unmutates + vmovdqu [esi + edi], ymm0 + lea esi, [esi + 32] + sub ecx, 32 + jg xloop + jmp xloop99 + + // Blend 25 / 75. + xloop25: + vmovdqu ymm0, [esi] + vmovdqu ymm1, [esi + edx] + vpavgb ymm0, ymm0, ymm1 + vpavgb ymm0, ymm0, ymm1 + vmovdqu [esi + edi], ymm0 + lea esi, [esi + 32] + sub ecx, 32 + jg xloop25 + jmp xloop99 + + // Blend 50 / 50. + xloop50: + vmovdqu ymm0, [esi] + vpavgb ymm0, ymm0, [esi + edx] + vmovdqu [esi + edi], ymm0 + lea esi, [esi + 32] + sub ecx, 32 + jg xloop50 + jmp xloop99 + + // Blend 75 / 25. + xloop75: + vmovdqu ymm1, [esi] + vmovdqu ymm0, [esi + edx] + vpavgb ymm0, ymm0, ymm1 + vpavgb ymm0, ymm0, ymm1 + vmovdqu [esi + edi], ymm0 + lea esi, [esi + 32] + sub ecx, 32 + jg xloop75 + jmp xloop99 + + // Blend 100 / 0 - Copy row unchanged. + xloop100: + rep movsb + + xloop99: + pop edi + pop esi + vzeroupper + ret + } +} +#endif // HAS_INTERPOLATEROW_AVX2 + +// Bilinear filter 16x2 -> 16x1 +__declspec(naked) +void InterpolateRow_SSSE3(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) { + __asm { + push esi + push edi + mov edi, [esp + 8 + 4] // dst_ptr + mov esi, [esp + 8 + 8] // src_ptr + mov edx, [esp + 8 + 12] // src_stride + mov ecx, [esp + 8 + 16] // dst_width + mov eax, [esp + 8 + 20] // source_y_fraction (0..255) + sub edi, esi + shr eax, 1 + // Dispatch to specialized filters if applicable. + cmp eax, 0 + je xloop100 // 0 / 128. Blend 100 / 0. + cmp eax, 32 + je xloop75 // 32 / 128 is 0.25. Blend 75 / 25. + cmp eax, 64 + je xloop50 // 64 / 128 is 0.50. Blend 50 / 50. + cmp eax, 96 + je xloop25 // 96 / 128 is 0.75. Blend 25 / 75. + + movd xmm0, eax // high fraction 0..127 + neg eax + add eax, 128 + movd xmm5, eax // low fraction 128..1 + punpcklbw xmm5, xmm0 + punpcklwd xmm5, xmm5 + pshufd xmm5, xmm5, 0 + + xloop: + movdqu xmm0, [esi] + movdqu xmm2, [esi + edx] + movdqu xmm1, xmm0 + punpcklbw xmm0, xmm2 + punpckhbw xmm1, xmm2 + pmaddubsw xmm0, xmm5 + pmaddubsw xmm1, xmm5 + psrlw xmm0, 7 + psrlw xmm1, 7 + packuswb xmm0, xmm1 + movdqu [esi + edi], xmm0 + lea esi, [esi + 16] + sub ecx, 16 + jg xloop + jmp xloop99 + + // Blend 25 / 75. + xloop25: + movdqu xmm0, [esi] + movdqu xmm1, [esi + edx] + pavgb xmm0, xmm1 + pavgb xmm0, xmm1 + movdqu [esi + edi], xmm0 + lea esi, [esi + 16] + sub ecx, 16 + jg xloop25 + jmp xloop99 + + // Blend 50 / 50. + xloop50: + movdqu xmm0, [esi] + movdqu xmm1, [esi + edx] + pavgb xmm0, xmm1 + movdqu [esi + edi], xmm0 + lea esi, [esi + 16] + sub ecx, 16 + jg xloop50 + jmp xloop99 + + // Blend 75 / 25. + xloop75: + movdqu xmm1, [esi] + movdqu xmm0, [esi + edx] + pavgb xmm0, xmm1 + pavgb xmm0, xmm1 + movdqu [esi + edi], xmm0 + lea esi, [esi + 16] + sub ecx, 16 + jg xloop75 + jmp xloop99 + + // Blend 100 / 0 - Copy row unchanged. + xloop100: + movdqu xmm0, [esi] + movdqu [esi + edi], xmm0 + lea esi, [esi + 16] + sub ecx, 16 + jg xloop100 + + xloop99: + pop edi + pop esi + ret + } +} + +#ifdef HAS_INTERPOLATEROW_SSE2 +// Bilinear filter 16x2 -> 16x1 +__declspec(naked) +void InterpolateRow_SSE2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) { + __asm { + push esi + push edi + mov edi, [esp + 8 + 4] // dst_ptr + mov esi, [esp + 8 + 8] // src_ptr + mov edx, [esp + 8 + 12] // src_stride + mov ecx, [esp + 8 + 16] // dst_width + mov eax, [esp + 8 + 20] // source_y_fraction (0..255) + sub edi, esi + // Dispatch to specialized filters if applicable. + cmp eax, 0 + je xloop100 // 0 / 256. Blend 100 / 0. + cmp eax, 64 + je xloop75 // 64 / 256 is 0.25. Blend 75 / 25. + cmp eax, 128 + je xloop50 // 128 / 256 is 0.50. Blend 50 / 50. + cmp eax, 192 + je xloop25 // 192 / 256 is 0.75. Blend 25 / 75. + + movd xmm5, eax // xmm5 = y fraction + punpcklbw xmm5, xmm5 + psrlw xmm5, 1 + punpcklwd xmm5, xmm5 + punpckldq xmm5, xmm5 + punpcklqdq xmm5, xmm5 + pxor xmm4, xmm4 + + xloop: + movdqu xmm0, [esi] // row0 + movdqu xmm2, [esi + edx] // row1 + movdqu xmm1, xmm0 + movdqu xmm3, xmm2 + punpcklbw xmm2, xmm4 + punpckhbw xmm3, xmm4 + punpcklbw xmm0, xmm4 + punpckhbw xmm1, xmm4 + psubw xmm2, xmm0 // row1 - row0 + psubw xmm3, xmm1 + paddw xmm2, xmm2 // 9 bits * 15 bits = 8.16 + paddw xmm3, xmm3 + pmulhw xmm2, xmm5 // scale diff + pmulhw xmm3, xmm5 + paddw xmm0, xmm2 // sum rows + paddw xmm1, xmm3 + packuswb xmm0, xmm1 + movdqu [esi + edi], xmm0 + lea esi, [esi + 16] + sub ecx, 16 + jg xloop + jmp xloop99 + + // Blend 25 / 75. + xloop25: + movdqu xmm0, [esi] + movdqu xmm1, [esi + edx] + pavgb xmm0, xmm1 + pavgb xmm0, xmm1 + movdqu [esi + edi], xmm0 + lea esi, [esi + 16] + sub ecx, 16 + jg xloop25 + jmp xloop99 + + // Blend 50 / 50. + xloop50: + movdqu xmm0, [esi] + movdqu xmm1, [esi + edx] + pavgb xmm0, xmm1 + movdqu [esi + edi], xmm0 + lea esi, [esi + 16] + sub ecx, 16 + jg xloop50 + jmp xloop99 + + // Blend 75 / 25. + xloop75: + movdqu xmm1, [esi] + movdqu xmm0, [esi + edx] + pavgb xmm0, xmm1 + pavgb xmm0, xmm1 + movdqu [esi + edi], xmm0 + lea esi, [esi + 16] + sub ecx, 16 + jg xloop75 + jmp xloop99 + + // Blend 100 / 0 - Copy row unchanged. + xloop100: + movdqu xmm0, [esi] + movdqu [esi + edi], xmm0 + lea esi, [esi + 16] + sub ecx, 16 + jg xloop100 + + xloop99: + pop edi + pop esi + ret + } +} +#endif // HAS_INTERPOLATEROW_SSE2 + +// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA. +__declspec(naked) +void ARGBShuffleRow_SSSE3(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix) { + __asm { + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // shuffler + movdqu xmm5, [ecx] + mov ecx, [esp + 16] // pix + + wloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + pshufb xmm0, xmm5 + pshufb xmm1, xmm5 + movdqu [edx], xmm0 + movdqu [edx + 16], xmm1 + lea edx, [edx + 32] + sub ecx, 8 + jg wloop + ret + } +} + +#ifdef HAS_ARGBSHUFFLEROW_AVX2 +__declspec(naked) +void ARGBShuffleRow_AVX2(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix) { + __asm { + mov eax, [esp + 4] // src_argb + mov edx, [esp + 8] // dst_argb + mov ecx, [esp + 12] // shuffler + vbroadcastf128 ymm5, [ecx] // same shuffle in high as low. + mov ecx, [esp + 16] // pix + + wloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + lea eax, [eax + 64] + vpshufb ymm0, ymm0, ymm5 + vpshufb ymm1, ymm1, ymm5 + vmovdqu [edx], ymm0 + vmovdqu [edx + 32], ymm1 + lea edx, [edx + 64] + sub ecx, 16 + jg wloop + + vzeroupper + ret + } +} +#endif // HAS_ARGBSHUFFLEROW_AVX2 + +__declspec(naked) +void ARGBShuffleRow_SSE2(const uint8* src_argb, uint8* dst_argb, + const uint8* shuffler, int pix) { + __asm { + push ebx + push esi + mov eax, [esp + 8 + 4] // src_argb + mov edx, [esp + 8 + 8] // dst_argb + mov esi, [esp + 8 + 12] // shuffler + mov ecx, [esp + 8 + 16] // pix + pxor xmm5, xmm5 + + mov ebx, [esi] // shuffler + cmp ebx, 0x03000102 + je shuf_3012 + cmp ebx, 0x00010203 + je shuf_0123 + cmp ebx, 0x00030201 + je shuf_0321 + cmp ebx, 0x02010003 + je shuf_2103 + + // TODO(fbarchard): Use one source pointer and 3 offsets. + shuf_any1: + movzx ebx, byte ptr [esi] + movzx ebx, byte ptr [eax + ebx] + mov [edx], bl + movzx ebx, byte ptr [esi + 1] + movzx ebx, byte ptr [eax + ebx] + mov [edx + 1], bl + movzx ebx, byte ptr [esi + 2] + movzx ebx, byte ptr [eax + ebx] + mov [edx + 2], bl + movzx ebx, byte ptr [esi + 3] + movzx ebx, byte ptr [eax + ebx] + mov [edx + 3], bl + lea eax, [eax + 4] + lea edx, [edx + 4] + sub ecx, 1 + jg shuf_any1 + jmp shuf99 + + shuf_0123: + movdqu xmm0, [eax] + lea eax, [eax + 16] + movdqa xmm1, xmm0 + punpcklbw xmm0, xmm5 + punpckhbw xmm1, xmm5 + pshufhw xmm0, xmm0, 01Bh // 1B = 00011011 = 0x0123 = BGRAToARGB + pshuflw xmm0, xmm0, 01Bh + pshufhw xmm1, xmm1, 01Bh + pshuflw xmm1, xmm1, 01Bh + packuswb xmm0, xmm1 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg shuf_0123 + jmp shuf99 + + shuf_0321: + movdqu xmm0, [eax] + lea eax, [eax + 16] + movdqa xmm1, xmm0 + punpcklbw xmm0, xmm5 + punpckhbw xmm1, xmm5 + pshufhw xmm0, xmm0, 039h // 39 = 00111001 = 0x0321 = RGBAToARGB + pshuflw xmm0, xmm0, 039h + pshufhw xmm1, xmm1, 039h + pshuflw xmm1, xmm1, 039h + packuswb xmm0, xmm1 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg shuf_0321 + jmp shuf99 + + shuf_2103: + movdqu xmm0, [eax] + lea eax, [eax + 16] + movdqa xmm1, xmm0 + punpcklbw xmm0, xmm5 + punpckhbw xmm1, xmm5 + pshufhw xmm0, xmm0, 093h // 93 = 10010011 = 0x2103 = ARGBToRGBA + pshuflw xmm0, xmm0, 093h + pshufhw xmm1, xmm1, 093h + pshuflw xmm1, xmm1, 093h + packuswb xmm0, xmm1 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg shuf_2103 + jmp shuf99 + + shuf_3012: + movdqu xmm0, [eax] + lea eax, [eax + 16] + movdqa xmm1, xmm0 + punpcklbw xmm0, xmm5 + punpckhbw xmm1, xmm5 + pshufhw xmm0, xmm0, 0C6h // C6 = 11000110 = 0x3012 = ABGRToARGB + pshuflw xmm0, xmm0, 0C6h + pshufhw xmm1, xmm1, 0C6h + pshuflw xmm1, xmm1, 0C6h + packuswb xmm0, xmm1 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg shuf_3012 + + shuf99: + pop esi + pop ebx + ret + } +} + +// YUY2 - Macro-pixel = 2 image pixels +// Y0U0Y1V0....Y2U2Y3V2...Y4U4Y5V4.... + +// UYVY - Macro-pixel = 2 image pixels +// U0Y0V0Y1 + +__declspec(naked) +void I422ToYUY2Row_SSE2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_frame, int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_y + mov esi, [esp + 8 + 8] // src_u + mov edx, [esp + 8 + 12] // src_v + mov edi, [esp + 8 + 16] // dst_frame + mov ecx, [esp + 8 + 20] // width + sub edx, esi + + convertloop: + movq xmm2, qword ptr [esi] // U + movq xmm3, qword ptr [esi + edx] // V + lea esi, [esi + 8] + punpcklbw xmm2, xmm3 // UV + movdqu xmm0, [eax] // Y + lea eax, [eax + 16] + movdqa xmm1, xmm0 + punpcklbw xmm0, xmm2 // YUYV + punpckhbw xmm1, xmm2 + movdqu [edi], xmm0 + movdqu [edi + 16], xmm1 + lea edi, [edi + 32] + sub ecx, 16 + jg convertloop + + pop edi + pop esi + ret + } +} + +__declspec(naked) +void I422ToUYVYRow_SSE2(const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_frame, int width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_y + mov esi, [esp + 8 + 8] // src_u + mov edx, [esp + 8 + 12] // src_v + mov edi, [esp + 8 + 16] // dst_frame + mov ecx, [esp + 8 + 20] // width + sub edx, esi + + convertloop: + movq xmm2, qword ptr [esi] // U + movq xmm3, qword ptr [esi + edx] // V + lea esi, [esi + 8] + punpcklbw xmm2, xmm3 // UV + movdqu xmm0, [eax] // Y + movdqa xmm1, xmm2 + lea eax, [eax + 16] + punpcklbw xmm1, xmm0 // UYVY + punpckhbw xmm2, xmm0 + movdqu [edi], xmm1 + movdqu [edi + 16], xmm2 + lea edi, [edi + 32] + sub ecx, 16 + jg convertloop + + pop edi + pop esi + ret + } +} + +#ifdef HAS_ARGBPOLYNOMIALROW_SSE2 +__declspec(naked) +void ARGBPolynomialRow_SSE2(const uint8* src_argb, + uint8* dst_argb, const float* poly, + int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] /* src_argb */ + mov edx, [esp + 4 + 8] /* dst_argb */ + mov esi, [esp + 4 + 12] /* poly */ + mov ecx, [esp + 4 + 16] /* width */ + pxor xmm3, xmm3 // 0 constant for zero extending bytes to ints. + + // 2 pixel loop. + convertloop: +// pmovzxbd xmm0, dword ptr [eax] // BGRA pixel +// pmovzxbd xmm4, dword ptr [eax + 4] // BGRA pixel + movq xmm0, qword ptr [eax] // BGRABGRA + lea eax, [eax + 8] + punpcklbw xmm0, xmm3 + movdqa xmm4, xmm0 + punpcklwd xmm0, xmm3 // pixel 0 + punpckhwd xmm4, xmm3 // pixel 1 + cvtdq2ps xmm0, xmm0 // 4 floats + cvtdq2ps xmm4, xmm4 + movdqa xmm1, xmm0 // X + movdqa xmm5, xmm4 + mulps xmm0, [esi + 16] // C1 * X + mulps xmm4, [esi + 16] + addps xmm0, [esi] // result = C0 + C1 * X + addps xmm4, [esi] + movdqa xmm2, xmm1 + movdqa xmm6, xmm5 + mulps xmm2, xmm1 // X * X + mulps xmm6, xmm5 + mulps xmm1, xmm2 // X * X * X + mulps xmm5, xmm6 + mulps xmm2, [esi + 32] // C2 * X * X + mulps xmm6, [esi + 32] + mulps xmm1, [esi + 48] // C3 * X * X * X + mulps xmm5, [esi + 48] + addps xmm0, xmm2 // result += C2 * X * X + addps xmm4, xmm6 + addps xmm0, xmm1 // result += C3 * X * X * X + addps xmm4, xmm5 + cvttps2dq xmm0, xmm0 + cvttps2dq xmm4, xmm4 + packuswb xmm0, xmm4 + packuswb xmm0, xmm0 + movq qword ptr [edx], xmm0 + lea edx, [edx + 8] + sub ecx, 2 + jg convertloop + pop esi + ret + } +} +#endif // HAS_ARGBPOLYNOMIALROW_SSE2 + +#ifdef HAS_ARGBPOLYNOMIALROW_AVX2 +__declspec(naked) +void ARGBPolynomialRow_AVX2(const uint8* src_argb, + uint8* dst_argb, const float* poly, + int width) { + __asm { + mov eax, [esp + 4] /* src_argb */ + mov edx, [esp + 8] /* dst_argb */ + mov ecx, [esp + 12] /* poly */ + vbroadcastf128 ymm4, [ecx] // C0 + vbroadcastf128 ymm5, [ecx + 16] // C1 + vbroadcastf128 ymm6, [ecx + 32] // C2 + vbroadcastf128 ymm7, [ecx + 48] // C3 + mov ecx, [esp + 16] /* width */ + + // 2 pixel loop. + convertloop: + vpmovzxbd ymm0, qword ptr [eax] // 2 BGRA pixels + lea eax, [eax + 8] + vcvtdq2ps ymm0, ymm0 // X 8 floats + vmulps ymm2, ymm0, ymm0 // X * X + vmulps ymm3, ymm0, ymm7 // C3 * X + vfmadd132ps ymm0, ymm4, ymm5 // result = C0 + C1 * X + vfmadd231ps ymm0, ymm2, ymm6 // result += C2 * X * X + vfmadd231ps ymm0, ymm2, ymm3 // result += C3 * X * X * X + vcvttps2dq ymm0, ymm0 + vpackusdw ymm0, ymm0, ymm0 // b0g0r0a0_00000000_b0g0r0a0_00000000 + vpermq ymm0, ymm0, 0xd8 // b0g0r0a0_b0g0r0a0_00000000_00000000 + vpackuswb xmm0, xmm0, xmm0 // bgrabgra_00000000_00000000_00000000 + vmovq qword ptr [edx], xmm0 + lea edx, [edx + 8] + sub ecx, 2 + jg convertloop + vzeroupper + ret + } +} +#endif // HAS_ARGBPOLYNOMIALROW_AVX2 + +#ifdef HAS_ARGBCOLORTABLEROW_X86 +// Tranform ARGB pixels with color table. +__declspec(naked) +void ARGBColorTableRow_X86(uint8* dst_argb, const uint8* table_argb, + int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] /* dst_argb */ + mov esi, [esp + 4 + 8] /* table_argb */ + mov ecx, [esp + 4 + 12] /* width */ + + // 1 pixel loop. + convertloop: + movzx edx, byte ptr [eax] + lea eax, [eax + 4] + movzx edx, byte ptr [esi + edx * 4] + mov byte ptr [eax - 4], dl + movzx edx, byte ptr [eax - 4 + 1] + movzx edx, byte ptr [esi + edx * 4 + 1] + mov byte ptr [eax - 4 + 1], dl + movzx edx, byte ptr [eax - 4 + 2] + movzx edx, byte ptr [esi + edx * 4 + 2] + mov byte ptr [eax - 4 + 2], dl + movzx edx, byte ptr [eax - 4 + 3] + movzx edx, byte ptr [esi + edx * 4 + 3] + mov byte ptr [eax - 4 + 3], dl + dec ecx + jg convertloop + pop esi + ret + } +} +#endif // HAS_ARGBCOLORTABLEROW_X86 + +#ifdef HAS_RGBCOLORTABLEROW_X86 +// Tranform RGB pixels with color table. +__declspec(naked) +void RGBColorTableRow_X86(uint8* dst_argb, const uint8* table_argb, int width) { + __asm { + push esi + mov eax, [esp + 4 + 4] /* dst_argb */ + mov esi, [esp + 4 + 8] /* table_argb */ + mov ecx, [esp + 4 + 12] /* width */ + + // 1 pixel loop. + convertloop: + movzx edx, byte ptr [eax] + lea eax, [eax + 4] + movzx edx, byte ptr [esi + edx * 4] + mov byte ptr [eax - 4], dl + movzx edx, byte ptr [eax - 4 + 1] + movzx edx, byte ptr [esi + edx * 4 + 1] + mov byte ptr [eax - 4 + 1], dl + movzx edx, byte ptr [eax - 4 + 2] + movzx edx, byte ptr [esi + edx * 4 + 2] + mov byte ptr [eax - 4 + 2], dl + dec ecx + jg convertloop + + pop esi + ret + } +} +#endif // HAS_RGBCOLORTABLEROW_X86 + +#ifdef HAS_ARGBLUMACOLORTABLEROW_SSSE3 +// Tranform RGB pixels with luma table. +__declspec(naked) +void ARGBLumaColorTableRow_SSSE3(const uint8* src_argb, uint8* dst_argb, + int width, + const uint8* luma, uint32 lumacoeff) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] /* src_argb */ + mov edi, [esp + 8 + 8] /* dst_argb */ + mov ecx, [esp + 8 + 12] /* width */ + movd xmm2, dword ptr [esp + 8 + 16] // luma table + movd xmm3, dword ptr [esp + 8 + 20] // lumacoeff + pshufd xmm2, xmm2, 0 + pshufd xmm3, xmm3, 0 + pcmpeqb xmm4, xmm4 // generate mask 0xff00ff00 + psllw xmm4, 8 + pxor xmm5, xmm5 + + // 4 pixel loop. + convertloop: + movdqu xmm0, qword ptr [eax] // generate luma ptr + pmaddubsw xmm0, xmm3 + phaddw xmm0, xmm0 + pand xmm0, xmm4 // mask out low bits + punpcklwd xmm0, xmm5 + paddd xmm0, xmm2 // add table base + movd esi, xmm0 + pshufd xmm0, xmm0, 0x39 // 00111001 to rotate right 32 + + movzx edx, byte ptr [eax] + movzx edx, byte ptr [esi + edx] + mov byte ptr [edi], dl + movzx edx, byte ptr [eax + 1] + movzx edx, byte ptr [esi + edx] + mov byte ptr [edi + 1], dl + movzx edx, byte ptr [eax + 2] + movzx edx, byte ptr [esi + edx] + mov byte ptr [edi + 2], dl + movzx edx, byte ptr [eax + 3] // copy alpha. + mov byte ptr [edi + 3], dl + + movd esi, xmm0 + pshufd xmm0, xmm0, 0x39 // 00111001 to rotate right 32 + + movzx edx, byte ptr [eax + 4] + movzx edx, byte ptr [esi + edx] + mov byte ptr [edi + 4], dl + movzx edx, byte ptr [eax + 5] + movzx edx, byte ptr [esi + edx] + mov byte ptr [edi + 5], dl + movzx edx, byte ptr [eax + 6] + movzx edx, byte ptr [esi + edx] + mov byte ptr [edi + 6], dl + movzx edx, byte ptr [eax + 7] // copy alpha. + mov byte ptr [edi + 7], dl + + movd esi, xmm0 + pshufd xmm0, xmm0, 0x39 // 00111001 to rotate right 32 + + movzx edx, byte ptr [eax + 8] + movzx edx, byte ptr [esi + edx] + mov byte ptr [edi + 8], dl + movzx edx, byte ptr [eax + 9] + movzx edx, byte ptr [esi + edx] + mov byte ptr [edi + 9], dl + movzx edx, byte ptr [eax + 10] + movzx edx, byte ptr [esi + edx] + mov byte ptr [edi + 10], dl + movzx edx, byte ptr [eax + 11] // copy alpha. + mov byte ptr [edi + 11], dl + + movd esi, xmm0 + + movzx edx, byte ptr [eax + 12] + movzx edx, byte ptr [esi + edx] + mov byte ptr [edi + 12], dl + movzx edx, byte ptr [eax + 13] + movzx edx, byte ptr [esi + edx] + mov byte ptr [edi + 13], dl + movzx edx, byte ptr [eax + 14] + movzx edx, byte ptr [esi + edx] + mov byte ptr [edi + 14], dl + movzx edx, byte ptr [eax + 15] // copy alpha. + mov byte ptr [edi + 15], dl + + lea eax, [eax + 16] + lea edi, [edi + 16] + sub ecx, 4 + jg convertloop + + pop edi + pop esi + ret + } +} +#endif // HAS_ARGBLUMACOLORTABLEROW_SSSE3 + +#endif // defined(_M_X64) +#endif // !defined(LIBYUV_DISABLE_X86) && (defined(_M_IX86) || defined(_M_X64)) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/row_x86.asm b/third_party/aom/third_party/libyuv/source/row_x86.asm new file mode 100644 index 0000000000..0cb326f8e5 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/row_x86.asm @@ -0,0 +1,146 @@ +; +; Copyright 2012 The LibYuv Project Authors. All rights reserved. +; +; Use of this source code is governed by a BSD-style license +; that can be found in the LICENSE file in the root of the source +; tree. An additional intellectual property rights grant can be found +; in the file PATENTS. All contributing project authors may +; be found in the AUTHORS file in the root of the source tree. +; + +%ifdef __YASM_VERSION_ID__ +%if __YASM_VERSION_ID__ < 01020000h +%error AVX2 is supported only by yasm 1.2.0 or later. +%endif +%endif +%include "x86inc.asm" + +SECTION .text + +; cglobal numeric constants are parameters, gpr regs, mm regs + +; void YUY2ToYRow_SSE2(const uint8* src_yuy2, uint8* dst_y, int pix) + +%macro YUY2TOYROW 2-3 +cglobal %1ToYRow%3, 3, 3, 3, src_yuy2, dst_y, pix +%ifidn %1,YUY2 + pcmpeqb m2, m2, m2 ; generate mask 0x00ff00ff + psrlw m2, m2, 8 +%endif + + ALIGN 4 +.convertloop: + mov%2 m0, [src_yuy2q] + mov%2 m1, [src_yuy2q + mmsize] + lea src_yuy2q, [src_yuy2q + mmsize * 2] +%ifidn %1,YUY2 + pand m0, m0, m2 ; YUY2 even bytes are Y + pand m1, m1, m2 +%else + psrlw m0, m0, 8 ; UYVY odd bytes are Y + psrlw m1, m1, 8 +%endif + packuswb m0, m0, m1 +%if cpuflag(AVX2) + vpermq m0, m0, 0xd8 +%endif + sub pixd, mmsize + mov%2 [dst_yq], m0 + lea dst_yq, [dst_yq + mmsize] + jg .convertloop + REP_RET +%endmacro + +; TODO(fbarchard): Remove MMX. Add SSSE3 pshufb version. +INIT_MMX MMX +YUY2TOYROW YUY2,a, +YUY2TOYROW YUY2,u,_Unaligned +YUY2TOYROW UYVY,a, +YUY2TOYROW UYVY,u,_Unaligned +INIT_XMM SSE2 +YUY2TOYROW YUY2,a, +YUY2TOYROW YUY2,u,_Unaligned +YUY2TOYROW UYVY,a, +YUY2TOYROW UYVY,u,_Unaligned +INIT_YMM AVX2 +YUY2TOYROW YUY2,a, +YUY2TOYROW UYVY,a, + +; void SplitUVRow_SSE2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix) + +%macro SplitUVRow 1-2 +cglobal SplitUVRow%2, 4, 4, 5, src_uv, dst_u, dst_v, pix + pcmpeqb m4, m4, m4 ; generate mask 0x00ff00ff + psrlw m4, m4, 8 + sub dst_vq, dst_uq + + ALIGN 4 +.convertloop: + mov%1 m0, [src_uvq] + mov%1 m1, [src_uvq + mmsize] + lea src_uvq, [src_uvq + mmsize * 2] + psrlw m2, m0, 8 ; odd bytes + psrlw m3, m1, 8 + pand m0, m0, m4 ; even bytes + pand m1, m1, m4 + packuswb m0, m0, m1 + packuswb m2, m2, m3 +%if cpuflag(AVX2) + vpermq m0, m0, 0xd8 + vpermq m2, m2, 0xd8 +%endif + mov%1 [dst_uq], m0 + mov%1 [dst_uq + dst_vq], m2 + lea dst_uq, [dst_uq + mmsize] + sub pixd, mmsize + jg .convertloop + REP_RET +%endmacro + +INIT_MMX MMX +SplitUVRow a, +SplitUVRow u,_Unaligned +INIT_XMM SSE2 +SplitUVRow a, +SplitUVRow u,_Unaligned +INIT_YMM AVX2 +SplitUVRow a, + +; void MergeUVRow_SSE2(const uint8* src_u, const uint8* src_v, uint8* dst_uv, +; int width); + +%macro MergeUVRow_ 1-2 +cglobal MergeUVRow_%2, 4, 4, 3, src_u, src_v, dst_uv, pix + sub src_vq, src_uq + + ALIGN 4 +.convertloop: + mov%1 m0, [src_uq] + mov%1 m1, [src_vq] + lea src_uq, [src_uq + mmsize] + punpcklbw m2, m0, m1 // first 8 UV pairs + punpckhbw m0, m0, m1 // next 8 UV pairs +%if cpuflag(AVX2) + vperm2i128 m1, m2, m0, 0x20 // low 128 of ymm2 and low 128 of ymm0 + vperm2i128 m2, m2, m0, 0x31 // high 128 of ymm2 and high 128 of ymm0 + mov%1 [dst_uvq], m1 + mov%1 [dst_uvq + mmsize], m2 +%else + mov%1 [dst_uvq], m2 + mov%1 [dst_uvq + mmsize], m0 +%endif + lea dst_uvq, [dst_uvq + mmsize * 2] + sub pixd, mmsize + jg .convertloop + REP_RET +%endmacro + +INIT_MMX MMX +MergeUVRow_ a, +MergeUVRow_ u,_Unaligned +INIT_XMM SSE2 +MergeUVRow_ a, +MergeUVRow_ u,_Unaligned +INIT_YMM AVX2 +MergeUVRow_ a, + diff --git a/third_party/aom/third_party/libyuv/source/scale.cc b/third_party/aom/third_party/libyuv/source/scale.cc new file mode 100644 index 0000000000..0a01304c41 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/scale.cc @@ -0,0 +1,1689 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/scale.h" + +#include +#include + +#include "libyuv/cpu_id.h" +#include "libyuv/planar_functions.h" // For CopyPlane +#include "libyuv/row.h" +#include "libyuv/scale_row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +static __inline int Abs(int v) { + return v >= 0 ? v : -v; +} + +#define SUBSAMPLE(v, a, s) (v < 0) ? (-((-v + a) >> s)) : ((v + a) >> s) + +// Scale plane, 1/2 +// This is an optimized version for scaling down a plane to 1/2 of +// its original size. + +static void ScalePlaneDown2(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_ptr, uint8* dst_ptr, + enum FilterMode filtering) { + int y; + void (*ScaleRowDown2)(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) = + filtering == kFilterNone ? ScaleRowDown2_C : + (filtering == kFilterLinear ? ScaleRowDown2Linear_C : ScaleRowDown2Box_C); + int row_stride = src_stride << 1; + if (!filtering) { + src_ptr += src_stride; // Point to odd rows. + src_stride = 0; + } + +#if defined(HAS_SCALEROWDOWN2_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_Any_NEON : + (filtering == kFilterLinear ? ScaleRowDown2Linear_Any_NEON : + ScaleRowDown2Box_Any_NEON); + if (IS_ALIGNED(dst_width, 16)) { + ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_NEON : + (filtering == kFilterLinear ? ScaleRowDown2Linear_NEON : + ScaleRowDown2Box_NEON); + } + } +#endif +#if defined(HAS_SCALEROWDOWN2_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_Any_SSE2 : + (filtering == kFilterLinear ? ScaleRowDown2Linear_Any_SSE2 : + ScaleRowDown2Box_Any_SSE2); + if (IS_ALIGNED(dst_width, 16)) { + ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_SSE2 : + (filtering == kFilterLinear ? ScaleRowDown2Linear_SSE2 : + ScaleRowDown2Box_SSE2); + } + } +#endif +#if defined(HAS_SCALEROWDOWN2_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_Any_AVX2 : + (filtering == kFilterLinear ? ScaleRowDown2Linear_Any_AVX2 : + ScaleRowDown2Box_Any_AVX2); + if (IS_ALIGNED(dst_width, 32)) { + ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_AVX2 : + (filtering == kFilterLinear ? ScaleRowDown2Linear_AVX2 : + ScaleRowDown2Box_AVX2); + } + } +#endif +#if defined(HAS_SCALEROWDOWN2_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(src_ptr, 4) && + IS_ALIGNED(src_stride, 4) && IS_ALIGNED(row_stride, 4) && + IS_ALIGNED(dst_ptr, 4) && IS_ALIGNED(dst_stride, 4)) { + ScaleRowDown2 = filtering ? + ScaleRowDown2Box_MIPS_DSPR2 : ScaleRowDown2_MIPS_DSPR2; + } +#endif + + if (filtering == kFilterLinear) { + src_stride = 0; + } + // TODO(fbarchard): Loop through source height to allow odd height. + for (y = 0; y < dst_height; ++y) { + ScaleRowDown2(src_ptr, src_stride, dst_ptr, dst_width); + src_ptr += row_stride; + dst_ptr += dst_stride; + } +} + +static void ScalePlaneDown2_16(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint16* src_ptr, uint16* dst_ptr, + enum FilterMode filtering) { + int y; + void (*ScaleRowDown2)(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst_ptr, int dst_width) = + filtering == kFilterNone ? ScaleRowDown2_16_C : + (filtering == kFilterLinear ? ScaleRowDown2Linear_16_C : + ScaleRowDown2Box_16_C); + int row_stride = src_stride << 1; + if (!filtering) { + src_ptr += src_stride; // Point to odd rows. + src_stride = 0; + } + +#if defined(HAS_SCALEROWDOWN2_16_NEON) + if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 16)) { + ScaleRowDown2 = filtering ? ScaleRowDown2Box_16_NEON : + ScaleRowDown2_16_NEON; + } +#endif +#if defined(HAS_SCALEROWDOWN2_16_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 16)) { + ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_SSE2 : + (filtering == kFilterLinear ? ScaleRowDown2Linear_16_SSE2 : + ScaleRowDown2Box_16_SSE2); + } +#endif +#if defined(HAS_SCALEROWDOWN2_16_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(src_ptr, 4) && + IS_ALIGNED(src_stride, 4) && IS_ALIGNED(row_stride, 4) && + IS_ALIGNED(dst_ptr, 4) && IS_ALIGNED(dst_stride, 4)) { + ScaleRowDown2 = filtering ? + ScaleRowDown2Box_16_MIPS_DSPR2 : ScaleRowDown2_16_MIPS_DSPR2; + } +#endif + + if (filtering == kFilterLinear) { + src_stride = 0; + } + // TODO(fbarchard): Loop through source height to allow odd height. + for (y = 0; y < dst_height; ++y) { + ScaleRowDown2(src_ptr, src_stride, dst_ptr, dst_width); + src_ptr += row_stride; + dst_ptr += dst_stride; + } +} + +// Scale plane, 1/4 +// This is an optimized version for scaling down a plane to 1/4 of +// its original size. + +static void ScalePlaneDown4(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_ptr, uint8* dst_ptr, + enum FilterMode filtering) { + int y; + void (*ScaleRowDown4)(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) = + filtering ? ScaleRowDown4Box_C : ScaleRowDown4_C; + int row_stride = src_stride << 2; + if (!filtering) { + src_ptr += src_stride * 2; // Point to row 2. + src_stride = 0; + } +#if defined(HAS_SCALEROWDOWN4_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ScaleRowDown4 = filtering ? + ScaleRowDown4Box_Any_NEON : ScaleRowDown4_Any_NEON; + if (IS_ALIGNED(dst_width, 8)) { + ScaleRowDown4 = filtering ? ScaleRowDown4Box_NEON : ScaleRowDown4_NEON; + } + } +#endif +#if defined(HAS_SCALEROWDOWN4_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ScaleRowDown4 = filtering ? + ScaleRowDown4Box_Any_SSE2 : ScaleRowDown4_Any_SSE2; + if (IS_ALIGNED(dst_width, 8)) { + ScaleRowDown4 = filtering ? ScaleRowDown4Box_SSE2 : ScaleRowDown4_SSE2; + } + } +#endif +#if defined(HAS_SCALEROWDOWN4_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleRowDown4 = filtering ? + ScaleRowDown4Box_Any_AVX2 : ScaleRowDown4_Any_AVX2; + if (IS_ALIGNED(dst_width, 16)) { + ScaleRowDown4 = filtering ? ScaleRowDown4Box_AVX2 : ScaleRowDown4_AVX2; + } + } +#endif +#if defined(HAS_SCALEROWDOWN4_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(row_stride, 4) && + IS_ALIGNED(src_ptr, 4) && IS_ALIGNED(src_stride, 4) && + IS_ALIGNED(dst_ptr, 4) && IS_ALIGNED(dst_stride, 4)) { + ScaleRowDown4 = filtering ? + ScaleRowDown4Box_MIPS_DSPR2 : ScaleRowDown4_MIPS_DSPR2; + } +#endif + + if (filtering == kFilterLinear) { + src_stride = 0; + } + for (y = 0; y < dst_height; ++y) { + ScaleRowDown4(src_ptr, src_stride, dst_ptr, dst_width); + src_ptr += row_stride; + dst_ptr += dst_stride; + } +} + +static void ScalePlaneDown4_16(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint16* src_ptr, uint16* dst_ptr, + enum FilterMode filtering) { + int y; + void (*ScaleRowDown4)(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst_ptr, int dst_width) = + filtering ? ScaleRowDown4Box_16_C : ScaleRowDown4_16_C; + int row_stride = src_stride << 2; + if (!filtering) { + src_ptr += src_stride * 2; // Point to row 2. + src_stride = 0; + } +#if defined(HAS_SCALEROWDOWN4_16_NEON) + if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 8)) { + ScaleRowDown4 = filtering ? ScaleRowDown4Box_16_NEON : + ScaleRowDown4_16_NEON; + } +#endif +#if defined(HAS_SCALEROWDOWN4_16_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { + ScaleRowDown4 = filtering ? ScaleRowDown4Box_16_SSE2 : + ScaleRowDown4_16_SSE2; + } +#endif +#if defined(HAS_SCALEROWDOWN4_16_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(row_stride, 4) && + IS_ALIGNED(src_ptr, 4) && IS_ALIGNED(src_stride, 4) && + IS_ALIGNED(dst_ptr, 4) && IS_ALIGNED(dst_stride, 4)) { + ScaleRowDown4 = filtering ? + ScaleRowDown4Box_16_MIPS_DSPR2 : ScaleRowDown4_16_MIPS_DSPR2; + } +#endif + + if (filtering == kFilterLinear) { + src_stride = 0; + } + for (y = 0; y < dst_height; ++y) { + ScaleRowDown4(src_ptr, src_stride, dst_ptr, dst_width); + src_ptr += row_stride; + dst_ptr += dst_stride; + } +} + +// Scale plane down, 3/4 + +static void ScalePlaneDown34(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_ptr, uint8* dst_ptr, + enum FilterMode filtering) { + int y; + void (*ScaleRowDown34_0)(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); + void (*ScaleRowDown34_1)(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); + const int filter_stride = (filtering == kFilterLinear) ? 0 : src_stride; + assert(dst_width % 3 == 0); + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_C; + ScaleRowDown34_1 = ScaleRowDown34_C; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_C; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_C; + } +#if defined(HAS_SCALEROWDOWN34_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_Any_NEON; + ScaleRowDown34_1 = ScaleRowDown34_Any_NEON; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_NEON; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_NEON; + } + if (dst_width % 24 == 0) { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_NEON; + ScaleRowDown34_1 = ScaleRowDown34_NEON; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_NEON; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_NEON; + } + } + } +#endif +#if defined(HAS_SCALEROWDOWN34_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_Any_SSSE3; + ScaleRowDown34_1 = ScaleRowDown34_Any_SSSE3; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_SSSE3; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_SSSE3; + } + if (dst_width % 24 == 0) { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_SSSE3; + ScaleRowDown34_1 = ScaleRowDown34_SSSE3; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_SSSE3; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_SSSE3; + } + } + } +#endif +#if defined(HAS_SCALEROWDOWN34_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && (dst_width % 24 == 0) && + IS_ALIGNED(src_ptr, 4) && IS_ALIGNED(src_stride, 4) && + IS_ALIGNED(dst_ptr, 4) && IS_ALIGNED(dst_stride, 4)) { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_MIPS_DSPR2; + ScaleRowDown34_1 = ScaleRowDown34_MIPS_DSPR2; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_MIPS_DSPR2; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_MIPS_DSPR2; + } + } +#endif + + for (y = 0; y < dst_height - 2; y += 3) { + ScaleRowDown34_0(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride; + dst_ptr += dst_stride; + ScaleRowDown34_1(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride; + dst_ptr += dst_stride; + ScaleRowDown34_0(src_ptr + src_stride, -filter_stride, + dst_ptr, dst_width); + src_ptr += src_stride * 2; + dst_ptr += dst_stride; + } + + // Remainder 1 or 2 rows with last row vertically unfiltered + if ((dst_height % 3) == 2) { + ScaleRowDown34_0(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride; + dst_ptr += dst_stride; + ScaleRowDown34_1(src_ptr, 0, dst_ptr, dst_width); + } else if ((dst_height % 3) == 1) { + ScaleRowDown34_0(src_ptr, 0, dst_ptr, dst_width); + } +} + +static void ScalePlaneDown34_16(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint16* src_ptr, uint16* dst_ptr, + enum FilterMode filtering) { + int y; + void (*ScaleRowDown34_0)(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst_ptr, int dst_width); + void (*ScaleRowDown34_1)(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst_ptr, int dst_width); + const int filter_stride = (filtering == kFilterLinear) ? 0 : src_stride; + assert(dst_width % 3 == 0); + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_16_C; + ScaleRowDown34_1 = ScaleRowDown34_16_C; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_C; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_C; + } +#if defined(HAS_SCALEROWDOWN34_16_NEON) + if (TestCpuFlag(kCpuHasNEON) && (dst_width % 24 == 0)) { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_16_NEON; + ScaleRowDown34_1 = ScaleRowDown34_16_NEON; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_NEON; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_NEON; + } + } +#endif +#if defined(HAS_SCALEROWDOWN34_16_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3) && (dst_width % 24 == 0)) { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_16_SSSE3; + ScaleRowDown34_1 = ScaleRowDown34_16_SSSE3; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_SSSE3; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_SSSE3; + } + } +#endif +#if defined(HAS_SCALEROWDOWN34_16_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && (dst_width % 24 == 0) && + IS_ALIGNED(src_ptr, 4) && IS_ALIGNED(src_stride, 4) && + IS_ALIGNED(dst_ptr, 4) && IS_ALIGNED(dst_stride, 4)) { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_16_MIPS_DSPR2; + ScaleRowDown34_1 = ScaleRowDown34_16_MIPS_DSPR2; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_MIPS_DSPR2; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_MIPS_DSPR2; + } + } +#endif + + for (y = 0; y < dst_height - 2; y += 3) { + ScaleRowDown34_0(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride; + dst_ptr += dst_stride; + ScaleRowDown34_1(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride; + dst_ptr += dst_stride; + ScaleRowDown34_0(src_ptr + src_stride, -filter_stride, + dst_ptr, dst_width); + src_ptr += src_stride * 2; + dst_ptr += dst_stride; + } + + // Remainder 1 or 2 rows with last row vertically unfiltered + if ((dst_height % 3) == 2) { + ScaleRowDown34_0(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride; + dst_ptr += dst_stride; + ScaleRowDown34_1(src_ptr, 0, dst_ptr, dst_width); + } else if ((dst_height % 3) == 1) { + ScaleRowDown34_0(src_ptr, 0, dst_ptr, dst_width); + } +} + + +// Scale plane, 3/8 +// This is an optimized version for scaling down a plane to 3/8 +// of its original size. +// +// Uses box filter arranges like this +// aaabbbcc -> abc +// aaabbbcc def +// aaabbbcc ghi +// dddeeeff +// dddeeeff +// dddeeeff +// ggghhhii +// ggghhhii +// Boxes are 3x3, 2x3, 3x2 and 2x2 + +static void ScalePlaneDown38(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_ptr, uint8* dst_ptr, + enum FilterMode filtering) { + int y; + void (*ScaleRowDown38_3)(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); + void (*ScaleRowDown38_2)(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width); + const int filter_stride = (filtering == kFilterLinear) ? 0 : src_stride; + assert(dst_width % 3 == 0); + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_C; + ScaleRowDown38_2 = ScaleRowDown38_C; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_C; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_C; + } + +#if defined(HAS_SCALEROWDOWN38_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_Any_NEON; + ScaleRowDown38_2 = ScaleRowDown38_Any_NEON; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_NEON; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_NEON; + } + if (dst_width % 12 == 0) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_NEON; + ScaleRowDown38_2 = ScaleRowDown38_NEON; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_NEON; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_NEON; + } + } + } +#endif +#if defined(HAS_SCALEROWDOWN38_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_Any_SSSE3; + ScaleRowDown38_2 = ScaleRowDown38_Any_SSSE3; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_SSSE3; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_SSSE3; + } + if (dst_width % 12 == 0 && !filtering) { + ScaleRowDown38_3 = ScaleRowDown38_SSSE3; + ScaleRowDown38_2 = ScaleRowDown38_SSSE3; + } + if (dst_width % 6 == 0 && filtering) { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_SSSE3; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_SSSE3; + } + } +#endif +#if defined(HAS_SCALEROWDOWN38_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && (dst_width % 12 == 0) && + IS_ALIGNED(src_ptr, 4) && IS_ALIGNED(src_stride, 4) && + IS_ALIGNED(dst_ptr, 4) && IS_ALIGNED(dst_stride, 4)) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_MIPS_DSPR2; + ScaleRowDown38_2 = ScaleRowDown38_MIPS_DSPR2; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_MIPS_DSPR2; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_MIPS_DSPR2; + } + } +#endif + + for (y = 0; y < dst_height - 2; y += 3) { + ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride * 3; + dst_ptr += dst_stride; + ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride * 3; + dst_ptr += dst_stride; + ScaleRowDown38_2(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride * 2; + dst_ptr += dst_stride; + } + + // Remainder 1 or 2 rows with last row vertically unfiltered + if ((dst_height % 3) == 2) { + ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride * 3; + dst_ptr += dst_stride; + ScaleRowDown38_3(src_ptr, 0, dst_ptr, dst_width); + } else if ((dst_height % 3) == 1) { + ScaleRowDown38_3(src_ptr, 0, dst_ptr, dst_width); + } +} + +static void ScalePlaneDown38_16(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint16* src_ptr, uint16* dst_ptr, + enum FilterMode filtering) { + int y; + void (*ScaleRowDown38_3)(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst_ptr, int dst_width); + void (*ScaleRowDown38_2)(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst_ptr, int dst_width); + const int filter_stride = (filtering == kFilterLinear) ? 0 : src_stride; + assert(dst_width % 3 == 0); + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_16_C; + ScaleRowDown38_2 = ScaleRowDown38_16_C; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_C; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_C; + } +#if defined(HAS_SCALEROWDOWN38_16_NEON) + if (TestCpuFlag(kCpuHasNEON) && (dst_width % 12 == 0)) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_16_NEON; + ScaleRowDown38_2 = ScaleRowDown38_16_NEON; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_NEON; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_NEON; + } + } +#endif +#if defined(HAS_SCALEROWDOWN38_16_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3) && (dst_width % 24 == 0)) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_16_SSSE3; + ScaleRowDown38_2 = ScaleRowDown38_16_SSSE3; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_SSSE3; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_SSSE3; + } + } +#endif +#if defined(HAS_SCALEROWDOWN38_16_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && (dst_width % 12 == 0) && + IS_ALIGNED(src_ptr, 4) && IS_ALIGNED(src_stride, 4) && + IS_ALIGNED(dst_ptr, 4) && IS_ALIGNED(dst_stride, 4)) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_16_MIPS_DSPR2; + ScaleRowDown38_2 = ScaleRowDown38_16_MIPS_DSPR2; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_MIPS_DSPR2; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_MIPS_DSPR2; + } + } +#endif + + for (y = 0; y < dst_height - 2; y += 3) { + ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride * 3; + dst_ptr += dst_stride; + ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride * 3; + dst_ptr += dst_stride; + ScaleRowDown38_2(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride * 2; + dst_ptr += dst_stride; + } + + // Remainder 1 or 2 rows with last row vertically unfiltered + if ((dst_height % 3) == 2) { + ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width); + src_ptr += src_stride * 3; + dst_ptr += dst_stride; + ScaleRowDown38_3(src_ptr, 0, dst_ptr, dst_width); + } else if ((dst_height % 3) == 1) { + ScaleRowDown38_3(src_ptr, 0, dst_ptr, dst_width); + } +} + +#define MIN1(x) ((x) < 1 ? 1 : (x)) + +static __inline uint32 SumPixels(int iboxwidth, const uint16* src_ptr) { + uint32 sum = 0u; + int x; + assert(iboxwidth > 0); + for (x = 0; x < iboxwidth; ++x) { + sum += src_ptr[x]; + } + return sum; +} + +static __inline uint32 SumPixels_16(int iboxwidth, const uint32* src_ptr) { + uint32 sum = 0u; + int x; + assert(iboxwidth > 0); + for (x = 0; x < iboxwidth; ++x) { + sum += src_ptr[x]; + } + return sum; +} + +static void ScaleAddCols2_C(int dst_width, int boxheight, int x, int dx, + const uint16* src_ptr, uint8* dst_ptr) { + int i; + int scaletbl[2]; + int minboxwidth = dx >> 16; + int* scaleptr = scaletbl - minboxwidth; + int boxwidth; + scaletbl[0] = 65536 / (MIN1(minboxwidth) * boxheight); + scaletbl[1] = 65536 / (MIN1(minboxwidth + 1) * boxheight); + for (i = 0; i < dst_width; ++i) { + int ix = x >> 16; + x += dx; + boxwidth = MIN1((x >> 16) - ix); + *dst_ptr++ = SumPixels(boxwidth, src_ptr + ix) * scaleptr[boxwidth] >> 16; + } +} + +static void ScaleAddCols2_16_C(int dst_width, int boxheight, int x, int dx, + const uint32* src_ptr, uint16* dst_ptr) { + int i; + int scaletbl[2]; + int minboxwidth = dx >> 16; + int* scaleptr = scaletbl - minboxwidth; + int boxwidth; + scaletbl[0] = 65536 / (MIN1(minboxwidth) * boxheight); + scaletbl[1] = 65536 / (MIN1(minboxwidth + 1) * boxheight); + for (i = 0; i < dst_width; ++i) { + int ix = x >> 16; + x += dx; + boxwidth = MIN1((x >> 16) - ix); + *dst_ptr++ = + SumPixels_16(boxwidth, src_ptr + ix) * scaleptr[boxwidth] >> 16; + } +} + +static void ScaleAddCols0_C(int dst_width, int boxheight, int x, int, + const uint16* src_ptr, uint8* dst_ptr) { + int scaleval = 65536 / boxheight; + int i; + src_ptr += (x >> 16); + for (i = 0; i < dst_width; ++i) { + *dst_ptr++ = src_ptr[i] * scaleval >> 16; + } +} + +static void ScaleAddCols1_C(int dst_width, int boxheight, int x, int dx, + const uint16* src_ptr, uint8* dst_ptr) { + int boxwidth = MIN1(dx >> 16); + int scaleval = 65536 / (boxwidth * boxheight); + int i; + x >>= 16; + for (i = 0; i < dst_width; ++i) { + *dst_ptr++ = SumPixels(boxwidth, src_ptr + x) * scaleval >> 16; + x += boxwidth; + } +} + +static void ScaleAddCols1_16_C(int dst_width, int boxheight, int x, int dx, + const uint32* src_ptr, uint16* dst_ptr) { + int boxwidth = MIN1(dx >> 16); + int scaleval = 65536 / (boxwidth * boxheight); + int i; + for (i = 0; i < dst_width; ++i) { + *dst_ptr++ = SumPixels_16(boxwidth, src_ptr + x) * scaleval >> 16; + x += boxwidth; + } +} + +// Scale plane down to any dimensions, with interpolation. +// (boxfilter). +// +// Same method as SimpleScale, which is fixed point, outputting +// one pixel of destination using fixed point (16.16) to step +// through source, sampling a box of pixel with simple +// averaging. +static void ScalePlaneBox(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_ptr, uint8* dst_ptr) { + int j, k; + // Initial source x/y coordinate and step values as 16.16 fixed point. + int x = 0; + int y = 0; + int dx = 0; + int dy = 0; + const int max_y = (src_height << 16); + ScaleSlope(src_width, src_height, dst_width, dst_height, kFilterBox, + &x, &y, &dx, &dy); + src_width = Abs(src_width); + { + // Allocate a row buffer of uint16. + align_buffer_64(row16, src_width * 2); + void (*ScaleAddCols)(int dst_width, int boxheight, int x, int dx, + const uint16* src_ptr, uint8* dst_ptr) = + (dx & 0xffff) ? ScaleAddCols2_C: + ((dx != 0x10000) ? ScaleAddCols1_C : ScaleAddCols0_C); + void (*ScaleAddRow)(const uint8* src_ptr, uint16* dst_ptr, int src_width) = + ScaleAddRow_C; +#if defined(HAS_SCALEADDROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ScaleAddRow = ScaleAddRow_Any_SSE2; + if (IS_ALIGNED(src_width, 16)) { + ScaleAddRow = ScaleAddRow_SSE2; + } + } +#endif +#if defined(HAS_SCALEADDROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleAddRow = ScaleAddRow_Any_AVX2; + if (IS_ALIGNED(src_width, 32)) { + ScaleAddRow = ScaleAddRow_AVX2; + } + } +#endif +#if defined(HAS_SCALEADDROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ScaleAddRow = ScaleAddRow_Any_NEON; + if (IS_ALIGNED(src_width, 16)) { + ScaleAddRow = ScaleAddRow_NEON; + } + } +#endif + + for (j = 0; j < dst_height; ++j) { + int boxheight; + int iy = y >> 16; + const uint8* src = src_ptr + iy * src_stride; + y += dy; + if (y > max_y) { + y = max_y; + } + boxheight = MIN1((y >> 16) - iy); + memset(row16, 0, src_width * 2); + for (k = 0; k < boxheight; ++k) { + ScaleAddRow(src, (uint16 *)(row16), src_width); + src += src_stride; + } + ScaleAddCols(dst_width, boxheight, x, dx, (uint16*)(row16), dst_ptr); + dst_ptr += dst_stride; + } + free_aligned_buffer_64(row16); + } +} + +static void ScalePlaneBox_16(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint16* src_ptr, uint16* dst_ptr) { + int j, k; + // Initial source x/y coordinate and step values as 16.16 fixed point. + int x = 0; + int y = 0; + int dx = 0; + int dy = 0; + const int max_y = (src_height << 16); + ScaleSlope(src_width, src_height, dst_width, dst_height, kFilterBox, + &x, &y, &dx, &dy); + src_width = Abs(src_width); + { + // Allocate a row buffer of uint32. + align_buffer_64(row32, src_width * 4); + void (*ScaleAddCols)(int dst_width, int boxheight, int x, int dx, + const uint32* src_ptr, uint16* dst_ptr) = + (dx & 0xffff) ? ScaleAddCols2_16_C: ScaleAddCols1_16_C; + void (*ScaleAddRow)(const uint16* src_ptr, uint32* dst_ptr, int src_width) = + ScaleAddRow_16_C; + +#if defined(HAS_SCALEADDROW_16_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(src_width, 16)) { + ScaleAddRow = ScaleAddRow_16_SSE2; + } +#endif + + for (j = 0; j < dst_height; ++j) { + int boxheight; + int iy = y >> 16; + const uint16* src = src_ptr + iy * src_stride; + y += dy; + if (y > max_y) { + y = max_y; + } + boxheight = MIN1((y >> 16) - iy); + memset(row32, 0, src_width * 4); + for (k = 0; k < boxheight; ++k) { + ScaleAddRow(src, (uint32 *)(row32), src_width); + src += src_stride; + } + ScaleAddCols(dst_width, boxheight, x, dx, (uint32*)(row32), dst_ptr); + dst_ptr += dst_stride; + } + free_aligned_buffer_64(row32); + } +} + +// Scale plane down with bilinear interpolation. +void ScalePlaneBilinearDown(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_ptr, uint8* dst_ptr, + enum FilterMode filtering) { + // Initial source x/y coordinate and step values as 16.16 fixed point. + int x = 0; + int y = 0; + int dx = 0; + int dy = 0; + // TODO(fbarchard): Consider not allocating row buffer for kFilterLinear. + // Allocate a row buffer. + align_buffer_64(row, src_width); + + const int max_y = (src_height - 1) << 16; + int j; + void (*ScaleFilterCols)(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx) = + (src_width >= 32768) ? ScaleFilterCols64_C : ScaleFilterCols_C; + void (*InterpolateRow)(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, int source_y_fraction) = + InterpolateRow_C; + ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, + &x, &y, &dx, &dy); + src_width = Abs(src_width); + +#if defined(HAS_INTERPOLATEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + InterpolateRow = InterpolateRow_Any_SSE2; + if (IS_ALIGNED(src_width, 16)) { + InterpolateRow = InterpolateRow_SSE2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + InterpolateRow = InterpolateRow_Any_SSSE3; + if (IS_ALIGNED(src_width, 16)) { + InterpolateRow = InterpolateRow_SSSE3; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(src_width, 32)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_NEON; + if (IS_ALIGNED(src_width, 16)) { + InterpolateRow = InterpolateRow_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2)) { + InterpolateRow = InterpolateRow_Any_MIPS_DSPR2; + if (IS_ALIGNED(src_width, 4)) { + InterpolateRow = InterpolateRow_MIPS_DSPR2; + } + } +#endif + + +#if defined(HAS_SCALEFILTERCOLS_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_SSSE3; + } +#endif +#if defined(HAS_SCALEFILTERCOLS_NEON) + if (TestCpuFlag(kCpuHasNEON) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_Any_NEON; + if (IS_ALIGNED(dst_width, 8)) { + ScaleFilterCols = ScaleFilterCols_NEON; + } + } +#endif + if (y > max_y) { + y = max_y; + } + + for (j = 0; j < dst_height; ++j) { + int yi = y >> 16; + const uint8* src = src_ptr + yi * src_stride; + if (filtering == kFilterLinear) { + ScaleFilterCols(dst_ptr, src, dst_width, x, dx); + } else { + int yf = (y >> 8) & 255; + InterpolateRow(row, src, src_stride, src_width, yf); + ScaleFilterCols(dst_ptr, row, dst_width, x, dx); + } + dst_ptr += dst_stride; + y += dy; + if (y > max_y) { + y = max_y; + } + } + free_aligned_buffer_64(row); +} + +void ScalePlaneBilinearDown_16(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint16* src_ptr, uint16* dst_ptr, + enum FilterMode filtering) { + // Initial source x/y coordinate and step values as 16.16 fixed point. + int x = 0; + int y = 0; + int dx = 0; + int dy = 0; + // TODO(fbarchard): Consider not allocating row buffer for kFilterLinear. + // Allocate a row buffer. + align_buffer_64(row, src_width * 2); + + const int max_y = (src_height - 1) << 16; + int j; + void (*ScaleFilterCols)(uint16* dst_ptr, const uint16* src_ptr, + int dst_width, int x, int dx) = + (src_width >= 32768) ? ScaleFilterCols64_16_C : ScaleFilterCols_16_C; + void (*InterpolateRow)(uint16* dst_ptr, const uint16* src_ptr, + ptrdiff_t src_stride, int dst_width, int source_y_fraction) = + InterpolateRow_16_C; + ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, + &x, &y, &dx, &dy); + src_width = Abs(src_width); + +#if defined(HAS_INTERPOLATEROW_16_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + InterpolateRow = InterpolateRow_Any_16_SSE2; + if (IS_ALIGNED(src_width, 16)) { + InterpolateRow = InterpolateRow_16_SSE2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + InterpolateRow = InterpolateRow_Any_16_SSSE3; + if (IS_ALIGNED(src_width, 16)) { + InterpolateRow = InterpolateRow_16_SSSE3; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_16_AVX2; + if (IS_ALIGNED(src_width, 32)) { + InterpolateRow = InterpolateRow_16_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_16_NEON; + if (IS_ALIGNED(src_width, 16)) { + InterpolateRow = InterpolateRow_16_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2)) { + InterpolateRow = InterpolateRow_Any_16_MIPS_DSPR2; + if (IS_ALIGNED(src_width, 4)) { + InterpolateRow = InterpolateRow_16_MIPS_DSPR2; + } + } +#endif + + +#if defined(HAS_SCALEFILTERCOLS_16_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_16_SSSE3; + } +#endif + if (y > max_y) { + y = max_y; + } + + for (j = 0; j < dst_height; ++j) { + int yi = y >> 16; + const uint16* src = src_ptr + yi * src_stride; + if (filtering == kFilterLinear) { + ScaleFilterCols(dst_ptr, src, dst_width, x, dx); + } else { + int yf = (y >> 8) & 255; + InterpolateRow((uint16*)row, src, src_stride, src_width, yf); + ScaleFilterCols(dst_ptr, (uint16*)row, dst_width, x, dx); + } + dst_ptr += dst_stride; + y += dy; + if (y > max_y) { + y = max_y; + } + } + free_aligned_buffer_64(row); +} + +// Scale up down with bilinear interpolation. +void ScalePlaneBilinearUp(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_ptr, uint8* dst_ptr, + enum FilterMode filtering) { + int j; + // Initial source x/y coordinate and step values as 16.16 fixed point. + int x = 0; + int y = 0; + int dx = 0; + int dy = 0; + const int max_y = (src_height - 1) << 16; + void (*InterpolateRow)(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, int source_y_fraction) = + InterpolateRow_C; + void (*ScaleFilterCols)(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx) = + filtering ? ScaleFilterCols_C : ScaleCols_C; + ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, + &x, &y, &dx, &dy); + src_width = Abs(src_width); + +#if defined(HAS_INTERPOLATEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + InterpolateRow = InterpolateRow_Any_SSE2; + if (IS_ALIGNED(dst_width, 16)) { + InterpolateRow = InterpolateRow_SSE2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + InterpolateRow = InterpolateRow_Any_SSSE3; + if (IS_ALIGNED(dst_width, 16)) { + InterpolateRow = InterpolateRow_SSSE3; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(dst_width, 32)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_NEON; + if (IS_ALIGNED(dst_width, 16)) { + InterpolateRow = InterpolateRow_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2)) { + InterpolateRow = InterpolateRow_Any_MIPS_DSPR2; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_MIPS_DSPR2; + } + } +#endif + + if (filtering && src_width >= 32768) { + ScaleFilterCols = ScaleFilterCols64_C; + } +#if defined(HAS_SCALEFILTERCOLS_SSSE3) + if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_SSSE3; + } +#endif +#if defined(HAS_SCALEFILTERCOLS_NEON) + if (filtering && TestCpuFlag(kCpuHasNEON) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_Any_NEON; + if (IS_ALIGNED(dst_width, 8)) { + ScaleFilterCols = ScaleFilterCols_NEON; + } + } +#endif + if (!filtering && src_width * 2 == dst_width && x < 0x8000) { + ScaleFilterCols = ScaleColsUp2_C; +#if defined(HAS_SCALECOLS_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { + ScaleFilterCols = ScaleColsUp2_SSE2; + } +#endif + } + + if (y > max_y) { + y = max_y; + } + { + int yi = y >> 16; + const uint8* src = src_ptr + yi * src_stride; + + // Allocate 2 row buffers. + const int kRowSize = (dst_width + 31) & ~31; + align_buffer_64(row, kRowSize * 2); + + uint8* rowptr = row; + int rowstride = kRowSize; + int lasty = yi; + + ScaleFilterCols(rowptr, src, dst_width, x, dx); + if (src_height > 1) { + src += src_stride; + } + ScaleFilterCols(rowptr + rowstride, src, dst_width, x, dx); + src += src_stride; + + for (j = 0; j < dst_height; ++j) { + yi = y >> 16; + if (yi != lasty) { + if (y > max_y) { + y = max_y; + yi = y >> 16; + src = src_ptr + yi * src_stride; + } + if (yi != lasty) { + ScaleFilterCols(rowptr, src, dst_width, x, dx); + rowptr += rowstride; + rowstride = -rowstride; + lasty = yi; + src += src_stride; + } + } + if (filtering == kFilterLinear) { + InterpolateRow(dst_ptr, rowptr, 0, dst_width, 0); + } else { + int yf = (y >> 8) & 255; + InterpolateRow(dst_ptr, rowptr, rowstride, dst_width, yf); + } + dst_ptr += dst_stride; + y += dy; + } + free_aligned_buffer_64(row); + } +} + +void ScalePlaneBilinearUp_16(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint16* src_ptr, uint16* dst_ptr, + enum FilterMode filtering) { + int j; + // Initial source x/y coordinate and step values as 16.16 fixed point. + int x = 0; + int y = 0; + int dx = 0; + int dy = 0; + const int max_y = (src_height - 1) << 16; + void (*InterpolateRow)(uint16* dst_ptr, const uint16* src_ptr, + ptrdiff_t src_stride, int dst_width, int source_y_fraction) = + InterpolateRow_16_C; + void (*ScaleFilterCols)(uint16* dst_ptr, const uint16* src_ptr, + int dst_width, int x, int dx) = + filtering ? ScaleFilterCols_16_C : ScaleCols_16_C; + ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, + &x, &y, &dx, &dy); + src_width = Abs(src_width); + +#if defined(HAS_INTERPOLATEROW_16_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + InterpolateRow = InterpolateRow_Any_16_SSE2; + if (IS_ALIGNED(dst_width, 16)) { + InterpolateRow = InterpolateRow_16_SSE2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + InterpolateRow = InterpolateRow_Any_16_SSSE3; + if (IS_ALIGNED(dst_width, 16)) { + InterpolateRow = InterpolateRow_16_SSSE3; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_16_AVX2; + if (IS_ALIGNED(dst_width, 32)) { + InterpolateRow = InterpolateRow_16_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_16_NEON; + if (IS_ALIGNED(dst_width, 16)) { + InterpolateRow = InterpolateRow_16_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2)) { + InterpolateRow = InterpolateRow_Any_16_MIPS_DSPR2; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_16_MIPS_DSPR2; + } + } +#endif + + if (filtering && src_width >= 32768) { + ScaleFilterCols = ScaleFilterCols64_16_C; + } +#if defined(HAS_SCALEFILTERCOLS_16_SSSE3) + if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_16_SSSE3; + } +#endif + if (!filtering && src_width * 2 == dst_width && x < 0x8000) { + ScaleFilterCols = ScaleColsUp2_16_C; +#if defined(HAS_SCALECOLS_16_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { + ScaleFilterCols = ScaleColsUp2_16_SSE2; + } +#endif + } + + if (y > max_y) { + y = max_y; + } + { + int yi = y >> 16; + const uint16* src = src_ptr + yi * src_stride; + + // Allocate 2 row buffers. + const int kRowSize = (dst_width + 31) & ~31; + align_buffer_64(row, kRowSize * 4); + + uint16* rowptr = (uint16*)row; + int rowstride = kRowSize; + int lasty = yi; + + ScaleFilterCols(rowptr, src, dst_width, x, dx); + if (src_height > 1) { + src += src_stride; + } + ScaleFilterCols(rowptr + rowstride, src, dst_width, x, dx); + src += src_stride; + + for (j = 0; j < dst_height; ++j) { + yi = y >> 16; + if (yi != lasty) { + if (y > max_y) { + y = max_y; + yi = y >> 16; + src = src_ptr + yi * src_stride; + } + if (yi != lasty) { + ScaleFilterCols(rowptr, src, dst_width, x, dx); + rowptr += rowstride; + rowstride = -rowstride; + lasty = yi; + src += src_stride; + } + } + if (filtering == kFilterLinear) { + InterpolateRow(dst_ptr, rowptr, 0, dst_width, 0); + } else { + int yf = (y >> 8) & 255; + InterpolateRow(dst_ptr, rowptr, rowstride, dst_width, yf); + } + dst_ptr += dst_stride; + y += dy; + } + free_aligned_buffer_64(row); + } +} + +// Scale Plane to/from any dimensions, without interpolation. +// Fixed point math is used for performance: The upper 16 bits +// of x and dx is the integer part of the source position and +// the lower 16 bits are the fixed decimal part. + +static void ScalePlaneSimple(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_ptr, uint8* dst_ptr) { + int i; + void (*ScaleCols)(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx) = ScaleCols_C; + // Initial source x/y coordinate and step values as 16.16 fixed point. + int x = 0; + int y = 0; + int dx = 0; + int dy = 0; + ScaleSlope(src_width, src_height, dst_width, dst_height, kFilterNone, + &x, &y, &dx, &dy); + src_width = Abs(src_width); + + if (src_width * 2 == dst_width && x < 0x8000) { + ScaleCols = ScaleColsUp2_C; +#if defined(HAS_SCALECOLS_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { + ScaleCols = ScaleColsUp2_SSE2; + } +#endif + } + + for (i = 0; i < dst_height; ++i) { + ScaleCols(dst_ptr, src_ptr + (y >> 16) * src_stride, dst_width, x, dx); + dst_ptr += dst_stride; + y += dy; + } +} + +static void ScalePlaneSimple_16(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint16* src_ptr, uint16* dst_ptr) { + int i; + void (*ScaleCols)(uint16* dst_ptr, const uint16* src_ptr, + int dst_width, int x, int dx) = ScaleCols_16_C; + // Initial source x/y coordinate and step values as 16.16 fixed point. + int x = 0; + int y = 0; + int dx = 0; + int dy = 0; + ScaleSlope(src_width, src_height, dst_width, dst_height, kFilterNone, + &x, &y, &dx, &dy); + src_width = Abs(src_width); + + if (src_width * 2 == dst_width && x < 0x8000) { + ScaleCols = ScaleColsUp2_16_C; +#if defined(HAS_SCALECOLS_16_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { + ScaleCols = ScaleColsUp2_16_SSE2; + } +#endif + } + + for (i = 0; i < dst_height; ++i) { + ScaleCols(dst_ptr, src_ptr + (y >> 16) * src_stride, + dst_width, x, dx); + dst_ptr += dst_stride; + y += dy; + } +} + +// Scale a plane. +// This function dispatches to a specialized scaler based on scale factor. + +LIBYUV_API +void ScalePlane(const uint8* src, int src_stride, + int src_width, int src_height, + uint8* dst, int dst_stride, + int dst_width, int dst_height, + enum FilterMode filtering) { + // Simplify filtering when possible. + filtering = ScaleFilterReduce(src_width, src_height, + dst_width, dst_height, filtering); + + // Negative height means invert the image. + if (src_height < 0) { + src_height = -src_height; + src = src + (src_height - 1) * src_stride; + src_stride = -src_stride; + } + + // Use specialized scales to improve performance for common resolutions. + // For example, all the 1/2 scalings will use ScalePlaneDown2() + if (dst_width == src_width && dst_height == src_height) { + // Straight copy. + CopyPlane(src, src_stride, dst, dst_stride, dst_width, dst_height); + return; + } + if (dst_width == src_width && filtering != kFilterBox) { + int dy = FixedDiv(src_height, dst_height); + // Arbitrary scale vertically, but unscaled horizontally. + ScalePlaneVertical(src_height, + dst_width, dst_height, + src_stride, dst_stride, src, dst, + 0, 0, dy, 1, filtering); + return; + } + if (dst_width <= Abs(src_width) && dst_height <= src_height) { + // Scale down. + if (4 * dst_width == 3 * src_width && + 4 * dst_height == 3 * src_height) { + // optimized, 3/4 + ScalePlaneDown34(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst, filtering); + return; + } + if (2 * dst_width == src_width && 2 * dst_height == src_height) { + // optimized, 1/2 + ScalePlaneDown2(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst, filtering); + return; + } + // 3/8 rounded up for odd sized chroma height. + if (8 * dst_width == 3 * src_width && + dst_height == ((src_height * 3 + 7) / 8)) { + // optimized, 3/8 + ScalePlaneDown38(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst, filtering); + return; + } + if (4 * dst_width == src_width && 4 * dst_height == src_height && + (filtering == kFilterBox || filtering == kFilterNone)) { + // optimized, 1/4 + ScalePlaneDown4(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst, filtering); + return; + } + } + if (filtering == kFilterBox && dst_height * 2 < src_height) { + ScalePlaneBox(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst); + return; + } + if (filtering && dst_height > src_height) { + ScalePlaneBilinearUp(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst, filtering); + return; + } + if (filtering) { + ScalePlaneBilinearDown(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst, filtering); + return; + } + ScalePlaneSimple(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst); +} + +LIBYUV_API +void ScalePlane_16(const uint16* src, int src_stride, + int src_width, int src_height, + uint16* dst, int dst_stride, + int dst_width, int dst_height, + enum FilterMode filtering) { + // Simplify filtering when possible. + filtering = ScaleFilterReduce(src_width, src_height, + dst_width, dst_height, filtering); + + // Negative height means invert the image. + if (src_height < 0) { + src_height = -src_height; + src = src + (src_height - 1) * src_stride; + src_stride = -src_stride; + } + + // Use specialized scales to improve performance for common resolutions. + // For example, all the 1/2 scalings will use ScalePlaneDown2() + if (dst_width == src_width && dst_height == src_height) { + // Straight copy. + CopyPlane_16(src, src_stride, dst, dst_stride, dst_width, dst_height); + return; + } + if (dst_width == src_width) { + int dy = FixedDiv(src_height, dst_height); + // Arbitrary scale vertically, but unscaled vertically. + ScalePlaneVertical_16(src_height, + dst_width, dst_height, + src_stride, dst_stride, src, dst, + 0, 0, dy, 1, filtering); + return; + } + if (dst_width <= Abs(src_width) && dst_height <= src_height) { + // Scale down. + if (4 * dst_width == 3 * src_width && + 4 * dst_height == 3 * src_height) { + // optimized, 3/4 + ScalePlaneDown34_16(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst, filtering); + return; + } + if (2 * dst_width == src_width && 2 * dst_height == src_height) { + // optimized, 1/2 + ScalePlaneDown2_16(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst, filtering); + return; + } + // 3/8 rounded up for odd sized chroma height. + if (8 * dst_width == 3 * src_width && + dst_height == ((src_height * 3 + 7) / 8)) { + // optimized, 3/8 + ScalePlaneDown38_16(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst, filtering); + return; + } + if (4 * dst_width == src_width && 4 * dst_height == src_height && + filtering != kFilterBilinear) { + // optimized, 1/4 + ScalePlaneDown4_16(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst, filtering); + return; + } + } + if (filtering == kFilterBox && dst_height * 2 < src_height) { + ScalePlaneBox_16(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst); + return; + } + if (filtering && dst_height > src_height) { + ScalePlaneBilinearUp_16(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst, filtering); + return; + } + if (filtering) { + ScalePlaneBilinearDown_16(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst, filtering); + return; + } + ScalePlaneSimple_16(src_width, src_height, dst_width, dst_height, + src_stride, dst_stride, src, dst); +} + +// Scale an I420 image. +// This function in turn calls a scaling function for each plane. + +LIBYUV_API +int I420Scale(const uint8* src_y, int src_stride_y, + const uint8* src_u, int src_stride_u, + const uint8* src_v, int src_stride_v, + int src_width, int src_height, + uint8* dst_y, int dst_stride_y, + uint8* dst_u, int dst_stride_u, + uint8* dst_v, int dst_stride_v, + int dst_width, int dst_height, + enum FilterMode filtering) { + int src_halfwidth = SUBSAMPLE(src_width, 1, 1); + int src_halfheight = SUBSAMPLE(src_height, 1, 1); + int dst_halfwidth = SUBSAMPLE(dst_width, 1, 1); + int dst_halfheight = SUBSAMPLE(dst_height, 1, 1); + if (!src_y || !src_u || !src_v || src_width == 0 || src_height == 0 || + src_width > 32768 || src_height > 32768 || + !dst_y || !dst_u || !dst_v || dst_width <= 0 || dst_height <= 0) { + return -1; + } + + ScalePlane(src_y, src_stride_y, src_width, src_height, + dst_y, dst_stride_y, dst_width, dst_height, + filtering); + ScalePlane(src_u, src_stride_u, src_halfwidth, src_halfheight, + dst_u, dst_stride_u, dst_halfwidth, dst_halfheight, + filtering); + ScalePlane(src_v, src_stride_v, src_halfwidth, src_halfheight, + dst_v, dst_stride_v, dst_halfwidth, dst_halfheight, + filtering); + return 0; +} + +LIBYUV_API +int I420Scale_16(const uint16* src_y, int src_stride_y, + const uint16* src_u, int src_stride_u, + const uint16* src_v, int src_stride_v, + int src_width, int src_height, + uint16* dst_y, int dst_stride_y, + uint16* dst_u, int dst_stride_u, + uint16* dst_v, int dst_stride_v, + int dst_width, int dst_height, + enum FilterMode filtering) { + int src_halfwidth = SUBSAMPLE(src_width, 1, 1); + int src_halfheight = SUBSAMPLE(src_height, 1, 1); + int dst_halfwidth = SUBSAMPLE(dst_width, 1, 1); + int dst_halfheight = SUBSAMPLE(dst_height, 1, 1); + if (!src_y || !src_u || !src_v || src_width == 0 || src_height == 0 || + src_width > 32768 || src_height > 32768 || + !dst_y || !dst_u || !dst_v || dst_width <= 0 || dst_height <= 0) { + return -1; + } + + ScalePlane_16(src_y, src_stride_y, src_width, src_height, + dst_y, dst_stride_y, dst_width, dst_height, + filtering); + ScalePlane_16(src_u, src_stride_u, src_halfwidth, src_halfheight, + dst_u, dst_stride_u, dst_halfwidth, dst_halfheight, + filtering); + ScalePlane_16(src_v, src_stride_v, src_halfwidth, src_halfheight, + dst_v, dst_stride_v, dst_halfwidth, dst_halfheight, + filtering); + return 0; +} + +// Deprecated api +LIBYUV_API +int Scale(const uint8* src_y, const uint8* src_u, const uint8* src_v, + int src_stride_y, int src_stride_u, int src_stride_v, + int src_width, int src_height, + uint8* dst_y, uint8* dst_u, uint8* dst_v, + int dst_stride_y, int dst_stride_u, int dst_stride_v, + int dst_width, int dst_height, + LIBYUV_BOOL interpolate) { + return I420Scale(src_y, src_stride_y, + src_u, src_stride_u, + src_v, src_stride_v, + src_width, src_height, + dst_y, dst_stride_y, + dst_u, dst_stride_u, + dst_v, dst_stride_v, + dst_width, dst_height, + interpolate ? kFilterBox : kFilterNone); +} + +// Deprecated api +LIBYUV_API +int ScaleOffset(const uint8* src, int src_width, int src_height, + uint8* dst, int dst_width, int dst_height, int dst_yoffset, + LIBYUV_BOOL interpolate) { + // Chroma requires offset to multiple of 2. + int dst_yoffset_even = dst_yoffset & ~1; + int src_halfwidth = SUBSAMPLE(src_width, 1, 1); + int src_halfheight = SUBSAMPLE(src_height, 1, 1); + int dst_halfwidth = SUBSAMPLE(dst_width, 1, 1); + int dst_halfheight = SUBSAMPLE(dst_height, 1, 1); + int aheight = dst_height - dst_yoffset_even * 2; // actual output height + const uint8* src_y = src; + const uint8* src_u = src + src_width * src_height; + const uint8* src_v = src + src_width * src_height + + src_halfwidth * src_halfheight; + uint8* dst_y = dst + dst_yoffset_even * dst_width; + uint8* dst_u = dst + dst_width * dst_height + + (dst_yoffset_even >> 1) * dst_halfwidth; + uint8* dst_v = dst + dst_width * dst_height + dst_halfwidth * dst_halfheight + + (dst_yoffset_even >> 1) * dst_halfwidth; + if (!src || src_width <= 0 || src_height <= 0 || + !dst || dst_width <= 0 || dst_height <= 0 || dst_yoffset_even < 0 || + dst_yoffset_even >= dst_height) { + return -1; + } + return I420Scale(src_y, src_width, + src_u, src_halfwidth, + src_v, src_halfwidth, + src_width, src_height, + dst_y, dst_width, + dst_u, dst_halfwidth, + dst_v, dst_halfwidth, + dst_width, aheight, + interpolate ? kFilterBox : kFilterNone); +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/scale_any.cc b/third_party/aom/third_party/libyuv/source/scale_any.cc new file mode 100644 index 0000000000..2f6a2c8baf --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/scale_any.cc @@ -0,0 +1,200 @@ +/* + * Copyright 2015 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/scale.h" +#include "libyuv/scale_row.h" + +#include "libyuv/basic_types.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Definition for ScaleFilterCols, ScaleARGBCols and ScaleARGBFilterCols +#define CANY(NAMEANY, TERP_SIMD, TERP_C, BPP, MASK) \ + void NAMEANY(uint8* dst_ptr, const uint8* src_ptr, \ + int dst_width, int x, int dx) { \ + int n = dst_width & ~MASK; \ + if (n > 0) { \ + TERP_SIMD(dst_ptr, src_ptr, n, x, dx); \ + } \ + TERP_C(dst_ptr + n * BPP, src_ptr, \ + dst_width & MASK, x + n * dx, dx); \ + } + +#ifdef HAS_SCALEFILTERCOLS_NEON +CANY(ScaleFilterCols_Any_NEON, ScaleFilterCols_NEON, ScaleFilterCols_C, 1, 7) +#endif +#ifdef HAS_SCALEARGBCOLS_NEON +CANY(ScaleARGBCols_Any_NEON, ScaleARGBCols_NEON, ScaleARGBCols_C, 4, 7) +#endif +#ifdef HAS_SCALEARGBFILTERCOLS_NEON +CANY(ScaleARGBFilterCols_Any_NEON, ScaleARGBFilterCols_NEON, + ScaleARGBFilterCols_C, 4, 3) +#endif +#undef CANY + +// Fixed scale down. +#define SDANY(NAMEANY, SCALEROWDOWN_SIMD, SCALEROWDOWN_C, FACTOR, BPP, MASK) \ + void NAMEANY(const uint8* src_ptr, ptrdiff_t src_stride, \ + uint8* dst_ptr, int dst_width) { \ + int r = (int)((unsigned int)dst_width % (MASK + 1)); \ + int n = dst_width - r; \ + if (n > 0) { \ + SCALEROWDOWN_SIMD(src_ptr, src_stride, dst_ptr, n); \ + } \ + SCALEROWDOWN_C(src_ptr + (n * FACTOR) * BPP, src_stride, \ + dst_ptr + n * BPP, r); \ + } + +#ifdef HAS_SCALEROWDOWN2_SSE2 +SDANY(ScaleRowDown2_Any_SSE2, ScaleRowDown2_SSE2, ScaleRowDown2_C, 2, 1, 15) +SDANY(ScaleRowDown2Linear_Any_SSE2, ScaleRowDown2Linear_SSE2, + ScaleRowDown2Linear_C, 2, 1, 15) +SDANY(ScaleRowDown2Box_Any_SSE2, ScaleRowDown2Box_SSE2, ScaleRowDown2Box_C, + 2, 1, 15) +#endif +#ifdef HAS_SCALEROWDOWN2_AVX2 +SDANY(ScaleRowDown2_Any_AVX2, ScaleRowDown2_AVX2, ScaleRowDown2_C, 2, 1, 31) +SDANY(ScaleRowDown2Linear_Any_AVX2, ScaleRowDown2Linear_AVX2, + ScaleRowDown2Linear_C, 2, 1, 31) +SDANY(ScaleRowDown2Box_Any_AVX2, ScaleRowDown2Box_AVX2, ScaleRowDown2Box_C, + 2, 1, 31) +#endif +#ifdef HAS_SCALEROWDOWN2_NEON +SDANY(ScaleRowDown2_Any_NEON, ScaleRowDown2_NEON, ScaleRowDown2_C, 2, 1, 15) +SDANY(ScaleRowDown2Linear_Any_NEON, ScaleRowDown2Linear_NEON, + ScaleRowDown2Linear_C, 2, 1, 15) +SDANY(ScaleRowDown2Box_Any_NEON, ScaleRowDown2Box_NEON, + ScaleRowDown2Box_C, 2, 1, 15) +#endif +#ifdef HAS_SCALEROWDOWN4_SSE2 +SDANY(ScaleRowDown4_Any_SSE2, ScaleRowDown4_SSE2, ScaleRowDown4_C, 4, 1, 7) +SDANY(ScaleRowDown4Box_Any_SSE2, ScaleRowDown4Box_SSE2, ScaleRowDown4Box_C, + 4, 1, 7) +#endif +#ifdef HAS_SCALEROWDOWN4_AVX2 +SDANY(ScaleRowDown4_Any_AVX2, ScaleRowDown4_AVX2, ScaleRowDown4_C, 4, 1, 15) +SDANY(ScaleRowDown4Box_Any_AVX2, ScaleRowDown4Box_AVX2, ScaleRowDown4Box_C, + 4, 1, 15) +#endif +#ifdef HAS_SCALEROWDOWN4_NEON +SDANY(ScaleRowDown4_Any_NEON, ScaleRowDown4_NEON, ScaleRowDown4_C, 4, 1, 7) +SDANY(ScaleRowDown4Box_Any_NEON, ScaleRowDown4Box_NEON, ScaleRowDown4Box_C, + 4, 1, 7) +#endif +#ifdef HAS_SCALEROWDOWN34_SSSE3 +SDANY(ScaleRowDown34_Any_SSSE3, ScaleRowDown34_SSSE3, + ScaleRowDown34_C, 4 / 3, 1, 23) +SDANY(ScaleRowDown34_0_Box_Any_SSSE3, ScaleRowDown34_0_Box_SSSE3, + ScaleRowDown34_0_Box_C, 4 / 3, 1, 23) +SDANY(ScaleRowDown34_1_Box_Any_SSSE3, ScaleRowDown34_1_Box_SSSE3, + ScaleRowDown34_1_Box_C, 4 / 3, 1, 23) +#endif +#ifdef HAS_SCALEROWDOWN34_NEON +SDANY(ScaleRowDown34_Any_NEON, ScaleRowDown34_NEON, + ScaleRowDown34_C, 4 / 3, 1, 23) +SDANY(ScaleRowDown34_0_Box_Any_NEON, ScaleRowDown34_0_Box_NEON, + ScaleRowDown34_0_Box_C, 4 / 3, 1, 23) +SDANY(ScaleRowDown34_1_Box_Any_NEON, ScaleRowDown34_1_Box_NEON, + ScaleRowDown34_1_Box_C, 4 / 3, 1, 23) +#endif +#ifdef HAS_SCALEROWDOWN38_SSSE3 +SDANY(ScaleRowDown38_Any_SSSE3, ScaleRowDown38_SSSE3, + ScaleRowDown38_C, 8 / 3, 1, 11) +SDANY(ScaleRowDown38_3_Box_Any_SSSE3, ScaleRowDown38_3_Box_SSSE3, + ScaleRowDown38_3_Box_C, 8 / 3, 1, 5) +SDANY(ScaleRowDown38_2_Box_Any_SSSE3, ScaleRowDown38_2_Box_SSSE3, + ScaleRowDown38_2_Box_C, 8 / 3, 1, 5) +#endif +#ifdef HAS_SCALEROWDOWN38_NEON +SDANY(ScaleRowDown38_Any_NEON, ScaleRowDown38_NEON, + ScaleRowDown38_C, 8 / 3, 1, 11) +SDANY(ScaleRowDown38_3_Box_Any_NEON, ScaleRowDown38_3_Box_NEON, + ScaleRowDown38_3_Box_C, 8 / 3, 1, 11) +SDANY(ScaleRowDown38_2_Box_Any_NEON, ScaleRowDown38_2_Box_NEON, + ScaleRowDown38_2_Box_C, 8 / 3, 1, 11) +#endif + +#ifdef HAS_SCALEARGBROWDOWN2_SSE2 +SDANY(ScaleARGBRowDown2_Any_SSE2, ScaleARGBRowDown2_SSE2, + ScaleARGBRowDown2_C, 2, 4, 3) +SDANY(ScaleARGBRowDown2Linear_Any_SSE2, ScaleARGBRowDown2Linear_SSE2, + ScaleARGBRowDown2Linear_C, 2, 4, 3) +SDANY(ScaleARGBRowDown2Box_Any_SSE2, ScaleARGBRowDown2Box_SSE2, + ScaleARGBRowDown2Box_C, 2, 4, 3) +#endif +#ifdef HAS_SCALEARGBROWDOWN2_NEON +SDANY(ScaleARGBRowDown2_Any_NEON, ScaleARGBRowDown2_NEON, + ScaleARGBRowDown2_C, 2, 4, 7) +SDANY(ScaleARGBRowDown2Linear_Any_NEON, ScaleARGBRowDown2Linear_NEON, + ScaleARGBRowDown2Linear_C, 2, 4, 7) +SDANY(ScaleARGBRowDown2Box_Any_NEON, ScaleARGBRowDown2Box_NEON, + ScaleARGBRowDown2Box_C, 2, 4, 7) +#endif +#undef SDANY + +// Scale down by even scale factor. +#define SDAANY(NAMEANY, SCALEROWDOWN_SIMD, SCALEROWDOWN_C, BPP, MASK) \ + void NAMEANY(const uint8* src_ptr, ptrdiff_t src_stride, int src_stepx, \ + uint8* dst_ptr, int dst_width) { \ + int r = (int)((unsigned int)dst_width % (MASK + 1)); \ + int n = dst_width - r; \ + if (n > 0) { \ + SCALEROWDOWN_SIMD(src_ptr, src_stride, src_stepx, dst_ptr, n); \ + } \ + SCALEROWDOWN_C(src_ptr + (n * src_stepx) * BPP, src_stride, \ + src_stepx, dst_ptr + n * BPP, r); \ + } + +#ifdef HAS_SCALEARGBROWDOWNEVEN_SSE2 +SDAANY(ScaleARGBRowDownEven_Any_SSE2, ScaleARGBRowDownEven_SSE2, + ScaleARGBRowDownEven_C, 4, 3) +SDAANY(ScaleARGBRowDownEvenBox_Any_SSE2, ScaleARGBRowDownEvenBox_SSE2, + ScaleARGBRowDownEvenBox_C, 4, 3) +#endif +#ifdef HAS_SCALEARGBROWDOWNEVEN_NEON +SDAANY(ScaleARGBRowDownEven_Any_NEON, ScaleARGBRowDownEven_NEON, + ScaleARGBRowDownEven_C, 4, 3) +SDAANY(ScaleARGBRowDownEvenBox_Any_NEON, ScaleARGBRowDownEvenBox_NEON, + ScaleARGBRowDownEvenBox_C, 4, 3) +#endif + +// Add rows box filter scale down. +#define SAANY(NAMEANY, SCALEADDROW_SIMD, SCALEADDROW_C, MASK) \ + void NAMEANY(const uint8* src_ptr, uint16* dst_ptr, int src_width) { \ + int n = src_width & ~MASK; \ + if (n > 0) { \ + SCALEADDROW_SIMD(src_ptr, dst_ptr, n); \ + } \ + SCALEADDROW_C(src_ptr + n, dst_ptr + n, src_width & MASK); \ + } + +#ifdef HAS_SCALEADDROW_SSE2 +SAANY(ScaleAddRow_Any_SSE2, ScaleAddRow_SSE2, ScaleAddRow_C, 15) +#endif +#ifdef HAS_SCALEADDROW_AVX2 +SAANY(ScaleAddRow_Any_AVX2, ScaleAddRow_AVX2, ScaleAddRow_C, 31) +#endif +#ifdef HAS_SCALEADDROW_NEON +SAANY(ScaleAddRow_Any_NEON, ScaleAddRow_NEON, ScaleAddRow_C, 15) +#endif +#undef SAANY + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + + + + + diff --git a/third_party/aom/third_party/libyuv/source/scale_argb.cc b/third_party/aom/third_party/libyuv/source/scale_argb.cc new file mode 100644 index 0000000000..40a2d1ab20 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/scale_argb.cc @@ -0,0 +1,853 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/scale.h" + +#include +#include + +#include "libyuv/cpu_id.h" +#include "libyuv/planar_functions.h" // For CopyARGB +#include "libyuv/row.h" +#include "libyuv/scale_row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +static __inline int Abs(int v) { + return v >= 0 ? v : -v; +} + +// ScaleARGB ARGB, 1/2 +// This is an optimized version for scaling down a ARGB to 1/2 of +// its original size. +static void ScaleARGBDown2(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_argb, uint8* dst_argb, + int x, int dx, int y, int dy, + enum FilterMode filtering) { + int j; + int row_stride = src_stride * (dy >> 16); + void (*ScaleARGBRowDown2)(const uint8* src_argb, ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) = + filtering == kFilterNone ? ScaleARGBRowDown2_C : + (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_C : + ScaleARGBRowDown2Box_C); + assert(dx == 65536 * 2); // Test scale factor of 2. + assert((dy & 0x1ffff) == 0); // Test vertical scale is multiple of 2. + // Advance to odd row, even column. + if (filtering == kFilterBilinear) { + src_argb += (y >> 16) * src_stride + (x >> 16) * 4; + } else { + src_argb += (y >> 16) * src_stride + ((x >> 16) - 1) * 4; + } + +#if defined(HAS_SCALEARGBROWDOWN2_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ScaleARGBRowDown2 = filtering == kFilterNone ? ScaleARGBRowDown2_Any_SSE2 : + (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_SSE2 : + ScaleARGBRowDown2Box_Any_SSE2); + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBRowDown2 = filtering == kFilterNone ? ScaleARGBRowDown2_SSE2 : + (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_SSE2 : + ScaleARGBRowDown2Box_SSE2); + } + } +#endif +#if defined(HAS_SCALEARGBROWDOWN2_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ScaleARGBRowDown2 = filtering == kFilterNone ? ScaleARGBRowDown2_Any_NEON : + (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_NEON : + ScaleARGBRowDown2Box_Any_NEON); + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBRowDown2 = filtering == kFilterNone ? ScaleARGBRowDown2_NEON : + (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_NEON : + ScaleARGBRowDown2Box_NEON); + } + } +#endif + + if (filtering == kFilterLinear) { + src_stride = 0; + } + for (j = 0; j < dst_height; ++j) { + ScaleARGBRowDown2(src_argb, src_stride, dst_argb, dst_width); + src_argb += row_stride; + dst_argb += dst_stride; + } +} + +// ScaleARGB ARGB, 1/4 +// This is an optimized version for scaling down a ARGB to 1/4 of +// its original size. +static void ScaleARGBDown4Box(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_argb, uint8* dst_argb, + int x, int dx, int y, int dy) { + int j; + // Allocate 2 rows of ARGB. + const int kRowSize = (dst_width * 2 * 4 + 31) & ~31; + align_buffer_64(row, kRowSize * 2); + int row_stride = src_stride * (dy >> 16); + void (*ScaleARGBRowDown2)(const uint8* src_argb, ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) = ScaleARGBRowDown2Box_C; + // Advance to odd row, even column. + src_argb += (y >> 16) * src_stride + (x >> 16) * 4; + assert(dx == 65536 * 4); // Test scale factor of 4. + assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4. +#if defined(HAS_SCALEARGBROWDOWN2_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_SSE2; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_SSE2; + } + } +#endif +#if defined(HAS_SCALEARGBROWDOWN2_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_NEON; + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_NEON; + } + } +#endif + + for (j = 0; j < dst_height; ++j) { + ScaleARGBRowDown2(src_argb, src_stride, row, dst_width * 2); + ScaleARGBRowDown2(src_argb + src_stride * 2, src_stride, + row + kRowSize, dst_width * 2); + ScaleARGBRowDown2(row, kRowSize, dst_argb, dst_width); + src_argb += row_stride; + dst_argb += dst_stride; + } + free_aligned_buffer_64(row); +} + +// ScaleARGB ARGB Even +// This is an optimized version for scaling down a ARGB to even +// multiple of its original size. +static void ScaleARGBDownEven(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_argb, uint8* dst_argb, + int x, int dx, int y, int dy, + enum FilterMode filtering) { + int j; + int col_step = dx >> 16; + int row_stride = (dy >> 16) * src_stride; + void (*ScaleARGBRowDownEven)(const uint8* src_argb, ptrdiff_t src_stride, + int src_step, uint8* dst_argb, int dst_width) = + filtering ? ScaleARGBRowDownEvenBox_C : ScaleARGBRowDownEven_C; + assert(IS_ALIGNED(src_width, 2)); + assert(IS_ALIGNED(src_height, 2)); + src_argb += (y >> 16) * src_stride + (x >> 16) * 4; +#if defined(HAS_SCALEARGBROWDOWNEVEN_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_SSE2 : + ScaleARGBRowDownEven_Any_SSE2; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_SSE2 : + ScaleARGBRowDownEven_SSE2; + } + } +#endif +#if defined(HAS_SCALEARGBROWDOWNEVEN_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_NEON : + ScaleARGBRowDownEven_Any_NEON; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_NEON : + ScaleARGBRowDownEven_NEON; + } + } +#endif + + if (filtering == kFilterLinear) { + src_stride = 0; + } + for (j = 0; j < dst_height; ++j) { + ScaleARGBRowDownEven(src_argb, src_stride, col_step, dst_argb, dst_width); + src_argb += row_stride; + dst_argb += dst_stride; + } +} + +// Scale ARGB down with bilinear interpolation. +static void ScaleARGBBilinearDown(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_argb, uint8* dst_argb, + int x, int dx, int y, int dy, + enum FilterMode filtering) { + int j; + void (*InterpolateRow)(uint8* dst_argb, const uint8* src_argb, + ptrdiff_t src_stride, int dst_width, int source_y_fraction) = + InterpolateRow_C; + void (*ScaleARGBFilterCols)(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) = + (src_width >= 32768) ? ScaleARGBFilterCols64_C : ScaleARGBFilterCols_C; + int64 xlast = x + (int64)(dst_width - 1) * dx; + int64 xl = (dx >= 0) ? x : xlast; + int64 xr = (dx >= 0) ? xlast : x; + int clip_src_width; + xl = (xl >> 16) & ~3; // Left edge aligned. + xr = (xr >> 16) + 1; // Right most pixel used. Bilinear uses 2 pixels. + xr = (xr + 1 + 3) & ~3; // 1 beyond 4 pixel aligned right most pixel. + if (xr > src_width) { + xr = src_width; + } + clip_src_width = (int)(xr - xl) * 4; // Width aligned to 4. + src_argb += xl * 4; + x -= (int)(xl << 16); +#if defined(HAS_INTERPOLATEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + InterpolateRow = InterpolateRow_Any_SSE2; + if (IS_ALIGNED(clip_src_width, 16)) { + InterpolateRow = InterpolateRow_SSE2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + InterpolateRow = InterpolateRow_Any_SSSE3; + if (IS_ALIGNED(clip_src_width, 16)) { + InterpolateRow = InterpolateRow_SSSE3; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(clip_src_width, 32)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_NEON; + if (IS_ALIGNED(clip_src_width, 16)) { + InterpolateRow = InterpolateRow_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && + IS_ALIGNED(src_argb, 4) && IS_ALIGNED(src_stride, 4)) { + InterpolateRow = InterpolateRow_Any_MIPS_DSPR2; + if (IS_ALIGNED(clip_src_width, 4)) { + InterpolateRow = InterpolateRow_MIPS_DSPR2; + } + } +#endif +#if defined(HAS_SCALEARGBFILTERCOLS_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3; + } +#endif +#if defined(HAS_SCALEARGBFILTERCOLS_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ScaleARGBFilterCols = ScaleARGBFilterCols_Any_NEON; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBFilterCols = ScaleARGBFilterCols_NEON; + } + } +#endif + // TODO(fbarchard): Consider not allocating row buffer for kFilterLinear. + // Allocate a row of ARGB. + { + align_buffer_64(row, clip_src_width * 4); + + const int max_y = (src_height - 1) << 16; + if (y > max_y) { + y = max_y; + } + for (j = 0; j < dst_height; ++j) { + int yi = y >> 16; + const uint8* src = src_argb + yi * src_stride; + if (filtering == kFilterLinear) { + ScaleARGBFilterCols(dst_argb, src, dst_width, x, dx); + } else { + int yf = (y >> 8) & 255; + InterpolateRow(row, src, src_stride, clip_src_width, yf); + ScaleARGBFilterCols(dst_argb, row, dst_width, x, dx); + } + dst_argb += dst_stride; + y += dy; + if (y > max_y) { + y = max_y; + } + } + free_aligned_buffer_64(row); + } +} + +// Scale ARGB up with bilinear interpolation. +static void ScaleARGBBilinearUp(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_argb, uint8* dst_argb, + int x, int dx, int y, int dy, + enum FilterMode filtering) { + int j; + void (*InterpolateRow)(uint8* dst_argb, const uint8* src_argb, + ptrdiff_t src_stride, int dst_width, int source_y_fraction) = + InterpolateRow_C; + void (*ScaleARGBFilterCols)(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) = + filtering ? ScaleARGBFilterCols_C : ScaleARGBCols_C; + const int max_y = (src_height - 1) << 16; +#if defined(HAS_INTERPOLATEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + InterpolateRow = InterpolateRow_Any_SSE2; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_SSE2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + InterpolateRow = InterpolateRow_Any_SSSE3; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_SSSE3; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_NEON; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && + IS_ALIGNED(dst_argb, 4) && IS_ALIGNED(dst_stride, 4)) { + InterpolateRow = InterpolateRow_MIPS_DSPR2; + } +#endif + if (src_width >= 32768) { + ScaleARGBFilterCols = filtering ? + ScaleARGBFilterCols64_C : ScaleARGBCols64_C; + } +#if defined(HAS_SCALEARGBFILTERCOLS_SSSE3) + if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3; + } +#endif +#if defined(HAS_SCALEARGBFILTERCOLS_NEON) + if (filtering && TestCpuFlag(kCpuHasNEON)) { + ScaleARGBFilterCols = ScaleARGBFilterCols_Any_NEON; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBFilterCols = ScaleARGBFilterCols_NEON; + } + } +#endif +#if defined(HAS_SCALEARGBCOLS_SSE2) + if (!filtering && TestCpuFlag(kCpuHasSSE2) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBCols_SSE2; + } +#endif +#if defined(HAS_SCALEARGBCOLS_NEON) + if (!filtering && TestCpuFlag(kCpuHasNEON)) { + ScaleARGBFilterCols = ScaleARGBCols_Any_NEON; + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBFilterCols = ScaleARGBCols_NEON; + } + } +#endif + if (!filtering && src_width * 2 == dst_width && x < 0x8000) { + ScaleARGBFilterCols = ScaleARGBColsUp2_C; +#if defined(HAS_SCALEARGBCOLSUP2_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { + ScaleARGBFilterCols = ScaleARGBColsUp2_SSE2; + } +#endif + } + + if (y > max_y) { + y = max_y; + } + + { + int yi = y >> 16; + const uint8* src = src_argb + yi * src_stride; + + // Allocate 2 rows of ARGB. + const int kRowSize = (dst_width * 4 + 31) & ~31; + align_buffer_64(row, kRowSize * 2); + + uint8* rowptr = row; + int rowstride = kRowSize; + int lasty = yi; + + ScaleARGBFilterCols(rowptr, src, dst_width, x, dx); + if (src_height > 1) { + src += src_stride; + } + ScaleARGBFilterCols(rowptr + rowstride, src, dst_width, x, dx); + src += src_stride; + + for (j = 0; j < dst_height; ++j) { + yi = y >> 16; + if (yi != lasty) { + if (y > max_y) { + y = max_y; + yi = y >> 16; + src = src_argb + yi * src_stride; + } + if (yi != lasty) { + ScaleARGBFilterCols(rowptr, src, dst_width, x, dx); + rowptr += rowstride; + rowstride = -rowstride; + lasty = yi; + src += src_stride; + } + } + if (filtering == kFilterLinear) { + InterpolateRow(dst_argb, rowptr, 0, dst_width * 4, 0); + } else { + int yf = (y >> 8) & 255; + InterpolateRow(dst_argb, rowptr, rowstride, dst_width * 4, yf); + } + dst_argb += dst_stride; + y += dy; + } + free_aligned_buffer_64(row); + } +} + +#ifdef YUVSCALEUP +// Scale YUV to ARGB up with bilinear interpolation. +static void ScaleYUVToARGBBilinearUp(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride_y, + int src_stride_u, + int src_stride_v, + int dst_stride_argb, + const uint8* src_y, + const uint8* src_u, + const uint8* src_v, + uint8* dst_argb, + int x, int dx, int y, int dy, + enum FilterMode filtering) { + int j; + void (*I422ToARGBRow)(const uint8* y_buf, + const uint8* u_buf, + const uint8* v_buf, + uint8* rgb_buf, + int width) = I422ToARGBRow_C; +#if defined(HAS_I422TOARGBROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + I422ToARGBRow = I422ToARGBRow_Any_SSSE3; + if (IS_ALIGNED(src_width, 8)) { + I422ToARGBRow = I422ToARGBRow_SSSE3; + } + } +#endif +#if defined(HAS_I422TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + I422ToARGBRow = I422ToARGBRow_Any_AVX2; + if (IS_ALIGNED(src_width, 16)) { + I422ToARGBRow = I422ToARGBRow_AVX2; + } + } +#endif +#if defined(HAS_I422TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + I422ToARGBRow = I422ToARGBRow_Any_NEON; + if (IS_ALIGNED(src_width, 8)) { + I422ToARGBRow = I422ToARGBRow_NEON; + } + } +#endif +#if defined(HAS_I422TOARGBROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(src_width, 4) && + IS_ALIGNED(src_y, 4) && IS_ALIGNED(src_stride_y, 4) && + IS_ALIGNED(src_u, 2) && IS_ALIGNED(src_stride_u, 2) && + IS_ALIGNED(src_v, 2) && IS_ALIGNED(src_stride_v, 2) && + IS_ALIGNED(dst_argb, 4) && IS_ALIGNED(dst_stride_argb, 4)) { + I422ToARGBRow = I422ToARGBRow_MIPS_DSPR2; + } +#endif + + void (*InterpolateRow)(uint8* dst_argb, const uint8* src_argb, + ptrdiff_t src_stride, int dst_width, int source_y_fraction) = + InterpolateRow_C; +#if defined(HAS_INTERPOLATEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + InterpolateRow = InterpolateRow_Any_SSE2; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_SSE2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + InterpolateRow = InterpolateRow_Any_SSSE3; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_SSSE3; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_NEON; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && + IS_ALIGNED(dst_argb, 4) && IS_ALIGNED(dst_stride_argb, 4)) { + InterpolateRow = InterpolateRow_MIPS_DSPR2; + } +#endif + + void (*ScaleARGBFilterCols)(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) = + filtering ? ScaleARGBFilterCols_C : ScaleARGBCols_C; + if (src_width >= 32768) { + ScaleARGBFilterCols = filtering ? + ScaleARGBFilterCols64_C : ScaleARGBCols64_C; + } +#if defined(HAS_SCALEARGBFILTERCOLS_SSSE3) + if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3; + } +#endif +#if defined(HAS_SCALEARGBFILTERCOLS_NEON) + if (filtering && TestCpuFlag(kCpuHasNEON)) { + ScaleARGBFilterCols = ScaleARGBFilterCols_Any_NEON; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBFilterCols = ScaleARGBFilterCols_NEON; + } + } +#endif +#if defined(HAS_SCALEARGBCOLS_SSE2) + if (!filtering && TestCpuFlag(kCpuHasSSE2) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBCols_SSE2; + } +#endif +#if defined(HAS_SCALEARGBCOLS_NEON) + if (!filtering && TestCpuFlag(kCpuHasNEON)) { + ScaleARGBFilterCols = ScaleARGBCols_Any_NEON; + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBFilterCols = ScaleARGBCols_NEON; + } + } +#endif + if (!filtering && src_width * 2 == dst_width && x < 0x8000) { + ScaleARGBFilterCols = ScaleARGBColsUp2_C; +#if defined(HAS_SCALEARGBCOLSUP2_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { + ScaleARGBFilterCols = ScaleARGBColsUp2_SSE2; + } +#endif + } + + const int max_y = (src_height - 1) << 16; + if (y > max_y) { + y = max_y; + } + const int kYShift = 1; // Shift Y by 1 to convert Y plane to UV coordinate. + int yi = y >> 16; + int uv_yi = yi >> kYShift; + const uint8* src_row_y = src_y + yi * src_stride_y; + const uint8* src_row_u = src_u + uv_yi * src_stride_u; + const uint8* src_row_v = src_v + uv_yi * src_stride_v; + + // Allocate 2 rows of ARGB. + const int kRowSize = (dst_width * 4 + 31) & ~31; + align_buffer_64(row, kRowSize * 2); + + // Allocate 1 row of ARGB for source conversion. + align_buffer_64(argb_row, src_width * 4); + + uint8* rowptr = row; + int rowstride = kRowSize; + int lasty = yi; + + // TODO(fbarchard): Convert first 2 rows of YUV to ARGB. + ScaleARGBFilterCols(rowptr, src_row_y, dst_width, x, dx); + if (src_height > 1) { + src_row_y += src_stride_y; + if (yi & 1) { + src_row_u += src_stride_u; + src_row_v += src_stride_v; + } + } + ScaleARGBFilterCols(rowptr + rowstride, src_row_y, dst_width, x, dx); + if (src_height > 2) { + src_row_y += src_stride_y; + if (!(yi & 1)) { + src_row_u += src_stride_u; + src_row_v += src_stride_v; + } + } + + for (j = 0; j < dst_height; ++j) { + yi = y >> 16; + if (yi != lasty) { + if (y > max_y) { + y = max_y; + yi = y >> 16; + uv_yi = yi >> kYShift; + src_row_y = src_y + yi * src_stride_y; + src_row_u = src_u + uv_yi * src_stride_u; + src_row_v = src_v + uv_yi * src_stride_v; + } + if (yi != lasty) { + // TODO(fbarchard): Convert the clipped region of row. + I422ToARGBRow(src_row_y, src_row_u, src_row_v, argb_row, src_width); + ScaleARGBFilterCols(rowptr, argb_row, dst_width, x, dx); + rowptr += rowstride; + rowstride = -rowstride; + lasty = yi; + src_row_y += src_stride_y; + if (yi & 1) { + src_row_u += src_stride_u; + src_row_v += src_stride_v; + } + } + } + if (filtering == kFilterLinear) { + InterpolateRow(dst_argb, rowptr, 0, dst_width * 4, 0); + } else { + int yf = (y >> 8) & 255; + InterpolateRow(dst_argb, rowptr, rowstride, dst_width * 4, yf); + } + dst_argb += dst_stride_argb; + y += dy; + } + free_aligned_buffer_64(row); + free_aligned_buffer_64(row_argb); +} +#endif + +// Scale ARGB to/from any dimensions, without interpolation. +// Fixed point math is used for performance: The upper 16 bits +// of x and dx is the integer part of the source position and +// the lower 16 bits are the fixed decimal part. + +static void ScaleARGBSimple(int src_width, int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_argb, uint8* dst_argb, + int x, int dx, int y, int dy) { + int j; + void (*ScaleARGBCols)(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) = + (src_width >= 32768) ? ScaleARGBCols64_C : ScaleARGBCols_C; +#if defined(HAS_SCALEARGBCOLS_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && src_width < 32768) { + ScaleARGBCols = ScaleARGBCols_SSE2; + } +#endif +#if defined(HAS_SCALEARGBCOLS_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ScaleARGBCols = ScaleARGBCols_Any_NEON; + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBCols = ScaleARGBCols_NEON; + } + } +#endif + if (src_width * 2 == dst_width && x < 0x8000) { + ScaleARGBCols = ScaleARGBColsUp2_C; +#if defined(HAS_SCALEARGBCOLSUP2_SSE2) + if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { + ScaleARGBCols = ScaleARGBColsUp2_SSE2; + } +#endif + } + + for (j = 0; j < dst_height; ++j) { + ScaleARGBCols(dst_argb, src_argb + (y >> 16) * src_stride, + dst_width, x, dx); + dst_argb += dst_stride; + y += dy; + } +} + +// ScaleARGB a ARGB. +// This function in turn calls a scaling function +// suitable for handling the desired resolutions. +static void ScaleARGB(const uint8* src, int src_stride, + int src_width, int src_height, + uint8* dst, int dst_stride, + int dst_width, int dst_height, + int clip_x, int clip_y, int clip_width, int clip_height, + enum FilterMode filtering) { + // Initial source x/y coordinate and step values as 16.16 fixed point. + int x = 0; + int y = 0; + int dx = 0; + int dy = 0; + // ARGB does not support box filter yet, but allow the user to pass it. + // Simplify filtering when possible. + filtering = ScaleFilterReduce(src_width, src_height, + dst_width, dst_height, + filtering); + + // Negative src_height means invert the image. + if (src_height < 0) { + src_height = -src_height; + src = src + (src_height - 1) * src_stride; + src_stride = -src_stride; + } + ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, + &x, &y, &dx, &dy); + src_width = Abs(src_width); + if (clip_x) { + int64 clipf = (int64)(clip_x) * dx; + x += (clipf & 0xffff); + src += (clipf >> 16) * 4; + dst += clip_x * 4; + } + if (clip_y) { + int64 clipf = (int64)(clip_y) * dy; + y += (clipf & 0xffff); + src += (clipf >> 16) * src_stride; + dst += clip_y * dst_stride; + } + + // Special case for integer step values. + if (((dx | dy) & 0xffff) == 0) { + if (!dx || !dy) { // 1 pixel wide and/or tall. + filtering = kFilterNone; + } else { + // Optimized even scale down. ie 2, 4, 6, 8, 10x. + if (!(dx & 0x10000) && !(dy & 0x10000)) { + if (dx == 0x20000) { + // Optimized 1/2 downsample. + ScaleARGBDown2(src_width, src_height, + clip_width, clip_height, + src_stride, dst_stride, src, dst, + x, dx, y, dy, filtering); + return; + } + if (dx == 0x40000 && filtering == kFilterBox) { + // Optimized 1/4 box downsample. + ScaleARGBDown4Box(src_width, src_height, + clip_width, clip_height, + src_stride, dst_stride, src, dst, + x, dx, y, dy); + return; + } + ScaleARGBDownEven(src_width, src_height, + clip_width, clip_height, + src_stride, dst_stride, src, dst, + x, dx, y, dy, filtering); + return; + } + // Optimized odd scale down. ie 3, 5, 7, 9x. + if ((dx & 0x10000) && (dy & 0x10000)) { + filtering = kFilterNone; + if (dx == 0x10000 && dy == 0x10000) { + // Straight copy. + ARGBCopy(src + (y >> 16) * src_stride + (x >> 16) * 4, src_stride, + dst, dst_stride, clip_width, clip_height); + return; + } + } + } + } + if (dx == 0x10000 && (x & 0xffff) == 0) { + // Arbitrary scale vertically, but unscaled vertically. + ScalePlaneVertical(src_height, + clip_width, clip_height, + src_stride, dst_stride, src, dst, + x, y, dy, 4, filtering); + return; + } + if (filtering && dy < 65536) { + ScaleARGBBilinearUp(src_width, src_height, + clip_width, clip_height, + src_stride, dst_stride, src, dst, + x, dx, y, dy, filtering); + return; + } + if (filtering) { + ScaleARGBBilinearDown(src_width, src_height, + clip_width, clip_height, + src_stride, dst_stride, src, dst, + x, dx, y, dy, filtering); + return; + } + ScaleARGBSimple(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst, + x, dx, y, dy); +} + +LIBYUV_API +int ARGBScaleClip(const uint8* src_argb, int src_stride_argb, + int src_width, int src_height, + uint8* dst_argb, int dst_stride_argb, + int dst_width, int dst_height, + int clip_x, int clip_y, int clip_width, int clip_height, + enum FilterMode filtering) { + if (!src_argb || src_width == 0 || src_height == 0 || + !dst_argb || dst_width <= 0 || dst_height <= 0 || + clip_x < 0 || clip_y < 0 || + clip_width > 32768 || clip_height > 32768 || + (clip_x + clip_width) > dst_width || + (clip_y + clip_height) > dst_height) { + return -1; + } + ScaleARGB(src_argb, src_stride_argb, src_width, src_height, + dst_argb, dst_stride_argb, dst_width, dst_height, + clip_x, clip_y, clip_width, clip_height, filtering); + return 0; +} + +// Scale an ARGB image. +LIBYUV_API +int ARGBScale(const uint8* src_argb, int src_stride_argb, + int src_width, int src_height, + uint8* dst_argb, int dst_stride_argb, + int dst_width, int dst_height, + enum FilterMode filtering) { + if (!src_argb || src_width == 0 || src_height == 0 || + src_width > 32768 || src_height > 32768 || + !dst_argb || dst_width <= 0 || dst_height <= 0) { + return -1; + } + ScaleARGB(src_argb, src_stride_argb, src_width, src_height, + dst_argb, dst_stride_argb, dst_width, dst_height, + 0, 0, dst_width, dst_height, filtering); + return 0; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/scale_common.cc b/third_party/aom/third_party/libyuv/source/scale_common.cc new file mode 100644 index 0000000000..1711f3d54c --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/scale_common.cc @@ -0,0 +1,1137 @@ +/* + * Copyright 2013 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/scale.h" + +#include +#include + +#include "libyuv/cpu_id.h" +#include "libyuv/planar_functions.h" // For CopyARGB +#include "libyuv/row.h" +#include "libyuv/scale_row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +static __inline int Abs(int v) { + return v >= 0 ? v : -v; +} + +// CPU agnostic row functions +void ScaleRowDown2_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + int x; + for (x = 0; x < dst_width - 1; x += 2) { + dst[0] = src_ptr[1]; + dst[1] = src_ptr[3]; + dst += 2; + src_ptr += 4; + } + if (dst_width & 1) { + dst[0] = src_ptr[1]; + } +} + +void ScaleRowDown2_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width) { + int x; + for (x = 0; x < dst_width - 1; x += 2) { + dst[0] = src_ptr[1]; + dst[1] = src_ptr[3]; + dst += 2; + src_ptr += 4; + } + if (dst_width & 1) { + dst[0] = src_ptr[1]; + } +} + +void ScaleRowDown2Linear_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + const uint8* s = src_ptr; + int x; + for (x = 0; x < dst_width - 1; x += 2) { + dst[0] = (s[0] + s[1] + 1) >> 1; + dst[1] = (s[2] + s[3] + 1) >> 1; + dst += 2; + s += 4; + } + if (dst_width & 1) { + dst[0] = (s[0] + s[1] + 1) >> 1; + } +} + +void ScaleRowDown2Linear_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width) { + const uint16* s = src_ptr; + int x; + for (x = 0; x < dst_width - 1; x += 2) { + dst[0] = (s[0] + s[1] + 1) >> 1; + dst[1] = (s[2] + s[3] + 1) >> 1; + dst += 2; + s += 4; + } + if (dst_width & 1) { + dst[0] = (s[0] + s[1] + 1) >> 1; + } +} + +void ScaleRowDown2Box_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + const uint8* s = src_ptr; + const uint8* t = src_ptr + src_stride; + int x; + for (x = 0; x < dst_width - 1; x += 2) { + dst[0] = (s[0] + s[1] + t[0] + t[1] + 2) >> 2; + dst[1] = (s[2] + s[3] + t[2] + t[3] + 2) >> 2; + dst += 2; + s += 4; + t += 4; + } + if (dst_width & 1) { + dst[0] = (s[0] + s[1] + t[0] + t[1] + 2) >> 2; + } +} + +void ScaleRowDown2Box_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width) { + const uint16* s = src_ptr; + const uint16* t = src_ptr + src_stride; + int x; + for (x = 0; x < dst_width - 1; x += 2) { + dst[0] = (s[0] + s[1] + t[0] + t[1] + 2) >> 2; + dst[1] = (s[2] + s[3] + t[2] + t[3] + 2) >> 2; + dst += 2; + s += 4; + t += 4; + } + if (dst_width & 1) { + dst[0] = (s[0] + s[1] + t[0] + t[1] + 2) >> 2; + } +} + +void ScaleRowDown4_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + int x; + for (x = 0; x < dst_width - 1; x += 2) { + dst[0] = src_ptr[2]; + dst[1] = src_ptr[6]; + dst += 2; + src_ptr += 8; + } + if (dst_width & 1) { + dst[0] = src_ptr[2]; + } +} + +void ScaleRowDown4_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width) { + int x; + for (x = 0; x < dst_width - 1; x += 2) { + dst[0] = src_ptr[2]; + dst[1] = src_ptr[6]; + dst += 2; + src_ptr += 8; + } + if (dst_width & 1) { + dst[0] = src_ptr[2]; + } +} + +void ScaleRowDown4Box_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + intptr_t stride = src_stride; + int x; + for (x = 0; x < dst_width - 1; x += 2) { + dst[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[3] + + src_ptr[stride + 0] + src_ptr[stride + 1] + + src_ptr[stride + 2] + src_ptr[stride + 3] + + src_ptr[stride * 2 + 0] + src_ptr[stride * 2 + 1] + + src_ptr[stride * 2 + 2] + src_ptr[stride * 2 + 3] + + src_ptr[stride * 3 + 0] + src_ptr[stride * 3 + 1] + + src_ptr[stride * 3 + 2] + src_ptr[stride * 3 + 3] + + 8) >> 4; + dst[1] = (src_ptr[4] + src_ptr[5] + src_ptr[6] + src_ptr[7] + + src_ptr[stride + 4] + src_ptr[stride + 5] + + src_ptr[stride + 6] + src_ptr[stride + 7] + + src_ptr[stride * 2 + 4] + src_ptr[stride * 2 + 5] + + src_ptr[stride * 2 + 6] + src_ptr[stride * 2 + 7] + + src_ptr[stride * 3 + 4] + src_ptr[stride * 3 + 5] + + src_ptr[stride * 3 + 6] + src_ptr[stride * 3 + 7] + + 8) >> 4; + dst += 2; + src_ptr += 8; + } + if (dst_width & 1) { + dst[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[3] + + src_ptr[stride + 0] + src_ptr[stride + 1] + + src_ptr[stride + 2] + src_ptr[stride + 3] + + src_ptr[stride * 2 + 0] + src_ptr[stride * 2 + 1] + + src_ptr[stride * 2 + 2] + src_ptr[stride * 2 + 3] + + src_ptr[stride * 3 + 0] + src_ptr[stride * 3 + 1] + + src_ptr[stride * 3 + 2] + src_ptr[stride * 3 + 3] + + 8) >> 4; + } +} + +void ScaleRowDown4Box_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width) { + intptr_t stride = src_stride; + int x; + for (x = 0; x < dst_width - 1; x += 2) { + dst[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[3] + + src_ptr[stride + 0] + src_ptr[stride + 1] + + src_ptr[stride + 2] + src_ptr[stride + 3] + + src_ptr[stride * 2 + 0] + src_ptr[stride * 2 + 1] + + src_ptr[stride * 2 + 2] + src_ptr[stride * 2 + 3] + + src_ptr[stride * 3 + 0] + src_ptr[stride * 3 + 1] + + src_ptr[stride * 3 + 2] + src_ptr[stride * 3 + 3] + + 8) >> 4; + dst[1] = (src_ptr[4] + src_ptr[5] + src_ptr[6] + src_ptr[7] + + src_ptr[stride + 4] + src_ptr[stride + 5] + + src_ptr[stride + 6] + src_ptr[stride + 7] + + src_ptr[stride * 2 + 4] + src_ptr[stride * 2 + 5] + + src_ptr[stride * 2 + 6] + src_ptr[stride * 2 + 7] + + src_ptr[stride * 3 + 4] + src_ptr[stride * 3 + 5] + + src_ptr[stride * 3 + 6] + src_ptr[stride * 3 + 7] + + 8) >> 4; + dst += 2; + src_ptr += 8; + } + if (dst_width & 1) { + dst[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[3] + + src_ptr[stride + 0] + src_ptr[stride + 1] + + src_ptr[stride + 2] + src_ptr[stride + 3] + + src_ptr[stride * 2 + 0] + src_ptr[stride * 2 + 1] + + src_ptr[stride * 2 + 2] + src_ptr[stride * 2 + 3] + + src_ptr[stride * 3 + 0] + src_ptr[stride * 3 + 1] + + src_ptr[stride * 3 + 2] + src_ptr[stride * 3 + 3] + + 8) >> 4; + } +} + +void ScaleRowDown34_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + int x; + assert((dst_width % 3 == 0) && (dst_width > 0)); + for (x = 0; x < dst_width; x += 3) { + dst[0] = src_ptr[0]; + dst[1] = src_ptr[1]; + dst[2] = src_ptr[3]; + dst += 3; + src_ptr += 4; + } +} + +void ScaleRowDown34_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width) { + int x; + assert((dst_width % 3 == 0) && (dst_width > 0)); + for (x = 0; x < dst_width; x += 3) { + dst[0] = src_ptr[0]; + dst[1] = src_ptr[1]; + dst[2] = src_ptr[3]; + dst += 3; + src_ptr += 4; + } +} + +// Filter rows 0 and 1 together, 3 : 1 +void ScaleRowDown34_0_Box_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* d, int dst_width) { + const uint8* s = src_ptr; + const uint8* t = src_ptr + src_stride; + int x; + assert((dst_width % 3 == 0) && (dst_width > 0)); + for (x = 0; x < dst_width; x += 3) { + uint8 a0 = (s[0] * 3 + s[1] * 1 + 2) >> 2; + uint8 a1 = (s[1] * 1 + s[2] * 1 + 1) >> 1; + uint8 a2 = (s[2] * 1 + s[3] * 3 + 2) >> 2; + uint8 b0 = (t[0] * 3 + t[1] * 1 + 2) >> 2; + uint8 b1 = (t[1] * 1 + t[2] * 1 + 1) >> 1; + uint8 b2 = (t[2] * 1 + t[3] * 3 + 2) >> 2; + d[0] = (a0 * 3 + b0 + 2) >> 2; + d[1] = (a1 * 3 + b1 + 2) >> 2; + d[2] = (a2 * 3 + b2 + 2) >> 2; + d += 3; + s += 4; + t += 4; + } +} + +void ScaleRowDown34_0_Box_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* d, int dst_width) { + const uint16* s = src_ptr; + const uint16* t = src_ptr + src_stride; + int x; + assert((dst_width % 3 == 0) && (dst_width > 0)); + for (x = 0; x < dst_width; x += 3) { + uint16 a0 = (s[0] * 3 + s[1] * 1 + 2) >> 2; + uint16 a1 = (s[1] * 1 + s[2] * 1 + 1) >> 1; + uint16 a2 = (s[2] * 1 + s[3] * 3 + 2) >> 2; + uint16 b0 = (t[0] * 3 + t[1] * 1 + 2) >> 2; + uint16 b1 = (t[1] * 1 + t[2] * 1 + 1) >> 1; + uint16 b2 = (t[2] * 1 + t[3] * 3 + 2) >> 2; + d[0] = (a0 * 3 + b0 + 2) >> 2; + d[1] = (a1 * 3 + b1 + 2) >> 2; + d[2] = (a2 * 3 + b2 + 2) >> 2; + d += 3; + s += 4; + t += 4; + } +} + +// Filter rows 1 and 2 together, 1 : 1 +void ScaleRowDown34_1_Box_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* d, int dst_width) { + const uint8* s = src_ptr; + const uint8* t = src_ptr + src_stride; + int x; + assert((dst_width % 3 == 0) && (dst_width > 0)); + for (x = 0; x < dst_width; x += 3) { + uint8 a0 = (s[0] * 3 + s[1] * 1 + 2) >> 2; + uint8 a1 = (s[1] * 1 + s[2] * 1 + 1) >> 1; + uint8 a2 = (s[2] * 1 + s[3] * 3 + 2) >> 2; + uint8 b0 = (t[0] * 3 + t[1] * 1 + 2) >> 2; + uint8 b1 = (t[1] * 1 + t[2] * 1 + 1) >> 1; + uint8 b2 = (t[2] * 1 + t[3] * 3 + 2) >> 2; + d[0] = (a0 + b0 + 1) >> 1; + d[1] = (a1 + b1 + 1) >> 1; + d[2] = (a2 + b2 + 1) >> 1; + d += 3; + s += 4; + t += 4; + } +} + +void ScaleRowDown34_1_Box_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* d, int dst_width) { + const uint16* s = src_ptr; + const uint16* t = src_ptr + src_stride; + int x; + assert((dst_width % 3 == 0) && (dst_width > 0)); + for (x = 0; x < dst_width; x += 3) { + uint16 a0 = (s[0] * 3 + s[1] * 1 + 2) >> 2; + uint16 a1 = (s[1] * 1 + s[2] * 1 + 1) >> 1; + uint16 a2 = (s[2] * 1 + s[3] * 3 + 2) >> 2; + uint16 b0 = (t[0] * 3 + t[1] * 1 + 2) >> 2; + uint16 b1 = (t[1] * 1 + t[2] * 1 + 1) >> 1; + uint16 b2 = (t[2] * 1 + t[3] * 3 + 2) >> 2; + d[0] = (a0 + b0 + 1) >> 1; + d[1] = (a1 + b1 + 1) >> 1; + d[2] = (a2 + b2 + 1) >> 1; + d += 3; + s += 4; + t += 4; + } +} + +// Scales a single row of pixels using point sampling. +void ScaleCols_C(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx) { + int j; + for (j = 0; j < dst_width - 1; j += 2) { + dst_ptr[0] = src_ptr[x >> 16]; + x += dx; + dst_ptr[1] = src_ptr[x >> 16]; + x += dx; + dst_ptr += 2; + } + if (dst_width & 1) { + dst_ptr[0] = src_ptr[x >> 16]; + } +} + +void ScaleCols_16_C(uint16* dst_ptr, const uint16* src_ptr, + int dst_width, int x, int dx) { + int j; + for (j = 0; j < dst_width - 1; j += 2) { + dst_ptr[0] = src_ptr[x >> 16]; + x += dx; + dst_ptr[1] = src_ptr[x >> 16]; + x += dx; + dst_ptr += 2; + } + if (dst_width & 1) { + dst_ptr[0] = src_ptr[x >> 16]; + } +} + +// Scales a single row of pixels up by 2x using point sampling. +void ScaleColsUp2_C(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx) { + int j; + for (j = 0; j < dst_width - 1; j += 2) { + dst_ptr[1] = dst_ptr[0] = src_ptr[0]; + src_ptr += 1; + dst_ptr += 2; + } + if (dst_width & 1) { + dst_ptr[0] = src_ptr[0]; + } +} + +void ScaleColsUp2_16_C(uint16* dst_ptr, const uint16* src_ptr, + int dst_width, int x, int dx) { + int j; + for (j = 0; j < dst_width - 1; j += 2) { + dst_ptr[1] = dst_ptr[0] = src_ptr[0]; + src_ptr += 1; + dst_ptr += 2; + } + if (dst_width & 1) { + dst_ptr[0] = src_ptr[0]; + } +} + +// (1-f)a + fb can be replaced with a + f(b-a) +#define BLENDER(a, b, f) (uint8)((int)(a) + \ + ((int)(f) * ((int)(b) - (int)(a)) >> 16)) + +void ScaleFilterCols_C(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx) { + int j; + for (j = 0; j < dst_width - 1; j += 2) { + int xi = x >> 16; + int a = src_ptr[xi]; + int b = src_ptr[xi + 1]; + dst_ptr[0] = BLENDER(a, b, x & 0xffff); + x += dx; + xi = x >> 16; + a = src_ptr[xi]; + b = src_ptr[xi + 1]; + dst_ptr[1] = BLENDER(a, b, x & 0xffff); + x += dx; + dst_ptr += 2; + } + if (dst_width & 1) { + int xi = x >> 16; + int a = src_ptr[xi]; + int b = src_ptr[xi + 1]; + dst_ptr[0] = BLENDER(a, b, x & 0xffff); + } +} + +void ScaleFilterCols64_C(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x32, int dx) { + int64 x = (int64)(x32); + int j; + for (j = 0; j < dst_width - 1; j += 2) { + int64 xi = x >> 16; + int a = src_ptr[xi]; + int b = src_ptr[xi + 1]; + dst_ptr[0] = BLENDER(a, b, x & 0xffff); + x += dx; + xi = x >> 16; + a = src_ptr[xi]; + b = src_ptr[xi + 1]; + dst_ptr[1] = BLENDER(a, b, x & 0xffff); + x += dx; + dst_ptr += 2; + } + if (dst_width & 1) { + int64 xi = x >> 16; + int a = src_ptr[xi]; + int b = src_ptr[xi + 1]; + dst_ptr[0] = BLENDER(a, b, x & 0xffff); + } +} +#undef BLENDER + +#define BLENDER(a, b, f) (uint16)((int)(a) + \ + ((int)(f) * ((int)(b) - (int)(a)) >> 16)) + +void ScaleFilterCols_16_C(uint16* dst_ptr, const uint16* src_ptr, + int dst_width, int x, int dx) { + int j; + for (j = 0; j < dst_width - 1; j += 2) { + int xi = x >> 16; + int a = src_ptr[xi]; + int b = src_ptr[xi + 1]; + dst_ptr[0] = BLENDER(a, b, x & 0xffff); + x += dx; + xi = x >> 16; + a = src_ptr[xi]; + b = src_ptr[xi + 1]; + dst_ptr[1] = BLENDER(a, b, x & 0xffff); + x += dx; + dst_ptr += 2; + } + if (dst_width & 1) { + int xi = x >> 16; + int a = src_ptr[xi]; + int b = src_ptr[xi + 1]; + dst_ptr[0] = BLENDER(a, b, x & 0xffff); + } +} + +void ScaleFilterCols64_16_C(uint16* dst_ptr, const uint16* src_ptr, + int dst_width, int x32, int dx) { + int64 x = (int64)(x32); + int j; + for (j = 0; j < dst_width - 1; j += 2) { + int64 xi = x >> 16; + int a = src_ptr[xi]; + int b = src_ptr[xi + 1]; + dst_ptr[0] = BLENDER(a, b, x & 0xffff); + x += dx; + xi = x >> 16; + a = src_ptr[xi]; + b = src_ptr[xi + 1]; + dst_ptr[1] = BLENDER(a, b, x & 0xffff); + x += dx; + dst_ptr += 2; + } + if (dst_width & 1) { + int64 xi = x >> 16; + int a = src_ptr[xi]; + int b = src_ptr[xi + 1]; + dst_ptr[0] = BLENDER(a, b, x & 0xffff); + } +} +#undef BLENDER + +void ScaleRowDown38_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + int x; + assert(dst_width % 3 == 0); + for (x = 0; x < dst_width; x += 3) { + dst[0] = src_ptr[0]; + dst[1] = src_ptr[3]; + dst[2] = src_ptr[6]; + dst += 3; + src_ptr += 8; + } +} + +void ScaleRowDown38_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst, int dst_width) { + int x; + assert(dst_width % 3 == 0); + for (x = 0; x < dst_width; x += 3) { + dst[0] = src_ptr[0]; + dst[1] = src_ptr[3]; + dst[2] = src_ptr[6]; + dst += 3; + src_ptr += 8; + } +} + +// 8x3 -> 3x1 +void ScaleRowDown38_3_Box_C(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + intptr_t stride = src_stride; + int i; + assert((dst_width % 3 == 0) && (dst_width > 0)); + for (i = 0; i < dst_width; i += 3) { + dst_ptr[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + + src_ptr[stride + 0] + src_ptr[stride + 1] + + src_ptr[stride + 2] + src_ptr[stride * 2 + 0] + + src_ptr[stride * 2 + 1] + src_ptr[stride * 2 + 2]) * + (65536 / 9) >> 16; + dst_ptr[1] = (src_ptr[3] + src_ptr[4] + src_ptr[5] + + src_ptr[stride + 3] + src_ptr[stride + 4] + + src_ptr[stride + 5] + src_ptr[stride * 2 + 3] + + src_ptr[stride * 2 + 4] + src_ptr[stride * 2 + 5]) * + (65536 / 9) >> 16; + dst_ptr[2] = (src_ptr[6] + src_ptr[7] + + src_ptr[stride + 6] + src_ptr[stride + 7] + + src_ptr[stride * 2 + 6] + src_ptr[stride * 2 + 7]) * + (65536 / 6) >> 16; + src_ptr += 8; + dst_ptr += 3; + } +} + +void ScaleRowDown38_3_Box_16_C(const uint16* src_ptr, + ptrdiff_t src_stride, + uint16* dst_ptr, int dst_width) { + intptr_t stride = src_stride; + int i; + assert((dst_width % 3 == 0) && (dst_width > 0)); + for (i = 0; i < dst_width; i += 3) { + dst_ptr[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + + src_ptr[stride + 0] + src_ptr[stride + 1] + + src_ptr[stride + 2] + src_ptr[stride * 2 + 0] + + src_ptr[stride * 2 + 1] + src_ptr[stride * 2 + 2]) * + (65536 / 9) >> 16; + dst_ptr[1] = (src_ptr[3] + src_ptr[4] + src_ptr[5] + + src_ptr[stride + 3] + src_ptr[stride + 4] + + src_ptr[stride + 5] + src_ptr[stride * 2 + 3] + + src_ptr[stride * 2 + 4] + src_ptr[stride * 2 + 5]) * + (65536 / 9) >> 16; + dst_ptr[2] = (src_ptr[6] + src_ptr[7] + + src_ptr[stride + 6] + src_ptr[stride + 7] + + src_ptr[stride * 2 + 6] + src_ptr[stride * 2 + 7]) * + (65536 / 6) >> 16; + src_ptr += 8; + dst_ptr += 3; + } +} + +// 8x2 -> 3x1 +void ScaleRowDown38_2_Box_C(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + intptr_t stride = src_stride; + int i; + assert((dst_width % 3 == 0) && (dst_width > 0)); + for (i = 0; i < dst_width; i += 3) { + dst_ptr[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + + src_ptr[stride + 0] + src_ptr[stride + 1] + + src_ptr[stride + 2]) * (65536 / 6) >> 16; + dst_ptr[1] = (src_ptr[3] + src_ptr[4] + src_ptr[5] + + src_ptr[stride + 3] + src_ptr[stride + 4] + + src_ptr[stride + 5]) * (65536 / 6) >> 16; + dst_ptr[2] = (src_ptr[6] + src_ptr[7] + + src_ptr[stride + 6] + src_ptr[stride + 7]) * + (65536 / 4) >> 16; + src_ptr += 8; + dst_ptr += 3; + } +} + +void ScaleRowDown38_2_Box_16_C(const uint16* src_ptr, ptrdiff_t src_stride, + uint16* dst_ptr, int dst_width) { + intptr_t stride = src_stride; + int i; + assert((dst_width % 3 == 0) && (dst_width > 0)); + for (i = 0; i < dst_width; i += 3) { + dst_ptr[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + + src_ptr[stride + 0] + src_ptr[stride + 1] + + src_ptr[stride + 2]) * (65536 / 6) >> 16; + dst_ptr[1] = (src_ptr[3] + src_ptr[4] + src_ptr[5] + + src_ptr[stride + 3] + src_ptr[stride + 4] + + src_ptr[stride + 5]) * (65536 / 6) >> 16; + dst_ptr[2] = (src_ptr[6] + src_ptr[7] + + src_ptr[stride + 6] + src_ptr[stride + 7]) * + (65536 / 4) >> 16; + src_ptr += 8; + dst_ptr += 3; + } +} + +void ScaleAddRow_C(const uint8* src_ptr, uint16* dst_ptr, int src_width) { + int x; + assert(src_width > 0); + for (x = 0; x < src_width - 1; x += 2) { + dst_ptr[0] += src_ptr[0]; + dst_ptr[1] += src_ptr[1]; + src_ptr += 2; + dst_ptr += 2; + } + if (src_width & 1) { + dst_ptr[0] += src_ptr[0]; + } +} + +void ScaleAddRow_16_C(const uint16* src_ptr, uint32* dst_ptr, int src_width) { + int x; + assert(src_width > 0); + for (x = 0; x < src_width - 1; x += 2) { + dst_ptr[0] += src_ptr[0]; + dst_ptr[1] += src_ptr[1]; + src_ptr += 2; + dst_ptr += 2; + } + if (src_width & 1) { + dst_ptr[0] += src_ptr[0]; + } +} + +void ScaleARGBRowDown2_C(const uint8* src_argb, + ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) { + const uint32* src = (const uint32*)(src_argb); + uint32* dst = (uint32*)(dst_argb); + + int x; + for (x = 0; x < dst_width - 1; x += 2) { + dst[0] = src[1]; + dst[1] = src[3]; + src += 4; + dst += 2; + } + if (dst_width & 1) { + dst[0] = src[1]; + } +} + +void ScaleARGBRowDown2Linear_C(const uint8* src_argb, + ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) { + int x; + for (x = 0; x < dst_width; ++x) { + dst_argb[0] = (src_argb[0] + src_argb[4] + 1) >> 1; + dst_argb[1] = (src_argb[1] + src_argb[5] + 1) >> 1; + dst_argb[2] = (src_argb[2] + src_argb[6] + 1) >> 1; + dst_argb[3] = (src_argb[3] + src_argb[7] + 1) >> 1; + src_argb += 8; + dst_argb += 4; + } +} + +void ScaleARGBRowDown2Box_C(const uint8* src_argb, ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) { + int x; + for (x = 0; x < dst_width; ++x) { + dst_argb[0] = (src_argb[0] + src_argb[4] + + src_argb[src_stride] + src_argb[src_stride + 4] + 2) >> 2; + dst_argb[1] = (src_argb[1] + src_argb[5] + + src_argb[src_stride + 1] + src_argb[src_stride + 5] + 2) >> 2; + dst_argb[2] = (src_argb[2] + src_argb[6] + + src_argb[src_stride + 2] + src_argb[src_stride + 6] + 2) >> 2; + dst_argb[3] = (src_argb[3] + src_argb[7] + + src_argb[src_stride + 3] + src_argb[src_stride + 7] + 2) >> 2; + src_argb += 8; + dst_argb += 4; + } +} + +void ScaleARGBRowDownEven_C(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width) { + const uint32* src = (const uint32*)(src_argb); + uint32* dst = (uint32*)(dst_argb); + + int x; + for (x = 0; x < dst_width - 1; x += 2) { + dst[0] = src[0]; + dst[1] = src[src_stepx]; + src += src_stepx * 2; + dst += 2; + } + if (dst_width & 1) { + dst[0] = src[0]; + } +} + +void ScaleARGBRowDownEvenBox_C(const uint8* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width) { + int x; + for (x = 0; x < dst_width; ++x) { + dst_argb[0] = (src_argb[0] + src_argb[4] + + src_argb[src_stride] + src_argb[src_stride + 4] + 2) >> 2; + dst_argb[1] = (src_argb[1] + src_argb[5] + + src_argb[src_stride + 1] + src_argb[src_stride + 5] + 2) >> 2; + dst_argb[2] = (src_argb[2] + src_argb[6] + + src_argb[src_stride + 2] + src_argb[src_stride + 6] + 2) >> 2; + dst_argb[3] = (src_argb[3] + src_argb[7] + + src_argb[src_stride + 3] + src_argb[src_stride + 7] + 2) >> 2; + src_argb += src_stepx * 4; + dst_argb += 4; + } +} + +// Scales a single row of pixels using point sampling. +void ScaleARGBCols_C(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + const uint32* src = (const uint32*)(src_argb); + uint32* dst = (uint32*)(dst_argb); + int j; + for (j = 0; j < dst_width - 1; j += 2) { + dst[0] = src[x >> 16]; + x += dx; + dst[1] = src[x >> 16]; + x += dx; + dst += 2; + } + if (dst_width & 1) { + dst[0] = src[x >> 16]; + } +} + +void ScaleARGBCols64_C(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x32, int dx) { + int64 x = (int64)(x32); + const uint32* src = (const uint32*)(src_argb); + uint32* dst = (uint32*)(dst_argb); + int j; + for (j = 0; j < dst_width - 1; j += 2) { + dst[0] = src[x >> 16]; + x += dx; + dst[1] = src[x >> 16]; + x += dx; + dst += 2; + } + if (dst_width & 1) { + dst[0] = src[x >> 16]; + } +} + +// Scales a single row of pixels up by 2x using point sampling. +void ScaleARGBColsUp2_C(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + const uint32* src = (const uint32*)(src_argb); + uint32* dst = (uint32*)(dst_argb); + int j; + for (j = 0; j < dst_width - 1; j += 2) { + dst[1] = dst[0] = src[0]; + src += 1; + dst += 2; + } + if (dst_width & 1) { + dst[0] = src[0]; + } +} + +// Mimics SSSE3 blender +#define BLENDER1(a, b, f) ((a) * (0x7f ^ f) + (b) * f) >> 7 +#define BLENDERC(a, b, f, s) (uint32)( \ + BLENDER1(((a) >> s) & 255, ((b) >> s) & 255, f) << s) +#define BLENDER(a, b, f) \ + BLENDERC(a, b, f, 24) | BLENDERC(a, b, f, 16) | \ + BLENDERC(a, b, f, 8) | BLENDERC(a, b, f, 0) + +void ScaleARGBFilterCols_C(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + const uint32* src = (const uint32*)(src_argb); + uint32* dst = (uint32*)(dst_argb); + int j; + for (j = 0; j < dst_width - 1; j += 2) { + int xi = x >> 16; + int xf = (x >> 9) & 0x7f; + uint32 a = src[xi]; + uint32 b = src[xi + 1]; + dst[0] = BLENDER(a, b, xf); + x += dx; + xi = x >> 16; + xf = (x >> 9) & 0x7f; + a = src[xi]; + b = src[xi + 1]; + dst[1] = BLENDER(a, b, xf); + x += dx; + dst += 2; + } + if (dst_width & 1) { + int xi = x >> 16; + int xf = (x >> 9) & 0x7f; + uint32 a = src[xi]; + uint32 b = src[xi + 1]; + dst[0] = BLENDER(a, b, xf); + } +} + +void ScaleARGBFilterCols64_C(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x32, int dx) { + int64 x = (int64)(x32); + const uint32* src = (const uint32*)(src_argb); + uint32* dst = (uint32*)(dst_argb); + int j; + for (j = 0; j < dst_width - 1; j += 2) { + int64 xi = x >> 16; + int xf = (x >> 9) & 0x7f; + uint32 a = src[xi]; + uint32 b = src[xi + 1]; + dst[0] = BLENDER(a, b, xf); + x += dx; + xi = x >> 16; + xf = (x >> 9) & 0x7f; + a = src[xi]; + b = src[xi + 1]; + dst[1] = BLENDER(a, b, xf); + x += dx; + dst += 2; + } + if (dst_width & 1) { + int64 xi = x >> 16; + int xf = (x >> 9) & 0x7f; + uint32 a = src[xi]; + uint32 b = src[xi + 1]; + dst[0] = BLENDER(a, b, xf); + } +} +#undef BLENDER1 +#undef BLENDERC +#undef BLENDER + +// Scale plane vertically with bilinear interpolation. +void ScalePlaneVertical(int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint8* src_argb, uint8* dst_argb, + int x, int y, int dy, + int bpp, enum FilterMode filtering) { + // TODO(fbarchard): Allow higher bpp. + int dst_width_bytes = dst_width * bpp; + void (*InterpolateRow)(uint8* dst_argb, const uint8* src_argb, + ptrdiff_t src_stride, int dst_width, int source_y_fraction) = + InterpolateRow_C; + const int max_y = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0; + int j; + assert(bpp >= 1 && bpp <= 4); + assert(src_height != 0); + assert(dst_width > 0); + assert(dst_height > 0); + src_argb += (x >> 16) * bpp; +#if defined(HAS_INTERPOLATEROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + InterpolateRow = InterpolateRow_Any_SSE2; + if (IS_ALIGNED(dst_width_bytes, 16)) { + InterpolateRow = InterpolateRow_SSE2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + InterpolateRow = InterpolateRow_Any_SSSE3; + if (IS_ALIGNED(dst_width_bytes, 16)) { + InterpolateRow = InterpolateRow_SSSE3; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(dst_width_bytes, 32)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_NEON; + if (IS_ALIGNED(dst_width_bytes, 16)) { + InterpolateRow = InterpolateRow_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && + IS_ALIGNED(src_argb, 4) && IS_ALIGNED(src_stride, 4) && + IS_ALIGNED(dst_argb, 4) && IS_ALIGNED(dst_stride, 4)) { + InterpolateRow = InterpolateRow_Any_MIPS_DSPR2; + if (IS_ALIGNED(dst_width_bytes, 4)) { + InterpolateRow = InterpolateRow_MIPS_DSPR2; + } + } +#endif + for (j = 0; j < dst_height; ++j) { + int yi; + int yf; + if (y > max_y) { + y = max_y; + } + yi = y >> 16; + yf = filtering ? ((y >> 8) & 255) : 0; + InterpolateRow(dst_argb, src_argb + yi * src_stride, + src_stride, dst_width_bytes, yf); + dst_argb += dst_stride; + y += dy; + } +} +void ScalePlaneVertical_16(int src_height, + int dst_width, int dst_height, + int src_stride, int dst_stride, + const uint16* src_argb, uint16* dst_argb, + int x, int y, int dy, + int wpp, enum FilterMode filtering) { + // TODO(fbarchard): Allow higher wpp. + int dst_width_words = dst_width * wpp; + void (*InterpolateRow)(uint16* dst_argb, const uint16* src_argb, + ptrdiff_t src_stride, int dst_width, int source_y_fraction) = + InterpolateRow_16_C; + const int max_y = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0; + int j; + assert(wpp >= 1 && wpp <= 2); + assert(src_height != 0); + assert(dst_width > 0); + assert(dst_height > 0); + src_argb += (x >> 16) * wpp; +#if defined(HAS_INTERPOLATEROW_16_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + InterpolateRow = InterpolateRow_Any_16_SSE2; + if (IS_ALIGNED(dst_width_bytes, 16)) { + InterpolateRow = InterpolateRow_16_SSE2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + InterpolateRow = InterpolateRow_Any_16_SSSE3; + if (IS_ALIGNED(dst_width_bytes, 16)) { + InterpolateRow = InterpolateRow_16_SSSE3; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_16_AVX2; + if (IS_ALIGNED(dst_width_bytes, 32)) { + InterpolateRow = InterpolateRow_16_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_16_NEON; + if (IS_ALIGNED(dst_width_bytes, 16)) { + InterpolateRow = InterpolateRow_16_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_MIPS_DSPR2) + if (TestCpuFlag(kCpuHasMIPS_DSPR2) && + IS_ALIGNED(src_argb, 4) && IS_ALIGNED(src_stride, 4) && + IS_ALIGNED(dst_argb, 4) && IS_ALIGNED(dst_stride, 4)) { + InterpolateRow = InterpolateRow_Any_16_MIPS_DSPR2; + if (IS_ALIGNED(dst_width_bytes, 4)) { + InterpolateRow = InterpolateRow_16_MIPS_DSPR2; + } + } +#endif + for (j = 0; j < dst_height; ++j) { + int yi; + int yf; + if (y > max_y) { + y = max_y; + } + yi = y >> 16; + yf = filtering ? ((y >> 8) & 255) : 0; + InterpolateRow(dst_argb, src_argb + yi * src_stride, + src_stride, dst_width_words, yf); + dst_argb += dst_stride; + y += dy; + } +} + +// Simplify the filtering based on scale factors. +enum FilterMode ScaleFilterReduce(int src_width, int src_height, + int dst_width, int dst_height, + enum FilterMode filtering) { + if (src_width < 0) { + src_width = -src_width; + } + if (src_height < 0) { + src_height = -src_height; + } + if (filtering == kFilterBox) { + // If scaling both axis to 0.5 or larger, switch from Box to Bilinear. + if (dst_width * 2 >= src_width && dst_height * 2 >= src_height) { + filtering = kFilterBilinear; + } + } + if (filtering == kFilterBilinear) { + if (src_height == 1) { + filtering = kFilterLinear; + } + // TODO(fbarchard): Detect any odd scale factor and reduce to Linear. + if (dst_height == src_height || dst_height * 3 == src_height) { + filtering = kFilterLinear; + } + // TODO(fbarchard): Remove 1 pixel wide filter restriction, which is to + // avoid reading 2 pixels horizontally that causes memory exception. + if (src_width == 1) { + filtering = kFilterNone; + } + } + if (filtering == kFilterLinear) { + if (src_width == 1) { + filtering = kFilterNone; + } + // TODO(fbarchard): Detect any odd scale factor and reduce to None. + if (dst_width == src_width || dst_width * 3 == src_width) { + filtering = kFilterNone; + } + } + return filtering; +} + +// Divide num by div and return as 16.16 fixed point result. +int FixedDiv_C(int num, int div) { + return (int)(((int64)(num) << 16) / div); +} + +// Divide num by div and return as 16.16 fixed point result. +int FixedDiv1_C(int num, int div) { + return (int)((((int64)(num) << 16) - 0x00010001) / + (div - 1)); +} + +#define CENTERSTART(dx, s) (dx < 0) ? -((-dx >> 1) + s) : ((dx >> 1) + s) + +// Compute slope values for stepping. +void ScaleSlope(int src_width, int src_height, + int dst_width, int dst_height, + enum FilterMode filtering, + int* x, int* y, int* dx, int* dy) { + assert(x != NULL); + assert(y != NULL); + assert(dx != NULL); + assert(dy != NULL); + assert(src_width != 0); + assert(src_height != 0); + assert(dst_width > 0); + assert(dst_height > 0); + // Check for 1 pixel and avoid FixedDiv overflow. + if (dst_width == 1 && src_width >= 32768) { + dst_width = src_width; + } + if (dst_height == 1 && src_height >= 32768) { + dst_height = src_height; + } + if (filtering == kFilterBox) { + // Scale step for point sampling duplicates all pixels equally. + *dx = FixedDiv(Abs(src_width), dst_width); + *dy = FixedDiv(src_height, dst_height); + *x = 0; + *y = 0; + } else if (filtering == kFilterBilinear) { + // Scale step for bilinear sampling renders last pixel once for upsample. + if (dst_width <= Abs(src_width)) { + *dx = FixedDiv(Abs(src_width), dst_width); + *x = CENTERSTART(*dx, -32768); // Subtract 0.5 (32768) to center filter. + } else if (dst_width > 1) { + *dx = FixedDiv1(Abs(src_width), dst_width); + *x = 0; + } + if (dst_height <= src_height) { + *dy = FixedDiv(src_height, dst_height); + *y = CENTERSTART(*dy, -32768); // Subtract 0.5 (32768) to center filter. + } else if (dst_height > 1) { + *dy = FixedDiv1(src_height, dst_height); + *y = 0; + } + } else if (filtering == kFilterLinear) { + // Scale step for bilinear sampling renders last pixel once for upsample. + if (dst_width <= Abs(src_width)) { + *dx = FixedDiv(Abs(src_width), dst_width); + *x = CENTERSTART(*dx, -32768); // Subtract 0.5 (32768) to center filter. + } else if (dst_width > 1) { + *dx = FixedDiv1(Abs(src_width), dst_width); + *x = 0; + } + *dy = FixedDiv(src_height, dst_height); + *y = *dy >> 1; + } else { + // Scale step for point sampling duplicates all pixels equally. + *dx = FixedDiv(Abs(src_width), dst_width); + *dy = FixedDiv(src_height, dst_height); + *x = CENTERSTART(*dx, 0); + *y = CENTERSTART(*dy, 0); + } + // Negative src_width means horizontally mirror. + if (src_width < 0) { + *x += (dst_width - 1) * *dx; + *dx = -*dx; + // src_width = -src_width; // Caller must do this. + } +} +#undef CENTERSTART + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/scale_gcc.cc b/third_party/aom/third_party/libyuv/source/scale_gcc.cc new file mode 100644 index 0000000000..8a6ac54592 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/scale_gcc.cc @@ -0,0 +1,1089 @@ +/* + * Copyright 2013 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for GCC x86 and x64. +#if !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__)) + +// Offsets for source bytes 0 to 9 +static uvec8 kShuf0 = + { 0, 1, 3, 4, 5, 7, 8, 9, 128, 128, 128, 128, 128, 128, 128, 128 }; + +// Offsets for source bytes 11 to 20 with 8 subtracted = 3 to 12. +static uvec8 kShuf1 = + { 3, 4, 5, 7, 8, 9, 11, 12, 128, 128, 128, 128, 128, 128, 128, 128 }; + +// Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31. +static uvec8 kShuf2 = + { 5, 7, 8, 9, 11, 12, 13, 15, 128, 128, 128, 128, 128, 128, 128, 128 }; + +// Offsets for source bytes 0 to 10 +static uvec8 kShuf01 = + { 0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10 }; + +// Offsets for source bytes 10 to 21 with 8 subtracted = 3 to 13. +static uvec8 kShuf11 = + { 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10, 10, 11, 12, 13 }; + +// Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31. +static uvec8 kShuf21 = + { 5, 6, 6, 7, 8, 9, 9, 10, 10, 11, 12, 13, 13, 14, 14, 15 }; + +// Coefficients for source bytes 0 to 10 +static uvec8 kMadd01 = + { 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2 }; + +// Coefficients for source bytes 10 to 21 +static uvec8 kMadd11 = + { 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1 }; + +// Coefficients for source bytes 21 to 31 +static uvec8 kMadd21 = + { 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3 }; + +// Coefficients for source bytes 21 to 31 +static vec16 kRound34 = + { 2, 2, 2, 2, 2, 2, 2, 2 }; + +static uvec8 kShuf38a = + { 0, 3, 6, 8, 11, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }; + +static uvec8 kShuf38b = + { 128, 128, 128, 128, 128, 128, 0, 3, 6, 8, 11, 14, 128, 128, 128, 128 }; + +// Arrange words 0,3,6 into 0,1,2 +static uvec8 kShufAc = + { 0, 1, 6, 7, 12, 13, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }; + +// Arrange words 0,3,6 into 3,4,5 +static uvec8 kShufAc3 = + { 128, 128, 128, 128, 128, 128, 0, 1, 6, 7, 12, 13, 128, 128, 128, 128 }; + +// Scaling values for boxes of 3x3 and 2x3 +static uvec16 kScaleAc33 = + { 65536 / 9, 65536 / 9, 65536 / 6, 65536 / 9, 65536 / 9, 65536 / 6, 0, 0 }; + +// Arrange first value for pixels 0,1,2,3,4,5 +static uvec8 kShufAb0 = + { 0, 128, 3, 128, 6, 128, 8, 128, 11, 128, 14, 128, 128, 128, 128, 128 }; + +// Arrange second value for pixels 0,1,2,3,4,5 +static uvec8 kShufAb1 = + { 1, 128, 4, 128, 7, 128, 9, 128, 12, 128, 15, 128, 128, 128, 128, 128 }; + +// Arrange third value for pixels 0,1,2,3,4,5 +static uvec8 kShufAb2 = + { 2, 128, 5, 128, 128, 128, 10, 128, 13, 128, 128, 128, 128, 128, 128, 128 }; + +// Scaling values for boxes of 3x2 and 2x2 +static uvec16 kScaleAb2 = + { 65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3, 65536 / 3, 65536 / 2, 0, 0 }; + +// GCC versions of row functions are verbatim conversions from Visual C. +// Generated using gcc disassembly on Visual C object file: +// objdump -D yuvscaler.obj >yuvscaler.txt + +void ScaleRowDown2_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "psrlw $0x8,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + :: "memory", "cc", "xmm0", "xmm1" + ); +} + +void ScaleRowDown2Linear_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" + "psrlw $0x8,%%xmm5 \n" + + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10, 0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "movdqa %%xmm0,%%xmm2 \n" + "psrlw $0x8,%%xmm0 \n" + "movdqa %%xmm1,%%xmm3 \n" + "psrlw $0x8,%%xmm1 \n" + "pand %%xmm5,%%xmm2 \n" + "pand %%xmm5,%%xmm3 \n" + "pavgw %%xmm2,%%xmm0 \n" + "pavgw %%xmm3,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + :: "memory", "cc", "xmm0", "xmm1", "xmm5" + ); +} + +void ScaleRowDown2Box_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" + "psrlw $0x8,%%xmm5 \n" + + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + MEMOPREG(movdqu,0x00,0,3,1,xmm2) // movdqu (%0,%3,1),%%xmm2 + MEMOPREG(movdqu,0x10,0,3,1,xmm3) // movdqu 0x10(%0,%3,1),%%xmm3 + "lea " MEMLEA(0x20,0) ",%0 \n" + "pavgb %%xmm2,%%xmm0 \n" + "pavgb %%xmm3,%%xmm1 \n" + "movdqa %%xmm0,%%xmm2 \n" + "psrlw $0x8,%%xmm0 \n" + "movdqa %%xmm1,%%xmm3 \n" + "psrlw $0x8,%%xmm1 \n" + "pand %%xmm5,%%xmm2 \n" + "pand %%xmm5,%%xmm3 \n" + "pavgw %%xmm2,%%xmm0 \n" + "pavgw %%xmm3,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : "r"((intptr_t)(src_stride)) // %3 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" + ); +} + +void ScaleRowDown4_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "pcmpeqb %%xmm5,%%xmm5 \n" + "psrld $0x18,%%xmm5 \n" + "pslld $0x10,%%xmm5 \n" + + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "pand %%xmm5,%%xmm0 \n" + "pand %%xmm5,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "psrlw $0x8,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "movq %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + :: "memory", "cc", "xmm0", "xmm1", "xmm5" + ); +} + +void ScaleRowDown4Box_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + intptr_t stridex3 = 0; + asm volatile ( + "pcmpeqb %%xmm7,%%xmm7 \n" + "psrlw $0x8,%%xmm7 \n" + "lea " MEMLEA4(0x00,4,4,2) ",%3 \n" + + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + MEMOPREG(movdqu,0x00,0,4,1,xmm2) // movdqu (%0,%4,1),%%xmm2 + MEMOPREG(movdqu,0x10,0,4,1,xmm3) // movdqu 0x10(%0,%4,1),%%xmm3 + "pavgb %%xmm2,%%xmm0 \n" + "pavgb %%xmm3,%%xmm1 \n" + MEMOPREG(movdqu,0x00,0,4,2,xmm2) // movdqu (%0,%4,2),%%xmm2 + MEMOPREG(movdqu,0x10,0,4,2,xmm3) // movdqu 0x10(%0,%4,2),%%xmm3 + MEMOPREG(movdqu,0x00,0,3,1,xmm4) // movdqu (%0,%3,1),%%xmm4 + MEMOPREG(movdqu,0x10,0,3,1,xmm5) // movdqu 0x10(%0,%3,1),%%xmm5 + "lea " MEMLEA(0x20,0) ",%0 \n" + "pavgb %%xmm4,%%xmm2 \n" + "pavgb %%xmm2,%%xmm0 \n" + "pavgb %%xmm5,%%xmm3 \n" + "pavgb %%xmm3,%%xmm1 \n" + "movdqa %%xmm0,%%xmm2 \n" + "psrlw $0x8,%%xmm0 \n" + "movdqa %%xmm1,%%xmm3 \n" + "psrlw $0x8,%%xmm1 \n" + "pand %%xmm7,%%xmm2 \n" + "pand %%xmm7,%%xmm3 \n" + "pavgw %%xmm2,%%xmm0 \n" + "pavgw %%xmm3,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqa %%xmm0,%%xmm2 \n" + "psrlw $0x8,%%xmm0 \n" + "pand %%xmm7,%%xmm2 \n" + "pavgw %%xmm2,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "movq %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x8,1) ",%1 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width), // %2 + "+r"(stridex3) // %3 + : "r"((intptr_t)(src_stride)) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm7" + ); +} + +void ScaleRowDown34_SSSE3(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "movdqa %0,%%xmm3 \n" + "movdqa %1,%%xmm4 \n" + "movdqa %2,%%xmm5 \n" + : + : "m"(kShuf0), // %0 + "m"(kShuf1), // %1 + "m"(kShuf2) // %2 + ); + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm2 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "movdqa %%xmm2,%%xmm1 \n" + "palignr $0x8,%%xmm0,%%xmm1 \n" + "pshufb %%xmm3,%%xmm0 \n" + "pshufb %%xmm4,%%xmm1 \n" + "pshufb %%xmm5,%%xmm2 \n" + "movq %%xmm0," MEMACCESS(1) " \n" + "movq %%xmm1," MEMACCESS2(0x8,1) " \n" + "movq %%xmm2," MEMACCESS2(0x10,1) " \n" + "lea " MEMLEA(0x18,1) ",%1 \n" + "sub $0x18,%2 \n" + "jg 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + :: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + ); +} + +void ScaleRowDown34_1_Box_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "movdqa %0,%%xmm2 \n" // kShuf01 + "movdqa %1,%%xmm3 \n" // kShuf11 + "movdqa %2,%%xmm4 \n" // kShuf21 + : + : "m"(kShuf01), // %0 + "m"(kShuf11), // %1 + "m"(kShuf21) // %2 + ); + asm volatile ( + "movdqa %0,%%xmm5 \n" // kMadd01 + "movdqa %1,%%xmm0 \n" // kMadd11 + "movdqa %2,%%xmm1 \n" // kRound34 + : + : "m"(kMadd01), // %0 + "m"(kMadd11), // %1 + "m"(kRound34) // %2 + ); + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm6 \n" + MEMOPREG(movdqu,0x00,0,3,1,xmm7) // movdqu (%0,%3),%%xmm7 + "pavgb %%xmm7,%%xmm6 \n" + "pshufb %%xmm2,%%xmm6 \n" + "pmaddubsw %%xmm5,%%xmm6 \n" + "paddsw %%xmm1,%%xmm6 \n" + "psrlw $0x2,%%xmm6 \n" + "packuswb %%xmm6,%%xmm6 \n" + "movq %%xmm6," MEMACCESS(1) " \n" + "movdqu " MEMACCESS2(0x8,0) ",%%xmm6 \n" + MEMOPREG(movdqu,0x8,0,3,1,xmm7) // movdqu 0x8(%0,%3),%%xmm7 + "pavgb %%xmm7,%%xmm6 \n" + "pshufb %%xmm3,%%xmm6 \n" + "pmaddubsw %%xmm0,%%xmm6 \n" + "paddsw %%xmm1,%%xmm6 \n" + "psrlw $0x2,%%xmm6 \n" + "packuswb %%xmm6,%%xmm6 \n" + "movq %%xmm6," MEMACCESS2(0x8,1) " \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm6 \n" + MEMOPREG(movdqu,0x10,0,3,1,xmm7) // movdqu 0x10(%0,%3),%%xmm7 + "lea " MEMLEA(0x20,0) ",%0 \n" + "pavgb %%xmm7,%%xmm6 \n" + "pshufb %%xmm4,%%xmm6 \n" + "pmaddubsw %4,%%xmm6 \n" + "paddsw %%xmm1,%%xmm6 \n" + "psrlw $0x2,%%xmm6 \n" + "packuswb %%xmm6,%%xmm6 \n" + "movq %%xmm6," MEMACCESS2(0x10,1) " \n" + "lea " MEMLEA(0x18,1) ",%1 \n" + "sub $0x18,%2 \n" + "jg 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : "r"((intptr_t)(src_stride)), // %3 + "m"(kMadd21) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} + +void ScaleRowDown34_0_Box_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "movdqa %0,%%xmm2 \n" // kShuf01 + "movdqa %1,%%xmm3 \n" // kShuf11 + "movdqa %2,%%xmm4 \n" // kShuf21 + : + : "m"(kShuf01), // %0 + "m"(kShuf11), // %1 + "m"(kShuf21) // %2 + ); + asm volatile ( + "movdqa %0,%%xmm5 \n" // kMadd01 + "movdqa %1,%%xmm0 \n" // kMadd11 + "movdqa %2,%%xmm1 \n" // kRound34 + : + : "m"(kMadd01), // %0 + "m"(kMadd11), // %1 + "m"(kRound34) // %2 + ); + + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm6 \n" + MEMOPREG(movdqu,0x00,0,3,1,xmm7) // movdqu (%0,%3,1),%%xmm7 + "pavgb %%xmm6,%%xmm7 \n" + "pavgb %%xmm7,%%xmm6 \n" + "pshufb %%xmm2,%%xmm6 \n" + "pmaddubsw %%xmm5,%%xmm6 \n" + "paddsw %%xmm1,%%xmm6 \n" + "psrlw $0x2,%%xmm6 \n" + "packuswb %%xmm6,%%xmm6 \n" + "movq %%xmm6," MEMACCESS(1) " \n" + "movdqu " MEMACCESS2(0x8,0) ",%%xmm6 \n" + MEMOPREG(movdqu,0x8,0,3,1,xmm7) // movdqu 0x8(%0,%3,1),%%xmm7 + "pavgb %%xmm6,%%xmm7 \n" + "pavgb %%xmm7,%%xmm6 \n" + "pshufb %%xmm3,%%xmm6 \n" + "pmaddubsw %%xmm0,%%xmm6 \n" + "paddsw %%xmm1,%%xmm6 \n" + "psrlw $0x2,%%xmm6 \n" + "packuswb %%xmm6,%%xmm6 \n" + "movq %%xmm6," MEMACCESS2(0x8,1) " \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm6 \n" + MEMOPREG(movdqu,0x10,0,3,1,xmm7) // movdqu 0x10(%0,%3,1),%%xmm7 + "lea " MEMLEA(0x20,0) ",%0 \n" + "pavgb %%xmm6,%%xmm7 \n" + "pavgb %%xmm7,%%xmm6 \n" + "pshufb %%xmm4,%%xmm6 \n" + "pmaddubsw %4,%%xmm6 \n" + "paddsw %%xmm1,%%xmm6 \n" + "psrlw $0x2,%%xmm6 \n" + "packuswb %%xmm6,%%xmm6 \n" + "movq %%xmm6," MEMACCESS2(0x10,1) " \n" + "lea " MEMLEA(0x18,1) ",%1 \n" + "sub $0x18,%2 \n" + "jg 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : "r"((intptr_t)(src_stride)), // %3 + "m"(kMadd21) // %4 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} + +void ScaleRowDown38_SSSE3(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "movdqa %3,%%xmm4 \n" + "movdqa %4,%%xmm5 \n" + + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "pshufb %%xmm4,%%xmm0 \n" + "pshufb %%xmm5,%%xmm1 \n" + "paddusb %%xmm1,%%xmm0 \n" + "movq %%xmm0," MEMACCESS(1) " \n" + "movhlps %%xmm0,%%xmm1 \n" + "movd %%xmm1," MEMACCESS2(0x8,1) " \n" + "lea " MEMLEA(0xc,1) ",%1 \n" + "sub $0xc,%2 \n" + "jg 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : "m"(kShuf38a), // %3 + "m"(kShuf38b) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5" + ); +} + +void ScaleRowDown38_2_Box_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "movdqa %0,%%xmm2 \n" + "movdqa %1,%%xmm3 \n" + "movdqa %2,%%xmm4 \n" + "movdqa %3,%%xmm5 \n" + : + : "m"(kShufAb0), // %0 + "m"(kShufAb1), // %1 + "m"(kShufAb2), // %2 + "m"(kScaleAb2) // %3 + ); + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,0,3,1,xmm1) // movdqu (%0,%3,1),%%xmm1 + "lea " MEMLEA(0x10,0) ",%0 \n" + "pavgb %%xmm1,%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "pshufb %%xmm2,%%xmm1 \n" + "movdqa %%xmm0,%%xmm6 \n" + "pshufb %%xmm3,%%xmm6 \n" + "paddusw %%xmm6,%%xmm1 \n" + "pshufb %%xmm4,%%xmm0 \n" + "paddusw %%xmm0,%%xmm1 \n" + "pmulhuw %%xmm5,%%xmm1 \n" + "packuswb %%xmm1,%%xmm1 \n" + "movd %%xmm1," MEMACCESS(1) " \n" + "psrlq $0x10,%%xmm1 \n" + "movd %%xmm1," MEMACCESS2(0x2,1) " \n" + "lea " MEMLEA(0x6,1) ",%1 \n" + "sub $0x6,%2 \n" + "jg 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : "r"((intptr_t)(src_stride)) // %3 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" + ); +} + +void ScaleRowDown38_3_Box_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "movdqa %0,%%xmm2 \n" + "movdqa %1,%%xmm3 \n" + "movdqa %2,%%xmm4 \n" + "pxor %%xmm5,%%xmm5 \n" + : + : "m"(kShufAc), // %0 + "m"(kShufAc3), // %1 + "m"(kScaleAc33) // %2 + ); + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movdqu,0x00,0,3,1,xmm6) // movdqu (%0,%3,1),%%xmm6 + "movhlps %%xmm0,%%xmm1 \n" + "movhlps %%xmm6,%%xmm7 \n" + "punpcklbw %%xmm5,%%xmm0 \n" + "punpcklbw %%xmm5,%%xmm1 \n" + "punpcklbw %%xmm5,%%xmm6 \n" + "punpcklbw %%xmm5,%%xmm7 \n" + "paddusw %%xmm6,%%xmm0 \n" + "paddusw %%xmm7,%%xmm1 \n" + MEMOPREG(movdqu,0x00,0,3,2,xmm6) // movdqu (%0,%3,2),%%xmm6 + "lea " MEMLEA(0x10,0) ",%0 \n" + "movhlps %%xmm6,%%xmm7 \n" + "punpcklbw %%xmm5,%%xmm6 \n" + "punpcklbw %%xmm5,%%xmm7 \n" + "paddusw %%xmm6,%%xmm0 \n" + "paddusw %%xmm7,%%xmm1 \n" + "movdqa %%xmm0,%%xmm6 \n" + "psrldq $0x2,%%xmm0 \n" + "paddusw %%xmm0,%%xmm6 \n" + "psrldq $0x2,%%xmm0 \n" + "paddusw %%xmm0,%%xmm6 \n" + "pshufb %%xmm2,%%xmm6 \n" + "movdqa %%xmm1,%%xmm7 \n" + "psrldq $0x2,%%xmm1 \n" + "paddusw %%xmm1,%%xmm7 \n" + "psrldq $0x2,%%xmm1 \n" + "paddusw %%xmm1,%%xmm7 \n" + "pshufb %%xmm3,%%xmm7 \n" + "paddusw %%xmm7,%%xmm6 \n" + "pmulhuw %%xmm4,%%xmm6 \n" + "packuswb %%xmm6,%%xmm6 \n" + "movd %%xmm6," MEMACCESS(1) " \n" + "psrlq $0x10,%%xmm6 \n" + "movd %%xmm6," MEMACCESS2(0x2,1) " \n" + "lea " MEMLEA(0x6,1) ",%1 \n" + "sub $0x6,%2 \n" + "jg 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : "r"((intptr_t)(src_stride)) // %3 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" + ); +} + +// Reads 16xN bytes and produces 16 shorts at a time. +void ScaleAddRows_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint16* dst_ptr, int src_width, int src_height) { + int tmp_height = 0; + intptr_t tmp_src = 0; + asm volatile ( + "mov %0,%3 \n" // row pointer + "mov %5,%2 \n" // height + "pxor %%xmm0,%%xmm0 \n" // clear accumulators + "pxor %%xmm1,%%xmm1 \n" + "pxor %%xmm4,%%xmm4 \n" + + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(3) ",%%xmm2 \n" + "add %6,%3 \n" + "movdqa %%xmm2,%%xmm3 \n" + "punpcklbw %%xmm4,%%xmm2 \n" + "punpckhbw %%xmm4,%%xmm3 \n" + "paddusw %%xmm2,%%xmm0 \n" + "paddusw %%xmm3,%%xmm1 \n" + "sub $0x1,%2 \n" + "jg 1b \n" + + "movdqu %%xmm0," MEMACCESS(1) " \n" + "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" + "lea " MEMLEA(0x20,1) ",%1 \n" + "lea " MEMLEA(0x10,0) ",%0 \n" // src_ptr += 16 + "mov %0,%3 \n" // row pointer + "mov %5,%2 \n" // height + "pxor %%xmm0,%%xmm0 \n" // clear accumulators + "pxor %%xmm1,%%xmm1 \n" + "sub $0x10,%4 \n" + "jg 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(tmp_height), // %2 + "+r"(tmp_src), // %3 + "+r"(src_width), // %4 + "+rm"(src_height) // %5 + : "rm"((intptr_t)(src_stride)) // %6 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4" + ); +} + +// Bilinear column filtering. SSSE3 version. +void ScaleFilterCols_SSSE3(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx) { + intptr_t x0 = 0, x1 = 0, temp_pixel = 0; + asm volatile ( + "movd %6,%%xmm2 \n" + "movd %7,%%xmm3 \n" + "movl $0x04040000,%k2 \n" + "movd %k2,%%xmm5 \n" + "pcmpeqb %%xmm6,%%xmm6 \n" + "psrlw $0x9,%%xmm6 \n" + "pextrw $0x1,%%xmm2,%k3 \n" + "subl $0x2,%5 \n" + "jl 29f \n" + "movdqa %%xmm2,%%xmm0 \n" + "paddd %%xmm3,%%xmm0 \n" + "punpckldq %%xmm0,%%xmm2 \n" + "punpckldq %%xmm3,%%xmm3 \n" + "paddd %%xmm3,%%xmm3 \n" + "pextrw $0x3,%%xmm2,%k4 \n" + + LABELALIGN + "2: \n" + "movdqa %%xmm2,%%xmm1 \n" + "paddd %%xmm3,%%xmm2 \n" + MEMOPARG(movzwl,0x00,1,3,1,k2) // movzwl (%1,%3,1),%k2 + "movd %k2,%%xmm0 \n" + "psrlw $0x9,%%xmm1 \n" + MEMOPARG(movzwl,0x00,1,4,1,k2) // movzwl (%1,%4,1),%k2 + "movd %k2,%%xmm4 \n" + "pshufb %%xmm5,%%xmm1 \n" + "punpcklwd %%xmm4,%%xmm0 \n" + "pxor %%xmm6,%%xmm1 \n" + "pmaddubsw %%xmm1,%%xmm0 \n" + "pextrw $0x1,%%xmm2,%k3 \n" + "pextrw $0x3,%%xmm2,%k4 \n" + "psrlw $0x7,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "movd %%xmm0,%k2 \n" + "mov %w2," MEMACCESS(0) " \n" + "lea " MEMLEA(0x2,0) ",%0 \n" + "sub $0x2,%5 \n" + "jge 2b \n" + + LABELALIGN + "29: \n" + "addl $0x1,%5 \n" + "jl 99f \n" + MEMOPARG(movzwl,0x00,1,3,1,k2) // movzwl (%1,%3,1),%k2 + "movd %k2,%%xmm0 \n" + "psrlw $0x9,%%xmm2 \n" + "pshufb %%xmm5,%%xmm2 \n" + "pxor %%xmm6,%%xmm2 \n" + "pmaddubsw %%xmm2,%%xmm0 \n" + "psrlw $0x7,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "movd %%xmm0,%k2 \n" + "mov %b2," MEMACCESS(0) " \n" + "99: \n" + : "+r"(dst_ptr), // %0 + "+r"(src_ptr), // %1 + "+a"(temp_pixel), // %2 + "+r"(x0), // %3 + "+r"(x1), // %4 + "+rm"(dst_width) // %5 + : "rm"(x), // %6 + "rm"(dx) // %7 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" + ); +} + +// Reads 4 pixels, duplicates them and writes 8 pixels. +// Alignment requirement: src_argb 16 byte aligned, dst_argb 16 byte aligned. +void ScaleColsUp2_SSE2(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx) { + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(1) ",%%xmm0 \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklbw %%xmm0,%%xmm0 \n" + "punpckhbw %%xmm1,%%xmm1 \n" + "movdqu %%xmm0," MEMACCESS(0) " \n" + "movdqu %%xmm1," MEMACCESS2(0x10,0) " \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + + : "+r"(dst_ptr), // %0 + "+r"(src_ptr), // %1 + "+r"(dst_width) // %2 + :: "memory", "cc", "xmm0", "xmm1" + ); +} + +void ScaleARGBRowDown2_SSE2(const uint8* src_argb, + ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) { + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "shufps $0xdd,%%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x4,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(dst_width) // %2 + :: "memory", "cc", "xmm0", "xmm1" + ); +} + +void ScaleARGBRowDown2Linear_SSE2(const uint8* src_argb, + ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) { + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "movdqa %%xmm0,%%xmm2 \n" + "shufps $0x88,%%xmm1,%%xmm0 \n" + "shufps $0xdd,%%xmm1,%%xmm2 \n" + "pavgb %%xmm2,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x4,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(dst_width) // %2 + :: "memory", "cc", "xmm0", "xmm1" + ); +} + +void ScaleARGBRowDown2Box_SSE2(const uint8* src_argb, + ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) { + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(0) ",%%xmm0 \n" + "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" + MEMOPREG(movdqu,0x00,0,3,1,xmm2) // movdqu (%0,%3,1),%%xmm2 + MEMOPREG(movdqu,0x10,0,3,1,xmm3) // movdqu 0x10(%0,%3,1),%%xmm3 + "lea " MEMLEA(0x20,0) ",%0 \n" + "pavgb %%xmm2,%%xmm0 \n" + "pavgb %%xmm3,%%xmm1 \n" + "movdqa %%xmm0,%%xmm2 \n" + "shufps $0x88,%%xmm1,%%xmm0 \n" + "shufps $0xdd,%%xmm1,%%xmm2 \n" + "pavgb %%xmm2,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(1) " \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "sub $0x4,%2 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(dst_width) // %2 + : "r"((intptr_t)(src_stride)) // %3 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3" + ); +} + +// Reads 4 pixels at a time. +// Alignment requirement: dst_argb 16 byte aligned. +void ScaleARGBRowDownEven_SSE2(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, uint8* dst_argb, int dst_width) { + intptr_t src_stepx_x4 = (intptr_t)(src_stepx); + intptr_t src_stepx_x12 = 0; + asm volatile ( + "lea " MEMLEA3(0x00,1,4) ",%1 \n" + "lea " MEMLEA4(0x00,1,1,2) ",%4 \n" + LABELALIGN + "1: \n" + "movd " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movd,0x00,0,1,1,xmm1) // movd (%0,%1,1),%%xmm1 + "punpckldq %%xmm1,%%xmm0 \n" + MEMOPREG(movd,0x00,0,1,2,xmm2) // movd (%0,%1,2),%%xmm2 + MEMOPREG(movd,0x00,0,4,1,xmm3) // movd (%0,%4,1),%%xmm3 + "lea " MEMLEA4(0x00,0,1,4) ",%0 \n" + "punpckldq %%xmm3,%%xmm2 \n" + "punpcklqdq %%xmm2,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "sub $0x4,%3 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_stepx_x4), // %1 + "+r"(dst_argb), // %2 + "+r"(dst_width), // %3 + "+r"(src_stepx_x12) // %4 + :: "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3" + ); +} + +// Blends four 2x2 to 4x1. +// Alignment requirement: dst_argb 16 byte aligned. +void ScaleARGBRowDownEvenBox_SSE2(const uint8* src_argb, + ptrdiff_t src_stride, int src_stepx, + uint8* dst_argb, int dst_width) { + intptr_t src_stepx_x4 = (intptr_t)(src_stepx); + intptr_t src_stepx_x12 = 0; + intptr_t row1 = (intptr_t)(src_stride); + asm volatile ( + "lea " MEMLEA3(0x00,1,4) ",%1 \n" + "lea " MEMLEA4(0x00,1,1,2) ",%4 \n" + "lea " MEMLEA4(0x00,0,5,1) ",%5 \n" + + LABELALIGN + "1: \n" + "movq " MEMACCESS(0) ",%%xmm0 \n" + MEMOPREG(movhps,0x00,0,1,1,xmm0) // movhps (%0,%1,1),%%xmm0 + MEMOPREG(movq,0x00,0,1,2,xmm1) // movq (%0,%1,2),%%xmm1 + MEMOPREG(movhps,0x00,0,4,1,xmm1) // movhps (%0,%4,1),%%xmm1 + "lea " MEMLEA4(0x00,0,1,4) ",%0 \n" + "movq " MEMACCESS(5) ",%%xmm2 \n" + MEMOPREG(movhps,0x00,5,1,1,xmm2) // movhps (%5,%1,1),%%xmm2 + MEMOPREG(movq,0x00,5,1,2,xmm3) // movq (%5,%1,2),%%xmm3 + MEMOPREG(movhps,0x00,5,4,1,xmm3) // movhps (%5,%4,1),%%xmm3 + "lea " MEMLEA4(0x00,5,1,4) ",%5 \n" + "pavgb %%xmm2,%%xmm0 \n" + "pavgb %%xmm3,%%xmm1 \n" + "movdqa %%xmm0,%%xmm2 \n" + "shufps $0x88,%%xmm1,%%xmm0 \n" + "shufps $0xdd,%%xmm1,%%xmm2 \n" + "pavgb %%xmm2,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "sub $0x4,%3 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_stepx_x4), // %1 + "+r"(dst_argb), // %2 + "+rm"(dst_width), // %3 + "+r"(src_stepx_x12), // %4 + "+r"(row1) // %5 + :: "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3" + ); +} + +void ScaleARGBCols_SSE2(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + intptr_t x0 = 0, x1 = 0; + asm volatile ( + "movd %5,%%xmm2 \n" + "movd %6,%%xmm3 \n" + "pshufd $0x0,%%xmm2,%%xmm2 \n" + "pshufd $0x11,%%xmm3,%%xmm0 \n" + "paddd %%xmm0,%%xmm2 \n" + "paddd %%xmm3,%%xmm3 \n" + "pshufd $0x5,%%xmm3,%%xmm0 \n" + "paddd %%xmm0,%%xmm2 \n" + "paddd %%xmm3,%%xmm3 \n" + "pshufd $0x0,%%xmm3,%%xmm3 \n" + "pextrw $0x1,%%xmm2,%k0 \n" + "pextrw $0x3,%%xmm2,%k1 \n" + "cmp $0x0,%4 \n" + "jl 99f \n" + "sub $0x4,%4 \n" + "jl 49f \n" + + LABELALIGN + "40: \n" + MEMOPREG(movd,0x00,3,0,4,xmm0) // movd (%3,%0,4),%%xmm0 + MEMOPREG(movd,0x00,3,1,4,xmm1) // movd (%3,%1,4),%%xmm1 + "pextrw $0x5,%%xmm2,%k0 \n" + "pextrw $0x7,%%xmm2,%k1 \n" + "paddd %%xmm3,%%xmm2 \n" + "punpckldq %%xmm1,%%xmm0 \n" + MEMOPREG(movd,0x00,3,0,4,xmm1) // movd (%3,%0,4),%%xmm1 + MEMOPREG(movd,0x00,3,1,4,xmm4) // movd (%3,%1,4),%%xmm4 + "pextrw $0x1,%%xmm2,%k0 \n" + "pextrw $0x3,%%xmm2,%k1 \n" + "punpckldq %%xmm4,%%xmm1 \n" + "punpcklqdq %%xmm1,%%xmm0 \n" + "movdqu %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x10,2) ",%2 \n" + "sub $0x4,%4 \n" + "jge 40b \n" + + "49: \n" + "test $0x2,%4 \n" + "je 29f \n" + MEMOPREG(movd,0x00,3,0,4,xmm0) // movd (%3,%0,4),%%xmm0 + MEMOPREG(movd,0x00,3,1,4,xmm1) // movd (%3,%1,4),%%xmm1 + "pextrw $0x5,%%xmm2,%k0 \n" + "punpckldq %%xmm1,%%xmm0 \n" + "movq %%xmm0," MEMACCESS(2) " \n" + "lea " MEMLEA(0x8,2) ",%2 \n" + "29: \n" + "test $0x1,%4 \n" + "je 99f \n" + MEMOPREG(movd,0x00,3,0,4,xmm0) // movd (%3,%0,4),%%xmm0 + "movd %%xmm0," MEMACCESS(2) " \n" + "99: \n" + : "+a"(x0), // %0 + "+d"(x1), // %1 + "+r"(dst_argb), // %2 + "+r"(src_argb), // %3 + "+r"(dst_width) // %4 + : "rm"(x), // %5 + "rm"(dx) // %6 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4" + ); +} + +// Reads 4 pixels, duplicates them and writes 8 pixels. +// Alignment requirement: src_argb 16 byte aligned, dst_argb 16 byte aligned. +void ScaleARGBColsUp2_SSE2(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + asm volatile ( + LABELALIGN + "1: \n" + "movdqu " MEMACCESS(1) ",%%xmm0 \n" + "lea " MEMLEA(0x10,1) ",%1 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpckldq %%xmm0,%%xmm0 \n" + "punpckhdq %%xmm1,%%xmm1 \n" + "movdqu %%xmm0," MEMACCESS(0) " \n" + "movdqu %%xmm1," MEMACCESS2(0x10,0) " \n" + "lea " MEMLEA(0x20,0) ",%0 \n" + "sub $0x8,%2 \n" + "jg 1b \n" + + : "+r"(dst_argb), // %0 + "+r"(src_argb), // %1 + "+r"(dst_width) // %2 + :: "memory", "cc", NACL_R14 + "xmm0", "xmm1" + ); +} + +// Shuffle table for arranging 2 pixels into pairs for pmaddubsw +static uvec8 kShuffleColARGB = { + 0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u, // bbggrraa 1st pixel + 8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u // bbggrraa 2nd pixel +}; + +// Shuffle table for duplicating 2 fractions into 8 bytes each +static uvec8 kShuffleFractions = { + 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, +}; + +// Bilinear row filtering combines 4x2 -> 4x1. SSSE3 version +void ScaleARGBFilterCols_SSSE3(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + intptr_t x0 = 0, x1 = 0; + asm volatile ( + "movdqa %0,%%xmm4 \n" + "movdqa %1,%%xmm5 \n" + : + : "m"(kShuffleColARGB), // %0 + "m"(kShuffleFractions) // %1 + ); + + asm volatile ( + "movd %5,%%xmm2 \n" + "movd %6,%%xmm3 \n" + "pcmpeqb %%xmm6,%%xmm6 \n" + "psrlw $0x9,%%xmm6 \n" + "pextrw $0x1,%%xmm2,%k3 \n" + "sub $0x2,%2 \n" + "jl 29f \n" + "movdqa %%xmm2,%%xmm0 \n" + "paddd %%xmm3,%%xmm0 \n" + "punpckldq %%xmm0,%%xmm2 \n" + "punpckldq %%xmm3,%%xmm3 \n" + "paddd %%xmm3,%%xmm3 \n" + "pextrw $0x3,%%xmm2,%k4 \n" + + LABELALIGN + "2: \n" + "movdqa %%xmm2,%%xmm1 \n" + "paddd %%xmm3,%%xmm2 \n" + MEMOPREG(movq,0x00,1,3,4,xmm0) // movq (%1,%3,4),%%xmm0 + "psrlw $0x9,%%xmm1 \n" + MEMOPREG(movhps,0x00,1,4,4,xmm0) // movhps (%1,%4,4),%%xmm0 + "pshufb %%xmm5,%%xmm1 \n" + "pshufb %%xmm4,%%xmm0 \n" + "pxor %%xmm6,%%xmm1 \n" + "pmaddubsw %%xmm1,%%xmm0 \n" + "psrlw $0x7,%%xmm0 \n" + "pextrw $0x1,%%xmm2,%k3 \n" + "pextrw $0x3,%%xmm2,%k4 \n" + "packuswb %%xmm0,%%xmm0 \n" + "movq %%xmm0," MEMACCESS(0) " \n" + "lea " MEMLEA(0x8,0) ",%0 \n" + "sub $0x2,%2 \n" + "jge 2b \n" + + LABELALIGN + "29: \n" + "add $0x1,%2 \n" + "jl 99f \n" + "psrlw $0x9,%%xmm2 \n" + MEMOPREG(movq,0x00,1,3,4,xmm0) // movq (%1,%3,4),%%xmm0 + "pshufb %%xmm5,%%xmm2 \n" + "pshufb %%xmm4,%%xmm0 \n" + "pxor %%xmm6,%%xmm2 \n" + "pmaddubsw %%xmm2,%%xmm0 \n" + "psrlw $0x7,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "movd %%xmm0," MEMACCESS(0) " \n" + + LABELALIGN + "99: \n" + : "+r"(dst_argb), // %0 + "+r"(src_argb), // %1 + "+rm"(dst_width), // %2 + "+r"(x0), // %3 + "+r"(x1) // %4 + : "rm"(x), // %5 + "rm"(dx) // %6 + : "memory", "cc", NACL_R14 + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" + ); +} + +// Divide num by div and return as 16.16 fixed point result. +int FixedDiv_X86(int num, int div) { + asm volatile ( + "cdq \n" + "shld $0x10,%%eax,%%edx \n" + "shl $0x10,%%eax \n" + "idiv %1 \n" + "mov %0, %%eax \n" + : "+a"(num) // %0 + : "c"(div) // %1 + : "memory", "cc", "edx" + ); + return num; +} + +// Divide num - 1 by div - 1 and return as 16.16 fixed point result. +int FixedDiv1_X86(int num, int div) { + asm volatile ( + "cdq \n" + "shld $0x10,%%eax,%%edx \n" + "shl $0x10,%%eax \n" + "sub $0x10001,%%eax \n" + "sbb $0x0,%%edx \n" + "sub $0x1,%1 \n" + "idiv %1 \n" + "mov %0, %%eax \n" + : "+a"(num) // %0 + : "c"(div) // %1 + : "memory", "cc", "edx" + ); + return num; +} + +#endif // defined(__x86_64__) || defined(__i386__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/scale_mips.cc b/third_party/aom/third_party/libyuv/source/scale_mips.cc new file mode 100644 index 0000000000..3eb4f27c45 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/scale_mips.cc @@ -0,0 +1,654 @@ +/* + * Copyright 2012 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/basic_types.h" +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for GCC MIPS DSPR2 +#if !defined(LIBYUV_DISABLE_MIPS) && \ + defined(__mips_dsp) && (__mips_dsp_rev >= 2) && \ + (_MIPS_SIM == _MIPS_SIM_ABI32) + +void ScaleRowDown2_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + __asm__ __volatile__( + ".set push \n" + ".set noreorder \n" + + "srl $t9, %[dst_width], 4 \n" // iterations -> by 16 + "beqz $t9, 2f \n" + " nop \n" + + ".p2align 2 \n" + "1: \n" + "lw $t0, 0(%[src_ptr]) \n" // |3|2|1|0| + "lw $t1, 4(%[src_ptr]) \n" // |7|6|5|4| + "lw $t2, 8(%[src_ptr]) \n" // |11|10|9|8| + "lw $t3, 12(%[src_ptr]) \n" // |15|14|13|12| + "lw $t4, 16(%[src_ptr]) \n" // |19|18|17|16| + "lw $t5, 20(%[src_ptr]) \n" // |23|22|21|20| + "lw $t6, 24(%[src_ptr]) \n" // |27|26|25|24| + "lw $t7, 28(%[src_ptr]) \n" // |31|30|29|28| + // TODO(fbarchard): Use odd pixels instead of even. + "precr.qb.ph $t8, $t1, $t0 \n" // |6|4|2|0| + "precr.qb.ph $t0, $t3, $t2 \n" // |14|12|10|8| + "precr.qb.ph $t1, $t5, $t4 \n" // |22|20|18|16| + "precr.qb.ph $t2, $t7, $t6 \n" // |30|28|26|24| + "addiu %[src_ptr], %[src_ptr], 32 \n" + "addiu $t9, $t9, -1 \n" + "sw $t8, 0(%[dst]) \n" + "sw $t0, 4(%[dst]) \n" + "sw $t1, 8(%[dst]) \n" + "sw $t2, 12(%[dst]) \n" + "bgtz $t9, 1b \n" + " addiu %[dst], %[dst], 16 \n" + + "2: \n" + "andi $t9, %[dst_width], 0xf \n" // residue + "beqz $t9, 3f \n" + " nop \n" + + "21: \n" + "lbu $t0, 0(%[src_ptr]) \n" + "addiu %[src_ptr], %[src_ptr], 2 \n" + "addiu $t9, $t9, -1 \n" + "sb $t0, 0(%[dst]) \n" + "bgtz $t9, 21b \n" + " addiu %[dst], %[dst], 1 \n" + + "3: \n" + ".set pop \n" + : [src_ptr] "+r" (src_ptr), + [dst] "+r" (dst) + : [dst_width] "r" (dst_width) + : "t0", "t1", "t2", "t3", "t4", "t5", + "t6", "t7", "t8", "t9" + ); +} + +void ScaleRowDown2Box_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + const uint8* t = src_ptr + src_stride; + + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + + "srl $t9, %[dst_width], 3 \n" // iterations -> step 8 + "bltz $t9, 2f \n" + " nop \n" + + ".p2align 2 \n" + "1: \n" + "lw $t0, 0(%[src_ptr]) \n" // |3|2|1|0| + "lw $t1, 4(%[src_ptr]) \n" // |7|6|5|4| + "lw $t2, 8(%[src_ptr]) \n" // |11|10|9|8| + "lw $t3, 12(%[src_ptr]) \n" // |15|14|13|12| + "lw $t4, 0(%[t]) \n" // |19|18|17|16| + "lw $t5, 4(%[t]) \n" // |23|22|21|20| + "lw $t6, 8(%[t]) \n" // |27|26|25|24| + "lw $t7, 12(%[t]) \n" // |31|30|29|28| + "addiu $t9, $t9, -1 \n" + "srl $t8, $t0, 16 \n" // |X|X|3|2| + "ins $t0, $t4, 16, 16 \n" // |17|16|1|0| + "ins $t4, $t8, 0, 16 \n" // |19|18|3|2| + "raddu.w.qb $t0, $t0 \n" // |17+16+1+0| + "raddu.w.qb $t4, $t4 \n" // |19+18+3+2| + "shra_r.w $t0, $t0, 2 \n" // |t0+2|>>2 + "shra_r.w $t4, $t4, 2 \n" // |t4+2|>>2 + "srl $t8, $t1, 16 \n" // |X|X|7|6| + "ins $t1, $t5, 16, 16 \n" // |21|20|5|4| + "ins $t5, $t8, 0, 16 \n" // |22|23|7|6| + "raddu.w.qb $t1, $t1 \n" // |21+20+5+4| + "raddu.w.qb $t5, $t5 \n" // |23+22+7+6| + "shra_r.w $t1, $t1, 2 \n" // |t1+2|>>2 + "shra_r.w $t5, $t5, 2 \n" // |t5+2|>>2 + "srl $t8, $t2, 16 \n" // |X|X|11|10| + "ins $t2, $t6, 16, 16 \n" // |25|24|9|8| + "ins $t6, $t8, 0, 16 \n" // |27|26|11|10| + "raddu.w.qb $t2, $t2 \n" // |25+24+9+8| + "raddu.w.qb $t6, $t6 \n" // |27+26+11+10| + "shra_r.w $t2, $t2, 2 \n" // |t2+2|>>2 + "shra_r.w $t6, $t6, 2 \n" // |t5+2|>>2 + "srl $t8, $t3, 16 \n" // |X|X|15|14| + "ins $t3, $t7, 16, 16 \n" // |29|28|13|12| + "ins $t7, $t8, 0, 16 \n" // |31|30|15|14| + "raddu.w.qb $t3, $t3 \n" // |29+28+13+12| + "raddu.w.qb $t7, $t7 \n" // |31+30+15+14| + "shra_r.w $t3, $t3, 2 \n" // |t3+2|>>2 + "shra_r.w $t7, $t7, 2 \n" // |t7+2|>>2 + "addiu %[src_ptr], %[src_ptr], 16 \n" + "addiu %[t], %[t], 16 \n" + "sb $t0, 0(%[dst]) \n" + "sb $t4, 1(%[dst]) \n" + "sb $t1, 2(%[dst]) \n" + "sb $t5, 3(%[dst]) \n" + "sb $t2, 4(%[dst]) \n" + "sb $t6, 5(%[dst]) \n" + "sb $t3, 6(%[dst]) \n" + "sb $t7, 7(%[dst]) \n" + "bgtz $t9, 1b \n" + " addiu %[dst], %[dst], 8 \n" + + "2: \n" + "andi $t9, %[dst_width], 0x7 \n" // x = residue + "beqz $t9, 3f \n" + " nop \n" + + "21: \n" + "lwr $t1, 0(%[src_ptr]) \n" + "lwl $t1, 3(%[src_ptr]) \n" + "lwr $t2, 0(%[t]) \n" + "lwl $t2, 3(%[t]) \n" + "srl $t8, $t1, 16 \n" + "ins $t1, $t2, 16, 16 \n" + "ins $t2, $t8, 0, 16 \n" + "raddu.w.qb $t1, $t1 \n" + "raddu.w.qb $t2, $t2 \n" + "shra_r.w $t1, $t1, 2 \n" + "shra_r.w $t2, $t2, 2 \n" + "sb $t1, 0(%[dst]) \n" + "sb $t2, 1(%[dst]) \n" + "addiu %[src_ptr], %[src_ptr], 4 \n" + "addiu $t9, $t9, -2 \n" + "addiu %[t], %[t], 4 \n" + "bgtz $t9, 21b \n" + " addiu %[dst], %[dst], 2 \n" + + "3: \n" + ".set pop \n" + + : [src_ptr] "+r" (src_ptr), + [dst] "+r" (dst), [t] "+r" (t) + : [dst_width] "r" (dst_width) + : "t0", "t1", "t2", "t3", "t4", "t5", + "t6", "t7", "t8", "t9" + ); +} + +void ScaleRowDown4_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + + "srl $t9, %[dst_width], 3 \n" + "beqz $t9, 2f \n" + " nop \n" + + ".p2align 2 \n" + "1: \n" + "lw $t1, 0(%[src_ptr]) \n" // |3|2|1|0| + "lw $t2, 4(%[src_ptr]) \n" // |7|6|5|4| + "lw $t3, 8(%[src_ptr]) \n" // |11|10|9|8| + "lw $t4, 12(%[src_ptr]) \n" // |15|14|13|12| + "lw $t5, 16(%[src_ptr]) \n" // |19|18|17|16| + "lw $t6, 20(%[src_ptr]) \n" // |23|22|21|20| + "lw $t7, 24(%[src_ptr]) \n" // |27|26|25|24| + "lw $t8, 28(%[src_ptr]) \n" // |31|30|29|28| + "precr.qb.ph $t1, $t2, $t1 \n" // |6|4|2|0| + "precr.qb.ph $t2, $t4, $t3 \n" // |14|12|10|8| + "precr.qb.ph $t5, $t6, $t5 \n" // |22|20|18|16| + "precr.qb.ph $t6, $t8, $t7 \n" // |30|28|26|24| + "precr.qb.ph $t1, $t2, $t1 \n" // |12|8|4|0| + "precr.qb.ph $t5, $t6, $t5 \n" // |28|24|20|16| + "addiu %[src_ptr], %[src_ptr], 32 \n" + "addiu $t9, $t9, -1 \n" + "sw $t1, 0(%[dst]) \n" + "sw $t5, 4(%[dst]) \n" + "bgtz $t9, 1b \n" + " addiu %[dst], %[dst], 8 \n" + + "2: \n" + "andi $t9, %[dst_width], 7 \n" // residue + "beqz $t9, 3f \n" + " nop \n" + + "21: \n" + "lbu $t1, 0(%[src_ptr]) \n" + "addiu %[src_ptr], %[src_ptr], 4 \n" + "addiu $t9, $t9, -1 \n" + "sb $t1, 0(%[dst]) \n" + "bgtz $t9, 21b \n" + " addiu %[dst], %[dst], 1 \n" + + "3: \n" + ".set pop \n" + : [src_ptr] "+r" (src_ptr), + [dst] "+r" (dst) + : [dst_width] "r" (dst_width) + : "t1", "t2", "t3", "t4", "t5", + "t6", "t7", "t8", "t9" + ); +} + +void ScaleRowDown4Box_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + intptr_t stride = src_stride; + const uint8* s1 = src_ptr + stride; + const uint8* s2 = s1 + stride; + const uint8* s3 = s2 + stride; + + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + + "srl $t9, %[dst_width], 1 \n" + "andi $t8, %[dst_width], 1 \n" + + ".p2align 2 \n" + "1: \n" + "lw $t0, 0(%[src_ptr]) \n" // |3|2|1|0| + "lw $t1, 0(%[s1]) \n" // |7|6|5|4| + "lw $t2, 0(%[s2]) \n" // |11|10|9|8| + "lw $t3, 0(%[s3]) \n" // |15|14|13|12| + "lw $t4, 4(%[src_ptr]) \n" // |19|18|17|16| + "lw $t5, 4(%[s1]) \n" // |23|22|21|20| + "lw $t6, 4(%[s2]) \n" // |27|26|25|24| + "lw $t7, 4(%[s3]) \n" // |31|30|29|28| + "raddu.w.qb $t0, $t0 \n" // |3 + 2 + 1 + 0| + "raddu.w.qb $t1, $t1 \n" // |7 + 6 + 5 + 4| + "raddu.w.qb $t2, $t2 \n" // |11 + 10 + 9 + 8| + "raddu.w.qb $t3, $t3 \n" // |15 + 14 + 13 + 12| + "raddu.w.qb $t4, $t4 \n" // |19 + 18 + 17 + 16| + "raddu.w.qb $t5, $t5 \n" // |23 + 22 + 21 + 20| + "raddu.w.qb $t6, $t6 \n" // |27 + 26 + 25 + 24| + "raddu.w.qb $t7, $t7 \n" // |31 + 30 + 29 + 28| + "add $t0, $t0, $t1 \n" + "add $t1, $t2, $t3 \n" + "add $t0, $t0, $t1 \n" + "add $t4, $t4, $t5 \n" + "add $t6, $t6, $t7 \n" + "add $t4, $t4, $t6 \n" + "shra_r.w $t0, $t0, 4 \n" + "shra_r.w $t4, $t4, 4 \n" + "sb $t0, 0(%[dst]) \n" + "sb $t4, 1(%[dst]) \n" + "addiu %[src_ptr], %[src_ptr], 8 \n" + "addiu %[s1], %[s1], 8 \n" + "addiu %[s2], %[s2], 8 \n" + "addiu %[s3], %[s3], 8 \n" + "addiu $t9, $t9, -1 \n" + "bgtz $t9, 1b \n" + " addiu %[dst], %[dst], 2 \n" + "beqz $t8, 2f \n" + " nop \n" + + "lw $t0, 0(%[src_ptr]) \n" // |3|2|1|0| + "lw $t1, 0(%[s1]) \n" // |7|6|5|4| + "lw $t2, 0(%[s2]) \n" // |11|10|9|8| + "lw $t3, 0(%[s3]) \n" // |15|14|13|12| + "raddu.w.qb $t0, $t0 \n" // |3 + 2 + 1 + 0| + "raddu.w.qb $t1, $t1 \n" // |7 + 6 + 5 + 4| + "raddu.w.qb $t2, $t2 \n" // |11 + 10 + 9 + 8| + "raddu.w.qb $t3, $t3 \n" // |15 + 14 + 13 + 12| + "add $t0, $t0, $t1 \n" + "add $t1, $t2, $t3 \n" + "add $t0, $t0, $t1 \n" + "shra_r.w $t0, $t0, 4 \n" + "sb $t0, 0(%[dst]) \n" + + "2: \n" + ".set pop \n" + + : [src_ptr] "+r" (src_ptr), + [dst] "+r" (dst), + [s1] "+r" (s1), + [s2] "+r" (s2), + [s3] "+r" (s3) + : [dst_width] "r" (dst_width) + : "t0", "t1", "t2", "t3", "t4", "t5", + "t6","t7", "t8", "t9" + ); +} + +void ScaleRowDown34_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + ".p2align 2 \n" + "1: \n" + "lw $t1, 0(%[src_ptr]) \n" // |3|2|1|0| + "lw $t2, 4(%[src_ptr]) \n" // |7|6|5|4| + "lw $t3, 8(%[src_ptr]) \n" // |11|10|9|8| + "lw $t4, 12(%[src_ptr]) \n" // |15|14|13|12| + "lw $t5, 16(%[src_ptr]) \n" // |19|18|17|16| + "lw $t6, 20(%[src_ptr]) \n" // |23|22|21|20| + "lw $t7, 24(%[src_ptr]) \n" // |27|26|25|24| + "lw $t8, 28(%[src_ptr]) \n" // |31|30|29|28| + "precrq.qb.ph $t0, $t2, $t4 \n" // |7|5|15|13| + "precrq.qb.ph $t9, $t6, $t8 \n" // |23|21|31|30| + "addiu %[dst_width], %[dst_width], -24 \n" + "ins $t1, $t1, 8, 16 \n" // |3|1|0|X| + "ins $t4, $t0, 8, 16 \n" // |X|15|13|12| + "ins $t5, $t5, 8, 16 \n" // |19|17|16|X| + "ins $t8, $t9, 8, 16 \n" // |X|31|29|28| + "addiu %[src_ptr], %[src_ptr], 32 \n" + "packrl.ph $t0, $t3, $t0 \n" // |9|8|7|5| + "packrl.ph $t9, $t7, $t9 \n" // |25|24|23|21| + "prepend $t1, $t2, 8 \n" // |4|3|1|0| + "prepend $t3, $t4, 24 \n" // |15|13|12|11| + "prepend $t5, $t6, 8 \n" // |20|19|17|16| + "prepend $t7, $t8, 24 \n" // |31|29|28|27| + "sw $t1, 0(%[dst]) \n" + "sw $t0, 4(%[dst]) \n" + "sw $t3, 8(%[dst]) \n" + "sw $t5, 12(%[dst]) \n" + "sw $t9, 16(%[dst]) \n" + "sw $t7, 20(%[dst]) \n" + "bnez %[dst_width], 1b \n" + " addiu %[dst], %[dst], 24 \n" + ".set pop \n" + : [src_ptr] "+r" (src_ptr), + [dst] "+r" (dst), + [dst_width] "+r" (dst_width) + : + : "t0", "t1", "t2", "t3", "t4", "t5", + "t6","t7", "t8", "t9" + ); +} + +void ScaleRowDown34_0_Box_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* d, int dst_width) { + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + "repl.ph $t3, 3 \n" // 0x00030003 + + ".p2align 2 \n" + "1: \n" + "lw $t0, 0(%[src_ptr]) \n" // |S3|S2|S1|S0| + "lwx $t1, %[src_stride](%[src_ptr]) \n" // |T3|T2|T1|T0| + "rotr $t2, $t0, 8 \n" // |S0|S3|S2|S1| + "rotr $t6, $t1, 8 \n" // |T0|T3|T2|T1| + "muleu_s.ph.qbl $t4, $t2, $t3 \n" // |S0*3|S3*3| + "muleu_s.ph.qbl $t5, $t6, $t3 \n" // |T0*3|T3*3| + "andi $t0, $t2, 0xFFFF \n" // |0|0|S2|S1| + "andi $t1, $t6, 0xFFFF \n" // |0|0|T2|T1| + "raddu.w.qb $t0, $t0 \n" + "raddu.w.qb $t1, $t1 \n" + "shra_r.w $t0, $t0, 1 \n" + "shra_r.w $t1, $t1, 1 \n" + "preceu.ph.qbr $t2, $t2 \n" // |0|S2|0|S1| + "preceu.ph.qbr $t6, $t6 \n" // |0|T2|0|T1| + "rotr $t2, $t2, 16 \n" // |0|S1|0|S2| + "rotr $t6, $t6, 16 \n" // |0|T1|0|T2| + "addu.ph $t2, $t2, $t4 \n" + "addu.ph $t6, $t6, $t5 \n" + "sll $t5, $t0, 1 \n" + "add $t0, $t5, $t0 \n" + "shra_r.ph $t2, $t2, 2 \n" + "shra_r.ph $t6, $t6, 2 \n" + "shll.ph $t4, $t2, 1 \n" + "addq.ph $t4, $t4, $t2 \n" + "addu $t0, $t0, $t1 \n" + "addiu %[src_ptr], %[src_ptr], 4 \n" + "shra_r.w $t0, $t0, 2 \n" + "addu.ph $t6, $t6, $t4 \n" + "shra_r.ph $t6, $t6, 2 \n" + "srl $t1, $t6, 16 \n" + "addiu %[dst_width], %[dst_width], -3 \n" + "sb $t1, 0(%[d]) \n" + "sb $t0, 1(%[d]) \n" + "sb $t6, 2(%[d]) \n" + "bgtz %[dst_width], 1b \n" + " addiu %[d], %[d], 3 \n" + "3: \n" + ".set pop \n" + : [src_ptr] "+r" (src_ptr), + [src_stride] "+r" (src_stride), + [d] "+r" (d), + [dst_width] "+r" (dst_width) + : + : "t0", "t1", "t2", "t3", + "t4", "t5", "t6" + ); +} + +void ScaleRowDown34_1_Box_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* d, int dst_width) { + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + "repl.ph $t2, 3 \n" // 0x00030003 + + ".p2align 2 \n" + "1: \n" + "lw $t0, 0(%[src_ptr]) \n" // |S3|S2|S1|S0| + "lwx $t1, %[src_stride](%[src_ptr]) \n" // |T3|T2|T1|T0| + "rotr $t4, $t0, 8 \n" // |S0|S3|S2|S1| + "rotr $t6, $t1, 8 \n" // |T0|T3|T2|T1| + "muleu_s.ph.qbl $t3, $t4, $t2 \n" // |S0*3|S3*3| + "muleu_s.ph.qbl $t5, $t6, $t2 \n" // |T0*3|T3*3| + "andi $t0, $t4, 0xFFFF \n" // |0|0|S2|S1| + "andi $t1, $t6, 0xFFFF \n" // |0|0|T2|T1| + "raddu.w.qb $t0, $t0 \n" + "raddu.w.qb $t1, $t1 \n" + "shra_r.w $t0, $t0, 1 \n" + "shra_r.w $t1, $t1, 1 \n" + "preceu.ph.qbr $t4, $t4 \n" // |0|S2|0|S1| + "preceu.ph.qbr $t6, $t6 \n" // |0|T2|0|T1| + "rotr $t4, $t4, 16 \n" // |0|S1|0|S2| + "rotr $t6, $t6, 16 \n" // |0|T1|0|T2| + "addu.ph $t4, $t4, $t3 \n" + "addu.ph $t6, $t6, $t5 \n" + "shra_r.ph $t6, $t6, 2 \n" + "shra_r.ph $t4, $t4, 2 \n" + "addu.ph $t6, $t6, $t4 \n" + "addiu %[src_ptr], %[src_ptr], 4 \n" + "shra_r.ph $t6, $t6, 1 \n" + "addu $t0, $t0, $t1 \n" + "addiu %[dst_width], %[dst_width], -3 \n" + "shra_r.w $t0, $t0, 1 \n" + "srl $t1, $t6, 16 \n" + "sb $t1, 0(%[d]) \n" + "sb $t0, 1(%[d]) \n" + "sb $t6, 2(%[d]) \n" + "bgtz %[dst_width], 1b \n" + " addiu %[d], %[d], 3 \n" + "3: \n" + ".set pop \n" + : [src_ptr] "+r" (src_ptr), + [src_stride] "+r" (src_stride), + [d] "+r" (d), + [dst_width] "+r" (dst_width) + : + : "t0", "t1", "t2", "t3", + "t4", "t5", "t6" + ); +} + +void ScaleRowDown38_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + + ".p2align 2 \n" + "1: \n" + "lw $t0, 0(%[src_ptr]) \n" // |3|2|1|0| + "lw $t1, 4(%[src_ptr]) \n" // |7|6|5|4| + "lw $t2, 8(%[src_ptr]) \n" // |11|10|9|8| + "lw $t3, 12(%[src_ptr]) \n" // |15|14|13|12| + "lw $t4, 16(%[src_ptr]) \n" // |19|18|17|16| + "lw $t5, 20(%[src_ptr]) \n" // |23|22|21|20| + "lw $t6, 24(%[src_ptr]) \n" // |27|26|25|24| + "lw $t7, 28(%[src_ptr]) \n" // |31|30|29|28| + "wsbh $t0, $t0 \n" // |2|3|0|1| + "wsbh $t6, $t6 \n" // |26|27|24|25| + "srl $t0, $t0, 8 \n" // |X|2|3|0| + "srl $t3, $t3, 16 \n" // |X|X|15|14| + "srl $t5, $t5, 16 \n" // |X|X|23|22| + "srl $t7, $t7, 16 \n" // |X|X|31|30| + "ins $t1, $t2, 24, 8 \n" // |8|6|5|4| + "ins $t6, $t5, 0, 8 \n" // |26|27|24|22| + "ins $t1, $t0, 0, 16 \n" // |8|6|3|0| + "ins $t6, $t7, 24, 8 \n" // |30|27|24|22| + "prepend $t2, $t3, 24 \n" // |X|15|14|11| + "ins $t4, $t4, 16, 8 \n" // |19|16|17|X| + "ins $t4, $t2, 0, 16 \n" // |19|16|14|11| + "addiu %[src_ptr], %[src_ptr], 32 \n" + "addiu %[dst_width], %[dst_width], -12 \n" + "addiu $t8,%[dst_width], -12 \n" + "sw $t1, 0(%[dst]) \n" + "sw $t4, 4(%[dst]) \n" + "sw $t6, 8(%[dst]) \n" + "bgez $t8, 1b \n" + " addiu %[dst], %[dst], 12 \n" + ".set pop \n" + : [src_ptr] "+r" (src_ptr), + [dst] "+r" (dst), + [dst_width] "+r" (dst_width) + : + : "t0", "t1", "t2", "t3", "t4", + "t5", "t6", "t7", "t8" + ); +} + +void ScaleRowDown38_2_Box_MIPS_DSPR2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + intptr_t stride = src_stride; + const uint8* t = src_ptr + stride; + const int c = 0x2AAA; + + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + + ".p2align 2 \n" + "1: \n" + "lw $t0, 0(%[src_ptr]) \n" // |S3|S2|S1|S0| + "lw $t1, 4(%[src_ptr]) \n" // |S7|S6|S5|S4| + "lw $t2, 0(%[t]) \n" // |T3|T2|T1|T0| + "lw $t3, 4(%[t]) \n" // |T7|T6|T5|T4| + "rotr $t1, $t1, 16 \n" // |S5|S4|S7|S6| + "packrl.ph $t4, $t1, $t3 \n" // |S7|S6|T7|T6| + "packrl.ph $t5, $t3, $t1 \n" // |T5|T4|S5|S4| + "raddu.w.qb $t4, $t4 \n" // S7+S6+T7+T6 + "raddu.w.qb $t5, $t5 \n" // T5+T4+S5+S4 + "precrq.qb.ph $t6, $t0, $t2 \n" // |S3|S1|T3|T1| + "precrq.qb.ph $t6, $t6, $t6 \n" // |S3|T3|S3|T3| + "srl $t4, $t4, 2 \n" // t4 / 4 + "srl $t6, $t6, 16 \n" // |0|0|S3|T3| + "raddu.w.qb $t6, $t6 \n" // 0+0+S3+T3 + "addu $t6, $t5, $t6 \n" + "mul $t6, $t6, %[c] \n" // t6 * 0x2AAA + "sll $t0, $t0, 8 \n" // |S2|S1|S0|0| + "sll $t2, $t2, 8 \n" // |T2|T1|T0|0| + "raddu.w.qb $t0, $t0 \n" // S2+S1+S0+0 + "raddu.w.qb $t2, $t2 \n" // T2+T1+T0+0 + "addu $t0, $t0, $t2 \n" + "mul $t0, $t0, %[c] \n" // t0 * 0x2AAA + "addiu %[src_ptr], %[src_ptr], 8 \n" + "addiu %[t], %[t], 8 \n" + "addiu %[dst_width], %[dst_width], -3 \n" + "addiu %[dst_ptr], %[dst_ptr], 3 \n" + "srl $t6, $t6, 16 \n" + "srl $t0, $t0, 16 \n" + "sb $t4, -1(%[dst_ptr]) \n" + "sb $t6, -2(%[dst_ptr]) \n" + "bgtz %[dst_width], 1b \n" + " sb $t0, -3(%[dst_ptr]) \n" + ".set pop \n" + : [src_ptr] "+r" (src_ptr), + [dst_ptr] "+r" (dst_ptr), + [t] "+r" (t), + [dst_width] "+r" (dst_width) + : [c] "r" (c) + : "t0", "t1", "t2", "t3", "t4", "t5", "t6" + ); +} + +void ScaleRowDown38_3_Box_MIPS_DSPR2(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + intptr_t stride = src_stride; + const uint8* s1 = src_ptr + stride; + stride += stride; + const uint8* s2 = src_ptr + stride; + const int c1 = 0x1C71; + const int c2 = 0x2AAA; + + __asm__ __volatile__ ( + ".set push \n" + ".set noreorder \n" + + ".p2align 2 \n" + "1: \n" + "lw $t0, 0(%[src_ptr]) \n" // |S3|S2|S1|S0| + "lw $t1, 4(%[src_ptr]) \n" // |S7|S6|S5|S4| + "lw $t2, 0(%[s1]) \n" // |T3|T2|T1|T0| + "lw $t3, 4(%[s1]) \n" // |T7|T6|T5|T4| + "lw $t4, 0(%[s2]) \n" // |R3|R2|R1|R0| + "lw $t5, 4(%[s2]) \n" // |R7|R6|R5|R4| + "rotr $t1, $t1, 16 \n" // |S5|S4|S7|S6| + "packrl.ph $t6, $t1, $t3 \n" // |S7|S6|T7|T6| + "raddu.w.qb $t6, $t6 \n" // S7+S6+T7+T6 + "packrl.ph $t7, $t3, $t1 \n" // |T5|T4|S5|S4| + "raddu.w.qb $t7, $t7 \n" // T5+T4+S5+S4 + "sll $t8, $t5, 16 \n" // |R5|R4|0|0| + "raddu.w.qb $t8, $t8 \n" // R5+R4 + "addu $t7, $t7, $t8 \n" + "srl $t8, $t5, 16 \n" // |0|0|R7|R6| + "raddu.w.qb $t8, $t8 \n" // R7 + R6 + "addu $t6, $t6, $t8 \n" + "mul $t6, $t6, %[c2] \n" // t6 * 0x2AAA + "precrq.qb.ph $t8, $t0, $t2 \n" // |S3|S1|T3|T1| + "precrq.qb.ph $t8, $t8, $t4 \n" // |S3|T3|R3|R1| + "srl $t8, $t8, 8 \n" // |0|S3|T3|R3| + "raddu.w.qb $t8, $t8 \n" // S3 + T3 + R3 + "addu $t7, $t7, $t8 \n" + "mul $t7, $t7, %[c1] \n" // t7 * 0x1C71 + "sll $t0, $t0, 8 \n" // |S2|S1|S0|0| + "sll $t2, $t2, 8 \n" // |T2|T1|T0|0| + "sll $t4, $t4, 8 \n" // |R2|R1|R0|0| + "raddu.w.qb $t0, $t0 \n" + "raddu.w.qb $t2, $t2 \n" + "raddu.w.qb $t4, $t4 \n" + "addu $t0, $t0, $t2 \n" + "addu $t0, $t0, $t4 \n" + "mul $t0, $t0, %[c1] \n" // t0 * 0x1C71 + "addiu %[src_ptr], %[src_ptr], 8 \n" + "addiu %[s1], %[s1], 8 \n" + "addiu %[s2], %[s2], 8 \n" + "addiu %[dst_width], %[dst_width], -3 \n" + "addiu %[dst_ptr], %[dst_ptr], 3 \n" + "srl $t6, $t6, 16 \n" + "srl $t7, $t7, 16 \n" + "srl $t0, $t0, 16 \n" + "sb $t6, -1(%[dst_ptr]) \n" + "sb $t7, -2(%[dst_ptr]) \n" + "bgtz %[dst_width], 1b \n" + " sb $t0, -3(%[dst_ptr]) \n" + ".set pop \n" + : [src_ptr] "+r" (src_ptr), + [dst_ptr] "+r" (dst_ptr), + [s1] "+r" (s1), + [s2] "+r" (s2), + [dst_width] "+r" (dst_width) + : [c1] "r" (c1), [c2] "r" (c2) + : "t0", "t1", "t2", "t3", "t4", + "t5", "t6", "t7", "t8" + ); +} + +#endif // defined(__mips_dsp) && (__mips_dsp_rev >= 2) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + diff --git a/third_party/aom/third_party/libyuv/source/scale_neon.cc b/third_party/aom/third_party/libyuv/source/scale_neon.cc new file mode 100644 index 0000000000..7825878e98 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/scale_neon.cc @@ -0,0 +1,1037 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for GCC Neon. +#if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \ + !defined(__aarch64__) + +// NEON downscalers with interpolation. +// Provided by Fritz Koenig + +// Read 32x1 throw away even pixels, and write 16x1. +void ScaleRowDown2_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + // load even pixels into q0, odd into q1 + MEMACCESS(0) + "vld2.8 {q0, q1}, [%0]! \n" + "subs %2, %2, #16 \n" // 16 processed per loop + MEMACCESS(1) + "vst1.8 {q1}, [%1]! \n" // store odd pixels + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst), // %1 + "+r"(dst_width) // %2 + : + : "q0", "q1" // Clobber List + ); +} + +// Read 32x1 average down and write 16x1. +void ScaleRowDown2Linear_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0, q1}, [%0]! \n" // load pixels and post inc + "subs %2, %2, #16 \n" // 16 processed per loop + "vpaddl.u8 q0, q0 \n" // add adjacent + "vpaddl.u8 q1, q1 \n" + "vrshrn.u16 d0, q0, #1 \n" // downshift, round and pack + "vrshrn.u16 d1, q1, #1 \n" + MEMACCESS(1) + "vst1.8 {q0}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst), // %1 + "+r"(dst_width) // %2 + : + : "q0", "q1" // Clobber List + ); +} + +// Read 32x2 average down and write 16x1. +void ScaleRowDown2Box_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + asm volatile ( + // change the stride to row 2 pointer + "add %1, %0 \n" + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0, q1}, [%0]! \n" // load row 1 and post inc + MEMACCESS(1) + "vld1.8 {q2, q3}, [%1]! \n" // load row 2 and post inc + "subs %3, %3, #16 \n" // 16 processed per loop + "vpaddl.u8 q0, q0 \n" // row 1 add adjacent + "vpaddl.u8 q1, q1 \n" + "vpadal.u8 q0, q2 \n" // row 2 add adjacent + row1 + "vpadal.u8 q1, q3 \n" + "vrshrn.u16 d0, q0, #2 \n" // downshift, round and pack + "vrshrn.u16 d1, q1, #2 \n" + MEMACCESS(2) + "vst1.8 {q0}, [%2]! \n" + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(src_stride), // %1 + "+r"(dst), // %2 + "+r"(dst_width) // %3 + : + : "q0", "q1", "q2", "q3" // Clobber List + ); +} + +void ScaleRowDown4_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0 + "subs %2, %2, #8 \n" // 8 processed per loop + MEMACCESS(1) + "vst1.8 {d2}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : + : "q0", "q1", "memory", "cc" + ); +} + +void ScaleRowDown4Box_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + const uint8* src_ptr1 = src_ptr + src_stride; + const uint8* src_ptr2 = src_ptr + src_stride * 2; + const uint8* src_ptr3 = src_ptr + src_stride * 3; +asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {q0}, [%0]! \n" // load up 16x4 + MEMACCESS(3) + "vld1.8 {q1}, [%3]! \n" + MEMACCESS(4) + "vld1.8 {q2}, [%4]! \n" + MEMACCESS(5) + "vld1.8 {q3}, [%5]! \n" + "subs %2, %2, #4 \n" + "vpaddl.u8 q0, q0 \n" + "vpadal.u8 q0, q1 \n" + "vpadal.u8 q0, q2 \n" + "vpadal.u8 q0, q3 \n" + "vpaddl.u16 q0, q0 \n" + "vrshrn.u32 d0, q0, #4 \n" // divide by 16 w/rounding + "vmovn.u16 d0, q0 \n" + MEMACCESS(1) + "vst1.32 {d0[0]}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width), // %2 + "+r"(src_ptr1), // %3 + "+r"(src_ptr2), // %4 + "+r"(src_ptr3) // %5 + : + : "q0", "q1", "q2", "q3", "memory", "cc" + ); +} + +// Down scale from 4 to 3 pixels. Use the neon multilane read/write +// to load up the every 4th pixel into a 4 different registers. +// Point samples 32 pixels to 24 pixels. +void ScaleRowDown34_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0 + "subs %2, %2, #24 \n" + "vmov d2, d3 \n" // order d0, d1, d2 + MEMACCESS(1) + "vst3.8 {d0, d1, d2}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : + : "d0", "d1", "d2", "d3", "memory", "cc" + ); +} + +void ScaleRowDown34_0_Box_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "vmov.u8 d24, #3 \n" + "add %3, %0 \n" + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0 + MEMACCESS(3) + "vld4.8 {d4, d5, d6, d7}, [%3]! \n" // src line 1 + "subs %2, %2, #24 \n" + + // filter src line 0 with src line 1 + // expand chars to shorts to allow for room + // when adding lines together + "vmovl.u8 q8, d4 \n" + "vmovl.u8 q9, d5 \n" + "vmovl.u8 q10, d6 \n" + "vmovl.u8 q11, d7 \n" + + // 3 * line_0 + line_1 + "vmlal.u8 q8, d0, d24 \n" + "vmlal.u8 q9, d1, d24 \n" + "vmlal.u8 q10, d2, d24 \n" + "vmlal.u8 q11, d3, d24 \n" + + // (3 * line_0 + line_1) >> 2 + "vqrshrn.u16 d0, q8, #2 \n" + "vqrshrn.u16 d1, q9, #2 \n" + "vqrshrn.u16 d2, q10, #2 \n" + "vqrshrn.u16 d3, q11, #2 \n" + + // a0 = (src[0] * 3 + s[1] * 1) >> 2 + "vmovl.u8 q8, d1 \n" + "vmlal.u8 q8, d0, d24 \n" + "vqrshrn.u16 d0, q8, #2 \n" + + // a1 = (src[1] * 1 + s[2] * 1) >> 1 + "vrhadd.u8 d1, d1, d2 \n" + + // a2 = (src[2] * 1 + s[3] * 3) >> 2 + "vmovl.u8 q8, d2 \n" + "vmlal.u8 q8, d3, d24 \n" + "vqrshrn.u16 d2, q8, #2 \n" + + MEMACCESS(1) + "vst3.8 {d0, d1, d2}, [%1]! \n" + + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width), // %2 + "+r"(src_stride) // %3 + : + : "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "d24", "memory", "cc" + ); +} + +void ScaleRowDown34_1_Box_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "vmov.u8 d24, #3 \n" + "add %3, %0 \n" + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0 + MEMACCESS(3) + "vld4.8 {d4, d5, d6, d7}, [%3]! \n" // src line 1 + "subs %2, %2, #24 \n" + // average src line 0 with src line 1 + "vrhadd.u8 q0, q0, q2 \n" + "vrhadd.u8 q1, q1, q3 \n" + + // a0 = (src[0] * 3 + s[1] * 1) >> 2 + "vmovl.u8 q3, d1 \n" + "vmlal.u8 q3, d0, d24 \n" + "vqrshrn.u16 d0, q3, #2 \n" + + // a1 = (src[1] * 1 + s[2] * 1) >> 1 + "vrhadd.u8 d1, d1, d2 \n" + + // a2 = (src[2] * 1 + s[3] * 3) >> 2 + "vmovl.u8 q3, d2 \n" + "vmlal.u8 q3, d3, d24 \n" + "vqrshrn.u16 d2, q3, #2 \n" + + MEMACCESS(1) + "vst3.8 {d0, d1, d2}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width), // %2 + "+r"(src_stride) // %3 + : + : "r4", "q0", "q1", "q2", "q3", "d24", "memory", "cc" + ); +} + +#define HAS_SCALEROWDOWN38_NEON +static uvec8 kShuf38 = + { 0, 3, 6, 8, 11, 14, 16, 19, 22, 24, 27, 30, 0, 0, 0, 0 }; +static uvec8 kShuf38_2 = + { 0, 8, 16, 2, 10, 17, 4, 12, 18, 6, 14, 19, 0, 0, 0, 0 }; +static vec16 kMult38_Div6 = + { 65536 / 12, 65536 / 12, 65536 / 12, 65536 / 12, + 65536 / 12, 65536 / 12, 65536 / 12, 65536 / 12 }; +static vec16 kMult38_Div9 = + { 65536 / 18, 65536 / 18, 65536 / 18, 65536 / 18, + 65536 / 18, 65536 / 18, 65536 / 18, 65536 / 18 }; + +// 32 -> 12 +void ScaleRowDown38_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + MEMACCESS(3) + "vld1.8 {q3}, [%3] \n" + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {d0, d1, d2, d3}, [%0]! \n" + "subs %2, %2, #12 \n" + "vtbl.u8 d4, {d0, d1, d2, d3}, d6 \n" + "vtbl.u8 d5, {d0, d1, d2, d3}, d7 \n" + MEMACCESS(1) + "vst1.8 {d4}, [%1]! \n" + MEMACCESS(1) + "vst1.32 {d5[0]}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : "r"(&kShuf38) // %3 + : "d0", "d1", "d2", "d3", "d4", "d5", "memory", "cc" + ); +} + +// 32x3 -> 12x1 +void OMITFP ScaleRowDown38_3_Box_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + const uint8* src_ptr1 = src_ptr + src_stride * 2; + + asm volatile ( + MEMACCESS(5) + "vld1.16 {q13}, [%5] \n" + MEMACCESS(6) + "vld1.8 {q14}, [%6] \n" + MEMACCESS(7) + "vld1.8 {q15}, [%7] \n" + "add %3, %0 \n" + ".p2align 2 \n" + "1: \n" + + // d0 = 00 40 01 41 02 42 03 43 + // d1 = 10 50 11 51 12 52 13 53 + // d2 = 20 60 21 61 22 62 23 63 + // d3 = 30 70 31 71 32 72 33 73 + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" + MEMACCESS(3) + "vld4.8 {d4, d5, d6, d7}, [%3]! \n" + MEMACCESS(4) + "vld4.8 {d16, d17, d18, d19}, [%4]! \n" + "subs %2, %2, #12 \n" + + // Shuffle the input data around to get align the data + // so adjacent data can be added. 0,1 - 2,3 - 4,5 - 6,7 + // d0 = 00 10 01 11 02 12 03 13 + // d1 = 40 50 41 51 42 52 43 53 + "vtrn.u8 d0, d1 \n" + "vtrn.u8 d4, d5 \n" + "vtrn.u8 d16, d17 \n" + + // d2 = 20 30 21 31 22 32 23 33 + // d3 = 60 70 61 71 62 72 63 73 + "vtrn.u8 d2, d3 \n" + "vtrn.u8 d6, d7 \n" + "vtrn.u8 d18, d19 \n" + + // d0 = 00+10 01+11 02+12 03+13 + // d2 = 40+50 41+51 42+52 43+53 + "vpaddl.u8 q0, q0 \n" + "vpaddl.u8 q2, q2 \n" + "vpaddl.u8 q8, q8 \n" + + // d3 = 60+70 61+71 62+72 63+73 + "vpaddl.u8 d3, d3 \n" + "vpaddl.u8 d7, d7 \n" + "vpaddl.u8 d19, d19 \n" + + // combine source lines + "vadd.u16 q0, q2 \n" + "vadd.u16 q0, q8 \n" + "vadd.u16 d4, d3, d7 \n" + "vadd.u16 d4, d19 \n" + + // dst_ptr[3] = (s[6 + st * 0] + s[7 + st * 0] + // + s[6 + st * 1] + s[7 + st * 1] + // + s[6 + st * 2] + s[7 + st * 2]) / 6 + "vqrdmulh.s16 q2, q2, q13 \n" + "vmovn.u16 d4, q2 \n" + + // Shuffle 2,3 reg around so that 2 can be added to the + // 0,1 reg and 3 can be added to the 4,5 reg. This + // requires expanding from u8 to u16 as the 0,1 and 4,5 + // registers are already expanded. Then do transposes + // to get aligned. + // q2 = xx 20 xx 30 xx 21 xx 31 xx 22 xx 32 xx 23 xx 33 + "vmovl.u8 q1, d2 \n" + "vmovl.u8 q3, d6 \n" + "vmovl.u8 q9, d18 \n" + + // combine source lines + "vadd.u16 q1, q3 \n" + "vadd.u16 q1, q9 \n" + + // d4 = xx 20 xx 30 xx 22 xx 32 + // d5 = xx 21 xx 31 xx 23 xx 33 + "vtrn.u32 d2, d3 \n" + + // d4 = xx 20 xx 21 xx 22 xx 23 + // d5 = xx 30 xx 31 xx 32 xx 33 + "vtrn.u16 d2, d3 \n" + + // 0+1+2, 3+4+5 + "vadd.u16 q0, q1 \n" + + // Need to divide, but can't downshift as the the value + // isn't a power of 2. So multiply by 65536 / n + // and take the upper 16 bits. + "vqrdmulh.s16 q0, q0, q15 \n" + + // Align for table lookup, vtbl requires registers to + // be adjacent + "vmov.u8 d2, d4 \n" + + "vtbl.u8 d3, {d0, d1, d2}, d28 \n" + "vtbl.u8 d4, {d0, d1, d2}, d29 \n" + + MEMACCESS(1) + "vst1.8 {d3}, [%1]! \n" + MEMACCESS(1) + "vst1.32 {d4[0]}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width), // %2 + "+r"(src_stride), // %3 + "+r"(src_ptr1) // %4 + : "r"(&kMult38_Div6), // %5 + "r"(&kShuf38_2), // %6 + "r"(&kMult38_Div9) // %7 + : "q0", "q1", "q2", "q3", "q8", "q9", "q13", "q14", "q15", "memory", "cc" + ); +} + +// 32x2 -> 12x1 +void ScaleRowDown38_2_Box_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + MEMACCESS(4) + "vld1.16 {q13}, [%4] \n" + MEMACCESS(5) + "vld1.8 {q14}, [%5] \n" + "add %3, %0 \n" + ".p2align 2 \n" + "1: \n" + + // d0 = 00 40 01 41 02 42 03 43 + // d1 = 10 50 11 51 12 52 13 53 + // d2 = 20 60 21 61 22 62 23 63 + // d3 = 30 70 31 71 32 72 33 73 + MEMACCESS(0) + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" + MEMACCESS(3) + "vld4.8 {d4, d5, d6, d7}, [%3]! \n" + "subs %2, %2, #12 \n" + + // Shuffle the input data around to get align the data + // so adjacent data can be added. 0,1 - 2,3 - 4,5 - 6,7 + // d0 = 00 10 01 11 02 12 03 13 + // d1 = 40 50 41 51 42 52 43 53 + "vtrn.u8 d0, d1 \n" + "vtrn.u8 d4, d5 \n" + + // d2 = 20 30 21 31 22 32 23 33 + // d3 = 60 70 61 71 62 72 63 73 + "vtrn.u8 d2, d3 \n" + "vtrn.u8 d6, d7 \n" + + // d0 = 00+10 01+11 02+12 03+13 + // d2 = 40+50 41+51 42+52 43+53 + "vpaddl.u8 q0, q0 \n" + "vpaddl.u8 q2, q2 \n" + + // d3 = 60+70 61+71 62+72 63+73 + "vpaddl.u8 d3, d3 \n" + "vpaddl.u8 d7, d7 \n" + + // combine source lines + "vadd.u16 q0, q2 \n" + "vadd.u16 d4, d3, d7 \n" + + // dst_ptr[3] = (s[6] + s[7] + s[6+st] + s[7+st]) / 4 + "vqrshrn.u16 d4, q2, #2 \n" + + // Shuffle 2,3 reg around so that 2 can be added to the + // 0,1 reg and 3 can be added to the 4,5 reg. This + // requires expanding from u8 to u16 as the 0,1 and 4,5 + // registers are already expanded. Then do transposes + // to get aligned. + // q2 = xx 20 xx 30 xx 21 xx 31 xx 22 xx 32 xx 23 xx 33 + "vmovl.u8 q1, d2 \n" + "vmovl.u8 q3, d6 \n" + + // combine source lines + "vadd.u16 q1, q3 \n" + + // d4 = xx 20 xx 30 xx 22 xx 32 + // d5 = xx 21 xx 31 xx 23 xx 33 + "vtrn.u32 d2, d3 \n" + + // d4 = xx 20 xx 21 xx 22 xx 23 + // d5 = xx 30 xx 31 xx 32 xx 33 + "vtrn.u16 d2, d3 \n" + + // 0+1+2, 3+4+5 + "vadd.u16 q0, q1 \n" + + // Need to divide, but can't downshift as the the value + // isn't a power of 2. So multiply by 65536 / n + // and take the upper 16 bits. + "vqrdmulh.s16 q0, q0, q13 \n" + + // Align for table lookup, vtbl requires registers to + // be adjacent + "vmov.u8 d2, d4 \n" + + "vtbl.u8 d3, {d0, d1, d2}, d28 \n" + "vtbl.u8 d4, {d0, d1, d2}, d29 \n" + + MEMACCESS(1) + "vst1.8 {d3}, [%1]! \n" + MEMACCESS(1) + "vst1.32 {d4[0]}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width), // %2 + "+r"(src_stride) // %3 + : "r"(&kMult38_Div6), // %4 + "r"(&kShuf38_2) // %5 + : "q0", "q1", "q2", "q3", "q13", "q14", "memory", "cc" + ); +} + +void ScaleAddRows_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint16* dst_ptr, int src_width, int src_height) { + const uint8* src_tmp = NULL; + asm volatile ( + ".p2align 2 \n" + "1: \n" + "mov %0, %1 \n" + "mov r12, %5 \n" + "veor q2, q2, q2 \n" + "veor q3, q3, q3 \n" + "2: \n" + // load 16 pixels into q0 + MEMACCESS(0) + "vld1.8 {q0}, [%0], %3 \n" + "vaddw.u8 q3, q3, d1 \n" + "vaddw.u8 q2, q2, d0 \n" + "subs r12, r12, #1 \n" + "bgt 2b \n" + MEMACCESS(2) + "vst1.16 {q2, q3}, [%2]! \n" // store pixels + "add %1, %1, #16 \n" + "subs %4, %4, #16 \n" // 16 processed per loop + "bgt 1b \n" + : "+r"(src_tmp), // %0 + "+r"(src_ptr), // %1 + "+r"(dst_ptr), // %2 + "+r"(src_stride), // %3 + "+r"(src_width), // %4 + "+r"(src_height) // %5 + : + : "memory", "cc", "r12", "q0", "q1", "q2", "q3" // Clobber List + ); +} + +// TODO(Yang Zhang): Investigate less load instructions for +// the x/dx stepping +#define LOAD2_DATA8_LANE(n) \ + "lsr %5, %3, #16 \n" \ + "add %6, %1, %5 \n" \ + "add %3, %3, %4 \n" \ + MEMACCESS(6) \ + "vld2.8 {d6["#n"], d7["#n"]}, [%6] \n" + +void ScaleFilterCols_NEON(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx) { + int dx_offset[4] = {0, 1, 2, 3}; + int* tmp = dx_offset; + const uint8* src_tmp = src_ptr; + asm volatile ( + ".p2align 2 \n" + "vdup.32 q0, %3 \n" // x + "vdup.32 q1, %4 \n" // dx + "vld1.32 {q2}, [%5] \n" // 0 1 2 3 + "vshl.i32 q3, q1, #2 \n" // 4 * dx + "vmul.s32 q1, q1, q2 \n" + // x , x + 1 * dx, x + 2 * dx, x + 3 * dx + "vadd.s32 q1, q1, q0 \n" + // x + 4 * dx, x + 5 * dx, x + 6 * dx, x + 7 * dx + "vadd.s32 q2, q1, q3 \n" + "vshl.i32 q0, q3, #1 \n" // 8 * dx + "1: \n" + LOAD2_DATA8_LANE(0) + LOAD2_DATA8_LANE(1) + LOAD2_DATA8_LANE(2) + LOAD2_DATA8_LANE(3) + LOAD2_DATA8_LANE(4) + LOAD2_DATA8_LANE(5) + LOAD2_DATA8_LANE(6) + LOAD2_DATA8_LANE(7) + "vmov q10, q1 \n" + "vmov q11, q2 \n" + "vuzp.16 q10, q11 \n" + "vmovl.u8 q8, d6 \n" + "vmovl.u8 q9, d7 \n" + "vsubl.s16 q11, d18, d16 \n" + "vsubl.s16 q12, d19, d17 \n" + "vmovl.u16 q13, d20 \n" + "vmovl.u16 q10, d21 \n" + "vmul.s32 q11, q11, q13 \n" + "vmul.s32 q12, q12, q10 \n" + "vshrn.s32 d18, q11, #16 \n" + "vshrn.s32 d19, q12, #16 \n" + "vadd.s16 q8, q8, q9 \n" + "vmovn.s16 d6, q8 \n" + + MEMACCESS(0) + "vst1.8 {d6}, [%0]! \n" // store pixels + "vadd.s32 q1, q1, q0 \n" + "vadd.s32 q2, q2, q0 \n" + "subs %2, %2, #8 \n" // 8 processed per loop + "bgt 1b \n" + : "+r"(dst_ptr), // %0 + "+r"(src_ptr), // %1 + "+r"(dst_width), // %2 + "+r"(x), // %3 + "+r"(dx), // %4 + "+r"(tmp), // %5 + "+r"(src_tmp) // %6 + : + : "memory", "cc", "q0", "q1", "q2", "q3", + "q8", "q9", "q10", "q11", "q12", "q13" + ); +} + +#undef LOAD2_DATA8_LANE + +// 16x2 -> 16x1 +void ScaleFilterRows_NEON(uint8* dst_ptr, + const uint8* src_ptr, ptrdiff_t src_stride, + int dst_width, int source_y_fraction) { + asm volatile ( + "cmp %4, #0 \n" + "beq 100f \n" + "add %2, %1 \n" + "cmp %4, #64 \n" + "beq 75f \n" + "cmp %4, #128 \n" + "beq 50f \n" + "cmp %4, #192 \n" + "beq 25f \n" + + "vdup.8 d5, %4 \n" + "rsb %4, #256 \n" + "vdup.8 d4, %4 \n" + // General purpose row blend. + "1: \n" + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" + MEMACCESS(2) + "vld1.8 {q1}, [%2]! \n" + "subs %3, %3, #16 \n" + "vmull.u8 q13, d0, d4 \n" + "vmull.u8 q14, d1, d4 \n" + "vmlal.u8 q13, d2, d5 \n" + "vmlal.u8 q14, d3, d5 \n" + "vrshrn.u16 d0, q13, #8 \n" + "vrshrn.u16 d1, q14, #8 \n" + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" + "bgt 1b \n" + "b 99f \n" + + // Blend 25 / 75. + "25: \n" + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" + MEMACCESS(2) + "vld1.8 {q1}, [%2]! \n" + "subs %3, %3, #16 \n" + "vrhadd.u8 q0, q1 \n" + "vrhadd.u8 q0, q1 \n" + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" + "bgt 25b \n" + "b 99f \n" + + // Blend 50 / 50. + "50: \n" + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" + MEMACCESS(2) + "vld1.8 {q1}, [%2]! \n" + "subs %3, %3, #16 \n" + "vrhadd.u8 q0, q1 \n" + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" + "bgt 50b \n" + "b 99f \n" + + // Blend 75 / 25. + "75: \n" + MEMACCESS(1) + "vld1.8 {q1}, [%1]! \n" + MEMACCESS(2) + "vld1.8 {q0}, [%2]! \n" + "subs %3, %3, #16 \n" + "vrhadd.u8 q0, q1 \n" + "vrhadd.u8 q0, q1 \n" + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" + "bgt 75b \n" + "b 99f \n" + + // Blend 100 / 0 - Copy row unchanged. + "100: \n" + MEMACCESS(1) + "vld1.8 {q0}, [%1]! \n" + "subs %3, %3, #16 \n" + MEMACCESS(0) + "vst1.8 {q0}, [%0]! \n" + "bgt 100b \n" + + "99: \n" + MEMACCESS(0) + "vst1.8 {d1[7]}, [%0] \n" + : "+r"(dst_ptr), // %0 + "+r"(src_ptr), // %1 + "+r"(src_stride), // %2 + "+r"(dst_width), // %3 + "+r"(source_y_fraction) // %4 + : + : "q0", "q1", "d4", "d5", "q13", "q14", "memory", "cc" + ); +} + +void ScaleARGBRowDown2_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + // load even pixels into q0, odd into q1 + MEMACCESS(0) + "vld2.32 {q0, q1}, [%0]! \n" + MEMACCESS(0) + "vld2.32 {q2, q3}, [%0]! \n" + "subs %2, %2, #8 \n" // 8 processed per loop + MEMACCESS(1) + "vst1.8 {q1}, [%1]! \n" // store odd pixels + MEMACCESS(1) + "vst1.8 {q3}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst), // %1 + "+r"(dst_width) // %2 + : + : "memory", "cc", "q0", "q1", "q2", "q3" // Clobber List + ); +} + +void ScaleARGBRowDown2Linear_NEON(const uint8* src_argb, ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) { + asm volatile ( + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels. + MEMACCESS(0) + "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels. + "subs %2, %2, #8 \n" // 8 processed per loop + "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts. + "vpaddl.u8 q3, q3 \n" // A 16 bytes -> 8 shorts. + "vrshrn.u16 d0, q0, #1 \n" // downshift, round and pack + "vrshrn.u16 d1, q1, #1 \n" + "vrshrn.u16 d2, q2, #1 \n" + "vrshrn.u16 d3, q3, #1 \n" + MEMACCESS(1) + "vst4.8 {d0, d1, d2, d3}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(dst_width) // %2 + : + : "memory", "cc", "q0", "q1", "q2", "q3" // Clobber List + ); +} + +void ScaleARGBRowDown2Box_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + asm volatile ( + // change the stride to row 2 pointer + "add %1, %1, %0 \n" + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels. + MEMACCESS(0) + "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels. + "subs %3, %3, #8 \n" // 8 processed per loop. + "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts. + "vpaddl.u8 q3, q3 \n" // A 16 bytes -> 8 shorts. + MEMACCESS(1) + "vld4.8 {d16, d18, d20, d22}, [%1]! \n" // load 8 more ARGB pixels. + MEMACCESS(1) + "vld4.8 {d17, d19, d21, d23}, [%1]! \n" // load last 8 ARGB pixels. + "vpadal.u8 q0, q8 \n" // B 16 bytes -> 8 shorts. + "vpadal.u8 q1, q9 \n" // G 16 bytes -> 8 shorts. + "vpadal.u8 q2, q10 \n" // R 16 bytes -> 8 shorts. + "vpadal.u8 q3, q11 \n" // A 16 bytes -> 8 shorts. + "vrshrn.u16 d0, q0, #2 \n" // downshift, round and pack + "vrshrn.u16 d1, q1, #2 \n" + "vrshrn.u16 d2, q2, #2 \n" + "vrshrn.u16 d3, q3, #2 \n" + MEMACCESS(2) + "vst4.8 {d0, d1, d2, d3}, [%2]! \n" + "bgt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(src_stride), // %1 + "+r"(dst), // %2 + "+r"(dst_width) // %3 + : + : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11" + ); +} + +// Reads 4 pixels at a time. +// Alignment requirement: src_argb 4 byte aligned. +void ScaleARGBRowDownEven_NEON(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, uint8* dst_argb, int dst_width) { + asm volatile ( + "mov r12, %3, lsl #2 \n" + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.32 {d0[0]}, [%0], r12 \n" + MEMACCESS(0) + "vld1.32 {d0[1]}, [%0], r12 \n" + MEMACCESS(0) + "vld1.32 {d1[0]}, [%0], r12 \n" + MEMACCESS(0) + "vld1.32 {d1[1]}, [%0], r12 \n" + "subs %2, %2, #4 \n" // 4 pixels per loop. + MEMACCESS(1) + "vst1.8 {q0}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(dst_width) // %2 + : "r"(src_stepx) // %3 + : "memory", "cc", "r12", "q0" + ); +} + +// Reads 4 pixels at a time. +// Alignment requirement: src_argb 4 byte aligned. +void ScaleARGBRowDownEvenBox_NEON(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width) { + asm volatile ( + "mov r12, %4, lsl #2 \n" + "add %1, %1, %0 \n" + ".p2align 2 \n" + "1: \n" + MEMACCESS(0) + "vld1.8 {d0}, [%0], r12 \n" // Read 4 2x2 blocks -> 2x1 + MEMACCESS(1) + "vld1.8 {d1}, [%1], r12 \n" + MEMACCESS(0) + "vld1.8 {d2}, [%0], r12 \n" + MEMACCESS(1) + "vld1.8 {d3}, [%1], r12 \n" + MEMACCESS(0) + "vld1.8 {d4}, [%0], r12 \n" + MEMACCESS(1) + "vld1.8 {d5}, [%1], r12 \n" + MEMACCESS(0) + "vld1.8 {d6}, [%0], r12 \n" + MEMACCESS(1) + "vld1.8 {d7}, [%1], r12 \n" + "vaddl.u8 q0, d0, d1 \n" + "vaddl.u8 q1, d2, d3 \n" + "vaddl.u8 q2, d4, d5 \n" + "vaddl.u8 q3, d6, d7 \n" + "vswp.8 d1, d2 \n" // ab_cd -> ac_bd + "vswp.8 d5, d6 \n" // ef_gh -> eg_fh + "vadd.u16 q0, q0, q1 \n" // (a+b)_(c+d) + "vadd.u16 q2, q2, q3 \n" // (e+f)_(g+h) + "vrshrn.u16 d0, q0, #2 \n" // first 2 pixels. + "vrshrn.u16 d1, q2, #2 \n" // next 2 pixels. + "subs %3, %3, #4 \n" // 4 pixels per loop. + MEMACCESS(2) + "vst1.8 {q0}, [%2]! \n" + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_stride), // %1 + "+r"(dst_argb), // %2 + "+r"(dst_width) // %3 + : "r"(src_stepx) // %4 + : "memory", "cc", "r12", "q0", "q1", "q2", "q3" + ); +} + +// TODO(Yang Zhang): Investigate less load instructions for +// the x/dx stepping +#define LOAD1_DATA32_LANE(dn, n) \ + "lsr %5, %3, #16 \n" \ + "add %6, %1, %5, lsl #2 \n" \ + "add %3, %3, %4 \n" \ + MEMACCESS(6) \ + "vld1.32 {"#dn"["#n"]}, [%6] \n" + +void ScaleARGBCols_NEON(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + int tmp = 0; + const uint8* src_tmp = src_argb; + asm volatile ( + ".p2align 2 \n" + "1: \n" + LOAD1_DATA32_LANE(d0, 0) + LOAD1_DATA32_LANE(d0, 1) + LOAD1_DATA32_LANE(d1, 0) + LOAD1_DATA32_LANE(d1, 1) + LOAD1_DATA32_LANE(d2, 0) + LOAD1_DATA32_LANE(d2, 1) + LOAD1_DATA32_LANE(d3, 0) + LOAD1_DATA32_LANE(d3, 1) + + MEMACCESS(0) + "vst1.32 {q0, q1}, [%0]! \n" // store pixels + "subs %2, %2, #8 \n" // 8 processed per loop + "bgt 1b \n" + : "+r"(dst_argb), // %0 + "+r"(src_argb), // %1 + "+r"(dst_width), // %2 + "+r"(x), // %3 + "+r"(dx), // %4 + "+r"(tmp), // %5 + "+r"(src_tmp) // %6 + : + : "memory", "cc", "q0", "q1" + ); +} + +#undef LOAD1_DATA32_LANE + +// TODO(Yang Zhang): Investigate less load instructions for +// the x/dx stepping +#define LOAD2_DATA32_LANE(dn1, dn2, n) \ + "lsr %5, %3, #16 \n" \ + "add %6, %1, %5, lsl #2 \n" \ + "add %3, %3, %4 \n" \ + MEMACCESS(6) \ + "vld2.32 {"#dn1"["#n"], "#dn2"["#n"]}, [%6] \n" + +void ScaleARGBFilterCols_NEON(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + int dx_offset[4] = {0, 1, 2, 3}; + int* tmp = dx_offset; + const uint8* src_tmp = src_argb; + asm volatile ( + ".p2align 2 \n" + "vdup.32 q0, %3 \n" // x + "vdup.32 q1, %4 \n" // dx + "vld1.32 {q2}, [%5] \n" // 0 1 2 3 + "vshl.i32 q9, q1, #2 \n" // 4 * dx + "vmul.s32 q1, q1, q2 \n" + "vmov.i8 q3, #0x7f \n" // 0x7F + "vmov.i16 q15, #0x7f \n" // 0x7F + // x , x + 1 * dx, x + 2 * dx, x + 3 * dx + "vadd.s32 q8, q1, q0 \n" + "1: \n" + // d0, d1: a + // d2, d3: b + LOAD2_DATA32_LANE(d0, d2, 0) + LOAD2_DATA32_LANE(d0, d2, 1) + LOAD2_DATA32_LANE(d1, d3, 0) + LOAD2_DATA32_LANE(d1, d3, 1) + "vshrn.i32 d22, q8, #9 \n" + "vand.16 d22, d22, d30 \n" + "vdup.8 d24, d22[0] \n" + "vdup.8 d25, d22[2] \n" + "vdup.8 d26, d22[4] \n" + "vdup.8 d27, d22[6] \n" + "vext.8 d4, d24, d25, #4 \n" + "vext.8 d5, d26, d27, #4 \n" // f + "veor.8 q10, q2, q3 \n" // 0x7f ^ f + "vmull.u8 q11, d0, d20 \n" + "vmull.u8 q12, d1, d21 \n" + "vmull.u8 q13, d2, d4 \n" + "vmull.u8 q14, d3, d5 \n" + "vadd.i16 q11, q11, q13 \n" + "vadd.i16 q12, q12, q14 \n" + "vshrn.i16 d0, q11, #7 \n" + "vshrn.i16 d1, q12, #7 \n" + + MEMACCESS(0) + "vst1.32 {d0, d1}, [%0]! \n" // store pixels + "vadd.s32 q8, q8, q9 \n" + "subs %2, %2, #4 \n" // 4 processed per loop + "bgt 1b \n" + : "+r"(dst_argb), // %0 + "+r"(src_argb), // %1 + "+r"(dst_width), // %2 + "+r"(x), // %3 + "+r"(dx), // %4 + "+r"(tmp), // %5 + "+r"(src_tmp) // %6 + : + : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", + "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +#undef LOAD2_DATA32_LANE + +#endif // defined(__ARM_NEON__) && !defined(__aarch64__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/scale_neon64.cc b/third_party/aom/third_party/libyuv/source/scale_neon64.cc new file mode 100644 index 0000000000..1d55193579 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/scale_neon64.cc @@ -0,0 +1,1042 @@ +/* + * Copyright 2014 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/scale.h" +#include "libyuv/row.h" +#include "libyuv/scale_row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for GCC Neon armv8 64 bit. +#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) + +// Read 32x1 throw away even pixels, and write 16x1. +void ScaleRowDown2_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + asm volatile ( + "1: \n" + // load even pixels into v0, odd into v1 + MEMACCESS(0) + "ld2 {v0.16b,v1.16b}, [%0], #32 \n" + "subs %w2, %w2, #16 \n" // 16 processed per loop + MEMACCESS(1) + "st1 {v1.16b}, [%1], #16 \n" // store odd pixels + "b.gt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst), // %1 + "+r"(dst_width) // %2 + : + : "v0", "v1" // Clobber List + ); +} + +// Read 32x1 average down and write 16x1. +void ScaleRowDown2Linear_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b,v1.16b}, [%0], #32 \n" // load pixels and post inc + "subs %w2, %w2, #16 \n" // 16 processed per loop + "uaddlp v0.8h, v0.16b \n" // add adjacent + "uaddlp v1.8h, v1.16b \n" + "rshrn v0.8b, v0.8h, #1 \n" // downshift, round and pack + "rshrn2 v0.16b, v1.8h, #1 \n" + MEMACCESS(1) + "st1 {v0.16b}, [%1], #16 \n" + "b.gt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst), // %1 + "+r"(dst_width) // %2 + : + : "v0", "v1" // Clobber List + ); +} + +// Read 32x2 average down and write 16x1. +void ScaleRowDown2Box_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + asm volatile ( + // change the stride to row 2 pointer + "add %1, %1, %0 \n" + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b,v1.16b}, [%0], #32 \n" // load row 1 and post inc + MEMACCESS(1) + "ld1 {v2.16b, v3.16b}, [%1], #32 \n" // load row 2 and post inc + "subs %w3, %w3, #16 \n" // 16 processed per loop + "uaddlp v0.8h, v0.16b \n" // row 1 add adjacent + "uaddlp v1.8h, v1.16b \n" + "uadalp v0.8h, v2.16b \n" // row 2 add adjacent + row1 + "uadalp v1.8h, v3.16b \n" + "rshrn v0.8b, v0.8h, #2 \n" // downshift, round and pack + "rshrn2 v0.16b, v1.8h, #2 \n" + MEMACCESS(2) + "st1 {v0.16b}, [%2], #16 \n" + "b.gt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(src_stride), // %1 + "+r"(dst), // %2 + "+r"(dst_width) // %3 + : + : "v0", "v1", "v2", "v3" // Clobber List + ); +} + +void ScaleRowDown4_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // src line 0 + "subs %w2, %w2, #8 \n" // 8 processed per loop + MEMACCESS(1) + "st1 {v2.8b}, [%1], #8 \n" + "b.gt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : + : "v0", "v1", "v2", "v3", "memory", "cc" + ); +} + +void ScaleRowDown4Box_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + const uint8* src_ptr1 = src_ptr + src_stride; + const uint8* src_ptr2 = src_ptr + src_stride * 2; + const uint8* src_ptr3 = src_ptr + src_stride * 3; +asm volatile ( + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b}, [%0], #16 \n" // load up 16x4 + MEMACCESS(3) + "ld1 {v1.16b}, [%2], #16 \n" + MEMACCESS(4) + "ld1 {v2.16b}, [%3], #16 \n" + MEMACCESS(5) + "ld1 {v3.16b}, [%4], #16 \n" + "subs %w5, %w5, #4 \n" + "uaddlp v0.8h, v0.16b \n" + "uadalp v0.8h, v1.16b \n" + "uadalp v0.8h, v2.16b \n" + "uadalp v0.8h, v3.16b \n" + "addp v0.8h, v0.8h, v0.8h \n" + "rshrn v0.8b, v0.8h, #4 \n" // divide by 16 w/rounding + MEMACCESS(1) + "st1 {v0.s}[0], [%1], #4 \n" + "b.gt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(src_ptr1), // %2 + "+r"(src_ptr2), // %3 + "+r"(src_ptr3), // %4 + "+r"(dst_width) // %5 + : + : "v0", "v1", "v2", "v3", "memory", "cc" + ); +} + +// Down scale from 4 to 3 pixels. Use the neon multilane read/write +// to load up the every 4th pixel into a 4 different registers. +// Point samples 32 pixels to 24 pixels. +void ScaleRowDown34_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // src line 0 + "subs %w2, %w2, #24 \n" + "orr v2.16b, v3.16b, v3.16b \n" // order v0, v1, v2 + MEMACCESS(1) + "st3 {v0.8b,v1.8b,v2.8b}, [%1], #24 \n" + "b.gt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : + : "v0", "v1", "v2", "v3", "memory", "cc" + ); +} + +void ScaleRowDown34_0_Box_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "movi v20.8b, #3 \n" + "add %3, %3, %0 \n" + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // src line 0 + MEMACCESS(3) + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%3], #32 \n" // src line 1 + "subs %w2, %w2, #24 \n" + + // filter src line 0 with src line 1 + // expand chars to shorts to allow for room + // when adding lines together + "ushll v16.8h, v4.8b, #0 \n" + "ushll v17.8h, v5.8b, #0 \n" + "ushll v18.8h, v6.8b, #0 \n" + "ushll v19.8h, v7.8b, #0 \n" + + // 3 * line_0 + line_1 + "umlal v16.8h, v0.8b, v20.8b \n" + "umlal v17.8h, v1.8b, v20.8b \n" + "umlal v18.8h, v2.8b, v20.8b \n" + "umlal v19.8h, v3.8b, v20.8b \n" + + // (3 * line_0 + line_1) >> 2 + "uqrshrn v0.8b, v16.8h, #2 \n" + "uqrshrn v1.8b, v17.8h, #2 \n" + "uqrshrn v2.8b, v18.8h, #2 \n" + "uqrshrn v3.8b, v19.8h, #2 \n" + + // a0 = (src[0] * 3 + s[1] * 1) >> 2 + "ushll v16.8h, v1.8b, #0 \n" + "umlal v16.8h, v0.8b, v20.8b \n" + "uqrshrn v0.8b, v16.8h, #2 \n" + + // a1 = (src[1] * 1 + s[2] * 1) >> 1 + "urhadd v1.8b, v1.8b, v2.8b \n" + + // a2 = (src[2] * 1 + s[3] * 3) >> 2 + "ushll v16.8h, v2.8b, #0 \n" + "umlal v16.8h, v3.8b, v20.8b \n" + "uqrshrn v2.8b, v16.8h, #2 \n" + + MEMACCESS(1) + "st3 {v0.8b,v1.8b,v2.8b}, [%1], #24 \n" + + "b.gt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width), // %2 + "+r"(src_stride) // %3 + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", + "v20", "memory", "cc" + ); +} + +void ScaleRowDown34_1_Box_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + "movi v20.8b, #3 \n" + "add %3, %3, %0 \n" + "1: \n" + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // src line 0 + MEMACCESS(3) + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%3], #32 \n" // src line 1 + "subs %w2, %w2, #24 \n" + // average src line 0 with src line 1 + "urhadd v0.8b, v0.8b, v4.8b \n" + "urhadd v1.8b, v1.8b, v5.8b \n" + "urhadd v2.8b, v2.8b, v6.8b \n" + "urhadd v3.8b, v3.8b, v7.8b \n" + + // a0 = (src[0] * 3 + s[1] * 1) >> 2 + "ushll v4.8h, v1.8b, #0 \n" + "umlal v4.8h, v0.8b, v20.8b \n" + "uqrshrn v0.8b, v4.8h, #2 \n" + + // a1 = (src[1] * 1 + s[2] * 1) >> 1 + "urhadd v1.8b, v1.8b, v2.8b \n" + + // a2 = (src[2] * 1 + s[3] * 3) >> 2 + "ushll v4.8h, v2.8b, #0 \n" + "umlal v4.8h, v3.8b, v20.8b \n" + "uqrshrn v2.8b, v4.8h, #2 \n" + + MEMACCESS(1) + "st3 {v0.8b,v1.8b,v2.8b}, [%1], #24 \n" + "b.gt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width), // %2 + "+r"(src_stride) // %3 + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", "memory", "cc" + ); +} + +static uvec8 kShuf38 = + { 0, 3, 6, 8, 11, 14, 16, 19, 22, 24, 27, 30, 0, 0, 0, 0 }; +static uvec8 kShuf38_2 = + { 0, 16, 32, 2, 18, 33, 4, 20, 34, 6, 22, 35, 0, 0, 0, 0 }; +static vec16 kMult38_Div6 = + { 65536 / 12, 65536 / 12, 65536 / 12, 65536 / 12, + 65536 / 12, 65536 / 12, 65536 / 12, 65536 / 12 }; +static vec16 kMult38_Div9 = + { 65536 / 18, 65536 / 18, 65536 / 18, 65536 / 18, + 65536 / 18, 65536 / 18, 65536 / 18, 65536 / 18 }; + +// 32 -> 12 +void ScaleRowDown38_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + asm volatile ( + MEMACCESS(3) + "ld1 {v3.16b}, [%3] \n" + "1: \n" + MEMACCESS(0) + "ld1 {v0.16b,v1.16b}, [%0], #32 \n" + "subs %w2, %w2, #12 \n" + "tbl v2.16b, {v0.16b,v1.16b}, v3.16b \n" + MEMACCESS(1) + "st1 {v2.8b}, [%1], #8 \n" + MEMACCESS(1) + "st1 {v2.s}[2], [%1], #4 \n" + "b.gt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : "r"(&kShuf38) // %3 + : "v0", "v1", "v2", "v3", "memory", "cc" + ); +} + +// 32x3 -> 12x1 +void OMITFP ScaleRowDown38_3_Box_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + const uint8* src_ptr1 = src_ptr + src_stride * 2; + ptrdiff_t tmp_src_stride = src_stride; + + asm volatile ( + MEMACCESS(5) + "ld1 {v29.8h}, [%5] \n" + MEMACCESS(6) + "ld1 {v30.16b}, [%6] \n" + MEMACCESS(7) + "ld1 {v31.8h}, [%7] \n" + "add %2, %2, %0 \n" + "1: \n" + + // 00 40 01 41 02 42 03 43 + // 10 50 11 51 12 52 13 53 + // 20 60 21 61 22 62 23 63 + // 30 70 31 71 32 72 33 73 + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" + MEMACCESS(3) + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%2], #32 \n" + MEMACCESS(4) + "ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%3], #32 \n" + "subs %w4, %w4, #12 \n" + + // Shuffle the input data around to get align the data + // so adjacent data can be added. 0,1 - 2,3 - 4,5 - 6,7 + // 00 10 01 11 02 12 03 13 + // 40 50 41 51 42 52 43 53 + "trn1 v20.8b, v0.8b, v1.8b \n" + "trn2 v21.8b, v0.8b, v1.8b \n" + "trn1 v22.8b, v4.8b, v5.8b \n" + "trn2 v23.8b, v4.8b, v5.8b \n" + "trn1 v24.8b, v16.8b, v17.8b \n" + "trn2 v25.8b, v16.8b, v17.8b \n" + + // 20 30 21 31 22 32 23 33 + // 60 70 61 71 62 72 63 73 + "trn1 v0.8b, v2.8b, v3.8b \n" + "trn2 v1.8b, v2.8b, v3.8b \n" + "trn1 v4.8b, v6.8b, v7.8b \n" + "trn2 v5.8b, v6.8b, v7.8b \n" + "trn1 v16.8b, v18.8b, v19.8b \n" + "trn2 v17.8b, v18.8b, v19.8b \n" + + // 00+10 01+11 02+12 03+13 + // 40+50 41+51 42+52 43+53 + "uaddlp v20.4h, v20.8b \n" + "uaddlp v21.4h, v21.8b \n" + "uaddlp v22.4h, v22.8b \n" + "uaddlp v23.4h, v23.8b \n" + "uaddlp v24.4h, v24.8b \n" + "uaddlp v25.4h, v25.8b \n" + + // 60+70 61+71 62+72 63+73 + "uaddlp v1.4h, v1.8b \n" + "uaddlp v5.4h, v5.8b \n" + "uaddlp v17.4h, v17.8b \n" + + // combine source lines + "add v20.4h, v20.4h, v22.4h \n" + "add v21.4h, v21.4h, v23.4h \n" + "add v20.4h, v20.4h, v24.4h \n" + "add v21.4h, v21.4h, v25.4h \n" + "add v2.4h, v1.4h, v5.4h \n" + "add v2.4h, v2.4h, v17.4h \n" + + // dst_ptr[3] = (s[6 + st * 0] + s[7 + st * 0] + // + s[6 + st * 1] + s[7 + st * 1] + // + s[6 + st * 2] + s[7 + st * 2]) / 6 + "sqrdmulh v2.8h, v2.8h, v29.8h \n" + "xtn v2.8b, v2.8h \n" + + // Shuffle 2,3 reg around so that 2 can be added to the + // 0,1 reg and 3 can be added to the 4,5 reg. This + // requires expanding from u8 to u16 as the 0,1 and 4,5 + // registers are already expanded. Then do transposes + // to get aligned. + // xx 20 xx 30 xx 21 xx 31 xx 22 xx 32 xx 23 xx 33 + "ushll v16.8h, v16.8b, #0 \n" + "uaddl v0.8h, v0.8b, v4.8b \n" + + // combine source lines + "add v0.8h, v0.8h, v16.8h \n" + + // xx 20 xx 21 xx 22 xx 23 + // xx 30 xx 31 xx 32 xx 33 + "trn1 v1.8h, v0.8h, v0.8h \n" + "trn2 v4.8h, v0.8h, v0.8h \n" + "xtn v0.4h, v1.4s \n" + "xtn v4.4h, v4.4s \n" + + // 0+1+2, 3+4+5 + "add v20.8h, v20.8h, v0.8h \n" + "add v21.8h, v21.8h, v4.8h \n" + + // Need to divide, but can't downshift as the the value + // isn't a power of 2. So multiply by 65536 / n + // and take the upper 16 bits. + "sqrdmulh v0.8h, v20.8h, v31.8h \n" + "sqrdmulh v1.8h, v21.8h, v31.8h \n" + + // Align for table lookup, vtbl requires registers to + // be adjacent + "tbl v3.16b, {v0.16b, v1.16b, v2.16b}, v30.16b \n" + + MEMACCESS(1) + "st1 {v3.8b}, [%1], #8 \n" + MEMACCESS(1) + "st1 {v3.s}[2], [%1], #4 \n" + "b.gt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(tmp_src_stride), // %2 + "+r"(src_ptr1), // %3 + "+r"(dst_width) // %4 + : "r"(&kMult38_Div6), // %5 + "r"(&kShuf38_2), // %6 + "r"(&kMult38_Div9) // %7 + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", + "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v29", + "v30", "v31", "memory", "cc" + ); +} + +// 32x2 -> 12x1 +void ScaleRowDown38_2_Box_NEON(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + // TODO(fbarchard): use src_stride directly for clang 3.5+. + ptrdiff_t tmp_src_stride = src_stride; + asm volatile ( + MEMACCESS(4) + "ld1 {v30.8h}, [%4] \n" + MEMACCESS(5) + "ld1 {v31.16b}, [%5] \n" + "add %2, %2, %0 \n" + "1: \n" + + // 00 40 01 41 02 42 03 43 + // 10 50 11 51 12 52 13 53 + // 20 60 21 61 22 62 23 63 + // 30 70 31 71 32 72 33 73 + MEMACCESS(0) + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" + MEMACCESS(3) + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%2], #32 \n" + "subs %w3, %w3, #12 \n" + + // Shuffle the input data around to get align the data + // so adjacent data can be added. 0,1 - 2,3 - 4,5 - 6,7 + // 00 10 01 11 02 12 03 13 + // 40 50 41 51 42 52 43 53 + "trn1 v16.8b, v0.8b, v1.8b \n" + "trn2 v17.8b, v0.8b, v1.8b \n" + "trn1 v18.8b, v4.8b, v5.8b \n" + "trn2 v19.8b, v4.8b, v5.8b \n" + + // 20 30 21 31 22 32 23 33 + // 60 70 61 71 62 72 63 73 + "trn1 v0.8b, v2.8b, v3.8b \n" + "trn2 v1.8b, v2.8b, v3.8b \n" + "trn1 v4.8b, v6.8b, v7.8b \n" + "trn2 v5.8b, v6.8b, v7.8b \n" + + // 00+10 01+11 02+12 03+13 + // 40+50 41+51 42+52 43+53 + "uaddlp v16.4h, v16.8b \n" + "uaddlp v17.4h, v17.8b \n" + "uaddlp v18.4h, v18.8b \n" + "uaddlp v19.4h, v19.8b \n" + + // 60+70 61+71 62+72 63+73 + "uaddlp v1.4h, v1.8b \n" + "uaddlp v5.4h, v5.8b \n" + + // combine source lines + "add v16.4h, v16.4h, v18.4h \n" + "add v17.4h, v17.4h, v19.4h \n" + "add v2.4h, v1.4h, v5.4h \n" + + // dst_ptr[3] = (s[6] + s[7] + s[6+st] + s[7+st]) / 4 + "uqrshrn v2.8b, v2.8h, #2 \n" + + // Shuffle 2,3 reg around so that 2 can be added to the + // 0,1 reg and 3 can be added to the 4,5 reg. This + // requires expanding from u8 to u16 as the 0,1 and 4,5 + // registers are already expanded. Then do transposes + // to get aligned. + // xx 20 xx 30 xx 21 xx 31 xx 22 xx 32 xx 23 xx 33 + + // combine source lines + "uaddl v0.8h, v0.8b, v4.8b \n" + + // xx 20 xx 21 xx 22 xx 23 + // xx 30 xx 31 xx 32 xx 33 + "trn1 v1.8h, v0.8h, v0.8h \n" + "trn2 v4.8h, v0.8h, v0.8h \n" + "xtn v0.4h, v1.4s \n" + "xtn v4.4h, v4.4s \n" + + // 0+1+2, 3+4+5 + "add v16.8h, v16.8h, v0.8h \n" + "add v17.8h, v17.8h, v4.8h \n" + + // Need to divide, but can't downshift as the the value + // isn't a power of 2. So multiply by 65536 / n + // and take the upper 16 bits. + "sqrdmulh v0.8h, v16.8h, v30.8h \n" + "sqrdmulh v1.8h, v17.8h, v30.8h \n" + + // Align for table lookup, vtbl requires registers to + // be adjacent + + "tbl v3.16b, {v0.16b, v1.16b, v2.16b}, v31.16b \n" + + MEMACCESS(1) + "st1 {v3.8b}, [%1], #8 \n" + MEMACCESS(1) + "st1 {v3.s}[2], [%1], #4 \n" + "b.gt 1b \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(tmp_src_stride), // %2 + "+r"(dst_width) // %3 + : "r"(&kMult38_Div6), // %4 + "r"(&kShuf38_2) // %5 + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", + "v18", "v19", "v30", "v31", "memory", "cc" + ); +} + +void ScaleAddRows_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint16* dst_ptr, int src_width, int src_height) { + const uint8* src_tmp = NULL; + asm volatile ( + "1: \n" + "mov %0, %1 \n" + "mov w12, %w5 \n" + "eor v2.16b, v2.16b, v2.16b \n" + "eor v3.16b, v3.16b, v3.16b \n" + "2: \n" + // load 16 pixels into q0 + MEMACCESS(0) + "ld1 {v0.16b}, [%0], %3 \n" + "uaddw2 v3.8h, v3.8h, v0.16b \n" + "uaddw v2.8h, v2.8h, v0.8b \n" + "subs w12, w12, #1 \n" + "b.gt 2b \n" + MEMACCESS(2) + "st1 {v2.8h, v3.8h}, [%2], #32 \n" // store pixels + "add %1, %1, #16 \n" + "subs %w4, %w4, #16 \n" // 16 processed per loop + "b.gt 1b \n" + : "+r"(src_tmp), // %0 + "+r"(src_ptr), // %1 + "+r"(dst_ptr), // %2 + "+r"(src_stride), // %3 + "+r"(src_width), // %4 + "+r"(src_height) // %5 + : + : "memory", "cc", "w12", "v0", "v1", "v2", "v3" // Clobber List + ); +} + +// TODO(Yang Zhang): Investigate less load instructions for +// the x/dx stepping +#define LOAD2_DATA8_LANE(n) \ + "lsr %5, %3, #16 \n" \ + "add %6, %1, %5 \n" \ + "add %3, %3, %4 \n" \ + MEMACCESS(6) \ + "ld2 {v4.b, v5.b}["#n"], [%6] \n" + +void ScaleFilterCols_NEON(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx) { + int dx_offset[4] = {0, 1, 2, 3}; + int* tmp = dx_offset; + const uint8* src_tmp = src_ptr; + int64 dst_width64 = (int64) dst_width; // Work around ios 64 bit warning. + int64 x64 = (int64) x; + int64 dx64 = (int64) dx; + asm volatile ( + "dup v0.4s, %w3 \n" // x + "dup v1.4s, %w4 \n" // dx + "ld1 {v2.4s}, [%5] \n" // 0 1 2 3 + "shl v3.4s, v1.4s, #2 \n" // 4 * dx + "mul v1.4s, v1.4s, v2.4s \n" + // x , x + 1 * dx, x + 2 * dx, x + 3 * dx + "add v1.4s, v1.4s, v0.4s \n" + // x + 4 * dx, x + 5 * dx, x + 6 * dx, x + 7 * dx + "add v2.4s, v1.4s, v3.4s \n" + "shl v0.4s, v3.4s, #1 \n" // 8 * dx + "1: \n" + LOAD2_DATA8_LANE(0) + LOAD2_DATA8_LANE(1) + LOAD2_DATA8_LANE(2) + LOAD2_DATA8_LANE(3) + LOAD2_DATA8_LANE(4) + LOAD2_DATA8_LANE(5) + LOAD2_DATA8_LANE(6) + LOAD2_DATA8_LANE(7) + "mov v6.16b, v1.16b \n" + "mov v7.16b, v2.16b \n" + "uzp1 v6.8h, v6.8h, v7.8h \n" + "ushll v4.8h, v4.8b, #0 \n" + "ushll v5.8h, v5.8b, #0 \n" + "ssubl v16.4s, v5.4h, v4.4h \n" + "ssubl2 v17.4s, v5.8h, v4.8h \n" + "ushll v7.4s, v6.4h, #0 \n" + "ushll2 v6.4s, v6.8h, #0 \n" + "mul v16.4s, v16.4s, v7.4s \n" + "mul v17.4s, v17.4s, v6.4s \n" + "shrn v6.4h, v16.4s, #16 \n" + "shrn2 v6.8h, v17.4s, #16 \n" + "add v4.8h, v4.8h, v6.8h \n" + "xtn v4.8b, v4.8h \n" + + MEMACCESS(0) + "st1 {v4.8b}, [%0], #8 \n" // store pixels + "add v1.4s, v1.4s, v0.4s \n" + "add v2.4s, v2.4s, v0.4s \n" + "subs %w2, %w2, #8 \n" // 8 processed per loop + "b.gt 1b \n" + : "+r"(dst_ptr), // %0 + "+r"(src_ptr), // %1 + "+r"(dst_width64), // %2 + "+r"(x64), // %3 + "+r"(dx64), // %4 + "+r"(tmp), // %5 + "+r"(src_tmp) // %6 + : + : "memory", "cc", "v0", "v1", "v2", "v3", + "v4", "v5", "v6", "v7", "v16", "v17" + ); +} + +#undef LOAD2_DATA8_LANE + +// 16x2 -> 16x1 +void ScaleFilterRows_NEON(uint8* dst_ptr, + const uint8* src_ptr, ptrdiff_t src_stride, + int dst_width, int source_y_fraction) { + int y_fraction = 256 - source_y_fraction; + asm volatile ( + "cmp %w4, #0 \n" + "b.eq 100f \n" + "add %2, %2, %1 \n" + "cmp %w4, #64 \n" + "b.eq 75f \n" + "cmp %w4, #128 \n" + "b.eq 50f \n" + "cmp %w4, #192 \n" + "b.eq 25f \n" + + "dup v5.8b, %w4 \n" + "dup v4.8b, %w5 \n" + // General purpose row blend. + "1: \n" + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" + MEMACCESS(2) + "ld1 {v1.16b}, [%2], #16 \n" + "subs %w3, %w3, #16 \n" + "umull v6.8h, v0.8b, v4.8b \n" + "umull2 v7.8h, v0.16b, v4.16b \n" + "umlal v6.8h, v1.8b, v5.8b \n" + "umlal2 v7.8h, v1.16b, v5.16b \n" + "rshrn v0.8b, v6.8h, #8 \n" + "rshrn2 v0.16b, v7.8h, #8 \n" + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" + "b.gt 1b \n" + "b 99f \n" + + // Blend 25 / 75. + "25: \n" + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" + MEMACCESS(2) + "ld1 {v1.16b}, [%2], #16 \n" + "subs %w3, %w3, #16 \n" + "urhadd v0.16b, v0.16b, v1.16b \n" + "urhadd v0.16b, v0.16b, v1.16b \n" + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" + "b.gt 25b \n" + "b 99f \n" + + // Blend 50 / 50. + "50: \n" + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" + MEMACCESS(2) + "ld1 {v1.16b}, [%2], #16 \n" + "subs %w3, %w3, #16 \n" + "urhadd v0.16b, v0.16b, v1.16b \n" + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" + "b.gt 50b \n" + "b 99f \n" + + // Blend 75 / 25. + "75: \n" + MEMACCESS(1) + "ld1 {v1.16b}, [%1], #16 \n" + MEMACCESS(2) + "ld1 {v0.16b}, [%2], #16 \n" + "subs %w3, %w3, #16 \n" + "urhadd v0.16b, v0.16b, v1.16b \n" + "urhadd v0.16b, v0.16b, v1.16b \n" + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" + "b.gt 75b \n" + "b 99f \n" + + // Blend 100 / 0 - Copy row unchanged. + "100: \n" + MEMACCESS(1) + "ld1 {v0.16b}, [%1], #16 \n" + "subs %w3, %w3, #16 \n" + MEMACCESS(0) + "st1 {v0.16b}, [%0], #16 \n" + "b.gt 100b \n" + + "99: \n" + MEMACCESS(0) + "st1 {v0.b}[15], [%0] \n" + : "+r"(dst_ptr), // %0 + "+r"(src_ptr), // %1 + "+r"(src_stride), // %2 + "+r"(dst_width), // %3 + "+r"(source_y_fraction),// %4 + "+r"(y_fraction) // %5 + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory", "cc" + ); +} + +void ScaleARGBRowDown2_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + asm volatile ( + "1: \n" + // load even pixels into q0, odd into q1 + MEMACCESS (0) + "ld2 {v0.4s, v1.4s}, [%0], #32 \n" + MEMACCESS (0) + "ld2 {v2.4s, v3.4s}, [%0], #32 \n" + "subs %w2, %w2, #8 \n" // 8 processed per loop + MEMACCESS (1) + "st1 {v1.16b}, [%1], #16 \n" // store odd pixels + MEMACCESS (1) + "st1 {v3.16b}, [%1], #16 \n" + "b.gt 1b \n" + : "+r" (src_ptr), // %0 + "+r" (dst), // %1 + "+r" (dst_width) // %2 + : + : "memory", "cc", "v0", "v1", "v2", "v3" // Clobber List + ); +} + +void ScaleARGBRowDown2Linear_NEON(const uint8* src_argb, ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) { + asm volatile ( + "1: \n" + MEMACCESS (0) + // load 8 ARGB pixels. + "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" + "subs %w2, %w2, #8 \n" // 8 processed per loop. + "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts. + "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts. + "uaddlp v3.8h, v3.16b \n" // A 16 bytes -> 8 shorts. + "rshrn v0.8b, v0.8h, #1 \n" // downshift, round and pack + "rshrn v1.8b, v1.8h, #1 \n" + "rshrn v2.8b, v2.8h, #1 \n" + "rshrn v3.8b, v3.8h, #1 \n" + MEMACCESS (1) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(dst_width) // %2 + : + : "memory", "cc", "v0", "v1", "v2", "v3" // Clobber List + ); +} + +void ScaleARGBRowDown2Box_NEON(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst, int dst_width) { + asm volatile ( + // change the stride to row 2 pointer + "add %1, %1, %0 \n" + "1: \n" + MEMACCESS (0) + "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 8 ARGB pixels. + "subs %w3, %w3, #8 \n" // 8 processed per loop. + "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts. + "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts. + "uaddlp v3.8h, v3.16b \n" // A 16 bytes -> 8 shorts. + MEMACCESS (1) + "ld4 {v16.16b,v17.16b,v18.16b,v19.16b}, [%1], #64 \n" // load 8 more ARGB pixels. + "uadalp v0.8h, v16.16b \n" // B 16 bytes -> 8 shorts. + "uadalp v1.8h, v17.16b \n" // G 16 bytes -> 8 shorts. + "uadalp v2.8h, v18.16b \n" // R 16 bytes -> 8 shorts. + "uadalp v3.8h, v19.16b \n" // A 16 bytes -> 8 shorts. + "rshrn v0.8b, v0.8h, #2 \n" // downshift, round and pack + "rshrn v1.8b, v1.8h, #2 \n" + "rshrn v2.8b, v2.8h, #2 \n" + "rshrn v3.8b, v3.8h, #2 \n" + MEMACCESS (2) + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" + "b.gt 1b \n" + : "+r" (src_ptr), // %0 + "+r" (src_stride), // %1 + "+r" (dst), // %2 + "+r" (dst_width) // %3 + : + : "memory", "cc", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19" + ); +} + +// Reads 4 pixels at a time. +// Alignment requirement: src_argb 4 byte aligned. +void ScaleARGBRowDownEven_NEON(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, uint8* dst_argb, int dst_width) { + asm volatile ( + "1: \n" + MEMACCESS(0) + "ld1 {v0.s}[0], [%0], %3 \n" + MEMACCESS(0) + "ld1 {v0.s}[1], [%0], %3 \n" + MEMACCESS(0) + "ld1 {v0.s}[2], [%0], %3 \n" + MEMACCESS(0) + "ld1 {v0.s}[3], [%0], %3 \n" + "subs %w2, %w2, #4 \n" // 4 pixels per loop. + MEMACCESS(1) + "st1 {v0.16b}, [%1], #16 \n" + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(dst_width) // %2 + : "r"((int64)(src_stepx * 4)) // %3 + : "memory", "cc", "v0" + ); +} + +// Reads 4 pixels at a time. +// Alignment requirement: src_argb 4 byte aligned. +// TODO(Yang Zhang): Might be worth another optimization pass in future. +// It could be upgraded to 8 pixels at a time to start with. +void ScaleARGBRowDownEvenBox_NEON(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width) { + asm volatile ( + "add %1, %1, %0 \n" + "1: \n" + MEMACCESS(0) + "ld1 {v0.8b}, [%0], %4 \n" // Read 4 2x2 blocks -> 2x1 + MEMACCESS(1) + "ld1 {v1.8b}, [%1], %4 \n" + MEMACCESS(0) + "ld1 {v2.8b}, [%0], %4 \n" + MEMACCESS(1) + "ld1 {v3.8b}, [%1], %4 \n" + MEMACCESS(0) + "ld1 {v4.8b}, [%0], %4 \n" + MEMACCESS(1) + "ld1 {v5.8b}, [%1], %4 \n" + MEMACCESS(0) + "ld1 {v6.8b}, [%0], %4 \n" + MEMACCESS(1) + "ld1 {v7.8b}, [%1], %4 \n" + "uaddl v0.8h, v0.8b, v1.8b \n" + "uaddl v2.8h, v2.8b, v3.8b \n" + "uaddl v4.8h, v4.8b, v5.8b \n" + "uaddl v6.8h, v6.8b, v7.8b \n" + "mov v16.d[1], v0.d[1] \n" // ab_cd -> ac_bd + "mov v0.d[1], v2.d[0] \n" + "mov v2.d[0], v16.d[1] \n" + "mov v16.d[1], v4.d[1] \n" // ef_gh -> eg_fh + "mov v4.d[1], v6.d[0] \n" + "mov v6.d[0], v16.d[1] \n" + "add v0.8h, v0.8h, v2.8h \n" // (a+b)_(c+d) + "add v4.8h, v4.8h, v6.8h \n" // (e+f)_(g+h) + "rshrn v0.8b, v0.8h, #2 \n" // first 2 pixels. + "rshrn2 v0.16b, v4.8h, #2 \n" // next 2 pixels. + "subs %w3, %w3, #4 \n" // 4 pixels per loop. + MEMACCESS(2) + "st1 {v0.16b}, [%2], #16 \n" + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_stride), // %1 + "+r"(dst_argb), // %2 + "+r"(dst_width) // %3 + : "r"((int64)(src_stepx * 4)) // %4 + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16" + ); +} + +// TODO(Yang Zhang): Investigate less load instructions for +// the x/dx stepping +#define LOAD1_DATA32_LANE(vn, n) \ + "lsr %5, %3, #16 \n" \ + "add %6, %1, %5, lsl #2 \n" \ + "add %3, %3, %4 \n" \ + MEMACCESS(6) \ + "ld1 {"#vn".s}["#n"], [%6] \n" + +void ScaleARGBCols_NEON(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + const uint8* src_tmp = src_argb; + int64 dst_width64 = (int64) dst_width; // Work around ios 64 bit warning. + int64 x64 = (int64) x; + int64 dx64 = (int64) dx; + int64 tmp64 = 0; + asm volatile ( + "1: \n" + LOAD1_DATA32_LANE(v0, 0) + LOAD1_DATA32_LANE(v0, 1) + LOAD1_DATA32_LANE(v0, 2) + LOAD1_DATA32_LANE(v0, 3) + LOAD1_DATA32_LANE(v1, 0) + LOAD1_DATA32_LANE(v1, 1) + LOAD1_DATA32_LANE(v1, 2) + LOAD1_DATA32_LANE(v1, 3) + + MEMACCESS(0) + "st1 {v0.4s, v1.4s}, [%0], #32 \n" // store pixels + "subs %w2, %w2, #8 \n" // 8 processed per loop + "b.gt 1b \n" + : "+r"(dst_argb), // %0 + "+r"(src_argb), // %1 + "+r"(dst_width64), // %2 + "+r"(x64), // %3 + "+r"(dx64), // %4 + "+r"(tmp64), // %5 + "+r"(src_tmp) // %6 + : + : "memory", "cc", "v0", "v1" + ); +} + +#undef LOAD1_DATA32_LANE + +// TODO(Yang Zhang): Investigate less load instructions for +// the x/dx stepping +#define LOAD2_DATA32_LANE(vn1, vn2, n) \ + "lsr %5, %3, #16 \n" \ + "add %6, %1, %5, lsl #2 \n" \ + "add %3, %3, %4 \n" \ + MEMACCESS(6) \ + "ld2 {"#vn1".s, "#vn2".s}["#n"], [%6] \n" + +void ScaleARGBFilterCols_NEON(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + int dx_offset[4] = {0, 1, 2, 3}; + int* tmp = dx_offset; + const uint8* src_tmp = src_argb; + int64 dst_width64 = (int64) dst_width; // Work around ios 64 bit warning. + int64 x64 = (int64) x; + int64 dx64 = (int64) dx; + asm volatile ( + "dup v0.4s, %w3 \n" // x + "dup v1.4s, %w4 \n" // dx + "ld1 {v2.4s}, [%5] \n" // 0 1 2 3 + "shl v6.4s, v1.4s, #2 \n" // 4 * dx + "mul v1.4s, v1.4s, v2.4s \n" + "movi v3.16b, #0x7f \n" // 0x7F + "movi v4.8h, #0x7f \n" // 0x7F + // x , x + 1 * dx, x + 2 * dx, x + 3 * dx + "add v5.4s, v1.4s, v0.4s \n" + "1: \n" + // d0, d1: a + // d2, d3: b + LOAD2_DATA32_LANE(v0, v1, 0) + LOAD2_DATA32_LANE(v0, v1, 1) + LOAD2_DATA32_LANE(v0, v1, 2) + LOAD2_DATA32_LANE(v0, v1, 3) + "shrn v2.4h, v5.4s, #9 \n" + "and v2.8b, v2.8b, v4.8b \n" + "dup v16.8b, v2.b[0] \n" + "dup v17.8b, v2.b[2] \n" + "dup v18.8b, v2.b[4] \n" + "dup v19.8b, v2.b[6] \n" + "ext v2.8b, v16.8b, v17.8b, #4 \n" + "ext v17.8b, v18.8b, v19.8b, #4 \n" + "ins v2.d[1], v17.d[0] \n" // f + "eor v7.16b, v2.16b, v3.16b \n" // 0x7f ^ f + "umull v16.8h, v0.8b, v7.8b \n" + "umull2 v17.8h, v0.16b, v7.16b \n" + "umull v18.8h, v1.8b, v2.8b \n" + "umull2 v19.8h, v1.16b, v2.16b \n" + "add v16.8h, v16.8h, v18.8h \n" + "add v17.8h, v17.8h, v19.8h \n" + "shrn v0.8b, v16.8h, #7 \n" + "shrn2 v0.16b, v17.8h, #7 \n" + + MEMACCESS(0) + "st1 {v0.4s}, [%0], #16 \n" // store pixels + "add v5.4s, v5.4s, v6.4s \n" + "subs %w2, %w2, #4 \n" // 4 processed per loop + "b.gt 1b \n" + : "+r"(dst_argb), // %0 + "+r"(src_argb), // %1 + "+r"(dst_width64), // %2 + "+r"(x64), // %3 + "+r"(dx64), // %4 + "+r"(tmp), // %5 + "+r"(src_tmp) // %6 + : + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", + "v6", "v7", "v16", "v17", "v18", "v19" + ); +} + +#undef LOAD2_DATA32_LANE + +#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/scale_win.cc b/third_party/aom/third_party/libyuv/source/scale_win.cc new file mode 100644 index 0000000000..c3896ebad2 --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/scale_win.cc @@ -0,0 +1,1354 @@ +/* + * Copyright 2013 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" +#include "libyuv/scale_row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// This module is for Visual C x86. +#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && \ + defined(_MSC_VER) && !defined(__clang__) + +// Offsets for source bytes 0 to 9 +static uvec8 kShuf0 = + { 0, 1, 3, 4, 5, 7, 8, 9, 128, 128, 128, 128, 128, 128, 128, 128 }; + +// Offsets for source bytes 11 to 20 with 8 subtracted = 3 to 12. +static uvec8 kShuf1 = + { 3, 4, 5, 7, 8, 9, 11, 12, 128, 128, 128, 128, 128, 128, 128, 128 }; + +// Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31. +static uvec8 kShuf2 = + { 5, 7, 8, 9, 11, 12, 13, 15, 128, 128, 128, 128, 128, 128, 128, 128 }; + +// Offsets for source bytes 0 to 10 +static uvec8 kShuf01 = + { 0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10 }; + +// Offsets for source bytes 10 to 21 with 8 subtracted = 3 to 13. +static uvec8 kShuf11 = + { 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10, 10, 11, 12, 13 }; + +// Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31. +static uvec8 kShuf21 = + { 5, 6, 6, 7, 8, 9, 9, 10, 10, 11, 12, 13, 13, 14, 14, 15 }; + +// Coefficients for source bytes 0 to 10 +static uvec8 kMadd01 = + { 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2 }; + +// Coefficients for source bytes 10 to 21 +static uvec8 kMadd11 = + { 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1 }; + +// Coefficients for source bytes 21 to 31 +static uvec8 kMadd21 = + { 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3 }; + +// Coefficients for source bytes 21 to 31 +static vec16 kRound34 = + { 2, 2, 2, 2, 2, 2, 2, 2 }; + +static uvec8 kShuf38a = + { 0, 3, 6, 8, 11, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }; + +static uvec8 kShuf38b = + { 128, 128, 128, 128, 128, 128, 0, 3, 6, 8, 11, 14, 128, 128, 128, 128 }; + +// Arrange words 0,3,6 into 0,1,2 +static uvec8 kShufAc = + { 0, 1, 6, 7, 12, 13, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }; + +// Arrange words 0,3,6 into 3,4,5 +static uvec8 kShufAc3 = + { 128, 128, 128, 128, 128, 128, 0, 1, 6, 7, 12, 13, 128, 128, 128, 128 }; + +// Scaling values for boxes of 3x3 and 2x3 +static uvec16 kScaleAc33 = + { 65536 / 9, 65536 / 9, 65536 / 6, 65536 / 9, 65536 / 9, 65536 / 6, 0, 0 }; + +// Arrange first value for pixels 0,1,2,3,4,5 +static uvec8 kShufAb0 = + { 0, 128, 3, 128, 6, 128, 8, 128, 11, 128, 14, 128, 128, 128, 128, 128 }; + +// Arrange second value for pixels 0,1,2,3,4,5 +static uvec8 kShufAb1 = + { 1, 128, 4, 128, 7, 128, 9, 128, 12, 128, 15, 128, 128, 128, 128, 128 }; + +// Arrange third value for pixels 0,1,2,3,4,5 +static uvec8 kShufAb2 = + { 2, 128, 5, 128, 128, 128, 10, 128, 13, 128, 128, 128, 128, 128, 128, 128 }; + +// Scaling values for boxes of 3x2 and 2x2 +static uvec16 kScaleAb2 = + { 65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3, 65536 / 3, 65536 / 2, 0, 0 }; + +// Reads 32 pixels, throws half away and writes 16 pixels. +__declspec(naked) +void ScaleRowDown2_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + mov eax, [esp + 4] // src_ptr + // src_stride ignored + mov edx, [esp + 12] // dst_ptr + mov ecx, [esp + 16] // dst_width + + wloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + psrlw xmm0, 8 // isolate odd pixels. + psrlw xmm1, 8 + packuswb xmm0, xmm1 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg wloop + + ret + } +} + +// Blends 32x1 rectangle to 16x1. +__declspec(naked) +void ScaleRowDown2Linear_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + mov eax, [esp + 4] // src_ptr + // src_stride + mov edx, [esp + 12] // dst_ptr + mov ecx, [esp + 16] // dst_width + pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff + psrlw xmm5, 8 + + wloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + + movdqa xmm2, xmm0 // average columns (32 to 16 pixels) + psrlw xmm0, 8 + movdqa xmm3, xmm1 + psrlw xmm1, 8 + pand xmm2, xmm5 + pand xmm3, xmm5 + pavgw xmm0, xmm2 + pavgw xmm1, xmm3 + packuswb xmm0, xmm1 + + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg wloop + + ret + } +} + +// Blends 32x2 rectangle to 16x1. +__declspec(naked) +void ScaleRowDown2Box_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_ptr + mov esi, [esp + 4 + 8] // src_stride + mov edx, [esp + 4 + 12] // dst_ptr + mov ecx, [esp + 4 + 16] // dst_width + pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff + psrlw xmm5, 8 + + wloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + esi] + movdqu xmm3, [eax + esi + 16] + lea eax, [eax + 32] + pavgb xmm0, xmm2 // average rows + pavgb xmm1, xmm3 + + movdqa xmm2, xmm0 // average columns (32 to 16 pixels) + psrlw xmm0, 8 + movdqa xmm3, xmm1 + psrlw xmm1, 8 + pand xmm2, xmm5 + pand xmm3, xmm5 + pavgw xmm0, xmm2 + pavgw xmm1, xmm3 + packuswb xmm0, xmm1 + + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg wloop + + pop esi + ret + } +} + +#ifdef HAS_SCALEROWDOWN2_AVX2 +// Reads 64 pixels, throws half away and writes 32 pixels. +__declspec(naked) +void ScaleRowDown2_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + mov eax, [esp + 4] // src_ptr + // src_stride ignored + mov edx, [esp + 12] // dst_ptr + mov ecx, [esp + 16] // dst_width + + wloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + lea eax, [eax + 64] + vpsrlw ymm0, ymm0, 8 // isolate odd pixels. + vpsrlw ymm1, ymm1, 8 + vpackuswb ymm0, ymm0, ymm1 + vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb + vmovdqu [edx], ymm0 + lea edx, [edx + 32] + sub ecx, 32 + jg wloop + + vzeroupper + ret + } +} + +// Blends 64x1 rectangle to 32x1. +__declspec(naked) +void ScaleRowDown2Linear_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + mov eax, [esp + 4] // src_ptr + // src_stride + mov edx, [esp + 12] // dst_ptr + mov ecx, [esp + 16] // dst_width + + vpcmpeqb ymm4, ymm4, ymm4 // '1' constant, 8b + vpsrlw ymm4, ymm4, 15 + vpackuswb ymm4, ymm4, ymm4 + vpxor ymm5, ymm5, ymm5 // constant 0 + + wloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + lea eax, [eax + 64] + + vpmaddubsw ymm0, ymm0, ymm4 // average horizontally + vpmaddubsw ymm1, ymm1, ymm4 + vpavgw ymm0, ymm0, ymm5 // (x + 1) / 2 + vpavgw ymm1, ymm1, ymm5 + vpackuswb ymm0, ymm0, ymm1 + vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb + + vmovdqu [edx], ymm0 + lea edx, [edx + 32] + sub ecx, 32 + jg wloop + + vzeroupper + ret + } +} + +// Blends 64x2 rectangle to 32x1. +__declspec(naked) +void ScaleRowDown2Box_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_ptr + mov esi, [esp + 4 + 8] // src_stride + mov edx, [esp + 4 + 12] // dst_ptr + mov ecx, [esp + 4 + 16] // dst_width + + vpcmpeqb ymm4, ymm4, ymm4 // '1' constant, 8b + vpsrlw ymm4, ymm4, 15 + vpackuswb ymm4, ymm4, ymm4 + vpxor ymm5, ymm5, ymm5 // constant 0 + + wloop: + vmovdqu ymm0, [eax] // average rows + vmovdqu ymm1, [eax + 32] + vpavgb ymm0, ymm0, [eax + esi] + vpavgb ymm1, ymm1, [eax + esi + 32] + lea eax, [eax + 64] + + vpmaddubsw ymm0, ymm0, ymm4 // average horizontally + vpmaddubsw ymm1, ymm1, ymm4 + vpavgw ymm0, ymm0, ymm5 // (x + 1) / 2 + vpavgw ymm1, ymm1, ymm5 + vpackuswb ymm0, ymm0, ymm1 + vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb + + vmovdqu [edx], ymm0 + lea edx, [edx + 32] + sub ecx, 32 + jg wloop + + pop esi + vzeroupper + ret + } +} +#endif // HAS_SCALEROWDOWN2_AVX2 + +// Point samples 32 pixels to 8 pixels. +__declspec(naked) +void ScaleRowDown4_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + mov eax, [esp + 4] // src_ptr + // src_stride ignored + mov edx, [esp + 12] // dst_ptr + mov ecx, [esp + 16] // dst_width + pcmpeqb xmm5, xmm5 // generate mask 0x00ff0000 + psrld xmm5, 24 + pslld xmm5, 16 + + wloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + pand xmm0, xmm5 + pand xmm1, xmm5 + packuswb xmm0, xmm1 + psrlw xmm0, 8 + packuswb xmm0, xmm0 + movq qword ptr [edx], xmm0 + lea edx, [edx + 8] + sub ecx, 8 + jg wloop + + ret + } +} + +// Blends 32x4 rectangle to 8x1. +__declspec(naked) +void ScaleRowDown4Box_SSE2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_ptr + mov esi, [esp + 8 + 8] // src_stride + mov edx, [esp + 8 + 12] // dst_ptr + mov ecx, [esp + 8 + 16] // dst_width + lea edi, [esi + esi * 2] // src_stride * 3 + pcmpeqb xmm7, xmm7 // generate mask 0x00ff00ff + psrlw xmm7, 8 + + wloop: + movdqu xmm0, [eax] // average rows + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + esi] + movdqu xmm3, [eax + esi + 16] + pavgb xmm0, xmm2 + pavgb xmm1, xmm3 + movdqu xmm2, [eax + esi * 2] + movdqu xmm3, [eax + esi * 2 + 16] + movdqu xmm4, [eax + edi] + movdqu xmm5, [eax + edi + 16] + lea eax, [eax + 32] + pavgb xmm2, xmm4 + pavgb xmm3, xmm5 + pavgb xmm0, xmm2 + pavgb xmm1, xmm3 + + movdqa xmm2, xmm0 // average columns (32 to 16 pixels) + psrlw xmm0, 8 + movdqa xmm3, xmm1 + psrlw xmm1, 8 + pand xmm2, xmm7 + pand xmm3, xmm7 + pavgw xmm0, xmm2 + pavgw xmm1, xmm3 + packuswb xmm0, xmm1 + + movdqa xmm2, xmm0 // average columns (16 to 8 pixels) + psrlw xmm0, 8 + pand xmm2, xmm7 + pavgw xmm0, xmm2 + packuswb xmm0, xmm0 + + movq qword ptr [edx], xmm0 + lea edx, [edx + 8] + sub ecx, 8 + jg wloop + + pop edi + pop esi + ret + } +} + +#ifdef HAS_SCALEROWDOWN4_AVX2 +// Point samples 64 pixels to 16 pixels. +__declspec(naked) +void ScaleRowDown4_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + mov eax, [esp + 4] // src_ptr + // src_stride ignored + mov edx, [esp + 12] // dst_ptr + mov ecx, [esp + 16] // dst_width + vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff0000 + vpsrld ymm5, ymm5, 24 + vpslld ymm5, ymm5, 16 + + wloop: + vmovdqu ymm0, [eax] + vmovdqu ymm1, [eax + 32] + lea eax, [eax + 64] + vpand ymm0, ymm0, ymm5 + vpand ymm1, ymm1, ymm5 + vpackuswb ymm0, ymm0, ymm1 + vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb + vpsrlw ymm0, ymm0, 8 + vpackuswb ymm0, ymm0, ymm0 + vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb + vmovdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg wloop + + vzeroupper + ret + } +} + +// Blends 64x4 rectangle to 16x1. +__declspec(naked) +void ScaleRowDown4Box_AVX2(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + push esi + push edi + mov eax, [esp + 8 + 4] // src_ptr + mov esi, [esp + 8 + 8] // src_stride + mov edx, [esp + 8 + 12] // dst_ptr + mov ecx, [esp + 8 + 16] // dst_width + lea edi, [esi + esi * 2] // src_stride * 3 + vpcmpeqb ymm7, ymm7, ymm7 // generate mask 0x00ff00ff + vpsrlw ymm7, ymm7, 8 + + wloop: + vmovdqu ymm0, [eax] // average rows + vmovdqu ymm1, [eax + 32] + vpavgb ymm0, ymm0, [eax + esi] + vpavgb ymm1, ymm1, [eax + esi + 32] + vmovdqu ymm2, [eax + esi * 2] + vmovdqu ymm3, [eax + esi * 2 + 32] + vpavgb ymm2, ymm2, [eax + edi] + vpavgb ymm3, ymm3, [eax + edi + 32] + lea eax, [eax + 64] + vpavgb ymm0, ymm0, ymm2 + vpavgb ymm1, ymm1, ymm3 + + vpand ymm2, ymm0, ymm7 // average columns (64 to 32 pixels) + vpand ymm3, ymm1, ymm7 + vpsrlw ymm0, ymm0, 8 + vpsrlw ymm1, ymm1, 8 + vpavgw ymm0, ymm0, ymm2 + vpavgw ymm1, ymm1, ymm3 + vpackuswb ymm0, ymm0, ymm1 + vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb + + vpand ymm2, ymm0, ymm7 // average columns (32 to 16 pixels) + vpsrlw ymm0, ymm0, 8 + vpavgw ymm0, ymm0, ymm2 + vpackuswb ymm0, ymm0, ymm0 + vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb + + vmovdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 16 + jg wloop + + pop edi + pop esi + vzeroupper + ret + } +} +#endif // HAS_SCALEROWDOWN4_AVX2 + +// Point samples 32 pixels to 24 pixels. +// Produces three 8 byte values. For each 8 bytes, 16 bytes are read. +// Then shuffled to do the scaling. + +__declspec(naked) +void ScaleRowDown34_SSSE3(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + mov eax, [esp + 4] // src_ptr + // src_stride ignored + mov edx, [esp + 12] // dst_ptr + mov ecx, [esp + 16] // dst_width + movdqa xmm3, kShuf0 + movdqa xmm4, kShuf1 + movdqa xmm5, kShuf2 + + wloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + movdqa xmm2, xmm1 + palignr xmm1, xmm0, 8 + pshufb xmm0, xmm3 + pshufb xmm1, xmm4 + pshufb xmm2, xmm5 + movq qword ptr [edx], xmm0 + movq qword ptr [edx + 8], xmm1 + movq qword ptr [edx + 16], xmm2 + lea edx, [edx + 24] + sub ecx, 24 + jg wloop + + ret + } +} + +// Blends 32x2 rectangle to 24x1 +// Produces three 8 byte values. For each 8 bytes, 16 bytes are read. +// Then shuffled to do the scaling. + +// Register usage: +// xmm0 src_row 0 +// xmm1 src_row 1 +// xmm2 shuf 0 +// xmm3 shuf 1 +// xmm4 shuf 2 +// xmm5 madd 0 +// xmm6 madd 1 +// xmm7 kRound34 + +// Note that movdqa+palign may be better than movdqu. +__declspec(naked) +void ScaleRowDown34_1_Box_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_ptr + mov esi, [esp + 4 + 8] // src_stride + mov edx, [esp + 4 + 12] // dst_ptr + mov ecx, [esp + 4 + 16] // dst_width + movdqa xmm2, kShuf01 + movdqa xmm3, kShuf11 + movdqa xmm4, kShuf21 + movdqa xmm5, kMadd01 + movdqa xmm6, kMadd11 + movdqa xmm7, kRound34 + + wloop: + movdqu xmm0, [eax] // pixels 0..7 + movdqu xmm1, [eax + esi] + pavgb xmm0, xmm1 + pshufb xmm0, xmm2 + pmaddubsw xmm0, xmm5 + paddsw xmm0, xmm7 + psrlw xmm0, 2 + packuswb xmm0, xmm0 + movq qword ptr [edx], xmm0 + movdqu xmm0, [eax + 8] // pixels 8..15 + movdqu xmm1, [eax + esi + 8] + pavgb xmm0, xmm1 + pshufb xmm0, xmm3 + pmaddubsw xmm0, xmm6 + paddsw xmm0, xmm7 + psrlw xmm0, 2 + packuswb xmm0, xmm0 + movq qword ptr [edx + 8], xmm0 + movdqu xmm0, [eax + 16] // pixels 16..23 + movdqu xmm1, [eax + esi + 16] + lea eax, [eax + 32] + pavgb xmm0, xmm1 + pshufb xmm0, xmm4 + movdqa xmm1, kMadd21 + pmaddubsw xmm0, xmm1 + paddsw xmm0, xmm7 + psrlw xmm0, 2 + packuswb xmm0, xmm0 + movq qword ptr [edx + 16], xmm0 + lea edx, [edx + 24] + sub ecx, 24 + jg wloop + + pop esi + ret + } +} + +// Note that movdqa+palign may be better than movdqu. +__declspec(naked) +void ScaleRowDown34_0_Box_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_ptr + mov esi, [esp + 4 + 8] // src_stride + mov edx, [esp + 4 + 12] // dst_ptr + mov ecx, [esp + 4 + 16] // dst_width + movdqa xmm2, kShuf01 + movdqa xmm3, kShuf11 + movdqa xmm4, kShuf21 + movdqa xmm5, kMadd01 + movdqa xmm6, kMadd11 + movdqa xmm7, kRound34 + + wloop: + movdqu xmm0, [eax] // pixels 0..7 + movdqu xmm1, [eax + esi] + pavgb xmm1, xmm0 + pavgb xmm0, xmm1 + pshufb xmm0, xmm2 + pmaddubsw xmm0, xmm5 + paddsw xmm0, xmm7 + psrlw xmm0, 2 + packuswb xmm0, xmm0 + movq qword ptr [edx], xmm0 + movdqu xmm0, [eax + 8] // pixels 8..15 + movdqu xmm1, [eax + esi + 8] + pavgb xmm1, xmm0 + pavgb xmm0, xmm1 + pshufb xmm0, xmm3 + pmaddubsw xmm0, xmm6 + paddsw xmm0, xmm7 + psrlw xmm0, 2 + packuswb xmm0, xmm0 + movq qword ptr [edx + 8], xmm0 + movdqu xmm0, [eax + 16] // pixels 16..23 + movdqu xmm1, [eax + esi + 16] + lea eax, [eax + 32] + pavgb xmm1, xmm0 + pavgb xmm0, xmm1 + pshufb xmm0, xmm4 + movdqa xmm1, kMadd21 + pmaddubsw xmm0, xmm1 + paddsw xmm0, xmm7 + psrlw xmm0, 2 + packuswb xmm0, xmm0 + movq qword ptr [edx + 16], xmm0 + lea edx, [edx+24] + sub ecx, 24 + jg wloop + + pop esi + ret + } +} + +// 3/8 point sampler + +// Scale 32 pixels to 12 +__declspec(naked) +void ScaleRowDown38_SSSE3(const uint8* src_ptr, ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + mov eax, [esp + 4] // src_ptr + // src_stride ignored + mov edx, [esp + 12] // dst_ptr + mov ecx, [esp + 16] // dst_width + movdqa xmm4, kShuf38a + movdqa xmm5, kShuf38b + + xloop: + movdqu xmm0, [eax] // 16 pixels -> 0,1,2,3,4,5 + movdqu xmm1, [eax + 16] // 16 pixels -> 6,7,8,9,10,11 + lea eax, [eax + 32] + pshufb xmm0, xmm4 + pshufb xmm1, xmm5 + paddusb xmm0, xmm1 + + movq qword ptr [edx], xmm0 // write 12 pixels + movhlps xmm1, xmm0 + movd [edx + 8], xmm1 + lea edx, [edx + 12] + sub ecx, 12 + jg xloop + + ret + } +} + +// Scale 16x3 pixels to 6x1 with interpolation +__declspec(naked) +void ScaleRowDown38_3_Box_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_ptr + mov esi, [esp + 4 + 8] // src_stride + mov edx, [esp + 4 + 12] // dst_ptr + mov ecx, [esp + 4 + 16] // dst_width + movdqa xmm2, kShufAc + movdqa xmm3, kShufAc3 + movdqa xmm4, kScaleAc33 + pxor xmm5, xmm5 + + xloop: + movdqu xmm0, [eax] // sum up 3 rows into xmm0/1 + movdqu xmm6, [eax + esi] + movhlps xmm1, xmm0 + movhlps xmm7, xmm6 + punpcklbw xmm0, xmm5 + punpcklbw xmm1, xmm5 + punpcklbw xmm6, xmm5 + punpcklbw xmm7, xmm5 + paddusw xmm0, xmm6 + paddusw xmm1, xmm7 + movdqu xmm6, [eax + esi * 2] + lea eax, [eax + 16] + movhlps xmm7, xmm6 + punpcklbw xmm6, xmm5 + punpcklbw xmm7, xmm5 + paddusw xmm0, xmm6 + paddusw xmm1, xmm7 + + movdqa xmm6, xmm0 // 8 pixels -> 0,1,2 of xmm6 + psrldq xmm0, 2 + paddusw xmm6, xmm0 + psrldq xmm0, 2 + paddusw xmm6, xmm0 + pshufb xmm6, xmm2 + + movdqa xmm7, xmm1 // 8 pixels -> 3,4,5 of xmm6 + psrldq xmm1, 2 + paddusw xmm7, xmm1 + psrldq xmm1, 2 + paddusw xmm7, xmm1 + pshufb xmm7, xmm3 + paddusw xmm6, xmm7 + + pmulhuw xmm6, xmm4 // divide by 9,9,6, 9,9,6 + packuswb xmm6, xmm6 + + movd [edx], xmm6 // write 6 pixels + psrlq xmm6, 16 + movd [edx + 2], xmm6 + lea edx, [edx + 6] + sub ecx, 6 + jg xloop + + pop esi + ret + } +} + +// Scale 16x2 pixels to 6x1 with interpolation +__declspec(naked) +void ScaleRowDown38_2_Box_SSSE3(const uint8* src_ptr, + ptrdiff_t src_stride, + uint8* dst_ptr, int dst_width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_ptr + mov esi, [esp + 4 + 8] // src_stride + mov edx, [esp + 4 + 12] // dst_ptr + mov ecx, [esp + 4 + 16] // dst_width + movdqa xmm2, kShufAb0 + movdqa xmm3, kShufAb1 + movdqa xmm4, kShufAb2 + movdqa xmm5, kScaleAb2 + + xloop: + movdqu xmm0, [eax] // average 2 rows into xmm0 + movdqu xmm1, [eax + esi] + lea eax, [eax + 16] + pavgb xmm0, xmm1 + + movdqa xmm1, xmm0 // 16 pixels -> 0,1,2,3,4,5 of xmm1 + pshufb xmm1, xmm2 + movdqa xmm6, xmm0 + pshufb xmm6, xmm3 + paddusw xmm1, xmm6 + pshufb xmm0, xmm4 + paddusw xmm1, xmm0 + + pmulhuw xmm1, xmm5 // divide by 3,3,2, 3,3,2 + packuswb xmm1, xmm1 + + movd [edx], xmm1 // write 6 pixels + psrlq xmm1, 16 + movd [edx + 2], xmm1 + lea edx, [edx + 6] + sub ecx, 6 + jg xloop + + pop esi + ret + } +} + +// Reads 16 bytes and accumulates to 16 shorts at a time. +__declspec(naked) +void ScaleAddRow_SSE2(const uint8* src_ptr, uint16* dst_ptr, int src_width) { + __asm { + mov eax, [esp + 4] // src_ptr + mov edx, [esp + 8] // dst_ptr + mov ecx, [esp + 12] // src_width + pxor xmm5, xmm5 + + // sum rows + xloop: + movdqu xmm3, [eax] // read 16 bytes + lea eax, [eax + 16] + movdqu xmm0, [edx] // read 16 words from destination + movdqu xmm1, [edx + 16] + movdqa xmm2, xmm3 + punpcklbw xmm2, xmm5 + punpckhbw xmm3, xmm5 + paddusw xmm0, xmm2 // sum 16 words + paddusw xmm1, xmm3 + movdqu [edx], xmm0 // write 16 words to destination + movdqu [edx + 16], xmm1 + lea edx, [edx + 32] + sub ecx, 16 + jg xloop + ret + } +} + +#ifdef HAS_SCALEADDROW_AVX2 +// Reads 32 bytes and accumulates to 32 shorts at a time. +__declspec(naked) +void ScaleAddRow_AVX2(const uint8* src_ptr, uint16* dst_ptr, int src_width) { + __asm { + mov eax, [esp + 4] // src_ptr + mov edx, [esp + 8] // dst_ptr + mov ecx, [esp + 12] // src_width + vpxor ymm5, ymm5, ymm5 + + // sum rows + xloop: + vmovdqu ymm3, [eax] // read 32 bytes + lea eax, [eax + 32] + vpermq ymm3, ymm3, 0xd8 // unmutate for vpunpck + vpunpcklbw ymm2, ymm3, ymm5 + vpunpckhbw ymm3, ymm3, ymm5 + vpaddusw ymm0, ymm2, [edx] // sum 16 words + vpaddusw ymm1, ymm3, [edx + 32] + vmovdqu [edx], ymm0 // write 32 words to destination + vmovdqu [edx + 32], ymm1 + lea edx, [edx + 64] + sub ecx, 32 + jg xloop + + vzeroupper + ret + } +} +#endif // HAS_SCALEADDROW_AVX2 + +// Bilinear column filtering. SSSE3 version. +__declspec(naked) +void ScaleFilterCols_SSSE3(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx) { + __asm { + push ebx + push esi + push edi + mov edi, [esp + 12 + 4] // dst_ptr + mov esi, [esp + 12 + 8] // src_ptr + mov ecx, [esp + 12 + 12] // dst_width + movd xmm2, [esp + 12 + 16] // x + movd xmm3, [esp + 12 + 20] // dx + mov eax, 0x04040000 // shuffle to line up fractions with pixel. + movd xmm5, eax + pcmpeqb xmm6, xmm6 // generate 0x007f for inverting fraction. + psrlw xmm6, 9 + pextrw eax, xmm2, 1 // get x0 integer. preroll + sub ecx, 2 + jl xloop29 + + movdqa xmm0, xmm2 // x1 = x0 + dx + paddd xmm0, xmm3 + punpckldq xmm2, xmm0 // x0 x1 + punpckldq xmm3, xmm3 // dx dx + paddd xmm3, xmm3 // dx * 2, dx * 2 + pextrw edx, xmm2, 3 // get x1 integer. preroll + + // 2 Pixel loop. + xloop2: + movdqa xmm1, xmm2 // x0, x1 fractions. + paddd xmm2, xmm3 // x += dx + movzx ebx, word ptr [esi + eax] // 2 source x0 pixels + movd xmm0, ebx + psrlw xmm1, 9 // 7 bit fractions. + movzx ebx, word ptr [esi + edx] // 2 source x1 pixels + movd xmm4, ebx + pshufb xmm1, xmm5 // 0011 + punpcklwd xmm0, xmm4 + pxor xmm1, xmm6 // 0..7f and 7f..0 + pmaddubsw xmm0, xmm1 // 16 bit, 2 pixels. + pextrw eax, xmm2, 1 // get x0 integer. next iteration. + pextrw edx, xmm2, 3 // get x1 integer. next iteration. + psrlw xmm0, 7 // 8.7 fixed point to low 8 bits. + packuswb xmm0, xmm0 // 8 bits, 2 pixels. + movd ebx, xmm0 + mov [edi], bx + lea edi, [edi + 2] + sub ecx, 2 // 2 pixels + jge xloop2 + + xloop29: + + add ecx, 2 - 1 + jl xloop99 + + // 1 pixel remainder + movzx ebx, word ptr [esi + eax] // 2 source x0 pixels + movd xmm0, ebx + psrlw xmm2, 9 // 7 bit fractions. + pshufb xmm2, xmm5 // 0011 + pxor xmm2, xmm6 // 0..7f and 7f..0 + pmaddubsw xmm0, xmm2 // 16 bit + psrlw xmm0, 7 // 8.7 fixed point to low 8 bits. + packuswb xmm0, xmm0 // 8 bits + movd ebx, xmm0 + mov [edi], bl + + xloop99: + + pop edi + pop esi + pop ebx + ret + } +} + +// Reads 16 pixels, duplicates them and writes 32 pixels. +__declspec(naked) +void ScaleColsUp2_SSE2(uint8* dst_ptr, const uint8* src_ptr, + int dst_width, int x, int dx) { + __asm { + mov edx, [esp + 4] // dst_ptr + mov eax, [esp + 8] // src_ptr + mov ecx, [esp + 12] // dst_width + + wloop: + movdqu xmm0, [eax] + lea eax, [eax + 16] + movdqa xmm1, xmm0 + punpcklbw xmm0, xmm0 + punpckhbw xmm1, xmm1 + movdqu [edx], xmm0 + movdqu [edx + 16], xmm1 + lea edx, [edx + 32] + sub ecx, 32 + jg wloop + + ret + } +} + +// Reads 8 pixels, throws half away and writes 4 even pixels (0, 2, 4, 6) +__declspec(naked) +void ScaleARGBRowDown2_SSE2(const uint8* src_argb, + ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) { + __asm { + mov eax, [esp + 4] // src_argb + // src_stride ignored + mov edx, [esp + 12] // dst_argb + mov ecx, [esp + 16] // dst_width + + wloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + shufps xmm0, xmm1, 0xdd + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg wloop + + ret + } +} + +// Blends 8x1 rectangle to 4x1. +__declspec(naked) +void ScaleARGBRowDown2Linear_SSE2(const uint8* src_argb, + ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) { + __asm { + mov eax, [esp + 4] // src_argb + // src_stride ignored + mov edx, [esp + 12] // dst_argb + mov ecx, [esp + 16] // dst_width + + wloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + lea eax, [eax + 32] + movdqa xmm2, xmm0 + shufps xmm0, xmm1, 0x88 // even pixels + shufps xmm2, xmm1, 0xdd // odd pixels + pavgb xmm0, xmm2 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg wloop + + ret + } +} + +// Blends 8x2 rectangle to 4x1. +__declspec(naked) +void ScaleARGBRowDown2Box_SSE2(const uint8* src_argb, + ptrdiff_t src_stride, + uint8* dst_argb, int dst_width) { + __asm { + push esi + mov eax, [esp + 4 + 4] // src_argb + mov esi, [esp + 4 + 8] // src_stride + mov edx, [esp + 4 + 12] // dst_argb + mov ecx, [esp + 4 + 16] // dst_width + + wloop: + movdqu xmm0, [eax] + movdqu xmm1, [eax + 16] + movdqu xmm2, [eax + esi] + movdqu xmm3, [eax + esi + 16] + lea eax, [eax + 32] + pavgb xmm0, xmm2 // average rows + pavgb xmm1, xmm3 + movdqa xmm2, xmm0 // average columns (8 to 4 pixels) + shufps xmm0, xmm1, 0x88 // even pixels + shufps xmm2, xmm1, 0xdd // odd pixels + pavgb xmm0, xmm2 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg wloop + + pop esi + ret + } +} + +// Reads 4 pixels at a time. +__declspec(naked) +void ScaleARGBRowDownEven_SSE2(const uint8* src_argb, ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width) { + __asm { + push ebx + push edi + mov eax, [esp + 8 + 4] // src_argb + // src_stride ignored + mov ebx, [esp + 8 + 12] // src_stepx + mov edx, [esp + 8 + 16] // dst_argb + mov ecx, [esp + 8 + 20] // dst_width + lea ebx, [ebx * 4] + lea edi, [ebx + ebx * 2] + + wloop: + movd xmm0, [eax] + movd xmm1, [eax + ebx] + punpckldq xmm0, xmm1 + movd xmm2, [eax + ebx * 2] + movd xmm3, [eax + edi] + lea eax, [eax + ebx * 4] + punpckldq xmm2, xmm3 + punpcklqdq xmm0, xmm2 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg wloop + + pop edi + pop ebx + ret + } +} + +// Blends four 2x2 to 4x1. +__declspec(naked) +void ScaleARGBRowDownEvenBox_SSE2(const uint8* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8* dst_argb, int dst_width) { + __asm { + push ebx + push esi + push edi + mov eax, [esp + 12 + 4] // src_argb + mov esi, [esp + 12 + 8] // src_stride + mov ebx, [esp + 12 + 12] // src_stepx + mov edx, [esp + 12 + 16] // dst_argb + mov ecx, [esp + 12 + 20] // dst_width + lea esi, [eax + esi] // row1 pointer + lea ebx, [ebx * 4] + lea edi, [ebx + ebx * 2] + + wloop: + movq xmm0, qword ptr [eax] // row0 4 pairs + movhps xmm0, qword ptr [eax + ebx] + movq xmm1, qword ptr [eax + ebx * 2] + movhps xmm1, qword ptr [eax + edi] + lea eax, [eax + ebx * 4] + movq xmm2, qword ptr [esi] // row1 4 pairs + movhps xmm2, qword ptr [esi + ebx] + movq xmm3, qword ptr [esi + ebx * 2] + movhps xmm3, qword ptr [esi + edi] + lea esi, [esi + ebx * 4] + pavgb xmm0, xmm2 // average rows + pavgb xmm1, xmm3 + movdqa xmm2, xmm0 // average columns (8 to 4 pixels) + shufps xmm0, xmm1, 0x88 // even pixels + shufps xmm2, xmm1, 0xdd // odd pixels + pavgb xmm0, xmm2 + movdqu [edx], xmm0 + lea edx, [edx + 16] + sub ecx, 4 + jg wloop + + pop edi + pop esi + pop ebx + ret + } +} + +// Column scaling unfiltered. SSE2 version. +__declspec(naked) +void ScaleARGBCols_SSE2(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + __asm { + push edi + push esi + mov edi, [esp + 8 + 4] // dst_argb + mov esi, [esp + 8 + 8] // src_argb + mov ecx, [esp + 8 + 12] // dst_width + movd xmm2, [esp + 8 + 16] // x + movd xmm3, [esp + 8 + 20] // dx + + pshufd xmm2, xmm2, 0 // x0 x0 x0 x0 + pshufd xmm0, xmm3, 0x11 // dx 0 dx 0 + paddd xmm2, xmm0 + paddd xmm3, xmm3 // 0, 0, 0, dx * 2 + pshufd xmm0, xmm3, 0x05 // dx * 2, dx * 2, 0, 0 + paddd xmm2, xmm0 // x3 x2 x1 x0 + paddd xmm3, xmm3 // 0, 0, 0, dx * 4 + pshufd xmm3, xmm3, 0 // dx * 4, dx * 4, dx * 4, dx * 4 + + pextrw eax, xmm2, 1 // get x0 integer. + pextrw edx, xmm2, 3 // get x1 integer. + + cmp ecx, 0 + jle xloop99 + sub ecx, 4 + jl xloop49 + + // 4 Pixel loop. + xloop4: + movd xmm0, [esi + eax * 4] // 1 source x0 pixels + movd xmm1, [esi + edx * 4] // 1 source x1 pixels + pextrw eax, xmm2, 5 // get x2 integer. + pextrw edx, xmm2, 7 // get x3 integer. + paddd xmm2, xmm3 // x += dx + punpckldq xmm0, xmm1 // x0 x1 + + movd xmm1, [esi + eax * 4] // 1 source x2 pixels + movd xmm4, [esi + edx * 4] // 1 source x3 pixels + pextrw eax, xmm2, 1 // get x0 integer. next iteration. + pextrw edx, xmm2, 3 // get x1 integer. next iteration. + punpckldq xmm1, xmm4 // x2 x3 + punpcklqdq xmm0, xmm1 // x0 x1 x2 x3 + movdqu [edi], xmm0 + lea edi, [edi + 16] + sub ecx, 4 // 4 pixels + jge xloop4 + + xloop49: + test ecx, 2 + je xloop29 + + // 2 Pixels. + movd xmm0, [esi + eax * 4] // 1 source x0 pixels + movd xmm1, [esi + edx * 4] // 1 source x1 pixels + pextrw eax, xmm2, 5 // get x2 integer. + punpckldq xmm0, xmm1 // x0 x1 + + movq qword ptr [edi], xmm0 + lea edi, [edi + 8] + + xloop29: + test ecx, 1 + je xloop99 + + // 1 Pixels. + movd xmm0, [esi + eax * 4] // 1 source x2 pixels + movd dword ptr [edi], xmm0 + xloop99: + + pop esi + pop edi + ret + } +} + +// Bilinear row filtering combines 2x1 -> 1x1. SSSE3 version. +// TODO(fbarchard): Port to Neon + +// Shuffle table for arranging 2 pixels into pairs for pmaddubsw +static uvec8 kShuffleColARGB = { + 0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u, // bbggrraa 1st pixel + 8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u // bbggrraa 2nd pixel +}; + +// Shuffle table for duplicating 2 fractions into 8 bytes each +static uvec8 kShuffleFractions = { + 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, +}; + +__declspec(naked) +void ScaleARGBFilterCols_SSSE3(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + __asm { + push esi + push edi + mov edi, [esp + 8 + 4] // dst_argb + mov esi, [esp + 8 + 8] // src_argb + mov ecx, [esp + 8 + 12] // dst_width + movd xmm2, [esp + 8 + 16] // x + movd xmm3, [esp + 8 + 20] // dx + movdqa xmm4, kShuffleColARGB + movdqa xmm5, kShuffleFractions + pcmpeqb xmm6, xmm6 // generate 0x007f for inverting fraction. + psrlw xmm6, 9 + pextrw eax, xmm2, 1 // get x0 integer. preroll + sub ecx, 2 + jl xloop29 + + movdqa xmm0, xmm2 // x1 = x0 + dx + paddd xmm0, xmm3 + punpckldq xmm2, xmm0 // x0 x1 + punpckldq xmm3, xmm3 // dx dx + paddd xmm3, xmm3 // dx * 2, dx * 2 + pextrw edx, xmm2, 3 // get x1 integer. preroll + + // 2 Pixel loop. + xloop2: + movdqa xmm1, xmm2 // x0, x1 fractions. + paddd xmm2, xmm3 // x += dx + movq xmm0, qword ptr [esi + eax * 4] // 2 source x0 pixels + psrlw xmm1, 9 // 7 bit fractions. + movhps xmm0, qword ptr [esi + edx * 4] // 2 source x1 pixels + pshufb xmm1, xmm5 // 0000000011111111 + pshufb xmm0, xmm4 // arrange pixels into pairs + pxor xmm1, xmm6 // 0..7f and 7f..0 + pmaddubsw xmm0, xmm1 // argb_argb 16 bit, 2 pixels. + pextrw eax, xmm2, 1 // get x0 integer. next iteration. + pextrw edx, xmm2, 3 // get x1 integer. next iteration. + psrlw xmm0, 7 // argb 8.7 fixed point to low 8 bits. + packuswb xmm0, xmm0 // argb_argb 8 bits, 2 pixels. + movq qword ptr [edi], xmm0 + lea edi, [edi + 8] + sub ecx, 2 // 2 pixels + jge xloop2 + + xloop29: + + add ecx, 2 - 1 + jl xloop99 + + // 1 pixel remainder + psrlw xmm2, 9 // 7 bit fractions. + movq xmm0, qword ptr [esi + eax * 4] // 2 source x0 pixels + pshufb xmm2, xmm5 // 00000000 + pshufb xmm0, xmm4 // arrange pixels into pairs + pxor xmm2, xmm6 // 0..7f and 7f..0 + pmaddubsw xmm0, xmm2 // argb 16 bit, 1 pixel. + psrlw xmm0, 7 + packuswb xmm0, xmm0 // argb 8 bits, 1 pixel. + movd [edi], xmm0 + + xloop99: + + pop edi + pop esi + ret + } +} + +// Reads 4 pixels, duplicates them and writes 8 pixels. +__declspec(naked) +void ScaleARGBColsUp2_SSE2(uint8* dst_argb, const uint8* src_argb, + int dst_width, int x, int dx) { + __asm { + mov edx, [esp + 4] // dst_argb + mov eax, [esp + 8] // src_argb + mov ecx, [esp + 12] // dst_width + + wloop: + movdqu xmm0, [eax] + lea eax, [eax + 16] + movdqa xmm1, xmm0 + punpckldq xmm0, xmm0 + punpckhdq xmm1, xmm1 + movdqu [edx], xmm0 + movdqu [edx + 16], xmm1 + lea edx, [edx + 32] + sub ecx, 8 + jg wloop + + ret + } +} + +// Divide num by div and return as 16.16 fixed point result. +__declspec(naked) +int FixedDiv_X86(int num, int div) { + __asm { + mov eax, [esp + 4] // num + cdq // extend num to 64 bits + shld edx, eax, 16 // 32.16 + shl eax, 16 + idiv dword ptr [esp + 8] + ret + } +} + +// Divide num by div and return as 16.16 fixed point result. +__declspec(naked) +int FixedDiv1_X86(int num, int div) { + __asm { + mov eax, [esp + 4] // num + mov ecx, [esp + 8] // denom + cdq // extend num to 64 bits + shld edx, eax, 16 // 32.16 + shl eax, 16 + sub eax, 0x00010001 + sbb edx, 0 + sub ecx, 1 + idiv ecx + ret + } +} +#endif // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/third_party/aom/third_party/libyuv/source/video_common.cc b/third_party/aom/third_party/libyuv/source/video_common.cc new file mode 100644 index 0000000000..379a0669ae --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/video_common.cc @@ -0,0 +1,64 @@ +/* + * Copyright 2011 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +#include "libyuv/video_common.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#define ARRAY_SIZE(x) (int)(sizeof(x) / sizeof(x[0])) + +struct FourCCAliasEntry { + uint32 alias; + uint32 canonical; +}; + +static const struct FourCCAliasEntry kFourCCAliases[] = { + {FOURCC_IYUV, FOURCC_I420}, + {FOURCC_YU16, FOURCC_I422}, + {FOURCC_YU24, FOURCC_I444}, + {FOURCC_YUYV, FOURCC_YUY2}, + {FOURCC_YUVS, FOURCC_YUY2}, // kCMPixelFormat_422YpCbCr8_yuvs + {FOURCC_HDYC, FOURCC_UYVY}, + {FOURCC_2VUY, FOURCC_UYVY}, // kCMPixelFormat_422YpCbCr8 + {FOURCC_JPEG, FOURCC_MJPG}, // Note: JPEG has DHT while MJPG does not. + {FOURCC_DMB1, FOURCC_MJPG}, + {FOURCC_BA81, FOURCC_BGGR}, // deprecated. + {FOURCC_RGB3, FOURCC_RAW }, + {FOURCC_BGR3, FOURCC_24BG}, + {FOURCC_CM32, FOURCC_BGRA}, // kCMPixelFormat_32ARGB + {FOURCC_CM24, FOURCC_RAW }, // kCMPixelFormat_24RGB + {FOURCC_L555, FOURCC_RGBO}, // kCMPixelFormat_16LE555 + {FOURCC_L565, FOURCC_RGBP}, // kCMPixelFormat_16LE565 + {FOURCC_5551, FOURCC_RGBO}, // kCMPixelFormat_16LE5551 +}; +// TODO(fbarchard): Consider mapping kCMPixelFormat_32BGRA to FOURCC_ARGB. +// {FOURCC_BGRA, FOURCC_ARGB}, // kCMPixelFormat_32BGRA + +LIBYUV_API +uint32 CanonicalFourCC(uint32 fourcc) { + int i; + for (i = 0; i < ARRAY_SIZE(kFourCCAliases); ++i) { + if (kFourCCAliases[i].alias == fourcc) { + return kFourCCAliases[i].canonical; + } + } + // Not an alias, so return it as-is. + return fourcc; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + diff --git a/third_party/aom/third_party/libyuv/source/x86inc.asm b/third_party/aom/third_party/libyuv/source/x86inc.asm new file mode 100644 index 0000000000..cb5c32df3a --- /dev/null +++ b/third_party/aom/third_party/libyuv/source/x86inc.asm @@ -0,0 +1,1136 @@ +;***************************************************************************** +;* x86inc.asm: x264asm abstraction layer +;***************************************************************************** +;* Copyright (C) 2005-2012 x264 project +;* +;* Authors: Loren Merritt +;* Anton Mitrofanov +;* Jason Garrett-Glaser +;* Henrik Gramner +;* +;* Permission to use, copy, modify, and/or distribute this software for any +;* purpose with or without fee is hereby granted, provided that the above +;* copyright notice and this permission notice appear in all copies. +;* +;* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +;* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +;* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +;* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +;* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +;* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +;***************************************************************************** + +; This is a header file for the x264ASM assembly language, which uses +; NASM/YASM syntax combined with a large number of macros to provide easy +; abstraction between different calling conventions (x86_32, win64, linux64). +; It also has various other useful features to simplify writing the kind of +; DSP functions that are most often used in x264. + +; Unlike the rest of x264, this file is available under an ISC license, as it +; has significant usefulness outside of x264 and we want it to be available +; to the largest audience possible. Of course, if you modify it for your own +; purposes to add a new feature, we strongly encourage contributing a patch +; as this feature might be useful for others as well. Send patches or ideas +; to x264-devel@videolan.org . + +; Local changes for libyuv: +; remove %define program_name and references in labels +; rename cpus to uppercase + +%define WIN64 0 +%define UNIX64 0 +%if ARCH_X86_64 + %ifidn __OUTPUT_FORMAT__,win32 + %define WIN64 1 + %elifidn __OUTPUT_FORMAT__,win64 + %define WIN64 1 + %else + %define UNIX64 1 + %endif +%endif + +%ifdef PREFIX + %define mangle(x) _ %+ x +%else + %define mangle(x) x +%endif + +; Name of the .rodata section. +; Kludge: Something on OS X fails to align .rodata even given an align attribute, +; so use a different read-only section. +%macro SECTION_RODATA 0-1 16 + %ifidn __OUTPUT_FORMAT__,macho64 + SECTION .text align=%1 + %elifidn __OUTPUT_FORMAT__,macho + SECTION .text align=%1 + fakegot: + %elifidn __OUTPUT_FORMAT__,aout + section .text + %else + SECTION .rodata align=%1 + %endif +%endmacro + +; aout does not support align= +%macro SECTION_TEXT 0-1 16 + %ifidn __OUTPUT_FORMAT__,aout + SECTION .text + %else + SECTION .text align=%1 + %endif +%endmacro + +%if WIN64 + %define PIC +%elif ARCH_X86_64 == 0 +; x86_32 doesn't require PIC. +; Some distros prefer shared objects to be PIC, but nothing breaks if +; the code contains a few textrels, so we'll skip that complexity. + %undef PIC +%endif +%ifdef PIC + default rel +%endif + +; Always use long nops (reduces 0x90 spam in disassembly on x86_32) +CPU amdnop + +; Macros to eliminate most code duplication between x86_32 and x86_64: +; Currently this works only for leaf functions which load all their arguments +; into registers at the start, and make no other use of the stack. Luckily that +; covers most of x264's asm. + +; PROLOGUE: +; %1 = number of arguments. loads them from stack if needed. +; %2 = number of registers used. pushes callee-saved regs if needed. +; %3 = number of xmm registers used. pushes callee-saved xmm regs if needed. +; %4 = list of names to define to registers +; PROLOGUE can also be invoked by adding the same options to cglobal + +; e.g. +; cglobal foo, 2,3,0, dst, src, tmp +; declares a function (foo), taking two args (dst and src) and one local variable (tmp) + +; TODO Some functions can use some args directly from the stack. If they're the +; last args then you can just not declare them, but if they're in the middle +; we need more flexible macro. + +; RET: +; Pops anything that was pushed by PROLOGUE, and returns. + +; REP_RET: +; Same, but if it doesn't pop anything it becomes a 2-byte ret, for athlons +; which are slow when a normal ret follows a branch. + +; registers: +; rN and rNq are the native-size register holding function argument N +; rNd, rNw, rNb are dword, word, and byte size +; rNh is the high 8 bits of the word size +; rNm is the original location of arg N (a register or on the stack), dword +; rNmp is native size + +%macro DECLARE_REG 2-3 + %define r%1q %2 + %define r%1d %2d + %define r%1w %2w + %define r%1b %2b + %define r%1h %2h + %if %0 == 2 + %define r%1m %2d + %define r%1mp %2 + %elif ARCH_X86_64 ; memory + %define r%1m [rsp + stack_offset + %3] + %define r%1mp qword r %+ %1m + %else + %define r%1m [esp + stack_offset + %3] + %define r%1mp dword r %+ %1m + %endif + %define r%1 %2 +%endmacro + +%macro DECLARE_REG_SIZE 3 + %define r%1q r%1 + %define e%1q r%1 + %define r%1d e%1 + %define e%1d e%1 + %define r%1w %1 + %define e%1w %1 + %define r%1h %3 + %define e%1h %3 + %define r%1b %2 + %define e%1b %2 +%if ARCH_X86_64 == 0 + %define r%1 e%1 +%endif +%endmacro + +DECLARE_REG_SIZE ax, al, ah +DECLARE_REG_SIZE bx, bl, bh +DECLARE_REG_SIZE cx, cl, ch +DECLARE_REG_SIZE dx, dl, dh +DECLARE_REG_SIZE si, sil, null +DECLARE_REG_SIZE di, dil, null +DECLARE_REG_SIZE bp, bpl, null + +; t# defines for when per-arch register allocation is more complex than just function arguments + +%macro DECLARE_REG_TMP 1-* + %assign %%i 0 + %rep %0 + CAT_XDEFINE t, %%i, r%1 + %assign %%i %%i+1 + %rotate 1 + %endrep +%endmacro + +%macro DECLARE_REG_TMP_SIZE 0-* + %rep %0 + %define t%1q t%1 %+ q + %define t%1d t%1 %+ d + %define t%1w t%1 %+ w + %define t%1h t%1 %+ h + %define t%1b t%1 %+ b + %rotate 1 + %endrep +%endmacro + +DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 + +%if ARCH_X86_64 + %define gprsize 8 +%else + %define gprsize 4 +%endif + +%macro PUSH 1 + push %1 + %assign stack_offset stack_offset+gprsize +%endmacro + +%macro POP 1 + pop %1 + %assign stack_offset stack_offset-gprsize +%endmacro + +%macro PUSH_IF_USED 1-* + %rep %0 + %if %1 < regs_used + PUSH r%1 + %endif + %rotate 1 + %endrep +%endmacro + +%macro POP_IF_USED 1-* + %rep %0 + %if %1 < regs_used + pop r%1 + %endif + %rotate 1 + %endrep +%endmacro + +%macro LOAD_IF_USED 1-* + %rep %0 + %if %1 < num_args + mov r%1, r %+ %1 %+ mp + %endif + %rotate 1 + %endrep +%endmacro + +%macro SUB 2 + sub %1, %2 + %ifidn %1, rsp + %assign stack_offset stack_offset+(%2) + %endif +%endmacro + +%macro ADD 2 + add %1, %2 + %ifidn %1, rsp + %assign stack_offset stack_offset-(%2) + %endif +%endmacro + +%macro movifnidn 2 + %ifnidn %1, %2 + mov %1, %2 + %endif +%endmacro + +%macro movsxdifnidn 2 + %ifnidn %1, %2 + movsxd %1, %2 + %endif +%endmacro + +%macro ASSERT 1 + %if (%1) == 0 + %error assert failed + %endif +%endmacro + +%macro DEFINE_ARGS 0-* + %ifdef n_arg_names + %assign %%i 0 + %rep n_arg_names + CAT_UNDEF arg_name %+ %%i, q + CAT_UNDEF arg_name %+ %%i, d + CAT_UNDEF arg_name %+ %%i, w + CAT_UNDEF arg_name %+ %%i, h + CAT_UNDEF arg_name %+ %%i, b + CAT_UNDEF arg_name %+ %%i, m + CAT_UNDEF arg_name %+ %%i, mp + CAT_UNDEF arg_name, %%i + %assign %%i %%i+1 + %endrep + %endif + + %xdefine %%stack_offset stack_offset + %undef stack_offset ; so that the current value of stack_offset doesn't get baked in by xdefine + %assign %%i 0 + %rep %0 + %xdefine %1q r %+ %%i %+ q + %xdefine %1d r %+ %%i %+ d + %xdefine %1w r %+ %%i %+ w + %xdefine %1h r %+ %%i %+ h + %xdefine %1b r %+ %%i %+ b + %xdefine %1m r %+ %%i %+ m + %xdefine %1mp r %+ %%i %+ mp + CAT_XDEFINE arg_name, %%i, %1 + %assign %%i %%i+1 + %rotate 1 + %endrep + %xdefine stack_offset %%stack_offset + %assign n_arg_names %0 +%endmacro + +%if WIN64 ; Windows x64 ;================================================= + +DECLARE_REG 0, rcx +DECLARE_REG 1, rdx +DECLARE_REG 2, R8 +DECLARE_REG 3, R9 +DECLARE_REG 4, R10, 40 +DECLARE_REG 5, R11, 48 +DECLARE_REG 6, rax, 56 +DECLARE_REG 7, rdi, 64 +DECLARE_REG 8, rsi, 72 +DECLARE_REG 9, rbx, 80 +DECLARE_REG 10, rbp, 88 +DECLARE_REG 11, R12, 96 +DECLARE_REG 12, R13, 104 +DECLARE_REG 13, R14, 112 +DECLARE_REG 14, R15, 120 + +%macro PROLOGUE 2-4+ 0 ; #args, #regs, #xmm_regs, arg_names... + %assign num_args %1 + %assign regs_used %2 + ASSERT regs_used >= num_args + ASSERT regs_used <= 15 + PUSH_IF_USED 7, 8, 9, 10, 11, 12, 13, 14 + %if mmsize == 8 + %assign xmm_regs_used 0 + %else + WIN64_SPILL_XMM %3 + %endif + LOAD_IF_USED 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 + DEFINE_ARGS %4 +%endmacro + +%macro WIN64_SPILL_XMM 1 + %assign xmm_regs_used %1 + ASSERT xmm_regs_used <= 16 + %if xmm_regs_used > 6 + SUB rsp, (xmm_regs_used-6)*16+16 + %assign %%i xmm_regs_used + %rep (xmm_regs_used-6) + %assign %%i %%i-1 + movdqa [rsp + (%%i-6)*16+(~stack_offset&8)], xmm %+ %%i + %endrep + %endif +%endmacro + +%macro WIN64_RESTORE_XMM_INTERNAL 1 + %if xmm_regs_used > 6 + %assign %%i xmm_regs_used + %rep (xmm_regs_used-6) + %assign %%i %%i-1 + movdqa xmm %+ %%i, [%1 + (%%i-6)*16+(~stack_offset&8)] + %endrep + add %1, (xmm_regs_used-6)*16+16 + %endif +%endmacro + +%macro WIN64_RESTORE_XMM 1 + WIN64_RESTORE_XMM_INTERNAL %1 + %assign stack_offset stack_offset-(xmm_regs_used-6)*16+16 + %assign xmm_regs_used 0 +%endmacro + +%define has_epilogue regs_used > 7 || xmm_regs_used > 6 || mmsize == 32 + +%macro RET 0 + WIN64_RESTORE_XMM_INTERNAL rsp + POP_IF_USED 14, 13, 12, 11, 10, 9, 8, 7 +%if mmsize == 32 + vzeroupper +%endif + ret +%endmacro + +%elif ARCH_X86_64 ; *nix x64 ;============================================= + +DECLARE_REG 0, rdi +DECLARE_REG 1, rsi +DECLARE_REG 2, rdx +DECLARE_REG 3, rcx +DECLARE_REG 4, R8 +DECLARE_REG 5, R9 +DECLARE_REG 6, rax, 8 +DECLARE_REG 7, R10, 16 +DECLARE_REG 8, R11, 24 +DECLARE_REG 9, rbx, 32 +DECLARE_REG 10, rbp, 40 +DECLARE_REG 11, R12, 48 +DECLARE_REG 12, R13, 56 +DECLARE_REG 13, R14, 64 +DECLARE_REG 14, R15, 72 + +%macro PROLOGUE 2-4+ ; #args, #regs, #xmm_regs, arg_names... + %assign num_args %1 + %assign regs_used %2 + ASSERT regs_used >= num_args + ASSERT regs_used <= 15 + PUSH_IF_USED 9, 10, 11, 12, 13, 14 + LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14 + DEFINE_ARGS %4 +%endmacro + +%define has_epilogue regs_used > 9 || mmsize == 32 + +%macro RET 0 + POP_IF_USED 14, 13, 12, 11, 10, 9 +%if mmsize == 32 + vzeroupper +%endif + ret +%endmacro + +%else ; X86_32 ;============================================================== + +DECLARE_REG 0, eax, 4 +DECLARE_REG 1, ecx, 8 +DECLARE_REG 2, edx, 12 +DECLARE_REG 3, ebx, 16 +DECLARE_REG 4, esi, 20 +DECLARE_REG 5, edi, 24 +DECLARE_REG 6, ebp, 28 +%define rsp esp + +%macro DECLARE_ARG 1-* + %rep %0 + %define r%1m [esp + stack_offset + 4*%1 + 4] + %define r%1mp dword r%1m + %rotate 1 + %endrep +%endmacro + +DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 + +%macro PROLOGUE 2-4+ ; #args, #regs, #xmm_regs, arg_names... + %assign num_args %1 + %assign regs_used %2 + %if regs_used > 7 + %assign regs_used 7 + %endif + ASSERT regs_used >= num_args + PUSH_IF_USED 3, 4, 5, 6 + LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6 + DEFINE_ARGS %4 +%endmacro + +%define has_epilogue regs_used > 3 || mmsize == 32 + +%macro RET 0 + POP_IF_USED 6, 5, 4, 3 +%if mmsize == 32 + vzeroupper +%endif + ret +%endmacro + +%endif ;====================================================================== + +%if WIN64 == 0 +%macro WIN64_SPILL_XMM 1 +%endmacro +%macro WIN64_RESTORE_XMM 1 +%endmacro +%endif + +%macro REP_RET 0 + %if has_epilogue + RET + %else + rep ret + %endif +%endmacro + +%macro TAIL_CALL 2 ; callee, is_nonadjacent + %if has_epilogue + call %1 + RET + %elif %2 + jmp %1 + %endif +%endmacro + +;============================================================================= +; arch-independent part +;============================================================================= + +%assign function_align 16 + +; Begin a function. +; Applies any symbol mangling needed for C linkage, and sets up a define such that +; subsequent uses of the function name automatically refer to the mangled version. +; Appends cpuflags to the function name if cpuflags has been specified. +%macro cglobal 1-2+ ; name, [PROLOGUE args] +%if %0 == 1 + cglobal_internal %1 %+ SUFFIX +%else + cglobal_internal %1 %+ SUFFIX, %2 +%endif +%endmacro +%macro cglobal_internal 1-2+ + %ifndef cglobaled_%1 + %xdefine %1 mangle(%1) + %xdefine %1.skip_prologue %1 %+ .skip_prologue + CAT_XDEFINE cglobaled_, %1, 1 + %endif + %xdefine current_function %1 + %ifidn __OUTPUT_FORMAT__,elf + global %1:function hidden + %else + global %1 + %endif + align function_align + %1: + RESET_MM_PERMUTATION ; not really needed, but makes disassembly somewhat nicer + %assign stack_offset 0 + %if %0 > 1 + PROLOGUE %2 + %endif +%endmacro + +%macro cextern 1 + %xdefine %1 mangle(%1) + CAT_XDEFINE cglobaled_, %1, 1 + extern %1 +%endmacro + +; like cextern, but without the prefix +%macro cextern_naked 1 + %xdefine %1 mangle(%1) + CAT_XDEFINE cglobaled_, %1, 1 + extern %1 +%endmacro + +%macro const 2+ + %xdefine %1 mangle(%1) + global %1 + %1: %2 +%endmacro + +; This is needed for ELF, otherwise the GNU linker assumes the stack is +; executable by default. +%ifidn __OUTPUT_FORMAT__,elf +SECTION .note.GNU-stack noalloc noexec nowrite progbits +%endif +%ifidn __OUTPUT_FORMAT__,elf32 +section .note.GNU-stack noalloc noexec nowrite progbits +%endif +%ifidn __OUTPUT_FORMAT__,elf64 +section .note.GNU-stack noalloc noexec nowrite progbits +%endif + +; cpuflags + +%assign cpuflags_MMX (1<<0) +%assign cpuflags_MMX2 (1<<1) | cpuflags_MMX +%assign cpuflags_3dnow (1<<2) | cpuflags_MMX +%assign cpuflags_3dnow2 (1<<3) | cpuflags_3dnow +%assign cpuflags_SSE (1<<4) | cpuflags_MMX2 +%assign cpuflags_SSE2 (1<<5) | cpuflags_SSE +%assign cpuflags_SSE2slow (1<<6) | cpuflags_SSE2 +%assign cpuflags_SSE3 (1<<7) | cpuflags_SSE2 +%assign cpuflags_SSSE3 (1<<8) | cpuflags_SSE3 +%assign cpuflags_SSE4 (1<<9) | cpuflags_SSSE3 +%assign cpuflags_SSE42 (1<<10)| cpuflags_SSE4 +%assign cpuflags_AVX (1<<11)| cpuflags_SSE42 +%assign cpuflags_xop (1<<12)| cpuflags_AVX +%assign cpuflags_fma4 (1<<13)| cpuflags_AVX +%assign cpuflags_AVX2 (1<<14)| cpuflags_AVX +%assign cpuflags_fma3 (1<<15)| cpuflags_AVX + +%assign cpuflags_cache32 (1<<16) +%assign cpuflags_cache64 (1<<17) +%assign cpuflags_slowctz (1<<18) +%assign cpuflags_lzcnt (1<<19) +%assign cpuflags_misalign (1<<20) +%assign cpuflags_aligned (1<<21) ; not a cpu feature, but a function variant +%assign cpuflags_atom (1<<22) +%assign cpuflags_bmi1 (1<<23) +%assign cpuflags_bmi2 (1<<24)|cpuflags_bmi1 +%assign cpuflags_tbm (1<<25)|cpuflags_bmi1 + +%define cpuflag(x) ((cpuflags & (cpuflags_ %+ x)) == (cpuflags_ %+ x)) +%define notcpuflag(x) ((cpuflags & (cpuflags_ %+ x)) != (cpuflags_ %+ x)) + +; Takes up to 2 cpuflags from the above list. +; All subsequent functions (up to the next INIT_CPUFLAGS) is built for the specified cpu. +; You shouldn't need to invoke this macro directly, it's a subroutine for INIT_MMX &co. +%macro INIT_CPUFLAGS 0-2 + %if %0 >= 1 + %xdefine cpuname %1 + %assign cpuflags cpuflags_%1 + %if %0 >= 2 + %xdefine cpuname %1_%2 + %assign cpuflags cpuflags | cpuflags_%2 + %endif + %xdefine SUFFIX _ %+ cpuname + %if cpuflag(AVX) + %assign AVX_enabled 1 + %endif + %if mmsize == 16 && notcpuflag(SSE2) + %define mova movaps + %define movu movups + %define movnta movntps + %endif + %if cpuflag(aligned) + %define movu mova + %elifidn %1, SSE3 + %define movu lddqu + %endif + %else + %xdefine SUFFIX + %undef cpuname + %undef cpuflags + %endif +%endmacro + +; merge MMX and SSE* + +%macro CAT_XDEFINE 3 + %xdefine %1%2 %3 +%endmacro + +%macro CAT_UNDEF 2 + %undef %1%2 +%endmacro + +%macro INIT_MMX 0-1+ + %assign AVX_enabled 0 + %define RESET_MM_PERMUTATION INIT_MMX %1 + %define mmsize 8 + %define num_mmregs 8 + %define mova movq + %define movu movq + %define movh movd + %define movnta movntq + %assign %%i 0 + %rep 8 + CAT_XDEFINE m, %%i, mm %+ %%i + CAT_XDEFINE nmm, %%i, %%i + %assign %%i %%i+1 + %endrep + %rep 8 + CAT_UNDEF m, %%i + CAT_UNDEF nmm, %%i + %assign %%i %%i+1 + %endrep + INIT_CPUFLAGS %1 +%endmacro + +%macro INIT_XMM 0-1+ + %assign AVX_enabled 0 + %define RESET_MM_PERMUTATION INIT_XMM %1 + %define mmsize 16 + %define num_mmregs 8 + %if ARCH_X86_64 + %define num_mmregs 16 + %endif + %define mova movdqa + %define movu movdqu + %define movh movq + %define movnta movntdq + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE m, %%i, xmm %+ %%i + CAT_XDEFINE nxmm, %%i, %%i + %assign %%i %%i+1 + %endrep + INIT_CPUFLAGS %1 +%endmacro + +%macro INIT_YMM 0-1+ + %assign AVX_enabled 1 + %define RESET_MM_PERMUTATION INIT_YMM %1 + %define mmsize 32 + %define num_mmregs 8 + %if ARCH_X86_64 + %define num_mmregs 16 + %endif + %define mova vmovaps + %define movu vmovups + %undef movh + %define movnta vmovntps + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE m, %%i, ymm %+ %%i + CAT_XDEFINE nymm, %%i, %%i + %assign %%i %%i+1 + %endrep + INIT_CPUFLAGS %1 +%endmacro + +INIT_XMM + +; I often want to use macros that permute their arguments. e.g. there's no +; efficient way to implement butterfly or transpose or dct without swapping some +; arguments. +; +; I would like to not have to manually keep track of the permutations: +; If I insert a permutation in the middle of a function, it should automatically +; change everything that follows. For more complex macros I may also have multiple +; implementations, e.g. the SSE2 and SSSE3 versions may have different permutations. +; +; Hence these macros. Insert a PERMUTE or some SWAPs at the end of a macro that +; permutes its arguments. It's equivalent to exchanging the contents of the +; registers, except that this way you exchange the register names instead, so it +; doesn't cost any cycles. + +%macro PERMUTE 2-* ; takes a list of pairs to swap +%rep %0/2 + %xdefine tmp%2 m%2 + %xdefine ntmp%2 nm%2 + %rotate 2 +%endrep +%rep %0/2 + %xdefine m%1 tmp%2 + %xdefine nm%1 ntmp%2 + %undef tmp%2 + %undef ntmp%2 + %rotate 2 +%endrep +%endmacro + +%macro SWAP 2-* ; swaps a single chain (sometimes more concise than pairs) +%rep %0-1 +%ifdef m%1 + %xdefine tmp m%1 + %xdefine m%1 m%2 + %xdefine m%2 tmp + CAT_XDEFINE n, m%1, %1 + CAT_XDEFINE n, m%2, %2 +%else + ; If we were called as "SWAP m0,m1" rather than "SWAP 0,1" infer the original numbers here. + ; Be careful using this mode in nested macros though, as in some cases there may be + ; other copies of m# that have already been dereferenced and don't get updated correctly. + %xdefine %%n1 n %+ %1 + %xdefine %%n2 n %+ %2 + %xdefine tmp m %+ %%n1 + CAT_XDEFINE m, %%n1, m %+ %%n2 + CAT_XDEFINE m, %%n2, tmp + CAT_XDEFINE n, m %+ %%n1, %%n1 + CAT_XDEFINE n, m %+ %%n2, %%n2 +%endif + %undef tmp + %rotate 1 +%endrep +%endmacro + +; If SAVE_MM_PERMUTATION is placed at the end of a function, then any later +; calls to that function will automatically load the permutation, so values can +; be returned in mmregs. +%macro SAVE_MM_PERMUTATION 0-1 + %if %0 + %xdefine %%f %1_m + %else + %xdefine %%f current_function %+ _m + %endif + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE %%f, %%i, m %+ %%i + %assign %%i %%i+1 + %endrep +%endmacro + +%macro LOAD_MM_PERMUTATION 1 ; name to load from + %ifdef %1_m0 + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE m, %%i, %1_m %+ %%i + CAT_XDEFINE n, m %+ %%i, %%i + %assign %%i %%i+1 + %endrep + %endif +%endmacro + +; Append cpuflags to the callee's name iff the appended name is known and the plain name isn't +%macro call 1 + call_internal %1, %1 %+ SUFFIX +%endmacro +%macro call_internal 2 + %xdefine %%i %1 + %ifndef cglobaled_%1 + %ifdef cglobaled_%2 + %xdefine %%i %2 + %endif + %endif + call %%i + LOAD_MM_PERMUTATION %%i +%endmacro + +; Substitutions that reduce instruction size but are functionally equivalent +%macro add 2 + %ifnum %2 + %if %2==128 + sub %1, -128 + %else + add %1, %2 + %endif + %else + add %1, %2 + %endif +%endmacro + +%macro sub 2 + %ifnum %2 + %if %2==128 + add %1, -128 + %else + sub %1, %2 + %endif + %else + sub %1, %2 + %endif +%endmacro + +;============================================================================= +; AVX abstraction layer +;============================================================================= + +%assign i 0 +%rep 16 + %if i < 8 + CAT_XDEFINE sizeofmm, i, 8 + %endif + CAT_XDEFINE sizeofxmm, i, 16 + CAT_XDEFINE sizeofymm, i, 32 +%assign i i+1 +%endrep +%undef i + +%macro CHECK_AVX_INSTR_EMU 3-* + %xdefine %%opcode %1 + %xdefine %%dst %2 + %rep %0-2 + %ifidn %%dst, %3 + %error non-AVX emulation of ``%%opcode'' is not supported + %endif + %rotate 1 + %endrep +%endmacro + +;%1 == instruction +;%2 == 1 if float, 0 if int +;%3 == 1 if 4-operand (xmm, xmm, xmm, imm), 0 if 2- or 3-operand (xmm, xmm, xmm) +;%4 == number of operands given +;%5+: operands +%macro RUN_AVX_INSTR 6-7+ + %ifid %6 + %define %%sizeofreg sizeof%6 + %elifid %5 + %define %%sizeofreg sizeof%5 + %else + %define %%sizeofreg mmsize + %endif + %if %%sizeofreg==32 + %if %4>=3 + v%1 %5, %6, %7 + %else + v%1 %5, %6 + %endif + %else + %if %%sizeofreg==8 + %define %%regmov movq + %elif %2 + %define %%regmov movaps + %else + %define %%regmov movdqa + %endif + + %if %4>=3+%3 + %ifnidn %5, %6 + %if AVX_enabled && %%sizeofreg==16 + v%1 %5, %6, %7 + %else + CHECK_AVX_INSTR_EMU {%1 %5, %6, %7}, %5, %7 + %%regmov %5, %6 + %1 %5, %7 + %endif + %else + %1 %5, %7 + %endif + %elif %4>=3 + %1 %5, %6, %7 + %else + %1 %5, %6 + %endif + %endif +%endmacro + +; 3arg AVX ops with a memory arg can only have it in src2, +; whereas SSE emulation of 3arg prefers to have it in src1 (i.e. the mov). +; So, if the op is symmetric and the wrong one is memory, swap them. +%macro RUN_AVX_INSTR1 8 + %assign %%swap 0 + %if AVX_enabled + %ifnid %6 + %assign %%swap 1 + %endif + %elifnidn %5, %6 + %ifnid %7 + %assign %%swap 1 + %endif + %endif + %if %%swap && %3 == 0 && %8 == 1 + RUN_AVX_INSTR %1, %2, %3, %4, %5, %7, %6 + %else + RUN_AVX_INSTR %1, %2, %3, %4, %5, %6, %7 + %endif +%endmacro + +;%1 == instruction +;%2 == 1 if float, 0 if int +;%3 == 1 if 4-operand (xmm, xmm, xmm, imm), 0 if 2- or 3-operand (xmm, xmm, xmm) +;%4 == 1 if symmetric (i.e. doesn't matter which src arg is which), 0 if not +%macro AVX_INSTR 4 + %macro %1 2-9 fnord, fnord, fnord, %1, %2, %3, %4 + %ifidn %3, fnord + RUN_AVX_INSTR %6, %7, %8, 2, %1, %2 + %elifidn %4, fnord + RUN_AVX_INSTR1 %6, %7, %8, 3, %1, %2, %3, %9 + %elifidn %5, fnord + RUN_AVX_INSTR %6, %7, %8, 4, %1, %2, %3, %4 + %else + RUN_AVX_INSTR %6, %7, %8, 5, %1, %2, %3, %4, %5 + %endif + %endmacro +%endmacro + +AVX_INSTR addpd, 1, 0, 1 +AVX_INSTR addps, 1, 0, 1 +AVX_INSTR addsd, 1, 0, 1 +AVX_INSTR addss, 1, 0, 1 +AVX_INSTR addsubpd, 1, 0, 0 +AVX_INSTR addsubps, 1, 0, 0 +AVX_INSTR andpd, 1, 0, 1 +AVX_INSTR andps, 1, 0, 1 +AVX_INSTR andnpd, 1, 0, 0 +AVX_INSTR andnps, 1, 0, 0 +AVX_INSTR blendpd, 1, 0, 0 +AVX_INSTR blendps, 1, 0, 0 +AVX_INSTR blendvpd, 1, 0, 0 +AVX_INSTR blendvps, 1, 0, 0 +AVX_INSTR cmppd, 1, 0, 0 +AVX_INSTR cmpps, 1, 0, 0 +AVX_INSTR cmpsd, 1, 0, 0 +AVX_INSTR cmpss, 1, 0, 0 +AVX_INSTR cvtdq2ps, 1, 0, 0 +AVX_INSTR cvtps2dq, 1, 0, 0 +AVX_INSTR divpd, 1, 0, 0 +AVX_INSTR divps, 1, 0, 0 +AVX_INSTR divsd, 1, 0, 0 +AVX_INSTR divss, 1, 0, 0 +AVX_INSTR dppd, 1, 1, 0 +AVX_INSTR dpps, 1, 1, 0 +AVX_INSTR haddpd, 1, 0, 0 +AVX_INSTR haddps, 1, 0, 0 +AVX_INSTR hsubpd, 1, 0, 0 +AVX_INSTR hsubps, 1, 0, 0 +AVX_INSTR maxpd, 1, 0, 1 +AVX_INSTR maxps, 1, 0, 1 +AVX_INSTR maxsd, 1, 0, 1 +AVX_INSTR maxss, 1, 0, 1 +AVX_INSTR minpd, 1, 0, 1 +AVX_INSTR minps, 1, 0, 1 +AVX_INSTR minsd, 1, 0, 1 +AVX_INSTR minss, 1, 0, 1 +AVX_INSTR movhlps, 1, 0, 0 +AVX_INSTR movlhps, 1, 0, 0 +AVX_INSTR movsd, 1, 0, 0 +AVX_INSTR movss, 1, 0, 0 +AVX_INSTR mpsadbw, 0, 1, 0 +AVX_INSTR mulpd, 1, 0, 1 +AVX_INSTR mulps, 1, 0, 1 +AVX_INSTR mulsd, 1, 0, 1 +AVX_INSTR mulss, 1, 0, 1 +AVX_INSTR orpd, 1, 0, 1 +AVX_INSTR orps, 1, 0, 1 +AVX_INSTR pabsb, 0, 0, 0 +AVX_INSTR pabsw, 0, 0, 0 +AVX_INSTR pabsd, 0, 0, 0 +AVX_INSTR packsswb, 0, 0, 0 +AVX_INSTR packssdw, 0, 0, 0 +AVX_INSTR packuswb, 0, 0, 0 +AVX_INSTR packusdw, 0, 0, 0 +AVX_INSTR paddb, 0, 0, 1 +AVX_INSTR paddw, 0, 0, 1 +AVX_INSTR paddd, 0, 0, 1 +AVX_INSTR paddq, 0, 0, 1 +AVX_INSTR paddsb, 0, 0, 1 +AVX_INSTR paddsw, 0, 0, 1 +AVX_INSTR paddusb, 0, 0, 1 +AVX_INSTR paddusw, 0, 0, 1 +AVX_INSTR palignr, 0, 1, 0 +AVX_INSTR pand, 0, 0, 1 +AVX_INSTR pandn, 0, 0, 0 +AVX_INSTR pavgb, 0, 0, 1 +AVX_INSTR pavgw, 0, 0, 1 +AVX_INSTR pblendvb, 0, 0, 0 +AVX_INSTR pblendw, 0, 1, 0 +AVX_INSTR pcmpestri, 0, 0, 0 +AVX_INSTR pcmpestrm, 0, 0, 0 +AVX_INSTR pcmpistri, 0, 0, 0 +AVX_INSTR pcmpistrm, 0, 0, 0 +AVX_INSTR pcmpeqb, 0, 0, 1 +AVX_INSTR pcmpeqw, 0, 0, 1 +AVX_INSTR pcmpeqd, 0, 0, 1 +AVX_INSTR pcmpeqq, 0, 0, 1 +AVX_INSTR pcmpgtb, 0, 0, 0 +AVX_INSTR pcmpgtw, 0, 0, 0 +AVX_INSTR pcmpgtd, 0, 0, 0 +AVX_INSTR pcmpgtq, 0, 0, 0 +AVX_INSTR phaddw, 0, 0, 0 +AVX_INSTR phaddd, 0, 0, 0 +AVX_INSTR phaddsw, 0, 0, 0 +AVX_INSTR phsubw, 0, 0, 0 +AVX_INSTR phsubd, 0, 0, 0 +AVX_INSTR phsubsw, 0, 0, 0 +AVX_INSTR pmaddwd, 0, 0, 1 +AVX_INSTR pmaddubsw, 0, 0, 0 +AVX_INSTR pmaxsb, 0, 0, 1 +AVX_INSTR pmaxsw, 0, 0, 1 +AVX_INSTR pmaxsd, 0, 0, 1 +AVX_INSTR pmaxub, 0, 0, 1 +AVX_INSTR pmaxuw, 0, 0, 1 +AVX_INSTR pmaxud, 0, 0, 1 +AVX_INSTR pminsb, 0, 0, 1 +AVX_INSTR pminsw, 0, 0, 1 +AVX_INSTR pminsd, 0, 0, 1 +AVX_INSTR pminub, 0, 0, 1 +AVX_INSTR pminuw, 0, 0, 1 +AVX_INSTR pminud, 0, 0, 1 +AVX_INSTR pmovmskb, 0, 0, 0 +AVX_INSTR pmulhuw, 0, 0, 1 +AVX_INSTR pmulhrsw, 0, 0, 1 +AVX_INSTR pmulhw, 0, 0, 1 +AVX_INSTR pmullw, 0, 0, 1 +AVX_INSTR pmulld, 0, 0, 1 +AVX_INSTR pmuludq, 0, 0, 1 +AVX_INSTR pmuldq, 0, 0, 1 +AVX_INSTR por, 0, 0, 1 +AVX_INSTR psadbw, 0, 0, 1 +AVX_INSTR pshufb, 0, 0, 0 +AVX_INSTR pshufd, 0, 1, 0 +AVX_INSTR pshufhw, 0, 1, 0 +AVX_INSTR pshuflw, 0, 1, 0 +AVX_INSTR psignb, 0, 0, 0 +AVX_INSTR psignw, 0, 0, 0 +AVX_INSTR psignd, 0, 0, 0 +AVX_INSTR psllw, 0, 0, 0 +AVX_INSTR pslld, 0, 0, 0 +AVX_INSTR psllq, 0, 0, 0 +AVX_INSTR pslldq, 0, 0, 0 +AVX_INSTR psraw, 0, 0, 0 +AVX_INSTR psrad, 0, 0, 0 +AVX_INSTR psrlw, 0, 0, 0 +AVX_INSTR psrld, 0, 0, 0 +AVX_INSTR psrlq, 0, 0, 0 +AVX_INSTR psrldq, 0, 0, 0 +AVX_INSTR psubb, 0, 0, 0 +AVX_INSTR psubw, 0, 0, 0 +AVX_INSTR psubd, 0, 0, 0 +AVX_INSTR psubq, 0, 0, 0 +AVX_INSTR psubsb, 0, 0, 0 +AVX_INSTR psubsw, 0, 0, 0 +AVX_INSTR psubusb, 0, 0, 0 +AVX_INSTR psubusw, 0, 0, 0 +AVX_INSTR ptest, 0, 0, 0 +AVX_INSTR punpckhbw, 0, 0, 0 +AVX_INSTR punpckhwd, 0, 0, 0 +AVX_INSTR punpckhdq, 0, 0, 0 +AVX_INSTR punpckhqdq, 0, 0, 0 +AVX_INSTR punpcklbw, 0, 0, 0 +AVX_INSTR punpcklwd, 0, 0, 0 +AVX_INSTR punpckldq, 0, 0, 0 +AVX_INSTR punpcklqdq, 0, 0, 0 +AVX_INSTR pxor, 0, 0, 1 +AVX_INSTR shufps, 1, 1, 0 +AVX_INSTR subpd, 1, 0, 0 +AVX_INSTR subps, 1, 0, 0 +AVX_INSTR subsd, 1, 0, 0 +AVX_INSTR subss, 1, 0, 0 +AVX_INSTR unpckhpd, 1, 0, 0 +AVX_INSTR unpckhps, 1, 0, 0 +AVX_INSTR unpcklpd, 1, 0, 0 +AVX_INSTR unpcklps, 1, 0, 0 +AVX_INSTR xorpd, 1, 0, 1 +AVX_INSTR xorps, 1, 0, 1 + +; 3DNow instructions, for sharing code between AVX, SSE and 3DN +AVX_INSTR pfadd, 1, 0, 1 +AVX_INSTR pfsub, 1, 0, 0 +AVX_INSTR pfmul, 1, 0, 1 + +; base-4 constants for shuffles +%assign i 0 +%rep 256 + %assign j ((i>>6)&3)*1000 + ((i>>4)&3)*100 + ((i>>2)&3)*10 + (i&3) + %if j < 10 + CAT_XDEFINE q000, j, i + %elif j < 100 + CAT_XDEFINE q00, j, i + %elif j < 1000 + CAT_XDEFINE q0, j, i + %else + CAT_XDEFINE q, j, i + %endif +%assign i i+1 +%endrep +%undef i +%undef j + +%macro FMA_INSTR 3 + %macro %1 4-7 %1, %2, %3 + %if cpuflag(xop) + v%5 %1, %2, %3, %4 + %else + %6 %1, %2, %3 + %7 %1, %4 + %endif + %endmacro +%endmacro + +FMA_INSTR pmacsdd, pmulld, paddd +FMA_INSTR pmacsww, pmullw, paddw +FMA_INSTR pmadcswd, pmaddwd, paddd + +; tzcnt is equivalent to "rep bsf" and is backwards-compatible with bsf. +; This lets us use tzcnt without bumping the yasm version requirement yet. +%define tzcnt rep bsf diff --git a/third_party/aom/third_party/x86inc/LICENSE b/third_party/aom/third_party/x86inc/LICENSE new file mode 100644 index 0000000000..7d07645a17 --- /dev/null +++ b/third_party/aom/third_party/x86inc/LICENSE @@ -0,0 +1,18 @@ +Copyright (C) 2005-2012 x264 project + +Authors: Loren Merritt + Anton Mitrofanov + Jason Garrett-Glaser + Henrik Gramner + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/third_party/aom/third_party/x86inc/README.libaom b/third_party/aom/third_party/x86inc/README.libaom new file mode 100644 index 0000000000..07c4dad20a --- /dev/null +++ b/third_party/aom/third_party/x86inc/README.libaom @@ -0,0 +1,20 @@ +URL: https://git.videolan.org/git/x264.git +Version: d23d18655249944c1ca894b451e2c82c7a584c62 +License: ISC +License File: LICENSE + +Description: +x264/libav's framework for x86 assembly. Contains a variety of macros and +defines that help automatically allow assembly to work cross-platform. + +Local Modifications: +Get configuration from aom_config.asm. +Prefix functions with aom by default. +Manage name mangling (prefixing with '_') manually because 'PREFIX' does not + exist in libaom. +Expand PIC default to macho64 and respect CONFIG_PIC from libaom +Set 'private_extern' visibility for macho targets. +Copy PIC 'GLOBAL' macros from x86_abi_support.asm +Use .text instead of .rodata on macho to avoid broken tables in PIC mode. +Use .text with no alignment for aout +Only use 'hidden' visibility with Chromium diff --git a/third_party/aom/third_party/x86inc/x86inc.asm b/third_party/aom/third_party/x86inc/x86inc.asm new file mode 100644 index 0000000000..cfee99c788 --- /dev/null +++ b/third_party/aom/third_party/x86inc/x86inc.asm @@ -0,0 +1,1649 @@ +;***************************************************************************** +;* x86inc.asm: x264asm abstraction layer +;***************************************************************************** +;* Copyright (C) 2005-2016 x264 project +;* +;* Authors: Loren Merritt +;* Anton Mitrofanov +;* Fiona Glaser +;* Henrik Gramner +;* +;* Permission to use, copy, modify, and/or distribute this software for any +;* purpose with or without fee is hereby granted, provided that the above +;* copyright notice and this permission notice appear in all copies. +;* +;* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +;* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +;* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +;* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +;* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +;* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +;***************************************************************************** + +; This is a header file for the x264ASM assembly language, which uses +; NASM/YASM syntax combined with a large number of macros to provide easy +; abstraction between different calling conventions (x86_32, win64, linux64). +; It also has various other useful features to simplify writing the kind of +; DSP functions that are most often used in x264. + +; Unlike the rest of x264, this file is available under an ISC license, as it +; has significant usefulness outside of x264 and we want it to be available +; to the largest audience possible. Of course, if you modify it for your own +; purposes to add a new feature, we strongly encourage contributing a patch +; as this feature might be useful for others as well. Send patches or ideas +; to x264-devel@videolan.org . + +%include "aom_config.asm" + +%ifndef private_prefix + %define private_prefix aom +%endif + +%ifndef public_prefix + %define public_prefix private_prefix +%endif + +%ifndef STACK_ALIGNMENT + %if ARCH_X86_64 + %define STACK_ALIGNMENT 16 + %else + %define STACK_ALIGNMENT 4 + %endif +%endif + +%define WIN64 0 +%define UNIX64 0 +%if ARCH_X86_64 + %ifidn __OUTPUT_FORMAT__,win32 + %define WIN64 1 + %elifidn __OUTPUT_FORMAT__,win64 + %define WIN64 1 + %elifidn __OUTPUT_FORMAT__,x64 + %define WIN64 1 + %else + %define UNIX64 1 + %endif +%endif + +%define FORMAT_ELF 0 +%ifidn __OUTPUT_FORMAT__,elf + %define FORMAT_ELF 1 +%elifidn __OUTPUT_FORMAT__,elf32 + %define FORMAT_ELF 1 +%elifidn __OUTPUT_FORMAT__,elf64 + %define FORMAT_ELF 1 +%endif + +%define FORMAT_MACHO 0 +%ifidn __OUTPUT_FORMAT__,macho32 + %define FORMAT_MACHO 1 +%elifidn __OUTPUT_FORMAT__,macho64 + %define FORMAT_MACHO 1 +%endif + +; Set PREFIX for libaom builds. +%if FORMAT_ELF + %undef PREFIX +%elif WIN64 + %undef PREFIX +%else + %define PREFIX +%endif + +%ifdef PREFIX + %define mangle(x) _ %+ x +%else + %define mangle(x) x +%endif + +; In some instances macho32 tables get misaligned when using .rodata. +; When looking at the disassembly it appears that the offset is either +; correct or consistently off by 90. Placing them in the .text section +; works around the issue. It appears to be specific to the way libaom +; handles the tables. +%macro SECTION_RODATA 0-1 16 + %ifidn __OUTPUT_FORMAT__,macho32 + SECTION .text align=%1 + fakegot: + %elifidn __OUTPUT_FORMAT__,aout + SECTION .text + %else + SECTION .rodata align=%1 + %endif +%endmacro + +; PIC macros are copied from aom_ports/x86_abi_support.asm. The "define PIC" +; from original code is added in for 64bit. +%ifidn __OUTPUT_FORMAT__,elf32 +%define ABI_IS_32BIT 1 +%elifidn __OUTPUT_FORMAT__,macho32 +%define ABI_IS_32BIT 1 +%elifidn __OUTPUT_FORMAT__,win32 +%define ABI_IS_32BIT 1 +%elifidn __OUTPUT_FORMAT__,aout +%define ABI_IS_32BIT 1 +%else +%define ABI_IS_32BIT 0 +%endif + +%if ABI_IS_32BIT + %if CONFIG_PIC=1 + %ifidn __OUTPUT_FORMAT__,elf32 + %define GET_GOT_DEFINED 1 + %define WRT_PLT wrt ..plt + %macro GET_GOT 1 + extern _GLOBAL_OFFSET_TABLE_ + push %1 + call %%get_got + %%sub_offset: + jmp %%exitGG + %%get_got: + mov %1, [esp] + add %1, _GLOBAL_OFFSET_TABLE_ + $$ - %%sub_offset wrt ..gotpc + ret + %%exitGG: + %undef GLOBAL + %define GLOBAL(x) x + %1 wrt ..gotoff + %undef RESTORE_GOT + %define RESTORE_GOT pop %1 + %endmacro + %elifidn __OUTPUT_FORMAT__,macho32 + %define GET_GOT_DEFINED 1 + %macro GET_GOT 1 + push %1 + call %%get_got + %%get_got: + pop %1 + %undef GLOBAL + %define GLOBAL(x) x + %1 - %%get_got + %undef RESTORE_GOT + %define RESTORE_GOT pop %1 + %endmacro + %else + %define GET_GOT_DEFINED 0 + %endif + %endif + + %if ARCH_X86_64 == 0 + %undef PIC + %endif + +%else + %macro GET_GOT 1 + %endmacro + %define GLOBAL(x) rel x + %define WRT_PLT wrt ..plt + + %if WIN64 + %define PIC + %elifidn __OUTPUT_FORMAT__,macho64 + %define PIC + %elif CONFIG_PIC + %define PIC + %endif +%endif + +%ifnmacro GET_GOT + %macro GET_GOT 1 + %endmacro + %define GLOBAL(x) x +%endif +%ifndef RESTORE_GOT + %define RESTORE_GOT +%endif +%ifndef WRT_PLT + %define WRT_PLT +%endif + +%ifdef PIC + default rel +%endif + +%ifndef GET_GOT_DEFINED + %define GET_GOT_DEFINED 0 +%endif +; Done with PIC macros + +%ifdef __NASM_VER__ + %use smartalign +%endif + +; Macros to eliminate most code duplication between x86_32 and x86_64: +; Currently this works only for leaf functions which load all their arguments +; into registers at the start, and make no other use of the stack. Luckily that +; covers most of x264's asm. + +; PROLOGUE: +; %1 = number of arguments. loads them from stack if needed. +; %2 = number of registers used. pushes callee-saved regs if needed. +; %3 = number of xmm registers used. pushes callee-saved xmm regs if needed. +; %4 = (optional) stack size to be allocated. The stack will be aligned before +; allocating the specified stack size. If the required stack alignment is +; larger than the known stack alignment the stack will be manually aligned +; and an extra register will be allocated to hold the original stack +; pointer (to not invalidate r0m etc.). To prevent the use of an extra +; register as stack pointer, request a negative stack size. +; %4+/%5+ = list of names to define to registers +; PROLOGUE can also be invoked by adding the same options to cglobal + +; e.g. +; cglobal foo, 2,3,7,0x40, dst, src, tmp +; declares a function (foo) that automatically loads two arguments (dst and +; src) into registers, uses one additional register (tmp) plus 7 vector +; registers (m0-m6) and allocates 0x40 bytes of stack space. + +; TODO Some functions can use some args directly from the stack. If they're the +; last args then you can just not declare them, but if they're in the middle +; we need more flexible macro. + +; RET: +; Pops anything that was pushed by PROLOGUE, and returns. + +; REP_RET: +; Use this instead of RET if it's a branch target. + +; registers: +; rN and rNq are the native-size register holding function argument N +; rNd, rNw, rNb are dword, word, and byte size +; rNh is the high 8 bits of the word size +; rNm is the original location of arg N (a register or on the stack), dword +; rNmp is native size + +%macro DECLARE_REG 2-3 + %define r%1q %2 + %define r%1d %2d + %define r%1w %2w + %define r%1b %2b + %define r%1h %2h + %define %2q %2 + %if %0 == 2 + %define r%1m %2d + %define r%1mp %2 + %elif ARCH_X86_64 ; memory + %define r%1m [rstk + stack_offset + %3] + %define r%1mp qword r %+ %1 %+ m + %else + %define r%1m [rstk + stack_offset + %3] + %define r%1mp dword r %+ %1 %+ m + %endif + %define r%1 %2 +%endmacro + +%macro DECLARE_REG_SIZE 3 + %define r%1q r%1 + %define e%1q r%1 + %define r%1d e%1 + %define e%1d e%1 + %define r%1w %1 + %define e%1w %1 + %define r%1h %3 + %define e%1h %3 + %define r%1b %2 + %define e%1b %2 + %if ARCH_X86_64 == 0 + %define r%1 e%1 + %endif +%endmacro + +DECLARE_REG_SIZE ax, al, ah +DECLARE_REG_SIZE bx, bl, bh +DECLARE_REG_SIZE cx, cl, ch +DECLARE_REG_SIZE dx, dl, dh +DECLARE_REG_SIZE si, sil, null +DECLARE_REG_SIZE di, dil, null +DECLARE_REG_SIZE bp, bpl, null + +; t# defines for when per-arch register allocation is more complex than just function arguments + +%macro DECLARE_REG_TMP 1-* + %assign %%i 0 + %rep %0 + CAT_XDEFINE t, %%i, r%1 + %assign %%i %%i+1 + %rotate 1 + %endrep +%endmacro + +%macro DECLARE_REG_TMP_SIZE 0-* + %rep %0 + %define t%1q t%1 %+ q + %define t%1d t%1 %+ d + %define t%1w t%1 %+ w + %define t%1h t%1 %+ h + %define t%1b t%1 %+ b + %rotate 1 + %endrep +%endmacro + +DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 + +%if ARCH_X86_64 + %define gprsize 8 +%else + %define gprsize 4 +%endif + +%macro PUSH 1 + push %1 + %ifidn rstk, rsp + %assign stack_offset stack_offset+gprsize + %endif +%endmacro + +%macro POP 1 + pop %1 + %ifidn rstk, rsp + %assign stack_offset stack_offset-gprsize + %endif +%endmacro + +%macro PUSH_IF_USED 1-* + %rep %0 + %if %1 < regs_used + PUSH r%1 + %endif + %rotate 1 + %endrep +%endmacro + +%macro POP_IF_USED 1-* + %rep %0 + %if %1 < regs_used + pop r%1 + %endif + %rotate 1 + %endrep +%endmacro + +%macro LOAD_IF_USED 1-* + %rep %0 + %if %1 < num_args + mov r%1, r %+ %1 %+ mp + %endif + %rotate 1 + %endrep +%endmacro + +%macro SUB 2 + sub %1, %2 + %ifidn %1, rstk + %assign stack_offset stack_offset+(%2) + %endif +%endmacro + +%macro ADD 2 + add %1, %2 + %ifidn %1, rstk + %assign stack_offset stack_offset-(%2) + %endif +%endmacro + +%macro movifnidn 2 + %ifnidn %1, %2 + mov %1, %2 + %endif +%endmacro + +%macro movsxdifnidn 2 + %ifnidn %1, %2 + movsxd %1, %2 + %endif +%endmacro + +%macro ASSERT 1 + %if (%1) == 0 + %error assertion ``%1'' failed + %endif +%endmacro + +%macro DEFINE_ARGS 0-* + %ifdef n_arg_names + %assign %%i 0 + %rep n_arg_names + CAT_UNDEF arg_name %+ %%i, q + CAT_UNDEF arg_name %+ %%i, d + CAT_UNDEF arg_name %+ %%i, w + CAT_UNDEF arg_name %+ %%i, h + CAT_UNDEF arg_name %+ %%i, b + CAT_UNDEF arg_name %+ %%i, m + CAT_UNDEF arg_name %+ %%i, mp + CAT_UNDEF arg_name, %%i + %assign %%i %%i+1 + %endrep + %endif + + %xdefine %%stack_offset stack_offset + %undef stack_offset ; so that the current value of stack_offset doesn't get baked in by xdefine + %assign %%i 0 + %rep %0 + %xdefine %1q r %+ %%i %+ q + %xdefine %1d r %+ %%i %+ d + %xdefine %1w r %+ %%i %+ w + %xdefine %1h r %+ %%i %+ h + %xdefine %1b r %+ %%i %+ b + %xdefine %1m r %+ %%i %+ m + %xdefine %1mp r %+ %%i %+ mp + CAT_XDEFINE arg_name, %%i, %1 + %assign %%i %%i+1 + %rotate 1 + %endrep + %xdefine stack_offset %%stack_offset + %assign n_arg_names %0 +%endmacro + +%define required_stack_alignment ((mmsize + 15) & ~15) + +%macro ALLOC_STACK 1-2 0 ; stack_size, n_xmm_regs (for win64 only) + %ifnum %1 + %if %1 != 0 + %assign %%pad 0 + %assign stack_size %1 + %if stack_size < 0 + %assign stack_size -stack_size + %endif + %if WIN64 + %assign %%pad %%pad + 32 ; shadow space + %if mmsize != 8 + %assign xmm_regs_used %2 + %if xmm_regs_used > 8 + %assign %%pad %%pad + (xmm_regs_used-8)*16 ; callee-saved xmm registers + %endif + %endif + %endif + %if required_stack_alignment <= STACK_ALIGNMENT + ; maintain the current stack alignment + %assign stack_size_padded stack_size + %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1)) + SUB rsp, stack_size_padded + %else + %assign %%reg_num (regs_used - 1) + %xdefine rstk r %+ %%reg_num + ; align stack, and save original stack location directly above + ; it, i.e. in [rsp+stack_size_padded], so we can restore the + ; stack in a single instruction (i.e. mov rsp, rstk or mov + ; rsp, [rsp+stack_size_padded]) + %if %1 < 0 ; need to store rsp on stack + %xdefine rstkm [rsp + stack_size + %%pad] + %assign %%pad %%pad + gprsize + %else ; can keep rsp in rstk during whole function + %xdefine rstkm rstk + %endif + %assign stack_size_padded stack_size + ((%%pad + required_stack_alignment-1) & ~(required_stack_alignment-1)) + mov rstk, rsp + and rsp, ~(required_stack_alignment-1) + sub rsp, stack_size_padded + movifnidn rstkm, rstk + %endif + WIN64_PUSH_XMM + %endif + %endif +%endmacro + +%macro SETUP_STACK_POINTER 1 + %ifnum %1 + %if %1 != 0 && required_stack_alignment > STACK_ALIGNMENT + %if %1 > 0 + %assign regs_used (regs_used + 1) + %endif + %if ARCH_X86_64 && regs_used < 5 + UNIX64 * 3 + ; Ensure that we don't clobber any registers containing arguments + %assign regs_used 5 + UNIX64 * 3 + %endif + %endif + %endif +%endmacro + +%macro DEFINE_ARGS_INTERNAL 3+ + %ifnum %2 + DEFINE_ARGS %3 + %elif %1 == 4 + DEFINE_ARGS %2 + %elif %1 > 4 + DEFINE_ARGS %2, %3 + %endif +%endmacro + +%if WIN64 ; Windows x64 ;================================================= + +DECLARE_REG 0, rcx +DECLARE_REG 1, rdx +DECLARE_REG 2, R8 +DECLARE_REG 3, R9 +DECLARE_REG 4, R10, 40 +DECLARE_REG 5, R11, 48 +DECLARE_REG 6, rax, 56 +DECLARE_REG 7, rdi, 64 +DECLARE_REG 8, rsi, 72 +DECLARE_REG 9, rbx, 80 +DECLARE_REG 10, rbp, 88 +DECLARE_REG 11, R12, 96 +DECLARE_REG 12, R13, 104 +DECLARE_REG 13, R14, 112 +DECLARE_REG 14, R15, 120 + +%macro PROLOGUE 2-5+ 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... + %assign num_args %1 + %assign regs_used %2 + ASSERT regs_used >= num_args + SETUP_STACK_POINTER %4 + ASSERT regs_used <= 15 + PUSH_IF_USED 7, 8, 9, 10, 11, 12, 13, 14 + ALLOC_STACK %4, %3 + %if mmsize != 8 && stack_size == 0 + WIN64_SPILL_XMM %3 + %endif + LOAD_IF_USED 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 + DEFINE_ARGS_INTERNAL %0, %4, %5 +%endmacro + +%macro WIN64_PUSH_XMM 0 + ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated. + %if xmm_regs_used > 6 + movaps [rstk + stack_offset + 8], xmm6 + %endif + %if xmm_regs_used > 7 + movaps [rstk + stack_offset + 24], xmm7 + %endif + %if xmm_regs_used > 8 + %assign %%i 8 + %rep xmm_regs_used-8 + movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i + %assign %%i %%i+1 + %endrep + %endif +%endmacro + +%macro WIN64_SPILL_XMM 1 + %assign xmm_regs_used %1 + ASSERT xmm_regs_used <= 16 + %if xmm_regs_used > 8 + ; Allocate stack space for callee-saved xmm registers plus shadow space and align the stack. + %assign %%pad (xmm_regs_used-8)*16 + 32 + %assign stack_size_padded %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1)) + SUB rsp, stack_size_padded + %endif + WIN64_PUSH_XMM +%endmacro + +%macro WIN64_RESTORE_XMM_INTERNAL 1 + %assign %%pad_size 0 + %if xmm_regs_used > 8 + %assign %%i xmm_regs_used + %rep xmm_regs_used-8 + %assign %%i %%i-1 + movaps xmm %+ %%i, [%1 + (%%i-8)*16 + stack_size + 32] + %endrep + %endif + %if stack_size_padded > 0 + %if stack_size > 0 && required_stack_alignment > STACK_ALIGNMENT + mov rsp, rstkm + %else + add %1, stack_size_padded + %assign %%pad_size stack_size_padded + %endif + %endif + %if xmm_regs_used > 7 + movaps xmm7, [%1 + stack_offset - %%pad_size + 24] + %endif + %if xmm_regs_used > 6 + movaps xmm6, [%1 + stack_offset - %%pad_size + 8] + %endif +%endmacro + +%macro WIN64_RESTORE_XMM 1 + WIN64_RESTORE_XMM_INTERNAL %1 + %assign stack_offset (stack_offset-stack_size_padded) + %assign xmm_regs_used 0 +%endmacro + +%define has_epilogue regs_used > 7 || xmm_regs_used > 6 || mmsize == 32 || stack_size > 0 + +%macro RET 0 + WIN64_RESTORE_XMM_INTERNAL rsp + POP_IF_USED 14, 13, 12, 11, 10, 9, 8, 7 + %if mmsize == 32 + vzeroupper + %endif + AUTO_REP_RET +%endmacro + +%elif ARCH_X86_64 ; *nix x64 ;============================================= + +DECLARE_REG 0, rdi +DECLARE_REG 1, rsi +DECLARE_REG 2, rdx +DECLARE_REG 3, rcx +DECLARE_REG 4, R8 +DECLARE_REG 5, R9 +DECLARE_REG 6, rax, 8 +DECLARE_REG 7, R10, 16 +DECLARE_REG 8, R11, 24 +DECLARE_REG 9, rbx, 32 +DECLARE_REG 10, rbp, 40 +DECLARE_REG 11, R12, 48 +DECLARE_REG 12, R13, 56 +DECLARE_REG 13, R14, 64 +DECLARE_REG 14, R15, 72 + +%macro PROLOGUE 2-5+ ; #args, #regs, #xmm_regs, [stack_size,] arg_names... + %assign num_args %1 + %assign regs_used %2 + ASSERT regs_used >= num_args + SETUP_STACK_POINTER %4 + ASSERT regs_used <= 15 + PUSH_IF_USED 9, 10, 11, 12, 13, 14 + ALLOC_STACK %4 + LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14 + DEFINE_ARGS_INTERNAL %0, %4, %5 +%endmacro + +%define has_epilogue regs_used > 9 || mmsize == 32 || stack_size > 0 + +%macro RET 0 + %if stack_size_padded > 0 + %if required_stack_alignment > STACK_ALIGNMENT + mov rsp, rstkm + %else + add rsp, stack_size_padded + %endif + %endif + POP_IF_USED 14, 13, 12, 11, 10, 9 + %if mmsize == 32 + vzeroupper + %endif + AUTO_REP_RET +%endmacro + +%else ; X86_32 ;============================================================== + +DECLARE_REG 0, eax, 4 +DECLARE_REG 1, ecx, 8 +DECLARE_REG 2, edx, 12 +DECLARE_REG 3, ebx, 16 +DECLARE_REG 4, esi, 20 +DECLARE_REG 5, edi, 24 +DECLARE_REG 6, ebp, 28 +%define rsp esp + +%macro DECLARE_ARG 1-* + %rep %0 + %define r%1m [rstk + stack_offset + 4*%1 + 4] + %define r%1mp dword r%1m + %rotate 1 + %endrep +%endmacro + +DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 + +%macro PROLOGUE 2-5+ ; #args, #regs, #xmm_regs, [stack_size,] arg_names... + %assign num_args %1 + %assign regs_used %2 + ASSERT regs_used >= num_args + %if num_args > 7 + %assign num_args 7 + %endif + %if regs_used > 7 + %assign regs_used 7 + %endif + SETUP_STACK_POINTER %4 + ASSERT regs_used <= 7 + PUSH_IF_USED 3, 4, 5, 6 + ALLOC_STACK %4 + LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6 + DEFINE_ARGS_INTERNAL %0, %4, %5 +%endmacro + +%define has_epilogue regs_used > 3 || mmsize == 32 || stack_size > 0 + +%macro RET 0 + %if stack_size_padded > 0 + %if required_stack_alignment > STACK_ALIGNMENT + mov rsp, rstkm + %else + add rsp, stack_size_padded + %endif + %endif + POP_IF_USED 6, 5, 4, 3 + %if mmsize == 32 + vzeroupper + %endif + AUTO_REP_RET +%endmacro + +%endif ;====================================================================== + +%if WIN64 == 0 + %macro WIN64_SPILL_XMM 1 + %endmacro + %macro WIN64_RESTORE_XMM 1 + %endmacro + %macro WIN64_PUSH_XMM 0 + %endmacro +%endif + +; On AMD cpus <=K10, an ordinary ret is slow if it immediately follows either +; a branch or a branch target. So switch to a 2-byte form of ret in that case. +; We can automatically detect "follows a branch", but not a branch target. +; (SSSE3 is a sufficient condition to know that your cpu doesn't have this problem.) +%macro REP_RET 0 + %if has_epilogue + RET + %else + rep ret + %endif + annotate_function_size +%endmacro + +%define last_branch_adr $$ +%macro AUTO_REP_RET 0 + %if notcpuflag(ssse3) + times ((last_branch_adr-$)>>31)+1 rep ; times 1 iff $ == last_branch_adr. + %endif + ret + annotate_function_size +%endmacro + +%macro BRANCH_INSTR 0-* + %rep %0 + %macro %1 1-2 %1 + %2 %1 + %if notcpuflag(ssse3) + %%branch_instr equ $ + %xdefine last_branch_adr %%branch_instr + %endif + %endmacro + %rotate 1 + %endrep +%endmacro + +BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, jna, jnae, jb, jbe, jnb, jnbe, jc, jnc, js, jns, jo, jno, jp, jnp + +%macro TAIL_CALL 2 ; callee, is_nonadjacent + %if has_epilogue + call %1 + RET + %elif %2 + jmp %1 + %endif + annotate_function_size +%endmacro + +;============================================================================= +; arch-independent part +;============================================================================= + +%assign function_align 16 + +; Begin a function. +; Applies any symbol mangling needed for C linkage, and sets up a define such that +; subsequent uses of the function name automatically refer to the mangled version. +; Appends cpuflags to the function name if cpuflags has been specified. +; The "" empty default parameter is a workaround for nasm, which fails if SUFFIX +; is empty and we call cglobal_internal with just %1 %+ SUFFIX (without %2). +%macro cglobal 1-2+ "" ; name, [PROLOGUE args] + cglobal_internal 1, %1 %+ SUFFIX, %2 +%endmacro +%macro cvisible 1-2+ "" ; name, [PROLOGUE args] + cglobal_internal 0, %1 %+ SUFFIX, %2 +%endmacro +%macro cglobal_internal 2-3+ + annotate_function_size + %if %1 + %xdefine %%FUNCTION_PREFIX private_prefix + ; libaom explicitly sets visibility in shared object builds. Avoid + ; setting visibility to hidden as it may break builds that split + ; sources on e.g., directory boundaries. + %ifdef CHROMIUM + %xdefine %%VISIBILITY hidden + %else + %xdefine %%VISIBILITY + %endif + %else + %xdefine %%FUNCTION_PREFIX public_prefix + %xdefine %%VISIBILITY + %endif + %ifndef cglobaled_%2 + %xdefine %2 mangle(%%FUNCTION_PREFIX %+ _ %+ %2) + %xdefine %2.skip_prologue %2 %+ .skip_prologue + CAT_XDEFINE cglobaled_, %2, 1 + %endif + %xdefine current_function %2 + %xdefine current_function_section __SECT__ + %if FORMAT_ELF + global %2:function %%VISIBILITY + %elif FORMAT_MACHO + %ifdef __NASM_VER__ + global %2 + %else + global %2:private_extern + %endif + %else + global %2 + %endif + align function_align + %2: + RESET_MM_PERMUTATION ; needed for x86-64, also makes disassembly somewhat nicer + %xdefine rstk rsp ; copy of the original stack pointer, used when greater alignment than the known stack alignment is required + %assign stack_offset 0 ; stack pointer offset relative to the return address + %assign stack_size 0 ; amount of stack space that can be freely used inside a function + %assign stack_size_padded 0 ; total amount of allocated stack space, including space for callee-saved xmm registers on WIN64 and alignment padding + %assign xmm_regs_used 0 ; number of XMM registers requested, used for dealing with callee-saved registers on WIN64 + %ifnidn %3, "" + PROLOGUE %3 + %endif +%endmacro + +%macro cextern 1 + %xdefine %1 mangle(private_prefix %+ _ %+ %1) + CAT_XDEFINE cglobaled_, %1, 1 + extern %1 +%endmacro + +; like cextern, but without the prefix +%macro cextern_naked 1 + %ifdef PREFIX + %xdefine %1 mangle(%1) + %endif + CAT_XDEFINE cglobaled_, %1, 1 + extern %1 +%endmacro + +%macro const 1-2+ + %xdefine %1 mangle(private_prefix %+ _ %+ %1) + %if FORMAT_ELF + global %1:data hidden + %else + global %1 + %endif + %1: %2 +%endmacro + +; This is needed for ELF, otherwise the GNU linker assumes the stack is executable by default. +%if FORMAT_ELF + [SECTION .note.GNU-stack noalloc noexec nowrite progbits] +%endif + +; Tell debuggers how large the function was. +; This may be invoked multiple times per function; we rely on later instances overriding earlier ones. +; This is invoked by RET and similar macros, and also cglobal does it for the previous function, +; but if the last function in a source file doesn't use any of the standard macros for its epilogue, +; then its size might be unspecified. +%macro annotate_function_size 0 + %ifdef __YASM_VER__ + %ifdef current_function + %if FORMAT_ELF + current_function_section + %%ecf equ $ + size current_function %%ecf - current_function + __SECT__ + %endif + %endif + %endif +%endmacro + +; cpuflags + +%assign cpuflags_mmx (1<<0) +%assign cpuflags_mmx2 (1<<1) | cpuflags_mmx +%assign cpuflags_3dnow (1<<2) | cpuflags_mmx +%assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow +%assign cpuflags_sse (1<<4) | cpuflags_mmx2 +%assign cpuflags_sse2 (1<<5) | cpuflags_sse +%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2 +%assign cpuflags_sse3 (1<<7) | cpuflags_sse2 +%assign cpuflags_ssse3 (1<<8) | cpuflags_sse3 +%assign cpuflags_sse4 (1<<9) | cpuflags_ssse3 +%assign cpuflags_sse42 (1<<10)| cpuflags_sse4 +%assign cpuflags_avx (1<<11)| cpuflags_sse42 +%assign cpuflags_xop (1<<12)| cpuflags_avx +%assign cpuflags_fma4 (1<<13)| cpuflags_avx +%assign cpuflags_fma3 (1<<14)| cpuflags_avx +%assign cpuflags_avx2 (1<<15)| cpuflags_fma3 + +%assign cpuflags_cache32 (1<<16) +%assign cpuflags_cache64 (1<<17) +%assign cpuflags_slowctz (1<<18) +%assign cpuflags_lzcnt (1<<19) +%assign cpuflags_aligned (1<<20) ; not a cpu feature, but a function variant +%assign cpuflags_atom (1<<21) +%assign cpuflags_bmi1 (1<<22)|cpuflags_lzcnt +%assign cpuflags_bmi2 (1<<23)|cpuflags_bmi1 + +; Returns a boolean value expressing whether or not the specified cpuflag is enabled. +%define cpuflag(x) (((((cpuflags & (cpuflags_ %+ x)) ^ (cpuflags_ %+ x)) - 1) >> 31) & 1) +%define notcpuflag(x) (cpuflag(x) ^ 1) + +; Takes an arbitrary number of cpuflags from the above list. +; All subsequent functions (up to the next INIT_CPUFLAGS) is built for the specified cpu. +; You shouldn't need to invoke this macro directly, it's a subroutine for INIT_MMX &co. +%macro INIT_CPUFLAGS 0-* + %xdefine SUFFIX + %undef cpuname + %assign cpuflags 0 + + %if %0 >= 1 + %rep %0 + %ifdef cpuname + %xdefine cpuname cpuname %+ _%1 + %else + %xdefine cpuname %1 + %endif + %assign cpuflags cpuflags | cpuflags_%1 + %rotate 1 + %endrep + %xdefine SUFFIX _ %+ cpuname + + %if cpuflag(avx) + %assign avx_enabled 1 + %endif + %if (mmsize == 16 && notcpuflag(sse2)) || (mmsize == 32 && notcpuflag(avx2)) + %define mova movaps + %define movu movups + %define movnta movntps + %endif + %if cpuflag(aligned) + %define movu mova + %elif cpuflag(sse3) && notcpuflag(ssse3) + %define movu lddqu + %endif + %endif + + %if ARCH_X86_64 || cpuflag(sse2) + %ifdef __NASM_VER__ + ALIGNMODE k8 + %else + CPU amdnop + %endif + %else + %ifdef __NASM_VER__ + ALIGNMODE nop + %else + CPU basicnop + %endif + %endif +%endmacro + +; Merge mmx and sse* +; m# is a simd register of the currently selected size +; xm# is the corresponding xmm register if mmsize >= 16, otherwise the same as m# +; ym# is the corresponding ymm register if mmsize >= 32, otherwise the same as m# +; (All 3 remain in sync through SWAP.) + +%macro CAT_XDEFINE 3 + %xdefine %1%2 %3 +%endmacro + +%macro CAT_UNDEF 2 + %undef %1%2 +%endmacro + +%macro INIT_MMX 0-1+ + %assign avx_enabled 0 + %define RESET_MM_PERMUTATION INIT_MMX %1 + %define mmsize 8 + %define num_mmregs 8 + %define mova movq + %define movu movq + %define movh movd + %define movnta movntq + %assign %%i 0 + %rep 8 + CAT_XDEFINE m, %%i, mm %+ %%i + CAT_XDEFINE nnmm, %%i, %%i + %assign %%i %%i+1 + %endrep + %rep 8 + CAT_UNDEF m, %%i + CAT_UNDEF nnmm, %%i + %assign %%i %%i+1 + %endrep + INIT_CPUFLAGS %1 +%endmacro + +%macro INIT_XMM 0-1+ + %assign avx_enabled 0 + %define RESET_MM_PERMUTATION INIT_XMM %1 + %define mmsize 16 + %define num_mmregs 8 + %if ARCH_X86_64 + %define num_mmregs 16 + %endif + %define mova movdqa + %define movu movdqu + %define movh movq + %define movnta movntdq + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE m, %%i, xmm %+ %%i + CAT_XDEFINE nnxmm, %%i, %%i + %assign %%i %%i+1 + %endrep + INIT_CPUFLAGS %1 +%endmacro + +%macro INIT_YMM 0-1+ + %assign avx_enabled 1 + %define RESET_MM_PERMUTATION INIT_YMM %1 + %define mmsize 32 + %define num_mmregs 8 + %if ARCH_X86_64 + %define num_mmregs 16 + %endif + %define mova movdqa + %define movu movdqu + %undef movh + %define movnta movntdq + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE m, %%i, ymm %+ %%i + CAT_XDEFINE nnymm, %%i, %%i + %assign %%i %%i+1 + %endrep + INIT_CPUFLAGS %1 +%endmacro + +INIT_XMM + +%macro DECLARE_MMCAST 1 + %define mmmm%1 mm%1 + %define mmxmm%1 mm%1 + %define mmymm%1 mm%1 + %define xmmmm%1 mm%1 + %define xmmxmm%1 xmm%1 + %define xmmymm%1 xmm%1 + %define ymmmm%1 mm%1 + %define ymmxmm%1 xmm%1 + %define ymmymm%1 ymm%1 + %define xm%1 xmm %+ m%1 + %define ym%1 ymm %+ m%1 +%endmacro + +%assign i 0 +%rep 16 + DECLARE_MMCAST i + %assign i i+1 +%endrep + +; I often want to use macros that permute their arguments. e.g. there's no +; efficient way to implement butterfly or transpose or dct without swapping some +; arguments. +; +; I would like to not have to manually keep track of the permutations: +; If I insert a permutation in the middle of a function, it should automatically +; change everything that follows. For more complex macros I may also have multiple +; implementations, e.g. the SSE2 and SSSE3 versions may have different permutations. +; +; Hence these macros. Insert a PERMUTE or some SWAPs at the end of a macro that +; permutes its arguments. It's equivalent to exchanging the contents of the +; registers, except that this way you exchange the register names instead, so it +; doesn't cost any cycles. + +%macro PERMUTE 2-* ; takes a list of pairs to swap + %rep %0/2 + %xdefine %%tmp%2 m%2 + %rotate 2 + %endrep + %rep %0/2 + %xdefine m%1 %%tmp%2 + CAT_XDEFINE nn, m%1, %1 + %rotate 2 + %endrep +%endmacro + +%macro SWAP 2+ ; swaps a single chain (sometimes more concise than pairs) + %ifnum %1 ; SWAP 0, 1, ... + SWAP_INTERNAL_NUM %1, %2 + %else ; SWAP m0, m1, ... + SWAP_INTERNAL_NAME %1, %2 + %endif +%endmacro + +%macro SWAP_INTERNAL_NUM 2-* + %rep %0-1 + %xdefine %%tmp m%1 + %xdefine m%1 m%2 + %xdefine m%2 %%tmp + CAT_XDEFINE nn, m%1, %1 + CAT_XDEFINE nn, m%2, %2 + %rotate 1 + %endrep +%endmacro + +%macro SWAP_INTERNAL_NAME 2-* + %xdefine %%args nn %+ %1 + %rep %0-1 + %xdefine %%args %%args, nn %+ %2 + %rotate 1 + %endrep + SWAP_INTERNAL_NUM %%args +%endmacro + +; If SAVE_MM_PERMUTATION is placed at the end of a function, then any later +; calls to that function will automatically load the permutation, so values can +; be returned in mmregs. +%macro SAVE_MM_PERMUTATION 0-1 + %if %0 + %xdefine %%f %1_m + %else + %xdefine %%f current_function %+ _m + %endif + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE %%f, %%i, m %+ %%i + %assign %%i %%i+1 + %endrep +%endmacro + +%macro LOAD_MM_PERMUTATION 1 ; name to load from + %ifdef %1_m0 + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE m, %%i, %1_m %+ %%i + CAT_XDEFINE nn, m %+ %%i, %%i + %assign %%i %%i+1 + %endrep + %endif +%endmacro + +; Append cpuflags to the callee's name iff the appended name is known and the plain name isn't +%macro call 1 + call_internal %1 %+ SUFFIX, %1 +%endmacro +%macro call_internal 2 + %xdefine %%i %2 + %ifndef cglobaled_%2 + %ifdef cglobaled_%1 + %xdefine %%i %1 + %endif + %endif + call %%i + LOAD_MM_PERMUTATION %%i +%endmacro + +; Substitutions that reduce instruction size but are functionally equivalent +%macro add 2 + %ifnum %2 + %if %2==128 + sub %1, -128 + %else + add %1, %2 + %endif + %else + add %1, %2 + %endif +%endmacro + +%macro sub 2 + %ifnum %2 + %if %2==128 + add %1, -128 + %else + sub %1, %2 + %endif + %else + sub %1, %2 + %endif +%endmacro + +;============================================================================= +; AVX abstraction layer +;============================================================================= + +%assign i 0 +%rep 16 + %if i < 8 + CAT_XDEFINE sizeofmm, i, 8 + %endif + CAT_XDEFINE sizeofxmm, i, 16 + CAT_XDEFINE sizeofymm, i, 32 + %assign i i+1 +%endrep +%undef i + +%macro CHECK_AVX_INSTR_EMU 3-* + %xdefine %%opcode %1 + %xdefine %%dst %2 + %rep %0-2 + %ifidn %%dst, %3 + %error non-avx emulation of ``%%opcode'' is not supported + %endif + %rotate 1 + %endrep +%endmacro + +;%1 == instruction +;%2 == minimal instruction set +;%3 == 1 if float, 0 if int +;%4 == 1 if non-destructive or 4-operand (xmm, xmm, xmm, imm), 0 otherwise +;%5 == 1 if commutative (i.e. doesn't matter which src arg is which), 0 if not +;%6+: operands +%macro RUN_AVX_INSTR 6-9+ + %ifnum sizeof%7 + %assign __sizeofreg sizeof%7 + %elifnum sizeof%6 + %assign __sizeofreg sizeof%6 + %else + %assign __sizeofreg mmsize + %endif + %assign __emulate_avx 0 + %if avx_enabled && __sizeofreg >= 16 + %xdefine __instr v%1 + %else + %xdefine __instr %1 + %if %0 >= 8+%4 + %assign __emulate_avx 1 + %endif + %endif + %ifnidn %2, fnord + %ifdef cpuname + %if notcpuflag(%2) + %error use of ``%1'' %2 instruction in cpuname function: current_function + %elif cpuflags_%2 < cpuflags_sse && notcpuflag(sse2) && __sizeofreg > 8 + %error use of ``%1'' sse2 instruction in cpuname function: current_function + %endif + %endif + %endif + + %if __emulate_avx + %xdefine __src1 %7 + %xdefine __src2 %8 + %ifnidn %6, %7 + %if %0 >= 9 + CHECK_AVX_INSTR_EMU {%1 %6, %7, %8, %9}, %6, %8, %9 + %else + CHECK_AVX_INSTR_EMU {%1 %6, %7, %8}, %6, %8 + %endif + %if %5 && %4 == 0 + %ifnid %8 + ; 3-operand AVX instructions with a memory arg can only have it in src2, + ; whereas SSE emulation prefers to have it in src1 (i.e. the mov). + ; So, if the instruction is commutative with a memory arg, swap them. + %xdefine __src1 %8 + %xdefine __src2 %7 + %endif + %endif + %if __sizeofreg == 8 + MOVQ %6, __src1 + %elif %3 + MOVAPS %6, __src1 + %else + MOVDQA %6, __src1 + %endif + %endif + %if %0 >= 9 + %1 %6, __src2, %9 + %else + %1 %6, __src2 + %endif + %elif %0 >= 9 + __instr %6, %7, %8, %9 + %elif %0 == 8 + __instr %6, %7, %8 + %elif %0 == 7 + __instr %6, %7 + %else + __instr %6 + %endif +%endmacro + +;%1 == instruction +;%2 == minimal instruction set +;%3 == 1 if float, 0 if int +;%4 == 1 if non-destructive or 4-operand (xmm, xmm, xmm, imm), 0 otherwise +;%5 == 1 if commutative (i.e. doesn't matter which src arg is which), 0 if not +%macro AVX_INSTR 1-5 fnord, 0, 1, 0 + %macro %1 1-10 fnord, fnord, fnord, fnord, %1, %2, %3, %4, %5 + %ifidn %2, fnord + RUN_AVX_INSTR %6, %7, %8, %9, %10, %1 + %elifidn %3, fnord + RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2 + %elifidn %4, fnord + RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2, %3 + %elifidn %5, fnord + RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2, %3, %4 + %else + RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2, %3, %4, %5 + %endif + %endmacro +%endmacro + +; Instructions with both VEX and non-VEX encodings +; Non-destructive instructions are written without parameters +AVX_INSTR addpd, sse2, 1, 0, 1 +AVX_INSTR addps, sse, 1, 0, 1 +AVX_INSTR addsd, sse2, 1, 0, 1 +AVX_INSTR addss, sse, 1, 0, 1 +AVX_INSTR addsubpd, sse3, 1, 0, 0 +AVX_INSTR addsubps, sse3, 1, 0, 0 +AVX_INSTR aesdec, fnord, 0, 0, 0 +AVX_INSTR aesdeclast, fnord, 0, 0, 0 +AVX_INSTR aesenc, fnord, 0, 0, 0 +AVX_INSTR aesenclast, fnord, 0, 0, 0 +AVX_INSTR aesimc +AVX_INSTR aeskeygenassist +AVX_INSTR andnpd, sse2, 1, 0, 0 +AVX_INSTR andnps, sse, 1, 0, 0 +AVX_INSTR andpd, sse2, 1, 0, 1 +AVX_INSTR andps, sse, 1, 0, 1 +AVX_INSTR blendpd, sse4, 1, 0, 0 +AVX_INSTR blendps, sse4, 1, 0, 0 +AVX_INSTR blendvpd, sse4, 1, 0, 0 +AVX_INSTR blendvps, sse4, 1, 0, 0 +AVX_INSTR cmppd, sse2, 1, 1, 0 +AVX_INSTR cmpps, sse, 1, 1, 0 +AVX_INSTR cmpsd, sse2, 1, 1, 0 +AVX_INSTR cmpss, sse, 1, 1, 0 +AVX_INSTR comisd, sse2 +AVX_INSTR comiss, sse +AVX_INSTR cvtdq2pd, sse2 +AVX_INSTR cvtdq2ps, sse2 +AVX_INSTR cvtpd2dq, sse2 +AVX_INSTR cvtpd2ps, sse2 +AVX_INSTR cvtps2dq, sse2 +AVX_INSTR cvtps2pd, sse2 +AVX_INSTR cvtsd2si, sse2 +AVX_INSTR cvtsd2ss, sse2 +AVX_INSTR cvtsi2sd, sse2 +AVX_INSTR cvtsi2ss, sse +AVX_INSTR cvtss2sd, sse2 +AVX_INSTR cvtss2si, sse +AVX_INSTR cvttpd2dq, sse2 +AVX_INSTR cvttps2dq, sse2 +AVX_INSTR cvttsd2si, sse2 +AVX_INSTR cvttss2si, sse +AVX_INSTR divpd, sse2, 1, 0, 0 +AVX_INSTR divps, sse, 1, 0, 0 +AVX_INSTR divsd, sse2, 1, 0, 0 +AVX_INSTR divss, sse, 1, 0, 0 +AVX_INSTR dppd, sse4, 1, 1, 0 +AVX_INSTR dpps, sse4, 1, 1, 0 +AVX_INSTR extractps, sse4 +AVX_INSTR haddpd, sse3, 1, 0, 0 +AVX_INSTR haddps, sse3, 1, 0, 0 +AVX_INSTR hsubpd, sse3, 1, 0, 0 +AVX_INSTR hsubps, sse3, 1, 0, 0 +AVX_INSTR insertps, sse4, 1, 1, 0 +AVX_INSTR lddqu, sse3 +AVX_INSTR ldmxcsr, sse +AVX_INSTR maskmovdqu, sse2 +AVX_INSTR maxpd, sse2, 1, 0, 1 +AVX_INSTR maxps, sse, 1, 0, 1 +AVX_INSTR maxsd, sse2, 1, 0, 1 +AVX_INSTR maxss, sse, 1, 0, 1 +AVX_INSTR minpd, sse2, 1, 0, 1 +AVX_INSTR minps, sse, 1, 0, 1 +AVX_INSTR minsd, sse2, 1, 0, 1 +AVX_INSTR minss, sse, 1, 0, 1 +AVX_INSTR movapd, sse2 +AVX_INSTR movaps, sse +AVX_INSTR movd, mmx +AVX_INSTR movddup, sse3 +AVX_INSTR movdqa, sse2 +AVX_INSTR movdqu, sse2 +AVX_INSTR movhlps, sse, 1, 0, 0 +AVX_INSTR movhpd, sse2, 1, 0, 0 +AVX_INSTR movhps, sse, 1, 0, 0 +AVX_INSTR movlhps, sse, 1, 0, 0 +AVX_INSTR movlpd, sse2, 1, 0, 0 +AVX_INSTR movlps, sse, 1, 0, 0 +AVX_INSTR movmskpd, sse2 +AVX_INSTR movmskps, sse +AVX_INSTR movntdq, sse2 +AVX_INSTR movntdqa, sse4 +AVX_INSTR movntpd, sse2 +AVX_INSTR movntps, sse +AVX_INSTR movq, mmx +AVX_INSTR movsd, sse2, 1, 0, 0 +AVX_INSTR movshdup, sse3 +AVX_INSTR movsldup, sse3 +AVX_INSTR movss, sse, 1, 0, 0 +AVX_INSTR movupd, sse2 +AVX_INSTR movups, sse +AVX_INSTR mpsadbw, sse4 +AVX_INSTR mulpd, sse2, 1, 0, 1 +AVX_INSTR mulps, sse, 1, 0, 1 +AVX_INSTR mulsd, sse2, 1, 0, 1 +AVX_INSTR mulss, sse, 1, 0, 1 +AVX_INSTR orpd, sse2, 1, 0, 1 +AVX_INSTR orps, sse, 1, 0, 1 +AVX_INSTR pabsb, ssse3 +AVX_INSTR pabsd, ssse3 +AVX_INSTR pabsw, ssse3 +AVX_INSTR packsswb, mmx, 0, 0, 0 +AVX_INSTR packssdw, mmx, 0, 0, 0 +AVX_INSTR packuswb, mmx, 0, 0, 0 +AVX_INSTR packusdw, sse4, 0, 0, 0 +AVX_INSTR paddb, mmx, 0, 0, 1 +AVX_INSTR paddw, mmx, 0, 0, 1 +AVX_INSTR paddd, mmx, 0, 0, 1 +AVX_INSTR paddq, sse2, 0, 0, 1 +AVX_INSTR paddsb, mmx, 0, 0, 1 +AVX_INSTR paddsw, mmx, 0, 0, 1 +AVX_INSTR paddusb, mmx, 0, 0, 1 +AVX_INSTR paddusw, mmx, 0, 0, 1 +AVX_INSTR palignr, ssse3 +AVX_INSTR pand, mmx, 0, 0, 1 +AVX_INSTR pandn, mmx, 0, 0, 0 +AVX_INSTR pavgb, mmx2, 0, 0, 1 +AVX_INSTR pavgw, mmx2, 0, 0, 1 +AVX_INSTR pblendvb, sse4, 0, 0, 0 +AVX_INSTR pblendw, sse4 +AVX_INSTR pclmulqdq +AVX_INSTR pcmpestri, sse42 +AVX_INSTR pcmpestrm, sse42 +AVX_INSTR pcmpistri, sse42 +AVX_INSTR pcmpistrm, sse42 +AVX_INSTR pcmpeqb, mmx, 0, 0, 1 +AVX_INSTR pcmpeqw, mmx, 0, 0, 1 +AVX_INSTR pcmpeqd, mmx, 0, 0, 1 +AVX_INSTR pcmpeqq, sse4, 0, 0, 1 +AVX_INSTR pcmpgtb, mmx, 0, 0, 0 +AVX_INSTR pcmpgtw, mmx, 0, 0, 0 +AVX_INSTR pcmpgtd, mmx, 0, 0, 0 +AVX_INSTR pcmpgtq, sse42, 0, 0, 0 +AVX_INSTR pextrb, sse4 +AVX_INSTR pextrd, sse4 +AVX_INSTR pextrq, sse4 +AVX_INSTR pextrw, mmx2 +AVX_INSTR phaddw, ssse3, 0, 0, 0 +AVX_INSTR phaddd, ssse3, 0, 0, 0 +AVX_INSTR phaddsw, ssse3, 0, 0, 0 +AVX_INSTR phminposuw, sse4 +AVX_INSTR phsubw, ssse3, 0, 0, 0 +AVX_INSTR phsubd, ssse3, 0, 0, 0 +AVX_INSTR phsubsw, ssse3, 0, 0, 0 +AVX_INSTR pinsrb, sse4 +AVX_INSTR pinsrd, sse4 +AVX_INSTR pinsrq, sse4 +AVX_INSTR pinsrw, mmx2 +AVX_INSTR pmaddwd, mmx, 0, 0, 1 +AVX_INSTR pmaddubsw, ssse3, 0, 0, 0 +AVX_INSTR pmaxsb, sse4, 0, 0, 1 +AVX_INSTR pmaxsw, mmx2, 0, 0, 1 +AVX_INSTR pmaxsd, sse4, 0, 0, 1 +AVX_INSTR pmaxub, mmx2, 0, 0, 1 +AVX_INSTR pmaxuw, sse4, 0, 0, 1 +AVX_INSTR pmaxud, sse4, 0, 0, 1 +AVX_INSTR pminsb, sse4, 0, 0, 1 +AVX_INSTR pminsw, mmx2, 0, 0, 1 +AVX_INSTR pminsd, sse4, 0, 0, 1 +AVX_INSTR pminub, mmx2, 0, 0, 1 +AVX_INSTR pminuw, sse4, 0, 0, 1 +AVX_INSTR pminud, sse4, 0, 0, 1 +AVX_INSTR pmovmskb, mmx2 +AVX_INSTR pmovsxbw, sse4 +AVX_INSTR pmovsxbd, sse4 +AVX_INSTR pmovsxbq, sse4 +AVX_INSTR pmovsxwd, sse4 +AVX_INSTR pmovsxwq, sse4 +AVX_INSTR pmovsxdq, sse4 +AVX_INSTR pmovzxbw, sse4 +AVX_INSTR pmovzxbd, sse4 +AVX_INSTR pmovzxbq, sse4 +AVX_INSTR pmovzxwd, sse4 +AVX_INSTR pmovzxwq, sse4 +AVX_INSTR pmovzxdq, sse4 +AVX_INSTR pmuldq, sse4, 0, 0, 1 +AVX_INSTR pmulhrsw, ssse3, 0, 0, 1 +AVX_INSTR pmulhuw, mmx2, 0, 0, 1 +AVX_INSTR pmulhw, mmx, 0, 0, 1 +AVX_INSTR pmullw, mmx, 0, 0, 1 +AVX_INSTR pmulld, sse4, 0, 0, 1 +AVX_INSTR pmuludq, sse2, 0, 0, 1 +AVX_INSTR por, mmx, 0, 0, 1 +AVX_INSTR psadbw, mmx2, 0, 0, 1 +AVX_INSTR pshufb, ssse3, 0, 0, 0 +AVX_INSTR pshufd, sse2 +AVX_INSTR pshufhw, sse2 +AVX_INSTR pshuflw, sse2 +AVX_INSTR psignb, ssse3, 0, 0, 0 +AVX_INSTR psignw, ssse3, 0, 0, 0 +AVX_INSTR psignd, ssse3, 0, 0, 0 +AVX_INSTR psllw, mmx, 0, 0, 0 +AVX_INSTR pslld, mmx, 0, 0, 0 +AVX_INSTR psllq, mmx, 0, 0, 0 +AVX_INSTR pslldq, sse2, 0, 0, 0 +AVX_INSTR psraw, mmx, 0, 0, 0 +AVX_INSTR psrad, mmx, 0, 0, 0 +AVX_INSTR psrlw, mmx, 0, 0, 0 +AVX_INSTR psrld, mmx, 0, 0, 0 +AVX_INSTR psrlq, mmx, 0, 0, 0 +AVX_INSTR psrldq, sse2, 0, 0, 0 +AVX_INSTR psubb, mmx, 0, 0, 0 +AVX_INSTR psubw, mmx, 0, 0, 0 +AVX_INSTR psubd, mmx, 0, 0, 0 +AVX_INSTR psubq, sse2, 0, 0, 0 +AVX_INSTR psubsb, mmx, 0, 0, 0 +AVX_INSTR psubsw, mmx, 0, 0, 0 +AVX_INSTR psubusb, mmx, 0, 0, 0 +AVX_INSTR psubusw, mmx, 0, 0, 0 +AVX_INSTR ptest, sse4 +AVX_INSTR punpckhbw, mmx, 0, 0, 0 +AVX_INSTR punpckhwd, mmx, 0, 0, 0 +AVX_INSTR punpckhdq, mmx, 0, 0, 0 +AVX_INSTR punpckhqdq, sse2, 0, 0, 0 +AVX_INSTR punpcklbw, mmx, 0, 0, 0 +AVX_INSTR punpcklwd, mmx, 0, 0, 0 +AVX_INSTR punpckldq, mmx, 0, 0, 0 +AVX_INSTR punpcklqdq, sse2, 0, 0, 0 +AVX_INSTR pxor, mmx, 0, 0, 1 +AVX_INSTR rcpps, sse, 1, 0, 0 +AVX_INSTR rcpss, sse, 1, 0, 0 +AVX_INSTR roundpd, sse4 +AVX_INSTR roundps, sse4 +AVX_INSTR roundsd, sse4 +AVX_INSTR roundss, sse4 +AVX_INSTR rsqrtps, sse, 1, 0, 0 +AVX_INSTR rsqrtss, sse, 1, 0, 0 +AVX_INSTR shufpd, sse2, 1, 1, 0 +AVX_INSTR shufps, sse, 1, 1, 0 +AVX_INSTR sqrtpd, sse2, 1, 0, 0 +AVX_INSTR sqrtps, sse, 1, 0, 0 +AVX_INSTR sqrtsd, sse2, 1, 0, 0 +AVX_INSTR sqrtss, sse, 1, 0, 0 +AVX_INSTR stmxcsr, sse +AVX_INSTR subpd, sse2, 1, 0, 0 +AVX_INSTR subps, sse, 1, 0, 0 +AVX_INSTR subsd, sse2, 1, 0, 0 +AVX_INSTR subss, sse, 1, 0, 0 +AVX_INSTR ucomisd, sse2 +AVX_INSTR ucomiss, sse +AVX_INSTR unpckhpd, sse2, 1, 0, 0 +AVX_INSTR unpckhps, sse, 1, 0, 0 +AVX_INSTR unpcklpd, sse2, 1, 0, 0 +AVX_INSTR unpcklps, sse, 1, 0, 0 +AVX_INSTR xorpd, sse2, 1, 0, 1 +AVX_INSTR xorps, sse, 1, 0, 1 + +; 3DNow instructions, for sharing code between AVX, SSE and 3DN +AVX_INSTR pfadd, 3dnow, 1, 0, 1 +AVX_INSTR pfsub, 3dnow, 1, 0, 0 +AVX_INSTR pfmul, 3dnow, 1, 0, 1 + +; base-4 constants for shuffles +%assign i 0 +%rep 256 + %assign j ((i>>6)&3)*1000 + ((i>>4)&3)*100 + ((i>>2)&3)*10 + (i&3) + %if j < 10 + CAT_XDEFINE q000, j, i + %elif j < 100 + CAT_XDEFINE q00, j, i + %elif j < 1000 + CAT_XDEFINE q0, j, i + %else + CAT_XDEFINE q, j, i + %endif + %assign i i+1 +%endrep +%undef i +%undef j + +%macro FMA_INSTR 3 + %macro %1 4-7 %1, %2, %3 + %if cpuflag(xop) + v%5 %1, %2, %3, %4 + %elifnidn %1, %4 + %6 %1, %2, %3 + %7 %1, %4 + %else + %error non-xop emulation of ``%5 %1, %2, %3, %4'' is not supported + %endif + %endmacro +%endmacro + +FMA_INSTR pmacsww, pmullw, paddw +FMA_INSTR pmacsdd, pmulld, paddd ; sse4 emulation +FMA_INSTR pmacsdql, pmuldq, paddq ; sse4 emulation +FMA_INSTR pmadcswd, pmaddwd, paddd + +; Macros for consolidating FMA3 and FMA4 using 4-operand (dst, src1, src2, src3) syntax. +; FMA3 is only possible if dst is the same as one of the src registers. +; Either src2 or src3 can be a memory operand. +%macro FMA4_INSTR 2-* + %push fma4_instr + %xdefine %$prefix %1 + %rep %0 - 1 + %macro %$prefix%2 4-6 %$prefix, %2 + %if notcpuflag(fma3) && notcpuflag(fma4) + %error use of ``%5%6'' fma instruction in cpuname function: current_function + %elif cpuflag(fma4) + v%5%6 %1, %2, %3, %4 + %elifidn %1, %2 + ; If %3 or %4 is a memory operand it needs to be encoded as the last operand. + %ifid %3 + v%{5}213%6 %2, %3, %4 + %else + v%{5}132%6 %2, %4, %3 + %endif + %elifidn %1, %3 + v%{5}213%6 %3, %2, %4 + %elifidn %1, %4 + v%{5}231%6 %4, %2, %3 + %else + %error fma3 emulation of ``%5%6 %1, %2, %3, %4'' is not supported + %endif + %endmacro + %rotate 1 + %endrep + %pop +%endmacro + +FMA4_INSTR fmadd, pd, ps, sd, ss +FMA4_INSTR fmaddsub, pd, ps +FMA4_INSTR fmsub, pd, ps, sd, ss +FMA4_INSTR fmsubadd, pd, ps +FMA4_INSTR fnmadd, pd, ps, sd, ss +FMA4_INSTR fnmsub, pd, ps, sd, ss + +; workaround: vpbroadcastq is broken in x86_32 due to a yasm bug (fixed in 1.3.0) +%ifdef __YASM_VER__ + %if __YASM_VERSION_ID__ < 0x01030000 && ARCH_X86_64 == 0 + %macro vpbroadcastq 2 + %if sizeof%1 == 16 + movddup %1, %2 + %else + vbroadcastsd %1, %2 + %endif + %endmacro + %endif +%endif diff --git a/third_party/aom/tools/all_builds.py b/third_party/aom/tools/all_builds.py new file mode 100755 index 0000000000..d1f0c80c03 --- /dev/null +++ b/third_party/aom/tools/all_builds.py @@ -0,0 +1,72 @@ +#!/usr/bin/python + +import getopt +import subprocess +import sys + +LONG_OPTIONS = ["shard=", "shards="] +BASE_COMMAND = "./configure --enable-internal-stats --enable-experimental" + +def RunCommand(command): + run = subprocess.Popen(command, shell=True) + output = run.communicate() + if run.returncode: + print "Non-zero return code: " + str(run.returncode) + " => exiting!" + sys.exit(1) + +def list_of_experiments(): + experiments = [] + configure_file = open("configure") + list_start = False + for line in configure_file.read().split("\n"): + if line == 'EXPERIMENT_LIST="': + list_start = True + elif line == '"': + list_start = False + elif list_start: + currently_broken = ["csm"] + experiment = line[4:] + if experiment not in currently_broken: + experiments.append(experiment) + return experiments + +def main(argv): + # Parse arguments + options = {"--shard": 0, "--shards": 1} + if "--" in argv: + opt_end_index = argv.index("--") + else: + opt_end_index = len(argv) + try: + o, _ = getopt.getopt(argv[1:opt_end_index], None, LONG_OPTIONS) + except getopt.GetoptError, err: + print str(err) + print "Usage: %s [--shard= --shards=] -- [configure flag ...]"%argv[0] + sys.exit(2) + + options.update(o) + extra_args = argv[opt_end_index + 1:] + + # Shard experiment list + shard = int(options["--shard"]) + shards = int(options["--shards"]) + experiments = list_of_experiments() + base_command = " ".join([BASE_COMMAND] + extra_args) + configs = [base_command] + configs += ["%s --enable-%s" % (base_command, e) for e in experiments] + my_configs = zip(configs, range(len(configs))) + my_configs = filter(lambda x: x[1] % shards == shard, my_configs) + my_configs = [e[0] for e in my_configs] + + # Run configs for this shard + for config in my_configs: + test_build(config) + +def test_build(configure_command): + print "\033[34m\033[47mTesting %s\033[0m" % (configure_command) + RunCommand(configure_command) + RunCommand("make clean") + RunCommand("make") + +if __name__ == "__main__": + main(sys.argv) diff --git a/third_party/aom/tools/author_first_release.sh b/third_party/aom/tools/author_first_release.sh new file mode 100755 index 0000000000..7b0b797212 --- /dev/null +++ b/third_party/aom/tools/author_first_release.sh @@ -0,0 +1,15 @@ +#!/bin/bash +## +## List the release each author first contributed to. +## +## Usage: author_first_release.sh [TAGS] +## +## If the TAGS arguments are unspecified, all tags reported by `git tag` +## will be considered. +## +tags=${@:-$(git tag)} +for tag in $tags; do + git shortlog -n -e -s $tag | + cut -f2- | + awk "{print \"${tag#v}\t\"\$0}" +done | sort -k2 | uniq -f2 diff --git a/third_party/aom/tools/build_inspector.sh b/third_party/aom/tools/build_inspector.sh new file mode 100755 index 0000000000..a67aa15f6c --- /dev/null +++ b/third_party/aom/tools/build_inspector.sh @@ -0,0 +1,16 @@ +if ! [ -x "$(command -v emcc)" ] || ! [ -x "$(command -v emconfigure)" ] || ! [ -x "$(command -v emmake)" ]; then + echo 'Emscripten SDK is not available (emcc, emconfigure or emmake is missing). Install it from https://kripken.github.io/emscripten-site/docs/getting_started/downloads.html and try again.' >&2 + exit 1 +fi + +echo 'Building JS Inspector' +if [ ! -d ".inspect" ]; then + mkdir .inspect + cd .inspect && emconfigure ../../configure --disable-multithread --disable-runtime-cpu-detect --target=generic-gnu --enable-accounting --enable-inspection --enable-aom_highbitdepth --extra-cflags="-D_POSIX_SOURCE" +fi + +cd .inspect +emmake make -j 8 +cp examples/inspect inspect.bc +emcc -O3 inspect.bc -o inspect.js -s TOTAL_MEMORY=134217728 -s MODULARIZE=1 -s EXPORT_NAME="'DecoderModule'" --post-js "../inspect-post.js" --memory-init-file 0 +cp inspect.js ../inspect.js diff --git a/third_party/aom/tools/cpplint.py b/third_party/aom/tools/cpplint.py new file mode 100755 index 0000000000..25fbef73d8 --- /dev/null +++ b/third_party/aom/tools/cpplint.py @@ -0,0 +1,4756 @@ +#!/usr/bin/python +# +# Copyright (c) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Does google-lint on c++ files. + +The goal of this script is to identify places in the code that *may* +be in non-compliance with google style. It does not attempt to fix +up these problems -- the point is to educate. It does also not +attempt to find all problems, or to ensure that everything it does +find is legitimately a problem. + +In particular, we can get very confused by /* and // inside strings! +We do a small hack, which is to ignore //'s with "'s after them on the +same line, but it is far from perfect (in either direction). +""" + +import codecs +import copy +import getopt +import math # for log +import os +import re +import sre_compile +import string +import sys +import unicodedata + + +_USAGE = """ +Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] + [--counting=total|toplevel|detailed] [--root=subdir] + [--linelength=digits] + [file] ... + + The style guidelines this tries to follow are those in + http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml + + Every problem is given a confidence score from 1-5, with 5 meaning we are + certain of the problem, and 1 meaning it could be a legitimate construct. + This will miss some errors, and is not a substitute for a code review. + + To suppress false-positive errors of a certain category, add a + 'NOLINT(category)' comment to the line. NOLINT or NOLINT(*) + suppresses errors of all categories on that line. + + The files passed in will be linted; at least one file must be provided. + Default linted extensions are .cc, .cpp, .cu, .cuh and .h. Change the + extensions with the --extensions flag. + + Flags: + + output=vs7 + By default, the output is formatted to ease emacs parsing. Visual Studio + compatible output (vs7) may also be used. Other formats are unsupported. + + verbose=# + Specify a number 0-5 to restrict errors to certain verbosity levels. + + filter=-x,+y,... + Specify a comma-separated list of category-filters to apply: only + error messages whose category names pass the filters will be printed. + (Category names are printed with the message and look like + "[whitespace/indent]".) Filters are evaluated left to right. + "-FOO" and "FOO" means "do not print categories that start with FOO". + "+FOO" means "do print categories that start with FOO". + + Examples: --filter=-whitespace,+whitespace/braces + --filter=whitespace,runtime/printf,+runtime/printf_format + --filter=-,+build/include_what_you_use + + To see a list of all the categories used in cpplint, pass no arg: + --filter= + + counting=total|toplevel|detailed + The total number of errors found is always printed. If + 'toplevel' is provided, then the count of errors in each of + the top-level categories like 'build' and 'whitespace' will + also be printed. If 'detailed' is provided, then a count + is provided for each category like 'build/class'. + + root=subdir + The root directory used for deriving header guard CPP variable. + By default, the header guard CPP variable is calculated as the relative + path to the directory that contains .git, .hg, or .svn. When this flag + is specified, the relative path is calculated from the specified + directory. If the specified directory does not exist, this flag is + ignored. + + Examples: + Assuing that src/.git exists, the header guard CPP variables for + src/chrome/browser/ui/browser.h are: + + No flag => CHROME_BROWSER_UI_BROWSER_H_ + --root=chrome => BROWSER_UI_BROWSER_H_ + --root=chrome/browser => UI_BROWSER_H_ + + linelength=digits + This is the allowed line length for the project. The default value is + 80 characters. + + Examples: + --linelength=120 + + extensions=extension,extension,... + The allowed file extensions that cpplint will check + + Examples: + --extensions=hpp,cpp +""" + +# We categorize each error message we print. Here are the categories. +# We want an explicit list so we can list them all in cpplint --filter=. +# If you add a new error message with a new category, add it to the list +# here! cpplint_unittest.py should tell you if you forget to do this. +_ERROR_CATEGORIES = [ + 'build/class', + 'build/deprecated', + 'build/endif_comment', + 'build/explicit_make_pair', + 'build/forward_decl', + 'build/header_guard', + 'build/include', + 'build/include_alpha', + 'build/include_order', + 'build/include_what_you_use', + 'build/namespaces', + 'build/printf_format', + 'build/storage_class', + 'legal/copyright', + 'readability/alt_tokens', + 'readability/braces', + 'readability/casting', + 'readability/check', + 'readability/constructors', + 'readability/fn_size', + 'readability/function', + 'readability/multiline_comment', + 'readability/multiline_string', + 'readability/namespace', + 'readability/nolint', + 'readability/nul', + 'readability/streams', + 'readability/todo', + 'readability/utf8', + 'runtime/arrays', + 'runtime/casting', + 'runtime/explicit', + 'runtime/int', + 'runtime/init', + 'runtime/invalid_increment', + 'runtime/member_string_references', + 'runtime/memset', + 'runtime/operator', + 'runtime/printf', + 'runtime/printf_format', + 'runtime/references', + 'runtime/sizeof', + 'runtime/string', + 'runtime/threadsafe_fn', + 'runtime/vlog', + 'whitespace/blank_line', + 'whitespace/braces', + 'whitespace/comma', + 'whitespace/comments', + 'whitespace/empty_conditional_body', + 'whitespace/empty_loop_body', + 'whitespace/end_of_line', + 'whitespace/ending_newline', + 'whitespace/forcolon', + 'whitespace/indent', + 'whitespace/line_length', + 'whitespace/newline', + 'whitespace/operators', + 'whitespace/parens', + 'whitespace/semicolon', + 'whitespace/tab', + 'whitespace/todo' + ] + +# The default state of the category filter. This is overrided by the --filter= +# flag. By default all errors are on, so only add here categories that should be +# off by default (i.e., categories that must be enabled by the --filter= flags). +# All entries here should start with a '-' or '+', as in the --filter= flag. +_DEFAULT_FILTERS = ['-build/include_alpha'] + +# We used to check for high-bit characters, but after much discussion we +# decided those were OK, as long as they were in UTF-8 and didn't represent +# hard-coded international strings, which belong in a separate i18n file. + + +# C++ headers +_CPP_HEADERS = frozenset([ + # Legacy + 'algobase.h', + 'algo.h', + 'alloc.h', + 'builtinbuf.h', + 'bvector.h', + 'complex.h', + 'defalloc.h', + 'deque.h', + 'editbuf.h', + 'fstream.h', + 'function.h', + 'hash_map', + 'hash_map.h', + 'hash_set', + 'hash_set.h', + 'hashtable.h', + 'heap.h', + 'indstream.h', + 'iomanip.h', + 'iostream.h', + 'istream.h', + 'iterator.h', + 'list.h', + 'map.h', + 'multimap.h', + 'multiset.h', + 'ostream.h', + 'pair.h', + 'parsestream.h', + 'pfstream.h', + 'procbuf.h', + 'pthread_alloc', + 'pthread_alloc.h', + 'rope', + 'rope.h', + 'ropeimpl.h', + 'set.h', + 'slist', + 'slist.h', + 'stack.h', + 'stdiostream.h', + 'stl_alloc.h', + 'stl_relops.h', + 'streambuf.h', + 'stream.h', + 'strfile.h', + 'strstream.h', + 'tempbuf.h', + 'tree.h', + 'type_traits.h', + 'vector.h', + # 17.6.1.2 C++ library headers + 'algorithm', + 'array', + 'atomic', + 'bitset', + 'chrono', + 'codecvt', + 'complex', + 'condition_variable', + 'deque', + 'exception', + 'forward_list', + 'fstream', + 'functional', + 'future', + 'initializer_list', + 'iomanip', + 'ios', + 'iosfwd', + 'iostream', + 'istream', + 'iterator', + 'limits', + 'list', + 'locale', + 'map', + 'memory', + 'mutex', + 'new', + 'numeric', + 'ostream', + 'queue', + 'random', + 'ratio', + 'regex', + 'set', + 'sstream', + 'stack', + 'stdexcept', + 'streambuf', + 'string', + 'strstream', + 'system_error', + 'thread', + 'tuple', + 'typeindex', + 'typeinfo', + 'type_traits', + 'unordered_map', + 'unordered_set', + 'utility', + 'valarray', + 'vector', + # 17.6.1.2 C++ headers for C library facilities + 'cassert', + 'ccomplex', + 'cctype', + 'cerrno', + 'cfenv', + 'cfloat', + 'cinttypes', + 'ciso646', + 'climits', + 'clocale', + 'cmath', + 'csetjmp', + 'csignal', + 'cstdalign', + 'cstdarg', + 'cstdbool', + 'cstddef', + 'cstdint', + 'cstdio', + 'cstdlib', + 'cstring', + 'ctgmath', + 'ctime', + 'cuchar', + 'cwchar', + 'cwctype', + ]) + +# Assertion macros. These are defined in base/logging.h and +# testing/base/gunit.h. Note that the _M versions need to come first +# for substring matching to work. +_CHECK_MACROS = [ + 'DCHECK', 'CHECK', + 'EXPECT_TRUE_M', 'EXPECT_TRUE', + 'ASSERT_TRUE_M', 'ASSERT_TRUE', + 'EXPECT_FALSE_M', 'EXPECT_FALSE', + 'ASSERT_FALSE_M', 'ASSERT_FALSE', + ] + +# Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE +_CHECK_REPLACEMENT = dict([(m, {}) for m in _CHECK_MACROS]) + +for op, replacement in [('==', 'EQ'), ('!=', 'NE'), + ('>=', 'GE'), ('>', 'GT'), + ('<=', 'LE'), ('<', 'LT')]: + _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement + _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE_M'][op] = 'EXPECT_%s_M' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE_M'][op] = 'ASSERT_%s_M' % replacement + +for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), + ('>=', 'LT'), ('>', 'LE'), + ('<=', 'GT'), ('<', 'GE')]: + _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement + _CHECK_REPLACEMENT['EXPECT_FALSE_M'][op] = 'EXPECT_%s_M' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE_M'][op] = 'ASSERT_%s_M' % inv_replacement + +# Alternative tokens and their replacements. For full list, see section 2.5 +# Alternative tokens [lex.digraph] in the C++ standard. +# +# Digraphs (such as '%:') are not included here since it's a mess to +# match those on a word boundary. +_ALT_TOKEN_REPLACEMENT = { + 'and': '&&', + 'bitor': '|', + 'or': '||', + 'xor': '^', + 'compl': '~', + 'bitand': '&', + 'and_eq': '&=', + 'or_eq': '|=', + 'xor_eq': '^=', + 'not': '!', + 'not_eq': '!=' + } + +# Compile regular expression that matches all the above keywords. The "[ =()]" +# bit is meant to avoid matching these keywords outside of boolean expressions. +# +# False positives include C-style multi-line comments and multi-line strings +# but those have always been troublesome for cpplint. +_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile( + r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)') + + +# These constants define types of headers for use with +# _IncludeState.CheckNextIncludeOrder(). +_C_SYS_HEADER = 1 +_CPP_SYS_HEADER = 2 +_LIKELY_MY_HEADER = 3 +_POSSIBLE_MY_HEADER = 4 +_OTHER_HEADER = 5 + +# These constants define the current inline assembly state +_NO_ASM = 0 # Outside of inline assembly block +_INSIDE_ASM = 1 # Inside inline assembly block +_END_ASM = 2 # Last line of inline assembly block +_BLOCK_ASM = 3 # The whole block is an inline assembly block + +# Match start of assembly blocks +_MATCH_ASM = re.compile(r'^\s*(?:asm|_asm|__asm|__asm__)' + r'(?:\s+(volatile|__volatile__))?' + r'\s*[{(]') + + +_regexp_compile_cache = {} + +# Finds occurrences of NOLINT or NOLINT(...). +_RE_SUPPRESSION = re.compile(r'\bNOLINT\b(\([^)]*\))?') + +# {str, set(int)}: a map from error categories to sets of linenumbers +# on which those errors are expected and should be suppressed. +_error_suppressions = {} + +# The root directory used for deriving header guard CPP variable. +# This is set by --root flag. +_root = None + +# The allowed line length of files. +# This is set by --linelength flag. +_line_length = 80 + +# The allowed extensions for file names +# This is set by --extensions flag. +_valid_extensions = set(['cc', 'h', 'cpp', 'cu', 'cuh']) + +def ParseNolintSuppressions(filename, raw_line, linenum, error): + """Updates the global list of error-suppressions. + + Parses any NOLINT comments on the current line, updating the global + error_suppressions store. Reports an error if the NOLINT comment + was malformed. + + Args: + filename: str, the name of the input file. + raw_line: str, the line of input text, with comments. + linenum: int, the number of the current line. + error: function, an error handler. + """ + # FIXME(adonovan): "NOLINT(" is misparsed as NOLINT(*). + matched = _RE_SUPPRESSION.search(raw_line) + if matched: + category = matched.group(1) + if category in (None, '(*)'): # => "suppress all" + _error_suppressions.setdefault(None, set()).add(linenum) + else: + if category.startswith('(') and category.endswith(')'): + category = category[1:-1] + if category in _ERROR_CATEGORIES: + _error_suppressions.setdefault(category, set()).add(linenum) + else: + error(filename, linenum, 'readability/nolint', 5, + 'Unknown NOLINT error category: %s' % category) + + +def ResetNolintSuppressions(): + "Resets the set of NOLINT suppressions to empty." + _error_suppressions.clear() + + +def IsErrorSuppressedByNolint(category, linenum): + """Returns true if the specified error category is suppressed on this line. + + Consults the global error_suppressions map populated by + ParseNolintSuppressions/ResetNolintSuppressions. + + Args: + category: str, the category of the error. + linenum: int, the current line number. + Returns: + bool, True iff the error should be suppressed due to a NOLINT comment. + """ + return (linenum in _error_suppressions.get(category, set()) or + linenum in _error_suppressions.get(None, set())) + +def Match(pattern, s): + """Matches the string with the pattern, caching the compiled regexp.""" + # The regexp compilation caching is inlined in both Match and Search for + # performance reasons; factoring it out into a separate function turns out + # to be noticeably expensive. + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].match(s) + + +def ReplaceAll(pattern, rep, s): + """Replaces instances of pattern in a string with a replacement. + + The compiled regex is kept in a cache shared by Match and Search. + + Args: + pattern: regex pattern + rep: replacement text + s: search string + + Returns: + string with replacements made (or original string if no replacements) + """ + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].sub(rep, s) + + +def Search(pattern, s): + """Searches the string for the pattern, caching the compiled regexp.""" + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].search(s) + + +class _IncludeState(dict): + """Tracks line numbers for includes, and the order in which includes appear. + + As a dict, an _IncludeState object serves as a mapping between include + filename and line number on which that file was included. + + Call CheckNextIncludeOrder() once for each header in the file, passing + in the type constants defined above. Calls in an illegal order will + raise an _IncludeError with an appropriate error message. + + """ + # self._section will move monotonically through this set. If it ever + # needs to move backwards, CheckNextIncludeOrder will raise an error. + _INITIAL_SECTION = 0 + _MY_H_SECTION = 1 + _C_SECTION = 2 + _CPP_SECTION = 3 + _OTHER_H_SECTION = 4 + + _TYPE_NAMES = { + _C_SYS_HEADER: 'C system header', + _CPP_SYS_HEADER: 'C++ system header', + _LIKELY_MY_HEADER: 'header this file implements', + _POSSIBLE_MY_HEADER: 'header this file may implement', + _OTHER_HEADER: 'other header', + } + _SECTION_NAMES = { + _INITIAL_SECTION: "... nothing. (This can't be an error.)", + _MY_H_SECTION: 'a header this file implements', + _C_SECTION: 'C system header', + _CPP_SECTION: 'C++ system header', + _OTHER_H_SECTION: 'other header', + } + + def __init__(self): + dict.__init__(self) + self.ResetSection() + + def ResetSection(self): + # The name of the current section. + self._section = self._INITIAL_SECTION + # The path of last found header. + self._last_header = '' + + def SetLastHeader(self, header_path): + self._last_header = header_path + + def CanonicalizeAlphabeticalOrder(self, header_path): + """Returns a path canonicalized for alphabetical comparison. + + - replaces "-" with "_" so they both cmp the same. + - removes '-inl' since we don't require them to be after the main header. + - lowercase everything, just in case. + + Args: + header_path: Path to be canonicalized. + + Returns: + Canonicalized path. + """ + return header_path.replace('-inl.h', '.h').replace('-', '_').lower() + + def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): + """Check if a header is in alphabetical order with the previous header. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + header_path: Canonicalized header to be checked. + + Returns: + Returns true if the header is in alphabetical order. + """ + # If previous section is different from current section, _last_header will + # be reset to empty string, so it's always less than current header. + # + # If previous line was a blank line, assume that the headers are + # intentionally sorted the way they are. + if (self._last_header > header_path and + not Match(r'^\s*$', clean_lines.elided[linenum - 1])): + return False + return True + + def CheckNextIncludeOrder(self, header_type): + """Returns a non-empty error message if the next header is out of order. + + This function also updates the internal state to be ready to check + the next include. + + Args: + header_type: One of the _XXX_HEADER constants defined above. + + Returns: + The empty string if the header is in the right order, or an + error message describing what's wrong. + + """ + error_message = ('Found %s after %s' % + (self._TYPE_NAMES[header_type], + self._SECTION_NAMES[self._section])) + + last_section = self._section + + if header_type == _C_SYS_HEADER: + if self._section <= self._C_SECTION: + self._section = self._C_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _CPP_SYS_HEADER: + if self._section <= self._CPP_SECTION: + self._section = self._CPP_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _LIKELY_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + self._section = self._OTHER_H_SECTION + elif header_type == _POSSIBLE_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + # This will always be the fallback because we're not sure + # enough that the header is associated with this file. + self._section = self._OTHER_H_SECTION + else: + assert header_type == _OTHER_HEADER + self._section = self._OTHER_H_SECTION + + if last_section != self._section: + self._last_header = '' + + return '' + + +class _CppLintState(object): + """Maintains module-wide state..""" + + def __init__(self): + self.verbose_level = 1 # global setting. + self.error_count = 0 # global count of reported errors + # filters to apply when emitting error messages + self.filters = _DEFAULT_FILTERS[:] + self.counting = 'total' # In what way are we counting errors? + self.errors_by_category = {} # string to int dict storing error counts + + # output format: + # "emacs" - format that emacs can parse (default) + # "vs7" - format that Microsoft Visual Studio 7 can parse + self.output_format = 'emacs' + + def SetOutputFormat(self, output_format): + """Sets the output format for errors.""" + self.output_format = output_format + + def SetVerboseLevel(self, level): + """Sets the module's verbosity, and returns the previous setting.""" + last_verbose_level = self.verbose_level + self.verbose_level = level + return last_verbose_level + + def SetCountingStyle(self, counting_style): + """Sets the module's counting options.""" + self.counting = counting_style + + def SetFilters(self, filters): + """Sets the error-message filters. + + These filters are applied when deciding whether to emit a given + error message. + + Args: + filters: A string of comma-separated filters (eg "+whitespace/indent"). + Each filter should start with + or -; else we die. + + Raises: + ValueError: The comma-separated filters did not all start with '+' or '-'. + E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter" + """ + # Default filters always have less priority than the flag ones. + self.filters = _DEFAULT_FILTERS[:] + for filt in filters.split(','): + clean_filt = filt.strip() + if clean_filt: + self.filters.append(clean_filt) + for filt in self.filters: + if not (filt.startswith('+') or filt.startswith('-')): + raise ValueError('Every filter in --filters must start with + or -' + ' (%s does not)' % filt) + + def ResetErrorCounts(self): + """Sets the module's error statistic back to zero.""" + self.error_count = 0 + self.errors_by_category = {} + + def IncrementErrorCount(self, category): + """Bumps the module's error statistic.""" + self.error_count += 1 + if self.counting in ('toplevel', 'detailed'): + if self.counting != 'detailed': + category = category.split('/')[0] + if category not in self.errors_by_category: + self.errors_by_category[category] = 0 + self.errors_by_category[category] += 1 + + def PrintErrorCounts(self): + """Print a summary of errors by category, and the total.""" + for category, count in self.errors_by_category.iteritems(): + sys.stderr.write('Category \'%s\' errors found: %d\n' % + (category, count)) + sys.stderr.write('Total errors found: %d\n' % self.error_count) + +_cpplint_state = _CppLintState() + + +def _OutputFormat(): + """Gets the module's output format.""" + return _cpplint_state.output_format + + +def _SetOutputFormat(output_format): + """Sets the module's output format.""" + _cpplint_state.SetOutputFormat(output_format) + + +def _VerboseLevel(): + """Returns the module's verbosity setting.""" + return _cpplint_state.verbose_level + + +def _SetVerboseLevel(level): + """Sets the module's verbosity, and returns the previous setting.""" + return _cpplint_state.SetVerboseLevel(level) + + +def _SetCountingStyle(level): + """Sets the module's counting options.""" + _cpplint_state.SetCountingStyle(level) + + +def _Filters(): + """Returns the module's list of output filters, as a list.""" + return _cpplint_state.filters + + +def _SetFilters(filters): + """Sets the module's error-message filters. + + These filters are applied when deciding whether to emit a given + error message. + + Args: + filters: A string of comma-separated filters (eg "whitespace/indent"). + Each filter should start with + or -; else we die. + """ + _cpplint_state.SetFilters(filters) + + +class _FunctionState(object): + """Tracks current function name and the number of lines in its body.""" + + _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. + _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. + + def __init__(self): + self.in_a_function = False + self.lines_in_function = 0 + self.current_function = '' + + def Begin(self, function_name): + """Start analyzing function body. + + Args: + function_name: The name of the function being tracked. + """ + self.in_a_function = True + self.lines_in_function = 0 + self.current_function = function_name + + def Count(self): + """Count line in current function body.""" + if self.in_a_function: + self.lines_in_function += 1 + + def Check(self, error, filename, linenum): + """Report if too many lines in function body. + + Args: + error: The function to call with any errors found. + filename: The name of the current file. + linenum: The number of the line to check. + """ + if Match(r'T(EST|est)', self.current_function): + base_trigger = self._TEST_TRIGGER + else: + base_trigger = self._NORMAL_TRIGGER + trigger = base_trigger * 2**_VerboseLevel() + + if self.lines_in_function > trigger: + error_level = int(math.log(self.lines_in_function / base_trigger, 2)) + # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... + if error_level > 5: + error_level = 5 + error(filename, linenum, 'readability/fn_size', error_level, + 'Small and focused functions are preferred:' + ' %s has %d non-comment lines' + ' (error triggered by exceeding %d lines).' % ( + self.current_function, self.lines_in_function, trigger)) + + def End(self): + """Stop analyzing function body.""" + self.in_a_function = False + + +class _IncludeError(Exception): + """Indicates a problem with the include order in a file.""" + pass + + +class FileInfo: + """Provides utility functions for filenames. + + FileInfo provides easy access to the components of a file's path + relative to the project root. + """ + + def __init__(self, filename): + self._filename = filename + + def FullName(self): + """Make Windows paths like Unix.""" + return os.path.abspath(self._filename).replace('\\', '/') + + def RepositoryName(self): + """FullName after removing the local path to the repository. + + If we have a real absolute path name here we can try to do something smart: + detecting the root of the checkout and truncating /path/to/checkout from + the name so that we get header guards that don't include things like + "C:\Documents and Settings\..." or "/home/username/..." in them and thus + people on different computers who have checked the source out to different + locations won't see bogus errors. + """ + fullname = self.FullName() + + if os.path.exists(fullname): + project_dir = os.path.dirname(fullname) + + if os.path.exists(os.path.join(project_dir, ".svn")): + # If there's a .svn file in the current directory, we recursively look + # up the directory tree for the top of the SVN checkout + root_dir = project_dir + one_up_dir = os.path.dirname(root_dir) + while os.path.exists(os.path.join(one_up_dir, ".svn")): + root_dir = os.path.dirname(root_dir) + one_up_dir = os.path.dirname(one_up_dir) + + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by + # searching up from the current path. + root_dir = os.path.dirname(fullname) + while (root_dir != os.path.dirname(root_dir) and + not os.path.exists(os.path.join(root_dir, ".git")) and + not os.path.exists(os.path.join(root_dir, ".hg")) and + not os.path.exists(os.path.join(root_dir, ".svn"))): + root_dir = os.path.dirname(root_dir) + + if (os.path.exists(os.path.join(root_dir, ".git")) or + os.path.exists(os.path.join(root_dir, ".hg")) or + os.path.exists(os.path.join(root_dir, ".svn"))): + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Don't know what to do; header guard warnings may be wrong... + return fullname + + def Split(self): + """Splits the file into the directory, basename, and extension. + + For 'chrome/browser/browser.cc', Split() would + return ('chrome/browser', 'browser', '.cc') + + Returns: + A tuple of (directory, basename, extension). + """ + + googlename = self.RepositoryName() + project, rest = os.path.split(googlename) + return (project,) + os.path.splitext(rest) + + def BaseName(self): + """File base name - text after the final slash, before the final period.""" + return self.Split()[1] + + def Extension(self): + """File extension - text following the final period.""" + return self.Split()[2] + + def NoExtension(self): + """File has no source file extension.""" + return '/'.join(self.Split()[0:2]) + + def IsSource(self): + """File has a source file extension.""" + return self.Extension()[1:] in ('c', 'cc', 'cpp', 'cxx') + + +def _ShouldPrintError(category, confidence, linenum): + """If confidence >= verbose, category passes filter and is not suppressed.""" + + # There are three ways we might decide not to print an error message: + # a "NOLINT(category)" comment appears in the source, + # the verbosity level isn't high enough, or the filters filter it out. + if IsErrorSuppressedByNolint(category, linenum): + return False + if confidence < _cpplint_state.verbose_level: + return False + + is_filtered = False + for one_filter in _Filters(): + if one_filter.startswith('-'): + if category.startswith(one_filter[1:]): + is_filtered = True + elif one_filter.startswith('+'): + if category.startswith(one_filter[1:]): + is_filtered = False + else: + assert False # should have been checked for in SetFilter. + if is_filtered: + return False + + return True + + +def Error(filename, linenum, category, confidence, message): + """Logs the fact we've found a lint error. + + We log where the error was found, and also our confidence in the error, + that is, how certain we are this is a legitimate style regression, and + not a misidentification or a use that's sometimes justified. + + False positives can be suppressed by the use of + "cpplint(category)" comments on the offending line. These are + parsed into _error_suppressions. + + Args: + filename: The name of the file containing the error. + linenum: The number of the line containing the error. + category: A string used to describe the "category" this bug + falls under: "whitespace", say, or "runtime". Categories + may have a hierarchy separated by slashes: "whitespace/indent". + confidence: A number from 1-5 representing a confidence score for + the error, with 5 meaning that we are certain of the problem, + and 1 meaning that it could be a legitimate construct. + message: The error message. + """ + if _ShouldPrintError(category, confidence, linenum): + _cpplint_state.IncrementErrorCount(category) + if _cpplint_state.output_format == 'vs7': + sys.stderr.write('%s(%s): %s [%s] [%d]\n' % ( + filename, linenum, message, category, confidence)) + elif _cpplint_state.output_format == 'eclipse': + sys.stderr.write('%s:%s: warning: %s [%s] [%d]\n' % ( + filename, linenum, message, category, confidence)) + else: + sys.stderr.write('%s:%s: %s [%s] [%d]\n' % ( + filename, linenum, message, category, confidence)) + + +# Matches standard C++ escape sequences per 2.13.2.3 of the C++ standard. +_RE_PATTERN_CLEANSE_LINE_ESCAPES = re.compile( + r'\\([abfnrtv?"\\\']|\d+|x[0-9a-fA-F]+)') +# Matches strings. Escape codes should already be removed by ESCAPES. +_RE_PATTERN_CLEANSE_LINE_DOUBLE_QUOTES = re.compile(r'"[^"]*"') +# Matches characters. Escape codes should already be removed by ESCAPES. +_RE_PATTERN_CLEANSE_LINE_SINGLE_QUOTES = re.compile(r"'.'") +# Matches multi-line C++ comments. +# This RE is a little bit more complicated than one might expect, because we +# have to take care of space removals tools so we can handle comments inside +# statements better. +# The current rule is: We only clear spaces from both sides when we're at the +# end of the line. Otherwise, we try to remove spaces from the right side, +# if this doesn't work we try on left side but only if there's a non-character +# on the right. +_RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile( + r"""(\s*/\*.*\*/\s*$| + /\*.*\*/\s+| + \s+/\*.*\*/(?=\W)| + /\*.*\*/)""", re.VERBOSE) + + +def IsCppString(line): + """Does line terminate so, that the next symbol is in string constant. + + This function does not consider single-line nor multi-line comments. + + Args: + line: is a partial line of code starting from the 0..n. + + Returns: + True, if next character appended to 'line' is inside a + string constant. + """ + + line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" + return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 + + +def CleanseRawStrings(raw_lines): + """Removes C++11 raw strings from lines. + + Before: + static const char kData[] = R"( + multi-line string + )"; + + After: + static const char kData[] = "" + (replaced by blank line) + ""; + + Args: + raw_lines: list of raw lines. + + Returns: + list of lines with C++11 raw strings replaced by empty strings. + """ + + delimiter = None + lines_without_raw_strings = [] + for line in raw_lines: + if delimiter: + # Inside a raw string, look for the end + end = line.find(delimiter) + if end >= 0: + # Found the end of the string, match leading space for this + # line and resume copying the original lines, and also insert + # a "" on the last line. + leading_space = Match(r'^(\s*)\S', line) + line = leading_space.group(1) + '""' + line[end + len(delimiter):] + delimiter = None + else: + # Haven't found the end yet, append a blank line. + line = '' + + else: + # Look for beginning of a raw string. + # See 2.14.15 [lex.string] for syntax. + matched = Match(r'^(.*)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line) + if matched: + delimiter = ')' + matched.group(2) + '"' + + end = matched.group(3).find(delimiter) + if end >= 0: + # Raw string ended on same line + line = (matched.group(1) + '""' + + matched.group(3)[end + len(delimiter):]) + delimiter = None + else: + # Start of a multi-line raw string + line = matched.group(1) + '""' + + lines_without_raw_strings.append(line) + + # TODO(unknown): if delimiter is not None here, we might want to + # emit a warning for unterminated string. + return lines_without_raw_strings + + +def FindNextMultiLineCommentStart(lines, lineix): + """Find the beginning marker for a multiline comment.""" + while lineix < len(lines): + if lines[lineix].strip().startswith('/*'): + # Only return this marker if the comment goes beyond this line + if lines[lineix].strip().find('*/', 2) < 0: + return lineix + lineix += 1 + return len(lines) + + +def FindNextMultiLineCommentEnd(lines, lineix): + """We are inside a comment, find the end marker.""" + while lineix < len(lines): + if lines[lineix].strip().endswith('*/'): + return lineix + lineix += 1 + return len(lines) + + +def RemoveMultiLineCommentsFromRange(lines, begin, end): + """Clears a range of lines for multi-line comments.""" + # Having // dummy comments makes the lines non-empty, so we will not get + # unnecessary blank line warnings later in the code. + for i in range(begin, end): + lines[i] = '// dummy' + + +def RemoveMultiLineComments(filename, lines, error): + """Removes multiline (c-style) comments from lines.""" + lineix = 0 + while lineix < len(lines): + lineix_begin = FindNextMultiLineCommentStart(lines, lineix) + if lineix_begin >= len(lines): + return + lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) + if lineix_end >= len(lines): + error(filename, lineix_begin + 1, 'readability/multiline_comment', 5, + 'Could not find end of multi-line comment') + return + RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) + lineix = lineix_end + 1 + + +def CleanseComments(line): + """Removes //-comments and single-line C-style /* */ comments. + + Args: + line: A line of C++ source. + + Returns: + The line with single-line comments removed. + """ + commentpos = line.find('//') + if commentpos != -1 and not IsCppString(line[:commentpos]): + line = line[:commentpos].rstrip() + # get rid of /* ... */ + return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) + + +class CleansedLines(object): + """Holds 3 copies of all lines with different preprocessing applied to them. + + 1) elided member contains lines without strings and comments, + 2) lines member contains lines without comments, and + 3) raw_lines member contains all the lines without processing. + All these three members are of , and of the same length. + """ + + def __init__(self, lines): + self.elided = [] + self.lines = [] + self.raw_lines = lines + self.num_lines = len(lines) + self.lines_without_raw_strings = CleanseRawStrings(lines) + for linenum in range(len(self.lines_without_raw_strings)): + self.lines.append(CleanseComments( + self.lines_without_raw_strings[linenum])) + elided = self._CollapseStrings(self.lines_without_raw_strings[linenum]) + self.elided.append(CleanseComments(elided)) + + def NumLines(self): + """Returns the number of lines represented.""" + return self.num_lines + + @staticmethod + def _CollapseStrings(elided): + """Collapses strings and chars on a line to simple "" or '' blocks. + + We nix strings first so we're not fooled by text like '"http://"' + + Args: + elided: The line being processed. + + Returns: + The line with collapsed strings. + """ + if not _RE_PATTERN_INCLUDE.match(elided): + # Remove escaped characters first to make quote/single quote collapsing + # basic. Things that look like escaped characters shouldn't occur + # outside of strings and chars. + elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) + elided = _RE_PATTERN_CLEANSE_LINE_SINGLE_QUOTES.sub("''", elided) + elided = _RE_PATTERN_CLEANSE_LINE_DOUBLE_QUOTES.sub('""', elided) + return elided + + +def FindEndOfExpressionInLine(line, startpos, depth, startchar, endchar): + """Find the position just after the matching endchar. + + Args: + line: a CleansedLines line. + startpos: start searching at this position. + depth: nesting level at startpos. + startchar: expression opening character. + endchar: expression closing character. + + Returns: + On finding matching endchar: (index just after matching endchar, 0) + Otherwise: (-1, new depth at end of this line) + """ + for i in xrange(startpos, len(line)): + if line[i] == startchar: + depth += 1 + elif line[i] == endchar: + depth -= 1 + if depth == 0: + return (i + 1, 0) + return (-1, depth) + + +def CloseExpression(clean_lines, linenum, pos): + """If input points to ( or { or [ or <, finds the position that closes it. + + If lines[linenum][pos] points to a '(' or '{' or '[' or '<', finds the + linenum/pos that correspond to the closing of the expression. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + pos: A position on the line. + + Returns: + A tuple (line, linenum, pos) pointer *past* the closing brace, or + (line, len(lines), -1) if we never find a close. Note we ignore + strings and comments when matching; and the line we return is the + 'cleansed' line at linenum. + """ + + line = clean_lines.elided[linenum] + startchar = line[pos] + if startchar not in '({[<': + return (line, clean_lines.NumLines(), -1) + if startchar == '(': endchar = ')' + if startchar == '[': endchar = ']' + if startchar == '{': endchar = '}' + if startchar == '<': endchar = '>' + + # Check first line + (end_pos, num_open) = FindEndOfExpressionInLine( + line, pos, 0, startchar, endchar) + if end_pos > -1: + return (line, linenum, end_pos) + + # Continue scanning forward + while linenum < clean_lines.NumLines() - 1: + linenum += 1 + line = clean_lines.elided[linenum] + (end_pos, num_open) = FindEndOfExpressionInLine( + line, 0, num_open, startchar, endchar) + if end_pos > -1: + return (line, linenum, end_pos) + + # Did not find endchar before end of file, give up + return (line, clean_lines.NumLines(), -1) + + +def FindStartOfExpressionInLine(line, endpos, depth, startchar, endchar): + """Find position at the matching startchar. + + This is almost the reverse of FindEndOfExpressionInLine, but note + that the input position and returned position differs by 1. + + Args: + line: a CleansedLines line. + endpos: start searching at this position. + depth: nesting level at endpos. + startchar: expression opening character. + endchar: expression closing character. + + Returns: + On finding matching startchar: (index at matching startchar, 0) + Otherwise: (-1, new depth at beginning of this line) + """ + for i in xrange(endpos, -1, -1): + if line[i] == endchar: + depth += 1 + elif line[i] == startchar: + depth -= 1 + if depth == 0: + return (i, 0) + return (-1, depth) + + +def ReverseCloseExpression(clean_lines, linenum, pos): + """If input points to ) or } or ] or >, finds the position that opens it. + + If lines[linenum][pos] points to a ')' or '}' or ']' or '>', finds the + linenum/pos that correspond to the opening of the expression. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + pos: A position on the line. + + Returns: + A tuple (line, linenum, pos) pointer *at* the opening brace, or + (line, 0, -1) if we never find the matching opening brace. Note + we ignore strings and comments when matching; and the line we + return is the 'cleansed' line at linenum. + """ + line = clean_lines.elided[linenum] + endchar = line[pos] + if endchar not in ')}]>': + return (line, 0, -1) + if endchar == ')': startchar = '(' + if endchar == ']': startchar = '[' + if endchar == '}': startchar = '{' + if endchar == '>': startchar = '<' + + # Check last line + (start_pos, num_open) = FindStartOfExpressionInLine( + line, pos, 0, startchar, endchar) + if start_pos > -1: + return (line, linenum, start_pos) + + # Continue scanning backward + while linenum > 0: + linenum -= 1 + line = clean_lines.elided[linenum] + (start_pos, num_open) = FindStartOfExpressionInLine( + line, len(line) - 1, num_open, startchar, endchar) + if start_pos > -1: + return (line, linenum, start_pos) + + # Did not find startchar before beginning of file, give up + return (line, 0, -1) + + +def CheckForCopyright(filename, lines, error): + """Logs an error if no Copyright message appears at the top of the file.""" + + # We'll say it should occur by line 10. Don't forget there's a + # dummy line at the front. + for line in xrange(1, min(len(lines), 11)): + if re.search(r'Copyright', lines[line], re.I): break + else: # means no copyright line was found + error(filename, 0, 'legal/copyright', 5, + 'No copyright message found. ' + 'You should have a line: "Copyright [year] "') + + +def GetHeaderGuardCPPVariable(filename): + """Returns the CPP variable that should be used as a header guard. + + Args: + filename: The name of a C++ header file. + + Returns: + The CPP variable that should be used as a header guard in the + named file. + + """ + + # Restores original filename in case that cpplint is invoked from Emacs's + # flymake. + filename = re.sub(r'_flymake\.h$', '.h', filename) + filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename) + + fileinfo = FileInfo(filename) + file_path_from_root = fileinfo.RepositoryName() + if _root: + file_path_from_root = re.sub('^' + _root + os.sep, '', file_path_from_root) + return re.sub(r'[-./\s]', '_', file_path_from_root).upper() + '_' + + +def CheckForHeaderGuard(filename, lines, error): + """Checks that the file contains a header guard. + + Logs an error if no #ifndef header guard is present. For other + headers, checks that the full pathname is used. + + Args: + filename: The name of the C++ header file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + + cppvar = GetHeaderGuardCPPVariable(filename) + + ifndef = None + ifndef_linenum = 0 + define = None + endif = None + endif_linenum = 0 + for linenum, line in enumerate(lines): + linesplit = line.split() + if len(linesplit) >= 2: + # find the first occurrence of #ifndef and #define, save arg + if not ifndef and linesplit[0] == '#ifndef': + # set ifndef to the header guard presented on the #ifndef line. + ifndef = linesplit[1] + ifndef_linenum = linenum + if not define and linesplit[0] == '#define': + define = linesplit[1] + # find the last occurrence of #endif, save entire line + if line.startswith('#endif'): + endif = line + endif_linenum = linenum + + if not ifndef: + error(filename, 0, 'build/header_guard', 5, + 'No #ifndef header guard found, suggested CPP variable is: %s' % + cppvar) + return + + if not define: + error(filename, 0, 'build/header_guard', 5, + 'No #define header guard found, suggested CPP variable is: %s' % + cppvar) + return + + # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ + # for backward compatibility. + if ifndef != cppvar: + error_level = 0 + if ifndef != cppvar + '_': + error_level = 5 + + ParseNolintSuppressions(filename, lines[ifndef_linenum], ifndef_linenum, + error) + error(filename, ifndef_linenum, 'build/header_guard', error_level, + '#ifndef header guard has wrong style, please use: %s' % cppvar) + + if define != ifndef: + error(filename, 0, 'build/header_guard', 5, + '#ifndef and #define don\'t match, suggested CPP variable is: %s' % + cppvar) + return + + if endif != ('#endif // %s' % cppvar): + error_level = 0 + if endif != ('#endif // %s' % (cppvar + '_')): + error_level = 5 + + ParseNolintSuppressions(filename, lines[endif_linenum], endif_linenum, + error) + error(filename, endif_linenum, 'build/header_guard', error_level, + '#endif line should be "#endif // %s"' % cppvar) + + +def CheckForBadCharacters(filename, lines, error): + """Logs an error for each line containing bad characters. + + Two kinds of bad characters: + + 1. Unicode replacement characters: These indicate that either the file + contained invalid UTF-8 (likely) or Unicode replacement characters (which + it shouldn't). Note that it's possible for this to throw off line + numbering if the invalid UTF-8 occurred adjacent to a newline. + + 2. NUL bytes. These are problematic for some tools. + + Args: + filename: The name of the current file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + for linenum, line in enumerate(lines): + if u'\ufffd' in line: + error(filename, linenum, 'readability/utf8', 5, + 'Line contains invalid UTF-8 (or Unicode replacement character).') + if '\0' in line: + error(filename, linenum, 'readability/nul', 5, 'Line contains NUL byte.') + + +def CheckForNewlineAtEOF(filename, lines, error): + """Logs an error if there is no newline char at the end of the file. + + Args: + filename: The name of the current file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + + # The array lines() was created by adding two newlines to the + # original file (go figure), then splitting on \n. + # To verify that the file ends in \n, we just have to make sure the + # last-but-two element of lines() exists and is empty. + if len(lines) < 3 or lines[-2]: + error(filename, len(lines) - 2, 'whitespace/ending_newline', 5, + 'Could not find a newline character at the end of the file.') + + +def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): + """Logs an error if we see /* ... */ or "..." that extend past one line. + + /* ... */ comments are legit inside macros, for one line. + Otherwise, we prefer // comments, so it's ok to warn about the + other. Likewise, it's ok for strings to extend across multiple + lines, as long as a line continuation character (backslash) + terminates each line. Although not currently prohibited by the C++ + style guide, it's ugly and unnecessary. We don't do well with either + in this lint program, so we warn about both. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Remove all \\ (escaped backslashes) from the line. They are OK, and the + # second (escaped) slash may trigger later \" detection erroneously. + line = line.replace('\\\\', '') + + if line.count('/*') > line.count('*/'): + error(filename, linenum, 'readability/multiline_comment', 5, + 'Complex multi-line /*...*/-style comment found. ' + 'Lint may give bogus warnings. ' + 'Consider replacing these with //-style comments, ' + 'with #if 0...#endif, ' + 'or with more clearly structured multi-line comments.') + + if (line.count('"') - line.count('\\"')) % 2: + error(filename, linenum, 'readability/multiline_string', 5, + 'Multi-line string ("...") found. This lint script doesn\'t ' + 'do well with such strings, and may give bogus warnings. ' + 'Use C++11 raw strings or concatenation instead.') + + +threading_list = ( + ('asctime(', 'asctime_r('), + ('ctime(', 'ctime_r('), + ('getgrgid(', 'getgrgid_r('), + ('getgrnam(', 'getgrnam_r('), + ('getlogin(', 'getlogin_r('), + ('getpwnam(', 'getpwnam_r('), + ('getpwuid(', 'getpwuid_r('), + ('gmtime(', 'gmtime_r('), + ('localtime(', 'localtime_r('), + ('rand(', 'rand_r('), + ('strtok(', 'strtok_r('), + ('ttyname(', 'ttyname_r('), + ) + + +def CheckPosixThreading(filename, clean_lines, linenum, error): + """Checks for calls to thread-unsafe functions. + + Much code has been originally written without consideration of + multi-threading. Also, engineers are relying on their old experience; + they have learned posix before threading extensions were added. These + tests guide the engineers to use thread-safe functions (when using + posix directly). + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + for single_thread_function, multithread_safe_function in threading_list: + ix = line.find(single_thread_function) + # Comparisons made explicit for clarity -- pylint: disable=g-explicit-bool-comparison + if ix >= 0 and (ix == 0 or (not line[ix - 1].isalnum() and + line[ix - 1] not in ('_', '.', '>'))): + error(filename, linenum, 'runtime/threadsafe_fn', 2, + 'Consider using ' + multithread_safe_function + + '...) instead of ' + single_thread_function + + '...) for improved thread safety.') + + +def CheckVlogArguments(filename, clean_lines, linenum, error): + """Checks that VLOG() is only used for defining a logging level. + + For example, VLOG(2) is correct. VLOG(INFO), VLOG(WARNING), VLOG(ERROR), and + VLOG(FATAL) are not. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line): + error(filename, linenum, 'runtime/vlog', 5, + 'VLOG() should be used with numeric verbosity level. ' + 'Use LOG() if you want symbolic severity levels.') + + +# Matches invalid increment: *count++, which moves pointer instead of +# incrementing a value. +_RE_PATTERN_INVALID_INCREMENT = re.compile( + r'^\s*\*\w+(\+\+|--);') + + +def CheckInvalidIncrement(filename, clean_lines, linenum, error): + """Checks for invalid increment *count++. + + For example following function: + void increment_counter(int* count) { + *count++; + } + is invalid, because it effectively does count++, moving pointer, and should + be replaced with ++*count, (*count)++ or *count += 1. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + if _RE_PATTERN_INVALID_INCREMENT.match(line): + error(filename, linenum, 'runtime/invalid_increment', 5, + 'Changing pointer instead of value (or unused value of operator*).') + + +class _BlockInfo(object): + """Stores information about a generic block of code.""" + + def __init__(self, seen_open_brace): + self.seen_open_brace = seen_open_brace + self.open_parentheses = 0 + self.inline_asm = _NO_ASM + + def CheckBegin(self, filename, clean_lines, linenum, error): + """Run checks that applies to text up to the opening brace. + + This is mostly for checking the text after the class identifier + and the "{", usually where the base class is specified. For other + blocks, there isn't much to check, so we always pass. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + pass + + def CheckEnd(self, filename, clean_lines, linenum, error): + """Run checks that applies to text after the closing brace. + + This is mostly used for checking end of namespace comments. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + pass + + +class _ClassInfo(_BlockInfo): + """Stores information about a class.""" + + def __init__(self, name, class_or_struct, clean_lines, linenum): + _BlockInfo.__init__(self, False) + self.name = name + self.starting_linenum = linenum + self.is_derived = False + if class_or_struct == 'struct': + self.access = 'public' + self.is_struct = True + else: + self.access = 'private' + self.is_struct = False + + # Remember initial indentation level for this class. Using raw_lines here + # instead of elided to account for leading comments. + initial_indent = Match(r'^( *)\S', clean_lines.raw_lines[linenum]) + if initial_indent: + self.class_indent = len(initial_indent.group(1)) + else: + self.class_indent = 0 + + # Try to find the end of the class. This will be confused by things like: + # class A { + # } *x = { ... + # + # But it's still good enough for CheckSectionSpacing. + self.last_line = 0 + depth = 0 + for i in range(linenum, clean_lines.NumLines()): + line = clean_lines.elided[i] + depth += line.count('{') - line.count('}') + if not depth: + self.last_line = i + break + + def CheckBegin(self, filename, clean_lines, linenum, error): + # Look for a bare ':' + if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]): + self.is_derived = True + + def CheckEnd(self, filename, clean_lines, linenum, error): + # Check that closing brace is aligned with beginning of the class. + # Only do this if the closing brace is indented by only whitespaces. + # This means we will not check single-line class definitions. + indent = Match(r'^( *)\}', clean_lines.elided[linenum]) + if indent and len(indent.group(1)) != self.class_indent: + if self.is_struct: + parent = 'struct ' + self.name + else: + parent = 'class ' + self.name + error(filename, linenum, 'whitespace/indent', 3, + 'Closing brace should be aligned with beginning of %s' % parent) + + +class _NamespaceInfo(_BlockInfo): + """Stores information about a namespace.""" + + def __init__(self, name, linenum): + _BlockInfo.__init__(self, False) + self.name = name or '' + self.starting_linenum = linenum + + def CheckEnd(self, filename, clean_lines, linenum, error): + """Check end of namespace comments.""" + line = clean_lines.raw_lines[linenum] + + # Check how many lines is enclosed in this namespace. Don't issue + # warning for missing namespace comments if there aren't enough + # lines. However, do apply checks if there is already an end of + # namespace comment and it's incorrect. + # + # TODO(unknown): We always want to check end of namespace comments + # if a namespace is large, but sometimes we also want to apply the + # check if a short namespace contained nontrivial things (something + # other than forward declarations). There is currently no logic on + # deciding what these nontrivial things are, so this check is + # triggered by namespace size only, which works most of the time. + if (linenum - self.starting_linenum < 10 + and not Match(r'};*\s*(//|/\*).*\bnamespace\b', line)): + return + + # Look for matching comment at end of namespace. + # + # Note that we accept C style "/* */" comments for terminating + # namespaces, so that code that terminate namespaces inside + # preprocessor macros can be cpplint clean. + # + # We also accept stuff like "// end of namespace ." with the + # period at the end. + # + # Besides these, we don't accept anything else, otherwise we might + # get false negatives when existing comment is a substring of the + # expected namespace. + if self.name: + # Named namespace + if not Match((r'};*\s*(//|/\*).*\bnamespace\s+' + re.escape(self.name) + + r'[\*/\.\\\s]*$'), + line): + error(filename, linenum, 'readability/namespace', 5, + 'Namespace should be terminated with "// namespace %s"' % + self.name) + else: + # Anonymous namespace + if not Match(r'};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line): + error(filename, linenum, 'readability/namespace', 5, + 'Namespace should be terminated with "// namespace"') + + +class _PreprocessorInfo(object): + """Stores checkpoints of nesting stacks when #if/#else is seen.""" + + def __init__(self, stack_before_if): + # The entire nesting stack before #if + self.stack_before_if = stack_before_if + + # The entire nesting stack up to #else + self.stack_before_else = [] + + # Whether we have already seen #else or #elif + self.seen_else = False + + +class _NestingState(object): + """Holds states related to parsing braces.""" + + def __init__(self): + # Stack for tracking all braces. An object is pushed whenever we + # see a "{", and popped when we see a "}". Only 3 types of + # objects are possible: + # - _ClassInfo: a class or struct. + # - _NamespaceInfo: a namespace. + # - _BlockInfo: some other type of block. + self.stack = [] + + # Stack of _PreprocessorInfo objects. + self.pp_stack = [] + + def SeenOpenBrace(self): + """Check if we have seen the opening brace for the innermost block. + + Returns: + True if we have seen the opening brace, False if the innermost + block is still expecting an opening brace. + """ + return (not self.stack) or self.stack[-1].seen_open_brace + + def InNamespaceBody(self): + """Check if we are currently one level inside a namespace body. + + Returns: + True if top of the stack is a namespace block, False otherwise. + """ + return self.stack and isinstance(self.stack[-1], _NamespaceInfo) + + def UpdatePreprocessor(self, line): + """Update preprocessor stack. + + We need to handle preprocessors due to classes like this: + #ifdef SWIG + struct ResultDetailsPageElementExtensionPoint { + #else + struct ResultDetailsPageElementExtensionPoint : public Extension { + #endif + + We make the following assumptions (good enough for most files): + - Preprocessor condition evaluates to true from #if up to first + #else/#elif/#endif. + + - Preprocessor condition evaluates to false from #else/#elif up + to #endif. We still perform lint checks on these lines, but + these do not affect nesting stack. + + Args: + line: current line to check. + """ + if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line): + # Beginning of #if block, save the nesting stack here. The saved + # stack will allow us to restore the parsing state in the #else case. + self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack))) + elif Match(r'^\s*#\s*(else|elif)\b', line): + # Beginning of #else block + if self.pp_stack: + if not self.pp_stack[-1].seen_else: + # This is the first #else or #elif block. Remember the + # whole nesting stack up to this point. This is what we + # keep after the #endif. + self.pp_stack[-1].seen_else = True + self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack) + + # Restore the stack to how it was before the #if + self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if) + else: + # TODO(unknown): unexpected #else, issue warning? + pass + elif Match(r'^\s*#\s*endif\b', line): + # End of #if or #else blocks. + if self.pp_stack: + # If we saw an #else, we will need to restore the nesting + # stack to its former state before the #else, otherwise we + # will just continue from where we left off. + if self.pp_stack[-1].seen_else: + # Here we can just use a shallow copy since we are the last + # reference to it. + self.stack = self.pp_stack[-1].stack_before_else + # Drop the corresponding #if + self.pp_stack.pop() + else: + # TODO(unknown): unexpected #endif, issue warning? + pass + + def Update(self, filename, clean_lines, linenum, error): + """Update nesting state with current line. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Update pp_stack first + self.UpdatePreprocessor(line) + + # Count parentheses. This is to avoid adding struct arguments to + # the nesting stack. + if self.stack: + inner_block = self.stack[-1] + depth_change = line.count('(') - line.count(')') + inner_block.open_parentheses += depth_change + + # Also check if we are starting or ending an inline assembly block. + if inner_block.inline_asm in (_NO_ASM, _END_ASM): + if (depth_change != 0 and + inner_block.open_parentheses == 1 and + _MATCH_ASM.match(line)): + # Enter assembly block + inner_block.inline_asm = _INSIDE_ASM + else: + # Not entering assembly block. If previous line was _END_ASM, + # we will now shift to _NO_ASM state. + inner_block.inline_asm = _NO_ASM + elif (inner_block.inline_asm == _INSIDE_ASM and + inner_block.open_parentheses == 0): + # Exit assembly block + inner_block.inline_asm = _END_ASM + + # Consume namespace declaration at the beginning of the line. Do + # this in a loop so that we catch same line declarations like this: + # namespace proto2 { namespace bridge { class MessageSet; } } + while True: + # Match start of namespace. The "\b\s*" below catches namespace + # declarations even if it weren't followed by a whitespace, this + # is so that we don't confuse our namespace checker. The + # missing spaces will be flagged by CheckSpacing. + namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', line) + if not namespace_decl_match: + break + + new_namespace = _NamespaceInfo(namespace_decl_match.group(1), linenum) + self.stack.append(new_namespace) + + line = namespace_decl_match.group(2) + if line.find('{') != -1: + new_namespace.seen_open_brace = True + line = line[line.find('{') + 1:] + + # Look for a class declaration in whatever is left of the line + # after parsing namespaces. The regexp accounts for decorated classes + # such as in: + # class LOCKABLE API Object { + # }; + # + # Templates with class arguments may confuse the parser, for example: + # template , + # class Vector = vector > + # class HeapQueue { + # + # Because this parser has no nesting state about templates, by the + # time it saw "class Comparator", it may think that it's a new class. + # Nested templates have a similar problem: + # template < + # typename ExportedType, + # typename TupleType, + # template class ImplTemplate> + # + # To avoid these cases, we ignore classes that are followed by '=' or '>' + class_decl_match = Match( + r'\s*(template\s*<[\w\s<>,:]*>\s*)?' + r'(class|struct)\s+([A-Z_]+\s+)*(\w+(?:::\w+)*)' + r'(([^=>]|<[^<>]*>|<[^<>]*<[^<>]*>\s*>)*)$', line) + if (class_decl_match and + (not self.stack or self.stack[-1].open_parentheses == 0)): + self.stack.append(_ClassInfo( + class_decl_match.group(4), class_decl_match.group(2), + clean_lines, linenum)) + line = class_decl_match.group(5) + + # If we have not yet seen the opening brace for the innermost block, + # run checks here. + if not self.SeenOpenBrace(): + self.stack[-1].CheckBegin(filename, clean_lines, linenum, error) + + # Update access control if we are inside a class/struct + if self.stack and isinstance(self.stack[-1], _ClassInfo): + classinfo = self.stack[-1] + access_match = Match( + r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?' + r':(?:[^:]|$)', + line) + if access_match: + classinfo.access = access_match.group(2) + + # Check that access keywords are indented +1 space. Skip this + # check if the keywords are not preceded by whitespaces. + indent = access_match.group(1) + if (len(indent) != classinfo.class_indent + 1 and + Match(r'^\s*$', indent)): + if classinfo.is_struct: + parent = 'struct ' + classinfo.name + else: + parent = 'class ' + classinfo.name + slots = '' + if access_match.group(3): + slots = access_match.group(3) + error(filename, linenum, 'whitespace/indent', 3, + '%s%s: should be indented +1 space inside %s' % ( + access_match.group(2), slots, parent)) + + # Consume braces or semicolons from what's left of the line + while True: + # Match first brace, semicolon, or closed parenthesis. + matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line) + if not matched: + break + + token = matched.group(1) + if token == '{': + # If namespace or class hasn't seen a opening brace yet, mark + # namespace/class head as complete. Push a new block onto the + # stack otherwise. + if not self.SeenOpenBrace(): + self.stack[-1].seen_open_brace = True + else: + self.stack.append(_BlockInfo(True)) + if _MATCH_ASM.match(line): + self.stack[-1].inline_asm = _BLOCK_ASM + elif token == ';' or token == ')': + # If we haven't seen an opening brace yet, but we already saw + # a semicolon, this is probably a forward declaration. Pop + # the stack for these. + # + # Similarly, if we haven't seen an opening brace yet, but we + # already saw a closing parenthesis, then these are probably + # function arguments with extra "class" or "struct" keywords. + # Also pop these stack for these. + if not self.SeenOpenBrace(): + self.stack.pop() + else: # token == '}' + # Perform end of block checks and pop the stack. + if self.stack: + self.stack[-1].CheckEnd(filename, clean_lines, linenum, error) + self.stack.pop() + line = matched.group(2) + + def InnermostClass(self): + """Get class info on the top of the stack. + + Returns: + A _ClassInfo object if we are inside a class, or None otherwise. + """ + for i in range(len(self.stack), 0, -1): + classinfo = self.stack[i - 1] + if isinstance(classinfo, _ClassInfo): + return classinfo + return None + + def CheckCompletedBlocks(self, filename, error): + """Checks that all classes and namespaces have been completely parsed. + + Call this when all lines in a file have been processed. + Args: + filename: The name of the current file. + error: The function to call with any errors found. + """ + # Note: This test can result in false positives if #ifdef constructs + # get in the way of brace matching. See the testBuildClass test in + # cpplint_unittest.py for an example of this. + for obj in self.stack: + if isinstance(obj, _ClassInfo): + error(filename, obj.starting_linenum, 'build/class', 5, + 'Failed to find complete declaration of class %s' % + obj.name) + elif isinstance(obj, _NamespaceInfo): + error(filename, obj.starting_linenum, 'build/namespaces', 5, + 'Failed to find complete declaration of namespace %s' % + obj.name) + + +def CheckForNonStandardConstructs(filename, clean_lines, linenum, + nesting_state, error): + r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2. + + Complain about several constructs which gcc-2 accepts, but which are + not standard C++. Warning about these in lint is one way to ease the + transition to new compilers. + - put storage class first (e.g. "static const" instead of "const static"). + - "%lld" instead of %qd" in printf-type functions. + - "%1$d" is non-standard in printf-type functions. + - "\%" is an undefined character escape sequence. + - text after #endif is not allowed. + - invalid inner-style forward declaration. + - >? and ?= and )\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', + line): + error(filename, linenum, 'build/deprecated', 3, + '>? and ))?' + # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;' + error(filename, linenum, 'runtime/member_string_references', 2, + 'const string& members are dangerous. It is much better to use ' + 'alternatives, such as pointers or simple constants.') + + # Everything else in this function operates on class declarations. + # Return early if the top of the nesting stack is not a class, or if + # the class head is not completed yet. + classinfo = nesting_state.InnermostClass() + if not classinfo or not classinfo.seen_open_brace: + return + + # The class may have been declared with namespace or classname qualifiers. + # The constructor and destructor will not have those qualifiers. + base_classname = classinfo.name.split('::')[-1] + + # Look for single-argument constructors that aren't marked explicit. + # Technically a valid construct, but against style. + args = Match(r'\s+(?:inline\s+)?%s\s*\(([^,()]+)\)' + % re.escape(base_classname), + line) + if (args and + args.group(1) != 'void' and + not Match(r'(const\s+)?%s(\s+const)?\s*(?:<\w+>\s*)?&' + % re.escape(base_classname), args.group(1).strip())): + error(filename, linenum, 'runtime/explicit', 5, + 'Single-argument constructors should be marked explicit.') + + +def CheckSpacingForFunctionCall(filename, line, linenum, error): + """Checks for the correctness of various spacing around function calls. + + Args: + filename: The name of the current file. + line: The text of the line to check. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + # Since function calls often occur inside if/for/while/switch + # expressions - which have their own, more liberal conventions - we + # first see if we should be looking inside such an expression for a + # function call, to which we can apply more strict standards. + fncall = line # if there's no control flow construct, look at whole line + for pattern in (r'\bif\s*\((.*)\)\s*{', + r'\bfor\s*\((.*)\)\s*{', + r'\bwhile\s*\((.*)\)\s*[{;]', + r'\bswitch\s*\((.*)\)\s*{'): + match = Search(pattern, line) + if match: + fncall = match.group(1) # look inside the parens for function calls + break + + # Except in if/for/while/switch, there should never be space + # immediately inside parens (eg "f( 3, 4 )"). We make an exception + # for nested parens ( (a+b) + c ). Likewise, there should never be + # a space before a ( when it's a function argument. I assume it's a + # function argument when the char before the whitespace is legal in + # a function name (alnum + _) and we're not starting a macro. Also ignore + # pointers and references to arrays and functions coz they're too tricky: + # we use a very simple way to recognize these: + # " (something)(maybe-something)" or + # " (something)(maybe-something," or + # " (something)[something]" + # Note that we assume the contents of [] to be short enough that + # they'll never need to wrap. + if ( # Ignore control structures. + not Search(r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b', + fncall) and + # Ignore pointers/references to functions. + not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and + # Ignore pointers/references to arrays. + not Search(r' \([^)]+\)\[[^\]]+\]', fncall)): + if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call + error(filename, linenum, 'whitespace/parens', 4, + 'Extra space after ( in function call') + elif Search(r'\(\s+(?!(\s*\\)|\()', fncall): + error(filename, linenum, 'whitespace/parens', 2, + 'Extra space after (') + if (Search(r'\w\s+\(', fncall) and + not Search(r'#\s*define|typedef', fncall) and + not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall)): + error(filename, linenum, 'whitespace/parens', 4, + 'Extra space before ( in function call') + # If the ) is followed only by a newline or a { + newline, assume it's + # part of a control statement (if/while/etc), and don't complain + if Search(r'[^)]\s+\)\s*[^{\s]', fncall): + # If the closing parenthesis is preceded by only whitespaces, + # try to give a more descriptive error message. + if Search(r'^\s+\)', fncall): + error(filename, linenum, 'whitespace/parens', 2, + 'Closing ) should be moved to the previous line') + else: + error(filename, linenum, 'whitespace/parens', 2, + 'Extra space before )') + + +def IsBlankLine(line): + """Returns true if the given line is blank. + + We consider a line to be blank if the line is empty or consists of + only white spaces. + + Args: + line: A line of a string. + + Returns: + True, if the given line is blank. + """ + return not line or line.isspace() + + +def CheckForFunctionLengths(filename, clean_lines, linenum, + function_state, error): + """Reports for long function bodies. + + For an overview why this is done, see: + http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions + + Uses a simplistic algorithm assuming other style guidelines + (especially spacing) are followed. + Only checks unindented functions, so class members are unchecked. + Trivial bodies are unchecked, so constructors with huge initializer lists + may be missed. + Blank/comment lines are not counted so as to avoid encouraging the removal + of vertical space and comments just to get through a lint check. + NOLINT *on the last line of a function* disables this check. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + function_state: Current function name and lines in body so far. + error: The function to call with any errors found. + """ + lines = clean_lines.lines + line = lines[linenum] + raw = clean_lines.raw_lines + raw_line = raw[linenum] + joined_line = '' + + starting_func = False + regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... + match_result = Match(regexp, line) + if match_result: + # If the name is all caps and underscores, figure it's a macro and + # ignore it, unless it's TEST or TEST_F. + function_name = match_result.group(1).split()[-1] + if function_name == 'TEST' or function_name == 'TEST_F' or ( + not Match(r'[A-Z_]+$', function_name)): + starting_func = True + + if starting_func: + body_found = False + for start_linenum in xrange(linenum, clean_lines.NumLines()): + start_line = lines[start_linenum] + joined_line += ' ' + start_line.lstrip() + if Search(r'(;|})', start_line): # Declarations and trivial functions + body_found = True + break # ... ignore + elif Search(r'{', start_line): + body_found = True + function = Search(r'((\w|:)*)\(', line).group(1) + if Match(r'TEST', function): # Handle TEST... macros + parameter_regexp = Search(r'(\(.*\))', joined_line) + if parameter_regexp: # Ignore bad syntax + function += parameter_regexp.group(1) + else: + function += '()' + function_state.Begin(function) + break + if not body_found: + # No body for the function (or evidence of a non-function) was found. + error(filename, linenum, 'readability/fn_size', 5, + 'Lint failed to find start of function body.') + elif Match(r'^\}\s*$', line): # function end + function_state.Check(error, filename, linenum) + function_state.End() + elif not Match(r'^\s*$', line): + function_state.Count() # Count non-blank/non-comment lines. + + +_RE_PATTERN_TODO = re.compile(r'^//(\s*)TODO(\(.+?\))?:?(\s|$)?') + + +def CheckComment(comment, filename, linenum, error): + """Checks for common mistakes in TODO comments. + + Args: + comment: The text of the comment from the line in question. + filename: The name of the current file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + match = _RE_PATTERN_TODO.match(comment) + if match: + # One whitespace is correct; zero whitespace is handled elsewhere. + leading_whitespace = match.group(1) + if len(leading_whitespace) > 1: + error(filename, linenum, 'whitespace/todo', 2, + 'Too many spaces before TODO') + + username = match.group(2) + if not username: + error(filename, linenum, 'readability/todo', 2, + 'Missing username in TODO; it should look like ' + '"// TODO(my_username): Stuff."') + + middle_whitespace = match.group(3) + # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison + if middle_whitespace != ' ' and middle_whitespace != '': + error(filename, linenum, 'whitespace/todo', 2, + 'TODO(my_username) should be followed by a space') + +def CheckAccess(filename, clean_lines, linenum, nesting_state, error): + """Checks for improper use of DISALLOW* macros. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + nesting_state: A _NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] # get rid of comments and strings + + matched = Match((r'\s*(DISALLOW_COPY_AND_ASSIGN|' + r'DISALLOW_EVIL_CONSTRUCTORS|' + r'DISALLOW_IMPLICIT_CONSTRUCTORS)'), line) + if not matched: + return + if nesting_state.stack and isinstance(nesting_state.stack[-1], _ClassInfo): + if nesting_state.stack[-1].access != 'private': + error(filename, linenum, 'readability/constructors', 3, + '%s must be in the private: section' % matched.group(1)) + + else: + # Found DISALLOW* macro outside a class declaration, or perhaps it + # was used inside a function when it should have been part of the + # class declaration. We could issue a warning here, but it + # probably resulted in a compiler error already. + pass + + +def FindNextMatchingAngleBracket(clean_lines, linenum, init_suffix): + """Find the corresponding > to close a template. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: Current line number. + init_suffix: Remainder of the current line after the initial <. + + Returns: + True if a matching bracket exists. + """ + line = init_suffix + nesting_stack = ['<'] + while True: + # Find the next operator that can tell us whether < is used as an + # opening bracket or as a less-than operator. We only want to + # warn on the latter case. + # + # We could also check all other operators and terminate the search + # early, e.g. if we got something like this "a(),;\[\]]*([<>(),;\[\]])(.*)$', line) + if match: + # Found an operator, update nesting stack + operator = match.group(1) + line = match.group(2) + + if nesting_stack[-1] == '<': + # Expecting closing angle bracket + if operator in ('<', '(', '['): + nesting_stack.append(operator) + elif operator == '>': + nesting_stack.pop() + if not nesting_stack: + # Found matching angle bracket + return True + elif operator == ',': + # Got a comma after a bracket, this is most likely a template + # argument. We have not seen a closing angle bracket yet, but + # it's probably a few lines later if we look for it, so just + # return early here. + return True + else: + # Got some other operator. + return False + + else: + # Expecting closing parenthesis or closing bracket + if operator in ('<', '(', '['): + nesting_stack.append(operator) + elif operator in (')', ']'): + # We don't bother checking for matching () or []. If we got + # something like (] or [), it would have been a syntax error. + nesting_stack.pop() + + else: + # Scan the next line + linenum += 1 + if linenum >= len(clean_lines.elided): + break + line = clean_lines.elided[linenum] + + # Exhausted all remaining lines and still no matching angle bracket. + # Most likely the input was incomplete, otherwise we should have + # seen a semicolon and returned early. + return True + + +def FindPreviousMatchingAngleBracket(clean_lines, linenum, init_prefix): + """Find the corresponding < that started a template. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: Current line number. + init_prefix: Part of the current line before the initial >. + + Returns: + True if a matching bracket exists. + """ + line = init_prefix + nesting_stack = ['>'] + while True: + # Find the previous operator + match = Search(r'^(.*)([<>(),;\[\]])[^<>(),;\[\]]*$', line) + if match: + # Found an operator, update nesting stack + operator = match.group(2) + line = match.group(1) + + if nesting_stack[-1] == '>': + # Expecting opening angle bracket + if operator in ('>', ')', ']'): + nesting_stack.append(operator) + elif operator == '<': + nesting_stack.pop() + if not nesting_stack: + # Found matching angle bracket + return True + elif operator == ',': + # Got a comma before a bracket, this is most likely a + # template argument. The opening angle bracket is probably + # there if we look for it, so just return early here. + return True + else: + # Got some other operator. + return False + + else: + # Expecting opening parenthesis or opening bracket + if operator in ('>', ')', ']'): + nesting_stack.append(operator) + elif operator in ('(', '['): + nesting_stack.pop() + + else: + # Scan the previous line + linenum -= 1 + if linenum < 0: + break + line = clean_lines.elided[linenum] + + # Exhausted all earlier lines and still no matching angle bracket. + return False + + +def CheckSpacing(filename, clean_lines, linenum, nesting_state, error): + """Checks for the correctness of various spacing issues in the code. + + Things we check for: spaces around operators, spaces after + if/for/while/switch, no spaces around parens in function calls, two + spaces between code and comment, don't start a block with a blank + line, don't end a function with a blank line, don't add a blank line + after public/protected/private, don't have too many blank lines in a row. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + nesting_state: A _NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + + # Don't use "elided" lines here, otherwise we can't check commented lines. + # Don't want to use "raw" either, because we don't want to check inside C++11 + # raw strings, + raw = clean_lines.lines_without_raw_strings + line = raw[linenum] + + # Before nixing comments, check if the line is blank for no good + # reason. This includes the first line after a block is opened, and + # blank lines at the end of a function (ie, right before a line like '}' + # + # Skip all the blank line checks if we are immediately inside a + # namespace body. In other words, don't issue blank line warnings + # for this block: + # namespace { + # + # } + # + # A warning about missing end of namespace comments will be issued instead. + if IsBlankLine(line) and not nesting_state.InNamespaceBody(): + elided = clean_lines.elided + prev_line = elided[linenum - 1] + prevbrace = prev_line.rfind('{') + # TODO(unknown): Don't complain if line before blank line, and line after, + # both start with alnums and are indented the same amount. + # This ignores whitespace at the start of a namespace block + # because those are not usually indented. + if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1: + # OK, we have a blank line at the start of a code block. Before we + # complain, we check if it is an exception to the rule: The previous + # non-empty line has the parameters of a function header that are indented + # 4 spaces (because they did not fit in a 80 column line when placed on + # the same line as the function name). We also check for the case where + # the previous line is indented 6 spaces, which may happen when the + # initializers of a constructor do not fit into a 80 column line. + exception = False + if Match(r' {6}\w', prev_line): # Initializer list? + # We are looking for the opening column of initializer list, which + # should be indented 4 spaces to cause 6 space indentation afterwards. + search_position = linenum-2 + while (search_position >= 0 + and Match(r' {6}\w', elided[search_position])): + search_position -= 1 + exception = (search_position >= 0 + and elided[search_position][:5] == ' :') + else: + # Search for the function arguments or an initializer list. We use a + # simple heuristic here: If the line is indented 4 spaces; and we have a + # closing paren, without the opening paren, followed by an opening brace + # or colon (for initializer lists) we assume that it is the last line of + # a function header. If we have a colon indented 4 spaces, it is an + # initializer list. + exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', + prev_line) + or Match(r' {4}:', prev_line)) + + if not exception: + error(filename, linenum, 'whitespace/blank_line', 2, + 'Redundant blank line at the start of a code block ' + 'should be deleted.') + # Ignore blank lines at the end of a block in a long if-else + # chain, like this: + # if (condition1) { + # // Something followed by a blank line + # + # } else if (condition2) { + # // Something else + # } + if linenum + 1 < clean_lines.NumLines(): + next_line = raw[linenum + 1] + if (next_line + and Match(r'\s*}', next_line) + and next_line.find('} else ') == -1): + error(filename, linenum, 'whitespace/blank_line', 3, + 'Redundant blank line at the end of a code block ' + 'should be deleted.') + + matched = Match(r'\s*(public|protected|private):', prev_line) + if matched: + error(filename, linenum, 'whitespace/blank_line', 3, + 'Do not leave a blank line after "%s:"' % matched.group(1)) + + # Next, we complain if there's a comment too near the text + commentpos = line.find('//') + if commentpos != -1: + # Check if the // may be in quotes. If so, ignore it + # Comparisons made explicit for clarity -- pylint: disable=g-explicit-bool-comparison + if (line.count('"', 0, commentpos) - + line.count('\\"', 0, commentpos)) % 2 == 0: # not in quotes + # Allow one space for new scopes, two spaces otherwise: + if (not Match(r'^\s*{ //', line) and + ((commentpos >= 1 and + line[commentpos-1] not in string.whitespace) or + (commentpos >= 2 and + line[commentpos-2] not in string.whitespace))): + error(filename, linenum, 'whitespace/comments', 2, + 'At least two spaces is best between code and comments') + # There should always be a space between the // and the comment + commentend = commentpos + 2 + if commentend < len(line) and not line[commentend] == ' ': + # but some lines are exceptions -- e.g. if they're big + # comment delimiters like: + # //---------------------------------------------------------- + # or are an empty C++ style Doxygen comment, like: + # /// + # or C++ style Doxygen comments placed after the variable: + # ///< Header comment + # //!< Header comment + # or they begin with multiple slashes followed by a space: + # //////// Header comment + match = (Search(r'[=/-]{4,}\s*$', line[commentend:]) or + Search(r'^/$', line[commentend:]) or + Search(r'^!< ', line[commentend:]) or + Search(r'^/< ', line[commentend:]) or + Search(r'^/+ ', line[commentend:])) + if not match: + error(filename, linenum, 'whitespace/comments', 4, + 'Should have a space between // and comment') + CheckComment(line[commentpos:], filename, linenum, error) + + line = clean_lines.elided[linenum] # get rid of comments and strings + + # Don't try to do spacing checks for operator methods + line = re.sub(r'operator(==|!=|<|<<|<=|>=|>>|>)\(', 'operator\(', line) + + # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". + # Otherwise not. Note we only check for non-spaces on *both* sides; + # sometimes people put non-spaces on one side when aligning ='s among + # many lines (not that this is behavior that I approve of...) + if Search(r'[\w.]=[\w.]', line) and not Search(r'\b(if|while) ', line): + error(filename, linenum, 'whitespace/operators', 4, + 'Missing spaces around =') + + # It's ok not to have spaces around binary operators like + - * /, but if + # there's too little whitespace, we get concerned. It's hard to tell, + # though, so we punt on this one for now. TODO. + + # You should always have whitespace around binary operators. + # + # Check <= and >= first to avoid false positives with < and >, then + # check non-include lines for spacing around < and >. + match = Search(r'[^<>=!\s](==|!=|<=|>=)[^<>=!\s]', line) + if match: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around %s' % match.group(1)) + # We allow no-spaces around << when used like this: 10<<20, but + # not otherwise (particularly, not when used as streams) + # Also ignore using ns::operator<<; + match = Search(r'(operator|\S)(?:L|UL|ULL|l|ul|ull)?<<(\S)', line) + if (match and + not (match.group(1).isdigit() and match.group(2).isdigit()) and + not (match.group(1) == 'operator' and match.group(2) == ';')): + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around <<') + elif not Match(r'#.*include', line): + # Avoid false positives on -> + reduced_line = line.replace('->', '') + + # Look for < that is not surrounded by spaces. This is only + # triggered if both sides are missing spaces, even though + # technically should should flag if at least one side is missing a + # space. This is done to avoid some false positives with shifts. + match = Search(r'[^\s<]<([^\s=<].*)', reduced_line) + if (match and + not FindNextMatchingAngleBracket(clean_lines, linenum, match.group(1))): + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around <') + + # Look for > that is not surrounded by spaces. Similar to the + # above, we only trigger if both sides are missing spaces to avoid + # false positives with shifts. + match = Search(r'^(.*[^\s>])>[^\s=>]', reduced_line) + if (match and + not FindPreviousMatchingAngleBracket(clean_lines, linenum, + match.group(1))): + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around >') + + # We allow no-spaces around >> for almost anything. This is because + # C++11 allows ">>" to close nested templates, which accounts for + # most cases when ">>" is not followed by a space. + # + # We still warn on ">>" followed by alpha character, because that is + # likely due to ">>" being used for right shifts, e.g.: + # value >> alpha + # + # When ">>" is used to close templates, the alphanumeric letter that + # follows would be part of an identifier, and there should still be + # a space separating the template type and the identifier. + # type> alpha + match = Search(r'>>[a-zA-Z_]', line) + if match: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around >>') + + # There shouldn't be space around unary operators + match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) + if match: + error(filename, linenum, 'whitespace/operators', 4, + 'Extra space for operator %s' % match.group(1)) + + # A pet peeve of mine: no spaces after an if, while, switch, or for + match = Search(r' (if\(|for\(|while\(|switch\()', line) + if match: + error(filename, linenum, 'whitespace/parens', 5, + 'Missing space before ( in %s' % match.group(1)) + + # For if/for/while/switch, the left and right parens should be + # consistent about how many spaces are inside the parens, and + # there should either be zero or one spaces inside the parens. + # We don't want: "if ( foo)" or "if ( foo )". + # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. + match = Search(r'\b(if|for|while|switch)\s*' + r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', + line) + if match: + if len(match.group(2)) != len(match.group(4)): + if not (match.group(3) == ';' and + len(match.group(2)) == 1 + len(match.group(4)) or + not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): + error(filename, linenum, 'whitespace/parens', 5, + 'Mismatching spaces inside () in %s' % match.group(1)) + if len(match.group(2)) not in [0, 1]: + error(filename, linenum, 'whitespace/parens', 5, + 'Should have zero or one spaces inside ( and ) in %s' % + match.group(1)) + + # You should always have a space after a comma (either as fn arg or operator) + # + # This does not apply when the non-space character following the + # comma is another comma, since the only time when that happens is + # for empty macro arguments. + # + # We run this check in two passes: first pass on elided lines to + # verify that lines contain missing whitespaces, second pass on raw + # lines to confirm that those missing whitespaces are not due to + # elided comments. + if Search(r',[^,\s]', line) and Search(r',[^,\s]', raw[linenum]): + error(filename, linenum, 'whitespace/comma', 3, + 'Missing space after ,') + + # You should always have a space after a semicolon + # except for few corner cases + # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more + # space after ; + if Search(r';[^\s};\\)/]', line): + error(filename, linenum, 'whitespace/semicolon', 3, + 'Missing space after ;') + + # Next we will look for issues with function calls. + CheckSpacingForFunctionCall(filename, line, linenum, error) + + # Except after an opening paren, or after another opening brace (in case of + # an initializer list, for instance), you should have spaces before your + # braces. And since you should never have braces at the beginning of a line, + # this is an easy test. + match = Match(r'^(.*[^ ({]){', line) + if match: + # Try a bit harder to check for brace initialization. This + # happens in one of the following forms: + # Constructor() : initializer_list_{} { ... } + # Constructor{}.MemberFunction() + # Type variable{}; + # FunctionCall(type{}, ...); + # LastArgument(..., type{}); + # LOG(INFO) << type{} << " ..."; + # map_of_type[{...}] = ...; + # + # We check for the character following the closing brace, and + # silence the warning if it's one of those listed above, i.e. + # "{.;,)<]". + # + # To account for nested initializer list, we allow any number of + # closing braces up to "{;,)<". We can't simply silence the + # warning on first sight of closing brace, because that would + # cause false negatives for things that are not initializer lists. + # Silence this: But not this: + # Outer{ if (...) { + # Inner{...} if (...){ // Missing space before { + # }; } + # + # There is a false negative with this approach if people inserted + # spurious semicolons, e.g. "if (cond){};", but we will catch the + # spurious semicolon with a separate check. + (endline, endlinenum, endpos) = CloseExpression( + clean_lines, linenum, len(match.group(1))) + trailing_text = '' + if endpos > -1: + trailing_text = endline[endpos:] + for offset in xrange(endlinenum + 1, + min(endlinenum + 3, clean_lines.NumLines() - 1)): + trailing_text += clean_lines.elided[offset] + if not Match(r'^[\s}]*[{.;,)<\]]', trailing_text): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before {') + + # Make sure '} else {' has spaces. + if Search(r'}else', line): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before else') + + # You shouldn't have spaces before your brackets, except maybe after + # 'delete []' or 'new char * []'. + if Search(r'\w\s+\[', line) and not Search(r'delete\s+\[', line): + error(filename, linenum, 'whitespace/braces', 5, + 'Extra space before [') + + # You shouldn't have a space before a semicolon at the end of the line. + # There's a special case for "for" since the style guide allows space before + # the semicolon there. + if Search(r':\s*;\s*$', line): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Semicolon defining empty statement. Use {} instead.') + elif Search(r'^\s*;\s*$', line): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Line contains only semicolon. If this should be an empty statement, ' + 'use {} instead.') + elif (Search(r'\s+;\s*$', line) and + not Search(r'\bfor\b', line)): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Extra space before last semicolon. If this should be an empty ' + 'statement, use {} instead.') + + # In range-based for, we wanted spaces before and after the colon, but + # not around "::" tokens that might appear. + if (Search('for *\(.*[^:]:[^: ]', line) or + Search('for *\(.*[^: ]:[^:]', line)): + error(filename, linenum, 'whitespace/forcolon', 2, + 'Missing space around colon in range-based for loop') + + +def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error): + """Checks for additional blank line issues related to sections. + + Currently the only thing checked here is blank line before protected/private. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + class_info: A _ClassInfo objects. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + # Skip checks if the class is small, where small means 25 lines or less. + # 25 lines seems like a good cutoff since that's the usual height of + # terminals, and any class that can't fit in one screen can't really + # be considered "small". + # + # Also skip checks if we are on the first line. This accounts for + # classes that look like + # class Foo { public: ... }; + # + # If we didn't find the end of the class, last_line would be zero, + # and the check will be skipped by the first condition. + if (class_info.last_line - class_info.starting_linenum <= 24 or + linenum <= class_info.starting_linenum): + return + + matched = Match(r'\s*(public|protected|private):', clean_lines.lines[linenum]) + if matched: + # Issue warning if the line before public/protected/private was + # not a blank line, but don't do this if the previous line contains + # "class" or "struct". This can happen two ways: + # - We are at the beginning of the class. + # - We are forward-declaring an inner class that is semantically + # private, but needed to be public for implementation reasons. + # Also ignores cases where the previous line ends with a backslash as can be + # common when defining classes in C macros. + prev_line = clean_lines.lines[linenum - 1] + if (not IsBlankLine(prev_line) and + not Search(r'\b(class|struct)\b', prev_line) and + not Search(r'\\$', prev_line)): + # Try a bit harder to find the beginning of the class. This is to + # account for multi-line base-specifier lists, e.g.: + # class Derived + # : public Base { + end_class_head = class_info.starting_linenum + for i in range(class_info.starting_linenum, linenum): + if Search(r'\{\s*$', clean_lines.lines[i]): + end_class_head = i + break + if end_class_head < linenum - 1: + error(filename, linenum, 'whitespace/blank_line', 3, + '"%s:" should be preceded by a blank line' % matched.group(1)) + + +def GetPreviousNonBlankLine(clean_lines, linenum): + """Return the most recent non-blank line and its line number. + + Args: + clean_lines: A CleansedLines instance containing the file contents. + linenum: The number of the line to check. + + Returns: + A tuple with two elements. The first element is the contents of the last + non-blank line before the current line, or the empty string if this is the + first non-blank line. The second is the line number of that line, or -1 + if this is the first non-blank line. + """ + + prevlinenum = linenum - 1 + while prevlinenum >= 0: + prevline = clean_lines.elided[prevlinenum] + if not IsBlankLine(prevline): # if not a blank line... + return (prevline, prevlinenum) + prevlinenum -= 1 + return ('', -1) + + +def CheckBraces(filename, clean_lines, linenum, error): + """Looks for misplaced braces (e.g. at the end of line). + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + line = clean_lines.elided[linenum] # get rid of comments and strings + + if Match(r'\s*{\s*$', line): + # We allow an open brace to start a line in the case where someone is using + # braces in a block to explicitly create a new scope, which is commonly used + # to control the lifetime of stack-allocated variables. Braces are also + # used for brace initializers inside function calls. We don't detect this + # perfectly: we just don't complain if the last non-whitespace character on + # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the + # previous line starts a preprocessor block. + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if (not Search(r'[,;:}{(]\s*$', prevline) and + not Match(r'\s*#', prevline)): + error(filename, linenum, 'whitespace/braces', 4, + '{ should almost always be at the end of the previous line') + + # An else clause should be on the same line as the preceding closing brace. + if Match(r'\s*else\s*', line): + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if Match(r'\s*}\s*$', prevline): + error(filename, linenum, 'whitespace/newline', 4, + 'An else should appear on the same line as the preceding }') + + # If braces come on one side of an else, they should be on both. + # However, we have to worry about "else if" that spans multiple lines! + if Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): + if Search(r'}\s*else if([^{]*)$', line): # could be multi-line if + # find the ( after the if + pos = line.find('else if') + pos = line.find('(', pos) + if pos > 0: + (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) + if endline[endpos:].find('{') == -1: # must be brace after if + error(filename, linenum, 'readability/braces', 5, + 'If an else has a brace on one side, it should have it on both') + else: # common case: else not followed by a multi-line if + error(filename, linenum, 'readability/braces', 5, + 'If an else has a brace on one side, it should have it on both') + + # Likewise, an else should never have the else clause on the same line + if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): + error(filename, linenum, 'whitespace/newline', 4, + 'Else clause should never be on same line as else (use 2 lines)') + + # In the same way, a do/while should never be on one line + if Match(r'\s*do [^\s{]', line): + error(filename, linenum, 'whitespace/newline', 4, + 'do/while clauses should not be on a single line') + + # Block bodies should not be followed by a semicolon. Due to C++11 + # brace initialization, there are more places where semicolons are + # required than not, so we use a whitelist approach to check these + # rather than a blacklist. These are the places where "};" should + # be replaced by just "}": + # 1. Some flavor of block following closing parenthesis: + # for (;;) {}; + # while (...) {}; + # switch (...) {}; + # Function(...) {}; + # if (...) {}; + # if (...) else if (...) {}; + # + # 2. else block: + # if (...) else {}; + # + # 3. const member function: + # Function(...) const {}; + # + # 4. Block following some statement: + # x = 42; + # {}; + # + # 5. Block at the beginning of a function: + # Function(...) { + # {}; + # } + # + # Note that naively checking for the preceding "{" will also match + # braces inside multi-dimensional arrays, but this is fine since + # that expression will not contain semicolons. + # + # 6. Block following another block: + # while (true) {} + # {}; + # + # 7. End of namespaces: + # namespace {}; + # + # These semicolons seems far more common than other kinds of + # redundant semicolons, possibly due to people converting classes + # to namespaces. For now we do not warn for this case. + # + # Try matching case 1 first. + match = Match(r'^(.*\)\s*)\{', line) + if match: + # Matched closing parenthesis (case 1). Check the token before the + # matching opening parenthesis, and don't warn if it looks like a + # macro. This avoids these false positives: + # - macro that defines a base class + # - multi-line macro that defines a base class + # - macro that defines the whole class-head + # + # But we still issue warnings for macros that we know are safe to + # warn, specifically: + # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P + # - TYPED_TEST + # - INTERFACE_DEF + # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED: + # + # We implement a whitelist of safe macros instead of a blacklist of + # unsafe macros, even though the latter appears less frequently in + # google code and would have been easier to implement. This is because + # the downside for getting the whitelist wrong means some extra + # semicolons, while the downside for getting the blacklist wrong + # would result in compile errors. + # + # In addition to macros, we also don't want to warn on compound + # literals. + closing_brace_pos = match.group(1).rfind(')') + opening_parenthesis = ReverseCloseExpression( + clean_lines, linenum, closing_brace_pos) + if opening_parenthesis[2] > -1: + line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]] + macro = Search(r'\b([A-Z_]+)\s*$', line_prefix) + if ((macro and + macro.group(1) not in ( + 'TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST', + 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED', + 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or + Search(r'\s+=\s*$', line_prefix)): + match = None + + else: + # Try matching cases 2-3. + match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line) + if not match: + # Try matching cases 4-6. These are always matched on separate lines. + # + # Note that we can't simply concatenate the previous line to the + # current line and do a single match, otherwise we may output + # duplicate warnings for the blank line case: + # if (cond) { + # // blank line + # } + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if prevline and Search(r'[;{}]\s*$', prevline): + match = Match(r'^(\s*)\{', line) + + # Check matching closing brace + if match: + (endline, endlinenum, endpos) = CloseExpression( + clean_lines, linenum, len(match.group(1))) + if endpos > -1 and Match(r'^\s*;', endline[endpos:]): + # Current {} pair is eligible for semicolon check, and we have found + # the redundant semicolon, output warning here. + # + # Note: because we are scanning forward for opening braces, and + # outputting warnings for the matching closing brace, if there are + # nested blocks with trailing semicolons, we will get the error + # messages in reversed order. + error(filename, endlinenum, 'readability/braces', 4, + "You don't need a ; after a }") + + +def CheckEmptyBlockBody(filename, clean_lines, linenum, error): + """Look for empty loop/conditional body with only a single semicolon. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + # Search for loop keywords at the beginning of the line. Because only + # whitespaces are allowed before the keywords, this will also ignore most + # do-while-loops, since those lines should start with closing brace. + # + # We also check "if" blocks here, since an empty conditional block + # is likely an error. + line = clean_lines.elided[linenum] + matched = Match(r'\s*(for|while|if)\s*\(', line) + if matched: + # Find the end of the conditional expression + (end_line, end_linenum, end_pos) = CloseExpression( + clean_lines, linenum, line.find('(')) + + # Output warning if what follows the condition expression is a semicolon. + # No warning for all other cases, including whitespace or newline, since we + # have a separate check for semicolons preceded by whitespace. + if end_pos >= 0 and Match(r';', end_line[end_pos:]): + if matched.group(1) == 'if': + error(filename, end_linenum, 'whitespace/empty_conditional_body', 5, + 'Empty conditional bodies should use {}') + else: + error(filename, end_linenum, 'whitespace/empty_loop_body', 5, + 'Empty loop bodies should use {} or continue') + + +def CheckCheck(filename, clean_lines, linenum, error): + """Checks the use of CHECK and EXPECT macros. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + # Decide the set of replacement macros that should be suggested + lines = clean_lines.elided + check_macro = None + start_pos = -1 + for macro in _CHECK_MACROS: + i = lines[linenum].find(macro) + if i >= 0: + check_macro = macro + + # Find opening parenthesis. Do a regular expression match here + # to make sure that we are matching the expected CHECK macro, as + # opposed to some other macro that happens to contain the CHECK + # substring. + matched = Match(r'^(.*\b' + check_macro + r'\s*)\(', lines[linenum]) + if not matched: + continue + start_pos = len(matched.group(1)) + break + if not check_macro or start_pos < 0: + # Don't waste time here if line doesn't contain 'CHECK' or 'EXPECT' + return + + # Find end of the boolean expression by matching parentheses + (last_line, end_line, end_pos) = CloseExpression( + clean_lines, linenum, start_pos) + if end_pos < 0: + return + if linenum == end_line: + expression = lines[linenum][start_pos + 1:end_pos - 1] + else: + expression = lines[linenum][start_pos + 1:] + for i in xrange(linenum + 1, end_line): + expression += lines[i] + expression += last_line[0:end_pos - 1] + + # Parse expression so that we can take parentheses into account. + # This avoids false positives for inputs like "CHECK((a < 4) == b)", + # which is not replaceable by CHECK_LE. + lhs = '' + rhs = '' + operator = None + while expression: + matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||' + r'==|!=|>=|>|<=|<|\()(.*)$', expression) + if matched: + token = matched.group(1) + if token == '(': + # Parenthesized operand + expression = matched.group(2) + (end, _) = FindEndOfExpressionInLine(expression, 0, 1, '(', ')') + if end < 0: + return # Unmatched parenthesis + lhs += '(' + expression[0:end] + expression = expression[end:] + elif token in ('&&', '||'): + # Logical and/or operators. This means the expression + # contains more than one term, for example: + # CHECK(42 < a && a < b); + # + # These are not replaceable with CHECK_LE, so bail out early. + return + elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'): + # Non-relational operator + lhs += token + expression = matched.group(2) + else: + # Relational operator + operator = token + rhs = matched.group(2) + break + else: + # Unparenthesized operand. Instead of appending to lhs one character + # at a time, we do another regular expression match to consume several + # characters at once if possible. Trivial benchmark shows that this + # is more efficient when the operands are longer than a single + # character, which is generally the case. + matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression) + if not matched: + matched = Match(r'^(\s*\S)(.*)$', expression) + if not matched: + break + lhs += matched.group(1) + expression = matched.group(2) + + # Only apply checks if we got all parts of the boolean expression + if not (lhs and operator and rhs): + return + + # Check that rhs do not contain logical operators. We already know + # that lhs is fine since the loop above parses out && and ||. + if rhs.find('&&') > -1 or rhs.find('||') > -1: + return + + # At least one of the operands must be a constant literal. This is + # to avoid suggesting replacements for unprintable things like + # CHECK(variable != iterator) + # + # The following pattern matches decimal, hex integers, strings, and + # characters (in that order). + lhs = lhs.strip() + rhs = rhs.strip() + match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$' + if Match(match_constant, lhs) or Match(match_constant, rhs): + # Note: since we know both lhs and rhs, we can provide a more + # descriptive error message like: + # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42) + # Instead of: + # Consider using CHECK_EQ instead of CHECK(a == b) + # + # We are still keeping the less descriptive message because if lhs + # or rhs gets long, the error message might become unreadable. + error(filename, linenum, 'readability/check', 2, + 'Consider using %s instead of %s(a %s b)' % ( + _CHECK_REPLACEMENT[check_macro][operator], + check_macro, operator)) + + +def CheckAltTokens(filename, clean_lines, linenum, error): + """Check alternative keywords being used in boolean expressions. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Avoid preprocessor lines + if Match(r'^\s*#', line): + return + + # Last ditch effort to avoid multi-line comments. This will not help + # if the comment started before the current line or ended after the + # current line, but it catches most of the false positives. At least, + # it provides a way to workaround this warning for people who use + # multi-line comments in preprocessor macros. + # + # TODO(unknown): remove this once cpplint has better support for + # multi-line comments. + if line.find('/*') >= 0 or line.find('*/') >= 0: + return + + for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line): + error(filename, linenum, 'readability/alt_tokens', 2, + 'Use operator %s instead of %s' % ( + _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1))) + + +def GetLineWidth(line): + """Determines the width of the line in column positions. + + Args: + line: A string, which may be a Unicode string. + + Returns: + The width of the line in column positions, accounting for Unicode + combining characters and wide characters. + """ + if isinstance(line, unicode): + width = 0 + for uc in unicodedata.normalize('NFC', line): + if unicodedata.east_asian_width(uc) in ('W', 'F'): + width += 2 + elif not unicodedata.combining(uc): + width += 1 + return width + else: + return len(line) + + +def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, + error): + """Checks rules from the 'C++ style rules' section of cppguide.html. + + Most of these rules are hard to test (naming, comment style), but we + do what we can. In particular we check for 2-space indents, line lengths, + tab usage, spaces inside code, etc. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + file_extension: The extension (without the dot) of the filename. + nesting_state: A _NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + + # Don't use "elided" lines here, otherwise we can't check commented lines. + # Don't want to use "raw" either, because we don't want to check inside C++11 + # raw strings, + raw_lines = clean_lines.lines_without_raw_strings + line = raw_lines[linenum] + + if line.find('\t') != -1: + error(filename, linenum, 'whitespace/tab', 1, + 'Tab found; better to use spaces') + + # One or three blank spaces at the beginning of the line is weird; it's + # hard to reconcile that with 2-space indents. + # NOTE: here are the conditions rob pike used for his tests. Mine aren't + # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces + # if(RLENGTH > 20) complain = 0; + # if(match($0, " +(error|private|public|protected):")) complain = 0; + # if(match(prev, "&& *$")) complain = 0; + # if(match(prev, "\\|\\| *$")) complain = 0; + # if(match(prev, "[\",=><] *$")) complain = 0; + # if(match($0, " <<")) complain = 0; + # if(match(prev, " +for \\(")) complain = 0; + # if(prevodd && match(prevprev, " +for \\(")) complain = 0; + initial_spaces = 0 + cleansed_line = clean_lines.elided[linenum] + while initial_spaces < len(line) and line[initial_spaces] == ' ': + initial_spaces += 1 + if line and line[-1].isspace(): + error(filename, linenum, 'whitespace/end_of_line', 4, + 'Line ends in whitespace. Consider deleting these extra spaces.') + # There are certain situations we allow one space, notably for section labels + elif ((initial_spaces == 1 or initial_spaces == 3) and + not Match(r'\s*\w+\s*:\s*$', cleansed_line)): + error(filename, linenum, 'whitespace/indent', 3, + 'Weird number of spaces at line-start. ' + 'Are you using a 2-space indent?') + + # Check if the line is a header guard. + is_header_guard = False + if file_extension == 'h': + cppvar = GetHeaderGuardCPPVariable(filename) + if (line.startswith('#ifndef %s' % cppvar) or + line.startswith('#define %s' % cppvar) or + line.startswith('#endif // %s' % cppvar)): + is_header_guard = True + # #include lines and header guards can be long, since there's no clean way to + # split them. + # + # URLs can be long too. It's possible to split these, but it makes them + # harder to cut&paste. + # + # The "$Id:...$" comment may also get very long without it being the + # developers fault. + if (not line.startswith('#include') and not is_header_guard and + not Match(r'^\s*//.*http(s?)://\S*$', line) and + not Match(r'^// \$Id:.*#[0-9]+ \$$', line)): + line_width = GetLineWidth(line) + extended_length = int((_line_length * 1.25)) + if line_width > extended_length: + error(filename, linenum, 'whitespace/line_length', 4, + 'Lines should very rarely be longer than %i characters' % + extended_length) + elif line_width > _line_length: + error(filename, linenum, 'whitespace/line_length', 2, + 'Lines should be <= %i characters long' % _line_length) + + if (cleansed_line.count(';') > 1 and + # for loops are allowed two ;'s (and may run over two lines). + cleansed_line.find('for') == -1 and + (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or + GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and + # It's ok to have many commands in a switch case that fits in 1 line + not ((cleansed_line.find('case ') != -1 or + cleansed_line.find('default:') != -1) and + cleansed_line.find('break;') != -1)): + error(filename, linenum, 'whitespace/newline', 0, + 'More than one command on the same line') + + # Some more style checks + CheckBraces(filename, clean_lines, linenum, error) + CheckEmptyBlockBody(filename, clean_lines, linenum, error) + CheckAccess(filename, clean_lines, linenum, nesting_state, error) + CheckSpacing(filename, clean_lines, linenum, nesting_state, error) + CheckCheck(filename, clean_lines, linenum, error) + CheckAltTokens(filename, clean_lines, linenum, error) + classinfo = nesting_state.InnermostClass() + if classinfo: + CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error) + + +_RE_PATTERN_INCLUDE_NEW_STYLE = re.compile(r'#include +"[^/]+\.h"') +_RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$') +# Matches the first component of a filename delimited by -s and _s. That is: +# _RE_FIRST_COMPONENT.match('foo').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo.cc').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo-bar_baz.cc').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo_bar-baz.cc').group(0) == 'foo' +_RE_FIRST_COMPONENT = re.compile(r'^[^-_.]+') + + +def _DropCommonSuffixes(filename): + """Drops common suffixes like _test.cc or -inl.h from filename. + + For example: + >>> _DropCommonSuffixes('foo/foo-inl.h') + 'foo/foo' + >>> _DropCommonSuffixes('foo/bar/foo.cc') + 'foo/bar/foo' + >>> _DropCommonSuffixes('foo/foo_internal.h') + 'foo/foo' + >>> _DropCommonSuffixes('foo/foo_unusualinternal.h') + 'foo/foo_unusualinternal' + + Args: + filename: The input filename. + + Returns: + The filename with the common suffix removed. + """ + for suffix in ('test.cc', 'regtest.cc', 'unittest.cc', + 'inl.h', 'impl.h', 'internal.h'): + if (filename.endswith(suffix) and len(filename) > len(suffix) and + filename[-len(suffix) - 1] in ('-', '_')): + return filename[:-len(suffix) - 1] + return os.path.splitext(filename)[0] + + +def _IsTestFilename(filename): + """Determines if the given filename has a suffix that identifies it as a test. + + Args: + filename: The input filename. + + Returns: + True if 'filename' looks like a test, False otherwise. + """ + if (filename.endswith('_test.cc') or + filename.endswith('_unittest.cc') or + filename.endswith('_regtest.cc')): + return True + else: + return False + + +def _ClassifyInclude(fileinfo, include, is_system): + """Figures out what kind of header 'include' is. + + Args: + fileinfo: The current file cpplint is running over. A FileInfo instance. + include: The path to a #included file. + is_system: True if the #include used <> rather than "". + + Returns: + One of the _XXX_HEADER constants. + + For example: + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'stdio.h', True) + _C_SYS_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'string', True) + _CPP_SYS_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/foo.h', False) + _LIKELY_MY_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo_unknown_extension.cc'), + ... 'bar/foo_other_ext.h', False) + _POSSIBLE_MY_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False) + _OTHER_HEADER + """ + # This is a list of all standard c++ header files, except + # those already checked for above. + is_cpp_h = include in _CPP_HEADERS + + if is_system: + if is_cpp_h: + return _CPP_SYS_HEADER + else: + return _C_SYS_HEADER + + # If the target file and the include we're checking share a + # basename when we drop common extensions, and the include + # lives in . , then it's likely to be owned by the target file. + target_dir, target_base = ( + os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName()))) + include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) + if target_base == include_base and ( + include_dir == target_dir or + include_dir == os.path.normpath(target_dir + '/../public')): + return _LIKELY_MY_HEADER + + # If the target and include share some initial basename + # component, it's possible the target is implementing the + # include, so it's allowed to be first, but we'll never + # complain if it's not there. + target_first_component = _RE_FIRST_COMPONENT.match(target_base) + include_first_component = _RE_FIRST_COMPONENT.match(include_base) + if (target_first_component and include_first_component and + target_first_component.group(0) == + include_first_component.group(0)): + return _POSSIBLE_MY_HEADER + + return _OTHER_HEADER + + + +def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): + """Check rules that are applicable to #include lines. + + Strings on #include lines are NOT removed from elided line, to make + certain tasks easier. However, to prevent false positives, checks + applicable to #include lines in CheckLanguage must be put here. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + include_state: An _IncludeState instance in which the headers are inserted. + error: The function to call with any errors found. + """ + fileinfo = FileInfo(filename) + + line = clean_lines.lines[linenum] + + # "include" should use the new style "foo/bar.h" instead of just "bar.h" + if _RE_PATTERN_INCLUDE_NEW_STYLE.search(line): + error(filename, linenum, 'build/include', 4, + 'Include the directory when naming .h files') + + # we shouldn't include a file more than once. actually, there are a + # handful of instances where doing so is okay, but in general it's + # not. + match = _RE_PATTERN_INCLUDE.search(line) + if match: + include = match.group(2) + is_system = (match.group(1) == '<') + if include in include_state: + error(filename, linenum, 'build/include', 4, + '"%s" already included at %s:%s' % + (include, filename, include_state[include])) + else: + include_state[include] = linenum + + # We want to ensure that headers appear in the right order: + # 1) for foo.cc, foo.h (preferred location) + # 2) c system files + # 3) cpp system files + # 4) for foo.cc, foo.h (deprecated location) + # 5) other google headers + # + # We classify each include statement as one of those 5 types + # using a number of techniques. The include_state object keeps + # track of the highest type seen, and complains if we see a + # lower type after that. + error_message = include_state.CheckNextIncludeOrder( + _ClassifyInclude(fileinfo, include, is_system)) + if error_message: + error(filename, linenum, 'build/include_order', 4, + '%s. Should be: %s.h, c system, c++ system, other.' % + (error_message, fileinfo.BaseName())) + canonical_include = include_state.CanonicalizeAlphabeticalOrder(include) + if not include_state.IsInAlphabeticalOrder( + clean_lines, linenum, canonical_include): + error(filename, linenum, 'build/include_alpha', 4, + 'Include "%s" not in alphabetical order' % include) + include_state.SetLastHeader(canonical_include) + + # Look for any of the stream classes that are part of standard C++. + match = _RE_PATTERN_INCLUDE.match(line) + if match: + include = match.group(2) + if Match(r'(f|ind|io|i|o|parse|pf|stdio|str|)?stream$', include): + # Many unit tests use cout, so we exempt them. + if not _IsTestFilename(filename): + error(filename, linenum, 'readability/streams', 3, + 'Streams are highly discouraged.') + + +def _GetTextInside(text, start_pattern): + r"""Retrieves all the text between matching open and close parentheses. + + Given a string of lines and a regular expression string, retrieve all the text + following the expression and between opening punctuation symbols like + (, [, or {, and the matching close-punctuation symbol. This properly nested + occurrences of the punctuations, so for the text like + printf(a(), b(c())); + a call to _GetTextInside(text, r'printf\(') will return 'a(), b(c())'. + start_pattern must match string having an open punctuation symbol at the end. + + Args: + text: The lines to extract text. Its comments and strings must be elided. + It can be single line and can span multiple lines. + start_pattern: The regexp string indicating where to start extracting + the text. + Returns: + The extracted text. + None if either the opening string or ending punctuation could not be found. + """ + # TODO(sugawarayu): Audit cpplint.py to see what places could be profitably + # rewritten to use _GetTextInside (and use inferior regexp matching today). + + # Give opening punctuations to get the matching close-punctuations. + matching_punctuation = {'(': ')', '{': '}', '[': ']'} + closing_punctuation = set(matching_punctuation.itervalues()) + + # Find the position to start extracting text. + match = re.search(start_pattern, text, re.M) + if not match: # start_pattern not found in text. + return None + start_position = match.end(0) + + assert start_position > 0, ( + 'start_pattern must ends with an opening punctuation.') + assert text[start_position - 1] in matching_punctuation, ( + 'start_pattern must ends with an opening punctuation.') + # Stack of closing punctuations we expect to have in text after position. + punctuation_stack = [matching_punctuation[text[start_position - 1]]] + position = start_position + while punctuation_stack and position < len(text): + if text[position] == punctuation_stack[-1]: + punctuation_stack.pop() + elif text[position] in closing_punctuation: + # A closing punctuation without matching opening punctuations. + return None + elif text[position] in matching_punctuation: + punctuation_stack.append(matching_punctuation[text[position]]) + position += 1 + if punctuation_stack: + # Opening punctuations left without matching close-punctuations. + return None + # punctuations match. + return text[start_position:position - 1] + + +# Patterns for matching call-by-reference parameters. +# +# Supports nested templates up to 2 levels deep using this messy pattern: +# < (?: < (?: < [^<>]* +# > +# | [^<>] )* +# > +# | [^<>] )* +# > +_RE_PATTERN_IDENT = r'[_a-zA-Z]\w*' # =~ [[:alpha:]][[:alnum:]]* +_RE_PATTERN_TYPE = ( + r'(?:const\s+)?(?:typename\s+|class\s+|struct\s+|union\s+|enum\s+)?' + r'(?:\w|' + r'\s*<(?:<(?:<[^<>]*>|[^<>])*>|[^<>])*>|' + r'::)+') +# A call-by-reference parameter ends with '& identifier'. +_RE_PATTERN_REF_PARAM = re.compile( + r'(' + _RE_PATTERN_TYPE + r'(?:\s*(?:\bconst\b|[*]))*\s*' + r'&\s*' + _RE_PATTERN_IDENT + r')\s*(?:=[^,()]+)?[,)]') +# A call-by-const-reference parameter either ends with 'const& identifier' +# or looks like 'const type& identifier' when 'type' is atomic. +_RE_PATTERN_CONST_REF_PARAM = ( + r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT + + r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')') + + +def CheckLanguage(filename, clean_lines, linenum, file_extension, + include_state, nesting_state, error): + """Checks rules from the 'C++ language rules' section of cppguide.html. + + Some of these rules are hard to test (function overloading, using + uint32 inappropriately), but we do the best we can. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + file_extension: The extension (without the dot) of the filename. + include_state: An _IncludeState instance in which the headers are inserted. + nesting_state: A _NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + # If the line is empty or consists of entirely a comment, no need to + # check it. + line = clean_lines.elided[linenum] + if not line: + return + + match = _RE_PATTERN_INCLUDE.search(line) + if match: + CheckIncludeLine(filename, clean_lines, linenum, include_state, error) + return + + # Reset include state across preprocessor directives. This is meant + # to silence warnings for conditional includes. + if Match(r'^\s*#\s*(?:ifdef|elif|else|endif)\b', line): + include_state.ResetSection() + + # Make Windows paths like Unix. + fullname = os.path.abspath(filename).replace('\\', '/') + + # TODO(unknown): figure out if they're using default arguments in fn proto. + + # Check to see if they're using an conversion function cast. + # I just try to capture the most common basic types, though there are more. + # Parameterless conversion functions, such as bool(), are allowed as they are + # probably a member operator declaration or default constructor. + match = Search( + r'(\bnew\s+)?\b' # Grab 'new' operator, if it's there + r'(int|float|double|bool|char|int32|uint32|int64|uint64)' + r'(\([^)].*)', line) + if match: + matched_new = match.group(1) + matched_type = match.group(2) + matched_funcptr = match.group(3) + + # gMock methods are defined using some variant of MOCK_METHODx(name, type) + # where type may be float(), int(string), etc. Without context they are + # virtually indistinguishable from int(x) casts. Likewise, gMock's + # MockCallback takes a template parameter of the form return_type(arg_type), + # which looks much like the cast we're trying to detect. + # + # std::function<> wrapper has a similar problem. + # + # Return types for function pointers also look like casts if they + # don't have an extra space. + if (matched_new is None and # If new operator, then this isn't a cast + not (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or + Search(r'\bMockCallback<.*>', line) or + Search(r'\bstd::function<.*>', line)) and + not (matched_funcptr and + Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(', + matched_funcptr))): + # Try a bit harder to catch gmock lines: the only place where + # something looks like an old-style cast is where we declare the + # return type of the mocked method, and the only time when we + # are missing context is if MOCK_METHOD was split across + # multiple lines. The missing MOCK_METHOD is usually one or two + # lines back, so scan back one or two lines. + # + # It's not possible for gmock macros to appear in the first 2 + # lines, since the class head + section name takes up 2 lines. + if (linenum < 2 or + not (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$', + clean_lines.elided[linenum - 1]) or + Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$', + clean_lines.elided[linenum - 2]))): + error(filename, linenum, 'readability/casting', 4, + 'Using deprecated casting style. ' + 'Use static_cast<%s>(...) instead' % + matched_type) + + CheckCStyleCast(filename, linenum, line, clean_lines.raw_lines[linenum], + 'static_cast', + r'\((int|float|double|bool|char|u?int(16|32|64))\)', error) + + # This doesn't catch all cases. Consider (const char * const)"hello". + # + # (char *) "foo" should always be a const_cast (reinterpret_cast won't + # compile). + if CheckCStyleCast(filename, linenum, line, clean_lines.raw_lines[linenum], + 'const_cast', r'\((char\s?\*+\s?)\)\s*"', error): + pass + else: + # Check pointer casts for other than string constants + CheckCStyleCast(filename, linenum, line, clean_lines.raw_lines[linenum], + 'reinterpret_cast', r'\((\w+\s?\*+\s?)\)', error) + + # In addition, we look for people taking the address of a cast. This + # is dangerous -- casts can assign to temporaries, so the pointer doesn't + # point where you think. + match = Search( + r'(?:&\(([^)]+)\)[\w(])|' + r'(?:&(static|dynamic|down|reinterpret)_cast\b)', line) + if match and match.group(1) != '*': + error(filename, linenum, 'runtime/casting', 4, + ('Are you taking an address of a cast? ' + 'This is dangerous: could be a temp var. ' + 'Take the address before doing the cast, rather than after')) + + # Create an extended_line, which is the concatenation of the current and + # next lines, for more effective checking of code that may span more than one + # line. + if linenum + 1 < clean_lines.NumLines(): + extended_line = line + clean_lines.elided[linenum + 1] + else: + extended_line = line + + # Check for people declaring static/global STL strings at the top level. + # This is dangerous because the C++ language does not guarantee that + # globals with constructors are initialized before the first access. + match = Match( + r'((?:|static +)(?:|const +))string +([a-zA-Z0-9_:]+)\b(.*)', + line) + # Make sure it's not a function. + # Function template specialization looks like: "string foo(...". + # Class template definitions look like: "string Foo::Method(...". + # + # Also ignore things that look like operators. These are matched separately + # because operator names cross non-word boundaries. If we change the pattern + # above, we would decrease the accuracy of matching identifiers. + if (match and + not Search(r'\boperator\W', line) and + not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)?\s*\(([^"]|$)', match.group(3))): + error(filename, linenum, 'runtime/string', 4, + 'For a static/global string constant, use a C style string instead: ' + '"%schar %s[]".' % + (match.group(1), match.group(2))) + + if Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line): + error(filename, linenum, 'runtime/init', 4, + 'You seem to be initializing a member variable with itself.') + + if file_extension == 'h': + # TODO(unknown): check that 1-arg constructors are explicit. + # How to tell it's a constructor? + # (handled in CheckForNonStandardConstructs for now) + # TODO(unknown): check that classes have DISALLOW_EVIL_CONSTRUCTORS + # (level 1 error) + pass + + # Check if people are using the verboten C basic types. The only exception + # we regularly allow is "unsigned short port" for port. + if Search(r'\bshort port\b', line): + if not Search(r'\bunsigned short port\b', line): + error(filename, linenum, 'runtime/int', 4, + 'Use "unsigned short" for ports, not "short"') + else: + match = Search(r'\b(short|long(?! +double)|long long)\b', line) + if match: + error(filename, linenum, 'runtime/int', 4, + 'Use int16/int64/etc, rather than the C type %s' % match.group(1)) + + # When snprintf is used, the second argument shouldn't be a literal. + match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) + if match and match.group(2) != '0': + # If 2nd arg is zero, snprintf is used to calculate size. + error(filename, linenum, 'runtime/printf', 3, + 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' + 'to snprintf.' % (match.group(1), match.group(2))) + + # Check if some verboten C functions are being used. + if Search(r'\bsprintf\b', line): + error(filename, linenum, 'runtime/printf', 5, + 'Never use sprintf. Use snprintf instead.') + match = Search(r'\b(strcpy|strcat)\b', line) + if match: + error(filename, linenum, 'runtime/printf', 4, + 'Almost always, snprintf is better than %s' % match.group(1)) + + # Check if some verboten operator overloading is going on + # TODO(unknown): catch out-of-line unary operator&: + # class X {}; + # int operator&(const X& x) { return 42; } // unary operator& + # The trick is it's hard to tell apart from binary operator&: + # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& + if Search(r'\boperator\s*&\s*\(\s*\)', line): + error(filename, linenum, 'runtime/operator', 4, + 'Unary operator& is dangerous. Do not use it.') + + # Check for suspicious usage of "if" like + # } if (a == b) { + if Search(r'\}\s*if\s*\(', line): + error(filename, linenum, 'readability/braces', 4, + 'Did you mean "else if"? If not, start a new line for "if".') + + # Check for potential format string bugs like printf(foo). + # We constrain the pattern not to pick things like DocidForPrintf(foo). + # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) + # TODO(sugawarayu): Catch the following case. Need to change the calling + # convention of the whole function to process multiple line to handle it. + # printf( + # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line); + printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(') + if printf_args: + match = Match(r'([\w.\->()]+)$', printf_args) + if match and match.group(1) != '__VA_ARGS__': + function_name = re.search(r'\b((?:string)?printf)\s*\(', + line, re.I).group(1) + error(filename, linenum, 'runtime/printf', 4, + 'Potential format string bug. Do %s("%%s", %s) instead.' + % (function_name, match.group(1))) + + # Check for potential memset bugs like memset(buf, sizeof(buf), 0). + match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) + if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): + error(filename, linenum, 'runtime/memset', 4, + 'Did you mean "memset(%s, 0, %s)"?' + % (match.group(1), match.group(2))) + + if Search(r'\busing namespace\b', line): + error(filename, linenum, 'build/namespaces', 5, + 'Do not use namespace using-directives. ' + 'Use using-declarations instead.') + + # Detect variable-length arrays. + match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) + if (match and match.group(2) != 'return' and match.group(2) != 'delete' and + match.group(3).find(']') == -1): + # Split the size using space and arithmetic operators as delimiters. + # If any of the resulting tokens are not compile time constants then + # report the error. + tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) + is_const = True + skip_next = False + for tok in tokens: + if skip_next: + skip_next = False + continue + + if Search(r'sizeof\(.+\)', tok): continue + if Search(r'arraysize\(\w+\)', tok): continue + + tok = tok.lstrip('(') + tok = tok.rstrip(')') + if not tok: continue + if Match(r'\d+', tok): continue + if Match(r'0[xX][0-9a-fA-F]+', tok): continue + if Match(r'k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue + # A catch all for tricky sizeof cases, including 'sizeof expression', + # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' + # requires skipping the next token because we split on ' ' and '*'. + if tok.startswith('sizeof'): + skip_next = True + continue + is_const = False + break + if not is_const: + error(filename, linenum, 'runtime/arrays', 1, + 'Do not use variable-length arrays. Use an appropriately named ' + "('k' followed by CamelCase) compile-time constant for the size.") + + # If DISALLOW_EVIL_CONSTRUCTORS, DISALLOW_COPY_AND_ASSIGN, or + # DISALLOW_IMPLICIT_CONSTRUCTORS is present, then it should be the last thing + # in the class declaration. + match = Match( + (r'\s*' + r'(DISALLOW_(EVIL_CONSTRUCTORS|COPY_AND_ASSIGN|IMPLICIT_CONSTRUCTORS))' + r'\(.*\);$'), + line) + if match and linenum + 1 < clean_lines.NumLines(): + next_line = clean_lines.elided[linenum + 1] + # We allow some, but not all, declarations of variables to be present + # in the statement that defines the class. The [\w\*,\s]* fragment of + # the regular expression below allows users to declare instances of + # the class or pointers to instances, but not less common types such + # as function pointers or arrays. It's a tradeoff between allowing + # reasonable code and avoiding trying to parse more C++ using regexps. + if not Search(r'^\s*}[\w\*,\s]*;', next_line): + error(filename, linenum, 'readability/constructors', 3, + match.group(1) + ' should be the last thing in the class') + + # Check for use of unnamed namespaces in header files. Registration + # macros are typically OK, so we allow use of "namespace {" on lines + # that end with backslashes. + if (file_extension == 'h' + and Search(r'\bnamespace\s*{', line) + and line[-1] != '\\'): + error(filename, linenum, 'build/namespaces', 4, + 'Do not use unnamed namespaces in header files. See ' + 'http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces' + ' for more information.') + +def CheckForNonConstReference(filename, clean_lines, linenum, + nesting_state, error): + """Check for non-const references. + + Separate from CheckLanguage since it scans backwards from current + line, instead of scanning forward. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + nesting_state: A _NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + # Do nothing if there is no '&' on current line. + line = clean_lines.elided[linenum] + if '&' not in line: + return + + # Long type names may be broken across multiple lines, usually in one + # of these forms: + # LongType + # ::LongTypeContinued &identifier + # LongType:: + # LongTypeContinued &identifier + # LongType< + # ...>::LongTypeContinued &identifier + # + # If we detected a type split across two lines, join the previous + # line to current line so that we can match const references + # accordingly. + # + # Note that this only scans back one line, since scanning back + # arbitrary number of lines would be expensive. If you have a type + # that spans more than 2 lines, please use a typedef. + if linenum > 1: + previous = None + if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line): + # previous_line\n + ::current_line + previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$', + clean_lines.elided[linenum - 1]) + elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line): + # previous_line::\n + current_line + previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$', + clean_lines.elided[linenum - 1]) + if previous: + line = previous.group(1) + line.lstrip() + else: + # Check for templated parameter that is split across multiple lines + endpos = line.rfind('>') + if endpos > -1: + (_, startline, startpos) = ReverseCloseExpression( + clean_lines, linenum, endpos) + if startpos > -1 and startline < linenum: + # Found the matching < on an earlier line, collect all + # pieces up to current line. + line = '' + for i in xrange(startline, linenum + 1): + line += clean_lines.elided[i].strip() + + # Check for non-const references in function parameters. A single '&' may + # found in the following places: + # inside expression: binary & for bitwise AND + # inside expression: unary & for taking the address of something + # inside declarators: reference parameter + # We will exclude the first two cases by checking that we are not inside a + # function body, including one that was just introduced by a trailing '{'. + # TODO(unknwon): Doesn't account for preprocessor directives. + # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare]. + check_params = False + if not nesting_state.stack: + check_params = True # top level + elif (isinstance(nesting_state.stack[-1], _ClassInfo) or + isinstance(nesting_state.stack[-1], _NamespaceInfo)): + check_params = True # within class or namespace + elif Match(r'.*{\s*$', line): + if (len(nesting_state.stack) == 1 or + isinstance(nesting_state.stack[-2], _ClassInfo) or + isinstance(nesting_state.stack[-2], _NamespaceInfo)): + check_params = True # just opened global/class/namespace block + # We allow non-const references in a few standard places, like functions + # called "swap()" or iostream operators like "<<" or ">>". Do not check + # those function parameters. + # + # We also accept & in static_assert, which looks like a function but + # it's actually a declaration expression. + whitelisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|' + r'operator\s*[<>][<>]|' + r'static_assert|COMPILE_ASSERT' + r')\s*\(') + if Search(whitelisted_functions, line): + check_params = False + elif not Search(r'\S+\([^)]*$', line): + # Don't see a whitelisted function on this line. Actually we + # didn't see any function name on this line, so this is likely a + # multi-line parameter list. Try a bit harder to catch this case. + for i in xrange(2): + if (linenum > i and + Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])): + check_params = False + break + + if check_params: + decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body + for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls): + if not Match(_RE_PATTERN_CONST_REF_PARAM, parameter): + error(filename, linenum, 'runtime/references', 2, + 'Is this a non-const reference? ' + 'If so, make const or use a pointer: ' + + ReplaceAll(' *<', '<', parameter)) + + +def CheckCStyleCast(filename, linenum, line, raw_line, cast_type, pattern, + error): + """Checks for a C-style cast by looking for the pattern. + + Args: + filename: The name of the current file. + linenum: The number of the line to check. + line: The line of code to check. + raw_line: The raw line of code to check, with comments. + cast_type: The string for the C++ cast to recommend. This is either + reinterpret_cast, static_cast, or const_cast, depending. + pattern: The regular expression used to find C-style casts. + error: The function to call with any errors found. + + Returns: + True if an error was emitted. + False otherwise. + """ + match = Search(pattern, line) + if not match: + return False + + # e.g., sizeof(int) + sizeof_match = Match(r'.*sizeof\s*$', line[0:match.start(1) - 1]) + if sizeof_match: + error(filename, linenum, 'runtime/sizeof', 1, + 'Using sizeof(type). Use sizeof(varname) instead if possible') + return True + + # operator++(int) and operator--(int) + if (line[0:match.start(1) - 1].endswith(' operator++') or + line[0:match.start(1) - 1].endswith(' operator--')): + return False + + # A single unnamed argument for a function tends to look like old + # style cast. If we see those, don't issue warnings for deprecated + # casts, instead issue warnings for unnamed arguments where + # appropriate. + # + # These are things that we want warnings for, since the style guide + # explicitly require all parameters to be named: + # Function(int); + # Function(int) { + # ConstMember(int) const; + # ConstMember(int) const { + # ExceptionMember(int) throw (...); + # ExceptionMember(int) throw (...) { + # PureVirtual(int) = 0; + # + # These are functions of some sort, where the compiler would be fine + # if they had named parameters, but people often omit those + # identifiers to reduce clutter: + # (FunctionPointer)(int); + # (FunctionPointer)(int) = value; + # Function((function_pointer_arg)(int)) + # ; + # <(FunctionPointerTemplateArgument)(int)>; + remainder = line[match.end(0):] + if Match(r'^\s*(?:;|const\b|throw\b|=|>|\{|\))', remainder): + # Looks like an unnamed parameter. + + # Don't warn on any kind of template arguments. + if Match(r'^\s*>', remainder): + return False + + # Don't warn on assignments to function pointers, but keep warnings for + # unnamed parameters to pure virtual functions. Note that this pattern + # will also pass on assignments of "0" to function pointers, but the + # preferred values for those would be "nullptr" or "NULL". + matched_zero = Match(r'^\s=\s*(\S+)\s*;', remainder) + if matched_zero and matched_zero.group(1) != '0': + return False + + # Don't warn on function pointer declarations. For this we need + # to check what came before the "(type)" string. + if Match(r'.*\)\s*$', line[0:match.start(0)]): + return False + + # Don't warn if the parameter is named with block comments, e.g.: + # Function(int /*unused_param*/); + if '/*' in raw_line: + return False + + # Passed all filters, issue warning here. + error(filename, linenum, 'readability/function', 3, + 'All parameters should be named in a function') + return True + + # At this point, all that should be left is actual casts. + error(filename, linenum, 'readability/casting', 4, + 'Using C-style cast. Use %s<%s>(...) instead' % + (cast_type, match.group(1))) + + return True + + +_HEADERS_CONTAINING_TEMPLATES = ( + ('', ('deque',)), + ('', ('unary_function', 'binary_function', + 'plus', 'minus', 'multiplies', 'divides', 'modulus', + 'negate', + 'equal_to', 'not_equal_to', 'greater', 'less', + 'greater_equal', 'less_equal', + 'logical_and', 'logical_or', 'logical_not', + 'unary_negate', 'not1', 'binary_negate', 'not2', + 'bind1st', 'bind2nd', + 'pointer_to_unary_function', + 'pointer_to_binary_function', + 'ptr_fun', + 'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t', + 'mem_fun_ref_t', + 'const_mem_fun_t', 'const_mem_fun1_t', + 'const_mem_fun_ref_t', 'const_mem_fun1_ref_t', + 'mem_fun_ref', + )), + ('', ('numeric_limits',)), + ('', ('list',)), + ('', ('map', 'multimap',)), + ('', ('allocator',)), + ('', ('queue', 'priority_queue',)), + ('', ('set', 'multiset',)), + ('', ('stack',)), + ('', ('char_traits', 'basic_string',)), + ('', ('pair',)), + ('', ('vector',)), + + # gcc extensions. + # Note: std::hash is their hash, ::hash is our hash + ('', ('hash_map', 'hash_multimap',)), + ('', ('hash_set', 'hash_multiset',)), + ('', ('slist',)), + ) + +_RE_PATTERN_STRING = re.compile(r'\bstring\b') + +_re_pattern_algorithm_header = [] +for _template in ('copy', 'max', 'min', 'min_element', 'sort', 'swap', + 'transform'): + # Match max(..., ...), max(..., ...), but not foo->max, foo.max or + # type::max(). + _re_pattern_algorithm_header.append( + (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'), + _template, + '')) + +_re_pattern_templates = [] +for _header, _templates in _HEADERS_CONTAINING_TEMPLATES: + for _template in _templates: + _re_pattern_templates.append( + (re.compile(r'(\<|\b)' + _template + r'\s*\<'), + _template + '<>', + _header)) + + +def FilesBelongToSameModule(filename_cc, filename_h): + """Check if these two filenames belong to the same module. + + The concept of a 'module' here is a as follows: + foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the + same 'module' if they are in the same directory. + some/path/public/xyzzy and some/path/internal/xyzzy are also considered + to belong to the same module here. + + If the filename_cc contains a longer path than the filename_h, for example, + '/absolute/path/to/base/sysinfo.cc', and this file would include + 'base/sysinfo.h', this function also produces the prefix needed to open the + header. This is used by the caller of this function to more robustly open the + header file. We don't have access to the real include paths in this context, + so we need this guesswork here. + + Known bugs: tools/base/bar.cc and base/bar.h belong to the same module + according to this implementation. Because of this, this function gives + some false positives. This should be sufficiently rare in practice. + + Args: + filename_cc: is the path for the .cc file + filename_h: is the path for the header path + + Returns: + Tuple with a bool and a string: + bool: True if filename_cc and filename_h belong to the same module. + string: the additional prefix needed to open the header file. + """ + + if not filename_cc.endswith('.cc'): + return (False, '') + filename_cc = filename_cc[:-len('.cc')] + if filename_cc.endswith('_unittest'): + filename_cc = filename_cc[:-len('_unittest')] + elif filename_cc.endswith('_test'): + filename_cc = filename_cc[:-len('_test')] + filename_cc = filename_cc.replace('/public/', '/') + filename_cc = filename_cc.replace('/internal/', '/') + + if not filename_h.endswith('.h'): + return (False, '') + filename_h = filename_h[:-len('.h')] + if filename_h.endswith('-inl'): + filename_h = filename_h[:-len('-inl')] + filename_h = filename_h.replace('/public/', '/') + filename_h = filename_h.replace('/internal/', '/') + + files_belong_to_same_module = filename_cc.endswith(filename_h) + common_path = '' + if files_belong_to_same_module: + common_path = filename_cc[:-len(filename_h)] + return files_belong_to_same_module, common_path + + +def UpdateIncludeState(filename, include_state, io=codecs): + """Fill up the include_state with new includes found from the file. + + Args: + filename: the name of the header to read. + include_state: an _IncludeState instance in which the headers are inserted. + io: The io factory to use to read the file. Provided for testability. + + Returns: + True if a header was succesfully added. False otherwise. + """ + headerfile = None + try: + headerfile = io.open(filename, 'r', 'utf8', 'replace') + except IOError: + return False + linenum = 0 + for line in headerfile: + linenum += 1 + clean_line = CleanseComments(line) + match = _RE_PATTERN_INCLUDE.search(clean_line) + if match: + include = match.group(2) + # The value formatting is cute, but not really used right now. + # What matters here is that the key is in include_state. + include_state.setdefault(include, '%s:%d' % (filename, linenum)) + return True + + +def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, + io=codecs): + """Reports for missing stl includes. + + This function will output warnings to make sure you are including the headers + necessary for the stl containers and functions that you use. We only give one + reason to include a header. For example, if you use both equal_to<> and + less<> in a .h file, only one (the latter in the file) of these will be + reported as a reason to include the . + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + include_state: An _IncludeState instance. + error: The function to call with any errors found. + io: The IO factory to use to read the header file. Provided for unittest + injection. + """ + required = {} # A map of header name to linenumber and the template entity. + # Example of required: { '': (1219, 'less<>') } + + for linenum in xrange(clean_lines.NumLines()): + line = clean_lines.elided[linenum] + if not line or line[0] == '#': + continue + + # String is special -- it is a non-templatized type in STL. + matched = _RE_PATTERN_STRING.search(line) + if matched: + # Don't warn about strings in non-STL namespaces: + # (We check only the first match per line; good enough.) + prefix = line[:matched.start()] + if prefix.endswith('std::') or not prefix.endswith('::'): + required[''] = (linenum, 'string') + + for pattern, template, header in _re_pattern_algorithm_header: + if pattern.search(line): + required[header] = (linenum, template) + + # The following function is just a speed up, no semantics are changed. + if not '<' in line: # Reduces the cpu time usage by skipping lines. + continue + + for pattern, template, header in _re_pattern_templates: + if pattern.search(line): + required[header] = (linenum, template) + + # The policy is that if you #include something in foo.h you don't need to + # include it again in foo.cc. Here, we will look at possible includes. + # Let's copy the include_state so it is only messed up within this function. + include_state = include_state.copy() + + # Did we find the header for this file (if any) and succesfully load it? + header_found = False + + # Use the absolute path so that matching works properly. + abs_filename = FileInfo(filename).FullName() + + # For Emacs's flymake. + # If cpplint is invoked from Emacs's flymake, a temporary file is generated + # by flymake and that file name might end with '_flymake.cc'. In that case, + # restore original file name here so that the corresponding header file can be + # found. + # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h' + # instead of 'foo_flymake.h' + abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename) + + # include_state is modified during iteration, so we iterate over a copy of + # the keys. + header_keys = include_state.keys() + for header in header_keys: + (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) + fullpath = common_path + header + if same_module and UpdateIncludeState(fullpath, include_state, io): + header_found = True + + # If we can't find the header file for a .cc, assume it's because we don't + # know where to look. In that case we'll give up as we're not sure they + # didn't include it in the .h file. + # TODO(unknown): Do a better job of finding .h files so we are confident that + # not having the .h file means there isn't one. + if filename.endswith('.cc') and not header_found: + return + + # All the lines have been processed, report the errors found. + for required_header_unstripped in required: + template = required[required_header_unstripped][1] + if required_header_unstripped.strip('<>"') not in include_state: + error(filename, required[required_header_unstripped][0], + 'build/include_what_you_use', 4, + 'Add #include ' + required_header_unstripped + ' for ' + template) + + +_RE_PATTERN_EXPLICIT_MAKEPAIR = re.compile(r'\bmake_pair\s*<') + + +def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error): + """Check that make_pair's template arguments are deduced. + + G++ 4.6 in C++0x mode fails badly if make_pair's template arguments are + specified explicitly, and such use isn't intended in any case. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line) + if match: + error(filename, linenum, 'build/explicit_make_pair', + 4, # 4 = high confidence + 'For C++11-compatibility, omit template arguments from make_pair' + ' OR use pair directly OR if appropriate, construct a pair directly') + + +def ProcessLine(filename, file_extension, clean_lines, line, + include_state, function_state, nesting_state, error, + extra_check_functions=[]): + """Processes a single line in the file. + + Args: + filename: Filename of the file that is being processed. + file_extension: The extension (dot not included) of the file. + clean_lines: An array of strings, each representing a line of the file, + with comments stripped. + line: Number of line being processed. + include_state: An _IncludeState instance in which the headers are inserted. + function_state: A _FunctionState instance which counts function lines, etc. + nesting_state: A _NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: A callable to which errors are reported, which takes 4 arguments: + filename, line number, error level, and message + extra_check_functions: An array of additional check functions that will be + run on each source line. Each function takes 4 + arguments: filename, clean_lines, line, error + """ + raw_lines = clean_lines.raw_lines + ParseNolintSuppressions(filename, raw_lines[line], line, error) + nesting_state.Update(filename, clean_lines, line, error) + if nesting_state.stack and nesting_state.stack[-1].inline_asm != _NO_ASM: + return + CheckForFunctionLengths(filename, clean_lines, line, function_state, error) + CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) + CheckStyle(filename, clean_lines, line, file_extension, nesting_state, error) + CheckLanguage(filename, clean_lines, line, file_extension, include_state, + nesting_state, error) + CheckForNonConstReference(filename, clean_lines, line, nesting_state, error) + CheckForNonStandardConstructs(filename, clean_lines, line, + nesting_state, error) + CheckVlogArguments(filename, clean_lines, line, error) + CheckPosixThreading(filename, clean_lines, line, error) + CheckInvalidIncrement(filename, clean_lines, line, error) + CheckMakePairUsesDeduction(filename, clean_lines, line, error) + for check_fn in extra_check_functions: + check_fn(filename, clean_lines, line, error) + +def ProcessFileData(filename, file_extension, lines, error, + extra_check_functions=[]): + """Performs lint checks and reports any errors to the given error function. + + Args: + filename: Filename of the file that is being processed. + file_extension: The extension (dot not included) of the file. + lines: An array of strings, each representing a line of the file, with the + last element being empty if the file is terminated with a newline. + error: A callable to which errors are reported, which takes 4 arguments: + filename, line number, error level, and message + extra_check_functions: An array of additional check functions that will be + run on each source line. Each function takes 4 + arguments: filename, clean_lines, line, error + """ + lines = (['// marker so line numbers and indices both start at 1'] + lines + + ['// marker so line numbers end in a known way']) + + include_state = _IncludeState() + function_state = _FunctionState() + nesting_state = _NestingState() + + ResetNolintSuppressions() + + CheckForCopyright(filename, lines, error) + + if file_extension == 'h': + CheckForHeaderGuard(filename, lines, error) + + RemoveMultiLineComments(filename, lines, error) + clean_lines = CleansedLines(lines) + for line in xrange(clean_lines.NumLines()): + ProcessLine(filename, file_extension, clean_lines, line, + include_state, function_state, nesting_state, error, + extra_check_functions) + nesting_state.CheckCompletedBlocks(filename, error) + + CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) + + # We check here rather than inside ProcessLine so that we see raw + # lines rather than "cleaned" lines. + CheckForBadCharacters(filename, lines, error) + + CheckForNewlineAtEOF(filename, lines, error) + +def ProcessFile(filename, vlevel, extra_check_functions=[]): + """Does google-lint on a single file. + + Args: + filename: The name of the file to parse. + + vlevel: The level of errors to report. Every error of confidence + >= verbose_level will be reported. 0 is a good default. + + extra_check_functions: An array of additional check functions that will be + run on each source line. Each function takes 4 + arguments: filename, clean_lines, line, error + """ + + _SetVerboseLevel(vlevel) + + try: + # Support the UNIX convention of using "-" for stdin. Note that + # we are not opening the file with universal newline support + # (which codecs doesn't support anyway), so the resulting lines do + # contain trailing '\r' characters if we are reading a file that + # has CRLF endings. + # If after the split a trailing '\r' is present, it is removed + # below. If it is not expected to be present (i.e. os.linesep != + # '\r\n' as in Windows), a warning is issued below if this file + # is processed. + + if filename == '-': + lines = codecs.StreamReaderWriter(sys.stdin, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace').read().split('\n') + else: + lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n') + + carriage_return_found = False + # Remove trailing '\r'. + for linenum in range(len(lines)): + if lines[linenum].endswith('\r'): + lines[linenum] = lines[linenum].rstrip('\r') + carriage_return_found = True + + except IOError: + sys.stderr.write( + "Skipping input '%s': Can't open for reading\n" % filename) + return + + # Note, if no dot is found, this will give the entire filename as the ext. + file_extension = filename[filename.rfind('.') + 1:] + + # When reading from stdin, the extension is unknown, so no cpplint tests + # should rely on the extension. + if filename != '-' and file_extension not in _valid_extensions: + sys.stderr.write('Ignoring %s; not a valid file name ' + '(%s)\n' % (filename, ', '.join(_valid_extensions))) + else: + ProcessFileData(filename, file_extension, lines, Error, + extra_check_functions) + if carriage_return_found and os.linesep != '\r\n': + # Use 0 for linenum since outputting only one error for potentially + # several lines. + Error(filename, 0, 'whitespace/newline', 1, + 'One or more unexpected \\r (^M) found;' + 'better to use only a \\n') + + sys.stderr.write('Done processing %s\n' % filename) + + +def PrintUsage(message): + """Prints a brief usage string and exits, optionally with an error message. + + Args: + message: The optional error message. + """ + sys.stderr.write(_USAGE) + if message: + sys.exit('\nFATAL ERROR: ' + message) + else: + sys.exit(1) + + +def PrintCategories(): + """Prints a list of all the error-categories used by error messages. + + These are the categories used to filter messages via --filter. + """ + sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES)) + sys.exit(0) + + +def ParseArguments(args): + """Parses the command line arguments. + + This may set the output format and verbosity level as side-effects. + + Args: + args: The command line arguments: + + Returns: + The list of filenames to lint. + """ + try: + (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=', + 'counting=', + 'filter=', + 'root=', + 'linelength=', + 'extensions=']) + except getopt.GetoptError: + PrintUsage('Invalid arguments.') + + verbosity = _VerboseLevel() + output_format = _OutputFormat() + filters = '' + counting_style = '' + + for (opt, val) in opts: + if opt == '--help': + PrintUsage(None) + elif opt == '--output': + if val not in ('emacs', 'vs7', 'eclipse'): + PrintUsage('The only allowed output formats are emacs, vs7 and eclipse.') + output_format = val + elif opt == '--verbose': + verbosity = int(val) + elif opt == '--filter': + filters = val + if not filters: + PrintCategories() + elif opt == '--counting': + if val not in ('total', 'toplevel', 'detailed'): + PrintUsage('Valid counting options are total, toplevel, and detailed') + counting_style = val + elif opt == '--root': + global _root + _root = val + elif opt == '--linelength': + global _line_length + try: + _line_length = int(val) + except ValueError: + PrintUsage('Line length must be digits.') + elif opt == '--extensions': + global _valid_extensions + try: + _valid_extensions = set(val.split(',')) + except ValueError: + PrintUsage('Extensions must be comma seperated list.') + + if not filenames: + PrintUsage('No files were specified.') + + _SetOutputFormat(output_format) + _SetVerboseLevel(verbosity) + _SetFilters(filters) + _SetCountingStyle(counting_style) + + return filenames + + +def main(): + filenames = ParseArguments(sys.argv[1:]) + + # Change stderr to write with replacement characters so we don't die + # if we try to print something containing non-ASCII characters. + sys.stderr = codecs.StreamReaderWriter(sys.stderr, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace') + + _cpplint_state.ResetErrorCounts() + for filename in filenames: + ProcessFile(filename, _cpplint_state.verbose_level) + _cpplint_state.PrintErrorCounts() + + sys.exit(_cpplint_state.error_count > 0) + + +if __name__ == '__main__': + main() diff --git a/third_party/aom/tools/diff.py b/third_party/aom/tools/diff.py new file mode 100644 index 0000000000..bac6aabdc0 --- /dev/null +++ b/third_party/aom/tools/diff.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +"""Classes for representing diff pieces.""" + +__author__ = "jkoleszar@google.com" + +import re + + +class DiffLines(object): + """A container for one half of a diff.""" + + def __init__(self, filename, offset, length): + self.filename = filename + self.offset = offset + self.length = length + self.lines = [] + self.delta_line_nums = [] + + def Append(self, line): + l = len(self.lines) + if line[0] != " ": + self.delta_line_nums.append(self.offset + l) + self.lines.append(line[1:]) + assert l+1 <= self.length + + def Complete(self): + return len(self.lines) == self.length + + def __contains__(self, item): + return item >= self.offset and item <= self.offset + self.length - 1 + + +class DiffHunk(object): + """A container for one diff hunk, consisting of two DiffLines.""" + + def __init__(self, header, file_a, file_b, start_a, len_a, start_b, len_b): + self.header = header + self.left = DiffLines(file_a, start_a, len_a) + self.right = DiffLines(file_b, start_b, len_b) + self.lines = [] + + def Append(self, line): + """Adds a line to the DiffHunk and its DiffLines children.""" + if line[0] == "-": + self.left.Append(line) + elif line[0] == "+": + self.right.Append(line) + elif line[0] == " ": + self.left.Append(line) + self.right.Append(line) + elif line[0] == "\\": + # Ignore newline messages from git diff. + pass + else: + assert False, ("Unrecognized character at start of diff line " + "%r" % line[0]) + self.lines.append(line) + + def Complete(self): + return self.left.Complete() and self.right.Complete() + + def __repr__(self): + return "DiffHunk(%s, %s, len %d)" % ( + self.left.filename, self.right.filename, + max(self.left.length, self.right.length)) + + +def ParseDiffHunks(stream): + """Walk a file-like object, yielding DiffHunks as they're parsed.""" + + file_regex = re.compile(r"(\+\+\+|---) (\S+)") + range_regex = re.compile(r"@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))?") + hunk = None + while True: + line = stream.readline() + if not line: + break + + if hunk is None: + # Parse file names + diff_file = file_regex.match(line) + if diff_file: + if line.startswith("---"): + a_line = line + a = diff_file.group(2) + continue + if line.startswith("+++"): + b_line = line + b = diff_file.group(2) + continue + + # Parse offset/lengths + diffrange = range_regex.match(line) + if diffrange: + if diffrange.group(2): + start_a = int(diffrange.group(1)) + len_a = int(diffrange.group(3)) + else: + start_a = 1 + len_a = int(diffrange.group(1)) + + if diffrange.group(5): + start_b = int(diffrange.group(4)) + len_b = int(diffrange.group(6)) + else: + start_b = 1 + len_b = int(diffrange.group(4)) + + header = [a_line, b_line, line] + hunk = DiffHunk(header, a, b, start_a, len_a, start_b, len_b) + else: + # Add the current line to the hunk + hunk.Append(line) + + # See if the whole hunk has been parsed. If so, yield it and prepare + # for the next hunk. + if hunk.Complete(): + yield hunk + hunk = None + + # Partial hunks are a parse error + assert hunk is None diff --git a/third_party/aom/tools/ftfy.sh b/third_party/aom/tools/ftfy.sh new file mode 100755 index 0000000000..315da1af5d --- /dev/null +++ b/third_party/aom/tools/ftfy.sh @@ -0,0 +1,158 @@ +#!/bin/sh +self="$0" +dirname_self=$(dirname "$self") + +usage() { + cat <&2 +Usage: $self [option] + +This script applies a whitespace transformation to the commit at HEAD. If no +options are given, then the modified files are left in the working tree. + +Options: + -h, --help Shows this message + -n, --dry-run Shows a diff of the changes to be made. + --amend Squashes the changes into the commit at HEAD + This option will also reformat the commit message. + --commit Creates a new commit containing only the whitespace changes + --msg-only Reformat the commit message only, ignore the patch itself. + +EOF + rm -f ${CLEAN_FILES} + exit 1 +} + + +log() { + echo "${self##*/}: $@" >&2 +} + + +aom_style() { + for f; do + case "$f" in + *.h|*.c|*.cc) + clang-format -i --style=file "$f" + ;; + esac + done +} + + +apply() { + [ $INTERSECT_RESULT -ne 0 ] && patch -p1 < "$1" +} + + +commit() { + LAST_CHANGEID=$(git show | awk '/Change-Id:/{print $2}') + if [ -z "$LAST_CHANGEID" ]; then + log "HEAD doesn't have a Change-Id, unable to generate a new commit" + exit 1 + fi + + # Build a deterministic Change-Id from the parent's + NEW_CHANGEID=${LAST_CHANGEID}-styled + NEW_CHANGEID=I$(echo $NEW_CHANGEID | git hash-object --stdin) + + # Commit, preserving authorship from the parent commit. + git commit -a -C HEAD > /dev/null + git commit --amend -F- << EOF +Cosmetic: Fix whitespace in change ${LAST_CHANGEID:0:9} + +Change-Id: ${NEW_CHANGEID} +EOF +} + + +show_commit_msg_diff() { + if [ $DIFF_MSG_RESULT -ne 0 ]; then + log "Modified commit message:" + diff -u "$ORIG_COMMIT_MSG" "$NEW_COMMIT_MSG" | tail -n +3 + fi +} + + +amend() { + show_commit_msg_diff + if [ $DIFF_MSG_RESULT -ne 0 ] || [ $INTERSECT_RESULT -ne 0 ]; then + git commit -a --amend -F "$NEW_COMMIT_MSG" + fi +} + + +diff_msg() { + git log -1 --format=%B > "$ORIG_COMMIT_MSG" + "${dirname_self}"/wrap-commit-msg.py \ + < "$ORIG_COMMIT_MSG" > "$NEW_COMMIT_MSG" + cmp -s "$ORIG_COMMIT_MSG" "$NEW_COMMIT_MSG" + DIFF_MSG_RESULT=$? +} + + +# Temporary files +ORIG_DIFF=orig.diff.$$ +MODIFIED_DIFF=modified.diff.$$ +FINAL_DIFF=final.diff.$$ +ORIG_COMMIT_MSG=orig.commit-msg.$$ +NEW_COMMIT_MSG=new.commit-msg.$$ +CLEAN_FILES="${ORIG_DIFF} ${MODIFIED_DIFF} ${FINAL_DIFF}" +CLEAN_FILES="${CLEAN_FILES} ${ORIG_COMMIT_MSG} ${NEW_COMMIT_MSG}" + +# Preconditions +[ $# -lt 2 ] || usage + +if ! clang-format -version >/dev/null 2>&1; then + log "clang-format not found" + exit 1 +fi + +if ! git diff --quiet HEAD; then + log "Working tree is dirty, commit your changes first" + exit 1 +fi + +# Need to be in the root +cd "$(git rev-parse --show-toplevel)" + +# Collect the original diff +git show > "${ORIG_DIFF}" + +# Apply the style guide on new and modified files and collect its diff +for f in $(git diff HEAD^ --name-only -M90 --diff-filter=AM); do + case "$f" in + third_party/*) continue;; + esac + aom_style "$f" +done +git diff --no-color --no-ext-diff > "${MODIFIED_DIFF}" + +# Intersect the two diffs +"${dirname_self}"/intersect-diffs.py \ + "${ORIG_DIFF}" "${MODIFIED_DIFF}" > "${FINAL_DIFF}" +INTERSECT_RESULT=$? +git reset --hard >/dev/null + +# Fixup the commit message +diff_msg + +# Handle options +if [ -n "$1" ]; then + case "$1" in + -h|--help) usage;; + -n|--dry-run) cat "${FINAL_DIFF}"; show_commit_msg_diff;; + --commit) apply "${FINAL_DIFF}"; commit;; + --amend) apply "${FINAL_DIFF}"; amend;; + --msg-only) amend;; + *) usage;; + esac +else + apply "${FINAL_DIFF}" + if ! git diff --quiet; then + log "Formatting changes applied, verify and commit." + log "See also: http://www.webmproject.org/code/contribute/conventions/" + git diff --stat + fi +fi + +rm -f ${CLEAN_FILES} diff --git a/third_party/aom/tools/gen_authors.sh b/third_party/aom/tools/gen_authors.sh new file mode 100755 index 0000000000..5def8bc898 --- /dev/null +++ b/third_party/aom/tools/gen_authors.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Add organization names manually. + +cat <" | sort | uniq | grep -v "corp.google\|clang-format") +EOF diff --git a/third_party/aom/tools/gen_constrained_tokenset.py b/third_party/aom/tools/gen_constrained_tokenset.py new file mode 100755 index 0000000000..5d12ee1ef5 --- /dev/null +++ b/third_party/aom/tools/gen_constrained_tokenset.py @@ -0,0 +1,120 @@ +#!/usr/bin/python +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +"""Generate the probability model for the constrained token set. + +Model obtained from a 2-sided zero-centered distribution derived +from a Pareto distribution. The cdf of the distribution is: +cdf(x) = 0.5 + 0.5 * sgn(x) * [1 - {alpha/(alpha + |x|)} ^ beta] + +For a given beta and a given probability of the 1-node, the alpha +is first solved, and then the {alpha, beta} pair is used to generate +the probabilities for the rest of the nodes. +""" + +import heapq +import sys +import numpy as np +import scipy.optimize +import scipy.stats + + +def cdf_spareto(x, xm, beta): + p = 1 - (xm / (np.abs(x) + xm))**beta + p = 0.5 + 0.5 * np.sign(x) * p + return p + + +def get_spareto(p, beta): + cdf = cdf_spareto + + def func(x): + return ((cdf(1.5, x, beta) - cdf(0.5, x, beta)) / + (1 - cdf(0.5, x, beta)) - p)**2 + + alpha = scipy.optimize.fminbound(func, 1e-12, 10000, xtol=1e-12) + parray = np.zeros(11) + parray[0] = 2 * (cdf(0.5, alpha, beta) - 0.5) + parray[1] = (2 * (cdf(1.5, alpha, beta) - cdf(0.5, alpha, beta))) + parray[2] = (2 * (cdf(2.5, alpha, beta) - cdf(1.5, alpha, beta))) + parray[3] = (2 * (cdf(3.5, alpha, beta) - cdf(2.5, alpha, beta))) + parray[4] = (2 * (cdf(4.5, alpha, beta) - cdf(3.5, alpha, beta))) + parray[5] = (2 * (cdf(6.5, alpha, beta) - cdf(4.5, alpha, beta))) + parray[6] = (2 * (cdf(10.5, alpha, beta) - cdf(6.5, alpha, beta))) + parray[7] = (2 * (cdf(18.5, alpha, beta) - cdf(10.5, alpha, beta))) + parray[8] = (2 * (cdf(34.5, alpha, beta) - cdf(18.5, alpha, beta))) + parray[9] = (2 * (cdf(66.5, alpha, beta) - cdf(34.5, alpha, beta))) + parray[10] = 2 * (1. - cdf(66.5, alpha, beta)) + return parray + + +def quantize_probs(p, save_first_bin, bits): + """Quantize probability precisely. + + Quantize probabilities minimizing dH (Kullback-Leibler divergence) + approximated by: sum (p_i-q_i)^2/p_i. + References: + https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence + https://github.com/JarekDuda/AsymmetricNumeralSystemsToolkit + """ + num_sym = p.size + p = np.clip(p, 1e-16, 1) + L = 2**bits + pL = p * L + ip = 1. / p # inverse probability + q = np.clip(np.round(pL), 1, L + 1 - num_sym) + quant_err = (pL - q)**2 * ip + sgn = np.sign(L - q.sum()) # direction of correction + if sgn != 0: # correction is needed + v = [] # heap of adjustment results (adjustment err, index) of each symbol + for i in range(1 if save_first_bin else 0, num_sym): + q_adj = q[i] + sgn + if q_adj > 0 and q_adj < L: + adj_err = (pL[i] - q_adj)**2 * ip[i] - quant_err[i] + heapq.heappush(v, (adj_err, i)) + while q.sum() != L: + # apply lowest error adjustment + (adj_err, i) = heapq.heappop(v) + quant_err[i] += adj_err + q[i] += sgn + # calculate the cost of adjusting this symbol again + q_adj = q[i] + sgn + if q_adj > 0 and q_adj < L: + adj_err = (pL[i] - q_adj)**2 * ip[i] - quant_err[i] + heapq.heappush(v, (adj_err, i)) + return q + + +def get_quantized_spareto(p, beta, bits, first_token): + parray = get_spareto(p, beta) + parray = parray[1:] / (1 - parray[0]) + # CONFIG_NEW_TOKENSET + if first_token > 1: + parray = parray[1:] / (1 - parray[0]) + qarray = quantize_probs(parray, first_token == 1, bits) + return qarray.astype(np.int) + + +def main(bits=15, first_token=1): + beta = 8 + for q in range(1, 256): + parray = get_quantized_spareto(q / 256., beta, bits, first_token) + assert parray.sum() == 2**bits + print '{', ', '.join('%d' % i for i in parray), '},' + + +if __name__ == '__main__': + if len(sys.argv) > 2: + main(int(sys.argv[1]), int(sys.argv[2])) + elif len(sys.argv) > 1: + main(int(sys.argv[1])) + else: + main() diff --git a/third_party/aom/tools/inspect-cli.js b/third_party/aom/tools/inspect-cli.js new file mode 100644 index 0000000000..a14c08111a --- /dev/null +++ b/third_party/aom/tools/inspect-cli.js @@ -0,0 +1,39 @@ +/** + * This tool lets you test if the compiled Javascript decoder is functioning properly. You'll + * need to download a SpiderMonkey js-shell to run this script. + * https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/ + * + * Example: + * js-shell inspect-cli.js video.ivf + */ +load("inspect.js"); +var buffer = read(scriptArgs[0], "binary"); +var Module = { + noExitRuntime: true, + noInitialRun: true, + preInit: [], + preRun: [], + postRun: [function () { + printErr(`Loaded Javascript Decoder OK`); + }], + memoryInitializerPrefixURL: "bin/", + arguments: ['input.ivf', 'output.raw'], + on_frame_decoded_json: function (jsonString) { + let json = JSON.parse("[" + Module.UTF8ToString(jsonString) + "null]"); + json.forEach(frame => { + if (frame) { + print(frame.frame); + } + }); + } +}; +DecoderModule(Module); +Module.FS.writeFile("/tmp/input.ivf", buffer, { encoding: "binary" }); +Module._open_file(); +Module._set_layers(0xFFFFFFFF); // Set this to zero if you want to benchmark decoding. +while(true) { + printErr("Decoding Frame ..."); + if (Module._read_frame()) { + break; + } +} diff --git a/third_party/aom/tools/inspect-post.js b/third_party/aom/tools/inspect-post.js new file mode 100644 index 0000000000..31c40bb82c --- /dev/null +++ b/third_party/aom/tools/inspect-post.js @@ -0,0 +1 @@ +Module["FS"] = FS; diff --git a/third_party/aom/tools/intersect-diffs.py b/third_party/aom/tools/intersect-diffs.py new file mode 100755 index 0000000000..df13c4ef70 --- /dev/null +++ b/third_party/aom/tools/intersect-diffs.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +"""Calculates the "intersection" of two unified diffs. + +Given two diffs, A and B, it finds all hunks in B that had non-context lines +in A and prints them to stdout. This is useful to determine the hunks in B that +are relevant to A. The resulting file can be applied with patch(1) on top of A. +""" + +__author__ = "jkoleszar@google.com" + +import sys + +import diff + + +def FormatDiffHunks(hunks): + """Re-serialize a list of DiffHunks.""" + r = [] + last_header = None + for hunk in hunks: + this_header = hunk.header[0:2] + if last_header != this_header: + r.extend(hunk.header) + last_header = this_header + else: + r.extend(hunk.header[2]) + r.extend(hunk.lines) + r.append("\n") + return "".join(r) + + +def ZipHunks(rhs_hunks, lhs_hunks): + """Join two hunk lists on filename.""" + for rhs_hunk in rhs_hunks: + rhs_file = rhs_hunk.right.filename.split("/")[1:] + + for lhs_hunk in lhs_hunks: + lhs_file = lhs_hunk.left.filename.split("/")[1:] + if lhs_file != rhs_file: + continue + yield (rhs_hunk, lhs_hunk) + + +def main(): + old_hunks = [x for x in diff.ParseDiffHunks(open(sys.argv[1], "r"))] + new_hunks = [x for x in diff.ParseDiffHunks(open(sys.argv[2], "r"))] + out_hunks = [] + + # Join the right hand side of the older diff with the left hand side of the + # newer diff. + for old_hunk, new_hunk in ZipHunks(old_hunks, new_hunks): + if new_hunk in out_hunks: + continue + old_lines = old_hunk.right + new_lines = new_hunk.left + + # Determine if this hunk overlaps any non-context line from the other + for i in old_lines.delta_line_nums: + if i in new_lines: + out_hunks.append(new_hunk) + break + + if out_hunks: + print FormatDiffHunks(out_hunks) + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/third_party/aom/tools/lint-hunks.py b/third_party/aom/tools/lint-hunks.py new file mode 100755 index 0000000000..d02bee16ce --- /dev/null +++ b/third_party/aom/tools/lint-hunks.py @@ -0,0 +1,146 @@ +#!/usr/bin/python +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +"""Performs style checking on each diff hunk.""" +import getopt +import os +import StringIO +import subprocess +import sys + +import diff + + +SHORT_OPTIONS = "h" +LONG_OPTIONS = ["help"] + +TOPLEVEL_CMD = ["git", "rev-parse", "--show-toplevel"] +DIFF_CMD = ["git", "diff"] +DIFF_INDEX_CMD = ["git", "diff-index", "-u", "HEAD", "--"] +SHOW_CMD = ["git", "show"] +CPPLINT_FILTERS = ["-readability/casting"] + + +class Usage(Exception): + pass + + +class SubprocessException(Exception): + def __init__(self, args): + msg = "Failed to execute '%s'"%(" ".join(args)) + super(SubprocessException, self).__init__(msg) + + +class Subprocess(subprocess.Popen): + """Adds the notion of an expected returncode to Popen.""" + + def __init__(self, args, expected_returncode=0, **kwargs): + self._args = args + self._expected_returncode = expected_returncode + super(Subprocess, self).__init__(args, **kwargs) + + def communicate(self, *args, **kwargs): + result = super(Subprocess, self).communicate(*args, **kwargs) + if self._expected_returncode is not None: + try: + ok = self.returncode in self._expected_returncode + except TypeError: + ok = self.returncode == self._expected_returncode + if not ok: + raise SubprocessException(self._args) + return result + + +def main(argv=None): + if argv is None: + argv = sys.argv + try: + try: + opts, args = getopt.getopt(argv[1:], SHORT_OPTIONS, LONG_OPTIONS) + except getopt.error, msg: + raise Usage(msg) + + # process options + for o, _ in opts: + if o in ("-h", "--help"): + print __doc__ + sys.exit(0) + + if args and len(args) > 1: + print __doc__ + sys.exit(0) + + # Find the fully qualified path to the root of the tree + tl = Subprocess(TOPLEVEL_CMD, stdout=subprocess.PIPE) + tl = tl.communicate()[0].strip() + + # See if we're working on the index or not. + if args: + diff_cmd = DIFF_CMD + [args[0] + "^!"] + else: + diff_cmd = DIFF_INDEX_CMD + + # Build the command line to execute cpplint + cpplint_cmd = [os.path.join(tl, "tools", "cpplint.py"), + "--filter=" + ",".join(CPPLINT_FILTERS), + "-"] + + # Get a list of all affected lines + file_affected_line_map = {} + p = Subprocess(diff_cmd, stdout=subprocess.PIPE) + stdout = p.communicate()[0] + for hunk in diff.ParseDiffHunks(StringIO.StringIO(stdout)): + filename = hunk.right.filename[2:] + if filename not in file_affected_line_map: + file_affected_line_map[filename] = set() + file_affected_line_map[filename].update(hunk.right.delta_line_nums) + + # Run each affected file through cpplint + lint_failed = False + for filename, affected_lines in file_affected_line_map.iteritems(): + if filename.split(".")[-1] not in ("c", "h", "cc"): + continue + + if args: + # File contents come from git + show_cmd = SHOW_CMD + [args[0] + ":" + filename] + show = Subprocess(show_cmd, stdout=subprocess.PIPE) + lint = Subprocess(cpplint_cmd, expected_returncode=(0, 1), + stdin=show.stdout, stderr=subprocess.PIPE) + lint_out = lint.communicate()[1] + else: + # File contents come from the working tree + lint = Subprocess(cpplint_cmd, expected_returncode=(0, 1), + stdin=subprocess.PIPE, stderr=subprocess.PIPE) + stdin = open(os.path.join(tl, filename)).read() + lint_out = lint.communicate(stdin)[1] + + for line in lint_out.split("\n"): + fields = line.split(":") + if fields[0] != "-": + continue + warning_line_num = int(fields[1]) + if warning_line_num in affected_lines: + print "%s:%d:%s"%(filename, warning_line_num, + ":".join(fields[2:])) + lint_failed = True + + # Set exit code if any relevant lint errors seen + if lint_failed: + return 1 + + except Usage, err: + print >>sys.stderr, err + print >>sys.stderr, "for help use --help" + return 2 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/third_party/aom/tools/wrap-commit-msg.py b/third_party/aom/tools/wrap-commit-msg.py new file mode 100755 index 0000000000..1c78824439 --- /dev/null +++ b/third_party/aom/tools/wrap-commit-msg.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +## +## Copyright (c) 2016, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +"""Wraps paragraphs of text, preserving manual formatting + +This is like fold(1), but has the special convention of not modifying lines +that start with whitespace. This allows you to intersperse blocks with +special formatting, like code blocks, with written prose. The prose will +be wordwrapped, and the manual formatting will be preserved. + + * This won't handle the case of a bulleted (or ordered) list specially, so + manual wrapping must be done. + +Occasionally it's useful to put something with explicit formatting that +doesn't look at all like a block of text inline. + + indicator = has_leading_whitespace(line); + if (indicator) + preserve_formatting(line); + +The intent is that this docstring would make it through the transform +and still be legible and presented as it is in the source. If additional +cases are handled, update this doc to describe the effect. +""" + +__author__ = "jkoleszar@google.com" +import textwrap +import sys + +def wrap(text): + if text: + return textwrap.fill(text, break_long_words=False) + '\n' + return "" + + +def main(fileobj): + text = "" + output = "" + while True: + line = fileobj.readline() + if not line: + break + + if line.lstrip() == line: + text += line + else: + output += wrap(text) + text="" + output += line + output += wrap(text) + + # Replace the file or write to stdout. + if fileobj == sys.stdin: + fileobj = sys.stdout + else: + fileobj.seek(0) + fileobj.truncate(0) + fileobj.write(output) + +if __name__ == "__main__": + if len(sys.argv) > 1: + main(open(sys.argv[1], "r+")) + else: + main(sys.stdin) diff --git a/third_party/aom/tools_common.c b/third_party/aom/tools_common.c new file mode 100644 index 0000000000..353021093f --- /dev/null +++ b/third_party/aom/tools_common.c @@ -0,0 +1,454 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include +#include + +#include "./tools_common.h" + +#if CONFIG_AV1_ENCODER +#include "aom/aomcx.h" +#endif + +#if CONFIG_AV1_DECODER +#include "aom/aomdx.h" +#endif + +#if defined(_WIN32) || defined(__OS2__) +#include +#include + +#ifdef __OS2__ +#define _setmode setmode +#define _fileno fileno +#define _O_BINARY O_BINARY +#endif +#endif + +#define LOG_ERROR(label) \ + do { \ + const char *l = label; \ + va_list ap; \ + va_start(ap, fmt); \ + if (l) fprintf(stderr, "%s: ", l); \ + vfprintf(stderr, fmt, ap); \ + fprintf(stderr, "\n"); \ + va_end(ap); \ + } while (0) + +FILE *set_binary_mode(FILE *stream) { + (void)stream; +#if defined(_WIN32) || defined(__OS2__) + _setmode(_fileno(stream), _O_BINARY); +#endif + return stream; +} + +void die(const char *fmt, ...) { + LOG_ERROR(NULL); + usage_exit(); +} + +void fatal(const char *fmt, ...) { + LOG_ERROR("Fatal"); + exit(EXIT_FAILURE); +} + +void warn(const char *fmt, ...) { LOG_ERROR("Warning"); } + +void die_codec(aom_codec_ctx_t *ctx, const char *s) { + const char *detail = aom_codec_error_detail(ctx); + + printf("%s: %s\n", s, aom_codec_error(ctx)); + if (detail) printf(" %s\n", detail); + exit(EXIT_FAILURE); +} + +int read_yuv_frame(struct AvxInputContext *input_ctx, aom_image_t *yuv_frame) { + FILE *f = input_ctx->file; + struct FileTypeDetectionBuffer *detect = &input_ctx->detect; + int plane = 0; + int shortread = 0; + const int bytespp = (yuv_frame->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1; + + for (plane = 0; plane < 3; ++plane) { + uint8_t *ptr; + const int w = aom_img_plane_width(yuv_frame, plane); + const int h = aom_img_plane_height(yuv_frame, plane); + int r; + + /* Determine the correct plane based on the image format. The for-loop + * always counts in Y,U,V order, but this may not match the order of + * the data on disk. + */ + switch (plane) { + case 1: + ptr = + yuv_frame->planes[yuv_frame->fmt == AOM_IMG_FMT_YV12 ? AOM_PLANE_V + : AOM_PLANE_U]; + break; + case 2: + ptr = + yuv_frame->planes[yuv_frame->fmt == AOM_IMG_FMT_YV12 ? AOM_PLANE_U + : AOM_PLANE_V]; + break; + default: ptr = yuv_frame->planes[plane]; + } + + for (r = 0; r < h; ++r) { + size_t needed = w * bytespp; + size_t buf_position = 0; + const size_t left = detect->buf_read - detect->position; + if (left > 0) { + const size_t more = (left < needed) ? left : needed; + memcpy(ptr, detect->buf + detect->position, more); + buf_position = more; + needed -= more; + detect->position += more; + } + if (needed > 0) { + shortread |= (fread(ptr + buf_position, 1, needed, f) < needed); + } + + ptr += yuv_frame->stride[plane]; + } + } + + return shortread; +} + +#if CONFIG_ENCODERS + +static const AvxInterface aom_encoders[] = { +#if CONFIG_AV1_ENCODER + { "av1", AV1_FOURCC, &aom_codec_av1_cx }, +#endif +}; + +int get_aom_encoder_count(void) { + return sizeof(aom_encoders) / sizeof(aom_encoders[0]); +} + +const AvxInterface *get_aom_encoder_by_index(int i) { return &aom_encoders[i]; } + +const AvxInterface *get_aom_encoder_by_name(const char *name) { + int i; + + for (i = 0; i < get_aom_encoder_count(); ++i) { + const AvxInterface *encoder = get_aom_encoder_by_index(i); + if (strcmp(encoder->name, name) == 0) return encoder; + } + + return NULL; +} + +#endif // CONFIG_ENCODERS + +#if CONFIG_DECODERS + +static const AvxInterface aom_decoders[] = { +#if CONFIG_AV1_DECODER + { "av1", AV1_FOURCC, &aom_codec_av1_dx }, +#endif +}; + +int get_aom_decoder_count(void) { + return sizeof(aom_decoders) / sizeof(aom_decoders[0]); +} + +const AvxInterface *get_aom_decoder_by_index(int i) { return &aom_decoders[i]; } + +const AvxInterface *get_aom_decoder_by_name(const char *name) { + int i; + + for (i = 0; i < get_aom_decoder_count(); ++i) { + const AvxInterface *const decoder = get_aom_decoder_by_index(i); + if (strcmp(decoder->name, name) == 0) return decoder; + } + + return NULL; +} + +const AvxInterface *get_aom_decoder_by_fourcc(uint32_t fourcc) { + int i; + + for (i = 0; i < get_aom_decoder_count(); ++i) { + const AvxInterface *const decoder = get_aom_decoder_by_index(i); + if (decoder->fourcc == fourcc) return decoder; + } + + return NULL; +} + +#endif // CONFIG_DECODERS + +// TODO(dkovalev): move this function to aom_image.{c, h}, so it will be part +// of aom_image_t support +int aom_img_plane_width(const aom_image_t *img, int plane) { + if (plane > 0 && img->x_chroma_shift > 0) + return (img->d_w + 1) >> img->x_chroma_shift; + else + return img->d_w; +} + +int aom_img_plane_height(const aom_image_t *img, int plane) { + if (plane > 0 && img->y_chroma_shift > 0) + return (img->d_h + 1) >> img->y_chroma_shift; + else + return img->d_h; +} + +void aom_img_write(const aom_image_t *img, FILE *file) { + int plane; + + for (plane = 0; plane < 3; ++plane) { + const unsigned char *buf = img->planes[plane]; + const int stride = img->stride[plane]; + const int w = aom_img_plane_width(img, plane) * + ((img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1); + const int h = aom_img_plane_height(img, plane); + int y; + + for (y = 0; y < h; ++y) { + fwrite(buf, 1, w, file); + buf += stride; + } + } +} + +int aom_img_read(aom_image_t *img, FILE *file) { + int plane; + + for (plane = 0; plane < 3; ++plane) { + unsigned char *buf = img->planes[plane]; + const int stride = img->stride[plane]; + const int w = aom_img_plane_width(img, plane) * + ((img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1); + const int h = aom_img_plane_height(img, plane); + int y; + + for (y = 0; y < h; ++y) { + if (fread(buf, 1, w, file) != (size_t)w) return 0; + buf += stride; + } + } + + return 1; +} + +// TODO(dkovalev) change sse_to_psnr signature: double -> int64_t +double sse_to_psnr(double samples, double peak, double sse) { + static const double kMaxPSNR = 100.0; + + if (sse > 0.0) { + const double psnr = 10.0 * log10(samples * peak * peak / sse); + return psnr > kMaxPSNR ? kMaxPSNR : psnr; + } else { + return kMaxPSNR; + } +} + +// TODO(debargha): Consolidate the functions below into a separate file. +#if CONFIG_HIGHBITDEPTH +static void highbd_img_upshift(aom_image_t *dst, aom_image_t *src, + int input_shift) { + // Note the offset is 1 less than half. + const int offset = input_shift > 0 ? (1 << (input_shift - 1)) - 1 : 0; + int plane; + if (dst->d_w != src->d_w || dst->d_h != src->d_h || + dst->x_chroma_shift != src->x_chroma_shift || + dst->y_chroma_shift != src->y_chroma_shift || dst->fmt != src->fmt || + input_shift < 0) { + fatal("Unsupported image conversion"); + } + switch (src->fmt) { + case AOM_IMG_FMT_I42016: + case AOM_IMG_FMT_I42216: + case AOM_IMG_FMT_I44416: + case AOM_IMG_FMT_I44016: break; + default: fatal("Unsupported image conversion"); break; + } + for (plane = 0; plane < 3; plane++) { + int w = src->d_w; + int h = src->d_h; + int x, y; + if (plane) { + w = (w + src->x_chroma_shift) >> src->x_chroma_shift; + h = (h + src->y_chroma_shift) >> src->y_chroma_shift; + } + for (y = 0; y < h; y++) { + uint16_t *p_src = + (uint16_t *)(src->planes[plane] + y * src->stride[plane]); + uint16_t *p_dst = + (uint16_t *)(dst->planes[plane] + y * dst->stride[plane]); + for (x = 0; x < w; x++) *p_dst++ = (*p_src++ << input_shift) + offset; + } + } +} + +static void lowbd_img_upshift(aom_image_t *dst, aom_image_t *src, + int input_shift) { + // Note the offset is 1 less than half. + const int offset = input_shift > 0 ? (1 << (input_shift - 1)) - 1 : 0; + int plane; + if (dst->d_w != src->d_w || dst->d_h != src->d_h || + dst->x_chroma_shift != src->x_chroma_shift || + dst->y_chroma_shift != src->y_chroma_shift || + dst->fmt != src->fmt + AOM_IMG_FMT_HIGHBITDEPTH || input_shift < 0) { + fatal("Unsupported image conversion"); + } + switch (src->fmt) { + case AOM_IMG_FMT_I420: + case AOM_IMG_FMT_I422: + case AOM_IMG_FMT_I444: + case AOM_IMG_FMT_I440: break; + default: fatal("Unsupported image conversion"); break; + } + for (plane = 0; plane < 3; plane++) { + int w = src->d_w; + int h = src->d_h; + int x, y; + if (plane) { + w = (w + src->x_chroma_shift) >> src->x_chroma_shift; + h = (h + src->y_chroma_shift) >> src->y_chroma_shift; + } + for (y = 0; y < h; y++) { + uint8_t *p_src = src->planes[plane] + y * src->stride[plane]; + uint16_t *p_dst = + (uint16_t *)(dst->planes[plane] + y * dst->stride[plane]); + for (x = 0; x < w; x++) { + *p_dst++ = (*p_src++ << input_shift) + offset; + } + } + } +} + +void aom_img_upshift(aom_image_t *dst, aom_image_t *src, int input_shift) { + if (src->fmt & AOM_IMG_FMT_HIGHBITDEPTH) { + highbd_img_upshift(dst, src, input_shift); + } else { + lowbd_img_upshift(dst, src, input_shift); + } +} + +void aom_img_truncate_16_to_8(aom_image_t *dst, aom_image_t *src) { + int plane; + if (dst->fmt + AOM_IMG_FMT_HIGHBITDEPTH != src->fmt || dst->d_w != src->d_w || + dst->d_h != src->d_h || dst->x_chroma_shift != src->x_chroma_shift || + dst->y_chroma_shift != src->y_chroma_shift) { + fatal("Unsupported image conversion"); + } + switch (dst->fmt) { + case AOM_IMG_FMT_I420: + case AOM_IMG_FMT_I422: + case AOM_IMG_FMT_I444: + case AOM_IMG_FMT_I440: break; + default: fatal("Unsupported image conversion"); break; + } + for (plane = 0; plane < 3; plane++) { + int w = src->d_w; + int h = src->d_h; + int x, y; + if (plane) { + w = (w + src->x_chroma_shift) >> src->x_chroma_shift; + h = (h + src->y_chroma_shift) >> src->y_chroma_shift; + } + for (y = 0; y < h; y++) { + uint16_t *p_src = + (uint16_t *)(src->planes[plane] + y * src->stride[plane]); + uint8_t *p_dst = dst->planes[plane] + y * dst->stride[plane]; + for (x = 0; x < w; x++) { + *p_dst++ = (uint8_t)(*p_src++); + } + } + } +} + +static void highbd_img_downshift(aom_image_t *dst, aom_image_t *src, + int down_shift) { + int plane; + if (dst->d_w != src->d_w || dst->d_h != src->d_h || + dst->x_chroma_shift != src->x_chroma_shift || + dst->y_chroma_shift != src->y_chroma_shift || dst->fmt != src->fmt || + down_shift < 0) { + fatal("Unsupported image conversion"); + } + switch (src->fmt) { + case AOM_IMG_FMT_I42016: + case AOM_IMG_FMT_I42216: + case AOM_IMG_FMT_I44416: + case AOM_IMG_FMT_I44016: break; + default: fatal("Unsupported image conversion"); break; + } + for (plane = 0; plane < 3; plane++) { + int w = src->d_w; + int h = src->d_h; + int x, y; + if (plane) { + w = (w + src->x_chroma_shift) >> src->x_chroma_shift; + h = (h + src->y_chroma_shift) >> src->y_chroma_shift; + } + for (y = 0; y < h; y++) { + uint16_t *p_src = + (uint16_t *)(src->planes[plane] + y * src->stride[plane]); + uint16_t *p_dst = + (uint16_t *)(dst->planes[plane] + y * dst->stride[plane]); + for (x = 0; x < w; x++) *p_dst++ = *p_src++ >> down_shift; + } + } +} + +static void lowbd_img_downshift(aom_image_t *dst, aom_image_t *src, + int down_shift) { + int plane; + if (dst->d_w != src->d_w || dst->d_h != src->d_h || + dst->x_chroma_shift != src->x_chroma_shift || + dst->y_chroma_shift != src->y_chroma_shift || + src->fmt != dst->fmt + AOM_IMG_FMT_HIGHBITDEPTH || down_shift < 0) { + fatal("Unsupported image conversion"); + } + switch (dst->fmt) { + case AOM_IMG_FMT_I420: + case AOM_IMG_FMT_I422: + case AOM_IMG_FMT_I444: + case AOM_IMG_FMT_I440: break; + default: fatal("Unsupported image conversion"); break; + } + for (plane = 0; plane < 3; plane++) { + int w = src->d_w; + int h = src->d_h; + int x, y; + if (plane) { + w = (w + src->x_chroma_shift) >> src->x_chroma_shift; + h = (h + src->y_chroma_shift) >> src->y_chroma_shift; + } + for (y = 0; y < h; y++) { + uint16_t *p_src = + (uint16_t *)(src->planes[plane] + y * src->stride[plane]); + uint8_t *p_dst = dst->planes[plane] + y * dst->stride[plane]; + for (x = 0; x < w; x++) { + *p_dst++ = *p_src++ >> down_shift; + } + } + } +} + +void aom_img_downshift(aom_image_t *dst, aom_image_t *src, int down_shift) { + if (dst->fmt & AOM_IMG_FMT_HIGHBITDEPTH) { + highbd_img_downshift(dst, src, down_shift); + } else { + lowbd_img_downshift(dst, src, down_shift); + } +} +#endif // CONFIG_HIGHBITDEPTH diff --git a/third_party/aom/tools_common.h b/third_party/aom/tools_common.h new file mode 100644 index 0000000000..8acb3709f3 --- /dev/null +++ b/third_party/aom/tools_common.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef TOOLS_COMMON_H_ +#define TOOLS_COMMON_H_ + +#include + +#include "./aom_config.h" +#include "aom/aom_codec.h" +#include "aom/aom_image.h" +#include "aom/aom_integer.h" +#include "aom_ports/msvc.h" + +#if CONFIG_ENCODERS +#include "./y4minput.h" +#endif + +#if defined(_MSC_VER) +/* MSVS uses _f{seek,tell}i64. */ +#define fseeko _fseeki64 +#define ftello _ftelli64 +typedef int64_t FileOffset; +#elif defined(_WIN32) +#include /* NOLINT*/ +/* MinGW uses f{seek,tell}o64 for large files. */ +#define fseeko fseeko64 +#define ftello ftello64 +typedef off64_t FileOffset; +#elif CONFIG_OS_SUPPORT +#include /* NOLINT*/ +typedef off_t FileOffset; +/* Use 32-bit file operations in WebM file format when building ARM + * executables (.axf) with RVCT. */ +#else +#define fseeko fseek +#define ftello ftell +typedef long FileOffset; /* NOLINT */ +#endif /* CONFIG_OS_SUPPORT */ + +#if CONFIG_OS_SUPPORT +#if defined(_MSC_VER) +#include /* NOLINT */ +#define isatty _isatty +#define fileno _fileno +#else +#include /* NOLINT */ +#endif /* _MSC_VER */ +#endif /* CONFIG_OS_SUPPORT */ + +#define LITERALU64(hi, lo) ((((uint64_t)hi) << 32) | lo) + +#ifndef PATH_MAX +#define PATH_MAX 512 +#endif + +#define IVF_FRAME_HDR_SZ (4 + 8) /* 4 byte size + 8 byte timestamp */ +#define IVF_FILE_HDR_SZ 32 + +#define RAW_FRAME_HDR_SZ sizeof(uint32_t) + +#define AV1_FOURCC 0x31305641 + +enum VideoFileType { + FILE_TYPE_RAW, + FILE_TYPE_IVF, + FILE_TYPE_Y4M, + FILE_TYPE_WEBM +}; + +struct FileTypeDetectionBuffer { + char buf[4]; + size_t buf_read; + size_t position; +}; + +struct AvxRational { + int numerator; + int denominator; +}; + +struct AvxInputContext { + const char *filename; + FILE *file; + int64_t length; + struct FileTypeDetectionBuffer detect; + enum VideoFileType file_type; + uint32_t width; + uint32_t height; + struct AvxRational pixel_aspect_ratio; + aom_img_fmt_t fmt; + aom_bit_depth_t bit_depth; + int only_i420; + uint32_t fourcc; + struct AvxRational framerate; +#if CONFIG_ENCODERS + y4m_input y4m; +#endif +}; + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(__GNUC__) +#define AOM_NO_RETURN __attribute__((noreturn)) +#else +#define AOM_NO_RETURN +#endif + +/* Sets a stdio stream into binary mode */ +FILE *set_binary_mode(FILE *stream); + +void die(const char *fmt, ...) AOM_NO_RETURN; +void fatal(const char *fmt, ...) AOM_NO_RETURN; +void warn(const char *fmt, ...); + +void die_codec(aom_codec_ctx_t *ctx, const char *s) AOM_NO_RETURN; + +/* The tool including this file must define usage_exit() */ +void usage_exit(void) AOM_NO_RETURN; + +#undef AOM_NO_RETURN + +int read_yuv_frame(struct AvxInputContext *input_ctx, aom_image_t *yuv_frame); + +typedef struct AvxInterface { + const char *const name; + const uint32_t fourcc; + aom_codec_iface_t *(*const codec_interface)(); +} AvxInterface; + +int get_aom_encoder_count(void); +const AvxInterface *get_aom_encoder_by_index(int i); +const AvxInterface *get_aom_encoder_by_name(const char *name); + +int get_aom_decoder_count(void); +const AvxInterface *get_aom_decoder_by_index(int i); +const AvxInterface *get_aom_decoder_by_name(const char *name); +const AvxInterface *get_aom_decoder_by_fourcc(uint32_t fourcc); + +// TODO(dkovalev): move this function to aom_image.{c, h}, so it will be part +// of aom_image_t support +int aom_img_plane_width(const aom_image_t *img, int plane); +int aom_img_plane_height(const aom_image_t *img, int plane); +void aom_img_write(const aom_image_t *img, FILE *file); +int aom_img_read(aom_image_t *img, FILE *file); + +double sse_to_psnr(double samples, double peak, double mse); + +#if CONFIG_HIGHBITDEPTH +void aom_img_upshift(aom_image_t *dst, aom_image_t *src, int input_shift); +void aom_img_downshift(aom_image_t *dst, aom_image_t *src, int down_shift); +void aom_img_truncate_16_to_8(aom_image_t *dst, aom_image_t *src); +#endif + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif // TOOLS_COMMON_H_ diff --git a/third_party/aom/usage.dox b/third_party/aom/usage.dox new file mode 100644 index 0000000000..59239e8f18 --- /dev/null +++ b/third_party/aom/usage.dox @@ -0,0 +1,135 @@ +/*!\page usage Usage + + The aom multi-format codec SDK provides a unified interface amongst its + supported codecs. This abstraction allows applications using this SDK to + easily support multiple video formats with minimal code duplication or + "special casing." This section describes the interface common to all codecs. + For codec-specific details, see the \ref codecs page. + + The following sections are common to all codecs: + - \ref usage_types + - \ref usage_features + - \ref usage_init + - \ref usage_errors + + For more information on decoder and encoder specific usage, see the + following pages: + \if decoder + \li \subpage usage_decode + \endif + \if encoder + \li \subpage usage_encode + \endif + + \section usage_types Important Data Types + There are two important data structures to consider in this interface. + + \subsection usage_ctxs Contexts + A context is a storage area allocated by the calling application that the + codec may write into to store details about a single instance of that codec. + Most of the context is implementation specific, and thus opaque to the + application. The context structure as seen by the application is of fixed + size, and thus can be allocated with automatic storage or dynamically + on the heap. + + Most operations require an initialized codec context. Codec context + instances are codec specific. That is, the codec to be used for the encoded + video must be known at initialization time. See #aom_codec_ctx_t for further + information. + + \subsection usage_ifaces Interfaces + A codec interface is an opaque structure that controls how function calls + into the generic interface are dispatched to their codec-specific + implementations. Applications \ref MUSTNOT attempt to examine or override + this storage, as it contains internal implementation details likely to + change from release to release. + + Each supported codec will expose an interface structure to the application + as an extern reference to a structure of the incomplete type + #aom_codec_iface_t. + + \section usage_features Features + Several "features" are defined that are optionally implemented by codec + algorithms. Indeed, the same algorithm may support different features on + different platforms. The purpose of defining these features is that when + they are implemented, they conform to a common interface. The features, or + capabilities, of an algorithm can be queried from it's interface by using + the aom_codec_get_caps() method. Attempts to invoke features not supported + by an algorithm will generally result in #AOM_CODEC_INCAPABLE. + + \if decoder + Currently defined decoder features include: + - \ref usage_cb + \endif + + \section usage_init Initialization + To initialize a codec instance, the address of the codec context + and interface structures are passed to an initialization function. Depending + on the \ref usage_features that the codec supports, the codec could be + initialized in different modes. + + To prevent cases of confusion where the ABI of the library changes, + the ABI is versioned. The ABI version number must be passed at + initialization time to ensure the application is using a header file that + matches the library. The current ABI version number is stored in the + preprocessor macros #AOM_CODEC_ABI_VERSION, #AOM_ENCODER_ABI_VERSION, and + #AOM_DECODER_ABI_VERSION. For convenience, each initialization function has + a wrapper macro that inserts the correct version number. These macros are + named like the initialization methods, but without the _ver suffix. + + + The available initialization methods are: + \if encoder + \li #aom_codec_enc_init (calls aom_codec_enc_init_ver()) + \li #aom_codec_enc_init_multi (calls aom_codec_enc_init_multi_ver()) + \endif + \if decoder + \li #aom_codec_dec_init (calls aom_codec_dec_init_ver()) + \endif + + + \section usage_errors Error Handling + Almost all codec functions return an error status of type #aom_codec_err_t. + The semantics of how each error condition should be processed is clearly + defined in the definitions of each enumerated value. Error values can be + converted into ASCII strings with the aom_codec_error() and + aom_codec_err_to_string() methods. The difference between these two methods is + that aom_codec_error() returns the error state from an initialized context, + whereas aom_codec_err_to_string() can be used in cases where an error occurs + outside any context. The enumerated value returned from the last call can be + retrieved from the err member of the decoder context as well. + Finally, more detailed error information may be able to be obtained by using + the aom_codec_error_detail() method. Not all errors produce detailed error + information. + + In addition to error information, the codec library's build configuration + is available at runtime on some platforms. This information can be returned + by calling aom_codec_build_config(), and is formatted as a base64 coded string + (comprised of characters in the set [a-z_a-Z0-9+/]). This information is not + useful to an application at runtime, but may be of use to aom for support. + + + \section usage_deadline Deadline + Both the encoding and decoding functions have a deadline + parameter. This parameter indicates the amount of time, in microseconds + (us), that the application wants the codec to spend processing before + returning. This is a soft deadline -- that is, the semantics of the + requested operation take precedence over meeting the deadline. If, for + example, an application sets a deadline of 1000us, and the + frame takes 2000us to decode, the call to aom_codec_decode() will return + after 2000us. In this case the deadline is not met, but the semantics of the + function are preserved. If, for the same frame, an application instead sets + a deadline of 5000us, the decoder will see that it has 3000us + remaining in its time slice when decoding completes. It could then choose to + run a set of \ref usage_postproc filters, and perhaps would return after + 4000us (instead of the allocated 5000us). In this case the deadline is met, + and the semantics of the call are preserved, as before. + + The special value 0 is reserved to represent an infinite + deadline. In this case, the codec will perform as much processing as + possible to yield the highest quality frame. + + By convention, the value 1 is used to mean "return as fast as + possible." + +*/ diff --git a/third_party/aom/usage_cx.dox b/third_party/aom/usage_cx.dox new file mode 100644 index 0000000000..dcf267ce43 --- /dev/null +++ b/third_party/aom/usage_cx.dox @@ -0,0 +1,13 @@ +/*! \page usage_encode Encoding + + The aom_codec_encode() function is at the core of the encode loop. It + processes raw images passed by the application, producing packets of + compressed data. The deadline parameter controls the amount + of time in microseconds the encoder should spend working on the frame. For + more information on the deadline parameter, see + \ref usage_deadline. + + + \ref samples + +*/ diff --git a/third_party/aom/usage_dx.dox b/third_party/aom/usage_dx.dox new file mode 100644 index 0000000000..6b76bf7b00 --- /dev/null +++ b/third_party/aom/usage_dx.dox @@ -0,0 +1,62 @@ +/*! \page usage_decode Decoding + + The aom_codec_decode() function is at the core of the decode loop. It + processes packets of compressed data passed by the application, producing + decoded images. The decoder expects packets to comprise exactly one image + frame of data. Packets \ref MUST be passed in decode order. If the + application wishes to associate some data with the frame, the + user_priv member may be set. The deadline + parameter controls the amount of time in microseconds the decoder should + spend working on the frame. This is typically used to support adaptive + \ref usage_postproc based on the amount of free CPU time. For more + information on the deadline parameter, see \ref usage_deadline. + + \ref samples + + + \section usage_cb Callback Based Decoding + There are two methods for the application to access decoded frame data. Some + codecs support asynchronous (callback-based) decoding \ref usage_features + that allow the application to register a callback to be invoked by the + decoder when decoded data becomes available. Decoders are not required to + support this feature, however. Like all \ref usage_features, support can be + determined by calling aom_codec_get_caps(). Callbacks are available in both + frame-based and slice-based variants. Frame based callbacks conform to the + signature of #aom_codec_put_frame_cb_fn_t and are invoked once the entire + frame has been decoded. Slice based callbacks conform to the signature of + #aom_codec_put_slice_cb_fn_t and are invoked after a subsection of the frame + is decoded. For example, a slice callback could be issued for each + macroblock row. However, the number and size of slices to return is + implementation specific. Also, the image data passed in a slice callback is + not necessarily in the same memory segment as the data will be when it is + assembled into a full frame. For this reason, the application \ref MUST + examine the rectangles that describe what data is valid to access and what + data has been updated in this call. For all their additional complexity, + slice based decoding callbacks provide substantial speed gains to the + overall application in some cases, due to improved cache behavior. + + + \section usage_frame_iter Frame Iterator Based Decoding + If the codec does not support callback based decoding, or the application + chooses not to make use of that feature, decoded frames are made available + through the aom_codec_get_frame() iterator. The application initializes the + iterator storage (of type #aom_codec_iter_t) to NULL, then calls + aom_codec_get_frame repeatedly until it returns NULL, indicating that all + images have been returned. This process may result in zero, one, or many + frames that are ready for display, depending on the codec. + + + \section usage_postproc Postprocessing + Postprocessing is a process that is applied after a frame is decoded to + enhance the image's appearance by removing artifacts introduced in the + compression process. It is not required to properly decode the frame, and + is generally done only when there is enough spare CPU time to execute + the required filters. Codecs may support a number of different + postprocessing filters, and the available filters may differ from platform + to platform. Embedded devices often do not have enough CPU to implement + postprocessing in software. The filter selection is generally handled + automatically by the codec, depending on the amount of time remaining before + hitting the user-specified \ref usage_deadline after decoding the frame. + + +*/ diff --git a/third_party/aom/video_common.h b/third_party/aom/video_common.h new file mode 100644 index 0000000000..1aa82f613a --- /dev/null +++ b/third_party/aom/video_common.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef VIDEO_COMMON_H_ +#define VIDEO_COMMON_H_ + +#include "./tools_common.h" + +typedef struct { + uint32_t codec_fourcc; + int frame_width; + int frame_height; + struct AvxRational time_base; +} AvxVideoInfo; + +#endif // VIDEO_COMMON_H_ diff --git a/third_party/aom/video_reader.c b/third_party/aom/video_reader.c new file mode 100644 index 0000000000..6a96af967e --- /dev/null +++ b/third_party/aom/video_reader.c @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include + +#include "./ivfdec.h" +#include "./video_reader.h" + +#include "aom_ports/mem_ops.h" + +static const char *const kIVFSignature = "DKIF"; + +struct AvxVideoReaderStruct { + AvxVideoInfo info; + FILE *file; + uint8_t *buffer; + size_t buffer_size; + size_t frame_size; +}; + +AvxVideoReader *aom_video_reader_open(const char *filename) { + char header[32]; + AvxVideoReader *reader = NULL; + FILE *const file = fopen(filename, "rb"); + if (!file) return NULL; // Can't open file + + if (fread(header, 1, 32, file) != 32) return NULL; // Can't read file header + + if (memcmp(kIVFSignature, header, 4) != 0) + return NULL; // Wrong IVF signature + + if (mem_get_le16(header + 4) != 0) return NULL; // Wrong IVF version + + reader = (AvxVideoReader *)calloc(1, sizeof(*reader)); + if (!reader) return NULL; // Can't allocate AvxVideoReader + + reader->file = file; + reader->info.codec_fourcc = mem_get_le32(header + 8); + reader->info.frame_width = mem_get_le16(header + 12); + reader->info.frame_height = mem_get_le16(header + 14); + reader->info.time_base.numerator = mem_get_le32(header + 16); + reader->info.time_base.denominator = mem_get_le32(header + 20); + + return reader; +} + +void aom_video_reader_close(AvxVideoReader *reader) { + if (reader) { + fclose(reader->file); + free(reader->buffer); + free(reader); + } +} + +int aom_video_reader_read_frame(AvxVideoReader *reader) { + return !ivf_read_frame(reader->file, &reader->buffer, &reader->frame_size, + &reader->buffer_size); +} + +const uint8_t *aom_video_reader_get_frame(AvxVideoReader *reader, + size_t *size) { + if (size) *size = reader->frame_size; + + return reader->buffer; +} + +const AvxVideoInfo *aom_video_reader_get_info(AvxVideoReader *reader) { + return &reader->info; +} diff --git a/third_party/aom/video_reader.h b/third_party/aom/video_reader.h new file mode 100644 index 0000000000..962c6653b5 --- /dev/null +++ b/third_party/aom/video_reader.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef VIDEO_READER_H_ +#define VIDEO_READER_H_ + +#include "./video_common.h" + +// The following code is work in progress. It is going to support transparent +// reading of input files. Right now only IVF format is supported for +// simplicity. The main goal the API is to be simple and easy to use in example +// code and in aomenc/aomdec later. All low-level details like memory +// buffer management are hidden from API users. +struct AvxVideoReaderStruct; +typedef struct AvxVideoReaderStruct AvxVideoReader; + +#ifdef __cplusplus +extern "C" { +#endif + +// Opens the input file for reading and inspects it to determine file type. +// Returns an opaque AvxVideoReader* upon success, or NULL upon failure. +// Right now only IVF format is supported. +AvxVideoReader *aom_video_reader_open(const char *filename); + +// Frees all resources associated with AvxVideoReader* returned from +// aom_video_reader_open() call. +void aom_video_reader_close(AvxVideoReader *reader); + +// Reads frame from the file and stores it in internal buffer. +int aom_video_reader_read_frame(AvxVideoReader *reader); + +// Returns the pointer to memory buffer with frame data read by last call to +// aom_video_reader_read_frame(). +const uint8_t *aom_video_reader_get_frame(AvxVideoReader *reader, size_t *size); + +// Fills AvxVideoInfo with information from opened video file. +const AvxVideoInfo *aom_video_reader_get_info(AvxVideoReader *reader); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // VIDEO_READER_H_ diff --git a/third_party/aom/video_writer.c b/third_party/aom/video_writer.c new file mode 100644 index 0000000000..4e072c7dcc --- /dev/null +++ b/third_party/aom/video_writer.c @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./ivfenc.h" +#include "./video_writer.h" +#include "aom/aom_encoder.h" + +struct AvxVideoWriterStruct { + AvxVideoInfo info; + FILE *file; + int frame_count; +}; + +static void write_header(FILE *file, const AvxVideoInfo *info, + int frame_count) { + struct aom_codec_enc_cfg cfg; + cfg.g_w = info->frame_width; + cfg.g_h = info->frame_height; + cfg.g_timebase.num = info->time_base.numerator; + cfg.g_timebase.den = info->time_base.denominator; + + ivf_write_file_header(file, &cfg, info->codec_fourcc, frame_count); +} + +AvxVideoWriter *aom_video_writer_open(const char *filename, + AvxContainer container, + const AvxVideoInfo *info) { + if (container == kContainerIVF) { + AvxVideoWriter *writer = NULL; + FILE *const file = fopen(filename, "wb"); + if (!file) return NULL; + + writer = malloc(sizeof(*writer)); + if (!writer) return NULL; + + writer->frame_count = 0; + writer->info = *info; + writer->file = file; + + write_header(writer->file, info, 0); + + return writer; + } + + return NULL; +} + +void aom_video_writer_close(AvxVideoWriter *writer) { + if (writer) { + // Rewriting frame header with real frame count + rewind(writer->file); + write_header(writer->file, &writer->info, writer->frame_count); + + fclose(writer->file); + free(writer); + } +} + +int aom_video_writer_write_frame(AvxVideoWriter *writer, const uint8_t *buffer, + size_t size, int64_t pts) { + ivf_write_frame_header(writer->file, pts, size); + if (fwrite(buffer, 1, size, writer->file) != size) return 0; + + ++writer->frame_count; + + return 1; +} diff --git a/third_party/aom/video_writer.h b/third_party/aom/video_writer.h new file mode 100644 index 0000000000..ad9e6ebb98 --- /dev/null +++ b/third_party/aom/video_writer.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef VIDEO_WRITER_H_ +#define VIDEO_WRITER_H_ + +#include "./video_common.h" + +typedef enum { kContainerIVF } AvxContainer; + +struct AvxVideoWriterStruct; +typedef struct AvxVideoWriterStruct AvxVideoWriter; + +#ifdef __cplusplus +extern "C" { +#endif + +// Finds and opens writer for specified container format. +// Returns an opaque AvxVideoWriter* upon success, or NULL upon failure. +// Right now only IVF format is supported. +AvxVideoWriter *aom_video_writer_open(const char *filename, + AvxContainer container, + const AvxVideoInfo *info); + +// Frees all resources associated with AvxVideoWriter* returned from +// aom_video_writer_open() call. +void aom_video_writer_close(AvxVideoWriter *writer); + +// Writes frame bytes to the file. +int aom_video_writer_write_frame(AvxVideoWriter *writer, const uint8_t *buffer, + size_t size, int64_t pts); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // VIDEO_WRITER_H_ diff --git a/third_party/aom/warnings.c b/third_party/aom/warnings.c new file mode 100644 index 0000000000..16d5c6c18a --- /dev/null +++ b/third_party/aom/warnings.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "./warnings.h" + +#include +#include +#include +#include + +#include "aom/aom_encoder.h" + +#include "./tools_common.h" +#include "./aomenc.h" + +static const char quantizer_warning_string[] = + "Bad quantizer values. Quantizer values should not be equal, and should " + "differ by at least 8."; + +struct WarningListNode { + const char *warning_string; + struct WarningListNode *next_warning; +}; + +struct WarningList { + struct WarningListNode *warning_node; +}; + +static void add_warning(const char *warning_string, + struct WarningList *warning_list) { + struct WarningListNode **node = &warning_list->warning_node; + + struct WarningListNode *new_node = malloc(sizeof(*new_node)); + if (new_node == NULL) { + fatal("Unable to allocate warning node."); + } + + new_node->warning_string = warning_string; + new_node->next_warning = NULL; + + while (*node != NULL) node = &(*node)->next_warning; + + *node = new_node; +} + +static void free_warning_list(struct WarningList *warning_list) { + while (warning_list->warning_node != NULL) { + struct WarningListNode *const node = warning_list->warning_node; + warning_list->warning_node = node->next_warning; + free(node); + } +} + +static int continue_prompt(int num_warnings) { + int c; + fprintf(stderr, + "%d encoder configuration warning(s). Continue? (y to continue) ", + num_warnings); + c = getchar(); + return c == 'y'; +} + +static void check_quantizer(int min_q, int max_q, + struct WarningList *warning_list) { + const int lossless = min_q == 0 && max_q == 0; + if (!lossless && (min_q == max_q || abs(max_q - min_q) < 8)) + add_warning(quantizer_warning_string, warning_list); +} + +void check_encoder_config(int disable_prompt, + const struct AvxEncoderConfig *global_config, + const struct aom_codec_enc_cfg *stream_config) { + int num_warnings = 0; + struct WarningListNode *warning = NULL; + struct WarningList warning_list = { 0 }; + (void)global_config; + check_quantizer(stream_config->rc_min_quantizer, + stream_config->rc_max_quantizer, &warning_list); + /* Count and print warnings. */ + for (warning = warning_list.warning_node; warning != NULL; + warning = warning->next_warning, ++num_warnings) { + warn(warning->warning_string); + } + + free_warning_list(&warning_list); + + if (num_warnings) { + if (!disable_prompt && !continue_prompt(num_warnings)) exit(EXIT_FAILURE); + } +} diff --git a/third_party/aom/warnings.h b/third_party/aom/warnings.h new file mode 100644 index 0000000000..61db2dcf8a --- /dev/null +++ b/third_party/aom/warnings.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef WARNINGS_H_ +#define WARNINGS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +struct aom_codec_enc_cfg; +struct AvxEncoderConfig; + +/* + * Checks config for improperly used settings. Warns user upon encountering + * settings that will lead to poor output quality. Prompts user to continue + * when warnings are issued. + */ +void check_encoder_config(int disable_prompt, + const struct AvxEncoderConfig *global_config, + const struct aom_codec_enc_cfg *stream_config); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WARNINGS_H_ diff --git a/third_party/aom/webmdec.cc b/third_party/aom/webmdec.cc new file mode 100644 index 0000000000..39ecef7063 --- /dev/null +++ b/third_party/aom/webmdec.cc @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "./webmdec.h" + +#include +#include + +#include "third_party/libwebm/mkvparser/mkvparser.h" +#include "third_party/libwebm/mkvparser/mkvreader.h" + +namespace { + +void reset(struct WebmInputContext *const webm_ctx) { + if (webm_ctx->reader != NULL) { + mkvparser::MkvReader *const reader = + reinterpret_cast(webm_ctx->reader); + delete reader; + } + if (webm_ctx->segment != NULL) { + mkvparser::Segment *const segment = + reinterpret_cast(webm_ctx->segment); + delete segment; + } + if (webm_ctx->buffer != NULL) { + delete[] webm_ctx->buffer; + } + webm_ctx->reader = NULL; + webm_ctx->segment = NULL; + webm_ctx->buffer = NULL; + webm_ctx->cluster = NULL; + webm_ctx->block_entry = NULL; + webm_ctx->block = NULL; + webm_ctx->block_frame_index = 0; + webm_ctx->video_track_index = 0; + webm_ctx->timestamp_ns = 0; + webm_ctx->is_key_frame = false; +} + +void get_first_cluster(struct WebmInputContext *const webm_ctx) { + mkvparser::Segment *const segment = + reinterpret_cast(webm_ctx->segment); + const mkvparser::Cluster *const cluster = segment->GetFirst(); + webm_ctx->cluster = cluster; +} + +void rewind_and_reset(struct WebmInputContext *const webm_ctx, + struct AvxInputContext *const aom_ctx) { + rewind(aom_ctx->file); + reset(webm_ctx); +} + +} // namespace + +int file_is_webm(struct WebmInputContext *webm_ctx, + struct AvxInputContext *aom_ctx) { + mkvparser::MkvReader *const reader = new mkvparser::MkvReader(aom_ctx->file); + webm_ctx->reader = reader; + webm_ctx->reached_eos = 0; + + mkvparser::EBMLHeader header; + long long pos = 0; + if (header.Parse(reader, pos) < 0) { + rewind_and_reset(webm_ctx, aom_ctx); + return 0; + } + + mkvparser::Segment *segment; + if (mkvparser::Segment::CreateInstance(reader, pos, segment)) { + rewind_and_reset(webm_ctx, aom_ctx); + return 0; + } + webm_ctx->segment = segment; + if (segment->Load() < 0) { + rewind_and_reset(webm_ctx, aom_ctx); + return 0; + } + + const mkvparser::Tracks *const tracks = segment->GetTracks(); + const mkvparser::VideoTrack *video_track = NULL; + for (unsigned long i = 0; i < tracks->GetTracksCount(); ++i) { + const mkvparser::Track *const track = tracks->GetTrackByIndex(i); + if (track->GetType() == mkvparser::Track::kVideo) { + video_track = static_cast(track); + webm_ctx->video_track_index = static_cast(track->GetNumber()); + break; + } + } + + if (video_track == NULL || video_track->GetCodecId() == NULL) { + rewind_and_reset(webm_ctx, aom_ctx); + return 0; + } + + if (!strncmp(video_track->GetCodecId(), "V_AV1", 5)) { + aom_ctx->fourcc = AV1_FOURCC; + } else { + rewind_and_reset(webm_ctx, aom_ctx); + return 0; + } + + aom_ctx->framerate.denominator = 0; + aom_ctx->framerate.numerator = 0; + aom_ctx->width = static_cast(video_track->GetWidth()); + aom_ctx->height = static_cast(video_track->GetHeight()); + + get_first_cluster(webm_ctx); + + return 1; +} + +int webm_read_frame(struct WebmInputContext *webm_ctx, uint8_t **buffer, + size_t *buffer_size) { + // This check is needed for frame parallel decoding, in which case this + // function could be called even after it has reached end of input stream. + if (webm_ctx->reached_eos) { + return 1; + } + mkvparser::Segment *const segment = + reinterpret_cast(webm_ctx->segment); + const mkvparser::Cluster *cluster = + reinterpret_cast(webm_ctx->cluster); + const mkvparser::Block *block = + reinterpret_cast(webm_ctx->block); + const mkvparser::BlockEntry *block_entry = + reinterpret_cast(webm_ctx->block_entry); + bool block_entry_eos = false; + do { + long status = 0; + bool get_new_block = false; + if (block_entry == NULL && !block_entry_eos) { + status = cluster->GetFirst(block_entry); + get_new_block = true; + } else if (block_entry_eos || block_entry->EOS()) { + cluster = segment->GetNext(cluster); + if (cluster == NULL || cluster->EOS()) { + *buffer_size = 0; + webm_ctx->reached_eos = 1; + return 1; + } + status = cluster->GetFirst(block_entry); + block_entry_eos = false; + get_new_block = true; + } else if (block == NULL || + webm_ctx->block_frame_index == block->GetFrameCount() || + block->GetTrackNumber() != webm_ctx->video_track_index) { + status = cluster->GetNext(block_entry, block_entry); + if (block_entry == NULL || block_entry->EOS()) { + block_entry_eos = true; + continue; + } + get_new_block = true; + } + if (status || block_entry == NULL) { + return -1; + } + if (get_new_block) { + block = block_entry->GetBlock(); + if (block == NULL) return -1; + webm_ctx->block_frame_index = 0; + } + } while (block_entry_eos || + block->GetTrackNumber() != webm_ctx->video_track_index); + + webm_ctx->cluster = cluster; + webm_ctx->block_entry = block_entry; + webm_ctx->block = block; + + const mkvparser::Block::Frame &frame = + block->GetFrame(webm_ctx->block_frame_index); + ++webm_ctx->block_frame_index; + if (frame.len > static_cast(*buffer_size)) { + delete[] * buffer; + *buffer = new uint8_t[frame.len]; + if (*buffer == NULL) { + return -1; + } + webm_ctx->buffer = *buffer; + } + *buffer_size = frame.len; + webm_ctx->timestamp_ns = block->GetTime(cluster); + webm_ctx->is_key_frame = block->IsKey(); + + mkvparser::MkvReader *const reader = + reinterpret_cast(webm_ctx->reader); + return frame.Read(reader, *buffer) ? -1 : 0; +} + +int webm_guess_framerate(struct WebmInputContext *webm_ctx, + struct AvxInputContext *aom_ctx) { + uint32_t i = 0; + uint8_t *buffer = NULL; + size_t buffer_size = 0; + while (webm_ctx->timestamp_ns < 1000000000 && i < 50) { + if (webm_read_frame(webm_ctx, &buffer, &buffer_size)) { + break; + } + ++i; + } + aom_ctx->framerate.numerator = (i - 1) * 1000000; + aom_ctx->framerate.denominator = + static_cast(webm_ctx->timestamp_ns / 1000); + delete[] buffer; + + get_first_cluster(webm_ctx); + webm_ctx->block = NULL; + webm_ctx->block_entry = NULL; + webm_ctx->block_frame_index = 0; + webm_ctx->timestamp_ns = 0; + webm_ctx->reached_eos = 0; + + return 0; +} + +void webm_free(struct WebmInputContext *webm_ctx) { reset(webm_ctx); } diff --git a/third_party/aom/webmdec.h b/third_party/aom/webmdec.h new file mode 100644 index 0000000000..329908eebf --- /dev/null +++ b/third_party/aom/webmdec.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef WEBMDEC_H_ +#define WEBMDEC_H_ + +#include "./tools_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct AvxInputContext; + +struct WebmInputContext { + void *reader; + void *segment; + uint8_t *buffer; + const void *cluster; + const void *block_entry; + const void *block; + int block_frame_index; + int video_track_index; + uint64_t timestamp_ns; + int is_key_frame; + int reached_eos; +}; + +// Checks if the input is a WebM file. If so, initializes WebMInputContext so +// that webm_read_frame can be called to retrieve a video frame. +// Returns 1 on success and 0 on failure or input is not WebM file. +// TODO(vigneshv): Refactor this function into two smaller functions specific +// to their task. +int file_is_webm(struct WebmInputContext *webm_ctx, + struct AvxInputContext *aom_ctx); + +// Reads a WebM Video Frame. Memory for the buffer is created, owned and managed +// by this function. For the first call, |buffer| should be NULL and +// |*buffer_size| should be 0. Once all the frames are read and used, +// webm_free() should be called, otherwise there will be a leak. +// Parameters: +// webm_ctx - WebmInputContext object +// buffer - pointer where the frame data will be filled. +// buffer_size - pointer to buffer size. +// Return values: +// 0 - Success +// 1 - End of Stream +// -1 - Error +int webm_read_frame(struct WebmInputContext *webm_ctx, uint8_t **buffer, + size_t *buffer_size); + +// Guesses the frame rate of the input file based on the container timestamps. +int webm_guess_framerate(struct WebmInputContext *webm_ctx, + struct AvxInputContext *aom_ctx); + +// Resets the WebMInputContext. +void webm_free(struct WebmInputContext *webm_ctx); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBMDEC_H_ diff --git a/third_party/aom/webmenc.cc b/third_party/aom/webmenc.cc new file mode 100644 index 0000000000..e3d209a276 --- /dev/null +++ b/third_party/aom/webmenc.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. +*/ + +#include "./webmenc.h" + +#include + +#include "third_party/libwebm/mkvmuxer/mkvmuxer.h" +#include "third_party/libwebm/mkvmuxer/mkvmuxerutil.h" +#include "third_party/libwebm/mkvmuxer/mkvwriter.h" + +namespace { +const uint64_t kDebugTrackUid = 0xDEADBEEF; +const int kVideoTrackNumber = 1; +} // namespace + +void write_webm_file_header(struct WebmOutputContext *webm_ctx, + const aom_codec_enc_cfg_t *cfg, + stereo_format_t stereo_fmt, unsigned int fourcc, + const struct AvxRational *par) { + mkvmuxer::MkvWriter *const writer = new mkvmuxer::MkvWriter(webm_ctx->stream); + mkvmuxer::Segment *const segment = new mkvmuxer::Segment(); + segment->Init(writer); + segment->set_mode(mkvmuxer::Segment::kFile); + segment->OutputCues(true); + + mkvmuxer::SegmentInfo *const info = segment->GetSegmentInfo(); + const uint64_t kTimecodeScale = 1000000; + info->set_timecode_scale(kTimecodeScale); + std::string version = "aomenc"; + if (!webm_ctx->debug) { + version.append(std::string(" ") + aom_codec_version_str()); + } + info->set_writing_app(version.c_str()); + + const uint64_t video_track_id = + segment->AddVideoTrack(static_cast(cfg->g_w), + static_cast(cfg->g_h), kVideoTrackNumber); + mkvmuxer::VideoTrack *const video_track = static_cast( + segment->GetTrackByNumber(video_track_id)); + video_track->SetStereoMode(stereo_fmt); + const char *codec_id; + switch (fourcc) { + case AV1_FOURCC: codec_id = "V_AV1"; break; + default: codec_id = "V_AV1"; break; + } + video_track->set_codec_id(codec_id); + if (par->numerator > 1 || par->denominator > 1) { + // TODO(fgalligan): Add support of DisplayUnit, Display Aspect Ratio type + // to WebM format. + const uint64_t display_width = static_cast( + ((cfg->g_w * par->numerator * 1.0) / par->denominator) + .5); + video_track->set_display_width(display_width); + video_track->set_display_height(cfg->g_h); + } + if (webm_ctx->debug) { + video_track->set_uid(kDebugTrackUid); + } + webm_ctx->writer = writer; + webm_ctx->segment = segment; +} + +void write_webm_block(struct WebmOutputContext *webm_ctx, + const aom_codec_enc_cfg_t *cfg, + const aom_codec_cx_pkt_t *pkt) { + mkvmuxer::Segment *const segment = + reinterpret_cast(webm_ctx->segment); + int64_t pts_ns = pkt->data.frame.pts * 1000000000ll * cfg->g_timebase.num / + cfg->g_timebase.den; + if (pts_ns <= webm_ctx->last_pts_ns) pts_ns = webm_ctx->last_pts_ns + 1000000; + webm_ctx->last_pts_ns = pts_ns; + + segment->AddFrame(static_cast(pkt->data.frame.buf), + pkt->data.frame.sz, kVideoTrackNumber, pts_ns, + pkt->data.frame.flags & AOM_FRAME_IS_KEY); +} + +void write_webm_file_footer(struct WebmOutputContext *webm_ctx) { + mkvmuxer::MkvWriter *const writer = + reinterpret_cast(webm_ctx->writer); + mkvmuxer::Segment *const segment = + reinterpret_cast(webm_ctx->segment); + segment->Finalize(); + delete segment; + delete writer; + webm_ctx->writer = NULL; + webm_ctx->segment = NULL; +} diff --git a/third_party/aom/webmenc.h b/third_party/aom/webmenc.h new file mode 100644 index 0000000000..74387fb8d1 --- /dev/null +++ b/third_party/aom/webmenc.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef WEBMENC_H_ +#define WEBMENC_H_ + +#include +#include + +#include "tools_common.h" +#include "aom/aom_encoder.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct WebmOutputContext { + int debug; + FILE *stream; + int64_t last_pts_ns; + void *writer; + void *segment; +}; + +/* Stereo 3D packed frame format */ +typedef enum stereo_format { + STEREO_FORMAT_MONO = 0, + STEREO_FORMAT_LEFT_RIGHT = 1, + STEREO_FORMAT_BOTTOM_TOP = 2, + STEREO_FORMAT_TOP_BOTTOM = 3, + STEREO_FORMAT_RIGHT_LEFT = 11 +} stereo_format_t; + +void write_webm_file_header(struct WebmOutputContext *webm_ctx, + const aom_codec_enc_cfg_t *cfg, + stereo_format_t stereo_fmt, unsigned int fourcc, + const struct AvxRational *par); + +void write_webm_block(struct WebmOutputContext *webm_ctx, + const aom_codec_enc_cfg_t *cfg, + const aom_codec_cx_pkt_t *pkt); + +void write_webm_file_footer(struct WebmOutputContext *webm_ctx); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBMENC_H_ diff --git a/third_party/aom/y4menc.c b/third_party/aom/y4menc.c new file mode 100644 index 0000000000..b094f74fed --- /dev/null +++ b/third_party/aom/y4menc.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "./y4menc.h" + +int y4m_write_file_header(char *buf, size_t len, int width, int height, + const struct AvxRational *framerate, + aom_img_fmt_t fmt, unsigned int bit_depth) { + const char *color; + switch (bit_depth) { + case 8: + color = fmt == AOM_IMG_FMT_444A + ? "C444alpha\n" + : fmt == AOM_IMG_FMT_I444 + ? "C444\n" + : fmt == AOM_IMG_FMT_I422 ? "C422\n" : "C420jpeg\n"; + break; + case 9: + color = fmt == AOM_IMG_FMT_I44416 + ? "C444p9 XYSCSS=444P9\n" + : fmt == AOM_IMG_FMT_I42216 ? "C422p9 XYSCSS=422P9\n" + : "C420p9 XYSCSS=420P9\n"; + break; + case 10: + color = fmt == AOM_IMG_FMT_I44416 + ? "C444p10 XYSCSS=444P10\n" + : fmt == AOM_IMG_FMT_I42216 ? "C422p10 XYSCSS=422P10\n" + : "C420p10 XYSCSS=420P10\n"; + break; + case 12: + color = fmt == AOM_IMG_FMT_I44416 + ? "C444p12 XYSCSS=444P12\n" + : fmt == AOM_IMG_FMT_I42216 ? "C422p12 XYSCSS=422P12\n" + : "C420p12 XYSCSS=420P12\n"; + break; + case 14: + color = fmt == AOM_IMG_FMT_I44416 + ? "C444p14 XYSCSS=444P14\n" + : fmt == AOM_IMG_FMT_I42216 ? "C422p14 XYSCSS=422P14\n" + : "C420p14 XYSCSS=420P14\n"; + break; + case 16: + color = fmt == AOM_IMG_FMT_I44416 + ? "C444p16 XYSCSS=444P16\n" + : fmt == AOM_IMG_FMT_I42216 ? "C422p16 XYSCSS=422P16\n" + : "C420p16 XYSCSS=420P16\n"; + break; + default: color = NULL; assert(0); + } + return snprintf(buf, len, "YUV4MPEG2 W%u H%u F%u:%u I%c %s", width, height, + framerate->numerator, framerate->denominator, 'p', color); +} + +int y4m_write_frame_header(char *buf, size_t len) { + return snprintf(buf, len, "FRAME\n"); +} diff --git a/third_party/aom/y4menc.h b/third_party/aom/y4menc.h new file mode 100644 index 0000000000..cb75eeb42e --- /dev/null +++ b/third_party/aom/y4menc.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef Y4MENC_H_ +#define Y4MENC_H_ + +#include "./tools_common.h" + +#include "aom/aom_decoder.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define Y4M_BUFFER_SIZE 128 + +int y4m_write_file_header(char *buf, size_t len, int width, int height, + const struct AvxRational *framerate, + aom_img_fmt_t fmt, unsigned int bit_depth); +int y4m_write_frame_header(char *buf, size_t len); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // Y4MENC_H_ diff --git a/third_party/aom/y4minput.c b/third_party/aom/y4minput.c new file mode 100644 index 0000000000..1919189245 --- /dev/null +++ b/third_party/aom/y4minput.c @@ -0,0 +1,1127 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + * + * Based on code from the OggTheora software codec source code, + * Copyright (C) 2002-2010 The Xiph.Org Foundation and contributors. + */ +#include +#include +#include + +#include "aom/aom_integer.h" +#include "y4minput.h" + +// Reads 'size' bytes from 'file' into 'buf' with some fault tolerance. +// Returns true on success. +static int file_read(void *buf, size_t size, FILE *file) { + const int kMaxRetries = 5; + int retry_count = 0; + int file_error; + size_t len = 0; + do { + const size_t n = fread((uint8_t *)buf + len, 1, size - len, file); + len += n; + file_error = ferror(file); + if (file_error) { + if (errno == EINTR || errno == EAGAIN) { + clearerr(file); + continue; + } else { + fprintf(stderr, "Error reading file: %u of %u bytes read, %d: %s\n", + (uint32_t)len, (uint32_t)size, errno, strerror(errno)); + return 0; + } + } + } while (!feof(file) && len < size && ++retry_count < kMaxRetries); + + if (!feof(file) && len != size) { + fprintf(stderr, + "Error reading file: %u of %u bytes read," + " error: %d, retries: %d, %d: %s\n", + (uint32_t)len, (uint32_t)size, file_error, retry_count, errno, + strerror(errno)); + } + return len == size; +} + +static int y4m_parse_tags(y4m_input *_y4m, char *_tags) { + int got_w; + int got_h; + int got_fps; + int got_interlace; + int got_par; + int got_chroma; + char *p; + char *q; + got_w = got_h = got_fps = got_interlace = got_par = got_chroma = 0; + for (p = _tags;; p = q) { + /*Skip any leading spaces.*/ + while (*p == ' ') p++; + /*If that's all we have, stop.*/ + if (p[0] == '\0') break; + /*Find the end of this tag.*/ + for (q = p + 1; *q != '\0' && *q != ' '; q++) { + } + /*Process the tag.*/ + switch (p[0]) { + case 'W': { + if (sscanf(p + 1, "%d", &_y4m->pic_w) != 1) return -1; + got_w = 1; + } break; + case 'H': { + if (sscanf(p + 1, "%d", &_y4m->pic_h) != 1) return -1; + got_h = 1; + } break; + case 'F': { + if (sscanf(p + 1, "%d:%d", &_y4m->fps_n, &_y4m->fps_d) != 2) { + return -1; + } + got_fps = 1; + } break; + case 'I': { + _y4m->interlace = p[1]; + got_interlace = 1; + } break; + case 'A': { + if (sscanf(p + 1, "%d:%d", &_y4m->par_n, &_y4m->par_d) != 2) { + return -1; + } + got_par = 1; + } break; + case 'C': { + if (q - p > 16) return -1; + memcpy(_y4m->chroma_type, p + 1, q - p - 1); + _y4m->chroma_type[q - p - 1] = '\0'; + got_chroma = 1; + } break; + /*Ignore unknown tags.*/ + } + } + if (!got_w || !got_h || !got_fps) return -1; + if (!got_interlace) _y4m->interlace = '?'; + if (!got_par) _y4m->par_n = _y4m->par_d = 0; + /*Chroma-type is not specified in older files, e.g., those generated by + mplayer.*/ + if (!got_chroma) strcpy(_y4m->chroma_type, "420"); + return 0; +} + +/*All anti-aliasing filters in the following conversion functions are based on + one of two window functions: + The 6-tap Lanczos window (for down-sampling and shifts): + sinc(\pi*t)*sinc(\pi*t/3), |t|<3 (sinc(t)==sin(t)/t) + 0, |t|>=3 + The 4-tap Mitchell window (for up-sampling): + 7|t|^3-12|t|^2+16/3, |t|<1 + -(7/3)|x|^3+12|x|^2-20|x|+32/3, |t|<2 + 0, |t|>=2 + The number of taps is intentionally kept small to reduce computational + overhead and limit ringing. + + The taps from these filters are scaled so that their sum is 1, and the result + is scaled by 128 and rounded to integers to create a filter whose + intermediate values fit inside 16 bits. + Coefficients are rounded in such a way as to ensure their sum is still 128, + which is usually equivalent to normal rounding. + + Conversions which require both horizontal and vertical filtering could + have these steps pipelined, for less memory consumption and better cache + performance, but we do them separately for simplicity.*/ + +#define OC_MINI(_a, _b) ((_a) > (_b) ? (_b) : (_a)) +#define OC_MAXI(_a, _b) ((_a) < (_b) ? (_b) : (_a)) +#define OC_CLAMPI(_a, _b, _c) (OC_MAXI(_a, OC_MINI(_b, _c))) + +/*420jpeg chroma samples are sited like: + Y-------Y-------Y-------Y------- + | | | | + | BR | | BR | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | | | | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | BR | | BR | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | | | | + | | | | + + 420mpeg2 chroma samples are sited like: + Y-------Y-------Y-------Y------- + | | | | + BR | BR | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | | | | + | | | | + Y-------Y-------Y-------Y------- + | | | | + BR | BR | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | | | | + | | | | + + We use a resampling filter to shift the site locations one quarter pixel (at + the chroma plane's resolution) to the right. + The 4:2:2 modes look exactly the same, except there are twice as many chroma + lines, and they are vertically co-sited with the luma samples in both the + mpeg2 and jpeg cases (thus requiring no vertical resampling).*/ +static void y4m_42xmpeg2_42xjpeg_helper(unsigned char *_dst, + const unsigned char *_src, int _c_w, + int _c_h) { + int y; + int x; + for (y = 0; y < _c_h; y++) { + /*Filter: [4 -17 114 35 -9 1]/128, derived from a 6-tap Lanczos + window.*/ + for (x = 0; x < OC_MINI(_c_w, 2); x++) { + _dst[x] = (unsigned char)OC_CLAMPI( + 0, (4 * _src[0] - 17 * _src[OC_MAXI(x - 1, 0)] + 114 * _src[x] + + 35 * _src[OC_MINI(x + 1, _c_w - 1)] - + 9 * _src[OC_MINI(x + 2, _c_w - 1)] + + _src[OC_MINI(x + 3, _c_w - 1)] + 64) >> + 7, + 255); + } + for (; x < _c_w - 3; x++) { + _dst[x] = (unsigned char)OC_CLAMPI( + 0, (4 * _src[x - 2] - 17 * _src[x - 1] + 114 * _src[x] + + 35 * _src[x + 1] - 9 * _src[x + 2] + _src[x + 3] + 64) >> + 7, + 255); + } + for (; x < _c_w; x++) { + _dst[x] = (unsigned char)OC_CLAMPI( + 0, (4 * _src[x - 2] - 17 * _src[x - 1] + 114 * _src[x] + + 35 * _src[OC_MINI(x + 1, _c_w - 1)] - + 9 * _src[OC_MINI(x + 2, _c_w - 1)] + _src[_c_w - 1] + 64) >> + 7, + 255); + } + _dst += _c_w; + _src += _c_w; + } +} + +/*Handles both 422 and 420mpeg2 to 422jpeg and 420jpeg, respectively.*/ +static void y4m_convert_42xmpeg2_42xjpeg(y4m_input *_y4m, unsigned char *_dst, + unsigned char *_aux) { + int c_w; + int c_h; + int c_sz; + int pli; + /*Skip past the luma data.*/ + _dst += _y4m->pic_w * _y4m->pic_h; + /*Compute the size of each chroma plane.*/ + c_w = (_y4m->pic_w + _y4m->dst_c_dec_h - 1) / _y4m->dst_c_dec_h; + c_h = (_y4m->pic_h + _y4m->dst_c_dec_v - 1) / _y4m->dst_c_dec_v; + c_sz = c_w * c_h; + for (pli = 1; pli < 3; pli++) { + y4m_42xmpeg2_42xjpeg_helper(_dst, _aux, c_w, c_h); + _dst += c_sz; + _aux += c_sz; + } +} + +/*This format is only used for interlaced content, but is included for + completeness. + + 420jpeg chroma samples are sited like: + Y-------Y-------Y-------Y------- + | | | | + | BR | | BR | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | | | | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | BR | | BR | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | | | | + | | | | + + 420paldv chroma samples are sited like: + YR------Y-------YR------Y------- + | | | | + | | | | + | | | | + YB------Y-------YB------Y------- + | | | | + | | | | + | | | | + YR------Y-------YR------Y------- + | | | | + | | | | + | | | | + YB------Y-------YB------Y------- + | | | | + | | | | + | | | | + + We use a resampling filter to shift the site locations one quarter pixel (at + the chroma plane's resolution) to the right. + Then we use another filter to move the C_r location down one quarter pixel, + and the C_b location up one quarter pixel.*/ +static void y4m_convert_42xpaldv_42xjpeg(y4m_input *_y4m, unsigned char *_dst, + unsigned char *_aux) { + unsigned char *tmp; + int c_w; + int c_h; + int c_sz; + int pli; + int y; + int x; + /*Skip past the luma data.*/ + _dst += _y4m->pic_w * _y4m->pic_h; + /*Compute the size of each chroma plane.*/ + c_w = (_y4m->pic_w + 1) / 2; + c_h = (_y4m->pic_h + _y4m->dst_c_dec_h - 1) / _y4m->dst_c_dec_h; + c_sz = c_w * c_h; + tmp = _aux + 2 * c_sz; + for (pli = 1; pli < 3; pli++) { + /*First do the horizontal re-sampling. + This is the same as the mpeg2 case, except that after the horizontal + case, we need to apply a second vertical filter.*/ + y4m_42xmpeg2_42xjpeg_helper(tmp, _aux, c_w, c_h); + _aux += c_sz; + switch (pli) { + case 1: { + /*Slide C_b up a quarter-pel. + This is the same filter used above, but in the other order.*/ + for (x = 0; x < c_w; x++) { + for (y = 0; y < OC_MINI(c_h, 3); y++) { + _dst[y * c_w] = (unsigned char)OC_CLAMPI( + 0, (tmp[0] - 9 * tmp[OC_MAXI(y - 2, 0) * c_w] + + 35 * tmp[OC_MAXI(y - 1, 0) * c_w] + 114 * tmp[y * c_w] - + 17 * tmp[OC_MINI(y + 1, c_h - 1) * c_w] + + 4 * tmp[OC_MINI(y + 2, c_h - 1) * c_w] + 64) >> + 7, + 255); + } + for (; y < c_h - 2; y++) { + _dst[y * c_w] = (unsigned char)OC_CLAMPI( + 0, (tmp[(y - 3) * c_w] - 9 * tmp[(y - 2) * c_w] + + 35 * tmp[(y - 1) * c_w] + 114 * tmp[y * c_w] - + 17 * tmp[(y + 1) * c_w] + 4 * tmp[(y + 2) * c_w] + 64) >> + 7, + 255); + } + for (; y < c_h; y++) { + _dst[y * c_w] = (unsigned char)OC_CLAMPI( + 0, (tmp[(y - 3) * c_w] - 9 * tmp[(y - 2) * c_w] + + 35 * tmp[(y - 1) * c_w] + 114 * tmp[y * c_w] - + 17 * tmp[OC_MINI(y + 1, c_h - 1) * c_w] + + 4 * tmp[(c_h - 1) * c_w] + 64) >> + 7, + 255); + } + _dst++; + tmp++; + } + _dst += c_sz - c_w; + tmp -= c_w; + } break; + case 2: { + /*Slide C_r down a quarter-pel. + This is the same as the horizontal filter.*/ + for (x = 0; x < c_w; x++) { + for (y = 0; y < OC_MINI(c_h, 2); y++) { + _dst[y * c_w] = (unsigned char)OC_CLAMPI( + 0, + (4 * tmp[0] - 17 * tmp[OC_MAXI(y - 1, 0) * c_w] + + 114 * tmp[y * c_w] + 35 * tmp[OC_MINI(y + 1, c_h - 1) * c_w] - + 9 * tmp[OC_MINI(y + 2, c_h - 1) * c_w] + + tmp[OC_MINI(y + 3, c_h - 1) * c_w] + 64) >> + 7, + 255); + } + for (; y < c_h - 3; y++) { + _dst[y * c_w] = (unsigned char)OC_CLAMPI( + 0, (4 * tmp[(y - 2) * c_w] - 17 * tmp[(y - 1) * c_w] + + 114 * tmp[y * c_w] + 35 * tmp[(y + 1) * c_w] - + 9 * tmp[(y + 2) * c_w] + tmp[(y + 3) * c_w] + 64) >> + 7, + 255); + } + for (; y < c_h; y++) { + _dst[y * c_w] = (unsigned char)OC_CLAMPI( + 0, + (4 * tmp[(y - 2) * c_w] - 17 * tmp[(y - 1) * c_w] + + 114 * tmp[y * c_w] + 35 * tmp[OC_MINI(y + 1, c_h - 1) * c_w] - + 9 * tmp[OC_MINI(y + 2, c_h - 1) * c_w] + tmp[(c_h - 1) * c_w] + + 64) >> + 7, + 255); + } + _dst++; + tmp++; + } + } break; + } + /*For actual interlaced material, this would have to be done separately on + each field, and the shift amounts would be different. + C_r moves down 1/8, C_b up 3/8 in the top field, and C_r moves down 3/8, + C_b up 1/8 in the bottom field. + The corresponding filters would be: + Down 1/8 (reverse order for up): [3 -11 125 15 -4 0]/128 + Down 3/8 (reverse order for up): [4 -19 98 56 -13 2]/128*/ + } +} + +/*Perform vertical filtering to reduce a single plane from 4:2:2 to 4:2:0. + This is used as a helper by several converation routines.*/ +static void y4m_422jpeg_420jpeg_helper(unsigned char *_dst, + const unsigned char *_src, int _c_w, + int _c_h) { + int y; + int x; + /*Filter: [3 -17 78 78 -17 3]/128, derived from a 6-tap Lanczos window.*/ + for (x = 0; x < _c_w; x++) { + for (y = 0; y < OC_MINI(_c_h, 2); y += 2) { + _dst[(y >> 1) * _c_w] = + OC_CLAMPI(0, (64 * _src[0] + 78 * _src[OC_MINI(1, _c_h - 1) * _c_w] - + 17 * _src[OC_MINI(2, _c_h - 1) * _c_w] + + 3 * _src[OC_MINI(3, _c_h - 1) * _c_w] + 64) >> + 7, + 255); + } + for (; y < _c_h - 3; y += 2) { + _dst[(y >> 1) * _c_w] = + OC_CLAMPI(0, (3 * (_src[(y - 2) * _c_w] + _src[(y + 3) * _c_w]) - + 17 * (_src[(y - 1) * _c_w] + _src[(y + 2) * _c_w]) + + 78 * (_src[y * _c_w] + _src[(y + 1) * _c_w]) + 64) >> + 7, + 255); + } + for (; y < _c_h; y += 2) { + _dst[(y >> 1) * _c_w] = OC_CLAMPI( + 0, + (3 * (_src[(y - 2) * _c_w] + _src[(_c_h - 1) * _c_w]) - + 17 * (_src[(y - 1) * _c_w] + _src[OC_MINI(y + 2, _c_h - 1) * _c_w]) + + 78 * (_src[y * _c_w] + _src[OC_MINI(y + 1, _c_h - 1) * _c_w]) + + 64) >> + 7, + 255); + } + _src++; + _dst++; + } +} + +/*420jpeg chroma samples are sited like: + Y-------Y-------Y-------Y------- + | | | | + | BR | | BR | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | | | | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | BR | | BR | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | | | | + | | | | + + 422jpeg chroma samples are sited like: + Y---BR--Y-------Y---BR--Y------- + | | | | + | | | | + | | | | + Y---BR--Y-------Y---BR--Y------- + | | | | + | | | | + | | | | + Y---BR--Y-------Y---BR--Y------- + | | | | + | | | | + | | | | + Y---BR--Y-------Y---BR--Y------- + | | | | + | | | | + | | | | + + We use a resampling filter to decimate the chroma planes by two in the + vertical direction.*/ +static void y4m_convert_422jpeg_420jpeg(y4m_input *_y4m, unsigned char *_dst, + unsigned char *_aux) { + int c_w; + int c_h; + int c_sz; + int dst_c_w; + int dst_c_h; + int dst_c_sz; + int pli; + /*Skip past the luma data.*/ + _dst += _y4m->pic_w * _y4m->pic_h; + /*Compute the size of each chroma plane.*/ + c_w = (_y4m->pic_w + _y4m->src_c_dec_h - 1) / _y4m->src_c_dec_h; + c_h = _y4m->pic_h; + dst_c_w = (_y4m->pic_w + _y4m->dst_c_dec_h - 1) / _y4m->dst_c_dec_h; + dst_c_h = (_y4m->pic_h + _y4m->dst_c_dec_v - 1) / _y4m->dst_c_dec_v; + c_sz = c_w * c_h; + dst_c_sz = dst_c_w * dst_c_h; + for (pli = 1; pli < 3; pli++) { + y4m_422jpeg_420jpeg_helper(_dst, _aux, c_w, c_h); + _aux += c_sz; + _dst += dst_c_sz; + } +} + +/*420jpeg chroma samples are sited like: + Y-------Y-------Y-------Y------- + | | | | + | BR | | BR | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | | | | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | BR | | BR | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | | | | + | | | | + + 422 chroma samples are sited like: + YBR-----Y-------YBR-----Y------- + | | | | + | | | | + | | | | + YBR-----Y-------YBR-----Y------- + | | | | + | | | | + | | | | + YBR-----Y-------YBR-----Y------- + | | | | + | | | | + | | | | + YBR-----Y-------YBR-----Y------- + | | | | + | | | | + | | | | + + We use a resampling filter to shift the original site locations one quarter + pixel (at the original chroma resolution) to the right. + Then we use a second resampling filter to decimate the chroma planes by two + in the vertical direction.*/ +static void y4m_convert_422_420jpeg(y4m_input *_y4m, unsigned char *_dst, + unsigned char *_aux) { + unsigned char *tmp; + int c_w; + int c_h; + int c_sz; + int dst_c_h; + int dst_c_sz; + int pli; + /*Skip past the luma data.*/ + _dst += _y4m->pic_w * _y4m->pic_h; + /*Compute the size of each chroma plane.*/ + c_w = (_y4m->pic_w + _y4m->src_c_dec_h - 1) / _y4m->src_c_dec_h; + c_h = _y4m->pic_h; + dst_c_h = (_y4m->pic_h + _y4m->dst_c_dec_v - 1) / _y4m->dst_c_dec_v; + c_sz = c_w * c_h; + dst_c_sz = c_w * dst_c_h; + tmp = _aux + 2 * c_sz; + for (pli = 1; pli < 3; pli++) { + /*In reality, the horizontal and vertical steps could be pipelined, for + less memory consumption and better cache performance, but we do them + separately for simplicity.*/ + /*First do horizontal filtering (convert to 422jpeg)*/ + y4m_42xmpeg2_42xjpeg_helper(tmp, _aux, c_w, c_h); + /*Now do the vertical filtering.*/ + y4m_422jpeg_420jpeg_helper(_dst, tmp, c_w, c_h); + _aux += c_sz; + _dst += dst_c_sz; + } +} + +/*420jpeg chroma samples are sited like: + Y-------Y-------Y-------Y------- + | | | | + | BR | | BR | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | | | | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | BR | | BR | + | | | | + Y-------Y-------Y-------Y------- + | | | | + | | | | + | | | | + + 411 chroma samples are sited like: + YBR-----Y-------Y-------Y------- + | | | | + | | | | + | | | | + YBR-----Y-------Y-------Y------- + | | | | + | | | | + | | | | + YBR-----Y-------Y-------Y------- + | | | | + | | | | + | | | | + YBR-----Y-------Y-------Y------- + | | | | + | | | | + | | | | + + We use a filter to resample at site locations one eighth pixel (at the source + chroma plane's horizontal resolution) and five eighths of a pixel to the + right. + Then we use another filter to decimate the planes by 2 in the vertical + direction.*/ +static void y4m_convert_411_420jpeg(y4m_input *_y4m, unsigned char *_dst, + unsigned char *_aux) { + unsigned char *tmp; + int c_w; + int c_h; + int c_sz; + int dst_c_w; + int dst_c_h; + int dst_c_sz; + int tmp_sz; + int pli; + int y; + int x; + /*Skip past the luma data.*/ + _dst += _y4m->pic_w * _y4m->pic_h; + /*Compute the size of each chroma plane.*/ + c_w = (_y4m->pic_w + _y4m->src_c_dec_h - 1) / _y4m->src_c_dec_h; + c_h = _y4m->pic_h; + dst_c_w = (_y4m->pic_w + _y4m->dst_c_dec_h - 1) / _y4m->dst_c_dec_h; + dst_c_h = (_y4m->pic_h + _y4m->dst_c_dec_v - 1) / _y4m->dst_c_dec_v; + c_sz = c_w * c_h; + dst_c_sz = dst_c_w * dst_c_h; + tmp_sz = dst_c_w * c_h; + tmp = _aux + 2 * c_sz; + for (pli = 1; pli < 3; pli++) { + /*In reality, the horizontal and vertical steps could be pipelined, for + less memory consumption and better cache performance, but we do them + separately for simplicity.*/ + /*First do horizontal filtering (convert to 422jpeg)*/ + for (y = 0; y < c_h; y++) { + /*Filters: [1 110 18 -1]/128 and [-3 50 86 -5]/128, both derived from a + 4-tap Mitchell window.*/ + for (x = 0; x < OC_MINI(c_w, 1); x++) { + tmp[x << 1] = (unsigned char)OC_CLAMPI( + 0, (111 * _aux[0] + 18 * _aux[OC_MINI(1, c_w - 1)] - + _aux[OC_MINI(2, c_w - 1)] + 64) >> + 7, + 255); + tmp[x << 1 | 1] = (unsigned char)OC_CLAMPI( + 0, (47 * _aux[0] + 86 * _aux[OC_MINI(1, c_w - 1)] - + 5 * _aux[OC_MINI(2, c_w - 1)] + 64) >> + 7, + 255); + } + for (; x < c_w - 2; x++) { + tmp[x << 1] = + (unsigned char)OC_CLAMPI(0, (_aux[x - 1] + 110 * _aux[x] + + 18 * _aux[x + 1] - _aux[x + 2] + 64) >> + 7, + 255); + tmp[x << 1 | 1] = (unsigned char)OC_CLAMPI( + 0, (-3 * _aux[x - 1] + 50 * _aux[x] + 86 * _aux[x + 1] - + 5 * _aux[x + 2] + 64) >> + 7, + 255); + } + for (; x < c_w; x++) { + tmp[x << 1] = (unsigned char)OC_CLAMPI( + 0, (_aux[x - 1] + 110 * _aux[x] + + 18 * _aux[OC_MINI(x + 1, c_w - 1)] - _aux[c_w - 1] + 64) >> + 7, + 255); + if ((x << 1 | 1) < dst_c_w) { + tmp[x << 1 | 1] = (unsigned char)OC_CLAMPI( + 0, + (-3 * _aux[x - 1] + 50 * _aux[x] + + 86 * _aux[OC_MINI(x + 1, c_w - 1)] - 5 * _aux[c_w - 1] + 64) >> + 7, + 255); + } + } + tmp += dst_c_w; + _aux += c_w; + } + tmp -= tmp_sz; + /*Now do the vertical filtering.*/ + y4m_422jpeg_420jpeg_helper(_dst, tmp, dst_c_w, c_h); + _dst += dst_c_sz; + } +} + +/*Convert 444 to 420jpeg.*/ +static void y4m_convert_444_420jpeg(y4m_input *_y4m, unsigned char *_dst, + unsigned char *_aux) { + unsigned char *tmp; + int c_w; + int c_h; + int c_sz; + int dst_c_w; + int dst_c_h; + int dst_c_sz; + int tmp_sz; + int pli; + int y; + int x; + /*Skip past the luma data.*/ + _dst += _y4m->pic_w * _y4m->pic_h; + /*Compute the size of each chroma plane.*/ + c_w = (_y4m->pic_w + _y4m->src_c_dec_h - 1) / _y4m->src_c_dec_h; + c_h = _y4m->pic_h; + dst_c_w = (_y4m->pic_w + _y4m->dst_c_dec_h - 1) / _y4m->dst_c_dec_h; + dst_c_h = (_y4m->pic_h + _y4m->dst_c_dec_v - 1) / _y4m->dst_c_dec_v; + c_sz = c_w * c_h; + dst_c_sz = dst_c_w * dst_c_h; + tmp_sz = dst_c_w * c_h; + tmp = _aux + 2 * c_sz; + for (pli = 1; pli < 3; pli++) { + /*Filter: [3 -17 78 78 -17 3]/128, derived from a 6-tap Lanczos window.*/ + for (y = 0; y < c_h; y++) { + for (x = 0; x < OC_MINI(c_w, 2); x += 2) { + tmp[x >> 1] = + OC_CLAMPI(0, (64 * _aux[0] + 78 * _aux[OC_MINI(1, c_w - 1)] - + 17 * _aux[OC_MINI(2, c_w - 1)] + + 3 * _aux[OC_MINI(3, c_w - 1)] + 64) >> + 7, + 255); + } + for (; x < c_w - 3; x += 2) { + tmp[x >> 1] = OC_CLAMPI(0, (3 * (_aux[x - 2] + _aux[x + 3]) - + 17 * (_aux[x - 1] + _aux[x + 2]) + + 78 * (_aux[x] + _aux[x + 1]) + 64) >> + 7, + 255); + } + for (; x < c_w; x += 2) { + tmp[x >> 1] = OC_CLAMPI( + 0, (3 * (_aux[x - 2] + _aux[c_w - 1]) - + 17 * (_aux[x - 1] + _aux[OC_MINI(x + 2, c_w - 1)]) + + 78 * (_aux[x] + _aux[OC_MINI(x + 1, c_w - 1)]) + 64) >> + 7, + 255); + } + tmp += dst_c_w; + _aux += c_w; + } + tmp -= tmp_sz; + /*Now do the vertical filtering.*/ + y4m_422jpeg_420jpeg_helper(_dst, tmp, dst_c_w, c_h); + _dst += dst_c_sz; + } +} + +/*The image is padded with empty chroma components at 4:2:0.*/ +static void y4m_convert_mono_420jpeg(y4m_input *_y4m, unsigned char *_dst, + unsigned char *_aux) { + int c_sz; + (void)_aux; + _dst += _y4m->pic_w * _y4m->pic_h; + c_sz = ((_y4m->pic_w + _y4m->dst_c_dec_h - 1) / _y4m->dst_c_dec_h) * + ((_y4m->pic_h + _y4m->dst_c_dec_v - 1) / _y4m->dst_c_dec_v); + memset(_dst, 128, c_sz * 2); +} + +/*No conversion function needed.*/ +static void y4m_convert_null(y4m_input *_y4m, unsigned char *_dst, + unsigned char *_aux) { + (void)_y4m; + (void)_dst; + (void)_aux; +} + +int y4m_input_open(y4m_input *_y4m, FILE *_fin, char *_skip, int _nskip, + int only_420) { + char buffer[80] = { 0 }; + int ret; + int i; + /*Read until newline, or 80 cols, whichever happens first.*/ + for (i = 0; i < 79; i++) { + if (_nskip > 0) { + buffer[i] = *_skip++; + _nskip--; + } else { + if (!file_read(buffer + i, 1, _fin)) return -1; + } + if (buffer[i] == '\n') break; + } + /*We skipped too much header data.*/ + if (_nskip > 0) return -1; + if (i == 79) { + fprintf(stderr, "Error parsing header; not a YUV2MPEG2 file?\n"); + return -1; + } + buffer[i] = '\0'; + if (memcmp(buffer, "YUV4MPEG", 8)) { + fprintf(stderr, "Incomplete magic for YUV4MPEG file.\n"); + return -1; + } + if (buffer[8] != '2') { + fprintf(stderr, "Incorrect YUV input file version; YUV4MPEG2 required.\n"); + } + ret = y4m_parse_tags(_y4m, buffer + 5); + if (ret < 0) { + fprintf(stderr, "Error parsing YUV4MPEG2 header.\n"); + return ret; + } + if (_y4m->interlace == '?') { + fprintf(stderr, + "Warning: Input video interlacing format unknown; " + "assuming progressive scan.\n"); + } else if (_y4m->interlace != 'p') { + fprintf(stderr, + "Input video is interlaced; " + "Only progressive scan handled.\n"); + return -1; + } + _y4m->aom_fmt = AOM_IMG_FMT_I420; + _y4m->bps = 12; + _y4m->bit_depth = 8; + if (strcmp(_y4m->chroma_type, "420") == 0 || + strcmp(_y4m->chroma_type, "420jpeg") == 0) { + _y4m->src_c_dec_h = _y4m->dst_c_dec_h = _y4m->src_c_dec_v = + _y4m->dst_c_dec_v = 2; + _y4m->dst_buf_read_sz = + _y4m->pic_w * _y4m->pic_h + + 2 * ((_y4m->pic_w + 1) / 2) * ((_y4m->pic_h + 1) / 2); + /* Natively supported: no conversion required. */ + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 0; + _y4m->convert = y4m_convert_null; + } else if (strcmp(_y4m->chroma_type, "420p10") == 0) { + _y4m->src_c_dec_h = 2; + _y4m->dst_c_dec_h = 2; + _y4m->src_c_dec_v = 2; + _y4m->dst_c_dec_v = 2; + _y4m->dst_buf_read_sz = + 2 * (_y4m->pic_w * _y4m->pic_h + + 2 * ((_y4m->pic_w + 1) / 2) * ((_y4m->pic_h + 1) / 2)); + /* Natively supported: no conversion required. */ + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 0; + _y4m->convert = y4m_convert_null; + _y4m->bit_depth = 10; + _y4m->bps = 15; + _y4m->aom_fmt = AOM_IMG_FMT_I42016; + if (only_420) { + fprintf(stderr, "Unsupported conversion from 420p10 to 420jpeg\n"); + return -1; + } + } else if (strcmp(_y4m->chroma_type, "420p12") == 0) { + _y4m->src_c_dec_h = 2; + _y4m->dst_c_dec_h = 2; + _y4m->src_c_dec_v = 2; + _y4m->dst_c_dec_v = 2; + _y4m->dst_buf_read_sz = + 2 * (_y4m->pic_w * _y4m->pic_h + + 2 * ((_y4m->pic_w + 1) / 2) * ((_y4m->pic_h + 1) / 2)); + /* Natively supported: no conversion required. */ + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 0; + _y4m->convert = y4m_convert_null; + _y4m->bit_depth = 12; + _y4m->bps = 18; + _y4m->aom_fmt = AOM_IMG_FMT_I42016; + if (only_420) { + fprintf(stderr, "Unsupported conversion from 420p12 to 420jpeg\n"); + return -1; + } + } else if (strcmp(_y4m->chroma_type, "420mpeg2") == 0) { + _y4m->src_c_dec_h = _y4m->dst_c_dec_h = _y4m->src_c_dec_v = + _y4m->dst_c_dec_v = 2; + _y4m->dst_buf_read_sz = _y4m->pic_w * _y4m->pic_h; + /*Chroma filter required: read into the aux buf first.*/ + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = + 2 * ((_y4m->pic_w + 1) / 2) * ((_y4m->pic_h + 1) / 2); + _y4m->convert = y4m_convert_42xmpeg2_42xjpeg; + } else if (strcmp(_y4m->chroma_type, "420paldv") == 0) { + _y4m->src_c_dec_h = _y4m->dst_c_dec_h = _y4m->src_c_dec_v = + _y4m->dst_c_dec_v = 2; + _y4m->dst_buf_read_sz = _y4m->pic_w * _y4m->pic_h; + /*Chroma filter required: read into the aux buf first. + We need to make two filter passes, so we need some extra space in the + aux buffer.*/ + _y4m->aux_buf_sz = 3 * ((_y4m->pic_w + 1) / 2) * ((_y4m->pic_h + 1) / 2); + _y4m->aux_buf_read_sz = + 2 * ((_y4m->pic_w + 1) / 2) * ((_y4m->pic_h + 1) / 2); + _y4m->convert = y4m_convert_42xpaldv_42xjpeg; + } else if (strcmp(_y4m->chroma_type, "422jpeg") == 0) { + _y4m->src_c_dec_h = _y4m->dst_c_dec_h = 2; + _y4m->src_c_dec_v = 1; + _y4m->dst_c_dec_v = 2; + _y4m->dst_buf_read_sz = _y4m->pic_w * _y4m->pic_h; + /*Chroma filter required: read into the aux buf first.*/ + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = + 2 * ((_y4m->pic_w + 1) / 2) * _y4m->pic_h; + _y4m->convert = y4m_convert_422jpeg_420jpeg; + } else if (strcmp(_y4m->chroma_type, "422") == 0) { + _y4m->src_c_dec_h = 2; + _y4m->src_c_dec_v = 1; + if (only_420) { + _y4m->dst_c_dec_h = 2; + _y4m->dst_c_dec_v = 2; + _y4m->dst_buf_read_sz = _y4m->pic_w * _y4m->pic_h; + /*Chroma filter required: read into the aux buf first. + We need to make two filter passes, so we need some extra space in the + aux buffer.*/ + _y4m->aux_buf_read_sz = 2 * ((_y4m->pic_w + 1) / 2) * _y4m->pic_h; + _y4m->aux_buf_sz = + _y4m->aux_buf_read_sz + ((_y4m->pic_w + 1) / 2) * _y4m->pic_h; + _y4m->convert = y4m_convert_422_420jpeg; + } else { + _y4m->aom_fmt = AOM_IMG_FMT_I422; + _y4m->bps = 16; + _y4m->dst_c_dec_h = _y4m->src_c_dec_h; + _y4m->dst_c_dec_v = _y4m->src_c_dec_v; + _y4m->dst_buf_read_sz = + _y4m->pic_w * _y4m->pic_h + 2 * ((_y4m->pic_w + 1) / 2) * _y4m->pic_h; + /*Natively supported: no conversion required.*/ + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 0; + _y4m->convert = y4m_convert_null; + } + } else if (strcmp(_y4m->chroma_type, "422p10") == 0) { + _y4m->src_c_dec_h = 2; + _y4m->src_c_dec_v = 1; + _y4m->aom_fmt = AOM_IMG_FMT_I42216; + _y4m->bps = 20; + _y4m->bit_depth = 10; + _y4m->dst_c_dec_h = _y4m->src_c_dec_h; + _y4m->dst_c_dec_v = _y4m->src_c_dec_v; + _y4m->dst_buf_read_sz = 2 * (_y4m->pic_w * _y4m->pic_h + + 2 * ((_y4m->pic_w + 1) / 2) * _y4m->pic_h); + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 0; + _y4m->convert = y4m_convert_null; + if (only_420) { + fprintf(stderr, "Unsupported conversion from 422p10 to 420jpeg\n"); + return -1; + } + } else if (strcmp(_y4m->chroma_type, "422p12") == 0) { + _y4m->src_c_dec_h = 2; + _y4m->src_c_dec_v = 1; + _y4m->aom_fmt = AOM_IMG_FMT_I42216; + _y4m->bps = 24; + _y4m->bit_depth = 12; + _y4m->dst_c_dec_h = _y4m->src_c_dec_h; + _y4m->dst_c_dec_v = _y4m->src_c_dec_v; + _y4m->dst_buf_read_sz = 2 * (_y4m->pic_w * _y4m->pic_h + + 2 * ((_y4m->pic_w + 1) / 2) * _y4m->pic_h); + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 0; + _y4m->convert = y4m_convert_null; + if (only_420) { + fprintf(stderr, "Unsupported conversion from 422p12 to 420jpeg\n"); + return -1; + } + } else if (strcmp(_y4m->chroma_type, "411") == 0) { + _y4m->src_c_dec_h = 4; + _y4m->dst_c_dec_h = 2; + _y4m->src_c_dec_v = 1; + _y4m->dst_c_dec_v = 2; + _y4m->dst_buf_read_sz = _y4m->pic_w * _y4m->pic_h; + /*Chroma filter required: read into the aux buf first. + We need to make two filter passes, so we need some extra space in the + aux buffer.*/ + _y4m->aux_buf_read_sz = 2 * ((_y4m->pic_w + 3) / 4) * _y4m->pic_h; + _y4m->aux_buf_sz = + _y4m->aux_buf_read_sz + ((_y4m->pic_w + 1) / 2) * _y4m->pic_h; + _y4m->convert = y4m_convert_411_420jpeg; + } else if (strcmp(_y4m->chroma_type, "444") == 0) { + _y4m->src_c_dec_h = 1; + _y4m->src_c_dec_v = 1; + if (only_420) { + _y4m->dst_c_dec_h = 2; + _y4m->dst_c_dec_v = 2; + _y4m->dst_buf_read_sz = _y4m->pic_w * _y4m->pic_h; + /*Chroma filter required: read into the aux buf first. + We need to make two filter passes, so we need some extra space in the + aux buffer.*/ + _y4m->aux_buf_read_sz = 2 * _y4m->pic_w * _y4m->pic_h; + _y4m->aux_buf_sz = + _y4m->aux_buf_read_sz + ((_y4m->pic_w + 1) / 2) * _y4m->pic_h; + _y4m->convert = y4m_convert_444_420jpeg; + } else { + _y4m->aom_fmt = AOM_IMG_FMT_I444; + _y4m->bps = 24; + _y4m->dst_c_dec_h = _y4m->src_c_dec_h; + _y4m->dst_c_dec_v = _y4m->src_c_dec_v; + _y4m->dst_buf_read_sz = 3 * _y4m->pic_w * _y4m->pic_h; + /*Natively supported: no conversion required.*/ + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 0; + _y4m->convert = y4m_convert_null; + } + } else if (strcmp(_y4m->chroma_type, "444p10") == 0) { + _y4m->src_c_dec_h = 1; + _y4m->src_c_dec_v = 1; + _y4m->aom_fmt = AOM_IMG_FMT_I44416; + _y4m->bps = 30; + _y4m->bit_depth = 10; + _y4m->dst_c_dec_h = _y4m->src_c_dec_h; + _y4m->dst_c_dec_v = _y4m->src_c_dec_v; + _y4m->dst_buf_read_sz = 2 * 3 * _y4m->pic_w * _y4m->pic_h; + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 0; + _y4m->convert = y4m_convert_null; + if (only_420) { + fprintf(stderr, "Unsupported conversion from 444p10 to 420jpeg\n"); + return -1; + } + } else if (strcmp(_y4m->chroma_type, "444p12") == 0) { + _y4m->src_c_dec_h = 1; + _y4m->src_c_dec_v = 1; + _y4m->aom_fmt = AOM_IMG_FMT_I44416; + _y4m->bps = 36; + _y4m->bit_depth = 12; + _y4m->dst_c_dec_h = _y4m->src_c_dec_h; + _y4m->dst_c_dec_v = _y4m->src_c_dec_v; + _y4m->dst_buf_read_sz = 2 * 3 * _y4m->pic_w * _y4m->pic_h; + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 0; + _y4m->convert = y4m_convert_null; + if (only_420) { + fprintf(stderr, "Unsupported conversion from 444p12 to 420jpeg\n"); + return -1; + } + } else if (strcmp(_y4m->chroma_type, "444alpha") == 0) { + _y4m->src_c_dec_h = 1; + _y4m->src_c_dec_v = 1; + if (only_420) { + _y4m->dst_c_dec_h = 2; + _y4m->dst_c_dec_v = 2; + _y4m->dst_buf_read_sz = _y4m->pic_w * _y4m->pic_h; + /*Chroma filter required: read into the aux buf first. + We need to make two filter passes, so we need some extra space in the + aux buffer. + The extra plane also gets read into the aux buf. + It will be discarded.*/ + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 3 * _y4m->pic_w * _y4m->pic_h; + _y4m->convert = y4m_convert_444_420jpeg; + } else { + _y4m->aom_fmt = AOM_IMG_FMT_444A; + _y4m->bps = 32; + _y4m->dst_c_dec_h = _y4m->src_c_dec_h; + _y4m->dst_c_dec_v = _y4m->src_c_dec_v; + _y4m->dst_buf_read_sz = 4 * _y4m->pic_w * _y4m->pic_h; + /*Natively supported: no conversion required.*/ + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 0; + _y4m->convert = y4m_convert_null; + } + } else if (strcmp(_y4m->chroma_type, "mono") == 0) { + _y4m->src_c_dec_h = _y4m->src_c_dec_v = 0; + _y4m->dst_c_dec_h = _y4m->dst_c_dec_v = 2; + _y4m->dst_buf_read_sz = _y4m->pic_w * _y4m->pic_h; + /*No extra space required, but we need to clear the chroma planes.*/ + _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 0; + _y4m->convert = y4m_convert_mono_420jpeg; + } else { + fprintf(stderr, "Unknown chroma sampling type: %s\n", _y4m->chroma_type); + return -1; + } + /*The size of the final frame buffers is always computed from the + destination chroma decimation type.*/ + _y4m->dst_buf_sz = + _y4m->pic_w * _y4m->pic_h + + 2 * ((_y4m->pic_w + _y4m->dst_c_dec_h - 1) / _y4m->dst_c_dec_h) * + ((_y4m->pic_h + _y4m->dst_c_dec_v - 1) / _y4m->dst_c_dec_v); + if (_y4m->bit_depth == 8) + _y4m->dst_buf = (unsigned char *)malloc(_y4m->dst_buf_sz); + else + _y4m->dst_buf = (unsigned char *)malloc(2 * _y4m->dst_buf_sz); + + if (_y4m->aux_buf_sz > 0) + _y4m->aux_buf = (unsigned char *)malloc(_y4m->aux_buf_sz); + return 0; +} + +void y4m_input_close(y4m_input *_y4m) { + free(_y4m->dst_buf); + free(_y4m->aux_buf); +} + +int y4m_input_fetch_frame(y4m_input *_y4m, FILE *_fin, aom_image_t *_img) { + char frame[6]; + int pic_sz; + int c_w; + int c_h; + int c_sz; + int bytes_per_sample = _y4m->bit_depth > 8 ? 2 : 1; + /*Read and skip the frame header.*/ + if (!file_read(frame, 6, _fin)) return 0; + if (memcmp(frame, "FRAME", 5)) { + fprintf(stderr, "Loss of framing in Y4M input data\n"); + return -1; + } + if (frame[5] != '\n') { + char c; + int j; + for (j = 0; j < 79 && file_read(&c, 1, _fin) && c != '\n'; j++) { + } + if (j == 79) { + fprintf(stderr, "Error parsing Y4M frame header\n"); + return -1; + } + } + /*Read the frame data that needs no conversion.*/ + if (!file_read(_y4m->dst_buf, _y4m->dst_buf_read_sz, _fin)) { + fprintf(stderr, "Error reading Y4M frame data.\n"); + return -1; + } + /*Read the frame data that does need conversion.*/ + if (!file_read(_y4m->aux_buf, _y4m->aux_buf_read_sz, _fin)) { + fprintf(stderr, "Error reading Y4M frame data.\n"); + return -1; + } + /*Now convert the just read frame.*/ + (*_y4m->convert)(_y4m, _y4m->dst_buf, _y4m->aux_buf); + /*Fill in the frame buffer pointers. + We don't use aom_img_wrap() because it forces padding for odd picture + sizes, which would require a separate fread call for every row.*/ + memset(_img, 0, sizeof(*_img)); + /*Y4M has the planes in Y'CbCr order, which libaom calls Y, U, and V.*/ + _img->fmt = _y4m->aom_fmt; + _img->w = _img->d_w = _y4m->pic_w; + _img->h = _img->d_h = _y4m->pic_h; + _img->x_chroma_shift = _y4m->dst_c_dec_h >> 1; + _img->y_chroma_shift = _y4m->dst_c_dec_v >> 1; + _img->bps = _y4m->bps; + + /*Set up the buffer pointers.*/ + pic_sz = _y4m->pic_w * _y4m->pic_h * bytes_per_sample; + c_w = (_y4m->pic_w + _y4m->dst_c_dec_h - 1) / _y4m->dst_c_dec_h; + c_w *= bytes_per_sample; + c_h = (_y4m->pic_h + _y4m->dst_c_dec_v - 1) / _y4m->dst_c_dec_v; + c_sz = c_w * c_h; + _img->stride[AOM_PLANE_Y] = _img->stride[AOM_PLANE_ALPHA] = + _y4m->pic_w * bytes_per_sample; + _img->stride[AOM_PLANE_U] = _img->stride[AOM_PLANE_V] = c_w; + _img->planes[AOM_PLANE_Y] = _y4m->dst_buf; + _img->planes[AOM_PLANE_U] = _y4m->dst_buf + pic_sz; + _img->planes[AOM_PLANE_V] = _y4m->dst_buf + pic_sz + c_sz; + _img->planes[AOM_PLANE_ALPHA] = _y4m->dst_buf + pic_sz + 2 * c_sz; + return 1; +} diff --git a/third_party/aom/y4minput.h b/third_party/aom/y4minput.h new file mode 100644 index 0000000000..db20190db6 --- /dev/null +++ b/third_party/aom/y4minput.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + * + * Based on code from the OggTheora software codec source code, + * Copyright (C) 2002-2010 The Xiph.Org Foundation and contributors. + */ + +#ifndef Y4MINPUT_H_ +#define Y4MINPUT_H_ + +#include +#include "aom/aom_image.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct y4m_input y4m_input; + +/*The function used to perform chroma conversion.*/ +typedef void (*y4m_convert_func)(y4m_input *_y4m, unsigned char *_dst, + unsigned char *_src); + +struct y4m_input { + int pic_w; + int pic_h; + int fps_n; + int fps_d; + int par_n; + int par_d; + char interlace; + int src_c_dec_h; + int src_c_dec_v; + int dst_c_dec_h; + int dst_c_dec_v; + char chroma_type[16]; + /*The size of each converted frame buffer.*/ + size_t dst_buf_sz; + /*The amount to read directly into the converted frame buffer.*/ + size_t dst_buf_read_sz; + /*The size of the auxilliary buffer.*/ + size_t aux_buf_sz; + /*The amount to read into the auxilliary buffer.*/ + size_t aux_buf_read_sz; + y4m_convert_func convert; + unsigned char *dst_buf; + unsigned char *aux_buf; + enum aom_img_fmt aom_fmt; + int bps; + unsigned int bit_depth; +}; + +int y4m_input_open(y4m_input *_y4m, FILE *_fin, char *_skip, int _nskip, + int only_420); +void y4m_input_close(y4m_input *_y4m); +int y4m_input_fetch_frame(y4m_input *_y4m, FILE *_fin, aom_image_t *img); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // Y4MINPUT_H_ -- cgit v1.2.3